diff --git a/Documentation/devicetree/bindings/hwmon/sensylink,ctf2301.yaml b/Documentation/devicetree/bindings/hwmon/sensylink,ctf2301.yaml
new file mode 100644
index 000000000000..389d9745e8fb
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/sensylink,ctf2301.yaml
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/sensylink,ctf2301.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Sensylink CTF2301 system-level thermal management solution chip
+
+maintainers:
+  - Troy Mitchell <troy.mitchell@linux.dev>
+
+allOf:
+  - $ref: hwmon-common.yaml#
+
+description: |
+  The CTF2301B is an I2C/SMBus compatible device featuring:
+    - One local temperature sensor with ±0.5°C accuracy and 0.0625°C resolution.
+    - One remote temperature sensor for external diode-connected transistors,
+      offering ±1°C accuracy and 0.125°C resolution.
+      (temperature range: -40°C to +125°C).
+    - An integrated PWM fan controller.
+    - A 1-channel fan speed monitor (TACH input) for RPM measurement.
+
+  Datasheets:
+    https://www.sensylink.com/upload/1/net.sensylink.portal/1689557281035.pdf
+
+properties:
+  compatible:
+    const: sensylink,ctf2301
+
+  reg:
+    maxItems: 1
+
+  sensylink,temp-signed:
+    description: Temperature value is signed
+    type: boolean
+
+  sensylink,pwm-freq:
+    description: |
+      The initial PWM output frequency.
+      This will likely only produce an approximation of the value you expect,
+      not the exact one.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 360000
+    maximum: 5800
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        ctf2301@4c {
+            compatible = "sensylink,ctf2301";
+            reg = <0x4c>;
+            sensylink,temp-signed;
+            sensylink,pwm-freq = <360000>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/i2c/spacemit,k1-i2c.yaml b/Documentation/devicetree/bindings/i2c/spacemit,k1-i2c.yaml
index b7220fff2235..f310b9c86f14 100644
--- a/Documentation/devicetree/bindings/i2c/spacemit,k1-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/spacemit,k1-i2c.yaml
@@ -41,6 +41,15 @@ properties:
     default: 400000
     maximum: 3300000
 
+  spacemit,sda-glitch-nofix:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description: |
+      Bypass the controller's SDA glitch fix logic.
+
+      The glitch fix may suppress short SDA pulses, but can introduce a
+      small delay on repeated-start signals. If this property is present,
+      the controller will bypass the glitch fix.
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/pinctrl/spacemit,k1-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/spacemit,k1-pinctrl.yaml
index d80e88aa07b4..2635a4cf9d37 100644
--- a/Documentation/devicetree/bindings/pinctrl/spacemit,k1-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/spacemit,k1-pinctrl.yaml
@@ -30,6 +30,15 @@ properties:
   resets:
     maxItems: 1
 
+  spacemit,apbc:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      - items:
+          - description: phandle to syscon that access the protected register
+          - description: offset of access secure registers
+    description:
+      A phandle to syscon with byte offset to access the protected register
+
 patternProperties:
   '-cfg$':
     type: object
@@ -74,8 +83,14 @@ patternProperties:
           drive-strength:
             description: |
               typical current when output high level.
-              1.8V output: 11, 21, 32, 42 (mA)
-              3.3V output: 7, 10, 13, 16, 19, 23, 26, 29 (mA)
+              For k1:
+                1.8V output: 11, 21, 32, 42 (mA)
+                3.3V output: 7, 10, 13, 16, 19, 23, 26, 29 (mA)
+              For k3:
+                1.8V output: 2, 4, 6, 7, 9, 11, 13, 14,
+                             21, 23, 25, 26, 28, 30, 31, 33 (mA)
+                3.3V output: 3, 5, 7, 9, 11, 13, 15, 17,
+                             25, 27, 29, 31, 33, 35, 37, 38 (mA)
 
           input-schmitt:
             description: |
@@ -109,11 +124,13 @@ required:
   - reg
   - clocks
   - clock-names
+  - spacemit,apbc
 
 additionalProperties: false
 
 examples:
   - |
+    #include <dt-bindings/clock/spacemit,k1-syscon.h>
     #define K1_PADCONF(pin, func) (((pin) << 16) | (func))
 
     soc {
@@ -126,6 +143,7 @@ examples:
             clocks = <&syscon_apbc 42>,
                      <&syscon_apbc 94>;
             clock-names = "func", "bus";
+            spacemit,apbc = <&syscon_apbc 0x50>;
 
             uart0_2_cfg: uart0-2-cfg {
                 uart0-2-pins {
diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml
index 153d0dac57fb..33f3811dfd8e 100644
--- a/Documentation/devicetree/bindings/riscv/cpus.yaml
+++ b/Documentation/devicetree/bindings/riscv/cpus.yaml
@@ -141,6 +141,24 @@ properties:
       DMIPS/MHz, relative to highest capacity-dmips-mhz
       in the system.
 
+  debug:
+    type: object
+    properties:
+      compatible:
+        const: riscv,debug-v1.0.0
+      trigger-module:
+        type: object
+        description: |
+          An indication set of optional CSR existence from
+          riscv-debug-spec Sdtrig extension
+        properties:
+          mcontext-present:
+            type: boolean
+          hcontext-present:
+            type: boolean
+          scontext-present:
+            type: boolean
+
 anyOf:
   - required:
       - riscv,isa
diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml
index 543ac94718e8..dd4b9a5a17e4 100644
--- a/Documentation/devicetree/bindings/riscv/extensions.yaml
+++ b/Documentation/devicetree/bindings/riscv/extensions.yaml
@@ -121,6 +121,13 @@ properties:
             version of the privileged ISA specification.
 
         # multi-letter extensions, sorted alphanumerically
+        - const: sdtrig
+          description: |
+            The standard Sdtrig extension for introduce trigger CSRs for
+            cause a breakpoint exception, entry into Debug Mode,
+            or trace action as frozen at commit 359bedc ("Freeze Candidate")
+            of riscv-debug-spec
+
         - const: smaia
           description: |
             The standard Smaia supervisor-level extension for the advanced
diff --git a/Documentation/devicetree/bindings/riscv/spacemit.yaml b/Documentation/devicetree/bindings/riscv/spacemit.yaml
index c56b62a6299a..5ab3e855a911 100644
--- a/Documentation/devicetree/bindings/riscv/spacemit.yaml
+++ b/Documentation/devicetree/bindings/riscv/spacemit.yaml
@@ -24,6 +24,10 @@ properties:
               - milkv,jupiter
               - xunlong,orangepi-rv2
           - const: spacemit,k1
+      - items:
+          - enum:
+              - deepcomputing,fml13v05
+          - const: spacemit,k3
 
 additionalProperties: true
 
diff --git a/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml b/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml
index f2dd20370dbb..1d10cfbad86c 100644
--- a/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml
+++ b/Documentation/devicetree/bindings/spi/fsl,spi-fsl-qspi.yaml
@@ -9,9 +9,6 @@ title: Freescale Quad Serial Peripheral Interface (QuadSPI)
 maintainers:
   - Han Xu <han.xu@nxp.com>
 
-allOf:
-  - $ref: spi-controller.yaml#
-
 properties:
   compatible:
     oneOf:
@@ -22,6 +19,7 @@ properties:
           - fsl,imx6ul-qspi
           - fsl,ls1021a-qspi
           - fsl,ls2080a-qspi
+          - spacemit,k1-qspi
       - items:
           - enum:
               - fsl,ls1043a-qspi
@@ -54,6 +52,11 @@ properties:
       - const: qspi_en
       - const: qspi
 
+  resets:
+    items:
+      - description: SoC QSPI reset
+      - description: SoC QSPI bus reset
+
 required:
   - compatible
   - reg
@@ -62,6 +65,18 @@ required:
   - clocks
   - clock-names
 
+allOf:
+  - $ref: spi-controller.yaml#
+  - if:
+      properties:
+        compatible:
+          not:
+            contains:
+              const: spacemit,k1-qspi
+    then:
+      properties:
+        resets: false
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/spi/spacemit,k1-spi.yaml b/Documentation/devicetree/bindings/spi/spacemit,k1-spi.yaml
new file mode 100644
index 000000000000..e82c7f8d0b98
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/spacemit,k1-spi.yaml
@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/spacemit,k1-spi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: SpacemiT K1 SoC Serial Peripheral Interface (SPI)
+
+maintainers:
+  - Alex Elder <elder@kernel.org>
+
+description:
+  The SpacemiT K1 SoC implements a SPI controller that has two 32-entry
+  FIFOs, for transmit and receive.  Details are currently available in
+  section 18.2.1 of the K1 User Manual, found in the SpacemiT Keystone
+  K1 Documentation[1].  The controller transfers words using PIO.  DMA
+  transfers are supported as well, if both TX and RX DMA channels are
+  specified,
+
+  [1] https://developer.spacemit.com/documentation
+
+allOf:
+  - $ref: /schemas/spi/spi-controller.yaml#
+
+properties:
+  compatible:
+    const: spacemit,k1-spi
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Core clock
+      - description: Bus clock
+
+  clock-names:
+    items:
+      - const: core
+      - const: bus
+
+  resets:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  dmas:
+    items:
+      - description: RX DMA channel
+      - description: TX DMA channel
+
+  dma-names:
+    items:
+      - const: rx
+      - const: tx
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - resets
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+
+    #include <dt-bindings/clock/spacemit,k1-syscon.h>
+    spi@d401c000 {
+        compatible = "spacemit,k1-spi";
+        reg = <0xd401c000 0x30>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+        clocks = <&syscon_apbc CLK_SSP3>,
+                 <&syscon_apbc CLK_SSP3_BUS>;
+        clock-names = "core", "bus";
+        resets = <&syscon_apbc RESET_SSP3>;
+        interrupts = <55>;
+        dmas = <&pdma 20>, <&pdma 19>;
+        dma-names = "rx", "tx";
+    };
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index f1d1882009ba..bd8d14abe2de 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -1407,6 +1407,8 @@ patternProperties:
     description: Sensirion AG
   "^sensortek,.*":
     description: Sensortek Technology Corporation
+  "^sensylink,.*":
+    description: Sensylink Microelectronics Technology Co., Ltd.
   "^sercomm,.*":
     description: Sercomm (Suzhou) Corporation
   "^sff,.*":
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index fadec20b87a8..17da8ac6ffcb 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -168,6 +168,7 @@ config RISCV
 	select HAVE_FUNCTION_ERROR_INJECTION
 	select HAVE_GCC_PLUGINS
 	select HAVE_GENERIC_VDSO if MMU
+	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IRQ_TIME_ACCOUNTING
 	select HAVE_KERNEL_BZIP2 if !XIP_KERNEL && !EFI_ZBOOT
 	select HAVE_KERNEL_GZIP if !XIP_KERNEL && !EFI_ZBOOT
@@ -225,7 +226,7 @@ config RISCV
 	select UACCESS_MEMCPY if !MMU
 	select VDSO_GETRANDOM if HAVE_GENERIC_VDSO && 64BIT
 	select USER_STACKTRACE_SUPPORT
-	select ZONE_DMA32 if 64BIT
+	select ZONE_DMA32 if 64BIT && !SOC_SPACEMIT_K3
 
 config RUSTC_SUPPORTS_RISCV
 	def_bool y
@@ -1323,6 +1324,30 @@ config BUILTIN_DTB_NAME
 	  for the DTS file that will be used to produce the DTB linked into the
 	  kernel.
 
+config RISCV_BUILD_ITB_IMAGE
+	bool "Build FIT Image (ITB) support"
+	default n
+	help
+	  Enable building Flattened Image Tree (FIT/ITB) format images.
+	  This format allows packaging kernel image with device trees and
+	  other data in a single file, with optional signing support.
+
+	  If you need to build Image.itb or signed boot images, say Y.
+	  Otherwise, say N.
+
+config FIT_SIGNATURE
+	bool "Enable FIT image signature support"
+	depends on RISCV_BUILD_ITB_IMAGE
+	default n
+	help
+	  Enable signature verification for FIT images. This requires
+	  RSA keys to be present in arch/riscv/boot/key directory.
+
+	  Compressed images (.gz, .bz2, etc.) are not supported when
+	  signature is enabled.
+
+	  If unsure, say N.
+
 endmenu # "Boot options"
 
 config PORTABLE
diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs
index 848e7149e443..e9ca0b0cc9cb 100644
--- a/arch/riscv/Kconfig.socs
+++ b/arch/riscv/Kconfig.socs
@@ -38,10 +38,7 @@ config ARCH_SOPHGO
 	  This enables support for Sophgo SoC platform hardware.
 
 config ARCH_SPACEMIT
-	bool "SpacemiT SoCs"
-	select PINCTRL
-	help
-	  This enables support for SpacemiT SoC platform hardware.
+	def_bool SOC_SPACEMIT
 
 config ARCH_STARFIVE
 	def_bool SOC_STARFIVE
@@ -98,4 +95,40 @@ config SOC_CANAAN_K210
 	help
 	  This enables support for Canaan Kendryte K210 SoC platform hardware.
 
+config SOC_SPACEMIT
+	bool "Spacemit SoCs"
+	select SIFIVE_PLIC
+	help
+	  This enables support for Spacemit SoCs platform hardware.
+
+config SOC_SPACEMIT_K1
+        bool "enable spacemit K1 SoC platforms"
+        depends on SOC_SPACEMIT
+        help
+          Enable Spacemit k1 Platform SOCs.
+
+config SOC_SPACEMIT_K3
+	bool "enable spacemit K3 SoC platforms"
+	depends on SOC_SPACEMIT
+	help
+	  Enable Spacemit k3 Platform SOCs.
+
+config SOC_SPACEMIT_K3_FPGA
+	bool "Spacemit K3 serial SoC FPGA platform"
+	depends on SOC_SPACEMIT_K3
+	default n
+	help
+	  This enable FPGA platform for K3 SoCs.
+
+config FIT_SIGNATURE
+	bool "Enable signature build for kernel FIT Images"
+	default n
+	help
+	  This option enables build signature for FIT Images,
+	  using a hash signed and verified using RSA.
+
+	  WARNING: When relying on signed FIT images with a required signature
+	  check the legacy image format is disabled by default, so that
+	  unsigned images can be loaded.
+
 endmenu # "SoC selection"
diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile
index bfc3d0b75b9b..bec21ae981dd 100644
--- a/arch/riscv/boot/Makefile
+++ b/arch/riscv/boot/Makefile
@@ -72,3 +72,8 @@ EFI_ZBOOT_BFD_TARGET	:= elf$(BITS)-littleriscv
 EFI_ZBOOT_MACH_TYPE	:= RISCV$(BITS)
 
 include $(srctree)/drivers/firmware/efi/libstub/Makefile.zboot
+
+# Include custom ITB image build rules if enabled
+ifdef CONFIG_RISCV_BUILD_ITB_IMAGE
+include $(src)/Makefile.itb
+endif
diff --git a/arch/riscv/boot/Makefile.itb b/arch/riscv/boot/Makefile.itb
new file mode 100755
index 000000000000..2f96730a84ff
--- /dev/null
+++ b/arch/riscv/boot/Makefile.itb
@@ -0,0 +1,39 @@
+#
+# arch/riscv/boot/Makefile.itb
+#
+# Custom ITB image build rules
+# This file is included when CONFIG_RISCV_BUILD_ITB_IMAGE is enabled
+#
+
+# Select .its template based on signature config
+ifeq ($(CONFIG_FIT_SIGNATURE),)
+its_template := $(srctree)/arch/riscv/generic/Image.its.S
+else
+its_template := $(srctree)/arch/riscv/generic/Image_sign.its.S
+endif
+
+# ITB image generation command
+quiet_cmd_itb-image = ITB     $@
+      cmd_itb-image = \
+	env PATH="$(objtree)/scripts/dtc:$(PATH)" \
+	$(BASH) $(MKIMAGE) \
+	-D "-I dts -O dtb -p 500 \
+		--include $(objtree)/arch/riscv \
+		--warning no-unit_address_vs_reg" \
+	-f $(2) $(3) $@
+
+# FIT signature support
+ifneq ($(CONFIG_FIT_SIGNATURE),)
+key_para := -k $(srctree)/arch/riscv/boot/key
+endif
+
+# Build Image.itb
+$(obj)/Image.itb: $(its_template) $(obj)/Image FORCE
+	$(call if_changed,itb-image,$<,$(key_para))
+
+# Build compressed Image.%.itb
+# NOT support compress image when enable signature
+ifeq ($(CONFIG_FIT_SIGNATURE),)
+$(obj)/Image.%.itb: $(its_template) $(obj)/Image.% FORCE
+	$(call if_changed,itb-image,$<,$(key_para))
+endif
diff --git a/arch/riscv/boot/dts/spacemit/Makefile b/arch/riscv/boot/dts/spacemit/Makefile
index 152832644870..daecd30d7b65 100644
--- a/arch/riscv/boot/dts/spacemit/Makefile
+++ b/arch/riscv/boot/dts/spacemit/Makefile
@@ -1,4 +1,14 @@
 # SPDX-License-Identifier: GPL-2.0
-dtb-$(CONFIG_ARCH_SPACEMIT) += k1-bananapi-f3.dtb
-dtb-$(CONFIG_ARCH_SPACEMIT) += k1-milkv-jupiter.dtb
-dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-rv2.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K1) += k1-bananapi-f3.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K1) += k1-milkv-jupiter.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K1) += k1-orangepi-rv2.dtb
+
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_fpga_1x1.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_evb.dtb k3_evb2-1.dtb k3_evb2-2.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_deb1.dtb k3-pico-itx.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_com260.dtb k3_com260_ifx.dtb k3_com260_ifx2.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3-deepcomputing-fml13v05.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_com260_kit_v02.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_gemini_c0.dtb k3_gemini_c1.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_BS01DCMA.dtb
+dtb-$(CONFIG_SOC_SPACEMIT_K3) += k3_com260_tq.dtb
diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
index 2aaaff77831e..d9d865fbe320 100644
--- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
@@ -14,6 +14,7 @@ aliases {
 		ethernet0 = &eth0;
 		ethernet1 = &eth1;
 		serial0 = &uart0;
+		spi3 = &spi3;
 	};
 
 	chosen {
@@ -92,6 +93,12 @@ &pdma {
 	status = "okay";
 };
 
+&spi3 {
+	pinctrl-0 = <&ssp3_0_cfg>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_2_cfg>;
diff --git a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi
index aff19c86d5ff..6ce56584ff47 100644
--- a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi
@@ -76,4 +76,24 @@ pwm14-1-pins {
 			drive-strength = <32>;
 		};
 	};
+
+	ssp3_0_cfg: ssp3-0-cfg {
+		ssp3-0-pins {
+			pinmux = <K1_PADCONF(75, 2)>,	/* SCLK */
+				 <K1_PADCONF(77, 2)>,	/* MOSI  */
+				 <K1_PADCONF(78, 2)>;	/* MISO */
+
+			bias-disable;
+			drive-strength = <19>;
+			power-source = <3300>;
+		};
+
+		ssp3-0-frm-pins {
+			pinmux = <K1_PADCONF(76, 2)>;	/* FRM (frame) */
+
+			bias-pull-up = <0>;
+			drive-strength = <19>;
+			power-source = <3300>;
+		};
+	};
 };
diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi
index 6cdcd80a7c83..ce55897d34fa 100644
--- a/arch/riscv/boot/dts/spacemit/k1.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k1.dtsi
@@ -797,6 +797,21 @@ uart9: serial@d4017800 {
 				status = "disabled";
 			};
 
+			spi3: spi@d401c000 {
+				compatible = "spacemit,k1-spi";
+				reg = <0x0 0xd401c000 0x0 0x30>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				clocks = <&syscon_apbc CLK_SSP3>,
+					 <&syscon_apbc CLK_SSP3_BUS>;
+				clock-names = "core", "bus";
+				resets = <&syscon_apbc RESET_SSP3>;
+				interrupts = <55>;
+				dmas = <&pdma 20>, <&pdma 19>;
+				dma-names = "rx", "tx";
+				status = "disabled";
+			};
+
 			/* sec_uart1: 0xf0612000, not available from Linux */
 		};
 
diff --git a/arch/riscv/boot/dts/spacemit/k3-camera-sensor-fpga.dtsi b/arch/riscv/boot/dts/spacemit/k3-camera-sensor-fpga.dtsi
new file mode 100755
index 000000000000..e0d6e9623978
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-camera-sensor-fpga.dtsi
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2023 SPACEMIT Micro Limited
+ */
+#include <dt-bindings/gpio/gpio.h>
+
+&soc {
+	sensor0: cam_sensor@0 {
+		cell-index = <0>;
+		compatible = "spacemit,vcam-sensor";
+		clocks = <&syscon_apmu CLK_APMU_ISIM_VCLK2>;
+		clock-names = "cam_mclk0";
+		reg = <0x0 0x0 0x0 0x0>;
+		dphy-entries = <5>;
+		dphy-settings = <0x00000001 0xa2848888 0x00001500 0x000000ff 0x1001>;
+		pwdn-gpios = <&gpio 0 7 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio 0 6 GPIO_ACTIVE_HIGH>;
+		dptc-gpios = <&gpio 0 8 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+	sensor1: cam_sensor@1 {
+		cell-index = <1>;
+		compatible = "spacemit,vcam-sensor";
+		clocks = <&syscon_apmu CLK_APMU_ISIM_VCLK2>;
+		clock-names = "cam_mclk1";
+		reg = <0x0 0x1 0x0 0x0>;
+		dphy-entries = <5>;
+		dphy-settings = <0x00000001 0xa2848888 0x00001500 0x000000ff 0x1001>;
+		pwdn-gpios = <&gpio 0 3 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio 0 2 GPIO_ACTIVE_HIGH>;
+		dptc-gpios = <&gpio 0 4 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-camera.dtsi b/arch/riscv/boot/dts/spacemit/k3-camera.dtsi
new file mode 100755
index 000000000000..0c7b03fd9b3c
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-camera.dtsi
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2023 SPACEMIT Micro Limited
+ */
+
+&soc {
+	csiphy0: csiphy@d420a000 {
+		compatible = "spacemit,csi-dphy";
+		cell-index = <0>;
+		spacemit,apmu = <&syscon_apmu>;
+		spacemit,ctrl-offset = <0x37c>;
+		spacemit,bifmode-enable = <0>;
+		reg = <0x0 0xd420a000 0x0 0x13f>;
+		reg-names = "csiphy-regs";
+		clocks = <&syscon_apmu CLK_APMU_CCIC1PHY>;
+		clock-names = "csi_dphy";
+		resets = <&syscon_apmu RESET_APMU_CCIC1_PHY>;
+		reset-names = "csi_dphy_reset";
+		status = "okay";
+	};
+
+	csiphy1: csiphy@d420a800 {
+		compatible = "spacemit,csi-dphy";
+		cell-index = <1>;
+		spacemit,apmu = <&syscon_apmu>;
+		spacemit,ctrl-offset = <0x37c>;
+		spacemit,bifmode-enable = <0>;
+		reg = <0x0 0xd420a800 0x0 0x13f>;
+		reg-names = "csiphy-regs";
+		clocks = <&syscon_apmu CLK_APMU_CCIC2PHY>;
+		clock-names = "csi_dphy";
+		resets = <&syscon_apmu RESET_APMU_CCIC2PHY>;
+		reset-names = "csi_dphy_reset";
+		status = "okay";
+	};
+
+	csiphy2: csiphy@d4206000 {
+		compatible = "spacemit,csi-dphy";
+		cell-index = <2>;
+		spacemit,apmu = <&syscon_apmu>;
+		spacemit,ctrl-offset = <0x37c>;
+		spacemit,bifmode-enable = <1>;
+		reg = <0x0 0xd4206000 0x0 0x13f>;
+		reg-names = "csiphy-regs";
+		clocks = <&syscon_apmu CLK_APMU_CCIC3PHY>;
+		clock-names = "csi_dphy";
+		resets = <&syscon_apmu RESET_APMU_CCIC3PHY>;
+		reset-names = "csi_dphy_reset";
+		status = "okay";
+	};
+
+	ccic0: cam_ccic@d420a000 {
+		status = "okay";
+		compatible = "spacemit,ccic";
+		cell-index = <0>;
+		spacemit,csiphy = <&csiphy0>;
+		reg = <0x0 0xd420a000 0x0 0x3ff>;
+		reg-names = "ccic-regs";
+		interrupt-parent = <&saplic>;
+		interrupts = <81 4>;
+		interrupt-names = "ccic-irq";
+		clocks = <&syscon_apmu CLK_APMU_CSI>,
+			 <&syscon_apmu CLK_APMU_CCIC_4X>,
+			 <&syscon_apmu CLK_APMU_SC2_HCLK>,
+			 <&syscon_apmu CLK_APMU_ISP_BUS>;
+		clock-names = "csi_func", "ccic_func", "sc2_ahb", "sc2_axi";
+		clock-rates = <1000000000>, <1000000000>, <409600000>, <409600000>;
+		resets = <&syscon_apmu RESET_APMU_CSI>,
+			 <&syscon_apmu RESET_APMU_CCIC_4X>,
+			 <&syscon_apmu RESET_APMU_SC2_HCLK>,
+			 <&syscon_apmu RESET_APMU_ISP_CIBUS>;
+		reset-names = "csi_reset", "ccic_4x_reset",
+			      "sc2_hclk_reset", "isp_cibus_reset";
+	};
+
+	ccic1: cam_ccic@d420a800 {
+		status = "okay";
+		compatible = "spacemit,ccic";
+		cell-index = <1>;
+		spacemit,csiphy = <&csiphy1>;
+		reg = <0x0 0xd420a800 0x0 0x3ff>;
+		reg-names = "ccic-regs";
+		interrupt-parent = <&saplic>;
+		interrupts = <82 4>;
+		interrupt-names = "ccic-irq";
+		clocks = <&syscon_apmu CLK_APMU_CSI>,
+			 <&syscon_apmu CLK_APMU_CCIC_4X>,
+			 <&syscon_apmu CLK_APMU_SC2_HCLK>,
+			 <&syscon_apmu CLK_APMU_ISP_BUS>;
+		clock-names = "csi_func", "ccic_func", "sc2_ahb", "sc2_axi";
+		clock-rates = <1000000000>, <1000000000>, <307200000>, <409600000>;
+		resets = <&syscon_apmu RESET_APMU_CSI>,
+			 <&syscon_apmu RESET_APMU_CCIC_4X>,
+			 <&syscon_apmu RESET_APMU_SC2_HCLK>,
+			 <&syscon_apmu RESET_APMU_ISP_CIBUS>;
+		reset-names = "csi_reset", "ccic_4x_reset",
+			      "sc2_hclk_reset", "isp_cibus_reset";
+	};
+
+	ccic2: cam_ccic@d4206000 {
+		status = "okay";
+		compatible = "spacemit,ccic";
+		cell-index = <2>;
+		spacemit,csiphy = <&csiphy2>;
+		reg = <0x0 0xd4206000 0x0 0x3ff>;
+		reg-names = "ccic-regs";
+		interrupt-parent = <&saplic>;
+		interrupts = <83 4>;
+		interrupt-names = "ccic-irq";
+		clocks = <&syscon_apmu CLK_APMU_CSI>,
+			 <&syscon_apmu CLK_APMU_CCIC_4X>,
+			 <&syscon_apmu CLK_APMU_SC2_HCLK>,
+			 <&syscon_apmu CLK_APMU_ISP_BUS>;
+		clock-names = "csi_func", "ccic_func", "sc2_ahb", "sc2_axi";
+		clock-rates = <1000000000>, <1000000000>, <307200000>, <409600000>;
+		resets = <&syscon_apmu RESET_APMU_CSI>,
+			 <&syscon_apmu RESET_APMU_CCIC_4X>,
+			 <&syscon_apmu RESET_APMU_SC2_HCLK>,
+			 <&syscon_apmu RESET_APMU_ISP_CIBUS>;
+		reset-names = "csi_reset", "ccic_4x_reset",
+			      "sc2_hclk_reset", "isp_cibus_reset";
+	};
+
+	ccic3: cam_ccic@d4206800 {
+		status = "okay";
+		compatible = "spacemit,ccic";
+		cell-index = <3>;
+		spacemit,csiphy = <&csiphy2>;
+		reg = <0x0 0xd4206800 0x0 0x3ff>;
+		reg-names = "ccic-regs";
+		interrupt-parent = <&saplic>;
+		interrupts = <71 4>;
+		interrupt-names = "ccic-irq";
+		clocks = <&syscon_apmu CLK_APMU_CSI>,
+			 <&syscon_apmu CLK_APMU_CCIC_4X>,
+			 <&syscon_apmu CLK_APMU_SC2_HCLK>,
+			 <&syscon_apmu CLK_APMU_ISP_BUS>;
+		clock-names = "csi_func", "ccic_func", "sc2_ahb", "sc2_axi";
+		clock-rates = <1000000000>, <1000000000>, <307200000>, <409600000>;
+		resets = <&syscon_apmu RESET_APMU_CSI>,
+			 <&syscon_apmu RESET_APMU_CCIC_4X>,
+			 <&syscon_apmu RESET_APMU_SC2_HCLK>,
+			 <&syscon_apmu RESET_APMU_ISP_CIBUS>;
+		reset-names = "csi_reset", "ccic_4x_reset",
+			      "sc2_hclk_reset", "isp_cibus_reset";
+	};
+
+	ccic_dma: cam_ccic_dma@d420f000 {
+		status = "okay";
+		compatible = "spacemit,ccic-dma";
+		cell-index = <0>;
+		reg = <0x0 0xd420f000 0x0 0x1080>;
+		reg-names = "ccic-dma-regs";
+		interrupt-parent = <&saplic>;
+		interrupts = <93 4>, <80 4>;
+		interrupt-names = "ccic-dma-irq", "ccic-dma-mmu-irq";
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-cpus.dtsi b/arch/riscv/boot/dts/spacemit/k3-cpus.dtsi
new file mode 100644
index 000000000000..968140e5ae99
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-cpus.dtsi
@@ -0,0 +1,902 @@
+/ {
+	cpus: cpus {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			timebase-frequency = <24000000>;
+
+			cpu_0: cpu@0 {
+				device_type = "cpu";
+				reg = <0>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache1>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu0_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_1: cpu@1 {
+				device_type = "cpu";
+				reg = <1>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache1>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu1_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_2: cpu@2 {
+				device_type = "cpu";
+				reg = <2>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache1>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu2_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_3: cpu@3 {
+				device_type = "cpu";
+				reg = <3>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache1>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu3_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_4: cpu@4 {
+				device_type = "cpu";
+				reg = <4>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache2>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu4_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_5: cpu@5 {
+				device_type = "cpu";
+				reg = <5>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache2>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu5_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_6: cpu@6 {
+				device_type = "cpu";
+				reg = <6>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache2>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu6_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_7: cpu@7 {
+				device_type = "cpu";
+				reg = <7>;
+				model = "Spacemit(R) X100";
+				status = "okay";
+				compatible = "riscv";
+				riscv,isa = "rv64imafdcvh";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v", "h",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache2>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu7_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_8: cpu@8 {
+				device_type = "cpu";
+				reg = <8>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache3>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu8_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_9: cpu@9 {
+				device_type = "cpu";
+				reg = <9>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache3>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu9_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_10: cpu@10 {
+				device_type = "cpu";
+				reg = <10>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache3>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu10_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_11: cpu@11 {
+				device_type = "cpu";
+				reg = <11>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache3>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu11_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_12: cpu@12 {
+				device_type = "cpu";
+				reg = <12>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache4>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu12_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_13: cpu@13 {
+				device_type = "cpu";
+				reg = <13>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache4>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu13_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_14: cpu@14 {
+				device_type = "cpu";
+				reg = <14>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache4>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu14_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			cpu_15: cpu@15 {
+				device_type = "cpu";
+				reg = <15>;
+				model = "Spacemit(R) A100";
+				status = "okay";
+				compatible = "riscv";
+				cpu-ai = "true";
+				riscv,isa = "rv64imafdcv";
+				riscv,isa-base = "rv64i";
+				riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "v",
+						"smaia", "smstateen", "ssaia", "sscofpmf",
+						"sstc", "svinval", "svnapot", "svpbmt",
+						"zawrs", "zba", "zbb", "zbc", "zbs", "zcb", "zca",
+						"zcmop", "zfa", "zfh", "zfhmin", "zkt", "zicbom",
+						"zicbop", "zicboz", "zicntr", "zicond", "zicsr",
+						"zifencei", "zihintpause", "zihintntl", "zihpm",
+						"zimop", "zvbb", "zvbc", "zvfh", "zvfhmin", "zvkb",
+						"zvkg", "zvkn", "zvknc", "zvkned", "zvkng", "zvknha",
+						"zvknhb", "zvks", "zvksc", "zvksed", "zvksh", "zvksg",
+						"zvkt", "zcd", "sdtrig";
+				riscv,cbom-block-size = <64>;
+				riscv,cboz-block-size = <64>;
+				riscv,cbop-block-size = <64>;
+				i-cache-block-size = <64>;
+				i-cache-size = <65536>;
+				i-cache-sets = <256>;
+				d-cache-block-size = <64>;
+				d-cache-size = <65536>;
+				d-cache-sets = <256>;
+				next-level-cache = <&l2_cache4>;
+				cpu-idle-states = <&CPU_NONRET>, <&CLUSTER_NONRET>;
+				mmu-type = "riscv,sv39";
+				#cooling-cells = <2>;
+				cpu15_intc: interrupt-controller {
+					#interrupt-cells = <1>;
+					compatible = "riscv,cpu-intc";
+					interrupt-controller;
+				};
+				debug {
+					compatible = "riscv,debug-v1.0.0";
+					trigger-module {
+						mcontext-present;
+						hcontext-present;
+						scontext-present;
+					};
+				};
+			};
+
+			l2_cache1: l2-cache1 {
+				compatible = "cache";
+				cache-block-size = <64>;
+				cache-level = <2>;
+				cache-size = <4194304>;
+				cache-sets = <4096>;
+				cache-unified;
+			};
+
+			l2_cache2: l2-cache2 {
+				compatible = "cache";
+				cache-block-size = <64>;
+				cache-level = <2>;
+				cache-size = <4194304>;
+				cache-sets = <4096>;
+				cache-unified;
+			};
+
+			l2_cache3: l2-cache3 {
+				compatible = "cache";
+				cache-block-size = <64>;
+				cache-level = <2>;
+				cache-size = <1048576>;
+				cache-sets = <1024>;
+				cache-unified;
+			};
+
+			l2_cache4: l2-cache4 {
+				compatible = "cache";
+				cache-block-size = <64>;
+				cache-level = <2>;
+				cache-size = <1048576>;
+				cache-sets = <1024>;
+				cache-unified;
+			};
+
+			cpu-map {
+				cluster0 {
+					core0 {
+						cpu = <&cpu_0>;
+					};
+
+					core1 {
+						cpu = <&cpu_1>;
+					};
+
+					core2 {
+						cpu = <&cpu_2>;
+					};
+
+					core3 {
+						cpu = <&cpu_3>;
+					};
+				};
+				cluster1 {
+					core0 {
+						cpu = <&cpu_4>;
+					};
+
+					core1 {
+						cpu = <&cpu_5>;
+					};
+
+					core2 {
+						cpu = <&cpu_6>;
+					};
+
+					core3 {
+						cpu = <&cpu_7>;
+					};
+				};
+				cluster2 {
+					core0 {
+						cpu = <&cpu_8>;
+					};
+
+					core1 {
+						cpu = <&cpu_9>;
+					};
+
+					core2 {
+						cpu = <&cpu_10>;
+					};
+
+					core3 {
+						cpu = <&cpu_11>;
+					};
+				};
+				cluster3 {
+					core0 {
+						cpu = <&cpu_12>;
+					};
+
+					core1 {
+						cpu = <&cpu_13>;
+					};
+
+					core2 {
+						cpu = <&cpu_14>;
+					};
+
+					core3 {
+						cpu = <&cpu_15>;
+					};
+				};
+			};
+	};
+
+	riscv-timer {
+		compatible = "riscv,timer";
+		riscv,timer-cannot-wake-cpu;
+	};
+
+	idle-states {
+		CPU_NONRET: cpu-nonret {
+			compatible = "riscv,idle-state";
+			idle-state-name = "riscv,cpu-nonret";
+			riscv,sbi-suspend-param = <0x90000000>;
+			local-timer-stop;
+			entry-latency-us = <200>;
+			exit-latency-us = <500>;
+			min-residency-us = <950>;
+		};
+
+		CLUSTER_NONRET: cluster-nonret {
+			compatible = "riscv,idle-state";
+			idle-state-name = "riscv,cluster-nonret";
+			riscv,sbi-suspend-param = <0x91000000>;
+			local-timer-stop;
+			entry-latency-us = <600>;
+			exit-latency-us = <1100>;
+			min-residency-us = <2700>;
+			wakeup-latency-us = <1500>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts b/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts
new file mode 100644
index 000000000000..81633504a946
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts
@@ -0,0 +1,1454 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "k3-edp0.dtsi"
+#include "k3-dp1.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/input/cros-ec-keyboard.h>
+
+/ {
+	model = "DeepComputing FML13V05";
+	compatible = "deepcomputing,fml13v05", "spacemit,k3";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	aux_5v: aux_5v {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	aux_3v3: aux_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&aux_5v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+                bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1";
+                rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	fan0: pwm-fan {
+		compatible = "pwm-fan";
+		pwms = <&cros_ec_pwm 0>;
+		/*
+		* Cooling states (0-8) -> PWM duty (0-255)
+		* Full speed is treated as ~6000RPM.
+		* Custom ramp to balance noise and thermals
+		*/
+		/* Keep headroom by capping max duty below 255. */
+		cooling-levels = <0 30 60 90 120 140 179 204 242>;
+		#cooling-cells = <2>;
+	};
+
+	vcc_wifi_pwr: vcc-wifi-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_wifi_3v3";
+		gpio = <&gpio 1 9 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+		regulator-boot-on;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rfkill_gpio_cfg>;
+	};
+
+	rfkill_wlan: rfkill-wlan {
+		compatible = "rfkill-gpio";
+		label = "spacemit-wlan";
+		radio-type = "wlan";
+		shutdown-gpio = <&gpio 1 3 GPIO_ACTIVE_HIGH>;
+		dev-pwr-supply = <&vcc_wifi_pwr>;
+		status = "okay";
+	};
+
+	rfkill_bt: rfkill-bt {
+		compatible = "rfkill-gpio";
+		label = "spacemit-bt";
+		radio-type = "bluetooth";
+		shutdown-gpio = <&gpio 0 30 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	vcc_cam_dmic: vcc-cam-dmic {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_cam_dmic";
+		gpio = <&gpio 3 4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		regulator-always-on;
+		regulator-boot-on;
+
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_dmic_cfg>;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm18 0 20000 0>;
+		brightness-levels = <
+			  0  20  20  20  21  21  21  21
+			 22  22  22  22  23  23  23  24
+			 24  24  25  25  25  26  26  27
+			 27  28  28  29  29  30  30  31
+			 31  32  32  33  33  34  34  35
+			 36  36  37  38  38  39  40  41
+			 41  42  43  44  45  45  46  47
+			 48  49  50  51  52  53  54  55
+			 56  57  58  59  60  61  62  63
+			 64  65  67  68  69  70  72  73
+			 74  75  77  78  79  81  82  84
+			 85  87  88  90  91  93  94  96
+			 98  99 101 103 104 106 108 110
+			112 113 115 117 119 121 123 125
+			127 129 131 133 135 137 139 141
+			143 145 147 149 152 154 156 158
+			161 163 165 168 170 173 175 178
+			180 183 186 188 191 194 196 199
+			202 205 208 210 213 216 219 222
+			225 228 231 234 238 241 244 247
+			251 255
+		>;
+		default-brightness-level = <128>;
+		status = "okay";
+	};
+
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+
+	pcie_dma_pool: pcie-dma-pool@1f0000000 {
+		compatible = "restricted-dma-pool";
+		reg = <0x1 0xf0000000 0x0 0x02000000>;
+	};
+};
+
+&pinctrl {
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c1-3-cfg {
+		i2c1-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c3-1-cfg {
+		i2c3-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	usb30_drd_dir-0-cfg {
+		usb30_drd_dir-pins {
+			power-source = <1800>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(85, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,
+				 <K3_PADCONF(81, 5)>;
+
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <1800>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <1800>;
+		};
+	};
+
+	sspa1_1_cfg: sspa1-1-cfg {
+		sspa1-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	rfkill_gpio_cfg: rfkill-gpio-cfg {
+		rfkill-pins {
+			pinmux = <K3_PADCONF(30, 0)>,	/* BT_EN: gpio30 */
+				 <K3_PADCONF(35, 0)>,	/* WLAN_W_DISABLE1: gpio35 */
+				 <K3_PADCONF(41, 0)>;	/* WLAN_PWREN: gpio41 */
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <3300>;
+		};
+	};
+
+	cam_dmic_cfg: cam-dmic-cfg {
+		cam-dmic-pins {
+			pinmux = <K3_PADCONF(100, 0)>;	/* gpio[5]_100 */
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <3300>;
+		};
+	};
+
+	dp0_1_cfg: dp0-1-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(23, 6)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	dp1_1_cfg: dp1-1-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(24, 6)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pwm18_2_cfg: pwm18-2-cfg {
+		pwm18-0-pins {
+			pinmux = <K3_PADCONF(106, 6)>;	/* pwm18 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(88, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+	};
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 2 24 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&emmc {
+	bus-width = <8>;
+	non-removable;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	clock-frequency = <375000000>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reset-on-resume;
+	status = "okay";
+
+	/* Ensure no deferred probe cause USB disconnect */
+	/* For BT, FG, CAM */
+	hub@1 {
+		/* FE1.1S USB2.0 hub (4 ports) */
+		compatible = "usb5e3,608";
+		reg = <1>;
+	};
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	/* dual phy selection is auto detect from GPIO86(SS_SW_3V3) */
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb30_drd_dir_0_cfg>;
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	reset-on-resume;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	/* TypeC */
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "okay";
+};
+
+&usb3_portc {
+	/* TypeC */
+	reset-on-resume;
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	/* TypeC */
+	status = "okay";
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy2 {
+	status = "disabled";
+};
+
+&phy3 {
+	status = "disabled";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy0";
+	dma-ranges = <0x02000000 0x0 0x80000000 0x1 0xf0000000 0x0 0x02000000>;
+	memory-region = <&pcie_dma_pool>;
+	status = "okay";
+};
+
+&espi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&espi0_2_cfg>;
+	status = "okay";
+	ec: ec@0 {
+		status = "okay";
+		interrupt-parent = <&gpio>;
+		interrupts = <1 11 IRQ_TYPE_EDGE_RISING>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		i2c_tunnel1: i2c-tunnel@1 {
+			compatible = "google,cros-ec-i2c-tunnel";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			google,remote-bus = <1>;
+		};
+
+		i2c_tunnel4: i2c-tunnel@4 {
+			compatible = "google,cros-ec-i2c-tunnel";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			google,remote-bus = <4>;
+
+			battery: battery@b {
+				compatible = "sbs,sbs-battery";
+				reg = <0x0b>;
+				sbs,i2c-retry-count = <2>;
+				sbs,poll-retry-count = <10>;
+				status = "okay";
+			};
+		};
+
+		/* EC PWM provider for fan control */
+		cros_ec_pwm: pwm {
+			compatible = "google,cros-ec-pwm";
+			#pwm-cells = <1>;
+		};
+
+		keyboard {
+			compatible = "google,cros-ec-keyb";
+			keypad,num-rows = <8>;
+			keypad,num-columns = <16>;
+			linux,keymap = <
+			/* Row 0 */
+			MATRIX_KEY(0x00, 0x00, KEY_C)           /* ID: 48 */
+			MATRIX_KEY(0x00, 0x01, KEY_DELETE)      /* ID: 76 */
+			MATRIX_KEY(0x00, 0x02, KEY_Q)           /* ID: 17 */
+			MATRIX_KEY(0x00, 0x03, KEY_RIGHTALT)    /* ID: 62 */
+			MATRIX_KEY(0x00, 0x05, KEY_X)           /* ID: 47 */
+			MATRIX_KEY(0x00, 0x06, KEY_V)           /* ID: 49 */
+			MATRIX_KEY(0x00, 0x07, KEY_M)           /* ID: 52 */
+			MATRIX_KEY(0x00, 0x08, KEY_DOT)         /* ID: 54 */
+			MATRIX_KEY(0x00, 0x09, KEY_RIGHTSHIFT)  /* ID: 57 */
+			MATRIX_KEY(0x00, 0x0a, KEY_COMMA)       /* ID: 53 */
+			MATRIX_KEY(0x00, 0x0c, KEY_RIGHTCTRL)   /* ID: 64 */
+			MATRIX_KEY(0x00, 0x0d, KEY_SLASH)       /* ID: 55 */
+			MATRIX_KEY(0x00, 0x0e, KEY_APOSTROPHE)  /* ID: 41 (' ) */
+
+			/* Row 1 */
+			MATRIX_KEY(0x01, 0x03, KEY_LEFTALT)     /* ID: 60 */
+			MATRIX_KEY(0x01, 0x04, KEY_SPACE)       /* ID: 61 */
+			MATRIX_KEY(0x01, 0x05, KEY_Z)           /* ID: 46 */
+			MATRIX_KEY(0x01, 0x06, KEY_B)           /* ID: 50 */
+			MATRIX_KEY(0x01, 0x07, KEY_N)           /* ID: 51 */
+			MATRIX_KEY(0x01, 0x08, KEY_DOWN)        /* ID: 84-1 */
+			MATRIX_KEY(0x01, 0x09, KEY_LEFTSHIFT)   /* ID: 44 */
+			MATRIX_KEY(0x01, 0x0c, KEY_LEFTCTRL)    /* ID: 58 */
+			MATRIX_KEY(0x01, 0x0d, KEY_UP)          /* ID: 83-1 */
+			MATRIX_KEY(0x01, 0x0e, KEY_ENTER)       /* ID: 43 */
+			MATRIX_KEY(0x01, 0x0f, KEY_F8)          /* ID: 119 */
+
+			/* Row 2 */
+			MATRIX_KEY(0x02, 0x02, KEY_FN)          /* ID: 59 */
+			MATRIX_KEY(0x02, 0x04, KEY_E)           /* ID: 19 */
+			MATRIX_KEY(0x02, 0x05, KEY_F2)          /* ID: 113 */
+			MATRIX_KEY(0x02, 0x06, KEY_G)           /* ID: 35 */
+			MATRIX_KEY(0x02, 0x07, KEY_H)           /* ID: 36 */
+			MATRIX_KEY(0x02, 0x08, KEY_BACKSLASH)   /* ID: 29/42 */
+			MATRIX_KEY(0x02, 0x0a, KEY_F7)          /* ID: 118 */
+			MATRIX_KEY(0x02, 0x0d, KEY_MINUS)       /* ID: 12 */
+			MATRIX_KEY(0x02, 0x0f, KEY_RIGHT)       /* ID: 89-1 */
+
+			/* Row 3 */
+			MATRIX_KEY(0x03, 0x01, KEY_LEFTMETA)    /* ID: 127 (Win) */
+			MATRIX_KEY(0x03, 0x02, KEY_TAB)         /* ID: 16 */
+			MATRIX_KEY(0x03, 0x04, KEY_F4)          /* ID: 115 */
+			MATRIX_KEY(0x03, 0x05, KEY_F1)          /* ID: 112 */
+			MATRIX_KEY(0x03, 0x06, KEY_T)           /* ID: 21 */
+			MATRIX_KEY(0x03, 0x07, KEY_Y)           /* ID: 22 */
+			MATRIX_KEY(0x03, 0x08, KEY_O)           /* ID: 25 */
+			MATRIX_KEY(0x03, 0x0a, KEY_F6)          /* ID: 117 */
+			MATRIX_KEY(0x03, 0x0b, KEY_F9)          /* ID: 120 */
+			MATRIX_KEY(0x03, 0x0d, KEY_F12)         /* ID: 123 */
+
+			/* Row 4 */
+			MATRIX_KEY(0x04, 0x02, KEY_GRAVE)       /* ID: 1 (~) */
+			MATRIX_KEY(0x04, 0x04, KEY_CAPSLOCK)    /* ID: 30 */
+			MATRIX_KEY(0x04, 0x05, KEY_S)           /* ID: 32 */
+			MATRIX_KEY(0x04, 0x06, KEY_5)           /* ID: 6 */
+			MATRIX_KEY(0x04, 0x07, KEY_6)           /* ID: 7 */
+			MATRIX_KEY(0x04, 0x08, KEY_F10)         /* ID: 121 */
+			MATRIX_KEY(0x04, 0x0a, KEY_F5)          /* ID: 116 */
+			MATRIX_KEY(0x04, 0x0d, KEY_0)           /* ID: 11 */
+			MATRIX_KEY(0x04, 0x0e, KEY_EQUAL)       /* ID: 13 */
+
+			/* Row 5 */
+			MATRIX_KEY(0x05, 0x02, KEY_1)           /* ID: 2 */
+			MATRIX_KEY(0x05, 0x04, KEY_3)           /* ID: 4 */
+			MATRIX_KEY(0x05, 0x05, KEY_2)           /* ID: 3 */
+			MATRIX_KEY(0x05, 0x06, KEY_4)           /* ID: 5 */
+			MATRIX_KEY(0x05, 0x07, KEY_7)           /* ID: 8 */
+			MATRIX_KEY(0x05, 0x08, KEY_9)           /* ID: 10 */
+			MATRIX_KEY(0x05, 0x0a, KEY_8)           /* ID: 9 */
+			MATRIX_KEY(0x05, 0x0b, KEY_102ND)       /* ID: 45 */
+			MATRIX_KEY(0x05, 0x0d, KEY_P)           /* ID: 26 */
+			MATRIX_KEY(0x05, 0x0e, KEY_BACKSPACE)   /* ID: 15 */
+
+			/* Row 6 */
+			MATRIX_KEY(0x06, 0x04, KEY_F3)          /* ID: 114 */
+			MATRIX_KEY(0x06, 0x05, KEY_W)           /* ID: 18 */
+			MATRIX_KEY(0x06, 0x06, KEY_R)           /* ID: 20 */
+			MATRIX_KEY(0x06, 0x07, KEY_U)           /* ID: 23 */
+			MATRIX_KEY(0x06, 0x08, KEY_F11)         /* ID: 122 */
+			MATRIX_KEY(0x06, 0x0a, KEY_I)           /* ID: 24 */
+			MATRIX_KEY(0x06, 0x0b, KEY_LEFT)        /* ID: 79-1 */
+			MATRIX_KEY(0x06, 0x0d, KEY_LEFTBRACE)   /* ID: 27 ([) */
+			MATRIX_KEY(0x06, 0x0e, KEY_RIGHTBRACE)  /* ID: 28 (]) */
+
+			/* Row 7 */
+			MATRIX_KEY(0x07, 0x02, KEY_A)           /* ID: 31 */
+			MATRIX_KEY(0x07, 0x05, KEY_ESC)         /* ID: 110 */
+			MATRIX_KEY(0x07, 0x06, KEY_F)           /* ID: 34 */
+			MATRIX_KEY(0x07, 0x07, KEY_J)           /* ID: 37 */
+			MATRIX_KEY(0x07, 0x08, KEY_L)           /* ID: 39 */
+			MATRIX_KEY(0x07, 0x0a, KEY_K)           /* ID: 38 */
+			MATRIX_KEY(0x07, 0x0d, KEY_SEMICOLON)   /* ID: 40 */
+			MATRIX_KEY(0x07, 0x0e, KEY_D)           /* ID: 33 */
+
+			/*
+			 * Fn layer (row + 8):
+			 *  F1..F12 -> media/system keys when Fn is held.
+			 */
+			MATRIX_KEY(0x08, 0x01, KEY_INSERT)        /* Fn+Delete = Insert */
+			MATRIX_KEY(0x09, 0x08, KEY_PAGEDOWN)      /* Fn+Down */
+			MATRIX_KEY(0x09, 0x0d, KEY_PAGEUP)        /* Fn+Up */
+			MATRIX_KEY(0x09, 0x0f, KEY_BRIGHTNESSUP)  /* Fn+F8 */
+			MATRIX_KEY(0x0a, 0x05, KEY_VOLUMEDOWN)    /* Fn+F2 */
+			MATRIX_KEY(0x0a, 0x0a, KEY_BRIGHTNESSDOWN)/* Fn+F7 */
+			MATRIX_KEY(0x0a, 0x0f, KEY_END)           /* Fn+Right */
+			MATRIX_KEY(0x0b, 0x04, KEY_PREVIOUSSONG)  /* Fn+F4 */
+			MATRIX_KEY(0x0b, 0x05, KEY_MUTE)          /* Fn+F1 */
+			MATRIX_KEY(0x0b, 0x0a, KEY_NEXTSONG)      /* Fn+F6 */
+			MATRIX_KEY(0x0b, 0x0b, KEY_SWITCHVIDEOMODE)/* Fn+F9 */
+			MATRIX_KEY(0x0b, 0x0d, KEY_CONFIG)        /* Fn+F12 */
+			MATRIX_KEY(0x0c, 0x08, KEY_RFKILL)        /* Fn+F10 */
+			MATRIX_KEY(0x0c, 0x0a, KEY_PLAYPAUSE)     /* Fn+F5 */
+			MATRIX_KEY(0x0e, 0x04, KEY_VOLUMEUP)      /* Fn+F3 */
+			MATRIX_KEY(0x0e, 0x08, KEY_SYSRQ)         /* Fn+F11 */
+			MATRIX_KEY(0x0e, 0x0b, KEY_HOME)          /* Fn+Left */
+			>;
+		};
+	};
+};
+
+&ec_master {
+	status = "okay";
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <45000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <60000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			thermal_gpu_trip0: thermal_gpu-trip0 {
+				temperature = <60000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <45000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <55000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip5: thermal_cluster0-trip5 {
+				temperature = <90000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip6: thermal_cluster0-trip6 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&fan0 0 1>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&fan0 2 3>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&fan0 3 4>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&fan0 4 5>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&fan0 6 6>;
+			};
+
+			map5 {
+				trip = <&thermal_cluster0_trip5>;
+				cooling-device = <&fan0 7 7>;
+			};
+
+			map6 {
+				trip = <&thermal_cluster0_trip6>;
+				cooling-device = <&fan0 8 8>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_3_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 2 21 GPIO_ACTIVE_LOW>;
+		suspend-vbus-off;
+		wakeup-source;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "source";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_1_cfg>;
+	status = "okay";
+
+	es8326: es8326@19{
+		compatible = "everest,es8326";
+		reg = <0x19>;
+		#sound-dai-cells = <0>;
+		interrupt-parent = <&gpio>;
+		interrupts = <3 31 IRQ_TYPE_EDGE_RISING>;
+		spk-ctl-gpio = <&gpio 3 23 0>;
+		everest,mic1-src = [44];
+		everest,mic2-src = [66];
+		everest,jack-detect-inverted;
+		status = "okay";
+	};
+};
+
+&i2c4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c4_2_cfg>;
+	status = "okay";
+
+	touchpad: hid@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+		hid-descr-addr = <0x0020>;
+		post-power-on-delay-ms = <200>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&touchpad_int_cfg>;
+		interrupt-parent = <&gpio>;
+		interrupts = <0 26 IRQ_TYPE_EDGE_FALLING>;
+		status = "okay";
+	};
+
+	light_sensor: cm32183@29 {
+		compatible = "capella,cm32183";
+		reg = <0x29>;
+		status = "okay";
+	};
+
+        light_sensor2: cm32181@10 {
+		compatible = "capella,cm32181";
+		reg = <0x10>;
+		status = "okay";
+	};
+
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&aux_5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&pwm18 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm18_2_cfg>;
+	status = "okay";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "okay";
+};
+
+&edp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_1_cfg>;
+	backlight = <&backlight>;
+	gpios-power = <101>;
+	gpios-enable = <118>;
+	status = "okay";
+};
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_1_cfg>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
+
+&i2s1 {
+	status = "okay";
+	pinctrl-0 = <&sspa1_1_cfg>;
+	pinctrl-names = "default";
+};
+
+&sound_card1 {
+	status = "okay";
+	simple-audio-card,mclk-fs = <256>;
+	simple-audio-card,dai-link@0 {
+		format = "i2s";
+		frame-master = <&link1_cpu>;
+		bitclock-master = <&link1_cpu>;
+
+		link1_cpu: cpu {
+			sound-dai = <&i2s1>;
+		};
+
+		link1_codec: codec {
+			sound-dai = <&es8326>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-dp0.dtsi b/arch/riscv/boot/dts/spacemit/k3-dp0.dtsi
new file mode 100644
index 000000000000..7b0225a1c230
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-dp0.dtsi
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+&soc {
+	display-subsystem-dp0 {
+		compatible = "spacemit,saturn-hee";
+		reg = <0 0xc0340000 0 0x54000>;
+		hw_ver = <2>;
+		ports = <&dpu0_crtc0>;
+	};
+
+	dpu0_crtc0: dpu0_crtc0 {
+		compatible = "spacemit,dpu-saturn";
+		interrupt-parent = <&saplic>;
+		interrupts = <90 IRQ_TYPE_LEVEL_HIGH>,
+			     <89 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "ONLINE_IRQ",
+				  "OFFLINE_IRQ";
+		clocks = <&syscon_apmu CLK_APMU_LCD_PXCLK>,
+			<&syscon_apmu CLK_APMU_LCD_MCLK>,
+			<&syscon_apmu CLK_APMU_LCD_HCLK>,
+			<&syscon_apmu CLK_APMU_DSI_ESC>,
+			<&syscon_apmu CLK_APMU_DPU_ACLK>,
+			<&syscon_apmu CLK_APMU_LCD_DSC>;
+		clock-names = "pxclk", "mclk", "hclk", "escclk", "aclk", "dscclk";
+		resets = <&syscon_apmu RESET_APMU_LCD_MCLK>,
+			<&syscon_apmu RESET_APMU_LCD>,
+			<&syscon_apmu RESET_APMU_DSI_ESC>,
+			<&syscon_apmu RESET_APMU_DPU_ACLK>,
+			<&syscon_apmu RESET_APMU_LCD_DSCCLK>;
+		reset-names= "mclk_reset", "lcd_reset", "esc_reset", "aclk_reset", "dsc_reset";
+		power-domains = <&power K3_PMU_LCD0_PWR_DOMAIN>;
+		pipeline-id = <1>;
+		is_edp = <1>;
+		dpu-id = <0>;
+		ip = "spacemit-saturn";
+		spacemit-dpu-min-mclk = <40960000>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				dpu0_crtc0_out0: endpoint@0 {
+					remote-endpoint = <&dp0_in>;
+				};
+			};
+		};
+	};
+
+	dp0: dp0@cac84000 {
+		compatible = "spacemit,inno-dp0";
+		reg = <0x0 0xcac84000 0x0 0x4000>;
+		spacemit,apmu = <&syscon_apmu>;
+		interrupt-parent = <&saplic>;
+		interrupts = <132 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_apmu CLK_APMU_EDP0_PXCLK>;
+		clock-names = "pxclk";
+		resets = <&syscon_apmu RESET_APMU_EDP0>;
+		reset-names= "reset";
+		dp-id = <0>;
+		#sound-dai-cells = <0>;
+		status = "disabled";
+
+		port {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dp0_in: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&dpu0_crtc0_out0>;
+			};
+		};
+	};
+
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-dp1.dtsi b/arch/riscv/boot/dts/spacemit/k3-dp1.dtsi
new file mode 100644
index 000000000000..9f880b169ffe
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-dp1.dtsi
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+&soc {
+	display-subsystem-dp1 {
+		compatible = "spacemit,saturn-edp";
+		reg = <0 0xc0440000 0 0x54000>;
+		hw_ver = <2>;
+		ports = <&dpu1_crtc0>;
+	};
+
+	dpu1_crtc0: dpu1_crtc0 {
+		compatible = "spacemit,dpu-saturn";
+		interrupt-parent = <&saplic>;
+		interrupts = <139 IRQ_TYPE_LEVEL_HIGH>,
+			     <138 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "ONLINE_IRQ",
+				  "OFFLINE_IRQ";
+		clocks = <&syscon_apmu CLK_APMU_DSI4LN2_LCD_PXCLK>,
+			<&syscon_apmu CLK_APMU_DSI4LN2_LCD_MCLK>,
+			<&syscon_apmu CLK_APMU_DSI4LN2_DPU_ACLK>,
+			<&syscon_apmu CLK_APMU_DSI4LN2_DSI_ESC>,
+			<&syscon_apmu CLK_APMU_DSI4LN2_LCD_DSC>;
+		clock-names = "pxclk", "mclk", "aclk", "escclk", "dscclk";
+		resets = <&syscon_apmu RESET_APMU_DSI4LN2_LCD_MCLK>,
+			<&syscon_apmu RESET_APMU_DSI4LN2_ESCCLK>,
+			<&syscon_apmu RESET_APMU_DSI4LN2_LCD_SW>,
+			<&syscon_apmu RESET_APMU_DSI4LN2_DPU_ACLK>,
+			<&syscon_apmu RESET_APMU_DSI4LN2_LCD_DSCCLK>;
+		reset-names= "mclk_reset", "esc_reset", "lcd_reset", "aclk_reset", "dsc_reset";
+		power-domains = <&power K3_PMU_LCD1_PWR_DOMAIN>;
+		pipeline-id = <1>;
+		is_edp = <1>;
+		dpu-id = <1>;
+		ip = "spacemit-saturn";
+		spacemit-dpu-min-mclk = <40960000>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				dpu1_crtc0_out0: endpoint@0 {
+					remote-endpoint = <&dp1_in>;
+				};
+			};
+		};
+	};
+
+	dp1: dp1@cac88000 {
+		compatible = "spacemit,inno-dp1";
+		reg = <0x0 0xcac88000 0x0 0x4000>;
+		spacemit,apmu = <&syscon_apmu>;
+		interrupt-parent = <&saplic>;
+		interrupts = <140 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_apmu CLK_APMU_EDP1_PXCLK>;
+		clock-names = "pxclk";
+		resets = <&syscon_apmu RESET_APMU_EDP1>;
+		reset-names= "reset";
+		dp-id = <1>;
+		#sound-dai-cells = <0>;
+		status = "disabled";
+
+		port {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dp1_in: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&dpu1_crtc0_out0>;
+			};
+		};
+	};
+
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-edp0.dtsi b/arch/riscv/boot/dts/spacemit/k3-edp0.dtsi
new file mode 100755
index 000000000000..f2ccdef951dc
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-edp0.dtsi
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+&soc {
+	display-subsystem-edp0 {
+		compatible = "spacemit,saturn-hee";
+		reg = <0 0xc0340000 0 0x54000>;
+		hw_ver = <2>;
+		ports = <&dpu0_crtc0>;
+	};
+
+	dpu0_crtc0: dpu0_crtc0 {
+		compatible = "spacemit,dpu-saturn";
+		interrupt-parent = <&saplic>;
+		interrupts = <90 IRQ_TYPE_LEVEL_HIGH>,
+			     <89 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "ONLINE_IRQ",
+				  "OFFLINE_IRQ";
+		clocks = <&syscon_apmu CLK_APMU_LCD_PXCLK>,
+			<&syscon_apmu CLK_APMU_LCD_MCLK>,
+			<&syscon_apmu CLK_APMU_LCD_HCLK>,
+			<&syscon_apmu CLK_APMU_DSI_ESC>,
+			<&syscon_apmu CLK_APMU_DPU_ACLK>,
+			<&syscon_apmu CLK_APMU_LCD_DSC>;
+		clock-names = "pxclk", "mclk", "hclk", "escclk", "aclk", "dscclk";
+		resets = <&syscon_apmu RESET_APMU_LCD_MCLK>,
+			<&syscon_apmu RESET_APMU_LCD>,
+			<&syscon_apmu RESET_APMU_DSI_ESC>,
+			<&syscon_apmu RESET_APMU_DPU_ACLK>,
+			<&syscon_apmu RESET_APMU_LCD_DSCCLK>;
+		reset-names= "mclk_reset", "lcd_reset", "esc_reset", "aclk_reset", "dsc_reset";
+		power-domains = <&power K3_PMU_LCD0_PWR_DOMAIN>;
+		pipeline-id = <1>;
+		is_edp = <1>;
+		dpu-id = <0>;
+		ip = "spacemit-saturn";
+		spacemit-dpu-min-mclk = <40960000>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				dpu0_crtc0_out0: endpoint@0 {
+					remote-endpoint = <&edp0_in>;
+				};
+			};
+		};
+	};
+
+	edp0: edp0@cac84000 {
+		compatible = "spacemit,inno-edp0";
+		reg = <0x0 0xcac84000 0x0 0x4000>;
+		spacemit,apmu = <&syscon_apmu>;
+		interrupt-parent = <&saplic>;
+		interrupts = <132 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_apmu CLK_APMU_EDP0_PXCLK>;
+		clock-names = "pxclk";
+		resets = <&syscon_apmu RESET_APMU_EDP0>;
+		reset-names= "reset";
+		edp-id = <0>;
+		status = "disabled";
+
+		port {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			edp0_in: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&dpu0_crtc0_out0>;
+			};
+		};
+	};
+
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
new file mode 100644
index 000000000000..951278bb0200
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
@@ -0,0 +1,1505 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "k3-edp0.dtsi"
+#include "k3-dp1.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/input/cros-ec-keyboard.h>
+
+/ {
+	model = "SpacemiT K3 Pico ITX";
+	compatible = "spacemit,k3-pico-itx";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	aux_5v: aux_5v {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	aux_3v3: aux_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&aux_5v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+                bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+                rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	rfkill-pcie-wlan {
+		compatible = "rfkill-gpio";
+		label = "rfkill-pcie-wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wlan_en_cfg>;
+		radio-type = "wlan";
+		shutdown-gpios = <&gpio 1 3 GPIO_ACTIVE_HIGH>;
+	};
+
+	rfkill-usb-bt {
+		compatible = "rfkill-gpio";
+		label = "rfkill-usb-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_en_cfg>;
+		radio-type = "bluetooth";
+		shutdown-gpios = <&gpio 0 30 GPIO_ACTIVE_HIGH>;
+	};
+
+	rfkill-usb-wwan {
+		compatible = "rfkill-gpio";
+		label = "m.2 WWAN"; /* B-Key */
+		radio-type = "wwan";
+		/* MODULE_TURN_ON_1V8 (FC_PWROFF#) */
+		shutdown-gpios = <&gpio 0 18 GPIO_ACTIVE_HIGH>;
+		/* WWAN_RESET#_1V8 (WWAN1V8_RST#) */
+		reset-gpios = <&gpio 0 19 GPIO_ACTIVE_HIGH>;
+		/* GPIO84 for CPU_W_DISABLE#1 (WWAN_DIS#) is no needed*/
+	};
+
+	fan0: pwm-fan {
+		compatible = "pwm-fan";
+		pwms = <&cros_ec_pwm 0>;
+		/*
+		* Cooling states (0-8) -> PWM duty (0-255)
+		* Full speed is treated as ~6000RPM.
+		* Custom ramp to balance noise and thermals
+		*/
+		/* Keep headroom by capping max duty below 255. */
+		cooling-levels = <0 30 60 90 120 149 170 191 213>;
+		#cooling-cells = <2>;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm18 0 20000 0>;
+		brightness-levels = <
+			0   1   2   3   4   5   6   7
+			8   9  10  11  12  13  14  15
+			16  17  18  19  20  21  22  23
+			24  25  26  27  28  29  30  31
+			32  33  34  35  36  37  38  39
+			40  41  42  43  44  45  46  47
+			48  49  50  51  52  53  54  55
+			56  57  58  59  60  61  62  63
+			64  65  66  67  68  69  70  71
+			72  73  74  75  76  77  78  79
+			80  81  82  83  84  85  86  87
+			88  89  90  91  92  93  94  95
+			96  97  98  99 100 101 102 103
+			104 105 106 107 108 109 110 111
+			112 113 114 115 116 117 118 119
+			120 121 122 123 124 125 126 127
+			128 129 130 131 132 133 134 135
+			136 137 138 139 140 141 142 143
+			144 145 146 147 148 149 150 151
+			152 153 154 155 156 157 158 159
+			160 161 162 163 164 165 166 167
+			168 169 170 171 172 173 174 175
+			176 177 178 179 180 181 182 183
+			184 185 186 187 188 189 190 191
+			192 193 194 195 196 197 198 199
+			200 201 202 203 204 205 206 207
+			208 209 210 211 212 213 214 215
+			216 217 218 219 220 221 222 223
+			224 225 226 227 228 229 230 231
+			232 233 234 235 236 237 238 239
+			240 241 242 243 244 245 246 247
+			248 249 250 251 252 253 254 255>;
+		default-brightness-level = <128>;
+		status = "okay";
+	};
+
+	hub_vbus: hub-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "hub-vbus";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		regulator-boot-on;
+		gpio = <&ec 8 GPIO_ACTIVE_HIGH>;
+	};
+
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	gmac0-cfg {
+		/delete-node/ gmac0-2-pins;
+		/delete-node/ gmac0-4-pins;
+		/delete-node/ gmac0-5-pins;
+
+		gmac0-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-6-pins {
+			pinmux = <K3_PADCONF(15, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c1-3-cfg {
+		i2c1-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c3-1-cfg {
+		i2c3-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	usb30_drd_dir-0-cfg {
+		usb30_drd_dir-pins {
+			power-source = <1800>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(85, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,
+				 <K3_PADCONF(81, 5)>;
+
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <1800>;
+		};
+	};
+
+	pcie1-1-cfg {
+		pcie1-0-pins {
+			pinmux = <K3_PADCONF(88, 4)>,	/* pcie1 perst */
+				 <K3_PADCONF(90, 4)>;	/* pcie1 clkreq */
+
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <1800>;
+		};
+	};
+
+	pcie2-1-cfg {
+		pcie2-0-pins {
+			pinmux = <K3_PADCONF(25, 4)>,	/* pcie2 perst */
+				 <K3_PADCONF(27, 4)>;	/* pcie2 clkreq */
+
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	pcie4_0_cfg: pcie4-0-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				 <K3_PADCONF(77, 5)>,	/* pcie4 wake */
+				 <K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <33>;		/* DS8 */
+			power-source = <1800>;
+		};
+	};
+
+	wlan_en_cfg: wl-en-cfg {
+		wlan-en-pins {
+			pinmux = <K3_PADCONF(35, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	bt_en_cfg: bt-en-cfg {
+		bt-en-pins {
+			pinmux = <K3_PADCONF(30, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	dp1_1_cfg: dp1-1-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(24, 6)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	dp0_1_cfg: dp0-1-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(23, 6)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pwm18_2_cfg: pwm18-2-cfg {
+		pwm18-0-pins {
+			pinmux = <K3_PADCONF(106, 6)>;	/* pwm18 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	sspa1_1_cfg: sspa1-1-cfg {
+		sspa1-0-pins {
+			power-source = <3300>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			bootinfo@0 {
+				reg = <0x00000000 0x00020000>;
+			};
+			fsbl@20000 {
+				reg = <0x00020000 0x00080000>;
+			};
+			env@a0000 {
+				reg = <0x000a0000 0x00010000>;
+			};
+			esos@b0000 {
+				reg = <0x000b0000 0x00100000>;
+			};
+			opensbi@1b0000 {
+				reg = <0x001b0000 0x00060000>;
+			};
+			uboot@210000 {
+				reg = <0x00210000 0x005f0000>;
+			};
+		};
+	};
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reset-on-resume;
+	status = "okay";
+	/* Ensure no defered probe cause USB disconnect */
+	function-supply = <&hub_vbus>;
+
+	hub@1 {
+		/* FE1.1S USB2.0 hub (4 ports) */
+		compatible = "usb1a40,101";
+		reg = <1>;
+		vdd-supply = <&hub_vbus>;
+	};
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	/* dual phy selection is auto detect from GPIO86(SS_SW_3V3) */
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb30_drd_dir_0_cfg>;
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	reset-on-resume;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	/* M2.B Socket, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	reset-on-resume;
+	phys = <&usb3_portb_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "disabled";
+};
+
+&usb3_portc {
+	/* Bluetooth, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	reset-on-resume;
+	phys = <&usb3_portc_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	/* TypeC (USB3/DP/Power) (with ANX7447 chip) */
+	status = "okay";
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy2 {
+	status = "okay";
+};
+
+&phy3 {
+	status = "okay";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+	num-lanes = <4>;
+	spacemit,bifurcation-gpios = <&gpio 2 25 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&pcie1_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_1_cfg>;
+	phys = <&phy1>;
+	phy-names = "phy1";
+	num-lanes = <2>;
+	spacemit,device-detect-gpios = <&gpio 2 25 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&pcie2_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_1_cfg>;
+	phys = <&phy2>, <&phy3>;
+	phy-names = "phy2", "phy3";
+	num-lanes = <2>;
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_0_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+	num-lanes = <1>;
+	status = "okay";
+};
+
+&espi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&espi0_2_cfg>;
+	status = "okay";
+	ec: ec@0 {
+		status = "okay";
+		interrupt-parent = <&gpio>;
+		interrupts = <1 11 IRQ_TYPE_EDGE_RISING>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		i2c_tunnel1: i2c-tunnel@1 {
+			compatible = "google,cros-ec-i2c-tunnel";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			google,remote-bus = <1>;
+		};
+
+		i2c_tunnel4: i2c-tunnel@4 {
+			compatible = "google,cros-ec-i2c-tunnel";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			google,remote-bus = <4>;
+		};
+
+		/* EC PWM provider for fan control */
+		cros_ec_pwm: pwm {
+			compatible = "google,cros-ec-pwm";
+			#pwm-cells = <1>;
+		};
+
+		keyboard {
+			compatible = "google,cros-ec-keyb";
+			keypad,num-rows = <8>;
+			keypad,num-columns = <16>;
+			linux,keymap = <
+					CROS_STD_TOP_ROW_KEYMAP
+					CROS_STD_MAIN_KEYMAP
+			>;
+		};
+	};
+};
+
+&ec_master {
+	status = "okay";
+};
+
+&eth0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac0_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac0_phy>;
+	snps,reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <73>;
+	spacemit,rx-phase = <61>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac0_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <45000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <60000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 2 3>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 4 5>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <45000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <55000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <78000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip5: thermal_cluster0-trip5 {
+				temperature = <82000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip6: thermal_cluster0-trip6 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip7: thermal_cluster0-trip7 {
+				temperature = <88000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&fan0 0 1>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&fan0 2 3>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&fan0 3 4>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&fan0 4 4>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&fan0 5 5>;
+			};
+
+			map5 {
+				trip = <&thermal_cluster0_trip5>;
+				cooling-device = <&fan0 6 6>;
+			};
+
+			map6 {
+				trip = <&thermal_cluster0_trip6>;
+				cooling-device = <&fan0 7 7>;
+			};
+
+			map7 {
+				trip = <&thermal_cluster0_trip7>;
+				cooling-device = <&fan0 8 8>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	adcdc2: adcdc2 {
+		regulator-min-microvolt = <540000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	adcdc1: adcdc1 {
+		regulator-min-microvolt = <540000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_3_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 2 21 GPIO_ACTIVE_LOW>;
+		suspend-vbus-off;
+		wakeup-source;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_1_cfg>;
+	status = "okay";
+
+	es8326: es8326@19{
+		compatible = "everest,es8326";
+		reg = <0x19>;
+		#sound-dai-cells = <0>;
+		interrupt-parent = <&gpio>;
+		interrupts = <3 31 IRQ_TYPE_EDGE_RISING>;
+		/* spk-ctl-gpio = <&gpio 127 0>; */
+		everest,mic1-src = [44];
+		everest,mic2-src = [66];
+		everest,jack-detect-inverted;
+		status = "okay";
+	};
+};
+
+&i2c8 {
+	status = "okay";
+	spacemit,sda-glitch-nofix;
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&aux_5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&pwm18 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm18_2_cfg>;
+	status = "okay";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "okay";
+};
+
+&edp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_1_cfg>;
+	backlight = <&backlight>;
+	gpios-power = <101>;
+	gpios-enable = <118>;
+	status = "okay";
+};
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_1_cfg>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
+
+&i2s1 {
+	status = "okay";
+	pinctrl-0 = <&sspa1_1_cfg>;
+	pinctrl-names = "default";
+};
+
+&sound_card1 {
+	status = "okay";
+	simple-audio-card,name = "snd-es8326";
+	simple-audio-card,mclk-fs = <256>;
+	simple-audio-card,dai-link@0 {
+		format = "i2s";
+		frame-master = <&link1_cpu>;
+		bitclock-master = <&link1_cpu>;
+
+		link1_cpu: cpu {
+			sound-dai = <&i2s1>;
+		};
+
+		link1_codec: codec {
+			sound-dai = <&es8326>;
+		};
+	};
+};
+
+&cpu_0 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_1 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_2 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_3 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_4 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_5 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_6 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_7 {
+	clst-supply = <&adcdc1>;
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
new file mode 100644
index 000000000000..fe4aea8bfbf5
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
@@ -0,0 +1,3350 @@
+#include <dt-bindings/gpio/gpio.h>
+
+#define K3_PADCONF(pin, func) (((pin) << 16) | (func))
+
+&pinctrl {
+	uart0_0_cfg: uart0-0-cfg {
+		uart0-0-pins {
+			pinmux = <K3_PADCONF(149, 2)>,	/* uart0 tx */
+				 <K3_PADCONF(150, 2)>;	/* uart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart0_1_cfg: uart0-1-cfg {
+		uart0-0-pins {
+			pinmux = <K3_PADCONF(132, 2)>,	/* uart0 tx */
+				 <K3_PADCONF(133, 2)>;	/* uart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart0_2_cfg: uart0-2-cfg {
+		uart0-0-pins {
+			pinmux = <K3_PADCONF(145, 5)>,	/* uart0 tx */
+				 <K3_PADCONF(146, 5)>;	/* uart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart0_3_cfg: uart0-3-cfg {
+		uart0-0-pins {
+			pinmux = <K3_PADCONF(42, 2)>,	/* uart0 tx */
+				 <K3_PADCONF(43, 2)>;	/* uart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart0_4_cfg: uart0-4-cfg {
+		uart0-0-pins {
+			pinmux = <K3_PADCONF(93, 3)>,	/* uart0 tx */
+				 <K3_PADCONF(94, 3)>;	/* uart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart1_0_cfg: uart1-0-cfg {
+		uart1-0-pins {
+			pinmux = <K3_PADCONF(95, 2)>,	/* uart1 tx */
+				 <K3_PADCONF(96, 2)>,	/* uart1 rx */
+				 <K3_PADCONF(97, 2)>,	/* uart1 cts */
+				 <K3_PADCONF(98, 2)>;	/* uart1 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart1_1_cfg: uart1-1-cfg {
+		uart1-0-pins {
+			pinmux = <K3_PADCONF(118, 1)>,	/* uart1 rts */
+				 <K3_PADCONF(119, 1)>,	/* uart1 cts */
+				 <K3_PADCONF(120, 1)>,	/* uart1 rx */
+				 <K3_PADCONF(121, 1)>;	/* uart1 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart2_0_cfg: uart2-0-cfg {
+		uart2-0-pins {
+			pinmux = <K3_PADCONF(134, 2)>,	/* uart2 tx */
+				 <K3_PADCONF(135, 2)>,	/* uart2 rx */
+				 <K3_PADCONF(136, 2)>,	/* uart2 cts */
+				 <K3_PADCONF(137, 2)>;	/* uart2 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart2_1_cfg: uart2-1-cfg {
+		uart2-0-pins {
+			pinmux = <K3_PADCONF(97, 1)>,	/* uart2 tx */
+				 <K3_PADCONF(98, 1)>;	/* uart2 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart3_0_cfg: uart3-0-cfg {
+		uart3-0-pins {
+			pinmux = <K3_PADCONF(26, 2)>,	/* uart3 tx */
+				 <K3_PADCONF(27, 2)>,	/* uart3 rx */
+				 <K3_PADCONF(28, 2)>,	/* uart3 cts */
+				 <K3_PADCONF(29, 2)>;	/* uart3 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart3_1_cfg: uart3-1-cfg {
+		uart3-0-pins {
+			pinmux = <K3_PADCONF(53, 2)>,	/* uart3 cts */
+				 <K3_PADCONF(54, 2)>,	/* uart3 rts */
+				 <K3_PADCONF(55, 2)>,	/* uart3 rx */
+				 <K3_PADCONF(56, 2)>;	/* uart3 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart4_0_cfg: uart4-0-cfg {
+		uart4-0-pins {
+			pinmux = <K3_PADCONF(86, 3)>,	/* uart4 tx */
+				 <K3_PADCONF(87, 3)>,	/* uart4 rx */
+				 <K3_PADCONF(90, 3)>,	/* uart4 cts */
+				 <K3_PADCONF(91, 3)>;	/* uart4 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart4_1_cfg: uart4-1-cfg {
+		uart4-0-pins {
+			pinmux = <K3_PADCONF(99, 3)>,	/* uart4 tx */
+				 <K3_PADCONF(100, 3)>,	/* uart4 rx */
+				 <K3_PADCONF(101, 3)>,	/* uart4 cts */
+				 <K3_PADCONF(102, 3)>;	/* uart4 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart5_0_cfg: uart5-0-cfg {
+		uart5-0-pins {
+			pinmux = <K3_PADCONF(145, 4)>,	/* uart5 tx */
+				 <K3_PADCONF(146, 4)>;	/* uart5 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart5_1_cfg: uart5-1-cfg {
+		uart5-0-pins {
+			pinmux = <K3_PADCONF(21, 2)>,	/* uart5 tx */
+				 <K3_PADCONF(22, 2)>,	/* uart5 rx */
+				 <K3_PADCONF(23, 2)>,	/* uart5 cts */
+				 <K3_PADCONF(24, 2)>;	/* uart5 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart5_2_cfg: uart5-2-cfg {
+		uart5-0-pins {
+			pinmux = <K3_PADCONF(82, 4)>,	/* uart5 rx */
+				 <K3_PADCONF(83, 4)>;	/* uart5 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart6_0_cfg: uart6-0-cfg {
+		uart6-0-pins {
+			pinmux = <K3_PADCONF(48, 2)>,	/* uart6 tx */
+				 <K3_PADCONF(49, 2)>,	/* uart6 rx */
+				 <K3_PADCONF(50, 2)>,	/* uart6 cts */
+				 <K3_PADCONF(51, 2)>;	/* uart6 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart6_1_cfg: uart6-1-cfg {
+		uart6-0-pins {
+			pinmux = <K3_PADCONF(122, 3)>,	/* uart6 tx */
+				 <K3_PADCONF(123, 3)>;	/* uart6 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart7_0_cfg: uart7-0-cfg {
+		uart7-0-pins {
+			pinmux = <K3_PADCONF(11, 2)>,	/* uart7 rts */
+				 <K3_PADCONF(12, 2)>,	/* uart7 cts */
+				 <K3_PADCONF(13, 2)>,	/* uart7 tx */
+				 <K3_PADCONF(14, 2)>;	/* uart7 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart7_1_cfg: uart7-1-cfg {
+		uart7-0-pins {
+			pinmux = <K3_PADCONF(23, 5)>,	/* uart7 tx */
+				 <K3_PADCONF(24, 5)>,	/* uart7 rx */
+				 <K3_PADCONF(25, 5)>,	/* uart7 cts */
+				 <K3_PADCONF(26, 5)>;	/* uart7 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart8_0_cfg: uart8-0-cfg {
+		uart8-0-pins {
+			pinmux = <K3_PADCONF(11, 5)>,	/* uart8 rx */
+				 <K3_PADCONF(12, 5)>;	/* uart8 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart8_1_cfg: uart8-1-cfg {
+		uart8-0-pins {
+			pinmux = <K3_PADCONF(76, 3)>,	/* uart8 tx */
+				 <K3_PADCONF(77, 3)>,	/* uart8 rx */
+				 <K3_PADCONF(78, 3)>,	/* uart8 cts */
+				 <K3_PADCONF(79, 3)>;	/* uart8 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart9_0_cfg: uart9-0-cfg {
+		uart9-0-pins {
+			pinmux = <K3_PADCONF(147, 4)>,	/* uart9 tx */
+				 <K3_PADCONF(148, 4)>;	/* uart9 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart9_1_cfg: uart9-1-cfg {
+		uart9-0-pins {
+			pinmux = <K3_PADCONF(82, 3)>,	/* uart9 cts */
+				 <K3_PADCONF(83, 3)>,	/* uart9 rts */
+				 <K3_PADCONF(84, 3)>,	/* uart9 tx */
+				 <K3_PADCONF(85, 3)>;	/* uart9 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart10_0_cfg: uart10-0-cfg {
+		uart10-0-pins {
+			pinmux = <K3_PADCONF(31, 2)>,	/* uart10 tx */
+				 <K3_PADCONF(32, 2)>,	/* uart10 rx */
+				 <K3_PADCONF(33, 2)>,	/* uart10 cts */
+				 <K3_PADCONF(34, 2)>;	/* uart10 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	uart10_1_cfg: uart10-1-cfg {
+		uart10-0-pins {
+			pinmux = <K3_PADCONF(44, 2)>,	/* uart10 tx */
+				 <K3_PADCONF(45, 2)>,	/* uart10 rx */
+				 <K3_PADCONF(46, 2)>,	/* uart10 cts */
+				 <K3_PADCONF(47, 2)>;	/* uart10 rts */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart0_0_cfg: ruart0-0-cfg {
+		ruart0-0-pins {
+			pinmux = <K3_PADCONF(132, 4)>,	/* ruart0 tx */
+				 <K3_PADCONF(133, 4)>;	/* ruart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart0_1_cfg: ruart0-1-cfg {
+		ruart0-0-pins {
+			pinmux = <K3_PADCONF(145, 6)>,	/* ruart0 tx */
+				 <K3_PADCONF(146, 6)>;	/* ruart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart0_2_cfg: ruart0-2-cfg {
+		ruart0-0-pins {
+			pinmux = <K3_PADCONF(109, 3)>,	/* ruart0 tx */
+				 <K3_PADCONF(110, 3)>;	/* ruart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart0_3_cfg: ruart0-3-cfg {
+		ruart0-0-pins {
+			pinmux = <K3_PADCONF(122, 4)>,	/* ruart0 tx */
+				 <K3_PADCONF(123, 4)>;	/* ruart0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart1_0_cfg: ruart1_0-cfg {
+		ruart1-0-pins {
+			pinmux = <K3_PADCONF(138, 2)>,	/* ruart1 rts */
+				 <K3_PADCONF(139, 2)>,	/* ruart1 cts */
+				 <K3_PADCONF(140, 2)>,	/* ruart1 rx */
+				 <K3_PADCONF(141, 2)>;	/* ruart1 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart1_1_cfg: ruart1-1-cfg {
+		ruart1-0-pins {
+			pinmux = <K3_PADCONF(17, 5)>,	/* ruart1 tx */
+				 <K3_PADCONF(18, 5)>,	/* ruart1 rx */
+				 <K3_PADCONF(19, 5)>,	/* ruart1 cts */
+				 <K3_PADCONF(20, 5)>;	/* ruart1 rts */
+
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart2_0_cfg: ruart2-0-cfg {
+		ruart2-0-pins {
+			pinmux = <K3_PADCONF(57, 2)>,	/* ruart2 tx */
+				 <K3_PADCONF(58, 2)>;	/* ruart2 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart3_0_cfg: ruart3-0-cfg {
+		ruart3-0-pins {
+			pinmux = <K3_PADCONF(55, 4)>,	/* ruart3 rx */
+				 <K3_PADCONF(56, 4)>;	/* ruart3 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart3_1_cfg: ruart3-1-cfg {
+		ruart3-0-pins {
+			pinmux = <K3_PADCONF(88, 3)>,	/* ruart3 tx */
+				 <K3_PADCONF(89, 3)>;	/* ruart3 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart4_0_cfg: ruart4-0-cfg {
+		ruart4-0-pins {
+			pinmux = <K3_PADCONF(21, 5)>,	/* ruart4 tx */
+				 <K3_PADCONF(22, 5)>;	/* ruart4 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart4_1_cfg: ruart4-1-cfg {
+		ruart4-0-pins {
+			pinmux = <K3_PADCONF(80, 3)>,	/* ruart4 tx */
+				 <K3_PADCONF(81, 3)>;	/* ruart4 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ruart5_0_cfg: ruart5-0-cfg {
+		ruart5-0-pins {
+			pinmux = <K3_PADCONF(59, 2)>,	/* ruart5 rx */
+				 <K3_PADCONF(60, 2)>;	/* ruart5 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c0_0_cfg: i2c0-0-cfg {
+		i2c0-0-pins {
+			pinmux = <K3_PADCONF(0, 6)>,	/* i2c0 scl */
+				 <K3_PADCONF(1, 6)>;	/* i2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c0_1_cfg: i2c0-1-cfg {
+		i2c0-0-pins {
+			pinmux = <K3_PADCONF(42, 5)>,	/* i2c0 scl */
+				 <K3_PADCONF(43, 5)>;	/* i2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c0_2_cfg: i2c0-2-cfg {
+		i2c0-0-pins {
+			pinmux = <K3_PADCONF(48, 5)>,	/* i2c0 scl */
+				 <K3_PADCONF(49, 5)>;	/* i2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c0_3_cfg: i2c0-3-cfg {
+		i2c0-0-pins {
+			pinmux = <K3_PADCONF(76, 6)>,	/* i2c0 scl */
+				 <K3_PADCONF(77, 6)>;	/* i2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c0_4_cfg: i2c0-4-cfg {
+		i2c0-0-pins {
+			pinmux = <K3_PADCONF(115, 4)>,	/* i2c0 scl */
+				 <K3_PADCONF(116, 4)>;	/* i2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c1_0_cfg: i2c1-0-cfg {
+		i2c1-0-pins {
+			pinmux = <K3_PADCONF(2, 6)>,	/* i2c1 scl */
+				 <K3_PADCONF(3, 6)>;	/* i2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c1_1_cfg: i2c1-1-cfg {
+		i2c1-0-pins {
+			pinmux = <K3_PADCONF(54, 5)>,	/* i2c1 scl */
+				 <K3_PADCONF(55, 5)>;	/* i2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c1_2_cfg: i2c1-2-cfg {
+		i2c1-0-pins {
+			pinmux = <K3_PADCONF(78, 6)>,	/* i2c1 scl */
+				 <K3_PADCONF(79, 6)>;	/* i2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c1_3_cfg: i2c1-3-cfg {
+		i2c1-0-pins {
+			pinmux = <K3_PADCONF(102, 6)>,	/* i2c1 scl */
+				 <K3_PADCONF(103, 6)>;	/* i2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c2_0_cfg: i2c2-0-cfg {
+		i2c2-0-pins {
+			pinmux = <K3_PADCONF(5, 6)>,	/* i2c2 scl */
+				 <K3_PADCONF(6, 6)>;	/* i2c2 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c2_1_cfg: i2c2-1-cfg {
+		i2c2-0-pins {
+			pinmux = <K3_PADCONF(46, 5)>,	/* i2c2 scl */
+				 <K3_PADCONF(47, 5)>;	/* i2c2 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c2_2_cfg: i2c2-2-cfg {
+		i2c2-0-pins {
+			pinmux = <K3_PADCONF(80, 6)>,	/* i2c2 scl */
+				 <K3_PADCONF(81, 6)>;	/* i2c2 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c2_3_cfg: i2c2-3-cfg {
+		i2c2-0-pins {
+			pinmux = <K3_PADCONF(120, 2)>,	/* i2c2 scl */
+				 <K3_PADCONF(121, 2)>;	/* i2c2 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c3_0_cfg: i2c3-0-cfg {
+		i2c3-0-pins {
+			pinmux = <K3_PADCONF(142, 5)>,	/* i2c3 scl */
+				 <K3_PADCONF(143, 5)>;	/* i2c3 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c3_1_cfg: i2c3-1-cfg {
+		i2c3-0-pins {
+			pinmux = <K3_PADCONF(36, 4)>,	/* i2c3 scl */
+				 <K3_PADCONF(37, 4)>;	/* i2c3 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c3_2_cfg: i2c3-2-cfg {
+		i2c3-0-pins {
+			pinmux = <K3_PADCONF(82, 6)>,	/* i2c3 scl */
+				 <K3_PADCONF(83, 6)>;	/* i2c3 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c3_3_cfg: i2c3-3-cfg {
+		i2c3-0-pins {
+			pinmux = <K3_PADCONF(105, 4)>,	/* i2c3 scl */
+				 <K3_PADCONF(106, 4)>;	/* i2c3 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c4_0_cfg: i2c4-0-cfg {
+		i2c4-0-pins {
+			pinmux = <K3_PADCONF(136, 6)>,	/* i2c4 scl */
+				 <K3_PADCONF(137, 6)>;	/* i2c4 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c4_1_cfg: i2c4-1-cfg {
+		i2c4-0-pins {
+			pinmux = <K3_PADCONF(11, 6)>,	/* i2c4 scl */
+				 <K3_PADCONF(12, 6)>;	/* i2c4 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c4_2_cfg: i2c4-2-cfg {
+		i2c4-0-pins {
+			pinmux = <K3_PADCONF(50, 5)>,	/* i2c4 scl */
+				 <K3_PADCONF(51, 5)>;	/* i2c4 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c4_3_cfg: i2c4-3-cfg {
+		i2c4-0-pins {
+			pinmux = <K3_PADCONF(111, 4)>,	/* i2c4 scl */
+				 <K3_PADCONF(112, 4)>;	/* i2c4 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	touchpad_int_cfg: touchpad-int-cfg {
+		touchpad-int-pins {
+			pinmux = <K3_PADCONF(26, 0)>;	/* gpio26 */
+
+			bias-pull-up;
+			power-source = <3300>;
+		};
+	};
+
+	i2c5_0_cfg: i2c5-0-cfg {
+		i2c5-0-pins {
+			pinmux = <K3_PADCONF(25, 6)>,	/* i2c5 sda */
+				 <K3_PADCONF(26, 6)>;	/* i2c5 scl */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c5_1_cfg: i2c5-1-cfg {
+		i2c5-0-pins {
+			pinmux = <K3_PADCONF(63, 5)>,	/* i2c5 scl */
+				 <K3_PADCONF(64, 5)>;	/* i2c5 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c5_2_cfg: i2c5-2-cfg {
+		i2c5-0-pins {
+			pinmux = <K3_PADCONF(93, 5)>,	/* i2c5 scl */
+				 <K3_PADCONF(94, 5)>;	/* i2c5 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c5_3_cfg: i2c5-3-cfg {
+		i2c5-0-pins {
+			pinmux = <K3_PADCONF(126, 4)>,	/* i2c5 scl */
+				 <K3_PADCONF(127, 4)>;	/* i2c5 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c6_0_cfg: i2c6-0-cfg {
+		i2c6-0-pins {
+			pinmux = <K3_PADCONF(151, 2)>,	/* i2c6 scl */
+				 <K3_PADCONF(152, 2)>;	/* i2c6 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c6_1_cfg: i2c6-1-cfg {
+		i2c6-0-pins {
+			pinmux = <K3_PADCONF(7, 6)>,	/* i2c6 scl */
+				 <K3_PADCONF(8, 6)>;	/* i2c6 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <18>;		/* 3V_DS4 */
+		};
+	};
+
+	i2c6_2_cfg: i2c6-2-cfg {
+		i2c6-0-pins {
+			pinmux = <K3_PADCONF(14, 6)>,	/* i2c6 scl */
+				 <K3_PADCONF(15, 6)>;	/* i2c6 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <18>;		/* 3V_DS4 */
+		};
+	};
+
+	i2c6_3_cfg: i2c6-3-cfg {
+		i2c6-0-pins {
+			pinmux = <K3_PADCONF(61, 5)>,	/* i2c6 scl */
+				 <K3_PADCONF(62, 5)>;	/* i2c6 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c8_cfg: i2c8-cfg {
+		i2c8-pins {
+			pinmux = <K3_PADCONF(128, 0)>,	/* i2c8 scl */
+				 <K3_PADCONF(129, 0)>;	/* i2c8 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	i2c8_cfg_rcpu: i2c8-cfg-rcpu {
+		i2c8-pins-rcpu {
+			pinmux = <K3_PADCONF(128, 1)>,	/* ri2c2 scl */
+				 <K3_PADCONF(129, 1)>;	/* ri2c2 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c0_0_cfg: ri2c0-0-cfg {
+		ri2c0-0-pins {
+			pinmux = <K3_PADCONF(27, 6)>,	/* ri2c0 scl */
+				 <K3_PADCONF(28, 6)>;	/* ri2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c0_1_cfg: ri2c0-1-cfg {
+		ri2c0-0-pins {
+			pinmux = <K3_PADCONF(57, 5)>,	/* ri2c0 scl */
+				 <K3_PADCONF(58, 5)>;	/* ri2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c0_2_cfg: ri2c0-2-cfg {
+		ri2c0-0-pins {
+			pinmux = <K3_PADCONF(74, 6)>,	/* ri2c0 scl */
+				 <K3_PADCONF(75, 6)>;	/* ri2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c0_3_cfg: ri2c0-3-cfg {
+		ri2c0-0-pins {
+			pinmux = <K3_PADCONF(115, 6)>,	/* ri2c0 scl */
+				 <K3_PADCONF(116, 6)>;	/* ri2c0 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c1_0_cfg: ri2c1-0-cfg {
+		ri2c1-0-pins {
+			pinmux = <K3_PADCONF(33, 6)>,	/* ri2c1 scl */
+				 <K3_PADCONF(34, 6)>;	/* ri2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c1_1_cfg: ri2c1-1-cfg {
+		ri2c1-0-pins {
+			pinmux = <K3_PADCONF(39, 4)>,	/* ri2c1 scl */
+				 <K3_PADCONF(40, 4)>;	/* ri2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c1_2_cfg: ri2c1-2-cfg {
+		ri2c1-0-pins {
+			pinmux = <K3_PADCONF(59, 5)>,	/* ri2c1 scl */
+				 <K3_PADCONF(60, 5)>;	/* ri2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ri2c1_3_cfg: ri2c1-3-cfg {
+		ri2c1-0-pins {
+			pinmux = <K3_PADCONF(105, 3)>,	/* ri2c1 scl */
+				 <K3_PADCONF(106, 3)>;	/* ri2c1 sda */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm0_0_cfg: pwm0-0-cfg {
+		pwm0-0-pins {
+			pinmux = <K3_PADCONF(0, 3)>;	/* pwm0 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm0_1_cfg: pwm0-1-cfg {
+		pwm0-0-pins {
+			pinmux = <K3_PADCONF(42, 6)>;	/* pwm0 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm1_0_cfg: pwm1-0-cfg {
+		pwm1-0-pins {
+			pinmux = <K3_PADCONF(1, 3)>;	/* pwm1 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm1_1_cfg: pwm0-0-cfg {
+		pwm0-0-pins {
+			pinmux = <K3_PADCONF(43, 6)>;	/* pwm1 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm1_2_cfg: pwm1-2-cfg {
+		pwm1-0-pins {
+			pinmux = <K3_PADCONF(95, 6)>;	/* pwm1 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm2_0_cfg: pwm2-0-cfg {
+		pwm2-0-pins {
+			pinmux = <K3_PADCONF(2, 3)>;	/* pwm2 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm2_1_cfg: pwm2-1-cfg {
+		pwm2-0-pins {
+			pinmux = <K3_PADCONF(44, 6)>;	/* pwm2 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm2_2_cfg: pwm2-2-cfg {
+		pwm2-0-pins {
+			pinmux = <K3_PADCONF(96, 6)>;	/* pwm2 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm2_3_cfg: pwm2-3-cfg {
+		pwm2-0-pins {
+			pinmux = <K3_PADCONF(134, 4)>;	/* pwm2 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm3_0_cfg: pwm3-0-cfg {
+		pwm3-0-pins {
+			pinmux = <K3_PADCONF(3, 3)>;	/* pwm3 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm3_1_cfg: pwm3-1-cfg {
+		pwm3-0-pins {
+			pinmux = <K3_PADCONF(45, 6)>;	/* pwm3 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm3_2_cfg: pwm3-2-cfg {
+		pwm3-0-pins {
+			pinmux = <K3_PADCONF(97, 6)>;	/* pwm3 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm3_3_cfg: pwm3-3-cfg {
+		pwm3-0-pins {
+			pinmux = <K3_PADCONF(135, 4)>;	/* pwm3 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm4_0_cfg: pwm4-0-cfg {
+		pwm4-0-pins {
+			pinmux = <K3_PADCONF(4, 3)>;	/* pwm4 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm4_1_cfg: pwm4-1-cfg {
+		pwm4-0-pins {
+			pinmux = <K3_PADCONF(46, 6)>;	/* pwm4 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm4_2_cfg: pwm4-2-cfg {
+		pwm4-0-pins {
+			pinmux = <K3_PADCONF(136, 4)>;	/* pwm4 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm5_0_cfg: pwm5-0-cfg {
+		pwm5-0-pins {
+			pinmux = <K3_PADCONF(5, 3)>;	/* pwm5 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm5_1_cfg: pwm5-1-cfg {
+		pwm5-0-pins {
+			pinmux = <K3_PADCONF(47, 6)>;	/* pwm5 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm5_2_cfg: pwm5-2-cfg {
+		pwm5-0-pins {
+			pinmux = <K3_PADCONF(137, 4)>;	/* pwm5 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm6_0_cfg: pwm6-0-cfg {
+		pwm6-0-pins {
+			pinmux = <K3_PADCONF(6, 3)>;	/* pwm6 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm6_1_cfg: pwm6-1-cfg {
+		pwm6-0-pins {
+			pinmux = <K3_PADCONF(145, 3)>;	/* pwm6 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm6_2_cfg: pwm6-2-cfg {
+		pwm6-0-pins {
+			pinmux = <K3_PADCONF(48, 6)>;	/* pwm6 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm7_0_cfg: pwm7-0-cfg {
+		pwm7-0-pins {
+			pinmux = <K3_PADCONF(7, 3)>;	/* pwm7 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm7_1_cfg: pwm7-1-cfg {
+		pwm7-0-pins {
+			pinmux = <K3_PADCONF(146, 3)>;	/* pwm7 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm7_2_cfg: pwm7-2-cfg {
+		pwm7-0-pins {
+			pinmux = <K3_PADCONF(49, 6)>;	/* pwm7 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm8_0_cfg: pwm8-0-cfg {
+		pwm8-0-pins {
+			pinmux = <K3_PADCONF(8, 3)>;	/* pwm8 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm8_1_cfg: pwm8-1-cfg {
+		pwm8-0-pins {
+			pinmux = <K3_PADCONF(147, 3)>;	/* pwm8 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm8_2_cfg: pwm8-2-cfg {
+		pwm8-0-pins {
+			pinmux = <K3_PADCONF(50, 6)>;	/* pwm8 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm9_0_cfg: pwm9-0-cfg {
+		pwm9-0-pins {
+			pinmux = <K3_PADCONF(9, 3)>;	/* pwm9 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm9_1_cfg: pwm9-1-cfg {
+		pwm9-0-pins {
+			pinmux = <K3_PADCONF(148, 3)>;	/* pwm9 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm9_2_cfg: pwm9-2-cfg {
+		pwm9-0-pins {
+			pinmux = <K3_PADCONF(51, 6)>;	/* pwm9 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm10_0_cfg: pwm10-0-cfg {
+		pwm10-0-pins {
+			pinmux = <K3_PADCONF(10, 3)>;	/* pwm10 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm10_1_cfg: pwm10-1-cfg {
+		pwm10-0-pins {
+			pinmux = <K3_PADCONF(52, 6)>;	/* pwm10 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm10_2_cfg: pwm10-2-cfg {
+		pwm10-0-pins {
+			pinmux = <K3_PADCONF(141, 4)>;	/* pwm10 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm11_0_cfg: pwm11-0-cfg {
+		pwm11-0-pins {
+			pinmux = <K3_PADCONF(53, 6)>;	/* pwm11 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm11_1_cfg: pwm11-1-cfg {
+		pwm11-0-pins {
+			pinmux = <K3_PADCONF(140, 4)>;	/* pwm11 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm12_0_cfg: pwm12-0-cfg {
+		pwm12-0-pins {
+			pinmux = <K3_PADCONF(54, 6)>;	/* pwm12 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm12_1_cfg: pwm12-1-cfg {
+		pwm12-0-pins {
+			pinmux = <K3_PADCONF(139, 4)>;	/* pwm12 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm13_0_cfg: pwm13-0-cfg {
+		pwm13-0-pins {
+			pinmux = <K3_PADCONF(55, 6)>;	/* pwm13 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm13_1_cfg: pwm13-1-cfg {
+		pwm13-0-pins {
+			pinmux = <K3_PADCONF(138, 4)>;	/* pwm13 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm13_2_cfg: pwm13-2-cfg {
+		pwm13-0-pins {
+			pinmux = <K3_PADCONF(13, 3)>;	/* pwm13 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm14_0_cfg: pwm14-0-cfg {
+		pwm14-0-pins {
+			pinmux = <K3_PADCONF(56, 6)>;	/* pwm14 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm14_1_cfg: pwm14-1-cfg {
+		pwm14-0-pins {
+			pinmux = <K3_PADCONF(144, 4)>;	/* pwm14 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm14_2_cfg: pwm14-2-cfg {
+		pwm14-0-pins {
+			pinmux = <K3_PADCONF(14, 3)>;	/* pwm14 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm15_0_cfg: pwm15-0-cfg {
+		pwm15-0-pins {
+			pinmux = <K3_PADCONF(57, 6)>;	/* pwm15 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm15_1_cfg: pwm15-1-cfg {
+		pwm15-0-pins {
+			pinmux = <K3_PADCONF(142, 4)>;	/* pwm15 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm15_2_cfg: pwm15-2-cfg {
+		pwm15-0-pins {
+			pinmux = <K3_PADCONF(21, 3)>;	/* pwm15 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm16_0_cfg: pwm16-0-cfg {
+		pwm16-0-pins {
+			pinmux = <K3_PADCONF(58, 6)>;	/* pwm16 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm16_1_cfg: pwm16-1-cfg {
+		pwm16-0-pins {
+			pinmux = <K3_PADCONF(143, 4)>;	/* pwm16 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm16_2_cfg: pwm16-2-cfg {
+		pwm16-0-pins {
+			pinmux = <K3_PADCONF(22, 3)>;	/* pwm16 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm17_0_cfg: pwm17-0-cfg {
+		pwm17-0-pins {
+			pinmux = <K3_PADCONF(23, 3)>;	/* pwm17 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm17_1_cfg: pwm17-1-cfg {
+		pwm17-0-pins {
+			pinmux = <K3_PADCONF(59, 6)>;	/* pwm17 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm17_2_cfg: pwm17-2-cfg {
+		pwm17-0-pins {
+			pinmux = <K3_PADCONF(105, 6)>;	/* pwm17 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm18_0_cfg: pwm18-0-cfg {
+		pwm18-0-pins {
+			pinmux = <K3_PADCONF(24, 3)>;	/* pwm18 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm18_1_cfg: pwm18-1-cfg {
+		pwm18-0-pins {
+			pinmux = <K3_PADCONF(60, 6)>;	/* pwm18 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm18_2_cfg: pwm18-2-cfg {
+		pwm18-0-pins {
+			pinmux = <K3_PADCONF(106, 6)>;	/* pwm18 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm19_0_cfg: pwm19-0-cfg {
+		pwm19-0-pins {
+			pinmux = <K3_PADCONF(25, 3)>;	/* pwm19 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm19_1_cfg: pwm19-1-cfg {
+		pwm19-0-pins {
+			pinmux = <K3_PADCONF(61, 6)>;	/* pwm19 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwm19_2_cfg: pwm19-2-cfg {
+		pwm19-0-pins {
+			pinmux = <K3_PADCONF(107, 6)>;	/* pwm19 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm0_0_cfg: rpwm0-0-cfg {
+		rpwm0-0-pins {
+			pinmux = <K3_PADCONF(15, 3)>;	/* rpwm0 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm0_1_cfg: rpwm0-1-cfg {
+		rpwm0-0-pins {
+			pinmux = <K3_PADCONF(27, 3)>;	/* rpwm0 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm0_2_cfg: rpwm0-2-cfg {
+		rpwm0-0-pins {
+			pinmux = <K3_PADCONF(64, 6)>;	/* rpwm0 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm1_0_cfg: rpwm1-0-cfg {
+		rpwm1-0-pins {
+			pinmux = <K3_PADCONF(16, 3)>;	/* rpwm1 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm1_1_cfg: rpwm1-1-cfg {
+		rpwm1-0-pins {
+			pinmux = <K3_PADCONF(28, 3)>;	/* rpwm1 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm1_2_cfg: rpwm1-2-cfg {
+		rpwm1-0-pins {
+			pinmux = <K3_PADCONF(65, 6)>;	/* rpwm1 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm2_0_cfg: rpwm2-0-cfg {
+		rpwm2-0-pins {
+			pinmux = <K3_PADCONF(17, 3)>;	/* rpwm2 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm2_1_cfg: rpwm2-1-cfg {
+		rpwm2-0-pins {
+			pinmux = <K3_PADCONF(29, 3)>;	/* rpwm2 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm2_2_cfg: rpwm2-2-cfg {
+		rpwm2-0-pins {
+			pinmux = <K3_PADCONF(66, 6)>;	/* rpwm2 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm3_0_cfg: rpwm3-0-cfg {
+		rpwm3-0-pins {
+			pinmux = <K3_PADCONF(18, 3)>;	/* rpwm3 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm3_1_cfg: rpwm3-1-cfg {
+		rpwm3-0-pins {
+			pinmux = <K3_PADCONF(30, 3)>;	/* rpwm3 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm3_2_cfg: rpwm3-2-cfg {
+		rpwm3-0-pins {
+			pinmux = <K3_PADCONF(67, 6)>;	/* rpwm3 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm4_0_cfg: rpwm4-0-cfg {
+		rpwm4-0-pins {
+			pinmux = <K3_PADCONF(19, 3)>;	/* rpwm4 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm4_1_cfg: rpwm4-1-cfg {
+		rpwm4-0-pins {
+			pinmux = <K3_PADCONF(31, 3)>;	/* rpwm4 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm4_2_cfg: rpwm4-2-cfg {
+		rpwm4-0-pins {
+			pinmux = <K3_PADCONF(93, 6)>;	/* rpwm4 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm5_0_cfg: rpwm5-0-cfg {
+		rpwm5-0-pins {
+			pinmux = <K3_PADCONF(20, 3)>;	/* rpwm5 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm5_1_cfg: rpwm5-1-cfg {
+		rpwm5-0-pins {
+			pinmux = <K3_PADCONF(32, 3)>;	/* rpwm5 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm5_2_cfg: rpwm5-2-cfg {
+		rpwm5-0-pins {
+			pinmux = <K3_PADCONF(92, 5)>;	/* rpwm5 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm6_0_cfg: rpwm6-0-cfg {
+		rpwm6-0-pins {
+			pinmux = <K3_PADCONF(33, 3)>;	/* rpwm6 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm6_1_cfg: rpwm6-1-cfg {
+		rpwm6-0-pins {
+			pinmux = <K3_PADCONF(94, 6)>;	/* rpwm6 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm6_2_cfg: rpwm6-2-cfg {
+		rpwm6-0-pins {
+			pinmux = <K3_PADCONF(109, 6)>;	/* rpwm6 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm7_0_cfg: rpwm7-0-cfg {
+		rpwm7-0-pins {
+			pinmux = <K3_PADCONF(34, 3)>;	/* rpwm7 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm7_1_cfg: rpwm7-1-cfg {
+		rpwm7-0-pins {
+			pinmux = <K3_PADCONF(110, 6)>;	/* rpwm7 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm8_0_cfg: rpwm8-0-cfg {
+		rpwm8-0-pins {
+			pinmux = <K3_PADCONF(35, 3)>;	/* rpwm8 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm8_1_cfg: rpwm8-1-cfg {
+		rpwm8-0-pins {
+			pinmux = <K3_PADCONF(111, 6)>;	/* rpwm8 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm9_0_cfg: rpwm9-0-cfg {
+		rpwm9-0-pins {
+			pinmux = <K3_PADCONF(36, 3)>;	/* rpwm9 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rpwm9_1_cfg: rpwm9-1-cfg {
+		rpwm9-0-pins {
+			pinmux = <K3_PADCONF(112, 6)>;	/* rpwm9 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ir0_0_cfg: ir0-0-cfg {
+		ir0-0-pins {
+			pinmux = <K3_PADCONF(41, 5)>;	/* ir0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ir0_1_cfg: ir0-1-cfg {
+		ir0-0-pins {
+			pinmux = <K3_PADCONF(108, 4)>;	/* ir0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ir1_0_cfg: ir1-0-cfg {
+		ir1-0-pins {
+			pinmux = <K3_PADCONF(0, 4)>;	/* ir1 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ir1_1_cfg: ir1-1-cfg {
+		ir1-0-pins {
+			pinmux = <K3_PADCONF(70, 4)>;	/* ir1 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rir0_0_cfg: rir0-0-cfg {
+		rir0-0-pins {
+			pinmux = <K3_PADCONF(40, 5)>;	/* rir0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rir0_1_cfg: rir0-1-cfg {
+		rir0-0-pins {
+			pinmux = <K3_PADCONF(71, 4)>;	/* rir0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rir1_0_cfg: rir1-0-cfg {
+		rir1-0-pins {
+			pinmux = <K3_PADCONF(1, 4)>;	/* rir1 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rir1_1_cfg: rir1-1-cfg {
+		rir1-0-pins {
+			pinmux = <K3_PADCONF(73, 4)>;	/* rir1 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can0_0_cfg: can0-0-cfg {
+		can0-0-pins {
+			pinmux = <K3_PADCONF(11, 3)>,	/* can0 tx */
+				 <K3_PADCONF(12, 3)>;	/* can0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can0_1_cfg: can0-1-cfg {
+		can0-0-pins {
+			pinmux = <K3_PADCONF(44, 3)>,	/* can0 tx */
+				 <K3_PADCONF(45, 3)>;	/* can0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can0_2_cfg: can0-2-cfg {
+		can0-0-pins {
+			pinmux = <K3_PADCONF(76, 4)>,	/* can0 tx */
+				 <K3_PADCONF(77, 4)>;	/* can0 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can1_0_cfg: can1-0-cfg {
+		can1-0-pins {
+			pinmux = <K3_PADCONF(48, 3)>,	/* can1 rx */
+				 <K3_PADCONF(49, 3)>;	/* can1 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can1_1_cfg: can1-1-cfg {
+		can1-0-pins {
+			pinmux = <K3_PADCONF(88, 6)>,	/* can1 rx */
+				 <K3_PADCONF(89, 6)>;	/* can1 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can1_2_cfg: can1-2-cfg {
+		can1-0-pins {
+			pinmux = <K3_PADCONF(109, 4)>,	/* can1 tx */
+				 <K3_PADCONF(110, 4)>;	/* can1 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can2_0_cfg: can2-0-cfg {
+		can2-0-pins {
+			pinmux = <K3_PADCONF(50, 3)>,	/* can2 tx */
+				 <K3_PADCONF(51, 3)>;	/* can2 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can2_1_cfg: can2-1-cfg {
+		can2-0-pins {
+			pinmux = <K3_PADCONF(86, 4)>,	/* can2 tx */
+				 <K3_PADCONF(87, 4)>;	/* can2 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can3_0_cfg: can3-0-cfg {
+		can3-0-pins {
+			pinmux = <K3_PADCONF(80, 4)>,	/* can3 rx */
+				 <K3_PADCONF(81, 4)>;	/* can3 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can3_1_cfg: can3-1-cfg {
+		can3-0-pins {
+			pinmux = <K3_PADCONF(103, 4)>,	/* can3 tx */
+				 <K3_PADCONF(104, 4)>;	/* can3 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can4_0_cfg: can4-0-cfg {
+		can4-0-pins {
+			pinmux = <K3_PADCONF(40, 6)>,	/* can4 tx */
+				 <K3_PADCONF(41, 6)>;	/* can4 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can4_1_cfg: can4-1-cfg {
+		can4-0-pins {
+			pinmux = <K3_PADCONF(101, 4)>,	/* can4 rx */
+				 <K3_PADCONF(102, 4)>;	/* can4 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	can4_2_cfg: can4-2-cfg {
+		can4-0-pins {
+			pinmux = <K3_PADCONF(120, 4)>,	/* can4 tx */
+				 <K3_PADCONF(121, 4)>;	/* can4 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rcan0_0_cfg: rcan0-0-cfg {
+		rcan0-0-pins {
+			pinmux = <K3_PADCONF(57, 3)>,	/* rcan0 rx */
+				 <K3_PADCONF(58, 3)>;	/* rcan0 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rcan0_1_cfg: rcan0-1-cfg {
+		rcan0-0-pins {
+			pinmux = <K3_PADCONF(90, 6)>,	/* rcan0 rx */
+				 <K3_PADCONF(91, 6)>;	/* rcan0 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rcan1_0_cfg: rcan1-0-cfg {
+		rcan1-0-pins {
+			pinmux = <K3_PADCONF(142, 2)>,	/* rcan1 rx */
+				 <K3_PADCONF(144, 2)>;	/* rcan1 tx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rcan2_0_cfg: rcan2-0-cfg {
+		rcan2-0-pins {
+			pinmux = <K3_PADCONF(99, 4)>,	/* rcan2 tx */
+				 <K3_PADCONF(100, 4)>;	/* rcan2 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rcan3_0_cfg: rcan3-0-cfg {
+		rcan3-0-pins {
+			pinmux = <K3_PADCONF(120, 3)>,	/* rcan3 tx */
+				 <K3_PADCONF(121, 3)>;	/* rcan3 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rcan4_0_cfg: rcan4-0-cfg {
+		rcan4-0-pins {
+			pinmux = <K3_PADCONF(107, 3)>,	/* rcan4 tx */
+				 <K3_PADCONF(108, 3)>;	/* rcan4 rx */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa0_0_cfg: sspa0-0-cfg {
+		sspa0-0-pins {
+			pinmux = <K3_PADCONF(81, 2)>,	/* sspa0 clk */
+				 <K3_PADCONF(82, 2)>,	/* sspa0 frm */
+				 <K3_PADCONF(83, 2)>,	/* sspa0 tx */
+				 <K3_PADCONF(84, 2)>,	/* sspa0 rx */
+				 <K3_PADCONF(85, 2)>;	/* sspa0 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa0_1_cfg: sspa0-1-cfg {
+		sspa0-0-pins {
+			pinmux = <K3_PADCONF(111, 2)>,	/* sspa0 clk */
+				 <K3_PADCONF(112, 2)>,	/* sspa0 frm */
+				 <K3_PADCONF(113, 2)>,	/* sspa0 tx */
+				 <K3_PADCONF(114, 2)>,	/* sspa0 rx */
+				 <K3_PADCONF(115, 2)>;	/* sspa0 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa1_0_cfg: sspa1-0-cfg {
+		sspa1-0-pins {
+			pinmux = <K3_PADCONF(15, 2)>,	/* sspa1 clk */
+				 <K3_PADCONF(16, 2)>,	/* sspa1 frm */
+				 <K3_PADCONF(17, 2)>,	/* sspa1 tx */
+				 <K3_PADCONF(18, 2)>,	/* sspa1 rx */
+				 <K3_PADCONF(19, 2)>;	/* sspa1 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa1_1_cfg: sspa1-1-cfg {
+		sspa1-0-pins {
+			pinmux = <K3_PADCONF(122, 2)>,	/* sspa1 clk */
+				 <K3_PADCONF(123, 2)>,	/* sspa1 frm */
+				 <K3_PADCONF(124, 2)>,	/* sspa1 tx */
+				 <K3_PADCONF(125, 2)>,	/* sspa1 rx */
+				 <K3_PADCONF(126, 2)>;	/* sspa1 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa2_0_cfg: sspa2-0-cfg {
+		sspa2-0-pins {
+			pinmux = <K3_PADCONF(76, 2)>,	/* sspa2 clk */
+				 <K3_PADCONF(77, 2)>,	/* sspa2 frm */
+				 <K3_PADCONF(78, 2)>,	/* sspa2 tx */
+				 <K3_PADCONF(79, 2)>,	/* sspa2 rx */
+				 <K3_PADCONF(80, 2)>;	/* sspa2 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa3_0_cfg: sspa3-0-cfg {
+		sspa3-0-pins {
+			pinmux = <K3_PADCONF(99, 2)>,	/* sspa3 clk */
+				 <K3_PADCONF(100, 2)>,	/* sspa3 frm */
+				 <K3_PADCONF(101, 2)>,	/* sspa3 tx */
+				 <K3_PADCONF(102, 2)>,	/* sspa3 rx */
+				 <K3_PADCONF(103, 2)>;	/* sspa3 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa4_0_cfg: sspa4-0-cfg {
+		sspa4-0-pins {
+			pinmux = <K3_PADCONF(69, 2)>,	/* sspa4 clk */
+				 <K3_PADCONF(70, 2)>,	/* sspa4 frm */
+				 <K3_PADCONF(71, 2)>,	/* sspa4 tx */
+				 <K3_PADCONF(72, 2)>,	/* sspa4 rx */
+				 <K3_PADCONF(73, 2)>;	/* sspa4 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	sspa5_0_cfg: sspa5-0-cfg {
+		sspa5-0-pins {
+			pinmux = <K3_PADCONF(0, 2)>,	/* sspa5 clk */
+				 <K3_PADCONF(1, 2)>,	/* sspa5 frm */
+				 <K3_PADCONF(2, 2)>,	/* sspa5 tx */
+				 <K3_PADCONF(3, 2)>,	/* sspa5 rx */
+				 <K3_PADCONF(4, 2)>;	/* sspa5 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rsspa0_0_cfg: rsspa0-0-cfg {
+		rsspa0-0-pins {
+			pinmux = <K3_PADCONF(6, 2)>,	/* rsspa0 clk */
+				 <K3_PADCONF(7, 2)>,	/* rsspa0 frm */
+				 <K3_PADCONF(8, 2)>,	/* rsspa0 tx */
+				 <K3_PADCONF(9, 2)>,	/* rsspa0 rx */
+				 <K3_PADCONF(10, 2)>;	/* rsspa0 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rsspa0_1_cfg: rsspa0-1-cfg {
+		rsspa0-0-pins {
+			pinmux = <K3_PADCONF(76, 1)>,	/* rsspa0 clk */
+				 <K3_PADCONF(77, 1)>,	/* rsspa0 frm */
+				 <K3_PADCONF(78, 1)>,	/* rsspa0 tx */
+				 <K3_PADCONF(79, 1)>,	/* rsspa0 rx */
+				 <K3_PADCONF(80, 1)>;	/* rsspa0 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rsspa1_0_cfg: rsspa1-0-cfg {
+		rsspa1-0-pins {
+			pinmux = <K3_PADCONF(36, 2)>,	/* rsspa1 clk */
+				 <K3_PADCONF(37, 2)>,	/* rsspa1 frm */
+				 <K3_PADCONF(38, 2)>,	/* rsspa1 tx */
+				 <K3_PADCONF(39, 2)>,	/* rsspa1 rx */
+				 <K3_PADCONF(40, 2)>;	/* rsspa1 sysclk */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie0_0_cfg: pcie0-0-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(42, 4)>,	/* pcie0 perst */
+				 <K3_PADCONF(43, 4)>,	/* pcie0 wake */
+				 <K3_PADCONF(44, 4)>;	/* pcie0 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie0_1_cfg: pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,	/* pcie0 perst */
+				 <K3_PADCONF(80, 5)>,	/* pcie0 wake */
+				 <K3_PADCONF(81, 5)>;	/* pcie0 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie0_2_cfg: pcie0-2-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(108, 5)>,	/* pcie0 perst */
+				 <K3_PADCONF(109, 5)>,	/* pcie0 wake */
+				 <K3_PADCONF(110, 5)>;	/* pcie0 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie1_0_cfg: pcie1-0-cfg {
+		pcie1-0-pins {
+			pinmux = <K3_PADCONF(21, 4)>,	/* pcie1 perst */
+				 <K3_PADCONF(22, 4)>,	/* pcie1 wake */
+				 <K3_PADCONF(23, 4)>;	/* pcie1 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie1_1_cfg: pcie1-1-cfg {
+		pcie1-0-pins {
+			pinmux = <K3_PADCONF(88, 4)>,	/* pcie1 perst */
+				 <K3_PADCONF(89, 4)>,	/* pcie1 wake */
+				 <K3_PADCONF(90, 4)>;	/* pcie1 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie1_2_cfg: pcie1-2-cfg {
+		pcie1-0-pins {
+			pinmux = <K3_PADCONF(113, 5)>,	/* pcie1 perst */
+				 <K3_PADCONF(114, 5)>,	/* pcie1 wake */
+				 <K3_PADCONF(115, 5)>;	/* pcie1 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie2_0_cfg: pcie2-0-cfg {
+		pcie2-0-pins {
+			pinmux = <K3_PADCONF(12, 4)>,	/* pcie2 perst */
+				 <K3_PADCONF(13, 4)>,	/* pcie2 wake */
+				 <K3_PADCONF(14, 4)>;	/* pcie2 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie2_1_cfg: pcie2-1-cfg {
+		pcie2-0-pins {
+			pinmux = <K3_PADCONF(25, 4)>,	/* pcie2 perst */
+				 <K3_PADCONF(26, 4)>,	/* pcie2 wake */
+				 <K3_PADCONF(27, 4)>;	/* pcie2 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie2_2_cfg: pcie2-2-cfg {
+		pcie2-0-pins {
+			pinmux = <K3_PADCONF(58, 4)>,	/* pcie2 perst */
+				 <K3_PADCONF(59, 4)>,	/* pcie2 wake */
+				 <K3_PADCONF(60, 4)>;	/* pcie2 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie3_0_cfg: pcie3-0-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(3, 4)>,	/* pcie3 perst */
+				 <K3_PADCONF(4, 4)>,	/* pcie3 wake */
+				 <K3_PADCONF(5, 4)>;	/* pcie3 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie3_1_cfg: pcie3-1-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(105, 5)>,	/* pcie3 perst */
+				 <K3_PADCONF(106, 5)>,	/* pcie3 wake */
+				 <K3_PADCONF(107, 5)>;	/* pcie3 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie3_2_cfg: pcie3-2-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(124, 3)>,	/* pcie3 perst */
+				 <K3_PADCONF(125, 3)>,	/* pcie3 wake */
+				 <K3_PADCONF(126, 3)>;	/* pcie3 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie4_0_cfg: pcie4-0-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(31, 4)>,	/* pcie4 perst */
+				 <K3_PADCONF(32, 4)>,	/* pcie4 wake */
+				 <K3_PADCONF(33, 4)>;	/* pcie4 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pcie4_1_cfg: pcie4-1-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				 <K3_PADCONF(77, 5)>,	/* pcie4 wake */
+				 <K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	gmac0_cfg: gmac0-cfg {
+		/* Base pins: - Used by RMII directly */
+		gmac0_base_pins: gmac0-0-pins {
+			pinmux = <K3_PADCONF(0, 1)>,	/* gmac0: MII=rxdv | RMII=crs_dv | RGMII=rx_ctl */
+				 <K3_PADCONF(1, 1)>,	/* gmac0 rx d0 */
+				 <K3_PADCONF(2, 1)>,	/* gmac0 rx d1 */
+				 <K3_PADCONF(3, 1)>,	/* gmac0: MII=rxc | RMII=ref_clk | RGMII=rxc */
+				 <K3_PADCONF(6, 1)>,	/* gmac0 tx d0 */
+				 <K3_PADCONF(7, 1)>,	/* gmac0 tx d1 */
+				 <K3_PADCONF(11, 1)>,	/* gmac0: MII=tx_en | RMII=tx_en | RGMII=tx_ctl */
+				 <K3_PADCONF(12, 1)>,	/* gmac0 mdc */
+				 <K3_PADCONF(13, 1)>;	/* gmac0 mdio */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* RGMII extra pins: add on top of base pins */
+		gmac0_rgmii_add_pins: gmac0-1-pins {
+			pinmux = <K3_PADCONF(4, 1)>,	/* gmac0 rx d2 */
+				 <K3_PADCONF(5, 1)>,	/* gmac0 rx d3 */
+				 <K3_PADCONF(8, 1)>,	/* gmac0 tx clk */
+				 <K3_PADCONF(9, 1)>,	/* gmac0 tx d2 */
+				 <K3_PADCONF(10, 1)>;	/* gmac0 tx d3 */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* MII extra pins: add on top of (base + rgmii extra pins) */
+		gmac0_mii_add_pins: gmac0-2-pins {
+			pinmux = <K3_PADCONF(15, 1)>,	/* gmac0 rxer */
+				 <K3_PADCONF(16, 1)>,	/* gmac0 txer */
+				 <K3_PADCONF(17, 1)>,	/* gmac0 crs */
+				 <K3_PADCONF(18, 1)>;	/* gmac0 col */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional int pins */
+		gmac0_int_pins: gmac0-3-pins {
+			pinmux = <K3_PADCONF(14, 1)>;	/* gmac0 int (from phy) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional pps pins */
+		gmac0_pps_pins: gmac0-4-pins {
+			pinmux = <K3_PADCONF(19, 1)>;	/* gmac0 pps (tsn/1588) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional reference clock output */
+		gmac0_refclk_pins: gmac0-5-pins {
+			pinmux = <K3_PADCONF(20, 1)>;	/* gmac0 clk ref */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+	};
+
+	gmac1_cfg: gmac1-cfg {
+		/* Base pins: - Used by RMII directly */
+		gmac1_base_pins: gmac1-0-pins {
+			pinmux = <K3_PADCONF(21, 1)>,	/* gmac1: MII=rxdv | RMII=crs_dv | RGMII=rx_ctl */
+				 <K3_PADCONF(22, 1)>,	/* gmac1 rx d0 */
+				 <K3_PADCONF(23, 1)>,	/* gmac1 rx d1 */
+				 <K3_PADCONF(24, 1)>,	/* gmac1: MII=rxc | RMII=ref_clk | RGMII=rxc */
+				 <K3_PADCONF(27, 1)>,	/* gmac1 tx d0 */
+				 <K3_PADCONF(28, 1)>,	/* gmac1 tx d1 */
+				 <K3_PADCONF(32, 1)>,	/* gmac1: MII=tx_en | RMII=tx_en | RGMII=tx_ctl */
+				 <K3_PADCONF(33, 1)>,	/* gmac1 mdc */
+				 <K3_PADCONF(34, 1)>;	/* gmac1 mdio */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* RGMII extra pins: add on top of base pins */
+		gmac1_rgmii_add_pins: gmac1-1-pins {
+			pinmux = <K3_PADCONF(25, 1)>,	/* gmac1 rx d2 */
+				 <K3_PADCONF(26, 1)>,	/* gmac1 rx d3 */
+				 <K3_PADCONF(29, 1)>,	/* gmac1 tx clk */
+				 <K3_PADCONF(30, 1)>,	/* gmac1 tx d2 */
+				 <K3_PADCONF(31, 1)>;	/* gmac1 tx d3 */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* MII extra pins: add on top of (base + rgmii extra pins) */
+		gmac1_mii_add_pins: gmac1-2-pins {
+			pinmux = <K3_PADCONF(37, 1)>,	/* gmac1 rxer */
+				 <K3_PADCONF(38, 1)>,	/* gmac1 txer */
+				 <K3_PADCONF(39, 1)>,	/* gmac1 crs */
+				 <K3_PADCONF(40, 1)>;	/* gmac1 col */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional int pins */
+		gmac1_int_pins: gmac1-3-pins {
+			pinmux = <K3_PADCONF(35, 1)>;	/* gmac1 int (from phy) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional pps pins */
+		gmac1_pps_pins: gmac1-4-pins {
+			pinmux = <K3_PADCONF(41, 1)>;	/* gmac1 pps (tsn/1588) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional reference clock output */
+		gmac1_refclk_pins: gmac1-5-pins {
+			pinmux = <K3_PADCONF(36, 1)>;	/* gmac1 clk ref */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+	};
+
+	gmac2_cfg: gmac2-cfg {
+		/* Base pins: - Used by RMII directly */
+		gmac2_base_pins: gmac2-0-pins {
+			pinmux = <K3_PADCONF(42, 1)>,	/* gmac2: RMII=crs_dv | RGMII=rx_ctl */
+				 <K3_PADCONF(43, 1)>,	/* gmac2 rx d0 */
+				 <K3_PADCONF(44, 1)>,	/* gmac2 rx d1 */
+				 <K3_PADCONF(45, 1)>,	/* gmac2: RMII=ref_clk | RGMII=rxc */
+				 <K3_PADCONF(48, 1)>,	/* gmac2 tx d0 */
+				 <K3_PADCONF(49, 1)>,	/* gmac2 tx d1 */
+				 <K3_PADCONF(53, 1)>,	/* gmac2: RMII=tx_en | RGMII=tx_ctl */
+				 <K3_PADCONF(54, 1)>,	/* gmac2 mdc */
+				 <K3_PADCONF(55, 1)>;	/* gmac2 mdio */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* RGMII extra pins: add on top of base pins */
+		gmac2_rgmii_add_pins: gmac2-1-pins {
+			pinmux = <K3_PADCONF(46, 1)>,	/* gmac2 rx d2 */
+				 <K3_PADCONF(47, 1)>,	/* gmac2 rx d3 */
+				 <K3_PADCONF(50, 1)>,	/* gmac2 tx clk */
+				 <K3_PADCONF(51, 1)>,	/* gmac2 tx d2 */
+				 <K3_PADCONF(52, 1)>;	/* gmac2 tx d3 */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional int pins */
+		gmac2_int_pins: gmac2-2-pins {
+			pinmux = <K3_PADCONF(56, 1)>;	/* gmac2 int (from phy) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional pps pins */
+		gmac2_pps_pins: gmac2-3-pins {
+			pinmux = <K3_PADCONF(58, 1)>;	/* gmac2 pps (tsn/1588) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional reference clock output */
+		gmac2_refclk_pins: gmac2-4-pins {
+			pinmux = <K3_PADCONF(57, 1)>;	/* gmac2 clk ref */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+	};
+
+	gmac3_cfg: gmac3-cfg {
+		/* Base pins: - Used by RMII directly */
+		gmac3_base_pins: gmac3-0-pins {
+			pinmux = <K3_PADCONF(59, 1)>,	/* gmac3: RMII=crs_dv | RGMII=rx_ctl */
+				 <K3_PADCONF(60, 1)>,	/* gmac3 rx d0 */
+				 <K3_PADCONF(61, 1)>,	/* gmac3 rx d1 */
+				 <K3_PADCONF(62, 1)>,	/* gmac3: RMII=ref_clk | RGMII=rxc */
+				 <K3_PADCONF(65, 1)>,	/* gmac3 tx d0 */
+				 <K3_PADCONF(66, 1)>,	/* gmac3 tx d1 */
+				 <K3_PADCONF(70, 1)>,	/* gmac3: RMII=tx_en | RGMII=tx_ctl */
+				 <K3_PADCONF(71, 1)>,	/* gmac3 mdc */
+				 <K3_PADCONF(72, 1)>;	/* gmac3 mdio */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* RGMII extra pins: add on top of base pins */
+		gmac3_rgmii_add_pins: gmac3-1-pins {
+			pinmux = <K3_PADCONF(63, 1)>,	/* gmac3 rx d2 */
+				 <K3_PADCONF(64, 1)>,	/* gmac3 rx d3 */
+				 <K3_PADCONF(67, 1)>,	/* gmac3 tx clk */
+				 <K3_PADCONF(68, 1)>,	/* gmac3 tx d2 */
+				 <K3_PADCONF(69, 1)>;	/* gmac3 tx d3 */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional int pins */
+		gmac3_int_pins: gmac3-2-pins {
+			pinmux = <K3_PADCONF(73, 1)>;	/* gmac3 int (from phy) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional pps pins */
+		gmac3_pps_pins: gmac3-3-pins {
+			pinmux = <K3_PADCONF(75, 1)>;	/* gmac3 pps (tsn/1588) */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		/* Optional reference clock output */
+		gmac3_refclk_pins: gmac3-4-pins {
+			pinmux = <K3_PADCONF(74, 1)>;	/* gmac3 clk ref */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <9>;		/* DS4 */
+		};
+	};
+
+	ssp0_0_cfg: ssp0-0-cfg {
+		ssp0-0-pins {
+			pinmux = <K3_PADCONF(53, 3)>,	/* ssp0 tx */
+				 <K3_PADCONF(54, 3)>,	/* ssp0 rx */
+				 <K3_PADCONF(55, 3)>;	/* ssp0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp0-1-pins {
+			pinmux = <K3_PADCONF(56, 3)>;	/* ssp0 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp0_1_cfg: ssp0-1-cfg {
+		ssp0-0-pins {
+			pinmux = <K3_PADCONF(81, 1)>,	/* ssp0 tx */
+				 <K3_PADCONF(82, 1)>,	/* ssp0 rx */
+				 <K3_PADCONF(83, 1)>;	/* ssp0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp0-1-pins {
+			pinmux = <K3_PADCONF(84, 1)>;	/* ssp0 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp0_2_cfg: ssp0-2-cfg {
+		ssp0-0-pins {
+			pinmux = <K3_PADCONF(104, 1)>,	/* ssp0 tx */
+				 <K3_PADCONF(105, 1)>,	/* ssp0 rx */
+				 <K3_PADCONF(106, 1)>;	/* ssp0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp0-1-pins {
+			pinmux = <K3_PADCONF(107, 1)>;	/* ssp0 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp1_0_cfg: ssp1-0-cfg {
+		ssp1-0-pins {
+			pinmux = <K3_PADCONF(32, 5)>,	/* ssp1 tx */
+				 <K3_PADCONF(33, 5)>,	/* ssp1 rx */
+				 <K3_PADCONF(34, 5)>;	/* ssp1 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp1-1-pins {
+			pinmux = <K3_PADCONF(35, 5)>;	/* ssp1 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp1_1_cfg: ssp1-1-cfg {
+		ssp1-0-pins {
+			pinmux = <K3_PADCONF(111, 1)>,	/* ssp1 tx */
+				 <K3_PADCONF(112, 1)>,	/* ssp1 rx */
+				 <K3_PADCONF(113, 1)>;	/* ssp1 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp1-1-pins {
+			pinmux = <K3_PADCONF(114, 1)>;	/* ssp1 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp2_0_cfg: ssp2-0-cfg {
+		ssp2-0-pins {
+			pinmux = <K3_PADCONF(28, 5)>,	/* ssp2 tx */
+				 <K3_PADCONF(29, 5)>,	/* ssp2 rx */
+				 <K3_PADCONF(30, 5)>;	/* ssp2 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp2-1-pins {
+			pinmux = <K3_PADCONF(31, 5)>;	/* ssp2 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp2_1_cfg: ssp2-1-cfg {
+		ssp2-0-pins {
+			pinmux = <K3_PADCONF(104, 2)>,	/* ssp2 tx */
+				 <K3_PADCONF(105, 2)>,	/* ssp2 rx */
+				 <K3_PADCONF(106, 2)>;	/* ssp2 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp2-1-pins {
+			pinmux = <K3_PADCONF(107, 2)>;	/* ssp2 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp3_0_cfg: ssp3-0-cfg {
+		ssp3-0-pins {
+			pinmux = <K3_PADCONF(68, 6)>,	/* ssp3 tx */
+				 <K3_PADCONF(69, 6)>,	/* ssp3 rx */
+				 <K3_PADCONF(70, 6)>;	/* ssp3 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp3-1-pins {
+			pinmux = <K3_PADCONF(71, 6)>;	/* ssp3 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	ssp3_1_cfg: ssp3-1-cfg {
+		ssp3-0-pins {
+			pinmux = <K3_PADCONF(99, 1)>,	/* ssp3 tx */
+				 <K3_PADCONF(100, 1)>,	/* ssp3 rx */
+				 <K3_PADCONF(101, 1)>;	/* ssp3 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		ssp3-1-pins {
+			pinmux = <K3_PADCONF(102, 1)>;	/* ssp3 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rssp0_0_cfg: rssp0-0-cfg {
+		rssp0-0-pins {
+			pinmux = <K3_PADCONF(60, 3)>,	/* rssp0 tx */
+				 <K3_PADCONF(61, 3)>,	/* rssp0 rx */
+				 <K3_PADCONF(62, 3)>;	/* rssp0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		rssp0-1-pins {
+			pinmux = <K3_PADCONF(63, 3)>;	/* rssp0 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rssp0_1_cfg: rssp0-1-cfg {
+		rssp0-0-pins {
+			pinmux = <K3_PADCONF(86, 1)>,	/* rssp0 tx */
+				 <K3_PADCONF(87, 1)>,	/* rssp0 rx */
+				 <K3_PADCONF(88, 1)>;	/* rssp0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		rssp0-1-pins {
+			pinmux = <K3_PADCONF(89, 1)>;	/* rssp0 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rssp1_0_cfg: rssp1-0-cfg {
+		rssp1-0-pins {
+			pinmux = <K3_PADCONF(64, 3)>,	/* rssp1 tx */
+				 <K3_PADCONF(65, 3)>,	/* rssp1 rx */
+				 <K3_PADCONF(66, 3)>;	/* rssp1 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		rssp1-1-pins {
+			pinmux = <K3_PADCONF(67, 3)>;	/* rssp1 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	rssp1_1_cfg: rssp1-1-cfg {
+		rssp1-0-pins {
+			pinmux = <K3_PADCONF(108, 1)>,	/* rssp1 tx */
+				 <K3_PADCONF(109, 1)>,	/* rssp1 rx */
+				 <K3_PADCONF(116, 1)>;	/* rssp1 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		rssp1-1-pins {
+			pinmux = <K3_PADCONF(117, 1)>;	/* rssp1 frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	pwr_ssp_cfg: pwr_ssp-cfg {
+		pwr_ssp-0-pins {
+			pinmux = <K3_PADCONF(151, 0)>,	/* pwr ssp tx */
+				 <K3_PADCONF(152, 0)>,	/* pwr ssp rx */
+				 <K3_PADCONF(149, 0)>;	/* pwr ssp clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		pwr_ssp-1-pins {
+			pinmux = <K3_PADCONF(150, 0)>;	/* pwr ssp frm */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	qspi_cfg: qspi-cfg {
+		qspi-pins {
+			pinmux = <K3_PADCONF(138, 0)>,	/* qspi dat0 */
+				 <K3_PADCONF(139, 0)>,	/* qspi dat1 */
+				 <K3_PADCONF(140, 0)>,	/* qspi dat2 */
+				 <K3_PADCONF(141, 0)>,	/* qspi dat3 */
+				 <K3_PADCONF(144, 0)>;	/* qspi clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	qspi_cs0_cfg: qspi-cs0-cfg {
+		qspi-cs0-pins {
+			pinmux = <K3_PADCONF(142, 0)>;	/* qspi cs0 */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	qspi_cs1_cfg: qspi-cs1-cfg {
+		qspi-cs1-pins {
+			pinmux = <K3_PADCONF(143, 0)>;	/* qspi cs1 */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	espi0_0_cfg: espi0-0-cfg {
+		espi0-0-pins {
+			pinmux = <K3_PADCONF(0, 5)>,	/* espi0 d0 */
+				 <K3_PADCONF(1, 5)>,	/* espi0 d1 */
+				 <K3_PADCONF(2, 5)>,	/* espi0 d2 */
+				 <K3_PADCONF(3, 5)>,	/* espi0 d3 */
+				 <K3_PADCONF(4, 5)>,	/* espi0 cs */
+				 <K3_PADCONF(6, 5)>,	/* espi0 reset */
+				 <K3_PADCONF(7, 5)>;	/* espi0 alert */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		espi0-1-pins {
+			pinmux = <K3_PADCONF(5, 5)>;	/* espi0 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	espi0_1_cfg: espi0-1-cfg {
+		espi0-0-pins {
+			pinmux = <K3_PADCONF(68, 3)>,	/* espi0 d0 */
+				 <K3_PADCONF(69, 3)>,	/* espi0 d1 */
+				 <K3_PADCONF(70, 3)>,	/* espi0 d2 */
+				 <K3_PADCONF(71, 3)>,	/* espi0 d3 */
+				 <K3_PADCONF(72, 3)>,	/* espi0 cs */
+				 <K3_PADCONF(74, 3)>,	/* espi0 reset */
+				 <K3_PADCONF(75, 3)>;	/* espi0 alert */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		espi0-1-pins {
+			pinmux = <K3_PADCONF(73, 3)>;	/* espi0 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	espi0_2_cfg: espi0-2-cfg {
+		espi0-0-pins {
+			pinmux = <K3_PADCONF(91, 4)>,	/* espi0 d0 */
+				 <K3_PADCONF(92, 4)>,	/* espi0 d1 */
+				 <K3_PADCONF(93, 4)>,	/* espi0 d2 */
+				 <K3_PADCONF(94, 4)>,	/* espi0 d3 */
+				 <K3_PADCONF(95, 4)>,	/* espi0 cs */
+				 <K3_PADCONF(97, 4)>,	/* espi0 reset */
+				 <K3_PADCONF(98, 4)>;	/* espi0 alert */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <1800>;
+		};
+
+		espi0-1-pins {
+			pinmux = <K3_PADCONF(96, 4)>;	/* espi0 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <1800>;
+		};
+	};
+
+	respi0_cfg: respi0-cfg {
+		respi0-0-pins {
+			pinmux = <K3_PADCONF(86, 2)>,	/* respi0 d0 */
+				 <K3_PADCONF(87, 2)>,	/* respi0 d1 */
+				 <K3_PADCONF(88, 2)>,	/* respi0 d2 */
+				 <K3_PADCONF(89, 2)>,	/* respi0 d3 */
+				 <K3_PADCONF(90, 2)>,	/* respi0 cs */
+				 <K3_PADCONF(92, 2)>,	/* respi0 reset */
+				 <K3_PADCONF(93, 2)>;	/* respi0 alert */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		respi0-1-pins {
+			pinmux = <K3_PADCONF(91, 2)>;	/* respi0 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	mmc1_cfg: mmc1-cfg {
+		mmc1-0-pins {
+			pinmux = <K3_PADCONF(132, 0)>,	/* mmc1 dat3 */
+				 <K3_PADCONF(133, 0)>,	/* mmc1 dat2 */
+				 <K3_PADCONF(134, 0)>,	/* mmc1 dat1 */
+				 <K3_PADCONF(135, 0)>,	/* mmc1 dat0 */
+				 <K3_PADCONF(136, 0)>;	/* mmc1 cmd */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <8>;		/* DS3 */
+			power-source = <3300>;
+		};
+
+		mmc1-1-pins {
+			pinmux = <K3_PADCONF(137, 0)>;	/* mmc1 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <8>;		/* DS3 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_uhs_cfg: mmc1-uhs-cfg {
+		mmc1-0-pins {
+			pinmux = <K3_PADCONF(132, 0)>,	/* mmc1 dat3 */
+				 <K3_PADCONF(133, 0)>,	/* mmc1 dat2 */
+				 <K3_PADCONF(134, 0)>,	/* mmc1 dat1 */
+				 <K3_PADCONF(135, 0)>,	/* mmc1 dat0 */
+				 <K3_PADCONF(136, 0)>;	/* mmc1 cmd */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <7>;		/* DS3 */
+			power-source = <1800>;
+		};
+
+		mmc1-1-pins {
+			pinmux = <K3_PADCONF(137, 0)>;	/* mmc1 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <6>;		/* DS2 */
+			power-source = <1800>;
+		};
+	};
+
+	mmc1_debug_cfg: mmc1-debug-cfg {
+		mmc1-0-pins {
+			pinmux = <K3_PADCONF(132, 2)>,	/* uart0 tx */
+				 <K3_PADCONF(133, 2)>,	/* uart0 rx */
+				 <K3_PADCONF(134, 0)>,	/* mmc1 dat1 */
+				 <K3_PADCONF(135, 0)>,	/* mmc1 dat0 */
+				 <K3_PADCONF(136, 0)>;	/* mmc1 cmd */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <8>;		/* DS3 */
+			power-source = <3300>;
+		};
+
+		mmc1-1-pins {
+			pinmux = <K3_PADCONF(137, 0)>;	/* mmc1 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <8>;		/* DS3 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_jtag_cfg: mmc1-jtag-cfg {
+		mmc1-0-pins {
+			pinmux = <K3_PADCONF(132, 5)>,	/* pri tdi */
+				 <K3_PADCONF(133, 5)>,	/* pri tms */
+				 <K3_PADCONF(134, 5)>,	/* pri tdo */
+				 <K3_PADCONF(137, 5)>;	/* pri tck */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <8>;		/* DS3 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc2_cfg: mmc2-cfg {
+		mmc2-0-pins {
+			pinmux = <K3_PADCONF(122, 1)>,	/* mmc2 dat3 */
+				 <K3_PADCONF(123, 1)>,	/* mmc2 dat2 */
+				 <K3_PADCONF(124, 1)>,	/* mmc2 dat1 */
+				 <K3_PADCONF(125, 1)>,	/* mmc2 dat0 */
+				 <K3_PADCONF(126, 1)>;	/* mmc2 cmd */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+		};
+
+		mmc2-1-pins {
+			pinmux = <K3_PADCONF(127, 1)>;	/* mmc2 clk */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam0_0_cfg: cam0-0-cfg {
+		cam0-0-pins {
+			pinmux = <K3_PADCONF(13, 5)>;	/* cam0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam0_1_cfg: cam0-1-cfg {
+		cam0-0-pins {
+			pinmux = <K3_PADCONF(46, 3)>;	/* cam0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam0_2_cfg: cam0-2-cfg {
+		cam1-0-pins {
+			pinmux = <K3_PADCONF(75, 2)>;	/* cam0 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam1_0_cfg: cam1-0-cfg {
+		cam1-0-pins {
+			pinmux = <K3_PADCONF(34, 4)>;	/* cam1 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam1_1_cfg: cam1-1-cfg {
+		cam1-0-pins {
+			pinmux = <K3_PADCONF(47, 3)>;	/* cam1 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam1_2_cfg: cam1-2-cfg {
+		cam0-0-pins {
+			pinmux = <K3_PADCONF(74, 2)>;	/* cam1 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam2_0_cfg: cam2-0-cfg {
+		cam2-0-pins {
+			pinmux = <K3_PADCONF(35, 4)>;	/* cam2 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam2_1_cfg: cam2-1-cfg {
+		cam2-0-pins {
+			pinmux = <K3_PADCONF(52, 5)>;	/* cam2 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam2_2_cfg: cam2-2-cfg {
+		cam2-0-pins {
+			pinmux = <K3_PADCONF(85, 1)>;	/* cam2 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam3_0_cfg: cam3-0-cfg {
+		cam3-0-pins {
+			pinmux = <K3_PADCONF(43, 3)>;	/* cam3 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam3_1_cfg: cam3-1-cfg {
+		cam3-0-pins {
+			pinmux = <K3_PADCONF(67, 4)>;	/* cam3 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	cam3_2_cfg: cam3-2-cfg {
+		cam3-0-pins {
+			pinmux = <K3_PADCONF(99, 6)>;	/* cam3 clk */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dsi_0_cfg: dsi-0-cfg {
+		dsi-0-pins {
+			pinmux = <K3_PADCONF(13, 6)>;	/* dsi te */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dsi_1_cfg: dsi-1-cfg {
+		dsi-0-pins {
+			pinmux = <K3_PADCONF(38, 6)>;	/* dsi te */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dsi_2_cfg: dsi-2-cfg {
+		dsi-0-pins {
+			pinmux = <K3_PADCONF(69, 5)>;	/* dsi te */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dsi_3_cfg: dsi-3-cfg {
+		dsi-0-pins {
+			pinmux = <K3_PADCONF(72, 5)>;	/* dsi te */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dsi_4_cfg: dsi-4-cfg {
+		dsi-0-pins {
+			pinmux = <K3_PADCONF(84, 6)>;	/* dsi te */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dsi_5_cfg: dsi-5-cfg {
+		dsi-0-pins {
+			pinmux = <K3_PADCONF(90, 1)>;	/* dsi te */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dsi_6_cfg: dsi-6-cfg {
+		dsi-0-pins {
+			pinmux = <K3_PADCONF(92, 6)>;	/* dsi te */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	/* edp only for edp */
+	edp0_0_cfg: edp0-0-cfg {
+		edp0-0-pins {
+			pinmux = <K3_PADCONF(30, 6)>;	/* edp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	edp0_1_cfg: edp0-1-cfg {
+		edp0-0-pins {
+			pinmux = <K3_PADCONF(57, 4)>;	/* edp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	edp1_0_cfg: edp1-0-cfg {
+		edp1-0-pins {
+			pinmux = <K3_PADCONF(31, 6)>;	/* edp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	/* dp only for edp&dp */
+	dp0_0_cfg: dp0-0-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(9, 6)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dp0_1_cfg: dp0-1-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(23, 6)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dp0_2_cfg: dp0-2-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(97, 5)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source= <3300>;
+		};
+	};
+
+	dp0_3_cfg: dp0-3-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(124, 4)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dp1_0_cfg: dp1-0-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(10, 6)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dp1_1_cfg: dp1-1-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(24, 6)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dp1_2_cfg: dp1-2-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(69, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dp1_3_cfg: dp1-3-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(72, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	dp1_4_cfg: dp1-4-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(98, 5)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	dp1_5_cfg: dp1-5-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(125, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb20_host_drv_0_cfg: usb20_host_drv-0-cfg {
+		usb20_host_drv-pins {
+			pinmux = <K3_PADCONF(16, 6)>;	/* usb20 host drive */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb20_host_drv_1_cfg: usb20_host_drv-1-cfg {
+		usb20_host_drv-pins {
+			pinmux = <K3_PADCONF(39, 5)>;	/* usb20 host drive */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb20_host_drv_2_cfg: usb20_host_drv-2-cfg {
+		usb20_host_drv-pins {
+			pinmux = <K3_PADCONF(73, 5)>;	/* usb20 host drive */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb20_host_drv_3_cfg: usb20_host_drv-3-cfg {
+		usb20_host_drv-pins {
+			pinmux = <K3_PADCONF(103, 3)>;	/* usb20 host drive */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb20_host_drv_4_cfg: usb20_host_drv-4-cfg {
+		usb20_host_drv-pins {
+			pinmux = <K3_PADCONF(108, 2)>;	/* usb20 host drive */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_drd_0_cfg: usb30_drd-0-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(17, 6)>,	/* usb30 drd id */
+				 <K3_PADCONF(18, 6)>,	/* usb30 drd vbuson */
+				 <K3_PADCONF(19, 6)>;	/* usb30 drd drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_drd_1_cfg: usb30_drd-1-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(95, 3)>,	/* usb30 drd id */
+				 <K3_PADCONF(96, 3)>,	/* usb30 drd vbuson */
+				 <K3_PADCONF(97, 3)>;	/* usb30 drd drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_drd_2_cfg: usb30_drd-2-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(116, 2)>,	/* usb30 drd id */
+				 <K3_PADCONF(117, 2)>,	/* usb30 drd vbuson */
+				 <K3_PADCONF(118, 2)>;	/* usb30 drd drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_drd_int_0_cfg: usb30_drd_int-0-cfg {
+		usb30_drd_int-pins {
+			pinmux = <K3_PADCONF(111, 5)>;	/* usb30 drd int */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_drd_int_1_cfg: usb30_drd_int-1-cfg {
+		usb30_drd_int-pins {
+			pinmux = <K3_PADCONF(119, 2)>;	/* usb30 drd int */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_b_drv_0_cfg: usb30_b_drv-0-cfg {
+		usb30_c_drv-pins {
+			pinmux = <K3_PADCONF(84, 4)>;	/* usb30 b drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_c_drv_0_cfg: usb30_c_drv-0-cfg {
+		usb30_c_drv-pins {
+			pinmux = <K3_PADCONF(85, 4)>;	/* usb30 c drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_c_drv_1_cfg: usb30_c_drv-1-cfg {
+		usb30_c_drv-pins {
+			pinmux = <K3_PADCONF(127, 6)>;	/* usb30 c drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_d_drv_0_cfg: usb30_d_drv-0-cfg {
+		usb30_d_drv-pins {
+			pinmux = <K3_PADCONF(20, 6)>;	/* usb30 d drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_d_drv_1_cfg: usb30_d_drv-1-cfg {
+		usb30_d_drv-pins {
+			pinmux = <K3_PADCONF(27, 5)>;	/* usb30 d drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_d_drv_2_cfg: usb30_d_drv-2-cfg {
+		usb30_d_drv-pins {
+			pinmux = <K3_PADCONF(112, 5)>;	/* usb30 d drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30h1_drv_0_cfg: usb30h1_drv-0-cfg {
+		usb30h1_drv-pins {
+			pinmux = <K3_PADCONF(74, 5)>;	/* usb30h-1 drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30h1_drv_1_cfg: usb30h1_drv-1-cfg {
+		usb30h1_drv-pins {
+			pinmux = <K3_PADCONF(104, 3)>;	/* usb30h-1 drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30h2_drv_0_cfg: usb30h2_drv-0-cfg {
+		usb30h2_drv-pins {
+			pinmux = <K3_PADCONF(75, 5)>;	/* usb30h-2 drv */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_drd_dir_0_cfg: usb30_drd_dir-0-cfg {
+		usb30_drd_dir-pins {
+			pinmux = <K3_PADCONF(86, 6)>;	/* usb30 drd dir */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+
+	usb30_drd_dir_1_cfg: usb30_drd_dir-1-cfg {
+		usb30_drd_dir-pins {
+			pinmux = <K3_PADCONF(107, 4)>;	/* usb30 drd dir */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <25>;		/* DS8 */
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-rdomain.dtsi b/arch/riscv/boot/dts/spacemit/k3-rdomain.dtsi
new file mode 100644
index 000000000000..8392837a1163
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-rdomain.dtsi
@@ -0,0 +1,551 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+&soc {
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	r_uart0: serial@c0881000 {
+		compatible = "spacemit,k1-uart",
+			     "intel,xscale-uart";
+		reg = <0x0 0xc0881000 0x0 0x100>;
+		clocks = <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART0>,
+			 <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART0_BUS>,
+			 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+		clock-names = "core", "bus", "gate";
+		spacemit,rcpu-uart;
+		resets = <&syscon_rcpu_uartctrl RESET_RCPU_UARTCTRL_RUART0>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		fifo-size = <256>;
+		tx-threshold = <32>;
+		interrupt-parent = <&saplic>;
+		interrupts = <251 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+	};
+
+	r_uart1: serial@c0881100 {
+		compatible = "spacemit,k1-uart",
+			     "intel,xscale-uart";
+		reg = <0x0 0xc0881100 0x0 0x100>;
+		clocks = <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART1>,
+			 <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART1_BUS>,
+			 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+		clock-names = "core", "bus", "gate";
+		spacemit,rcpu-uart;
+		resets = <&syscon_rcpu_uartctrl RESET_RCPU_UARTCTRL_RUART1>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		fifo-size = <256>;
+		tx-threshold = <32>;
+		interrupt-parent = <&saplic>;
+		interrupts = <252 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+	};
+
+	r_uart2: serial@c0881200 {
+		compatible = "spacemit,k1-uart",
+			     "intel,xscale-uart";
+		reg = <0x0 0xc0881200 0x0 0x100>;
+		clocks = <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART2>,
+			 <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART2_BUS>,
+			 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+		clock-names = "core", "bus", "gate";
+		spacemit,rcpu-uart;
+		resets = <&syscon_rcpu_uartctrl RESET_RCPU_UARTCTRL_RUART2>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		fifo-size = <256>;
+		tx-threshold = <32>;
+		interrupt-parent = <&saplic>;
+		interrupts = <253 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+	};
+
+	r_uart3: serial@c0881300 {
+		compatible = "spacemit,k1-uart",
+			     "intel,xscale-uart";
+		reg = <0x0 0xc0881300 0x0 0x100>;
+		clocks = <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART3>,
+			 <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART3_BUS>,
+			 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+		clock-names = "core", "bus", "gate";
+		spacemit,rcpu-uart;
+		resets = <&syscon_rcpu_uartctrl RESET_RCPU_UARTCTRL_RUART3>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		fifo-size = <256>;
+		tx-threshold = <32>;
+		interrupt-parent = <&saplic>;
+		interrupts = <254 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+	};
+
+	r_uart4: serial@c0881400 {
+		compatible = "spacemit,k1-uart",
+			     "intel,xscale-uart";
+		reg = <0x0 0xc0881400 0x0 0x100>;
+		clocks = <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART4>,
+			 <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART4_BUS>,
+			 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+		clock-names = "core", "bus", "gate";
+		spacemit,rcpu-uart;
+		resets = <&syscon_rcpu_uartctrl RESET_RCPU_UARTCTRL_RUART4>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		fifo-size = <256>;
+		tx-threshold = <32>;
+		interrupt-parent = <&saplic>;
+		interrupts = <255 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+	};
+
+	r_uart5: serial@c0881500 {
+		compatible = "spacemit,k1-uart",
+			     "intel,xscale-uart";
+		reg = <0x0 0xc0881500 0x0 0x100>;
+		clocks = <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART5>,
+			 <&syscon_rcpu_uartctrl CLK_RCPU_UARTCTRL_RUART5_BUS>,
+			 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+		clock-names = "core", "bus", "gate";
+		spacemit,rcpu-uart;
+		resets = <&syscon_rcpu_uartctrl RESET_RCPU_UARTCTRL_RUART5>;
+		reg-shift = <2>;
+		reg-io-width = <4>;
+		fifo-size = <256>;
+		tx-threshold = <32>;
+		interrupt-parent = <&saplic>;
+		interrupts = <256 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+	};
+
+	r_flexcan0: fdcan@c0710000 {
+		compatible = "spacemit,k1-flexcan";
+		reg = <0x0 0xc0710000 0x0 0x4000>;
+		clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN0>,
+			 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN0_BUS>;
+		clock-names = "per","ipg";
+		interrupts = <241 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&saplic>;
+		fsl,clk-source = <0>;
+		resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RCAN0>;
+		status = "disabled";
+	};
+
+	r_flexcan1: fdcan@c0720000 {
+		compatible = "spacemit,k1-flexcan";
+		reg = <0x0 0xc0720000 0x0 0x4000>;
+		clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN1>,
+			 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN1_BUS>;
+		clock-names = "per","ipg";
+		interrupts = <243 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&saplic>;
+		fsl,clk-source = <0>;
+		resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RCAN1>;
+		status = "disabled";
+	};
+
+	r_flexcan2: fdcan@c0730000 {
+		compatible = "spacemit,k1-flexcan";
+		reg = <0x0 0xc0730000 0x0 0x4000>;
+		clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN2>,
+			 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN2_BUS>;
+		clock-names = "per","ipg";
+		interrupts = <245 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&saplic>;
+		fsl,clk-source = <0>;
+		resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RCAN2>;
+		status = "disabled";
+	};
+
+	r_flexcan3: fdcan@c0740000 {
+		compatible = "spacemit,k1-flexcan";
+		reg = <0x0 0xc0740000 0x0 0x4000>;
+		clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN3>,
+			 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN3_BUS>;
+		clock-names = "per","ipg";
+		interrupts = <247 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&saplic>;
+		fsl,clk-source = <0>;
+		resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RCAN3>;
+		status = "disabled";
+	};
+
+	r_flexcan4: fdcan@c0750000 {
+		compatible = "spacemit,k1-flexcan";
+		reg = <0x0 0xc0750000 0x0 0x4000>;
+		clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN4>,
+			 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RCAN4_BUS>;
+		clock-names = "per","ipg";
+		interrupts = <249 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&saplic>;
+		fsl,clk-source = <0>;
+		resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RCAN4>;
+		status = "disabled";
+	};
+
+	rpwm0: pwm@c088d100 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d100 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM0>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM0_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM0>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm1: pwm@c088d200 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d200 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM1>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM1_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM1>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm2: pwm@c088d300 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d300 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM2>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM2_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM2>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm3: pwm@c088d400 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d400 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM3>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM3_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM3>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm4: pwm@c088d500 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d500 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM4>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM4_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM4>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm5: pwm@c088d600 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d600 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM5>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM5_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM5>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm6: pwm@c088d700 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d700 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM6>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM6_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM6>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm7: pwm@c088d800 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d800 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM7>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM7_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM7>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm8: pwm@c088d900 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088d900 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM8>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM8_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM8>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	rpwm9: pwm@c088da00 {
+		compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+		reg = <0x0 0xc088da00 0x0 0x10>;
+		clocks = <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM9>,
+			 <&syscon_rcpu_pwmctrl CLK_RCPU_PWMCTRL_RPWM9_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_pwmctrl RESET_RCPU_PWMCTRL_PWM9>;
+		#pwm-cells = <3>;
+		k1,pwm-disable-fd;
+		status = "disabled";
+	};
+
+	r_i2c0: ri2c@c0886000 {
+		compatible = "spacemit,k1-i2c";
+		reg = <0x0 0xc0886000 0x0 0x38>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		interrupt-parent = <&saplic>;
+		interrupts = <272 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_rcpu_i2cctrl CLK_RCPU_I2CCTRL_RI2C0>,
+			 <&syscon_rcpu_i2cctrl CLK_RCPU_I2CCTRL_RI2C0_BUS>;
+		clock-names = "func", "bus";
+		clock-frequency = <400000>;
+		resets = <&syscon_rcpu_i2cctrl RESET_RCPU_I2CCTRL_RI2C0>;
+		status = "disabled";
+	};
+
+	r_i2c1: ri2c@c0886100 {
+		compatible = "spacemit,k1-i2c";
+		reg = <0x0 0xc0886100 0x0 0x38>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		interrupt-parent = <&saplic>;
+		interrupts = <273 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_rcpu_i2cctrl CLK_RCPU_I2CCTRL_RI2C1>,
+			 <&syscon_rcpu_i2cctrl CLK_RCPU_I2CCTRL_RI2C1_BUS>;
+		clock-names = "func", "bus";
+		clock-frequency = <400000>;
+		resets = <&syscon_rcpu_i2cctrl RESET_RCPU_I2CCTRL_RI2C1>;
+		status = "disabled";
+	};
+
+	adma0: adma0@C0883000 {
+		compatible = "spacemit,k3-adma";
+		reg = <0x0 0xc0883000 0x0 0x100>,
+		      <0x0 0xc08d0000 0x0 0x100>;
+		reg-names = "adma_reg", "buf_addr";
+		interrupt-parent = <&saplic>;
+		interrupts = <257 IRQ_TYPE_LEVEL_HIGH>,
+			     <258 IRQ_TYPE_LEVEL_HIGH>;
+		#dma-cells= <1>;
+		#dma-channels = <2>;
+		status = "disabled";
+	};
+
+	adma1: adma1@C0883400 {
+		compatible = "spacemit,k3-adma";
+		reg = <0x0 0xc0883400 0x0 0x100>,
+		      <0x0 0xc08d0100 0x0 0x100>;
+		reg-names = "adma_reg", "buf_addr";
+		interrupt-parent = <&saplic>;
+		interrupts = <260 IRQ_TYPE_LEVEL_HIGH>,
+			     <261 IRQ_TYPE_LEVEL_HIGH>;
+		#dma-cells= <1>;
+		#dma-channels = <2>;
+		rpmsg-services = "adma-service";
+		rpmsg-channels = <1>;
+		status = "disabled";
+	};
+
+	adma2: adma2@C0883800 {
+		compatible = "spacemit,k3-adma";
+		reg = <0x0 0xc0883800 0x0 0x100>,
+		      <0x0 0xc08d0200 0x0 0x100>;
+		reg-names = "adma_reg", "buf_addr";
+		interrupt-parent = <&saplic>;
+		interrupts = <263 IRQ_TYPE_LEVEL_HIGH>,
+			     <264 IRQ_TYPE_LEVEL_HIGH>;
+		#dma-cells= <1>;
+		#dma-channels = <2>;
+		status = "disabled";
+	};
+
+	adma3: adma3@C0883C00 {
+		compatible = "spacemit,k3-adma";
+		reg = <0x0 0xc0883C00 0x0 0x100>,
+		      <0x0 0xc08d0300 0x0 0x100>;
+		reg-names = "adma_reg", "buf_addr";
+		interrupt-parent = <&saplic>;
+		interrupts = <266 IRQ_TYPE_LEVEL_HIGH>,
+			     <267 IRQ_TYPE_LEVEL_HIGH>;
+		#dma-cells= <1>;
+		#dma-channels = <2>;
+		status = "disabled";
+	};
+
+	ri2s0: ri2s0@c0883100 {
+		compatible = "spacemit,k3-ri2s";
+		#sound-dai-cells = <0>;
+		reg = <0x0 0xc0883100 0x0 0x300>,
+		      <0x0 0xc08d0400 0x0 0x4000>;
+		clocks = <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S0>,
+			 <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S0_BUS>,
+			 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RI2S0_SYSCLK>;
+		clock-names = "func", "bus", "sysclk";
+		resets = <&syscon_rcpu_i2sctrl RESET_RCPU_I2SCTRL_RI2S0>,
+			 <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RI2S0_SYSCLK>;
+		reset-names = "reset", "reset-sys";
+		dmas = <&adma0 1>, <&adma0 0>;
+		dma-names = "rx", "tx";
+		status = "disabled";
+	};
+
+	ri2s1: ri2s1@c0883500 {
+		compatible = "spacemit,k3-ri2s";
+		#sound-dai-cells = <0>;
+		reg = <0x0 0xc0883500 0x0 0x300>,
+		      <0x0 0xc08d4400 0x0 0x4000>;
+		clocks = <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S1>,
+			 <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S1_BUS>,
+			 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RI2S1_SYSCLK>;
+		clock-names = "func", "bus", "sysclk";
+		resets = <&syscon_rcpu_i2sctrl RESET_RCPU_I2SCTRL_RI2S1>,
+			 <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RI2S1_SYSCLK>;
+		reset-names = "reset", "reset-sys";
+		dmas = <&adma1 1>, <&adma1 0>;
+		dma-names = "rx", "tx";
+		status = "disabled";
+	};
+
+	ri2s2: ri2s2@c0883900 {
+		compatible = "spacemit,k3-ri2s";
+		#sound-dai-cells = <0>;
+		reg = <0x0 0xc0883900 0x0 0x300>,
+		      <0x0 0xc08d8400 0x0 0x4000>;
+		clocks = <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S2>,
+			 <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S2_BUS>,
+			 <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S2_SYSCLK>;
+		clock-names = "func", "bus", "sysclk";
+		resets = <&syscon_rcpu_i2sctrl RESET_RCPU_I2SCTRL_RI2S2>,
+			 <&syscon_rcpu_i2sctrl RESET_RCPU_I2SCTRL_RI2S2_SYSCLK>;
+		reset-names = "reset", "reset-sys";
+		dmas = <&adma2 1>, <&adma2 0>;
+		dma-names = "rx", "tx";
+		status = "disabled";
+	};
+
+	ri2s3: ri2s3@c0883D00 {
+		compatible = "spacemit,k3-ri2s";
+		#sound-dai-cells = <0>;
+		reg = <0x0 0xc0883d00 0x0 0x300>,
+		      <0x0 0xc08dc400 0x0 0x4000>;
+		clocks = <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S3>,
+			 <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S3_BUS>,
+			 <&syscon_rcpu_i2sctrl CLK_RCPU_I2SCTRL_RI2S3_SYSCLK>;
+		clock-names = "func", "bus", "sysclk";
+		resets = <&syscon_rcpu_i2sctrl RESET_RCPU_I2SCTRL_RI2S3>,
+			 <&syscon_rcpu_i2sctrl RESET_RCPU_I2SCTRL_RI2S3_SYSCLK>;
+		reset-names = "reset", "reset-sys";
+		dmas = <&adma3 1>, <&adma3 0>;
+		dma-names = "rx", "tx";
+		status = "disabled";
+	};
+
+	sound_card_dp0: sound-card-dp0@0 {
+		compatible = "simple-audio-card";
+		simple-audio-card,mclk-fs = <512>;
+		status = "disabled";
+		simple-audio-card,dai-link@0 {
+			format = "left_j";
+			frame-master = <&dp0_link_cpu>;
+			bitclock-master = <&dp0_link_cpu>;
+			dp0_link_cpu: cpu {
+				sound-dai = <&ri2s2>;
+			};
+		};
+	};
+
+	sound_card_dp1: sound-card-dp1@1 {
+		compatible = "simple-audio-card";
+		simple-audio-card,mclk-fs = <512>;
+		status = "disabled";
+		simple-audio-card,dai-link@0 {
+			format = "left_j";
+			frame-master = <&dp1_link_cpu>;
+			bitclock-master = <&dp1_link_cpu>;
+			dp1_link_cpu: cpu {
+				sound-dai = <&ri2s3>;
+			};
+		};
+	};
+
+	r_ircrx0: r-irc-rx0@c0887000 {
+		compatible = "spacemit-k1,irc";
+		reg = <0x0 0xc0887000 0x0 0x100>;
+		interrupts = <274 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&saplic>;
+		clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RIRC0>;
+		resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RIRC0>;
+		status = "disabled";
+	};
+
+	r_ircrx1: r-irc-rx1@c088e000 {
+		compatible = "spacemit-k1,irc";
+		reg = <0x0 0xc088e000 0x0 0x100>;
+		interrupts = <275 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&saplic>;
+		clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_RIRC1>;
+		resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RIRC1>;
+		status = "disabled";
+	};
+
+	rspi0: rspi0@c0885000 {
+		compatible = "spacemit,k3-spi";
+		reg = <0x0 0xc0885000 0x0 0x30>;
+		interrupt-parent = <&saplic>;
+		interrupts = <269 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_rcpu_spictrl  CLK_RCPU_SPICTRL_RSPI0>,
+			 <&syscon_rcpu_spictrl CLK_RCPU_SPICTRL_RSPI0_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_spictrl RESET_RCPU_SPICTRL_RSPI0>;
+		spi-max-frequency = <51200000>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	rspi1: rspi1@c0885100 {
+		compatible = "spacemit,k3-spi";
+		reg = <0x0 0xc0885100 0x0 0x30>;
+		interrupt-parent = <&saplic>;
+		interrupts = <270 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_rcpu_spictrl  CLK_RCPU_SPICTRL_RSPI1>,
+			 <&syscon_rcpu_spictrl CLK_RCPU_SPICTRL_RSPI1_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_spictrl RESET_RCPU_SPICTRL_RSPI1>;
+		spi-max-frequency = <51200000>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
+	rspi2: rspi2@c0885200 {
+		compatible = "spacemit,k3-spi";
+		reg = <0x0 0xc0885200 0x0 0x30>;
+		interrupt-parent = <&saplic>;
+		interrupts = <271 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&syscon_rcpu_spictrl  CLK_RCPU_SPICTRL_RSPI2>,
+			 <&syscon_rcpu_spictrl CLK_RCPU_SPICTRL_RSPI2_BUS>;
+		clock-names = "func", "bus";
+		resets = <&syscon_rcpu_spictrl RESET_RCPU_SPICTRL_RSPI2>;
+		spi-max-frequency = <51200000>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-trace.dtsi b/arch/riscv/boot/dts/spacemit/k3-trace.dtsi
new file mode 100644
index 000000000000..aad1a5850077
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-trace.dtsi
@@ -0,0 +1,803 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2026 Spacemit, Inc */
+
+/{
+	soc: soc {
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		encoder0: encoder@d9002000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9002000 0x0 0x1000>;
+			cpus = <&cpu_0>;
+			out-ports {
+				port {
+					cluster0_encoder0_out_port: endpoint {
+						remote-endpoint = <&cluster0_bridge0_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge0: bridge@d9003000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9003000 0x0 0x1000>;
+			cpus = <&cpu_0>;
+			out-ports {
+				port {
+					cluster0_bridge0_out_port: endpoint {
+						remote-endpoint = <&cluster0_funnel_in_port0>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster0_bridge0_in_port: endpoint {
+						remote-endpoint = <&cluster0_encoder0_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder1: encoder@d9004000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9004000 0x0 0x1000>;
+			cpus = <&cpu_1>;
+			out-ports {
+				port {
+					cluster0_encoder1_out_port: endpoint {
+						remote-endpoint = <&cluster0_bridge1_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge1: bridge@d9005000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9005000 0x0 0x1000>;
+			cpus = <&cpu_1>;
+			out-ports {
+				port {
+					cluster0_bridge1_out_port: endpoint {
+						remote-endpoint = <&cluster0_funnel_in_port1>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster0_bridge1_in_port: endpoint {
+						remote-endpoint = <&cluster0_encoder1_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder2: encoder@d9006000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9006000 0x0 0x1000>;
+			cpus = <&cpu_2>;
+			out-ports {
+				port {
+					cluster0_encoder2_out_port: endpoint {
+						remote-endpoint = <&cluster0_bridge2_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge2: bridge@d9007000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9007000 0x0 0x1000>;
+			cpus = <&cpu_2>;
+			out-ports {
+				port {
+					cluster0_bridge2_out_port: endpoint {
+						remote-endpoint = <&cluster0_funnel_in_port2>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster0_bridge2_in_port: endpoint {
+						remote-endpoint = <&cluster0_encoder2_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder3: encoder@d9008000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9008000 0x0 0x1000>;
+			cpus = <&cpu_3>;
+			out-ports {
+				port {
+					cluster0_encoder3_out_port: endpoint {
+						remote-endpoint = <&cluster0_bridge3_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge3: bridge@d9009000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9009000 0x0 0x1000>;
+			cpus = <&cpu_3>;
+			out-ports {
+				port {
+					cluster0_bridge3_out_port: endpoint {
+						remote-endpoint = <&cluster0_funnel_in_port3>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster0_bridge3_in_port: endpoint {
+						remote-endpoint = <&cluster0_encoder3_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder4: encoder@d9012000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9012000 0x0 0x1000>;
+			cpus = <&cpu_4>;
+			out-ports {
+				port {
+					cluster1_encoder0_out_port: endpoint {
+						remote-endpoint = <&cluster1_bridge0_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge4: bridge@d9013000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9013000 0x0 0x1000>;
+			cpus = <&cpu_4>;
+			out-ports {
+				port {
+					cluster1_bridge0_out_port: endpoint {
+						remote-endpoint = <&cluster1_funnel_in_port0>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster1_bridge0_in_port: endpoint {
+						remote-endpoint = <&cluster1_encoder0_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder5: encoder@d9014000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9014000 0x0 0x1000>;
+			cpus = <&cpu_5>;
+			out-ports {
+				port {
+					cluster1_encoder1_out_port: endpoint {
+						remote-endpoint = <&cluster1_bridge1_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge5: bridge@d9015000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9015000 0x0 0x1000>;
+			cpus = <&cpu_5>;
+			out-ports {
+				port {
+					cluster1_bridge1_out_port: endpoint {
+						remote-endpoint = <&cluster1_funnel_in_port1>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster1_bridge1_in_port: endpoint {
+						remote-endpoint = <&cluster1_encoder1_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder6: encoder@d9016000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9016000 0x0 0x1000>;
+			cpus = <&cpu_6>;
+			out-ports {
+				port {
+					cluster1_encoder2_out_port: endpoint {
+						remote-endpoint = <&cluster1_bridge2_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge6: bridge@d9017000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9017000 0x0 0x1000>;
+			cpus = <&cpu_6>;
+			out-ports {
+				port {
+					cluster1_bridge2_out_port: endpoint {
+						remote-endpoint = <&cluster1_funnel_in_port2>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster1_bridge2_in_port: endpoint {
+						remote-endpoint = <&cluster1_encoder2_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder7: encoder@d9018000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9018000 0x0 0x1000>;
+			cpus = <&cpu_7>;
+			out-ports {
+				port {
+					cluster1_encoder3_out_port: endpoint {
+						remote-endpoint = <&cluster1_bridge3_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge7: bridge@d9019000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9019000 0x0 0x1000>;
+			cpus = <&cpu_7>;
+			out-ports {
+				port {
+					cluster1_bridge3_out_port: endpoint {
+						remote-endpoint = <&cluster1_funnel_in_port3>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster1_bridge3_in_port: endpoint {
+						remote-endpoint = <&cluster1_encoder3_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder8: encoder@d9022000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9022000 0x0 0x1000>;
+			cpus = <&cpu_8>;
+			out-ports {
+				port {
+					cluster2_encoder0_out_port: endpoint {
+						remote-endpoint = <&cluster2_bridge0_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge8: bridge@d9023000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9023000 0x0 0x1000>;
+			cpus = <&cpu_8>;
+			out-ports {
+				port {
+					cluster2_bridge0_out_port: endpoint {
+						remote-endpoint = <&cluster2_funnel_in_port0>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster2_bridge0_in_port: endpoint {
+						remote-endpoint = <&cluster2_encoder0_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder9: encoder@d9024000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9024000 0x0 0x1000>;
+			cpus = <&cpu_9>;
+			out-ports {
+				port {
+					cluster2_encoder1_out_port: endpoint {
+						remote-endpoint = <&cluster2_bridge1_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge9: bridge@d9025000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9025000 0x0 0x1000>;
+			cpus = <&cpu_9>;
+			out-ports {
+				port {
+					cluster2_bridge1_out_port: endpoint {
+						remote-endpoint = <&cluster2_funnel_in_port1>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster2_bridge1_in_port: endpoint {
+						remote-endpoint = <&cluster2_encoder1_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder10: encoder@d9026000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9026000 0x0 0x1000>;
+			cpus = <&cpu_10>;
+			out-ports {
+				port {
+					cluster2_encoder2_out_port: endpoint {
+						remote-endpoint = <&cluster2_bridge2_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge10: bridge@d9027000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9027000 0x0 0x1000>;
+			cpus = <&cpu_10>;
+			out-ports {
+				port {
+					cluster2_bridge2_out_port: endpoint {
+						remote-endpoint = <&cluster2_funnel_in_port2>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster2_bridge2_in_port: endpoint {
+						remote-endpoint = <&cluster2_encoder2_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder11: encoder@d9028000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9028000 0x0 0x1000>;
+			cpus = <&cpu_11>;
+			out-ports {
+				port {
+					cluster2_encoder3_out_port: endpoint {
+						remote-endpoint = <&cluster2_bridge3_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge11: bridge@d9029000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9029000 0x0 0x1000>;
+			cpus = <&cpu_11>;
+			out-ports {
+				port {
+					cluster2_bridge3_out_port: endpoint {
+						remote-endpoint = <&cluster2_funnel_in_port3>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster2_bridge3_in_port: endpoint {
+						remote-endpoint = <&cluster2_encoder3_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder12: encoder@d9032000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9032000 0x0 0x1000>;
+			cpus = <&cpu_12>;
+			out-ports {
+				port {
+					cluster3_encoder0_out_port: endpoint {
+						remote-endpoint = <&cluster3_bridge0_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge12: bridge@d9033000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9033000 0x0 0x1000>;
+			cpus = <&cpu_12>;
+			out-ports {
+				port {
+					cluster3_bridge0_out_port: endpoint {
+						remote-endpoint = <&cluster3_funnel_in_port0>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster3_bridge0_in_port: endpoint {
+						remote-endpoint = <&cluster3_encoder0_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder13: encoder@d9034000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9034000 0x0 0x1000>;
+			cpus = <&cpu_13>;
+			out-ports {
+				port {
+					cluster3_encoder1_out_port: endpoint {
+						remote-endpoint = <&cluster3_bridge1_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge13: bridge@d9035000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9035000 0x0 0x1000>;
+			cpus = <&cpu_13>;
+			out-ports {
+				port {
+					cluster3_bridge1_out_port: endpoint {
+						remote-endpoint = <&cluster3_funnel_in_port1>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster3_bridge1_in_port: endpoint {
+						remote-endpoint = <&cluster3_encoder1_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder14: encoder@d9036000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9036000 0x0 0x1000>;
+			cpus = <&cpu_14>;
+			out-ports {
+				port {
+					cluster3_encoder2_out_port: endpoint {
+						remote-endpoint = <&cluster3_bridge2_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge14: bridge@d9037000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9037000 0x0 0x1000>;
+			cpus = <&cpu_14>;
+			out-ports {
+				port {
+					cluster3_bridge2_out_port: endpoint {
+						remote-endpoint = <&cluster3_funnel_in_port2>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster3_bridge2_in_port: endpoint {
+						remote-endpoint = <&cluster3_encoder2_out_port>;
+					};
+				};
+			};
+		};
+
+		encoder15: encoder@d9038000 {
+			compatible = "riscv,trace-encoder";
+			reg = <0x0 0xd9038000 0x0 0x1000>;
+			cpus = <&cpu_15>;
+			out-ports {
+				port {
+					cluster3_encoder3_out_port: endpoint {
+						remote-endpoint = <&cluster3_bridge3_in_port>;
+					};
+				};
+			};
+		};
+
+		bridge15: bridge@d9039000 {
+			compatible = "riscv,trace-atbbridge";
+			reg = <0x0 0xd9039000 0x0 0x1000>;
+			cpus = <&cpu_15>;
+			out-ports {
+				port {
+					cluster3_bridge3_out_port: endpoint {
+						remote-endpoint = <&cluster3_funnel_in_port3>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					cluster3_bridge3_in_port: endpoint {
+						remote-endpoint = <&cluster3_encoder3_out_port>;
+					};
+				};
+			};
+		};
+
+		cluster0_funnel: funnel@d9000000 {
+			compatible = "riscv,trace-funnel";
+			reg = <0x0 0xd9000000 0x0 0x1000>;
+			cpus = <&cpu_0>, <&cpu_1>, <&cpu_2>, <&cpu_3>;
+			riscv,timestamp-present;
+			out-ports {
+				port {
+					cluster0_funnel_out_port: endpoint {
+						remote-endpoint = <&main_funnel_in_port0>;
+					};
+				};
+			};
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					cluster0_funnel_in_port0: endpoint {
+						remote-endpoint = <&cluster0_bridge0_out_port>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					cluster0_funnel_in_port1: endpoint {
+						remote-endpoint = <&cluster0_bridge1_out_port>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+					cluster0_funnel_in_port2: endpoint {
+						remote-endpoint = <&cluster0_bridge2_out_port>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+					cluster0_funnel_in_port3: endpoint {
+						remote-endpoint = <&cluster0_bridge3_out_port>;
+					};
+				};
+			};
+		};
+
+		cluster1_funnel: funnel@d9010000 {
+			compatible = "riscv,trace-funnel";
+			reg = <0x0 0xd9010000 0x0 0x1000>;
+			cpus = <&cpu_4>, <&cpu_5>, <&cpu_6>, <&cpu_7>;
+			riscv,timestamp-present;
+			out-ports {
+				port {
+					cluster1_funnel_out_port: endpoint {
+						remote-endpoint = <&main_funnel_in_port1>;
+					};
+				};
+			};
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					cluster1_funnel_in_port0: endpoint {
+						remote-endpoint = <&cluster1_bridge0_out_port>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					cluster1_funnel_in_port1: endpoint {
+						remote-endpoint = <&cluster1_bridge1_out_port>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+					cluster1_funnel_in_port2: endpoint {
+						remote-endpoint = <&cluster1_bridge2_out_port>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+					cluster1_funnel_in_port3: endpoint {
+						remote-endpoint = <&cluster1_bridge3_out_port>;
+					};
+				};
+			};
+		};
+
+		cluster2_funnel: funnel@d9020000 {
+			compatible = "riscv,trace-funnel";
+			reg = <0x0 0xd9020000 0x0 0x1000>;
+			cpus = <&cpu_8>, <&cpu_9>, <&cpu_10>, <&cpu_11>;
+			riscv,timestamp-present;
+			out-ports {
+				port {
+					cluster2_funnel_out_port: endpoint {
+						remote-endpoint = <&main_funnel_in_port2>;
+					};
+				};
+			};
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					cluster2_funnel_in_port0: endpoint {
+						remote-endpoint = <&cluster2_bridge0_out_port>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					cluster2_funnel_in_port1: endpoint {
+						remote-endpoint = <&cluster2_bridge1_out_port>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+					cluster2_funnel_in_port2: endpoint {
+						remote-endpoint = <&cluster2_bridge2_out_port>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+					cluster2_funnel_in_port3: endpoint {
+						remote-endpoint = <&cluster2_bridge3_out_port>;
+					};
+				};
+			};
+		};
+
+		cluster3_funnel: funnel@d9030000 {
+			compatible = "riscv,trace-funnel";
+			reg = <0x0 0xd9030000 0x0 0x1000>;
+			cpus = <&cpu_12>, <&cpu_13>, <&cpu_14>, <&cpu_15>;
+			riscv,timestamp-present;
+			out-ports {
+				port {
+					cluster3_funnel_out_port: endpoint {
+						remote-endpoint = <&main_funnel_in_port3>;
+					};
+				};
+			};
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					cluster3_funnel_in_port0: endpoint {
+						remote-endpoint = <&cluster3_bridge0_out_port>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					cluster3_funnel_in_port1: endpoint {
+						remote-endpoint = <&cluster3_bridge1_out_port>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+					cluster3_funnel_in_port2: endpoint {
+						remote-endpoint = <&cluster3_bridge2_out_port>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+					cluster3_funnel_in_port3: endpoint {
+						remote-endpoint = <&cluster3_bridge3_out_port>;
+					};
+				};
+			};
+		};
+
+		main_funnel: funnel@d9042000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0xd9042000 0x0 0x1000>;
+			clocks = <&dummy_clk>;
+			clock-names = "apb_pclk";
+			out-ports {
+				port {
+					main_funnel_out_port: endpoint {
+						remote-endpoint = <&etf_in_port>;
+					};
+				};
+			};
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					main_funnel_in_port0: endpoint {
+						remote-endpoint = <&cluster0_funnel_out_port>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					main_funnel_in_port1: endpoint {
+						remote-endpoint = <&cluster1_funnel_out_port>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+					main_funnel_in_port2: endpoint {
+						remote-endpoint = <&cluster2_funnel_out_port>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+					main_funnel_in_port3: endpoint {
+						remote-endpoint = <&cluster3_funnel_out_port>;
+					};
+				};
+			};
+		};
+
+		etf: etf@d9043000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0xd9043000 0x0 0x1000>;
+			clocks = <&dummy_clk>;
+			clock-names = "apb_pclk";
+			out-ports {
+				port {
+					etf_out_port: endpoint {
+						remote-endpoint = <&etr_in_port>;
+					};
+				};
+			};
+			in-ports {
+				port {
+					etf_in_port: endpoint {
+						remote-endpoint = <&main_funnel_out_port>;
+					};
+				};
+			};
+		};
+
+		etr: etr@d9044000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0xd9044000 0x0 0x1000>;
+			clocks = <&dummy_clk>;
+			clock-names = "apb_pclk";
+			arm,scatter-gather;
+			in-ports {
+				port {
+					etr_in_port: endpoint {
+						remote-endpoint = <&etf_out_port>;
+					};
+				};
+			};
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi
new file mode 100644
index 000000000000..a9dad319bca5
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3.dtsi
@@ -0,0 +1,2681 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+/dts-v1/;
+
+/* Inline SoC GPIO helper: expand a linear GPIO to <bank,offset> */
+#define K3_GPIO(x) ((x) / 32) ((x) % 32)
+#include "k3-cpus.dtsi"
+#include "k3-trace.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/clock/spacemit,k3-syscon.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pmu/k3_pmu.h>
+#include <dt-bindings/dma/k3-dmac.h>
+#include <dt-bindings/leds/common.h>
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		serial0 = &uart0;
+		serial1 = &uart1;
+		serial2 = &uart2;
+		serial3 = &uart3;
+		serial4 = &uart4;
+		serial5 = &uart5;
+		serial6 = &uart6;
+		serial7 = &uart7;
+		serial8 = &uart8;
+		serial9 = &uart9;
+		serial10 = &uart10;
+		serial11 = &r_uart0;
+		serial12 = &r_uart1;
+		serial13 = &r_uart2;
+		serial14 = &r_uart3;
+		serial15 = &r_uart4;
+		serial16 = &r_uart5;
+		mmc0 = &sdcard;
+		mmc1 = &sdio;
+		mmc2 = &emmc;
+		ethernet0 = &eth0;
+		ethernet1 = &eth1;
+		ethernet2 = &eth2;
+		ethernet3 = &eth3;
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
+		i2c8 = &i2c8;
+		spi0 = &spi0;
+		spi1 = &spi1;
+		spi2 = &spi2;
+		spi3 = &spi3;
+	};
+
+	clocks {
+		vctcxo_1m: clock-1m {
+			compatible = "fixed-clock";
+			clock-frequency = <1000000>;
+			clock-output-names = "vctcxo_1m";
+			#clock-cells = <0>;
+		};
+
+		vctcxo_24m: clock-24m {
+			compatible = "fixed-clock";
+			clock-frequency = <24000000>;
+			clock-output-names = "vctcxo_24m";
+			#clock-cells = <0>;
+		};
+
+		vctcxo_3m: clock-3m {
+			compatible = "fixed-clock";
+			clock-frequency = <3000000>;
+			clock-output-names = "vctcxo_3m";
+			#clock-cells = <0>;
+		};
+
+		osc_32k: clock-32k {
+			compatible = "fixed-clock";
+			clock-frequency = <32768>;
+			clock-output-names = "osc_32k";
+			#clock-cells = <0>;
+		};
+
+		reserved_clk: clock-reserved {
+			compatible = "fixed-clock";
+			clock-frequency = <0>;
+			clock-output-names = "reserved_clk";
+			#clock-cells = <0>;
+		};
+
+		external_clk: clock-external {
+			compatible = "fixed-clock";
+			clock-frequency = <0>;
+			clock-output-names = "external_clk";
+			#clock-cells = <0>;
+		};
+	};
+
+	memory@102000000 {
+		device_type = "memory";
+		reg = <0x1 0x02000000 0x1 0xfe000000>;
+	};
+
+	dummy_clk: apb-pclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0x0>;
+		clock-output-names = "clk14mhz";
+		clock-frequency = <14000000>;
+	};
+
+	resmem: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		/* used for rcpu0 code & data & bss space */
+		rcpu0_pwr: rcpu0_pwr@100200000 {
+			reg = <0x1 0x200000 0x0 0x300000>;
+			no-map;
+		};
+
+		/* rcpu0's heap */
+		rcpu0_heap: rcpu0_heap@100500000 {
+			reg = <0x1 0x500000 0x0 0x200000>;
+			no-map;
+		};
+
+		/* vring0 */
+		rcpu0_vdev0vring0: vdev0vring0@c0876000 {
+			reg = <0x0 0xc0876000 0x0 0x3000>;
+			no-map;
+		};
+
+		/* vring1 */
+		rcpu0_vdev0vring1: vdev0vring1@c0879000 {
+			reg = <0x0 0xc0879000 0x0 0x3000>;
+			no-map;
+		};
+
+		/* vring2 */
+		rcpu0_vdev0vring2: vdev0vring2@c087c000 {
+			reg = <0x0 0xc087c000 0x0 0x1000>;
+			no-map;
+		};
+
+		/* the rcpu0's share memory buffer */
+		rcpu0_vdev0buffer: vdev0buffer@100700000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x700000 0x0 0xfc000>;
+			no-map;
+		};
+
+		/* the rcpu0's resource table */
+		rcpu0_rsc_table: rcpu0_rsc_table@1007fc000 {
+			reg = <0x1 0x7fc000 0x0 0x4000>;
+			no-map;
+		};
+
+		/*****************rcpu1**********************/
+		/* used for rcp1 code & data & bss space */
+		rcpu1_runtime: rcpu1_runtime@100800000 {
+			reg = <0x1 0x800000 0x0 0x300000>;
+			no-map;
+		};
+
+		/* rcpu1's heap */
+		rcpu1_heap: rcpu1_heap@100B00000 {
+			reg = <0x1 0xb00000 0x0 0x200000>;
+			no-map;
+		};
+
+		/* vring0 */
+		rcpu1_vdev0vring0: vdev0vring0@c086c000 {
+			reg = <0x0 0xc086c000 0x0 0x3000>;
+			no-map;
+		};
+
+		/* vring1 */
+		rcpu1_vdev0vring1: vdev0vring1@c086f000 {
+			reg = <0x0 0xc086f000 0x0 0x3000>;
+			no-map;
+		};
+
+		/* the rcpu1's share memory buffer */
+		rcpu1_vdev0buffer: vdev0buffer@100d00000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0xd00000 0x0 0xfc000>;
+			no-map;
+		};
+
+		/* the rcpu1's resource table */
+		rcpu1_rsc_table: rcpu1_rsc_table@100dfc000 {
+			reg = <0x1 0xdfc000 0x0 0x4000>;
+			no-map;
+		};
+
+		/* rpmi */
+		rpmi0: rpmi0@100e00000 {
+			reg = <0x1 0xe00000 0x0 0x4000>;
+			no-map;
+		};
+
+		/* reseved rpmi for feture use */
+
+		/* reserve 3M for rcpu's dtb space */
+		rcpu_dtb_table: rcpu_dtb_table@100f00000 {
+			reg = <0x1 0xf00000 0x0 0x300000>;
+			no-map;
+		};
+
+		/* reserved 5M for dpu, including mmu table and cmdlist */
+		dpu_resv0: dpu_reserved@101b00000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x01b00000 0x0 0x00280000>;
+			no-map;
+		};
+
+		dpu_resv1: dpu_reserved1@101d80000 {
+			compatible = "shared-dma-pool";
+			reg = <0x1 0x01d80000 0x0 0x00280000>;
+			no-map;
+		};
+	};
+
+	soc: soc {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		compatible = "simple-bus";
+		dma-noncoherent;
+		ranges;
+
+		clint0: clint@e081c000 {
+			compatible = "riscv,clint0";
+			interrupts-extended = <
+				&cpu0_intc  3 &cpu0_intc  7
+				&cpu1_intc  3 &cpu1_intc  7
+				&cpu2_intc  3 &cpu2_intc  7
+				&cpu3_intc  3 &cpu3_intc  7
+				&cpu4_intc  3 &cpu4_intc  7
+				&cpu5_intc  3 &cpu5_intc  7
+				&cpu6_intc  3 &cpu6_intc  7
+				&cpu7_intc  3 &cpu7_intc  7
+				&cpu8_intc  3 &cpu8_intc  7
+				&cpu9_intc  3 &cpu9_intc  7
+				&cpu10_intc  3 &cpu10_intc  7
+				&cpu11_intc  3 &cpu11_intc  7
+				&cpu12_intc  3 &cpu12_intc  7
+				&cpu13_intc  3 &cpu13_intc  7
+				&cpu14_intc  3 &cpu14_intc  7
+				&cpu15_intc  3 &cpu15_intc  7
+			>;
+			reg = <0x0 0xe081c000 0x0 0x0004000>;
+		};
+
+		simsic: interrupt-controller@e0400000 {
+			compatible = "riscv,imsics";
+			interrupts-extended = <
+				&cpu0_intc 9 &cpu1_intc 9
+				&cpu2_intc 9 &cpu3_intc 9
+				&cpu4_intc 9 &cpu5_intc 9
+				&cpu6_intc 9 &cpu7_intc 9
+				&cpu8_intc 9 &cpu9_intc 9
+				&cpu10_intc 9 &cpu11_intc 9
+				&cpu12_intc 9 &cpu13_intc 9
+				&cpu14_intc 9 &cpu15_intc 9>;
+
+			reg = <0x0 0xe0400000 0x0 0x400000>;
+			interrupt-controller;
+			#interrupt-cells = <0>;
+			#msi-cells = <0>;
+			msi-controller;
+			/* Number of interrupt identities: 511 */
+			riscv,num-ids = <511>;
+			/* Number of guest interrupt identities: 63 */
+			riscv,num-guest-ids = <63>;
+			/* Number of HART index bits: 4 (2^4 = 16 harts) */
+			riscv,hart-index-bits = <4>;
+			/*
+			 * Number of guest index bits: 6
+			 *    2^6 = 64 interrupt files
+			 *        = 1 s-level interrupt file + 63 vs-level interrupt files
+			 *
+			 * But on K3,
+			 *   X100: only 7 of the 63 vs-level interrupt files are valid.
+			 *   A100: no valid vs-level interrupt files.
+			 */
+			riscv,guest-index-bits = <6>;
+			status = "okay";
+		};
+
+		saplic: interrupt-controller@e0804000 {
+			compatible = "riscv,aplic";
+			/* aplic config */
+			reg = <0x0 0xe0804000 0x0 0x4000>;
+			msi-parent = <&simsic>;
+			#interrupt-cells = <2>;
+			interrupt-controller;
+			/* the number of wired interrupt sources:512 */
+			riscv,num-sources = <512>;
+			status = "okay";
+		};
+
+		syscon_mpmu: system-controller@d4050000 {
+			compatible = "spacemit,k3-syscon-mpmu";
+			reg = <0x0 0xd4050000 0x0 0x10000>;
+			clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>,
+				 <&vctcxo_24m>, <&reserved_clk>, <&external_clk>;
+			clock-names = "osc_32k", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m",
+				      "reserved_clk", "external_clk";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		pll: clock-controller@d4090000 {
+			compatible = "spacemit,k3-pll";
+			reg = <0x0 0xd4090000 0x0 0x10000>;
+			clocks = <&vctcxo_24m>;
+			spacemit,mpmu = <&syscon_mpmu>;
+			#clock-cells = <1>;
+		};
+
+		syscon_apmu: system-controller@d4282800 {
+			compatible = "spacemit,k3-syscon-apmu";
+			reg = <0x0 0xd4282800 0x0 0x400>;
+			clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>,
+				 <&reserved_clk>, <&external_clk>;
+			clock-names = "osc_32k", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m",
+				      "reserved_clk", "external_clk";
+			#clock-cells = <1>;
+			#power-domain-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_apbc: system-controller@d4015000 {
+			compatible = "spacemit,k3-syscon-apbc";
+			reg = <0x0 0xd4015000 0x0 0x1000>;
+			clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>,
+				 <&reserved_clk>, <&external_clk>;
+			clock-names = "osc_32k", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m",
+				      "reserved_clk", "external_clk";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_apbc2: system-controller@f0610000 {
+			compatible = "spacemit,k3-syscon-apbc2";
+			reg = <0x0 0xf0610000 0x0 0x2000>;
+			clocks = <&osc_32k>, <&vctcxo_1m>, <&vctcxo_3m>, <&vctcxo_24m>,
+				 <&reserved_clk>, <&external_clk>;
+			clock-names = "osc_32k", "vctcxo_1m", "vctcxo_3m", "vctcxo_24m",
+				      "reserved_clk", "external_clk";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_dciu: system-controller@d8440000 {
+			compatible = "spacemit,k3-syscon-dciu";
+			reg = <0x0 0xd8440000 0x0 0xc000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_rcpu_sysctrl: system-controller@c0880000 {
+			compatible = "spacemit,k3-syscon-rcpu-sysctrl";
+			reg = <0x0 0xc0880000 0x0 0x1000>;
+			clocks = <&vctcxo_24m>, <&external_clk>;
+			clock-names = "vctcxo_24m", "external_clk";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_rcpu_uartctrl: system-controller@c0881f00 {
+			compatible = "spacemit,k3-syscon-rcpu-uartctrl";
+			reg = <0x0 0xc0881f00 0x0 0x100>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_rcpu_i2sctrl: system-controller@c0882000 {
+			compatible = "spacemit,k3-syscon-rcpu-i2sctrl";
+			reg = <0x0 0xc0882000 0x0 0x1000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_rcpu_spictrl: system-controller@c0885f00 {
+			compatible = "spacemit,k3-syscon-rcpu-spictrl";
+			reg = <0x0 0xc0885f00 0x0 0x100>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_rcpu_i2cctrl: system-controller@c0886f00 {
+			compatible = "spacemit,k3-syscon-rcpu-i2cctrl";
+			reg = <0x0 0xc0886f00 0x0 0x100>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_rpmu: system-controller@c088c000 {
+			compatible = "spacemit,k3-syscon-rpmu";
+			reg = <0x0 0xc088c000 0x0 0x800>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_rcpu_pwmctrl: system-controller@c088d000 {
+			compatible = "spacemit,k3-syscon-rcpu-pwmctrl";
+			reg = <0x0 0xc088d000 0x0 0x100>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		syscon_gpio: syscon-gpio{
+			reg = <0x0 0xd4019000 0x0 0x800>;
+		};
+
+		syscon_gpio_edge: syscon-gpio-edge{
+			compatible = "syscon";
+			reg = <0x0 0xd4019800 0x0 0x10>;
+		};
+
+		gpio: gpio@d4019000 {
+			compatible = "spacemit,k3-gpio";
+			reg = <0x0 0xd4019000 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_GPIO>,
+				 <&syscon_apbc CLK_APBC_GPIO_BUS>;
+			clock-names = "core", "bus";
+			gpio-controller;
+			#gpio-cells = <3>;
+			interrupts = <58 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+			syscon-gpio-regs = <&syscon_gpio>;
+			syscon-gpio-edge = <&syscon_gpio_edge>;
+			gpio-ranges = <&pinctrl 0 0 0 32>,
+				      <&pinctrl 1 0 32 32>,
+				      <&pinctrl 2 0 64 32>,
+				      <&pinctrl 3 0 96 32>;
+		};
+
+		pinctrl: pinctrl@d401e000 {
+			compatible = "spacemit,k3-pinctrl";
+			reg = <0x0 0xd401e000 0x0 0x400>,
+			      <0x0 0xd401e800 0x0 0x34>;
+			clocks = <&syscon_apbc CLK_APBC_AIB>,
+                                <&syscon_apbc CLK_APBC_AIB_BUS>;
+			clock-names = "func", "bus";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupts = <60 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			spacemit,gpio-edge = <&syscon_gpio_edge>;
+			spacemit,apbc = <&syscon_apbc 0x50>;
+			spacemit,apmu = <&syscon_apmu>;
+		};
+
+		pdma: pdma@d4000000 {
+			compatible = "spacemit,k1-pdma";
+			reg = <0x0 0xd4000000 0x0 0x4000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <72 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apmu CLK_APMU_DMA>;
+			resets = <&syscon_apmu RESET_APMU_DMA>;
+			#dma-cells= <1>;
+			#dma-channels = <16>;
+			max-burst-size = <64>;
+			status = "okay";
+		};
+
+		pdma1: pdma@f0600000 {
+			compatible = "spacemit,k1-pdma";
+			reg = <0x0 0xf0600000 0x0 0x4000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <150 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells= <1>;
+			#dma-channels = <16>;
+			max-burst-size = <64>;
+			status = "disabled";
+		};
+
+		udma: udma {
+			compatible = "spacemit,udma";
+			status = "okay";
+		};
+
+		uart0: serial@d4017000 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017000 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART0>,
+				 <&syscon_apbc CLK_APBC_UART0_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART0>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <42 IRQ_TYPE_LEVEL_HIGH>;
+			status = "okay";
+		};
+
+		uart1: serial@f0612000 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xf0612000 0x0 0x100>;
+			clocks = <&syscon_apbc2 CLK_APBC2_SEC_UART1>,
+				 <&syscon_apbc2 CLK_APBC2_SEC_UART1_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc2 RESET_APBC2_SEC_UART1>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <43 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart2: serial@d4017100 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017100 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART2>,
+				 <&syscon_apbc CLK_APBC_UART2_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART2>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <44 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart3: serial@d4017200 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017200 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART3>,
+				 <&syscon_apbc CLK_APBC_UART3_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART3>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <45 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart4: serial@d4017300 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017300 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART4>,
+				 <&syscon_apbc CLK_APBC_UART4_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART4>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <46 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart5: serial@d4017400 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017400 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART5>,
+				 <&syscon_apbc CLK_APBC_UART5_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART5>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <47 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart6: serial@d4017500 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017500 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART6>,
+				 <&syscon_apbc CLK_APBC_UART6_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART6>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <48 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart7: serial@d4017600 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017600 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART7>,
+				 <&syscon_apbc CLK_APBC_UART7_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART7>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <49 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart8: serial@d4017700 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017700 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART8>,
+				 <&syscon_apbc CLK_APBC_UART8_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART8>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <50 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart9: serial@d4017800 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd4017800 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART9>,
+				 <&syscon_apbc CLK_APBC_UART9_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART9>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <51 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart10: serial@d401f000 {
+			compatible = "spacemit,k1-uart",
+				     "intel,xscale-uart";
+			reg = <0x0 0xd401f000 0x0 0x100>;
+			clocks = <&syscon_apbc CLK_APBC_UART10>,
+				 <&syscon_apbc CLK_APBC_UART10_BUS>,
+				 <&syscon_mpmu CLK_MPMU_SLOW_UART>;
+			clock-names = "core", "bus", "gate";
+			resets = <&syscon_apbc RESET_APBC_UART10>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			fifo-size = <256>;
+			tx-threshold = <32>;
+			interrupt-parent = <&saplic>;
+			interrupts = <281 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		i2c0: i2c@d4010800 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xd4010800 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <36 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_TWSI0>,
+				 <&syscon_apbc CLK_APBC_TWSI0_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc RESET_APBC_TWSI0>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		i2c1: i2c@d4011000 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xd4011000 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <37 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_TWSI1>,
+				 <&syscon_apbc CLK_APBC_TWSI1_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc RESET_APBC_TWSI1>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@d4012000 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xd4012000 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <38 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_TWSI2>,
+				 <&syscon_apbc CLK_APBC_TWSI2_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc RESET_APBC_TWSI2>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@f0614000 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xf0614000 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <39 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc2 CLK_APBC2_SEC_TWSI3>,
+				 <&syscon_apbc2 CLK_APBC2_SEC_TWSI3_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc2 RESET_APBC2_SEC_TWSI3>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@d4012800 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xd4012800 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <40 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_TWSI4>,
+				 <&syscon_apbc CLK_APBC_TWSI4_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc RESET_APBC_TWSI4>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		i2c5: i2c@d4013800 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xd4013800 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <41 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_TWSI5>,
+				 <&syscon_apbc CLK_APBC_TWSI5_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc RESET_APBC_TWSI5>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		i2c6: i2c@d4018800 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xd4018800 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <70 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_TWSI6>,
+				 <&syscon_apbc CLK_APBC_TWSI6_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc RESET_APBC_TWSI6>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		i2c8: i2c@d401d800 {
+			compatible = "spacemit,k1-i2c";
+			reg = <0x0 0xd401d800 0x0 0x38>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <19 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_TWSI8>,
+				 <&syscon_apbc CLK_APBC_TWSI8_BUS>;
+			clock-names = "func", "bus";
+			clock-frequency = <400000>;
+			resets = <&syscon_apbc RESET_APBC_TWSI8>;
+			power-domains = <&power K3_PMU_DUMMY_PWR_DOMAIN>;
+			status = "disabled";
+		};
+
+		pwm0: pwm@d401a000 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401a000 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM0>,<&syscon_apbc CLK_APBC_PWM0_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM0>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm1: pwm@d401a400 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401a400 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM1>,<&syscon_apbc CLK_APBC_PWM1_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM1>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm2: pwm@d401a800 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401a800 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM2>,<&syscon_apbc CLK_APBC_PWM2_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM2>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm3: pwm@d401ac00 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401ac00 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM3>,<&syscon_apbc CLK_APBC_PWM3_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM3>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm4: pwm@d401b000 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401b000 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM4>,<&syscon_apbc CLK_APBC_PWM4_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM4>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm5: pwm@d401b400 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401b400 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM5>,<&syscon_apbc CLK_APBC_PWM5_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM5>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm6: pwm@d401b800 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401b800 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM6>,<&syscon_apbc CLK_APBC_PWM6_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM6>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm7: pwm@d401bc00 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd401bc00 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM7>,<&syscon_apbc CLK_APBC_PWM7_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM7>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm8: pwm@d4020000 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4020000 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM8>,<&syscon_apbc CLK_APBC_PWM8_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM8>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm9: pwm@d4020400 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4020400 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM9>,<&syscon_apbc CLK_APBC_PWM9_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM9>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm10: pwm@d4020800 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4020800 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM10>,<&syscon_apbc CLK_APBC_PWM10_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM10>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm11: pwm@d4020c00 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4020c00 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM11>,<&syscon_apbc CLK_APBC_PWM11_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM11>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm12: pwm@d4021000 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4021000 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM12>,<&syscon_apbc CLK_APBC_PWM12_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM12>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm13: pwm@d4021400 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4021400 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM13>,<&syscon_apbc CLK_APBC_PWM13_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM13>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm14: pwm@d4021800 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4021800 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM14>,<&syscon_apbc CLK_APBC_PWM14_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM14>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm15: pwm@d4021c00 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4021c00 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM15>,<&syscon_apbc CLK_APBC_PWM15_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM15>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm16: pwm@d4022000 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4022000 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM16>,<&syscon_apbc CLK_APBC_PWM16_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM16>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm17: pwm@d4022400 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4022400 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM17>,<&syscon_apbc CLK_APBC_PWM17_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM17>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm18: pwm@d4022800 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4022800 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM18>,<&syscon_apbc CLK_APBC_PWM18_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM18>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		pwm19: pwm@d4022c00 {
+			compatible = "spacemit,k1-pwm", "marvell,pxa910-pwm";
+			reg = <0x0 0xd4022c00 0x0 0x10>;
+			clocks = <&syscon_apbc CLK_APBC_PWM19>,<&syscon_apbc CLK_APBC_PWM19_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_PWM19>;
+			#pwm-cells = <3>;
+			k1,pwm-disable-fd;
+			status = "disabled";
+		};
+
+		spi0: spi@d4040000 {
+			compatible = "spacemit,k3-spi";
+			reg = <0x0 0xd4040000 0x0 0x30>;
+			dmas = <&pdma DMA_SSP0_TX
+				&pdma DMA_SSP0_RX>;
+			dma-names = "tx", "rx";
+			interrupt-parent = <&saplic>;
+			interrupts = <283 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc  CLK_APBC_SPI0>,
+				<&syscon_apbc CLK_APBC_SPI0_BUS>;
+			clock-names = "core", "bus";
+			resets = <&syscon_apbc RESET_APBC_SPI0>;
+			spi-max-frequency = <51200000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		spi1: spi@d4040800 {
+			compatible = "spacemit,k3-spi";
+			reg = <0x0 0xd4040800 0x0 0x30>;
+			dmas = <&pdma DMA_SSP1_TX
+				&pdma DMA_SSP1_RX>;
+			dma-names = "tx", "rx";
+			interrupt-parent = <&saplic>;
+			interrupts = <282 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc  CLK_APBC_SPI1>,
+				<&syscon_apbc CLK_APBC_SPI1_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_SPI1>;
+			spi-max-frequency = <51200000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		spi2: spi@f0613000 {
+			compatible = "spacemit,k3-spi";
+			reg = <0x0 0xf0613000 0x0 0x30>;
+			// dmas = <&pdma 19
+			// 	&pdma 20>;
+			// dma-names = "rx", "tx";
+			interrupt-parent = <&saplic>;
+			interrupts = <54 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc2 CLK_APBC2_SEC_SPI2>,
+				 <&syscon_apbc2 CLK_APBC2_SEC_SPI2_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc2 RESET_APBC2_SEC_SPI2>;
+			spi-max-frequency = <51200000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		spi3: spi@d401c000 {
+			compatible = "spacemit,k3-spi";
+			reg = <0x0 0xd401c000 0x0 0x30>;
+			dmas = <&pdma DMA_SSP3_TX
+				&pdma DMA_SSP3_RX>;
+			dma-names = "tx", "rx";
+			interrupt-parent = <&saplic>;
+			interrupts = <55 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apbc CLK_APBC_SPI3>,
+				<&syscon_apbc CLK_APBC_SPI3_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_SPI3>;
+			spi-max-frequency = <51200000>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		qspi: spi@d420c000 {
+			compatible = "spacemit,k3-qspi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0xd420c000 0x0 0x1000>,
+			      <0x0 0xb8000000 0x0 0xc00000>;
+			reg-names = "QuadSPI", "QuadSPI-memory";
+			clocks = <&syscon_apmu CLK_APMU_QSPI>,
+				 <&syscon_apmu CLK_APMU_QSPI_BUS>;
+			clock-names = "qspi", "qspi_en";
+			resets = <&syscon_apmu RESET_APMU_QSPI>,
+				 <&syscon_apmu RESET_APMU_QSPI_BUS>;
+			reset-names = "qspi_reset", "qspi_bus_reset";
+			interrupts = <117 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			status = "disabled";
+		};
+
+		v2d: v2d@c0100000 {
+			compatible = "spacemit,v2d";
+			reg =  <0x0 0xc0100000 0x0 0x1000>;
+			reg-names = "v2dreg";
+			clocks = <&syscon_apmu CLK_APMU_LCD_MCLK>,
+				 <&syscon_apmu CLK_APMU_V2D>;
+			clock-names = "v2d-io", "v2d-core";
+			resets = <&syscon_apmu RESET_APMU_V2D>;
+			reset-names = "v2d_reset";
+			interrupt-parent = <&saplic>;
+			interrupts = <86 IRQ_TYPE_LEVEL_HIGH>;
+			status = "okay";
+		};
+
+		mailbox0: mailbox0@cac90000 {
+			compatible = "spacemit,k3-mailbox";
+			reg = <0x0 0xcac90000 0x0 0x400>;
+			interrupt-parent = <&saplic>;
+			interrupts = <210 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			status = "okay";
+		};
+
+		mailbox1: mailbox1@cac90400 {
+			compatible = "spacemit,k3-mailbox";
+			reg = <0x0 0xcac90400 0x0 0x400>;
+			interrupt-parent = <&saplic>;
+			interrupts = <212 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			status = "okay";
+		};
+
+		mailbox2: mailbox2@cac90800 {
+			compatible = "spacemit,k3-mailbox";
+			reg = <0x0 0xcac90800 0x0 0x400>;
+			interrupt-parent = <&saplic>;
+			interrupts = <214 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			status = "okay";
+		};
+
+		mailbox3: mailbox3@cac90c00 {
+			compatible = "spacemit,k3-mailbox";
+			reg = <0x0 0xcac90c00 0x0 0x400>;
+			interrupt-parent = <&saplic>;
+			interrupts = <216 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			status = "okay";
+		};
+
+		mailbox4: mailbox4@cac91000 {
+			compatible = "spacemit,k3-mailbox";
+			reg = <0x0 0xcac91000 0x0 0x400>;
+			interrupt-parent = <&saplic>;
+			interrupts = <217 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			status = "okay";
+		};
+
+		mailbox5: mailbox5@cac91400 {
+			compatible = "spacemit,k3-mailbox";
+			reg = <0x0 0xcac91400 0x0 0x400>;
+			interrupt-parent = <&saplic>;
+			interrupts = <226 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			status = "okay";
+		};
+
+		mailbox6: mailbox6@cac91800 {
+			compatible = "spacemit,k3-mailbox";
+			reg = <0x0 0xcac91800 0x0 0x400>;
+			interrupt-parent = <&saplic>;
+			interrupts = <227 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+			status = "okay";
+		};
+
+		rcpu0: rcpu_rproc0@0 {
+			compatible = "spacemit,k3-rproc";
+			reg =  <0 0 0 0>;
+			mboxes = <&mailbox3 1>, <&mailbox3 2>;
+			mbox-names = "vq0", "vq1";
+			firmware-name = "rt24_os0_rcpu.elf";
+			memory-region = <&rcpu0_pwr>, <&rcpu0_heap>, <&rcpu0_vdev0vring0>, <&rcpu0_vdev0buffer>,
+					<&rcpu0_vdev0vring1>, <&rcpu0_rsc_table>;
+			status = "disabled";
+		};
+
+		flexcan0: fdcan@d4028000 {
+			compatible = "spacemit,k1-flexcan";
+			reg = <0x0 0xd4028000 0x0 0x4000>;
+			interrupts = <161 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			fsl,clk-source = <0>;
+			clocks = <&syscon_apbc CLK_APBC_CAN0>,<&syscon_apbc CLK_APBC_CAN0_BUS>;
+			clock-names = "per","ipg";
+			resets = <&syscon_apbc RESET_APBC_CAN0>;
+			status = "disabled";
+		};
+
+		flexcan1: fdcan@d402c000 {
+			compatible = "spacemit,k1-flexcan";
+			reg = <0x0 0xd402c000 0x0 0x4000>;
+			interrupts = <163 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			fsl,clk-source = <0>;
+			clocks = <&syscon_apbc CLK_APBC_CAN1>,<&syscon_apbc CLK_APBC_CAN1_BUS>;
+			clock-names = "per","ipg";
+			resets = <&syscon_apbc RESET_APBC_CAN1>;
+			status = "disabled";
+		};
+
+		flexcan2: fdcan@d4034000 {
+			compatible = "spacemit,k1-flexcan";
+			reg = <0x0 0xd4034000 0x0 0x4000>;
+			interrupts = <165 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			fsl,clk-source = <0>;
+			clocks = <&syscon_apbc CLK_APBC_CAN2>,<&syscon_apbc CLK_APBC_CAN2_BUS>;
+			clock-names = "per","ipg";
+			resets = <&syscon_apbc RESET_APBC_CAN2>;
+			status = "disabled";
+		};
+
+		flexcan3: fdcan@d4038000 {
+			compatible = "spacemit,k1-flexcan";
+			reg = <0x0 0xd4038000 0x0 0x4000>;
+			interrupts = <167 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			fsl,clk-source = <0>;
+			clocks = <&syscon_apbc CLK_APBC_CAN3>,<&syscon_apbc CLK_APBC_CAN3_BUS>;
+			clock-names = "per","ipg";
+			resets = <&syscon_apbc RESET_APBC_CAN3>;
+			status = "disabled";
+		};
+
+		flexcan4: fdcan@d403c000 {
+			compatible = "spacemit,k1-flexcan";
+			reg = <0x0 0xd403c000 0x0 0x4000>;
+			interrupts = <169 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			fsl,clk-source = <0>;
+			clocks = <&syscon_apbc CLK_APBC_CAN4>,<&syscon_apbc CLK_APBC_CAN4_BUS>;
+			clock-names = "per","ipg";
+			resets = <&syscon_apbc RESET_APBC_CAN4>;
+			status = "disabled";
+		};
+
+		rcpu1: rcpu_rproc1@1 {
+			compatible = "spacemit,k3-rproc";
+			reg =  <0 0 0 0>;
+			mboxes = <&mailbox4 0>, <&mailbox4 1>;
+			mbox-names = "vq0", "vq1";
+			firmware-name = "rt24_os1_rcpu.elf";
+			memory-region = <&rcpu1_runtime>, <&rcpu1_heap>, <&rcpu1_vdev0vring0>, <&rcpu1_vdev0buffer>,
+					<&rcpu1_vdev0vring1>, <&rcpu1_rsc_table>;
+			status = "okay";
+		};
+
+		mpxy_mbox: mpxy_mbox@0 {
+			compatible = "riscv,sbi-mpxy-mbox";
+			#mbox-cells = <2>;
+			status = "okay";
+		};
+
+		rpmi_regulator: rpmi_regulator@0 {
+			compatible = "riscv,rpmi-regulator";
+			mboxes = <&mpxy_mbox 0x0007 0x0>;
+			status = "okay";
+		};
+
+		rpmi_rtc: rpmi_rtc@0 {
+			compatible = "riscv,rpmi-rtc";
+			mboxes = <&mpxy_mbox 0xe 0x0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <64 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "rpmi rtc";
+			status = "okay";
+		};
+
+		rpmi_pwrkey: rpmi_pwrkey@0 {
+			compatible = "riscv,rpmi-pwrkey";
+			mboxes = <&mpxy_mbox 0xf 0x0>;
+			interrupt-parent = <&saplic>;
+			interrupts = <64 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "rpmi pwrkey";
+			status = "okay";
+		};
+
+		pwrkey_monitor: pwrkey_monitor@1101 {
+			compatible = "spacemit,k3-pwrkey-monitor";
+			status = "disabled";
+		};
+
+		pmu: power-management@0 {
+			compatible = "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			power: power-controller {
+				compatible = "spacemit,power-controller";
+				mboxes = <&mpxy_mbox 0x9 0x0>;
+				#power-domain-cells = <1>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "okay";
+
+				vpu: vpu@0 {
+					#power-domain-cells = <0>;
+				};
+
+				gpu: gpu@1 {
+					#power-domain-cells = <0>;
+				};
+
+				audio: audio@2 {
+					#power-domain-cells = <0>;
+				};
+
+				lcd0: lcd@0 {
+					#power-domain-cells = <0>;
+				};
+
+				lcd1: lcd@1 {
+					#power-domain-cells = <0>;
+				};
+
+				dummy: dummy@0 {
+					#power-domain-cells = <0>;
+				};
+			};
+		};
+
+		thermal: thermal@d4018000 {
+			compatible = "spacemit,k3-tsensor";
+			reg = <0x0 0xd4018000 0x0 0x100>;
+			interrupt-parent = <&saplic>;
+			interrupts = <61 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "tsensor";
+			clocks = <&syscon_apbc CLK_APBC_TSEN>,
+				<&syscon_apbc CLK_APBC_TSEN_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_TSEN>;
+			sensor_range = <0x0 0x7>;
+			tsensor_map = <1 0 1 1 1 1 1 1>;
+			temperature_offset = <274>;
+			#thermal-sensor-cells = <1>;
+			status = "okay";
+		};
+
+		thermal_zones: thermal-zones {
+
+		};
+
+		timer0: timer@d4016000 {
+			compatible = "spacemit,soc-timer";
+			reg = <0x0 0xd4016000 0x0 0xc8>;
+			spacemit,timer-id = <0>;
+			spacemit,timer-fastclk-frequency = <1000000>;
+			spacemit,timer-apb-frequency = <102400000>;
+			spacemit,timer-frequency = <1000000>;
+			clocks = <&syscon_apbc CLK_APBC_TIMERS1>,
+				<&syscon_apbc CLK_APBC_TIMERS1_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_TIMERS1>;
+			status = "okay";
+
+			counter0 {
+				compatible = "spacemit,timer-match";
+				interrupts = <26 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-parent = <&saplic>;
+				spacemit,timer-broadcast;
+				spacemit,timer-counter-id = <0>;
+				status = "okay";
+			};
+		};
+
+		rtc: rtc@d4010000 {
+			compatible = "mrvl,mmp-rtc";
+			reg = <0x0 0xd4010000 0x0 0x100>;
+			interrupt-parent = <&saplic>;
+			interrupts = <21 IRQ_TYPE_LEVEL_HIGH>, <22 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "rtc 1Hz", "rtc alarm";
+			clocks = <&syscon_apbc CLK_APBC_RTC>,
+				 <&syscon_apbc CLK_APBC_RTC_BUS>;
+			clock-names = "func", "bus";
+			resets = <&syscon_apbc RESET_APBC_RTC>;
+			status = "okay";
+		};
+
+		ircrx0: irc-rx0@d4017e00 {
+			compatible = "spacemit-k1,irc";
+			reg = <0x0 0xd4017e00 0x0 0x100>;
+			interrupts = <69 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			clocks = <&syscon_apbc CLK_APBC_IR0>;
+			resets = <&syscon_apbc RESET_APBC_IR0>;
+			clock-frequency = <102400000>;
+			status = "disabled";
+		};
+
+		ircrx1: irc-rx1@d4017f00 {
+			compatible = "spacemit-k1,irc";
+			reg = <0x0 0xd4017f00 0x0 0x100>;
+			interrupts = <20 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			clocks = <&syscon_apbc CLK_APBC_IR1>;
+			resets = <&syscon_apbc RESET_APBC_IR1>;
+			clock-frequency = <102400000>;
+			status = "disabled";
+		};
+
+		tcm: tcm@00000000 {
+			compatible = "spacemit,k1-tcm";
+			reg = <0x0 0 0x0 0x4000000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0 0x4000000>;
+			no-memory-wc;
+			/* the first range 0x00000~0x180000 */
+			cluster0_tcm@0 {
+				reg = <0x00000 0x60000>;
+				cpus = <&cpu_8>, <&cpu_9>;
+				pool;
+			};
+			cluster0_tcm@60000 {
+				reg = <0x60000 0x60000>;
+				cpus = <&cpu_8>, <&cpu_9>;
+				pool;
+			};
+			cluster0_tcm@c0000 {
+				reg = <0xc0000 0x60000>;
+				cpus = <&cpu_10>, <&cpu_11>;
+				pool;
+			};
+			cluster0_tcm@120000 {
+				reg = <0x120000 0x60000>;
+				cpus = <&cpu_10>, <&cpu_11>;
+				pool;
+			};
+
+			/* the second range 0x2000000~0x2180000 */
+			cluster1_tcm@2000000 {
+				reg = <0x2000000 0x60000>;
+				cpus = <&cpu_12>, <&cpu_13>;
+				pool;
+			};
+			cluster1_tcm@2060000 {
+				reg = <0x2060000 0x60000>;
+				cpus = <&cpu_12>, <&cpu_13>;
+				pool;
+			};
+			cluster1_tcm@20c0000 {
+				reg = <0x20c0000 0x60000>;
+				cpus = <&cpu_14>, <&cpu_15>;
+				pool;
+			};
+			cluster1_tcm@2120000 {
+				reg = <0x2120000 0x60000>;
+				cpus = <&cpu_14>, <&cpu_15>;
+				pool;
+			};
+		};
+
+		usb2_host_u2phy: phy@c0a20000 {
+			compatible = "spacemit,k3-usb2-phy";
+			reg = <0x0 0xc0a20000 0x0 0x200>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb2_host: usb2@c0a00000 {
+			compatible = "spacemit,k1-dwc3";
+			reg = <0x0 0xc0a00000 0x0 0x10000>;
+			spacemit,syscon-apmu = <&syscon_apmu 0x7c>;
+			interrupts = <105 IRQ_TYPE_LEVEL_HIGH>,
+				     <155 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc3", "wakeup";
+			interrupt-parent = <&saplic>;
+			clocks = <&syscon_apmu CLK_APMU_USB2_BUS>;
+			resets = <&syscon_apmu RESET_APMU_USB2>;
+			phys = <&usb2_host_u2phy>;
+			phy-names = "usb2-phy";
+			phy_type = "utmi";
+			snps,dis_enblslpm_quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,dis-tx-ipgap-linecheck-quirk;
+			dr_mode = "host";
+			maximum-speed = "high-speed";
+			status = "disabled";
+		};
+
+		usb3_porta_u2phy: phy@cad20000 {
+			compatible = "spacemit,k3-usb2-phy";
+			reg = <0x0 0xcad20000 0x0 0x200>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_porta_u3phy: phy@cad30000 {
+			compatible = "spacemit,k3-usb3-phy",
+				     "spacemit,k3-typec-switch";
+			spacemit,syscon-apmu = <&syscon_apmu>;
+			spacemit,syscon-apb-spare = <&pll>;
+			clocks = <&syscon_apmu CLK_APMU_USB3_PORTA_BUS>;
+			reg = <0x0 0xcad30000 0x0 0x200>,
+			      <0x0 0xcad40000 0x0 0x200>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_porta: usb3@cad00000 {
+			compatible = "spacemit,k1-dwc3";
+			reg = <0x0 0xcad00000 0x0 0x10000>;
+			spacemit,syscon-apmu = <&syscon_apmu 0x3c4>;
+			interrupts = <118 IRQ_TYPE_LEVEL_HIGH>,
+				     <156 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc3", "wakeup";
+			interrupt-parent = <&saplic>;
+			clocks = <&syscon_apmu CLK_APMU_USB3_PORTA_BUS>;
+			resets = <&syscon_apmu RESET_APMU_USB3_PORTA>;
+			phys = <&usb3_porta_u3phy>, <&usb3_porta_u2phy>;
+			phy-names = "usb3-phy", "usb2-phy";
+			phy_type = "utmi";
+			snps,dis_enblslpm_quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_u3_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,dis-tx-ipgap-linecheck-quirk;
+			snps,parkmode-disable-ss-quirk;
+			status = "disabled";
+		};
+
+		usb3_portb_u2phy: phy@81500000 {
+			compatible = "spacemit,k3-usb2-phy";
+			reg = <0x0 0x81500000 0x0 0x200>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_portb_u3phy: phy@81f00000{
+			compatible = "spacemit,k3-usb3-phy";
+			spacemit,syscon-apmu = <&syscon_apmu>;
+			spacemit,syscon-apb-spare = <&pll>;
+			reg = <0x0 0x81f00000 0x0 0x200>;
+			combo-usb-bit = <2>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_portb: usb3@81400000 {
+			compatible = "spacemit,k1-dwc3";
+			reg = <0x0 0x81400000 0x0 0x10000>;
+			spacemit,syscon-apmu = <&syscon_apmu 0x3c8>;
+			interrupts = <125 IRQ_TYPE_LEVEL_HIGH>,
+				     <157 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc3", "wakeup";
+			interrupt-parent = <&saplic>;
+			clocks = <&syscon_apmu CLK_APMU_USB3_PORTB_BUS>;
+			resets = <&syscon_apmu RESET_APMU_USB3_PORTB>;
+			phys = <&usb3_portb_u2phy>, <&usb3_portb_u3phy>;
+			phy-names = "usb2-phy", "usb3-phy";
+			phy_type = "utmi";
+			snps,dis_enblslpm_quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_u3_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,dis-tx-ipgap-linecheck-quirk;
+			snps,parkmode-disable-ss-quirk;
+			dr_mode = "host";
+			status = "disabled";
+		};
+
+		usb3_portc_u2phy: phy@81800000 {
+			compatible = "spacemit,k3-usb2-phy";
+			reg = <0x0 0x81800000 0x0 0x200>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_portc_u3phy: phy@82000000{
+			compatible = "spacemit,k3-usb3-phy";
+			spacemit,syscon-apmu = <&syscon_apmu>;
+			spacemit,syscon-apb-spare = <&pll>;
+			reg = <0x0 0x82000000 0x0 0x200>;
+			combo-usb-bit = <1>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_portc: usb3@81700000 {
+			compatible = "spacemit,k1-dwc3";
+			reg = <0x0 0x81700000 0x0 0x10000>;
+			spacemit,syscon-apmu = <&syscon_apmu 0x3cc>;
+			interrupts = <148 IRQ_TYPE_LEVEL_HIGH>,
+				     <158 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc3", "wakeup";
+			interrupt-parent = <&saplic>;
+			clocks = <&syscon_apmu CLK_APMU_USB3_PORTC_BUS>;
+			resets = <&syscon_apmu RESET_APMU_USB3_PORTC>;
+			phys = <&usb3_portc_u2phy>, <&usb3_portc_u3phy>;
+			phy-names = "usb2-phy", "usb3-phy";
+			phy_type = "utmi";
+			snps,dis_enblslpm_quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_u3_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,dis-tx-ipgap-linecheck-quirk;
+			snps,parkmode-disable-ss-quirk;
+			dr_mode = "host";
+			status = "disabled";
+		};
+
+		usb3_portd_u2phy: phy@81b00000 {
+			compatible = "spacemit,k3-usb2-phy";
+			reg = <0x0 0x81b00000 0x0 0x200>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_portd_u3phy: phy@82100000{
+			compatible = "spacemit,k3-usb3-phy";
+			spacemit,syscon-apmu = <&syscon_apmu>;
+			spacemit,syscon-apb-spare = <&pll>;
+			reg = <0x0 0x82100000 0x0 0x200>;
+			combo-usb-bit = <0>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		usb3_portd: usb3@81a00000 {
+			compatible = "spacemit,k1-dwc3";
+			reg = <0x0 0x81a00000 0x0 0x10000>;
+			spacemit,syscon-apmu = <&syscon_apmu 0x3d0>;
+			interrupts = <149 IRQ_TYPE_LEVEL_HIGH>,
+				     <159 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc3", "wakeup";
+			interrupt-parent = <&saplic>;
+			clocks = <&syscon_apmu CLK_APMU_USB3_PORTD_BUS>;
+			resets = <&syscon_apmu RESET_APMU_USB3_PORTD>;
+			phys = <&usb3_portd_u2phy>, <&usb3_portd_u3phy>;
+			phy-names = "usb2-phy", "usb3-phy";
+			phy_type = "utmi";
+			snps,dis_enblslpm_quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_u3_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,dis-tx-ipgap-linecheck-quirk;
+			snps,parkmode-disable-ss-quirk;
+			dr_mode = "host";
+			status = "disabled";
+		};
+
+		gmac_axi_setup: stmmac-axi-config {
+			snps,wr_osr_lmt = <0xf>;
+			snps,rd_osr_lmt = <0xf>;
+			/* max axi burst len is 256 */
+			snps,blen = <256 128 64 32 16 0 0>;
+		};
+
+		imggpu: imggpu@cac00000 {
+			compatible = "img,rgx";
+			interrupt-names = "rgxirq";
+			interrupt-parent = <&saplic>;
+			interrupts = <75 IRQ_TYPE_LEVEL_HIGH>;
+			reg = <0x0 0xcac00000 0x0 0x80000>;
+			reg-names = "rgxregs";
+			power-domains = <&power K3_PMU_GPU_PWR_DOMAIN>;
+			clocks = <&syscon_apmu CLK_APMU_GPU>;
+			clock-names = "clk_rgx";
+			resets = <&syscon_apmu RESET_APMU_GPU>;
+			operating-points-v2 = <&imggpu_opp_table>;
+			#cooling-cells = <2>;
+			thermal-zone = "thermal_gpu";
+			status = "okay";
+
+			imggpu_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-409000000 {
+					opp-hz = /bits/ 64 <409000000>;
+					opp-microvolt = <800000>;
+					clock-latency-ns = <100000>;
+				};
+
+				opp-491000000 {
+					opp-hz = /bits/ 64 <491000000>;
+					opp-microvolt = <800000>;
+					clock-latency-ns = <100000>;
+				};
+
+				opp-614000000 {
+					opp-hz = /bits/ 64 <614000000>;
+					opp-microvolt = <900000>;
+					clock-latency-ns = <100000>;
+				};
+
+				opp-819000000 {
+					opp-hz = /bits/ 64 <819000000>;
+					opp-microvolt = <900000>;
+					clock-latency-ns = <100000>;
+				};
+
+				opp-1000000000 {
+					opp-hz = /bits/ 64 <1000000000>;
+					opp-microvolt = <950000>;
+					clock-latency-ns = <100000>;
+				};
+
+				opp-1228000000 {
+					opp-hz = /bits/ 64 <1228000000>;
+					opp-microvolt = <950000>;
+					clock-latency-ns = <100000>;
+				};
+			};
+		};
+
+                ec_master: ethercat_master {
+                        compatible = "spacemit,igh-ec-master";
+                        master-count = <1>;
+                        status = "disabled";
+
+                        master0 {
+                                main-device = <&eth0>;
+                        };
+		};
+
+		gmac_default_rxq_setup: rx-queues-config {
+			snps,rx-queues-to-use = <2>;
+			snps,rx-sched-sp;
+
+			queue0 {
+				snps,dcb-algorithm;
+				snps,priority = <0x0f>;
+				snps,map-to-dma-channel = <0>;
+			};
+
+			queue1 {
+				snps,dcb-algorithm;
+				snps,priority = <0xf0>;
+				snps,map-to-dma-channel = <1>;
+			};
+		};
+
+		gmac_default_txq_setup: tx-queues-config {
+			snps,tx-queues-to-use = <2>;
+
+			queue0 {
+				snps,dcb-algorithm;
+			};
+
+			queue1 {
+				snps,dcb-algorithm;
+			};
+		};
+
+		eth0: ethernet@cac80000 {
+			compatible = "spacemit,k3-gmac", "snps,dwmac-5.10a";
+			reg = <0x0 0xcac80000 0x0 0x2000>;
+			spacemit,apmu = <&syscon_apmu>;
+			spacemit,ctrl-offset = <0x3e4>;
+			spacemit,dline-offset = <0x3e8>;
+			clocks = <&syscon_apmu CLK_APMU_EMAC0_BUS>,
+				 <&syscon_apmu CLK_APMU_EMAC0_1588>;
+			clock-names = "stmmaceth", "ptp_ref";
+			resets = <&syscon_apmu RESET_APMU_EMAC0>;
+			reset-names = "stmmaceth";
+			interrupt-parent = <&saplic>;
+			interrupts = <131 IRQ_TYPE_LEVEL_HIGH>, <276 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq", "eth_wake_irq";
+			mac-address = [00 00 00 00 00 00];
+			tx-fifo-depth = <8192>;
+			rx-fifo-depth = <8192>;
+			snps,tso;
+			snps,force_sf_dma_mode;
+			snps,axi-config = <&gmac_axi_setup>;
+			snps,mtl-rx-config = <&gmac_default_rxq_setup>;
+			snps,mtl-tx-config = <&gmac_default_txq_setup>;
+			status = "disabled";
+		};
+
+		eth1: ethernet@cac82000 {
+			compatible = "spacemit,k3-gmac", "snps,dwmac-5.10a";
+			reg = <0x0 0xcac82000 0x0 0x2000>;
+			spacemit,apmu = <&syscon_apmu>;
+			spacemit,ctrl-offset = <0x3ec>;
+			spacemit,dline-offset = <0x3f0>;
+			clocks = <&syscon_apmu CLK_APMU_EMAC1_BUS>,
+				 <&syscon_apmu CLK_APMU_EMAC1_1588>;
+			clock-names = "stmmaceth", "ptp_ref";
+			resets = <&syscon_apmu RESET_APMU_EMAC1>;
+			reset-names = "stmmaceth";
+			interrupt-parent = <&saplic>;
+			interrupts = <133 IRQ_TYPE_LEVEL_HIGH>, <277 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq", "eth_wake_irq";
+			mac-address = [00 00 00 00 00 00];
+			tx-fifo-depth = <8192>;
+			rx-fifo-depth = <8192>;
+			snps,tso;
+			snps,force_sf_dma_mode;
+			snps,axi-config = <&gmac_axi_setup>;
+			snps,mtl-rx-config = <&gmac_default_rxq_setup>;
+			snps,mtl-tx-config = <&gmac_default_txq_setup>;
+			status = "disabled";
+		};
+
+		eth2: ethernet@cac8e000 {
+			compatible = "spacemit,k3-gmac", "snps,dwmac-5.10a";
+			reg = <0x0 0xcac8e000 0x0 0x2000>;
+			spacemit,apmu = <&syscon_apmu>;
+			spacemit,ctrl-offset = <0x248>;
+			spacemit,dline-offset = <0x24c>;
+			clocks = <&syscon_apmu CLK_APMU_EMAC2_BUS>,
+				 <&syscon_apmu CLK_APMU_EMAC2_1588>;
+			clock-names = "stmmaceth", "ptp_ref";
+			resets = <&syscon_apmu RESET_APMU_EMAC2>;
+			reset-names = "stmmaceth";
+			interrupt-parent = <&saplic>;
+			interrupts = <130 IRQ_TYPE_LEVEL_HIGH>, <278 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq", "eth_wake_irq";
+			mac-address = [00 00 00 00 00 00];
+			tx-fifo-depth = <8192>;
+			rx-fifo-depth = <8192>;
+			snps,tso;
+			snps,force_sf_dma_mode;
+			snps,axi-config = <&gmac_axi_setup>;
+			snps,mtl-rx-config = <&gmac_default_rxq_setup>;
+			snps,mtl-tx-config = <&gmac_default_txq_setup>;
+			status = "disabled";
+		};
+
+		eth3: ethernet@a0000000 {
+			compatible = "spacemit,k3-gmac", "snps,dwmac-5.10a";
+			reg = <0x0 0xa0000000 0x0 0x2000>;
+			spacemit,apmu = <&syscon_rcpu_sysctrl>;
+			spacemit,ctrl-offset = <0x0e4>;
+			spacemit,dline-offset = <0x0e8>;
+			clocks = <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_REMAC_BUS>,
+				 <&syscon_rcpu_sysctrl CLK_RCPU_SYSCTRL_REMAC_1588>;
+			clock-names = "stmmaceth", "ptp_ref";
+			resets = <&syscon_rcpu_sysctrl RESET_RCPU_SYSCTRL_RGMAC>;
+			reset-names = "stmmaceth";
+			interrupt-parent = <&saplic>;
+			interrupts = <237 IRQ_TYPE_LEVEL_HIGH>, <236 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq", "eth_wake_irq";
+			mac-address = [00 00 00 00 00 00];
+			tx-fifo-depth = <4096>;
+			rx-fifo-depth = <4096>;
+			snps,tso;
+			snps,force_sf_dma_mode;
+			snps,axi-config = <&gmac_axi_setup>;
+			status = "disabled";
+		};
+
+		sdcard: mmc@d4280000 {
+			compatible = "spacemit,k3-sdhci";
+			reg = <0x0 0xd4280000 0x0 0x200>;
+			clocks = <&syscon_apmu CLK_APMU_SDH_AXI>,
+				 <&syscon_apmu CLK_APMU_SDH0>;
+			clock-names = "core", "io";
+			interrupt-parent = <&saplic>;
+			interrupts = <99 IRQ_TYPE_LEVEL_HIGH>;
+			resets = <&syscon_apmu RESET_APMU_SDH_AXI>,
+				 <&syscon_apmu RESET_APMU_SDH0>;
+			reset-names = "sdh_axi", "sdh0";
+			status = "disabled";
+		};
+
+		sdio: mmc@d4280800 {
+			compatible = "spacemit,k3-sdhci";
+			reg = <0x0 0xd4280800 0x0 0x200>;
+			clocks = <&syscon_apmu CLK_APMU_SDH_AXI>,
+				 <&syscon_apmu CLK_APMU_SDH1>;
+			clock-names = "core", "io";
+			interrupt-parent = <&saplic>;
+			interrupts = <100 IRQ_TYPE_LEVEL_HIGH>;
+			resets = <&syscon_apmu RESET_APMU_SDH_AXI>,
+				 <&syscon_apmu RESET_APMU_SDH1>;
+			reset-names = "sdh_axi", "sdh1";
+			status = "disabled";
+		};
+
+		emmc: mmc@d4281000 {
+			compatible = "spacemit,k3-sdhci";
+			reg = <0x0 0xd4281000 0x0 0x200>;
+			clocks = <&syscon_apmu CLK_APMU_SDH_AXI>,
+				 <&syscon_apmu CLK_APMU_SDH2>;
+			clock-names = "core", "io";
+			interrupt-parent = <&saplic>;
+			interrupts = <101 IRQ_TYPE_LEVEL_HIGH>;
+			resets = <&syscon_apmu RESET_APMU_SDH_AXI>,
+				 <&syscon_apmu RESET_APMU_SDH2>;
+			reset-names = "sdh_axi", "sdh2";
+			status = "disabled";
+		};
+
+		i2s0: i2s0@d4026000 {
+			compatible = "spacemit,k1-i2s";
+			#sound-dai-cells = <0>;
+			reg = <0x0 0xD4026000 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S0_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S0_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S0_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S0_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S0>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk_div", "sysclk",
+				      "bclk", "sspa_bus", "sspa", "c_sysclk", "c_bclk";
+			resets = <&syscon_apbc RESET_APBC_I2S0>;
+			assigned-clocks = <&syscon_mpmu CLK_MPMU_I2S_153P6_BASE>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK_SRC>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S_BCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S0_SYSCLK_SEL>;
+			assigned-clock-rates = <307200000>, <307200000>,
+					       <12288000>, <1536000>, <614400000>;
+			/* must set to same value when enable multi i2s: 8000/16000/48000Hz */
+			/* spacemit,fixed-sample-rate = <48000>; */
+			dmas = <&pdma 22>, <&pdma 21>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		i2s1: i2s1@d4026800 {
+			compatible = "spacemit,k1-i2s";
+			#sound-dai-cells = <0>;
+			reg = <0x0 0xD4026800 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S1_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S1_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S1_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S1>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk", "bclk", "sspa_bus", "sspa", "c_sysclk", "c_bclk";
+			resets = <&syscon_apbc RESET_APBC_I2S1>;
+			assigned-clocks = <&syscon_mpmu CLK_MPMU_I2S_153P6_BASE>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK_SRC>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S_BCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S1_SYSCLK_SRC>;
+			assigned-clock-rates = <307200000>, <307200000>,
+					       <12288000>, <1536000>, <307200000>;
+			/* must set to same value when enable multi i2s: 8000/16000/48000Hz */
+			/* spacemit,fixed-sample-rate = <48000>; */
+			dmas = <&pdma 24>, <&pdma 23>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		i2s2: i2s2@d4027000 {
+			compatible = "spacemit,k1-i2s";
+			#sound-dai-cells = <0>;
+			reg = <0x0 0xD4027000 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S2_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S2_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S2_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S2_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S2>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk_div", "sysclk",
+				      "bclk", "sspa_bus", "sspa", "c_sysclk", "c_bclk";
+			resets = <&syscon_apbc RESET_APBC_I2S2>;
+			assigned-clocks = <&syscon_mpmu CLK_MPMU_I2S_153P6_BASE>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK_SRC>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S_BCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S2_SYSCLK_SEL>;
+			assigned-clock-rates = <307200000>, <307200000>,
+					       <12288000>, <1536000>, <614400000>;
+			/* must set to same value when enable multi i2s: 8000/16000/48000Hz */
+			/* spacemit,fixed-sample-rate = <48000>; */
+			dmas = <&pdma 57>, <&pdma 56>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		i2s3: i2s3@d4027800 {
+			compatible = "spacemit,k1-i2s";
+			#sound-dai-cells = <0>;
+			reg = <0x0 0xD4027800 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S3_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S3_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S3_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S3_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S3>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk_div", "sysclk",
+				      "bclk", "sspa_bus", "sspa", "c_sysclk", "c_bclk";
+			resets = <&syscon_apbc RESET_APBC_I2S3>;
+			assigned-clocks = <&syscon_mpmu CLK_MPMU_I2S_153P6_BASE>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK_SRC>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S_BCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S3_SYSCLK_SEL>;
+			assigned-clock-rates = <307200000>, <307200000>,
+					       <12288000>, <1536000>, <614400000>;
+			/* must set to same value when enable multi i2s: 8000/16000/48000Hz */
+			/* spacemit,fixed-sample-rate = <48000>; */
+			dmas = <&pdma 59>, <&pdma 58>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		i2s4: i2s4@d4041000 {
+			compatible = "spacemit,k1-i2s";
+			#sound-dai-cells = <0>;
+			reg = <0x0 0xD4041000 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S4_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S4_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S4_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S4_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S4>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk_div", "sysclk",
+				      "bclk", "sspa_bus", "sspa", "c_sysclk", "c_bclk";
+			resets = <&syscon_apbc RESET_APBC_I2S4>;
+			assigned-clocks = <&syscon_mpmu CLK_MPMU_I2S_153P6_BASE>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK_SRC>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S_BCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S4_SYSCLK_SEL>;
+			assigned-clock-rates = <307200000>, <307200000>,
+					       <12288000>, <1536000>, <614400000>;
+			/* must set to same value when enable multi i2s: 8000/16000/48000Hz */
+			/* spacemit,fixed-sample-rate = <48000>; */
+			dmas = <&pdma 61>, <&pdma 60>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		i2s5: i2s5@d4041800 {
+			compatible = "spacemit,k1-i2s";
+			#sound-dai-cells = <0>;
+			reg = <0x0 0xD4041800 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S5_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S5_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S5_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S5_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S5>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk_div", "sysclk",
+				      "bclk", "sspa_bus", "sspa", "c_sysclk", "c_bclk";
+			resets = <&syscon_apbc RESET_APBC_I2S5>;
+			assigned-clocks = <&syscon_mpmu CLK_MPMU_I2S_153P6_BASE>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK_SRC>,
+					  <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S_BCLK>,
+					  <&syscon_mpmu CLK_MPMU_I2S5_SYSCLK_SEL>;
+			assigned-clock-rates = <307200000>, <307200000>,
+					       <12288000>, <1536000>, <614400000>;
+			/* must set to same value when enable multi i2s: 8000/16000/48000Hz */
+			/* spacemit,fixed-sample-rate = <48000>; */
+			dmas = <&pdma 63>, <&pdma 62>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		sound_card0: sound-card@0 {
+			status = "disabled";
+			compatible = "simple-audio-card";
+		};
+
+		sound_card1: sound-card@1 {
+			status = "disabled";
+			compatible = "simple-audio-card";
+		};
+
+		hwspinlock: hwspin@cac91c00 {
+			compatible = "spacemit,hwspinlock";
+			reg = <0x0 0xcac91c00 0x0 0x100>;
+			status = "disabled";
+		};
+
+		ai_dmac0: hdma0@d8804000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd8804000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <7 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA0>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status  = "okay";
+		};
+
+		ai_dmac1: hdma1@d8805000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd8805000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <8 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA1>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status = "okay";
+		};
+
+		ai_dmac2: hdma2@d8806000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd8806000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <9 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA2>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status = "okay";
+		};
+
+		ai_dmac3: hdma3@d8807000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd8807000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <10 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA3>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status = "okay";
+		};
+
+		ai_dmac4: hdma4@d8808000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd8808000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <11 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA4>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status  = "okay";
+		};
+
+		ai_dmac5: hdma5@d8809000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd8809000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <12 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA5>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status = "okay";
+		};
+
+		ai_dmac6: hdma6@d880a000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd880a000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <13 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA6>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status = "okay";
+		};
+
+		ai_dmac7: hdma7@d880b000 {
+			compatible = "spacemit-ai-dmac";
+			reg = <0x0 0xd880b000 0x0 0x1000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <14 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_dciu CLK_DCIU_HDMA>;
+			resets = <&syscon_dciu RESET_DCIU_HDMA>, <&syscon_dciu RESET_DCIU_AXIDMA7>;
+			reset-names = "bus", "dma";
+			dma-coherent;
+			status = "okay";
+		};
+
+		ai_dmadev: aidma-dev@0 {
+			compatible = "ai-dma-dev";
+			msi-parent = <&simsic>;
+			status = "okay";
+		};
+
+		phy0: phy-pcie@81d00000 {
+			compatible = "spacemit,k3-pcie-phy";
+			reg = <0x0 0x81d00000 0x0 0x1000>;
+			spacemit,syscon-apb-spare = <&pll>;
+			num-lanes = <2>;
+			spacemit,phy-id = <0>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		phy1: phy-pcie@81e00000 {
+			compatible = "spacemit,k3-pcie-phy";
+			reg = <0x0 0x81e00000 0x0 0x1000>;
+			spacemit,syscon-apb-spare = <&pll>;
+			num-lanes = <2>;
+			spacemit,phy-id = <1>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		phy2: phy-pcie@81f00000 {
+			compatible = "spacemit,k3-pcie-phy";
+			reg = <0x0 0x81f00000 0x0 0x1000>;
+			spacemit,syscon-apb-spare = <&pll>;
+			num-lanes = <1>;
+			spacemit,phy-id = <2>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		phy3: phy-pcie@82000000 {
+			compatible = "spacemit,k3-pcie-phy";
+			reg = <0x0 0x82000000 0x0 0x1000>;
+			spacemit,syscon-apb-spare = <&pll>;
+			num-lanes = <1>;
+			spacemit,phy-id = <3>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		phy4: phy-pcie@82100000 {
+			compatible = "spacemit,k3-pcie-phy";
+			reg = <0x0 0x82100000 0x0 0x1000>;
+			spacemit,syscon-apb-spare = <&pll>;
+			num-lanes = <1>;
+			spacemit,phy-id = <4>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		phy5: phy-pcie@82200000 {
+			compatible = "spacemit,k3-pcie-phy";
+			reg = <0x0 0x82200000 0x0 0x1000>;
+			spacemit,syscon-apb-spare = <&pll>;
+			num-lanes = <1>;
+			spacemit,phy-id = <5>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
+		iommu: iommu@c0f00000 {
+			compatible = "spacemit,k3-iommu";
+			reg = <0x0 0xc0f00000 0x0 0x1000>;
+			interrupts = <234 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			#iommu-cells = <0x01>;
+			clocks = <&syscon_apmu CLK_APMU_IOMMU>;
+			status = "disabled";
+		};
+
+		pcie0_rc: pcie@80000000 {
+			compatible = "spacemit,k1-pcie";
+			reg = <0x0 0x80000000 0x0 0x00001000>, /* dbi */
+				<0x0 0x80100000 0x0 0x00001000>, /* dbi2 */
+				<0x0 0x80300000 0x0 0x00003f20>, /* atu registers */
+				<0x11 0x00000000 0x0 0x00010000>, /* config space */
+				<0x0 0x82900000 0x0 0x00001000>; /* phy ahb (link) */
+			reg-names = "dbi", "dbi2", "atu", "config", "link";
+			bus-range = <0x00 0xff>;
+			max-link-speed = <3>;
+			num-lanes = <8>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x00 0x00010000 0x11 0x00010000 0x0 0x00100000>,
+				<0x02000000 0x0 0x00110000 0x11 0x00110000 0x0 0x7fef0000>,
+				<0x43000000 0x18 0x00000000 0x18 0x00000000 0x1 0x00000000>;
+
+			interrupt-parent = <&saplic>;
+			interrupts = <141 IRQ_TYPE_LEVEL_HIGH>, <219 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pcie_irq", "wakeup";
+				clocks = <&syscon_apmu CLK_APMU_PCIE_PORTA_BUS>,
+					<&syscon_apmu CLK_APMU_PCIE_PORTA_MSTR>,
+					<&syscon_apmu CLK_APMU_PCIE_PORTA_SLV>;
+				clock-names = "dbi", "mstr", "slv";
+				resets = <&syscon_apmu RESET_APMU_PCIE_PORTA_DBI>,
+					<&syscon_apmu RESET_APMU_PCIE_PORTA_MSTR>,
+					<&syscon_apmu RESET_APMU_PCIE_PORTA_SLV>;
+				reset-names = "dbi", "mstr", "slv";
+			linux,pci-domain = <0>;
+			msi-parent = <&simsic>;
+			spacemit,apmu = <&syscon_apmu 0x1f0>;
+
+			iommu-map = <0x0 &iommu 0x00000 0x10000>;
+
+			status = "disabled";
+		};
+
+		pcie1_rc: pcie@80400000 {
+			compatible = "spacemit,k1-pcie";
+			reg = <0x0 0x80400000 0x0 0x00001000>, /* dbi */
+				<0x0 0x80500000 0x0 0x00001000>, /* dbi2 */
+				<0x0 0x80700000 0x0 0x00003f20>, /* atu registers */
+				<0x11 0x80000000 0x0 0x00010000>, /* config space */
+				<0x0 0x82c00000 0x0 0x00001000>; /* phy ahb (link) */
+			reg-names = "dbi", "dbi2", "atu", "config", "link";
+
+			bus-range = <0x00 0xff>;
+			max-link-speed = <3>;
+			num-lanes = <2>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00010000 0x11 0x80010000 0x0 0x00100000>,
+				<0x02000000 0x0 0x80110000 0x11 0x80110000 0x0 0x7fef0000>,
+				<0x43000000 0x16 0x00000000 0x16 0x00000000 0x1 0x00000000>;
+
+			interrupt-parent = <&saplic>;
+			interrupts = <142 IRQ_TYPE_LEVEL_HIGH>, <220 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pcie_irq", "wakeup";
+
+			clocks = <&syscon_apmu CLK_APMU_PCIE_PORTB_BUS>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTB_MSTR>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTB_SLV>;
+			clock-names = "dbi", "mstr", "slv";
+			resets = <&syscon_apmu RESET_APMU_PCIE_PORTB_DBI>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTB_MSTR>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTB_SLV>;
+			reset-names = "dbi", "mstr", "slv";
+
+			linux,pci-domain = <1>;
+			msi-parent = <&simsic>;
+			spacemit,pcie-port = <1>;
+			spacemit,apmu = <&syscon_apmu 0x1d0>;
+
+			iommu-map = <0x0 &iommu 0x10000 0x10000>;
+
+			status = "disabled";
+		};
+
+		pcie2_rc: pcie@80800000 {
+			compatible = "spacemit,k1-pcie";
+			reg = <0x0 0x80800000 0x0 0x00001000>, /* dbi */
+				<0x0 0x80900000 0x0 0x00001000>, /* dbi2 */
+				<0x0 0x80b00000 0x0 0x00003f20>, /* atu registers */
+				<0x12 0x00000000 0x0 0x00010000>, /* config space */
+				<0x0 0x82d00000 0x0 0x00001000>; /* phy ahb (link) */
+			reg-names = "dbi", "dbi2", "atu", "config", "link";
+
+			bus-range = <0x00 0xff>;
+			max-link-speed = <3>;
+			num-lanes = <2>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x00 0x00000000 0x12 0x00010000 0x0 0x00100000>,
+				<0x02000000 0x0 0x00110000 0x12 0x00110000 0x0 0x7fef0000>,
+				<0x43000000 0x15 0x00000000 0x15 0x00000000 0x1 0x00000000>;
+
+			interrupt-parent = <&saplic>;
+			interrupts = <143 IRQ_TYPE_LEVEL_HIGH>, <221 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pcie_irq", "wakeup";
+
+			clocks = <&syscon_apmu CLK_APMU_PCIE_PORTC_BUS>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTC_MSTR>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTC_SLV>;
+			clock-names = "dbi", "mstr", "slv";
+			resets = <&syscon_apmu RESET_APMU_PCIE_PORTC_DBI>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTC_MSTR>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTC_SLV>;
+			reset-names = "dbi", "mstr", "slv";
+
+			linux,pci-domain = <2>;
+			msi-parent = <&simsic>;
+			spacemit,pcie-port = <2>;
+			spacemit,apmu = <&syscon_apmu 0x1c8>;
+
+			iommu-map = <0x0 &iommu 0x20000 0x10000>;
+
+			status = "disabled";
+		};
+
+		pcie3_rc: pcie@80c00000 {
+			compatible = "spacemit,k1-pcie";
+			reg = <0x0 0x80c00000 0x0 0x00001000>, /* dbi */
+			      <0x0 0x80d00000 0x0 0x00001000>, /* dbi2 */
+			      <0x0 0x80f00000 0x0 0x00003f20>, /* atu registers */
+			      <0x12 0x80000000 0x0 0x00010000>, /* config space */
+			      <0x0 0x82a00000 0x0 0x00001000>; /* phy ahb (link) */
+			reg-names = "dbi", "dbi2", "atu", "config", "link";
+
+			bus-range = <0x00 0xff>;
+			max-link-speed = <3>;
+			num-lanes = <1>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00010000 0x12 0x80010000 0x0 0x100000>,
+				<0x02000000 0x0 0x80110000 0x12 0x80110000 0x0 0x3fef0000>,
+				<0x43000000 0x14 0x00000000 0x14 0x00000000 0x1 0x00000000>;
+
+			interrupt-parent = <&saplic>;
+			interrupts = <145 IRQ_TYPE_LEVEL_HIGH>, <222 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pcie_irq", "wakeup";
+
+			clocks = <&syscon_apmu CLK_APMU_PCIE_PORTD_BUS>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTD_MSTR>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTD_SLV>;
+			clock-names = "dbi", "mstr", "slv";
+			resets = <&syscon_apmu RESET_APMU_PCIE_PORTD_DBI>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTD_MSTR>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTD_SLV>;
+			reset-names = "dbi", "mstr", "slv";
+
+			linux,pci-domain = <3>;
+			msi-parent = <&simsic>;
+			spacemit,pcie-port = <3>;
+			spacemit,apmu = <&syscon_apmu 0x1e0>;
+			status = "disabled";
+		};
+
+		pcie4_rc: pcie@81000000 {
+			compatible = "spacemit,k1-pcie";
+			reg = <0x0 0x81000000 0x0 0x00001000>, /* dbi */
+			      <0x0 0x81100000 0x0 0x00001000>, /* dbi2 */
+			      <0x0 0x81300000 0x0 0x00003f20>, /* atu registers */
+			      <0x12 0xc0000000 0x0 0x00010000>, /* config space */
+			      <0x0 0x82b00000 0x0 0x00001000>; /* phy ahb (link) */
+			reg-names = "dbi", "dbi2", "atu", "config", "link";
+
+			bus-range = <0x00 0xff>;
+			max-link-speed = <3>;
+			num-lanes = <1>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x12 0xc0010000 0x0 0x100000>,
+				<0x02000000 0x0 0xc0110000 0x12 0xc0110000 0x0 0x3fef0000>,
+				<0x43000000 0x13 0x00000000 0x13 0x00000000 0x1 0x00000000>;
+
+			interrupt-parent = <&saplic>;
+			interrupts = <146 IRQ_TYPE_LEVEL_HIGH>, <223 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pcie_irq", "wakeup";
+
+			clocks = <&syscon_apmu CLK_APMU_PCIE_PORTE_BUS>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTE_MSTR>,
+				<&syscon_apmu CLK_APMU_PCIE_PORTE_SLV>;
+			clock-names = "dbi", "mstr", "slv";
+			resets = <&syscon_apmu RESET_APMU_PCIE_PORTE_DBI>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTE_MSTR>,
+				<&syscon_apmu RESET_APMU_PCIE_PORTE_SLV>;
+			reset-names = "dbi", "mstr", "slv";
+
+			linux,pci-domain = <4>;
+			msi-parent = <&simsic>;
+			spacemit,pcie-port = <4>;
+			spacemit,apmu = <&syscon_apmu 0x1e8>;
+			status = "disabled";
+		};
+
+		pcie0_ep: pcie_ep@80000000 {
+			compatible = "spacemit,k3-pcie-ep";
+			reg = <0x0 0x80000000 0x0 0x00100000>, /* dbi */
+				<0x0 0x80100000 0x0 0x00100000>, /* dbi2 */
+				<0x0 0x80200000 0x0 0x00001000>, /* elbi */
+				<0x0 0x80300000 0x0 0x00001000>, /* atu */
+				<0x0 0x80380000 0x0 0x00001000>, /* dma */
+				<0x11 0x00000000 0x0 0x10000000>, /* addr_space */
+				<0x0 0x82900000 0x0 0x00001000>; /* phy ahb */
+			reg-names = "dbi", "dbi2", "elbi", "atu", "dma", "addr_space", "link";
+
+			interrupt-parent = <&saplic>;
+			interrupts = <141 IRQ_TYPE_LEVEL_HIGH>, <219 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dma", "global";
+
+			clocks = <&syscon_apmu CLK_APMU_PCIE_PORTA_BUS>,
+				 <&syscon_apmu CLK_APMU_PCIE_PORTA_MSTR>,
+				 <&syscon_apmu CLK_APMU_PCIE_PORTA_SLV>;
+			clock-names = "dbi", "mstr", "slv";
+			resets = <&syscon_apmu RESET_APMU_PCIE_PORTA_DBI>,
+				 <&syscon_apmu RESET_APMU_PCIE_PORTA_MSTR>,
+				 <&syscon_apmu RESET_APMU_PCIE_PORTA_SLV>;
+			reset-names = "dbi", "mstr", "slv";
+
+			spacemit,pcie-port = <0>;
+			spacemit,apmu = <&syscon_apmu 0x1f0>;
+
+			msi-parent = <&simsic>;
+			max-link-speed = <3>;
+			num-lanes = <8>;
+			status = "disabled";
+		};
+
+		ufshc: ufshc@0xc0e00000 {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			compatible = "spacemit,k3-ufshcd";
+			reg = <0x0 0xc0e00000 0x0 0x40000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <135 0x4>;
+			clocks = <&syscon_apmu CLK_APMU_UFS_ACLK>,
+				 <&syscon_apmu CLK_APMU_UFS_REFCLK>;
+			clock-names = "ufs-aclk", "ref_clk";
+			resets = <&syscon_apmu RESET_APMU_UFS_ACLK>;
+			reset-names = "ufs-aclk-rst";
+			clock-freq = <491520000>;
+			freq-table-hz = <491520000 491520000 19200000 19200000>;
+			lanes-per-direction = <2>;
+			status = "disabled";
+		};
+
+		espi0: espi@cac8c000 {
+			compatible = "spacemit,k3-espi";
+			reg = <0x0 0xcac8c000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x84000000 0x1000>;
+			pr-mem-base0 = <0x84000000>;
+			pr-mem-base1 = <0xcac8c000>;
+			clocks = <&syscon_apmu CLK_APMU_ESPI_SCLK_SRC>,
+				 <&syscon_apmu CLK_APMU_ESPI_SCLK>,
+				 <&syscon_apmu CLK_APMU_ESPI_MCLK>;
+			clock-names = "sclk_src", "sclk", "mclk";
+			resets = <&syscon_apmu RESET_APMU_ESPI_MCLK>,
+				 <&syscon_apmu RESET_APMU_ESPI_SCLK>;
+			assigned-clocks = <&syscon_apmu CLK_APMU_ESPI_SCLK>;
+			assigned-clock-parents = <&syscon_apmu CLK_APMU_ESPI_SCLK_SRC>;
+			operating-frequency = <20>;
+			io-mode = "single";
+			alert-mode = "io1";
+			alert-type = "open-drain";
+			watchdog-timeout = <0xffff>;
+			crc-enable;
+			auto-gating-enable;
+			peripheral-channel-enable;
+			vwire-channel-enable;
+			oob-channel-enable;
+			flash-channel-enable;
+			status = "disabled";
+
+			ec: ec@0 {
+				compatible = "google,cros-ec-espi";
+				reg = <0x0 0x1000>;
+				status = "disabled";
+			};
+		};
+
+		linlon-v5@c0500000 {
+			compatible = "arm china,linlon-v5";
+			reg = <0x0 0xc0500000 0x0 0x200000>;
+			interrupt-parent = <&saplic>;
+			interrupts = <74 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&syscon_apmu CLK_APMU_VPU>;
+			clock-names = "vpu_clk";
+			power-domains = <&power K3_PMU_VPU_PWR_DOMAIN>;
+			resets = <&syscon_apmu RESET_APMU_VPU>;
+			status = "okay";
+		};
+
+		ccic0: cam_ccic@d420a000 {
+			status = "disabled";
+		};
+
+		ccic1: cam_ccic@d420a800 {
+			status = "disabled";
+		};
+
+		ccic2: cam_ccic@d4206000 {
+			status = "disabled";
+		};
+
+		ccic3: cam_ccic@d4206800 {
+			status = "disabled";
+		};
+
+		ccic_dma: cam_ccic_dma@d420f000 {
+			status = "disabled";
+		};
+
+		watchdog: watchdog@d4014000 {
+			compatible = "spacemit-k1,wdt";
+			clocks = <&syscon_apbc CLK_APBC_TIMERS0>,
+				 <&syscon_apbc CLK_APBC_TIMERS0_BUS>;
+			clock-names = "clk", "clk-bus";
+			resets = <&syscon_apbc RESET_APBC_TIMERS0>;
+			reg = <0x0 0xd4014000 0x0 0xff>,
+			      <0x0 0xd4050000 0x0 0x1024>;
+			interrupts = <35 4>;
+			interrupt-parent = <&saplic>;
+			spa,wdt-disabled;
+			spa,wdt-enable-restart-handler;
+			status = "okay";
+		};
+
+		efuse: fuse@f0702800 {
+			compatible = "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x0 0xf0702800 0x400>;
+			status = "okay";
+
+			efuse_bank7: efuse_bank@7 {
+				compatible = "spacemit,efuse";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				reg = <0x190 0x20>;
+				resets = <&syscon_apmu RESET_APMU_AES_WTM>;
+				reset-names = "aes_reset";
+				clocks = <&syscon_apmu CLK_APMU_AES_WTM>;
+				clock-names = "aes_core";
+				status = "okay";
+
+				soc_die_id: bank7@11,3 {
+					reg = <0x11 3>;
+					bits = <3 16>;
+				};
+
+				soc_ver_id: bank7@1f,6 {
+					reg = <0x1F 1>;
+					bits = <6 2>;
+				};
+
+				soc_pack_id: bank7@16,6 {
+					reg = <0x16 2>;
+					bits = <6 9>;
+				};
+
+				soc_svt_dro: bank7@15,5 {
+					reg = <0x15 2>;
+					bits = <5 9>;
+				};
+
+				soc_chip_id: bank7@17,7 {
+					reg = <0x17 8>;
+					bits = <7 55>;
+				};
+			};
+		};
+
+		socinfo: socinfo@0 {
+			compatible = "spacemit,socinfo";
+			status = "okay";
+
+			nvmem-cells = <&soc_die_id>, <&soc_ver_id>, <&soc_pack_id>,
+				      <&soc_svt_dro>, <&soc_chip_id>;
+			nvmem-cell-names = "soc_die_id", "soc_ver_id", "soc_pack_id",
+					   "soc_svt_dro", "soc_chip_id";
+		};
+
+		reboot_mode_helper: reboot-mode-helper@c087c000 {
+			compatible = "spacemit,k3-reboot";
+			reg = <0x0 0xc087c000 0x0 0x1000>;
+		};
+
+		irq_affinity: irq-aff@0 {
+			compatible = "spacemit,irq-affinity-restore";
+			msi-parent = <&simsic>;
+			status = "okay";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_BS01DCMA.dts b/arch/riscv/boot/dts/spacemit/k3_BS01DCMA.dts
new file mode 100644
index 000000000000..b8cad8f60b85
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_BS01DCMA.dts
@@ -0,0 +1,942 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "k3-edp0.dtsi"
+
+/ {
+	model = "SpacemiT K3 BS01DCMA";
+	compatible = "spacemit,k3-bs01dcma";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p4v: p4v {
+		compatible = "regulator-fixed";
+		regulator-name = "p4v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 clk_ignore_unused loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm18 0 20000 0>;
+		brightness-levels = <
+			0   1   2   3   4   5   6   7
+			8   9  10  11  12  13  14  15
+			16  17  18  19  20  21  22  23
+			24  25  26  27  28  29  30  31
+			32  33  34  35  36  37  38  39
+			40  41  42  43  44  45  46  47
+			48  49  50  51  52  53  54  55
+			56  57  58  59  60  61  62  63
+			64  65  66  67  68  69  70  71
+			72  73  74  75  76  77  78  79
+			80  81  82  83  84  85  86  87
+			88  89  90  91  92  93  94  95
+			96  97  98  99 100 101 102 103
+			104 105 106 107 108 109 110 111
+			112 113 114 115 116 117 118 119
+			120 121 122 123 124 125 126 127
+			128 129 130 131 132 133 134 135
+			136 137 138 139 140 141 142 143
+			144 145 146 147 148 149 150 151
+			152 153 154 155 156 157 158 159
+			160 161 162 163 164 165 166 167
+			168 169 170 171 172 173 174 175
+			176 177 178 179 180 181 182 183
+			184 185 186 187 188 189 190 191
+			192 193 194 195 196 197 198 199
+			200 201 202 203 204 205 206 207
+			208 209 210 211 212 213 214 215
+			216 217 218 219 220 221 222 223
+			224 225 226 227 228 229 230 231
+			232 233 234 235 236 237 238 239
+			240 241 242 243 244 245 246 247
+			248 249 250 251 252 253 254 255>;
+		default-brightness-level = <128>;
+		status = "okay";
+	};
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	i2c0-2-cfg {
+		i2c0-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-3-cfg {
+		i2c3-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c6-0-cfg {
+		i2c6-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			bias-pull-up = <1>;		/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie1-1-cfg {
+		pcie1-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie2-1-cfg {
+		pcie2-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pwm18_2_cfg: pwm18-2-cfg {
+		pwm18-0-pins {
+			pinmux = <K3_PADCONF(106, 6)>;	/* pwm18 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	dp0_1_cfg: dp0-1-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(23, 6)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_0_cfg>;
+	status = "disabled";
+
+	bluetooth {
+		compatible = "realtek,rtl8852bs-bt";
+		device-wake-gpios = <&gpio 2 30 GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio 2 29 GPIO_ACTIVE_HIGH>;
+		host-wake-gpios = <&gpio 2 28 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "disabled";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	/* Bringup stage, only enable USB2 phy */
+	maximum-speed = "high-speed";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	status = "okay";
+};
+
+&phy0{
+	status = "okay";
+};
+
+&phy1{
+	status = "okay";
+};
+
+&phy2{
+	status = "okay";
+};
+
+&phy3{
+	status = "okay";
+};
+
+&phy4{
+	status = "okay";
+};
+
+&phy5{
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie2_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_1_cfg>;
+	phys = <&phy2>, <&phy3>;
+	phy-names = "phy2", "phy3";
+
+	status = "okay";
+};
+
+&pcie3_rc {
+	phys = <&phy4>;
+	phy-names = "phy4";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+
+	status = "okay";
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			thermal_gpu_trip0: thermal_gpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	adcdc2: adcdc2 {
+		regulator-min-microvolt = <540000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	adcdc1: adcdc1 {
+		regulator-min-microvolt = <540000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_3_cfg>;
+	status = "okay";
+
+	spacemit@51 {
+		status = "okay";
+		compatible = "spacemit,i2c-slave";
+		reg = <0x51>;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_0_cfg>;
+	status = "okay";
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p4v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&pwm18 {
+	status = "okay";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "okay";
+};
+
+&edp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_1_cfg>;
+	backlight = <&backlight>;
+	gpios-power = <101>;
+	gpios-enable = <118>;
+	status = "okay";
+};
+
+&cpu_0 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_1 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_2 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_3 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_4 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_5 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_6 {
+	clst-supply = <&adcdc1>;
+};
+
+&cpu_7 {
+	clst-supply = <&adcdc1>;
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_com260.dts b/arch/riscv/boot/dts/spacemit/k3_com260.dts
new file mode 100644
index 000000000000..d7f6db526ea8
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_com260.dts
@@ -0,0 +1,1419 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd/lcd_tc358762xbg_dpi_800x480.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 Com260";
+	compatible = "spacemit,k3-com260";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	rfkill-pcie-wlan {
+		compatible = "rfkill-gpio";
+		label = "rfkill-pcie-wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis1_cfg>;
+		radio-type = "wlan";
+		shutdown-gpios = <&gpio 2 9 GPIO_ACTIVE_LOW>;
+	};
+
+	rfkill-m2-bt {
+		compatible = "rfkill-gpio";
+		label = "rfkill-m2-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis2_cfg>;
+		radio-type = "bluetooth";
+		shutdown-gpios = <&gpio 1 25 GPIO_ACTIVE_LOW>;
+	};
+
+	hub_reset: hub-reset {
+		compatible = "regulator-fixed";
+		regulator-name = "hub-reset";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		regulator-boot-on;
+		gpio = <&gpio 1 21 0>;
+	};
+};
+
+&rpmi_pwrkey {
+	status = "disabled";
+};
+
+&pwrkey_monitor {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwrkey_monitor_cfg>;
+	monitor-gpios = <&gpio 2 20 GPIO_ACTIVE_HIGH>;
+	interrupts-extended = <&pinctrl 84 IRQ_TYPE_EDGE_BOTH>;
+	status = "okay";
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	can2-1-cfg {
+		can2-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	i2c0-0-cfg {
+		i2c0-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source= <3300>;
+		};
+	};
+
+	i2c5_2_cfg: i2c5-2-cfg {
+		i2c5-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c6-1-cfg {
+		i2c6-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	pwrkey_monitor_cfg: pwrkey_monitor-cfg {
+		pwrkey-monitor-pins {
+			pinmux = <K3_PADCONF(84, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	dp1_3_cfg: dp1-3-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(72, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,	/* pcie0 perst */
+				<K3_PADCONF(81, 5)>;	/* pcie0 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie3-0-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(3, 4)>,	/* pcie3 perst */
+				 <K3_PADCONF(5, 4)>;	/* pcie3 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				<K3_PADCONF(80, 5)>,	/* pcie4 wake */
+				<K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis1_cfg: m2-wdis1-cfg {
+		m2-wdis1-pins {
+			pinmux = <K3_PADCONF(73, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis2_cfg: m2-wdis2-cfg {
+		m2-wdis2-pins {
+			pinmux = <K3_PADCONF(57, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	cam0_2_cfg: cam0-2-cfg {
+               cam1-0-pins {
+                        power-source = <1800>;
+                };
+        };
+
+	cam_mux_cfg: cam-mux-cfg {
+		cam-mux-pins {
+			pinmux = <K3_PADCONF(103, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(4, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+	};
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs","debug";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	pinctrl-2 = <&mmc1_jtag_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 0 4 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	/* M2 BT */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	orientation-switch;
+	status = "okay";
+
+	port {
+		usb3_phy_switch: endpoint {
+			remote-endpoint = <&fusb301_sw_ep>;
+		};
+	};
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	reset-on-resume;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reset-on-resume;
+	/* wakeup-source; */
+
+	/* VL817 USB2.0 hub (4 ports) */
+	hub_2_0: hub@1 {
+		compatible = "usb2109,2817";
+		reg = <1>;
+		peer-hub = <&hub_3_0>;
+		vdd-supply = <&hub_reset>;
+	};
+
+	/* VL817 USB3.0 hub (4 ports) */
+	hub_3_0: hub@2 {
+		compatible = "usb2109,817";
+		reg = <2>;
+		peer-hub = <&hub_2_0>;
+	};
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy4 {
+	status = "okay";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_0_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy4";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+
+        master0 {
+		main-device = <&eth1>;
+        };
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac1_phy>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <47>;
+	spacemit,rx-phase = <53>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac1_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 2 3>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 4 5>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <50000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <60000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <70000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&ctf2301 0 50>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&ctf2301 64 75>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&ctf2301 102 128>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&ctf2301 179 204>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&ctf2301 255 255>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_0_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+		suspend-vbus-off;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+
+				port@1 {
+					reg = <0x1>;
+					fusb301_sw_ep: endpoint {
+						remote-endpoint = <&usb3_phy_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c5_2_cfg>;
+
+	imx219@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 13 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5640@3c {
+		compatible = "spacemit,ov5640-max96724-max9295";
+		reg = <0x3c>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		clock-names = "xclk";
+		clocks = <&syscon_apmu CLK_APMU_ISIM_VCLK0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam0_2_cfg>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		deserializer-addr = <0x27>;
+		serializer-base-addr = <0x41>;
+		status = "okay";
+	};
+
+	imx415@37 {
+		compatible = "sony,imx415";
+		reg = <0x37>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+};
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	spacemit-dpu-bitclk = <700000000>;
+	status = "okay";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dsi1_output: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	panel0: panel0@0 {
+			status = "ok";
+			compatible = "raspberrypi,dpi-panel";
+			force-attached = "lcd_tc358762xbg_dpi_800x480";
+	};
+};
+
+&lcds {
+	status = "okay";
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_2_cfg>;
+	status = "okay";
+
+	raspits-panel@45 {
+		status = "okay";
+		compatible = "raspberrypi,7inch-touchscreen-panel";
+		reg = <0x45>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dsi1_output>;
+			};
+		};
+	};
+
+	raspits-touch-ft5426@38 {
+		status = "okay";
+		compatible = "raspits_ft5426";
+		reg = <0x38>;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_1_cfg>;
+	status = "okay";
+
+	ctf2301: ctf2301@4c {
+		compatible = "sensylink,ctf2301";
+		reg = <0x4c>;
+		#cooling-cells = <2>;
+	};
+};
+
+&flexcan2 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can2_1_cfg>;
+	status = "okay";
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_3_cfg>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
+
+&ccic0 {
+	status = "okay";
+};
+
+&ccic1 {
+	status = "okay";
+};
+
+&ccic2 {
+	status = "okay";
+};
+
+&ccic3 {
+	status = "disabled";
+};
+
+&csiphy0 {
+	status = "okay";
+};
+
+&csiphy1 {
+	status = "okay";
+};
+
+&csiphy2 {
+	spacemit,bifmode-enable = <0>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_com260_ifx.dts b/arch/riscv/boot/dts/spacemit/k3_com260_ifx.dts
new file mode 100644
index 000000000000..cd9eb42c955e
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_com260_ifx.dts
@@ -0,0 +1,1416 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd/lcd_tc358762xbg_dpi_800x480.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 Com260 IFX";
+	compatible = "spacemit,k3-com260-ifx";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	rfkill-pcie-wlan {
+		compatible = "rfkill-gpio";
+		label = "rfkill-pcie-wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis1_cfg>;
+		radio-type = "wlan";
+		shutdown-gpios = <&gpio 2 9 GPIO_ACTIVE_LOW>;
+	};
+
+	rfkill-m2-bt {
+		compatible = "rfkill-gpio";
+		label = "rfkill-m2-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis2_cfg>;
+		radio-type = "bluetooth";
+		shutdown-gpios = <&gpio 1 25 GPIO_ACTIVE_LOW>;
+	};
+
+	hub_reset: hub-reset {
+		compatible = "regulator-fixed";
+		regulator-name = "hub-reset";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		regulator-boot-on;
+		gpio = <&gpio 1 21 0>;
+	};
+};
+
+&rpmi_pwrkey {
+	status = "disabled";
+};
+
+&pwrkey_monitor {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwrkey_monitor_cfg>;
+	monitor-gpios = <&gpio 2 20 GPIO_ACTIVE_HIGH>;
+	interrupts-extended = <&pinctrl 84 IRQ_TYPE_EDGE_BOTH>;
+	status = "okay";
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	can2-1-cfg {
+		can2-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	i2c0-0-cfg {
+		i2c0-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source= <3300>;
+		};
+	};
+
+	i2c5_2_cfg: i2c5-2-cfg {
+		i2c5-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c6-1-cfg {
+		i2c6-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	pwrkey_monitor_cfg: pwrkey_monitor-cfg {
+		pwrkey-monitor-pins {
+			pinmux = <K3_PADCONF(84, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	dp1_3_cfg: dp1-3-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(72, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,	/* pcie0 perst */
+				<K3_PADCONF(81, 5)>;	/* pcie0 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie3-0-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(3, 4)>,	/* pcie3 perst */
+				 <K3_PADCONF(5, 4)>;	/* pcie3 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				<K3_PADCONF(80, 5)>,	/* pcie4 wake */
+				<K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis1_cfg: m2-wdis1-cfg {
+		m2-wdis1-pins {
+			pinmux = <K3_PADCONF(73, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis2_cfg: m2-wdis2-cfg {
+		m2-wdis2-pins {
+			pinmux = <K3_PADCONF(57, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	cam0_2_cfg: cam0-2-cfg {
+               cam1-0-pins {
+                        power-source = <1800>;
+                };
+        };
+
+	cam_mux_cfg: cam-mux-cfg {
+		cam-mux-pins {
+			pinmux = <K3_PADCONF(103, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(4, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+	};
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 0 4 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	/* M2 BT */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	orientation-switch;
+	status = "okay";
+
+	port {
+		usb3_phy_switch: endpoint {
+			remote-endpoint = <&fusb301_sw_ep>;
+		};
+	};
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	reset-on-resume;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reset-on-resume;
+	/* wakeup-source; */
+
+	/* VL817 USB2.0 hub (4 ports) */
+	hub_2_0: hub@1 {
+		compatible = "usb2109,2817";
+		reg = <1>;
+		peer-hub = <&hub_3_0>;
+		vdd-supply = <&hub_reset>;
+	};
+
+	/* VL817 USB3.0 hub (4 ports) */
+	hub_3_0: hub@2 {
+		compatible = "usb2109,817";
+		reg = <2>;
+		peer-hub = <&hub_2_0>;
+	};
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy4 {
+	status = "okay";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_0_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy4";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+
+        master0 {
+		main-device = <&eth1>;
+        };
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac1_phy>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <47>;
+	spacemit,rx-phase = <53>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac1_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 2 3>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 4 5>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <50000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <60000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <70000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&ctf2301 0 50>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&ctf2301 64 75>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&ctf2301 102 128>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&ctf2301 179 204>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&ctf2301 255 255>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_0_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+		suspend-vbus-off;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+
+				port@1 {
+					reg = <0x1>;
+					fusb301_sw_ep: endpoint {
+						remote-endpoint = <&usb3_phy_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c5_2_cfg>;
+
+	imx219@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+		csi-id = <1>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 13 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5640@3c {
+		compatible = "spacemit,ov5640-max96724-max9295";
+		reg = <0x3c>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		clock-names = "xclk";
+		clocks = <&syscon_apmu CLK_APMU_ISIM_VCLK0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam0_2_cfg>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		deserializer-addr = <0x27>;
+		serializer-base-addr = <0x41>;
+		status = "okay";
+	};
+
+	imx415@37 {
+		compatible = "sony,imx415";
+		reg = <0x37>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+};
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	spacemit-dpu-bitclk = <700000000>;
+	status = "okay";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dsi1_output: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	panel0: panel0@0 {
+			status = "ok";
+			compatible = "raspberrypi,dpi-panel";
+			force-attached = "lcd_tc358762xbg_dpi_800x480";
+	};
+};
+
+&lcds {
+	status = "okay";
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_2_cfg>;
+	status = "okay";
+
+	raspits-panel@45 {
+		status = "okay";
+		compatible = "raspberrypi,7inch-touchscreen-panel";
+		reg = <0x45>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dsi1_output>;
+			};
+		};
+	};
+
+	raspits-touch-ft5426@38 {
+		status = "okay";
+		compatible = "raspits_ft5426";
+		reg = <0x38>;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_1_cfg>;
+	status = "okay";
+
+	ctf2301: ctf2301@4c {
+		compatible = "sensylink,ctf2301";
+		reg = <0x4c>;
+		#cooling-cells = <2>;
+	};
+};
+
+&flexcan2 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can2_1_cfg>;
+	status = "okay";
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_3_cfg>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
+
+&ccic0 {
+	status = "okay";
+};
+
+&ccic1 {
+	status = "okay";
+};
+
+&ccic2 {
+	status = "okay";
+};
+
+&ccic3 {
+	status = "disabled";
+};
+
+&csiphy0 {
+	status = "okay";
+};
+
+&csiphy1 {
+	status = "okay";
+};
+
+&csiphy2 {
+	spacemit,bifmode-enable = <0>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_com260_ifx2.dts b/arch/riscv/boot/dts/spacemit/k3_com260_ifx2.dts
new file mode 100644
index 000000000000..42873db89440
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_com260_ifx2.dts
@@ -0,0 +1,1463 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd/lcd_tc358762xbg_dpi_800x480.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 Com260 IFX2";
+	compatible = "spacemit,k3-com260-ifx";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	rfkill-pcie-wlan {
+		compatible = "rfkill-gpio";
+		label = "rfkill-pcie-wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis1_cfg>;
+		radio-type = "wlan";
+		shutdown-gpios = <&gpio 2 9 GPIO_ACTIVE_LOW>;
+	};
+
+	rfkill-m2-bt {
+		compatible = "rfkill-gpio";
+		label = "rfkill-m2-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis2_cfg>;
+		radio-type = "bluetooth";
+		shutdown-gpios = <&gpio 1 25 GPIO_ACTIVE_LOW>;
+	};
+
+	hub_reset: hub-reset {
+		compatible = "regulator-fixed";
+		regulator-name = "hub-reset";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		regulator-boot-on;
+		gpio = <&gpio 1 21 0>;
+	};
+};
+
+&rpmi_pwrkey {
+	status = "disabled";
+};
+
+&pwrkey_monitor {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwrkey_monitor_cfg>;
+	monitor-gpios = <&gpio 2 20 GPIO_ACTIVE_HIGH>;
+	interrupts-extended = <&pinctrl 84 IRQ_TYPE_EDGE_BOTH>;
+	status = "okay";
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	can3-1-cfg {
+		can3-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	can4-1-cfg {
+		can4-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	rcan2-0-cfg {
+		rcan2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	i2c0-0-cfg {
+		i2c0-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source= <3300>;
+		};
+	};
+
+	i2c5_2_cfg: i2c5-2-cfg {
+		i2c5-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c6-1-cfg {
+		i2c6-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	pwrkey_monitor_cfg: pwrkey_monitor-cfg {
+		pwrkey-monitor-pins {
+			pinmux = <K3_PADCONF(84, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	dp1_3_cfg: dp1-3-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(72, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,	/* pcie0 perst */
+				<K3_PADCONF(81, 5)>;	/* pcie0 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie3-0-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(3, 4)>,	/* pcie3 perst */
+				 <K3_PADCONF(5, 4)>;	/* pcie3 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				<K3_PADCONF(80, 5)>,	/* pcie4 wake */
+				<K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis1_cfg: m2-wdis1-cfg {
+		m2-wdis1-pins {
+			pinmux = <K3_PADCONF(73, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis2_cfg: m2-wdis2-cfg {
+		m2-wdis2-pins {
+			pinmux = <K3_PADCONF(57, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	cam0_2_cfg: cam0-2-cfg {
+               cam1-0-pins {
+                        power-source = <1800>;
+                };
+        };
+
+	cam_mux_cfg: cam-mux-cfg {
+		cam-mux-pins {
+			pinmux = <K3_PADCONF(103, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(4, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+	};
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 0 4 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	/* M2 BT */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	orientation-switch;
+	status = "okay";
+
+	port {
+		usb3_phy_switch: endpoint {
+			remote-endpoint = <&fusb301_sw_ep>;
+		};
+	};
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	reset-on-resume;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reset-on-resume;
+	/* wakeup-source; */
+
+	/* VL817 USB2.0 hub (4 ports) */
+	hub_2_0: hub@1 {
+		compatible = "usb2109,2817";
+		reg = <1>;
+		peer-hub = <&hub_3_0>;
+		vdd-supply = <&hub_reset>;
+	};
+
+	/* VL817 USB3.0 hub (4 ports) */
+	hub_3_0: hub@2 {
+		compatible = "usb2109,817";
+		reg = <2>;
+		peer-hub = <&hub_2_0>;
+	};
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy4 {
+	status = "okay";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_0_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy4";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+
+        master0 {
+		main-device = <&eth1>;
+        };
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac1_phy>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <47>;
+	spacemit,rx-phase = <53>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac1_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 2 3>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 4 5>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <50000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <60000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <70000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&ctf2301 0 50>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&ctf2301 64 75>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&ctf2301 102 128>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&ctf2301 179 204>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&ctf2301 255 255>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_0_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+		suspend-vbus-off;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+
+				port@1 {
+					reg = <0x1>;
+					fusb301_sw_ep: endpoint {
+						remote-endpoint = <&usb3_phy_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c5_2_cfg>;
+
+	imx219@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+		csi-id = <1>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 13 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5640@3c {
+		compatible = "spacemit,ov5640-max96724-max9295";
+		reg = <0x3c>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		clock-names = "xclk";
+		clocks = <&syscon_apmu CLK_APMU_ISIM_VCLK0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam0_2_cfg>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		deserializer-addr = <0x27>;
+		serializer-base-addr = <0x41>;
+		status = "okay";
+	};
+
+	imx415@37 {
+		compatible = "sony,imx415";
+		reg = <0x37>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+};
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	spacemit-dpu-bitclk = <700000000>;
+	status = "okay";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dsi1_output: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	panel0: panel0@0 {
+			status = "ok";
+			compatible = "raspberrypi,dpi-panel";
+			force-attached = "lcd_tc358762xbg_dpi_800x480";
+	};
+};
+
+&lcds {
+	status = "okay";
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_2_cfg>;
+	status = "okay";
+
+	raspits-panel@45 {
+		status = "okay";
+		compatible = "raspberrypi,7inch-touchscreen-panel";
+		reg = <0x45>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dsi1_output>;
+			};
+		};
+	};
+
+	raspits-touch-ft5426@38 {
+		status = "okay";
+		compatible = "raspits_ft5426";
+		reg = <0x38>;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_1_cfg>;
+	status = "okay";
+
+	ctf2301: ctf2301@4c {
+		compatible = "sensylink,ctf2301";
+		reg = <0x4c>;
+		#cooling-cells = <2>;
+	};
+};
+
+&flexcan0 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can0_1_cfg>;
+	status = "okay";
+};
+
+&flexcan1 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can1_0_cfg>;
+	status = "okay";
+};
+
+&flexcan2 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can2_0_cfg>;
+	status = "okay";
+};
+
+&flexcan3 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can3_1_cfg>;
+	status = "okay";
+};
+
+&flexcan4 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can4_1_cfg>;
+	status = "okay";
+};
+
+&r_flexcan2 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&rcan2_0_cfg>;
+	status = "okay";
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_3_cfg>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
+
+&ccic0 {
+	status = "okay";
+};
+
+&ccic1 {
+	status = "okay";
+};
+
+&ccic2 {
+	status = "okay";
+};
+
+&ccic3 {
+	status = "disabled";
+};
+
+&csiphy0 {
+	status = "okay";
+};
+
+&csiphy1 {
+	status = "okay";
+};
+
+&csiphy2 {
+	spacemit,bifmode-enable = <0>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_com260_kit_v02.dts b/arch/riscv/boot/dts/spacemit/k3_com260_kit_v02.dts
new file mode 100644
index 000000000000..6efb692b4671
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_com260_kit_v02.dts
@@ -0,0 +1,1358 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd/lcd_tc358762xbg_dpi_800x480.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "k3-dp1.dtsi"
+
+/ {
+	model = "SpacemiT K3 Com260 Kit V02";
+	compatible = "spacemit,k3-com260-kit-v02";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	rfkill-pcie-wlan {
+		compatible = "rfkill-gpio";
+		label = "rfkill-pcie-wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis1_cfg>;
+		radio-type = "wlan";
+		shutdown-gpios = <&gpio 2 9 GPIO_ACTIVE_LOW>;
+	};
+
+	rfkill-m2-bt {
+		compatible = "rfkill-gpio";
+		label = "rfkill-m2-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&m2_wdis2_cfg>;
+		radio-type = "bluetooth";
+		shutdown-gpios = <&gpio 1 25 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&rpmi_pwrkey {
+	status = "disabled";
+};
+
+&pwrkey_monitor {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwrkey_monitor_cfg>;
+	monitor-gpios = <&gpio 2 20 GPIO_ACTIVE_HIGH>;
+	interrupts-extended = <&pinctrl 84 IRQ_TYPE_EDGE_BOTH>;
+	status = "okay";
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	uart4-0-cfg {
+		uart4-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	uart5-2-cfg {
+		uart5-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	can3-1-cfg {
+		can3-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	can4-1-cfg {
+		can4-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	rcan2-0-cfg {
+		rcan2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	i2c0-0-cfg {
+		i2c0-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source= <3300>;
+		};
+	};
+
+	i2c6-1-cfg {
+		i2c6-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	pwrkey_monitor_cfg: pwrkey_monitor-cfg {
+		pwrkey-monitor-pins {
+			pinmux = <K3_PADCONF(84, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	dp1_3_cfg: dp1-3-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(72, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,	/* pcie0 perst */
+				<K3_PADCONF(81, 5)>;	/* pcie0 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie3-0-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(3, 4)>,	/* pcie3 perst */
+				 <K3_PADCONF(5, 4)>;	/* pcie3 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				<K3_PADCONF(80, 5)>,	/* pcie4 wake */
+				<K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis1_cfg: m2-wdis1-cfg {
+		m2-wdis1-pins {
+			pinmux = <K3_PADCONF(73, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	m2_wdis2_cfg: m2-wdis2-cfg {
+		m2-wdis2-pins {
+			pinmux = <K3_PADCONF(57, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(4, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart4_0_cfg>;
+	status = "okay";
+};
+
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart5_2_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	spacemit,qspi-tx-dma = <0>;
+	spacemit,qspi-rx-dma = <0>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+	};
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 0 4 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	/* M2 BT */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	orientation-switch;
+	status = "okay";
+
+	port {
+		usb3_phy_switch: endpoint {
+			remote-endpoint = <&fusb301_sw_ep>;
+		};
+	};
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	/* VL817 USB2.0 hub (4 ports) */
+	hub_2_0: hub@1 {
+		compatible = "usb2109,2817";
+		reg = <1>;
+		peer-hub = <&hub_3_0>;
+		/* GPIO53(MIPI_CSI3_PWDN) reserved for HUB 3V3 RST,
+		 * now reset will be done by HW */
+	};
+
+	/* VL817 USB3.0 hub (4 ports) */
+	hub_3_0: hub@2 {
+		compatible = "usb2109,817";
+		reg = <2>;
+		peer-hub = <&hub_2_0>;
+		/* GPIO53(MIPI_CSI3_PWDN) reserved for HUB 3V3 RST,
+		 * now reset will be done by HW */
+	};
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy4 {
+	status = "okay";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_0_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy4";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+
+        master0 {
+		main-device = <&eth1>;
+        };
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&rgmii1>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <47>;
+	spacemit,rx-phase = <53>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		rgmii1: phy@1 {
+			compatible = "ethernet-phy-id001c.c916";
+			reg = <0x1>;
+			device_type = "ethernet-phy";
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 1 2>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 2 3>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <50000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <60000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <70000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&ctf2301 0 50>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&ctf2301 64 75>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&ctf2301 102 128>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&ctf2301 179 204>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&ctf2301 255 255>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_0_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+		wakeup-source;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+
+				port@1 {
+					reg = <0x1>;
+					fusb301_sw_ep: endpoint {
+						remote-endpoint = <&usb3_phy_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+};
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	spacemit-dpu-bitclk = <700000000>;
+	status = "okay";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dsi1_output: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	panel0: panel0@0 {
+			status = "ok";
+			compatible = "raspberrypi,dpi-panel";
+			force-attached = "lcd_tc358762xbg_dpi_800x480";
+	};
+};
+
+&lcds {
+	status = "okay";
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_2_cfg>;
+	status = "okay";
+
+	raspits-panel@45 {
+		status = "okay";
+		compatible = "raspberrypi,7inch-touchscreen-panel";
+		reg = <0x45>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dsi1_output>;
+			};
+		};
+	};
+
+	raspits-touch-ft5426@38 {
+		status = "okay";
+		compatible = "raspits_ft5426";
+		reg = <0x38>;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_1_cfg>;
+	status = "okay";
+
+	ctf2301: ctf2301@4c {
+		compatible = "sensylink,ctf2301";
+		reg = <0x4c>;
+		#cooling-cells = <2>;
+	};
+};
+
+&flexcan0 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can0_1_cfg>;
+	status = "okay";
+};
+
+&flexcan1 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can1_0_cfg>;
+	status = "okay";
+};
+
+&flexcan2 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can2_0_cfg>;
+	status = "okay";
+};
+
+&flexcan3 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can3_1_cfg>;
+	status = "okay";
+};
+
+&flexcan4 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can4_1_cfg>;
+	status = "okay";
+};
+
+&r_flexcan2 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&rcan2_0_cfg>;
+	status = "okay";
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_3_cfg>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_com260_tq.dts b/arch/riscv/boot/dts/spacemit/k3_com260_tq.dts
new file mode 100644
index 000000000000..b0cc5a251c3d
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_com260_tq.dts
@@ -0,0 +1,1405 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd/lcd_tc358762xbg_dpi_800x480.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 Com260 TQ";
+	compatible = "spacemit,k3-com260-tq";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p5v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	pcie0_pwr_en: regulator-pcie0-pwr-en {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie0 pwr en";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pcie0_pwr_en_cfg>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio 2 9 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-boot-on;
+		status = "okay";
+	};
+
+	hub_reset: hub-reset {
+		compatible = "regulator-fixed";
+		regulator-name = "hub-reset";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		regulator-boot-on;
+		gpio = <&gpio 1 21 0>;
+	};
+};
+
+&rpmi_pwrkey {
+	status = "disabled";
+};
+
+&pwrkey_monitor {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwrkey_monitor_cfg>;
+	monitor-gpios = <&gpio 2 20 GPIO_ACTIVE_HIGH>;
+	interrupts-extended = <&pinctrl 84 IRQ_TYPE_EDGE_BOTH>;
+	status = "okay";
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	can2-1-cfg {
+		can2-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	i2c0-0-cfg {
+		i2c0-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c5_2_cfg: i2c5-2-cfg {
+		i2c5-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c6-1-cfg {
+		i2c6-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	pwrkey_monitor_cfg: pwrkey_monitor-cfg {
+		pwrkey-monitor-pins {
+			pinmux = <K3_PADCONF(84, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	dp1_3_cfg: dp1-3-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(72, 4)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,	/* pcie0 perst */
+				<K3_PADCONF(81, 5)>;	/* pcie0 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie3-0-cfg {
+		pcie3-0-pins {
+			pinmux = <K3_PADCONF(3, 4)>,	/* pcie3 perst */
+				 <K3_PADCONF(5, 4)>;	/* pcie3 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				<K3_PADCONF(80, 5)>,	/* pcie4 wake */
+				<K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <38>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie0_pwr_en_cfg: pcie0-pwr-en-cfg {
+		pcie0-pwr-en-pins {
+			pinmux = <K3_PADCONF(73, 0)>;
+
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	cam0_2_cfg: cam0-2-cfg {
+		cam1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	cam_mux_cfg: cam-mux-cfg {
+		cam-mux-pins {
+			pinmux = <K3_PADCONF(103, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(4, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+	};
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs","debug";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	pinctrl-2 = <&mmc1_jtag_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 0 4 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	/* M2 BT */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	orientation-switch;
+	status = "okay";
+
+	port {
+		usb3_phy_switch: endpoint {
+			remote-endpoint = <&fusb301_sw_ep>;
+		};
+	};
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	reset-on-resume;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	reset-on-resume;
+	/* wakeup-source; */
+
+	/* VL817 USB2.0 hub (4 ports) */
+	hub_2_0: hub@1 {
+		compatible = "usb2109,2817";
+		reg = <1>;
+		peer-hub = <&hub_3_0>;
+		vdd-supply = <&hub_reset>;
+	};
+
+	/* VL817 USB3.0 hub (4 ports) */
+	hub_3_0: hub@2 {
+		compatible = "usb2109,817";
+		reg = <2>;
+		peer-hub = <&hub_2_0>;
+	};
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy4 {
+	status = "okay";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	vpcie3v3-supply = <&pcie0_pwr_en>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_0_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy4";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+
+	master0 {
+		main-device = <&eth1>;
+	};
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac1_phy>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <47>;
+	spacemit,rx-phase = <53>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac1_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 2 3>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 4 5>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <50000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <60000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <70000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&ctf2301 0 50>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&ctf2301 64 75>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&ctf2301 102 128>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&ctf2301 179 204>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&ctf2301 255 255>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_0_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+		suspend-vbus-off;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+
+				port@1 {
+					reg = <0x1>;
+					fusb301_sw_ep: endpoint {
+						remote-endpoint = <&usb3_phy_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c5_2_cfg>;
+
+	imx219@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 13 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+
+	ov5640@3c {
+		compatible = "spacemit,ov5640-max96724-max9295";
+		reg = <0x3c>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		clock-names = "xclk";
+		clocks = <&syscon_apmu CLK_APMU_ISIM_VCLK0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam0_2_cfg>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		deserializer-addr = <0x27>;
+		serializer-base-addr = <0x41>;
+		status = "okay";
+	};
+
+	imx415@37 {
+		compatible = "sony,imx415";
+		reg = <0x37>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_mux_cfg>;
+		i2c-mux-gpios = <&gpio 3 7 GPIO_ACTIVE_HIGH>;
+		pwdn-gpios = <&gpio 1 12 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+};
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	spacemit-dpu-bitclk = <700000000>;
+	status = "okay";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dsi1_output: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+
+	panel0: panel0@0 {
+		status = "ok";
+		compatible = "raspberrypi,dpi-panel";
+		force-attached = "lcd_tc358762xbg_dpi_800x480";
+	};
+};
+
+&lcds {
+	status = "okay";
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_2_cfg>;
+	status = "okay";
+
+	raspits-panel@45 {
+		status = "okay";
+		compatible = "raspberrypi,7inch-touchscreen-panel";
+		reg = <0x45>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dsi1_output>;
+			};
+		};
+	};
+
+	raspits-touch-ft5426@38 {
+		status = "okay";
+		compatible = "raspits_ft5426";
+		reg = <0x38>;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_1_cfg>;
+	status = "okay";
+
+	ctf2301: ctf2301@4c {
+		compatible = "sensylink,ctf2301";
+		reg = <0x4c>;
+		#cooling-cells = <2>;
+	};
+};
+
+&flexcan2 {
+	clock-frequency = <80000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can2_1_cfg>;
+	status = "okay";
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_3_cfg>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
+
+&ccic0 {
+	status = "okay";
+};
+
+&ccic1 {
+	status = "okay";
+};
+
+&ccic2 {
+	status = "okay";
+};
+
+&ccic3 {
+	status = "disabled";
+};
+
+&csiphy0 {
+	status = "okay";
+};
+
+&csiphy1 {
+	status = "okay";
+};
+
+&csiphy2 {
+	spacemit,bifmode-enable = <0>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_deb1.dts b/arch/riscv/boot/dts/spacemit/k3_deb1.dts
new file mode 100644
index 000000000000..a2755ac0aa11
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_deb1.dts
@@ -0,0 +1,1475 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "k3-edp0.dtsi"
+#include "k3-dp1.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/input/cros-ec-keyboard.h>
+
+/ {
+	model = "SpacemiT K3 Deb1";
+	compatible = "spacemit,k3-deb1";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	aux_5v: aux_5v {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	aux_3v3: aux_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&aux_5v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+                bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+                rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	rfkill-pcie-wlan {
+		compatible = "rfkill-gpio";
+		label = "rfkill-pcie-wlan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wlan_en_cfg>;
+		radio-type = "wlan";
+		shutdown-gpios = <&gpio 1 3 GPIO_ACTIVE_HIGH>;
+	};
+
+	rfkill-usb-bt {
+		compatible = "rfkill-gpio";
+		label = "rfkill-usb-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_en_cfg>;
+		radio-type = "bluetooth";
+		shutdown-gpios = <&gpio 0 30 GPIO_ACTIVE_HIGH>;
+	};
+
+	rfkill-usb-wwan {
+		compatible = "rfkill-gpio";
+		label = "m.2 WWAN"; /* B-Key */
+		radio-type = "wwan";
+		/* MODULE_TURN_ON_1V8 (FC_PWROFF#) */
+		shutdown-gpios = <&gpio 0 18 GPIO_ACTIVE_HIGH>;
+		/* WWAN_RESET#_1V8 (WWAN1V8_RST#) */
+		reset-gpios = <&gpio 0 19 GPIO_ACTIVE_HIGH>;
+		/* GPIO84 for CPU_W_DISABLE#1 (WWAN_DIS#) is no needed*/
+	};
+
+	fan0: pwm-fan {
+		compatible = "pwm-fan";
+		pwms = <&cros_ec_pwm 0>;
+		/*
+		* Cooling states (0-8) -> PWM duty (0-255)
+		* Full speed is treated as ~6000RPM.
+		* Custom ramp to balance noise and thermals
+		*/
+		/* Keep headroom by capping max duty below 255. */
+		cooling-levels = <0 30 60 90 120 149 170 191 213>;
+		#cooling-cells = <2>;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm18 0 20000 0>;
+		brightness-levels = <
+			0   1   2   3   4   5   6   7
+			8   9  10  11  12  13  14  15
+			16  17  18  19  20  21  22  23
+			24  25  26  27  28  29  30  31
+			32  33  34  35  36  37  38  39
+			40  41  42  43  44  45  46  47
+			48  49  50  51  52  53  54  55
+			56  57  58  59  60  61  62  63
+			64  65  66  67  68  69  70  71
+			72  73  74  75  76  77  78  79
+			80  81  82  83  84  85  86  87
+			88  89  90  91  92  93  94  95
+			96  97  98  99 100 101 102 103
+			104 105 106 107 108 109 110 111
+			112 113 114 115 116 117 118 119
+			120 121 122 123 124 125 126 127
+			128 129 130 131 132 133 134 135
+			136 137 138 139 140 141 142 143
+			144 145 146 147 148 149 150 151
+			152 153 154 155 156 157 158 159
+			160 161 162 163 164 165 166 167
+			168 169 170 171 172 173 174 175
+			176 177 178 179 180 181 182 183
+			184 185 186 187 188 189 190 191
+			192 193 194 195 196 197 198 199
+			200 201 202 203 204 205 206 207
+			208 209 210 211 212 213 214 215
+			216 217 218 219 220 221 222 223
+			224 225 226 227 228 229 230 231
+			232 233 234 235 236 237 238 239
+			240 241 242 243 244 245 246 247
+			248 249 250 251 252 253 254 255>;
+		default-brightness-level = <128>;
+		status = "okay";
+	};
+
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	gmac0-cfg {
+		/delete-node/ gmac0-2-pins;
+		/delete-node/ gmac0-4-pins;
+		/delete-node/ gmac0-5-pins;
+
+		gmac0-0-pins {
+			power-source = <1800>;
+			drive-strength = <11>;
+		};
+
+		gmac0-1-pins {
+			power-source = <1800>;
+			drive-strength = <11>;
+		};
+
+		gmac0-3-pins {
+			power-source = <1800>;
+			drive-strength = <11>;
+		};
+
+		gmac0-6-pins {
+			pinmux = <K3_PADCONF(15, 0)>;
+
+			bias-disable;
+			drive-strength = <11>;
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+			drive-strength = <9>;		/* DS4 */
+		};
+	};
+
+	i2c1-3-cfg {
+		i2c1-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c3-1-cfg {
+		i2c3-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	usb30_drd_dir-0-cfg {
+		usb30_drd_dir-pins {
+			power-source = <1800>;
+		};
+	};
+
+	fusb301_cfg: fusb301-cfg {
+		fusb301-0-pins {
+			pinmux = <K3_PADCONF(85, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+			drive-strength = <11>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			pinmux = <K3_PADCONF(79, 5)>,
+				 <K3_PADCONF(81, 5)>;
+
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <1800>;
+		};
+	};
+
+	pcie1-1-cfg {
+		pcie1-0-pins {
+			pinmux = <K3_PADCONF(88, 4)>,	/* pcie1 perst */
+				 <K3_PADCONF(90, 4)>;	/* pcie1 clkreq */
+
+			bias-pull-up;
+			drive-strength = <33>;
+			power-source = <1800>;
+		};
+	};
+
+	pcie2-1-cfg {
+		pcie2-0-pins {
+			pinmux = <K3_PADCONF(25, 4)>,	/* pcie2 perst */
+				 <K3_PADCONF(27, 4)>;	/* pcie2 clkreq */
+
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	pcie4_0_cfg: pcie4-0-cfg {
+		pcie4-0-pins {
+			pinmux = <K3_PADCONF(76, 5)>,	/* pcie4 perst */
+				 <K3_PADCONF(77, 5)>,	/* pcie4 wake */
+				 <K3_PADCONF(78, 5)>;	/* pcie4 clkreq */
+
+			bias-pull-up;			/* normal bias disable */
+			drive-strength = <33>;		/* DS8 */
+			power-source = <1800>;
+		};
+	};
+
+	wlan_en_cfg: wl-en-cfg {
+		wlan-en-pins {
+			pinmux = <K3_PADCONF(35, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	bt_en_cfg: bt-en-cfg {
+		bt-en-pins {
+			pinmux = <K3_PADCONF(30, 0)>;
+
+			bias-pull-up;
+			drive-strength = <38>;
+			power-source = <3300>;
+		};
+	};
+
+	dp1_1_cfg: dp1-1-cfg {
+		dp1-0-pins {
+			pinmux = <K3_PADCONF(24, 6)>;	/* dp1 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	dp0_1_cfg: dp0-1-cfg {
+		dp0-0-pins {
+			pinmux = <K3_PADCONF(23, 6)>;	/* dp0 hpd */
+
+			bias-disable;			/* normal bias disable */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	pwm18_2_cfg: pwm18-2-cfg {
+		pwm18-0-pins {
+			pinmux = <K3_PADCONF(106, 6)>;	/* pwm18 */
+
+			bias-pull-up;			/* normal pull-up */
+			drive-strength = <25>;		/* DS8 */
+			power-source = <3300>;
+		};
+	};
+
+	sspa1_1_cfg: sspa1-1-cfg {
+		sspa1-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	espi0_2_cfg: espi0-2-cfg {
+		espi0-0-pins {
+			drive-strength = <9>;		/* DS4 */
+		};
+
+		espi0-1-pins {
+			drive-strength = <9>;		/* DS4 */
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		m25p,fast-read;
+		broken-flash-reset;
+		status = "okay";
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			bootinfo@0 {
+				reg = <0x00000000 0x00020000>;
+			};
+			fsbl@20000 {
+				reg = <0x00020000 0x00080000>;
+			};
+			env@a0000 {
+				reg = <0x000a0000 0x00010000>;
+			};
+			esos@b0000 {
+				reg = <0x000b0000 0x00100000>;
+			};
+			opensbi@1b0000 {
+				reg = <0x001b0000 0x00060000>;
+			};
+			uboot@210000 {
+				reg = <0x00210000 0x005f0000>;
+			};
+		};
+	};
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	hub@1 {
+		/* FE1.1S USB2.0 hub (4 ports) */
+		compatible = "usb1a40,101";
+		reg = <1>;
+	};
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	/* dual phy selection is auto detect from GPIO86(SS_SW_3V3) */
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb30_drd_dir_0_cfg>;
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	monitor-vbus;
+	reset-on-resume;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0x0>;
+			porta_role_switch: endpoint {
+				remote-endpoint = <&fusb301_ep>;
+			};
+		};
+	};
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	/* M2.B Socket, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	reset-on-resume;
+	phys = <&usb3_portb_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "disabled";
+};
+
+&usb3_portc {
+	/* Bluetooth, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	reset-on-resume;
+	phys = <&usb3_portc_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	/* TypeC (USB3/DP/Power) (with ANX7447 chip) */
+	status = "okay";
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&phy0 {
+	status = "okay";
+};
+
+&phy1 {
+	status = "okay";
+};
+
+&phy2 {
+	status = "okay";
+};
+
+&phy3 {
+	status = "okay";
+};
+
+&phy5 {
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>, <&phy1>;
+	phy-names = "phy0", "phy1";
+	num-lanes = <4>;
+	spacemit,bifurcation-gpios = <&gpio 2 25 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&pcie1_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_1_cfg>;
+	phys = <&phy1>;
+	phy-names = "phy1";
+	num-lanes = <2>;
+	spacemit,device-detect-gpios = <&gpio 2 25 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&pcie2_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_1_cfg>;
+	phys = <&phy2>, <&phy3>;
+	phy-names = "phy2", "phy3";
+	num-lanes = <2>;
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_0_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy5";
+	num-lanes = <1>;
+	status = "okay";
+};
+
+&espi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&espi0_2_cfg>;
+	status = "okay";
+	ec@0 {
+		status = "okay";
+		interrupt-parent = <&gpio>;
+		interrupts = <1 11 IRQ_TYPE_EDGE_RISING>;
+
+		i2c_tunnel1: i2c-tunnel@1 {
+			compatible = "google,cros-ec-i2c-tunnel";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			google,remote-bus = <1>;
+
+			rt7451: retimer@13 {
+				compatible = "retimer,rt7451";
+				reg = <0x13>;
+				primary-addr = <0x13>;
+				secondary-addr = <0x29>;
+				status = "okay";
+			};
+		};
+
+		i2c_tunnel4: i2c-tunnel@4 {
+			compatible = "google,cros-ec-i2c-tunnel";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			google,remote-bus = <4>;
+		};
+
+		/* EC PWM provider for fan control */
+		cros_ec_pwm: pwm {
+			compatible = "google,cros-ec-pwm";
+			#pwm-cells = <1>;
+		};
+
+		keyboard {
+			compatible = "google,cros-ec-keyb";
+			keypad,num-rows = <8>;
+			keypad,num-columns = <16>;
+			linux,keymap = <
+					CROS_STD_TOP_ROW_KEYMAP
+					CROS_STD_MAIN_KEYMAP
+			>;
+		};
+	};
+};
+
+&ec_master {
+	status = "okay";
+};
+
+&eth0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac0_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac0_phy>;
+	snps,reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <73>;
+	spacemit,rx-phase = <61>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac0_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <45000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <60000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 2 3>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 4 5>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <45000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster0_trip1: thermal_cluster0-trip1 {
+				temperature = <55000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip2: thermal_cluster0-trip2 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip3: thermal_cluster0-trip3 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip4: thermal_cluster0-trip4 {
+				temperature = <78000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip5: thermal_cluster0-trip5 {
+				temperature = <82000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip6: thermal_cluster0-trip6 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster0_trip7: thermal_cluster0-trip7 {
+				temperature = <88000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster0_trip0>;
+				cooling-device = <&fan0 0 1>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster0_trip1>;
+				cooling-device = <&fan0 2 3>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster0_trip2>;
+				cooling-device = <&fan0 3 4>;
+			};
+
+			map3 {
+				trip = <&thermal_cluster0_trip3>;
+				cooling-device = <&fan0 4 4>;
+			};
+
+			map4 {
+				trip = <&thermal_cluster0_trip4>;
+				cooling-device = <&fan0 5 5>;
+			};
+
+			map5 {
+				trip = <&thermal_cluster0_trip5>;
+				cooling-device = <&fan0 6 6>;
+			};
+
+			map6 {
+				trip = <&thermal_cluster0_trip6>;
+				cooling-device = <&fan0 7 7>;
+			};
+
+			map7 {
+				trip = <&thermal_cluster0_trip7>;
+				cooling-device = <&fan0 8 8>;
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_3_cfg>;
+	status = "okay";
+
+	tcpc@25 {
+		compatible = "onsemi,fusb301";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb301_cfg>;
+		irq-gpios = <&gpio 2 21 GPIO_ACTIVE_LOW>;
+		suspend-vbus-off;
+		wakeup-source;
+		status = "okay";
+
+		typec_0: connector@0 {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			typec-power-opmode = "default";
+			pd-disable;
+
+			ports {
+				#address-cells = <0x1>;
+				#size-cells = <0x0>;
+
+				port@0 {
+					reg = <0x0>;
+					fusb301_ep: endpoint {
+						remote-endpoint = <&porta_role_switch>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_1_cfg>;
+	status = "okay";
+
+	es8326: es8326@19{
+		compatible = "everest,es8326";
+		reg = <0x19>;
+		#sound-dai-cells = <0>;
+		interrupt-parent = <&gpio>;
+		interrupts = <3 31 IRQ_TYPE_EDGE_RISING>;
+		/* spk-ctl-gpio = <&gpio 127 0>; */
+		everest,mic1-src = [44];
+		everest,mic2-src = [66];
+		everest,jack-detect-inverted;
+		status = "okay";
+	};
+};
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&aux_5v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&pwm18 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm18_2_cfg>;
+	status = "okay";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "okay";
+};
+
+&edp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_1_cfg>;
+	backlight = <&backlight>;
+	gpios-power = <101>;
+	gpios-enable = <118>;
+	status = "okay";
+};
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_1_cfg>;
+	retimer-supply = <&rt7451>;
+	status = "okay";
+};
+
+&adma3 {
+	status = "okay";
+};
+
+&ri2s3 {
+	status = "okay";
+};
+
+&sound_card_dp1 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp1";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp1_link_cpu>;
+		bitclock-master = <&dp1_link_cpu>;
+		dp1_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp1>;
+		};
+	};
+};
+
+&i2s1 {
+	status = "okay";
+	pinctrl-0 = <&sspa1_1_cfg>;
+	pinctrl-names = "default";
+};
+
+&sound_card1 {
+	status = "okay";
+	simple-audio-card,name = "snd-es8326";
+	simple-audio-card,mclk-fs = <256>;
+	simple-audio-card,dai-link@0 {
+		format = "i2s";
+		frame-master = <&link1_cpu>;
+		bitclock-master = <&link1_cpu>;
+
+		link1_cpu: cpu {
+			sound-dai = <&i2s1>;
+		};
+
+		link1_codec: codec {
+			sound-dai = <&es8326>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_evb.dts b/arch/riscv/boot/dts/spacemit/k3_evb.dts
new file mode 100644
index 000000000000..94cb2523dbd6
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_evb.dts
@@ -0,0 +1,1551 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "lcd/lcd_icnl9911c_mipi.dtsi"
+#include "k3-dp0.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 EVB";
+	compatible = "spacemit,k3-evb";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p4v: p4v {
+		compatible = "regulator-fixed";
+		regulator-name = "p4v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	vmmc_sdio: regulator-vmmc-sdio {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmc-sdio";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpio = <&gpio 3 6 GPIO_ACTIVE_HIGH>;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+
+		reset-gpios = <&gpio 3 4 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	i2c0-2-cfg {
+		i2c0-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c6-0-cfg {
+		i2c6-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	uart2-0-cfg {
+		uart2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	ssp0-2-cfg  {
+		ssp0-0-pins {
+			power-source = <3300>;
+		};
+
+		ssp0-1-pins {
+			power-source = <3300>;
+		};
+	};
+
+	usb20_host_drv-3-cfg {
+		usb20_host_drv-pins {
+			power-source  = <1800>;
+		};
+	};
+
+	usb30_drd-id-vbus-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(116, 2)>,	/* usb30 drd id */
+				 <K3_PADCONF(117, 2)>;	/* usb30 drd vbuson */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <47>;		/* DS8 */
+		};
+	};
+
+	gmac0-cfg {
+		/delete-node/ gmac0-2-pins;
+		/delete-node/ gmac0-4-pins;
+		/delete-node/ gmac0-5-pins;
+
+		gmac0-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-6-pins {
+			pinmux = <K3_PADCONF(15, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac2-cfg {
+		/delete-node/ gmac2-3-pins;
+		/delete-node/ gmac2-4-pins;
+
+		gmac2-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-2-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-5-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac3-cfg {
+		/delete-node/ gmac3-3-pins;
+		/delete-node/ gmac3-4-pins;
+
+		gmac3-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-2-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-5-pins {
+			pinmux = <K3_PADCONF(75, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	mmc2-cfg {
+		mmc2-0-pins {
+			power-source = <1800>;
+		};
+
+		mmc2-1-pins {
+			power-source = <1800>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			bias-pull-up = <1>;		/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie1-1-cfg {
+		pcie1-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie2-2-cfg {
+		pcie2-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie3-1-cfg {
+		pcie3-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(86, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	wifi_hostwake_cfg: wifi-hostwake-cfg {
+		wifi-hostwake-pins {
+			pinmux = <K3_PADCONF(101, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	bt_hostwake_cfg: bt-hostwake-cfg {
+		bt-hostwake-pins {
+			pinmux = <K3_PADCONF(62, 0)>;
+
+			bias-pull-up;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_0_cfg>;
+	status = "disabled";
+
+	bluetooth {
+		compatible = "realtek,rtl8852bs-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_hostwake_cfg>;
+		device-wake-gpios = <&gpio 1 31 GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio 2 29 GPIO_ACTIVE_HIGH>;
+		interrupts-extended = <&pinctrl 62 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "disabled";
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs","debug";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	pinctrl-2 = <&mmc1_debug_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 2 22 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&sdio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc2_cfg>;
+	bus-width = <4>;
+	non-removable;
+	vmmc-supply = <&vmmc_sdio>;
+	vqmmc-supply = <&p1v8>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	no-mmc;
+	no-sd;
+	keep-power-in-suspend;
+	clock-frequency = <375000000>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "disabled";
+
+	wifi@1 {
+		reg = <1>;
+		compatible = "realtek,rtl8852bs";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_hostwake_cfg>;
+		interrupts-extended = <&pinctrl 101 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&emmc {
+	bus-width = <8>;
+	non-removable;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	clock-frequency = <375000000>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	pinctrl-names = "default";
+	reset-on-resume;
+	pinctrl-0 = <&usb20_host_drv_3_cfg>; /* GPIO103 */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	// pinctrl-names = "default";
+	// pinctrl-0 = <&usb30_drd_int_1_cfg>, /* GPIO119 J12511 */
+	// 	    <&usb30_drd_dir_0_cfg>, /* GPIO86 J12512 */
+	// 	    <&usb30_drd_id_vbus_cfg>; /* GPIO116 J12602, GPIO117 J12515 */
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	/* Bringup stage, only enable USB2 phy */
+	maximum-speed = "high-speed";
+	reset-on-resume;
+	// wakeup-source;
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	status = "okay";
+};
+
+/* For USB verify, Must disable PCIe phy2/3/4 and related controllers first
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "okay";
+};
+
+&usb3_portc {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+*/
+
+&ufshc {
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+};
+
+&eth0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac0_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac0_phy>;
+	snps,reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <65>;
+	spacemit,rx-phase = <50>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac0_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac1_phy>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <58>;
+	spacemit,rx-phase = <70>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac1_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac2_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac2_phy>;
+	snps,reset-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <53>;
+	spacemit,rx-phase = <52>;
+
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac2_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac3_cfg>;
+
+	max-speed = <1000>;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac3_phy>;
+	snps,reset-gpios = <&gpio 2 11 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <51>;
+	spacemit,rx-phase = <54>;
+
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac3_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&phy0{
+	status = "okay";
+};
+
+&phy1{
+	status = "okay";
+};
+
+&phy2{
+	status = "okay";
+};
+
+&phy3{
+	status = "okay";
+};
+
+&phy4{
+	status = "okay";
+};
+
+&phy5{
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>;
+	phy-names = "phy0";
+
+	num-lanes = <2>;
+	status = "okay";
+};
+
+&pcie1_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_1_cfg>;
+	phys = <&phy1>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&pcie2_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_2_cfg>;
+	phys = <&phy2>, <&phy3>;
+	phy-names = "phy0", "phy1";
+
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_1_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 1 2>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 2 3>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <10000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_2_cfg>;
+	status = "okay";
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 18 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <1>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 10 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	imx219@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 13 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_0_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p4v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "disabled";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "disabled";
+
+	panel0: panel0@0 {
+		status = "ok";
+		compatible = "spacemit,mipi-panel";
+		reg = <0>;
+		reset-gpios = <&gpio 1 31 GPIO_ACTIVE_HIGH>;
+		bl-gpios = <&gpio 1 29 GPIO_ACTIVE_HIGH>;
+		dc0-gpios = <&gpio 1 24 GPIO_ACTIVE_HIGH>;
+		dc1-gpios = <&gpio 1 25 GPIO_ACTIVE_HIGH>;
+		id = <2>;
+		force-attached = "lcd_icnl9911c_mipi";
+	};
+};
+
+&lcds {
+	status = "disabled";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "okay";
+};
+
+&dp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_2_cfg>;
+	status = "okay";
+};
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_4_cfg>;
+	status = "okay";
+};
+
+&adma2 {
+	status = "okay";
+};
+
+&ri2s2 {
+	status = "okay";
+};
+
+&sound_card_dp0 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp0";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp0_link_cpu>;
+		bitclock-master = <&dp0_link_cpu>;
+		dp0_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp0>;
+		};
+	};
+};
+
+&ccic0 {
+	status = "okay";
+};
+
+&ccic1 {
+	status = "okay";
+};
+
+&ccic2 {
+	status = "okay";
+};
+
+&ccic3 {
+	status = "okay";
+};
+
+&csiphy0 {
+	status = "okay";
+};
+
+&csiphy1 {
+	status = "okay";
+};
+
+&csiphy2 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_evb2-1.dts b/arch/riscv/boot/dts/spacemit/k3_evb2-1.dts
new file mode 100644
index 000000000000..3fe62a97f17e
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_evb2-1.dts
@@ -0,0 +1,1533 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "lcd/lcd_icnl9911c_mipi.dtsi"
+#include "k3-dp0.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 EVB2-1";
+	compatible = "spacemit,k3-evb2-1";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p4v: p4v {
+		compatible = "regulator-fixed";
+		regulator-name = "p4v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 clk_ignore_unused loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	vmmc_sdio: regulator-vmmc-sdio {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmc-sdio";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpio = <&gpio 3 6 GPIO_ACTIVE_HIGH>;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+
+		reset-gpios = <&gpio 3 4 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	i2c0-2-cfg {
+		i2c0-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c6-0-cfg {
+		i2c6-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	uart2-0-cfg {
+		uart2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	ssp0-2-cfg  {
+		ssp0-0-pins {
+			power-source = <3300>;
+		};
+
+		ssp0-1-pins {
+			power-source = <3300>;
+		};
+	};
+
+	usb20_host_drv-3-cfg {
+		usb20_host_drv-pins {
+			power-source  = <1800>;
+		};
+	};
+
+	usb30_drd-id-vbus-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(116, 2)>,	/* usb30 drd id */
+				 <K3_PADCONF(117, 2)>;	/* usb30 drd vbuson */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <47>;		/* DS8 */
+		};
+	};
+
+	gmac0-cfg {
+		/delete-node/ gmac0-2-pins;
+		/delete-node/ gmac0-4-pins;
+		/delete-node/ gmac0-5-pins;
+
+		gmac0-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-6-pins {
+			pinmux = <K3_PADCONF(15, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac2-cfg {
+		/delete-node/ gmac2-3-pins;
+		/delete-node/ gmac2-4-pins;
+
+		gmac2-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-2-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-5-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac3-cfg {
+		/delete-node/ gmac3-3-pins;
+		/delete-node/ gmac3-4-pins;
+
+		gmac3-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-2-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-5-pins {
+			pinmux = <K3_PADCONF(75, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	mmc2-cfg {
+		mmc2-0-pins {
+			power-source = <1800>;
+		};
+
+		mmc2-1-pins {
+			power-source = <1800>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			bias-pull-up = <1>;		/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie1-1-cfg {
+		pcie1-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie2-2-cfg {
+		pcie2-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie3-1-cfg {
+		pcie3-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(86, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_0_cfg>;
+	status = "disabled";
+
+	bluetooth {
+		compatible = "realtek,rtl8852bs-bt";
+		device-wake-gpios = <&gpio 2 30 GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio 2 29 GPIO_ACTIVE_HIGH>;
+		host-wake-gpios = <&gpio 2 28 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "disabled";
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 2 22 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&sdio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc2_cfg>;
+	bus-width = <4>;
+	non-removable;
+	vmmc-supply = <&vmmc_sdio>;
+	vqmmc-supply = <&p1v8>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	no-mmc;
+	no-sd;
+	keep-power-in-suspend;
+	clock-frequency = <375000000>;
+	status = "disabled";
+};
+
+&emmc {
+	bus-width = <8>;
+	non-removable;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	clock-frequency = <375000000>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb20_host_drv_3_cfg>; /* GPIO103 */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	// pinctrl-names = "default";
+	// pinctrl-0 = <&usb30_drd_int_1_cfg>, /* GPIO119 J12511 */
+	// 	    <&usb30_drd_dir_0_cfg>, /* GPIO86 J12512 */
+	// 	    <&usb30_drd_id_vbus_cfg>; /* GPIO116 J12602, GPIO117 J12515 */
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	/* Bringup stage, only enable USB2 phy */
+	maximum-speed = "high-speed";
+	reset-on-resume;
+	// wakeup-source;
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	status = "okay";
+};
+
+/* For USB verify, Must disable PCIe phy2/3/4 and related controllers first
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "okay";
+};
+
+&usb3_portc {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+*/
+
+&ufshc {
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+};
+
+&eth0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac0_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac0_phy>;
+	snps,reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <65>;
+	spacemit,rx-phase = <50>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac0_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac1_phy>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <58>;
+	spacemit,rx-phase = <70>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac1_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac2_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac2_phy>;
+	snps,reset-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <53>;
+	spacemit,rx-phase = <52>;
+
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac2_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac3_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <4096>;
+	rx-fifo-depth = <4096>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac3_phy>;
+	snps,reset-gpios = <&gpio 2 11 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <51>;
+	spacemit,rx-phase = <54>;
+
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac3_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&phy0{
+	status = "okay";
+};
+
+&phy1{
+	status = "okay";
+};
+
+&phy2{
+	status = "okay";
+};
+
+&phy3{
+	status = "okay";
+};
+
+&phy4{
+	status = "okay";
+};
+
+&phy5{
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>;
+	phy-names = "phy0";
+
+	num-lanes = <2>;
+	status = "okay";
+};
+
+&pcie1_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_1_cfg>;
+	phys = <&phy1>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&pcie2_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_2_cfg>;
+	phys = <&phy2>, <&phy3>;
+	phy-names = "phy0", "phy1";
+
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_1_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 1 2>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 2 3>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_2_cfg>;
+	status = "okay";
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 18 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <1>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 10 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	imx219@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 13 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_0_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p4v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "disabled";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "disabled";
+
+	panel0: panel0@0 {
+		status = "ok";
+		compatible = "spacemit,mipi-panel";
+		reg = <0>;
+		reset-gpios = <&gpio 1 31 GPIO_ACTIVE_HIGH>;
+		bl-gpios = <&gpio 1 29 GPIO_ACTIVE_HIGH>;
+		dc0-gpios = <&gpio 1 24 GPIO_ACTIVE_HIGH>;
+		dc1-gpios = <&gpio 1 25 GPIO_ACTIVE_HIGH>;
+		id = <2>;
+		force-attached = "lcd_icnl9911c_mipi";
+	};
+};
+
+&lcds {
+	status = "disabled";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "okay";
+};
+
+&dp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_2_cfg>;
+	status = "okay";
+};
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_4_cfg>;
+	status = "okay";
+};
+
+&adma2 {
+	status = "okay";
+};
+
+&ri2s2 {
+	status = "okay";
+};
+
+&sound_card_dp0 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp0";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp0_link_cpu>;
+		bitclock-master = <&dp0_link_cpu>;
+		dp0_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp0>;
+		};
+	};
+};
+
+&ccic0 {
+	status = "okay";
+};
+
+&ccic1 {
+	status = "okay";
+};
+
+&ccic2 {
+	status = "okay";
+};
+
+&ccic3 {
+	status = "okay";
+};
+
+&csiphy0 {
+	status = "okay";
+};
+
+&csiphy1 {
+	status = "okay";
+};
+
+&csiphy2 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_evb2-2.dts b/arch/riscv/boot/dts/spacemit/k3_evb2-2.dts
new file mode 100644
index 000000000000..c6da547ee94e
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_evb2-2.dts
@@ -0,0 +1,1533 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "lcd/lcd_icnl9911c_mipi.dtsi"
+#include "k3-dp0.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 EVB2-2";
+	compatible = "spacemit,k3-evb2-2";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p4v: p4v {
+		compatible = "regulator-fixed";
+		regulator-name = "p4v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 clk_ignore_unused loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+
+	vmmc_sdio: regulator-vmmc-sdio {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmc-sdio";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpio = <&gpio 3 6 GPIO_ACTIVE_HIGH>;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+
+		reset-gpios = <&gpio 3 4 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	i2c0-2-cfg {
+		i2c0-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c1-1-cfg {
+		i2c1-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c2-1-cfg {
+		i2c2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	i2c3-2-cfg {
+		i2c3-0-pins {
+			power-source = <3300>;
+		};
+	};
+
+	i2c6-0-cfg {
+		i2c6-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	uart2-0-cfg {
+		uart2-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	ssp0-2-cfg  {
+		ssp0-0-pins {
+			power-source = <3300>;
+		};
+
+		ssp0-1-pins {
+			power-source = <3300>;
+		};
+	};
+
+	usb20_host_drv-3-cfg {
+		usb20_host_drv-pins {
+			power-source  = <1800>;
+		};
+	};
+
+	usb30_drd-id-vbus-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(116, 2)>,	/* usb30 drd id */
+				 <K3_PADCONF(117, 2)>;	/* usb30 drd vbuson */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <47>;		/* DS8 */
+		};
+	};
+
+	gmac0-cfg {
+		/delete-node/ gmac0-2-pins;
+		/delete-node/ gmac0-4-pins;
+		/delete-node/ gmac0-5-pins;
+
+		gmac0-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-6-pins {
+			pinmux = <K3_PADCONF(15, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac2-cfg {
+		/delete-node/ gmac2-3-pins;
+		/delete-node/ gmac2-4-pins;
+
+		gmac2-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-2-pins {
+			power-source = <1800>;
+		};
+
+		gmac2-5-pins {
+			pinmux = <K3_PADCONF(58, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac3-cfg {
+		/delete-node/ gmac3-3-pins;
+		/delete-node/ gmac3-4-pins;
+
+		gmac3-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-2-pins {
+			power-source = <1800>;
+		};
+
+		gmac3-5-pins {
+			pinmux = <K3_PADCONF(75, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	mmc2-cfg {
+		mmc2-0-pins {
+			power-source = <1800>;
+		};
+
+		mmc2-1-pins {
+			power-source = <1800>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			bias-pull-up = <1>;		/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie1-1-cfg {
+		pcie1-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie2-2-cfg {
+		pcie2-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie3-1-cfg {
+		pcie3-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(86, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_0_cfg>;
+	status = "disabled";
+
+	bluetooth {
+		compatible = "realtek,rtl8852bs-bt";
+		device-wake-gpios = <&gpio 2 30 GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio 2 29 GPIO_ACTIVE_HIGH>;
+		host-wake-gpios = <&gpio 2 28 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "disabled";
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 2 22 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&sdio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc2_cfg>;
+	bus-width = <4>;
+	non-removable;
+	vmmc-supply = <&vmmc_sdio>;
+	vqmmc-supply = <&p1v8>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	no-mmc;
+	no-sd;
+	keep-power-in-suspend;
+	clock-frequency = <375000000>;
+	status = "disabled";
+};
+
+&emmc {
+	bus-width = <8>;
+	non-removable;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	clock-frequency = <375000000>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb20_host_drv_3_cfg>; /* GPIO103 */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	// pinctrl-names = "default";
+	// pinctrl-0 = <&usb30_drd_int_1_cfg>, /* GPIO119 J12511 */
+	// 	    <&usb30_drd_dir_0_cfg>, /* GPIO86 J12512 */
+	// 	    <&usb30_drd_id_vbus_cfg>; /* GPIO116 J12602, GPIO117 J12515 */
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	/* Bringup stage, only enable USB2 phy */
+	maximum-speed = "high-speed";
+	reset-on-resume;
+	// wakeup-source;
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	status = "okay";
+};
+
+/* For USB verify, Must disable PCIe phy2/3/4 and related controllers first
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "okay";
+};
+
+&usb3_portc {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	reset-on-resume;
+	// wakeup-source;
+	status = "okay";
+};
+*/
+
+&ufshc {
+	status = "okay";
+};
+
+&ec_master {
+	status = "okay";
+};
+
+&eth0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac0_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac0_phy>;
+	snps,reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <65>;
+	spacemit,rx-phase = <50>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac0_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac1_phy>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <58>;
+	spacemit,rx-phase = <70>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac1_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac2_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac2_phy>;
+	snps,reset-gpios = <&gpio 1 26 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <53>;
+	spacemit,rx-phase = <52>;
+
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac2_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&eth3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac3_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <4096>;
+	rx-fifo-depth = <4096>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&gmac3_phy>;
+	snps,reset-gpios = <&gpio 2 11 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <51>;
+	spacemit,rx-phase = <54>;
+
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		gmac3_phy: ethernet-phy@1 {
+			compatible = "ethernet-phy-id001c.c916",
+				     "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+			device_type = "ethernet-phy";
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@1 {
+					reg = <1>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_GREEN>;
+					default-state = "keep";
+				};
+
+				led@2 {
+					reg = <2>;
+					function = LED_FUNCTION_LAN;
+					color = <LED_COLOR_ID_YELLOW>;
+					default-state = "keep";
+				};
+			};
+		};
+	};
+};
+
+&phy0{
+	status = "okay";
+};
+
+&phy1{
+	status = "okay";
+};
+
+&phy2{
+	status = "okay";
+};
+
+&phy3{
+	status = "okay";
+};
+
+&phy4{
+	status = "okay";
+};
+
+&phy5{
+	status = "okay";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>;
+	phy-names = "phy0";
+
+	num-lanes = <2>;
+	status = "okay";
+};
+
+&pcie1_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_1_cfg>;
+	phys = <&phy1>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&pcie2_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_2_cfg>;
+	phys = <&phy2>, <&phy3>;
+	phy-names = "phy0", "phy1";
+
+	status = "okay";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_1_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy0";
+
+	status = "okay";
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <500>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			gpu_alert0: gpu-alert0 {
+				temperature = <65000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert1: gpu-alert1 {
+				temperature = <75000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_alert2: gpu-alert2 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			gpu_crit: gpu-crit {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&gpu_alert0>;
+				cooling-device = <&imggpu 0 1>;
+			};
+
+			map1 {
+				trip = <&gpu_alert1>;
+				cooling-device = <&imggpu 1 2>;
+			};
+
+			map2 {
+				trip = <&gpu_alert2>;
+				cooling-device = <&imggpu 2 3>;
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <1005000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_2_cfg>;
+	status = "okay";
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <0>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 18 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_1_cfg>;
+	status = "okay";
+
+	ov5647@36 {
+		compatible = "ovti,ov5647";
+		reg = <0x36>;
+		csi-id = <1>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 10 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	imx219@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+		csi-id = <2>;
+		vdd-supply = <&aldo3>;
+		pwdn-gpios = <&gpio 1 13 GPIO_ACTIVE_HIGH>;
+		status = "okay";
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_0_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p4v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "disabled";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "disabled";
+
+	panel0: panel0@0 {
+		status = "ok";
+		compatible = "spacemit,mipi-panel";
+		reg = <0>;
+		reset-gpios = <&gpio 1 31 GPIO_ACTIVE_HIGH>;
+		bl-gpios = <&gpio 1 29 GPIO_ACTIVE_HIGH>;
+		dc0-gpios = <&gpio 1 24 GPIO_ACTIVE_HIGH>;
+		dc1-gpios = <&gpio 1 25 GPIO_ACTIVE_HIGH>;
+		id = <2>;
+		force-attached = "lcd_icnl9911c_mipi";
+	};
+};
+
+&lcds {
+	status = "disabled";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "okay";
+};
+
+&dp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_2_cfg>;
+	status = "okay";
+};
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "okay";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_4_cfg>;
+	status = "okay";
+};
+
+&adma2 {
+	status = "okay";
+};
+
+&ri2s2 {
+	status = "okay";
+};
+
+&sound_card_dp0 {
+	status = "okay";
+	simple-audio-card,name = "snd-dp0";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp0_link_cpu>;
+		bitclock-master = <&dp0_link_cpu>;
+		dp0_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp0>;
+		};
+	};
+};
+
+&ccic0 {
+	status = "okay";
+};
+
+&ccic1 {
+	status = "okay";
+};
+
+&ccic2 {
+	status = "okay";
+};
+
+&ccic3 {
+	status = "okay";
+};
+
+&csiphy0 {
+	status = "okay";
+};
+
+&csiphy1 {
+	status = "okay";
+};
+
+&csiphy2 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_fpga_1x1.dts b/arch/riscv/boot/dts/spacemit/k3_fpga_1x1.dts
new file mode 100644
index 000000000000..3be86687989b
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_fpga_1x1.dts
@@ -0,0 +1,472 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "lcd/lcd_gc9503v_mipi.dtsi"
+#include "k3-camera.dtsi"
+#include "k3-camera-sensor-fpga.dtsi"
+
+/ {
+	model = "SpacemiT K3 FPGA";
+	compatible = "spacemit,k3-fpga";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <10000000>;
+	};
+
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	aux_5v0: aux_5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_5v0";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p1v8>;
+		status = "disabled";
+	};
+
+	aux_3v3: aux_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "aux_3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&aux_5v0>;
+		status = "disabled";
+	};
+
+	firmware {
+		optee {
+			compatible = "linaro,optee-tz";
+			method = "smc";
+			mboxes = <&mpxy_mbox 0x1000 0x2>,
+				<&mpxy_mbox 0x1001 0x2>,
+				<&mpxy_mbox 0x1002 0x2>,
+				<&mpxy_mbox 0x1003 0x2>,
+				<&mpxy_mbox 0x1004 0x2>,
+				<&mpxy_mbox 0x1005 0x2>,
+				<&mpxy_mbox 0x1006 0x2>,
+				<&mpxy_mbox 0x1007 0x2>,
+				<&mpxy_mbox 0x1008 0x2>,
+				<&mpxy_mbox 0x1009 0x2>,
+				<&mpxy_mbox 0x100a 0x2>,
+				<&mpxy_mbox 0x100b 0x2>,
+				<&mpxy_mbox 0x100c 0x2>,
+				<&mpxy_mbox 0x100d 0x2>,
+				<&mpxy_mbox 0x100e 0x2>,
+				<&mpxy_mbox 0x100f 0x2>;
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&qspi {
+	status = "disabled";
+};
+
+&usb3_porta {
+	status = "disabled";
+	maximum-speed = "high-speed";
+	dr_mode = "peripheral";
+	/delete-property/ phy_type;
+	phy_type = "utmi_wide";
+};
+
+&eth0 {
+	max-speed = <100>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&rgmii0>;
+	snps,reset-gpios = <&gpio 1 8 GPIO_ACTIVE_LOW>;
+
+	spacemit,wake-irq-enable;
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-reg;
+	spacemit,tx-phase = <0>;
+	spacemit,rx-phase = <0>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		rgmii0: phy@0 {
+			compatible = "ethernet-phy-id0.11a", "ethernet-phy-ieee802.3-c22";
+			reg = <0x0>;
+
+			motorcomm,clk-out-frequency-hz = <125000000>;
+			motorcomm,keep-pll-enabled;
+			motorcomm,auto-sleep-disabled;
+		};
+	};
+};
+
+&eth3 {
+	max-speed = <100>;
+	tx-fifo-depth = <4096>;
+	rx-fifo-depth = <4096>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&rgmii3>;
+	snps,reset-gpios = <&gpio 2 16 GPIO_ACTIVE_LOW>;
+
+	spacemit,wake-irq-enable;
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-reg;
+	spacemit,tx-phase = <0>;
+	spacemit,rx-phase = <0>;
+
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		rgmii3: phy@3 {
+			compatible = "ethernet-phy-id0.11a", "ethernet-phy-ieee802.3-c22";
+			reg = <0x0>;
+
+			motorcomm,clk-out-frequency-hz = <125000000>;
+			motorcomm,keep-pll-enabled;
+			motorcomm,auto-sleep-disabled;
+		};
+	};
+};
+
+&sdcard {
+	bus-width = <4>;
+	broken-cd;
+	cap-sd-highspeed;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <10000000>;
+	status = "disabled";
+};
+
+&emmc {
+	bus-width = <8>;
+	non-removable;
+	cap-mmc-highspeed;
+	no-sd;
+	no-sdio;
+	clock-frequency = <10000000>;
+	status = "disabled";
+};
+
+&rpmi_regulator {
+	status = "disabled";
+/*
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3450000>;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3450000>;
+	};
+
+	dcdc3: dcdc3 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <1800000>;
+	};
+
+	dcdc4: dcdc4 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3300000>;
+	};
+
+	dcdc5: dcdc5 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3450000>;
+	};
+
+	dcdc6: dcdc6 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3450000>;
+	};
+
+	aldo1: aldo1 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	aldo2: aldo2 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	aldo3: aldo3 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	dldo1: dldo1 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	dldo2: dldo2 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	dldo3: dldo3 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	dldo4: dldo4 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	dldo5: dldo5 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	dldo6: dldo6 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	dldo7: dldo7 {
+                regulator-min-microvolt = <500000>;
+                regulator-max-microvolt = <3400000>;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+*/
+};
+
+&i2c8 {
+	/* pinctrl-names = "default"; */
+	/* pinctrl-0 = <&pinctrl_i2c8>; */
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&aux_5v0>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			/* buck */
+			buck1: buck1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+			};
+
+			/* aldo */
+			aldo1: aldo1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			/* dldo */
+			dldo1: dldo1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+		};
+	};
+};
+
+&pcie0_rc {
+	max-link-speed = <1>;
+	num-lanes = <1>;
+	status = "disabled";
+};
+
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "disabled";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "disabled";
+
+	panel0: panel0@0 {
+		status = "ok";
+		compatible = "spacemit,mipi-panel";
+		reg = <0>;
+		gpios-dc = <16 18>;
+		gpios-reset = <17>;
+		id = <2>;
+		force-attached = "lcd_gc9503v_mipi";
+	};
+};
+
+&lcds {
+	status = "disabled";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_gemini_c0.dts b/arch/riscv/boot/dts/spacemit/k3_gemini_c0.dts
new file mode 100644
index 000000000000..94dcea350661
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_gemini_c0.dts
@@ -0,0 +1,1134 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "lcd/lcd_icnl9911c_mipi.dtsi"
+#include "k3-dp0.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 Gemini C0";
+	compatible = "spacemit,k3-gemini-c0";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p4v: p4v {
+		compatible = "regulator-fixed";
+		regulator-name = "p4v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	i2c6-0-cfg {
+		i2c6-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	ssp0-2-cfg  {
+		ssp0-0-pins {
+			power-source = <3300>;
+		};
+
+		ssp0-1-pins {
+			power-source = <3300>;
+		};
+	};
+
+	usb20_host_drv-3-cfg {
+		usb20_host_drv-pins {
+			power-source  = <1800>;
+		};
+	};
+
+	usb30_drd-id-vbus-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(116, 2)>,	/* usb30 drd id */
+				 <K3_PADCONF(117, 2)>;	/* usb30 drd vbuson */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <47>;		/* DS8 */
+		};
+	};
+
+	gmac0-cfg {
+		/delete-node/ gmac0-2-pins;
+		/delete-node/ gmac0-4-pins;
+		/delete-node/ gmac0-5-pins;
+
+		gmac0-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac0-6-pins {
+			pinmux = <K3_PADCONF(15, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	gmac1-cfg {
+		/delete-node/ gmac1-2-pins;
+		/delete-node/ gmac1-4-pins;
+		/delete-node/ gmac1-5-pins;
+
+		gmac1-0-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-1-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-3-pins {
+			power-source = <1800>;
+		};
+
+		gmac1-6-pins {
+			pinmux = <K3_PADCONF(37, 0)>;
+
+			bias-disable;
+			drive-strength = <25>;
+			power-source = <1800>;
+		};
+	};
+
+	pcie0-1-cfg {
+		pcie0-0-pins {
+			bias-pull-up = <1>;		/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie1-1-cfg {
+		pcie1-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	pcie2-2-cfg {
+		pcie2-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie3-1-cfg {
+		pcie3-0-pins {
+			bias-pull-up;			/* pull up  */
+			drive-strength = <33>;		/* DS15 */
+			power-source = <1800>;
+		};
+	};
+
+	pcie4-1-cfg {
+		pcie4-0-pins {
+			bias-pull-up;			/* pull up */
+			drive-strength = <38>;		/* DS15 */
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(86, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "disabled";
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 2 22 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&emmc {
+	bus-width = <8>;
+	non-removable;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	clock-frequency = <375000000>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb20_host_drv_3_cfg>; /* GPIO103 */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	// pinctrl-names = "default";
+	// pinctrl-0 = <&usb30_drd_int_1_cfg>, /* GPIO119 J12511 */
+	// 	    <&usb30_drd_dir_0_cfg>, /* GPIO86 J12512 */
+	// 	    <&usb30_drd_id_vbus_cfg>; /* GPIO116 J12602, GPIO117 J12515 */
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	/* Bringup stage, only enable USB2 phy */
+	maximum-speed = "high-speed";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	status = "okay";
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	/* Bringup stage, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	phys = <&usb3_portb_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "okay";
+};
+
+&usb3_portc {
+	/* Bringup stage, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	phys = <&usb3_portc_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	/* Bringup stage, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	phys = <&usb3_portd_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&eth0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac0_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&rgmii0>;
+	snps,reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <65>;
+	spacemit,rx-phase = <50>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		rgmii0: phy@0 {
+			compatible = "ethernet-phy-id001c.c916";
+			reg = <0x1>;
+			device_type = "ethernet-phy";
+		};
+	};
+};
+
+&eth1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_cfg>;
+
+	max-speed = <1000>;
+	tx-fifo-depth = <8192>;
+	rx-fifo-depth = <8192>;
+	snps,tso;
+	snps,force_sf_dma_mode;
+	phy-mode = "rgmii";
+	phy-handle = <&rgmii1>;
+	snps,reset-gpios = <&gpio 1 5 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <0 20000 100000>;
+
+	spacemit,wake-irq-enable;
+
+	spacemit,clk-tuning-enable;
+	spacemit,clk-tuning-by-delayline;
+	spacemit,tx-phase = <58>;
+	spacemit,rx-phase = <70>;
+
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		rgmii1: phy@1 {
+			compatible = "ethernet-phy-id001c.c916";
+			reg = <0x1>;
+			device_type = "ethernet-phy";
+		};
+	};
+};
+
+&phy0{
+	status = "disabled";
+};
+
+&phy1{
+	status = "disabled";
+};
+
+&phy2{
+	status = "disabled";
+};
+
+&phy3{
+	status = "disabled";
+};
+
+&phy4{
+	status = "disabled";
+};
+
+&phy5{
+	status = "disabled";
+};
+
+&pcie0_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_1_cfg>;
+	phys = <&phy0>;
+	phy-names = "phy0";
+
+	num-lanes = <2>;
+	status = "disabled";
+};
+
+&pcie1_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_1_cfg>;
+	phys = <&phy1>;
+	phy-names = "phy0";
+
+	status = "disabled";
+};
+
+&pcie2_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_2_cfg>;
+	phys = <&phy2>, <&phy3>;
+	phy-names = "phy0", "phy1";
+
+	status = "disabled";
+};
+
+&pcie3_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3_1_cfg>;
+	phys = <&phy4>;
+	phy-names = "phy0";
+
+	status = "disabled";
+};
+
+&pcie4_rc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie4_1_cfg>;
+	phys = <&phy5>;
+	phy-names = "phy0";
+
+	status = "disabled";
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			thermal_gpu_trip0: thermal_gpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <10000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_0_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p4v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
+
+&dpu_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "disabled";
+};
+
+&dpu_crtc1 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dsi0 {
+	status = "disabled";
+
+	panel0: panel0@0 {
+		status = "ok";
+		compatible = "spacemit,mipi-panel";
+		reg = <0>;
+		reset-gpios = <&gpio 1 31 GPIO_ACTIVE_HIGH>;
+		bl-gpios = <&gpio 1 29 GPIO_ACTIVE_HIGH>;
+		dc0-gpios = <&gpio 1 24 GPIO_ACTIVE_HIGH>;
+		dc1-gpios = <&gpio 1 25 GPIO_ACTIVE_HIGH>;
+		id = <2>;
+		force-attached = "lcd_icnl9911c_mipi";
+	};
+};
+
+&lcds {
+	status = "disabled";
+};
+
+&dpu0_crtc0 {
+	memory-region = <&dpu_resv0>;
+	status = "disabled";
+};
+
+&dp0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp0_2_cfg>;
+	status = "disabled";
+};
+
+&dpu1_crtc0 {
+	memory-region = <&dpu_resv1>;
+	status = "disabled";
+};
+
+&dp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dp1_4_cfg>;
+	status = "disabled";
+};
+
+&adma2 {
+	status = "disabled";
+};
+
+&ri2s2 {
+	status = "disabled";
+};
+
+&sound_card_dp0 {
+	status = "disabled";
+	simple-audio-card,name = "snd-dp0";
+	simple-audio-card,mclk-fs = <512>;
+	simple-audio-card,dai-link@0 {
+		format = "left_j";
+		frame-master = <&dp0_link_cpu>;
+		bitclock-master = <&dp0_link_cpu>;
+		dp0_link_codec: codec {
+			playback-only;
+			sound-dai = <&dp0>;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3_gemini_c1.dts b/arch/riscv/boot/dts/spacemit/k3_gemini_c1.dts
new file mode 100644
index 000000000000..1f2d7bc6c42c
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_gemini_c1.dts
@@ -0,0 +1,815 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+#include "k3.dtsi"
+#include "k3-rdomain.dtsi"
+#include "k3-pinctrl.dtsi"
+#include "k3_opp_table.dtsi"
+#include "lcd_dsi_panel.dtsi"
+#include "lcd/lcd_icnl9911c_mipi.dtsi"
+#include "k3-dp0.dtsi"
+#include "k3-dp1.dtsi"
+#include "k3-camera.dtsi"
+
+/ {
+	model = "SpacemiT K3 Gemini C1";
+	compatible = "spacemit,k3-gemini-c1";
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <24000000>;
+	};
+
+/*
+	p12v: p12v {
+		compatible = "regulator-fixed";
+		regulator-name = "p12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		status = "disabled";
+	};
+
+	p5v: p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "p5v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p4v: p4v {
+		compatible = "regulator-fixed";
+		regulator-name = "p4v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p1v8: p1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "p1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+
+	p3v3: p3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "p3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&p12v>;
+		status = "disabled";
+	};
+*/
+	chosen {
+		bootargs = "earlycon=sbi console=ttyS0,115200 loglevel=8 random.trust_bootloader=1 unaligned_scalar_speed=fast unaligned_vector_speed=fast";
+		rng-seed = <0x25d69b2 0xff555073 0xd23238ea 0x57aa5455 0x792478ed 0xa744f28e 0x6ba4fc54 0xa2bf20fc>;
+		stdout-path = "serial0:115200";
+	};
+};
+
+&resmem {
+	/* global autoconfigured region for contiguous allocations */
+	cmamem: linux,cma {
+		compatible = "shared-dma-pool";
+		reusable;
+		size = <0 0x20000000>;
+		alloc-ranges = <1 0x40000000 0 0x20000000>;
+		linux,cma-default;
+	};
+};
+
+&pinctrl {
+	i2c6-0-cfg {
+		i2c6-0-pins {
+			power-source = <1800>;
+		};
+	};
+
+	usb20_host_drv-3-cfg {
+		usb20_host_drv-pins {
+			power-source  = <1800>;
+		};
+	};
+
+	usb30_drd-id-vbus-cfg {
+		usb30_drd-pins {
+			pinmux = <K3_PADCONF(116, 2)>,	/* usb30 drd id */
+				 <K3_PADCONF(117, 2)>;	/* usb30 drd vbuson */
+
+			bias-pull-down;			/* normal pull-down */
+			drive-strength = <47>;		/* DS8 */
+		};
+	};
+
+	mmc1_cd_cfg: mmc1-cd-cfg {
+		mmc1-cd-pins {
+			pinmux = <K3_PADCONF(86, 0)>;	/* cd gpio */
+
+			bias-pull-up = <1>;
+			drive-strength = <8>;
+			power-source = <3300>;
+		};
+	};
+
+	qspi-cfg {
+		qspi-pins {
+			power-source = <1800>;
+		};
+	};
+
+	qspi-cs0-cfg {
+		qspi-cs0-pins {
+			power-source = <1800>;
+		};
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg &qspi_cs0_cfg>;
+	status = "disabled";
+};
+
+&sdcard {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg &mmc1_cd_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg &mmc1_cd_cfg>;
+	bus-width = <4>;
+	wp-inverted;
+	cd-gpios = <&gpio 2 22 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&p3v3>;
+	vqmmc-supply = <&aldo1>;
+	no-mmc;
+	no-sdio;
+	clock-frequency = <204800000>;
+	spacemit,tx_delaycode = <0x1f>;
+	status = "okay";
+};
+
+&emmc {
+	bus-width = <8>;
+	non-removable;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	clock-frequency = <375000000>;
+	status = "okay";
+};
+
+&usb2_host_u2phy {
+	status = "okay";
+};
+
+&usb2_host {
+	pinctrl-names = "default";
+	pinctrl-0 = <&usb20_host_drv_3_cfg>; /* GPIO103 */
+	status = "okay";
+};
+
+&usb3_porta_u2phy {
+	status = "okay";
+};
+
+&usb3_porta_u3phy {
+	// pinctrl-names = "default";
+	// pinctrl-0 = <&usb30_drd_int_1_cfg>, /* GPIO119 J12511 */
+	// 	    <&usb30_drd_dir_0_cfg>, /* GPIO86 J12512 */
+	// 	    <&usb30_drd_id_vbus_cfg>; /* GPIO116 J12602, GPIO117 J12515 */
+	status = "okay";
+};
+
+&usb3_porta {
+	dr_mode = "otg";
+	/* Bringup stage, only enable USB2 phy */
+	maximum-speed = "high-speed";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	status = "okay";
+};
+
+&usb3_portb_u2phy {
+	status = "okay";
+};
+
+&usb3_portb_u3phy {
+	status = "okay";
+};
+
+&usb3_portb {
+	/* Bringup stage, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	phys = <&usb3_portb_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portc_u2phy {
+	status = "okay";
+};
+
+&usb3_portc_u3phy {
+	status = "okay";
+};
+
+&usb3_portc {
+	/* Bringup stage, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	phys = <&usb3_portc_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb3_portd_u2phy {
+	status = "okay";
+};
+
+&usb3_portd_u3phy {
+	status = "okay";
+};
+
+&usb3_portd {
+	/* Bringup stage, only enable USB2 phy */
+	/delete-property/ phys;
+	/delete-property/ phy-names;
+	maximum-speed = "high-speed";
+	phys = <&usb3_portd_u2phy>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&ufshc {
+	status = "okay";
+};
+
+&thermal_zones {
+	thermal_top {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 0>;
+
+		trips {
+			thermal_top_trip0: thermal_top-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_vpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 2>;
+
+		trips {
+			thermal_vpu_trip0: thermal_vpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_gpu {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 3>;
+
+		trips {
+			thermal_gpu_trip0: thermal_gpu-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	thermal_cluster0 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 4>;
+
+		trips {
+			thermal_cluster0_trip0: thermal_cluster0-trip0 {
+				temperature = <35000>;
+				hysteresis = <5000>;
+				type = "active";
+			};
+		};
+	};
+
+	thermal_cluster1 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 5>;
+
+		trips {
+			thermal_cluster1_trip0: thermal_cluster1-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster1_trip1: thermal_cluster1-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip2: thermal_cluster1-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster1_trip3: thermal_cluster1-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster1_trip0>;
+				cooling-device = <&cpu_0 2 2>,
+						<&cpu_1 2 2>,
+						<&cpu_2 2 2>,
+						<&cpu_3 2 2>,
+						<&cpu_4 2 2>,
+						<&cpu_5 2 2>,
+						<&cpu_6 2 2>,
+						<&cpu_7 2 2>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster1_trip1>;
+				cooling-device = <&cpu_0 3 5>,
+						<&cpu_1 3 5>,
+						<&cpu_2 3 5>,
+						<&cpu_3 3 5>,
+						<&cpu_4 3 5>,
+						<&cpu_5 3 5>,
+						<&cpu_6 3 5>,
+						<&cpu_7 3 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster1_trip2>;
+				cooling-device = <&cpu_0 6 8>,
+						<&cpu_1 6 8>,
+						<&cpu_2 6 8>,
+						<&cpu_3 6 8>,
+						<&cpu_4 6 8>,
+						<&cpu_5 6 8>,
+						<&cpu_6 6 8>,
+						<&cpu_7 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster2 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 6>;
+
+		trips {
+			thermal_cluster2_trip0: thermal_cluster2-trip0 {
+				temperature = <85000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+
+			thermal_cluster2_trip1: thermal_cluster2-trip1 {
+				temperature = <95000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip2: thermal_cluster2-trip2 {
+				temperature = <105000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			thermal_cluster2_trip3: thermal_cluster2-trip3 {
+				temperature = <115000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&thermal_cluster2_trip0>;
+				cooling-device = <&cpu_8 3 3>,
+						<&cpu_9 3 3>,
+						<&cpu_10 3 3>,
+						<&cpu_11 3 3>,
+						<&cpu_12 3 3>,
+						<&cpu_13 3 3>,
+						<&cpu_14 3 3>,
+						<&cpu_15 3 3>;
+			};
+
+			map1 {
+				trip = <&thermal_cluster2_trip1>;
+				cooling-device = <&cpu_8 4 5>,
+						<&cpu_9 4 5>,
+						<&cpu_10 4 5>,
+						<&cpu_11 4 5>,
+						<&cpu_12 4 5>,
+						<&cpu_13 4 5>,
+						<&cpu_14 4 5>,
+						<&cpu_15 4 5>;
+			};
+
+			map2 {
+				trip = <&thermal_cluster2_trip2>;
+				cooling-device = <&cpu_8 6 8>,
+						<&cpu_9 6 8>,
+						<&cpu_10 6 8>,
+						<&cpu_11 6 8>,
+						<&cpu_12 6 8>,
+						<&cpu_13 6 8>,
+						<&cpu_14 6 8>,
+						<&cpu_15 6 8>;
+			};
+		};
+	};
+
+	thermal_cluster3 {
+		polling-delay = <1000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&thermal 7>;
+
+		trips {
+			thermal_cluster7_trip0: thermal_cluster7-trip0 {
+				temperature = <40000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+};
+
+&rpmi_regulator {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg_rcpu>;
+
+	pvin: pvin {
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc2: edcdc2 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	edcdc1: edcdc1 {
+		regulator-min-microvolt = <534000>;
+		regulator-max-microvolt = <10000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p3v3: p3v3 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	p1v8: p1v8 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc1: dcdc1 {
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc3: dcdc3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc4: dcdc4 {
+		regulator-min-microvolt = <2100000>;
+		regulator-max-microvolt = <2100000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc5: dcdc5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dcdc6: dcdc6 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <600000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo1: aldo1 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo2: aldo2 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	aldo3: aldo3 {
+		regulator-min-microvolt = <500000>;
+		regulator-max-microvolt = <3400000>;
+	};
+
+	aldo4: aldo4 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo1: dldo1 {
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo2: dldo2 {
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo3: dldo3 {
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo4: dldo4 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo5: dldo5 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo6: dldo6 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	dldo7: dldo7 {
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_x100: pwr_x100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_a100: pwr_a100 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	pwr_ext: pwr_ext {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_0_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		read-only;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product_name: product-name {
+			};
+		};
+	};
+};
+
+/*
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "disabled";
+
+	p1@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		status = "disabled";
+
+		vcc-supply = <&p4v>;
+
+		regulators {
+			compatible = "spacemit-p1-regulator";
+
+			buck1: buck1 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck2: buck2 {
+				regulator-min-microvolt = <1050000>;
+				regulator-max-microvolt = <1050000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck3: buck3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck4: buck4 {
+				regulator-min-microvolt = <2100000>;
+				regulator-max-microvolt = <2100000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <500000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo2: aldo2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			aldo3: aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4: aldo4 {
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo1: dldo1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo2: dldo2 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo3: dldo3 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo4: dldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo5: dldo5 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo6: dldo6 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			dldo7: dldo7 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+*/
diff --git a/arch/riscv/boot/dts/spacemit/k3_opp_table.dtsi b/arch/riscv/boot/dts/spacemit/k3_opp_table.dtsi
new file mode 100644
index 000000000000..928b722a8e9e
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3_opp_table.dtsi
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2023 Spacemit, Inc */
+
+&cpus {
+	clst_core_opp_table0_x100: opp_table0_x100 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		clocks = <&pll CLK_PLL3>, <&pll CLK_PLL4>, <&pll CLK_PLL3_D1>, <&syscon_apmu CLK_APMU_CPU_C1_PLL_SRC>;
+		clock-names = "pll_clst0", "pll_clst1", "pll_src", "clt_pll_src";
+
+		opp2400000000 {
+			opp-hz = /bits/ 64 <2400000000>, /bits/ 64 <2400000000>;
+			opp-microvolt = <1000000 1000000 1004000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp2300000000 {
+			opp-hz = /bits/ 64 <2300000000>, /bits/ 64 <2300000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp2200000000 {
+			opp-hz = /bits/ 64 <2200000000>, /bits/ 64 <2200000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp2150000000 {
+			opp-hz = /bits/ 64 <2150000000>, /bits/ 64 <2150000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp2100000000 {
+			opp-hz = /bits/ 64 <2100000000>, /bits/ 64 <2100000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp2000000000 {
+			opp-hz = /bits/ 64 <2000000000>, /bits/ 64 <2000000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1900000000 {
+			opp-hz = /bits/ 64 <1900000000>, /bits/ 64 <1900000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1850000000 {
+			opp-hz = /bits/ 64 <1850000000>, /bits/ 64 <1850000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1800000000 {
+			opp-hz = /bits/ 64 <1800000000>, /bits/ 64 <1800000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1700000000 {
+			opp-hz = /bits/ 64 <1700000000>, /bits/ 64 <1700000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1600000000 {
+			opp-hz = /bits/ 64 <1600000000>, /bits/ 64 <1600000000>;
+			opp-microvolt = <880000 880000 884000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1500000000 {
+			opp-hz = /bits/ 64 <1500000000>, /bits/ 64 <1500000000>;
+			opp-microvolt = <850000 850000 854000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1400000000 {
+			opp-hz = /bits/ 64 <1400000000>, /bits/ 64 <1400000000>;
+			opp-microvolt = <850000 850000 854000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1300000000 {
+			opp-hz = /bits/ 64 <1300000000>, /bits/ 64 <1300000000>;
+			opp-microvolt = <850000 850000 854000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1200000000 {
+			opp-hz = /bits/ 64 <1200000000>, /bits/ 64 <1200000000>;
+			opp-microvolt = <850000 850000 854000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1100000000 {
+			opp-hz = /bits/ 64 <1100000000>, /bits/ 64 <1100000000>;
+			opp-microvolt = <850000 850000 854000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1000000000 {
+			opp-hz = /bits/ 64 <1000000000>, /bits/ 64 <1000000000>;
+			opp-microvolt = <800000 800000 804000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp819000000 {
+			opp-hz = /bits/ 64 <819200000>, /bits/ 64 <819200000>;
+			opp-microvolt = <800000 800000 804000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp614400000 {
+			opp-hz = /bits/ 64 <614400000>, /bits/ 64 <614400000>;
+			opp-microvolt = <800000 800000 804000>;
+			clock-latency-ns = <200000>;
+		};
+	};
+
+	clst_core_opp_table0_a100: opp_table0_a100 {
+		compatible = "operating-points-v2";
+		opp-shared;
+		clocks = <&pll CLK_PLL5>, <&pll CLK_PLL8>, <&pll CLK_PLL5_D1>, <&syscon_apmu CLK_APMU_CPU_C3_PLL_SRC>;
+		clock-names = "pll_clst0", "pll_clst1", "pll_src", "clt_pll_src";
+
+		opp2000000000 {
+			opp-hz = /bits/ 64 <2000000000>, /bits/ 64 <2000000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1900000000 {
+			opp-hz = /bits/ 64 <1900000000>, /bits/ 64 <1900000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1850000000 {
+			opp-hz = /bits/ 64 <1850000000>, /bits/ 64 <1850000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1800000000 {
+			opp-hz = /bits/ 64 <1800000000>, /bits/ 64 <1800000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1700000000 {
+			opp-hz = /bits/ 64 <1700000000>, /bits/ 64 <1700000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1600000000 {
+			opp-hz = /bits/ 64 <1600000000>, /bits/ 64 <1600000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1500000000 {
+			opp-hz = /bits/ 64 <1500000000>, /bits/ 64 <1500000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1400000000 {
+			opp-hz = /bits/ 64 <1400000000>, /bits/ 64 <1400000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1300000000 {
+			opp-hz = /bits/ 64 <1300000000>, /bits/ 64 <1300000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1200000000 {
+			opp-hz = /bits/ 64 <1200000000>, /bits/ 64 <1200000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1100000000 {
+			opp-hz = /bits/ 64 <1100000000>, /bits/ 64 <1100000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp1000000000 {
+			opp-hz = /bits/ 64 <1000000000>, /bits/ 64 <1000000000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp819000000 {
+			opp-hz = /bits/ 64 <819200000>, /bits/ 64 <819200000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp614400000 {
+			opp-hz = /bits/ 64 <614400000>, /bits/ 64 <614400000>;
+			clock-latency-ns = <200000>;
+		};
+	};
+};
+
+&cpu_0 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_1 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_2 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_3 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_4 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_5 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_6 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_7 {
+	clst-supply = <&edcdc1>;
+	clocks = <&syscon_apmu CLK_APMU_CPU_C0_CORE>, <&syscon_apmu CLK_APMU_CPU_C1_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_x100>;
+};
+
+&cpu_8 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
+
+&cpu_9 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
+
+&cpu_10 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
+
+&cpu_11 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
+
+&cpu_12 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
+
+&cpu_13 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
+
+&cpu_14 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
+
+&cpu_15 {
+	clocks = <&syscon_apmu CLK_APMU_CPU_C2_CORE>, <&syscon_apmu CLK_APMU_CPU_C3_CORE>;
+	clock-names = "cls0", "cls1";
+	operating-points-v2 = <&clst_core_opp_table0_a100>;
+};
diff --git a/arch/riscv/boot/dts/spacemit/lcd/lcd_gc9503v_mipi.dtsi b/arch/riscv/boot/dts/spacemit/lcd/lcd_gc9503v_mipi.dtsi
new file mode 100644
index 000000000000..3d175db0e20b
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/lcd/lcd_gc9503v_mipi.dtsi
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/ { lcds: lcds  {
+	lcd_gc9503v_mipi: lcd_gc9503v_mipi {
+		dsi-work-mode = <1>; /* video burst mode*/
+		dsi-lane-number = <4>;
+		dsi-color-format = "rgb888";
+
+		phy-bit-clock = <911000>;	/* kbps */
+		phy-escape-clock = <52000>;/* kHz */
+
+		width-mm = <72>;
+		height-mm = <126>;
+
+		use-dcs-write;
+
+		/*mipi info*/
+		height = <640>;
+		width = <480>;
+		hfp = <60>;
+		hbp = <60>;
+		hsync = <8>;
+		vfp = <30>;
+		vbp = <30>;
+		vsync = <8>;
+		fps = <60>;
+		work-mode = <0>;
+		rgb-mode = <3>;
+		lane-number = <2>;
+		phy-freq = <314000>;
+		split-enable = <0>;
+		eotp-enable = <0>;
+		burst-mode = <2>;
+		esd-check-enable = <0>;
+
+		te-esd-en = <0>;
+
+		/* DSI_CMD, DSI_MODE, timeout, len, cmd  */
+		initial-command = [
+			39 01 00 06 F0 55 AA 52 08 00
+			39 01 00 03 F6 5A 87
+			39 01 00 02 C1 3F
+			39 01 00 02 C2 0E
+			39 01 00 02 C6 F8
+			39 01 00 02 C9 10
+			39 01 00 02 CD 25
+			39 01 00 04 FA 08 08 08
+			39 01 00 02 F8 8A
+			39 01 00 02 71 48
+			39 01 00 02 72 48
+			39 01 00 03 73 00 44
+			39 01 00 02 97 EE
+			39 01 00 02 83 93
+			39 01 00 02 A3 22
+			39 01 00 04 FD 28 3C 00
+			39 01 00 02 AC 65
+			39 01 00 02 A7 47
+			39 01 00 02 A0 DD
+			39 01 00 02 9A C0
+			39 01 00 02 9B 40
+			39 01 00 03 82 3E 3E
+			39 01 00 02 B1 10
+			39 01 00 03 7A 13 1A
+			39 01 00 03 7B 13 1A
+			39 01 00 21 6D 00 1F 19 1A 10 0E 0C 0A 02 08 1E 1E 1E 1E 1E 1E 1E 1E 1E 1E 1E 1E 07 01 09 0B 0D 0F 1A 19 1F 00
+			39 01 00 11 64 28 04 02 7C 03 03 28 03 02 7D 03 03 7A 7A 7A 7A
+			39 01 00 11 65 28 02 02 7E 03 03 28 01 02 7F 03 03 7A 7A 7A 7A
+			39 01 00 11 66 28 00 02 80 03 03 20 01 02 81 03 03 7A 7A 7A 7A
+			39 01 00 11 67 20 02 02 82 03 03 20 03 02 83 03 03 7A 7A 7A 7A
+			39 01 00 0e 68 77 08 06 08 05 7A 7A 08 06 08 05 7A 7A
+			39 01 00 09 60 28 06 7A 7A 28 05 7A 7A
+			39 01 00 09 63 22 7E 7A 7A 22 7F 7A 7A
+			39 01 00 08 69 14 22 14 22 14 22 08
+			39 01 00 02 6B 07
+			39 01 00 35 D1 00 00 00 80 00 BF 00 F8 01 20 01 4D 01 68 01 88 01 B8 01 F0 02 28 02 7D 02 B8 02 BA 03 00 03 4D 03 6A 03 88 03 A0 03 B5 03 C8 03 D0 03 E8 03 F0 03 F8 03 FF
+			39 01 00 35 D2 00 00 00 80 00 BF 00 F8 01 20 01 4D 01 68 01 88 01 B8 01 F0 02 28 02 7D 02 B8 02 BA 03 00 03 4D 03 6A 03 88 03 A0 03 B5 03 C8 03 D0 03 E8 03 F0 03 F8 03 FF
+			39 01 00 35 D3 00 00 00 80 00 BF 00 F8 01 20 01 4D 01 68 01 88 01 B8 01 F0 02 28 02 7D 02 B8 02 BA 03 00 03 4D 03 6A 03 88 03 A0 03 B5 03 C8 03 D0 03 E8 03 F0 03 F8 03 FF
+			39 01 00 35 D4 00 00 00 80 00 BF 00 F8 01 20 01 4D 01 68 01 88 01 B8 01 F0 02 28 02 7D 02 B8 02 BA 03 00 03 4D 03 6A 03 88 03 A0 03 B5 03 C8 03 D0 03 E8 03 F0 03 F8 03 FF
+			39 01 00 35 D5 00 00 00 80 00 BF 00 F8 01 20 01 4D 01 68 01 88 01 B8 01 F0 02 28 02 7D 02 B8 02 BA 03 00 03 4D 03 6A 03 88 03 A0 03 B5 03 C8 03 D0 03 E8 03 F0 03 F8 03 FF
+			39 01 00 35 D6 00 00 00 80 00 BF 00 F8 01 20 01 4D 01 68 01 88 01 B8 01 F0 02 28 02 7D 02 B8 02 BA 03 00 03 4D 03 6A 03 88 03 A0 03 B5 03 C8 03 D0 03 E8 03 F0 03 F8 03 FF
+			39 01 78 02 11 00
+			39 01 14 02 29 00
+		];
+		sleep-in-command = [
+			39 01 14 01 28
+			39 01 78 01 10
+		];
+		sleep-out-command = [
+			05 01 78 01 11
+			05 01 14 01 29
+		];
+		read-id-command = [
+			37 01 00 01 01
+			14 01 00 01 04
+		];
+
+		display-timings {
+			timing0 {
+				clock-frequency = <98304000>;
+				hactive = <480>;
+				hfront-porch = <60>;
+				hback-porch = <60>;
+				hsync-len = <8>;
+				vactive = <640>;
+				vfront-porch = <30>;
+				vback-porch = <30>;
+				vsync-len = <8>;
+				vsync-active = <1>;
+				hsync-active = <1>;
+			};
+		};
+	};
+};};
diff --git a/arch/riscv/boot/dts/spacemit/lcd/lcd_icnl9911c_mipi.dtsi b/arch/riscv/boot/dts/spacemit/lcd/lcd_icnl9911c_mipi.dtsi
new file mode 100644
index 000000000000..f39c4b354598
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/lcd/lcd_icnl9911c_mipi.dtsi
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/ { lcds: lcds {
+	lcd_icnl9911c_mipi: lcd_icnl9911c_mipi {
+		dsi-work-mode = <1>; /* video burst mode*/
+		dsi-lane-number = <4>;
+		dsi-color-format = "rgb888";
+		width-mm = <72>;
+		height-mm = <126>;
+		use-dcs-write;
+
+		/*mipi info*/
+		height = <1600>;
+		width = <720>;
+		hfp = <48>;
+		hbp = <48>;
+		hsync = <4>;
+		vfp = <150>;
+		vbp = <32>;
+		vsync = <4>;
+		fps = <60>;
+		work-mode = <0>;
+		rgb-mode = <3>;
+		lane-number = <4>;
+		phy-freq = <624000>;
+		phy-escape-clock = <52000>;
+		phy-bit-clock = <624000000>;
+		split-enable = <0>;
+		eotp-enable = <0>;
+		burst-mode = <2>;
+		esd-check-enable = <0>;
+		vpn-tx-dly-cnt = <76>;
+		vpn-dly-cnt = <540>;
+
+		/* DSI_CMD, DSI_MODE, timeout, len, cmd */
+		initial-command = [
+			39 01 00 03 F0 5A 59
+			39 01 00 03 F1 A5 A6
+			39 01 00 21 B0 83 82 86 87 06 07 04 05 33 33 33 33 20 00 00 77 00 00 3F 05 04 03 02 01 02 03 04 00 00 00 00 00
+			39 01 00 1e B1 13 91 8E 81 20 00 00 77 00 00 04 08 54 00 00 00 44 40 02 01 40 02 01 40 02 01 40 02 01
+			39 01 00 12 B2 54 C4 82 05 40 02 01 40 02 01 05 05 54 0C 0C 0D 0B
+			39 01 00 20 B3 12 00 00 00 00 26 26 91 91 91 91 3C 26 00 18 01 02 08 20 30 08 09 44 20 40 20 40 08 09 22 33
+			39 01 00 1D B4 03 00 00 06 1E 1F 0C 0E 10 12 14 16 04 03 03 03 03 03 03 03 03 03 FF FF FC 00 00 00
+			39 01 00 1D B5 03 00 00 07 1E 1F 0D 0F 11 13 15 17 05 03 03 03 03 03 03 03 03 03 FF FF FC 00 00 00
+			39 01 00 19 B8 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+			39 01 00 03 BA 6B 6B
+			39 01 00 0E BB 01 05 09 11 0D 19 1D 55 25 69 00 21 25
+			39 01 00 0F BC 00 00 00 00 02 20 FF 00 03 33 01 73 33 02
+			39 01 00 0B BD E9 02 4F CF 72 A4 08 44 AE 15
+			39 01 00 0D BE 7D 7D 5A 46 0C 77 43 07 0E 0E 00 00
+			39 01 00 09 BF 07 25 07 25 7F 00 11 04
+			39 01 00 0D C0 10 FF FF FF FF FF 00 FF 00 00 00 00
+			39 01 00 14 C1 C0 20 20 96 04 32 32 04 2A 40 36 00 07 CF FF FF C0 00 C0
+			39 01 00 02 C2 00
+			39 01 00 10 C2 CC 01 10 00 01 30 02 21 43 00 01 30 02 21 43
+			39 01 00 0D C3 06 00 FF 00 FF 00 00 81 01 00 00 00
+			39 01 00 0D C4 84 03 2B 41 00 3C 00 03 03 3E 00 00
+			39 01 00 0C C5 03 1C C0 C0 40 10 42 44 0F 0A 14
+			39 01 00 0B C6 87 A0 2A 29 29 00 64 37 08 04
+			39 01 00 17 C7 F7 D3 BA A5 80 63 36 8B 56 2A FF CE 23 F4 D3 A4 86 5A 1A 7F E4 00
+			39 01 00 17 C8 F7 D3 BA A5 80 63 36 8B 56 2A FF CE 23 F4 D3 A4 86 5A 1A 7F E4 00
+			39 01 00 09 D0 80 0D FF 0F 61 0B 08 0C
+			39 01 00 0E D2 42 0C 30 01 80 26 04 00 00 C3 00 00 00
+			39 01 00 03 F1 5A 59
+			39 01 00 03 F0 A5 A6
+			39 01 00 02 35 00
+			39 01 96 01 11
+			39 01 32 01 29
+		];
+		sleep-in-command = [
+			39 01 00 02 26 08
+			39 01 78 01 28
+			39 01 78 01 10
+		];
+		sleep-out-command = [
+			39 01 96 01 11
+			39 01 32 01 29
+		];
+		read-id-command = [
+			37 01 00 01 01
+			14 01 00 01 04
+		];
+
+		display-timings {
+			timing0 {
+				clock-frequency = <87870000>;
+				hactive = <720>;
+				hfront-porch = <48>;
+				hback-porch = <48>;
+				hsync-len = <4>;
+				vactive = <1600>;
+				vfront-porch = <150>;
+				vback-porch = <32>;
+				vsync-len = <4>;
+				vsync-active = <1>;
+				hsync-active = <1>;
+			};
+		};
+	};
+};};
diff --git a/arch/riscv/boot/dts/spacemit/lcd/lcd_nt36523_mipi.dtsi b/arch/riscv/boot/dts/spacemit/lcd/lcd_nt36523_mipi.dtsi
new file mode 100644
index 000000000000..e4b10223fb68
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/lcd/lcd_nt36523_mipi.dtsi
@@ -0,0 +1,408 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/ { lcds: lcds {
+	lcd_nt36523_mipi: lcd_nt36523_mipi {
+		dsi-work-mode = <1>; /* video burst mode*/
+		dsi-lane-number = <8>;
+		dsi-color-format = "rgb888";
+
+		phy-bit-clock = <1200000000>;	/* kbps */
+		phy-escape-clock = <52000>;/* kHz */
+
+		width-mm = <72>;
+		height-mm = <126>;
+
+		use-dcs-write;
+
+		/*mipi info*/
+		height = <2560>;
+		width = <1600>;
+		hfp = <120>;
+		hbp = <50>;
+		hsync = <5>;
+		vfp = <26>;
+		vbp = <198>;
+		vsync = <2>;
+		fps = <60>;
+		work-mode = <0>;
+		rgb-mode = <3>;
+		lane-number = <8>;
+		phy-freq = <1050000>;
+		split-enable = <1>;
+		eotp-enable = <0>;
+		burst-mode = <2>;
+		esd-check-enable = <0>;
+
+		te-esd-en = <0>;
+
+		/* DSI_CMD, DSI_MODE, timeout, len, cmd  */
+		initial-command = [
+			39 01 00 02 FF 20
+			39 01 00 02 FB 01
+			39 01 00 02 05 69
+			39 01 00 02 07 32
+			39 01 00 02 08 32
+			39 01 00 02 0D 63
+			39 01 00 02 0E 41
+			39 01 00 02 0F 41
+			39 01 00 02 94 00
+			39 01 00 02 95 C3
+			39 01 00 02 96 EB
+			39 01 00 02 30 11
+			39 01 00 02 58 40
+			39 01 00 02 6D AA
+			39 01 00 02 75 C4
+			39 01 00 02 77 B3
+			39 01 00 02 FF 22
+			39 01 00 02 FB 01
+			39 01 00 02 9F 5E
+			39 01 00 02 6F 01
+			39 01 00 02 70 11
+			39 01 00 02 73 01
+			39 01 00 02 74 87
+			39 01 00 02 76 01
+			39 01 00 02 77 87
+			39 01 00 02 A0 3F
+			39 01 00 02 A9 50
+			39 01 00 02 AA 28
+			39 01 00 02 AB 28
+			39 01 00 02 AD 10
+			39 01 00 02 B8 00
+			39 01 00 02 B9 87
+			39 01 00 02 BA 87
+			39 01 00 02 BB 87
+			39 01 00 02 BE 04
+			39 01 00 02 BF 87
+			39 01 00 02 C0 04
+			39 01 00 02 C1 97
+			39 01 00 02 C2 00
+			39 01 00 02 C5 00
+			39 01 00 02 C6 01
+			39 01 00 02 C7 48
+			39 01 00 02 CA 43
+			39 01 00 02 CB 3C
+			39 01 00 02 CE 00
+			39 01 00 02 CF 43
+			39 01 00 02 D0 3C
+			39 01 00 02 D3 43
+			39 01 00 02 D4 3C
+			39 01 00 02 D7 00
+			39 01 00 02 DC 43
+			39 01 00 02 DD 3C
+			39 01 00 02 E1 43
+			39 01 00 02 E2 3C
+			39 01 00 02 F2 00
+			39 01 00 02 F3 01
+			39 01 00 02 F4 48
+			39 01 00 02 FF 24
+			39 01 00 02 FB 01
+			39 01 00 02 00 03
+			39 01 00 02 01 27
+			39 01 00 02 02 27
+			39 01 00 02 03 27
+			39 01 00 02 04 00
+			39 01 00 02 05 00
+			39 01 00 02 06 00
+			39 01 00 02 07 04
+			39 01 00 02 08 05
+			39 01 00 02 09 23
+			39 01 00 02 0A 22
+			39 01 00 02 0B 28
+			39 01 00 02 0C 28
+			39 01 00 02 0D 21
+			39 01 00 02 0E 29
+			39 01 00 02 0F 29
+			39 01 00 02 10 0C
+			39 01 00 02 11 0D
+			39 01 00 02 12 0E
+			39 01 00 02 13 0F
+			39 01 00 02 14 26
+			39 01 00 02 15 26
+			39 01 00 02 16 03
+			39 01 00 02 17 27
+			39 01 00 02 18 27
+			39 01 00 02 19 27
+			39 01 00 02 1A 00
+			39 01 00 02 1B 00
+			39 01 00 02 1C 00
+			39 01 00 02 1D 04
+			39 01 00 02 1E 05
+			39 01 00 02 1F 23
+			39 01 00 02 20 22
+			39 01 00 02 21 28
+			39 01 00 02 22 28
+			39 01 00 02 23 21
+			39 01 00 02 24 29
+			39 01 00 02 25 29
+			39 01 00 02 26 0C
+			39 01 00 02 27 0D
+			39 01 00 02 28 0E
+			39 01 00 02 29 0F
+			39 01 00 02 2A 26
+			39 01 00 02 2B 26
+			39 01 00 02 2F 02
+			39 01 00 02 30 01
+			39 01 00 02 33 01
+			39 01 00 02 34 02
+			39 01 00 02 37 22
+			39 01 00 02 39 00
+			39 01 00 02 3A 01
+			39 01 00 02 3B 86
+			39 01 00 02 3D 02
+			39 01 00 02 4D 43
+			39 01 00 02 4E 21
+			39 01 00 02 51 12
+			39 01 00 02 52 34
+			39 01 00 03 55 0B 0B
+			39 01 00 02 56 04
+			39 01 00 02 58 10
+			39 01 00 02 59 20
+			39 01 00 02 5A 6A
+			39 01 00 02 5B 63
+			39 01 00 03 5E 00 0B
+			39 01 00 02 5F 00
+			39 01 00 02 65 82
+			39 01 00 02 96 73
+			39 01 00 02 B1 0E
+			39 01 00 02 B2 0D
+			39 01 00 02 C4 2F
+			39 01 00 0D B6 05 00 05 00 00 00 00 00 05 05 00 00
+			39 01 00 02 60 C8
+			39 01 00 02 61 00
+			39 01 00 02 63 A0
+			39 01 00 02 91 40
+			39 01 00 02 92 87
+			39 01 00 02 93 1A
+			39 01 00 02 94 C8
+			39 01 00 02 95 80
+			39 01 00 02 AB 22
+			39 01 00 02 D7 55
+			39 01 00 02 DA 0A
+			39 01 00 02 DE 08
+			39 01 00 02 DB 71
+			39 01 00 02 DC 87
+			39 01 00 02 DD 22
+			39 01 00 02 DF 01
+			39 01 00 02 E0 87
+			39 01 00 02 E1 01
+			39 01 00 02 E2 87
+			39 01 00 02 E3 01
+			39 01 00 02 E4 87
+			39 01 00 02 E5 01
+			39 01 00 02 E6 87
+			39 01 00 02 5C 00
+			39 01 00 02 5D 00
+			39 01 00 02 8D 00
+			39 01 00 02 8E 00
+			39 01 00 02 B5 90
+			39 01 00 02 C2 DF
+			39 01 00 02 65 C2
+			39 01 00 02 B1 0E
+			39 01 00 02 B2 0D
+			39 01 00 02 FF 25
+			39 01 00 02 FB 01
+			39 01 00 02 05 00
+			39 01 00 02 19 07
+			39 01 00 02 1F 6A
+			39 01 00 02 20 63
+			39 01 00 02 26 6A
+			39 01 00 02 27 63
+			39 01 00 02 33 6A
+			39 01 00 02 34 63
+			39 01 00 02 3F 00
+			39 01 00 02 44 01
+			39 01 00 02 45 48
+			39 01 00 02 48 6A
+			39 01 00 02 49 63
+			39 01 00 02 5B 00
+			39 01 00 02 5C 00
+			39 01 00 02 5D 00
+			39 01 00 02 5E D0
+			39 01 00 02 61 6A
+			39 01 00 02 62 63
+			39 01 00 02 F1 10
+			39 01 00 02 FF 2A
+			39 01 00 02 FB 01
+			39 01 00 02 64 96
+			39 01 00 02 6A 9E
+			39 01 00 02 76 9E
+			39 01 00 02 79 96
+			39 01 00 02 7C 96
+			39 01 00 02 7F 96
+			39 01 00 02 82 16
+			39 01 00 02 85 9E
+			39 01 00 02 88 96
+			39 01 00 02 A2 FF
+			39 01 00 02 A3 FF
+			39 01 00 02 A4 F3
+			39 01 00 02 A5 FF
+			39 01 00 02 D6 08
+			39 01 00 02 FF 26
+			39 01 00 02 FB 01
+			39 01 00 02 00 81
+			39 01 00 02 01 30
+			39 01 00 02 02 31
+			39 01 00 02 0A F2
+			39 01 00 02 04 5E
+			39 01 00 02 06 32
+			39 01 00 02 0A F5
+			39 01 00 02 0C 16
+			39 01 00 02 0D 0D
+			39 01 00 02 0F 00
+			39 01 00 02 11 00
+			39 01 00 02 12 50
+			39 01 00 02 13 60
+			39 01 00 02 14 70
+			39 01 00 02 15 00
+			39 01 00 02 16 10
+			39 01 00 02 17 A0
+			39 01 00 02 18 86
+			39 01 00 02 19 16
+			39 01 00 02 1A 7F
+			39 01 00 02 1B 15
+			39 01 00 02 1C 90
+			39 01 00 02 22 00
+			39 01 00 02 23 00
+			39 01 00 02 2A 16
+			39 01 00 02 2B 7F
+			39 01 00 02 1D 00
+			39 01 00 02 1E 87
+			39 01 00 02 1F 87
+			39 01 00 02 24 00
+			39 01 00 02 25 87
+			39 01 00 02 2F 04
+			39 01 00 02 30 87
+			39 01 00 02 31 04
+			39 01 00 02 32 9F
+			39 01 00 02 39 00
+			39 01 00 02 3A 87
+			39 01 00 02 20 01
+			39 01 00 02 33 66
+			39 01 00 02 34 66
+			39 01 00 02 35 66
+			39 01 00 02 8B 50
+			39 01 00 02 FF 25
+			39 01 00 02 FB 01
+			39 01 00 02 13 02
+			39 01 00 02 14 41
+			39 01 00 02 BC 02
+			39 01 00 02 BD 41
+			39 01 00 02 FF 2A
+			39 01 00 02 FB 01
+			39 01 00 02 99 95
+			39 01 00 02 9A 03
+			39 01 00 02 FF 27
+			39 01 00 02 FB 01
+			39 01 00 02 56 06
+			39 01 00 02 58 80
+			39 01 00 02 59 93
+			39 01 00 02 5A 00
+			39 01 00 02 5B 02
+			39 01 00 02 5C 00
+			39 01 00 02 5D 00
+			39 01 00 02 5E 20
+			39 01 00 02 5F 10
+			39 01 00 02 60 00
+			39 01 00 02 61 2A
+			39 01 00 02 62 00
+			39 01 00 02 63 01
+			39 01 00 02 64 56
+			39 01 00 02 65 29
+			39 01 00 02 66 00
+			39 01 00 02 67 01
+			39 01 00 02 68 57
+			39 01 00 02 3F 40
+			39 01 00 02 76 02
+			39 01 00 02 FF 2A
+			39 01 00 02 FB 01
+			39 01 00 02 22 2F
+			39 01 00 02 23 08
+			39 01 00 02 24 00
+			39 01 00 02 25 87
+			39 01 00 02 27 00
+			39 01 00 02 28 1A
+			39 01 00 02 29 00
+			39 01 00 02 2A 1A
+			39 01 00 02 2B 00
+			39 01 00 02 2D 1A
+			39 01 00 02 2F 00
+			39 01 00 02 30 87
+			39 01 00 02 32 00
+			39 01 00 02 33 1A
+			39 01 00 02 34 00
+			39 01 00 02 35 1A
+			39 01 00 02 36 00
+			39 01 00 02 37 1A
+			39 01 00 02 38 03
+			39 01 00 02 39 87
+			39 01 00 02 3B 00
+			39 01 00 02 3D 1A
+			39 01 00 02 3F 00
+			39 01 00 02 40 1A
+			39 01 00 02 41 00
+			39 01 00 02 42 1A
+			39 01 00 02 FF 27
+			39 01 00 02 FB 01
+			39 01 00 02 30 02
+			39 01 00 02 76 02
+			39 01 00 02 96 02
+			39 01 00 02 F8 02
+			39 01 00 02 FF E0
+			39 01 00 02 FB 01
+			39 01 00 02 14 60
+			39 01 00 02 16 C0
+			39 01 00 02 4F 02
+			39 01 00 02 FF F0
+			39 01 00 02 FB 01
+			39 01 00 02 3A 08
+			39 01 00 02 FF D0
+			39 01 00 02 FB 01
+			39 01 00 02 02 AF
+			39 01 00 02 09 EE
+			39 01 00 02 1C 88
+			39 01 00 02 1D 08
+			39 01 00 02 FF 10
+			39 01 00 02 FB 01
+			39 01 00 02 B9 05
+			39 01 00 02 FF 20
+			39 01 00 02 FB 01
+			39 01 00 02 18 40
+			39 01 00 02 FF 10
+			39 01 00 02 FB 01
+			39 01 00 02 B9 02
+			39 01 00 02 35 00
+			39 01 00 02 BB 13
+			39 01 00 06 3B 03 C8 1A 04 04
+			39 01 78 01 11
+			39 01 14 01 29
+		];
+		sleep-in-command = [
+			39 01 14 01 28
+			39 01 78 01 10
+		];
+		sleep-out-command = [
+			05 01 78 01 11
+			05 01 14 01 29
+		];
+		read-id-command = [
+			37 01 00 01 01
+			14 01 00 01 04
+		];
+
+		display-timings {
+			timing0 {
+				clock-frequency = <163000000>;
+				hactive = <1600>;
+				hfront-porch = <120>;
+				hback-porch = <50>;
+				hsync-len = <5>;
+				vactive = <2560>;
+				vfront-porch = <26>;
+				vback-porch = <198>;
+				vsync-len = <2>;
+				vsync-active = <1>;
+				hsync-active = <1>;
+			};
+		};
+	};
+};};
diff --git a/arch/riscv/boot/dts/spacemit/lcd/lcd_tc358762xbg_dpi_800x480.dtsi b/arch/riscv/boot/dts/spacemit/lcd/lcd_tc358762xbg_dpi_800x480.dtsi
new file mode 100644
index 000000000000..6ad649dc2346
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/lcd/lcd_tc358762xbg_dpi_800x480.dtsi
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/ { lcds: lcds {
+	lcd_tc358762xbg_dpi_800x480: lcd_tc358762xbg_dpi_800x480 {
+		dsi-work-mode = <1>; /* video burst mode*/
+		dsi-lane-number = <1>;
+		dsi-color-format = "rgb888";
+		width-mm = <95>;
+		height-mm = <53>;
+		use-dcs-write;
+
+		/*mipi info*/
+		height = <480>;
+		width = <800>;
+		hfp = <1>;
+		hbp = <46>;
+		hsync = <2>;
+		vfp = <7>;
+		vbp = <21>;
+		vsync = <2>;
+		fps = <55>;
+		work-mode = <0>;
+		rgb-mode = <3>;
+		lane-number = <1>;
+		phy-freq = <700000>;
+		phy-escape-clock = <52000>;
+		phy-bit-clock = <700000000>;
+		split-enable = <0>;
+		eotp-enable = <0>;
+		burst-mode = <2>;
+		esd-check-enable = <0>;
+
+		/* DSI_CMD, DSI_MODE, timeout, len, cmd */
+		initial-command = [
+			39 01 F0 01 11
+			39 01 28 01 29
+		];
+		sleep-in-command = [
+			39 01 78 01 28
+			39 01 78 01 10
+		];
+		sleep-out-command = [
+			39 01 96 01 11
+			39 01 32 01 29
+		];
+
+		display-timings {
+			timing0 {
+				clock-frequency = <25000000>;
+				hactive = <800>;
+				hfront-porch = <1>;
+				hback-porch = <46>;
+				hsync-len = <2>;
+				vactive = <480>;
+				vfront-porch = <7>;
+				vback-porch = <21>;
+				vsync-len = <2>;
+				vsync-active = <1>;
+				hsync-active = <1>;
+			};
+		};
+	};
+};};
diff --git a/arch/riscv/boot/dts/spacemit/lcd_dsi_panel.dtsi b/arch/riscv/boot/dts/spacemit/lcd_dsi_panel.dtsi
new file mode 100644
index 000000000000..cef6ec6885f3
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/lcd_dsi_panel.dtsi
@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Copyright (c) 2025 Spacemit, Inc */
+
+&soc {
+
+	display-subsystem-dsi {
+		compatible = "spacemit,saturn-hee";
+		reg = <0 0xc0340000 0 0x54000>;
+		hw_ver = <2>;
+		ports = <&dpu_crtc0>, <&dpu_crtc1>;
+	};
+
+	dpu_crtc0: dpu_crtc0 {
+		compatible = "spacemit,dpu-saturn";
+		reg = <0x0 0xd421a800 0x0 0x200>,
+			<0x0 0xd421aa00 0x0 0x200>;
+		interrupt-parent = <&saplic>;
+		interrupts = <90 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "ONLINE_IRQ";
+		clocks = <&syscon_apmu CLK_APMU_LCD_PXCLK>,
+			<&syscon_apmu CLK_APMU_LCD_MCLK>,
+			<&syscon_apmu CLK_APMU_LCD_HCLK>,
+			<&syscon_apmu CLK_APMU_DSI_ESC>,
+			<&syscon_apmu CLK_APMU_DPU_ACLK>,
+			<&syscon_apmu CLK_APMU_LCD_DSC>;
+		clock-names = "pxclk", "mclk", "hclk", "escclk", "aclk", "dscclk";
+		resets = <&syscon_apmu RESET_APMU_LCD_MCLK>,
+			<&syscon_apmu RESET_APMU_LCD>,
+			<&syscon_apmu RESET_APMU_DSI_ESC>,
+			<&syscon_apmu RESET_APMU_DPU_ACLK>,
+			<&syscon_apmu RESET_APMU_LCD_DSCCLK>;
+		reset-names= "mclk_reset", "lcd_reset", "esc_reset", "aclk_reset", "dsc_reset";
+		power-domains = <&power K3_PMU_LCD0_PWR_DOMAIN>;
+		pipeline-id = <1>;
+		is_edp = <0>;
+		dpu-id = <0>;
+		ip = "spacemit-saturn";
+		spacemit-dpu-min-mclk = <40960000>;
+		spacemit-dpu-dsipll;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				dpu_crtc0_out0: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&dsi0_in>;
+				};
+
+				dpu_crtc0_out1: endpoint@1 {
+					reg = <1>;
+					remote-endpoint = <&wb0_in0>;
+				};
+			};
+		};
+	};
+
+	dpu_crtc1: dpu_crtc1 {
+		compatible = "spacemit,dpu-saturn";
+		reg = <0x0 0xd421a800 0x0 0x200>,
+			<0x0 0xd421aa00 0x0 0x200>;
+		interrupt-parent = <&saplic>;
+		interrupts = <89 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "OFFLINE_IRQ";
+		clocks = <&syscon_apmu CLK_APMU_LCD_PXCLK>,
+			<&syscon_apmu CLK_APMU_LCD_MCLK>,
+			<&syscon_apmu CLK_APMU_LCD_HCLK>,
+			<&syscon_apmu CLK_APMU_DSI_ESC>,
+			<&syscon_apmu CLK_APMU_DPU_ACLK>,
+			<&syscon_apmu CLK_APMU_LCD_DSC>;
+		clock-names = "pxclk", "mclk", "hclk", "escclk", "aclk", "dscclk";
+		resets = <&syscon_apmu RESET_APMU_LCD_MCLK>,
+			<&syscon_apmu RESET_APMU_LCD>,
+			<&syscon_apmu RESET_APMU_DSI_ESC>,
+			<&syscon_apmu RESET_APMU_DPU_ACLK>,
+			<&syscon_apmu RESET_APMU_LCD_DSCCLK>;
+		reset-names= "mclk_reset", "lcd_reset", "esc_reset", "aclk_reset", "dsc_reset";
+		power-domains = <&power K3_PMU_LCD0_PWR_DOMAIN>;
+		pipeline-id = <2>;
+		is_edp = <1>;
+		ip = "spacemit-saturn";
+		spacemit-dpu-min-mclk = <40960000>;
+		status = "disabled";
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				dpu_crtc1_out0: endpoint@0 {
+					remote-endpoint = <&wb0_in1>;
+				};
+			};
+		};
+	};
+
+	dsi0: dsi0@d421a800 {
+		compatible = "spacemit,dsi-host";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0 0xd421a800 0 0x200>;
+		interrupt-parent = <&saplic>;
+		interrupts = <79 IRQ_TYPE_LEVEL_HIGH>;
+		ip = "synopsys-dhost";
+		dev-id = <2>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				dsi0_out: endpoint {
+					remote-endpoint = <&dphy0_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				dsi0_in: endpoint {
+					remote-endpoint = <&dpu_crtc0_out0>;
+				};
+			};
+		};
+	};
+
+	wb0 {
+		compatible = "spacemit,wb0";
+		dev-id = <0>;
+		status = "ok";
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			port@0 {
+				reg = <0>;
+				wb0_in0: endpoint {
+					remote-endpoint = <&dpu_crtc0_out1>;
+				};
+			};
+			port@1 {
+				reg = <1>;
+				wb0_in1: endpoint {
+					remote-endpoint = <&dpu_crtc1_out0>;
+				};
+			};
+		};
+	};
+
+	dphy0: dphy0@d421a800 {
+		compatible = "spacemit,dsi-phy";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0 0xd421a800 0 0x200>;
+		ip = "spacemit-dphy";
+		dev-id = <2>;
+		status = "ok";
+
+		port@0 {
+			reg = <0>;
+			dphy0_in: endpoint {
+				remote-endpoint = <&dsi0_out>;
+			};
+		};
+	};
+};
diff --git a/arch/riscv/boot/key/kernel_key_prv.crt b/arch/riscv/boot/key/kernel_key_prv.crt
new file mode 100644
index 000000000000..aa3132d28ae7
--- /dev/null
+++ b/arch/riscv/boot/key/kernel_key_prv.crt
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUNtgOy59vVVj1v7e/m5B08fiMuFwwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yNDA0MDcxMTQ4NTlaFw0yNDA1
+MDcxMTQ4NTlaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC9phd1E8t2251AcI1oIVIvJYiDXamtK7oeQaIs2neX
+AEhi/DIrBgCxXN04zh9LJWSMdH5q2x/ElgREhibH3Xo/XhIdxTalGBeAtz+1P7kE
+QMgVG0zQaf7ophmkQUi/S9j9Gxy+zugLtxGIrkCy/nG9OBMcKzd3+kDqNVDWSD1w
+RLmDFZ/IDFGCd0VlwcN7ZLfh4G56JktHRdxvHgI4GFG+S8zn6EkaBJbsHORd/TQZ
+CYOYe3vHq9+mjh8UqWcQmJwt4aWOYh5DIiYFa+Ju8xb4NIHJovdekeW7DF/pGEqX
+Jn/rxznG6ZXY2wUAYUXW9g30dYcrPxmY2a2Z46ORxCUlAgMBAAGjUzBRMB0GA1Ud
+DgQWBBQvshLHuxR6SdJgA4NtXTtte1PTlDAfBgNVHSMEGDAWgBQvshLHuxR6SdJg
+A4NtXTtte1PTlDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCq
+ehG2VE2CFelqHYe+KzQosCLWxfqlv86tffhm7bNohcTngdleRPAl93GdvCs+Jnoy
+C+TarPOEW8sAFsUh9mSmUnBBFkG9jOGr5z8cWdQ5h0zNc17TpA0iNcYzbmzYv12I
+JANw+w+iKardKbDYquQZ0tsE+aMi8h2HQ3A02UfllL6Ja2rCvgv/M/Cyt9j2VwDH
+bXYxbWWM69cMdaZpV+2cjfs2RBqklkSPQHNRA7DW1v5nFTeLkIuWj6D2d85k1zAo
+eXIv8+UpQ5NKuZ5ojsjMtsGJflLd3ZTXDuC9dUqL6nNbSvdbzY1OcwI7wXBWi3db
+6lSM7JdHF06axiPMUvsT
+-----END CERTIFICATE-----
diff --git a/arch/riscv/boot/key/kernel_key_prv.key b/arch/riscv/boot/key/kernel_key_prv.key
new file mode 100644
index 000000000000..45c600570206
--- /dev/null
+++ b/arch/riscv/boot/key/kernel_key_prv.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvaYXdRPLdtudQHCNaCFSLyWIg12prSu6HkGiLNp3lwBIYvwy
+KwYAsVzdOM4fSyVkjHR+atsfxJYERIYmx916P14SHcU2pRgXgLc/tT+5BEDIFRtM
+0Gn+6KYZpEFIv0vY/Rscvs7oC7cRiK5Asv5xvTgTHCs3d/pA6jVQ1kg9cES5gxWf
+yAxRgndFZcHDe2S34eBueiZLR0Xcbx4COBhRvkvM5+hJGgSW7BzkXf00GQmDmHt7
+x6vfpo4fFKlnEJicLeGljmIeQyImBWvibvMW+DSByaL3XpHluwxf6RhKlyZ/68c5
+xumV2NsFAGFF1vYN9HWHKz8ZmNmtmeOjkcQlJQIDAQABAoIBAFvrphD8IDEi7wwt
+wYxIIkJxycIKYtRpjlNd8pIU7cQFwhAgNty9kxZkcWE4HtjYbezaBfAa90HciIMC
+Fsfm5HrUSKmfTgYlMDcWGs3p7e5Oy+vXCJs31PLEl0cS0YTO6YR8YKlnNfd/qwL5
+mqhnRY+2HyUXGVKQvGgIoX7LEuRXxIK/gaFNE0udzO87eD/GCfwf7hAE6x3wdVGb
+ZJ+m7oA7AdCT7c8VjsaJY1OQfyAxWYV5Frzll2F7v87YxVPioUfWzPJmR51Q9XID
+mZ06RmneeUrmkF0tQnFrcjQC5TD2jaqiI4SJT/cmWXW/FV3JW67Jspp/ocAPxRXF
+3FjIwr0CgYEA8z2wzh1J2wNsNNUPFNlWCevF7ZuHJAWc9GJkjQkEa8Lk1NzEESL2
+IexLdKRSATTvQH6pFqNSuOcKkVfqO8auNGZmH/U1WZ1gFrzp4d3SayEyo/Z6zWtd
+Z9LFI4gIjs4SAJzP4eSm2kmFthmBQABJNIuHffBZRlj4hYadnwtS5h8CgYEAx5i/
+9RCEayCSm5HdAU9smFyFNJsgTPmAU4z26kyy5ZPo/tUr0sgqSVmYmqo8oRhGH1wc
+nlR/zQx8cW/LAbKzleHNmzxSvB/HDN2eVU+EyQuR+LL+bFf66XnR6c0R4jqakqg1
+BN5o5ZvnLK6JeWyoqaqFPURCuK5NYdTCQ+GbZDsCgYA7ijw/F7g+XgYcoJKCffir
+kQTv9oQL8zvKLlOCmlR0tE4jNK7hqvcosph9wsM3/GccGGT3X58eF5/kl+qEG0wF
+E5On+vPGLdmZd0CgwxWAl623qNvCcKd6hIMxAVlm61ZIxBm2p6uX/ys1ueSDcA57
+SDQwRab0q06aEKuAhNsT7QKBgAN9fk3QviAJb8bcpBqzwthfNc0XsVbnPgE4A9FR
+CgfNsFNUcvG6olKqCGsb8vd2LzO5Lp7uJ1X0k/B/3fLnpxBF5IyEX4X+oEJvVad0
+PoQteZ35s009tJuEj9EgzuE/2RgbbOHq6baqJT64xpqi9cgRvUa2/Nux4JSHXeZF
++aulAoGBAIJhoN8R1WM9INYtbFbYvLn0mLWXB7ZjvjlXNUmEKDibX57qHOq4VNS2
+5uI2qXd/JIORXwn2u+jB3+e/ebPWem5tH4xl0J0TnjAmcjiBGV8ihn90Bi3nDvQy
+XtUdODYyzDvecPt7B3SfvQct7qwlnj2AJr8jDSmd6nuGvL/dHS+i
+-----END RSA PRIVATE KEY-----
diff --git a/arch/riscv/boot/key/kernel_key_pub.key b/arch/riscv/boot/key/kernel_key_pub.key
new file mode 100644
index 000000000000..d3329893223c
--- /dev/null
+++ b/arch/riscv/boot/key/kernel_key_pub.key
@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvaYXdRPLdtudQHCNaCFS
+LyWIg12prSu6HkGiLNp3lwBIYvwyKwYAsVzdOM4fSyVkjHR+atsfxJYERIYmx916
+P14SHcU2pRgXgLc/tT+5BEDIFRtM0Gn+6KYZpEFIv0vY/Rscvs7oC7cRiK5Asv5x
+vTgTHCs3d/pA6jVQ1kg9cES5gxWfyAxRgndFZcHDe2S34eBueiZLR0Xcbx4COBhR
+vkvM5+hJGgSW7BzkXf00GQmDmHt7x6vfpo4fFKlnEJicLeGljmIeQyImBWvibvMW
++DSByaL3XpHluwxf6RhKlyZ/68c5xumV2NsFAGFF1vYN9HWHKz8ZmNmtmeOjkcQl
+JQIDAQAB
+-----END PUBLIC KEY-----
diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig
index fc2725cbca18..8eae620cb023 100644
--- a/arch/riscv/configs/defconfig
+++ b/arch/riscv/configs/defconfig
@@ -158,6 +158,7 @@ CONFIG_I2C_DESIGNWARE_CORE=y
 CONFIG_I2C_MV64XXX=m
 CONFIG_SPI=y
 CONFIG_SPI_CADENCE_QUADSPI=m
+CONFIG_SPI_FSL_QUADSPI=m
 CONFIG_SPI_PL022=m
 CONFIG_SPI_SIFIVE=y
 CONFIG_SPI_SUN6I=y
@@ -304,3 +305,9 @@ CONFIG_DEBUG_KERNEL=y
 CONFIG_DEBUG_FS=y
 # CONFIG_RUNTIME_TESTING_MENU is not set
 CONFIG_MEMTEST=y
+CONFIG_TEE=y
+CONFIG_OPTEE=y
+CONFIG_DEBUG_INFO_DWARF5=y
+CONFIG_READABLE_ASM=y
+CONFIG_GDB_SCRIPTS=y
+CONFIG_RISCV_SBI_MPXY_MBOX=y
diff --git a/arch/riscv/configs/k3_bianbu_defconfig b/arch/riscv/configs/k3_bianbu_defconfig
new file mode 100644
index 000000000000..a02ecae94db0
--- /dev/null
+++ b/arch/riscv/configs/k3_bianbu_defconfig
@@ -0,0 +1,1212 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_JIT=y
+CONFIG_BPF_JIT_ALWAYS_ON=y
+CONFIG_PREEMPT_VOLUNTARY=y
+CONFIG_PREEMPT_DYNAMIC=y
+CONFIG_BSD_PROCESS_ACCT=y
+CONFIG_BSD_PROCESS_ACCT_V3=y
+CONFIG_PSI=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_IKHEADERS=m
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_SCHED_AUTOGROUP=y
+CONFIG_BOOT_CONFIG=y
+CONFIG_EXPERT=y
+CONFIG_SGETMASK_SYSCALL=y
+CONFIG_PROFILING=y
+CONFIG_KEXEC=y
+CONFIG_KEXEC_FILE=y
+CONFIG_SOC_SPACEMIT=y
+CONFIG_SOC_SPACEMIT_K3=y
+CONFIG_SMP=y
+CONFIG_NR_CPUS=16
+# CONFIG_RISCV_ISA_VENDOR_EXT_ANDES is not set
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_DMI is not set
+CONFIG_RISCV_BUILD_ITB_IMAGE=y
+CONFIG_PM_DEBUG=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_SPACEMIT_K3_CPUFREQ=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=y
+CONFIG_KPROBES=y
+# CONFIG_STACKPROTECTOR_STRONG is not set
+# CONFIG_RANDOMIZE_KSTACK_OFFSET is not set
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_ZSTD=y
+CONFIG_MODULE_DECOMPRESS=y
+CONFIG_BLK_DEV_BSGLIB=y
+CONFIG_BLK_DEV_ZONED=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BLK_CGROUP_IOPRIO=y
+CONFIG_BLK_SED_OPAL=y
+CONFIG_BLK_INLINE_ENCRYPTION=y
+CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y
+CONFIG_PARTITION_ADVANCED=y
+CONFIG_AIX_PARTITION=y
+CONFIG_OSF_PARTITION=y
+CONFIG_AMIGA_PARTITION=y
+CONFIG_ATARI_PARTITION=y
+CONFIG_MAC_PARTITION=y
+CONFIG_BSD_DISKLABEL=y
+CONFIG_MINIX_SUBPARTITION=y
+CONFIG_SOLARIS_X86_PARTITION=y
+CONFIG_UNIXWARE_DISKLABEL=y
+CONFIG_LDM_PARTITION=y
+CONFIG_SGI_PARTITION=y
+CONFIG_ULTRIX_PARTITION=y
+CONFIG_SUN_PARTITION=y
+CONFIG_KARMA_PARTITION=y
+CONFIG_SYSV68_PARTITION=y
+CONFIG_CMDLINE_PARTITION=y
+CONFIG_MQ_IOSCHED_KYBER=m
+CONFIG_IOSCHED_BFQ=m
+CONFIG_BINFMT_MISC=m
+CONFIG_SLAB_BUCKETS=y
+CONFIG_RANDOM_KMALLOC_CACHES=y
+CONFIG_SHUFFLE_PAGE_ALLOCATOR=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_PAGE_REPORTING=y
+CONFIG_KSM=y
+CONFIG_TRANSPARENT_HUGEPAGE=y
+CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
+CONFIG_CMA=y
+CONFIG_CMA_DEBUGFS=y
+CONFIG_CMA_SYSFS=y
+CONFIG_IDLE_PAGE_TRACKING=y
+CONFIG_ANON_VMA_NAME=y
+CONFIG_USERFAULTFD=y
+CONFIG_LRU_GEN=y
+CONFIG_LRU_GEN_ENABLED=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_XFRM_USER=m
+CONFIG_XFRM_INTERFACE=m
+CONFIG_XFRM_STATISTICS=y
+CONFIG_NET_KEY=m
+CONFIG_XDP_SOCKETS=y
+CONFIG_XDP_SOCKETS_DIAG=m
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_FIB_TRIE_STATS=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_IP_ROUTE_MULTIPATH=y
+CONFIG_IP_ROUTE_VERBOSE=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPGRE_DEMUX=m
+CONFIG_NET_IPGRE=m
+CONFIG_NET_IPGRE_BROADCAST=y
+CONFIG_IP_MROUTE=y
+CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IP_PIMSM_V1=y
+CONFIG_IP_PIMSM_V2=y
+CONFIG_NET_IPVTI=m
+CONFIG_NET_FOU_IP_TUNNELS=y
+CONFIG_INET_AH=m
+CONFIG_INET_ESP=m
+CONFIG_INET_ESP_OFFLOAD=m
+CONFIG_INET_ESPINTCP=y
+CONFIG_INET_IPCOMP=m
+CONFIG_INET_DIAG=m
+CONFIG_INET_UDP_DIAG=m
+CONFIG_INET_RAW_DIAG=m
+CONFIG_INET_DIAG_DESTROY=y
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=m
+CONFIG_TCP_AO=y
+CONFIG_TCP_MD5SIG=y
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_ESP_OFFLOAD=m
+CONFIG_INET6_ESPINTCP=y
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_SIT=m
+CONFIG_IPV6_SIT_6RD=y
+CONFIG_IPV6_GRE=m
+CONFIG_IPV6_SUBTREES=y
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IPV6_PIMSM_V2=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETLABEL=y
+CONFIG_MPTCP=y
+CONFIG_NETWORK_SECMARK=y
+CONFIG_NETWORK_PHY_TIMESTAMPING=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_SECMARK=y
+CONFIG_NF_CONNTRACK_ZONES=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_TIMEOUT=y
+CONFIG_NF_CONNTRACK_TIMESTAMP=y
+CONFIG_NF_CONNTRACK_AMANDA=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_H323=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_NETBIOS_NS=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SANE=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_CT_NETLINK_TIMEOUT=m
+CONFIG_NF_CT_NETLINK_HELPER=m
+CONFIG_NETFILTER_NETLINK_GLUE_CT=y
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NF_FLOW_TABLE_INET=m
+CONFIG_NF_FLOW_TABLE=m
+CONFIG_NETFILTER_XTABLES_COMPAT=y
+CONFIG_NETFILTER_XTABLES_LEGACY=y
+CONFIG_NETFILTER_XT_SET=m
+CONFIG_NETFILTER_XT_TARGET_AUDIT=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_TRACE=m
+CONFIG_NETFILTER_XT_TARGET_SECMARK=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_SET_BITMAP_IP=m
+CONFIG_IP_SET_BITMAP_IPMAC=m
+CONFIG_IP_SET_BITMAP_PORT=m
+CONFIG_IP_SET_HASH_IP=m
+CONFIG_IP_SET_HASH_IPMARK=m
+CONFIG_IP_SET_HASH_IPPORT=m
+CONFIG_IP_SET_HASH_IPPORTIP=m
+CONFIG_IP_SET_HASH_IPPORTNET=m
+CONFIG_IP_SET_HASH_IPMAC=m
+CONFIG_IP_SET_HASH_MAC=m
+CONFIG_IP_SET_HASH_NETPORTNET=m
+CONFIG_IP_SET_HASH_NET=m
+CONFIG_IP_SET_HASH_NETNET=m
+CONFIG_IP_SET_HASH_NETPORT=m
+CONFIG_IP_SET_HASH_NETIFACE=m
+CONFIG_IP_SET_LIST_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_PROTO_AH=y
+CONFIG_IP_VS_PROTO_SCTP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_WRR=m
+CONFIG_IP_VS_LC=m
+CONFIG_IP_VS_WLC=m
+CONFIG_IP_VS_FO=m
+CONFIG_IP_VS_OVF=m
+CONFIG_IP_VS_LBLC=m
+CONFIG_IP_VS_LBLCR=m
+CONFIG_IP_VS_DH=m
+CONFIG_IP_VS_SH=m
+CONFIG_IP_VS_MH=m
+CONFIG_IP_VS_SED=m
+CONFIG_IP_VS_NQ=m
+CONFIG_IP_VS_TWOS=m
+CONFIG_IP_VS_FTP=m
+CONFIG_IP_VS_PE_SIP=m
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_NF_LOG_IPV4=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_NAT=m
+CONFIG_IP_NF_TARGET_MASQUERADE=m
+CONFIG_IP_NF_TARGET_NETMAP=m
+CONFIG_IP_NF_TARGET_REDIRECT=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_TARGET_TTL=m
+CONFIG_IP_NF_RAW=m
+CONFIG_IP_NF_ARPFILTER=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_HL=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_RAW=m
+CONFIG_IP6_NF_SECURITY=m
+CONFIG_IP6_NF_NAT=m
+CONFIG_IP6_NF_TARGET_MASQUERADE=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NFT_BRIDGE_META=m
+CONFIG_NFT_BRIDGE_REJECT=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_BROUTE=m
+CONFIG_BRIDGE_EBT_T_FILTER=m
+CONFIG_BRIDGE_EBT_T_NAT=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_RDS=m
+CONFIG_RDS_TCP=m
+CONFIG_TIPC=m
+CONFIG_ATM=m
+CONFIG_ATM_CLIP=m
+CONFIG_ATM_LANE=m
+CONFIG_ATM_MPOA=m
+CONFIG_ATM_BR2684=m
+CONFIG_L2TP=m
+CONFIG_L2TP_DEBUGFS=m
+CONFIG_L2TP_V3=y
+CONFIG_L2TP_IP=m
+CONFIG_L2TP_ETH=m
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_NET_DSA=m
+CONFIG_NET_DSA_TAG_NONE=m
+CONFIG_NET_DSA_TAG_AR9331=m
+CONFIG_NET_DSA_TAG_BRCM=m
+CONFIG_NET_DSA_TAG_BRCM_LEGACY=m
+CONFIG_NET_DSA_TAG_BRCM_PREPEND=m
+CONFIG_NET_DSA_TAG_HELLCREEK=m
+CONFIG_NET_DSA_TAG_GSWIP=m
+CONFIG_NET_DSA_TAG_DSA=m
+CONFIG_NET_DSA_TAG_EDSA=m
+CONFIG_NET_DSA_TAG_MTK=m
+CONFIG_NET_DSA_TAG_KSZ=m
+CONFIG_NET_DSA_TAG_OCELOT=m
+CONFIG_NET_DSA_TAG_OCELOT_8021Q=m
+CONFIG_NET_DSA_TAG_QCA=m
+CONFIG_NET_DSA_TAG_RTL4_A=m
+CONFIG_NET_DSA_TAG_RTL8_4=m
+CONFIG_NET_DSA_TAG_RZN1_A5PSW=m
+CONFIG_NET_DSA_TAG_LAN9303=m
+CONFIG_NET_DSA_TAG_SJA1105=m
+CONFIG_NET_DSA_TAG_TRAILER=m
+CONFIG_NET_DSA_TAG_VSC73XX_8021Q=m
+CONFIG_NET_DSA_TAG_XRS700X=m
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_VLAN_8021Q_MVRP=y
+CONFIG_LLC2=m
+CONFIG_ATALK=m
+CONFIG_X25=m
+CONFIG_LAPB=m
+CONFIG_PHONET=m
+CONFIG_6LOWPAN=m
+CONFIG_IEEE802154=m
+CONFIG_IEEE802154_6LOWPAN=m
+CONFIG_MAC802154=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_HTB=m
+CONFIG_NET_SCH_HFSC=m
+CONFIG_NET_SCH_PRIO=m
+CONFIG_NET_SCH_MULTIQ=m
+CONFIG_NET_SCH_RED=m
+CONFIG_NET_SCH_SFB=m
+CONFIG_NET_SCH_SFQ=m
+CONFIG_NET_SCH_TEQL=m
+CONFIG_NET_SCH_TBF=m
+CONFIG_NET_SCH_CBS=m
+CONFIG_NET_SCH_ETF=m
+CONFIG_NET_SCH_TAPRIO=m
+CONFIG_NET_SCH_GRED=m
+CONFIG_NET_SCH_NETEM=m
+CONFIG_NET_SCH_DRR=m
+CONFIG_NET_SCH_MQPRIO=m
+CONFIG_NET_SCH_SKBPRIO=m
+CONFIG_NET_SCH_CHOKE=m
+CONFIG_NET_SCH_QFQ=m
+CONFIG_NET_SCH_CODEL=m
+CONFIG_NET_SCH_FQ_CODEL=m
+CONFIG_NET_SCH_CAKE=m
+CONFIG_NET_SCH_FQ=m
+CONFIG_NET_SCH_HHF=m
+CONFIG_NET_SCH_PIE=m
+CONFIG_NET_SCH_FQ_PIE=m
+CONFIG_NET_SCH_PLUG=m
+CONFIG_NET_SCH_ETS=m
+CONFIG_NET_CLS_BASIC=m
+CONFIG_NET_CLS_ROUTE4=m
+CONFIG_NET_CLS_FW=m
+CONFIG_NET_CLS_U32=m
+CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_FLOW=m
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_BPF=m
+CONFIG_NET_CLS_FLOWER=m
+CONFIG_NET_CLS_MATCHALL=m
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_CMP=m
+CONFIG_NET_EMATCH_NBYTE=m
+CONFIG_NET_EMATCH_U32=m
+CONFIG_NET_EMATCH_META=m
+CONFIG_NET_EMATCH_TEXT=m
+CONFIG_NET_EMATCH_IPSET=m
+CONFIG_NET_EMATCH_IPT=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_ACT_POLICE=m
+CONFIG_NET_ACT_GACT=m
+CONFIG_GACT_PROB=y
+CONFIG_NET_ACT_MIRRED=m
+CONFIG_NET_ACT_SAMPLE=m
+CONFIG_NET_ACT_NAT=m
+CONFIG_NET_ACT_PEDIT=m
+CONFIG_NET_ACT_SIMP=m
+CONFIG_NET_ACT_SKBEDIT=m
+CONFIG_NET_ACT_CSUM=m
+CONFIG_NET_ACT_MPLS=m
+CONFIG_NET_ACT_VLAN=m
+CONFIG_NET_ACT_BPF=m
+CONFIG_NET_ACT_CONNMARK=m
+CONFIG_NET_ACT_CTINFO=m
+CONFIG_NET_ACT_SKBMOD=m
+CONFIG_NET_ACT_TUNNEL_KEY=m
+CONFIG_NET_ACT_CT=m
+CONFIG_NET_ACT_GATE=m
+CONFIG_NET_TC_SKB_EXT=y
+CONFIG_DCB=y
+CONFIG_DNS_RESOLVER=y
+CONFIG_BATMAN_ADV=m
+# CONFIG_BATMAN_ADV_BATMAN_V is not set
+CONFIG_OPENVSWITCH=m
+CONFIG_VSOCKETS=m
+CONFIG_VIRTIO_VSOCKETS=m
+CONFIG_NETLINK_DIAG=m
+CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
+CONFIG_HSR=m
+CONFIG_QRTR=m
+CONFIG_QRTR_TUN=m
+CONFIG_NET_NCSI=y
+CONFIG_NCSI_OEM_CMD_GET_MAC=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BPF_STREAM_PARSER=y
+CONFIG_NET_PKTGEN=m
+CONFIG_HAMRADIO=y
+CONFIG_AX25=m
+CONFIG_NETROM=m
+CONFIG_ROSE=m
+CONFIG_MKISS=m
+CONFIG_6PACK=m
+CONFIG_BPQETHER=m
+CONFIG_BAYCOM_SER_FDX=m
+CONFIG_BAYCOM_SER_HDX=m
+CONFIG_YAM=m
+CONFIG_CAN=m
+CONFIG_CAN_J1939=m
+CONFIG_CAN_ISOTP=m
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HCIBTUSB=m
+CONFIG_BT_HCIBTUSB_MTK=y
+CONFIG_BT_HCIBTSDIO=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_H4=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_AF_RXRPC=y
+CONFIG_AF_RXRPC_IPV6=y
+CONFIG_RXKAD=y
+CONFIG_RXPERF=m
+CONFIG_AF_KCM=m
+CONFIG_MCTP=y
+CONFIG_CFG80211=m
+CONFIG_CFG80211_DEBUGFS=y
+CONFIG_MAC80211=m
+CONFIG_MAC80211_MESH=y
+CONFIG_MAC80211_DEBUGFS=y
+CONFIG_MAC80211_MESSAGE_TRACING=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_NET_9P=m
+CONFIG_NET_9P_VIRTIO=m
+CONFIG_CAIF=m
+CONFIG_CAIF_USB=m
+CONFIG_CEPH_LIB_USE_DNS_RESOLVER=y
+CONFIG_NFC=m
+CONFIG_NFC_DIGITAL=m
+CONFIG_NFC_NCI=m
+CONFIG_NFC_NCI_UART=m
+CONFIG_NFC_HCI=m
+CONFIG_NFC_SHDLC=y
+CONFIG_NFC_SIM=m
+CONFIG_NFC_PORT100=m
+CONFIG_NFC_VIRTUAL_NCI=m
+CONFIG_NFC_FDP=m
+CONFIG_NFC_FDP_I2C=m
+CONFIG_NFC_PN544_I2C=m
+CONFIG_NFC_PN533_USB=m
+CONFIG_NFC_PN533_I2C=m
+CONFIG_NFC_MICROREAD_I2C=m
+CONFIG_NFC_MRVL_USB=m
+CONFIG_NFC_MRVL_UART=m
+CONFIG_NFC_MRVL_I2C=m
+CONFIG_NFC_ST21NFCA_I2C=m
+CONFIG_NFC_ST_NCI_I2C=m
+CONFIG_NFC_NXP_NCI=m
+CONFIG_NFC_NXP_NCI_I2C=m
+CONFIG_NFC_S3FWRN5_I2C=m
+CONFIG_NET_IFE=m
+CONFIG_PAGE_POOL_STATS=y
+CONFIG_FAILOVER=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIE_SPACEMIT_EP=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_PCI_ENDPOINT_CONFIGFS=y
+CONFIG_PCI_EPF_TEST=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+# CONFIG_STANDALONE is not set
+CONFIG_FW_LOADER_COMPRESS=y
+# CONFIG_FW_LOADER_COMPRESS_XZ is not set
+CONFIG_FW_LOADER_COMPRESS_ZSTD=y
+# CONFIG_ALLOW_DEV_COREDUMP is not set
+CONFIG_MTD=y
+CONFIG_MTD_CMDLINE_PARTS=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_PARTITIONED_MASTER=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_VIRTIO_BLK=m
+CONFIG_BLK_DEV_NVME=y
+CONFIG_RT7451_RETIMER=y
+CONFIG_EEPROM_AT24=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=y
+CONFIG_TARGET_CORE=y
+CONFIG_TCM_IBLOCK=y
+CONFIG_TCM_FILEIO=y
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_EQUALIZER=m
+CONFIG_IFB=m
+CONFIG_NET_TEAM=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_BAREUDP=m
+CONFIG_GTP=m
+CONFIG_PFCP=m
+CONFIG_AMT=m
+CONFIG_MACSEC=m
+CONFIG_NETCONSOLE=m
+CONFIG_NETCONSOLE_DYNAMIC=y
+CONFIG_TUN=y
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+CONFIG_NLMON=m
+CONFIG_NET_VRF=m
+CONFIG_R8169=m
+CONFIG_R8127=m
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_SPACEMIT_ETHQOS=y
+CONFIG_ETHERCAT=y
+CONFIG_EC_MASTER=y
+CONFIG_EC_DEVICE=y
+CONFIG_EC_K3_GMAC=y
+CONFIG_LED_TRIGGER_PHY=y
+CONFIG_SFP=y
+CONFIG_REALTEK_PHY=y
+CONFIG_CAN_FLEXCAN=m
+CONFIG_USB_RTL8152=m
+CONFIG_USB_USBNET=y
+# CONFIG_USB_NET_AX8817X is not set
+CONFIG_USB_NET_CDC_EEM=m
+CONFIG_USB_NET_CDC_MBIM=m
+# CONFIG_USB_NET_NET1080 is not set
+CONFIG_USB_NET_RNDIS_HOST=m
+# CONFIG_USB_BELKIN is not set
+# CONFIG_USB_ARMLINUX is not set
+# CONFIG_USB_NET_ZAURUS is not set
+CONFIG_USB_NET_QMI_WWAN=m
+CONFIG_IWLWIFI=m
+CONFIG_IWLMVM=m
+CONFIG_MT7921E=m
+CONFIG_MT7925E=m
+CONFIG_RTW89=m
+CONFIG_RTW89_8852BE=m
+CONFIG_RTL8852BS=m
+CONFIG_INPUT_EVDEV=y
+CONFIG_KEYBOARD_CROS_EC=y
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_RASPITS_FT5426=y
+CONFIG_INPUT_MISC=y
+CONFIG_INPUT_UINPUT=m
+CONFIG_INPUT_DRV_RPMI_PWRKEY=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_NR_UARTS=17
+CONFIG_SERIAL_8250_RUNTIME_UARTS=17
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+# CONFIG_DEVPORT is not set
+CONFIG_TCG_TPM=y
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_CROS_EC_TUNNEL=y
+CONFIG_I2C_SLAVE=y
+CONFIG_I2C_SPACEMIT_SLAVE=y
+CONFIG_SPI=y
+CONFIG_SPI_FSL_QUADSPI=y
+CONFIG_SPI_SPACEMIT_K1=y
+CONFIG_SPI_SPIDEV=y
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_SINGLE=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_SPACEMIT_K1=y
+CONFIG_GPIO_CROS_EC=y
+CONFIG_BATTERY_SBS=y
+CONFIG_CHARGER_CROS_USBPD=m
+# CONFIG_CHARGER_CROS_PCHG is not set
+CONFIG_SENSORS_CTF2301=y
+CONFIG_SENSORS_PWM_FAN=y
+CONFIG_THERMAL=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_K3_THERMAL=y
+CONFIG_WATCHDOG=y
+CONFIG_SPACEMIT_K1_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+# CONFIG_REGULATOR_SPACEMIT_P1 is not set
+# CONFIG_REGULATOR_SPACEMIT_MPQ8655 is not set
+CONFIG_RC_CORE=y
+CONFIG_RC_DECODERS=y
+CONFIG_IR_NEC_DECODER=y
+CONFIG_RC_DEVICES=y
+CONFIG_IR_SPACEMIT_K1=y
+# CONFIG_MEDIA_CEC_SUPPORT is not set
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=y
+CONFIG_VIDEO_LINLON_K3=y
+CONFIG_SPACEMIT_K3_CCIC=y
+CONFIG_SPACEMIT_K3_CCIC_IOMMU=y
+CONFIG_SPACEMIT_K3_CAM_IMX219=y
+CONFIG_SPACEMIT_K3_CAM_IMX415=y
+CONFIG_SPACEMIT_K3_CAM_OV5647=y
+CONFIG_SPACEMIT_K3_CAM_OV2735=y
+CONFIG_SPACEMIT_K3_CAM_OV5640_MAX96724_MAX9295=y
+CONFIG_DRM=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_POWERVR_ROGUE=y
+CONFIG_POWERVR_DVFS=y
+CONFIG_POWERVR_THERMAL=y
+CONFIG_DRM_SPACEMIT=y
+CONFIG_SPACEMIT_INNO_DP=y
+CONFIG_SPACEMIT_MIPI_PANEL=y
+CONFIG_FB=y
+CONFIG_BACKLIGHT_PWM=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_USB_AUDIO=y
+CONFIG_SND_USB_AUDIO_MIDI_V2=y
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_K1_I2S=y
+CONFIG_SND_SOC_K3_RI2S=y
+CONFIG_SND_SOC_ES8326=m
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_HIDRAW=y
+CONFIG_UHID=y
+CONFIG_HID_MULTITOUCH=m
+CONFIG_I2C_HID_OF=m
+CONFIG_USB_HIDDEV=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_ACM=m
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+# CONFIG_USB_DWC3_OF_SIMPLE is not set
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_SIMPLE=m
+CONFIG_USB_SERIAL_AIRCABLE=m
+CONFIG_USB_SERIAL_ARK3116=m
+CONFIG_USB_SERIAL_BELKIN=m
+CONFIG_USB_SERIAL_CH341=m
+CONFIG_USB_SERIAL_WHITEHEAT=m
+CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m
+CONFIG_USB_SERIAL_CP210X=m
+CONFIG_USB_SERIAL_CYPRESS_M8=m
+CONFIG_USB_SERIAL_EMPEG=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_SERIAL_VISOR=m
+CONFIG_USB_SERIAL_IPAQ=m
+CONFIG_USB_SERIAL_IR=m
+CONFIG_USB_SERIAL_EDGEPORT=m
+CONFIG_USB_SERIAL_EDGEPORT_TI=m
+CONFIG_USB_SERIAL_F81232=m
+CONFIG_USB_SERIAL_F8153X=m
+CONFIG_USB_SERIAL_GARMIN=m
+CONFIG_USB_SERIAL_IPW=m
+CONFIG_USB_SERIAL_IUU=m
+CONFIG_USB_SERIAL_KEYSPAN_PDA=m
+CONFIG_USB_SERIAL_KEYSPAN=m
+CONFIG_USB_SERIAL_KLSI=m
+CONFIG_USB_SERIAL_KOBIL_SCT=m
+CONFIG_USB_SERIAL_MCT_U232=m
+CONFIG_USB_SERIAL_METRO=m
+CONFIG_USB_SERIAL_MOS7720=m
+CONFIG_USB_SERIAL_MOS7840=m
+CONFIG_USB_SERIAL_MXUPORT=m
+CONFIG_USB_SERIAL_NAVMAN=m
+CONFIG_USB_SERIAL_PL2303=m
+CONFIG_USB_SERIAL_OTI6858=m
+CONFIG_USB_SERIAL_QCAUX=m
+CONFIG_USB_SERIAL_QUALCOMM=m
+CONFIG_USB_SERIAL_SPCP8X5=m
+CONFIG_USB_SERIAL_SAFE=m
+CONFIG_USB_SERIAL_SAFE_PADDED=y
+CONFIG_USB_SERIAL_SIERRAWIRELESS=m
+CONFIG_USB_SERIAL_SYMBOL=m
+CONFIG_USB_SERIAL_TI=m
+CONFIG_USB_SERIAL_CYBERJACK=m
+CONFIG_USB_SERIAL_OPTION=m
+CONFIG_USB_SERIAL_OMNINET=m
+CONFIG_USB_SERIAL_OPTICON=m
+CONFIG_USB_SERIAL_XSENS_MT=m
+CONFIG_USB_SERIAL_WISHBONE=m
+CONFIG_USB_SERIAL_SSU100=m
+CONFIG_USB_SERIAL_QT2=m
+CONFIG_USB_SERIAL_UPD78F0730=m
+CONFIG_USB_SERIAL_XR=m
+CONFIG_USB_SERIAL_DEBUG=m
+CONFIG_USB_ONBOARD_DEV=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_RNDIS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_USB_CONFIGFS_F_TCM=y
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB301=m
+CONFIG_MMC=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_K1=y
+CONFIG_SCSI_UFSHCD=y
+CONFIG_SCSI_UFSHCD_PLATFORM=y
+CONFIG_SCSI_UFS_SPACEMIT_K3=y
+CONFIG_NEW_LEDS=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_TRIGGER_NETDEV=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_CROS_EC=y
+CONFIG_RTC_DRV_RPMI=y
+CONFIG_DMADEVICES=y
+CONFIG_SPACEMIT_HDMA=y
+CONFIG_USERSPACE_DMA=y
+CONFIG_DW_EDMA=y
+CONFIG_DMATEST=y
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VFIO=y
+CONFIG_VFIO_PCI=y
+# CONFIG_VIRTIO_MENU is not set
+CONFIG_VHOST_VSOCK=m
+CONFIG_CHROME_PLATFORMS=y
+CONFIG_CROS_EC=y
+CONFIG_CROS_EC_ESPI=y
+CONFIG_CROS_EC_TYPEC=m
+CONFIG_SPACEMIT_CCU=y
+CONFIG_SPACEMIT_K3_CCU=y
+CONFIG_HWSPINLOCK=y
+CONFIG_HWSPINLOCK_SPACEMIT=y
+CONFIG_SPACEMIT_K1X_TIMER=y
+CONFIG_MAILBOX=y
+CONFIG_SPACEMIT_MAILBOX=y
+CONFIG_K3_MAILBOX=y
+CONFIG_RISCV_IOMMU_SPACEMIT_K3=y
+CONFIG_REMOTEPROC=y
+CONFIG_REMOTEPROC_CDEV=y
+CONFIG_SPACEMIT_K3_REMOTEPROC=y
+CONFIG_RPMSG_CHAR=y
+CONFIG_RPMSG_CTRL=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_SPACEMIT_HMP=y
+CONFIG_SPACEMIT_PM_DOMAINS=y
+CONFIG_SPACEMIT_REBOOT_CONTROL=y
+CONFIG_SPACEMIT_SOCINFO=y
+CONFIG_SPACEMIT_POWERKEY_MONITOR=y
+CONFIG_SPACEMIT_DDR_BW=y
+CONFIG_SPACEMIT_V2D=y
+CONFIG_IIO=m
+CONFIG_CM32181=m
+CONFIG_PWM=y
+CONFIG_PWM_CROS_EC=y
+CONFIG_PWM_PXA=y
+CONFIG_PHY_SPACEMIT_K1_USB2=y
+CONFIG_PHY_SPACEMIT_K3_USB3=y
+CONFIG_PHY_SPACEMIT_K3_PCIE=y
+CONFIG_NVMEM_SPACEMIT_EFUSE=y
+CONFIG_CORESIGHT=y
+CONFIG_CORESIGHT_LINKS_AND_SINKS=y
+CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y
+CONFIG_RVTRACE=y
+CONFIG_RVTRACE_ENCODER=y
+CONFIG_VALIDATE_FS_PARSER=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_JFS_FS=m
+CONFIG_JFS_POSIX_ACL=y
+CONFIG_JFS_SECURITY=y
+CONFIG_JFS_STATISTICS=y
+CONFIG_XFS_FS=m
+CONFIG_XFS_QUOTA=y
+CONFIG_XFS_POSIX_ACL=y
+CONFIG_GFS2_FS=m
+CONFIG_GFS2_FS_LOCKING_DLM=y
+CONFIG_OCFS2_FS=m
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_NILFS2_FS=m
+CONFIG_F2FS_FS=m
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_F2FS_FS_COMPRESSION=y
+# CONFIG_F2FS_IOSTAT is not set
+CONFIG_F2FS_UNFAIR_RWSEM=y
+CONFIG_ZONEFS_FS=m
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y
+CONFIG_FS_VERITY=y
+CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y
+CONFIG_FANOTIFY=y
+CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y
+CONFIG_QUOTA_NETLINK_INTERFACE=y
+CONFIG_QFMT_V1=m
+CONFIG_QFMT_V2=m
+CONFIG_AUTOFS_FS=m
+CONFIG_FUSE_FS=y
+CONFIG_CUSE=m
+CONFIG_VIRTIO_FS=m
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_NETFS_DEBUG=y
+CONFIG_FSCACHE_STATS=y
+CONFIG_CACHEFILES=m
+CONFIG_CACHEFILES_ERROR_INJECTION=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_UDF_FS=m
+CONFIG_MSDOS_FS=m
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS3_LZX_XPRESS=y
+CONFIG_NTFS3_FS_POSIX_ACL=y
+CONFIG_NTFS_FS=m
+CONFIG_PROC_KCORE=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_TMPFS_INODE64=y
+CONFIG_TMPFS_QUOTA=y
+CONFIG_HUGETLBFS=y
+CONFIG_EFIVAR_FS=y
+CONFIG_ORANGEFS_FS=m
+CONFIG_ADFS_FS=m
+CONFIG_AFFS_FS=m
+CONFIG_ECRYPT_FS=y
+CONFIG_ECRYPT_FS_MESSAGING=y
+CONFIG_HFS_FS=m
+CONFIG_HFSPLUS_FS=m
+CONFIG_BEFS_FS=m
+CONFIG_BFS_FS=m
+CONFIG_EFS_FS=m
+CONFIG_CRAMFS=m
+CONFIG_SQUASHFS=y
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_VXFS_FS=m
+CONFIG_MINIX_FS=m
+CONFIG_OMFS_FS=m
+CONFIG_HPFS_FS=m
+CONFIG_QNX4FS_FS=m
+CONFIG_QNX6FS_FS=m
+CONFIG_ROMFS_FS=m
+CONFIG_PSTORE=y
+CONFIG_PSTORE_RAM=m
+CONFIG_PSTORE_BLK=m
+CONFIG_UFS_FS=m
+CONFIG_EROFS_FS=m
+CONFIG_EROFS_FS_ZIP_LZMA=y
+CONFIG_EROFS_FS_ZIP_DEFLATE=y
+CONFIG_EROFS_FS_ZIP_ZSTD=y
+CONFIG_EROFS_FS_PCPU_KTHREAD=y
+# CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI is not set
+CONFIG_NFS_FS=m
+CONFIG_NFS_V2=m
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=m
+CONFIG_NFS_SWAP=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_NFS_V4_1_MIGRATION=y
+CONFIG_NFS_FSCACHE=y
+# CONFIG_NFS_V4_2_READ_PLUS is not set
+CONFIG_NFSD=m
+CONFIG_NFSD_V3_ACL=y
+CONFIG_NFSD_V4=y
+CONFIG_NFSD_BLOCKLAYOUT=y
+CONFIG_NFSD_SCSILAYOUT=y
+CONFIG_NFSD_FLEXFILELAYOUT=y
+CONFIG_NFSD_V4_2_INTER_SSC=y
+CONFIG_SUNRPC_DEBUG=y
+CONFIG_CEPH_FS=m
+CONFIG_CEPH_FSCACHE=y
+CONFIG_CEPH_FS_POSIX_ACL=y
+CONFIG_CIFS=m
+# CONFIG_CIFS_STATS2 is not set
+CONFIG_CIFS_UPCALL=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_CIFS_DFS_UPCALL=y
+CONFIG_CIFS_SWN_UPCALL=y
+CONFIG_CIFS_FSCACHE=y
+CONFIG_SMB_SERVER=m
+CONFIG_CODA_FS=m
+CONFIG_AFS_FS=m
+CONFIG_AFS_FSCACHE=y
+CONFIG_9P_FS=m
+CONFIG_9P_FSCACHE=y
+CONFIG_9P_FS_POSIX_ACL=y
+CONFIG_9P_FS_SECURITY=y
+CONFIG_NLS_DEFAULT="utf8"
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_737=m
+CONFIG_NLS_CODEPAGE_775=m
+CONFIG_NLS_CODEPAGE_850=m
+CONFIG_NLS_CODEPAGE_852=m
+CONFIG_NLS_CODEPAGE_855=m
+CONFIG_NLS_CODEPAGE_857=m
+CONFIG_NLS_CODEPAGE_860=m
+CONFIG_NLS_CODEPAGE_861=m
+CONFIG_NLS_CODEPAGE_862=m
+CONFIG_NLS_CODEPAGE_863=m
+CONFIG_NLS_CODEPAGE_864=m
+CONFIG_NLS_CODEPAGE_865=m
+CONFIG_NLS_CODEPAGE_866=m
+CONFIG_NLS_CODEPAGE_869=m
+CONFIG_NLS_CODEPAGE_936=m
+CONFIG_NLS_CODEPAGE_950=m
+CONFIG_NLS_CODEPAGE_932=m
+CONFIG_NLS_CODEPAGE_949=m
+CONFIG_NLS_CODEPAGE_874=m
+CONFIG_NLS_ISO8859_8=m
+CONFIG_NLS_CODEPAGE_1250=m
+CONFIG_NLS_CODEPAGE_1251=m
+CONFIG_NLS_ASCII=m
+CONFIG_NLS_ISO8859_1=m
+CONFIG_NLS_ISO8859_2=m
+CONFIG_NLS_ISO8859_3=m
+CONFIG_NLS_ISO8859_4=m
+CONFIG_NLS_ISO8859_5=m
+CONFIG_NLS_ISO8859_6=m
+CONFIG_NLS_ISO8859_7=m
+CONFIG_NLS_ISO8859_9=m
+CONFIG_NLS_ISO8859_13=m
+CONFIG_NLS_ISO8859_14=m
+CONFIG_NLS_ISO8859_15=m
+CONFIG_NLS_KOI8_R=m
+CONFIG_NLS_KOI8_U=m
+CONFIG_NLS_MAC_ROMAN=m
+CONFIG_NLS_MAC_CELTIC=m
+CONFIG_NLS_MAC_CENTEURO=m
+CONFIG_NLS_MAC_CROATIAN=m
+CONFIG_NLS_MAC_CYRILLIC=m
+CONFIG_NLS_MAC_GAELIC=m
+CONFIG_NLS_MAC_GREEK=m
+CONFIG_NLS_MAC_ICELAND=m
+CONFIG_NLS_MAC_INUIT=m
+CONFIG_NLS_MAC_ROMANIAN=m
+CONFIG_NLS_MAC_TURKISH=m
+CONFIG_DLM=m
+CONFIG_UNICODE=y
+CONFIG_KEYS_REQUEST_CACHE=y
+CONFIG_PERSISTENT_KEYRINGS=y
+CONFIG_SECURITY_DMESG_RESTRICT=y
+CONFIG_SECURITY=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_SECURITY_LANDLOCK=y
+# CONFIG_INTEGRITY is not set
+CONFIG_LSM="landlock,apparmor"
+CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y
+CONFIG_ZERO_CALL_USED_REGS=y
+CONFIG_FORTIFY_SOURCE=y
+CONFIG_HARDENED_USERCOPY=y
+CONFIG_LIST_HARDENED=y
+CONFIG_CRYPTO_USER=m
+CONFIG_CRYPTO_PCRYPT=m
+CONFIG_CRYPTO_CRYPTD=m
+CONFIG_CRYPTO_DH=y
+CONFIG_CRYPTO_DH_RFC7919_GROUPS=y
+CONFIG_CRYPTO_ECDSA=m
+CONFIG_CRYPTO_ECRDSA=m
+CONFIG_CRYPTO_AES_TI=m
+CONFIG_CRYPTO_ARIA=m
+CONFIG_CRYPTO_BLOWFISH=m
+CONFIG_CRYPTO_CAST5=m
+CONFIG_CRYPTO_CAST6=m
+CONFIG_CRYPTO_DES=m
+CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_TWOFISH=m
+CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
+CONFIG_CRYPTO_LRW=m
+CONFIG_CRYPTO_AEGIS128=m
+CONFIG_CRYPTO_GCM=y
+CONFIG_CRYPTO_SEQIV=y
+CONFIG_CRYPTO_ESSIV=m
+CONFIG_CRYPTO_MD4=m
+CONFIG_CRYPTO_MICHAEL_MIC=m
+CONFIG_CRYPTO_RMD160=m
+CONFIG_CRYPTO_WP512=m
+CONFIG_CRYPTO_XCBC=m
+CONFIG_CRYPTO_DEFLATE=y
+CONFIG_CRYPTO_LZO=y
+CONFIG_CRYPTO_842=m
+CONFIG_CRYPTO_LZ4=m
+CONFIG_CRYPTO_LZ4HC=m
+CONFIG_CRYPTO_ZSTD=m
+CONFIG_CRYPTO_ANSI_CPRNG=m
+CONFIG_CRYPTO_USER_API_HASH=m
+CONFIG_CRYPTO_USER_API_SKCIPHER=m
+CONFIG_CRYPTO_USER_API_RNG=m
+CONFIG_CRYPTO_USER_API_AEAD=m
+# CONFIG_CRYPTO_USER_API_ENABLE_OBSOLETE is not set
+CONFIG_CRYPTO_DEV_ATMEL_ECC=m
+CONFIG_CRYPTO_DEV_ATMEL_SHA204A=m
+CONFIG_CRYPTO_DEV_VIRTIO=m
+CONFIG_CRYPTO_DEV_SAFEXCEL=m
+CONFIG_CRYPTO_DEV_CCREE=m
+CONFIG_CRYPTO_DEV_AMLOGIC_GXL=m
+CONFIG_PKCS8_PRIVATE_KEY_PARSER=m
+CONFIG_PKCS7_TEST_KEY=m
+CONFIG_SIGNED_PE_FILE_VERIFICATION=y
+CONFIG_CORDIC=m
+CONFIG_XZ_DEC_TEST=m
+CONFIG_DMA_RESTRICTED_POOL=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_BOOT_PRINTK_DELAY=y
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_DEBUG_INFO_BTF=y
+CONFIG_GDB_SCRIPTS=y
+CONFIG_VMLINUX_MAP=y
+CONFIG_KGDB=y
+# CONFIG_KGDB_HONOUR_BLOCKLIST is not set
+CONFIG_KGDB_KDB=y
+CONFIG_KDB_KEYBOARD=y
+CONFIG_PAGE_POISONING=y
+CONFIG_DEBUG_WX=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DETECT_HUNG_TASK=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_RCU_CPU_STALL_CPUTIME=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_LATENCYTOP=y
+CONFIG_SCHED_TRACER=y
+CONFIG_FTRACE_SYSCALLS=y
+CONFIG_BLK_DEV_IO_TRACE=y
+# CONFIG_UPROBE_EVENTS is not set
+CONFIG_SAMPLES=y
+CONFIG_SAMPLE_TRACE_PRINTK=m
+CONFIG_SAMPLE_TRACE_ARRAY=m
+# CONFIG_RUNTIME_KERNEL_TESTING_MENU is not set
+CONFIG_NOTIFIER_ERROR_INJECTION=m
+CONFIG_TEST_BPF=m
+CONFIG_MEMTEST=y
diff --git a/arch/riscv/configs/k3_defconfig b/arch/riscv/configs/k3_defconfig
new file mode 100644
index 000000000000..c00377ecf3f5
--- /dev/null
+++ b/arch/riscv/configs/k3_defconfig
@@ -0,0 +1,562 @@
+CONFIG_SYSVIPC=y
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
+CONFIG_PREEMPT=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_CGROUPS=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_INITRAMFS_SOURCE="rootfs.cpio.gz"
+# CONFIG_RD_BZIP2 is not set
+# CONFIG_RD_LZMA is not set
+# CONFIG_RD_XZ is not set
+# CONFIG_RD_LZO is not set
+# CONFIG_RD_LZ4 is not set
+# CONFIG_RD_ZSTD is not set
+CONFIG_EXPERT=y
+CONFIG_SOC_SPACEMIT=y
+CONFIG_SOC_SPACEMIT_K3=y
+CONFIG_SMP=y
+CONFIG_NR_CPUS=16
+# CONFIG_RISCV_ISA_VENDOR_EXT_ANDES is not set
+CONFIG_HZ_100=y
+CONFIG_RISCV_SBI_V01=y
+CONFIG_CMDLINE="console=ttyS0"
+# CONFIG_DMI is not set
+CONFIG_RISCV_BUILD_ITB_IMAGE=y
+CONFIG_PM_DEBUG=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_SPACEMIT_K3_CPUFREQ=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=y
+CONFIG_KPROBES=y
+CONFIG_JUMP_LABEL=y
+# CONFIG_GCC_PLUGINS is not set
+CONFIG_MODULES=y
+# CONFIG_COREDUMP is not set
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_PAGE_REPORTING=y
+CONFIG_CMA=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_FIB_TRIE_STATS=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_IP_ROUTE_MULTIPATH=y
+CONFIG_IP_ROUTE_VERBOSE=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_IP_PNP_RARP=y
+CONFIG_NETFILTER=y
+CONFIG_NETLINK_DIAG=y
+CONFIG_CAN=y
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HCIBTUSB=m
+CONFIG_BT_HCIBTSDIO=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_H4=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_CFG80211=y
+CONFIG_MAC80211=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_PCI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_PCIE_SPACEMIT_EP=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_PCI_ENDPOINT_CONFIGFS=y
+CONFIG_PCI_EPF_TEST=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+# CONFIG_ALLOW_DEV_COREDUMP is not set
+CONFIG_MTD=y
+CONFIG_MTD_CMDLINE_PARTS=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_PARTITIONED_MASTER=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_NVME=y
+CONFIG_RPMB=y
+CONFIG_RT7451_RETIMER=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_TARGET_CORE=y
+CONFIG_TCM_IBLOCK=y
+CONFIG_TCM_FILEIO=y
+CONFIG_NETDEVICES=y
+CONFIG_R8169=m
+CONFIG_R8127=m
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_SPACEMIT_ETHQOS=y
+CONFIG_ETHERCAT=y
+CONFIG_EC_MASTER=y
+CONFIG_EC_DEVICE=y
+CONFIG_EC_K3_GMAC=y
+CONFIG_LED_TRIGGER_PHY=y
+CONFIG_SFP=y
+CONFIG_REALTEK_PHY=y
+CONFIG_CAN_FLEXCAN=y
+CONFIG_USB_RTL8152=m
+CONFIG_USB_USBNET=m
+# CONFIG_USB_NET_AX8817X is not set
+CONFIG_USB_NET_CDC_EEM=m
+CONFIG_USB_NET_CDC_MBIM=m
+# CONFIG_USB_NET_NET1080 is not set
+CONFIG_USB_NET_RNDIS_HOST=m
+# CONFIG_USB_BELKIN is not set
+# CONFIG_USB_ARMLINUX is not set
+# CONFIG_USB_NET_ZAURUS is not set
+CONFIG_USB_NET_QMI_WWAN=m
+CONFIG_RTW89=y
+CONFIG_RTW89_8852BE=m
+CONFIG_RTL8852BS=m
+CONFIG_INPUT_EVDEV=y
+CONFIG_KEYBOARD_CROS_EC=y
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_RASPITS_FT5426=y
+CONFIG_INPUT_MISC=y
+CONFIG_INPUT_UINPUT=y
+CONFIG_INPUT_DRV_RPMI_PWRKEY=y
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_NR_UARTS=17
+CONFIG_SERIAL_8250_RUNTIME_UARTS=17
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+# CONFIG_HW_RANDOM is not set
+# CONFIG_DEVPORT is not set
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_CROS_EC_TUNNEL=y
+CONFIG_I2C_SLAVE=y
+CONFIG_I2C_SPACEMIT_SLAVE=y
+CONFIG_SPI=y
+CONFIG_SPI_FSL_QUADSPI=y
+CONFIG_SPI_SPACEMIT_K1=y
+CONFIG_SPI_SPIDEV=y
+CONFIG_PINCTRL=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_SPACEMIT_K1=y
+CONFIG_GPIO_CROS_EC=y
+CONFIG_POWER_SUPPLY=y
+CONFIG_SENSORS_CTF2301=y
+CONFIG_SENSORS_PWM_FAN=y
+CONFIG_THERMAL=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_K3_THERMAL=y
+CONFIG_WATCHDOG=y
+CONFIG_SPACEMIT_K1_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+# CONFIG_REGULATOR_SPACEMIT_P1 is not set
+# CONFIG_REGULATOR_SPACEMIT_MPQ8655 is not set
+CONFIG_RC_CORE=y
+CONFIG_RC_DECODERS=y
+CONFIG_IR_NEC_DECODER=y
+CONFIG_RC_DEVICES=y
+CONFIG_IR_SPACEMIT_K1=y
+# CONFIG_MEDIA_CEC_SUPPORT is not set
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=y
+CONFIG_VIDEO_LINLON_K3=y
+CONFIG_SPACEMIT_K3_CCIC=y
+CONFIG_SPACEMIT_K3_CCIC_IOMMU=y
+CONFIG_SPACEMIT_K3_CAM_IMX219=y
+CONFIG_SPACEMIT_K3_CAM_IMX415=y
+CONFIG_SPACEMIT_K3_CAM_OV5647=y
+CONFIG_SPACEMIT_K3_CAM_OV2735=y
+CONFIG_SPACEMIT_K3_CAM_OV5640_MAX96724_MAX9295=y
+# CONFIG_MEDIA_TUNER_E4000 is not set
+# CONFIG_MEDIA_TUNER_FC0011 is not set
+# CONFIG_MEDIA_TUNER_FC0012 is not set
+# CONFIG_MEDIA_TUNER_FC0013 is not set
+# CONFIG_MEDIA_TUNER_FC2580 is not set
+# CONFIG_MEDIA_TUNER_IT913X is not set
+# CONFIG_MEDIA_TUNER_M88RS6000T is not set
+# CONFIG_MEDIA_TUNER_MAX2165 is not set
+# CONFIG_MEDIA_TUNER_MC44S803 is not set
+# CONFIG_MEDIA_TUNER_MT2060 is not set
+# CONFIG_MEDIA_TUNER_MT2063 is not set
+# CONFIG_MEDIA_TUNER_MT20XX is not set
+# CONFIG_MEDIA_TUNER_MT2131 is not set
+# CONFIG_MEDIA_TUNER_MT2266 is not set
+# CONFIG_MEDIA_TUNER_MXL301RF is not set
+# CONFIG_MEDIA_TUNER_MXL5005S is not set
+# CONFIG_MEDIA_TUNER_MXL5007T is not set
+# CONFIG_MEDIA_TUNER_QM1D1B0004 is not set
+# CONFIG_MEDIA_TUNER_QM1D1C0042 is not set
+# CONFIG_MEDIA_TUNER_QT1010 is not set
+# CONFIG_MEDIA_TUNER_R820T is not set
+# CONFIG_MEDIA_TUNER_SI2157 is not set
+# CONFIG_MEDIA_TUNER_SIMPLE is not set
+# CONFIG_MEDIA_TUNER_TDA18212 is not set
+# CONFIG_MEDIA_TUNER_TDA18218 is not set
+# CONFIG_MEDIA_TUNER_TDA18250 is not set
+# CONFIG_MEDIA_TUNER_TDA18271 is not set
+# CONFIG_MEDIA_TUNER_TDA827X is not set
+# CONFIG_MEDIA_TUNER_TDA8290 is not set
+# CONFIG_MEDIA_TUNER_TDA9887 is not set
+# CONFIG_MEDIA_TUNER_TEA5761 is not set
+# CONFIG_MEDIA_TUNER_TEA5767 is not set
+# CONFIG_MEDIA_TUNER_TUA9001 is not set
+# CONFIG_MEDIA_TUNER_XC2028 is not set
+# CONFIG_MEDIA_TUNER_XC4000 is not set
+# CONFIG_MEDIA_TUNER_XC5000 is not set
+# CONFIG_DVB_MXL5XX is not set
+# CONFIG_DVB_STB0899 is not set
+# CONFIG_DVB_STB6100 is not set
+# CONFIG_DVB_STV090x is not set
+# CONFIG_DVB_STV0910 is not set
+# CONFIG_DVB_STV6110x is not set
+# CONFIG_DVB_STV6111 is not set
+# CONFIG_DVB_DRXK is not set
+# CONFIG_DVB_MN88472 is not set
+# CONFIG_DVB_MN88473 is not set
+# CONFIG_DVB_SI2165 is not set
+# CONFIG_DVB_TDA18271C2DD is not set
+# CONFIG_DVB_CX24110 is not set
+# CONFIG_DVB_CX24116 is not set
+# CONFIG_DVB_CX24117 is not set
+# CONFIG_DVB_CX24120 is not set
+# CONFIG_DVB_CX24123 is not set
+# CONFIG_DVB_DS3000 is not set
+# CONFIG_DVB_MB86A16 is not set
+# CONFIG_DVB_MT312 is not set
+# CONFIG_DVB_S5H1420 is not set
+# CONFIG_DVB_SI21XX is not set
+# CONFIG_DVB_STB6000 is not set
+# CONFIG_DVB_STV0288 is not set
+# CONFIG_DVB_STV0299 is not set
+# CONFIG_DVB_STV0900 is not set
+# CONFIG_DVB_STV6110 is not set
+# CONFIG_DVB_TDA10071 is not set
+# CONFIG_DVB_TDA10086 is not set
+# CONFIG_DVB_TDA8083 is not set
+# CONFIG_DVB_TDA8261 is not set
+# CONFIG_DVB_TDA826X is not set
+# CONFIG_DVB_TS2020 is not set
+# CONFIG_DVB_TUA6100 is not set
+# CONFIG_DVB_TUNER_CX24113 is not set
+# CONFIG_DVB_TUNER_ITD1000 is not set
+# CONFIG_DVB_VES1X93 is not set
+# CONFIG_DVB_ZL10036 is not set
+# CONFIG_DVB_ZL10039 is not set
+# CONFIG_DVB_CX22700 is not set
+# CONFIG_DVB_CX22702 is not set
+# CONFIG_DVB_CXD2820R is not set
+# CONFIG_DVB_CXD2841ER is not set
+# CONFIG_DVB_DIB3000MB is not set
+# CONFIG_DVB_DIB3000MC is not set
+# CONFIG_DVB_DIB7000M is not set
+# CONFIG_DVB_DIB7000P is not set
+# CONFIG_DVB_DIB9000 is not set
+# CONFIG_DVB_DRXD is not set
+# CONFIG_DVB_EC100 is not set
+# CONFIG_DVB_L64781 is not set
+# CONFIG_DVB_MT352 is not set
+# CONFIG_DVB_NXT6000 is not set
+# CONFIG_DVB_S5H1432 is not set
+# CONFIG_DVB_SP887X is not set
+# CONFIG_DVB_STV0367 is not set
+# CONFIG_DVB_TDA10048 is not set
+# CONFIG_DVB_TDA1004X is not set
+# CONFIG_DVB_ZD1301_DEMOD is not set
+# CONFIG_DVB_ZL10353 is not set
+# CONFIG_DVB_STV0297 is not set
+# CONFIG_DVB_TDA10021 is not set
+# CONFIG_DVB_TDA10023 is not set
+# CONFIG_DVB_VES1820 is not set
+# CONFIG_DVB_AU8522_DTV is not set
+# CONFIG_DVB_AU8522_V4L is not set
+# CONFIG_DVB_BCM3510 is not set
+# CONFIG_DVB_LG2160 is not set
+# CONFIG_DVB_LGDT3305 is not set
+# CONFIG_DVB_LGDT330X is not set
+# CONFIG_DVB_MXL692 is not set
+# CONFIG_DVB_NXT200X is not set
+# CONFIG_DVB_OR51132 is not set
+# CONFIG_DVB_OR51211 is not set
+# CONFIG_DVB_S5H1409 is not set
+# CONFIG_DVB_S5H1411 is not set
+# CONFIG_DVB_DIB8000 is not set
+# CONFIG_DVB_MB86A20S is not set
+# CONFIG_DVB_S921 is not set
+# CONFIG_DVB_MN88443X is not set
+# CONFIG_DVB_TC90522 is not set
+# CONFIG_DVB_PLL is not set
+# CONFIG_DVB_TUNER_DIB0070 is not set
+# CONFIG_DVB_TUNER_DIB0090 is not set
+# CONFIG_DVB_A8293 is not set
+# CONFIG_DVB_AF9033 is not set
+# CONFIG_DVB_ASCOT2E is not set
+# CONFIG_DVB_ATBM8830 is not set
+# CONFIG_DVB_HELENE is not set
+# CONFIG_DVB_HORUS3A is not set
+# CONFIG_DVB_ISL6405 is not set
+# CONFIG_DVB_ISL6421 is not set
+# CONFIG_DVB_ISL6423 is not set
+# CONFIG_DVB_IX2505V is not set
+# CONFIG_DVB_LGS8GL5 is not set
+# CONFIG_DVB_LGS8GXX is not set
+# CONFIG_DVB_LNBH25 is not set
+# CONFIG_DVB_LNBH29 is not set
+# CONFIG_DVB_LNBP21 is not set
+# CONFIG_DVB_LNBP22 is not set
+# CONFIG_DVB_M88RS2000 is not set
+# CONFIG_DVB_TDA665x is not set
+# CONFIG_DVB_DRX39XYJ is not set
+# CONFIG_DVB_CXD2099 is not set
+# CONFIG_DVB_SP2 is not set
+CONFIG_DRM=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_POWERVR_ROGUE=y
+CONFIG_POWERVR_DVFS=y
+CONFIG_POWERVR_THERMAL=y
+CONFIG_DRM_SPACEMIT=y
+CONFIG_SPACEMIT_INNO_DP=y
+CONFIG_SPACEMIT_MIPI_PANEL=y
+CONFIG_DRM_SPACEMIT_BACKLIGHT=y
+CONFIG_FB=y
+CONFIG_BACKLIGHT_PWM=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_USB_AUDIO=y
+CONFIG_SND_USB_AUDIO_MIDI_V2=y
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_K1_I2S=y
+CONFIG_SND_SOC_K3_RI2S=y
+CONFIG_SND_SOC_ES8326=y
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_HIDRAW=y
+CONFIG_UHID=y
+CONFIG_HID_MULTITOUCH=y
+CONFIG_I2C_HID_OF=y
+CONFIG_USB_HIDDEV=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_ACM=m
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+# CONFIG_USB_DWC3_OF_SIMPLE is not set
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_SIMPLE=m
+CONFIG_USB_SERIAL_AIRCABLE=m
+CONFIG_USB_SERIAL_ARK3116=m
+CONFIG_USB_SERIAL_BELKIN=m
+CONFIG_USB_SERIAL_CH341=m
+CONFIG_USB_SERIAL_WHITEHEAT=m
+CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m
+CONFIG_USB_SERIAL_CP210X=m
+CONFIG_USB_SERIAL_CYPRESS_M8=m
+CONFIG_USB_SERIAL_EMPEG=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_SERIAL_VISOR=m
+CONFIG_USB_SERIAL_IPAQ=m
+CONFIG_USB_SERIAL_IR=m
+CONFIG_USB_SERIAL_EDGEPORT=m
+CONFIG_USB_SERIAL_EDGEPORT_TI=m
+CONFIG_USB_SERIAL_F81232=m
+CONFIG_USB_SERIAL_F8153X=m
+CONFIG_USB_SERIAL_GARMIN=m
+CONFIG_USB_SERIAL_IPW=m
+CONFIG_USB_SERIAL_IUU=m
+CONFIG_USB_SERIAL_KEYSPAN_PDA=m
+CONFIG_USB_SERIAL_KEYSPAN=m
+CONFIG_USB_SERIAL_KLSI=m
+CONFIG_USB_SERIAL_KOBIL_SCT=m
+CONFIG_USB_SERIAL_MCT_U232=m
+CONFIG_USB_SERIAL_METRO=m
+CONFIG_USB_SERIAL_MOS7720=m
+CONFIG_USB_SERIAL_MOS7840=m
+CONFIG_USB_SERIAL_MXUPORT=m
+CONFIG_USB_SERIAL_NAVMAN=m
+CONFIG_USB_SERIAL_PL2303=m
+CONFIG_USB_SERIAL_OTI6858=m
+CONFIG_USB_SERIAL_QCAUX=m
+CONFIG_USB_SERIAL_QUALCOMM=m
+CONFIG_USB_SERIAL_SPCP8X5=m
+CONFIG_USB_SERIAL_SAFE=m
+CONFIG_USB_SERIAL_SAFE_PADDED=y
+CONFIG_USB_SERIAL_SIERRAWIRELESS=m
+CONFIG_USB_SERIAL_SYMBOL=m
+CONFIG_USB_SERIAL_TI=m
+CONFIG_USB_SERIAL_CYBERJACK=m
+CONFIG_USB_SERIAL_OPTION=m
+CONFIG_USB_SERIAL_OMNINET=m
+CONFIG_USB_SERIAL_OPTICON=m
+CONFIG_USB_SERIAL_XSENS_MT=m
+CONFIG_USB_SERIAL_WISHBONE=m
+CONFIG_USB_SERIAL_SSU100=m
+CONFIG_USB_SERIAL_QT2=m
+CONFIG_USB_SERIAL_UPD78F0730=m
+CONFIG_USB_SERIAL_XR=m
+CONFIG_USB_SERIAL_DEBUG=m
+CONFIG_USB_ONBOARD_DEV=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_RNDIS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_USB_CONFIGFS_F_TCM=y
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB301=m
+CONFIG_MMC=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_K1=y
+CONFIG_SCSI_UFSHCD=y
+CONFIG_SCSI_UFSHCD_PLATFORM=y
+CONFIG_SCSI_UFS_SPACEMIT_K3=y
+CONFIG_NEW_LEDS=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_TRIGGERS=y
+CONFIG_LEDS_TRIGGER_NETDEV=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_CROS_EC=y
+CONFIG_RTC_DRV_RPMI=y
+CONFIG_DMADEVICES=y
+CONFIG_SPACEMIT_HDMA=y
+CONFIG_USERSPACE_DMA=y
+CONFIG_DW_EDMA=y
+CONFIG_DMATEST=y
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VFIO=y
+CONFIG_VFIO_PCI=y
+# CONFIG_VIRTIO_MENU is not set
+# CONFIG_VHOST_MENU is not set
+CONFIG_CHROME_PLATFORMS=y
+CONFIG_CROS_EC=y
+CONFIG_CROS_EC_ESPI=y
+CONFIG_SPACEMIT_CCU=y
+CONFIG_SPACEMIT_K3_CCU=y
+CONFIG_HWSPINLOCK=y
+CONFIG_SPACEMIT_K1X_TIMER=y
+CONFIG_MAILBOX=y
+CONFIG_SPACEMIT_MAILBOX=y
+CONFIG_K3_MAILBOX=y
+CONFIG_RISCV_IOMMU_SPACEMIT_K3=y
+CONFIG_REMOTEPROC=y
+CONFIG_REMOTEPROC_CDEV=y
+CONFIG_SPACEMIT_K3_REMOTEPROC=y
+CONFIG_RPMSG_CHAR=y
+CONFIG_RPMSG_CTRL=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_SPACEMIT_HMP=y
+CONFIG_SPACEMIT_PM_DOMAINS=y
+CONFIG_SPACEMIT_REBOOT_CONTROL=y
+CONFIG_SPACEMIT_SOCINFO=y
+CONFIG_SPACEMIT_POWERKEY_MONITOR=y
+CONFIG_SPACEMIT_DDR_BW=y
+CONFIG_SPACEMIT_V2D=y
+CONFIG_PWM=y
+CONFIG_PWM_CROS_EC=y
+CONFIG_PWM_PXA=y
+CONFIG_PHY_SPACEMIT_K1_USB2=y
+CONFIG_PHY_SPACEMIT_K3_USB3=y
+CONFIG_PHY_SPACEMIT_K3_PCIE=y
+CONFIG_NVMEM_SPACEMIT_EFUSE=y
+CONFIG_CORESIGHT=y
+CONFIG_CORESIGHT_LINKS_AND_SINKS=y
+CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y
+CONFIG_RVTRACE=y
+CONFIG_RVTRACE_ENCODER=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+# CONFIG_DNOTIFY is not set
+CONFIG_AUTOFS_FS=m
+CONFIG_FUSE_FS=y
+CONFIG_CUSE=m
+CONFIG_VIRTIO_FS=m
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_MSDOS_FS=y
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_PROC_CHILDREN=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+# CONFIG_EFIVAR_FS is not set
+# CONFIG_MISC_FILESYSTEMS is not set
+CONFIG_NFS_FS=y
+CONFIG_NFS_V2=y
+CONFIG_NFS_V4=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_ROOT_NFS=y
+CONFIG_NFSD=y
+CONFIG_NFSD_V3_ACL=y
+CONFIG_NFSD_V4=y
+CONFIG_NFSD_BLOCKLAYOUT=y
+CONFIG_NFSD_SCSILAYOUT=y
+CONFIG_NFSD_FLEXFILELAYOUT=y
+CONFIG_NFSD_V4_2_INTER_SSC=y
+CONFIG_SUNRPC_DEBUG=y
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ASCII=m
+CONFIG_NLS_ISO8859_1=m
+CONFIG_NLS_ISO8859_2=m
+CONFIG_NLS_ISO8859_3=m
+CONFIG_NLS_ISO8859_4=m
+CONFIG_NLS_ISO8859_5=m
+CONFIG_NLS_ISO8859_6=m
+CONFIG_NLS_ISO8859_7=m
+CONFIG_NLS_ISO8859_9=m
+CONFIG_NLS_ISO8859_13=m
+CONFIG_NLS_ISO8859_14=m
+CONFIG_NLS_ISO8859_15=m
+CONFIG_NLS_UTF8=m
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_DYNAMIC_DEBUG=y
+# CONFIG_DEBUG_MISC is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_DEBUG_INFO_BTF=y
+CONFIG_DEBUG_PAGEALLOC=y
+CONFIG_SCHED_STACK_END_CHECK=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VM_PGFLAGS=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DEBUG_LIST=y
+CONFIG_DEBUG_PLIST=y
+CONFIG_DEBUG_SG=y
+CONFIG_RCU_CPU_STALL_CPUTIME=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_RCU_EQS_DEBUG=y
+CONFIG_FTRACE_SYSCALLS=y
+# CONFIG_RUNTIME_KERNEL_TESTING_MENU is not set
+# CONFIG_RUNTIME_TESTING_MENU is not set
diff --git a/arch/riscv/configs/k3_rt_bianbu_defconfig b/arch/riscv/configs/k3_rt_bianbu_defconfig
new file mode 100644
index 000000000000..dae101274510
--- /dev/null
+++ b/arch/riscv/configs/k3_rt_bianbu_defconfig
@@ -0,0 +1,1191 @@
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_SYSVIPC=y
+CONFIG_POSIX_MQUEUE=y
+CONFIG_WATCH_QUEUE=y
+CONFIG_NO_HZ=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_JIT=y
+CONFIG_BPF_JIT_ALWAYS_ON=y
+CONFIG_PREEMPT_RT=y
+CONFIG_PREEMPT_DYNAMIC=y
+CONFIG_BSD_PROCESS_ACCT=y
+CONFIG_BSD_PROCESS_ACCT_V3=y
+CONFIG_PSI=y
+CONFIG_IKCONFIG=y
+CONFIG_IKCONFIG_PROC=y
+CONFIG_IKHEADERS=m
+CONFIG_MEMCG=y
+CONFIG_BLK_CGROUP=y
+CONFIG_CFS_BANDWIDTH=y
+CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_FREEZER=y
+CONFIG_CPUSETS=y
+CONFIG_CGROUP_DEVICE=y
+CONFIG_CGROUP_CPUACCT=y
+CONFIG_CGROUP_PERF=y
+CONFIG_CGROUP_BPF=y
+CONFIG_CGROUP_MISC=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_CHECKPOINT_RESTORE=y
+CONFIG_SCHED_AUTOGROUP=y
+CONFIG_BOOT_CONFIG=y
+CONFIG_EXPERT=y
+CONFIG_SGETMASK_SYSCALL=y
+CONFIG_PROFILING=y
+CONFIG_KEXEC=y
+CONFIG_KEXEC_FILE=y
+CONFIG_SOC_SPACEMIT=y
+CONFIG_SOC_SPACEMIT_K3=y
+CONFIG_SMP=y
+CONFIG_NR_CPUS=16
+# CONFIG_RISCV_ISA_VENDOR_EXT_ANDES is not set
+CONFIG_RISCV_SBI_V01=y
+# CONFIG_DMI is not set
+CONFIG_RISCV_BUILD_ITB_IMAGE=y
+CONFIG_PM_DEBUG=y
+CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y
+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
+CONFIG_CPU_FREQ_GOV_POWERSAVE=y
+CONFIG_CPU_FREQ_GOV_ONDEMAND=y
+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y
+CONFIG_SPACEMIT_K3_CPUFREQ=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=y
+CONFIG_KPROBES=y
+# CONFIG_STACKPROTECTOR_STRONG is not set
+# CONFIG_RANDOMIZE_KSTACK_OFFSET is not set
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODULE_COMPRESS=y
+CONFIG_MODULE_COMPRESS_ZSTD=y
+CONFIG_MODULE_DECOMPRESS=y
+CONFIG_BLK_DEV_BSGLIB=y
+CONFIG_BLK_DEV_ZONED=y
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BLK_CGROUP_IOPRIO=y
+CONFIG_BLK_SED_OPAL=y
+CONFIG_BLK_INLINE_ENCRYPTION=y
+CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y
+CONFIG_PARTITION_ADVANCED=y
+CONFIG_AIX_PARTITION=y
+CONFIG_OSF_PARTITION=y
+CONFIG_AMIGA_PARTITION=y
+CONFIG_ATARI_PARTITION=y
+CONFIG_MAC_PARTITION=y
+CONFIG_BSD_DISKLABEL=y
+CONFIG_MINIX_SUBPARTITION=y
+CONFIG_SOLARIS_X86_PARTITION=y
+CONFIG_UNIXWARE_DISKLABEL=y
+CONFIG_LDM_PARTITION=y
+CONFIG_SGI_PARTITION=y
+CONFIG_ULTRIX_PARTITION=y
+CONFIG_SUN_PARTITION=y
+CONFIG_KARMA_PARTITION=y
+CONFIG_SYSV68_PARTITION=y
+CONFIG_CMDLINE_PARTITION=y
+CONFIG_MQ_IOSCHED_KYBER=m
+CONFIG_IOSCHED_BFQ=m
+CONFIG_BINFMT_MISC=m
+CONFIG_SLAB_BUCKETS=y
+CONFIG_RANDOM_KMALLOC_CACHES=y
+CONFIG_SHUFFLE_PAGE_ALLOCATOR=y
+# CONFIG_COMPAT_BRK is not set
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_PAGE_REPORTING=y
+CONFIG_KSM=y
+CONFIG_CMA=y
+CONFIG_CMA_DEBUGFS=y
+CONFIG_CMA_SYSFS=y
+CONFIG_IDLE_PAGE_TRACKING=y
+CONFIG_ANON_VMA_NAME=y
+CONFIG_USERFAULTFD=y
+CONFIG_LRU_GEN=y
+CONFIG_LRU_GEN_ENABLED=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_PACKET_DIAG=m
+CONFIG_UNIX_DIAG=m
+CONFIG_TLS=m
+CONFIG_TLS_DEVICE=y
+CONFIG_XFRM_USER=m
+CONFIG_XFRM_INTERFACE=m
+CONFIG_XFRM_STATISTICS=y
+CONFIG_NET_KEY=m
+CONFIG_XDP_SOCKETS=y
+CONFIG_XDP_SOCKETS_DIAG=m
+CONFIG_IP_MULTICAST=y
+CONFIG_IP_ADVANCED_ROUTER=y
+CONFIG_IP_FIB_TRIE_STATS=y
+CONFIG_IP_MULTIPLE_TABLES=y
+CONFIG_IP_ROUTE_MULTIPATH=y
+CONFIG_IP_ROUTE_VERBOSE=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_NET_IPIP=m
+CONFIG_NET_IPGRE_DEMUX=m
+CONFIG_NET_IPGRE=m
+CONFIG_NET_IPGRE_BROADCAST=y
+CONFIG_IP_MROUTE=y
+CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IP_PIMSM_V1=y
+CONFIG_IP_PIMSM_V2=y
+CONFIG_NET_IPVTI=m
+CONFIG_NET_FOU_IP_TUNNELS=y
+CONFIG_INET_AH=m
+CONFIG_INET_ESP=m
+CONFIG_INET_ESP_OFFLOAD=m
+CONFIG_INET_ESPINTCP=y
+CONFIG_INET_IPCOMP=m
+CONFIG_INET_DIAG=m
+CONFIG_INET_UDP_DIAG=m
+CONFIG_INET_RAW_DIAG=m
+CONFIG_INET_DIAG_DESTROY=y
+CONFIG_TCP_CONG_ADVANCED=y
+CONFIG_TCP_CONG_HSTCP=m
+CONFIG_TCP_CONG_HYBLA=m
+CONFIG_TCP_CONG_NV=m
+CONFIG_TCP_CONG_SCALABLE=m
+CONFIG_TCP_CONG_LP=m
+CONFIG_TCP_CONG_VENO=m
+CONFIG_TCP_CONG_YEAH=m
+CONFIG_TCP_CONG_ILLINOIS=m
+CONFIG_TCP_CONG_DCTCP=m
+CONFIG_TCP_CONG_CDG=m
+CONFIG_TCP_CONG_BBR=m
+CONFIG_TCP_AO=y
+CONFIG_TCP_MD5SIG=y
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_INET6_AH=m
+CONFIG_INET6_ESP=m
+CONFIG_INET6_ESP_OFFLOAD=m
+CONFIG_INET6_ESPINTCP=y
+CONFIG_INET6_IPCOMP=m
+CONFIG_IPV6_MIP6=m
+CONFIG_IPV6_ILA=m
+CONFIG_IPV6_VTI=m
+CONFIG_IPV6_SIT=m
+CONFIG_IPV6_SIT_6RD=y
+CONFIG_IPV6_GRE=m
+CONFIG_IPV6_SUBTREES=y
+CONFIG_IPV6_MROUTE=y
+CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y
+CONFIG_IPV6_PIMSM_V2=y
+CONFIG_IPV6_SEG6_LWTUNNEL=y
+CONFIG_IPV6_SEG6_HMAC=y
+CONFIG_IPV6_IOAM6_LWTUNNEL=y
+CONFIG_NETLABEL=y
+CONFIG_MPTCP=y
+CONFIG_NETWORK_SECMARK=y
+CONFIG_NETWORK_PHY_TIMESTAMPING=y
+CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
+CONFIG_NF_CONNTRACK=m
+CONFIG_NF_CONNTRACK_SECMARK=y
+CONFIG_NF_CONNTRACK_ZONES=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CONNTRACK_TIMEOUT=y
+CONFIG_NF_CONNTRACK_TIMESTAMP=y
+CONFIG_NF_CONNTRACK_AMANDA=m
+CONFIG_NF_CONNTRACK_FTP=m
+CONFIG_NF_CONNTRACK_H323=m
+CONFIG_NF_CONNTRACK_IRC=m
+CONFIG_NF_CONNTRACK_NETBIOS_NS=m
+CONFIG_NF_CONNTRACK_SNMP=m
+CONFIG_NF_CONNTRACK_PPTP=m
+CONFIG_NF_CONNTRACK_SANE=m
+CONFIG_NF_CONNTRACK_SIP=m
+CONFIG_NF_CONNTRACK_TFTP=m
+CONFIG_NF_CT_NETLINK=m
+CONFIG_NF_CT_NETLINK_TIMEOUT=m
+CONFIG_NF_CT_NETLINK_HELPER=m
+CONFIG_NETFILTER_NETLINK_GLUE_CT=y
+CONFIG_NF_TABLES=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NUMGEN=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_CONNLIMIT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_NAT=m
+CONFIG_NFT_TUNNEL=m
+CONFIG_NFT_QUEUE=m
+CONFIG_NFT_QUOTA=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_COMPAT=m
+CONFIG_NFT_HASH=m
+CONFIG_NFT_XFRM=m
+CONFIG_NFT_SOCKET=m
+CONFIG_NFT_OSF=m
+CONFIG_NFT_TPROXY=m
+CONFIG_NFT_SYNPROXY=m
+CONFIG_NFT_DUP_NETDEV=m
+CONFIG_NFT_FWD_NETDEV=m
+CONFIG_NFT_REJECT_NETDEV=m
+CONFIG_NF_FLOW_TABLE_INET=m
+CONFIG_NF_FLOW_TABLE=m
+CONFIG_NETFILTER_XTABLES_COMPAT=y
+CONFIG_NETFILTER_XT_SET=m
+CONFIG_NETFILTER_XT_TARGET_AUDIT=m
+CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
+CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m
+CONFIG_NETFILTER_XT_TARGET_CONNMARK=m
+CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m
+CONFIG_NETFILTER_XT_TARGET_CT=m
+CONFIG_NETFILTER_XT_TARGET_DSCP=m
+CONFIG_NETFILTER_XT_TARGET_HL=m
+CONFIG_NETFILTER_XT_TARGET_HMARK=m
+CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m
+CONFIG_NETFILTER_XT_TARGET_LOG=m
+CONFIG_NETFILTER_XT_TARGET_MARK=m
+CONFIG_NETFILTER_XT_NAT=m
+CONFIG_NETFILTER_XT_TARGET_NETMAP=m
+CONFIG_NETFILTER_XT_TARGET_NFLOG=m
+CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m
+CONFIG_NETFILTER_XT_TARGET_REDIRECT=m
+CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m
+CONFIG_NETFILTER_XT_TARGET_TEE=m
+CONFIG_NETFILTER_XT_TARGET_TPROXY=m
+CONFIG_NETFILTER_XT_TARGET_SECMARK=m
+CONFIG_NETFILTER_XT_TARGET_TCPMSS=m
+CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
+CONFIG_NETFILTER_XT_MATCH_BPF=m
+CONFIG_NETFILTER_XT_MATCH_CGROUP=m
+CONFIG_NETFILTER_XT_MATCH_CLUSTER=m
+CONFIG_NETFILTER_XT_MATCH_COMMENT=m
+CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m
+CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m
+CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_CONNMARK=m
+CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_CPU=m
+CONFIG_NETFILTER_XT_MATCH_DCCP=m
+CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m
+CONFIG_NETFILTER_XT_MATCH_DSCP=m
+CONFIG_NETFILTER_XT_MATCH_ESP=m
+CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m
+CONFIG_NETFILTER_XT_MATCH_HELPER=m
+CONFIG_NETFILTER_XT_MATCH_IPCOMP=m
+CONFIG_NETFILTER_XT_MATCH_IPRANGE=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NETFILTER_XT_MATCH_LENGTH=m
+CONFIG_NETFILTER_XT_MATCH_LIMIT=m
+CONFIG_NETFILTER_XT_MATCH_MAC=m
+CONFIG_NETFILTER_XT_MATCH_MARK=m
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m
+CONFIG_NETFILTER_XT_MATCH_NFACCT=m
+CONFIG_NETFILTER_XT_MATCH_OSF=m
+CONFIG_NETFILTER_XT_MATCH_OWNER=m
+CONFIG_NETFILTER_XT_MATCH_POLICY=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m
+CONFIG_NETFILTER_XT_MATCH_QUOTA=m
+CONFIG_NETFILTER_XT_MATCH_RATEEST=m
+CONFIG_NETFILTER_XT_MATCH_REALM=m
+CONFIG_NETFILTER_XT_MATCH_RECENT=m
+CONFIG_NETFILTER_XT_MATCH_SOCKET=m
+CONFIG_NETFILTER_XT_MATCH_STATE=m
+CONFIG_NETFILTER_XT_MATCH_STATISTIC=m
+CONFIG_NETFILTER_XT_MATCH_STRING=m
+CONFIG_NETFILTER_XT_MATCH_TCPMSS=m
+CONFIG_NETFILTER_XT_MATCH_TIME=m
+CONFIG_NETFILTER_XT_MATCH_U32=m
+CONFIG_IP_SET=m
+CONFIG_IP_SET_BITMAP_IP=m
+CONFIG_IP_SET_BITMAP_IPMAC=m
+CONFIG_IP_SET_BITMAP_PORT=m
+CONFIG_IP_SET_HASH_IP=m
+CONFIG_IP_SET_HASH_IPMARK=m
+CONFIG_IP_SET_HASH_IPPORT=m
+CONFIG_IP_SET_HASH_IPPORTIP=m
+CONFIG_IP_SET_HASH_IPPORTNET=m
+CONFIG_IP_SET_HASH_IPMAC=m
+CONFIG_IP_SET_HASH_MAC=m
+CONFIG_IP_SET_HASH_NETPORTNET=m
+CONFIG_IP_SET_HASH_NET=m
+CONFIG_IP_SET_HASH_NETNET=m
+CONFIG_IP_SET_HASH_NETPORT=m
+CONFIG_IP_SET_HASH_NETIFACE=m
+CONFIG_IP_SET_LIST_SET=m
+CONFIG_IP_VS=m
+CONFIG_IP_VS_IPV6=y
+CONFIG_IP_VS_PROTO_TCP=y
+CONFIG_IP_VS_PROTO_UDP=y
+CONFIG_IP_VS_PROTO_ESP=y
+CONFIG_IP_VS_PROTO_AH=y
+CONFIG_IP_VS_PROTO_SCTP=y
+CONFIG_IP_VS_RR=m
+CONFIG_IP_VS_WRR=m
+CONFIG_IP_VS_LC=m
+CONFIG_IP_VS_WLC=m
+CONFIG_IP_VS_FO=m
+CONFIG_IP_VS_OVF=m
+CONFIG_IP_VS_LBLC=m
+CONFIG_IP_VS_LBLCR=m
+CONFIG_IP_VS_DH=m
+CONFIG_IP_VS_SH=m
+CONFIG_IP_VS_MH=m
+CONFIG_IP_VS_SED=m
+CONFIG_IP_VS_NQ=m
+CONFIG_IP_VS_TWOS=m
+CONFIG_IP_VS_FTP=m
+CONFIG_IP_VS_PE_SIP=m
+CONFIG_NFT_DUP_IPV4=m
+CONFIG_NFT_FIB_IPV4=m
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_LOG_ARP=m
+CONFIG_NF_LOG_IPV4=m
+CONFIG_IP_NF_IPTABLES=m
+CONFIG_IP_NF_MATCH_AH=m
+CONFIG_IP_NF_MATCH_ECN=m
+CONFIG_IP_NF_MATCH_RPFILTER=m
+CONFIG_IP_NF_MATCH_TTL=m
+CONFIG_IP_NF_TARGET_REJECT=m
+CONFIG_IP_NF_TARGET_SYNPROXY=m
+CONFIG_IP_NF_TARGET_ECN=m
+CONFIG_IP_NF_ARP_MANGLE=m
+CONFIG_NFT_DUP_IPV6=m
+CONFIG_NFT_FIB_IPV6=m
+CONFIG_IP6_NF_IPTABLES=m
+CONFIG_IP6_NF_MATCH_AH=m
+CONFIG_IP6_NF_MATCH_EUI64=m
+CONFIG_IP6_NF_MATCH_FRAG=m
+CONFIG_IP6_NF_MATCH_OPTS=m
+CONFIG_IP6_NF_MATCH_HL=m
+CONFIG_IP6_NF_MATCH_IPV6HEADER=m
+CONFIG_IP6_NF_MATCH_MH=m
+CONFIG_IP6_NF_MATCH_RPFILTER=m
+CONFIG_IP6_NF_MATCH_RT=m
+CONFIG_IP6_NF_MATCH_SRH=m
+CONFIG_IP6_NF_TARGET_REJECT=m
+CONFIG_IP6_NF_TARGET_SYNPROXY=m
+CONFIG_IP6_NF_TARGET_NPT=m
+CONFIG_NF_TABLES_BRIDGE=m
+CONFIG_NFT_BRIDGE_META=m
+CONFIG_NFT_BRIDGE_REJECT=m
+CONFIG_NF_CONNTRACK_BRIDGE=m
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_BRIDGE_EBT_802_3=m
+CONFIG_BRIDGE_EBT_AMONG=m
+CONFIG_BRIDGE_EBT_ARP=m
+CONFIG_BRIDGE_EBT_IP=m
+CONFIG_BRIDGE_EBT_IP6=m
+CONFIG_BRIDGE_EBT_LIMIT=m
+CONFIG_BRIDGE_EBT_MARK=m
+CONFIG_BRIDGE_EBT_PKTTYPE=m
+CONFIG_BRIDGE_EBT_STP=m
+CONFIG_BRIDGE_EBT_VLAN=m
+CONFIG_BRIDGE_EBT_ARPREPLY=m
+CONFIG_BRIDGE_EBT_DNAT=m
+CONFIG_BRIDGE_EBT_MARK_T=m
+CONFIG_BRIDGE_EBT_REDIRECT=m
+CONFIG_BRIDGE_EBT_SNAT=m
+CONFIG_BRIDGE_EBT_LOG=m
+CONFIG_BRIDGE_EBT_NFLOG=m
+CONFIG_RDS=m
+CONFIG_RDS_TCP=m
+CONFIG_TIPC=m
+CONFIG_ATM=m
+CONFIG_ATM_CLIP=m
+CONFIG_ATM_LANE=m
+CONFIG_ATM_MPOA=m
+CONFIG_ATM_BR2684=m
+CONFIG_L2TP=m
+CONFIG_L2TP_DEBUGFS=m
+CONFIG_L2TP_V3=y
+CONFIG_L2TP_IP=m
+CONFIG_L2TP_ETH=m
+CONFIG_BRIDGE=m
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_BRIDGE_MRP=y
+CONFIG_BRIDGE_CFM=y
+CONFIG_NET_DSA=m
+CONFIG_NET_DSA_TAG_NONE=m
+CONFIG_NET_DSA_TAG_AR9331=m
+CONFIG_NET_DSA_TAG_BRCM=m
+CONFIG_NET_DSA_TAG_BRCM_LEGACY=m
+CONFIG_NET_DSA_TAG_BRCM_PREPEND=m
+CONFIG_NET_DSA_TAG_HELLCREEK=m
+CONFIG_NET_DSA_TAG_GSWIP=m
+CONFIG_NET_DSA_TAG_DSA=m
+CONFIG_NET_DSA_TAG_EDSA=m
+CONFIG_NET_DSA_TAG_MTK=m
+CONFIG_NET_DSA_TAG_KSZ=m
+CONFIG_NET_DSA_TAG_OCELOT=m
+CONFIG_NET_DSA_TAG_OCELOT_8021Q=m
+CONFIG_NET_DSA_TAG_QCA=m
+CONFIG_NET_DSA_TAG_RTL4_A=m
+CONFIG_NET_DSA_TAG_RTL8_4=m
+CONFIG_NET_DSA_TAG_RZN1_A5PSW=m
+CONFIG_NET_DSA_TAG_LAN9303=m
+CONFIG_NET_DSA_TAG_SJA1105=m
+CONFIG_NET_DSA_TAG_TRAILER=m
+CONFIG_NET_DSA_TAG_VSC73XX_8021Q=m
+CONFIG_NET_DSA_TAG_XRS700X=m
+CONFIG_VLAN_8021Q=m
+CONFIG_VLAN_8021Q_GVRP=y
+CONFIG_VLAN_8021Q_MVRP=y
+CONFIG_LLC2=m
+CONFIG_ATALK=m
+CONFIG_X25=m
+CONFIG_LAPB=m
+CONFIG_PHONET=m
+CONFIG_6LOWPAN=m
+CONFIG_IEEE802154=m
+CONFIG_IEEE802154_6LOWPAN=m
+CONFIG_MAC802154=m
+CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_HTB=m
+CONFIG_NET_SCH_HFSC=m
+CONFIG_NET_SCH_PRIO=m
+CONFIG_NET_SCH_MULTIQ=m
+CONFIG_NET_SCH_RED=m
+CONFIG_NET_SCH_SFB=m
+CONFIG_NET_SCH_SFQ=m
+CONFIG_NET_SCH_TEQL=m
+CONFIG_NET_SCH_TBF=m
+CONFIG_NET_SCH_CBS=m
+CONFIG_NET_SCH_ETF=m
+CONFIG_NET_SCH_TAPRIO=m
+CONFIG_NET_SCH_GRED=m
+CONFIG_NET_SCH_NETEM=m
+CONFIG_NET_SCH_DRR=m
+CONFIG_NET_SCH_MQPRIO=m
+CONFIG_NET_SCH_SKBPRIO=m
+CONFIG_NET_SCH_CHOKE=m
+CONFIG_NET_SCH_QFQ=m
+CONFIG_NET_SCH_CODEL=m
+CONFIG_NET_SCH_FQ_CODEL=m
+CONFIG_NET_SCH_CAKE=m
+CONFIG_NET_SCH_FQ=m
+CONFIG_NET_SCH_HHF=m
+CONFIG_NET_SCH_PIE=m
+CONFIG_NET_SCH_FQ_PIE=m
+CONFIG_NET_SCH_PLUG=m
+CONFIG_NET_SCH_ETS=m
+CONFIG_NET_CLS_BASIC=m
+CONFIG_NET_CLS_ROUTE4=m
+CONFIG_NET_CLS_FW=m
+CONFIG_NET_CLS_U32=m
+CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_FLOW=m
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_BPF=m
+CONFIG_NET_CLS_FLOWER=m
+CONFIG_NET_CLS_MATCHALL=m
+CONFIG_NET_EMATCH=y
+CONFIG_NET_EMATCH_CMP=m
+CONFIG_NET_EMATCH_NBYTE=m
+CONFIG_NET_EMATCH_U32=m
+CONFIG_NET_EMATCH_META=m
+CONFIG_NET_EMATCH_TEXT=m
+CONFIG_NET_EMATCH_IPSET=m
+CONFIG_NET_EMATCH_IPT=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_ACT_POLICE=m
+CONFIG_NET_ACT_GACT=m
+CONFIG_GACT_PROB=y
+CONFIG_NET_ACT_MIRRED=m
+CONFIG_NET_ACT_SAMPLE=m
+CONFIG_NET_ACT_NAT=m
+CONFIG_NET_ACT_PEDIT=m
+CONFIG_NET_ACT_SIMP=m
+CONFIG_NET_ACT_SKBEDIT=m
+CONFIG_NET_ACT_CSUM=m
+CONFIG_NET_ACT_MPLS=m
+CONFIG_NET_ACT_VLAN=m
+CONFIG_NET_ACT_BPF=m
+CONFIG_NET_ACT_CONNMARK=m
+CONFIG_NET_ACT_CTINFO=m
+CONFIG_NET_ACT_SKBMOD=m
+CONFIG_NET_ACT_TUNNEL_KEY=m
+CONFIG_NET_ACT_CT=m
+CONFIG_NET_ACT_GATE=m
+CONFIG_NET_TC_SKB_EXT=y
+CONFIG_DCB=y
+CONFIG_DNS_RESOLVER=y
+CONFIG_BATMAN_ADV=m
+# CONFIG_BATMAN_ADV_BATMAN_V is not set
+CONFIG_OPENVSWITCH=m
+CONFIG_VSOCKETS=m
+CONFIG_VIRTIO_VSOCKETS=m
+CONFIG_NETLINK_DIAG=m
+CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=m
+CONFIG_HSR=m
+CONFIG_QRTR=m
+CONFIG_QRTR_TUN=m
+CONFIG_NET_NCSI=y
+CONFIG_NCSI_OEM_CMD_GET_MAC=y
+CONFIG_CGROUP_NET_PRIO=y
+CONFIG_BPF_STREAM_PARSER=y
+CONFIG_NET_PKTGEN=m
+CONFIG_HAMRADIO=y
+CONFIG_AX25=m
+CONFIG_NETROM=m
+CONFIG_ROSE=m
+CONFIG_MKISS=m
+CONFIG_6PACK=m
+CONFIG_BPQETHER=m
+CONFIG_BAYCOM_SER_FDX=m
+CONFIG_BAYCOM_SER_HDX=m
+CONFIG_YAM=m
+CONFIG_CAN=m
+CONFIG_CAN_J1939=m
+CONFIG_CAN_ISOTP=m
+CONFIG_BT=m
+CONFIG_BT_RFCOMM=m
+CONFIG_BT_RFCOMM_TTY=y
+CONFIG_BT_BNEP=y
+CONFIG_BT_BNEP_MC_FILTER=y
+CONFIG_BT_BNEP_PROTO_FILTER=y
+CONFIG_BT_HIDP=y
+CONFIG_BT_HCIBTUSB=m
+CONFIG_BT_HCIBTUSB_MTK=y
+CONFIG_BT_HCIBTSDIO=m
+CONFIG_BT_HCIUART=m
+CONFIG_BT_HCIUART_H4=y
+CONFIG_BT_HCIUART_RTL=y
+CONFIG_AF_RXRPC=y
+CONFIG_AF_RXRPC_IPV6=y
+CONFIG_RXKAD=y
+CONFIG_RXPERF=m
+CONFIG_AF_KCM=m
+CONFIG_MCTP=y
+CONFIG_CFG80211=m
+CONFIG_CFG80211_DEBUGFS=y
+CONFIG_MAC80211=m
+CONFIG_MAC80211_MESH=y
+CONFIG_MAC80211_DEBUGFS=y
+CONFIG_MAC80211_MESSAGE_TRACING=y
+CONFIG_RFKILL=y
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=y
+CONFIG_NET_9P=m
+CONFIG_NET_9P_VIRTIO=m
+CONFIG_CAIF=m
+CONFIG_CAIF_USB=m
+CONFIG_CEPH_LIB_USE_DNS_RESOLVER=y
+CONFIG_NFC=m
+CONFIG_NFC_DIGITAL=m
+CONFIG_NFC_NCI=m
+CONFIG_NFC_NCI_UART=m
+CONFIG_NFC_HCI=m
+CONFIG_NFC_SHDLC=y
+CONFIG_NFC_SIM=m
+CONFIG_NFC_PORT100=m
+CONFIG_NFC_VIRTUAL_NCI=m
+CONFIG_NFC_FDP=m
+CONFIG_NFC_FDP_I2C=m
+CONFIG_NFC_PN544_I2C=m
+CONFIG_NFC_PN533_USB=m
+CONFIG_NFC_PN533_I2C=m
+CONFIG_NFC_MICROREAD_I2C=m
+CONFIG_NFC_MRVL_USB=m
+CONFIG_NFC_MRVL_UART=m
+CONFIG_NFC_MRVL_I2C=m
+CONFIG_NFC_ST21NFCA_I2C=m
+CONFIG_NFC_ST_NCI_I2C=m
+CONFIG_NFC_NXP_NCI=m
+CONFIG_NFC_NXP_NCI_I2C=m
+CONFIG_NFC_S3FWRN5_I2C=m
+CONFIG_NET_IFE=m
+CONFIG_PAGE_POOL_STATS=y
+CONFIG_FAILOVER=y
+CONFIG_PCI=y
+CONFIG_PCIE_SPACEMIT_EP=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_PCI_ENDPOINT_CONFIGFS=y
+CONFIG_PCI_EPF_TEST=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+# CONFIG_STANDALONE is not set
+CONFIG_FW_LOADER_COMPRESS=y
+# CONFIG_FW_LOADER_COMPRESS_XZ is not set
+CONFIG_FW_LOADER_COMPRESS_ZSTD=y
+# CONFIG_ALLOW_DEV_COREDUMP is not set
+# CONFIG_EFI_DISABLE_RUNTIME is not set
+CONFIG_MTD=y
+CONFIG_MTD_CMDLINE_PARTS=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_PARTITIONED_MASTER=y
+CONFIG_MTD_SPI_NOR=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_VIRTIO_BLK=m
+CONFIG_BLK_DEV_NVME=y
+CONFIG_RT7451_RETIMER=y
+CONFIG_EEPROM_AT24=y
+CONFIG_BLK_DEV_SD=y
+CONFIG_ATA=y
+CONFIG_SATA_AHCI=y
+CONFIG_MD=y
+CONFIG_BLK_DEV_DM=y
+CONFIG_TARGET_CORE=y
+CONFIG_TCM_IBLOCK=y
+CONFIG_TCM_FILEIO=y
+CONFIG_NETDEVICES=y
+CONFIG_BONDING=m
+CONFIG_DUMMY=m
+CONFIG_WIREGUARD=m
+CONFIG_EQUALIZER=m
+CONFIG_IFB=m
+CONFIG_NET_TEAM=m
+CONFIG_MACVLAN=m
+CONFIG_MACVTAP=m
+CONFIG_IPVLAN=m
+CONFIG_IPVTAP=m
+CONFIG_VXLAN=m
+CONFIG_GENEVE=m
+CONFIG_BAREUDP=m
+CONFIG_GTP=m
+CONFIG_PFCP=m
+CONFIG_AMT=m
+CONFIG_MACSEC=m
+CONFIG_NETCONSOLE=m
+CONFIG_NETCONSOLE_DYNAMIC=y
+CONFIG_TUN=y
+CONFIG_VETH=m
+CONFIG_VIRTIO_NET=m
+CONFIG_NLMON=m
+CONFIG_NET_VRF=m
+CONFIG_R8169=m
+CONFIG_R8127=m
+CONFIG_STMMAC_ETH=y
+CONFIG_STMMAC_SELFTESTS=y
+CONFIG_DWMAC_SPACEMIT_ETHQOS=y
+CONFIG_ETHERCAT=y
+CONFIG_EC_MASTER=y
+CONFIG_EC_DEVICE=y
+CONFIG_EC_K3_GMAC=y
+CONFIG_LED_TRIGGER_PHY=y
+CONFIG_SFP=y
+CONFIG_REALTEK_PHY=y
+CONFIG_CAN_FLEXCAN=m
+CONFIG_USB_RTL8152=m
+CONFIG_USB_USBNET=y
+# CONFIG_USB_NET_AX8817X is not set
+CONFIG_USB_NET_CDC_EEM=m
+CONFIG_USB_NET_CDC_MBIM=m
+# CONFIG_USB_NET_NET1080 is not set
+CONFIG_USB_NET_RNDIS_HOST=m
+# CONFIG_USB_BELKIN is not set
+# CONFIG_USB_ARMLINUX is not set
+# CONFIG_USB_NET_ZAURUS is not set
+CONFIG_USB_NET_QMI_WWAN=m
+CONFIG_IWLWIFI=m
+CONFIG_IWLMVM=m
+CONFIG_MT7921E=m
+CONFIG_MT7925E=m
+CONFIG_RTW89=m
+CONFIG_RTW89_8852BE=m
+CONFIG_RTL8852BS=m
+CONFIG_INPUT_EVDEV=y
+CONFIG_KEYBOARD_CROS_EC=y
+# CONFIG_INPUT_MOUSE is not set
+CONFIG_INPUT_TOUCHSCREEN=y
+CONFIG_TOUCHSCREEN_RASPITS_FT5426=y
+CONFIG_INPUT_MISC=y
+CONFIG_INPUT_UINPUT=m
+CONFIG_INPUT_DRV_RPMI_PWRKEY=y
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_NR_UARTS=17
+CONFIG_SERIAL_8250_RUNTIME_UARTS=17
+CONFIG_SERIAL_OF_PLATFORM=y
+CONFIG_SERIAL_EARLYCON_RISCV_SBI=y
+CONFIG_SERIAL_DEV_BUS=y
+# CONFIG_DEVPORT is not set
+CONFIG_TCG_TPM=y
+CONFIG_I2C_CHARDEV=y
+CONFIG_I2C_CROS_EC_TUNNEL=y
+CONFIG_I2C_SLAVE=y
+CONFIG_SPI=y
+CONFIG_SPI_K1=y
+CONFIG_SPI_FSL_QUADSPI=y
+CONFIG_SPI_SPIDEV=y
+CONFIG_PINCTRL=y
+CONFIG_PINCTRL_SINGLE=y
+CONFIG_GPIOLIB=y
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_SPACEMIT_K1=y
+CONFIG_GPIO_CROS_EC=y
+CONFIG_POWER_SUPPLY=y
+CONFIG_BATTERY_SBS=y
+CONFIG_CHARGER_CROS_USBPD=m
+# CONFIG_CHARGER_CROS_PCHG is not set
+CONFIG_SENSORS_CTF2301=y
+CONFIG_SENSORS_PWM_FAN=y
+CONFIG_THERMAL=y
+CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
+CONFIG_K3_THERMAL=y
+CONFIG_WATCHDOG=y
+CONFIG_SPACEMIT_K1_WATCHDOG=y
+CONFIG_REGULATOR=y
+CONFIG_REGULATOR_FIXED_VOLTAGE=y
+# CONFIG_REGULATOR_SPACEMIT_P1 is not set
+# CONFIG_REGULATOR_SPACEMIT_MPQ8655 is not set
+CONFIG_RC_CORE=y
+CONFIG_RC_DECODERS=y
+CONFIG_IR_NEC_DECODER=y
+CONFIG_RC_DEVICES=y
+CONFIG_IR_SPACEMIT_K1=y
+# CONFIG_MEDIA_CEC_SUPPORT is not set
+CONFIG_MEDIA_SUPPORT=y
+CONFIG_MEDIA_USB_SUPPORT=y
+CONFIG_USB_VIDEO_CLASS=y
+CONFIG_VIDEO_LINLON_K3=y
+CONFIG_SPACEMIT_K3_CCIC=y
+CONFIG_SPACEMIT_K3_CCIC_IOMMU=y
+CONFIG_SPACEMIT_K3_CAM_IMX219=y
+CONFIG_SPACEMIT_K3_CAM_IMX415=y
+CONFIG_SPACEMIT_K3_CAM_OV5647=y
+CONFIG_SPACEMIT_K3_CAM_OV2735=y
+CONFIG_SPACEMIT_K3_CAM_OV5640_MAX96724_MAX9295=y
+CONFIG_DRM=y
+CONFIG_POWERVR_ROGUE=y
+CONFIG_POWERVR_DVFS=y
+CONFIG_POWERVR_THERMAL=y
+CONFIG_DRM_SPACEMIT=y
+CONFIG_SPACEMIT_INNO_DP=y
+CONFIG_SPACEMIT_MIPI_PANEL=y
+CONFIG_FB=y
+CONFIG_BACKLIGHT_PWM=y
+CONFIG_SOUND=y
+CONFIG_SND=y
+CONFIG_SND_USB_AUDIO=y
+CONFIG_SND_USB_AUDIO_MIDI_V2=y
+CONFIG_SND_SOC=y
+CONFIG_SND_SOC_K1_I2S=y
+CONFIG_SND_SOC_K3_RI2S=y
+CONFIG_SND_SOC_ES8326=m
+CONFIG_SND_SIMPLE_CARD=y
+CONFIG_HIDRAW=y
+CONFIG_UHID=y
+CONFIG_HID_MULTITOUCH=m
+CONFIG_I2C_HID_OF=m
+CONFIG_USB_HIDDEV=y
+CONFIG_USB=y
+CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
+CONFIG_USB_XHCI_HCD=y
+CONFIG_USB_ACM=m
+CONFIG_USB_STORAGE=y
+CONFIG_USB_UAS=y
+CONFIG_USB_DWC3=y
+# CONFIG_USB_DWC3_OF_SIMPLE is not set
+CONFIG_USB_SERIAL=m
+CONFIG_USB_SERIAL_GENERIC=y
+CONFIG_USB_SERIAL_SIMPLE=m
+CONFIG_USB_SERIAL_AIRCABLE=m
+CONFIG_USB_SERIAL_ARK3116=m
+CONFIG_USB_SERIAL_BELKIN=m
+CONFIG_USB_SERIAL_CH341=m
+CONFIG_USB_SERIAL_WHITEHEAT=m
+CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m
+CONFIG_USB_SERIAL_CP210X=m
+CONFIG_USB_SERIAL_CYPRESS_M8=m
+CONFIG_USB_SERIAL_EMPEG=m
+CONFIG_USB_SERIAL_FTDI_SIO=m
+CONFIG_USB_SERIAL_VISOR=m
+CONFIG_USB_SERIAL_IPAQ=m
+CONFIG_USB_SERIAL_IR=m
+CONFIG_USB_SERIAL_EDGEPORT=m
+CONFIG_USB_SERIAL_EDGEPORT_TI=m
+CONFIG_USB_SERIAL_F81232=m
+CONFIG_USB_SERIAL_F8153X=m
+CONFIG_USB_SERIAL_GARMIN=m
+CONFIG_USB_SERIAL_IPW=m
+CONFIG_USB_SERIAL_IUU=m
+CONFIG_USB_SERIAL_KEYSPAN_PDA=m
+CONFIG_USB_SERIAL_KEYSPAN=m
+CONFIG_USB_SERIAL_KLSI=m
+CONFIG_USB_SERIAL_KOBIL_SCT=m
+CONFIG_USB_SERIAL_MCT_U232=m
+CONFIG_USB_SERIAL_METRO=m
+CONFIG_USB_SERIAL_MOS7720=m
+CONFIG_USB_SERIAL_MOS7840=m
+CONFIG_USB_SERIAL_MXUPORT=m
+CONFIG_USB_SERIAL_NAVMAN=m
+CONFIG_USB_SERIAL_PL2303=m
+CONFIG_USB_SERIAL_OTI6858=m
+CONFIG_USB_SERIAL_QCAUX=m
+CONFIG_USB_SERIAL_QUALCOMM=m
+CONFIG_USB_SERIAL_SPCP8X5=m
+CONFIG_USB_SERIAL_SAFE=m
+CONFIG_USB_SERIAL_SAFE_PADDED=y
+CONFIG_USB_SERIAL_SIERRAWIRELESS=m
+CONFIG_USB_SERIAL_SYMBOL=m
+CONFIG_USB_SERIAL_TI=m
+CONFIG_USB_SERIAL_CYBERJACK=m
+CONFIG_USB_SERIAL_OPTION=m
+CONFIG_USB_SERIAL_OMNINET=m
+CONFIG_USB_SERIAL_OPTICON=m
+CONFIG_USB_SERIAL_XSENS_MT=m
+CONFIG_USB_SERIAL_WISHBONE=m
+CONFIG_USB_SERIAL_SSU100=m
+CONFIG_USB_SERIAL_QT2=m
+CONFIG_USB_SERIAL_UPD78F0730=m
+CONFIG_USB_SERIAL_XR=m
+CONFIG_USB_SERIAL_DEBUG=m
+CONFIG_USB_ONBOARD_DEV=y
+CONFIG_USB_GADGET=y
+CONFIG_USB_CONFIGFS=y
+CONFIG_USB_CONFIGFS_SERIAL=y
+CONFIG_USB_CONFIGFS_NCM=y
+CONFIG_USB_CONFIGFS_RNDIS=y
+CONFIG_USB_CONFIGFS_MASS_STORAGE=y
+CONFIG_USB_CONFIGFS_F_LB_SS=y
+CONFIG_USB_CONFIGFS_F_FS=y
+CONFIG_USB_CONFIGFS_F_HID=y
+CONFIG_USB_CONFIGFS_F_UVC=y
+CONFIG_USB_CONFIGFS_F_TCM=y
+CONFIG_TYPEC=y
+CONFIG_TYPEC_FUSB301=m
+CONFIG_MMC=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_OF_K1=y
+CONFIG_SCSI_UFSHCD=y
+CONFIG_SCSI_UFSHCD_PLATFORM=y
+CONFIG_SCSI_UFS_SPACEMIT_K3=y
+CONFIG_NEW_LEDS=y
+CONFIG_LEDS_CLASS=y
+CONFIG_LEDS_TRIGGER_NETDEV=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_CROS_EC=y
+CONFIG_RTC_DRV_SA1100=y
+CONFIG_DMADEVICES=y
+CONFIG_MMP_PDMA=y
+CONFIG_SPACEMIT_HDMA=y
+CONFIG_USERSPACE_DMA=y
+CONFIG_DW_EDMA=y
+CONFIG_DMATEST=y
+CONFIG_DMABUF_HEAPS=y
+CONFIG_DMABUF_HEAPS_SYSTEM=y
+CONFIG_DMABUF_HEAPS_CMA=y
+CONFIG_VFIO=y
+CONFIG_VFIO_PCI=y
+# CONFIG_VIRTIO_MENU is not set
+CONFIG_VHOST_VSOCK=m
+CONFIG_CHROME_PLATFORMS=y
+CONFIG_CROS_EC=y
+CONFIG_CROS_EC_ESPI=y
+CONFIG_CROS_EC_TYPEC=m
+CONFIG_SPACEMIT_CCU=y
+CONFIG_SPACEMIT_K3_CCU=y
+CONFIG_HWSPINLOCK=y
+CONFIG_HWSPINLOCK_SPACEMIT=y
+CONFIG_SPACEMIT_K1X_TIMER=y
+CONFIG_MAILBOX=y
+CONFIG_SPACEMIT_MAILBOX=y
+CONFIG_K3_MAILBOX=y
+CONFIG_RISCV_IOMMU_SPACEMIT_K3=y
+CONFIG_REMOTEPROC=y
+CONFIG_REMOTEPROC_CDEV=y
+CONFIG_SPACEMIT_K3_REMOTEPROC=y
+CONFIG_RPMSG_CHAR=y
+CONFIG_RPMSG_CTRL=y
+CONFIG_RPMSG_VIRTIO=y
+CONFIG_SPACEMIT_HMP=y
+CONFIG_SPACEMIT_PM_DOMAINS=y
+CONFIG_SPACEMIT_REBOOT_CONTROL=y
+CONFIG_SPACEMIT_V2D=y
+CONFIG_IIO=m
+CONFIG_CM32181=m
+CONFIG_PWM=y
+CONFIG_PWM_CROS_EC=y
+CONFIG_PWM_PXA=y
+CONFIG_PHY_SPACEMIT_K1_USB2=y
+CONFIG_PHY_SPACEMIT_K3_USB3=y
+CONFIG_PHY_SPACEMIT_K3_PCIE=y
+CONFIG_CORESIGHT=y
+CONFIG_CORESIGHT_LINKS_AND_SINKS=y
+CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y
+CONFIG_RVTRACE=y
+CONFIG_RVTRACE_ENCODER=y
+CONFIG_VALIDATE_FS_PARSER=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_JFS_FS=m
+CONFIG_JFS_POSIX_ACL=y
+CONFIG_JFS_SECURITY=y
+CONFIG_JFS_STATISTICS=y
+CONFIG_XFS_FS=m
+CONFIG_XFS_QUOTA=y
+CONFIG_XFS_POSIX_ACL=y
+CONFIG_GFS2_FS=m
+CONFIG_GFS2_FS_LOCKING_DLM=y
+CONFIG_OCFS2_FS=m
+CONFIG_BTRFS_FS=m
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_NILFS2_FS=m
+CONFIG_F2FS_FS=m
+CONFIG_F2FS_FS_SECURITY=y
+CONFIG_F2FS_FS_COMPRESSION=y
+# CONFIG_F2FS_IOSTAT is not set
+CONFIG_F2FS_UNFAIR_RWSEM=y
+CONFIG_ZONEFS_FS=m
+CONFIG_FS_ENCRYPTION=y
+CONFIG_FS_ENCRYPTION_INLINE_CRYPT=y
+CONFIG_FS_VERITY=y
+CONFIG_FS_VERITY_BUILTIN_SIGNATURES=y
+CONFIG_FANOTIFY=y
+CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y
+CONFIG_QUOTA_NETLINK_INTERFACE=y
+CONFIG_QFMT_V1=m
+CONFIG_QFMT_V2=m
+CONFIG_AUTOFS_FS=m
+CONFIG_FUSE_FS=y
+CONFIG_CUSE=m
+CONFIG_VIRTIO_FS=m
+CONFIG_OVERLAY_FS=m
+CONFIG_OVERLAY_FS_XINO_AUTO=y
+CONFIG_NETFS_DEBUG=y
+CONFIG_FSCACHE_STATS=y
+CONFIG_CACHEFILES=m
+CONFIG_CACHEFILES_ERROR_INJECTION=y
+CONFIG_ISO9660_FS=m
+CONFIG_JOLIET=y
+CONFIG_ZISOFS=y
+CONFIG_UDF_FS=m
+CONFIG_MSDOS_FS=m
+CONFIG_VFAT_FS=y
+CONFIG_EXFAT_FS=m
+CONFIG_NTFS3_LZX_XPRESS=y
+CONFIG_NTFS3_FS_POSIX_ACL=y
+CONFIG_NTFS_FS=m
+CONFIG_PROC_KCORE=y
+CONFIG_TMPFS=y
+CONFIG_TMPFS_POSIX_ACL=y
+CONFIG_TMPFS_INODE64=y
+CONFIG_TMPFS_QUOTA=y
+CONFIG_HUGETLBFS=y
+CONFIG_EFIVAR_FS=y
+CONFIG_ORANGEFS_FS=m
+CONFIG_ADFS_FS=m
+CONFIG_AFFS_FS=m
+CONFIG_ECRYPT_FS=y
+CONFIG_ECRYPT_FS_MESSAGING=y
+CONFIG_HFS_FS=m
+CONFIG_HFSPLUS_FS=m
+CONFIG_BEFS_FS=m
+CONFIG_BFS_FS=m
+CONFIG_EFS_FS=m
+CONFIG_CRAMFS=m
+CONFIG_SQUASHFS=y
+CONFIG_SQUASHFS_FILE_DIRECT=y
+CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT=y
+CONFIG_SQUASHFS_XATTR=y
+CONFIG_SQUASHFS_LZ4=y
+CONFIG_SQUASHFS_LZO=y
+CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
+CONFIG_VXFS_FS=m
+CONFIG_MINIX_FS=m
+CONFIG_OMFS_FS=m
+CONFIG_HPFS_FS=m
+CONFIG_QNX4FS_FS=m
+CONFIG_QNX6FS_FS=m
+CONFIG_ROMFS_FS=m
+CONFIG_PSTORE=y
+CONFIG_PSTORE_RAM=m
+CONFIG_PSTORE_BLK=m
+CONFIG_UFS_FS=m
+CONFIG_EROFS_FS=m
+CONFIG_EROFS_FS_ZIP_LZMA=y
+CONFIG_EROFS_FS_ZIP_DEFLATE=y
+CONFIG_EROFS_FS_ZIP_ZSTD=y
+CONFIG_EROFS_FS_PCPU_KTHREAD=y
+# CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI is not set
+CONFIG_NFS_FS=m
+CONFIG_NFS_V2=m
+CONFIG_NFS_V3_ACL=y
+CONFIG_NFS_V4=m
+CONFIG_NFS_SWAP=y
+CONFIG_NFS_V4_1=y
+CONFIG_NFS_V4_2=y
+CONFIG_NFS_V4_1_MIGRATION=y
+CONFIG_NFS_FSCACHE=y
+# CONFIG_NFS_V4_2_READ_PLUS is not set
+CONFIG_NFSD=m
+CONFIG_NFSD_V3_ACL=y
+CONFIG_NFSD_V4=y
+CONFIG_NFSD_BLOCKLAYOUT=y
+CONFIG_NFSD_SCSILAYOUT=y
+CONFIG_NFSD_FLEXFILELAYOUT=y
+CONFIG_NFSD_V4_2_INTER_SSC=y
+CONFIG_SUNRPC_DEBUG=y
+CONFIG_CEPH_FS=m
+CONFIG_CEPH_FSCACHE=y
+CONFIG_CEPH_FS_POSIX_ACL=y
+CONFIG_CIFS=m
+# CONFIG_CIFS_STATS2 is not set
+CONFIG_CIFS_UPCALL=y
+CONFIG_CIFS_XATTR=y
+CONFIG_CIFS_POSIX=y
+CONFIG_CIFS_DFS_UPCALL=y
+CONFIG_CIFS_SWN_UPCALL=y
+CONFIG_CIFS_FSCACHE=y
+CONFIG_SMB_SERVER=m
+CONFIG_CODA_FS=m
+CONFIG_AFS_FS=m
+CONFIG_AFS_FSCACHE=y
+CONFIG_9P_FS=m
+CONFIG_9P_FSCACHE=y
+CONFIG_9P_FS_POSIX_ACL=y
+CONFIG_9P_FS_SECURITY=y
+CONFIG_NLS_DEFAULT="utf8"
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_CODEPAGE_737=m
+CONFIG_NLS_CODEPAGE_775=m
+CONFIG_NLS_CODEPAGE_850=m
+CONFIG_NLS_CODEPAGE_852=m
+CONFIG_NLS_CODEPAGE_855=m
+CONFIG_NLS_CODEPAGE_857=m
+CONFIG_NLS_CODEPAGE_860=m
+CONFIG_NLS_CODEPAGE_861=m
+CONFIG_NLS_CODEPAGE_862=m
+CONFIG_NLS_CODEPAGE_863=m
+CONFIG_NLS_CODEPAGE_864=m
+CONFIG_NLS_CODEPAGE_865=m
+CONFIG_NLS_CODEPAGE_866=m
+CONFIG_NLS_CODEPAGE_869=m
+CONFIG_NLS_CODEPAGE_936=m
+CONFIG_NLS_CODEPAGE_950=m
+CONFIG_NLS_CODEPAGE_932=m
+CONFIG_NLS_CODEPAGE_949=m
+CONFIG_NLS_CODEPAGE_874=m
+CONFIG_NLS_ISO8859_8=m
+CONFIG_NLS_CODEPAGE_1250=m
+CONFIG_NLS_CODEPAGE_1251=m
+CONFIG_NLS_ASCII=m
+CONFIG_NLS_ISO8859_1=m
+CONFIG_NLS_ISO8859_2=m
+CONFIG_NLS_ISO8859_3=m
+CONFIG_NLS_ISO8859_4=m
+CONFIG_NLS_ISO8859_5=m
+CONFIG_NLS_ISO8859_6=m
+CONFIG_NLS_ISO8859_7=m
+CONFIG_NLS_ISO8859_9=m
+CONFIG_NLS_ISO8859_13=m
+CONFIG_NLS_ISO8859_14=m
+CONFIG_NLS_ISO8859_15=m
+CONFIG_NLS_KOI8_R=m
+CONFIG_NLS_KOI8_U=m
+CONFIG_NLS_MAC_ROMAN=m
+CONFIG_NLS_MAC_CELTIC=m
+CONFIG_NLS_MAC_CENTEURO=m
+CONFIG_NLS_MAC_CROATIAN=m
+CONFIG_NLS_MAC_CYRILLIC=m
+CONFIG_NLS_MAC_GAELIC=m
+CONFIG_NLS_MAC_GREEK=m
+CONFIG_NLS_MAC_ICELAND=m
+CONFIG_NLS_MAC_INUIT=m
+CONFIG_NLS_MAC_ROMANIAN=m
+CONFIG_NLS_MAC_TURKISH=m
+CONFIG_DLM=m
+CONFIG_UNICODE=y
+CONFIG_KEYS_REQUEST_CACHE=y
+CONFIG_PERSISTENT_KEYRINGS=y
+CONFIG_SECURITY_DMESG_RESTRICT=y
+CONFIG_SECURITY=y
+CONFIG_SECURITY_APPARMOR=y
+CONFIG_SECURITY_LANDLOCK=y
+# CONFIG_INTEGRITY is not set
+CONFIG_LSM="landlock,apparmor"
+CONFIG_INIT_STACK_ALL_PATTERN=y
+CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y
+CONFIG_ZERO_CALL_USED_REGS=y
+CONFIG_FORTIFY_SOURCE=y
+CONFIG_HARDENED_USERCOPY=y
+CONFIG_LIST_HARDENED=y
+CONFIG_RANDSTRUCT_FULL=y
+CONFIG_CRYPTO_USER=m
+CONFIG_CRYPTO_PCRYPT=m
+CONFIG_CRYPTO_CRYPTD=m
+CONFIG_CRYPTO_DH=y
+CONFIG_CRYPTO_DH_RFC7919_GROUPS=y
+CONFIG_CRYPTO_ECDSA=m
+CONFIG_CRYPTO_ECRDSA=m
+CONFIG_CRYPTO_AES_TI=m
+CONFIG_CRYPTO_ARIA=m
+CONFIG_CRYPTO_BLOWFISH=m
+CONFIG_CRYPTO_CAST5=m
+CONFIG_CRYPTO_CAST6=m
+CONFIG_CRYPTO_DES=m
+CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_TWOFISH=m
+CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
+CONFIG_CRYPTO_LRW=m
+CONFIG_CRYPTO_AEGIS128=m
+CONFIG_CRYPTO_GCM=y
+CONFIG_CRYPTO_SEQIV=y
+CONFIG_CRYPTO_ESSIV=m
+CONFIG_CRYPTO_MD4=m
+CONFIG_CRYPTO_MICHAEL_MIC=m
+CONFIG_CRYPTO_RMD160=m
+CONFIG_CRYPTO_WP512=m
+CONFIG_CRYPTO_XCBC=m
+CONFIG_CRYPTO_DEFLATE=y
+CONFIG_CRYPTO_LZO=y
+CONFIG_CRYPTO_842=m
+CONFIG_CRYPTO_LZ4=m
+CONFIG_CRYPTO_LZ4HC=m
+CONFIG_CRYPTO_ZSTD=m
+CONFIG_CRYPTO_ANSI_CPRNG=m
+CONFIG_CRYPTO_USER_API_HASH=m
+CONFIG_CRYPTO_USER_API_SKCIPHER=m
+CONFIG_CRYPTO_USER_API_RNG=m
+CONFIG_CRYPTO_USER_API_AEAD=m
+# CONFIG_CRYPTO_USER_API_ENABLE_OBSOLETE is not set
+CONFIG_CRYPTO_DEV_ATMEL_ECC=m
+CONFIG_CRYPTO_DEV_ATMEL_SHA204A=m
+CONFIG_CRYPTO_DEV_VIRTIO=m
+CONFIG_CRYPTO_DEV_SAFEXCEL=m
+CONFIG_CRYPTO_DEV_CCREE=m
+CONFIG_CRYPTO_DEV_AMLOGIC_GXL=m
+CONFIG_PKCS8_PRIVATE_KEY_PARSER=m
+CONFIG_PKCS7_TEST_KEY=m
+CONFIG_SIGNED_PE_FILE_VERIFICATION=y
+CONFIG_CORDIC=m
+CONFIG_XZ_DEC_TEST=m
+CONFIG_DMA_RESTRICTED_POOL=y
+CONFIG_DMA_CMA=y
+CONFIG_PRINTK_TIME=y
+CONFIG_BOOT_PRINTK_DELAY=y
+CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_GDB_SCRIPTS=y
+CONFIG_VMLINUX_MAP=y
+CONFIG_KGDB=y
+# CONFIG_KGDB_HONOUR_BLOCKLIST is not set
+CONFIG_KGDB_KDB=y
+CONFIG_KDB_KEYBOARD=y
+CONFIG_PAGE_POISONING=y
+CONFIG_DEBUG_WX=y
+CONFIG_DEBUG_MEMORY_INIT=y
+CONFIG_DETECT_HUNG_TASK=y
+CONFIG_DEBUG_ATOMIC_SLEEP=y
+CONFIG_RCU_CPU_STALL_CPUTIME=y
+# CONFIG_RCU_TRACE is not set
+CONFIG_LATENCYTOP=y
+CONFIG_SCHED_TRACER=y
+CONFIG_FTRACE_SYSCALLS=y
+CONFIG_BLK_DEV_IO_TRACE=y
+# CONFIG_UPROBE_EVENTS is not set
+CONFIG_SAMPLES=y
+CONFIG_SAMPLE_TRACE_PRINTK=m
+CONFIG_SAMPLE_TRACE_ARRAY=m
+# CONFIG_RUNTIME_KERNEL_TESTING_MENU is not set
+CONFIG_NOTIFIER_ERROR_INJECTION=m
+CONFIG_TEST_BPF=m
+CONFIG_MEMTEST=y
diff --git a/arch/riscv/generic/Image.its.S b/arch/riscv/generic/Image.its.S
new file mode 100755
index 000000000000..49448dc008f4
--- /dev/null
+++ b/arch/riscv/generic/Image.its.S
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/dts-v1/;
+/ {
+	description = KERNEL_NAME;
+	#address-cells = <ADDR_CELLS>;
+	images {
+		kernel {
+			description = KERNEL_NAME;
+			data = /incbin/(IMAGE_BINARY);
+			type = "kernel";
+			arch = "riscv";
+			os = "linux";
+			compression = IMAGE_COMPRESSION;
+			load = /bits/ ADDR_BITS <IMAGE_LOAD_ADDRESS>;
+			entry = /bits/ ADDR_BITS <IMAGE_ENTRY_ADDRESS>;
+			hash {
+				algo = "sha256";
+			};
+		};
+	};
+	configurations {
+		default = "conf-default";
+		conf-default {
+			description = "Generic Linux kernel";
+			kernel = "kernel";
+		};
+	};
+};
diff --git a/arch/riscv/generic/Image_sign.its.S b/arch/riscv/generic/Image_sign.its.S
new file mode 100644
index 000000000000..101cf9fcff1c
--- /dev/null
+++ b/arch/riscv/generic/Image_sign.its.S
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/dts-v1/;
+/ {
+	description = KERNEL_NAME;
+	#address-cells = <ADDR_CELLS>;
+	images {
+		kernel {
+			description = KERNEL_NAME;
+			data = /incbin/(IMAGE_BINARY);
+			type = "kernel";
+			arch = "riscv";
+			os = "linux";
+			compression = IMAGE_COMPRESSION;
+			load = /bits/ ADDR_BITS <IMAGE_LOAD_ADDRESS>;
+			entry = /bits/ ADDR_BITS <IMAGE_ENTRY_ADDRESS>;
+			hash {
+				algo = "sha256";
+			};
+		};
+	};
+	configurations {
+		default = "conf-default";
+		conf-default {
+			description = "Generic Linux kernel";
+			kernel = "kernel";
+			signature {
+				algo = "sha256,rsa2048";
+				key-name-hint = "kernel_key_prv";
+				sign-images = "kernel";
+			};
+		};
+	};
+};
+
diff --git a/arch/riscv/generic/Platform b/arch/riscv/generic/Platform
new file mode 100644
index 000000000000..520f42e16b97
--- /dev/null
+++ b/arch/riscv/generic/Platform
@@ -0,0 +1,19 @@
+#
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (C) 2024 Spacemit
+#
+# This software is licensed under the terms of the GNU General Public
+# License version 2, as published by the Free Software Foundation, and
+# may be copied, distributed, and modified under those terms.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ifeq ($(CONFIG_FIT_SIGNATURE),)
+its-y	:= Image.its.S
+else
+its-y	:= Image_sign.its.S
+endif
+
diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index 122e1485d39a..390c17df2125 100644
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -389,7 +389,11 @@ static __always_inline void __cmpwait(volatile void *ptr,
 		"	and	%0, %0, %3\n"
 		"	xor	%0, %0, %2\n"
 		"	bnez	%0, 1f\n"
-			ZAWRS_WRS_NTO "\n"
+#ifdef CONFIG_SOC_SPACEMIT_K3
+		ZAWRS_WRS_STO "\n"
+#else
+		ZAWRS_WRS_NTO "\n"
+#endif
 		"1:"
 		: "=&r" (tmp), "+A" (*(__ptr32b))
 		: "r" (__val), "r" (__mask)
@@ -406,7 +410,11 @@ static __always_inline void __cmpwait(volatile void *ptr,
 		"	and	%0, %0, %3\n"
 		"	xor	%0, %0, %2\n"
 		"	bnez	%0, 1f\n"
-			ZAWRS_WRS_NTO "\n"
+#ifdef CONFIG_SOC_SPACEMIT_K3
+		ZAWRS_WRS_STO "\n"
+#else
+		ZAWRS_WRS_NTO "\n"
+#endif
 		"1:"
 		: "=&r" (tmp), "+A" (*(__ptr32b))
 		: "r" (__val), "r" (__mask)
@@ -417,7 +425,11 @@ static __always_inline void __cmpwait(volatile void *ptr,
 		"	lr.w	%0, %1\n"
 		"	xor	%0, %0, %2\n"
 		"	bnez	%0, 1f\n"
-			ZAWRS_WRS_NTO "\n"
+#ifdef CONFIG_SOC_SPACEMIT_K3
+		ZAWRS_WRS_STO "\n"
+#else
+		ZAWRS_WRS_NTO "\n"
+#endif
 		"1:"
 		: "=&r" (tmp), "+A" (*(u32 *)ptr)
 		: "r" (val));
@@ -428,7 +440,11 @@ static __always_inline void __cmpwait(volatile void *ptr,
 		"	lr.d	%0, %1\n"
 		"	xor	%0, %0, %2\n"
 		"	bnez	%0, 1f\n"
-			ZAWRS_WRS_NTO "\n"
+#ifdef CONFIG_SOC_SPACEMIT_K3
+		ZAWRS_WRS_STO "\n"
+#else
+		ZAWRS_WRS_NTO "\n"
+#endif
 		"1:"
 		: "=&r" (tmp), "+A" (*(u64 *)ptr)
 		: "r" (val));
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 4a37a98398ad..04285e5b9295 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -74,6 +74,9 @@
 #define SATP_ASID_SHIFT	44
 #define SATP_ASID_MASK	_AC(0xFFFF, UL)
 #endif
+#define SATP_MODE_MASK	(~(SATP_PPN | \
+			  ((unsigned long)SATP_ASID_MASK << \
+			   SATP_ASID_SHIFT)))
 
 /* Exception cause high bit - is an interrupt if set */
 #define CAUSE_IRQ_FLAG		(_AC(1, UL) << (__riscv_xlen - 1))
@@ -220,6 +223,8 @@
 #define ENVCFG_FIOM			_AC(0x1, UL)
 
 /* Smstateen bits */
+#define SMSTATEEN0_HSCONTEXT_SHIFT	57
+#define SMSTATEEN0_HSCONTEXT		(_ULL(1) << SMSTATEEN0_HSCONTEXT_SHIFT)
 #define SMSTATEEN0_AIA_IMSIC_SHIFT	58
 #define SMSTATEEN0_AIA_IMSIC		(_ULL(1) << SMSTATEEN0_AIA_IMSIC_SHIFT)
 #define SMSTATEEN0_AIA_SHIFT		59
@@ -444,6 +449,10 @@
 #define CSR_VTYPE		0xc21
 #define CSR_VLENB		0xc22
 
+/* riscv-debug-spec: Sdtrig extension */
+#define CSR_SCONTEXT		0x5a8
+#define CSR_HCONTEXT		0x6a8
+
 /* Scalar Crypto Extension - Entropy */
 #define CSR_SEED		0x015
 #define SEED_OPST_MASK		_AC(0xC0000000, UL)
diff --git a/arch/riscv/include/asm/hw_breakpoint.h b/arch/riscv/include/asm/hw_breakpoint.h
new file mode 100644
index 000000000000..a5ef70fa14a1
--- /dev/null
+++ b/arch/riscv/include/asm/hw_breakpoint.h
@@ -0,0 +1,332 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2026 Qualcomm Technologies, Inc.
+ */
+
+#ifndef __RISCV_HW_BREAKPOINT_H
+#define __RISCV_HW_BREAKPOINT_H
+
+struct task_struct;
+
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+
+#include <uapi/linux/hw_breakpoint.h>
+
+#if __riscv_xlen == 64
+#define cpu_to_le cpu_to_le64
+#define le_to_cpu le64_to_cpu
+#elif __riscv_xlen == 32
+#define cpu_to_le cpu_to_le32
+#define le_to_cpu le32_to_cpu
+#else
+#error "Unexpected __riscv_xlen"
+#endif
+
+#define RV_DBTR_BIT(_prefix, _name)		\
+	RV_DBTR_##_prefix##_##_name##_BIT
+
+#define RV_DBTR_BIT_MASK(_prefix, _name)		\
+	RV_DBTR_##_prefix##_name##_BIT_MASK
+
+#define RV_DBTR_BIT_MASK_VAL(_prefix, _name, _width)	\
+	(((1UL << (_width)) - 1) << RV_DBTR_BIT(_prefix, _name))
+
+#define CLEAR_DBTR_BIT(_target, _prefix, _bit_name)	\
+	__clear_bit(RV_DBTR_BIT(_prefix, _bit_name), &(_target))
+
+#define SET_DBTR_BIT(_target, _prefix, _bit_name)	\
+	__set_bit(RV_DBTR_BIT(_prefix, _bit_name), &(_target))
+
+enum {
+	RV_DBTR_BP	= 0,
+	RV_DBTR_WP	= 1,
+};
+
+enum {
+	RV_DBTR_TRIG_NONE = 0,
+	RV_DBTR_TRIG_LEGACY,
+	RV_DBTR_TRIG_MCONTROL,
+	RV_DBTR_TRIG_ICOUNT,
+	RV_DBTR_TRIG_ITRIGGER,
+	RV_DBTR_TRIG_ETRIGGER,
+	RV_DBTR_TRIG_MCONTROL6,
+};
+
+/* Trigger Data 1 */
+enum {
+	RV_DBTR_BIT(TDATA1, DATA)   = 0,
+#if __riscv_xlen == 64
+	RV_DBTR_BIT(TDATA1, DMODE)  = 59,
+	RV_DBTR_BIT(TDATA1, TYPE)   = 60,
+#elif __riscv_xlen == 32
+	RV_DBTR_BIT(TDATA1, DMODE)  = 27,
+	RV_DBTR_BIT(TDATA1, TYPE)   = 28,
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+};
+
+enum {
+#if __riscv_xlen == 64
+	RV_DBTR_BIT_MASK(TDATA1, DATA) = RV_DBTR_BIT_MASK_VAL(TDATA1, DATA, 59),
+#elif __riscv_xlen == 32
+	RV_DBTR_BIT_MASK(TDATA1, DATA) = RV_DBTR_BIT_MASK_VAL(TDATA1, DATA,  27),
+#else
+	#error "Unknown __riscv_xlen"
+#endif
+	RV_DBTR_BIT_MASK(TDAT1, DMODE) = RV_DBTR_BIT_MASK_VAL(TDATA1, DMODE, 1),
+	RV_DBTR_BIT_MASK(TDATA1, TYPE) = RV_DBTR_BIT_MASK_VAL(TDATA1, TYPE,  4),
+};
+
+/* MC - Match Control Type Register */
+enum {
+	RV_DBTR_BIT(MC, LOAD)     = 0,
+	RV_DBTR_BIT(MC, STORE)    = 1,
+	RV_DBTR_BIT(MC, EXEC)     = 2,
+	RV_DBTR_BIT(MC, U)        = 3,
+	RV_DBTR_BIT(MC, S)        = 4,
+	RV_DBTR_BIT(MC, RES2)     = 5,
+	RV_DBTR_BIT(MC, M)        = 6,
+	RV_DBTR_BIT(MC, MATCH)    = 7,
+	RV_DBTR_BIT(MC, CHAIN)    = 11,
+	RV_DBTR_BIT(MC, ACTION)   = 12,
+	RV_DBTR_BIT(MC, SIZELO)   = 16,
+	RV_DBTR_BIT(MC, TIMING)   = 18,
+	RV_DBTR_BIT(MC, SELECT)   = 19,
+	RV_DBTR_BIT(MC, HIT)      = 20,
+#if __riscv_xlen >= 64
+	RV_DBTR_BIT(MC, SIZEHI) = 21,
+#endif
+#if __riscv_xlen == 64
+	RV_DBTR_BIT(MC, MASKMAX) = 53,
+	RV_DBTR_BIT(MC, DMODE)   = 59,
+	RV_DBTR_BIT(MC, TYPE)    = 60,
+#elif __riscv_xlen == 32
+	RV_DBTR_BIT(MC, MASKMAX) = 21,
+	RV_DBTR_BIT(MC, DMODE)   = 27,
+	RV_DBTR_BIT(MC, TYPE)    = 28,
+#else
+	#error "Unknown riscv xlen"
+#endif
+};
+
+enum {
+	RV_DBTR_BIT_MASK(MC, LOAD) = RV_DBTR_BIT_MASK_VAL(MC, LOAD, 1),
+	RV_DBTR_BIT_MASK(MC, STORE) = RV_DBTR_BIT_MASK_VAL(MC, STORE, 1),
+	RV_DBTR_BIT_MASK(MC, EXEC) = RV_DBTR_BIT_MASK_VAL(MC, EXEC, 1),
+	RV_DBTR_BIT_MASK(MC, U) = RV_DBTR_BIT_MASK_VAL(MC, U, 1),
+	RV_DBTR_BIT_MASK(MC, S) = RV_DBTR_BIT_MASK_VAL(MC, S, 1),
+	RV_DBTR_BIT_MASK(MC, RES2) = RV_DBTR_BIT_MASK_VAL(MC, RES2, 1),
+	RV_DBTR_BIT_MASK(MC, M) = RV_DBTR_BIT_MASK_VAL(MC, M, 1),
+	RV_DBTR_BIT_MASK(MC, MATCH) = RV_DBTR_BIT_MASK_VAL(MC, MATCH, 4),
+	RV_DBTR_BIT_MASK(MC, CHAIN) = RV_DBTR_BIT_MASK_VAL(MC, CHAIN, 1),
+	RV_DBTR_BIT_MASK(MC, ACTION) = RV_DBTR_BIT_MASK_VAL(MC, ACTION, 4),
+	RV_DBTR_BIT_MASK(MC, SIZELO) = RV_DBTR_BIT_MASK_VAL(MC, SIZELO, 2),
+	RV_DBTR_BIT_MASK(MC, TIMING) = RV_DBTR_BIT_MASK_VAL(MC, TIMING, 1),
+	RV_DBTR_BIT_MASK(MC, SELECT) = RV_DBTR_BIT_MASK_VAL(MC, SELECT, 1),
+	RV_DBTR_BIT_MASK(MC, HIT) = RV_DBTR_BIT_MASK_VAL(MC, HIT, 1),
+#if __riscv_xlen >= 64
+	RV_DBTR_BIT_MASK(MC, SIZEHI) = RV_DBTR_BIT_MASK_VAL(MC, SIZEHI,  2),
+#endif
+	RV_DBTR_BIT_MASK(MC, MASKMAX) = RV_DBTR_BIT_MASK_VAL(MC, MASKMAX, 6),
+	RV_DBTR_BIT_MASK(MC, DMODE) = RV_DBTR_BIT_MASK_VAL(MC, DMODE, 1),
+	RV_DBTR_BIT_MASK(MC, TYPE) = RV_DBTR_BIT_MASK_VAL(MC, TYPE, 4),
+};
+
+/* MC6 - Match Control 6 Type Register */
+enum {
+	RV_DBTR_BIT(MC6, LOAD)    = 0,
+	RV_DBTR_BIT(MC6, STORE)   = 1,
+	RV_DBTR_BIT(MC6, EXEC)    = 2,
+	RV_DBTR_BIT(MC6, U)       = 3,
+	RV_DBTR_BIT(MC6, S)       = 4,
+	RV_DBTR_BIT(MC6, RES2)    = 5,
+	RV_DBTR_BIT(MC6, M)       = 6,
+	RV_DBTR_BIT(MC6, MATCH)   = 7,
+	RV_DBTR_BIT(MC6, CHAIN)   = 11,
+	RV_DBTR_BIT(MC6, ACTION)  = 12,
+	RV_DBTR_BIT(MC6, SIZE)    = 16,
+	RV_DBTR_BIT(MC6, TIMING)  = 20,
+	RV_DBTR_BIT(MC6, SELECT)  = 21,
+	RV_DBTR_BIT(MC6, HIT)     = 22,
+	RV_DBTR_BIT(MC6, VU)      = 23,
+	RV_DBTR_BIT(MC6, VS)      = 24,
+#if __riscv_xlen == 64
+	RV_DBTR_BIT(MC6, DMODE)   = 59,
+	RV_DBTR_BIT(MC6, TYPE)    = 60,
+#elif __riscv_xlen == 32
+	RV_DBTR_BIT(MC6, DMODE)   = 27,
+	RV_DBTR_BIT(MC6, TYPE)    = 28,
+#else
+	#error "Unknown riscv xlen"
+#endif
+};
+
+enum {
+	RV_DBTR_BIT_MASK(MC6, LOAD) = RV_DBTR_BIT_MASK_VAL(MC6, LOAD, 1),
+	RV_DBTR_BIT_MASK(MC6, STORE) = RV_DBTR_BIT_MASK_VAL(MC6, STORE,  1),
+	RV_DBTR_BIT_MASK(MC6, EXEC) = RV_DBTR_BIT_MASK_VAL(MC6, EXEC, 1),
+	RV_DBTR_BIT_MASK(MC6, U) = RV_DBTR_BIT_MASK_VAL(MC6, U, 1),
+	RV_DBTR_BIT_MASK(MC6, S) = RV_DBTR_BIT_MASK_VAL(MC6, S, 1),
+	RV_DBTR_BIT_MASK(MC6, RES2) = RV_DBTR_BIT_MASK_VAL(MC6, RES2, 1),
+	RV_DBTR_BIT_MASK(MC6, M) = RV_DBTR_BIT_MASK_VAL(MC6, M, 1),
+	RV_DBTR_BIT_MASK(MC6, MATCH) = RV_DBTR_BIT_MASK_VAL(MC6, MATCH, 4),
+	RV_DBTR_BIT_MASK(MC6, CHAIN) = RV_DBTR_BIT_MASK_VAL(MC6, CHAIN, 1),
+	RV_DBTR_BIT_MASK(MC6, ACTION) = RV_DBTR_BIT_MASK_VAL(MC6, ACTION, 4),
+	RV_DBTR_BIT_MASK(MC6, SIZE) = RV_DBTR_BIT_MASK_VAL(MC6, SIZE, 4),
+	RV_DBTR_BIT_MASK(MC6, TIMING) = RV_DBTR_BIT_MASK_VAL(MC6, TIMING, 1),
+	RV_DBTR_BIT_MASK(MC6, SELECT) = RV_DBTR_BIT_MASK_VAL(MC6, SELECT, 1),
+	RV_DBTR_BIT_MASK(MC6, HIT) = RV_DBTR_BIT_MASK_VAL(MC6, HIT, 1),
+	RV_DBTR_BIT_MASK(MC6, VU) = RV_DBTR_BIT_MASK_VAL(MC6, VU, 1),
+	RV_DBTR_BIT_MASK(MC6, VS) = RV_DBTR_BIT_MASK_VAL(MC6, VS, 1),
+#if __riscv_xlen == 64
+	RV_DBTR_BIT_MASK(MC6, DMODE) = RV_DBTR_BIT_MASK_VAL(MC6, DMODE, 1),
+	RV_DBTR_BIT_MASK(MC6, TYPE) = RV_DBTR_BIT_MASK_VAL(MC6, TYPE, 4),
+#elif __riscv_xlen == 32
+	RV_DBTR_BIT_MASK(MC6, DMODE) = RV_DBTR_BIT_MASK_VAL(MC6, DMODE, 1),
+	RV_DBTR_BIT_MASK(MC6, TYPE) = RV_DBTR_BIT_MASK_VAL(MC6, TYPE, 4),
+#else
+	#error "Unknown riscv xlen"
+#endif
+};
+
+#define RV_DBTR_SET_TDATA1_TYPE(_t1, _type)				\
+	({								\
+		typeof(_t1) (td1t1) = (_t1);				\
+		(td1t1) &= ~RV_DBTR_BIT_MASK(TDATA1, TYPE);		\
+		(td1t1) |= (((unsigned long)(_type)			\
+			     << RV_DBTR_BIT(TDATA1, TYPE))		\
+			    & RV_DBTR_BIT_MASK(TDATA1, TYPE));		\
+		(td1t1);						\
+	})
+
+#define RV_DBTR_SET_MC_TYPE(_t1, _type)				\
+	({							\
+		typeof(_t1) (mct1) = (_t1);			\
+		(mct1) &= ~RV_DBTR_BIT_MASK(MC, TYPE);		\
+		(mct1) |= (((unsigned long)(_type)		\
+			    << RV_DBTR_BIT(MC, TYPE))		\
+			   & RV_DBTR_BIT_MASK(MC, TYPE));	\
+		(mct1);						\
+	})
+
+#define RV_DBTR_SET_MC6_TYPE(_t1, _type)				\
+	({								\
+		typeof(_t1) (mc6t1) = (_t1);				\
+		(mc6t1) &= ~RV_DBTR_BIT_MASK(MC6, TYPE);		\
+		(mc6t1) |= (((unsigned long)(_type)			\
+			     << RV_DBTR_BIT(MC6, TYPE))			\
+			    & RV_DBTR_BIT_MASK(MC6, TYPE));		\
+		(mc6t1);						\
+	})
+
+#define RV_DBTR_SET_MC_EXEC_BIT(_t1)		\
+	SET_DBTR_BIT(_t1, MC, EXEC)
+
+#define RV_DBTR_SET_MC_LOAD_BIT(_t1)		\
+	SET_DBTR_BIT(_t1, MC, LOAD)
+
+#define RV_DBTR_SET_MC_STORE_BIT(_t1)		\
+	SET_DBTR_BIT(_t1, MC, STORE)
+
+#define RV_DBTR_SET_MC_SIZELO(_t1, _val)			\
+	({							\
+		typeof(_t1) (mcslt1) = (_t1);			\
+		mcslt1 &= ~RV_DBTR_BIT_MASK(MC, SIZELO);	\
+		mcslt1 |= (((_val) << RV_DBTR_BIT(MC, SIZELO))	\
+			   & RV_DBTR_BIT_MASK(MC, SIZELO));	\
+		(mcslt1);					\
+	})
+
+#define RV_DBTR_SET_MC_SIZEHI(_t1, _val)			\
+	({							\
+		typeof(_t1) (mcsht1) = (_t1);			\
+		mcsht1 &= ~RV_DBTR_BIT_MASK(MC, SIZEHI);	\
+		mcsht1 |= (((_val) << RV_DBTR_BIT(MC, SIZEHI))	\
+			    & RV_DBTR_BIT_MASK(MC, SIZEHI));	\
+		(mcsht1);					\
+	})
+
+#define RV_DBTR_SET_MC6_EXEC_BIT(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, EXEC)
+
+#define RV_DBTR_SET_MC6_LOAD_BIT(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, LOAD)
+
+#define RV_DBTR_SET_MC6_STORE_BIT(_t1)		\
+	SET_DBTR_BIT(_t1, MC6, STORE)
+
+#define RV_DBTR_SET_MC6_SIZE(_t1, _val)					\
+	({								\
+		typeof(_t1) (mc6szt1) = (_t1);				\
+		(mc6szt1) &= ~RV_DBTR_BIT_MASK(MC6, SIZE);		\
+		(mc6szt1) |= (((_val) << RV_DBTR_BIT(MC6, SIZE))	\
+			      & RV_DBTR_BIT_MASK(MC6, SIZE));		\
+		(mc6szt1);						\
+	})
+
+struct arch_hw_breakpoint {
+	unsigned long address;
+	unsigned long len;
+	unsigned int type;
+
+	/* Trigger configuration data */
+	unsigned long tdata1;
+	unsigned long tdata2;
+	unsigned long tdata3;
+};
+
+/* Maximum number of hardware breakpoints supported */
+#define HW_BP_NUM_MAX 32
+
+struct perf_event_attr;
+struct notifier_block;
+struct perf_event;
+struct pt_regs;
+
+int hw_breakpoint_slots(int type);
+int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw);
+int hw_breakpoint_arch_parse(struct perf_event *bp,
+			     const struct perf_event_attr *attr,
+			     struct arch_hw_breakpoint *hw);
+int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
+				    unsigned long val, void *data);
+
+void arch_enable_hw_breakpoint(struct perf_event *bp);
+void arch_update_hw_breakpoint(struct perf_event *bp);
+void arch_disable_hw_breakpoint(struct perf_event *bp);
+int arch_install_hw_breakpoint(struct perf_event *bp);
+void arch_uninstall_hw_breakpoint(struct perf_event *bp);
+void hw_breakpoint_pmu_read(struct perf_event *bp);
+void clear_ptrace_hw_breakpoint(struct task_struct *tsk);
+void flush_ptrace_hw_breakpoint(struct task_struct *tsk);
+
+#else
+
+int hw_breakpoint_slots(int type)
+{
+	return 0;
+}
+
+static inline void clear_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+}
+
+static inline void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+}
+
+void arch_enable_hw_breakpoint(struct perf_event *bp)
+{
+}
+
+void arch_update_hw_breakpoint(struct perf_event *bp)
+{
+}
+
+void arch_disable_hw_breakpoint(struct perf_event *bp)
+{
+}
+
+#endif /* CONFIG_HAVE_HW_BREAKPOINT */
+#endif /* __RISCV_HW_BREAKPOINT_H */
diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h
index affd63e11b0a..ba4b62905670 100644
--- a/arch/riscv/include/asm/hwcap.h
+++ b/arch/riscv/include/asm/hwcap.h
@@ -106,6 +106,7 @@
 #define RISCV_ISA_EXT_ZAAMO		97
 #define RISCV_ISA_EXT_ZALRSC		98
 #define RISCV_ISA_EXT_ZICBOP		99
+#define RISCV_ISA_EXT_SDTRIG		100
 
 #define RISCV_ISA_EXT_XLINUXENVCFG	127
 
diff --git a/arch/riscv/include/asm/insn.h b/arch/riscv/include/asm/insn.h
index c3005573e8c9..3c29379b6270 100644
--- a/arch/riscv/include/asm/insn.h
+++ b/arch/riscv/include/asm/insn.h
@@ -228,6 +228,15 @@
 #define RVG_MASK_EBREAK		0xffffffff
 #define RVG_MASK_SRET		0xffffffff
 
+#define RV_INSN_MATCH_SFENCE_VMA	0x12000073
+#define RV_INSN_MASK_SFENCE_VMA		0xfe00707f
+#define RV_INSN_MATCH_SINVAL_VMA	0x16000073
+#define RV_INSN_MASK_SINVAL_VMA		0xfe00707f
+#define RV_INSN_MATCH_SFENCE_W_INVAL	0x18000073
+#define RV_INSN_MASK_SFENCE_W_INVAL	0xffffffff
+#define RV_INSN_MATCH_SFENCE_INVAL_IR	0x18100073
+#define RV_INSN_MASK_SFENCE_INVAL_IR	0xffffffff
+
 #define __INSN_LENGTH_MASK	_UL(0x3)
 #define __INSN_LENGTH_GE_32	_UL(0x3)
 #define __INSN_OPCODE_MASK	_UL(0x7F)
diff --git a/arch/riscv/include/asm/kdebug.h b/arch/riscv/include/asm/kdebug.h
index 85ac00411f6e..53e989781aa1 100644
--- a/arch/riscv/include/asm/kdebug.h
+++ b/arch/riscv/include/asm/kdebug.h
@@ -6,7 +6,8 @@
 enum die_val {
 	DIE_UNUSED,
 	DIE_TRAP,
-	DIE_OOPS
+	DIE_OOPS,
+	DIE_DEBUG
 };
 
 #endif
diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
index 4d794573e3db..1898f041a8f4 100644
--- a/arch/riscv/include/asm/kvm_host.h
+++ b/arch/riscv/include/asm/kvm_host.h
@@ -24,6 +24,7 @@
 #include <asm/kvm_vcpu_sbi_fwft.h>
 #include <asm/kvm_vcpu_timer.h>
 #include <asm/kvm_vcpu_pmu.h>
+#include <asm/kvm_vcpu_debug.h>
 
 #define KVM_MAX_VCPUS			1024
 
@@ -100,6 +101,9 @@ struct kvm_arch {
 
 	/* KVM_CAP_RISCV_MP_STATE_RESET */
 	bool mp_state_reset;
+
+	/* hcontext ID for guest VM */
+	unsigned long hcontext;
 };
 
 struct kvm_cpu_trap {
@@ -180,6 +184,10 @@ struct kvm_vcpu_reset_state {
 	unsigned long a1;
 };
 
+struct kvm_vcpu_sdtrig_csr {
+	unsigned long scontext;
+};
+
 struct kvm_vcpu_arch {
 	/* VCPU ran at least once */
 	bool ran_atleast_once;
@@ -202,6 +210,9 @@ struct kvm_vcpu_arch {
 	unsigned long host_senvcfg;
 	unsigned long host_sstateen0;
 
+	/* SCONTEXT of Host */
+	unsigned long host_scontext;
+
 	/* CPU context of Host */
 	struct kvm_cpu_context host_context;
 
@@ -214,6 +225,9 @@ struct kvm_vcpu_arch {
 	/* CPU Smstateen CSR context of Guest VCPU */
 	struct kvm_vcpu_smstateen_csr smstateen_csr;
 
+	/* CPU Sdtrig CSR context of Guest VCPU */
+	struct kvm_vcpu_sdtrig_csr sdtrig_csr;
+
 	/* CPU reset state of Guest VCPU */
 	struct kvm_vcpu_reset_state reset_state;
 
diff --git a/arch/riscv/include/asm/kvm_vcpu_debug.h b/arch/riscv/include/asm/kvm_vcpu_debug.h
new file mode 100644
index 000000000000..0a025fc4e6dd
--- /dev/null
+++ b/arch/riscv/include/asm/kvm_vcpu_debug.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2024 SiFive
+ *
+ * Authors:
+ *	Yong-Xuan Wang <yongxuan.wang@sifive.com>
+ */
+
+#ifndef __KVM_VCPU_RISCV_DEBUG_H
+#define __KVM_VCPU_RISCV_DEBUG_H
+
+#include <linux/types.h>
+
+DECLARE_STATIC_KEY_FALSE(use_hcontext);
+extern atomic_long_t hcontext_id_share;
+
+void kvm_riscv_debug_init(void);
+void kvm_riscv_debug_exit(void);
+void kvm_riscv_debug_get_hcontext_id(struct kvm *kvm);
+void kvm_riscv_debug_return_hcontext_id(struct kvm *kvm);
+void kvm_riscv_debug_vcpu_swap_in_guest_context(struct kvm_vcpu *vcpu);
+void kvm_riscv_debug_vcpu_swap_in_host_context(struct kvm_vcpu *vcpu);
+
+#endif
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index da5426122d28..d9c270ed1bb4 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -7,6 +7,7 @@
 #define _ASM_RISCV_PROCESSOR_H
 
 #include <linux/const.h>
+#include <asm/hw_breakpoint.h>
 #include <linux/cache.h>
 #include <linux/prctl.h>
 
@@ -122,6 +123,9 @@ struct thread_struct {
 	/* A forced icache flush is not needed if migrating to the previous cpu. */
 	unsigned int prev_cpu;
 #endif
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+	struct perf_event *ptrace_bps[HW_BP_NUM_MAX];
+#endif
 };
 
 /* Whitelist the fstate from the task_struct for hardened usercopy */
diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h
index ccc77a89b1e2..b6e1fb52963e 100644
--- a/arch/riscv/include/asm/sbi.h
+++ b/arch/riscv/include/asm/sbi.h
@@ -37,6 +37,7 @@ enum sbi_ext_id {
 	SBI_EXT_NACL = 0x4E41434C,
 	SBI_EXT_FWFT = 0x46574654,
 	SBI_EXT_MPXY = 0x4D505859,
+	SBI_EXT_DBTR = 0x44425452,
 
 	/* Experimentals extensions must lie within this range */
 	SBI_EXT_EXPERIMENTAL_START = 0x08000000,
@@ -55,6 +56,9 @@ enum sbi_ext_base_fid {
 	SBI_EXT_BASE_GET_MVENDORID,
 	SBI_EXT_BASE_GET_MARCHID,
 	SBI_EXT_BASE_GET_MIMPID,
+#if defined(CONFIG_SOC_SPACEMIT)
+       SBI_EXT_BASE_FLUSH_CACHE_ALL,
+#endif
 };
 
 enum sbi_ext_time_fid {
@@ -481,6 +485,8 @@ enum sbi_mpxy_attribute_id {
 /* Possible values of MSG_PROT_ID attribute as-per SBI v3.0 (or higher) */
 enum sbi_mpxy_msgproto_id {
 	SBI_MPXY_MSGPROTO_RPMI_ID = 0x0,
+	SBI_MPXY_MSGPROTO_STMM_ID = 0x1,
+	SBI_MPXY_MSGPROTO_TEE_ID = 0x2,
 };
 
 /* RPMI message protocol specific MPXY attributes */
@@ -505,6 +511,34 @@ enum sbi_mpxy_rpmi_attribute_id {
 #define SBI_MPXY_CHAN_CAP_SEND_WITHOUT_RESP	BIT(4)
 #define SBI_MPXY_CHAN_CAP_GET_NOTIFICATIONS	BIT(5)
 
+/* SBI debug triggers function IDs */
+enum sbi_ext_dbtr_fid {
+	SBI_EXT_DBTR_NUM_TRIGGERS = 0,
+	SBI_EXT_DBTR_SETUP_SHMEM,
+	SBI_EXT_DBTR_TRIG_READ,
+	SBI_EXT_DBTR_TRIG_INSTALL,
+	SBI_EXT_DBTR_TRIG_UPDATE,
+	SBI_EXT_DBTR_TRIG_UNINSTALL,
+	SBI_EXT_DBTR_TRIG_ENABLE,
+	SBI_EXT_DBTR_TRIG_DISABLE,
+};
+
+struct sbi_dbtr_data_msg {
+	unsigned long tstate;
+	unsigned long tdata1;
+	unsigned long tdata2;
+	unsigned long tdata3;
+};
+
+struct sbi_dbtr_id_msg {
+	unsigned long idx;
+};
+
+union sbi_dbtr_shmem_entry {
+	struct sbi_dbtr_data_msg data;
+	struct sbi_dbtr_id_msg id;
+};
+
 /* SBI spec version fields */
 #define SBI_SPEC_VERSION_DEFAULT	0x1
 #define SBI_SPEC_VERSION_MAJOR_SHIFT	24
@@ -581,6 +615,11 @@ long sbi_probe_extension(int ext);
 int sbi_fwft_set(u32 feature, unsigned long value, unsigned long flags);
 int sbi_fwft_set_cpumask(const cpumask_t *mask, u32 feature,
 			 unsigned long value, unsigned long flags);
+
+#if defined(CONFIG_SOC_SPACEMIT)
+void sbi_flush_local_dcache_all(void);
+#endif
+
 /**
  * sbi_fwft_set_online_cpus() - Set a feature on all online cpus
  * @feature: The feature to be set
diff --git a/arch/riscv/include/asm/suspend.h b/arch/riscv/include/asm/suspend.h
index dc5782b5fbad..53160891da88 100644
--- a/arch/riscv/include/asm/suspend.h
+++ b/arch/riscv/include/asm/suspend.h
@@ -13,9 +13,16 @@ struct suspend_context {
 	/* Saved and restored by low-level functions */
 	struct pt_regs regs;
 	/* Saved and restored by high-level functions */
+	unsigned long scontext;
 	unsigned long envcfg;
 	unsigned long tvec;
 	unsigned long ie;
+#if __riscv_xlen < 64
+	unsigned long hstateen0h;
+#endif
+	unsigned long hstateen0;
+	unsigned long sstateen0;
+
 #ifdef CONFIG_MMU
 	unsigned long satp;
 	unsigned long stimecmp;
diff --git a/arch/riscv/include/asm/switch_to.h b/arch/riscv/include/asm/switch_to.h
index 0e71eb82f920..8f19f58a3e52 100644
--- a/arch/riscv/include/asm/switch_to.h
+++ b/arch/riscv/include/asm/switch_to.h
@@ -8,6 +8,7 @@
 
 #include <linux/jump_label.h>
 #include <linux/sched/task_stack.h>
+#include <linux/pid.h>
 #include <linux/mm_types.h>
 #include <asm/vector.h>
 #include <asm/cpufeature.h>
@@ -70,6 +71,12 @@ static __always_inline bool has_fpu(void) { return false; }
 #define __switch_to_fpu(__prev, __next) do { } while (0)
 #endif
 
+DECLARE_STATIC_KEY_FALSE(use_scontext);
+static __always_inline bool has_scontext(void)
+{
+	return static_branch_likely(&use_scontext);
+}
+
 static inline void envcfg_update_bits(struct task_struct *task,
 				      unsigned long mask, unsigned long val)
 {
@@ -88,6 +95,12 @@ static inline void __switch_to_envcfg(struct task_struct *next)
 			:: "r" (next->thread.envcfg) : "memory");
 }
 
+static __always_inline void __switch_to_scontext(struct task_struct *__prev,
+						 struct task_struct *__next)
+{
+	csr_write(CSR_SCONTEXT, task_pid_nr(__next));
+}
+
 extern struct task_struct *__switch_to(struct task_struct *,
 				       struct task_struct *);
 
@@ -119,6 +132,8 @@ do {							\
 		__switch_to_fpu(__prev, __next);	\
 	if (has_vector() || has_xtheadvector())		\
 		__switch_to_vector(__prev, __next);	\
+	if (has_scontext())				\
+		__switch_to_scontext(__prev, __next);	\
 	if (switch_to_should_flush_icache(__next))	\
 		local_flush_icache_all();		\
 	__switch_to_envcfg(__next);			\
diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h
index f5f4f7f85543..9ea79aae2483 100644
--- a/arch/riscv/include/asm/uaccess.h
+++ b/arch/riscv/include/asm/uaccess.h
@@ -477,6 +477,11 @@ static inline void user_access_restore(unsigned long enabled) { }
 	if (__asm_copy_from_user_sum_enabled(_dst, _src, _len))		\
 		goto label;
 
+/* Memory cacheability check for vector uaccess optimization */
+#ifdef CONFIG_RISCV_ISA_V
+int is_cacheable_safe(const void *addr);
+#endif
+
 #else /* CONFIG_MMU */
 #include <asm-generic/uaccess.h>
 #endif /* CONFIG_MMU */
diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h
index 759a4852c09a..eeb1be3a634d 100644
--- a/arch/riscv/include/uapi/asm/kvm.h
+++ b/arch/riscv/include/uapi/asm/kvm.h
@@ -100,6 +100,11 @@ struct kvm_riscv_smstateen_csr {
 	unsigned long sstateen0;
 };
 
+/* Sdtrig CSR for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
+struct kvm_riscv_sdtrig_csr {
+	unsigned long scontext;
+};
+
 /* TIMER registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
 struct kvm_riscv_timer {
 	__u64 frequency;
@@ -190,6 +195,7 @@ enum KVM_RISCV_ISA_EXT_ID {
 	KVM_RISCV_ISA_EXT_ZFBFMIN,
 	KVM_RISCV_ISA_EXT_ZVFBFMIN,
 	KVM_RISCV_ISA_EXT_ZVFBFWMA,
+	KVM_RISCV_ISA_EXT_SDTRIG,
 	KVM_RISCV_ISA_EXT_MAX,
 };
 
@@ -257,12 +263,15 @@ struct kvm_riscv_sbi_fwft {
 #define KVM_REG_RISCV_CSR_GENERAL	(0x0 << KVM_REG_RISCV_SUBTYPE_SHIFT)
 #define KVM_REG_RISCV_CSR_AIA		(0x1 << KVM_REG_RISCV_SUBTYPE_SHIFT)
 #define KVM_REG_RISCV_CSR_SMSTATEEN	(0x2 << KVM_REG_RISCV_SUBTYPE_SHIFT)
+#define KVM_REG_RISCV_CSR_SDTRIG	(0x3 << KVM_REG_RISCV_SUBTYPE_SHIFT)
 #define KVM_REG_RISCV_CSR_REG(name)	\
 		(offsetof(struct kvm_riscv_csr, name) / sizeof(unsigned long))
 #define KVM_REG_RISCV_CSR_AIA_REG(name)	\
 	(offsetof(struct kvm_riscv_aia_csr, name) / sizeof(unsigned long))
 #define KVM_REG_RISCV_CSR_SMSTATEEN_REG(name)  \
 	(offsetof(struct kvm_riscv_smstateen_csr, name) / sizeof(unsigned long))
+#define KVM_REG_RISCV_CSR_SDTRIG_REG(name)  \
+	(offsetof(struct kvm_riscv_sdtrig_csr, name) / sizeof(unsigned long))
 
 /* Timer registers are mapped as type 4 */
 #define KVM_REG_RISCV_TIMER		(0x04 << KVM_REG_RISCV_TYPE_SHIFT)
diff --git a/arch/riscv/include/uapi/asm/ptrace.h b/arch/riscv/include/uapi/asm/ptrace.h
index beff8df80ac9..9709734ca8eb 100644
--- a/arch/riscv/include/uapi/asm/ptrace.h
+++ b/arch/riscv/include/uapi/asm/ptrace.h
@@ -120,6 +120,15 @@ struct __riscv_v_regset_state {
 	char vreg[];
 };
 
+struct __riscv_hwdebug_state {
+	__u64     dbg_info;
+	struct {
+		__u64 addr;
+		__u64 type;
+		__u64 len;
+	}       dbg_regs[16];
+};
+
 /*
  * According to spec: The number of bits in a single vector register,
  * VLEN >= ELEN, which must be a power of 2, and must be no greater than
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index f60fce69b725..486ba6a12664 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -98,6 +98,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE)	+= mcount-dyn.o
 
 obj-$(CONFIG_PERF_EVENTS)	+= perf_callchain.o
 obj-$(CONFIG_HAVE_PERF_REGS)	+= perf_regs.o
+obj-$(CONFIG_HAVE_HW_BREAKPOINT)	+= hw_breakpoint.o
 obj-$(CONFIG_RISCV_SBI)		+= sbi.o sbi_ecall.o
 ifeq ($(CONFIG_RISCV_SBI), y)
 obj-$(CONFIG_SMP)		+= sbi-ipi.o
diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c
index 3f50d3dd76c6..2cf05d6d20cc 100644
--- a/arch/riscv/kernel/cpu-hotplug.c
+++ b/arch/riscv/kernel/cpu-hotplug.c
@@ -15,6 +15,7 @@
 #include <asm/cpu_ops.h>
 #include <asm/numa.h>
 #include <asm/smp.h>
+#include <asm/sbi.h>
 
 bool cpu_has_hotplug(unsigned int cpu)
 {
@@ -69,8 +70,14 @@ void __noreturn arch_cpu_idle_dead(void)
 {
 	idle_task_exit();
 
+#if defined(CONFIG_SOC_SPACEMIT)
+	sbi_flush_local_dcache_all();
+#endif
 	cpuhp_ap_report_dead();
 
+#if defined(CONFIG_SOC_SPACEMIT)
+	sbi_flush_local_dcache_all();
+#endif
 	cpu_ops->cpu_stop();
 	/* It should never reach here */
 	BUG();
diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
index 3dbc8cc557dd..28be4e614fff 100644
--- a/arch/riscv/kernel/cpu.c
+++ b/arch/riscv/kernel/cpu.c
@@ -330,12 +330,15 @@ static int c_show(struct seq_file *m, void *v)
 {
 	unsigned long cpu_id = (unsigned long)v - 1;
 	struct riscv_cpuinfo *ci = per_cpu_ptr(&riscv_cpuinfo, cpu_id);
-	struct device_node *node;
-	const char *compat;
+	struct device_node *node = of_get_cpu_node(cpu_id, NULL);
+	const char *compat, *model;
 
 	seq_printf(m, "processor\t: %lu\n", cpu_id);
 	seq_printf(m, "hart\t\t: %lu\n", cpuid_to_hartid_map(cpu_id));
 
+	if (!of_property_read_string(node, "model", &model))
+		seq_printf(m, "model name\t: %s\n", model);
+
 	/*
 	 * For historical raisins, the isa: line is limited to the lowest common
 	 * denominator of extensions supported across all harts. A true list of
@@ -347,13 +350,10 @@ static int c_show(struct seq_file *m, void *v)
 	print_mmu(m);
 
 	if (acpi_disabled) {
-		node = of_get_cpu_node(cpu_id, NULL);
-
 		if (!of_property_read_string(node, "compatible", &compat) &&
 		    strcmp(compat, "riscv"))
 			seq_printf(m, "uarch\t\t: %s\n", compat);
 
-		of_node_put(node);
 	}
 
 	seq_printf(m, "mvendorid\t: 0x%lx\n", ci->mvendorid);
@@ -367,6 +367,7 @@ static int c_show(struct seq_file *m, void *v)
 	seq_puts(m, "hart isa\t: ");
 	print_isa(m, hart_isa[cpu_id].isa, cpu_id);
 	seq_puts(m, "\n");
+	of_node_put(node);
 
 	return 0;
 }
diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c
index 72ca768f4e91..903e27cf908f 100644
--- a/arch/riscv/kernel/cpufeature.c
+++ b/arch/riscv/kernel/cpufeature.c
@@ -28,6 +28,7 @@
 #include <asm/vector.h>
 #include <asm/vendor_extensions.h>
 #include <asm/vendor_extensions/thead.h>
+#include <linux/soc/spacemit/spacemit-hmp.h>
 
 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
 
@@ -45,6 +46,19 @@ struct riscv_isainfo hart_isa[NR_CPUS];
 
 u32 thead_vlenb_of;
 
+atomic_t hcontext_disable;
+atomic_t scontext_disable;
+
+DEFINE_STATIC_KEY_FALSE_RO(use_hcontext);
+EXPORT_SYMBOL(use_hcontext);
+
+DEFINE_STATIC_KEY_FALSE_RO(use_scontext);
+EXPORT_SYMBOL(use_scontext);
+
+/* Record the maximum number that the hcontext CSR allowed to hold */
+atomic_long_t hcontext_id_share;
+EXPORT_SYMBOL(hcontext_id_share);
+
 /**
  * riscv_isa_extension_base() - Get base extension word
  *
@@ -76,6 +90,16 @@ bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned i
 	if (bit >= RISCV_ISA_EXT_MAX)
 		return false;
 
+#ifdef CONFIG_SPACEMIT_HMP
+	if ((bit == RISCV_ISA_EXT_h) && test_bit(bit, bmap)) {
+		struct cpumask	cpumask;
+		unsigned int cpu = smp_processor_id();
+
+		hmp_get_cpumask(&cpumask, HMP_REGULAR);
+		return cpumask_test_cpu(cpu, &cpumask);
+	}
+#endif
+
 	return test_bit(bit, bmap);
 }
 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
@@ -540,6 +564,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = {
 	__RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
 	__RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
 	__RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC),
+	__RISCV_ISA_EXT_DATA(sdtrig, RISCV_ISA_EXT_SDTRIG),
 };
 
 const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext);
@@ -859,14 +884,36 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap)
 		 * have.
 		 */
 		if (elf_hwcap)
+#ifndef CONFIG_SPACEMIT_HMP
 			elf_hwcap &= this_hwcap;
+#else
+			/*
+			 * The hypervisor extension is supported on X100 cores,
+			 * but not A100 cores. So, make some special processing
+			 * to support hypervisor on spacemit k3 soc.
+			 */
+			elf_hwcap &= (this_hwcap | isa2hwcap[RISCV_ISA_EXT_h]);
+#endif
 		else
 			elf_hwcap = this_hwcap;
 
 		if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
 		else
+#ifndef CONFIG_SPACEMIT_HMP
 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
+#else
+		{
+			/*
+			 * The hypervisor extension is supported on X100 cores,
+			 * but not A100 cores. So, make some special processing
+			 * to support hypervisor on spacemit k3 soc.
+			 */
+			bitmap_copy(source_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
+			bitmap_set(source_isa, RISCV_ISA_EXT_h, 1);
+			bitmap_and(riscv_isa, riscv_isa, source_isa, RISCV_ISA_EXT_MAX);
+		}
+#endif
 	}
 
 	if (!acpi_disabled && rhct)
@@ -1010,15 +1057,36 @@ static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap)
 		 * common capabilities of every "okay" hart, in case they don't.
 		 */
 		if (elf_hwcap)
+#ifndef CONFIG_SPACEMIT_HMP
 			elf_hwcap &= this_hwcap;
+#else
+			/*
+			 * The hypervisor extension is supported on X100 cores,
+			 * but not A100 cores. So, make some special processing
+			 * to support hypervisor on spacemit k3 soc.
+			 */
+			elf_hwcap &= (this_hwcap | isa2hwcap[RISCV_ISA_EXT_h]);
+#endif
 		else
 			elf_hwcap = this_hwcap;
 
 		if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
 		else
+#ifndef CONFIG_SPACEMIT_HMP
 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
-
+#else
+		{
+			/*
+			 * The hypervisor extension is supported on X100 cores,
+			 * but not A100 cores. So, make some special processing
+			 * to support hypervisor on spacemit k3 soc.
+			 */
+			bitmap_copy(source_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
+			bitmap_set(source_isa, RISCV_ISA_EXT_h, 1);
+			bitmap_and(riscv_isa, riscv_isa, source_isa, RISCV_ISA_EXT_MAX);
+		}
+#endif
 		riscv_fill_vendor_ext_list(cpu);
 	}
 
@@ -1118,6 +1186,147 @@ unsigned long riscv_get_elf_hwcap(void)
 	return hwcap;
 }
 
+static void __init sdtrig_percpu_csrs_check(void *data)
+{
+	struct device_node *node;
+	struct device_node *debug_node;
+	struct device_node *trigger_module;
+
+	unsigned int cpu = smp_processor_id();
+
+	/*
+	 * Expect every cpu node has the [h/s]context-present property
+	 * otherwise, jump to sdtrig_csrs_disable_all to disable all access to
+	 * [h/s]context CSRs
+	 */
+	node = of_cpu_device_node_get(cpu);
+	if (!node)
+		goto sdtrig_csrs_disable_all;
+
+	debug_node = of_get_compatible_child(node, "riscv,debug-v1.0.0");
+	of_node_put(node);
+
+	if (!debug_node)
+		goto sdtrig_csrs_disable_all;
+
+	trigger_module = of_get_child_by_name(debug_node, "trigger-module");
+	of_node_put(debug_node);
+
+	if (!trigger_module)
+		goto sdtrig_csrs_disable_all;
+
+	if (!(IS_ENABLED(CONFIG_KVM) &&
+	      of_property_read_bool(trigger_module, "hcontext-present")))
+		atomic_inc(&hcontext_disable);
+
+	if (!of_property_read_bool(trigger_module, "scontext-present"))
+		atomic_inc(&scontext_disable);
+
+	of_node_put(trigger_module);
+
+	/*
+	 * Before access to hcontext/scontext CSRs, if the smstateen
+	 * extension is present, the accessibility will be controlled
+	 * by the hstateen0[H]/sstateen0 CSRs.
+	 */
+	if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SMSTATEEN)) {
+		u64 hstateen_bit;
+
+		if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_h)) {
+#if __riscv_xlen > 32
+			csr_set(CSR_HSTATEEN0, SMSTATEEN0_HSCONTEXT);
+			hstateen_bit = csr_read(CSR_HSTATEEN0);
+#else
+			csr_set(CSR_HSTATEEN0H, SMSTATEEN0_HSCONTEXT >> 32);
+			hstateen_bit = csr_read(CSR_HSTATEEN0H) << 32;
+#endif
+			if (!(hstateen_bit & SMSTATEEN0_HSCONTEXT))
+				goto sdtrig_csrs_disable_all;
+
+		} else {
+			if (IS_ENABLED(CONFIG_KVM))
+				atomic_inc(&hcontext_disable);
+
+			/*
+			 * The smstateen extension doesn't provide
+			 * high 32 bits of sstateen0 CSR which represent
+			 * accessibility for scontext CSR;
+			 * The decision is left on whether the dts has the
+			 * property to access the scontext CSR.
+			 */
+		}
+	}
+
+	/*
+	 * The code can only access hcontext/scontext CSRs if:
+	 * The cpu dts node have [h/s]context-present;
+	 * If Smstateen extension is presented, then the accessibility bit
+	 * toward hcontext/scontext CSRs is enabled; Or the Smstateen extension
+	 * isn't available, thus the access won't be blocked by it.
+	 *
+	 * With writing 1 to the every bit of these CSRs, we retrieve the
+	 * maximum bits that is available on the CSRs. and decide
+	 * whether it's suit for its context recording operation.
+	 */
+	if (IS_ENABLED(CONFIG_KVM) &&
+	    !atomic_read(&hcontext_disable)) {
+		unsigned long hcontext_available_bits = 0;
+
+		csr_write(CSR_HCONTEXT, -1UL);
+		hcontext_available_bits = csr_swap(CSR_HCONTEXT, hcontext_available_bits);
+
+		/* hcontext CSR is required by at least 1 bit */
+		if (hcontext_available_bits)
+			atomic_long_and(hcontext_available_bits, &hcontext_id_share);
+		else
+			atomic_inc(&hcontext_disable);
+	}
+
+	if (!atomic_read(&scontext_disable)) {
+		unsigned long scontext_available_bits = 0;
+
+		csr_write(CSR_SCONTEXT, -1UL);
+		scontext_available_bits = csr_swap(CSR_SCONTEXT, scontext_available_bits);
+
+		/* scontext CSR is required by at least the sizeof pid_t */
+		if (scontext_available_bits < ((1UL << (sizeof(pid_t) << 3)) - 1))
+			atomic_inc(&scontext_disable);
+	}
+
+	return;
+
+sdtrig_csrs_disable_all:
+	if (IS_ENABLED(CONFIG_KVM))
+		atomic_inc(&hcontext_disable);
+
+	atomic_inc(&scontext_disable);
+}
+
+static int __init sdtrig_enable_csrs_fill(void)
+{
+	if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SDTRIG)) {
+		atomic_long_set(&hcontext_id_share, -1UL);
+
+		/* check every CPUs sdtrig extension optional CSRs */
+		sdtrig_percpu_csrs_check(NULL);
+		smp_call_function(sdtrig_percpu_csrs_check, NULL, 1);
+
+		if (IS_ENABLED(CONFIG_KVM) &&
+		    !atomic_read(&hcontext_disable)) {
+			pr_info("riscv-sdtrig: Writing 'GuestOS ID' to hcontext CSR is enabled\n");
+			static_branch_enable(&use_hcontext);
+		}
+
+		if (!atomic_read(&scontext_disable)) {
+			pr_info("riscv-sdtrig: Writing 'PID' to scontext CSR is enabled\n");
+			static_branch_enable(&use_scontext);
+		}
+	}
+	return 0;
+}
+
+arch_initcall(sdtrig_enable_csrs_fill);
+
 void __init riscv_user_isa_enable(void)
 {
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ))
@@ -1126,7 +1335,7 @@ void __init riscv_user_isa_enable(void)
 		pr_warn("Zicboz disabled as it is unavailable on some harts\n");
 
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM))
-		current->thread.envcfg |= ENVCFG_CBCFE;
+		current->thread.envcfg |= (ENVCFG_CBCFE | ENVCFG_CBIE);
 	else if (any_cpu_has_zicbom)
 		pr_warn("Zicbom disabled as it is unavailable on some harts\n");
 
diff --git a/arch/riscv/kernel/hw_breakpoint.c b/arch/riscv/kernel/hw_breakpoint.c
new file mode 100644
index 000000000000..cce68f0ad653
--- /dev/null
+++ b/arch/riscv/kernel/hw_breakpoint.c
@@ -0,0 +1,817 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Qualcomm Technologies, Inc.
+ */
+
+#include <linux/hw_breakpoint.h>
+#include <linux/perf_event.h>
+#include <linux/spinlock.h>
+#include <linux/percpu.h>
+#include <linux/kdebug.h>
+#include <linux/bitops.h>
+#include <linux/cpu.h>
+#include <linux/cpuhotplug.h>
+
+#include <asm/sbi.h>
+#include <asm/insn.h>
+
+/* Registered per-cpu bp/wp */
+static DEFINE_PER_CPU(struct perf_event *, pcpu_hw_bp_events[HW_BP_NUM_MAX]);
+static DEFINE_PER_CPU(unsigned long, ecall_lock_flags);
+static DEFINE_PER_CPU(raw_spinlock_t, ecall_lock);
+
+/* Per-cpu shared memory between S and M mode */
+static union sbi_dbtr_shmem_entry __percpu *sbi_dbtr_shmem;
+
+/* number of debug triggers on this cpu . */
+static int dbtr_total_num __ro_after_init;
+static int dbtr_type __ro_after_init;
+static int dbtr_init __ro_after_init;
+
+#if __riscv_xlen == 64
+#define MEM_HI(_m)	0
+#define MEM_LO(_m)	((u64)(_m))
+#elif __riscv_xlen == 32
+#define MEM_HI(_m)	((u64)(_m) >> 32)
+#define MEM_LO(_m)	((u64)(_m) & 0xFFFFFFFFUL)
+#else
+#error "Unknown __riscv_xlen"
+#endif
+
+static int arch_smp_setup_sbi_shmem(unsigned int cpu)
+{
+	union sbi_dbtr_shmem_entry *dbtr_shmem;
+	unsigned long shmem_pa;
+	struct sbiret ret;
+	int rc = 0;
+
+	dbtr_shmem = per_cpu_ptr(sbi_dbtr_shmem, cpu);
+	if (!dbtr_shmem) {
+		pr_err("Invalid per-cpu shared memory for debug triggers\n");
+		return -ENODEV;
+	}
+
+	shmem_pa = __pa(dbtr_shmem);
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_SETUP_SHMEM,
+			MEM_LO(shmem_pa), MEM_HI(shmem_pa), 0, 0, 0, 0);
+
+	if (ret.error) {
+		switch (ret.error) {
+		case SBI_ERR_DENIED:
+			pr_warn("%s: Access denied for shared memory at %lx\n",
+				__func__, shmem_pa);
+			rc = -EPERM;
+			break;
+
+		case SBI_ERR_INVALID_PARAM:
+		case SBI_ERR_INVALID_ADDRESS:
+			pr_warn("%s: Invalid address parameter (%lu)\n",
+				__func__, ret.error);
+			rc = -EINVAL;
+			break;
+
+		case SBI_ERR_ALREADY_AVAILABLE:
+			pr_warn("%s: Shared memory is already set\n",
+				__func__);
+			rc = -EADDRINUSE;
+			break;
+
+		case SBI_ERR_FAILURE:
+			pr_err("%s: Internal sdtrig state error\n",
+			       __func__);
+			rc = -ENXIO;
+			break;
+
+		default:
+			pr_warn("%s: Unknown error %lu\n", __func__, ret.error);
+			rc = -ENXIO;
+			break;
+		}
+	}
+
+	pr_info("CPU %d: HW Breakpoint shared memory registered.\n", cpu);
+
+	return rc;
+}
+
+static int arch_smp_teardown_sbi_shmem(unsigned int cpu)
+{
+	struct sbiret ret;
+
+	/* Disable shared memory */
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_SETUP_SHMEM,
+			-1UL, -1UL, 0, 0, 0, 0);
+
+	if (ret.error) {
+		switch (ret.error) {
+		case SBI_ERR_DENIED:
+			pr_err("%s: Access denied for shared memory.\n",
+			       __func__);
+			break;
+
+		case SBI_ERR_INVALID_PARAM:
+		case SBI_ERR_INVALID_ADDRESS:
+			pr_err("%s: Invalid address parameter (%lu)\n",
+			       __func__, ret.error);
+			break;
+
+		case SBI_ERR_ALREADY_AVAILABLE:
+			pr_err("%s: Shared memory is already set\n",
+			       __func__);
+			break;
+		case SBI_ERR_FAILURE:
+			pr_err("%s: Internal sdtrig state error\n",
+			       __func__);
+			break;
+		default:
+			pr_err("%s: Unknown error %lu\n", __func__, ret.error);
+			break;
+		}
+	}
+
+	pr_warn("CPU %d: HW Breakpoint shared memory disabled.\n", cpu);
+
+	return 0;
+}
+
+static void init_sbi_dbtr(void)
+{
+	unsigned long tdata1;
+	struct sbiret ret;
+
+	if (sbi_probe_extension(SBI_EXT_DBTR) <= 0) {
+		pr_warn("%s: SBI_EXT_DBTR is not supported\n", __func__);
+		dbtr_total_num = 0;
+		goto done;
+	}
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_NUM_TRIGGERS,
+			0, 0, 0, 0, 0, 0);
+	if (ret.error) {
+		pr_warn("%s: Failed to detect triggers\n", __func__);
+		dbtr_total_num = 0;
+		goto done;
+	}
+
+	tdata1 = 0;
+	tdata1 = RV_DBTR_SET_TDATA1_TYPE(tdata1, RV_DBTR_TRIG_MCONTROL6);
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_NUM_TRIGGERS,
+			tdata1, 0, 0, 0, 0, 0);
+	if (ret.error) {
+		pr_warn("%s: failed to detect mcontrol6 triggers\n", __func__);
+	} else if (!ret.value) {
+		pr_warn("%s: type 6 triggers not available\n", __func__);
+	} else {
+		dbtr_total_num = ret.value;
+		dbtr_type = RV_DBTR_TRIG_MCONTROL6;
+		pr_warn("%s: mcontrol6 trigger available.\n", __func__);
+		goto done;
+	}
+
+	/* fallback to type 2 triggers if type 6 is not available */
+
+	tdata1 = 0;
+	tdata1 = RV_DBTR_SET_TDATA1_TYPE(tdata1, RV_DBTR_TRIG_MCONTROL);
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_NUM_TRIGGERS,
+			tdata1, 0, 0, 0, 0, 0);
+	if (ret.error) {
+		pr_warn("%s: failed to detect mcontrol triggers\n", __func__);
+	} else if (!ret.value) {
+		pr_warn("%s: type 2 triggers not available\n", __func__);
+	} else {
+		dbtr_total_num = ret.value;
+		dbtr_type = RV_DBTR_TRIG_MCONTROL;
+		goto done;
+	}
+
+done:
+	dbtr_init = 1;
+}
+
+int hw_breakpoint_slots(int type)
+{
+	/*
+	 * We can be called early, so don't rely on
+	 * static variables being initialised.
+	 */
+
+	if (!dbtr_init)
+		init_sbi_dbtr();
+
+	return dbtr_total_num;
+}
+
+int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
+{
+	unsigned int len;
+	unsigned long va;
+
+	va = hw->address;
+	len = hw->len;
+
+	return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
+}
+
+static int rv_init_mcontrol_trigger(const struct perf_event_attr *attr,
+				    struct arch_hw_breakpoint *hw)
+{
+	switch (attr->bp_type) {
+	case HW_BREAKPOINT_X:
+		hw->type = RV_DBTR_BP;
+		RV_DBTR_SET_MC_EXEC_BIT(hw->tdata1);
+		break;
+	case HW_BREAKPOINT_R:
+		hw->type = RV_DBTR_WP;
+		RV_DBTR_SET_MC_LOAD_BIT(hw->tdata1);
+		break;
+	case HW_BREAKPOINT_W:
+		hw->type = RV_DBTR_WP;
+		RV_DBTR_SET_MC_STORE_BIT(hw->tdata1);
+		break;
+	case HW_BREAKPOINT_RW:
+		hw->type = RV_DBTR_WP;
+		RV_DBTR_SET_MC_LOAD_BIT(hw->tdata1);
+		RV_DBTR_SET_MC_STORE_BIT(hw->tdata1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (attr->bp_len) {
+	case HW_BREAKPOINT_LEN_1:
+		hw->len = 1;
+		break;
+	case HW_BREAKPOINT_LEN_2:
+		hw->len = 2;
+		break;
+	case HW_BREAKPOINT_LEN_4:
+		hw->len = 4;
+		break;
+#if __riscv_xlen >= 64
+	case HW_BREAKPOINT_LEN_8:
+		hw->len = 8;
+		break;
+#endif
+	default:
+		return -EINVAL;
+	}
+
+	/* any size */
+	hw->tdata1 = RV_DBTR_SET_MC_SIZELO(hw->tdata1, 0);
+	hw->tdata1 = RV_DBTR_SET_MC_SIZEHI(hw->tdata1, 0);
+	hw->tdata1 = RV_DBTR_SET_MC_TYPE(hw->tdata1, RV_DBTR_TRIG_MCONTROL);
+
+	CLEAR_DBTR_BIT(hw->tdata1, MC, DMODE);
+	CLEAR_DBTR_BIT(hw->tdata1, MC, TIMING);
+	CLEAR_DBTR_BIT(hw->tdata1, MC, SELECT);
+	CLEAR_DBTR_BIT(hw->tdata1, MC, ACTION);
+	CLEAR_DBTR_BIT(hw->tdata1, MC, CHAIN);
+	CLEAR_DBTR_BIT(hw->tdata1, MC, MATCH);
+	CLEAR_DBTR_BIT(hw->tdata1, MC, M);
+
+	SET_DBTR_BIT(hw->tdata1, MC, S);
+	SET_DBTR_BIT(hw->tdata1, MC, U);
+
+	return 0;
+}
+
+static int rv_init_mcontrol6_trigger(const struct perf_event_attr *attr,
+				     struct arch_hw_breakpoint *hw)
+{
+	switch (attr->bp_type) {
+	case HW_BREAKPOINT_X:
+		hw->type = RV_DBTR_BP;
+		RV_DBTR_SET_MC6_EXEC_BIT(hw->tdata1);
+		break;
+	case HW_BREAKPOINT_R:
+		hw->type = RV_DBTR_WP;
+		RV_DBTR_SET_MC6_LOAD_BIT(hw->tdata1);
+		break;
+	case HW_BREAKPOINT_W:
+		hw->type = RV_DBTR_WP;
+		RV_DBTR_SET_MC6_STORE_BIT(hw->tdata1);
+		break;
+	case HW_BREAKPOINT_RW:
+		hw->type = RV_DBTR_WP;
+		RV_DBTR_SET_MC6_STORE_BIT(hw->tdata1);
+		RV_DBTR_SET_MC6_LOAD_BIT(hw->tdata1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (attr->bp_len) {
+	case HW_BREAKPOINT_LEN_1:
+		hw->len = 1;
+		break;
+	case HW_BREAKPOINT_LEN_2:
+		hw->len = 2;
+		break;
+	case HW_BREAKPOINT_LEN_4:
+		hw->len = 4;
+		break;
+	case HW_BREAKPOINT_LEN_8:
+		hw->len = 8;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* any size */
+	hw->tdata1 = RV_DBTR_SET_MC6_SIZE(hw->tdata1, 0);
+	hw->tdata1 = RV_DBTR_SET_MC6_TYPE(hw->tdata1, RV_DBTR_TRIG_MCONTROL6);
+
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, DMODE);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, TIMING);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, SELECT);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, ACTION);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, CHAIN);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, MATCH);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, M);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, VS);
+	CLEAR_DBTR_BIT(hw->tdata1, MC6, VU);
+
+	SET_DBTR_BIT(hw->tdata1, MC6, S);
+	SET_DBTR_BIT(hw->tdata1, MC6, U);
+
+	return 0;
+}
+
+int hw_breakpoint_arch_parse(struct perf_event *bp,
+			     const struct perf_event_attr *attr,
+			     struct arch_hw_breakpoint *hw)
+{
+	int ret;
+
+	/* Breakpoint address */
+	hw->address = attr->bp_addr;
+	hw->tdata2 = attr->bp_addr;
+	hw->tdata3 = 0x0;
+
+	switch (dbtr_type) {
+	case RV_DBTR_TRIG_MCONTROL:
+		ret = rv_init_mcontrol_trigger(attr, hw);
+		break;
+	case RV_DBTR_TRIG_MCONTROL6:
+		ret = rv_init_mcontrol6_trigger(attr, hw);
+		break;
+	default:
+		pr_warn("unsupported trigger type\n");
+		ret = -EOPNOTSUPP;
+		break;
+	}
+
+	return ret;
+}
+
+/* Get effective address from epc load/store instruction */
+static unsigned long get_load_store_badaddr(struct pt_regs *regs, unsigned long *badaddr)
+{
+	int ret = -EINVAL;
+	unsigned long epc = regs->epc;
+	u32 insn;
+	int rs1, imm;
+	unsigned long rs1_val;
+
+	/* Read insnuction */
+	if (user_mode(regs)) {
+		if (__copy_from_user_inatomic(&insn, (void __user *)epc, sizeof(insn))) {
+			pr_err("Failed to read user insnuction at epc=0x%lx\n", epc);
+			return ret;
+		}
+	} else {
+		if (copy_from_kernel_nofault(&insn, (void *)epc, sizeof(insn))) {
+			pr_err("Failed to read kernel insnuction at epc=0x%lx\n", epc);
+			return ret;
+		}
+	}
+
+	static const unsigned int regs_offs[] = {
+		offsetof(struct pt_regs, epc),
+		offsetof(struct pt_regs, ra),
+		offsetof(struct pt_regs, sp),
+		offsetof(struct pt_regs, gp),
+		offsetof(struct pt_regs, tp),
+		offsetof(struct pt_regs, t0),
+		offsetof(struct pt_regs, t1),
+		offsetof(struct pt_regs, t2),
+		offsetof(struct pt_regs, s0),
+		offsetof(struct pt_regs, s1),
+		offsetof(struct pt_regs, a0),
+		offsetof(struct pt_regs, a1),
+		offsetof(struct pt_regs, a2),
+		offsetof(struct pt_regs, a3),
+		offsetof(struct pt_regs, a4),
+		offsetof(struct pt_regs, a5),
+		offsetof(struct pt_regs, a6),
+		offsetof(struct pt_regs, a7),
+		offsetof(struct pt_regs, s2),
+		offsetof(struct pt_regs, s3),
+		offsetof(struct pt_regs, s4),
+		offsetof(struct pt_regs, s5),
+		offsetof(struct pt_regs, s6),
+		offsetof(struct pt_regs, s7),
+		offsetof(struct pt_regs, s8),
+		offsetof(struct pt_regs, s9),
+		offsetof(struct pt_regs, s10),
+		offsetof(struct pt_regs, s11),
+		offsetof(struct pt_regs, t3),
+		offsetof(struct pt_regs, t4),
+		offsetof(struct pt_regs, t5),
+		offsetof(struct pt_regs, status),
+		offsetof(struct pt_regs, badaddr),
+		offsetof(struct pt_regs, cause),
+		offsetof(struct pt_regs, orig_a0),
+	};
+
+	/* Check if 32-bit insnuction */
+	if (INSN_LEN(insn) == 4) {
+		rs1 = RV_EXTRACT_RS1_REG(insn);
+
+		if ((insn & INSN_MASK_LB) == INSN_MATCH_LB ||
+		    (insn & INSN_MASK_LH) == INSN_MATCH_LH ||
+		    (insn & INSN_MASK_LW) == INSN_MATCH_LW ||
+		    (insn & INSN_MASK_LD) == INSN_MATCH_LD ||
+		    (insn & INSN_MASK_LBU) == INSN_MATCH_LBU ||
+		    (insn & INSN_MASK_LHU) == INSN_MATCH_LHU ||
+		    (insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
+			imm = IMM_I(insn);
+			rs1_val = regs_get_register(regs, regs_offs[rs1]);
+			*badaddr = rs1_val + imm;
+			ret = 0;
+		}
+
+		if ((insn & INSN_MASK_SB) == INSN_MATCH_SB ||
+		    (insn & INSN_MASK_SH) == INSN_MATCH_SH ||
+		    (insn & INSN_MASK_SW) == INSN_MATCH_SW ||
+		    (insn & INSN_MASK_SD) == INSN_MATCH_SD) {
+			imm = IMM_S(insn);
+			rs1_val = regs_get_register(regs, regs_offs[rs1]);
+			*badaddr = rs1_val + imm;
+			ret = 0;
+		}
+	} else {
+		/* C.LDSP */
+		if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP) {
+			imm = RVC_LDSP_IMM(insn);
+			*badaddr = regs->sp + imm;
+			ret = 0;
+		}
+
+		/* C.SDSP */
+		if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
+			imm = RVC_SDSP_IMM(insn);
+			*badaddr = regs->sp + imm;
+			ret = 0;
+		}
+
+		/* C.LWSP */
+		if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP) {
+			imm = RVC_LWSP_IMM(insn);
+			*badaddr = regs->sp + imm;
+			ret = 0;
+		}
+
+		/* C.SWSP */
+		if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
+			imm = RVC_SWSP_IMM(insn);
+			*badaddr = regs->sp + imm;
+			ret = 0;
+		}
+
+		rs1 = RVC_RS1S(insn);
+
+		/* C.LD, C.SD, C.FLD, C.SLD */
+		if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD ||
+		    (insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD ||
+		    (insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD ||
+		    (insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
+			imm = RVC_LD_IMM(insn);
+			rs1_val = regs_get_register(regs, regs_offs[rs1]);
+			*badaddr = rs1_val + imm;
+			ret = 0;
+		}
+
+		/* C.LW, C.SW, C.FLW, C.SLW */
+		if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW ||
+		    (insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW ||
+		    (insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW ||
+		    (insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
+			imm = RVC_LW_IMM(insn);
+			rs1_val = regs_get_register(regs, regs_offs[rs1]);
+			*badaddr = rs1_val + imm;
+			ret = 0;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * HW Breakpoint/watchpoint handler
+ */
+static int hw_breakpoint_handler(struct die_args *args)
+{
+	int ret = NOTIFY_DONE;
+	struct arch_hw_breakpoint *bp;
+	struct perf_event *event;
+	bool match;
+	unsigned long badaddr;
+	int i;
+
+	for (i = 0; i < dbtr_total_num; i++) {
+		event = this_cpu_read(pcpu_hw_bp_events[i]);
+		if (!event)
+			continue;
+
+		bp = counter_arch_bp(event);
+		switch (bp->type) {
+		/* Breakpoint */
+		case RV_DBTR_BP:
+			if (bp->address == args->regs->epc) {
+				perf_bp_event(event, args->regs);
+				ret = NOTIFY_STOP;
+			}
+			break;
+
+		/* Watchpoint */
+		case RV_DBTR_WP:
+			match = (bp->address == csr_read(CSR_STVAL));
+			if (!match) {
+				/* Hardware bug: When a breakpoint exception occurs
+				 * on an instruction load, or store, stval does not
+				 * contain the correct faulting virtual address.
+				 * So we can get the correct fault virtual address
+				 * by parsing the load/store instruction at the epc
+				 * address.
+				 */
+				if (!get_load_store_badaddr(args->regs, &badaddr))
+					match = (bp->address == badaddr);
+			}
+			if (match) {
+				perf_bp_event(event, args->regs);
+				ret = NOTIFY_STOP;
+			}
+			break;
+
+		default:
+			pr_warn("%s: Unknown type: %u\n", __func__, bp->type);
+			break;
+		}
+	}
+
+	return ret;
+}
+
+int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
+				    unsigned long val, void *data)
+{
+	if (val != DIE_DEBUG)
+		return NOTIFY_DONE;
+
+	return hw_breakpoint_handler(data);
+}
+
+/* atomic: counter->ctx->lock is held */
+int arch_install_hw_breakpoint(struct perf_event *event)
+{
+	struct arch_hw_breakpoint *bp = counter_arch_bp(event);
+	union sbi_dbtr_shmem_entry *shmem = this_cpu_ptr(sbi_dbtr_shmem);
+	struct sbi_dbtr_data_msg *xmit;
+	struct sbi_dbtr_id_msg *recv;
+	struct perf_event **slot;
+	unsigned long idx;
+	struct sbiret ret;
+	int err = 0;
+
+	raw_spin_lock_irqsave(this_cpu_ptr(&ecall_lock),
+			      *this_cpu_ptr(&ecall_lock_flags));
+
+	xmit = &shmem->data;
+	recv = &shmem->id;
+	xmit->tdata1 = cpu_to_le(bp->tdata1);
+	xmit->tdata2 = cpu_to_le(bp->tdata2);
+	xmit->tdata3 = cpu_to_le(bp->tdata3);
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_TRIG_INSTALL,
+			1, 0, 0, 0, 0, 0);
+
+	if (ret.error) {
+		pr_warn("%s: failed to install trigger\n", __func__);
+		err = -EIO;
+		goto done;
+	}
+
+	idx = le_to_cpu(recv->idx);
+	if (idx >= dbtr_total_num) {
+		pr_warn("%s: invalid trigger index %lu\n", __func__, idx);
+		err = -EINVAL;
+		goto done;
+	}
+
+	slot = this_cpu_ptr(&pcpu_hw_bp_events[idx]);
+	if (*slot) {
+		pr_warn("%s: slot %lu is in use\n", __func__, idx);
+		err = -EBUSY;
+		goto done;
+	}
+
+	pr_debug("Trigger %lu installed at index 0x%lx\n", bp->tdata2, idx);
+
+	/* Save the event - to be looked up in handler */
+	*slot = event;
+
+done:
+	raw_spin_unlock_irqrestore(this_cpu_ptr(&ecall_lock),
+				   *this_cpu_ptr(&ecall_lock_flags));
+	return err;
+}
+
+/* atomic: counter->ctx->lock is held */
+void arch_uninstall_hw_breakpoint(struct perf_event *event)
+{
+	struct sbiret ret;
+	int i;
+
+	for (i = 0; i < dbtr_total_num; i++) {
+		struct perf_event **slot = this_cpu_ptr(&pcpu_hw_bp_events[i]);
+
+		if (*slot == event) {
+			*slot = NULL;
+			break;
+		}
+	}
+
+	if (i == dbtr_total_num) {
+		pr_warn("%s: Breakpoint not installed.\n", __func__);
+		return;
+	}
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_TRIG_UNINSTALL,
+			i, 1, 0, 0, 0, 0);
+	if (ret.error)
+		pr_warn("%s: Failed to uninstall trigger %d.\n", __func__, i);
+}
+
+void arch_enable_hw_breakpoint(struct perf_event *event)
+{
+	struct sbiret ret;
+	int i;
+	struct perf_event **slot;
+
+	for (i = 0; i < dbtr_total_num; i++) {
+		slot = this_cpu_ptr(&pcpu_hw_bp_events[i]);
+
+		if (*slot == event)
+			break;
+	}
+
+	if (i == dbtr_total_num) {
+		pr_warn("%s: Breakpoint not installed.\n", __func__);
+		return;
+	}
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_TRIG_ENABLE,
+			i, 1, 0, 0, 0, 0);
+
+	if (ret.error) {
+		pr_warn("%s: Failed to install trigger %d\n", __func__, i);
+		return;
+	}
+}
+EXPORT_SYMBOL_GPL(arch_enable_hw_breakpoint);
+
+void arch_update_hw_breakpoint(struct perf_event *event)
+{
+	struct arch_hw_breakpoint *bp = counter_arch_bp(event);
+	union sbi_dbtr_shmem_entry *shmem = this_cpu_ptr(sbi_dbtr_shmem);
+	struct sbi_dbtr_data_msg *xmit;
+	struct perf_event **slot;
+	struct sbiret ret;
+	int i;
+
+	for (i = 0; i < dbtr_total_num; i++) {
+		slot = this_cpu_ptr(&pcpu_hw_bp_events[i]);
+
+		if (*slot == event)
+			break;
+	}
+
+	if (i == dbtr_total_num) {
+		pr_warn("%s: Breakpoint not installed.\n", __func__);
+		return;
+	}
+
+	raw_spin_lock_irqsave(this_cpu_ptr(&ecall_lock),
+			      *this_cpu_ptr(&ecall_lock_flags));
+
+	xmit = &shmem->data;
+	xmit->tdata1 = cpu_to_le(bp->tdata1);
+	xmit->tdata2 = cpu_to_le(bp->tdata2);
+	xmit->tdata3 = cpu_to_le(bp->tdata3);
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_TRIG_UPDATE,
+			i, 1, 0, 0, 0, 0);
+	if (ret.error)
+		pr_warn("%s: Failed to update trigger %d.\n", __func__, i);
+
+	raw_spin_unlock_irqrestore(this_cpu_ptr(&ecall_lock),
+				   *this_cpu_ptr(&ecall_lock_flags));
+}
+EXPORT_SYMBOL_GPL(arch_update_hw_breakpoint);
+
+void arch_disable_hw_breakpoint(struct perf_event *event)
+{
+	struct sbiret ret;
+	int i;
+
+	for (i = 0; i < dbtr_total_num; i++) {
+		struct perf_event **slot = this_cpu_ptr(&pcpu_hw_bp_events[i]);
+
+		if (*slot == event)
+			break;
+	}
+
+	if (i == dbtr_total_num) {
+		pr_warn("%s: Breakpoint not installed.\n", __func__);
+		return;
+	}
+
+	ret = sbi_ecall(SBI_EXT_DBTR, SBI_EXT_DBTR_TRIG_DISABLE,
+			i, 1, 0, 0, 0, 0);
+
+	if (ret.error) {
+		pr_warn("%s: Failed to uninstall trigger %d.\n", __func__, i);
+		return;
+	}
+}
+EXPORT_SYMBOL_GPL(arch_disable_hw_breakpoint);
+
+void hw_breakpoint_pmu_read(struct perf_event *bp)
+{
+	/* TODO */
+}
+
+void clear_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+	memset(tsk->thread.ptrace_bps, 0, sizeof(tsk->thread.ptrace_bps));
+}
+
+void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+	int i;
+	struct thread_struct *t = &tsk->thread;
+
+	for (i = 0; i < dbtr_total_num; i++) {
+		unregister_hw_breakpoint(t->ptrace_bps[i]);
+		t->ptrace_bps[i] = NULL;
+	}
+}
+
+static int __init arch_hw_breakpoint_init(void)
+{
+	unsigned int cpu;
+	int rc = 0;
+
+	for_each_possible_cpu(cpu)
+		raw_spin_lock_init(&per_cpu(ecall_lock, cpu));
+
+	if (!dbtr_init)
+		init_sbi_dbtr();
+
+	if (dbtr_total_num) {
+		pr_info("%s: total number of type %d triggers: %u\n",
+			__func__, dbtr_type, dbtr_total_num);
+	} else {
+		pr_info("%s: No hardware triggers available\n", __func__);
+		goto out;
+	}
+
+	/* Allocate per-cpu shared memory */
+	sbi_dbtr_shmem = __alloc_percpu(sizeof(*sbi_dbtr_shmem) * dbtr_total_num,
+					PAGE_SIZE);
+
+	if (!sbi_dbtr_shmem) {
+		pr_warn("%s: Failed to allocate shared memory.\n", __func__);
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	/* Hotplug handler to register/unregister shared memory with SBI */
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
+			       "riscv/hw_breakpoint:prepare",
+			       arch_smp_setup_sbi_shmem,
+			       arch_smp_teardown_sbi_shmem);
+
+	if (rc < 0) {
+		pr_warn("%s: Failed to setup CPU hotplug state\n", __func__);
+		free_percpu(sbi_dbtr_shmem);
+		return rc;
+	}
+ out:
+	return rc;
+}
+arch_initcall(arch_hw_breakpoint_init);
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 31a392993cb4..71402a96af0e 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -188,6 +188,9 @@ void flush_thread(void)
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
 		envcfg_update_bits(current, ENVCFG_PMM, ENVCFG_PMM_PMLEN_0);
 #endif
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+	flush_ptrace_hw_breakpoint(current);
+#endif
 }
 
 void arch_release_task_struct(struct task_struct *tsk)
@@ -259,6 +262,8 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 	if (has_vector() || has_xtheadvector())
 		riscv_v_thread_alloc(p);
 	p->thread.sp = (unsigned long)childregs; /* kernel sp */
+	if (IS_ENABLED(CONFIG_HAVE_HW_BREAKPOINT))
+		clear_ptrace_hw_breakpoint(p);
 	return 0;
 }
 
diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c
index 8e86305831ea..1f736dfd5a50 100644
--- a/arch/riscv/kernel/ptrace.c
+++ b/arch/riscv/kernel/ptrace.c
@@ -19,6 +19,7 @@
 #include <linux/regset.h>
 #include <linux/sched.h>
 #include <linux/sched/task_stack.h>
+#include <linux/hw_breakpoint.h>
 
 enum riscv_regset {
 	REGSET_X,
@@ -31,6 +32,9 @@ enum riscv_regset {
 #ifdef CONFIG_RISCV_ISA_SUPM
 	REGSET_TAGGED_ADDR_CTRL,
 #endif
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+	REGSET_HW_BREAK,
+#endif
 };
 
 static int riscv_gpr_get(struct task_struct *target,
@@ -118,7 +122,11 @@ static int riscv_vr_get(struct task_struct *target,
 	membuf_write(&to, &ptrace_vstate, sizeof(struct __riscv_v_regset_state));
 
 	/* Copy all the vector registers from vstate. */
+#ifdef CONFIG_SPACEMIT_HMP
+	return membuf_write(&to, vstate->datap, vstate->vlenb * 32);
+#else
 	return membuf_write(&to, vstate->datap, riscv_v_vsize);
+#endif
 }
 
 static int riscv_vr_set(struct task_struct *target,
@@ -149,8 +157,13 @@ static int riscv_vr_set(struct task_struct *target,
 
 	/* Copy all the vector registers. */
 	pos = 0;
+#ifdef CONFIG_SPACEMIT_HMP
+	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vstate->datap,
+				 0, vstate->vlenb * 32);
+#else
 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, vstate->datap,
 				 0, riscv_v_vsize);
+#endif
 	return ret;
 }
 #endif
@@ -184,6 +197,261 @@ static int tagged_addr_ctrl_set(struct task_struct *target,
 }
 #endif
 
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+static void ptrace_hbptriggered(struct perf_event *bp,
+				struct perf_sample_data *data,
+				struct pt_regs *regs)
+{
+	int i;
+	struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
+
+	for (i = 0; i < HW_BP_NUM_MAX; i++)
+		if (current->thread.ptrace_bps[i] == bp)
+			break;
+
+	force_sig_ptrace_errno_trap(i, (void __user *)bkpt->address);
+}
+
+static int ptrace_hbp_get_resource_info(unsigned int note_type, u64 *info)
+{
+	u8 num;
+	u64 reg = 0;
+
+	num = hw_breakpoint_slots(note_type);
+
+	*info = reg | num;
+	return 0;
+}
+
+static int ptrace_hbp_get_addr(unsigned int note_type,
+			       struct task_struct *tsk,
+			       unsigned long idx, u64 *addr)
+{
+	struct perf_event *bp;
+
+	if (idx >= HW_BP_NUM_MAX)
+		return -EINVAL;
+
+	bp = tsk->thread.ptrace_bps[idx];
+
+	*addr = bp ? counter_arch_bp(bp)->address : 0;
+
+	return 0;
+}
+
+static int ptrace_hbp_get_type(unsigned int note_type,
+			       struct task_struct *tsk,
+			       unsigned long idx, u64 *type)
+{
+	struct perf_event *bp;
+
+	if (idx >= HW_BP_NUM_MAX)
+		return -EINVAL;
+
+	bp = tsk->thread.ptrace_bps[idx];
+
+	*type = bp ? counter_arch_bp(bp)->type : 0;
+
+	return 0;
+}
+
+static int ptrace_hbp_get_len(unsigned int note_type,
+			      struct task_struct *tsk,
+			      unsigned long idx, u64 *len)
+{
+	struct perf_event *bp;
+
+	if (idx >= HW_BP_NUM_MAX)
+		return -EINVAL;
+
+	bp = tsk->thread.ptrace_bps[idx];
+
+	*len = bp ? counter_arch_bp(bp)->len : 0;
+
+	return 0;
+}
+
+#define PTRACE_HBP_ADDR_SZ	sizeof(u64)
+#define PTRACE_HBP_TYPE_SZ	sizeof(u64)
+#define PTRACE_HBP_SIZE_SZ	sizeof(u64)
+
+static int hw_break_get(struct task_struct *target,
+			const struct user_regset *regset,
+			struct membuf to)
+{
+	u64 info;
+	u64 addr, type, len;
+	int ret, idx = 0;
+	unsigned int note_type = regset->core_note_type;
+
+	/* Resource info */
+	ret = ptrace_hbp_get_resource_info(note_type, &info);
+	if (ret)
+		return ret;
+
+	membuf_write(&to, &info, sizeof(info));
+	/* (addr, type, len) registers */
+	while (to.left) {
+		ret = ptrace_hbp_get_addr(note_type, target, idx, &addr);
+		if (ret)
+			return ret;
+
+		ret = ptrace_hbp_get_type(note_type, target, idx, &type);
+		if (ret)
+			return ret;
+
+		ret = ptrace_hbp_get_len(note_type, target, idx, &len);
+		if (ret)
+			return ret;
+
+		membuf_store(&to, addr);
+		membuf_store(&to, type);
+		membuf_store(&to, len);
+		idx++;
+	}
+
+	return 0;
+}
+
+static inline int hw_break_empty(u64 addr, u64 type, u64 size)
+{
+	/* TODO: for now adjusted to current riscv-gdb behavior */
+	return (!addr && !size);
+}
+
+static int ptrace_hbp_set(struct task_struct *target, u64 addr,
+			  u64 type, u64 size, int idx)
+{
+	struct perf_event *bp;
+	struct perf_event_attr attr;
+	u32 bp_type;
+	u64 bp_len;
+
+	if (!hw_break_empty(addr, type, size)) {
+		/* bp size: gdb to kernel */
+		switch (size) {
+		case 1:
+			bp_len = HW_BREAKPOINT_LEN_1;
+			break;
+		case 2:
+			bp_len = HW_BREAKPOINT_LEN_2;
+			break;
+		case 4:
+			bp_len = HW_BREAKPOINT_LEN_4;
+			break;
+		case 8:
+			bp_len = HW_BREAKPOINT_LEN_8;
+			break;
+		default:
+			pr_warn("%s: unsupported size: %llu\n", __func__, size);
+			return -EINVAL;
+		}
+
+		/* bp type: gdb to kernel */
+		switch (type) {
+		case 0:
+			bp_type = HW_BREAKPOINT_X;
+			break;
+		case 1:
+			bp_type = HW_BREAKPOINT_R;
+			break;
+		case 2:
+			bp_type = HW_BREAKPOINT_W;
+			break;
+		case 3:
+			bp_type = HW_BREAKPOINT_RW;
+			break;
+		default:
+			pr_warn("%s: unsupported type: %llu\n", __func__, type);
+			return -EINVAL;
+		}
+	}
+
+	bp = target->thread.ptrace_bps[idx];
+	if (bp) {
+		attr = bp->attr;
+
+		if (hw_break_empty(addr, type, size)) {
+			attr.disabled = 1;
+		} else {
+			attr.bp_addr = addr;
+			attr.bp_type = bp_type;
+			attr.bp_len = bp_len;
+			attr.disabled = 0;
+		}
+
+		return modify_user_hw_breakpoint(bp, &attr);
+	}
+
+	if (hw_break_empty(addr, type, size))
+		return 0;
+
+	ptrace_breakpoint_init(&attr);
+	attr.bp_addr = addr;
+	attr.bp_type = bp_type;
+	attr.bp_len = bp_len;
+
+	bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered,
+					 NULL, target);
+	if (IS_ERR(bp))
+		return PTR_ERR(bp);
+
+	target->thread.ptrace_bps[idx] = bp;
+	return 0;
+}
+
+static int hw_break_set(struct task_struct *target,
+			const struct user_regset *regset,
+			unsigned int pos, unsigned int count,
+			const void *kbuf, const void __user *ubuf)
+{
+	int ret, idx = 0, offset, limit;
+	u64 addr;
+	u64 type;
+	u64 size;
+
+	/* Resource info */
+	offset = offsetof(struct __riscv_hwdebug_state, dbg_regs);
+	user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset);
+
+	/* (addr, type, len) registers */
+	limit = regset->n * regset->size;
+	while (count && offset < limit) {
+		if (count < PTRACE_HBP_ADDR_SZ)
+			return -EINVAL;
+		ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr,
+					 offset, offset + PTRACE_HBP_ADDR_SZ);
+		if (ret)
+			return ret;
+
+		offset += PTRACE_HBP_ADDR_SZ;
+
+		if (!count)
+			break;
+		ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &type,
+					 offset, offset + PTRACE_HBP_TYPE_SZ);
+		if (ret)
+			return ret;
+
+		offset += PTRACE_HBP_TYPE_SZ;
+
+		ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &size,
+					 offset, offset + PTRACE_HBP_SIZE_SZ);
+		if (ret)
+			return ret;
+
+		offset += PTRACE_HBP_SIZE_SZ;
+
+		ret = ptrace_hbp_set(target, addr, type, size, idx);
+		if (ret)
+			return ret;
+
+		idx++;
+	}
+
+	return 0;
+}
+#endif
 static const struct user_regset riscv_user_regset[] = {
 	[REGSET_X] = {
 		USER_REGSET_NOTE_TYPE(PRSTATUS),
@@ -224,6 +492,16 @@ static const struct user_regset riscv_user_regset[] = {
 		.set = tagged_addr_ctrl_set,
 	},
 #endif
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+	[REGSET_HW_BREAK] = {
+		USER_REGSET_NOTE_TYPE(RISCV_HW_BREAK),
+		.n = sizeof(struct __riscv_hwdebug_state) / sizeof(u32),
+		.size = sizeof(u32),
+		.align = sizeof(u32),
+		.regset_get = hw_break_get,
+		.set = hw_break_set,
+	},
+#endif
 };
 
 static const struct user_regset_view riscv_user_native_view = {
diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c
index c443337056ab..5858fd24eced 100644
--- a/arch/riscv/kernel/sbi.c
+++ b/arch/riscv/kernel/sbi.c
@@ -76,6 +76,15 @@ int sbi_console_getchar(void)
 }
 EXPORT_SYMBOL(sbi_console_getchar);
 
+#if defined(CONFIG_SOC_SPACEMIT)
+void sbi_flush_local_dcache_all(void)
+{
+       sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_FLUSH_CACHE_ALL, 0,
+                       0, 0, 0, 0, 0);
+}
+EXPORT_SYMBOL(sbi_flush_local_dcache_all);
+#endif
+
 /**
  * sbi_shutdown() - Remove all the harts from executing supervisor code.
  *
diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c
index 08378fea3a11..0315d912f871 100644
--- a/arch/riscv/kernel/signal.c
+++ b/arch/riscv/kernel/signal.c
@@ -96,7 +96,11 @@ static long save_v_state(struct pt_regs *regs, void __user **sc_vec)
 	/* Copy the pointer datap itself. */
 	err |= __put_user((__force void *)datap, &state->v_state.datap);
 	/* Copy the whole vector content to user space datap. */
+#ifdef CONFIG_SPACEMIT_HMP
+	err |= __copy_to_user(datap, current->thread.vstate.datap, current->thread.vstate.vlenb * 32);
+#else
 	err |= __copy_to_user(datap, current->thread.vstate.datap, riscv_v_vsize);
+#endif
 	/* Copy magic to the user space after saving  all vector conetext */
 	err |= __put_user(RISCV_V_MAGIC, &hdr->magic);
 	err |= __put_user(riscv_v_sc_size, &hdr->size);
@@ -140,7 +144,11 @@ static long __restore_v_state(struct pt_regs *regs, void __user *sc_vec)
 	 * Copy the whole vector content from user space datap. Use
 	 * copy_from_user to prevent information leak.
 	 */
+#ifdef CONFIG_SPACEMIT_HMP
+	return copy_from_user(current->thread.vstate.datap, datap, current->thread.vstate.vlenb * 32);
+#else
 	return copy_from_user(current->thread.vstate.datap, datap, riscv_v_vsize);
+#endif
 }
 #else
 #define save_v_state(task, regs) (0)
diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c
index 24b3f57d467f..79b41533cc0e 100644
--- a/arch/riscv/kernel/suspend.c
+++ b/arch/riscv/kernel/suspend.c
@@ -11,13 +11,26 @@
 #include <asm/csr.h>
 #include <asm/sbi.h>
 #include <asm/suspend.h>
+#include <asm/switch_to.h>
 
 void suspend_save_csrs(struct suspend_context *context)
 {
+	if (has_scontext())
+		context->scontext = csr_read(CSR_SCONTEXT);
+
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_XLINUXENVCFG))
 		context->envcfg = csr_read(CSR_ENVCFG);
 	context->tvec = csr_read(CSR_TVEC);
 	context->ie = csr_read(CSR_IE);
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) {
+		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_h)) {
+#if __riscv_xlen < 64
+			context->hstateen0h = csr_read(CSR_HSTATEEN0H);
+#endif
+			context->hstateen0 = csr_read(CSR_HSTATEEN0);
+		}
+		context->sstateen0 = csr_read(CSR_SSTATEEN0);
+	}
 
 	/*
 	 * No need to save/restore IP CSR (i.e. MIP or SIP) because:
@@ -43,11 +56,23 @@ void suspend_save_csrs(struct suspend_context *context)
 
 void suspend_restore_csrs(struct suspend_context *context)
 {
+	if (has_scontext())
+		csr_write(CSR_SCONTEXT, context->scontext);
+
 	csr_write(CSR_SCRATCH, 0);
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_XLINUXENVCFG))
 		csr_write(CSR_ENVCFG, context->envcfg);
 	csr_write(CSR_TVEC, context->tvec);
 	csr_write(CSR_IE, context->ie);
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) {
+		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_h)) {
+#if __riscv_xlen < 64
+			csr_write(CSR_HSTATEEN0H, context->hstateen0h);
+#endif
+			csr_write(CSR_HSTATEEN0, context->hstateen0);
+		}
+		csr_write(CSR_SSTATEEN0, context->sstateen0);
+	}
 
 #ifdef CONFIG_MMU
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SSTC)) {
@@ -114,6 +139,11 @@ static int sbi_system_suspend(unsigned long sleep_type,
 {
 	struct sbiret ret;
 
+#if defined(CONFIG_SOC_SPACEMIT)
+        /* flush the local cache */
+        sbi_flush_local_dcache_all();
+#endif
+
 	ret = sbi_ecall(SBI_EXT_SUSP, SBI_EXT_SUSP_SYSTEM_SUSPEND,
 			sleep_type, resume_addr, opaque, 0, 0, 0);
 	if (ret.error)
@@ -152,6 +182,11 @@ static int sbi_suspend_finisher(unsigned long suspend_type,
 {
 	struct sbiret ret;
 
+#if defined(CONFIG_SOC_SPACEMIT)
+        /* flush the local cache */
+        sbi_flush_local_dcache_all();
+#endif
+
 	ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_SUSPEND,
 			suspend_type, resume_addr, opaque, 0, 0, 0);
 
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 80230de167de..d6e37f06dc29 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -288,6 +288,12 @@ void handle_break(struct pt_regs *regs)
 	if (probe_breakpoint_handler(regs))
 		return;
 
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+	if (notify_die(DIE_DEBUG, "EBREAK", regs, 0, regs->cause, SIGTRAP)
+	    == NOTIFY_STOP)
+		return;
+#endif
+
 	current->thread.bad_cause = regs->cause;
 
 	if (user_mode(regs))
diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c
index 901e67adf576..c82b3382a31a 100644
--- a/arch/riscv/kernel/vector.c
+++ b/arch/riscv/kernel/vector.c
@@ -31,6 +31,14 @@ EXPORT_SYMBOL_GPL(riscv_v_vsize);
 
 int riscv_v_setup_vsize(void)
 {
+#ifdef CONFIG_SPACEMIT_HMP
+	/*
+	 * The vector context size of X100 is 32*32=1024,
+	 * and the vector context size of A100 is 128*32=4096,
+	 * so, config to 4096, it'll be ok for both X100 and A100.
+	*/
+	riscv_v_vsize = 4096;
+#else
 	unsigned long this_vsize;
 
 	/*
@@ -53,10 +61,12 @@ int riscv_v_setup_vsize(void)
 		return 0;
 	}
 
+	/* the different vsize is allowed on spacemit hmp architecture */
 	if (riscv_v_vsize != this_vsize) {
 		WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
 		return -EOPNOTSUPP;
 	}
+#endif
 
 	return 0;
 }
diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig
index c50328212917..9e25e0f47662 100644
--- a/arch/riscv/kvm/Kconfig
+++ b/arch/riscv/kvm/Kconfig
@@ -39,4 +39,14 @@ config KVM
 
 	  If unsure, say N.
 
+config KVM_RISCV_VTVM
+	bool "Trap VS satp/TLB management with HSTATUS.VTVM"
+	depends on KVM && SOC_SPACEMIT_K3
+	default y
+	help
+	  Trap VS-mode page-table and TLB-management instructions through
+	  HSTATUS.VTVM and emulate them in HS-mode.
+
+	  If unsure, say Y.
+
 endif # VIRTUALIZATION
diff --git a/arch/riscv/kvm/Makefile b/arch/riscv/kvm/Makefile
index 07197395750e..14b2b1387aef 100644
--- a/arch/riscv/kvm/Makefile
+++ b/arch/riscv/kvm/Makefile
@@ -20,6 +20,7 @@ kvm-y += mmu.o
 kvm-y += nacl.o
 kvm-y += tlb.o
 kvm-y += vcpu.o
+kvm-y += vcpu_debug.o
 kvm-y += vcpu_exit.o
 kvm-y += vcpu_fp.o
 kvm-y += vcpu_insn.o
diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c
index 77dc1655b442..2ee839953a30 100644
--- a/arch/riscv/kvm/main.c
+++ b/arch/riscv/kvm/main.c
@@ -14,6 +14,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_nacl.h>
 #include <asm/sbi.h>
+#include <linux/soc/spacemit/spacemit-hmp.h>
 
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
@@ -25,6 +26,15 @@ int kvm_arch_enable_virtualization_cpu(void)
 {
 	int rc;
 
+#ifdef CONFIG_SPACEMIT_HMP
+	struct cpumask	cpumask;
+	unsigned int cpu = smp_processor_id();
+
+	hmp_get_cpumask(&cpumask, HMP_REGULAR);
+	if (!cpumask_test_cpu(cpu, &cpumask))
+		return 0;
+#endif
+
 	rc = kvm_riscv_nacl_enable();
 	if (rc)
 		return rc;
@@ -44,6 +54,15 @@ int kvm_arch_enable_virtualization_cpu(void)
 
 void kvm_arch_disable_virtualization_cpu(void)
 {
+#ifdef CONFIG_SPACEMIT_HMP
+	struct cpumask	cpumask;
+	unsigned int cpu = smp_processor_id();
+
+	hmp_get_cpumask(&cpumask, HMP_REGULAR);
+	if (!cpumask_test_cpu(cpu, &cpumask))
+		return;
+#endif
+
 	kvm_riscv_aia_disable();
 
 	/*
@@ -168,12 +187,16 @@ static int __init riscv_kvm_init(void)
 		return rc;
 	}
 
+	kvm_riscv_debug_init();
+
 	return 0;
 }
 module_init(riscv_kvm_init);
 
 static void __exit riscv_kvm_exit(void)
 {
+	kvm_riscv_debug_exit();
+
 	kvm_exit();
 
 	kvm_riscv_teardown();
diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c
index 3c5a70a2b927..0bb4f792b618 100644
--- a/arch/riscv/kvm/tlb.c
+++ b/arch/riscv/kvm/tlb.c
@@ -90,7 +90,8 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid,
 		return;
 	}
 
-	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
+	hgatp = csr_swap(CSR_HGATP, (csr_read(CSR_HGATP) & ~HGATP_VMID) |
+				(vmid << HGATP_VMID_SHIFT));
 
 	if (has_svinval()) {
 		asm volatile (SFENCE_W_INVAL() ::: "memory");
@@ -112,7 +113,8 @@ void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid,
 {
 	unsigned long hgatp;
 
-	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
+	hgatp = csr_swap(CSR_HGATP, (csr_read(CSR_HGATP) & ~HGATP_VMID) |
+				(vmid << HGATP_VMID_SHIFT));
 
 	asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory");
 
@@ -130,7 +132,8 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
 		return;
 	}
 
-	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
+	hgatp = csr_swap(CSR_HGATP, (csr_read(CSR_HGATP) & ~HGATP_VMID) |
+				(vmid << HGATP_VMID_SHIFT));
 
 	if (has_svinval()) {
 		asm volatile (SFENCE_W_INVAL() ::: "memory");
@@ -151,7 +154,8 @@ void kvm_riscv_local_hfence_vvma_all(unsigned long vmid)
 {
 	unsigned long hgatp;
 
-	hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT);
+	hgatp = csr_swap(CSR_HGATP, (csr_read(CSR_HGATP) & ~HGATP_VMID) |
+				(vmid << HGATP_VMID_SHIFT));
 
 	asm volatile(HFENCE_VVMA(zero, zero) : : : "memory");
 
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 5ce35aba6069..ae4457b488a1 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -20,6 +20,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_nacl.h>
 #include <asm/kvm_vcpu_vector.h>
+#include <asm/switch_to.h>
 
 #define CREATE_TRACE_POINTS
 #include "trace.h"
@@ -74,6 +75,9 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu,
 	cntx->hstatus |= HSTATUS_VTW;
 	cntx->hstatus |= HSTATUS_SPVP;
 	cntx->hstatus |= HSTATUS_SPV;
+#ifdef CONFIG_KVM_RISCV_VTVM
+	cntx->hstatus |= HSTATUS_VTVM;
+#endif
 }
 
 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)
@@ -569,6 +573,9 @@ static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu)
 					  SMSTATEEN0_AIA_ISEL;
 		if (riscv_isa_extension_available(isa, SMSTATEEN))
 			cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0;
+
+		if (has_scontext())
+			cfg->hstateen0 |= SMSTATEEN0_HSCONTEXT;
 	}
 
 	if (vcpu->guest_debug)
@@ -758,6 +765,8 @@ static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *
 	    (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0))
 		vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0,
 						     smcsr->sstateen0);
+
+	kvm_riscv_debug_vcpu_swap_in_guest_context(vcpu);
 }
 
 static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu)
@@ -772,6 +781,8 @@ static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *v
 	    (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0))
 		smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0,
 					    vcpu->arch.host_sstateen0);
+
+	kvm_riscv_debug_vcpu_swap_in_host_context(vcpu);
 }
 
 /*
diff --git a/arch/riscv/kvm/vcpu_debug.c b/arch/riscv/kvm/vcpu_debug.c
new file mode 100644
index 000000000000..5081c272f01d
--- /dev/null
+++ b/arch/riscv/kvm/vcpu_debug.c
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2024 SiFive
+ */
+
+#include <linux/kvm_host.h>
+#include <asm/switch_to.h>
+
+DEFINE_SPINLOCK(hcontext_lock);
+unsigned long *hcontext_bitmap;
+unsigned long hcontext_bitmap_len;
+
+static __always_inline bool has_hcontext(void)
+{
+	return static_branch_likely(&use_hcontext);
+}
+
+void kvm_riscv_debug_init(void)
+{
+	/*
+	 * As from riscv-debug-spec, Chapter 5.7.9:
+	 * If the H extension is implemented, it’s recommended to
+	 * implement no more than 7 bits on RV32 and 14 on RV64.
+	 * Allocating bit array according to spec size.
+	 */
+#if __riscv_xlen > 32
+	unsigned long tmp = atomic_long_read(&hcontext_id_share) & GENMASK(13, 0);
+#else
+	unsigned long tmp = atomic_long_read(&hcontext_id_share) & GENMASK(6, 0);
+#endif
+	if (has_hcontext()) {
+		while (tmp) {
+			kvm_info("hcontext: try to allocate 0x%lx-bit array\n", tmp);
+			hcontext_bitmap_len = tmp + 1;
+			hcontext_bitmap = bitmap_zalloc(tmp, 0);
+			if (hcontext_bitmap)
+				break;
+			tmp = tmp >> 1;
+		}
+
+		if (tmp == 0) {
+			/* We can't allocate any space for hcontext bitmap */
+			static_branch_disable(&use_hcontext);
+		} else {
+			/* ID 0 is hypervisor */
+			set_bit(0, hcontext_bitmap);
+		}
+	}
+}
+
+void kvm_riscv_debug_exit(void)
+{
+	if (has_hcontext()) {
+		static_branch_disable(&use_hcontext);
+		kfree(hcontext_bitmap);
+	}
+}
+
+void kvm_riscv_debug_get_hcontext_id(struct kvm *kvm)
+{
+	if (has_hcontext()) {
+		unsigned long free_id;
+
+		spin_lock(&hcontext_lock);
+		free_id = find_first_zero_bit(hcontext_bitmap, hcontext_bitmap_len);
+
+		/* share the maximum ID when we run out of the hcontext ID */
+		if (free_id <= hcontext_bitmap_len)
+			set_bit(free_id, hcontext_bitmap);
+		else
+			free_id -= 1;
+
+		kvm->arch.hcontext = free_id;
+		spin_unlock(&hcontext_lock);
+	}
+}
+
+void kvm_riscv_debug_return_hcontext_id(struct kvm *kvm)
+{
+	if (has_hcontext()) {
+		spin_lock(&hcontext_lock);
+		clear_bit(kvm->arch.hcontext, hcontext_bitmap);
+		spin_unlock(&hcontext_lock);
+	}
+}
+
+void kvm_riscv_debug_vcpu_swap_in_guest_context(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu_sdtrig_csr *csr = &vcpu->arch.sdtrig_csr;
+	unsigned long hcontext = vcpu->kvm->arch.hcontext;
+
+	if (has_hcontext())
+		csr_write(CSR_HCONTEXT, hcontext);
+	if (has_scontext())
+		vcpu->arch.host_scontext = csr_swap(CSR_SCONTEXT, csr->scontext);
+}
+
+void kvm_riscv_debug_vcpu_swap_in_host_context(struct kvm_vcpu *vcpu)
+{
+	struct kvm_vcpu_sdtrig_csr *csr = &vcpu->arch.sdtrig_csr;
+
+	/* Hypervisor uses the hcontext ID 0 */
+	if (has_hcontext())
+		csr_write(CSR_HCONTEXT, 0);
+	if (has_scontext())
+		csr->scontext = csr_swap(CSR_SCONTEXT, vcpu->arch.host_scontext);
+}
diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c
index 4d89b94128ae..28f9d6bcc0df 100644
--- a/arch/riscv/kvm/vcpu_insn.c
+++ b/arch/riscv/kvm/vcpu_insn.c
@@ -8,7 +8,9 @@
 #include <linux/kvm_host.h>
 
 #include <asm/cpufeature.h>
+#include <asm/insn-def.h>
 #include <asm/insn.h>
+#include <asm/pgtable.h>
 
 struct insn_func {
 	unsigned long mask;
@@ -87,6 +89,74 @@ static int wrs_insn(struct kvm_vcpu *vcpu, struct kvm_run *run, ulong insn)
 	return KVM_INSN_CONTINUE_NEXT_SEPC;
 }
 
+#ifdef CONFIG_KVM_RISCV_VTVM
+static int sfence_vma_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
+			   ulong insn)
+{
+	unsigned long vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
+	unsigned long gva = GET_RS1(insn, &vcpu->arch.guest_context);
+	unsigned long asid = GET_RS2(insn, &vcpu->arch.guest_context);
+	unsigned int rs1_num = (insn >> SH_RS1) & MASK_RX;
+	unsigned int rs2_num = (insn >> SH_RS2) & MASK_RX;
+
+	asid &= SATP_ASID_MASK;
+
+	if (!rs1_num && !rs2_num) {
+		kvm_riscv_local_hfence_vvma_all(vmid);
+		kvm_riscv_local_hfence_vvma_all(vmid);
+	}
+	else if (!rs1_num) {
+		kvm_riscv_local_hfence_vvma_asid_all(vmid, asid);
+		kvm_riscv_local_hfence_vvma_asid_all(vmid, asid);
+	}
+	else if (!rs2_num) {
+		kvm_riscv_local_hfence_vvma_gva(vmid, gva, PAGE_SIZE,
+						PAGE_SHIFT);
+		kvm_riscv_local_hfence_vvma_gva(vmid, gva, PAGE_SIZE,
+						PAGE_SHIFT);
+	}
+	else {
+		kvm_riscv_local_hfence_vvma_asid_gva(vmid, asid, gva,
+						     PAGE_SIZE, PAGE_SHIFT);
+		kvm_riscv_local_hfence_vvma_asid_gva(vmid, asid, gva,
+						     PAGE_SIZE, PAGE_SHIFT);
+	}
+
+	return KVM_INSN_CONTINUE_NEXT_SEPC;
+}
+
+static int sinval_vma_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
+			   ulong insn)
+{
+	if (!riscv_isa_extension_available(vcpu->arch.isa, SVINVAL))
+		return KVM_INSN_ILLEGAL_TRAP;
+
+	return sfence_vma_insn(vcpu, run, insn);
+}
+
+static int sfence_w_inval_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
+			       ulong insn)
+{
+	if (!riscv_isa_extension_available(vcpu->arch.isa, SVINVAL))
+		return KVM_INSN_ILLEGAL_TRAP;
+
+	asm volatile (SFENCE_W_INVAL() ::: "memory");
+
+	return KVM_INSN_CONTINUE_NEXT_SEPC;
+}
+
+static int sfence_inval_ir_insn(struct kvm_vcpu *vcpu, struct kvm_run *run,
+				ulong insn)
+{
+	if (!riscv_isa_extension_available(vcpu->arch.isa, SVINVAL))
+		return KVM_INSN_ILLEGAL_TRAP;
+
+	asm volatile (SFENCE_INVAL_IR() ::: "memory");
+
+	return KVM_INSN_CONTINUE_NEXT_SEPC;
+}
+#endif
+
 struct csr_func {
 	unsigned int base;
 	unsigned int count;
@@ -109,9 +179,39 @@ static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num,
 	return KVM_INSN_EXIT_TO_USER_SPACE;
 }
 
+#ifdef CONFIG_KVM_RISCV_VTVM
+static int satp_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num,
+			unsigned long *val, unsigned long new_val,
+			unsigned long wr_mask)
+{
+	unsigned long old_vsatp, next_vsatp, next_mode;
+
+	old_vsatp = csr_read(CSR_VSATP);
+	vcpu->arch.guest_csr.vsatp = old_vsatp;
+	if (val)
+		*val = old_vsatp;
+
+	if (!wr_mask)
+		return KVM_INSN_CONTINUE_NEXT_SEPC;
+
+	next_vsatp = (old_vsatp & ~wr_mask) | (new_val & wr_mask);
+	next_mode = next_vsatp & SATP_MODE_MASK;
+	if (next_mode && next_mode != (unsigned long)satp_mode)
+		return KVM_INSN_CONTINUE_NEXT_SEPC;
+
+	csr_write(CSR_VSATP, next_vsatp);
+	vcpu->arch.guest_csr.vsatp = csr_read(CSR_VSATP);
+
+	return KVM_INSN_CONTINUE_NEXT_SEPC;
+}
+#endif
+
 static const struct csr_func csr_funcs[] = {
 	KVM_RISCV_VCPU_AIA_CSR_FUNCS
 	KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS
+#ifdef CONFIG_KVM_RISCV_VTVM
+	{ .base = CSR_SATP, .count = 1, .func = satp_csr_rmw },
+#endif
 	{ .base = CSR_SEED, .count = 1, .func = seed_csr_rmw },
 };
 
@@ -257,6 +357,28 @@ static const struct insn_func system_opcode_funcs[] = {
 		.match = INSN_MATCH_CSRRCI,
 		.func  = csr_insn,
 	},
+#ifdef CONFIG_KVM_RISCV_VTVM
+	{
+		.mask  = RV_INSN_MASK_SFENCE_VMA,
+		.match = RV_INSN_MATCH_SFENCE_VMA,
+		.func  = sfence_vma_insn,
+	},
+	{
+		.mask  = RV_INSN_MASK_SINVAL_VMA,
+		.match = RV_INSN_MATCH_SINVAL_VMA,
+		.func  = sinval_vma_insn,
+	},
+	{
+		.mask  = RV_INSN_MASK_SFENCE_W_INVAL,
+		.match = RV_INSN_MATCH_SFENCE_W_INVAL,
+		.func  = sfence_w_inval_insn,
+	},
+	{
+		.mask  = RV_INSN_MASK_SFENCE_INVAL_IR,
+		.match = RV_INSN_MATCH_SFENCE_INVAL_IR,
+		.func  = sfence_inval_ir_insn,
+	},
+#endif
 	{
 		.mask  = INSN_MASK_WFI,
 		.match = INSN_MATCH_WFI,
diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c
index 865dae903aa0..ca1dbc5b7365 100644
--- a/arch/riscv/kvm/vcpu_onereg.c
+++ b/arch/riscv/kvm/vcpu_onereg.c
@@ -35,6 +35,7 @@ static const unsigned long kvm_isa_ext_arr[] = {
 	[KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
 	[KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v,
 	/* Multi letter extensions (alphabetically sorted) */
+	KVM_ISA_EXT_ARR(SDTRIG),
 	KVM_ISA_EXT_ARR(SMNPM),
 	KVM_ISA_EXT_ARR(SMSTATEEN),
 	KVM_ISA_EXT_ARR(SSAIA),
@@ -567,6 +568,34 @@ static int kvm_riscv_vcpu_smstateen_get_csr(struct kvm_vcpu *vcpu,
 	return 0;
 }
 
+static inline int kvm_riscv_vcpu_sdtrig_set_csr(struct kvm_vcpu *vcpu,
+						unsigned long reg_num,
+						unsigned long reg_val)
+{
+	struct kvm_vcpu_sdtrig_csr *csr = &vcpu->arch.sdtrig_csr;
+
+	if (reg_num >= sizeof(struct kvm_riscv_sdtrig_csr) /
+		sizeof(unsigned long))
+		return -EINVAL;
+
+	((unsigned long *)csr)[reg_num] = reg_val;
+	return 0;
+}
+
+static int kvm_riscv_vcpu_sdtrig_get_csr(struct kvm_vcpu *vcpu,
+					 unsigned long reg_num,
+					 unsigned long *out_val)
+{
+	struct kvm_vcpu_sdtrig_csr *csr = &vcpu->arch.sdtrig_csr;
+
+	if (reg_num >= sizeof(struct kvm_riscv_sdtrig_csr) /
+		sizeof(unsigned long))
+		return -EINVAL;
+
+	*out_val = ((unsigned long *)csr)[reg_num];
+	return 0;
+}
+
 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu,
 				      const struct kvm_one_reg *reg)
 {
@@ -596,6 +625,11 @@ static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu,
 			rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num,
 							      &reg_val);
 		break;
+	case KVM_REG_RISCV_CSR_SDTRIG:
+		rc = -EINVAL;
+		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SDTRIG))
+			rc = kvm_riscv_vcpu_sdtrig_get_csr(vcpu, reg_num, &reg_val);
+		break;
 	default:
 		rc = -ENOENT;
 		break;
@@ -641,6 +675,11 @@ static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu,
 			rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num,
 							      reg_val);
 		break;
+	case KVM_REG_RISCV_CSR_SDTRIG:
+		rc = -EINVAL;
+		if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SDTRIG))
+			rc = kvm_riscv_vcpu_sdtrig_set_csr(vcpu, reg_num, reg_val);
+		break;
 	default:
 		rc = -ENOENT;
 		break;
@@ -896,6 +935,8 @@ static inline unsigned long num_csr_regs(const struct kvm_vcpu *vcpu)
 		n += sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long);
 	if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN))
 		n += sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long);
+	if (riscv_isa_extension_available(vcpu->arch.isa, SDTRIG))
+		n += sizeof(struct kvm_riscv_sdtrig_csr) / sizeof(unsigned long);
 
 	return n;
 }
@@ -904,7 +945,7 @@ static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu,
 				u64 __user *uindices)
 {
 	int n1 = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long);
-	int n2 = 0, n3 = 0;
+	int n2 = 0, n3 = 0, n4 = 0;
 
 	/* copy general csr regs */
 	for (int i = 0; i < n1; i++) {
@@ -956,7 +997,25 @@ static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu,
 		}
 	}
 
-	return n1 + n2 + n3;
+	/* copy Sdtrig csr regs */
+	if (riscv_isa_extension_available(vcpu->arch.isa, SDTRIG)) {
+		n4 = sizeof(struct kvm_riscv_sdtrig_csr) / sizeof(unsigned long);
+
+		for (int i = 0; i < n4; i++) {
+			u64 size = IS_ENABLED(CONFIG_32BIT) ?
+				   KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
+			u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR |
+					  KVM_REG_RISCV_CSR_SDTRIG | i;
+
+			if (uindices) {
+				if (put_user(reg, uindices))
+					return -EFAULT;
+				uindices++;
+			}
+		}
+	}
+
+	return n1 + n2 + n3 + n4;
 }
 
 static inline unsigned long num_timer_regs(void)
diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c
index 66d91ae6e9b2..0e08a71bd454 100644
--- a/arch/riscv/kvm/vm.c
+++ b/arch/riscv/kvm/vm.c
@@ -46,6 +46,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
 	kvm_riscv_guest_timer_init(kvm);
 
+	kvm_riscv_debug_get_hcontext_id(kvm);
+
 	return 0;
 }
 
@@ -54,6 +56,8 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 	kvm_destroy_vcpus(kvm);
 
 	kvm_riscv_aia_destroy_vm(kvm);
+
+	kvm_riscv_debug_return_hcontext_id(kvm);
 }
 
 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irql,
diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c
index abb1c2bf2542..78bf09bbb06b 100644
--- a/arch/riscv/kvm/vmid.c
+++ b/arch/riscv/kvm/vmid.c
@@ -17,6 +17,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_tlb.h>
 #include <asm/kvm_vmid.h>
+#include <linux/soc/spacemit/spacemit-hmp.h>
 
 static unsigned long vmid_version = 1;
 static unsigned long vmid_next;
@@ -106,8 +107,14 @@ void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)
 		 * running, we force VM exits on all host CPUs using IPI and
 		 * flush all Guest TLBs.
 		 */
-		on_each_cpu_mask(cpu_online_mask, __local_hfence_gvma_all,
-				 NULL, 1);
+#ifndef CONFIG_SPACEMIT_HMP
+		on_each_cpu_mask(cpu_online_mask, __local_hfence_gvma_all, NULL, 1);
+#else
+		struct cpumask	cpumask;
+		hmp_get_cpumask(&cpumask, HMP_REGULAR);
+		cpumask_and(&cpumask, &cpumask, cpu_online_mask);
+		on_each_cpu_mask(&cpumask, __local_hfence_gvma_all, NULL, 1);
+#endif
 	}
 
 	vmid->vmid = vmid_next;
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index bbc031124974..bac8e899f1a7 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -11,6 +11,7 @@ endif
 lib-y			+= csum.o
 ifeq ($(CONFIG_MMU), y)
 lib-$(CONFIG_RISCV_ISA_V)	+= uaccess_vector.o
+lib-$(CONFIG_RISCV_ISA_V)	+= uaccess_cache_check.o
 endif
 lib-$(CONFIG_MMU)	+= uaccess.o
 lib-$(CONFIG_64BIT)	+= tishift.o
diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S
index 4efea1b3326c..69a2d6f087b5 100644
--- a/arch/riscv/lib/uaccess.S
+++ b/arch/riscv/lib/uaccess.S
@@ -17,8 +17,42 @@ SYM_FUNC_START(__asm_copy_to_user)
 	ALTERNATIVE("j fallback_scalar_usercopy", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V)
 	REG_L	t0, riscv_v_usercopy_threshold
 	bltu	a2, t0, fallback_scalar_usercopy
+
+	/* Check if both src and dst buffers are cacheable */
+	/* Save caller-saved registers that we need to preserve */
+	addi	sp, sp, -32
+	REG_S	ra, 0(sp)
+	REG_S	a0, 8(sp)	/* dst */
+	REG_S	a1, 16(sp)	/* src */
+	REG_S	a2, 24(sp)	/* len */
+
+	/* Check dst buffer (a0) */
+	call	is_cacheable_safe
+	beqz	a0, .Lnon_cacheable_1	/* if dst is non-cacheable, skip vector */
+
+	/* Check src buffer (a1 has original src) */
+	REG_L	a0, 16(sp)	/* load src to a0 */
+	call	is_cacheable_safe
+	beqz	a0, .Lnon_cacheable_1	/* if src is non-cacheable, skip vector */
+
+	/* Both buffers are cacheable - restore registers and use vector */
+	REG_L	ra, 0(sp)
+	REG_L	a0, 8(sp)
+	REG_L	a1, 16(sp)
+	REG_L	a2, 24(sp)
+	addi	sp, sp, 32
 	li	a3, 1
 	tail 	enter_vector_usercopy
+
+.Lnon_cacheable_1:
+	/* Non-cacheable buffer detected - restore and use scalar copy */
+	REG_L	ra, 0(sp)
+	REG_L	a0, 8(sp)
+	REG_L	a1, 16(sp)
+	REG_L	a2, 24(sp)
+	addi	sp, sp, 32
+	/* Fall through to scalar copy */
+	tail	fallback_scalar_usercopy
 #endif
 SYM_FUNC_END(__asm_copy_to_user)
 EXPORT_SYMBOL(__asm_copy_to_user)
@@ -45,8 +79,42 @@ SYM_FUNC_START(__asm_copy_to_user_sum_enabled)
 	ALTERNATIVE("j fallback_scalar_usercopy_sum_enabled", "nop", 0, RISCV_ISA_EXT_ZVE32X, CONFIG_RISCV_ISA_V)
 	REG_L	t0, riscv_v_usercopy_threshold
 	bltu	a2, t0, fallback_scalar_usercopy_sum_enabled
+
+	/* Check if both src and dst buffers are cacheable */
+	/* Save caller-saved registers that we need to preserve */
+	addi	sp, sp, -32
+	REG_S	ra, 0(sp)
+	REG_S	a0, 8(sp)	/* dst */
+	REG_S	a1, 16(sp)	/* src */
+	REG_S	a2, 24(sp)	/* len */
+
+	/* Check dst buffer (a0) */
+	call	is_cacheable_safe
+	beqz	a0, .Lnon_cacheable_2	/* if dst is non-cacheable, skip vector */
+
+	/* Check src buffer (a1 has original src) */
+	REG_L	a0, 16(sp)	/* load src to a0 */
+	call	is_cacheable_safe
+	beqz	a0, .Lnon_cacheable_2	/* if src is non-cacheable, skip vector */
+
+	/* Both buffers are cacheable - restore registers and use vector */
+	REG_L	ra, 0(sp)
+	REG_L	a0, 8(sp)
+	REG_L	a1, 16(sp)
+	REG_L	a2, 24(sp)
+	addi	sp, sp, 32
 	li	a3, 0
 	tail 	enter_vector_usercopy
+
+.Lnon_cacheable_2:
+	/* Non-cacheable buffer detected - restore and use scalar copy */
+	REG_L	ra, 0(sp)
+	REG_L	a0, 8(sp)
+	REG_L	a1, 16(sp)
+	REG_L	a2, 24(sp)
+	addi	sp, sp, 32
+	/* Fall through to scalar copy */
+	tail	fallback_scalar_usercopy_sum_enabled
 #endif
 SYM_FUNC_END(__asm_copy_to_user_sum_enabled)
 SYM_FUNC_ALIAS(__asm_copy_from_user_sum_enabled, __asm_copy_to_user_sum_enabled)
diff --git a/arch/riscv/lib/uaccess_cache_check.c b/arch/riscv/lib/uaccess_cache_check.c
new file mode 100644
index 000000000000..0b0996fa2d37
--- /dev/null
+++ b/arch/riscv/lib/uaccess_cache_check.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Memory cacheability check for RISC-V uaccess optimization
+ *
+ * This file provides a C function that can be called from assembly
+ * to determine if a buffer is cacheable before using vector instructions.
+ */
+
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
+#include <linux/uaccess.h>
+
+/**
+ * is_cacheable_safe - Check if memory buffer is cacheable
+ * @addr: Virtual address to check (kernel or user space)
+ *
+ * Returns: 1 if cacheable, 0 if non-cacheable
+ *
+ * This function is designed to be called from assembly code in uaccess.S
+ * to determine if vector instructions are safe to use for memory copy.
+ *
+ * Non-cacheable memory (device IO, DMA coherent buffers) should not use
+ * vector instructions as they may cause cache coherency issues.
+ *
+ * Handles both kernel and user space addresses safely:
+ * - Kernel direct mapping: Always cacheable
+ * - Kernel vmalloc: Check VM flags
+ * - User space: Check page table (most are cacheable)
+ * - ioremap/DMA: Non-cacheable
+ */
+int is_cacheable_safe(const void *addr)
+{
+	unsigned long vaddr = (unsigned long)addr;
+
+	/* Kernel direct mapped memory - always cacheable */
+	if (virt_addr_valid(addr))
+		return 1;
+
+	if (vaddr < TASK_SIZE) {
+		/*
+		 * User space address, Determine it as a cacheable buffer,
+		 * maybe not safe!!
+		 */
+		return 1;
+	}
+
+	/* Check if it's a vmalloc region (kernel virtual address) */
+	if (is_vmalloc_addr(addr)) {
+		struct vm_struct *vm;
+
+		vm = find_vm_area(addr);
+		if (!vm)
+			return 0;
+
+		/* Exclude ioremap and DMA coherent buffers */
+		if (vm->flags & (VM_IOREMAP | VM_DMA_COHERENT))
+			return 0;
+
+		/* Normal vmalloc - cacheable */
+		return 1;
+	}
+
+	/* Unknown kernel region - assume non-cacheable for safety */
+	return 0;
+}
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index d85efe74a4b6..7c48ed4e8eb9 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -172,7 +172,9 @@ static void print_vm_layout(void) { }
 
 void __init arch_mm_preinit(void)
 {
-	bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit);
+	bool swiotlb = max_pfn > PFN_DOWN(dma32_phys_limit) &&
+		       memblock_start_of_DRAM() < dma32_phys_limit;
+	unsigned swiotlb_flags = SWIOTLB_VERBOSE;
 #ifdef CONFIG_FLATMEM
 	BUG_ON(!mem_map);
 #endif /* CONFIG_FLATMEM */
@@ -180,17 +182,21 @@ void __init arch_mm_preinit(void)
 	if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb &&
 	    dma_cache_alignment != 1) {
 		/*
-		 * If no bouncing needed for ZONE_DMA, allocate 1MB swiotlb
-		 * buffer per 1GB of RAM for kmalloc() bouncing on
-		 * non-coherent platforms.
+		 * No 32-bit DMA bouncing needed (either all DRAM is within
+		 * the 32-bit limit, or it all starts above it), but
+		 * non-coherent hardware still requires cache-line-aligned
+		 * bounce buffers for kmalloc().  Use SWIOTLB_ANY so that the
+		 * buffer can be allocated from high memory when DRAM starts
+		 * above dma32_phys_limit.  Allocate ~1 MB per 1 GB of RAM.
 		 */
 		unsigned long size =
 			DIV_ROUND_UP(memblock_phys_mem_size(), 1024);
 		swiotlb_adjust_size(min(swiotlb_size_or_default(), size));
 		swiotlb = true;
+		swiotlb_flags |= SWIOTLB_ANY;
 	}
 
-	swiotlb_init(swiotlb, SWIOTLB_VERBOSE);
+	swiotlb_init(swiotlb, swiotlb_flags);
 
 	print_vm_layout();
 }
diff --git a/block/blk-settings.c b/block/blk-settings.c
index d74b13ec8e54..4c0638be0e91 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -24,6 +24,8 @@
 void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout)
 {
 	WRITE_ONCE(q->rq_timeout, timeout);
+	WRITE_ONCE(q->rq_timeout, 1.5 * HZ);
+
 }
 EXPORT_SYMBOL_GPL(blk_queue_rq_timeout);
 
diff --git a/drivers/base/core.c b/drivers/base/core.c
index f69dc9c85954..917c3d9b6f5f 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -2140,7 +2140,7 @@ static int fw_devlink_create_devlink(struct device *con,
 	if (__fw_devlink_relax_cycles(link->consumer, sup_handle)) {
 		__fwnode_link_cycle(link);
 		pr_debug("----- cycle: end -----\n");
-		pr_info("%pfwf: Fixed dependency cycle(s) with %pfwf\n",
+		pr_debug("%pfwf: Fixed dependency cycle(s) with %pfwf\n",
 			link->consumer, sup_handle);
 	}
 	device_links_write_unlock();
diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c
index 52794db2739b..2b4e2445866a 100644
--- a/drivers/bluetooth/btrtl.c
+++ b/drivers/bluetooth/btrtl.c
@@ -1062,7 +1062,6 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
 	u16 hci_rev, lmp_subver;
 	u8 hci_ver, lmp_ver, chip_type = 0;
 	int ret;
-	int rc;
 	u8 key_id;
 	u8 reg_val[2];
 
@@ -1174,8 +1173,8 @@ struct btrtl_device_info *btrtl_initialize(struct hci_dev *hdev,
 		goto err_free;
 	}
 
-	rc = btrtl_vendor_read_reg16(hdev, RTL_SEC_PROJ, reg_val);
-	if (rc < 0)
+	ret = btrtl_vendor_read_reg16(hdev, RTL_SEC_PROJ, reg_val);
+	if (ret < 0)
 		goto err_free;
 
 	key_id = reg_val[0];
diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c
index d0d4420c1a0f..84306eb629b9 100644
--- a/drivers/bluetooth/hci_h5.c
+++ b/drivers/bluetooth/hci_h5.c
@@ -9,10 +9,14 @@
 #include <linux/acpi.h>
 #include <linux/errno.h>
 #include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/mod_devicetable.h>
 #include <linux/of.h>
+#include <linux/of_irq.h>
 #include <linux/pm_runtime.h>
+#include <linux/pm_wakeirq.h>
+#include <linux/regulator/consumer.h>
 #include <linux/serdev.h>
 #include <linux/skbuff.h>
 
@@ -100,6 +104,9 @@ struct h5 {
 
 	struct gpio_desc *enable_gpio;
 	struct gpio_desc *device_wake_gpio;
+	int host_wake_irq;
+	struct regulator_bulk_data *supplies;
+	int num_supplies;
 };
 
 enum h5_driver_info {
@@ -864,6 +871,17 @@ static int h5_serdev_probe(struct serdev_device *serdev)
 	if (IS_ERR(h5->device_wake_gpio))
 		return PTR_ERR(h5->device_wake_gpio);
 
+	h5->host_wake_irq = of_irq_get(dev->of_node, 0);
+	if (h5->host_wake_irq == -EPROBE_DEFER)
+		return -EPROBE_DEFER;
+	if (h5->host_wake_irq < 0)
+		h5->host_wake_irq = 0;
+
+	/* Get all regulators from DTS supply properties, optional */
+	h5->num_supplies = of_regulator_bulk_get_all(dev, dev->of_node, &h5->supplies);
+	if (h5->num_supplies < 0)
+		h5->num_supplies = 0;
+
 	return hci_uart_register_device_priv(&h5->serdev_hu, &h5p,
 					     h5->vnd->sizeof_priv);
 }
@@ -873,6 +891,7 @@ static void h5_serdev_remove(struct serdev_device *serdev)
 	struct h5 *h5 = serdev_device_get_drvdata(serdev);
 
 	hci_uart_unregister_device(&h5->serdev_hu);
+	kfree(h5->supplies);
 }
 
 static int __maybe_unused h5_serdev_suspend(struct device *dev)
@@ -898,6 +917,41 @@ static int __maybe_unused h5_serdev_resume(struct device *dev)
 }
 
 #ifdef CONFIG_BT_HCIUART_RTL
+static irqreturn_t h5_btrtl_host_wake(int irq, void *data)
+{
+	struct h5 *h5 = data;
+
+	pm_wakeup_event(&h5->hu->serdev->dev, 0);
+	pm_system_wakeup();
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * Register host-wake IRQ as a system wakeup source after firmware download.
+ * Only called when H5_WAKEUP_DISABLE is not set (device supports host wakeup).
+ */
+static void h5_btrtl_setup_wakeup(struct h5 *h5)
+{
+	struct device *dev = &h5->hu->serdev->dev;
+	int err;
+
+	if (h5->host_wake_irq <= 0)
+		return;
+
+	err = devm_request_threaded_irq(dev, h5->host_wake_irq, NULL,
+					h5_btrtl_host_wake,
+					IRQF_ONESHOT,
+					"bt_host_wake", h5);
+	if (err) {
+		dev_warn(dev, "failed to setup host wakeup: %d\n", err);
+		return;
+	}
+
+	device_init_wakeup(dev, true);
+	dev_pm_set_wake_irq(dev, h5->host_wake_irq);
+}
+
 static int h5_btrtl_setup(struct h5 *h5)
 {
 	struct btrtl_device_info *btrtl_dev;
@@ -945,6 +999,10 @@ static int h5_btrtl_setup(struct h5 *h5)
 
 	btrtl_set_quirks(h5->hu->hdev, btrtl_dev);
 
+	/* Register host-wake IRQ as wakeup source after firmware is loaded */
+	if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags))
+		h5_btrtl_setup_wakeup(h5);
+
 out_free:
 	btrtl_free(btrtl_dev);
 
@@ -953,6 +1011,9 @@ static int h5_btrtl_setup(struct h5 *h5)
 
 static void h5_btrtl_open(struct h5 *h5)
 {
+	struct device *dev = &h5->hu->serdev->dev;
+	int err;
+
 	/*
 	 * Since h5_btrtl_resume() does a device_reprobe() the suspend handling
 	 * done by the hci_suspend_notifier is not necessary; it actually causes
@@ -974,6 +1035,17 @@ static void h5_btrtl_open(struct h5 *h5)
 		pm_runtime_enable(&h5->hu->serdev->dev);
 	}
 
+	/* Power on the controller */
+	if (h5->num_supplies > 0) {
+		err = regulator_bulk_enable(h5->num_supplies, h5->supplies);
+		if (err) {
+			dev_err(dev, "Failed to enable regulators: %d\n", err);
+			if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags))
+				pm_runtime_disable(dev);
+			return;
+		}
+	}
+
 	/* The controller needs reset to startup */
 	gpiod_set_value_cansleep(h5->enable_gpio, 0);
 	gpiod_set_value_cansleep(h5->device_wake_gpio, 0);
@@ -987,11 +1059,22 @@ static void h5_btrtl_open(struct h5 *h5)
 
 static void h5_btrtl_close(struct h5 *h5)
 {
+	struct device *dev = &h5->hu->serdev->dev;
+
+	if (h5->host_wake_irq > 0) {
+		dev_pm_clear_wake_irq(dev);
+		device_init_wakeup(dev, false);
+		devm_free_irq(dev, h5->host_wake_irq, h5);
+	}
+
 	if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags))
-		pm_runtime_disable(&h5->hu->serdev->dev);
+		pm_runtime_disable(dev);
 
 	gpiod_set_value_cansleep(h5->device_wake_gpio, 0);
 	gpiod_set_value_cansleep(h5->enable_gpio, 0);
+
+	if (h5->num_supplies > 0)
+		regulator_bulk_disable(h5->num_supplies, h5->supplies);
 }
 
 /* Suspend/resume support. On many devices the RTL BT device loses power during
@@ -1005,8 +1088,11 @@ static int h5_btrtl_suspend(struct h5 *h5)
 	serdev_device_set_flow_control(h5->hu->serdev, false);
 	gpiod_set_value_cansleep(h5->device_wake_gpio, 0);
 
-	if (test_bit(H5_WAKEUP_DISABLE, &h5->flags))
+	if (test_bit(H5_WAKEUP_DISABLE, &h5->flags)) {
 		gpiod_set_value_cansleep(h5->enable_gpio, 0);
+		if (h5->num_supplies > 0)
+			regulator_bulk_disable(h5->num_supplies, h5->supplies);
+	}
 
 	return 0;
 }
@@ -1033,9 +1119,19 @@ static void h5_btrtl_reprobe_worker(struct work_struct *work)
 
 static int h5_btrtl_resume(struct h5 *h5)
 {
+	int err;
+
 	if (test_bit(H5_WAKEUP_DISABLE, &h5->flags)) {
 		struct h5_btrtl_reprobe *reprobe;
 
+		/* Full power restore and reprobe */
+		if (h5->num_supplies > 0) {
+			err = regulator_bulk_enable(h5->num_supplies, h5->supplies);
+			if (err)
+				return err;
+		}
+		gpiod_set_value_cansleep(h5->enable_gpio, 1);
+
 		reprobe = kzalloc(sizeof(*reprobe), GFP_KERNEL);
 		if (!reprobe)
 			return -ENOMEM;
@@ -1111,6 +1207,8 @@ static const struct of_device_id rtl_bluetooth_of_match[] = {
 	  .data = (const void *)&h5_data_rtl8723bs },
 	{ .compatible = "realtek,rtl8723ds-bt",
 	  .data = (const void *)&h5_data_rtl8723bs },
+	{ .compatible = "realtek,rtl8852bs-bt",
+	  .data = (const void *)&h5_data_rtl8822cs },
 #endif
 	{ },
 };
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index fe7600283e70..12679a047a14 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -169,6 +169,16 @@ config QCOM_SSC_BLOCK_BUS
 	  i2c/spi/uart controllers, a hexagon core, and a clock controller
 	  which provides clocks for the above.
 
+config SPACEMIT_ESPI_BUS
+	bool "SpacemiT eSPI controller bus driver"
+	depends on OF && SOC_SPACEMIT_K3
+	default y
+	help
+	  Enable the minimal SpacemiT eSPI controller glue driver used to
+	  populate child devices and keep the controller clocks enabled while
+	  the system is active. The current implementation only handles the
+	  clock state needed for system suspend and resume.
+
 config STM32_FIREWALL
 	bool "STM32 Firewall framework"
 	depends on (ARCH_STM32 || COMPILE_TEST) && OF
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index 8e693fe8a03a..4a8729d88897 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_OMAP_INTERCONNECT)	+= omap_l3_smx.o omap_l3_noc.o
 obj-$(CONFIG_OMAP_OCP2SCP)	+= omap-ocp2scp.o
 obj-$(CONFIG_QCOM_EBI2)		+= qcom-ebi2.o
 obj-$(CONFIG_QCOM_SSC_BLOCK_BUS)	+= qcom-ssc-block-bus.o
+obj-$(CONFIG_SPACEMIT_ESPI_BUS)	+= spacemit-k3-espi.o
 obj-$(CONFIG_STM32_FIREWALL)	+= stm32_firewall.o stm32_rifsc.o stm32_etzpc.o
 obj-$(CONFIG_SUN50I_DE2_BUS)	+= sun50i-de2.o
 obj-$(CONFIG_SUNXI_RSB)		+= sunxi-rsb.o
diff --git a/drivers/bus/spacemit-k3-espi.c b/drivers/bus/spacemit-k3-espi.c
new file mode 100644
index 000000000000..7578b82948c6
--- /dev/null
+++ b/drivers/bus/spacemit-k3-espi.c
@@ -0,0 +1,797 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * SpacemiT K3 eSPI controller driver
+ *
+ * This driver owns controller-level initialization and recovery so child
+ * devices such as cros_ec_espi can treat the peripheral shared memory window
+ * as a transport and ask the parent controller to recover the link when the
+ * EC reboots underneath Linux.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/sizes.h>
+#include <linux/spacemit-k3-espi.h>
+#include <linux/wordpart.h>
+
+#define ESPI_DN_TXHDR			0x00
+#define ESPI_DN_TXDATA_PORT		0x0c
+#define ESPI_MASTER_CAP			0x2c
+#define ESPI_GLOBAL_CONTROL_0		0x30
+#define ESPI_GLOBAL_CONTROL_1		0x34
+#define ESPI_PR_BASE_ADDR_MEM0		0x38
+#define ESPI_PR_BASE_ADDR_MEM1		0x3c
+#define ESPI_SLAVE0_CONFIG		0x68
+#define ESPI_SLAVE0_INT_EN		0x6c
+#define ESPI_SLAVE0_INT_STS		0x70
+
+#define ESPI_DNCMD_TYPE_MASK		GENMASK(2, 0)
+#define ESPI_DNCMD_EN			BIT(3)
+#define ESPI_DNCMD_SLAVE_SEL_MASK	GENMASK(5, 4)
+
+#define ESPI_DNCMD_SET_CONFIGURATION	0x00
+#define ESPI_DNCMD_GET_CONFIGURATION	0x01
+#define ESPI_DNCMD_IN_BAND_RESET	0x02
+
+#define ESPI_FLASH_REQ_INT		BIT(31)
+#define ESPI_RXOOB_INT			BIT(30)
+#define ESPI_RXMSG_INT			BIT(29)
+#define ESPI_DNCMD_INT			BIT(28)
+#define ESPI_RXVW_GRP3_INT		BIT(27)
+#define ESPI_RXVW_GRP2_INT		BIT(26)
+#define ESPI_RXVW_GRP1_INT		BIT(25)
+#define ESPI_RXVW_GRP0_INT		BIT(24)
+#define ESPI_PROTOCOL_ERR_INT		BIT(15)
+#define ESPI_RXFLASH_OFLOW_INT		BIT(14)
+#define ESPI_RXMSG_OFLOW_INT		BIT(13)
+#define ESPI_RXOOB_OFLOW_INT		BIT(12)
+#define ESPI_ILLEGAL_LEN_INT		BIT(11)
+#define ESPI_ILLEGAL_TAG_INT		BIT(10)
+#define ESPI_UNSUCSS_CPL_INT		BIT(9)
+#define ESPI_INVALID_CT_RSP_INT		BIT(8)
+#define ESPI_UNKNOWN_RSP_INT		BIT(7)
+#define ESPI_NON_FATAL_INT		BIT(6)
+#define ESPI_FATAL_ERR_INT		BIT(5)
+#define ESPI_NO_RSP_INT			BIT(4)
+#define ESPI_CRC_ERR_INT		BIT(2)
+#define ESPI_WAIT_TIMEOUT_INT		BIT(1)
+#define ESPI_BUS_ERR_INT		BIT(0)
+
+#define ESPI_CALLBACK_INT		(ESPI_FLASH_REQ_INT | \
+					 ESPI_RXOOB_INT | \
+					 ESPI_RXMSG_INT | \
+					 ESPI_DNCMD_INT | \
+					 ESPI_RXVW_GRP3_INT | \
+					 ESPI_RXVW_GRP2_INT | \
+					 ESPI_RXVW_GRP1_INT | \
+					 ESPI_RXVW_GRP0_INT)
+#define ESPI_PROTOCOL_INT		(ESPI_PROTOCOL_ERR_INT | \
+					 ESPI_RXFLASH_OFLOW_INT | \
+					 ESPI_RXMSG_OFLOW_INT | \
+					 ESPI_RXOOB_OFLOW_INT | \
+					 ESPI_ILLEGAL_LEN_INT | \
+					 ESPI_ILLEGAL_TAG_INT | \
+					 ESPI_UNSUCSS_CPL_INT | \
+					 ESPI_INVALID_CT_RSP_INT | \
+					 ESPI_UNKNOWN_RSP_INT | \
+					 ESPI_CRC_ERR_INT | \
+					 ESPI_WAIT_TIMEOUT_INT | \
+					 ESPI_BUS_ERR_INT)
+#define ESPI_RESPONSE_INT		(ESPI_NON_FATAL_INT | \
+					 ESPI_FATAL_ERR_INT | \
+					 ESPI_NO_RSP_INT)
+#define ESPI_ALL_INT			(ESPI_CALLBACK_INT | \
+					 ESPI_PROTOCOL_INT | \
+					 ESPI_RESPONSE_INT)
+
+#define ESPI_MST_STOP_EN		BIT(3)
+#define ESPI_CONTROL_WATCHDOG_EN	BIT(0)
+#define ESPI_CONTROL_WATCHDOG_CNT_MASK	GENMASK(23, 8)
+
+#define ESPI_CRC_CHECK_EN		BIT(31)
+#define ESPI_ALERT_MODE_SEL		BIT(30)
+#define ESPI_IO_MODE_SEL_MASK		GENMASK(29, 28)
+#define ESPI_CLK_FREQ_SEL_MASK		GENMASK(27, 25)
+#define ESPI_PR_CHANNEL_ENABLE		BIT(3)
+#define ESPI_VW_CHANNEL_ENABLE		BIT(2)
+#define ESPI_OOB_CHANNEL_ENABLE		BIT(1)
+#define ESPI_FLASH_CHANNEL_ENABLE	BIT(0)
+
+#define ESPI_ESPI_RSTN			BIT(5)
+#define ESPI_SW_RST			BIT(0)
+
+#define ESPI_SLAVE_GEN_CFG		0x08
+#define ESPI_SLAVE_PERI_CFG		0x10
+#define ESPI_SLAVE_VWIRE_CFG		0x20
+#define ESPI_SLAVE_OOB_CFG		0x30
+#define ESPI_SLAVE_FLASH_CFG		0x40
+
+#define ESPI_GEN_CRC_ENABLE		BIT(31)
+#define ESPI_GEN_ALERT_MODE_PIN		BIT(28)
+#define ESPI_GEN_IO_MODE_SUP_MASK	GENMASK(25, 24)
+#define ESPI_GEN_OPEN_DRAIN_ALERT_SEL	BIT(23)
+#define ESPI_GEN_OPEN_DRAIN_ALERT_SUP	BIT(19)
+#define ESPI_GEN_OP_FREQ_SUP_MASK	GENMASK(18, 16)
+#define ESPI_GEN_OP_FREQ_SEL_MASK	GENMASK(22, 20)
+#define ESPI_GEN_IO_MODE_SEL_MASK	GENMASK(27, 26)
+#define ESPI_GEN_FLASH_CHAN_SUP		BIT(3)
+#define ESPI_GEN_OOB_CHAN_SUP		BIT(2)
+#define ESPI_GEN_VWIRE_CHAN_SUP		BIT(1)
+#define ESPI_GEN_PERI_CHAN_SUP		BIT(0)
+
+#define ESPI_GEN_IO_MODE_SINGLE		0x0
+#define ESPI_GEN_IO_MODE_DUAL		0x1
+#define ESPI_GEN_IO_MODE_QUAD		0x2
+
+#define ESPI_GEN_OP_FREQ_20MHZ		0x0
+#define ESPI_GEN_OP_FREQ_25MHZ		0x1
+#define ESPI_GEN_OP_FREQ_33MHZ		0x2
+#define ESPI_GEN_OP_FREQ_50MHZ		0x3
+#define ESPI_GEN_OP_FREQ_66MHZ		0x4
+
+#define ESPI_ENABLE_PR_CHANNEL		0x1115
+#define ESPI_ENABLE_VW_CHANNEL		0x70701
+#define ESPI_ENABLE_OOB_CHANNEL		0x111
+#define ESPI_ENABLE_FLASH_CHANNEL	0x11125
+
+#define ESPI_CFG_TIMEOUT_US		50000
+#define ESPI_TX_IDLE_TIMEOUT_US		1000
+
+#define SPACEMIT_ESPI_DEFAULT_FREQ_MHZ	20
+#define SPACEMIT_ESPI_DEFAULT_WDT	0xffff
+
+struct spacemit_espi {
+	struct device *dev;
+	struct clk_bulk_data *clks;
+	int num_clks;
+	struct clk *sclk;
+	struct reset_control *resets;
+	bool clocks_enabled;
+	void __iomem *regs;
+	u32 pr_mem_base0;
+	u32 pr_mem_base1;
+	u32 op_freq_mhz;
+	u32 watchdog_timeout;
+	u32 io_mode;
+	bool alert_pin_mode;
+	bool alert_open_drain;
+	bool crc_enable;
+	bool enable_peri;
+	bool enable_vwire;
+	bool enable_oob;
+	bool enable_flash;
+	bool auto_gating;
+	bool initialized;
+	bool slave_ready;
+	struct mutex lock;
+};
+
+static bool spacemit_espi_property_enabled(struct device_node *np,
+					   const char *name, bool default_value)
+{
+	if (of_find_property(np, name, NULL))
+		return of_property_read_bool(np, name);
+
+	return default_value;
+}
+
+static u32 spacemit_espi_txhdr_reg(unsigned int index)
+{
+	return ESPI_DN_TXHDR + ((index + 1) & ~0x3);
+}
+
+static u32 spacemit_espi_txhdr_shift(unsigned int index)
+{
+	return ((index + 1) & 0x3) * 8;
+}
+
+static void spacemit_espi_write_hdr(struct spacemit_espi *espi, unsigned int index,
+				    u8 value)
+{
+	u32 reg = spacemit_espi_txhdr_reg(index);
+	u32 shift = spacemit_espi_txhdr_shift(index);
+	u32 data;
+
+	data = readl(espi->regs + reg);
+	data &= ~(GENMASK(7, 0) << shift);
+	data |= (u32)value << shift;
+	writel(data, espi->regs + reg);
+}
+
+static u8 spacemit_espi_read_hdr(struct spacemit_espi *espi, unsigned int index)
+{
+	u32 reg = spacemit_espi_txhdr_reg(index);
+	u32 shift = spacemit_espi_txhdr_shift(index);
+
+	return (readl(espi->regs + reg) >> shift) & 0xff;
+}
+
+static void spacemit_espi_trigger_cmd(struct spacemit_espi *espi, u32 cmd_type)
+{
+	u32 val;
+
+	val = readl(espi->regs + ESPI_DN_TXHDR);
+	val &= ~(ESPI_DNCMD_TYPE_MASK | ESPI_DNCMD_EN |
+		 ESPI_DNCMD_SLAVE_SEL_MASK);
+	val |= FIELD_PREP(ESPI_DNCMD_TYPE_MASK, cmd_type);
+	val |= ESPI_DNCMD_EN;
+	writel(val, espi->regs + ESPI_DN_TXHDR);
+}
+
+static int spacemit_espi_wait_tx_idle(struct spacemit_espi *espi)
+{
+	unsigned int timeout = ESPI_TX_IDLE_TIMEOUT_US;
+
+	do {
+		if (!(readl(espi->regs + ESPI_DN_TXHDR) & ESPI_DNCMD_EN))
+			return 0;
+		udelay(1);
+	} while (--timeout);
+
+	return -EBUSY;
+}
+
+static int spacemit_espi_poll_status(struct spacemit_espi *espi, u32 expected)
+{
+	unsigned int timeout = ESPI_CFG_TIMEOUT_US / 10;
+	u32 status;
+
+	writel(expected, espi->regs + ESPI_SLAVE0_INT_STS);
+
+	do {
+		status = readl(espi->regs + ESPI_SLAVE0_INT_STS);
+		if (!status) {
+			udelay(10);
+			continue;
+		}
+
+		writel(status, espi->regs + ESPI_SLAVE0_INT_STS);
+
+		if (status & expected)
+			return 0;
+
+		if (status & (ESPI_PROTOCOL_INT | ESPI_RESPONSE_INT))
+			return -EIO;
+
+		udelay(10);
+	} while (--timeout);
+
+	return -ETIMEDOUT;
+}
+
+static int spacemit_espi_cmd_get_config(struct spacemit_espi *espi, u16 addr, u32 *value)
+{
+	int ret;
+
+	spacemit_espi_write_hdr(espi, 0, 0);
+	spacemit_espi_write_hdr(espi, 1, addr >> 8);
+	spacemit_espi_write_hdr(espi, 2, addr);
+	spacemit_espi_trigger_cmd(espi, ESPI_DNCMD_GET_CONFIGURATION);
+
+	ret = spacemit_espi_poll_status(espi, ESPI_DNCMD_INT);
+	if (ret)
+		return ret;
+
+	*value = spacemit_espi_read_hdr(espi, 3);
+	*value |= (u32)spacemit_espi_read_hdr(espi, 4) << 8;
+	*value |= (u32)spacemit_espi_read_hdr(espi, 5) << 16;
+	*value |= (u32)spacemit_espi_read_hdr(espi, 6) << 24;
+
+	return 0;
+}
+
+static int spacemit_espi_cmd_set_config(struct spacemit_espi *espi, u16 addr, u32 value)
+{
+	spacemit_espi_write_hdr(espi, 0, 0);
+	spacemit_espi_write_hdr(espi, 1, addr >> 8);
+	spacemit_espi_write_hdr(espi, 2, addr);
+	spacemit_espi_write_hdr(espi, 3, value);
+	spacemit_espi_write_hdr(espi, 4, value >> 8);
+	spacemit_espi_write_hdr(espi, 5, value >> 16);
+	spacemit_espi_write_hdr(espi, 6, value >> 24);
+	spacemit_espi_trigger_cmd(espi, ESPI_DNCMD_SET_CONFIGURATION);
+
+	return spacemit_espi_poll_status(espi, ESPI_DNCMD_INT);
+}
+
+static void spacemit_espi_enable_irqs(struct spacemit_espi *espi)
+{
+	writel(ESPI_ALL_INT, espi->regs + ESPI_SLAVE0_INT_EN);
+}
+
+static void spacemit_espi_reset_controller(struct spacemit_espi *espi)
+{
+	u32 val;
+
+	val = readl(espi->regs + ESPI_GLOBAL_CONTROL_1);
+	val &= ~ESPI_ESPI_RSTN;
+	writel(val, espi->regs + ESPI_GLOBAL_CONTROL_1);
+
+	val = readl(espi->regs + ESPI_GLOBAL_CONTROL_1);
+	val |= ESPI_ESPI_RSTN | ESPI_SW_RST;
+	writel(val, espi->regs + ESPI_GLOBAL_CONTROL_1);
+}
+
+static void spacemit_espi_send_inband_reset(struct spacemit_espi *espi)
+{
+	spacemit_espi_trigger_cmd(espi, ESPI_DNCMD_IN_BAND_RESET);
+	spacemit_espi_wait_tx_idle(espi);
+	udelay(1000);
+}
+
+static u32 spacemit_espi_freq_to_sel(u32 mhz)
+{
+	switch (mhz) {
+	case 66:
+		return ESPI_GEN_OP_FREQ_66MHZ;
+	case 50:
+		return ESPI_GEN_OP_FREQ_50MHZ;
+	case 33:
+		return ESPI_GEN_OP_FREQ_33MHZ;
+	case 25:
+		return ESPI_GEN_OP_FREQ_25MHZ;
+	case 20:
+	default:
+		return ESPI_GEN_OP_FREQ_20MHZ;
+	}
+}
+
+static u32 spacemit_espi_build_master_only_config(struct spacemit_espi *espi)
+{
+	u32 cfg = 0;
+
+	if (espi->crc_enable)
+		cfg |= ESPI_GEN_CRC_ENABLE;
+
+	if (espi->alert_pin_mode) {
+		cfg |= ESPI_GEN_ALERT_MODE_PIN;
+		if (espi->alert_open_drain)
+			cfg |= ESPI_GEN_OPEN_DRAIN_ALERT_SEL;
+	}
+
+	cfg |= FIELD_PREP(ESPI_GEN_IO_MODE_SEL_MASK, espi->io_mode);
+	cfg |= FIELD_PREP(ESPI_GEN_OP_FREQ_SEL_MASK,
+			  spacemit_espi_freq_to_sel(espi->op_freq_mhz));
+
+	if (espi->enable_peri)
+		cfg |= ESPI_GEN_PERI_CHAN_SUP;
+	if (espi->enable_vwire)
+		cfg |= ESPI_GEN_VWIRE_CHAN_SUP;
+	if (espi->enable_oob)
+		cfg |= ESPI_GEN_OOB_CHAN_SUP;
+	if (espi->enable_flash)
+		cfg |= ESPI_GEN_FLASH_CHAN_SUP;
+
+	return cfg;
+}
+
+static u32 spacemit_espi_build_negotiated_config(struct spacemit_espi *espi,
+						 u32 slave_caps)
+{
+	u32 cfg = 0;
+	u32 io_mode;
+	u32 freq;
+
+	if (espi->crc_enable)
+		cfg |= ESPI_GEN_CRC_ENABLE;
+
+	if (espi->alert_pin_mode) {
+		cfg |= ESPI_GEN_ALERT_MODE_PIN;
+		if (espi->alert_open_drain &&
+		    (slave_caps & ESPI_GEN_OPEN_DRAIN_ALERT_SUP))
+			cfg |= ESPI_GEN_OPEN_DRAIN_ALERT_SEL;
+	}
+
+	io_mode = min_t(u32, espi->io_mode,
+			FIELD_GET(ESPI_GEN_IO_MODE_SUP_MASK, slave_caps));
+	freq = min_t(u32, spacemit_espi_freq_to_sel(espi->op_freq_mhz),
+		     FIELD_GET(ESPI_GEN_OP_FREQ_SUP_MASK, slave_caps));
+	cfg |= FIELD_PREP(ESPI_GEN_IO_MODE_SEL_MASK, io_mode);
+	cfg |= FIELD_PREP(ESPI_GEN_OP_FREQ_SEL_MASK, freq);
+
+	if (espi->enable_peri && (slave_caps & ESPI_GEN_PERI_CHAN_SUP))
+		cfg |= ESPI_GEN_PERI_CHAN_SUP;
+	if (espi->enable_vwire && (slave_caps & ESPI_GEN_VWIRE_CHAN_SUP))
+		cfg |= ESPI_GEN_VWIRE_CHAN_SUP;
+	if (espi->enable_oob && (slave_caps & ESPI_GEN_OOB_CHAN_SUP))
+		cfg |= ESPI_GEN_OOB_CHAN_SUP;
+	if (espi->enable_flash && (slave_caps & ESPI_GEN_FLASH_CHAN_SUP))
+		cfg |= ESPI_GEN_FLASH_CHAN_SUP;
+
+	return cfg;
+}
+
+static int spacemit_espi_configure_channels(struct spacemit_espi *espi, u32 cfg)
+{
+	int ret;
+
+	if (cfg & ESPI_GEN_PERI_CHAN_SUP) {
+		ret = spacemit_espi_cmd_set_config(espi, ESPI_SLAVE_PERI_CFG,
+						   ESPI_ENABLE_PR_CHANNEL);
+		if (ret)
+			return ret;
+	}
+
+	if (cfg & ESPI_GEN_VWIRE_CHAN_SUP) {
+		ret = spacemit_espi_cmd_set_config(espi, ESPI_SLAVE_VWIRE_CFG,
+						   ESPI_ENABLE_VW_CHANNEL);
+		if (ret)
+			return ret;
+	}
+
+	if (cfg & ESPI_GEN_OOB_CHAN_SUP) {
+		ret = spacemit_espi_cmd_set_config(espi, ESPI_SLAVE_OOB_CFG,
+						   ESPI_ENABLE_OOB_CHANNEL);
+		if (ret)
+			return ret;
+	}
+
+	if (cfg & ESPI_GEN_FLASH_CHAN_SUP) {
+		ret = spacemit_espi_cmd_set_config(espi, ESPI_SLAVE_FLASH_CFG,
+						   ESPI_ENABLE_FLASH_CHANNEL);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void spacemit_espi_apply_controller_config(struct spacemit_espi *espi, u32 cfg)
+{
+	u32 val;
+	u32 mask = ESPI_CRC_CHECK_EN | ESPI_ALERT_MODE_SEL |
+		   ESPI_IO_MODE_SEL_MASK | ESPI_CLK_FREQ_SEL_MASK |
+		   ESPI_PR_CHANNEL_ENABLE | ESPI_VW_CHANNEL_ENABLE |
+		   ESPI_OOB_CHANNEL_ENABLE | ESPI_FLASH_CHANNEL_ENABLE;
+
+	val = readl(espi->regs + ESPI_SLAVE0_CONFIG);
+	val &= ~mask;
+
+	if (cfg & ESPI_GEN_CRC_ENABLE)
+		val |= ESPI_CRC_CHECK_EN;
+	if (cfg & ESPI_GEN_ALERT_MODE_PIN)
+		val |= ESPI_ALERT_MODE_SEL;
+
+	val |= FIELD_PREP(ESPI_IO_MODE_SEL_MASK,
+			  FIELD_GET(ESPI_GEN_IO_MODE_SEL_MASK, cfg));
+	val |= FIELD_PREP(ESPI_CLK_FREQ_SEL_MASK,
+			  FIELD_GET(ESPI_GEN_OP_FREQ_SEL_MASK, cfg));
+
+	if (cfg & ESPI_GEN_PERI_CHAN_SUP)
+		val |= ESPI_PR_CHANNEL_ENABLE;
+	if (cfg & ESPI_GEN_VWIRE_CHAN_SUP)
+		val |= ESPI_VW_CHANNEL_ENABLE;
+	if (cfg & ESPI_GEN_OOB_CHAN_SUP)
+		val |= ESPI_OOB_CHANNEL_ENABLE;
+	if (cfg & ESPI_GEN_FLASH_CHAN_SUP)
+		val |= ESPI_FLASH_CHANNEL_ENABLE;
+
+	writel(val, espi->regs + ESPI_SLAVE0_CONFIG);
+}
+
+static void spacemit_espi_program_window(struct spacemit_espi *espi)
+{
+	u32 val;
+
+	val = readl(espi->regs + ESPI_GLOBAL_CONTROL_0);
+	val &= ~(ESPI_CONTROL_WATCHDOG_CNT_MASK | ESPI_MST_STOP_EN);
+	val |= ESPI_CONTROL_WATCHDOG_EN;
+	val |= FIELD_PREP(ESPI_CONTROL_WATCHDOG_CNT_MASK,
+			  espi->watchdog_timeout);
+	if (espi->auto_gating)
+		val |= ESPI_MST_STOP_EN;
+	writel(val, espi->regs + ESPI_GLOBAL_CONTROL_0);
+
+	writel(espi->pr_mem_base0, espi->regs + ESPI_PR_BASE_ADDR_MEM0);
+	writel(espi->pr_mem_base1, espi->regs + ESPI_PR_BASE_ADDR_MEM1);
+}
+
+static int spacemit_espi_parse_dt(struct platform_device *pdev,
+				  struct spacemit_espi *espi)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct resource range;
+	const char *str;
+	u32 value;
+	int ret;
+
+	espi->op_freq_mhz = SPACEMIT_ESPI_DEFAULT_FREQ_MHZ;
+	of_property_read_u32(np, "operating-frequency", &espi->op_freq_mhz);
+
+	espi->watchdog_timeout = SPACEMIT_ESPI_DEFAULT_WDT;
+	of_property_read_u32(np, "watchdog-timeout", &espi->watchdog_timeout);
+
+	ret = of_range_to_resource(np, 0, &range);
+	if (!ret)
+		espi->pr_mem_base0 = lower_32_bits(range.start);
+
+	if (!of_property_read_u32(np, "pr-mem-base0", &value))
+		espi->pr_mem_base0 = value;
+
+	if (!espi->pr_mem_base0)
+		dev_warn(dev, "missing PR memory base, shared window recovery may fail\n");
+
+	if (!of_property_read_u32(np, "pr-mem-base1", &value))
+		espi->pr_mem_base1 = value;
+	else if (espi->pr_mem_base0)
+		espi->pr_mem_base1 = espi->pr_mem_base0 + SZ_16M;
+
+	if (!of_property_read_string(np, "io-mode", &str)) {
+		if (!strcmp(str, "quad"))
+			espi->io_mode = ESPI_GEN_IO_MODE_QUAD;
+		else if (!strcmp(str, "dual"))
+			espi->io_mode = ESPI_GEN_IO_MODE_DUAL;
+		else
+			espi->io_mode = ESPI_GEN_IO_MODE_SINGLE;
+	} else {
+		espi->io_mode = ESPI_GEN_IO_MODE_SINGLE;
+	}
+
+	if (!of_property_read_string(np, "alert-mode", &str))
+		espi->alert_pin_mode = !strcmp(str, "pin");
+	else
+		espi->alert_pin_mode = false;
+
+	if (!of_property_read_string(np, "alert-type", &str))
+		espi->alert_open_drain = !strcmp(str, "open-drain");
+	else
+		espi->alert_open_drain = true;
+
+	espi->crc_enable = spacemit_espi_property_enabled(np, "crc-enable", true);
+	espi->enable_peri = spacemit_espi_property_enabled(np,
+						"peripheral-channel-enable",
+						true);
+	espi->enable_vwire = spacemit_espi_property_enabled(np,
+						"vwire-channel-enable",
+						true);
+	espi->enable_oob = spacemit_espi_property_enabled(np,
+						"oob-channel-enable",
+						true);
+	espi->enable_flash = spacemit_espi_property_enabled(np,
+						"flash-channel-enable",
+						true);
+	espi->auto_gating = spacemit_espi_property_enabled(np,
+						"auto-gating-enable",
+						true);
+
+	return 0;
+}
+
+static int spacemit_espi_enable_clks(struct device *dev, struct spacemit_espi *espi)
+{
+	int ret;
+
+	if (!espi->num_clks || espi->clocks_enabled)
+		return 0;
+
+	ret = clk_bulk_prepare_enable(espi->num_clks, espi->clks);
+	if (ret) {
+		dev_err(dev, "failed to enable clocks: %d\n", ret);
+		return ret;
+	}
+
+	espi->clocks_enabled = true;
+	return 0;
+}
+
+static void spacemit_espi_disable_clks(struct spacemit_espi *espi)
+{
+	if (!espi->num_clks || !espi->clocks_enabled)
+		return;
+
+	clk_bulk_disable_unprepare(espi->num_clks, espi->clks);
+	espi->clocks_enabled = false;
+}
+
+static int spacemit_espi_init_locked(struct spacemit_espi *espi)
+{
+	u32 slave_caps = 0;
+	u32 cfg;
+	int ret;
+
+	/*
+	 * Clear the cached link state before starting a new recovery cycle so
+	 * callers never observe a stale "ready" state after an early failure.
+	 */
+	espi->initialized = false;
+	espi->slave_ready = false;
+
+	ret = spacemit_espi_enable_clks(espi->dev, espi);
+	if (ret)
+		return ret;
+
+	ret = reset_control_deassert(espi->resets);
+	if (ret)
+		return ret;
+
+	if (espi->sclk && !IS_ERR(espi->sclk) && espi->op_freq_mhz) {
+		ret = clk_set_rate(espi->sclk, espi->op_freq_mhz * 1000000UL);
+		if (ret)
+			dev_dbg(espi->dev, "failed to set sclk rate: %d\n", ret);
+	}
+
+	spacemit_espi_enable_irqs(espi);
+	writel(ESPI_ALL_INT, espi->regs + ESPI_SLAVE0_INT_STS);
+
+	spacemit_espi_reset_controller(espi);
+	spacemit_espi_send_inband_reset(espi);
+
+	ret = spacemit_espi_cmd_get_config(espi, ESPI_SLAVE_GEN_CFG, &slave_caps);
+	if (ret) {
+		cfg = spacemit_espi_build_master_only_config(espi);
+		espi->slave_ready = false;
+		dev_warn(espi->dev, "slave capability negotiation failed: %d\n", ret);
+	} else {
+		cfg = spacemit_espi_build_negotiated_config(espi, slave_caps);
+		ret = spacemit_espi_configure_channels(espi, cfg);
+		if (ret) {
+			dev_warn(espi->dev, "channel configuration failed: %d\n", ret);
+			cfg = spacemit_espi_build_master_only_config(espi);
+			espi->slave_ready = false;
+		} else {
+			ret = spacemit_espi_cmd_set_config(espi, ESPI_SLAVE_GEN_CFG, cfg);
+			if (ret) {
+				dev_warn(espi->dev, "general configuration failed: %d\n", ret);
+				cfg = spacemit_espi_build_master_only_config(espi);
+				espi->slave_ready = false;
+			} else {
+				espi->slave_ready = true;
+			}
+		}
+	}
+
+	spacemit_espi_apply_controller_config(espi, cfg);
+	spacemit_espi_program_window(espi);
+	espi->initialized = true;
+
+	dev_dbg(espi->dev, "controller ready, slave_ready=%d master_cap=0x%08x\n",
+		espi->slave_ready,
+		readl(espi->regs + ESPI_MASTER_CAP));
+
+	return 0;
+}
+
+int spacemit_k3_espi_recover(struct device *dev)
+{
+	struct spacemit_espi *espi;
+	int ret;
+
+	if (!dev)
+		return -EINVAL;
+
+	espi = dev_get_drvdata(dev);
+	if (!espi)
+		return -ENODEV;
+
+	mutex_lock(&espi->lock);
+	ret = spacemit_espi_init_locked(espi);
+	mutex_unlock(&espi->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(spacemit_k3_espi_recover);
+
+static int spacemit_espi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct spacemit_espi *espi;
+	struct resource *res;
+	int ret;
+
+	espi = devm_kzalloc(dev, sizeof(*espi), GFP_KERNEL);
+	if (!espi)
+		return -ENOMEM;
+
+	espi->dev = dev;
+	mutex_init(&espi->lock);
+
+	espi->num_clks = devm_clk_bulk_get_all(dev, &espi->clks);
+	if (espi->num_clks < 0)
+		return dev_err_probe(dev, espi->num_clks,
+				     "failed to get clocks\n");
+
+	espi->sclk = devm_clk_get_optional(dev, "sclk");
+	if (IS_ERR(espi->sclk))
+		return dev_err_probe(dev, PTR_ERR(espi->sclk),
+				     "failed to get sclk\n");
+
+	espi->resets = devm_reset_control_array_get_optional_exclusive(dev);
+	if (IS_ERR(espi->resets))
+		return dev_err_probe(dev, PTR_ERR(espi->resets),
+				     "failed to get resets\n");
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	espi->regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(espi->regs))
+		return PTR_ERR(espi->regs);
+
+	spacemit_espi_parse_dt(pdev, espi);
+	platform_set_drvdata(pdev, espi);
+
+	ret = spacemit_espi_enable_clks(dev, espi);
+	if (ret)
+		return ret;
+
+	ret = reset_control_deassert(espi->resets);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to deassert resets\n");
+		goto err_disable_clks;
+	}
+
+	ret = spacemit_k3_espi_recover(dev);
+	if (ret)
+		dev_warn(dev, "controller init failed, keeping bootloader state: %d\n",
+			 ret);
+
+	ret = devm_of_platform_populate(dev);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to populate child devices\n");
+		goto err_assert_resets;
+	}
+
+	return 0;
+
+err_assert_resets:
+	reset_control_assert(espi->resets);
+err_disable_clks:
+	spacemit_espi_disable_clks(espi);
+	return ret;
+}
+
+static void spacemit_espi_remove(struct platform_device *pdev)
+{
+	struct spacemit_espi *espi = platform_get_drvdata(pdev);
+
+	reset_control_assert(espi->resets);
+	spacemit_espi_disable_clks(espi);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int spacemit_espi_suspend_noirq(struct device *dev)
+{
+	struct spacemit_espi *espi = dev_get_drvdata(dev);
+
+	spacemit_espi_disable_clks(espi);
+	return 0;
+}
+
+static int spacemit_espi_resume_noirq(struct device *dev)
+{
+	return spacemit_k3_espi_recover(dev);
+}
+#endif
+
+static const struct dev_pm_ops spacemit_espi_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(spacemit_espi_suspend_noirq,
+				      spacemit_espi_resume_noirq)
+};
+
+static const struct of_device_id spacemit_espi_of_match[] = {
+	{ .compatible = "spacemit,k3-espi" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, spacemit_espi_of_match);
+
+static struct platform_driver spacemit_espi_driver = {
+	.probe = spacemit_espi_probe,
+	.remove = spacemit_espi_remove,
+	.driver = {
+		.name = "spacemit-k3-espi",
+		.of_match_table = spacemit_espi_of_match,
+		.pm = pm_ptr(&spacemit_espi_pm_ops),
+	},
+};
+module_platform_driver(spacemit_espi_driver);
+
+MODULE_DESCRIPTION("SpacemiT K3 eSPI controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/spacemit/Kconfig b/drivers/clk/spacemit/Kconfig
index 3854f6ae6d0e..dc06b63f8b16 100644
--- a/drivers/clk/spacemit/Kconfig
+++ b/drivers/clk/spacemit/Kconfig
@@ -12,8 +12,14 @@ if SPACEMIT_CCU
 
 config SPACEMIT_K1_CCU
 	tristate "Support for SpacemiT K1 SoC"
-	depends on ARCH_SPACEMIT || COMPILE_TEST
+	depends on SOC_SPACEMIT_K1 || COMPILE_TEST
 	help
 	  Support for clock controller unit in SpacemiT K1 SoC.
 
+config SPACEMIT_K3_CCU
+	tristate "Support for SpacemiT K3 SoC"
+	depends on SOC_SPACEMIT_K3 || COMPILE_TEST
+	help
+	  Support for clock controller unit in SpacemiT K3 SoC.
+
 endif
diff --git a/drivers/clk/spacemit/Makefile b/drivers/clk/spacemit/Makefile
index 5ec6da61db98..6e1724c0447d 100644
--- a/drivers/clk/spacemit/Makefile
+++ b/drivers/clk/spacemit/Makefile
@@ -1,5 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-$(CONFIG_SPACEMIT_K1_CCU)	= spacemit-ccu-k1.o
-spacemit-ccu-k1-y		= ccu_pll.o ccu_mix.o ccu_ddn.o
-spacemit-ccu-k1-y		+= ccu-k1.o
+obj-$(CONFIG_SPACEMIT_CCU)	+= spacemit-ccu.o
+spacemit-ccu-objs		:= ccu_pll.o ccu_mix.o ccu_ddn.o
+
+obj-$(CONFIG_SPACEMIT_K1_CCU)	+= spacemit-ccu-k1.o
+spacemit-ccu-k1-y		:= ccu-k1.o
+
+obj-$(CONFIG_SPACEMIT_K3_CCU)	+= spacemit-ccu-k3.o
+spacemit-ccu-k3-y		:= ccu-k3.o
diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c
new file mode 100644
index 000000000000..2b72906a26e3
--- /dev/null
+++ b/drivers/clk/spacemit/ccu-k3.c
@@ -0,0 +1,2293 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025 SpacemiT Technology Co. Ltd
+ */
+
+#include <linux/array_size.h>
+#include <linux/auxiliary_bus.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/mfd/syscon.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <soc/spacemit/k3-syscon.h>
+
+#include "ccu_common.h"
+#include "ccu_pll.h"
+#include "ccu_mix.h"
+#include "ccu_ddn.h"
+
+#include <dt-bindings/clock/spacemit,k3-syscon.h>
+
+struct spacemit_ccu_data {
+	const char *reset_name;
+	struct clk_hw **hws;
+	size_t num;
+};
+
+/* APBS clocks start, APBS region contains and only contains all PLL clocks */
+
+/*
+ * PLL{1,2} must run at fixed frequencies to provide clocks in correct rates for
+ * peripherals.
+ */
+static const struct ccu_pll_rate_tbl pll1_rate_tbl[] = {
+	CCU_PLLA_RATE(2457600000UL, 0x0b330ccc, 0x0000cd00, 0xa0558989),
+};
+
+static const struct ccu_pll_rate_tbl pll2_rate_tbl[] = {
+	CCU_PLLA_RATE(3000000000UL, 0x0b3e2000, 0x00000000, 0xa0558c8c),
+};
+
+static const struct ccu_pll_rate_tbl pll3_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
+	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
+	CCU_PLLA_RATE(2050000000UL, 0x0b2a2d55, 0x00005500, 0xa0558686),
+	CCU_PLLA_RATE(2100000000UL, 0x0b2b3000, 0x00000000, 0xa0558787),
+	CCU_PLLA_RATE(2150000000UL, 0x0b2c32aa, 0x0000ab00, 0xa0558787),
+	CCU_PLLA_RATE(2200000000UL, 0x0b2d3555, 0x00005500, 0xa0558787),
+	CCU_PLLA_RATE(2250000000UL, 0x0b2e3800, 0x00000000, 0xa0558888),
+	CCU_PLLA_RATE(2300000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0558888),
+	CCU_PLLA_RATE(2350000000UL, 0x0b303d55, 0x00005500, 0xa0558888),
+	CCU_PLLA_RATE(2400000000UL, 0x0b320000, 0x00000000, 0xa0558989),
+};
+
+static const struct ccu_pll_rate_tbl pll4_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
+	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
+	CCU_PLLA_RATE(2050000000UL, 0x0b2a2d55, 0x00005500, 0xa0558686),
+	CCU_PLLA_RATE(2100000000UL, 0x0b2b3000, 0x00000000, 0xa0558787),
+	CCU_PLLA_RATE(2150000000UL, 0x0b2c32aa, 0x0000ab00, 0xa0558787),
+	CCU_PLLA_RATE(2200000000UL, 0x0b2d3555, 0x00005500, 0xa0558787),
+	CCU_PLLA_RATE(2250000000UL, 0x0b2e3800, 0x00000000, 0xa0558888),
+	CCU_PLLA_RATE(2300000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0558888),
+	CCU_PLLA_RATE(2350000000UL, 0x0b303d55, 0x00005500, 0xa0558888),
+	CCU_PLLA_RATE(2400000000UL, 0x0b320000, 0x00000000, 0xa0558989),
+};
+
+static const struct ccu_pll_rate_tbl pll5_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
+	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
+};
+
+static const struct ccu_pll_rate_tbl pll6_rate_tbl[] = {
+	CCU_PLLA_RATE(3200000000UL, 0x0b422aaa, 0x0000ab00, 0xa0558e8e),
+};
+
+static const struct ccu_pll_rate_tbl pll7_rate_tbl[] = {
+	CCU_PLLA_RATE(2800000000UL, 0x0b3a1555, 0x00005500, 0xa0558b8b),
+};
+
+static const struct ccu_pll_rate_tbl pll8_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
+	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
+};
+
+CCU_PLLA_DEFINE(pll1, pll1_rate_tbl, APBS_PLL1_SWCR1, APBS_PLL1_SWCR2, APBS_PLL1_SWCR3,
+		MPMU_POSR, POSR_PLL1_LOCK, CLK_SET_RATE_GATE);
+CCU_PLLA_DEFINE(pll2, pll2_rate_tbl, APBS_PLL2_SWCR1, APBS_PLL2_SWCR2, APBS_PLL2_SWCR3,
+		MPMU_POSR, POSR_PLL2_LOCK, CLK_SET_RATE_GATE);
+CCU_PLLA_DEFINE(pll3, pll3_rate_tbl, APBS_PLL3_SWCR1, APBS_PLL3_SWCR2, APBS_PLL3_SWCR3,
+		MPMU_POSR, POSR_PLL3_LOCK, CLK_SET_RATE_GATE);
+CCU_PLLA_DEFINE(pll4, pll4_rate_tbl, APBS_PLL4_SWCR1, APBS_PLL4_SWCR2, APBS_PLL4_SWCR3,
+		MPMU_POSR, POSR_PLL4_LOCK, CLK_SET_RATE_GATE);
+CCU_PLLA_DEFINE(pll5, pll5_rate_tbl, APBS_PLL5_SWCR1, APBS_PLL5_SWCR2, APBS_PLL5_SWCR3,
+		MPMU_POSR, POSR_PLL5_LOCK, CLK_SET_RATE_GATE);
+CCU_PLLA_DEFINE(pll6, pll6_rate_tbl, APBS_PLL6_SWCR1, APBS_PLL6_SWCR2, APBS_PLL6_SWCR3,
+		MPMU_POSR, POSR_PLL6_LOCK, CLK_SET_RATE_GATE);
+CCU_PLLA_DEFINE(pll7, pll7_rate_tbl, APBS_PLL7_SWCR1, APBS_PLL7_SWCR2, APBS_PLL7_SWCR3,
+		MPMU_POSR, POSR_PLL7_LOCK, CLK_SET_RATE_GATE);
+CCU_PLLA_DEFINE(pll8, pll8_rate_tbl, APBS_PLL8_SWCR1, APBS_PLL8_SWCR2, APBS_PLL8_SWCR3,
+		MPMU_POSR, POSR_PLL8_LOCK, CLK_SET_RATE_GATE);
+
+CCU_FACTOR_GATE_DEFINE(pll1_d2, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d3, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(2), 3, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d4, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d5, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d6, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d7, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_FLAGS_DEFINE(pll1_d8, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(7), 8, 1,
+			     CLK_IS_CRITICAL);
+CCU_DIV_GATE_DEFINE(pll1_dx, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, 23, 5, BIT(22), 0);
+CCU_FACTOR_GATE_FLAGS_DEFINE(pll1_d64_38p4, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2,
+			     BIT(31), 64, 1, CLK_IGNORE_UNUSED);
+CCU_FACTOR_GATE_DEFINE(pll1_aud_245p7, CCU_PARENT_HW(pll1), APBS_PLL1_SWCR2, BIT(21), 10, 1);
+CCU_FACTOR_DEFINE(pll1_aud_24p5, CCU_PARENT_HW(pll1_aud_245p7), 10, 1);
+
+CCU_FACTOR_GATE_DEFINE(pll2_d1, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(0), 1, 1);
+CCU_FACTOR_GATE_DEFINE(pll2_d2, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_DEFINE(pll2_d3, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(2), 3, 1);
+CCU_FACTOR_GATE_DEFINE(pll2_d4, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll2_d5, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll2_d6, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll2_d7, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_DEFINE(pll2_d8, CCU_PARENT_HW(pll2), APBS_PLL2_SWCR2, BIT(7), 8, 1);
+CCU_FACTOR_DEFINE(pll2_66, CCU_PARENT_HW(pll2_d5), 9, 1);
+CCU_FACTOR_DEFINE(pll2_33, CCU_PARENT_HW(pll2_66), 2, 1);
+CCU_FACTOR_DEFINE(pll2_50, CCU_PARENT_HW(pll2_d5), 12, 1);
+CCU_FACTOR_DEFINE(pll2_25, CCU_PARENT_HW(pll2_50), 2, 1);
+CCU_FACTOR_DEFINE(pll2_20, CCU_PARENT_HW(pll2_d5), 30, 1);
+CCU_FACTOR_DEFINE(pll2_d24_125, CCU_PARENT_HW(pll2_d3), 8, 1);
+CCU_FACTOR_DEFINE(pll2_d120_25, CCU_PARENT_HW(pll2_d3), 40, 1);
+
+CCU_FACTOR_GATE_DEFINE(pll3_d1, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(0), 1, 1);
+CCU_FACTOR_GATE_DEFINE(pll3_d2, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_DEFINE(pll3_d3, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(2), 3, 1);
+CCU_FACTOR_GATE_DEFINE(pll3_d4, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll3_d5, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll3_d6, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll3_d7, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_DEFINE(pll3_d8, CCU_PARENT_HW(pll3), APBS_PLL3_SWCR2, BIT(7), 8, 1);
+
+CCU_FACTOR_GATE_DEFINE(pll4_d1, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(0), 1, 1);
+CCU_FACTOR_GATE_DEFINE(pll4_d2, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_DEFINE(pll4_d3, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(2), 3, 1);
+CCU_FACTOR_GATE_DEFINE(pll4_d4, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll4_d5, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll4_d6, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll4_d7, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_DEFINE(pll4_d8, CCU_PARENT_HW(pll4), APBS_PLL4_SWCR2, BIT(7), 8, 1);
+
+CCU_FACTOR_GATE_DEFINE(pll5_d1, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(0), 1, 1);
+CCU_FACTOR_GATE_DEFINE(pll5_d2, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_DEFINE(pll5_d3, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(2), 3, 1);
+CCU_FACTOR_GATE_DEFINE(pll5_d4, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll5_d5, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll5_d6, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll5_d7, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_DEFINE(pll5_d8, CCU_PARENT_HW(pll5), APBS_PLL5_SWCR2, BIT(7), 8, 1);
+
+CCU_FACTOR_GATE_DEFINE(pll6_d1, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(0), 1, 1);
+CCU_FACTOR_GATE_DEFINE(pll6_d2, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_FLAGS_DEFINE(pll6_d3, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(2), 3, 1, CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_DEFINE(pll6_d4, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll6_d5, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll6_d6, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll6_d7, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_DEFINE(pll6_d8, CCU_PARENT_HW(pll6), APBS_PLL6_SWCR2, BIT(7), 8, 1);
+CCU_FACTOR_DEFINE(pll6_80, CCU_PARENT_HW(pll6_d5), 8, 1);
+CCU_FACTOR_DEFINE(pll6_40, CCU_PARENT_HW(pll6_d5), 16, 1);
+CCU_FACTOR_DEFINE(pll6_20, CCU_PARENT_HW(pll6_d5), 32, 1);
+
+CCU_FACTOR_GATE_DEFINE(pll7_d1, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(0), 1, 1);
+CCU_FACTOR_GATE_DEFINE(pll7_d2, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_DEFINE(pll7_d3, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(2), 3, 1);
+CCU_FACTOR_GATE_DEFINE(pll7_d4, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll7_d5, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll7_d6, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll7_d7, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_DEFINE(pll7_d8, CCU_PARENT_HW(pll7), APBS_PLL7_SWCR2, BIT(7), 8, 1);
+
+CCU_FACTOR_GATE_DEFINE(pll8_d1, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(0), 1, 1);
+CCU_FACTOR_GATE_DEFINE(pll8_d2, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(1), 2, 1);
+CCU_FACTOR_GATE_DEFINE(pll8_d3, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(2), 3, 1);
+CCU_FACTOR_GATE_DEFINE(pll8_d4, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(3), 4, 1);
+CCU_FACTOR_GATE_DEFINE(pll8_d5, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(4), 5, 1);
+CCU_FACTOR_GATE_DEFINE(pll8_d6, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(5), 6, 1);
+CCU_FACTOR_GATE_DEFINE(pll8_d7, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(6), 7, 1);
+CCU_FACTOR_GATE_DEFINE(pll8_d8, CCU_PARENT_HW(pll8), APBS_PLL8_SWCR2, BIT(7), 8, 1);
+/* APBS clocks end */
+
+/* MPMU clocks start */
+CCU_GATE_DEFINE(pll1_d8_307p2, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(13), CLK_IS_CRITICAL);
+CCU_FACTOR_DEFINE(pll1_d32_76p8, CCU_PARENT_HW(pll1_d8_307p2), 4, 1);
+CCU_FACTOR_DEFINE(pll1_d40_61p44, CCU_PARENT_HW(pll1_d8_307p2), 5, 1);
+CCU_FACTOR_DEFINE(pll1_d16_153p6, CCU_PARENT_HW(pll1_d8), 2, 1);
+CCU_FACTOR_GATE_FLAGS_DEFINE(pll1_d24_102p4, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(12), 3, 1, CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_FLAGS_DEFINE(pll1_d48_51p2, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(7), 6, 1, CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_FLAGS_DEFINE(pll1_d48_51p2_ap, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(11), 6, 1, CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_DEFINE(pll1_m3d128_57p6, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(8), 16, 3);
+CCU_FACTOR_GATE_FLAGS_DEFINE(pll1_d96_25p6, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(4), 12, 1, CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_DEFINE(pll1_d192_12p8, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(3), 24, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d192_12p8_wdt, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(19), 24, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d384_6p4, CCU_PARENT_HW(pll1_d8), MPMU_ACGR, BIT(2), 48, 1);
+
+CCU_FACTOR_DEFINE(pll1_d768_3p2, CCU_PARENT_HW(pll1_d384_6p4), 2, 1);
+CCU_FACTOR_DEFINE(pll1_d1536_1p6, CCU_PARENT_HW(pll1_d384_6p4), 4, 1);
+CCU_FACTOR_DEFINE(pll1_d3072_0p8, CCU_PARENT_HW(pll1_d384_6p4), 8, 1);
+
+CCU_GATE_DEFINE(pll1_d6_409p6, CCU_PARENT_HW(pll1_d6), MPMU_ACGR, BIT(0), CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_DEFINE(pll1_d12_204p8, CCU_PARENT_HW(pll1_d6), MPMU_ACGR, BIT(5), 2, 1);
+
+CCU_GATE_DEFINE(pll1_d5_491p52, CCU_PARENT_HW(pll1_d5), MPMU_ACGR, BIT(21), CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_DEFINE(pll1_d10_245p76, CCU_PARENT_HW(pll1_d5), MPMU_ACGR, BIT(18), 2, 1);
+
+CCU_GATE_DEFINE(pll1_d4_614p4, CCU_PARENT_HW(pll1_d4), MPMU_ACGR, BIT(15), CLK_IS_CRITICAL);
+CCU_FACTOR_GATE_DEFINE(pll1_d52_47p26, CCU_PARENT_HW(pll1_d4), MPMU_ACGR, BIT(10), 13, 1);
+CCU_FACTOR_GATE_DEFINE(pll1_d78_31p5, CCU_PARENT_HW(pll1_d4), MPMU_ACGR, BIT(6), 39, 2);
+
+CCU_GATE_DEFINE(pll1_d3_819p2, CCU_PARENT_HW(pll1_d3), MPMU_ACGR, BIT(14), CLK_IS_CRITICAL);
+
+CCU_GATE_DEFINE(pll1_d2_1228p8, CCU_PARENT_HW(pll1_d2), MPMU_ACGR, BIT(16), CLK_IS_CRITICAL);
+
+static const struct clk_parent_data apb_parents[] = {
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d24_102p4),
+};
+CCU_MUX_DEFINE(apb_clk, apb_parents, MPMU_APBCSCR, 0, 2, CLK_IS_CRITICAL);
+
+CCU_GATE_DEFINE(slow_uart, CCU_PARENT_NAME(osc_32k), MPMU_ACGR, BIT(1), CLK_IGNORE_UNUSED);
+CCU_DDN_DEFINE(slow_uart1_14p74, pll1_d16_153p6, MPMU_SUCCR, 16, 13, 0, 13, 2, 0);
+CCU_DDN_DEFINE(slow_uart2_48, pll1_d4_614p4, MPMU_SUCCR_1, 16, 13, 0, 13, 2, 0);
+
+CCU_GATE_DEFINE(wdt_clk, CCU_PARENT_HW(pll1_d96_25p6), MPMU_WDTPCR, BIT(1), 0);
+CCU_GATE_DEFINE(wdt_bus_clk, CCU_PARENT_HW(apb_clk), MPMU_WDTPCR, BIT(0), 0);
+
+CCU_GATE_DEFINE(r_ipc_clk, CCU_PARENT_HW(apb_clk), MPMU_RIPCCR, BIT(0), 0);
+
+CCU_FACTOR_DEFINE(i2s_153p6, CCU_PARENT_HW(pll1_d8_307p2), 2, 1);
+
+static const struct clk_parent_data i2s_153p6_base_parents[] = {
+	CCU_PARENT_HW(i2s_153p6),
+	CCU_PARENT_HW(pll1_d8_307p2),
+};
+CCU_MUX_DEFINE(i2s_153p6_base, i2s_153p6_base_parents, MPMU_FCCR, 29, 1, 0);
+
+static const struct clk_parent_data i2s_sysclk_src_parents[] = {
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(i2s_153p6_base),
+};
+CCU_MUX_GATE_DEFINE(i2s_sysclk_src, i2s_sysclk_src_parents, MPMU_ISCCR1, 30, 1, BIT(31), 0);
+
+CCU_DDN_DEFINE(i2s_sysclk, i2s_sysclk_src, MPMU_ISCCR1, 0, 15, 15, 12, 1, 0);
+
+CCU_FACTOR_DEFINE(i2s_bclk_factor, CCU_PARENT_HW(i2s_sysclk), 2, 1);
+/*
+ * Divider of i2s_bclk always implies a 1/2 factor, which is
+ * described by i2s_bclk_factor.
+ */
+CCU_DIV_GATE_DEFINE(i2s_bclk, CCU_PARENT_HW(i2s_bclk_factor), MPMU_ISCCR1, 27, 2, BIT(29), 0);
+
+CCU_MUX_GATE_DEFINE(i2s1_sysclk_src, i2s_sysclk_src_parents, MPMU_ISCCR0, 30, 1, BIT(31), 0);
+CCU_DDN_DEFINE(i2s1_sysclk, i2s1_sysclk_src, MPMU_ISCCR0, 0, 15, 15, 12, 1, 0);
+
+static const struct clk_parent_data i2s_sysclk_parents[] = {
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_NAME(vctcxo_24m),
+	CCU_PARENT_HW(pll2_d5),
+	CCU_PARENT_NAME(vctcxo_24m),
+};
+CCU_MUX_DEFINE(i2s0_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 0, 2, 0);
+CCU_MUX_DEFINE(i2s2_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 4, 2, 0);
+CCU_MUX_DEFINE(i2s3_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 12, 2, 0);
+CCU_MUX_DEFINE(i2s4_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 16, 2, 0);
+CCU_MUX_DEFINE(i2s5_sysclk_sel, i2s_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 20, 2, 0);
+
+CCU_DDN_DEFINE(i2s0_sysclk_div, i2s0_sysclk_sel, MPMU_I2S0_SYSCLK, 0, 16, 16, 16, 2, 0);
+CCU_DDN_DEFINE(i2s2_sysclk_div, i2s2_sysclk_sel, MPMU_I2S2_SYSCLK, 0, 16, 16, 16, 2, 0);
+CCU_DDN_DEFINE(i2s3_sysclk_div, i2s3_sysclk_sel, MPMU_I2S3_SYSCLK, 0, 16, 16, 16, 2, 0);
+CCU_DDN_DEFINE(i2s4_sysclk_div, i2s4_sysclk_sel, MPMU_I2S4_SYSCLK, 0, 16, 16, 16, 2, 0);
+CCU_DDN_DEFINE(i2s5_sysclk_div, i2s5_sysclk_sel, MPMU_I2S5_SYSCLK, 0, 16, 16, 16, 2, 0);
+
+static const struct clk_parent_data i2s2_sysclk_parents[] = {
+	CCU_PARENT_HW(i2s_sysclk),
+	CCU_PARENT_HW(i2s2_sysclk_div),
+};
+
+CCU_GATE_DEFINE(i2s0_sysclk, CCU_PARENT_HW(i2s0_sysclk_div), MPMU_I2S_SYSCLK_CTRL, BIT(2), 0);
+CCU_MUX_GATE_DEFINE(i2s2_sysclk, i2s2_sysclk_parents, MPMU_I2S_SYSCLK_CTRL, 8, 1, BIT(6), 0);
+CCU_GATE_DEFINE(i2s3_sysclk, CCU_PARENT_HW(i2s3_sysclk_div), MPMU_I2S_SYSCLK_CTRL, BIT(14), 0);
+CCU_GATE_DEFINE(i2s4_sysclk, CCU_PARENT_HW(i2s4_sysclk_div), MPMU_I2S_SYSCLK_CTRL, BIT(18), 0);
+CCU_GATE_DEFINE(i2s5_sysclk, CCU_PARENT_HW(i2s5_sysclk_div), MPMU_I2S_SYSCLK_CTRL, BIT(22), 0);
+/* MPMU clocks end */
+
+/* APBC clocks start */
+static const struct clk_parent_data uart_clk_parents[] = {
+	CCU_PARENT_HW(pll1_m3d128_57p6),
+	CCU_PARENT_HW(slow_uart1_14p74),
+	CCU_PARENT_HW(slow_uart2_48),
+};
+CCU_MUX_GATE_DEFINE(uart0_clk, uart_clk_parents, APBC_UART0_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart2_clk, uart_clk_parents, APBC_UART2_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart3_clk, uart_clk_parents, APBC_UART3_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart4_clk, uart_clk_parents, APBC_UART4_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart5_clk, uart_clk_parents, APBC_UART5_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart6_clk, uart_clk_parents, APBC_UART6_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart7_clk, uart_clk_parents, APBC_UART7_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart8_clk, uart_clk_parents, APBC_UART8_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart9_clk, uart_clk_parents, APBC_UART9_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(uart10_clk, uart_clk_parents, APBC_UART10_CLK_RST, 4, 3, BIT(1), 0);
+
+CCU_GATE_DEFINE(uart0_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart2_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART2_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart3_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART3_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart4_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART4_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart5_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART5_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart6_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART6_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart7_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART7_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart8_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART8_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart9_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART9_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(uart10_bus_clk, CCU_PARENT_HW(apb_clk), APBC_UART10_CLK_RST, BIT(0), 0);
+
+CCU_GATE_DEFINE(gpio_clk, CCU_PARENT_NAME(vctcxo_24m), APBC_GPIO_CLK_RST, BIT(1), 0);
+CCU_GATE_DEFINE(gpio_bus_clk, CCU_PARENT_HW(apb_clk), APBC_GPIO_CLK_RST, BIT(0), 0);
+
+static const struct clk_parent_data pwm_parents[] = {
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_NAME(osc_32k),
+};
+CCU_MUX_GATE_DEFINE(pwm0_clk, pwm_parents, APBC_PWM0_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm1_clk, pwm_parents, APBC_PWM1_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm2_clk, pwm_parents, APBC_PWM2_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm3_clk, pwm_parents, APBC_PWM3_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm4_clk, pwm_parents, APBC_PWM4_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm5_clk, pwm_parents, APBC_PWM5_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm6_clk, pwm_parents, APBC_PWM6_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm7_clk, pwm_parents, APBC_PWM7_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm8_clk, pwm_parents, APBC_PWM8_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm9_clk, pwm_parents, APBC_PWM9_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm10_clk, pwm_parents, APBC_PWM10_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm11_clk, pwm_parents, APBC_PWM11_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm12_clk, pwm_parents, APBC_PWM12_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm13_clk, pwm_parents, APBC_PWM13_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm14_clk, pwm_parents, APBC_PWM14_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm15_clk, pwm_parents, APBC_PWM15_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm16_clk, pwm_parents, APBC_PWM16_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm17_clk, pwm_parents, APBC_PWM17_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm18_clk, pwm_parents, APBC_PWM18_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(pwm19_clk, pwm_parents, APBC_PWM19_CLK_RST, 4, 3, BIT(1), 0);
+
+CCU_GATE_DEFINE(pwm0_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm1_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM1_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm2_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM2_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm3_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM3_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm4_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM4_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm5_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM5_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm6_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM6_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm7_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM7_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm8_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM8_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm9_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM9_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm10_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM10_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm11_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM11_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm12_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM12_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm13_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM13_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm14_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM14_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm15_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM15_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm16_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM16_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm17_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM17_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm18_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM18_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(pwm19_bus_clk, CCU_PARENT_HW(apb_clk), APBC_PWM19_CLK_RST, BIT(0), 0);
+
+static const struct clk_parent_data i2s_bclk_parents[] = {
+	CCU_PARENT_NAME(vctcxo_1m),
+	CCU_PARENT_HW(i2s_bclk),
+};
+
+CCU_MUX_DEFINE(spi0_i2s_bclk, i2s_bclk_parents, APBC_SSP0_CLK_RST, 3, 1, 0);
+CCU_MUX_DEFINE(spi1_i2s_bclk, i2s_bclk_parents, APBC_SSP1_CLK_RST, 3, 1, 0);
+CCU_MUX_DEFINE(spi3_i2s_bclk, i2s_bclk_parents, APBC_SSP3_CLK_RST, 3, 1, 0);
+
+static const struct clk_parent_data spi0_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(spi0_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(spi0_clk, spi0_parents, APBC_SSP0_CLK_RST, 4, 3, BIT(1), 0);
+static const struct clk_parent_data spi1_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(spi1_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(spi1_clk, spi1_parents, APBC_SSP1_CLK_RST, 4, 3, BIT(1), 0);
+static const struct clk_parent_data spi3_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(spi3_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(spi3_clk, spi3_parents, APBC_SSP3_CLK_RST, 4, 3, BIT(1), 0);
+
+CCU_GATE_DEFINE(spi0_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSP0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(spi1_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSP1_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(spi3_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSP3_CLK_RST, BIT(0), 0);
+
+
+CCU_GATE_DEFINE(rtc_clk, CCU_PARENT_NAME(osc_32k), APBC_RTC_CLK_RST,
+		BIT(7) | BIT(1), 0);
+CCU_GATE_DEFINE(rtc_bus_clk, CCU_PARENT_HW(apb_clk), APBC_RTC_CLK_RST, BIT(0), 0);
+
+static const struct clk_parent_data twsi_parents[] = {
+	CCU_PARENT_HW(pll1_d78_31p5),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d40_61p44),
+};
+CCU_MUX_GATE_DEFINE(twsi0_clk, twsi_parents, APBC_TWSI0_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(twsi1_clk, twsi_parents, APBC_TWSI1_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(twsi2_clk, twsi_parents, APBC_TWSI2_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(twsi4_clk, twsi_parents, APBC_TWSI4_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(twsi5_clk, twsi_parents, APBC_TWSI5_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(twsi6_clk, twsi_parents, APBC_TWSI6_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(twsi8_clk, twsi_parents, APBC_TWSI8_CLK_RST, 4, 3, BIT(1), 0);
+
+CCU_GATE_DEFINE(twsi0_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TWSI0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(twsi1_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TWSI1_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(twsi2_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TWSI2_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(twsi4_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TWSI4_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(twsi5_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TWSI5_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(twsi6_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TWSI6_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(twsi8_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TWSI8_CLK_RST, BIT(0), 0);
+
+static const struct clk_parent_data timer_parents[] = {
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_NAME(osc_32k),
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_NAME(vctcxo_3m),
+	CCU_PARENT_NAME(vctcxo_1m),
+};
+CCU_MUX_GATE_DEFINE(timers0_clk, timer_parents, APBC_TIMERS0_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(timers1_clk, timer_parents, APBC_TIMERS1_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(timers2_clk, timer_parents, APBC_TIMERS2_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(timers3_clk, timer_parents, APBC_TIMERS3_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(timers4_clk, timer_parents, APBC_TIMERS4_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(timers5_clk, timer_parents, APBC_TIMERS5_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(timers6_clk, timer_parents, APBC_TIMERS6_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(timers7_clk, timer_parents, APBC_TIMERS7_CLK_RST, 4, 3, BIT(1), 0);
+
+CCU_GATE_DEFINE(timers0_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(timers1_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS1_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(timers2_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS2_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(timers3_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS3_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(timers4_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS4_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(timers5_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS5_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(timers6_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS6_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(timers7_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TIMERS7_CLK_RST, BIT(0), 0);
+
+
+CCU_GATE_DEFINE(aib_clk, CCU_PARENT_NAME(vctcxo_24m), APBC_AIB_CLK_RST, BIT(1), 0);
+CCU_GATE_DEFINE(aib_bus_clk, CCU_PARENT_HW(apb_clk), APBC_AIB_CLK_RST, BIT(0), 0);
+
+CCU_GATE_DEFINE(onewire_clk, CCU_PARENT_NAME(vctcxo_24m), APBC_ONEWIRE_CLK_RST, BIT(1), 0);
+CCU_GATE_DEFINE(onewire_bus_clk, CCU_PARENT_HW(apb_clk), APBC_ONEWIRE_CLK_RST, BIT(0), 0);
+
+/*
+ * When i2s_bclk is selected as the parent clock of sspa,
+ * the hardware requires bit3 to be set
+ */
+
+CCU_MUX_DEFINE(i2s0_i2s_bclk, i2s_bclk_parents, APBC_SSPA0_CLK_RST, 3, 1, 0);
+CCU_MUX_DEFINE(i2s1_i2s_bclk, i2s_bclk_parents, APBC_SSPA1_CLK_RST, 3, 1, 0);
+CCU_MUX_DEFINE(i2s2_i2s_bclk, i2s_bclk_parents, APBC_SSPA2_CLK_RST, 3, 1, 0);
+CCU_MUX_DEFINE(i2s3_i2s_bclk, i2s_bclk_parents, APBC_SSPA3_CLK_RST, 3, 1, 0);
+CCU_MUX_DEFINE(i2s4_i2s_bclk, i2s_bclk_parents, APBC_SSPA4_CLK_RST, 3, 1, 0);
+CCU_MUX_DEFINE(i2s5_i2s_bclk, i2s_bclk_parents, APBC_SSPA5_CLK_RST, 3, 1, 0);
+
+static const struct clk_parent_data i2s0_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(i2s0_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(i2s0_clk, i2s0_parents, APBC_SSPA0_CLK_RST, 4, 3, BIT(1), 0);
+
+static const struct clk_parent_data i2s1_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(i2s1_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(i2s1_clk, i2s1_parents, APBC_SSPA1_CLK_RST, 4, 3, BIT(1), 0);
+
+static const struct clk_parent_data i2s2_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(i2s2_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(i2s2_clk, i2s2_parents, APBC_SSPA2_CLK_RST, 4, 3, BIT(1), 0);
+
+static const struct clk_parent_data i2s3_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(i2s3_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(i2s3_clk, i2s3_parents, APBC_SSPA3_CLK_RST, 4, 3, BIT(1), 0);
+
+static const struct clk_parent_data i2s4_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(i2s4_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(i2s4_clk, i2s4_parents, APBC_SSPA4_CLK_RST, 4, 3, BIT(1), 0);
+
+static const struct clk_parent_data i2s5_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(i2s5_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(i2s5_clk, i2s5_parents, APBC_SSPA5_CLK_RST, 4, 3, BIT(1), 0);
+
+CCU_GATE_DEFINE(i2s0_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSPA0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(i2s1_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSPA1_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(i2s2_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSPA2_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(i2s3_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSPA3_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(i2s4_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSPA4_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(i2s5_bus_clk, CCU_PARENT_HW(apb_clk), APBC_SSPA5_CLK_RST, BIT(0), 0);
+
+CCU_GATE_DEFINE(dro_clk, CCU_PARENT_HW(apb_clk), APBC_DRO_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(ir0_clk, CCU_PARENT_HW(apb_clk), APBC_IR0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(ir1_clk, CCU_PARENT_HW(apb_clk), APBC_IR1_CLK_RST, BIT(0), 0);
+
+CCU_GATE_DEFINE(tsen_clk, CCU_PARENT_HW(apb_clk), APBC_TSEN_CLK_RST, BIT(1), 0);
+CCU_GATE_DEFINE(tsen_bus_clk, CCU_PARENT_HW(apb_clk), APBC_TSEN_CLK_RST, BIT(0), 0);
+
+CCU_GATE_DEFINE(ipc_ap2rcpu_clk, CCU_PARENT_HW(apb_clk), APBC_IPC_AP2AUD_CLK_RST, BIT(1), 0);
+CCU_GATE_DEFINE(ipc_ap2rcpu_bus_clk, CCU_PARENT_HW(apb_clk), APBC_IPC_AP2AUD_CLK_RST, BIT(0), 0);
+
+static const struct clk_parent_data can_parents[] = {
+	CCU_PARENT_HW(pll6_20),
+	CCU_PARENT_HW(pll6_40),
+	CCU_PARENT_HW(pll6_80),
+};
+CCU_MUX_GATE_DEFINE(can0_clk, can_parents, APBC_CAN0_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(can1_clk, can_parents, APBC_CAN1_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(can2_clk, can_parents, APBC_CAN2_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(can3_clk, can_parents, APBC_CAN3_CLK_RST, 4, 3, BIT(1), 0);
+CCU_MUX_GATE_DEFINE(can4_clk, can_parents, APBC_CAN4_CLK_RST, 4, 3, BIT(1), 0);
+
+CCU_GATE_DEFINE(can0_bus_clk, CCU_PARENT_HW(apb_clk), APBC_CAN0_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(can1_bus_clk, CCU_PARENT_HW(apb_clk), APBC_CAN1_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(can2_bus_clk, CCU_PARENT_HW(apb_clk), APBC_CAN2_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(can3_bus_clk, CCU_PARENT_HW(apb_clk), APBC_CAN3_CLK_RST, BIT(0), 0);
+CCU_GATE_DEFINE(can4_bus_clk, CCU_PARENT_HW(apb_clk), APBC_CAN4_CLK_RST, BIT(0), 0);
+/* APBC clocks end */
+
+/* APMU clocks start */
+static const struct clk_parent_data axi_clk_parents[] = {
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll1_d6_409p6),
+};
+CCU_MUX_DIV_FC_DEFINE(axi_clk, axi_clk_parents, APMU_ACLK_CLK_CTRL, 1, 2, BIT(4), 0, 1, CLK_IS_CRITICAL);
+
+static const struct clk_parent_data cci550_clk_parents[] = {
+	CCU_PARENT_HW(pll1_d10_245p76),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll7_d3),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+	CCU_PARENT_HW(pll7_d2),
+};
+CCU_MUX_GATE_FC_DEFINE(cci550_clk, cci550_clk_parents, APMU_CCI550_CLK_CTRL, BIT(12), 0, 3, BIT(13),
+		      CLK_IS_CRITICAL);
+
+static const struct clk_parent_data cpu_c0_clk_parents[] = {
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW_NULL(),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+	CCU_PARENT_HW(pll2_d2),
+	CCU_PARENT_HW(pll3_d1),
+};
+CCU_MUX_DIV_FC_DEFINE(cpu_c0_core_clk, cpu_c0_clk_parents, APMU_CPU_C0_CLK_CTRL,
+		      3, 3, BIT(12), 0, 3, CLK_IS_CRITICAL | CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+static const struct clk_parent_data cpu_c1_pll_src_parents[] = {
+	CCU_PARENT_HW(pll4_d1),
+	CCU_PARENT_HW(pll3_d1),
+};
+CCU_MUX_DEFINE(cpu_c1_pll_src, cpu_c1_pll_src_parents, APMU_CPU_C1_CLK_CTRL, 13, 1, 0);
+
+static const struct clk_parent_data cpu_c1_clk_parents[] = {
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW_NULL(),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+	CCU_PARENT_HW(pll2_d2),
+	CCU_PARENT_HW(cpu_c1_pll_src),
+};
+CCU_MUX_DIV_FC_DEFINE(cpu_c1_core_clk, cpu_c1_clk_parents, APMU_CPU_C1_CLK_CTRL,
+		      3, 3, BIT(12), 0, 3, CLK_IS_CRITICAL | CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+static const struct clk_parent_data cpu_c2_clk_parents[] = {
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW_NULL(),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+	CCU_PARENT_HW(pll2_d2),
+	CCU_PARENT_HW(pll5_d1),
+};
+CCU_MUX_DIV_FC_DEFINE(cpu_c2_core_clk, cpu_c2_clk_parents, APMU_CPU_C2_CLK_CTRL,
+		      3, 3, BIT(12), 0, 3, CLK_IS_CRITICAL | CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+static const struct clk_parent_data cpu_c3_pll_src_parents[] = {
+	CCU_PARENT_HW(pll8_d1),
+	CCU_PARENT_HW(pll5_d1),
+};
+CCU_MUX_DEFINE(cpu_c3_pll_src, cpu_c3_pll_src_parents, APMU_CPU_C3_CLK_CTRL, 13, 1, 0);
+
+static const struct clk_parent_data cpu_c3_clk_parents[] = {
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW_NULL(),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+	CCU_PARENT_HW(pll2_d2),
+	CCU_PARENT_HW(cpu_c3_pll_src),
+};
+CCU_MUX_DIV_FC_DEFINE(cpu_c3_core_clk, cpu_c3_clk_parents, APMU_CPU_C3_CLK_CTRL,
+		      3, 3, BIT(12), 0, 3, CLK_IS_CRITICAL | CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+static const struct clk_parent_data ccic2phy_parents[] = {
+	CCU_PARENT_HW(pll1_d24_102p4),
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+};
+CCU_MUX_GATE_DEFINE(ccic2phy_clk, ccic2phy_parents, APMU_CSI_CCIC2_CLK_RES_CTRL, 7, 1, BIT(5), 0);
+
+static const struct clk_parent_data ccic3phy_parents[] = {
+	CCU_PARENT_HW(pll1_d24_102p4),
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+};
+CCU_MUX_GATE_DEFINE(ccic3phy_clk, ccic3phy_parents, APMU_CSI_CCIC2_CLK_RES_CTRL, 31, 1, BIT(30), 0);
+
+static const struct clk_parent_data csi_parents[] = {
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll2_d2),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(csi_clk, csi_parents, APMU_CSI_CCIC2_CLK_RES_CTRL, 20, 3, BIT(15),
+			   16, 3, BIT(4), CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+static const struct clk_parent_data isp_bus_parents[] = {
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d10_245p76),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(isp_bus_clk, isp_bus_parents, APMU_ISP_CLK_RES_CTRL, 18, 3, BIT(23),
+			   21, 2, BIT(17), 0);
+
+CCU_GATE_DEFINE(d1p_1228p8, CCU_PARENT_HW(pll1_d2_1228p8), APMU_PMU_CLK_GATE_CTRL, BIT(31), 0);
+CCU_GATE_DEFINE(d1p_819p2, CCU_PARENT_HW(pll1_d3_819p2), APMU_PMU_CLK_GATE_CTRL, BIT(30), 0);
+CCU_GATE_DEFINE(d1p_614p4, CCU_PARENT_HW(pll1_d4_614p4), APMU_PMU_CLK_GATE_CTRL, BIT(29), 0);
+CCU_GATE_DEFINE(d1p_491p52, CCU_PARENT_HW(pll1_d5_491p52), APMU_PMU_CLK_GATE_CTRL, BIT(28), 0);
+CCU_GATE_DEFINE(d1p_409p6, CCU_PARENT_HW(pll1_d6_409p6), APMU_PMU_CLK_GATE_CTRL, BIT(27), 0);
+CCU_GATE_DEFINE(d1p_307p2, CCU_PARENT_HW(pll1_d8_307p2), APMU_PMU_CLK_GATE_CTRL, BIT(26), 0);
+CCU_GATE_DEFINE(d1p_245p76, CCU_PARENT_HW(pll1_d10_245p76), APMU_PMU_CLK_GATE_CTRL, BIT(22), 0);
+
+static const struct clk_parent_data v2d_parents[] = {
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll1_d4_614p4),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(v2d_clk, v2d_parents, APMU_LCD_CLK_RES_CTRL1, 9, 3, BIT(28), 12, 2,
+			   BIT(8), 0);
+
+static const struct clk_parent_data dsiesc_parents[] = {
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+	CCU_PARENT_HW(pll1_d52_47p26),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d32_76p8),
+};
+CCU_MUX_GATE_DEFINE(dsi_esc_clk, dsiesc_parents, APMU_LCD_CLK_RES_CTRL1, 0, 2, BIT(2), 0);
+
+CCU_GATE_DEFINE(lcd_hclk, CCU_PARENT_HW(axi_clk), APMU_LCD_CLK_RES_CTRL1, BIT(5), 0);
+
+static const struct clk_parent_data lcd_dsc_parents[] = {
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d10_245p76),
+	CCU_PARENT_HW(pll7_d5),
+	CCU_PARENT_HW(pll2_d7),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+	CCU_PARENT_HW(pll2_d8),
+};
+CCU_MUX_DIV_GATE_SPLIT_FC_DEFINE(lcd_dsc_clk, lcd_dsc_parents, APMU_LCD_CLK_RES_CTRL2,
+				 APMU_LCD_CLK_RES_CTRL1, 25, 3, BIT(26), 29, 3, BIT(14), 0);
+
+static const struct clk_parent_data lcdpx_parents[] = {
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll7_d5),
+	CCU_PARENT_HW(pll6_d6),
+	CCU_PARENT_HW(pll2_d7),
+	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+	CCU_PARENT_HW(pll2_d8),
+};
+CCU_MUX_DIV_GATE_SPLIT_FC_DEFINE(lcd_pxclk, lcdpx_parents, APMU_LCD_CLK_RES_CTRL2,
+				 APMU_LCD_CLK_RES_CTRL1, 17, 3, BIT(30), 21, 3, BIT(16), 0);
+
+static const struct clk_parent_data lcdmclk_parents[] = {
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d8_307p2),
+};
+CCU_MUX_DIV_GATE_SPLIT_FC_DEFINE(lcd_mclk, lcdmclk_parents, APMU_LCD_CLK_RES_CTRL2,
+				 APMU_LCD_CLK_RES_CTRL1, 1, 4, BIT(29), 5, 3, BIT(0),
+				 CCU_DIV_4BIT_NO_0F_FLAG);
+
+static const struct clk_parent_data ccic_4x_parents[] = {
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll2_d2),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(ccic_4x_clk, ccic_4x_parents, APMU_CCIC_CLK_RES_CTRL, 18, 3,
+			   BIT(15), 23, 2, BIT(4), CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+static const struct clk_parent_data ccic1phy_parents[] = {
+	CCU_PARENT_HW(pll1_d24_102p4),
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+};
+CCU_MUX_GATE_DEFINE(ccic1phy_clk, ccic1phy_parents, APMU_CCIC_CLK_RES_CTRL, 7, 1, BIT(5), 0);
+
+
+static const struct clk_parent_data sc2hclk_parents[] = {
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll2_d4),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(sc2_hclk, sc2hclk_parents, APMU_CCIC_CLK_RES_CTRL, 10, 3,
+			   BIT(16), 8, 2, BIT(3), 0);
+
+CCU_GATE_DEFINE(sdh_axi_aclk, CCU_PARENT_HW(axi_clk), APMU_SDH0_CLK_RES_CTRL, BIT(3), 0);
+static const struct clk_parent_data sdh01_parents[] = {
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll2_d8),
+	CCU_PARENT_HW(pll2_d5),
+	CCU_PARENT_NAME(reserved_clk),
+	CCU_PARENT_NAME(reserved_clk),
+	CCU_PARENT_HW(pll1_dx),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(sdh0_clk, sdh01_parents, APMU_SDH0_CLK_RES_CTRL, 8, 3,
+			   BIT(11), 5, 3, BIT(4), 0);
+CCU_MUX_DIV_GATE_FC_DEFINE(sdh1_clk, sdh01_parents, APMU_SDH1_CLK_RES_CTRL, 8, 3,
+			   BIT(11), 5, 3, BIT(4), 0);
+static const struct clk_parent_data sdh2_parents[] = {
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll2_d8),
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_NAME(reserved_clk),
+	CCU_PARENT_NAME(reserved_clk),
+	CCU_PARENT_HW(pll1_dx),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(sdh2_clk, sdh2_parents, APMU_SDH2_CLK_RES_CTRL, 8, 3,
+			   BIT(11), 5, 3, BIT(4), 0);
+
+CCU_GATE_DEFINE(usb2_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(1), 0);
+CCU_GATE_DEFINE(usb3_porta_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(4), 0);
+CCU_GATE_DEFINE(usb3_portb_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(8), 0);
+CCU_GATE_DEFINE(usb3_portc_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(12), 0);
+CCU_GATE_DEFINE(usb3_portd_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(16), 0);
+
+static const struct clk_parent_data qspi_parents[] = {
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll2_d8),
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll1_d10_245p76),
+	CCU_PARENT_NAME(reserved_clk),
+	CCU_PARENT_HW(pll1_dx),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_NAME(reserved_clk),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(qspi_clk, qspi_parents, APMU_QSPI_CLK_RES_CTRL, 9, 3,
+			   BIT(12), 6, 3, BIT(4), 0);
+CCU_GATE_DEFINE(qspi_bus_clk, CCU_PARENT_HW(axi_clk), APMU_QSPI_CLK_RES_CTRL, BIT(3), 0);
+
+CCU_GATE_DEFINE(dma_clk, CCU_PARENT_HW(axi_clk), APMU_DMA_CLK_RES_CTRL, BIT(3), 0);
+
+static const struct clk_parent_data aes_wtm_parents[] = {
+	CCU_PARENT_HW(pll1_d12_204p8),
+	CCU_PARENT_HW(pll1_d24_102p4),
+};
+CCU_MUX_GATE_DEFINE(aes_wtm_clk, aes_wtm_parents, APMU_AES_CLK_RES_CTRL, 6, 1, BIT(5), 0);
+
+static const struct clk_parent_data vpu_parents[] = {
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll2_d5),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(vpu_clk, vpu_parents, APMU_VPU_CLK_RES_CTRL, 13, 3,
+			   BIT(21), 10, 3, BIT(3), CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+CCU_GATE_DEFINE(dtc_clk, CCU_PARENT_HW(axi_clk), APMU_DTC_CLK_RES_CTRL, BIT(3), 0);
+
+static const struct clk_parent_data gpu_parents[] = {
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d3_819p2),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d2_1228p8),
+	CCU_PARENT_HW(pll2_d3),
+	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll2_d5),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(gpu_clk, gpu_parents, APMU_GPU_CLK_RES_CTRL, 12, 3,
+			   BIT(15), 18, 3, BIT(4), CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+CCU_GATE_DEFINE(mc_ahb_clk, CCU_PARENT_HW(axi_clk), APMU_PMUA_MC_CTRL, BIT(1), 0);
+
+static const struct clk_parent_data top_parents[] = {
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll3_d4),
+	CCU_PARENT_HW(pll6_d5),
+	CCU_PARENT_HW(pll7_d4),
+	CCU_PARENT_HW(pll6_d4),
+	CCU_PARENT_HW(pll7_d3),
+	CCU_PARENT_HW(pll6_d3),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(top_dclk, top_parents, APMU_TOP_DCLK_CTRL, 5, 3,
+			   BIT(8), 2, 3, BIT(1), CLK_IS_CRITICAL | CCU_DIV_VALID_FIRST4_SRC_FLAG);
+
+static const struct clk_parent_data ucie_parents[] = {
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll3_d4),
+	CCU_PARENT_HW(pll6_d5),
+	CCU_PARENT_HW(pll7_d4),
+	CCU_PARENT_HW(pll6_d4),
+};
+CCU_MUX_GATE_DEFINE(ucie_clk, ucie_parents, APMU_UCIE_CTRL, 4, 3, BIT(0), 0);
+CCU_GATE_DEFINE(ucie_sbclk, CCU_PARENT_HW(axi_clk), APMU_UCIE_CTRL, BIT(8), 0);
+
+static const struct clk_parent_data rcpu_clk_parents[] = {
+	CCU_PARENT_HW(pll1_aud_245p7),
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d6_409p6),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(rcpu_clk, rcpu_clk_parents, APMU_RCPU_CLK_RES_CTRL,
+			   4, 3, BIT(15), 7, 3, BIT(12), 0);
+
+static const struct clk_parent_data dsi4ln2_dsi_esc_parents[] = {
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+	CCU_PARENT_HW(pll1_d52_47p26),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d32_76p8),
+};
+CCU_MUX_GATE_DEFINE(dsi4ln2_dsi_esc_clk, dsi4ln2_dsi_esc_parents, APMU_LCD_CLK_RES_CTRL3,
+		    0, 1, BIT(2), 0);
+
+static const struct clk_parent_data dsi4ln2_lcd_dsc_parents[] = {
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll7_d5),
+	CCU_PARENT_HW(pll6_d6),
+	CCU_PARENT_HW(pll2_d7),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+};
+CCU_MUX_DIV_GATE_SPLIT_FC_DEFINE(dsi4ln2_lcd_dsc_clk, dsi4ln2_lcd_dsc_parents,
+				 APMU_LCD_CLK_RES_CTRL4, APMU_LCD_CLK_RES_CTRL3,
+				 25, 3, BIT(26), 29, 3, BIT(14), 0);
+
+static const struct clk_parent_data dsi4ln2_lcdpx_parents[] = {
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll7_d5),
+	CCU_PARENT_HW(pll6_d6),
+	CCU_PARENT_HW(pll2_d7),
+	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll1_d48_51p2_ap),
+	CCU_PARENT_HW(pll2_d8),
+};
+CCU_MUX_DIV_GATE_SPLIT_FC_DEFINE(dsi4ln2_lcd_pxclk, dsi4ln2_lcdpx_parents, APMU_LCD_CLK_RES_CTRL4,
+				 APMU_LCD_CLK_RES_CTRL3, 17, 3, BIT(30), 21, 3, BIT(16), 0);
+
+static const struct clk_parent_data dsi4ln2_lcd_mclk_parents[] = {
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d8_307p2),
+};
+CCU_MUX_DIV_GATE_SPLIT_FC_DEFINE(dsi4ln2_lcd_mclk, dsi4ln2_lcd_mclk_parents, APMU_LCD_CLK_RES_CTRL4,
+				 APMU_LCD_CLK_RES_CTRL3, 1, 4, BIT(29), 5, 3, BIT(0),
+				 CCU_DIV_4BIT_NO_0F_FLAG);
+
+static const struct clk_parent_data dpu_aclk_parents[] = {
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d8_307p2),
+	CCU_PARENT_HW(pll2_d4),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(dsi4ln2_dpu_aclk, dpu_aclk_parents, APMU_LCD_CLK_RES_CTRL5,
+			   2, 3, BIT(30), 5, 3, BIT(1), 0);
+
+CCU_MUX_DIV_GATE_FC_DEFINE(dpu_aclk, dpu_aclk_parents, APMU_LCD_CLK_RES_CTRL5, 17, 3, BIT(31),
+			   20, 3, BIT(16), 0);
+
+static const struct clk_parent_data ufs_aclk_parents[] = {
+	CCU_PARENT_HW(pll1_d5_491p52),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll2_d6),
+	CCU_PARENT_HW(pll2_d5),
+};
+CCU_MUX_DIV_GATE_FC_DEFINE(ufs_aclk, ufs_aclk_parents, APMU_UFS_CLK_RES_CTRL, 5, 3, BIT(8),
+			   2, 3, BIT(1), 0);
+
+CCU_FACTOR_DEFINE(ufs_refclk, CCU_PARENT_HW(pll1_d64_38p4), 2, 1);
+
+static const struct clk_parent_data edp0_pclk_parents[] = {
+	CCU_PARENT_HW(lcd_pxclk),
+	CCU_PARENT_NAME(external_clk),
+};
+CCU_MUX_GATE_DEFINE(edp0_pxclk, edp0_pclk_parents, APMU_LCD_EDP_CTRL, 2, 1, BIT(1), 0);
+
+static const struct clk_parent_data edp1_pclk_parents[] = {
+	CCU_PARENT_HW(dsi4ln2_lcd_pxclk),
+	CCU_PARENT_NAME(external_clk),
+};
+CCU_MUX_GATE_DEFINE(edp1_pxclk, edp1_pclk_parents, APMU_LCD_EDP_CTRL, 18, 1, BIT(17), 0);
+
+CCU_GATE_DEFINE(pciea_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_A, BIT(2), 0);
+CCU_GATE_DEFINE(pciea_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_A, BIT(1), 0);
+CCU_GATE_DEFINE(pciea_bus_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_A, BIT(0), 0);
+CCU_GATE_DEFINE(pcieb_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_B, BIT(2), 0);
+CCU_GATE_DEFINE(pcieb_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_B, BIT(1), 0);
+CCU_GATE_DEFINE(pcieb_bus_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_B, BIT(0), 0);
+CCU_GATE_DEFINE(pciec_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_C, BIT(2), 0);
+CCU_GATE_DEFINE(pciec_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_C, BIT(1), 0);
+CCU_GATE_DEFINE(pciec_bus_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_C, BIT(0), 0);
+CCU_GATE_DEFINE(pcied_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_D, BIT(2), 0);
+CCU_GATE_DEFINE(pcied_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_D, BIT(1), 0);
+CCU_GATE_DEFINE(pcied_bus_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_D, BIT(0), 0);
+CCU_GATE_DEFINE(pciee_mstr_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_E, BIT(2), 0);
+CCU_GATE_DEFINE(pciee_slv_clk, CCU_PARENT_HW(pll2_d6), APMU_PCIE_CLK_RES_CTRL_E, BIT(1), 0);
+CCU_GATE_DEFINE(pciee_bus_clk, CCU_PARENT_HW(axi_clk), APMU_PCIE_CLK_RES_CTRL_E, BIT(0), 0);
+
+CCU_FACTOR_DEFINE(iommu_clk, CCU_PARENT_HW(pll2_d6), 1, 1);
+
+static const struct clk_parent_data emac_1588_parents[] = {
+	CCU_PARENT_NAME(vctcxo_24m),
+	CCU_PARENT_HW(pll2_d24_125),
+};
+
+CCU_GATE_DEFINE(emac0_bus_clk, CCU_PARENT_HW(axi_clk), APMU_EMAC0_CLK_RES_CTRL, BIT(0), 0);
+CCU_GATE_FLAGS_DEFINE(emac0_ref_clk, CCU_PARENT_HW(pll2_d120_25), APMU_EMAC0_CLK_RES_CTRL,
+		      BIT(14), CCU_GATE_INVERT_FLAG);
+CCU_MUX_DEFINE(emac0_1588_clk, emac_1588_parents, APMU_EMAC0_CLK_RES_CTRL, 15, 1, 0);
+CCU_GATE_DEFINE(emac0_rgmii_tx_clk, CCU_PARENT_HW(pll2_d24_125), APMU_EMAC0_CLK_RES_CTRL,
+		BIT(8), 0);
+CCU_GATE_DEFINE(emac1_bus_clk, CCU_PARENT_HW(axi_clk), APMU_EMAC1_CLK_RES_CTRL, BIT(0), 0);
+CCU_GATE_FLAGS_DEFINE(emac1_ref_clk, CCU_PARENT_HW(pll2_d120_25), APMU_EMAC1_CLK_RES_CTRL,
+		      BIT(14), CCU_GATE_INVERT_FLAG);
+CCU_MUX_DEFINE(emac1_1588_clk, emac_1588_parents, APMU_EMAC1_CLK_RES_CTRL, 15, 1, 0);
+CCU_GATE_DEFINE(emac1_rgmii_tx_clk, CCU_PARENT_HW(pll2_d24_125), APMU_EMAC1_CLK_RES_CTRL,
+		BIT(8), 0);
+CCU_GATE_DEFINE(emac2_bus_clk, CCU_PARENT_HW(axi_clk), APMU_EMAC2_CLK_RES_CTRL, BIT(0), 0);
+CCU_GATE_FLAGS_DEFINE(emac2_ref_clk, CCU_PARENT_HW(pll2_d120_25), APMU_EMAC2_CLK_RES_CTRL,
+		      BIT(14), CCU_GATE_INVERT_FLAG);
+CCU_MUX_DEFINE(emac2_1588_clk, emac_1588_parents, APMU_EMAC2_CLK_RES_CTRL, 15, 1, 0);
+CCU_GATE_DEFINE(emac2_rgmii_tx_clk, CCU_PARENT_HW(pll2_d24_125), APMU_EMAC2_CLK_RES_CTRL,
+		BIT(8), 0);
+
+static const struct clk_parent_data espi_sclk_src_parents[] = {
+	CCU_PARENT_HW(pll2_20),
+	CCU_PARENT_HW(pll2_25),
+	CCU_PARENT_HW(pll2_33),
+	CCU_PARENT_HW(pll2_50),
+	CCU_PARENT_HW(pll2_66),
+};
+CCU_MUX_DEFINE(espi_sclk_src, espi_sclk_src_parents, APMU_ESPI_CLK_RES_CTRL, 4, 3, 0);
+
+static const struct clk_parent_data espi_sclk_parents[] = {
+	CCU_PARENT_NAME(external_clk),
+	CCU_PARENT_HW(espi_sclk_src),
+};
+CCU_MUX_GATE_DEFINE(espi_sclk, espi_sclk_parents, APMU_ESPI_CLK_RES_CTRL, 7, 1, BIT(3), 0);
+
+CCU_GATE_DEFINE(espi_mclk, CCU_PARENT_HW(axi_clk), APMU_ESPI_CLK_RES_CTRL, BIT(1), 0);
+
+CCU_FACTOR_DEFINE(cam_src1_clk, CCU_PARENT_HW(pll1_d6_409p6), 15, 1);
+CCU_FACTOR_DEFINE(cam_src2_clk, CCU_PARENT_HW(pll2_d5), 25, 1);
+CCU_FACTOR_DEFINE(cam_src3_clk, CCU_PARENT_HW(pll2_d6), 20, 1);
+CCU_FACTOR_DEFINE(cam_src4_clk, CCU_PARENT_HW(pll1_d6_409p6), 16, 1);
+
+static const struct clk_parent_data isim_vclk_parents[] = {
+	CCU_PARENT_HW(cam_src1_clk),
+	CCU_PARENT_HW(cam_src2_clk),
+	CCU_PARENT_HW(cam_src3_clk),
+	CCU_PARENT_HW(cam_src4_clk),
+};
+CCU_MUX_DIV_GATE_DEFINE(isim_vclk_out0, isim_vclk_parents, APMU_SNR_ISIM_VCLK_CTRL, 3, 4,
+			1, 2, BIT(0), CCU_DIV_4BIT_NO_0F_FLAG);
+CCU_MUX_DIV_GATE_DEFINE(isim_vclk_out1, isim_vclk_parents, APMU_SNR_ISIM_VCLK_CTRL, 11, 4,
+			9, 2, BIT(8), CCU_DIV_4BIT_NO_0F_FLAG);
+CCU_MUX_DIV_GATE_DEFINE(isim_vclk_out2, isim_vclk_parents, APMU_SNR_ISIM_VCLK_CTRL, 19, 4,
+			17, 2, BIT(16), CCU_DIV_4BIT_NO_0F_FLAG);
+CCU_MUX_DIV_GATE_DEFINE(isim_vclk_out3, isim_vclk_parents, APMU_SNR_ISIM_VCLK_CTRL, 27, 4,
+			25, 2, BIT(24), CCU_DIV_4BIT_NO_0F_FLAG);
+/* APMU clocks end */
+
+/* DCIU clocks start */
+CCU_GATE_DEFINE(hdma_clk, CCU_PARENT_HW(axi_clk), DCIU_DMASYS_CLK_EN, BIT(0), 0);
+CCU_GATE_DEFINE(dma350_clk, CCU_PARENT_HW(axi_clk), DCIU_DMASYS_SDMA_CLK_EN, BIT(0), 0);
+CCU_GATE_DEFINE(c2_tcm_pipe_clk, CCU_PARENT_HW(axi_clk), DCIU_C2_TCM_PIPE_CLK, BIT(0), 0);
+CCU_GATE_DEFINE(c3_tcm_pipe_clk, CCU_PARENT_HW(axi_clk), DCIU_C3_TCM_PIPE_CLK, BIT(0), 0);
+/* DCIU clocks end */
+
+/* RPMU clocks start */
+CCU_DIV_FC_DEFINE(rcpu_axi_clk,  CCU_PARENT_HW(rcpu_clk), RCPU5_RCPU_BUS_CLK_CTRL,
+		  BIT(8), 3, 3, CLK_IGNORE_UNUSED);
+CCU_DIV_FC_DEFINE(rcpu_apb_clk,  CCU_PARENT_HW(rcpu_axi_clk), RCPU5_RCPU_BUS_CLK_CTRL,
+		  BIT(8), 0, 3, CLK_IGNORE_UNUSED);
+
+CCU_GATE_DEFINE(ripc2msa_clk, CCU_PARENT_HW(rcpu_clk), RCPU5_AON_PER_CLK_RST_CTRL,
+		BIT(5), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ripc2cp_clk, CCU_PARENT_HW(rcpu_clk), RCPU5_AON_PER_CLK_RST_CTRL,
+		BIT(3), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ripc2ap_clk, CCU_PARENT_HW(rcpu_clk), RCPU5_AON_PER_CLK_RST_CTRL,
+		BIT(1), CLK_IGNORE_UNUSED);
+
+static const struct clk_parent_data rtimer_parents[] = {
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d768_3p2),
+};
+CCU_MUX_DIV_GATE_DEFINE(rtimer1_clk, rtimer_parents, RCPU5_TIMER1_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rtimer1_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU5_TIMER1_CLK_RST,
+			BIT(2), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rtimer2_clk, rtimer_parents, RCPU5_TIMER2_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rtimer2_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU5_TIMER2_CLK_RST,
+			BIT(2), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rtimer3_clk, rtimer_parents, RCPU5_TIMER3_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rtimer3_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU5_TIMER3_CLK_RST,
+			BIT(2), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rtimer4_clk, rtimer_parents, RCPU5_TIMER4_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rtimer4_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU5_TIMER4_CLK_RST,
+			BIT(2), CLK_IGNORE_UNUSED);
+
+static const struct clk_parent_data rt24_core_parents[] = {
+	CCU_PARENT_HW(rcpu_clk),
+	CCU_PARENT_HW(pll1_d4_614p4),
+	CCU_PARENT_HW(pll1_d5_491p52),
+};
+CCU_MUX_DIV_FC_DEFINE(rt24_core0_clk, rt24_core_parents, RCPU5_RT24_CORE0_CLK_CTRL,
+		      0, 2, BIT(8), 4, 2, CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_FC_DEFINE(rt24_core1_clk, rt24_core_parents, RCPU5_RT24_CORE1_CLK_CTRL,
+		      0, 2, BIT(8), 4, 2, CLK_IGNORE_UNUSED);
+
+CCU_GATE_DEFINE(rgpio_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU5_GPIO_AND_EDGE_CLK_RST,
+		BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rgpio_edge_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU5_GPIO_AND_EDGE_CLK_RST,
+		BIT(3), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rgpio_lp_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU5_GPIO_AND_EDGE_CLK_RST,
+		BIT(4), CLK_IGNORE_UNUSED);
+/* RPMU clocks end */
+
+/* RCPU SYSCTRL clocks start */
+static const struct clk_parent_data rcan_parents[] = {
+	CCU_PARENT_HW(pll6_20),
+	CCU_PARENT_HW(pll6_40),
+	CCU_PARENT_HW(pll6_80),
+};
+CCU_MUX_GATE_DEFINE(rcan0_clk, rcan_parents, RCPU_CAN_CLK_RST, 4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_GATE_DEFINE(rcan1_clk, rcan_parents, RCPU_CAN1_CLK_RST, 4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_GATE_DEFINE(rcan2_clk, rcan_parents, RCPU_CAN2_CLK_RST, 4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_GATE_DEFINE(rcan3_clk, rcan_parents, RCPU_CAN3_CLK_RST, 4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_GATE_DEFINE(rcan4_clk, rcan_parents, RCPU_CAN4_CLK_RST, 4, 2, BIT(1), CLK_IGNORE_UNUSED);
+
+CCU_GATE_DEFINE(rcan0_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU_CAN_CLK_RST,
+		BIT(2), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rcan1_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU_CAN1_CLK_RST,
+		BIT(2), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rcan2_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU_CAN2_CLK_RST,
+		BIT(2), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rcan3_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU_CAN3_CLK_RST,
+		BIT(2), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rcan4_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU_CAN4_CLK_RST,
+		BIT(2), CLK_IGNORE_UNUSED);
+
+CCU_GATE_DEFINE(rirc0_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU_IRC_CLK_RST,
+		BIT(2), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rirc1_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU_IRC1_CLK_RST,
+		BIT(2), CLK_IGNORE_UNUSED);
+
+CCU_MUX_DEFINE(respi_sclk_src, espi_sclk_src_parents, RCPU_ESPI_CLK_RST,
+		4, 3, CLK_IGNORE_UNUSED);
+CCU_MUX_GATE_DEFINE(respi_sclk, espi_sclk_parents, RCPU_ESPI_CLK_RST,
+		8, 1, BIT(1), CLK_IGNORE_UNUSED);
+
+CCU_GATE_DEFINE(remac_bus_clk, CCU_PARENT_HW(rcpu_axi_clk), RCPU_GMAC_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_FLAGS_DEFINE(remac_ref_clk, CCU_PARENT_HW(pll2_d120_25), RCPU_GMAC_CLK_RST,
+		BIT(14), CCU_GATE_INVERT_FLAG | CLK_IGNORE_UNUSED);
+CCU_MUX_DEFINE(remac_1588_clk, emac_1588_parents, RCPU_GMAC_CLK_RST,
+		15, 1, CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(remac_rgmii_tx_clk, CCU_PARENT_HW(pll2_d24_125), RCPU_GMAC_CLK_RST,
+		BIT(8), CLK_IGNORE_UNUSED);
+
+static const struct clk_parent_data ri2s01_sysclk_parents[] = {
+	CCU_PARENT_HW(pll1_aud_24p5),
+	CCU_PARENT_HW(pll1_aud_245p7),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d768_3p2),
+};
+CCU_MUX_DIV_GATE_DEFINE(ri2s0_sysclk, ri2s01_sysclk_parents, RCPU_AUDIO_I2S0_SYS_CLK_CTRL,
+			8, 11, 4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ri2s1_sysclk, ri2s01_sysclk_parents, RCPU_AUDIO_I2S1_SYS_CLK_CTRL,
+			8, 11, 4, 2, BIT(1), CLK_IGNORE_UNUSED);
+
+CCU_GATE_DEFINE(ruart_14_src, CCU_PARENT_HW(pll1_d5_491p52), RCPU_UART_NM_CLK_14M_CTRL,
+		BIT(31), CLK_IGNORE_UNUSED);
+CCU_DDN_DEFINE(ruart_14, ruart_14_src, RCPU_UART_NM_CLK_14M_CTRL,
+		0, 13, 16, 13, 2, CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ruart_58_src, CCU_PARENT_HW(pll1_d5_491p52), RCPU_UART_NM_CLK_58M_CTRL,
+		BIT(31), CLK_IGNORE_UNUSED);
+CCU_DDN_DEFINE(ruart_58, ruart_58_src, RCPU_UART_NM_CLK_58M_CTRL,
+		0, 13, 16, 13, 2, CLK_IGNORE_UNUSED);
+/* RCPU SYSCTRL clocks end */
+
+/* RCPU UARTCTRL clocks start */
+static const struct clk_parent_data ruart_clk_parents[] = {
+	CCU_PARENT_HW(ruart_14),
+	CCU_PARENT_HW(pll1_aud_245p7),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(ruart_58),
+};
+CCU_MUX_DIV_GATE_DEFINE(ruart0_clk, ruart_clk_parents, RCPU1_UART0_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ruart1_clk, ruart_clk_parents, RCPU1_UART1_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ruart2_clk, ruart_clk_parents, RCPU1_UART2_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ruart3_clk, ruart_clk_parents, RCPU1_UART3_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ruart4_clk, ruart_clk_parents, RCPU1_UART4_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ruart5_clk, ruart_clk_parents, RCPU1_UART5_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ruart0_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU1_UART0_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ruart1_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU1_UART1_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ruart2_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU1_UART2_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ruart3_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU1_UART3_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ruart4_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU1_UART4_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ruart5_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU1_UART5_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+/* RCPU UARTCTRLclocks end */
+
+/* RCPU I2SCTRL clocks start */
+static const struct clk_parent_data ri2s_clk_parents[] = {
+	CCU_PARENT_HW(pll1_aud_24p5),
+	CCU_PARENT_HW(pll1_aud_245p7),
+};
+CCU_MUX_DIV_GATE_DEFINE(ri2s0_clk, ri2s_clk_parents, RCPU2_AUDIO_I2S0_TX_RX_CLK_CTRL, 4, 11,
+			16, 2, BIT(2), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ri2s1_clk, ri2s_clk_parents, RCPU2_AUDIO_I2S1_TX_RX_CLK_CTRL, 4, 11,
+			16, 2, BIT(2), CLK_IGNORE_UNUSED);
+
+
+CCU_GATE_DEFINE(ri2s0_bus_clk, CCU_PARENT_HW(rcpu_apb_clk),
+		RCPU2_AUDIO_I2S0_TX_RX_CLK_CTRL, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ri2s1_bus_clk, CCU_PARENT_HW(rcpu_apb_clk),
+		RCPU2_AUDIO_I2S1_TX_RX_CLK_CTRL, BIT(1), CLK_IGNORE_UNUSED);
+
+static const struct clk_parent_data ri2s23_sysclk_parents[] = {
+	CCU_PARENT_HW(pll1_aud_24p5),
+	CCU_PARENT_HW(pll1_aud_245p7),
+};
+CCU_MUX_DIV_GATE_DEFINE(ri2s2_sysclk, ri2s23_sysclk_parents, RCPU2_AUDIO_I2S2_SYS_CLK_CTRL,
+			4, 11, 16, 2, BIT(2), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ri2s3_sysclk, ri2s23_sysclk_parents, RCPU2_AUDIO_I2S3_SYS_CLK_CTRL,
+			4, 11, 16, 2, BIT(2), CLK_IGNORE_UNUSED);
+
+CCU_DIV_GATE_DEFINE(ri2s2_clk, CCU_PARENT_HW(ri2s2_sysclk), RCPU2_AUDIO_I2S2_TX_RX_CLK_CTRL,
+		    4, 11, BIT(2), CLK_IGNORE_UNUSED);
+CCU_DIV_GATE_DEFINE(ri2s3_clk, CCU_PARENT_HW(ri2s3_sysclk), RCPU2_AUDIO_I2S3_TX_RX_CLK_CTRL,
+		    4, 11, BIT(2), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ri2s2_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU2_AUDIO_I2S2_TX_RX_CLK_CTRL,
+		BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ri2s3_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU2_AUDIO_I2S3_TX_RX_CLK_CTRL,
+		BIT(1), CLK_IGNORE_UNUSED);
+/* RCPU I2SCTRL clocks end */
+
+/* RCPU SPICTRL clocks start */
+static const struct clk_parent_data rspi_parents[] = {
+	CCU_PARENT_HW(pll1_aud_24p5),
+	CCU_PARENT_HW(pll1_aud_245p7),
+	CCU_PARENT_HW(pll1_d96_25p6),
+};
+CCU_MUX_DIV_GATE_DEFINE(rspi0_clk, rspi_parents, RCPU3_SSP0_CLK_RST, 8, 11,
+		   4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rspi1_clk, rspi_parents, RCPU3_SSP1_CLK_RST, 8, 11,
+		   4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rspi2_clk, rspi_parents, RCPU3_PWR_SSP_CLK_RST, 8, 11,
+		   4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rspi0_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU3_SSP0_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rspi1_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU3_SSP1_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rspi2_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU3_PWR_SSP_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+/* RCPU SPICTRL clocks end */
+
+/* RCPU I2CCTRL clocks start */
+static const struct clk_parent_data ri2c_parents[] = {
+	CCU_PARENT_HW(pll1_aud_24p5),
+	CCU_PARENT_HW(pll1_aud_245p7),
+	CCU_PARENT_HW(pll1_d96_25p6),
+};
+CCU_MUX_DIV_GATE_DEFINE(ri2c0_clk, ri2c_parents, RCPU4_I2C0_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ri2c1_clk, ri2c_parents, RCPU4_I2C1_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(ri2c2_clk, ri2c_parents, RCPU4_PWR_I2C_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ri2c0_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU4_I2C0_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ri2c1_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU4_I2C1_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(ri2c2_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU4_PWR_I2C_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+/* RCPU I2CCTRL clocks end */
+
+/* RCPU PWMCTRL clocks start */
+static const struct clk_parent_data rpwm_parents[] = {
+	CCU_PARENT_HW(pll1_aud_245p7),
+	CCU_PARENT_HW(pll1_aud_24p5),
+};
+CCU_MUX_DIV_GATE_DEFINE(rpwm0_clk, rpwm_parents, RCPU6_PWM0_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm1_clk, rpwm_parents, RCPU6_PWM1_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm2_clk, rpwm_parents, RCPU6_PWM2_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm3_clk, rpwm_parents, RCPU6_PWM3_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm4_clk, rpwm_parents, RCPU6_PWM4_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm5_clk, rpwm_parents, RCPU6_PWM5_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm6_clk, rpwm_parents, RCPU6_PWM6_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm7_clk, rpwm_parents, RCPU6_PWM7_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm8_clk, rpwm_parents, RCPU6_PWM8_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_MUX_DIV_GATE_DEFINE(rpwm9_clk, rpwm_parents, RCPU6_PWM9_CLK_RST, 8, 11,
+			4, 2, BIT(1), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm0_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM0_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm1_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM1_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm2_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM2_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm3_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM3_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm4_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM4_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm5_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM5_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm6_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM6_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm7_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM7_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm8_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM8_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+CCU_GATE_DEFINE(rpwm9_bus_clk, CCU_PARENT_HW(rcpu_apb_clk), RCPU6_PWM9_CLK_RST,
+		BIT(0), CLK_IGNORE_UNUSED);
+/* RCPU PWMCTRL clocks end */
+
+/* APBC2 clocks start */
+CCU_MUX_GATE_DEFINE(uart1_sec_clk, uart_clk_parents, APBC2_UART1_CLK_RST, 4, 3, BIT(1), 0);
+CCU_GATE_DEFINE(uart1_sec_bus_clk, CCU_PARENT_HW(apb_clk), APBC2_UART1_CLK_RST, BIT(0), 0);
+
+CCU_MUX_DEFINE(spi2_i2s_bclk, i2s_bclk_parents, APBC2_SSP2_CLK_RST, 3, 1, 0);
+
+static const struct clk_parent_data spi2_parents[] = {
+	CCU_PARENT_HW(pll1_d384_6p4),
+	CCU_PARENT_HW(pll1_d192_12p8),
+	CCU_PARENT_HW(pll1_d96_25p6),
+	CCU_PARENT_HW(pll1_d48_51p2),
+	CCU_PARENT_HW(pll1_d768_3p2),
+	CCU_PARENT_HW(pll1_d1536_1p6),
+	CCU_PARENT_HW(pll1_d3072_0p8),
+	CCU_PARENT_HW(spi2_i2s_bclk),
+};
+CCU_MUX_GATE_DEFINE(spi2_sec_clk, spi2_parents, APBC2_SSP2_CLK_RST, 4, 3, BIT(1), 0);
+CCU_GATE_DEFINE(spi2_sec_bus_clk, CCU_PARENT_HW(apb_clk), APBC2_SSP2_CLK_RST, BIT(0), 0);
+
+CCU_MUX_GATE_DEFINE(twsi3_sec_clk, twsi_parents, APBC2_TWSI3_CLK_RST, 4, 3, BIT(1), 0);
+CCU_GATE_DEFINE(twsi3_sec_bus_clk, CCU_PARENT_HW(apb_clk), APBC2_TWSI3_CLK_RST, BIT(0), 0);
+
+CCU_GATE_DEFINE(rtc_sec_clk, CCU_PARENT_NAME(osc_32k), APBC2_RTC_CLK_RST, BIT(7) | BIT(1), 0);
+CCU_GATE_DEFINE(rtc_sec_bus_clk, CCU_PARENT_HW(apb_clk), APBC2_RTC_CLK_RST, BIT(0), 0);
+
+CCU_MUX_GATE_DEFINE(timers_sec_clk, timer_parents, APBC2_TIMERS_CLK_RST, 4, 3, BIT(1), 0);
+CCU_GATE_DEFINE(timers_sec_bus_clk, CCU_PARENT_HW(apb_clk), APBC2_TIMERS_CLK_RST, BIT(0), 0);
+
+CCU_GATE_DEFINE(gpio_sec_clk, CCU_PARENT_NAME(vctcxo_24m), APBC2_GPIO_CLK_RST, BIT(1), 0);
+CCU_GATE_DEFINE(gpio_sec_bus_clk, CCU_PARENT_HW(apb_clk), APBC2_GPIO_CLK_RST, BIT(0), 0);
+/* APBC2 clocks end */
+
+static struct clk_hw *k3_ccu_pll_hws[] = {
+	[CLK_PLL1]		= &pll1.common.hw,
+	[CLK_PLL2]		= &pll2.common.hw,
+	[CLK_PLL3]		= &pll3.common.hw,
+	[CLK_PLL4]		= &pll4.common.hw,
+	[CLK_PLL5]		= &pll5.common.hw,
+	[CLK_PLL6]		= &pll6.common.hw,
+	[CLK_PLL7]		= &pll7.common.hw,
+	[CLK_PLL8]		= &pll8.common.hw,
+	[CLK_PLL1_D2]		= &pll1_d2.common.hw,
+	[CLK_PLL1_D3]		= &pll1_d3.common.hw,
+	[CLK_PLL1_D4]		= &pll1_d4.common.hw,
+	[CLK_PLL1_D5]		= &pll1_d5.common.hw,
+	[CLK_PLL1_D6]		= &pll1_d6.common.hw,
+	[CLK_PLL1_D7]		= &pll1_d7.common.hw,
+	[CLK_PLL1_D8]		= &pll1_d8.common.hw,
+	[CLK_PLL1_DX]		= &pll1_dx.common.hw,
+	[CLK_PLL1_D64]		= &pll1_d64_38p4.common.hw,
+	[CLK_PLL1_D10_AUD]	= &pll1_aud_245p7.common.hw,
+	[CLK_PLL1_D100_AUD]	= &pll1_aud_24p5.common.hw,
+	[CLK_PLL2_D1]		= &pll2_d1.common.hw,
+	[CLK_PLL2_D2]		= &pll2_d2.common.hw,
+	[CLK_PLL2_D3]		= &pll2_d3.common.hw,
+	[CLK_PLL2_D4]		= &pll2_d4.common.hw,
+	[CLK_PLL2_D5]		= &pll2_d5.common.hw,
+	[CLK_PLL2_D6]		= &pll2_d6.common.hw,
+	[CLK_PLL2_D7]		= &pll2_d7.common.hw,
+	[CLK_PLL2_D8]		= &pll2_d8.common.hw,
+	[CLK_PLL2_66]		= &pll2_66.common.hw,
+	[CLK_PLL2_33]		= &pll2_33.common.hw,
+	[CLK_PLL2_50]		= &pll2_50.common.hw,
+	[CLK_PLL2_25]		= &pll2_25.common.hw,
+	[CLK_PLL2_20]		= &pll2_20.common.hw,
+	[CLK_PLL2_D24_125]	= &pll2_d24_125.common.hw,
+	[CLK_PLL2_D120_25]	= &pll2_d120_25.common.hw,
+	[CLK_PLL3_D1]		= &pll3_d1.common.hw,
+	[CLK_PLL3_D2]		= &pll3_d2.common.hw,
+	[CLK_PLL3_D3]		= &pll3_d3.common.hw,
+	[CLK_PLL3_D4]		= &pll3_d4.common.hw,
+	[CLK_PLL3_D5]		= &pll3_d5.common.hw,
+	[CLK_PLL3_D6]		= &pll3_d6.common.hw,
+	[CLK_PLL3_D7]		= &pll3_d7.common.hw,
+	[CLK_PLL3_D8]		= &pll3_d8.common.hw,
+	[CLK_PLL4_D1]		= &pll4_d1.common.hw,
+	[CLK_PLL4_D2]		= &pll4_d2.common.hw,
+	[CLK_PLL4_D3]		= &pll4_d3.common.hw,
+	[CLK_PLL4_D4]		= &pll4_d4.common.hw,
+	[CLK_PLL4_D5]		= &pll4_d5.common.hw,
+	[CLK_PLL4_D6]		= &pll4_d6.common.hw,
+	[CLK_PLL4_D7]		= &pll4_d7.common.hw,
+	[CLK_PLL4_D8]		= &pll4_d8.common.hw,
+	[CLK_PLL5_D1]		= &pll5_d1.common.hw,
+	[CLK_PLL5_D2]		= &pll5_d2.common.hw,
+	[CLK_PLL5_D3]		= &pll5_d3.common.hw,
+	[CLK_PLL5_D4]		= &pll5_d4.common.hw,
+	[CLK_PLL5_D5]		= &pll5_d5.common.hw,
+	[CLK_PLL5_D6]		= &pll5_d6.common.hw,
+	[CLK_PLL5_D7]		= &pll5_d7.common.hw,
+	[CLK_PLL5_D8]		= &pll5_d8.common.hw,
+	[CLK_PLL6_D1]		= &pll6_d1.common.hw,
+	[CLK_PLL6_D2]		= &pll6_d2.common.hw,
+	[CLK_PLL6_D3]		= &pll6_d3.common.hw,
+	[CLK_PLL6_D4]		= &pll6_d4.common.hw,
+	[CLK_PLL6_D5]		= &pll6_d5.common.hw,
+	[CLK_PLL6_D6]		= &pll6_d6.common.hw,
+	[CLK_PLL6_D7]		= &pll6_d7.common.hw,
+	[CLK_PLL6_D8]		= &pll6_d8.common.hw,
+	[CLK_PLL6_80]		= &pll6_80.common.hw,
+	[CLK_PLL6_40]		= &pll6_40.common.hw,
+	[CLK_PLL6_20]		= &pll6_20.common.hw,
+	[CLK_PLL7_D1]		= &pll7_d1.common.hw,
+	[CLK_PLL7_D2]		= &pll7_d2.common.hw,
+	[CLK_PLL7_D3]		= &pll7_d3.common.hw,
+	[CLK_PLL7_D4]		= &pll7_d4.common.hw,
+	[CLK_PLL7_D5]		= &pll7_d5.common.hw,
+	[CLK_PLL7_D6]		= &pll7_d6.common.hw,
+	[CLK_PLL7_D7]		= &pll7_d7.common.hw,
+	[CLK_PLL7_D8]		= &pll7_d8.common.hw,
+	[CLK_PLL8_D1]		= &pll8_d1.common.hw,
+	[CLK_PLL8_D2]		= &pll8_d2.common.hw,
+	[CLK_PLL8_D3]		= &pll8_d3.common.hw,
+	[CLK_PLL8_D4]		= &pll8_d4.common.hw,
+	[CLK_PLL8_D5]		= &pll8_d5.common.hw,
+	[CLK_PLL8_D6]		= &pll8_d6.common.hw,
+	[CLK_PLL8_D7]		= &pll8_d7.common.hw,
+	[CLK_PLL8_D8]		= &pll8_d8.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_pll_data = {
+	/* The PLL CCU implements no resets */
+	.hws		= k3_ccu_pll_hws,
+	.num		= ARRAY_SIZE(k3_ccu_pll_hws),
+};
+
+static struct clk_hw *k3_ccu_mpmu_hws[] = {
+	[CLK_MPMU_PLL1_307P2]		= &pll1_d8_307p2.common.hw,
+	[CLK_MPMU_PLL1_76P8]		= &pll1_d32_76p8.common.hw,
+	[CLK_MPMU_PLL1_61P44]		= &pll1_d40_61p44.common.hw,
+	[CLK_MPMU_PLL1_153P6]		= &pll1_d16_153p6.common.hw,
+	[CLK_MPMU_PLL1_102P4]		= &pll1_d24_102p4.common.hw,
+	[CLK_MPMU_PLL1_51P2]		= &pll1_d48_51p2.common.hw,
+	[CLK_MPMU_PLL1_51P2_AP]		= &pll1_d48_51p2_ap.common.hw,
+	[CLK_MPMU_PLL1_57P6]		= &pll1_m3d128_57p6.common.hw,
+	[CLK_MPMU_PLL1_25P6]		= &pll1_d96_25p6.common.hw,
+	[CLK_MPMU_PLL1_12P8]		= &pll1_d192_12p8.common.hw,
+	[CLK_MPMU_PLL1_12P8_WDT]	= &pll1_d192_12p8_wdt.common.hw,
+	[CLK_MPMU_PLL1_6P4]		= &pll1_d384_6p4.common.hw,
+	[CLK_MPMU_PLL1_3P2]		= &pll1_d768_3p2.common.hw,
+	[CLK_MPMU_PLL1_1P6]		= &pll1_d1536_1p6.common.hw,
+	[CLK_MPMU_PLL1_0P8]		= &pll1_d3072_0p8.common.hw,
+	[CLK_MPMU_PLL1_409P6]		= &pll1_d6_409p6.common.hw,
+	[CLK_MPMU_PLL1_204P8]		= &pll1_d12_204p8.common.hw,
+	[CLK_MPMU_PLL1_491]		= &pll1_d5_491p52.common.hw,
+	[CLK_MPMU_PLL1_245P76]		= &pll1_d10_245p76.common.hw,
+	[CLK_MPMU_PLL1_614]		= &pll1_d4_614p4.common.hw,
+	[CLK_MPMU_PLL1_47P26]		= &pll1_d52_47p26.common.hw,
+	[CLK_MPMU_PLL1_31P5]		= &pll1_d78_31p5.common.hw,
+	[CLK_MPMU_PLL1_819]		= &pll1_d3_819p2.common.hw,
+	[CLK_MPMU_PLL1_1228]		= &pll1_d2_1228p8.common.hw,
+	[CLK_MPMU_APB]			= &apb_clk.common.hw,
+	[CLK_MPMU_SLOW_UART]		= &slow_uart.common.hw,
+	[CLK_MPMU_SLOW_UART1]		= &slow_uart1_14p74.common.hw,
+	[CLK_MPMU_SLOW_UART2]		= &slow_uart2_48.common.hw,
+	[CLK_MPMU_WDT]			= &wdt_clk.common.hw,
+	[CLK_MPMU_WDT_BUS]		= &wdt_bus_clk.common.hw,
+	[CLK_MPMU_RIPC]			= &r_ipc_clk.common.hw,
+	[CLK_MPMU_I2S_153P6]		= &i2s_153p6.common.hw,
+	[CLK_MPMU_I2S_153P6_BASE]	= &i2s_153p6_base.common.hw,
+	[CLK_MPMU_I2S_SYSCLK_SRC]	= &i2s_sysclk_src.common.hw,
+	[CLK_MPMU_I2S_SYSCLK]		= &i2s_sysclk.common.hw,
+	[CLK_MPMU_I2S_BCLK_FACTOR]	= &i2s_bclk_factor.common.hw,
+	[CLK_MPMU_I2S_BCLK]		= &i2s_bclk.common.hw,
+	[CLK_MPMU_I2S1_SYSCLK_SRC]	= &i2s1_sysclk_src.common.hw,
+	[CLK_MPMU_I2S1_SYSCLK]		= &i2s1_sysclk.common.hw,
+	[CLK_MPMU_I2S0_SYSCLK_SEL]	= &i2s0_sysclk_sel.common.hw,
+	[CLK_MPMU_I2S2_SYSCLK_SEL]	= &i2s2_sysclk_sel.common.hw,
+	[CLK_MPMU_I2S3_SYSCLK_SEL]	= &i2s3_sysclk_sel.common.hw,
+	[CLK_MPMU_I2S4_SYSCLK_SEL]	= &i2s4_sysclk_sel.common.hw,
+	[CLK_MPMU_I2S5_SYSCLK_SEL]	= &i2s5_sysclk_sel.common.hw,
+	[CLK_MPMU_I2S0_SYSCLK_DIV]	= &i2s0_sysclk_div.common.hw,
+	[CLK_MPMU_I2S2_SYSCLK_DIV]	= &i2s2_sysclk_div.common.hw,
+	[CLK_MPMU_I2S3_SYSCLK_DIV]	= &i2s3_sysclk_div.common.hw,
+	[CLK_MPMU_I2S4_SYSCLK_DIV]	= &i2s4_sysclk_div.common.hw,
+	[CLK_MPMU_I2S5_SYSCLK_DIV]	= &i2s5_sysclk_div.common.hw,
+	[CLK_MPMU_I2S0_SYSCLK]		= &i2s0_sysclk.common.hw,
+	[CLK_MPMU_I2S2_SYSCLK]		= &i2s2_sysclk.common.hw,
+	[CLK_MPMU_I2S3_SYSCLK]		= &i2s3_sysclk.common.hw,
+	[CLK_MPMU_I2S4_SYSCLK]		= &i2s4_sysclk.common.hw,
+	[CLK_MPMU_I2S5_SYSCLK]		= &i2s5_sysclk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_mpmu_data = {
+	.reset_name	= "mpmu-reset",
+	.hws		= k3_ccu_mpmu_hws,
+	.num		= ARRAY_SIZE(k3_ccu_mpmu_hws),
+};
+
+static struct clk_hw *k3_ccu_apbc_hws[] = {
+	[CLK_APBC_UART0]		= &uart0_clk.common.hw,
+	[CLK_APBC_UART2]		= &uart2_clk.common.hw,
+	[CLK_APBC_UART3]		= &uart3_clk.common.hw,
+	[CLK_APBC_UART4]		= &uart4_clk.common.hw,
+	[CLK_APBC_UART5]		= &uart5_clk.common.hw,
+	[CLK_APBC_UART6]		= &uart6_clk.common.hw,
+	[CLK_APBC_UART7]		= &uart7_clk.common.hw,
+	[CLK_APBC_UART8]		= &uart8_clk.common.hw,
+	[CLK_APBC_UART9]		= &uart9_clk.common.hw,
+	[CLK_APBC_UART10]		= &uart10_clk.common.hw,
+	[CLK_APBC_UART0_BUS]		= &uart0_bus_clk.common.hw,
+	[CLK_APBC_UART2_BUS]		= &uart2_bus_clk.common.hw,
+	[CLK_APBC_UART3_BUS]		= &uart3_bus_clk.common.hw,
+	[CLK_APBC_UART4_BUS]		= &uart4_bus_clk.common.hw,
+	[CLK_APBC_UART5_BUS]		= &uart5_bus_clk.common.hw,
+	[CLK_APBC_UART6_BUS]		= &uart6_bus_clk.common.hw,
+	[CLK_APBC_UART7_BUS]		= &uart7_bus_clk.common.hw,
+	[CLK_APBC_UART8_BUS]		= &uart8_bus_clk.common.hw,
+	[CLK_APBC_UART9_BUS]		= &uart9_bus_clk.common.hw,
+	[CLK_APBC_UART10_BUS]		= &uart10_bus_clk.common.hw,
+	[CLK_APBC_GPIO]			= &gpio_clk.common.hw,
+	[CLK_APBC_GPIO_BUS]		= &gpio_bus_clk.common.hw,
+	[CLK_APBC_PWM0]			= &pwm0_clk.common.hw,
+	[CLK_APBC_PWM1]			= &pwm1_clk.common.hw,
+	[CLK_APBC_PWM2]			= &pwm2_clk.common.hw,
+	[CLK_APBC_PWM3]			= &pwm3_clk.common.hw,
+	[CLK_APBC_PWM4]			= &pwm4_clk.common.hw,
+	[CLK_APBC_PWM5]			= &pwm5_clk.common.hw,
+	[CLK_APBC_PWM6]			= &pwm6_clk.common.hw,
+	[CLK_APBC_PWM7]			= &pwm7_clk.common.hw,
+	[CLK_APBC_PWM8]			= &pwm8_clk.common.hw,
+	[CLK_APBC_PWM9]			= &pwm9_clk.common.hw,
+	[CLK_APBC_PWM10]		= &pwm10_clk.common.hw,
+	[CLK_APBC_PWM11]		= &pwm11_clk.common.hw,
+	[CLK_APBC_PWM12]		= &pwm12_clk.common.hw,
+	[CLK_APBC_PWM13]		= &pwm13_clk.common.hw,
+	[CLK_APBC_PWM14]		= &pwm14_clk.common.hw,
+	[CLK_APBC_PWM15]		= &pwm15_clk.common.hw,
+	[CLK_APBC_PWM16]		= &pwm16_clk.common.hw,
+	[CLK_APBC_PWM17]		= &pwm17_clk.common.hw,
+	[CLK_APBC_PWM18]		= &pwm18_clk.common.hw,
+	[CLK_APBC_PWM19]		= &pwm19_clk.common.hw,
+	[CLK_APBC_PWM0_BUS]		= &pwm0_bus_clk.common.hw,
+	[CLK_APBC_PWM1_BUS]		= &pwm1_bus_clk.common.hw,
+	[CLK_APBC_PWM2_BUS]		= &pwm2_bus_clk.common.hw,
+	[CLK_APBC_PWM3_BUS]		= &pwm3_bus_clk.common.hw,
+	[CLK_APBC_PWM4_BUS]		= &pwm4_bus_clk.common.hw,
+	[CLK_APBC_PWM5_BUS]		= &pwm5_bus_clk.common.hw,
+	[CLK_APBC_PWM6_BUS]		= &pwm6_bus_clk.common.hw,
+	[CLK_APBC_PWM7_BUS]		= &pwm7_bus_clk.common.hw,
+	[CLK_APBC_PWM8_BUS]		= &pwm8_bus_clk.common.hw,
+	[CLK_APBC_PWM9_BUS]		= &pwm9_bus_clk.common.hw,
+	[CLK_APBC_PWM10_BUS]		= &pwm10_bus_clk.common.hw,
+	[CLK_APBC_PWM11_BUS]		= &pwm11_bus_clk.common.hw,
+	[CLK_APBC_PWM12_BUS]		= &pwm12_bus_clk.common.hw,
+	[CLK_APBC_PWM13_BUS]		= &pwm13_bus_clk.common.hw,
+	[CLK_APBC_PWM14_BUS]		= &pwm14_bus_clk.common.hw,
+	[CLK_APBC_PWM15_BUS]		= &pwm15_bus_clk.common.hw,
+	[CLK_APBC_PWM16_BUS]		= &pwm16_bus_clk.common.hw,
+	[CLK_APBC_PWM17_BUS]		= &pwm17_bus_clk.common.hw,
+	[CLK_APBC_PWM18_BUS]		= &pwm18_bus_clk.common.hw,
+	[CLK_APBC_PWM19_BUS]		= &pwm19_bus_clk.common.hw,
+	[CLK_APBC_SPI0_I2S_BCLK]	= &spi0_i2s_bclk.common.hw,
+	[CLK_APBC_SPI1_I2S_BCLK]	= &spi1_i2s_bclk.common.hw,
+	[CLK_APBC_SPI3_I2S_BCLK]	= &spi3_i2s_bclk.common.hw,
+	[CLK_APBC_SPI0]			= &spi0_clk.common.hw,
+	[CLK_APBC_SPI1]			= &spi1_clk.common.hw,
+	[CLK_APBC_SPI3]			= &spi3_clk.common.hw,
+	[CLK_APBC_SPI0_BUS]		= &spi0_bus_clk.common.hw,
+	[CLK_APBC_SPI1_BUS]		= &spi1_bus_clk.common.hw,
+	[CLK_APBC_SPI3_BUS]		= &spi3_bus_clk.common.hw,
+	[CLK_APBC_RTC]			= &rtc_clk.common.hw,
+	[CLK_APBC_RTC_BUS]		= &rtc_bus_clk.common.hw,
+	[CLK_APBC_TWSI0]		= &twsi0_clk.common.hw,
+	[CLK_APBC_TWSI1]		= &twsi1_clk.common.hw,
+	[CLK_APBC_TWSI2]		= &twsi2_clk.common.hw,
+	[CLK_APBC_TWSI4]		= &twsi4_clk.common.hw,
+	[CLK_APBC_TWSI5]		= &twsi5_clk.common.hw,
+	[CLK_APBC_TWSI6]		= &twsi6_clk.common.hw,
+	[CLK_APBC_TWSI8]		= &twsi8_clk.common.hw,
+	[CLK_APBC_TWSI0_BUS]		= &twsi0_bus_clk.common.hw,
+	[CLK_APBC_TWSI1_BUS]		= &twsi1_bus_clk.common.hw,
+	[CLK_APBC_TWSI2_BUS]		= &twsi2_bus_clk.common.hw,
+	[CLK_APBC_TWSI4_BUS]		= &twsi4_bus_clk.common.hw,
+	[CLK_APBC_TWSI5_BUS]		= &twsi5_bus_clk.common.hw,
+	[CLK_APBC_TWSI6_BUS]		= &twsi6_bus_clk.common.hw,
+	[CLK_APBC_TWSI8_BUS]		= &twsi8_bus_clk.common.hw,
+	[CLK_APBC_TIMERS0]		= &timers0_clk.common.hw,
+	[CLK_APBC_TIMERS1]		= &timers1_clk.common.hw,
+	[CLK_APBC_TIMERS2]		= &timers2_clk.common.hw,
+	[CLK_APBC_TIMERS3]		= &timers3_clk.common.hw,
+	[CLK_APBC_TIMERS4]		= &timers4_clk.common.hw,
+	[CLK_APBC_TIMERS5]		= &timers5_clk.common.hw,
+	[CLK_APBC_TIMERS6]		= &timers6_clk.common.hw,
+	[CLK_APBC_TIMERS7]		= &timers7_clk.common.hw,
+	[CLK_APBC_TIMERS0_BUS]		= &timers0_bus_clk.common.hw,
+	[CLK_APBC_TIMERS1_BUS]		= &timers1_bus_clk.common.hw,
+	[CLK_APBC_TIMERS2_BUS]		= &timers2_bus_clk.common.hw,
+	[CLK_APBC_TIMERS3_BUS]		= &timers3_bus_clk.common.hw,
+	[CLK_APBC_TIMERS4_BUS]		= &timers4_bus_clk.common.hw,
+	[CLK_APBC_TIMERS5_BUS]		= &timers5_bus_clk.common.hw,
+	[CLK_APBC_TIMERS6_BUS]		= &timers6_bus_clk.common.hw,
+	[CLK_APBC_TIMERS7_BUS]		= &timers7_bus_clk.common.hw,
+	[CLK_APBC_AIB]			= &aib_clk.common.hw,
+	[CLK_APBC_AIB_BUS]		= &aib_bus_clk.common.hw,
+	[CLK_APBC_ONEWIRE]		= &onewire_clk.common.hw,
+	[CLK_APBC_ONEWIRE_BUS]		= &onewire_bus_clk.common.hw,
+	[CLK_APBC_I2S0_BCLK]		= &i2s0_i2s_bclk.common.hw,
+	[CLK_APBC_I2S1_BCLK]		= &i2s1_i2s_bclk.common.hw,
+	[CLK_APBC_I2S2_BCLK]		= &i2s2_i2s_bclk.common.hw,
+	[CLK_APBC_I2S3_BCLK]		= &i2s3_i2s_bclk.common.hw,
+	[CLK_APBC_I2S4_BCLK]		= &i2s4_i2s_bclk.common.hw,
+	[CLK_APBC_I2S5_BCLK]		= &i2s5_i2s_bclk.common.hw,
+	[CLK_APBC_I2S0]			= &i2s0_clk.common.hw,
+	[CLK_APBC_I2S1]			= &i2s1_clk.common.hw,
+	[CLK_APBC_I2S2]			= &i2s2_clk.common.hw,
+	[CLK_APBC_I2S3]			= &i2s3_clk.common.hw,
+	[CLK_APBC_I2S4]			= &i2s4_clk.common.hw,
+	[CLK_APBC_I2S5]			= &i2s5_clk.common.hw,
+	[CLK_APBC_I2S0_BUS]		= &i2s0_bus_clk.common.hw,
+	[CLK_APBC_I2S1_BUS]		= &i2s1_bus_clk.common.hw,
+	[CLK_APBC_I2S2_BUS]		= &i2s2_bus_clk.common.hw,
+	[CLK_APBC_I2S3_BUS]		= &i2s3_bus_clk.common.hw,
+	[CLK_APBC_I2S4_BUS]		= &i2s4_bus_clk.common.hw,
+	[CLK_APBC_I2S5_BUS]		= &i2s5_bus_clk.common.hw,
+	[CLK_APBC_DRO]			= &dro_clk.common.hw,
+	[CLK_APBC_IR0]			= &ir0_clk.common.hw,
+	[CLK_APBC_IR1]			= &ir1_clk.common.hw,
+	[CLK_APBC_TSEN]			= &tsen_clk.common.hw,
+	[CLK_APBC_TSEN_BUS]		= &tsen_bus_clk.common.hw,
+	[CLK_APBC_IPC_AP2RCPU]		= &ipc_ap2rcpu_clk.common.hw,
+	[CLK_APBC_IPC_AP2RCPU_BUS]	= &ipc_ap2rcpu_bus_clk.common.hw,
+	[CLK_APBC_CAN0]			= &can0_clk.common.hw,
+	[CLK_APBC_CAN1]			= &can1_clk.common.hw,
+	[CLK_APBC_CAN2]			= &can2_clk.common.hw,
+	[CLK_APBC_CAN3]			= &can3_clk.common.hw,
+	[CLK_APBC_CAN4]			= &can4_clk.common.hw,
+	[CLK_APBC_CAN0_BUS]		= &can0_bus_clk.common.hw,
+	[CLK_APBC_CAN1_BUS]		= &can1_bus_clk.common.hw,
+	[CLK_APBC_CAN2_BUS]		= &can2_bus_clk.common.hw,
+	[CLK_APBC_CAN3_BUS]		= &can3_bus_clk.common.hw,
+	[CLK_APBC_CAN4_BUS]		= &can4_bus_clk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_apbc_data = {
+	.reset_name	= "apbc-reset",
+	.hws		= k3_ccu_apbc_hws,
+	.num		= ARRAY_SIZE(k3_ccu_apbc_hws),
+};
+
+static struct clk_hw *k3_ccu_apmu_hws[] = {
+	[CLK_APMU_AXICLK]		= &axi_clk.common.hw,
+	[CLK_APMU_CCI550]		= &cci550_clk.common.hw,
+	[CLK_APMU_CPU_C1_PLL_SRC]	= &cpu_c1_pll_src.common.hw,
+	[CLK_APMU_CPU_C3_PLL_SRC]	= &cpu_c3_pll_src.common.hw,
+	[CLK_APMU_CPU_C0_CORE]		= &cpu_c0_core_clk.common.hw,
+	[CLK_APMU_CPU_C1_CORE]		= &cpu_c1_core_clk.common.hw,
+	[CLK_APMU_CPU_C2_CORE]		= &cpu_c2_core_clk.common.hw,
+	[CLK_APMU_CPU_C3_CORE]		= &cpu_c3_core_clk.common.hw,
+	[CLK_APMU_CCIC2PHY]		= &ccic2phy_clk.common.hw,
+	[CLK_APMU_CCIC3PHY]		= &ccic3phy_clk.common.hw,
+	[CLK_APMU_CSI]			= &csi_clk.common.hw,
+	[CLK_APMU_ISP_BUS]		= &isp_bus_clk.common.hw,
+	[CLK_APMU_D1P_1228P8]		= &d1p_1228p8.common.hw,
+	[CLK_APMU_D1P_819P2]		= &d1p_819p2.common.hw,
+	[CLK_APMU_D1P_614P4]		= &d1p_614p4.common.hw,
+	[CLK_APMU_D1P_491P52]		= &d1p_491p52.common.hw,
+	[CLK_APMU_D1P_409P6]		= &d1p_409p6.common.hw,
+	[CLK_APMU_D1P_307P2]		= &d1p_307p2.common.hw,
+	[CLK_APMU_D1P_245P76]		= &d1p_245p76.common.hw,
+	[CLK_APMU_V2D]			= &v2d_clk.common.hw,
+	[CLK_APMU_DSI_ESC]		= &dsi_esc_clk.common.hw,
+	[CLK_APMU_LCD_HCLK]		= &lcd_hclk.common.hw,
+	[CLK_APMU_LCD_DSC]		= &lcd_dsc_clk.common.hw,
+	[CLK_APMU_LCD_PXCLK]		= &lcd_pxclk.common.hw,
+	[CLK_APMU_LCD_MCLK]		= &lcd_mclk.common.hw,
+	[CLK_APMU_CCIC_4X]		= &ccic_4x_clk.common.hw,
+	[CLK_APMU_CCIC1PHY]		= &ccic1phy_clk.common.hw,
+	[CLK_APMU_SC2_HCLK]		= &sc2_hclk.common.hw,
+	[CLK_APMU_SDH_AXI]		= &sdh_axi_aclk.common.hw,
+	[CLK_APMU_SDH0]			= &sdh0_clk.common.hw,
+	[CLK_APMU_SDH1]			= &sdh1_clk.common.hw,
+	[CLK_APMU_SDH2]			= &sdh2_clk.common.hw,
+	[CLK_APMU_USB2_BUS]		= &usb2_bus_clk.common.hw,
+	[CLK_APMU_USB3_PORTA_BUS]	= &usb3_porta_bus_clk.common.hw,
+	[CLK_APMU_USB3_PORTB_BUS]	= &usb3_portb_bus_clk.common.hw,
+	[CLK_APMU_USB3_PORTC_BUS]	= &usb3_portc_bus_clk.common.hw,
+	[CLK_APMU_USB3_PORTD_BUS]	= &usb3_portd_bus_clk.common.hw,
+	[CLK_APMU_QSPI]			= &qspi_clk.common.hw,
+	[CLK_APMU_QSPI_BUS]		= &qspi_bus_clk.common.hw,
+	[CLK_APMU_DMA]			= &dma_clk.common.hw,
+	[CLK_APMU_AES_WTM]		= &aes_wtm_clk.common.hw,
+	[CLK_APMU_VPU]			= &vpu_clk.common.hw,
+	[CLK_APMU_DTC]			= &dtc_clk.common.hw,
+	[CLK_APMU_GPU]			= &gpu_clk.common.hw,
+	[CLK_APMU_MC_AHB]		= &mc_ahb_clk.common.hw,
+	[CLK_APMU_TOP_DCLK]		= &top_dclk.common.hw,
+	[CLK_APMU_UCIE]			= &ucie_clk.common.hw,
+	[CLK_APMU_UCIE_SBCLK]		= &ucie_sbclk.common.hw,
+	[CLK_APMU_RCPU]			= &rcpu_clk.common.hw,
+	[CLK_APMU_DSI4LN2_DSI_ESC]	= &dsi4ln2_dsi_esc_clk.common.hw,
+	[CLK_APMU_DSI4LN2_LCD_DSC]	= &dsi4ln2_lcd_dsc_clk.common.hw,
+	[CLK_APMU_DSI4LN2_LCD_PXCLK]	= &dsi4ln2_lcd_pxclk.common.hw,
+	[CLK_APMU_DSI4LN2_LCD_MCLK]	= &dsi4ln2_lcd_mclk.common.hw,
+	[CLK_APMU_DSI4LN2_DPU_ACLK]	= &dsi4ln2_dpu_aclk.common.hw,
+	[CLK_APMU_DPU_ACLK]		= &dpu_aclk.common.hw,
+	[CLK_APMU_UFS_ACLK]		= &ufs_aclk.common.hw,
+	[CLK_APMU_UFS_REFCLK]		= &ufs_refclk.common.hw,
+	[CLK_APMU_EDP0_PXCLK]		= &edp0_pxclk.common.hw,
+	[CLK_APMU_EDP1_PXCLK]		= &edp1_pxclk.common.hw,
+	[CLK_APMU_PCIE_PORTA_MSTR]	= &pciea_mstr_clk.common.hw,
+	[CLK_APMU_PCIE_PORTA_SLV]	= &pciea_slv_clk.common.hw,
+	[CLK_APMU_PCIE_PORTA_BUS]	= &pciea_bus_clk.common.hw,
+	[CLK_APMU_PCIE_PORTB_MSTR]	= &pcieb_mstr_clk.common.hw,
+	[CLK_APMU_PCIE_PORTB_SLV]	= &pcieb_slv_clk.common.hw,
+	[CLK_APMU_PCIE_PORTB_BUS]	= &pcieb_bus_clk.common.hw,
+	[CLK_APMU_PCIE_PORTC_MSTR]	= &pciec_mstr_clk.common.hw,
+	[CLK_APMU_PCIE_PORTC_SLV]	= &pciec_slv_clk.common.hw,
+	[CLK_APMU_PCIE_PORTC_BUS]	= &pciec_bus_clk.common.hw,
+	[CLK_APMU_PCIE_PORTD_MSTR]	= &pcied_mstr_clk.common.hw,
+	[CLK_APMU_PCIE_PORTD_SLV]	= &pcied_slv_clk.common.hw,
+	[CLK_APMU_PCIE_PORTD_BUS]	= &pcied_bus_clk.common.hw,
+	[CLK_APMU_PCIE_PORTE_MSTR]	= &pciee_mstr_clk.common.hw,
+	[CLK_APMU_PCIE_PORTE_SLV]	= &pciee_slv_clk.common.hw,
+	[CLK_APMU_PCIE_PORTE_BUS]	= &pciee_bus_clk.common.hw,
+	[CLK_APMU_IOMMU]		= &iommu_clk.common.hw,
+	[CLK_APMU_EMAC0_BUS]		= &emac0_bus_clk.common.hw,
+	[CLK_APMU_EMAC0_REF]		= &emac0_ref_clk.common.hw,
+	[CLK_APMU_EMAC0_1588]		= &emac0_1588_clk.common.hw,
+	[CLK_APMU_EMAC0_RGMII_TX]	= &emac0_rgmii_tx_clk.common.hw,
+	[CLK_APMU_EMAC1_BUS]		= &emac1_bus_clk.common.hw,
+	[CLK_APMU_EMAC1_REF]		= &emac1_ref_clk.common.hw,
+	[CLK_APMU_EMAC1_1588]		= &emac1_1588_clk.common.hw,
+	[CLK_APMU_EMAC1_RGMII_TX]	= &emac1_rgmii_tx_clk.common.hw,
+	[CLK_APMU_EMAC2_BUS]		= &emac2_bus_clk.common.hw,
+	[CLK_APMU_EMAC2_REF]		= &emac2_ref_clk.common.hw,
+	[CLK_APMU_EMAC2_1588]		= &emac2_1588_clk.common.hw,
+	[CLK_APMU_EMAC2_RGMII_TX]	= &emac2_rgmii_tx_clk.common.hw,
+	[CLK_APMU_ESPI_SCLK_SRC]	= &espi_sclk_src.common.hw,
+	[CLK_APMU_ESPI_SCLK]		= &espi_sclk.common.hw,
+	[CLK_APMU_ESPI_MCLK]		= &espi_mclk.common.hw,
+	[CLK_APMU_CAM_SRC1]		= &cam_src1_clk.common.hw,
+	[CLK_APMU_CAM_SRC2]		= &cam_src2_clk.common.hw,
+	[CLK_APMU_CAM_SRC3]		= &cam_src3_clk.common.hw,
+	[CLK_APMU_CAM_SRC4]		= &cam_src4_clk.common.hw,
+	[CLK_APMU_ISIM_VCLK0]		= &isim_vclk_out0.common.hw,
+	[CLK_APMU_ISIM_VCLK1]		= &isim_vclk_out1.common.hw,
+	[CLK_APMU_ISIM_VCLK2]		= &isim_vclk_out2.common.hw,
+	[CLK_APMU_ISIM_VCLK3]		= &isim_vclk_out3.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_apmu_data = {
+	.reset_name	= "apmu-reset",
+	.hws		= k3_ccu_apmu_hws,
+	.num		= ARRAY_SIZE(k3_ccu_apmu_hws),
+};
+
+static struct clk_hw *k3_ccu_dciu_hws[] = {
+	[CLK_DCIU_HDMA]			= &hdma_clk.common.hw,
+	[CLK_DCIU_DMA350]		= &dma350_clk.common.hw,
+	[CLK_DCIU_C2_TCM_PIPE]		= &c2_tcm_pipe_clk.common.hw,
+	[CLK_DCIU_C3_TCM_PIPE]		= &c3_tcm_pipe_clk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_dciu_data = {
+	.reset_name	= "dciu-reset",
+	.hws		= k3_ccu_dciu_hws,
+	.num		= ARRAY_SIZE(k3_ccu_dciu_hws),
+};
+
+static struct clk_hw *k3_ccu_rcpu_sysctrl_hws[] = {
+	[CLK_RCPU_SYSCTRL_RCAN0]		= &rcan0_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN1]		= &rcan1_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN2]		= &rcan2_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN3]		= &rcan3_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN4]		= &rcan4_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN0_BUS]		= &rcan0_bus_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN1_BUS]		= &rcan1_bus_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN2_BUS]		= &rcan2_bus_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN3_BUS]		= &rcan3_bus_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RCAN4_BUS]		= &rcan4_bus_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RIRC0]		= &rirc0_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RIRC1]		= &rirc1_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RESPI_SCLK_SRC]	= &respi_sclk_src.common.hw,
+	[CLK_RCPU_SYSCTRL_RESPI_SCLK]		= &respi_sclk.common.hw,
+	[CLK_RCPU_SYSCTRL_REMAC_BUS]		= &remac_bus_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_REMAC_REF]		= &remac_ref_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_REMAC_1588]		= &remac_1588_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_REMAC_RGMII_TX]	= &remac_rgmii_tx_clk.common.hw,
+	[CLK_RCPU_SYSCTRL_RI2S0_SYSCLK]		= &ri2s0_sysclk.common.hw,
+	[CLK_RCPU_SYSCTRL_RI2S1_SYSCLK]		= &ri2s1_sysclk.common.hw,
+	[CLK_RCPU_SYSCTRL_RUART_14_SRC]		= &ruart_14_src.common.hw,
+	[CLK_RCPU_SYSCTRL_RUART_58_SRC]		= &ruart_58_src.common.hw,
+	[CLK_RCPU_SYSCTRL_RUART_14]		= &ruart_14.common.hw,
+	[CLK_RCPU_SYSCTRL_RUART_58]		= &ruart_58.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_rcpu_sysctrl_data = {
+	.reset_name	= "rsysctrl-reset",
+	.hws		= k3_ccu_rcpu_sysctrl_hws,
+	.num		= ARRAY_SIZE(k3_ccu_rcpu_sysctrl_hws),
+};
+
+static struct clk_hw *k3_ccu_rcpu_uartctrl_hws[] = {
+	[CLK_RCPU_UARTCTRL_RUART0]		= &ruart0_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART1]		= &ruart1_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART2]		= &ruart2_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART3]		= &ruart3_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART4]		= &ruart4_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART5]		= &ruart5_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART0_BUS]		= &ruart0_bus_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART1_BUS]		= &ruart1_bus_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART2_BUS]		= &ruart2_bus_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART3_BUS]		= &ruart3_bus_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART4_BUS]		= &ruart4_bus_clk.common.hw,
+	[CLK_RCPU_UARTCTRL_RUART5_BUS]		= &ruart5_bus_clk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_rcpu_uartctrl_data = {
+	.reset_name	= "ruartctrl-reset",
+	.hws		= k3_ccu_rcpu_uartctrl_hws,
+	.num		= ARRAY_SIZE(k3_ccu_rcpu_uartctrl_hws),
+};
+
+static struct clk_hw *k3_ccu_rcpu_i2sctrl_hws[] = {
+	[CLK_RCPU_I2SCTRL_RI2S0]		= &ri2s0_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S1]		= &ri2s1_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S2]		= &ri2s2_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S3]		= &ri2s3_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S0_BUS]		= &ri2s0_bus_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S1_BUS]		= &ri2s1_bus_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S2_BUS]		= &ri2s2_bus_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S3_BUS]		= &ri2s3_bus_clk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S2_SYSCLK]		= &ri2s2_sysclk.common.hw,
+	[CLK_RCPU_I2SCTRL_RI2S3_SYSCLK]		= &ri2s3_sysclk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_rcpu_i2sctrl_data = {
+	.reset_name	= "ri2sctrl-reset",
+	.hws		= k3_ccu_rcpu_i2sctrl_hws,
+	.num		= ARRAY_SIZE(k3_ccu_rcpu_i2sctrl_hws),
+};
+
+static struct clk_hw *k3_ccu_rcpu_spictrl_hws[] = {
+	[CLK_RCPU_SPICTRL_RSPI0]		= &rspi0_clk.common.hw,
+	[CLK_RCPU_SPICTRL_RSPI1]		= &rspi1_clk.common.hw,
+	[CLK_RCPU_SPICTRL_RSPI2]		= &rspi2_clk.common.hw,
+	[CLK_RCPU_SPICTRL_RSPI0_BUS]		= &rspi0_bus_clk.common.hw,
+	[CLK_RCPU_SPICTRL_RSPI1_BUS]		= &rspi1_bus_clk.common.hw,
+	[CLK_RCPU_SPICTRL_RSPI2_BUS]		= &rspi2_bus_clk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_rcpu_spictrl_data = {
+	.reset_name	= "rspictrl-reset",
+	.hws		= k3_ccu_rcpu_spictrl_hws,
+	.num		= ARRAY_SIZE(k3_ccu_rcpu_spictrl_hws),
+};
+
+static struct clk_hw *k3_ccu_rcpu_i2cctrl_hws[] = {
+	[CLK_RCPU_I2CCTRL_RI2C0]		= &ri2c0_clk.common.hw,
+	[CLK_RCPU_I2CCTRL_RI2C1]		= &ri2c1_clk.common.hw,
+	[CLK_RCPU_I2CCTRL_RI2C2]		= &ri2c2_clk.common.hw,
+	[CLK_RCPU_I2CCTRL_RI2C0_BUS]		= &ri2c0_bus_clk.common.hw,
+	[CLK_RCPU_I2CCTRL_RI2C1_BUS]		= &ri2c1_bus_clk.common.hw,
+	[CLK_RCPU_I2CCTRL_RI2C2_BUS]		= &ri2c2_bus_clk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_rcpu_i2cctrl_data = {
+	.reset_name	= "ri2cctrl-reset",
+	.hws		= k3_ccu_rcpu_i2cctrl_hws,
+	.num		= ARRAY_SIZE(k3_ccu_rcpu_i2cctrl_hws),
+};
+
+static struct clk_hw *k3_ccu_rpmu_hws[] = {
+	[CLK_RPMU_RCPU_AXI]		= &rcpu_axi_clk.common.hw,
+	[CLK_RPMU_RCPU_APB]		= &rcpu_apb_clk.common.hw,
+	[CLK_RPMU_RIPC2MSA]		= &ripc2msa_clk.common.hw,
+	[CLK_RPMU_RIPC2CP]		= &ripc2cp_clk.common.hw,
+	[CLK_RPMU_RIPC2AP]		= &ripc2ap_clk.common.hw,
+	[CLK_RPMU_RTIMER1]		= &rtimer1_clk.common.hw,
+	[CLK_RPMU_RTIMER2]		= &rtimer2_clk.common.hw,
+	[CLK_RPMU_RTIMER3]		= &rtimer3_clk.common.hw,
+	[CLK_RPMU_RTIMER4]		= &rtimer4_clk.common.hw,
+	[CLK_RPMU_RTIMER1_BUS]		= &rtimer1_bus_clk.common.hw,
+	[CLK_RPMU_RTIMER2_BUS]		= &rtimer2_bus_clk.common.hw,
+	[CLK_RPMU_RTIMER3_BUS]		= &rtimer3_bus_clk.common.hw,
+	[CLK_RPMU_RTIMER4_BUS]		= &rtimer4_bus_clk.common.hw,
+	[CLK_RPMU_RT24_CORE0]		= &rt24_core0_clk.common.hw,
+	[CLK_RPMU_RT24_CORE1]		= &rt24_core1_clk.common.hw,
+	[CLK_RPMU_RGPIO]		= &rgpio_clk.common.hw,
+	[CLK_RPMU_RGPIO_EDGE]		= &rgpio_edge_clk.common.hw,
+	[CLK_RPMU_RGPIO_LP]		= &rgpio_lp_clk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_rpmu_data = {
+	.reset_name	= "rpmu-reset",
+	.hws		= k3_ccu_rpmu_hws,
+	.num		= ARRAY_SIZE(k3_ccu_rpmu_hws),
+};
+
+static struct clk_hw *k3_ccu_rcpu_pwmctrl_hws[] = {
+	[CLK_RCPU_PWMCTRL_RPWM0]	= &rpwm0_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM1]	= &rpwm1_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM2]	= &rpwm2_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM3]	= &rpwm3_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM4]	= &rpwm4_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM5]	= &rpwm5_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM6]	= &rpwm6_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM7]	= &rpwm7_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM8]	= &rpwm8_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM9]	= &rpwm9_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM0_BUS]	= &rpwm0_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM1_BUS]	= &rpwm1_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM2_BUS]	= &rpwm2_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM3_BUS]	= &rpwm3_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM4_BUS]	= &rpwm4_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM5_BUS]	= &rpwm5_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM6_BUS]	= &rpwm6_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM7_BUS]	= &rpwm7_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM8_BUS]	= &rpwm8_bus_clk.common.hw,
+	[CLK_RCPU_PWMCTRL_RPWM9_BUS]	= &rpwm9_bus_clk.common.hw,
+};
+static const struct spacemit_ccu_data k3_ccu_rcpu_pwmctrl_data = {
+	.reset_name	= "rpwmctrl-reset",
+	.hws		= k3_ccu_rcpu_pwmctrl_hws,
+	.num		= ARRAY_SIZE(k3_ccu_rcpu_pwmctrl_hws),
+};
+
+static struct clk_hw *k3_ccu_apbc2_hws[] = {
+	[CLK_APBC2_SEC_UART1]		= &uart1_sec_clk.common.hw,
+	[CLK_APBC2_SEC_UART1_BUS]	= &uart1_sec_bus_clk.common.hw,
+	[CLK_APBC2_SEC_SPI2_I2S_BCLK]	= &spi2_i2s_bclk.common.hw,
+	[CLK_APBC2_SEC_SPI2]		= &spi2_sec_clk.common.hw,
+	[CLK_APBC2_SEC_SPI2_BUS]	= &spi2_sec_bus_clk.common.hw,
+	[CLK_APBC2_SEC_TWSI3]		= &twsi3_sec_clk.common.hw,
+	[CLK_APBC2_SEC_TWSI3_BUS]	= &twsi3_sec_bus_clk.common.hw,
+	[CLK_APBC2_SEC_RTC]		= &rtc_sec_clk.common.hw,
+	[CLK_APBC2_SEC_RTC_BUS]		= &rtc_sec_bus_clk.common.hw,
+	[CLK_APBC2_SEC_TIMERS]		= &timers_sec_clk.common.hw,
+	[CLK_APBC2_SEC_TIMERS_BUS]	= &timers_sec_bus_clk.common.hw,
+	[CLK_APBC2_SEC_GPIO]		= &gpio_sec_clk.common.hw,
+	[CLK_APBC2_SEC_GPIO_BUS]	= &gpio_sec_bus_clk.common.hw,
+};
+
+static const struct spacemit_ccu_data k3_ccu_apbc2_data = {
+	.reset_name	= "apbc2-reset",
+	.hws		= k3_ccu_apbc2_hws,
+	.num		= ARRAY_SIZE(k3_ccu_apbc2_hws),
+};
+
+static int spacemit_ccu_register(struct device *dev,
+				 struct regmap *regmap,
+				 struct regmap *lock_regmap,
+				 const struct spacemit_ccu_data *data)
+{
+	struct clk_hw_onecell_data *clk_data;
+	int i, ret;
+
+	/* Nothing to do if the CCU does not implement any clocks */
+	if (!data->hws)
+		return 0;
+
+	clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, data->num),
+				GFP_KERNEL);
+	if (!clk_data)
+		return -ENOMEM;
+
+	for (i = 0; i < data->num; i++) {
+		struct clk_hw *hw = data->hws[i];
+		struct ccu_common *common;
+		const char *name;
+
+		if (!hw) {
+			clk_data->hws[i] = ERR_PTR(-ENOENT);
+			continue;
+		}
+
+		name = hw->init->name;
+
+		common = hw_to_ccu_common(hw);
+		common->regmap		= regmap;
+		common->lock_regmap	= lock_regmap;
+
+		ret = devm_clk_hw_register(dev, hw);
+		if (ret) {
+			dev_err(dev, "Cannot register clock %d - %s\n",
+				i, name);
+			return ret;
+		}
+
+		clk_data->hws[i] = hw;
+	}
+
+	clk_data->num = data->num;
+
+	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data);
+	if (ret)
+		dev_err(dev, "failed to add clock hardware provider (%d)\n", ret);
+
+	return ret;
+}
+
+static void spacemit_cadev_release(struct device *dev)
+{
+	struct auxiliary_device *adev = to_auxiliary_dev(dev);
+
+	kfree(to_spacemit_ccu_adev(adev));
+}
+
+static void spacemit_adev_unregister(void *data)
+{
+	struct auxiliary_device *adev = data;
+
+	auxiliary_device_delete(adev);
+	auxiliary_device_uninit(adev);
+}
+
+static int spacemit_ccu_reset_register(struct device *dev,
+				       struct regmap *regmap,
+				       const char *reset_name)
+{
+	struct spacemit_ccu_adev *cadev;
+	struct auxiliary_device *adev;
+	static u32 next_id;
+	int ret;
+
+	/* Nothing to do if the CCU does not implement a reset controller */
+	if (!reset_name)
+		return 0;
+
+	cadev = devm_kzalloc(dev, sizeof(*cadev), GFP_KERNEL);
+	if (!cadev)
+		return -ENOMEM;
+	cadev->regmap = regmap;
+
+	adev = &cadev->adev;
+	adev->name = reset_name;
+	adev->dev.parent = dev;
+	adev->dev.release = spacemit_cadev_release;
+	adev->dev.of_node = dev->of_node;
+	adev->id = next_id++;
+
+	ret = auxiliary_device_init(adev);
+	if (ret)
+		return ret;
+
+	ret = auxiliary_device_add(adev);
+	if (ret) {
+		auxiliary_device_uninit(adev);
+		return ret;
+	}
+
+	return devm_add_action_or_reset(dev, spacemit_adev_unregister, adev);
+}
+
+static int k3_ccu_probe(struct platform_device *pdev)
+{
+	struct regmap *base_regmap, *lock_regmap = NULL;
+	const struct spacemit_ccu_data *data;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	base_regmap = device_node_to_regmap(dev->of_node);
+	if (IS_ERR(base_regmap))
+		return dev_err_probe(dev, PTR_ERR(base_regmap),
+				     "failed to get regmap\n");
+	/*
+	 * The lock status of PLLs locate in MPMU region, while PLLs themselves
+	 * are in APBS region. Reference to MPMU syscon is required to check PLL
+	 * status.
+	 */
+	if (of_device_is_compatible(dev->of_node, "spacemit,k3-pll")) {
+		struct device_node *mpmu = of_parse_phandle(dev->of_node,
+							    "spacemit,mpmu", 0);
+
+		if (!mpmu)
+			return dev_err_probe(dev, -ENODEV,
+					     "Cannot parse MPMU region\n");
+
+		lock_regmap = device_node_to_regmap(mpmu);
+		of_node_put(mpmu);
+
+		if (IS_ERR(lock_regmap))
+			return dev_err_probe(dev, PTR_ERR(lock_regmap),
+					     "failed to get lock regmap\n");
+	}
+
+	data = of_device_get_match_data(dev);
+
+	ret = spacemit_ccu_register(dev, base_regmap, lock_regmap, data);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register clocks\n");
+
+	ret = spacemit_ccu_reset_register(dev, base_regmap, data->reset_name);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register resets\n");
+
+	return 0;
+}
+
+static const struct of_device_id of_k3_ccu_match[] = {
+	{
+		.compatible	= "spacemit,k3-pll",
+		.data		= &k3_ccu_pll_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-mpmu",
+		.data		= &k3_ccu_mpmu_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-apbc",
+		.data		= &k3_ccu_apbc_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-apmu",
+		.data		= &k3_ccu_apmu_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-dciu",
+		.data		= &k3_ccu_dciu_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-rcpu-sysctrl",
+		.data		= &k3_ccu_rcpu_sysctrl_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-rcpu-uartctrl",
+		.data		= &k3_ccu_rcpu_uartctrl_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-rcpu-i2sctrl",
+		.data		= &k3_ccu_rcpu_i2sctrl_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-rcpu-spictrl",
+		.data		= &k3_ccu_rcpu_spictrl_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-rcpu-i2cctrl",
+		.data		= &k3_ccu_rcpu_i2cctrl_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-rpmu",
+		.data		= &k3_ccu_rpmu_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-rcpu-pwmctrl",
+		.data		= &k3_ccu_rcpu_pwmctrl_data,
+	},
+	{
+		.compatible	= "spacemit,k3-syscon-apbc2",
+		.data		= &k3_ccu_apbc2_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, of_k3_ccu_match);
+
+static struct platform_driver k3_ccu_driver = {
+	.driver = {
+		.name		= "spacemit,k3-ccu",
+		.of_match_table = of_k3_ccu_match,
+	},
+	.probe	= k3_ccu_probe,
+};
+module_platform_driver(k3_ccu_driver);
+
+MODULE_DESCRIPTION("SpacemiT K3 CCU driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/spacemit/ccu_common.h b/drivers/clk/spacemit/ccu_common.h
index da72f3836e0b..b725fa6de0b9 100644
--- a/drivers/clk/spacemit/ccu_common.h
+++ b/drivers/clk/spacemit/ccu_common.h
@@ -9,6 +9,14 @@
 
 #include <linux/regmap.h>
 
+/* invert en-bit value of gate clock, 1: disable, 0: enable */
+#define CCU_GATE_INVERT_FLAG		BIT(31)
+/* The maximum divisor (0xF) is invalid to the 4-bit divider in PMUAP base */
+#define CCU_DIV_4BIT_NO_0F_FLAG		BIT(30)
+/* Some dividers in PMUAP are only valid for the first four clock sources */
+#define CCU_DIV_VALID_FIRST4_SRC_FLAG	BIT(29)
+#define CCU_DIV_VALID_SRC_MAX		4
+
 struct ccu_common {
 	struct regmap *regmap;
 	struct regmap *lock_regmap;
@@ -24,6 +32,7 @@ struct ccu_common {
 		/* For PLL */
 		struct {
 			u32 reg_swcr1;
+			u32 reg_swcr2;
 			u32 reg_swcr3;
 		};
 	};
diff --git a/drivers/clk/spacemit/ccu_mix.c b/drivers/clk/spacemit/ccu_mix.c
index 7b7990875372..e7cdbe10fbcd 100644
--- a/drivers/clk/spacemit/ccu_mix.c
+++ b/drivers/clk/spacemit/ccu_mix.c
@@ -16,8 +16,12 @@
 static void ccu_gate_disable(struct clk_hw *hw)
 {
 	struct ccu_mix *mix = hw_to_ccu_mix(hw);
+	struct ccu_gate_config *gate = &mix->gate;
 
-	ccu_update(&mix->common, ctrl, mix->gate.mask, 0);
+	if (gate->flags & CCU_GATE_INVERT_FLAG)
+		ccu_update(&mix->common, ctrl, gate->mask, gate->mask);
+	else
+		ccu_update(&mix->common, ctrl, mix->gate.mask, 0);
 }
 
 static int ccu_gate_enable(struct clk_hw *hw)
@@ -25,7 +29,10 @@ static int ccu_gate_enable(struct clk_hw *hw)
 	struct ccu_mix *mix = hw_to_ccu_mix(hw);
 	struct ccu_gate_config *gate = &mix->gate;
 
-	ccu_update(&mix->common, ctrl, gate->mask, gate->mask);
+	if (gate->flags & CCU_GATE_INVERT_FLAG)
+		ccu_update(&mix->common, ctrl, gate->mask, 0);
+	else
+		ccu_update(&mix->common, ctrl, gate->mask, gate->mask);
 
 	return 0;
 }
@@ -35,7 +42,10 @@ static int ccu_gate_is_enabled(struct clk_hw *hw)
 	struct ccu_mix *mix = hw_to_ccu_mix(hw);
 	struct ccu_gate_config *gate = &mix->gate;
 
-	return (ccu_read(&mix->common, ctrl) & gate->mask) == gate->mask;
+	if (gate->flags & CCU_GATE_INVERT_FLAG)
+		return (ccu_read(&mix->common, ctrl) & gate->mask) == 0;
+	else
+		return (ccu_read(&mix->common, ctrl) & gate->mask) == gate->mask;
 }
 
 static unsigned long ccu_factor_recalc_rate(struct clk_hw *hw,
@@ -52,10 +62,21 @@ static unsigned long ccu_div_recalc_rate(struct clk_hw *hw,
 	struct ccu_mix *mix = hw_to_ccu_mix(hw);
 	struct ccu_div_config *div = &mix->div;
 	unsigned long val;
+	struct ccu_mux_config *mux = &mix->mux;
+	u8 parent_id = 0;
 
 	val = ccu_read(&mix->common, ctrl) >> div->shift;
 	val &= (1 << div->width) - 1;
 
+	if (div->flags & CCU_DIV_VALID_FIRST4_SRC_FLAG) {
+		if (!mux) {
+			parent_id = ccu_read(&mix->common, ctrl) >> mux->shift;
+			parent_id &= (1 << mux->width) - 1;
+			if (parent_id >= CCU_DIV_VALID_SRC_MAX)
+				return parent_rate;
+		}
+	}
+
 	return divider_recalc_rate(hw, parent_rate, val, NULL, 0, div->width);
 }
 
@@ -69,7 +90,7 @@ static int ccu_mix_trigger_fc(struct clk_hw *hw)
 	struct ccu_common *common = hw_to_ccu_common(hw);
 	unsigned int val;
 
-	if (common->reg_fc)
+	if (!common->reg_fc)
 		return 0;
 
 	ccu_update(common, fc, common->mask_fc, common->mask_fc);
@@ -104,7 +125,10 @@ ccu_mix_calc_best_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned int parent_num = clk_hw_get_num_parents(hw);
 	struct ccu_div_config *div = &mix->div;
 	u32 div_max = 1 << div->width;
-	unsigned long best_rate = 0;
+	unsigned long best_rate = clk_hw_get_rate(hw);
+
+	if (div->flags & CCU_DIV_4BIT_NO_0F_FLAG)
+		div_max -= 1;
 
 	for (int i = 0; i < parent_num; i++) {
 		struct clk_hw *parent = clk_hw_get_parent_by_index(hw, i);
@@ -113,6 +137,10 @@ ccu_mix_calc_best_rate(struct clk_hw *hw, unsigned long rate,
 		if (!parent)
 			continue;
 
+		if ((div->flags & CCU_DIV_VALID_FIRST4_SRC_FLAG)
+		     && i >= CCU_DIV_VALID_SRC_MAX)
+			div_max = 1;
+
 		parent_rate = clk_hw_get_rate(parent);
 
 		for (int j = 1; j <= div_max; j++) {
diff --git a/drivers/clk/spacemit/ccu_mix.h b/drivers/clk/spacemit/ccu_mix.h
index 54d40cd39b27..7865ba055ca2 100644
--- a/drivers/clk/spacemit/ccu_mix.h
+++ b/drivers/clk/spacemit/ccu_mix.h
@@ -19,6 +19,7 @@
  */
 struct ccu_gate_config {
 	u32 mask;
+	u32 flags;
 };
 
 struct ccu_factor_config {
@@ -34,6 +35,7 @@ struct ccu_mux_config {
 struct ccu_div_config {
 	u8 shift;
 	u8 width;
+	u32 flags;
 };
 
 struct ccu_mix {
@@ -48,7 +50,11 @@ struct ccu_mix {
 #define CCU_FACTOR_INIT(_div, _mul)	{ .div = _div, .mul = _mul }
 #define CCU_MUX_INIT(_shift, _width)	{ .shift = _shift, .width = _width }
 #define CCU_DIV_INIT(_shift, _width)	{ .shift = _shift, .width = _width }
+#define CCU_DIV_FLAGS_INIT(_shift, _width, _flags)	\
+	{ .shift = _shift, .width = _width, .flags = _flags }
+#define CCU_GATE_FLAGS_INIT(_mask, _flags)	{ .mask = _mask, .flags = _flags }
 
+#define CCU_PARENT_HW_NULL()          { .hw = NULL}
 #define CCU_PARENT_HW(_parent)		{ .hw = &_parent.common.hw }
 #define CCU_PARENT_NAME(_name)		{ .fw_name = #_name }
 
@@ -101,6 +107,15 @@ static struct ccu_mix _name = {							\
 	}									\
 }
 
+#define CCU_GATE_FLAGS_DEFINE(_name, _parent, _reg_ctrl, _mask_gate, _flags)		\
+static struct ccu_mix _name = {							\
+	.gate	= CCU_GATE_FLAGS_INIT(_mask_gate, _flags),					\
+	.common	= {								\
+		.reg_ctrl	= _reg_ctrl,					\
+		CCU_MIX_INITHW(_name, _parent, spacemit_ccu_gate_ops, _flags),	\
+	}									\
+}
+
 #define CCU_FACTOR_GATE_FLAGS_DEFINE(_name, _parent, _reg_ctrl, _mask_gate, _div,	\
 			       _mul, _flags)					\
 static struct ccu_mix _name = {							\
@@ -145,7 +160,7 @@ static struct ccu_mix _name = {							\
 				 _muxshift, _muxwidth, _mask_gate, _flags)	\
 static struct ccu_mix _name = {							\
 	.gate	= CCU_GATE_INIT(_mask_gate),					\
-	.div	= CCU_DIV_INIT(_mshift, _mwidth),				\
+	.div	= CCU_DIV_FLAGS_INIT(_mshift, _mwidth, _flags),			\
 	.mux	= CCU_MUX_INIT(_muxshift, _muxwidth),				\
 	.common	= {								\
 		.reg_ctrl	= _reg_ctrl,					\
@@ -159,7 +174,7 @@ static struct ccu_mix _name = {							\
 					 _muxwidth, _mask_gate, _flags)		\
 static struct ccu_mix _name = {							\
 	.gate	= CCU_GATE_INIT(_mask_gate),					\
-	.div	= CCU_DIV_INIT(_mshift, _mwidth),				\
+	.div	= CCU_DIV_FLAGS_INIT(_mshift, _mwidth, _flags),			\
 	.mux	= CCU_MUX_INIT(_muxshift, _muxwidth),				\
 	.common = {								\
 		.reg_ctrl	= _reg_ctrl,					\
@@ -180,7 +195,7 @@ CCU_MUX_DIV_GATE_SPLIT_FC_DEFINE(_name, _parents, _reg_ctrl, _reg_ctrl, _mshift,
 #define CCU_MUX_DIV_FC_DEFINE(_name, _parents, _reg_ctrl, _mshift, _mwidth,	\
 			      _mask_fc, _muxshift, _muxwidth, _flags)		\
 static struct ccu_mix _name = {							\
-	.div	= CCU_DIV_INIT(_mshift, _mwidth),				\
+	.div	= CCU_DIV_FLAGS_INIT(_mshift, _mwidth, _flags),			\
 	.mux	= CCU_MUX_INIT(_muxshift, _muxwidth),				\
 	.common = {								\
 		.reg_ctrl	= _reg_ctrl,					\
@@ -191,6 +206,20 @@ static struct ccu_mix _name = {							\
 	},									\
 }
 
+#define CCU_MUX_GATE_FC_DEFINE(_name, _parents, _reg_ctrl, _mask_fc, _muxshift,	\
+			  _muxwidth, _mask_gate, _flags)\
+static struct ccu_mix _name = {							\
+	.gate	= CCU_GATE_INIT(_mask_gate),					\
+	.mux	= CCU_MUX_INIT(_muxshift, _muxwidth),				\
+	.common = {								\
+		.reg_ctrl	= _reg_ctrl,					\
+		.reg_fc		= _reg_ctrl,					\
+		.mask_fc	= _mask_fc,					\
+		CCU_MIX_INITHW_PARENTS(_name, _parents, spacemit_ccu_mux_ops,	\
+				       _flags)					\
+	},									\
+}
+
 #define CCU_MUX_FC_DEFINE(_name, _parents, _reg_ctrl, _mask_fc,	_muxshift,	\
 			  _muxwidth, _flags)					\
 static struct ccu_mix _name = {							\
@@ -204,6 +233,18 @@ static struct ccu_mix _name = {							\
 	},									\
 }
 
+#define CCU_DIV_FC_DEFINE(_name, _parent, _reg_ctrl, _mask_fc,	_mshift,	\
+			  _mwidth, _flags)					\
+static struct ccu_mix _name = {							\
+	.div	= CCU_DIV_INIT(_mshift, _mwidth),				\
+	.common = {								\
+		.reg_ctrl	= _reg_ctrl,					\
+		.reg_fc		= _reg_ctrl,					\
+		.mask_fc	= _mask_fc,					\
+		CCU_MIX_INITHW(_name, _parent, spacemit_ccu_div_ops, _flags)		\
+	},									\
+}
+
 static inline struct ccu_mix *hw_to_ccu_mix(struct clk_hw *hw)
 {
 	struct ccu_common *common = hw_to_ccu_common(hw);
diff --git a/drivers/clk/spacemit/ccu_pll.c b/drivers/clk/spacemit/ccu_pll.c
index d92f0dae65a4..cde0d41dc399 100644
--- a/drivers/clk/spacemit/ccu_pll.c
+++ b/drivers/clk/spacemit/ccu_pll.c
@@ -17,6 +17,9 @@
 #define PLL_SWCR3_EN		((u32)BIT(31))
 #define PLL_SWCR3_MASK		GENMASK(30, 0)
 
+#define PLLA_SWCR2_EN		((u32)BIT(16))
+#define PLLA_SWCR2_MASK 	GENMASK(15, 8)
+
 static const struct ccu_pll_rate_tbl *ccu_pll_lookup_best_rate(struct ccu_pll *pll,
 							       unsigned long rate)
 {
@@ -148,6 +151,114 @@ static int ccu_pll_init(struct clk_hw *hw)
 	return 0;
 }
 
+static const struct ccu_pll_rate_tbl *ccu_plla_lookup_matched_entry(struct ccu_pll *pll)
+{
+	struct ccu_pll_config *config = &pll->config;
+	u32 swcr1, swcr2, swcr3;
+	int i;
+
+	swcr1 = ccu_read(&pll->common, swcr1);
+	swcr2 = ccu_read(&pll->common, swcr2);
+	swcr3 = ccu_read(&pll->common, swcr3);
+	swcr2 &= PLLA_SWCR2_MASK;
+
+	for (i = 0; i < config->tbl_num; i++) {
+		const struct ccu_pll_rate_tbl *entry = &config->rate_tbl[i];
+
+		if (swcr1 == entry->swcr1 && swcr2 == entry->swcr2 && swcr3 == entry->swcr3)
+			return entry;
+	}
+
+	return NULL;
+}
+
+static void ccu_plla_update_param(struct ccu_pll *pll, const struct ccu_pll_rate_tbl *entry)
+{
+	struct ccu_common *common = &pll->common;
+
+	regmap_write(common->regmap, common->reg_swcr1, entry->swcr1);
+	regmap_write(common->regmap, common->reg_swcr3, entry->swcr3);
+	ccu_update(common, swcr2, PLLA_SWCR2_MASK, entry->swcr2);
+}
+
+static int ccu_plla_is_enabled(struct clk_hw *hw)
+{
+	struct ccu_common *common = hw_to_ccu_common(hw);
+
+	return ccu_read(common, swcr2) & PLLA_SWCR2_EN;
+}
+
+static int ccu_plla_enable(struct clk_hw *hw)
+{
+	struct ccu_pll *pll = hw_to_ccu_pll(hw);
+	struct ccu_common *common = &pll->common;
+#ifndef CONFIG_SOC_SPACEMIT_K3_FPGA
+	unsigned int tmp;
+#endif
+
+	ccu_update(common, swcr2, PLLA_SWCR2_EN, PLLA_SWCR2_EN);
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	return 0;
+#else
+	/* check lock status */
+	return regmap_read_poll_timeout_atomic(common->lock_regmap,
+					       pll->config.reg_lock,
+					       tmp,
+					       tmp & pll->config.mask_lock,
+					       PLL_DELAY_US, PLL_TIMEOUT_US);
+#endif
+}
+
+static void ccu_plla_disable(struct clk_hw *hw)
+{
+	struct ccu_common *common = hw_to_ccu_common(hw);
+
+	ccu_update(common, swcr2, PLLA_SWCR2_EN, 0);
+}
+
+/*
+ * PLLAs must be gated before changing rate, which is ensured by
+ * flag CLK_SET_RATE_GATE.
+ */
+static int ccu_plla_set_rate(struct clk_hw *hw, unsigned long rate,
+			     unsigned long parent_rate)
+{
+	struct ccu_pll *pll = hw_to_ccu_pll(hw);
+	const struct ccu_pll_rate_tbl *entry;
+
+	entry = ccu_pll_lookup_best_rate(pll, rate);
+	ccu_plla_update_param(pll, entry);
+
+	return 0;
+}
+
+static unsigned long ccu_plla_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct ccu_pll *pll = hw_to_ccu_pll(hw);
+	const struct ccu_pll_rate_tbl *entry;
+
+	entry = ccu_plla_lookup_matched_entry(pll);
+
+	WARN_ON_ONCE(!entry);
+
+	return entry ? entry->rate : -EINVAL;
+}
+
+static int ccu_plla_init(struct clk_hw *hw)
+{
+	struct ccu_pll *pll = hw_to_ccu_pll(hw);
+
+	if (ccu_plla_lookup_matched_entry(pll))
+		return 0;
+
+	ccu_plla_disable(hw);
+	ccu_plla_update_param(pll, &pll->config.rate_tbl[0]);
+
+	return 0;
+}
+
 const struct clk_ops spacemit_ccu_pll_ops = {
 	.init		= ccu_pll_init,
 	.enable		= ccu_pll_enable,
@@ -157,3 +268,14 @@ const struct clk_ops spacemit_ccu_pll_ops = {
 	.determine_rate = ccu_pll_determine_rate,
 	.is_enabled	= ccu_pll_is_enabled,
 };
+
+const struct clk_ops spacemit_ccu_plla_ops = {
+	.init		= ccu_plla_init,
+	.enable		= ccu_plla_enable,
+	.disable	= ccu_plla_disable,
+	.set_rate	= ccu_plla_set_rate,
+	.recalc_rate	= ccu_plla_recalc_rate,
+	.determine_rate	= ccu_pll_determine_rate,
+	.is_enabled	= ccu_plla_is_enabled,
+};
+
diff --git a/drivers/clk/spacemit/ccu_pll.h b/drivers/clk/spacemit/ccu_pll.h
index 0592f4c3068c..8727379f6ce6 100644
--- a/drivers/clk/spacemit/ccu_pll.h
+++ b/drivers/clk/spacemit/ccu_pll.h
@@ -15,15 +15,25 @@
  * struct ccu_pll_rate_tbl - Structure mapping between PLL rate and register
  * configuration.
  *
+ * Usual PLL type
  * @rate:	PLL rate
  * @swcr1:	Register value of PLLX_SW1_CTRL (PLLx_SWCR1).
+ * @swcr2:	Not use.
  * @swcr3:	Register value of the PLLx_SW3_CTRL's lowest 31 bits of
  *		PLLx_SW3_CTRL (PLLx_SWCR3). This highest bit is for enabling
  *		the PLL and not contained in this field.
+ *
+ * Special PLL type A
+ * @rate:	PLLA rate
+ * @swcr1:	Register value of PLLAX_SW1_CTRL (PLLAx_SWCR1).
+ * @swcr2:	Register value of PLLAX_SW2_CTRL[15:8] (PLLAx_SWCR2). PLLAX_SW2_CTRL[16]
+ *		is for enabling the PLLA and not contained in this field.
+ * @swcr3:	Register value of PLLAX_SW3_CTRL (PLLAx_SWCR3).
  */
 struct ccu_pll_rate_tbl {
 	unsigned long rate;
 	u32 swcr1;
+	u32 swcr2;
 	u32 swcr3;
 };
 
@@ -36,8 +46,16 @@ struct ccu_pll_config {
 
 #define CCU_PLL_RATE(_rate, _swcr1, _swcr3) \
 	{									\
-		.rate	= _rate,							\
+		.rate	= _rate,						\
+		.swcr1	= _swcr1,						\
+		.swcr3	= _swcr3,						\
+	}
+
+#define CCU_PLLA_RATE(_rate, _swcr1, _swcr2, _swcr3) \
+	{									\
+		.rate	= _rate,						\
 		.swcr1	= _swcr1,						\
+		.swcr2	= _swcr2,						\
 		.swcr3	= _swcr3,						\
 	}
 
@@ -63,6 +81,15 @@ struct ccu_pll {
 		.flags		= _flags,					\
 	})
 
+#define CCU_PLLA_HWINIT(_name, _flags)						\
+	(&(struct clk_init_data) {						\
+		.name		= #_name,					\
+		.ops		= &spacemit_ccu_plla_ops,			\
+		.parent_data	= &(struct clk_parent_data) { .index = 0 },	\
+		.num_parents	= 1,						\
+		.flags		= _flags,					\
+	})
+
 #define CCU_PLL_DEFINE(_name, _table, _reg_swcr1, _reg_swcr3, _reg_lock,	\
 		       _mask_lock, _flags)					\
 static struct ccu_pll _name = {							\
@@ -74,6 +101,18 @@ static struct ccu_pll _name = {							\
 	}									\
 }
 
+#define CCU_PLLA_DEFINE(_name, _table, _reg_swcr1, _reg_swcr2, _reg_swcr3,	\
+		       _reg_lock, _mask_lock, _flags)				\
+static struct ccu_pll _name = {							\
+	.config	= CCU_PLL_CONFIG(_table, _reg_lock, _mask_lock),		\
+	.common = {								\
+		.reg_swcr1	= _reg_swcr1,					\
+		.reg_swcr2	= _reg_swcr2,					\
+		.reg_swcr3	= _reg_swcr3,					\
+		.hw.init	= CCU_PLLA_HWINIT(_name, _flags)		\
+	}									\
+}
+
 static inline struct ccu_pll *hw_to_ccu_pll(struct clk_hw *hw)
 {
 	struct ccu_common *common = hw_to_ccu_common(hw);
@@ -82,5 +121,6 @@ static inline struct ccu_pll *hw_to_ccu_pll(struct clk_hw *hw)
 }
 
 extern const struct clk_ops spacemit_ccu_pll_ops;
+extern const struct clk_ops spacemit_ccu_plla_ops;
 
 #endif
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index ffcd23668763..fa45f2126564 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -152,6 +152,14 @@ config REALTEK_OTTO_TIMER
 	  RT8391, RTL8392, RTL8393 and RTL8396 and chips of the RTL930x series
 	  such as RTL9301, RTL9302 or RTL9303.
 
+config SPACEMIT_K1X_TIMER
+	bool "Spacemit k1x/k3 timer driver"
+	depends on SOC_SPACEMIT_K1 || SOC_SPACEMIT_K3 || COMPILE_TEST
+	select CLKSRC_MMIO
+	select TIMER_OF
+	help
+	  Enables the support for the spacemit k1x timer driver.
+
 config SUN4I_TIMER
 	bool "Sun4i timer driver" if COMPILE_TEST
 	depends on HAS_IOMEM
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index ec4452ee958f..c95fad68821d 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -95,3 +95,4 @@ obj-$(CONFIG_CLKSRC_LOONGSON1_PWM)	+= timer-loongson1-pwm.o
 obj-$(CONFIG_EP93XX_TIMER)		+= timer-ep93xx.o
 obj-$(CONFIG_RALINK_TIMER)		+= timer-ralink.o
 obj-$(CONFIG_NXP_STM_TIMER)		+= timer-nxp-stm.o
+obj-$(CONFIG_SPACEMIT_K1X_TIMER)	+= timer-k1x.o
diff --git a/drivers/clocksource/timer-k1x.c b/drivers/clocksource/timer-k1x.c
new file mode 100644
index 000000000000..afb0974a8087
--- /dev/null
+++ b/drivers/clocksource/timer-k1x.c
@@ -0,0 +1,837 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * spacemit-k1x timer driver
+ *
+ * Copyright (C) 2025 Spacemit
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/clockchips.h>
+
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/cpu.h>
+#include <linux/clockchips.h>
+#include <linux/sched_clock.h>
+#include <linux/stat.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/reset.h>
+#include <linux/platform_device.h>
+
+#define TMR_CCR		(0x000c)
+#define TMR_TN_MM(n, m)	(0x0010 + ((n) << 4) + ((m) << 2))
+#define TMR_CR(n)	(0x0090 + ((n) << 2))
+#define TMR_SR(n)	(0x0080 + ((n) << 2))
+#define TMR_IER(n)	(0x0060 + ((n) << 2))
+#define TMR_PLVR(n)	(0x0040 + ((n) << 2))
+#define TMR_PLCR(n)	(0x0050 + ((n) << 2))
+#define TMR_WMER	(0x0068)
+#define TMR_WMR		(0x006c)
+#define TMR_WVR		(0x00cc)
+#define TMR_WSR		(0x00c0)
+#define TMR_ICR(n)	(0x0070 + ((n) << 2))
+#define TMR_WICR	(0x00c4)
+#define TMR_CER		(0x0000)
+#define TMR_CMR		(0x0004)
+#define TMR_WCR		(0x00c8)
+#define TMR_WFAR	(0x00b0)
+#define TMR_WSAR	(0x00b4)
+#define TMR_CRSR        (0x0008)
+
+#define TMR_CCR_CS_0(x)	(((x) & 0x3) << 0)
+#define TMR_CCR_CS_1(x)	(((x) & 0x3) << 2)
+#define TMR_CCR_CS_2(x)	(((x) & 0x3) << 5)
+
+#define MAX_EVT_NUM		5
+
+#define MAX_DELTA		(0xfffffffe)
+#define MIN_DELTA		(5)
+
+#define SPACEMIT_MAX_COUNTER		3
+#define SPACEMIT_MAX_TIMER		3
+
+#define TMR_CER_COUNTER(cid)	(1 << (cid))
+#define SPACEMIT_ALL_COUNTERS	((1 << SPACEMIT_MAX_COUNTER) - 1)
+
+#define	SPACEMIT_TIMER_CLOCK_32KHZ	32768
+
+#define SPACEMIT_TIMER_COUNTER_CLKSRC	(1 << 0)
+#define SPACEMIT_TIMER_COUNTER_CLKEVT	(1 << 1)
+#define SPACEMIT_TIMER_COUNTER_DELAY		(1 << 2)
+
+#define SPACEMIT_TIMER_ALL_CPU	(0xFFFFFFFF)
+
+struct spacemit_timer;
+
+struct spacemit_timer_evt {
+	struct clock_event_device ced;
+	struct irqaction irqa;
+	unsigned int freq;
+	unsigned int irq;
+	unsigned int cid;
+	unsigned int tid;
+	int cpu;
+	bool timer_enabled;
+	/* 0: timer set; 1: timer timeout(irq comes) */
+	int timer_status;
+	unsigned int timeout;
+	struct spacemit_timer *timer;
+};
+
+struct spacemit_timer {
+	unsigned int id;
+	void __iomem *base;
+	struct spacemit_timer_evt evt[SPACEMIT_MAX_COUNTER];
+	unsigned int flag;
+	int loop_delay_fastclk;
+	unsigned int fc_freq;
+	unsigned int freq;
+	struct clk *clk;
+	struct clk *clk_bus;
+	/* lock to protect hw operation. */
+	raw_spinlock_t tm_lock;
+};
+
+struct timer_werror_info {
+	u32 reg;
+	u32 target;
+	u32 val;
+	u32 mask;
+};
+
+/* record the last x write failures */
+#define TIMER_ERR_NUM			10
+static struct timer_werror_info werr_info[TIMER_ERR_NUM];
+static int werr_index;
+
+static struct spacemit_timer *spacemit_timers[SPACEMIT_MAX_TIMER];
+static int timer_counter_switch_clock(struct spacemit_timer *tm, unsigned int freq);
+
+/* To silence -Wmissing-prototypes. */
+void timer_dump_hwinfo(int tid);
+void timer_dump_hwinfo(int tid)
+{
+	struct spacemit_timer_evt *t_evt = &spacemit_timers[tid]->evt[0];
+	void __iomem *base = spacemit_timers[tid]->base;
+	unsigned int sr, cid, cer, cmr, ccr, mr, ier, cr;
+
+	cid = t_evt->cid;
+
+	cer = __raw_readl(base + TMR_CER);
+	cmr = __raw_readl(base + TMR_CMR);
+	ccr = __raw_readl(base + TMR_CCR);
+	mr = __raw_readl(base + TMR_TN_MM(cid, 0));
+	ier = __raw_readl(base + TMR_IER(cid));
+	sr = __raw_readl(base + TMR_SR(cid));
+	cr = __raw_readl(base + TMR_CR(cid));
+
+	pr_err("timer enable: %d. timeout: %d cycles. next event: %lld\n", !t_evt->timer_status, t_evt->timeout, t_evt->ced.next_event);
+
+	pr_err("cer/cmr/ccr/mr/ier/sr/cr: (0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x)\n", cer, cmr, ccr, mr, ier, sr, cr);
+
+	return;
+}
+
+static void timer_write_error(u32 reg, u32 target, u32 val, u32 mask)
+{
+	werr_info[werr_index].reg = reg;
+	werr_info[werr_index].target = target;
+	werr_info[werr_index].val = val;
+	werr_info[werr_index].mask = mask;
+	werr_index = (werr_index+1) % TIMER_ERR_NUM;
+
+	pr_err("timer write fail: register = 0x%x: (0x%x, 0x%x, 0x%x)\n", reg, target, val, mask);
+}
+
+static void timer_write_check(struct spacemit_timer *tm, u32 reg, u32 val, u32 mask, bool clr, bool clk_switch)
+{
+	int loop = 3, retry = 100;
+	u32 t_read, t_check = clr ? !val : val;
+
+reg_re_write:
+	__raw_writel(val, tm->base + reg);
+
+	if (clk_switch)
+		timer_counter_switch_clock(tm, tm->fc_freq);
+
+	t_read = __raw_readl(tm->base + reg);
+
+	while (((t_read & mask) != (t_check & mask)) && loop) {
+		/* avoid trying frequently to worsen bus contention */
+		udelay(30);
+		t_read = __raw_readl(tm->base + reg);
+		loop--;
+
+		if (!loop) {
+			timer_write_error(reg, t_check, t_read, mask);
+			loop = 3;
+			if (--retry)
+				goto reg_re_write;
+			else
+				return;
+		}
+	}
+}
+
+static int timer_counter_switch_clock(struct spacemit_timer *tm, unsigned int freq)
+{
+	u32 ccr, val, mask, tid;
+
+	tid = tm->id;
+
+	ccr = __raw_readl(tm->base + TMR_CCR);
+
+	switch (tid) {
+	case 0:
+		mask = TMR_CCR_CS_0(3);
+		break;
+	case 1:
+		mask = TMR_CCR_CS_1(3);
+		break;
+	case 2:
+		mask = TMR_CCR_CS_2(3);
+		break;
+	default:
+		pr_err("wrong timer id: 0x%x\n", tid);
+		return -EINVAL;
+	}
+
+	ccr &= ~mask;
+
+	if (freq == tm->fc_freq)
+		val = 0;
+	else if (freq == SPACEMIT_TIMER_CLOCK_32KHZ)
+		val = 1;
+	else {
+		pr_err("Timer %d: invalid clock rate %d\n", tid, freq);
+		return -EINVAL;
+	}
+
+	switch (tid) {
+	case 0:
+		ccr |= TMR_CCR_CS_0(val);
+		break;
+	case 1:
+		ccr |= TMR_CCR_CS_1(val);
+		break;
+	case 2:
+		ccr |= TMR_CCR_CS_2(val);
+		break;
+	}
+
+	timer_write_check(tm, TMR_CCR, ccr, mask, false, false);
+
+	return 0;
+}
+
+static void timer_counter_disable(struct spacemit_timer_evt *evt)
+{
+	struct spacemit_timer *tm = evt->timer;
+	u32 cer;
+	bool clk_switch = false;
+
+	if (evt->freq != tm->fc_freq)
+		clk_switch = true;
+	/*
+	 * Stop the counter will need multiple timer clock to take effect.
+	 * Some operations can only be done when counter is disabled. So
+	 * add delay here.
+	 */
+	/* Step1: disable counter */
+	cer = __raw_readl(tm->base + TMR_CER);
+	timer_write_check(tm, TMR_CER, (cer & ~(1 << evt->cid)), (1 << evt->cid), false, clk_switch);
+
+	/* remove unnecesary write, check explicitly: 2 cycles (32k) */
+
+	evt->timer_status = 1;
+}
+
+static void timer_counter_enable(struct spacemit_timer_evt *evt)
+{
+	struct spacemit_timer *tm = evt->timer;
+	u32 cer;
+
+	/* Switch to original clock */
+	if (evt->freq != tm->fc_freq)
+		timer_counter_switch_clock(tm, evt->freq);
+
+	/* Enable timer */
+	cer = __raw_readl(tm->base + TMR_CER);
+
+	timer_write_check(tm, TMR_CER, (cer | (1 << evt->cid)), (1 << evt->cid), false, false);
+
+	evt->timer_status = 0;
+}
+
+static irqreturn_t timer_interrupt(int irq, void *dev_id)
+{
+	struct clock_event_device *c = dev_id;
+	struct spacemit_timer_evt *evt;
+	unsigned int cnt;
+	unsigned long flags;
+	void __iomem *base;
+
+	evt = container_of(c, struct spacemit_timer_evt, ced);
+	cnt = evt->cid;
+	base = evt->timer->base;
+
+	raw_spin_lock_irqsave(&(evt->timer->tm_lock), flags);
+	/* We only use match #0 for the counter. */
+	if (__raw_readl(base + TMR_SR(cnt)) & 0x1) {
+		timer_counter_disable(evt);
+
+		/* Disable the interrupt. */
+		timer_write_check(evt->timer, TMR_IER(cnt), 0, 0x7, false, false);
+		/* Clear interrupt status */
+		timer_write_check(evt->timer, TMR_ICR(cnt), 0x1, 0x7, true, false);
+
+		raw_spin_unlock_irqrestore(&(evt->timer->tm_lock), flags);
+
+		c->event_handler(c);
+
+		return IRQ_HANDLED;
+	}
+
+	raw_spin_unlock_irqrestore(&(evt->timer->tm_lock), flags);
+	return IRQ_NONE;
+}
+
+static int timer_set_shutdown(struct clock_event_device *dev)
+{
+	struct spacemit_timer_evt *evt;
+	unsigned long flags;
+
+	evt = container_of(dev, struct spacemit_timer_evt, ced);
+
+	raw_spin_lock_irqsave(&(evt->timer->tm_lock), flags);
+
+	evt->timer_enabled = !evt->timer_status;
+
+	/* disable counter */
+	timer_counter_disable(evt);
+
+	raw_spin_unlock_irqrestore(&(evt->timer->tm_lock), flags);
+
+	return 0;
+}
+
+static int timer_resume(struct clock_event_device *dev)
+{
+	struct spacemit_timer_evt *evt;
+	struct spacemit_timer *tm;
+	unsigned long flags;
+	int ret = 0;
+
+	evt = container_of(dev, struct spacemit_timer_evt, ced);
+	tm = evt->timer;
+
+	if (tm->clk_bus && !__clk_is_enabled(tm->clk_bus)) {
+		ret = clk_prepare_enable(tm->clk_bus);
+		if (ret) {
+			pr_err("Timer %d: Failed to enable bus clock in resume: %d\n", tm->id, ret);
+			return ret;
+		}
+	}
+	if (tm->clk && !__clk_is_enabled(tm->clk)) {
+		ret = clk_prepare_enable(tm->clk);
+		if (ret) {
+			pr_err("Timer %d: Failed to enable clock in resume: %d\n", tm->id, ret);
+			return ret;
+		}
+	}
+
+	raw_spin_lock_irqsave(&(tm->tm_lock), flags);
+
+	/* check whether need to enable timer */
+	if (evt->timer_enabled)
+		timer_counter_enable(evt);
+	raw_spin_unlock_irqrestore(&(tm->tm_lock), flags);
+
+	return 0;
+}
+
+static int timer_set_next_event(unsigned long delta,
+				struct clock_event_device *dev)
+{
+	struct spacemit_timer_evt *evt;
+	unsigned int cid;
+	unsigned long flags;
+	u32 cer;
+	void __iomem *base;
+
+	evt = container_of(dev, struct spacemit_timer_evt, ced);
+	cid = evt->cid;
+	base = evt->timer->base;
+
+	raw_spin_lock_irqsave(&(evt->timer->tm_lock), flags);
+
+	cer = __raw_readl(base + TMR_CER);
+
+	/* If the timer counter is enabled, first disable it. */
+	if (cer & (1 << cid))
+		timer_counter_disable(evt);
+
+	/* Setup new counter value */
+	timer_write_check(evt->timer, TMR_TN_MM(cid, 0), (delta - 1), (u32)(-1), false, false);
+
+	/* enable the matching interrupt */
+	timer_write_check(evt->timer, TMR_IER(cid), 0x1, 0x1, false, false);
+
+	timer_counter_enable(evt);
+
+	evt->timeout = delta - 1;
+
+	raw_spin_unlock_irqrestore(&(evt->timer->tm_lock), flags);
+	return 0;
+}
+
+static int __init spacemit_timer_init(struct device_node *np, int tid, void __iomem *base,
+			  unsigned int flag, unsigned int fc_freq,
+			  unsigned int apb_freq, unsigned int freq)
+{
+	struct spacemit_timer *tm = spacemit_timers[tid];
+	struct clk *clk, *clk_bus;
+	struct reset_control *resets;
+	u32 tmp, delay;
+
+	if (tm)
+		return -EINVAL;
+
+	tm = kzalloc(sizeof(*tm), GFP_KERNEL);
+	if (!tm)
+		return -ENOMEM;
+
+	clk = of_clk_get(np, 0);
+	if (!clk) {
+		pr_err("%s: get clk failed! %s\n", __func__, np->name);
+		goto out;
+	}
+
+	if (IS_ERR(clk)) {
+		pr_err("Timer %d: fail to get clock!\n", tid);
+		goto out;
+	}
+
+	clk_bus = of_clk_get(np, 1);
+	if (!clk_bus) {
+		pr_err("%s: get bus clk failed! %s\n", __func__, np->name);
+		clk_disable_unprepare(clk);
+		goto out;
+	}
+
+	if (clk_prepare_enable(clk_bus)) {
+		pr_err("Timer %d: fail to enable clk bus clock!\n", tid);
+		goto out;
+	}
+
+	if (clk_prepare_enable(clk)) {
+		clk_disable_unprepare(clk_bus);
+		pr_err("Timer %d: fail to enable clock!\n", tid);
+		goto out;
+	}
+
+	if (clk_set_rate(clk, fc_freq)) {
+		pr_err("Timer %d: fail to set clock rate to %uHz!\n", tid, fc_freq);
+		goto out;
+	}
+
+	tm->clk = clk;
+	tm->clk_bus = clk_bus;
+
+	resets = of_reset_control_get(np, 0);
+	if(IS_ERR(resets)) {
+		pr_err("%s: get resets failed! %s\n", __func__, np->name);
+		clk_disable_unprepare(clk);
+		clk_disable_unprepare(clk_bus);
+		return PTR_ERR(resets);
+	}
+	reset_control_deassert(resets);
+	/*
+	 * The calculation formula for the loop cycle is:
+	 *
+	 * (1) need wait for 2 timer's clock cycle:
+	 *        1             2
+	 *     ------- x 2 = -------
+	 *     fc_freq       fc_freq
+	 *
+	 * (2) convert to apb clock cycle:
+	 *        2          1        apb_freq * 2
+	 *     ------- / -------- = ----------------
+	 *     fc_freq   apb_freq       fc_freq
+	 *
+	 * (3) every apb register's accessing will take 8 apb clock cycle,
+	 *     also consider add extral one more time for safe way;
+	 *     so finally need loop times for the apb register accessing:
+	 *
+	 *       (apb_freq * 2)
+	 *     ------------------ / 8 + 1
+	 *          fc_freq
+	 */
+	delay = ((apb_freq * 2) / fc_freq / 8) + 1;
+	pr_err("Timer %d: loop_delay_fastclk is %d\n", tid, delay);
+
+	tm->id = tid;
+	tm->base = base;
+	tm->flag = flag;
+	tm->loop_delay_fastclk = delay;
+	tm->fc_freq = fc_freq;
+	tm->freq = freq;
+	raw_spin_lock_init(&(tm->tm_lock));
+
+	spacemit_timers[tid] = tm;
+
+	/* We will disable all counters. Switch to fastclk first. */
+	timer_counter_switch_clock(tm, fc_freq);
+
+	/* disalbe all counters */
+	tmp = __raw_readl(base + TMR_CER) & ~SPACEMIT_ALL_COUNTERS;
+	__raw_writel(tmp, base + TMR_CER);
+
+	/* disable matching interrupt */
+	__raw_writel(0x00, base + TMR_IER(0));
+	__raw_writel(0x00, base + TMR_IER(1));
+	__raw_writel(0x00, base + TMR_IER(2));
+
+	while (delay--) {
+		/* Clear pending interrupt status */
+		__raw_writel(0x1, base + TMR_ICR(0));
+		__raw_writel(0x1, base + TMR_ICR(1));
+		__raw_writel(0x1, base + TMR_ICR(2));
+		__raw_writel(tmp, base + TMR_CER);
+	}
+
+	return 0;
+out:
+	kfree(tm);
+	return -EINVAL;
+}
+
+static int __init spacemit_timer_hw_init(struct spacemit_timer_evt *evt)
+{
+	struct spacemit_timer *tm = evt->timer;
+	unsigned int tmp, delay, freq, cid, ratio;
+	int ret;
+
+	cid = evt->cid;
+	freq = evt->freq;
+
+	ret = timer_counter_switch_clock(tm, freq);
+	if (ret)
+		return ret;
+
+	ratio = tm->fc_freq / freq;
+	delay = tm->loop_delay_fastclk * ratio;
+
+	/* set timer to free-running mode */
+	tmp = __raw_readl(tm->base + TMR_CMR) | TMR_CER_COUNTER(cid);
+	__raw_writel(tmp, tm->base + TMR_CMR);
+
+	/* free-running */
+	__raw_writel(0x0, tm->base + TMR_PLCR(cid));
+	/* clear status */
+	__raw_writel(0x7, tm->base + TMR_ICR(cid));
+
+	/* enable counter */
+	tmp = __raw_readl(tm->base + TMR_CER) | TMR_CER_COUNTER(cid);
+	__raw_writel(tmp, tm->base + TMR_CER);
+
+	while (delay--)
+		__raw_writel(tmp, tm->base + TMR_CER);
+
+	return 0;
+}
+
+static int __init spacemit_timer_setup(struct spacemit_timer_evt *evt)
+{
+	int broadcast = 0;
+	int ret;
+
+	if (evt->cpu == SPACEMIT_TIMER_ALL_CPU)
+		broadcast = 1;
+	else if (evt->cpu >= num_possible_cpus())
+		return -EINVAL;
+
+	evt->ced.name = "timer-spacemit";
+	evt->ced.features = CLOCK_EVT_FEAT_ONESHOT;
+	evt->ced.rating = 200;
+	evt->ced.set_next_event = timer_set_next_event;
+	evt->ced.set_state_shutdown = timer_set_shutdown;
+	evt->ced.tick_resume = timer_resume;
+	evt->ced.irq = evt->irq;
+
+	evt->irqa.flags = IRQF_TIMER | IRQF_IRQPOLL;
+	evt->irqa.handler = timer_interrupt;
+	evt->irqa.dev_id = &(evt->ced);
+
+	ret = spacemit_timer_hw_init(evt);
+	if (ret)
+		return ret;
+
+	if (broadcast) {
+		evt->irqa.name = "broadcast-timer";
+		/* evt->ced.features |= CLOCK_EVT_FEAT_DYNIRQ; */
+		evt->ced.cpumask = cpu_possible_mask;
+		ret = request_irq(evt->ced.irq, timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL | IRQF_ONESHOT, "broadcast-timer", evt->irqa.dev_id);
+		if (ret < 0)
+			return -EPROBE_DEFER;
+		clockevents_config_and_register(&evt->ced,
+						evt->freq, MIN_DELTA, MAX_DELTA);
+	} else {
+		evt->irqa.name = "local-timer";
+		evt->ced.cpumask = cpumask_of(evt->cpu);
+		evt->irqa.flags |= IRQF_PERCPU;
+		ret = request_irq(evt->ced.irq, timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL, "local-timer", evt->irqa.dev_id);
+		if (ret < 0)
+			return ret;
+		/* Enable clock event device for boot CPU. */
+		if (evt->cpu == smp_processor_id()) {
+			clockevents_config_and_register(&evt->ced,
+							evt->freq, MIN_DELTA,
+							MAX_DELTA);
+			/* Only online CPU can be set affinity. */
+			irq_set_affinity_hint(evt->ced.irq, cpumask_of(evt->cpu));
+		} else {
+			/* disable none boot CPU's irq at first */
+			disable_irq(evt->ced.irq);
+		}
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+
+const struct of_device_id spacemit_counter_of_id[] = {
+	{
+		.compatible = "spacemit,timer-match",
+	},
+	{ },
+};
+
+static int __init spacemit_of_counter_init(struct device_node *np, int tid)
+{
+	int irq, ret;
+	unsigned int cid, cpu;
+	struct spacemit_timer_evt *evt;
+
+	if (!np)
+		return -EINVAL;
+
+	ret = of_property_read_u32(np, "spacemit,timer-counter-id", &cid);
+	if (ret || cid >= SPACEMIT_MAX_TIMER) {
+		pr_err("Timer %d: fail to get counter id 0x%x\n", tid, cid);
+		return ret;
+	}
+
+	if (of_property_read_bool(np, "spacemit,timer-broadcast"))
+		cpu = SPACEMIT_TIMER_ALL_CPU;
+	else {
+		ret = of_property_read_u32(np,
+					   "spacemit,timer-counter-cpu",
+					   &cpu);
+		if (ret) {
+			pr_err("Timer %d:%d: fail to get cpu\n",
+			       tid, cid);
+			return ret;
+		}
+	}
+	irq = irq_of_parse_and_map(np, 0);
+	evt = &spacemit_timers[tid]->evt[cid];
+	evt->timer = spacemit_timers[tid];
+	evt->freq = spacemit_timers[tid]->freq;
+	evt->irq = irq;
+	evt->cpu = cpu;
+	evt->cid = cid;
+	evt->tid = tid;
+	ret = spacemit_timer_setup(evt);
+	if (ret) {
+		pr_err("Timer %d:%d: fail to create clkevt\n",
+		       tid, cid);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int __init spacemit_of_timer_init(struct device_node *np)
+{
+	unsigned int flag, tid, fc_freq, apb_freq, freq;
+	void __iomem *base;
+	struct device_node *child_np;
+	const struct of_device_id *match;
+	int ret = 0;
+
+	/* timer initialization */
+	base = of_iomap(np, 0);
+	if (!base) {
+		pr_err("Timer: fail to map register space\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	flag = 0;
+
+	/* get timer id */
+	ret = of_property_read_u32(np, "spacemit,timer-id", &tid);
+	if (ret || tid >= SPACEMIT_MAX_TIMER) {
+		pr_err("Timer %d: fail to get timer-id with err %d\n", tid, ret);
+		goto out;
+	}
+
+	/* timer's fast clock and apb frequency */
+	ret = of_property_read_u32(np, "spacemit,timer-fastclk-frequency", &fc_freq);
+	if (ret) {
+		pr_err("Timer %d: fail to get fastclk-frequency with err %d\n",
+		       tid, ret);
+		goto out;
+	}
+
+	ret = of_property_read_u32(np, "spacemit,timer-apb-frequency", &apb_freq);
+	if (ret) {
+		pr_err("Timer %d: fail to get apb-frequency with err %d\n",
+		       tid, ret);
+		goto out;
+	}
+
+	ret = of_property_read_u32(np, "spacemit,timer-frequency", &freq);
+	if (ret) {
+		pr_err("Timer %d: fail to get timer frequency with err %d\n",
+		       tid, ret);
+		goto out;
+	}
+
+	/*
+	 * Need use loop for more safe register's accessing,
+	 * so at here dynamically calculate the loop time.
+	 */
+	if (!fc_freq || !apb_freq) {
+		pr_err("mmp timer's fast clock or apb freq are incorrect!\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = spacemit_timer_init(np, tid, base, flag, fc_freq, apb_freq, freq);
+	if (ret)
+		goto out;
+
+	/* counter initialization */
+	for_each_child_of_node(np, child_np) {
+		match = of_match_node(spacemit_counter_of_id, child_np);
+		if (!of_device_is_available(child_np))
+			continue;
+		ret = spacemit_of_counter_init(child_np, tid);
+		if (ret)
+			goto out;
+	}
+	return 0;
+out:
+	if (ret)
+		pr_err("Failed to get timer from dtb with error:%d\n", ret);
+	return ret;
+}
+static int spacemit_timer_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	int ret = 0;
+
+	if (!of_device_is_available(np))
+		return -ENODEV;
+
+	ret = spacemit_of_timer_init(np);
+	if (ret)
+		dev_err(&pdev->dev, "spacemit_timer: probe failed for node %s\n", np->name);
+
+	return ret;
+}
+
+static int spacemit_timer_suspend_noirq(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct device_node *np = pdev->dev.of_node;
+	unsigned int tid;
+	struct spacemit_timer *tm;
+
+	if (of_property_read_u32(np, "spacemit,timer-id", &tid))
+		return 0;
+
+	if (tid >= SPACEMIT_MAX_TIMER)
+		return 0;
+
+	tm = spacemit_timers[tid];
+	if (tm) {
+		if (tm->clk)
+			clk_disable_unprepare(tm->clk);
+		if (tm->clk_bus)
+			clk_disable_unprepare(tm->clk_bus);
+	}
+
+	return 0;
+}
+
+static int spacemit_timer_resume_noirq(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct device_node *np = pdev->dev.of_node;
+	unsigned int tid;
+	struct spacemit_timer *tm;
+	int ret = 0;
+
+	if (of_property_read_u32(np, "spacemit,timer-id", &tid))
+		return 0;
+
+	if (tid >= SPACEMIT_MAX_TIMER)
+		return 0;
+
+	tm = spacemit_timers[tid];
+	if (tm) {
+		/* Only restore clocks if they are not already enabled
+		 * (timer_resume may have already restored them) */
+		if (tm->clk_bus && !__clk_is_enabled(tm->clk_bus)) {
+			ret = clk_prepare_enable(tm->clk_bus);
+			if (ret) {
+				dev_err(dev, "Failed to enable bus clock: %d\n", ret);
+				return ret;
+			}
+		}
+		if (tm->clk && !__clk_is_enabled(tm->clk)) {
+			ret = clk_prepare_enable(tm->clk);
+			if (ret) {
+				dev_err(dev, "Failed to enable clock: %d\n", ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops spacemit_timer_pm_ops = {
+	.suspend_noirq = spacemit_timer_suspend_noirq,
+	.resume_noirq = spacemit_timer_resume_noirq,
+};
+
+static const struct of_device_id spacemit_timer_of_match[] = {
+	{ .compatible = "spacemit,soc-timer" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, spacemit_timer_of_match);
+
+static struct platform_driver spacemit_timer_driver = {
+	.probe  = spacemit_timer_probe,
+	.driver = {
+		.name           = "spacemit-timer",
+		.of_match_table = spacemit_timer_of_match,
+		.pm             = &spacemit_timer_pm_ops,
+	},
+};
+
+module_platform_driver(spacemit_timer_driver);
+#endif
diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c
index 4d7cf338824a..a5001db3e28c 100644
--- a/drivers/clocksource/timer-riscv.c
+++ b/drivers/clocksource/timer-riscv.c
@@ -74,6 +74,7 @@ static DEFINE_PER_CPU(struct clock_event_device, riscv_clock_event) = {
 	.rating			= 100,
 	.set_next_event		= riscv_clock_next_event,
 	.set_state_shutdown	= riscv_clock_shutdown,
+	.set_state_oneshot_stopped = riscv_clock_shutdown,
 };
 
 /*
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index 78702a08364f..c9a8522e2a75 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -253,6 +253,15 @@ config CPUFREQ_DT_PLATDEV
 
 	  If in doubt, say N.
 
+config SPACEMIT_K3_CPUFREQ
+	tristate "CPU frequency scaling driver for Spacemit K1X"
+	depends on OF && COMMON_CLK
+	select CPUFREQ_DT
+	select CPUFREQ_DT_PLATDEV
+	help
+	  This adds the CPUFreq driver support for Spacemit K3 SoC
+	  which are capable of changing the CPU's frequency dynamically.
+
 if X86
 source "drivers/cpufreq/Kconfig.x86"
 endif
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 681d687b5a18..587a40a22b47 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -107,3 +107,4 @@ obj-$(CONFIG_LOONGSON3_CPUFREQ)		+= loongson3_cpufreq.o
 obj-$(CONFIG_SH_CPU_FREQ)		+= sh-cpufreq.o
 obj-$(CONFIG_SPARC_US2E_CPUFREQ)	+= sparc-us2e-cpufreq.o
 obj-$(CONFIG_SPARC_US3_CPUFREQ)		+= sparc-us3-cpufreq.o
+obj-$(CONFIG_SPACEMIT_K3_CPUFREQ)	+= spacemit-k3-cpufreq.o
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index dc11b62399ad..89911be7e929 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -242,4 +242,9 @@ static int __init cpufreq_dt_platdev_init(void)
 			       -1, data,
 			       sizeof(struct cpufreq_dt_platform_data)));
 }
+
+#if defined(CONFIG_SOC_SPACEMIT)
+late_initcall(cpufreq_dt_platdev_init);
+#else
 core_initcall(cpufreq_dt_platdev_init);
+#endif
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 7d5079fd1688..6a28e39a46a8 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -24,6 +24,7 @@
 
 #include "cpufreq-dt.h"
 
+#ifndef CONFIG_SOC_SPACEMIT
 struct private_data {
 	struct list_head node;
 
@@ -33,9 +34,11 @@ struct private_data {
 	bool have_static_opps;
 	int opp_token;
 };
+#endif
 
 static LIST_HEAD(priv_list);
 
+#ifndef CONFIG_SOC_SPACEMIT
 static struct private_data *cpufreq_dt_find_data(int cpu)
 {
 	struct private_data *priv;
@@ -47,6 +50,24 @@ static struct private_data *cpufreq_dt_find_data(int cpu)
 
 	return NULL;
 }
+#else
+struct private_data *cpufreq_dt_find_data(int cpu)
+{
+	struct private_data *priv;
+
+	list_for_each_entry(priv, &priv_list, node) {
+		if (cpumask_test_cpu(cpu, priv->cpus))
+			return priv;
+	}
+
+	return NULL;
+}
+
+void cpufreq_dt_add_data(struct private_data *priv)
+{
+	list_add(&priv->node, &priv_list);
+}
+#endif
 
 static int set_target(struct cpufreq_policy *policy, unsigned int index)
 {
diff --git a/drivers/cpufreq/cpufreq-dt.h b/drivers/cpufreq/cpufreq-dt.h
index fc1889aeb4f1..2488469e9330 100644
--- a/drivers/cpufreq/cpufreq-dt.h
+++ b/drivers/cpufreq/cpufreq-dt.h
@@ -24,4 +24,19 @@ struct cpufreq_dt_platform_data {
 
 struct platform_device *cpufreq_dt_pdev_register(struct device *dev);
 
+#ifdef CONFIG_SOC_SPACEMIT
+struct private_data {
+	struct list_head node;
+
+	cpumask_var_t cpus;
+	struct device *cpu_dev;
+	struct cpufreq_frequency_table *freq_table;
+	bool have_static_opps;
+	int opp_token;
+};
+
+struct private_data *cpufreq_dt_find_data(int cpu);
+void cpufreq_dt_add_data(struct private_data *priv);
+#endif
+
 #endif /* __CPUFREQ_DT_H__ */
diff --git a/drivers/cpufreq/spacemit-k3-cpufreq.c b/drivers/cpufreq/spacemit-k3-cpufreq.c
new file mode 100644
index 000000000000..2fe2324c2f32
--- /dev/null
+++ b/drivers/cpufreq/spacemit-k3-cpufreq.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/cpufreq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/cpumask.h>
+#include <linux/clk/clk-conf.h>
+#include <linux/pm_qos.h>
+#include <linux/notifier.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/mutex.h>
+#include <linux/pm_opp.h>
+#include <linux/device.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include "../opp/opp.h"
+#include "cpufreq-dt.h"
+
+#define TURBO0_FREQUENCY		(1000000000)
+#define STABLE_FREQUENCY		(819200000)
+
+static int spacemit_processor_notifier(struct notifier_block *nb,
+                                  unsigned long event, void *data)
+{
+	int cpu;
+	struct device *cpu_dev;
+	struct cpufreq_freqs *freqs = (struct cpufreq_freqs *)data;
+	struct cpufreq_policy *policy = ( struct cpufreq_policy *)freqs->policy;
+	struct opp_table *opp_table;
+	struct device_node *np;
+	struct clk *pll_clst0, *pll_clst1, *pll_src, *clt_pll_src;
+	u64 rates;
+	u32 microvol;
+	int i;
+
+	cpu = cpumask_first(policy->related_cpus);
+	cpu_dev = get_cpu_device(cpu);
+	opp_table = _find_opp_table(cpu_dev);
+
+	for_each_available_child_of_node(opp_table->np, np) {
+		of_property_read_u64_array(np, "opp-hz", &rates, 1);
+		if (rates == freqs->new * 1000) {
+			of_property_read_u32(np, "opp-microvolt", &microvol);
+			break;
+		}
+	}
+
+	/* get the pll clk handler */
+	pll_clst0 = of_clk_get_by_name(opp_table->np, "pll_clst0");
+	pll_clst1 = of_clk_get_by_name(opp_table->np, "pll_clst1");
+	pll_src = of_clk_get_by_name(opp_table->np, "pll_src");
+	clt_pll_src = of_clk_get_by_name(opp_table->np, "clt_pll_src");
+
+	if (event == CPUFREQ_PRECHANGE) {
+
+		if (freqs->new * 1000 > TURBO0_FREQUENCY) {
+			if (freqs->old * 1000 > TURBO0_FREQUENCY) {
+				for (i = 0; i < opp_table->clk_count; ++i)
+					clk_set_rate(opp_table->clks[i], STABLE_FREQUENCY);
+			}
+
+			if (freqs->new * 1000 > TURBO0_FREQUENCY) {
+				/* 2.4G */
+				if (!IS_ERR(pll_clst0))
+					clk_set_rate(pll_clst0, freqs->new * 1000);
+
+				/* 2.4G */
+				if (!IS_ERR(pll_clst1))
+					clk_set_rate(pll_clst1, freqs->new * 1000);
+			}
+		}
+	}
+
+	i = clk_set_parent(clt_pll_src, pll_src);
+
+	if (event == CPUFREQ_POSTCHANGE) {
+		/* TODO */
+	}
+
+	if (!IS_ERR(pll_clst0))
+		clk_put(pll_clst0);
+	if (!IS_ERR(pll_clst1))
+		clk_put(pll_clst1);
+
+	dev_pm_opp_put_opp_table(opp_table);
+
+	return 0;
+}
+static struct notifier_block spacemit_processor_notifier_block = {
+       .notifier_call = spacemit_processor_notifier,
+};
+
+static int spacemit_policy_notifier(struct notifier_block *nb,
+                                  unsigned long event, void *data)
+{
+	int cpu;
+	struct device *cpu_dev;
+	struct cpufreq_policy *policy = data;
+	struct opp_table *opp_table;
+
+	cpu = cpumask_first(policy->related_cpus);
+	cpu_dev = get_cpu_device(cpu);
+	opp_table = _find_opp_table(cpu_dev);
+
+	if (policy->clk)
+		clk_put(policy->clk);
+
+	/* cover the policy->clk & opp_table->clk which has been set before */
+	policy->clk = opp_table->clks[0];
+	opp_table->clk = opp_table->clks[0];
+
+	return 0;
+}
+
+static struct notifier_block spacemit_policy_notifier_block = {
+       .notifier_call = spacemit_policy_notifier,
+};
+
+static int spacemit_dt_cpufreq_pre_early_init(struct device *dev, int cpu)
+{
+	struct private_data *priv;
+	struct device *cpu_dev;
+	const char *reg_name[] = { "clst", NULL };
+	const char *clk_name[] = { "cls0", "cls1", NULL };
+	struct dev_pm_opp_config config = {
+		.regulator_names = reg_name,
+		.clk_names = clk_name,
+		.config_clks = dev_pm_opp_config_clks_simple,
+	};
+	int ret;
+
+	/* Check if this CPU is already covered by some other policy */
+	if (cpufreq_dt_find_data(cpu))
+		return 0;
+
+	cpu_dev = get_cpu_device(cpu);
+	if (!cpu_dev)
+		return -EPROBE_DEFER;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	if (!alloc_cpumask_var(&priv->cpus, GFP_KERNEL))
+		return -ENOMEM;
+
+	cpumask_set_cpu(cpu, priv->cpus);
+	priv->cpu_dev = cpu_dev;
+
+	if (cpu >= 8)
+		config.regulator_names = NULL;
+	/*
+	 * OPP layer will be taking care of regulators now, but it needs to know
+	 * the name of the regulator first.
+	 */
+	priv->opp_token = dev_pm_opp_set_config(cpu_dev, &config);
+	if (priv->opp_token < 0) {
+		ret = -EPROBE_DEFER;
+		goto free_cpumask;
+	}
+
+	/* Get OPP-sharing information from "operating-points-v2" bindings */
+	ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, priv->cpus);
+	if (ret)
+		goto out;
+
+	/*
+	 * Initialize OPP tables for all priv->cpus. They will be shared by
+	 * all CPUs which have marked their CPUs shared with OPP bindings.
+	 *
+	 * For platforms not using operating-points-v2 bindings, we do this
+	 * before updating priv->cpus. Otherwise, we will end up creating
+	 * duplicate OPPs for the CPUs.
+	 *
+	 * OPPs might be populated at runtime, don't fail for error here unless
+	 * it is -EPROBE_DEFER.
+	 */
+	ret = dev_pm_opp_of_cpumask_add_table(priv->cpus);
+	if (!ret) {
+		priv->have_static_opps = true;
+	} else if (ret == -EPROBE_DEFER) {
+		goto out;
+	}
+
+	/*
+	 * The OPP table must be initialized, statically or dynamically, by this
+	 * point.
+	 */
+	ret = dev_pm_opp_get_opp_count(cpu_dev);
+	if (ret <= 0) {
+		dev_err(cpu_dev, "OPP table can't be empty\n");
+		ret = -ENODEV;
+		goto out;
+	}
+
+	ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &priv->freq_table);
+	if (ret) {
+		dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
+		goto out;
+	}
+
+	cpufreq_dt_add_data(priv);
+
+	return 0;
+
+out:
+	if (priv->have_static_opps)
+		dev_pm_opp_of_cpumask_remove_table(priv->cpus);
+	dev_pm_opp_put_regulators(priv->opp_token);
+free_cpumask:
+	free_cpumask_var(priv->cpus);
+	return ret;
+}
+
+static int spacemit_dt_cpufreq_pre_probe(struct platform_device *pdev)
+{
+	int cpu;
+
+	if (strncmp(pdev->name, "cpufreq-dt", 10) != 0)
+		return 0;
+
+	for_each_possible_cpu(cpu)
+		spacemit_dt_cpufreq_pre_early_init(&pdev->dev, cpu);
+
+	return 0;
+}
+
+static int __device_notifier_call(struct notifier_block *nb,
+				      unsigned long event, void *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+
+	switch (event) {
+	case BUS_NOTIFY_REMOVED_DEVICE:
+		break;
+	case BUS_NOTIFY_UNBOUND_DRIVER:
+		break;
+	case BUS_NOTIFY_BIND_DRIVER:
+		/* here */
+		spacemit_dt_cpufreq_pre_probe(pdev);
+		break;
+	case BUS_NOTIFY_ADD_DEVICE:
+		break;
+	default:
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block spacemit_platform_nb = {
+	.notifier_call = __device_notifier_call,
+};
+
+static int __init spacemit_processor_driver_init(void)
+{
+       int ret;
+
+	ret = cpufreq_register_notifier(&spacemit_processor_notifier_block, CPUFREQ_TRANSITION_NOTIFIER);
+	if (ret) {
+		pr_err("register cpufreq notifier failed\n");
+		return -EINVAL;
+	}
+
+       ret = cpufreq_register_notifier(&spacemit_policy_notifier_block, CPUFREQ_POLICY_NOTIFIER);
+       if (ret) {
+               pr_err("register cpufreq notifier failed\n");
+               return -EINVAL;
+       }
+
+	bus_register_notifier(&platform_bus_type, &spacemit_platform_nb);
+
+       return 0;
+}
+arch_initcall(spacemit_processor_driver_init);
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index b8a74b1798ba..909fefaab4f1 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -455,6 +455,21 @@ config MMP_PDMA
 	help
 	  Support the MMP PDMA engine for PXA and MMP platform.
 
+config MMP_PDMA_SPACEMIT
+	bool "Spacemit mmp_pdma support"
+	depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST || SOC_SPACEMIT
+	select DMA_ENGINE
+	help
+	  Support the MMP PDMA engine for Spacemit-k1x platform.
+
+config SPACEMIT_PDMA_SUPPORT_64BIT
+       bool "SpacemiT PDMA 64-bit support"
+       depends on MMP_PDMA_SPACEMIT
+       help
+         Enable 64-bit DMA addressing support for SpacemiT PDMA controller.
+         This allows the PDMA to access memory above 4GB boundary.
+         If unsure, say N.
+
 config MMP_TDMA
 	tristate "MMP Two-Channel DMA support"
 	depends on ARCH_MMP || COMPILE_TEST
@@ -471,7 +486,7 @@ config MOXART_DMA
 	select DMA_VIRTUAL_CHANNELS
 	help
 	  Enable support for the MOXA ART SoC DMA controller.
- 
+
 	  Say Y here if you enabled MMP ADMA, otherwise say N.
 
 config MPC512X_DMA
@@ -749,6 +764,27 @@ config XILINX_ZYNQMP_DPDMA
 	  driver provides the dmaengine required by the DisplayPort subsystem
 	  display driver.
 
+config SPACEMIT_HDMA
+	bool "spacemit hdma driver support"
+	depends on DMA_ENGINE || SOC_SPACEMIT
+	help
+	  Enable spacemit hdma(high-speed dma) driver
+
+config SPACEMIT_ADMA
+	tristate "Spacmit ADMA support"
+	depends on DMA_ENGINE && SOC_SPACEMIT
+	help
+	  Enable support for Spacemit Audio DMA Engine.
+
+config USERSPACE_DMA
+	tristate "Userspace DMA driver support"
+	depends on DMA_ENGINE
+	help
+	  Enable a simple userspace DMA test driver.
+	  This driver exposes a character device (e.g. /dev/udma) that
+	  allows userspace to allocate and map DMA buffers and trigger
+	  memcpy operations using the DMA engine.
+
 # driver files
 source "drivers/dma/amd/Kconfig"
 
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index a54d7688392b..2f1d08400473 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -56,6 +56,7 @@ obj-$(CONFIG_LPC32XX_DMAMUX) += lpc32xx-dmamux.o
 obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o
 obj-$(CONFIG_MILBEAUT_XDMAC) += milbeaut-xdmac.o
 obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o
+obj-$(CONFIG_MMP_PDMA_SPACEMIT) += mmp_pdma_spacemit.o
 obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o
 obj-$(CONFIG_MOXART_DMA) += moxart-dma.o
 obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o
@@ -83,8 +84,11 @@ obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o
 obj-$(CONFIG_UNIPHIER_XDMAC) += uniphier-xdmac.o
 obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
 obj-$(CONFIG_ST_FDMA) += st_fdma.o
+obj-$(CONFIG_USERSPACE_DMA) += udma.o
 obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/
 obj-$(CONFIG_INTEL_LDMA) += lgm/
+obj-$(CONFIG_SPACEMIT_HDMA) += ai_dma_ctrl.o ai_dma_dev.o
+obj-$(CONFIG_SPACEMIT_ADMA) += adma-spacemit.o
 
 obj-y += amd/
 obj-y += mediatek/
diff --git a/drivers/dma/adma-spacemit.c b/drivers/dma/adma-spacemit.c
new file mode 100644
index 000000000000..e3dde04d0931
--- /dev/null
+++ b/drivers/dma/adma-spacemit.c
@@ -0,0 +1,1056 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2025 Spacemit K3 Adma Driver
+ */
+
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/dmaengine.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/dmapool.h>
+#include <linux/genalloc.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+#include <linux/of.h>
+#include <linux/delay.h>
+#include <linux/bitfield.h>
+#include <linux/rpmsg.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/string.h>
+#include "dmaengine.h"
+
+#define BCR	0x00  /* channel byte count register */
+#define SAR	0x10  /* channel source address register */
+#define DAR	0x20  /* channel destination address register */
+#define NDR	0x30  /* channel next descriptor address register */
+#define DCR	0x40  /* channel control register */
+#define CDR	0x70  /* channel current descriptor address register */
+#define IER	0x80  /* channel interrupt mask register */
+#define ISR	0xA0  /* channel interrupt status register */
+
+#define ADMA_SAMPLE_BITS_MASK		GENMASK(24, 22)
+#define ADMA_SAMPLE_BITS(x)		FIELD_PREP(ADMA_SAMPLE_BITS_MASK, x)
+#define ADMA_CH_ABORT			BIT(20)
+#define ADMA_CLOSE_DESC_EN		BIT(17)
+#define ADMA_UNPACK_SAMPLES		BIT(16)
+#define ADMA_CH_ACTIVE			BIT(14)
+#define ADMA_FETCH_NEXT_DESC		BIT(13)
+#define ADMA_CH_EN			BIT(12)
+#define ADMA_INTRRUPT_MODE		BIT(10)
+
+#define ADMA_BURST_LIMIT_MASK		GENMASK(8, 6)
+#define ADMA_BURST_1BYTE		FIELD_PREP(ADMA_BURST_LIMIT_MASK, 0x5)
+#define ADMA_BURST_2BYTE		FIELD_PREP(ADMA_BURST_LIMIT_MASK, 0x6)
+#define ADMA_BURST_4BYTE		FIELD_PREP(ADMA_BURST_LIMIT_MASK, 0x0)
+#define ADMA_BURST_8BYTE		FIELD_PREP(ADMA_BURST_LIMIT_MASK, 0x1)
+#define ADMA_BURST_16BYTE		FIELD_PREP(ADMA_BURST_LIMIT_MASK, 0x3)
+#define ADMA_BURST_32BYTE		FIELD_PREP(ADMA_BURST_LIMIT_MASK, 0x7)
+
+#define ADMA_DEST_ADDR_DIR_MASK		GENMASK(5, 4)
+#define ADMA_DEST_ADDR_INCREMENT	FIELD_PREP(ADMA_DEST_ADDR_DIR_MASK, 0x0)
+#define ADMA_DEST_ADDR_DECREMENT	FIELD_PREP(ADMA_DEST_ADDR_DIR_MASK, 0x1)
+#define ADMA_DEST_ADDR_HOLD		FIELD_PREP(ADMA_DEST_ADDR_DIR_MASK, 0x2)
+
+#define ADMA_SRC_ADDR_DIR_MASK		GENMASK(3, 2)
+#define ADMA_SRC_ADDR_INCREMENT		FIELD_PREP(ADMA_SRC_ADDR_DIR_MASK, 0x0)
+#define ADMA_SRC_ADDR_DECREMENT		FIELD_PREP(ADMA_SRC_ADDR_DIR_MASK, 0x1)
+#define ADMA_SRC_ADDR_HOLD		FIELD_PREP(ADMA_SRC_ADDR_DIR_MASK, 0x2)
+
+#define tx_to_adma_desc(tx)	\
+		container_of(tx, struct adma_desc_sw, async_tx)
+#define to_adma_chan(dchan)	\
+		container_of(dchan, struct adma_ch, chan)
+#define to_adma_dev(dmadev)	\
+		container_of(dmadev, struct adma_dev, device)
+
+#define STARTUP_MSG		"startup"
+#define STARTUP_OK_MSG		"startup-ok"
+
+enum {
+	AUDIO_SAMPLE_WORD_8BITS = 0x0,
+	AUDIO_SAMPLE_WORD_12BITS,
+	AUDIO_SAMPLE_WORD_16BITS,
+	AUDIO_SAMPLE_WORD_20BITS,
+	AUDIO_SAMPLE_WORD_24BITS,
+	AUDIO_SAMPLE_WORD_32BITS,
+};
+
+struct adma_desc_hw {
+	u32 byte_cnt;
+	u32 src_addr;
+	u32 dst_addr;
+	u32 nxt_desc;
+};
+
+struct adma_desc_sw {
+	struct adma_desc_hw *desc;
+	struct list_head node;
+	struct list_head tx_list;
+	struct dma_async_tx_descriptor async_tx;
+	dma_addr_t dma_addr;
+};
+
+struct adma_pchan;
+
+struct adma_ch {
+	struct device	*dev;
+	struct dma_chan	chan;
+	struct dma_async_tx_descriptor	desc;
+	struct adma_pchan	*phy;
+	struct dma_slave_config	slave_config;
+	enum dma_transfer_direction	dir;
+	struct adma_desc_sw *cyclic_first;
+
+	struct tasklet_struct	tasklet;
+	u32	dev_addr;
+
+	spinlock_t desc_lock;
+	struct list_head chain_pending;
+	struct list_head chain_running;
+	enum dma_status	status;
+
+	struct dma_pool *sw_desc_pool;
+	struct gen_pool *hw_desc_pool;
+	bool init;
+};
+
+struct adma_pchan {
+	int idx;
+	int irq;
+	void __iomem *base;
+	struct adma_ch *vchan;
+	bool use_rpmsg;  /* true if this channel uses rpmsg for interrupts */
+	struct rpmsg_device *rpdev;  /* rpmsg device for this channel (if use_rpmsg) */
+	char rpmsg_service[32];  /* rpmsg service name for this channel */
+};
+
+struct adma_dev {
+	int		dma_channels;
+	void __iomem	*base;
+	void __iomem	*desc_base;
+	dma_addr_t	desc_addr;
+	unsigned long	desc_size;
+
+	struct dma_device	device;
+	struct device		*dev;
+	struct adma_pchan	*phy;
+};
+
+struct adma_rpmsg_data {
+	struct rpmsg_device *rpdev;
+	struct list_head node;
+	struct adma_pchan *phy;  /* the channel using this rpmsg service */
+	char service_name[32];
+};
+
+static DEFINE_MUTEX(adma_rpmsg_lock);
+static LIST_HEAD(adma_rpmsg_list);
+
+static void adma_ch_write_reg(struct adma_pchan *phy, u32 reg_offset, u32 value)
+{
+	writel(value, phy->base + reg_offset + (phy->idx * 4));
+}
+
+static u32 adma_ch_read_reg(struct adma_pchan *phy, u32 reg_offset)
+{
+	return readl(phy->base + reg_offset + (phy->idx * 4));
+}
+
+static dma_cookie_t adma_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+	struct adma_ch *achan = to_adma_chan(tx->chan);
+	struct adma_desc_sw *desc = tx_to_adma_desc(tx);
+	struct adma_desc_sw *child;
+	unsigned long flags;
+	dma_cookie_t cookie = -EBUSY;
+
+	spin_lock_irqsave(&achan->desc_lock, flags);
+	list_for_each_entry(child, &desc->tx_list, node) {
+		cookie = dma_cookie_assign(&child->async_tx);
+	}
+
+	list_splice_tail_init(&desc->tx_list, &achan->chain_pending);
+	spin_unlock_irqrestore(&achan->desc_lock, flags);
+
+	return cookie;
+}
+
+static int adma_alloc_chan_resources(struct dma_chan *dchan)
+{
+	struct adma_ch *achan = to_adma_chan(dchan);
+	struct adma_dev *adev = to_adma_dev(achan->chan.device);
+	u32 buf_offset = 0;
+
+	achan->sw_desc_pool = dma_pool_create(dev_name(&dchan->dev->device),
+					      achan->dev,
+					      sizeof(struct adma_desc_sw),
+					      __alignof__(struct adma_desc_sw),
+					      0);
+	if (!achan->sw_desc_pool) {
+		dev_err(achan->dev, "unable to allocate sw desc pool\n");
+		return -ENOMEM;
+	}
+
+	achan->hw_desc_pool = gen_pool_create(4, -1);
+	if (!achan->hw_desc_pool) {
+		pr_err("unable to allocate hw desc pool\n");
+		dma_pool_destroy(achan->sw_desc_pool);
+		return -ENOMEM;
+	}
+
+	buf_offset = achan->phy->idx * adev->desc_size / 2;
+	if (gen_pool_add_virt(achan->hw_desc_pool,
+		(long)adev->desc_base + buf_offset,
+		adev->desc_addr + buf_offset,
+		adev->desc_size / 2,
+		-1) != 0) {
+		pr_err("gen_pool_add mem error!\n");
+		gen_pool_destroy(achan->hw_desc_pool);
+		dma_pool_destroy(achan->sw_desc_pool);
+		return -ENOMEM;
+	}
+
+	achan->status = DMA_COMPLETE;
+	achan->dir = 0;
+	achan->dev_addr = 0;
+
+	return 1;
+}
+
+static void adma_free_desc_list(struct adma_ch *chan,
+					struct list_head *list)
+{
+	struct adma_desc_sw *desc, *_desc;
+
+	list_for_each_entry_safe(desc, _desc, list, node) {
+		list_del(&desc->node);
+		desc->desc->nxt_desc = 0;
+		gen_pool_free(chan->hw_desc_pool, (long)desc->desc, sizeof(struct adma_desc_hw));
+		dma_pool_free(chan->sw_desc_pool, desc, desc->dma_addr);
+	}
+}
+
+static void adma_free_chan_resources(struct dma_chan *dchan)
+{
+	struct adma_ch *achan = to_adma_chan(dchan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&achan->desc_lock, flags);
+	adma_free_desc_list(achan, &achan->chain_pending);
+	adma_free_desc_list(achan, &achan->chain_running);
+	spin_unlock_irqrestore(&achan->desc_lock, flags);
+	gen_pool_destroy(achan->hw_desc_pool);
+	dma_pool_destroy(achan->sw_desc_pool);
+	achan->hw_desc_pool = NULL;
+	achan->sw_desc_pool = NULL;
+	achan->status = DMA_COMPLETE;
+	achan->dir = 0;
+	achan->dev_addr = 0;
+}
+
+static struct adma_desc_sw *alloc_descriptor(struct adma_ch *achan)
+{
+	struct adma_desc_sw *desc;
+	dma_addr_t pdesc;
+
+	desc = dma_pool_zalloc(achan->sw_desc_pool, GFP_ATOMIC, &pdesc);
+	if (!desc) {
+		dev_err(achan->dev, "can't alloc for sw descriptor\n");
+		return NULL;
+	}
+	desc->dma_addr = pdesc;
+
+	desc->desc = (struct adma_desc_hw *)gen_pool_alloc(achan->hw_desc_pool,
+				sizeof(struct adma_desc_hw));
+	if (!desc->desc) {
+		dev_err(achan->dev, "can't alloc for hw descriptor\n");
+		dma_pool_free(achan->sw_desc_pool, desc, desc->dma_addr);
+		return NULL;
+	}
+
+	pdesc = (dma_addr_t)gen_pool_virt_to_phys(achan->hw_desc_pool, (long)desc->desc);
+
+	INIT_LIST_HEAD(&desc->tx_list);
+	dma_async_tx_descriptor_init(&desc->async_tx, &achan->chan);
+	desc->async_tx.tx_submit = adma_tx_submit;
+	desc->async_tx.phys = pdesc;
+
+	return desc;
+}
+
+static struct dma_async_tx_descriptor *
+adma_prep_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr,
+				size_t len, size_t period_len,
+				enum dma_transfer_direction direction,
+				unsigned long flags)
+{
+	struct adma_ch *achan;
+	struct adma_desc_sw *first = NULL, *prev = NULL, *new;
+	dma_addr_t adma_src, adma_dst;
+
+	achan = to_adma_chan(dchan);
+
+	switch (direction) {
+	case DMA_MEM_TO_DEV:
+		adma_src = buf_addr & 0xffffffff;
+		achan->dev_addr = achan->slave_config.dst_addr;
+		adma_dst = achan->dev_addr;
+		break;
+	case DMA_DEV_TO_MEM:
+		adma_dst = buf_addr & 0xffffffff;
+		achan->dev_addr = achan->slave_config.src_addr;
+		adma_src = achan->dev_addr;
+		break;
+	default:
+		dev_err(achan->dev, "Unsupported direction for cyclic DMA\n");
+		return NULL;
+	}
+	achan->dir = direction;
+
+	do {
+		new = alloc_descriptor(achan);
+		if (!new) {
+			dev_err(achan->dev, "no memory for desc\n");
+			goto fail;
+		}
+		new->desc->byte_cnt = period_len;
+		new->desc->src_addr = adma_src;
+		new->desc->dst_addr = adma_dst;
+
+		dev_dbg(achan->dev, "desc:0x%llx, src:0x%llx, dst:0x%llx",
+			   new->async_tx.phys, adma_src, adma_dst);
+
+		if (!first)
+			first = new;
+		else
+			prev->desc->nxt_desc = new->async_tx.phys;
+		new->async_tx.cookie = 0;
+		prev = new;
+		len -= period_len;
+
+		if (achan->dir == DMA_MEM_TO_DEV)
+			adma_src += period_len;
+		else
+			adma_dst += period_len;
+
+		list_add_tail(&new->node, &first->tx_list);
+	} while (len);
+
+	first->async_tx.flags = flags;
+	first->async_tx.cookie = -EBUSY;
+	new->desc->nxt_desc = first->async_tx.phys;
+	achan->cyclic_first = first;
+
+	return &first->async_tx;
+
+fail:
+	if (first)
+		adma_free_desc_list(achan, &first->tx_list);
+
+	return NULL;
+}
+
+static int adma_config(struct dma_chan *dchan,
+						struct dma_slave_config *cfg)
+{
+	struct adma_ch *achan = to_adma_chan(dchan);
+
+	memcpy(&achan->slave_config, cfg, sizeof(*cfg));
+	return 0;
+}
+
+static void set_desc(struct adma_pchan *phy, dma_addr_t addr)
+{
+	adma_ch_write_reg(phy, NDR, addr);
+}
+
+static void set_ctrl_reg(struct adma_pchan *phy)
+{
+	u32 ctrl_reg_val = 0;
+	u32 maxburst = 0, sample_bits = 0;
+	enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
+	struct adma_ch *achan = phy->vchan;
+
+	if (achan->dir == DMA_MEM_TO_DEV) {
+		maxburst = achan->slave_config.dst_maxburst;
+		width = achan->slave_config.dst_addr_width;
+		ctrl_reg_val |= ADMA_DEST_ADDR_HOLD | ADMA_SRC_ADDR_INCREMENT;
+	} else if (achan->dir == DMA_DEV_TO_MEM) {
+		maxburst = achan->slave_config.src_maxburst;
+		width = achan->slave_config.src_addr_width;
+		ctrl_reg_val |= ADMA_SRC_ADDR_HOLD | ADMA_DEST_ADDR_INCREMENT;
+	} else
+		ctrl_reg_val |= ADMA_SRC_ADDR_HOLD | ADMA_DEST_ADDR_HOLD;
+
+	if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
+		sample_bits = AUDIO_SAMPLE_WORD_8BITS;
+	else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
+		sample_bits = AUDIO_SAMPLE_WORD_16BITS;
+	else if (width == DMA_SLAVE_BUSWIDTH_3_BYTES)
+		sample_bits = AUDIO_SAMPLE_WORD_24BITS;
+	else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES)
+		sample_bits = AUDIO_SAMPLE_WORD_32BITS;
+
+	ctrl_reg_val |= ADMA_SAMPLE_BITS(sample_bits);
+
+	switch (maxburst) {
+	case 8:
+		ctrl_reg_val |= ADMA_BURST_1BYTE;
+		break;
+	case 16:
+		ctrl_reg_val |= ADMA_BURST_2BYTE;
+		break;
+	case 32:
+		ctrl_reg_val |= ADMA_BURST_4BYTE;
+		break;
+	case 64:
+		ctrl_reg_val |= ADMA_BURST_8BYTE;
+		break;
+	case 128:
+		ctrl_reg_val |= ADMA_BURST_16BYTE;
+		break;
+	case 256:
+		ctrl_reg_val |= ADMA_BURST_32BYTE;
+		break;
+	default:
+		ctrl_reg_val |= ADMA_BURST_4BYTE;
+		break;
+	}
+
+	ctrl_reg_val |= ADMA_UNPACK_SAMPLES;
+	ctrl_reg_val |= ADMA_CH_ABORT;
+
+	adma_ch_write_reg(phy, DCR, ctrl_reg_val);
+}
+
+static void enable_chan(struct adma_pchan *phy)
+{
+	u32 ctrl_val;
+	struct adma_ch *achan = phy->vchan;
+
+	if (adma_ch_read_reg(phy, DCR) & BIT(14)) {
+		dev_err(achan->dev, "chan%d is not idle!!!!", phy->idx);
+		return;
+	}
+
+	if (!phy->use_rpmsg)
+		enable_irq(phy->irq);
+
+	if (achan->dir == DMA_MEM_TO_DEV)
+		adma_ch_write_reg(phy, DAR, achan->dev_addr);
+	else if (achan->dir == DMA_DEV_TO_MEM)
+		adma_ch_write_reg(phy, SAR, achan->dev_addr);
+
+	adma_ch_write_reg(phy, IER, 1);
+	ctrl_val = adma_ch_read_reg(phy, DCR);
+	ctrl_val |= ADMA_FETCH_NEXT_DESC;
+	ctrl_val |= ADMA_CH_EN;
+	adma_ch_write_reg(phy, DCR, ctrl_val);
+}
+
+static void adma_issue_pending(struct dma_chan *dchan)
+{
+	struct adma_ch *achan = to_adma_chan(dchan);
+	struct adma_pchan *phy;
+	struct adma_desc_sw *desc;
+	unsigned long flags;
+
+	spin_lock_irqsave(&achan->desc_lock, flags);
+	if (achan->status == DMA_IN_PROGRESS) {
+		dev_err(achan->dev, "DMA controller still busy\n");
+		spin_unlock_irqrestore(&achan->desc_lock, flags);
+		return;
+	}
+	phy = achan->phy;
+	desc = list_first_entry(&achan->chain_pending,
+							struct adma_desc_sw, node);
+	list_splice_tail_init(&achan->chain_pending, &achan->chain_running);
+	set_desc(phy, desc->async_tx.phys);
+	set_ctrl_reg(phy);
+	enable_chan(phy);
+	achan->status = DMA_IN_PROGRESS;
+	spin_unlock_irqrestore(&achan->desc_lock, flags);
+}
+
+static unsigned int adma_residue(struct adma_ch *chan,
+				dma_cookie_t cookie)
+{
+	struct adma_desc_sw *sw;
+	u32 curr, residue = 0;
+	bool passed = false;
+	bool cyclic = chan->cyclic_first != NULL;
+
+	/*
+	 * If the channel does not have a phy pointer anymore, it has already
+	 * been completed. Therefore, its residue is 0.
+	 */
+	if (!chan->phy)
+		return 0;
+
+	if (chan->dir == DMA_DEV_TO_MEM)
+		curr = adma_ch_read_reg(chan->phy, DAR);
+	else
+		curr = adma_ch_read_reg(chan->phy, SAR);
+
+	list_for_each_entry(sw, &chan->chain_running, node) {
+		u32 start, end, len;
+
+		if (chan->dir == DMA_DEV_TO_MEM)
+			start = sw->desc->dst_addr;
+		else
+			start = sw->desc->src_addr;
+
+		len = sw->desc->byte_cnt;
+		end = start + len;
+
+		/*
+		 * 'passed' will be latched once we found the descriptor which
+		 * lies inside the boundaries of the curr pointer. All
+		 * descriptors that occur in the list _after_ we found that
+		 * partially handled descriptor are still to be processed and
+		 * are hence added to the residual bytes counter.
+		 */
+
+		if (passed) {
+			residue += len;
+		} else if (curr >= start && curr <= end) {
+			residue += end - curr;
+			passed = true;
+		}
+
+		if (cyclic)
+			continue;
+
+		if (sw->async_tx.cookie == cookie) {
+			return residue;
+		} else {
+			residue = 0;
+			passed = false;
+		}
+	}
+
+	/* We should only get here in case of cyclic transactions */
+	return residue;
+}
+
+static enum dma_status adma_tx_status(struct dma_chan *dchan,
+							dma_cookie_t cookie,
+							struct dma_tx_state *txstate)
+{
+	struct adma_ch *chan = to_adma_chan(dchan);
+	enum dma_status ret;
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	ret = dma_cookie_status(dchan, cookie, txstate);
+	if (likely(ret != DMA_ERROR))
+		dma_set_residue(txstate, adma_residue(chan, cookie));
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+	if (ret == DMA_COMPLETE)
+		return ret;
+	else
+		return chan->status;
+}
+
+#if 0
+static void disable_chan(struct adma_pchan *phy)
+{
+	u32 reg_val = adma_ch_read_reg(phy, DCR);
+
+	reg_val |= ADMA_CH_ABORT;
+	adma_ch_write_reg(phy, DCR, reg_val);
+
+	udelay(500);
+	reg_val = adma_ch_read_reg(phy, DCR);
+	reg_val &= ~ADMA_CH_EN;
+	adma_ch_write_reg(phy, DCR, reg_val);
+	adma_ch_write_reg(phy, IER, 0);
+}
+#endif
+
+static int adma_terminate_all(struct dma_chan *dchan)
+{
+	struct adma_ch *achan = to_adma_chan(dchan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&achan->desc_lock, flags);
+
+	if (!achan->phy->use_rpmsg) {
+		adma_ch_write_reg(achan->phy, ISR, 0);
+		disable_irq_nosync(achan->phy->irq);
+	}
+	adma_ch_write_reg(achan->phy, IER, 0);
+	adma_free_desc_list(achan, &achan->chain_pending);
+	adma_free_desc_list(achan, &achan->chain_running);
+	achan->status = DMA_COMPLETE;
+	spin_unlock_irqrestore(&achan->desc_lock, flags);
+
+	return 0;
+}
+
+static void dma_do_tasklet(struct tasklet_struct *t)
+{
+	struct adma_ch *chan = from_tasklet(chan, t, tasklet);
+	struct adma_desc_sw *desc;
+	LIST_HEAD(chain_cleanup);
+	unsigned long flags;
+	struct dmaengine_desc_callback cb;
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	if (chan->status == DMA_COMPLETE) {
+		spin_unlock_irqrestore(&chan->desc_lock, flags);
+		return;
+	}
+	desc = chan->cyclic_first;
+	dmaengine_desc_get_callback(&desc->async_tx, &cb);
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+	dmaengine_desc_callback_invoke(&cb, NULL);
+}
+
+static struct dma_chan *adma_dma_xlate(struct of_phandle_args *dma_spec,
+				struct of_dma *ofdma)
+{
+	struct adma_dev *adev = ofdma->of_dma_data;
+	struct dma_chan *chan;
+	u32 idx = dma_spec->args[0];
+
+	chan = &adev->phy[idx].vchan->chan;
+	if (!chan)
+		return NULL;
+
+	chan = dma_get_slave_channel(chan);
+	if (!chan)
+		return NULL;
+
+	return chan;
+}
+
+static int clear_chan_irq(struct adma_pchan *phy)
+{
+	u32 status = adma_ch_read_reg(phy, ISR);
+
+	if (!(status & BIT(0)))
+		return -EAGAIN;
+
+	/* clear irq */
+	adma_ch_write_reg(phy, ISR, 0);
+
+	return 0;
+}
+
+static irqreturn_t adma_chan_handler(int irq, void *dev_id)
+{
+	struct adma_pchan *phy = dev_id;
+
+	if (clear_chan_irq(phy) != 0)
+		return IRQ_NONE;
+
+	tasklet_schedule(&phy->vchan->tasklet);
+	return IRQ_HANDLED;
+}
+
+/* rpmsg callback for handling remote interrupts */
+static int rpmsg_adma_client_cb(struct rpmsg_device *rpdev, void *data,
+				int len, void *priv, u32 src)
+{
+	struct adma_rpmsg_data *rpmsg_data;
+	struct adma_pchan *phy;
+
+	if (strcmp(data, STARTUP_OK_MSG) == 0) {
+		dev_info(&rpdev->dev, "channel: 0x%x -> 0x%x startup ok!\n",
+			rpdev->src, rpdev->dst);
+		return 0;
+	}
+
+	rpmsg_data = dev_get_drvdata(&rpdev->dev);
+	if (!rpmsg_data || !rpmsg_data->phy)
+		return 0;
+
+	phy = rpmsg_data->phy;
+	if (phy->vchan && phy->vchan->init)
+		tasklet_schedule(&phy->vchan->tasklet);
+
+	return 0;
+}
+
+const char *irq_names[] = { "tx", "rx" };
+static int adma_chan_init(struct adma_dev *adev, int idx, int irq)
+{
+	struct adma_pchan *phy  = &adev->phy[idx];
+	struct adma_ch *chan;
+	char *irq_name;
+	int ret;
+
+	chan = devm_kzalloc(adev->dev, sizeof(*chan), GFP_KERNEL);
+	if (!chan)
+		return -ENOMEM;
+
+	phy->idx = idx;
+	phy->base = adev->base;
+	phy->vchan = chan;
+	chan->phy = phy;
+
+	/* Only register hardware IRQ if not using rpmsg */
+	if (irq && !phy->use_rpmsg) {
+		irq_name = devm_kasprintf(adev->dev, GFP_KERNEL, "%s-%s",
+					  dev_name(adev->dev), irq_names[idx]);
+		if (!irq_name)
+			return -ENOMEM;
+
+		ret = devm_request_irq(adev->dev, irq, adma_chan_handler,
+				       0, irq_name, phy);
+		if (ret) {
+			dev_err(adev->dev, "channel request irq fail!\n");
+			devm_kfree(adev->dev, chan);
+			return ret;
+		}
+		phy->irq = irq;
+		disable_irq(phy->irq);
+	}
+
+	spin_lock_init(&chan->desc_lock);
+	chan->dev = adev->dev;
+	chan->chan.device = &adev->device;
+	tasklet_setup(&chan->tasklet, dma_do_tasklet);
+	INIT_LIST_HEAD(&chan->chain_pending);
+	INIT_LIST_HEAD(&chan->chain_running);
+
+	/* register virt channel to dma engine */
+	list_add_tail(&chan->chan.device_node, &adev->device.channels);
+	chan->init = true;
+
+	return 0;
+}
+
+static const struct of_device_id adma_id_table[] = {
+	{ .compatible = "spacemit,k3-adma"},
+	{},
+};
+
+static int adma_probe(struct platform_device *pdev)
+{
+	struct adma_dev *adev;
+	struct device *dev;
+	const struct of_device_id *of_id;
+	int dma_channels = 0;
+	int i, ret, irq = 0;
+	int channel_count;
+	struct resource *res;
+	const enum dma_slave_buswidth widths =
+		DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
+		DMA_SLAVE_BUSWIDTH_3_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES;
+
+	of_id = of_match_device(adma_id_table, &pdev->dev);
+	if (!of_id) {
+		dev_err(&pdev->dev, "Unable to match OF ID\n");
+		return -ENODEV;
+	}
+
+	dev = &pdev->dev;
+	adev = devm_kzalloc(dev, sizeof(*adev), GFP_KERNEL);
+	if (!adev) {
+		dev_err(&pdev->dev, "can't alloc memory for adma_dev\n");
+		return -ENOMEM;
+	}
+	adev->dev = dev;
+	adev->base = devm_platform_ioremap_resource_byname(pdev, "adma_reg");
+	if (IS_ERR(adev->base))
+		return PTR_ERR(adev->base);
+
+	ret = of_property_match_string(dev->of_node, "reg-names", "buf_addr");
+	if (ret < 0) {
+		dev_err(&pdev->dev, "can't find buf_addr\n");
+		return -EINVAL;
+	}
+	adev->desc_base = devm_platform_get_and_ioremap_resource(pdev, ret, &res);
+	if (IS_ERR(adev->desc_base))
+		return dev_err_probe(&pdev->dev, PTR_ERR(adev->desc_base), "failed to map desc buf addr\n");
+	adev->desc_addr = res->start;
+	adev->desc_size = res->end - res->start + 1;
+
+	if (dev->of_node) {
+		/* Parse dma-channels properties */
+		if (of_property_read_u32(dev->of_node, "dma-channels",
+					&dma_channels))
+			of_property_read_u32(dev->of_node, "#dma-channels",
+					&dma_channels);
+	} else {
+		dma_channels = 2;
+	}
+	adev->dma_channels = dma_channels;
+
+	adev->phy = devm_kcalloc(dev, dma_channels, sizeof(*adev->phy),
+				 GFP_KERNEL);
+	if (adev->phy == NULL)
+		return -ENOMEM;
+
+	/* Configure rpmsg for channels before initializing them */
+	channel_count = of_property_count_u32_elems(dev->of_node, "rpmsg-channels");
+	if (channel_count > 0) {
+		const char **service_names;
+		u32 *channels;
+		int service_count, j;
+
+		/* Validate rpmsg-service property */
+		service_count = of_property_count_strings(dev->of_node, "rpmsg-services");
+		if (service_count != channel_count) {
+			dev_err(dev, "rpmsg-services count (%d) must match rpmsg-channels count (%d)\n",
+				service_count, channel_count);
+			return -EINVAL;
+		}
+
+		/* Allocate arrays for service names and channel indices */
+		service_names = devm_kcalloc(dev, channel_count, sizeof(char *), GFP_KERNEL);
+		channels = devm_kcalloc(dev, channel_count, sizeof(u32), GFP_KERNEL);
+		if (!service_names || !channels)
+			return -ENOMEM;
+
+		/* Read service names and channel indices */
+		for (j = 0; j < channel_count; j++) {
+			ret = of_property_read_string_index(dev->of_node, "rpmsg-services",
+							     j, &service_names[j]);
+			if (ret) {
+				dev_err(dev, "failed to read rpmsg-service[%d]\n", j);
+				return ret;
+			}
+		}
+
+		ret = of_property_read_u32_array(dev->of_node, "rpmsg-channels",
+						  channels, channel_count);
+		if (ret) {
+			dev_err(dev, "failed to read rpmsg-channels\n");
+			return ret;
+		}
+
+		/* Configure each channel with its rpmsg service */
+		for (j = 0; j < channel_count; j++) {
+			struct adma_rpmsg_data *rpmsg_data;
+			struct adma_pchan *phy;
+			bool found = false;
+			int chan_idx = channels[j];
+
+			if (chan_idx >= dma_channels) {
+				dev_err(dev, "invalid channel index %d (max %d)\n",
+					chan_idx, dma_channels - 1);
+				return -EINVAL;
+			}
+
+			phy = &adev->phy[chan_idx];
+
+			/* Find matching rpmsg device */
+			mutex_lock(&adma_rpmsg_lock);
+			list_for_each_entry(rpmsg_data, &adma_rpmsg_list, node) {
+				if (strcmp(rpmsg_data->service_name, service_names[j]) == 0) {
+					phy->use_rpmsg = true;
+					phy->rpdev = rpmsg_data->rpdev;
+					strscpy(phy->rpmsg_service, service_names[j],
+						sizeof(phy->rpmsg_service));
+
+					rpmsg_data->phy = phy;
+					found = true;
+					dev_info(dev, "channel %d using rpmsg service: %s\n",
+						 chan_idx, service_names[j]);
+					break;
+				}
+			}
+			mutex_unlock(&adma_rpmsg_lock);
+
+			if (!found) {
+				dev_err(dev, "channel %d: rpmsg service '%s' not found, deferring probe\n",
+					chan_idx, service_names[j]);
+				return -EPROBE_DEFER;
+			}
+
+			/* Send startup message for this channel */
+			if (phy->rpdev && phy->rpdev->ept) {
+				ret = rpmsg_send(phy->rpdev->ept, STARTUP_MSG, strlen(STARTUP_MSG));
+				if (ret)
+					dev_warn(dev, "channel %d: rpmsg_send startup failed: %d\n",
+						 chan_idx, ret);
+			}
+		}
+	}
+
+	/* Initialize adma channels */
+	INIT_LIST_HEAD(&adev->device.channels);
+
+	for (i = 0; i < dma_channels; i++) {
+		irq = platform_get_irq(pdev, i);
+		ret = adma_chan_init(adev, i, irq);
+		if (ret)
+			return ret;
+	}
+
+	dma_cap_set(DMA_SLAVE, adev->device.cap_mask);
+	dma_cap_set(DMA_CYCLIC, adev->device.cap_mask);
+	adev->device.dev = dev;
+	adev->device.device_tx_status = adma_tx_status;
+	adev->device.device_alloc_chan_resources = adma_alloc_chan_resources;
+	adev->device.device_free_chan_resources = adma_free_chan_resources;
+	adev->device.device_prep_dma_cyclic = adma_prep_cyclic;
+	adev->device.device_issue_pending = adma_issue_pending;
+	adev->device.device_config = adma_config;
+	adev->device.device_terminate_all = adma_terminate_all;
+	adev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
+	adev->device.src_addr_widths = widths;
+	adev->device.dst_addr_widths = widths;
+	adev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
+	dma_set_mask(adev->dev, adev->dev->coherent_dma_mask);
+
+	ret = dma_async_device_register(&adev->device);
+	if (ret) {
+		dev_err(adev->device.dev, "unable to register\n");
+		return ret;
+	}
+
+	if (pdev->dev.of_node) {
+		ret = of_dma_controller_register(pdev->dev.of_node,
+						adma_dma_xlate, adev);
+		if (ret < 0) {
+			dev_err(dev, "of_dma_controller_register failed\n");
+			dma_async_device_unregister(&adev->device);
+			return ret;
+		}
+	}
+
+	platform_set_drvdata(pdev, adev);
+	return 0;
+}
+
+static void adma_remove(struct platform_device *pdev)
+{
+	struct adma_dev *adev = platform_get_drvdata(pdev);
+	int i;
+
+	if (pdev->dev.of_node)
+		of_dma_controller_free(pdev->dev.of_node);
+
+	dma_async_device_unregister(&adev->device);
+
+	/* Clear rpmsg channel associations */
+	mutex_lock(&adma_rpmsg_lock);
+	for (i = 0; i < adev->dma_channels; i++) {
+		struct adma_pchan *phy = &adev->phy[i];
+
+		if (phy->rpdev) {
+			struct adma_rpmsg_data *rpmsg_data;
+
+			list_for_each_entry(rpmsg_data, &adma_rpmsg_list, node) {
+				if (rpmsg_data->phy == phy) {
+					rpmsg_data->phy = NULL;
+					break;
+				}
+			}
+		}
+	}
+	mutex_unlock(&adma_rpmsg_lock);
+
+	platform_set_drvdata(pdev, NULL);
+}
+
+static struct platform_driver adma_driver = {
+	.driver	= {
+		.name	= "k3-adma",
+		.of_match_table	= adma_id_table,
+	},
+	.probe	= adma_probe,
+	.remove	= adma_remove,
+};
+
+/* rpmsg driver - supports multiple service names */
+static const struct rpmsg_device_id rpmsg_driver_adma_id_table[] = {
+	{ .name = "adma-service" },
+	{ },
+};
+MODULE_DEVICE_TABLE(rpmsg, rpmsg_driver_adma_id_table);
+
+static int rpmsg_adma_client_probe(struct rpmsg_device *rpdev)
+{
+	struct adma_rpmsg_data *rpmsg_data;
+	const char *service_name;
+
+	dev_info(&rpdev->dev, "new channel: 0x%x -> 0x%x!\n",
+		rpdev->src, rpdev->dst);
+
+	rpmsg_data = devm_kzalloc(&rpdev->dev, sizeof(*rpmsg_data), GFP_KERNEL);
+	if (!rpmsg_data)
+		return -ENOMEM;
+
+	rpmsg_data->rpdev = rpdev;
+	rpmsg_data->phy = NULL;
+
+	/* Get service name from rpmsg device id */
+	service_name = rpdev->id.name;
+	strscpy(rpmsg_data->service_name, service_name, sizeof(rpmsg_data->service_name));
+
+	dev_set_drvdata(&rpdev->dev, rpmsg_data);
+
+	/* Add to global list for platform driver to find */
+	mutex_lock(&adma_rpmsg_lock);
+	list_add_tail(&rpmsg_data->node, &adma_rpmsg_list);
+	mutex_unlock(&adma_rpmsg_lock);
+
+	dev_info(&rpdev->dev, "rpmsg service '%s' registered\n", service_name);
+
+	return 0;
+}
+
+static void rpmsg_adma_client_remove(struct rpmsg_device *rpdev)
+{
+	struct adma_rpmsg_data *rpmsg_data = dev_get_drvdata(&rpdev->dev);
+
+	dev_info(&rpdev->dev, "rpmsg adma client driver is removed\n");
+
+	/* Remove from global list */
+	mutex_lock(&adma_rpmsg_lock);
+	list_del(&rpmsg_data->node);
+	mutex_unlock(&adma_rpmsg_lock);
+}
+
+static struct rpmsg_driver rpmsg_adma_client = {
+	.drv.name	= KBUILD_MODNAME,
+	.id_table	= rpmsg_driver_adma_id_table,
+	.probe		= rpmsg_adma_client_probe,
+	.callback	= rpmsg_adma_client_cb,
+	.remove		= rpmsg_adma_client_remove,
+};
+
+static int __init adma_init(void)
+{
+	int ret;
+
+	ret = register_rpmsg_driver(&rpmsg_adma_client);
+	if (ret) {
+		pr_err("Failed to register rpmsg driver: %d\n", ret);
+		return ret;
+	}
+
+	ret = platform_driver_register(&adma_driver);
+	if (ret) {
+		unregister_rpmsg_driver(&rpmsg_adma_client);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit adma_exit(void)
+{
+	platform_driver_unregister(&adma_driver);
+	unregister_rpmsg_driver(&rpmsg_adma_client);
+}
+
+subsys_initcall(adma_init);
+module_exit(adma_exit);
+
+MODULE_DESCRIPTION("ADMA Driver for SpacemiT K3 SoC");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/ai_dma.h b/drivers/dma/ai_dma.h
new file mode 100644
index 000000000000..1a208d51d9a5
--- /dev/null
+++ b/drivers/dma/ai_dma.h
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Spacemit hdmac driver support
+ *
+ * Copyright (c) 2025 SPACEMIT, Co. Ltd.
+ */
+
+struct ai_dmac {
+	struct device *dev;
+	void __iomem *base;
+	int irq;
+	int id;
+
+	struct clk *clk;
+	struct reset_control *resets;
+
+	unsigned int src_width;
+	unsigned int dest_width;
+	unsigned int max_length;
+	unsigned int address_align_mask;
+	unsigned int length_align_mask;
+};
+
+enum aipack_element {
+	ELEMENT_SIZE_BIT4,
+	ELEMENT_SIZE_BIT8,
+	ELEMENT_SIZE_BIT16,
+};
+
+enum dma_req_status {
+	DMA_REQ_FREE = 0,
+	DMA_REQ_SUBMIT = 1,
+	DMA_REQ_PROCESS = 2,
+	DMA_REQ_DONE = 3,
+};
+
+struct ai_pack_param {
+	bool sgdg;
+	bool pack;
+	bool transpose;
+	bool pad;
+	enum aipack_element ele_size;
+	u32 pad_value;
+	u32 m_size;
+	u32 k_size;
+	u32 mr_size;
+	u32 kr_size;
+};
+
+struct req_addr_info {
+	void *req_list;
+	dma_addr_t req_addr;
+};
+
+struct req_node {
+	struct req_addr_info *info;
+	struct list_head list;
+};
+
+struct aipack_param {
+	size_t k_size;
+	size_t m_size;
+	size_t kr_size;
+	size_t mr_size;
+	bool pack;
+	bool transpose;
+	unsigned int pad_value;
+	enum aipack_element ele_size;
+};
+
+struct dma_transfer_param {
+	void *src;
+	void *dst;
+	pid_t pid;
+	bool is_sgdg;
+	size_t size;
+	struct aipack_param ai_param;
+};
+
+struct aidma_req {
+	struct dma_transfer_param params;
+	enum dma_req_status status;
+} __attribute__((aligned(64)));
+
+struct axi_dmac_info {
+	struct ai_dmac *dma;
+	int work_id;
+	bool work;
+	void *req;
+};
+
+#define AXI_DMAC_NUM	8
+#define AIDMA_MAX_REQ	32
+extern struct axi_dmac_info aidma_info[AXI_DMAC_NUM];
+void start_transfer(void);
+
+int ai_dmac_pack_start(struct ai_dmac *c, struct ai_pack_param *param,
+		       dma_addr_t dma_dst, dma_addr_t dma_src);
+
+int ai_dmac_memcpy(struct ai_dmac *c, dma_addr_t dma_dst,
+		   dma_addr_t dma_src, size_t len);
+
+int ai_dmac_memcpy_by_2d(struct ai_dmac *c, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len);
diff --git a/drivers/dma/ai_dma_ctrl.c b/drivers/dma/ai_dma_ctrl.c
new file mode 100644
index 000000000000..2fb1e5d49ba9
--- /dev/null
+++ b/drivers/dma/ai_dma_ctrl.c
@@ -0,0 +1,457 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Spacemit hdma controller driver support
+ *
+ * Copyright (c) 2025 SPACEMIT, Co. Ltd.
+ */
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_dma.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/adi-axi-common.h>
+
+#include "ai_dma.h"
+
+#define AXI_DMAC_REG_INTERFACE_DESC	0x10
+#define AXI_DMAC_DMA_SRC_TYPE_MSK	GENMASK(13, 12)
+#define AXI_DMAC_DMA_SRC_TYPE_GET(x)	FIELD_GET(AXI_DMAC_DMA_SRC_TYPE_MSK, x)
+#define AXI_DMAC_DMA_SRC_WIDTH_MSK	GENMASK(11, 8)
+#define AXI_DMAC_DMA_SRC_WIDTH_GET(x)	FIELD_GET(AXI_DMAC_DMA_SRC_WIDTH_MSK, x)
+#define AXI_DMAC_DMA_DST_TYPE_MSK	GENMASK(5, 4)
+#define AXI_DMAC_DMA_DST_TYPE_GET(x)	FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x)
+#define AXI_DMAC_DMA_DST_WIDTH_MSK	GENMASK(3, 0)
+#define AXI_DMAC_DMA_DST_WIDTH_GET(x)	FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x)
+#define AXI_DMAC_REG_COHERENCY_DESC	0x14
+#define AXI_DMAC_DST_COHERENT_MSK	BIT(0)
+#define AXI_DMAC_DST_COHERENT_GET(x)	FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x)
+
+#define AXI_DMAC_REG_IRQ_MASK		0x80
+#define AXI_DMAC_REG_IRQ_PENDING	0x84
+#define AXI_DMAC_REG_IRQ_SOURCE		0x88
+
+#define AXI_DMAC_REG_CTRL		0x400
+#define AXI_DMAC_REG_TRANSFER_ID	0x404
+#define AXI_DMAC_REG_START_TRANSFER	0x408
+#define AXI_DMAC_REG_FLAGS		0x40c
+#define AXI_DMAC_REG_DEST_ADDRESS	0x410
+#define AXI_DMAC_REG_SRC_ADDRESS	0x414
+#define AXI_DMAC_REG_X_LENGTH		0x418
+#define AXI_DMAC_REG_Y_LENGTH		0x41c
+#define AXI_DMAC_REG_DEST_STRIDE	0x420
+#define AXI_DMAC_REG_SRC_STRIDE		0x424
+#define AXI_DMAC_REG_TRANSFER_DONE	0x428
+#define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c
+#define AXI_DMAC_REG_STATUS		0x430
+#define AXI_DMAC_REG_CURRENT_SRC_ADDR	0x434
+#define AXI_DMAC_REG_CURRENT_DEST_ADDR	0x438
+#define AXI_DMAC_REG_PARTIAL_XFER_LEN	0x44c
+#define AXI_DMAC_REG_PARTIAL_XFER_ID	0x450
+#define AXI_DMAC_REG_CURRENT_SG_ID	0x454
+#define AXI_DMAC_REG_SG_ADDRESS		0x47c
+#define AXI_DMAC_REG_DEST_ADDRESS_HIGH	0x490
+#define AXI_DMAC_REG_SRC_ADDRESS_HIGH	0x494
+#define AXI_DMAC_REG_CUR_DEST_ADDR_HIGH	0x498
+#define AXI_DMAC_REG_CUR_SRC_ADDR_HIGH	0x498
+#define AXI_DMAC_REG_SG_ADDRESS_HIGH	0x4bc
+#define AXI_DMAC_REG_SGDG_CFG		0x600
+#define AXI_DMAC_REG_PAD_VALUE		0x604
+#define AXI_DMAC_REG_SGDG_M_SIZE	0x608
+#define AXI_DMAC_REG_SGDG_K_SIZE	0x60c
+#define AXI_DMAC_REG_SGDG_MR_SIZE	0x610
+#define AXI_DMAC_REG_SGDG_KR_SIZE	0x614
+#define AXI_DMAC_REG_SGDG_MP_SIZE	0x618
+#define AXI_DMAC_REG_SGDG_KP_SIZE	0x61c
+#define AXI_DMAC_REG_SGDG_MB_SIZE	0x624
+
+#define AXI_DMAC_REG_SGDG_MODE		BIT(0)
+#define AXI_DMAC_REG_PACK_MODE		BIT(1)
+#define AXI_DMAC_REG_TRANSPOSE		BIT(2)
+#define AXI_DMAC_REG_PAD_MODE		BIT(3) | BIT(8)
+#define AXI_DMAC_REG_ELE_SIZE_OFFSET	4
+
+#define AXI_DMAC_CTRL_ENABLE		BIT(0)
+#define AXI_DMAC_CTRL_PAUSE		BIT(1)
+#define AXI_DMAC_CTRL_ENABLE_SG		BIT(2)
+
+#define AXI_DMAC_IRQ_SOT		BIT(0)
+#define AXI_DMAC_IRQ_EOT		BIT(1)
+
+#define AXI_DMAC_FLAG_CYCLIC		BIT(0)
+#define AXI_DMAC_FLAG_LAST		BIT(1)
+#define AXI_DMAC_FLAG_PARTIAL_REPORT	BIT(2)
+
+#define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31)
+
+/* The maximum ID allocated by the hardware is 31 */
+#define AXI_DMAC_SG_UNUSED 32U
+
+/* Flags for axi_dmac_hw_desc.flags */
+#define AXI_DMAC_HW_FLAG_LAST		BIT(0)
+#define AXI_DMAC_HW_FLAG_IRQ		BIT(1)
+
+int dma_id = 0;
+struct axi_dmac_info aidma_info[AXI_DMAC_NUM];
+extern spinlock_t aidma_lock;
+
+struct ai_dmac_hw_desc {
+	u32 flags;
+	u32 id;
+	u64 dest_addr;
+	u64 src_addr;
+	u64 next_sg_addr;
+	u32 y_len;
+	u32 x_len;
+	u32 src_stride;
+	u32 dst_stride;
+	u64 __pad[2];
+};
+
+struct ai_dmac_sg {
+	struct ai_dmac_hw_desc *hw;
+	dma_addr_t hw_phys;
+};
+
+struct ai_dmac_desc {
+	unsigned int num_sgs;
+	struct ai_dmac_sg sg[] __counted_by(num_sgs);
+};
+
+static void ai_dmac_write(struct ai_dmac *ai_dmac, unsigned int reg, unsigned int val)
+{
+	writel(val, ai_dmac->base + reg);
+}
+
+static int ai_dmac_read(struct ai_dmac *ai_dmac, unsigned int reg)
+{
+	return readl(ai_dmac->base + reg);
+}
+
+static struct ai_dmac_desc *axi_dmac_alloc_desc(struct ai_dmac *c, unsigned int num_sgs)
+{
+	struct ai_dmac_desc *desc;
+	struct ai_dmac_hw_desc *hws;
+	struct device *dev = c->dev;
+	dma_addr_t hw_phys;
+	unsigned int i;
+
+	desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT);
+	if (!desc)
+		return NULL;
+
+	desc->num_sgs = num_sgs;
+
+	hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)),
+				&hw_phys, GFP_ATOMIC);
+	if (!hws) {
+		kfree(desc);
+		return NULL;
+	}
+
+	for (i = 0; i < num_sgs; i++) {
+		desc->sg[i].hw = &hws[i];
+		desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws);
+		hws[i].id = AXI_DMAC_SG_UNUSED;
+		hws[i].flags = 0;
+		/* Link hardware descriptors */
+		hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws);
+	}
+
+	desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ;
+	return desc;
+}
+
+int ai_dmac_memcpy(struct ai_dmac *c, dma_addr_t dma_dst,
+		   dma_addr_t dma_src, size_t len)
+{
+	struct ai_dmac_desc *desc;
+	struct ai_dmac_sg *dsg;
+	int num_sgs = DIV_ROUND_UP(len, c->max_length);
+	unsigned int i;
+	unsigned int segment_size = DIV_ROUND_UP(len, num_sgs);
+	unsigned int flags = 0;
+	segment_size = ((segment_size - 1) | c->length_align_mask) + 1;
+
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_CFG, 0);
+	if (!c || !len)
+		return -1;
+
+	desc = axi_dmac_alloc_desc(c, num_sgs);
+	if (!desc)
+		return -1;
+	dsg = desc->sg;
+	for (i = 0; i < num_sgs; i++) {
+		dsg->hw->x_len = len > segment_size ? segment_size : len - 1;
+		dsg->hw->y_len = 0;
+		dsg->hw->src_addr = dma_src;
+		dsg->hw->dest_addr = dma_dst;
+		dma_src += segment_size;
+		dma_dst += segment_size;
+		dsg++;
+		len -= segment_size;
+	}
+
+	dsg = desc->sg;
+	dsg->hw->id = ai_dmac_read(c, AXI_DMAC_REG_TRANSFER_ID);
+	ai_dmac_write(c, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE | AXI_DMAC_CTRL_ENABLE_SG);
+
+	ai_dmac_write(c, AXI_DMAC_REG_SG_ADDRESS, (u32)dsg->hw_phys);
+	ai_dmac_write(c, AXI_DMAC_REG_SG_ADDRESS_HIGH, (u64)dsg->hw_phys >> 32);
+	ai_dmac_write(c, AXI_DMAC_REG_IRQ_PENDING, 3);
+	ai_dmac_write(c, AXI_DMAC_REG_FLAGS, flags);
+	ai_dmac_write(c, AXI_DMAC_REG_START_TRANSFER, 1);
+	return 0;
+}
+
+int ai_dmac_memcpy_by_2d(struct ai_dmac *c, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len)
+{
+	struct ai_dmac_desc *desc;
+	struct ai_dmac_sg *dsg;
+	u32 y_length = len / 512;
+	u32 residue = len % 512;
+	int num_sgs = residue == 0 ? 2 : 3;
+	unsigned int i;
+	unsigned int flags = 0;
+
+	desc = axi_dmac_alloc_desc(c, num_sgs);
+	if (!desc)
+		return -1;
+	dsg = desc->sg;
+	for (i = 0; i < 2; i++) {
+		dsg->hw->x_len = 255;
+		dsg->hw->y_len = y_length - 1;
+		dsg->hw->src_stride = 512;
+		dsg->hw->dst_stride = 512;
+		if (c->id % 2) {
+			if (i == 0) {
+				dsg->hw->src_addr = dma_src;
+				dsg->hw->dest_addr = dma_dst;
+			} else {
+				dsg->hw->src_addr = dma_src + 256;
+				dsg->hw->dest_addr = dma_dst + 256;
+			}
+		} else {
+			if (i == 0) {
+				dsg->hw->src_addr = dma_src + 256;
+				dsg->hw->dest_addr = dma_dst + 256;
+			} else {
+				dsg->hw->src_addr = dma_src;
+				dsg->hw->dest_addr = dma_dst;
+			}
+		}
+		dsg++;
+	}
+
+	if (num_sgs == 3) {
+		dsg->hw->x_len = residue;
+		dsg->hw->y_len = 0;
+		dsg->hw->src_addr = dma_src + len - residue;
+		dsg->hw->dest_addr = dma_dst + len - residue;
+	}
+
+	dsg = desc->sg;
+	dsg->hw->id = ai_dmac_read(c, AXI_DMAC_REG_TRANSFER_ID);
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_CFG, 0);
+	ai_dmac_write(c, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE | AXI_DMAC_CTRL_ENABLE_SG);
+
+	ai_dmac_write(c, AXI_DMAC_REG_SG_ADDRESS, (u32)dsg->hw_phys);
+	ai_dmac_write(c, AXI_DMAC_REG_SG_ADDRESS_HIGH, (u64)dsg->hw_phys >> 32);
+	ai_dmac_write(c, AXI_DMAC_REG_IRQ_PENDING, 3);
+	ai_dmac_write(c, AXI_DMAC_REG_FLAGS, flags);
+	ai_dmac_write(c, AXI_DMAC_REG_START_TRANSFER, 1);
+
+	return 0;
+}
+
+int ai_dmac_pack_start(struct ai_dmac *c, struct ai_pack_param *param,
+		       dma_addr_t dma_dst, dma_addr_t dma_src)
+{
+	u32 kp, mp;
+	u32 reg = 0;
+
+	kp = (param->k_size % param->kr_size) ? (param->kr_size - (param->k_size % param->kr_size)) : 0;
+	mp = (param->m_size % param->mr_size) ? (param->mr_size - (param->m_size % param->mr_size)) : 0;
+
+	ai_dmac_write(c, AXI_DMAC_REG_SRC_ADDRESS, dma_src & 0xffffffff);
+	ai_dmac_write(c, AXI_DMAC_REG_DEST_ADDRESS, dma_dst & 0xffffffff);
+	ai_dmac_write(c, AXI_DMAC_REG_SRC_ADDRESS_HIGH, dma_src >> 32);
+	ai_dmac_write(c, AXI_DMAC_REG_DEST_ADDRESS_HIGH, dma_dst >> 32);
+	if (param->sgdg == true)
+		reg |= AXI_DMAC_REG_SGDG_MODE;
+	if (param->pack == false)
+		reg |= AXI_DMAC_REG_PACK_MODE;
+	if (param->transpose == true)
+		reg |= AXI_DMAC_REG_TRANSPOSE;
+	if (kp || mp)
+		reg |= AXI_DMAC_REG_PAD_MODE;
+	reg |= param->ele_size << AXI_DMAC_REG_ELE_SIZE_OFFSET;
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_CFG, reg);
+	ai_dmac_write(c, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE | AXI_DMAC_CTRL_ENABLE_SG);
+
+	ai_dmac_write(c, AXI_DMAC_REG_PAD_VALUE, param->pad_value);
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_M_SIZE, param->m_size);
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_K_SIZE, param->k_size);
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_MR_SIZE, param->mr_size);
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_KR_SIZE, param->kr_size);
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_MP_SIZE, mp);
+	ai_dmac_write(c, AXI_DMAC_REG_SGDG_KP_SIZE, kp);
+	if ((param->pack == false) && (mp == 0))
+		ai_dmac_write(c, AXI_DMAC_REG_SGDG_MB_SIZE, param->m_size / param->mr_size);
+	else if ((param->pack == false) && (mp != 0))
+		ai_dmac_write(c, AXI_DMAC_REG_SGDG_MB_SIZE, (param->m_size / param->mr_size) + 1);
+
+	ai_dmac_write(c, AXI_DMAC_REG_FLAGS, 0);
+	ai_dmac_write(c, AXI_DMAC_REG_IRQ_PENDING, 3);
+	ai_dmac_write(c, AXI_DMAC_REG_SRC_STRIDE, 0);
+	ai_dmac_write(c, AXI_DMAC_REG_DEST_STRIDE, 0);
+	ai_dmac_write(c, AXI_DMAC_REG_X_LENGTH, 0);
+	ai_dmac_write(c, AXI_DMAC_REG_Y_LENGTH, 0);
+	ai_dmac_write(c, AXI_DMAC_REG_START_TRANSFER, 1);
+
+	return 0;
+}
+
+static irqreturn_t axi_dmac_start_next_handler(int irq, void *devid)
+{
+	struct ai_dmac *dma = (struct ai_dmac *)devid;
+	struct aidma_req *req;
+	unsigned long flags;
+	unsigned int pending;
+	int id;
+
+	spin_lock_irqsave(&aidma_lock, flags);
+	pending = ai_dmac_read(dma, AXI_DMAC_REG_IRQ_PENDING);
+	if (!pending) {
+		spin_unlock_irqrestore(&aidma_lock, flags);
+		return IRQ_NONE;
+	}
+
+	ai_dmac_write(dma, AXI_DMAC_REG_IRQ_PENDING, pending);
+	req = aidma_info[dma->id].req;
+	id = aidma_info[dma->id].work_id;
+	req[id].status = DMA_REQ_DONE;
+	aidma_info[dma->id].work = false;
+	spin_unlock_irqrestore(&aidma_lock, flags);
+	start_transfer();
+
+	return IRQ_HANDLED;
+}
+
+static int axi_dmac_get_config(struct device *dev, struct ai_dmac *dma)
+{
+	unsigned int val, reg;
+
+	reg = ai_dmac_read(dma, AXI_DMAC_REG_INTERFACE_DESC);
+	if (reg == 0) {
+		dev_err(dev, "DMA interface register reads zero\n");
+		return -EFAULT;
+	}
+
+	val = AXI_DMAC_DMA_SRC_WIDTH_GET(reg);
+	if (val == 0) {
+		dev_err(dev, "Source bus width is zero\n");
+		return -EINVAL;
+	}
+	dma->src_width = 1 << val;
+
+	val = AXI_DMAC_DMA_DST_WIDTH_GET(reg);
+        if (val == 0) {
+		dev_err(dev, "Destination bus width is zero\n");
+		return -EINVAL;
+	}
+	dma->dest_width = 1 << val;
+
+	ai_dmac_write(dma, AXI_DMAC_REG_X_LENGTH, 0xffffffff);
+	dma->max_length = ai_dmac_read(dma, AXI_DMAC_REG_X_LENGTH);
+	if (dma->max_length != UINT_MAX)
+		dma->max_length++;
+
+	ai_dmac_write(dma, AXI_DMAC_REG_X_LENGTH, 0x00);
+	dma->length_align_mask =
+		ai_dmac_read(dma, AXI_DMAC_REG_X_LENGTH);
+
+	return 0;
+}
+
+static int ai_dmac_probe(struct platform_device *pdev)
+{
+	struct ai_dmac *dma_dev;
+	u32 irq_mask = 0;
+	int ret;
+
+	dma_dev = devm_kzalloc(&pdev->dev, sizeof(struct ai_dmac), GFP_KERNEL);
+	if (!dma_dev)
+		return -ENOMEM;
+
+	dma_dev->irq = platform_get_irq(pdev, 0);
+	if (dma_dev->irq < 0)
+		return dma_dev->irq;
+
+	dma_dev->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(dma_dev->base))
+		return PTR_ERR(dma_dev->base);
+
+	dma_dev->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(dma_dev->clk))
+		return PTR_ERR(dma_dev->clk);
+	clk_prepare_enable(dma_dev->clk);
+
+	dma_dev->resets = devm_reset_control_array_get_optional_shared(&pdev->dev);
+	if (IS_ERR(dma_dev->resets))
+		return PTR_ERR(dma_dev->resets);
+	ret = reset_control_deassert(dma_dev->resets);
+	if (ret)
+		return ret;
+
+	dma_dev->dev = &pdev->dev;
+	ret = axi_dmac_get_config(&pdev->dev, dma_dev);
+	if (ret < 0)
+		return ret;
+
+	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
+
+	irq_mask |= AXI_DMAC_IRQ_SOT;
+	ai_dmac_write(dma_dev, AXI_DMAC_REG_IRQ_MASK, irq_mask);
+
+	ret = devm_request_irq(&pdev->dev, dma_dev->irq, axi_dmac_start_next_handler,
+				IRQF_SHARED, dev_name(&pdev->dev), dma_dev);
+	if (ret)
+		return ret;
+
+	aidma_info[dma_id].dma = dma_dev;
+	aidma_info[dma_id].work_id = -1;
+	aidma_info[dma_id].req = NULL;
+	aidma_info[dma_id].work = false;
+	dma_dev->id = dma_id;
+	dma_id++;
+
+	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(38));
+	return 0;
+}
+
+static const struct of_device_id ai_dmac_of_match_table[] = {
+	{ .compatible = "spacemit-ai-dmac" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ai_dmac_of_match_table);
+
+static struct platform_driver ai_dmac_driver = {
+	.driver = {
+		.name = "ai-dmac",
+		.of_match_table = ai_dmac_of_match_table,
+	},
+	.probe = ai_dmac_probe,
+};
+module_platform_driver(ai_dmac_driver);
diff --git a/drivers/dma/ai_dma_dev.c b/drivers/dma/ai_dma_dev.c
new file mode 100644
index 000000000000..64b079b5786d
--- /dev/null
+++ b/drivers/dma/ai_dma_dev.c
@@ -0,0 +1,565 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Spacemit hdma service driver support
+ *
+ * Copyright (c) 2025 SPACEMIT, Co. Ltd.
+ */
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/sched.h>
+#include <linux/device.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/msi.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <asm/uaccess.h>
+#include <linux/mutex.h>
+#include <linux/pid.h>
+#include "ai_dma.h"
+
+#define DEVICE_NAME	"ai_dma"
+#define REQLIST_NAME	"aidma_list"
+#define DMAMSI_NAME	"dma_msi"
+#define IOC_MAGIC	"c"
+#define MAP_SIZE	4096
+#define GET_MSI_HWIRQ	0
+
+spinlock_t aidma_lock;
+static struct list_head dma_req_list;
+unsigned long msi_addr;
+unsigned int msi_hwirq;
+
+typedef struct {
+	size_t			size;
+	unsigned long		user_addr;
+	void			*kern_addr;
+	dma_addr_t		dma_addr;
+	struct list_head	list;
+} dma_map_info_t;
+
+typedef struct {
+	dma_addr_t		addr;
+	size_t			size;
+} va2pa_t;
+
+static int va2pa(void *va, size_t size, va2pa_t **va2pa, pid_t pid)
+{
+	pmd_t *pmd;
+	pte_t *pte;
+	struct task_struct *task;
+	struct mm_struct *mm;
+	unsigned long pg_offset;
+	unsigned long pg_address;
+	int flag = 0;
+	va2pa_t *p;
+
+	unsigned long vaddr = (unsigned long)va;
+
+	task = get_pid_task(find_get_pid(pid), PIDTYPE_PID);
+	if (!task) {
+		return -ESRCH;
+	}
+	mm = get_task_mm(task);
+	if (!mm) {
+		return -EINVAL;
+	}
+
+	p = kmalloc(sizeof(va2pa_t), GFP_ATOMIC);
+	if (!p)
+		printk("failed to alloc for pages\n");
+	*va2pa = p;
+
+	memset(p, 0x00, sizeof(va2pa_t));
+	pmd = pmd_off(mm, vaddr);
+	if (pmd_none(*pmd)) {
+		printk("not in the pmd!\n");
+		flag = -1;
+	}
+
+	pte = pte_offset_map(pmd, vaddr);
+	if (pte_none(*pte)) {
+		printk("not in the pte!\n");
+		flag = -1;
+	}
+
+	pg_offset = offset_in_page(vaddr);
+	pg_address = pte_pfn(__pte(pte_val(*pte))) << PAGE_SHIFT;
+	p->addr = pg_address | pg_offset;
+	p->size = size;
+	pte_unmap(pte);
+
+	return flag;
+}
+
+static int dma_malloc(struct ai_dmac *dma, dma_map_info_t *dma_info, struct vm_area_struct *vma)
+{
+	dma_info->kern_addr = kmalloc(dma_info->size, GFP_KERNEL);
+	if (!dma_info->kern_addr) {
+		dev_err(dma->dev,"kmalloc failed\n");
+		return -ENOMEM;
+	}
+
+	dma_info->dma_addr = dma_map_single(dma->dev, dma_info->kern_addr, dma_info->size, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dma->dev, dma_info->dma_addr)) {
+		dev_err(dma->dev,"mapping buffer failed\n");
+		return -1;
+	}
+
+	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	if (remap_pfn_range(vma, vma->vm_start, (virt_to_phys(dma_info->kern_addr) >> PAGE_SHIFT),
+			    vma->vm_end - vma->vm_start, vma->vm_page_prot)) {
+		dma_unmap_single(dma->dev, dma_info->dma_addr, dma_info->size, DMA_BIDIRECTIONAL);
+		kfree(dma_info->kern_addr);
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
+static void dma_free(struct ai_dmac *dma, dma_map_info_t *dma_info)
+{
+	dma_unmap_single(dma->dev, dma_info->dma_addr, dma_info->size, DMA_BIDIRECTIONAL);
+	kfree(dma_info->kern_addr);
+}
+
+static int dma_open(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static int dma_release(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static void dma_vma_close(struct vm_area_struct *vma)
+{
+	dma_map_info_t *dma_info;
+	struct ai_dmac *dma = aidma_info[0].dma;;
+
+	dma_info = vma->vm_private_data;
+	dma_free(dma, dma_info);
+}
+
+static const struct vm_operations_struct dma_vm_ops = {
+	.close = dma_vma_close,
+};
+
+static int dma_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	int ret;
+	dma_map_info_t *dma_info;
+	struct ai_dmac *dma;
+
+	if (aidma_info[0].dma == NULL)
+		return -1;
+	else
+		dma = aidma_info[0].dma;
+
+	dma_info = kmalloc(sizeof(*dma_info), GFP_KERNEL);
+	if (dma_info == NULL) {
+		printk("Unable to allocate VMA data structure.\n");
+		return -ENOMEM;
+	}
+	dma_info->size = vma->vm_end - vma->vm_start;
+	dma_info->user_addr = vma->vm_start;
+	ret = dma_malloc(dma, dma_info, vma);
+	if (ret < 0)
+		return -1;
+
+	vma->vm_ops = &dma_vm_ops;
+	vma->vm_private_data = dma_info;
+	vm_flags_set(vma, VM_DONTCOPY);
+
+	return 0;
+}
+
+static int dma_list_open(struct inode *inode, struct file *filp)
+{
+	struct ai_dmac *dma;
+	struct req_addr_info *info;
+	struct req_node *node;
+	unsigned long flags;
+
+	if (aidma_info[0].dma == NULL)
+		return -1;
+	else
+		dma = aidma_info[0].dma;
+
+	info = kmalloc(sizeof(struct req_addr_info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	node = kmalloc(sizeof(struct req_node), GFP_KERNEL);
+	if (!node) {
+		kfree(info);
+		return -ENOMEM;
+	}
+
+	info->req_list = dma_alloc_coherent(dma->dev, AIDMA_MAX_REQ * sizeof(struct aidma_req),
+					    &info->req_addr, GFP_KERNEL);
+	if (!info->req_list) {
+		kfree(node);
+		kfree(info);
+		return -ENOMEM;
+	}
+
+	node->info = info;
+	INIT_LIST_HEAD(&node->list);
+
+	spin_lock_irqsave(&aidma_lock, flags);
+	list_add_tail(&node->list, &dma_req_list);
+	spin_unlock_irqrestore(&aidma_lock, flags);
+
+	filp->private_data = info;
+
+	return 0;
+}
+
+static int dma_list_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct ai_dmac *dma;
+	struct req_addr_info *info = file->private_data;
+
+	if (aidma_info[0].dma == NULL)
+		return -1;
+	else
+		dma = aidma_info[0].dma;
+
+	dma_mmap_coherent(dma->dev, vma, info->req_list, info->req_addr, AIDMA_MAX_REQ * sizeof(struct aidma_req));
+	return 0;
+}
+
+static int dma_list_release(struct inode *inode, struct file *filp)
+{
+	struct req_addr_info *info = filp->private_data;
+	struct req_node *node, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&aidma_lock, flags);
+	list_for_each_entry_safe(node, tmp, &dma_req_list, list) {
+		if (node->info->req_addr == info->req_addr) {
+			list_del(&node->list);
+			kfree(node);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&aidma_lock, flags);
+
+	dma_free_coherent(aidma_info[0].dma->dev, AIDMA_MAX_REQ * sizeof(struct aidma_req),
+				info->req_list, info->req_addr);
+	kfree(info);
+	return 0;
+}
+
+static const struct file_operations ai_dma_fops = {
+	.owner		= THIS_MODULE,
+	.open		= dma_open,
+	.release	= dma_release,
+	.mmap		= dma_mmap,
+};
+
+static const struct file_operations req_fops = {
+	.owner		= THIS_MODULE,
+	.open		= dma_list_open,
+	.release	= dma_list_release,
+	.mmap		= dma_list_mmap,
+};
+
+void start_transfer() {
+	struct aidma_req *req;
+	struct dma_transfer_param *param;
+	struct ai_pack_param *params;
+	struct ai_dmac *dma;
+	struct req_node *node;
+	int i, j, ret;
+	size_t size = 0;
+	dma_addr_t src_addr, dst_addr;
+	va2pa_t *s_pa_l, *d_pa_l;
+	unsigned long flags;
+
+	spin_lock_irqsave(&aidma_lock, flags);
+	list_for_each_entry(node, &dma_req_list, list) {
+		req = node->info->req_list;
+		for (i = 0; i < AIDMA_MAX_REQ; i++) {
+			if (req[i].status == DMA_REQ_SUBMIT) {
+				for (j = 0; j < AXI_DMAC_NUM; j++) {
+					if (aidma_info[j].work != true) {
+						aidma_info[j].work = true;
+						aidma_info[j].work_id = i;
+						aidma_info[j].req = req;
+						dma = aidma_info[j].dma;
+						req[i].status = DMA_REQ_PROCESS;
+						break;
+					}
+				}
+				if (dma == NULL) {
+					spin_unlock_irqrestore(&aidma_lock, flags);
+					return;
+				}
+				param = &req[i].params;
+				if (param->is_sgdg == true) {
+					size = param->ai_param.m_size * param->ai_param.k_size;
+				} else {
+					size = param->size;
+				}
+
+				ret = va2pa(param->src, size, &s_pa_l, param->pid);
+				if (ret != 0) {
+					pr_info("src va2pa error, addr:%llx, ret = %d\n",(unsigned long long)param->src,ret);
+					if (s_pa_l)
+						kfree(s_pa_l);
+					spin_unlock_irqrestore(&aidma_lock, flags);
+					return;
+				}
+
+				ret = va2pa(param->dst, size, &d_pa_l, param->pid);
+				if (ret != 0) {
+					pr_info("dst va2pa error, addr:%llx, ret = %d\n", (unsigned long long)param->dst,ret);
+					if (d_pa_l)
+						kfree(d_pa_l);
+					kfree(s_pa_l);
+					spin_unlock_irqrestore(&aidma_lock, flags);
+					return;
+				}
+				src_addr = s_pa_l->addr;
+				dst_addr = d_pa_l->addr;
+				pr_debug("src_addr:%llx dst_addr:%llx\n",src_addr,dst_addr);
+
+				if (param->is_sgdg == true) {
+					params = kmalloc(sizeof(struct ai_pack_param), GFP_ATOMIC);
+					if (!params) {
+						kfree(s_pa_l);
+						kfree(d_pa_l);
+						spin_unlock_irqrestore(&aidma_lock, flags);
+						return;
+					}
+					params->sgdg = true;
+					params->pack = param->ai_param.pack;
+					params->pad_value = param->ai_param.pad_value;
+					params->transpose = param->ai_param.transpose;
+					params->ele_size = param->ai_param.ele_size;
+					params->m_size = param->ai_param.m_size;
+					params->k_size = param->ai_param.k_size;
+					params->mr_size = param->ai_param.mr_size;
+					params->kr_size = param->ai_param.kr_size;
+					ai_dmac_pack_start(dma, params, dst_addr, src_addr);
+				} else {
+					ai_dmac_memcpy_by_2d(dma, dst_addr, src_addr, param->size);
+				}
+			}
+		}
+	}
+	spin_unlock_irqrestore(&aidma_lock, flags);
+	return;
+}
+
+static void dma_msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	unsigned int msi_addr_lo = msg->address_lo;
+	unsigned int msi_addr_hi = msg->address_hi;
+	msi_addr = ((unsigned long)msi_addr_hi << 32) + msi_addr_lo;
+	msi_hwirq = msg->data;
+}
+
+static int dma_msi_open(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static int dma_msi_release(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static int dma_msi_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long pfn = msi_addr >> PAGE_SHIFT;
+	unsigned long vsize = vma->vm_end - vma->vm_start;
+	unsigned long psize = MAP_SIZE - offset;
+	int ret;
+
+	if (vsize > psize)
+		return -EINVAL;
+
+	ret = remap_pfn_range(vma, vma->vm_start, pfn, vsize, vma->vm_page_prot);
+	if (ret)
+		return -EAGAIN;
+
+	pr_debug("Successfully mapped physical 0x%lx to user virtual 0x%lx\n",
+			msi_addr, vma->vm_start);
+	return 0;
+}
+
+static long dma_msi_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	switch (cmd) {
+	case GET_MSI_HWIRQ:
+		if (copy_to_user((unsigned int __user *)arg, &msi_hwirq, sizeof(msi_hwirq)))
+			return -EFAULT;
+		return 0;
+	default:
+		return -ENOTTY;
+	};
+}
+
+static const struct file_operations dma_msi_fops = {
+	.owner		= THIS_MODULE,
+	.open		= dma_msi_open,
+	.release	= dma_msi_release,
+	.mmap		= dma_msi_mmap,
+	.unlocked_ioctl	= dma_msi_ioctl,
+};
+
+static irqreturn_t ai_dma_start_transfer(int irq, void *devid)
+{
+	start_transfer();
+	return IRQ_HANDLED;
+}
+
+static char *ai_dma_devnode(const struct device *dev, umode_t *mode)
+{
+	if (mode)
+		*mode = 0666;
+	return NULL;
+}
+
+static int ai_dmadev_probe(struct platform_device *pdev) {
+	struct device *dev;
+	static unsigned char dma_major, dma_req, msi_major;
+	static struct class *dma_class, *req_class, *msi_class;
+	int msi_irq, ret = 0;
+
+	dev = &pdev->dev;
+	if (!dev_get_msi_domain(dev)) {
+		if (is_of_node(dev->fwnode))
+			of_msi_configure(dev, to_of_node(dev->fwnode));
+	}
+	ret = platform_device_msi_init_and_alloc_irqs(dev, 1, dma_msi_write);
+	if (ret) {
+		return dev_err_probe(&pdev->dev, ret, "Failed to allocate 1 MSI\n");
+	}
+	msi_irq = msi_get_virq(&pdev->dev, 0);
+	ret = request_irq(msi_irq, ai_dma_start_transfer, IRQF_SHARED, dev_name(&pdev->dev), &pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request msi irq\n");
+		goto err_free_msi;
+	}
+
+	msi_major = register_chrdev(0, DMAMSI_NAME, &dma_msi_fops);
+	if (msi_major < 0) {
+		ret = msi_major;
+		goto err_free_irq;
+	}
+
+	msi_class = class_create(DMAMSI_NAME);
+	if (IS_ERR(msi_class)) {
+		ret = PTR_ERR(msi_class);
+		goto err_unregister_msi_chrdev;
+	}
+
+	msi_class->devnode = ai_dma_devnode;
+
+	dev = device_create(msi_class, NULL, MKDEV(msi_major, 0), NULL, DMAMSI_NAME);
+	if (IS_ERR(dev)) {
+		ret = PTR_ERR(dev);
+		goto err_destroy_msi_class;
+	}
+
+	dma_major = register_chrdev(0 , DEVICE_NAME, &ai_dma_fops);
+	if (dma_major < 0) {
+		ret = dma_major;
+		goto err_destroy_msi_device;
+	}
+
+	dma_class = class_create(DEVICE_NAME);
+	if (IS_ERR(dma_class)) {
+		ret = PTR_ERR(dma_class);
+		goto err_unregister_dma_chrdev;
+	}
+
+	dma_class->devnode = ai_dma_devnode;
+
+	dev = device_create(dma_class, NULL, MKDEV(dma_major, 0), NULL, DEVICE_NAME);
+	if (IS_ERR(dev)) {
+		ret = PTR_ERR(dev);
+		goto err_destroy_dma_class;
+	}
+
+	dma_req = register_chrdev(0, REQLIST_NAME, &req_fops);
+	if (dma_req < 0) {
+		ret = dma_req;
+		goto err_destroy_dma_device;
+	}
+
+	req_class = class_create(REQLIST_NAME);
+	if (IS_ERR(req_class)) {
+		ret = PTR_ERR(req_class);
+		goto err_unregister_req_chrdev;
+	}
+
+	req_class->devnode = ai_dma_devnode;
+
+	dev = device_create(req_class, NULL, MKDEV(dma_req, 0), NULL, REQLIST_NAME);
+	if (IS_ERR(dev)) {
+		ret = PTR_ERR(dev);
+		goto err_destroy_req_class;
+	}
+
+	INIT_LIST_HEAD(&dma_req_list);
+	spin_lock_init(&aidma_lock);
+
+	return 0;
+
+err_destroy_req_class:
+	class_destroy(req_class);
+err_unregister_req_chrdev:
+	unregister_chrdev(dma_req, REQLIST_NAME);
+err_destroy_dma_device:
+	device_destroy(dma_class, MKDEV(dma_major, 0));
+err_destroy_dma_class:
+	class_destroy(dma_class);
+err_unregister_dma_chrdev:
+	unregister_chrdev(dma_major, DEVICE_NAME);
+err_destroy_msi_device:
+	device_destroy(msi_class, MKDEV(msi_major, 0));
+err_destroy_msi_class:
+	class_destroy(msi_class);
+err_unregister_msi_chrdev:
+	unregister_chrdev(msi_major, DMAMSI_NAME);
+err_free_irq:
+	free_irq(msi_irq, &pdev->dev);
+err_free_msi:
+	platform_device_msi_free_irqs_all(dev);
+
+	return ret;
+}
+
+static void ai_dmadev_remove(struct platform_device *pdev)
+{
+	platform_device_msi_free_irqs_all(&pdev->dev);
+}
+
+static const struct of_device_id ai_dmadev_of_match[] = {
+	{ .compatible = "ai-dma-dev", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ai_dmadev_of_match);
+
+static struct platform_driver ai_dma_driver = {
+	.driver		= {
+		.name	= "ai_dma_dev",
+		.of_match_table = ai_dmadev_of_match,
+	},
+	.probe		= ai_dmadev_probe,
+	.remove		= ai_dmadev_remove,
+};
+module_platform_driver(ai_dma_driver);
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index d07229a74886..3d81208634c0 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -19,6 +19,7 @@
 #include <linux/reset.h>
 #include <linux/of_dma.h>
 #include <linux/of.h>
+#include <linux/delay.h>
 
 #include "dmaengine.h"
 
@@ -68,6 +69,7 @@
 #define DCMD_BURST8	(1 << 16)	/* 8 byte burst */
 #define DCMD_BURST16	(2 << 16)	/* 16 byte burst */
 #define DCMD_BURST32	(3 << 16)	/* 32 byte burst */
+#define DCMD_BURST64	(4 << 16)	/* 64 byte burst */
 #define DCMD_WIDTH1	(1 << 14)	/* 1 byte width */
 #define DCMD_WIDTH2	(2 << 14)	/* 2 byte width (HalfWord) */
 #define DCMD_WIDTH4	(3 << 14)	/* 4 byte width (Word) */
@@ -125,6 +127,7 @@ struct mmp_pdma_chan {
 	bool idle;			/* channel statue machine */
 	bool byte_align;
 
+	u32 bytes_residue;
 	struct dma_pool *desc_pool;	/* Descriptors pool */
 };
 
@@ -152,8 +155,8 @@ struct mmp_pdma_phy {
  *
  * Controller Configuration:
  * @run_bits:   Control bits in DCSR register for channel start/stop
- * @dma_mask:   DMA addressing capability of controller. 0 to use OF/platform
- *              settings, or explicit mask like DMA_BIT_MASK(32/64)
+ * @dma_width:  DMA addressing width in bits (32 or 64). Determines the
+ *              DMA mask capability of the controller hardware.
  */
 struct mmp_pdma_ops {
 	/* Hardware Register Operations */
@@ -173,7 +176,7 @@ struct mmp_pdma_ops {
 
 	/* Controller Configuration */
 	u32 run_bits;
-	u64 dma_mask;
+	u32 dma_width;
 };
 
 struct mmp_pdma_device {
@@ -183,6 +186,8 @@ struct mmp_pdma_device {
 	struct dma_device		device;
 	struct mmp_pdma_phy		*phy;
 	const struct mmp_pdma_ops	*ops;
+	struct clk			*clk;
+	struct reset_control		*rst;
 	spinlock_t phy_lock; /* protect alloc/free phy channels */
 };
 
@@ -318,7 +323,7 @@ static void enable_chan(struct mmp_pdma_phy *phy)
 
 static void disable_chan(struct mmp_pdma_phy *phy)
 {
-	u32 reg, dcsr;
+	u32 reg, dcsr, cnt = 1000;
 
 	if (!phy)
 		return;
@@ -335,6 +340,19 @@ static void disable_chan(struct mmp_pdma_phy *phy)
 		/* If no vchan, just clear the RUN bit */
 		writel(dcsr & ~DCSR_RUN, phy->base + reg);
 	}
+
+	/* ensure dma is stopped. */
+	do {
+		u32 dcsr = readl(phy->base + reg);
+
+		if (dcsr & DCSR_STOPSTATE)
+			return;
+		udelay(10);
+	} while (--cnt);
+
+	if (!cnt)
+		dev_warn(phy->vchan ? phy->vchan->dev : NULL,
+			 "DMA channel %d failed to stop within timeout\n", phy->idx);
 }
 
 static int clear_chan_irq(struct mmp_pdma_phy *phy)
@@ -362,6 +380,9 @@ static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id)
 	if (clear_chan_irq(phy) != 0)
 		return IRQ_NONE;
 
+	if (!phy->vchan)
+		return IRQ_HANDLED;
+
 	tasklet_schedule(&phy->vchan->tasklet);
 	return IRQ_HANDLED;
 }
@@ -433,14 +454,16 @@ static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
 	unsigned long flags;
 	u32 reg;
 
-	if (!pchan->phy)
+	spin_lock_irqsave(&pdev->phy_lock, flags);
+	if (!pchan->phy) {
+		spin_unlock_irqrestore(&pdev->phy_lock, flags);
 		return;
+	}
 
 	/* clear the channel mapping in DRCMR */
 	reg = DRCMR(pchan->drcmr);
 	writel(0, pchan->phy->base + reg);
 
-	spin_lock_irqsave(&pdev->phy_lock, flags);
 	pchan->phy->vchan = NULL;
 	pchan->phy = NULL;
 	spin_unlock_irqrestore(&pdev->phy_lock, flags);
@@ -491,6 +514,7 @@ static void start_pending_queue(struct mmp_pdma_chan *chan)
 	pdev->ops->write_next_addr(chan->phy, desc->async_tx.phys);
 	enable_chan(chan->phy);
 	chan->idle = false;
+	chan->bytes_residue = 0;
 }
 
 
@@ -695,13 +719,13 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 	if ((sgl == NULL) || (sg_len == 0))
 		return NULL;
 
-	chan->byte_align = false;
+	chan->byte_align = true;
 
 	mmp_pdma_config_write(dchan, &chan->slave_config, dir);
 
 	for_each_sg(sgl, sg, sg_len, i) {
 		addr = sg_dma_address(sg);
-		avail = sg_dma_len(sgl);
+		avail = sg_dma_len(sg);
 
 		do {
 			len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
@@ -888,6 +912,8 @@ static int mmp_pdma_config_write(struct dma_chan *dchan,
 		chan->dcmd |= DCMD_BURST16;
 	else if (maxburst == 32)
 		chan->dcmd |= DCMD_BURST32;
+	else if (maxburst == 64)
+		chan->dcmd |= DCMD_BURST64;
 
 	chan->dir = direction;
 	chan->dev_addr = addr;
@@ -904,6 +930,17 @@ static int mmp_pdma_config(struct dma_chan *dchan,
 	return 0;
 }
 
+static int mmp_pdma_pause(struct dma_chan *dchan)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+
+	if (!chan->phy)
+		return -EINVAL;
+
+	disable_chan(chan->phy);
+	return 0;
+}
+
 static int mmp_pdma_terminate_all(struct dma_chan *dchan)
 {
 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
@@ -919,6 +956,7 @@ static int mmp_pdma_terminate_all(struct dma_chan *dchan)
 	mmp_pdma_free_desc_list(chan, &chan->chain_running);
 	spin_unlock_irqrestore(&chan->desc_lock, flags);
 	chan->idle = true;
+	chan->bytes_residue = 0;
 
 	return 0;
 }
@@ -938,7 +976,7 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
 	 * been completed. Therefore, its residue is 0.
 	 */
 	if (!chan->phy)
-		return 0;
+		return chan->bytes_residue; /* special case for EORIRQEN */
 
 	if (chan->dir == DMA_DEV_TO_MEM)
 		curr = pdev->ops->read_dst_addr(chan->phy);
@@ -1006,10 +1044,13 @@ static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan,
 {
 	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
 	enum dma_status ret;
+	unsigned long flags;
 
+	spin_lock_irqsave(&chan->desc_lock, flags);
 	ret = dma_cookie_status(dchan, cookie, txstate);
 	if (likely(ret != DMA_ERROR))
 		dma_set_residue(txstate, mmp_pdma_residue(chan, cookie));
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
 
 	return ret;
 }
@@ -1054,6 +1095,13 @@ static void dma_do_tasklet(struct tasklet_struct *t)
 
 	/* submit pending list; callback for each desc; free desc */
 	spin_lock_irqsave(&chan->desc_lock, flags);
+	list_for_each_entry(desc, &chan->chain_running, node) {
+		if (desc->desc.dcmd & DCMD_ENDIRQEN) {
+			chan->bytes_residue =
+				mmp_pdma_residue(chan, desc->async_tx.cookie);
+			break;
+		}
+	}
 
 	list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) {
 		/*
@@ -1156,6 +1204,7 @@ static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq)
 	INIT_LIST_HEAD(&chan->chain_pending);
 	INIT_LIST_HEAD(&chan->chain_running);
 
+	chan->bytes_residue = 0;
 	/* register virt channel to dma engine */
 	list_add_tail(&chan->chan.device_node, &pdev->device.channels);
 
@@ -1172,7 +1221,7 @@ static const struct mmp_pdma_ops marvell_pdma_v1_ops = {
 	.get_desc_src_addr = get_desc_src_addr_32,
 	.get_desc_dst_addr = get_desc_dst_addr_32,
 	.run_bits = (DCSR_RUN),
-	.dma_mask = 0,			/* let OF/platform set DMA mask */
+	.dma_width = 32,
 };
 
 static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
@@ -1184,8 +1233,8 @@ static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
 	.set_desc_dst_addr = set_desc_dst_addr_64,
 	.get_desc_src_addr = get_desc_src_addr_64,
 	.get_desc_dst_addr = get_desc_dst_addr_64,
-	.run_bits = (DCSR_RUN | DCSR_LPAEEN),
-	.dma_mask = DMA_BIT_MASK(64),	/* force 64-bit DMA addr capability */
+	.run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN),
+	.dma_width = 64,
 };
 
 static const struct of_device_id mmp_pdma_dt_ids[] = {
@@ -1216,12 +1265,15 @@ static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec,
 	return chan;
 }
 
+static void mmp_pdma_reset_control_assert(void *data)
+{
+	reset_control_assert(data);
+}
+
 static int mmp_pdma_probe(struct platform_device *op)
 {
 	struct mmp_pdma_device *pdev;
 	struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev);
-	struct clk *clk;
-	struct reset_control *rst;
 	int i, ret, irq = 0;
 	int dma_channels = 0, irq_num = 0;
 	const enum dma_slave_buswidth widths =
@@ -1240,14 +1292,25 @@ static int mmp_pdma_probe(struct platform_device *op)
 	if (IS_ERR(pdev->base))
 		return PTR_ERR(pdev->base);
 
-	clk = devm_clk_get_optional_enabled(pdev->dev, NULL);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	pdev->clk = devm_clk_get_optional_enabled(pdev->dev, NULL);
+	if (IS_ERR(pdev->clk))
+		return PTR_ERR(pdev->clk);
+
+	pdev->rst = devm_reset_control_get_optional_exclusive(pdev->dev, NULL);
+	if (IS_ERR(pdev->rst))
+		return PTR_ERR(pdev->rst);
 
-	rst = devm_reset_control_get_optional_exclusive_deasserted(pdev->dev,
-								   NULL);
-	if (IS_ERR(rst))
-		return PTR_ERR(rst);
+	if (pdev->rst) {
+		ret = reset_control_deassert(pdev->rst);
+		if (ret)
+			return ret;
+
+		ret = devm_add_action_or_reset(pdev->dev,
+					       mmp_pdma_reset_control_assert,
+					       pdev->rst);
+		if (ret)
+			return ret;
+	}
 
 	pdev->ops = of_device_get_match_data(&op->dev);
 	if (!pdev->ops)
@@ -1307,20 +1370,17 @@ static int mmp_pdma_probe(struct platform_device *op)
 	pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic;
 	pdev->device.device_issue_pending = mmp_pdma_issue_pending;
 	pdev->device.device_config = mmp_pdma_config;
+	pdev->device.device_pause = mmp_pdma_pause;
 	pdev->device.device_terminate_all = mmp_pdma_terminate_all;
 	pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
 	pdev->device.src_addr_widths = widths;
 	pdev->device.dst_addr_widths = widths;
 	pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
-	pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
+	pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 
-	/* Set DMA mask based on ops->dma_mask, or OF/platform */
-	if (pdev->ops->dma_mask)
-		dma_set_mask(pdev->dev, pdev->ops->dma_mask);
-	else if (pdev->dev->coherent_dma_mask)
-		dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask);
-	else
-		dma_set_mask(pdev->dev, DMA_BIT_MASK(64));
+	/* Set DMA mask based on controller hardware capabilities */
+	dma_set_mask_and_coherent(pdev->dev,
+				  DMA_BIT_MASK(pdev->ops->dma_width));
 
 	ret = dma_async_device_register(&pdev->device);
 	if (ret) {
@@ -1349,9 +1409,42 @@ static const struct platform_device_id mmp_pdma_id_table[] = {
 	{ },
 };
 
+#ifdef CONFIG_PM_SLEEP
+static int mmp_pdma_suspend_noirq(struct device *dev)
+{
+	struct mmp_pdma_device *pdev = dev_get_drvdata(dev);
+
+	if (pdev->clk)
+		clk_disable_unprepare(pdev->clk);
+
+	return 0;
+}
+
+static int mmp_pdma_resume_noirq(struct device *dev)
+{
+	struct mmp_pdma_device *pdev = dev_get_drvdata(dev);
+	int ret;
+
+	if (pdev->clk) {
+		ret = clk_prepare_enable(pdev->clk);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops mmp_pdma_pm_ops = {
+	NOIRQ_SYSTEM_SLEEP_PM_OPS(mmp_pdma_suspend_noirq, mmp_pdma_resume_noirq)
+};
+#endif
+
 static struct platform_driver mmp_pdma_driver = {
 	.driver		= {
 		.name	= "mmp-pdma",
+#ifdef CONFIG_PM_SLEEP
+		.pm	= &mmp_pdma_pm_ops,
+#endif
 		.of_match_table = mmp_pdma_dt_ids,
 	},
 	.id_table	= mmp_pdma_id_table,
diff --git a/drivers/dma/mmp_pdma_spacemit.c b/drivers/dma/mmp_pdma_spacemit.c
new file mode 100644
index 000000000000..822ac1625051
--- /dev/null
+++ b/drivers/dma/mmp_pdma_spacemit.c
@@ -0,0 +1,1628 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2012 Marvell International Ltd.
+ */
+
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/dmaengine.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/platform_data/mmp_dma.h>
+#include <linux/dmapool.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/of_device.h>
+#include <linux/of_dma.h>
+#include <linux/of.h>
+
+#include <linux/delay.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_qos.h>
+
+#include "dmaengine.h"
+
+#define DDADRH(n)	(0x0300 + ((n) << 4))
+#define DSADRH(n)	(0x0304 + ((n) << 4))
+#define DTADRH(n)	(0x0308 + ((n) << 4))
+#define DCSR_LPAEEN	BIT(21)	/* Long Physical Address Extension enable */
+#define DRCMR_INVALID	100		/* Max DMA request number + 1 */
+#define DCMD_BURST64	(4 << 16)	/* 64 byte burst */
+
+#define DCSR		0x0000
+#define DALGN		0x00a0
+#define DINT		0x00f0
+#define DDADR		0x0200
+#define DSADR(n)	(0x0204 + ((n) << 4))
+#define DTADR(n)	(0x0208 + ((n) << 4))
+#define DCMD		0x020c
+
+#define DCSR_RUN	BIT(31)	/* Run Bit (read / write) */
+#define DCSR_NODESC	BIT(30)	/* No-Descriptor Fetch (read / write) */
+#define DCSR_STOPIRQEN	BIT(29)	/* Stop Interrupt Enable (read / write) */
+#define DCSR_REQPEND	BIT(8)	/* Request Pending (read-only) */
+#define DCSR_STOPSTATE	BIT(3)	/* Stop State (read-only) */
+#define DCSR_ENDINTR	BIT(2)	/* End Interrupt (read / write) */
+#define DCSR_STARTINTR	BIT(1)	/* Start Interrupt (read / write) */
+#define DCSR_BUSERR	BIT(0)	/* Bus Error Interrupt (read / write) */
+
+#define DCSR_EORIRQEN	BIT(28)	/* End of Receive Interrupt Enable (R/W) */
+#define DCSR_EORJMPEN	BIT(27)	/* Jump to next descriptor on EOR */
+#define DCSR_EORSTOPEN	BIT(26)	/* STOP on an EOR */
+#define DCSR_SETCMPST	BIT(25)	/* Set Descriptor Compare Status */
+#define DCSR_CLRCMPST	BIT(24)	/* Clear Descriptor Compare Status */
+#define DCSR_CMPST	BIT(10)	/* The Descriptor Compare Status */
+#define DCSR_EORINTR	BIT(9)	/* The end of Receive */
+
+#define DRCMR(n)	((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2))
+#define DRCMR_MAPVLD	BIT(7)	/* Map Valid (read / write) */
+#define DRCMR_CHLNUM	0x1f	/* mask for Channel Number (read / write) */
+
+#define DDADR_DESCADDR	0xfffffff0	/* Address of next descriptor (mask) */
+#define DDADR_STOP	BIT(0)	/* Stop (read / write) */
+
+#define DCMD_INCSRCADDR	BIT(31)	/* Source Address Increment Setting. */
+#define DCMD_INCTRGADDR	BIT(30)	/* Target Address Increment Setting. */
+#define DCMD_FLOWSRC	BIT(29)	/* Flow Control by the source. */
+#define DCMD_FLOWTRG	BIT(28)	/* Flow Control by the target. */
+#define DCMD_STARTIRQEN	BIT(22)	/* Start Interrupt Enable */
+#define DCMD_ENDIRQEN	BIT(21)	/* End Interrupt Enable */
+#define DCMD_ENDIAN	BIT(18)	/* Device Endian-ness. */
+#define DCMD_BURST8	(1 << 16)	/* 8 byte burst */
+#define DCMD_BURST16	(2 << 16)	/* 16 byte burst */
+#define DCMD_BURST32	(3 << 16)	/* 32 byte burst */
+#define DCMD_WIDTH1	(1 << 14)	/* 1 byte width */
+#define DCMD_WIDTH2	(2 << 14)	/* 2 byte width (HalfWord) */
+#define DCMD_WIDTH4	(3 << 14)	/* 4 byte width (Word) */
+#define DCMD_LENGTH	0x01fff		/* length mask (max = 8K - 1) */
+
+#define PDMA_MAX_DESC_BYTES	DCMD_LENGTH
+
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+struct mmp_pdma_desc_hw {
+	u32 ddadr;	/* Points to the next descriptor + flags */
+	u32 dsadr;	/* DSADR value for the current transfer */
+	u32 dtadr;	/* DTADR value for the current transfer */
+	u32 dcmd;	/* DCMD value for the current transfer */
+	u32 ddadrh;	/* Points to the next descriptor + flags */
+	u32 dsadrh;	/* DSADR value for the current transfer */
+	u32 dtadrh;	/* DTADR value for the current transfer */
+	u32 rsvd;	/* DCMD value for the current transfer */
+} __aligned(64);
+#else
+struct mmp_pdma_desc_hw {
+	u32 ddadr;	/* Points to the next descriptor + flags */
+	u32 dsadr;	/* DSADR value for the current transfer */
+	u32 dtadr;	/* DTADR value for the current transfer */
+	u32 dcmd;	/* DCMD value for the current transfer */
+} __aligned(32);
+#endif
+
+struct mmp_pdma_desc_sw {
+	struct mmp_pdma_desc_hw desc;
+	struct list_head node;
+	struct list_head tx_list;
+	struct dma_async_tx_descriptor async_tx;
+};
+
+struct mmp_pdma_phy;
+
+struct mmp_pdma_chan {
+	struct device *dev;
+	struct dma_chan chan;
+	struct dma_async_tx_descriptor desc;
+	struct mmp_pdma_phy *phy;
+	enum dma_transfer_direction dir;
+	struct dma_slave_config slave_config;
+
+	struct mmp_pdma_desc_sw *cyclic_first;	/* first desc_sw if channel
+						 * is in cyclic mode */
+
+	/* channel's basic info */
+	struct tasklet_struct tasklet;
+	u32 dcmd;
+	u32 drcmr;
+	u32 dev_addr;
+
+	/* list for desc */
+	spinlock_t desc_lock;		/* Descriptor list lock */
+	struct list_head chain_pending;	/* Link descriptors queue for pending */
+	struct list_head chain_running;	/* Link descriptors queue for running */
+	bool idle;			/* channel statue machine */
+	bool byte_align;
+
+	int user_do_qos;
+	int qos_count; /* Per-channel qos count */
+	enum dma_status status; /* channel state machine */
+	u32 bytes_residue;
+
+	struct dma_pool *desc_pool;	/* Descriptors pool */
+};
+
+struct mmp_pdma_phy {
+	int idx;
+	void __iomem *base;
+	struct mmp_pdma_chan *vchan;
+};
+
+struct reserved_chan{
+	int	chan_id;
+	int	drcmr;
+};
+
+struct mmp_pdma_device {
+	int				dma_channels;
+	int				nr_reserved_channels;
+	struct reserved_chan		*reserved_channels;
+	s32				lpm_qos;
+	struct clk			*clk;
+	struct reset_control		*resets;
+	int				max_burst_size;
+	void __iomem			*base;
+	struct device			*dev;
+	struct dma_device		device;
+	struct mmp_pdma_phy		*phy;
+	spinlock_t phy_lock; /* protect alloc/free phy channels */
+};
+
+#define tx_to_mmp_pdma_desc(tx)					\
+	container_of(tx, struct mmp_pdma_desc_sw, async_tx)
+#define to_mmp_pdma_desc(lh)					\
+	container_of(lh, struct mmp_pdma_desc_sw, node)
+#define to_mmp_pdma_chan(dchan)					\
+	container_of(dchan, struct mmp_pdma_chan, chan)
+#define to_mmp_pdma_dev(dmadev)					\
+	container_of(dmadev, struct mmp_pdma_device, device)
+
+static void mmp_pdma_qos_put(struct mmp_pdma_chan *chan);
+
+#define QSPI_PHY_CHAN	15
+
+static int mmp_pdma_config_write(struct dma_chan *dchan,
+			   struct dma_slave_config *cfg,
+			   enum dma_transfer_direction direction);
+
+static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr)
+{
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+	u32 ddadrh;
+#endif
+	u32 reg = (phy->idx << 4) + DDADR;
+
+	writel(addr & 0xffffffff, phy->base + reg);
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+	/* config higher bits for desc address */
+	ddadrh = (addr >> 32);
+	writel(ddadrh, phy->base + DDADRH(phy->idx));
+#endif
+}
+
+static void enable_chan(struct mmp_pdma_phy *phy)
+{
+	u32 reg, dalgn;
+	u32 dcsr;
+	unsigned long flags;
+	struct mmp_pdma_device *pdev;
+
+	if (phy == NULL)
+		return;
+
+	if (!phy->vchan)
+		return;
+
+	pdev = to_mmp_pdma_dev(phy->vchan->chan.device);
+
+	spin_lock_irqsave(&pdev->phy_lock, flags);
+
+	reg = DRCMR(phy->vchan->drcmr);
+	writel(DRCMR_MAPVLD | phy->idx, phy->base + reg);
+
+	dalgn = readl(phy->base + DALGN);
+	if (phy->vchan->byte_align)
+		dalgn |= 1 << phy->idx;
+	else
+		dalgn &= ~(1 << phy->idx);
+	writel(dalgn, phy->base + DALGN);
+
+	reg = (phy->idx << 2) + DCSR;
+
+	dcsr = readl(phy->base + reg);
+	dcsr |= (DCSR_RUN | DCSR_EORIRQEN | DCSR_EORSTOPEN);
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+	/* use long descriptor mode: set DCSR_LPAEEN bit */
+	dcsr |= DCSR_LPAEEN;
+#endif
+	writel(dcsr, phy->base + reg);
+
+	spin_unlock_irqrestore(&pdev->phy_lock, flags);
+}
+
+static void disable_chan(struct mmp_pdma_phy *phy)
+{
+	u32 reg;
+	u32 dcsr, cnt = 1000;
+
+	if (!phy)
+		return;
+
+	reg = (phy->idx << 2) + DCSR;
+
+	dcsr = readl(phy->base + reg);
+	dcsr &= ~(DCSR_RUN | DCSR_EORIRQEN | DCSR_EORSTOPEN);
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+	/* use long descriptor mode: set DCSR_LPAEEN bit */
+	dcsr &= ~DCSR_LPAEEN;
+#endif
+	writel(dcsr, phy->base + reg);
+
+	/* ensure dma is stopped. */
+	dcsr = readl(phy->base + reg);
+	while (!(dcsr & (0x1 << 3)) && --cnt) {
+		udelay(10);
+		dcsr = readl(phy->base + reg);
+	}
+
+	WARN_ON(!cnt);
+}
+
+static int clear_chan_irq(struct mmp_pdma_phy *phy)
+{
+	u32 dcsr;
+	u32 dint = readl(phy->base + DINT);
+	u32 reg = (phy->idx << 2) + DCSR;
+
+	if (!(dint & BIT(phy->idx)))
+		return -EAGAIN;
+
+	/* clear irq */
+	dcsr = readl(phy->base + reg);
+	writel(dcsr, phy->base + reg);
+	if ((dcsr & DCSR_BUSERR) && (phy->vchan))
+		dev_warn(phy->vchan->dev, "DCSR_BUSERR\n");
+
+	return 0;
+}
+
+static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id)
+{
+	struct mmp_pdma_phy *phy = dev_id;
+	struct mmp_pdma_chan *pchan = phy->vchan;
+
+	if (clear_chan_irq(phy) != 0)
+		return IRQ_NONE;
+
+	if (pchan)
+		tasklet_schedule(&pchan->tasklet);
+
+	return IRQ_HANDLED;
+}
+
+static bool is_channel_reserved(struct mmp_pdma_device *pdev, int chan_id)
+{
+	int i;
+
+	for (i = 0; i < pdev->nr_reserved_channels; i++) {
+		if (chan_id == pdev->reserved_channels[i].chan_id)
+			return true;
+	}
+
+	return false;
+}
+
+static struct mmp_pdma_phy * lookup_phy_for_drcmr(struct mmp_pdma_device *pdev, int drcmr)
+{
+	int i;
+	int chan_id;
+	struct mmp_pdma_phy *phy;
+
+	for (i = 0; i < pdev->nr_reserved_channels; i++) {
+		if (drcmr == pdev->reserved_channels[i].drcmr) {
+			chan_id = pdev->reserved_channels[i].chan_id;
+			phy = &pdev->phy[chan_id];
+			return phy;
+		}
+	}
+
+	return NULL;
+}
+
+static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id)
+{
+	struct mmp_pdma_device *pdev = dev_id;
+	struct mmp_pdma_phy *phy;
+	u32 dint = readl(pdev->base + DINT);
+	int i, ret;
+	int irq_num = 0;
+	unsigned long flags;
+
+	while (dint) {
+		i = __ffs(dint);
+		/* only handle interrupts belonging to pdma driver*/
+		if (i >= pdev->dma_channels)
+			break;
+
+		dint &= (dint - 1);
+		phy = &pdev->phy[i];
+		spin_lock_irqsave(&pdev->phy_lock, flags);
+
+		ret = mmp_pdma_chan_handler(irq, phy);
+
+		spin_unlock_irqrestore(&pdev->phy_lock, flags);
+		if (ret == IRQ_HANDLED)
+			irq_num++;
+	}
+
+	if (irq_num)
+		return IRQ_HANDLED;
+
+	return IRQ_NONE;
+}
+
+/* lookup free phy channel as descending priority */
+static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan)
+{
+	int prio, i;
+	struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
+	struct mmp_pdma_phy *phy, *found = NULL;
+	unsigned long flags;
+
+	/*
+	 * dma channel priorities
+	 * ch 0 - 3,  16 - 19  <--> (0)
+	 * ch 4 - 7,  20 - 23  <--> (1)
+	 * ch 8 - 11, 24 - 27  <--> (2)
+	 * ch 12 - 15, 28 - 31  <--> (3)
+	 */
+
+	spin_lock_irqsave(&pdev->phy_lock, flags);
+
+	phy = lookup_phy_for_drcmr(pdev, pchan->drcmr);
+
+	if (phy != NULL) {
+		if (!phy->vchan) {
+			phy->vchan = pchan;
+			found = phy;
+		}
+
+		goto out_unlock;
+	}
+
+	for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) {
+		for (i = 0; i < pdev->dma_channels; i++) {
+			if (prio != (i & 0xf) >> 2)
+				continue;
+
+			if (is_channel_reserved(pdev, i))
+				continue;
+			phy = &pdev->phy[i];
+			if (!phy->vchan) {
+				phy->vchan = pchan;
+				found = phy;
+				goto out_unlock;
+			}
+		}
+	}
+
+out_unlock:
+	spin_unlock_irqrestore(&pdev->phy_lock, flags);
+	return found;
+}
+
+static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
+{
+	struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
+	unsigned long flags;
+	u32 reg;
+
+	if (!pchan->phy)
+		return;
+
+	/* clear the channel mapping in DRCMR */
+	reg = DRCMR(pchan->drcmr);
+	writel(0, pchan->phy->base + reg);
+
+	spin_lock_irqsave(&pdev->phy_lock, flags);
+	pchan->phy->vchan = NULL;
+	pchan->phy = NULL;
+
+	spin_unlock_irqrestore(&pdev->phy_lock, flags);
+}
+
+/*
+ * start_pending_queue - transfer any pending transactions
+ * pending list ==> running list
+ */
+static void start_pending_queue(struct mmp_pdma_chan *chan)
+{
+	struct mmp_pdma_desc_sw *desc;
+	struct mmp_pdma_desc_sw *_desc;
+
+	/* still in running, irq will start the pending list */
+	if (chan->status == DMA_IN_PROGRESS) {
+		dev_dbg(chan->dev, "DMA controller still busy\n");
+		return;
+	}
+
+	if (list_empty(&chan->chain_pending)) {
+		/* chance to re-fetch phy channel with higher prio */
+		mmp_pdma_free_phy(chan);
+		dev_dbg(chan->dev, "no pending list\n");
+		return;
+	}
+
+	if (!chan->phy) {
+		chan->phy = lookup_phy(chan);
+		if (!chan->phy) {
+			dev_dbg(chan->dev, "no free dma channel\n");
+			return;
+		}
+	}
+
+	/*
+	 * pending -> running
+	 * reintilize pending list
+	 */
+	list_for_each_entry_safe(desc, _desc, &chan->chain_pending, node) {
+		list_del(&desc->node);
+		list_add_tail(&desc->node, &chan->chain_running);
+		if (desc->desc.ddadr & DDADR_STOP)
+			break;
+	}
+
+	desc = list_first_entry(&chan->chain_running,
+				struct mmp_pdma_desc_sw, node);
+
+	/*
+	 * Program the descriptor's address into the DMA controller,
+	 * then start the DMA transaction
+	 */
+	set_desc(chan->phy, desc->async_tx.phys);
+	enable_chan(chan->phy);
+	chan->idle = false;
+	chan->status = DMA_IN_PROGRESS;
+	chan->bytes_residue = 0;
+}
+
+
+/* desc->tx_list ==> pending list */
+static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan);
+	struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx);
+	struct mmp_pdma_desc_sw *child;
+	unsigned long flags;
+	dma_cookie_t cookie = -EBUSY;
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+
+	list_for_each_entry(child, &desc->tx_list, node) {
+		cookie = dma_cookie_assign(&child->async_tx);
+	}
+
+	/* softly link to pending list - desc->tx_list ==> pending list */
+	list_splice_tail_init(&desc->tx_list, &chan->chain_pending);
+
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+	return cookie;
+}
+
+static struct mmp_pdma_desc_sw *
+mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan)
+{
+	struct mmp_pdma_desc_sw *desc;
+	dma_addr_t pdesc;
+
+	desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
+	if (!desc) {
+		dev_err(chan->dev, "out of memory for link descriptor\n");
+		return NULL;
+	}
+
+	INIT_LIST_HEAD(&desc->tx_list);
+	dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
+	/* each desc has submit */
+	desc->async_tx.tx_submit = mmp_pdma_tx_submit;
+	desc->async_tx.phys = pdesc;
+
+	return desc;
+}
+
+/*
+ * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel.
+ *
+ * This function will create a dma pool for descriptor allocation.
+ * Request irq only when channel is requested
+ * Return - The number of allocated descriptors.
+ */
+
+static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+
+	if (chan->desc_pool)
+		return 1;
+
+	chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device),
+					  chan->dev,
+					  sizeof(struct mmp_pdma_desc_sw),
+					  __alignof__(struct mmp_pdma_desc_sw),
+					  0);
+	if (!chan->desc_pool) {
+		dev_err(chan->dev, "unable to allocate descriptor pool\n");
+		return -ENOMEM;
+	}
+
+	chan->status = DMA_COMPLETE;
+	chan->dir = 0;
+	chan->dcmd = 0;
+
+	mmp_pdma_free_phy(chan);
+
+	chan->idle = true;
+	chan->dev_addr = 0;
+	return 1;
+}
+
+static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan,
+				    struct list_head *list)
+{
+	struct mmp_pdma_desc_sw *desc, *_desc;
+
+	list_for_each_entry_safe(desc, _desc, list, node) {
+		list_del(&desc->node);
+		dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
+	}
+}
+
+static void mmp_pdma_free_chan_resources(struct dma_chan *dchan)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+	unsigned long flags;
+
+	/* wait until task ends if necessary */
+	tasklet_kill(&chan->tasklet);
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	mmp_pdma_free_desc_list(chan, &chan->chain_pending);
+	mmp_pdma_free_desc_list(chan, &chan->chain_running);
+
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+	dma_pool_destroy(chan->desc_pool);
+	chan->desc_pool = NULL;
+	chan->idle = true;
+	chan->dev_addr = 0;
+
+	chan->status = DMA_COMPLETE;
+	chan->dir = 0;
+	chan->dcmd = 0;
+
+	mmp_pdma_free_phy(chan);
+	return;
+}
+
+#define INVALID_BURST_SETTING	-1
+#define DEFAULT_MAX_BURST_SIZE	32
+
+static int get_max_burst_setting(unsigned int max_burst_size)
+{
+	switch (max_burst_size) {
+	case 8:
+		return DCMD_BURST8;
+	case 16:
+		return DCMD_BURST16;
+	case 32:
+		return DCMD_BURST32;
+	case 64:
+		return DCMD_BURST64;
+	default:
+		return INVALID_BURST_SETTING;
+	}
+}
+
+static struct dma_async_tx_descriptor *
+mmp_pdma_prep_memcpy(struct dma_chan *dchan,
+		     dma_addr_t dma_dst, dma_addr_t dma_src,
+		     size_t len, unsigned long flags)
+{
+	struct mmp_pdma_chan *chan;
+	struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
+	size_t copy = 0;
+	struct mmp_pdma_device *dev;
+	int value;
+
+	if (!dchan)
+		return NULL;
+
+	if (!len)
+		return NULL;
+
+	chan = to_mmp_pdma_chan(dchan);
+	chan->byte_align = false;
+
+	if (!chan->dir) {
+		chan->dir = DMA_MEM_TO_MEM;
+		chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR;
+		dev = to_mmp_pdma_dev(dchan->device);
+		value = get_max_burst_setting(dev->max_burst_size);
+
+		BUG_ON(value == INVALID_BURST_SETTING);
+
+		chan->dcmd |= value;
+	}
+
+	do {
+		/* Allocate the link descriptor from DMA pool */
+		new = mmp_pdma_alloc_descriptor(chan);
+		if (!new) {
+			dev_err(chan->dev, "no memory for desc\n");
+			goto fail;
+		}
+
+		copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES);
+		if (dma_src & 0x7 || dma_dst & 0x7)
+			chan->byte_align = true;
+
+		new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy);
+
+		/*
+		 * Check whether descriptor/source-addr/target-addr is in
+		 * region higher than 4G. If so, set related higher bits to 1.
+		 */
+		if (chan->dir == DMA_MEM_TO_DEV) {
+			new->desc.dsadr = dma_src & 0xffffffff;
+			new->desc.dtadr = dma_dst;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+			new->desc.dsadrh = (dma_src >> 32);
+			new->desc.dtadrh = 0;
+#endif
+		} else if (chan->dir == DMA_DEV_TO_MEM) {
+			new->desc.dsadr = dma_src;
+			new->desc.dtadr = dma_dst & 0xffffffff;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+			new->desc.dsadrh = 0;
+			new->desc.dtadrh = (dma_dst >> 32);
+#endif
+		} else if (chan->dir == DMA_MEM_TO_MEM) {
+			new->desc.dsadr = dma_src & 0xffffffff;
+			new->desc.dtadr = dma_dst & 0xffffffff;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+			new->desc.dsadrh = (dma_src >> 32);
+			new->desc.dtadrh = (dma_dst >> 32);
+#endif
+		} else {
+			dev_err(chan->dev, "wrong direction: 0x%x\n", chan->dir);
+			goto fail;
+		}
+
+		if (!first)
+			first = new;
+		else {
+			prev->desc.ddadr = new->async_tx.phys;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+			prev->desc.ddadrh = (new->async_tx.phys >> 32);
+#endif
+		}
+
+		new->async_tx.cookie = 0;
+		async_tx_ack(&new->async_tx);
+
+		prev = new;
+		len -= copy;
+
+		if (chan->dir == DMA_MEM_TO_DEV) {
+			dma_src += copy;
+		} else if (chan->dir == DMA_DEV_TO_MEM) {
+			dma_dst += copy;
+		} else if (chan->dir == DMA_MEM_TO_MEM) {
+			dma_src += copy;
+			dma_dst += copy;
+		}
+
+		/* Insert the link descriptor to the LD ring */
+		list_add_tail(&new->node, &first->tx_list);
+	} while (len);
+
+	first->async_tx.flags = flags; /* client is in control of this ack */
+	first->async_tx.cookie = -EBUSY;
+
+	/* last desc and fire IRQ */
+	new->desc.ddadr = DDADR_STOP;
+	new->desc.dcmd |= DCMD_ENDIRQEN;
+
+	chan->cyclic_first = NULL;
+
+	return &first->async_tx;
+
+fail:
+	if (first)
+		mmp_pdma_free_desc_list(chan, &first->tx_list);
+	return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
+		       unsigned int sg_len, enum dma_transfer_direction dir,
+		       unsigned long flags, void *context)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+	struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL;
+	size_t len, avail;
+	struct scatterlist *sg;
+	dma_addr_t addr;
+	int i;
+
+	if ((sgl == NULL) || (sg_len == 0))
+		return NULL;
+
+	chan->byte_align = true;
+
+	mmp_pdma_config_write(dchan, &chan->slave_config, dir);
+
+	for_each_sg(sgl, sg, sg_len, i) {
+		addr = sg_dma_address(sg);
+		avail = sg_dma_len(sgl);
+
+		do {
+			len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
+			if (addr & 0x7)
+				chan->byte_align = true;
+
+			/* allocate and populate the descriptor */
+			new = mmp_pdma_alloc_descriptor(chan);
+			if (!new) {
+				dev_err(chan->dev, "no memory for desc\n");
+				goto fail;
+			}
+
+			new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len);
+
+			/*
+			 * Check whether descriptor/source-addr/target-addr is in
+			 * region higher than 4G. If so, set related higher bits to 1.
+			 */
+			if (dir == DMA_MEM_TO_DEV) {
+				new->desc.dsadr = addr & 0xffffffff;
+				new->desc.dtadr = chan->dev_addr;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+				new->desc.dsadrh = (addr >> 32);
+				new->desc.dtadrh = 0;
+#endif
+			} else if (dir == DMA_DEV_TO_MEM) {
+				new->desc.dsadr = chan->dev_addr;
+				new->desc.dtadr = addr & 0xffffffff;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+				new->desc.dsadrh = 0;
+				new->desc.dtadrh = (addr >> 32);
+#endif
+			} else {
+				dev_err(chan->dev, "wrong direction: 0x%x\n", chan->dir);
+				goto fail;
+			}
+
+			if (!first)
+				first = new;
+			else {
+				prev->desc.ddadr = new->async_tx.phys;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+				prev->desc.ddadrh = (new->async_tx.phys >> 32);
+#endif
+			}
+
+			new->async_tx.cookie = 0;
+			async_tx_ack(&new->async_tx);
+			prev = new;
+
+			/* Insert the link descriptor to the LD ring */
+			list_add_tail(&new->node, &first->tx_list);
+
+			/* update metadata */
+			addr += len;
+			avail -= len;
+		} while (avail);
+	}
+
+	first->async_tx.cookie = -EBUSY;
+	first->async_tx.flags = flags;
+
+	/* last desc and fire IRQ */
+	new->desc.ddadr = DDADR_STOP;
+	new->desc.dcmd |= DCMD_ENDIRQEN;
+
+	chan->dir = dir;
+	chan->cyclic_first = NULL;
+
+	return &first->async_tx;
+
+fail:
+	if (first)
+		mmp_pdma_free_desc_list(chan, &first->tx_list);
+	return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan,
+			 dma_addr_t buf_addr, size_t len, size_t period_len,
+			 enum dma_transfer_direction direction,
+			 unsigned long flags)
+{
+	struct mmp_pdma_chan *chan;
+	struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
+	dma_addr_t dma_src, dma_dst;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+	dma_addr_t dma_srch, dma_dsth;
+#endif
+
+	if (!dchan || !len || !period_len)
+		return NULL;
+
+	/* the buffer length must be a multiple of period_len */
+	if (len % period_len != 0)
+		return NULL;
+
+	if (period_len > PDMA_MAX_DESC_BYTES)
+		return NULL;
+
+	chan = to_mmp_pdma_chan(dchan);
+	mmp_pdma_config_write(dchan, &chan->slave_config, direction);
+
+	switch (direction) {
+	case DMA_MEM_TO_DEV:
+		dma_src = buf_addr & 0xffffffff;
+		dma_dst = chan->dev_addr;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+		dma_srch = (buf_addr >> 32);
+		dma_dsth = 0;
+#endif
+		break;
+	case DMA_DEV_TO_MEM:
+		dma_dst = buf_addr & 0xffffffff;
+		dma_src = chan->dev_addr;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+		dma_dsth = (buf_addr >> 32);
+		dma_srch = 0;
+#endif
+		break;
+	default:
+		dev_err(chan->dev, "Unsupported direction for cyclic DMA\n");
+		return NULL;
+	}
+
+	chan->dir = direction;
+
+	do {
+		/* Allocate the link descriptor from DMA pool */
+		new = mmp_pdma_alloc_descriptor(chan);
+		if (!new) {
+			dev_err(chan->dev, "no memory for desc\n");
+			goto fail;
+		}
+
+		new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN |
+				  (DCMD_LENGTH & period_len));
+		new->desc.dsadr = dma_src;
+		new->desc.dtadr = dma_dst;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+		new->desc.dsadrh = dma_dsth;
+		new->desc.dtadrh = dma_srch;
+#endif
+
+		if (!first)
+			first = new;
+		else {
+			prev->desc.ddadr = new->async_tx.phys;
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+			prev->desc.ddadrh = (new->async_tx.phys >> 32);
+#endif
+		}
+
+		new->async_tx.cookie = 0;
+		async_tx_ack(&new->async_tx);
+
+		prev = new;
+		len -= period_len;
+
+		if (chan->dir == DMA_MEM_TO_DEV)
+			dma_src += period_len;
+		else
+			dma_dst += period_len;
+
+		/* Insert the link descriptor to the LD ring */
+		list_add_tail(&new->node, &first->tx_list);
+	} while (len);
+
+	first->async_tx.flags = flags; /* client is in control of this ack */
+	first->async_tx.cookie = -EBUSY;
+
+	/* make the cyclic link */
+	new->desc.ddadr = first->async_tx.phys;
+	chan->cyclic_first = first;
+
+	return &first->async_tx;
+
+fail:
+	if (first)
+		mmp_pdma_free_desc_list(chan, &first->tx_list);
+	return NULL;
+}
+
+static int mmp_pdma_config_write(struct dma_chan *dchan,
+			   struct dma_slave_config *cfg,
+			   enum dma_transfer_direction direction)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+	u32 maxburst = 0, addr = 0;
+	enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
+
+	if (!dchan)
+		return -EINVAL;
+
+	if (direction == DMA_DEV_TO_MEM) {
+		chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC;
+		maxburst = cfg->src_maxburst;
+		width = cfg->src_addr_width;
+		addr = cfg->src_addr;
+	} else if (direction == DMA_MEM_TO_DEV) {
+		chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG;
+		maxburst = cfg->dst_maxburst;
+		width = cfg->dst_addr_width;
+		addr = cfg->dst_addr;
+	}
+
+	if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
+		chan->dcmd |= DCMD_WIDTH1;
+	else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
+		chan->dcmd |= DCMD_WIDTH2;
+	else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES)
+		chan->dcmd |= DCMD_WIDTH4;
+
+	if (maxburst == 8)
+		chan->dcmd |= DCMD_BURST8;
+	else if (maxburst == 16)
+		chan->dcmd |= DCMD_BURST16;
+	else if (maxburst == 32)
+		chan->dcmd |= DCMD_BURST32;
+
+	chan->dir = direction;
+	chan->dev_addr = addr;
+
+	return 0;
+}
+
+static int mmp_pdma_pause_chan(struct dma_chan *dchan)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+
+	if (!chan->phy)
+		return -1;
+
+	disable_chan(chan->phy);
+	chan->status = DMA_PAUSED;
+
+	return 0;
+}
+
+static int mmp_pdma_config(struct dma_chan *dchan,
+			   struct dma_slave_config *cfg)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+
+	memcpy(&chan->slave_config, cfg, sizeof(*cfg));
+	return 0;
+}
+
+static int mmp_pdma_terminate_all(struct dma_chan *dchan)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+	unsigned long flags;
+
+	if (!dchan)
+		return -EINVAL;
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	disable_chan(chan->phy);
+	chan->status = DMA_COMPLETE;
+	mmp_pdma_free_phy(chan);
+
+	mmp_pdma_free_desc_list(chan, &chan->chain_pending);
+	mmp_pdma_free_desc_list(chan, &chan->chain_running);
+	chan->bytes_residue = 0;
+
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+	chan->idle = true;
+
+	mmp_pdma_qos_put(chan);
+
+	return 0;
+}
+
+static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
+				     dma_cookie_t cookie)
+{
+	struct mmp_pdma_desc_sw *sw;
+	u32 curr, residue = 0;
+	bool passed = false;
+	bool cyclic = chan->cyclic_first != NULL;
+
+	/*
+	 * If the channel does not have a phy pointer anymore, it has already
+	 * been completed. Therefore, its residue is 0.
+	 */
+	if (!chan->phy)
+		return chan->bytes_residue; /* special case for EORIRQEN */
+
+	if (chan->dir == DMA_DEV_TO_MEM)
+		curr = readl(chan->phy->base + DTADR(chan->phy->idx));
+	else
+		curr = readl(chan->phy->base + DSADR(chan->phy->idx));
+
+	list_for_each_entry(sw, &chan->chain_running, node) {
+		u32 start, end, len;
+
+		if (chan->dir == DMA_DEV_TO_MEM)
+			start = sw->desc.dtadr;
+		else
+			start = sw->desc.dsadr;
+
+		len = sw->desc.dcmd & DCMD_LENGTH;
+		end = start + len;
+
+		/*
+		 * 'passed' will be latched once we found the descriptor which
+		 * lies inside the boundaries of the curr pointer. All
+		 * descriptors that occur in the list _after_ we found that
+		 * partially handled descriptor are still to be processed and
+		 * are hence added to the residual bytes counter.
+		 */
+
+		if (passed) {
+			residue += len;
+		} else if (curr >= start && curr <= end) {
+			residue += end - curr;
+			passed = true;
+		}
+
+		/*
+		 * Descriptors that have the ENDIRQEN bit set mark the end of a
+		 * transaction chain, and the cookie assigned with it has been
+		 * returned previously from mmp_pdma_tx_submit().
+		 *
+		 * In case we have multiple transactions in the running chain,
+		 * and the cookie does not match the one the user asked us
+		 * about, reset the state variables and start over.
+		 *
+		 * This logic does not apply to cyclic transactions, where all
+		 * descriptors have the ENDIRQEN bit set, and for which we
+		 * can't have multiple transactions on one channel anyway.
+		 */
+		if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN))
+			continue;
+
+		if (sw->async_tx.cookie == cookie) {
+			return residue;
+		} else {
+			residue = 0;
+			passed = false;
+		}
+	}
+
+	/* We should only get here in case of cyclic transactions */
+	return residue;
+}
+
+static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan,
+					  dma_cookie_t cookie,
+					  struct dma_tx_state *txstate)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+	enum dma_status ret;
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	ret = dma_cookie_status(dchan, cookie, txstate);
+	if (likely(ret != DMA_ERROR))
+		dma_set_residue(txstate, mmp_pdma_residue(chan, cookie));
+
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+	if (ret == DMA_COMPLETE)
+		return ret;
+	else
+		return chan->status;
+}
+
+/*
+ * mmp_pdma_issue_pending - Issue the DMA start command
+ * pending list ==> running list
+ */
+static void mmp_pdma_issue_pending(struct dma_chan *dchan)
+{
+	struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
+	unsigned long flags;
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	start_pending_queue(chan);
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+}
+
+/*
+ * dma_do_tasklet
+ * Do call back
+ * Start pending list
+ */
+static void dma_do_tasklet(struct tasklet_struct *t)
+{
+	struct mmp_pdma_chan *chan = from_tasklet(chan, t, tasklet);
+	struct mmp_pdma_desc_sw *desc, *_desc;
+	LIST_HEAD(chain_cleanup);
+	unsigned long flags;
+	struct dmaengine_desc_callback cb;
+
+	/* return if this channel has been stopped */
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	if (chan->status == DMA_COMPLETE) {
+		spin_unlock_irqrestore(&chan->desc_lock, flags);
+		return;
+	}
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+	if (chan->cyclic_first) {
+		spin_lock_irqsave(&chan->desc_lock, flags);
+		desc = chan->cyclic_first;
+		dmaengine_desc_get_callback(&desc->async_tx, &cb);
+		spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+		dmaengine_desc_callback_invoke(&cb, NULL);
+
+		return;
+	}
+
+	/* submit pending list; callback for each desc; free desc */
+	spin_lock_irqsave(&chan->desc_lock, flags);
+
+	/* special for the EORIRQEN case, residue is not 0 */
+	list_for_each_entry(desc, &chan->chain_running, node) {
+		if (desc->desc.dcmd & DCMD_ENDIRQEN) {
+			chan->bytes_residue =
+				mmp_pdma_residue(chan, desc->async_tx.cookie);
+			break;
+		}
+	}
+
+	list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) {
+		/*
+		 * move the descriptors to a temporary list so we can drop
+		 * the lock during the entire cleanup operation
+		 */
+		list_move(&desc->node, &chain_cleanup);
+
+		/*
+		 * Look for the first list entry which has the ENDIRQEN flag
+		 * set. That is the descriptor we got an interrupt for, so
+		 * complete that transaction and its cookie.
+		 */
+		if (desc->desc.dcmd & DCMD_ENDIRQEN) {
+			dma_cookie_t cookie = desc->async_tx.cookie;
+			dma_cookie_complete(&desc->async_tx);
+			dev_dbg(chan->dev, "completed_cookie=%d\n", cookie);
+			break;
+		}
+	}
+
+	/*
+	 * The hardware is idle and ready for more when the
+	 * chain_running list is empty.
+	 */
+	chan->status = list_empty(&chan->chain_running) ?
+		DMA_COMPLETE : DMA_IN_PROGRESS;
+
+	/* Start any pending transactions automatically */
+	start_pending_queue(chan);
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+
+	/* Run the callback for each descriptor, in order */
+	list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) {
+		struct dma_async_tx_descriptor *txd = &desc->async_tx;
+
+		/* Remove from the list of transactions */
+		list_del(&desc->node);
+		/* Run the link descriptor callback function */
+		dmaengine_desc_get_callback(txd, &cb);
+		dmaengine_desc_callback_invoke(&cb, NULL);
+
+		dma_pool_free(chan->desc_pool, desc, txd->phys);
+	}
+}
+
+static void mmp_pdma_remove(struct platform_device *op)
+{
+	struct mmp_pdma_device *pdev = platform_get_drvdata(op);
+	struct mmp_pdma_phy *phy;
+	int i, irq = 0, irq_num = 0;
+
+	if (op->dev.of_node)
+		of_dma_controller_free(op->dev.of_node);
+
+	for (i = 0; i < pdev->dma_channels; i++) {
+		if (platform_get_irq(op, i) > 0)
+			irq_num++;
+	}
+
+	if (irq_num != pdev->dma_channels) {
+		irq = platform_get_irq(op, 0);
+		devm_free_irq(&op->dev, irq, pdev);
+	} else {
+		for (i = 0; i < pdev->dma_channels; i++) {
+			phy = &pdev->phy[i];
+			irq = platform_get_irq(op, i);
+			devm_free_irq(&op->dev, irq, phy);
+		}
+	}
+
+	dma_async_device_unregister(&pdev->device);
+
+#ifndef CONFIG_SOC_SPACEMIT_K3_FPGA
+	reset_control_assert(pdev->resets);
+	clk_disable_unprepare(pdev->clk);
+#endif
+
+	kfree(pdev->reserved_channels);
+	platform_set_drvdata(op, NULL);
+}
+
+static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq)
+{
+	struct mmp_pdma_phy *phy  = &pdev->phy[idx];
+	struct mmp_pdma_chan *chan;
+	int ret;
+
+	chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL);
+	if (chan == NULL)
+		return -ENOMEM;
+
+	phy->idx = idx;
+	phy->base = pdev->base;
+
+	if (irq) {
+		ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler,
+				       IRQF_SHARED, "pdma", phy);
+		if (ret) {
+			dev_err(pdev->dev, "channel request irq fail!\n");
+			return ret;
+		}
+	}
+
+	spin_lock_init(&chan->desc_lock);
+	chan->dev = pdev->dev;
+	chan->chan.device = &pdev->device;
+	tasklet_setup(&chan->tasklet, dma_do_tasklet);
+	INIT_LIST_HEAD(&chan->chain_pending);
+	INIT_LIST_HEAD(&chan->chain_running);
+
+	chan->status = DMA_COMPLETE;
+	chan->bytes_residue = 0;
+	chan->qos_count = 0;
+	chan->user_do_qos = 1;
+
+	/* register virt channel to dma engine */
+	list_add_tail(&chan->chan.device_node, &pdev->device.channels);
+
+	return 0;
+}
+
+static const struct of_device_id mmp_pdma_dt_ids[] = {
+	{ .compatible = "spacemit,pdma-1.0", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids);
+
+static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec,
+					   struct of_dma *ofdma)
+{
+	struct mmp_pdma_device *d = ofdma->of_dma_data;
+	struct dma_chan *chan;
+#ifdef CONFIG_PM
+	struct mmp_pdma_chan *c;
+#endif
+
+	chan = dma_get_any_slave_channel(&d->device);
+	if (!chan)
+		return NULL;
+
+	to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0];
+#ifdef CONFIG_PM
+	if (unlikely(dma_spec->args_count != 2))
+		dev_err(d->dev, "#dma-cells should be 2!\n");
+
+	c = to_mmp_pdma_chan(chan);
+	c->user_do_qos = dma_spec->args[1] ? 1 : 0;
+
+	if (c->user_do_qos)
+		dev_dbg(d->dev, "channel %d: user does qos itself\n",
+			 c->chan.chan_id);
+	else
+		dev_dbg(d->dev, "channel %d: pdma does qos\n",
+			 c->chan.chan_id);
+#endif
+
+	return chan;
+}
+
+static int mmp_pdma_probe(struct platform_device *op)
+{
+	struct mmp_pdma_device *pdev;
+	const struct of_device_id *of_id;
+	struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev);
+	int i, ret, irq = 0;
+	int dma_channels = 0, irq_num = 0;
+	const enum dma_slave_buswidth widths =
+		DMA_SLAVE_BUSWIDTH_1_BYTE   | DMA_SLAVE_BUSWIDTH_2_BYTES |
+		DMA_SLAVE_BUSWIDTH_4_BYTES;
+
+	int nr_reserved_channels;
+	const int *list;
+	unsigned int max_burst_size = DEFAULT_MAX_BURST_SIZE;
+
+	pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL);
+	if (!pdev)
+		return -ENOMEM;
+
+	pdev->dev = &op->dev;
+
+	spin_lock_init(&pdev->phy_lock);
+
+	pdev->base = devm_platform_ioremap_resource(op, 0);
+	if (IS_ERR(pdev->base))
+		return PTR_ERR(pdev->base);
+
+#ifndef CONFIG_SOC_SPACEMIT_K3_FPGA
+	pdev->clk = devm_clk_get(pdev->dev,NULL);
+	if(IS_ERR(pdev->clk))
+		return PTR_ERR(pdev->clk);
+
+	ret = clk_prepare_enable(pdev->clk);
+	if (ret)
+		return dev_err_probe(pdev->dev, ret, "could not enable dma bus clock\n");
+
+	pdev->resets = devm_reset_control_get_optional(pdev->dev,NULL);
+	if(IS_ERR(pdev->resets)) {
+		ret = PTR_ERR(pdev->resets);
+		goto err_rst;
+	}
+	ret = reset_control_deassert(pdev->resets);
+	if(ret)
+		goto err_rst;
+#endif
+
+	of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev);
+
+	if (of_id) {
+		int n;
+		of_property_read_u32(pdev->dev->of_node, "#dma-channels",
+				&dma_channels);
+
+		list = of_get_property(pdev->dev->of_node, "reserved-channels",
+			&n);
+
+		if (of_property_read_u32(pdev->dev->of_node, "max-burst-size",
+		    &max_burst_size)) {
+			dev_err(pdev->dev, "Cannot find the max-burst-size node "
+				       "in the device tree, set it to %d\n",
+				       DEFAULT_MAX_BURST_SIZE);
+			max_burst_size = DEFAULT_MAX_BURST_SIZE;
+		}
+
+		if (get_max_burst_setting(max_burst_size) == INVALID_BURST_SETTING) {
+			dev_err(pdev->dev, "Unsupported max-burst-size value %d "
+				       "in the device tree, set it to %d\n",
+					max_burst_size, DEFAULT_MAX_BURST_SIZE);
+			max_burst_size = DEFAULT_MAX_BURST_SIZE;
+		}
+
+		if (list) {
+			int num_args = 2;
+
+			nr_reserved_channels = n / (sizeof(u32) * num_args);
+
+			pdev->nr_reserved_channels = nr_reserved_channels;
+
+			pdev->reserved_channels = kzalloc(nr_reserved_channels * sizeof(struct reserved_chan),
+							GFP_KERNEL);
+
+			if (pdev->reserved_channels == NULL)
+				return -ENOMEM;
+
+			for (i = 0; i < nr_reserved_channels; i++) {
+				int value;
+
+				of_property_read_u32_index(pdev->dev->of_node, "reserved-channels", i * num_args, &value);
+				pdev->reserved_channels[i].chan_id = value;
+				of_property_read_u32_index(pdev->dev->of_node, "reserved-channels", i * num_args + 1, &value);
+				pdev->reserved_channels[i].drcmr   = value;
+			}
+		}
+	} else if (pdata && pdata->dma_channels) {
+		dma_channels = pdata->dma_channels;
+	} else {
+		dma_channels = 32;	/* default 32 channel */
+	}
+	pdev->dma_channels = dma_channels;
+
+	pdev->max_burst_size = max_burst_size;
+	dev_dbg(pdev->dev, "set max burst size to %d\n", max_burst_size);
+
+#ifdef CONFIG_PM
+	pm_runtime_enable(&op->dev);
+	/*
+	 * We can't ensure the pm operations are always in non-atomic context.
+	 * Actually it depends on the drivers' behavior. So mark it as irq safe.
+	 */
+	pm_runtime_irq_safe(&op->dev);
+#endif
+	for (i = 0; i < dma_channels; i++) {
+		if (platform_get_irq_optional(op, i) > 0)
+			irq_num++;
+	}
+
+	pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy),
+				 GFP_KERNEL);
+	if (pdev->phy == NULL)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&pdev->device.channels);
+
+	if (irq_num != dma_channels) {
+		/* all chan share one irq, demux inside */
+		irq = platform_get_irq(op, 0);
+		ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler,
+				       IRQF_SHARED, "pdma", pdev);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < dma_channels; i++) {
+		irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i);
+		ret = mmp_pdma_chan_init(pdev, i, irq);
+		if (ret)
+			return ret;
+	}
+
+	dma_cap_set(DMA_SLAVE, pdev->device.cap_mask);
+	dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask);
+	dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask);
+	dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask);
+	pdev->device.dev = &op->dev;
+	pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources;
+	pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources;
+	pdev->device.device_tx_status = mmp_pdma_tx_status;
+	pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy;
+	pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg;
+	pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic;
+	pdev->device.device_issue_pending = mmp_pdma_issue_pending;
+	pdev->device.device_config = mmp_pdma_config;
+	pdev->device.device_pause = mmp_pdma_pause_chan;
+	pdev->device.device_terminate_all = mmp_pdma_terminate_all;
+	pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
+	pdev->device.src_addr_widths = widths;
+	pdev->device.dst_addr_widths = widths;
+	pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
+	pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
+
+#ifdef CONFIG_SPACEMIT_PDMA_SUPPORT_64BIT
+	dma_set_mask(pdev->dev, DMA_BIT_MASK(64));
+#else
+	dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask);
+#endif
+
+	ret = dma_async_device_register(&pdev->device);
+	if (ret) {
+		dev_err(pdev->device.dev, "unable to register\n");
+		return ret;
+	}
+
+	if (op->dev.of_node) {
+		/* Device-tree DMA controller registration */
+		ret = of_dma_controller_register(op->dev.of_node,
+						 mmp_pdma_dma_xlate, pdev);
+		if (ret < 0) {
+			dev_err(&op->dev, "of_dma_controller_register failed\n");
+			dma_async_device_unregister(&pdev->device);
+			return ret;
+		}
+	}
+
+	platform_set_drvdata(op, pdev);
+	dev_dbg(pdev->device.dev, "initialized %d channels\n", dma_channels);
+	return 0;
+
+#ifndef CONFIG_SOC_SPACEMIT_K3_FPGA
+err_rst:
+	clk_disable_unprepare(pdev->clk);
+#endif
+	return ret;
+}
+
+static void mmp_pdma_qos_put(struct mmp_pdma_chan *chan)
+{
+	unsigned long flags;
+
+	if (chan->user_do_qos)
+		return;
+
+	spin_lock_irqsave(&chan->desc_lock, flags);
+	if (chan->qos_count == 1) {
+		chan->qos_count = 0;
+		pm_runtime_put_autosuspend(chan->dev);
+	}
+
+	spin_unlock_irqrestore(&chan->desc_lock, flags);
+}
+
+static const struct platform_device_id mmp_pdma_id_table[] = {
+	{ "mmp-pdma", },
+	{ },
+};
+
+#ifdef CONFIG_PM_SLEEP
+static int mmp_pdma_suspend_noirq(struct device *dev)
+{
+	struct mmp_pdma_device *pdev = dev_get_drvdata(dev);
+
+#ifndef CONFIG_SOC_SPACEMIT_K3_FPGA
+	clk_disable_unprepare(pdev->clk);
+#endif
+
+	return 0;
+}
+
+static int mmp_pdma_resume_noirq(struct device *dev)
+{
+	struct mmp_pdma_device *pdev = dev_get_drvdata(dev);
+
+#ifndef CONFIG_SOC_SPACEMIT_K3_FPGA
+	clk_prepare_enable(pdev->clk);
+#endif
+
+	return 0;
+}
+
+static const struct dev_pm_ops spacemit_mmp_pdma_pm_qos = {
+	NOIRQ_SYSTEM_SLEEP_PM_OPS(mmp_pdma_suspend_noirq, mmp_pdma_resume_noirq)
+};
+#endif
+
+static struct platform_driver spacemit_mmp_pdma_driver = {
+	.driver		= {
+		.name	= "mmp-pdma",
+#ifdef CONFIG_PM_SLEEP
+		.pm	= &spacemit_mmp_pdma_pm_qos,
+#endif
+		.of_match_table = mmp_pdma_dt_ids,
+	},
+	.id_table	= mmp_pdma_id_table,
+	.probe		= mmp_pdma_probe,
+	.remove_new	= mmp_pdma_remove,
+};
+
+static int __init mmp_pdma_init(void)
+{
+	return platform_driver_register(&spacemit_mmp_pdma_driver);
+}
+
+static void __exit mmp_pdma_exit(void)
+{
+	platform_driver_unregister(&spacemit_mmp_pdma_driver);
+}
+
+subsys_initcall(mmp_pdma_init);
+module_exit(mmp_pdma_exit);
+
+MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver");
+MODULE_AUTHOR("Marvell International Ltd.");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/udma.c b/drivers/dma/udma.c
new file mode 100644
index 000000000000..8fafd61a574d
--- /dev/null
+++ b/drivers/dma/udma.c
@@ -0,0 +1,470 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/sched.h>
+#include <linux/device.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/uaccess.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+
+#define USE_DMA_MALLOC
+// #define DMA_CONFIG_DEBUG
+
+#define DEVICE_NAME	"udma"
+#define IOC_MAGIC	'c'
+#define DMA_MEMCPY_CMD	_IOR(IOC_MAGIC, 0, int)
+#define DMA_VA_TO_PA	_IOR(IOC_MAGIC, 1, int)
+
+static int dma_major;
+static struct class *dma_class;
+static struct dma_device *dma_dev;
+static struct dma_chan *dma_chan;
+static struct dma_async_tx_descriptor *dma_tx;
+static struct list_head dmabuf_list;
+static struct completion dma_m2m_ok;
+static struct mutex dma_mutex;
+
+typedef struct {
+	void			*src;
+	void			*dst;
+	size_t			size;
+#ifdef DMA_CONFIG_DEBUG
+	long long		time[10];
+	int			time_cnt;
+	int			dma_irq_subscript;
+#endif
+} memcpy_msg_t;
+
+typedef struct {
+	void			*user_addr;
+	void			*dma_addr;
+} va_to_pa_msg_t;
+
+typedef struct {
+	size_t			size;		// Size of the buffer
+	unsigned long		user_addr;	// User virtual address of the buffer
+	void			*kern_addr;	// Kernel virtual address of the buffer
+	dma_addr_t		dma_addr;	// DMA bus address of the buffer
+	struct list_head	list;		// List node pointers for dma alloc list
+} dma_map_info_t;
+
+typedef struct {
+	dma_addr_t		addr;
+	size_t			size;
+	int			dirty;
+} va2pa_t;
+
+#ifdef DMA_CONFIG_DEBUG
+#include <linux/time.h>
+
+#define DMA_TIME_STAMP()			\
+	do {					\
+		g_time[g_time_cnt++] = getus();	\
+	} while (0)
+
+static volatile int g_time_cnt;
+static volatile long long g_time[10];
+static int g_dma_irq_subscript;
+
+static long long getus(void)
+{
+	return ktime_to_us(ktime_get());
+}
+#else
+#define DMA_TIME_STAMP()
+#endif
+
+static void dma_callback_func(void *priv)
+{
+#ifdef DMA_CONFIG_DEBUG
+	g_dma_irq_subscript = g_time_cnt;
+#endif
+	DMA_TIME_STAMP();
+	complete(&dma_m2m_ok);
+}
+
+static int dma_open(struct inode *inode, struct file *filp)
+{
+	return dma_dev ? 0 : -EPERM;
+}
+
+static int dma_release(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static ssize_t dma_read(struct file *filp, char __user *buf, size_t size, loff_t *ppos)
+{
+	return size;
+}
+
+static ssize_t dma_write(struct file *filp, const char __user *buf, size_t size, loff_t *ppos)
+{
+	return size;
+}
+
+#ifdef USE_DMA_MALLOC
+static int dma_malloc(dma_map_info_t *dma_info, struct vm_area_struct *vma)
+{
+	int ret;
+
+	dma_info->kern_addr = dma_alloc_coherent(dma_dev->dev, dma_info->size,
+						 &dma_info->dma_addr, GFP_KERNEL);
+	if (!dma_info->kern_addr) {
+		dev_err(dma_dev->dev,
+			"Unable to allocate contiguous DMA memory region of size %zu\n",
+			dma_info->size);
+		return -ENOMEM;
+	}
+
+	ret = dma_mmap_coherent(dma_dev->dev, vma, dma_info->kern_addr,
+				dma_info->dma_addr, dma_info->size);
+	if (ret < 0) {
+		dev_err(dma_dev->dev,
+			"Unable to remap address %p to userspace address 0x%lx, size %zu\n",
+			dma_info->kern_addr, dma_info->user_addr, dma_info->size);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dma_free(dma_map_info_t *dma_info)
+{
+	dma_free_coherent(dma_dev->dev, dma_info->size, dma_info->kern_addr, dma_info->dma_addr);
+
+	return 0;
+}
+#else
+static int kernel_malloc(dma_map_info_t *dma_info, struct vm_area_struct *vma)
+{
+	dma_info->kern_addr = kmalloc(dma_info->size, GFP_KERNEL);
+	if (!dma_info->kern_addr) {
+		dev_err(dma_dev->dev, "kmalloc failed\n");
+		return -ENOMEM;
+	}
+
+	if (remap_pfn_range(vma, vma->vm_start,
+			    (virt_to_phys(dma_info->kern_addr) >> PAGE_SHIFT),
+			    vma->vm_end - vma->vm_start, vma->vm_page_prot)) {
+		return -EAGAIN;
+	}
+
+	dma_info->dma_addr = dma_map_single(dma_dev->dev, dma_info->kern_addr,
+					    dma_info->size, DMA_FROM_DEVICE);
+	if (dma_mapping_error(dma_dev->dev, dma_info->dma_addr)) {
+		dev_err(dma_dev->dev, "mapping buffer failed\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int kernel_free(dma_map_info_t *dma_info)
+{
+	dma_unmap_single(dma_dev->dev, dma_info->dma_addr, dma_info->size, DMA_FROM_DEVICE);
+	kfree(dma_info->kern_addr);
+
+	return 0;
+}
+#endif
+
+static void dma_vma_close(struct vm_area_struct *vma)
+{
+	dma_map_info_t *dma_info;
+
+	dma_info = vma->vm_private_data;
+#ifdef USE_DMA_MALLOC
+	dma_free(dma_info);
+#else
+	kernel_free(dma_info);
+#endif
+	kfree(dma_info);
+}
+
+static const struct vm_operations_struct dma_vm_ops = {
+	.close = dma_vma_close,
+};
+
+static int dma_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	int		ret;
+	dma_map_info_t *dma_info;
+
+	dma_info = kmalloc(sizeof(*dma_info), GFP_KERNEL);
+	if (!dma_info) {
+		dev_err(dma_dev->dev, "Unable to allocate VMA data structure\n");
+		return -ENOMEM;
+	}
+
+	dma_info->size		= vma->vm_end - vma->vm_start;
+	dma_info->user_addr	= vma->vm_start;
+
+#ifdef USE_DMA_MALLOC
+	ret = dma_malloc(dma_info, vma);
+#else
+	ret = kernel_malloc(dma_info, vma);
+#endif
+
+	if (ret < 0)
+		return -1;
+
+	vma->vm_ops		= &dma_vm_ops;
+	vma->vm_private_data	= dma_info;
+	vm_flags_set(vma, VM_DONTCOPY);
+
+	list_add(&dma_info->list, &dmabuf_list);
+	return 0;
+}
+
+static int va2pa(void *va, size_t size, va2pa_t **va2pa)
+{
+	pmd_t *pmd;
+	pte_t *pte;
+
+	unsigned long pg_offset;
+	unsigned long pg_address;
+	unsigned long old_pfn;
+	unsigned long paddr;
+	size_t total;
+	int flag = 0;
+
+	va2pa_t *p;
+	int num = size / sizeof(PAGE_SIZE);
+	unsigned long vaddr = (unsigned long)va;
+	int i, j;
+
+	p = kmalloc_array(num, sizeof(*p), GFP_KERNEL);
+	*va2pa = p;
+
+	j = 0;
+	old_pfn = 0;
+	total = 0;
+
+	for (i = 0; i < num; i++) {
+		memset(p + i, 0x00, sizeof(va2pa_t));
+		pmd = pmd_off(current->mm, vaddr);
+		if (pmd_none(*pmd)) {
+			dev_err(dma_dev->dev, "not in the pmd!");
+			flag = -1;
+			break;
+		}
+
+		pte = pte_offset_map(pmd, vaddr);
+		if (pte_none(*pte)) {
+			dev_err(dma_dev->dev, "not in the pte!");
+			flag = -1;
+			break;
+		}
+
+		pg_offset = offset_in_page(vaddr);
+		pg_address = pte_pfn(__pte(pte_val(*pte))) << PAGE_SHIFT;
+		paddr = pg_address | pg_offset;
+		pte_unmap(pte);
+		if ((old_pfn + PAGE_SIZE) == pg_address) {
+			p[j].size += (PAGE_SIZE - (vaddr - (vaddr & PAGE_MASK)));
+			total += (PAGE_SIZE - (vaddr - (vaddr & PAGE_MASK)));
+			p[j].dirty = 1;
+		} else {
+			if (p[j].dirty == 1)
+				j++;
+			p[j].addr = paddr;
+			p[j].size = (PAGE_SIZE - (vaddr - (vaddr & PAGE_MASK)));
+			total += p[j].size;
+			p[j].dirty = 1;
+		}
+		if (total >= size) {
+			j++;
+			break;
+		}
+		old_pfn = pg_address;
+		vaddr = (vaddr & PAGE_MASK) + PAGE_SIZE;
+	}
+
+	if (flag == -1) {
+		kfree(p);
+		*va2pa = NULL;
+		j = 0;
+	}
+
+	return j;
+}
+
+static int dma_memcpy(dma_addr_t dst, dma_addr_t src, size_t size)
+{
+#ifndef USE_DMA_MALLOC
+	dma_sync_single_for_cpu(dma_dev->dev, (dma_addr_t)src, size, DMA_FROM_DEVICE);
+#endif
+	mutex_lock(&dma_mutex);
+	dma_tx = dma_dev->device_prep_dma_memcpy(dma_chan,
+						dst,
+						src,
+						size,
+						DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	if (!dma_tx) {
+		mutex_unlock(&dma_mutex);
+		dev_err(dma_dev->dev, "Failed to prepare DMA memcpy");
+		return -1;
+	}
+
+	dma_tx->callback		= dma_callback_func;//set call back function
+	dma_tx->callback_param		= NULL;
+	if (dma_submit_error(dma_tx->tx_submit(dma_tx))) {
+		mutex_unlock(&dma_mutex);
+		dev_err(dma_dev->dev, "Failed to do DMA tx_submit");
+		return -1;
+	}
+
+	init_completion(&dma_m2m_ok);
+	dma_async_issue_pending(dma_chan);//begin dma transfer
+	wait_for_completion(&dma_m2m_ok);
+	reinit_completion(&dma_m2m_ok);
+
+#ifndef USE_DMA_MALLOC
+	dma_sync_single_for_device(dma_dev->dev, (dma_addr_t)msg.dst, msg.size, DMA_FROM_DEVICE);
+#endif
+	mutex_unlock(&dma_mutex);
+
+	return 0;
+}
+
+static long dma_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	if (cmd == DMA_MEMCPY_CMD) {
+		memcpy_msg_t msg;
+		va2pa_t *s_pa_l, *d_pa_l;
+		int i, loop, ret;
+		size_t off;
+
+		if (copy_from_user(&msg, (void *)arg, sizeof(memcpy_msg_t)))
+			return -EFAULT;
+
+		ret = va2pa(msg.src, msg.size, &s_pa_l);
+		if (ret != 1) {
+			kfree(s_pa_l);
+			return -1;
+		}
+		ret = va2pa(msg.dst, msg.size, &d_pa_l);
+		if (ret == 0) {
+			kfree(s_pa_l);
+			return -1;
+		}
+
+		loop = ret;
+		off = 0;
+
+		for (i = 0; i < loop; i++) {
+			dma_memcpy(d_pa_l[i].addr, s_pa_l[0].addr + off, d_pa_l[i].size);
+			off += d_pa_l[i].size;
+		}
+
+		kfree(s_pa_l);
+		kfree(d_pa_l);
+	}
+	return 0;
+}
+
+static const struct file_operations dma_fops = {
+	.owner		= THIS_MODULE,
+	.read		= dma_read,
+	.write		= dma_write,
+	.open		= dma_open,
+	.release	= dma_release,
+	.mmap		= dma_mmap,
+	.unlocked_ioctl = dma_ioctl,
+};
+
+static const struct of_device_id udma_of_match[] = {
+	{ .compatible = "spacemit,udma", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, udma_of_match);
+
+static int udma_probe(struct platform_device *pdev)
+{
+	dma_cap_mask_t mask;
+	struct device *dev_ret;
+
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_MEMCPY, mask);
+	dma_chan = dma_request_channel(mask, NULL, NULL);
+	if (!dma_chan) {
+		dev_dbg(&pdev->dev,
+			"no DMA MEMCPY channel, deferring probe\n");
+		return -EPROBE_DEFER;
+	}
+
+	dma_dev = dma_chan->device;
+
+	INIT_LIST_HEAD(&dmabuf_list);
+	mutex_init(&dma_mutex);
+
+	dma_major = register_chrdev(0, DEVICE_NAME, &dma_fops);
+	if (dma_major < 0) {
+		dev_err(&pdev->dev, "register_chrdev failed: %d\n", dma_major);
+		goto err_release_chan;
+	}
+
+	dma_class = class_create(DEVICE_NAME);
+	if (IS_ERR(dma_class)) {
+		dev_err(&pdev->dev, "class_create failed\n");
+		goto err_unregister_chrdev;
+	}
+
+	dev_ret = device_create(dma_class, NULL, MKDEV(dma_major, 0), NULL,
+				DEVICE_NAME);
+	if (IS_ERR(dev_ret)) {
+		dev_err(&pdev->dev, "device_create failed: %ld\n",
+			PTR_ERR(dev_ret));
+		goto err_destroy_class;
+	}
+
+	dev_dbg(dma_dev->dev, "dma channel id = %d\n", dma_chan->chan_id);
+
+	return 0;
+
+err_destroy_class:
+	class_destroy(dma_class);
+	dma_class = NULL;
+err_unregister_chrdev:
+	unregister_chrdev(dma_major, DEVICE_NAME);
+	dma_major = 0;
+err_release_chan:
+	dma_release_channel(dma_chan);
+	dma_chan = NULL;
+	dma_dev = NULL;
+
+	return -ENODEV;
+}
+
+static void udma_remove(struct platform_device *pdev)
+{
+	device_destroy(dma_class, MKDEV(dma_major, 0));
+	class_destroy(dma_class);
+	unregister_chrdev(dma_major, DEVICE_NAME);
+	dma_release_channel(dma_chan);
+	dma_chan = NULL;
+	dma_dev = NULL;
+}
+
+static struct platform_driver udma_driver = {
+	.probe		= udma_probe,
+	.remove		= udma_remove,
+	.driver		= {
+		.name		= "udma",
+		.of_match_table = udma_of_match,
+	},
+};
+
+module_platform_driver(udma_driver);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index e053524c5e35..46ef8571b59a 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -682,13 +682,13 @@ config GPIO_SNPS_CREG
 	  each GPIO line owns a field with different length and on/off value.
 
 config GPIO_SPACEMIT_K1
-	tristate "SPACEMIT K1 GPIO support"
+	tristate "SPACEMIT K1/K3 GPIO support"
 	depends on ARCH_SPACEMIT || COMPILE_TEST
 	depends on OF_GPIO
 	select GPIO_GENERIC
 	select GPIOLIB_IRQCHIP
 	help
-	  Say yes here to support the SpacemiT's K1 GPIO device.
+	  Say yes here to support the SpacemiT's K1/K3 GPIO device.
 
 config GPIO_SPEAR_SPICS
 	bool "ST SPEAr13xx SPI Chip Select as GPIO support"
diff --git a/drivers/gpio/gpio-spacemit-k1.c b/drivers/gpio/gpio-spacemit-k1.c
index eb66a15c002f..4c8a9132a1ee 100644
--- a/drivers/gpio/gpio-spacemit-k1.c
+++ b/drivers/gpio/gpio-spacemit-k1.c
@@ -12,36 +12,105 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/seq_file.h>
-
-/* register offset */
-#define SPACEMIT_GPLR		0x00 /* port level - R */
-#define SPACEMIT_GPDR		0x0c /* port direction - R/W */
-#define SPACEMIT_GPSR		0x18 /* port set - W */
-#define SPACEMIT_GPCR		0x24 /* port clear - W */
-#define SPACEMIT_GRER		0x30 /* port rising edge R/W */
-#define SPACEMIT_GFER		0x3c /* port falling edge R/W */
-#define SPACEMIT_GEDR		0x48 /* edge detect status - R/W1C */
-#define SPACEMIT_GSDR		0x54 /* (set) direction - W */
-#define SPACEMIT_GCDR		0x60 /* (clear) direction - W */
-#define SPACEMIT_GSRER		0x6c /* (set) rising edge detect enable - W */
-#define SPACEMIT_GCRER		0x78 /* (clear) rising edge detect enable - W */
-#define SPACEMIT_GSFER		0x84 /* (set) falling edge detect enable - W */
-#define SPACEMIT_GCFER		0x90 /* (clear) falling edge detect enable - W */
-#define SPACEMIT_GAPMASK	0x9c /* interrupt mask , 0 disable, 1 enable - R/W */
+#include <linux/version.h>
+
+/* register offset structure */
+struct spacemit_gpio_reg_offsets {
+	u32 gplr;      /* port level - R */
+	u32 gpdr;      /* port direction - R/W */
+	u32 gpsr;      /* port set - W */
+	u32 gpcr;      /* port clear - W */
+	u32 grer;      /* port rising edge R/W */
+	u32 gfer;      /* port falling edge R/W */
+	u32 gedr;      /* edge detect status - R/W1C */
+	u32 gsdr;      /* (set) direction - W */
+	u32 gcdr;      /* (clear) direction - W */
+	u32 gsrer;     /* (set) rising edge detect enable - W */
+	u32 gcrer;     /* (clear) rising edge detect enable - W */
+	u32 gsfer;     /* (set) falling edge detect enable - W */
+	u32 gcfer;     /* (clear) falling edge detect enable - W */
+	u32 gapmask;   /* interrupt mask , 0 disable, 1 enable - R/W */
+	u32 gcpmask;   /* interrupt mask for K3 */
+};
 
 #define SPACEMIT_NR_BANKS		4
 #define SPACEMIT_NR_GPIOS_PER_BANK	32
 
+/* K1 reg_offsets */
+static const struct spacemit_gpio_reg_offsets k1_regs = {
+	.gplr    = 0x00,
+	.gpdr    = 0x0c,
+	.gpsr    = 0x18,
+	.gpcr    = 0x24,
+	.grer    = 0x30,
+	.gfer    = 0x3c,
+	.gedr    = 0x48,
+	.gsdr    = 0x54,
+	.gcdr    = 0x60,
+	.gsrer   = 0x6c,
+	.gcrer   = 0x78,
+	.gsfer   = 0x84,
+	.gcfer   = 0x90,
+	.gapmask = 0x9c,
+	.gcpmask = 0xA8,
+};
+
+/* K3 reg_offsets */
+static const struct spacemit_gpio_reg_offsets k3_regs = {
+	.gplr    = 0x0,
+	.gpdr    = 0x4,
+	.gpsr    = 0x8,
+	.gpcr    = 0xc,
+	.grer    = 0x10,
+	.gfer    = 0x14,
+	.gedr    = 0x18,
+	.gsdr    = 0x1c,
+	.gcdr    = 0x20,
+	.gsrer   = 0x24,
+	.gcrer   = 0x28,
+	.gsfer   = 0x2c,
+	.gcfer   = 0x30,
+	.gapmask = 0x34,
+	.gcpmask = 0x38,
+};
+
+/* K1 bank_offsets */
+static const u32 k1_bank_offsets[] = { 0x0, 0x4, 0x8, 0x100 };
+
+/* K3 bank_offsets */
+static const u32 k3_bank_offsets[] = { 0x0, 0x40, 0x80, 0x100 };
+
+/*
+ * Use raw spinlock so the regmap can be accessed from irq chip callbacks
+ * (irq_set_type, irq_mask, irq_unmask, irq_ack) which run in atomic context
+ * on PREEMPT_RT where regular spinlock_t becomes a sleeping rt_spin_lock.
+ */
+static const struct regmap_config spacemit_gpio_regmap_config = {
+	.reg_bits        = 32,
+	.val_bits        = 32,
+	.reg_stride      = 4,
+	.use_raw_spinlock = true,
+};
+
 #define to_spacemit_gpio_bank(x) container_of((x), struct spacemit_gpio_bank, gc)
 
 struct spacemit_gpio;
 
+struct spacemit_gpio_data {
+	const struct spacemit_gpio_reg_offsets *reg_offsets;
+	const u32 *bank_offsets;
+};
+
 struct spacemit_gpio_bank {
 	struct gpio_generic_chip chip;
 	struct spacemit_gpio *sg;
-	void __iomem *base;
+	u32 bank_offset;
 	u32 irq_mask;
 	u32 irq_rising_edge;
 	u32 irq_falling_edge;
@@ -49,6 +118,9 @@ struct spacemit_gpio_bank {
 
 struct spacemit_gpio {
 	struct device *dev;
+	struct regmap *rm_gpio;
+	struct regmap *rm_gpio_edge;
+	const struct spacemit_gpio_data *data;
 	struct spacemit_gpio_bank sgb[SPACEMIT_NR_BANKS];
 };
 
@@ -60,13 +132,14 @@ static u32 spacemit_gpio_bank_index(struct spacemit_gpio_bank *gb)
 static irqreturn_t spacemit_gpio_irq_handler(int irq, void *dev_id)
 {
 	struct spacemit_gpio_bank *gb = dev_id;
+	u32 reg_offset = gb->bank_offset + gb->sg->data->reg_offsets->gedr;
 	unsigned long pending;
 	u32 n, gedr;
 
-	gedr = readl(gb->base + SPACEMIT_GEDR);
+	regmap_read(gb->sg->rm_gpio, reg_offset, &gedr);
 	if (!gedr)
 		return IRQ_NONE;
-	writel(gedr, gb->base + SPACEMIT_GEDR);
+	regmap_write(gb->sg->rm_gpio, reg_offset, gedr);
 
 	pending = gedr & gb->irq_mask;
 	if (!pending)
@@ -81,8 +154,9 @@ static irqreturn_t spacemit_gpio_irq_handler(int irq, void *dev_id)
 static void spacemit_gpio_irq_ack(struct irq_data *d)
 {
 	struct spacemit_gpio_bank *gb = irq_data_get_irq_chip_data(d);
+	u32 reg_offset = gb->bank_offset + gb->sg->data->reg_offsets->gedr;
 
-	writel(BIT(irqd_to_hwirq(d)), gb->base + SPACEMIT_GEDR);
+	regmap_write(gb->sg->rm_gpio, reg_offset, BIT(irqd_to_hwirq(d)));
 }
 
 static void spacemit_gpio_irq_mask(struct irq_data *d)
@@ -91,13 +165,20 @@ static void spacemit_gpio_irq_mask(struct irq_data *d)
 	u32 bit = BIT(irqd_to_hwirq(d));
 
 	gb->irq_mask &= ~bit;
-	writel(gb->irq_mask, gb->base + SPACEMIT_GAPMASK);
+	regmap_write(gb->sg->rm_gpio,
+		     gb->bank_offset + gb->sg->data->reg_offsets->gapmask,
+		     gb->irq_mask);
 
-	if (bit & gb->irq_rising_edge)
-		writel(bit, gb->base + SPACEMIT_GCRER);
+	if (gb->irq_rising_edge & bit)
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gcrer,
+			     bit);
+
+	if (gb->irq_falling_edge & bit)
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gcfer,
+			     bit);
 
-	if (bit & gb->irq_falling_edge)
-		writel(bit, gb->base + SPACEMIT_GCFER);
 }
 
 static void spacemit_gpio_irq_unmask(struct irq_data *d)
@@ -107,13 +188,19 @@ static void spacemit_gpio_irq_unmask(struct irq_data *d)
 
 	gb->irq_mask |= bit;
 
-	if (bit & gb->irq_rising_edge)
-		writel(bit, gb->base + SPACEMIT_GSRER);
+	if (gb->irq_rising_edge & bit)
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gsrer,
+			     bit);
 
-	if (bit & gb->irq_falling_edge)
-		writel(bit, gb->base + SPACEMIT_GSFER);
+	if (gb->irq_falling_edge & bit)
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gsfer,
+			     bit);
 
-	writel(gb->irq_mask, gb->base + SPACEMIT_GAPMASK);
+	regmap_write(gb->sg->rm_gpio,
+		     gb->bank_offset + gb->sg->data->reg_offsets->gapmask,
+		     gb->irq_mask);
 }
 
 static int spacemit_gpio_irq_set_type(struct irq_data *d, unsigned int type)
@@ -123,18 +210,26 @@ static int spacemit_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 
 	if (type & IRQ_TYPE_EDGE_RISING) {
 		gb->irq_rising_edge |= bit;
-		writel(bit, gb->base + SPACEMIT_GSRER);
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gsrer,
+			     bit);
 	} else {
 		gb->irq_rising_edge &= ~bit;
-		writel(bit, gb->base + SPACEMIT_GCRER);
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gcrer,
+			     bit);
 	}
 
 	if (type & IRQ_TYPE_EDGE_FALLING) {
 		gb->irq_falling_edge |= bit;
-		writel(bit, gb->base + SPACEMIT_GSFER);
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gsfer,
+			     bit);
 	} else {
 		gb->irq_falling_edge &= ~bit;
-		writel(bit, gb->base + SPACEMIT_GCFER);
+		regmap_write(gb->sg->rm_gpio,
+			     gb->bank_offset + gb->sg->data->reg_offsets->gcfer,
+			     bit);
 	}
 
 	return 0;
@@ -148,7 +243,7 @@ static void spacemit_gpio_irq_print_chip(struct irq_data *data, struct seq_file
 }
 
 static struct irq_chip spacemit_gpio_chip = {
-	.name		= "k1-gpio-irqchip",
+	.name		= "spacemit-gpio-irqchip",
 	.irq_ack	= spacemit_gpio_irq_ack,
 	.irq_mask	= spacemit_gpio_irq_mask,
 	.irq_unmask	= spacemit_gpio_irq_unmask,
@@ -165,54 +260,74 @@ static bool spacemit_of_node_instance_match(struct gpio_chip *gc, unsigned int i
 
 	if (i >= SPACEMIT_NR_BANKS)
 		return false;
-
 	return (gc == &sg->sgb[i].chip.gc);
 }
 
+static int spacemit_gpio_get(struct gpio_chip *gc, unsigned int offset)
+{
+	struct spacemit_gpio_bank *gb = gpiochip_get_data(gc);
+	u32 reg_offset = gb->bank_offset + gb->sg->data->reg_offsets->gplr;
+	u32 val;
+
+	regmap_read(gb->sg->rm_gpio, reg_offset, &val);
+	return !!(val & BIT(offset));
+}
+
+static int spacemit_gpio_set(struct gpio_chip *gc, unsigned int offset, int value)
+{
+	struct spacemit_gpio_bank *gb = gpiochip_get_data(gc);
+	u32 reg_offset;
+
+	if (value)
+		reg_offset = gb->bank_offset + gb->sg->data->reg_offsets->gpsr;
+	else
+		reg_offset = gb->bank_offset + gb->sg->data->reg_offsets->gpcr;
+
+	return regmap_write(gb->sg->rm_gpio, reg_offset, BIT(offset));
+}
+
+static int spacemit_gpio_direction_input(struct gpio_chip *gc, unsigned int offset)
+{
+	struct spacemit_gpio_bank *gb = gpiochip_get_data(gc);
+	u32 reg_offset = gb->bank_offset + gb->sg->data->reg_offsets->gcdr;
+
+	regmap_write(gb->sg->rm_gpio, reg_offset, BIT(offset));
+	return 0;
+}
+
+static int spacemit_gpio_direction_output(struct gpio_chip *gc, unsigned int offset, int value)
+{
+	struct spacemit_gpio_bank *gb = gpiochip_get_data(gc);
+	u32 reg_offset = gb->bank_offset + gb->sg->data->reg_offsets->gsdr;
+
+	regmap_write(gb->sg->rm_gpio, reg_offset, BIT(offset));
+	spacemit_gpio_set(gc, offset, value);
+	return 0;
+}
+
 static int spacemit_gpio_add_bank(struct spacemit_gpio *sg,
-				  void __iomem *regs,
 				  int index, int irq)
 {
 	struct spacemit_gpio_bank *gb = &sg->sgb[index];
-	struct gpio_generic_chip_config config;
 	struct gpio_chip *gc = &gb->chip.gc;
 	struct device *dev = sg->dev;
 	struct gpio_irq_chip *girq;
-	void __iomem *dat, *set, *clr, *dirin, *dirout;
-	int ret, bank_base[] = { 0x0, 0x4, 0x8, 0x100 };
-
-	gb->base = regs + bank_base[index];
-
-	dat	= gb->base + SPACEMIT_GPLR;
-	set	= gb->base + SPACEMIT_GPSR;
-	clr	= gb->base + SPACEMIT_GPCR;
-	dirin	= gb->base + SPACEMIT_GCDR;
-	dirout	= gb->base + SPACEMIT_GSDR;
-
-	config = (struct gpio_generic_chip_config) {
-		.dev = dev,
-		.sz = 4,
-		.dat = dat,
-		.set = set,
-		.clr = clr,
-		.dirout = dirout,
-		.dirin = dirin,
-		.flags = GPIO_GENERIC_UNREADABLE_REG_SET |
-			 GPIO_GENERIC_UNREADABLE_REG_DIR,
-	};
-
-	/* This registers 32 GPIO lines per bank */
-	ret = gpio_generic_chip_init(&gb->chip, &config);
-	if (ret)
-		return dev_err_probe(dev, ret, "failed to init gpio chip\n");
+	int ret;
+
+	gb->bank_offset = sg->data->bank_offsets[index];
+	gc->label	= devm_kasprintf(dev, GFP_KERNEL, "%s-bank%d", dev_name(dev), index);
+	gc->request	= gpiochip_generic_request;
+	gc->free	= gpiochip_generic_free;
+	gc->get		= spacemit_gpio_get;
+	gc->set		= spacemit_gpio_set;
+	gc->parent	= dev;
+	gc->direction_input	= spacemit_gpio_direction_input;
+	gc->direction_output	= spacemit_gpio_direction_output;
 
 	gb->sg = sg;
 
-	gc->label		= dev_name(dev);
-	gc->request		= gpiochip_generic_request;
-	gc->free		= gpiochip_generic_free;
+	gc->base		= index * SPACEMIT_NR_GPIOS_PER_BANK;
 	gc->ngpio		= SPACEMIT_NR_GPIOS_PER_BANK;
-	gc->base		= -1;
 	gc->of_gpio_n_cells	= 3;
 	gc->of_node_instance_match = spacemit_of_node_instance_match;
 
@@ -223,13 +338,17 @@ static int spacemit_gpio_add_bank(struct spacemit_gpio *sg,
 	gpio_irq_chip_set_chip(girq, &spacemit_gpio_chip);
 
 	/* Disable Interrupt */
-	writel(0, gb->base + SPACEMIT_GAPMASK);
+	regmap_write(gb->sg->rm_gpio, gb->bank_offset + gb->sg->data->reg_offsets->gapmask, 0);
 	/* Disable Edge Detection Settings */
-	writel(0x0, gb->base + SPACEMIT_GRER);
-	writel(0x0, gb->base + SPACEMIT_GFER);
+	regmap_write(gb->sg->rm_gpio, gb->bank_offset + gb->sg->data->reg_offsets->grer, 0x0);
+	regmap_write(gb->sg->rm_gpio, gb->bank_offset + gb->sg->data->reg_offsets->gfer, 0x0);
 	/* Clear Interrupt */
-	writel(0xffffffff, gb->base + SPACEMIT_GCRER);
-	writel(0xffffffff, gb->base + SPACEMIT_GCFER);
+	regmap_write(gb->sg->rm_gpio,
+		     gb->bank_offset + gb->sg->data->reg_offsets->gcrer,
+		     0xffffffff);
+	regmap_write(gb->sg->rm_gpio,
+		     gb->bank_offset + gb->sg->data->reg_offsets->gcfer,
+		     0xffffffff);
 
 	ret = devm_request_threaded_irq(dev, irq, NULL,
 					spacemit_gpio_irq_handler,
@@ -251,25 +370,55 @@ static int spacemit_gpio_add_bank(struct spacemit_gpio *sg,
 static int spacemit_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct device_node *gpio_np;
 	struct spacemit_gpio *sg;
 	struct clk *core_clk, *bus_clk;
-	void __iomem *regs;
+	const struct spacemit_gpio_data *data;
+	void __iomem *base;
 	int i, irq, ret;
 
 	sg = devm_kzalloc(dev, sizeof(*sg), GFP_KERNEL);
 	if (!sg)
 		return -ENOMEM;
 
-	regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(regs))
-		return PTR_ERR(regs);
+	sg->dev = dev;
+
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return dev_err_probe(dev, -EINVAL, "Unsupported compatible\n");
+
+	sg->data = data;
+
+	/*
+	 * Create a private MMIO regmap with use_raw_spinlock instead of using
+	 * the syscon regmap. On PREEMPT_RT, the MMIO regmap bus sets fast_io=true
+	 * which selects spinlock_t; spinlock_t becomes rt_spin_lock (a sleeping
+	 * lock) on PREEMPT_RT, making it illegal to call from the atomic context
+	 * of irq chip callbacks (__setup_irq holds desc->lock, a raw_spinlock).
+	 * use_raw_spinlock keeps raw_spin_lock_irqsave semantics on all configs.
+	 */
+	gpio_np = of_parse_phandle(np, "syscon-gpio-regs", 0);
+	base = devm_of_iomap(dev, gpio_np, 0, NULL);
+	of_node_put(gpio_np);
+	if (IS_ERR(base))
+		return dev_err_probe(dev, PTR_ERR(base),
+				     "Failed to iomap gpio registers\n");
+
+	sg->rm_gpio = devm_regmap_init_mmio(dev, base, &spacemit_gpio_regmap_config);
+	if (IS_ERR(sg->rm_gpio))
+		return dev_err_probe(dev, PTR_ERR(sg->rm_gpio),
+				     "Failed to init gpio regmap\n");
+
+	sg->rm_gpio_edge = syscon_regmap_lookup_by_phandle(np, "syscon-gpio-edge");
+	if (IS_ERR(sg->rm_gpio_edge))
+		return dev_err_probe(dev, PTR_ERR(sg->rm_gpio_edge),
+				     "Failed to get syscon-gpio-edge regmap\n");
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
 
-	sg->dev	= dev;
-
 	core_clk = devm_clk_get_enabled(dev, "core");
 	if (IS_ERR(core_clk))
 		return dev_err_probe(dev, PTR_ERR(core_clk), "failed to get clock\n");
@@ -279,7 +428,7 @@ static int spacemit_gpio_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(bus_clk), "failed to get bus clock\n");
 
 	for (i = 0; i < SPACEMIT_NR_BANKS; i++) {
-		ret = spacemit_gpio_add_bank(sg, regs, i, irq);
+		ret = spacemit_gpio_add_bank(sg, i, irq);
 		if (ret)
 			return ret;
 	}
@@ -287,8 +436,19 @@ static int spacemit_gpio_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct spacemit_gpio_data k1_data = {
+	.reg_offsets = &k1_regs,
+	.bank_offsets = k1_bank_offsets,
+};
+
+static const struct spacemit_gpio_data k3_data = {
+	.reg_offsets = &k3_regs,
+	.bank_offsets = k3_bank_offsets,
+};
+
 static const struct of_device_id spacemit_gpio_dt_ids[] = {
-	{ .compatible = "spacemit,k1-gpio" },
+	{ .compatible = "spacemit,k1-gpio", .data = &k1_data },
+	{ .compatible = "spacemit,k3-gpio", .data = &k3_data },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, spacemit_gpio_dt_ids);
@@ -296,12 +456,12 @@ MODULE_DEVICE_TABLE(of, spacemit_gpio_dt_ids);
 static struct platform_driver spacemit_gpio_driver = {
 	.probe		= spacemit_gpio_probe,
 	.driver		= {
-		.name	= "k1-gpio",
+		.name	= "spacemit-gpio",
 		.of_match_table = spacemit_gpio_dt_ids,
 	},
 };
 module_platform_driver(spacemit_gpio_driver);
 
 MODULE_AUTHOR("Yixun Lan <dlan@gentoo.org>");
-MODULE_DESCRIPTION("GPIO driver for SpacemiT K1 SoC");
+MODULE_DESCRIPTION("GPIO driver for SpacemiT K1/K3 SoC");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 7e6bc0b3a589..2b821542152f 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -398,6 +398,12 @@ source "drivers/gpu/drm/imagination/Kconfig"
 
 source "drivers/gpu/drm/tyr/Kconfig"
 
+source "drivers/gpu/drm/img-rogue/Kconfig"
+
+source "drivers/gpu/drm/nulldisp/Kconfig"
+
+source "drivers/gpu/drm/spacemit/Kconfig"
+
 config DRM_HYPERV
 	tristate "DRM Support for Hyper-V synthetic video device"
 	depends on DRM && PCI && HYPERV_VMBUS
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index da2565e6de71..663552aba78d 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -232,6 +232,9 @@ obj-y			+= solomon/
 obj-$(CONFIG_DRM_SPRD) += sprd/
 obj-$(CONFIG_DRM_LOONGSON) += loongson/
 obj-$(CONFIG_DRM_POWERVR) += imagination/
+obj-y			+= img-rogue/
+obj-$(CONFIG_POWERVR_ROGUE_NULLDRMDISP) += nulldisp/
+obj-$(CONFIG_DRM_SPACEMIT) += spacemit/
 
 # Ensure drm headers are self-contained and pass kernel-doc
 hdrtest-files := \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 7333e19291cf..159975c91782 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -175,7 +175,11 @@ int amdgpu_exp_hw_support;
 int amdgpu_dc = -1;
 int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
+#ifdef CONFIG_SOC_SPACEMIT_K3
+uint amdgpu_pcie_gen_cap = 0x00040004;
+#else
 uint amdgpu_pcie_gen_cap;
+#endif
 uint amdgpu_pcie_lane_cap;
 u64 amdgpu_cg_mask = 0xffffffffffffffff;
 uint amdgpu_pg_mask = 0xffffffff;
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index 09b12c30df69..c25a804f6e5a 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -650,7 +650,7 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
 
 	if (count == 0 && (connector->status == connector_status_connected ||
 			   connector->status == connector_status_unknown)) {
-		count = drm_add_modes_noedid(connector, 1024, 768);
+		count = drm_add_modes_noedid(connector, 1920, 1080);
 
 		/*
 		 * Section 4.2.2.6 (EDID Corruption Detection) of the DP 1.4a
diff --git a/drivers/gpu/drm/img-rogue/Kconfig b/drivers/gpu/drm/img-rogue/Kconfig
new file mode 100644
index 000000000000..8446137bc16a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/Kconfig
@@ -0,0 +1,40 @@
+config POWERVR_ROGUE
+	tristate "PowerVR GPU"
+	default n
+	depends on DRM
+	help
+	  Driver for PowerVR graphics processor hardware.
+
+	  Say Y here if your SoC contains a PowerVR GPU. For more
+	  information, see <http://www.imgtec.com/powervr/>.
+
+config POWERVR_DVFS
+	tristate "PowerVR GPU DVFS"
+	default n
+	depends on POWERVR_ROGUE && PM_DEVFREQ && PM_OPP
+	select DEVFREQ_GOV_SIMPLE_ONDEMAND
+	help
+	  Enable Dynamic Voltage and Frequency Scaling (DVFS) support
+	  for PowerVR GPU. This allows the GPU frequency and voltage to
+	  be adjusted dynamically based on workload to optimize power
+	  consumption and performance.
+
+	  Requires kernel support for devfreq (CONFIG_PM_DEVFREQ) and
+	  Operating Performance Points (CONFIG_PM_OPP).
+
+	  Say Y here if you want to enable DVFS.
+
+config POWERVR_THERMAL
+	tristate "PowerVR GPU Thermal"
+	default n
+	depends on POWERVR_ROGUE && POWERVR_DVFS && DEVFREQ_THERMAL && K3_THERMAL
+	help
+	  Enable thermal management support for PowerVR GPU.
+	  This allows the GPU to adjust frequency and voltage
+	  based on thermal conditions.
+
+	  Requires kernel support for devfreq thermal
+	  (CONFIG_DEVFREQ_THERMAL) and K3 thermal
+	  (CONFIG_K3_THERMAL).
+
+	  Say Y here if you want to enable PowerVR thermal management.
diff --git a/drivers/gpu/drm/img-rogue/Makefile b/drivers/gpu/drm/img-rogue/Makefile
new file mode 100644
index 000000000000..3679ae30e0a9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/Makefile
@@ -0,0 +1,167 @@
+include $(srctree)/drivers/gpu/drm/img-rogue/config_kernel.mk
+ccflags-y += -include $(srctree)/drivers/gpu/drm/img-rogue/config_kernel.h
+ccflags-y += -I$(src)/km
+ccflags-y += -I$(src)/spacemit
+ccflags-y += -I$(src)
+
+ccflags-y += $(call cc-option,-Wno-unused-const-variable)
+ccflags-y += $(call cc-option,-Wno-unused-but-set-variable)
+ccflags-y += $(call cc-option,-Wno-missing-format-attribute)
+ccflags-y += $(call cc-option,-Wno-error=designated-init)
+ccflags-y += $(call cc-option,-Wno-incompatible-pointer-types)
+
+ccflags-$(CONFIG_POWERVR_DVFS) += -I$(srctree)/drivers/devfreq
+
+obj-$(CONFIG_POWERVR_ROGUE) += pvrsrvkm.o
+pvrsrvkm-y += \
+ client_cache_direct_bridge.o \
+ server_cache_bridge.o \
+ server_cmm_bridge.o \
+ client_devicememhistory_direct_bridge.o \
+ server_devicememhistory_bridge.o \
+ server_di_bridge.o \
+ server_dmabuf_bridge.o \
+ client_mm_direct_bridge.o \
+ server_mm_bridge.o \
+ server_mmextmem_bridge.o \
+ client_pvrtl_direct_bridge.o \
+ server_pvrtl_bridge.o \
+ server_rgxbreakpoint_bridge.o \
+ server_rgxcmp_bridge.o \
+ server_rgxfwdbg_bridge.o \
+ server_rgxhwperf_bridge.o \
+ server_rgxregconfig_bridge.o \
+ server_rgxta3d_bridge.o \
+ server_rgxtimerquery_bridge.o \
+ server_rgxtq2_bridge.o \
+ server_rgxtq_bridge.o \
+ server_srvcore_bridge.o \
+ client_sync_direct_bridge.o \
+ server_sync_bridge.o \
+ client_synctracking_direct_bridge.o \
+ server_synctracking_bridge.o \
+ cache_km.o \
+ connection_server.o \
+ debug_common.o \
+ devicemem_heapcfg.o \
+ devicemem_history_server.o \
+ devicemem_server.o \
+ di_impl_brg.o \
+ di_server.o \
+ handle.o \
+ info_page_km.o \
+ lists.o \
+ mmu_common.o \
+ physheap.o \
+ physmem.o \
+ physmem_extmem.o \
+ physmem_hostmem.o \
+ physmem_lma.o \
+ physmem_osmem.o \
+ physmem_ramem.o \
+ pmr.o \
+ power.o \
+ process_stats.o \
+ pvr_notifier.o \
+ pvrsrv.o \
+ pvrsrv_bridge_init.o \
+ pvrsrv_pool.o \
+ srvcore.o \
+ sync_checkpoint.o \
+ sync_server.o \
+ tlintern.o \
+ tlserver.o \
+ tlstream.o \
+ vmm_pvz_client.o \
+ vmm_pvz_server.o \
+ vz_vmm_pvz.o \
+ vz_vmm_vm.o \
+ rgx_bridge_init.o \
+ rgxbreakpoint.o \
+ rgxbvnc.o \
+ rgxccb.o \
+ rgxcompute.o \
+ rgxdebug_common.o \
+ rgxfwcmnctx.o \
+ rgxfwdbg.o \
+ rgxfwimageutils.o \
+ rgxfwriscv.o \
+ rgxfwtrace_strings.o \
+ rgxhwperf_common.o \
+ rgxlayer_impl_common.o \
+ rgxmem.o \
+ rgxmmuinit.o \
+ rgxpower.o \
+ rgxregconfig.o \
+ rgxshader.o \
+ rgxsyncutils.o \
+ rgxtdmtransfer.o \
+ rgxtimecorr.o \
+ rgxtimerquery.o \
+ rgxutils.o \
+ rgxdebug.o \
+ rgxfwutils.o \
+ rgxhwperf.o \
+ rgxinit.o \
+ rgxlayer_impl.o \
+ rgxmipsmmuinit.o \
+ rgxmulticore.o \
+ rgxsrvinit.o \
+ rgxstartstop.o \
+ rgxta3d.o \
+ rgxtransfer.o \
+ allocmem.o \
+ dkf_server.o \
+ event.o \
+ fwload.o \
+ handle_idr.o \
+ km_apphint.o \
+ module_common.o \
+ osconnection_server.o \
+ osfunc.o \
+ osmmap_stub.o \
+ physmem_dmabuf.o \
+ physmem_extmem_linux.o \
+ physmem_osmem_linux.o \
+ physmem_test.o \
+ pmr_env.o \
+ pmr_os.o \
+ pvr_bridge_k.o \
+ pvr_buffer_sync.o \
+ pvr_counting_timeline.o \
+ pvr_debug.o \
+ pvr_debugfs.o \
+ pvr_drm.o \
+ pvr_export_fence.o \
+ pvr_fence.o \
+ pvr_platform_drv.o \
+ pvr_sw_fence.o \
+ pvr_sync_file.o \
+ pvr_sync_ioctl_common.o \
+ pvr_sync_ioctl_drm.o \
+ devicemem.o \
+ devicemem_utils.o \
+ hash.o \
+ mem_utils.o \
+ pvrsrv_error.o \
+ ra.o \
+ sync.o \
+ tlclient.o \
+ uniq_key_splay_tree.o \
+ rgx_hwperf_table.o \
+ interrupt_support.o \
+ sysconfig_cmn.o \
+ dma_support.o \
+ vmm_type_stub.o \
+ spacemit/sysconfig.o \
+ spacemit/spacemit_init.o \
+ server_rgxkicksync_bridge.o \
+ rgxkicksync.o \
+ pvr_sync_ioctl_common.o \
+ pvr_sync_ioctl_drm.o
+pvrsrvkm-$(CONFIG_POWERVR_DVFS)   += pvr_dvfs_device.o pvr_dvfs_common.o
+pvrsrvkm-$(CONFIG_ARM)   += osfunc_arm.o
+pvrsrvkm-$(CONFIG_ARM64) += osfunc_arm64.o
+pvrsrvkm-$(CONFIG_EVENT_TRACING) += trace_events.o pvr_gputrace.o
+pvrsrvkm-$(CONFIG_RISCV) += osfunc_riscv.o
+pvrsrvkm-$(CONFIG_X86)   += osfunc_x86.o
diff --git a/drivers/gpu/drm/img-rogue/allocmem.c b/drivers/gpu/drm/img-rogue/allocmem.c
new file mode 100644
index 000000000000..817680376d79
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/allocmem.c
@@ -0,0 +1,377 @@
+/*************************************************************************/ /*!
+@File
+@Title          Host memory management implementation for Linux
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/string.h>
+
+#include "img_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "process_stats.h"
+#include "osfunc.h"
+
+
+/*
+ * When memory statistics are disabled, memory records are used instead.
+ * In order for these to work, the PID of the process that requested the
+ * allocation needs to be stored at the end of the kmalloc'd memory, making
+ * sure 4 extra bytes are allocated to fit the PID.
+ *
+ * There is no need for this extra allocation when memory statistics are
+ * enabled, since all allocations are tracked in DebugFS mem_area files.
+ */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+/* kmalloc guarantees a minimal alignment which is ARCH_KMALLOC_MINALIGN. This
+ * alignment is architecture specific and can be quite big, e.g. on Aarch64
+ * it can be 64 bytes. This is too much for keeping a single PID field and could
+ * lead to a lot of wasted memory. This is a reason why we're defaulting to 8
+ * bytes alignment which should be enough for any architecture.
+ */
+#define ALLOCMEM_PID_SIZE_PADDING PVR_ALIGN(sizeof(IMG_UINT32), 8)
+#else
+#define ALLOCMEM_PID_SIZE_PADDING 0UL
+#endif
+
+/* How many times kmalloc can fail before the allocation threshold is reduced */
+static const IMG_UINT32 g_ui32kmallocFailLimit = 10;
+/* How many kmalloc failures happened since the last allocation threshold change */
+static IMG_UINT32 g_ui32kmallocFailCount = 0;
+/* Current kmalloc threshold value in bytes */
+static IMG_UINT32 g_ui32kmallocThreshold = PVR_LINUX_KMALLOC_ALLOCATION_THRESHOLD;
+/* Spinlock used so that the global variables above may not be modified by more than 1 thread at a time */
+static DEFINE_SPINLOCK(kmalloc_lock);
+
+
+static inline void OSTryDecreaseKmallocThreshold(void)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&kmalloc_lock, flags);
+
+	g_ui32kmallocFailCount++;
+
+	if (g_ui32kmallocFailCount >= g_ui32kmallocFailLimit)
+	{
+		g_ui32kmallocFailCount = 0;
+		if (g_ui32kmallocThreshold > PAGE_SIZE)
+		{
+			g_ui32kmallocThreshold >>= 1;
+			printk(KERN_INFO "Threshold is now set to %d\n", g_ui32kmallocThreshold);
+		}
+	}
+
+	spin_unlock_irqrestore(&kmalloc_lock, flags);
+}
+
+static inline void OSResetKmallocFailCount(void)
+{
+	unsigned long flags;
+	spin_lock_irqsave(&kmalloc_lock, flags);
+
+	g_ui32kmallocFailCount = 0;
+
+	spin_unlock_irqrestore(&kmalloc_lock, flags);
+}
+
+static inline void _pvr_vfree(const void* pvAddr)
+{
+#if defined(DEBUG)
+	/* Size harder to come by for vmalloc and since vmalloc allocates
+	 * a whole number of pages, poison the minimum size known to have
+	 * been allocated.
+	 */
+	OSCachedMemSet((void*)pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE,
+	               PAGE_SIZE);
+#endif
+	vfree(pvAddr);
+}
+
+static inline void _pvr_kfree(const void* pvAddr)
+{
+#if defined(DEBUG)
+	/* Poison whole memory block */
+	OSCachedMemSet((void*)pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE,
+	               ksize(pvAddr));
+#endif
+	kfree(pvAddr);
+}
+
+static inline void *_pvr_alloc_stats_add(void *pvAddr, IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+#if !defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVR_UNREFERENCED_PARAMETER(pvAddr);
+#else
+	if (!is_vmalloc_addr(pvAddr))
+	{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_KMALLOC,
+									  pvAddr,
+									  sCpuPAddr,
+									  ksize(pvAddr),
+									  OSGetCurrentClientProcessIDKM()
+									  DEBUG_MEMSTATS_ARGS);
+#else
+		/* because clang has some features that allow detection out-of-bounds
+		 * access we need to put the metadata in the beginning of the allocation */
+		*(IMG_UINT32 *) pvAddr = OSGetCurrentClientProcessIDKM();
+		PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_KMALLOC, ksize(pvAddr),
+		                            *(IMG_UINT32 *) pvAddr);
+
+		/* because metadata is kept in the beginning of the allocation we need
+		 * to return address offset by the ALLOCMEM_PID_SIZE_PADDING */
+		pvAddr = (IMG_UINT8 *) pvAddr + ALLOCMEM_PID_SIZE_PADDING;
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+	else
+	{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+									  pvAddr,
+									  sCpuPAddr,
+									  PVR_ALIGN(ui32Size, PAGE_SIZE),
+									  OSGetCurrentClientProcessIDKM()
+									  DEBUG_MEMSTATS_ARGS);
+#else
+		PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                    PVR_ALIGN(ui32Size, PAGE_SIZE),
+		                                    (IMG_UINT64)(uintptr_t) pvAddr,
+		                                    OSGetCurrentClientProcessIDKM());
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#endif /* !defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return pvAddr;
+}
+
+static inline void *_pvr_alloc_stats_remove(void *pvAddr)
+{
+#if !defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVR_UNREFERENCED_PARAMETER(pvAddr);
+#else
+	if (!is_vmalloc_addr(pvAddr))
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		/* because metadata is kept in the beginning of the allocation we need
+		 * shift address offset by the ALLOCMEM_PID_SIZE_PADDING to the original
+		 * value */
+		pvAddr = (IMG_UINT8 *) pvAddr - ALLOCMEM_PID_SIZE_PADDING;
+
+		/* first 4 bytes of the allocation are the process' PID */
+		PVRSRVStatsDecrMemKAllocStat(ksize(pvAddr), *(IMG_UINT32 *) pvAddr);
+#else
+		PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_KMALLOC,
+		                                (IMG_UINT64)(uintptr_t) pvAddr,
+		                                OSGetCurrentClientProcessIDKM());
+#endif
+	}
+	else
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                      (IMG_UINT64)(uintptr_t) pvAddr);
+#else
+		PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMALLOC,
+		                                (IMG_UINT64)(uintptr_t) pvAddr,
+		                                OSGetCurrentClientProcessIDKM());
+#endif
+	}
+#endif /* !defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return pvAddr;
+}
+
+void *(OSAllocMem)(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+	void *pvRet = NULL;
+
+	if ((ui32Size + ALLOCMEM_PID_SIZE_PADDING) <= g_ui32kmallocThreshold)
+	{
+		pvRet = kmalloc(ui32Size + ALLOCMEM_PID_SIZE_PADDING, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vmalloc(ui32Size);
+	}
+
+	if (pvRet != NULL)
+	{
+		pvRet = _pvr_alloc_stats_add(pvRet, ui32Size DEBUG_MEMSTATS_ARGS);
+	}
+
+	return pvRet;
+}
+
+void *(OSAllocZMem)(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS)
+{
+	void *pvRet = NULL;
+
+	if ((ui32Size + ALLOCMEM_PID_SIZE_PADDING) <= g_ui32kmallocThreshold)
+	{
+		pvRet = kzalloc(ui32Size + ALLOCMEM_PID_SIZE_PADDING, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vzalloc(ui32Size);
+	}
+
+	if (pvRet != NULL)
+	{
+		pvRet = _pvr_alloc_stats_add(pvRet, ui32Size DEBUG_MEMSTATS_ARGS);
+	}
+
+	return pvRet;
+}
+
+/*
+ * The parentheses around OSFreeMem prevent the macro in allocmem.h from
+ * applying, as it would break the function's definition.
+ */
+void (OSFreeMem)(void *pvMem)
+{
+	if (pvMem != NULL)
+	{
+		pvMem = _pvr_alloc_stats_remove(pvMem);
+
+		if (!is_vmalloc_addr(pvMem))
+		{
+			_pvr_kfree(pvMem);
+		}
+		else
+		{
+			_pvr_vfree(pvMem);
+		}
+	}
+}
+
+void *OSAllocMemNoStats(IMG_UINT32 ui32Size)
+{
+	void *pvRet = NULL;
+
+	if (ui32Size <= g_ui32kmallocThreshold)
+	{
+		pvRet = kmalloc(ui32Size, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vmalloc(ui32Size);
+	}
+
+	return pvRet;
+}
+
+void *OSAllocZMemNoStats(IMG_UINT32 ui32Size)
+{
+	void *pvRet = NULL;
+
+	if (ui32Size <= g_ui32kmallocThreshold)
+	{
+		pvRet = kzalloc(ui32Size, GFP_KERNEL);
+		if (pvRet == NULL)
+		{
+			OSTryDecreaseKmallocThreshold();
+		}
+		else
+		{
+			OSResetKmallocFailCount();
+		}
+	}
+
+	if (pvRet == NULL)
+	{
+		pvRet = vzalloc(ui32Size);
+	}
+
+	return pvRet;
+}
+
+/*
+ * The parentheses around OSFreeMemNoStats prevent the macro in allocmem.h from
+ * applying, as it would break the function's definition.
+ */
+void (OSFreeMemNoStats)(void *pvMem)
+{
+	if (pvMem != NULL)
+	{
+		if (!is_vmalloc_addr(pvMem))
+		{
+			_pvr_kfree(pvMem);
+		}
+		else
+		{
+			_pvr_vfree(pvMem);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/allocmem.h b/drivers/gpu/drm/img-rogue/allocmem.h
new file mode 100644
index 000000000000..3de9e6781fbd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/allocmem.h
@@ -0,0 +1,224 @@
+/*************************************************************************/ /*!
+@File           allocmem.h
+@Title          memory allocation header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory-Allocation API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef ALLOCMEM_H
+#define ALLOCMEM_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * PVRSRV_ENABLE_PROCESS_STATS enables process statistics regarding events,
+ *     resources and memory across all processes
+ * PVRSRV_ENABLE_MEMORY_STATS enables recording of Linux kernel memory
+ *     allocations, provided that PVRSRV_ENABLE_PROCESS_STATS is enabled
+ *   - Output can be found in:
+ *     /(sys/kernel/debug|proc)/pvr/proc_stats/[live|retired]_pids_stats/mem_area
+ * PVRSRV_DEBUG_LINUX_MEMORY_STATS provides more details about memory
+ *     statistics in conjunction with PVRSRV_ENABLE_MEMORY_STATS
+ * PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON is defined to encompass both memory
+ *     allocation statistics functionalities described above in a single macro
+ */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS) && defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) && defined(DEBUG)
+#define PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+#endif
+
+/*
+ * When using detailed memory allocation statistics, the line number and
+ * file name where the allocation happened are also provided.
+ * When this feature is not used, these parameters are not needed.
+ */
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+#define DEBUG_MEMSTATS_PARAMS ,void *pvAllocFromFile, IMG_UINT32 ui32AllocFromLine
+#define DEBUG_MEMSTATS_ARGS   ,pvAllocFromFile, ui32AllocFromLine
+#define DEBUG_MEMSTATS_UNREF  (void)pvAllocFromFile; (void)ui32AllocFromLine;
+#define DEBUG_MEMSTATS_VALUES ,__FILE__, __LINE__
+#else
+#define DEBUG_MEMSTATS_PARAMS /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_ARGS   /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_UNREF  /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#define DEBUG_MEMSTATS_VALUES  /*!<
+                                 * Used for PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON
+                                 * build option. */
+#endif
+
+
+/**************************************************************************/ /*!
+@Function       OSAllocMem
+@Description    Allocates CPU memory. Contents are uninitialized.
+                If passed a size of zero, function should not assert,
+                but just return a NULL pointer.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+#if defined(DOXYGEN)
+void *OSAllocMem(IMG_UINT32 ui32Size);
+#else
+void *OSAllocMem(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS);
+#define OSAllocMem(_size)	(OSAllocMem)((_size) DEBUG_MEMSTATS_VALUES)
+#endif
+
+/**************************************************************************/ /*!
+@Function       OSAllocZMem
+@Description    Allocates CPU memory and initializes the contents to zero.
+                If passed a size of zero, function should not assert,
+                but just return a NULL pointer.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+#if defined(DOXYGEN)
+void *OSAllocZMem(IMG_UINT32 ui32Size);
+#else
+void *OSAllocZMem(IMG_UINT32 ui32Size DEBUG_MEMSTATS_PARAMS);
+#define OSAllocZMem(_size)	(OSAllocZMem)((_size) DEBUG_MEMSTATS_VALUES)
+#endif
+
+
+/**************************************************************************/ /*!
+@Function       OSAllocMemNoStats
+@Description    Allocates CPU memory. Contents are uninitialized.
+                 If passed a size of zero, function should not assert,
+                 but just return a NULL pointer.
+                 The allocated memory is not accounted for by process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSAllocMem() and OSAllocMemNoStats() equate to
+                 the same operation.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+void *OSAllocMemNoStats(IMG_UINT32 ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSAllocZMemNoStats
+@Description    Allocates CPU memory and initializes the contents to zero.
+                 If passed a size of zero, function should not assert,
+                 but just return a NULL pointer.
+                 The allocated memory is not accounted for by process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSAllocZMem() and OSAllocZMemNoStats() equate to
+                 the same operation.
+@Input          ui32Size        Size of required allocation (in bytes)
+@Return         Pointer to allocated memory on success.
+                Otherwise NULL.
+ */ /**************************************************************************/
+void *OSAllocZMemNoStats(IMG_UINT32 ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSFreeMem
+@Description    Frees previously allocated CPU memory.
+@Input          pvCpuVAddr       Pointer to the memory to be freed.
+@Return         None.
+ */ /**************************************************************************/
+void OSFreeMem(void *pvCpuVAddr);
+
+/**************************************************************************/ /*!
+@Function       OSFreeMemNoStats
+@Description    Frees previously allocated CPU memory.
+                 The freed memory does not update the figures in process stats.
+                 Process stats are an optional feature (enabled only when
+                 PVRSRV_ENABLE_PROCESS_STATS is defined) which track the amount
+                 of memory allocated to help in debugging. Where this is not
+                 required, OSFreeMem() and OSFreeMemNoStats() equate to the
+                 same operation.
+@Input          pvCpuVAddr       Pointer to the memory to be freed.
+@Return         None.
+ */ /**************************************************************************/
+void OSFreeMemNoStats(void *pvCpuVAddr);
+
+/*
+ * These macros allow us to catch double-free bugs on DEBUG builds and
+ * prevent crashes on RELEASE builds.
+ */
+
+/*! @cond Doxygen_Suppress */
+#if defined(DEBUG)
+#define double_free_sentinel ((void *)&OSFreeMem)
+#define ALLOCMEM_ASSERT(exp) PVR_ASSERT(exp)
+#else
+#define double_free_sentinel NULL
+#define ALLOCMEM_ASSERT(exp) do {} while (0)
+#endif
+/*! @endcond */
+
+/*! Frees memory allocated by OSAllocMem(). */
+#define OSFreeMem(_ptr) do { \
+		ALLOCMEM_ASSERT((_ptr) != double_free_sentinel); \
+		(OSFreeMem)(_ptr); \
+		(_ptr) = double_free_sentinel; \
+		MSC_SUPPRESS_4127 \
+	} while (0)
+
+/*! Frees memory allocated by OSAllocMemNoStats(). */
+#define OSFreeMemNoStats(_ptr) do { \
+		ALLOCMEM_ASSERT((_ptr) != double_free_sentinel); \
+		(OSFreeMemNoStats)(_ptr); \
+		(_ptr) = double_free_sentinel; \
+		MSC_SUPPRESS_4127 \
+	} while (0)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* ALLOCMEM_H */
+
+/******************************************************************************
+ End of file (allocmem.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/apollo.mk b/drivers/gpu/drm/img-rogue/apollo/apollo.mk
new file mode 100644
index 000000000000..6d34673e40be
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/apollo.mk
@@ -0,0 +1,4 @@
+apollo-y += \
+ tc_apollo.o \
+ tc_drv.o \
+ tc_odin.o
diff --git a/drivers/gpu/drm/img-rogue/apollo/apollo_regs.h b/drivers/gpu/drm/img-rogue/apollo/apollo_regs.h
new file mode 100644
index 000000000000..4081e2123ac7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/apollo_regs.h
@@ -0,0 +1,108 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(APOLLO_REGS_H)
+#define APOLLO_REGS_H
+
+#include "tc_clocks.h"
+
+/* TC TCF5 */
+#define TC5_SYS_APOLLO_REG_PCI_BASENUM (1)
+#define TC5_SYS_APOLLO_REG_PDP2_OFFSET (0x800000)
+#define TC5_SYS_APOLLO_REG_PDP2_SIZE   (0x7C4)
+
+#define TC5_SYS_APOLLO_REG_PDP2_FBDC_OFFSET (0xA00000)
+#define TC5_SYS_APOLLO_REG_PDP2_FBDC_SIZE   (0x14)
+
+#define TC5_SYS_APOLLO_REG_HDMI_OFFSET (0xC00000)
+#define TC5_SYS_APOLLO_REG_HDMI_SIZE   (0x1C)
+
+/* TC ES2 */
+#define TCF_TEMP_SENSOR_SPI_OFFSET	0xe
+#define TCF_TEMP_SENSOR_TO_C(raw)	(((raw) * 248 / 4096) - 54)
+
+/* Number of bytes that are broken */
+#define SYS_DEV_MEM_BROKEN_BYTES	(1024 * 1024)
+#define SYS_DEV_MEM_REGION_SIZE		(0x40000000 - SYS_DEV_MEM_BROKEN_BYTES)
+
+/* Apollo reg on base register 0 */
+#define SYS_APOLLO_REG_PCI_BASENUM	(0)
+#define SYS_APOLLO_REG_REGION_SIZE	(0x00010000)
+
+#define SYS_APOLLO_REG_SYS_OFFSET	(0x0000)
+#define SYS_APOLLO_REG_SYS_SIZE		(0x0400)
+
+#define SYS_APOLLO_REG_PLL_OFFSET	(0x1000)
+#define SYS_APOLLO_REG_PLL_SIZE		(0x0400)
+
+#define SYS_APOLLO_REG_HOST_OFFSET	(0x4050)
+#define SYS_APOLLO_REG_HOST_SIZE	(0x0014)
+
+#define SYS_APOLLO_REG_PDP1_OFFSET	(0xC000)
+#define SYS_APOLLO_REG_PDP1_SIZE	(0x2000)
+
+/* Offsets for flashing Apollo PROMs from base 0 */
+#define APOLLO_FLASH_STAT_OFFSET	(0x4058)
+#define APOLLO_FLASH_DATA_WRITE_OFFSET	(0x4050)
+#define APOLLO_FLASH_RESET_OFFSET	(0x4060)
+
+#define APOLLO_FLASH_FIFO_STATUS_MASK	 (0xF)
+#define APOLLO_FLASH_FIFO_STATUS_SHIFT	 (0)
+#define APOLLO_FLASH_PROGRAM_STATUS_MASK (0xF)
+#define APOLLO_FLASH_PROGRAM_STATUS_SHIFT (16)
+
+#define APOLLO_FLASH_PROG_COMPLETE_BIT	(0x1)
+#define APOLLO_FLASH_PROG_PROGRESS_BIT	(0x2)
+#define APOLLO_FLASH_PROG_FAILED_BIT	(0x4)
+#define APOLLO_FLASH_INV_FILETYPE_BIT	(0x8)
+
+#define APOLLO_FLASH_FIFO_SIZE		(8)
+
+/* RGX reg on base register 1 */
+#define SYS_RGX_REG_PCI_BASENUM		(1)
+#define SYS_RGX_REG_REGION_SIZE		(0x7FFFF)
+
+/* Device memory (including HP mapping) on base register 2 */
+#define SYS_DEV_MEM_PCI_BASENUM		(2)
+
+#endif /* APOLLO_REGS_H */
diff --git a/drivers/gpu/drm/img-rogue/apollo/bonnie_tcf.h b/drivers/gpu/drm/img-rogue/apollo/bonnie_tcf.h
new file mode 100644
index 000000000000..fc87ec790df9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/bonnie_tcf.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* bonnie_tcf.h - Bonnie TCF register definitions */
+
+/* tab size 4 */
+
+#ifndef BONNIE_TCF_DEFS_H
+#define BONNIE_TCF_DEFS_H
+
+#define BONNIE_TCF_OFFSET_BONNIETC_REGBANK							0x00000000
+#define BONNIE_TCF_OFFSET_TC_IFACE_COUNTERS							0x00004000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_IMGV4_RTM_TOP				0x00008000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_TCF_SCRATCH_PAD_SECN		0x0000C000
+#define BONNIE_TCF_OFFSET_TC_TEST_MODULE_TCF_SCRATCH_PAD_DBG		0x00010000
+#define BONNIE_TCF_OFFSET_MULTI_CLK_ALIGN							0x00014000
+#define BONNIE_TCF_OFFSET_ALIGN_DATA_TX								0x00018000
+#define BONNIE_TCF_OFFSET_SAI_RX_1									0x0001C000
+#define BONNIE_TCF_OFFSET_SAI_RX_SDR								0x00040000
+#define BONNIE_TCF_OFFSET_SAI_TX_1									0x00044000
+#define BONNIE_TCF_OFFSET_SAI_TX_SDR								0x00068000
+
+#define BONNIE_TCF_OFFSET_SAI_RX_DELTA								0x00004000
+#define BONNIE_TCF_OFFSET_SAI_TX_DELTA								0x00004000
+
+#define BONNIE_TCF_OFFSET_SAI_CLK_TAPS								0x0000000C
+#define BONNIE_TCF_OFFSET_SAI_EYES									0x00000010
+#define BONNIE_TCF_OFFSET_SAI_TRAIN_ACK								0x00000018
+
+
+#endif /* BONNIE_TCF_DEFS_H */
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_crtc.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_crtc.c
new file mode 100644
index 000000000000..3e39517fe308
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_crtc.c
@@ -0,0 +1,1086 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "pvr_linux_fence.h"
+#include "drm_pdp_drv.h"
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_vblank.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include "pvr_dma_resv.h"
+#include "drm_pdp_gem.h"
+
+#include "pdp_apollo.h"
+#include "pdp_odin.h"
+#include "pdp_plato.h"
+
+#include "plato_drv.h"
+
+#if defined(PDP_USE_ATOMIC)
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#endif
+
+#include "kernel_compatibility.h"
+
+enum pdp_crtc_flip_status {
+	PDP_CRTC_FLIP_STATUS_NONE = 0,
+	PDP_CRTC_FLIP_STATUS_PENDING,
+	PDP_CRTC_FLIP_STATUS_DONE,
+};
+
+struct pdp_flip_data {
+	struct dma_fence_cb base;
+	struct drm_crtc *crtc;
+	struct dma_fence *wait_fence;
+};
+
+/* returns true for ok, false for fail */
+static bool pdp_clocks_set(struct drm_crtc *crtc,
+				struct drm_display_mode *adjusted_mode)
+{
+	struct pdp_drm_private *dev_priv = crtc->dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	bool res;
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN: {
+		pdp_odin_set_updates_enabled(crtc->dev->dev,
+						pdp_crtc->pdp_reg, false);
+		res = pdp_odin_clocks_set(crtc->dev->dev,
+				pdp_crtc->pdp_reg, pdp_crtc->pll_reg,
+				0,                       /* apollo only */
+				dev_priv->outdev - 1,
+				pdp_crtc->odn_core_reg,  /* odin only */
+				adjusted_mode->hdisplay,
+				adjusted_mode->vdisplay,
+				dev_priv->subversion);
+		pdp_odin_set_updates_enabled(crtc->dev->dev,
+					     pdp_crtc->pdp_reg, true);
+
+		break;
+	}
+	case PDP_VERSION_APOLLO: {
+		int clock_in_mhz = adjusted_mode->clock / 1000;
+
+		pdp_apollo_set_updates_enabled(crtc->dev->dev,
+					       pdp_crtc->pdp_reg, false);
+		res = pdp_apollo_clocks_set(crtc->dev->dev,
+				pdp_crtc->pdp_reg, pdp_crtc->pll_reg,
+				clock_in_mhz,           /* apollo only */
+				NULL,                   /* odin only */
+				adjusted_mode->hdisplay,
+				adjusted_mode->vdisplay);
+		pdp_apollo_set_updates_enabled(crtc->dev->dev,
+					       pdp_crtc->pdp_reg, true);
+
+		DRM_DEBUG_DRIVER("pdp clock set to %dMhz\n", clock_in_mhz);
+
+		break;
+	}
+	case PDP_VERSION_PLATO:
+#if defined(SUPPORT_PLATO_DISPLAY)
+		plato_enable_pdp_clock(dev_priv->dev->dev->parent);
+		res = true;
+#else
+		DRM_ERROR("Trying to enable plato PDP clock on non-Plato build\n");
+		res = false;
+#endif
+		break;
+	default:
+		BUG();
+	}
+
+	return res;
+}
+
+void pdp_crtc_set_plane_enabled(struct drm_crtc *crtc, bool enable)
+{
+	struct pdp_drm_private *dev_priv = crtc->dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN:
+		pdp_odin_set_plane_enabled(crtc->dev->dev,
+					   pdp_crtc->pdp_reg,
+					   0, enable);
+		break;
+	case PDP_VERSION_APOLLO:
+		pdp_apollo_set_plane_enabled(crtc->dev->dev,
+					     pdp_crtc->pdp_reg,
+					     0, enable);
+		break;
+	case PDP_VERSION_PLATO:
+		pdp_plato_set_plane_enabled(crtc->dev->dev,
+					    pdp_crtc->pdp_reg,
+					    0, enable);
+		break;
+	default:
+		BUG();
+	}
+}
+
+static void pdp_crtc_set_syncgen_enabled(struct drm_crtc *crtc, bool enable)
+{
+	struct pdp_drm_private *dev_priv = crtc->dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN:
+		pdp_odin_set_syncgen_enabled(crtc->dev->dev,
+					     pdp_crtc->pdp_reg,
+					     enable);
+		break;
+	case PDP_VERSION_APOLLO:
+		pdp_apollo_set_syncgen_enabled(crtc->dev->dev,
+					       pdp_crtc->pdp_reg,
+					       enable);
+		break;
+	case PDP_VERSION_PLATO:
+		pdp_plato_set_syncgen_enabled(crtc->dev->dev,
+					      pdp_crtc->pdp_reg,
+					      enable);
+		break;
+	default:
+		BUG();
+	}
+}
+
+static void pdp_crtc_set_enabled(struct drm_crtc *crtc, bool enable)
+{
+	struct pdp_drm_private *dev_priv = crtc->dev->dev_private;
+
+	if (enable) {
+		pdp_crtc_set_syncgen_enabled(crtc, enable);
+		pdp_crtc_set_plane_enabled(crtc, dev_priv->display_enabled);
+		drm_crtc_vblank_on(crtc);
+	} else {
+		drm_crtc_vblank_off(crtc);
+		pdp_crtc_set_plane_enabled(crtc, enable);
+		pdp_crtc_set_syncgen_enabled(crtc, enable);
+	}
+}
+
+static void pdp_crtc_mode_set(struct drm_crtc *crtc,
+			      struct drm_display_mode *adjusted_mode)
+{
+	/*
+	 * ht   = horizontal total
+	 * hbps = horizontal back porch start
+	 * has  = horizontal active start
+	 * hlbs = horizontal left border start
+	 * hfps = horizontal front porch start
+	 * hrbs = horizontal right border start
+	 *
+	 * vt   = vertical total
+	 * vbps = vertical back porch start
+	 * vas  = vertical active start
+	 * vtbs = vertical top border start
+	 * vfps = vertical front porch start
+	 * vbbs = vertical bottom border start
+	 */
+	struct pdp_drm_private *dev_priv = crtc->dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	uint32_t ht = adjusted_mode->htotal;
+	uint32_t hbps = adjusted_mode->hsync_end - adjusted_mode->hsync_start;
+	uint32_t has = (adjusted_mode->htotal - adjusted_mode->hsync_start);
+	uint32_t hlbs = has;
+	uint32_t hfps = (hlbs + adjusted_mode->hdisplay);
+	uint32_t hrbs = hfps;
+	uint32_t vt = adjusted_mode->vtotal;
+	uint32_t vbps = adjusted_mode->vsync_end - adjusted_mode->vsync_start;
+	uint32_t vas = (adjusted_mode->vtotal - adjusted_mode->vsync_start);
+	uint32_t vtbs = vas;
+	uint32_t vfps = (vtbs + adjusted_mode->vdisplay);
+	uint32_t vbbs = vfps;
+	bool ok;
+
+	ok = pdp_clocks_set(crtc, adjusted_mode);
+
+	if (!ok) {
+		dev_info(crtc->dev->dev, "%s failed\n", __func__);
+		return;
+	}
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN:
+		pdp_odin_set_updates_enabled(crtc->dev->dev,
+					     pdp_crtc->pdp_reg, false);
+		pdp_odin_reset_planes(crtc->dev->dev,
+				      pdp_crtc->pdp_reg);
+		pdp_odin_mode_set(crtc->dev->dev,
+			     pdp_crtc->pdp_reg,
+			     adjusted_mode->hdisplay, adjusted_mode->vdisplay,
+			     hbps, ht, has,
+			     hlbs, hfps, hrbs,
+			     vbps, vt, vas,
+			     vtbs, vfps, vbbs,
+			     adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC,
+			     adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC,
+			     pdp_crtc->pfim_reg);
+		pdp_odin_set_powerdwn_enabled(crtc->dev->dev,
+					      pdp_crtc->pdp_reg, false);
+		pdp_odin_set_updates_enabled(crtc->dev->dev,
+					     pdp_crtc->pdp_reg, true);
+		break;
+	case PDP_VERSION_APOLLO:
+		pdp_apollo_set_updates_enabled(crtc->dev->dev,
+					       pdp_crtc->pdp_reg, false);
+		pdp_apollo_reset_planes(crtc->dev->dev,
+					pdp_crtc->pdp_reg);
+		pdp_apollo_mode_set(crtc->dev->dev,
+			     pdp_crtc->pdp_reg,
+			     adjusted_mode->hdisplay, adjusted_mode->vdisplay,
+			     hbps, ht, has,
+			     hlbs, hfps, hrbs,
+			     vbps, vt, vas,
+			     vtbs, vfps, vbbs,
+			     adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC,
+			     adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC);
+		pdp_apollo_set_powerdwn_enabled(crtc->dev->dev,
+						pdp_crtc->pdp_reg, false);
+		pdp_apollo_set_updates_enabled(crtc->dev->dev,
+					       pdp_crtc->pdp_reg, true);
+		break;
+	case PDP_VERSION_PLATO:
+		pdp_plato_mode_set(crtc->dev->dev,
+				   pdp_crtc->pdp_reg,
+				   adjusted_mode->hdisplay,
+				   adjusted_mode->vdisplay,
+				   hbps, ht, has,
+				   hlbs, hfps, hrbs,
+				   vbps, vt, vas,
+				   vtbs, vfps, vbbs,
+				   adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC,
+				   adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC);
+		break;
+	default:
+		BUG();
+	}
+}
+
+
+static bool pdp_crtc_helper_mode_fixup(struct drm_crtc *crtc,
+					const struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode)
+{
+	struct pdp_drm_private *dev_priv = crtc->dev->dev_private;
+
+	if (dev_priv->version == PDP_VERSION_ODIN
+		&& mode->hdisplay == 1920
+		&& mode->vdisplay == 1080) {
+
+		/* 1080p 60Hz */
+		const int h_total = 2200;
+		const int h_active_start = 192;
+		const int h_back_porch_start = 44;
+		const int v_total = 1125;
+		const int v_active_start = 41;
+		const int v_back_porch_start = 5;
+
+		adjusted_mode->htotal = h_total;
+		adjusted_mode->hsync_start = adjusted_mode->htotal -
+						h_active_start;
+		adjusted_mode->hsync_end = adjusted_mode->hsync_start +
+						h_back_porch_start;
+		adjusted_mode->vtotal = v_total;
+		adjusted_mode->vsync_start = adjusted_mode->vtotal -
+						v_active_start;
+		adjusted_mode->vsync_end = adjusted_mode->vsync_start +
+						v_back_porch_start;
+	}
+	return true;
+}
+
+static void pdp_crtc_flip_complete(struct drm_crtc *crtc);
+
+#if defined(PDP_USE_ATOMIC)
+static void pdp_crtc_helper_mode_set_nofb(struct drm_crtc *crtc)
+{
+	pdp_crtc_mode_set(crtc, &crtc->state->adjusted_mode);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void pdp_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+					 struct drm_crtc_state *old_crtc_state)
+{
+#else
+static void pdp_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+					 struct drm_atomic_state *state)
+{
+	struct drm_crtc_state *old_crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+	struct drm_crtc_state *new_crtc_state = crtc->state;
+
+	if (!new_crtc_state->active || !old_crtc_state->active)
+		return;
+
+	if (crtc->state->event) {
+		struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+		unsigned long flags;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+		pdp_crtc->flip_async = new_crtc_state->async_flip;
+#else
+		pdp_crtc->flip_async = !!(new_crtc_state->pageflip_flags
+					  & DRM_MODE_PAGE_FLIP_ASYNC);
+#endif
+		if (pdp_crtc->flip_async)
+			WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		pdp_crtc->flip_event = crtc->state->event;
+		crtc->state->event = NULL;
+
+		atomic_set(&pdp_crtc->flip_status, PDP_CRTC_FLIP_STATUS_DONE);
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+		if (pdp_crtc->flip_async)
+			pdp_crtc_flip_complete(crtc);
+	}
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void pdp_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					  struct drm_crtc_state *old_crtc_state)
+#else
+static void pdp_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					  struct drm_atomic_state *state)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+{
+	pdp_crtc_set_enabled(crtc, true);
+
+	if (crtc->state->event) {
+		struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+		unsigned long flags;
+
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		pdp_crtc->flip_event = crtc->state->event;
+		crtc->state->event = NULL;
+
+		atomic_set(&pdp_crtc->flip_status, PDP_CRTC_FLIP_STATUS_DONE);
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+	}
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void pdp_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					   struct drm_crtc_state *old_crtc_state)
+#else
+static void pdp_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					   struct drm_atomic_state *state)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+{
+	pdp_crtc_set_enabled(crtc, false);
+
+	if (crtc->state->event) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+	}
+}
+#else
+static void pdp_crtc_helper_dpms(struct drm_crtc *crtc, int mode)
+{
+}
+
+static void pdp_crtc_helper_prepare(struct drm_crtc *crtc)
+{
+	pdp_crtc_set_enabled(crtc, false);
+}
+
+static void pdp_crtc_helper_commit(struct drm_crtc *crtc)
+{
+	pdp_crtc_set_enabled(crtc, true);
+}
+
+static int pdp_crtc_helper_mode_set_base_atomic(struct drm_crtc *crtc,
+						struct drm_framebuffer *fb,
+						int x, int y,
+						enum mode_set_atomic atomic)
+{
+	if (x < 0 || y < 0)
+		return -EINVAL;
+
+	pdp_plane_set_surface(crtc, crtc->primary, fb,
+			      (uint32_t) x, (uint32_t) y);
+
+	return 0;
+}
+
+static int pdp_crtc_helper_mode_set_base(struct drm_crtc *crtc,
+					 int x, int y,
+					 struct drm_framebuffer *old_fb)
+{
+	if (!crtc->primary->fb) {
+		DRM_ERROR("no framebuffer\n");
+		return 0;
+	}
+
+	return pdp_crtc_helper_mode_set_base_atomic(crtc,
+						    crtc->primary->fb,
+						    x, y,
+						    0);
+}
+
+static int pdp_crtc_helper_mode_set(struct drm_crtc *crtc,
+				    struct drm_display_mode *mode,
+				    struct drm_display_mode *adjusted_mode,
+				    int x, int y,
+				    struct drm_framebuffer *old_fb)
+{
+	pdp_crtc_mode_set(crtc, adjusted_mode);
+
+	return pdp_crtc_helper_mode_set_base(crtc, x, y, old_fb);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+static void pdp_crtc_helper_load_lut(struct drm_crtc *crtc)
+{
+}
+#endif
+
+static void pdp_crtc_helper_disable(struct drm_crtc *crtc)
+{
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	enum pdp_crtc_flip_status status;
+
+	pdp_crtc_set_enabled(crtc, false);
+
+	status = atomic_read(&pdp_crtc->flip_status);
+	if (status != PDP_CRTC_FLIP_STATUS_NONE) {
+		long lerr;
+
+		lerr = wait_event_timeout(
+			pdp_crtc->flip_pending_wait_queue,
+			atomic_read(&pdp_crtc->flip_status)
+					!= PDP_CRTC_FLIP_STATUS_PENDING,
+			30 * HZ);
+		if (!lerr)
+			DRM_ERROR("Failed to wait for pending flip\n");
+		else if (!pdp_crtc->flip_async)
+			pdp_crtc_flip_complete(crtc);
+	}
+}
+#endif /* defined(PDP_USE_ATOMIC) */
+
+static int pfim_init(struct drm_device *dev,
+		     struct pdp_crtc *pdp_crtc,
+		     const char *crtc_name)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	struct resource *regs;
+	int err;
+
+	if (!dev_priv->pfim_capable) {
+		pdp_crtc->pfim_reg = NULL;
+		return 0;
+	}
+
+	regs = platform_get_resource_byname(to_platform_device(dev->dev),
+					    IORESOURCE_MEM,
+					    "pfim-regs");
+	if (!regs) {
+		DRM_ERROR("missing pfim register info\n");
+		return -ENXIO;
+	}
+
+	pdp_crtc->pfim_reg_phys_base = regs->start;
+	pdp_crtc->pfim_reg_size = resource_size(regs);
+
+	if (!request_mem_region(pdp_crtc->pfim_reg_phys_base,
+				pdp_crtc->pfim_reg_size,
+				crtc_name)) {
+		DRM_ERROR("failed to reserve pfim registers\n");
+		return -EBUSY;
+	}
+
+	pdp_crtc->pfim_reg =
+		ioremap(pdp_crtc->pfim_reg_phys_base, pdp_crtc->pfim_reg_size);
+	if (!pdp_crtc->pfim_reg) {
+		DRM_ERROR("failed to map pfim registers\n");
+		err = -ENOMEM;
+		goto err_release_mem;
+	}
+	return 0;
+
+err_release_mem:
+	release_mem_region(pdp_crtc->pfim_reg_phys_base,
+			   pdp_crtc->pfim_reg_size);
+	pdp_crtc->pfim_reg = NULL;
+	return err;
+}
+
+static void pfim_deinit(struct pdp_crtc *pdp_crtc)
+{
+	if (pdp_crtc->pfim_reg) {
+		iounmap(pdp_crtc->pfim_reg);
+		release_mem_region(pdp_crtc->pfim_reg_phys_base,
+				   pdp_crtc->pfim_reg_size);
+		pdp_crtc->pfim_reg = NULL;
+	}
+}
+
+static void pdp_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+
+	DRM_DEBUG_DRIVER("[CRTC:%d]\n", crtc->base.id);
+
+	drm_crtc_cleanup(crtc);
+
+	iounmap(pdp_crtc->pll_reg);
+
+	iounmap(pdp_crtc->pdp_reg);
+	release_mem_region(pdp_crtc->pdp_reg_phys_base, pdp_crtc->pdp_reg_size);
+
+	pfim_deinit(pdp_crtc);
+
+	kfree(pdp_crtc);
+	dev_priv->crtc = NULL;
+}
+
+static void pdp_crtc_flip_complete(struct drm_crtc *crtc)
+{
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	unsigned long flags;
+
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
+
+	/* The flipping process has been completed so reset the flip state */
+	atomic_set(&pdp_crtc->flip_status, PDP_CRTC_FLIP_STATUS_NONE);
+	pdp_crtc->flip_async = false;
+
+#if !defined(PDP_USE_ATOMIC)
+	if (pdp_crtc->flip_data) {
+		dma_fence_put(pdp_crtc->flip_data->wait_fence);
+		kfree(pdp_crtc->flip_data);
+		pdp_crtc->flip_data = NULL;
+	}
+#endif
+
+	if (pdp_crtc->flip_event) {
+		drm_crtc_send_vblank_event(crtc, pdp_crtc->flip_event);
+		pdp_crtc->flip_event = NULL;
+	}
+
+	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+}
+
+#if !defined(PDP_USE_ATOMIC)
+static void pdp_crtc_flip(struct drm_crtc *crtc)
+{
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	struct drm_framebuffer *old_fb;
+
+	WARN_ON(atomic_read(&to_pdp_crtc(crtc)->flip_status)
+			!= PDP_CRTC_FLIP_STATUS_PENDING);
+
+	old_fb = pdp_crtc->old_fb;
+	pdp_crtc->old_fb = NULL;
+
+	/*
+	 * The graphics stream registers latch on vsync so we can go ahead and
+	 * do the flip now.
+	 */
+	(void) pdp_crtc_helper_mode_set_base(crtc, crtc->x, crtc->y, old_fb);
+
+	atomic_set(&pdp_crtc->flip_status, PDP_CRTC_FLIP_STATUS_DONE);
+	wake_up(&pdp_crtc->flip_pending_wait_queue);
+
+	if (pdp_crtc->flip_async)
+		pdp_crtc_flip_complete(crtc);
+}
+
+static void pdp_crtc_flip_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
+{
+	struct pdp_flip_data *flip_data =
+		container_of(cb, struct pdp_flip_data, base);
+
+	pdp_crtc_flip(flip_data->crtc);
+}
+
+static void pdp_crtc_flip_schedule_cb(struct dma_fence *fence,
+				      struct dma_fence_cb *cb)
+{
+	struct pdp_flip_data *flip_data =
+		container_of(cb, struct pdp_flip_data, base);
+	int err = 0;
+
+	if (flip_data->wait_fence)
+		err = dma_fence_add_callback(flip_data->wait_fence,
+					     &flip_data->base,
+					     pdp_crtc_flip_cb);
+
+	if (!flip_data->wait_fence || err) {
+		if (err && err != -ENOENT)
+			DRM_ERROR("flip failed to wait on old buffer\n");
+		pdp_crtc_flip_cb(flip_data->wait_fence, &flip_data->base);
+	}
+}
+
+static int pdp_crtc_flip_schedule(struct drm_crtc *crtc,
+				  struct drm_gem_object *obj,
+				  struct drm_gem_object *old_obj)
+{
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	struct dma_resv *resv = pdp_gem_get_resv(obj);
+	struct dma_resv *old_resv = pdp_gem_get_resv(old_obj);
+	struct pdp_flip_data *flip_data;
+	struct dma_fence *fence;
+	int err;
+
+	flip_data = kmalloc(sizeof(*flip_data), GFP_KERNEL);
+	if (!flip_data)
+		return -ENOMEM;
+
+	flip_data->crtc = crtc;
+
+	ww_mutex_lock(&old_resv->lock, NULL);
+	flip_data->wait_fence =
+		dma_fence_get(dma_resv_get_excl(old_resv));
+
+	if (old_resv != resv) {
+		ww_mutex_unlock(&old_resv->lock);
+		ww_mutex_lock(&resv->lock, NULL);
+	}
+
+	fence = dma_fence_get(dma_resv_get_excl(resv));
+	ww_mutex_unlock(&resv->lock);
+
+	pdp_crtc->flip_data = flip_data;
+	atomic_set(&pdp_crtc->flip_status, PDP_CRTC_FLIP_STATUS_PENDING);
+
+	if (fence) {
+		err = dma_fence_add_callback(fence, &flip_data->base,
+					     pdp_crtc_flip_schedule_cb);
+		dma_fence_put(fence);
+		if (err && err != -ENOENT)
+			goto err_set_flip_status_none;
+	}
+
+	if (!fence || err == -ENOENT) {
+		pdp_crtc_flip_schedule_cb(fence, &flip_data->base);
+		err = 0;
+	}
+
+	return err;
+
+err_set_flip_status_none:
+	atomic_set(&pdp_crtc->flip_status, PDP_CRTC_FLIP_STATUS_NONE);
+	dma_fence_put(flip_data->wait_fence);
+	kfree(flip_data);
+	return err;
+}
+
+static int pdp_crtc_page_flip(struct drm_crtc *crtc,
+			      struct drm_framebuffer *fb,
+			      struct drm_pending_vblank_event *event,
+			      uint32_t page_flip_flags
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+			      , struct drm_modeset_acquire_ctx *ctx
+#endif
+			     )
+{
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	struct pdp_framebuffer *pdp_fb = to_pdp_framebuffer(fb);
+	struct pdp_framebuffer *pdp_old_fb =
+		to_pdp_framebuffer(crtc->primary->fb);
+	enum pdp_crtc_flip_status status;
+	unsigned long flags;
+	int err;
+
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
+	status = atomic_read(&pdp_crtc->flip_status);
+	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+	if (status != PDP_CRTC_FLIP_STATUS_NONE)
+		return -EBUSY;
+
+	if (!(page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC)) {
+		err = drm_crtc_vblank_get(crtc);
+		if (err)
+			return err;
+	}
+
+	pdp_crtc->old_fb = crtc->primary->fb;
+	pdp_crtc->flip_event = event;
+	pdp_crtc->flip_async = !!(page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC);
+
+	/* Set the crtc primary plane to point to the new framebuffer */
+	crtc->primary->fb = fb;
+
+	err = pdp_crtc_flip_schedule(crtc, pdp_fb->obj[0], pdp_old_fb->obj[0]);
+	if (err) {
+		crtc->primary->fb = pdp_crtc->old_fb;
+		pdp_crtc->old_fb = NULL;
+		pdp_crtc->flip_event = NULL;
+		pdp_crtc->flip_async = false;
+
+		DRM_ERROR("failed to schedule flip (err=%d)\n", err);
+		goto err_vblank_put;
+	}
+
+	return 0;
+
+err_vblank_put:
+	if (!(page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC))
+		drm_crtc_vblank_put(crtc);
+	return err;
+}
+#endif /* !defined(PDP_USE_ATOMIC) */
+
+static const struct drm_crtc_helper_funcs pdp_crtc_helper_funcs = {
+	.mode_fixup = pdp_crtc_helper_mode_fixup,
+#if defined(PDP_USE_ATOMIC)
+	.mode_set_nofb = pdp_crtc_helper_mode_set_nofb,
+	.atomic_flush = pdp_crtc_helper_atomic_flush,
+	.atomic_enable = pdp_crtc_helper_atomic_enable,
+	.atomic_disable = pdp_crtc_helper_atomic_disable,
+#else
+	.dpms = pdp_crtc_helper_dpms,
+	.prepare = pdp_crtc_helper_prepare,
+	.commit = pdp_crtc_helper_commit,
+	.mode_set = pdp_crtc_helper_mode_set,
+	.mode_set_base = pdp_crtc_helper_mode_set_base,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	.load_lut = pdp_crtc_helper_load_lut,
+#endif
+	.mode_set_base_atomic = pdp_crtc_helper_mode_set_base_atomic,
+	.disable = pdp_crtc_helper_disable,
+#endif
+};
+
+static const struct drm_crtc_funcs pdp_crtc_funcs = {
+	.destroy = pdp_crtc_destroy,
+#if defined(PDP_USE_ATOMIC)
+	.reset = drm_atomic_helper_crtc_reset,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+#else
+	.set_config = drm_crtc_helper_set_config,
+	.page_flip = pdp_crtc_page_flip,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	.enable_vblank  = pdp_enable_vblank,
+	.disable_vblank = pdp_disable_vblank,
+#endif
+};
+
+
+struct drm_crtc *pdp_crtc_create(struct drm_device *dev, uint32_t number,
+				 struct drm_plane *primary_plane)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	struct pdp_crtc *pdp_crtc;
+	const char *crtc_name = "crtc-0";
+	int err;
+
+	pdp_crtc = kzalloc(sizeof(*pdp_crtc), GFP_KERNEL);
+	if (!pdp_crtc) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+
+	init_waitqueue_head(&pdp_crtc->flip_pending_wait_queue);
+	atomic_set(&pdp_crtc->flip_status, PDP_CRTC_FLIP_STATUS_NONE);
+	pdp_crtc->number = number;
+
+	switch (number) {
+	case 0:
+	{
+		struct resource *regs;
+		const char *pdp_resname = NULL;
+
+		if (dev_priv->version == PDP_VERSION_ODIN) {
+			switch (dev_priv->outdev) {
+			case PDP_OUTPUT_PDP1: {
+				pdp_resname = "pdp-regs";
+				break;
+			}
+			case PDP_OUTPUT_PDP2: {
+				pdp_resname = "pdp2-regs";
+				break;
+			}
+			default:
+				DRM_ERROR("wrong PDP output device\n");
+				err = -ENODEV;
+				goto err_exit;
+			}
+		} else {
+			pdp_resname = "pdp-regs";
+		}
+
+		regs = platform_get_resource_byname(
+				    to_platform_device(dev->dev),
+				    IORESOURCE_MEM,
+				    pdp_resname);
+		if (!regs) {
+			DRM_ERROR("missing pdp register info\n");
+			err = -ENXIO;
+			goto err_crtc_free;
+		}
+
+		pdp_crtc->pdp_reg_phys_base = regs->start;
+		pdp_crtc->pdp_reg_size = resource_size(regs);
+
+		if (dev_priv->version == PDP_VERSION_ODIN ||
+			dev_priv->version == PDP_VERSION_APOLLO) {
+			regs = platform_get_resource_byname(
+					    to_platform_device(dev->dev),
+					    IORESOURCE_MEM,
+					    "pll-regs");
+			if (!regs) {
+				DRM_ERROR("missing pll register info\n");
+				err = -ENXIO;
+				goto err_crtc_free;
+			}
+
+			pdp_crtc->pll_reg_phys_base = regs->start;
+			pdp_crtc->pll_reg_size = resource_size(regs);
+
+			pdp_crtc->pll_reg = ioremap(pdp_crtc->pll_reg_phys_base,
+						    pdp_crtc->pll_reg_size);
+			if (!pdp_crtc->pll_reg) {
+				DRM_ERROR("failed to map pll registers\n");
+				err = -ENOMEM;
+				goto err_crtc_free;
+			}
+		} else if (dev_priv->version == PDP_VERSION_PLATO) {
+			regs = platform_get_resource_byname(
+				    to_platform_device(dev->dev),
+				    IORESOURCE_MEM,
+				    PLATO_PDP_RESOURCE_BIF_REGS);
+			if (!regs) {
+				DRM_ERROR("missing pdp-bif register info\n");
+				err = -ENXIO;
+				goto err_crtc_free;
+			}
+
+			pdp_crtc->pdp_bif_reg_phys_base = regs->start;
+			pdp_crtc->pdp_bif_reg_size = resource_size(regs);
+
+			if (!request_mem_region(pdp_crtc->pdp_bif_reg_phys_base,
+					pdp_crtc->pdp_bif_reg_size,
+					crtc_name)) {
+				DRM_ERROR("failed to reserve pdp-bif registers\n");
+				err = -EBUSY;
+				goto err_crtc_free;
+			}
+
+			pdp_crtc->pdp_bif_reg =
+				ioremap(pdp_crtc->pdp_bif_reg_phys_base,
+					pdp_crtc->pdp_bif_reg_size);
+			if (!pdp_crtc->pdp_bif_reg) {
+				DRM_ERROR("failed to map pdp-bif registers\n");
+				err = -ENOMEM;
+				goto err_iounmap_regs;
+			}
+		}
+
+		if (dev_priv->version == PDP_VERSION_ODIN) {
+			regs = platform_get_resource_byname(
+					    to_platform_device(dev->dev),
+					    IORESOURCE_MEM,
+					    "odn-core");
+			if (!regs) {
+				DRM_ERROR("missing odn-core info\n");
+				err = -ENXIO;
+				goto err_crtc_free;
+			}
+
+			pdp_crtc->odn_core_phys_base = regs->start;
+			pdp_crtc->odn_core_size = resource_size(regs);
+
+			pdp_crtc->odn_core_reg
+				= ioremap(pdp_crtc->odn_core_phys_base,
+					  pdp_crtc->odn_core_size);
+			if (!pdp_crtc->odn_core_reg) {
+				DRM_ERROR("failed to map pdp reset register\n");
+				err = -ENOMEM;
+				goto err_iounmap_regs;
+			}
+
+			err = pfim_init(dev, pdp_crtc, crtc_name);
+			if (err) {
+				DRM_ERROR("failed to initialise PFIM\n");
+				goto err_iounmap_regs;
+			}
+		}
+
+		break;
+	}
+	default:
+		DRM_ERROR("invalid crtc number %u\n", number);
+		err = -EINVAL;
+		goto err_crtc_free;
+	}
+
+	if (!request_mem_region(pdp_crtc->pdp_reg_phys_base,
+				pdp_crtc->pdp_reg_size,
+				crtc_name)) {
+		DRM_ERROR("failed to reserve pdp registers\n");
+		err = -EBUSY;
+		goto err_crtc_free;
+	}
+
+	pdp_crtc->pdp_reg = ioremap(pdp_crtc->pdp_reg_phys_base,
+				    pdp_crtc->pdp_reg_size);
+	if (!pdp_crtc->pdp_reg) {
+		DRM_ERROR("failed to map pdp registers\n");
+		err = -ENOMEM;
+		goto err_release_mem_region;
+	}
+
+	err = drm_crtc_init_with_planes(dev, &pdp_crtc->base, primary_plane,
+					NULL, &pdp_crtc_funcs, NULL);
+	if (err) {
+		DRM_ERROR("CRTC init with planes failed");
+		goto err_iounmap_regs;
+	}
+
+	drm_crtc_helper_add(&pdp_crtc->base, &pdp_crtc_helper_funcs);
+
+	DRM_DEBUG_DRIVER("[CRTC:%d]\n", pdp_crtc->base.base.id);
+
+	return &pdp_crtc->base;
+
+err_iounmap_regs:
+	iounmap(pdp_crtc->pdp_reg);
+	if (pdp_crtc->odn_core_reg)
+		iounmap(pdp_crtc->odn_core_reg);
+	if (pdp_crtc->pdp_bif_reg)
+		iounmap(pdp_crtc->pdp_bif_reg);
+err_release_mem_region:
+	release_mem_region(pdp_crtc->pdp_reg_phys_base, pdp_crtc->pdp_reg_size);
+	pfim_deinit(pdp_crtc);
+err_crtc_free:
+	kfree(pdp_crtc);
+err_exit:
+	return ERR_PTR(err);
+}
+
+void pdp_crtc_set_vblank_enabled(struct drm_crtc *crtc, bool enable)
+{
+	struct pdp_drm_private *dev_priv = crtc->dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN:
+		pdp_odin_set_vblank_enabled(crtc->dev->dev,
+					    pdp_crtc->pdp_reg,
+					    enable);
+		break;
+	case PDP_VERSION_APOLLO:
+		pdp_apollo_set_vblank_enabled(crtc->dev->dev,
+					    pdp_crtc->pdp_reg,
+					    enable);
+		break;
+	case PDP_VERSION_PLATO:
+		pdp_plato_set_vblank_enabled(crtc->dev->dev,
+					     pdp_crtc->pdp_reg,
+					     enable);
+		break;
+	default:
+		BUG();
+	}
+}
+
+void pdp_crtc_irq_handler(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	bool handled;
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN:
+		handled = pdp_odin_check_and_clear_vblank(dev->dev,
+							  pdp_crtc->pdp_reg);
+		break;
+	case PDP_VERSION_APOLLO:
+		handled = pdp_apollo_check_and_clear_vblank(dev->dev,
+							    pdp_crtc->pdp_reg);
+		break;
+	case PDP_VERSION_PLATO:
+		handled = pdp_plato_check_and_clear_vblank(dev->dev,
+							   pdp_crtc->pdp_reg);
+		break;
+	default:
+		handled = false;
+		break;
+	}
+
+	if (handled) {
+		enum pdp_crtc_flip_status status;
+
+		drm_handle_vblank(dev, pdp_crtc->number);
+
+		status = atomic_read(&pdp_crtc->flip_status);
+		if (status == PDP_CRTC_FLIP_STATUS_DONE) {
+			if (!pdp_crtc->flip_async) {
+				pdp_crtc_flip_complete(crtc);
+#if !defined(PDP_USE_ATOMIC)
+				drm_crtc_vblank_put(crtc);
+#endif
+			}
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_debugfs.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_debugfs.c
new file mode 100644
index 000000000000..949577c9758b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_debugfs.c
@@ -0,0 +1,194 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#include <linux/debugfs.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_debugfs.h>
+#include <drm/drm_file.h>
+#include <linux/uaccess.h>
+#endif
+
+#include "drm_pdp_drv.h"
+
+#define PDP_DEBUGFS_DISPLAY_ENABLED "display_enabled"
+
+static int display_enabled_open(struct inode *inode, struct file *file)
+{
+	file->private_data = inode->i_private;
+
+	return 0;
+}
+
+static ssize_t display_enabled_read(struct file *file,
+				    char __user *user_buffer,
+				    size_t count,
+				    loff_t *position_ptr)
+{
+	struct drm_device *dev = file->private_data;
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	loff_t position = *position_ptr;
+	char buffer[] = "N\n";
+	size_t buffer_size = ARRAY_SIZE(buffer);
+	int err;
+
+	if (position < 0)
+		return -EINVAL;
+	else if (position >= buffer_size || count == 0)
+		return 0;
+
+	if (dev_priv->display_enabled)
+		buffer[0] = 'Y';
+
+	if (count > buffer_size - position)
+		count = buffer_size - position;
+
+	err = copy_to_user(user_buffer, &buffer[position], count);
+	if (err)
+		return -EFAULT;
+
+	*position_ptr = position + count;
+
+	return count;
+}
+
+static ssize_t display_enabled_write(struct file *file,
+				     const char __user *user_buffer,
+				     size_t count,
+				     loff_t *position)
+{
+	struct drm_device *dev = file->private_data;
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	char buffer[3];
+	int err;
+
+	count = min(count, ARRAY_SIZE(buffer) - 1);
+
+	err = copy_from_user(buffer, user_buffer, count);
+	if (err)
+		return -EFAULT;
+	buffer[count] = '\0';
+
+	if (!kstrtobool(buffer, &dev_priv->display_enabled) && dev_priv->crtc)
+		pdp_crtc_set_plane_enabled(dev_priv->crtc, dev_priv->display_enabled);
+
+	return count;
+}
+
+static const struct file_operations pdp_display_enabled_fops = {
+	.owner = THIS_MODULE,
+	.open = display_enabled_open,
+	.read = display_enabled_read,
+	.write = display_enabled_write,
+	.llseek = default_llseek,
+};
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+static int pdp_debugfs_create(struct drm_minor *minor, const char *name,
+			      umode_t mode, const struct file_operations *fops)
+{
+	struct drm_info_node *node;
+
+	/*
+	 * We can't get access to our driver private data when this function is
+	 * called so we fake up a node so that we can clean up entries later on.
+	 */
+	node = kzalloc(sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+
+	node->dent = debugfs_create_file(name, mode, minor->debugfs_root,
+					 minor->dev, fops);
+	if (!node->dent) {
+		kfree(node);
+		return -ENOMEM;
+	}
+
+	node->minor = minor;
+	node->info_ent = (void *) fops;
+
+	mutex_lock(&minor->debugfs_lock);
+	list_add(&node->list, &minor->debugfs_list);
+	mutex_unlock(&minor->debugfs_lock);
+
+	return 0;
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+int pdp_debugfs_init(struct drm_minor *minor)
+#else
+void pdp_debugfs_init(struct drm_minor *minor)
+#endif
+{
+	int err;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	err = pdp_debugfs_create(minor, PDP_DEBUGFS_DISPLAY_ENABLED,
+				 0100644,
+				 &pdp_display_enabled_fops);
+#else
+	struct dentry *dent = debugfs_create_file(PDP_DEBUGFS_DISPLAY_ENABLED,
+						  0100644,
+						  minor->debugfs_root,
+						  minor->dev,
+						  &pdp_display_enabled_fops);
+	err = !dent;
+#endif
+	if (err) {
+		DRM_INFO("failed to create '%s' debugfs entry\n",
+			 PDP_DEBUGFS_DISPLAY_ENABLED);
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	return err;
+#endif
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+void pdp_debugfs_cleanup(struct drm_minor *minor)
+{
+	drm_debugfs_remove_files((struct drm_info_list *) &pdp_display_enabled_fops,
+				 1, minor);
+}
+#endif
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_drv.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_drv.c
new file mode 100644
index 000000000000..d36a0b986f9d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_drv.c
@@ -0,0 +1,795 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/component.h>
+#include <linux/platform_device.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_prime.h>
+#include <drm/drm_print.h>
+#include <drm/drm_vblank.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include "tc_drv.h"
+#include "pvrversion.h"
+
+#include "drm_pdp_drv.h"
+#include "drm_pdp_gem.h"
+#include "pdp_drm.h"
+
+#include "odin_defs.h"
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+#include "plato_drv.h"
+#include "pdp2_regs.h"
+#include "pdp2_mmu_regs.h"
+#endif
+
+#define DRIVER_NAME "pdp"
+#define DRIVER_DESC "Imagination Technologies PDP DRM Display Driver"
+#define DRIVER_DATE "20150612"
+
+#if defined(PDP_USE_ATOMIC)
+#include <drm/drm_atomic_helper.h>
+
+#define PVR_DRIVER_ATOMIC DRIVER_ATOMIC
+#else
+#define PVR_DRIVER_ATOMIC 0
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#define	PVR_DRIVER_PRIME 0
+#else
+#define	PVR_DRIVER_PRIME DRIVER_PRIME
+#endif
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+static bool display_enable = true;
+static unsigned int output_device = 1;
+
+module_param(display_enable, bool, 0444);
+MODULE_PARM_DESC(display_enable, "Enable all displays (default: Y)");
+
+module_param(output_device, uint, 0444);
+MODULE_PARM_DESC(output_device, "PDP output device (default: PDP1)");
+
+struct pdp_gem_private *pdp_gem_get_private(struct drm_device *dev)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+
+	return dev_priv->gem_priv;
+}
+
+static void pdp_irq_handler(void *data)
+{
+	struct drm_device *dev = data;
+	struct drm_crtc *crtc;
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+		pdp_crtc_irq_handler(crtc);
+}
+
+static int pdp_early_load(struct drm_device *dev)
+{
+	struct pdp_drm_private *dev_priv;
+	int err;
+
+	DRM_DEBUG("loading %s device\n", to_platform_device(dev->dev)->name);
+
+	platform_set_drvdata(to_platform_device(dev->dev), dev);
+
+	dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
+	if (!dev_priv)
+		return -ENOMEM;
+
+	dev->dev_private = dev_priv;
+	dev_priv->dev = dev;
+	dev_priv->version = (enum pdp_version)
+		to_platform_device(dev->dev)->id_entry->driver_data;
+	dev_priv->display_enabled = display_enable;
+
+#if !defined(SUPPORT_PLATO_DISPLAY)
+	/* PDP output device selection */
+	dev_priv->outdev = (enum pdp_output_device)output_device;
+	if (dev_priv->outdev == PDP_OUTPUT_PDP2 &&
+	    !tc_pdp2_compatible(dev->dev->parent)) {
+		DRM_ERROR("TC doesn't support PDP2\n");
+		err = -ENODEV;
+		goto err_dev_priv_free;
+	}
+
+	if (dev_priv->outdev == PDP_OUTPUT_PDP1) {
+		dev_priv->pdp_interrupt = TC_INTERRUPT_PDP;
+	} else if (dev_priv->outdev == PDP_OUTPUT_PDP2) {
+		dev_priv->pdp_interrupt = TC_INTERRUPT_PDP2;
+	} else {
+		DRM_ERROR("wrong PDP device number (outdev=%u)\n",
+			  dev_priv->outdev);
+		err = -ENODEV;
+		goto err_dev_priv_free;
+	}
+
+	/* PDP FBC module support detection */
+	dev_priv->pfim_capable = (dev_priv->outdev == PDP_OUTPUT_PDP2 &&
+				  tc_pfim_capable(dev->dev->parent));
+#endif
+
+	if (dev_priv->version == PDP_VERSION_APOLLO ||
+		dev_priv->version == PDP_VERSION_ODIN) {
+#if !defined(SUPPORT_PLATO_DISPLAY)
+		err = tc_enable(dev->dev->parent);
+		if (err) {
+			DRM_ERROR("failed to enable parent device (err=%d)\n", err);
+			goto err_dev_priv_free;
+		}
+
+		/*
+		 * check whether it's Orion PDP for picking
+		 * the right display mode list later on
+		 */
+		if (dev_priv->version == PDP_VERSION_ODIN)
+			dev_priv->subversion = (enum pdp_odin_subversion)
+				tc_odin_subvers(dev->dev->parent);
+#endif
+	}
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+	else if (dev_priv->version == PDP_VERSION_PLATO) {
+// XXX do we need to do this? Plato driver has already enabled device.
+		err = plato_enable(dev->dev->parent);
+		if (err) {
+			DRM_ERROR("failed to enable parent device (err=%d)\n", err);
+			goto err_dev_priv_free;
+		}
+	}
+#endif
+
+	dev_priv->gem_priv = pdp_gem_init(dev, 0);
+	if (!dev_priv->gem_priv) {
+		DRM_ERROR("gem initialisation failed\n");
+		err = -ENOMEM;
+		goto err_disable_parent_device;
+	}
+
+	err = pdp_modeset_early_init(dev_priv);
+	if (err) {
+		DRM_ERROR("early modeset initialisation failed (err=%d)\n",
+			  err);
+		goto err_gem_cleanup;
+	}
+
+	err = drm_vblank_init(dev_priv->dev, 1);
+	if (err) {
+		DRM_ERROR("failed to complete vblank init (err=%d)\n", err);
+		goto err_modeset_late_cleanup;
+	}
+
+	if (dev_priv->version == PDP_VERSION_APOLLO ||
+		dev_priv->version == PDP_VERSION_ODIN) {
+#if !defined(SUPPORT_PLATO_DISPLAY)
+		err = tc_set_interrupt_handler(dev->dev->parent,
+					   dev_priv->pdp_interrupt,
+					   pdp_irq_handler,
+					   dev);
+		if (err) {
+			DRM_ERROR("failed to set interrupt handler (err=%d)\n",
+				  err);
+			goto err_vblank_cleanup;
+		}
+
+		err = tc_enable_interrupt(dev->dev->parent,
+					  dev_priv->pdp_interrupt);
+		if (err) {
+			DRM_ERROR("failed to enable pdp interrupts (err=%d)\n",
+				  err);
+			goto err_uninstall_interrupt_handle;
+		}
+#endif
+	}
+#if defined(SUPPORT_PLATO_DISPLAY)
+	else if (dev_priv->version == PDP_VERSION_PLATO) {
+		err = plato_set_interrupt_handler(dev->dev->parent,
+							PLATO_INTERRUPT_PDP,
+							pdp_irq_handler,
+							dev);
+		if (err) {
+			DRM_ERROR("failed to set interrupt handler (err=%d)\n",
+				  err);
+			goto err_vblank_cleanup;
+		}
+
+		err = plato_enable_interrupt(dev->dev->parent, PLATO_INTERRUPT_PDP);
+		if (err) {
+			DRM_ERROR("failed to enable pdp interrupts (err=%d)\n",
+				  err);
+			goto err_uninstall_interrupt_handle;
+		}
+	}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0))
+	dev->irq_enabled = true;
+#endif
+
+
+	return 0;
+
+err_uninstall_interrupt_handle:
+	if (dev_priv->version == PDP_VERSION_APOLLO ||
+		dev_priv->version == PDP_VERSION_ODIN) {
+#if !defined(SUPPORT_PLATO_DISPLAY)
+		tc_set_interrupt_handler(dev->dev->parent,
+					     dev_priv->pdp_interrupt,
+					     NULL,
+					     NULL);
+#endif
+	}
+#if defined(SUPPORT_PLATO_DISPLAY)
+	else if (dev_priv->version == PDP_VERSION_PLATO) {
+		plato_set_interrupt_handler(dev->dev->parent,
+				PLATO_INTERRUPT_PDP,
+				NULL,
+				NULL);
+	}
+#endif
+err_vblank_cleanup:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	/* Called by drm_dev_fini in Linux 4.11.0 and later */
+	drm_vblank_cleanup(dev_priv->dev);
+#endif
+err_modeset_late_cleanup:
+	pdp_modeset_late_cleanup(dev_priv);
+err_gem_cleanup:
+	pdp_gem_cleanup(dev_priv->gem_priv);
+err_disable_parent_device:
+	if (dev_priv->version == PDP_VERSION_APOLLO ||
+		dev_priv->version == PDP_VERSION_ODIN) {
+#if !defined(SUPPORT_PLATO_DISPLAY)
+		tc_disable(dev->dev->parent);
+#endif
+	}
+#if defined(SUPPORT_PLATO_DISPLAY)
+	else if (dev_priv->version == PDP_VERSION_PLATO)
+		plato_disable(dev->dev->parent);
+#endif
+err_dev_priv_free:
+	kfree(dev_priv);
+	return err;
+}
+
+static int pdp_late_load(struct drm_device *dev)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	int err;
+
+	err = pdp_modeset_late_init(dev_priv);
+	if (err) {
+		DRM_ERROR("late modeset initialisation failed (err=%d)\n",
+			  err);
+		return err;
+	}
+
+	return 0;
+}
+
+static void pdp_early_unload(struct drm_device *dev)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+
+#if defined(CONFIG_DRM_FBDEV_EMULATION) && defined(PDP_USE_ATOMIC)
+	drm_atomic_helper_shutdown(dev);
+#endif
+	pdp_modeset_early_cleanup(dev_priv);
+}
+
+static void pdp_late_unload(struct drm_device *dev)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+
+	DRM_INFO("unloading %s device.\n", to_platform_device(dev->dev)->name);
+	if (dev_priv->version == PDP_VERSION_APOLLO ||
+		dev_priv->version == PDP_VERSION_ODIN) {
+#if !defined(SUPPORT_PLATO_DISPLAY)
+		tc_disable_interrupt(dev->dev->parent, dev_priv->pdp_interrupt);
+		tc_set_interrupt_handler(dev->dev->parent,
+					     dev_priv->pdp_interrupt,
+					     NULL,
+					     NULL);
+#endif
+	}
+#if defined(SUPPORT_PLATO_DISPLAY)
+	else if (dev_priv->version == PDP_VERSION_PLATO) {
+		plato_disable_interrupt(dev->dev->parent, PLATO_INTERRUPT_PDP);
+		plato_set_interrupt_handler(dev->dev->parent,
+						PLATO_INTERRUPT_PDP,
+						NULL,
+						NULL);
+	}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	/* Called by drm_dev_fini in Linux 4.11.0 and later */
+	drm_vblank_cleanup(dev_priv->dev);
+#endif
+	pdp_modeset_late_cleanup(dev_priv);
+	pdp_gem_cleanup(dev_priv->gem_priv);
+
+	if (dev_priv->version == PDP_VERSION_APOLLO ||
+		dev_priv->version == PDP_VERSION_ODIN) {
+#if !defined(SUPPORT_PLATO_DISPLAY)
+		tc_disable(dev->dev->parent);
+#endif
+	}
+#if defined(SUPPORT_PLATO_DISPLAY)
+	else if (dev_priv->version == PDP_VERSION_PLATO)
+		plato_disable(dev->dev->parent);
+#endif
+
+	kfree(dev_priv);
+}
+
+
+#if !defined(CONFIG_DRM_FBDEV_EMULATION)
+static inline void pdp_teardown_drm_config(struct drm_device *dev)
+{
+#if defined(PDP_USE_ATOMIC)
+	drm_atomic_helper_shutdown(dev);
+#else
+	struct drm_crtc *crtc;
+
+	DRM_INFO("%s: %s device\n", __func__, to_platform_device(dev->dev)->name);
+
+	/*
+	 * When non atomic driver is in use, manually trigger ->set_config
+	 * with an empty mode set associated to this crtc.
+	 */
+	drm_modeset_lock_all(dev);
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		if (crtc->primary->fb) {
+			struct drm_mode_set mode_set = { .crtc = crtc };
+			int err;
+
+			err = drm_mode_set_config_internal(&mode_set);
+			if (err)
+				DRM_ERROR("failed to disable crtc %p (err=%d)\n",
+					  crtc, err);
+		}
+	}
+	drm_modeset_unlock_all(dev);
+#endif
+}
+#endif /* !defined(CONFIG_DRM_FBDEV_EMULATION) */
+
+static void pdp_lastclose(struct drm_device *dev)
+{
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	struct pdp_fbdev *fbdev = dev_priv->fbdev;
+	int err;
+
+	if (fbdev) {
+		/*
+		 * This is a fbdev driver, therefore never attempt to shutdown
+		 * on a client disconnecting.
+		 */
+		err = drm_fb_helper_restore_fbdev_mode_unlocked(&fbdev->helper);
+		if (err)
+			DRM_ERROR("failed to restore mode (err=%d)\n", err);
+	}
+#else
+	pdp_teardown_drm_config(dev);
+#endif
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+int pdp_enable_vblank(struct drm_crtc *crtc)
+#else
+static int pdp_enable_vblank(struct drm_device *dev, unsigned int pipe)
+#endif
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	struct drm_device *dev = crtc->dev;
+	unsigned int pipe      = drm_crtc_index(crtc);
+#endif
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+
+	switch (pipe) {
+	case 0:
+		pdp_crtc_set_vblank_enabled(dev_priv->crtc, true);
+		break;
+	default:
+		DRM_ERROR("invalid crtc %u\n", pipe);
+		return -EINVAL;
+	}
+
+	DRM_DEBUG("vblank interrupts enabled for crtc %d\n", pipe);
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+void pdp_disable_vblank(struct drm_crtc *crtc)
+#else
+static void pdp_disable_vblank(struct drm_device *dev, unsigned int pipe)
+#endif
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	struct drm_device *dev = crtc->dev;
+	unsigned int pipe      = drm_crtc_index(crtc);
+#endif
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+
+	switch (pipe) {
+	case 0:
+		pdp_crtc_set_vblank_enabled(dev_priv->crtc, false);
+		break;
+	default:
+		DRM_ERROR("invalid crtc %u\n", pipe);
+		return;
+	}
+
+	DRM_DEBUG("vblank interrupts disabled for crtc %d\n", pipe);
+}
+
+static int pdp_gem_object_create_ioctl(struct drm_device *dev,
+				       void *data,
+				       struct drm_file *file)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+
+	return pdp_gem_object_create_ioctl_priv(dev,
+						dev_priv->gem_priv,
+						data,
+						file);
+}
+
+static int pdp_gem_dumb_create(struct drm_file *file,
+			       struct drm_device *dev,
+			       struct drm_mode_create_dumb *args)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+
+	return pdp_gem_dumb_create_priv(file,
+					dev,
+					dev_priv->gem_priv,
+					args);
+}
+
+void pdp_gem_object_free(struct drm_gem_object *obj)
+{
+	struct pdp_drm_private *dev_priv = obj->dev->dev_private;
+
+	pdp_gem_object_free_priv(dev_priv->gem_priv, obj);
+}
+
+static const struct drm_ioctl_desc pdp_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(PDP_GEM_CREATE, pdp_gem_object_create_ioctl,
+				DRM_AUTH | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PDP_GEM_MMAP, pdp_gem_object_mmap_ioctl,
+				DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(PDP_GEM_CPU_PREP, pdp_gem_object_cpu_prep_ioctl,
+				DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(PDP_GEM_CPU_FINI, pdp_gem_object_cpu_fini_ioctl,
+				DRM_AUTH),
+};
+
+static const struct file_operations pdp_driver_fops = {
+	.owner		= THIS_MODULE,
+	.open		= drm_open,
+	.release	= drm_release,
+	.unlocked_ioctl	= drm_ioctl,
+	.mmap		= drm_gem_mmap,
+	.poll		= drm_poll,
+	.read		= drm_read,
+	.llseek		= noop_llseek,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= drm_compat_ioctl,
+#endif
+};
+
+static struct drm_driver pdp_drm_driver = {
+	.load				= NULL,
+	.unload				= NULL,
+	.lastclose			= pdp_lastclose,
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	.get_vblank_counter		= drm_vblank_no_hw_counter,
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0))
+	.enable_vblank			= pdp_enable_vblank,
+	.disable_vblank			= pdp_disable_vblank,
+#endif
+
+	.debugfs_init			= pdp_debugfs_init,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	.debugfs_cleanup		= pdp_debugfs_cleanup,
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_prime_export		= pdp_gem_prime_export,
+	.gem_free_object		= pdp_gem_object_free,
+	.gem_vm_ops			= &pdp_gem_vm_ops,
+#endif
+
+	.gem_prime_import		= pdp_gem_prime_import,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	.prime_handle_to_fd		= drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle		= drm_gem_prime_fd_to_handle,
+#endif
+	.gem_prime_import_sg_table	= pdp_gem_prime_import_sg_table,
+
+    // Set dumb_create to NULL to avoid xorg owning the display (if xorg is running).
+	.dumb_create			= pdp_gem_dumb_create,
+	.dumb_map_offset		= pdp_gem_dumb_map_offset,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	.dumb_destroy			= drm_gem_dumb_destroy,
+#endif
+
+	.name				= DRIVER_NAME,
+	.desc				= DRIVER_DESC,
+	.date				= DRIVER_DATE,
+	.major				= PVRVERSION_MAJ,
+	.minor				= PVRVERSION_MIN,
+	.patchlevel			= PVRVERSION_BUILD,
+
+	.driver_features		= DRIVER_GEM |
+					  DRIVER_MODESET |
+					  PVR_DRIVER_PRIME |
+					  PVR_DRIVER_ATOMIC,
+	.ioctls				= pdp_ioctls,
+	.num_ioctls			= ARRAY_SIZE(pdp_ioctls),
+	.fops				= &pdp_driver_fops,
+};
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+
+static int compare_parent_dev(struct device *dev, void *data)
+{
+	struct device *pdp_dev = data;
+
+	return dev->parent && dev->parent == pdp_dev->parent;
+}
+
+static int pdp_component_bind(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *ddev;
+	int ret;
+
+	dev_info(dev, "Loading platform device\n");
+	ddev = drm_dev_alloc(&pdp_drm_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	// XXX no need to do this as happens in pdp_early_load
+	platform_set_drvdata(pdev, ddev);
+
+	BUG_ON(pdp_drm_driver.load != NULL);
+
+	ret = pdp_early_load(ddev);
+	if (ret)
+		goto err_drm_dev_put;
+
+	DRM_DEBUG_DRIVER("Binding other components\n");
+	/* Bind other components, including HDMI encoder/connector */
+	ret = component_bind_all(dev, ddev);
+	if (ret) {
+		DRM_ERROR("Failed to bind other components (ret=%d)\n", ret);
+		goto err_drm_dev_late_unload;
+	}
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret)
+		goto err_drm_dev_late_unload;
+
+	ret = pdp_late_load(ddev);
+	if (ret)
+		goto err_drm_dev_unregister;
+
+	return 0;
+
+err_drm_dev_unregister:
+	drm_dev_unregister(ddev);
+err_drm_dev_late_unload:
+	pdp_late_unload(ddev);
+err_drm_dev_put:
+	drm_dev_put(ddev);
+	return	ret;
+}
+
+static void pdp_component_unbind(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	dev_info(dev, "Unloading platform device\n");
+	BUG_ON(pdp_drm_driver.unload != NULL);
+	pdp_early_unload(ddev);
+	drm_dev_unregister(ddev);
+	pdp_late_unload(ddev);
+	component_unbind_all(dev, ddev);
+	drm_dev_put(ddev);
+}
+
+static const struct component_master_ops pdp_component_ops = {
+	.bind	= pdp_component_bind,
+	.unbind = pdp_component_unbind,
+};
+
+
+static int pdp_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct component_match *match = NULL;
+
+	component_match_add(dev, &match, compare_parent_dev, dev);
+	return component_master_add_with_match(dev, &pdp_component_ops, match);
+}
+
+static int pdp_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &pdp_component_ops);
+	return 0;
+}
+
+#else // !SUPPORT_PLATO_DISPLAY
+
+static int pdp_probe(struct platform_device *pdev)
+{
+	struct drm_device *ddev;
+	int ret;
+
+	ddev = drm_dev_alloc(&pdp_drm_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	/*
+	 * The load callback, called from drm_dev_register, is deprecated,
+	 * because of potential race conditions.
+	 */
+	BUG_ON(pdp_drm_driver.load != NULL);
+
+	ret = pdp_early_load(ddev);
+	if (ret)
+		goto err_drm_dev_put;
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret)
+		goto err_drm_dev_late_unload;
+
+	ret = pdp_late_load(ddev);
+	if (ret)
+		goto err_drm_dev_unregister;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+		pdp_drm_driver.name,
+		pdp_drm_driver.major,
+		pdp_drm_driver.minor,
+		pdp_drm_driver.patchlevel,
+		pdp_drm_driver.date,
+		ddev->primary->index);
+#endif
+	return 0;
+
+err_drm_dev_unregister:
+	drm_dev_unregister(ddev);
+err_drm_dev_late_unload:
+	pdp_late_unload(ddev);
+err_drm_dev_put:
+	drm_dev_put(ddev);
+	return	ret;
+}
+
+static int pdp_remove(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	/*
+	 * The unload callback, called from drm_dev_unregister, is
+	 * deprecated.
+	 */
+	BUG_ON(pdp_drm_driver.unload != NULL);
+
+	pdp_early_unload(ddev);
+
+	drm_dev_unregister(ddev);
+
+	pdp_late_unload(ddev);
+
+	drm_dev_put(ddev);
+
+	return 0;
+}
+
+#endif  // SUPPORT_PLATO_DISPLAY
+
+static void pdp_shutdown(struct platform_device *pdev)
+{
+}
+
+static struct platform_device_id pdp_platform_device_id_table[] = {
+	{ .name = APOLLO_DEVICE_NAME_PDP, .driver_data = PDP_VERSION_APOLLO },
+	{ .name = ODN_DEVICE_NAME_PDP, .driver_data = PDP_VERSION_ODIN },
+#if defined(SUPPORT_PLATO_DISPLAY)
+#if defined(PLATO_MULTI_DEVICE)
+	{ .name = PLATO_MAKE_DEVICE_NAME_PDP(0),
+	  .driver_data = PDP_VERSION_PLATO },
+	{ .name = PLATO_MAKE_DEVICE_NAME_PDP(1),
+	  .driver_data = PDP_VERSION_PLATO },
+	{ .name = PLATO_MAKE_DEVICE_NAME_PDP(2),
+	  .driver_data = PDP_VERSION_PLATO },
+	{ .name = PLATO_MAKE_DEVICE_NAME_PDP(3),
+	  .driver_data = PDP_VERSION_PLATO },
+#else
+	{ .name = PLATO_DEVICE_NAME_PDP, .driver_data = PDP_VERSION_PLATO },
+#endif
+#endif  // SUPPORT_PLATO_DISPLAY
+	{ },
+};
+
+static struct platform_driver pdp_platform_driver = {
+	.probe		= pdp_probe,
+	.remove		= pdp_remove,
+	.shutdown	= pdp_shutdown,
+	.driver		= {
+		.owner  = THIS_MODULE,
+		.name	= DRIVER_NAME,
+	},
+	.id_table	= pdp_platform_device_id_table,
+};
+
+module_platform_driver(pdp_platform_driver);
+
+MODULE_AUTHOR("Imagination Technologies Ltd. <gpl-support@imgtec.com>");
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_DEVICE_TABLE(platform, pdp_platform_device_id_table);
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_drv.h b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_drv.h
new file mode 100644
index 000000000000..fe6083a7a3cb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_drv.h
@@ -0,0 +1,237 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__DRM_PDP_DRV_H__)
+#define __DRM_PDP_DRV_H__
+
+#include <linux/version.h>
+#include <linux/wait.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_fourcc.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_mm.h>
+
+#include <drm/drm_plane.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+#include <drm/drm_framebuffer.h>
+#endif
+
+#include "pdp_common.h"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && \
+	!defined(PVR_ANDROID_USE_PDP_LEGACY)
+#define PDP_USE_ATOMIC
+#endif
+
+struct pdp_gem_context;
+enum pdp_crtc_flip_status;
+struct pdp_flip_data;
+struct pdp_gem_private;
+
+#if !defined(SUPPORT_PLATO_DISPLAY)
+struct tc_pdp_platform_data;
+#else
+struct plato_pdp_platform_data;
+#endif
+
+struct pdp_drm_private {
+	struct drm_device *dev;
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+	struct pdp_fbdev *fbdev;
+#endif
+
+	enum pdp_version version;
+
+	/* differentiate Orion from base Odin PDP */
+	enum pdp_odin_subversion subversion;
+
+	/* created by pdp_gem_init */
+	struct pdp_gem_private	*gem_priv;
+
+	/* preferred output device */
+	enum pdp_output_device outdev;
+	uint32_t pdp_interrupt;
+
+	/* PDP FBC Decompression module support */
+	bool pfim_capable;
+
+	/* initialised by pdp_modeset_early_init */
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	struct drm_connector *connector;
+	struct drm_encoder *encoder;
+
+	bool display_enabled;
+};
+
+struct pdp_crtc {
+	struct drm_crtc base;
+
+	uint32_t number;
+
+	resource_size_t pdp_reg_size;
+	resource_size_t pdp_reg_phys_base;
+	void __iomem *pdp_reg;
+
+	resource_size_t pdp_bif_reg_size;
+	resource_size_t pdp_bif_reg_phys_base;
+	void __iomem *pdp_bif_reg;
+
+	resource_size_t pll_reg_size;
+	resource_size_t pll_reg_phys_base;
+	void __iomem *pll_reg;
+
+	resource_size_t odn_core_size; /* needed for odin pdp clk reset */
+	resource_size_t odn_core_phys_base;
+	void __iomem *odn_core_reg;
+
+	resource_size_t pfim_reg_size;
+	resource_size_t pfim_reg_phys_base;
+	void __iomem *pfim_reg;
+
+	wait_queue_head_t flip_pending_wait_queue;
+
+	/* Reuse the drm_device event_lock to protect these */
+	atomic_t flip_status;
+	struct drm_pending_vblank_event *flip_event;
+	struct drm_framebuffer *old_fb;
+	struct pdp_flip_data *flip_data;
+	bool flip_async;
+};
+
+#define to_pdp_crtc(crtc) container_of(crtc, struct pdp_crtc, base)
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+struct drm_gem_object;
+
+struct pdp_framebuffer {
+	struct drm_framebuffer base;
+	struct drm_gem_object *obj[1];
+};
+
+#define to_pdp_framebuffer(fb) container_of(fb, struct pdp_framebuffer, base)
+#define to_drm_framebuffer(fb) (&(fb)->base)
+#else
+#define pdp_framebuffer drm_framebuffer
+#define to_pdp_framebuffer(fb) (fb)
+#define to_drm_framebuffer(fb) (fb)
+#endif
+
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+struct pdp_fbdev {
+	struct drm_fb_helper helper;
+	struct pdp_framebuffer fb;
+	struct pdp_drm_private *priv;
+};
+#endif
+
+static inline u32 pdp_drm_fb_cpp(struct drm_framebuffer *fb)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	return fb->format->cpp[0];
+#else
+	return fb->bits_per_pixel / 8;
+#endif
+}
+
+static inline u32 pdp_drm_fb_format(struct drm_framebuffer *fb)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	return fb->format->format;
+#else
+	return fb->pixel_format;
+#endif
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+int pdp_debugfs_init(struct drm_minor *minor);
+#else
+void pdp_debugfs_init(struct drm_minor *minor);
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+void pdp_debugfs_cleanup(struct drm_minor *minor);
+#endif
+
+struct drm_plane *pdp_plane_create(struct drm_device *dev,
+				   enum drm_plane_type type);
+void pdp_plane_set_surface(struct drm_crtc *crtc, struct drm_plane *plane,
+			   struct drm_framebuffer *fb,
+			   const uint32_t src_x, const uint32_t src_y);
+
+struct drm_crtc *pdp_crtc_create(struct drm_device *dev, uint32_t number,
+				 struct drm_plane *primary_plane);
+void pdp_crtc_set_plane_enabled(struct drm_crtc *crtc, bool enable);
+void pdp_crtc_set_vblank_enabled(struct drm_crtc *crtc, bool enable);
+void pdp_crtc_irq_handler(struct drm_crtc *crtc);
+
+
+struct drm_connector *pdp_dvi_connector_create(struct drm_device *dev);
+
+struct drm_encoder *pdp_tmds_encoder_create(struct drm_device *dev);
+
+int pdp_modeset_early_init(struct pdp_drm_private *dev_priv);
+int pdp_modeset_late_init(struct pdp_drm_private *dev_priv);
+void pdp_modeset_early_cleanup(struct pdp_drm_private *dev_priv);
+void pdp_modeset_late_cleanup(struct pdp_drm_private *dev_priv);
+
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+struct pdp_fbdev *pdp_fbdev_create(struct pdp_drm_private *dev);
+void pdp_fbdev_destroy(struct pdp_fbdev *fbdev);
+#endif
+
+int pdp_modeset_validate_init(struct pdp_drm_private *dev_priv,
+			      struct drm_mode_fb_cmd2 *mode_cmd,
+			      struct pdp_framebuffer *pdp_fb,
+			      struct drm_gem_object *obj);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+int pdp_enable_vblank(struct drm_crtc *crtc);
+void pdp_disable_vblank(struct drm_crtc *crtc);
+#endif
+
+#endif /* !defined(__DRM_PDP_DRV_H__) */
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_dvi.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_dvi.c
new file mode 100644
index 000000000000..92a99a81a906
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_dvi.c
@@ -0,0 +1,293 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "drm_pdp_drv.h"
+
+#include <linux/moduleparam.h>
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+#include <drm/drm_edid.h>
+#endif
+
+#if defined(PDP_USE_ATOMIC)
+#include <drm/drm_atomic_helper.h>
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0))
+#include <drm/drm_probe_helper.h>
+#endif
+
+#include "kernel_compatibility.h"
+
+struct pdp_mode_data {
+	int hdisplay;
+	int vdisplay;
+	int vrefresh;
+	bool reduced_blanking;
+	bool interlaced;
+	bool margins;
+};
+
+static const struct pdp_mode_data pdp_extra_modes[] = {
+};
+
+static char preferred_mode_name[DRM_DISPLAY_MODE_LEN] = "\0";
+
+module_param_string(dvi_preferred_mode,
+		    preferred_mode_name,
+		    DRM_DISPLAY_MODE_LEN,
+		    0444);
+
+MODULE_PARM_DESC(dvi_preferred_mode,
+		 "Specify the preferred mode (if supported), e.g. 1280x1024.");
+
+
+static int pdp_dvi_add_extra_modes(struct drm_connector *connector)
+{
+	struct drm_display_mode *mode;
+	int num_modes;
+	int i;
+
+	for (i = 0, num_modes = 0; i < ARRAY_SIZE(pdp_extra_modes); i++) {
+		mode = drm_cvt_mode(connector->dev,
+				    pdp_extra_modes[i].hdisplay,
+				    pdp_extra_modes[i].vdisplay,
+				    pdp_extra_modes[i].vrefresh,
+				    pdp_extra_modes[i].reduced_blanking,
+				    pdp_extra_modes[i].interlaced,
+				    pdp_extra_modes[i].margins);
+		if (mode) {
+			drm_mode_probed_add(connector, mode);
+			num_modes++;
+		}
+	}
+
+	return num_modes;
+}
+
+static int pdp_dvi_connector_helper_get_modes(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	int num_modes;
+	int len = strlen(preferred_mode_name);
+
+	if (len)
+		dev_info(dev->dev, "detected dvi_preferred_mode=%s\n",
+					preferred_mode_name);
+	else
+		dev_info(dev->dev, "no dvi_preferred_mode\n");
+
+	num_modes = drm_add_modes_noedid(connector,
+					 dev->mode_config.max_width,
+					 dev->mode_config.max_height);
+
+	num_modes += pdp_dvi_add_extra_modes(connector);
+	if (num_modes) {
+		struct drm_display_mode *pref_mode = NULL;
+
+		if (len) {
+			struct drm_display_mode *mode;
+			struct list_head *entry;
+
+			list_for_each(entry, &connector->probed_modes) {
+				mode = list_entry(entry,
+						  struct drm_display_mode,
+						  head);
+				if (!strcmp(mode->name, preferred_mode_name)) {
+					pref_mode = mode;
+					break;
+				}
+			}
+		}
+
+		if (pref_mode)
+			pref_mode->type |= DRM_MODE_TYPE_PREFERRED;
+		else
+			drm_set_preferred_mode(connector,
+					       dev->mode_config.max_width,
+					       dev->mode_config.max_height);
+	}
+
+	drm_mode_sort(&connector->probed_modes);
+
+	DRM_DEBUG_DRIVER("[CONNECTOR:%d:%s] found %d modes\n",
+			 connector->base.id,
+			 connector->name,
+			 num_modes);
+
+	return num_modes;
+}
+
+static enum drm_mode_status
+pdp_dvi_connector_helper_mode_valid(struct drm_connector *connector,
+				    struct drm_display_mode *mode)
+{
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return MODE_NO_INTERLACE;
+	else if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
+		return MODE_NO_DBLESCAN;
+
+	return MODE_OK;
+}
+
+#if !defined(PDP_USE_ATOMIC)
+static struct drm_encoder *
+pdp_dvi_connector_helper_best_encoder(struct drm_connector *connector)
+{
+	/* Pick the first encoder we find */
+	if (connector->encoder_ids[0] != 0) {
+		struct drm_encoder *encoder;
+
+		encoder = drm_encoder_find(connector->dev,
+					   NULL,
+					   connector->encoder_ids[0]);
+		if (encoder) {
+			DRM_DEBUG_DRIVER("[ENCODER:%d:%s] best for [CONNECTOR:%d:%s]\n",
+					 encoder->base.id,
+					 encoder->name,
+					 connector->base.id,
+					 connector->name);
+			return encoder;
+		}
+	}
+
+	return NULL;
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+static enum drm_connector_status
+pdp_dvi_connector_detect(struct drm_connector *connector,
+			 bool force)
+{
+	/*
+	 * It appears that there is no way to determine if a monitor
+	 * is connected. This needs to be set to connected otherwise
+	 * DPMS never gets set to ON.
+	 */
+	return connector_status_connected;
+}
+#endif
+
+static void pdp_dvi_connector_destroy(struct drm_connector *connector)
+{
+	struct pdp_drm_private *dev_priv = connector->dev->dev_private;
+
+	DRM_DEBUG_DRIVER("[CONNECTOR:%d:%s]\n",
+			 connector->base.id,
+			 connector->name);
+
+	drm_connector_cleanup(connector);
+
+	kfree(connector);
+	dev_priv->connector = NULL;
+}
+
+static void pdp_dvi_connector_force(struct drm_connector *connector)
+{
+}
+
+static struct drm_connector_helper_funcs pdp_dvi_connector_helper_funcs = {
+	.get_modes = pdp_dvi_connector_helper_get_modes,
+	.mode_valid = pdp_dvi_connector_helper_mode_valid,
+	/*
+	 * For atomic, don't set atomic_best_encoder or best_encoder. This will
+	 * cause the DRM core to fallback to drm_atomic_helper_best_encoder().
+	 * This is fine as we only have a single connector and encoder.
+	 */
+#if !defined(PDP_USE_ATOMIC)
+	.best_encoder = pdp_dvi_connector_helper_best_encoder,
+#endif
+};
+
+static const struct drm_connector_funcs pdp_dvi_connector_funcs = {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+	.detect = pdp_dvi_connector_detect,
+#endif
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = pdp_dvi_connector_destroy,
+	.force = pdp_dvi_connector_force,
+#if defined(PDP_USE_ATOMIC)
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+#else
+	.dpms = drm_helper_connector_dpms,
+#endif
+};
+
+
+struct drm_connector *
+pdp_dvi_connector_create(struct drm_device *dev)
+{
+	struct drm_connector *connector;
+
+	connector = kzalloc(sizeof(*connector), GFP_KERNEL);
+	if (!connector)
+		return ERR_PTR(-ENOMEM);
+
+	drm_connector_init(dev,
+			   connector,
+			   &pdp_dvi_connector_funcs,
+			   DRM_MODE_CONNECTOR_DVID);
+	drm_connector_helper_add(connector, &pdp_dvi_connector_helper_funcs);
+
+	connector->dpms = DRM_MODE_DPMS_OFF;
+	connector->interlace_allowed = false;
+	connector->doublescan_allowed = false;
+	connector->display_info.subpixel_order = SubPixelHorizontalRGB;
+
+	DRM_DEBUG_DRIVER("[CONNECTOR:%d:%s]\n",
+			 connector->base.id,
+			 connector->name);
+
+	return connector;
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_fb.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_fb.c
new file mode 100644
index 000000000000..1faf6716b66d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_fb.c
@@ -0,0 +1,294 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+#include <linux/version.h>
+#include <linux/export.h>
+#include <linux/mm.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0))
+#include <drm/drmP.h>
+#endif
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "drm_pdp_gem.h"
+#include "kernel_compatibility.h"
+
+#define FBDEV_NAME "pdpdrmfb"
+
+static struct fb_ops pdp_fbdev_ops = {
+	.owner = THIS_MODULE,
+	.fb_check_var = drm_fb_helper_check_var,
+	.fb_set_par = drm_fb_helper_set_par,
+	.fb_fillrect = cfb_fillrect,
+	.fb_copyarea = cfb_copyarea,
+	.fb_imageblit = cfb_imageblit,
+	.fb_pan_display = drm_fb_helper_pan_display,
+	.fb_blank = drm_fb_helper_blank,
+	.fb_setcmap = drm_fb_helper_setcmap,
+	.fb_debug_enter = drm_fb_helper_debug_enter,
+	.fb_debug_leave = drm_fb_helper_debug_leave,
+};
+
+
+static struct fb_info *
+pdp_fbdev_helper_alloc(struct drm_fb_helper *helper)
+{
+	return drm_fb_helper_alloc_info(helper);
+}
+
+static inline void
+pdp_fbdev_helper_fill_info(struct drm_fb_helper *helper,
+			   struct drm_fb_helper_surface_size *sizes,
+			   struct fb_info *info,
+			   struct drm_mode_fb_cmd2 __maybe_unused *mode_cmd)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	drm_fb_helper_fill_fix(info, mode_cmd->pitches[0], helper->fb->depth);
+	drm_fb_helper_fill_var(info, helper, sizes->fb_width,
+			       sizes->fb_height);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	drm_fb_helper_fill_fix(info, mode_cmd->pitches[0],
+			       helper->fb->format->depth);
+	drm_fb_helper_fill_var(info, helper, helper->fb->width,
+			       helper->fb->height);
+#else
+	drm_fb_helper_fill_info(info, helper, sizes);
+#endif
+}
+
+static int pdp_fbdev_probe(struct drm_fb_helper *helper,
+			   struct drm_fb_helper_surface_size *sizes)
+{
+	struct pdp_fbdev *pdp_fbdev =
+		container_of(helper, struct pdp_fbdev, helper);
+	struct drm_framebuffer *fb =
+		to_drm_framebuffer(&pdp_fbdev->fb);
+	struct pdp_gem_private *gem_priv = pdp_fbdev->priv->gem_priv;
+	struct drm_device *dev = helper->dev;
+	struct drm_mode_fb_cmd2 mode_cmd;
+	struct pdp_gem_object *pdp_obj;
+	struct drm_gem_object *obj;
+	struct fb_info *info;
+	void __iomem *vaddr;
+	size_t obj_size;
+	int err;
+
+	if (helper->fb)
+		return 0;
+
+	mutex_lock(&dev->struct_mutex);
+
+	/* Create a framebuffer */
+	info = pdp_fbdev_helper_alloc(helper);
+	if (!info) {
+		err = -ENOMEM;
+		goto err_unlock_dev;
+	}
+
+	memset(&mode_cmd, 0, sizeof(mode_cmd));
+	mode_cmd.pitches[0] =
+		sizes->surface_width * DIV_ROUND_UP(sizes->surface_bpp, 8);
+	mode_cmd.width = sizes->surface_width;
+	mode_cmd.height = sizes->surface_height;
+	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
+							  sizes->surface_depth);
+	obj_size = PAGE_ALIGN(mode_cmd.height * mode_cmd.pitches[0]);
+
+	obj = pdp_gem_object_create(dev, gem_priv, obj_size, 0);
+	if (IS_ERR(obj)) {
+		err = PTR_ERR(obj);
+		goto err_unlock_dev;
+	}
+
+	pdp_obj = to_pdp_obj(obj);
+
+	vaddr = ioremap(pdp_obj->cpu_addr, obj->size);
+	if (!vaddr) {
+		err = PTR_ERR(vaddr);
+		goto err_gem_destroy;
+	}
+
+	/* Zero fb memory, fb_memset accounts for iomem address space */
+	fb_memset(vaddr, 0, obj_size);
+
+	err = pdp_modeset_validate_init(pdp_fbdev->priv, &mode_cmd,
+					&pdp_fbdev->fb, obj);
+	if (err)
+		goto err_gem_unmap;
+
+	helper->fb = fb;
+	helper->COMPAT_FB_INFO = info;
+
+	/* Fill out the Linux framebuffer info */
+	strscpy(info->fix.id, FBDEV_NAME, sizeof(info->fix.id));
+	pdp_fbdev_helper_fill_info(helper, sizes, info, &mode_cmd);
+	info->par = helper;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	info->flags = FBINFO_DEFAULT | FBINFO_HWACCEL_DISABLED;
+#else
+	info->flags = FBINFO_HWACCEL_DISABLED;
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0))
+	info->flags |= FBINFO_CAN_FORCE_OUTPUT;
+#endif
+	info->fbops = &pdp_fbdev_ops;
+	info->fix.smem_start = pdp_obj->cpu_addr;
+	info->fix.smem_len = obj_size;
+	info->screen_base = vaddr;
+	info->screen_size = obj_size;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0))
+	info->apertures->ranges[0].base = pdp_obj->cpu_addr;
+	info->apertures->ranges[0].size = obj_size;
+#endif
+
+	mutex_unlock(&dev->struct_mutex);
+	return 0;
+
+err_gem_unmap:
+	iounmap(vaddr);
+
+err_gem_destroy:
+	pdp_gem_object_free_priv(gem_priv, obj);
+
+err_unlock_dev:
+	mutex_unlock(&dev->struct_mutex);
+
+	DRM_ERROR(FBDEV_NAME " - %s failed (err=%d)\n", __func__, err);
+	return err;
+}
+
+static const struct drm_fb_helper_funcs pdp_fbdev_helper_funcs = {
+	.fb_probe = pdp_fbdev_probe,
+};
+
+struct pdp_fbdev *pdp_fbdev_create(struct pdp_drm_private *dev_priv)
+{
+	struct pdp_fbdev *pdp_fbdev;
+	int err;
+	const u8 preferred_bpp = 32;
+
+	pdp_fbdev = kzalloc(sizeof(*pdp_fbdev), GFP_KERNEL);
+	if (!pdp_fbdev)
+		return ERR_PTR(-ENOMEM);
+
+	drm_fb_helper_prepare(dev_priv->dev, &pdp_fbdev->helper,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0))
+			      preferred_bpp,
+#endif
+			      &pdp_fbdev_helper_funcs);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	err = drm_fb_helper_init(dev_priv->dev, &pdp_fbdev->helper, 1, 1);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0))
+	err = drm_fb_helper_init(dev_priv->dev, &pdp_fbdev->helper, 1);
+#else
+	err = drm_fb_helper_init(dev_priv->dev, &pdp_fbdev->helper);
+#endif
+	if (err)
+		goto err_free_fbdev;
+
+	pdp_fbdev->priv = dev_priv;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+	drm_fb_helper_single_add_all_connectors(&pdp_fbdev->helper);
+#endif
+
+	err = drm_fb_helper_initial_config(&pdp_fbdev->helper
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0))
+					   , preferred_bpp
+#endif
+					  );
+	if (err)
+		goto err_fb_helper_fini;
+
+	DRM_DEBUG_DRIVER(FBDEV_NAME " - fb device registered\n");
+	return pdp_fbdev;
+
+err_fb_helper_fini:
+	drm_fb_helper_fini(&pdp_fbdev->helper);
+
+err_free_fbdev:
+	kfree(pdp_fbdev);
+
+	DRM_ERROR(FBDEV_NAME " - %s, failed (err=%d)\n", __func__, err);
+	return ERR_PTR(err);
+}
+
+void pdp_fbdev_destroy(struct pdp_fbdev *pdp_fbdev)
+{
+	struct pdp_framebuffer *pdp_fb;
+	struct pdp_gem_object *pdp_obj;
+	struct drm_framebuffer *fb;
+	struct fb_info *info;
+
+	if (!pdp_fbdev)
+		return;
+
+	drm_fb_helper_unregister_info(&pdp_fbdev->helper);
+	pdp_fb = &pdp_fbdev->fb;
+
+	pdp_obj = to_pdp_obj(pdp_fb->obj[0]);
+	if (pdp_obj) {
+		info = pdp_fbdev->helper.COMPAT_FB_INFO;
+		iounmap((void __iomem *)info->screen_base);
+	}
+
+	drm_gem_object_put(pdp_fb->obj[0]);
+
+	drm_fb_helper_fini(&pdp_fbdev->helper);
+
+	fb = to_drm_framebuffer(pdp_fb);
+
+	/**
+	 * If the driver's probe function hasn't been called
+	 * (due to deferred setup of the framebuffer device),
+	 * then the framebuffer won't have been initialised.
+	 * Check this before attempting to clean it up.
+	 */
+	if (fb && fb->dev)
+		drm_framebuffer_cleanup(fb);
+
+	kfree(pdp_fbdev);
+}
+#endif /* CONFIG_DRM_FBDEV_EMULATION */
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_gem.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_gem.c
new file mode 100644
index 000000000000..88fa77c56c99
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_gem.c
@@ -0,0 +1,890 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_prime.h>
+#include <linux/platform_device.h>
+#endif
+
+#include <linux/dma-buf.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/capability.h>
+
+#include <drm/drm_mm.h>
+
+#if defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+#include "physheap.h"
+#include "pvrsrv.h"
+#include "pvr_debug.h"
+#else
+#if defined(SUPPORT_PLATO_DISPLAY)
+#include "plato_drv.h"
+#else
+#include "tc_drv.h"
+#endif
+#endif
+
+#include "drm_pdp_gem.h"
+#include "pdp_drm.h"
+#include "kernel_compatibility.h"
+
+#if !defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+#if defined(SUPPORT_PLATO_DISPLAY)
+#define pdp_gem_platform_data plato_pdp_platform_data
+#else
+#define pdp_gem_platform_data tc_pdp_platform_data
+#endif
+#endif
+
+const struct vm_operations_struct pdp_gem_vm_ops = {
+	.fault	= pdp_gem_object_vm_fault,
+	.open	= drm_gem_vm_open,
+	.close	= drm_gem_vm_close,
+};
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+const struct drm_gem_object_funcs pdp_gem_funcs = {
+	.export = pdp_gem_prime_export,
+	.free = pdp_gem_object_free,
+	.vm_ops = &pdp_gem_vm_ops,
+};
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0) */
+
+struct pdp_gem_private {
+	struct mutex			vram_lock;
+	struct				drm_mm vram;
+	resource_size_t			memory_base;
+	bool				dma_map_export_host_addr;
+#if defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+	PVRSRV_DEVICE_NODE		*pvr_dev_node;
+	PHYS_HEAP			*pvr_phys_heap;
+#endif
+};
+
+static struct pdp_gem_object *
+pdp_gem_private_object_create(struct drm_device *dev,
+			      size_t size,
+			      struct dma_resv *resv)
+{
+	struct pdp_gem_object *pdp_obj;
+
+	WARN_ON(PAGE_ALIGN(size) != size);
+
+	pdp_obj = kzalloc(sizeof(*pdp_obj), GFP_KERNEL);
+	if (!pdp_obj)
+		return ERR_PTR(-ENOMEM);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	if (!resv)
+		dma_resv_init(&pdp_obj->_resv);
+#else
+	pdp_obj->base.resv = resv;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	pdp_obj->base.funcs = &pdp_gem_funcs;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0) */
+
+	drm_gem_private_object_init(dev, &pdp_obj->base, size);
+
+	return pdp_obj;
+}
+
+struct drm_gem_object *pdp_gem_object_create(struct drm_device *dev,
+					     struct pdp_gem_private *gem_priv,
+					     size_t size,
+					     u32 flags)
+{
+	struct pdp_gem_object *pdp_obj;
+	struct drm_mm_node *node;
+	int err = 0;
+
+	pdp_obj = pdp_gem_private_object_create(dev, size, NULL);
+	if (!pdp_obj) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+
+	node = kzalloc(sizeof(*node), GFP_KERNEL);
+	if (!node) {
+		err = -ENOMEM;
+		goto err_unref;
+	}
+
+	mutex_lock(&gem_priv->vram_lock);
+	err = drm_mm_insert_node(&gem_priv->vram, node, size);
+	mutex_unlock(&gem_priv->vram_lock);
+	if (err)
+		goto err_free_node;
+
+	pdp_obj->vram = node;
+	pdp_obj->dev_addr = pdp_obj->vram->start;
+	pdp_obj->cpu_addr = gem_priv->memory_base + pdp_obj->dev_addr;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	pdp_obj->resv = &pdp_obj->_resv;
+#else
+	pdp_obj->resv = pdp_obj->base.resv;
+#endif
+	pdp_obj->dma_map_export_host_addr = gem_priv->dma_map_export_host_addr;
+
+	return &pdp_obj->base;
+
+err_free_node:
+	kfree(node);
+err_unref:
+	pdp_gem_object_free_priv(gem_priv, &pdp_obj->base);
+err_exit:
+	return ERR_PTR(err);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+vm_fault_t pdp_gem_object_vm_fault(struct vm_fault *vmf)
+#else
+int pdp_gem_object_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+#endif
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	struct vm_area_struct *vma = vmf->vma;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	unsigned long addr = vmf->address;
+#else
+	unsigned long addr = (unsigned long)vmf->virtual_address;
+#endif
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct pdp_gem_object *pdp_obj = to_pdp_obj(obj);
+	unsigned long off;
+	unsigned long pfn;
+
+	off = addr - vma->vm_start;
+	pfn = (pdp_obj->cpu_addr + off) >> PAGE_SHIFT;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0))
+	return vmf_insert_pfn(vma, addr, pfn);
+#else
+	{
+		int err;
+
+		err = vm_insert_pfn(vma, addr, pfn);
+		switch (err) {
+		case 0:
+		case -EBUSY:
+			return VM_FAULT_NOPAGE;
+		case -ENOMEM:
+			return VM_FAULT_OOM;
+		default:
+			return VM_FAULT_SIGBUS;
+		}
+	}
+#endif
+}
+
+void pdp_gem_object_free_priv(struct pdp_gem_private *gem_priv,
+			      struct drm_gem_object *obj)
+{
+	struct pdp_gem_object *pdp_obj = to_pdp_obj(obj);
+
+	drm_gem_free_mmap_offset(obj);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	if (&pdp_obj->_resv == pdp_obj->resv)
+		dma_resv_fini(&pdp_obj->_resv);
+#endif
+	if (pdp_obj->vram) {
+		mutex_lock(&gem_priv->vram_lock);
+		drm_mm_remove_node(pdp_obj->vram);
+		mutex_unlock(&gem_priv->vram_lock);
+
+		kfree(pdp_obj->vram);
+	} else if (obj->import_attach) {
+		drm_prime_gem_destroy(obj, pdp_obj->sgt);
+	}
+
+	drm_gem_object_release(&pdp_obj->base);
+	kfree(pdp_obj);
+}
+
+static int pdp_gem_prime_attach(struct dma_buf *dma_buf,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+				struct device *dev,
+#endif
+				struct dma_buf_attachment *attach)
+{
+#if defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+	(void) dma_buf;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+	(void) dev;
+#endif
+	/* Restrict access to PVR Services */
+	if (strcmp(attach->dev->driver->name,
+		   PVR_LDM_DRIVER_REGISTRATION_NAME))
+		return -EPERM;
+#else
+	struct drm_gem_object *obj = dma_buf->priv;
+
+	/* Restrict access to Rogue */
+	if (WARN_ON(!obj->dev->dev->parent) ||
+	    obj->dev->dev->parent != attach->dev->parent)
+		return -EPERM;
+#endif
+	return 0;
+}
+
+static struct sg_table *
+pdp_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
+			  enum dma_data_direction dir)
+{
+	struct drm_gem_object *obj = attach->dmabuf->priv;
+	struct pdp_gem_object *pdp_obj = to_pdp_obj(obj);
+	struct sg_table *sgt;
+
+	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt)
+		return NULL;
+
+	if (sg_alloc_table(sgt, 1, GFP_KERNEL))
+		goto err_free_sgt;
+
+	if (pdp_obj->dma_map_export_host_addr)
+		sg_dma_address(sgt->sgl) = pdp_obj->cpu_addr;
+	else
+		sg_dma_address(sgt->sgl) = pdp_obj->dev_addr;
+
+	sg_dma_len(sgt->sgl) = obj->size;
+
+	return sgt;
+
+err_free_sgt:
+	kfree(sgt);
+	return NULL;
+}
+
+static void pdp_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
+					struct sg_table *sgt,
+					enum dma_data_direction dir)
+{
+	sg_free_table(sgt);
+	kfree(sgt);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+static void *pdp_gem_prime_kmap_atomic(struct dma_buf *dma_buf,
+				       unsigned long page_num)
+{
+	return NULL;
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *pdp_gem_prime_kmap(struct dma_buf *dma_buf,
+				unsigned long page_num)
+{
+	return NULL;
+}
+#endif
+
+static int pdp_gem_prime_mmap(struct dma_buf *dma_buf,
+			      struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	int err;
+
+	mutex_lock(&obj->dev->struct_mutex);
+	err = drm_gem_mmap_obj(obj, obj->size, vma);
+	mutex_unlock(&obj->dev->struct_mutex);
+
+	return err;
+}
+
+#if defined(CONFIG_X86)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *pdp_gem_prime_vmap(struct dma_buf *dma_buf)
+#else
+static int pdp_gem_prime_vmap(struct dma_buf *dma_buf, struct iosys_map *map)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct pdp_gem_object *pdp_obj = to_pdp_obj(obj);
+	void __iomem *vaddr;
+	__maybe_unused int ret = 0;
+
+	mutex_lock(&obj->dev->struct_mutex);
+
+	/*
+	 * On x86 platforms, the pointer returned by ioremap can be dereferenced
+	 * directly. As such, explicitly cast away the __ioremap qualifier.
+	 */
+	vaddr = ioremap(pdp_obj->cpu_addr, obj->size);
+	if (vaddr == NULL) {
+		DRM_DEBUG_DRIVER("ioremap failed");
+		ret = -ENOMEM;
+	}
+
+	mutex_unlock(&obj->dev->struct_mutex);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0))
+	if (ret == 0)
+		iosys_map_set_vaddr_iomem(map, vaddr);
+	return ret;
+#else
+	return (void __force *) vaddr;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) */
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void pdp_gem_prime_vunmap(struct dma_buf *dma_buf, void *vaddr)
+#else
+static void pdp_gem_prime_vunmap(struct dma_buf *dma_buf, struct iosys_map *map)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0))
+	void __iomem *vaddr = map->vaddr_iomem;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) */
+
+	mutex_lock(&obj->dev->struct_mutex);
+	iounmap((void __iomem *)vaddr);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0))
+	iosys_map_clear(map);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0) */
+
+	mutex_unlock(&obj->dev->struct_mutex);
+}
+#endif
+
+static const struct dma_buf_ops pdp_gem_prime_dmabuf_ops = {
+	.attach		= pdp_gem_prime_attach,
+	.map_dma_buf	= pdp_gem_prime_map_dma_buf,
+	.unmap_dma_buf	= pdp_gem_prime_unmap_dma_buf,
+	.release	= drm_gem_dmabuf_release,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+	.map_atomic	= pdp_gem_prime_kmap_atomic,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	.map		= pdp_gem_prime_kmap,
+#endif
+#else
+	.kmap_atomic	= pdp_gem_prime_kmap_atomic,
+	.kmap		= pdp_gem_prime_kmap,
+#endif
+	.mmap		= pdp_gem_prime_mmap,
+#if defined(CONFIG_X86)
+	.vmap		= pdp_gem_prime_vmap,
+	.vunmap		= pdp_gem_prime_vunmap
+#endif
+};
+
+
+static int
+pdp_gem_lookup_our_object(struct drm_file *file, u32 handle,
+			  struct drm_gem_object **objp)
+
+{
+	struct drm_gem_object *obj;
+
+	obj = drm_gem_object_lookup(file, handle);
+	if (!obj)
+		return -ENOENT;
+
+	if (obj->import_attach) {
+		/*
+		 * The dmabuf associated with the object is not one of ours.
+		 * Our own buffers are handled differently on import.
+		 */
+		drm_gem_object_put(obj);
+		return -EINVAL;
+	}
+
+	*objp = obj;
+	return 0;
+}
+
+struct dma_buf *pdp_gem_prime_export(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+				     struct drm_device *dev,
+#endif
+				     struct drm_gem_object *obj,
+				     int flags)
+{
+	struct pdp_gem_object *pdp_obj = to_pdp_obj(obj);
+	DEFINE_DMA_BUF_EXPORT_INFO(export_info);
+
+	export_info.ops = &pdp_gem_prime_dmabuf_ops;
+	export_info.size = obj->size;
+	export_info.flags = flags;
+	export_info.resv = pdp_obj->resv;
+	export_info.priv = obj;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	return drm_gem_dmabuf_export(obj->dev, &export_info);
+#else
+	return drm_gem_dmabuf_export(dev, &export_info);
+#endif
+}
+
+struct drm_gem_object *
+pdp_gem_prime_import(struct drm_device *dev,
+		     struct dma_buf *dma_buf)
+{
+	if (dma_buf->ops == &pdp_gem_prime_dmabuf_ops) {
+		struct drm_gem_object *obj = dma_buf->priv;
+
+		if (obj->dev == dev) {
+			/*
+			 * The dmabuf is one of ours, so return the associated
+			 * PDP GEM object, rather than create a new one.
+			 */
+			drm_gem_object_get(obj);
+
+			return obj;
+		}
+	}
+
+	return drm_gem_prime_import(dev, dma_buf);
+}
+
+struct drm_gem_object *
+pdp_gem_prime_import_sg_table(struct drm_device *dev,
+			      struct dma_buf_attachment *attach,
+			      struct sg_table *sgt)
+{
+	struct pdp_gem_private *gem_priv = pdp_gem_get_private(dev);
+	struct pdp_gem_object *pdp_obj;
+	int err;
+
+	pdp_obj = pdp_gem_private_object_create(dev,
+						attach->dmabuf->size,
+						attach->dmabuf->resv);
+	if (!pdp_obj) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+
+	pdp_obj->sgt = sgt;
+
+	/* We only expect a single entry for card memory */
+	if (pdp_obj->sgt->nents != 1) {
+		err = -EINVAL;
+		goto err_obj_unref;
+	}
+
+	pdp_obj->dev_addr = sg_dma_address(pdp_obj->sgt->sgl);
+	pdp_obj->cpu_addr = gem_priv->memory_base + pdp_obj->dev_addr;
+	pdp_obj->resv = attach->dmabuf->resv;
+
+	return &pdp_obj->base;
+
+err_obj_unref:
+	drm_gem_object_put(&pdp_obj->base);
+err_exit:
+	return ERR_PTR(err);
+}
+
+int pdp_gem_dumb_create_priv(struct drm_file *file,
+			     struct drm_device *dev,
+			     struct pdp_gem_private *gem_priv,
+			     struct drm_mode_create_dumb *args)
+{
+	struct drm_gem_object *obj;
+	u32 handle;
+	u32 pitch;
+	size_t size;
+	int err;
+
+	pitch = args->width * (ALIGN(args->bpp, 8) >> 3);
+	size = PAGE_ALIGN(pitch * args->height);
+
+	obj = pdp_gem_object_create(dev, gem_priv, size, 0);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	err = drm_gem_handle_create(file, obj, &handle);
+	if (err)
+		goto exit;
+
+	args->handle = handle;
+	args->pitch = pitch;
+	args->size = size;
+
+exit:
+	drm_gem_object_put(obj);
+	return err;
+}
+
+int pdp_gem_dumb_map_offset(struct drm_file *file,
+			    struct drm_device *dev,
+			    uint32_t handle,
+			    uint64_t *offset)
+{
+	struct drm_gem_object *obj;
+	int err;
+
+	mutex_lock(&dev->struct_mutex);
+
+	err = pdp_gem_lookup_our_object(file, handle, &obj);
+	if (err)
+		goto exit_unlock;
+
+	err = drm_gem_create_mmap_offset(obj);
+	if (err)
+		goto exit_obj_unref;
+
+	*offset = drm_vma_node_offset_addr(&obj->vma_node);
+
+exit_obj_unref:
+	drm_gem_object_put(obj);
+exit_unlock:
+	mutex_unlock(&dev->struct_mutex);
+	return err;
+}
+
+#if defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+static bool
+pdp_gem_init_platform(struct drm_device *dev,
+		      struct pdp_gem_private *gem_priv,
+		      unsigned int instance)
+{
+	IMG_CPU_PHYADDR heap_cpu_paddr;
+	IMG_CPU_PHYADDR lma_cpu_paddr;
+	IMG_DEV_PHYADDR heap_dev_paddr;
+	IMG_UINT64 heap_size;
+	PVRSRV_ERROR pvr_err;
+
+	gem_priv->pvr_dev_node = PVRSRVGetDeviceInstance(instance);
+	if (!gem_priv->pvr_dev_node) {
+		DRM_ERROR("%s can't get PVR device node for instance %d\n",
+			  dev->driver->name, instance);
+		return false;
+	}
+
+	pvr_err = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_DISPLAY,
+					 gem_priv->pvr_dev_node,
+					 &gem_priv->pvr_phys_heap);
+	if (pvr_err != PVRSRV_OK) {
+		DRM_ERROR("%s couldn't acquire display heap: %s\n",
+			  dev->driver->name, PVRSRVGetErrorString(pvr_err));
+		return false;
+	}
+
+	if (PhysHeapGetType(gem_priv->pvr_phys_heap) != PHYS_HEAP_TYPE_LMA) {
+		DRM_ERROR("%s display heap is not LMA\n", dev->driver->name);
+		goto exit_release_heap;
+	}
+
+	pvr_err = PhysHeapGetCpuPAddr(gem_priv->pvr_phys_heap, &heap_cpu_paddr);
+	if (pvr_err != PVRSRV_OK) {
+		DRM_ERROR("%s couldn't get display heap base CPU physical address: %s\n",
+			  dev->driver->name, PVRSRVGetErrorString(pvr_err));
+		goto exit_release_heap;
+	}
+
+	PhysHeapCpuPAddrToDevPAddr(gem_priv->pvr_phys_heap, 1,
+				   &heap_dev_paddr, &heap_cpu_paddr);
+
+	pvr_err = PhysHeapGetSize(gem_priv->pvr_phys_heap, &heap_size);
+	if (pvr_err != PVRSRV_OK) {
+		DRM_ERROR("%s couldn't get display heap size: %s\n",
+			  dev->driver->name, PVRSRVGetErrorString(pvr_err));
+		goto exit_release_heap;
+	}
+
+	lma_cpu_paddr = heap_cpu_paddr;
+	lma_cpu_paddr.uiAddr -= heap_dev_paddr.uiAddr;
+
+	drm_mm_init(&gem_priv->vram, heap_dev_paddr.uiAddr, heap_size);
+
+	DRM_INFO("%s has 0x%llx bytes of allocatable memory at LMA offset 0x%llx (CPU PA 0x%llx)\n",
+		 dev->driver->name, (u64)heap_size,
+		 (u64)heap_dev_paddr.uiAddr, (u64)heap_cpu_paddr.uiAddr);
+
+	gem_priv->memory_base = lma_cpu_paddr.uiAddr;
+	gem_priv->dma_map_export_host_addr = false;
+
+	return true;
+
+exit_release_heap:
+	PhysHeapRelease(gem_priv->pvr_phys_heap);
+	return false;
+}
+
+static void
+pdp_gem_cleanup_platform(struct pdp_gem_private *gem_priv)
+{
+	drm_mm_takedown(&gem_priv->vram);
+	PhysHeapRelease(gem_priv->pvr_phys_heap);
+}
+#else
+static bool
+pdp_gem_init_platform(struct drm_device *dev,
+		      struct pdp_gem_private *gem_priv,
+		      unsigned int instance)
+{
+	struct pdp_gem_platform_data *pdata =
+		to_platform_device(dev->dev)->dev.platform_data;
+
+	/* Instance is only used by SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE */
+	WARN_ON(instance != 0);
+
+#if defined(SUPPORT_ION) && !defined(SUPPORT_GEM_ALLOC)
+	drm_mm_init(&gem_priv->vram, 0, 0);
+	DRM_INFO("%s has no directly allocatable memory; the memory is managed by ION\n",
+		 dev->driver->name);
+
+#else
+	drm_mm_init(&gem_priv->vram,
+		    pdata->pdp_heap_memory_base - pdata->memory_base,
+		    pdata->pdp_heap_memory_size);
+
+	DRM_INFO("%s has %pa bytes of allocatable memory at 0x%llx = (0x%llx - 0x%llx)\n",
+		 dev->driver->name, &pdata->pdp_heap_memory_size,
+		 (u64)(pdata->pdp_heap_memory_base - pdata->memory_base),
+		 (u64)pdata->pdp_heap_memory_base, (u64)pdata->memory_base);
+#endif
+	gem_priv->memory_base = pdata->memory_base;
+	gem_priv->dma_map_export_host_addr = pdata->dma_map_export_host_addr;
+
+	return true;
+}
+
+static void
+pdp_gem_cleanup_platform(struct pdp_gem_private *gem_priv)
+{
+	drm_mm_takedown(&gem_priv->vram);
+}
+#endif
+
+struct pdp_gem_private *pdp_gem_init(struct drm_device *dev, unsigned int instance)
+{
+	struct pdp_gem_private *gem_priv = kmalloc(sizeof(*gem_priv), GFP_KERNEL);
+
+	if (!gem_priv)
+		return NULL;
+
+	memset(&gem_priv->vram, 0, sizeof(gem_priv->vram));
+
+	mutex_init(&gem_priv->vram_lock);
+
+	if (!pdp_gem_init_platform(dev, gem_priv, instance)) {
+		mutex_destroy(&gem_priv->vram_lock);
+		kfree(gem_priv);
+		return NULL;
+	}
+
+	return gem_priv;
+}
+
+void pdp_gem_cleanup(struct pdp_gem_private *gem_priv)
+{
+	pdp_gem_cleanup_platform(gem_priv);
+
+	mutex_destroy(&gem_priv->vram_lock);
+
+	kfree(gem_priv);
+}
+
+struct dma_resv *pdp_gem_get_resv(struct drm_gem_object *obj)
+{
+	return (to_pdp_obj(obj)->resv);
+}
+
+u64 pdp_gem_get_dev_addr(struct drm_gem_object *obj)
+{
+	struct pdp_gem_object *pdp_obj = to_pdp_obj(obj);
+
+	return pdp_obj->dev_addr;
+}
+
+int pdp_gem_object_create_ioctl_priv(struct drm_device *dev,
+				struct pdp_gem_private *gem_priv,
+				void *data,
+				struct drm_file *file)
+{
+	struct drm_pdp_gem_create *args = data;
+	struct drm_gem_object *obj;
+	int err;
+
+	if (args->flags) {
+		DRM_ERROR("invalid flags: %#08x\n", args->flags);
+		return -EINVAL;
+	}
+
+	if (args->handle) {
+		DRM_ERROR("invalid handle (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	obj = pdp_gem_object_create(dev,
+					gem_priv,
+					PAGE_ALIGN(args->size),
+					args->flags);
+	if (IS_ERR(obj))
+		return PTR_ERR(obj);
+
+	err = drm_gem_handle_create(file, obj, &args->handle);
+	drm_gem_object_put(obj);
+
+	return err;
+
+}
+
+int pdp_gem_object_mmap_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file)
+{
+	struct drm_pdp_gem_mmap *args = (struct drm_pdp_gem_mmap *)data;
+
+	if (args->pad) {
+		DRM_ERROR("invalid pad (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	if (args->offset) {
+		DRM_ERROR("invalid offset (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	return pdp_gem_dumb_map_offset(file, dev, args->handle, &args->offset);
+}
+
+int pdp_gem_object_cpu_prep_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file)
+{
+	struct drm_pdp_gem_cpu_prep *args = (struct drm_pdp_gem_cpu_prep *)data;
+	struct drm_gem_object *obj;
+	struct pdp_gem_object *pdp_obj;
+	bool write = !!(args->flags & PDP_GEM_CPU_PREP_WRITE);
+	bool wait = !(args->flags & PDP_GEM_CPU_PREP_NOWAIT);
+	int err = 0;
+
+	if (args->flags & ~(PDP_GEM_CPU_PREP_READ |
+			    PDP_GEM_CPU_PREP_WRITE |
+			    PDP_GEM_CPU_PREP_NOWAIT)) {
+		DRM_ERROR("invalid flags: %#08x\n", args->flags);
+		return -EINVAL;
+	}
+
+	mutex_lock(&dev->struct_mutex);
+
+	err = pdp_gem_lookup_our_object(file, args->handle, &obj);
+	if (err)
+		goto exit_unlock;
+
+	pdp_obj = to_pdp_obj(obj);
+
+	if (pdp_obj->cpu_prep) {
+		err = -EBUSY;
+		goto exit_unref;
+	}
+
+	if (wait) {
+		long lerr;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0))
+		lerr = dma_resv_wait_timeout(pdp_obj->resv, write,
+					     true, 30 * HZ);
+#else
+		lerr = dma_resv_wait_timeout_rcu(pdp_obj->resv, write,
+						 true, 30 * HZ);
+#endif
+		if (!lerr)
+			err = -EBUSY;
+		else if (lerr < 0)
+			err = lerr;
+	} else {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0))
+		if (!dma_resv_test_signaled(pdp_obj->resv, write))
+#else
+		if (!dma_resv_test_signaled_rcu(pdp_obj->resv, write))
+#endif
+			err = -EBUSY;
+	}
+
+	if (!err)
+		pdp_obj->cpu_prep = true;
+
+exit_unref:
+	drm_gem_object_put(obj);
+exit_unlock:
+	mutex_unlock(&dev->struct_mutex);
+	return err;
+}
+
+int pdp_gem_object_cpu_fini_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file)
+{
+	struct drm_pdp_gem_cpu_fini *args = (struct drm_pdp_gem_cpu_fini *)data;
+	struct drm_gem_object *obj;
+	struct pdp_gem_object *pdp_obj;
+	int err = 0;
+
+	if (args->pad) {
+		DRM_ERROR("invalid pad (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	mutex_lock(&dev->struct_mutex);
+
+	err = pdp_gem_lookup_our_object(file, args->handle, &obj);
+	if (err)
+		goto exit_unlock;
+
+	pdp_obj = to_pdp_obj(obj);
+
+	if (!pdp_obj->cpu_prep) {
+		err = -EINVAL;
+		goto exit_unref;
+	}
+
+	pdp_obj->cpu_prep = false;
+
+exit_unref:
+	drm_gem_object_put(obj);
+exit_unlock:
+	mutex_unlock(&dev->struct_mutex);
+	return err;
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_gem.h b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_gem.h
new file mode 100644
index 000000000000..f8c04dad9521
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_gem.h
@@ -0,0 +1,156 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__DRM_PDP_GEM_H__)
+#define __DRM_PDP_GEM_H__
+
+#include <linux/dma-buf.h>
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_gem.h>
+
+#include "drm_pdp_drv.h"
+#include "pvr_dma_resv.h"
+
+extern const struct vm_operations_struct pdp_gem_vm_ops;
+
+struct pdp_gem_private;
+
+struct pdp_gem_object {
+	struct drm_gem_object base;
+
+	/* Non-null if backing originated from this driver */
+	struct drm_mm_node *vram;
+
+	/* Non-null if backing was imported */
+	struct sg_table *sgt;
+
+	bool dma_map_export_host_addr;
+	phys_addr_t cpu_addr;
+	dma_addr_t dev_addr;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	struct dma_resv _resv;
+#endif
+	struct dma_resv *resv;
+
+	bool cpu_prep;
+};
+
+#define to_pdp_obj(obj) container_of(obj, struct pdp_gem_object, base)
+
+struct pdp_gem_private *pdp_gem_init(struct drm_device *dev,
+				     unsigned int instance);
+
+void pdp_gem_cleanup(struct pdp_gem_private *dev_priv);
+
+/* ioctl functions */
+int pdp_gem_object_create_ioctl_priv(struct drm_device *dev,
+				     struct pdp_gem_private *gem_priv,
+				     void *data,
+				     struct drm_file *file);
+int pdp_gem_object_mmap_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file);
+int pdp_gem_object_cpu_prep_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file);
+int pdp_gem_object_cpu_fini_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file);
+
+/* drm driver functions */
+struct drm_gem_object *pdp_gem_object_create(struct drm_device *dev,
+					     struct pdp_gem_private *gem_priv,
+					     size_t size,
+					     u32 flags);
+
+void pdp_gem_object_free_priv(struct pdp_gem_private *gem_priv,
+			      struct drm_gem_object *obj);
+
+void pdp_gem_object_free(struct drm_gem_object *obj);
+
+struct dma_buf *pdp_gem_prime_export(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+				     struct drm_device *dev,
+#endif
+				     struct drm_gem_object *obj,
+				     int flags);
+
+struct drm_gem_object *pdp_gem_prime_import(struct drm_device *dev,
+					    struct dma_buf *dma_buf);
+
+struct drm_gem_object *
+pdp_gem_prime_import_sg_table(struct drm_device *dev,
+			      struct dma_buf_attachment *attach,
+			      struct sg_table *sgt);
+
+int pdp_gem_dumb_create_priv(struct drm_file *file,
+			     struct drm_device *dev,
+			     struct pdp_gem_private *gem_priv,
+			     struct drm_mode_create_dumb *args);
+
+int pdp_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
+			    uint32_t handle, uint64_t *offset);
+
+/* vm operation functions */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0))
+typedef int vm_fault_t;
+#endif
+vm_fault_t pdp_gem_object_vm_fault(struct vm_fault *vmf);
+#else
+int pdp_gem_object_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
+#endif
+
+/* internal interfaces */
+struct dma_resv *pdp_gem_get_resv(struct drm_gem_object *obj);
+u64 pdp_gem_get_dev_addr(struct drm_gem_object *obj);
+
+/* functions provided by clients of PDP GEM */
+struct pdp_gem_private *pdp_gem_get_private(struct drm_device *dev);
+
+#endif /* !defined(__DRM_PDP_GEM_H__) */
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_modeset.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_modeset.c
new file mode 100644
index 000000000000..6586209b3aea
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_modeset.c
@@ -0,0 +1,434 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "drm_pdp_drv.h"
+
+#include <linux/moduleparam.h>
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_print.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+#define drm_gem_fb_create(...) pdp_framebuffer_create(__VA_ARGS__)
+#else
+#include <drm/drm_gem_framebuffer_helper.h>
+#endif
+
+#if defined(PDP_USE_ATOMIC)
+#include <drm/drm_atomic_helper.h>
+#endif
+
+#include <powervr/img_drm_fourcc.h>
+
+#include "kernel_compatibility.h"
+
+#define PDP_WIDTH_MIN			640
+#define PDP_WIDTH_MAX			1280
+#define PDP_HEIGHT_MIN			480
+#define PDP_HEIGHT_MAX			1024
+
+#define ODIN_PDP_WIDTH_MAX		1920
+#define ODIN_PDP_HEIGHT_MAX		1080
+
+#define ORION_PDP_WIDTH_MAX		1280
+#define ORION_PDP_HEIGHT_MAX		720
+
+#define PLATO_PDP_WIDTH_MAX		1920
+#define PLATO_PDP_HEIGHT_MAX	1080
+
+static bool async_flip_enable = true;
+
+module_param(async_flip_enable, bool, 0444);
+
+MODULE_PARM_DESC(async_flip_enable,
+		 "Enable support for 'faked' async flipping (default: Y)");
+
+static inline int
+drm_mode_fb_cmd2_validate(const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	switch (mode_cmd->pixel_format) {
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_RGB565:
+		break;
+	default:
+		DRM_ERROR_RATELIMITED("pixel format not supported (format = %u)\n",
+				      mode_cmd->pixel_format);
+		return -EINVAL;
+	}
+
+	if (mode_cmd->flags & DRM_MODE_FB_INTERLACED) {
+		DRM_ERROR_RATELIMITED("interlaced framebuffers not supported\n");
+		return -EINVAL;
+	}
+
+	switch (mode_cmd->modifier[0]) {
+	case DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12:
+	case DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12:
+	case DRM_FORMAT_MOD_LINEAR:
+		break;
+	default:
+		DRM_ERROR_RATELIMITED("format modifier 0x%llx is not supported\n",
+			  mode_cmd->modifier[0]);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void pdp_framebuffer_destroy(struct drm_framebuffer *fb)
+{
+	struct pdp_framebuffer *pdp_fb = to_pdp_framebuffer(fb);
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", fb->base.id);
+
+	drm_framebuffer_cleanup(fb);
+
+	drm_gem_object_put(pdp_fb->obj[0]);
+
+	kfree(pdp_fb);
+}
+
+static int pdp_framebuffer_create_handle(struct drm_framebuffer *fb,
+					 struct drm_file *file,
+					 unsigned int *handle)
+{
+	struct pdp_framebuffer *pdp_fb = to_pdp_framebuffer(fb);
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", fb->base.id);
+
+	return drm_gem_handle_create(file, pdp_fb->obj[0], handle);
+}
+
+static const struct drm_framebuffer_funcs pdp_framebuffer_funcs = {
+	.destroy = pdp_framebuffer_destroy,
+	.create_handle = pdp_framebuffer_create_handle,
+	.dirty = NULL,
+};
+
+static inline int
+pdp_framebuffer_init(struct pdp_drm_private *dev_priv,
+		     const struct drm_mode_fb_cmd2 *mode_cmd,
+		     struct pdp_framebuffer *pdp_fb,
+		     struct drm_gem_object *obj)
+{
+	struct drm_framebuffer *fb;
+
+	if (!pdp_fb)
+		return -EINVAL;
+
+	fb = to_drm_framebuffer(pdp_fb);
+	pdp_fb->obj[0] = obj;
+
+	drm_helper_mode_fill_fb_struct(dev_priv->dev, fb, mode_cmd);
+
+	return drm_framebuffer_init(dev_priv->dev, fb, &pdp_framebuffer_funcs);
+}
+
+int pdp_modeset_validate_init(struct pdp_drm_private *dev_priv,
+			      struct drm_mode_fb_cmd2 *mode_cmd,
+			      struct pdp_framebuffer *pdp_fb,
+			      struct drm_gem_object *obj)
+{
+	int err;
+
+	err = drm_mode_fb_cmd2_validate(mode_cmd);
+	if (err)
+		return err;
+
+	return pdp_framebuffer_init(dev_priv, mode_cmd, pdp_fb, obj);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+static struct drm_framebuffer *
+pdp_framebuffer_create(struct drm_device *dev,
+		       struct drm_file *file,
+		       const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	struct drm_gem_object *obj;
+	struct pdp_framebuffer *pdp_fb;
+	int err;
+
+	obj = drm_gem_object_lookup(file, mode_cmd->handles[0]);
+	if (!obj) {
+		DRM_ERROR("failed to find buffer with handle %u\n",
+			  mode_cmd->handles[0]);
+		err = -ENOENT;
+		goto err_out;
+	}
+
+	pdp_fb = kzalloc(sizeof(*pdp_fb), GFP_KERNEL);
+	if (!pdp_fb) {
+		err = -ENOMEM;
+		goto err_obj_put;
+	}
+
+	err = pdp_framebuffer_init(dev_priv, mode_cmd, pdp_fb, obj);
+	if (err) {
+		DRM_ERROR("failed to initialise framebuffer (err=%d)\n", err);
+		goto err_free_fb;
+	}
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", pdp_fb->base.base.id);
+
+	return &pdp_fb->base;
+
+err_free_fb:
+	kfree(pdp_fb);
+err_obj_put:
+	drm_gem_object_put(obj);
+err_out:
+	return ERR_PTR(err);
+}
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)) */
+
+
+/*************************************************************************
+ * DRM mode config callbacks
+ **************************************************************************/
+
+static struct drm_framebuffer *
+pdp_fb_create(struct drm_device *dev,
+			struct drm_file *file,
+			const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_framebuffer *fb;
+	int err;
+
+	err = drm_mode_fb_cmd2_validate(mode_cmd);
+	if (err)
+		return ERR_PTR(err);
+
+	fb = drm_gem_fb_create(dev, file, mode_cmd);
+	if (IS_ERR(fb))
+		goto out;
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", fb->base.id);
+
+out:
+	return fb;
+}
+
+static const struct drm_mode_config_funcs pdp_mode_config_funcs = {
+	.fb_create = pdp_fb_create,
+	.output_poll_changed = NULL,
+#if defined(PDP_USE_ATOMIC)
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+#endif
+};
+
+
+int pdp_modeset_early_init(struct pdp_drm_private *dev_priv)
+{
+	struct drm_device *dev = dev_priv->dev;
+	int err;
+
+	drm_mode_config_init(dev);
+
+	dev->mode_config.funcs = &pdp_mode_config_funcs;
+	dev->mode_config.min_width = PDP_WIDTH_MIN;
+	dev->mode_config.min_height = PDP_HEIGHT_MIN;
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_APOLLO:
+		dev->mode_config.max_width = PDP_WIDTH_MAX;
+		dev->mode_config.max_height = PDP_HEIGHT_MAX;
+		break;
+	case PDP_VERSION_ODIN:
+		if (dev_priv->subversion == PDP_ODIN_ORION) {
+			dev->mode_config.max_width = ORION_PDP_WIDTH_MAX;
+			dev->mode_config.max_height = ORION_PDP_HEIGHT_MAX;
+		} else {
+			dev->mode_config.max_width = ODIN_PDP_WIDTH_MAX;
+			dev->mode_config.max_height = ODIN_PDP_HEIGHT_MAX;
+		}
+		break;
+	case PDP_VERSION_PLATO:
+		dev->mode_config.max_width = PLATO_PDP_WIDTH_MAX;
+		dev->mode_config.max_height = PLATO_PDP_HEIGHT_MAX;
+		break;
+	default:
+		BUG();
+	}
+
+	DRM_INFO("max_width is %d\n",
+		dev->mode_config.max_width);
+	DRM_INFO("max_height is %d\n",
+		dev->mode_config.max_height);
+
+	dev->mode_config.async_page_flip = async_flip_enable;
+
+	DRM_INFO("%s async flip support is %s\n",
+		 dev->driver->name, async_flip_enable ? "enabled" : "disabled");
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+	dev->mode_config.allow_fb_modifiers = true;
+#endif
+
+	dev_priv->plane = pdp_plane_create(dev, DRM_PLANE_TYPE_PRIMARY);
+	if (IS_ERR(dev_priv->plane)) {
+		DRM_ERROR("failed to create a primary plane\n");
+		err = PTR_ERR(dev_priv->plane);
+		goto err_config_cleanup;
+	}
+
+	dev_priv->crtc = pdp_crtc_create(dev, 0, dev_priv->plane);
+	if (IS_ERR(dev_priv->crtc)) {
+		DRM_ERROR("failed to create a CRTC\n");
+		err = PTR_ERR(dev_priv->crtc);
+		goto err_config_cleanup;
+	}
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_APOLLO:
+	case PDP_VERSION_ODIN:
+		dev_priv->connector = pdp_dvi_connector_create(dev);
+		if (IS_ERR(dev_priv->connector)) {
+			DRM_ERROR("failed to create a connector\n");
+			err = PTR_ERR(dev_priv->connector);
+			goto err_config_cleanup;
+		}
+
+		dev_priv->encoder = pdp_tmds_encoder_create(dev);
+		if (IS_ERR(dev_priv->encoder)) {
+			DRM_ERROR("failed to create an encoder\n");
+			err = PTR_ERR(dev_priv->encoder);
+			goto err_config_cleanup;
+		}
+
+		err = drm_connector_attach_encoder(dev_priv->connector,
+						   dev_priv->encoder);
+		if (err) {
+			DRM_ERROR("can't attach [ENCODER:%d:%s] to [CONNECTOR:%d:%s] (err=%d)\n",
+				  dev_priv->encoder->base.id,
+				  dev_priv->encoder->name,
+				  dev_priv->connector->base.id,
+				  dev_priv->connector->name,
+				  err);
+			goto err_config_cleanup;
+		}
+		break;
+	case PDP_VERSION_PLATO:
+		// PLATO connectors are created in HDMI component driver
+		break;
+	default:
+		BUG();
+	}
+
+	DRM_DEBUG_DRIVER("initialised\n");
+
+	return 0;
+
+err_config_cleanup:
+	drm_mode_config_cleanup(dev);
+
+	return err;
+}
+
+static inline int pdp_modeset_init_fbdev(struct pdp_drm_private *dev_priv)
+{
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+	struct pdp_fbdev *fbdev;
+	int err;
+
+	fbdev = pdp_fbdev_create(dev_priv);
+	if (IS_ERR(fbdev)) {
+		DRM_ERROR("failed to create a fb device");
+		return PTR_ERR(fbdev);
+	}
+	dev_priv->fbdev = fbdev;
+
+	/*
+	 * pdpdrmfb is registered and available for userspace to use. If this
+	 * is the only or primary device, fbcon has already bound a tty to it,
+	 * and the following call will take no effect. However, this may be
+	 * essential in order to sync the display when fbcon was already bound
+	 * to a different tty (and fbdev). This triggers ->set_config() which
+	 * will in turn set up a config and then do a modeset.
+	 */
+	err = drm_fb_helper_restore_fbdev_mode_unlocked(&dev_priv->fbdev->helper);
+	if (err) {
+		DRM_ERROR("failed to set mode (err=%d)\n", err);
+		return err;
+	}
+#endif
+	return 0;
+}
+
+int pdp_modeset_late_init(struct pdp_drm_private *dev_priv)
+{
+	struct drm_device *ddev = dev_priv->dev;
+	int err;
+
+	drm_mode_config_reset(ddev);
+
+	err = pdp_modeset_init_fbdev(dev_priv);
+	if (err)
+		DRM_INFO("fbdev init failure is not fatal, continue anyway.\n");
+
+	return 0;
+}
+
+void pdp_modeset_early_cleanup(struct pdp_drm_private *dev_priv)
+{
+}
+
+void pdp_modeset_late_cleanup(struct pdp_drm_private *dev_priv)
+{
+#if defined(CONFIG_DRM_FBDEV_EMULATION)
+	pdp_fbdev_destroy(dev_priv->fbdev);
+#endif
+	drm_mode_config_cleanup(dev_priv->dev);
+
+	DRM_DEBUG_DRIVER("cleaned up\n");
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_plane.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_plane.c
new file mode 100644
index 000000000000..57c0bea8c233
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_plane.c
@@ -0,0 +1,385 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#include "drm_pdp_drv.h"
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0))
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_plane_helper.h>
+
+#if defined(PDP_USE_ATOMIC)
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
+#include <drm/drm_gem_atomic_helper.h>
+#else
+#include <drm/drm_gem_framebuffer_helper.h>
+#endif
+#endif
+
+#include <powervr/img_drm_fourcc.h>
+
+#include "drm_pdp_gem.h"
+#include "pdp_apollo.h"
+#include "pdp_odin.h"
+#include "pdp_plato.h"
+#include "pfim_defs.h"
+
+#include "kernel_compatibility.h"
+
+
+static const uint32_t apollo_plato_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+};
+static const uint32_t odin_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGB565,
+};
+
+static const uint64_t default_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+static const uint64_t odin_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12,
+	DRM_FORMAT_MOD_INVALID
+};
+
+#if defined(PDP_USE_ATOMIC)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
+static int pdp_plane_helper_atomic_check(struct drm_plane *plane,
+					 struct drm_atomic_state *atomic_state)
+#else
+static int pdp_plane_helper_atomic_check(struct drm_plane *plane,
+					 struct drm_plane_state *state)
+#endif
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
+	struct drm_plane_state *state = drm_atomic_get_new_plane_state(atomic_state,
+								       plane);
+#endif
+	struct drm_crtc_state *crtc_new_state;
+
+	if (!state->crtc)
+		return 0;
+
+	crtc_new_state = drm_atomic_get_new_crtc_state(state->state,
+						       state->crtc);
+
+	return drm_atomic_helper_check_plane_state(state, crtc_new_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, true);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
+static void pdp_plane_helper_atomic_update(struct drm_plane *plane,
+					   struct drm_atomic_state *atomic_state)
+#else
+static void pdp_plane_helper_atomic_update(struct drm_plane *plane,
+					   struct drm_plane_state *old_state)
+#endif
+{
+	struct drm_plane_state *plane_state = plane->state;
+	struct drm_framebuffer *fb = plane_state->fb;
+
+	if (fb) {
+		pdp_plane_set_surface(plane_state->crtc, plane, fb,
+				      plane_state->src_x, plane_state->src_y);
+	}
+}
+
+static bool pdp_plane_format_mod_supported(struct drm_plane *plane,
+					   uint32_t format,
+					   uint64_t modifier)
+{
+	struct pdp_drm_private *dev_priv = plane->dev->dev_private;
+	const uint32_t *supported_formats;
+	const uint64_t *supported_modifiers;
+	unsigned int num_supported_formats;
+	unsigned int num_supported_modifiers;
+	unsigned int i;
+
+	if (modifier == DRM_FORMAT_MOD_INVALID)
+		return false;
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN:
+		supported_formats = odin_formats;
+		num_supported_formats = ARRAY_SIZE(odin_formats);
+		supported_modifiers = odin_modifiers;
+		num_supported_modifiers = ARRAY_SIZE(odin_modifiers);
+		break;
+	case PDP_VERSION_APOLLO:
+	case PDP_VERSION_PLATO:
+		supported_formats = apollo_plato_formats;
+		num_supported_formats = ARRAY_SIZE(apollo_plato_formats);
+		supported_modifiers = default_modifiers;
+		num_supported_modifiers = ARRAY_SIZE(default_modifiers);
+		break;
+	default:
+		DRM_ERROR("Unsupported PDP version\n");
+		return false;
+	}
+
+	for (i = 0; i < num_supported_formats; i++) {
+		if (supported_formats[i] == format) {
+			unsigned int j;
+
+			for (j = 0; j < num_supported_modifiers; j++)
+				if (supported_modifiers[j] == modifier)
+					return true;
+		}
+	}
+
+	return false;
+}
+
+static const struct drm_plane_helper_funcs pdp_plane_helper_funcs = {
+	.prepare_fb = drm_gem_plane_helper_prepare_fb,
+	.atomic_check = pdp_plane_helper_atomic_check,
+	.atomic_update = pdp_plane_helper_atomic_update,
+};
+
+static const struct drm_plane_funcs pdp_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = drm_plane_helper_destroy,
+	.reset = drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	.format_mod_supported = pdp_plane_format_mod_supported,
+};
+#else
+#define pdp_plane_funcs drm_primary_helper_funcs
+#endif
+
+struct drm_plane *pdp_plane_create(struct drm_device *dev,
+				   enum drm_plane_type type)
+{
+	struct pdp_drm_private *dev_priv = dev->dev_private;
+	struct drm_plane *plane;
+	const uint32_t *supported_formats;
+	const uint64_t *supported_modifiers;
+	uint32_t num_supported_formats;
+	int err;
+
+	switch (dev_priv->version) {
+	case PDP_VERSION_ODIN:
+		supported_formats = odin_formats;
+		num_supported_formats = ARRAY_SIZE(odin_formats);
+		supported_modifiers = odin_modifiers;
+		break;
+	case PDP_VERSION_APOLLO:
+	case PDP_VERSION_PLATO:
+		supported_formats = apollo_plato_formats;
+		num_supported_formats = ARRAY_SIZE(apollo_plato_formats);
+		supported_modifiers = default_modifiers;
+		break;
+	default:
+		DRM_ERROR("Unsupported PDP version\n");
+		err = -EINVAL;
+		goto err_exit;
+	}
+
+	plane = kzalloc(sizeof(*plane), GFP_KERNEL);
+	if (!plane) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+
+	err = drm_universal_plane_init(dev, plane, 0, &pdp_plane_funcs,
+				       supported_formats,
+				       num_supported_formats,
+				       supported_modifiers, type, NULL);
+	if (err)
+		goto err_plane_free;
+
+#if defined(PDP_USE_ATOMIC)
+	drm_plane_helper_add(plane, &pdp_plane_helper_funcs);
+#endif
+
+	DRM_DEBUG_DRIVER("[PLANE:%d]\n", plane->base.id);
+
+	return plane;
+
+err_plane_free:
+	kfree(plane);
+err_exit:
+	return ERR_PTR(err);
+}
+
+void pdp_plane_set_surface(struct drm_crtc *crtc, struct drm_plane *plane,
+			   struct drm_framebuffer *fb,
+			   const uint32_t src_x, const uint32_t src_y)
+{
+	struct pdp_drm_private *dev_priv = plane->dev->dev_private;
+	struct pdp_crtc *pdp_crtc = to_pdp_crtc(crtc);
+	struct pdp_framebuffer *pdp_fb = to_pdp_framebuffer(fb);
+	unsigned int pitch = fb->pitches[0];
+	uint64_t address = pdp_gem_get_dev_addr(pdp_fb->obj[0]);
+	uint64_t modifier = 0;
+	uint32_t format;
+	uint32_t fbc_mode;
+
+	/*
+	 * User space specifies 'x' and 'y' and this is used to tell the display
+	 * to scan out from part way through a buffer.
+	 */
+	address += ((src_y * pitch) + (src_x * (pdp_drm_fb_cpp(fb))));
+
+	/*
+	 * NOTE: If the buffer dimensions are less than the current mode then
+	 * the output will appear in the top left of the screen. This can be
+	 * centered by adjusting horizontal active start, right border start,
+	 * vertical active start and bottom border start. At this point it's
+	 * not entirely clear where this should be done. On the one hand it's
+	 * related to pdp_crtc_helper_mode_set but on the other hand there
+	 * might not always be a call to pdp_crtc_helper_mode_set. This needs
+	 * to be investigated.
+	 */
+	switch (dev_priv->version) {
+	case PDP_VERSION_APOLLO:
+		switch (pdp_drm_fb_format(fb)) {
+		case DRM_FORMAT_ARGB8888:
+		case DRM_FORMAT_XRGB8888:
+			format = 0xE;
+			break;
+		default:
+			DRM_ERROR("unsupported pixel format (format = %d)\n",
+				  pdp_drm_fb_format(fb));
+			return;
+		}
+
+		pdp_apollo_set_surface(plane->dev->dev,
+				       pdp_crtc->pdp_reg,
+				       0,
+				       address,
+				       0, 0,
+				       fb->width, fb->height, pitch,
+				       format,
+				       255,
+				       false);
+		break;
+	case PDP_VERSION_ODIN:
+		switch (pdp_drm_fb_format(fb)) {
+		case DRM_FORMAT_ARGB8888:
+		case DRM_FORMAT_XRGB8888:
+			format = ODN_PDP_SURF_PIXFMT_ARGB8888;
+			break;
+		case DRM_FORMAT_RGB565:
+			format = ODN_PDP_SURF_PIXFMT_RGB565;
+			break;
+		default:
+			DRM_ERROR("unsupported pixel format (format = %d)\n",
+				  pdp_drm_fb_format(fb));
+			return;
+		}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+		modifier = fb->modifier;
+#else
+		modifier = fb->modifier[0];
+#endif
+
+		switch (modifier) {
+		case DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12:
+			fbc_mode = ODIN_PFIM_FBCDC_8X8_V12;
+			break;
+		case DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12:
+			fbc_mode = ODIN_PFIM_FBCDC_16X4_V12;
+			break;
+		case DRM_FORMAT_MOD_LINEAR:
+			fbc_mode = ODIN_PFIM_MOD_LINEAR;
+			break;
+		default:
+			DRM_ERROR("unsupported fbc format (format = %llu)\n",
+				  modifier);
+			return;
+		}
+
+		pdp_odin_set_surface(plane->dev->dev,
+				     pdp_crtc->pdp_reg,
+				     0,
+				     address, fb->offsets[0],
+				     0, 0,
+				     fb->width, fb->height, pitch,
+				     format,
+				     255,
+				     false,
+				     pdp_crtc->pfim_reg, fbc_mode);
+		break;
+	case PDP_VERSION_PLATO:
+		switch (pdp_drm_fb_format(fb)) {
+		case DRM_FORMAT_ARGB8888:
+		case DRM_FORMAT_XRGB8888:
+			format = PLATO_PDP_PIXEL_FORMAT_ARGB8;
+			break;
+		default:
+			DRM_ERROR("unsupported pixel format (format = %d)\n",
+				  pdp_drm_fb_format(fb));
+			return;
+		}
+
+		pdp_plato_set_surface(crtc->dev->dev,
+				      pdp_crtc->pdp_reg,
+				      pdp_crtc->pdp_bif_reg,
+				      0,
+				      address,
+				      0, 0,
+				      fb->width, fb->height, pitch,
+				      format,
+				      255,
+				      false);
+		break;
+	default:
+			BUG();
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/drm_pdp_tmds.c b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_tmds.c
new file mode 100644
index 000000000000..b9c30f5453a4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/drm_pdp_tmds.c
@@ -0,0 +1,144 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "drm_pdp_drv.h"
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0))
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_encoder.h>
+#include <drm/drm_modeset_helper_vtables.h>
+
+#include "kernel_compatibility.h"
+
+static void pdp_tmds_encoder_helper_dpms(struct drm_encoder *encoder, int mode)
+{
+}
+
+static bool
+pdp_tmds_encoder_helper_mode_fixup(struct drm_encoder *encoder,
+				   const struct drm_display_mode *mode,
+				   struct drm_display_mode *adjusted_mode)
+{
+	return true;
+}
+
+static void pdp_tmds_encoder_helper_prepare(struct drm_encoder *encoder)
+{
+}
+
+static void pdp_tmds_encoder_helper_commit(struct drm_encoder *encoder)
+{
+}
+
+static void
+pdp_tmds_encoder_helper_mode_set(struct drm_encoder *encoder,
+				 struct drm_display_mode *mode,
+				 struct drm_display_mode *adjusted_mode)
+{
+}
+
+static void pdp_tmds_encoder_destroy(struct drm_encoder *encoder)
+{
+	struct pdp_drm_private *dev_priv = encoder->dev->dev_private;
+
+	DRM_DEBUG_DRIVER("[ENCODER:%d:%s]\n",
+			 encoder->base.id,
+			 encoder->name);
+
+	drm_encoder_cleanup(encoder);
+
+	kfree(encoder);
+	dev_priv->encoder = NULL;
+}
+
+static const struct drm_encoder_helper_funcs pdp_tmds_encoder_helper_funcs = {
+	.dpms = pdp_tmds_encoder_helper_dpms,
+	.mode_fixup = pdp_tmds_encoder_helper_mode_fixup,
+	.prepare = pdp_tmds_encoder_helper_prepare,
+	.commit = pdp_tmds_encoder_helper_commit,
+	.mode_set = pdp_tmds_encoder_helper_mode_set,
+	.detect = NULL,
+	.disable = NULL,
+};
+
+static const struct drm_encoder_funcs pdp_tmds_encoder_funcs = {
+	.reset = NULL,
+	.destroy = pdp_tmds_encoder_destroy,
+};
+
+struct drm_encoder *
+pdp_tmds_encoder_create(struct drm_device *dev)
+{
+	struct drm_encoder *encoder;
+	int err;
+
+	encoder = kzalloc(sizeof(*encoder), GFP_KERNEL);
+	if (!encoder)
+		return ERR_PTR(-ENOMEM);
+
+	err = drm_encoder_init(dev,
+			       encoder,
+			       &pdp_tmds_encoder_funcs,
+			       DRM_MODE_ENCODER_TMDS,
+			       NULL);
+	if (err) {
+		DRM_ERROR("Failed to initialise encoder");
+		return ERR_PTR(err);
+	}
+	drm_encoder_helper_add(encoder, &pdp_tmds_encoder_helper_funcs);
+
+	/*
+	 * This is a bit field that's used to determine which
+	 * CRTCs can drive this encoder.
+	 */
+	encoder->possible_crtcs = 0x1;
+
+	DRM_DEBUG_DRIVER("[ENCODER:%d:%s]\n",
+			 encoder->base.id,
+			 encoder->name);
+
+	return encoder;
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/odin_defs.h b/drivers/gpu/drm/img-rogue/apollo/odin_defs.h
new file mode 100644
index 000000000000..10ca9b6d4382
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/odin_defs.h
@@ -0,0 +1,338 @@
+/****************************************************************************
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin Memory Map - View from PCIe
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+****************************************************************************/
+
+#ifndef _ODIN_DEFS_H_
+#define _ODIN_DEFS_H_
+
+/* These defines have not been autogenerated */
+
+#define PCI_VENDOR_ID_ODIN                  (0x1AEE)
+#define DEVICE_ID_ODIN                      (0x1010)
+#define DEVICE_ID_VALI                      (0x2010)
+#define DEVICE_ID_TBA                       (0x1CF2)
+
+/* PCI BAR 0 contains the PDP regs and the Odin system regs */
+#define ODN_SYS_BAR                         0
+#define ODN_SYS_REGION_SIZE                 0x000800000 /* 8MB */
+
+#define ODN_SYS_REGS_OFFSET                 0
+#define ODN_SYS_REGS_SIZE                   0x000400000 /* 4MB */
+
+#define ODN_PDP_REGS_OFFSET                 0x000440000
+#define ODN_PDP_REGS_SIZE                   0x000040000 /* 256k */
+
+#define ODN_PDP2_REGS_OFFSET                0x000480000
+#define ODN_PDP2_REGS_SIZE                  0x000040000 /* 256k */
+
+#define ODN_PDP2_PFIM_OFFSET                0x000500000
+#define ODN_PDP2_PFIM_SIZE                  0x000040000 /* 256k */
+
+#define ODIN_DMA_REGS_OFFSET                0x0004C0000
+#define ODIN_DMA_REGS_SIZE                  0x000040000 /* 256k */
+
+#define ODIN_DMA_CHAN_REGS_SIZE             0x000001000 /*   4k */
+
+/* PCI BAR 2 contains the Device Under Test SOCIF 64MB region */
+#define ODN_DUT_SOCIF_BAR                   2
+#define ODN_DUT_SOCIF_OFFSET                0x000000000
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* Reserve only the strictly required register range for each OSID */
+#define ODN_DUT_SOCIF_SIZE                  0x000010000 /* 64k */
+#else
+/* Grant the driver access to the entire register IO range */
+#define ODN_DUT_SOCIF_SIZE                  0x004000000 /* 64MB */
+#endif
+
+/* PCI BAR 4 contains the on-board 1GB DDR memory */
+#define ODN_DDR_BAR                         4
+#define ODN_DDR_MEM_OFFSET                  0x000000000
+#define ODN_DDR_MEM_SIZE                    0x040000000 /* 1GB */
+
+/* Odin system register banks */
+#define ODN_REG_BANK_CORE                   0x00000
+#define ODN_REG_BANK_TCF_SPI_MASTER         0x02000
+#define ODN_REG_BANK_ODN_CLK_BLK            0x0A000
+#define ODN_REG_BANK_ODN_MCU_COMMUNICATOR   0x0C000
+#define ODN_REG_BANK_DB_TYPE_ID             0x0C200
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_TCFVUOCTA   0x000000C6U
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_MASK   0x000000C0U
+#define ODN_REG_BANK_DB_TYPE_ID_TYPE_SHIFT  0x6
+#define ODN_REG_BANK_ODN_I2C                0x0E000
+#define ODN_REG_BANK_MULTI_CLK_ALIGN        0x20000
+#define ODN_REG_BANK_ALIGN_DATA_TX          0x22000
+#define ODN_REG_BANK_SAI_RX_DDR_0           0x24000
+#define ODN_REG_BANK_SAI_RX_DDR(n)          (ODN_REG_BANK_SAI_RX_DDR_0 + (0x02000*n))
+#define ODN_REG_BANK_SAI_TX_DDR_0           0x3A000
+#define ODN_REG_BANK_SAI_TX_DDR(n)          (ODN_REG_BANK_SAI_TX_DDR_0 + (0x02000*n))
+#define ODN_REG_BANK_SAI_TX_SDR             0x4E000
+
+/* Odin SPI regs */
+#define ODN_SPI_MST_ADDR_RDNWR              0x0000
+#define ODN_SPI_MST_WDATA                   0x0004
+#define ODN_SPI_MST_RDATA                   0x0008
+#define ODN_SPI_MST_STATUS                  0x000C
+#define ODN_SPI_MST_GO                      0x0010
+
+
+/*
+   Odin CLK regs - the odn_clk_blk module defs are not auto generated
+ */
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG1           0x620
+#define ODN_PDP_PCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define ODN_PDP_PCLK_ODIV1_LO_TIME_SHIFT         0
+#define ODN_PDP_PCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define ODN_PDP_PCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG2           0x624
+#define ODN_PDP_PCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define ODN_PDP_PCLK_ODIV2_NOCOUNT_SHIFT         6
+#define ODN_PDP_PCLK_ODIV2_EDGE_MASK             0x00000080U
+#define ODN_PDP_PCLK_ODIV2_EDGE_SHIFT            7
+
+#define ODN_PDP_P_CLK_OUT_DIVIDER_REG3           0x61C
+
+#define ODN_PDP_M_CLK_OUT_DIVIDER_REG1           0x628
+#define ODN_PDP_MCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define ODN_PDP_MCLK_ODIV1_LO_TIME_SHIFT         0
+#define ODN_PDP_MCLK_ODIV1_HI_TIME_MASK	         0x00000FC0U
+#define ODN_PDP_MCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define ODN_PDP_M_CLK_OUT_DIVIDER_REG2           0x62C
+#define ODN_PDP_MCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define ODN_PDP_MCLK_ODIV2_NOCOUNT_SHIFT         6
+#define ODN_PDP_MCLK_ODIV2_EDGE_MASK             0x00000080U
+#define ODN_PDP_MCLK_ODIV2_EDGE_SHIFT            7
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG1            0x650
+#define ODN_PDP_PCLK_MUL1_LO_TIME_MASK           0x0000003FU
+#define ODN_PDP_PCLK_MUL1_LO_TIME_SHIFT          0
+#define ODN_PDP_PCLK_MUL1_HI_TIME_MASK           0x00000FC0U
+#define ODN_PDP_PCLK_MUL1_HI_TIME_SHIFT          6
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG2            0x654
+#define ODN_PDP_PCLK_MUL2_NOCOUNT_MASK           0x00000040U
+#define ODN_PDP_PCLK_MUL2_NOCOUNT_SHIFT          6
+#define ODN_PDP_PCLK_MUL2_EDGE_MASK              0x00000080U
+#define ODN_PDP_PCLK_MUL2_EDGE_SHIFT             7
+
+#define ODN_PDP_P_CLK_MULTIPLIER_REG3            0x64C
+
+#define ODN_PDP_P_CLK_IN_DIVIDER_REG             0x658
+#define ODN_PDP_PCLK_IDIV_LO_TIME_MASK           0x0000003FU
+#define ODN_PDP_PCLK_IDIV_LO_TIME_SHIFT          0
+#define ODN_PDP_PCLK_IDIV_HI_TIME_MASK           0x00000FC0U
+#define ODN_PDP_PCLK_IDIV_HI_TIME_SHIFT          6
+#define ODN_PDP_PCLK_IDIV_NOCOUNT_MASK           0x00001000U
+#define ODN_PDP_PCLK_IDIV_NOCOUNT_SHIFT          12
+#define ODN_PDP_PCLK_IDIV_EDGE_MASK              0x00002000U
+#define ODN_PDP_PCLK_IDIV_EDGE_SHIFT             13
+
+/*
+ * DUT core clock input divider, multiplier and out divider.
+ */
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1                (0x0028)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  (0)
+
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2                (0x002C)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_MASK      (0x00000080U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_SHIFT     (7)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_MASK   (0x00000040U)
+#define ODN_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  (6)
+
+#define ODN_DUT_CORE_CLK_MULTIPLIER1                 (0x0050)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME_MASK    (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME_SHIFT   (6)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME_MASK    (0x0000003FU)
+#define ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME_SHIFT   (0)
+
+#define ODN_DUT_CORE_CLK_MULTIPLIER2                 (0x0054)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_MASK       (0x00007000U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_SHIFT      (12)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_EN_MASK    (0x00000800U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_FRAC_EN_SHIFT   (11)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE_MASK       (0x00000080U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE_SHIFT      (7)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT_MASK    (0x00000040U)
+#define ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT_SHIFT   (6)
+
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1                 (0x0058)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE_MASK       (0x00002000U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE_SHIFT      (13)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT_MASK    (0x00001000U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT_SHIFT   (12)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME_MASK    (0x00000FC0U)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME_SHIFT   (6)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME_MASK    (0x0000003FU)
+#define ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME_SHIFT   (0)
+
+/*
+ * DUT interface clock input divider, multiplier and out divider.
+ */
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1               (0x0220)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME_MASK  (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT (6)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME_MASK  (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT (0)
+
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2               (0x0224)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE_MASK     (0x00000080U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE_SHIFT    (7)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT_MASK  (0x00000040U)
+#define ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT (6)
+
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1                (0x0250)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME_SHIFT  (0)
+
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2                (0x0254)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_MASK      (0x00007000U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_SHIFT     (12)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_EN_MASK   (0x00000800U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_FRAC_EN_SHIFT  (11)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE_MASK      (0x00000080U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE_SHIFT     (7)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT_MASK   (0x00000040U)
+#define ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT_SHIFT  (6)
+
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1                (0x0258)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE_MASK      (0x00002000U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE_SHIFT     (13)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT_MASK   (0x00001000U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT_SHIFT  (12)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME_MASK   (0x00000FC0U)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME_SHIFT  (6)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME_MASK   (0x0000003FU)
+#define ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME_SHIFT  (0)
+
+
+/*
+ * Min max values from Xilinx Virtex7 data sheet DS183, for speed grade 2
+ * All in Hz
+ */
+#define ODN_INPUT_CLOCK_SPEED                        (100000000U)
+#define ODN_INPUT_CLOCK_SPEED_MIN                    (10000000U)
+#define ODN_INPUT_CLOCK_SPEED_MAX                    (933000000U)
+#define ODN_OUTPUT_CLOCK_SPEED_MIN                   (4690000U)
+#define ODN_OUTPUT_CLOCK_SPEED_MAX                   (933000000U)
+#define ODN_VCO_MIN                                  (600000000U)
+#define ODN_VCO_MAX                                  (1440000000U)
+#define ODN_PFD_MIN                                  (10000000U)
+#define ODN_PFD_MAX                                  (500000000U)
+
+/*
+ * Max values that can be set in DRP registers
+ */
+#define ODN_OREG_VALUE_MAX                            (126.875f)
+#define ODN_MREG_VALUE_MAX                            (126.875f)
+#define ODN_DREG_VALUE_MAX                            (126U)
+
+
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE                (0x00000001U)
+#define ODN_MMCM_LOCK_STATUS_DUT_IF                  (0x00000002U)
+#define ODN_MMCM_LOCK_STATUS_PDPP                    (0x00000008U)
+
+/*
+    Odin interrupt flags
+*/
+#define ODN_INTERRUPT_ENABLE_PDP1           (1 << ODN_INTERRUPT_ENABLE_PDP1_SHIFT)
+#define ODN_INTERRUPT_ENABLE_PDP2           (1 << ODN_INTERRUPT_ENABLE_PDP2_SHIFT)
+#define ODN_INTERRUPT_ENABLE_DUT            (1 << ODN_INTERRUPT_ENABLE_DUT_SHIFT)
+#define ODN_INTERRUPT_STATUS_PDP1           (1 << ODN_INTERRUPT_STATUS_PDP1_SHIFT)
+#define ODN_INTERRUPT_STATUS_PDP2           (1 << ODN_INTERRUPT_STATUS_PDP2_SHIFT)
+#define ODN_INTERRUPT_STATUS_DUT            (1 << ODN_INTERRUPT_STATUS_DUT_SHIFT)
+#define ODN_INTERRUPT_CLEAR_PDP1            (1 << ODN_INTERRUPT_CLR_PDP1_SHIFT)
+#define ODN_INTERRUPT_CLEAR_PDP2            (1 << ODN_INTERRUPT_CLR_PDP2_SHIFT)
+#define ODN_INTERRUPT_CLEAR_DUT             (1 << ODN_INTERRUPT_CLR_DUT_SHIFT)
+
+#define ODN_INTERRUPT_ENABLE_CDMA           (1 << ODN_INTERRUPT_ENABLE_CDMA_SHIFT)
+#define ODN_INTERRUPT_STATUS_CDMA           (1 << ODN_INTERRUPT_STATUS_CDMA_SHIFT)
+#define ODN_INTERRUPT_CLEAR_CDMA            (1 << ODN_INTERRUPT_CLR_CDMA_SHIFT)
+
+#define ODN_INTERRUPT_ENABLE_CDMA2          (1 << (ODN_INTERRUPT_ENABLE_CDMA_SHIFT + 1))
+#define ODN_INTERRUPT_STATUS_CDMA2          (1 << (ODN_INTERRUPT_STATUS_CDMA_SHIFT + 1))
+#define ODN_INTERRUPT_CLEAR_CDMA2           (1 << (ODN_INTERRUPT_CLR_CDMA_SHIFT + 1))
+
+#define ODN_INTERRUPT_ENABLE_OSID(n)        (1 << (ODN_INTERRUPT_ENABLE_OS_IRQ_SHIFT + (n)))
+#define ODN_INTERRUPT_STATUS_OSID(n)        (1 << (ODN_INTERRUPT_STATUS_OS_IRQ_SHIFT + (n)))
+#define ODN_INTERRUPT_CLEAR_OSID(n)         (1 << (ODN_INTERRUPT_CLR_OS_IRQ_SHIFT + (n)))
+
+/*
+   Other defines
+*/
+#define ODN_STREAM_OFF                      0
+#define ODN_STREAM_ON                       1
+#define ODN_SYNC_GEN_DISABLE                0
+#define ODN_SYNC_GEN_ENABLE                 1
+#define ODN_INTERLACE_DISABLE               0
+#define ODN_INTERLACE_ENABLE                1
+#define ODN_PIXEL_CLOCK_INVERTED            1
+#define ODN_HSYNC_POLARITY_ACTIVE_HIGH      1
+
+#define ODN_PDP_INTCLR_ALL                  0x000FFFFFU
+#define	ODN_PDP_INTSTAT_ALL_OURUN_MASK      0x000FFFF0U
+
+/*
+   DMA defs
+*/
+#define ODN_CDMA_ADDR_WIDTH                35
+#define ODN_DMA_HW_DESC_HEAP_SIZE          0x100000
+#define ODN_DMA_CHAN_RX                    0
+#define ODN_DMA_CHAN_TX                    1
+
+#define ODIN_DMA_TX_CHAN_NAME              "tx"
+#define ODIN_DMA_RX_CHAN_NAME              "rx"
+
+/*
+   FBC defs
+*/
+#define ODIN_PFIM_RELNUM                   (005U)
+
+#endif /* _ODIN_DEFS_H_ */
+
+/*****************************************************************************
+ End of file (odn_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/odin_pdp_regs.h b/drivers/gpu/drm/img-rogue/apollo/odin_pdp_regs.h
new file mode 100644
index 000000000000..da47a253db31
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/odin_pdp_regs.h
@@ -0,0 +1,8540 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* tab size 4 */
+
+#ifndef ODN_PDP_REGS_H
+#define ODN_PDP_REGS_H
+
+/* Odin-PDP hardware register definitions */
+
+
+#define ODN_PDP_GRPH1SURF_OFFSET					(0x0000)
+
+/* PDP, GRPH1SURF, GRPH1PIXFMT
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH1SURF_GRPH1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USEGAMMA
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USECSC
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USELUT
+*/
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH1SURF_GRPH1USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2SURF_OFFSET					(0x0004)
+
+/* PDP, GRPH2SURF, GRPH2PIXFMT
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH2SURF_GRPH2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USEGAMMA
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USECSC
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USELUT
+*/
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH2SURF_GRPH2USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3SURF_OFFSET					(0x0008)
+
+/* PDP, GRPH3SURF, GRPH3PIXFMT
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH3SURF_GRPH3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USEGAMMA
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USECSC
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USELUT
+*/
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH3SURF_GRPH3USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4SURF_OFFSET					(0x000C)
+
+/* PDP, GRPH4SURF, GRPH4PIXFMT
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_LSBMASK		(0x0000001F)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_SHIFT			(27)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_LENGTH		(5)
+#define ODN_PDP_GRPH4SURF_GRPH4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USEGAMMA
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_MASK		(0x04000000)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_SHIFT		(26)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USEGAMMA_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USECSC
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_MASK			(0x02000000)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_SHIFT			(25)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4LUTRWCHOICE
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_MASK		(0x01000000)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SHIFT	(24)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LENGTH	(1)
+#define ODN_PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USELUT
+*/
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_MASK			(0x00800000)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_SHIFT			(23)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_LENGTH		(1)
+#define ODN_PDP_GRPH4SURF_GRPH4USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SURF_OFFSET						(0x0010)
+
+/* PDP, VID1SURF, VID1PIXFMT
+*/
+#define ODN_PDP_VID1SURF_VID1PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID1SURF_VID1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEGAMMA
+*/
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID1SURF_VID1USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USECSC
+*/
+#define ODN_PDP_VID1SURF_VID1USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID1SURF_VID1USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USECSC_SHIFT			(25)
+#define ODN_PDP_VID1SURF_VID1USECSC_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEI2P
+*/
+#define ODN_PDP_VID1SURF_VID1USEI2P_MASK			(0x01000000)
+#define ODN_PDP_VID1SURF_VID1USEI2P_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEI2P_SHIFT			(24)
+#define ODN_PDP_VID1SURF_VID1USEI2P_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USEI2P_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1COSITED
+*/
+#define ODN_PDP_VID1SURF_VID1COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID1SURF_VID1COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1COSITED_SHIFT			(23)
+#define ODN_PDP_VID1SURF_VID1COSITED_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEHQCD
+*/
+#define ODN_PDP_VID1SURF_VID1USEHQCD_MASK			(0x00400000)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_SHIFT			(22)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_LENGTH			(1)
+#define ODN_PDP_VID1SURF_VID1USEHQCD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEINSTREAM
+*/
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_MASK		(0x00200000)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_LSBMASK	(0x00000001)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_SHIFT		(21)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_LENGTH		(1)
+#define ODN_PDP_VID1SURF_VID1USEINSTREAM_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2SURF_OFFSET						(0x0014)
+
+/* PDP, VID2SURF, VID2PIXFMT
+*/
+#define ODN_PDP_VID2SURF_VID2PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID2SURF_VID2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2COSITED
+*/
+#define ODN_PDP_VID2SURF_VID2COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID2SURF_VID2COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SURF_VID2COSITED_SHIFT			(23)
+#define ODN_PDP_VID2SURF_VID2COSITED_LENGTH			(1)
+#define ODN_PDP_VID2SURF_VID2COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USEGAMMA
+*/
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID2SURF_VID2USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USECSC
+*/
+#define ODN_PDP_VID2SURF_VID2USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID2SURF_VID2USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2SURF_VID2USECSC_SHIFT			(25)
+#define ODN_PDP_VID2SURF_VID2USECSC_LENGTH			(1)
+#define ODN_PDP_VID2SURF_VID2USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SURF_OFFSET						(0x0018)
+
+/* PDP, VID3SURF, VID3PIXFMT
+*/
+#define ODN_PDP_VID3SURF_VID3PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID3SURF_VID3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3COSITED
+*/
+#define ODN_PDP_VID3SURF_VID3COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID3SURF_VID3COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SURF_VID3COSITED_SHIFT			(23)
+#define ODN_PDP_VID3SURF_VID3COSITED_LENGTH			(1)
+#define ODN_PDP_VID3SURF_VID3COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USEGAMMA
+*/
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID3SURF_VID3USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USECSC
+*/
+#define ODN_PDP_VID3SURF_VID3USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID3SURF_VID3USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3SURF_VID3USECSC_SHIFT			(25)
+#define ODN_PDP_VID3SURF_VID3USECSC_LENGTH			(1)
+#define ODN_PDP_VID3SURF_VID3USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SURF_OFFSET						(0x001C)
+
+/* PDP, VID4SURF, VID4PIXFMT
+*/
+#define ODN_PDP_VID4SURF_VID4PIXFMT_MASK			(0xF8000000)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_LSBMASK			(0x0000001F)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_SHIFT			(27)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_LENGTH			(5)
+#define ODN_PDP_VID4SURF_VID4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4COSITED
+*/
+#define ODN_PDP_VID4SURF_VID4COSITED_MASK			(0x00800000)
+#define ODN_PDP_VID4SURF_VID4COSITED_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4SURF_VID4COSITED_SHIFT			(23)
+#define ODN_PDP_VID4SURF_VID4COSITED_LENGTH			(1)
+#define ODN_PDP_VID4SURF_VID4COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USEGAMMA
+*/
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_MASK			(0x04000000)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_SHIFT			(26)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_LENGTH		(1)
+#define ODN_PDP_VID4SURF_VID4USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USECSC
+*/
+#define ODN_PDP_VID4SURF_VID4USECSC_MASK			(0x02000000)
+#define ODN_PDP_VID4SURF_VID4USECSC_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4SURF_VID4USECSC_SHIFT			(25)
+#define ODN_PDP_VID4SURF_VID4USECSC_LENGTH			(1)
+#define ODN_PDP_VID4SURF_VID4USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1CTRL_OFFSET					(0x0020)
+
+/* PDP, GRPH1CTRL, GRPH1STREN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_SHIFT			(31)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_LENGTH			(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYEN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYSRC
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLEND
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLENDPOS
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1DITHEREN
+*/
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH1CTRL_GRPH1DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CTRL_OFFSET					(0x0024)
+
+/* PDP, GRPH2CTRL, GRPH2STREN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_SHIFT			(31)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_LENGTH			(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYEN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYSRC
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLEND
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLENDPOS
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2DITHEREN
+*/
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH2CTRL_GRPH2DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CTRL_OFFSET					(0x0028)
+
+/* PDP, GRPH3CTRL, GRPH3STREN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_SHIFT			(31)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_LENGTH			(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYEN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYSRC
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLEND
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLENDPOS
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH3CTRL_GRPH3BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3DITHEREN
+*/
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH3CTRL_GRPH3DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CTRL_OFFSET					(0x002C)
+
+/* PDP, GRPH4CTRL, GRPH4STREN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_MASK			(0x80000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_SHIFT			(31)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_LENGTH			(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYEN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_SHIFT			(30)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYSRC
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_SHIFT		(29)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLEND
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_MASK			(0x18000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_LSBMASK		(0x00000003)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_SHIFT			(27)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_LENGTH			(2)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLENDPOS
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_MASK		(0x07000000)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_SHIFT		(24)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_LENGTH		(3)
+#define ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4DITHEREN
+*/
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_MASK		(0x00800000)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_SHIFT		(23)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_LENGTH		(1)
+#define ODN_PDP_GRPH4CTRL_GRPH4DITHEREN_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1CTRL_OFFSET						(0x0030)
+
+/* PDP, VID1CTRL, VID1STREN
+*/
+#define ODN_PDP_VID1CTRL_VID1STREN_MASK				(0x80000000)
+#define ODN_PDP_VID1CTRL_VID1STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1STREN_SHIFT			(31)
+#define ODN_PDP_VID1CTRL_VID1STREN_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYEN
+*/
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYSRC
+*/
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID1CTRL_VID1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLEND
+*/
+#define ODN_PDP_VID1CTRL_VID1BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID1CTRL_VID1BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID1CTRL_VID1BLEND_SHIFT			(27)
+#define ODN_PDP_VID1CTRL_VID1BLEND_LENGTH			(2)
+#define ODN_PDP_VID1CTRL_VID1BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLENDPOS
+*/
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID1CTRL_VID1BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1DITHEREN
+*/
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID1CTRL_VID1DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CTRL_OFFSET						(0x0034)
+
+/* PDP, VID2CTRL, VID2STREN
+*/
+#define ODN_PDP_VID2CTRL_VID2STREN_MASK				(0x80000000)
+#define ODN_PDP_VID2CTRL_VID2STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2STREN_SHIFT			(31)
+#define ODN_PDP_VID2CTRL_VID2STREN_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYEN
+*/
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYSRC
+*/
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID2CTRL_VID2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLEND
+*/
+#define ODN_PDP_VID2CTRL_VID2BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID2CTRL_VID2BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID2CTRL_VID2BLEND_SHIFT			(27)
+#define ODN_PDP_VID2CTRL_VID2BLEND_LENGTH			(2)
+#define ODN_PDP_VID2CTRL_VID2BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLENDPOS
+*/
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID2CTRL_VID2BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2DITHEREN
+*/
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID2CTRL_VID2DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CTRL_OFFSET						(0x0038)
+
+/* PDP, VID3CTRL, VID3STREN
+*/
+#define ODN_PDP_VID3CTRL_VID3STREN_MASK				(0x80000000)
+#define ODN_PDP_VID3CTRL_VID3STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3STREN_SHIFT			(31)
+#define ODN_PDP_VID3CTRL_VID3STREN_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYEN
+*/
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYSRC
+*/
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID3CTRL_VID3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLEND
+*/
+#define ODN_PDP_VID3CTRL_VID3BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID3CTRL_VID3BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID3CTRL_VID3BLEND_SHIFT			(27)
+#define ODN_PDP_VID3CTRL_VID3BLEND_LENGTH			(2)
+#define ODN_PDP_VID3CTRL_VID3BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLENDPOS
+*/
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID3CTRL_VID3BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3DITHEREN
+*/
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID3CTRL_VID3DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CTRL_OFFSET						(0x003C)
+
+/* PDP, VID4CTRL, VID4STREN
+*/
+#define ODN_PDP_VID4CTRL_VID4STREN_MASK				(0x80000000)
+#define ODN_PDP_VID4CTRL_VID4STREN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4STREN_SHIFT			(31)
+#define ODN_PDP_VID4CTRL_VID4STREN_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4STREN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYEN
+*/
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_MASK			(0x40000000)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_LSBMASK			(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_SHIFT			(30)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYSRC
+*/
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_MASK			(0x20000000)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_SHIFT			(29)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_LENGTH			(1)
+#define ODN_PDP_VID4CTRL_VID4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLEND
+*/
+#define ODN_PDP_VID4CTRL_VID4BLEND_MASK				(0x18000000)
+#define ODN_PDP_VID4CTRL_VID4BLEND_LSBMASK			(0x00000003)
+#define ODN_PDP_VID4CTRL_VID4BLEND_SHIFT			(27)
+#define ODN_PDP_VID4CTRL_VID4BLEND_LENGTH			(2)
+#define ODN_PDP_VID4CTRL_VID4BLEND_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLENDPOS
+*/
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_MASK			(0x07000000)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_LSBMASK		(0x00000007)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_SHIFT			(24)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_LENGTH		(3)
+#define ODN_PDP_VID4CTRL_VID4BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4DITHEREN
+*/
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_MASK			(0x00800000)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_SHIFT			(23)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_LENGTH		(1)
+#define ODN_PDP_VID4CTRL_VID4DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1UCTRL_OFFSET					(0x0050)
+
+/* PDP, VID1UCTRL, VID1UVHALFSTR
+*/
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID1UCTRL_VID1UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2UCTRL_OFFSET					(0x0054)
+
+/* PDP, VID2UCTRL, VID2UVHALFSTR
+*/
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID2UCTRL_VID2UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3UCTRL_OFFSET					(0x0058)
+
+/* PDP, VID3UCTRL, VID3UVHALFSTR
+*/
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID3UCTRL_VID3UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4UCTRL_OFFSET					(0x005C)
+
+/* PDP, VID4UCTRL, VID4UVHALFSTR
+*/
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_MASK		(0xC0000000)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_LSBMASK		(0x00000003)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_SHIFT		(30)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_LENGTH		(2)
+#define ODN_PDP_VID4UCTRL_VID4UVHALFSTR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1STRIDE_OFFSET					(0x0060)
+
+/* PDP, GRPH1STRIDE, GRPH1STRIDE
+*/
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2STRIDE_OFFSET					(0x0064)
+
+/* PDP, GRPH2STRIDE, GRPH2STRIDE
+*/
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3STRIDE_OFFSET					(0x0068)
+
+/* PDP, GRPH3STRIDE, GRPH3STRIDE
+*/
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH3STRIDE_GRPH3STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4STRIDE_OFFSET					(0x006C)
+
+/* PDP, GRPH4STRIDE, GRPH4STRIDE
+*/
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_MASK		(0xFFC00000)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_SHIFT		(22)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_LENGTH		(10)
+#define ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1STRIDE_OFFSET					(0x0070)
+
+/* PDP, VID1STRIDE, VID1STRIDE
+*/
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_SHIFT			(22)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_LENGTH		(10)
+#define ODN_PDP_VID1STRIDE_VID1STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2STRIDE_OFFSET					(0x0074)
+
+/* PDP, VID2STRIDE, VID2STRIDE
+*/
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_SHIFT			(22)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_LENGTH		(10)
+#define ODN_PDP_VID2STRIDE_VID2STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3STRIDE_OFFSET					(0x0078)
+
+/* PDP, VID3STRIDE, VID3STRIDE
+*/
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_SHIFT			(22)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_LENGTH		(10)
+#define ODN_PDP_VID3STRIDE_VID3STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4STRIDE_OFFSET					(0x007C)
+
+/* PDP, VID4STRIDE, VID4STRIDE
+*/
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_MASK			(0xFFC00000)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_SHIFT			(22)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_LENGTH		(10)
+#define ODN_PDP_VID4STRIDE_VID4STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1SIZE_OFFSET					(0x0080)
+
+/* PDP, GRPH1SIZE, GRPH1WIDTH
+*/
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH1SIZE_GRPH1WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SIZE, GRPH1HEIGHT
+*/
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2SIZE_OFFSET					(0x0084)
+
+/* PDP, GRPH2SIZE, GRPH2WIDTH
+*/
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH2SIZE_GRPH2WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SIZE, GRPH2HEIGHT
+*/
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3SIZE_OFFSET					(0x0088)
+
+/* PDP, GRPH3SIZE, GRPH3WIDTH
+*/
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH3SIZE_GRPH3WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SIZE, GRPH3HEIGHT
+*/
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH3SIZE_GRPH3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4SIZE_OFFSET					(0x008C)
+
+/* PDP, GRPH4SIZE, GRPH4WIDTH
+*/
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_SHIFT			(16)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_LENGTH			(12)
+#define ODN_PDP_GRPH4SIZE_GRPH4WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SIZE, GRPH4HEIGHT
+*/
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_SHIFT			(0)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_LENGTH		(12)
+#define ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SIZE_OFFSET						(0x0090)
+
+/* PDP, VID1SIZE, VID1WIDTH
+*/
+#define ODN_PDP_VID1SIZE_VID1WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_SHIFT			(16)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_LENGTH			(12)
+#define ODN_PDP_VID1SIZE_VID1WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID1SIZE, VID1HEIGHT
+*/
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID1SIZE_VID1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SIZE_OFFSET						(0x0094)
+
+/* PDP, VID2SIZE, VID2WIDTH
+*/
+#define ODN_PDP_VID2SIZE_VID2WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_SHIFT			(16)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_LENGTH			(12)
+#define ODN_PDP_VID2SIZE_VID2WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID2SIZE, VID2HEIGHT
+*/
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID2SIZE_VID2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SIZE_OFFSET						(0x0098)
+
+/* PDP, VID3SIZE, VID3WIDTH
+*/
+#define ODN_PDP_VID3SIZE_VID3WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_SHIFT			(16)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_LENGTH			(12)
+#define ODN_PDP_VID3SIZE_VID3WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID3SIZE, VID3HEIGHT
+*/
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID3SIZE_VID3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SIZE_OFFSET						(0x009C)
+
+/* PDP, VID4SIZE, VID4WIDTH
+*/
+#define ODN_PDP_VID4SIZE_VID4WIDTH_MASK				(0x0FFF0000)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_SHIFT			(16)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_LENGTH			(12)
+#define ODN_PDP_VID4SIZE_VID4WIDTH_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VID4SIZE, VID4HEIGHT
+*/
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_MASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_SHIFT			(0)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_LENGTH			(12)
+#define ODN_PDP_VID4SIZE_VID4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1POSN_OFFSET					(0x00A0)
+
+/* PDP, GRPH1POSN, GRPH1XSTART
+*/
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH1POSN_GRPH1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1POSN, GRPH1YSTART
+*/
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH1POSN_GRPH1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2POSN_OFFSET					(0x00A4)
+
+/* PDP, GRPH2POSN, GRPH2XSTART
+*/
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH2POSN_GRPH2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2POSN, GRPH2YSTART
+*/
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH2POSN_GRPH2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3POSN_OFFSET					(0x00A8)
+
+/* PDP, GRPH3POSN, GRPH3XSTART
+*/
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH3POSN_GRPH3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3POSN, GRPH3YSTART
+*/
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH3POSN_GRPH3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4POSN_OFFSET					(0x00AC)
+
+/* PDP, GRPH4POSN, GRPH4XSTART
+*/
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_SHIFT			(16)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_LENGTH		(12)
+#define ODN_PDP_GRPH4POSN_GRPH4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4POSN, GRPH4YSTART
+*/
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_LSBMASK		(0x00000FFF)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_SHIFT			(0)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_LENGTH		(12)
+#define ODN_PDP_GRPH4POSN_GRPH4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1POSN_OFFSET						(0x00B0)
+
+/* PDP, VID1POSN, VID1XSTART
+*/
+#define ODN_PDP_VID1POSN_VID1XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID1POSN_VID1XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1XSTART_SHIFT			(16)
+#define ODN_PDP_VID1POSN_VID1XSTART_LENGTH			(12)
+#define ODN_PDP_VID1POSN_VID1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSN, VID1YSTART
+*/
+#define ODN_PDP_VID1POSN_VID1YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID1POSN_VID1YSTART_SHIFT			(0)
+#define ODN_PDP_VID1POSN_VID1YSTART_LENGTH			(12)
+#define ODN_PDP_VID1POSN_VID1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2POSN_OFFSET						(0x00B4)
+
+/* PDP, VID2POSN, VID2XSTART
+*/
+#define ODN_PDP_VID2POSN_VID2XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID2POSN_VID2XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2XSTART_SHIFT			(16)
+#define ODN_PDP_VID2POSN_VID2XSTART_LENGTH			(12)
+#define ODN_PDP_VID2POSN_VID2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSN, VID2YSTART
+*/
+#define ODN_PDP_VID2POSN_VID2YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID2POSN_VID2YSTART_SHIFT			(0)
+#define ODN_PDP_VID2POSN_VID2YSTART_LENGTH			(12)
+#define ODN_PDP_VID2POSN_VID2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3POSN_OFFSET						(0x00B8)
+
+/* PDP, VID3POSN, VID3XSTART
+*/
+#define ODN_PDP_VID3POSN_VID3XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID3POSN_VID3XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3XSTART_SHIFT			(16)
+#define ODN_PDP_VID3POSN_VID3XSTART_LENGTH			(12)
+#define ODN_PDP_VID3POSN_VID3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSN, VID3YSTART
+*/
+#define ODN_PDP_VID3POSN_VID3YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID3POSN_VID3YSTART_SHIFT			(0)
+#define ODN_PDP_VID3POSN_VID3YSTART_LENGTH			(12)
+#define ODN_PDP_VID3POSN_VID3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4POSN_OFFSET						(0x00BC)
+
+/* PDP, VID4POSN, VID4XSTART
+*/
+#define ODN_PDP_VID4POSN_VID4XSTART_MASK			(0x0FFF0000)
+#define ODN_PDP_VID4POSN_VID4XSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4XSTART_SHIFT			(16)
+#define ODN_PDP_VID4POSN_VID4XSTART_LENGTH			(12)
+#define ODN_PDP_VID4POSN_VID4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSN, VID4YSTART
+*/
+#define ODN_PDP_VID4POSN_VID4YSTART_MASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4YSTART_LSBMASK			(0x00000FFF)
+#define ODN_PDP_VID4POSN_VID4YSTART_SHIFT			(0)
+#define ODN_PDP_VID4POSN_VID4YSTART_LENGTH			(12)
+#define ODN_PDP_VID4POSN_VID4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1GALPHA_OFFSET					(0x00C0)
+
+/* PDP, GRPH1GALPHA, GRPH1GALPHA
+*/
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2GALPHA_OFFSET					(0x00C4)
+
+/* PDP, GRPH2GALPHA, GRPH2GALPHA
+*/
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3GALPHA_OFFSET					(0x00C8)
+
+/* PDP, GRPH3GALPHA, GRPH3GALPHA
+*/
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH3GALPHA_GRPH3GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4GALPHA_OFFSET					(0x00CC)
+
+/* PDP, GRPH4GALPHA, GRPH4GALPHA
+*/
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_SHIFT		(0)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_LENGTH		(10)
+#define ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1GALPHA_OFFSET					(0x00D0)
+
+/* PDP, VID1GALPHA, VID1GALPHA
+*/
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_SHIFT			(0)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_LENGTH		(10)
+#define ODN_PDP_VID1GALPHA_VID1GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2GALPHA_OFFSET					(0x00D4)
+
+/* PDP, VID2GALPHA, VID2GALPHA
+*/
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_SHIFT			(0)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_LENGTH		(10)
+#define ODN_PDP_VID2GALPHA_VID2GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3GALPHA_OFFSET					(0x00D8)
+
+/* PDP, VID3GALPHA, VID3GALPHA
+*/
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_SHIFT			(0)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_LENGTH		(10)
+#define ODN_PDP_VID3GALPHA_VID3GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4GALPHA_OFFSET				    (0x00DC)
+
+/* PDP, VID4GALPHA, VID4GALPHA
+*/
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_MASK			(0x000003FF)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_SHIFT			(0)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_LENGTH		(10)
+#define ODN_PDP_VID4GALPHA_VID4GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1CKEY_R_OFFSET					(0x00E0)
+
+/* PDP, GRPH1CKEY_R, GRPH1CKEY_R
+*/
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_R_GRPH1CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1CKEY_GB_OFFSET			        (0x00E4)
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_G
+*/
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_B
+*/
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CKEY_R_OFFSET					(0x00E8)
+
+/* PDP, GRPH2CKEY_R, GRPH2CKEY_R
+*/
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_R_GRPH2CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2CKEY_GB_OFFSET					(0x00EC)
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_G
+*/
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_B
+*/
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CKEY_R_OFFSET					(0x00F0)
+
+/* PDP, GRPH3CKEY_R, GRPH3CKEY_R
+*/
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_R_GRPH3CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3CKEY_GB_OFFSET					(0x00F4)
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_G
+*/
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_B
+*/
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CKEY_R_OFFSET					(0x00F8)
+
+/* PDP, GRPH4CKEY_R, GRPH4CKEY_R
+*/
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_SHIFT		(0)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_R_GRPH4CKEY_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4CKEY_GB_OFFSET					(0x00FC)
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_G
+*/
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_MASK		(0x03FF0000)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SHIFT		(16)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_B
+*/
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_MASK		(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SHIFT		(0)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LENGTH		(10)
+#define ODN_PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1CKEY_R_OFFSET					(0x0100)
+
+/* PDP, VID1CKEY_R, VID1CKEY_R
+*/
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_R_VID1CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1CKEY_GB_OFFSET					(0x0104)
+
+/* PDP, VID1CKEY_GB, VID1CKEY_G
+*/
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CKEY_GB, VID1CKEY_B
+*/
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID1CKEY_GB_VID1CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CKEY_R_OFFSET					(0x0108)
+
+/* PDP, VID2CKEY_R, VID2CKEY_R
+*/
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_R_VID2CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2CKEY_GB_OFFSET					(0x010C)
+
+/* PDP, VID2CKEY_GB, VID2CKEY_G
+*/
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CKEY_GB, VID2CKEY_B
+*/
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID2CKEY_GB_VID2CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CKEY_R_OFFSET					(0x0110)
+
+/* PDP, VID3CKEY_R, VID3CKEY_R
+*/
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_R_VID3CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3CKEY_GB_OFFSET					(0x0114)
+
+/* PDP, VID3CKEY_GB, VID3CKEY_G
+*/
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CKEY_GB, VID3CKEY_B
+*/
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID3CKEY_GB_VID3CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CKEY_R_OFFSET					(0x0118)
+
+/* PDP, VID4CKEY_R, VID4CKEY_R
+*/
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_MASK			(0x000003FF)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_SHIFT			(0)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_R_VID4CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4CKEY_GB_OFFSET					(0x011C)
+
+/* PDP, VID4CKEY_GB, VID4CKEY_G
+*/
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_MASK			(0x03FF0000)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_SHIFT		(16)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CKEY_GB, VID4CKEY_B
+*/
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_MASK			(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_SHIFT		(0)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_LENGTH		(10)
+#define ODN_PDP_VID4CKEY_GB_VID4CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1BLND2_R_OFFSET					(0x0120)
+
+/* PDP, GRPH1BLND2_R, GRPH1PIXDBL
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1LINDBL
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1CKEYMASK_R
+*/
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1BLND2_GB_OFFSET				(0x0124)
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_G
+*/
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_B
+*/
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BLND2_R_OFFSET					(0x0128)
+
+/* PDP, GRPH2BLND2_R, GRPH2PIXDBL
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2LINDBL
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2CKEYMASK_R
+*/
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BLND2_GB_OFFSET				(0x012C)
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_G
+*/
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_B
+*/
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BLND2_R_OFFSET					(0x0130)
+
+/* PDP, GRPH3BLND2_R, GRPH3PIXDBL
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3LINDBL
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3CKEYMASK_R
+*/
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BLND2_GB_OFFSET				(0x0134)
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_G
+*/
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_B
+*/
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BLND2_R_OFFSET					(0x0138)
+
+/* PDP, GRPH4BLND2_R, GRPH4PIXDBL
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_MASK		(0x80000000)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_SHIFT		(31)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_LENGTH		(1)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4PIXDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4LINDBL
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_MASK		(0x20000000)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_LSBMASK	(0x00000001)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_SHIFT		(29)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_LENGTH		(1)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4LINDBL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4CKEYMASK_R
+*/
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_MASK	(0x000003FF)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BLND2_GB_OFFSET				(0x013C)
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_G
+*/
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LENGTH (10)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_B
+*/
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LSBMASK (0x000003FF)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LENGTH (10)
+#define ODN_PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1BLND2_R_OFFSET					(0x0140)
+
+/* PDP, VID1BLND2_R, VID1CKEYMASK_R
+*/
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_R_VID1CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1BLND2_GB_OFFSET					(0x0144)
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_G
+*/
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_B
+*/
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID1BLND2_GB_VID1CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2BLND2_R_OFFSET		            (0x0148)
+
+/* PDP, VID2BLND2_R, VID2CKEYMASK_R
+*/
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_R_VID2CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2BLND2_GB_OFFSET		            (0x014C)
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_G
+*/
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_B
+*/
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID2BLND2_GB_VID2CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3BLND2_R_OFFSET		            (0x0150)
+
+/* PDP, VID3BLND2_R, VID3CKEYMASK_R
+*/
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_R_VID3CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3BLND2_GB_OFFSET		            (0x0154)
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_G
+*/
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_B
+*/
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID3BLND2_GB_VID3CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4BLND2_R_OFFSET		            (0x0158)
+
+/* PDP, VID4BLND2_R, VID4CKEYMASK_R
+*/
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_MASK		(0x000003FF)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_SHIFT	(0)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_R_VID4CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4BLND2_GB_OFFSET		            (0x015C)
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_G
+*/
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_MASK	(0x03FF0000)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_SHIFT	(16)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_G_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_B
+*/
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_MASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_SHIFT	(0)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_LENGTH	(10)
+#define ODN_PDP_VID4BLND2_GB_VID4CKEYMASK_B_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_OFFSET		    (0x0160)
+
+/* PDP, GRPH1INTERLEAVE_CTRL, GRPH1INTFIELD
+*/
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_MASK	(0x00000001)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_OFFSET		    (0x0164)
+
+/* PDP, GRPH2INTERLEAVE_CTRL, GRPH2INTFIELD
+*/
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_OFFSET		    (0x0168)
+
+/* PDP, GRPH3INTERLEAVE_CTRL, GRPH3INTFIELD
+*/
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_OFFSET		    (0x016C)
+
+/* PDP, GRPH4INTERLEAVE_CTRL, GRPH4INTFIELD
+*/
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_MASK (0x00000001)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SHIFT (0)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LENGTH (1)
+#define ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1INTERLEAVE_CTRL_OFFSET		    (0x0170)
+
+/* PDP, VID1INTERLEAVE_CTRL, VID1INTFIELD
+*/
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SHIFT (0)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LENGTH (1)
+#define ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2INTERLEAVE_CTRL_OFFSET		    (0x0174)
+
+/* PDP, VID2INTERLEAVE_CTRL, VID2INTFIELD
+*/
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SHIFT (0)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LENGTH (1)
+#define ODN_PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3INTERLEAVE_CTRL_OFFSET		    (0x0178)
+
+/* PDP, VID3INTERLEAVE_CTRL, VID3INTFIELD
+*/
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SHIFT (0)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LENGTH (1)
+#define ODN_PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4INTERLEAVE_CTRL_OFFSET		    (0x017C)
+
+/* PDP, VID4INTERLEAVE_CTRL, VID4INTFIELD
+*/
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_MASK (0x00000001)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LSBMASK (0x00000001)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SHIFT (0)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LENGTH (1)
+#define ODN_PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH1BASEADDR_OFFSET		        (0x0180)
+
+/* PDP, GRPH1BASEADDR, GRPH1BASEADDR
+*/
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH1BASEADDR_GRPH1BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH2BASEADDR_OFFSET		        (0x0184)
+
+/* PDP, GRPH2BASEADDR, GRPH2BASEADDR
+*/
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH2BASEADDR_GRPH2BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH3BASEADDR_OFFSET		        (0x0188)
+
+/* PDP, GRPH3BASEADDR, GRPH3BASEADDR
+*/
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH3BASEADDR_GRPH3BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_GRPH4BASEADDR_OFFSET		        (0x018C)
+
+/* PDP, GRPH4BASEADDR, GRPH4BASEADDR
+*/
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_SHIFT	(5)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_LENGTH	(27)
+#define ODN_PDP_GRPH4BASEADDR_GRPH4BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1BASEADDR_OFFSET		            (0x0190)
+
+/* PDP, VID1BASEADDR, VID1BASEADDR
+*/
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID1BASEADDR_VID1BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2BASEADDR_OFFSET		            (0x0194)
+
+/* PDP, VID2BASEADDR, VID2BASEADDR
+*/
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID2BASEADDR_VID2BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID3BASEADDR_OFFSET		            (0x0198)
+
+/* PDP, VID3BASEADDR, VID3BASEADDR
+*/
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID3BASEADDR_VID3BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID4BASEADDR_OFFSET		            (0x019C)
+
+/* PDP, VID4BASEADDR, VID4BASEADDR
+*/
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_LENGTH	(27)
+#define ODN_PDP_VID4BASEADDR_VID4BASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID1UBASEADDR_OFFSET				(0x01B0)
+
+/* PDP, VID1UBASEADDR, VID1UBASEADDR
+*/
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_MASK	(0xFFFFFFE0)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LSBMASK	(0x07FFFFFF)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_SHIFT	(5)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH	(27)
+#define ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_VID2UBASEADDR_OFFSET		        (0x01B4)
+
+/* PDP, VID2UBASEADDR, VID2UBASEADDR
+*/
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID2UBASEADDR_VID2UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3UBASEADDR_OFFSET		        (0x01B8)
+
+/* PDP, VID3UBASEADDR, VID3UBASEADDR
+*/
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID3UBASEADDR_VID3UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4UBASEADDR_OFFSET		        (0x01BC)
+
+/* PDP, VID4UBASEADDR, VID4UBASEADDR
+*/
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID4UBASEADDR_VID4UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VBASEADDR_OFFSET		        (0x01D0)
+
+/* PDP, VID1VBASEADDR, VID1VBASEADDR
+*/
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VBASEADDR_OFFSET		        (0x01D4)
+
+/* PDP, VID2VBASEADDR, VID2VBASEADDR
+*/
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID2VBASEADDR_VID2VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VBASEADDR_OFFSET		        (0x01D8)
+
+/* PDP, VID3VBASEADDR, VID3VBASEADDR
+*/
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID3VBASEADDR_VID3VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VBASEADDR_OFFSET		(0x01DC)
+
+/* PDP, VID4VBASEADDR, VID4VBASEADDR
+*/
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_MASK		(0xFFFFFFE0)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_SHIFT		(5)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_LENGTH		(27)
+#define ODN_PDP_VID4VBASEADDR_VID4VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1POSTSKIPCTRL_OFFSET		(0x0230)
+
+/* PDP, VID1POSTSKIPCTRL, VID1HPOSTCLIP
+*/
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSTSKIPCTRL, VID1VPOSTCLIP
+*/
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2POSTSKIPCTRL_OFFSET		(0x0234)
+
+/* PDP, VID2POSTSKIPCTRL, VID2HPOSTCLIP
+*/
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSTSKIPCTRL, VID2VPOSTCLIP
+*/
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3POSTSKIPCTRL_OFFSET		(0x0238)
+
+/* PDP, VID3POSTSKIPCTRL, VID3HPOSTCLIP
+*/
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSTSKIPCTRL, VID3VPOSTCLIP
+*/
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4POSTSKIPCTRL_OFFSET		(0x023C)
+
+/* PDP, VID4POSTSKIPCTRL, VID4HPOSTCLIP
+*/
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_MASK		(0x007F0000)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SHIFT		(16)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LENGTH		(7)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSTSKIPCTRL, VID4VPOSTCLIP
+*/
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_MASK		(0x0000003F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LSBMASK		(0x0000003F)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SHIFT		(0)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LENGTH		(6)
+#define ODN_PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1DECIMATE_CTRL_OFFSET		(0x0240)
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_EN
+*/
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2DECIMATE_CTRL_OFFSET		(0x0244)
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_EN
+*/
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3DECIMATE_CTRL_OFFSET		(0x0248)
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_EN
+*/
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4DECIMATE_CTRL_OFFSET		(0x024C)
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_EN
+*/
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1DECIMATE_CTRL_OFFSET		(0x0250)
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_EN
+*/
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2DECIMATE_CTRL_OFFSET		(0x0254)
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_EN
+*/
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3DECIMATE_CTRL_OFFSET		(0x0258)
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_EN
+*/
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4DECIMATE_CTRL_OFFSET		(0x025C)
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_MODE
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_PIXEL_HALVE
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_EN
+*/
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_MASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SHIFT		(0)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LENGTH		(1)
+#define ODN_PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SKIPCTRL_OFFSET		(0x0270)
+
+/* PDP, VID1SKIPCTRL, VID1HSKIP
+*/
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_SHIFT		(16)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_LENGTH		(12)
+#define ODN_PDP_VID1SKIPCTRL_VID1HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SKIPCTRL, VID1VSKIP
+*/
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_SHIFT		(0)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_LENGTH		(12)
+#define ODN_PDP_VID1SKIPCTRL_VID1VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SKIPCTRL_OFFSET		(0x0274)
+
+/* PDP, VID2SKIPCTRL, VID2HSKIP
+*/
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_SHIFT		(16)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_LENGTH		(12)
+#define ODN_PDP_VID2SKIPCTRL_VID2HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SKIPCTRL, VID2VSKIP
+*/
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_SHIFT		(0)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_LENGTH		(12)
+#define ODN_PDP_VID2SKIPCTRL_VID2VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SKIPCTRL_OFFSET		(0x0278)
+
+/* PDP, VID3SKIPCTRL, VID3HSKIP
+*/
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_SHIFT		(16)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_LENGTH		(12)
+#define ODN_PDP_VID3SKIPCTRL_VID3HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SKIPCTRL, VID3VSKIP
+*/
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_SHIFT		(0)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_LENGTH		(12)
+#define ODN_PDP_VID3SKIPCTRL_VID3VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SKIPCTRL_OFFSET		(0x027C)
+
+/* PDP, VID4SKIPCTRL, VID4HSKIP
+*/
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_SHIFT		(16)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_LENGTH		(12)
+#define ODN_PDP_VID4SKIPCTRL_VID4HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SKIPCTRL, VID4VSKIP
+*/
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_MASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_SHIFT		(0)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_LENGTH		(12)
+#define ODN_PDP_VID4SKIPCTRL_VID4VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SCALECTRL_OFFSET		(0x0460)
+
+/* PDP, VID1SCALECTRL, VID1HSCALEBP
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSCALEBP
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1HSBEFOREVS
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSURUNCTRL
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1PAN_EN
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID1SCALECTRL_VID1PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VORDER
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_SHIFT		(16)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_LENGTH		(2)
+#define ODN_PDP_VID1SCALECTRL_VID1VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VPITCH
+*/
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_SHIFT		(0)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_LENGTH		(16)
+#define ODN_PDP_VID1SCALECTRL_VID1VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VSINIT_OFFSET		(0x0464)
+
+/* PDP, VID1VSINIT, VID1VINITIAL1
+*/
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1VSINIT, VID1VINITIAL0
+*/
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID1VSINIT_VID1VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF0_OFFSET		(0x0468)
+
+/* PDP, VID1VCOEFF0, VID1VCOEFF0
+*/
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF0_VID1VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF1_OFFSET		(0x046C)
+
+/* PDP, VID1VCOEFF1, VID1VCOEFF1
+*/
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF1_VID1VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF2_OFFSET		(0x0470)
+
+/* PDP, VID1VCOEFF2, VID1VCOEFF2
+*/
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF2_VID1VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF3_OFFSET		(0x0474)
+
+/* PDP, VID1VCOEFF3, VID1VCOEFF3
+*/
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF3_VID1VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF4_OFFSET		(0x0478)
+
+/* PDP, VID1VCOEFF4, VID1VCOEFF4
+*/
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF4_VID1VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF5_OFFSET		(0x047C)
+
+/* PDP, VID1VCOEFF5, VID1VCOEFF5
+*/
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF5_VID1VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF6_OFFSET		(0x0480)
+
+/* PDP, VID1VCOEFF6, VID1VCOEFF6
+*/
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF6_VID1VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF7_OFFSET		(0x0484)
+
+/* PDP, VID1VCOEFF7, VID1VCOEFF7
+*/
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID1VCOEFF7_VID1VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1VCOEFF8_OFFSET		(0x0488)
+
+/* PDP, VID1VCOEFF8, VID1VCOEFF8
+*/
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID1VCOEFF8_VID1VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HSINIT_OFFSET		(0x048C)
+
+/* PDP, VID1HSINIT, VID1HINITIAL
+*/
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID1HSINIT_VID1HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1HSINIT, VID1HPITCH
+*/
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_SHIFT		(0)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_LENGTH		(16)
+#define ODN_PDP_VID1HSINIT_VID1HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF0_OFFSET		(0x0490)
+
+/* PDP, VID1HCOEFF0, VID1HCOEFF0
+*/
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF0_VID1HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF1_OFFSET		(0x0494)
+
+/* PDP, VID1HCOEFF1, VID1HCOEFF1
+*/
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF1_VID1HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF2_OFFSET		(0x0498)
+
+/* PDP, VID1HCOEFF2, VID1HCOEFF2
+*/
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF2_VID1HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF3_OFFSET		(0x049C)
+
+/* PDP, VID1HCOEFF3, VID1HCOEFF3
+*/
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF3_VID1HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF4_OFFSET		(0x04A0)
+
+/* PDP, VID1HCOEFF4, VID1HCOEFF4
+*/
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF4_VID1HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF5_OFFSET		(0x04A4)
+
+/* PDP, VID1HCOEFF5, VID1HCOEFF5
+*/
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF5_VID1HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF6_OFFSET		(0x04A8)
+
+/* PDP, VID1HCOEFF6, VID1HCOEFF6
+*/
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF6_VID1HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF7_OFFSET		(0x04AC)
+
+/* PDP, VID1HCOEFF7, VID1HCOEFF7
+*/
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF7_VID1HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF8_OFFSET		(0x04B0)
+
+/* PDP, VID1HCOEFF8, VID1HCOEFF8
+*/
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF8_VID1HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF9_OFFSET		(0x04B4)
+
+/* PDP, VID1HCOEFF9, VID1HCOEFF9
+*/
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF9_VID1HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF10_OFFSET		(0x04B8)
+
+/* PDP, VID1HCOEFF10, VID1HCOEFF10
+*/
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF10_VID1HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF11_OFFSET		(0x04BC)
+
+/* PDP, VID1HCOEFF11, VID1HCOEFF11
+*/
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF11_VID1HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF12_OFFSET		(0x04C0)
+
+/* PDP, VID1HCOEFF12, VID1HCOEFF12
+*/
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF12_VID1HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF13_OFFSET		(0x04C4)
+
+/* PDP, VID1HCOEFF13, VID1HCOEFF13
+*/
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF13_VID1HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF14_OFFSET		(0x04C8)
+
+/* PDP, VID1HCOEFF14, VID1HCOEFF14
+*/
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF14_VID1HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF15_OFFSET		(0x04CC)
+
+/* PDP, VID1HCOEFF15, VID1HCOEFF15
+*/
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID1HCOEFF15_VID1HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1HCOEFF16_OFFSET		(0x04D0)
+
+/* PDP, VID1HCOEFF16, VID1HCOEFF16
+*/
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID1HCOEFF16_VID1HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1SCALESIZE_OFFSET		(0x04D4)
+
+/* PDP, VID1SCALESIZE, VID1SCALEWIDTH
+*/
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALESIZE, VID1SCALEHEIGHT
+*/
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CORE_ID_OFFSET		(0x04E0)
+
+/* PDP, PVR_ODN_PDP_CORE_ID, GROUP_ID
+*/
+#define ODN_PDP_CORE_ID_GROUP_ID_MASK		(0xFF000000)
+#define ODN_PDP_CORE_ID_GROUP_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_ID_GROUP_ID_SHIFT		(24)
+#define ODN_PDP_CORE_ID_GROUP_ID_LENGTH		(8)
+#define ODN_PDP_CORE_ID_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_ID, CORE_ID
+*/
+#define ODN_PDP_CORE_ID_CORE_ID_MASK		(0x00FF0000)
+#define ODN_PDP_CORE_ID_CORE_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_ID_CORE_ID_SHIFT		(16)
+#define ODN_PDP_CORE_ID_CORE_ID_LENGTH		(8)
+#define ODN_PDP_CORE_ID_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_ID, CONFIG_ID
+*/
+#define ODN_PDP_CORE_ID_CONFIG_ID_MASK		(0x0000FFFF)
+#define ODN_PDP_CORE_ID_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_CORE_ID_CONFIG_ID_SHIFT		(0)
+#define ODN_PDP_CORE_ID_CONFIG_ID_LENGTH		(16)
+#define ODN_PDP_CORE_ID_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CORE_REV_OFFSET		(0x04F0)
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MAJOR_REV
+*/
+#define ODN_PDP_CORE_REV_MAJOR_REV_MASK		(0x00FF0000)
+#define ODN_PDP_CORE_REV_MAJOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAJOR_REV_SHIFT		(16)
+#define ODN_PDP_CORE_REV_MAJOR_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MINOR_REV
+*/
+#define ODN_PDP_CORE_REV_MINOR_REV_MASK		(0x0000FF00)
+#define ODN_PDP_CORE_REV_MINOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MINOR_REV_SHIFT		(8)
+#define ODN_PDP_CORE_REV_MINOR_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_CORE_REV, MAINT_REV
+*/
+#define ODN_PDP_CORE_REV_MAINT_REV_MASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAINT_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_CORE_REV_MAINT_REV_SHIFT		(0)
+#define ODN_PDP_CORE_REV_MAINT_REV_LENGTH		(8)
+#define ODN_PDP_CORE_REV_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SCALECTRL_OFFSET		(0x0500)
+
+/* PDP, VID2SCALECTRL, VID2HSCALEBP
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSCALEBP
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2HSBEFOREVS
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSURUNCTRL
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2PAN_EN
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID2SCALECTRL_VID2PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VORDER
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_SHIFT		(16)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_LENGTH		(2)
+#define ODN_PDP_VID2SCALECTRL_VID2VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VPITCH
+*/
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_SHIFT		(0)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_LENGTH		(16)
+#define ODN_PDP_VID2SCALECTRL_VID2VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VSINIT_OFFSET		(0x0504)
+
+/* PDP, VID2VSINIT, VID2VINITIAL1
+*/
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2VSINIT, VID2VINITIAL0
+*/
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID2VSINIT_VID2VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF0_OFFSET		(0x0508)
+
+/* PDP, VID2VCOEFF0, VID2VCOEFF0
+*/
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF0_VID2VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF1_OFFSET		(0x050C)
+
+/* PDP, VID2VCOEFF1, VID2VCOEFF1
+*/
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF1_VID2VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF2_OFFSET		(0x0510)
+
+/* PDP, VID2VCOEFF2, VID2VCOEFF2
+*/
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF2_VID2VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF3_OFFSET		(0x0514)
+
+/* PDP, VID2VCOEFF3, VID2VCOEFF3
+*/
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF3_VID2VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF4_OFFSET		(0x0518)
+
+/* PDP, VID2VCOEFF4, VID2VCOEFF4
+*/
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF4_VID2VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF5_OFFSET		(0x051C)
+
+/* PDP, VID2VCOEFF5, VID2VCOEFF5
+*/
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF5_VID2VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF6_OFFSET		(0x0520)
+
+/* PDP, VID2VCOEFF6, VID2VCOEFF6
+*/
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF6_VID2VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF7_OFFSET		(0x0524)
+
+/* PDP, VID2VCOEFF7, VID2VCOEFF7
+*/
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID2VCOEFF7_VID2VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2VCOEFF8_OFFSET		(0x0528)
+
+/* PDP, VID2VCOEFF8, VID2VCOEFF8
+*/
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID2VCOEFF8_VID2VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HSINIT_OFFSET		(0x052C)
+
+/* PDP, VID2HSINIT, VID2HINITIAL
+*/
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID2HSINIT_VID2HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2HSINIT, VID2HPITCH
+*/
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_SHIFT		(0)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_LENGTH		(16)
+#define ODN_PDP_VID2HSINIT_VID2HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF0_OFFSET		(0x0530)
+
+/* PDP, VID2HCOEFF0, VID2HCOEFF0
+*/
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF0_VID2HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF1_OFFSET		(0x0534)
+
+/* PDP, VID2HCOEFF1, VID2HCOEFF1
+*/
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF1_VID2HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF2_OFFSET		(0x0538)
+
+/* PDP, VID2HCOEFF2, VID2HCOEFF2
+*/
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF2_VID2HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF3_OFFSET		(0x053C)
+
+/* PDP, VID2HCOEFF3, VID2HCOEFF3
+*/
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF3_VID2HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF4_OFFSET		(0x0540)
+
+/* PDP, VID2HCOEFF4, VID2HCOEFF4
+*/
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF4_VID2HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF5_OFFSET		(0x0544)
+
+/* PDP, VID2HCOEFF5, VID2HCOEFF5
+*/
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF5_VID2HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF6_OFFSET		(0x0548)
+
+/* PDP, VID2HCOEFF6, VID2HCOEFF6
+*/
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF6_VID2HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF7_OFFSET		(0x054C)
+
+/* PDP, VID2HCOEFF7, VID2HCOEFF7
+*/
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF7_VID2HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF8_OFFSET		(0x0550)
+
+/* PDP, VID2HCOEFF8, VID2HCOEFF8
+*/
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF8_VID2HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF9_OFFSET		(0x0554)
+
+/* PDP, VID2HCOEFF9, VID2HCOEFF9
+*/
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF9_VID2HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF10_OFFSET		(0x0558)
+
+/* PDP, VID2HCOEFF10, VID2HCOEFF10
+*/
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF10_VID2HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF11_OFFSET		(0x055C)
+
+/* PDP, VID2HCOEFF11, VID2HCOEFF11
+*/
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF11_VID2HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF12_OFFSET		(0x0560)
+
+/* PDP, VID2HCOEFF12, VID2HCOEFF12
+*/
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF12_VID2HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF13_OFFSET		(0x0564)
+
+/* PDP, VID2HCOEFF13, VID2HCOEFF13
+*/
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF13_VID2HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF14_OFFSET		(0x0568)
+
+/* PDP, VID2HCOEFF14, VID2HCOEFF14
+*/
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF14_VID2HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF15_OFFSET		(0x056C)
+
+/* PDP, VID2HCOEFF15, VID2HCOEFF15
+*/
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID2HCOEFF15_VID2HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2HCOEFF16_OFFSET		(0x0570)
+
+/* PDP, VID2HCOEFF16, VID2HCOEFF16
+*/
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID2HCOEFF16_VID2HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2SCALESIZE_OFFSET		(0x0574)
+
+/* PDP, VID2SCALESIZE, VID2SCALEWIDTH
+*/
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALESIZE, VID2SCALEHEIGHT
+*/
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SCALECTRL_OFFSET		(0x0578)
+
+/* PDP, VID3SCALECTRL, VID3HSCALEBP
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_SHIFT		(31)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSCALEBP
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_SHIFT		(30)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3HSBEFOREVS
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_MASK		(0x20000000)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_SHIFT		(29)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSURUNCTRL
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_MASK		(0x08000000)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_SHIFT		(27)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3PAN_EN
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID3SCALECTRL_VID3PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VORDER
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_LSBMASK		(0x00000003)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_SHIFT		(16)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_LENGTH		(2)
+#define ODN_PDP_VID3SCALECTRL_VID3VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VPITCH
+*/
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_SHIFT		(0)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_LENGTH		(16)
+#define ODN_PDP_VID3SCALECTRL_VID3VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VSINIT_OFFSET		(0x057C)
+
+/* PDP, VID3VSINIT, VID3VINITIAL1
+*/
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3VSINIT, VID3VINITIAL0
+*/
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID3VSINIT_VID3VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF0_OFFSET		(0x0580)
+
+/* PDP, VID3VCOEFF0, VID3VCOEFF0
+*/
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF0_VID3VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF1_OFFSET		(0x0584)
+
+/* PDP, VID3VCOEFF1, VID3VCOEFF1
+*/
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF1_VID3VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF2_OFFSET		(0x0588)
+
+/* PDP, VID3VCOEFF2, VID3VCOEFF2
+*/
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF2_VID3VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF3_OFFSET		(0x058C)
+
+/* PDP, VID3VCOEFF3, VID3VCOEFF3
+*/
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF3_VID3VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF4_OFFSET		(0x0590)
+
+/* PDP, VID3VCOEFF4, VID3VCOEFF4
+*/
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF4_VID3VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF5_OFFSET		(0x0594)
+
+/* PDP, VID3VCOEFF5, VID3VCOEFF5
+*/
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF5_VID3VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF6_OFFSET		(0x0598)
+
+/* PDP, VID3VCOEFF6, VID3VCOEFF6
+*/
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF6_VID3VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF7_OFFSET		(0x059C)
+
+/* PDP, VID3VCOEFF7, VID3VCOEFF7
+*/
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID3VCOEFF7_VID3VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3VCOEFF8_OFFSET		(0x05A0)
+
+/* PDP, VID3VCOEFF8, VID3VCOEFF8
+*/
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID3VCOEFF8_VID3VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HSINIT_OFFSET		(0x05A4)
+
+/* PDP, VID3HSINIT, VID3HINITIAL
+*/
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID3HSINIT_VID3HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3HSINIT, VID3HPITCH
+*/
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_SHIFT		(0)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_LENGTH		(16)
+#define ODN_PDP_VID3HSINIT_VID3HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF0_OFFSET		(0x05A8)
+
+/* PDP, VID3HCOEFF0, VID3HCOEFF0
+*/
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF0_VID3HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF1_OFFSET		(0x05AC)
+
+/* PDP, VID3HCOEFF1, VID3HCOEFF1
+*/
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF1_VID3HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF2_OFFSET		(0x05B0)
+
+/* PDP, VID3HCOEFF2, VID3HCOEFF2
+*/
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF2_VID3HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF3_OFFSET		(0x05B4)
+
+/* PDP, VID3HCOEFF3, VID3HCOEFF3
+*/
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF3_VID3HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF4_OFFSET		(0x05B8)
+
+/* PDP, VID3HCOEFF4, VID3HCOEFF4
+*/
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF4_VID3HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF5_OFFSET		(0x05BC)
+
+/* PDP, VID3HCOEFF5, VID3HCOEFF5
+*/
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF5_VID3HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF6_OFFSET		(0x05C0)
+
+/* PDP, VID3HCOEFF6, VID3HCOEFF6
+*/
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF6_VID3HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF7_OFFSET		(0x05C4)
+
+/* PDP, VID3HCOEFF7, VID3HCOEFF7
+*/
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF7_VID3HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF8_OFFSET		(0x05C8)
+
+/* PDP, VID3HCOEFF8, VID3HCOEFF8
+*/
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF8_VID3HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF9_OFFSET		(0x05CC)
+
+/* PDP, VID3HCOEFF9, VID3HCOEFF9
+*/
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF9_VID3HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF10_OFFSET		(0x05D0)
+
+/* PDP, VID3HCOEFF10, VID3HCOEFF10
+*/
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_LENGTH	(32)
+#define ODN_PDP_VID3HCOEFF10_VID3HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF11_OFFSET		(0x05D4)
+
+/* PDP, VID3HCOEFF11, VID3HCOEFF11
+*/
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_LENGTH	(32)
+#define ODN_PDP_VID3HCOEFF11_VID3HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF12_OFFSET		(0x05D8)
+
+/* PDP, VID3HCOEFF12, VID3HCOEFF12
+*/
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF12_VID3HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF13_OFFSET		(0x05DC)
+
+/* PDP, VID3HCOEFF13, VID3HCOEFF13
+*/
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF13_VID3HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF14_OFFSET		(0x05E0)
+
+/* PDP, VID3HCOEFF14, VID3HCOEFF14
+*/
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF14_VID3HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF15_OFFSET		(0x05E4)
+
+/* PDP, VID3HCOEFF15, VID3HCOEFF15
+*/
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_LENGTH		(32)
+#define ODN_PDP_VID3HCOEFF15_VID3HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3HCOEFF16_OFFSET		(0x05E8)
+
+/* PDP, VID3HCOEFF16, VID3HCOEFF16
+*/
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_LENGTH		(8)
+#define ODN_PDP_VID3HCOEFF16_VID3HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3SCALESIZE_OFFSET		(0x05EC)
+
+/* PDP, VID3SCALESIZE, VID3SCALEWIDTH
+*/
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALESIZE, VID3SCALEHEIGHT
+*/
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SCALECTRL_OFFSET		(0x05F0)
+
+/* PDP, VID4SCALECTRL, VID4HSCALEBP
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_MASK		(0x80000000)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_SHIFT	(31)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSCALEBP
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_MASK		(0x40000000)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_SHIFT	(30)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4HSBEFOREVS
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_MASK	(0x20000000)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_SHIFT	(29)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4HSBEFOREVS_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSURUNCTRL
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_MASK	(0x08000000)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_SHIFT	(27)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_LENGTH	(1)
+#define ODN_PDP_VID4SCALECTRL_VID4VSURUNCTRL_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4PAN_EN
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_MASK		(0x00040000)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_SHIFT		(18)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_LENGTH		(1)
+#define ODN_PDP_VID4SCALECTRL_VID4PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VORDER
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_MASK		(0x00030000)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_LSBMASK	(0x00000003)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_SHIFT		(16)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_LENGTH		(2)
+#define ODN_PDP_VID4SCALECTRL_VID4VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VPITCH
+*/
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_LSBMASK	(0x0000FFFF)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_SHIFT		(0)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_LENGTH		(16)
+#define ODN_PDP_VID4SCALECTRL_VID4VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VSINIT_OFFSET			(0x05F4)
+
+/* PDP, VID4VSINIT, VID4VINITIAL1
+*/
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_MASK		(0xFFFF0000)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_SHIFT		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_LENGTH		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4VSINIT, VID4VINITIAL0
+*/
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_SHIFT		(0)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_LENGTH		(16)
+#define ODN_PDP_VID4VSINIT_VID4VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF0_OFFSET			(0x05F8)
+
+/* PDP, VID4VCOEFF0, VID4VCOEFF0
+*/
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF0_VID4VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF1_OFFSET			(0x05FC)
+
+/* PDP, VID4VCOEFF1, VID4VCOEFF1
+*/
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF1_VID4VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF2_OFFSET			(0x0600)
+
+/* PDP, VID4VCOEFF2, VID4VCOEFF2
+*/
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF2_VID4VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF3_OFFSET			(0x0604)
+
+/* PDP, VID4VCOEFF3, VID4VCOEFF3
+*/
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF3_VID4VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF4_OFFSET			(0x0608)
+
+/* PDP, VID4VCOEFF4, VID4VCOEFF4
+*/
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF4_VID4VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF5_OFFSET			(0x060C)
+
+/* PDP, VID4VCOEFF5, VID4VCOEFF5
+*/
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF5_VID4VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF6_OFFSET			(0x0610)
+
+/* PDP, VID4VCOEFF6, VID4VCOEFF6
+*/
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF6_VID4VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF7_OFFSET			(0x0614)
+
+/* PDP, VID4VCOEFF7, VID4VCOEFF7
+*/
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID4VCOEFF7_VID4VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4VCOEFF8_OFFSET			(0x0618)
+
+/* PDP, VID4VCOEFF8, VID4VCOEFF8
+*/
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_MASK		(0x000000FF)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_LENGTH		(8)
+#define ODN_PDP_VID4VCOEFF8_VID4VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HSINIT_OFFSET			(0x061C)
+
+/* PDP, VID4HSINIT, VID4HINITIAL
+*/
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_MASK		(0xFFFF0000)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_SHIFT		(16)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_LENGTH		(16)
+#define ODN_PDP_VID4HSINIT_VID4HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4HSINIT, VID4HPITCH
+*/
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_MASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_SHIFT		(0)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_LENGTH		(16)
+#define ODN_PDP_VID4HSINIT_VID4HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF0_OFFSET			(0x0620)
+
+/* PDP, VID4HCOEFF0, VID4HCOEFF0
+*/
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF0_VID4HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF1_OFFSET			(0x0624)
+
+/* PDP, VID4HCOEFF1, VID4HCOEFF1
+*/
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF1_VID4HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF2_OFFSET			(0x0628)
+
+/* PDP, VID4HCOEFF2, VID4HCOEFF2
+*/
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF2_VID4HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF3_OFFSET			(0x062C)
+
+/* PDP, VID4HCOEFF3, VID4HCOEFF3
+*/
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF3_VID4HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF4_OFFSET			(0x0630)
+
+/* PDP, VID4HCOEFF4, VID4HCOEFF4
+*/
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF4_VID4HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF5_OFFSET			(0x0634)
+
+/* PDP, VID4HCOEFF5, VID4HCOEFF5
+*/
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF5_VID4HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF6_OFFSET			(0x0638)
+
+/* PDP, VID4HCOEFF6, VID4HCOEFF6
+*/
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF6_VID4HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF7_OFFSET			(0x063C)
+
+/* PDP, VID4HCOEFF7, VID4HCOEFF7
+*/
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF7_VID4HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF8_OFFSET			(0x0640)
+
+/* PDP, VID4HCOEFF8, VID4HCOEFF8
+*/
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF8_VID4HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF9_OFFSET			(0x0644)
+
+/* PDP, VID4HCOEFF9, VID4HCOEFF9
+*/
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_LENGTH		(32)
+#define ODN_PDP_VID4HCOEFF9_VID4HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF10_OFFSET			(0x0648)
+
+/* PDP, VID4HCOEFF10, VID4HCOEFF10
+*/
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF10_VID4HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF11_OFFSET			(0x064C)
+
+/* PDP, VID4HCOEFF11, VID4HCOEFF11
+*/
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF11_VID4HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF12_OFFSET			(0x0650)
+
+/* PDP, VID4HCOEFF12, VID4HCOEFF12
+*/
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF12_VID4HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF13_OFFSET			(0x0654)
+
+/* PDP, VID4HCOEFF13, VID4HCOEFF13
+*/
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF13_VID4HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF14_OFFSET			(0x0658)
+
+/* PDP, VID4HCOEFF14, VID4HCOEFF14
+*/
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF14_VID4HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF15_OFFSET			(0x065C)
+
+/* PDP, VID4HCOEFF15, VID4HCOEFF15
+*/
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_MASK		(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_LSBMASK	(0xFFFFFFFF)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_LENGTH	(32)
+#define ODN_PDP_VID4HCOEFF15_VID4HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4HCOEFF16_OFFSET			(0x0660)
+
+/* PDP, VID4HCOEFF16, VID4HCOEFF16
+*/
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_MASK		(0x000000FF)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_LSBMASK	(0x000000FF)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_SHIFT		(0)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_LENGTH	(8)
+#define ODN_PDP_VID4HCOEFF16_VID4HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4SCALESIZE_OFFSET			(0x0664)
+
+/* PDP, VID4SCALESIZE, VID4SCALEWIDTH
+*/
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_SHIFT		(16)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_LENGTH		(12)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALESIZE, VID4SCALEHEIGHT
+*/
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_MASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SHIFT		(0)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LENGTH		(12)
+#define ODN_PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND0_OFFSET				(0x0668)
+
+/* PDP, PORTER_BLND0, BLND0BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND0_BLND0BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND0, BLND0PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND0_BLND0PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND1_OFFSET				(0x066C)
+
+/* PDP, PORTER_BLND1, BLND1BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND1_BLND1BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND1, BLND1PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND1_BLND1PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND2_OFFSET				(0x0670)
+
+/* PDP, PORTER_BLND2, BLND2BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND2_BLND2BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND2, BLND2PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND2_BLND2PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND3_OFFSET				(0x0674)
+
+/* PDP, PORTER_BLND3, BLND3BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND3_BLND3BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND3, BLND3PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND3_BLND3PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND4_OFFSET				(0x0678)
+
+/* PDP, PORTER_BLND4, BLND4BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND4_BLND4BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND4, BLND4PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND4_BLND4PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND5_OFFSET				(0x067C)
+
+/* PDP, PORTER_BLND5, BLND5BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND5_BLND5BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND5, BLND5PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND5_BLND5PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND6_OFFSET				(0x0680)
+
+/* PDP, PORTER_BLND6, BLND6BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND6_BLND6BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND6, BLND6PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND6_BLND6PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PORTER_BLND7_OFFSET				(0x0684)
+
+/* PDP, PORTER_BLND7, BLND7BLENDTYPE
+*/
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_MASK		(0x00000010)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_LSBMASK		(0x00000001)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_SHIFT		(4)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_LENGTH		(1)
+#define ODN_PDP_PORTER_BLND7_BLND7BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND7, BLND7PORTERMODE
+*/
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_MASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_LSBMASK		(0x0000000F)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_SHIFT		(0)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_LENGTH		(4)
+#define ODN_PDP_PORTER_BLND7_BLND7PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06C8)
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06CC)
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMAX
+*/
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMIN
+*/
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_C_RG_OFFSET		(0x06D0)
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_R
+*/
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_G
+*/
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1LUMAKEY_C_B_OFFSET		(0x06D4)
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYEN
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYC_B
+*/
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06D8)
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_OFFSET				(0x06DC)
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMAX
+*/
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMIN
+*/
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_C_RG_OFFSET				(0x06E0)
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_R
+*/
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_G
+*/
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2LUMAKEY_C_B_OFFSET				(0x06E4)
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYEN
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYC_B
+*/
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06E8)
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_OFFSET			(0x06EC)
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMAX
+*/
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMIN
+*/
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_C_RG_OFFSET		(0x06F0)
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_R
+*/
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_G
+*/
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3LUMAKEY_C_B_OFFSET		(0x06F4)
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SHIFT		(29)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LENGTH		(1)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYEN
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYC_B
+*/
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06F8)
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_TRANS
+*/
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_OPAQUE
+*/
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06FC)
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMAX
+*/
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMIN
+*/
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_MASK		(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LENGTH		(10)
+#define ODN_PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_C_RG_OFFSET			(0x0700)
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_R
+*/
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_MASK		(0x0FFF0000)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_G
+*/
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_MASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4LUMAKEY_C_B_OFFSET		(0x0704)
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYALPHAMULT
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_MASK	(0x20000000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LSBMASK	(0x00000001)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SHIFT	(29)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LENGTH	(1)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SIGNED_FIELD IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYEN
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_MASK		(0x10000000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SHIFT		(28)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LENGTH		(1)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYOUTOFF
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LSBMASK	(0x000003FF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SHIFT		(16)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LENGTH	(10)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYC_B
+*/
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_MASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SHIFT		(0)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LENGTH		(12)
+#define ODN_PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF0_OFFSET			(0x0708)
+
+/* PDP, CSCCOEFF0, CSCCOEFFRU
+*/
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF0, CSCCOEFFRY
+*/
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF0_CSCCOEFFRY_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF1_OFFSET			(0x070C)
+
+/* PDP, CSCCOEFF1, CSCCOEFFGY
+*/
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFGY_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF1, CSCCOEFFRV
+*/
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF1_CSCCOEFFRV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF2_OFFSET			(0x0710)
+
+/* PDP, CSCCOEFF2, CSCCOEFFGV
+*/
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF2, CSCCOEFFGU
+*/
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF2_CSCCOEFFGU_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF3_OFFSET			(0x0714)
+
+/* PDP, CSCCOEFF3, CSCCOEFFBU
+*/
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_MASK		(0x003FF800)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_SHIFT		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF3, CSCCOEFFBY
+*/
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF3_CSCCOEFFBY_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CSCCOEFF4_OFFSET			(0x0718)
+
+/* PDP, CSCCOEFF4, CSCCOEFFBV
+*/
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_MASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_LSBMASK		(0x000007FF)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_SHIFT		(0)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_LENGTH		(11)
+#define ODN_PDP_CSCCOEFF4_CSCCOEFFBV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BGNDCOL_AR_OFFSET			(0x071C)
+
+/* PDP, BGNDCOL_AR, BGNDCOL_A
+*/
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_MASK		(0x03FF0000)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_SHIFT		(16)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_A_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_AR, BGNDCOL_R
+*/
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_MASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_SHIFT		(0)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_AR_BGNDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BGNDCOL_GB_OFFSET			(0x0720)
+
+/* PDP, BGNDCOL_GB, BGNDCOL_G
+*/
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_MASK		(0x03FF0000)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_SHIFT		(16)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_GB, BGNDCOL_B
+*/
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_MASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_SHIFT		(0)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_LENGTH		(10)
+#define ODN_PDP_BGNDCOL_GB_BGNDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BORDCOL_R_OFFSET			(0x0724)
+
+/* PDP, BORDCOL_R, BORDCOL_R
+*/
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_MASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_SHIFT		(0)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_LENGTH		(10)
+#define ODN_PDP_BORDCOL_R_BORDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BORDCOL_GB_OFFSET			(0x0728)
+
+/* PDP, BORDCOL_GB, BORDCOL_G
+*/
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_MASK		(0x03FF0000)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_SHIFT		(16)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_LENGTH		(10)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BORDCOL_GB, BORDCOL_B
+*/
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_MASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_SHIFT		(0)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_LENGTH		(10)
+#define ODN_PDP_BORDCOL_GB_BORDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_LINESTAT_OFFSET				(0x0734)
+
+/* PDP, LINESTAT, LINENO
+*/
+#define ODN_PDP_LINESTAT_LINENO_MASK			(0x00001FFF)
+#define ODN_PDP_LINESTAT_LINENO_LSBMASK			(0x00001FFF)
+#define ODN_PDP_LINESTAT_LINENO_SHIFT			(0)
+#define ODN_PDP_LINESTAT_LINENO_LENGTH			(13)
+#define ODN_PDP_LINESTAT_LINENO_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_OFFSET	(0x0738)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C11C12, CR_PROCAMP_C12
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C11C12, CR_PROCAMP_C11
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_OFFSET		(0x073C)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C13C21, CR_PROCAMP_C21
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C13C21, CR_PROCAMP_C13
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_OFFSET		(0x0740)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C22C23, CR_PROCAMP_C23
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C22C23, CR_PROCAMP_C22
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_OFFSET		(0x0744)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C31C32, CR_PROCAMP_C32
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C31C32, CR_PROCAMP_C31
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_MASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LSBMASK	(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_OFFSET		(0x0748)
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_C33
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_MASK		(0x3FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_LSBMASK		(0x00003FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_LENGTH		(14)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_C33_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_RANGE
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_MASK		(0x00000030)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LSBMASK		(0x00000003)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SHIFT		(4)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LENGTH		(2)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_C33, CR_PROCAMP_EN
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_MASK		(0x00000001)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_LENGTH		(1)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_C33_CR_PROCAMP_EN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_OFFSET		(0x074C)
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_G
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_MASK		(0x0FFF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_B
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_MASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_OFFSET		(0x0750)
+
+/* PDP, CR_ODN_PDP_PROCAMP_OUTOFFSET_R, CR_PROCAMP_OUTOFF_R
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_MASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LSBMASK		(0x00000FFF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LENGTH		(12)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_OFFSET		(0x0754)
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_G
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_MASK		(0x03FF0000)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SHIFT		(16)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_B
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_MASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_OFFSET		(0x0758)
+
+/* PDP, CR_ODN_PDP_PROCAMP_INOFFSET_R, CR_PROCAMP_INOFF_R
+*/
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_MASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SHIFT		(0)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LENGTH		(10)
+#define ODN_PDP_CR_ODN_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SIGNAT_R_OFFSET		(0x075C)
+
+/* PDP, SIGNAT_R, SIGNATURE_R
+*/
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_MASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_SHIFT		(0)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_LENGTH		(10)
+#define ODN_PDP_SIGNAT_R_SIGNATURE_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SIGNAT_GB_OFFSET		(0x0760)
+
+/* PDP, SIGNAT_GB, SIGNATURE_G
+*/
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_MASK		(0x03FF0000)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_SHIFT		(16)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_LENGTH		(10)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SIGNAT_GB, SIGNATURE_B
+*/
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_MASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_SHIFT		(0)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_LENGTH		(10)
+#define ODN_PDP_SIGNAT_GB_SIGNATURE_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGISTER_UPDATE_CTRL_OFFSET		(0x0764)
+
+/* PDP, REGISTER_UPDATE_CTRL, BYPASS_DOUBLE_BUFFERING
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_MASK		(0x00000004)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SHIFT		(2)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, REGISTERS_VALID
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_MASK		(0x00000002)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SHIFT		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, USE_VBLANK
+*/
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_MASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SHIFT		(0)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGISTER_UPDATE_STATUS_OFFSET		(0x0768)
+
+/* PDP, REGISTER_UPDATE_STATUS, REGISTERS_UPDATED
+*/
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_MASK		(0x00000002)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LSBMASK		(0x00000001)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SHIFT		(1)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LENGTH		(1)
+#define ODN_PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGCTRL_OFFSET		(0x076C)
+
+/* PDP, DBGCTRL, DBG_READ
+*/
+#define ODN_PDP_DBGCTRL_DBG_READ_MASK		(0x00000002)
+#define ODN_PDP_DBGCTRL_DBG_READ_LSBMASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_READ_SHIFT		(1)
+#define ODN_PDP_DBGCTRL_DBG_READ_LENGTH		(1)
+#define ODN_PDP_DBGCTRL_DBG_READ_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGCTRL, DBG_ENAB
+*/
+#define ODN_PDP_DBGCTRL_DBG_ENAB_MASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_LSBMASK		(0x00000001)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_SHIFT		(0)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_LENGTH		(1)
+#define ODN_PDP_DBGCTRL_DBG_ENAB_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGDATA_R_OFFSET		(0x0770)
+
+/* PDP, DBGDATA_R, DBG_DATA_R
+*/
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_MASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_SHIFT		(0)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_LENGTH		(10)
+#define ODN_PDP_DBGDATA_R_DBG_DATA_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGDATA_GB_OFFSET		(0x0774)
+
+/* PDP, DBGDATA_GB, DBG_DATA_G
+*/
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_MASK		(0x03FF0000)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_SHIFT		(16)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_LENGTH		(10)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGDATA_GB, DBG_DATA_B
+*/
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_MASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_SHIFT		(0)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_LENGTH		(10)
+#define ODN_PDP_DBGDATA_GB_DBG_DATA_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DBGSIDE_OFFSET				(0x0778)
+
+/* PDP, DBGSIDE, DBG_VAL
+*/
+#define ODN_PDP_DBGSIDE_DBG_VAL_MASK			(0x00000008)
+#define ODN_PDP_DBGSIDE_DBG_VAL_LSBMASK			(0x00000001)
+#define ODN_PDP_DBGSIDE_DBG_VAL_SHIFT			(3)
+#define ODN_PDP_DBGSIDE_DBG_VAL_LENGTH			(1)
+#define ODN_PDP_DBGSIDE_DBG_VAL_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, DBGSIDE, DBG_SIDE
+*/
+#define ODN_PDP_DBGSIDE_DBG_SIDE_MASK			(0x00000007)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_LSBMASK		(0x00000007)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_SHIFT			(0)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_LENGTH			(3)
+#define ODN_PDP_DBGSIDE_DBG_SIDE_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_OUTPUT_OFFSET				(0x077C)
+
+/* PDP, OUTPUT, EIGHT_BIT_OUTPUT
+*/
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_MASK		(0x00000002)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_LSBMASK		(0x00000001)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_SHIFT		(1)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_LENGTH		(1)
+#define ODN_PDP_OUTPUT_EIGHT_BIT_OUTPUT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OUTPUT, OUTPUT_CONFIG
+*/
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_MASK		(0x00000001)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_LSBMASK		(0x00000001)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_SHIFT		(0)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_LENGTH		(1)
+#define ODN_PDP_OUTPUT_OUTPUT_CONFIG_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_SYNCCTRL_OFFSET				(0x0780)
+
+/* PDP, SYNCCTRL, SYNCACTIVE
+*/
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_MASK		(0x80000000)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_SHIFT		(31)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_SYNCACTIVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, ODN_PDP_RST
+*/
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_MASK		(0x20000000)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_SHIFT		(29)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_ODN_PDP_RST_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, POWERDN
+*/
+#define ODN_PDP_SYNCCTRL_POWERDN_MASK			(0x10000000)
+#define ODN_PDP_SYNCCTRL_POWERDN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_POWERDN_SHIFT			(28)
+#define ODN_PDP_SYNCCTRL_POWERDN_LENGTH			(1)
+#define ODN_PDP_SYNCCTRL_POWERDN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, LOWPWRMODE
+*/
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_MASK		(0x08000000)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_SHIFT		(27)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_LOWPWRMODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDSYNCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_MASK		(0x04000000)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_SHIFT		(26)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_UPDSYNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDINTCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_MASK		(0x02000000)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_SHIFT		(25)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_UPDINTCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDCTRL
+*/
+#define ODN_PDP_SYNCCTRL_UPDCTRL_MASK			(0x01000000)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_SHIFT			(24)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_LENGTH			(1)
+#define ODN_PDP_SYNCCTRL_UPDCTRL_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDWAIT
+*/
+#define ODN_PDP_SYNCCTRL_UPDWAIT_MASK			(0x000F0000)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_LSBMASK		(0x0000000F)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_SHIFT			(16)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_LENGTH			(4)
+#define ODN_PDP_SYNCCTRL_UPDWAIT_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, FIELD_EN
+*/
+#define ODN_PDP_SYNCCTRL_FIELD_EN_MASK			(0x00002000)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_SHIFT			(13)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_FIELD_EN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, CSYNC_EN
+*/
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_MASK			(0x00001000)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_SHIFT			(12)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_CSYNC_EN_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, SYNCCTRL, CLKPOL
+*/
+#define ODN_PDP_SYNCCTRL_CLKPOL_MASK		(0x00000800)
+#define ODN_PDP_SYNCCTRL_CLKPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_CLKPOL_SHIFT		(11)
+#define ODN_PDP_SYNCCTRL_CLKPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_CLKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VS_SLAVE
+*/
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_MASK		(0x00000080)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_SHIFT		(7)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HS_SLAVE
+*/
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_MASK		(0x00000040)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_SHIFT		(6)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKPOL
+*/
+#define ODN_PDP_SYNCCTRL_BLNKPOL_MASK		(0x00000020)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_SHIFT		(5)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_BLNKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKDIS
+*/
+#define ODN_PDP_SYNCCTRL_BLNKDIS_MASK		(0x00000010)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_SHIFT		(4)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_BLNKDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSPOL
+*/
+#define ODN_PDP_SYNCCTRL_VSPOL_MASK		(0x00000008)
+#define ODN_PDP_SYNCCTRL_VSPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VSPOL_SHIFT		(3)
+#define ODN_PDP_SYNCCTRL_VSPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSDIS
+*/
+#define ODN_PDP_SYNCCTRL_VSDIS_MASK		(0x00000004)
+#define ODN_PDP_SYNCCTRL_VSDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_VSDIS_SHIFT		(2)
+#define ODN_PDP_SYNCCTRL_VSDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_VSDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSPOL
+*/
+#define ODN_PDP_SYNCCTRL_HSPOL_MASK		(0x00000002)
+#define ODN_PDP_SYNCCTRL_HSPOL_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSPOL_SHIFT		(1)
+#define ODN_PDP_SYNCCTRL_HSPOL_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSDIS
+*/
+#define ODN_PDP_SYNCCTRL_HSDIS_MASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSDIS_LSBMASK		(0x00000001)
+#define ODN_PDP_SYNCCTRL_HSDIS_SHIFT		(0)
+#define ODN_PDP_SYNCCTRL_HSDIS_LENGTH		(1)
+#define ODN_PDP_SYNCCTRL_HSDIS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_HSYNC1_OFFSET			(0x0784)
+
+/* PDP, HSYNC1, HBPS
+*/
+#define ODN_PDP_HSYNC1_HBPS_MASK		(0x1FFF0000)
+#define ODN_PDP_HSYNC1_HBPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC1_HBPS_SHIFT		(16)
+#define ODN_PDP_HSYNC1_HBPS_LENGTH		(13)
+#define ODN_PDP_HSYNC1_HBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC1, HT
+*/
+#define ODN_PDP_HSYNC1_HT_MASK			(0x00001FFF)
+#define ODN_PDP_HSYNC1_HT_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC1_HT_SHIFT			(0)
+#define ODN_PDP_HSYNC1_HT_LENGTH		(13)
+#define ODN_PDP_HSYNC1_HT_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_HSYNC2_OFFSET			(0x0788)
+
+/* PDP, HSYNC2, HAS
+*/
+#define ODN_PDP_HSYNC2_HAS_MASK			(0x1FFF0000)
+#define ODN_PDP_HSYNC2_HAS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HAS_SHIFT		(16)
+#define ODN_PDP_HSYNC2_HAS_LENGTH		(13)
+#define ODN_PDP_HSYNC2_HAS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC2, HLBS
+*/
+#define ODN_PDP_HSYNC2_HLBS_MASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HLBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC2_HLBS_SHIFT		(0)
+#define ODN_PDP_HSYNC2_HLBS_LENGTH		(13)
+#define ODN_PDP_HSYNC2_HLBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_HSYNC3_OFFSET			(0x078C)
+
+/* PDP, HSYNC3, HFPS
+*/
+#define ODN_PDP_HSYNC3_HFPS_MASK		(0x1FFF0000)
+#define ODN_PDP_HSYNC3_HFPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HFPS_SHIFT		(16)
+#define ODN_PDP_HSYNC3_HFPS_LENGTH		(13)
+#define ODN_PDP_HSYNC3_HFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC3, HRBS
+*/
+#define ODN_PDP_HSYNC3_HRBS_MASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HRBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HSYNC3_HRBS_SHIFT		(0)
+#define ODN_PDP_HSYNC3_HRBS_LENGTH		(13)
+#define ODN_PDP_HSYNC3_HRBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VSYNC1_OFFSET			(0x0790)
+
+/* PDP, VSYNC1, VBPS
+*/
+#define ODN_PDP_VSYNC1_VBPS_MASK		(0x1FFF0000)
+#define ODN_PDP_VSYNC1_VBPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC1_VBPS_SHIFT		(16)
+#define ODN_PDP_VSYNC1_VBPS_LENGTH		(13)
+#define ODN_PDP_VSYNC1_VBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC1, VT
+*/
+#define ODN_PDP_VSYNC1_VT_MASK			(0x00001FFF)
+#define ODN_PDP_VSYNC1_VT_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC1_VT_SHIFT			(0)
+#define ODN_PDP_VSYNC1_VT_LENGTH		(13)
+#define ODN_PDP_VSYNC1_VT_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_VSYNC2_OFFSET			(0x0794)
+
+/* PDP, VSYNC2, VAS
+*/
+#define ODN_PDP_VSYNC2_VAS_MASK			(0x1FFF0000)
+#define ODN_PDP_VSYNC2_VAS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VAS_SHIFT		(16)
+#define ODN_PDP_VSYNC2_VAS_LENGTH		(13)
+#define ODN_PDP_VSYNC2_VAS_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VSYNC2, VTBS
+*/
+#define ODN_PDP_VSYNC2_VTBS_MASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VTBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC2_VTBS_SHIFT		(0)
+#define ODN_PDP_VSYNC2_VTBS_LENGTH		(13)
+#define ODN_PDP_VSYNC2_VTBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VSYNC3_OFFSET			(0x0798)
+
+/* PDP, VSYNC3, VFPS
+*/
+#define ODN_PDP_VSYNC3_VFPS_MASK		(0x1FFF0000)
+#define ODN_PDP_VSYNC3_VFPS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VFPS_SHIFT		(16)
+#define ODN_PDP_VSYNC3_VFPS_LENGTH		(13)
+#define ODN_PDP_VSYNC3_VFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC3, VBBS
+*/
+#define ODN_PDP_VSYNC3_VBBS_MASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VBBS_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VSYNC3_VBBS_SHIFT		(0)
+#define ODN_PDP_VSYNC3_VBBS_LENGTH		(13)
+#define ODN_PDP_VSYNC3_VBBS_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTSTAT_OFFSET			(0x079C)
+
+/* PDP, INTSTAT, INTS_VID4ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1ORUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID4URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1URUN
+*/
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK1
+*/
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK0
+*/
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK1
+*/
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK0
+*/
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTSTAT_INTS_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTENAB_OFFSET				(0x07A0)
+
+/* PDP, INTENAB, INTEN_VID4ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1ORUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID4URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1URUN
+*/
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK1
+*/
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK0
+*/
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK1
+*/
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK0
+*/
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTENAB_INTEN_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTCLR_OFFSET		(0x07A4)
+
+/* PDP, INTCLR, INTCLR_VID4ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_MASK		(0x00080000)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_SHIFT		(19)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_MASK		(0x00040000)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_SHIFT		(18)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_MASK		(0x00020000)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_SHIFT		(17)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_MASK		(0x00010000)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_SHIFT		(16)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_MASK		(0x00008000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_SHIFT		(15)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_MASK		(0x00004000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_SHIFT		(14)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_MASK		(0x00002000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_SHIFT		(13)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1ORUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_MASK		(0x00001000)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_SHIFT		(12)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID4URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_MASK		(0x00000800)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_SHIFT		(11)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_MASK		(0x00000400)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_SHIFT		(10)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_MASK		(0x00000200)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_SHIFT		(9)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_MASK		(0x00000100)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_SHIFT		(8)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_MASK		(0x00000080)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_SHIFT		(7)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_MASK		(0x00000040)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_SHIFT		(6)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_MASK		(0x00000020)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_SHIFT		(5)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1URUN
+*/
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_MASK		(0x00000010)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_SHIFT		(4)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK1
+*/
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_MASK		(0x00000008)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_SHIFT		(3)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK0
+*/
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_MASK		(0x00000004)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_SHIFT		(2)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK1
+*/
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_MASK		(0x00000002)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_SHIFT		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK0
+*/
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_MASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_LSBMASK		(0x00000001)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_SHIFT		(0)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_LENGTH		(1)
+#define ODN_PDP_INTCLR_INTCLR_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_MEMCTRL_OFFSET		(0x07A8)
+
+/* PDP, MEMCTRL, MEMREFRESH
+*/
+#define ODN_PDP_MEMCTRL_MEMREFRESH_MASK		(0xC0000000)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_LSBMASK		(0x00000003)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_SHIFT		(30)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_LENGTH		(2)
+#define ODN_PDP_MEMCTRL_MEMREFRESH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEMCTRL, BURSTLEN
+*/
+#define ODN_PDP_MEMCTRL_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_MEMCTRL_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_MEMCTRL_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_MEMCTRL_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_MEMCTRL_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_MEM_THRESH_OFFSET		(0x07AC)
+
+/* PDP, MEM_THRESH, UVTHRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_MEM_THRESH_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, YTHRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_MEM_THRESH_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, THRESHOLD
+*/
+#define ODN_PDP_MEM_THRESH_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_MEM_THRESH_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_ALTERNATE_3D_CTRL_OFFSET		(0x07B0)
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_ON
+*/
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_MASK		(0x00000010)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LSBMASK		(0x00000001)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SHIFT		(4)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LENGTH		(1)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_BLENDSEL
+*/
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_MASK		(0x00000007)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LSBMASK		(0x00000007)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SHIFT		(0)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LENGTH		(3)
+#define ODN_PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA0_R_OFFSET		(0x07B4)
+
+/* PDP, GAMMA0_R, GAMMA0_R
+*/
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_SHIFT		(0)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_LENGTH		(10)
+#define ODN_PDP_GAMMA0_R_GAMMA0_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA0_GB_OFFSET		(0x07B8)
+
+/* PDP, GAMMA0_GB, GAMMA0_G
+*/
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_SHIFT		(16)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_LENGTH		(10)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA0_GB, GAMMA0_B
+*/
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_SHIFT		(0)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_LENGTH		(10)
+#define ODN_PDP_GAMMA0_GB_GAMMA0_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA1_R_OFFSET		(0x07BC)
+
+/* PDP, GAMMA1_R, GAMMA1_R
+*/
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_SHIFT		(0)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_LENGTH		(10)
+#define ODN_PDP_GAMMA1_R_GAMMA1_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA1_GB_OFFSET		(0x07C0)
+
+/* PDP, GAMMA1_GB, GAMMA1_G
+*/
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_SHIFT		(16)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_LENGTH		(10)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA1_GB, GAMMA1_B
+*/
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_SHIFT		(0)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_LENGTH		(10)
+#define ODN_PDP_GAMMA1_GB_GAMMA1_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA2_R_OFFSET		(0x07C4)
+
+/* PDP, GAMMA2_R, GAMMA2_R
+*/
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_SHIFT		(0)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_LENGTH		(10)
+#define ODN_PDP_GAMMA2_R_GAMMA2_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA2_GB_OFFSET		(0x07C8)
+
+/* PDP, GAMMA2_GB, GAMMA2_G
+*/
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_SHIFT		(16)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_LENGTH		(10)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA2_GB, GAMMA2_B
+*/
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_SHIFT		(0)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_LENGTH		(10)
+#define ODN_PDP_GAMMA2_GB_GAMMA2_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA3_R_OFFSET		(0x07CC)
+
+/* PDP, GAMMA3_R, GAMMA3_R
+*/
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_SHIFT		(0)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_LENGTH		(10)
+#define ODN_PDP_GAMMA3_R_GAMMA3_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA3_GB_OFFSET		(0x07D0)
+
+/* PDP, GAMMA3_GB, GAMMA3_G
+*/
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_SHIFT		(16)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_LENGTH		(10)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA3_GB, GAMMA3_B
+*/
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_SHIFT		(0)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_LENGTH		(10)
+#define ODN_PDP_GAMMA3_GB_GAMMA3_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA4_R_OFFSET		(0x07D4)
+
+/* PDP, GAMMA4_R, GAMMA4_R
+*/
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_SHIFT		(0)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_LENGTH		(10)
+#define ODN_PDP_GAMMA4_R_GAMMA4_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA4_GB_OFFSET		(0x07D8)
+
+/* PDP, GAMMA4_GB, GAMMA4_G
+*/
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_SHIFT		(16)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_LENGTH		(10)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA4_GB, GAMMA4_B
+*/
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_SHIFT		(0)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_LENGTH		(10)
+#define ODN_PDP_GAMMA4_GB_GAMMA4_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA5_R_OFFSET		(0x07DC)
+
+/* PDP, GAMMA5_R, GAMMA5_R
+*/
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_SHIFT		(0)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_LENGTH		(10)
+#define ODN_PDP_GAMMA5_R_GAMMA5_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA5_GB_OFFSET		(0x07E0)
+
+/* PDP, GAMMA5_GB, GAMMA5_G
+*/
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_SHIFT		(16)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_LENGTH		(10)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA5_GB, GAMMA5_B
+*/
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_SHIFT		(0)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_LENGTH		(10)
+#define ODN_PDP_GAMMA5_GB_GAMMA5_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA6_R_OFFSET		(0x07E4)
+
+/* PDP, GAMMA6_R, GAMMA6_R
+*/
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_SHIFT		(0)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_LENGTH		(10)
+#define ODN_PDP_GAMMA6_R_GAMMA6_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA6_GB_OFFSET		(0x07E8)
+
+/* PDP, GAMMA6_GB, GAMMA6_G
+*/
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_SHIFT		(16)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_LENGTH		(10)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA6_GB, GAMMA6_B
+*/
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_SHIFT		(0)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_LENGTH		(10)
+#define ODN_PDP_GAMMA6_GB_GAMMA6_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA7_R_OFFSET		(0x07EC)
+
+/* PDP, GAMMA7_R, GAMMA7_R
+*/
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_SHIFT		(0)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_LENGTH		(10)
+#define ODN_PDP_GAMMA7_R_GAMMA7_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA7_GB_OFFSET		(0x07F0)
+
+/* PDP, GAMMA7_GB, GAMMA7_G
+*/
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_SHIFT		(16)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_LENGTH		(10)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA7_GB, GAMMA7_B
+*/
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_SHIFT		(0)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_LENGTH		(10)
+#define ODN_PDP_GAMMA7_GB_GAMMA7_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA8_R_OFFSET		(0x07F4)
+
+/* PDP, GAMMA8_R, GAMMA8_R
+*/
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_SHIFT		(0)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_LENGTH		(10)
+#define ODN_PDP_GAMMA8_R_GAMMA8_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA8_GB_OFFSET		(0x07F8)
+
+/* PDP, GAMMA8_GB, GAMMA8_G
+*/
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_SHIFT		(16)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_LENGTH		(10)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA8_GB, GAMMA8_B
+*/
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_SHIFT		(0)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_LENGTH		(10)
+#define ODN_PDP_GAMMA8_GB_GAMMA8_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA9_R_OFFSET		(0x07FC)
+
+/* PDP, GAMMA9_R, GAMMA9_R
+*/
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_SHIFT		(0)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_LENGTH		(10)
+#define ODN_PDP_GAMMA9_R_GAMMA9_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA9_GB_OFFSET		(0x0800)
+
+/* PDP, GAMMA9_GB, GAMMA9_G
+*/
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_SHIFT		(16)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_LENGTH		(10)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA9_GB, GAMMA9_B
+*/
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_SHIFT		(0)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_LENGTH		(10)
+#define ODN_PDP_GAMMA9_GB_GAMMA9_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA10_R_OFFSET		(0x0804)
+
+/* PDP, GAMMA10_R, GAMMA10_R
+*/
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_SHIFT		(0)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_LENGTH		(10)
+#define ODN_PDP_GAMMA10_R_GAMMA10_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA10_GB_OFFSET		(0x0808)
+
+/* PDP, GAMMA10_GB, GAMMA10_G
+*/
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_SHIFT		(16)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_LENGTH		(10)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA10_GB, GAMMA10_B
+*/
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_SHIFT		(0)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_LENGTH		(10)
+#define ODN_PDP_GAMMA10_GB_GAMMA10_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA11_R_OFFSET		(0x080C)
+
+/* PDP, GAMMA11_R, GAMMA11_R
+*/
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_SHIFT		(0)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_LENGTH		(10)
+#define ODN_PDP_GAMMA11_R_GAMMA11_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA11_GB_OFFSET		(0x0810)
+
+/* PDP, GAMMA11_GB, GAMMA11_G
+*/
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_SHIFT		(16)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_LENGTH		(10)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA11_GB, GAMMA11_B
+*/
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_SHIFT		(0)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_LENGTH		(10)
+#define ODN_PDP_GAMMA11_GB_GAMMA11_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA12_R_OFFSET		(0x0814)
+
+/* PDP, GAMMA12_R, GAMMA12_R
+*/
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_SHIFT		(0)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_LENGTH		(10)
+#define ODN_PDP_GAMMA12_R_GAMMA12_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA12_GB_OFFSET		(0x0818)
+
+/* PDP, GAMMA12_GB, GAMMA12_G
+*/
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_SHIFT		(16)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_LENGTH		(10)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA12_GB, GAMMA12_B
+*/
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_SHIFT		(0)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_LENGTH		(10)
+#define ODN_PDP_GAMMA12_GB_GAMMA12_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA13_R_OFFSET		(0x081C)
+
+/* PDP, GAMMA13_R, GAMMA13_R
+*/
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_SHIFT		(0)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_LENGTH		(10)
+#define ODN_PDP_GAMMA13_R_GAMMA13_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA13_GB_OFFSET		(0x0820)
+
+/* PDP, GAMMA13_GB, GAMMA13_G
+*/
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_SHIFT		(16)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_LENGTH		(10)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA13_GB, GAMMA13_B
+*/
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_SHIFT		(0)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_LENGTH		(10)
+#define ODN_PDP_GAMMA13_GB_GAMMA13_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA14_R_OFFSET		(0x0824)
+
+/* PDP, GAMMA14_R, GAMMA14_R
+*/
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_SHIFT		(0)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_LENGTH		(10)
+#define ODN_PDP_GAMMA14_R_GAMMA14_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA14_GB_OFFSET		(0x0828)
+
+/* PDP, GAMMA14_GB, GAMMA14_G
+*/
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_SHIFT		(16)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_LENGTH		(10)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA14_GB, GAMMA14_B
+*/
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_SHIFT		(0)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_LENGTH		(10)
+#define ODN_PDP_GAMMA14_GB_GAMMA14_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA15_R_OFFSET		(0x082C)
+
+/* PDP, GAMMA15_R, GAMMA15_R
+*/
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_SHIFT		(0)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_LENGTH		(10)
+#define ODN_PDP_GAMMA15_R_GAMMA15_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA15_GB_OFFSET		(0x0830)
+
+/* PDP, GAMMA15_GB, GAMMA15_G
+*/
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_SHIFT		(16)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_LENGTH		(10)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA15_GB, GAMMA15_B
+*/
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_SHIFT		(0)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_LENGTH		(10)
+#define ODN_PDP_GAMMA15_GB_GAMMA15_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA16_R_OFFSET		(0x0834)
+
+/* PDP, GAMMA16_R, GAMMA16_R
+*/
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_SHIFT		(0)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_LENGTH		(10)
+#define ODN_PDP_GAMMA16_R_GAMMA16_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA16_GB_OFFSET		(0x0838)
+
+/* PDP, GAMMA16_GB, GAMMA16_G
+*/
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_SHIFT		(16)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_LENGTH		(10)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA16_GB, GAMMA16_B
+*/
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_SHIFT		(0)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_LENGTH		(10)
+#define ODN_PDP_GAMMA16_GB_GAMMA16_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA17_R_OFFSET		(0x083C)
+
+/* PDP, GAMMA17_R, GAMMA17_R
+*/
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_SHIFT		(0)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_LENGTH		(10)
+#define ODN_PDP_GAMMA17_R_GAMMA17_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA17_GB_OFFSET		(0x0840)
+
+/* PDP, GAMMA17_GB, GAMMA17_G
+*/
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_SHIFT		(16)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_LENGTH		(10)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA17_GB, GAMMA17_B
+*/
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_SHIFT		(0)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_LENGTH		(10)
+#define ODN_PDP_GAMMA17_GB_GAMMA17_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA18_R_OFFSET		(0x0844)
+
+/* PDP, GAMMA18_R, GAMMA18_R
+*/
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_SHIFT		(0)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_LENGTH		(10)
+#define ODN_PDP_GAMMA18_R_GAMMA18_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA18_GB_OFFSET		(0x0848)
+
+/* PDP, GAMMA18_GB, GAMMA18_G
+*/
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_SHIFT		(16)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_LENGTH		(10)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA18_GB, GAMMA18_B
+*/
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_SHIFT		(0)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_LENGTH		(10)
+#define ODN_PDP_GAMMA18_GB_GAMMA18_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA19_R_OFFSET		(0x084C)
+
+/* PDP, GAMMA19_R, GAMMA19_R
+*/
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_SHIFT		(0)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_LENGTH		(10)
+#define ODN_PDP_GAMMA19_R_GAMMA19_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA19_GB_OFFSET		(0x0850)
+
+/* PDP, GAMMA19_GB, GAMMA19_G
+*/
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_SHIFT		(16)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_LENGTH		(10)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA19_GB, GAMMA19_B
+*/
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_SHIFT		(0)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_LENGTH		(10)
+#define ODN_PDP_GAMMA19_GB_GAMMA19_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA20_R_OFFSET		(0x0854)
+
+/* PDP, GAMMA20_R, GAMMA20_R
+*/
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_SHIFT		(0)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_LENGTH		(10)
+#define ODN_PDP_GAMMA20_R_GAMMA20_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA20_GB_OFFSET		(0x0858)
+
+/* PDP, GAMMA20_GB, GAMMA20_G
+*/
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_SHIFT		(16)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_LENGTH		(10)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA20_GB, GAMMA20_B
+*/
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_SHIFT		(0)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_LENGTH		(10)
+#define ODN_PDP_GAMMA20_GB_GAMMA20_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA21_R_OFFSET		(0x085C)
+
+/* PDP, GAMMA21_R, GAMMA21_R
+*/
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_SHIFT		(0)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_LENGTH		(10)
+#define ODN_PDP_GAMMA21_R_GAMMA21_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA21_GB_OFFSET		(0x0860)
+
+/* PDP, GAMMA21_GB, GAMMA21_G
+*/
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_SHIFT		(16)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_LENGTH		(10)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA21_GB, GAMMA21_B
+*/
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_SHIFT		(0)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_LENGTH		(10)
+#define ODN_PDP_GAMMA21_GB_GAMMA21_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA22_R_OFFSET		(0x0864)
+
+/* PDP, GAMMA22_R, GAMMA22_R
+*/
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_SHIFT		(0)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_LENGTH		(10)
+#define ODN_PDP_GAMMA22_R_GAMMA22_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA22_GB_OFFSET		(0x0868)
+
+/* PDP, GAMMA22_GB, GAMMA22_G
+*/
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_SHIFT		(16)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_LENGTH		(10)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA22_GB, GAMMA22_B
+*/
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_SHIFT		(0)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_LENGTH		(10)
+#define ODN_PDP_GAMMA22_GB_GAMMA22_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA23_R_OFFSET		(0x086C)
+
+/* PDP, GAMMA23_R, GAMMA23_R
+*/
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_SHIFT		(0)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_LENGTH		(10)
+#define ODN_PDP_GAMMA23_R_GAMMA23_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA23_GB_OFFSET		(0x0870)
+
+/* PDP, GAMMA23_GB, GAMMA23_G
+*/
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_SHIFT		(16)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_LENGTH		(10)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA23_GB, GAMMA23_B
+*/
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_SHIFT		(0)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_LENGTH		(10)
+#define ODN_PDP_GAMMA23_GB_GAMMA23_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA24_R_OFFSET		(0x0874)
+
+/* PDP, GAMMA24_R, GAMMA24_R
+*/
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_SHIFT		(0)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_LENGTH		(10)
+#define ODN_PDP_GAMMA24_R_GAMMA24_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA24_GB_OFFSET		(0x0878)
+
+/* PDP, GAMMA24_GB, GAMMA24_G
+*/
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_SHIFT		(16)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_LENGTH		(10)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA24_GB, GAMMA24_B
+*/
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_SHIFT		(0)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_LENGTH		(10)
+#define ODN_PDP_GAMMA24_GB_GAMMA24_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA25_R_OFFSET		(0x087C)
+
+/* PDP, GAMMA25_R, GAMMA25_R
+*/
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_SHIFT		(0)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_LENGTH		(10)
+#define ODN_PDP_GAMMA25_R_GAMMA25_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA25_GB_OFFSET		(0x0880)
+
+/* PDP, GAMMA25_GB, GAMMA25_G
+*/
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_SHIFT		(16)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_LENGTH		(10)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA25_GB, GAMMA25_B
+*/
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_SHIFT		(0)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_LENGTH		(10)
+#define ODN_PDP_GAMMA25_GB_GAMMA25_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA26_R_OFFSET		(0x0884)
+
+/* PDP, GAMMA26_R, GAMMA26_R
+*/
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_SHIFT		(0)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_LENGTH		(10)
+#define ODN_PDP_GAMMA26_R_GAMMA26_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA26_GB_OFFSET		(0x0888)
+
+/* PDP, GAMMA26_GB, GAMMA26_G
+*/
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_SHIFT		(16)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_LENGTH		(10)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA26_GB, GAMMA26_B
+*/
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_SHIFT		(0)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_LENGTH		(10)
+#define ODN_PDP_GAMMA26_GB_GAMMA26_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA27_R_OFFSET		(0x088C)
+
+/* PDP, GAMMA27_R, GAMMA27_R
+*/
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_SHIFT		(0)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_LENGTH		(10)
+#define ODN_PDP_GAMMA27_R_GAMMA27_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA27_GB_OFFSET		(0x0890)
+
+/* PDP, GAMMA27_GB, GAMMA27_G
+*/
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_SHIFT		(16)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_LENGTH		(10)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA27_GB, GAMMA27_B
+*/
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_SHIFT		(0)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_LENGTH		(10)
+#define ODN_PDP_GAMMA27_GB_GAMMA27_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA28_R_OFFSET		(0x0894)
+
+/* PDP, GAMMA28_R, GAMMA28_R
+*/
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_SHIFT		(0)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_LENGTH		(10)
+#define ODN_PDP_GAMMA28_R_GAMMA28_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA28_GB_OFFSET		(0x0898)
+
+/* PDP, GAMMA28_GB, GAMMA28_G
+*/
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_SHIFT		(16)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_LENGTH		(10)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA28_GB, GAMMA28_B
+*/
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_SHIFT		(0)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_LENGTH		(10)
+#define ODN_PDP_GAMMA28_GB_GAMMA28_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA29_R_OFFSET		(0x089C)
+
+/* PDP, GAMMA29_R, GAMMA29_R
+*/
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_SHIFT		(0)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_LENGTH		(10)
+#define ODN_PDP_GAMMA29_R_GAMMA29_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA29_GB_OFFSET		(0x08A0)
+
+/* PDP, GAMMA29_GB, GAMMA29_G
+*/
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_SHIFT		(16)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_LENGTH		(10)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA29_GB, GAMMA29_B
+*/
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_SHIFT		(0)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_LENGTH		(10)
+#define ODN_PDP_GAMMA29_GB_GAMMA29_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA30_R_OFFSET		(0x08A4)
+
+/* PDP, GAMMA30_R, GAMMA30_R
+*/
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_SHIFT		(0)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_LENGTH		(10)
+#define ODN_PDP_GAMMA30_R_GAMMA30_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA30_GB_OFFSET		(0x08A8)
+
+/* PDP, GAMMA30_GB, GAMMA30_G
+*/
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_SHIFT		(16)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_LENGTH		(10)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA30_GB, GAMMA30_B
+*/
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_SHIFT		(0)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_LENGTH		(10)
+#define ODN_PDP_GAMMA30_GB_GAMMA30_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA31_R_OFFSET		(0x08AC)
+
+/* PDP, GAMMA31_R, GAMMA31_R
+*/
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_SHIFT		(0)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_LENGTH		(10)
+#define ODN_PDP_GAMMA31_R_GAMMA31_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA31_GB_OFFSET		(0x08B0)
+
+/* PDP, GAMMA31_GB, GAMMA31_G
+*/
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_SHIFT		(16)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_LENGTH		(10)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA31_GB, GAMMA31_B
+*/
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_SHIFT		(0)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_LENGTH		(10)
+#define ODN_PDP_GAMMA31_GB_GAMMA31_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA32_R_OFFSET			(0x08B4)
+
+/* PDP, GAMMA32_R, GAMMA32_R
+*/
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_SHIFT		(0)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_LENGTH		(10)
+#define ODN_PDP_GAMMA32_R_GAMMA32_R_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GAMMA32_GB_OFFSET			(0x08B8)
+
+/* PDP, GAMMA32_GB, GAMMA32_G
+*/
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_MASK		(0x03FF0000)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_SHIFT		(16)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_LENGTH		(10)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA32_GB, GAMMA32_B
+*/
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_MASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_LSBMASK		(0x000003FF)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_SHIFT		(0)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_LENGTH		(10)
+#define ODN_PDP_GAMMA32_GB_GAMMA32_B_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VEVENT_OFFSET				(0x08BC)
+
+/* PDP, VEVENT, VEVENT
+*/
+#define ODN_PDP_VEVENT_VEVENT_MASK			(0x1FFF0000)
+#define ODN_PDP_VEVENT_VEVENT_LSBMASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VEVENT_SHIFT			(16)
+#define ODN_PDP_VEVENT_VEVENT_LENGTH			(13)
+#define ODN_PDP_VEVENT_VEVENT_SIGNED_FIELD		IMG_FALSE
+
+/* PDP, VEVENT, VFETCH
+*/
+#define ODN_PDP_VEVENT_VFETCH_MASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VFETCH_LSBMASK			(0x00001FFF)
+#define ODN_PDP_VEVENT_VFETCH_SHIFT			(0)
+#define ODN_PDP_VEVENT_VFETCH_LENGTH			(13)
+#define ODN_PDP_VEVENT_VFETCH_SIGNED_FIELD		IMG_FALSE
+
+#define ODN_PDP_HDECTRL_OFFSET				(0x08C0)
+
+/* PDP, HDECTRL, HDES
+*/
+#define ODN_PDP_HDECTRL_HDES_MASK		(0x1FFF0000)
+#define ODN_PDP_HDECTRL_HDES_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDES_SHIFT		(16)
+#define ODN_PDP_HDECTRL_HDES_LENGTH		(13)
+#define ODN_PDP_HDECTRL_HDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HDECTRL, HDEF
+*/
+#define ODN_PDP_HDECTRL_HDEF_MASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDEF_LSBMASK		(0x00001FFF)
+#define ODN_PDP_HDECTRL_HDEF_SHIFT		(0)
+#define ODN_PDP_HDECTRL_HDEF_LENGTH		(13)
+#define ODN_PDP_HDECTRL_HDEF_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VDECTRL_OFFSET			(0x08C4)
+
+/* PDP, VDECTRL, VDES
+*/
+#define ODN_PDP_VDECTRL_VDES_MASK		(0x1FFF0000)
+#define ODN_PDP_VDECTRL_VDES_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDES_SHIFT		(16)
+#define ODN_PDP_VDECTRL_VDES_LENGTH		(13)
+#define ODN_PDP_VDECTRL_VDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VDECTRL, VDEF
+*/
+#define ODN_PDP_VDECTRL_VDEF_MASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDEF_LSBMASK		(0x00001FFF)
+#define ODN_PDP_VDECTRL_VDEF_SHIFT		(0)
+#define ODN_PDP_VDECTRL_VDEF_LENGTH		(13)
+#define ODN_PDP_VDECTRL_VDEF_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_OPMASK_R_OFFSET			(0x08C8)
+
+/* PDP, OPMASK_R, MASKLEVEL
+*/
+#define ODN_PDP_OPMASK_R_MASKLEVEL_MASK		(0x80000000)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_LSBMASK		(0x00000001)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_SHIFT		(31)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_LENGTH		(1)
+#define ODN_PDP_OPMASK_R_MASKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, BLANKLEVEL
+*/
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_MASK		(0x40000000)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_LSBMASK		(0x00000001)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_SHIFT		(30)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_LENGTH		(1)
+#define ODN_PDP_OPMASK_R_BLANKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, MASKR
+*/
+#define ODN_PDP_OPMASK_R_MASKR_MASK		(0x000003FF)
+#define ODN_PDP_OPMASK_R_MASKR_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_R_MASKR_SHIFT		(0)
+#define ODN_PDP_OPMASK_R_MASKR_LENGTH		(10)
+#define ODN_PDP_OPMASK_R_MASKR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_OPMASK_GB_OFFSET		(0x08CC)
+
+/* PDP, OPMASK_GB, MASKG
+*/
+#define ODN_PDP_OPMASK_GB_MASKG_MASK		(0x03FF0000)
+#define ODN_PDP_OPMASK_GB_MASKG_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKG_SHIFT		(16)
+#define ODN_PDP_OPMASK_GB_MASKG_LENGTH		(10)
+#define ODN_PDP_OPMASK_GB_MASKG_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_GB, MASKB
+*/
+#define ODN_PDP_OPMASK_GB_MASKB_MASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKB_LSBMASK		(0x000003FF)
+#define ODN_PDP_OPMASK_GB_MASKB_SHIFT		(0)
+#define ODN_PDP_OPMASK_GB_MASKB_LENGTH		(10)
+#define ODN_PDP_OPMASK_GB_MASKB_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_ADDR_CTRL_OFFSET		(0x08D0)
+
+/* PDP, REGLD_ADDR_CTRL, REGLD_ADDRIN
+*/
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_MASK		(0xFFFFFFF0)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LSBMASK		(0x0FFFFFFF)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SHIFT		(4)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LENGTH		(28)
+#define ODN_PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_ADDR_STAT_OFFSET		(0x08D4)
+
+/* PDP, REGLD_ADDR_STAT, REGLD_ADDROUT
+*/
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_MASK		(0xFFFFFFF0)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LSBMASK		(0x0FFFFFFF)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SHIFT		(4)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LENGTH		(28)
+#define ODN_PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_STAT_OFFSET		(0x08D8)
+
+/* PDP, REGLD_STAT, REGLD_ADDREN
+*/
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_MASK		(0x00800000)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_LSBMASK		(0x00000001)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_SHIFT		(23)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_LENGTH		(1)
+#define ODN_PDP_REGLD_STAT_REGLD_ADDREN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_REGLD_CTRL_OFFSET		(0x08DC)
+
+/* PDP, REGLD_CTRL, REGLD_ADDRLEN
+*/
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_MASK		(0xFF000000)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_LSBMASK	(0x000000FF)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_SHIFT		(24)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_LENGTH		(8)
+#define ODN_PDP_REGLD_CTRL_REGLD_ADDRLEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGLD_CTRL, REGLD_VAL
+*/
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_MASK		(0x00800000)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_LSBMASK		(0x00000001)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_SHIFT		(23)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_LENGTH		(1)
+#define ODN_PDP_REGLD_CTRL_REGLD_VAL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_UPDCTRL_OFFSET			(0x08E0)
+
+/* PDP, UPDCTRL, UPDFIELD
+*/
+#define ODN_PDP_UPDCTRL_UPDFIELD_MASK		(0x00000001)
+#define ODN_PDP_UPDCTRL_UPDFIELD_LSBMASK	(0x00000001)
+#define ODN_PDP_UPDCTRL_UPDFIELD_SHIFT		(0)
+#define ODN_PDP_UPDCTRL_UPDFIELD_LENGTH		(1)
+#define ODN_PDP_UPDCTRL_UPDFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_INTCTRL_OFFSET			(0x08E4)
+
+/* PDP, PVR_ODN_PDP_INTCTRL, HBLNK_LINE
+*/
+#define ODN_PDP_INTCTRL_HBLNK_LINE_MASK		(0x00010000)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_LSBMASK	(0x00000001)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_SHIFT	(16)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_LENGTH	(1)
+#define ODN_PDP_INTCTRL_HBLNK_LINE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_ODN_PDP_INTCTRL, HBLNK_LINENO
+*/
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_MASK	(0x00001FFF)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_LSBMASK	(0x00001FFF)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_SHIFT	(0)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_LENGTH	(13)
+#define ODN_PDP_INTCTRL_HBLNK_LINENO_SIGNED_FIELD IMG_FALSE
+
+#define ODN_PDP_PDISETUP_OFFSET		(0x0900)
+
+/* PDP, PDISETUP, PDI_BLNKLVL
+*/
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_MASK		(0x00000040)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_SHIFT		(6)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_BLNKLVL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_BLNK
+*/
+#define ODN_PDP_PDISETUP_PDI_BLNK_MASK		(0x00000020)
+#define ODN_PDP_PDISETUP_PDI_BLNK_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_BLNK_SHIFT		(5)
+#define ODN_PDP_PDISETUP_PDI_BLNK_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_BLNK_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_PWR
+*/
+#define ODN_PDP_PDISETUP_PDI_PWR_MASK		(0x00000010)
+#define ODN_PDP_PDISETUP_PDI_PWR_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_PWR_SHIFT		(4)
+#define ODN_PDP_PDISETUP_PDI_PWR_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_PWR_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_EN
+*/
+#define ODN_PDP_PDISETUP_PDI_EN_MASK		(0x00000008)
+#define ODN_PDP_PDISETUP_PDI_EN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_EN_SHIFT		(3)
+#define ODN_PDP_PDISETUP_PDI_EN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_GDEN
+*/
+#define ODN_PDP_PDISETUP_PDI_GDEN_MASK		(0x00000004)
+#define ODN_PDP_PDISETUP_PDI_GDEN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_GDEN_SHIFT		(2)
+#define ODN_PDP_PDISETUP_PDI_GDEN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_GDEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_NFEN
+*/
+#define ODN_PDP_PDISETUP_PDI_NFEN_MASK		(0x00000002)
+#define ODN_PDP_PDISETUP_PDI_NFEN_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_NFEN_SHIFT		(1)
+#define ODN_PDP_PDISETUP_PDI_NFEN_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_NFEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_CR
+*/
+#define ODN_PDP_PDISETUP_PDI_CR_MASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_CR_LSBMASK		(0x00000001)
+#define ODN_PDP_PDISETUP_PDI_CR_SHIFT		(0)
+#define ODN_PDP_PDISETUP_PDI_CR_LENGTH		(1)
+#define ODN_PDP_PDISETUP_PDI_CR_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDITIMING0_OFFSET		(0x0904)
+
+/* PDP, PDITIMING0, PDI_PWRSVGD
+*/
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_MASK		(0x0F000000)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_LSBMASK		(0x0000000F)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_SHIFT		(24)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_LENGTH		(4)
+#define ODN_PDP_PDITIMING0_PDI_PWRSVGD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_LSDEL
+*/
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_MASK		(0x007F0000)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_LSBMASK		(0x0000007F)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_SHIFT		(16)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_LENGTH		(7)
+#define ODN_PDP_PDITIMING0_PDI_LSDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_PWRSV2GD2
+*/
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_MASK		(0x000003FF)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_LSBMASK		(0x000003FF)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_SHIFT		(0)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_LENGTH		(10)
+#define ODN_PDP_PDITIMING0_PDI_PWRSV2GD2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDITIMING1_OFFSET		(0x0908)
+
+/* PDP, PDITIMING1, PDI_NLDEL
+*/
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_MASK		(0x000F0000)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_LSBMASK		(0x0000000F)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_SHIFT		(16)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_LENGTH		(4)
+#define ODN_PDP_PDITIMING1_PDI_NLDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING1, PDI_ACBDEL
+*/
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_MASK		(0x000003FF)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_LSBMASK		(0x000003FF)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_SHIFT		(0)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_LENGTH		(10)
+#define ODN_PDP_PDITIMING1_PDI_ACBDEL_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDICOREID_OFFSET		(0x090C)
+
+/* PDP, PDICOREID, PDI_GROUP_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_MASK		(0xFF000000)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_SHIFT		(24)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_LENGTH		(8)
+#define ODN_PDP_PDICOREID_PDI_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CORE_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_MASK		(0x00FF0000)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_SHIFT		(16)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_LENGTH		(8)
+#define ODN_PDP_PDICOREID_PDI_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CONFIG_ID
+*/
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_MASK		(0x0000FFFF)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_SHIFT		(0)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_LENGTH		(16)
+#define ODN_PDP_PDICOREID_PDI_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_PDICOREREV_OFFSET		(0x0910)
+
+/* PDP, PDICOREREV, PDI_MAJOR_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_MASK		(0x00FF0000)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_SHIFT		(16)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MINOR_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_MASK		(0x0000FF00)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_SHIFT		(8)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MAINT_REV
+*/
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_MASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_LSBMASK		(0x000000FF)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_SHIFT		(0)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_LENGTH		(8)
+#define ODN_PDP_PDICOREREV_PDI_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX2_OFFSET		(0x0920)
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_MASK		(0x000000C0)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_MASK		(0x00000030)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_MASK		(0x0000000C)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SHIFT		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_MASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LSBMASK		(0x00000003)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LENGTH		(2)
+#define ODN_PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX4_0_OFFSET		(0x0924)
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_MASK		(0xF0000000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SHIFT		(28)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_MASK		(0x0F000000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_MASK		(0x00F00000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SHIFT		(20)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_MASK		(0x000F0000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SHIFT		(16)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_MASK		(0x0000F000)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_MASK		(0x00000F00)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SHIFT		(8)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_MASK		(0x000000F0)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_MASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX4_1_OFFSET		(0x0928)
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_MASK		(0xF0000000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SHIFT		(28)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_MASK		(0x0F000000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_MASK		(0x00F00000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SHIFT		(20)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y3
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_MASK		(0x000F0000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SHIFT		(16)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_MASK		(0x0000F000)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_MASK		(0x00000F00)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SHIFT		(8)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_MASK		(0x000000F0)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SHIFT		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y2
+*/
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_MASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LSBMASK		(0x0000000F)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LENGTH		(4)
+#define ODN_PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_0_OFFSET		(0x092C)
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X4Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X3Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X2Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X1Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X0Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_1_OFFSET		(0x0930)
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X1Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X0Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X7Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X6Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X5Y0
+*/
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_2_OFFSET		(0x0934)
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X6Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X5Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X4Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X3Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X2Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_3_OFFSET		(0x0938)
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X3Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X2Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X1Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X0Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X7Y1
+*/
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_4_OFFSET		(0x093C)
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X0Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X7Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X6Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X5Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X4Y2
+*/
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_5_OFFSET		(0x0940)
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X5Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X4Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X3Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X2Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X1Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_6_OFFSET		(0x0944)
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X2Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X1Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X0Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X7Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X6Y3
+*/
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_7_OFFSET		(0x0948)
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X7Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X6Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X5Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X4Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X3Y4
+*/
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_8_OFFSET		(0x094C)
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X4Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X3Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X2Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X1Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X0Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_9_OFFSET		(0x0950)
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X1Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X0Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X7Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X6Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X5Y5
+*/
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_10_OFFSET		(0x0954)
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X6Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X5Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X4Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X3Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X2Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_11_OFFSET		(0x0958)
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X3Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_MASK		(0x3F000000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SHIFT		(24)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X2Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X1Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X0Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X7Y6
+*/
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_DITHERMATRIX8_12_OFFSET		(0x095C)
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X7Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_MASK		(0x00FC0000)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SHIFT		(18)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X6Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_MASK		(0x0003F000)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SHIFT		(12)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X5Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_MASK		(0x00000FC0)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SHIFT		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X4Y7
+*/
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_MASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LSBMASK		(0x0000003F)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SHIFT		(0)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LENGTH		(6)
+#define ODN_PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_MEMCTRL_OFFSET		(0x0960)
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_BURSTLEN
+*/
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_MEM_THRESH_OFFSET		(0x0964)
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_THRESHOLD
+*/
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_MEMCTRL_OFFSET		(0x0968)
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_BURSTLEN
+*/
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_MEM_THRESH_OFFSET		(0x096C)
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_THRESHOLD
+*/
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_MEMCTRL_OFFSET		(0x0970)
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_BURSTLEN
+*/
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_MEM_THRESH_OFFSET		(0x0974)
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_THRESHOLD
+*/
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_MEMCTRL_OFFSET		(0x0978)
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_BURSTLEN
+*/
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_MEM_THRESH_OFFSET		(0x097C)
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_UVTHRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_YTHRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_THRESHOLD
+*/
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_MEMCTRL_OFFSET		(0x0980)
+
+/* PDP, VID1_MEMCTRL, VID1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEMCTRL, VID1_BURSTLEN
+*/
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID1_MEMCTRL_VID1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_MEM_THRESH_OFFSET		(0x0984)
+
+/* PDP, VID1_MEM_THRESH, VID1_UVTHRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_YTHRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_THRESHOLD
+*/
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_MEMCTRL_OFFSET		(0x0988)
+
+/* PDP, VID2_MEMCTRL, VID2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEMCTRL, VID2_BURSTLEN
+*/
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID2_MEMCTRL_VID2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_MEM_THRESH_OFFSET		(0x098C)
+
+/* PDP, VID2_MEM_THRESH, VID2_UVTHRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_YTHRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_THRESHOLD
+*/
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_MEMCTRL_OFFSET		(0x0990)
+
+/* PDP, VID3_MEMCTRL, VID3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEMCTRL, VID3_BURSTLEN
+*/
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID3_MEMCTRL_VID3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_MEM_THRESH_OFFSET		(0x0994)
+
+/* PDP, VID3_MEM_THRESH, VID3_UVTHRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_YTHRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_THRESHOLD
+*/
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_MEMCTRL_OFFSET		(0x0998)
+
+/* PDP, VID4_MEMCTRL, VID4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define ODN_PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEMCTRL, VID4_BURSTLEN
+*/
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_MASK		(0x000000FF)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_SHIFT		(0)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_LENGTH		(8)
+#define ODN_PDP_VID4_MEMCTRL_VID4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_MEM_THRESH_OFFSET		(0x099C)
+
+/* PDP, VID4_MEM_THRESH, VID4_UVTHRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_MASK		(0xFF000000)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SHIFT		(24)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LENGTH		(8)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_YTHRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_MASK		(0x001FF000)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SHIFT		(12)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_THRESHOLD
+*/
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_MASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LSBMASK		(0x000001FF)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SHIFT		(0)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LENGTH		(9)
+#define ODN_PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH1_PANIC_THRESH_OFFSET		(0x09A0)
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH2_PANIC_THRESH_OFFSET		(0x09A4)
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH3_PANIC_THRESH_OFFSET		(0x09A8)
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_GRPH4_PANIC_THRESH_OFFSET		(0x09AC)
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID1_PANIC_THRESH_OFFSET		(0x09B0)
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID2_PANIC_THRESH_OFFSET		(0x09B4)
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID3_PANIC_THRESH_OFFSET		(0x09B8)
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_VID4_PANIC_THRESH_OFFSET		(0x09BC)
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_ENABLE
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SHIFT		(31)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_ENABLE
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SHIFT		(30)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LENGTH		(1)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MAX
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MIN
+*/
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define ODN_PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define ODN_PDP_BURST_BOUNDARY_OFFSET		(0x09C0)
+
+/* PDP, BURST_BOUNDARY, BURST_BOUNDARY
+*/
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_MASK		(0x0000003F)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_LSBMASK		(0x0000003F)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_SHIFT		(0)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_LENGTH		(6)
+#define ODN_PDP_BURST_BOUNDARY_BURST_BOUNDARY_SIGNED_FIELD	IMG_FALSE
+
+
+/* ---------------------- End of register definitions ---------------------- */
+
+/* NUMREG defines the extent of register address space.
+*/
+
+#define		ODN_PDP_NUMREG	   ((0x09C0 >> 2)+1)
+
+/* Info about video plane addresses */
+#define ODN_PDP_YADDR_BITS		(ODN_PDP_VID1BASEADDR_VID1BASEADDR_LENGTH)
+#define ODN_PDP_YADDR_ALIGN		5
+#define ODN_PDP_UADDR_BITS		(ODN_PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH)
+#define ODN_PDP_UADDR_ALIGN		5
+#define ODN_PDP_VADDR_BITS		(ODN_PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH)
+#define ODN_PDP_VADDR_ALIGN		5
+
+#define ODN_PDP_YSTRIDE_BITS	(ODN_PDP_VID1STRIDE_VID1STRIDE_LENGTH)
+#define ODN_PDP_YSTRIDE_ALIGN	5
+
+#define ODN_PDP_MAX_INPUT_WIDTH (ODN_PDP_VID1SIZE_VID1WIDTH_LSBMASK + 1)
+#define ODN_PDP_MAX_INPUT_HEIGHT (ODN_PDP_VID1SIZE_VID1HEIGHT_LSBMASK + 1)
+
+/* Maximum 6 bytes per pixel for RGB161616 */
+#define ODN_PDP_MAX_IMAGE_BYTES (ODN_PDP_MAX_INPUT_WIDTH * ODN_PDP_MAX_INPUT_HEIGHT * 6)
+
+/* Round up */
+#define ODN_PDP_MAX_IMAGE_PAGES ((ODN_PDP_MAX_IMAGE_BYTES+PAGE_SIZE-1)/PAGE_SIZE)
+
+#define ODN_PDP_YADDR_MAX		(((1 << ODN_PDP_YADDR_BITS) - 1) << ODN_PDP_YADDR_ALIGN)
+#define ODN_PDP_UADDR_MAX		(((1 << ODN_PDP_UADDR_BITS) - 1) << ODN_PDP_UADDR_ALIGN)
+#define ODN_PDP_VADDR_MAX		(((1 << ODN_PDP_VADDR_BITS) - 1) << ODN_PDP_VADDR_ALIGN)
+#define ODN_PDP_YSTRIDE_MAX		((1 << ODN_PDP_YSTRIDE_BITS) << ODN_PDP_YSTRIDE_ALIGN)
+#define ODN_PDP_YADDR_ALIGNMASK		((1 << ODN_PDP_YADDR_ALIGN) - 1)
+#define ODN_PDP_UADDR_ALIGNMASK		((1 << ODN_PDP_UADDR_ALIGN) - 1)
+#define ODN_PDP_VADDR_ALIGNMASK		((1 << ODN_PDP_VADDR_ALIGN) - 1)
+#define ODN_PDP_YSTRIDE_ALIGNMASK	((1 << ODN_PDP_YSTRIDE_ALIGN) - 1)
+
+/* Field Values (some are reserved for future use) */
+#define ODN_PDP_SURF_PIXFMT_RGB332					0x3
+#define ODN_PDP_SURF_PIXFMT_ARGB4444				0x4
+#define ODN_PDP_SURF_PIXFMT_ARGB1555				0x5
+#define ODN_PDP_SURF_PIXFMT_RGB888					0x6
+#define ODN_PDP_SURF_PIXFMT_RGB565					0x7
+#define ODN_PDP_SURF_PIXFMT_ARGB8888				0x8
+#define ODN_PDP_SURF_PIXFMT_420_PL8					0x9
+#define ODN_PDP_SURF_PIXFMT_420_PL8IVU				0xA
+#define ODN_PDP_SURF_PIXFMT_420_PL8IUV				0xB
+#define ODN_PDP_SURF_PIXFMT_422_UY0VY1_8888			0xC
+#define ODN_PDP_SURF_PIXFMT_422_VY0UY1_8888			0xD
+#define ODN_PDP_SURF_PIXFMT_422_Y0UY1V_8888			0xE
+#define ODN_PDP_SURF_PIXFMT_422_Y0VY1U_8888			0xF
+#define ODN_PDP_SURF_PIXFMT_AYUV8888				0x10
+#define ODN_PDP_SURF_PIXFMT_YUV101010				0x15
+#define ODN_PDP_SURF_PIXFMT_RGB101010				0x17
+#define ODN_PDP_SURF_PIXFMT_420_PL10IUV				0x18
+#define ODN_PDP_SURF_PIXFMT_420_PL10IVU				0x19
+#define ODN_PDP_SURF_PIXFMT_422_PL10IUV				0x1A
+#define ODN_PDP_SURF_PIXFMT_422_PL10IVU				0x1B
+#define ODN_PDP_SURF_PIXFMT_RGB121212				0x1E
+#define ODN_PDP_SURF_PIXFMT_RGB161616				0x1F
+
+#define ODN_PDP_CTRL_CKEYSRC_PREV					0x0
+#define ODN_PDP_CTRL_CKEYSRC_CUR					0x1
+
+#define ODN_PDP_MEMCTRL_MEMREFRESH_ALWAYS			0x0
+#define ODN_PDP_MEMCTRL_MEMREFRESH_HBLNK			0x1
+#define ODN_PDP_MEMCTRL_MEMREFRESH_VBLNK			0x2
+#define ODN_PDP_MEMCTRL_MEMREFRESH_BOTH				0x3
+
+#define ODN_PDP_3D_CTRL_BLENDSEL_BGND_WITH_POS0		0x0
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS0_WITH_POS1		0x1
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS1_WITH_POS2		0x2
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS2_WITH_POS3		0x3
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS3_WITH_POS4		0x4
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS4_WITH_POS5		0x5
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS5_WITH_POS6		0x6
+#define ODN_PDP_3D_CTRL_BLENDSEL_POS6_WITH_POS7		0x7
+
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_Y_STRIDE	0x0
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_DOUBLE_Y_STRIDE 0x1
+#define ODN_PDP_UADDR_UV_STRIDE_EQUAL_TO_HALF_Y_STRIDE 0x2
+
+#define ODN_PDP_PROCAMP_OUTPUT_OFFSET_FRACTIONAL_BITS 1
+#define ODN_PDP_PROCAMP_COEFFICIENT_FRACTIONAL_BITS	10
+
+/*---------------------------------------------------------------------------*/
+
+#endif /* ODN_PDP_REGS_H */
diff --git a/drivers/gpu/drm/img-rogue/apollo/odin_regs.h b/drivers/gpu/drm/img-rogue/apollo/odin_regs.h
new file mode 100644
index 000000000000..5d5821623ac7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/odin_regs.h
@@ -0,0 +1,1026 @@
+/******************************************************************************
+@Title          Odin system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin FPGA register defs for IMG 3rd generation TCF
+
+	Auto generated headers, eg. odn_core.h:
+		regconv -d . -a 8 odn_core.def
+
+	Source files :
+		odn_core.def
+		mca_debug.def
+		sai_rx_debug.def
+		sai_tx_debug.def
+		ad_tx.def
+
+	Changes:
+		Removed obsolete copyright dates
+		Changed lower case to upper case
+			(eg. odn_core changed to ODN_CORE)
+		Changed PVR5__ to ODN_
+		Merged multiple .def files into one header
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+/* tab size 4 */
+
+#ifndef _ODIN_REGS_H_
+#define _ODIN_REGS_H_
+
+/******************************
+  Generated from: odn_core.def
+*******************************/
+
+/*
+	Register ID
+*/
+#define ODN_CORE_ID                             0x0000
+#define ODN_ID_VARIANT_MASK                     0x0000FFFFU
+#define ODN_ID_VARIANT_SHIFT                    0
+#define ODN_ID_VARIANT_SIGNED                   0
+
+#define ODN_ID_ID_MASK                          0xFFFF0000U
+#define ODN_ID_ID_SHIFT                         16
+#define ODN_ID_ID_SIGNED                        0
+
+/*
+	Register REL
+*/
+#define ODN_CORE_REL                            0x0004
+#define ODN_REL_MINOR_MASK                      0x0000FFFFU
+#define ODN_REL_MINOR_SHIFT                     0
+#define ODN_REL_MINOR_SIGNED                    0
+
+#define ODN_REL_MAJOR_MASK                      0xFFFF0000U
+#define ODN_REL_MAJOR_SHIFT                     16
+#define ODN_REL_MAJOR_SIGNED                    0
+
+/*
+	Register CHANGE_SET
+*/
+#define ODN_CORE_CHANGE_SET                     0x0008
+#define ODN_CHANGE_SET_SET_MASK                 0xFFFFFFFFU
+#define ODN_CHANGE_SET_SET_SHIFT                0
+#define ODN_CHANGE_SET_SET_SIGNED               0
+
+/*
+	Register USER_ID
+*/
+#define ODN_CORE_USER_ID                        0x000C
+#define ODN_USER_ID_ID_MASK                     0x000000FFU
+#define ODN_USER_ID_ID_SHIFT                    0
+#define ODN_USER_ID_ID_SIGNED                   0
+
+/*
+	Register USER_BUILD
+*/
+#define ODN_CORE_USER_BUILD                     0x0010
+#define ODN_USER_BUILD_BUILD_MASK               0xFFFFFFFFU
+#define ODN_USER_BUILD_BUILD_SHIFT              0
+#define ODN_USER_BUILD_BUILD_SIGNED             0
+
+/*
+	Register SW_IF_VERSION
+*/
+#define ODN_CORE_SW_IF_VERSION                  0x0014
+#define ODN_SW_IF_VERSION_VERSION_MASK          0x0000FFFFU
+#define ODN_SW_IF_VERSION_VERSION_SHIFT         0
+#define ODN_SW_IF_VERSION_VERSION_SIGNED        0
+
+/*
+	Register INTERNAL_RESETN
+*/
+#define ODN_CORE_INTERNAL_RESETN                0x0080
+#define ODN_INTERNAL_RESETN_DDR_MASK            0x00000001U
+#define ODN_INTERNAL_RESETN_DDR_SHIFT           0
+#define ODN_INTERNAL_RESETN_DDR_SIGNED          0
+
+#define ODN_INTERNAL_RESETN_MIG0_MASK           0x00000002U
+#define ODN_INTERNAL_RESETN_MIG0_SHIFT          1
+#define ODN_INTERNAL_RESETN_MIG0_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_MIG1_MASK           0x00000004U
+#define ODN_INTERNAL_RESETN_MIG1_SHIFT          2
+#define ODN_INTERNAL_RESETN_MIG1_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PDP1_MASK           0x00000008U
+#define ODN_INTERNAL_RESETN_PDP1_SHIFT          3
+#define ODN_INTERNAL_RESETN_PDP1_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PDP2_MASK           0x00000010U
+#define ODN_INTERNAL_RESETN_PDP2_SHIFT          4
+#define ODN_INTERNAL_RESETN_PDP2_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PERIP_MASK          0x00000020U
+#define ODN_INTERNAL_RESETN_PERIP_SHIFT         5
+#define ODN_INTERNAL_RESETN_PERIP_SIGNED        0
+
+#define ODN_INTERNAL_RESETN_GIST_MASK           0x00000040U
+#define ODN_INTERNAL_RESETN_GIST_SHIFT          6
+#define ODN_INTERNAL_RESETN_GIST_SIGNED         0
+
+#define ODN_INTERNAL_RESETN_PIKE_MASK           0x00000080U
+#define ODN_INTERNAL_RESETN_PIKE_SHIFT          7
+#define ODN_INTERNAL_RESETN_PIKE_SIGNED         0
+
+/*
+	Register EXTERNAL_RESETN
+*/
+#define ODN_CORE_EXTERNAL_RESETN                0x0084
+#define ODN_EXTERNAL_RESETN_DUT_MASK            0x00000001U
+#define ODN_EXTERNAL_RESETN_DUT_SHIFT           0
+#define ODN_EXTERNAL_RESETN_DUT_SIGNED          0
+
+#define ODN_EXTERNAL_RESETN_DUT_SPI_MASK        0x00000002U
+#define ODN_EXTERNAL_RESETN_DUT_SPI_SHIFT       1
+#define ODN_EXTERNAL_RESETN_DUT_SPI_SIGNED      0
+
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_MASK    0x00000004U
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_SHIFT   2
+#define ODN_EXTERNAL_RESETN_DUT_PEP_DDR_SIGNED  0
+
+#define ODN_EXTERNAL_RESETN_DUT_IF_MASK         0x00000008U
+#define ODN_EXTERNAL_RESETN_DUT_IF_SHIFT        3
+#define ODN_EXTERNAL_RESETN_DUT_IF_SIGNED       0
+
+#define ODN_EXTERNAL_RESETN_DUT1_MASK           0x00000010U
+#define ODN_EXTERNAL_RESETN_DUT1_SHIFT          4
+#define ODN_EXTERNAL_RESETN_DUT1_SIGNED         0
+
+#define ODN_EXTERNAL_RESETN_DUT2_MASK           0x00000020U
+#define ODN_EXTERNAL_RESETN_DUT2_SHIFT          5
+#define ODN_EXTERNAL_RESETN_DUT2_SIGNED         0
+
+/*
+	Register EXTERNAL_RESET
+*/
+#define ODN_CORE_EXTERNAL_RESET                 0x0088
+#define ODN_EXTERNAL_RESET_PVT_CAL_MASK         0x00000001U
+#define ODN_EXTERNAL_RESET_PVT_CAL_SHIFT        0
+#define ODN_EXTERNAL_RESET_PVT_CAL_SIGNED       0
+
+#define ODN_EXTERNAL_RESET_PLL_MASK             0x00000002U
+#define ODN_EXTERNAL_RESET_PLL_SHIFT            1
+#define ODN_EXTERNAL_RESET_PLL_SIGNED           0
+
+/*
+	Register INTERNAL_AUTO_RESETN
+*/
+#define ODN_CORE_INTERNAL_AUTO_RESETN           0x008C
+#define ODN_INTERNAL_AUTO_RESETN_AUX_MASK       0x00000001U
+#define ODN_INTERNAL_AUTO_RESETN_AUX_SHIFT      0
+#define ODN_INTERNAL_AUTO_RESETN_AUX_SIGNED     0
+
+/*
+	Register CLK_GEN_RESET
+*/
+#define ODN_CORE_CLK_GEN_RESET                  0x0090
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_MASK    0x00000001U
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_SHIFT   0
+#define ODN_CLK_GEN_RESET_DUT_CORE_MMCM_SIGNED  0
+
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_MASK      0x00000002U
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_SHIFT     1
+#define ODN_CLK_GEN_RESET_DUT_IF_MMCM_SIGNED    0
+
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_MASK       0x00000004U
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_SHIFT      2
+#define ODN_CLK_GEN_RESET_MULTI_MMCM_SIGNED     0
+
+#define ODN_CLK_GEN_RESET_PDP_MMCM_MASK         0x00000008U
+#define ODN_CLK_GEN_RESET_PDP_MMCM_SHIFT        3
+#define ODN_CLK_GEN_RESET_PDP_MMCM_SIGNED       0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define ODN_CORE_INTERRUPT_STATUS               0x0100
+#define ODN_INTERRUPT_STATUS_DUT_MASK           0x00000001U
+#define ODN_INTERRUPT_STATUS_DUT_SHIFT          0
+#define ODN_INTERRUPT_STATUS_DUT_SIGNED         0
+
+#define ODN_INTERRUPT_STATUS_PDP1_MASK          0x00000002U
+#define ODN_INTERRUPT_STATUS_PDP1_SHIFT         1
+#define ODN_INTERRUPT_STATUS_PDP1_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_PDP2_MASK          0x00000004U
+#define ODN_INTERRUPT_STATUS_PDP2_SHIFT         2
+#define ODN_INTERRUPT_STATUS_PDP2_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_PERIP_MASK         0x00000008U
+#define ODN_INTERRUPT_STATUS_PERIP_SHIFT        3
+#define ODN_INTERRUPT_STATUS_PERIP_SIGNED       0
+
+#define ODN_INTERRUPT_STATUS_UART_MASK          0x00000010U
+#define ODN_INTERRUPT_STATUS_UART_SHIFT         4
+#define ODN_INTERRUPT_STATUS_UART_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_MASK 0x00000020U
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_STATUS_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_MASK 0x00000040U
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_SHIFT 6
+#define ODN_INTERRUPT_STATUS_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_STATUS_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_MASK 0x00000100U
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_STATUS_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_DUT2_MASK          0x00000200U
+#define ODN_INTERRUPT_STATUS_DUT2_SHIFT         9
+#define ODN_INTERRUPT_STATUS_DUT2_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_STATUS_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_STATUS_CDMA_MASK          0x00001800U
+#define ODN_INTERRUPT_STATUS_CDMA_SHIFT         11
+#define ODN_INTERRUPT_STATUS_CDMA_SIGNED        0
+
+#define ODN_INTERRUPT_STATUS_OS_IRQ_MASK        0x001FE000U
+#define ODN_INTERRUPT_STATUS_OS_IRQ_SHIFT       13
+#define ODN_INTERRUPT_STATUS_OS_IRQ_SIGNED      0
+
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_MASK      0x40000000U
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_SHIFT     30
+#define ODN_INTERRUPT_STATUS_IRQ_TEST_SIGNED    0
+
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_MASK 0x80000000U
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_SHIFT 31
+#define ODN_INTERRUPT_STATUS_MASTER_STATUS_SIGNED 0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define ODN_CORE_INTERRUPT_ENABLE               0x0104
+#define ODN_INTERRUPT_ENABLE_DUT_MASK           0x00000001U
+#define ODN_INTERRUPT_ENABLE_DUT_SHIFT          0
+#define ODN_INTERRUPT_ENABLE_DUT_SIGNED         0
+
+#define ODN_INTERRUPT_ENABLE_PDP1_MASK          0x00000002U
+#define ODN_INTERRUPT_ENABLE_PDP1_SHIFT         1
+#define ODN_INTERRUPT_ENABLE_PDP1_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_PDP2_MASK          0x00000004U
+#define ODN_INTERRUPT_ENABLE_PDP2_SHIFT         2
+#define ODN_INTERRUPT_ENABLE_PDP2_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_PERIP_MASK         0x00000008U
+#define ODN_INTERRUPT_ENABLE_PERIP_SHIFT        3
+#define ODN_INTERRUPT_ENABLE_PERIP_SIGNED       0
+
+#define ODN_INTERRUPT_ENABLE_UART_MASK          0x00000010U
+#define ODN_INTERRUPT_ENABLE_UART_SHIFT         4
+#define ODN_INTERRUPT_ENABLE_UART_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_MASK 0x00000020U
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_ENABLE_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_MASK 0x00000040U
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_SHIFT 6
+#define ODN_INTERRUPT_ENABLE_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_MASK 0x00000100U
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_ENABLE_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_DUT2_MASK          0x00000200U
+#define ODN_INTERRUPT_ENABLE_DUT2_SHIFT         9
+#define ODN_INTERRUPT_ENABLE_DUT2_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_ENABLE_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_ENABLE_CDMA_MASK          0x00001800U
+#define ODN_INTERRUPT_ENABLE_CDMA_SHIFT         11
+#define ODN_INTERRUPT_ENABLE_CDMA_SIGNED        0
+
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_MASK        0x001FE000U
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_SHIFT       13
+#define ODN_INTERRUPT_ENABLE_OS_IRQ_SIGNED      0
+
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_MASK      0x40000000U
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_SHIFT     30
+#define ODN_INTERRUPT_ENABLE_IRQ_TEST_SIGNED    0
+
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_MASK 0x80000000U
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_SHIFT 31
+#define ODN_INTERRUPT_ENABLE_MASTER_ENABLE_SIGNED 0
+
+/*
+	Register INTERRUPT_CLR
+*/
+#define ODN_CORE_INTERRUPT_CLR                  0x010C
+#define ODN_INTERRUPT_CLR_DUT_MASK              0x00000001U
+#define ODN_INTERRUPT_CLR_DUT_SHIFT             0
+#define ODN_INTERRUPT_CLR_DUT_SIGNED            0
+
+#define ODN_INTERRUPT_CLR_PDP1_MASK             0x00000002U
+#define ODN_INTERRUPT_CLR_PDP1_SHIFT            1
+#define ODN_INTERRUPT_CLR_PDP1_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_PDP2_MASK             0x00000004U
+#define ODN_INTERRUPT_CLR_PDP2_SHIFT            2
+#define ODN_INTERRUPT_CLR_PDP2_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_PERIP_MASK            0x00000008U
+#define ODN_INTERRUPT_CLR_PERIP_SHIFT           3
+#define ODN_INTERRUPT_CLR_PERIP_SIGNED          0
+
+#define ODN_INTERRUPT_CLR_UART_MASK             0x00000010U
+#define ODN_INTERRUPT_CLR_UART_SHIFT            4
+#define ODN_INTERRUPT_CLR_UART_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_MASK  0x00000020U
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_SHIFT 5
+#define ODN_INTERRUPT_CLR_GIST_IN_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_MASK   0x00000040U
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_SHIFT  6
+#define ODN_INTERRUPT_CLR_GIST_IN_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_MASK 0x00000080U
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_SHIFT 7
+#define ODN_INTERRUPT_CLR_GIST_OUT_LNK_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_MASK  0x00000100U
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_SHIFT 8
+#define ODN_INTERRUPT_CLR_GIST_OUT_MB_ERR_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_DUT2_MASK             0x00000200U
+#define ODN_INTERRUPT_CLR_DUT2_SHIFT            9
+#define PVR5__INTERRUPT_CLR_DUT2_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_MASK 0x00000400U
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_SHIFT 10
+#define ODN_INTERRUPT_CLR_AXI_LOCKUP_PROTECTION_SIGNED 0
+
+#define ODN_INTERRUPT_CLR_CDMA_MASK             0x00001800U
+#define ODN_INTERRUPT_CLR_CDMA_SHIFT            11
+#define ODN_INTERRUPT_CLR_CDMA_SIGNED           0
+
+#define ODN_INTERRUPT_CLR_OS_IRQ_MASK           0x001FE000U
+#define ODN_INTERRUPT_CLR_OS_IRQ_SHIFT          13
+#define ODN_INTERRUPT_CLR_OS_IRQ_SIGNED         0
+
+#define ODN_INTERRUPT_CLR_IRQ_TEST_MASK         0x40000000U
+#define ODN_INTERRUPT_CLR_IRQ_TEST_SHIFT        30
+#define ODN_INTERRUPT_CLR_IRQ_TEST_SIGNED       0
+
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_MASK     0x80000000U
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_SHIFT    31
+#define ODN_INTERRUPT_CLR_MASTER_CLEAR_SIGNED   0
+
+/*
+	Register INTERRUPT_TEST
+*/
+#define ODN_CORE_INTERRUPT_TEST                 0x0110
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_MASK  0x00000001U
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_SHIFT 0
+#define ODN_INTERRUPT_TEST_INTERRUPT_TEST_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT_CLR
+*/
+#define ODN_CORE_INTERRUPT_TIMEOUT_CLR      0x0114
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_MASK 0x00000002U
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SHIFT 1
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SIGNED 0
+
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_MASK 0x00000001U
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SHIFT 0
+#define ODN_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT
+*/
+#define ODN_CORE_INTERRUPT_TIMEOUT          0x0118
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_MASK 0xFFFFFFFFU
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SHIFT 0
+#define ODN_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SIGNED 0
+/*
+	Register SYSTEM_ID
+*/
+#define ODN_CORE_SYSTEM_ID                      0x011C
+#define ODN_SYSTEM_ID_ID_MASK                   0x0000FFFFU
+#define ODN_SYSTEM_ID_ID_SHIFT                  0
+#define ODN_SYSTEM_ID_ID_SIGNED                 0
+
+/*
+	Register SUPPORTED_FEATURES
+*/
+#define ODN_CORE_SUPPORTED_FEATURES             0x0120
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_MASK 0xFFFFFFFEU
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_SHIFT 1
+#define ODN_SUPPORTED_FEATURES_UNIMPLEMENTED_FREATURES_SIGNED 0
+
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_MASK 0x00000001U
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_SHIFT 0
+#define ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS_SIGNED 0
+
+/*
+	Register NUM_GPIO
+*/
+#define ODN_CORE_NUM_GPIO                       0x0180
+#define ODN_NUM_GPIO_NUMBER_MASK                0x0000000FU
+#define ODN_NUM_GPIO_NUMBER_SHIFT               0
+#define ODN_NUM_GPIO_NUMBER_SIGNED              0
+
+/*
+	Register GPIO_EN
+*/
+#define ODN_CORE_GPIO_EN                        0x0184
+#define ODN_GPIO_EN_DIRECTION_MASK              0x000000FFU
+#define ODN_GPIO_EN_DIRECTION_SHIFT             0
+#define ODN_GPIO_EN_DIRECTION_SIGNED            0
+
+/*
+	Register GPIO
+*/
+#define ODN_CORE_GPIO                           0x0188
+#define ODN_GPIO_GPIO_MASK                      0x000000FFU
+#define ODN_GPIO_GPIO_SHIFT                     0
+#define ODN_GPIO_GPIO_SIGNED                    0
+
+/*
+	Register NUM_DUT_CTRL
+*/
+#define ODN_CORE_NUM_DUT_CTRL                   0x0190
+#define ODN_NUM_DUT_CTRL_NUM_PINS_MASK          0xFFFFFFFFU
+#define ODN_NUM_DUT_CTRL_NUM_PINS_SHIFT         0
+#define ODN_NUM_DUT_CTRL_NUM_PINS_SIGNED        0
+
+/*
+	Register DUT_CTRL1
+*/
+#define ODN_CORE_DUT_CTRL1                      0x0194
+#define ODN_DUT_CTRL1_CONTROL1_MASK             0x3FFFFFFFU
+#define ODN_DUT_CTRL1_CONTROL1_SHIFT            0
+#define ODN_DUT_CTRL1_CONTROL1_SIGNED           0
+
+#define ODN_DUT_CTRL1_FBDC_BYPASS_MASK          0x40000000U
+#define ODN_DUT_CTRL1_FBDC_BYPASS_SHIFT         30
+#define ODN_DUT_CTRL1_FBDC_BYPASS_SIGNED        0
+
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_MASK       0x80000000U
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_SHIFT      31
+#define ODN_DUT_CTRL1_DUT_MST_OFFSET_SIGNED     0
+
+/*
+	Register DUT_CTRL2
+*/
+#define ODN_CORE_DUT_CTRL2                      0x0198
+#define ODN_DUT_CTRL2_CONTROL2_MASK             0xFFFFFFFFU
+#define ODN_DUT_CTRL2_CONTROL2_SHIFT            0
+#define ODN_DUT_CTRL2_CONTROL2_SIGNED           0
+
+/*
+	Register NUM_DUT_STAT
+*/
+#define ODN_CORE_NUM_DUT_STAT                   0x019C
+#define ODN_NUM_DUT_STAT_NUM_PINS_MASK          0xFFFFFFFFU
+#define ODN_NUM_DUT_STAT_NUM_PINS_SHIFT         0
+#define ODN_NUM_DUT_STAT_NUM_PINS_SIGNED        0
+
+/*
+	Register DUT_STAT1
+*/
+#define ODN_CORE_DUT_STAT1                      0x01A0
+#define ODN_DUT_STAT1_STATUS1_MASK              0xFFFFFFFFU
+#define ODN_DUT_STAT1_STATUS1_SHIFT             0
+#define ODN_DUT_STAT1_STATUS1_SIGNED            0
+
+/*
+	Register DUT_STAT2
+*/
+#define ODN_CORE_DUT_STAT2                      0x01A4
+#define ODN_DUT_STAT2_STATUS2_MASK              0xFFFFFFFFU
+#define ODN_DUT_STAT2_STATUS2_SHIFT             0
+#define ODN_DUT_STAT2_STATUS2_SIGNED            0
+
+/*
+	Register DASH_LEDS
+*/
+#define ODN_CORE_DASH_LEDS                      0x01A8
+#define ODN_DASH_LEDS_REPA_MASK                 0xFFF00000U
+#define ODN_DASH_LEDS_REPA_SHIFT                20
+#define ODN_DASH_LEDS_REPA_SIGNED               0
+
+#define ODN_DASH_LEDS_PIKE_MASK                 0x00000FFFU
+#define ODN_DASH_LEDS_PIKE_SHIFT                0
+#define ODN_DASH_LEDS_PIKE_SIGNED               0
+
+/*
+	Register DUT_CLK_INFO
+*/
+#define ODN_CORE_DUT_CLK_INFO                   0x01B0
+#define ODN_DUT_CLK_INFO_CORE_MASK              0x0000FFFFU
+#define ODN_DUT_CLK_INFO_CORE_SHIFT             0
+#define ODN_DUT_CLK_INFO_CORE_SIGNED            0
+
+#define ODN_DUT_CLK_INFO_MEM_MASK               0xFFFF0000U
+#define ODN_DUT_CLK_INFO_MEM_SHIFT              16
+#define ODN_DUT_CLK_INFO_MEM_SIGNED             0
+
+/*
+	Register DUT_CLK_PHSE
+*/
+#define ODN_CORE_DUT_CLK_PHSE                   0x01B4
+#define ODN_DUT_CLK_PHSE_MEM_REQ_MASK           0x0000FFFFU
+#define ODN_DUT_CLK_PHSE_MEM_REQ_SHIFT          0
+#define ODN_DUT_CLK_PHSE_MEM_REQ_SIGNED         0
+
+#define ODN_DUT_CLK_PHSE_MEM_RD_MASK            0xFFFF0000U
+#define ODN_DUT_CLK_PHSE_MEM_RD_SHIFT           16
+#define ODN_DUT_CLK_PHSE_MEM_RD_SIGNED          0
+
+/*
+	Register CORE_STATUS
+*/
+#define ODN_CORE_CORE_STATUS                    0x0200
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_MASK   0x00000001U
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_SHIFT  0
+#define ODN_CORE_STATUS_PCIE_USER_LNK_UP_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_MASK 0x00000010U
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_SHIFT 4
+#define ODN_CORE_STATUS_MIG_C0_MMCM_LOCKED_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_MASK 0x00000020U
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_SHIFT 5
+#define ODN_CORE_STATUS_MIG_C0_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_MASK 0x00000040U
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_SHIFT 6
+#define ODN_CORE_STATUS_MIG_C1_MMCM_LOCKED_SIGNED 0
+
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_MASK 0x00000080U
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_SHIFT 7
+#define ODN_CORE_STATUS_MIG_C1_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_MASK 0x00000100U
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_SHIFT 8
+#define ODN_CORE_STATUS_PERIP_IMG2AXI_IDLE_SIGNED 0
+
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_MASK 0x00000200U
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_SHIFT 9
+#define ODN_CORE_STATUS_PERIP_AXI2IMG_IDLE_SIGNED 0
+
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_MASK 0x00001000U
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SHIFT 12
+#define ODN_CORE_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_MASK 0x00002000U
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SHIFT 13
+#define ODN_CORE_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+/*
+	Register CORE_CONTROL
+*/
+#define ODN_CORE_CORE_CONTROL                   0x0204
+#define ODN_CORE_CONTROL_BAR4_OFFSET_MASK       0x0000001FU
+#define ODN_CORE_CONTROL_BAR4_OFFSET_SHIFT      0
+#define ODN_CORE_CONTROL_BAR4_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_MASK 0x00000300U
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SHIFT 8
+#define ODN_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SIGNED 0
+
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_MASK    0x00001C00U
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_SHIFT   10
+#define ODN_CORE_CONTROL_HDMI_MODULE_EN_SIGNED  0
+
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_MASK 0x00002000U
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SHIFT 13
+#define ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SIGNED 0
+
+#define ODN_CORE_CONTROL_PDP1_OFFSET_MASK       0x00070000U
+#define ODN_CORE_CONTROL_PDP1_OFFSET_SHIFT      16
+#define ODN_CORE_CONTROL_PDP1_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_PDP2_OFFSET_MASK       0x00700000U
+#define ODN_CORE_CONTROL_PDP2_OFFSET_SHIFT      20
+#define ODN_CORE_CONTROL_PDP2_OFFSET_SIGNED     0
+
+#define ODN_CORE_CONTROL_DUT_OFFSET_MASK        0x07000000U
+#define ODN_CORE_CONTROL_DUT_OFFSET_SHIFT       24
+#define ODN_CORE_CONTROL_DUT_OFFSET_SIGNED      0
+
+/*
+	Register REG_BANK_STATUS
+*/
+#define ODN_CORE_REG_BANK_STATUS                0x0208
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_MASK 0xFFFFFFFFU
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SHIFT 0
+#define ODN_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SIGNED 0
+
+/*
+	Register MMCM_LOCK_STATUS
+*/
+#define ODN_CORE_MMCM_LOCK_STATUS               0x020C
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_MASK      0x00000001U
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_SHIFT     0
+#define ODN_MMCM_LOCK_STATUS_DUT_CORE_SIGNED    0
+
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_MASK        0x00000002U
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_SHIFT       1
+#define ODN_MMCM_LOCK_STATUS_DUT_IF_SIGNED      0
+
+#define ODN_MMCM_LOCK_STATUS_MULTI_MASK         0x00000004U
+#define ODN_MMCM_LOCK_STATUS_MULTI_SHIFT        2
+#define ODN_MMCM_LOCK_STATUS_MULTI_SIGNED       0
+
+#define ODN_MMCM_LOCK_STATUS_PDPP_MASK          0x00000008U
+#define ODN_MMCM_LOCK_STATUS_PDPP_SHIFT         3
+#define ODN_MMCM_LOCK_STATUS_PDPP_SIGNED        0
+
+/*
+	Register GIST_STATUS
+*/
+#define ODN_CORE_GIST_STATUS                    0x0210
+#define ODN_GIST_STATUS_MST_MASK                0x000001FFU
+#define ODN_GIST_STATUS_MST_SHIFT               0
+#define ODN_GIST_STATUS_MST_SIGNED              0
+
+#define ODN_GIST_STATUS_SLV_MASK                0x001FF000U
+#define ODN_GIST_STATUS_SLV_SHIFT               12
+#define ODN_GIST_STATUS_SLV_SIGNED              0
+
+#define ODN_GIST_STATUS_SLV_OUT_MASK            0x03000000U
+#define ODN_GIST_STATUS_SLV_OUT_SHIFT           24
+#define ODN_GIST_STATUS_SLV_OUT_SIGNED          0
+
+#define ODN_GIST_STATUS_MST_OUT_MASK            0x70000000U
+#define ODN_GIST_STATUS_MST_OUT_SHIFT           28
+#define ODN_GIST_STATUS_MST_OUT_SIGNED          0
+
+/*
+	Register DUT_MST_ADD
+*/
+#define ODN_CORE_DUT_MST_ADD                    0x0214
+#define ODN_DUT_MST_ADD_SLV_OUT_MASK            0x0000003FU
+#define ODN_DUT_MST_ADD_SLV_OUT_SHIFT           0
+#define ODN_DUT_MST_ADD_SLV_OUT_SIGNED          0
+
+/*
+	Register DUT_MULTIPLX_INFO
+*/
+#define ODN_CORE_DUT_MULTIPLX_INFO              0x0218
+#define ODN_DUT_MULTIPLX_INFO_MEM_MASK          0x000000FFU
+#define ODN_DUT_MULTIPLX_INFO_MEM_SHIFT         0
+#define ODN_DUT_MULTIPLX_INFO_MEM_SIGNED        0
+
+/****************************
+  Generated from: ad_tx.def
+*****************************/
+
+/*
+	Register ADT_CONTROL
+*/
+#define ODN_AD_TX_DEBUG_ADT_CONTROL             0x0000
+#define ODN_SET_ADTX_READY_MASK                 0x00000004U
+#define ODN_SET_ADTX_READY_SHIFT                2
+#define ODN_SET_ADTX_READY_SIGNED               0
+
+#define ODN_SEND_ALIGN_DATA_MASK                0x00000002U
+#define ODN_SEND_ALIGN_DATA_SHIFT               1
+#define ODN_SEND_ALIGN_DATA_SIGNED              0
+
+#define ODN_ENABLE_FLUSHING_MASK                0x00000001U
+#define ODN_ENABLE_FLUSHING_SHIFT               0
+#define ODN_ENABLE_FLUSHING_SIGNED              0
+
+/*
+	Register ADT_STATUS
+*/
+#define ODN_AD_TX_DEBUG_ADT_STATUS              0x0004
+#define ODN_REQUEST_COMPLETE_MASK               0x00000001U
+#define ODN_REQUEST_COMPLETE_SHIFT              0
+#define ODN_REQUEST_COMPLETE_SIGNED             0
+
+
+/******************************
+ Generated from: mca_debug.def
+*******************************/
+
+/*
+	Register MCA_CONTROL
+*/
+#define ODN_MCA_DEBUG_MCA_CONTROL               0x0000
+#define ODN_ALIGN_START_MASK                    0x00000001U
+#define ODN_ALIGN_START_SHIFT                   0
+#define ODN_ALIGN_START_SIGNED                  0
+
+/*
+	Register MCA_STATUS
+*/
+#define ODN_MCA_DEBUG_MCA_STATUS                0x0004
+#define ODN_TCHECK_SDEBUG_MASK                  0x40000000U
+#define ODN_TCHECK_SDEBUG_SHIFT                 30
+#define ODN_TCHECK_SDEBUG_SIGNED                0
+
+#define ODN_CHECK_SDEBUG_MASK                   0x20000000U
+#define ODN_CHECK_SDEBUG_SHIFT                  29
+#define ODN_CHECK_SDEBUG_SIGNED                 0
+
+#define ODN_ALIGN_SDEBUG_MASK                   0x10000000U
+#define ODN_ALIGN_SDEBUG_SHIFT                  28
+#define ODN_ALIGN_SDEBUG_SIGNED                 0
+
+#define ODN_FWAIT_SDEBUG_MASK                   0x08000000U
+#define ODN_FWAIT_SDEBUG_SHIFT                  27
+#define ODN_FWAIT_SDEBUG_SIGNED                 0
+
+#define ODN_IDLE_SDEBUG_MASK                    0x04000000U
+#define ODN_IDLE_SDEBUG_SHIFT                   26
+#define ODN_IDLE_SDEBUG_SIGNED                  0
+
+#define ODN_FIFO_FULL_MASK                      0x03FF0000U
+#define ODN_FIFO_FULL_SHIFT                     16
+#define ODN_FIFO_FULL_SIGNED                    0
+
+#define ODN_FIFO_EMPTY_MASK                     0x0000FFC0U
+#define ODN_FIFO_EMPTY_SHIFT                    6
+#define ODN_FIFO_EMPTY_SIGNED                   0
+
+#define ODN_TAG_CHECK_ERROR_MASK                0x00000020U
+#define ODN_TAG_CHECK_ERROR_SHIFT               5
+#define ODN_TAG_CHECK_ERROR_SIGNED              0
+
+#define ODN_ALIGN_CHECK_ERROR_MASK              0x00000010U
+#define ODN_ALIGN_CHECK_ERROR_SHIFT             4
+#define ODN_ALIGN_CHECK_ERROR_SIGNED            0
+
+#define ODN_ALIGN_ERROR_MASK                    0x00000008U
+#define ODN_ALIGN_ERROR_SHIFT                   3
+#define ODN_ALIGN_ERROR_SIGNED                  0
+
+#define ODN_TAG_CHECKING_OK_MASK                0x00000004U
+#define ODN_TAG_CHECKING_OK_SHIFT               2
+#define ODN_TAG_CHECKING_OK_SIGNED              0
+
+#define ODN_ALIGN_CHECK_OK_MASK                 0x00000002U
+#define ODN_ALIGN_CHECK_OK_SHIFT                1
+#define ODN_ALIGN_CHECK_OK_SIGNED               0
+
+#define ODN_ALIGNMENT_FOUND_MASK                0x00000001U
+#define ODN_ALIGNMENT_FOUND_SHIFT               0
+#define ODN_ALIGNMENT_FOUND_SIGNED              0
+
+
+/*********************************
+ Generated from: sai_rx_debug.def
+**********************************/
+
+/*
+	Register SIG_RESULT
+*/
+#define ODN_SAI_RX_DEBUG_SIG_RESULT             0x0000
+#define ODN_SIG_RESULT_VALUE_MASK               0xFFFFFFFFU
+#define ODN_SIG_RESULT_VALUE_SHIFT              0
+#define ODN_SIG_RESULT_VALUE_SIGNED             0
+
+/*
+	Register INIT_SIG
+*/
+#define ODN_SAI_RX_DEBUG_INIT_SIG               0x0004
+#define ODN_INIT_SIG_VALUE_MASK                 0x00000001U
+#define ODN_INIT_SIG_VALUE_SHIFT                0
+#define ODN_INIT_SIG_VALUE_SIGNED               0
+
+/*
+	Register SAI_BYPASS
+*/
+#define ODN_SAI_RX_DEBUG_SAI_BYPASS             0x0008
+#define ODN_BYPASS_CLK_TAPS_VALUE_MASK          0x000003FFU
+#define ODN_BYPASS_CLK_TAPS_VALUE_SHIFT         0
+#define ODN_BYPASS_CLK_TAPS_VALUE_SIGNED        0
+
+#define ODN_BYPASS_SET_MASK                     0x00010000U
+#define ODN_BYPASS_SET_SHIFT                    16
+#define ODN_BYPASS_SET_SIGNED                   0
+
+#define ODN_BYPASS_EN_MASK                      0x00100000U
+#define ODN_BYPASS_EN_SHIFT                     20
+#define ODN_BYPASS_EN_SIGNED                    0
+
+#define ODN_EN_STATUS_MASK                      0x01000000U
+#define ODN_EN_STATUS_SHIFT                     24
+#define ODN_EN_STATUS_SIGNED                    0
+
+/*
+	Register SAI_CLK_TAPS
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CLK_TAPS           0x000C
+#define ODN_CLK_TAPS_VALUE_MASK                 0x000003FFU
+#define ODN_CLK_TAPS_VALUE_SHIFT                0
+#define ODN_CLK_TAPS_VALUE_SIGNED               0
+
+#define ODN_TRAINING_COMPLETE_MASK              0x00010000U
+#define ODN_TRAINING_COMPLETE_SHIFT             16
+#define ODN_TRAINING_COMPLETE_SIGNED            0
+
+/*
+	Register SAI_EYES
+*/
+#define ODN_SAI_RX_DEBUG_SAI_EYES               0x0010
+#define ODN_MIN_EYE_END_MASK                    0x0000FFFFU
+#define ODN_MIN_EYE_END_SHIFT                   0
+#define ODN_MIN_EYE_END_SIGNED                  0
+
+#define ODN_MAX_EYE_START_MASK                  0xFFFF0000U
+#define ODN_MAX_EYE_START_SHIFT                 16
+#define ODN_MAX_EYE_START_SIGNED                0
+
+/*
+	Register SAI_DDR_INVERT
+*/
+#define ODN_SAI_RX_DEBUG_SAI_DDR_INVERT         0x0014
+#define ODN_DDR_INVERT_MASK                     0x00000001U
+#define ODN_DDR_INVERT_SHIFT                    0
+#define ODN_DDR_INVERT_SIGNED                   0
+
+#define ODN_OVERIDE_VALUE_MASK                  0x00010000U
+#define ODN_OVERIDE_VALUE_SHIFT                 16
+#define ODN_OVERIDE_VALUE_SIGNED                0
+
+#define ODN_INVERT_OVERIDE_MASK                 0x00100000U
+#define ODN_INVERT_OVERIDE_SHIFT                20
+#define ODN_INVERT_OVERIDE_SIGNED               0
+
+/*
+	Register SAI_TRAIN_ACK
+*/
+#define ODN_SAI_RX_DEBUG_SAI_TRAIN_ACK          0x0018
+#define ODN_TRAIN_ACK_FAIL_MASK                 0x00000001U
+#define ODN_TRAIN_ACK_FAIL_SHIFT                0
+#define ODN_TRAIN_ACK_FAIL_SIGNED               0
+
+#define ODN_TRAIN_ACK_FAIL_COUNT_MASK           0x000000F0U
+#define ODN_TRAIN_ACK_FAIL_COUNT_SHIFT          4
+#define ODN_TRAIN_ACK_FAIL_COUNT_SIGNED         0
+
+#define ODN_TRAIN_ACK_COMPLETE_MASK             0x00000100U
+#define ODN_TRAIN_ACK_COMPLETE_SHIFT            8
+#define ODN_TRAIN_ACK_COMPLETE_SIGNED           0
+
+#define ODN_TRAIN_ACK_OVERIDE_MASK              0x00001000U
+#define ODN_TRAIN_ACK_OVERIDE_SHIFT             12
+#define ODN_TRAIN_ACK_OVERIDE_SIGNED            0
+
+/*
+	Register SAI_TRAIN_ACK_COUNT
+*/
+#define ODN_SAI_RX_DEBUG_SAI_TRAIN_ACK_COUNT    0x001C
+#define ODN_TRAIN_COUNT_MASK                    0xFFFFFFFFU
+#define ODN_TRAIN_COUNT_SHIFT                   0
+#define ODN_TRAIN_COUNT_SIGNED                  0
+
+/*
+	Register SAI_CHANNEL_NUMBER
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_NUMBER     0x0020
+#define ODN_CHANNEL_NUMBER_MASK                 0x0000FFFFU
+#define ODN_CHANNEL_NUMBER_SHIFT                0
+#define ODN_CHANNEL_NUMBER_SIGNED               0
+
+/*
+	Register SAI_CHANNEL_EYE_START
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_START  0x0024
+#define ODN_CHANNEL_EYE_START_MASK              0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_START_SHIFT             0
+#define ODN_CHANNEL_EYE_START_SIGNED            0
+
+/*
+	Register SAI_CHANNEL_EYE_END
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_END    0x0028
+#define ODN_CHANNEL_EYE_END_MASK                0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_END_SHIFT               0
+#define ODN_CHANNEL_EYE_END_SIGNED              0
+
+/*
+	Register SAI_CHANNEL_EYE_PATTERN
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_PATTERN 0x002C
+#define ODN_CHANNEL_EYE_PATTERN_MASK            0xFFFFFFFFU
+#define ODN_CHANNEL_EYE_PATTERN_SHIFT           0
+#define ODN_CHANNEL_EYE_PATTERN_SIGNED          0
+
+/*
+	Register SAI_CHANNEL_EYE_DEBUG
+*/
+#define ODN_SAI_RX_DEBUG_SAI_CHANNEL_EYE_DEBUG  0x0030
+#define ODN_CHANNEL_EYE_SENSE_MASK              0x00000001U
+#define ODN_CHANNEL_EYE_SENSE_SHIFT             0
+#define ODN_CHANNEL_EYE_SENSE_SIGNED            0
+
+#define ODN_CHANNEL_EYE_COMPLETE_MASK           0x00000002U
+#define ODN_CHANNEL_EYE_COMPLETE_SHIFT          1
+#define ODN_CHANNEL_EYE_COMPLETE_SIGNED         0
+
+
+/*********************************
+ Generated from: sai_tx_debug.def
+**********************************/
+
+/*
+	Register SIG_RESULT
+*/
+#define ODN_SAI_TX_DEBUG_SIG_RESULT             0x0000
+#define ODN_TX_SIG_RESULT_VALUE_MASK            0xFFFFFFFFU
+#define ODN_TX_SIG_RESULT_VALUE_SHIFT           0
+#define ODN_TX_SIG_RESULT_VALUE_SIGNED          0
+
+/*
+	Register INIT_SIG
+*/
+#define ODN_SAI_TX_DEBUG_INIT_SIG               0x0004
+#define ODN_TX_INIT_SIG_VALUE_MASK              0x00000001U
+#define ODN_TX_INIT_SIG_VALUE_SHIFT             0
+#define ODN_TX_INIT_SIG_VALUE_SIGNED            0
+
+/*
+	Register SAI_BYPASS
+*/
+#define ODN_SAI_TX_DEBUG_SAI_BYPASS             0x0008
+#define ODN_TX_BYPASS_EN_MASK                   0x00000001U
+#define ODN_TX_BYPASS_EN_SHIFT                  0
+#define ODN_TX_BYPASS_EN_SIGNED                 0
+
+#define ODN_TX_ACK_RESEND_MASK                  0x00000002U
+#define ODN_TX_ACK_RESEND_SHIFT                 1
+#define ODN_TX_ACK_RESEND_SIGNED                0
+
+#define ODN_TX_DISABLE_ACK_SEND_MASK            0x00000004U
+#define ODN_TX_DISABLE_ACK_SEND_SHIFT           2
+#define ODN_TX_DISABLE_ACK_SEND_SIGNED          0
+
+/*
+	Register SAI_STATUS
+*/
+#define ODN_SAI_TX_DEBUG_SAI_STATUS             0x000C
+#define ODN_TX_TRAINING_COMPLETE_MASK           0x00000001U
+#define ODN_TX_TRAINING_COMPLETE_SHIFT          0
+#define ODN_TX_TRAINING_COMPLETE_SIGNED         0
+
+#define ODN_TX_TRAINING_ACK_COMPLETE_MASK       0x00000002U
+#define ODN_TX_TRAINING_ACK_COMPLETE_SHIFT      1
+#define ODN_TX_TRAINING_ACK_COMPLETE_SIGNED     0
+
+
+
+#endif /* _ODIN_REGS_H_ */
+
+/******************************************************************************
+ End of file (odin_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/orion_defs.h b/drivers/gpu/drm/img-rogue/apollo/orion_defs.h
new file mode 100644
index 000000000000..1691151de58d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/orion_defs.h
@@ -0,0 +1,183 @@
+/****************************************************************************
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Orion Memory Map - View from PCIe
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+****************************************************************************/
+
+#ifndef _ORION_DEFS_H_
+#define _ORION_DEFS_H_
+
+/*
+ * These defines have not been autogenerated
+ * Only values different from Odin will be included here
+ */
+
+#define DEVICE_ID_ORION                          0x1020
+
+/* Odin system register banks */
+#define SRS_REG_BANK_ODN_CLK_BLK                 0x02000
+
+/*
+ * Orion CLK regs - the srs_clk_blk module defs are not auto generated
+ */
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG1           0x620
+#define SRS_PDP_PCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define SRS_PDP_PCLK_ODIV1_LO_TIME_SHIFT         0
+#define SRS_PDP_PCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define SRS_PDP_PCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG2           0x624
+#define SRS_PDP_PCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define SRS_PDP_PCLK_ODIV2_NOCOUNT_SHIFT         6
+#define SRS_PDP_PCLK_ODIV2_EDGE_MASK             0x00000080U
+#define SRS_PDP_PCLK_ODIV2_EDGE_SHIFT            7
+#define SRS_PDP_PCLK_ODIV2_FRAC_MASK             0x00007C00U
+#define SRS_PDP_PCLK_ODIV2_FRAC_SHIFT            10
+
+#define SRS_PDP_P_CLK_OUT_DIVIDER_REG3           0x61C
+
+#define SRS_PDP_M_CLK_OUT_DIVIDER_REG1           0x628
+#define SRS_PDP_MCLK_ODIV1_LO_TIME_MASK          0x0000003FU
+#define SRS_PDP_MCLK_ODIV1_LO_TIME_SHIFT         0
+#define SRS_PDP_MCLK_ODIV1_HI_TIME_MASK          0x00000FC0U
+#define SRS_PDP_MCLK_ODIV1_HI_TIME_SHIFT         6
+
+#define SRS_PDP_M_CLK_OUT_DIVIDER_REG2           0x62C
+#define SRS_PDP_MCLK_ODIV2_NOCOUNT_MASK          0x00000040U
+#define SRS_PDP_MCLK_ODIV2_NOCOUNT_SHIFT         6
+#define SRS_PDP_MCLK_ODIV2_EDGE_MASK             0x00000080U
+#define SRS_PDP_MCLK_ODIV2_EDGE_SHIFT            7
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG1            0x650
+#define SRS_PDP_PCLK_MUL1_LO_TIME_MASK           0x0000003FU
+#define SRS_PDP_PCLK_MUL1_LO_TIME_SHIFT          0
+#define SRS_PDP_PCLK_MUL1_HI_TIME_MASK           0x00000FC0U
+#define SRS_PDP_PCLK_MUL1_HI_TIME_SHIFT          6
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG2            0x654
+#define SRS_PDP_PCLK_MUL2_NOCOUNT_MASK           0x00000040U
+#define SRS_PDP_PCLK_MUL2_NOCOUNT_SHIFT          6
+#define SRS_PDP_PCLK_MUL2_EDGE_MASK              0x00000080U
+#define SRS_PDP_PCLK_MUL2_EDGE_SHIFT             7
+#define SRS_PDP_PCLK_MUL2_FRAC_MASK              0x00007C00U
+#define SRS_PDP_PCLK_MUL2_FRAC_SHIFT             10
+
+#define SRS_PDP_P_CLK_MULTIPLIER_REG3            0x64C
+
+#define SRS_PDP_P_CLK_IN_DIVIDER_REG             0x658
+#define SRS_PDP_PCLK_IDIV_LO_TIME_MASK           0x0000003FU
+#define SRS_PDP_PCLK_IDIV_LO_TIME_SHIFT          0
+#define SRS_PDP_PCLK_IDIV_HI_TIME_MASK           0x00000FC0U
+#define SRS_PDP_PCLK_IDIV_HI_TIME_SHIFT          6
+#define SRS_PDP_PCLK_IDIV_NOCOUNT_MASK           0x00001000U
+#define SRS_PDP_PCLK_IDIV_NOCOUNT_SHIFT          12
+#define SRS_PDP_PCLK_IDIV_EDGE_MASK              0x00002000U
+#define SRS_PDP_PCLK_IDIV_EDGE_SHIFT             13
+
+/*
+ * DUT core clock input divider, DUT reference clock input divider
+ */
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1                0x0020
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_MASK   0x00000FC0U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  6
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_MASK   0x0000003FU
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  0
+
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2                0x0024
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_MASK      0x00000080U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_EDGE_SHIFT     7
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_MASK   0x00000040U
+#define SRS_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  6
+
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1                 0x0028
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_HI_TIME_MASK    0x00000FC0U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_HI_TIME_SHIFT   6
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_LO_TIME_MASK    0x0000003FU
+#define SRS_DUT_REF_CLK_OUT_DIVIDER1_LO_TIME_SHIFT   0
+
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2                 0x002C
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_EDGE_MASK       0x00000080U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_EDGE_SHIFT      7
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_NOCOUNT_MASK    0x00000040U
+#define SRS_DUT_REF_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT   6
+
+/*
+ * DUT interface reference clock input divider
+ */
+
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1                0x0228
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_HI_TIME_MASK   0x00000FC0U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_HI_TIME_SHIFT  6
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_LO_TIME_MASK   0x0000003FU
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER1_LO_TIME_SHIFT  0
+
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2                0x022C
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_EDGE_MASK      0x00000080U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_EDGE_SHIFT     7
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_NOCOUNT_MASK   0x00000040U
+#define SRS_DUT_MEM_CLK_OUT_DIVIDER2_NOCOUNT_SHIFT  6
+
+/*
+ * Min max values from Xilinx Virtex Ultrascale data sheet DS893,
+ * for speed grade 1. All in Hz.
+ */
+#define SRS_INPUT_CLOCK_SPEED                       100000000U
+#define SRS_INPUT_CLOCK_SPEED_MIN                   10000000U
+#define SRS_INPUT_CLOCK_SPEED_MAX                   800000000U
+#define SRS_OUTPUT_CLOCK_SPEED_MIN                  4690000U
+#define SRS_OUTPUT_CLOCK_SPEED_MAX                  630000000U
+#define SRS_VCO_MIN                                 600000000U
+#define SRS_VCO_MAX                                 1200000000U
+#define SRS_PFD_MIN                                 10000000U
+#define SRS_PFD_MAX                                 450000000U
+
+/*
+ * Orion interrupt flags
+ */
+#define SRS_INTERRUPT_ENABLE_PDP1          (1 << SRS_INTERRUPT_ENABLE_PDP_SHIFT)
+#define SRS_INTERRUPT_ENABLE_DUT           (1 << SRS_INTERRUPT_ENABLE_DUT_SHIFT)
+#define SRS_INTERRUPT_STATUS_PDP1          (1 << SRS_INTERRUPT_STATUS_PDP_SHIFT)
+#define SRS_INTERRUPT_STATUS_DUT           (1 << SRS_INTERRUPT_STATUS_DUT_SHIFT)
+#define SRS_INTERRUPT_CLEAR_PDP1           (1 << SRS_INTERRUPT_CLR_PDP_SHIFT)
+#define SRS_INTERRUPT_CLEAR_DUT            (1 << SRS_INTERRUPT_CLR_DUT_SHIFT)
+
+#endif /* _ORION_DEFS_H_ */
+
+/*****************************************************************************
+ End of file (orion_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/orion_regs.h b/drivers/gpu/drm/img-rogue/apollo/orion_regs.h
new file mode 100644
index 000000000000..2a626bd2b70a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/orion_regs.h
@@ -0,0 +1,439 @@
+/******************************************************************************
+@Title          Orion system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Orion FPGA register defs for Sirius RTL
+@Author         Autogenerated
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef _OUT_DRV_H_
+#define _OUT_DRV_H_
+
+/*
+	Register ID
+*/
+#define SRS_CORE_ID                         0x0000
+#define SRS_ID_VARIANT_MASK                     0x0000FFFFU
+#define SRS_ID_VARIANT_SHIFT                    0
+#define SRS_ID_VARIANT_SIGNED                   0
+
+#define SRS_ID_ID_MASK                          0xFFFF0000U
+#define SRS_ID_ID_SHIFT                         16
+#define SRS_ID_ID_SIGNED                        0
+
+/*
+	Register REVISION
+*/
+#define SRS_CORE_REVISION                   0x0004
+#define SRS_REVISION_MINOR_MASK                 0x000000FFU
+#define SRS_REVISION_MINOR_SHIFT                0
+#define SRS_REVISION_MINOR_SIGNED               0
+
+#define SRS_REVISION_MAJOR_MASK                 0x00000F00U
+#define SRS_REVISION_MAJOR_SHIFT                8
+#define SRS_REVISION_MAJOR_SIGNED               0
+
+/*
+	Register CHANGE_SET
+*/
+#define SRS_CORE_CHANGE_SET                 0x0008
+#define SRS_CHANGE_SET_SET_MASK                 0xFFFFFFFFU
+#define SRS_CHANGE_SET_SET_SHIFT                0
+#define SRS_CHANGE_SET_SET_SIGNED               0
+
+/*
+	Register USER_ID
+*/
+#define SRS_CORE_USER_ID                    0x000C
+#define SRS_USER_ID_ID_MASK                     0x0000000FU
+#define SRS_USER_ID_ID_SHIFT                    0
+#define SRS_USER_ID_ID_SIGNED                   0
+
+/*
+	Register USER_BUILD
+*/
+#define SRS_CORE_USER_BUILD                 0x0010
+#define SRS_USER_BUILD_BUILD_MASK               0xFFFFFFFFU
+#define SRS_USER_BUILD_BUILD_SHIFT              0
+#define SRS_USER_BUILD_BUILD_SIGNED             0
+
+/*
+	Register SOFT_RESETN
+*/
+#define SRS_CORE_SOFT_RESETN                0x0080
+#define SRS_SOFT_RESETN_DDR_MASK                0x00000001U
+#define SRS_SOFT_RESETN_DDR_SHIFT               0
+#define SRS_SOFT_RESETN_DDR_SIGNED              0
+
+#define SRS_SOFT_RESETN_USB_MASK                0x00000002U
+#define SRS_SOFT_RESETN_USB_SHIFT               1
+#define SRS_SOFT_RESETN_USB_SIGNED              0
+
+#define SRS_SOFT_RESETN_PDP_MASK                0x00000004U
+#define SRS_SOFT_RESETN_PDP_SHIFT               2
+#define SRS_SOFT_RESETN_PDP_SIGNED              0
+
+#define SRS_SOFT_RESETN_GIST_MASK               0x00000008U
+#define SRS_SOFT_RESETN_GIST_SHIFT              3
+#define SRS_SOFT_RESETN_GIST_SIGNED             0
+
+/*
+	Register DUT_SOFT_RESETN
+*/
+#define SRS_CORE_DUT_SOFT_RESETN            0x0084
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_MASK       0x00000001U
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_SHIFT      0
+#define SRS_DUT_SOFT_RESETN_EXTERNAL_SIGNED     0
+
+/*
+	Register SOFT_AUTO_RESETN
+*/
+#define SRS_CORE_SOFT_AUTO_RESETN           0x0088
+#define SRS_SOFT_AUTO_RESETN_CFG_MASK           0x00000001U
+#define SRS_SOFT_AUTO_RESETN_CFG_SHIFT          0
+#define SRS_SOFT_AUTO_RESETN_CFG_SIGNED         0
+
+/*
+	Register CLK_GEN_RESET
+*/
+#define SRS_CORE_CLK_GEN_RESET              0x0090
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_MASK    0x00000001U
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_SHIFT   0
+#define SRS_CLK_GEN_RESET_DUT_CORE_MMCM_SIGNED  0
+
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_MASK      0x00000002U
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_SHIFT     1
+#define SRS_CLK_GEN_RESET_DUT_IF_MMCM_SIGNED    0
+
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_MASK       0x00000004U
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_SHIFT      2
+#define SRS_CLK_GEN_RESET_MULTI_MMCM_SIGNED     0
+
+#define SRS_CLK_GEN_RESET_PDP_MMCM_MASK         0x00000008U
+#define SRS_CLK_GEN_RESET_PDP_MMCM_SHIFT        3
+#define SRS_CLK_GEN_RESET_PDP_MMCM_SIGNED       0
+
+/*
+	Register DUT_MEM
+*/
+#define SRS_CORE_DUT_MEM                    0x0120
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_MASK  0x0000FFFFU
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_SHIFT 0
+#define SRS_DUT_MEM_READ_RESPONSE_LATENCY_SIGNED 0
+
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_MASK 0xFFFF0000U
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_SHIFT 16
+#define SRS_DUT_MEM_WRITE_RESPONSE_LATENCY_SIGNED 0
+
+/*
+	Register APM
+*/
+#define SRS_CORE_APM                        0x0150
+#define SRS_APM_RESET_EVENT_MASK                0x00000001U
+#define SRS_APM_RESET_EVENT_SHIFT               0
+#define SRS_APM_RESET_EVENT_SIGNED              0
+
+#define SRS_APM_CAPTURE_EVENT_MASK              0x00000002U
+#define SRS_APM_CAPTURE_EVENT_SHIFT             1
+#define SRS_APM_CAPTURE_EVENT_SIGNED            0
+
+/*
+	Register NUM_GPIO
+*/
+#define SRS_CORE_NUM_GPIO                   0x0180
+#define SRS_NUM_GPIO_NUMBER_MASK                0x0000000FU
+#define SRS_NUM_GPIO_NUMBER_SHIFT               0
+#define SRS_NUM_GPIO_NUMBER_SIGNED              0
+
+/*
+	Register GPIO_EN
+*/
+#define SRS_CORE_GPIO_EN                    0x0184
+#define SRS_GPIO_EN_DIRECTION_MASK              0x000000FFU
+#define SRS_GPIO_EN_DIRECTION_SHIFT             0
+#define SRS_GPIO_EN_DIRECTION_SIGNED            0
+
+/*
+	Register GPIO
+*/
+#define SRS_CORE_GPIO                       0x0188
+#define SRS_GPIO_GPIO_MASK                      0x000000FFU
+#define SRS_GPIO_GPIO_SHIFT                     0
+#define SRS_GPIO_GPIO_SIGNED                    0
+
+/*
+	Register SPI_MASTER_IFACE
+*/
+#define SRS_CORE_SPI_MASTER_IFACE           0x018C
+#define SRS_SPI_MASTER_IFACE_ENABLE_MASK        0x00000001U
+#define SRS_SPI_MASTER_IFACE_ENABLE_SHIFT       0
+#define SRS_SPI_MASTER_IFACE_ENABLE_SIGNED      0
+
+/*
+	Register SRS_IP_STATUS
+*/
+#define SRS_CORE_SRS_IP_STATUS              0x0200
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_MASK 0x00000001U
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_SHIFT 0
+#define SRS_SRS_IP_STATUS_PCIE_USER_LNK_UP_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_MASK 0x00000002U
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_SHIFT 1
+#define SRS_SRS_IP_STATUS_MIG_INIT_CALIB_COMPLETE_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_MASK 0x00000004U
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SHIFT 2
+#define SRS_SRS_IP_STATUS_GIST_SLV_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_MASK 0x00000008U
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SHIFT 3
+#define SRS_SRS_IP_STATUS_GIST_MST_C2C_CONFIG_ERROR_OUT_SIGNED 0
+
+/*
+	Register CORE_CONTROL
+*/
+#define SRS_CORE_CORE_CONTROL               0x0204
+#define SRS_CORE_CONTROL_BAR4_OFFSET_MASK       0x0000001FU
+#define SRS_CORE_CONTROL_BAR4_OFFSET_SHIFT      0
+#define SRS_CORE_CONTROL_BAR4_OFFSET_SIGNED     0
+
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_MASK 0x00000300U
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SHIFT 8
+#define SRS_CORE_CONTROL_HDMI_MONITOR_OVERRIDE_SIGNED 0
+
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_MASK    0x00001C00U
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_SHIFT   10
+#define SRS_CORE_CONTROL_HDMI_MODULE_EN_SIGNED  0
+
+/*
+	Register REG_BANK_STATUS
+*/
+#define SRS_CORE_REG_BANK_STATUS            0x0208
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_MASK 0xFFFFFFFFU
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SHIFT 0
+#define SRS_REG_BANK_STATUS_ARB_SLV_RD_TIMEOUT_SIGNED 0
+
+/*
+	Register MMCM_LOCK_STATUS
+*/
+#define SRS_CORE_MMCM_LOCK_STATUS           0x020C
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_MASK      0x00000001U
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_SHIFT     0
+#define SRS_MMCM_LOCK_STATUS_DUT_CORE_SIGNED    0
+
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_MASK        0x00000002U
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_SHIFT       1
+#define SRS_MMCM_LOCK_STATUS_DUT_IF_SIGNED      0
+
+#define SRS_MMCM_LOCK_STATUS_MULTI_MASK         0x00000004U
+#define SRS_MMCM_LOCK_STATUS_MULTI_SHIFT        2
+#define SRS_MMCM_LOCK_STATUS_MULTI_SIGNED       0
+
+#define SRS_MMCM_LOCK_STATUS_PDP_MASK           0x00000008U
+#define SRS_MMCM_LOCK_STATUS_PDP_SHIFT          3
+#define SRS_MMCM_LOCK_STATUS_PDP_SIGNED         0
+
+/*
+	Register GIST_STATUS
+*/
+#define SRS_CORE_GIST_STATUS                0x0210
+#define SRS_GIST_STATUS_MST_MASK                0x000001FFU
+#define SRS_GIST_STATUS_MST_SHIFT               0
+#define SRS_GIST_STATUS_MST_SIGNED              0
+
+#define SRS_GIST_STATUS_SLV_MASK                0x001FF000U
+#define SRS_GIST_STATUS_SLV_SHIFT               12
+#define SRS_GIST_STATUS_SLV_SIGNED              0
+
+#define SRS_GIST_STATUS_SLV_OUT_MASK            0x03000000U
+#define SRS_GIST_STATUS_SLV_OUT_SHIFT           24
+#define SRS_GIST_STATUS_SLV_OUT_SIGNED          0
+
+#define SRS_GIST_STATUS_MST_OUT_MASK            0x70000000U
+#define SRS_GIST_STATUS_MST_OUT_SHIFT           28
+#define SRS_GIST_STATUS_MST_OUT_SIGNED          0
+
+/*
+	Register SENSOR_BOARD
+*/
+#define SRS_CORE_SENSOR_BOARD               0x0214
+#define SRS_SENSOR_BOARD_ID_MASK                0x00000003U
+#define SRS_SENSOR_BOARD_ID_SHIFT               0
+#define SRS_SENSOR_BOARD_ID_SIGNED              0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define SRS_CORE_INTERRUPT_STATUS           0x0218
+#define SRS_INTERRUPT_STATUS_DUT_MASK           0x00000001U
+#define SRS_INTERRUPT_STATUS_DUT_SHIFT          0
+#define SRS_INTERRUPT_STATUS_DUT_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_PDP_MASK           0x00000002U
+#define SRS_INTERRUPT_STATUS_PDP_SHIFT          1
+#define SRS_INTERRUPT_STATUS_PDP_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_I2C_MASK           0x00000004U
+#define SRS_INTERRUPT_STATUS_I2C_SHIFT          2
+#define SRS_INTERRUPT_STATUS_I2C_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_SPI_MASK           0x00000008U
+#define SRS_INTERRUPT_STATUS_SPI_SHIFT          3
+#define SRS_INTERRUPT_STATUS_SPI_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_APM_MASK           0x00000010U
+#define SRS_INTERRUPT_STATUS_APM_SHIFT          4
+#define SRS_INTERRUPT_STATUS_APM_SIGNED         0
+
+#define SRS_INTERRUPT_STATUS_OS_IRQ_MASK        0x00001FE0U
+#define SRS_INTERRUPT_STATUS_OS_IRQ_SHIFT       5
+#define SRS_INTERRUPT_STATUS_OS_IRQ_SIGNED      0
+
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_MASK      0x40000000U
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_SHIFT     30
+#define SRS_INTERRUPT_STATUS_IRQ_TEST_SIGNED    0
+
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_MASK 0x80000000U
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_SHIFT 31
+#define SRS_INTERRUPT_STATUS_MASTER_STATUS_SIGNED 0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define SRS_CORE_INTERRUPT_ENABLE           0x021C
+#define SRS_INTERRUPT_ENABLE_DUT_MASK           0x00000001U
+#define SRS_INTERRUPT_ENABLE_DUT_SHIFT          0
+#define SRS_INTERRUPT_ENABLE_DUT_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_PDP_MASK           0x00000002U
+#define SRS_INTERRUPT_ENABLE_PDP_SHIFT          1
+#define SRS_INTERRUPT_ENABLE_PDP_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_I2C_MASK           0x00000004U
+#define SRS_INTERRUPT_ENABLE_I2C_SHIFT          2
+#define SRS_INTERRUPT_ENABLE_I2C_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_SPI_MASK           0x00000008U
+#define SRS_INTERRUPT_ENABLE_SPI_SHIFT          3
+#define SRS_INTERRUPT_ENABLE_SPI_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_APM_MASK           0x00000010U
+#define SRS_INTERRUPT_ENABLE_APM_SHIFT          4
+#define SRS_INTERRUPT_ENABLE_APM_SIGNED         0
+
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_MASK        0x00001FE0U
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_SHIFT       5
+#define SRS_INTERRUPT_ENABLE_OS_IRQ_SIGNED      0
+
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_MASK      0x40000000U
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_SHIFT     30
+#define SRS_INTERRUPT_ENABLE_IRQ_TEST_SIGNED    0
+
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_MASK 0x80000000U
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_SHIFT 31
+#define SRS_INTERRUPT_ENABLE_MASTER_ENABLE_SIGNED 0
+
+/*
+	Register INTERRUPT_CLR
+*/
+#define SRS_CORE_INTERRUPT_CLR              0x0220
+#define SRS_INTERRUPT_CLR_DUT_MASK              0x00000001U
+#define SRS_INTERRUPT_CLR_DUT_SHIFT             0
+#define SRS_INTERRUPT_CLR_DUT_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_PDP_MASK              0x00000002U
+#define SRS_INTERRUPT_CLR_PDP_SHIFT             1
+#define SRS_INTERRUPT_CLR_PDP_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_I2C_MASK              0x00000004U
+#define SRS_INTERRUPT_CLR_I2C_SHIFT             2
+#define SRS_INTERRUPT_CLR_I2C_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_SPI_MASK              0x00000008U
+#define SRS_INTERRUPT_CLR_SPI_SHIFT             3
+#define SRS_INTERRUPT_CLR_SPI_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_APM_MASK              0x00000010U
+#define SRS_INTERRUPT_CLR_APM_SHIFT             4
+#define SRS_INTERRUPT_CLR_APM_SIGNED            0
+
+#define SRS_INTERRUPT_CLR_OS_IRQ_MASK           0x00001FE0U
+#define SRS_INTERRUPT_CLR_OS_IRQ_SHIFT          5
+#define SRS_INTERRUPT_CLR_OS_IRQ_SIGNED         0
+
+#define SRS_INTERRUPT_CLR_IRQ_TEST_MASK         0x40000000U
+#define SRS_INTERRUPT_CLR_IRQ_TEST_SHIFT        30
+#define SRS_INTERRUPT_CLR_IRQ_TEST_SIGNED       0
+
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_MASK     0x80000000U
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_SHIFT    31
+#define SRS_INTERRUPT_CLR_MASTER_CLEAR_SIGNED   0
+
+/*
+	Register INTERRUPT_TEST
+*/
+#define SRS_CORE_INTERRUPT_TEST             0x0224
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_MASK  0x00000001U
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_SHIFT 0
+#define SRS_INTERRUPT_TEST_INTERRUPT_TEST_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT_CLR
+*/
+#define SRS_CORE_INTERRUPT_TIMEOUT_CLR      0x0228
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_MASK 0x00000002U
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SHIFT 1
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_SIGNED 0
+
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_MASK 0x00000001U
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SHIFT 0
+#define SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_SIGNED 0
+
+/*
+	Register INTERRUPT_TIMEOUT
+*/
+#define SRS_CORE_INTERRUPT_TIMEOUT          0x022C
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_MASK 0xFFFFFFFFU
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SHIFT 0
+#define SRS_INTERRUPT_TIMEOUT_INTERRUPT_TIMEOUT_THRESHOLD_COUNTER_SIGNED 0
+
+#endif /* _OUT_DRV_H_ */
+
+/******************************************************************************
+ End of file (orion_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_apollo.c b/drivers/gpu/drm/img-rogue/apollo/pdp_apollo.c
new file mode 100644
index 000000000000..a55c6d83e13f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_apollo.c
@@ -0,0 +1,329 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/delay.h>
+
+#include "pdp_apollo.h"
+#include "pdp_common.h"
+#include "pdp_regs.h"
+#include "tcf_rgbpdp_regs.h"
+#include "tcf_pll.h"
+
+/* Map a register to the "pll-regs" region */
+#define PLL_REG(n) ((n) - TCF_PLL_PLL_PDP_CLK0)
+
+bool pdp_apollo_clocks_set(struct device *dev,
+			   void __iomem *pdp_reg, void __iomem *pll_reg,
+			   u32 clock_in_mhz,
+			   void __iomem *odn_core_reg,
+			   u32 hdisplay, u32 vdisplay)
+{
+	/*
+	 * Setup TCF_CR_PLL_PDP_CLK1TO5 based on the main clock speed
+	 * (clock 0 or 3)
+	 */
+	const u32 clock = (clock_in_mhz >= 50) ? 0 : 0x3;
+
+	/* Set phase 0, ratio 50:50 and frequency in MHz */
+	pll_wreg32(pll_reg, PLL_REG(TCF_PLL_PLL_PDP_CLK0), clock_in_mhz);
+
+	pll_wreg32(pll_reg, PLL_REG(TCF_PLL_PLL_PDP_CLK1TO5), clock);
+
+	/* Now initiate reprogramming of the PLLs */
+	pll_wreg32(pll_reg, PLL_REG(TCF_PLL_PLL_PDP_DRP_GO), 0x1);
+
+	udelay(1000);
+
+	pll_wreg32(pll_reg, PLL_REG(TCF_PLL_PLL_PDP_DRP_GO), 0x0);
+
+	return true;
+}
+
+void pdp_apollo_set_updates_enabled(struct device *dev, void __iomem *pdp_reg,
+				    bool enable)
+{
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set updates: %s\n", enable ? "enable" : "disable");
+#endif
+	/* nothing to do here */
+}
+
+void pdp_apollo_set_syncgen_enabled(struct device *dev, void __iomem *pdp_reg,
+				    bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set syncgen: %s\n", enable ? "enable" : "disable");
+#endif
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      SYNCACTIVE_SHIFT, SYNCACTIVE_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, value);
+}
+
+void pdp_apollo_set_powerdwn_enabled(struct device *dev, void __iomem *pdp_reg,
+				     bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set powerdwn: %s\n", enable ? "enable" : "disable");
+#endif
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      POWERDN_SHIFT, POWERDN_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, value);
+}
+
+void pdp_apollo_set_vblank_enabled(struct device *dev, void __iomem *pdp_reg,
+				   bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set vblank: %s\n", enable ? "enable" : "disable");
+#endif
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      INTEN_VBLNK0_SHIFT, INTEN_VBLNK0_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB, value);
+}
+
+bool pdp_apollo_check_and_clear_vblank(struct device *dev,
+				       void __iomem *pdp_reg)
+{
+	u32 value;
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_INTSTAT);
+
+	if (REG_VALUE_GET(value, INTS_VBLNK0_SHIFT, INTS_VBLNK0_MASK)) {
+		value = REG_VALUE_SET(0, 0x1,
+				      INTCLR_VBLNK0_SHIFT, INTCLR_VBLNK0_MASK);
+		pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_INTCLEAR, value);
+		return true;
+	}
+	return false;
+}
+
+void pdp_apollo_set_plane_enabled(struct device *dev, void __iomem *pdp_reg,
+				  u32 plane, bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set plane %u: %s\n",
+		 plane, enable ? "enable" : "disable");
+#endif
+
+	if (plane > 0) {
+		dev_err(dev, "Maximum of 1 plane is supported\n");
+		return;
+	}
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      STR1STREN_SHIFT, STR1STREN_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, value);
+}
+
+void pdp_apollo_reset_planes(struct device *dev, void __iomem *pdp_reg)
+{
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Reset planes\n");
+#endif
+
+	pdp_apollo_set_plane_enabled(dev, pdp_reg, 0, false);
+}
+
+void pdp_apollo_set_surface(struct device *dev, void __iomem *pdp_reg,
+			    u32 plane, u32 address,
+			    u32 posx, u32 posy,
+			    u32 width, u32 height, u32 stride,
+			    u32 format, u32 alpha, bool blend)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev,
+		 "Set surface: size=%dx%d stride=%d format=%d address=0x%x\n",
+		 width, height, stride, format, address);
+#endif
+
+	if (plane > 0) {
+		dev_err(dev, "Maximum of 1 plane is supported\n");
+		return;
+	}
+
+	/* Size & format */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1SURF);
+	value = REG_VALUE_SET(value, width - 1,
+			      STR1WIDTH_SHIFT, STR1WIDTH_MASK);
+	value = REG_VALUE_SET(value, height - 1,
+			      STR1HEIGHT_SHIFT, STR1HEIGHT_MASK);
+	value = REG_VALUE_SET(value, format,
+			      STR1PIXFMT_SHIFT, STR1PIXFMT_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1SURF, value);
+	/* Stride */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_PDP_STR1POSN);
+	value = REG_VALUE_SET(value,
+			      (stride >> DCPDP_STR1POSN_STRIDE_SHIFT) - 1,
+			      STR1STRIDE_SHIFT, STR1STRIDE_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_PDP_STR1POSN, value);
+	/* Disable interlaced output */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL);
+	value = REG_VALUE_SET(value, 0x0,
+			      STR1INTFIELD_SHIFT,
+			      STR1INTFIELD_MASK);
+	/* Frame buffer base address */
+	value = REG_VALUE_SET(value,
+			      address >> DCPDP_STR1ADDRCTRL_BASE_ADDR_SHIFT,
+			      STR1BASE_SHIFT, STR1BASE_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL, value);
+}
+
+void pdp_apollo_mode_set(struct device *dev, void __iomem *pdp_reg,
+			 u32 h_display, u32 v_display,
+			 u32 hbps, u32 ht, u32 has,
+			 u32 hlbs, u32 hfps, u32 hrbs,
+			 u32 vbps, u32 vt, u32 vas,
+			 u32 vtbs, u32 vfps, u32 vbbs,
+			 bool nhsync, bool nvsync)
+{
+	u32 value;
+
+	dev_info(dev, "Set mode: %dx%d\n", h_display, v_display);
+#ifdef PDP_VERBOSE
+	dev_info(dev, " ht: %d hbps %d has %d hlbs %d hfps %d hrbs %d\n",
+		 ht, hbps, has, hlbs, hfps, hrbs);
+	dev_info(dev, " vt: %d vbps %d vas %d vtbs %d vfps %d vbbs %d\n",
+		 vt, vbps, vas, vtbs, vfps, vbbs);
+#endif
+
+/*
+ * This gets set in dc_pdp_common.c, but hasn't been necessary here.
+ *
+ * if (pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL)
+ *     != 0x0000C010) {
+ * // Buffer request threshold
+ * pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL,
+ *            0x00001C10);
+ * }
+ */
+
+	/* Border colour */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_BORDCOL);
+	value = REG_VALUE_SET(value, 0x0, BORDCOL_SHIFT, BORDCOL_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_BORDCOL, value);
+
+	/* Update control */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_UPDCTRL);
+	value = REG_VALUE_SET(value, 0x0, UPDFIELD_SHIFT, UPDFIELD_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_UPDCTRL, value);
+
+	/* Set hsync timings */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC1);
+	value = REG_VALUE_SET(value, hbps, HBPS_SHIFT, HBPS_MASK);
+	value = REG_VALUE_SET(value, ht, HT_SHIFT, HT_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC1, value);
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC2);
+	value = REG_VALUE_SET(value, has, HAS_SHIFT, HAS_MASK);
+	value = REG_VALUE_SET(value, hlbs, HLBS_SHIFT, HLBS_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC2, value);
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC3);
+	value = REG_VALUE_SET(value, hfps, HFPS_SHIFT, HFPS_MASK);
+	value = REG_VALUE_SET(value, hrbs, HRBS_SHIFT, HRBS_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC3, value);
+
+	/* Set vsync timings */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC1);
+	value = REG_VALUE_SET(value, vbps, VBPS_SHIFT, VBPS_MASK);
+	value = REG_VALUE_SET(value, vt, VT_SHIFT, VT_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC1, value);
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC2);
+	value = REG_VALUE_SET(value, vas, VAS_SHIFT, VAS_MASK);
+	value = REG_VALUE_SET(value, vtbs, VTBS_SHIFT, VTBS_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC2, value);
+
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC3);
+	value = REG_VALUE_SET(value, vfps, VFPS_SHIFT, VFPS_MASK);
+	value = REG_VALUE_SET(value, vbbs, VBBS_SHIFT, VBBS_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC3, value);
+
+	/* Horizontal data enable */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HDECTRL);
+	value = REG_VALUE_SET(value, hlbs, HDES_SHIFT, HDES_MASK);
+	value = REG_VALUE_SET(value, hfps, HDEF_SHIFT, HDEF_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_HDECTRL, value);
+
+	/* Vertical data enable */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VDECTRL);
+	value = REG_VALUE_SET(value, vtbs, VDES_SHIFT, VDES_MASK);
+	value = REG_VALUE_SET(value, vfps, VDEF_SHIFT, VDEF_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VDECTRL, value);
+
+	/* Vertical event start and vertical fetch start */
+	value = pdp_rreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VEVENT);
+	value = REG_VALUE_SET(value, vbps, VFETCH_SHIFT, VFETCH_MASK);
+	value = REG_VALUE_SET(value, vfps, VEVENT_SHIFT, VEVENT_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_VEVENT, value);
+
+	/* Set up polarities of sync/blank */
+	value = REG_VALUE_SET(0, 0x1, BLNKPOL_SHIFT, BLNKPOL_MASK);
+
+/*
+ * Enable this if you want vblnk1. You also need to change to vblnk1
+ * in the interrupt handler.
+ *
+ * value = REG_VALUE_SET(value, 0x1, FIELDPOL_SHIFT, FIELDPOL_MASK);
+ */
+	if (nhsync)
+		value = REG_VALUE_SET(value, 0x1, HSPOL_SHIFT, HSPOL_MASK);
+	if (nvsync)
+		value = REG_VALUE_SET(value, 0x1, VSPOL_SHIFT, VSPOL_MASK);
+	pdp_wreg32(pdp_reg, TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL, value);
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_apollo.h b/drivers/gpu/drm/img-rogue/apollo/pdp_apollo.h
new file mode 100644
index 000000000000..fc7db08a041e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_apollo.h
@@ -0,0 +1,88 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_APOLLO_H__)
+#define __PDP_APOLLO_H__
+
+#include <linux/device.h>
+#include <linux/types.h>
+
+bool pdp_apollo_clocks_set(struct device *dev,
+			   void __iomem *pdp_reg, void __iomem *pll_reg,
+			   u32 clock_in_mhz,
+			   void __iomem *odn_core_reg,
+			   u32 hdisplay, u32 vdisplay);
+
+void pdp_apollo_set_updates_enabled(struct device *dev,	void __iomem *pdp_reg,
+				    bool enable);
+
+void pdp_apollo_set_syncgen_enabled(struct device *dev, void __iomem *pdp_reg,
+				    bool enable);
+
+void pdp_apollo_set_powerdwn_enabled(struct device *dev, void __iomem *pdp_reg,
+				     bool enable);
+
+void pdp_apollo_set_vblank_enabled(struct device *dev, void __iomem *pdp_reg,
+				   bool enable);
+
+bool pdp_apollo_check_and_clear_vblank(struct device *dev,
+				       void __iomem *pdp_reg);
+
+void pdp_apollo_set_plane_enabled(struct device *dev, void __iomem *pdp_reg,
+				  u32 plane, bool enable);
+
+void pdp_apollo_reset_planes(struct device *dev, void __iomem *pdp_reg);
+
+void pdp_apollo_set_surface(struct device *dev,	void __iomem *pdp_reg,
+			    u32 plane, u32 address,
+			    u32 posx, u32 posy,
+			    u32 width, u32 height, u32 stride,
+			    u32 format, u32 alpha, bool blend);
+
+void pdp_apollo_mode_set(struct device *dev, void __iomem *pdp_reg,
+			 u32 h_display, u32 v_display,
+			 u32 hbps, u32 ht, u32 has,
+			 u32 hlbs, u32 hfps, u32 hrbs,
+			 u32 vbps, u32 vt, u32 vas,
+			 u32 vtbs, u32 vfps, u32 vbbs,
+			 bool nhsync, bool nvsync);
+
+#endif /* __PDP_APOLLO_H__ */
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_common.h b/drivers/gpu/drm/img-rogue/apollo/pdp_common.h
new file mode 100644
index 000000000000..68e7130ecd9c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_common.h
@@ -0,0 +1,107 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_COMMON_H__)
+#define __PDP_COMMON_H__
+
+#include <linux/io.h>
+
+/*#define PDP_VERBOSE*/
+
+#define REG_VALUE_GET(v, s, m) \
+	(u32)(((v) & (m)) >> (s))
+#define REG_VALUE_SET(v, b, s, m) \
+	(u32)(((v) & (u32)~(m)) | (u32)(((b) << (s)) & (m)))
+/* Active low */
+#define REG_VALUE_LO(v, b, s, m) \
+	(u32)((v) & ~(u32)(((b) << (s)) & (m)))
+
+enum pdp_version {
+	PDP_VERSION_APOLLO,
+	PDP_VERSION_ODIN,
+	PDP_VERSION_PLATO,
+};
+
+enum pdp_odin_subversion {
+	PDP_ODIN_NONE = 0,
+	PDP_ODIN_ORION,
+};
+
+enum pdp_output_device {
+	PDP_OUTPUT_PDP1 = 1,
+	PDP_OUTPUT_PDP2,
+};
+
+/* Register R-W */
+static inline u32 core_rreg32(void __iomem *base, resource_size_t reg)
+{
+	return ioread32(base + reg);
+}
+
+static inline void core_wreg32(void __iomem *base, resource_size_t reg,
+			       u32 value)
+{
+	iowrite32(value, base + reg);
+}
+
+static inline u32 pdp_rreg32(void __iomem *base, resource_size_t reg)
+{
+	return ioread32(base + reg);
+}
+
+static inline void pdp_wreg32(void __iomem *base, resource_size_t reg,
+			      u32 value)
+{
+	iowrite32(value, base + reg);
+}
+
+static inline u32 pll_rreg32(void __iomem *base, resource_size_t reg)
+{
+	return ioread32(base + reg);
+}
+
+static inline void pll_wreg32(void __iomem *base, resource_size_t reg,
+			      u32 value)
+{
+	iowrite32(value, base + reg);
+}
+
+#endif /* __PDP_COMMON_H__ */
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_odin.c b/drivers/gpu/drm/img-rogue/apollo/pdp_odin.c
new file mode 100644
index 000000000000..e3a0d5c85e8a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_odin.c
@@ -0,0 +1,1231 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include <linux/kernel.h>
+
+#include "pdp_common.h"
+#include "pdp_odin.h"
+#include "odin_defs.h"
+#include "odin_regs.h"
+#include "orion_defs.h"
+#include "orion_regs.h"
+#include "pfim_defs.h"
+#include "pfim_regs.h"
+
+#define ODIN_PLL_REG(n)	((n) - ODN_PDP_P_CLK_OUT_DIVIDER_REG1)
+
+struct odin_displaymode {
+	int w;		/* display width */
+	int h;		/* display height */
+	int id;		/* pixel clock input divider */
+	int m;		/* pixel clock multiplier */
+	int od1;	/* pixel clock output divider */
+	int od2;	/* mem clock output divider */
+};
+
+struct pfim_property {
+	u32 tiles_per_line;
+	u32 tile_type;
+	u32 tile_xsize;
+	u32 tile_ysize;
+};
+
+/*
+ * For Odin, only the listed modes below are supported.
+ * 1080p id=5, m=37, od1=5, od2=5
+ * 720p id=5, m=37, od1=10, od2=5
+ * 1280x1024 id=1, m=14, od1=13, od2=8
+ * 1440x900 id=5, m=53, od1=10, od2=8
+ * 1280x960 id=3, m=40, od1=13, od2=9
+ * 1024x768 id=1, m=13, od1=20, od2=10
+ * 800x600 id=2, m=20, od1=25, od2=7
+ * 640x480 id=1, m=12, od1=48, od2=9
+ * ... where id is the PDP_P_CLK input divider,
+ * m is PDP_P_CLK multiplier regs 1 to 3
+ * od1 is PDP_P_clk output divider regs 1 to 3
+ * od2 is PDP_M_clk output divider regs 1 to 2
+ */
+static const struct odin_displaymode odin_modes[] = {
+	{.w = 1920, .h = 1080, .id = 5, .m = 37, .od1 = 5, .od2 = 5},
+	{.w = 1280, .h = 720, .id = 5, .m = 37, .od1 = 10, .od2 = 5},
+	{.w = 1280, .h = 1024, .id = 1, .m = 14, .od1 = 13, .od2 = 10},
+	{.w = 1440, .h = 900, .id = 5, .m = 53, .od1 = 10, .od2 = 8},
+	{.w = 1280, .h = 960, .id = 3, .m = 40, .od1 = 13, .od2 = 9},
+	{.w = 1024, .h = 768, .id = 1, .m = 13, .od1 = 20, .od2 = 10},
+	{.w = 800, .h = 600, .id = 2, .m = 20, .od1 = 25, .od2 = 7},
+	{.w = 640, .h = 480, .id = 1, .m = 12, .od1 = 48, .od2 = 9},
+	{.w = 0, .h = 0, .id = 0, .m = 0, .od1 = 0, .od2 = 0}
+};
+
+/*
+ * For Orion, only the listed modes below are supported.
+ * 1920x1080 mode is currently not supported.
+ */
+static const struct odin_displaymode orion_modes[] = {
+	{.w = 1280, .h = 720, .id = 5, .m = 37, .od1 = 10, .od2 = 7},
+	{.w = 1280, .h = 1024, .id = 1, .m = 12, .od1 = 11, .od2 = 10},
+	{.w = 1440, .h = 900, .id = 5, .m = 53, .od1 = 10, .od2 = 9},
+	{.w = 1280, .h = 960, .id = 5, .m = 51, .od1 = 10, .od2 = 9},
+	{.w = 1024, .h = 768, .id = 3, .m = 33, .od1 = 17, .od2 = 10},
+	{.w = 800, .h = 600, .id = 2, .m = 24, .od1 = 31, .od2 = 12},
+	{.w = 640, .h = 480, .id = 1, .m = 12, .od1 = 50, .od2 = 12},
+	{.w = 0, .h = 0, .id = 0, .m = 0, .od1 = 0, .od2 = 0}
+};
+
+static const struct pfim_property pfim_properties[] = {
+	[ODIN_PFIM_MOD_LINEAR]     = {0},
+	[ODIN_PFIM_FBCDC_8X8_V12]  = {.tiles_per_line = 8,
+				      .tile_type = ODN_PFIM_TILETYPE_8X8,
+				      .tile_xsize = 8,
+				      .tile_ysize = 8},
+	[ODIN_PFIM_FBCDC_16X4_V12] = {.tiles_per_line = 16,
+				      .tile_type = ODN_PFIM_TILETYPE_16X4,
+				      .tile_xsize = 16,
+				      .tile_ysize = 4},
+};
+
+static const u32 GRPH_SURF_OFFSET[] = {
+	ODN_PDP_GRPH1SURF_OFFSET,
+	ODN_PDP_GRPH2SURF_OFFSET,
+	ODN_PDP_VID1SURF_OFFSET,
+	ODN_PDP_GRPH4SURF_OFFSET
+};
+static const u32 GRPH_SURF_GRPH_PIXFMT_SHIFT[] = {
+	ODN_PDP_GRPH1SURF_GRPH1PIXFMT_SHIFT,
+	ODN_PDP_GRPH2SURF_GRPH2PIXFMT_SHIFT,
+	ODN_PDP_VID1SURF_VID1PIXFMT_SHIFT,
+	ODN_PDP_GRPH4SURF_GRPH4PIXFMT_SHIFT
+};
+static const u32 GRPH_SURF_GRPH_PIXFMT_MASK[] = {
+	ODN_PDP_GRPH1SURF_GRPH1PIXFMT_MASK,
+	ODN_PDP_GRPH2SURF_GRPH2PIXFMT_MASK,
+	ODN_PDP_VID1SURF_VID1PIXFMT_MASK,
+	ODN_PDP_GRPH4SURF_GRPH4PIXFMT_MASK
+};
+static const u32 GRPH_GALPHA_OFFSET[] = {
+	ODN_PDP_GRPH1GALPHA_OFFSET,
+	ODN_PDP_GRPH2GALPHA_OFFSET,
+	ODN_PDP_VID1GALPHA_OFFSET,
+	ODN_PDP_GRPH4GALPHA_OFFSET
+};
+static const u32 GRPH_GALPHA_GRPH_GALPHA_SHIFT[] = {
+	ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_SHIFT,
+	ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_SHIFT,
+	ODN_PDP_VID1GALPHA_VID1GALPHA_SHIFT,
+	ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_SHIFT
+};
+static const u32 GRPH_GALPHA_GRPH_GALPHA_MASK[] = {
+	ODN_PDP_GRPH1GALPHA_GRPH1GALPHA_MASK,
+	ODN_PDP_GRPH2GALPHA_GRPH2GALPHA_MASK,
+	ODN_PDP_VID1GALPHA_VID1GALPHA_MASK,
+	ODN_PDP_GRPH4GALPHA_GRPH4GALPHA_MASK
+};
+static const u32 GRPH_CTRL_OFFSET[] = {
+	ODN_PDP_GRPH1CTRL_OFFSET,
+	ODN_PDP_GRPH2CTRL_OFFSET,
+	ODN_PDP_VID1CTRL_OFFSET,
+	ODN_PDP_GRPH4CTRL_OFFSET,
+};
+static const u32 GRPH_CTRL_GRPH_BLEND_SHIFT[] = {
+	ODN_PDP_GRPH1CTRL_GRPH1BLEND_SHIFT,
+	ODN_PDP_GRPH2CTRL_GRPH2BLEND_SHIFT,
+	ODN_PDP_VID1CTRL_VID1BLEND_SHIFT,
+	ODN_PDP_GRPH4CTRL_GRPH4BLEND_SHIFT
+};
+static const u32 GRPH_CTRL_GRPH_BLEND_MASK[] = {
+	ODN_PDP_GRPH1CTRL_GRPH1BLEND_MASK,
+	ODN_PDP_GRPH2CTRL_GRPH2BLEND_MASK,
+	ODN_PDP_VID1CTRL_VID1BLEND_MASK,
+	ODN_PDP_GRPH4CTRL_GRPH4BLEND_MASK
+};
+static const u32 GRPH_CTRL_GRPH_BLENDPOS_SHIFT[] = {
+	ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_SHIFT,
+	ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_SHIFT,
+	ODN_PDP_VID1CTRL_VID1BLENDPOS_SHIFT,
+	ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_SHIFT
+};
+static const u32 GRPH_CTRL_GRPH_BLENDPOS_MASK[] = {
+	ODN_PDP_GRPH1CTRL_GRPH1BLENDPOS_MASK,
+	ODN_PDP_GRPH2CTRL_GRPH2BLENDPOS_MASK,
+	ODN_PDP_VID1CTRL_VID1BLENDPOS_MASK,
+	ODN_PDP_GRPH4CTRL_GRPH4BLENDPOS_MASK
+};
+static const u32 GRPH_CTRL_GRPH_STREN_SHIFT[] = {
+	ODN_PDP_GRPH1CTRL_GRPH1STREN_SHIFT,
+	ODN_PDP_GRPH2CTRL_GRPH2STREN_SHIFT,
+	ODN_PDP_VID1CTRL_VID1STREN_SHIFT,
+	ODN_PDP_GRPH4CTRL_GRPH4STREN_SHIFT
+};
+static const u32 GRPH_CTRL_GRPH_STREN_MASK[] = {
+	ODN_PDP_GRPH1CTRL_GRPH1STREN_MASK,
+	ODN_PDP_GRPH2CTRL_GRPH2STREN_MASK,
+	ODN_PDP_VID1CTRL_VID1STREN_MASK,
+	ODN_PDP_GRPH4CTRL_GRPH4STREN_MASK
+};
+static const u32 GRPH_POSN_OFFSET[] = {
+	ODN_PDP_GRPH1POSN_OFFSET,
+	ODN_PDP_GRPH2POSN_OFFSET,
+	ODN_PDP_VID1POSN_OFFSET,
+	ODN_PDP_GRPH4POSN_OFFSET
+};
+static const u32 GRPH_POSN_GRPH_XSTART_SHIFT[] = {
+	ODN_PDP_GRPH1POSN_GRPH1XSTART_SHIFT,
+	ODN_PDP_GRPH2POSN_GRPH2XSTART_SHIFT,
+	ODN_PDP_VID1POSN_VID1XSTART_SHIFT,
+	ODN_PDP_GRPH4POSN_GRPH4XSTART_SHIFT,
+};
+static const u32 GRPH_POSN_GRPH_XSTART_MASK[] = {
+	ODN_PDP_GRPH1POSN_GRPH1XSTART_MASK,
+	ODN_PDP_GRPH2POSN_GRPH2XSTART_MASK,
+	ODN_PDP_VID1POSN_VID1XSTART_MASK,
+	ODN_PDP_GRPH4POSN_GRPH4XSTART_MASK,
+};
+static const u32 GRPH_POSN_GRPH_YSTART_SHIFT[] = {
+	ODN_PDP_GRPH1POSN_GRPH1YSTART_SHIFT,
+	ODN_PDP_GRPH2POSN_GRPH2YSTART_SHIFT,
+	ODN_PDP_VID1POSN_VID1YSTART_SHIFT,
+	ODN_PDP_GRPH4POSN_GRPH4YSTART_SHIFT,
+};
+static const u32 GRPH_POSN_GRPH_YSTART_MASK[] = {
+	ODN_PDP_GRPH1POSN_GRPH1YSTART_MASK,
+	ODN_PDP_GRPH2POSN_GRPH2YSTART_MASK,
+	ODN_PDP_VID1POSN_VID1YSTART_MASK,
+	ODN_PDP_GRPH4POSN_GRPH4YSTART_MASK,
+};
+static const u32 GRPH_SIZE_OFFSET[] = {
+	ODN_PDP_GRPH1SIZE_OFFSET,
+	ODN_PDP_GRPH2SIZE_OFFSET,
+	ODN_PDP_VID1SIZE_OFFSET,
+	ODN_PDP_GRPH4SIZE_OFFSET,
+};
+static const u32 GRPH_SIZE_GRPH_WIDTH_SHIFT[] = {
+	ODN_PDP_GRPH1SIZE_GRPH1WIDTH_SHIFT,
+	ODN_PDP_GRPH2SIZE_GRPH2WIDTH_SHIFT,
+	ODN_PDP_VID1SIZE_VID1WIDTH_SHIFT,
+	ODN_PDP_GRPH4SIZE_GRPH4WIDTH_SHIFT
+};
+static const u32 GRPH_SIZE_GRPH_WIDTH_MASK[] = {
+	ODN_PDP_GRPH1SIZE_GRPH1WIDTH_MASK,
+	ODN_PDP_GRPH2SIZE_GRPH2WIDTH_MASK,
+	ODN_PDP_VID1SIZE_VID1WIDTH_MASK,
+	ODN_PDP_GRPH4SIZE_GRPH4WIDTH_MASK
+};
+static const u32 GRPH_SIZE_GRPH_HEIGHT_SHIFT[] = {
+	ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_SHIFT,
+	ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_SHIFT,
+	ODN_PDP_VID1SIZE_VID1HEIGHT_SHIFT,
+	ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_SHIFT
+};
+static const u32 GRPH_SIZE_GRPH_HEIGHT_MASK[] = {
+	ODN_PDP_GRPH1SIZE_GRPH1HEIGHT_MASK,
+	ODN_PDP_GRPH2SIZE_GRPH2HEIGHT_MASK,
+	ODN_PDP_VID1SIZE_VID1HEIGHT_MASK,
+	ODN_PDP_GRPH4SIZE_GRPH4HEIGHT_MASK
+};
+static const u32 GRPH_STRIDE_OFFSET[] = {
+	ODN_PDP_GRPH1STRIDE_OFFSET,
+	ODN_PDP_GRPH2STRIDE_OFFSET,
+	ODN_PDP_VID1STRIDE_OFFSET,
+	ODN_PDP_GRPH4STRIDE_OFFSET
+};
+static const u32 GRPH_STRIDE_GRPH_STRIDE_SHIFT[] = {
+	ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_SHIFT,
+	ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_SHIFT,
+	ODN_PDP_VID1STRIDE_VID1STRIDE_SHIFT,
+	ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_SHIFT
+};
+static const u32 GRPH_STRIDE_GRPH_STRIDE_MASK[] = {
+	ODN_PDP_GRPH1STRIDE_GRPH1STRIDE_MASK,
+	ODN_PDP_GRPH2STRIDE_GRPH2STRIDE_MASK,
+	ODN_PDP_VID1STRIDE_VID1STRIDE_MASK,
+	ODN_PDP_GRPH4STRIDE_GRPH4STRIDE_MASK
+};
+static const u32 GRPH_INTERLEAVE_CTRL_OFFSET[] = {
+	ODN_PDP_GRPH1INTERLEAVE_CTRL_OFFSET,
+	ODN_PDP_GRPH2INTERLEAVE_CTRL_OFFSET,
+	ODN_PDP_VID1INTERLEAVE_CTRL_OFFSET,
+	ODN_PDP_GRPH4INTERLEAVE_CTRL_OFFSET
+};
+static const u32 GRPH_INTERLEAVE_CTRL_GRPH_INTFIELD_SHIFT[] = {
+	ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SHIFT,
+	ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SHIFT,
+	ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SHIFT,
+	ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SHIFT
+};
+static const u32 GRPH_INTERLEAVE_CTRL_GRPH_INTFIELD_MASK[] = {
+	ODN_PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_MASK,
+	ODN_PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_MASK,
+	ODN_PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_MASK,
+	ODN_PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_MASK
+};
+static const u32 GRPH_BASEADDR_OFFSET[] = {
+	ODN_PDP_GRPH1BASEADDR_OFFSET,
+	ODN_PDP_GRPH2BASEADDR_OFFSET,
+	ODN_PDP_VID1BASEADDR_OFFSET,
+	ODN_PDP_GRPH4BASEADDR_OFFSET
+};
+
+static const u32 ODN_INTERNAL_RESETN_PDP_MASK[] = {
+	ODN_INTERNAL_RESETN_PDP1_MASK,
+	ODN_INTERNAL_RESETN_PDP2_MASK
+};
+
+static const u32 ODN_INTERNAL_RESETN_PDP_SHIFT[]  = {
+	ODN_INTERNAL_RESETN_PDP1_SHIFT,
+	ODN_INTERNAL_RESETN_PDP2_SHIFT
+};
+
+static void get_odin_clock_settings(u32 value, u32 *lo_time, u32 *hi_time,
+				u32 *no_count, u32 *edge)
+{
+	u32 lt, ht;
+
+	/* If the value is 1, High Time & Low Time are both set to 1
+	 * and the NOCOUNT bit is set to 1.
+	 */
+	if (value == 1) {
+		*lo_time = 1;
+		*hi_time = 1;
+
+		/* If od is an odd number then write 1 to NO_COUNT
+		 * otherwise write 0.
+		 */
+		*no_count = 1;
+
+		/* If m is and odd number then write 1 to EDGE bit of MR2
+		 * otherwise write 0.
+		 * If id is an odd number then write 1 to EDGE bit of ID
+		 * otherwise write 0.
+		 */
+		*edge = 0;
+		return;
+	}
+	*no_count = 0;
+
+	/* High Time & Low time is half the value listed for each PDP mode */
+	lt = value>>1;
+	ht = lt;
+
+	/* If the value is odd, Low Time is rounded up to nearest integer
+	 * and High Time is rounded down, and Edge is set to 1.
+	 */
+	if (value & 1) {
+		lt++;
+
+		/* If m is and odd number then write 1 to EDGE bit of MR2
+		 * otherwise write 0.
+		 * If id is an odd number then write 1 to EDGE bit of ID
+		 * otherwise write 0.
+		 */
+		*edge = 1;
+
+	} else {
+		*edge = 0;
+	}
+	*hi_time = ht;
+	*lo_time = lt;
+}
+
+static const struct odin_displaymode *get_odin_mode(int w, int h,
+						    enum pdp_odin_subversion pv)
+{
+	struct odin_displaymode *pdp_modes;
+	int n = 0;
+
+	if (pv == PDP_ODIN_ORION)
+		pdp_modes = (struct odin_displaymode *)orion_modes;
+	else
+		pdp_modes = (struct odin_displaymode *)odin_modes;
+
+	do {
+		if ((pdp_modes[n].w == w) && (pdp_modes[n].h == h))
+			return pdp_modes+n;
+
+	} while (pdp_modes[n++].w);
+
+	return NULL;
+}
+
+bool pdp_odin_clocks_set(struct device *dev,
+			 void __iomem *pdp_reg, void __iomem *pll_reg,
+			 u32 clock_freq, u32 dev_num,
+			 void __iomem *odn_core_reg,
+			 u32 hdisplay, u32 vdisplay,
+			 enum pdp_odin_subversion pdpsubv)
+{
+	u32 value;
+	const struct odin_displaymode *odispl;
+	u32 hi_time, lo_time, no_count, edge;
+	u32 core_id, core_rev;
+
+	core_id = pdp_rreg32(pdp_reg, ODN_PDP_CORE_ID_OFFSET);
+	dev_info(dev, "Odin-PDP CORE_ID  %08X\n", core_id);
+
+	core_rev = pdp_rreg32(odn_core_reg, ODN_PDP_CORE_REV_OFFSET);
+	dev_info(dev, "Odin-PDP CORE_REV %08X\n", core_rev);
+
+	odispl = get_odin_mode(hdisplay, vdisplay, pdpsubv);
+	if (!odispl) {
+		dev_err(dev, "Display mode not supported.\n");
+		return false;
+	}
+
+	/*
+	 * The PDP uses a Xilinx clock that requires read
+	 * modify write for all registers.
+	 * It is essential that only the specified bits are changed
+	 * because other bits are in use.
+	 * To change PDP clocks reset PDP & PDP mmcm (PLL) first,
+	 * then apply changes and then un-reset mmcm & PDP.
+	 * Warm reset will keep the changes.
+	 *    wr 0x000080 0x1f7 ; # reset pdp
+	 *    wr 0x000090 8 ; # reset pdp mmcm
+	 * then apply clock changes, then
+	 *    wr 0x000090 0x0 ; # un-reset pdp mmcm
+	 *    wr 0x000080 0x1ff ; # un-reset pdp
+	 */
+
+	/*
+	 * Hold Odin PDP in reset while changing the clock regs.
+	 * Set the PDP bit of ODN_CORE_INTERNAL_RESETN low to reset.
+	 * set bit 3 to 0 (active low)
+	 */
+	if (pdpsubv == PDP_ODIN_ORION) {
+		value = core_rreg32(odn_core_reg, SRS_CORE_SOFT_RESETN);
+		value = REG_VALUE_LO(value, 1, SRS_SOFT_RESETN_PDP_SHIFT,
+				     SRS_SOFT_RESETN_PDP_MASK);
+		core_wreg32(odn_core_reg, SRS_CORE_SOFT_RESETN, value);
+	} else {
+		value = core_rreg32(odn_core_reg, ODN_CORE_INTERNAL_RESETN);
+		value = REG_VALUE_LO(value, 1,
+				     ODN_INTERNAL_RESETN_PDP_SHIFT[dev_num],
+				     ODN_INTERNAL_RESETN_PDP_MASK[dev_num]);
+		core_wreg32(odn_core_reg, ODN_CORE_INTERNAL_RESETN, value);
+	}
+
+	/*
+	 * Hold the PDP MMCM in reset while changing the clock regs.
+	 * Set the PDP bit of ODN_CORE_CLK_GEN_RESET high to reset.
+	 */
+	value = core_rreg32(odn_core_reg, ODN_CORE_CLK_GEN_RESET);
+	value = REG_VALUE_SET(value, 0x1,
+			      ODN_CLK_GEN_RESET_PDP_MMCM_SHIFT,
+			      ODN_CLK_GEN_RESET_PDP_MMCM_MASK);
+	core_wreg32(odn_core_reg, ODN_CORE_CLK_GEN_RESET, value);
+
+	/* Pixel clock Input divider */
+	get_odin_clock_settings(odispl->id, &lo_time, &hi_time,
+				&no_count, &edge);
+
+	value = pll_rreg32(pll_reg,
+			   ODIN_PLL_REG(ODN_PDP_P_CLK_IN_DIVIDER_REG));
+	value = REG_VALUE_SET(value, lo_time,
+			      ODN_PDP_PCLK_IDIV_LO_TIME_SHIFT,
+			      ODN_PDP_PCLK_IDIV_LO_TIME_MASK);
+	value = REG_VALUE_SET(value, hi_time,
+			      ODN_PDP_PCLK_IDIV_HI_TIME_SHIFT,
+			      ODN_PDP_PCLK_IDIV_HI_TIME_MASK);
+	value = REG_VALUE_SET(value, no_count,
+			      ODN_PDP_PCLK_IDIV_NOCOUNT_SHIFT,
+			      ODN_PDP_PCLK_IDIV_NOCOUNT_MASK);
+	value = REG_VALUE_SET(value, edge,
+			      ODN_PDP_PCLK_IDIV_EDGE_SHIFT,
+			      ODN_PDP_PCLK_IDIV_EDGE_MASK);
+	pll_wreg32(pll_reg, ODIN_PLL_REG(ODN_PDP_P_CLK_IN_DIVIDER_REG),
+		   value);
+
+	/* Pixel clock Output divider */
+	get_odin_clock_settings(odispl->od1, &lo_time, &hi_time,
+				&no_count, &edge);
+
+	/* Pixel clock Output divider reg1 */
+	value = pll_rreg32(pll_reg,
+			   ODIN_PLL_REG(ODN_PDP_P_CLK_OUT_DIVIDER_REG1));
+	value = REG_VALUE_SET(value, lo_time,
+			      ODN_PDP_PCLK_ODIV1_LO_TIME_SHIFT,
+			      ODN_PDP_PCLK_ODIV1_LO_TIME_MASK);
+	value = REG_VALUE_SET(value, hi_time,
+			      ODN_PDP_PCLK_ODIV1_HI_TIME_SHIFT,
+			      ODN_PDP_PCLK_ODIV1_HI_TIME_MASK);
+	pll_wreg32(pll_reg, ODIN_PLL_REG(ODN_PDP_P_CLK_OUT_DIVIDER_REG1),
+		   value);
+
+	/* Pixel clock Output divider reg2 */
+	value = pll_rreg32(pll_reg,
+			   ODIN_PLL_REG(ODN_PDP_P_CLK_OUT_DIVIDER_REG2));
+	value = REG_VALUE_SET(value, no_count,
+			      ODN_PDP_PCLK_ODIV2_NOCOUNT_SHIFT,
+			      ODN_PDP_PCLK_ODIV2_NOCOUNT_MASK);
+	value = REG_VALUE_SET(value, edge,
+			      ODN_PDP_PCLK_ODIV2_EDGE_SHIFT,
+			      ODN_PDP_PCLK_ODIV2_EDGE_MASK);
+	if (pdpsubv == PDP_ODIN_ORION) {
+		/*
+		 * Fractional divide for PLL registers currently does not work
+		 * on Sirius, as duly mentioned on the TRM. However, owing to
+		 * what most likely is a design flaw in the RTL, the
+		 * following register and a later one have their fractional
+		 * divide fields set to values other than 0 by default,
+		 * unlike on Odin. This prevents the PDP device from working
+		 * on Orion
+		 */
+		value = REG_VALUE_LO(value, 0x1F, SRS_PDP_PCLK_ODIV2_FRAC_SHIFT,
+				     SRS_PDP_PCLK_ODIV2_FRAC_MASK);
+	}
+	pll_wreg32(pll_reg, ODIN_PLL_REG(ODN_PDP_P_CLK_OUT_DIVIDER_REG2),
+		   value);
+
+	/* Pixel clock Multiplier */
+	get_odin_clock_settings(odispl->m, &lo_time, &hi_time,
+				&no_count, &edge);
+
+	/* Pixel clock Multiplier reg1 */
+	value = pll_rreg32(pll_reg,
+			   ODIN_PLL_REG(ODN_PDP_P_CLK_MULTIPLIER_REG1));
+	value = REG_VALUE_SET(value, lo_time,
+			      ODN_PDP_PCLK_MUL1_LO_TIME_SHIFT,
+			      ODN_PDP_PCLK_MUL1_LO_TIME_MASK);
+	value = REG_VALUE_SET(value, hi_time,
+			      ODN_PDP_PCLK_MUL1_HI_TIME_SHIFT,
+			      ODN_PDP_PCLK_MUL1_HI_TIME_MASK);
+	pll_wreg32(pll_reg, ODIN_PLL_REG(ODN_PDP_P_CLK_MULTIPLIER_REG1),
+		   value);
+
+	/* Pixel clock Multiplier reg2 */
+	value = pll_rreg32(pll_reg,
+			   ODIN_PLL_REG(ODN_PDP_P_CLK_MULTIPLIER_REG2));
+	value = REG_VALUE_SET(value, no_count,
+			      ODN_PDP_PCLK_MUL2_NOCOUNT_SHIFT,
+			      ODN_PDP_PCLK_MUL2_NOCOUNT_MASK);
+	value = REG_VALUE_SET(value, edge,
+			      ODN_PDP_PCLK_MUL2_EDGE_SHIFT,
+			      ODN_PDP_PCLK_MUL2_EDGE_MASK);
+	if (pdpsubv == PDP_ODIN_ORION) {
+		/* Zero out fractional divide fields */
+		value = REG_VALUE_LO(value, 0x1F, SRS_PDP_PCLK_MUL2_FRAC_SHIFT,
+				     SRS_PDP_PCLK_MUL2_FRAC_MASK);
+	}
+	pll_wreg32(pll_reg, ODIN_PLL_REG(ODN_PDP_P_CLK_MULTIPLIER_REG2),
+		   value);
+
+	/* Mem clock Output divider */
+	get_odin_clock_settings(odispl->od2, &lo_time, &hi_time,
+				&no_count, &edge);
+
+	/* Mem clock Output divider reg1 */
+	value = pll_rreg32(pll_reg,
+			   ODIN_PLL_REG(ODN_PDP_M_CLK_OUT_DIVIDER_REG1));
+	value = REG_VALUE_SET(value, lo_time,
+			      ODN_PDP_MCLK_ODIV1_LO_TIME_SHIFT,
+			      ODN_PDP_MCLK_ODIV1_LO_TIME_MASK);
+	value = REG_VALUE_SET(value, hi_time,
+			      ODN_PDP_MCLK_ODIV1_HI_TIME_SHIFT,
+			      ODN_PDP_MCLK_ODIV1_HI_TIME_MASK);
+	pll_wreg32(pll_reg, ODIN_PLL_REG(ODN_PDP_M_CLK_OUT_DIVIDER_REG1),
+		   value);
+
+	/* Mem clock Output divider reg2 */
+	value = pll_rreg32(pll_reg,
+			   ODIN_PLL_REG(ODN_PDP_M_CLK_OUT_DIVIDER_REG2));
+	value = REG_VALUE_SET(value, no_count,
+			      ODN_PDP_MCLK_ODIV2_NOCOUNT_SHIFT,
+			      ODN_PDP_MCLK_ODIV2_NOCOUNT_MASK);
+	value = REG_VALUE_SET(value, edge,
+			      ODN_PDP_MCLK_ODIV2_EDGE_SHIFT,
+			      ODN_PDP_MCLK_ODIV2_EDGE_MASK);
+	pll_wreg32(pll_reg, ODIN_PLL_REG(ODN_PDP_M_CLK_OUT_DIVIDER_REG2),
+		   value);
+
+	/*
+	 * Take the PDP MMCM out of reset.
+	 * Set the PDP bit of ODN_CORE_CLK_GEN_RESET to 0.
+	 */
+	value = core_rreg32(odn_core_reg, ODN_CORE_CLK_GEN_RESET);
+	value = REG_VALUE_LO(value, 1, ODN_CLK_GEN_RESET_PDP_MMCM_SHIFT,
+			     ODN_CLK_GEN_RESET_PDP_MMCM_MASK);
+	core_wreg32(odn_core_reg, ODN_CORE_CLK_GEN_RESET, value);
+
+	/*
+	 * Wait until MMCM_LOCK_STATUS_PDPP bit is '1' in register
+	 * MMCM_LOCK_STATUS. Issue an error if this does not
+	 * go to '1' within 500ms.
+	 */
+	{
+		int count;
+		bool locked = false;
+
+		for (count = 0; count < 10; count++) {
+			value = core_rreg32(odn_core_reg,
+					    ODN_CORE_MMCM_LOCK_STATUS);
+			if (value & ODN_MMCM_LOCK_STATUS_PDPP) {
+				locked = true;
+				break;
+			}
+			msleep(50);
+		}
+
+		if (!locked) {
+			dev_err(dev, "The MMCM pll did not lock\n");
+			return false;
+		}
+	}
+
+	/*
+	 * Take Odin-PDP out of reset:
+	 * Set the PDP bit of ODN_CORE_INTERNAL_RESETN to 1.
+	 */
+	if (pdpsubv == PDP_ODIN_ORION) {
+		value = core_rreg32(odn_core_reg, SRS_CORE_SOFT_RESETN);
+		value = REG_VALUE_SET(value, 1, SRS_SOFT_RESETN_PDP_SHIFT,
+				     SRS_SOFT_RESETN_PDP_MASK);
+		core_wreg32(odn_core_reg, SRS_CORE_SOFT_RESETN, value);
+	} else {
+		value = core_rreg32(odn_core_reg, ODN_CORE_INTERNAL_RESETN);
+		value = REG_VALUE_SET(value, 1,
+				      ODN_INTERNAL_RESETN_PDP_SHIFT[dev_num],
+				      ODN_INTERNAL_RESETN_PDP_MASK[dev_num]);
+		core_wreg32(odn_core_reg, ODN_CORE_INTERNAL_RESETN, value);
+	}
+
+	return true;
+}
+
+void pdp_odin_set_updates_enabled(struct device *dev, void __iomem *pdp_reg,
+				  bool enable)
+{
+	u32 value = enable ?
+		(1 << ODN_PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SHIFT |
+		 1 << ODN_PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SHIFT) :
+		0x0;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set updates: %s\n", enable ? "enable" : "disable");
+#endif
+
+	pdp_wreg32(pdp_reg, ODN_PDP_REGISTER_UPDATE_CTRL_OFFSET, value);
+}
+
+void pdp_odin_set_syncgen_enabled(struct device *dev, void __iomem *pdp_reg,
+				  bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set syncgen: %s\n", enable ? "enable" : "disable");
+#endif
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_SYNCCTRL_OFFSET);
+
+	value = REG_VALUE_SET(value,
+			      enable ? ODN_SYNC_GEN_ENABLE : ODN_SYNC_GEN_DISABLE,
+			      ODN_PDP_SYNCCTRL_SYNCACTIVE_SHIFT,
+			      ODN_PDP_SYNCCTRL_SYNCACTIVE_MASK);
+
+	/* Invert the pixel clock */
+	value = REG_VALUE_SET(value, ODN_PIXEL_CLOCK_INVERTED,
+			      ODN_PDP_SYNCCTRL_CLKPOL_SHIFT,
+			      ODN_PDP_SYNCCTRL_CLKPOL_MASK);
+
+	/* Set the Horizontal Sync Polarity to active high */
+	value = REG_VALUE_LO(value, ODN_HSYNC_POLARITY_ACTIVE_HIGH,
+			     ODN_PDP_SYNCCTRL_HSPOL_SHIFT,
+			     ODN_PDP_SYNCCTRL_HSPOL_MASK);
+
+	pdp_wreg32(pdp_reg, ODN_PDP_SYNCCTRL_OFFSET, value);
+
+	/* Check for underruns when the sync generator
+	 * is being turned off.
+	 */
+	if (!enable) {
+		value = pdp_rreg32(pdp_reg, ODN_PDP_INTSTAT_OFFSET);
+		value &= ODN_PDP_INTSTAT_ALL_OURUN_MASK;
+
+		if (value) {
+			dev_warn(dev, "underruns detected. status=0x%08X\n",
+				 value);
+		} else {
+			dev_info(dev, "no underruns detected\n");
+		}
+	}
+}
+
+void pdp_odin_set_powerdwn_enabled(struct device *dev, void __iomem *pdp_reg,
+				   bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set powerdwn: %s\n", enable ? "enable" : "disable");
+#endif
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_SYNCCTRL_OFFSET);
+
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      ODN_PDP_SYNCCTRL_POWERDN_SHIFT,
+			      ODN_PDP_SYNCCTRL_POWERDN_MASK);
+
+	pdp_wreg32(pdp_reg, ODN_PDP_SYNCCTRL_OFFSET, value);
+}
+
+void pdp_odin_set_vblank_enabled(struct device *dev, void __iomem *pdp_reg,
+				 bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set vblank: %s\n", enable ? "enable" : "disable");
+#endif
+
+	pdp_wreg32(pdp_reg, ODN_PDP_INTCLR_OFFSET, ODN_PDP_INTCLR_ALL);
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_INTENAB_OFFSET);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      ODN_PDP_INTENAB_INTEN_VBLNK0_SHIFT,
+			      ODN_PDP_INTENAB_INTEN_VBLNK0_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_INTENAB_OFFSET, value);
+}
+
+bool pdp_odin_check_and_clear_vblank(struct device *dev,
+				     void __iomem *pdp_reg)
+{
+	u32 value;
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_INTSTAT_OFFSET);
+
+	if (REG_VALUE_GET(value,
+			  ODN_PDP_INTSTAT_INTS_VBLNK0_SHIFT,
+			  ODN_PDP_INTSTAT_INTS_VBLNK0_MASK)) {
+		pdp_wreg32(pdp_reg, ODN_PDP_INTCLR_OFFSET,
+			   (1 << ODN_PDP_INTCLR_INTCLR_VBLNK0_SHIFT));
+
+		return true;
+	}
+	return false;
+}
+
+void pdp_odin_set_plane_enabled(struct device *dev, void __iomem *pdp_reg,
+				u32 plane, bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set plane %u: %s\n",
+		 plane, enable ? "enable" : "disable");
+#endif
+
+	if (plane > 3) {
+		dev_err(dev, "Maximum of 4 planes are supported\n");
+		return;
+	}
+
+	value = pdp_rreg32(pdp_reg, GRPH_CTRL_OFFSET[plane]);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      GRPH_CTRL_GRPH_STREN_SHIFT[plane],
+			      GRPH_CTRL_GRPH_STREN_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_CTRL_OFFSET[plane], value);
+}
+
+void pdp_odin_reset_planes(struct device *dev, void __iomem *pdp_reg)
+{
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Reset planes\n");
+#endif
+
+	pdp_wreg32(pdp_reg, GRPH_CTRL_OFFSET[0], 0x00000000);
+	pdp_wreg32(pdp_reg, GRPH_CTRL_OFFSET[1], 0x01000000);
+	pdp_wreg32(pdp_reg, GRPH_CTRL_OFFSET[2], 0x02000000);
+	pdp_wreg32(pdp_reg, GRPH_CTRL_OFFSET[3], 0x03000000);
+}
+
+static unsigned int pfim_pixel_format(u32 pdp_format)
+{
+	u32 pfim_pixformat;
+
+	switch (pdp_format) {
+	case ODN_PDP_SURF_PIXFMT_ARGB8888:
+		pfim_pixformat = ODN_PFIM_PIXFMT_ARGB8888;
+		break;
+	case ODN_PDP_SURF_PIXFMT_RGB565:
+		pfim_pixformat = ODN_PFIM_PIXFMT_RGB565;
+		break;
+	default:
+		WARN(true, "Unknown Odin pixel format: %u defaulting to ARGB8888\n",
+		     pdp_format);
+		pfim_pixformat = ODN_PFIM_PIXFMT_ARGB8888;
+	}
+
+	return pfim_pixformat;
+}
+
+static unsigned int pfim_tiles_line(u32 width,
+				    u32 pfim_format,
+				    u32 fbc_mode)
+{
+	u32 bpp;
+	u32 tpl;
+
+	switch (pfim_format) {
+	case ODN_PFIM_PIXFMT_ARGB8888:
+		bpp = 32;
+		break;
+	case ODN_PFIM_PIXFMT_RGB565:
+		bpp = 16;
+		break;
+	default:
+		WARN(true, "Unknown PFIM pixel format: %u, defaulting to 32 bpp\n",
+		     pfim_format);
+		bpp = 32;
+	}
+
+	if (fbc_mode < ODIN_PFIM_FBCDC_MAX) {
+		tpl = pfim_properties[fbc_mode].tiles_per_line;
+	} else {
+		WARN(true, "Unknown FBC compression format: %u, defaulting to 8X8_V12\n",
+		     fbc_mode);
+		tpl = pfim_properties[ODIN_PFIM_FBCDC_8X8_V12].tiles_per_line;
+	}
+
+	return ((width/tpl) / (32/bpp));
+}
+
+static void pfim_modeset(void __iomem *pfim_reg)
+{
+	u32 value;
+
+	/*
+	 * Odin PDP can address up to 32 bits of PCI BAR4,
+	 * so this register is not necessary
+	 */
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB, 0x00);
+
+	/*
+	 * Following registers are only used with YUV buffers,
+	 * which we currently do not support
+	 */
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_UV_BASE_ADDR_LSB, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_UV_BASE_ADDR_MSB, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_PDP_Y_BASE_ADDR, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_PDP_UV_BASE_ADDR, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CR_Y_VAL0, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CR_UV_VAL0, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CR_Y_VAL1, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CR_UV_VAL1, 0x00);
+
+	/*
+	 * PFIM tags are used for distinguishing between Y and UV plane
+	 * request when that is the kind of format we use. Thus, any
+	 * random value will do, as explained in the TRM
+	 */
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_REQ_CONTEXT, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_REQ_TAG, PFIM_RND_TAG);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_REQ_SB_TAG, 0x00);
+
+	/* Default tile value if tile is found to be corrupted */
+	value = REG_VALUE_SET(0, 0x01,
+			      CR_PFIM_FBDC_FILTER_ENABLE_SHIFT,
+			      CR_PFIM_FBDC_FILTER_ENABLE_MASK);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_FILTER_ENABLE, value);
+
+	/* Recommended values for corrupt tile substitution */
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CR_CH0123_VAL0, 0x00);
+	value = REG_VALUE_SET(0, 0x01000000,
+			      CR_PFIM_FBDC_CR_CH0123_VAL1_SHIFT,
+			      CR_PFIM_FBDC_CR_CH0123_VAL1_MASK);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CR_CH0123_VAL1, value);
+
+	/* Only used when requesting a clear tile */
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CLEAR_COLOUR_LSB, 0x00);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_CLEAR_COLOUR_MSB, 0x00);
+
+	/* Current PDP revision does not support lossy formats */
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_REQ_LOSSY, 0x00);
+
+	/* Force invalidation of FBC headers at beginning of render */
+	value = REG_VALUE_SET(0, 0x01,
+			      CR_PFIM_FBDC_HDR_INVAL_REQ_SHIFT,
+			      CR_PFIM_FBDC_HDR_INVAL_REQ_MASK);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_HDR_INVAL_REQ, value);
+}
+
+static unsigned int pfim_num_tiles(struct device *dev, u32 width, u32 height,
+				   u32 pfim_format, u32 fbc_mode)
+{
+	u32 phys_width, phys_height;
+	u32 walign, halign;
+	u32 tile_mult;
+	u32 num_tiles;
+	u32 bpp;
+
+	switch (pfim_format) {
+	case ODN_PFIM_PIXFMT_ARGB8888:
+		bpp = 32;
+		tile_mult = 4;
+		break;
+	case ODN_PFIM_PIXFMT_RGB565:
+		bpp = 16;
+		tile_mult = 2;
+		break;
+	default:
+		dev_warn(dev, "WARNING: Wrong PFIM pixel format: %d\n",
+			 pfim_format);
+		return 0;
+	}
+
+	switch (fbc_mode) {
+	case ODIN_PFIM_FBCDC_8X8_V12:
+		switch (bpp) {
+		case 16:	/* 16x8 */
+			walign = 16;
+			break;
+		case 32:	/* 8x8 */
+			walign = 8;
+			break;
+		default:
+			dev_warn(dev, "WARNING: Wrong bit depth: %d\n",
+				 bpp);
+			return 0;
+		}
+		halign = 8;
+		break;
+	case ODIN_PFIM_FBCDC_16X4_V12:
+		switch (bpp) {
+		case 16:	/* 32x4 */
+			walign = 32;
+			break;
+		case 32:	/* 16x4 */
+			walign = 16;
+			break;
+		default:
+			dev_warn(dev, "WARNING: Wrong bit depth: %d\n",
+				 bpp);
+			return 0;
+		}
+		halign = 4;
+		break;
+	default:
+		dev_warn(dev, "WARNING: Wrong FBC compression format: %d\n",
+			 fbc_mode);
+		return 0;
+	}
+
+	phys_width = ALIGN(width, walign);
+	phys_height = ALIGN(height, halign);
+	num_tiles = phys_width / pfim_properties[fbc_mode].tile_xsize;
+	num_tiles *= phys_height / pfim_properties[fbc_mode].tile_ysize;
+	num_tiles *= tile_mult;
+	num_tiles /= 4;
+
+	return num_tiles ? num_tiles : 1;
+}
+
+static void pfim_set_surface(struct device *dev,
+			     void __iomem *pfim_reg,
+			     u32 width,
+			     u32 height,
+			     u32 pdp_format,
+			     u32 fbc_mode)
+{
+	u32 pfim_pixformat = pfim_pixel_format(pdp_format);
+	u32 tiles_line = pfim_tiles_line(width, pfim_pixformat, fbc_mode);
+	u32 tile_type = pfim_properties[fbc_mode].tile_type;
+	u32 num_tiles = pfim_num_tiles(dev, width, height,
+				       pfim_pixformat, fbc_mode);
+
+	pdp_wreg32(pfim_reg, CR_PFIM_NUM_TILES, num_tiles);
+	pdp_wreg32(pfim_reg, CR_PFIM_TILES_PER_LINE, tiles_line);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_PIX_FORMAT, pfim_pixformat);
+	pdp_wreg32(pfim_reg, CR_PFIM_FBDC_TILE_TYPE, tile_type);
+}
+
+void pdp_odin_set_surface(struct device *dev, void __iomem *pdp_reg,
+			  u32 plane, u32 address, u32 offset,
+			  u32 posx, u32 posy,
+			  u32 width, u32 height, u32 stride,
+			  u32 format, u32 alpha, bool blend,
+			  void __iomem *pfim_reg, u32 fbcm)
+{
+	/*
+	 * Use a blender based on the plane number (this defines the Z
+	 * ordering)
+	 */
+	static const int GRPH_BLEND_POS[] = { 0x0, 0x1, 0x2, 0x3 };
+	u32 blend_mode;
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev,
+		 "Set surface: plane=%d pos=%d:%d size=%dx%d stride=%d format=%d alpha=%d address=0x%x\n",
+		 plane, posx, posy, width, height, stride,
+		 format, alpha, address);
+#endif
+
+	if (plane > 3) {
+		dev_err(dev, "Maximum of 4 planes are supported\n");
+		return;
+	}
+
+	if (address & 0xf)
+		dev_warn(dev, "The frame buffer address is not aligned\n");
+
+	if (fbcm && pfim_reg) {
+		pfim_set_surface(dev, pfim_reg,
+				 width, height,
+				 format, fbcm);
+		pdp_wreg32(pfim_reg, CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB,
+			   (address + offset) >> 6);
+	} else
+		pdp_wreg32(pdp_reg, GRPH_BASEADDR_OFFSET[plane], address);
+
+	/* Pos */
+	value = REG_VALUE_SET(0x0, posx,
+			      GRPH_POSN_GRPH_XSTART_SHIFT[plane],
+			      GRPH_POSN_GRPH_XSTART_MASK[plane]);
+	value = REG_VALUE_SET(value, posy,
+			      GRPH_POSN_GRPH_YSTART_SHIFT[plane],
+			      GRPH_POSN_GRPH_YSTART_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_POSN_OFFSET[plane], value);
+
+	/* Size */
+	value = REG_VALUE_SET(0x0, width - 1,
+			      GRPH_SIZE_GRPH_WIDTH_SHIFT[plane],
+			      GRPH_SIZE_GRPH_WIDTH_MASK[plane]);
+	value = REG_VALUE_SET(value, height - 1,
+			      GRPH_SIZE_GRPH_HEIGHT_SHIFT[plane],
+			      GRPH_SIZE_GRPH_HEIGHT_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_SIZE_OFFSET[plane], value);
+
+	/* Stride */
+	value = REG_VALUE_SET(0x0, (stride >> 4) - 1,
+			      GRPH_STRIDE_GRPH_STRIDE_SHIFT[plane],
+			      GRPH_STRIDE_GRPH_STRIDE_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_STRIDE_OFFSET[plane], value);
+
+	/* Interlace mode: progressive */
+	value = REG_VALUE_SET(0x0, ODN_INTERLACE_DISABLE,
+			      GRPH_INTERLEAVE_CTRL_GRPH_INTFIELD_SHIFT[plane],
+			      GRPH_INTERLEAVE_CTRL_GRPH_INTFIELD_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_INTERLEAVE_CTRL_OFFSET[plane], value);
+
+	/* Format */
+	value = REG_VALUE_SET(0x0, format,
+			      GRPH_SURF_GRPH_PIXFMT_SHIFT[plane],
+			      GRPH_SURF_GRPH_PIXFMT_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_SURF_OFFSET[plane], value);
+
+	/* Global alpha (0...1023) */
+	value = REG_VALUE_SET(0x0, ((1024 * 256) / 255 * alpha) / 256,
+			      GRPH_GALPHA_GRPH_GALPHA_SHIFT[plane],
+			      GRPH_GALPHA_GRPH_GALPHA_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_GALPHA_OFFSET[plane], value);
+	value = pdp_rreg32(pdp_reg, GRPH_CTRL_OFFSET[plane]);
+
+	/* Blend mode */
+	if (blend) {
+		if (alpha != 255)
+			blend_mode = 0x2; /* 0b10 = global alpha blending */
+		else
+			blend_mode = 0x3; /* 0b11 = pixel alpha blending */
+	} else {
+		blend_mode = 0x0; /* 0b00 = no blending */
+	}
+	value = REG_VALUE_SET(value, blend_mode,
+			      GRPH_CTRL_GRPH_BLEND_SHIFT[plane],
+			      GRPH_CTRL_GRPH_BLEND_MASK[plane]);
+
+	/* Blend position */
+	value = REG_VALUE_SET(value, GRPH_BLEND_POS[plane],
+			      GRPH_CTRL_GRPH_BLENDPOS_SHIFT[plane],
+			      GRPH_CTRL_GRPH_BLENDPOS_MASK[plane]);
+	pdp_wreg32(pdp_reg, GRPH_CTRL_OFFSET[plane], value);
+}
+
+void pdp_odin_mode_set(struct device *dev, void __iomem *pdp_reg,
+		       u32 h_display, u32 v_display,
+		       u32 hbps, u32 ht, u32 has,
+		       u32 hlbs, u32 hfps, u32 hrbs,
+		       u32 vbps, u32 vt, u32 vas,
+		       u32 vtbs, u32 vfps, u32 vbbs,
+		       bool nhsync, bool nvsync,
+		       void __iomem *pfim_reg)
+{
+	u32 value;
+
+	dev_info(dev, "Set mode: %dx%d\n", h_display, v_display);
+#ifdef PDP_VERBOSE
+	dev_info(dev, " ht: %d hbps %d has %d hlbs %d hfps %d hrbs %d\n",
+		 ht, hbps, has, hlbs, hfps, hrbs);
+	dev_info(dev, " vt: %d vbps %d vas %d vtbs %d vfps %d vbbs %d\n",
+		 vt, vbps, vas, vtbs, vfps, vbbs);
+#endif
+
+	/* Border colour: 10bits per channel */
+	pdp_wreg32(pdp_reg, ODN_PDP_BORDCOL_R_OFFSET, 0x0);
+	pdp_wreg32(pdp_reg, ODN_PDP_BORDCOL_GB_OFFSET, 0x0);
+
+	/* Background: 10bits per channel */
+	value = pdp_rreg32(pdp_reg, ODN_PDP_BGNDCOL_AR_OFFSET);
+	value = REG_VALUE_SET(value, 0x3ff,
+			      ODN_PDP_BGNDCOL_AR_BGNDCOL_A_SHIFT,
+			      ODN_PDP_BGNDCOL_AR_BGNDCOL_A_MASK);
+	value = REG_VALUE_SET(value, 0x0,
+			      ODN_PDP_BGNDCOL_AR_BGNDCOL_R_SHIFT,
+			      ODN_PDP_BGNDCOL_AR_BGNDCOL_R_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_BGNDCOL_AR_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_BGNDCOL_GB_OFFSET);
+	value = REG_VALUE_SET(value, 0x0,
+			      ODN_PDP_BGNDCOL_GB_BGNDCOL_G_SHIFT,
+			      ODN_PDP_BGNDCOL_GB_BGNDCOL_G_MASK);
+	value = REG_VALUE_SET(value, 0x0,
+			      ODN_PDP_BGNDCOL_GB_BGNDCOL_B_SHIFT,
+			      ODN_PDP_BGNDCOL_GB_BGNDCOL_B_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_BGNDCOL_GB_OFFSET, value);
+	pdp_wreg32(pdp_reg, ODN_PDP_BORDCOL_GB_OFFSET, 0x0);
+
+	/* Update control */
+	value = pdp_rreg32(pdp_reg, ODN_PDP_UPDCTRL_OFFSET);
+	value = REG_VALUE_SET(value, 0x0,
+			      ODN_PDP_UPDCTRL_UPDFIELD_SHIFT,
+			      ODN_PDP_UPDCTRL_UPDFIELD_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_UPDCTRL_OFFSET, value);
+
+	/* Horizontal timing */
+	value = pdp_rreg32(pdp_reg, ODN_PDP_HSYNC1_OFFSET);
+	value = REG_VALUE_SET(value, hbps,
+			      ODN_PDP_HSYNC1_HBPS_SHIFT,
+			      ODN_PDP_HSYNC1_HBPS_MASK);
+	value = REG_VALUE_SET(value, ht,
+			      ODN_PDP_HSYNC1_HT_SHIFT,
+			      ODN_PDP_HSYNC1_HT_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_HSYNC1_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_HSYNC2_OFFSET);
+	value = REG_VALUE_SET(value, has,
+			      ODN_PDP_HSYNC2_HAS_SHIFT,
+			      ODN_PDP_HSYNC2_HAS_MASK);
+	value = REG_VALUE_SET(value, hlbs,
+			      ODN_PDP_HSYNC2_HLBS_SHIFT,
+			      ODN_PDP_HSYNC2_HLBS_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_HSYNC2_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_HSYNC3_OFFSET);
+	value = REG_VALUE_SET(value, hfps,
+			      ODN_PDP_HSYNC3_HFPS_SHIFT,
+			      ODN_PDP_HSYNC3_HFPS_MASK);
+	value = REG_VALUE_SET(value, hrbs,
+			      ODN_PDP_HSYNC3_HRBS_SHIFT,
+			      ODN_PDP_HSYNC3_HRBS_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_HSYNC3_OFFSET, value);
+
+	/* Vertical timing */
+	value = pdp_rreg32(pdp_reg, ODN_PDP_VSYNC1_OFFSET);
+	value = REG_VALUE_SET(value, vbps,
+			      ODN_PDP_VSYNC1_VBPS_SHIFT,
+			      ODN_PDP_VSYNC1_VBPS_MASK);
+	value = REG_VALUE_SET(value, vt,
+			      ODN_PDP_VSYNC1_VT_SHIFT,
+			      ODN_PDP_VSYNC1_VT_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_VSYNC1_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_VSYNC2_OFFSET);
+	value = REG_VALUE_SET(value, vas,
+			      ODN_PDP_VSYNC2_VAS_SHIFT,
+			      ODN_PDP_VSYNC2_VAS_MASK);
+	value = REG_VALUE_SET(value, vtbs,
+			      ODN_PDP_VSYNC2_VTBS_SHIFT,
+			      ODN_PDP_VSYNC2_VTBS_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_VSYNC2_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, ODN_PDP_VSYNC3_OFFSET);
+	value = REG_VALUE_SET(value, vfps,
+			      ODN_PDP_VSYNC3_VFPS_SHIFT,
+			      ODN_PDP_VSYNC3_VFPS_MASK);
+	value = REG_VALUE_SET(value, vbbs,
+			      ODN_PDP_VSYNC3_VBBS_SHIFT,
+			      ODN_PDP_VSYNC3_VBBS_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_VSYNC3_OFFSET, value);
+
+	/* Horizontal data enable */
+	value = pdp_rreg32(pdp_reg, ODN_PDP_HDECTRL_OFFSET);
+	value = REG_VALUE_SET(value, hlbs,
+			      ODN_PDP_HDECTRL_HDES_SHIFT,
+			      ODN_PDP_HDECTRL_HDES_MASK);
+	value = REG_VALUE_SET(value, hfps,
+			      ODN_PDP_HDECTRL_HDEF_SHIFT,
+			      ODN_PDP_HDECTRL_HDEF_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_HDECTRL_OFFSET, value);
+
+	/* Vertical data enable */
+	value = pdp_rreg32(pdp_reg, ODN_PDP_VDECTRL_OFFSET);
+	value = REG_VALUE_SET(value, vtbs,
+			      ODN_PDP_VDECTRL_VDES_SHIFT,
+			      ODN_PDP_VDECTRL_VDES_MASK);
+	value = REG_VALUE_SET(value, vfps,
+			      ODN_PDP_VDECTRL_VDEF_SHIFT,
+			      ODN_PDP_VDECTRL_VDEF_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_VDECTRL_OFFSET, value);
+
+	/* Vertical event start and vertical fetch start */
+	value = pdp_rreg32(pdp_reg, ODN_PDP_VEVENT_OFFSET);
+	value = REG_VALUE_SET(value, vbps,
+			      ODN_PDP_VEVENT_VFETCH_SHIFT,
+			      ODN_PDP_VEVENT_VFETCH_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_VEVENT_OFFSET, value);
+
+	/* Set up polarities of sync/blank */
+	value = REG_VALUE_SET(0, 0x1,
+			      ODN_PDP_SYNCCTRL_BLNKPOL_SHIFT,
+			      ODN_PDP_SYNCCTRL_BLNKPOL_MASK);
+	if (nhsync)
+		value = REG_VALUE_SET(value, 0x1,
+				      ODN_PDP_SYNCCTRL_HSPOL_SHIFT,
+				      ODN_PDP_SYNCCTRL_HSPOL_MASK);
+	if (nvsync)
+		value = REG_VALUE_SET(value, 0x1,
+				      ODN_PDP_SYNCCTRL_VSPOL_SHIFT,
+				      ODN_PDP_SYNCCTRL_VSPOL_MASK);
+	pdp_wreg32(pdp_reg, ODN_PDP_SYNCCTRL_OFFSET, value);
+
+	/* PDP framebuffer compression setup */
+	if (pfim_reg)
+		pfim_modeset(pfim_reg);
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_odin.h b/drivers/gpu/drm/img-rogue/apollo/pdp_odin.h
new file mode 100644
index 000000000000..0a625e231147
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_odin.h
@@ -0,0 +1,95 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_ODIN_H__)
+#define __PDP_ODIN_H__
+
+#include <linux/device.h>
+#include <linux/types.h>
+
+/* include here for ODN_PDP_SURF_PIXFMT_ARGB8888 as this is part of the API */
+#include "odin_pdp_regs.h"
+#include "pdp_common.h"
+
+bool pdp_odin_clocks_set(struct device *dev,
+			 void __iomem *pdp_reg, void __iomem *pll_reg,
+			 u32 clock_freq, u32 dev_num,
+			 void __iomem *odn_core_reg,
+			 u32 hdisplay, u32 vdisplay,
+			 enum pdp_odin_subversion);
+
+void pdp_odin_set_updates_enabled(struct device *dev, void __iomem *pdp_reg,
+				  bool enable);
+
+void pdp_odin_set_syncgen_enabled(struct device *dev, void __iomem *pdp_reg,
+				  bool enable);
+
+void pdp_odin_set_powerdwn_enabled(struct device *dev, void __iomem *pdp_reg,
+				   bool enable);
+
+void pdp_odin_set_vblank_enabled(struct device *dev, void __iomem *pdp_reg,
+				 bool enable);
+
+bool pdp_odin_check_and_clear_vblank(struct device *dev,
+				     void __iomem *pdp_reg);
+
+void pdp_odin_set_plane_enabled(struct device *dev, void __iomem *pdp_reg,
+				u32 plane, bool enable);
+
+void pdp_odin_reset_planes(struct device *dev, void __iomem *pdp_reg);
+
+void pdp_odin_set_surface(struct device *dev, void __iomem *pdp_reg,
+			  u32 plane, u32 address, u32 offset,
+			  u32 posx, u32 posy,
+			  u32 width, u32 height, u32 stride,
+			  u32 format, u32 alpha, bool blend,
+			  void __iomem *pfim_reg, u32 fbcf);
+
+void pdp_odin_mode_set(struct device *dev, void __iomem *pdp_reg,
+		       u32 h_display, u32 v_display,
+		       u32 hbps, u32 ht, u32 has,
+		       u32 hlbs, u32 hfps, u32 hrbs,
+		       u32 vbps, u32 vt, u32 vas,
+		       u32 vtbs, u32 vfps, u32 vbbs,
+		       bool nhsync, bool nvsync,
+		       void __iomem *pfim_reg);
+
+#endif /* __PDP_ODIN_H__ */
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_plato.c b/drivers/gpu/drm/img-rogue/apollo/pdp_plato.c
new file mode 100644
index 000000000000..df214f97bc6d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_plato.c
@@ -0,0 +1,339 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "pdp_common.h"
+#include "pdp_plato.h"
+#include "pdp2_mmu_regs.h"
+#include "pdp2_regs.h"
+
+#define PLATO_PDP_STRIDE_SHIFT 5
+
+
+void pdp_plato_set_syncgen_enabled(struct device *dev, void __iomem *pdp_reg,
+				   bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set syncgen: %s\n", enable ? "enable" : "disable");
+#endif
+
+	value = pdp_rreg32(pdp_reg, PDP_SYNCCTRL_OFFSET);
+	/* Starts Sync Generator. */
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      PDP_SYNCCTRL_SYNCACTIVE_SHIFT,
+			      PDP_SYNCCTRL_SYNCACTIVE_MASK);
+	/* Controls polarity of pixel clock: Pixel clock is inverted */
+	value = REG_VALUE_SET(value, 0x01,
+			      PDP_SYNCCTRL_CLKPOL_SHIFT,
+			      PDP_SYNCCTRL_CLKPOL_MASK);
+	pdp_wreg32(pdp_reg, PDP_SYNCCTRL_OFFSET, value);
+}
+
+void pdp_plato_set_vblank_enabled(struct device *dev, void __iomem *pdp_reg,
+				  bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set vblank: %s\n", enable ? "enable" : "disable");
+#endif
+
+	pdp_wreg32(pdp_reg, PDP_INTCLR_OFFSET, 0xFFFFFFFF);
+
+	value = pdp_rreg32(pdp_reg, PDP_INTENAB_OFFSET);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      PDP_INTENAB_INTEN_VBLNK0_SHIFT,
+			      PDP_INTENAB_INTEN_VBLNK0_MASK);
+	pdp_wreg32(pdp_reg, PDP_INTENAB_OFFSET, value);
+}
+
+bool pdp_plato_check_and_clear_vblank(struct device *dev,
+				      void __iomem *pdp_reg)
+{
+	u32 value;
+
+	value = pdp_rreg32(pdp_reg, PDP_INTSTAT_OFFSET);
+
+	if (REG_VALUE_GET(value,
+			  PDP_INTSTAT_INTS_VBLNK0_SHIFT,
+			  PDP_INTSTAT_INTS_VBLNK0_MASK)) {
+		pdp_wreg32(pdp_reg, PDP_INTCLR_OFFSET,
+			   (1 << PDP_INTCLR_INTCLR_VBLNK0_SHIFT));
+		return true;
+	}
+
+	return false;
+}
+
+void pdp_plato_set_plane_enabled(struct device *dev, void __iomem *pdp_reg,
+				 u32 plane, bool enable)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev, "Set plane %u: %s\n",
+		 plane, enable ? "enable" : "disable");
+#endif
+	value = pdp_rreg32(pdp_reg, PDP_GRPH1CTRL_OFFSET);
+	value = REG_VALUE_SET(value, enable ? 0x1 : 0x0,
+			      PDP_GRPH1CTRL_GRPH1STREN_SHIFT,
+			      PDP_GRPH1CTRL_GRPH1STREN_MASK);
+	pdp_wreg32(pdp_reg, PDP_GRPH1CTRL_OFFSET, value);
+}
+
+void pdp_plato_set_surface(struct device *dev,
+			   void __iomem *pdp_reg, void __iomem *pdp_bif_reg,
+			   u32 plane, u64 address,
+			   u32 posx, u32 posy,
+			   u32 width, u32 height, u32 stride,
+			   u32 format, u32 alpha, bool blend)
+{
+	u32 value;
+
+#ifdef PDP_VERBOSE
+	dev_info(dev,
+		 "Set surface: size=%dx%d stride=%d format=%d address=0x%llx\n",
+		 width, height, stride, format, address);
+#endif
+
+	pdp_wreg32(pdp_reg, PDP_REGISTER_UPDATE_CTRL_OFFSET, 0x0);
+	/*
+	 * Set the offset position to (0,0) as we've already added any offset
+	 * to the base address.
+	 */
+	pdp_wreg32(pdp_reg, PDP_GRPH1POSN_OFFSET, 0);
+
+	/* Set the frame buffer base address */
+	if (address & 0xF)
+		dev_warn(dev, "The frame buffer address is not aligned\n");
+
+	pdp_wreg32(pdp_reg, PDP_GRPH1BASEADDR_OFFSET,
+		   (u32)address & PDP_GRPH1BASEADDR_GRPH1BASEADDR_MASK);
+
+	/*
+	 * Write 8 msb of the address to address extension bits in the PDP
+	 * MMU control register.
+	 */
+	value = pdp_rreg32(pdp_bif_reg, PDP_BIF_ADDRESS_CONTROL_OFFSET);
+	value = REG_VALUE_SET(value, address >> 32,
+			      PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED_SHIFT,
+			      PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED_MASK);
+	value = REG_VALUE_SET(value, 0x00,
+			      PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING_SHIFT,
+			      PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING_MASK);
+	value = REG_VALUE_SET(value, 0x01,
+			      PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS_SHIFT,
+			      PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS_MASK);
+	pdp_wreg32(pdp_bif_reg, PDP_BIF_ADDRESS_CONTROL_OFFSET, value);
+
+	/* Set the framebuffer pixel format */
+	value = pdp_rreg32(pdp_reg, PDP_GRPH1SURF_OFFSET);
+	value = REG_VALUE_SET(value, format,
+			      PDP_GRPH1SURF_GRPH1PIXFMT_SHIFT,
+			      PDP_GRPH1SURF_GRPH1PIXFMT_MASK);
+	pdp_wreg32(pdp_reg, PDP_GRPH1SURF_OFFSET, value);
+	/*
+	 * Set the framebuffer size (this might be smaller than the resolution)
+	 */
+	value = REG_VALUE_SET(0, width - 1,
+			      PDP_GRPH1SIZE_GRPH1WIDTH_SHIFT,
+			      PDP_GRPH1SIZE_GRPH1WIDTH_MASK);
+	value = REG_VALUE_SET(value, height - 1,
+			      PDP_GRPH1SIZE_GRPH1HEIGHT_SHIFT,
+			      PDP_GRPH1SIZE_GRPH1HEIGHT_MASK);
+	pdp_wreg32(pdp_reg, PDP_GRPH1SIZE_OFFSET, value);
+
+	/* Set the framebuffer stride in 16byte words */
+	value = REG_VALUE_SET(0, (stride >> PLATO_PDP_STRIDE_SHIFT) - 1,
+			      PDP_GRPH1STRIDE_GRPH1STRIDE_SHIFT,
+			      PDP_GRPH1STRIDE_GRPH1STRIDE_MASK);
+	pdp_wreg32(pdp_reg, PDP_GRPH1STRIDE_OFFSET, value);
+
+	/* Enable the register writes on the next vblank */
+	pdp_wreg32(pdp_reg, PDP_REGISTER_UPDATE_CTRL_OFFSET, 0x3);
+
+	/*
+	 * Issues with NoC sending interleaved read responses to PDP require
+	 * burst to be 1.
+	 */
+	value = REG_VALUE_SET(0, 0x02,
+			      PDP_MEMCTRL_MEMREFRESH_SHIFT,
+			      PDP_MEMCTRL_MEMREFRESH_MASK);
+	value = REG_VALUE_SET(value, 0x01,
+			      PDP_MEMCTRL_BURSTLEN_SHIFT,
+			      PDP_MEMCTRL_BURSTLEN_MASK);
+	pdp_wreg32(pdp_reg, PDP_MEMCTRL_OFFSET, value);
+}
+
+void pdp_plato_mode_set(struct device *dev, void __iomem *pdp_reg,
+			u32 h_display, u32 v_display,
+			u32 hbps, u32 ht, u32 has,
+			u32 hlbs, u32 hfps, u32 hrbs,
+			u32 vbps, u32 vt, u32 vas,
+			u32 vtbs, u32 vfps, u32 vbbs,
+			bool nhsync, bool nvsync)
+{
+	u32 value;
+
+	dev_info(dev, "Set mode: %dx%d\n", h_display, v_display);
+#ifdef PDP_VERBOSE
+	dev_info(dev, " ht: %d hbps %d has %d hlbs %d hfps %d hrbs %d\n",
+		 ht, hbps, has, hlbs, hfps, hrbs);
+	dev_info(dev, " vt: %d vbps %d vas %d vtbs %d vfps %d vbbs %d\n",
+		 vt, vbps, vas, vtbs, vfps, vbbs);
+#endif
+
+	/* Update control */
+	value = pdp_rreg32(pdp_reg, PDP_REGISTER_UPDATE_CTRL_OFFSET);
+	value = REG_VALUE_SET(value, 0x0,
+			      PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SHIFT,
+			      PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_MASK);
+	pdp_wreg32(pdp_reg, PDP_REGISTER_UPDATE_CTRL_OFFSET, value);
+
+	/* Set hsync timings */
+	value = pdp_rreg32(pdp_reg, PDP_HSYNC1_OFFSET);
+	value = REG_VALUE_SET(value, hbps,
+			      PDP_HSYNC1_HBPS_SHIFT,
+			      PDP_HSYNC1_HBPS_MASK);
+	value = REG_VALUE_SET(value, ht,
+			      PDP_HSYNC1_HT_SHIFT,
+			      PDP_HSYNC1_HT_MASK);
+	pdp_wreg32(pdp_reg, PDP_HSYNC1_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, PDP_HSYNC2_OFFSET);
+	value = REG_VALUE_SET(value, has,
+			      PDP_HSYNC2_HAS_SHIFT,
+			      PDP_HSYNC2_HAS_MASK);
+	value = REG_VALUE_SET(value, hlbs,
+			      PDP_HSYNC2_HLBS_SHIFT,
+			      PDP_HSYNC2_HLBS_MASK);
+	pdp_wreg32(pdp_reg, PDP_HSYNC2_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, PDP_HSYNC3_OFFSET);
+	value = REG_VALUE_SET(value, hfps,
+			      PDP_HSYNC3_HFPS_SHIFT,
+			      PDP_HSYNC3_HFPS_MASK);
+	value = REG_VALUE_SET(value, hrbs,
+			      PDP_HSYNC3_HRBS_SHIFT,
+			      PDP_HSYNC3_HRBS_MASK);
+	pdp_wreg32(pdp_reg, PDP_HSYNC3_OFFSET, value);
+
+	/* Set vsync timings */
+	value = pdp_rreg32(pdp_reg, PDP_VSYNC1_OFFSET);
+	value = REG_VALUE_SET(value, vbps,
+			      PDP_VSYNC1_VBPS_SHIFT,
+			      PDP_VSYNC1_VBPS_MASK);
+	value = REG_VALUE_SET(value, vt,
+			      PDP_VSYNC1_VT_SHIFT,
+			      PDP_VSYNC1_VT_MASK);
+	pdp_wreg32(pdp_reg, PDP_VSYNC1_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, PDP_VSYNC2_OFFSET);
+	value = REG_VALUE_SET(value, vas,
+			      PDP_VSYNC2_VAS_SHIFT,
+			      PDP_VSYNC2_VAS_MASK);
+	value = REG_VALUE_SET(value, vtbs,
+			      PDP_VSYNC2_VTBS_SHIFT,
+			      PDP_VSYNC2_VTBS_MASK);
+	pdp_wreg32(pdp_reg, PDP_VSYNC2_OFFSET, value);
+
+	value = pdp_rreg32(pdp_reg, PDP_VSYNC3_OFFSET);
+	value = REG_VALUE_SET(value, vfps,
+			      PDP_VSYNC3_VFPS_SHIFT,
+			      PDP_VSYNC3_VFPS_MASK);
+	value = REG_VALUE_SET(value, vbbs,
+			      PDP_VSYNC3_VBBS_SHIFT,
+			      PDP_VSYNC3_VBBS_MASK);
+	pdp_wreg32(pdp_reg, PDP_VSYNC3_OFFSET, value);
+
+	/* Horizontal data enable */
+	value = pdp_rreg32(pdp_reg, PDP_HDECTRL_OFFSET);
+	value = REG_VALUE_SET(value, has,
+			      PDP_HDECTRL_HDES_SHIFT,
+			      PDP_HDECTRL_HDES_MASK);
+	value = REG_VALUE_SET(value, hrbs,
+			      PDP_HDECTRL_HDEF_SHIFT,
+			      PDP_HDECTRL_HDEF_MASK);
+	pdp_wreg32(pdp_reg, PDP_HDECTRL_OFFSET, value);
+
+	/* Vertical data enable */
+	value = pdp_rreg32(pdp_reg, PDP_VDECTRL_OFFSET);
+	value = REG_VALUE_SET(value, vtbs, /* XXX: we're setting this to VAS */
+			      PDP_VDECTRL_VDES_SHIFT,
+			      PDP_VDECTRL_VDES_MASK);
+	value = REG_VALUE_SET(value, vfps, /* XXX: set to VBBS */
+			      PDP_VDECTRL_VDEF_SHIFT,
+			      PDP_VDECTRL_VDEF_MASK);
+	pdp_wreg32(pdp_reg, PDP_VDECTRL_OFFSET, value);
+
+	/* Vertical event start and vertical fetch start */
+	value = 0;
+	value = REG_VALUE_SET(value, 0,
+			      PDP_VEVENT_VEVENT_SHIFT,
+			      PDP_VEVENT_VEVENT_MASK);
+	value = REG_VALUE_SET(value, vbps,
+			      PDP_VEVENT_VFETCH_SHIFT,
+			      PDP_VEVENT_VFETCH_MASK);
+	value = REG_VALUE_SET(value, vfps,
+			      PDP_VEVENT_VEVENT_SHIFT,
+			      PDP_VEVENT_VEVENT_MASK);
+	pdp_wreg32(pdp_reg, PDP_VEVENT_OFFSET, value);
+
+	/* Set up polarities of sync/blank */
+	value = REG_VALUE_SET(0, 0x1,
+			      PDP_SYNCCTRL_BLNKPOL_SHIFT,
+			      PDP_SYNCCTRL_BLNKPOL_MASK);
+
+	if (nhsync)
+		value = REG_VALUE_SET(value, 0x1,
+				      PDP_SYNCCTRL_HSPOL_SHIFT,
+				      PDP_SYNCCTRL_HSPOL_MASK);
+
+	if (nvsync)
+		value = REG_VALUE_SET(value, 0x1,
+				      PDP_SYNCCTRL_VSPOL_SHIFT,
+				      PDP_SYNCCTRL_VSPOL_MASK);
+
+	pdp_wreg32(pdp_reg,
+		   PDP_SYNCCTRL_OFFSET,
+		   value);
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_plato.h b/drivers/gpu/drm/img-rogue/apollo/pdp_plato.h
new file mode 100644
index 000000000000..3275259673c0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_plato.h
@@ -0,0 +1,86 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_PLATO_H__)
+#define __PDP_PLATO_H__
+
+#include <linux/device.h>
+#include <linux/types.h>
+
+#define PLATO_PDP_PIXEL_FORMAT_G        (0x00)
+#define PLATO_PDP_PIXEL_FORMAT_ARGB4    (0x04)
+#define PLATO_PDP_PIXEL_FORMAT_ARGB1555 (0x05)
+#define PLATO_PDP_PIXEL_FORMAT_RGB8     (0x06)
+#define PLATO_PDP_PIXEL_FORMAT_RGB565   (0x07)
+#define PLATO_PDP_PIXEL_FORMAT_ARGB8    (0x08)
+#define PLATO_PDP_PIXEL_FORMAT_AYUV8    (0x10)
+#define PLATO_PDP_PIXEL_FORMAT_YUV10    (0x15)
+#define PLATO_PDP_PIXEL_FORMAT_RGBA8    (0x16)
+
+
+void pdp_plato_set_syncgen_enabled(struct device *dev, void __iomem *pdp_reg,
+				   bool enable);
+
+void pdp_plato_set_vblank_enabled(struct device *dev, void __iomem *pdp_reg,
+				  bool enable);
+
+bool pdp_plato_check_and_clear_vblank(struct device *dev,
+				      void __iomem *pdp_reg);
+
+void pdp_plato_set_plane_enabled(struct device *dev, void __iomem *pdp_reg,
+				 u32 plane, bool enable);
+
+void pdp_plato_set_surface(struct device *dev,
+			   void __iomem *pdp_reg, void __iomem *pdp_bif_reg,
+			   u32 plane, u64 address,
+			   u32 posx, u32 posy,
+			   u32 width, u32 height, u32 stride,
+			   u32 format, u32 alpha, bool blend);
+
+void pdp_plato_mode_set(struct device *dev, void __iomem *pdp_reg,
+			u32 h_display, u32 v_display,
+			u32 hbps, u32 ht, u32 has,
+			u32 hlbs, u32 hfps, u32 hrbs,
+			u32 vbps, u32 vt, u32 vas,
+			u32 vtbs, u32 vfps, u32 vbbs,
+			bool nhsync, bool nvsync);
+
+#endif /* __PDP_PLATO_H__ */
diff --git a/drivers/gpu/drm/img-rogue/apollo/pdp_regs.h b/drivers/gpu/drm/img-rogue/apollo/pdp_regs.h
new file mode 100644
index 000000000000..bd26b0617f95
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pdp_regs.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__PDP_REGS_H__)
+#define __PDP_REGS_H__
+
+/*************************************************************************/ /*!
+ PCI Device Information
+*/ /**************************************************************************/
+
+#define DCPDP_VENDOR_ID_POWERVR			(0x1010)
+
+#define DCPDP_DEVICE_ID_PCI_APOLLO_FPGA		(0x1CF1)
+#define DCPDP_DEVICE_ID_PCIE_APOLLO_FPGA	(0x1CF2)
+
+/*************************************************************************/ /*!
+ PCI Device Base Address Information
+*/ /**************************************************************************/
+
+/* PLL and PDP registers on base address register 0 */
+#define DCPDP_REG_PCI_BASENUM			(0)
+
+#define DCPDP_PCI_PLL_REG_OFFSET		(0x1000)
+#define DCPDP_PCI_PLL_REG_SIZE			(0x0400)
+
+#define DCPDP_PCI_PDP_REG_OFFSET		(0xC000)
+#define DCPDP_PCI_PDP_REG_SIZE			(0x2000)
+
+/*************************************************************************/ /*!
+ Misc register information
+*/ /**************************************************************************/
+
+/* This information isn't captured in tcf_rgbpdp_regs.h so define it here */
+#define DCPDP_STR1SURF_FORMAT_ARGB8888		(0xE)
+#define DCPDP_STR1ADDRCTRL_BASE_ADDR_SHIFT	(4)
+#define DCPDP_STR1POSN_STRIDE_SHIFT		(4)
+
+#endif /* !defined(__PDP_REGS_H__) */
diff --git a/drivers/gpu/drm/img-rogue/apollo/pfim_defs.h b/drivers/gpu/drm/img-rogue/apollo/pfim_defs.h
new file mode 100644
index 000000000000..d39c06f703bd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pfim_defs.h
@@ -0,0 +1,69 @@
+/******************************************************************************
+@Title          Odin PFIM definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin register defs for PDP-FBDC Interface Module
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+#ifndef _PFIM_DEFS_H_
+#define _PFIM_DEFS_H_
+
+/* Supported FBC modes */
+#define ODIN_PFIM_MOD_LINEAR       (0x00)
+#define ODIN_PFIM_FBCDC_8X8_V12    (0x01)
+#define ODIN_PFIM_FBCDC_16X4_V12   (0x02)
+#define ODIN_PFIM_FBCDC_MAX        (0x03)
+
+/* Supported pixel formats */
+#define ODN_PFIM_PIXFMT_NONE       (0x00)
+#define ODN_PFIM_PIXFMT_ARGB8888   (0x0C)
+#define ODN_PFIM_PIXFMT_RGB565     (0x05)
+
+/* Tile types */
+#define ODN_PFIM_TILETYPE_8X8      (0x01)
+#define ODN_PFIM_TILETYPE_16X4     (0x02)
+#define ODN_PFIM_TILETYPE_32x2     (0x03)
+
+#define PFIM_ROUNDUP(X, Y)         (((X) + ((Y) - 1U)) & ~((Y) - 1U))
+#define PFIM_RND_TAG               (0x10)
+
+#endif /* _PFIM_DEFS_H_ */
+
+/******************************************************************************
+ End of file (pfim_defs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/pfim_regs.h b/drivers/gpu/drm/img-rogue/apollo/pfim_regs.h
new file mode 100644
index 000000000000..4b8ff82138b4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/pfim_regs.h
@@ -0,0 +1,265 @@
+/******************************************************************************
+@Title          Odin PFIM control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Odin register defs for PDP-FBDC Interface Module
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+#ifndef _PFIM_REGS_H_
+#define _PFIM_REGS_H_
+
+/*
+	Register CR_PFIM_NUM_TILES
+*/
+#define CR_PFIM_NUM_TILES 0x0000
+#define CR_PFIM_NUM_TILES_MASK 0x007FFFFFU
+#define CR_PFIM_NUM_TILES_SHIFT 0
+#define CR_PFIM_NUM_TILES_SIGNED 0
+
+/*
+	Register CR_PFIM_TILES_PER_LINE
+*/
+#define CR_PFIM_TILES_PER_LINE 0x0004
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_MASK 0x000000FFU
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_SHIFT 0
+#define CR_PFIM_TILES_PER_LINE_PFIM_TILES_PER_LINE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB
+*/
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB 0x0008
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB
+*/
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB 0x000C
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_MASK 0x00000003U
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_YARGB_BASE_ADDR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_UV_BASE_ADDR_LSB
+*/
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB 0x0010
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_UV_BASE_ADDR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_UV_BASE_ADDR_MSB
+*/
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB 0x0014
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_MASK 0x00000003U
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_UV_BASE_ADDR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_PDP_Y_BASE_ADDR
+*/
+#define CR_PFIM_PDP_Y_BASE_ADDR 0x0018
+#define CR_PFIM_PDP_Y_BASE_ADDR_MASK 0xFFFFFFFFU
+#define CR_PFIM_PDP_Y_BASE_ADDR_SHIFT 0
+#define CR_PFIM_PDP_Y_BASE_ADDR_SIGNED 0
+
+/*
+	Register CR_PFIM_PDP_UV_BASE_ADDR
+*/
+#define CR_PFIM_PDP_UV_BASE_ADDR 0x001C
+#define CR_PFIM_PDP_UV_BASE_ADDR_MASK 0xFFFFFFFFU
+#define CR_PFIM_PDP_UV_BASE_ADDR_SHIFT 0
+#define CR_PFIM_PDP_UV_BASE_ADDR_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_CONTEXT
+*/
+#define CR_PFIM_FBDC_REQ_CONTEXT 0x0020
+#define CR_PFIM_FBDC_REQ_CONTEXT_MASK 0x00000007U
+#define CR_PFIM_FBDC_REQ_CONTEXT_SHIFT 0
+#define CR_PFIM_FBDC_REQ_CONTEXT_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_TAG
+*/
+#define CR_PFIM_FBDC_REQ_TAG 0x0024
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_MASK     0x00000003U
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_SHIFT    0
+#define CR_PFIM_FBDC_REQ_TAG_YARGB_SIGNED   0
+
+#define CR_PFIM_FBDC_REQ_TAG_UV_MASK        0x00000030U
+#define CR_PFIM_FBDC_REQ_TAG_UV_SHIFT       4
+#define CR_PFIM_FBDC_REQ_TAG_UV_SIGNED      0
+
+/*
+	Register CR_PFIM_FBDC_REQ_SB_TAG
+*/
+#define CR_PFIM_FBDC_REQ_SB_TAG 0x0028
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_MASK  0x00000003U
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_SHIFT 0
+#define CR_PFIM_FBDC_REQ_SB_TAG_YARGB_SIGNED 0
+
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_MASK     0x00000030U
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_SHIFT    4
+#define CR_PFIM_FBDC_REQ_SB_TAG_UV_SIGNED   0
+
+/*
+	Register CR_PFIM_FBDC_HDR_INVAL_REQ
+*/
+#define CR_PFIM_FBDC_HDR_INVAL_REQ 0x002C
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_MASK 0x00000001U
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_SHIFT 0
+#define CR_PFIM_FBDC_HDR_INVAL_REQ_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_PIX_FORMAT
+*/
+#define CR_PFIM_FBDC_PIX_FORMAT 0x0030
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_MASK 0x0000007FU
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_SHIFT 0
+#define CR_PFIM_FBDC_PIX_FORMAT_FBDC_PIX_FMT_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_CH0123_VAL0
+*/
+#define CR_PFIM_FBDC_CR_CH0123_VAL0 0x0034
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_CH0123_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_CH0123_VAL1
+*/
+#define CR_PFIM_FBDC_CR_CH0123_VAL1 0x0038
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_CH0123_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_Y_VAL0
+*/
+#define CR_PFIM_FBDC_CR_Y_VAL0 0x003C
+#define CR_PFIM_FBDC_CR_Y_VAL0_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_Y_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_Y_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_UV_VAL0
+*/
+#define CR_PFIM_FBDC_CR_UV_VAL0 0x0040
+#define CR_PFIM_FBDC_CR_UV_VAL0_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_UV_VAL0_SHIFT 0
+#define CR_PFIM_FBDC_CR_UV_VAL0_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_Y_VAL1
+*/
+#define CR_PFIM_FBDC_CR_Y_VAL1 0x0044
+#define CR_PFIM_FBDC_CR_Y_VAL1_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_Y_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_Y_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CR_UV_VAL1
+*/
+#define CR_PFIM_FBDC_CR_UV_VAL1 0x0048
+#define CR_PFIM_FBDC_CR_UV_VAL1_MASK 0x000003FFU
+#define CR_PFIM_FBDC_CR_UV_VAL1_SHIFT 0
+#define CR_PFIM_FBDC_CR_UV_VAL1_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_ENABLE
+*/
+#define CR_PFIM_FBDC_FILTER_ENABLE 0x004C
+#define CR_PFIM_FBDC_FILTER_ENABLE_MASK 0x00000001U
+#define CR_PFIM_FBDC_FILTER_ENABLE_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_ENABLE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_STATUS
+*/
+#define CR_PFIM_FBDC_FILTER_STATUS 0x0050
+#define CR_PFIM_FBDC_FILTER_STATUS_MASK 0x0000000FU
+#define CR_PFIM_FBDC_FILTER_STATUS_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_STATUS_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_FILTER_CLEAR
+*/
+#define CR_PFIM_FBDC_FILTER_CLEAR 0x0054
+#define CR_PFIM_FBDC_FILTER_CLEAR_MASK 0x0000000FU
+#define CR_PFIM_FBDC_FILTER_CLEAR_SHIFT 0
+#define CR_PFIM_FBDC_FILTER_CLEAR_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_TILE_TYPE
+*/
+#define CR_PFIM_FBDC_TILE_TYPE 0x0058
+#define CR_PFIM_FBDC_TILE_TYPE_MASK 0x00000003U
+#define CR_PFIM_FBDC_TILE_TYPE_SHIFT 0
+#define CR_PFIM_FBDC_TILE_TYPE_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CLEAR_COLOUR_LSB
+*/
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB 0x005C
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_SHIFT 0
+#define CR_PFIM_FBDC_CLEAR_COLOUR_LSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_CLEAR_COLOUR_MSB
+*/
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB 0x0060
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_MASK 0xFFFFFFFFU
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_SHIFT 0
+#define CR_PFIM_FBDC_CLEAR_COLOUR_MSB_SIGNED 0
+
+/*
+	Register CR_PFIM_FBDC_REQ_LOSSY
+*/
+#define CR_PFIM_FBDC_REQ_LOSSY 0x0064
+#define CR_PFIM_FBDC_REQ_LOSSY_MASK 0x00000001U
+#define CR_PFIM_FBDC_REQ_LOSSY_SHIFT 0
+#define CR_PFIM_FBDC_REQ_LOSSY_SIGNED 0
+
+#endif /* _PFIM_REGS_H_ */
+
+/******************************************************************************
+ End of file (pfim_regs.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/sysconfig.c b/drivers/gpu/drm/img-rogue/apollo/sysconfig.c
new file mode 100644
index 000000000000..d198df48e292
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/sysconfig.c
@@ -0,0 +1,1740 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "sysinfo.h"
+#include "apollo_regs.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+#include "syscommon.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwutils.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_support.h"
+#include "ion_sys.h"
+#endif
+
+#include "vmm_pvz_server.h"
+#include "pvr_bridge_k.h"
+#include "pvr_drv.h"
+#include "tc_drv.h"
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+
+#define SECURE_FW_MEM_SIZE (0x400000)  /*  4MB */
+#define SECURE_MEM_SIZE    (0x4000000) /* 64MB */
+
+typedef struct
+{
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;
+	IMG_UINT64 uiSize;
+	IMG_BOOL bUsed;
+} CARD_PHYS_HEAP_CONFIG_SPEC;
+
+#define HEAP_SPEC_IDX_GPU_PRIVATE (0U)
+#define HEAP_SPEC_IDX_GPU_LOCAL   (1U)
+
+static const CARD_PHYS_HEAP_CONFIG_SPEC gasCardHeapTemplate[] =
+{
+	{
+	 PHYS_HEAP_USAGE_GPU_PRIVATE,
+	 0,					/* determined at runtime by apphints */
+	 false				/* determined at runtime by apphints */
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_LOCAL,
+	 0,					/* determined at runtime */
+	 true
+	},
+	{
+	 PHYS_HEAP_USAGE_GPU_SECURE,
+	 SECURE_MEM_SIZE,
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	 true
+#else
+	 false
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PRIVATE,
+	 SECURE_FW_MEM_SIZE,
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	 true
+#else
+	 false
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_SHARED,
+#if defined(SUPPORT_SECURITY_VALIDATION) && defined(RGX_PREMAP_FW_HEAPS)
+	 /* simultaneous virtualisation and security support requires premapped heaps,
+	  * i.e. FW_PRIVATE and FW_SHARED must fit contiguously into Fw's VA heap (RGX_FIRMWARE_RAW_HEAP_SIZE) */
+	 RGX_FIRMWARE_RAW_HEAP_SIZE - SECURE_FW_MEM_SIZE,
+#else
+	 RGX_FIRMWARE_RAW_HEAP_SIZE,
+#endif
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	true				/* VZ drivers need dedicated Fw heaps */
+#else
+	false				/* Native drivers can fallback on GPU_LOCAL for Fw mem */
+#endif
+	},
+	{
+	 PHYS_HEAP_USAGE_FW_PREMAP_PT,
+	 RGX_FIRMWARE_MAX_PAGETABLE_SIZE,
+#if defined(RGX_PREMAP_FW_HEAPS)
+	 true
+#else
+	 false
+#endif
+	}
+};
+
+#define ODIN_MEMORY_HYBRID_DEVICE_BASE 0x400000000
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (10)
+
+#define UI64_TOPWORD_IS_ZERO(ui64) ((ui64 >> 32) == 0)
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+
+/* Fake DVFS configuration used purely for testing purposes */
+
+static const IMG_OPP asOPPTable[] =
+{
+	{ 8,  25000000},
+	{ 16, 50000000},
+	{ 32, 75000000},
+	{ 64, 100000000},
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+static void SetFrequency(IMG_HANDLE hSysData, IMG_UINT32 ui32Frequency)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetFrequency %u", ui32Frequency));
+}
+
+static void SetVoltage(IMG_HANDLE hSysData, IMG_UINT32 ui32Voltage)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+	PVR_DPF((PVR_DBG_ERROR, "SetVoltage %u", ui32Voltage));
+}
+
+#endif
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsLocalPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCLocalCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCLocalDevPAddrToCpuPAddr,
+};
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 ui32NumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHostPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHostCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHostDevPAddrToCpuPAddr,
+};
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr);
+
+static PHYS_HEAP_FUNCTIONS gsHybridPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = TCHybridCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = TCHybridDevPAddrToCpuPAddr
+};
+
+typedef struct _SYS_DATA_ SYS_DATA;
+
+struct _SYS_DATA_
+{
+	struct platform_device *pdev;
+
+	struct tc_rogue_platform_data *pdata;
+
+	struct resource *registers;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_client *ion_client;
+	struct ion_handle *ion_rogue_allocation;
+#endif
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	void *pvS3Buffer;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+};
+
+#define SYSTEM_INFO_FORMAT_STRING "FPGA Revision: %s - TCF Core Revision: %s - TCF Core Target Build ID: %s - PCI Version: %s - Macro Version: %s"
+#define FPGA_REV_MAX_LEN      8 /* current longest format: "x.y.z" */
+#define TCF_CORE_REV_MAX_LEN  8 /* current longest format: "x.y.z" */
+#define TCF_CORE_CFG_MAX_LEN  4 /* current longest format: "x" */
+#define PCI_VERSION_MAX_LEN   4 /* current longest format: "x" */
+#define MACRO_VERSION_MAX_LEN 8 /* current longest format: "x.yz" */
+
+static IMG_CHAR *GetDeviceVersionString(SYS_DATA *psSysData)
+{
+	int err;
+	char str_fpga_rev[FPGA_REV_MAX_LEN]={0};
+	char str_tcf_core_rev[TCF_CORE_REV_MAX_LEN]={0};
+	char str_tcf_core_target_build_id[TCF_CORE_CFG_MAX_LEN]={0};
+	char str_pci_ver[PCI_VERSION_MAX_LEN]={0};
+	char str_macro_ver[MACRO_VERSION_MAX_LEN]={0};
+
+	IMG_CHAR *pszVersion;
+	IMG_UINT32 ui32StringLength;
+
+	err = tc_sys_strings(psSysData->pdev->dev.parent,
+							 str_fpga_rev, sizeof(str_fpga_rev),
+							 str_tcf_core_rev, sizeof(str_tcf_core_rev),
+							 str_tcf_core_target_build_id, sizeof(str_tcf_core_target_build_id),
+							 str_pci_ver, sizeof(str_pci_ver),
+							 str_macro_ver, sizeof(str_macro_ver));
+	if (err)
+	{
+		return NULL;
+	}
+
+	/* Calculate how much space we need to allocate for the string */
+	ui32StringLength = OSStringLength(SYSTEM_INFO_FORMAT_STRING);
+	ui32StringLength += OSStringLength(str_fpga_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_rev);
+	ui32StringLength += OSStringLength(str_tcf_core_target_build_id);
+	ui32StringLength += OSStringLength(str_pci_ver);
+	ui32StringLength += OSStringLength(str_macro_ver);
+
+	/* Create the version string */
+	pszVersion = OSAllocMem(ui32StringLength * sizeof(IMG_CHAR));
+	if (pszVersion)
+	{
+		OSSNPrintf(&pszVersion[0], ui32StringLength,
+				   SYSTEM_INFO_FORMAT_STRING,
+				   str_fpga_rev,
+				   str_tcf_core_rev,
+				   str_tcf_core_target_build_id,
+				   str_pci_ver,
+				   str_macro_ver);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to create format string", __func__));
+	}
+
+	return pszVersion;
+}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+static SYS_DATA *gpsIonPrivateData;
+
+PVRSRV_ERROR IonInit(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYS_DATA *psSysData = pvPrivateData;
+	gpsIonPrivateData = psSysData;
+
+	psSysData->ion_client = ion_client_create(psSysData->pdata->ion_device, SYS_RGX_DEV_NAME);
+	if (IS_ERR(psSysData->ion_client))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create ION client (%ld)", __func__, PTR_ERR(psSysData->ion_client)));
+		eError = PVRSRV_ERROR_ION_NO_CLIENT;
+		goto err_out;
+	}
+	/* Allocate the whole rogue ion heap and pass that to services to manage */
+	psSysData->ion_rogue_allocation = ion_alloc(psSysData->ion_client, psSysData->pdata->rogue_heap_memory_size, 4096, (1 << psSysData->pdata->ion_heap_id), 0);
+	if (IS_ERR(psSysData->ion_rogue_allocation))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to allocate ION rogue buffer (%ld)", __func__, PTR_ERR(psSysData->ion_rogue_allocation)));
+		eError = PVRSRV_ERROR_ION_FAILED_TO_ALLOC;
+		goto err_destroy_client;
+
+	}
+
+	return PVRSRV_OK;
+err_destroy_client:
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+err_out:
+	return eError;
+}
+
+void IonDeinit(void)
+{
+	SYS_DATA *psSysData = gpsIonPrivateData;
+	ion_free(psSysData->ion_client, psSysData->ion_rogue_allocation);
+	psSysData->ion_rogue_allocation = NULL;
+	ion_client_destroy(psSysData->ion_client);
+	psSysData->ion_client = NULL;
+}
+
+struct ion_device *IonDevAcquire(void)
+{
+	return gpsIonPrivateData->pdata->ion_device;
+}
+
+void IonDevRelease(struct ion_device *ion_device)
+{
+	PVR_ASSERT(ion_device == gpsIonPrivateData->pdata->ion_device);
+}
+#endif /* defined(SUPPORT_ION) */
+
+static void TCLocalCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+			psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCLocalDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+				      IMG_UINT32 ui32NumOfAddr,
+				      IMG_CPU_PHYADDR *psCpuPAddr,
+				      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+			psDevPAddr[ui32Idx].uiAddr + psSysData->pdata->tc_memory_base;
+	}
+}
+
+static void TCHostCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	if (sizeof(*psDevPAddr) == sizeof(*psCpuPAddr))
+	{
+		OSCachedMemCopy(psDevPAddr, psCpuPAddr, uiNumOfAddr * sizeof(*psDevPAddr));
+		return;
+	}
+
+	/* In this case we may have a 32bit host, so we can't do a memcpy */
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (uiNumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < uiNumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void TCHostDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+									 IMG_UINT32 uiNumOfAddr,
+									 IMG_CPU_PHYADDR *psCpuPAddr,
+									 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	if (sizeof(*psCpuPAddr) == sizeof(*psDevPAddr))
+	{
+		OSCachedMemCopy(psCpuPAddr, psDevPAddr, uiNumOfAddr * sizeof(*psCpuPAddr));
+		return;
+	}
+
+	/* In this case we may have a 32bit host, so we can't do a memcpy.
+	 * Check we are not dropping any data from the 64bit dev addr */
+	PVR_ASSERT(UI64_TOPWORD_IS_ZERO(psDevPAddr[0].uiAddr));
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (uiNumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < uiNumOfAddr; ++ui32Idx)
+		{
+			PVR_ASSERT(UI64_TOPWORD_IS_ZERO(psDevPAddr[ui32Idx].uiAddr));
+			psCpuPAddr[ui32Idx].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+
+}
+
+static inline
+IMG_CHAR* GetHeapName(PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_LOCAL))    return "lma_gpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_SECURE))   return "lma_gpu_secure";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_GPU_PRIVATE))  return "lma_gpu_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PRIVATE))   return "lma_fw_private";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_SHARED))    return "lma_fw_shared";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_FW_PREMAP_PT)) return "lma_fw_pagetables";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_CPU_LOCAL))    return "lma_cpu_local";
+	if (BITMASK_HAS(ui32Flags,PHYS_HEAP_USAGE_DISPLAY))      return "lma_gpu_display";
+	else                                                     return "Unexpected Heap";
+}
+
+static void TCHybridCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psDevPAddr[ui32Idx].uiAddr =
+		    (psCpuPAddr[ui32Idx].uiAddr - psSysData->pdata->tc_memory_base) +
+		    ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+}
+
+static void TCHybridDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                       IMG_UINT32 ui32NumOfAddr,
+                                       IMG_CPU_PHYADDR *psCpuPAddr,
+                                       IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = (PVRSRV_DEVICE_CONFIG *)hPrivData;
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32NumOfAddr; ui32Idx++)
+	{
+		psCpuPAddr[ui32Idx].uiAddr =
+		    (psDevPAddr[ui32Idx].uiAddr - ODIN_MEMORY_HYBRID_DEVICE_BASE) +
+		    psSysData->pdata->tc_memory_base;
+	}
+}
+
+static PVRSRV_ERROR
+InitLocalHeap(PHYS_HEAP_CONFIG *psPhysHeap,
+			  IMG_UINT64 uiBaseAddr, IMG_UINT64 uiStartAddr,
+			  IMG_UINT64 uiSize, PHYS_HEAP_FUNCTIONS *psFuncs,
+			  PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	psPhysHeap->eType = PHYS_HEAP_TYPE_LMA;
+	psPhysHeap->ui32UsageFlags = ui32Flags;
+	psPhysHeap->uConfig.sLMA.pszPDumpMemspaceName = "LMA";
+	psPhysHeap->uConfig.sLMA.psMemFuncs = psFuncs;
+	psPhysHeap->uConfig.sLMA.pszHeapName = GetHeapName(ui32Flags);
+	psPhysHeap->uConfig.sLMA.sCardBase.uiAddr = uiBaseAddr;
+	psPhysHeap->uConfig.sLMA.sStartAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(uiStartAddr);
+	psPhysHeap->uConfig.sLMA.uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+CreateCardGPUHeaps(const SYS_DATA *psSysData,
+				   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+				   PHYS_HEAP_CONFIG *pasPhysHeaps,
+				   PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				   IMG_UINT32 *pui32HeapIdx,
+				   IMG_UINT64 ui64CardAddr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->rogue_heap_memory_base;
+	IMG_UINT32 ui32SpecIdx;
+
+	for (ui32SpecIdx = 0; ui32SpecIdx < ARRAY_SIZE(gasCardHeapTemplate); ui32SpecIdx++)
+	{
+		if (pasCardHeapSpec[ui32SpecIdx].bUsed)
+		{
+			IMG_UINT64 ui64HeapSize = pasCardHeapSpec[ui32SpecIdx].uiSize;
+
+			eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+								   ui64CardAddr,
+								   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+								   ui64HeapSize,
+								   psHeapFuncs,
+								   pasCardHeapSpec[ui32SpecIdx].ui32UsageFlags);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			ui64CardAddr  += pasCardHeapSpec[ui32SpecIdx].uiSize;
+			ui64StartAddr += pasCardHeapSpec[ui32SpecIdx].uiSize;
+			(*pui32HeapIdx)++;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+static PVRSRV_ERROR
+CreateCardEXTHeap(const SYS_DATA *psSysData,
+				  PHYS_HEAP_CONFIG *pasPhysHeaps,
+				  PHYS_HEAP_FUNCTIONS *psHeapFuncs,
+				  IMG_UINT32 *pui32HeapIdx,
+				  IMG_UINT64 ui64CardBase)
+{
+	IMG_UINT64 ui64StartAddr = psSysData->pdata->pdp_heap_memory_base;
+	IMG_UINT64 ui64Size = psSysData->pdata->pdp_heap_memory_size;
+	PVRSRV_ERROR eError;
+
+	eError = InitLocalHeap(&pasPhysHeaps[*pui32HeapIdx],
+						   ui64CardBase + psSysData->pdata->rogue_heap_memory_size,
+						   IMG_CAST_TO_CPUPHYADDR_UINT(ui64StartAddr),
+						   ui64Size, psHeapFuncs,
+						   PHYS_HEAP_USAGE_EXTERNAL | PHYS_HEAP_USAGE_DISPLAY);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	(*pui32HeapIdx)++;
+
+	return PVRSRV_OK;
+}
+#endif
+
+static PVRSRV_ERROR
+InitLocalHeaps(const SYS_DATA *psSysData,
+			   CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			   PHYS_HEAP_CONFIG *pasPhysHeaps,
+			   IMG_UINT32 *pui32HeapIdx)
+{
+	PHYS_HEAP_FUNCTIONS *psHeapFuncs;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64CardBase;
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHybridPhysHeapFuncs;
+		ui64CardBase = ODIN_MEMORY_HYBRID_DEVICE_BASE;
+	}
+	else if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psHeapFuncs = &gsHostPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->tc_memory_base;
+	}
+	else
+	{
+		psHeapFuncs = &gsLocalPhysHeapFuncs;
+		ui64CardBase = psSysData->pdata->rogue_heap_memory_base - psSysData->pdata->tc_memory_base;
+	}
+
+	eError = CreateCardGPUHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+	eError = CreateCardEXTHeap(psSysData, pasPhysHeaps, psHeapFuncs, pui32HeapIdx, ui64CardBase);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+InitHostHeaps(const SYS_DATA *psSysData, PHYS_HEAP_CONFIG *pasPhysHeaps, IMG_UINT32 *pui32HeapIdx)
+{
+	if (psSysData->pdata->mem_mode != TC_MEMORY_LOCAL)
+	{
+		pasPhysHeaps[*pui32HeapIdx].eType = PHYS_HEAP_TYPE_UMA;
+		pasPhysHeaps[*pui32HeapIdx].ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.psMemFuncs = &gsHostPhysHeapFuncs;
+		pasPhysHeaps[*pui32HeapIdx].uConfig.sUMA.pszHeapName = "uma_cpu_local";
+
+		(*pui32HeapIdx)++;
+
+		PVR_DPF((PVR_DBG_WARNING,
+				 "Initialising CPU_LOCAL UMA Host PhysHeaps with memory mode: %d",
+				 psSysData->pdata->mem_mode));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsInit(const SYS_DATA *psSysData,
+			  CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+			  PHYS_HEAP_CONFIG *pasPhysHeaps,
+			  void *pvPrivData, IMG_UINT32 ui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	IMG_UINT32 ui32HeapCounter = 0;
+
+	eError = InitLocalHeaps(psSysData, pasCardHeapSpec, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	eError = InitHostHeaps(psSysData, pasPhysHeaps, &ui32HeapCounter);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE((ui32HeapCounter == ui32NumHeaps),
+							"Number of PhysHeapConfigs set up doesn't match the initial requirement.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	/* Initialise fields that don't change between memory modes.
+	 * Fix up heap IDs. This is needed for multi-testchip systems to
+	 * ensure the heap IDs are unique as this is what Services expects.
+	 */
+	for (i = 0; i < ui32NumHeaps; i++)
+	{
+		switch (pasPhysHeaps[i].eType)
+		{
+		case PHYS_HEAP_TYPE_UMA:
+			pasPhysHeaps[i].uConfig.sUMA.hPrivData = pvPrivData;
+			break;
+		case PHYS_HEAP_TYPE_LMA:
+			pasPhysHeaps[i].uConfig.sLMA.hPrivData = pvPrivData;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "Invalid PHYS_HEAP_TYPE: %u in %s",
+			                        pasPhysHeaps[i].eType,
+			                        __func__));
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapSetRequirements(const SYS_DATA *psSysData,
+						CARD_PHYS_HEAP_CONFIG_SPEC *pasCardHeapSpec,
+						IMG_UINT32 *pui32CardPhysHeapCfgCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64FreeCardMemory = psSysData->pdata->rogue_heap_memory_size;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_PRIVATE) &&
+		BITMASK_HAS(pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].ui32UsageFlags, PHYS_HEAP_USAGE_GPU_LOCAL),
+		"PhysHeapConfigs not set correctly in the system layer.", PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < ARRAY_SIZE(gasCardHeapTemplate); i++)
+	{
+		if (pasCardHeapSpec[i].bUsed)
+		{
+			/* Determine the memory requirements of heaps with a fixed size */
+			ui64FreeCardMemory -= pasCardHeapSpec[i].uiSize;
+
+			/* Count card physheap configs used by the system */
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	if (SysRestrictGpuLocalAddPrivateHeap())
+	{
+		IMG_UINT64 ui64GpuSharedMem = SysRestrictGpuLocalPhysheap(ui64FreeCardMemory);
+
+		if (ui64GpuSharedMem == ui64FreeCardMemory)
+		{
+			/* No memory reserved for GPU private use, special heap not needed */
+		}
+		else
+		{
+			/* Set up the GPU private heap */
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed = true;
+			pasCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].uiSize = ui64FreeCardMemory - ui64GpuSharedMem;
+			ui64FreeCardMemory = ui64GpuSharedMem;
+			(*pui32CardPhysHeapCfgCount)++;
+		}
+	}
+
+	/* all remaining memory card memory goes to GPU_LOCAL */
+	pasCardHeapSpec[HEAP_SPEC_IDX_GPU_LOCAL].uiSize = ui64FreeCardMemory;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PhysHeapsCreate(const SYS_DATA *psSysData, PVRSRV_DEVICE_CONFIG *psDevConfig,
+				PHYS_HEAP_CONFIG **ppasPhysHeapsOut,
+				IMG_UINT32 *puiPhysHeapCountOut)
+{
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps = 0;
+	CARD_PHYS_HEAP_CONFIG_SPEC asCardHeapSpec[ARRAY_SIZE(gasCardHeapTemplate)];
+
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->mem_mode == TC_MEMORY_LOCAL) ||
+							(psSysData->pdata->mem_mode == TC_MEMORY_HYBRID),
+							"Unsupported memory mode", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	/* Initialise the local heap specs with the build-time template */
+	memcpy(asCardHeapSpec, gasCardHeapTemplate, sizeof(gasCardHeapTemplate));
+
+	eError = PhysHeapSetRequirements(psSysData, asCardHeapSpec, &ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psDevConfig->bHasNonMappableLocalMemory = asCardHeapSpec[HEAP_SPEC_IDX_GPU_PRIVATE].bUsed;
+
+	if (psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* CPU_LOCAL heap */
+		ui32NumHeaps++;
+	}
+
+#if TC_DISPLAY_MEM_SIZE != 0
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL ||
+		psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		/* EXTERNAL / DISPLAY heap */
+		ui32NumHeaps++;
+	}
+#endif
+
+	pasPhysHeaps = OSAllocMem(sizeof(*pasPhysHeaps) * ui32NumHeaps);
+	if (!pasPhysHeaps)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = PhysHeapsInit(psSysData, asCardHeapSpec, pasPhysHeaps, psDevConfig, ui32NumHeaps);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pasPhysHeaps);
+		return eError;
+	}
+
+	*ppasPhysHeapsOut = pasPhysHeaps;
+	*puiPhysHeapCountOut = ui32NumHeaps;
+
+	return PVRSRV_OK;
+}
+
+static void DeviceConfigDestroy(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->pszVersion)
+	{
+		OSFreeMem(psDevConfig->pszVersion);
+	}
+
+	OSFreeMem(psDevConfig->pasPhysHeaps);
+
+	OSFreeMem(psDevConfig);
+}
+
+static void odinTCDevPhysAddr2DmaAddr(PVRSRV_DEVICE_CONFIG *psDevConfig,
+									  IMG_DMA_ADDR *psDmaAddr,
+									  IMG_DEV_PHYADDR *psDevPAddr,
+									  IMG_BOOL *pbValid,
+									  IMG_UINT32 ui32NumAddr,
+									  IMG_BOOL bSparseAlloc)
+{
+	IMG_CPU_PHYADDR sCpuPAddr = {0};
+	IMG_UINT32 ui32Idx;
+
+	/* Fast path */
+	if (!bSparseAlloc)
+	{
+		/* In Odin, DMA address space is the same as host CPU */
+		TCLocalDevPAddrToCpuPAddr(psDevConfig,
+								  1,
+								  &sCpuPAddr,
+								  psDevPAddr);
+		psDmaAddr->uiAddr = sCpuPAddr.uiAddr;
+	}
+	else
+	{
+		for (ui32Idx = 0; ui32Idx < ui32NumAddr; ui32Idx++)
+		{
+			if (pbValid[ui32Idx])
+			{
+				TCLocalDevPAddrToCpuPAddr(psDevConfig,
+										  1,
+										  &sCpuPAddr,
+										  &psDevPAddr[ui32Idx]);
+				psDmaAddr[ui32Idx].uiAddr = sCpuPAddr.uiAddr;
+			}
+			else
+			{
+				/* Invalid DMA address marker */
+				psDmaAddr[ui32Idx].uiAddr = ~((IMG_UINT64)0x0);
+			}
+		}
+	}
+}
+
+static void * odinTCgetCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig, char *name)
+{
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	return tc_dma_chan(psDev->parent, name);
+}
+
+static void odinTCFreeCDMAChan(PVRSRV_DEVICE_CONFIG *psDevConfig,
+							   void* channel)
+{
+
+	struct device* psDev = (struct device*) psDevConfig->pvOSDevice;
+	struct dma_chan *chan = (struct dma_chan*) channel;
+
+	tc_dma_chan_free(psDev->parent, chan);
+}
+
+static void GetDriverMode(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DeviceID;
+
+	/*
+	 * Drivers with virtualization support should check if the mode in which the
+	 * driver must control a device has been explicitly specified at load time
+	 * through module parameters.
+	 * Multi-device platforms must find the internal ID of the device currently
+	 * being created when checking for its associated DriverMode parameter.
+	 */
+	if (PVRSRVAcquireInternalID(&ui32DeviceID) != PVRSRV_OK)
+	{
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		return;
+	}
+
+	if (psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID] == DRIVER_MODE_DEFAULT)
+	{
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		void __iomem *pvRegBase;
+
+		pvRegBase = (void __iomem *) OSMapPhysToLin(psDevConfig->sRegsCpuPBase, psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (pvRegBase == NULL)
+		{
+			/* failed to map register bank, default to native mode */
+			psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+		}
+		else
+		{
+			IMG_UINT64 ui64ClkCtrl;
+
+			/* the CLK_CTRL register is valid only in the Os 0 (Host) register bank
+			 * if it reads 0 then we can conclude this Os is set up to run as Guest */
+#if defined(RGX_CR_CLK_CTRL)
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL);
+#else
+			ui64ClkCtrl = OSReadHWReg64(pvRegBase, RGX_CR_CLK_CTRL1);
+#endif
+			OSUnMapPhysToLin((void __force *) pvRegBase, psDevConfig->ui32RegsSize);
+
+			psDevConfig->eDriverMode = (ui64ClkCtrl != 0) ? (DRIVER_MODE_HOST) : (DRIVER_MODE_GUEST);
+		}
+#else
+		psDevConfig->eDriverMode = DRIVER_MODE_NATIVE;
+#endif
+	}
+	else
+	{
+		psDevConfig->eDriverMode = psPVRSRVData->aeModuleParamDriverMode[ui32DeviceID];
+	}
+}
+
+static PVRSRV_ERROR DeviceConfigCreate(SYS_DATA *psSysData,
+									   PVRSRV_DEVICE_CONFIG **ppsDevConfigOut)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	RGX_DATA *psRGXData;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 uiPhysHeapCount;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	PVR_LOG_RETURN_IF_FALSE((psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+							 psSysData->pdata->mem_mode == TC_MEMORY_LOCAL),
+							"Multidevice virtualization setup supported only on Odin device with TC_MEMORY_LOCAL",
+							PVRSRV_ERROR_INVALID_DEVICE);
+#endif
+
+	psDevConfig = OSAllocZMem(sizeof(*psDevConfig) +
+							  sizeof(*psRGXData) +
+							  sizeof(*psRGXTimingInfo));
+	if (!psDevConfig)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psRGXData = (RGX_DATA *) IMG_OFFSET_ADDR(psDevConfig, sizeof(*psDevConfig));
+	psRGXTimingInfo = (RGX_TIMING_INFORMATION *) IMG_OFFSET_ADDR(psRGXData, sizeof(*psRGXData));
+
+	/* Setup RGX specific timing data */
+#if defined(TC_APOLLO_BONNIE)
+	/* For BonnieTC there seems to be an additional 5x multiplier that occurs to the clock as measured speed is 540Mhz not 108Mhz. */
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent)  * 6 * 5;
+#elif defined(TC_APOLLO_ES2)
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent)  * 6;
+#else
+	psRGXTimingInfo->ui32CoreClockSpeed = tc_core_clock_speed(psSysData->pdev->dev.parent) /
+											tc_core_clock_multiplex(psSysData->pdev->dev.parent);
+#endif
+	psRGXTimingInfo->bEnableActivePM = IMG_FALSE;
+	psRGXTimingInfo->bEnableRDPowIsland = IMG_FALSE;
+	psRGXTimingInfo->ui32ActivePMLatencyms = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/* Set up the RGX data */
+	psRGXData->psRGXTimingInfo = psRGXTimingInfo;
+
+	/* Setup the device config */
+	psDevConfig->pvOSDevice = &psSysData->pdev->dev;
+	psDevConfig->pszName = "tc";
+	psDevConfig->pszVersion = GetDeviceVersionString(psSysData);
+
+	psDevConfig->sRegsCpuPBase.uiAddr = psSysData->registers->start;
+	psDevConfig->ui32RegsSize = resource_size(psSysData->registers);
+
+	PVRSRVAcquireInternalID(&ui32DeviceID);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* Rogue FPGA images are correctly routing the OSID interrupts
+	 * for cores with the IRQ_PER_OS feature */
+	psDevConfig->ui32IRQ = TC_INTERRUPT_OSID0 + ui32DeviceID;
+#else
+	psDevConfig->ui32IRQ = TC_INTERRUPT_EXT;
+#endif
+
+	GetDriverMode(psDevConfig);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* If there is device running in native mode, prevent any attempts at
+	 * creating any Guest devices, as there will be no Host to support them.
+	 * Currently the VZFPGA supports only one physical GPU. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig))
+	{
+		PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+		PVRSRV_DEVICE_NODE *psDN;
+
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		for (psDN = psPVRSRVData->psDeviceNodeList; psDN != NULL; psDN = psDN->psNext)
+		{
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDN))
+			{
+				OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+				PVR_DPF((PVR_DBG_ERROR, "%s() Device %u is already running in native mode, no other Guests supported in the system.",  __func__, psDN->sDevId.ui32InternalID));
+				eError = PVRSRV_ERROR_INVALID_DEVICE;
+				goto ErrorFreeDevConfig;
+			}
+		}
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+#endif
+
+	eError = PhysHeapsCreate(psSysData, psDevConfig, &pasPhysHeaps, &uiPhysHeapCount);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorFreeDevConfig;
+	}
+
+	if (psSysData->pdata->baseboard == TC_BASEBOARD_ODIN &&
+	    psSysData->pdata->mem_mode == TC_MEMORY_HYBRID)
+	{
+		psDevConfig->eDefaultHeap = SysDefaultToCpuLocalHeap() ?
+		    PVRSRV_PHYS_HEAP_CPU_LOCAL : PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+	else
+	{
+		psDevConfig->eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+	}
+
+	psDevConfig->eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+
+	psDevConfig->pasPhysHeaps = pasPhysHeaps;
+	psDevConfig->ui32PhysHeapCount = uiPhysHeapCount;
+
+	/* Only required for LMA but having this always set shouldn't be a problem */
+	psDevConfig->bDevicePA0IsValid = IMG_TRUE;
+
+	psDevConfig->hDevData = psRGXData;
+	psDevConfig->hSysData = psSysData;
+
+#if defined(SUPPORT_ALT_REGBASE)
+	if (psSysData->pdata->mem_mode != TC_MEMORY_LOCAL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: alternative GPU register base is "
+					"supported only in LMA mode", __func__));
+		goto ErrorFreeDevConfig;
+	}
+	else
+	{
+		/* Using display memory base as the alternative GPU register base,
+		 * since the display memory range is not used by the firmware. */
+		IMG_CPU_PHYADDR  sDisplayMemAddr;
+
+		sDisplayMemAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psSysData->pdata->pdp_heap_memory_base);
+		TCLocalCpuPAddrToDevPAddr(psDevConfig, 1,
+		                          &psDevConfig->sAltRegsGpuPBase,
+		                          &sDisplayMemAddr);
+	}
+#endif
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/* Fake DVFS configuration used purely for testing purposes */
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetFrequency = SetFrequency;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.pfnSetVoltage = SetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	psDevConfig->sDVFS.sDVFSDeviceCfg.ui32PollMs = 1000;
+	psDevConfig->sDVFS.sDVFSDeviceCfg.bIdleReq = IMG_TRUE;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32UpThreshold = 90;
+	psDevConfig->sDVFS.sDVFSGovernorCfg.ui32DownDifferential = 10;
+#endif
+
+	/* DMA channel config */
+	psDevConfig->pfnSlaveDMAGetChan = odinTCgetCDMAChan;
+	psDevConfig->pfnSlaveDMAFreeChan = odinTCFreeCDMAChan;
+	psDevConfig->pfnDevPhysAddr2DmaAddr = odinTCDevPhysAddr2DmaAddr;
+	psDevConfig->pszDmaTxChanName = psSysData->pdata->tc_dma_tx_chan_name;
+	psDevConfig->pszDmaRxChanName = psSysData->pdata->tc_dma_rx_chan_name;
+	psDevConfig->bHasDma = IMG_TRUE;
+	/* Following two values are expressed in number of bytes */
+	psDevConfig->ui32DmaTransferUnit = 1;
+	psDevConfig->ui32DmaAlignment = 1;
+
+	*ppsDevConfigOut = psDevConfig;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevConfig:
+	OSFreeMem(psDevConfig);
+	return eError;
+}
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+/* #define _DBG(...) PVR_LOG((__VA_ARGS__)) */
+#define _DBG(...)
+
+static PVRSRV_ERROR PrePower(IMG_HANDLE hSysData,
+                             PVRSRV_SYS_POWER_STATE eNewPowerState,
+                             PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                             PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize;
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags: 0x%08x", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags);
+
+	/* The transition might be both from ON or OFF states to OFF state so check
+	 * only for the *new* state. Also this is only valid for suspend requests. */
+	if (eNewPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = LMA_HeapIteratorCreate(psSysData->psDevConfig->psDevNode,
+	                                PVRSRV_PHYS_HEAP_GPU_LOCAL,
+	                                &psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorCreate", return_error);
+
+	eError = LMA_HeapIteratorGetHeapStats(psSysData->psHeapIter, &uiHeapTotalSize,
+	                                      &uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorGetHeapStats",
+	                      return_error);
+	uiHeapFreeSize = uiHeapTotalSize - uiHeapUsedSize;
+
+	_DBG("(%s()) heap stats: total=0x%" IMG_UINT64_FMTSPECx ", "
+	     "used=0x%" IMG_UINT64_FMTSPECx ", free=0x%" IMG_UINT64_FMTSPECx,
+	     __func__, uiHeapTotalSize, uiHeapUsedSize, uiHeapFreeSize);
+
+	psSysData->pvS3Buffer = OSAllocMem(uiHeapUsedSize);
+	PVR_LOG_GOTO_IF_NOMEM(psSysData->pvS3Buffer, eError, destroy_iterator);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		if (uiOffset + uiSize > uiHeapUsedSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "uiOffset = %" IMG_UINT64_FMTSPECx ", "
+			        "uiSize = %" IMG_UINT64_FMTSPECx, uiOffset, uiSize));
+
+			PVR_LOG_GOTO_WITH_ERROR("LMA_HeapIteratorNext", eError,
+			                        PVRSRV_ERROR_INVALID_OFFSET,
+			                        free_buffer);
+		}
+
+		TCLocalDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                          &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                            PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy((IMG_BYTE *) psSysData->pvS3Buffer + uiOffset, pvCpuVAddr,
+		       uiSize);
+		/* and now poison it */
+		memset(pvCpuVAddr, 0x9b, uiSize);
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+destroy_iterator:
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+return_error:
+	return eError;
+}
+
+static PVRSRV_ERROR PostPower(IMG_HANDLE hSysData,
+                              PVRSRV_SYS_POWER_STATE eNewPowerState,
+                              PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	SYS_DATA *psSysData = (SYS_DATA *) hSysData;
+	IMG_DEV_PHYADDR sDevPAddr = {0};
+	IMG_UINT64 uiSize = 0, uiOffset = 0;
+	PVRSRV_ERROR eError;
+
+	_DBG("(%s()) state: current=%s, new=%s; flags=0x%08x; buffer=%px", __func__,
+	     PVRSRVSysPowerStateToString(eCurrentPowerState),
+	     PVRSRVSysPowerStateToString(eNewPowerState), ePwrFlags,
+	     psSysData->pvS3Buffer);
+
+	/* The transition might be both to ON or OFF states from OFF state so check
+	 * only for the *current* state. Also this is only valid for resume requests. */
+	if (eCurrentPowerState != PVRSRV_SYS_POWER_STATE_OFF ||
+	    !BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ) ||
+	    psSysData->pvS3Buffer == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = LMA_HeapIteratorReset(psSysData->psHeapIter);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LMA_HeapIteratorReset", free_buffer);
+
+	while (LMA_HeapIteratorNext(psSysData->psHeapIter, &sDevPAddr, &uiSize))
+	{
+		void *pvCpuVAddr;
+		IMG_CPU_PHYADDR sCpuPAddr = {0};
+
+		TCLocalDevPAddrToCpuPAddr(psSysData->psDevConfig, 1, &sCpuPAddr,
+		                          &sDevPAddr);
+
+		pvCpuVAddr = OSMapPhysToLin(sCpuPAddr, uiSize,
+		                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+
+		_DBG("(%s()) iterator: dev_paddr=%px, cpu_paddr=%px, cpu_vaddr=%px, "
+		     "size=0x%05" IMG_UINT64_FMTSPECx, __func__,
+		     (void *) sDevPAddr.uiAddr, (void *) sCpuPAddr.uiAddr,
+		     pvCpuVAddr, uiSize);
+
+		/* copy memory */
+		memcpy(pvCpuVAddr, (IMG_BYTE *) psSysData->pvS3Buffer + uiOffset,
+		       uiSize);
+
+		uiOffset += uiSize;
+
+		OSUnMapPhysToLin(pvCpuVAddr, uiSize);
+	}
+
+	LMA_HeapIteratorDestroy(psSysData->psHeapIter);
+	psSysData->psHeapIter = NULL;
+
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+
+	return PVRSRV_OK;
+
+free_buffer:
+	OSFreeMem(psSysData->pvS3Buffer);
+	psSysData->pvS3Buffer = NULL;
+
+	return eError;
+}
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig;
+	SYS_DATA *psSysData;
+	resource_size_t uiRegistersSize;
+	IMG_UINT32 ui32MinRegBankSize;
+	PVRSRV_ERROR eError;
+	int err = 0;
+
+	PVR_ASSERT(pvOSDevice);
+
+	psSysData = OSAllocZMem(sizeof(*psSysData));
+	if (psSysData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psSysData->pdev = to_platform_device((struct device *)pvOSDevice);
+	psSysData->pdata = psSysData->pdev->dev.platform_data;
+
+	/*
+	 * The device cannot address system memory, so there is no DMA
+	 * limitation.
+	 */
+	if (psSysData->pdata->mem_mode == TC_MEMORY_LOCAL)
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(64));
+	}
+	else
+	{
+		dma_set_mask(pvOSDevice, DMA_BIT_MASK(32));
+	}
+
+	err = tc_enable(psSysData->pdev->dev.parent);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to enable PCI device (%d)", __func__, err));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+		goto ErrFreeSysData;
+	}
+
+	psSysData->registers = platform_get_resource_byname(psSysData->pdev,
+														IORESOURCE_MEM,
+														"rogue-regs");
+	if (!psSysData->registers)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get Rogue register information",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		goto ErrorDevDisable;
+	}
+
+	/* Check the address range is large enough. */
+	uiRegistersSize = resource_size(psSysData->registers);
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* each GPU instance gets the minimum 64kb register range */
+	ui32MinRegBankSize = RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE;
+#else
+	/* the GPU gets the entire 64MB IO range */
+	ui32MinRegBankSize = SYS_RGX_REG_REGION_SIZE;
+#endif
+
+	if (uiRegistersSize < ui32MinRegBankSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register region isn't big enough (was %pa, required 0x%08x)",
+				 __func__, &uiRegistersSize, ui32MinRegBankSize));
+
+		eError = PVRSRV_ERROR_PCI_REGION_TOO_SMALL;
+		goto ErrorDevDisable;
+	}
+
+	/* Reserve the address range */
+	if (!request_mem_region(psSysData->registers->start,
+							resource_size(psSysData->registers),
+							SYS_RGX_DEV_NAME))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Rogue register memory region not available",
+				 __func__));
+		eError = PVRSRV_ERROR_PCI_CALL_FAILED;
+
+		goto ErrorDevDisable;
+	}
+
+	eError = DeviceConfigCreate(psSysData, &psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseMemRegion;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	eError = IonInit(psSysData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise ION", __func__));
+		goto ErrorDeviceConfigDestroy;
+	}
+#endif
+
+	/* Set psDevConfig->pfnSysDevErrorNotify callback */
+	psDevConfig->pfnSysDevErrorNotify = SysRGXErrorNotify;
+
+#if defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__)
+	/* power functions */
+	psDevConfig->pfnPrePowerState = PrePower;
+	psDevConfig->pfnPostPowerState = PostPower;
+
+	psSysData->psDevConfig = psDevConfig;
+#endif /* defined(SUPPORT_LMA_SUSPEND_TO_RAM) && defined(__x86_64__) */
+
+	*ppsDevConfig = psDevConfig;
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+ErrorDeviceConfigDestroy:
+	DeviceConfigDestroy(psDevConfig);
+#endif
+ErrorReleaseMemRegion:
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+ErrorDevDisable:
+	tc_disable(psSysData->pdev->dev.parent);
+ErrFreeSysData:
+	OSFreeMem(psSysData);
+	return eError;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)psDevConfig->hSysData;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	IonDeinit();
+#endif
+
+	DeviceConfigDestroy(psDevConfig);
+
+	release_mem_region(psSysData->registers->start,
+					   resource_size(psSysData->registers));
+	tc_disable(psSysData->pdev->dev.parent);
+
+	OSFreeMem(psSysData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+#if defined(TC_APOLLO_TCF5)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	return PVRSRV_OK;
+#else
+	SYS_DATA *psSysData = psDevConfig->hSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	u32 tmp = 0;
+	u32 pll;
+
+	PVR_DUMPDEBUG_LOG("------[ rgx_tc system debug ]------");
+
+	if (tc_sys_info(psSysData->pdev->dev.parent, &tmp, &pll))
+		goto err_out;
+
+	if (tmp > 0)
+		PVR_DUMPDEBUG_LOG("Chip temperature: %d degrees C", tmp);
+	PVR_DUMPDEBUG_LOG("PLL status: %x", pll);
+
+err_out:
+	return eError;
+#endif
+}
+
+typedef struct
+{
+	struct device *psDev;
+	int iInterruptID;
+	void *pvData;
+	PFN_LISR pfnLISR;
+} LISR_DATA;
+
+static void TCInterruptHandler(void* pvData)
+{
+	LISR_DATA *psLISRData = pvData;
+	psLISRData->pfnLISR(psLISRData->pvData);
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+				  IMG_UINT32 ui32IRQ,
+				  const IMG_CHAR *pszName,
+				  PFN_LISR pfnLISR,
+				  void *pvData,
+				  IMG_HANDLE *phLISRData)
+{
+	SYS_DATA *psSysData = (SYS_DATA *)hSysData;
+	LISR_DATA *psLISRData;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if ((ui32IRQ != TC_INTERRUPT_EXT) &&
+		(ui32IRQ < TC_INTERRUPT_OSID0) && (ui32IRQ > TC_INTERRUPT_OSID7))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device matching IRQ %d", __func__, ui32IRQ));
+		return PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+	}
+
+	psLISRData = OSAllocZMem(sizeof(*psLISRData));
+	if (!psLISRData)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_out;
+	}
+
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+	psLISRData->iInterruptID = ui32IRQ;
+	psLISRData->psDev = psSysData->pdev->dev.parent;
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, TCInterruptHandler, psLISRData);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_free_data;
+	}
+
+	err = tc_enable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_enable_interrupt() failed (%d)", __func__, err));
+		eError = PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR;
+		goto err_unset_interrupt_handler;
+	}
+
+	*phLISRData = psLISRData;
+	eError = PVRSRV_OK;
+
+	PVR_TRACE(("Installed device LISR " IMG_PFN_FMTSPEC " with tc module to ID %u",
+			   pfnLISR, ui32IRQ));
+
+err_out:
+	return eError;
+err_unset_interrupt_handler:
+	tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+err_free_data:
+	OSFreeMem(psLISRData);
+	goto err_out;
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *) hLISRData;
+	int err;
+
+	err = tc_disable_interrupt(psLISRData->psDev, psLISRData->iInterruptID);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_disable_interrupt() failed (%d)", __func__, err));
+	}
+
+	err = tc_set_interrupt_handler(psLISRData->psDev, psLISRData->iInterruptID, NULL, NULL);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: tc_set_interrupt_handler() failed (%d)", __func__, err));
+	}
+
+	PVR_TRACE(("Uninstalled device LISR " IMG_PFN_FMTSPEC " with tc module from ID %u",
+			   psLISRData->pfnLISR, psLISRData->iInterruptID));
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
+
+/****************************************************************************************************/
+/****                                   VM migration test code                                   ****/
+/****************************************************************************************************/
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode);
+static void PreMigrationDeviceSuspend(struct drm_device *psDev);
+static void PostMigrationDeviceResume(struct drm_device *psDev);
+
+void PVRVMMigration(unsigned int src, unsigned int dest);
+EXPORT_SYMBOL(PVRVMMigration);
+
+#define SWAP_REGSBASE_PTR(a, b) do \
+	{ \
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		b = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+		a = (void __iomem *)(((uintptr_t)a)^((uintptr_t)b));	\
+	} while (0)
+
+static void SwapHyperlanes(PVRSRV_DEVICE_NODE *psSrcNode, PVRSRV_DEVICE_NODE *psDestNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psHostNode = PVRSRVGetDeviceInstance(0);
+	PVRSRV_RGXDEV_INFO *psSrcInfo = psSrcNode->pvDevice;
+	PVRSRV_RGXDEV_INFO *psDestInfo = psDestNode->pvDevice;
+	PVRSRV_DEVICE_CONFIG *psSrcConfig = psSrcNode->psDevConfig;
+	PVRSRV_DEVICE_CONFIG *psDestConfig = psDestNode->psDevConfig;
+	LISR_DATA *psSrcLISRData = (LISR_DATA *) psSrcInfo->pvLISRData;
+	void *pfnLISR = psSrcLISRData->pfnLISR;
+	IMG_UINT32 ui32SrcHyperLane, ui32DestHyperLane;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psHostNode != NULL) &&
+								  (psHostNode->psDevConfig != NULL)),
+								 "Device 0 (expected Host) not initialised.");
+
+	/* Determine the HyperLane ID used by a Guest Device from the Register Bank Base address used */
+	ui32SrcHyperLane = (psSrcConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psSrcConfig->ui32RegsSize;
+	ui32DestHyperLane = (psDestConfig->sRegsCpuPBase.uiAddr - psHostNode->psDevConfig->sRegsCpuPBase.uiAddr) / psDestConfig->ui32RegsSize;
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Swapping hyperlanes between Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32SrcHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32DestHyperLane));
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resulting configuration:    Dev%u (hyperlane%u) and Dev%u (hyperlane%u)", __func__,
+							psSrcNode->sDevId.ui32InternalID, ui32DestHyperLane,
+							psDestNode->sDevId.ui32InternalID, ui32SrcHyperLane));
+
+	/* swap the register bank details */
+	SWAP_REGSBASE_PTR(psSrcInfo->pvRegsBaseKM, psDestInfo->pvRegsBaseKM);
+	SWAP(psSrcConfig->sRegsCpuPBase.uiAddr, psDestConfig->sRegsCpuPBase.uiAddr);
+	/* DevConfig->ui32RegsSize remains the same */
+
+	/* Swap interrupt lines between devices */
+	eError = SysUninstallDeviceLISR(psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+	eError = SysUninstallDeviceLISR(psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysUninstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	SWAP(psSrcConfig->ui32IRQ, psDestConfig->ui32IRQ);
+
+	eError = SysInstallDeviceLISR(psSrcConfig->hSysData,
+								  psSrcConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psSrcNode,
+								  &psSrcInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psSrcConfig->ui32IRQ, ui32SrcHyperLane);
+
+	eError = SysInstallDeviceLISR(psDestConfig->hSysData,
+								  psDestConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  pfnLISR,
+								  psDestNode,
+								  &psDestInfo->pvLISRData);
+	PVR_LOG_IF_ERROR_VA(PVR_DBG_ERROR, eError, "SysInstallDeviceLISR(IRQ%u, Device %u)",
+												psDestConfig->ui32IRQ, ui32DestHyperLane);
+
+	/* Swap contents of LMA carveouts between virtual devices */
+	{
+		/* Guest Raw Fw Heap mapping is done using the Host Devices */
+		PHYS_HEAP *psSrcHeap = NULL;
+		PHYS_HEAP *psDestHeap = NULL;
+		IMG_DEV_PHYADDR sSrcHeapBase, sDestHeapBase;
+
+		psSrcHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32SrcHyperLane];
+		psDestHeap = psHostNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DestHyperLane];
+
+		PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcHeap != NULL) &&
+									  (psDestHeap != NULL)),
+									 "Guest firmware heaps not premapped by the Host Device.");
+
+		eError = PhysHeapGetDevPAddr(psSrcHeap, &sSrcHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(src fw heap)");
+		eError = PhysHeapGetDevPAddr(psDestHeap, &sDestHeapBase);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PhysHeapGetDevPAddr(dest fw heap)");
+
+		eError = PvzServerUnmapDevPhysHeap(ui32SrcHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(src fw heap)");
+		eError = PvzServerUnmapDevPhysHeap(ui32DestHyperLane, 0);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap(dest fw heap)");
+
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32SrcHyperLane]);
+		PhysHeapRelease(psHostNode->apsFWPremapPhysHeap[ui32DestHyperLane]);
+
+		/* create new heaps with new base addresses */
+		eError = PvzServerMapDevPhysHeap(ui32SrcHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sDestHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(src fw heap)");
+		eError = PvzServerMapDevPhysHeap(ui32DestHyperLane, 0, RGX_FIRMWARE_RAW_HEAP_SIZE, sSrcHeapBase.uiAddr);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PvzServerMapDevPhysHeap(dest fw heap)");
+	}
+}
+
+static void PreMigrationDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(eError == PVRSRV_OK,
+	                           "LinuxBridgeBlockClientsAccess()");
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * be declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+	}
+}
+
+static void PostMigrationDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+									PVRSRV_SYS_POWER_STATE_ON,
+									PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ);
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+}
+
+void PVRVMMigration(unsigned int src, unsigned int dest)
+{
+	PVRSRV_DEVICE_NODE *psSrcNode = PVRSRVGetDeviceInstance(src);
+	PVRSRV_DEVICE_NODE *psDestNode = PVRSRVGetDeviceInstance(dest);
+	struct device *psSrcDev, *psDestDev;
+	struct drm_device *psSrcDrmDev, *psDestDrmDev;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode != NULL) && (psDestNode != NULL) && (psSrcNode != psDestNode)),
+								 "Invalid Device IDs requested for migration.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE) &&
+								  (psDestNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)),
+								 "Devices not fully initialised.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig != NULL) &&
+								  (psDestNode->psDevConfig != NULL)),
+								 "Device config structure is NULL.");
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcNode->psDevConfig->pvOSDevice != NULL) &&
+								  (psDestNode->psDevConfig->pvOSDevice != NULL)),
+								 "Linux kernel device pointer is NULL.");
+
+	psSrcDev = psSrcNode->psDevConfig->pvOSDevice;
+	psDestDev = psDestNode->psDevConfig->pvOSDevice;
+	psSrcDrmDev = dev_get_drvdata(psSrcDev);
+	psDestDrmDev = dev_get_drvdata(psDestDev);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(((psSrcDrmDev != NULL) &&
+								  (psDestDrmDev != NULL)),
+								 "Linux kernel drm_device pointer is NULL.");
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psSrcNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psSrcDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Suspending device %u before migration",
+							__func__, psDestNode->sDevId.ui32InternalID));
+	PreMigrationDeviceSuspend(psDestDrmDev);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Migrating vGPU resources (regbank, irq, osid)", __func__));
+	SwapHyperlanes(psSrcNode, psDestNode);
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psSrcNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psSrcDrmDev);
+	PVR_DPF((PVR_DBG_WARNING, "%s: Resuming device %u", __func__,
+								psDestNode->sDevId.ui32InternalID));
+	PostMigrationDeviceResume(psDestDrmDev);
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/sysinfo.h b/drivers/gpu/drm/img-rogue/apollo/sysinfo.h
new file mode 100644
index 000000000000..6355a98482c3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/sysinfo.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#if defined(VIRTUAL_PLATFORM) || defined(FPGA)
+#define MAX_HW_TIME_US                           (240000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (120000)
+#else
+#define MAX_HW_TIME_US                           (500000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (1500)//(10000)
+#endif
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+
+#if defined(VIRTUAL_PLATFORM)
+#define EVENT_OBJECT_TIMEOUT_US                  (120000000)
+#elif defined(TC_APOLLO_TCF5)
+#define EVENT_OBJECT_TIMEOUT_US                  (2000000)
+#else
+#define EVENT_OBJECT_TIMEOUT_US                  (100000)
+#endif
+
+#define SYS_RGX_DEV_NAME "tc_rogue"
+
+#endif /* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_apollo.c b/drivers/gpu/drm/img-rogue/apollo/tc_apollo.c
new file mode 100644
index 000000000000..618c017b8c0f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_apollo.c
@@ -0,0 +1,1567 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This is a device driver for the apollo testchip framework. It creates
+ * platform devices for the pdp and ext sub-devices, and exports functions to
+ * manage the shared interrupt handling
+ */
+
+#include <linux/version.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/platform_device.h>
+#include <linux/version.h>
+#include <linux/thermal.h>
+
+#include "tc_drv_internal.h"
+#include "tc_apollo.h"
+
+#if defined(SUPPORT_DMA_HEAP)
+#include "tc_dmabuf_heap.h"
+#elif defined(SUPPORT_ION)
+#include "tc_ion.h"
+#endif
+
+#include "apollo_regs.h"
+#include "tcf_clk_ctrl.h"
+#include "tcf_pll.h"
+#include "tc_clocks.h"
+
+#if defined(SUPPORT_APOLLO_FPGA)
+#include "tc_apollo_debugfs.h"
+#endif /* defined(SUPPORT_APOLLO_FPGA) */
+
+/*
+ * kernel_compatibility.h: This header is a special case and should always be
+ * the last file included, as it can affect definitions/declarations in files
+ * included after it.
+ */
+#include "kernel_compatibility.h"
+
+#define TC_INTERRUPT_FLAG_PDP      (1 << PDP1_INT_SHIFT)
+#define TC_INTERRUPT_FLAG_EXT      (1 << EXT_INT_SHIFT)
+
+#define PCI_VENDOR_ID_POWERVR      0x1010
+#define DEVICE_ID_PCI_APOLLO_FPGA  0x1CF1
+#define DEVICE_ID_PCIE_APOLLO_FPGA 0x1CF2
+
+#define APOLLO_MEM_PCI_BASENUM	   (2)
+
+static struct {
+	struct thermal_zone_device *thermal_zone;
+
+#if defined(SUPPORT_APOLLO_FPGA)
+	struct tc_io_region fpga;
+	struct apollo_debugfs_fpga_entries fpga_entries;
+#endif
+} apollo_pdata;
+
+#if defined(SUPPORT_APOLLO_FPGA)
+
+#define APOLLO_DEVICE_NAME_FPGA "apollo_fpga"
+
+struct apollo_fpga_platform_data {
+	/* The testchip memory mode (LMA, HOST or HYBRID) */
+	int mem_mode;
+
+	resource_size_t tc_memory_base;
+
+	resource_size_t pdp_heap_memory_base;
+	resource_size_t pdp_heap_memory_size;
+};
+
+#endif /* defined(SUPPORT_APOLLO_FPGA) */
+
+static void spi_write(struct tc_device *tc, u32 off, u32 val)
+{
+	iowrite32(off, tc->tcf.registers
+		  + TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR);
+	iowrite32(val, tc->tcf.registers
+		  + TCF_CLK_CTRL_TCF_SPI_MST_WDATA);
+	iowrite32(TCF_SPI_MST_GO_MASK, tc->tcf.registers
+		  + TCF_CLK_CTRL_TCF_SPI_MST_GO);
+	udelay(1000);
+}
+
+static int spi_read(struct tc_device *tc, u32 off, u32 *val)
+{
+	int cnt = 0;
+	u32 spi_mst_status;
+
+	iowrite32(0x40000 | off, tc->tcf.registers
+		  + TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR);
+	iowrite32(TCF_SPI_MST_GO_MASK, tc->tcf.registers
+		  + TCF_CLK_CTRL_TCF_SPI_MST_GO);
+
+	udelay(100);
+
+	do {
+		spi_mst_status = ioread32(tc->tcf.registers
+					  + TCF_CLK_CTRL_TCF_SPI_MST_STATUS);
+
+		if (cnt++ > 10000) {
+			dev_err(&tc->pdev->dev,
+				"%s: Time out reading SPI reg (0x%x)\n",
+				__func__, off);
+			return -1;
+		}
+
+	} while (spi_mst_status != 0x08);
+
+	*val = ioread32(tc->tcf.registers
+			+ TCF_CLK_CTRL_TCF_SPI_MST_RDATA);
+
+	return 0;
+}
+
+static int apollo_thermal_get_temp(struct thermal_zone_device *thermal,
+				   int *t)
+{
+	struct tc_device *tc;
+	int err = -ENODEV;
+	u32 tmp;
+
+	if (!thermal)
+		goto err_out;
+
+	tc = (struct tc_device *)thermal->devdata;
+
+	if (!tc)
+		goto err_out;
+
+	if (spi_read(tc, TCF_TEMP_SENSOR_SPI_OFFSET, &tmp)) {
+		dev_err(&tc->pdev->dev,
+				"Failed to read apollo temperature sensor\n");
+
+		goto err_out;
+	}
+
+	/* Report this in millidegree Celsius */
+	*t = TCF_TEMP_SENSOR_TO_C(tmp) * 1000;
+
+	err = 0;
+
+err_out:
+	return err;
+}
+
+static struct thermal_zone_device_ops apollo_thermal_dev_ops = {
+	.get_temp = apollo_thermal_get_temp,
+};
+
+#if defined(SUPPORT_RGX)
+
+static void pll_write_reg(struct tc_device *tc,
+	resource_size_t reg_offset, u32 reg_value)
+{
+	BUG_ON(reg_offset < TCF_PLL_PLL_CORE_CLK0);
+	BUG_ON(reg_offset > tc->tcf_pll.region.size +
+		TCF_PLL_PLL_CORE_CLK0 - 4);
+
+	/* Tweak the offset because we haven't mapped the full pll region */
+	iowrite32(reg_value, tc->tcf_pll.registers +
+		reg_offset - TCF_PLL_PLL_CORE_CLK0);
+}
+
+static u32 sai_read_es2(struct tc_device *tc, u32 addr)
+{
+	iowrite32(0x200 | addr, tc->tcf.registers + 0x300);
+	iowrite32(0x1 | addr, tc->tcf.registers + 0x318);
+	return ioread32(tc->tcf.registers + 0x310);
+}
+
+static int apollo_align_interface_es2(struct tc_device *tc)
+{
+	u32 reg = 0;
+	u32 reg_reset_n;
+	int reset_cnt = 0;
+	int err = -EFAULT;
+	bool aligned = false;
+
+	/* Try to enable the core clock PLL */
+	spi_write(tc, 0x1, 0x0);
+	reg  = ioread32(tc->tcf.registers + 0x320);
+	reg |= 0x1;
+	iowrite32(reg, tc->tcf.registers + 0x320);
+	reg &= 0xfffffffe;
+	iowrite32(reg, tc->tcf.registers + 0x320);
+	msleep(1000);
+
+	if (spi_read(tc, 0x2, &reg)) {
+		dev_err(&tc->pdev->dev,
+				"Unable to read PLL status\n");
+		goto err_out;
+	}
+
+	if (reg == 0x1) {
+		/* Select DUT PLL as core clock */
+		reg  = ioread32(tc->tcf.registers +
+			TCF_CLK_CTRL_DUT_CONTROL_1);
+		reg &= 0xfffffff7;
+		iowrite32(reg, tc->tcf.registers +
+			TCF_CLK_CTRL_DUT_CONTROL_1);
+	} else {
+		dev_err(&tc->pdev->dev,
+			"PLL has failed to lock, status = %x\n", reg);
+		goto err_out;
+	}
+
+	reg_reset_n = ioread32(tc->tcf.registers +
+		TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+
+	while (!aligned && reset_cnt < 10 &&
+			tc->version != APOLLO_VERSION_TCF_5) {
+		int bank;
+		u32 eyes;
+		u32 clk_taps;
+		u32 train_ack;
+
+		++reset_cnt;
+
+		/* Reset the DUT to allow the SAI to retrain */
+		reg_reset_n &= ~(0x1 << DUT_RESETN_SHIFT);
+		iowrite32(reg_reset_n, tc->tcf.registers +
+			  TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+		udelay(100);
+		reg_reset_n |= (0x1 << DUT_RESETN_SHIFT);
+		iowrite32(reg_reset_n, tc->tcf.registers +
+			  TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+		udelay(100);
+
+		/* Assume alignment passed, if any bank fails on either DUT or
+		 * FPGA we will set this to false and try again for a max of 10
+		 * times.
+		 */
+		aligned = true;
+
+		/* For each of the banks */
+		for (bank = 0; bank < 10; bank++) {
+			int bank_aligned = 0;
+			/* Check alignment on the DUT */
+			u32 bank_base = 0x7000 + (0x1000 * bank);
+
+			spi_read(tc, bank_base + 0x4, &eyes);
+			spi_read(tc, bank_base + 0x3, &clk_taps);
+			spi_read(tc, bank_base + 0x6, &train_ack);
+
+			bank_aligned = tc_is_interface_aligned(
+					eyes, clk_taps, train_ack);
+			if (!bank_aligned) {
+				dev_warn(&tc->pdev->dev,
+					"Alignment check failed, retrying\n");
+				aligned = false;
+				break;
+			}
+
+			/* Check alignment on the FPGA */
+			bank_base = 0xb0 + (0x10 * bank);
+
+			eyes = sai_read_es2(tc, bank_base + 0x4);
+			clk_taps = sai_read_es2(tc, bank_base + 0x3);
+			train_ack = sai_read_es2(tc, bank_base + 0x6);
+
+			bank_aligned = tc_is_interface_aligned(
+					eyes, clk_taps, train_ack);
+
+			if (!bank_aligned) {
+				dev_warn(&tc->pdev->dev,
+					"Alignment check failed, retrying\n");
+				aligned = false;
+				break;
+			}
+		}
+	}
+
+	if (!aligned) {
+		dev_err(&tc->pdev->dev, "Unable to initialise the testchip (interface alignment failure), please restart the system.\n");
+		/* We are not returning an error here, cause VP doesn't
+		 * implement the necessary registers although they claim to be
+		 * TC compatible.
+		 */
+	}
+
+	if (reset_cnt > 1) {
+		dev_dbg(&tc->pdev->dev, "Note: The testchip required more than one reset to find a good interface alignment!\n");
+		dev_dbg(&tc->pdev->dev, "      This should be harmless, but if you do suspect foul play, please reset the machine.\n");
+		dev_dbg(&tc->pdev->dev, "      If you continue to see this message you may want to report it to PowerVR Verification Platforms.\n");
+	}
+
+	err = 0;
+err_out:
+	return err;
+}
+
+static void apollo_set_clocks(struct tc_device *tc,
+			      int core_clock, int mem_clock, int sys_clock)
+{
+	u32 val;
+
+	/* This is disabled for TCF2 since the current FPGA builds do not
+	 * like their core clocks being set (it takes apollo down).
+	 */
+	if (tc->version != APOLLO_VERSION_TCF_2) {
+		val = core_clock / 1000000;
+		pll_write_reg(tc, TCF_PLL_PLL_CORE_CLK0, val);
+
+		val = 0x1 << PLL_CORE_DRP_GO_SHIFT;
+		pll_write_reg(tc, TCF_PLL_PLL_CORE_DRP_GO, val);
+	}
+
+	val = mem_clock / 1000000;
+	pll_write_reg(tc, TCF_PLL_PLL_MEMIF_CLK0, val);
+
+	val = 0x1 << PLL_MEM_DRP_GO_SHIFT;
+	pll_write_reg(tc, TCF_PLL_PLL_MEM_DRP_GO, val);
+
+	if (tc->version == APOLLO_VERSION_TCF_5) {
+		val = sys_clock / 1000000;
+		pll_write_reg(tc, TCF_PLL_PLL_SYSIF_CLK0, val);
+
+		val = 0x1 << PLL_MEM_DRP_GO_SHIFT;
+		pll_write_reg(tc, TCF_PLL_PLL_SYS_DRP_GO, val);
+	}
+
+	udelay(400);
+}
+
+static void apollo_set_mem_latency(struct tc_device *tc,
+				   int mem_latency, int mem_wresp_latency)
+{
+	u32 regval = 0;
+
+	if (mem_latency <= 4) {
+		/* The total memory read latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a read.
+		 * Set the memory read latency to 0 cycles.
+		 */
+		mem_latency = 0;
+	} else {
+		mem_latency -= 4;
+
+		dev_info(&tc->pdev->dev,
+			 "Setting memory read latency to %i cycles\n",
+			 mem_latency);
+	}
+
+	if (mem_wresp_latency <= 2) {
+		/* The total memory write latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a write.
+		 * Set the memory write latency to 0 cycles.
+		 */
+		mem_wresp_latency = 0;
+	} else {
+		mem_wresp_latency -= 2;
+
+		dev_info(&tc->pdev->dev,
+			 "Setting memory write response latency to %i cycles\n",
+			 mem_wresp_latency);
+	}
+
+	mem_latency |= mem_wresp_latency << 16;
+
+	spi_write(tc, 0x1009, mem_latency);
+
+	if (spi_read(tc, 0x1009, &regval) != 0) {
+		dev_err(&tc->pdev->dev,
+			"Failed to read back memory latency register");
+		return;
+	}
+
+	if (mem_latency != regval) {
+		dev_err(&tc->pdev->dev,
+			"Memory latency register doesn't match requested value (actual: %#08x, expected: %#08x)\n",
+			regval, mem_latency);
+	}
+}
+
+static void apollo_fpga_update_dut_clk_freq(struct tc_device *tc,
+					    int *core_clock, int *mem_clock, int *clock_multiplex)
+{
+	struct device *dev = &tc->pdev->dev;
+	u32 reg = 0;
+
+#if defined(SUPPORT_FPGA_DUT_CLK_INFO)
+	/* DUT_CLK_INFO available only if SW_IF_VERSION >= 1 */
+	reg = ioread32(tc->tcf.registers + TCF_CLK_CTRL_SW_IF_VERSION);
+	reg = (reg & VERSION_MASK) >> VERSION_SHIFT;
+#endif
+	if (reg >= 1) {
+		reg = ioread32(tc->tcf.registers + TCF_CLK_CTRL_DUT_CLK_INFO);
+
+		if ((reg != 0) && (reg != 0xbaadface) && (reg != 0xffffffff)) {
+			dev_info(dev, "TCF_CLK_CTRL_DUT_CLK_INFO = %08x\n", reg);
+
+			if (*core_clock == 0) {
+				*core_clock = ((reg & CORE_MASK) >> CORE_SHIFT) * 1000000;
+				dev_info(dev, "Using register DUT core clock value: %i\n",
+							*core_clock);
+			} else {
+				dev_info(dev, "Using module param DUT core clock value: %i\n",
+							*core_clock);
+			}
+
+			if (*mem_clock == 0) {
+				*mem_clock = ((reg & MEM_MASK) >> MEM_SHIFT) * 1000000;
+				dev_info(dev, "Using register DUT mem clock value: %i\n",
+				 *mem_clock);
+			} else {
+				dev_info(dev, "Using module param DUT mem clock value: %i\n",
+							*mem_clock);
+			}
+
+			return;
+		}
+	}
+
+	if (*core_clock == 0) {
+		*core_clock = RGX_TC_CORE_CLOCK_SPEED;
+		dev_info(dev, "Using default DUT core clock value: %i\n",
+				 *core_clock);
+	} else {
+		dev_info(dev, "Using module param DUT core clock value: %i\n",
+					*core_clock);
+	}
+
+	if (*mem_clock == 0) {
+		*mem_clock = RGX_TC_MEM_CLOCK_SPEED;
+		dev_info(dev, "Using default DUT mem clock value: %i\n",
+				 *mem_clock);
+	} else {
+		dev_info(dev, "Using module param DUT mem clock value: %i\n",
+					*mem_clock);
+	}
+
+	if (*clock_multiplex == 0) {
+		*clock_multiplex = RGX_TC_CLOCK_MULTIPLEX;
+		dev_info(dev, "Using default DUT clock multiplex: %i\n",
+				 *clock_multiplex);
+	} else {
+		dev_info(dev, "Using module param DUT clock multiplex: %i\n",
+					*clock_multiplex);
+	}
+}
+
+#endif /* defined(SUPPORT_RGX) */
+
+static int apollo_hard_reset(struct tc_device *tc,
+			     int *core_clock, int *mem_clock, int sys_clock, int *clock_multiplex)
+{
+	u32 reg;
+	u32 reg_reset_n = 0;
+
+	int err = 0;
+
+	/* This is required for SPI reset which is not yet implemented. */
+	/*u32 aux_reset_n;*/
+
+	if (tc->version == APOLLO_VERSION_TCF_2) {
+		/* Power down */
+		reg = ioread32(tc->tcf.registers +
+			TCF_CLK_CTRL_DUT_CONTROL_1);
+		reg &= ~DUT_CTRL_VCC_0V9EN;
+		reg &= ~DUT_CTRL_VCC_1V8EN;
+		reg |= DUT_CTRL_VCC_IO_INH;
+		reg |= DUT_CTRL_VCC_CORE_INH;
+		iowrite32(reg, tc->tcf.registers +
+			TCF_CLK_CTRL_DUT_CONTROL_1);
+		msleep(500);
+	}
+
+	/* Put everything into reset */
+	iowrite32(reg_reset_n, tc->tcf.registers +
+		TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+
+	/* Take PDP1 and PDP2 out of reset */
+	reg_reset_n |= (0x1 << PDP1_RESETN_SHIFT);
+	reg_reset_n |= (0x1 << PDP2_RESETN_SHIFT);
+
+	iowrite32(reg_reset_n, tc->tcf.registers +
+		TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+	msleep(100);
+
+	/* Take DDR out of reset */
+	reg_reset_n |= (0x1 << DDR_RESETN_SHIFT);
+	iowrite32(reg_reset_n, tc->tcf.registers +
+		  TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+
+#if defined(SUPPORT_RGX)
+	if (tc->version == APOLLO_VERSION_TCF_5) {
+		apollo_fpga_update_dut_clk_freq(tc, core_clock, mem_clock, clock_multiplex);
+	} else {
+		struct device *dev = &tc->pdev->dev;
+
+		if (*core_clock == 0) {
+			*core_clock = RGX_TC_CORE_CLOCK_SPEED;
+			dev_info(dev, "Using default DUT core clock value: %i\n",
+					 *core_clock);
+		} else {
+			dev_info(dev, "Using module param DUT core clock value: %i\n",
+						*core_clock);
+		}
+
+		if (*mem_clock == 0) {
+			*mem_clock = RGX_TC_MEM_CLOCK_SPEED;
+			dev_info(dev, "Using default DUT mem clock value: %i\n",
+					 *mem_clock);
+		} else {
+			dev_info(dev, "Using module param DUT mem clock value: %i\n",
+						*mem_clock);
+		}
+
+		if (*clock_multiplex == 0) {
+			*clock_multiplex = RGX_TC_CLOCK_MULTIPLEX;
+			dev_info(dev, "Using default DUT clock multiplex: %i\n",
+					 *clock_multiplex);
+		} else {
+			dev_info(dev, "Using module param DUT clock multiplex: %i\n",
+						*clock_multiplex);
+		}
+	}
+
+	/* Set clock speed here, before reset. */
+	apollo_set_clocks(tc, *core_clock, *mem_clock, sys_clock);
+
+	/* Put take GLB_CLKG and SCB out of reset */
+	reg_reset_n |= (0x1 << GLB_CLKG_EN_SHIFT);
+	reg_reset_n |= (0x1 << SCB_RESETN_SHIFT);
+	iowrite32(reg_reset_n, tc->tcf.registers +
+		  TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+	msleep(100);
+
+	if (tc->version == APOLLO_VERSION_TCF_2) {
+		/* Enable the voltage control regulators on DUT */
+		reg = ioread32(tc->tcf.registers +
+			TCF_CLK_CTRL_DUT_CONTROL_1);
+		reg |= DUT_CTRL_VCC_0V9EN;
+		reg |= DUT_CTRL_VCC_1V8EN;
+		reg &= ~DUT_CTRL_VCC_IO_INH;
+		reg &= ~DUT_CTRL_VCC_CORE_INH;
+		iowrite32(reg, tc->tcf.registers +
+			TCF_CLK_CTRL_DUT_CONTROL_1);
+		msleep(300);
+	}
+
+	/* Take DUT_DCM out of reset */
+	reg_reset_n |= (0x1 << DUT_DCM_RESETN_SHIFT);
+	iowrite32(reg_reset_n, tc->tcf.registers +
+		  TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+	msleep(100);
+
+
+	err = tc_iopol32_nonzero(DCM_LOCK_STATUS_MASK,
+		tc->tcf.registers + TCF_CLK_CTRL_DCM_LOCK_STATUS);
+
+	if (err != 0)
+		goto err_out;
+
+	if (tc->version == APOLLO_VERSION_TCF_2) {
+		/* Set ODT to a specific value that seems to provide the most
+		 * stable signals.
+		 */
+		spi_write(tc, 0x11, 0x413130);
+	}
+
+	/* Take DUT out of reset */
+	reg_reset_n |= (0x1 << DUT_RESETN_SHIFT);
+	iowrite32(reg_reset_n, tc->tcf.registers +
+		  TCF_CLK_CTRL_CLK_AND_RST_CTRL);
+	msleep(100);
+
+	if (tc->version != APOLLO_VERSION_TCF_5) {
+		u32 hood_ctrl;
+
+		err = apollo_align_interface_es2(tc);
+		if (err)
+			goto err_out;
+
+		spi_read(tc, 0xF, &hood_ctrl);
+		hood_ctrl |= 0x1;
+		spi_write(tc, 0xF, hood_ctrl);
+	}
+
+#endif /* defined(SUPPORT_RGX) */
+
+	if (tc->version == APOLLO_VERSION_TCF_2) {
+		/* Enable the temperature sensor */
+		spi_write(tc, 0xc, 0); /* power up */
+		spi_write(tc, 0xc, 2); /* reset */
+		spi_write(tc, 0xc, 6); /* init & run */
+
+		/* Register a new thermal zone */
+		apollo_pdata.thermal_zone =
+			thermal_tripless_zone_device_register("apollo", tc,
+						     &apollo_thermal_dev_ops,
+						     NULL);
+		if (IS_ERR(apollo_pdata.thermal_zone)) {
+			dev_warn(&tc->pdev->dev, "Couldn't register thermal zone");
+			apollo_pdata.thermal_zone = NULL;
+		}
+	}
+
+	reg = ioread32(tc->tcf.registers + TCF_CLK_CTRL_SW_IF_VERSION);
+	reg = (reg & VERSION_MASK) >> VERSION_SHIFT;
+
+	if (reg == 0) {
+		u32 build_inc;
+		u32 build_owner;
+
+		/* Check the build */
+		reg = ioread32(tc->tcf.registers + TCF_CLK_CTRL_FPGA_DES_REV_1);
+		build_inc = (reg >> 12) & 0xff;
+		build_owner = (reg >> 20) & 0xf;
+
+		if (build_inc) {
+			dev_alert(&tc->pdev->dev,
+				"BE WARNED: You are not running a tagged release of the FPGA!\n");
+
+			dev_alert(&tc->pdev->dev, "Owner: 0x%01x, Inc: 0x%02x\n",
+				  build_owner, build_inc);
+		}
+
+		dev_info(&tc->pdev->dev, "FPGA Release: %u.%02u\n",
+			 reg >> 8 & 0xf, reg & 0xff);
+	}
+
+#if defined(SUPPORT_RGX)
+err_out:
+#endif /* defined(SUPPORT_RGX) */
+	return err;
+}
+
+static void apollo_set_mem_mode_lma(struct tc_device *tc)
+{
+	u32 val;
+
+	val = ioread32(tc->tcf.registers + TCF_CLK_CTRL_TEST_CTRL);
+	val &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK
+		| HOST_PHY_MODE_MASK);
+	val |= (0x1 << ADDRESS_FORCE_SHIFT);
+	iowrite32(val, tc->tcf.registers + TCF_CLK_CTRL_TEST_CTRL);
+}
+
+static void apollo_set_mem_mode_hybrid(struct tc_device *tc)
+{
+	u32 val;
+
+	val = ioread32(tc->tcf.registers + TCF_CLK_CTRL_TEST_CTRL);
+	val &= ~(ADDRESS_FORCE_MASK | PCI_TEST_MODE_MASK | HOST_ONLY_MODE_MASK
+		| HOST_PHY_MODE_MASK);
+	val |= ((0x1 << HOST_ONLY_MODE_SHIFT) | (0x1 << HOST_PHY_MODE_SHIFT));
+	iowrite32(val, tc->tcf.registers + TCF_CLK_CTRL_TEST_CTRL);
+
+	/* Setup apollo to pass 1GB window of address space to the local memory.
+	 * This is a sub-mode of the host only mode, meaning that the apollo TC
+	 * can address the system memory with a 1GB window of address space
+	 * routed to the device local memory. The simplest approach is to mirror
+	 * the CPU physical address space, by moving the device local memory
+	 * window where it is mapped in the CPU physical address space.
+	 */
+	iowrite32(tc->tc_mem.base,
+		  tc->tcf.registers + TCF_CLK_CTRL_HOST_PHY_OFFSET);
+}
+
+static int apollo_set_mem_mode(struct tc_device *tc, int mem_mode)
+{
+	switch (mem_mode) {
+	case TC_MEMORY_HYBRID:
+		apollo_set_mem_mode_hybrid(tc);
+		dev_info(&tc->pdev->dev, "Memory mode: TC_MEMORY_HYBRID\n");
+		break;
+	case TC_MEMORY_LOCAL:
+		apollo_set_mem_mode_lma(tc);
+		dev_info(&tc->pdev->dev, "Memory mode: TC_MEMORY_LOCAL\n");
+		break;
+	default:
+		dev_err(&tc->pdev->dev, "unsupported memory mode = %d\n",
+			mem_mode);
+		return -EINVAL;
+	};
+
+	tc->mem_mode = mem_mode;
+
+	return 0;
+}
+
+static bool apollo_pdp_export_host_addr(struct tc_device *tc)
+{
+	return tc->mem_mode == TC_MEMORY_HYBRID;
+}
+
+static u64 apollo_get_pdp_dma_mask(struct tc_device *tc)
+{
+	/* The PDP does not access system memory, so there is no
+	 * DMA limitation.
+	 */
+	if ((tc->mem_mode == TC_MEMORY_LOCAL) ||
+	    (tc->mem_mode == TC_MEMORY_HYBRID))
+		return DMA_BIT_MASK(64);
+
+	return DMA_BIT_MASK(32);
+}
+
+#if defined(SUPPORT_RGX) || defined(SUPPORT_APOLLO_FPGA)
+#if defined(SUPPORT_RGX)
+static u64 apollo_get_rogue_dma_mask(struct tc_device *tc)
+#else /* SUPPORT_APOLLO_FPGA */
+static u64 apollo_get_fpga_dma_mask(struct tc_device *tc)
+#endif /* defined(SUPPORT_RGX) */
+{
+	/* Does not access system memory, so there is no DMA limitation */
+	if (tc->mem_mode == TC_MEMORY_LOCAL)
+		return DMA_BIT_MASK(64);
+
+	return DMA_BIT_MASK(32);
+}
+#endif /* defined(SUPPORT_RGX) || defined(SUPPORT_APOLLO_FPGA) */
+
+static int apollo_hw_init(struct tc_device *tc,
+			  int *core_clock, int *mem_clock, int sys_clock, int *clock_multiplex,
+			  int mem_latency, int mem_wresp_latency, int mem_mode)
+{
+	int err = 0;
+
+	err = apollo_hard_reset(tc, core_clock, mem_clock, sys_clock, clock_multiplex);
+	if (err)
+		goto err_out;
+
+	err = apollo_set_mem_mode(tc, mem_mode);
+	if (err)
+		goto err_out;
+
+#if defined(SUPPORT_RGX)
+	if (tc->version == APOLLO_VERSION_TCF_BONNIE) {
+		u32 reg;
+		/* Enable ASTC via SPI */
+		if (spi_read(tc, 0xf, &reg)) {
+			dev_err(&tc->pdev->dev,
+				"Failed to read apollo ASTC register\n");
+			err = -ENODEV;
+			goto err_out;
+		}
+
+		reg |= 0x1 << 4;
+		spi_write(tc, 0xf, reg);
+	} else if (tc->version == APOLLO_VERSION_TCF_5) {
+		apollo_set_mem_latency(tc, mem_latency, mem_wresp_latency);
+	}
+#endif /* defined(SUPPORT_RGX) */
+
+err_out:
+	return err;
+}
+
+static int apollo_enable_irq(struct tc_device *tc)
+{
+	int err = 0;
+
+#if defined(TC_FAKE_INTERRUPTS)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	timer_setup(&tc->timer, tc_irq_fake_wrapper, 0);
+#else
+	setup_timer(&tc->timer, tc_irq_fake_wrapper, (unsigned long)tc);
+#endif
+
+	mod_timer(&tc->timer,
+		jiffies + msecs_to_jiffies(FAKE_INTERRUPT_TIME_MS));
+#else
+	{
+		u32 val;
+
+		iowrite32(0, tc->tcf.registers +
+			TCF_CLK_CTRL_INTERRUPT_ENABLE);
+		iowrite32(0xffffffff, tc->tcf.registers +
+			TCF_CLK_CTRL_INTERRUPT_CLEAR);
+
+		/* Set sense to active high */
+		val = ioread32(tc->tcf.registers +
+			TCF_CLK_CTRL_INTERRUPT_OP_CFG) & ~(INT_SENSE_MASK);
+		iowrite32(val, tc->tcf.registers +
+			TCF_CLK_CTRL_INTERRUPT_OP_CFG);
+
+		err = request_irq(tc->pdev->irq, apollo_irq_handler,
+			IRQF_SHARED, DRV_NAME, tc);
+	}
+#endif
+	return err;
+}
+
+static void apollo_disable_irq(struct tc_device *tc)
+{
+#if defined(TC_FAKE_INTERRUPTS)
+	del_timer_sync(&tc->timer);
+#else
+	iowrite32(0, tc->tcf.registers +
+		TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	iowrite32(0xffffffff, tc->tcf.registers +
+		TCF_CLK_CTRL_INTERRUPT_CLEAR);
+
+	free_irq(tc->pdev->irq, tc);
+#endif
+}
+
+static enum tc_version_t
+apollo_detect_tc_version(struct tc_device *tc)
+{
+	u32 val = ioread32(tc->tcf.registers +
+		       TCF_CLK_CTRL_TCF_CORE_TARGET_BUILD_CFG);
+
+	switch (val) {
+	default:
+		dev_err(&tc->pdev->dev,
+			"Unknown TCF core target build ID (0x%x) - assuming Hood ES2 - PLEASE REPORT TO ANDROID TEAM\n",
+			val);
+		__fallthrough;
+	case 5:
+		dev_err(&tc->pdev->dev, "Looks like a Hood ES2 TC\n");
+		return APOLLO_VERSION_TCF_2;
+	case 1:
+		dev_err(&tc->pdev->dev, "Looks like a TCF5\n");
+		return APOLLO_VERSION_TCF_5;
+	case 6:
+		dev_err(&tc->pdev->dev, "Looks like a Bonnie TC\n");
+		return APOLLO_VERSION_TCF_BONNIE;
+	}
+}
+
+static u32 apollo_interrupt_id_to_flag(int interrupt_id)
+{
+	switch (interrupt_id) {
+	case TC_INTERRUPT_PDP:
+		return TC_INTERRUPT_FLAG_PDP;
+	case TC_INTERRUPT_EXT:
+		return TC_INTERRUPT_FLAG_EXT;
+	default:
+		BUG();
+	}
+}
+
+/*
+ * GetMemorySize:
+ * For a given PCI phys addr and size, go through the memory to check the actual size.
+ * Do this by writing to address zero, and then checking positions in the memory to see if there
+ * is any aliasing. e.g. some devices may advertise 1GB of memory but in reality only have half of
+ * that.
+ */
+static uint64_t
+apollo_get_memory_size(struct tc_device *tc, uint64_t pci_bar_base, uint64_t pci_bar_size)
+{
+#define MEM_PROBE_VALUE 0xa5a5a5a5
+#define MEM_MAP_SIZE sizeof(uint32_t)
+	void __iomem *host_mapped_base;
+	uint32_t *mem_zero;
+	uint64_t mem_size = 0;
+
+	BUG_ON((pci_bar_size & (pci_bar_size-1)) != 0);	/* PCI BARs are always a power of 2 */
+	host_mapped_base = ioremap(pci_bar_base, MEM_MAP_SIZE);
+
+	if (host_mapped_base == NULL) {
+		dev_err(&tc->pdev->dev,
+			"%s: Cannot map PCI memory at phys 0x%llx for size 0x%llx",
+			__func__, pci_bar_base, pci_bar_size);
+	} else {
+		mem_zero = (uint32_t __force *)host_mapped_base;
+		mem_size = 1024*1024;	/* assume at least 1MB of memory */
+		while (mem_size < pci_bar_size) {
+			void __iomem *host_mapped = ioremap(pci_bar_base + mem_size, MEM_MAP_SIZE);
+			volatile uint32_t *mem_probe;
+
+			if (host_mapped == NULL) {
+				dev_err(&tc->pdev->dev,
+					"%s: Cannot map PCI memory at phys 0x%llx for size 0x%llx",
+					__func__, pci_bar_base + mem_size, pci_bar_size);
+				break;
+			}
+
+			mem_probe = (volatile uint32_t __force *)host_mapped;
+
+			/* check that the memory at offset actually exists and is not aliased */
+			*mem_probe = MEM_PROBE_VALUE;
+			*mem_zero = 0;
+			if (*mem_probe != MEM_PROBE_VALUE)
+				break;
+			*mem_probe = 0;	/* reset value */
+			mem_size <<= 1;	/* double the offset we probe each time. */
+			iounmap(host_mapped);
+		}
+		iounmap(host_mapped_base);
+		dev_info(&tc->pdev->dev,
+			"%s: Local memory size 0x%llx (PCI BAR size 0x%llx)",
+			__func__, mem_size, pci_bar_size);
+	}
+
+	return mem_size;
+}
+
+static int apollo_dev_init(struct tc_device *tc, struct pci_dev *pdev,
+			   int pdp_mem_size, int secure_mem_size)
+{
+	int err;
+	uint64_t pci_bar_size;
+	uint64_t tc_mem_size;
+
+	/* Reserve and map the tcf_clk / "sys" registers */
+	err = setup_io_region(pdev, &tc->tcf,
+		SYS_APOLLO_REG_PCI_BASENUM,
+		SYS_APOLLO_REG_SYS_OFFSET, SYS_APOLLO_REG_SYS_SIZE);
+	if (err)
+		goto err_out;
+
+	/* Reserve and map the tcf_pll registers */
+	err = setup_io_region(pdev, &tc->tcf_pll,
+		SYS_APOLLO_REG_PCI_BASENUM,
+		SYS_APOLLO_REG_PLL_OFFSET + TCF_PLL_PLL_CORE_CLK0,
+		TCF_PLL_PLL_DRP_STATUS - TCF_PLL_PLL_CORE_CLK0 + 4);
+	if (err)
+		goto err_unmap_sys_registers;
+
+#if defined(SUPPORT_APOLLO_FPGA)
+#define FPGA_REGISTERS_SIZE 4
+	/* If this is a special 'fgpa' build, have the apollo driver manage
+	 * the second register bar.
+	 */
+	err = setup_io_region(pdev, &apollo_pdata.fpga,
+		SYS_RGX_REG_PCI_BASENUM, 0, FPGA_REGISTERS_SIZE);
+	if (err)
+		goto err_unmap_pll_registers;
+#endif
+
+	/* Detect testchip version */
+	tc->version = apollo_detect_tc_version(tc);
+
+	/* Setup card memory */
+	tc->tc_mem.base =
+		pci_resource_start(pdev, APOLLO_MEM_PCI_BASENUM);
+
+	pci_bar_size = pci_resource_len(pdev, APOLLO_MEM_PCI_BASENUM);
+
+	tc_mem_size = apollo_get_memory_size(tc, tc->tc_mem.base, pci_bar_size);
+
+	if (tc_mem_size < pci_bar_size) {
+		dev_warn(&tc->pdev->dev,
+			"%s: Device memory region smaller than PCI BAR (got 0x%08llx, PCI BAR 0x%08llx)",
+			__func__, tc_mem_size, pci_bar_size);
+	}
+
+	tc->tc_mem.size = tc_mem_size;
+
+	if (tc->tc_mem.size < pdp_mem_size) {
+		dev_err(&pdev->dev,
+			"Apollo MEM region (bar %d) has size of %lu which is smaller than the requested PDP heap of %lu",
+			APOLLO_MEM_PCI_BASENUM,
+			(unsigned long)tc->tc_mem.size,
+			(unsigned long)pdp_mem_size);
+		err = -EIO;
+		goto err_unmap_fpga_registers;
+	}
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	if (tc->tc_mem.size <
+	    (pdp_mem_size + secure_mem_size)) {
+		dev_err(&pdev->dev,
+			"Apollo MEM region (bar %d) has size of %lu which is smaller than the requested PDP heap of %lu plus the requested secure heap size %lu",
+			APOLLO_MEM_PCI_BASENUM,
+			(unsigned long)tc->tc_mem.size,
+			(unsigned long)pdp_mem_size,
+			(unsigned long)secure_mem_size);
+		err = -EIO;
+		goto err_unmap_fpga_registers;
+	}
+#endif
+
+	err = tc_mtrr_setup(tc);
+	if (err)
+		goto err_unmap_fpga_registers;
+
+	/* Setup ranges for the device heaps */
+	tc->pdp_heap_mem_size = pdp_mem_size;
+
+	/* We know ext_heap_mem_size won't underflow as we've compared
+	 * tc_mem.size against the pdp_mem_size value earlier
+	 */
+	tc->ext_heap_mem_size =
+		tc->tc_mem.size - tc->pdp_heap_mem_size;
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	tc->ext_heap_mem_size -= secure_mem_size;
+#endif
+
+	if (tc->ext_heap_mem_size < TC_EXT_MINIMUM_MEM_SIZE) {
+		dev_warn(&pdev->dev,
+			"Apollo MEM region (bar %d) has size of %lu, with %lu pdp_mem_size only %lu bytes are left for ext device, which looks too small",
+			APOLLO_MEM_PCI_BASENUM,
+			(unsigned long)tc->tc_mem.size,
+			(unsigned long)pdp_mem_size,
+			(unsigned long)tc->ext_heap_mem_size);
+		/* Continue as this is only a 'helpful warning' not a hard
+		 * requirement
+		 */
+	}
+
+	tc->ext_heap_mem_base = tc->tc_mem.base;
+	tc->pdp_heap_mem_base =
+		tc->tc_mem.base + tc->ext_heap_mem_size;
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	tc->secure_heap_mem_base = tc->pdp_heap_mem_base +
+		tc->pdp_heap_mem_size;
+	tc->secure_heap_mem_size = secure_mem_size;
+#endif
+
+#if defined(SUPPORT_DMA_HEAP)
+	err = tc_dmabuf_heap_init(tc, APOLLO_MEM_PCI_BASENUM);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialise DMA heap\n");
+		goto err_unmap_fpga_registers;
+	}
+#elif defined(SUPPORT_ION)
+	err = tc_ion_init(tc, APOLLO_MEM_PCI_BASENUM);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialise ION\n");
+		goto err_unmap_fpga_registers;
+	}
+#endif /* defined(SUPPORT_ION) */
+
+#if defined(SUPPORT_APOLLO_FPGA)
+	apollo_debugfs_add_fpga_entries(tc, &apollo_pdata.fpga,
+					&apollo_pdata.fpga_entries);
+#endif /* defined(SUPPORT_APOLLO_FPGA) */
+
+err_out:
+	return err;
+err_unmap_fpga_registers:
+#if defined(SUPPORT_APOLLO_FPGA)
+	iounmap(apollo_pdata.fpga.registers);
+	release_pci_io_addr(pdev, SYS_RGX_REG_PCI_BASENUM,
+		apollo_pdata.fpga.region.base, apollo_pdata.fpga.region.size);
+err_unmap_pll_registers:
+#endif /* defined(SUPPORT_APOLLO_FPGA) */
+	iounmap(tc->tcf_pll.registers);
+	release_pci_io_addr(pdev, SYS_APOLLO_REG_PCI_BASENUM,
+		tc->tcf_pll.region.base, tc->tcf_pll.region.size);
+err_unmap_sys_registers:
+	iounmap(tc->tcf.registers);
+	release_pci_io_addr(pdev, SYS_APOLLO_REG_PCI_BASENUM,
+		tc->tcf.region.base, tc->tcf.region.size);
+	goto err_out;
+}
+
+static void apollo_dev_cleanup(struct tc_device *tc)
+{
+#if defined(SUPPORT_APOLLO_FPGA)
+	apollo_debugfs_remove_fpga_entries(&apollo_pdata.fpga_entries);
+#endif
+
+#if defined(SUPPORT_DMA_HEAP)
+	tc_dmabuf_heap_deinit(tc, APOLLO_MEM_PCI_BASENUM);
+#elif defined(SUPPORT_ION)
+	tc_ion_deinit(tc, APOLLO_MEM_PCI_BASENUM);
+#endif
+
+	tc_mtrr_cleanup(tc);
+
+#if defined(SUPPORT_APOLLO_FPGA)
+	iounmap(apollo_pdata.fpga.registers);
+	release_pci_io_addr(tc->pdev, SYS_RGX_REG_PCI_BASENUM,
+		apollo_pdata.fpga.region.base, apollo_pdata.fpga.region.size);
+#endif
+
+	iounmap(tc->tcf_pll.registers);
+	release_pci_io_addr(tc->pdev, SYS_APOLLO_REG_PCI_BASENUM,
+		tc->tcf_pll.region.base, tc->tcf_pll.region.size);
+
+	iounmap(tc->tcf.registers);
+	release_pci_io_addr(tc->pdev, SYS_APOLLO_REG_PCI_BASENUM,
+		tc->tcf.region.base, tc->tcf.region.size);
+
+	if (apollo_pdata.thermal_zone)
+		thermal_zone_device_unregister(apollo_pdata.thermal_zone);
+}
+
+int apollo_init(struct tc_device *tc, struct pci_dev *pdev,
+		int *core_clock, int *mem_clock, int sys_clock, int *clock_multiplex,
+		int pdp_mem_size, int secure_mem_size,
+		int mem_latency, int mem_wresp_latency, int mem_mode)
+{
+	int err = 0;
+
+	err = apollo_dev_init(tc, pdev, pdp_mem_size, secure_mem_size);
+	if (err) {
+		dev_err(&pdev->dev, "apollo_dev_init failed\n");
+		goto err_out;
+	}
+
+	err = apollo_hw_init(tc, core_clock, mem_clock, sys_clock, clock_multiplex,
+			     mem_latency, mem_wresp_latency, mem_mode);
+	if (err) {
+		dev_err(&pdev->dev, "apollo_hw_init failed\n");
+		goto err_dev_cleanup;
+	}
+
+	err = apollo_enable_irq(tc);
+	if (err) {
+		dev_err(&pdev->dev,
+			"Failed to initialise IRQ\n");
+		goto err_dev_cleanup;
+	}
+
+err_out:
+	return err;
+
+err_dev_cleanup:
+	apollo_dev_cleanup(tc);
+	goto err_out;
+}
+
+int apollo_cleanup(struct tc_device *tc)
+{
+	apollo_disable_irq(tc);
+	apollo_dev_cleanup(tc);
+
+	return 0;
+}
+
+int apollo_register_pdp_device(struct tc_device *tc)
+{
+	int err = 0;
+	resource_size_t reg_start =
+		pci_resource_start(tc->pdev,
+				   SYS_APOLLO_REG_PCI_BASENUM);
+	struct resource pdp_resources_es2[] = {
+		DEFINE_RES_MEM_NAMED(reg_start + SYS_APOLLO_REG_PDP1_OFFSET,
+				SYS_APOLLO_REG_PDP1_SIZE, "pdp-regs"),
+		DEFINE_RES_MEM_NAMED(reg_start +
+				SYS_APOLLO_REG_PLL_OFFSET +
+				TCF_PLL_PLL_PDP_CLK0,
+				TCF_PLL_PLL_PDP2_DRP_GO -
+				TCF_PLL_PLL_PDP_CLK0 + 4, "pll-regs"),
+	};
+	struct resource pdp_resources_tcf5[] = {
+		DEFINE_RES_MEM_NAMED(reg_start + SYS_APOLLO_REG_PDP1_OFFSET,
+				SYS_APOLLO_REG_PDP1_SIZE, "pdp-regs"),
+		DEFINE_RES_MEM_NAMED(reg_start +
+				SYS_APOLLO_REG_PLL_OFFSET +
+				TCF_PLL_PLL_PDP_CLK0,
+				TCF_PLL_PLL_PDP2_DRP_GO -
+				TCF_PLL_PLL_PDP_CLK0 + 4, "pll-regs"),
+		DEFINE_RES_MEM_NAMED(pci_resource_start(tc->pdev,
+				TC5_SYS_APOLLO_REG_PCI_BASENUM)
+				+ TC5_SYS_APOLLO_REG_PDP2_OFFSET,
+			TC5_SYS_APOLLO_REG_PDP2_SIZE, "tc5-pdp2-regs"),
+
+		DEFINE_RES_MEM_NAMED(pci_resource_start(tc->pdev,
+				TC5_SYS_APOLLO_REG_PCI_BASENUM)
+				+ TC5_SYS_APOLLO_REG_PDP2_FBDC_OFFSET,
+				TC5_SYS_APOLLO_REG_PDP2_FBDC_SIZE,
+				"tc5-pdp2-fbdc-regs"),
+
+		DEFINE_RES_MEM_NAMED(pci_resource_start(tc->pdev,
+				TC5_SYS_APOLLO_REG_PCI_BASENUM)
+				+ TC5_SYS_APOLLO_REG_HDMI_OFFSET,
+				TC5_SYS_APOLLO_REG_HDMI_SIZE,
+				"tc5-adv5711-regs"),
+	};
+
+	struct tc_pdp_platform_data pdata = {
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+		.ion_device = tc->ion_device,
+		.ion_heap_id = ION_HEAP_TC_PDP,
+#endif
+		.memory_base = tc->tc_mem.base,
+		.pdp_heap_memory_base = tc->pdp_heap_mem_base,
+		.pdp_heap_memory_size = tc->pdp_heap_mem_size,
+		.dma_map_export_host_addr = apollo_pdp_export_host_addr(tc),
+	};
+	struct platform_device_info pdp_device_info = {
+		.parent = &tc->pdev->dev,
+		.name = APOLLO_DEVICE_NAME_PDP,
+		.id = -2,
+		.data = &pdata,
+		.size_data = sizeof(pdata),
+		.dma_mask = apollo_get_pdp_dma_mask(tc),
+	};
+
+	if (tc->version == APOLLO_VERSION_TCF_5) {
+		pdp_device_info.res = pdp_resources_tcf5;
+		pdp_device_info.num_res = ARRAY_SIZE(pdp_resources_tcf5);
+	} else if (tc->version == APOLLO_VERSION_TCF_2 ||
+			tc->version == APOLLO_VERSION_TCF_BONNIE) {
+		pdp_device_info.res = pdp_resources_es2;
+		pdp_device_info.num_res = ARRAY_SIZE(pdp_resources_es2);
+	} else {
+		dev_err(&tc->pdev->dev,
+			"Unable to set PDP resource info for unknown apollo device\n");
+	}
+
+	tc->pdp_dev = platform_device_register_full(&pdp_device_info);
+	if (IS_ERR(tc->pdp_dev)) {
+		err = PTR_ERR(tc->pdp_dev);
+		dev_err(&tc->pdev->dev,
+			"Failed to register PDP device (%d)\n", err);
+		tc->pdp_dev = NULL;
+		goto err;
+	}
+err:
+	return err;
+}
+
+#if defined(SUPPORT_RGX)
+
+int apollo_register_ext_device(struct tc_device *tc)
+{
+	int err = 0;
+	struct resource rogue_resources[] = {
+		DEFINE_RES_MEM_NAMED(pci_resource_start(tc->pdev,
+				SYS_RGX_REG_PCI_BASENUM),
+			 SYS_RGX_REG_REGION_SIZE, "rogue-regs"),
+	};
+	struct tc_rogue_platform_data pdata = {
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+		.ion_device = tc->ion_device,
+		.ion_heap_id = ION_HEAP_TC_ROGUE,
+#endif
+		.mem_mode = tc->mem_mode,
+		.baseboard = TC_BASEBOARD_APOLLO,
+		.tc_memory_base = tc->tc_mem.base,
+		.pdp_heap_memory_base = tc->pdp_heap_mem_base,
+		.pdp_heap_memory_size = tc->pdp_heap_mem_size,
+		.rogue_heap_memory_base = tc->ext_heap_mem_base,
+		.rogue_heap_memory_size = tc->ext_heap_mem_size,
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+		.secure_heap_memory_base = tc->secure_heap_mem_base,
+		.secure_heap_memory_size = tc->secure_heap_mem_size,
+#endif
+	};
+	struct platform_device_info rogue_device_info = {
+		.parent = &tc->pdev->dev,
+		.name = TC_DEVICE_NAME_ROGUE,
+		.id = -2,
+		.res = rogue_resources,
+		.num_res = ARRAY_SIZE(rogue_resources),
+		.data = &pdata,
+		.size_data = sizeof(pdata),
+		.dma_mask = apollo_get_rogue_dma_mask(tc),
+	};
+
+	tc->ext_dev[0]
+		= platform_device_register_full(&rogue_device_info);
+
+	if (IS_ERR(tc->ext_dev[0])) {
+		err = PTR_ERR(tc->ext_dev[0]);
+		dev_err(&tc->pdev->dev,
+			"Failed to register rogue device (%d)\n", err);
+		tc->ext_dev[0] = NULL;
+	}
+	return err;
+}
+
+#elif defined(SUPPORT_APOLLO_FPGA)
+
+int apollo_register_ext_device(struct tc_device *tc)
+{
+	int err = 0;
+	struct resource fpga_resources[] = {
+		/* For the 'fpga' build, we don't use the Rogue, but reuse the
+		 * define that mentions RGX.
+		 */
+		DEFINE_RES_MEM_NAMED(pci_resource_start(tc->pdev,
+				SYS_RGX_REG_PCI_BASENUM),
+			 SYS_RGX_REG_REGION_SIZE, "fpga-regs"),
+	};
+	struct apollo_fpga_platform_data pdata = {
+		.mem_mode = tc->mem_mode,
+		.tc_memory_base = tc->tc_mem.base,
+		.pdp_heap_memory_base = tc->pdp_heap_mem_base,
+		.pdp_heap_memory_size = tc->pdp_heap_mem_size,
+	};
+	struct platform_device_info fpga_device_info = {
+		.parent = &tc->pdev->dev,
+		.name = APOLLO_DEVICE_NAME_FPGA,
+		.id = -1,
+		.res = fpga_resources,
+		.num_res = ARRAY_SIZE(fpga_resources),
+		.data = &pdata,
+		.size_data = sizeof(pdata),
+		.dma_mask = apollo_get_fpga_dma_mask(tc),
+	};
+
+	tc->ext_dev = platform_device_register_full(&fpga_device_info);
+	if (IS_ERR(tc->ext_dev)) {
+		err = PTR_ERR(tc->ext_dev);
+		dev_err(&tc->pdev->dev,
+			"Failed to register fpga device (%d)\n", err);
+		tc->ext_dev = NULL;
+		/* Fall through */
+	}
+
+	return err;
+}
+
+#else /* defined(SUPPORT_APOLLO_FPGA) */
+
+int apollo_register_ext_device(struct tc_device *tc)
+{
+	return 0;
+}
+
+#endif /* defined(SUPPORT_RGX) */
+
+void apollo_enable_interrupt_register(struct tc_device *tc,
+				      int interrupt_id)
+{
+	u32 val;
+
+	if (interrupt_id == TC_INTERRUPT_PDP ||
+		interrupt_id == TC_INTERRUPT_EXT) {
+		val = ioread32(
+			tc->tcf.registers + TCF_CLK_CTRL_INTERRUPT_ENABLE);
+		val |= apollo_interrupt_id_to_flag(interrupt_id);
+		iowrite32(val,
+			tc->tcf.registers + TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	}
+}
+
+void apollo_disable_interrupt_register(struct tc_device *tc,
+				       int interrupt_id)
+{
+	u32 val;
+
+	if (interrupt_id == TC_INTERRUPT_PDP ||
+		interrupt_id == TC_INTERRUPT_EXT) {
+		val = ioread32(
+			tc->tcf.registers + TCF_CLK_CTRL_INTERRUPT_ENABLE);
+		val &= ~(apollo_interrupt_id_to_flag(interrupt_id));
+		iowrite32(val,
+			tc->tcf.registers + TCF_CLK_CTRL_INTERRUPT_ENABLE);
+	}
+}
+
+irqreturn_t apollo_irq_handler(int irq, void *data)
+{
+	u32 interrupt_status;
+	u32 interrupt_clear = 0;
+	unsigned long flags;
+	irqreturn_t ret = IRQ_NONE;
+	struct tc_device *tc = (struct tc_device *)data;
+
+	spin_lock_irqsave(&tc->interrupt_handler_lock, flags);
+
+#if defined(TC_FAKE_INTERRUPTS)
+	/* If we're faking interrupts pretend we got both ext and PDP ints */
+	interrupt_status = TC_INTERRUPT_FLAG_EXT
+		| TC_INTERRUPT_FLAG_PDP;
+#else
+	interrupt_status = ioread32(tc->tcf.registers
+			+ TCF_CLK_CTRL_INTERRUPT_STATUS);
+#endif
+
+	if (interrupt_status & TC_INTERRUPT_FLAG_EXT) {
+		struct tc_interrupt_handler *ext_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_EXT];
+
+		if (ext_int->enabled && ext_int->handler_function) {
+			ext_int->handler_function(ext_int->handler_data);
+			interrupt_clear |= TC_INTERRUPT_FLAG_EXT;
+		}
+		ret = IRQ_HANDLED;
+	}
+	if (interrupt_status & TC_INTERRUPT_FLAG_PDP) {
+		struct tc_interrupt_handler *pdp_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_PDP];
+
+		if (pdp_int->enabled && pdp_int->handler_function) {
+			pdp_int->handler_function(pdp_int->handler_data);
+			interrupt_clear |= TC_INTERRUPT_FLAG_PDP;
+		}
+		ret = IRQ_HANDLED;
+	}
+
+	if (tc->version == APOLLO_VERSION_TCF_5) {
+		/* On TC5 the interrupt is not  by the TC framework, but
+		 * by the PDP itself. So we always have to callback to the tc5
+		 * pdp code regardless of the interrupt status of the TCF.
+		 */
+		struct tc_interrupt_handler *pdp_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_TC5_PDP];
+
+		if (pdp_int->enabled && pdp_int->handler_function) {
+			pdp_int->handler_function(pdp_int->handler_data);
+			ret = IRQ_HANDLED;
+		}
+	}
+
+	if (interrupt_clear)
+		iowrite32(0xffffffff,
+			tc->tcf.registers + TCF_CLK_CTRL_INTERRUPT_CLEAR);
+
+	spin_unlock_irqrestore(&tc->interrupt_handler_lock, flags);
+
+	return ret;
+}
+
+int apollo_sys_info(struct tc_device *tc, u32 *tmp, u32 *pll)
+{
+	int err = 0;
+
+	*tmp = 0;
+	*pll = 0;
+
+	if (tc->version == APOLLO_VERSION_TCF_5)
+		/* Not implemented on TCF5 */
+		goto err_out;
+	else if (tc->version == APOLLO_VERSION_TCF_2) {
+		int t;
+
+		err = apollo_thermal_get_temp(apollo_pdata.thermal_zone, &t);
+		if (err)
+			goto err_out;
+		*tmp = t / 1000;
+	}
+
+	if (spi_read(tc, 0x2, pll)) {
+		dev_err(&tc->pdev->dev, "Failed to read PLL status\n");
+		err = -ENODEV;
+		goto err_out;
+	}
+
+err_out:
+	return err;
+}
+
+int apollo_sys_strings(struct tc_device *tc,
+		       char *str_fpga_rev, size_t size_fpga_rev,
+		       char *str_tcf_core_rev, size_t size_tcf_core_rev,
+		       char *str_tcf_core_target_build_id,
+		       size_t size_tcf_core_target_build_id,
+		       char *str_pci_ver, size_t size_pci_ver,
+		       char *str_macro_ver, size_t size_macro_ver)
+{
+	int err = 0;
+	u32 val;
+	resource_size_t host_fpga_base;
+	void __iomem *host_fpga_registers;
+
+	/* To get some of the version information we need to read from a
+	 * register that we don't normally have mapped. Map it temporarily
+	 * (without trying to reserve it) to get the information we need.
+	 */
+	host_fpga_base =
+		pci_resource_start(tc->pdev, SYS_APOLLO_REG_PCI_BASENUM)
+				+ 0x40F0;
+
+	host_fpga_registers = ioremap(host_fpga_base, 0x04);
+	if (!host_fpga_registers) {
+		dev_err(&tc->pdev->dev,
+			"Failed to map host fpga registers\n");
+		err = -EIO;
+		goto err_out;
+	}
+
+	/* Create the components of the PCI and macro versions */
+	val = ioread32(host_fpga_registers);
+	snprintf(str_pci_ver, size_pci_ver, "%d",
+		 HEX2DEC((val & 0x00FF0000) >> 16));
+	snprintf(str_macro_ver, size_macro_ver, "%d.%d",
+		 (val & 0x00000F00) >> 8,
+		 HEX2DEC((val & 0x000000FF) >> 0));
+
+	/* Unmap the register now that we no longer need it */
+	iounmap(host_fpga_registers);
+
+	/*
+	 * Check bits 7:0 of register 0x28 (TCF_CORE_REV_REG or SW_IF_VERSION
+	 * depending on its own value) to find out how the driver should
+	 * generate the strings for FPGA and core revision.
+	 */
+	val = ioread32(tc->tcf.registers + TCF_CLK_CTRL_SW_IF_VERSION);
+	val = (val & VERSION_MASK) >> VERSION_SHIFT;
+
+	if (val == 0) {
+		/* Create the components of the TCF core revision number */
+		val = ioread32(tc->tcf.registers + TCF_CLK_CTRL_TCF_CORE_REV_REG);
+		snprintf(str_tcf_core_rev, size_tcf_core_rev, "%d.%d.%d",
+			 HEX2DEC((val & TCF_CORE_REV_REG_MAJOR_MASK)
+				 >> TCF_CORE_REV_REG_MAJOR_SHIFT),
+			 HEX2DEC((val & TCF_CORE_REV_REG_MINOR_MASK)
+				 >> TCF_CORE_REV_REG_MINOR_SHIFT),
+			 HEX2DEC((val & TCF_CORE_REV_REG_MAINT_MASK)
+				 >> TCF_CORE_REV_REG_MAINT_SHIFT));
+
+		/* Create the components of the FPGA revision number */
+		val = ioread32(tc->tcf.registers + TCF_CLK_CTRL_FPGA_REV_REG);
+		snprintf(str_fpga_rev, size_fpga_rev, "%d.%d.%d",
+			 HEX2DEC((val & FPGA_REV_REG_MAJOR_MASK)
+				 >> FPGA_REV_REG_MAJOR_SHIFT),
+			 HEX2DEC((val & FPGA_REV_REG_MINOR_MASK)
+				 >> FPGA_REV_REG_MINOR_SHIFT),
+			 HEX2DEC((val & FPGA_REV_REG_MAINT_MASK)
+				 >> FPGA_REV_REG_MAINT_SHIFT));
+	} else if (val == 1) {
+		/* Create the components of the TCF core revision number */
+		snprintf(str_tcf_core_rev, size_tcf_core_rev, "%d", val);
+
+		/* Create the components of the FPGA revision number */
+		val = ioread32(tc->tcf.registers + TCF_CLK_CTRL_REL);
+		snprintf(str_fpga_rev, size_fpga_rev, "%d.%d",
+			 HEX2DEC((val & MAJOR_MASK) >> MAJOR_SHIFT),
+			 HEX2DEC((val & MINOR_MASK) >> MINOR_SHIFT));
+	} else {
+		dev_warn(&tc->pdev->dev,
+			 "%s: unrecognised SW_IF_VERSION %#08x\n",
+			 __func__, val);
+
+		/* Create the components of the TCF core revision number */
+		snprintf(str_tcf_core_rev, size_tcf_core_rev, "%d", val);
+
+		/* Create the components of the FPGA revision number */
+		snprintf(str_fpga_rev, size_fpga_rev, "N/A");
+	}
+
+	/* Create the component of the TCF core target build ID */
+	val = ioread32(tc->tcf.registers +
+		       TCF_CLK_CTRL_TCF_CORE_TARGET_BUILD_CFG);
+	snprintf(str_tcf_core_target_build_id, size_tcf_core_target_build_id,
+		"%d",
+		(val & TCF_CORE_TARGET_BUILD_ID_MASK)
+		>> TCF_CORE_TARGET_BUILD_ID_SHIFT);
+
+err_out:
+	return err;
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_apollo.h b/drivers/gpu/drm/img-rogue/apollo/tc_apollo.h
new file mode 100644
index 000000000000..0090dd2e570e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_apollo.h
@@ -0,0 +1,77 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _APOLLO_DRV_H
+#define _APOLLO_DRV_H
+
+#include "tc_drv_internal.h"
+#include "apollo_regs.h"
+
+#if defined(SUPPORT_RGX) && defined(SUPPORT_APOLLO_FPGA)
+#error Define either SUPPORT_RGX or SUPPORT_APOLLO_FGPA, not both
+#endif
+
+int apollo_init(struct tc_device *tc, struct pci_dev *pdev,
+		int *core_clock, int *mem_clock, int sys_clock, int *clock_multiplex,
+		int pdp_mem_size, int secure_mem_size,
+		int mem_latency, int mem_wresp_latency, int mem_mode);
+int apollo_cleanup(struct tc_device *tc);
+
+int apollo_register_pdp_device(struct tc_device *tc);
+int apollo_register_ext_device(struct tc_device *tc);
+
+void apollo_enable_interrupt_register(struct tc_device *tc,
+				      int interrupt_id);
+void apollo_disable_interrupt_register(struct tc_device *tc,
+				       int interrupt_id);
+
+irqreturn_t apollo_irq_handler(int irq, void *data);
+
+int apollo_sys_info(struct tc_device *tc, u32 *tmp, u32 *pll);
+int apollo_sys_strings(struct tc_device *tc,
+		       char *str_fpga_rev, size_t size_fpga_rev,
+		       char *str_tcf_core_rev, size_t size_tcf_core_rev,
+		       char *str_tcf_core_target_build_id,
+		       size_t size_tcf_core_target_build_id,
+		       char *str_pci_ver, size_t size_pci_ver,
+		       char *str_macro_ver, size_t size_macro_ver);
+
+#endif /* _APOLLO_DRV_H */
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_clocks.h b/drivers/gpu/drm/img-rogue/apollo/tc_clocks.h
new file mode 100644
index 000000000000..431273de0827
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_clocks.h
@@ -0,0 +1,158 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TC_CLOCKS_H)
+#define TC_CLOCKS_H
+
+/*
+ * The core clock speed is passed through a multiplier depending on the TC
+ * version.
+ *
+ * On TC_ES1: Multiplier = x3, final speed = 270MHz
+ * On TC_ES2: Multiplier = x6, final speed = 540MHz
+ * On TCF5:   Multiplier = 1x final speed = 45MHz
+ *
+ *
+ * The base (unmultiplied speed) can be adjusted using a module parameter
+ * called "sys_core_clk_speed", a number in Hz.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_core_clk_speed=60000000" /etc/init.d/rc.pvr start
+ *
+ * would result in a core speed of 60MHz xMultiplier.
+ *
+ *
+ * The memory clock is unmultiplied and can be adjusted using a module
+ * parameter called "sys_mem_clk_speed", this should be the number in Hz for
+ * the memory clock speed.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=100000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the memory clock speed set to 100MHz.
+ *
+ *
+ * Same applies to the system interface clock speed, "sys_sysif_clk_speed".
+ * Needed for TCF5 but not for TC_ES2/ES1.
+ * As an example:
+ *
+ * PVR_SRVKM_PARAMS="sys_sysif_clk_speed=45000000" /etc/init.d/rc.pvr start
+ *
+ * would attempt to start the driver with the system clock speed set to 45MHz.
+ *
+ *
+ * All parameters can be specified at once, e.g.,
+ * PVR_SRVKM_PARAMS="sys_mem_clk_speed=MEMORY_SPEED sys_core_clk_speed=CORE_SPEED sys_sysif_clk_speed=SYSIF_SPEED" /etc/init.d/rc.pvr start
+ */
+
+#define RGX_TC_SYS_CLOCK_SPEED		(25000000) /*< At the moment just used for TCF5 */
+#define RGX_TC_CLOCK_MULTIPLEX    (1)
+
+#if defined(TC_APOLLO_TCF5_22_46_54_330)
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_22_49_21_16) || \
+      defined(TC_APOLLO_TCF5_22_60_22_29) || \
+      defined(TC_APOLLO_TCF5_22_75_22_25)
+ #define RGX_TC_CORE_CLOCK_SPEED	(20000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+#elif defined(TC_APOLLO_TCF5_22_67_54_30)
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_22_89_204_18)
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_22_86_104_218)
+ #define RGX_TC_CORE_CLOCK_SPEED	(30000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_88_104_318)
+ #define RGX_TC_CORE_CLOCK_SPEED	(28000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_98_54_230)
+ #define RGX_TC_CORE_CLOCK_SPEED	(100000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_22_102_54_38)
+ #define RGX_TC_CORE_CLOCK_SPEED	(80000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_BVNC_NOT_SUPPORTED)
+ /* TC TCF5 (22.*) fallback frequencies */
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(20000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_33_8_22_1)
+ #define RGX_TC_CORE_CLOCK_SPEED	(25000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_TCF5_REFERENCE)
+ /* TC TCF5 (Reference bitfile) */
+ #undef RGX_TC_SYS_CLOCK_SPEED
+ #define RGX_TC_CORE_CLOCK_SPEED	(50000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(50000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(45000000)
+#elif defined(TC_APOLLO_BONNIE)
+ /* TC Bonnie */
+ #define RGX_TC_CORE_CLOCK_SPEED	(18000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(65000000)
+#elif defined(TC_APOLLO_ES2)
+ /* TC ES2 */
+ #define RGX_TC_CORE_CLOCK_SPEED	(90000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(104000000)
+#elif defined(TC_ORION)
+ #define RGX_TC_CORE_CLOCK_SPEED	(40000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(100000000)
+ #define RGX_TC_SYS_CLOCK_SPEED		(25000000)
+#elif defined(TC_APOLLO_TCF5_29_19_52_202)
+ #define RGX_TC_CORE_CLOCK_SPEED	(25000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(40000000)
+#elif defined(TC_APOLLO_TCF5_29_18_204_508)
+ #define RGX_TC_CORE_CLOCK_SPEED	(15000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(35000000)
+#else
+ /* TC ES1 */
+ #define RGX_TC_CORE_CLOCK_SPEED	(90000000)
+ #define RGX_TC_MEM_CLOCK_SPEED		(65000000)
+#endif
+
+#endif /* if !defined(TC_CLOCKS_H) */
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_drv.c b/drivers/gpu/drm/img-rogue/apollo/tc_drv.c
new file mode 100644
index 000000000000..5be3eaca04c6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_drv.c
@@ -0,0 +1,857 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This is a device driver for the testchip framework. It creates platform
+ * devices for the pdp and ext sub-devices, and exports functions to manage the
+ * shared interrupt handling
+ */
+
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+
+#if defined(CONFIG_MTRR)
+#include <asm/mtrr.h>
+#endif
+
+#include "pvrmodule.h"
+
+#include "tc_apollo.h"
+#include "tc_odin.h"
+#include "kernel_compatibility.h"
+
+/* How much memory to give to the PDP heap (used for pdp buffers). */
+#define TC_PDP_MEM_SIZE_BYTES           ((TC_DISPLAY_MEM_SIZE)*1024*1024)
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+/* How much memory to give to the secure heap. */
+#define TC_SECURE_MEM_SIZE_BYTES        ((TC_SECURE_MEM_SIZE)*1024*1024)
+#endif
+
+#define PCI_VENDOR_ID_POWERVR		0x1010
+#define DEVICE_ID_PCI_APOLLO_FPGA	0x1CF1
+#define DEVICE_ID_PCIE_APOLLO_FPGA	0x1CF2
+
+MODULE_DESCRIPTION("PowerVR testchip framework driver");
+MODULE_IMPORT_NS(DMA_BUF);
+
+static int tc_core_clock;
+module_param(tc_core_clock, int, 0444);
+MODULE_PARM_DESC(tc_core_clock, "TC core clock speed");
+
+static int tc_mem_clock;
+module_param(tc_mem_clock, int, 0444);
+MODULE_PARM_DESC(tc_mem_clock, "TC memory clock speed");
+
+static int tc_clock_multiplex;
+module_param(tc_clock_multiplex, int, 0444);
+MODULE_PARM_DESC(tc_clock_multiplex, "TC core clock multiplex");
+
+static int tc_sys_clock = RGX_TC_SYS_CLOCK_SPEED;
+module_param(tc_sys_clock, int, 0444);
+MODULE_PARM_DESC(tc_sys_clock, "TC system clock speed (TCF5 only)");
+
+static int tc_mem_latency;
+module_param(tc_mem_latency, int, 0444);
+MODULE_PARM_DESC(tc_mem_latency, "TC memory read latency in cycles (TCF5 only)");
+
+static unsigned long tc_mem_mode = TC_MEMORY_CONFIG;
+module_param(tc_mem_mode, ulong, 0444);
+MODULE_PARM_DESC(tc_mem_mode, "TC memory mode (local = 1, hybrid = 2, host = 3)");
+
+static int tc_wresp_latency;
+module_param(tc_wresp_latency, int, 0444);
+MODULE_PARM_DESC(tc_wresp_latency, "TC memory write response latency in cycles (TCF5 only)");
+
+static unsigned long tc_pdp_mem_size = TC_PDP_MEM_SIZE_BYTES;
+module_param(tc_pdp_mem_size, ulong, 0444);
+MODULE_PARM_DESC(tc_pdp_mem_size,
+	"TC PDP reserved memory size in bytes");
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+static unsigned long tc_secure_mem_size = TC_SECURE_MEM_SIZE_BYTES;
+module_param(tc_secure_mem_size, ulong, 0444);
+MODULE_PARM_DESC(tc_secure_mem_size,
+	"TC secure reserved memory size in bytes");
+#endif
+
+static bool fbc_bypass;
+module_param(fbc_bypass, bool, 0444);
+MODULE_PARM_DESC(fbc_bypass, "Force bypass of PDP2 FBC decompression");
+
+static struct debugfs_blob_wrapper tc_debugfs_rogue_name_blobs[] = {
+	[APOLLO_VERSION_TCF_2] = {
+		.data = "hood", /* probably */
+		.size = sizeof("hood") - 1,
+	},
+	[APOLLO_VERSION_TCF_5] = {
+		.data = "fpga (unknown)",
+		.size = sizeof("fpga (unknown)") - 1,
+	},
+	[APOLLO_VERSION_TCF_BONNIE] = {
+		.data = "bonnie",
+		.size = sizeof("bonnie") - 1,
+	},
+	[ODIN_VERSION_TCF_BONNIE] = {
+		.data = "bonnie",
+		.size = sizeof("bonnie") - 1,
+	},
+	[ODIN_VERSION_FPGA] = {
+		.data = "fpga (unknown)",
+		.size = sizeof("fpga (unknown)") - 1,
+	},
+	[ODIN_VERSION_ORION] = {
+		.data = "orion",
+		.size = sizeof("orion") - 1,
+	},
+	[ODIN_VERSION_VALI] = {
+		.data = "vali",
+		.size = sizeof("vali") - 1,
+	},
+};
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+/* forward declaration */
+static void tc_devres_release(struct device *dev, void *res);
+
+static ssize_t rogue_name_show(struct device_driver *drv, char *buf)
+{
+	struct pci_dev *pci_dev;
+	struct tc_device *tc;
+	struct device *dev;
+
+	dev = driver_find_next_device(drv, NULL);
+	if (!dev)
+		return -ENODEV;
+
+	pci_dev = to_pci_dev(dev);
+	if (!pci_dev)
+		return -ENODEV;
+
+	tc = devres_find(&pci_dev->dev, tc_devres_release, NULL, NULL);
+	if (!tc)
+		return -ENODEV;
+
+	return sprintf(buf, "%s\n", (const char *)
+			tc_debugfs_rogue_name_blobs[tc->version].data);
+}
+
+static DRIVER_ATTR_RO(rogue_name);
+
+static struct attribute *tc_attrs[] = {
+	&driver_attr_rogue_name.attr,
+	NULL,
+};
+
+static struct attribute_group tc_attr_group = {
+	.attrs = tc_attrs,
+};
+
+static const struct attribute_group *tc_attr_groups[] = {
+	&tc_attr_group,
+	NULL,
+};
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)) */
+
+
+int tc_mtrr_setup(struct tc_device *tc)
+{
+	int err = 0;
+	/* Register the LMA as write combined */
+	err = arch_io_reserve_memtype_wc(tc->tc_mem.base,
+					 tc->tc_mem.size);
+	if (err)
+		return -ENODEV;
+	/* Enable write combining */
+	tc->mtrr = arch_phys_wc_add(tc->tc_mem.base,
+				    tc->tc_mem.size);
+	if (tc->mtrr < 0) {
+		err = -ENODEV;
+		goto err_out;
+	}
+
+	return err;
+
+err_out:
+	arch_io_free_memtype_wc(tc->tc_mem.base,
+				tc->tc_mem.size);
+	return err;
+}
+
+void tc_mtrr_cleanup(struct tc_device *tc)
+{
+	if (tc->mtrr >= 0) {
+		arch_phys_wc_del(tc->mtrr);
+		arch_io_free_memtype_wc(tc->tc_mem.base,
+					tc->tc_mem.size);
+	}
+}
+
+int tc_is_interface_aligned(u32 eyes, u32 clk_taps, u32 train_ack)
+{
+	u32	max_eye_start = eyes >> 16;
+	u32	min_eye_end   = eyes & 0xffff;
+
+	/* If either the training or training ack failed, we haven't aligned */
+	if (!(clk_taps & 0x10000) || !(train_ack & 0x100))
+		return 0;
+
+	/* If the max eye >= min eye it means the readings are nonsense */
+	if (max_eye_start >= min_eye_end)
+		return 0;
+
+	/* If we failed the ack pattern more than 4 times */
+	if (((train_ack & 0xf0) >> 4) > 4)
+		return 0;
+
+	/* If there is less than 7 taps (240ps @40ps/tap, this number should be
+	 * lower for the fpga, since its taps are bigger We should really
+	 * calculate the "7" based on the interface clock speed.
+	 */
+	if ((min_eye_end - max_eye_start) < 7)
+		return 0;
+
+	return 1;
+}
+
+int tc_iopol32_nonzero(u32 mask, void __iomem *addr)
+{
+	int polnum;
+	u32 read_value;
+
+	for (polnum = 0; polnum < 50; polnum++) {
+		read_value = ioread32(addr) & mask;
+		if (read_value != 0)
+			break;
+		msleep(20);
+	}
+	if (polnum == 50) {
+		pr_err(DRV_NAME " iopol32_nonzero timeout\n");
+		return -ETIME;
+	}
+	return 0;
+}
+
+int request_pci_io_addr(struct pci_dev *pdev, u32 index,
+	resource_size_t offset, resource_size_t length)
+{
+	resource_size_t start, end;
+
+	start = pci_resource_start(pdev, index);
+	end = pci_resource_end(pdev, index);
+
+	if ((start + offset + length - 1) > end)
+		return -EIO;
+	if (pci_resource_flags(pdev, index) & IORESOURCE_IO) {
+		if (request_region(start + offset, length, DRV_NAME) == NULL)
+			return -EIO;
+	} else {
+		if (request_mem_region(start + offset, length, DRV_NAME)
+			== NULL)
+			return -EIO;
+	}
+	return 0;
+}
+
+void release_pci_io_addr(struct pci_dev *pdev, u32 index,
+	resource_size_t start, resource_size_t length)
+{
+	if (pci_resource_flags(pdev, index) & IORESOURCE_IO)
+		release_region(start, length);
+	else
+		release_mem_region(start, length);
+}
+
+int setup_io_region(struct pci_dev *pdev,
+	struct tc_io_region *region, u32 index,
+	resource_size_t offset,	resource_size_t size)
+{
+	int err;
+	resource_size_t pci_phys_addr;
+
+	err = request_pci_io_addr(pdev, index, offset, size);
+	if (err) {
+		dev_err(&pdev->dev,
+			"Failed to request tc registers (err=%d)\n", err);
+		return -EIO;
+	}
+	pci_phys_addr = pci_resource_start(pdev, index);
+	region->region.base = pci_phys_addr + offset;
+	region->region.size = size;
+
+	region->registers = ioremap(region->region.base, region->region.size);
+
+	if (!region->registers) {
+		dev_err(&pdev->dev, "Failed to map tc registers\n");
+		release_pci_io_addr(pdev, index,
+			region->region.base, region->region.size);
+		return -EIO;
+	}
+	return 0;
+}
+
+#if defined(TC_FAKE_INTERRUPTS)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+void tc_irq_fake_wrapper(struct timer_list *t)
+{
+	struct tc_device *tc = from_timer(tc, t, timer);
+#else
+void tc_irq_fake_wrapper(unsigned long data)
+{
+	struct tc_device *tc = (struct tc_device *)data;
+#endif
+
+	if (tc->odin)
+		odin_irq_handler(0, tc);
+	else
+		apollo_irq_handler(0, tc);
+
+	mod_timer(&tc->timer,
+		jiffies + msecs_to_jiffies(FAKE_INTERRUPT_TIME_MS));
+}
+#endif
+
+static int tc_register_pdp_device(struct tc_device *tc)
+{
+	int err = 0;
+
+	if (tc->odin || tc->orion)
+		err = odin_register_pdp_device(tc);
+	else if (tc->vali)
+		err = 0;
+	else
+		err = apollo_register_pdp_device(tc);
+
+	return err;
+}
+
+static int tc_register_ext_device(struct tc_device *tc)
+{
+	int err = 0;
+
+	if (tc->odin || tc->orion || tc->vali)
+		err = odin_register_ext_device(tc);
+	else
+		err = apollo_register_ext_device(tc);
+
+	return err;
+}
+
+static int tc_register_dma_device(struct tc_device *tc)
+{
+	int err = 0;
+
+	if (tc->odin)
+		err = odin_register_dma_device(tc);
+
+	return err;
+}
+
+static void tc_devres_release(struct device *dev, void *res)
+{
+	/* No extra cleanup needed */
+}
+
+static int tc_cleanup(struct pci_dev *pdev)
+{
+	struct tc_device *tc = devres_find(&pdev->dev,
+					   tc_devres_release, NULL, NULL);
+	int i, err = 0;
+
+	if (!tc) {
+		dev_err(&pdev->dev, "No tc device resources found\n");
+		return -ENODEV;
+	}
+
+	debugfs_remove(tc->debugfs_rogue_name);
+
+	for (i = 0; i < TC_INTERRUPT_COUNT; i++)
+		if (tc->interrupt_handlers[i].enabled)
+			tc_disable_interrupt(&pdev->dev, i);
+
+	if (tc->odin || tc->orion)
+		err = odin_cleanup(tc);
+	else
+		err = apollo_cleanup(tc);
+
+	debugfs_remove(tc->debugfs_tc_dir);
+
+	return err;
+}
+
+static int tc_init(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct tc_device *tc;
+	int err = 0;
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	int sec_mem_size = TC_SECURE_MEM_SIZE_BYTES;
+#else /* defined(SUPPORT_FAKE_SECURE_ION_HEAP) */
+	int sec_mem_size = 0;
+#endif /* defined(SUPPORT_FAKE_SECURE_ION_HEAP) */
+
+	if (!devres_open_group(&pdev->dev, NULL, GFP_KERNEL))
+		return -ENOMEM;
+
+	tc = devres_alloc(tc_devres_release,
+		sizeof(*tc), GFP_KERNEL);
+	if (!tc) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	devres_add(&pdev->dev, tc);
+
+	err = tc_enable(&pdev->dev);
+	if (err) {
+		dev_err(&pdev->dev,
+			"tc_enable failed %d\n", err);
+		goto err_release;
+	}
+
+	tc->pdev = pdev;
+
+	spin_lock_init(&tc->interrupt_handler_lock);
+	spin_lock_init(&tc->interrupt_enable_lock);
+
+	tc->debugfs_tc_dir = debugfs_create_dir(DRV_NAME, NULL);
+
+	if (pdev->vendor == PCI_VENDOR_ID_ODIN) {
+
+		if (pdev->device == DEVICE_ID_ODIN)
+			tc->odin = true;
+		else if (pdev->device == DEVICE_ID_ORION)
+			tc->orion = true;
+		else if (pdev->device == DEVICE_ID_VALI)
+			tc->vali = true;
+
+		dev_info(&pdev->dev, "%s detected\n", odin_tc_name(tc));
+
+		err = odin_init(tc, pdev,
+				&tc_core_clock, &tc_mem_clock, &tc_clock_multiplex,
+				tc_pdp_mem_size, sec_mem_size,
+				tc_mem_latency, tc_wresp_latency,
+				tc_mem_mode, fbc_bypass);
+		if (err)
+			goto err_dev_cleanup;
+
+	} else {
+		dev_info(&pdev->dev, "Apollo detected");
+		tc->odin = false;
+
+		err = apollo_init(tc, pdev,
+				  &tc_core_clock, &tc_mem_clock, tc_sys_clock, &tc_clock_multiplex,
+				  tc_pdp_mem_size, sec_mem_size,
+				  tc_mem_latency, tc_wresp_latency,
+				  tc_mem_mode);
+		if (err)
+			goto err_dev_cleanup;
+	}
+
+	/* Add the rogue name debugfs entry */
+	tc->debugfs_rogue_name =
+		debugfs_create_blob("rogue-name", 0444,
+			tc->debugfs_tc_dir,
+			&tc_debugfs_rogue_name_blobs[tc->version]);
+
+#if defined(TC_FAKE_INTERRUPTS)
+	dev_warn(&pdev->dev, "WARNING: Faking interrupts every %d ms",
+		FAKE_INTERRUPT_TIME_MS);
+#endif
+
+	/* Register pdp and ext platform devices */
+	err = tc_register_pdp_device(tc);
+	if (err)
+		goto err_dev_cleanup;
+
+	err = tc_register_ext_device(tc);
+	if (err)
+		goto err_dev_cleanup;
+
+	err =  tc_register_dma_device(tc);
+	if (err)
+		goto err_dev_cleanup;
+
+	devres_remove_group(&pdev->dev, NULL);
+
+	pci_set_master(pdev);
+
+err_out:
+	if (err)
+		dev_err(&pdev->dev, "%s: failed\n", __func__);
+
+	return err;
+
+err_dev_cleanup:
+	tc_cleanup(pdev);
+	tc_disable(&pdev->dev);
+err_release:
+	devres_release_group(&pdev->dev, NULL);
+	goto err_out;
+}
+
+static void tc_exit(struct pci_dev *pdev)
+{
+	struct tc_device *tc = devres_find(&pdev->dev,
+					   tc_devres_release, NULL, NULL);
+	int osid;
+
+	if (!tc) {
+		dev_err(&pdev->dev, "No tc device resources found\n");
+		return;
+	}
+
+	if (tc->pdp_dev)
+		platform_device_unregister(tc->pdp_dev);
+
+	for (osid=0; osid < RGX_NUM_DRIVERS_SUPPORTED; osid++)
+	{
+		if (tc->ext_dev[osid])
+			platform_device_unregister(tc->ext_dev[osid]);
+	}
+
+	if (tc->dma_dev)
+		platform_device_unregister(tc->dma_dev);
+
+	pci_clear_master(pdev);
+
+	tc_cleanup(pdev);
+
+	tc_disable(&pdev->dev);
+}
+
+static struct pci_device_id tc_pci_tbl[] = {
+	{ PCI_VDEVICE(POWERVR, DEVICE_ID_PCI_APOLLO_FPGA) },
+	{ PCI_VDEVICE(POWERVR, DEVICE_ID_PCIE_APOLLO_FPGA) },
+	{ PCI_VDEVICE(POWERVR, DEVICE_ID_TBA) },
+	{ PCI_VDEVICE(ODIN, DEVICE_ID_ODIN) },
+	{ PCI_VDEVICE(ODIN, DEVICE_ID_VALI) },
+	{ PCI_VDEVICE(ODIN, DEVICE_ID_ORION) },
+	{ },
+};
+
+static struct pci_driver tc_pci_driver = {
+	.name           = DRV_NAME,
+	.id_table       = tc_pci_tbl,
+	.probe          = tc_init,
+	.remove         = tc_exit,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	.groups         = tc_attr_groups,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)) */
+};
+
+module_pci_driver(tc_pci_driver);
+
+MODULE_DEVICE_TABLE(pci, tc_pci_tbl);
+
+int tc_enable(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	return pci_enable_device(pdev);
+}
+EXPORT_SYMBOL(tc_enable);
+
+void tc_disable(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	pci_disable_device(pdev);
+}
+EXPORT_SYMBOL(tc_disable);
+
+int tc_set_interrupt_handler(struct device *dev, int interrupt_id,
+	void (*handler_function)(void *), void *data)
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+		NULL, NULL);
+	int err = 0;
+	unsigned long flags;
+
+	if (!tc) {
+		dev_err(dev, "No tc device resources found\n");
+		err = -ENODEV;
+		goto err_out;
+	}
+
+	if (interrupt_id < 0 || interrupt_id >= TC_INTERRUPT_COUNT) {
+		dev_err(dev, "Invalid interrupt ID (%d)\n", interrupt_id);
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	spin_lock_irqsave(&tc->interrupt_handler_lock, flags);
+
+	tc->interrupt_handlers[interrupt_id].handler_function =
+		handler_function;
+	tc->interrupt_handlers[interrupt_id].handler_data = data;
+
+	spin_unlock_irqrestore(&tc->interrupt_handler_lock, flags);
+
+err_out:
+	return err;
+}
+EXPORT_SYMBOL(tc_set_interrupt_handler);
+
+int tc_enable_interrupt(struct device *dev, int interrupt_id)
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+		NULL, NULL);
+	int err = 0;
+	unsigned long flags;
+
+	if (!tc) {
+		dev_err(dev, "No tc device resources found\n");
+		err = -ENODEV;
+		goto err_out;
+	}
+	if (interrupt_id < 0 || interrupt_id >= TC_INTERRUPT_COUNT) {
+		dev_err(dev, "Invalid interrupt ID (%d)\n", interrupt_id);
+		err = -EINVAL;
+		goto err_out;
+	}
+	spin_lock_irqsave(&tc->interrupt_enable_lock, flags);
+
+	if (tc->interrupt_handlers[interrupt_id].enabled) {
+		dev_warn(dev, "Interrupt ID %d already enabled\n",
+			interrupt_id);
+		err = -EEXIST;
+		goto err_unlock;
+	}
+	tc->interrupt_handlers[interrupt_id].enabled = true;
+
+	if (tc->odin || tc->orion || tc->vali)
+		odin_enable_interrupt_register(tc, interrupt_id);
+	else
+		apollo_enable_interrupt_register(tc, interrupt_id);
+
+err_unlock:
+	spin_unlock_irqrestore(&tc->interrupt_enable_lock, flags);
+err_out:
+	return err;
+}
+EXPORT_SYMBOL(tc_enable_interrupt);
+
+int tc_disable_interrupt(struct device *dev, int interrupt_id)
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+		NULL, NULL);
+	int err = 0;
+	unsigned long flags;
+
+	if (!tc) {
+		dev_err(dev, "No tc device resources found\n");
+		err = -ENODEV;
+		goto err_out;
+	}
+	if (interrupt_id < 0 || interrupt_id >= TC_INTERRUPT_COUNT) {
+		dev_err(dev, "Invalid interrupt ID (%d)\n", interrupt_id);
+		err = -EINVAL;
+		goto err_out;
+	}
+	spin_lock_irqsave(&tc->interrupt_enable_lock, flags);
+
+	if (!tc->interrupt_handlers[interrupt_id].enabled) {
+		dev_warn(dev, "Interrupt ID %d already disabled\n",
+			interrupt_id);
+	}
+	tc->interrupt_handlers[interrupt_id].enabled = false;
+
+	if (tc->odin || tc->orion || tc->vali)
+		odin_disable_interrupt_register(tc, interrupt_id);
+	else
+		apollo_disable_interrupt_register(tc, interrupt_id);
+
+	spin_unlock_irqrestore(&tc->interrupt_enable_lock, flags);
+err_out:
+	return err;
+}
+EXPORT_SYMBOL(tc_disable_interrupt);
+
+int tc_sys_info(struct device *dev, u32 *tmp, u32 *pll)
+{
+	int err = -ENODEV;
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+		NULL, NULL);
+
+	if (!tc) {
+		dev_err(dev, "No tc device resources found\n");
+		goto err_out;
+	}
+
+	if (tc->odin || tc->orion || tc->vali)
+		err = odin_sys_info(tc, tmp, pll);
+	else
+		err = apollo_sys_info(tc, tmp, pll);
+
+err_out:
+	return err;
+}
+EXPORT_SYMBOL(tc_sys_info);
+
+int tc_sys_strings(struct device *dev,
+		   char *str_fpga_rev, size_t size_fpga_rev,
+		   char *str_tcf_core_rev, size_t size_tcf_core_rev,
+		   char *str_tcf_core_target_build_id,
+		   size_t size_tcf_core_target_build_id,
+		   char *str_pci_ver, size_t size_pci_ver,
+		   char *str_macro_ver, size_t size_macro_ver)
+{
+	int err = -ENODEV;
+
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+		NULL, NULL);
+
+	if (!tc) {
+		dev_err(dev, "No tc device resources found\n");
+		goto err_out;
+	}
+
+	if (!str_fpga_rev ||
+	    !size_fpga_rev ||
+	    !str_tcf_core_rev ||
+	    !size_tcf_core_rev ||
+	    !str_tcf_core_target_build_id ||
+	    !size_tcf_core_target_build_id ||
+	    !str_pci_ver ||
+	    !size_pci_ver ||
+	    !str_macro_ver ||
+	    !size_macro_ver) {
+
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	if (tc->odin || tc->orion || tc->vali) {
+		err = odin_sys_strings(tc,
+				 str_fpga_rev, size_fpga_rev,
+				 str_tcf_core_rev, size_tcf_core_rev,
+				 str_tcf_core_target_build_id,
+				 size_tcf_core_target_build_id,
+				 str_pci_ver, size_pci_ver,
+				 str_macro_ver, size_macro_ver);
+	} else {
+		err = apollo_sys_strings(tc,
+				 str_fpga_rev, size_fpga_rev,
+				 str_tcf_core_rev, size_tcf_core_rev,
+				 str_tcf_core_target_build_id,
+				 size_tcf_core_target_build_id,
+				 str_pci_ver, size_pci_ver,
+				 str_macro_ver, size_macro_ver);
+	}
+
+err_out:
+	return err;
+}
+EXPORT_SYMBOL(tc_sys_strings);
+
+int tc_core_clock_speed(struct device *dev)
+{
+	return tc_core_clock;
+}
+EXPORT_SYMBOL(tc_core_clock_speed);
+
+int tc_core_clock_multiplex(struct device *dev)
+{
+	return tc_clock_multiplex;
+}
+EXPORT_SYMBOL(tc_core_clock_multiplex);
+
+unsigned int tc_odin_subvers(struct device *dev)
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+		NULL, NULL);
+
+	if (tc->orion)
+		return 1;
+	else
+		return 0;
+}
+EXPORT_SYMBOL(tc_odin_subvers);
+
+bool tc_pfim_capable(struct device *dev)
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+					   NULL, NULL);
+
+	if (tc->odin && !tc->orion)
+		return (!tc->fbc_bypass &&
+			odin_pfim_compatible(tc));
+
+	return false;
+}
+EXPORT_SYMBOL(tc_pfim_capable);
+
+bool tc_pdp2_compatible(struct device *dev)
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+					   NULL, NULL);
+
+	/* PDP2 is available in all versions of Sleipnir PCB / Odin RTL */
+	return (tc->odin && !tc->orion);
+}
+EXPORT_SYMBOL(tc_pdp2_compatible);
+
+struct dma_chan *tc_dma_chan(struct device *dev, char *name)
+
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+					   NULL, NULL);
+
+	if (tc->odin)
+		return odin_cdma_chan(tc, name);
+	else
+		return NULL;
+}
+EXPORT_SYMBOL(tc_dma_chan);
+
+void tc_dma_chan_free(struct device *dev,
+		      void *chan_prv)
+{
+	struct tc_device *tc = devres_find(dev, tc_devres_release,
+					   NULL, NULL);
+
+	if (tc->odin)
+		odin_cdma_chan_free(tc, chan_prv);
+}
+EXPORT_SYMBOL(tc_dma_chan_free);
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_drv.h b/drivers/gpu/drm/img-rogue/apollo/tc_drv.h
new file mode 100644
index 000000000000..84016d1351ca
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_drv.h
@@ -0,0 +1,201 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _TC_DRV_H
+#define _TC_DRV_H
+
+/*
+ * This contains the hooks for the testchip driver, as used by the Rogue and
+ * PDP sub-devices, and the platform data passed to each of their drivers
+ */
+
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+
+/* Valid values for the TC_MEMORY_CONFIG configuration option */
+#define TC_MEMORY_LOCAL		1
+#define TC_MEMORY_HOST		2
+#define TC_MEMORY_HYBRID	3
+
+/* Baseboard implementation enumeration */
+#define TC_BASEBOARD_APOLLO 1
+#define TC_BASEBOARD_ODIN 2
+#define TC_BASEBOARD_ORION 3
+#define TC_BASEBOARD_VALI 4
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+
+#include PVR_ANDROID_ION_HEADER
+
+/* NOTE: This should be kept in sync with the user side (in buffer_generic.c) */
+#if defined(SUPPORT_RGX)
+#define ION_HEAP_TC_ROGUE    (ION_HEAP_TYPE_CUSTOM+1)
+#endif
+#define ION_HEAP_TC_PDP      (ION_HEAP_TYPE_CUSTOM+2)
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+#define ION_HEAP_TC_SECURE   (ION_HEAP_TYPE_CUSTOM+3)
+#endif
+
+#endif /* defined(SUPPORT_ION) */
+
+#define TC_INTERRUPT_PDP     0
+#define TC_INTERRUPT_EXT     1
+#define TC_INTERRUPT_TC5_PDP 2
+#define TC_INTERRUPT_PDP2    3
+#define TC_INTERRUPT_CDMA    4
+#define TC_INTERRUPT_CDMA2   5
+#define TC_INTERRUPT_OSID0   6
+#define TC_INTERRUPT_OSID1   7
+#define TC_INTERRUPT_OSID2   8
+#define TC_INTERRUPT_OSID3   9
+#define TC_INTERRUPT_OSID4  10
+#define TC_INTERRUPT_OSID5  11
+#define TC_INTERRUPT_OSID6  12
+#define TC_INTERRUPT_OSID7  13
+#define TC_INTERRUPT_COUNT  14
+
+
+int tc_enable(struct device *dev);
+void tc_disable(struct device *dev);
+
+int tc_enable_interrupt(struct device *dev, int interrupt_id);
+int tc_disable_interrupt(struct device *dev, int interrupt_id);
+
+int tc_set_interrupt_handler(struct device *dev, int interrupt_id,
+	void (*handler_function)(void *), void *handler_data);
+
+int tc_sys_info(struct device *dev, u32 *tmp, u32 *pll);
+int tc_sys_strings(struct device *dev,
+	char *str_fpga_rev, size_t size_fpga_rev, char *str_tcf_core_rev,
+	size_t size_tcf_core_rev, char *str_tcf_core_target_build_id,
+	size_t size_tcf_core_target_build_id, char *str_pci_ver,
+	size_t size_pci_ver, char *str_macro_ver, size_t size_macro_ver);
+int tc_core_clock_speed(struct device *dev);
+int tc_core_clock_multiplex(struct device *dev);
+
+unsigned int tc_odin_subvers(struct device *dev);
+
+bool tc_pfim_capable(struct device *dev);
+bool tc_pdp2_compatible(struct device *dev);
+
+void tc_dma_chan_free(struct device *dev, void *chandata);
+struct dma_chan *tc_dma_chan(struct device *dev, char *name);
+
+#define APOLLO_DEVICE_NAME_PDP   "apollo_pdp"
+#define ODN_DEVICE_NAME_PDP      "odin_pdp"
+#define ODN_DEVICE_NAME_CDMA     "odin-cdma"
+
+/* The following structs are initialised and passed down by the parent tc
+ * driver to the respective sub-drivers
+ */
+
+struct tc_pdp_platform_data {
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_device *ion_device;
+	int ion_heap_id;
+#endif
+	resource_size_t memory_base;
+
+	/* The following is used by the drm_pdp driver as it manages the
+	 * pdp memory
+	 */
+	resource_size_t pdp_heap_memory_base;
+	resource_size_t pdp_heap_memory_size;
+
+	/* Used to export host address instead of pdp address, depends on the
+	 * TC memory mode.
+	 *
+	 * PDP phys address space is from 0 to end of local device memory,
+	 * however if the TC is configured to operate in hybrid mode then the
+	 * GPU is configured to match the CPU phys address space view.
+	 */
+	bool dma_map_export_host_addr;
+};
+
+struct tc_dma_platform_data {
+	u32 addr_width;
+	u32 num_dmas;
+	bool has_dre;
+	bool has_sg;
+};
+
+#if defined(SUPPORT_RGX)
+
+#define TC_DEVICE_NAME_ROGUE "tc_rogue"
+
+struct tc_rogue_platform_data {
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_device *ion_device;
+	int ion_heap_id;
+#endif
+	/* The testchip memory mode (LOCAL, HOST or HYBRID) */
+	int mem_mode;
+
+	/* The testchip baseboard type (APOLLO, ODIN or ORION) */
+	int baseboard;
+
+	/* The base address of the testchip memory (CPU physical address) -
+	 * used to convert from CPU-Physical to device-physical addresses
+	 */
+	resource_size_t tc_memory_base;
+
+	/* The following is used to setup the services heaps that map to the
+	 * ion heaps
+	 */
+	resource_size_t pdp_heap_memory_base;
+	resource_size_t pdp_heap_memory_size;
+	resource_size_t rogue_heap_memory_base;
+	resource_size_t rogue_heap_memory_size;
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	resource_size_t secure_heap_memory_base;
+	resource_size_t secure_heap_memory_size;
+#endif
+
+	/* DMA channel names for RGX usage */
+	char *tc_dma_tx_chan_name;
+	char *tc_dma_rx_chan_name;
+};
+
+#endif /* defined(SUPPORT_RGX) */
+
+#endif /* _TC_DRV_H */
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_drv_internal.h b/drivers/gpu/drm/img-rogue/apollo/tc_drv_internal.h
new file mode 100644
index 000000000000..b10ba49091b4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_drv_internal.h
@@ -0,0 +1,204 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _TC_DRV_INTERNAL_H
+#define _TC_DRV_INTERNAL_H
+
+#include "tc_drv.h"
+
+#include <linux/version.h>
+
+#if defined(TC_FAKE_INTERRUPTS)
+#define FAKE_INTERRUPT_TIME_MS 20
+#include <linux/timer.h>
+#include <linux/time.h>
+#endif
+
+#define DRV_NAME "tc"
+
+/* This is a guess of what's a minimum sensible size for the ext heap
+ * It is only used for a warning if the ext heap is smaller, and does
+ * not affect the functional logic in any way
+ */
+#define TC_EXT_MINIMUM_MEM_SIZE (10*1024*1024)
+
+#if defined(SUPPORT_DMA_HEAP)
+ #if defined(SUPPORT_FAKE_SECURE_DMA_HEAP)
+  #define TC_DMA_HEAP_COUNT 3
+ #else
+  #define TC_DMA_HEAP_COUNT 2
+ #endif
+#elif defined(SUPPORT_ION)
+ #if defined(SUPPORT_RGX) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+  #define TC_ION_HEAP_BASE_COUNT 3
+ #else
+  #define TC_ION_HEAP_BASE_COUNT 2
+ #endif
+
+ #if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+  #define TC_ION_HEAP_COUNT (TC_ION_HEAP_BASE_COUNT + 1)
+ #else
+  #define TC_ION_HEAP_COUNT TC_ION_HEAP_BASE_COUNT
+ #endif
+#endif /* defined(SUPPORT_ION) */
+
+/* Convert a byte offset to a 32 bit dword offset */
+#define DWORD_OFFSET(byte_offset)  ((byte_offset)>>2)
+
+#define HEX2DEC(v)                 ((((v) >> 4) * 10) + ((v) & 0x0F))
+
+enum tc_version_t {
+	TC_INVALID_VERSION,
+	APOLLO_VERSION_TCF_2,
+	APOLLO_VERSION_TCF_5,
+	APOLLO_VERSION_TCF_BONNIE,
+	ODIN_VERSION_TCF_BONNIE,
+	ODIN_VERSION_FPGA,
+	ODIN_VERSION_ORION,
+	ODIN_VERSION_VALI,
+};
+
+struct tc_interrupt_handler {
+	bool enabled;
+	void (*handler_function)(void *data);
+	void *handler_data;
+};
+
+struct tc_region {
+	resource_size_t base;
+	resource_size_t size;
+};
+
+struct tc_io_region {
+	struct tc_region region;
+	void __iomem *registers;
+};
+
+struct tc_device {
+	struct pci_dev *pdev;
+
+	enum tc_version_t version;
+	bool odin;
+	bool vali;
+	bool orion;
+
+	int mem_mode;
+
+	struct tc_io_region tcf;
+	struct tc_io_region tcf_pll;
+
+	struct tc_region tc_mem;
+
+	struct platform_device *pdp_dev;
+
+	resource_size_t pdp_heap_mem_base;
+	resource_size_t pdp_heap_mem_size;
+
+	struct platform_device *ext_dev[RGX_NUM_DRIVERS_SUPPORTED];
+
+	resource_size_t ext_heap_mem_base;
+	resource_size_t ext_heap_mem_size;
+
+	struct platform_device *dma_dev;
+
+	struct dma_chan *dma_chans[2];
+	unsigned int dma_refcnt[2];
+	unsigned int dma_nchan;
+	struct mutex dma_mutex;
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP) || \
+	defined(SUPPORT_FAKE_SECURE_DMA_HEAP)
+	resource_size_t secure_heap_mem_base;
+	resource_size_t secure_heap_mem_size;
+#endif
+
+	int mtrr;
+	spinlock_t interrupt_handler_lock;
+	spinlock_t interrupt_enable_lock;
+
+	struct tc_interrupt_handler
+		interrupt_handlers[TC_INTERRUPT_COUNT];
+
+#if defined(TC_FAKE_INTERRUPTS)
+	struct timer_list timer;
+#endif
+
+#if defined(SUPPORT_DMA_HEAP)
+	struct dma_heap *dma_heaps[TC_DMA_HEAP_COUNT];
+#elif defined(SUPPORT_ION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	struct ion_device *ion_device;
+#endif
+	struct ion_heap *ion_heaps[TC_ION_HEAP_COUNT];
+	int ion_heap_count;
+#endif /* defined(SUPPORT_ION) */
+
+	bool fbc_bypass;
+
+	struct dentry *debugfs_tc_dir;
+	struct dentry *debugfs_rogue_name;
+};
+
+int tc_mtrr_setup(struct tc_device *tc);
+void tc_mtrr_cleanup(struct tc_device *tc);
+
+int tc_is_interface_aligned(u32 eyes, u32 clk_taps, u32 train_ack);
+
+int tc_iopol32_nonzero(u32 mask, void __iomem *addr);
+
+int request_pci_io_addr(struct pci_dev *pdev, u32 index,
+	resource_size_t offset, resource_size_t length);
+void release_pci_io_addr(struct pci_dev *pdev, u32 index,
+	resource_size_t start, resource_size_t length);
+
+int setup_io_region(struct pci_dev *pdev,
+	struct tc_io_region *region, u32 index,
+	resource_size_t offset,	resource_size_t size);
+
+#if defined(TC_FAKE_INTERRUPTS)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+void tc_irq_fake_wrapper(struct timer_list *t);
+#else
+void tc_irq_fake_wrapper(unsigned long data);
+#endif
+#endif /* defined(TC_FAKE_INTERRUPTS) */
+
+#endif /* _TC_DRV_INTERNAL_H */
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_odin.c b/drivers/gpu/drm/img-rogue/apollo/tc_odin.c
new file mode 100644
index 000000000000..2e676ac7e342
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_odin.c
@@ -0,0 +1,2424 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This is a device driver for the odin testchip framework. It creates
+ * platform devices for the pdp and ext sub-devices, and exports functions
+ * to manage the shared interrupt handling
+ */
+
+#include <linux/version.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/version.h>
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+
+#include "tc_drv_internal.h"
+#include "tc_odin.h"
+#if defined(SUPPORT_DMA_HEAP)
+#include "tc_dmabuf_heap.h"
+#elif defined(SUPPORT_ION)
+#include "tc_ion.h"
+#endif
+
+/* Odin (3rd gen TCF FPGA) */
+#include "odin_defs.h"
+#include "odin_regs.h"
+#include "bonnie_tcf.h"
+#include "tc_clocks.h"
+
+/* Orion demo platform */
+#include "orion_defs.h"
+#include "orion_regs.h"
+
+/* Odin/Orion common registers */
+#include "tc_odin_common_regs.h"
+
+/* Macros to set and get register fields */
+#define REG_FIELD_GET(v, str) \
+	(u32)(((v) & (str##_MASK)) >> (str##_SHIFT))
+#define REG_FIELD_SET(v, f, str) \
+	(v = (u32)(((v) & (u32)~(str##_MASK)) |		\
+		   (u32)(((f) << (str##_SHIFT)) & (str##_MASK))))
+
+#define SAI_STATUS_UNALIGNED 0
+#define SAI_STATUS_ALIGNED   1
+#define SAI_STATUS_ERROR     2
+
+/* Odin/Orion shared masks */
+static const u32 CHANGE_SET_SET_MASK[] = {
+	ODN_CHANGE_SET_SET_MASK,
+	SRS_CHANGE_SET_SET_MASK
+};
+static const u32 CHANGE_SET_SET_SHIFT[] = {
+	ODN_CHANGE_SET_SET_SHIFT,
+	SRS_CHANGE_SET_SET_SHIFT
+};
+static const u32 USER_ID_ID_MASK[] = {
+	ODN_USER_ID_ID_MASK,
+	SRS_USER_ID_ID_MASK
+};
+static const u32 USER_ID_ID_SHIFT[] = {
+	ODN_USER_ID_ID_SHIFT,
+	SRS_USER_ID_ID_SHIFT
+};
+static const u32 USER_BUILD_BUILD_MASK[] = {
+	ODN_USER_BUILD_BUILD_MASK,
+	SRS_USER_BUILD_BUILD_MASK
+};
+static const u32 USER_BUILD_BUILD_SHIFT[] = {
+	ODN_USER_BUILD_BUILD_SHIFT,
+	SRS_USER_BUILD_BUILD_SHIFT
+};
+static const u32 INPUT_CLOCK_SPEED_MIN[] = {
+	ODN_INPUT_CLOCK_SPEED_MIN,
+	SRS_INPUT_CLOCK_SPEED_MIN
+};
+static const u32 INPUT_CLOCK_SPEED_MAX[] = {
+	ODN_INPUT_CLOCK_SPEED_MAX,
+	SRS_INPUT_CLOCK_SPEED_MAX
+};
+static const u32 OUTPUT_CLOCK_SPEED_MIN[] = {
+	ODN_OUTPUT_CLOCK_SPEED_MIN,
+	SRS_OUTPUT_CLOCK_SPEED_MIN
+};
+static const u32 OUTPUT_CLOCK_SPEED_MAX[] = {
+	ODN_OUTPUT_CLOCK_SPEED_MAX,
+	SRS_OUTPUT_CLOCK_SPEED_MAX
+};
+static const u32 VCO_MIN[] = {
+	ODN_VCO_MIN,
+	SRS_VCO_MIN
+};
+static const u32 VCO_MAX[] = {
+	ODN_VCO_MAX,
+	SRS_VCO_MAX
+};
+static const u32 PFD_MIN[] = {
+	ODN_PFD_MIN,
+	SRS_PFD_MIN
+};
+static const u32 PFD_MAX[] = {
+	ODN_PFD_MAX,
+	SRS_PFD_MAX
+};
+
+#if defined(SUPPORT_RGX)
+
+static void spi_write(struct tc_device *tc, u32 off, u32 val)
+{
+	iowrite32(off, tc->tcf.registers
+		  + ODN_REG_BANK_TCF_SPI_MASTER
+		  + ODN_SPI_MST_ADDR_RDNWR);
+	iowrite32(val, tc->tcf.registers
+		  + ODN_REG_BANK_TCF_SPI_MASTER
+		  + ODN_SPI_MST_WDATA);
+	iowrite32(0x1, tc->tcf.registers
+		  + ODN_REG_BANK_TCF_SPI_MASTER
+		  + ODN_SPI_MST_GO);
+	udelay(1000);
+}
+
+static int spi_read(struct tc_device *tc, u32 off, u32 *val)
+{
+	int cnt = 0;
+	u32 spi_mst_status;
+
+	iowrite32(0x40000 | off, tc->tcf.registers
+		  + ODN_REG_BANK_TCF_SPI_MASTER
+		  + ODN_SPI_MST_ADDR_RDNWR);
+	iowrite32(0x1, tc->tcf.registers
+		  + ODN_REG_BANK_TCF_SPI_MASTER
+		  + ODN_SPI_MST_GO);
+	udelay(100);
+
+	do {
+		spi_mst_status = ioread32(tc->tcf.registers
+					  + ODN_REG_BANK_TCF_SPI_MASTER
+					  + ODN_SPI_MST_STATUS);
+
+		if (cnt++ > 10000) {
+			dev_err(&tc->pdev->dev,
+				"%s: Time out reading SPI reg (0x%x)\n",
+				__func__, off);
+			return -1;
+		}
+
+	} while (spi_mst_status != 0x08);
+
+	*val = ioread32(tc->tcf.registers
+			+ ODN_REG_BANK_TCF_SPI_MASTER
+			+ ODN_SPI_MST_RDATA);
+
+	return 0;
+}
+
+/* Returns 1 for aligned, 0 for unaligned */
+static int get_odin_sai_status(struct tc_device *tc, int bank)
+{
+	void __iomem *bank_addr = tc->tcf.registers
+					+ ODN_REG_BANK_SAI_RX_DDR(bank);
+	void __iomem *reg_addr;
+	u32 eyes;
+	u32 clk_taps;
+	u32 train_ack;
+
+	reg_addr = bank_addr + ODN_SAI_RX_DEBUG_SAI_EYES;
+	eyes = ioread32(reg_addr);
+
+	reg_addr = bank_addr + ODN_SAI_RX_DEBUG_SAI_CLK_TAPS;
+	clk_taps = ioread32(reg_addr);
+
+	reg_addr = bank_addr + ODN_SAI_RX_DEBUG_SAI_TRAIN_ACK;
+	train_ack = ioread32(reg_addr);
+
+#if 0 /* enable this to get debug info if the board is not aligning */
+	dev_info(&tc->pdev->dev,
+		"odin bank %d align: eyes=%08x clk_taps=%08x train_ack=%08x\n",
+		bank, eyes, clk_taps, train_ack);
+#endif
+
+	if (tc_is_interface_aligned(eyes, clk_taps, train_ack))
+		return SAI_STATUS_ALIGNED;
+
+	dev_warn(&tc->pdev->dev, "odin bank %d is unaligned\n", bank);
+	return SAI_STATUS_UNALIGNED;
+}
+
+/* Read the odin multi clocked bank align status.
+ * Returns 1 for aligned, 0 for unaligned
+ */
+static int read_odin_mca_status(struct tc_device *tc)
+{
+	void __iomem *bank_addr = tc->tcf.registers
+					+ ODN_REG_BANK_MULTI_CLK_ALIGN;
+	void __iomem *reg_addr = bank_addr + ODN_MCA_DEBUG_MCA_STATUS;
+	u32 mca_status;
+
+	mca_status = ioread32(reg_addr);
+
+#if 0 /* Enable this if there are alignment issues */
+	dev_info(&tc->pdev->dev,
+		"Odin MCA_STATUS = %08x\n", mca_status);
+#endif
+	return mca_status & ODN_ALIGNMENT_FOUND_MASK;
+}
+
+/* Read the DUT multi clocked bank align status.
+ * Returns 1 for aligned, 0 for unaligned
+ */
+static int read_dut_mca_status(struct tc_device *tc)
+{
+	u32 mca_status;
+	const int mca_status_register_offset = 1; /* not in bonnie_tcf.h */
+	int spi_address = DWORD_OFFSET(BONNIE_TCF_OFFSET_MULTI_CLK_ALIGN);
+
+	spi_address = DWORD_OFFSET(BONNIE_TCF_OFFSET_MULTI_CLK_ALIGN)
+			+ mca_status_register_offset;
+
+	spi_read(tc, spi_address, &mca_status);
+
+#if 0 /* Enable this if there are alignment issues */
+	dev_info(&tc->pdev->dev,
+		"DUT MCA_STATUS = %08x\n", mca_status);
+#endif
+	return mca_status & 1; /* 'alignment found' status is in bit 1 */
+}
+
+/* Returns 1 for aligned, 0 for unaligned */
+static int get_dut_sai_status(struct tc_device *tc, int bank)
+{
+	u32 eyes;
+	u32 clk_taps;
+	u32 train_ack;
+	const u32 bank_base = DWORD_OFFSET(BONNIE_TCF_OFFSET_SAI_RX_1
+				+ (BONNIE_TCF_OFFSET_SAI_RX_DELTA * bank));
+	int spi_timeout;
+
+	spi_timeout = spi_read(tc, bank_base
+		+ DWORD_OFFSET(BONNIE_TCF_OFFSET_SAI_EYES), &eyes);
+	if (spi_timeout)
+		return SAI_STATUS_ERROR;
+
+	spi_read(tc, bank_base
+		+ DWORD_OFFSET(BONNIE_TCF_OFFSET_SAI_CLK_TAPS), &clk_taps);
+	spi_read(tc, bank_base
+		+ DWORD_OFFSET(BONNIE_TCF_OFFSET_SAI_TRAIN_ACK), &train_ack);
+
+#if 0 /* enable this to get debug info if the board is not aligning */
+	dev_info(&tc->pdev->dev,
+		"dut  bank %d align: eyes=%08x clk_taps=%08x train_ack=%08x\n",
+		bank, eyes, clk_taps, train_ack);
+#endif
+
+	if (tc_is_interface_aligned(eyes, clk_taps, train_ack))
+		return SAI_STATUS_ALIGNED;
+
+	dev_warn(&tc->pdev->dev, "dut bank %d is unaligned\n", bank);
+	return SAI_STATUS_UNALIGNED;
+}
+
+/*
+ * Returns the divider group register fields for the specified counter value.
+ * See Xilinx Application Note xapp888.
+ */
+static void odin_mmcm_reg_param_calc(u32 value, u32 *low, u32 *high,
+				     u32 *edge, u32 *no_count)
+{
+	if (value == 1U) {
+		*no_count = 1U;
+		*edge = 0;
+		*high = 0;
+		*low = 0;
+	} else {
+		*no_count = 0;
+		*edge = value % 2U;
+		*high = value >> 1;
+		*low = (value + *edge) >> 1U;
+	}
+}
+
+/*
+ * Returns the MMCM Input Divider, FB Multiplier and Output Divider values for
+ * the specified input frequency and target output frequency.
+ * Function doesn't support fractional values for multiplier and output divider
+ * As per Xilinx 7 series FPGAs clocking resources user guide, aims for highest
+ * VCO and smallest D and M.
+ * Configured for Xilinx Virtex7 speed grade 2.
+ */
+static int odin_mmcm_counter_calc(struct device *dev,
+				  u32 freq_input, u32 freq_output,
+				  u32 *d, u32 *m, u32 *o)
+{
+	u32 tcver = tc_odin_subvers(dev);
+	u32 best_diff, d_best, m_best, o_best;
+	u32 m_min, m_max, m_ideal;
+	u32 d_cur, m_cur, o_cur;
+	u32 d_min, d_max;
+
+	/*
+	 * Check specified input frequency is within range
+	 */
+	if (freq_input < INPUT_CLOCK_SPEED_MIN[tcver]) {
+		dev_err(dev, "Input frequency (%u hz) below minimum supported value (%u hz)\n",
+			freq_input, INPUT_CLOCK_SPEED_MIN[tcver]);
+		return -EINVAL;
+	}
+	if (freq_input > INPUT_CLOCK_SPEED_MAX[tcver]) {
+		dev_err(dev, "Input frequency (%u hz) above maximum supported value (%u hz)\n",
+			freq_input, INPUT_CLOCK_SPEED_MAX[tcver]);
+		return -EINVAL;
+	}
+
+	/*
+	 * Check specified target frequency is within range
+	 */
+	if (freq_output < OUTPUT_CLOCK_SPEED_MIN[tcver]) {
+		dev_err(dev, "Output frequency (%u hz) below minimum supported value (%u hz)\n",
+			freq_input, OUTPUT_CLOCK_SPEED_MIN[tcver]);
+		return -EINVAL;
+	}
+	if (freq_output > OUTPUT_CLOCK_SPEED_MAX[tcver]) {
+		dev_err(dev, "Output frequency (%u hz) above maximum supported value (%u hz)\n",
+			freq_output, OUTPUT_CLOCK_SPEED_MAX[tcver]);
+		return -EINVAL;
+	}
+
+	/*
+	 * Calculate min and max for Input Divider.
+	 * Refer Xilinx 7 series FPGAs clocking resources user guide
+	 * equation 3-6 and 3-7
+	 */
+	d_min = DIV_ROUND_UP(freq_input, PFD_MAX[tcver]);
+	d_max = min(freq_input/PFD_MIN[tcver], (u32)ODN_DREG_VALUE_MAX);
+
+	/*
+	 * Calculate min and max for Input Divider.
+	 * Refer Xilinx 7 series FPGAs clocking resources user guide.
+	 * equation 3-8 and 3-9
+	 */
+	m_min = DIV_ROUND_UP((VCO_MIN[tcver] * d_min), freq_input);
+	m_max = min(((VCO_MAX[tcver] * d_max) / freq_input),
+		    (u32)ODN_MREG_VALUE_MAX);
+
+	for (d_cur = d_min; d_cur <= d_max; d_cur++) {
+		/*
+		 * Refer Xilinx 7 series FPGAs clocking resources user guide.
+		 * equation 3-10
+		 */
+		m_ideal = min(((d_cur * VCO_MAX[tcver])/freq_input), m_max);
+
+		for (m_cur = m_ideal; m_cur >= m_min; m_cur -= 1) {
+			/**
+			 * Skip if VCO for given 'm' and 'd' value is not an
+			 * integer since fractional component is not supported
+			 */
+			if (((freq_input * m_cur) % d_cur) != 0)
+				continue;
+
+			/**
+			 * Skip if divider for given 'm' and 'd' value is not
+			 * an integer since fractional component is not
+			 * supported
+			 */
+			if ((freq_input * m_cur) % (d_cur * freq_output) != 0)
+				continue;
+
+			/**
+			 * Calculate output divider value.
+			 */
+			o_cur = (freq_input * m_cur)/(d_cur * freq_output);
+
+			*d = d_cur;
+			*m = m_cur;
+			*o = o_cur;
+			return 0;
+		}
+	}
+
+	/*
+	 * Failed to find exact optimal solution with high VCO. Brute-force find
+	 * a suitable config, again prioritising high VCO, to get lowest jitter
+	 */
+	d_min = 1; d_max = (u32)ODN_DREG_VALUE_MAX;
+	m_min = 1; m_max = (u32)ODN_MREG_VALUE_MAX;
+	best_diff = 0xFFFFFFFF;
+
+	for (d_cur = d_min; d_cur <= d_max; d_cur++) {
+		for (m_cur = m_max; m_cur >= m_min; m_cur -= 1) {
+			u32 pfd, vco, o_avg, o_min, o_max;
+
+			pfd = freq_input / d_cur;
+			vco = pfd * m_cur;
+
+			if (pfd < PFD_MIN[tcver])
+				continue;
+
+			if (pfd > PFD_MAX[tcver])
+				continue;
+
+			if (vco < VCO_MIN[tcver])
+				continue;
+
+			if (vco > VCO_MAX[tcver])
+				continue;
+
+			/*
+			 * A range of -1/+3 around o_avg gives us 100kHz granularity.
+			 * It can be extended further.
+			 */
+			o_avg = vco / freq_output;
+			o_min = (o_avg >= 2) ? (o_avg - 1) : 1;
+			o_max = o_avg + 3;
+			if (o_max > (u32)ODN_OREG_VALUE_MAX)
+				o_max = (u32)ODN_OREG_VALUE_MAX;
+
+			for (o_cur = o_min; o_cur <= o_max; o_cur++) {
+				u32 freq_cur, diff_cur;
+
+				freq_cur = vco / o_cur;
+
+				if (freq_cur > freq_output)
+					continue;
+
+				diff_cur = freq_output - freq_cur;
+
+				if (diff_cur == 0) {
+					/* Found an exact match */
+					*d = d_cur;
+					*m = m_cur;
+					*o = o_cur;
+					return 0;
+				}
+
+				if (diff_cur < best_diff) {
+					best_diff = diff_cur;
+					d_best = d_cur;
+					m_best = m_cur;
+					o_best = o_cur;
+				}
+			}
+		}
+	}
+
+	if (best_diff != 0xFFFFFFFF) {
+		dev_warn(dev, "Odin: Found similar freq of %u Hz\n", freq_output - best_diff);
+		*d = d_best;
+		*m = m_best;
+		*o = o_best;
+		return 0;
+	}
+
+	dev_err(dev, "Odin: Unable to find integer values for d, m and o for requested frequency (%u)\n",
+		freq_output);
+
+	return -ERANGE;
+}
+
+static int odin_fpga_set_dut_core_clk(struct tc_device *tc,
+				      u32 input_clk, u32 output_clk)
+{
+	int err = 0;
+	u32 in_div, mul, out_div;
+	u32 high_time, low_time, edge, no_count;
+	u32 value;
+	void __iomem *base = tc->tcf.registers;
+	void __iomem *clk_blk_base = base + ODN_REG_BANK_ODN_CLK_BLK;
+	struct device *dev = &tc->pdev->dev;
+
+	err = odin_mmcm_counter_calc(dev, input_clk, output_clk, &in_div,
+				     &mul, &out_div);
+	if (err != 0)
+		return err;
+
+	/* Put DUT into reset */
+	iowrite32(ODN_EXTERNAL_RESETN_DUT_SPI_MASK,
+		  base + ODN_CORE_EXTERNAL_RESETN);
+	msleep(20);
+
+	/* Put DUT Core MMCM into reset */
+	iowrite32(ODN_CLK_GEN_RESET_DUT_CORE_MMCM_MASK,
+		  base + ODN_CORE_CLK_GEN_RESET);
+	msleep(20);
+
+	/* Calculate the register fields for output divider */
+	odin_mmcm_reg_param_calc(out_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to output divider register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_OUT_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			ODN_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			ODN_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_OUT_DIVIDER1);
+
+	/* Read-modify-write the required fields to output divider register 2 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_OUT_DIVIDER2);
+	REG_FIELD_SET(value, edge,
+			ODN_DUT_CORE_CLK_OUT_DIVIDER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			ODN_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_OUT_DIVIDER2);
+
+	/* Calculate the register fields for multiplier */
+	odin_mmcm_reg_param_calc(mul, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to multiplier register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER1);
+	REG_FIELD_SET(value, high_time,
+			ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER1);
+
+	/* Read-modify-write the required fields to multiplier register 2 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER2);
+	REG_FIELD_SET(value, edge,
+			ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER2);
+
+	/* Calculate the register fields for input divider */
+	odin_mmcm_reg_param_calc(in_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to input divider register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_IN_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME);
+	REG_FIELD_SET(value, edge,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE);
+	REG_FIELD_SET(value, no_count,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_IN_DIVIDER1);
+
+	/* Bring DUT clock MMCM out of reset */
+	iowrite32(0, tc->tcf.registers + ODN_CORE_CLK_GEN_RESET);
+
+	err = tc_iopol32_nonzero(ODN_MMCM_LOCK_STATUS_DUT_CORE,
+				 base + ODN_CORE_MMCM_LOCK_STATUS);
+	if (err != 0) {
+		dev_err(dev, "MMCM failed to lock for DUT core\n");
+		return err;
+	}
+
+	/* Bring DUT out of reset */
+	iowrite32(ODN_EXTERNAL_RESETN_DUT_SPI_MASK |
+		  ODN_EXTERNAL_RESETN_DUT_MASK,
+		  tc->tcf.registers + ODN_CORE_EXTERNAL_RESETN);
+	msleep(20);
+
+	dev_info(dev, "DUT core clock set-up successful\n");
+
+	return err;
+}
+
+static int odin_fpga_set_dut_if_clk(struct tc_device *tc,
+				    u32 input_clk, u32 output_clk)
+{
+	int err = 0;
+	u32 in_div, mul, out_div;
+	u32 high_time, low_time, edge, no_count;
+	u32 value;
+	void __iomem *base = tc->tcf.registers;
+	void __iomem *clk_blk_base = base + ODN_REG_BANK_ODN_CLK_BLK;
+	struct device *dev = &tc->pdev->dev;
+
+	err = odin_mmcm_counter_calc(dev, input_clk, output_clk,
+				     &in_div, &mul, &out_div);
+	if (err != 0)
+		return err;
+
+	/* Put DUT into reset */
+	iowrite32(ODN_EXTERNAL_RESETN_DUT_SPI_MASK,
+		  base + ODN_CORE_EXTERNAL_RESETN);
+	msleep(20);
+
+	/* Put DUT Core MMCM into reset */
+	iowrite32(ODN_CLK_GEN_RESET_DUT_IF_MMCM_MASK,
+		  base + ODN_CORE_CLK_GEN_RESET);
+	msleep(20);
+
+	/* Calculate the register fields for output divider */
+	odin_mmcm_reg_param_calc(out_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to output divider register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER1);
+
+	/* Read-modify-write the required fields to output divider register 2 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER2);
+	REG_FIELD_SET(value, edge,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER2);
+
+	/* Calculate the register fields for multiplier */
+	odin_mmcm_reg_param_calc(mul, &high_time, &low_time, &edge, &no_count);
+
+	/* Read-modify-write the required fields to multiplier register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER1);
+	REG_FIELD_SET(value, high_time,
+			ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER1);
+
+	/* Read-modify-write the required fields to multiplier register 2 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER2);
+	REG_FIELD_SET(value, edge,
+			ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER2);
+
+	/* Calculate the register fields for input divider */
+	odin_mmcm_reg_param_calc(in_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to input divider register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_IN_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME);
+	REG_FIELD_SET(value, edge,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE);
+	REG_FIELD_SET(value, no_count,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_IN_DIVIDER1);
+
+	/* Bring DUT interface clock MMCM out of reset */
+	iowrite32(0, tc->tcf.registers + ODN_CORE_CLK_GEN_RESET);
+
+	err = tc_iopol32_nonzero(ODN_MMCM_LOCK_STATUS_DUT_IF,
+				 base + ODN_CORE_MMCM_LOCK_STATUS);
+	if (err != 0) {
+		dev_err(dev, "MMCM failed to lock for DUT IF\n");
+		return err;
+	}
+
+	/* Bring DUT out of reset */
+	iowrite32(ODN_EXTERNAL_RESETN_DUT_SPI_MASK |
+		  ODN_EXTERNAL_RESETN_DUT_MASK,
+		  tc->tcf.registers + ODN_CORE_EXTERNAL_RESETN);
+	msleep(20);
+
+	dev_info(dev, "DUT IF clock set-up successful\n");
+
+	return err;
+}
+
+static void odin_fpga_update_dut_clk_freq(struct tc_device *tc,
+					  int *core_clock, int *mem_clock, int *clock_multiplex)
+{
+	struct device *dev = &tc->pdev->dev;
+	int dut_clk_info = 0;
+	int dut_clk_multiplex = 0;
+
+#if defined(SUPPORT_FPGA_DUT_CLK_INFO)
+	dut_clk_info = ioread32(tc->tcf.registers + ODN_CORE_DUT_CLK_INFO);
+#endif
+
+	if ((dut_clk_info != 0) && (dut_clk_info != 0xbaadface)
+				&& (dut_clk_info != 0xffffffff)) {
+		dev_info(dev, "ODN_DUT_CLK_INFO = %08x\n", dut_clk_info);
+
+		if (*core_clock == 0) {
+			*core_clock = ((dut_clk_info & ODN_DUT_CLK_INFO_CORE_MASK)
+					   >> ODN_DUT_CLK_INFO_CORE_SHIFT) * 1000000;
+			dev_info(dev, "Using register DUT core clock value: %i\n",
+						*core_clock);
+		} else {
+			dev_info(dev, "Using module param DUT core clock value: %i\n",
+						*core_clock);
+		}
+
+		if (*mem_clock == 0) {
+			*mem_clock = ((dut_clk_info & ODN_DUT_CLK_INFO_MEM_MASK)
+				   >> ODN_DUT_CLK_INFO_MEM_SHIFT) * 1000000;
+			dev_info(dev, "Using register DUT mem clock value: %i\n",
+			 *mem_clock);
+		} else {
+			dev_info(dev, "Using module param DUT mem clock value: %i\n",
+						*mem_clock);
+		}
+	} else {
+		if (*core_clock == 0) {
+			*core_clock = RGX_TC_CORE_CLOCK_SPEED;
+			dev_info(dev, "Using default DUT core clock value: %i\n",
+					 *core_clock);
+		} else {
+			dev_info(dev, "Using module param DUT core clock value: %i\n",
+						*core_clock);
+		}
+
+		if (*mem_clock == 0) {
+			*mem_clock = RGX_TC_MEM_CLOCK_SPEED;
+			dev_info(dev, "Using default DUT mem clock value: %i\n",
+					 *mem_clock);
+		} else {
+			dev_info(dev, "Using module param DUT mem clock value: %i\n",
+						*mem_clock);
+		}
+	}
+
+#if defined(SUPPORT_FPGA_DUT_MULTIPLEX_INFO)
+	dut_clk_multiplex = ioread32(tc->tcf.registers + ODN_CORE_DUT_MULTIPLX_INFO);
+#endif
+
+	if ((dut_clk_multiplex != 0) && (dut_clk_multiplex != 0xbaadface)
+					&& (dut_clk_multiplex != 0xffffffff)) {
+		dev_info(dev, "ODN_DUT_MULTIPLX_INFO = %08x\n", dut_clk_multiplex);
+		if (*clock_multiplex == 0) {
+			*clock_multiplex = ((dut_clk_multiplex & ODN_DUT_MULTIPLX_INFO_MEM_MASK)
+					   >> ODN_DUT_MULTIPLX_INFO_MEM_SHIFT);
+			dev_info(dev, "Using register DUT clock multiplex: %i\n",
+						*clock_multiplex);
+		} else {
+			dev_info(dev, "Using module param DUT clock multiplex: %i\n",
+						*clock_multiplex);
+		}
+	} else {
+		if (*clock_multiplex == 0) {
+			*clock_multiplex = RGX_TC_CLOCK_MULTIPLEX;
+			dev_info(dev, "Using default DUT clock multiplex: %i\n",
+					 *clock_multiplex);
+		} else {
+			dev_info(dev, "Using module param DUT clock multiplex: %i\n",
+						*clock_multiplex);
+		}
+	}
+}
+
+static int odin_hard_reset_fpga(struct tc_device *tc,
+				int *core_clock, int *mem_clock, int *clock_mulitplex)
+{
+	int err = 0;
+
+	odin_fpga_update_dut_clk_freq(tc, core_clock, mem_clock, clock_mulitplex);
+
+	err = odin_fpga_set_dut_core_clk(tc, ODN_INPUT_CLOCK_SPEED, *core_clock);
+	if (err != 0)
+		goto err_out;
+
+	err = odin_fpga_set_dut_if_clk(tc, ODN_INPUT_CLOCK_SPEED, *mem_clock);
+
+err_out:
+	return err;
+}
+
+static int odin_get_default_clocks_vali(struct tc_device *tc,
+				int *core_clock, int *mem_clock, int *clock_mulitplex)
+{
+	/* For now use default TC values */
+	*core_clock = RGX_TC_CORE_CLOCK_SPEED;
+	*mem_clock = RGX_TC_MEM_CLOCK_SPEED;
+	*clock_mulitplex = RGX_TC_CLOCK_MULTIPLEX;
+	return 0;
+}
+
+static int odin_hard_reset_bonnie(struct tc_device *tc)
+{
+	int reset_cnt = 0;
+	bool aligned = false;
+	int alignment_found;
+
+	msleep(100);
+
+	/* It is essential to do an SPI reset once on power-up before
+	 * doing any DUT reads via the SPI interface.
+	 */
+	iowrite32(1, tc->tcf.registers		/* set bit 1 low */
+			+ ODN_CORE_EXTERNAL_RESETN);
+	msleep(20);
+
+	iowrite32(3, tc->tcf.registers		/* set bit 1 high */
+			+ ODN_CORE_EXTERNAL_RESETN);
+	msleep(20);
+
+	while (!aligned && (reset_cnt < 20)) {
+
+		int bank;
+
+		/* Reset the DUT to allow the SAI to retrain */
+		iowrite32(2, /* set bit 0 low */
+			tc->tcf.registers
+			+ ODN_CORE_EXTERNAL_RESETN);
+
+		/* Hold the DUT in reset for 50ms */
+		msleep(50);
+
+		/* Take the DUT out of reset */
+		iowrite32(3, /* set bit 0 hi */
+			tc->tcf.registers
+			+ ODN_CORE_EXTERNAL_RESETN);
+		reset_cnt++;
+
+		/* Wait 200ms for the DUT to stabilise */
+		msleep(200);
+
+		/* Check the odin Multi Clocked bank Align status */
+		alignment_found = read_odin_mca_status(tc);
+		dev_info(&tc->pdev->dev,
+				"Odin mca_status indicates %s\n",
+				(alignment_found)?"aligned":"UNALIGNED");
+
+		/* Check the DUT MCA status */
+		alignment_found = read_dut_mca_status(tc);
+		dev_info(&tc->pdev->dev,
+				"DUT mca_status indicates %s\n",
+				(alignment_found)?"aligned":"UNALIGNED");
+
+		/* If all banks have aligned then the reset was successful */
+		for (bank = 0; bank < 10; bank++) {
+
+			int dut_aligned = 0;
+			int odin_aligned = 0;
+
+			odin_aligned = get_odin_sai_status(tc, bank);
+			dut_aligned = get_dut_sai_status(tc, bank);
+
+			if (dut_aligned == SAI_STATUS_ERROR)
+				return SAI_STATUS_ERROR;
+
+			if (!dut_aligned || !odin_aligned) {
+				aligned = false;
+				break;
+			}
+			aligned = true;
+		}
+
+		if (aligned) {
+			dev_info(&tc->pdev->dev,
+				"all banks have aligned\n");
+			break;
+		}
+
+		dev_warn(&tc->pdev->dev,
+			"Warning- not all banks have aligned. Trying again.\n");
+	}
+
+	if (!aligned)
+		dev_warn(&tc->pdev->dev, "odin_hard_reset failed\n");
+
+	return (aligned) ? 0 : 1; /* return 0 for success */
+}
+
+static void odin_set_mem_latency(struct tc_device *tc,
+				 int mem_latency, int mem_wresp_latency)
+{
+	u32 regval = 0;
+
+	if (mem_latency <= 4) {
+		/* The total memory read latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a read.
+		 * Set the memory read latency to 0 cycles.
+		 */
+		mem_latency = 0;
+	} else {
+		mem_latency -= 4;
+
+		dev_info(&tc->pdev->dev,
+			 "Setting memory read latency to %i cycles\n",
+			 mem_latency);
+	}
+
+	if (mem_wresp_latency <= 2) {
+		/* The total memory write latency cannot be lower than the
+		 * amount of cycles consumed by the hardware to do a write.
+		 * Set the memory write latency to 0 cycles.
+		 */
+		mem_wresp_latency = 0;
+	} else {
+		mem_wresp_latency -= 2;
+
+		dev_info(&tc->pdev->dev,
+			 "Setting memory write response latency to %i cycles\n",
+			 mem_wresp_latency);
+	}
+
+	mem_latency |= mem_wresp_latency << 16;
+
+	spi_write(tc, 0x1009, mem_latency);
+
+	if (spi_read(tc, 0x1009, &regval) != 0) {
+		dev_err(&tc->pdev->dev,
+			"Failed to read back memory latency register");
+		return;
+	}
+
+	if (mem_latency != regval) {
+		dev_err(&tc->pdev->dev,
+			"Memory latency register doesn't match requested value (actual: %#08x, expected: %#08x)\n",
+			regval, mem_latency);
+	}
+}
+
+static int orion_set_dut_core_clk(struct tc_device *tc,
+				  u32 input_clk,
+				  u32 output_clk)
+{
+	void __iomem *base = tc->tcf.registers;
+	void __iomem *clk_blk_base = base + SRS_REG_BANK_ODN_CLK_BLK;
+	struct device *dev = &tc->pdev->dev;
+	u32 high_time, low_time, edge, no_count;
+	u32 in_div, mul, out_div;
+	u32 value;
+	int err;
+
+	err = odin_mmcm_counter_calc(dev, input_clk, output_clk, &in_div,
+				     &mul, &out_div);
+	if (err != 0)
+		return err;
+
+	/* Put DUT into reset */
+	iowrite32(0, base + SRS_CORE_DUT_SOFT_RESETN);
+	msleep(20);
+
+	/* Put DUT Core MMCM into reset */
+	iowrite32(SRS_CLK_GEN_RESET_DUT_CORE_MMCM_MASK,
+		  base + SRS_CORE_CLK_GEN_RESET);
+	msleep(20);
+
+	/* Calculate the register fields for input divider */
+	odin_mmcm_reg_param_calc(in_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to input divider register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_IN_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_LO_TIME);
+	REG_FIELD_SET(value, edge,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_EDGE);
+	REG_FIELD_SET(value, no_count,
+			 ODN_DUT_CORE_CLK_IN_DIVIDER1_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_IN_DIVIDER1);
+
+	/* Calculate the register fields for multiplier */
+	odin_mmcm_reg_param_calc(mul, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to multiplier register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER1);
+	REG_FIELD_SET(value, high_time,
+			ODN_DUT_CORE_CLK_MULTIPLIER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			ODN_DUT_CORE_CLK_MULTIPLIER1_LO_TIME);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER1);
+
+	/* Read-modify-write the required fields to multiplier register 2 */
+	value = ioread32(clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER2);
+	REG_FIELD_SET(value, edge,
+			ODN_DUT_CORE_CLK_MULTIPLIER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			ODN_DUT_CORE_CLK_MULTIPLIER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_CORE_CLK_MULTIPLIER2);
+
+	/* Calculate the register fields for output divider */
+	odin_mmcm_reg_param_calc(out_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/*
+	 * Read-modify-write the required fields to
+	 * core output divider register 1
+	 */
+	value = ioread32(clk_blk_base + SRS_DUT_CORE_CLK_OUT_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME);
+	iowrite32(value, clk_blk_base + SRS_DUT_CORE_CLK_OUT_DIVIDER1);
+
+	/*
+	 * Read-modify-write the required fields to core output
+	 * divider register 2
+	 */
+	value = ioread32(clk_blk_base + SRS_DUT_CORE_CLK_OUT_DIVIDER2);
+	REG_FIELD_SET(value, edge,
+			SRS_DUT_CORE_CLK_OUT_DIVIDER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			SRS_DUT_CORE_CLK_OUT_DIVIDER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + SRS_DUT_CORE_CLK_OUT_DIVIDER2);
+
+	/*
+	 * Read-modify-write the required fields to
+	 * reference output divider register 1
+	 */
+	value = ioread32(clk_blk_base + SRS_DUT_REF_CLK_OUT_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			SRS_DUT_CORE_CLK_OUT_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			SRS_DUT_CORE_CLK_OUT_DIVIDER1_LO_TIME);
+	iowrite32(value, clk_blk_base + SRS_DUT_REF_CLK_OUT_DIVIDER1);
+
+	/*
+	 * Read-modify-write the required fields to
+	 * reference output divider register 2
+	 */
+	value = ioread32(clk_blk_base + SRS_DUT_REF_CLK_OUT_DIVIDER2);
+	REG_FIELD_SET(value, edge,
+			SRS_DUT_REF_CLK_OUT_DIVIDER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			SRS_DUT_REF_CLK_OUT_DIVIDER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + SRS_DUT_REF_CLK_OUT_DIVIDER2);
+
+	/* Bring DUT IF clock MMCM out of reset */
+	iowrite32(0, tc->tcf.registers + SRS_CORE_CLK_GEN_RESET);
+
+	err = tc_iopol32_nonzero(SRS_MMCM_LOCK_STATUS_DUT_CORE_MASK,
+				 base + SRS_CORE_MMCM_LOCK_STATUS);
+	if (err != 0) {
+		dev_err(dev, "MMCM failed to lock for DUT core\n");
+		return err;
+	}
+
+	/* Bring DUT out of reset */
+	iowrite32(SRS_DUT_SOFT_RESETN_EXTERNAL_MASK,
+		  tc->tcf.registers + SRS_CORE_DUT_SOFT_RESETN);
+	msleep(20);
+
+	dev_info(dev, "DUT core clock set-up successful\n");
+
+	return err;
+}
+
+static int orion_set_dut_sys_mem_clk(struct tc_device *tc,
+				     u32 input_clk,
+				     u32 output_clk)
+{
+	void __iomem *base = tc->tcf.registers;
+	void __iomem *clk_blk_base = base + SRS_REG_BANK_ODN_CLK_BLK;
+	struct device *dev = &tc->pdev->dev;
+	u32 high_time, low_time, edge, no_count;
+	u32 in_div, mul, out_div;
+	u32 value;
+	int err;
+
+	err = odin_mmcm_counter_calc(dev, input_clk, output_clk, &in_div,
+				     &mul, &out_div);
+	if (err != 0)
+		return err;
+
+	/* Put DUT into reset */
+	iowrite32(0, base + SRS_CORE_DUT_SOFT_RESETN);
+	msleep(20);
+
+	/* Put DUT Core MMCM into reset */
+	iowrite32(SRS_CLK_GEN_RESET_DUT_IF_MMCM_MASK,
+		  base + SRS_CORE_CLK_GEN_RESET);
+	msleep(20);
+
+	/* Calculate the register fields for input divider */
+	odin_mmcm_reg_param_calc(in_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to input divider register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_IN_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_LO_TIME);
+	REG_FIELD_SET(value, edge,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_EDGE);
+	REG_FIELD_SET(value, no_count,
+			 ODN_DUT_IFACE_CLK_IN_DIVIDER1_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_IN_DIVIDER1);
+
+	/* Calculate the register fields for multiplier */
+	odin_mmcm_reg_param_calc(mul, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to multiplier register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER1);
+	REG_FIELD_SET(value, high_time,
+			ODN_DUT_IFACE_CLK_MULTIPLIER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			ODN_DUT_IFACE_CLK_MULTIPLIER1_LO_TIME);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER1);
+
+	/* Read-modify-write the required fields to multiplier register 2 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER2);
+	REG_FIELD_SET(value, edge,
+			ODN_DUT_IFACE_CLK_MULTIPLIER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			ODN_DUT_IFACE_CLK_MULTIPLIER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_MULTIPLIER2);
+
+	/* Calculate the register fields for output divider */
+	odin_mmcm_reg_param_calc(out_div, &high_time, &low_time,
+				 &edge, &no_count);
+
+	/* Read-modify-write the required fields to output divider register 1 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER1_LO_TIME);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER1);
+
+	/* Read-modify-write the required fields to output divider register 2 */
+	value = ioread32(clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER2);
+	REG_FIELD_SET(value, edge,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			ODN_DUT_IFACE_CLK_OUT_DIVIDER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + ODN_DUT_IFACE_CLK_OUT_DIVIDER2);
+
+	/*
+	 * New to Orion, registers undocumented in the TRM, assumed high_time,
+	 * low_time, edge and no_count are in the same bit fields as the
+	 * previous two registers Even though these registers seem to be
+	 * undocumented, setting them is essential for the DUT not to show
+	 * abnormal behaviour, like the firmware jumping to random addresses
+	 */
+
+	/*
+	 * Read-modify-write the required fields to memory clock output divider
+	 * register 1
+	 */
+	value = ioread32(clk_blk_base + SRS_DUT_MEM_CLK_OUT_DIVIDER1);
+	REG_FIELD_SET(value, high_time,
+			SRS_DUT_MEM_CLK_OUT_DIVIDER1_HI_TIME);
+	REG_FIELD_SET(value, low_time,
+			SRS_DUT_MEM_CLK_OUT_DIVIDER1_LO_TIME);
+	iowrite32(value, clk_blk_base + SRS_DUT_MEM_CLK_OUT_DIVIDER1);
+
+	/*
+	 * Read-modify-write the required fields to memory clock output divider
+	 * register 1
+	 */
+	value = ioread32(clk_blk_base + SRS_DUT_MEM_CLK_OUT_DIVIDER2);
+	REG_FIELD_SET(value, edge,
+			SRS_DUT_MEM_CLK_OUT_DIVIDER2_EDGE);
+	REG_FIELD_SET(value, no_count,
+			SRS_DUT_MEM_CLK_OUT_DIVIDER2_NOCOUNT);
+	iowrite32(value, clk_blk_base + SRS_DUT_MEM_CLK_OUT_DIVIDER2);
+
+	/* Bring DUT clock MMCM out of reset */
+	iowrite32(0, tc->tcf.registers + SRS_CORE_CLK_GEN_RESET);
+
+	err = tc_iopol32_nonzero(SRS_MMCM_LOCK_STATUS_DUT_IF_MASK,
+				 base + SRS_CORE_MMCM_LOCK_STATUS);
+	if (err != 0) {
+		dev_err(dev, "MMCM failed to lock for DUT IF\n");
+		return err;
+	}
+
+	/* Bring DUT out of reset */
+	iowrite32(SRS_DUT_SOFT_RESETN_EXTERNAL_MASK,
+		  tc->tcf.registers + SRS_CORE_DUT_SOFT_RESETN);
+	msleep(20);
+
+	dev_info(dev, "DUT IF clock set-up successful\n");
+
+	return err;
+}
+
+
+static int orion_hard_reset(struct tc_device *tc, int *core_clock, int *mem_clock)
+{
+	int err;
+	struct device *dev = &tc->pdev->dev;
+
+	if (*core_clock == 0) {
+		*core_clock = RGX_TC_CORE_CLOCK_SPEED;
+		dev_info(dev, "Using default DUT core clock value: %i\n",
+				 *core_clock);
+	} else {
+		dev_info(dev, "Using module param DUT core clock value: %i\n",
+					*core_clock);
+	}
+
+	if (*mem_clock == 0) {
+		*mem_clock = RGX_TC_MEM_CLOCK_SPEED;
+		dev_info(dev, "Using default DUT mem clock value: %i\n",
+				 *mem_clock);
+	} else {
+		dev_info(dev, "Using module param DUT mem clock value: %i\n",
+					*mem_clock);
+	}
+
+	err = orion_set_dut_core_clk(tc, SRS_INPUT_CLOCK_SPEED, *core_clock);
+	if (err != 0)
+		goto err_out;
+
+	err = orion_set_dut_sys_mem_clk(tc, SRS_INPUT_CLOCK_SPEED, *mem_clock);
+
+err_out:
+	return err;
+}
+
+#endif /* defined(SUPPORT_RGX) */
+
+/* Do a hard reset on the DUT */
+static int odin_hard_reset(struct tc_device *tc, int *core_clock, int *mem_clock,
+							int *clock_mulitplex)
+{
+#if defined(SUPPORT_RGX)
+	if (tc->version == ODIN_VERSION_TCF_BONNIE)
+		return odin_hard_reset_bonnie(tc);
+	if (tc->version == ODIN_VERSION_VALI)
+		return odin_get_default_clocks_vali(tc, core_clock, mem_clock, clock_mulitplex);
+	if (tc->version == ODIN_VERSION_FPGA)
+		return odin_hard_reset_fpga(tc, core_clock, mem_clock, clock_mulitplex);
+	if (tc->version == ODIN_VERSION_ORION)
+		return orion_hard_reset(tc, core_clock, mem_clock);
+
+	dev_err(&tc->pdev->dev, "Invalid Odin version");
+	return 1;
+#else /* defined(SUPPORT_RGX) */
+	return 0;
+#endif /* defined(SUPPORT_RGX) */
+}
+
+static void odin_set_mem_mode_lma(struct tc_device *tc)
+{
+	u32 val;
+
+	if (tc->version != ODIN_VERSION_FPGA)
+		return;
+
+	/* Enable memory offset to be applied to DUT and PDPs */
+	iowrite32(0x80000A10, tc->tcf.registers + ODN_CORE_DUT_CTRL1);
+
+	/* Apply memory offset to GPU and PDPs to point to DDR memory.
+	 * Enable HDMI.
+	 */
+	val = (0x4 << ODN_CORE_CONTROL_DUT_OFFSET_SHIFT) |
+	      (0x4 << ODN_CORE_CONTROL_PDP1_OFFSET_SHIFT) |
+	      (0x4 << ODN_CORE_CONTROL_PDP2_OFFSET_SHIFT) |
+	      (0x2 << ODN_CORE_CONTROL_HDMI_MODULE_EN_SHIFT) |
+	      (0x1 << ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SHIFT);
+	iowrite32(val, tc->tcf.registers + ODN_CORE_CORE_CONTROL);
+}
+
+/*
+ * For Hybrid mode we don't want to set the above DUT/PDP offset shifts
+ * because those are for 32bit address limitations or to make address translation
+ * easier by placing the device physical address range to start at 0.
+ * Odin will get 35 bits which will allow it to address all host memory
+ * through the bus master and all device memory.
+ */
+static void odin_set_mem_mode_hybrid(struct tc_device *tc)
+{
+	u32 val;
+
+	/* Disable memory offset to be applied to DUT and PDPs */
+	iowrite32(0x00000A10, tc->tcf.registers + ODN_CORE_DUT_CTRL1);
+
+	/* Enable HDMI */
+	val = (0x2 << ODN_CORE_CONTROL_HDMI_MODULE_EN_SHIFT) |
+	      (0x1 << ODN_CORE_CONTROL_MCU_COMMUNICATOR_EN_SHIFT);
+	iowrite32(val, tc->tcf.registers + ODN_CORE_CORE_CONTROL);
+}
+
+static int odin_set_mem_mode(struct tc_device *tc, int mem_mode)
+{
+	switch (mem_mode) {
+	case TC_MEMORY_LOCAL:
+		odin_set_mem_mode_lma(tc);
+		dev_info(&tc->pdev->dev, "Memory mode: TC_MEMORY_LOCAL\n");
+		break;
+	case TC_MEMORY_HYBRID:
+		odin_set_mem_mode_hybrid(tc);
+		dev_info(&tc->pdev->dev, "Memory mode: TC_MEMORY_HYBRID\n");
+		break;
+	default:
+		dev_err(&tc->pdev->dev, "unsupported memory mode = %d\n",
+			mem_mode);
+		return -EINVAL;
+	};
+
+	tc->mem_mode = mem_mode;
+
+	return 0;
+}
+
+static u64 odin_get_pdp_dma_mask(struct tc_device *tc)
+{
+	/* Does not access system memory, so there is no DMA limitation */
+	if ((tc->mem_mode == TC_MEMORY_LOCAL) ||
+	    (tc->mem_mode == TC_MEMORY_HYBRID))
+		return DMA_BIT_MASK(64);
+
+	return DMA_BIT_MASK(32);
+}
+
+#if defined(SUPPORT_RGX)
+static u64 odin_get_rogue_dma_mask(struct tc_device *tc)
+{
+	/* Does not access system memory, so there is no DMA limitation */
+	if (tc->mem_mode == TC_MEMORY_LOCAL)
+		return DMA_BIT_MASK(64);
+
+	return DMA_BIT_MASK(32);
+}
+#endif /* defined(SUPPORT_RGX) */
+
+static void odin_set_fbc_bypass(struct tc_device *tc, bool fbc_bypass)
+{
+	u32 val;
+
+	/* Register field is present whether TC has PFIM support or not */
+	val = ioread32(tc->tcf.registers + ODN_CORE_DUT_CTRL1);
+	REG_FIELD_SET(val, fbc_bypass ? 0x1 : 0x0,
+		      ODN_DUT_CTRL1_FBDC_BYPASS);
+	iowrite32(val, tc->tcf.registers + ODN_CORE_DUT_CTRL1);
+
+	tc->fbc_bypass = fbc_bypass;
+}
+
+static int odin_hw_init(struct tc_device *tc, int *core_clock,
+			int *mem_clock, int *clock_mulitplex, int mem_latency,
+			int mem_wresp_latency, int mem_mode,
+			bool fbc_bypass)
+{
+	int err;
+
+	err = odin_hard_reset(tc, core_clock, mem_clock, clock_mulitplex);
+	if (err) {
+		dev_err(&tc->pdev->dev, "Failed to initialise Odin");
+		goto err_out;
+	}
+
+	err = odin_set_mem_mode(tc, mem_mode);
+	if (err)
+		goto err_out;
+
+	if (!tc->vali)
+		odin_set_fbc_bypass(tc, fbc_bypass);
+
+#if defined(SUPPORT_RGX)
+	if (tc->version == ODIN_VERSION_FPGA)
+		odin_set_mem_latency(tc, mem_latency, mem_wresp_latency);
+#endif /* defined(SUPPORT_RGX) */
+
+err_out:
+	return err;
+}
+
+static int odin_enable_irq(struct tc_device *tc)
+{
+	int err = 0;
+
+#if defined(TC_FAKE_INTERRUPTS)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	timer_setup(&tc->timer, tc_irq_fake_wrapper, 0);
+#else
+	setup_timer(&tc->timer, tc_irq_fake_wrapper, (unsigned long)tc);
+#endif
+	mod_timer(&tc->timer,
+		jiffies + msecs_to_jiffies(FAKE_INTERRUPT_TIME_MS));
+#else
+	iowrite32(0, tc->tcf.registers +
+		  common_reg_offset(tc, CORE_INTERRUPT_ENABLE));
+	iowrite32(0xffffffff, tc->tcf.registers +
+		  common_reg_offset(tc, CORE_INTERRUPT_CLR));
+
+	dev_info(&tc->pdev->dev,
+		"Registering IRQ %d for use by %s\n",
+		 tc->pdev->irq,
+		 odin_tc_name(tc));
+
+	err = request_irq(tc->pdev->irq, odin_irq_handler,
+		IRQF_SHARED, DRV_NAME, tc);
+
+	if (err) {
+		dev_err(&tc->pdev->dev,
+			"Error - IRQ %d failed to register\n",
+			tc->pdev->irq);
+	} else {
+		dev_info(&tc->pdev->dev,
+			"IRQ %d was successfully registered for use by %s\n",
+			 tc->pdev->irq,
+			 odin_tc_name(tc));
+	}
+#endif
+	return err;
+}
+
+static void odin_disable_irq(struct tc_device *tc)
+{
+#if defined(TC_FAKE_INTERRUPTS)
+	del_timer_sync(&tc->timer);
+#else
+	iowrite32(0, tc->tcf.registers +
+			common_reg_offset(tc, CORE_INTERRUPT_ENABLE));
+	iowrite32(0xffffffff, tc->tcf.registers +
+			common_reg_offset(tc, CORE_INTERRUPT_CLR));
+
+	free_irq(tc->pdev->irq, tc);
+#endif
+}
+
+static enum tc_version_t
+odin_detect_daughterboard_version(struct tc_device *tc)
+{
+	u32 reg = ioread32(tc->tcf.registers + ODN_REG_BANK_DB_TYPE_ID);
+	u32 val = reg;
+
+	if (tc->orion)
+		return ODIN_VERSION_ORION;
+
+	if (tc->vali)
+		return ODIN_VERSION_VALI;
+
+	val = (val & ODN_REG_BANK_DB_TYPE_ID_TYPE_MASK) >>
+		ODN_REG_BANK_DB_TYPE_ID_TYPE_SHIFT;
+
+	switch (val) {
+	default:
+		dev_err(&tc->pdev->dev,
+			"Unknown odin version ID type %#x (DB_TYPE_ID: %#08x)\n",
+			val, reg);
+		return TC_INVALID_VERSION;
+	case 1:
+		dev_info(&tc->pdev->dev, "DUT: Bonnie TC\n");
+		return ODIN_VERSION_TCF_BONNIE;
+	case 2:
+	case 3:
+		dev_info(&tc->pdev->dev, "DUT: FPGA\n");
+		return ODIN_VERSION_FPGA;
+	}
+}
+
+#if defined(SUPPORT_BAR_RESIZE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0))
+/* Should only be called with all BAR's resources released */
+static int try_resize_mem_bar(struct pci_dev *pdev, int target_bar_num)
+{
+	int err = 0;
+	u16 command;
+	u32 sizes, max_size, i;
+
+	pci_disable_device(pdev);
+
+	/* Disable memory decoding */
+	pci_read_config_word(pdev, PCI_COMMAND, &command);
+	command &= ~PCI_COMMAND_MEMORY;
+	pci_write_config_word(pdev, PCI_COMMAND, command);
+
+	sizes = pci_rebar_get_possible_sizes(pdev, target_bar_num);
+	if (sizes) {
+		max_size = fls(sizes);
+		err = pci_resize_resource(pdev, target_bar_num, max_size);
+		if (err) {
+			dev_err(&pdev->dev,
+				"error %u in pci_resize_resource (size requested 0x%x)\n",
+				err, max_size);
+			goto re_enable;
+		}
+		pci_assign_unassigned_bus_resources(pdev->bus);
+	} else {
+		dev_info(&pdev->dev, "PCI resize not supported on this platform\n");
+	}
+
+re_enable:
+	/* Re-enable memory decoding */
+	pci_read_config_word(pdev, PCI_COMMAND, &command);
+	command &= PCI_COMMAND_MEMORY;
+	pci_write_config_word(pdev, PCI_COMMAND, command);
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(&pdev->dev, "error %u in pci_enable_device\n", err);
+		goto err_out;
+	}
+
+	return dev_err;
+
+}
+#endif
+
+static int odin_dev_init(struct tc_device *tc, struct pci_dev *pdev,
+			 int pdp_mem_size, int secure_mem_size)
+{
+	int err;
+	u32 val;
+
+#if defined(SUPPORT_BAR_RESIZE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0))
+	err = try_resize_mem_bar(pdev, ODN_DDR_BAR);
+	if (err) {
+		dev_err(&pdev->dev,
+			"Failed to resize BAR %d", ODN_DDR_BAR);
+		goto err_out;
+	}
+#endif
+
+	/* Reserve and map the tcf system registers */
+	err = setup_io_region(pdev, &tc->tcf,
+		ODN_SYS_BAR, ODN_SYS_REGS_OFFSET, ODN_SYS_REGS_SIZE);
+	if (err)
+		goto err_out;
+
+	tc->version = odin_detect_daughterboard_version(tc);
+	if (tc->version == TC_INVALID_VERSION) {
+		err = -EIO;
+		goto err_odin_unmap_sys_registers;
+	}
+
+	/* Setup card memory */
+	tc->tc_mem.base = pci_resource_start(pdev, ODN_DDR_BAR);
+	tc->tc_mem.size = pci_resource_len(pdev, ODN_DDR_BAR);
+
+	if (tc->tc_mem.size < pdp_mem_size) {
+		dev_err(&pdev->dev,
+			"%s MEM region (bar %d) has size of %lu which is smaller than the requested PDP heap of %lu",
+			odin_tc_name(tc),
+			ODN_DDR_BAR,
+			(unsigned long)tc->tc_mem.size,
+			(unsigned long)pdp_mem_size);
+
+		err = -EIO;
+		goto err_odin_unmap_sys_registers;
+	}
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	if (tc->tc_mem.size <
+	    (pdp_mem_size + secure_mem_size)) {
+		dev_err(&pdev->dev,
+			"Odin MEM region (bar %d) has size of %lu which is smaller than the requested PDP heap of %lu"
+			" plus the requested secure heap size %lu",
+			ODN_DDR_BAR,
+			(unsigned long)tc->tc_mem.size,
+			(unsigned long)pdp_mem_size,
+			(unsigned long)secure_mem_size);
+		err = -EIO;
+		goto err_odin_unmap_sys_registers;
+	}
+#endif
+
+	err = tc_mtrr_setup(tc);
+	if (err)
+		goto err_odin_unmap_sys_registers;
+
+	/* Setup ranges for the device heaps */
+	tc->pdp_heap_mem_size = pdp_mem_size;
+
+	/* We know ext_heap_mem_size won't underflow as we've compared
+	 * tc_mem.size against the pdp_mem_size value earlier
+	 */
+	tc->ext_heap_mem_size =
+		tc->tc_mem.size - tc->pdp_heap_mem_size;
+
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	tc->ext_heap_mem_size -= secure_mem_size;
+#endif
+
+	if (tc->ext_heap_mem_size < TC_EXT_MINIMUM_MEM_SIZE) {
+		dev_warn(&pdev->dev,
+			 "%s MEM region (bar 4) has size of %lu, with %lu pdp_mem_size only %lu bytes are left for "
+			 "ext device, which looks too small",
+			 odin_tc_name(tc),
+			 (unsigned long)tc->tc_mem.size,
+			 (unsigned long)pdp_mem_size,
+			 (unsigned long)tc->ext_heap_mem_size);
+		/* Continue as this is only a 'helpful warning' not a hard
+		 * requirement
+		 */
+	}
+	tc->ext_heap_mem_base = tc->tc_mem.base;
+	tc->pdp_heap_mem_base =
+		tc->tc_mem.base + tc->ext_heap_mem_size;
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+	tc->secure_heap_mem_base = tc->pdp_heap_mem_base +
+		tc->pdp_heap_mem_size;
+	tc->secure_heap_mem_size = secure_mem_size;
+#endif
+
+#if defined(SUPPORT_DMA_HEAP)
+	err = tc_dmabuf_heap_init(tc, ODN_DDR_BAR);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialise ION\n");
+		goto err_odin_unmap_sys_registers;
+	}
+#elif defined(SUPPORT_ION)
+	err = tc_ion_init(tc, ODN_DDR_BAR);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialise ION\n");
+		goto err_odin_unmap_sys_registers;
+	}
+#endif /* defined(SUPPORT_ION) */
+
+	/* CDMA initialisation */
+	val = ioread32(tc->tcf.registers + ODN_CORE_SUPPORTED_FEATURES);
+	tc->dma_nchan = REG_FIELD_GET(val,
+				       ODN_SUPPORTED_FEATURES_2X_CDMA_AND_IRQS);
+	tc->dma_nchan++;
+	dev_info(&tc->pdev->dev, "Odin RTL has %u DMA(s)\n", tc->dma_nchan);
+	mutex_init(&tc->dma_mutex);
+
+	if (tc->odin) {
+		val = ioread32(tc->tcf.registers +
+		       ODN_CORE_REL);
+		dev_info(&pdev->dev, "%s = 0x%08x\n",
+			"ODN_CORE_REL", val);
+	} else if (tc->vali) {
+		val = ioread32(tc->tcf.registers + ODN_CORE_ID);
+		dev_info(&pdev->dev, "%s = 0x%08x\n", "VALI_CORE_ID", val);
+		val = ioread32(tc->tcf.registers + ODN_CORE_REL);
+		dev_info(&pdev->dev, "%s = 0x%08x\n", "VALI_CORE_REL", val);
+	} else {
+		val = ioread32(tc->tcf.registers +
+		       SRS_CORE_REVISION);
+		dev_info(&pdev->dev, "%s = 0x%08x\n",
+			"SRS_CORE_REVISION", val);
+	}
+
+	val = ioread32(tc->tcf.registers +
+		       common_reg_offset(tc, CORE_CHANGE_SET));
+	dev_info(&pdev->dev, "%s = 0x%08x\n",
+		 common_reg_name(tc, CORE_CHANGE_SET), val);
+
+	val = ioread32(tc->tcf.registers +
+		       common_reg_offset(tc, CORE_USER_ID));
+	dev_info(&pdev->dev, "%s = 0x%08x\n",
+		 common_reg_name(tc, CORE_USER_ID), val);
+
+	val = ioread32(tc->tcf.registers +
+		       common_reg_offset(tc, CORE_USER_BUILD));
+	dev_info(&pdev->dev, "%s = 0x%08x\n",
+		 common_reg_name(tc, CORE_USER_BUILD), val);
+
+err_out:
+	return err;
+
+err_odin_unmap_sys_registers:
+	dev_info(&pdev->dev,
+		 "%s: failed - unmapping the io regions.\n", __func__);
+
+	iounmap(tc->tcf.registers);
+	release_pci_io_addr(pdev, ODN_SYS_BAR,
+			 tc->tcf.region.base, tc->tcf.region.size);
+	goto err_out;
+}
+
+static void odin_dev_cleanup(struct tc_device *tc)
+{
+#if defined(SUPPORT_DMA_HEAP)
+	tc_dmabuf_heap_deinit(tc, ODN_DDR_BAR);
+#elif defined(SUPPORT_ION)
+	tc_ion_deinit(tc, ODN_DDR_BAR);
+#endif
+
+	tc_mtrr_cleanup(tc);
+
+	iounmap(tc->tcf.registers);
+
+	release_pci_io_addr(tc->pdev,
+			ODN_SYS_BAR,
+			tc->tcf.region.base,
+			tc->tcf.region.size);
+}
+
+static u32 odin_interrupt_id_to_flag(int interrupt_id)
+{
+	switch (interrupt_id) {
+	case TC_INTERRUPT_PDP:
+		return ODN_INTERRUPT_ENABLE_PDP1;
+	case TC_INTERRUPT_EXT:
+		return ODN_INTERRUPT_ENABLE_DUT;
+	case TC_INTERRUPT_PDP2:
+		return ODN_INTERRUPT_ENABLE_PDP2;
+	case TC_INTERRUPT_CDMA:
+		return ODN_INTERRUPT_ENABLE_CDMA;
+	case TC_INTERRUPT_CDMA2:
+		return ODN_INTERRUPT_ENABLE_CDMA2;
+	case TC_INTERRUPT_OSID0:
+		return ODN_INTERRUPT_ENABLE_OSID(0);
+	case TC_INTERRUPT_OSID1:
+		return ODN_INTERRUPT_ENABLE_OSID(1);
+	case TC_INTERRUPT_OSID2:
+		return ODN_INTERRUPT_ENABLE_OSID(2);
+	case TC_INTERRUPT_OSID3:
+		return ODN_INTERRUPT_ENABLE_OSID(3);
+	case TC_INTERRUPT_OSID4:
+		return ODN_INTERRUPT_ENABLE_OSID(4);
+	case TC_INTERRUPT_OSID5:
+		return ODN_INTERRUPT_ENABLE_OSID(5);
+	case TC_INTERRUPT_OSID6:
+		return ODN_INTERRUPT_ENABLE_OSID(6);
+	case TC_INTERRUPT_OSID7:
+		return ODN_INTERRUPT_ENABLE_OSID(7);
+	default:
+		BUG();
+	}
+}
+
+int odin_init(struct tc_device *tc, struct pci_dev *pdev,
+	      int *core_clock, int *mem_clock, int *clock_mulitplex,
+	      int pdp_mem_size, int secure_mem_size,
+	      int mem_latency, int mem_wresp_latency, int mem_mode,
+	      bool fbc_bypass)
+{
+	int err = 0;
+
+	err = odin_dev_init(tc, pdev, pdp_mem_size, secure_mem_size);
+	if (err) {
+		dev_err(&pdev->dev, "odin_dev_init failed\n");
+		goto err_out;
+	}
+
+	err = odin_hw_init(tc, core_clock, mem_clock, clock_mulitplex,
+			   mem_latency, mem_wresp_latency, mem_mode,
+			   fbc_bypass);
+	if (err) {
+		dev_err(&pdev->dev, "odin_hw_init failed\n");
+		goto err_dev_cleanup;
+	}
+
+	err = odin_enable_irq(tc);
+	if (err) {
+		dev_err(&pdev->dev,
+			"Failed to initialise IRQ\n");
+		goto err_dev_cleanup;
+	}
+
+err_out:
+	return err;
+
+err_dev_cleanup:
+	odin_dev_cleanup(tc);
+	goto err_out;
+}
+
+int odin_cleanup(struct tc_device *tc)
+{
+	/*
+	 * Make sure we don't attempt to clean-up after an invalid device.
+	 * We'll have already unmapped the PCI i/o space so cannot access
+	 * anything now.
+	 */
+	if (tc->version != TC_INVALID_VERSION) {
+		odin_disable_irq(tc);
+		odin_dev_cleanup(tc);
+	}
+
+	return 0;
+}
+
+int odin_register_pdp_device(struct tc_device *tc)
+{
+	int err = 0;
+	resource_size_t reg_start = pci_resource_start(tc->pdev, ODN_SYS_BAR);
+	struct resource pdp_resources_odin[] = {
+		DEFINE_RES_MEM_NAMED(reg_start +
+				ODN_PDP_REGS_OFFSET, /* start */
+				ODN_PDP_REGS_SIZE, /* size */
+				"pdp-regs"),
+		DEFINE_RES_MEM_NAMED(reg_start +
+				ODN_PDP2_REGS_OFFSET, /* start */
+				ODN_PDP2_REGS_SIZE, /* size */
+				"pdp2-regs"),
+		DEFINE_RES_MEM_NAMED(reg_start +
+				ODN_SYS_REGS_OFFSET +
+				common_reg_offset(tc, REG_BANK_ODN_CLK_BLK) +
+				ODN_PDP_P_CLK_OUT_DIVIDER_REG1, /* start */
+				ODN_PDP_P_CLK_IN_DIVIDER_REG -
+				ODN_PDP_P_CLK_OUT_DIVIDER_REG1 + 4, /* size */
+				"pll-regs"),
+		DEFINE_RES_MEM_NAMED(reg_start +
+				ODN_PDP2_PFIM_OFFSET, /* start */
+				ODN_PDP2_PFIM_SIZE, /* size */
+				"pfim-regs"),
+		DEFINE_RES_MEM_NAMED(reg_start +
+				ODN_SYS_REGS_OFFSET +
+				ODN_REG_BANK_CORE, /* start */
+				ODN_CORE_MMCM_LOCK_STATUS + 4, /* size */
+				"odn-core"),
+	};
+
+	struct tc_pdp_platform_data pdata = {
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+		.ion_device = tc->ion_device,
+		.ion_heap_id = ION_HEAP_TC_PDP,
+#endif
+		.memory_base = tc->tc_mem.base,
+		.pdp_heap_memory_base = tc->pdp_heap_mem_base,
+		.pdp_heap_memory_size = tc->pdp_heap_mem_size,
+	};
+	struct platform_device_info pdp_device_info = {
+		.parent = &tc->pdev->dev,
+		.name = ODN_DEVICE_NAME_PDP,
+		.id = -2,
+		.data = &pdata,
+		.size_data = sizeof(pdata),
+		.dma_mask = odin_get_pdp_dma_mask(tc),
+	};
+
+	pdp_device_info.res = pdp_resources_odin;
+	pdp_device_info.num_res = ARRAY_SIZE(pdp_resources_odin);
+
+	tc->pdp_dev = platform_device_register_full(&pdp_device_info);
+	if (IS_ERR(tc->pdp_dev)) {
+		err = PTR_ERR(tc->pdp_dev);
+		dev_err(&tc->pdev->dev,
+			"Failed to register PDP device (%d)\n", err);
+		tc->pdp_dev = NULL;
+		goto err_out;
+	}
+
+err_out:
+	return err;
+}
+
+int odin_register_ext_device(struct tc_device *tc)
+{
+#if defined(SUPPORT_RGX)
+	int err = 0;
+	int osid;
+
+	unsigned long CarveoutSize = tc->tc_mem.size/RGX_NUM_DRIVERS_SUPPORTED;
+	unsigned long EXTHeapSize  = tc->ext_heap_mem_size/RGX_NUM_DRIVERS_SUPPORTED;
+	unsigned long PDPHeapSize  = tc->pdp_heap_mem_size/RGX_NUM_DRIVERS_SUPPORTED;
+
+	for (osid=0; osid < RGX_NUM_DRIVERS_SUPPORTED; osid++)
+	{
+		unsigned long EXTHeapBase = tc->tc_mem.base + osid*CarveoutSize;
+		unsigned long PDPHeapBase = EXTHeapBase + EXTHeapSize;
+
+		struct resource odin_rogue_resources[] = {
+			DEFINE_RES_MEM_NAMED(pci_resource_start(tc->pdev, ODN_DUT_SOCIF_BAR) +
+								 osid*ODN_DUT_SOCIF_SIZE,
+								 ODN_DUT_SOCIF_SIZE,
+								 "rogue-regs"),
+		};
+
+		struct tc_rogue_platform_data pdata = {
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+			.ion_device = tc->ion_device,
+			.ion_heap_id = ION_HEAP_TC_ROGUE,
+#endif
+			.mem_mode = tc->mem_mode,
+			.tc_memory_base = tc->tc_mem.base,
+			.pdp_heap_memory_base = PDPHeapBase,
+			.pdp_heap_memory_size = PDPHeapSize,
+			.rogue_heap_memory_base = EXTHeapBase,
+			.rogue_heap_memory_size = EXTHeapSize,
+#if defined(SUPPORT_FAKE_SECURE_ION_HEAP)
+			.secure_heap_memory_base = tc->secure_heap_mem_base,
+			.secure_heap_memory_size = tc->secure_heap_mem_size,
+#endif
+			.tc_dma_tx_chan_name = ODIN_DMA_TX_CHAN_NAME,
+			.tc_dma_rx_chan_name = ODIN_DMA_RX_CHAN_NAME,
+		};
+
+		struct platform_device_info odin_rogue_dev_info = {
+			.parent = &tc->pdev->dev,
+			.name = TC_DEVICE_NAME_ROGUE,
+			.id = -2,
+			.res = odin_rogue_resources,
+			.num_res = ARRAY_SIZE(odin_rogue_resources),
+			.data = &pdata,
+			.size_data = sizeof(pdata),
+			.dma_mask = odin_get_rogue_dma_mask(tc),
+		};
+
+		if (tc->odin)
+			pdata.baseboard = TC_BASEBOARD_ODIN;
+		else if (tc->orion)
+			pdata.baseboard = TC_BASEBOARD_ORION;
+		else if (tc->vali)
+			pdata.baseboard = TC_BASEBOARD_VALI;
+
+		tc->ext_dev[osid]
+			= platform_device_register_full(&odin_rogue_dev_info);
+
+		if (IS_ERR(tc->ext_dev[osid])) {
+			err = PTR_ERR(tc->ext_dev[osid]);
+			dev_err(&tc->pdev->dev,
+				"Failed to register rogue device[%u] (%d)\n", osid, err);
+			tc->ext_dev[osid] = NULL;
+		}
+	}
+	return err;
+#else /* defined(SUPPORT_RGX) */
+	return 0;
+#endif /* defined(SUPPORT_RGX) */
+}
+
+int odin_register_dma_device(struct tc_device *tc)
+{
+	resource_size_t reg_start = pci_resource_start(tc->pdev, ODN_SYS_BAR);
+	int err = 0;
+
+	struct resource odin_cdma_resources[] = {
+		DEFINE_RES_MEM_NAMED(reg_start +
+				     ODIN_DMA_REGS_OFFSET,     /* start */
+				     ODIN_DMA_REGS_SIZE,       /* size */
+				     "cdma-regs"),
+		DEFINE_RES_IRQ_NAMED(TC_INTERRUPT_CDMA,
+				     "cdma-irq"),
+		DEFINE_RES_IRQ_NAMED(TC_INTERRUPT_CDMA2,
+				     "cdma-irq2"),
+	};
+
+	struct tc_dma_platform_data pdata = {
+		.addr_width = ODN_CDMA_ADDR_WIDTH,
+		.num_dmas = tc->dma_nchan,
+		.has_dre = true,
+		.has_sg = true,
+	};
+
+	struct platform_device_info odin_cdma_dev_info = {
+		.parent = &tc->pdev->dev,
+		.name = ODN_DEVICE_NAME_CDMA,
+		.id = -2,
+		.res = odin_cdma_resources,
+		.num_res = ARRAY_SIZE(odin_cdma_resources),
+		.dma_mask = DMA_BIT_MASK(ODN_CDMA_ADDR_WIDTH),
+		.data = &pdata,
+		.size_data = sizeof(pdata),
+	};
+
+	tc->dma_dev
+		= platform_device_register_full(&odin_cdma_dev_info);
+
+	if (IS_ERR(tc->dma_dev)) {
+		err = PTR_ERR(tc->dma_dev);
+		dev_err(&tc->pdev->dev,
+			"Failed to register CDMA device (%d)\n", err);
+		tc->dma_dev = NULL;
+	}
+
+	return err;
+}
+
+void odin_enable_interrupt_register(struct tc_device *tc,
+				    int interrupt_id)
+{
+	u32 val;
+	u32 flag;
+
+	switch (interrupt_id) {
+	case TC_INTERRUPT_PDP:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin PDP interrupts\n");
+		break;
+	case TC_INTERRUPT_EXT:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin DUT interrupts\n");
+		break;
+	case TC_INTERRUPT_PDP2:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin PDP2 interrupts\n");
+		break;
+	case TC_INTERRUPT_CDMA:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin CDMA interrupts\n");
+		break;
+	case TC_INTERRUPT_CDMA2:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin CDMA2 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID0:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID0 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID1:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID1 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID2:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID2 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID3:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID3 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID4:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID4 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID5:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID5 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID6:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID6 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID7:
+		dev_info(&tc->pdev->dev,
+			"Enabling Odin OSID7 interrupts\n");
+		break;
+	default:
+		dev_err(&tc->pdev->dev,
+			"Error - illegal interrupt id\n");
+		return;
+	}
+
+	val = ioread32(tc->tcf.registers +
+		       common_reg_offset(tc, CORE_INTERRUPT_ENABLE));
+	flag = odin_interrupt_id_to_flag(interrupt_id);
+	val |= flag;
+	iowrite32(val, tc->tcf.registers +
+		  common_reg_offset(tc, CORE_INTERRUPT_ENABLE));
+}
+
+void odin_disable_interrupt_register(struct tc_device *tc,
+				     int interrupt_id)
+{
+	u32 val;
+
+	switch (interrupt_id) {
+	case TC_INTERRUPT_PDP:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin PDP interrupts\n");
+		break;
+	case TC_INTERRUPT_EXT:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin DUT interrupts\n");
+		break;
+	case TC_INTERRUPT_PDP2:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin PDP2 interrupts\n");
+		break;
+	case TC_INTERRUPT_CDMA:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin CDMA interrupts\n");
+		break;
+	case TC_INTERRUPT_CDMA2:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin CDMA2 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID0:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID0 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID1:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID1 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID2:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID2 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID3:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID3 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID4:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID4 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID5:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID5 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID6:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID6 interrupts\n");
+		break;
+	case TC_INTERRUPT_OSID7:
+		dev_info(&tc->pdev->dev,
+			"Disabling Odin OSID7 interrupts\n");
+		break;
+	default:
+		dev_err(&tc->pdev->dev,
+			"Error - illegal interrupt id\n");
+		return;
+	}
+	val = ioread32(tc->tcf.registers +
+		       common_reg_offset(tc, CORE_INTERRUPT_ENABLE));
+	val &= ~(odin_interrupt_id_to_flag(interrupt_id));
+	iowrite32(val, tc->tcf.registers +
+		  common_reg_offset(tc, CORE_INTERRUPT_ENABLE));
+}
+
+irqreturn_t odin_irq_handler(int irq, void *data)
+{
+	u32 interrupt_status;
+	u32 interrupt_clear = 0;
+	unsigned long flags;
+	irqreturn_t ret = IRQ_NONE;
+	struct tc_device *tc = (struct tc_device *)data;
+
+	spin_lock_irqsave(&tc->interrupt_handler_lock, flags);
+
+#if defined(TC_FAKE_INTERRUPTS)
+	/* If we're faking interrupts pretend we got both ext and PDP ints */
+	interrupt_status = ODN_INTERRUPT_STATUS_DUT
+		| ODN_INTERRUPT_STATUS_PDP1;
+#else
+	interrupt_status = ioread32(tc->tcf.registers +
+				    common_reg_offset(tc,
+						      CORE_INTERRUPT_STATUS));
+#endif
+
+	if (interrupt_status & ODN_INTERRUPT_STATUS_DUT) {
+		struct tc_interrupt_handler *ext_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_EXT];
+
+		if (ext_int->enabled && ext_int->handler_function) {
+			ext_int->handler_function(ext_int->handler_data);
+			interrupt_clear |= ODN_INTERRUPT_CLEAR_DUT;
+		}
+		ret = IRQ_HANDLED;
+	}
+	if (interrupt_status & ODN_INTERRUPT_STATUS_PDP1) {
+		struct tc_interrupt_handler *pdp_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_PDP];
+
+		if (pdp_int->enabled && pdp_int->handler_function) {
+			pdp_int->handler_function(pdp_int->handler_data);
+			interrupt_clear |= ODN_INTERRUPT_CLEAR_PDP1;
+		}
+		ret = IRQ_HANDLED;
+	}
+	if (interrupt_status & ODN_INTERRUPT_STATUS_PDP2) {
+		struct tc_interrupt_handler *pdp_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_PDP2];
+
+		if (pdp_int->enabled && pdp_int->handler_function) {
+			pdp_int->handler_function(pdp_int->handler_data);
+			interrupt_clear |= ODN_INTERRUPT_CLEAR_PDP2;
+		}
+		ret = IRQ_HANDLED;
+	}
+
+	if (interrupt_status & ODN_INTERRUPT_STATUS_CDMA) {
+		struct tc_interrupt_handler *cdma_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_CDMA];
+		if (cdma_int->enabled && cdma_int->handler_function) {
+			cdma_int->handler_function(cdma_int->handler_data);
+			interrupt_clear |= ODN_INTERRUPT_CLEAR_CDMA;
+		}
+		ret = IRQ_HANDLED;
+	}
+
+	if (interrupt_status & ODN_INTERRUPT_STATUS_CDMA2) {
+		struct tc_interrupt_handler *cdma_int =
+			&tc->interrupt_handlers[TC_INTERRUPT_CDMA2];
+		if (cdma_int->enabled && cdma_int->handler_function) {
+			cdma_int->handler_function(cdma_int->handler_data);
+			interrupt_clear |= ODN_INTERRUPT_CLEAR_CDMA2;
+		}
+		ret = IRQ_HANDLED;
+	}
+
+	if (interrupt_status & ODN_INTERRUPT_STATUS_OS_IRQ_MASK) {
+		unsigned osid;
+
+		for (osid=0; osid < RGX_NUM_DRIVERS_SUPPORTED; osid++)
+		{
+			struct tc_interrupt_handler *ext_int =
+				&tc->interrupt_handlers[TC_INTERRUPT_OSID0 + osid];
+
+			if (ext_int->enabled && ext_int->handler_function) {
+				ext_int->handler_function(ext_int->handler_data);
+				interrupt_clear |= ODN_INTERRUPT_CLEAR_OSID(osid);
+			}
+		}
+		ret = IRQ_HANDLED;
+	}
+
+	if (interrupt_clear)
+		iowrite32(interrupt_clear,
+			  tc->tcf.registers +
+			  common_reg_offset(tc, CORE_INTERRUPT_CLR));
+
+	/*
+	 * Orion PDP interrupts are occasionally masked because, for unknown
+	 * reasons, a vblank goes without being asserted for about 1000 ms. This
+	 * feature is not present on Odin, and setting the
+	 * INTERRUPT_TIMEOUT_THRESHOLD register to 0 does not seem to disable it
+	 * either. This is probably caused by a bug in some versions of Sirius
+	 * RTL. Also this bug seems to only affect PDP interrupts, but not the
+	 * DUT. This might sometimes lead to a sudden jitter effect in the
+	 * render. Further investigation is pending before this code can
+	 * be safely removed.
+	 */
+
+	if (tc->orion) {
+		if (REG_FIELD_GET(ioread32(tc->tcf.registers +
+					   SRS_CORE_INTERRUPT_TIMEOUT_CLR),
+				  SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT)) {
+			dev_warn(&tc->pdev->dev,
+				 "Orion PDP interrupts were masked, clearing now\n");
+			iowrite32(SRS_INTERRUPT_TIMEOUT_CLR_INTERRUPT_MST_TIMEOUT_CLR_MASK,
+				  tc->tcf.registers + SRS_CORE_INTERRUPT_TIMEOUT_CLR);
+		}
+	}
+
+	spin_unlock_irqrestore(&tc->interrupt_handler_lock, flags);
+
+	return ret;
+}
+
+int odin_sys_info(struct tc_device *tc, u32 *tmp, u32 *pll)
+{
+	*tmp = 0;
+	*pll = 0;
+	return 0;
+}
+
+int odin_sys_strings(struct tc_device *tc,
+		     char *str_fpga_rev, size_t size_fpga_rev,
+		     char *str_tcf_core_rev, size_t size_tcf_core_rev,
+		     char *str_tcf_core_target_build_id,
+		     size_t size_tcf_core_target_build_id,
+		     char *str_pci_ver, size_t size_pci_ver,
+		     char *str_macro_ver, size_t size_macro_ver)
+{
+	u32 tcver = tc_odin_subvers(&tc->pdev->dev);
+	char temp_str[12];
+	u32 val;
+
+	if (tc->odin) {
+		/* Read the Odin major and minor revision ID register Rx-xx */
+		val = ioread32(tc->tcf.registers +
+				   ODN_CORE_REL);
+
+		snprintf(str_tcf_core_rev,
+			 size_tcf_core_rev,
+			 "%d.%d",
+			 HEX2DEC((val & ODN_REL_MAJOR_MASK)
+				 >> ODN_REL_MAJOR_SHIFT),
+			 HEX2DEC((val & ODN_REL_MINOR_MASK)
+				 >> ODN_REL_MINOR_SHIFT));
+	} else {
+		/* Read the Orion major and minor revision ID register Rx-xx */
+		val = ioread32(tc->tcf.registers +
+				   SRS_CORE_REVISION);
+
+		snprintf(str_tcf_core_rev,
+			 size_tcf_core_rev,
+			 "%d.%d",
+			 HEX2DEC((val & SRS_REVISION_MAJOR_MASK)
+				 >> SRS_REVISION_MAJOR_SHIFT),
+			 HEX2DEC((val & SRS_REVISION_MINOR_MASK)
+				 >> SRS_REVISION_MINOR_SHIFT));
+	}
+
+	dev_info(&tc->pdev->dev, "%s core revision %s\n",
+		 odin_tc_name(tc), str_tcf_core_rev);
+
+	/* Read the Odin register containing the Perforce changelist
+	 * value that the FPGA build was generated from
+	 */
+	val = ioread32(tc->tcf.registers +
+		       common_reg_offset(tc, CORE_CHANGE_SET));
+
+	snprintf(str_tcf_core_target_build_id,
+		 size_tcf_core_target_build_id,
+		 "%d",
+		 (val & CHANGE_SET_SET_MASK[tcver])
+		 >> CHANGE_SET_SET_SHIFT[tcver]);
+
+	/* Read the Odin User_ID register containing the User ID for
+	 * identification of a modified build
+	 */
+	val = ioread32(tc->tcf.registers + common_reg_offset(tc, CORE_USER_ID));
+
+	snprintf(temp_str,
+		 sizeof(temp_str),
+		 "%d",
+		 HEX2DEC((val & USER_ID_ID_MASK[tcver])
+			 >> USER_ID_ID_SHIFT[tcver]));
+
+	/* Read the Odin User_Build register containing the User build
+	 * number for identification of modified builds
+	 */
+	val = ioread32(tc->tcf.registers +
+		       common_reg_offset(tc, CORE_USER_BUILD));
+
+	snprintf(temp_str,
+		 sizeof(temp_str),
+		 "%d",
+		 HEX2DEC((val & USER_BUILD_BUILD_MASK[tcver])
+			 >> USER_BUILD_BUILD_SHIFT[tcver]));
+
+	return 0;
+}
+
+const char *odin_tc_name(struct tc_device *tc)
+{
+	if (tc->odin)
+		return "Odin";
+	else if (tc->orion)
+		return "Orion";
+	else if (tc->vali)
+		return "Vali";
+	else
+		return "Unknown TC";
+}
+
+bool odin_pfim_compatible(struct tc_device *tc)
+{
+	u32 val;
+
+	val = ioread32(tc->tcf.registers +
+		       ODN_CORE_REL);
+
+	return ((REG_FIELD_GET(val, ODN_REL_MAJOR)
+		 >= ODIN_PFIM_RELNUM));
+}
+
+
+struct dma_chan *odin_cdma_chan(struct tc_device *tc, char *name)
+{
+	struct dma_chan *chan;
+	unsigned long chan_idx;
+	int err;
+
+	if (!(strcmp("rx", name)))
+		chan_idx = ODN_DMA_CHAN_RX;
+	else if (!(strcmp("tx", name))) {
+		/*
+		 * When Odin RTL has a single CDMA device, we simulate
+		 * a second channel by always opening the first one.
+		 * This is made possible because CDMA allows for
+		 * transfers in both directions
+		 */
+		if (tc->dma_nchan == 1) {
+			name = "rx";
+			chan_idx = ODN_DMA_CHAN_RX;
+		} else
+			chan_idx = ODN_DMA_CHAN_TX;
+	} else {
+		dev_err(&tc->pdev->dev, "Wrong CDMA channel name\n");
+		return NULL;
+	}
+
+	mutex_lock(&tc->dma_mutex);
+
+	if (tc->dma_refcnt[chan_idx]) {
+		tc->dma_refcnt[chan_idx]++;
+	} else {
+		chan = dma_request_chan(&tc->dma_dev->dev, name);
+		if (IS_ERR(chan)) {
+			err = PTR_ERR(chan);
+			dev_err(&tc->pdev->dev,
+				"dma channel request failed (%d)\n", err);
+			mutex_unlock(&tc->dma_mutex);
+			return NULL;
+		}
+		tc->dma_chans[chan_idx] = chan;
+		tc->dma_refcnt[chan_idx] = 1;
+	}
+
+	mutex_unlock(&tc->dma_mutex);
+
+	return tc->dma_chans[chan_idx];
+}
+
+void odin_cdma_chan_free(struct tc_device *tc,
+			 void *chan_priv)
+{
+	struct dma_chan *dma_chan = (struct dma_chan *)chan_priv;
+	u32 chan_idx;
+
+	BUG_ON(dma_chan == NULL);
+
+	mutex_lock(&tc->dma_mutex);
+
+	if (dma_chan == tc->dma_chans[ODN_DMA_CHAN_RX])
+		chan_idx = ODN_DMA_CHAN_RX;
+	else if (dma_chan == tc->dma_chans[ODN_DMA_CHAN_TX])
+		chan_idx = ODN_DMA_CHAN_TX;
+	else
+		goto cdma_chan_free_exit;
+
+	tc->dma_refcnt[chan_idx]--;
+	if (!tc->dma_refcnt[chan_idx]) {
+		tc->dma_chans[chan_idx] = NULL;
+		dma_release_channel(dma_chan);
+	}
+
+cdma_chan_free_exit:
+	mutex_unlock(&tc->dma_mutex);
+}
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_odin.h b/drivers/gpu/drm/img-rogue/apollo/tc_odin.h
new file mode 100644
index 000000000000..d2be3cde97a4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_odin.h
@@ -0,0 +1,82 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _ODIN_DRV_H
+#define _ODIN_DRV_H
+
+#include "tc_drv_internal.h"
+#include "odin_defs.h"
+#include "orion_defs.h"
+
+int odin_init(struct tc_device *tc, struct pci_dev *pdev,
+	      int *core_clock, int *mem_clock, int *clock_multiplex,
+	      int pdp_mem_size, int secure_mem_size,
+	      int mem_latency, int mem_wresp_latency, int mem_mode,
+	      bool fbc_bypass);
+int odin_cleanup(struct tc_device *tc);
+
+int odin_register_pdp_device(struct tc_device *tc);
+int odin_register_ext_device(struct tc_device *tc);
+int odin_register_dma_device(struct tc_device *tc);
+
+struct dma_chan *odin_cdma_chan(struct tc_device *tc, char *name);
+void odin_cdma_chan_free(struct tc_device *tc, void *chan_priv);
+
+void odin_enable_interrupt_register(struct tc_device *tc,
+				    int interrupt_id);
+void odin_disable_interrupt_register(struct tc_device *tc,
+				     int interrupt_id);
+
+irqreturn_t odin_irq_handler(int irq, void *data);
+
+int odin_sys_info(struct tc_device *tc, u32 *tmp, u32 *pll);
+int odin_sys_strings(struct tc_device *tc,
+		     char *str_fpga_rev, size_t size_fpga_rev,
+		     char *str_tcf_core_rev, size_t size_tcf_core_rev,
+		     char *str_tcf_core_target_build_id,
+		     size_t size_tcf_core_target_build_id,
+		     char *str_pci_ver, size_t size_pci_ver,
+		     char *str_macro_ver, size_t size_macro_ver);
+
+const char *odin_tc_name(struct tc_device *tc);
+
+bool odin_pfim_compatible(struct tc_device *tc);
+#endif /* _ODIN_DRV_H */
diff --git a/drivers/gpu/drm/img-rogue/apollo/tc_odin_common_regs.h b/drivers/gpu/drm/img-rogue/apollo/tc_odin_common_regs.h
new file mode 100644
index 000000000000..0914d08cbd49
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tc_odin_common_regs.h
@@ -0,0 +1,114 @@
+/*
+ * @File        odin_common_regs.h
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __TC_ODIN_COMMON_REGS_H__
+#define __TC_ODIN_COMMON_REGS_H__
+
+#include <linux/types.h>
+#include <linux/stringify.h>
+
+struct tc_device;
+
+enum odin_common_regs {
+	CORE_CHANGE_SET = 0,
+	CORE_USER_ID,
+	CORE_USER_BUILD,
+	CORE_INTERRUPT_ENABLE,
+	CORE_INTERRUPT_CLR,
+	CORE_INTERRUPT_STATUS,
+	REG_BANK_ODN_CLK_BLK,
+};
+
+#define ODIN_REGNAME(REG_NAME) "ODN_" __stringify(REG_NAME)
+#define VALI_REGNAME(REG_NAME) "VALI_" __stringify(REG_NAME)
+#define ORION_REGNAME(REG_NAME) "SRS_" __stringify(REG_NAME)
+
+struct odin_orion_reg {
+	u32 odin_offset;
+	u32 vali_offset;
+	u32 orion_offset;
+	const char *odin_name;
+	const char *vali_name;
+	const char *orion_name;
+};
+
+#define COMMON_REG_ENTRY(REG) \
+	[REG] = {				  \
+		.odin_offset = ODN_##REG,	  \
+		.vali_offset = ODN_##REG,	  \
+		.orion_offset = SRS_##REG,	  \
+		.odin_name = ODIN_REGNAME(REG),	  \
+		.vali_name = VALI_REGNAME(REG), \
+		.orion_name = ORION_REGNAME(REG), \
+	}
+
+static const struct odin_orion_reg common_regs[] = {
+	COMMON_REG_ENTRY(CORE_CHANGE_SET),
+	COMMON_REG_ENTRY(CORE_USER_ID),
+	COMMON_REG_ENTRY(CORE_USER_BUILD),
+	COMMON_REG_ENTRY(CORE_INTERRUPT_ENABLE),
+	COMMON_REG_ENTRY(CORE_INTERRUPT_CLR),
+	COMMON_REG_ENTRY(CORE_INTERRUPT_STATUS),
+	COMMON_REG_ENTRY(REG_BANK_ODN_CLK_BLK),
+};
+
+static inline const u32 common_reg_offset(struct tc_device *tc, u32 reg)
+{
+	if (tc->odin)
+		return common_regs[reg].odin_offset;
+	else if (tc->vali)
+		return common_regs[reg].vali_offset;
+	else
+		return common_regs[reg].orion_offset;
+}
+
+static inline const char *common_reg_name(struct tc_device *tc, u32 reg)
+{
+	if (tc->odin)
+		return common_regs[reg].odin_name;
+	else if (tc->vali)
+		return common_regs[reg].vali_name;
+	else
+		return common_regs[reg].orion_name;
+}
+
+#endif	/* __TC_ODIN_COMMON_REGS_H__ */
diff --git a/drivers/gpu/drm/img-rogue/apollo/tcf_clk_ctrl.h b/drivers/gpu/drm/img-rogue/apollo/tcf_clk_ctrl.h
new file mode 100644
index 000000000000..cc7b10fd8116
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tcf_clk_ctrl.h
@@ -0,0 +1,1018 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework system control register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from: tcf_clk_ctrl.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_CLK_CTRL_H_)
+#define _TCF_CLK_CTRL_H_
+
+/*
+ * The following register definitions are valid if register 0x28 has value 0.
+ */
+
+/*
+	Register FPGA_ID_REG
+*/
+#define TCF_CLK_CTRL_FPGA_ID_REG            0x0000
+#define FPGA_ID_REG_CORE_CFG_MASK           0x0000FFFFU
+#define FPGA_ID_REG_CORE_CFG_SHIFT          0
+#define FPGA_ID_REG_CORE_CFG_SIGNED         0
+
+#define FPGA_ID_REG_CORE_ID_MASK            0xFFFF0000U
+#define FPGA_ID_REG_CORE_ID_SHIFT           16
+#define FPGA_ID_REG_CORE_ID_SIGNED          0
+
+/*
+	Register FPGA_REV_REG
+*/
+#define TCF_CLK_CTRL_FPGA_REV_REG           0x0008
+#define FPGA_REV_REG_MAINT_MASK             0x000000FFU
+#define FPGA_REV_REG_MAINT_SHIFT            0
+#define FPGA_REV_REG_MAINT_SIGNED           0
+
+#define FPGA_REV_REG_MINOR_MASK             0x0000FF00U
+#define FPGA_REV_REG_MINOR_SHIFT            8
+#define FPGA_REV_REG_MINOR_SIGNED           0
+
+#define FPGA_REV_REG_MAJOR_MASK             0x00FF0000U
+#define FPGA_REV_REG_MAJOR_SHIFT            16
+#define FPGA_REV_REG_MAJOR_SIGNED           0
+
+#define FPGA_REV_REG_DESIGNER_MASK          0xFF000000U
+#define FPGA_REV_REG_DESIGNER_SHIFT         24
+#define FPGA_REV_REG_DESIGNER_SIGNED        0
+
+/*
+	Register FPGA_DES_REV_1
+*/
+#define TCF_CLK_CTRL_FPGA_DES_REV_1         0x0010
+#define FPGA_DES_REV_1_MASK                 0xFFFFFFFFU
+#define FPGA_DES_REV_1_SHIFT                0
+#define FPGA_DES_REV_1_SIGNED               0
+
+/*
+	Register FPGA_DES_REV_2
+*/
+#define TCF_CLK_CTRL_FPGA_DES_REV_2         0x0018
+#define FPGA_DES_REV_2_MASK                 0xFFFFFFFFU
+#define FPGA_DES_REV_2_SHIFT                0
+#define FPGA_DES_REV_2_SIGNED               0
+
+/*
+	Register TCF_CORE_ID_REG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_ID_REG        0x0020
+#define TCF_CORE_ID_REG_CORE_CFG_MASK       0x0000FFFFU
+#define TCF_CORE_ID_REG_CORE_CFG_SHIFT      0
+#define TCF_CORE_ID_REG_CORE_CFG_SIGNED     0
+
+#define TCF_CORE_ID_REG_CORE_ID_MASK        0xFFFF0000U
+#define TCF_CORE_ID_REG_CORE_ID_SHIFT       16
+#define TCF_CORE_ID_REG_CORE_ID_SIGNED      0
+
+/*
+	Register TCF_CORE_REV_REG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_REV_REG       0x0028
+#define TCF_CORE_REV_REG_MAINT_MASK         0x000000FFU
+#define TCF_CORE_REV_REG_MAINT_SHIFT        0
+#define TCF_CORE_REV_REG_MAINT_SIGNED       0
+
+#define TCF_CORE_REV_REG_MINOR_MASK         0x0000FF00U
+#define TCF_CORE_REV_REG_MINOR_SHIFT        8
+#define TCF_CORE_REV_REG_MINOR_SIGNED       0
+
+#define TCF_CORE_REV_REG_MAJOR_MASK         0x00FF0000U
+#define TCF_CORE_REV_REG_MAJOR_SHIFT        16
+#define TCF_CORE_REV_REG_MAJOR_SIGNED       0
+
+#define TCF_CORE_REV_REG_DESIGNER_MASK      0xFF000000U
+#define TCF_CORE_REV_REG_DESIGNER_SHIFT     24
+#define TCF_CORE_REV_REG_DESIGNER_SIGNED    0
+
+/*
+	Register TCF_CORE_DES_REV_1
+*/
+#define TCF_CLK_CTRL_TCF_CORE_DES_REV_1     0x0030
+#define TCF_CORE_DES_REV_1_MASK             0xFFFFFFFFU
+#define TCF_CORE_DES_REV_1_SHIFT            0
+#define TCF_CORE_DES_REV_1_SIGNED           0
+
+/*
+	Register TCF_CORE_DES_REV_2
+*/
+#define TCF_CLK_CTRL_TCF_CORE_DES_REV_2     0x0038
+#define TCF_CORE_DES_REV_2_MASK             0xFFFFFFFFU
+#define TCF_CORE_DES_REV_2_SHIFT            0
+#define TCF_CORE_DES_REV_2_SIGNED           0
+
+
+/*
+ * The following register definitions are valid if register 0x28 has value 1.
+ */
+
+/*
+	Register ID
+*/
+#define TCF_CLK_CTRL_ID                     0x0000
+#define VARIANT_MASK                        0x0000FFFFU
+#define VARIANT_SHIFT                       0
+#define VARIANT_SIGNED                      0
+
+#define ID_MASK                             0xFFFF0000U
+#define ID_SHIFT                            16
+#define ID_SIGNED                           0
+
+/*
+	Register REL
+*/
+#define TCF_CLK_CTRL_REL                    0x0008
+#define MINOR_MASK                          0x0000FFFFU
+#define MINOR_SHIFT                         0
+#define MINOR_SIGNED                        0
+
+#define MAJOR_MASK                          0xFFFF0000U
+#define MAJOR_SHIFT                         16
+#define MAJOR_SIGNED                        0
+
+/*
+	Register CHANGE_SET
+*/
+#define TCF_CLK_CTRL_CHANGE_SET             0x0010
+#define SET_MASK                            0xFFFFFFFFU
+#define SET_SHIFT                           0
+#define SET_SIGNED                          0
+
+/*
+	Register USER_ID
+*/
+#define TCF_CLK_CTRL_USER_ID                0x0018
+#define USER_ID_MASK                        0x0000000FU
+#define USER_ID_SHIFT                       0
+#define USER_ID_SIGNED                      0
+
+/*
+	Register USER_BUILD
+*/
+#define TCF_CLK_CTRL_USER_BUILD             0x0020
+#define BUILD_MASK                          0xFFFFFFFFU
+#define BUILD_SHIFT                         0
+#define BUILD_SIGNED                        0
+
+/*
+	Register SW_IF_VERSION
+*/
+#define TCF_CLK_CTRL_SW_IF_VERSION          0x0028
+#define VERSION_MASK                        0x0000FFFFU
+#define VERSION_SHIFT                       0
+#define VERSION_SIGNED                      0
+
+/*
+ * The following register definitions are valid for all Apollo builds,
+ * even if some of the registers are not available for certain cores.
+ */
+
+/*
+	Register SCB_GENERAL_CONTROL
+*/
+#define TCF_CLK_CTRL_SCB_GENERAL_CONTROL    0x0040
+#define SCB_GC_TRANS_HALT_MASK              0x00000200U
+#define SCB_GC_TRANS_HALT_SHIFT             9
+#define SCB_GC_TRANS_HALT_SIGNED            0
+
+#define SCB_GC_CKD_REGS_MASK                0x00000100U
+#define SCB_GC_CKD_REGS_SHIFT               8
+#define SCB_GC_CKD_REGS_SIGNED              0
+
+#define SCB_GC_CKD_SLAVE_MASK               0x00000080U
+#define SCB_GC_CKD_SLAVE_SHIFT              7
+#define SCB_GC_CKD_SLAVE_SIGNED             0
+
+#define SCB_GC_CKD_MASTER_MASK              0x00000040U
+#define SCB_GC_CKD_MASTER_SHIFT             6
+#define SCB_GC_CKD_MASTER_SIGNED            0
+
+#define SCB_GC_CKD_XDATA_MASK               0x00000020U
+#define SCB_GC_CKD_XDATA_SHIFT              5
+#define SCB_GC_CKD_XDATA_SIGNED             0
+
+#define SCB_GC_SFR_REG_MASK                 0x00000010U
+#define SCB_GC_SFR_REG_SHIFT                4
+#define SCB_GC_SFR_REG_SIGNED               0
+
+#define SCB_GC_SFR_SLAVE_MASK               0x00000008U
+#define SCB_GC_SFR_SLAVE_SHIFT              3
+#define SCB_GC_SFR_SLAVE_SIGNED             0
+
+#define SCB_GC_SFR_MASTER_MASK              0x00000004U
+#define SCB_GC_SFR_MASTER_SHIFT             2
+#define SCB_GC_SFR_MASTER_SIGNED            0
+
+#define SCB_GC_SFR_DET_DATA_MASK            0x00000002U
+#define SCB_GC_SFR_DET_DATA_SHIFT           1
+#define SCB_GC_SFR_DET_DATA_SIGNED          0
+
+#define SCB_GC_SFR_GEN_DATA_MASK            0x00000001U
+#define SCB_GC_SFR_GEN_DATA_SHIFT           0
+#define SCB_GC_SFR_GEN_DATA_SIGNED          0
+
+/*
+	Register SCB_MASTER_READ_COUNT
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_READ_COUNT  0x0048
+#define MASTER_READ_COUNT_MASK              0x0000FFFFU
+#define MASTER_READ_COUNT_SHIFT             0
+#define MASTER_READ_COUNT_SIGNED            0
+
+/*
+	Register SCB_MASTER_READ_DATA
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_READ_DATA   0x0050
+#define MASTER_READ_DATA_MASK               0x000000FFU
+#define MASTER_READ_DATA_SHIFT              0
+#define MASTER_READ_DATA_SIGNED             0
+
+/*
+	Register SCB_MASTER_ADDRESS
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_ADDRESS     0x0058
+#define SCB_MASTER_ADDRESS_MASK             0x000003FFU
+#define SCB_MASTER_ADDRESS_SHIFT            0
+#define SCB_MASTER_ADDRESS_SIGNED           0
+
+/*
+	Register SCB_MASTER_WRITE_DATA
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_WRITE_DATA  0x0060
+#define MASTER_WRITE_DATA_MASK              0x000000FFU
+#define MASTER_WRITE_DATA_SHIFT             0
+#define MASTER_WRITE_DATA_SIGNED            0
+
+/*
+	Register SCB_MASTER_WRITE_COUNT
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_WRITE_COUNT 0x0068
+#define MASTER_WRITE_COUNT_MASK             0x0000FFFFU
+#define MASTER_WRITE_COUNT_SHIFT            0
+#define MASTER_WRITE_COUNT_SIGNED           0
+
+/*
+	Register SCB_BUS_SELECT
+*/
+#define TCF_CLK_CTRL_SCB_BUS_SELECT         0x0070
+#define BUS_SELECT_MASK                     0x00000003U
+#define BUS_SELECT_SHIFT                    0
+#define BUS_SELECT_SIGNED                   0
+
+/*
+	Register SCB_MASTER_FILL_STATUS
+*/
+#define TCF_CLK_CTRL_SCB_MASTER_FILL_STATUS 0x0078
+#define MASTER_WRITE_FIFO_EMPTY_MASK        0x00000008U
+#define MASTER_WRITE_FIFO_EMPTY_SHIFT       3
+#define MASTER_WRITE_FIFO_EMPTY_SIGNED      0
+
+#define MASTER_WRITE_FIFO_FULL_MASK         0x00000004U
+#define MASTER_WRITE_FIFO_FULL_SHIFT        2
+#define MASTER_WRITE_FIFO_FULL_SIGNED       0
+
+#define MASTER_READ_FIFO_EMPTY_MASK         0x00000002U
+#define MASTER_READ_FIFO_EMPTY_SHIFT        1
+#define MASTER_READ_FIFO_EMPTY_SIGNED       0
+
+#define MASTER_READ_FIFO_FULL_MASK          0x00000001U
+#define MASTER_READ_FIFO_FULL_SHIFT         0
+#define MASTER_READ_FIFO_FULL_SIGNED        0
+
+/*
+	Register CLK_AND_RST_CTRL
+*/
+#define TCF_CLK_CTRL_CLK_AND_RST_CTRL       0x0080
+#define GLB_CLKG_EN_MASK                    0x00020000U
+#define GLB_CLKG_EN_SHIFT                   17
+#define GLB_CLKG_EN_SIGNED                  0
+
+#define CLK_GATE_CNTL_MASK                  0x00010000U
+#define CLK_GATE_CNTL_SHIFT                 16
+#define CLK_GATE_CNTL_SIGNED                0
+
+#define DUT_DCM_RESETN_MASK                 0x00000400U
+#define DUT_DCM_RESETN_SHIFT                10
+#define DUT_DCM_RESETN_SIGNED               0
+
+#define MEM_RESYNC_BYPASS_MASK              0x00000200U
+#define MEM_RESYNC_BYPASS_SHIFT             9
+#define MEM_RESYNC_BYPASS_SIGNED            0
+
+#define SYS_RESYNC_BYPASS_MASK              0x00000100U
+#define SYS_RESYNC_BYPASS_SHIFT             8
+#define SYS_RESYNC_BYPASS_SIGNED            0
+
+#define SCB_RESETN_MASK                     0x00000010U
+#define SCB_RESETN_SHIFT                    4
+#define SCB_RESETN_SIGNED                   0
+
+#define PDP2_RESETN_MASK                    0x00000008U
+#define PDP2_RESETN_SHIFT                   3
+#define PDP2_RESETN_SIGNED                  0
+
+#define PDP1_RESETN_MASK                    0x00000004U
+#define PDP1_RESETN_SHIFT                   2
+#define PDP1_RESETN_SIGNED                  0
+
+#define DDR_RESETN_MASK                     0x00000002U
+#define DDR_RESETN_SHIFT                    1
+#define DDR_RESETN_SIGNED                   0
+
+#define DUT_RESETN_MASK                     0x00000001U
+#define DUT_RESETN_SHIFT                    0
+#define DUT_RESETN_SIGNED                   0
+
+/*
+	Register TEST_REG_OUT
+*/
+#define TCF_CLK_CTRL_TEST_REG_OUT           0x0088
+#define TEST_REG_OUT_MASK                   0xFFFFFFFFU
+#define TEST_REG_OUT_SHIFT                  0
+#define TEST_REG_OUT_SIGNED                 0
+
+/*
+	Register TEST_REG_IN
+*/
+#define TCF_CLK_CTRL_TEST_REG_IN            0x0090
+#define TEST_REG_IN_MASK                    0xFFFFFFFFU
+#define TEST_REG_IN_SHIFT                   0
+#define TEST_REG_IN_SIGNED                  0
+
+/*
+	Register TEST_CTRL
+*/
+#define TCF_CLK_CTRL_TEST_CTRL              0x0098
+#define PCI_TEST_OFFSET_MASK                0xF8000000U
+#define PCI_TEST_OFFSET_SHIFT               27
+#define PCI_TEST_OFFSET_SIGNED              0
+
+#define PDP1_HOST_MEM_SELECT_MASK           0x00000200U
+#define PDP1_HOST_MEM_SELECT_SHIFT          9
+#define PDP1_HOST_MEM_SELECT_SIGNED         0
+
+#define HOST_PHY_MODE_MASK                  0x00000100U
+#define HOST_PHY_MODE_SHIFT                 8
+#define HOST_PHY_MODE_SIGNED                0
+
+#define HOST_ONLY_MODE_MASK                 0x00000080U
+#define HOST_ONLY_MODE_SHIFT                7
+#define HOST_ONLY_MODE_SIGNED               0
+
+#define PCI_TEST_MODE_MASK                  0x00000040U
+#define PCI_TEST_MODE_SHIFT                 6
+#define PCI_TEST_MODE_SIGNED                0
+
+#define TURN_OFF_DDR_MASK                   0x00000020U
+#define TURN_OFF_DDR_SHIFT                  5
+#define TURN_OFF_DDR_SIGNED                 0
+
+#define SYS_RD_CLK_INV_MASK                 0x00000010U
+#define SYS_RD_CLK_INV_SHIFT                4
+#define SYS_RD_CLK_INV_SIGNED               0
+
+#define MEM_REQ_CLK_INV_MASK                0x00000008U
+#define MEM_REQ_CLK_INV_SHIFT               3
+#define MEM_REQ_CLK_INV_SIGNED              0
+
+#define BURST_SPLIT_MASK                    0x00000004U
+#define BURST_SPLIT_SHIFT                   2
+#define BURST_SPLIT_SIGNED                  0
+
+#define CLK_INVERSION_MASK                  0x00000002U
+#define CLK_INVERSION_SHIFT                 1
+#define CLK_INVERSION_SIGNED                0
+
+#define ADDRESS_FORCE_MASK                  0x00000001U
+#define ADDRESS_FORCE_SHIFT                 0
+#define ADDRESS_FORCE_SIGNED                0
+
+/*
+	Register CLEAR_HOST_MEM_SIG
+*/
+#define TCF_CLK_CTRL_CLEAR_HOST_MEM_SIG     0x00A0
+#define SIGNATURE_TAG_ID_MASK               0x00000F00U
+#define SIGNATURE_TAG_ID_SHIFT              8
+#define SIGNATURE_TAG_ID_SIGNED             0
+
+#define CLEAR_HOST_MEM_SIGNATURE_MASK       0x00000001U
+#define CLEAR_HOST_MEM_SIGNATURE_SHIFT      0
+#define CLEAR_HOST_MEM_SIGNATURE_SIGNED     0
+
+/*
+	Register HOST_MEM_SIGNATURE
+*/
+#define TCF_CLK_CTRL_HOST_MEM_SIGNATURE     0x00A8
+#define HOST_MEM_SIGNATURE_MASK             0xFFFFFFFFU
+#define HOST_MEM_SIGNATURE_SHIFT            0
+#define HOST_MEM_SIGNATURE_SIGNED           0
+
+/*
+	Register INTERRUPT_STATUS
+*/
+#define TCF_CLK_CTRL_INTERRUPT_STATUS       0x00C8
+#define INTERRUPT_MASTER_STATUS_MASK        0x80000000U
+#define INTERRUPT_MASTER_STATUS_SHIFT       31
+#define INTERRUPT_MASTER_STATUS_SIGNED      0
+
+#define OTHER_INTS_MASK                     0x7FFE0000U
+#define OTHER_INTS_SHIFT                    17
+#define OTHER_INTS_SIGNED                   0
+
+#define HOST_MST_NORESPONSE_MASK            0x00010000U
+#define HOST_MST_NORESPONSE_SHIFT           16
+#define HOST_MST_NORESPONSE_SIGNED          0
+
+#define PDP2_INT_MASK                       0x00008000U
+#define PDP2_INT_SHIFT                      15
+#define PDP2_INT_SIGNED                     0
+
+#define PDP1_INT_MASK                       0x00004000U
+#define PDP1_INT_SHIFT                      14
+#define PDP1_INT_SIGNED                     0
+
+#define EXT_INT_MASK                        0x00002000U
+#define EXT_INT_SHIFT                       13
+#define EXT_INT_SIGNED                      0
+
+#define SCB_MST_HLT_BIT_MASK                0x00001000U
+#define SCB_MST_HLT_BIT_SHIFT               12
+#define SCB_MST_HLT_BIT_SIGNED              0
+
+#define SCB_SLV_EVENT_MASK                  0x00000800U
+#define SCB_SLV_EVENT_SHIFT                 11
+#define SCB_SLV_EVENT_SIGNED                0
+
+#define SCB_TDONE_RX_MASK                   0x00000400U
+#define SCB_TDONE_RX_SHIFT                  10
+#define SCB_TDONE_RX_SIGNED                 0
+
+#define SCB_SLV_WT_RD_DAT_MASK              0x00000200U
+#define SCB_SLV_WT_RD_DAT_SHIFT             9
+#define SCB_SLV_WT_RD_DAT_SIGNED            0
+
+#define SCB_SLV_WT_PRV_RD_MASK              0x00000100U
+#define SCB_SLV_WT_PRV_RD_SHIFT             8
+#define SCB_SLV_WT_PRV_RD_SIGNED            0
+
+#define SCB_SLV_WT_WR_DAT_MASK              0x00000080U
+#define SCB_SLV_WT_WR_DAT_SHIFT             7
+#define SCB_SLV_WT_WR_DAT_SIGNED            0
+
+#define SCB_MST_WT_RD_DAT_MASK              0x00000040U
+#define SCB_MST_WT_RD_DAT_SHIFT             6
+#define SCB_MST_WT_RD_DAT_SIGNED            0
+
+#define SCB_ADD_ACK_ERR_MASK                0x00000020U
+#define SCB_ADD_ACK_ERR_SHIFT               5
+#define SCB_ADD_ACK_ERR_SIGNED              0
+
+#define SCB_WR_ACK_ERR_MASK                 0x00000010U
+#define SCB_WR_ACK_ERR_SHIFT                4
+#define SCB_WR_ACK_ERR_SIGNED               0
+
+#define SCB_SDAT_LO_TIM_MASK                0x00000008U
+#define SCB_SDAT_LO_TIM_SHIFT               3
+#define SCB_SDAT_LO_TIM_SIGNED              0
+
+#define SCB_SCLK_LO_TIM_MASK                0x00000004U
+#define SCB_SCLK_LO_TIM_SHIFT               2
+#define SCB_SCLK_LO_TIM_SIGNED              0
+
+#define SCB_UNEX_START_BIT_MASK             0x00000002U
+#define SCB_UNEX_START_BIT_SHIFT            1
+#define SCB_UNEX_START_BIT_SIGNED           0
+
+#define SCB_BUS_INACTIVE_MASK               0x00000001U
+#define SCB_BUS_INACTIVE_SHIFT              0
+#define SCB_BUS_INACTIVE_SIGNED             0
+
+/*
+	Register INTERRUPT_OP_CFG
+*/
+#define TCF_CLK_CTRL_INTERRUPT_OP_CFG       0x00D0
+#define PULSE_NLEVEL_MASK                   0x80000000U
+#define PULSE_NLEVEL_SHIFT                  31
+#define PULSE_NLEVEL_SIGNED                 0
+
+#define INT_SENSE_MASK                      0x40000000U
+#define INT_SENSE_SHIFT                     30
+#define INT_SENSE_SIGNED                    0
+
+#define INTERRUPT_DEST_MASK                 0x0000000FU
+#define INTERRUPT_DEST_SHIFT                0
+#define INTERRUPT_DEST_SIGNED               0
+
+/*
+	Register INTERRUPT_ENABLE
+*/
+#define TCF_CLK_CTRL_INTERRUPT_ENABLE       0x00D8
+#define INTERRUPT_MASTER_ENABLE_MASK        0x80000000U
+#define INTERRUPT_MASTER_ENABLE_SHIFT       31
+#define INTERRUPT_MASTER_ENABLE_SIGNED      0
+
+#define INTERRUPT_ENABLE_MASK               0x7FFFFFFFU
+#define INTERRUPT_ENABLE_SHIFT              0
+#define INTERRUPT_ENABLE_SIGNED             0
+
+/*
+	Register INTERRUPT_CLEAR
+*/
+#define TCF_CLK_CTRL_INTERRUPT_CLEAR        0x00E0
+#define INTERRUPT_MASTER_CLEAR_MASK         0x80000000U
+#define INTERRUPT_MASTER_CLEAR_SHIFT        31
+#define INTERRUPT_MASTER_CLEAR_SIGNED       0
+
+#define INTERRUPT_CLEAR_MASK                0x7FFFFFFFU
+#define INTERRUPT_CLEAR_SHIFT               0
+#define INTERRUPT_CLEAR_SIGNED              0
+
+/*
+	Register YCC_RGB_CTRL
+*/
+#define TCF_CLK_CTRL_YCC_RGB_CTRL           0x00E8
+#define RGB_CTRL1_MASK                      0x000001FFU
+#define RGB_CTRL1_SHIFT                     0
+#define RGB_CTRL1_SIGNED                    0
+
+#define RGB_CTRL2_MASK                      0x01FF0000U
+#define RGB_CTRL2_SHIFT                     16
+#define RGB_CTRL2_SIGNED                    0
+
+/*
+	Register EXP_BRD_CTRL
+*/
+#define TCF_CLK_CTRL_EXP_BRD_CTRL           0x00F8
+#define PDP1_DATA_EN_MASK                   0x00000003U
+#define PDP1_DATA_EN_SHIFT                  0
+#define PDP1_DATA_EN_SIGNED                 0
+
+#define PDP2_DATA_EN_MASK                   0x00000030U
+#define PDP2_DATA_EN_SHIFT                  4
+#define PDP2_DATA_EN_SIGNED                 0
+
+#define EXP_BRD_OUTPUT_MASK                 0xFFFFFF00U
+#define EXP_BRD_OUTPUT_SHIFT                8
+#define EXP_BRD_OUTPUT_SIGNED               0
+
+/*
+	Register HOSTIF_CONTROL
+*/
+#define TCF_CLK_CTRL_HOSTIF_CONTROL         0x0100
+#define HOSTIF_CTRL_MASK                    0x000000FFU
+#define HOSTIF_CTRL_SHIFT                   0
+#define HOSTIF_CTRL_SIGNED                  0
+
+/*
+	Register DUT_CONTROL_1
+*/
+#define TCF_CLK_CTRL_DUT_CONTROL_1          0x0108
+#define DUT_CTRL_1_MASK                     0xFFFFFFFFU
+#define DUT_CTRL_1_SHIFT                    0
+#define DUT_CTRL_1_SIGNED                   0
+
+/* TC ES2 additional needs those: */
+#define DUT_CTRL_TEST_MODE_SHIFT            0
+#define DUT_CTRL_TEST_MODE_MASK             0x3
+
+#define DUT_CTRL_VCC_0V9EN                  (1<<12)
+#define DUT_CTRL_VCC_1V8EN                  (1<<13)
+#define DUT_CTRL_VCC_IO_INH                 (1<<14)
+#define DUT_CTRL_VCC_CORE_INH               (1<<15)
+
+/*
+	Register DUT_STATUS_1
+*/
+#define TCF_CLK_CTRL_DUT_STATUS_1           0x0110
+#define DUT_STATUS_1_MASK                   0xFFFFFFFFU
+#define DUT_STATUS_1_SHIFT                  0
+#define DUT_STATUS_1_SIGNED                 0
+
+/*
+	Register DUT_CTRL_NOT_STAT_1
+*/
+#define TCF_CLK_CTRL_DUT_CTRL_NOT_STAT_1    0x0118
+#define DUT_STAT_NOT_CTRL_1_MASK            0xFFFFFFFFU
+#define DUT_STAT_NOT_CTRL_1_SHIFT           0
+#define DUT_STAT_NOT_CTRL_1_SIGNED          0
+
+/*
+	Register DUT_CONTROL_2
+*/
+#define TCF_CLK_CTRL_DUT_CONTROL_2          0x0120
+#define DUT_CTRL_2_MASK                     0xFFFFFFFFU
+#define DUT_CTRL_2_SHIFT                    0
+#define DUT_CTRL_2_SIGNED                   0
+
+/*
+	Register DUT_STATUS_2
+*/
+#define TCF_CLK_CTRL_DUT_STATUS_2           0x0128
+#define DUT_STATUS_2_MASK                   0xFFFFFFFFU
+#define DUT_STATUS_2_SHIFT                  0
+#define DUT_STATUS_2_SIGNED                 0
+
+/*
+	Register DUT_CTRL_NOT_STAT_2
+*/
+#define TCF_CLK_CTRL_DUT_CTRL_NOT_STAT_2    0x0130
+#define DUT_CTRL_NOT_STAT_2_MASK            0xFFFFFFFFU
+#define DUT_CTRL_NOT_STAT_2_SHIFT           0
+#define DUT_CTRL_NOT_STAT_2_SIGNED          0
+
+/*
+	Register BUS_CAP_BASE_ADDR
+*/
+#define TCF_CLK_CTRL_BUS_CAP_BASE_ADDR      0x0138
+#define BUS_CAP_BASE_ADDR_MASK              0xFFFFFFFFU
+#define BUS_CAP_BASE_ADDR_SHIFT             0
+#define BUS_CAP_BASE_ADDR_SIGNED            0
+
+/*
+	Register BUS_CAP_ENABLE
+*/
+#define TCF_CLK_CTRL_BUS_CAP_ENABLE         0x0140
+#define BUS_CAP_ENABLE_MASK                 0x00000001U
+#define BUS_CAP_ENABLE_SHIFT                0
+#define BUS_CAP_ENABLE_SIGNED               0
+
+/*
+	Register BUS_CAP_COUNT
+*/
+#define TCF_CLK_CTRL_BUS_CAP_COUNT          0x0148
+#define BUS_CAP_COUNT_MASK                  0xFFFFFFFFU
+#define BUS_CAP_COUNT_SHIFT                 0
+#define BUS_CAP_COUNT_SIGNED                0
+
+/*
+	Register DCM_LOCK_STATUS
+*/
+#define TCF_CLK_CTRL_DCM_LOCK_STATUS        0x0150
+#define DCM_LOCK_STATUS_MASK                0x00000007U
+#define DCM_LOCK_STATUS_SHIFT               0
+#define DCM_LOCK_STATUS_SIGNED              0
+
+/*
+	Register AUX_DUT_RESETNS
+*/
+#define TCF_CLK_CTRL_AUX_DUT_RESETNS        0x0158
+#define AUX_DUT_RESETNS_MASK                0x0000000FU
+#define AUX_DUT_RESETNS_SHIFT               0
+#define AUX_DUT_RESETNS_SIGNED              0
+
+/*
+	Register TCF_SPI_MST_ADDR_RDNWR
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_ADDR_RDNWR 0x0160
+#define TCF_SPI_MST_ADDR_MASK               0x0003FFFFU
+#define TCF_SPI_MST_ADDR_SHIFT              0
+#define TCF_SPI_MST_ADDR_SIGNED             0
+
+#define TCF_SPI_MST_RDNWR_MASK              0x00040000U
+#define TCF_SPI_MST_RDNWR_SHIFT             18
+#define TCF_SPI_MST_RDNWR_SIGNED            0
+
+#define TCF_SPI_MST_SLAVE_ID_MASK           0x00080000U
+#define TCF_SPI_MST_SLAVE_ID_SHIFT          19
+#define TCF_SPI_MST_SLAVE_ID_SIGNED         0
+
+#define TCF_SPI_MST_MASTER_ID_MASK          0x00300000U
+#define TCF_SPI_MST_MASTER_ID_SHIFT         20
+#define TCF_SPI_MST_MASTER_ID_SIGNED        0
+
+/*
+	Register TCF_SPI_MST_WDATA
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_WDATA      0x0168
+#define TCF_SPI_MST_WDATA_MASK              0xFFFFFFFFU
+#define TCF_SPI_MST_WDATA_SHIFT             0
+#define TCF_SPI_MST_WDATA_SIGNED            0
+
+/*
+	Register TCF_SPI_MST_RDATA
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_RDATA      0x0170
+#define TCF_SPI_MST_RDATA_MASK              0xFFFFFFFFU
+#define TCF_SPI_MST_RDATA_SHIFT             0
+#define TCF_SPI_MST_RDATA_SIGNED            0
+
+/*
+	Register TCF_SPI_MST_STATUS
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_STATUS     0x0178
+#define TCF_SPI_MST_STATUS_MASK             0x0000000FU
+#define TCF_SPI_MST_STATUS_SHIFT            0
+#define TCF_SPI_MST_STATUS_SIGNED           0
+
+/*
+	Register TCF_SPI_MST_GO
+*/
+#define TCF_CLK_CTRL_TCF_SPI_MST_GO         0x0180
+#define TCF_SPI_MST_GO_MASK                 0x00000001U
+#define TCF_SPI_MST_GO_SHIFT                0
+#define TCF_SPI_MST_GO_SIGNED               0
+
+/*
+	Register EXT_SIG_CTRL
+*/
+#define TCF_CLK_CTRL_EXT_SIG_CTRL           0x0188
+#define EXT_SYS_REQ_SIG_START_MASK          0x00000001U
+#define EXT_SYS_REQ_SIG_START_SHIFT         0
+#define EXT_SYS_REQ_SIG_START_SIGNED        0
+
+#define EXT_SYS_RD_SIG_START_MASK           0x00000002U
+#define EXT_SYS_RD_SIG_START_SHIFT          1
+#define EXT_SYS_RD_SIG_START_SIGNED         0
+
+#define EXT_MEM_REQ_SIG_START_MASK          0x00000004U
+#define EXT_MEM_REQ_SIG_START_SHIFT         2
+#define EXT_MEM_REQ_SIG_START_SIGNED        0
+
+#define EXT_MEM_RD_SIG_START_MASK           0x00000008U
+#define EXT_MEM_RD_SIG_START_SHIFT          3
+#define EXT_MEM_RD_SIG_START_SIGNED         0
+
+/*
+	Register EXT_SYS_REQ_SIG
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_SIG        0x0190
+#define EXT_SYS_REQ_SIG_MASK                0xFFFFFFFFU
+#define EXT_SYS_REQ_SIG_SHIFT               0
+#define EXT_SYS_REQ_SIG_SIGNED              0
+
+/*
+	Register EXT_SYS_RD_SIG
+*/
+#define TCF_CLK_CTRL_EXT_SYS_RD_SIG         0x0198
+#define EXT_SYS_RD_SIG_MASK                 0xFFFFFFFFU
+#define EXT_SYS_RD_SIG_SHIFT                0
+#define EXT_SYS_RD_SIG_SIGNED               0
+
+/*
+	Register EXT_MEM_REQ_SIG
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_SIG        0x01A0
+#define EXT_MEM_REQ_SIG_MASK                0xFFFFFFFFU
+#define EXT_MEM_REQ_SIG_SHIFT               0
+#define EXT_MEM_REQ_SIG_SIGNED              0
+
+/*
+	Register EXT_MEM_RD_SIG
+*/
+#define TCF_CLK_CTRL_EXT_MEM_RD_SIG         0x01A8
+#define EXT_MEM_RD_SIG_MASK                 0xFFFFFFFFU
+#define EXT_MEM_RD_SIG_SHIFT                0
+#define EXT_MEM_RD_SIG_SIGNED               0
+
+/*
+	Register EXT_SYS_REQ_WR_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_WR_CNT     0x01B0
+#define EXT_SYS_REQ_WR_CNT_MASK             0xFFFFFFFFU
+#define EXT_SYS_REQ_WR_CNT_SHIFT            0
+#define EXT_SYS_REQ_WR_CNT_SIGNED           0
+
+/*
+	Register EXT_SYS_REQ_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_REQ_RD_CNT     0x01B8
+#define EXT_SYS_REQ_RD_CNT_MASK             0xFFFFFFFFU
+#define EXT_SYS_REQ_RD_CNT_SHIFT            0
+#define EXT_SYS_REQ_RD_CNT_SIGNED           0
+
+/*
+	Register EXT_SYS_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_SYS_RD_CNT         0x01C0
+#define EXT_SYS_RD_CNT_MASK                 0xFFFFFFFFU
+#define EXT_SYS_RD_CNT_SHIFT                0
+#define EXT_SYS_RD_CNT_SIGNED               0
+
+/*
+	Register EXT_MEM_REQ_WR_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_WR_CNT     0x01C8
+#define EXT_MEM_REQ_WR_CNT_MASK             0xFFFFFFFFU
+#define EXT_MEM_REQ_WR_CNT_SHIFT            0
+#define EXT_MEM_REQ_WR_CNT_SIGNED           0
+
+/*
+	Register EXT_MEM_REQ_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_REQ_RD_CNT     0x01D0
+#define EXT_MEM_REQ_RD_CNT_MASK             0xFFFFFFFFU
+#define EXT_MEM_REQ_RD_CNT_SHIFT            0
+#define EXT_MEM_REQ_RD_CNT_SIGNED           0
+
+/*
+	Register EXT_MEM_RD_CNT
+*/
+#define TCF_CLK_CTRL_EXT_MEM_RD_CNT         0x01D8
+#define EXT_MEM_RD_CNT_MASK                 0xFFFFFFFFU
+#define EXT_MEM_RD_CNT_SHIFT                0
+#define EXT_MEM_RD_CNT_SIGNED               0
+
+/*
+	Register TCF_CORE_TARGET_BUILD_CFG
+*/
+#define TCF_CLK_CTRL_TCF_CORE_TARGET_BUILD_CFG 0x01E0
+#define TCF_CORE_TARGET_BUILD_ID_MASK       0x000000FFU
+#define TCF_CORE_TARGET_BUILD_ID_SHIFT      0
+#define TCF_CORE_TARGET_BUILD_ID_SIGNED     0
+
+/*
+	Register MEM_THROUGH_SYS
+*/
+#define TCF_CLK_CTRL_MEM_THROUGH_SYS        0x01E8
+#define MEM_THROUGH_SYS_MASK                0x00000001U
+#define MEM_THROUGH_SYS_SHIFT               0
+#define MEM_THROUGH_SYS_SIGNED              0
+
+/*
+	Register HOST_PHY_OFFSET
+*/
+#define TCF_CLK_CTRL_HOST_PHY_OFFSET        0x01F0
+#define HOST_PHY_OFFSET_MASK                0xFFFFFFFFU
+#define HOST_PHY_OFFSET_SHIFT               0
+#define HOST_PHY_OFFSET_SIGNED              0
+
+/*
+	Register DEBUG_REG_SEL
+*/
+#define TCF_CLK_CTRL_DEBUG_REG_SEL          0x01F8
+#define DEBUG_REG_SELECT_MASK               0xFFFFFFFFU
+#define DEBUG_REG_SELECT_SHIFT              0
+#define DEBUG_REG_SELECT_SIGNED             0
+
+/*
+	Register DEBUG_REG
+*/
+#define TCF_CLK_CTRL_DEBUG_REG              0x0200
+#define DEBUG_REG_VALUE_MASK                0xFFFFFFFFU
+#define DEBUG_REG_VALUE_SHIFT               0
+#define DEBUG_REG_VALUE_SIGNED              0
+
+/*
+	Register JTAG_CTRL
+*/
+#define TCF_CLK_CTRL_JTAG_CTRL              0x0208
+#define JTAG_TRST_MASK                      0x00000001U
+#define JTAG_TRST_SHIFT                     0
+#define JTAG_TRST_SIGNED                    0
+
+#define JTAG_TMS_MASK                       0x00000002U
+#define JTAG_TMS_SHIFT                      1
+#define JTAG_TMS_SIGNED                     0
+
+#define JTAG_TCK_MASK                       0x00000004U
+#define JTAG_TCK_SHIFT                      2
+#define JTAG_TCK_SIGNED                     0
+
+#define JTAG_TDO_MASK                       0x00000008U
+#define JTAG_TDO_SHIFT                      3
+#define JTAG_TDO_SIGNED                     0
+
+#define JTAG_TDI_MASK                       0x00000010U
+#define JTAG_TDI_SHIFT                      4
+#define JTAG_TDI_SIGNED                     0
+
+#define JTAG_DASH_N_REG_MASK                0x40000000U
+#define JTAG_DASH_N_REG_SHIFT               30
+#define JTAG_DASH_N_REG_SIGNED              0
+
+#define JTAG_DISABLE_MASK                   0x80000000U
+#define JTAG_DISABLE_SHIFT                  31
+#define JTAG_DISABLE_SIGNED                 0
+
+/*
+	Register SAI_DEBUG_RDNWR
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_RDNWR        0x0300
+#define SAI_DEBUG_REG_ADDR_MASK             0x000001FFU
+#define SAI_DEBUG_REG_ADDR_SHIFT            0
+#define SAI_DEBUG_REG_ADDR_SIGNED           0
+
+#define SAI_DEBUG_REG_RDNWR_MASK            0x00000200U
+#define SAI_DEBUG_REG_RDNWR_SHIFT           9
+#define SAI_DEBUG_REG_RDNWR_SIGNED          0
+
+/*
+	Register SAI_DEBUG_WDATA
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_WDATA        0x0308
+#define SAI_DEBUG_REG_WDATA_MASK            0xFFFFFFFFU
+#define SAI_DEBUG_REG_WDATA_SHIFT           0
+#define SAI_DEBUG_REG_WDATA_SIGNED          0
+
+/*
+	Register SAI_DEBUG_RDATA
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_RDATA        0x0310
+#define SAI_DEBUG_REG_RDATA_MASK            0xFFFFFFFFU
+#define SAI_DEBUG_REG_RDATA_SHIFT           0
+#define SAI_DEBUG_REG_RDATA_SIGNED          0
+
+/*
+	Register SAI_DEBUG_GO
+*/
+#define TCF_CLK_CTRL_SAI_DEBUG_GO           0x0318
+#define SAI_DEBUG_REG_GO_MASK               0x00000001U
+#define SAI_DEBUG_REG_GO_SHIFT              0
+#define SAI_DEBUG_REG_GO_SIGNED             0
+
+/*
+	Register AUX_DUT_RESETS
+*/
+#define TCF_CLK_CTRL_AUX_DUT_RESETS         0x0320
+#define AUX_DUT_RESETS_MASK                 0x0000000FU
+#define AUX_DUT_RESETS_SHIFT                0
+#define AUX_DUT_RESETS_SIGNED               0
+
+/*
+	Register DUT_CLK_CTRL
+*/
+#define TCF_CLK_CTRL_DUT_CLK_CTRL           0x0328
+#define MEM_REQ_PHSE_MASK                   0x0000FFFFU
+#define MEM_REQ_PHSE_SHIFT                  0
+#define MEM_REQ_PHSE_SIGNED                 0
+
+/*
+	Register DUT_CLK_STATUS
+*/
+#define TCF_CLK_CTRL_DUT_CLK_STATUS         0x0330
+#define MEM_REQ_PHSE_SET_MASK               0x00000003U
+#define MEM_REQ_PHSE_SET_SHIFT              0
+#define MEM_REQ_PHSE_SET_SIGNED             0
+
+/*
+	Register DUT_CLK_INFO
+*/
+#define TCF_CLK_CTRL_DUT_CLK_INFO           0x0340
+#define CORE_MASK                           0x0000FFFFU
+#define CORE_SHIFT                          0
+#define CORE_SIGNED                         0
+
+#define MEM_MASK                            0xFFFF0000U
+#define MEM_SHIFT                           16
+#define MEM_SIGNED                          0
+
+/*
+	Register DUT_CLK_PHSE
+*/
+#define TCF_CLK_CTRL_DUT_CLK_PHSE           0x0348
+#define MEM_REQ_MASK                        0x0000FFFFU
+#define MEM_REQ_SHIFT                       0
+#define MEM_REQ_SIGNED                      0
+
+#define MEM_RD_MASK                         0xFFFF0000U
+#define MEM_RD_SHIFT                        16
+#define MEM_RD_SIGNED                       0
+
+#endif /* !defined(_TCF_CLK_CTRL_H_) */
+
+/*****************************************************************************
+ End of file (tcf_clk_ctrl.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/tcf_pll.h b/drivers/gpu/drm/img-rogue/apollo/tcf_pll.h
new file mode 100644
index 000000000000..71eaf924bbd6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tcf_pll.h
@@ -0,0 +1,311 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework PDP register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from tcf_pll.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_PLL_H_)
+#define _TCF_PLL_H_
+
+/*
+	Register PLL_DDR2_CLK0
+*/
+#define TCF_PLL_PLL_DDR2_CLK0               0x0000
+#define DDR2_PLL_CLK0_PHS_MASK              0x00300000U
+#define DDR2_PLL_CLK0_PHS_SHIFT             20
+#define DDR2_PLL_CLK0_PHS_SIGNED            0
+
+#define DDR2_PLL_CLK0_MS_MASK               0x00030000U
+#define DDR2_PLL_CLK0_MS_SHIFT              16
+#define DDR2_PLL_CLK0_MS_SIGNED             0
+
+#define DDR2_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define DDR2_PLL_CLK0_FREQ_SHIFT            0
+#define DDR2_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_DDR2_CLK1TO5
+*/
+#define TCF_PLL_PLL_DDR2_CLK1TO5            0x0008
+#define DDR2_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define DDR2_PLL_CLK1TO5_PHS_SHIFT          20
+#define DDR2_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define DDR2_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define DDR2_PLL_CLK1TO5_MS_SHIFT           10
+#define DDR2_PLL_CLK1TO5_MS_SIGNED          0
+
+#define DDR2_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define DDR2_PLL_CLK1TO5_FREQ_SHIFT         0
+#define DDR2_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_DDR2_DRP_GO
+*/
+#define TCF_PLL_PLL_DDR2_DRP_GO             0x0010
+#define PLL_DDR2_DRP_GO_MASK                0x00000001U
+#define PLL_DDR2_DRP_GO_SHIFT               0
+#define PLL_DDR2_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_PDP_CLK0
+*/
+#define TCF_PLL_PLL_PDP_CLK0                0x0018
+#define PDP_PLL_CLK0_PHS_MASK               0x00300000U
+#define PDP_PLL_CLK0_PHS_SHIFT              20
+#define PDP_PLL_CLK0_PHS_SIGNED             0
+
+#define PDP_PLL_CLK0_MS_MASK                0x00030000U
+#define PDP_PLL_CLK0_MS_SHIFT               16
+#define PDP_PLL_CLK0_MS_SIGNED              0
+
+#define PDP_PLL_CLK0_FREQ_MASK              0x000001FFU
+#define PDP_PLL_CLK0_FREQ_SHIFT             0
+#define PDP_PLL_CLK0_FREQ_SIGNED            0
+
+/*
+	Register PLL_PDP_CLK1TO5
+*/
+#define TCF_PLL_PLL_PDP_CLK1TO5             0x0020
+#define PDP_PLL_CLK1TO5_PHS_MASK            0x3FF00000U
+#define PDP_PLL_CLK1TO5_PHS_SHIFT           20
+#define PDP_PLL_CLK1TO5_PHS_SIGNED          0
+
+#define PDP_PLL_CLK1TO5_MS_MASK             0x000FFC00U
+#define PDP_PLL_CLK1TO5_MS_SHIFT            10
+#define PDP_PLL_CLK1TO5_MS_SIGNED           0
+
+#define PDP_PLL_CLK1TO5_FREQ_MASK           0x000003FFU
+#define PDP_PLL_CLK1TO5_FREQ_SHIFT          0
+#define PDP_PLL_CLK1TO5_FREQ_SIGNED         0
+
+/*
+	Register PLL_PDP_DRP_GO
+*/
+#define TCF_PLL_PLL_PDP_DRP_GO              0x0028
+#define PLL_PDP_DRP_GO_MASK                 0x00000001U
+#define PLL_PDP_DRP_GO_SHIFT                0
+#define PLL_PDP_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_PDP2_CLK0
+*/
+#define TCF_PLL_PLL_PDP2_CLK0               0x0030
+#define PDP2_PLL_CLK0_PHS_MASK              0x00300000U
+#define PDP2_PLL_CLK0_PHS_SHIFT             20
+#define PDP2_PLL_CLK0_PHS_SIGNED            0
+
+#define PDP2_PLL_CLK0_MS_MASK               0x00030000U
+#define PDP2_PLL_CLK0_MS_SHIFT              16
+#define PDP2_PLL_CLK0_MS_SIGNED             0
+
+#define PDP2_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define PDP2_PLL_CLK0_FREQ_SHIFT            0
+#define PDP2_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_PDP2_CLK1TO5
+*/
+#define TCF_PLL_PLL_PDP2_CLK1TO5            0x0038
+#define PDP2_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define PDP2_PLL_CLK1TO5_PHS_SHIFT          20
+#define PDP2_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define PDP2_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define PDP2_PLL_CLK1TO5_MS_SHIFT           10
+#define PDP2_PLL_CLK1TO5_MS_SIGNED          0
+
+#define PDP2_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define PDP2_PLL_CLK1TO5_FREQ_SHIFT         0
+#define PDP2_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_PDP2_DRP_GO
+*/
+#define TCF_PLL_PLL_PDP2_DRP_GO             0x0040
+#define PLL_PDP2_DRP_GO_MASK                0x00000001U
+#define PLL_PDP2_DRP_GO_SHIFT               0
+#define PLL_PDP2_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_CORE_CLK0
+*/
+#define TCF_PLL_PLL_CORE_CLK0               0x0048
+#define CORE_PLL_CLK0_PHS_MASK              0x00300000U
+#define CORE_PLL_CLK0_PHS_SHIFT             20
+#define CORE_PLL_CLK0_PHS_SIGNED            0
+
+#define CORE_PLL_CLK0_MS_MASK               0x00030000U
+#define CORE_PLL_CLK0_MS_SHIFT              16
+#define CORE_PLL_CLK0_MS_SIGNED             0
+
+#define CORE_PLL_CLK0_FREQ_MASK             0x000001FFU
+#define CORE_PLL_CLK0_FREQ_SHIFT            0
+#define CORE_PLL_CLK0_FREQ_SIGNED           0
+
+/*
+	Register PLL_CORE_CLK1TO5
+*/
+#define TCF_PLL_PLL_CORE_CLK1TO5            0x0050
+#define CORE_PLL_CLK1TO5_PHS_MASK           0x3FF00000U
+#define CORE_PLL_CLK1TO5_PHS_SHIFT          20
+#define CORE_PLL_CLK1TO5_PHS_SIGNED         0
+
+#define CORE_PLL_CLK1TO5_MS_MASK            0x000FFC00U
+#define CORE_PLL_CLK1TO5_MS_SHIFT           10
+#define CORE_PLL_CLK1TO5_MS_SIGNED          0
+
+#define CORE_PLL_CLK1TO5_FREQ_MASK          0x000003FFU
+#define CORE_PLL_CLK1TO5_FREQ_SHIFT         0
+#define CORE_PLL_CLK1TO5_FREQ_SIGNED        0
+
+/*
+	Register PLL_CORE_DRP_GO
+*/
+#define TCF_PLL_PLL_CORE_DRP_GO             0x0058
+#define PLL_CORE_DRP_GO_MASK                0x00000001U
+#define PLL_CORE_DRP_GO_SHIFT               0
+#define PLL_CORE_DRP_GO_SIGNED              0
+
+/*
+	Register PLL_SYSIF_CLK0
+*/
+#define TCF_PLL_PLL_SYSIF_CLK0              0x0060
+#define SYSIF_PLL_CLK0_PHS_MASK             0x00300000U
+#define SYSIF_PLL_CLK0_PHS_SHIFT            20
+#define SYSIF_PLL_CLK0_PHS_SIGNED           0
+
+#define SYSIF_PLL_CLK0_MS_MASK              0x00030000U
+#define SYSIF_PLL_CLK0_MS_SHIFT             16
+#define SYSIF_PLL_CLK0_MS_SIGNED            0
+
+#define SYSIF_PLL_CLK0_FREQ_MASK            0x000001FFU
+#define SYSIF_PLL_CLK0_FREQ_SHIFT           0
+#define SYSIF_PLL_CLK0_FREQ_SIGNED          0
+
+/*
+	Register PLL_SYSIF_CLK1TO5
+*/
+#define TCF_PLL_PLL_SYSIF_CLK1TO5           0x0068
+#define SYSIF_PLL_CLK1TO5_PHS_MASK          0x3FF00000U
+#define SYSIF_PLL_CLK1TO5_PHS_SHIFT         20
+#define SYSIF_PLL_CLK1TO5_PHS_SIGNED        0
+
+#define SYSIF_PLL_CLK1TO5_MS_MASK           0x000FFC00U
+#define SYSIF_PLL_CLK1TO5_MS_SHIFT          10
+#define SYSIF_PLL_CLK1TO5_MS_SIGNED         0
+
+#define SYSIF_PLL_CLK1TO5_FREQ_MASK         0x000003FFU
+#define SYSIF_PLL_CLK1TO5_FREQ_SHIFT        0
+#define SYSIF_PLL_CLK1TO5_FREQ_SIGNED       0
+
+/*
+	Register PLL_SYS_DRP_GO
+*/
+#define TCF_PLL_PLL_SYS_DRP_GO              0x0070
+#define PLL_SYS_DRP_GO_MASK                 0x00000001U
+#define PLL_SYS_DRP_GO_SHIFT                0
+#define PLL_SYS_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_MEMIF_CLK0
+*/
+#define TCF_PLL_PLL_MEMIF_CLK0              0x0078
+#define MEMIF_PLL_CLK0_PHS_MASK             0x00300000U
+#define MEMIF_PLL_CLK0_PHS_SHIFT            20
+#define MEMIF_PLL_CLK0_PHS_SIGNED           0
+
+#define MEMIF_PLL_CLK0_MS_MASK              0x00030000U
+#define MEMIF_PLL_CLK0_MS_SHIFT             16
+#define MEMIF_PLL_CLK0_MS_SIGNED            0
+
+#define MEMIF_PLL_CLK0_FREQ_MASK            0x000001FFU
+#define MEMIF_PLL_CLK0_FREQ_SHIFT           0
+#define MEMIF_PLL_CLK0_FREQ_SIGNED          0
+
+/*
+	Register PLL_MEMIF_CLK1TO5
+*/
+#define TCF_PLL_PLL_MEMIF_CLK1TO5           0x0080
+#define MEMIF_PLL_CLK1TO5_PHS_MASK          0x3FF00000U
+#define MEMIF_PLL_CLK1TO5_PHS_SHIFT         20
+#define MEMIF_PLL_CLK1TO5_PHS_SIGNED        0
+
+#define MEMIF_PLL_CLK1TO5_MS_MASK           0x000FFC00U
+#define MEMIF_PLL_CLK1TO5_MS_SHIFT          10
+#define MEMIF_PLL_CLK1TO5_MS_SIGNED         0
+
+#define MEMIF_PLL_CLK1TO5_FREQ_MASK         0x000003FFU
+#define MEMIF_PLL_CLK1TO5_FREQ_SHIFT        0
+#define MEMIF_PLL_CLK1TO5_FREQ_SIGNED       0
+
+/*
+	Register PLL_MEM_DRP_GO
+*/
+#define TCF_PLL_PLL_MEM_DRP_GO              0x0088
+#define PLL_MEM_DRP_GO_MASK                 0x00000001U
+#define PLL_MEM_DRP_GO_SHIFT                0
+#define PLL_MEM_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_ALL_DRP_GO
+*/
+#define TCF_PLL_PLL_ALL_DRP_GO              0x0090
+#define PLL_ALL_DRP_GO_MASK                 0x00000001U
+#define PLL_ALL_DRP_GO_SHIFT                0
+#define PLL_ALL_DRP_GO_SIGNED               0
+
+/*
+	Register PLL_DRP_STATUS
+*/
+#define TCF_PLL_PLL_DRP_STATUS              0x0098
+#define PLL_LOCKS_MASK                      0x00003F00U
+#define PLL_LOCKS_SHIFT                     8
+#define PLL_LOCKS_SIGNED                    0
+
+#define PLL_DRP_GOOD_MASK                   0x0000003FU
+#define PLL_DRP_GOOD_SHIFT                  0
+#define PLL_DRP_GOOD_SIGNED                 0
+
+#endif /* !defined(_TCF_PLL_H_) */
+
+/*****************************************************************************
+ End of file (tcf_pll.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/apollo/tcf_rgbpdp_regs.h b/drivers/gpu/drm/img-rogue/apollo/tcf_rgbpdp_regs.h
new file mode 100644
index 000000000000..e87ba6152411
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/apollo/tcf_rgbpdp_regs.h
@@ -0,0 +1,559 @@
+/*************************************************************************/ /*!
+@Title          Test Chip Framework PDP register definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Autogenerated C -- do not edit
+                Generated from: tcf_rgbpdp_regs.def
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(_TCF_RGBPDP_REGS_H_)
+#define _TCF_RGBPDP_REGS_H_
+
+/*
+	Register PVR_TCF_RGBPDP_STR1SURF
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STR1SURF  0x0000
+#define STR1HEIGHT_MASK                     0x000007FFU
+#define STR1HEIGHT_SHIFT                    0
+#define STR1HEIGHT_SIGNED                   0
+
+#define STR1WIDTH_MASK                      0x003FF800U
+#define STR1WIDTH_SHIFT                     11
+#define STR1WIDTH_SIGNED                    0
+
+#define STR1PIXFMT_MASK                     0x0F000000U
+#define STR1PIXFMT_SHIFT                    24
+#define STR1PIXFMT_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_STR1ADDRCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STR1ADDRCTRL 0x0004
+#define STR1BASE_MASK                       0x03FFFFFFU
+#define STR1BASE_SHIFT                      0
+#define STR1BASE_SIGNED                     0
+
+#define STR1INTFIELD_MASK                   0x40000000U
+#define STR1INTFIELD_SHIFT                  30
+#define STR1INTFIELD_SIGNED                 0
+
+#define STR1STREN_MASK                      0x80000000U
+#define STR1STREN_SHIFT                     31
+#define STR1STREN_SIGNED                    0
+
+/*
+	Register PVR_PDP_STR1POSN
+*/
+#define TCF_RGBPDP_PVR_PDP_STR1POSN         0x0008
+#define STR1STRIDE_MASK                     0x000003FFU
+#define STR1STRIDE_SHIFT                    0
+#define STR1STRIDE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_MEMCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_MEMCTRL   0x000C
+#define MEMREFRESH_MASK                     0xC0000000U
+#define MEMREFRESH_SHIFT                    30
+#define MEMREFRESH_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_STRCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_STRCTRL   0x0010
+#define BURSTLEN_GFX_MASK                   0x000000FFU
+#define BURSTLEN_GFX_SHIFT                  0
+#define BURSTLEN_GFX_SIGNED                 0
+
+#define THRESHOLD_GFX_MASK                  0x0000FF00U
+#define THRESHOLD_GFX_SHIFT                 8
+#define THRESHOLD_GFX_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_SYNCCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_SYNCCTRL  0x0014
+#define HSDIS_MASK                          0x00000001U
+#define HSDIS_SHIFT                         0
+#define HSDIS_SIGNED                        0
+
+#define HSPOL_MASK                          0x00000002U
+#define HSPOL_SHIFT                         1
+#define HSPOL_SIGNED                        0
+
+#define VSDIS_MASK                          0x00000004U
+#define VSDIS_SHIFT                         2
+#define VSDIS_SIGNED                        0
+
+#define VSPOL_MASK                          0x00000008U
+#define VSPOL_SHIFT                         3
+#define VSPOL_SIGNED                        0
+
+#define BLNKDIS_MASK                        0x00000010U
+#define BLNKDIS_SHIFT                       4
+#define BLNKDIS_SIGNED                      0
+
+#define BLNKPOL_MASK                        0x00000020U
+#define BLNKPOL_SHIFT                       5
+#define BLNKPOL_SIGNED                      0
+
+#define HS_SLAVE_MASK                       0x00000040U
+#define HS_SLAVE_SHIFT                      6
+#define HS_SLAVE_SIGNED                     0
+
+#define VS_SLAVE_MASK                       0x00000080U
+#define VS_SLAVE_SHIFT                      7
+#define VS_SLAVE_SIGNED                     0
+
+#define INTERLACE_MASK                      0x00000100U
+#define INTERLACE_SHIFT                     8
+#define INTERLACE_SIGNED                    0
+
+#define FIELDPOL_MASK                       0x00000200U
+#define FIELDPOL_SHIFT                      9
+#define FIELDPOL_SIGNED                     0
+
+#define CLKPOL_MASK                         0x00000800U
+#define CLKPOL_SHIFT                        11
+#define CLKPOL_SIGNED                       0
+
+#define CSYNC_EN_MASK                       0x00001000U
+#define CSYNC_EN_SHIFT                      12
+#define CSYNC_EN_SIGNED                     0
+
+#define FIELD_EN_MASK                       0x00002000U
+#define FIELD_EN_SHIFT                      13
+#define FIELD_EN_SIGNED                     0
+
+#define UPDWAIT_MASK                        0x000F0000U
+#define UPDWAIT_SHIFT                       16
+#define UPDWAIT_SIGNED                      0
+
+#define UPDCTRL_MASK                        0x01000000U
+#define UPDCTRL_SHIFT                       24
+#define UPDCTRL_SIGNED                      0
+
+#define UPDINTCTRL_MASK                     0x02000000U
+#define UPDINTCTRL_SHIFT                    25
+#define UPDINTCTRL_SIGNED                   0
+
+#define UPDSYNCTRL_MASK                     0x04000000U
+#define UPDSYNCTRL_SHIFT                    26
+#define UPDSYNCTRL_SIGNED                   0
+
+#define POWERDN_MASK                        0x10000000U
+#define POWERDN_SHIFT                       28
+#define POWERDN_SIGNED                      0
+
+#define DISP_RST_MASK                       0x20000000U
+#define DISP_RST_SHIFT                      29
+#define DISP_RST_SIGNED                     0
+
+#define SYNCACTIVE_MASK                     0x80000000U
+#define SYNCACTIVE_SHIFT                    31
+#define SYNCACTIVE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_BORDCOL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_BORDCOL   0x0018
+#define BORDCOL_MASK                        0x00FFFFFFU
+#define BORDCOL_SHIFT                       0
+#define BORDCOL_SIGNED                      0
+
+/*
+	Register PVR_TCF_RGBPDP_UPDCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_UPDCTRL   0x001C
+#define UPDFIELD_MASK                       0x00000001U
+#define UPDFIELD_SHIFT                      0
+#define UPDFIELD_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC1
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC1    0x0020
+#define HT_MASK                             0x00000FFFU
+#define HT_SHIFT                            0
+#define HT_SIGNED                           0
+
+#define HBPS_MASK                           0x0FFF0000U
+#define HBPS_SHIFT                          16
+#define HBPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC2
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC2    0x0024
+#define HLBS_MASK                           0x00000FFFU
+#define HLBS_SHIFT                          0
+#define HLBS_SIGNED                         0
+
+#define HAS_MASK                            0x0FFF0000U
+#define HAS_SHIFT                           16
+#define HAS_SIGNED                          0
+
+/*
+	Register PVR_TCF_RGBPDP_HSYNC3
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HSYNC3    0x0028
+#define HRBS_MASK                           0x00000FFFU
+#define HRBS_SHIFT                          0
+#define HRBS_SIGNED                         0
+
+#define HFPS_MASK                           0x0FFF0000U
+#define HFPS_SHIFT                          16
+#define HFPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC1
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC1    0x002C
+#define VT_MASK                             0x00000FFFU
+#define VT_SHIFT                            0
+#define VT_SIGNED                           0
+
+#define VBPS_MASK                           0x0FFF0000U
+#define VBPS_SHIFT                          16
+#define VBPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC2
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC2    0x0030
+#define VTBS_MASK                           0x00000FFFU
+#define VTBS_SHIFT                          0
+#define VTBS_SIGNED                         0
+
+#define VAS_MASK                            0x0FFF0000U
+#define VAS_SHIFT                           16
+#define VAS_SIGNED                          0
+
+/*
+	Register PVR_TCF_RGBPDP_VSYNC3
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VSYNC3    0x0034
+#define VBBS_MASK                           0x00000FFFU
+#define VBBS_SHIFT                          0
+#define VBBS_SIGNED                         0
+
+#define VFPS_MASK                           0x0FFF0000U
+#define VFPS_SHIFT                          16
+#define VFPS_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_HDECTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_HDECTRL   0x0038
+#define HDEF_MASK                           0x00000FFFU
+#define HDEF_SHIFT                          0
+#define HDEF_SIGNED                         0
+
+#define HDES_MASK                           0x0FFF0000U
+#define HDES_SHIFT                          16
+#define HDES_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VDECTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VDECTRL   0x003C
+#define VDEF_MASK                           0x00000FFFU
+#define VDEF_SHIFT                          0
+#define VDEF_SIGNED                         0
+
+#define VDES_MASK                           0x0FFF0000U
+#define VDES_SHIFT                          16
+#define VDES_SIGNED                         0
+
+/*
+	Register PVR_TCF_RGBPDP_VEVENT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_VEVENT    0x0040
+#define VFETCH_MASK                         0x00000FFFU
+#define VFETCH_SHIFT                        0
+#define VFETCH_SIGNED                       0
+
+#define VEVENT_MASK                         0x0FFF0000U
+#define VEVENT_SHIFT                        16
+#define VEVENT_SIGNED                       0
+
+/*
+	Register PVR_TCF_RGBPDP_OPMASK
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_OPMASK    0x0044
+#define MASKR_MASK                          0x000000FFU
+#define MASKR_SHIFT                         0
+#define MASKR_SIGNED                        0
+
+#define MASKG_MASK                          0x0000FF00U
+#define MASKG_SHIFT                         8
+#define MASKG_SIGNED                        0
+
+#define MASKB_MASK                          0x00FF0000U
+#define MASKB_SHIFT                         16
+#define MASKB_SIGNED                        0
+
+#define BLANKLEVEL_MASK                     0x40000000U
+#define BLANKLEVEL_SHIFT                    30
+#define BLANKLEVEL_SIGNED                   0
+
+#define MASKLEVEL_MASK                      0x80000000U
+#define MASKLEVEL_SHIFT                     31
+#define MASKLEVEL_SIGNED                    0
+
+/*
+	Register PVR_TCF_RGBPDP_INTSTAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTSTAT   0x0048
+#define INTS_HBLNK0_MASK                    0x00000001U
+#define INTS_HBLNK0_SHIFT                   0
+#define INTS_HBLNK0_SIGNED                  0
+
+#define INTS_HBLNK1_MASK                    0x00000002U
+#define INTS_HBLNK1_SHIFT                   1
+#define INTS_HBLNK1_SIGNED                  0
+
+#define INTS_VBLNK0_MASK                    0x00000004U
+#define INTS_VBLNK0_SHIFT                   2
+#define INTS_VBLNK0_SIGNED                  0
+
+#define INTS_VBLNK1_MASK                    0x00000008U
+#define INTS_VBLNK1_SHIFT                   3
+#define INTS_VBLNK1_SIGNED                  0
+
+#define INTS_STR1URUN_MASK                  0x00000010U
+#define INTS_STR1URUN_SHIFT                 4
+#define INTS_STR1URUN_SIGNED                0
+
+#define INTS_STR1ORUN_MASK                  0x00000020U
+#define INTS_STR1ORUN_SHIFT                 5
+#define INTS_STR1ORUN_SIGNED                0
+
+#define INTS_DISPURUN_MASK                  0x00000040U
+#define INTS_DISPURUN_SHIFT                 6
+#define INTS_DISPURUN_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_INTENAB
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTENAB   0x004C
+#define INTEN_HBLNK0_MASK                   0x00000001U
+#define INTEN_HBLNK0_SHIFT                  0
+#define INTEN_HBLNK0_SIGNED                 0
+
+#define INTEN_HBLNK1_MASK                   0x00000002U
+#define INTEN_HBLNK1_SHIFT                  1
+#define INTEN_HBLNK1_SIGNED                 0
+
+#define INTEN_VBLNK0_MASK                   0x00000004U
+#define INTEN_VBLNK0_SHIFT                  2
+#define INTEN_VBLNK0_SIGNED                 0
+
+#define INTEN_VBLNK1_MASK                   0x00000008U
+#define INTEN_VBLNK1_SHIFT                  3
+#define INTEN_VBLNK1_SIGNED                 0
+
+#define INTEN_STR1URUN_MASK                 0x00000010U
+#define INTEN_STR1URUN_SHIFT                4
+#define INTEN_STR1URUN_SIGNED               0
+
+#define INTEN_STR1ORUN_MASK                 0x00000020U
+#define INTEN_STR1ORUN_SHIFT                5
+#define INTEN_STR1ORUN_SIGNED               0
+
+#define INTEN_DISPURUN_MASK                 0x00000040U
+#define INTEN_DISPURUN_SHIFT                6
+#define INTEN_DISPURUN_SIGNED               0
+
+/*
+	Register PVR_TCF_RGBPDP_INTCLEAR
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTCLEAR  0x0050
+#define INTCLR_HBLNK0_MASK                  0x00000001U
+#define INTCLR_HBLNK0_SHIFT                 0
+#define INTCLR_HBLNK0_SIGNED                0
+
+#define INTCLR_HBLNK1_MASK                  0x00000002U
+#define INTCLR_HBLNK1_SHIFT                 1
+#define INTCLR_HBLNK1_SIGNED                0
+
+#define INTCLR_VBLNK0_MASK                  0x00000004U
+#define INTCLR_VBLNK0_SHIFT                 2
+#define INTCLR_VBLNK0_SIGNED                0
+
+#define INTCLR_VBLNK1_MASK                  0x00000008U
+#define INTCLR_VBLNK1_SHIFT                 3
+#define INTCLR_VBLNK1_SIGNED                0
+
+#define INTCLR_STR1URUN_MASK                0x00000010U
+#define INTCLR_STR1URUN_SHIFT               4
+#define INTCLR_STR1URUN_SIGNED              0
+
+#define INTCLR_STR1ORUN_MASK                0x00000020U
+#define INTCLR_STR1ORUN_SHIFT               5
+#define INTCLR_STR1ORUN_SIGNED              0
+
+#define INTCLR_DISPURUN_MASK                0x00000040U
+#define INTCLR_DISPURUN_SHIFT               6
+#define INTCLR_DISPURUN_SIGNED              0
+
+/*
+	Register PVR_TCF_RGBPDP_INTCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_INTCTRL   0x0054
+#define HBLNK_LINENO_MASK                   0x00000FFFU
+#define HBLNK_LINENO_SHIFT                  0
+#define HBLNK_LINENO_SIGNED                 0
+
+#define HBLNK_LINE_MASK                     0x00010000U
+#define HBLNK_LINE_SHIFT                    16
+#define HBLNK_LINE_SIGNED                   0
+
+/*
+	Register PVR_TCF_RGBPDP_SIGNAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_SIGNAT    0x0058
+#define SIGNATURE_MASK                      0xFFFFFFFFU
+#define SIGNATURE_SHIFT                     0
+#define SIGNATURE_SIGNED                    0
+
+/*
+	Register PVR_TCF_RGBPDP_LINESTAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_LINESTAT  0x005C
+#define LINENO_MASK                         0x00000FFFU
+#define LINENO_SHIFT                        0
+#define LINENO_SIGNED                       0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGCTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGCTRL   0x0060
+#define DBG_ENAB_MASK                       0x00000001U
+#define DBG_ENAB_SHIFT                      0
+#define DBG_ENAB_SIGNED                     0
+
+#define DBG_READ_MASK                       0x00000002U
+#define DBG_READ_SHIFT                      1
+#define DBG_READ_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGDATA
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGDATA   0x0064
+#define DBG_DATA_MASK                       0x00FFFFFFU
+#define DBG_DATA_SHIFT                      0
+#define DBG_DATA_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_DBGSIDE
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_DBGSIDE   0x0068
+#define DBG_SIDE_MASK                       0x00000007U
+#define DBG_SIDE_SHIFT                      0
+#define DBG_SIDE_SIGNED                     0
+
+#define DBG_VAL_MASK                        0x00000008U
+#define DBG_VAL_SHIFT                       3
+#define DBG_VAL_SIGNED                      0
+
+/*
+	Register PVR_TCF_RGBPDP_REGLD_STAT
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_REGLD_STAT 0x0070
+#define REGLD_ADDROUT_MASK                  0x00FFFFFFU
+#define REGLD_ADDROUT_SHIFT                 0
+#define REGLD_ADDROUT_SIGNED                0
+
+#define REGLD_ADDREN_MASK                   0x80000000U
+#define REGLD_ADDREN_SHIFT                  31
+#define REGLD_ADDREN_SIGNED                 0
+
+/*
+	Register PVR_TCF_RGBPDP_REGLD_CTRL
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_REGLD_CTRL 0x0074
+#define REGLD_ADDRIN_MASK                   0x00FFFFFFU
+#define REGLD_ADDRIN_SHIFT                  0
+#define REGLD_ADDRIN_SIGNED                 0
+
+#define REGLD_VAL_MASK                      0x01000000U
+#define REGLD_VAL_SHIFT                     24
+#define REGLD_VAL_SIGNED                    0
+
+#define REGLD_ADDRLEN_MASK                  0xFE000000U
+#define REGLD_ADDRLEN_SHIFT                 25
+#define REGLD_ADDRLEN_SIGNED                0
+
+/*
+	Register PVR_TCF_RGBPDP_CORE_ID
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_CORE_ID   0x0078
+#define CONFIG_ID_MASK                      0x0000FFFFU
+#define CONFIG_ID_SHIFT                     0
+#define CONFIG_ID_SIGNED                    0
+
+#define CORE_ID_MASK                        0x00FF0000U
+#define CORE_ID_SHIFT                       16
+#define CORE_ID_SIGNED                      0
+
+#define GROUP_ID_MASK                       0xFF000000U
+#define GROUP_ID_SHIFT                      24
+#define GROUP_ID_SIGNED                     0
+
+/*
+	Register PVR_TCF_RGBPDP_CORE_REV
+*/
+#define TCF_RGBPDP_PVR_TCF_RGBPDP_CORE_REV  0x007C
+#define MAINT_REV_MASK                      0x000000FFU
+#define MAINT_REV_SHIFT                     0
+#define MAINT_REV_SIGNED                    0
+
+#define MINOR_REV_MASK                      0x0000FF00U
+#define MINOR_REV_SHIFT                     8
+#define MINOR_REV_SIGNED                    0
+
+#define MAJOR_REV_MASK                      0x00FF0000U
+#define MAJOR_REV_SHIFT                     16
+#define MAJOR_REV_SIGNED                    0
+
+#endif /* !defined(_TCF_RGBPDP_REGS_H_) */
+
+/*****************************************************************************
+ End of file (tcf_rgbpdp_regs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/cache_km.c b/drivers/gpu/drm/img-rogue/cache_km.c
new file mode 100644
index 000000000000..b7cac8f45209
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cache_km.c
@@ -0,0 +1,1638 @@
+/*************************************************************************/ /*!
+@File           cache_km.c
+@Title          CPU d-cache maintenance operations framework
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements server side code for CPU d-cache maintenance taking
+                into account the idiosyncrasies of the various types of CPU
+                d-cache instruction-set architecture (ISA) maintenance
+                mechanisms.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if defined(__linux__)
+#include <linux/version.h>
+#include <linux/uaccess.h>
+#include <asm/current.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/highmem.h>
+#endif
+
+#include "pmr.h"
+#include "log2.h"
+#include "device.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "cache_km.h"
+#include "pvr_debug.h"
+#include "lock_types.h"
+#include "allocmem.h"
+#include "process_stats.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+#include "ri_server.h"
+#endif
+#include "devicemem.h"
+#include "pvrsrv_apphint.h"
+#include "pvrsrv_sync_server.h"
+#include "km_apphint_defs.h"
+#include "km_apphint_defs_common.h"
+#include "os_apphint.h"
+#include "di_server.h"
+
+/* This header must always be included last */
+#if defined(__linux__)
+#include "kernel_compatibility.h"
+#endif
+
+/* Top-level file-local build definitions */
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS) && defined(__linux__)
+#define CACHEOP_DEBUG
+#define CACHEOP_STATS_ITEMS_MAX				32
+#define INCR_WRAP(x)						((x+1) >= CACHEOP_STATS_ITEMS_MAX ? 0 : (x+1))
+#define DECR_WRAP(x)						((x-1) < 0 ? (CACHEOP_STATS_ITEMS_MAX-1) : (x-1))
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+/* Refer to CacheOpStatsExecLogHeader() for header item names */
+#define CACHEOP_RI_PRINTF_HEADER			"%-8s %-8s %-10s %-10s %-5s %-16s %-16s %-10s %-10s %-18s"
+#define CACHEOP_RI_PRINTF					"%-8d %-8d %-10s %-10s %-5s 0x%-14llx 0x%-14llx 0x%-8llx 0x%-8llx %-18llu\n"
+#else
+#define CACHEOP_PRINTF_HEADER				"%-8s %-8s %-10s %-10s %-5s %-10s %-10s %-18s"
+#define CACHEOP_PRINTF						"%-8d %-8d %-10s %-10s %-5s 0x%-8llx 0x%-8llx %-18llu\n"
+#endif
+#endif
+
+//#define CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING		/* Force OS page (not cache line) flush granularity */
+#define CACHEOP_PVR_ASSERT(x)							/* Define as PVR_ASSERT(x), enable for swdev & testing */
+#define CACHEOP_DEVMEM_OOR_ERROR_STRING		"cacheop device memory request is out of range"
+#define CACHEOP_MAX_DEBUG_MESSAGE_LEN		160
+
+typedef struct _CACHEOP_WORK_ITEM_
+{
+	PMR *psPMR;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_CACHE_OP uiCacheOp;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	PVRSRV_TIMELINE iTimeline;
+	SYNC_TIMELINE_OBJ sSWTimelineObj;
+	PVRSRV_DEVICE_NODE *psDevNode;
+#if defined(CACHEOP_DEBUG)
+	IMG_UINT64 ui64StartTime;
+	IMG_UINT64 ui64EndTime;
+	IMG_BOOL bKMReq;
+	IMG_PID pid;
+#endif
+} CACHEOP_WORK_ITEM;
+
+typedef struct _CACHEOP_STATS_EXEC_ITEM_
+{
+	IMG_UINT32 ui32DeviceID;
+	IMG_PID pid;
+	PVRSRV_CACHE_OP uiCacheOp;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT64 ui64StartTime;
+	IMG_UINT64 ui64EndTime;
+	IMG_BOOL bKMReq;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+#endif
+} CACHEOP_STATS_EXEC_ITEM;
+
+typedef enum _CACHEOP_CONFIG_
+{
+	CACHEOP_CONFIG_DEFAULT = 0,
+	/* cache flush mechanism types */
+	CACHEOP_CONFIG_URBF    = 4,
+	/* sw-emulated deferred flush mechanism */
+	CACHEOP_CONFIG_KDF     = 8,
+	/* pseudo configuration items */
+	CACHEOP_CONFIG_LAST    = 16,
+	CACHEOP_CONFIG_KLOG    = 16,
+	CACHEOP_CONFIG_ALL     = 31
+} CACHEOP_CONFIG;
+
+typedef struct _CACHEOP_WORK_QUEUE_
+{
+/*
+ * Init. state & primary device node framework
+ * is anchored on.
+ */
+	IMG_BOOL bInit;
+/*
+  MMU page size/shift & d-cache line size
+ */
+	size_t uiPageSize;
+	IMG_UINT32 uiLineSize;
+	IMG_UINT32 uiLineShift;
+	IMG_UINT32 uiPageShift;
+	PMR *psInfoPagePMR;
+	IMG_UINT32 *pui32InfoPage;
+
+#if defined(CACHEOP_DEBUG)
+/*
+  CacheOp statistics
+ */
+	DI_ENTRY *psDIEntry;
+	IMG_HANDLE hStatsExecLock;
+
+	IMG_UINT32 ui32ServerOps;
+	IMG_UINT32 ui32ClientOps;
+	IMG_UINT32 ui32TotalOps;
+	IMG_UINT32 ui32ServerOpUsedUMVA;
+	IMG_UINT32 ui32AvgExecTime;
+	IMG_UINT32 ui32AvgExecTimeRemainder;
+
+	IMG_INT32 i32StatsExecWriteIdx;
+	CACHEOP_STATS_EXEC_ITEM asStatsExecuted[CACHEOP_STATS_ITEMS_MAX];
+#endif
+
+	DI_ENTRY *psConfigTune;
+	IMG_HANDLE hConfigLock;
+	CACHEOP_CONFIG	eConfig;
+	IMG_UINT32		ui32Config;
+	IMG_BOOL		bSupportsUMFlush;
+} CACHEOP_WORK_QUEUE;
+
+/* Top-level CacheOp framework object */
+static CACHEOP_WORK_QUEUE gsCwq;
+
+#define CacheOpConfigSupports(e) ((gsCwq.eConfig & (e)) ? IMG_TRUE : IMG_FALSE)
+
+#if defined(CACHEOP_DEBUG)
+static INLINE void CacheOpStatsExecLogHeader(IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(szBuffer, CACHEOP_MAX_DEBUG_MESSAGE_LEN,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+				CACHEOP_RI_PRINTF_HEADER,
+#else
+				CACHEOP_PRINTF_HEADER,
+#endif
+				"DevID",
+				"Pid",
+				"CacheOp",
+				"Type",
+				"Origin",
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+				"DevVAddr",
+				"DevPAddr",
+#endif
+				"Offset",
+				"Size",
+				"xTime (us)");
+}
+
+static void CacheOpStatsExecLogWrite(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
+{
+	IMG_INT32 i32WriteOffset;
+	IMG_UINT32 ui32ExecTime;
+	printk("log write\n");
+	if (!psCacheOpWorkItem->uiCacheOp)
+	{
+		return;
+	}
+	else if (psCacheOpWorkItem->bKMReq && !CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
+	{
+		/* KM logs spams the history due to frequency, this removes it completely */
+		return;
+	}
+
+	OSLockAcquire(gsCwq.hStatsExecLock);
+
+	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
+	gsCwq.i32StatsExecWriteIdx = INCR_WRAP(gsCwq.i32StatsExecWriteIdx);
+	gsCwq.asStatsExecuted[i32WriteOffset].ui32DeviceID = psCacheOpWorkItem->psDevNode ? psCacheOpWorkItem->psDevNode->sDevId.ui32InternalID : -1;
+	gsCwq.asStatsExecuted[i32WriteOffset].pid = psCacheOpWorkItem->pid;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiSize = psCacheOpWorkItem->uiSize;
+	gsCwq.asStatsExecuted[i32WriteOffset].bKMReq = psCacheOpWorkItem->bKMReq;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiOffset	= psCacheOpWorkItem->uiOffset;
+	gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp = psCacheOpWorkItem->uiCacheOp;
+	gsCwq.asStatsExecuted[i32WriteOffset].ui64StartTime = psCacheOpWorkItem->ui64StartTime;
+	gsCwq.asStatsExecuted[i32WriteOffset].ui64EndTime = psCacheOpWorkItem->ui64EndTime;
+
+	CACHEOP_PVR_ASSERT(gsCwq.asStatsExecuted[i32WriteOffset].pid);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	if (gsCwq.bInit && psCacheOpWorkItem->psPMR)
+	{
+		IMG_CPU_PHYADDR sDevPAddr;
+		PVRSRV_ERROR eError, eLockError;
+		IMG_BOOL bValid;
+
+		/* Get more detailed information regarding the sub allocations that
+		   PMR has from RI manager for process that requested the CacheOp */
+		eError = RIDumpProcessListKM(psCacheOpWorkItem->psPMR,
+									 gsCwq.asStatsExecuted[i32WriteOffset].pid,
+									 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+									 &gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* (Re)lock here as some PMR might have not been locked */
+		eLockError = PMRLockSysPhysAddresses(psCacheOpWorkItem->psPMR);
+		PVR_GOTO_IF_ERROR(eLockError, e0);
+
+		eError = PMR_CpuPhysAddr(psCacheOpWorkItem->psPMR,
+								 gsCwq.uiPageShift,
+								 1,
+								 gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+								 &sDevPAddr,
+								 &bValid);
+
+		eLockError = PMRUnlockSysPhysAddresses(psCacheOpWorkItem->psPMR);
+		PVR_LOG_IF_ERROR(eLockError, "PMRUnlockSysPhysAddresses");
+
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+
+
+		gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr.uiAddr = sDevPAddr.uiAddr;
+	}
+#endif
+
+	/* Calculate the approximate cumulative moving average execution time.
+	 * This calculation is based on standard equation:
+	 *
+	 * CMAnext = (new + count * CMAprev) / (count + 1)
+	 *
+	 * but in simplified form:
+	 *
+	 * CMAnext = CMAprev + (new - CMAprev) / (count + 1)
+	 *
+	 * this gets rid of multiplication and prevents overflow.
+	 *
+	 * Also to increase accuracy that we lose with integer division,
+	 * we hold the moving remainder of the division and add it.
+	 *
+	 * CMAnext = CMAprev + (new - CMAprev + CMRprev) / (count + 1)
+	 *
+	 * Multiple tests proved it to be the best solution for approximating
+	 * CMA using integers.
+	 *
+	 */
+
+	ui32ExecTime =
+		gsCwq.asStatsExecuted[i32WriteOffset].ui64EndTime -
+		gsCwq.asStatsExecuted[i32WriteOffset].ui64StartTime;
+
+	{
+
+	IMG_INT32 i32Div =
+		(IMG_INT32) ui32ExecTime -
+		(IMG_INT32) gsCwq.ui32AvgExecTime +
+		(IMG_INT32) gsCwq.ui32AvgExecTimeRemainder;
+
+	gsCwq.ui32AvgExecTime += i32Div / (IMG_INT32)(gsCwq.ui32TotalOps + 1);
+	gsCwq.ui32AvgExecTimeRemainder = i32Div % (IMG_INT32)(gsCwq.ui32TotalOps + 1);
+
+	gsCwq.ui32TotalOps++;
+
+	}
+
+	if (!gsCwq.asStatsExecuted[i32WriteOffset].bKMReq)
+	{
+		/* This operation queues only UM CacheOp in per-PID process statistics database */
+		PVRSRVStatsUpdateCacheOpStats(
+						gsCwq.asStatsExecuted[i32WriteOffset].uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						gsCwq.asStatsExecuted[i32WriteOffset].sDevVAddr,
+						gsCwq.asStatsExecuted[i32WriteOffset].sDevPAddr,
+#endif
+						gsCwq.asStatsExecuted[i32WriteOffset].uiOffset,
+						gsCwq.asStatsExecuted[i32WriteOffset].uiSize,
+						ui32ExecTime,
+						!gsCwq.asStatsExecuted[i32WriteOffset].bKMReq,
+						psCacheOpWorkItem->pid);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+e0:
+#endif
+	OSLockRelease(gsCwq.hStatsExecLock);
+}
+
+static int CacheOpStatsExecLogRead(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	IMG_CHAR *pszFlushType;
+	IMG_CHAR *pszCacheOpType;
+	IMG_CHAR *pszFlushSource;
+	IMG_INT32 i32ReadOffset;
+	IMG_INT32 i32WriteOffset;
+
+	IMG_CHAR szBuffer[CACHEOP_MAX_DEBUG_MESSAGE_LEN] = {0};
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	OSLockAcquire(gsCwq.hStatsExecLock);
+
+	DIPrintf(psEntry,
+			"Primary CPU d-cache architecture: LSZ: 0x%x, URBF: %s\n",
+			gsCwq.uiLineSize,
+			gsCwq.bSupportsUMFlush ? "Yes" : "No");
+
+	DIPrintf(psEntry,
+			"Configuration: UKT: %d, URBF: %s\n",
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD],
+			gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No");
+
+	DIPrintf(psEntry,
+			"Summary: Total Ops [%d] - Server(using UMVA)/Client [%d(%d)/%d]. Avg execution time [%d]\n",
+			gsCwq.ui32TotalOps, gsCwq.ui32ServerOps, gsCwq.ui32ServerOpUsedUMVA, gsCwq.ui32ClientOps, gsCwq.ui32AvgExecTime);
+
+
+	CacheOpStatsExecLogHeader(szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	i32WriteOffset = gsCwq.i32StatsExecWriteIdx;
+	for (i32ReadOffset = DECR_WRAP(i32WriteOffset);
+		 i32ReadOffset != i32WriteOffset;
+		 i32ReadOffset = DECR_WRAP(i32ReadOffset))
+	{
+		IMG_UINT64 ui64ExecTime =
+			gsCwq.asStatsExecuted[i32ReadOffset].ui64EndTime -
+			gsCwq.asStatsExecuted[i32ReadOffset].ui64StartTime;
+
+		IMG_DEVMEM_SIZE_T ui64NumOfPages =
+			gsCwq.asStatsExecuted[i32ReadOffset].uiSize >> gsCwq.uiPageShift;
+
+
+		if (!gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
+		{
+			break;
+		}
+		if (ui64NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			pszFlushType = "RBF.Fast";
+		}
+		else
+		{
+			pszFlushType = "RBF.Slow";
+		}
+
+		pszFlushSource = gsCwq.asStatsExecuted[i32ReadOffset].bKMReq ? " KM" : " UM";
+
+		switch (gsCwq.asStatsExecuted[i32ReadOffset].uiCacheOp)
+		{
+			case PVRSRV_CACHE_OP_NONE:
+				pszCacheOpType = "None";
+				break;
+			case PVRSRV_CACHE_OP_CLEAN:
+				pszCacheOpType = "Clean";
+				break;
+			case PVRSRV_CACHE_OP_INVALIDATE:
+				pszCacheOpType = "Invalidate";
+				break;
+			case PVRSRV_CACHE_OP_FLUSH:
+				pszCacheOpType = "Flush";
+				break;
+			case PVRSRV_CACHE_OP_TIMELINE:
+				pszCacheOpType = "Timeline";
+				pszFlushType = "      ";
+				break;
+			default:
+				pszCacheOpType = "Unknown";
+				break;
+		}
+
+		DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						CACHEOP_RI_PRINTF,
+#else
+						CACHEOP_PRINTF,
+#endif
+						gsCwq.asStatsExecuted[i32ReadOffset].ui32DeviceID,
+						gsCwq.asStatsExecuted[i32ReadOffset].pid,
+						pszCacheOpType,
+						pszFlushType,
+						pszFlushSource,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+						gsCwq.asStatsExecuted[i32ReadOffset].sDevVAddr.uiAddr,
+						gsCwq.asStatsExecuted[i32ReadOffset].sDevPAddr.uiAddr,
+#endif
+						gsCwq.asStatsExecuted[i32ReadOffset].uiOffset,
+						gsCwq.asStatsExecuted[i32ReadOffset].uiSize,
+						ui64ExecTime);
+
+	}
+
+	OSLockRelease(gsCwq.hStatsExecLock);
+
+	return 0;
+}
+#endif /* defined(CACHEOP_DEBUG) */
+
+static INLINE void CacheOpStatsReset(void)
+{
+#if defined(CACHEOP_DEBUG)
+	gsCwq.ui32ServerOps = 0;
+	gsCwq.ui32ClientOps = 0;
+	gsCwq.ui32TotalOps = 0;
+	gsCwq.ui32ServerOpUsedUMVA = 0;
+	gsCwq.ui32AvgExecTime = 0;
+	gsCwq.ui32AvgExecTimeRemainder = 0;
+
+	gsCwq.i32StatsExecWriteIdx = 0;
+
+	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));
+#endif
+}
+
+static void CacheOpConfigUpdate(IMG_UINT32 ui32Config)
+{
+	OSLockAcquire(gsCwq.hConfigLock);
+
+	/* Step 0, set the gsCwq.eConfig bits */
+	if (!(ui32Config & (CACHEOP_CONFIG_LAST - 1)))
+	{
+		gsCwq.eConfig = CACHEOP_CONFIG_KDF;
+		if (gsCwq.bSupportsUMFlush)
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
+		}
+	}
+	else
+	{
+		if (ui32Config & CACHEOP_CONFIG_KDF)
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_KDF;
+		}
+		else
+		{
+			gsCwq.eConfig &= ~CACHEOP_CONFIG_KDF;
+		}
+
+		if (gsCwq.bSupportsUMFlush && (ui32Config & CACHEOP_CONFIG_URBF))
+		{
+			gsCwq.eConfig |= CACHEOP_CONFIG_URBF;
+		}
+		else
+		{
+			gsCwq.eConfig &= ~CACHEOP_CONFIG_URBF;
+		}
+	}
+
+	if (ui32Config & CACHEOP_CONFIG_KLOG)
+	{
+		/* Suppress logs from KM caller */
+		gsCwq.eConfig |= CACHEOP_CONFIG_KLOG;
+	}
+	else
+	{
+		gsCwq.eConfig &= ~CACHEOP_CONFIG_KLOG;
+	}
+
+	/* Step 1, set gsCwq.ui32Config based on gsCwq.eConfig */
+	ui32Config = 0;
+
+	if (gsCwq.eConfig & CACHEOP_CONFIG_KDF)
+	{
+		ui32Config |= CACHEOP_CONFIG_KDF;
+	}
+	if (gsCwq.eConfig & CACHEOP_CONFIG_URBF)
+	{
+		ui32Config |= CACHEOP_CONFIG_URBF;
+	}
+	if (gsCwq.eConfig & CACHEOP_CONFIG_KLOG)
+	{
+		ui32Config |= CACHEOP_CONFIG_KLOG;
+	}
+	gsCwq.ui32Config = ui32Config;
+
+
+	/* Step 3, in certain cases where a CacheOp/VA is provided, this threshold determines at what point
+	   the optimisation due to the presence of said VA (i.e. us not having to remap the PMR pages in KM)
+	   is clawed-back because of the overhead of maintaining such large request which might stalls the
+	   user thread; so to hide this latency have these CacheOps executed on deferred CacheOp thread */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_KMDFTHRESHLD] = (IMG_UINT32)(PVR_DIRTY_BYTES_FLUSH_THRESHOLD >> 2);
+
+	/* Step 4, if no UM support, all requests are done in KM so zero these forcing all client requests
+	   to come down into the KM for maintenance */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = 0;
+
+	if (gsCwq.bSupportsUMFlush)
+	{
+		/* With URBF enabled we never go to the kernel */
+		if (gsCwq.eConfig & CACHEOP_CONFIG_URBF)
+		{
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = (IMG_UINT32)~0;
+		}
+	}
+
+	/* Step 5, reset stats. */
+	CacheOpStatsReset();
+
+	OSLockRelease(gsCwq.hConfigLock);
+}
+
+static int CacheOpConfigRead(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	DIPrintf(psEntry, "URBF: %s\n",
+		gsCwq.eConfig & CACHEOP_CONFIG_URBF ? "Yes" : "No");
+
+	return 0;
+}
+
+static INLINE PVRSRV_ERROR CacheOpConfigQuery(const PVRSRV_DEVICE_NODE *psDevNode,
+											const void *psPrivate,
+											IMG_UINT32 *pui32Value)
+{
+	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	switch (ui32ID)
+	{
+		case APPHINT_ID_CacheOpConfig:
+			*pui32Value = gsCwq.ui32Config;
+			break;
+
+		case APPHINT_ID_CacheOpUMKMThresholdSize:
+			*pui32Value = gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD];
+			break;
+
+		default:
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR CacheOpConfigSet(const PVRSRV_DEVICE_NODE *psDevNode,
+											const void *psPrivate,
+											IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 ui32ID = (IMG_UINT32)(uintptr_t) psPrivate;
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	switch (ui32ID)
+	{
+		case APPHINT_ID_CacheOpConfig:
+			CacheOpConfigUpdate(ui32Value & CACHEOP_CONFIG_ALL);
+			break;
+
+
+		case APPHINT_ID_CacheOpUMKMThresholdSize:
+		{
+			if (!ui32Value || !gsCwq.bSupportsUMFlush)
+			{
+				/* CPU ISA does not support UM flush, therefore every request goes down into
+				   the KM, silently ignore request to adjust threshold */
+				PVR_ASSERT(! gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD]);
+				break;
+			}
+			else if (ui32Value < gsCwq.uiPageSize)
+			{
+				/* Silently round-up to OS page size */
+				ui32Value = gsCwq.uiPageSize;
+			}
+
+			/* Align to OS page size */
+			ui32Value &= ~(gsCwq.uiPageSize - 1);
+
+			gsCwq.pui32InfoPage[CACHEOP_INFO_UMKMTHRESHLD] = ui32Value;
+
+			break;
+		}
+
+		default:
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR CacheOpTimelineBind(PVRSRV_DEVICE_NODE *psDevNode,
+											   CACHEOP_WORK_ITEM *psCacheOpWorkItem,
+											   PVRSRV_TIMELINE iTimeline)
+{
+	PVRSRV_ERROR eError;
+
+	/* Always default the incoming CacheOp work-item to safe values */
+	SyncClearTimelineObj(&psCacheOpWorkItem->sSWTimelineObj);
+	psCacheOpWorkItem->iTimeline = PVRSRV_NO_TIMELINE;
+	psCacheOpWorkItem->psDevNode = psDevNode;
+	if (iTimeline == PVRSRV_NO_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+
+	psCacheOpWorkItem->iTimeline = iTimeline;
+	eError = SyncSWGetTimelineObj(iTimeline, &psCacheOpWorkItem->sSWTimelineObj);
+	PVR_LOG_IF_ERROR(eError, "SyncSWGetTimelineObj");
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR CacheOpTimelineExec(CACHEOP_WORK_ITEM *psCacheOpWorkItem)
+{
+	PVRSRV_ERROR eError;
+
+	if (psCacheOpWorkItem->iTimeline == PVRSRV_NO_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+	CACHEOP_PVR_ASSERT(psCacheOpWorkItem->sSWTimelineObj.pvTlObj);
+
+	eError = SyncSWTimelineAdvanceKM(psCacheOpWorkItem->psDevNode,
+	                                 &psCacheOpWorkItem->sSWTimelineObj);
+	(void) SyncSWTimelineReleaseKM(&psCacheOpWorkItem->sSWTimelineObj);
+
+	return eError;
+}
+
+static INLINE void CacheOpExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
+										PVRSRV_CACHE_OP uiCacheOp,
+										IMG_BYTE *pbCpuVirtAddr,
+										IMG_CPU_PHYADDR sCpuPhyAddr,
+										IMG_DEVMEM_OFFSET_T uiPgAlignedOffset,
+										IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset,
+										IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset)
+{
+	IMG_BYTE *pbCpuVirtAddrEnd;
+	IMG_BYTE *pbCpuVirtAddrStart;
+	IMG_CPU_PHYADDR sCpuPhyAddrEnd;
+	IMG_CPU_PHYADDR sCpuPhyAddrStart;
+	IMG_DEVMEM_SIZE_T uiRelFlushSize;
+	IMG_DEVMEM_OFFSET_T uiRelFlushOffset;
+	IMG_DEVMEM_SIZE_T uiNextPgAlignedOffset;
+
+	/* These quantities allows us to perform cache operations
+	   at cache-line granularity thereby ensuring we do not
+	   perform more than is necessary */
+	CACHEOP_PVR_ASSERT(uiPgAlignedOffset < uiCLAlignedEndOffset);
+	uiRelFlushSize = (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
+	uiRelFlushOffset = 0;
+
+	if (uiCLAlignedStartOffset > uiPgAlignedOffset)
+	{
+		/* Zero unless initially starting at an in-page offset */
+		uiRelFlushOffset = uiCLAlignedStartOffset - uiPgAlignedOffset;
+		uiRelFlushSize -= uiRelFlushOffset;
+	}
+
+	/* uiRelFlushSize is gsCwq.uiPageSize unless current outstanding CacheOp
+	   size is smaller. The 1st case handles in-page CacheOp range and
+	   the 2nd case handles multiple-page CacheOp range with a last
+	   CacheOp size that is less than gsCwq.uiPageSize */
+	uiNextPgAlignedOffset = uiPgAlignedOffset + (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize;
+	if (uiNextPgAlignedOffset < uiPgAlignedOffset)
+	{
+		/* uiNextPgAlignedOffset is greater than uiCLAlignedEndOffset
+		   by implication of this wrap-round; this only happens when
+		   uiPgAlignedOffset is the last page aligned offset */
+		uiRelFlushSize = uiRelFlushOffset ?
+				uiCLAlignedEndOffset - uiCLAlignedStartOffset :
+				uiCLAlignedEndOffset - uiPgAlignedOffset;
+	}
+	else
+	{
+		if (uiNextPgAlignedOffset > uiCLAlignedEndOffset)
+		{
+			uiRelFlushSize = uiRelFlushOffset ?
+					uiCLAlignedEndOffset - uiCLAlignedStartOffset :
+					uiCLAlignedEndOffset - uiPgAlignedOffset;
+		}
+	}
+
+	/* More efficient to request cache maintenance operation for full
+	   relative range as opposed to multiple cache-aligned ranges */
+	sCpuPhyAddrStart.uiAddr = sCpuPhyAddr.uiAddr + uiRelFlushOffset;
+	sCpuPhyAddrEnd.uiAddr = sCpuPhyAddrStart.uiAddr + uiRelFlushSize;
+	if (pbCpuVirtAddr)
+	{
+		pbCpuVirtAddrStart = pbCpuVirtAddr + uiRelFlushOffset;
+		pbCpuVirtAddrEnd = pbCpuVirtAddrStart + uiRelFlushSize;
+	}
+	else
+	{
+		/* Some OS/Env layer support functions expect NULL(s) */
+		pbCpuVirtAddrStart = NULL;
+		pbCpuVirtAddrEnd = NULL;
+	}
+
+	/* Perform requested CacheOp on the CPU data cache for successive cache
+	   line worth of bytes up to page or in-page cache-line boundary */
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pbCpuVirtAddrStart, pbCpuVirtAddrEnd,
+									sCpuPhyAddrStart, sCpuPhyAddrEnd);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					__func__, uiCacheOp));
+			break;
+	}
+
+}
+
+static INLINE void CacheOpExecRangeBasedVA(PVRSRV_DEVICE_NODE *psDevNode,
+										 IMG_CPU_VIRTADDR pvAddress,
+										 IMG_DEVMEM_SIZE_T uiSize,
+										 PVRSRV_CACHE_OP uiCacheOp)
+{
+	IMG_CPU_PHYADDR sCpuPhyAddrUnused =
+		{ IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) };
+	IMG_BYTE *pbEnd = (IMG_BYTE*)((uintptr_t)pvAddress + (uintptr_t)uiSize);
+	IMG_BYTE *pbStart = (IMG_BYTE*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiLineSize-1));
+
+	/*
+	  If the start/end address isn't aligned to cache line size, round it up to the
+	  nearest multiple; this ensures that we flush all the cache lines affected by
+	  unaligned start/end addresses.
+	 */
+	pbEnd = (IMG_BYTE *) PVR_ALIGN((uintptr_t)pbEnd, (uintptr_t)gsCwq.uiLineSize);
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pbStart, pbEnd, sCpuPhyAddrUnused, sCpuPhyAddrUnused);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					 __func__, uiCacheOp));
+			break;
+	}
+
+}
+
+static INLINE PVRSRV_ERROR CacheOpValidateUMVA(PMR *psPMR,
+											   IMG_CPU_VIRTADDR pvAddress,
+											   IMG_DEVMEM_OFFSET_T uiOffset,
+											   IMG_DEVMEM_SIZE_T uiSize,
+											   PVRSRV_CACHE_OP uiCacheOp,
+											   void **ppvOutAddress)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(__linux__) && !defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *vma;
+#endif
+	void __user *pvAddr;
+
+	IMG_BOOL bReadOnlyInvalidate =
+		(uiCacheOp == PVRSRV_CACHE_OP_INVALIDATE) &&
+		!PVRSRV_CHECK_CPU_WRITEABLE(PMR_Flags(psPMR));
+
+	if (!pvAddress || bReadOnlyInvalidate)
+	{
+		/* As pvAddress is optional, NULL is expected from UM/KM requests */
+		/* Also don't allow invalidates for UMVA of read-only memory */
+		pvAddr = NULL;
+		goto e0;
+	}
+
+
+
+#if !defined(__linux__) || defined(CACHEFLUSH_NO_KMRBF_USING_UMVA)
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	pvAddr = NULL;
+#else
+	/* Validate VA, assume most basic address limit access_ok() check */
+	pvAddr = (void __user *)(uintptr_t)((uintptr_t)pvAddress + uiOffset);
+	if (!access_ok(pvAddr, uiSize))
+	{
+		pvAddr = NULL;
+		if (! mm)
+		{
+			/* Bad KM request, don't silently ignore */
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_CPU_ADDR, e0);
+		}
+	}
+	else if (mm)
+	{
+		mmap_read_lock(mm);
+		vma = find_vma(mm, (unsigned long)(uintptr_t)pvAddr);
+
+		if (!vma ||
+			vma->vm_start > (unsigned long)(uintptr_t)pvAddr ||
+			vma->vm_end < (unsigned long)(uintptr_t)pvAddr + uiSize ||
+			vma->vm_private_data != psPMR)
+		{
+			/*
+			 * Request range is not fully mapped or is not matching the PMR
+			 * Ignore request's VA.
+			 */
+			pvAddr = NULL;
+		}
+		mmap_read_unlock(mm);
+	}
+#endif
+
+e0:
+	*ppvOutAddress = (IMG_CPU_VIRTADDR __force) pvAddr;
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpPMRExec (PMR *psPMR,
+									IMG_CPU_VIRTADDR pvAddress,
+									IMG_DEVMEM_OFFSET_T uiOffset,
+									IMG_DEVMEM_SIZE_T uiSize,
+									PVRSRV_CACHE_OP uiCacheOp,
+									IMG_BOOL bIsRequestValidated)
+
+{
+	IMG_HANDLE hPrivOut = NULL;
+	IMG_BOOL bPMRIsSparse;
+	IMG_UINT32 ui32PageIndex;
+	IMG_UINT32 ui32NumOfPages;
+	size_t uiOutSize;	/* Effectively unused */
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_DEVMEM_SIZE_T uiPgAlignedSize;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedOffset;
+	IMG_DEVMEM_OFFSET_T uiCLAlignedEndOffset;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedEndOffset;
+	IMG_DEVMEM_OFFSET_T uiCLAlignedStartOffset;
+	IMG_DEVMEM_OFFSET_T uiPgAlignedStartOffset;
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_CPU_PHYADDR asCpuPhyAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_CPU_PHYADDR *psCpuPhyAddr = asCpuPhyAddr;
+	IMG_BOOL bIsPMRInfoValid = IMG_FALSE;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BYTE *pbCpuVirtAddr = NULL;
+	IMG_BOOL *pbValid = abValid;
+	OS_CACHE_OP_ADDR_TYPE eCacheOpAddrType;
+
+	psDevNode = PMR_DeviceNode(psPMR);
+	eCacheOpAddrType = OSCPUCacheOpAddressType(psDevNode);
+
+	if (uiCacheOp == PVRSRV_CACHE_OP_NONE || uiCacheOp == PVRSRV_CACHE_OP_TIMELINE)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (! bIsRequestValidated)
+	{
+		/* Need to validate parameters before proceeding */
+		/* Check for size + offset overflow */
+		PVR_LOG_RETURN_IF_FALSE(((uiOffset + uiSize) >= uiSize),
+		                        "Overflow detected on offset + size parameters",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		/* Since size + offset is later aligned to page size check for overflow with alignment */
+		PVR_LOG_RETURN_IF_FALSE((((uiOffset + uiSize) + gsCwq.uiPageSize - 1) >= (uiOffset + uiSize)),
+		                        "Overflow detected on offset + size parameters with applied alignment",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		PVR_LOG_RETURN_IF_FALSE(((uiOffset+uiSize) <= PMR_PhysicalSize(psPMR)),
+		                        CACHEOP_DEVMEM_OOR_ERROR_STRING,
+		                        PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PMRLockSysPhysAddresses");
+	}
+
+	/* Fast track the request if a CPU VA is provided and CPU ISA supports VA only maintenance */
+	eError = CacheOpValidateUMVA(psPMR, pvAddress, uiOffset, uiSize, uiCacheOp, (void**)&pbCpuVirtAddr);
+	if (eError == PVRSRV_OK)
+	{
+		pvAddress = pbCpuVirtAddr;
+
+		if (pvAddress && eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pvAddress, uiSize, uiCacheOp);
+
+			if (!bIsRequestValidated)
+			{
+				eError = PMRUnlockSysPhysAddresses(psPMR);
+				PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+			}
+#if defined(CACHEOP_DEBUG)
+			gsCwq.ui32ServerOpUsedUMVA += 1;
+#endif
+			return PVRSRV_OK;
+		}
+		else if (pvAddress)
+		{
+			/* Round down the incoming VA (if any) down to the nearest page aligned VA */
+			pvAddress = (void*)((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1));
+#if defined(CACHEOP_DEBUG)
+			gsCwq.ui32ServerOpUsedUMVA += 1;
+#endif
+		}
+	}
+	else
+	{
+		/*
+		 * This validation pathway has been added to accommodate any/all requests that might
+		 * cause the kernel to Oops; essentially, KM requests should prevalidate cache maint.
+		 * parameters but if this fails then we would rather fail gracefully than cause the
+		 * kernel to Oops so instead we log the fact that an invalid KM virtual address was
+		 * supplied and what action was taken to mitigate against kernel Oops(ing) if any.
+		 */
+		CACHEOP_PVR_ASSERT(pbCpuVirtAddr == NULL);
+
+		if (eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, using paddress",
+					__func__,
+					pvAddress));
+
+			/* We can still proceed as kernel/cpu uses CPU PA for d-cache maintenance */
+			pvAddress = NULL;
+		}
+		else
+		{
+			/*
+			 * The approach here is to attempt a reacquisition of the PMR kernel VA and see if
+			 * said VA corresponds to the parameter VA, if so fail requested cache maint. op.
+			 * cause this indicates some kind of internal, memory and/or meta-data corruption
+			 * else we reissue the request using this (re)acquired alias PMR kernel VA.
+			 */
+			if (PMR_IsSparse(psPMR))
+			{
+				eError = PMRAcquireSparseKernelMappingData(psPMR,
+														   0,
+														   gsCwq.uiPageSize,
+														   (void **)&pbCpuVirtAddr,
+														   &uiOutSize,
+														   &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", e0);
+			}
+			else
+			{
+				eError = PMRAcquireKernelMappingData(psPMR,
+													 0,
+													 gsCwq.uiPageSize,
+													 (void **)&pbCpuVirtAddr,
+													 &uiOutSize,
+													 &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", e0);
+			}
+
+			/* Here, we only compare these CPU virtual addresses at granularity of the OS page size */
+			if ((uintptr_t)pbCpuVirtAddr == ((uintptr_t)pvAddress & ~((uintptr_t)gsCwq.uiPageSize-1)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid vaddress 0x%p in CPU d-cache maint. op, no alt. so failing request",
+						__func__,
+						pvAddress));
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_GOTO_WITH_ERROR("PMRReleaseKernelMappingData", eError, PVRSRV_ERROR_INVALID_CPU_ADDR, e0);
+			}
+			else if (eCacheOpAddrType == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s: Bad vaddress 0x%p in CPU d-cache maint. op, using reacquired vaddress 0x%p",
+						__func__,
+						pvAddress,
+						pbCpuVirtAddr));
+
+				/* Note that this might still fail if there is kernel memory/meta-data corruption;
+				   there is not much we can do here but at the least we will be informed of this
+				   before the kernel Oops(ing) */
+				CacheOpExecRangeBasedVA(PMR_DeviceNode(psPMR), pbCpuVirtAddr, uiSize, uiCacheOp);
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+				eError = PVRSRV_OK;
+				goto e0;
+			}
+			else
+			{
+				/* At this junction, we have exhausted every possible work-around possible but we do
+				   know that VA reacquisition returned another/alias page-aligned VA; so with this
+				   future expectation of PMRAcquireKernelMappingData(), we proceed */
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s: Bad vaddress %p in CPU d-cache maint. op, will use reacquired vaddress",
+						__func__,
+						pvAddress));
+
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+				/* NULL this to force per-page reacquisition down-stream */
+				pvAddress = NULL;
+			}
+		}
+	}
+
+	/* NULL clobbered var., OK to proceed */
+	pbCpuVirtAddr = NULL;
+	eError = PVRSRV_OK;
+
+	/* Need this for kernel mapping */
+	bPMRIsSparse = PMR_IsSparse(psPMR);
+
+	/* Round the incoming offset down to the nearest cache-line / page aligned-address */
+	uiCLAlignedEndOffset = uiOffset + uiSize;
+	uiCLAlignedEndOffset = PVR_ALIGN(uiCLAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiLineSize);
+	uiCLAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiLineSize-1));
+
+	uiPgAlignedEndOffset = uiCLAlignedEndOffset;
+	uiPgAlignedEndOffset = PVR_ALIGN(uiPgAlignedEndOffset, (IMG_DEVMEM_SIZE_T)gsCwq.uiPageSize);
+	uiPgAlignedStartOffset = (uiOffset & ~((IMG_DEVMEM_OFFSET_T)gsCwq.uiPageSize-1));
+	uiPgAlignedSize = uiPgAlignedEndOffset - uiPgAlignedStartOffset;
+
+#if defined(CACHEOP_NO_CACHE_LINE_ALIGNED_ROUNDING)
+	/* For internal debug if cache-line optimised
+	   flushing is suspected of causing data corruption */
+	uiCLAlignedStartOffset = uiPgAlignedStartOffset;
+	uiCLAlignedEndOffset = uiPgAlignedEndOffset;
+#endif
+
+	/* Type of allocation backing the PMR data */
+	ui32NumOfPages = uiPgAlignedSize >> gsCwq.uiPageShift;
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		/* The pbValid array is allocated first as it is needed in
+		   both physical/virtual cache maintenance methods */
+		pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+		if (! pbValid)
+		{
+			pbValid = abValid;
+		}
+		else if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			psCpuPhyAddr = OSAllocZMem(ui32NumOfPages * sizeof(IMG_CPU_PHYADDR));
+			if (! psCpuPhyAddr)
+			{
+				psCpuPhyAddr = asCpuPhyAddr;
+				OSFreeMem(pbValid);
+				pbValid = abValid;
+			}
+		}
+	}
+
+	/* We always retrieve PMR data in bulk, up-front if number of pages is within
+	   PMR_MAX_TRANSLATION_STACK_ALLOC limits else we check to ensure that a
+	   dynamic buffer has been allocated to satisfy requests outside limits */
+	if (ui32NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC || pbValid != abValid)
+	{
+		if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			/* Look-up PMR CpuPhyAddr once, if possible */
+			eError = PMR_CpuPhysAddr(psPMR,
+									 gsCwq.uiPageShift,
+									 ui32NumOfPages,
+									 uiPgAlignedStartOffset,
+									 psCpuPhyAddr,
+									 pbValid);
+			if (eError == PVRSRV_OK)
+			{
+				bIsPMRInfoValid = IMG_TRUE;
+			}
+		}
+		else
+		{
+			/* Look-up PMR per-page validity once, if possible */
+			eError = PMR_IsOffsetValid(psPMR,
+									   gsCwq.uiPageShift,
+									   ui32NumOfPages,
+									   uiPgAlignedStartOffset,
+									   pbValid);
+			bIsPMRInfoValid = (eError == PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
+		}
+	}
+
+	/* For each (possibly non-contiguous) PMR page(s), carry out the requested cache maint. op. */
+	for (uiPgAlignedOffset = uiPgAlignedStartOffset, ui32PageIndex = 0;
+		 uiPgAlignedOffset < uiPgAlignedEndOffset;
+		 uiPgAlignedOffset += (IMG_DEVMEM_OFFSET_T) gsCwq.uiPageSize, ui32PageIndex += 1)
+	{
+
+		if (! bIsPMRInfoValid)
+		{
+			/* Never cross page boundary without looking up corresponding PMR page physical
+			   address and/or page validity if these were not looked-up, in bulk, up-front */
+			ui32PageIndex = 0;
+			if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+			{
+				eError = PMR_CpuPhysAddr(psPMR,
+										 gsCwq.uiPageShift,
+										 1,
+										 uiPgAlignedOffset,
+										 psCpuPhyAddr,
+										 pbValid);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMR_CpuPhysAddr", e0);
+			}
+			else
+			{
+				eError = PMR_IsOffsetValid(psPMR,
+										  gsCwq.uiPageShift,
+										  1,
+										  uiPgAlignedOffset,
+										  pbValid);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMR_IsOffsetValid", e0);
+			}
+		}
+
+		/* Skip invalid PMR pages (i.e. sparse) */
+		if (pbValid[ui32PageIndex] == IMG_FALSE)
+		{
+			CACHEOP_PVR_ASSERT(bPMRIsSparse);
+			continue;
+		}
+
+		if (pvAddress)
+		{
+			/* The caller has supplied either a KM/UM CpuVA, so use it unconditionally */
+			pbCpuVirtAddr =
+				(void *)(uintptr_t)((uintptr_t)pvAddress + (uintptr_t)(uiPgAlignedOffset-uiPgAlignedStartOffset));
+		}
+		/* Skip CpuVA acquire if CacheOp can be maintained entirely using CpuPA */
+		else if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			if (bPMRIsSparse)
+			{
+				eError =
+					PMRAcquireSparseKernelMappingData(psPMR,
+													  uiPgAlignedOffset,
+													  gsCwq.uiPageSize,
+													  (void **)&pbCpuVirtAddr,
+													  &uiOutSize,
+													  &hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData", e0);
+			}
+			else
+			{
+				eError =
+					PMRAcquireKernelMappingData(psPMR,
+												uiPgAlignedOffset,
+												gsCwq.uiPageSize,
+												(void **)&pbCpuVirtAddr,
+												&uiOutSize,
+												&hPrivOut);
+				PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", e0);
+			}
+		}
+
+		/* Issue actual cache maintenance for PMR */
+		CacheOpExecRangeBased(psDevNode,
+							uiCacheOp,
+							pbCpuVirtAddr,
+							(eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_VIRTUAL) ?
+								psCpuPhyAddr[ui32PageIndex] : psCpuPhyAddr[0],
+							uiPgAlignedOffset,
+							uiCLAlignedStartOffset,
+							uiCLAlignedEndOffset);
+
+		if (! pvAddress)
+		{
+			/* The caller has not supplied either a KM/UM CpuVA, release mapping */
+			if (eCacheOpAddrType != OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+			{
+				eError = PMRReleaseKernelMappingData(psPMR, hPrivOut);
+				PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+			}
+		}
+	}
+
+e0:
+	if (psCpuPhyAddr != asCpuPhyAddr)
+	{
+		OSFreeMem(psCpuPhyAddr);
+	}
+
+	if (pbValid != abValid)
+	{
+		OSFreeMem(pbValid);
+	}
+
+	if (! bIsRequestValidated)
+	{
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpBatchExecTimeline(PVRSRV_DEVICE_NODE *psDevNode,
+											 PVRSRV_TIMELINE iTimeline)
+{
+	PVRSRV_ERROR eError;
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {NULL};
+
+	eError = CacheOpTimelineBind(psDevNode, &sCacheOpWorkItem, iTimeline);
+	PVR_LOG_RETURN_IF_ERROR(eError, "CacheOpTimelineBind");
+
+	eError = CacheOpTimelineExec(&sCacheOpWorkItem);
+	PVR_LOG_IF_ERROR(eError, "CacheOpTimelineExec");
+
+	return eError;
+}
+
+static PVRSRV_ERROR CacheOpBatchExecRangeBased(PVRSRV_DEVICE_NODE *psDevNode,
+											PMR **ppsPMR,
+											IMG_CPU_VIRTADDR *pvAddress,
+											IMG_DEVMEM_OFFSET_T *puiOffset,
+											IMG_DEVMEM_SIZE_T *puiSize,
+											PVRSRV_CACHE_OP *puiCacheOp,
+											IMG_UINT32 ui32NumCacheOps,
+											PVRSRV_TIMELINE uiTimeline)
+{
+	IMG_UINT32 ui32Idx;
+	IMG_BOOL bBatchHasTimeline;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+#endif
+
+	/* Check if batch has an associated timeline update */
+	bBatchHasTimeline = puiCacheOp[ui32NumCacheOps-1] & PVRSRV_CACHE_OP_TIMELINE;
+	puiCacheOp[ui32NumCacheOps-1] &= ~(PVRSRV_CACHE_OP_TIMELINE);
+
+	for (ui32Idx = 0; ui32Idx < ui32NumCacheOps; ui32Idx++)
+	{
+		/* Fail UM request, don't silently ignore */
+		PVR_GOTO_IF_INVALID_PARAM(puiSize[ui32Idx], eError, e0);
+
+#if defined(CACHEOP_DEBUG)
+		sCacheOpWorkItem.ui64StartTime = OSClockus64();
+#endif
+
+		eError = CacheOpPMRExec(ppsPMR[ui32Idx],
+								pvAddress[ui32Idx],
+								puiOffset[ui32Idx],
+								puiSize[ui32Idx],
+								puiCacheOp[ui32Idx],
+								IMG_FALSE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpExecPMR", e0);
+
+#if defined(CACHEOP_DEBUG)
+		sCacheOpWorkItem.ui64EndTime = OSClockus64();
+
+		sCacheOpWorkItem.psDevNode = psDevNode;
+		sCacheOpWorkItem.psPMR = ppsPMR[ui32Idx];
+		sCacheOpWorkItem.uiSize = puiSize[ui32Idx];
+		sCacheOpWorkItem.uiOffset = puiOffset[ui32Idx];
+		sCacheOpWorkItem.uiCacheOp = puiCacheOp[ui32Idx];
+		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+
+		gsCwq.ui32ServerOps += 1;
+#endif
+	}
+
+e0:
+	if (bBatchHasTimeline)
+	{
+		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline);
+	}
+
+	return eError;
+}
+
+
+PVRSRV_ERROR CacheOpExec (PPVRSRV_DEVICE_NODE psDevNode,
+						  void *pvVirtStart,
+						  void *pvVirtEnd,
+						  IMG_CPU_PHYADDR sCPUPhysStart,
+						  IMG_CPU_PHYADDR sCPUPhysEnd,
+						  PVRSRV_CACHE_OP uiCacheOp)
+{
+#if defined(CACHEOP_DEBUG)
+	IMG_UINT64 ui64StartTime = OSClockus64();
+#endif
+
+	switch (uiCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			OSCPUCacheCleanRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+			OSCPUCacheInvalidateRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		case PVRSRV_CACHE_OP_FLUSH:
+			OSCPUCacheFlushRangeKM(psDevNode, pvVirtStart, pvVirtEnd, sCPUPhysStart, sCPUPhysEnd);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,	"%s: Invalid cache operation type %d",
+					 __func__, uiCacheOp));
+			break;
+	}
+
+#if defined(CACHEOP_DEBUG)
+	if (CacheOpConfigSupports(CACHEOP_CONFIG_KLOG))
+	{
+		CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+
+		gsCwq.ui32ServerOps += 1;
+
+		sCacheOpWorkItem.uiOffset = 0;
+		sCacheOpWorkItem.bKMReq = IMG_TRUE;
+		sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+		/* Use information page PMR for logging KM request */
+		sCacheOpWorkItem.psPMR = gsCwq.psInfoPagePMR;
+		sCacheOpWorkItem.psDevNode = psDevNode;
+		sCacheOpWorkItem.ui64StartTime = ui64StartTime;
+		sCacheOpWorkItem.ui64EndTime = OSClockus64();
+		sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+		sCacheOpWorkItem.uiSize = (sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr);
+
+		CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR CacheOpValExec(PMR *psPMR,
+						    IMG_UINT64 uiAddress,
+						    IMG_DEVMEM_OFFSET_T uiOffset,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    PVRSRV_CACHE_OP uiCacheOp)
+{
+	PVRSRV_ERROR eError;
+	IMG_CPU_VIRTADDR pvAddress = (IMG_CPU_VIRTADDR)(uintptr_t)uiAddress;
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+
+	sCacheOpWorkItem.ui64StartTime = OSClockus64();
+#endif
+
+	eError = CacheOpPMRExec(psPMR,
+							pvAddress,
+							uiOffset,
+							uiSize,
+							uiCacheOp,
+							IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpPMRExec", e0);
+
+#if defined(CACHEOP_DEBUG)
+	sCacheOpWorkItem.ui64EndTime = OSClockus64();
+
+	sCacheOpWorkItem.psDevNode = PMR_DeviceNode(psPMR);
+	sCacheOpWorkItem.psPMR = psPMR;
+	sCacheOpWorkItem.uiSize = uiSize;
+	sCacheOpWorkItem.uiOffset = uiOffset;
+	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+
+	gsCwq.ui32ServerOps += 1;
+#endif
+
+e0:
+	return eError;
+}
+
+PVRSRV_ERROR CacheOpQueue (CONNECTION_DATA *psConnection,
+						   PVRSRV_DEVICE_NODE *psDevNode,
+						   IMG_UINT32 ui32NumCacheOps,
+						   PMR **ppsPMR,
+						   IMG_UINT64 *puiAddress,
+						   IMG_DEVMEM_OFFSET_T *puiOffset,
+						   IMG_DEVMEM_SIZE_T *puiSize,
+						   PVRSRV_CACHE_OP *puiCacheOp,
+						   IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_TIMELINE uiTimeline = (PVRSRV_TIMELINE)ui32OpTimeline;
+	IMG_CPU_VIRTADDR *pvAddress = (IMG_CPU_VIRTADDR*)(uintptr_t)puiAddress;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!gsCwq.bInit)
+	{
+		PVR_LOG(("CacheOp framework not initialised, failing request"));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+	else if (! ui32NumCacheOps)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	/* Ensure any single timeline CacheOp request is processed immediately */
+	else if (ui32NumCacheOps == 1 && puiCacheOp[0] == PVRSRV_CACHE_OP_TIMELINE)
+	{
+		eError = CacheOpBatchExecTimeline(psDevNode, uiTimeline);
+	}
+	/* This is the default entry for all client requests */
+	else
+	{
+		if (!(gsCwq.eConfig & (CACHEOP_CONFIG_LAST-1)))
+		{
+			/* default the configuration before execution */
+			CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
+		}
+
+		eError =
+			CacheOpBatchExecRangeBased(psDevNode,
+									   ppsPMR,
+									   pvAddress,
+									   puiOffset,
+									   puiSize,
+									   puiCacheOp,
+									   ui32NumCacheOps,
+									   uiTimeline);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR CacheOpLog (PMR *psPMR,
+						 IMG_UINT64 puiAddress,
+						 IMG_DEVMEM_OFFSET_T uiOffset,
+						 IMG_DEVMEM_SIZE_T uiSize,
+						 IMG_UINT64 ui64StartTime,
+						 IMG_UINT64 ui64EndTime,
+						 PVRSRV_CACHE_OP uiCacheOp)
+{
+#if defined(CACHEOP_DEBUG)
+	CACHEOP_WORK_ITEM sCacheOpWorkItem = {0};
+	PVR_UNREFERENCED_PARAMETER(puiAddress);
+
+	sCacheOpWorkItem.psDevNode = PMR_DeviceNode(psPMR);
+	sCacheOpWorkItem.psPMR = psPMR;
+	sCacheOpWorkItem.uiSize = uiSize;
+	sCacheOpWorkItem.uiOffset = uiOffset;
+	sCacheOpWorkItem.uiCacheOp = uiCacheOp;
+	sCacheOpWorkItem.pid = OSGetCurrentClientProcessIDKM();
+
+	sCacheOpWorkItem.ui64StartTime = ui64StartTime;
+	sCacheOpWorkItem.ui64EndTime = ui64EndTime;
+
+	gsCwq.ui32ClientOps += 1;
+
+	CacheOpStatsExecLogWrite(&sCacheOpWorkItem);
+#else
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiCacheOp);
+	PVR_UNREFERENCED_PARAMETER(puiAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64EndTime);
+#endif
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR CacheOpInit2 (void)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	/* Apphint read/write is not concurrent, so lock protects against this */
+	eError = OSLockCreate((POS_LOCK*)&gsCwq.hConfigLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+
+#if defined(CACHEFLUSH_ISA_SUPPORTS_UM_FLUSH)
+	gsCwq.bSupportsUMFlush = IMG_TRUE;
+#else
+	gsCwq.bSupportsUMFlush = IMG_FALSE;
+#endif
+
+	gsCwq.pui32InfoPage = psPVRSRVData->pui32InfoPage;
+	gsCwq.psInfoPagePMR = psPVRSRVData->psInfoPagePMR;
+
+	/* Normally, platforms should use their default configurations, put exceptions here */
+#if defined(__i386__) || defined(__x86_64__)
+#if !defined(TC_MEMORY_CONFIG)
+	CacheOpConfigUpdate(CACHEOP_CONFIG_URBF | CACHEOP_CONFIG_KDF);
+#else
+	CacheOpConfigUpdate(CACHEOP_CONFIG_KDF);
+#endif
+#else /* defined(__x86__) */
+	CacheOpConfigUpdate(CACHEOP_CONFIG_DEFAULT);
+#endif
+
+	/* Initialise the remaining occupants of the CacheOp information page */
+	gsCwq.pui32InfoPage[CACHEOP_INFO_PGSIZE]   = (IMG_UINT32)gsCwq.uiPageSize;
+	gsCwq.pui32InfoPage[CACHEOP_INFO_LINESIZE] = (IMG_UINT32)gsCwq.uiLineSize;
+
+	/* Set before spawning thread */
+	gsCwq.bInit = IMG_TRUE;
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = CacheOpConfigRead};
+		/* Writing the unsigned integer binary encoding of CACHEOP_CONFIG
+		   into this file cycles through avail. configuration(s) */
+		eError = DICreateEntry("cacheop_config", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gsCwq.psConfigTune);
+		PVR_LOG_GOTO_IF_FALSE(gsCwq.psConfigTune, "DICreateEntry", e0);
+	}
+
+	/* Register the CacheOp framework (re)configuration handlers */
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpConfig,
+										CacheOpConfigQuery,
+										CacheOpConfigSet,
+										APPHINT_OF_DRIVER_NO_DEVICE,
+										(void *) APPHINT_ID_CacheOpConfig);
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_CacheOpUMKMThresholdSize,
+										CacheOpConfigQuery,
+										CacheOpConfigSet,
+										APPHINT_OF_DRIVER_NO_DEVICE,
+										(void *) APPHINT_ID_CacheOpUMKMThresholdSize);
+
+	return PVRSRV_OK;
+e0:
+	CacheOpDeInit2();
+	return eError;
+}
+
+void CacheOpDeInit2 (void)
+{
+	gsCwq.bInit = IMG_FALSE;
+
+	if (gsCwq.hConfigLock)
+	{
+		OSLockDestroy(gsCwq.hConfigLock);
+		gsCwq.hConfigLock = NULL;
+	}
+
+	if (gsCwq.psConfigTune)
+	{
+		DIDestroyEntry(gsCwq.psConfigTune);
+		gsCwq.psConfigTune = NULL;
+	}
+
+	gsCwq.pui32InfoPage = NULL;
+	gsCwq.psInfoPagePMR = NULL;
+}
+
+PVRSRV_ERROR CacheOpInit (void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	gsCwq.uiPageSize = OSGetPageSize();
+	gsCwq.uiPageShift = OSGetPageShift();
+	gsCwq.uiLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+	gsCwq.uiLineShift = ExactLog2(gsCwq.uiLineSize);
+	PVR_LOG_RETURN_IF_FALSE((gsCwq.uiLineSize && gsCwq.uiPageSize && gsCwq.uiPageShift), "", PVRSRV_ERROR_INIT_FAILURE);
+
+#if defined(CACHEOP_DEBUG)
+	/* debugfs file read-out is not concurrent, so lock protects against this */
+	eError = OSLockCreate((POS_LOCK*)&gsCwq.hStatsExecLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	gsCwq.i32StatsExecWriteIdx = 0;
+	OSCachedMemSet(gsCwq.asStatsExecuted, 0, sizeof(gsCwq.asStatsExecuted));
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = CacheOpStatsExecLogRead};
+		/* File captures the most recent subset of CacheOp(s) executed */
+		eError = DICreateEntry("cacheop_history", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gsCwq.psDIEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", e0);
+	}
+e0:
+#endif
+	return eError;
+}
+
+void CacheOpDeInit (void)
+{
+#if defined(CACHEOP_DEBUG)
+	if (gsCwq.hStatsExecLock)
+	{
+		OSLockDestroy(gsCwq.hStatsExecLock);
+		gsCwq.hStatsExecLock = NULL;
+	}
+
+	if (gsCwq.psDIEntry)
+	{
+		DIDestroyEntry(gsCwq.psDIEntry);
+		gsCwq.psDIEntry = NULL;
+	}
+#endif
+}
diff --git a/drivers/gpu/drm/img-rogue/cache_km.h b/drivers/gpu/drm/img-rogue/cache_km.h
new file mode 100644
index 000000000000..282ff5bc5f0f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cache_km.h
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File           cache_km.h
+@Title          CPU cache management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef CACHE_KM_H
+#define CACHE_KM_H
+
+#if defined(__linux__)
+#include <linux/version.h>
+#else
+#define KERNEL_VERSION
+#endif
+
+#include "pvrsrv_error.h"
+#include "os_cpu_cache.h"
+#include "img_types.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "pmr.h"
+
+typedef IMG_UINT32 PVRSRV_CACHE_OP_ADDR_TYPE;	/*!< Represents CPU address type required for CPU d-cache maintenance */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_VIRTUAL	0x1	/*!< Operation requires CPU virtual address only */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_PHYSICAL	0x2	/*!< Operation requires CPU physical address only */
+#define PVRSRV_CACHE_OP_ADDR_TYPE_BOTH		0x3	/*!< Operation requires both CPU virtual & physical addresses */
+
+#include "connection_server.h"
+
+/*
+ * CacheOpInit() & CacheOpDeInit()
+ *
+ * This must be called to initialise the KM cache maintenance framework.
+ * This is called early during the driver/module (un)loading phase.
+ */
+PVRSRV_ERROR CacheOpInit(void);
+void CacheOpDeInit(void);
+
+/*
+ * CacheOpInit2() & CacheOpDeInit2()
+ *
+ * This must be called to initialise the UM cache maintenance framework.
+ * This is called when the driver is loaded/unloaded from the kernel.
+ */
+PVRSRV_ERROR CacheOpInit2(void);
+void CacheOpDeInit2(void);
+
+/*
+ * CacheOpExec()
+ *
+ * This is the primary CPU data-cache maintenance interface and it is
+ * always guaranteed to be synchronous; the arguments supplied must be
+ * pre-validated for performance reasons else the d-cache maintenance
+ * operation might cause the underlying OS kernel to fault.
+ */
+PVRSRV_ERROR CacheOpExec(PPVRSRV_DEVICE_NODE psDevNode,
+						void *pvVirtStart,
+						void *pvVirtEnd,
+						IMG_CPU_PHYADDR sCPUPhysStart,
+						IMG_CPU_PHYADDR sCPUPhysEnd,
+						PVRSRV_CACHE_OP uiCacheOp);
+
+/*
+ * CacheOpValExec()
+ *
+ * Same as CacheOpExec(), except arguments are _Validated_ before being
+ * presented to the underlying OS kernel for CPU data-cache maintenance.
+ * The uiAddress is the start CPU virtual address for the to-be d-cache
+ * maintained PMR, it can be NULL in which case a remap will be performed
+ * internally, if required for cache maintenance. This is primarily used
+ * as the services client bridge call handler for synchronous user-mode
+ * cache maintenance requests.
+ */
+PVRSRV_ERROR CacheOpValExec(PMR *psPMR,
+							IMG_UINT64 uiAddress,
+							IMG_DEVMEM_OFFSET_T uiOffset,
+							IMG_DEVMEM_SIZE_T uiSize,
+							PVRSRV_CACHE_OP uiCacheOp);
+
+/*
+ * CacheOpQueue()
+ *
+ * This is the secondary cache maintenance interface and it is not
+ * guaranteed to be synchronous in that requests could be deferred
+ * and executed asynchronously. This interface is primarily meant
+ * as services client bridge call handler. Both uiInfoPgGFSeqNum
+ * and ui32[Current,Next]FenceSeqNum implements an internal client
+ * server queueing protocol so making use of this interface outside
+ * of services client is not recommended and should not be done.
+ */
+PVRSRV_ERROR CacheOpQueue(CONNECTION_DATA *psConnection,
+						PPVRSRV_DEVICE_NODE psDevNode,
+						IMG_UINT32 ui32OpCount,
+						PMR **ppsPMR,
+						IMG_UINT64 *puiAddress,
+						IMG_DEVMEM_OFFSET_T *puiOffset,
+						IMG_DEVMEM_SIZE_T *puiSize,
+						PVRSRV_CACHE_OP *puiCacheOp,
+						IMG_UINT32 ui32OpTimeline);
+
+/*
+ * CacheOpLog()
+ *
+ * This is used for logging client cache maintenance operations that
+ * was executed in user-space.
+ */
+PVRSRV_ERROR CacheOpLog(PMR *psPMR,
+						IMG_UINT64 uiAddress,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						IMG_DEVMEM_SIZE_T uiSize,
+						IMG_UINT64 ui64StartTime,
+						IMG_UINT64 ui64EndTime,
+						PVRSRV_CACHE_OP uiCacheOp);
+
+#endif	/* CACHE_KM_H */
diff --git a/drivers/gpu/drm/img-rogue/cache_ops.h b/drivers/gpu/drm/img-rogue/cache_ops.h
new file mode 100644
index 000000000000..a1d714519d59
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cache_ops.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services cache management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines for cache management which are visible internally
+                and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef CACHE_OPS_H
+#define CACHE_OPS_H
+#include "img_types.h"
+/*!
+* @Defgroup CPUCacheAPIs
+* @{
+*/
+#define CACHE_BATCH_MAX (8U)
+#define MAX_DMA_OPS (34)
+typedef IMG_UINT32 PVRSRV_CACHE_OP;				/*!< Type represents cache maintenance operation */
+#define PVRSRV_CACHE_OP_NONE				0x0U	/*!< No operation */
+#define PVRSRV_CACHE_OP_CLEAN				0x1U	/*!< Flush w/o invalidate */
+#define PVRSRV_CACHE_OP_INVALIDATE			0x2U	/*!< Invalidate w/o flush */
+#define PVRSRV_CACHE_OP_FLUSH				0x3U	/*!< Flush w/ invalidate */
+/*! @} End of Defgroup CPUCacheAPIs */
+
+#endif	/* CACHE_OPS_H */
diff --git a/drivers/gpu/drm/img-rogue/client_cache_bridge.h b/drivers/gpu/drm/img-rogue/client_cache_bridge.h
new file mode 100644
index 000000000000..1dec13fccce7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_cache_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_CACHE_BRIDGE_H
+#define CLIENT_CACHE_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_cache_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp);
+
+#endif /* CLIENT_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_cache_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_cache_direct_bridge.c
new file mode 100644
index 000000000000..9691bae9352f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_cache_direct_bridge.c
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for cache
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_cache_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "cache_ops.h"
+
+#include "cache_km.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpQueue(IMG_HANDLE hBridge,
+					     IMG_UINT32 ui32NumCacheOps,
+					     IMG_HANDLE * phPMR,
+					     IMG_UINT64 * pui64Address,
+					     IMG_DEVMEM_OFFSET_T * puiOffset,
+					     IMG_DEVMEM_SIZE_T * puiSize,
+					     PVRSRV_CACHE_OP * piuCacheOp,
+					     IMG_UINT32 ui32OpTimeline)
+{
+	PVRSRV_ERROR eError;
+	PMR **psPMRInt;
+
+	psPMRInt = (PMR **) phPMR;
+
+	eError =
+	    CacheOpQueue(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			 ui32NumCacheOps,
+			 psPMRInt, pui64Address, puiOffset, puiSize, piuCacheOp, ui32OpTimeline);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpExec(IMG_HANDLE hBridge,
+					    IMG_HANDLE hPMR,
+					    IMG_UINT64 ui64Address,
+					    IMG_DEVMEM_OFFSET_T uiOffset,
+					    IMG_DEVMEM_SIZE_T uiSize, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = CacheOpValExec(psPMRInt, ui64Address, uiOffset, uiSize, iuCacheOp);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeCacheOpLog(IMG_HANDLE hBridge,
+					   IMG_HANDLE hPMR,
+					   IMG_UINT64 ui64Address,
+					   IMG_DEVMEM_OFFSET_T uiOffset,
+					   IMG_DEVMEM_SIZE_T uiSize,
+					   IMG_INT64 i64StartTime,
+					   IMG_INT64 i64EndTime, PVRSRV_CACHE_OP iuCacheOp)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    CacheOpLog(psPMRInt,
+		       ui64Address, uiOffset, uiSize, i64StartTime, i64EndTime, iuCacheOp);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/client_devicememhistory_bridge.h b/drivers/gpu/drm/img-rogue/client_devicememhistory_bridge.h
new file mode 100644
index 000000000000..bfa6bfb9c037
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_devicememhistory_bridge.h
@@ -0,0 +1,111 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+#define CLIENT_DEVICEMEMHISTORY_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_devicememhistory_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut);
+
+#endif /* CLIENT_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_devicememhistory_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_devicememhistory_direct_bridge.c
new file mode 100644
index 000000000000..a593a7e2cc7a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_devicememhistory_direct_bridge.c
@@ -0,0 +1,195 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for devicememhistory
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_devicememhistory_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMap(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMR,
+						    IMG_DEVMEM_SIZE_T uiOffset,
+						    IMG_DEV_VIRTADDR sDevVAddr,
+						    IMG_DEVMEM_SIZE_T uiSize,
+						    const IMG_CHAR * puiText,
+						    IMG_UINT32 ui32Log2PageSize,
+						    IMG_UINT32 ui32AllocationIndex,
+						    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  uiOffset,
+				  sDevVAddr,
+				  uiSize,
+				  puiText,
+				  ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmap(IMG_HANDLE hBridge,
+						      IMG_HANDLE hPMR,
+						      IMG_DEVMEM_SIZE_T uiOffset,
+						      IMG_DEV_VIRTADDR sDevVAddr,
+						      IMG_DEVMEM_SIZE_T uiSize,
+						      const IMG_CHAR * puiText,
+						      IMG_UINT32 ui32Log2PageSize,
+						      IMG_UINT32 ui32AllocationIndex,
+						      IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    uiOffset,
+				    sDevVAddr,
+				    uiSize,
+				    puiText,
+				    ui32Log2PageSize, ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryMapVRange(IMG_HANDLE hBridge,
+							  IMG_DEV_VIRTADDR sBaseDevVAddr,
+							  IMG_UINT32 ui32ui32StartPage,
+							  IMG_UINT32 ui32NumPages,
+							  IMG_DEVMEM_SIZE_T uiAllocSize,
+							  const IMG_CHAR * puiText,
+							  IMG_UINT32 ui32Log2PageSize,
+							  IMG_UINT32 ui32AllocationIndex,
+							  IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryMapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					sBaseDevVAddr,
+					ui32ui32StartPage,
+					ui32NumPages,
+					uiAllocSize,
+					puiText,
+					ui32Log2PageSize,
+					ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistoryUnmapVRange(IMG_HANDLE hBridge,
+							    IMG_DEV_VIRTADDR sBaseDevVAddr,
+							    IMG_UINT32 ui32ui32StartPage,
+							    IMG_UINT32 ui32NumPages,
+							    IMG_DEVMEM_SIZE_T uiAllocSize,
+							    const IMG_CHAR * puiText,
+							    IMG_UINT32 ui32Log2PageSize,
+							    IMG_UINT32 ui32AllocationIndex,
+							    IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    DevicememHistoryUnmapVRangeKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  sBaseDevVAddr,
+					  ui32ui32StartPage,
+					  ui32NumPages,
+					  uiAllocSize,
+					  puiText,
+					  ui32Log2PageSize,
+					  ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevicememHistorySparseChange(IMG_HANDLE hBridge,
+							     IMG_HANDLE hPMR,
+							     IMG_DEVMEM_SIZE_T uiOffset,
+							     IMG_DEV_VIRTADDR sDevVAddr,
+							     IMG_DEVMEM_SIZE_T uiSize,
+							     const IMG_CHAR * puiText,
+							     IMG_UINT32 ui32Log2PageSize,
+							     IMG_UINT32 ui32AllocPageCount,
+							     IMG_UINT32 * pui32AllocPageIndices,
+							     IMG_UINT32 ui32FreePageCount,
+							     IMG_UINT32 * pui32FreePageIndices,
+							     IMG_UINT32 ui32AllocationIndex,
+							     IMG_UINT32 * pui32AllocationIndexOut)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   uiOffset,
+					   sDevVAddr,
+					   uiSize,
+					   puiText,
+					   ui32Log2PageSize,
+					   ui32AllocPageCount,
+					   pui32AllocPageIndices,
+					   ui32FreePageCount,
+					   pui32FreePageIndices,
+					   ui32AllocationIndex, pui32AllocationIndexOut);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/client_htbuffer_bridge.h b/drivers/gpu/drm/img-rogue/client_htbuffer_bridge.h
new file mode 100644
index 000000000000..2c5d18843120
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_htbuffer_bridge.h
@@ -0,0 +1,64 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_HTBUFFER_BRIDGE_H
+#define CLIENT_HTBUFFER_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_htbuffer_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode);
+
+#endif /* CLIENT_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_htbuffer_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_htbuffer_direct_bridge.c
new file mode 100644
index 000000000000..3e6295cdfdf3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_htbuffer_direct_bridge.c
@@ -0,0 +1,70 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for htbuffer
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_htbuffer_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#include "htbserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHTBControl(IMG_HANDLE hBridge,
+					   IMG_UINT32 ui32NumGroups,
+					   IMG_UINT32 * pui32GroupEnable,
+					   IMG_UINT32 ui32LogLevel,
+					   IMG_UINT32 ui32EnablePID,
+					   IMG_UINT32 ui32LogMode, IMG_UINT32 ui32OpMode)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError =
+	    HTBControlKM(ui32NumGroups,
+			 pui32GroupEnable, ui32LogLevel, ui32EnablePID, ui32LogMode, ui32OpMode);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/client_mm_bridge.h b/drivers/gpu/drm/img-rogue/client_mm_bridge.h
new file mode 100644
index 000000000000..fe2494bc472e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_mm_bridge.h
@@ -0,0 +1,229 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_MM_BRIDGE_H
+#define CLIENT_MM_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_mm_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge,
+							IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags,
+						IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS * pasapPhysHeapMemStats);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset);
+
+#endif /* CLIENT_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_mm_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_mm_direct_bridge.c
new file mode 100644
index 000000000000..a1d1bd895f5f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_mm_direct_bridge.c
@@ -0,0 +1,705 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for mm
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_mm_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRExportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMR,
+					     IMG_HANDLE * phPMRExport,
+					     IMG_UINT64 * pui64Size,
+					     IMG_UINT32 * pui32Log2Contig,
+					     IMG_UINT64 * pui64Password)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRExportPMR(psPMRInt, &psPMRExportInt, pui64Size, pui32Log2Contig, pui64Password);
+
+	*phPMRExport = psPMRExportInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMR);
+	PVR_UNREFERENCED_PARAMETER(phPMRExport);
+	PVR_UNREFERENCED_PARAMETER(pui64Size);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2Contig);
+	PVR_UNREFERENCED_PARAMETER(pui64Password);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnexportPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMRExport)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError = PMRUnexportPMR(psPMRExportInt);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRGetUID(IMG_HANDLE hBridge,
+					  IMG_HANDLE hPMR, IMG_UINT64 * pui64UID)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRGetUID(psPMRInt, pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRMakeLocalImportHandle(IMG_HANDLE hBridge,
+							 IMG_HANDLE hBuffer, IMG_HANDLE * phExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psBufferInt;
+	PMR *psExtMemInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psBufferInt = (PMR *) hBuffer;
+
+	eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+
+	*phExtMem = psExtMemInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnmakeLocalImportHandle(IMG_HANDLE hBridge, IMG_HANDLE hExtMem)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtMemInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtMemInt = (PMR *) hExtMem;
+
+	eError = PMRUnmakeLocalImportHandle(psExtMemInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRImportPMR(IMG_HANDLE hBridge,
+					     IMG_HANDLE hPMRExport,
+					     IMG_UINT64 ui64uiPassword,
+					     IMG_UINT64 ui64uiSize,
+					     IMG_UINT32 ui32uiLog2Contig, IMG_HANDLE * phPMR)
+{
+#if defined(SUPPORT_INSECURE_EXPORT)
+	PVRSRV_ERROR eError;
+	PMR_EXPORT *psPMRExportInt;
+	PMR *psPMRInt = NULL;
+
+	psPMRExportInt = (PMR_EXPORT *) hPMRExport;
+
+	eError =
+	    PhysmemImportPMR(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     psPMRExportInt,
+			     ui64uiPassword, ui64uiSize, ui32uiLog2Contig, &psPMRInt);
+
+	*phPMR = psPMRInt;
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPMRExport);
+	PVR_UNREFERENCED_PARAMETER(ui64uiPassword);
+	PVR_UNREFERENCED_PARAMETER(ui64uiSize);
+	PVR_UNREFERENCED_PARAMETER(ui32uiLog2Contig);
+	PVR_UNREFERENCED_PARAMETER(phPMR);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRLocalImportPMR(IMG_HANDLE hBridge,
+						  IMG_HANDLE hExtHandle,
+						  IMG_HANDLE * phPMR,
+						  IMG_DEVMEM_SIZE_T * puiSize,
+						  IMG_DEVMEM_ALIGN_T * puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psExtHandleInt;
+	PMR *psPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psExtHandleInt = (PMR *) hExtHandle;
+
+	eError = PMRLocalImportPMR(psExtHandleInt, &psPMRInt, puiSize, puiAlign);
+
+	*phPMR = psPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePMRUnrefPMR(IMG_HANDLE hBridge, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRInt = (PMR *) hPMR;
+
+	eError = PMRUnrefPMR(psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysmemNewRamBackedPMR(IMG_HANDLE hBridge,
+						       IMG_DEVMEM_SIZE_T uiSize,
+						       IMG_UINT32 ui32NumPhysChunks,
+						       IMG_UINT32 ui32NumVirtChunks,
+						       IMG_UINT32 * pui32MappingTable,
+						       IMG_UINT32 ui32Log2PageSize,
+						       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						       IMG_UINT32 ui32AnnotationLength,
+						       const IMG_CHAR * puiAnnotation,
+						       IMG_PID ui32PID,
+						       IMG_HANDLE * phPMRPtr,
+						       IMG_UINT32 ui32PDumpFlags,
+						       PVRSRV_MEMALLOCFLAGS_T * puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRPtrInt = NULL;
+
+	eError =
+	    PhysmemNewRamBackedPMR_direct(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					  uiSize,
+					  ui32NumPhysChunks,
+					  ui32NumVirtChunks,
+					  pui32MappingTable,
+					  ui32Log2PageSize,
+					  uiFlags,
+					  ui32AnnotationLength,
+					  puiAnnotation,
+					  ui32PID, &psPMRPtrInt, ui32PDumpFlags, puiOutFlags);
+
+	*phPMRPtr = psPMRPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxCreate(IMG_HANDLE hBridge,
+						   IMG_BOOL bbKernelMemoryCtx,
+						   IMG_HANDLE * phDevMemServerContext,
+						   IMG_HANDLE * phPrivData,
+						   IMG_UINT32 * pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	eError =
+	    DevmemIntCtxCreate(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       bbKernelMemoryCtx,
+			       &psDevMemServerContextInt, &hPrivDataInt, pui32CPUCacheLineSize);
+
+	*phDevMemServerContext = psDevMemServerContextInt;
+	*phPrivData = hPrivDataInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntCtxDestroy(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemServerContext)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemServerContextInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerContextInt = (DEVMEMINT_CTX *) hDevmemServerContext;
+
+	eError = DevmemIntCtxDestroy(psDevmemServerContextInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapCreate(IMG_HANDLE hBridge,
+						    IMG_HANDLE hDevmemCtx,
+						    IMG_UINT32 ui32HeapConfigIndex,
+						    IMG_UINT32 ui32HeapIndex,
+						    IMG_HANDLE * phDevmemHeapPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				ui32HeapConfigIndex, ui32HeapIndex, &psDevmemHeapPtrInt);
+
+	*phDevmemHeapPtr = psDevmemHeapPtrInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntHeapDestroy(IMG_HANDLE hBridge, IMG_HANDLE hDevmemHeap)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeapInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemHeapInt = (DEVMEMINT_HEAP *) hDevmemHeap;
+
+	eError = DevmemIntHeapDestroy(psDevmemHeapInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntMapPMR(IMG_HANDLE hBridge,
+						IMG_HANDLE hReservation, IMG_HANDLE hPMR)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnmapPMR(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnmapPMR(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRange(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemServerHeap,
+						      IMG_DEV_VIRTADDR sAddress,
+						      IMG_DEVMEM_SIZE_T uiLength,
+						      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						      IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemIntReserveRange(psDevmemServerHeapInt,
+				  sAddress, uiLength, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntReserveRangeAndMapPMR(IMG_HANDLE hBridge,
+							       IMG_HANDLE hDevmemServerHeap,
+							       IMG_DEV_VIRTADDR sAddress,
+							       IMG_DEVMEM_SIZE_T uiLength,
+							       IMG_HANDLE hPMR,
+							       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	PMR *psPMRInt;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemIntReserveRangeAndMapPMR(psDevmemServerHeapInt,
+					   sAddress,
+					   uiLength, psPMRInt, uiFlags, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntUnreserveRange(IMG_HANDLE hBridge, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError = DevmemIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeChangeSparseMem(IMG_HANDLE hBridge,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 * pui32AllocPageIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 * pui32FreePageIndices,
+						IMG_UINT32 ui32SparseFlags, IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemIntChangeSparse(ui32AllocPageCount,
+				  pui32AllocPageIndices,
+				  ui32FreePageCount,
+				  pui32FreePageIndices, ui32SparseFlags, psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIsVDevAddrValid(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR sAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntIsVDevAddrValid(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, sAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemInvalidateFBSCTable(IMG_HANDLE hBridge,
+							  IMG_HANDLE hDevmemCtx,
+							  IMG_UINT64 ui64FBSCEntries)
+{
+#if defined(RGX_FEATURE_FBCDC)
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntInvalidateFBSCTable(psDevmemCtxInt, ui64FBSCEntries);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(ui64FBSCEntries);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigCount(IMG_HANDLE hBridge,
+						       IMG_UINT32 * pui32NumHeapConfigs)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   pui32NumHeapConfigs);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapCount(IMG_HANDLE hBridge,
+						 IMG_UINT32 ui32HeapConfigIndex,
+						 IMG_UINT32 * pui32NumHeaps)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapCount(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			     ui32HeapConfigIndex, pui32NumHeaps);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapConfigName(IMG_HANDLE hBridge,
+						      IMG_UINT32 ui32HeapConfigIndex,
+						      IMG_UINT32 ui32HeapConfigNameBufSz,
+						      IMG_CHAR * puiHeapConfigName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapConfigName(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  ui32HeapConfigIndex, ui32HeapConfigNameBufSz, puiHeapConfigName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeHeapCfgHeapDetails(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32HeapConfigIndex,
+						   IMG_UINT32 ui32HeapIndex,
+						   IMG_UINT32 ui32HeapNameBufSz,
+						   IMG_CHAR * puiHeapNameOut,
+						   IMG_DEV_VIRTADDR * psDevVAddrBase,
+						   IMG_DEVMEM_SIZE_T * puiHeapLength,
+						   IMG_DEVMEM_SIZE_T * puiReservedRegionLength,
+						   IMG_UINT32 * pui32Log2DataPageSizeOut,
+						   IMG_UINT32 * pui32Log2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    HeapCfgHeapDetails(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+			       ui32HeapConfigIndex,
+			       ui32HeapIndex,
+			       ui32HeapNameBufSz,
+			       puiHeapNameOut,
+			       psDevVAddrBase,
+			       puiHeapLength,
+			       puiReservedRegionLength,
+			       pui32Log2DataPageSizeOut, pui32Log2ImportAlignmentOut);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemIntRegisterPFNotifyKM(IMG_HANDLE hBridge,
+							    IMG_HANDLE hDevmemCtx,
+							    IMG_BOOL bRegister)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError = DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, bRegister);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePhysHeapGetMemInfo(IMG_HANDLE hBridge,
+						   IMG_UINT32 ui32PhysHeapCount,
+						   PVRSRV_PHYS_HEAP * peaPhysHeapID,
+						   PHYS_HEAP_MEM_STATS * pasapPhysHeapMemStats)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVPhysHeapGetMemInfoKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				       ui32PhysHeapCount, peaPhysHeapID, pasapPhysHeapMemStats);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeGetDefaultPhysicalHeap(IMG_HANDLE hBridge,
+						       PVRSRV_PHYS_HEAP * peHeap)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), peHeap);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemGetFaultAddress(IMG_HANDLE hBridge,
+						      IMG_HANDLE hDevmemCtx,
+						      IMG_DEV_VIRTADDR * psFaultAddress)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtxInt;
+
+	psDevmemCtxInt = (DEVMEMINT_CTX *) hDevmemCtx;
+
+	eError =
+	    DevmemIntGetFaultAddress(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     psDevmemCtxInt, psFaultAddress);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgePVRSRVStatsUpdateOOMStat(IMG_HANDLE hBridge,
+							 IMG_UINT32 ui32ui32StatType,
+							 IMG_PID ui32pid)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVStatsUpdateOOMStat(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				     ui32ui32StatType, ui32pid);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32ui32StatType);
+	PVR_UNREFERENCED_PARAMETER(ui32pid);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntReserveRange(IMG_HANDLE hBridge,
+						       IMG_HANDLE hDevmemServerHeap,
+						       IMG_DEV_VIRTADDR sAddress,
+						       IMG_DEVMEM_SIZE_T uiLength,
+						       IMG_HANDLE * phReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psDevmemServerHeapInt = (DEVMEMINT_HEAP *) hDevmemServerHeap;
+
+	eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt, sAddress, uiLength, &psReservationInt);
+
+	*phReservation = psReservationInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnreserveRange(IMG_HANDLE hBridge,
+							 IMG_HANDLE hReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnreserveRange(psReservationInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapPages(IMG_HANDLE hBridge,
+						   IMG_HANDLE hReservation,
+						   IMG_HANDLE hPMR,
+						   IMG_UINT32 ui32PageCount,
+						   IMG_UINT32 ui32PhysPageOffset,
+						   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						   IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PMR *psPMRInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+	psPMRInt = (PMR *) hPMR;
+
+	eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       ui32PageCount, ui32PhysPageOffset, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntUnmapPages(IMG_HANDLE hBridge,
+						     IMG_HANDLE hReservation,
+						     IMG_UINT32 ui32VirtPageOffset,
+						     IMG_UINT32 ui32PageCount)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError = DevmemXIntUnmapPages(psReservationInt, ui32VirtPageOffset, ui32PageCount);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeDevmemXIntMapVRangeToBackingPage(IMG_HANDLE hBridge,
+								 IMG_HANDLE hReservation,
+								 IMG_UINT32 ui32PageCount,
+								 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+								 IMG_UINT32 ui32VirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMXINT_RESERVATION *psReservationInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psReservationInt = (DEVMEMXINT_RESERVATION *) hReservation;
+
+	eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     ui32PageCount, uiFlags, ui32VirtPageOffset);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/client_pvrtl_bridge.h b/drivers/gpu/drm/img-rogue/client_pvrtl_bridge.h
new file mode 100644
index 000000000000..2cfafd5a83e1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_pvrtl_bridge.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_PVRTL_BRIDGE_H
+#define CLIENT_PVRTL_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_pvrtl_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams,
+						  IMG_UINT32 * pui32NumFound);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin,
+						IMG_UINT32 * pui32Available);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data);
+
+#endif /* CLIENT_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_pvrtl_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_pvrtl_direct_bridge.c
new file mode 100644
index 000000000000..fa2fbed9595b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_pvrtl_direct_bridge.c
@@ -0,0 +1,175 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for pvrtl
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_pvrtl_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#include "tlserver.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLOpenStream(IMG_HANDLE hBridge,
+					     const IMG_CHAR * puiName,
+					     IMG_UINT32 ui32Mode,
+					     IMG_HANDLE * phSD, IMG_HANDLE * phTLPMR)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerOpenStreamKM(puiName, ui32Mode, &psSDInt, &psTLPMRInt);
+
+	*phSD = psSDInt;
+	*phTLPMR = psTLPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCloseStream(IMG_HANDLE hBridge, IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCloseStreamKM(psSDInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLAcquireData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 * pui32ReadOffset,
+					      IMG_UINT32 * pui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerAcquireDataKM(psSDInt, pui32ReadOffset, pui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReleaseData(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSD,
+					      IMG_UINT32 ui32ReadOffset, IMG_UINT32 ui32ReadLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerReleaseDataKM(psSDInt, ui32ReadOffset, ui32ReadLen);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLDiscoverStreams(IMG_HANDLE hBridge,
+						  const IMG_CHAR * puiNamePattern,
+						  IMG_UINT32 ui32Size,
+						  IMG_CHAR * puiStreams, IMG_UINT32 * pui32NumFound)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = TLServerDiscoverStreamsKM(puiNamePattern, ui32Size, puiStreams, pui32NumFound);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLReserveStream(IMG_HANDLE hBridge,
+						IMG_HANDLE hSD,
+						IMG_UINT32 * pui32BufferOffset,
+						IMG_UINT32 ui32Size,
+						IMG_UINT32 ui32SizeMin, IMG_UINT32 * pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    pui32BufferOffset, ui32Size, ui32SizeMin, pui32Available);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLCommitStream(IMG_HANDLE hBridge,
+					       IMG_HANDLE hSD, IMG_UINT32 ui32ReqSize)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerCommitStreamKM(psSDInt, ui32ReqSize);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeTLWriteData(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSD,
+					    IMG_UINT32 ui32Size, IMG_BYTE * pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC *psSDInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSDInt = (TL_STREAM_DESC *) hSD;
+
+	eError = TLServerWriteDataKM(psSDInt, ui32Size, pui8Data);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/client_ri_bridge.h b/drivers/gpu/drm/img-rogue/client_ri_bridge.h
new file mode 100644
index 000000000000..c419e21e9d07
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_ri_bridge.h
@@ -0,0 +1,88 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_RI_BRIDGE_H
+#define CLIENT_RI_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_ri_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner);
+
+#endif /* CLIENT_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_ri_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_ri_direct_bridge.c
new file mode 100644
index 000000000000..6e60293d68e2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_ri_direct_bridge.c
@@ -0,0 +1,180 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for ri
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_ri_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "ri_typedefs.h"
+
+#include "ri_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntry(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteMEMDESCEntry(IMG_HANDLE hBridge,
+						    IMG_HANDLE hPMRHandle,
+						    IMG_UINT32 ui32TextBSize,
+						    const IMG_CHAR * puiTextB,
+						    IMG_UINT64 ui64Offset,
+						    IMG_UINT64 ui64Size,
+						    IMG_UINT64 ui64Flags, IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError =
+	    RIWriteMEMDESCEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  psPMRHandleInt,
+				  ui32TextBSize,
+				  puiTextB, ui64Offset, ui64Size, ui64Flags, &psRIHandleInt);
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWriteProcListEntry(IMG_HANDLE hBridge,
+						     IMG_UINT32 ui32TextBSize,
+						     const IMG_CHAR * puiTextB,
+						     IMG_UINT64 ui64Size,
+						     IMG_UINT64 ui64DevVAddr,
+						     IMG_HANDLE * phRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	eError =
+	    RIWriteProcListEntryKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   ui32TextBSize, puiTextB, ui64Size, ui64DevVAddr, &psRIHandleInt);
+
+	*phRIHandle = psRIHandleInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIUpdateMEMDESCAddr(IMG_HANDLE hBridge,
+						    IMG_HANDLE hRIHandle, IMG_DEV_VIRTADDR sAddr)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIUpdateMEMDESCAddrKM(psRIHandleInt, sAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDeleteMEMDESCEntry(IMG_HANDLE hBridge, IMG_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError;
+	RI_HANDLE psRIHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psRIHandleInt = (RI_HANDLE) hRIHandle;
+
+	eError = RIDeleteMEMDESCEntryKM(psRIHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpList(IMG_HANDLE hBridge, IMG_HANDLE hPMRHandle)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIDumpListKM(psPMRHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpAll(IMG_HANDLE hBridge)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpAllKM();
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIDumpProcess(IMG_HANDLE hBridge, IMG_PID ui32Pid)
+{
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = RIDumpProcessKM(ui32Pid);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeRIWritePMREntryWithOwner(IMG_HANDLE hBridge,
+							 IMG_HANDLE hPMRHandle, IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMRHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psPMRHandleInt = (PMR *) hPMRHandle;
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRHandleInt, ui32Owner);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/client_sync_bridge.h b/drivers/gpu/drm/img-rogue/client_sync_bridge.h
new file mode 100644
index 000000000000..19f1b0ece5b9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_sync_bridge.h
@@ -0,0 +1,102 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNC_BRIDGE_H
+#define CLIENT_SYNC_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_sync_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence);
+
+#endif /* CLIENT_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_sync_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_sync_direct_bridge.c
new file mode 100644
index 000000000000..d631aea421ad
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_sync_direct_bridge.c
@@ -0,0 +1,262 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for sync
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_sync_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeAllocSyncPrimitiveBlock(IMG_HANDLE hBridge,
+							IMG_HANDLE * phSyncHandle,
+							IMG_UINT32 * pui32SyncPrimVAddr,
+							IMG_UINT32 * pui32SyncPrimBlockSize,
+							IMG_HANDLE * phhSyncPMR)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+					    &psSyncHandleInt,
+					    pui32SyncPrimVAddr,
+					    pui32SyncPrimBlockSize, &pshSyncPMRInt);
+
+	*phSyncHandle = psSyncHandleInt;
+	*phhSyncPMR = pshSyncPMRInt;
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeFreeSyncPrimitiveBlock(IMG_HANDLE hBridge, IMG_HANDLE hSyncHandle)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimSet(IMG_HANDLE hBridge,
+					    IMG_HANDLE hSyncHandle,
+					    IMG_UINT32 ui32Index, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimSetKM(psSyncHandleInt, ui32Index, ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDump(IMG_HANDLE hBridge,
+					      IMG_HANDLE hSyncHandle, IMG_UINT32 ui32Offset)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpKM(psSyncHandleInt, ui32Offset);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpValue(IMG_HANDLE hBridge,
+						   IMG_HANDLE hSyncHandle,
+						   IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError = PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt, ui32Offset, ui32Value);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpPol(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value,
+						 IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiPDumpFlags)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     ui32Offset, ui32Value, ui32Mask, eOperator, uiPDumpFlags);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncPrimPDumpCBP(IMG_HANDLE hBridge,
+						 IMG_HANDLE hSyncHandle,
+						 IMG_UINT32 ui32Offset,
+						 IMG_DEVMEM_OFFSET_T uiWriteOffset,
+						 IMG_DEVMEM_SIZE_T uiPacketSize,
+						 IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	psSyncHandleInt = (SYNC_PRIMITIVE_BLOCK *) hSyncHandle;
+
+	eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     ui32Offset, uiWriteOffset, uiPacketSize, uiBufferSize);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hSyncHandle);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncAllocEvent(IMG_HANDLE hBridge,
+					       IMG_BOOL bServerSync,
+					       IMG_UINT32 ui32FWAddr,
+					       IMG_UINT32 ui32ClassNameSize,
+					       const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+
+	eError =
+	    PVRSRVSyncAllocEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				   bServerSync, ui32FWAddr, ui32ClassNameSize, puiClassName);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncFreeEvent(IMG_HANDLE hBridge, IMG_UINT32 ui32FWAddr)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVSyncFreeEventKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge), ui32FWAddr);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncCheckpointSignalledPDumpPol(IMG_HANDLE hBridge,
+								PVRSRV_FENCE hFence)
+{
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	eError = PVRSRVSyncCheckpointSignalledPDumpPolKM(hFence);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+	PVR_UNREFERENCED_PARAMETER(hFence);
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+#endif
+}
diff --git a/drivers/gpu/drm/img-rogue/client_synctracking_bridge.h b/drivers/gpu/drm/img-rogue/client_synctracking_bridge.h
new file mode 100644
index 000000000000..544efd962834
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_synctracking_bridge.h
@@ -0,0 +1,68 @@
+/*******************************************************************************
+@File
+@Title          Client bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exports the client bridge functions for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef CLIENT_SYNCTRACKING_BRIDGE_H
+#define CLIENT_SYNCTRACKING_BRIDGE_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(PVR_INDIRECT_BRIDGE_CLIENTS)
+#include "pvr_bridge_client.h"
+#include "pvr_bridge.h"
+#endif
+
+#include "common_synctracking_bridge.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord);
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName);
+
+#endif /* CLIENT_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/client_synctracking_direct_bridge.c b/drivers/gpu/drm/img-rogue/client_synctracking_direct_bridge.c
new file mode 100644
index 000000000000..baeb89a6a0ae
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/client_synctracking_direct_bridge.c
@@ -0,0 +1,92 @@
+/*******************************************************************************
+@File
+@Title          Direct client bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the client side of the bridge for synctracking
+                which is used in calls from Server context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include "client_synctracking_bridge.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+/* Module specific includes */
+
+#include "sync.h"
+#include "sync_server.h"
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordRemoveByHandle(IMG_HANDLE hBridge, IMG_HANDLE hhRecord)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt;
+	PVR_UNREFERENCED_PARAMETER(hBridge);
+
+	pshRecordInt = (SYNC_RECORD_HANDLE) hhRecord;
+
+	eError = PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR BridgeSyncRecordAdd(IMG_HANDLE hBridge,
+					      IMG_HANDLE * phhRecord,
+					      IMG_HANDLE hhServerSyncPrimBlock,
+					      IMG_UINT32 ui32ui32FwBlockAddr,
+					      IMG_UINT32 ui32ui32SyncOffset,
+					      IMG_BOOL bbServerSync,
+					      IMG_UINT32 ui32ClassNameSize,
+					      const IMG_CHAR * puiClassName)
+{
+	PVRSRV_ERROR eError;
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt;
+
+	pshServerSyncPrimBlockInt = (SYNC_PRIMITIVE_BLOCK *) hhServerSyncPrimBlock;
+
+	eError =
+	    PVRSRVSyncRecordAddKM(NULL, (PVRSRV_DEVICE_NODE *) ((void *)hBridge),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  ui32ui32FwBlockAddr,
+				  ui32ui32SyncOffset,
+				  bbServerSync, ui32ClassNameSize, puiClassName);
+
+	*phhRecord = pshRecordInt;
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/common_cache_bridge.h b/drivers/gpu/drm/img-rogue/common_cache_bridge.h
new file mode 100644
index 000000000000..cc848753f7fa
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_cache_bridge.h
@@ -0,0 +1,126 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CACHE_BRIDGE_H
+#define COMMON_CACHE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_CACHE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE			PVRSRV_BRIDGE_CACHE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CACHE_CACHEOPEXEC			PVRSRV_BRIDGE_CACHE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CACHE_CACHEOPLOG			PVRSRV_BRIDGE_CACHE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CACHE_CMD_LAST			(PVRSRV_BRIDGE_CACHE_CMD_FIRST+2)
+
+/*******************************************
+            CacheOpQueue
+ *******************************************/
+
+/* Bridge in structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPQUEUE_TAG
+{
+	PVRSRV_CACHE_OP *piuCacheOp;
+	IMG_UINT64 *pui64Address;
+	IMG_DEVMEM_OFFSET_T *puiOffset;
+	IMG_DEVMEM_SIZE_T *puiSize;
+	IMG_HANDLE *phPMR;
+	IMG_UINT32 ui32NumCacheOps;
+	IMG_UINT32 ui32OpTimeline;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPQUEUE;
+
+/* Bridge out structure for CacheOpQueue */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPQUEUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPQUEUE;
+
+/*******************************************
+            CacheOpExec
+ *******************************************/
+
+/* Bridge in structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPEXEC_TAG
+{
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPEXEC;
+
+/* Bridge out structure for CacheOpExec */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPEXEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPEXEC;
+
+/*******************************************
+            CacheOpLog
+ *******************************************/
+
+/* Bridge in structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_IN_CACHEOPLOG_TAG
+{
+	IMG_INT64 i64EndTime;
+	IMG_INT64 i64StartTime;
+	IMG_UINT64 ui64Address;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_CACHE_OP iuCacheOp;
+} __packed PVRSRV_BRIDGE_IN_CACHEOPLOG;
+
+/* Bridge out structure for CacheOpLog */
+typedef struct PVRSRV_BRIDGE_OUT_CACHEOPLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CACHEOPLOG;
+
+#endif /* COMMON_CACHE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_cmm_bridge.h b/drivers/gpu/drm/img-rogue/common_cmm_bridge.h
new file mode 100644
index 000000000000..da48de3598be
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_cmm_bridge.h
@@ -0,0 +1,114 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_CMM_BRIDGE_H
+#define COMMON_CMM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_CMM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX			PVRSRV_BRIDGE_CMM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_CMM_CMD_LAST			(PVRSRV_BRIDGE_CMM_CMD_FIRST+2)
+
+/*******************************************
+            DevmemIntExportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX;
+
+/* Bridge out structure for DevmemIntExportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX;
+
+/*******************************************
+            DevmemIntUnexportCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX_TAG
+{
+	IMG_HANDLE hContextExport;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX;
+
+/* Bridge out structure for DevmemIntUnexportCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX;
+
+/*******************************************
+            DevmemIntAcquireRemoteCtx
+ *******************************************/
+
+/* Bridge in structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX;
+
+/* Bridge out structure for DevmemIntAcquireRemoteCtx */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX;
+
+#endif /* COMMON_CMM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_devicememhistory_bridge.h b/drivers/gpu/drm/img-rogue/common_devicememhistory_bridge.h
new file mode 100644
index 000000000000..800f98dd2836
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_devicememhistory_bridge.h
@@ -0,0 +1,185 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DEVICEMEMHISTORY_BRIDGE_H
+#define COMMON_DEVICEMEMHISTORY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE			PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST			(PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_FIRST+4)
+
+/*******************************************
+            DevicememHistoryMap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP;
+
+/* Bridge out structure for DevicememHistoryMap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP;
+
+/*******************************************
+            DevicememHistoryUnmap
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP;
+
+/* Bridge out structure for DevicememHistoryUnmap */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP;
+
+/*******************************************
+            DevicememHistoryMapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryMapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE;
+
+/*******************************************
+            DevicememHistoryUnmapVRange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiAllocSize;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 ui32ui32StartPage;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/* Bridge out structure for DevicememHistoryUnmapVRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE;
+
+/*******************************************
+            DevicememHistorySparseChange
+ *******************************************/
+
+/* Bridge in structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	const IMG_CHAR *puiText;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32AllocationIndex;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32Log2PageSize;
+} __packed PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE;
+
+/* Bridge out structure for DevicememHistorySparseChange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32AllocationIndexOut;
+} __packed PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE;
+
+#endif /* COMMON_DEVICEMEMHISTORY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_di_bridge.h b/drivers/gpu/drm/img-rogue/common_di_bridge.h
new file mode 100644
index 000000000000..8591006140b6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_di_bridge.h
@@ -0,0 +1,153 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DI_BRIDGE_H
+#define COMMON_DI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "pvr_dicommon.h"
+
+#define PVRSRV_BRIDGE_DI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DI_DICREATECONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT			PVRSRV_BRIDGE_DI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DI_DIREADENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DI_DIWRITEENTRY			PVRSRV_BRIDGE_DI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DI_DILISTALLENTRIES			PVRSRV_BRIDGE_DI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_DI_CMD_LAST			(PVRSRV_BRIDGE_DI_CMD_FIRST+4)
+
+/*******************************************
+            DICreateContext
+ *******************************************/
+
+/* Bridge in structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_IN_DICREATECONTEXT_TAG
+{
+	IMG_CHAR *puiStreamName;
+} __packed PVRSRV_BRIDGE_IN_DICREATECONTEXT;
+
+/* Bridge out structure for DICreateContext */
+typedef struct PVRSRV_BRIDGE_OUT_DICREATECONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+	IMG_CHAR *puiStreamName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DICREATECONTEXT;
+
+/*******************************************
+            DIDestroyContext
+ *******************************************/
+
+/* Bridge in structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT;
+
+/* Bridge out structure for DIDestroyContext */
+typedef struct PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT;
+
+/*******************************************
+            DIReadEntry
+ *******************************************/
+
+/* Bridge in structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIREADENTRY_TAG
+{
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+} __packed PVRSRV_BRIDGE_IN_DIREADENTRY;
+
+/* Bridge out structure for DIReadEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIREADENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIREADENTRY;
+
+/*******************************************
+            DIWriteEntry
+ *******************************************/
+
+/* Bridge in structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_IN_DIWRITEENTRY_TAG
+{
+	IMG_HANDLE hContext;
+	const IMG_CHAR *puiEntryPath;
+	const IMG_CHAR *puiValue;
+	IMG_UINT32 ui32ValueSize;
+} __packed PVRSRV_BRIDGE_IN_DIWRITEENTRY;
+
+/* Bridge out structure for DIWriteEntry */
+typedef struct PVRSRV_BRIDGE_OUT_DIWRITEENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DIWRITEENTRY;
+
+/*******************************************
+            DIListAllEntries
+ *******************************************/
+
+/* Bridge in structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_IN_DILISTALLENTRIES_TAG
+{
+	IMG_HANDLE hContext;
+} __packed PVRSRV_BRIDGE_IN_DILISTALLENTRIES;
+
+/* Bridge out structure for DIListAllEntries */
+typedef struct PVRSRV_BRIDGE_OUT_DILISTALLENTRIES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DILISTALLENTRIES;
+
+#endif /* COMMON_DI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_dmabuf_bridge.h b/drivers/gpu/drm/img-rogue/common_dmabuf_bridge.h
new file mode 100644
index 000000000000..cc631e3e1116
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_dmabuf_bridge.h
@@ -0,0 +1,145 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_DMABUF_BRIDGE_H
+#define COMMON_DMABUF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_DMABUF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF			PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_DMABUF_CMD_LAST			(PVRSRV_BRIDGE_DMABUF_CMD_FIRST+3)
+
+/*******************************************
+            PhysmemImportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF;
+
+/* Bridge out structure for PhysmemImportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF;
+
+/*******************************************
+            PhysmemExportDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF;
+
+/* Bridge out structure for PhysmemExportDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF;
+
+/*******************************************
+            PhysmemExportGemHandle
+ *******************************************/
+
+/* Bridge in structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE;
+
+/* Bridge out structure for PhysmemExportGemHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Handle;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE;
+
+/*******************************************
+            PhysmemImportSparseDmaBuf
+ *******************************************/
+
+/* Bridge in structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_SIZE_T uiChunkSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiName;
+	IMG_INT ifd;
+	IMG_UINT32 ui32NameSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF;
+
+/* Bridge out structure for PhysmemImportSparseDmaBuf */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF;
+
+#endif /* COMMON_DMABUF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_htbuffer_bridge.h b/drivers/gpu/drm/img-rogue/common_htbuffer_bridge.h
new file mode 100644
index 000000000000..32be202186dc
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_htbuffer_bridge.h
@@ -0,0 +1,82 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_HTBUFFER_BRIDGE_H
+#define COMMON_HTBUFFER_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "htbuffer_types.h"
+
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST			0
+#define PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL			PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0
+#define PVRSRV_BRIDGE_HTBUFFER_CMD_LAST			(PVRSRV_BRIDGE_HTBUFFER_CMD_FIRST+0)
+
+/*******************************************
+            HTBControl
+ *******************************************/
+
+/* Bridge in structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_IN_HTBCONTROL_TAG
+{
+	IMG_UINT32 *pui32GroupEnable;
+	IMG_UINT32 ui32EnablePID;
+	IMG_UINT32 ui32LogLevel;
+	IMG_UINT32 ui32LogMode;
+	IMG_UINT32 ui32NumGroups;
+	IMG_UINT32 ui32OpMode;
+} __packed PVRSRV_BRIDGE_IN_HTBCONTROL;
+
+/* Bridge out structure for HTBControl */
+typedef struct PVRSRV_BRIDGE_OUT_HTBCONTROL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HTBCONTROL;
+
+#endif /* COMMON_HTBUFFER_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_mm_bridge.h b/drivers/gpu/drm/img-rogue/common_mm_bridge.h
new file mode 100644
index 000000000000..80fa1baeb390
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_mm_bridge.h
@@ -0,0 +1,737 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MM_BRIDGE_H
+#define COMMON_MM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MM_PMREXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+1
+#define PVRSRV_BRIDGE_MM_PMRGETUID			PVRSRV_BRIDGE_MM_CMD_FIRST+2
+#define PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+3
+#define PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE			PVRSRV_BRIDGE_MM_CMD_FIRST+4
+#define PVRSRV_BRIDGE_MM_PMRIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+5
+#define PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+6
+#define PVRSRV_BRIDGE_MM_PMRUNREFPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+7
+#define PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+8
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+9
+#define PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+10
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE			PVRSRV_BRIDGE_MM_CMD_FIRST+11
+#define PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY			PVRSRV_BRIDGE_MM_CMD_FIRST+12
+#define PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+13
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+14
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+15
+#define PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR			PVRSRV_BRIDGE_MM_CMD_FIRST+16
+#define PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+17
+#define PVRSRV_BRIDGE_MM_CHANGESPARSEMEM			PVRSRV_BRIDGE_MM_CMD_FIRST+18
+#define PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID			PVRSRV_BRIDGE_MM_CMD_FIRST+19
+#define PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE			PVRSRV_BRIDGE_MM_CMD_FIRST+20
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+21
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT			PVRSRV_BRIDGE_MM_CMD_FIRST+22
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME			PVRSRV_BRIDGE_MM_CMD_FIRST+23
+#define PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS			PVRSRV_BRIDGE_MM_CMD_FIRST+24
+#define PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM			PVRSRV_BRIDGE_MM_CMD_FIRST+25
+#define PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO			PVRSRV_BRIDGE_MM_CMD_FIRST+26
+#define PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP			PVRSRV_BRIDGE_MM_CMD_FIRST+27
+#define PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS			PVRSRV_BRIDGE_MM_CMD_FIRST+28
+#define PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT			PVRSRV_BRIDGE_MM_CMD_FIRST+29
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+30
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE			PVRSRV_BRIDGE_MM_CMD_FIRST+31
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+32
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES			PVRSRV_BRIDGE_MM_CMD_FIRST+33
+#define PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE			PVRSRV_BRIDGE_MM_CMD_FIRST+34
+#define PVRSRV_BRIDGE_MM_CMD_LAST			(PVRSRV_BRIDGE_MM_CMD_FIRST+34)
+
+/*******************************************
+            PMRExportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMREXPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMREXPORTPMR;
+
+/* Bridge out structure for PMRExportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMREXPORTPMR_TAG
+{
+	IMG_UINT64 ui64Password;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRExport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2Contig;
+} __packed PVRSRV_BRIDGE_OUT_PMREXPORTPMR;
+
+/*******************************************
+            PMRUnexportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR_TAG
+{
+	IMG_HANDLE hPMRExport;
+} __packed PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR;
+
+/* Bridge out structure for PMRUnexportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR;
+
+/*******************************************
+            PMRGetUID
+ *******************************************/
+
+/* Bridge in structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_IN_PMRGETUID_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRGETUID;
+
+/* Bridge out structure for PMRGetUID */
+typedef struct PVRSRV_BRIDGE_OUT_PMRGETUID_TAG
+{
+	IMG_UINT64 ui64UID;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRGETUID;
+
+/*******************************************
+            PMRMakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hBuffer;
+} __packed PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRMakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRUnmakeLocalImportHandle
+ *******************************************/
+
+/* Bridge in structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	IMG_HANDLE hExtMem;
+} __packed PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE;
+
+/* Bridge out structure for PMRUnmakeLocalImportHandle */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE;
+
+/*******************************************
+            PMRImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRIMPORTPMR_TAG
+{
+	IMG_UINT64 ui64uiPassword;
+	IMG_UINT64 ui64uiSize;
+	IMG_HANDLE hPMRExport;
+	IMG_UINT32 ui32uiLog2Contig;
+} __packed PVRSRV_BRIDGE_IN_PMRIMPORTPMR;
+
+/* Bridge out structure for PMRImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRIMPORTPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRIMPORTPMR;
+
+/*******************************************
+            PMRLocalImportPMR
+ *******************************************/
+
+/* Bridge in structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_HANDLE hExtHandle;
+} __packed PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR;
+
+/* Bridge out structure for PMRLocalImportPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR_TAG
+{
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR;
+
+/*******************************************
+            PMRUnrefPMR
+ *******************************************/
+
+/* Bridge in structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_IN_PMRUNREFPMR_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_PMRUNREFPMR;
+
+/* Bridge out structure for PMRUnrefPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PMRUNREFPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PMRUNREFPMR;
+
+/*******************************************
+            PhysmemNewRamBackedPMR
+ *******************************************/
+
+/* Bridge in structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 *pui32MappingTable;
+	const IMG_CHAR *puiAnnotation;
+	IMG_UINT32 ui32AnnotationLength;
+	IMG_UINT32 ui32Log2PageSize;
+	IMG_UINT32 ui32NumPhysChunks;
+	IMG_UINT32 ui32NumVirtChunks;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_PID ui32PID;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR;
+
+/* Bridge out structure for PhysmemNewRamBackedPMR */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR;
+
+/*******************************************
+            DevmemIntCtxCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_BOOL bbKernelMemoryCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE;
+
+/* Bridge out structure for DevmemIntCtxCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE_TAG
+{
+	IMG_HANDLE hDevMemServerContext;
+	IMG_HANDLE hPrivData;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CPUCacheLineSize;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE;
+
+/*******************************************
+            DevmemIntCtxDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY_TAG
+{
+	IMG_HANDLE hDevmemServerContext;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY;
+
+/* Bridge out structure for DevmemIntCtxDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY;
+
+/*******************************************
+            DevmemIntHeapCreate
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE;
+
+/* Bridge out structure for DevmemIntHeapCreate */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE_TAG
+{
+	IMG_HANDLE hDevmemHeapPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE;
+
+/*******************************************
+            DevmemIntHeapDestroy
+ *******************************************/
+
+/* Bridge in structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY_TAG
+{
+	IMG_HANDLE hDevmemHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY;
+
+/* Bridge out structure for DevmemIntHeapDestroy */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY;
+
+/*******************************************
+            DevmemIntMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR;
+
+/* Bridge out structure for DevmemIntMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR;
+
+/*******************************************
+            DevmemIntUnmapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR;
+
+/* Bridge out structure for DevmemIntUnmapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR;
+
+/*******************************************
+            DevmemIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE;
+
+/* Bridge out structure for DevmemIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE;
+
+/*******************************************
+            DevmemIntReserveRangeAndMapPMR
+ *******************************************/
+
+/* Bridge in structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+	IMG_HANDLE hPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/* Bridge out structure for DevmemIntReserveRangeAndMapPMR */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR;
+
+/*******************************************
+            DevmemIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE;
+
+/*******************************************
+            ChangeSparseMem
+ *******************************************/
+
+/* Bridge in structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_IN_CHANGESPARSEMEM_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 *pui32AllocPageIndices;
+	IMG_UINT32 *pui32FreePageIndices;
+	IMG_UINT32 ui32AllocPageCount;
+	IMG_UINT32 ui32FreePageCount;
+	IMG_UINT32 ui32SparseFlags;
+} __packed PVRSRV_BRIDGE_IN_CHANGESPARSEMEM;
+
+/* Bridge out structure for ChangeSparseMem */
+typedef struct PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM;
+
+/*******************************************
+            DevmemIsVDevAddrValid
+ *******************************************/
+
+/* Bridge in structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID;
+
+/* Bridge out structure for DevmemIsVDevAddrValid */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID;
+
+/*******************************************
+            DevmemInvalidateFBSCTable
+ *******************************************/
+
+/* Bridge in structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	IMG_UINT64 ui64FBSCEntries;
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE;
+
+/* Bridge out structure for DevmemInvalidateFBSCTable */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE;
+
+/*******************************************
+            HeapCfgHeapConfigCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT;
+
+/* Bridge out structure for HeapCfgHeapConfigCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeapConfigs;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT;
+
+/*******************************************
+            HeapCfgHeapCount
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT_TAG
+{
+	IMG_UINT32 ui32HeapConfigIndex;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT;
+
+/* Bridge out structure for HeapCfgHeapCount */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumHeaps;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT;
+
+/*******************************************
+            HeapCfgHeapConfigName
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapConfigNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME;
+
+/* Bridge out structure for HeapCfgHeapConfigName */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME_TAG
+{
+	IMG_CHAR *puiHeapConfigName;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME;
+
+/*******************************************
+            HeapCfgHeapDetails
+ *******************************************/
+
+/* Bridge in structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_CHAR *puiHeapNameOut;
+	IMG_UINT32 ui32HeapConfigIndex;
+	IMG_UINT32 ui32HeapIndex;
+	IMG_UINT32 ui32HeapNameBufSz;
+} __packed PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS;
+
+/* Bridge out structure for HeapCfgHeapDetails */
+typedef struct PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_CHAR *puiHeapNameOut;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2DataPageSizeOut;
+	IMG_UINT32 ui32Log2ImportAlignmentOut;
+} __packed PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS;
+
+/*******************************************
+            DevmemIntRegisterPFNotifyKM
+ *******************************************/
+
+/* Bridge in structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+	IMG_BOOL bRegister;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/* Bridge out structure for DevmemIntRegisterPFNotifyKM */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM;
+
+/*******************************************
+            PhysHeapGetMemInfo
+ *******************************************/
+
+/* Bridge in structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS *pasapPhysHeapMemStats;
+	PVRSRV_PHYS_HEAP *peaPhysHeapID;
+	IMG_UINT32 ui32PhysHeapCount;
+} __packed PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO;
+
+/* Bridge out structure for PhysHeapGetMemInfo */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO_TAG
+{
+	PHYS_HEAP_MEM_STATS *pasapPhysHeapMemStats;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO;
+
+/*******************************************
+            GetDefaultPhysicalHeap
+ *******************************************/
+
+/* Bridge in structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP;
+
+/* Bridge out structure for GetDefaultPhysicalHeap */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP eHeap;
+} __packed PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP;
+
+/*******************************************
+            DevmemGetFaultAddress
+ *******************************************/
+
+/* Bridge in structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_HANDLE hDevmemCtx;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS;
+
+/* Bridge out structure for DevmemGetFaultAddress */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS_TAG
+{
+	IMG_DEV_VIRTADDR sFaultAddress;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS;
+
+/*******************************************
+            PVRSRVStatsUpdateOOMStat
+ *******************************************/
+
+/* Bridge in structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	IMG_PID ui32pid;
+	IMG_UINT32 ui32ui32StatType;
+} __packed PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT;
+
+/* Bridge out structure for PVRSRVStatsUpdateOOMStat */
+typedef struct PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT;
+
+/*******************************************
+            DevmemXIntReserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_DEV_VIRTADDR sAddress;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_HANDLE hDevmemServerHeap;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntReserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE;
+
+/*******************************************
+            DevmemXIntUnreserveRange
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	IMG_HANDLE hReservation;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE;
+
+/* Bridge out structure for DevmemXIntUnreserveRange */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE;
+
+/*******************************************
+            DevmemXIntMapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32PhysPageOffset;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES;
+
+/* Bridge out structure for DevmemXIntMapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES;
+
+/*******************************************
+            DevmemXIntUnmapPages
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES;
+
+/* Bridge out structure for DevmemXIntUnmapPages */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES;
+
+/*******************************************
+            DevmemXIntMapVRangeToBackingPage
+ *******************************************/
+
+/* Bridge in structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	IMG_HANDLE hReservation;
+	IMG_UINT32 ui32PageCount;
+	IMG_UINT32 ui32VirtPageOffset;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+/* Bridge out structure for DevmemXIntMapVRangeToBackingPage */
+typedef struct PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE;
+
+#endif /* COMMON_MM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_mmextmem_bridge.h b/drivers/gpu/drm/img-rogue/common_mmextmem_bridge.h
new file mode 100644
index 000000000000..f52637853325
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_mmextmem_bridge.h
@@ -0,0 +1,80 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_MMEXTMEM_BRIDGE_H
+#define COMMON_MMEXTMEM_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST			0
+#define PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM			PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0
+#define PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST			(PVRSRV_BRIDGE_MMEXTMEM_CMD_FIRST+0)
+
+/*******************************************
+            PhysmemWrapExtMem
+ *******************************************/
+
+/* Bridge in structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_UINT64 ui64CpuVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+} __packed PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM;
+
+/* Bridge out structure for PhysmemWrapExtMem */
+typedef struct PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM_TAG
+{
+	IMG_HANDLE hPMRPtr;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM;
+
+#endif /* COMMON_MMEXTMEM_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_pvrtl_bridge.h b/drivers/gpu/drm/img-rogue/common_pvrtl_bridge.h
new file mode 100644
index 000000000000..edc822375222
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_pvrtl_bridge.h
@@ -0,0 +1,214 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_PVRTL_BRIDGE_H
+#define COMMON_PVRTL_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+
+#define PVRSRV_BRIDGE_PVRTL_CMD_FIRST			0
+#define PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+0
+#define PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+1
+#define PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+2
+#define PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+3
+#define PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+4
+#define PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+5
+#define PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+6
+#define PVRSRV_BRIDGE_PVRTL_TLWRITEDATA			PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7
+#define PVRSRV_BRIDGE_PVRTL_CMD_LAST			(PVRSRV_BRIDGE_PVRTL_CMD_FIRST+7)
+
+/*******************************************
+            TLOpenStream
+ *******************************************/
+
+/* Bridge in structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_IN_TLOPENSTREAM_TAG
+{
+	const IMG_CHAR *puiName;
+	IMG_UINT32 ui32Mode;
+} __packed PVRSRV_BRIDGE_IN_TLOPENSTREAM;
+
+/* Bridge out structure for TLOpenStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLOPENSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_HANDLE hTLPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLOPENSTREAM;
+
+/*******************************************
+            TLCloseStream
+ *******************************************/
+
+/* Bridge in structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCLOSESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLCLOSESTREAM;
+
+/* Bridge out structure for TLCloseStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCLOSESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCLOSESTREAM;
+
+/*******************************************
+            TLAcquireData
+ *******************************************/
+
+/* Bridge in structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_IN_TLACQUIREDATA_TAG
+{
+	IMG_HANDLE hSD;
+} __packed PVRSRV_BRIDGE_IN_TLACQUIREDATA;
+
+/* Bridge out structure for TLAcquireData */
+typedef struct PVRSRV_BRIDGE_OUT_TLACQUIREDATA_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLACQUIREDATA;
+
+/*******************************************
+            TLReleaseData
+ *******************************************/
+
+/* Bridge in structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_IN_TLRELEASEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReadLen;
+	IMG_UINT32 ui32ReadOffset;
+} __packed PVRSRV_BRIDGE_IN_TLRELEASEDATA;
+
+/* Bridge out structure for TLReleaseData */
+typedef struct PVRSRV_BRIDGE_OUT_TLRELEASEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLRELEASEDATA;
+
+/*******************************************
+            TLDiscoverStreams
+ *******************************************/
+
+/* Bridge in structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS_TAG
+{
+	const IMG_CHAR *puiNamePattern;
+	IMG_CHAR *puiStreams;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS;
+
+/* Bridge out structure for TLDiscoverStreams */
+typedef struct PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS_TAG
+{
+	IMG_CHAR *puiStreams;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumFound;
+} __packed PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS;
+
+/*******************************************
+            TLReserveStream
+ *******************************************/
+
+/* Bridge in structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_IN_TLRESERVESTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32SizeMin;
+} __packed PVRSRV_BRIDGE_IN_TLRESERVESTREAM;
+
+/* Bridge out structure for TLReserveStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLRESERVESTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Available;
+	IMG_UINT32 ui32BufferOffset;
+} __packed PVRSRV_BRIDGE_OUT_TLRESERVESTREAM;
+
+/*******************************************
+            TLCommitStream
+ *******************************************/
+
+/* Bridge in structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_IN_TLCOMMITSTREAM_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_UINT32 ui32ReqSize;
+} __packed PVRSRV_BRIDGE_IN_TLCOMMITSTREAM;
+
+/* Bridge out structure for TLCommitStream */
+typedef struct PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM;
+
+/*******************************************
+            TLWriteData
+ *******************************************/
+
+/* Bridge in structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_IN_TLWRITEDATA_TAG
+{
+	IMG_HANDLE hSD;
+	IMG_BYTE *pui8Data;
+	IMG_UINT32 ui32Size;
+} __packed PVRSRV_BRIDGE_IN_TLWRITEDATA;
+
+/* Bridge out structure for TLWriteData */
+typedef struct PVRSRV_BRIDGE_OUT_TLWRITEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_TLWRITEDATA;
+
+#endif /* COMMON_PVRTL_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxbreakpoint_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxbreakpoint_bridge.h
new file mode 100644
index 000000000000..ca5290a88266
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxbreakpoint_bridge.h
@@ -0,0 +1,150 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxbreakpoint
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxbreakpoint
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXBREAKPOINT_BRIDGE_H
+#define COMMON_RGXBREAKPOINT_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS			PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_LAST			(PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXSetBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT_TAG
+{
+	IMG_UINT64 ui64TempSpillingAddr;
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 eFWDataMaster;
+	IMG_UINT32 ui32BreakpointAddr;
+	IMG_UINT32 ui32DM;
+	IMG_UINT32 ui32HandlerAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT;
+
+/* Bridge out structure for RGXSetBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT;
+
+/*******************************************
+            RGXClearBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXClearBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT;
+
+/* Bridge out structure for RGXClearBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT;
+
+/*******************************************
+            RGXEnableBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXEnableBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT;
+
+/* Bridge out structure for RGXEnableBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT;
+
+/*******************************************
+            RGXDisableBreakpoint
+ *******************************************/
+
+/* Bridge in structure for RGXDisableBreakpoint */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT_TAG
+{
+	IMG_HANDLE hPrivData;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT;
+
+/* Bridge out structure for RGXDisableBreakpoint */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT;
+
+/*******************************************
+            RGXOverallocateBPRegisters
+ *******************************************/
+
+/* Bridge in structure for RGXOverallocateBPRegisters */
+typedef struct PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS_TAG
+{
+	IMG_UINT32 ui32SharedRegs;
+	IMG_UINT32 ui32TempRegs;
+} __packed PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS;
+
+/* Bridge out structure for RGXOverallocateBPRegisters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS;
+
+#endif /* COMMON_RGXBREAKPOINT_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxcmp_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxcmp_bridge.h
new file mode 100644
index 000000000000..aa1d2b73985a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxcmp_bridge.h
@@ -0,0 +1,309 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXCMP_BRIDGE_H
+#define COMMON_RGXCMP_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXCMP_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXCMP_CMD_LAST			(PVRSRV_BRIDGE_RGXCMP_CMD_FIRST+11)
+
+/*******************************************
+            RGXCreateComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticComputeContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+	IMG_UINT32 ui32StaticComputeContextStateSize;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT;
+
+/* Bridge out structure for RGXCreateComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT;
+
+/*******************************************
+            RGXDestroyComputeContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT;
+
+/* Bridge out structure for RGXDestroyComputeContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT;
+
+/*******************************************
+            RGXFlushComputeData
+ *******************************************/
+
+/* Bridge in structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA;
+
+/* Bridge out structure for RGXFlushComputeData */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA;
+
+/*******************************************
+            RGXSendCancelCmd
+ *******************************************/
+
+/* Bridge in structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32FirstIntJobRefToCancel;
+	IMG_INT32 i32LastIntJobRefToCancel;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD;
+
+/* Bridge out structure for RGXSendCancelCmd */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD;
+
+/*******************************************
+            RGXSetComputeContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetComputeContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY;
+
+/*******************************************
+            RGXNotifyComputeWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hComputeContext;
+} __packed PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXNotifyComputeWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXSetComputeContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetComputeContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY;
+
+/*******************************************
+            RGXGetLastDeviceError
+ *******************************************/
+
+/* Bridge in structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR;
+
+/* Bridge out structure for RGXGetLastDeviceError */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Error;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR;
+
+/*******************************************
+            RGXKickTimestampQuery
+ *******************************************/
+
+/* Bridge in structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY_TAG
+{
+	IMG_HANDLE hComputeContext;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY;
+
+/* Bridge out structure for RGXKickTimestampQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY;
+
+/*******************************************
+            RGXKickCDM
+ *******************************************/
+
+/* Bridge in structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKCDM_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hComputeContext;
+	IMG_UINT32 *pui32ClientUpdateOffset;
+	IMG_UINT32 *pui32ClientUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui8DMCmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phClientUpdateUFOSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFenceFd;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CmdSize;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumOfWorkgroups;
+	IMG_UINT32 ui32NumOfWorkitems;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKCDM;
+
+/* Bridge out structure for RGXKickCDM */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKCDM_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	IMG_UINT32 ui32IntJobRef;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKCDM;
+
+/*******************************************
+            RGXCDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXCDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXCDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY;
+
+#endif /* COMMON_RGXCMP_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxfwdbg_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxfwdbg_bridge.h
new file mode 100644
index 000000000000..534b8e9536ed
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxfwdbg_bridge.h
@@ -0,0 +1,339 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXFWDBG_BRIDGE_H
+#define COMMON_RGXFWDBG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "devicemem_typedefs.h"
+#include "rgx_bridge.h"
+#include "pvrsrv_memallocflags.h"
+
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT			PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15
+#define PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST			(PVRSRV_BRIDGE_RGXFWDBG_CMD_FIRST+15)
+
+/*******************************************
+            RGXFWDebugSetFWLog
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG_TAG
+{
+	IMG_UINT32 ui32RGXFWLogType;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG;
+
+/* Bridge out structure for RGXFWDebugSetFWLog */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG;
+
+/*******************************************
+            RGXFWDebugDumpFreelistPageList
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/* Bridge out structure for RGXFWDebugDumpFreelistPageList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST;
+
+/*******************************************
+            RGXFWDebugPowerOff
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF;
+
+/* Bridge out structure for RGXFWDebugPowerOff */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF;
+
+/*******************************************
+            RGXFWDebugPowerOn
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON;
+
+/* Bridge out structure for RGXFWDebugPowerOn */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON;
+
+/*******************************************
+            RGXFWDebugSetVzConnectionCooldownPeriodInSec
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	IMG_UINT32 ui32VzConne;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/* Bridge out structure for RGXFWDebugSetVzConnectionCooldownPeriodInSec */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC;
+
+/*******************************************
+            RGXFWDebugSetHCSDeadline
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	IMG_UINT32 ui32RGXHCSDeadline;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE;
+
+/* Bridge out structure for RGXFWDebugSetHCSDeadline */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE;
+
+/*******************************************
+            RGXFWDebugSetDriverPriority
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/* Bridge out structure for RGXFWDebugSetDriverPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSlice
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32TSPercentage;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSlice */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE;
+
+/*******************************************
+            RGXFWDebugSetDriverTimeSliceInterval
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	IMG_UINT32 ui32TSIntervalMs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/* Bridge out structure for RGXFWDebugSetDriverTimeSliceInterval */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL;
+
+/*******************************************
+            RGXFWDebugSetDriverIsolationGroup
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32IsolationGroup;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/* Bridge out structure for RGXFWDebugSetDriverIsolationGroup */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP;
+
+/*******************************************
+            RGXFWDebugSetOSNewOnlineState
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32OSNewState;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/* Bridge out structure for RGXFWDebugSetOSNewOnlineState */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE;
+
+/*******************************************
+            RGXFWDebugMapGuestHeap
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	IMG_UINT64 ui64ui64GuestHeapBase;
+	IMG_UINT32 ui32DriverID;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP;
+
+/* Bridge out structure for RGXFWDebugMapGuestHeap */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP;
+
+/*******************************************
+            RGXFWDebugPHRConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32PHRMode;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugPHRConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE;
+
+/*******************************************
+            RGXFWDebugWdgConfigure
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	IMG_UINT32 ui32ui32WdgPeriodUs;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE;
+
+/* Bridge out structure for RGXFWDebugWdgConfigure */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE;
+
+/*******************************************
+            RGXCurrentTime
+ *******************************************/
+
+/* Bridge in structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_IN_RGXCURRENTTIME_TAG
+{
+	IMG_UINT8 ui8TimerType;
+} __packed PVRSRV_BRIDGE_IN_RGXCURRENTTIME;
+
+/* Bridge out structure for RGXCurrentTime */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCURRENTTIME_TAG
+{
+	IMG_UINT64 ui64Time;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCURRENTTIME;
+
+/*******************************************
+            RGXFWDebugInjectFault
+ *******************************************/
+
+/* Bridge in structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXFWDEBUGINJECTFAULT;
+
+/* Bridge out structure for RGXFWDebugInjectFault */
+typedef struct PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT;
+
+#endif /* COMMON_RGXFWDBG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxhwperf_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxhwperf_bridge.h
new file mode 100644
index 000000000000..4d86216c8197
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxhwperf_bridge.h
@@ -0,0 +1,248 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXHWPERF_BRIDGE_H
+#define COMMON_RGXHWPERF_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_hwperf.h"
+
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS			PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST			(PVRSRV_BRIDGE_RGXHWPERF_CMD_FIRST+9)
+
+/*******************************************
+            RGXGetConfiguredHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS;
+
+/*******************************************
+            RGXGetEnabledHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS;
+
+/* Bridge out structure for RGXGetEnabledHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS_TAG
+{
+	IMG_UINT32 *pui32EnabledBlockIDs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32BlockCount;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetHWPerfTimeStamp
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP;
+
+/* Bridge out structure for RGXGetHWPerfTimeStamp */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP_TAG
+{
+	IMG_UINT64 ui64TimeStamp;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP;
+
+/*******************************************
+            RGXCtrlHWPerf
+ *******************************************/
+
+/* Bridge in structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_IN_RGXCTRLHWPERF_TAG
+{
+	IMG_UINT64 ui64Mask;
+	IMG_UINT32 ui32StreamId;
+	IMG_BOOL bToggle;
+} __packed PVRSRV_BRIDGE_IN_RGXCTRLHWPERF;
+
+/* Bridge out structure for RGXCtrlHWPerf */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF;
+
+/*******************************************
+            RGXGetHWPerfBvncFeatureFlags
+ *******************************************/
+
+/* Bridge in structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/* Bridge out structure for RGXGetHWPerfBvncFeatureFlags */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS_TAG
+{
+	RGX_HWPERF_BVNC sBVNC;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS;
+
+/*******************************************
+            RGXControlHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	IMG_UINT16 *pui16BlockIDs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_BOOL bEnable;
+} __packed PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS;
+
+/* Bridge out structure for RGXControlHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS;
+
+/*******************************************
+            RGXConfigMuxHWPerfCounters
+ *******************************************/
+
+/* Bridge in structure for RGXConfigMuxHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS;
+
+/* Bridge out structure for RGXConfigMuxHWPerfCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS;
+
+/*******************************************
+            RGXConfigCustomCounters
+ *******************************************/
+
+/* Bridge in structure for RGXConfigCustomCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS_TAG
+{
+	IMG_UINT32 *pui32CustomCounterIDs;
+	IMG_UINT16 ui16CustomBlockID;
+	IMG_UINT16 ui16NumCustomCounters;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS;
+
+/* Bridge out structure for RGXConfigCustomCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS;
+
+/*******************************************
+            RGXConfigureHWPerfBlocks
+ *******************************************/
+
+/* Bridge in structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigs;
+	IMG_UINT32 ui32ArrayLen;
+	IMG_UINT32 ui32CtrlWord;
+} __packed PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS;
+
+/* Bridge out structure for RGXConfigureHWPerfBlocks */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS;
+
+/*******************************************
+            RGXGetConfiguredHWPerfMuxCounters
+ *******************************************/
+
+/* Bridge in structure for RGXGetConfiguredHWPerfMuxCounters */
+typedef struct PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters;
+	IMG_UINT32 ui32BlockID;
+} __packed PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS;
+
+/* Bridge out structure for RGXGetConfiguredHWPerfMuxCounters */
+typedef struct PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS_TAG
+{
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS;
+
+#endif /* COMMON_RGXHWPERF_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxkicksync_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxkicksync_bridge.h
new file mode 100644
index 000000000000..afd882ca7529
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxkicksync_bridge.h
@@ -0,0 +1,143 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXKICKSYNC_BRIDGE_H
+#define COMMON_RGXKICKSYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST			(PVRSRV_BRIDGE_RGXKICKSYNC_CMD_FIRST+3)
+
+/*******************************************
+            RGXCreateKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hPrivData;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXCreateKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXDestroyKickSyncContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT;
+
+/* Bridge out structure for RGXDestroyKickSyncContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT;
+
+/*******************************************
+            RGXKickSync2
+ *******************************************/
+
+/* Bridge in structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKSYNC2_TAG
+{
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 *pui32UpdateDevVarOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phUpdateUFODevVarBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hTimelineFenceFD;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKSYNC2;
+
+/* Bridge out structure for RGXKickSync2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKSYNC2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFenceFD;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKSYNC2;
+
+/*******************************************
+            RGXSetKickSyncContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hKickSyncContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetKickSyncContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY;
+
+#endif /* COMMON_RGXKICKSYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxregconfig_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxregconfig_bridge.h
new file mode 100644
index 000000000000..942b7e4b06d0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxregconfig_bridge.h
@@ -0,0 +1,146 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXREGCONFIG_BRIDGE_H
+#define COMMON_RGXREGCONFIG_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG			PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST			(PVRSRV_BRIDGE_RGXREGCONFIG_CMD_FIRST+4)
+
+/*******************************************
+            RGXSetRegConfigType
+ *******************************************/
+
+/* Bridge in structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE_TAG
+{
+	IMG_UINT8 ui8RegPowerIsland;
+} __packed PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE;
+
+/* Bridge out structure for RGXSetRegConfigType */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE;
+
+/*******************************************
+            RGXAddRegconfig
+ *******************************************/
+
+/* Bridge in structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXADDREGCONFIG_TAG
+{
+	IMG_UINT64 ui64RegMask;
+	IMG_UINT64 ui64RegValue;
+	IMG_UINT32 ui32RegAddr;
+} __packed PVRSRV_BRIDGE_IN_RGXADDREGCONFIG;
+
+/* Bridge out structure for RGXAddRegconfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG;
+
+/*******************************************
+            RGXClearRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG;
+
+/* Bridge out structure for RGXClearRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG;
+
+/*******************************************
+            RGXEnableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG;
+
+/* Bridge out structure for RGXEnableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG;
+
+/*******************************************
+            RGXDisableRegConfig
+ *******************************************/
+
+/* Bridge in structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG;
+
+/* Bridge out structure for RGXDisableRegConfig */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG;
+
+#endif /* COMMON_RGXREGCONFIG_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxta3d_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxta3d_bridge.h
new file mode 100644
index 000000000000..849d18cc78a8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxta3d_bridge.h
@@ -0,0 +1,422 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTA3D_BRIDGE_H
+#define COMMON_RGXTA3D_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "rgx_fwif_shared.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+9
+#define PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+10
+#define PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+11
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+12
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+13
+#define PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT			PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+14
+#define PVRSRV_BRIDGE_RGXTA3D_CMD_LAST			(PVRSRV_BRIDGE_RGXTA3D_CMD_FIRST+14)
+
+/*******************************************
+            RGXDestroyHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET_TAG
+{
+	IMG_HANDLE hKmHwRTDataSet;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET;
+
+/* Bridge out structure for RGXDestroyHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET;
+
+/*******************************************
+            RGXCreateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hPMR;
+	IMG_HANDLE hReservation;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER;
+
+/* Bridge out structure for RGXCreateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferMemDesc;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER;
+
+/* Bridge out structure for RGXDestroyZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER;
+
+/*******************************************
+            RGXPopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsZSBufferKM;
+} __packed PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXPopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXUnpopulateZSBuffer
+ *******************************************/
+
+/* Bridge in structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER_TAG
+{
+	IMG_HANDLE hsPopulation;
+} __packed PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER;
+
+/* Bridge out structure for RGXUnpopulateZSBuffer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER;
+
+/*******************************************
+            RGXDestroyFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST;
+
+/* Bridge out structure for RGXDestroyFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST;
+
+/*******************************************
+            RGXDestroyRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT;
+
+/* Bridge out structure for RGXDestroyRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT;
+
+/*******************************************
+            RGXSendZSStoreDisable
+ *******************************************/
+
+/* Bridge in structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32ExtJobRefToDisableZSStore;
+	IMG_BOOL bDisableDepthStore;
+	IMG_BOOL bDisableStencilStore;
+} __packed PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE;
+
+/* Bridge out structure for RGXSendZSStoreDisable */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE;
+
+/*******************************************
+            RGXSetRenderContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hRenderContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetRenderContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXRenderContextStalled
+ *******************************************/
+
+/* Bridge in structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED_TAG
+{
+	IMG_HANDLE hRenderContext;
+} __packed PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED;
+
+/* Bridge out structure for RGXRenderContextStalled */
+typedef struct PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED;
+
+/*******************************************
+            RGXKickTA3D2
+ *******************************************/
+
+/* Bridge in structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXKICKTA3D2_TAG
+{
+	IMG_UINT64 ui64Deadline;
+	IMG_HANDLE hKMHWRTDataSet;
+	IMG_HANDLE hMSAAScratchBuffer;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock;
+	IMG_HANDLE hRenderContext;
+	IMG_HANDLE hZSBuffer;
+	IMG_UINT32 *pui32Client3DUpdateSyncOffset;
+	IMG_UINT32 *pui32Client3DUpdateValue;
+	IMG_UINT32 *pui32ClientTAFenceSyncOffset;
+	IMG_UINT32 *pui32ClientTAFenceValue;
+	IMG_UINT32 *pui32ClientTAUpdateSyncOffset;
+	IMG_UINT32 *pui32ClientTAUpdateValue;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_BYTE *pui83DCmd;
+	IMG_BYTE *pui83DPRCmd;
+	IMG_BYTE *pui8TACmd;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_CHAR *puiUpdateFenceName3D;
+	IMG_HANDLE *phClient3DUpdateSyncPrimBlock;
+	IMG_HANDLE *phClientTAFenceSyncPrimBlock;
+	IMG_HANDLE *phClientTAUpdateSyncPrimBlock;
+	IMG_HANDLE *phSyncPMRs;
+	PVRSRV_FENCE hCheckFence;
+	PVRSRV_FENCE hCheckFence3D;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	PVRSRV_TIMELINE hUpdateTimeline3D;
+	IMG_UINT32 ui323DCmdSize;
+	IMG_UINT32 ui323DPRCmdSize;
+	IMG_UINT32 ui32Client3DUpdateCount;
+	IMG_UINT32 ui32ClientTAFenceCount;
+	IMG_UINT32 ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32NumberOfDrawCalls;
+	IMG_UINT32 ui32NumberOfIndices;
+	IMG_UINT32 ui32NumberOfMRTs;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32PRFenceUFOSyncOffset;
+	IMG_UINT32 ui32PRFenceValue;
+	IMG_UINT32 ui32RenderTargetSize;
+	IMG_UINT32 ui32SyncPMRCount;
+	IMG_UINT32 ui32TACmdSize;
+	IMG_BOOL bbAbort;
+	IMG_BOOL bbKick3D;
+	IMG_BOOL bbKickPR;
+	IMG_BOOL bbKickTA;
+} __packed PVRSRV_BRIDGE_IN_RGXKICKTA3D2;
+
+/* Bridge out structure for RGXKickTA3D2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXKICKTA3D2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+	PVRSRV_FENCE hUpdateFence3D;
+} __packed PVRSRV_BRIDGE_OUT_RGXKICKTA3D2;
+
+/*******************************************
+            RGXSetRenderContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hRenderContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetRenderContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXCreateHWRTDataSet
+ *******************************************/
+
+/* Bridge in structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_UINT64 ui64FlippedMultiSampleCtl;
+	IMG_UINT64 ui64MultiSampleCtl;
+	IMG_DEV_VIRTADDR *psMacrotileArrayDevVAddr;
+	IMG_DEV_VIRTADDR *psPMMlistDevVAddr;
+	IMG_DEV_VIRTADDR *psRTCDevVAddr;
+	IMG_DEV_VIRTADDR *psRgnHeaderDevVAddr;
+	IMG_DEV_VIRTADDR *psTailPtrsDevVAddr;
+	IMG_DEV_VIRTADDR *psVHeapTableDevVAddr;
+	IMG_HANDLE *phKmHwRTDataSet;
+	IMG_HANDLE *phapsFreeLists;
+	IMG_UINT32 ui32ISPMergeLowerX;
+	IMG_UINT32 ui32ISPMergeLowerY;
+	IMG_UINT32 ui32ISPMergeScaleX;
+	IMG_UINT32 ui32ISPMergeScaleY;
+	IMG_UINT32 ui32ISPMergeUpperX;
+	IMG_UINT32 ui32ISPMergeUpperY;
+	IMG_UINT32 ui32ISPMtileSize;
+	IMG_UINT32 ui32MTileStride;
+	IMG_UINT32 ui32PPPScreen;
+	IMG_UINT32 ui32RgnHeaderSize;
+	IMG_UINT32 ui32TEAA;
+	IMG_UINT32 ui32TEMTILE1;
+	IMG_UINT32 ui32TEMTILE2;
+	IMG_UINT32 ui32TEScreen;
+	IMG_UINT32 ui32TPCSize;
+	IMG_UINT32 ui32TPCStride;
+	IMG_UINT16 ui16MaxRTs;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET;
+
+/* Bridge out structure for RGXCreateHWRTDataSet */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET_TAG
+{
+	IMG_HANDLE *phKmHwRTDataSet;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET;
+
+/*******************************************
+            RGXCreateFreeList
+ *******************************************/
+
+/* Bridge in structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hFreeListReservation;
+	IMG_HANDLE hMemCtxPrivData;
+	IMG_HANDLE hsGlobalFreeList;
+	IMG_UINT32 ui32GrowFLPages;
+	IMG_UINT32 ui32GrowParamThreshold;
+	IMG_UINT32 ui32InitFLPages;
+	IMG_UINT32 ui32MaxFLPages;
+	IMG_BOOL bbFreeListCheck;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATEFREELIST;
+
+/* Bridge out structure for RGXCreateFreeList */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST_TAG
+{
+	IMG_HANDLE hCleanupCookie;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST;
+
+/*******************************************
+            RGXCreateRenderContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_DEV_VIRTADDR sVDMCallStackAddr;
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_BYTE *pui8StaticRenderContextState;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32Max3DDeadlineMS;
+	IMG_UINT32 ui32MaxTADeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+	IMG_UINT32 ui32StaticRenderContextStateSize;
+	IMG_UINT32 ui32ui32CallStackDepth;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT;
+
+/* Bridge out structure for RGXCreateRenderContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT_TAG
+{
+	IMG_HANDLE hRenderContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT;
+
+#endif /* COMMON_RGXTA3D_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxtimerquery_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxtimerquery_bridge.h
new file mode 100644
index 000000000000..34d7c2c56a2c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxtimerquery_bridge.h
@@ -0,0 +1,112 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTIMERQUERY_BRIDGE_H
+#define COMMON_RGXTIMERQUERY_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER			PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST			(PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_FIRST+2)
+
+/*******************************************
+            RGXBeginTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY;
+
+/* Bridge out structure for RGXBeginTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY;
+
+/*******************************************
+            RGXEndTimerQuery
+ *******************************************/
+
+/* Bridge in structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY;
+
+/* Bridge out structure for RGXEndTimerQuery */
+typedef struct PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY;
+
+/*******************************************
+            RGXQueryTimer
+ *******************************************/
+
+/* Bridge in structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_IN_RGXQUERYTIMER_TAG
+{
+	IMG_UINT32 ui32QueryId;
+} __packed PVRSRV_BRIDGE_IN_RGXQUERYTIMER;
+
+/* Bridge out structure for RGXQueryTimer */
+typedef struct PVRSRV_BRIDGE_OUT_RGXQUERYTIMER_TAG
+{
+	IMG_UINT64 ui64EndTime;
+	IMG_UINT64 ui64StartTime;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXQUERYTIMER;
+
+#endif /* COMMON_RGXTIMERQUERY_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxtq2_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxtq2_bridge.h
new file mode 100644
index 000000000000..f20fca015dc0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxtq2_bridge.h
@@ -0,0 +1,264 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ2_BRIDGE_H
+#define COMMON_RGXTQ2_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RGXTQ2_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ2_CMD_FIRST+8)
+
+/*******************************************
+            RGXTDMCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdSize;
+	IMG_UINT32 ui32MaxDeadlineMS;
+	IMG_UINT32 ui32PackedCCBSizeU88;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXTDMDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXTDMSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXTDMSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXTDMNotifyWriteOffsetUpdate
+ *******************************************/
+
+/* Bridge in structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32PDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/* Bridge out structure for RGXTDMNotifyWriteOffsetUpdate */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE;
+
+/*******************************************
+            RGXTDMSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXTDMSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTDMGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTDMReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTDMReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXTDMSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXTDMSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXTDMSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3_TAG
+{
+	IMG_UINT64 ui64DeadlineInus;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32UpdateSyncOffset;
+	IMG_UINT32 *pui32UpdateValue;
+	IMG_UINT8 *pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE *phUpdateUFOSyncPrimBlock;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	PVRSRV_TIMELINE hUpdateTimeline;
+	IMG_UINT32 ui32Characteristic1;
+	IMG_UINT32 ui32Characteristic2;
+	IMG_UINT32 ui32ClientUpdateCount;
+	IMG_UINT32 ui32CommandSize;
+	IMG_UINT32 ui32ExternalJobReference;
+	IMG_UINT32 ui32PDumpFlags;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXTDMSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE hUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ2_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_rgxtq_bridge.h b/drivers/gpu/drm/img-rogue/common_rgxtq_bridge.h
new file mode 100644
index 000000000000..31d90dc0f993
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_rgxtq_bridge.h
@@ -0,0 +1,245 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for rgxtq
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for rgxtq
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RGXTQ_BRIDGE_H
+#define COMMON_RGXTQ_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "rgx_bridge.h"
+#include "pvrsrv_sync_km.h"
+
+#define PVRSRV_BRIDGE_RGXTQ_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3			PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RGXTQ_CMD_LAST			(PVRSRV_BRIDGE_RGXTQ_CMD_FIRST+7)
+
+/*******************************************
+            RGXCreateTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT_TAG
+{
+	IMG_UINT64 ui64RobustnessAddress;
+	IMG_HANDLE hPrivData;
+	IMG_BYTE *pui8FrameworkCmd;
+	IMG_INT32 i32Priority;
+	IMG_UINT32 ui32ContextFlags;
+	IMG_UINT32 ui32FrameworkCmdize;
+	IMG_UINT32 ui32PackedCCBSizeU8888;
+} __packed PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT;
+
+/* Bridge out structure for RGXCreateTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT;
+
+/*******************************************
+            RGXDestroyTransferContext
+ *******************************************/
+
+/* Bridge in structure for RGXDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT_TAG
+{
+	IMG_HANDLE hTransferContext;
+} __packed PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT;
+
+/* Bridge out structure for RGXDestroyTransferContext */
+typedef struct PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT;
+
+/*******************************************
+            RGXSetTransferContextPriority
+ *******************************************/
+
+/* Bridge in structure for RGXSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_INT32 i32Priority;
+} __packed PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY;
+
+/* Bridge out structure for RGXSetTransferContextPriority */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY;
+
+/*******************************************
+            RGXSubmitTransfer2
+ *******************************************/
+
+/* Bridge in structure for RGXSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32ClientUpdateCount;
+	IMG_UINT32 *pui32CommandSize;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32TQPrepareFlags;
+	IMG_UINT32 **pui32UpdateSyncOffset;
+	IMG_UINT32 **pui32UpdateValue;
+	IMG_UINT8 **pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE **phUpdateUFOSyncPrimBlock;
+	PVRSRV_TIMELINE h2DUpdateTimeline;
+	PVRSRV_TIMELINE h3DUpdateTimeline;
+	PVRSRV_FENCE hCheckFenceFD;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PrepareCount;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2;
+
+/* Bridge out structure for RGXSubmitTransfer2 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE h2DUpdateFence;
+	PVRSRV_FENCE h3DUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2;
+
+/*******************************************
+            RGXTQGetSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTQGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY;
+
+/* Bridge out structure for RGXTQGetSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY_TAG
+{
+	IMG_HANDLE hCLIPMRMem;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY;
+
+/*******************************************
+            RGXTQReleaseSharedMemory
+ *******************************************/
+
+/* Bridge in structure for RGXTQReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY_TAG
+{
+	IMG_HANDLE hPMRMem;
+} __packed PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY;
+
+/* Bridge out structure for RGXTQReleaseSharedMemory */
+typedef struct PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY;
+
+/*******************************************
+            RGXSetTransferContextProperty
+ *******************************************/
+
+/* Bridge in structure for RGXSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Input;
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 ui32Property;
+} __packed PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY;
+
+/* Bridge out structure for RGXSetTransferContextProperty */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY_TAG
+{
+	IMG_UINT64 ui64Output;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY;
+
+/*******************************************
+            RGXSubmitTransfer3
+ *******************************************/
+
+/* Bridge in structure for RGXSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3_TAG
+{
+	IMG_HANDLE hTransferContext;
+	IMG_UINT32 *pui32ClientUpdateCount;
+	IMG_UINT32 *pui32CommandSize;
+	IMG_UINT32 *pui32SyncPMRFlags;
+	IMG_UINT32 *pui32TQPrepareFlags;
+	IMG_UINT32 **pui32UpdateSyncOffset;
+	IMG_UINT32 **pui32UpdateValue;
+	IMG_UINT8 **pui8FWCommand;
+	IMG_CHAR *puiUpdateFenceName;
+	IMG_HANDLE *phSyncPMRs;
+	IMG_HANDLE **phUpdateUFOSyncPrimBlock;
+	PVRSRV_TIMELINE h2DUpdateTimeline;
+	PVRSRV_TIMELINE h3DUpdateTimeline;
+	PVRSRV_FENCE hCheckFenceFD;
+	PVRSRV_FENCE hExportFenceToSignal;
+	IMG_UINT32 ui32ExtJobRef;
+	IMG_UINT32 ui32PrepareCount;
+	IMG_UINT32 ui32SyncPMRCount;
+} __packed PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3;
+
+/* Bridge out structure for RGXSubmitTransfer3 */
+typedef struct PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3_TAG
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE h2DUpdateFence;
+	PVRSRV_FENCE h3DUpdateFence;
+} __packed PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3;
+
+#endif /* COMMON_RGXTQ_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_ri_bridge.h b/drivers/gpu/drm/img-rogue/common_ri_bridge.h
new file mode 100644
index 000000000000..baa41adccdfe
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_ri_bridge.h
@@ -0,0 +1,224 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_RI_BRIDGE_H
+#define COMMON_RI_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "ri_typedefs.h"
+
+#define PVRSRV_BRIDGE_RI_CMD_FIRST			0
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+0
+#define PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+1
+#define PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+2
+#define PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR			PVRSRV_BRIDGE_RI_CMD_FIRST+3
+#define PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY			PVRSRV_BRIDGE_RI_CMD_FIRST+4
+#define PVRSRV_BRIDGE_RI_RIDUMPLIST			PVRSRV_BRIDGE_RI_CMD_FIRST+5
+#define PVRSRV_BRIDGE_RI_RIDUMPALL			PVRSRV_BRIDGE_RI_CMD_FIRST+6
+#define PVRSRV_BRIDGE_RI_RIDUMPPROCESS			PVRSRV_BRIDGE_RI_CMD_FIRST+7
+#define PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER			PVRSRV_BRIDGE_RI_CMD_FIRST+8
+#define PVRSRV_BRIDGE_RI_CMD_LAST			(PVRSRV_BRIDGE_RI_CMD_FIRST+8)
+
+/*******************************************
+            RIWritePMREntry
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY;
+
+/* Bridge out structure for RIWritePMREntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY;
+
+/*******************************************
+            RIWriteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_UINT64 ui64Flags;
+	IMG_UINT64 ui64Offset;
+	IMG_UINT64 ui64Size;
+	IMG_HANDLE hPMRHandle;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY;
+
+/* Bridge out structure for RIWriteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY;
+
+/*******************************************
+            RIWriteProcListEntry
+ *******************************************/
+
+/* Bridge in structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_UINT64 ui64DevVAddr;
+	IMG_UINT64 ui64Size;
+	const IMG_CHAR *puiTextB;
+	IMG_UINT32 ui32TextBSize;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY;
+
+/* Bridge out structure for RIWriteProcListEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY;
+
+/*******************************************
+            RIUpdateMEMDESCAddr
+ *******************************************/
+
+/* Bridge in structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR_TAG
+{
+	IMG_DEV_VIRTADDR sAddr;
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR;
+
+/* Bridge out structure for RIUpdateMEMDESCAddr */
+typedef struct PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR;
+
+/*******************************************
+            RIDeleteMEMDESCEntry
+ *******************************************/
+
+/* Bridge in structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY_TAG
+{
+	IMG_HANDLE hRIHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY;
+
+/* Bridge out structure for RIDeleteMEMDESCEntry */
+typedef struct PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY;
+
+/*******************************************
+            RIDumpList
+ *******************************************/
+
+/* Bridge in structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPLIST_TAG
+{
+	IMG_HANDLE hPMRHandle;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPLIST;
+
+/* Bridge out structure for RIDumpList */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPLIST_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPLIST;
+
+/*******************************************
+            RIDumpAll
+ *******************************************/
+
+/* Bridge in structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPALL_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPALL;
+
+/* Bridge out structure for RIDumpAll */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPALL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPALL;
+
+/*******************************************
+            RIDumpProcess
+ *******************************************/
+
+/* Bridge in structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_IN_RIDUMPPROCESS_TAG
+{
+	IMG_PID ui32Pid;
+} __packed PVRSRV_BRIDGE_IN_RIDUMPPROCESS;
+
+/* Bridge out structure for RIDumpProcess */
+typedef struct PVRSRV_BRIDGE_OUT_RIDUMPPROCESS_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIDUMPPROCESS;
+
+/*******************************************
+            RIWritePMREntryWithOwner
+ *******************************************/
+
+/* Bridge in structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	IMG_HANDLE hPMRHandle;
+	IMG_PID ui32Owner;
+} __packed PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER;
+
+/* Bridge out structure for RIWritePMREntryWithOwner */
+typedef struct PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER;
+
+#endif /* COMMON_RI_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_srvcore_bridge.h b/drivers/gpu/drm/img-rogue/common_srvcore_bridge.h
new file mode 100644
index 000000000000..603ba5b1d89e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_srvcore_bridge.h
@@ -0,0 +1,369 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SRVCORE_BRIDGE_H
+#define COMMON_SRVCORE_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pvrsrv_device_types.h"
+#include "cache_ops.h"
+
+#define PVRSRV_BRIDGE_SRVCORE_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SRVCORE_CONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SRVCORE_DISCONNECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+10
+#define PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+11
+#define PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+12
+#define PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+13
+#define PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+14
+#define PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+15
+#define PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE			PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16
+#define PVRSRV_BRIDGE_SRVCORE_CMD_LAST			(PVRSRV_BRIDGE_SRVCORE_CMD_FIRST+16)
+
+/*******************************************
+            Connect
+ *******************************************/
+
+/* Bridge in structure for Connect */
+typedef struct PVRSRV_BRIDGE_IN_CONNECT_TAG
+{
+	IMG_UINT32 ui32ClientBuildOptions;
+	IMG_UINT32 ui32ClientDDKBuild;
+	IMG_UINT32 ui32ClientDDKVersion;
+	IMG_UINT32 ui32Flags;
+} __packed PVRSRV_BRIDGE_IN_CONNECT;
+
+/* Bridge out structure for Connect */
+typedef struct PVRSRV_BRIDGE_OUT_CONNECT_TAG
+{
+	IMG_UINT64 ui64PackedBvnc;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CapabilityFlags;
+	IMG_UINT8 ui8KernelArch;
+} __packed PVRSRV_BRIDGE_OUT_CONNECT;
+
+/*******************************************
+            Disconnect
+ *******************************************/
+
+/* Bridge in structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_IN_DISCONNECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_DISCONNECT;
+
+/* Bridge out structure for Disconnect */
+typedef struct PVRSRV_BRIDGE_OUT_DISCONNECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DISCONNECT;
+
+/*******************************************
+            AcquireGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT;
+
+/* Bridge out structure for AcquireGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT;
+
+/*******************************************
+            ReleaseGlobalEventObject
+ *******************************************/
+
+/* Bridge in structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	IMG_HANDLE hGlobalEventObject;
+} __packed PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT;
+
+/* Bridge out structure for ReleaseGlobalEventObject */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT;
+
+/*******************************************
+            EventObjectOpen
+ *******************************************/
+
+/* Bridge in structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hEventObject;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN;
+
+/* Bridge out structure for EventObjectOpen */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN_TAG
+{
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN;
+
+/*******************************************
+            EventObjectWait
+ *******************************************/
+
+/* Bridge in structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT;
+
+/* Bridge out structure for EventObjectWait */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT;
+
+/*******************************************
+            EventObjectClose
+ *******************************************/
+
+/* Bridge in structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE_TAG
+{
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE;
+
+/* Bridge out structure for EventObjectClose */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE;
+
+/*******************************************
+            DumpDebugInfo
+ *******************************************/
+
+/* Bridge in structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_IN_DUMPDEBUGINFO_TAG
+{
+	IMG_UINT32 ui32VerbLevel;
+} __packed PVRSRV_BRIDGE_IN_DUMPDEBUGINFO;
+
+/* Bridge out structure for DumpDebugInfo */
+typedef struct PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO;
+
+/*******************************************
+            GetDevClockSpeed
+ *******************************************/
+
+/* Bridge in structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED;
+
+/* Bridge out structure for GetDevClockSpeed */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ClockSpeed;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED;
+
+/*******************************************
+            HWOpTimeout
+ *******************************************/
+
+/* Bridge in structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_IN_HWOPTIMEOUT_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_HWOPTIMEOUT;
+
+/* Bridge out structure for HWOpTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_HWOPTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_HWOPTIMEOUT;
+
+/*******************************************
+            AlignmentCheck
+ *******************************************/
+
+/* Bridge in structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_IN_ALIGNMENTCHECK_TAG
+{
+	IMG_UINT32 *pui32AlignChecks;
+	IMG_UINT32 ui32AlignChecksSize;
+} __packed PVRSRV_BRIDGE_IN_ALIGNMENTCHECK;
+
+/* Bridge out structure for AlignmentCheck */
+typedef struct PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK;
+
+/*******************************************
+            GetDeviceStatus
+ *******************************************/
+
+/* Bridge in structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_IN_GETDEVICESTATUS_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_GETDEVICESTATUS;
+
+/* Bridge out structure for GetDeviceStatus */
+typedef struct PVRSRV_BRIDGE_OUT_GETDEVICESTATUS_TAG
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceSatus;
+} __packed PVRSRV_BRIDGE_OUT_GETDEVICESTATUS;
+
+/*******************************************
+            GetMultiCoreInfo
+ *******************************************/
+
+/* Bridge in structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_IN_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	IMG_UINT32 ui32CapsSize;
+} __packed PVRSRV_BRIDGE_IN_GETMULTICOREINFO;
+
+/* Bridge out structure for GetMultiCoreInfo */
+typedef struct PVRSRV_BRIDGE_OUT_GETMULTICOREINFO_TAG
+{
+	IMG_UINT64 *pui64Caps;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32NumCores;
+} __packed PVRSRV_BRIDGE_OUT_GETMULTICOREINFO;
+
+/*******************************************
+            EventObjectWaitTimeout
+ *******************************************/
+
+/* Bridge in structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	IMG_UINT64 ui64uiTimeoutus;
+	IMG_HANDLE hOSEventKM;
+} __packed PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT;
+
+/* Bridge out structure for EventObjectWaitTimeout */
+typedef struct PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT;
+
+/*******************************************
+            FindProcessMemStats
+ *******************************************/
+
+/* Bridge in structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	IMG_UINT32 ui32ArrSize;
+	IMG_UINT32 ui32PID;
+	IMG_BOOL bbAllProcessStats;
+} __packed PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS;
+
+/* Bridge out structure for FindProcessMemStats */
+typedef struct PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS_TAG
+{
+	IMG_UINT64 *pui64MemStatsArray;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS;
+
+/*******************************************
+            AcquireInfoPage
+ *******************************************/
+
+/* Bridge in structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE;
+
+/* Bridge out structure for AcquireInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE;
+
+/*******************************************
+            ReleaseInfoPage
+ *******************************************/
+
+/* Bridge in structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_IN_RELEASEINFOPAGE_TAG
+{
+	IMG_HANDLE hPMR;
+} __packed PVRSRV_BRIDGE_IN_RELEASEINFOPAGE;
+
+/* Bridge out structure for ReleaseInfoPage */
+typedef struct PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE;
+
+#endif /* COMMON_SRVCORE_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_sync_bridge.h b/drivers/gpu/drm/img-rogue/common_sync_bridge.h
new file mode 100644
index 000000000000..753d40b64f12
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_sync_bridge.h
@@ -0,0 +1,254 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNC_BRIDGE_H
+#define COMMON_SYNC_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "pdumpdefs.h"
+#include "devicemem_typedefs.h"
+#include "pvrsrv_sync_km.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#define PVRSRV_BRIDGE_SYNC_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK			PVRSRV_BRIDGE_SYNC_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMSET			PVRSRV_BRIDGE_SYNC_CMD_FIRST+2
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+3
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE			PVRSRV_BRIDGE_SYNC_CMD_FIRST+4
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+5
+#define PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP			PVRSRV_BRIDGE_SYNC_CMD_FIRST+6
+#define PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+7
+#define PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT			PVRSRV_BRIDGE_SYNC_CMD_FIRST+8
+#define PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL			PVRSRV_BRIDGE_SYNC_CMD_FIRST+9
+#define PVRSRV_BRIDGE_SYNC_CMD_LAST			(PVRSRV_BRIDGE_SYNC_CMD_FIRST+9)
+
+/*******************************************
+            AllocSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_UINT32 ui32EmptyStructPlaceholder;
+} __packed PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for AllocSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_HANDLE hhSyncPMR;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32SyncPrimBlockSize;
+	IMG_UINT32 ui32SyncPrimVAddr;
+} __packed PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            FreeSyncPrimitiveBlock
+ *******************************************/
+
+/* Bridge in structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	IMG_HANDLE hSyncHandle;
+} __packed PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK;
+
+/* Bridge out structure for FreeSyncPrimitiveBlock */
+typedef struct PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK;
+
+/*******************************************
+            SyncPrimSet
+ *******************************************/
+
+/* Bridge in structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMSET_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMSET;
+
+/* Bridge out structure for SyncPrimSet */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMSET_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMSET;
+
+/*******************************************
+            SyncPrimPDump
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP;
+
+/* Bridge out structure for SyncPrimPDump */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP;
+
+/*******************************************
+            SyncPrimPDumpValue
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE;
+
+/* Bridge out structure for SyncPrimPDumpValue */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE;
+
+/*******************************************
+            SyncPrimPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL_TAG
+{
+	IMG_HANDLE hSyncHandle;
+	PDUMP_POLL_OPERATOR eOperator;
+	IMG_UINT32 ui32Mask;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32Value;
+	PDUMP_FLAGS_T uiPDumpFlags;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL;
+
+/* Bridge out structure for SyncPrimPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL;
+
+/*******************************************
+            SyncPrimPDumpCBP
+ *******************************************/
+
+/* Bridge in structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP_TAG
+{
+	IMG_DEVMEM_SIZE_T uiBufferSize;
+	IMG_DEVMEM_SIZE_T uiPacketSize;
+	IMG_DEVMEM_OFFSET_T uiWriteOffset;
+	IMG_HANDLE hSyncHandle;
+	IMG_UINT32 ui32Offset;
+} __packed PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP;
+
+/* Bridge out structure for SyncPrimPDumpCBP */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP;
+
+/*******************************************
+            SyncAllocEvent
+ *******************************************/
+
+/* Bridge in structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCALLOCEVENT_TAG
+{
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32FWAddr;
+	IMG_BOOL bServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCALLOCEVENT;
+
+/* Bridge out structure for SyncAllocEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT;
+
+/*******************************************
+            SyncFreeEvent
+ *******************************************/
+
+/* Bridge in structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_IN_SYNCFREEEVENT_TAG
+{
+	IMG_UINT32 ui32FWAddr;
+} __packed PVRSRV_BRIDGE_IN_SYNCFREEEVENT;
+
+/* Bridge out structure for SyncFreeEvent */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCFREEEVENT_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCFREEEVENT;
+
+/*******************************************
+            SyncCheckpointSignalledPDumpPol
+ *******************************************/
+
+/* Bridge in structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_FENCE hFence;
+} __packed PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+/* Bridge out structure for SyncCheckpointSignalledPDumpPol */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL;
+
+#endif /* COMMON_SYNC_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/common_synctracking_bridge.h b/drivers/gpu/drm/img-rogue/common_synctracking_bridge.h
new file mode 100644
index 000000000000..9fd9fdb796a7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/common_synctracking_bridge.h
@@ -0,0 +1,97 @@
+/*******************************************************************************
+@File
+@Title          Common bridge header for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declares common defines and structures used by both the client
+                and server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#ifndef COMMON_SYNCTRACKING_BRIDGE_H
+#define COMMON_SYNCTRACKING_BRIDGE_H
+
+#include <powervr/mem_types.h>
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST			0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+0
+#define PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD			PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1
+#define PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST			(PVRSRV_BRIDGE_SYNCTRACKING_CMD_FIRST+1)
+
+/*******************************************
+            SyncRecordRemoveByHandle
+ *******************************************/
+
+/* Bridge in structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	IMG_HANDLE hhRecord;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE;
+
+/* Bridge out structure for SyncRecordRemoveByHandle */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE_TAG
+{
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE;
+
+/*******************************************
+            SyncRecordAdd
+ *******************************************/
+
+/* Bridge in structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_IN_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhServerSyncPrimBlock;
+	const IMG_CHAR *puiClassName;
+	IMG_UINT32 ui32ClassNameSize;
+	IMG_UINT32 ui32ui32FwBlockAddr;
+	IMG_UINT32 ui32ui32SyncOffset;
+	IMG_BOOL bbServerSync;
+} __packed PVRSRV_BRIDGE_IN_SYNCRECORDADD;
+
+/* Bridge out structure for SyncRecordAdd */
+typedef struct PVRSRV_BRIDGE_OUT_SYNCRECORDADD_TAG
+{
+	IMG_HANDLE hhRecord;
+	PVRSRV_ERROR eError;
+} __packed PVRSRV_BRIDGE_OUT_SYNCRECORDADD;
+
+#endif /* COMMON_SYNCTRACKING_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/config_kernel.h b/drivers/gpu/drm/img-rogue/config_kernel.h
new file mode 100644
index 000000000000..514ddd68a16b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/config_kernel.h
@@ -0,0 +1,315 @@
+#define PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY 5
+#define RGX_FW_FILENAME "rgx.fw"
+#define RGX_SH_FILENAME "rgx.sh"
+#define PVR_BUILD_DIR "spacemit"
+#define PVR_BUILD_TYPE "release"
+#define PVRSRV_MODNAME "pvrsrvkm"
+#define PVRSYNC_MODNAME "pvr_sync"
+#define SUPPORT_RGX 1
+#define DISPLAY_CONTROLLER drm_spacemit
+#define PVRSRV_MAX_DEVICES 8
+#define PVRSRV_HWPERF_COUNTERS_PERBLK 12
+#define RELEASE
+#define RGX_BVNC_CORE_KM_HEADER "cores/rgxcore_km_36.56.104.183.h"
+#define RGX_BNC_CONFIG_KM_HEADER "configs/rgxconfig_km_36.V.104.183.h"
+#define PVRSRV_NEED_PVR_DPF
+#define SUPPORT_LINUX_FDINFO
+#define PVRSRV_FORCE_HWPERF_TO_SCHED_CLK
+#define PVRSRV_TRACE_ROGUE_EVENTS
+#define SUPPORT_PHYSMEM_TEST
+#define SUPPORT_RGXTQ_BRIDGE
+#define PVRSRV_POISON_ON_ALLOC_VALUE 0xd9
+#define PVRSRV_POISON_ON_FREE_VALUE 0x63
+#define RGX_NUM_DRIVERS_SUPPORTED 1
+#define RGX_HCS_DEFAULT_DEADLINE_MS 0xFFFFFFFFU
+#define RGX_FW_HEAP_USES_FIRMWARE_OSID 0
+#define RGX_FW_HEAP_USES_HOST_OSID 1
+#define RGX_FW_HEAP_USES_DEDICATED_OSID 2
+#define RGX_FW_HEAP_OSID_ASSIGNMENT RGX_FW_HEAP_USES_FIRMWARE_OSID
+#define PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION 0
+#define RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION  512
+#define PVRSRV_APPHINT_DRIVERMODE "default"
+#define RGX_FW_HEAP_SHIFT 25
+#define PVRSRV_APPHINT_GUESTFWHEAPSTRIDE (1 << RGX_FW_HEAP_SHIFT)
+#define PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN IMG_FALSE
+#define RGX_VZ_CONNECTION_TIMEOUT_US 60000000
+#define GPUVIRT_VALIDATION_NUM_OS 8
+#define PVRSRV_ENABLE_CCCB_GROW
+#define FIX_DUSTS_POW_ON_INIT
+#define SUPPORT_POWMON_COMPONENT
+#define PVR_POWER_ACTOR_MEASUREMENT_PERIOD_MS 10U
+#define PVR_POWER_MONITOR_HWPERF
+#define PVR_LDM_PLATFORM_PRE_REGISTERED
+#define PVR_LDM_DRIVER_REGISTRATION_NAME "pvrsrvkm"
+#define PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN 256
+#define ION_DEFAULT_HEAP_NAME "ion_system_heap"
+#define ION_DEFAULT_HEAP_ID_MASK (1 << ION_HEAP_TYPE_SYSTEM)
+#define PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT APPHNT_BLDVAR_DBGDUMPLIMIT
+#define PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG IMG_FALSE
+#define PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE 0x4000
+#define PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE 786432
+#define PVRSRV_APPHINT_ENABLESIGNATURECHECKS APPHNT_BLDVAR_ENABLESIGNATURECHECKS
+#define PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE RGXFW_SIG_BUFFER_SIZE_MIN
+#define PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING IMG_FALSE
+#define PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG
+#define PVRSRV_APPHINT_VALIDATEIRQ 0
+#define PVRSRV_APPHINT_DISABLECLOCKGATING 0
+#define PVRSRV_APPHINT_DISABLEDMOVERLAP 0
+#define PVRSRV_APPHINT_ENABLECDMKILLINGRANDMODE 0
+#define PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH 0
+#define PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH 0
+#define PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL
+#define PVRSRV_APPHINT_ENABLERDPOWERISLAND RGX_RD_POWER_ISLAND_DEFAULT
+#define PVRSRV_APPHINT_ENABLESPUCLOCKGATING IMG_FALSE
+#define PVRSRV_APPHINT_FIRMWAREPERF FW_PERF_CONF_NONE
+#define PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN
+#define PVRSRV_APPHINT_HWPERFDISABLECUSTOMCOUNTERFILTER 0
+#define PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB 2048
+#define PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB 2048
+#define PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS 50
+#define PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP 1
+#define PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME 0
+#define PVRSRV_APPHINT_TFBCVERSION 0
+#define PVRSRV_APPHINT_NEWFILTERINGMODE 1
+#define PVRSRV_APPHINT_TRUNCATEMODE 0
+#define PVRSRV_APPHINT_EMUMAXFREQ 0
+#define PVRSRV_APPHINT_GPIOVALIDATIONMODE 0
+#define PVRSRV_APPHINT_RGXBVNC ""
+#define PVRSRV_APPHINT_CLEANUPTHREADPRIORITY 5
+#define PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY 0
+#define PVRSRV_APPHINT_CACHEOPTHREADPRIORITY 1
+#define PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 11
+#define PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES 10000
+#define PVRSRV_APPHINT_ASSERTONHWRTRIGGER IMG_FALSE
+#define PVRSRV_APPHINT_ASSERTOUTOFMEMORY IMG_FALSE
+#define PVRSRV_APPHINT_CHECKMLIST APPHNT_BLDVAR_DEBUG
+#define PVRSRV_APPHINT_DISABLEFEDLOGGING IMG_FALSE
+#define PVRSRV_APPHINT_KCCB_SIZE_LOG2 10
+#define PVRSRV_APPHINT_ENABLEAPM RGX_ACTIVEPM_DEFAULT
+#define PVRSRV_APPHINT_ENABLEHTBLOGGROUP 0
+#define PVRSRV_APPHINT_ENABLELOGGROUP RGXFWIF_LOG_TYPE_NONE
+#define PVRSRV_APPHINT_FIRMWARELOGTYPE 0
+#define PVRSRV_APPHINT_FWTRACEBUFSIZEINDWORDS RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS
+#define PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE 1
+#define PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING 0
+#define PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC 40000
+#define PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD 90
+#define PVRSRV_APPHINT_FBCDCVERSIONOVERRIDE 0
+#define PVRSRV_APPHINT_HTBOPERATIONMODE HTB_OPMODE_DROPOLDEST
+#define PVRSRV_APPHINT_HTBUFFERSIZE 64
+#define PVRSRV_APPHINT_ENABLEFTRACEGPU IMG_FALSE
+#define PVRSRV_APPHINT_HWPERFFWFILTER 0
+#define PVRSRV_APPHINT_HWPERFHOSTFILTER 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_SERVICES 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_EGL 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGLES 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENCL 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_VULKAN 0
+#define PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGL 0
+#define PVRSRV_APPHINT_TIMECORRCLOCK 2
+#define PVRSRV_APPHINT_ENABLEFWPOISONONFREE IMG_FALSE
+#define PVRSRV_APPHINT_FWPOISONONFREEVALUE 0xBD
+#define PVRSRV_APPHINT_ZEROFREELIST IMG_FALSE
+#define PVRSRV_APPHINT_GPUUNITSPOWERCHANGE IMG_FALSE
+#define PVRSRV_APPHINT_DISABLEPDUMPPANIC IMG_FALSE
+#define PVRSRV_APPHINT_CACHEOPCONFIG 0
+#define PVRSRV_APPHINT_CACHEOPUMKMHRESHOLDSIZE 0
+#define PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC IMG_FALSE
+#define PVRSRV_APPHINT_PHYSMEMTESTPASSES APPHNT_PHYSMEMTEST_ENABLE
+#define PVRSRV_APPHINT_TESTSLRINTERVAL 0
+#define PVRSRV_APPHINT_RISCVDMITEST 0
+#define PVRSRV_APPHINT_VALIDATESOCUSCTIMERS 0
+#define PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2 8
+#define PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2 7
+#define PDVFS_COM_HOST 1
+#define PDVFS_COM_AP 2
+#define PDVFS_COM_PMC 3
+#define PDVFS_COM_IMG_CLKDIV 4
+#define PDVFS_COM_SYSREG 5
+#define PDVFS_COM PDVFS_COM_HOST
+#define SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY
+#define PVR_GPIO_MODE_GENERAL 1
+#define PVR_GPIO_MODE PVR_GPIO_MODE_GENERAL
+#define PVRSRV_ENABLE_PROCESS_STATS
+#define SUPPORT_USC_BREAKPOINT
+#define SUPPORT_AGP
+#define RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US 2000000
+#define PVR_ANNOTATION_MAX_LEN 63
+#define PVRSRV_DEVICE_INIT_MODE PVRSRV_LINUX_DEV_INIT_ON_CONNECT
+#define SUPPORT_DI_BRG_IMPL
+#define PVR_LINUX_PHYSMEM_MAX_POOL_PAGES 10240
+#define PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES 20480
+#define PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES
+#define PVR_DIRTY_BYTES_FLUSH_THRESHOLD 524288
+#define PVR_LINUX_HIGHORDER_ALLOCATION_THRESHOLD 256
+#define PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM 2
+#define PVR_LINUX_KMALLOC_ALLOCATION_THRESHOLD 16384
+#define SUPPORT_PMR_DEFERRED_FREE
+#define SUPPORT_PMR_PAGES_DEFERRED_FREE
+#define SUPPORT_MMU_DEFERRED_FREE
+#define PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC 1
+#define SUPPORT_NATIVE_FENCE_SYNC
+#define PVRSRV_STALLED_CCB_ACTION
+#define UPDATE_FENCE_CHECKPOINT_COUNT 1
+#define RGX_VZ_CONNECTION_COOLDOWN_PERIOD 0
+#define PVR_DRM_NAME "pvr"
+#define DEVICE_MEMSETCPY_ALIGN_IN_BYTES 16
+#define RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS 0
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA 15
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D 16
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM 14
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ3D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ2D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_CDM 15
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TA 16
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_3D 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_KICKSYNC 13
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TDM 17
+#define PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_RDM 15
+#define RGX_DRIVERID_0_DEFAULT_PRIORITY (1 - 0)
+#define RGX_DRIVERID_1_DEFAULT_PRIORITY (1 - 1)
+#define RGX_DRIVERID_2_DEFAULT_PRIORITY (1 - 2)
+#define RGX_DRIVERID_3_DEFAULT_PRIORITY (1 - 3)
+#define RGX_DRIVERID_4_DEFAULT_PRIORITY (1 - 4)
+#define RGX_DRIVERID_5_DEFAULT_PRIORITY (1 - 5)
+#define RGX_DRIVERID_6_DEFAULT_PRIORITY (1 - 6)
+#define RGX_DRIVERID_7_DEFAULT_PRIORITY (1 - 7)
+#define RGX_DRIVERID_8_DEFAULT_PRIORITY (1 - 8)
+#define RGX_DRIVERID_9_DEFAULT_PRIORITY (1 - 9)
+#define RGX_DRIVERID_10_DEFAULT_PRIORITY (1 - 10)
+#define RGX_DRIVERID_11_DEFAULT_PRIORITY (1 - 11)
+#define RGX_DRIVERID_12_DEFAULT_PRIORITY (1 - 12)
+#define RGX_DRIVERID_13_DEFAULT_PRIORITY (1 - 13)
+#define RGX_DRIVERID_14_DEFAULT_PRIORITY (1 - 14)
+#define RGX_DRIVERID_15_DEFAULT_PRIORITY (1 - 15)
+#define RGX_DRIVERID_16_DEFAULT_PRIORITY (1 - 16)
+#define RGX_DRIVERID_17_DEFAULT_PRIORITY (1 - 17)
+#define RGX_DRIVERID_18_DEFAULT_PRIORITY (1 - 18)
+#define RGX_DRIVERID_19_DEFAULT_PRIORITY (1 - 19)
+#define RGX_DRIVERID_20_DEFAULT_PRIORITY (1 - 20)
+#define RGX_DRIVERID_21_DEFAULT_PRIORITY (1 - 21)
+#define RGX_DRIVERID_22_DEFAULT_PRIORITY (1 - 22)
+#define RGX_DRIVERID_23_DEFAULT_PRIORITY (1 - 23)
+#define RGX_DRIVERID_24_DEFAULT_PRIORITY (1 - 24)
+#define RGX_DRIVERID_25_DEFAULT_PRIORITY (1 - 25)
+#define RGX_DRIVERID_26_DEFAULT_PRIORITY (1 - 26)
+#define RGX_DRIVERID_27_DEFAULT_PRIORITY (1 - 27)
+#define RGX_DRIVERID_28_DEFAULT_PRIORITY (1 - 28)
+#define RGX_DRIVERID_29_DEFAULT_PRIORITY (1 - 29)
+#define RGX_DRIVERID_30_DEFAULT_PRIORITY (1 - 30)
+#define RGX_DRIVERID_31_DEFAULT_PRIORITY (1 - 31)
+#define RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_1_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_2_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_3_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_4_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_5_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_6_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_7_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_8_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_9_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_10_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_11_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_12_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_13_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_14_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_15_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_16_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_17_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_18_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_19_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_20_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_21_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_22_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_23_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_24_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_25_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_26_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_27_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_28_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_29_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_30_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_31_DEFAULT_ISOLATION_GROUP 0
+#define RGX_DRIVERID_0_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_1_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_2_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_3_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_4_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_5_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_6_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_7_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_8_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_9_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_10_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_11_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_12_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_13_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_14_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_15_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_16_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_17_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_18_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_19_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_20_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_21_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_22_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_23_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_24_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_25_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_26_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_27_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_28_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_29_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_30_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVERID_31_DEFAULT_TIME_SLICE 0
+#define RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL 0
+#define RGX_DRIVER_DEFAULT_TIME_SLICES_SUM ((((((((((((((((((((((((((((((((\
+						+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)\
+						+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)+0)
+#define DRIVER0_SECURITY_SUPPORT 0
+#define DRIVER1_SECURITY_SUPPORT 0
+#define DRIVER2_SECURITY_SUPPORT 0
+#define DRIVER3_SECURITY_SUPPORT 0
+#define DRIVER4_SECURITY_SUPPORT 0
+#define DRIVER5_SECURITY_SUPPORT 0
+#define DRIVER6_SECURITY_SUPPORT 0
+#define DRIVER7_SECURITY_SUPPORT 0
+#define DRIVER8_SECURITY_SUPPORT 0
+#define DRIVER9_SECURITY_SUPPORT 0
+#define DRIVER10_SECURITY_SUPPORT 0
+#define DRIVER11_SECURITY_SUPPORT 0
+#define DRIVER12_SECURITY_SUPPORT 0
+#define DRIVER13_SECURITY_SUPPORT 0
+#define DRIVER14_SECURITY_SUPPORT 0
+#define DRIVER15_SECURITY_SUPPORT 0
+#define DRIVER16_SECURITY_SUPPORT 0
+#define DRIVER17_SECURITY_SUPPORT 0
+#define DRIVER18_SECURITY_SUPPORT 0
+#define DRIVER19_SECURITY_SUPPORT 0
+#define DRIVER20_SECURITY_SUPPORT 0
+#define DRIVER21_SECURITY_SUPPORT 0
+#define DRIVER22_SECURITY_SUPPORT 0
+#define DRIVER23_SECURITY_SUPPORT 0
+#define DRIVER24_SECURITY_SUPPORT 0
+#define DRIVER25_SECURITY_SUPPORT 0
+#define DRIVER26_SECURITY_SUPPORT 0
+#define DRIVER27_SECURITY_SUPPORT 0
+#define DRIVER28_SECURITY_SUPPORT 0
+#define DRIVER29_SECURITY_SUPPORT 0
+#define DRIVER30_SECURITY_SUPPORT 0
+#define DRIVER31_SECURITY_SUPPORT 0
+#define SUPPORT_BUFFER_SYNC 1
+#define PVRSRV_ENABLE_SYNC_POISONING 1
+#define SUPPORT_RGXKICKSYNC_BRIDGE
+#define SUPPORT_TQ
+#define TRACK_FW_BOOT
+#define LMA
+#define PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY
+#if defined(CONFIG_POWERVR_DVFS)
+#define SUPPORT_LINUX_DVFS 1
+#endif
diff --git a/drivers/gpu/drm/img-rogue/config_kernel.mk b/drivers/gpu/drm/img-rogue/config_kernel.mk
new file mode 100644
index 000000000000..8aa7946ed36d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/config_kernel.mk
@@ -0,0 +1,56 @@
+override PVRSRV_DIR := services
+override HOST_PRIMARY_ARCH := host_x86_64
+override HOST_32BIT_ARCH := host_i386
+override HOST_FORCE_32BIT := -m32
+override HOST_ALL_ARCH := host_x86_64 host_i386
+override TARGET_PRIMARY_ARCH := target_riscv64
+override TARGET_SECONDARY_ARCH :=
+override TARGET_ALL_ARCH := target_riscv64
+override TARGET_FORCE_32BIT :=
+override PVR_ARCH := rogue
+override METAG_VERSION_NEEDED := 2.8.1.0.3
+override MIPS_VERSION_NEEDED := 2014.07-1
+override RISCV_VERSION_NEEDED := 1.7.1
+override KERNEL_ID := 6.12.16-gf26327ac7891-dirty
+override PVRSRV_MODULE_BASEDIR := /lib/modules/6.12.16-gf26327ac7891-dirty/extra/
+override KERNEL_COMPONENTS := srvkm drm_spacemit
+override KERNEL_CROSS_COMPILE := riscv64-unknown-linux-gnu-
+override WINDOW_SYSTEM := wayland
+override PVRSRV_MODNAME := pvrsrvkm
+override PVR_BUILD_DIR := spacemit
+override PVR_BUILD_TYPE := release
+override SUPPORT_RGX := 1
+override DISPLAY_CONTROLLER := drm_spacemit
+override PVR_SYSTEM := spacemit
+override PVR_LOADER :=
+override BUILD := release
+override SORT_BRIDGE_STRUCTS := 1
+override DEBUGLINK := 1
+override RGX_BNC := 36.V.104.183
+override SUPPORT_LINUX_FDINFO := 1
+override PVRSRV_FORCE_HWPERF_TO_SCHED_CLK := 1
+override PVRSRV_TRACE_ROGUE_EVENTS := 1
+override SUPPORT_PHYSMEM_TEST := 1
+override SUPPORT_MIPS_64K_PAGE_SIZE :=
+override RGX_NUM_DRIVERS_SUPPORTED := 1
+override RGX_FW_HEAP_OSID_ASSIGNMENT := RGX_FW_HEAP_USES_FIRMWARE_OSID
+override VMM_TYPE := stub
+override SUPPORT_POWMON_COMPONENT := 1
+override RGX_TIMECORR_CLOCK := sched
+override PDVFS_COM_HOST := 1
+override PDVFS_COM_AP := 2
+override PDVFS_COM_PMC := 3
+override PDVFS_COM_IMG_CLKDIV := 4
+override PDVFS_COM_SYSREG := 5
+override PDVFS_COM := PDVFS_COM_HOST
+override SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY := 1
+override PVR_GPIO_MODE_GENERAL := 1
+override PVR_GPIO_MODE := PVR_GPIO_MODE_GENERAL
+override PVR_HANDLE_BACKEND := idr
+override SUPPORT_DMABUF_BRIDGE := 1
+override SUPPORT_USC_BREAKPOINT := 1
+override SUPPORT_DI_BRG_IMPL := 1
+override SUPPORT_WRAP_EXTMEM := 1
+override SUPPORT_NATIVE_FENCE_SYNC := 1
+override SUPPORT_DMA_FENCE := 1
+override SUPPORT_BUFFER_SYNC := 1
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_1.V.4.19.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_1.V.4.19.h
new file mode 100644
index 000000000000..5f60752630c0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_1.V.4.19.h
@@ -0,0 +1,82 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.4.19 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_4_19_H
+#define RGXCONFIG_KM_1_V_4_19_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 19
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_4_19_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_1.V.4.5.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_1.V.4.5.h
new file mode 100644
index 000000000000..1af637872cf9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_1.V.4.5.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 1.V.4.5 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_1_V_4_5_H
+#define RGXCONFIG_KM_1_V_4_5_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 1
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 5
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (1U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (MTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+
+#endif /* RGXCONFIG_KM_1_V_4_5_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_15.V.1.64.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_15.V.1.64.h
new file mode 100644
index 000000000000..3ad2665eb9ee
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_15.V.1.64.h
@@ -0,0 +1,87 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 15.V.1.64 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_15_V_1_64_H
+#define RGXCONFIG_KM_15_V_1_64_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 15
+#define RGX_BNC_KM_N 1
+#define RGX_BNC_KM_C 64
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP217)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_15_V_1_64_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.104.18.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.104.18.h
new file mode 100644
index 000000000000..4594c6429129
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.104.18.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.104.18 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_104_18_H
+#define RGXCONFIG_KM_22_V_104_18_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 18
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_104_18_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.104.218.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.104.218.h
new file mode 100644
index 000000000000..e271c768760d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.104.218.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.104.218 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_104_218_H
+#define RGXCONFIG_KM_22_V_104_218_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 218
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_104_218_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.208.318.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.208.318.h
new file mode 100644
index 000000000000..8b76f2aa6cf0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.208.318.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.208.318 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_208_318_H
+#define RGXCONFIG_KM_22_V_208_318_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 318
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_208_318_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.21.16.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.21.16.h
new file mode 100644
index 000000000000..9d54fdd6a578
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.21.16.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.21.16 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_21_16_H
+#define RGXCONFIG_KM_22_V_21_16_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 21
+#define RGX_BNC_KM_C 16
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (8U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_21_16_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.25.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.25.h
new file mode 100644
index 000000000000..ffe2926153ff
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.25.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.25 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_25_H
+#define RGXCONFIG_KM_22_V_54_25_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 25
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (3U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_25_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.30.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.30.h
new file mode 100644
index 000000000000..d59101b73ad6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.30.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.30 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_30_H
+#define RGXCONFIG_KM_22_V_54_30_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 30
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_30_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.330.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.330.h
new file mode 100644
index 000000000000..fd7547bca34e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.330.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.330 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_330_H
+#define RGXCONFIG_KM_22_V_54_330_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 330
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (32U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_330_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.38.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.38.h
new file mode 100644
index 000000000000..6db0a9272d48
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_22.V.54.38.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 22.V.54.38 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_22_V_54_38_H
+#define RGXCONFIG_KM_22_V_54_38_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 22
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 38
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (1U)
+#define RGX_FEATURE_SINGLE_BIF
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_22_V_54_38_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.104.504.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.104.504.h
new file mode 100644
index 000000000000..e79b894f1299
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.104.504.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.104.504 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_104_504_H
+#define RGXCONFIG_KM_24_V_104_504_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 504
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_104_504_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.208.504.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.208.504.h
new file mode 100644
index 000000000000..2b92c75a2d80
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.208.504.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.208.504 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_208_504_H
+#define RGXCONFIG_KM_24_V_208_504_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 504
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_208_504_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.208.505.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.208.505.h
new file mode 100644
index 000000000000..d14c991870e1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.208.505.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.208.505 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_208_505_H
+#define RGXCONFIG_KM_24_V_208_505_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 208
+#define RGX_BNC_KM_C 505
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (12U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_208_505_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.54.204.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.54.204.h
new file mode 100644
index 000000000000..0274380eafdd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_24.V.54.204.h
@@ -0,0 +1,98 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 24.V.54.204 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_24_V_54_204_H
+#define RGXCONFIG_KM_24_V_54_204_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 24
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 204
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (3U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (3U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_24_V_54_204_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_29.V.108.208.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_29.V.108.208.h
new file mode 100644
index 000000000000..bae4231b4f9e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_29.V.108.208.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 29.V.108.208 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_29_V_108_208_H
+#define RGXCONFIG_KM_29_V_108_208_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 29
+#define RGX_BNC_KM_N 108
+#define RGX_BNC_KM_C 208
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (2U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_29_V_108_208_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_29.V.52.202.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_29.V.52.202.h
new file mode 100644
index 000000000000..805e6df66ab3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_29.V.52.202.h
@@ -0,0 +1,101 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 29.V.52.202 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_29_V_52_202_H
+#define RGXCONFIG_KM_29_V_52_202_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 29
+#define RGX_BNC_KM_N 52
+#define RGX_BNC_KM_C 202
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FASTRENDER_DM
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (4U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBE_CHECKSUM_2D
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TDM_PDS_CHECKSUM
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_29_V_52_202_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_33.V.11.3.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_33.V.11.3.h
new file mode 100644
index 000000000000..19155570aa2e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_33.V.11.3.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 33.V.11.3 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_33_V_11_3_H
+#define RGXCONFIG_KM_33_V_11_3_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 33
+#define RGX_BNC_KM_N 11
+#define RGX_BNC_KM_C 3
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (1U)
+#define RGX_FEATURE_NUM_OSIDS (2U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (2U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_33_V_11_3_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_33.V.22.1.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_33.V.22.1.h
new file mode 100644
index 000000000000..a375f749cf46
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_33.V.22.1.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 33.V.22.1 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_33_V_22_1_H
+#define RGXCONFIG_KM_33_V_22_1_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 33
+#define RGX_BNC_KM_N 22
+#define RGX_BNC_KM_C 1
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_PM_MMU_VFP
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_33_V_22_1_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.182.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.182.h
new file mode 100644
index 000000000000..48084d57d188
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.182.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_182_H
+#define RGXCONFIG_KM_36_V_104_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_182_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.183.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.183.h
new file mode 100644
index 000000000000..dc4a3590f0d9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.183.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.183 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_183_H
+#define RGXCONFIG_KM_36_V_104_183_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 183
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_183_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.796.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.796.h
new file mode 100644
index 000000000000..d4f87c4ffa4d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.104.796.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.104.796 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_104_796_H
+#define RGXCONFIG_KM_36_V_104_796_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 104
+#define RGX_BNC_KM_C 796
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (2U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (6U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SAFETY_IRQ
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_WATCHDOG_TIMER
+#define RGX_FEATURE_WORKGROUP_PROTECTION
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_104_796_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.52.182.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.52.182.h
new file mode 100644
index 000000000000..049939fbb452
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.52.182.h
@@ -0,0 +1,105 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.52.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_52_182_H
+#define RGXCONFIG_KM_36_V_52_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 52
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_52_182_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.103.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.103.h
new file mode 100644
index 000000000000..3f8b4ffab362
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.103.h
@@ -0,0 +1,103 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.103 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_103_H
+#define RGXCONFIG_KM_36_V_54_103_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 103
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+
+#endif /* RGXCONFIG_KM_36_V_54_103_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.182.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.182.h
new file mode 100644
index 000000000000..78b53d52cf4a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.182.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.182 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_182_H
+#define RGXCONFIG_KM_36_V_54_182_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 182
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_182_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.183.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.183.h
new file mode 100644
index 000000000000..d2f5dd4bab1b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.183.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.183 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_183_H
+#define RGXCONFIG_KM_36_V_54_183_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 183
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_VERSION (10U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_183_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.280.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.280.h
new file mode 100644
index 000000000000..30bbaa68dcb5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_36.V.54.280.h
@@ -0,0 +1,110 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 36.V.54.280 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_36_V_54_280_H
+#define RGXCONFIG_KM_36_V_54_280_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 36
+#define RGX_BNC_KM_N 54
+#define RGX_BNC_KM_C 280
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (4U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (11U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_36_V_54_280_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.2.51.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.2.51.h
new file mode 100644
index 000000000000..28114cad8c97
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.2.51.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.2.51 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_2_51_H
+#define RGXCONFIG_KM_4_V_2_51_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 51
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_2_51_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.2.58.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.2.58.h
new file mode 100644
index 000000000000..99ee53e0c197
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.2.58.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.2.58 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_2_58_H
+#define RGXCONFIG_KM_4_V_2_58_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 2
+#define RGX_BNC_KM_C 58
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (2U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_2_58_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.4.55.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.4.55.h
new file mode 100644
index 000000000000..c27adf56b55b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.4.55.h
@@ -0,0 +1,89 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.4.55 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_4_55_H
+#define RGXCONFIG_KM_4_V_4_55_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 55
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_4_55_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.6.62.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.6.62.h
new file mode 100644
index 000000000000..2d4aa9599a2b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_4.V.6.62.h
@@ -0,0 +1,93 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 4.V.6.62 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_4_V_6_62_H
+#define RGXCONFIG_KM_4_V_6_62_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 4
+#define RGX_BNC_KM_N 6
+#define RGX_BNC_KM_C 62
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (6U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (2U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_BANKS (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_4_V_6_62_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_46.V.102.389.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_46.V.102.389.h
new file mode 100644
index 000000000000..a8eab1e08e4d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_46.V.102.389.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 46.V.102.389 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_46_V_102_389_H
+#define RGXCONFIG_KM_46_V_102_389_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 46
+#define RGX_BNC_KM_N 102
+#define RGX_BNC_KM_C 389
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (2U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (20U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_VOLCANIC_TB
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_46_V_102_389_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_46.V.204.390.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_46.V.204.390.h
new file mode 100644
index 000000000000..96544ed32b48
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_46.V.204.390.h
@@ -0,0 +1,114 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 46.V.204.390 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_46_V_204_390_H
+#define RGXCONFIG_KM_46_V_204_390_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 46
+#define RGX_BNC_KM_N 204
+#define RGX_BNC_KM_C 390
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_COREID_PER_OS
+#define RGX_FEATURE_ECC_RAMS (0U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC (50U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (50U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (7U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT
+#define RGX_FEATURE_GPU_VIRTUALISATION
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_IRQ_PER_OS
+#define RGX_FEATURE_LAYOUT_MARS (1U)
+#define RGX_FEATURE_MIPS
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (7U)
+#define RGX_FEATURE_NUM_MEMBUS (1U)
+#define RGX_FEATURE_NUM_OSIDS (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PBE2_IN_XE
+#define RGX_FEATURE_PBVNC_COREID_REG
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (36U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION (2U)
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE /* Specifies the SLC is */
+                                          /* customer-configurable. True SLC */
+                                          /* size must be sourced from */
+                                          /* register. */
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (64U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET
+#define RGX_FEATURE_TB_GPU_COUNT (1U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT
+#define RGX_FEATURE_TFBC_NATIVE_YUV10
+#define RGX_FEATURE_TFBC_VERSION (20U)
+#define RGX_FEATURE_TILE_SIZE_X (16U)
+#define RGX_FEATURE_TILE_SIZE_Y (16U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_VOLCANIC_TB
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH (19U)
+#define RGX_FEATURE_XPU_MAX_SLAVES (3U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST (1U)
+
+#endif /* RGXCONFIG_KM_46_V_204_390_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_5.V.1.46.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_5.V.1.46.h
new file mode 100644
index 000000000000..1c9b7db878d3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_5.V.1.46.h
@@ -0,0 +1,85 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 5.V.1.46 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_5_V_1_46_H
+#define RGXCONFIG_KM_5_V_1_46_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 5
+#define RGX_BNC_KM_N 1
+#define RGX_BNC_KM_C 46
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (1U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP217)
+#define RGX_FEATURE_META_COREMEM_SIZE (0U)
+#define RGX_FEATURE_NUM_CLUSTERS (1U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (1U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_ROGUEXE
+#define RGX_FEATURE_SLC_BANKS (1U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (16U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XE_ARCHITECTURE (1U)
+
+#endif /* RGXCONFIG_KM_5_V_1_46_H */
diff --git a/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_6.V.4.35.h b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_6.V.4.35.h
new file mode 100644
index 000000000000..cbf5589ce79c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/configs/rgxconfig_km_6.V.4.35.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@Title          RGX Configuration for BVNC 6.V.4.35 (kernel defines)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCONFIG_KM_6_V_4_35_H
+#define RGXCONFIG_KM_6_V_4_35_H
+
+/***** Automatically generated file. Do not edit manually ********************/
+
+/******************************************************************************
+ * B.V.N.C Validation defines
+ *****************************************************************************/
+#define RGX_BNC_KM_B 6
+#define RGX_BNC_KM_N 4
+#define RGX_BNC_KM_C 35
+
+/******************************************************************************
+ * DDK Defines
+ *****************************************************************************/
+#define RGX_FEATURE_AXI_ACELITE
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING
+#define RGX_FEATURE_COMPUTE
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE
+#define RGX_FEATURE_COMPUTE_OVERLAP
+#define RGX_FEATURE_DYNAMIC_DUST_POWER
+#define RGX_FEATURE_FAULT_DECODE_VERSION (0U)
+#define RGX_FEATURE_FBCDC_ALGORITHM (2U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE (2U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS (0U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS (0U)
+#define RGX_FEATURE_GS_RTA_SUPPORT
+#define RGX_FEATURE_LAYOUT_MARS (0U)
+#define RGX_FEATURE_META (LTP218)
+#define RGX_FEATURE_META_COREMEM_SIZE (32U)
+#define RGX_FEATURE_NUM_CLUSTERS (4U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES (8U)
+#define RGX_FEATURE_NUM_RASTER_PIPES (1U)
+#define RGX_FEATURE_PERFBUS
+#define RGX_FEATURE_PERF_COUNTER_BATCH
+#define RGX_FEATURE_PHYS_BUS_WIDTH (40U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS (512U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES (128U)
+#define RGX_FEATURE_SOC_TIMER
+#define RGX_FEATURE_TILE_SIZE_X (32U)
+#define RGX_FEATURE_TILE_SIZE_Y (32U)
+#define RGX_FEATURE_TLA
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS (40U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE
+
+#endif /* RGXCONFIG_KM_6_V_4_35_H */
diff --git a/drivers/gpu/drm/img-rogue/connection_server.c b/drivers/gpu/drm/img-rogue/connection_server.c
new file mode 100644
index 000000000000..b519ee14de22
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/connection_server.c
@@ -0,0 +1,515 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Handles connections coming from the client and the management
+                connection based information
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "handle.h"
+#include "pvrsrv.h"
+#include "connection_server.h"
+#include "osconnection_server.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "sync_server.h"
+#include "process_stats.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "tlstream.h"
+#include "rgxhwperf_common.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+/* PID associated with Connection currently being purged by Cleanup thread */
+static IMG_PID gCurrentPurgeConnectionPid;
+
+static PVRSRV_ERROR ConnectionDataDestroy(CONNECTION_DATA *psConnection)
+{
+	PVRSRV_ERROR eError;
+	PROCESS_HANDLE_BASE *psProcessHandleBase;
+	IMG_UINT64 ui64MaxBridgeTime;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+
+	if (psPVRSRVData->bUnload)
+	{
+		/* driver is unloading so do not allow the bridge lock to be released */
+		ui64MaxBridgeTime = 0;
+	}
+	else
+	{
+		ui64MaxBridgeTime = CONNECTION_DEFERRED_CLEANUP_TIMESLICE_NS;
+	}
+
+	PVR_ASSERT(psConnection != NULL);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psConnection, "psConnection");
+
+	/* Close HWPerfClient stream here even though we created it in
+	 * PVRSRVConnectKM(). */
+	if (psConnection->hClientTLStream)
+	{
+		TLStreamClose(psConnection->hClientTLStream);
+		psConnection->hClientTLStream = NULL;
+		PVR_DPF((PVR_DBG_MESSAGE, "Destroyed private stream."));
+	}
+
+	/* Get process handle base to decrement the refcount */
+	psProcessHandleBase = psConnection->psProcessHandleBase;
+
+	if (psProcessHandleBase != NULL)
+	{
+		/* PVRSRVReleaseProcessHandleBase() calls PVRSRVFreeKernelHandles()
+		 * and PVRSRVFreeHandleBase() for the process handle base.
+		 * Releasing kernel handles can never return RETRY error because
+		 * release function for those handles are NOPs and PVRSRVFreeKernelHandles()
+		 * doesn't even call pfnReleaseData() callback.
+		 * Process handles can potentially return RETRY hence additional check
+		 * below. */
+		eError = PVRSRVReleaseProcessHandleBase(psProcessHandleBase, psConnection->pid,
+		                                        ui64MaxBridgeTime);
+		if (PVRSRVIsRetryError(eError))
+		{
+			return eError;
+		}
+		else
+		{
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVReleaseProcessHandleBase");
+		}
+
+		psConnection->psProcessHandleBase = NULL;
+	}
+
+	/* Free handle base for this connection */
+	if (psConnection->psHandleBase != NULL)
+	{
+		eError = PVRSRVFreeHandleBase(psConnection->psHandleBase, ui64MaxBridgeTime);
+		/*
+		 * If we get PVRSRV_ERROR_RETRY we need to pass this back to the caller
+		 * who will schedule a retry.
+		 * Do not log this as it is an expected exception.
+		 * This can occur if the Firmware is still processing a workload from
+		 * the client when a tear-down request is received.
+		 * Retrying will allow the in-flight work to be completed and the
+		 * tear-down request can be completed when the FW is no longer busy.
+		 */
+		if (PVRSRVIsRetryError(eError))
+		{
+			return eError;
+		}
+		else
+		{
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeHandleBase");
+		}
+
+		psConnection->psHandleBase = NULL;
+	}
+
+	if (psConnection->psSyncConnectionData != NULL)
+	{
+		SyncUnregisterConnection(psConnection->psSyncConnectionData);
+		psConnection->psSyncConnectionData = NULL;
+	}
+
+	if (psConnection->psPDumpConnectionData != NULL)
+	{
+		PDumpUnregisterConnection(psDevNode,
+		                          psConnection->psPDumpConnectionData);
+		psConnection->psPDumpConnectionData = NULL;
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDeviceDisconnect(psDevNode);
+#endif
+
+	/* Call environment specific connection data deinit function */
+	if (psConnection->hOsPrivateData != NULL)
+	{
+		eError = OSConnectionPrivateDataDeInit(psConnection->hOsPrivateData);
+		PVR_LOG_RETURN_IF_ERROR(eError, "OSConnectionPrivateDataDeInit");
+
+		psConnection->hOsPrivateData = NULL;
+	}
+
+	/* Close the PID stats entry as late as possible to catch all frees */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+	if (psConnection->hProcessStats != NULL)
+	{
+		PVRSRVStatsDeregisterProcess(psConnection->hProcessStats);
+		psConnection->hProcessStats = NULL;
+	}
+#endif
+
+	OSFreeMemNoStats(psConnection);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Kick the Firmware to invalidate caches to clear all the zombie PMRs.
+	 * If there are not zombie PMRs or no mappings were freed the kick will not
+	 * be executed.
+	 *
+	 * This is needed:
+	 * - when the process is killed and the connection cleanup has to clean up
+	 *   all dangling handles.
+	 * - there are any outstanding PMRs in the zombie list due to no
+	 *   invalidation being executed before connection destruction
+	 */
+	eError = MMU_CacheInvalidateKick(psDevNode, NULL);
+	PVR_LOG_IF_ERROR(eError, "MMU_CacheInvalidateKick");
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVCommonConnectionConnect(void **ppvPrivData, void *pvOSData)
+{
+	CONNECTION_DATA *psConnection;
+	PVRSRV_ERROR eError;
+	PROCESS_HANDLE_BASE *psProcessHandleBase;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* Allocate connection data area, no stats since process not registered yet */
+	psConnection = OSAllocZMemNoStats(sizeof(*psConnection));
+	PVR_LOG_RETURN_IF_NOMEM(psConnection, "psConnection");
+
+	/* Allocate process statistics as early as possible to catch all allocs */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+	eError = PVRSRVStatsRegisterProcess(&psConnection->hProcessStats);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsRegisterProcess", failure);
+#endif
+
+	/* Call environment specific connection data init function */
+	eError = OSConnectionPrivateDataInit(&psConnection->hOsPrivateData, pvOSData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSConnectionPrivateDataInit", failure);
+
+	/* Must come after OSConnectionPrivateDataInit */
+	psDevNode = OSGetDevNode(psConnection);
+	PVR_LOG_GOTO_IF_NOMEM(psDevNode, eError, failure);
+
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cannot connect to the device during deinit"));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_DEVICE, failure);
+	}
+
+	psConnection->pid = OSGetCurrentClientProcessIDKM();
+	psConnection->vpid = OSGetCurrentVirtualProcessID();
+	psConnection->tid = (IMG_UINT32)OSGetCurrentClientThreadIDKM();
+	OSStringSafeCopy(psConnection->pszProcName, OSGetCurrentClientProcessNameKM(), PVRSRV_CONNECTION_PROCESS_NAME_LEN);
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	eError = PVRSRVInitialiseDMA(psDevNode, psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVInitialiseDMA", failure);
+#endif
+
+	/* Register this connection with the sync core */
+	eError = SyncRegisterConnection(&psConnection->psSyncConnectionData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncRegisterConnection", failure);
+
+	/*
+	 * Register this connection and Sync PDump callback with
+	 * the pdump core. Pass in the Sync connection data.
+	 */
+	eError = PDumpRegisterConnection(psDevNode,
+	                                 psConnection->psSyncConnectionData,
+	                                 SyncConnectionPDumpSyncBlocks,
+	                                 &psConnection->psPDumpConnectionData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDumpRegisterConnection", failure);
+
+	/* Allocate handle base for this connection */
+	eError = PVRSRVAllocHandleBase(&psConnection->psHandleBase,
+	                               PVRSRV_HANDLE_BASE_TYPE_CONNECTION);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", failure);
+
+	/* get process handle base (if it doesn't exist it will be allocated) */
+	eError = PVRSRVAcquireProcessHandleBase(psConnection->pid, &psProcessHandleBase);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireProcessHandleBase", failure);
+
+	/* hConnectionsLock now resides in PVRSRV_DEVICE_NODE */
+	{
+		IMG_BOOL bHostStreamIsNull;
+		PVRSRV_RGXDEV_INFO  *psRgxDevInfo;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		eError = PVRSRVStatsDeviceConnect(psDevNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsDeviceConnect", failure);
+#endif
+
+		OSLockAcquire(psDevNode->hConnectionsLock);
+		dllist_add_to_tail(&psDevNode->sConnections, &psConnection->sConnectionListNode);
+#if defined(DEBUG) || defined(PDUMP)
+		PVR_LOG(("%s connected - (devID = %u)", psConnection->pszProcName,
+		        psDevNode->sDevId.ui32InternalID));
+#endif
+		OSLockRelease(psDevNode->hConnectionsLock);
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+		{
+			psRgxDevInfo = _RGX_DEVICE_INFO_FROM_NODE(psDevNode);
+
+			OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+			bHostStreamIsNull = (IMG_BOOL)(psRgxDevInfo->hHWPerfHostStream == NULL);
+			OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+			if (!bHostStreamIsNull)
+			{
+				if (TLStreamIsOpenForReading(psRgxDevInfo->hHWPerfHostStream))
+				{
+					/* Announce this client connection in the host stream, if event mask is set */
+					RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(psDevNode, psConnection->pid, psConnection->pszProcName);
+				}
+			}
+		}
+	}
+
+	psConnection->psProcessHandleBase = psProcessHandleBase;
+
+	*ppvPrivData = psConnection;
+
+	return PVRSRV_OK;
+
+failure:
+	ConnectionDataDestroy(psConnection);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _CleanupThreadPurgeConnectionData(void *pvConnectionData)
+{
+	PVRSRV_ERROR eErrorConnection, eErrorKernel;
+	CONNECTION_DATA *psConnectionData = pvConnectionData;
+
+	gCurrentPurgeConnectionPid = psConnectionData->pid;
+
+	eErrorConnection = ConnectionDataDestroy(psConnectionData);
+	if (eErrorConnection != PVRSRV_OK)
+	{
+		if (PVRSRVIsRetryError(eErrorConnection))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Failed to purge connection data %p "
+			        "(deferring destruction)", __func__, psConnectionData));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Connection data %p deferred destruction "
+		        "finished", __func__, psConnectionData));
+	}
+
+	/* Check if possible resize the global handle base */
+	eErrorKernel = PVRSRVPurgeHandles(KERNEL_HANDLE_BASE);
+	PVR_LOG_IF_ERROR(eErrorKernel, "PVRSRVPurgeHandles");
+
+	gCurrentPurgeConnectionPid = 0;
+
+	return eErrorConnection;
+}
+
+void PVRSRVCommonConnectionDisconnect(void *pvDataPtr)
+{
+	CONNECTION_DATA *psConnectionData = pvDataPtr;
+	PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnectionData);
+
+	OSLockAcquire(psDevNode->hConnectionsLock);
+	dllist_remove_node(&psConnectionData->sConnectionListNode);
+	OSLockRelease(psDevNode->hConnectionsLock);
+
+	/* Notify the PDump core if the pdump control client is disconnecting */
+	if (psConnectionData->ui32ClientFlags & SRV_FLAGS_PDUMPCTRL)
+	{
+		PDumpDisconnectionNotify(psDevNode);
+	}
+
+	/* Add a HOST_CLIENT_INFO event to match the one on connection */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		IMG_BOOL bHostStreamIsNull;
+		PVRSRV_RGXDEV_INFO  *psRgxDevInfo;
+
+		psRgxDevInfo = _RGX_DEVICE_INFO_FROM_NODE(psDevNode);
+
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+		bHostStreamIsNull = (IMG_BOOL)(psRgxDevInfo->hHWPerfHostStream == NULL);
+		OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+		if (!bHostStreamIsNull)
+		{
+			if (TLStreamIsOpenForReading(psRgxDevInfo->hHWPerfHostStream))
+			{
+				/* Announce this client connection in the host stream, if event mask is set */
+				RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(psDevNode, psConnectionData->pid, psConnectionData->pszProcName);
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Mark remaining resources for driver to free */
+	RIConnectionClosed(psConnectionData);
+#endif
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	PVRSRVDeInitialiseDMA(psDevNode, psConnectionData);
+#endif
+
+#if defined(DEBUG) || defined(PDUMP)
+	PVR_LOG(("%s disconnected - (devID = %u)", psConnectionData->pszProcName,
+	        psDevNode->sDevId.ui32InternalID));
+#endif
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK)
+#endif
+	{
+		/* Defer the release of the connection data */
+		psConnectionData->sCleanupThreadFn.pfnFree = _CleanupThreadPurgeConnectionData;
+		psConnectionData->sCleanupThreadFn.pvData = psConnectionData;
+		/* Some resources in HANDLE_BASE may need FW idle confirmation
+		 * hence setting to TRUE to use the global EO for retries which is
+		 * signalled by the device MISR */
+		psConnectionData->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+		psConnectionData->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_CONNECTION;
+		CLEANUP_THREAD_SET_RETRY_COUNT(&psConnectionData->sCleanupThreadFn,
+		                               CLEANUP_THREAD_RETRY_COUNT_DEFAULT);
+		PVRSRVCleanupThreadAddWork(psDevNode, &psConnectionData->sCleanupThreadFn);
+	}
+}
+
+IMG_PID PVRSRVGetPurgeConnectionPid(void)
+{
+	return gCurrentPurgeConnectionPid;
+}
+
+/* Prefix for debug messages about Active Connections */
+#define DEBUG_DUMP_CONNECTION_FORMAT_STR " P%d-V%d-T%d-%s,"
+#define CONNECTIONS_PREFIX               "Connections Device ID:%u(%d)"
+#define MAX_CONNECTIONS_PREFIX            (29)
+#define MAX_DEBUG_DUMP_CONNECTION_STR_LEN (1+10+10+10+7+PVRSRV_CONNECTION_PROCESS_NAME_LEN)
+#define MAX_DEBUG_DUMP_STRING_LEN         (1+MAX_CONNECTIONS_PREFIX+(3*MAX_DEBUG_DUMP_CONNECTION_STR_LEN))
+
+void PVRSRVConnectionDebugNotify(PVRSRV_DEVICE_NODE *psDevNode,
+								 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile)
+{
+	PDLLIST_NODE pNext, pNode;
+
+	/* We must check for an initialised device before accessing its mutex.
+	 * The mutex is initialised as part of DeviceInitialize() which occurs
+	 * on first access to the device node.
+	 */
+	if ((psDevNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE) &&
+	    (psDevNode->eDevState != PVRSRV_DEVICE_STATE_FROZEN))
+	{
+		PVR_DUMPDEBUG_LOG("Connections: No Devices: No active connections");
+		return;
+	}
+
+	OSLockAcquire(psDevNode->hConnectionsLock);
+	if (dllist_is_empty(&psDevNode->sConnections))
+	{
+		PVR_DUMPDEBUG_LOG(CONNECTIONS_PREFIX " No active connections",
+						  (unsigned char)psDevNode->sDevId.ui32InternalID,
+						  (unsigned char)psDevNode->sDevId.i32KernelDeviceID);
+	}
+	else
+	{
+		IMG_CHAR sActiveConnections[MAX_DEBUG_DUMP_STRING_LEN];
+		IMG_UINT16 i, uiPos = 0;
+		IMG_BOOL bPrinted = IMG_FALSE;
+		size_t uiSize = sizeof(sActiveConnections);
+
+		IMG_CHAR szTmpConBuff[MAX_CONNECTIONS_PREFIX + 1];
+		i = OSSNPrintf(szTmpConBuff,
+					   MAX_CONNECTIONS_PREFIX,
+					   CONNECTIONS_PREFIX,
+					   (unsigned char)psDevNode->sDevId.ui32InternalID,
+					   (unsigned char)psDevNode->sDevId.i32KernelDeviceID);
+		OSStringSafeCopy(sActiveConnections+uiPos, szTmpConBuff, uiSize);
+
+		/* Move the write offset to the end of the current string */
+		uiPos += i;
+		/* Update the amount of remaining space available to copy into */
+		uiSize -= i;
+
+		dllist_foreach_node(&psDevNode->sConnections, pNode, pNext)
+		{
+			CONNECTION_DATA *sData = IMG_CONTAINER_OF(pNode, CONNECTION_DATA, sConnectionListNode);
+
+			IMG_CHAR sTmpBuff[MAX_DEBUG_DUMP_CONNECTION_STR_LEN];
+			i = OSSNPrintf(sTmpBuff, MAX_DEBUG_DUMP_CONNECTION_STR_LEN,
+				DEBUG_DUMP_CONNECTION_FORMAT_STR, sData->pid, sData->vpid, sData->tid, sData->pszProcName);
+			i = MIN(MAX_DEBUG_DUMP_CONNECTION_STR_LEN, i);
+			bPrinted = IMG_FALSE;
+
+			OSStringSafeCopy(sActiveConnections+uiPos, sTmpBuff, uiSize);
+
+			/* Move the write offset to the end of the current string */
+			uiPos += i;
+			/* Update the amount of remaining space available to copy into */
+			uiSize -= i;
+
+			/* If there is not enough space to add another connection to this line, output the line */
+			if (uiSize <= MAX_DEBUG_DUMP_CONNECTION_STR_LEN)
+			{
+				PVR_DUMPDEBUG_LOG("%s", sActiveConnections);
+
+				/*
+				 * Remove the "Connections:" prefix from the buffer.
+				 * Leave the subsequent buffer contents indented by the same
+				 * amount to aid in interpreting the debug output.
+				 */
+				uiPos = sizeof(CONNECTIONS_PREFIX) - 1;
+				/* Reset the amount of space available to copy into */
+				uiSize = MAX_DEBUG_DUMP_STRING_LEN - uiPos;
+				bPrinted = IMG_TRUE;
+			}
+		}
+
+		/* Only print the current line if it hasn't already been printed */
+		if (!bPrinted)
+		{
+			/* Strip off the final comma */
+			sActiveConnections[OSStringNLength(sActiveConnections, MAX_DEBUG_DUMP_STRING_LEN) - 1] = '\0';
+			PVR_DUMPDEBUG_LOG("%s", sActiveConnections);
+		}
+#undef MAX_DEBUG_DUMP_STRING_LEN
+#undef MAX_DEBUG_DUMP_CONNECTIONS_PER_LINE
+	}
+	OSLockRelease(psDevNode->hConnectionsLock);
+}
diff --git a/drivers/gpu/drm/img-rogue/connection_server.h b/drivers/gpu/drm/img-rogue/connection_server.h
new file mode 100644
index 000000000000..673c37c02a3c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/connection_server.h
@@ -0,0 +1,143 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    API for server side connection management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(CONNECTION_SERVER_H)
+#define CONNECTION_SERVER_H
+
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "handle.h"
+#include "pvrsrv_cleanup.h"
+
+/* Variable used to hold in memory the timeout for the current time slice*/
+extern IMG_UINT64 gui64TimesliceLimit;
+/* Counter number of handle data freed during the current time slice */
+extern IMG_UINT32 gui32HandleDataFreeCounter;
+/* Set the maximum time the freeing of the resources can keep the lock */
+#define CONNECTION_DEFERRED_CLEANUP_TIMESLICE_NS (3000 * 1000) /* 3ms */
+
+typedef struct _CONNECTION_DATA_
+{
+	PVRSRV_HANDLE_BASE		*psHandleBase;
+	PROCESS_HANDLE_BASE		*psProcessHandleBase;
+	struct _SYNC_CONNECTION_DATA_	*psSyncConnectionData;
+	struct _PDUMP_CONNECTION_DATA_	*psPDumpConnectionData;
+
+	/* Holds the client flags supplied at connection time */
+	IMG_UINT32          ui32ClientFlags;
+
+	/*
+	 * OS specific data can be stored via this handle.
+	 * See osconnection_server.h for a generic mechanism
+	 * for initialising this field.
+	 */
+	IMG_HANDLE          hOsPrivateData;
+
+#define PVRSRV_CONNECTION_PROCESS_NAME_LEN (16)
+	IMG_PID             pid;
+	IMG_PID				vpid;
+	IMG_UINT32			tid;
+	IMG_CHAR            pszProcName[PVRSRV_CONNECTION_PROCESS_NAME_LEN];
+
+	IMG_HANDLE          hProcessStats;
+
+	IMG_HANDLE          hClientTLStream;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/*
+	 * Connection-based values per application which can be modified by the
+	 * AppHint settings 'OSid, OSidReg, bOSidAxiProtReg' for each application.
+	 * These control where the connection's memory allocation is sourced from.
+	 * ui32OSid, ui32OSidReg range from 0..(GPUVIRT_VALIDATION_NUM_OS - 1).
+	 */
+	IMG_UINT32          ui32OSid;
+	IMG_UINT32          ui32OSidReg;
+	IMG_BOOL            bOSidAxiProtReg;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	IMG_BOOL            bAcceptDmaRequests;
+	ATOMIC_T            ui32NumDmaTransfersInFlight;
+	IMG_HANDLE          hDmaEventObject;
+#endif
+	/* Structure which is hooked into the cleanup thread work list */
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+
+	DLLIST_NODE         sConnectionListNode;
+
+	/* List navigation for deferred freeing of connection data */
+	struct _CONNECTION_DATA_	**ppsThis;
+	struct _CONNECTION_DATA_	*psNext;
+} CONNECTION_DATA;
+
+#include "osconnection_server.h"
+
+PVRSRV_ERROR PVRSRVCommonConnectionConnect(void **ppvPrivData, void *pvOSData);
+void PVRSRVCommonConnectionDisconnect(void *pvPrivData);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVGetPurgeConnectionPid
+
+@Description    Returns PID associated with Connection currently being purged by
+                Cleanup Thread. If no Connection is purged 0 is returned.
+
+@Return         PID associated with currently purged connection or 0 if no
+                connection is being purged
+*/ /***************************************************************************/
+IMG_PID PVRSRVGetPurgeConnectionPid(void);
+
+void PVRSRVConnectionDebugNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile);
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVConnectionPrivateData)
+#endif
+static INLINE
+IMG_HANDLE PVRSRVConnectionPrivateData(CONNECTION_DATA *psConnection)
+{
+	return (psConnection != NULL) ? psConnection->hOsPrivateData : NULL;
+}
+
+#endif /* !defined(CONNECTION_SERVER_H) */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_1.39.4.19.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_1.39.4.19.h
new file mode 100644
index 000000000000..a64cb4f379e9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_1.39.4.19.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.39.4.19
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_39_4_19_H
+#define RGXCORE_KM_1_39_4_19_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2784771 */
+
+/******************************************************************************
+ * BVNC = 1.39.4.19
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 39
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 19
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344
+#define FIX_HW_BRN_42321
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_39_4_19_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_1.82.4.5.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_1.82.4.5.h
new file mode 100644
index 000000000000..ae53d7ceef27
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_1.82.4.5.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 1.82.4.5
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_1_82_4_5_H
+#define RGXCORE_KM_1_82_4_5_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2503111 */
+
+/******************************************************************************
+ * BVNC = 1.82.4.5
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 1
+#define RGX_BVNC_KM_V 82
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 5
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_1_82_4_5_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_15.5.1.64.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_15.5.1.64.h
new file mode 100644
index 000000000000..e354f5b8c8fe
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_15.5.1.64.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 15.5.1.64
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_15_5_1_64_H
+#define RGXCORE_KM_15_5_1_64_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3846532 */
+
+/******************************************************************************
+ * BVNC = 15.5.1.64
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 15
+#define RGX_BVNC_KM_V 5
+#define RGX_BVNC_KM_N 1
+#define RGX_BVNC_KM_C 64
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_15_5_1_64_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.102.54.38.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.102.54.38.h
new file mode 100644
index 000000000000..ee279a63a255
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.102.54.38.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.102.54.38
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_102_54_38_H
+#define RGXCORE_KM_22_102_54_38_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4831550 */
+
+/******************************************************************************
+ * BVNC = 22.102.54.38
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 102
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 38
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_102_54_38_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.104.208.318.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.104.208.318.h
new file mode 100644
index 000000000000..58472828a1b0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.104.208.318.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.104.208.318
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_104_208_318_H
+#define RGXCORE_KM_22_104_208_318_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5124208 */
+
+/******************************************************************************
+ * BVNC = 22.104.208.318
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 104
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 318
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_104_208_318_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.30.54.25.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.30.54.25.h
new file mode 100644
index 000000000000..34d2c176900a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.30.54.25.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.30.54.25
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_30_54_25_H
+#define RGXCORE_KM_22_30_54_25_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4086500 */
+
+/******************************************************************************
+ * BVNC = 22.30.54.25
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 30
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 25
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_30_54_25_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.40.54.30.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.40.54.30.h
new file mode 100644
index 000000000000..f9dfea7676f1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.40.54.30.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.40.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_40_54_30_H
+#define RGXCORE_KM_22_40_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4094817 */
+
+/******************************************************************************
+ * BVNC = 22.40.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 40
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_40_54_30_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.46.54.330.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.46.54.330.h
new file mode 100644
index 000000000000..f269fc0c8a12
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.46.54.330.h
@@ -0,0 +1,78 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.46.54.330
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_46_54_330_H
+#define RGXCORE_KM_22_46_54_330_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4136505 */
+
+/******************************************************************************
+ * BVNC = 22.46.54.330
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 46
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 330
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_22_46_54_330_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.49.21.16.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.49.21.16.h
new file mode 100644
index 000000000000..f1cc4df22992
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.49.21.16.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.49.21.16
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_49_21_16_H
+#define RGXCORE_KM_22_49_21_16_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4158766 */
+
+/******************************************************************************
+ * BVNC = 22.49.21.16
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 49
+#define RGX_BVNC_KM_N 21
+#define RGX_BVNC_KM_C 16
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_61450
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_22_49_21_16_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.67.54.30.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.67.54.30.h
new file mode 100644
index 000000000000..c60d20b87718
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.67.54.30.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.67.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_67_54_30_H
+#define RGXCORE_KM_22_67_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4339986 */
+
+/******************************************************************************
+ * BVNC = 22.67.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 67
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_60084
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_67_54_30_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.68.54.30.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.68.54.30.h
new file mode 100644
index 000000000000..b5f8459a1442
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.68.54.30.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.68.54.30
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_68_54_30_H
+#define RGXCORE_KM_22_68_54_30_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4339984 */
+
+/******************************************************************************
+ * BVNC = 22.68.54.30
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 68
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 30
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65273
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_68_54_30_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.86.104.218.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.86.104.218.h
new file mode 100644
index 000000000000..a0c3001f1147
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.86.104.218.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.86.104.218
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_86_104_218_H
+#define RGXCORE_KM_22_86_104_218_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4665024 */
+
+/******************************************************************************
+ * BVNC = 22.86.104.218
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 86
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 218
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_65101
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_86_104_218_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.87.104.18.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.87.104.18.h
new file mode 100644
index 000000000000..2a2c66ad95d4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_22.87.104.18.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 22.87.104.18
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_22_87_104_18_H
+#define RGXCORE_KM_22_87_104_18_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4658768 */
+
+/******************************************************************************
+ * BVNC = 22.87.104.18
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 22
+#define RGX_BVNC_KM_V 87
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 18
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_64502
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_61389
+
+
+
+#endif /* RGXCORE_KM_22_87_104_18_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.50.208.504.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.50.208.504.h
new file mode 100644
index 000000000000..1aca3f49a04c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.50.208.504.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.50.208.504
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_50_208_504_H
+#define RGXCORE_KM_24_50_208_504_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5086680 */
+
+/******************************************************************************
+ * BVNC = 24.50.208.504
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 50
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 504
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_50_208_504_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.56.208.505.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.56.208.505.h
new file mode 100644
index 000000000000..26060f932d47
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.56.208.505.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.56.208.505
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_56_208_505_H
+#define RGXCORE_KM_24_56_208_505_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5203837 */
+
+/******************************************************************************
+ * BVNC = 24.56.208.505
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 56
+#define RGX_BVNC_KM_N 208
+#define RGX_BVNC_KM_C 505
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_56_208_505_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.66.54.204.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.66.54.204.h
new file mode 100644
index 000000000000..4fb0ef0395b3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.66.54.204.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.66.54.204
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_66_54_204_H
+#define RGXCORE_KM_24_66_54_204_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5200207 */
+
+/******************************************************************************
+ * BVNC = 24.66.54.204
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 66
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 204
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_66_54_204_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.67.104.504.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.67.104.504.h
new file mode 100644
index 000000000000..4266f63edac9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_24.67.104.504.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 24.67.104.504
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_24_67_104_504_H
+#define RGXCORE_KM_24_67_104_504_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5203838 */
+
+/******************************************************************************
+ * BVNC = 24.67.104.504
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 24
+#define RGX_BVNC_KM_V 67
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 504
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_24_67_104_504_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_29.14.108.208.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_29.14.108.208.h
new file mode 100644
index 000000000000..6636f54022c7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_29.14.108.208.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 29.14.108.208
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_29_14_108_208_H
+#define RGXCORE_KM_29_14_108_208_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5682134 */
+
+/******************************************************************************
+ * BVNC = 29.14.108.208
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 29
+#define RGX_BVNC_KM_V 14
+#define RGX_BVNC_KM_N 108
+#define RGX_BVNC_KM_C 208
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_68186
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_29_14_108_208_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_29.19.52.202.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_29.19.52.202.h
new file mode 100644
index 000000000000..01683cb721ee
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_29.19.52.202.h
@@ -0,0 +1,77 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 29.19.52.202
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_29_19_52_202_H
+#define RGXCORE_KM_29_19_52_202_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5691248 */
+
+/******************************************************************************
+ * BVNC = 29.19.52.202
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 29
+#define RGX_BVNC_KM_V 19
+#define RGX_BVNC_KM_N 52
+#define RGX_BVNC_KM_C 202
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_68186
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+#define HW_ERN_66622
+
+
+
+#endif /* RGXCORE_KM_29_19_52_202_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_33.15.11.3.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_33.15.11.3.h
new file mode 100644
index 000000000000..9762fa901bda
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_33.15.11.3.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 33.15.11.3
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_33_15_11_3_H
+#define RGXCORE_KM_33_15_11_3_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5820045 */
+
+/******************************************************************************
+ * BVNC = 33.15.11.3
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 33
+#define RGX_BVNC_KM_V 15
+#define RGX_BVNC_KM_N 11
+#define RGX_BVNC_KM_C 3
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_33_15_11_3_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_33.8.22.1.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_33.8.22.1.h
new file mode 100644
index 000000000000..272e922e0191
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_33.8.22.1.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 33.8.22.1
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_33_8_22_1_H
+#define RGXCORE_KM_33_8_22_1_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5544349 */
+
+/******************************************************************************
+ * BVNC = 33.8.22.1
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 33
+#define RGX_BVNC_KM_V 8
+#define RGX_BVNC_KM_N 22
+#define RGX_BVNC_KM_C 1
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_33_8_22_1_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.29.52.182.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.29.52.182.h
new file mode 100644
index 000000000000..7a5a87c84f7b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.29.52.182.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.29.52.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_29_52_182_H
+#define RGXCORE_KM_36_29_52_182_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908879 */
+
+/******************************************************************************
+ * BVNC = 36.29.52.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 29
+#define RGX_BVNC_KM_N 52
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_29_52_182_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.50.54.182.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.50.54.182.h
new file mode 100644
index 000000000000..0eccc97305ab
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.50.54.182.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.50.54.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_50_54_182_H
+#define RGXCORE_KM_36_50_54_182_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5849603 */
+
+/******************************************************************************
+ * BVNC = 36.50.54.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 50
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_50_54_182_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.52.104.182.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.52.104.182.h
new file mode 100644
index 000000000000..d14b662b9730
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.52.104.182.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.52.104.182
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_52_104_182_H
+#define RGXCORE_KM_36_52_104_182_H
+
+/* Automatically generated file (14/08/2023 09:10:43): Do not edit manually */
+/* CS: @5849605 */
+
+/******************************************************************************
+ * BVNC = 36.52.104.182
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 52
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 182
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71242
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_52_104_182_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.53.104.796.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.53.104.796.h
new file mode 100644
index 000000000000..c3ec305d3830
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.53.104.796.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.53.104.796
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_53_104_796_H
+#define RGXCORE_KM_36_53_104_796_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5896094 */
+
+/******************************************************************************
+ * BVNC = 36.53.104.796
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 53
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 796
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_53_104_796_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.54.54.183.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.54.54.183.h
new file mode 100644
index 000000000000..6cac1b26d084
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.54.54.183.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.54.54.183
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_54_54_183_H
+#define RGXCORE_KM_36_54_54_183_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908021 */
+
+/******************************************************************************
+ * BVNC = 36.54.54.183
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 54
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 183
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_54_54_183_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.55.54.103.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.55.54.103.h
new file mode 100644
index 000000000000..809d5808d3ea
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.55.54.103.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.55.54.103
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_55_54_103_H
+#define RGXCORE_KM_36_55_54_103_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5908021 */
+
+/******************************************************************************
+ * BVNC = 36.55.54.103
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 55
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 103
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_55_54_103_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.56.104.183.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.56.104.183.h
new file mode 100644
index 000000000000..96e875e3b1f4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.56.104.183.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.56.104.183
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_56_104_183_H
+#define RGXCORE_KM_36_56_104_183_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @5942195 */
+
+/******************************************************************************
+ * BVNC = 36.56.104.183
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 56
+#define RGX_BVNC_KM_N 104
+#define RGX_BVNC_KM_C 183
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_56_104_183_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.60.54.280.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.60.54.280.h
new file mode 100644
index 000000000000..5f46e75d4f06
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_36.60.54.280.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 36.60.54.280
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_36_60_54_280_H
+#define RGXCORE_KM_36_60_54_280_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @6040409 */
+
+/******************************************************************************
+ * BVNC = 36.60.54.280
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 36
+#define RGX_BVNC_KM_V 60
+#define RGX_BVNC_KM_N 54
+#define RGX_BVNC_KM_C 280
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_36_60_54_280_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.31.4.55.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.31.4.55.h
new file mode 100644
index 000000000000..a39603ae35a1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.31.4.55.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.31.4.55
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_31_4_55_H
+#define RGXCORE_KM_4_31_4_55_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2919104 */
+
+/******************************************************************************
+ * BVNC = 4.31.4.55
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 31
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 55
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_31_4_55_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.40.2.51.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.40.2.51.h
new file mode 100644
index 000000000000..931fa0cd445c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.40.2.51.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.40.2.51
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_40_2_51_H
+#define RGXCORE_KM_4_40_2_51_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3254374 */
+
+/******************************************************************************
+ * BVNC = 4.40.2.51
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 40
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 51
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_40_2_51_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.43.6.62.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.43.6.62.h
new file mode 100644
index 000000000000..90d35a8360bb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.43.6.62.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.43.6.62
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_43_6_62_H
+#define RGXCORE_KM_4_43_6_62_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3253129 */
+
+/******************************************************************************
+ * BVNC = 4.43.6.62
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 43
+#define RGX_BVNC_KM_N 6
+#define RGX_BVNC_KM_C 62
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_43_6_62_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.45.2.58.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.45.2.58.h
new file mode 100644
index 000000000000..00e6b90f6207
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.45.2.58.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.45.2.58
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_45_2_58_H
+#define RGXCORE_KM_4_45_2_58_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3547765 */
+
+/******************************************************************************
+ * BVNC = 4.45.2.58
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 45
+#define RGX_BVNC_KM_N 2
+#define RGX_BVNC_KM_C 58
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_45_2_58_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.46.6.62.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.46.6.62.h
new file mode 100644
index 000000000000..909198f233b4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_4.46.6.62.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 4.46.6.62
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_4_46_6_62_H
+#define RGXCORE_KM_4_46_6_62_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @4015666 */
+
+/******************************************************************************
+ * BVNC = 4.46.6.62
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 4
+#define RGX_BVNC_KM_V 46
+#define RGX_BVNC_KM_N 6
+#define RGX_BVNC_KM_C 62
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_50767
+#define FIX_HW_BRN_63142
+#define FIX_HW_BRN_64502
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_4_46_6_62_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_46.72.102.389.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_46.72.102.389.h
new file mode 100644
index 000000000000..66eb259c0ebf
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_46.72.102.389.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 46.72.102.389
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_46_72_102_389_H
+#define RGXCORE_KM_46_72_102_389_H
+
+/* Automatically generated file (07/02/2023 09:11:39): Do not edit manually */
+/* CS: @6385487 */
+
+/******************************************************************************
+ * BVNC = 46.72.102.389
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 46
+#define RGX_BVNC_KM_V 72
+#define RGX_BVNC_KM_N 102
+#define RGX_BVNC_KM_C 389
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_66927
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_46_72_102_389_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_46.94.204.390.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_46.94.204.390.h
new file mode 100644
index 000000000000..548989eb518d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_46.94.204.390.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 46.94.204.390
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_46_94_204_390_H
+#define RGXCORE_KM_46_94_204_390_H
+
+/* Automatically generated file (02/03/2023 09:11:42): Do not edit manually */
+/* CS: @6389604 */
+
+/******************************************************************************
+ * BVNC = 46.94.204.390
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 46
+#define RGX_BVNC_KM_V 94
+#define RGX_BVNC_KM_N 204
+#define RGX_BVNC_KM_C 390
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63553
+#define FIX_HW_BRN_66927
+#define FIX_HW_BRN_71317
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+#define HW_ERN_47025
+#define HW_ERN_57596
+
+
+
+#endif /* RGXCORE_KM_46_94_204_390_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_5.9.1.46.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_5.9.1.46.h
new file mode 100644
index 000000000000..cba592e7b327
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_5.9.1.46.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 5.9.1.46
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_5_9_1_46_H
+#define RGXCORE_KM_5_9_1_46_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @2967148 */
+
+/******************************************************************************
+ * BVNC = 5.9.1.46
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 5
+#define RGX_BVNC_KM_V 9
+#define RGX_BVNC_KM_N 1
+#define RGX_BVNC_KM_C 46
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+
+
+
+#endif /* RGXCORE_KM_5_9_1_46_H */
diff --git a/drivers/gpu/drm/img-rogue/cores/rgxcore_km_6.34.4.35.h b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_6.34.4.35.h
new file mode 100644
index 000000000000..b45ed9e1547c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/cores/rgxcore_km_6.34.4.35.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@Title          RGX Core BVNC 6.34.4.35
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXCORE_KM_6_34_4_35_H
+#define RGXCORE_KM_6_34_4_35_H
+
+/* Automatically generated file (21/01/2022 09:01:15): Do not edit manually */
+/* CS: @3533654 */
+
+/******************************************************************************
+ * BVNC = 6.34.4.35
+ *****************************************************************************/
+#define RGX_BVNC_KM_B 6
+#define RGX_BVNC_KM_V 34
+#define RGX_BVNC_KM_N 4
+#define RGX_BVNC_KM_C 35
+
+/******************************************************************************
+ * Errata
+ *****************************************************************************/
+
+#define FIX_HW_BRN_63142
+
+
+
+/******************************************************************************
+ * Enhancements
+ *****************************************************************************/
+#define HW_ERN_42290
+#define HW_ERN_42606
+
+
+
+#endif /* RGXCORE_KM_6_34_4_35_H */
diff --git a/drivers/gpu/drm/img-rogue/debug_common.c b/drivers/gpu/drm/img-rogue/debug_common.c
new file mode 100644
index 000000000000..16746b5d550a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/debug_common.c
@@ -0,0 +1,1994 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Creates common debug info entries.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__linux__)
+#include <errno.h>
+#endif /* #if !defined(__linux__) */
+
+#include "debug_common.h"
+#include "pvrsrv.h"
+#include "di_server.h"
+#include "lists.h"
+#include "pvrversion.h"
+#include "rgx_options.h"
+#include "allocmem.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "osfunc.h"
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#include "rgxfwdbg.h"
+#endif
+
+#ifdef SUPPORT_RGX
+#include "rgxdevice.h"
+#include "rgxdebug_common.h"
+#include "rgxinit.h"
+#include "rgxmmudefs_km.h"
+#endif
+
+static DI_ENTRY *gpsVersionDIEntry;
+static DI_ENTRY *gpsStatusDIEntry;
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+static DI_ENTRY *gpsDebugLevelDIEntry;
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+struct DI_VZ_DATA {
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_UINT32 ui32DriverID;
+};
+#endif
+
+static void _DumpDebugDIPrintfWrapper(void *pvDumpDebugFile, const IMG_CHAR *pszFormat, ...)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list ArgList;
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN, "%s\n", pszFormat);
+
+	va_start(ArgList, pszFormat);
+	DIVPrintf(pvDumpDebugFile, szBuffer, ArgList);
+	va_end(ArgList);
+}
+
+/*************************************************************************/ /*!
+ Version DebugFS entry
+*/ /**************************************************************************/
+
+static void *_DebugVersionCompare_AnyVaCb(PVRSRV_DEVICE_NODE *psDevNode,
+                                          va_list va)
+{
+	IMG_UINT64 *pui64CurrentPosition = va_arg(va, IMG_UINT64 *);
+	IMG_UINT64 ui64Position = va_arg(va, IMG_UINT64);
+	IMG_UINT64 ui64CurrentPosition = *pui64CurrentPosition;
+
+	(*pui64CurrentPosition)++;
+
+	return (ui64CurrentPosition == ui64Position) ? psDevNode : NULL;
+}
+
+static void *_VersionDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+
+	if (psPVRSRVData == NULL) {
+		PVR_DPF((PVR_DBG_ERROR, "psPVRSRVData = NULL"));
+		return NULL;
+	}
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+	                                      _DebugVersionCompare_AnyVaCb,
+	                                      &uiCurrentPosition,
+	                                      *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+static void _VersionDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvPriv)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+}
+
+static void *_VersionDINext(OSDI_IMPL_ENTRY *psEntry,void *pvPriv,
+                            IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	(*pui64Pos)++;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+	                                      _DebugVersionCompare_AnyVaCb,
+	                                      &uiCurrentPosition,
+	                                      *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+#define DI_PRINT_VERSION_FMTSPEC \
+		"%s Version: %u.%u @ %u (%s) build options: 0x%08x %s\n"
+#define STR_DEBUG   "debug"
+#define STR_RELEASE "release"
+
+#if defined(DEBUG)
+#define BUILD_OPT_LEN 80
+
+static inline void _AppendOptionStr(IMG_CHAR pszBuildOptions[], const IMG_CHAR* str, OSDI_IMPL_ENTRY *psEntry, IMG_UINT32* pui32BuildOptionLen)
+{
+	IMG_UINT32 ui32BuildOptionLen = *pui32BuildOptionLen;
+	const IMG_UINT32 strLen = OSStringLength(str);
+	const IMG_UINT32 optStrLen = sizeof(IMG_CHAR) * (BUILD_OPT_LEN-1);
+
+	if ((ui32BuildOptionLen + strLen) > optStrLen)
+	{
+		pszBuildOptions[ui32BuildOptionLen] = '\0';
+		DIPrintf(psEntry, "%s\n", pszBuildOptions);
+		ui32BuildOptionLen = 0;
+	}
+	if (strLen < optStrLen)
+	{
+		OSStringSafeCopy(pszBuildOptions+ui32BuildOptionLen, str, strLen);
+		ui32BuildOptionLen += strLen - 1;
+	}
+	*pui32BuildOptionLen = ui32BuildOptionLen;
+}
+#endif /* DEBUG || SUPPORT_VALIDATION */
+
+static int _VersionDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvPriv)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+
+	if (pvPriv == DI_START_TOKEN)
+	{
+		if (psPVRSRVData->sDriverInfo.bIsNoMatch)
+		{
+			const BUILD_INFO *psBuildInfo;
+
+			psBuildInfo = &psPVRSRVData->sDriverInfo.sUMBuildInfo;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+			         "UM Driver",
+			         PVRVERSION_UNPACK_MAJ(psBuildInfo->ui32BuildVersion),
+			         PVRVERSION_UNPACK_MIN(psBuildInfo->ui32BuildVersion),
+			         psBuildInfo->ui32BuildRevision,
+			         (psBuildInfo->ui32BuildType == BUILD_TYPE_DEBUG) ?
+			                 STR_DEBUG : STR_RELEASE,
+			         psBuildInfo->ui32BuildOptions,
+			         PVR_BUILD_DIR);
+
+			psBuildInfo = &psPVRSRVData->sDriverInfo.sKMBuildInfo;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+			         "KM Driver (" PVR_ARCH_NAME ")",
+			         PVRVERSION_UNPACK_MAJ(psBuildInfo->ui32BuildVersion),
+			         PVRVERSION_UNPACK_MIN(psBuildInfo->ui32BuildVersion),
+			         psBuildInfo->ui32BuildRevision,
+			         (psBuildInfo->ui32BuildType == BUILD_TYPE_DEBUG) ?
+			                 STR_DEBUG : STR_RELEASE,
+			         psBuildInfo->ui32BuildOptions,
+			         PVR_BUILD_DIR);
+		}
+		else
+		{
+			/* bIsNoMatch is `false` in one of the following cases:
+			 * - UM & KM version parameters actually match.
+			 * - A comparison between UM & KM has not been made yet, because no
+			 *   client ever connected.
+			 *
+			 * In both cases, available (KM) version info is the best output we
+			 * can provide.
+			 */
+			DIPrintf(psEntry, "Driver Version: %s (%s) (%s) build options: "
+			         "0x%08lx %s\n", PVRVERSION_STRING, PVR_ARCH_NAME,
+			         PVR_BUILD_TYPE, RGX_BUILD_OPTIONS_KM, PVR_BUILD_DIR);
+		}
+	}
+	else if (pvPriv != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *) pvPriv;
+		PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+#ifdef SUPPORT_RGX
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+#if defined(DEBUG)
+		IMG_CHAR pszBuildOptions[BUILD_OPT_LEN];
+		IMG_UINT32 ui32BuildOptionLen = 0;
+		static const char* aszOptions[] = RGX_BUILD_OPTIONS_LIST;
+		int i = 0;
+#endif
+#endif /* SUPPORT_RGX */
+		IMG_BOOL bFwVersionInfoPrinted = IMG_FALSE;
+
+		DIPrintf(psEntry, "\nDevice Name: %s\n", psDevConfig->pszName);
+		DIPrintf(psEntry, "Device ID: %u:%d\n", psDevNode->sDevId.ui32InternalID,
+		                                        psDevNode->sDevId.i32KernelDeviceID);
+
+		if (psDevConfig->pszVersion)
+		{
+			DIPrintf(psEntry, "Device Version: %s\n",
+			          psDevConfig->pszVersion);
+		}
+
+		if (psDevNode->pfnDeviceVersionString)
+		{
+			IMG_CHAR *pszVerStr;
+
+			if (psDevNode->pfnDeviceVersionString(psDevNode,
+			                                      &pszVerStr) == PVRSRV_OK)
+			{
+				DIPrintf(psEntry, "%s\n", pszVerStr);
+
+				OSFreeMem(pszVerStr);
+			}
+		}
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+		{
+#ifdef SUPPORT_RGX
+			/* print device's firmware version info */
+			if (psDevInfo->psRGXFWIfOsInitMemDesc != NULL)
+			{
+				/* psDevInfo->psRGXFWIfOsInitMemDesc should be permanently mapped */
+				if (psDevInfo->psRGXFWIfOsInit != NULL)
+				{
+					RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+				                               INVALIDATE);
+					if (psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated)
+					{
+						const RGXFWIF_COMPCHECKS *psRGXCompChecks =
+						        &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks;
+						IMG_UINT32 ui32DDKVer = psRGXCompChecks->ui32DDKVersion;
+
+						DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+						         "Firmware",
+						         PVRVERSION_UNPACK_MAJ(ui32DDKVer),
+						         PVRVERSION_UNPACK_MIN(ui32DDKVer),
+						         psRGXCompChecks->ui32DDKBuild,
+						         ((psRGXCompChecks->ui32BuildOptions &
+						          OPTIONS_DEBUG_EN) ? STR_DEBUG : STR_RELEASE),
+						         psRGXCompChecks->ui32BuildOptions,
+						         PVR_BUILD_DIR);
+						bFwVersionInfoPrinted = IMG_TRUE;
+
+#if defined(DEBUG)
+						DIPrintf(psEntry, "Firmware Build Options:\n");
+
+						for (i = 0; i < ARRAY_SIZE(aszOptions); i++)
+						{
+							if ((psRGXCompChecks->ui32BuildOptions & 1<<i))
+							{
+								_AppendOptionStr(pszBuildOptions, aszOptions[i], psEntry, &ui32BuildOptionLen);
+							}
+						}
+
+						if (ui32BuildOptionLen != 0)
+						{
+							DIPrintf(psEntry, "%s", pszBuildOptions);
+						}
+						DIPrintf(psEntry, "\n");
+#endif
+					}
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Error acquiring CPU virtual "
+					        "address of FWInitMemDesc", __func__));
+				}
+			}
+#endif /* SUPPORT_RGX */
+		}
+		else
+		{
+#ifdef SUPPORT_RGX
+			RGX_FW_INFO_HEADER	*psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+			DIPrintf(psEntry, DI_PRINT_VERSION_FMTSPEC,
+					 "Firmware",
+					 psFWInfoHeader->ui16PVRVersionMajor,
+					 psFWInfoHeader->ui16PVRVersionMinor,
+					 psFWInfoHeader->ui32PVRVersionBuild,
+					 ((psFWInfoHeader->ui32Flags &
+					 OPTIONS_DEBUG_EN) ? STR_DEBUG : STR_RELEASE),
+					 psFWInfoHeader->ui32Flags,
+					 PVR_BUILD_DIR);
+
+			bFwVersionInfoPrinted = IMG_TRUE;
+#if defined(DEBUG)
+			DIPrintf(psEntry, "Firmware Build Options:\n");
+
+			for (i = 0; i < ARRAY_SIZE(aszOptions); i++)
+			{
+				if ((psFWInfoHeader->ui32Flags & 1<<i))
+				{
+					_AppendOptionStr(pszBuildOptions, aszOptions[i], psEntry, &ui32BuildOptionLen);
+				}
+			}
+
+			if (ui32BuildOptionLen != 0)
+			{
+				DIPrintf(psEntry, "%s", pszBuildOptions);
+			}
+			DIPrintf(psEntry, "\n");
+#endif
+
+#endif /* SUPPORT_RGX */
+		}
+
+		if (!bFwVersionInfoPrinted)
+		{
+			DIPrintf(psEntry, "Firmware Version: Info unavailable %s\n",
+#ifdef NO_HARDWARE
+			         "on NoHW driver"
+#else /* NO_HARDWARE */
+			         "(Is INIT complete?)"
+#endif /* NO_HARDWARE */
+			         );
+		}
+	}
+
+	return 0;
+}
+
+#if defined(SUPPORT_RGX) && defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+
+/*************************************************************************/ /*!
+ Power data DebugFS entry
+*/ /**************************************************************************/
+
+static PVRSRV_ERROR SendPowerCounterCommand(PVRSRV_DEVICE_NODE* psDeviceNode,
+                                            RGXFWIF_COUNTER_DUMP_REQUEST eRequestType,
+                                            IMG_UINT32 *pui32kCCBCommandSlot)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFWIF_KCCB_CMD sCounterDumpCmd;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCounterDumpCmd.eCmdType = RGXFWIF_KCCB_CMD_COUNTER_DUMP;
+	sCounterDumpCmd.uCmdData.sCounterDumpConfigData.eCounterDumpRequest = eRequestType;
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sCounterDumpCmd,
+				PDUMP_FLAGS_CONTINUOUS,
+				pui32kCCBCommandSlot);
+	PVR_LOG_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot");
+
+	return eError;
+}
+
+static int _DebugPowerDataDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	int eError = 0;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "power counter data was requested!"));
+		return -EIO;
+	}
+
+	OSLockAcquire(psDevInfo->hCounterDumpingLock);
+
+	eError = SendPowerCounterCommand(psDeviceNode,
+									 RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+									 &ui32kCCBCommandSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		OSLockRelease(psDevInfo->hCounterDumpingLock);
+		return -EIO;
+	}
+
+	/* Wait for FW complete completion */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo,
+									  ui32kCCBCommandSlot,
+									  PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+		OSLockRelease(psDevInfo->hCounterDumpingLock);
+		return -EIO;
+	}
+
+	/* Read back the buffer */
+	{
+		IMG_UINT32* pui32PowerBuffer;
+		IMG_UINT32 ui32NumOfRegs, ui32SamplePeriod, ui32NumOfCores;
+		IMG_UINT32 i, j;
+
+		if (!psDevInfo->psCounterBufferMemDesc)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Counter buffer not allocated!"));
+			return -EINVAL;
+		}
+
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCounterBufferMemDesc,
+										  (void**)&pui32PowerBuffer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "DevmemAcquireCpuVirtAddr");
+			OSLockRelease(psDevInfo->hCounterDumpingLock);
+			return -EIO;
+		}
+
+		RGXFwSharedMemCacheOpExec(pui32PowerBuffer, PAGE_SIZE, PVRSRV_CACHE_OP_INVALIDATE);
+
+		ui32NumOfRegs = *pui32PowerBuffer++;
+		ui32SamplePeriod = *pui32PowerBuffer++;
+		ui32NumOfCores = *pui32PowerBuffer++;
+		PVR_DPF((PVR_DBG_MESSAGE, "Number of power counters: %u.", ui32NumOfRegs));
+
+		if (ui32NumOfCores == 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "No GPU cores enabled!"));
+			eError = -EINVAL;
+		}
+
+		if (ui32NumOfRegs && ui32NumOfCores)
+		{
+			DIPrintf(psEntry, "Power counter data for device\n");
+			DIPrintf(psEntry, "Sample period: 0x%08x\n", ui32SamplePeriod);
+
+			for (i = 0; i < ui32NumOfRegs; i++)
+			{
+				IMG_UINT32 ui32High, ui32Low;
+				IMG_UINT32 ui32RegOffset = *pui32PowerBuffer++;
+				IMG_UINT32 ui32NumOfInstances = *pui32PowerBuffer++;
+
+				PVR_ASSERT(ui32NumOfInstances);
+
+				DIPrintf(psEntry, "0x%08x:", ui32RegOffset);
+
+				for (j = 0; j < ui32NumOfInstances * ui32NumOfCores; j++)
+				{
+					ui32Low = *pui32PowerBuffer++;
+#if defined(RGX_FEATURE_CATURIX_XTP_TOP_INFRASTRUCTURE_BIT_MASK)
+					if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CATURIX_XTP_TOP_INFRASTRUCTURE))
+					{
+						/* Power counters have 32-bit range */
+						DIPrintf(psEntry, " 0x%08x", ui32Low);
+					}
+					else
+#endif
+					{
+						/* Power counters have 64-bit range */
+						ui32High = *pui32PowerBuffer++;
+
+						DIPrintf(psEntry, " 0x%016" IMG_UINT64_FMTSPECx,
+								 (IMG_UINT64) ui32Low | (IMG_UINT64) ui32High << 32);
+					}
+				}
+
+				DIPrintf(psEntry, "\n");
+			}
+		}
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psCounterBufferMemDesc);
+	}
+
+	OSLockRelease(psDevInfo->hCounterDumpingLock);
+
+	return eError;
+}
+
+static IMG_INT64 PowerDataSet(const IMG_CHAR __user *pcBuffer,
+                              IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                              void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*)pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_COUNTER_DUMP_REQUEST eRequest;
+	IMG_UINT32 ui32kCCBCommandSlot;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "power counter data was requested!"));
+		return -EIO;
+	}
+
+	if (pcBuffer[0] == '1')
+	{
+		eRequest = RGXFWIF_PWR_COUNTER_DUMP_START;
+	}
+	else if (pcBuffer[0] == '0')
+	{
+		eRequest = RGXFWIF_PWR_COUNTER_DUMP_STOP;
+	}
+	else
+	{
+		return -EINVAL;
+	}
+
+	OSLockAcquire(psDevInfo->hCounterDumpingLock);
+
+	SendPowerCounterCommand(psDeviceNode,
+	                        eRequest,
+	                        &ui32kCCBCommandSlot);
+
+	OSLockRelease(psDevInfo->hCounterDumpingLock);
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+#endif /* defined(SUPPORT_RGX) && defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+/*************************************************************************/ /*!
+ Status DebugFS entry
+*/ /**************************************************************************/
+
+static void *_DebugStatusCompare_AnyVaCb(PVRSRV_DEVICE_NODE *psDevNode,
+										 va_list va)
+{
+	IMG_UINT64 *pui64CurrentPosition = va_arg(va, IMG_UINT64 *);
+	IMG_UINT64 ui64Position = va_arg(va, IMG_UINT64);
+	IMG_UINT64 ui64CurrentPosition = *pui64CurrentPosition;
+
+	(*pui64CurrentPosition)++;
+
+	return (ui64CurrentPosition == ui64Position) ? psDevNode : NULL;
+}
+
+static void *_DebugStatusDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =  List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+										  _DebugStatusCompare_AnyVaCb,
+										  &uiCurrentPosition,
+										  *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psDeviceNode;
+}
+
+static void _DebugStatusDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+}
+
+static void *_DebugStatusDINext(OSDI_IMPL_ENTRY *psEntry,
+								 void *pvData,
+								 IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+	IMG_UINT64 uiCurrentPosition = 1;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	(*pui64Pos)++;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =  List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+										  _DebugStatusCompare_AnyVaCb,
+										  &uiCurrentPosition,
+										  *pui64Pos);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+#ifdef SUPPORT_RGX
+	if (psDeviceNode && !PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+		if (psDevInfo && psDevInfo->pfnGetGpuUtilStats)
+		{
+			PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+			PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+			if (eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+			{
+				PVRSRV_ERROR eError;
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+				static IMG_BOOL bFirstTime = IMG_TRUE;
+#endif
+
+				OSLockAcquire(psDevInfo->hGpuUtilStatsLock);
+
+				eError = psDevInfo->pfnGetGpuUtilStats(psDeviceNode,
+													   psDebugInfo->hGpuUtilUserDebugFS,
+													   &psDevInfo->sGpuUtilStats);
+
+				OSLockRelease(psDevInfo->hGpuUtilStatsLock);
+
+				if (eError != PVRSRV_OK)
+				{
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+					if (bFirstTime)
+					{
+						bFirstTime = IMG_FALSE;
+#endif	/* defined(EMULATOR) || defined(VIRTUAL_PLATFORM) */
+					PVR_DPF((PVR_DBG_ERROR,
+					        "%s: Failed to get GPU statistics (%s)",
+					        __func__, PVRSRVGetErrorString(eError)));
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+					}
+#endif	/* defined(EMULATOR) || defined(VIRTUAL_PLATFORM) */
+				}
+			}
+		}
+	}
+#endif
+
+	return psDeviceNode;
+}
+
+static int _DebugStatusDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	if (pvData == DI_START_TOKEN)
+	{
+		PVRSRV_DATA *psPVRSRVData = DIGetPrivData(psEntry);
+
+		if (psPVRSRVData != NULL)
+		{
+			switch (psPVRSRVData->eServicesState)
+			{
+				case PVRSRV_SERVICES_STATE_OK:
+					DIPrintf(psEntry, "Driver Status:   OK\n");
+					break;
+				case PVRSRV_SERVICES_STATE_BAD:
+					DIPrintf(psEntry, "Driver Status:   BAD\n");
+					break;
+				case PVRSRV_SERVICES_STATE_UNDEFINED:
+					DIPrintf(psEntry, "Driver Status:   UNDEFINED\n");
+					break;
+				default:
+					DIPrintf(psEntry, "Driver Status:   UNKNOWN (%d)\n",
+					         psPVRSRVData->eServicesState);
+					break;
+			}
+		}
+	}
+	else if (pvData != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+		IMG_CHAR           *pszStatus = "";
+		IMG_CHAR           *pszReason = "";
+		PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus;
+		PVRSRV_DEVICE_HEALTH_REASON eHealthReason;
+
+		DIPrintf(psEntry, "\nDevice ID: %u:%d\n", psDeviceNode->sDevId.ui32InternalID,
+		                                          psDeviceNode->sDevId.i32KernelDeviceID);
+
+		/* Update the health status now if possible... */
+		if (psDeviceNode->pfnUpdateHealthStatus)
+		{
+			psDeviceNode->pfnUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+		}
+		eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+		eHealthReason = OSAtomicRead(&psDeviceNode->eHealthReason);
+
+		switch (eHealthStatus)
+		{
+			case PVRSRV_DEVICE_HEALTH_STATUS_OK:  pszStatus = "OK";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:  pszStatus = "NOT RESPONDING";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:  pszStatus = "DEAD";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:  pszStatus = "FAULT";  break;
+			case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:  pszStatus = "UNDEFINED";  break;
+			default:  pszStatus = "UNKNOWN";  break;
+		}
+
+		switch (eHealthReason)
+		{
+			case PVRSRV_DEVICE_HEALTH_REASON_NONE:  pszReason = "";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_ASSERTED:  pszReason = " (Asserted)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING:  pszReason = " (Poll failing)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS:  pszReason = " (Global Event Object timeouts rising)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT:  pszReason = " (KCCB offset invalid)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED:  pszReason = " (KCCB stalled)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_IDLING:  pszReason = " (Idling)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_RESTARTING:  pszReason = " (Restarting)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS:  pszReason = " (Missing interrupts)";  break;
+			case PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR:  pszReason = " (PCI error)";  break;
+			default:  pszReason = " (Unknown reason)";  break;
+		}
+
+		DIPrintf(psEntry, "Firmware Status: %s%s\n", pszStatus, pszReason);
+		if (PVRSRV_ERROR_LIMIT_REACHED)
+		{
+			DIPrintf(psEntry, "Server Errors:   %d+\n", IMG_UINT32_MAX);
+		}
+		else
+		{
+			DIPrintf(psEntry, "Server Errors:   %d\n", PVRSRV_KM_ERRORS);
+		}
+
+
+		/* Write other useful stats to aid the test cycle... */
+		if (psDeviceNode->pvDevice != NULL)
+		{
+#ifdef SUPPORT_RGX
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+			const RGXFWIF_HWRINFOBUF *psHWRInfoBuf;
+			const RGXFWIF_SYSDATA *psFwSysData;
+
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+			psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+			psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+#ifdef PVRSRV_DEBUG_LISR_EXECUTION
+			/* Show the detected #LISR, #MISR scheduled calls */
+			DIPrintf(psEntry, "RGX #LISR: %" IMG_UINT64_FMTSPEC "\n", psDeviceNode->ui64nLISR);
+			DIPrintf(psEntry, "RGX #MISR: %" IMG_UINT64_FMTSPEC "\n", psDeviceNode->ui64nMISR);
+#endif /* PVRSRV_DEBUG_LISR_EXECUTION */
+
+			/* Calculate the number of HWR events in total across all the DMs... */
+			if (psHWRInfoBuf != NULL)
+			{
+				IMG_UINT32 ui32HWREventCount = 0;
+				IMG_UINT32 ui32CRREventCount = 0;
+				IMG_UINT32 ui32DMIndex;
+
+				for (ui32DMIndex = 0; ui32DMIndex < RGXFWIF_DM_MAX; ui32DMIndex++)
+				{
+					ui32HWREventCount += psHWRInfoBuf->aui32HwrDmLockedUpCount[ui32DMIndex];
+					ui32CRREventCount += psHWRInfoBuf->aui32HwrDmOverranCount[ui32DMIndex];
+				}
+
+				DIPrintf(psEntry, "HWR Event Count: %d\n", ui32HWREventCount);
+				DIPrintf(psEntry, "CRR Event Count: %d\n", ui32CRREventCount);
+#ifdef PVRSRV_STALLED_CCB_ACTION
+				/* Write the number of Sync Lockup Recovery (SLR) events... */
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested,
+				                           INVALIDATE);
+				DIPrintf(psEntry, "SLR Event Count: %d\n", psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested);
+#endif /* PVRSRV_STALLED_CCB_ACTION */
+			}
+
+			/* Show error counts */
+			DIPrintf(psEntry, "WGP Error Count: %d\n", psDevInfo->sErrorCounts.ui32WGPErrorCount);
+			DIPrintf(psEntry, "TRP Error Count: %d\n", psDevInfo->sErrorCounts.ui32TRPErrorCount);
+
+			/*
+			 * Guest drivers do not support the following functionality:
+			 *	- Perform actual on-chip fw tracing.
+			 *	- Collect actual on-chip GPU utilization stats.
+			 *	- Perform actual on-chip GPU power/dvfs management.
+			 *	- As a result no more information can be provided.
+			 */
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				if (psFwSysData != NULL)
+				{
+					DIPrintf(psEntry, "FWF Event Count: %d\n", psFwSysData->ui32FWFaults);
+				}
+
+				/* Write the number of APM events... */
+				DIPrintf(psEntry, "APM Event Count: %d\n", psDevInfo->ui32ActivePMReqTotal);
+
+				/* Write the current GPU Utilisation values... */
+				if (psDevInfo->pfnGetGpuUtilStats &&
+					eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+				{
+					PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+					RGXFWIF_GPU_UTIL_STATS *psGpuUtilStats = &psDevInfo->sGpuUtilStats;
+
+					OSLockAcquire(psDevInfo->hGpuUtilStatsLock);
+
+					if ((IMG_UINT32)psGpuUtilStats->ui64GpuStatCumulative)
+					{
+						const IMG_CHAR *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+						IMG_UINT64 util;
+						IMG_UINT32 rem;
+						IMG_UINT32 ui32DriverID;
+						RGXFWIF_DM eDM;
+						IMG_INT    iDM_Util = 0;
+
+						if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+						{
+							apszDmNames[RGXFWIF_DM_TDM] = "2D";
+						}
+
+						util = 100 * psGpuUtilStats->ui64GpuStatActive;
+						util = OSDivide64(util, (IMG_UINT32)psGpuUtilStats->ui64GpuStatCumulative, &rem);
+
+						DIPrintf(psEntry, "GPU Utilisation: %u%%\n", (IMG_UINT32)util);
+
+						DIPrintf(psEntry, "DM Utilisation:");
+
+						FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+						{
+							DIPrintf(psEntry, "  VM%u", ui32DriverID);
+						}
+
+						DIPrintf(psEntry, "\n");
+
+						for (eDM = RGXFWIF_DM_TDM; eDM < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; eDM++,iDM_Util++)
+						{
+							DIPrintf(psEntry, "        %5s: ", apszDmNames[eDM]);
+
+							FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+							{
+								IMG_UINT32 uiDivisor = (IMG_UINT32)psGpuUtilStats->aaui64DMOSStatCumulative[iDM_Util][ui32DriverID];
+
+								if (uiDivisor == 0U)
+								{
+									DIPrintf(psEntry, "   - ");
+									continue;
+								}
+
+								util = 100 * psGpuUtilStats->aaui64DMOSStatActive[iDM_Util][ui32DriverID];
+								util = OSDivide64(util, uiDivisor, &rem);
+
+								DIPrintf(psEntry, "%3u%% ", (IMG_UINT32)util);
+							}
+
+
+							DIPrintf(psEntry, "\n");
+						}
+					}
+					else
+					{
+						DIPrintf(psEntry, "GPU Utilisation: -\n");
+					}
+
+					OSLockRelease(psDevInfo->hGpuUtilStatsLock);
+
+				}
+			}
+#endif /* SUPPORT_RGX */
+		}
+	}
+
+	return 0;
+}
+
+#if defined(DEBUG)
+static IMG_INT64 DebugStatusSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[0] == 'k' || pcBuffer[0] == 'K', -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	psPVRSRVData->eServicesState = PVRSRV_SERVICES_STATE_BAD;
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+#endif
+
+const IMG_CHAR *PVRSRVGetDebugDevStateString(PVRSRV_DEVICE_STATE eDevState)
+{
+	static const char *const _pszDeviceStateStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_STATE_LIST
+	#undef X
+	};
+
+	if (eDevState < 0 || eDevState >= PVRSRV_DEVICE_STATE_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceStateStrings[eDevState];
+}
+
+
+const IMG_CHAR *PVRSRVGetDebugHealthStatusString(PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus)
+{
+	static const char *const _pszDeviceHealthStatusStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_HEALTH_STATUS_LIST
+	#undef X
+	};
+
+	if (eHealthStatus < 0 || eHealthStatus >= PVRSRV_DEVICE_HEALTH_STATUS_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceHealthStatusStrings[eHealthStatus];
+}
+
+
+const IMG_CHAR *PVRSRVGetDebugHealthReasonString(PVRSRV_DEVICE_HEALTH_REASON eHealthReason)
+{
+	static const char *const _pszDeviceHealthReasonStrings[] = {
+	#define X(_name) #_name,
+		PVRSRV_DEVICE_HEALTH_REASON_LIST
+	#undef X
+	};
+
+	if (eHealthReason < 0 || eHealthReason >= PVRSRV_DEVICE_HEALTH_REASON_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszDeviceHealthReasonStrings[eHealthReason];
+}
+
+/*************************************************************************/ /*!
+ Dump Debug DebugFS entry
+*/ /**************************************************************************/
+
+static int _DebugDumpDebugDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDeviceNode->pvDevice != NULL)
+	{
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX,
+		                   _DumpDebugDIPrintfWrapper, psEntry);
+	}
+
+	return 0;
+}
+
+#ifdef SUPPORT_RGX
+
+/*************************************************************************/ /*!
+ Firmware Trace DebugFS entry
+*/ /**************************************************************************/
+
+static int _DebugFWTraceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDevInfo != NULL)
+	{
+		RGXDumpFirmwareTrace(_DumpDebugDIPrintfWrapper, psEntry, psDevInfo);
+	}
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+ Firmware Translated Page Tables DebugFS entry
+*/ /**************************************************************************/
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+static int _FirmwareMappingsDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32FwVA;
+	IMG_UINT32 ui32FwPageSize;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	psDeviceNode = DIGetPrivData(psEntry);
+
+	if ((psDeviceNode == NULL) ||
+	    (psDeviceNode->pvDevice == NULL) ||
+	    (((PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice)->psKernelMMUCtx == NULL))
+	{
+		/* The Kernel MMU context containing the Firmware mappings is not initialised */
+		return 0;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	DIPrintf(psEntry, "+-----------------+------------------------+------------------------+--------------+\n"
+					  "|    Firmware     |           CPU          |         Device         |      PTE     |\n"
+					  "| Virtual Address |    Physical Address    |    Physical Address    |     Flags    |\n"
+					  "+-----------------+------------------------+------------------------+              +\n");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		DIPrintf(psEntry, "|                                               RI/XI = Read / Execution Inhibit   |\n"
+		                  "|                                               C     = Cache Coherent             |\n"
+		                  "|                                               D     = Dirty Page Table Entry     |\n"
+		                  "|                                               V     = Valid Page Table Entry     |\n"
+		                  "|                                               G     = Global Page Table Entry    |\n"
+		                  "+-----------------+------------------------+------------------------+--------------+\n");
+
+		/* MIPS uses the same page size as the OS */
+		ui32FwPageSize = OSGetPageSize();
+	}
+	else
+#endif
+	{
+		DIPrintf(psEntry, "|                                               P     = Pending Page Table Entry   |\n"
+		                  "|                                               PM    = Parameter Manager Source   |\n"
+		                  "|                                               B     = Bypass SLC                 |\n"
+		                  "|                                               C     = Cache Coherent             |\n"
+		                  "|                                               RW/RO = Device Access Rights       |\n"
+		                  "|                                               V     = Valid Page Table Entry     |\n"
+		                  "+-----------------+------------------------+------------------------+--------------+\n");
+
+		ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+	}
+
+	FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+	{
+		IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) ((RGX_FIRMWARE_RAW_HEAP_BASE +
+		                             (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE)) & UINT_MAX);
+		IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_SIZE & UINT_MAX);
+
+		DIPrintf(psEntry, "|                                       OS ID %u                                    |\n"
+						  "+-----------------+------------------------+------------------------+--------------+\n", ui32DriverID);
+
+		for (ui32FwVA = ui32FwHeapBase;
+		     ui32FwVA < ui32FwHeapEnd;
+		     ui32FwVA += ui32FwPageSize)
+		{
+			PVRSRV_ERROR eError;
+			IMG_UINT64 ui64PTE = 0U;
+			IMG_CPU_PHYADDR sCpuPA = {0U};
+			IMG_DEV_PHYADDR sDevPA = {0U};
+
+			eError = RGXGetFwMapping(psDevInfo, ui32FwVA, &sCpuPA, &sDevPA, &ui64PTE);
+
+			if (eError == PVRSRV_OK)
+			{
+				RGXDocumentFwMapping(psDevInfo, _DumpDebugDIPrintfWrapper, psEntry,
+				                     ui32FwVA, sCpuPA, sDevPA, ui64PTE);
+			}
+			else if (eError != PVRSRV_ERROR_DEVICEMEM_NO_MAPPING)
+			{
+				PVR_LOG_ERROR(eError, "RGXGetFwMapping");
+				return -EIO;
+			}
+		}
+
+		DIPrintf(psEntry, "+-----------------+------------------------+------------------------+--------------+\n");
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+		{
+			break;
+		}
+#endif
+	}
+
+	return 0;
+}
+#endif
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+
+static void *_FirmwareGcovDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo != NULL)
+	{
+		if (psDevInfo->psFirmwareGcovBufferMemDesc != NULL)
+		{
+			void *pvCpuVirtAddr;
+			DevmemAcquireCpuVirtAddr(psDevInfo->psFirmwareGcovBufferMemDesc, &pvCpuVirtAddr);
+			return *pui64Pos ? NULL : pvCpuVirtAddr;
+		}
+	}
+
+	return NULL;
+}
+
+static void _FirmwareGcovDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psDevInfo != NULL)
+	{
+		if (psDevInfo->psFirmwareGcovBufferMemDesc != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psFirmwareGcovBufferMemDesc);
+		}
+	}
+}
+
+static void *_FirmwareGcovDINext(OSDI_IMPL_ENTRY *psEntry,
+								  void *pvData,
+								  IMG_UINT64 *pui64Pos)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+	PVR_UNREFERENCED_PARAMETER(pui64Pos);
+	return NULL;
+}
+
+static int _FirmwareGcovDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo != NULL)
+	{
+		DIWrite(psEntry, pvData, psDevInfo->ui32FirmwareGcovSize);
+	}
+	return 0;
+}
+
+#endif /* SUPPORT_FIRMWARE_GCOV */
+
+
+#if  defined(SUPPORT_RISCV_GDB)
+#define RISCV_DMI_SIZE  (8U)
+
+static IMG_INT64 _RiscvDmiRead(IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+
+	ui64Count = MIN(RISCV_DMI_SIZE, ui64Count);
+	memcpy(pcBuffer, &psDebugInfo->ui64RiscvDmi, ui64Count);
+
+	return ui64Count;
+}
+
+static IMG_INT64 _RiscvDmiWrite(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: devinfo is NULL", __func__));
+		return 0;
+	}
+
+	ui64Count -= 1; /* Drop `\0` */
+	ui64Count = MIN(RISCV_DMI_SIZE, ui64Count);
+
+	memcpy(&psDebugInfo->ui64RiscvDmi, pcBuffer, ui64Count);
+
+	RGXRiscvDmiOp(psDevInfo, &psDebugInfo->ui64RiscvDmi);
+
+	return ui64Count;
+}
+#endif
+
+#endif /* SUPPORT_RGX */
+
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+
+/*************************************************************************/ /*!
+ Debug level DebugFS entry
+*/ /**************************************************************************/
+
+static int DebugLevelDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DIPrintf(psEntry, "%u\n", OSDebugLevel());
+
+	return 0;
+}
+
+#ifndef __GNUC__
+static int __builtin_ffsl(long int x)
+{
+	for (size_t i = 0; i < sizeof(x) * 8; i++)
+	{
+		if (x & (1 << i))
+		{
+			return i + 1;
+		}
+	}
+	return 0;
+}
+#endif /* __GNUC__ */
+
+static IMG_INT64 DebugLevelSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const IMG_UINT uiMaxBufferSize = 6;
+	IMG_UINT32 ui32Level;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	if (sscanf(pcBuffer, "%u", &ui32Level) == 0)
+	{
+		return -EINVAL;
+	}
+
+	OSSetDebugLevel(ui32Level & ((1 << __builtin_ffsl(DBGPRIV_LAST)) - 1));
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+#if defined(SUPPORT_RGX) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static int VZPriorityDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+	                    -EINVAL);
+
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ai32DriverPriority[ui32DriverID], INVALIDATE);
+	DIPrintf(psEntry, "%u\n", psRuntimeCfg->ai32DriverPriority[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZPrioritySet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                               IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32Priority;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32Priority) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverPriorityKM(NULL, psVZDriverData->psDevNode,
+	                                             psVZDriverData->ui32DriverID, ui32Priority);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZTimeSliceIntervalDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	DIPrintf(psEntry, "%u ms (0: disable)\n", psRuntimeCfg->ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT64 VZTimeSliceIntervalSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+										IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32TSIntervalMs;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32TSIntervalMs) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(NULL, psVZDriverData->psDevNode,
+														  ui32TSIntervalMs);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZTimeSliceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+						-EINVAL);
+
+	DIPrintf(psEntry, "%u (0: auto; 1%% to 100%%)\n", psRuntimeCfg->aui32TSPercentage[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZTimeSliceSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+								IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32TSPercentage;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32TSPercentage) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverTimeSliceKM(NULL, psVZDriverData->psDevNode,
+												  psVZDriverData->ui32DriverID, ui32TSPercentage);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZIsolationGroupDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_UINT32 ui32DriverID;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	ui32DriverID = psVZDriverData->ui32DriverID;
+	PVR_RETURN_IF_FALSE(ui32DriverID < (RGXFW_HOST_DRIVER_ID + RGX_NUM_DRIVERS_SUPPORTED),
+	                    -EINVAL);
+
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID], INVALIDATE);
+	DIPrintf(psEntry, "%u\n", psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID]);
+
+	return 0;
+}
+
+static IMG_INT64 VZIsolationGroupSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+                                     IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32IsolationGroup;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32IsolationGroup) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetDriverIsolationGroupKM(NULL, psVZDriverData->psDevNode,
+	                                                   psVZDriverData->ui32DriverID, ui32IsolationGroup);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+static int VZConnectionCooldownPeriodDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	DI_VZ_DATA *psVZDriverData = DIGetPrivData(psEntry);
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	psDevInfo = psVZDriverData->psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, -EIO);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, -EIO);
+
+	DIPrintf(psEntry, "%u sec\n", psRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+	return 0;
+}
+
+static IMG_INT64 VZConnectionCooldownPeriodSet(const IMG_CHAR *pcBuffer, IMG_UINT64 ui64Count,
+										IMG_UINT64 *pui64Pos, void *pvData)
+{
+	const DI_VZ_DATA *psVZDriverData = (const DI_VZ_DATA*)pvData;
+	const IMG_UINT32 uiMaxBufferSize = 12;
+	IMG_UINT32 ui32VzConnectionCooldownPeriodInSec;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count > 0 && ui64Count < uiMaxBufferSize, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData != NULL, -EINVAL);
+	PVR_RETURN_IF_FALSE(psVZDriverData->psDevNode != NULL, -ENXIO);
+
+	if (OSStringToUINT32(pcBuffer, 10, &ui32VzConnectionCooldownPeriodInSec) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	eError = PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(NULL, psVZDriverData->psDevNode,
+														  ui32VzConnectionCooldownPeriodInSec);
+	if (eError != PVRSRV_OK)
+	{
+		return -EIO;
+	}
+
+	*pui64Pos += ui64Count;
+	return ui64Count;
+}
+
+#endif
+
+PVRSRV_ERROR DebugCommonInitDriver(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPVRSRVData != NULL);
+
+	/*
+	 * The DebugFS entries are designed to work in a single device system but
+	 * this function will be called multiple times in a multi-device system.
+	 * Return an error in this case.
+	 */
+	if (gpsVersionDIEntry)
+	{
+		return -EEXIST;
+	}
+
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnStart = _VersionDIStart,
+			.pfnStop = _VersionDIStop,
+			.pfnNext = _VersionDINext,
+			.pfnShow = _VersionDIShow
+		};
+
+		eError = DICreateEntry("version", NULL, &sIterator, psPVRSRVData,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsVersionDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnStart = _DebugStatusDIStart,
+			.pfnStop = _DebugStatusDIStop,
+			.pfnNext = _DebugStatusDINext,
+			.pfnShow = _DebugStatusDIShow,
+#if defined(DEBUG)
+			.pfnWrite = DebugStatusSet,
+			//'K' expected + Null terminator
+#endif
+			.ui32WriteLenMax= ((1U)+1U)
+		};
+		eError = DICreateEntry("status", NULL, &sIterator, psPVRSRVData,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsStatusDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = DebugLevelDIShow,
+			.pfnWrite = DebugLevelSet,
+			//Max value of 255(3 char) + Null terminator
+			.ui32WriteLenMax =((3U)+1U)
+		};
+		eError = DICreateEntry("debug_level", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &gpsDebugLevelDIEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+	return PVRSRV_OK;
+
+return_error_:
+	DebugCommonDeInitDriver();
+
+	return eError;
+}
+
+void DebugCommonDeInitDriver(void)
+{
+#if defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON)
+	if (gpsDebugLevelDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsDebugLevelDIEntry);
+	}
+#endif /* defined(DEBUG) || defined(PVR_DPF_ADHOC_DEBUG_ON) */
+
+
+	if (gpsStatusDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsStatusDIEntry);
+	}
+
+	if (gpsVersionDIEntry != NULL)
+	{
+		DIDestroyEntry(gpsVersionDIEntry);
+	}
+}
+
+PVRSRV_ERROR DebugCommonInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszDeviceId[sizeof("gpu4294967296")];
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSSNPrintf(pszDeviceId, sizeof(pszDeviceId), "gpu%02d",
+	           psDeviceNode->sDevId.ui32InternalID);
+	eError = DICreateGroup(pszDeviceId, NULL, &psDebugInfo->psGroup);
+	PVR_GOTO_IF_ERROR(eError, return_error_);
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	eError = SORgxGpuUtilStatsRegister(&psDebugInfo->hGpuUtilUserDebugFS);
+	PVR_GOTO_IF_ERROR(eError, return_error_);
+#endif
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = _DebugDumpDebugDIShow};
+		eError = DICreateEntry("debug_dump", psDebugInfo->psGroup, &sIterator,
+		                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+		                       &psDebugInfo->psDumpDebugEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+
+#ifdef SUPPORT_RGX
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		{
+			DI_ITERATOR_CB sIterator = {.pfnShow = _DebugFWTraceDIShow};
+			eError = DICreateEntry("firmware_trace", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWTraceEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+		{
+			DI_ITERATOR_CB sIterator = {
+				.pfnStart = _FirmwareGcovDIStart,
+				.pfnStop = _FirmwareGcovDIStop,
+				.pfnNext = _FirmwareGcovDINext,
+				.pfnShow = _FirmwareGcovDIShow
+			};
+
+			eError = DICreateEntry("firmware_gcov", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWGCOVEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+#endif /* SUPPORT_FIRMWARE_GCOV */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+		{
+			DI_ITERATOR_CB sIterator = {.pfnShow = _FirmwareMappingsDIShow};
+			eError = DICreateEntry("firmware_mappings", psDebugInfo->psGroup, &sIterator,
+			                       psDeviceNode, DI_ENTRY_TYPE_GENERIC,
+			                       &psDebugInfo->psFWMappingsEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+		}
+#endif
+
+#if  defined(SUPPORT_RISCV_GDB)
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			DI_ITERATOR_CB sIterator = {
+				.pfnRead = _RiscvDmiRead,
+				.pfnWrite = _RiscvDmiWrite,
+				.ui32WriteLenMax = ((RISCV_DMI_SIZE)+1U)
+			};
+			eError = DICreateEntry("riscv_dmi", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+			                       DI_ENTRY_TYPE_RANDOM_ACCESS, &psDebugInfo->psRiscvDmiDIEntry);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+			psDebugInfo->ui64RiscvDmi = 0ULL;
+		}
+#endif /* SUPPORT_VALIDATION || SUPPORT_RISCV_GDB */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+		{
+			eError = DICreateGroup("vz", psDebugInfo->psGroup, &psDebugInfo->psVZGroup);
+			PVR_GOTO_IF_ERROR(eError, return_error_);
+
+			{
+				IMG_UINT32 ui32DriverID;
+
+				DI_ITERATOR_CB sPriorityIterator = {
+					.pfnShow = VZPriorityDIShow,
+					.pfnWrite = VZPrioritySet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sTimeSliceIntervalIterator = {
+					.pfnShow = VZTimeSliceIntervalDIShow,
+					.pfnWrite = VZTimeSliceIntervalSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sTimeSliceIterator = {
+					.pfnShow = VZTimeSliceDIShow,
+					.pfnWrite = VZTimeSliceSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sIsolationGroupIterator = {
+					.pfnShow = VZIsolationGroupDIShow,
+					.pfnWrite = VZIsolationGroupSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				DI_ITERATOR_CB sVzConnectionCooldownPeriodIterator = {
+					.pfnShow = VZConnectionCooldownPeriodDIShow,
+					.pfnWrite = VZConnectionCooldownPeriodSet,
+					//Max value of UINT_MAX (10 chars) + Null terminator
+					.ui32WriteLenMax = sizeof("4294967295")
+				};
+
+				FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+				{
+					IMG_CHAR szDriverID[4];
+					OSSNPrintf(szDriverID, 4, "%u", ui32DriverID);
+
+					eError = DICreateGroup(szDriverID, psDebugInfo->psVZGroup, &psDebugInfo->apsVZDriverGroups[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					psDebugInfo->apsVZDriverData[ui32DriverID] = OSAllocMem(sizeof(PVRSRV_DEVICE_DEBUG_INFO));
+					PVR_GOTO_IF_NOMEM(psDebugInfo->apsVZDriverData[ui32DriverID], eError, return_error_);
+
+					psDebugInfo->apsVZDriverData[ui32DriverID]->psDevNode = psDeviceNode;
+					psDebugInfo->apsVZDriverData[ui32DriverID]->ui32DriverID = ui32DriverID;
+
+					eError = DICreateEntry("priority", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sPriorityIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					eError = DICreateEntry("time_slice", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sTimeSliceIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+
+					if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+					{
+						eError = DICreateEntry("time_slice_interval", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+											   &sTimeSliceIntervalIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+											   &psDebugInfo->psVZDriverTimeSliceIntervalDIEntry);
+						PVR_GOTO_IF_ERROR(eError, return_error_);
+
+						eError = DICreateEntry("vz_connection_cooldown_period", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+											   &sVzConnectionCooldownPeriodIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+											   &psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry);
+						PVR_GOTO_IF_ERROR(eError, return_error_);
+					}
+
+					eError = DICreateEntry("isolation_group", psDebugInfo->apsVZDriverGroups[ui32DriverID],
+					                       &sIsolationGroupIterator, psDebugInfo->apsVZDriverData[ui32DriverID], DI_ENTRY_TYPE_GENERIC,
+					                       &psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID]);
+					PVR_GOTO_IF_ERROR(eError, return_error_);
+				}
+			}
+		}
+#endif /* defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1 */
+	}
+
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = _DebugPowerDataDIShow,
+			.pfnWrite = PowerDataSet,
+			//Expects '0' or '1' plus Null terminator
+			.ui32WriteLenMax = ((1U)+1U)
+		};
+		eError = DICreateEntry("power_data", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+		                       DI_ENTRY_TYPE_GENERIC, &psDebugInfo->psPowerDataEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif /* SUPPORT_POWER_SAMPLING_VIA_DEBUGFS */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	{
+		DI_ITERATOR_CB sIterator = {
+			.pfnShow = PVRSRVPowerStatsPrintElements,
+		};
+		eError = DICreateEntry("power_timing_stats", psDebugInfo->psGroup, &sIterator, psDeviceNode,
+		                       DI_ENTRY_TYPE_GENERIC, &psDebugInfo->psPowerTimingStatsEntry);
+		PVR_GOTO_IF_ERROR(eError, return_error_);
+	}
+#endif
+#endif /* SUPPORT_RGX */
+
+	return PVRSRV_OK;
+
+return_error_:
+	DebugCommonDeInitDevice(psDeviceNode);
+
+	return eError;
+}
+
+void DebugCommonDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DEVICE_DEBUG_INFO *psDebugInfo = &psDeviceNode->sDebugInfo;
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	if (psDebugInfo->psPowerTimingStatsEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psPowerTimingStatsEntry);
+		psDebugInfo->psPowerTimingStatsEntry = NULL;
+	}
+#endif
+
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	if (psDebugInfo->psPowerDataEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psPowerDataEntry);
+		psDebugInfo->psPowerDataEntry = NULL;
+	}
+#endif /* SUPPORT_POWER_SAMPLING_VIA_DEBUGFS */
+
+
+#ifdef SUPPORT_RGX
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			if (psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverIsolationGroupDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverPriorityDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID] != NULL)
+			{
+				DIDestroyEntry(psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID]);
+				psDebugInfo->apsVZDriverTimeSliceDIEntries[ui32DriverID] = NULL;
+			}
+
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				if (psDebugInfo->psVZDriverTimeSliceIntervalDIEntry != NULL)
+				{
+					DIDestroyEntry(psDebugInfo->psVZDriverTimeSliceIntervalDIEntry);
+					psDebugInfo->psVZDriverTimeSliceIntervalDIEntry = NULL;
+				}
+
+				if (psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry != NULL)
+				{
+					DIDestroyEntry(psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry);
+					psDebugInfo->psVZDriverConnectionCooldownPeriodDIEntry = NULL;
+				}
+			}
+
+			if (psDebugInfo->apsVZDriverData[ui32DriverID] != NULL)
+			{
+				OSFreeMem(psDebugInfo->apsVZDriverData[ui32DriverID]);
+				psDebugInfo->apsVZDriverData[ui32DriverID] = NULL;
+			}
+
+			if (psDebugInfo->apsVZDriverGroups[ui32DriverID] != NULL)
+			{
+				DIDestroyGroup(psDebugInfo->apsVZDriverGroups[ui32DriverID]);
+				psDebugInfo->apsVZDriverGroups[ui32DriverID] = NULL;
+			}
+		}
+
+		if (psDebugInfo->psVZGroup != NULL)
+		{
+			DIDestroyGroup(psDebugInfo->psVZGroup);
+			psDebugInfo->psVZGroup = NULL;
+		}
+	}
+#endif
+
+	if (psDebugInfo->psFWTraceEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWTraceEntry);
+		psDebugInfo->psFWTraceEntry = NULL;
+	}
+
+#ifdef SUPPORT_FIRMWARE_GCOV
+	if (psDebugInfo->psFWGCOVEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWGCOVEntry);
+		psDebugInfo->psFWGCOVEntry = NULL;
+	}
+#endif
+
+	if (psDebugInfo->psFWMappingsEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psFWMappingsEntry);
+		psDebugInfo->psFWMappingsEntry = NULL;
+	}
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR) &&
+		(psDebugInfo->psRiscvDmiDIEntry != NULL))
+	{
+		DIDestroyEntry(psDebugInfo->psRiscvDmiDIEntry);
+		psDebugInfo->psRiscvDmiDIEntry = NULL;
+	}
+#endif
+#endif /* SUPPORT_RGX */
+
+	if (psDebugInfo->psDumpDebugEntry != NULL)
+	{
+		DIDestroyEntry(psDebugInfo->psDumpDebugEntry);
+		psDebugInfo->psDumpDebugEntry = NULL;
+	}
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	if (psDebugInfo->hGpuUtilUserDebugFS != NULL)
+	{
+		SORgxGpuUtilStatsUnregister(psDebugInfo->hGpuUtilUserDebugFS);
+		psDebugInfo->hGpuUtilUserDebugFS = NULL;
+	}
+#endif /* defined(SUPPORT_RGX) && !defined(NO_HARDWARE) */
+
+	if (psDebugInfo->psGroup != NULL)
+	{
+		DIDestroyGroup(psDebugInfo->psGroup);
+		psDebugInfo->psGroup = NULL;
+	}
+}
+
+/*
+	Appends flags strings to a null-terminated string buffer
+*/
+void DebugCommonFlagStrings(IMG_CHAR *psDesc,
+							IMG_UINT32 ui32DescSize,
+							const IMG_FLAGS2DESC *psConvTable,
+							IMG_UINT32 ui32TableSize,
+							IMG_UINT32 ui32Flags)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (BITMASK_HAS(ui32Flags, psConvTable[ui32Idx].uiFlag))
+		{
+			OSStringLCat(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/debug_common.h b/drivers/gpu/drm/img-rogue/debug_common.h
new file mode 100644
index 000000000000..aba576d273a1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/debug_common.h
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common debug definitions and functions.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEBUG_COMMON_H
+#define DEBUG_COMMON_H
+
+#include "pvrsrv_error.h"
+#include "device.h"
+
+PVRSRV_ERROR DebugCommonInitDriver(void);
+void DebugCommonDeInitDriver(void);
+
+const IMG_CHAR *PVRSRVGetDebugDevStateString(PVRSRV_DEVICE_STATE eDevState);
+const IMG_CHAR *PVRSRVGetDebugHealthStatusString(PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus);
+const IMG_CHAR *PVRSRVGetDebugHealthReasonString(PVRSRV_DEVICE_HEALTH_REASON eHealthReason);
+
+PVRSRV_ERROR DebugCommonInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+void DebugCommonDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+typedef struct _IMG_FLAGS2DESC_
+{
+	IMG_UINT32		uiFlag;
+	const IMG_CHAR	*pszLabel;
+} IMG_FLAGS2DESC;
+
+void DebugCommonFlagStrings(IMG_CHAR *psDesc,
+							IMG_UINT32 ui32DescSize,
+							const IMG_FLAGS2DESC *psConvTable,
+							IMG_UINT32 ui32TableSize,
+							IMG_UINT32 ui32Flags);
+
+#endif /* DEBUG_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/device.h b/drivers/gpu/drm/img-rogue/device.h
new file mode 100644
index 000000000000..4a6975b023cb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/device.h
@@ -0,0 +1,673 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common Device header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device related function templates and defines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DEVICE_H
+#define DEVICE_H
+
+#include "devicemem_heapcfg.h"
+#include "mmu_common.h"
+#include "ra.h"			/* RA_ARENA */
+#include "pvrsrv_device.h"
+#include "sync_checkpoint.h"
+#include "srvkm.h"
+#include "physheap.h"
+#include "sync_internal.h"
+#include "dllist.h"
+
+#include "rgx_bvnc_defs_km.h"
+
+#include "lock.h"
+
+#include "power.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+typedef struct _PVRSRV_POWER_DEV_TAG_ *PPVRSRV_POWER_DEV;
+
+struct SYNC_RECORD;
+
+struct _CONNECTION_DATA_;
+
+/*************************************************************************/ /*!
+ @Function      AllocUFOBlockCallback
+ @Description   Device specific callback for allocation of a UFO block
+
+ @Input         psDeviceNode          Pointer to device node to allocate
+                                      the UFO for.
+ @Input         ui32RequestedSize     Minimum size of allocation requested
+ @Output        ppsMemDesc            Pointer to pointer for the memdesc of
+                                      the allocation
+ @Output        pui32SyncAddr         FW Base address of the UFO block
+ @Output        puiSyncPrimBlockSize  Size of the UFO block
+
+ @Return        PVRSRV_OK if allocation was successful
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*AllocUFOBlockCallback)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+											  IMG_UINT32 ui32RequestedSize,
+											  DEVMEM_MEMDESC **ppsMemDesc,
+											  IMG_UINT32 *pui32SyncAddr,
+											  IMG_UINT32 *puiSyncPrimBlockSize);
+
+/*************************************************************************/ /*!
+ @Function      FreeUFOBlockCallback
+ @Description   Device specific callback for freeing of a UFO
+
+ @Input         psDeviceNode    Pointer to device node that the UFO block was
+                                allocated from.
+ @Input         psMemDesc       Pointer to pointer for the memdesc of the UFO
+                                block to free.
+*/ /**************************************************************************/
+typedef void (*FreeUFOBlockCallback)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+									 DEVMEM_MEMDESC *psMemDesc);
+
+typedef struct _PVRSRV_DEVICE_IDENTIFIER_
+{
+	/* Pdump memory and register bank names */
+	IMG_CHAR				*pszPDumpDevName;
+	IMG_CHAR				*pszPDumpRegName;
+
+	/* Under Linux, this is the minor number of RenderNode corresponding to this Device */
+	IMG_INT32				i32KernelDeviceID;
+	/* Services layer enumeration of the device used in pvrdebug */
+	IMG_UINT32				ui32InternalID;
+} PVRSRV_DEVICE_IDENTIFIER;
+
+typedef struct _DEVICE_MEMORY_INFO_
+{
+	/* Heap count. Doesn't include additional heaps from PVRSRVCreateDeviceMemHeap */
+	IMG_UINT32				ui32HeapCount;
+
+	/* Blueprints for creating new device memory contexts */
+	IMG_UINT32              uiNumHeapConfigs;
+	DEVMEM_HEAP_CONFIG      *psDeviceMemoryHeapConfigArray;
+	DEVMEM_HEAP_BLUEPRINT   *psDeviceMemoryHeap;
+} DEVICE_MEMORY_INFO;
+
+#define MMU_BAD_PHYS_ADDR (0xbadbad00badULL)
+
+typedef struct __DEFAULT_PAGE__
+{
+	/*Page handle for the page allocated (UMA/LMA)*/
+	PG_HANDLE	sPageHandle;
+	POS_LOCK	psPgLock;
+	/*Default page size in terms of log2 */
+	IMG_UINT32	ui32Log2PgSize;
+	IMG_UINT64	ui64PgPhysAddr;
+#if defined(PDUMP)
+	IMG_HANDLE hPdumpPg;
+#endif
+} PVRSRV_DEF_PAGE;
+
+#define PVRSRV_DEVICE_STATE_LIST \
+	X(UNDEFINED)                 \
+	X(CREATING)                  \
+	X(CREATED)                   \
+	X(ACTIVE)                    \
+	X(FROZEN)                    \
+	X(DEINIT)                    \
+	X(DESTRUCTING)               \
+	X(BAD)                       \
+	X(PCI_ERROR)                 \
+	X(LAST)                      \
+
+typedef enum _PVRSRV_DEVICE_STATE_
+{
+#define X(_name) PVRSRV_DEVICE_STATE_ ## _name,
+	PVRSRV_DEVICE_STATE_LIST
+#undef X
+
+} PVRSRV_DEVICE_STATE;
+
+#define PVRSRV_DEVICE_HEALTH_STATUS_LIST \
+	X(UNDEFINED)                         \
+	X(OK)                                \
+	X(NOT_RESPONDING)                    \
+	X(DEAD)                              \
+	X(FAULT)                             \
+	X(LAST)                              \
+
+typedef enum _PVRSRV_DEVICE_HEALTH_STATUS_
+{
+#define X(_name) PVRSRV_DEVICE_HEALTH_STATUS_ ## _name,
+	PVRSRV_DEVICE_HEALTH_STATUS_LIST
+#undef X
+
+} PVRSRV_DEVICE_HEALTH_STATUS;
+
+#define PVRSRV_DEVICE_HEALTH_REASON_LIST \
+	X(NONE)                              \
+	X(ASSERTED)                          \
+	X(POLL_FAILING)                      \
+	X(TIMEOUTS)                          \
+	X(QUEUE_CORRUPT)                     \
+	X(QUEUE_STALLED)                     \
+	X(IDLING)                            \
+	X(RESTARTING)                        \
+	X(MISSING_INTERRUPTS)                \
+	X(PCI_ERROR)                         \
+	X(LAST)                              \
+
+typedef enum _PVRSRV_DEVICE_HEALTH_REASON_
+{
+#define X(_name) PVRSRV_DEVICE_HEALTH_REASON_ ## _name,
+	PVRSRV_DEVICE_HEALTH_REASON_LIST
+#undef X
+
+} PVRSRV_DEVICE_HEALTH_REASON;
+
+typedef enum _PVRSRV_DEVICE_DEBUG_DUMP_STATUS_
+{
+	PVRSRV_DEVICE_DEBUG_DUMP_NONE = 0,
+	PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE
+} PVRSRV_DEVICE_DEBUG_DUMP_STATUS;
+
+#ifndef DI_GROUP_DEFINED
+#define DI_GROUP_DEFINED
+typedef struct DI_GROUP DI_GROUP;
+#endif
+#ifndef DI_ENTRY_DEFINED
+#define DI_ENTRY_DEFINED
+typedef struct DI_ENTRY DI_ENTRY;
+#endif
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#ifndef DI_VZ_DATA_DEFINED
+#define DI_VZ_DATA_DEFINED
+typedef struct DI_VZ_DATA DI_VZ_DATA;
+#endif
+#endif
+
+typedef struct _PVRSRV_DEVICE_DEBUG_INFO_
+{
+	DI_GROUP *psGroup;
+	DI_ENTRY *psDumpDebugEntry;
+#ifdef SUPPORT_RGX
+	DI_ENTRY *psFWTraceEntry;
+#ifdef SUPPORT_FIRMWARE_GCOV
+	DI_ENTRY *psFWGCOVEntry;
+#endif
+	DI_ENTRY *psFWMappingsEntry;
+#if  defined(SUPPORT_RISCV_GDB)
+	DI_ENTRY *psRiscvDmiDIEntry;
+	IMG_UINT64 ui64RiscvDmi;
+#endif
+	DI_ENTRY *psDevMemEntry;
+	IMG_HANDLE hGpuUtilUserDebugFS;
+#endif /* SUPPORT_RGX */
+#ifdef SUPPORT_POWER_SAMPLING_VIA_DEBUGFS
+	DI_ENTRY *psPowerDataEntry;
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	DI_ENTRY *psPowerTimingStatsEntry;
+#endif
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	DI_GROUP *psVZGroup;
+	DI_GROUP *apsVZDriverGroups[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *apsVZDriverPriorityDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *apsVZDriverTimeSliceDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *psVZDriverTimeSliceIntervalDIEntry;
+	DI_ENTRY *apsVZDriverIsolationGroupDIEntries[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_VZ_DATA *apsVZDriverData[RGX_NUM_DRIVERS_SUPPORTED];
+	DI_ENTRY *psVZDriverConnectionCooldownPeriodDIEntry;
+#endif
+} PVRSRV_DEVICE_DEBUG_INFO;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+#define RGX_LISR_INIT							(0U)
+#define RGX_LISR_DEVICE_NOT_POWERED				(1U)
+#define RGX_LISR_NOT_TRIGGERED_BY_HW			(2U)
+#define RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED		(3U)
+#define RGX_LISR_PROCESSED						(4U)
+
+typedef IMG_UINT32 LISR_STATUS;
+
+typedef struct _LISR_EXECUTION_INFO_
+{
+	/* status of last LISR invocation */
+	LISR_STATUS ui32Status;
+
+	/* snapshot from the last LISR invocation */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	IMG_UINT32 aui32InterruptCountSnapshot[RGX_NUM_DRIVERS_SUPPORTED];
+#else
+	IMG_UINT32 aui32InterruptCountSnapshot[RGXFW_THREAD_NUM];
+#endif
+
+	/* time of the last LISR invocation */
+	IMG_UINT64 ui64Clockns;
+} LISR_EXECUTION_INFO;
+
+#define UPDATE_LISR_DBG_STATUS(status)		\
+	do { \
+		psDeviceNode->sLISRExecutionInfo.ui32Status = (status); \
+		if ((status > RGX_LISR_INIT) && (status < RGX_LISR_PROCESSED)) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: IRQ %" IMG_UINT64_FMTSPEC " rejected: %s", __func__, psDeviceNode->ui64nLISR, #status)); \
+		} \
+	} while (0)
+
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)	psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[idx] = (val)
+#define UPDATE_LISR_DBG_TIMESTAMP()			psDeviceNode->sLISRExecutionInfo.ui64Clockns = OSClockns64()
+#define UPDATE_LISR_DBG_COUNTER()			psDeviceNode->ui64nLISR++
+#define UPDATE_MISR_DBG_COUNTER()			psDeviceNode->ui64nMISR++
+#else
+#define UPDATE_LISR_DBG_STATUS(status)
+#define UPDATE_LISR_DBG_SNAPSHOT(idx, val)
+#define UPDATE_LISR_DBG_TIMESTAMP()
+#define UPDATE_LISR_DBG_COUNTER()
+#define UPDATE_MISR_DBG_COUNTER()
+#endif /* defined(PVRSRV_DEBUG_LISR_EXECUTION) */
+
+typedef struct _PVRSRV_DEVICE_NODE_
+{
+	PVRSRV_DEVICE_IDENTIFIER	sDevId;
+	PVRSRV_DEVICE_STATE			eDevState; /* Set using PVRSRVDeviceSetState, not directly. */
+	PVRSRV_DEVICE_FABRIC_TYPE	eDevFabricType;
+
+	ATOMIC_T					eHealthStatus; /* Holds values from PVRSRV_DEVICE_HEALTH_STATUS */
+	ATOMIC_T					eHealthReason; /* Holds values from PVRSRV_DEVICE_HEALTH_REASON */
+	ATOMIC_T					eDebugDumpRequested; /* Holds values from PVRSRV_DEVICE_DEBUG_DUMP_STATUS */
+
+	IMG_HANDLE					*hDebugTable;
+
+	/* device specific MMU attributes */
+	MMU_DEVICEATTRIBS      *psMMUDevAttrs;
+	/* Device specific MMU firmware attributes, used only in some devices */
+	MMU_DEVICEATTRIBS      *psFirmwareMMUDevAttrs;
+	/* Physical Heap where MMU PT pages are allocated from, normally the
+	 * system's default physical heap but can be different for AutoVz driver. */
+	PHYS_HEAP              *psMMUPhysHeap;
+
+	/* lock for power state transitions */
+	POS_LOCK				hPowerLock;
+	IMG_PID                 uiPwrLockOwnerPID; /* Only valid between lock and corresponding unlock
+	                                              operations of hPowerLock */
+#if defined(DEBUG)
+	struct
+	{
+		const char        *pszFile;			    /* Power lock acquired location (File) */
+		IMG_UINT32         ui32LineNum;		    /* Power lock acquired location (Line number) */
+		IMG_UINT64         ui64Timestamp;       /* Power lock acquired timestamp */
+	} sPowerLockOwner;
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+	IMG_UINT32              uiPowerOffCounter; /* Counts how many times the device has been powered
+	                                              off. Incremented in PVRSRVSetDeviceCurrentPowerState().*/
+	IMG_UINT32              uiPowerOffCounterNext; /* Value of next update to uiPowerOffCounter. */
+#endif
+
+	/* current system device power state */
+	PVRSRV_SYS_POWER_STATE	eCurrentSysPowerState;
+	PPVRSRV_POWER_DEV	psPowerDev;
+
+	/*
+		callbacks the device must support:
+	*/
+
+	PVRSRV_ERROR (*pfnInvalFBSCTable)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+									  MMU_CONTEXT *psMMUContext,
+									  IMG_UINT64 ui64FBSCEntries);
+
+	PVRSRV_ERROR (*pfnValidateOrTweakPhysAddrs)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												MMU_DEVICEATTRIBS *psDevAttrs,
+												IMG_UINT64 *pui64Addr);
+
+	void (*pfnMMUCacheInvalidate)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+								  MMU_CONTEXT *psMMUContext,
+								  MMU_LEVEL eLevel,
+								  IMG_BOOL bUnmap);
+
+	PVRSRV_ERROR (*pfnMMUCacheInvalidateKick)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	                                          IMG_UINT32 *pui32NextMMUInvalidateUpdate);
+
+	IMG_UINT32 (*pfnMMUCacheGetInvalidateCounter)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	/* Callback pfnMMUTopLevelPxWorkarounds may be NULL if not required */
+	void (*pfnMMUTopLevelPxWorkarounds)(struct _CONNECTION_DATA_ *psConnection,
+			                            struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+			                            IMG_DEV_PHYADDR sPhysAddrL1Px,
+			                            void *pxL1PxCpuVAddr);
+
+	/* Callback pfnMMUTweakProtFlags may be NULL if not required */
+	void (*pfnMMUTweakProtFlags)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+			                     MMU_DEVICEATTRIBS *psDevAttrs,
+			                     PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+			                     MMU_PROTFLAGS_T *uiMMUProtFlags);
+
+
+	void (*pfnDumpDebugInfo)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnUpdateHealthStatus)(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	                                      IMG_BOOL bIsTimerPoll);
+
+#if defined(SUPPORT_AUTOVZ)
+	void (*pfnUpdateAutoVzWatchdog)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+#endif
+
+	PVRSRV_ERROR (*pfnValidationGPUUnitsPowerChange)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32NewState);
+
+	PVRSRV_ERROR (*pfnResetHWRLogs)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnVerifyBVNC)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask);
+
+	/* Method to drain device HWPerf packets from firmware buffer to host buffer */
+	PVRSRV_ERROR (*pfnServiceHWPerf)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_ERROR (*pfnDeviceVersionString)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_CHAR **ppszVersionString);
+
+	PVRSRV_ERROR (*pfnDeviceClockSpeed)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_PUINT32 pui32RGXClockSpeed);
+
+	PVRSRV_ERROR (*pfnSoftReset)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+
+	PVRSRV_ERROR (*pfnAlignmentCheck)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32FWAlignChecksSize, IMG_UINT32 aui32FWAlignChecks[]);
+	IMG_BOOL	(*pfnCheckDeviceFeature)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT64 ui64FeatureMask);
+
+	IMG_INT32	(*pfnGetDeviceFeatureValue)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, enum _RGX_FEATURE_WITH_VALUE_INDEX_ eFeatureIndex);
+
+    PVRSRV_ERROR (*pfnGetMultiCoreInfo)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores, IMG_UINT64 *pui64Caps);
+
+	IMG_BOOL (*pfnHasFBCDCVersion31)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	IMG_UINT32 (*pfnGetTFBCLossyGroup)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	MMU_DEVICEATTRIBS* (*pfnGetMMUDeviceAttributes)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_BOOL bKernelMemoryCtx);
+
+	PVRSRV_DEVICE_SNOOP_MODE (*pfnGetDeviceSnoopMode)(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	PVRSRV_DEVICE_CONFIG	*psDevConfig;
+
+	/* device post-finalise compatibility check */
+	PVRSRV_ERROR			(*pfnInitDeviceCompatCheck) (struct _PVRSRV_DEVICE_NODE_*);
+
+	/* initialise device-specific physheaps */
+	PVRSRV_ERROR			(*pfnPhysMemDeviceHeapsInit) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* determining the appropriate LMA allocation policy */
+	PHYS_HEAP_POLICY		(*pfnPhysHeapGetLMAPolicy) (PHYS_HEAP_USAGE_FLAGS, struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+	/* initialise fw mmu, if FW not using GPU mmu, NULL otherwise. */
+	PVRSRV_ERROR			(*pfnFwMMUInit) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* Check device's FW Main physheap free memory */
+	PVRSRV_ERROR			(*pfnCheckForSufficientFWPhysMem) (struct _PVRSRV_DEVICE_NODE_ *);
+
+	/* information about the device's address space and heaps */
+	DEVICE_MEMORY_INFO		sDevMemoryInfo;
+
+	/* device's shared-virtual-memory heap max virtual address */
+	IMG_UINT64				ui64GeneralSVMHeapTopVA;
+
+	ATOMIC_T				iNumClockSpeedChanges;
+
+	/* private device information */
+	void					*pvDevice;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	RA_ARENA				*psOSSharedArena;
+	RA_ARENA				*psOSidSubArena[GPUVIRT_VALIDATION_NUM_OS];
+#endif
+
+	/* When virtualisation support is enabled the Firmware heaps of virtualised
+	 * drivers can be entirely premapped into the Fw's VA space, during init
+	 * or during runtime on explicit request from Guest drivers. */
+	PHYS_HEAP               *apsFWPremapPhysHeap[RGX_NUM_DRIVERS_SUPPORTED];
+
+	/* Head of the physical heap list. Tracks PhysHeap objects created from
+	 * the PHYS_HEAP_CONFIG definitions supplied by the system layer at
+	 * device creation time. There could be 1 or more and varies from system
+	 * to system.
+	 */
+	struct _PHYS_HEAP_		*psPhysHeapList;
+	POS_LOCK				hPhysHeapLock;
+
+	/* The apsPhysHeap array is a mapping table to the system's, often fewer,
+	 * physical memory heaps defined for this device. It contains
+	 * PVRSRV_PHYS_HEAP_LAST entries, one for each possible physical
+	 * heaps allowed in the design. Each PhysHeap in the design is acquired
+	 * and stored in the mapping table during device create. Fall-back logic
+	 * is employed to ensure a valid heap is always found from the set defined
+	 * in the system layer for the device. Responsibility for this is shared
+	 * between the common layer (PhysHeapInitDeviceHeaps) and sub-device
+	 * layer (pfnPhysMemDeviceHeapsInit).
+	 * It is used in the PhysMem module to create PMRs from a given PhysHeap
+	 * of memory. See PhysHeapCreatePMR()
+	 */
+	PHYS_HEAP				*apsPhysHeap[PVRSRV_PHYS_HEAP_LAST];
+	IMG_UINT32				ui32UserAllocHeapCount;
+
+	/* Flag indicating if the firmware has been initialised during the
+	 * 1st boot of the Host driver according to the AutoVz life-cycle. */
+	IMG_BOOL				bAutoVzFwIsUp;
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL				bAutoVzAllowGPUPowerdown;
+#endif
+
+	/* Flags indicating VM state and if PVZ calls originating from it are valid */
+	IMG_UINT32				ui32VmState;
+
+	struct _PVRSRV_DEVICE_NODE_	*psNext;
+	struct _PVRSRV_DEVICE_NODE_	**ppsThis;
+
+	/* Functions for notification about memory contexts */
+	PVRSRV_ERROR			(*pfnRegisterMemoryContext)(struct _PVRSRV_DEVICE_NODE_	*psDeviceNode,
+														MMU_CONTEXT					*psMMUContext,
+														IMG_HANDLE					*hPrivData);
+	void					(*pfnUnregisterMemoryContext)(IMG_HANDLE hPrivData);
+
+	/* Functions for allocation/freeing of UFOs */
+	AllocUFOBlockCallback	pfnAllocUFOBlock;	/*!< Callback for allocation of a block of UFO memory */
+	FreeUFOBlockCallback	pfnFreeUFOBlock;	/*!< Callback for freeing of a block of UFO memory */
+
+	IMG_HANDLE				hSyncServerRecordNotify;
+	POS_LOCK				hSyncServerRecordLock;
+	IMG_UINT32				ui32SyncServerRecordCount;
+	IMG_UINT32				ui32SyncServerRecordCountHighWatermark;
+	DLLIST_NODE				sSyncServerRecordList;
+	struct SYNC_RECORD		*apsSyncServerRecordsFreed[PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN];
+	IMG_UINT32				uiSyncServerRecordFreeIdx;
+
+	IMG_HANDLE				hSyncCheckpointRecordNotify;
+	POS_LOCK				hSyncCheckpointRecordLock;
+	IMG_UINT32				ui32SyncCheckpointRecordCount;
+	IMG_UINT32				ui32SyncCheckpointRecordCountHighWatermark;
+	DLLIST_NODE				sSyncCheckpointRecordList;
+	struct SYNC_CHECKPOINT_RECORD	*apsSyncCheckpointRecordsFreed[PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN];
+	IMG_UINT32				uiSyncCheckpointRecordFreeIdx;
+
+	IMG_HANDLE				hSyncCheckpointNotify;
+	POS_SPINLOCK			hSyncCheckpointListLock; /*!< Protects sSyncCheckpointSyncsList */
+	DLLIST_NODE				sSyncCheckpointSyncsList;
+
+	PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext;
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+
+	/* With this sync-prim we make sure the MMU cache is flushed
+	 * before we free the page table memory */
+	PVRSRV_CLIENT_SYNC_PRIM	*psMMUCacheSyncPrim;
+	IMG_UINT32				ui32NextMMUInvalidateUpdate;
+
+	IMG_HANDLE				hCmdCompNotify;
+	IMG_HANDLE				hDbgReqNotify;
+	IMG_HANDLE				hAppHintDbgReqNotify;
+	IMG_HANDLE				hPhysHeapDbgReqNotify;
+
+	/* Device MMU common module can support one other larger page size
+	 * (e.g. 16KB) in addition to the default OS page size (often 4KB).
+	 * If supported, device code will store the size here as a log2 value
+	 * during device creation/registration. A 0 value will imply it is
+	 * not supported.
+	 */
+	IMG_UINT32              ui32Non4KPageSizeLog2;
+
+	PVRSRV_DEF_PAGE			sScratchPage;
+	PVRSRV_DEF_PAGE			sDevZeroPage;
+
+	/* Lock protects access to sMemoryContextPageFaultNotifyListHead and
+	 * per memory context DEVMEMINT_CTX::sProcessNotifyListHead lists. */
+	POSWR_LOCK				hPageFaultNotifyLock;
+	DLLIST_NODE				sMemoryContextPageFaultNotifyListHead;
+
+	/* System DMA channels */
+	IMG_UINT32				ui32RefCountDMA;
+	IMG_HANDLE				hDmaTxChan;
+	IMG_HANDLE				hDmaRxChan;
+	POS_LOCK				hDmaTxLock;
+	POS_LOCK				hDmaRxLock;
+
+#if defined(PDUMP)
+	/*
+	 * FBC clear color register default value to use.
+	 */
+	IMG_UINT64				ui64FBCClearColour;
+
+	/* Device-level callback which is called when pdump.exe starts.
+	 * Should be implemented in device-specific init code, e.g. rgxinit.c
+	 */
+	PVRSRV_ERROR			(*pfnPDumpInitDevice)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+	/* device-level callback to return pdump ID associated to a memory context */
+	IMG_UINT32				(*pfnMMUGetContextID)(IMG_HANDLE hDevMemContext);
+
+	IMG_UINT8			*pui8DeferredSyncCPSignal;	/*! Deferred fence events buffer */
+
+	IMG_UINT16			ui16SyncCPReadIdx;		/*! Read index in the above deferred fence events buffer */
+
+	IMG_UINT16			ui16SyncCPWriteIdx;		/*! Write index in the above deferred fence events buffer */
+
+	POS_LOCK			hSyncCheckpointSignalLock;	/*! Guards data shared between an sleepable-contexts */
+
+	void				*pvSyncCPMISR;			/*! MISR to emit pending/deferred fence signals */
+
+	void				*hTransition;			/*!< SyncCheckpoint PdumpTransition Cookie */
+
+	DLLIST_NODE			sSyncCheckpointContextListHead;	/*!< List head for the sync chkpt contexts */
+
+	POS_LOCK			hSyncCheckpointContextListLock;	/*! lock for accessing sync chkpt contexts list */
+
+#endif
+
+
+	/* Members for linking which connections are open on this device */
+	POS_LOCK                hConnectionsLock;    /*!< Lock protecting sConnections */
+	DLLIST_NODE             sConnections;        /*!< The list of currently active connection objects for this device node */
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	LISR_EXECUTION_INFO     sLISRExecutionInfo;  /*!< Information about the last execution of the LISR */
+	IMG_UINT64              ui64nLISR;           /*!< Number of LISR calls seen */
+	IMG_UINT64              ui64nMISR;           /*!< Number of MISR calls made */
+#endif
+
+	PVRSRV_DEVICE_DEBUG_INFO sDebugInfo;
+	IMG_BOOL                bEnablePFDebug;      /*!< EnablePageFaultDebug AppHint setting for device */
+
+	DLLIST_NODE             sCleanupThreadWorkList; /*!< List of work for the cleanup thread associated with the device */
+	ATOMIC_T                i32NumCleanupItems;   /*!< Number of cleanup thread work items. Includes items being freed. */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Data for the deferred freeing of a PMR physical pages for a given device */
+	DLLIST_NODE             sPMRZombieList;       /*!< List of PMRs to free */
+	POS_LOCK                hPMRZombieListLock;   /*!< List lock */
+	IMG_UINT32              uiPMRZombieCount;     /*!< Number of elements in the list */
+	IMG_UINT32              uiPMRZombieCountInCleanup; /*!< Number of elements in cleanup items */
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+	ATOMIC_T                eFrozen;              /*< Frozen / Unfrozen indicator */
+	IMG_HANDLE              hDeviceThreadEvObj;   /*< Event Object for Freeze indicator */
+	IMG_HANDLE              hDeviceFreezeThaw;    /*< Event handle for Freeze/Thaw */
+	POS_LOCK                hFreezeThawLock;      /*< Freeze/Thaw lock */
+	ATOMIC_T                iFreezeCount;         /*< Number of blocked on frozen tasks */
+	ATOMIC_T                iTotalFreezes;        /*< Total number of times device frozen */
+	ATOMIC_T                iThreadsActive;       /*< Number of threads active on this device */
+	IMG_UINT64              ui64LastDeviceOffTimestamp; /* Last device power off timestamp */
+	IMG_UINT64              ui64LastDeviceOffHostTimestampNs; /* Last device power off host timestamp */
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	IMG_BOOL bGPUWorkPeriodFTraceEnabled;
+#endif
+} PVRSRV_DEVICE_NODE;
+
+/*
+ * Macros to be used instead of calling directly the pfns since these macros
+ * will expand the feature passed as argument into the bitmask/index to work
+ * with the macros defined in rgx_bvnc_defs_km.h
+ */
+#define PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, Feature) \
+		psDevNode->pfnCheckDeviceFeature(psDevNode, RGX_FEATURE_##Feature##_BIT_MASK)
+#define PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, Feature) \
+		psDevNode->pfnGetDeviceFeatureValue(psDevNode, RGX_FEATURE_##Feature##_IDX)
+
+PVRSRV_ERROR PVRSRVDeviceFinalise(PVRSRV_DEVICE_NODE *psDeviceNode,
+											   IMG_BOOL bInitSuccessful);
+
+PVRSRV_ERROR PVRSRVDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE * psDeviceNode, IMG_UINT32 ui32ClientBuildOptions);
+
+void PVRSRVDeviceSetState(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_STATE eNewDevState);
+
+#define PVRSRVIsStatusRecoverable(eStatus) \
+	(((eStatus == PVRSRV_DEVICE_HEALTH_STATUS_DEAD)) ? \
+	 IMG_FALSE : IMG_TRUE)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+/* Determines if a 32-bit `uiCurrent` counter advanced to or beyond
+ * `uiRequired` value. The function takes into consideration that the
+ * counter could have wrapped around. */
+static INLINE IMG_BOOL PVRSRVHasCounter32Advanced(IMG_UINT32 uiCurrent,
+                                                  IMG_UINT32 uiRequired)
+{
+	return uiCurrent >= uiRequired ?
+	    /* ... with the counter wrapped around ...
+	     * There can't be ~4 billion transactions completed, so consider wrapped */
+	    (((uiCurrent - uiRequired) > 0xF0000000UL) ? IMG_FALSE : IMG_TRUE) :
+	    /* There can't be ~4 billion transactions pending, so consider wrapped */
+	    (((uiRequired - uiCurrent) > 0xF0000000UL) ? IMG_TRUE : IMG_FALSE);
+}
+#endif
+
+#endif /* DEVICE_H */
+
+/******************************************************************************
+ End of file (device.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/device_connection.h b/drivers/gpu/drm/img-rogue/device_connection.h
new file mode 100644
index 000000000000..b80983c2d87d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/device_connection.h
@@ -0,0 +1,130 @@
+/*************************************************************************/ /*!
+@File           device_connection.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(DEVICE_CONNECTION_H)
+#define DEVICE_CONNECTION_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(__KERNEL__)
+typedef struct _PVRSRV_DEVICE_NODE_ *SHARED_DEV_CONNECTION;
+#else
+#include "connection.h"
+typedef const struct PVRSRV_DEV_CONNECTION_TAG *SHARED_DEV_CONNECTION;
+#endif
+
+/******************************************************************************
+ * Device capability flags and masks
+ *
+ * Following bitmask shows allocated ranges and values for our device
+ * capability settings:
+ *
+ * 31 27  23  19  15  11   7   3  0
+ * |...|...|...|...|...|...|...|...
+ *                               ** CACHE_COHERENT                   [0x1..0x2]
+ *                                x  PVRSRV_CACHE_COHERENT_DEVICE_FLAG
+ *                               x.  PVRSRV_CACHE_COHERENT_CPU_FLAG
+ *                             *... NONMAPPABLE_MEMORY                    [0x8]
+ *                             x...  PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG
+ *                            *.... PDUMP_IS_RECORDING                   [0x10]
+ *                            x....  PVRSRV_PDUMP_IS_RECORDING
+ *                      ***........ DEVMEM_SVM_ALLOC             [0x100..0x400]
+ *                        x........  PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED
+ *                       x.........  PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED
+ *                      x..........  PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL
+ *                     *........... FBCDC_V3_1             [0x800]
+ *                     x...........  FBCDC_V3_1_USED
+ *                    *............ PVRSRV_SYSTEM_DMA
+ *                    x............  PVRSRV_SYSTEM_DMA_USED
+ *                   *............. TFBC_LOSSY_GROUP
+ *                   x.............  TFBC_LOSSY_GROUP_1
+ * |...|...|...|...|...|...|...|...
+ *****************************************************************************/
+
+/* Flag to be passed over the bridge during connection stating whether CPU cache coherent is available*/
+#define PVRSRV_CACHE_COHERENT_SHIFT (0)
+#define	PVRSRV_CACHE_COHERENT_DEVICE_FLAG (1U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define	PVRSRV_CACHE_COHERENT_CPU_FLAG (2U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define	PVRSRV_CACHE_COHERENT_EMULATE_FLAG (4U << PVRSRV_CACHE_COHERENT_SHIFT)
+#define PVRSRV_CACHE_COHERENT_MASK (7U << PVRSRV_CACHE_COHERENT_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether CPU non-mappable memory is present */
+#define PVRSRV_NONMAPPABLE_MEMORY_PRESENT_SHIFT (7)
+#define PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG (1U << PVRSRV_NONMAPPABLE_MEMORY_PRESENT_SHIFT)
+
+/* Flag to be passed over the bridge to indicate PDump activity */
+#define PVRSRV_PDUMP_IS_RECORDING_SHIFT (4)
+#define PVRSRV_PDUMP_IS_RECORDING (1U << PVRSRV_PDUMP_IS_RECORDING_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating SVM allocation availability */
+#define PVRSRV_DEVMEM_SVM_ALLOC_SHIFT (8)
+#define PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED (1U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+#define PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED (2U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+#define PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL (4U << PVRSRV_DEVMEM_SVM_ALLOC_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether GPU uses FBCDC v3.1 */
+#define PVRSRV_FBCDC_V3_1_USED_SHIFT (11)
+#define PVRSRV_FBCDC_V3_1_USED (1U << PVRSRV_FBCDC_V3_1_USED_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether System has
+   DMA transfer capability to and from device memory */
+#define PVRSRV_SYSTEM_DMA_SHIFT (12)
+#define PVRSRV_SYSTEM_DMA_USED (1U << PVRSRV_SYSTEM_DMA_SHIFT)
+
+/* Flag to be passed over the bridge during connection stating whether GPU supports TFBC and is
+   configured to use lossy compression control group 1 (25% / 37.5% / 50%) */
+#define PVRSRV_TFBC_LOSSY_GROUP_SHIFT (13)
+#define PVRSRV_TFBC_LOSSY_GROUP_1 (1U << PVRSRV_TFBC_LOSSY_GROUP_SHIFT)
+
+static INLINE IMG_HANDLE GetBridgeHandle(SHARED_DEV_CONNECTION hDevConnection)
+{
+#if defined(__KERNEL__)
+    return hDevConnection;
+#else
+    return hDevConnection->hServices;
+#endif
+}
+
+
+#endif /* !defined(DEVICE_CONNECTION_H) */
diff --git a/drivers/gpu/drm/img-rogue/devicemem.c b/drivers/gpu/drm/img-rogue/devicemem.c
new file mode 100644
index 000000000000..916485b77ec1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem.c
@@ -0,0 +1,2755 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Front End (nominally Client side part, but now invokable
+                from server too) of device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "devicemem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "osfunc.h"
+#include "osmmap.h"
+#include "devicemem_utils.h"
+#include "client_mm_bridge.h"
+#include "client_cache_bridge.h"
+#include "services_km.h"
+
+#if defined(PDUMP)
+#if defined(__KERNEL__)
+#include "pdump_km.h"
+#else
+#include "pdump_um.h"
+#endif
+#include "devicemem_pdump.h"
+#endif
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "client_ri_bridge.h"
+#endif
+#include "client_devicememhistory_bridge.h"
+#include "info_page_client.h"
+
+#include "rgx_heaps.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "pvr_ricommon.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "srvcore.h"
+#if defined(__linux__)
+#include "linux/kernel.h"
+#endif
+#else
+#include "srvcore_intern.h"
+#include "rgxdefs.h"
+#endif
+
+#if defined(__KERNEL__) && defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+extern PVRSRV_ERROR RIDumpAllKM(void);
+#endif
+
+#if defined(__KERNEL__)
+#define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError)
+#else
+#define GET_ERROR_STRING(eError) PVRSRVGetErrorString(eError)
+#endif
+
+#if defined(__KERNEL__)
+/* Derive the virtual from the hPMR */
+static
+IMG_UINT64 _GetPremappedVA(PMR *psPMR, PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+
+	IMG_DEV_PHYADDR sDevAddr;
+	IMG_BOOL bValid;
+	PHYS_HEAP *psPhysHeap = psDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+	IMG_DEV_PHYADDR sHeapAddr;
+
+	eError = PhysHeapGetDevPAddr(psPhysHeap, &sHeapAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", fail);
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+{
+	if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_UMA ||
+			PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_DMA)
+	{
+		IMG_DEV_PHYADDR sDevPAddrCorrected;
+
+		PhysHeapCpuPAddrToDevPAddr(psPhysHeap, 1, &sDevPAddrCorrected, (IMG_CPU_PHYADDR *)&sHeapAddr);
+		sHeapAddr.uiAddr = sDevPAddrCorrected.uiAddr;
+	}
+}
+#endif
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddr", fail);
+
+	eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sDevAddr, &bValid, DEVICE_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMR_DevPhysAddr");
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddr");
+		goto fail;
+	}
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddr");
+
+	ui64OptionalMapAddress = RGX_FIRMWARE_RAW_HEAP_BASE | (sDevAddr.uiAddr - sHeapAddr.uiAddr);
+
+	PVR_DPF((PVR_DBG_ALLOC, "%s: sDevAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" sHeapAddr.uiAddr = 0x%"IMG_UINT64_FMTSPECx" => ui64OptionalMapAddress = 0x%"IMG_UINT64_FMTSPECx,
+	         __func__, sDevAddr.uiAddr, sHeapAddr.uiAddr, ui64OptionalMapAddress));
+fail:
+	return ui64OptionalMapAddress;
+}
+#endif
+
+/*****************************************************************************
+ *                    Sub allocation internals                               *
+ *****************************************************************************/
+static INLINE PVRSRV_MEMALLOCFLAGS_T
+DevmemOverrideFlagsOrPassThrough(SHARED_DEV_CONNECTION hDevConnection, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+#if defined(__KERNEL__) && defined(RGX_FEATURE_GPU_CPU_COHERENCY_BIT_MASK)
+	/*
+	 *  Override the requested memory flags of FW allocations only,
+	 *  non-FW allocations pass-through unmodified.
+	 *
+	 *  On ace-lite platforms:
+	 *    - If the allocation is not CPU cached, then there is nothing
+	 *      for the GPU to snoop regardless of the GPU cache setting.
+	 *    - If the allocation is not GPU cached, then the SLC will not
+	 *      be used and will not snoop the CPU even if it is CPU cached.
+	 *    - Therefore only the GPU setting can be upgraded to coherent
+	 *      if it is already GPU cached incoherent and the CPU is cached.
+	 *
+	 *  All other platforms:
+	 *    - Do not modify the allocation flags.
+	 */
+
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDevConnection;
+
+	if (psDevNode->pvDevice != NULL && PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, GPU_CPU_COHERENCY))
+	{
+		if (PVRSRV_CHECK_FW_MAIN(uiFlags))
+		{
+			if ((PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) || PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags)) &&
+			         (PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags)) &&
+			         PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+			         psDevNode->eDevFabricType == PVRSRV_DEVICE_FABRIC_ACELITE)
+			{
+				/* Upgrade the allocation from GPU cached incoherent to GPU cached coherent. */
+				uiFlags &= ~PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK;
+				uiFlags |= PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hDevConnection);
+#endif
+
+	return uiFlags;
+}
+
+static INLINE void
+CheckAnnotationLength(const IMG_CHAR *pszAnnotation)
+{
+	IMG_UINT32 length = OSStringLength(pszAnnotation);
+
+	if (length >= DEVMEM_ANNOTATION_MAX_LEN)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Annotation \"%s\" has been truncated to %d characters from %d characters",
+				__func__, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1, length));
+	}
+}
+
+static PVRSRV_ERROR
+AllocateDeviceMemory(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiLog2Quantum,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_UINT32 ui32NumPhysChunks,
+		IMG_UINT32 ui32NumVirtChunks,
+		IMG_UINT32 *pui32MappingTable,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		IMG_BOOL bExportable,
+		const IMG_CHAR *pszAnnotation,
+		DEVMEM_IMPORT **ppsImport)
+{
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_MEMALLOCFLAGS_T uiOutFlags;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failAlloc);
+
+	/* check if shift value is not too big (sizeof(1ULL)) */
+	PVR_ASSERT(uiLog2Quantum < sizeof(unsigned long long) * 8);
+	/* Check the size is a multiple of the quantum */
+	PVR_ASSERT((uiSize & ((1ULL<<uiLog2Quantum)-1)) == 0);
+
+	CheckAnnotationLength(pszAnnotation);
+
+	/* Pass only the PMR flags down */
+	uiOutFlags = uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK;
+	eError = BridgePhysmemNewRamBackedPMR(GetBridgeHandle(hDevConnection),
+			uiSize,
+			ui32NumPhysChunks,
+			ui32NumVirtChunks,
+			pui32MappingTable,
+			uiLog2Quantum,
+			uiOutFlags,
+			OSStringNLength(pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN - 1) + 1,
+			pszAnnotation,
+			OSGetCurrentProcessID(),
+			&hPMR,
+			PDUMP_NONE,
+			&uiOutFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate memory for %s (%s)",
+				__func__,
+				pszAnnotation,
+				PVRSRVGETERRORSTRING(eError)));
+		goto failPMR;
+	}
+
+	uiFlags &= ~PVRSRV_PHYS_HEAP_HINT_MASK;
+	uiFlags |= (uiOutFlags & PVRSRV_PHYS_HEAP_HINT_MASK);
+
+	DevmemImportStructInit(psImport,
+			uiSize,
+			uiAlign,
+			uiFlags,
+			hPMR,
+			bExportable ? DEVMEM_PROPERTIES_EXPORTABLE : 0);
+
+	*ppsImport = psImport;
+	return PVRSRV_OK;
+
+failPMR:
+	DevmemImportDiscard(psImport);
+failAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*****************************************************************************
+ *                    Sub allocation internals                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc,
+		IMG_UINT32 ui32AllocPageCount,
+		IMG_UINT32 *paui32AllocPageIndices,
+		IMG_UINT32 ui32FreePageCount,
+		IMG_UINT32 *pauiFreePageIndices,
+		SPARSE_MEM_RESIZE_FLAGS uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_IMPORT *psImport;
+	SHARED_DEV_CONNECTION hDevConnection;
+	IMG_HANDLE hPMR;
+	DEVMEM_HEAP *psHeap;
+	IMG_HANDLE hReservation;
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	DEVMEM_PROPERTIES_T uiProperties;
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(psMemDesc->psImport != NULL);
+
+	psImport = psMemDesc->psImport;
+	hDevConnection = psImport->hDevConnection;
+	hPMR = psImport->hPMR;
+	psHeap = psImport->sDeviceImport.psHeap;
+	hReservation = psImport->sDeviceImport.hReservation;
+
+	PVR_ASSERT(hDevConnection != NULL);
+	PVR_ASSERT(hPMR != NULL);
+	PVR_ASSERT(psHeap != NULL);
+	PVR_ASSERT(BITMASK_ANY(uiSparseFlags, SPARSE_RESIZE_BOTH));
+	PVR_ASSERT(hReservation != LACK_OF_RESERVATION_POISON);
+
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	PVR_ASSERT(!BITMASK_HAS(uiProperties, DEVMEM_PROPERTIES_SECURE));
+	PVR_ASSERT(!BITMASK_HAS(uiProperties, DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE));
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount == 0);
+
+	OSLockAcquire(psImport->hLock);
+
+	eError = BridgeChangeSparseMem(GetBridgeHandle(hDevConnection),
+	                               ui32AllocPageCount,
+	                               paui32AllocPageIndices,
+	                               ui32FreePageCount,
+	                               pauiFreePageIndices,
+	                               uiSparseFlags,
+	                               hReservation);
+
+	OSLockRelease(psImport->hLock);
+
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (GetInfoPageDebugFlags(hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistorySparseChange(GetBridgeHandle(hDevConnection),
+		                                   hPMR,
+		                                   psMemDesc->uiOffset,
+		                                   psMemDesc->sDeviceMemDesc.sDevVAddr,
+		                                   psMemDesc->uiAllocSize,
+		                                   psMemDesc->szText,
+		                                   DevmemGetHeapLog2PageSize(psHeap),
+		                                   ui32AllocPageCount,
+		                                   paui32AllocPageIndices,
+		                                   ui32FreePageCount,
+		                                   pauiFreePageIndices,
+		                                   psMemDesc->ui32AllocationIndex,
+		                                   &psMemDesc->ui32AllocationIndex);
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+FreeDeviceMemory(DEVMEM_IMPORT *psImport)
+{
+	DevmemImportStructRelease(psImport);
+}
+
+static PVRSRV_ERROR
+SubAllocImportAlloc(RA_PERARENA_HANDLE hArena,
+		RA_LENGTH_T uiSize,
+		RA_FLAGS_T _flags,
+		RA_LENGTH_T uBaseAlignment,
+		const IMG_CHAR *pszAnnotation,
+		/* returned data */
+		RA_IMPORT *psRAImport)
+{
+	/* When suballocations need a new lump of memory, the RA calls
+	   back here.  Later, in the kernel, we must construct a new PMR
+	   and a pairing between the new lump of virtual memory and the
+	   PMR (whether or not such PMR is backed by physical memory) */
+	DEVMEM_HEAP *psHeap;
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags = (PVRSRV_MEMALLOCFLAGS_T) _flags;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+
+	/* Per-arena private handle is, for us, the heap */
+	psHeap = hArena;
+
+	/* The RA should not have invoked us with a size that is not a
+	   multiple of the quantum anyway */
+	PVR_ASSERT((uiSize & ((1ULL<<psHeap->uiLog2Quantum)-1)) == 0);
+
+	eError = AllocateDeviceMemory(psHeap->psCtx->hDevConnection,
+			psHeap->uiLog2Quantum,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			uBaseAlignment,
+			uiFlags,
+			IMG_FALSE,
+			"PMR sub-allocated",
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failAlloc);
+
+#if defined(PDUMP) && defined(DEBUG)
+#if defined(__KERNEL__)
+	PDUMPCOMMENTWITHFLAGS(PMR_DeviceNode((PMR*)psImport->hPMR), PDUMP_CONT,
+			"Created PMR for sub-allocations with handle ID: 0x%p Annotation: \"%s\" (PID %u)",
+			psImport->hPMR, pszAnnotation, OSGetCurrentProcessID());
+#else
+	PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS,
+			"Created PMR for sub-allocations with handle ID: %p Annotation: \"%s\" (PID %u)",
+			psImport->hPMR, pszAnnotation, OSGetCurrentProcessID());
+#endif
+#else
+	PVR_UNREFERENCED_PARAMETER(pszAnnotation);
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+#if defined(__KERNEL__)
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection;
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice;
+
+		PVR_ASSERT(PVRSRV_CHECK_FW_MAIN(uiFlags));
+
+		/* If allocation is made by the Kernel from the firmware heap, account for it
+		 * under the PVR_SYS_ALLOC_PID.
+		 */
+		if ((psHeap == psDevInfo->psFirmwareMainHeap) || (psHeap == psDevInfo->psFirmwareConfigHeap))
+		{
+			eError = BridgeRIWritePMREntryWithOwner (GetBridgeHandle(psImport->hDevConnection),
+					psImport->hPMR,
+					PVR_SYS_ALLOC_PID);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntryWithOwner");
+		}
+		else
+#endif
+		{
+			eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+					psImport->hPMR);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+		}
+	}
+#endif
+
+#if defined(__KERNEL__)
+	if (psHeap->bPremapped)
+	{
+		ui64OptionalMapAddress = _GetPremappedVA(psImport->hPMR, psHeap->psCtx->hDevConnection);
+	}
+#endif
+
+	/*
+		Suballocations always get mapped into the device was we need to
+		key the RA off something and as we can't export suballocations
+		there is no valid reason to request an allocation an not map it
+	 */
+	eError = DevmemImportStructDevMap(psHeap,
+			IMG_TRUE,
+			psImport,
+			ui64OptionalMapAddress);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	OSLockAcquire(psImport->hLock);
+	/* Mark this import struct as zeroed so we can save some PDump LDBs
+	 * and do not have to CPU map + mem set()*/
+	if (uiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC)
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_ZEROED;
+	}
+	else if (uiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC)
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_POISONED;
+	}
+	psImport->uiProperties |= DEVMEM_PROPERTIES_IMPORT_IS_CLEAN;
+	OSLockRelease(psImport->hLock);
+
+	psRAImport->base = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	psRAImport->uSize = uiSize;
+	psRAImport->hPriv = psImport;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failMap:
+	FreeDeviceMemory(psImport);
+failAlloc:
+
+	return eError;
+}
+
+static void
+SubAllocImportFree(RA_PERARENA_HANDLE hArena,
+		RA_BASE_T uiBase,
+		RA_PERISPAN_HANDLE hImport)
+{
+	DEVMEM_IMPORT *psImport = hImport;
+#if !defined(PVRSRV_NEED_PVR_ASSERT)
+	PVR_UNREFERENCED_PARAMETER(hArena);
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+#endif
+
+	PVR_ASSERT(psImport != NULL);
+	PVR_ASSERT(hArena == psImport->sDeviceImport.psHeap);
+	PVR_ASSERT(uiBase == psImport->sDeviceImport.sDevVAddr.uiAddr);
+
+	(void) DevmemImportStructDevUnmap(psImport);
+	(void) DevmemImportStructRelease(psImport);
+}
+
+/*****************************************************************************
+ *                    Devmem context internals                               *
+ *****************************************************************************/
+
+static PVRSRV_ERROR
+PopulateContextFromBlueprint(struct DEVMEM_CONTEXT_TAG *psCtx,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	struct DEVMEM_HEAP_TAG **ppsHeapArray;
+	IMG_UINT32 uiNumHeaps;
+	IMG_UINT32 uiHeapsToUnwindOnError;
+	IMG_UINT32 uiHeapIndex;
+	IMG_DEV_VIRTADDR sDevVAddrBase;
+	IMG_CHAR aszHeapName[DEVMEM_HEAPNAME_MAXLENGTH];
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2DataPageSize;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2ImportAlignment;
+
+	eError = DevmemHeapCount(psCtx->hDevConnection,
+			uiHeapBlueprintID,
+			&uiNumHeaps);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (uiNumHeaps == 0)
+	{
+		ppsHeapArray = NULL;
+	}
+	else
+	{
+		ppsHeapArray = OSAllocMem(sizeof(*ppsHeapArray) * uiNumHeaps);
+		PVR_GOTO_IF_NOMEM(ppsHeapArray, eError, e0);
+	}
+
+	uiHeapsToUnwindOnError = 0;
+
+	for (uiHeapIndex = 0; uiHeapIndex < uiNumHeaps; uiHeapIndex++)
+	{
+		eError = DevmemHeapDetails(psCtx->hDevConnection,
+				uiHeapBlueprintID,
+				uiHeapIndex,
+				&aszHeapName[0],
+				sizeof(aszHeapName),
+				&sDevVAddrBase,
+				&uiHeapLength,
+				&uiReservedRegionLength,
+				&uiLog2DataPageSize,
+				&uiLog2ImportAlignment);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		eError = DevmemCreateHeap(psCtx,
+				sDevVAddrBase,
+				uiHeapLength,
+				uiReservedRegionLength,
+				uiLog2DataPageSize,
+				uiLog2ImportAlignment,
+				aszHeapName,
+				uiHeapBlueprintID,
+				uiHeapIndex,
+				&ppsHeapArray[uiHeapIndex]);
+		PVR_GOTO_IF_ERROR(eError, e1);
+
+		uiHeapsToUnwindOnError = uiHeapIndex + 1;
+	}
+
+	psCtx->uiAutoHeapCount = uiNumHeaps;
+	psCtx->ppsAutoHeapArray = ppsHeapArray;
+
+	PVR_ASSERT(psCtx->uiNumHeaps >= psCtx->uiAutoHeapCount);
+	PVR_ASSERT(psCtx->uiAutoHeapCount == uiNumHeaps);
+
+	return PVRSRV_OK;
+
+	/* error exit paths */
+e1:
+	for (uiHeapIndex = 0; uiHeapIndex < uiHeapsToUnwindOnError; uiHeapIndex++)
+	{
+		eError2 = DevmemDestroyHeap(ppsHeapArray[uiHeapIndex]);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+	if (uiNumHeaps != 0)
+	{
+		OSFreeMem(ppsHeapArray);
+	}
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+UnpopulateContextFromBlueprint(struct DEVMEM_CONTEXT_TAG *psCtx)
+{
+	PVRSRV_ERROR eReturn = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+	IMG_UINT32 uiHeapIndex;
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+
+	for (uiHeapIndex = 0; uiHeapIndex < psCtx->uiAutoHeapCount; uiHeapIndex++)
+	{
+		if (!psCtx->ppsAutoHeapArray[uiHeapIndex])
+		{
+			continue;
+		}
+
+		eError2 = DevmemDestroyHeap(psCtx->ppsAutoHeapArray[uiHeapIndex]);
+		if (eError2 != PVRSRV_OK)
+		{
+			eReturn = eError2;
+		}
+		else
+		{
+			psCtx->ppsAutoHeapArray[uiHeapIndex] = NULL;
+		}
+	}
+
+	if ((!bDoCheck || (eReturn == PVRSRV_OK)) && psCtx->ppsAutoHeapArray)
+	{
+		OSFreeMem(psCtx->ppsAutoHeapArray);
+		psCtx->ppsAutoHeapArray = NULL;
+		psCtx->uiAutoHeapCount = 0;
+	}
+
+	return eReturn;
+}
+
+/*****************************************************************************
+ *                    Devmem context functions                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID,
+		DEVMEM_CONTEXT **ppsCtxPtr)
+{
+	PVRSRV_ERROR		eError;
+	DEVMEM_CONTEXT		*psCtx;
+	/* handle to the server-side counterpart of the device memory
+	   context (specifically, for handling mapping to device MMU) */
+	IMG_HANDLE			hDevMemServerContext;
+	IMG_HANDLE			hPrivData;
+	IMG_BOOL			bHeapCfgFWId = (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW);
+
+	PVR_GOTO_IF_NOMEM(ppsCtxPtr, eError, e0);
+
+	psCtx = OSAllocMem(sizeof(*psCtx));
+	PVR_GOTO_IF_NOMEM(psCtx, eError, e0);
+
+	psCtx->uiNumHeaps = 0;
+
+	psCtx->hDevConnection = hDevConnection;
+
+	/* Create (server-side) Device Memory context */
+	eError = BridgeDevmemIntCtxCreate(GetBridgeHandle(psCtx->hDevConnection),
+			bHeapCfgFWId,
+			&hDevMemServerContext,
+			&hPrivData,
+			&psCtx->ui32CPUCacheLineSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	psCtx->hDevMemServerContext = hDevMemServerContext;
+	psCtx->hPrivData = hPrivData;
+
+	/* automagic heap creation */
+	psCtx->uiAutoHeapCount = 0;
+
+	eError = PopulateContextFromBlueprint(psCtx, uiHeapBlueprintID);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	*ppsCtxPtr = psCtx;
+
+	PVR_ASSERT(psCtx->uiNumHeaps == psCtx->uiAutoHeapCount);
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+e2:
+	PVR_ASSERT(psCtx->uiAutoHeapCount == 0);
+	PVR_ASSERT(psCtx->uiNumHeaps == 0);
+	BridgeDevmemIntCtxDestroy(GetBridgeHandle(psCtx->hDevConnection), hDevMemServerContext);
+
+e1:
+	OSFreeMem(psCtx);
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
+		IMG_HANDLE *hPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(psCtx, eError, e0);
+	PVR_GOTO_IF_INVALID_PARAM(hPrivData, eError, e0);
+
+	*hPrivData = psCtx->hPrivData;
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(psCtx, eError, e0);
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemFindHeapByName(const struct DEVMEM_CONTEXT_TAG *psCtx,
+		const IMG_CHAR *pszHeapName,
+		struct DEVMEM_HEAP_TAG **ppsHeapRet)
+{
+	IMG_UINT32 uiHeapIndex;
+
+	/* N.B.  This func is only useful for finding "automagic" heaps by name */
+	for (uiHeapIndex = 0;
+			uiHeapIndex < psCtx->uiAutoHeapCount;
+			uiHeapIndex++)
+	{
+		if (!OSStringNCompare(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName, pszHeapName, OSStringLength(psCtx->ppsAutoHeapArray[uiHeapIndex]->pszName) + 1))
+		{
+			*ppsHeapRet = psCtx->ppsAutoHeapArray[uiHeapIndex];
+			return PVRSRV_OK;
+		}
+	}
+
+	return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemDestroyContext(DEVMEM_CONTEXT *psCtx)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bDoCheck = IMG_TRUE;
+
+#if defined(__KERNEL__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+
+	PVR_RETURN_IF_INVALID_PARAM(psCtx);
+
+	eError = UnpopulateContextFromBlueprint(psCtx);
+	if (bDoCheck && eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: UnpopulateContextFromBlueprint failed (%d) leaving %d heaps",
+				__func__, eError, psCtx->uiNumHeaps));
+		goto e1;
+	}
+
+	eError = DestroyServerResource(psCtx->hDevConnection,
+	                               NULL,
+	                               BridgeDevmemIntCtxDestroy,
+	                               psCtx->hDevMemServerContext);
+	if (bDoCheck)
+	{
+		PVR_LOG_GOTO_IF_ERROR(eError, "BridgeDevMemIntCtxDestroy", e1);
+
+		/* should be no more heaps left */
+		if (psCtx->uiNumHeaps)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Additional heaps remain in DEVMEM_CONTEXT",
+					__func__));
+			eError = PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT;
+			goto e1;
+		}
+	}
+
+	OSCachedMemSet(psCtx, 0, sizeof(*psCtx));
+	OSFreeMem(psCtx);
+
+e1:
+	return eError;
+}
+
+/*****************************************************************************
+ *                 Devmem heap query functions                               *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 *puiNumHeapConfigsOut)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapConfigCount(GetBridgeHandle(hDevConnection),
+			puiNumHeapConfigsOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_UINT32 *puiNumHeapsOut)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapCount(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			puiNumHeapsOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapConfigName(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_CHAR *pszConfigNameOut,
+		IMG_UINT32 uiConfigNameBufSz)
+{
+	PVRSRV_ERROR eError;
+	eError = BridgeHeapCfgHeapConfigName(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			uiConfigNameBufSz,
+			pszConfigNameOut);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_UINT32 uiHeapConfigIndex,
+		IMG_UINT32 uiHeapIndex,
+		IMG_CHAR *pszHeapNameOut,
+		IMG_UINT32 uiHeapNameBufSz,
+		IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+		IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+		IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+		IMG_UINT32 *puiLog2DataPageSizeOut,
+		IMG_UINT32 *puiLog2ImportAlignmentOut)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgeHeapCfgHeapDetails(GetBridgeHandle(hDevConnection),
+			uiHeapConfigIndex,
+			uiHeapIndex,
+			uiHeapNameBufSz,
+			pszHeapNameOut,
+			psDevVAddrBaseOut,
+			puiHeapLengthOut,
+			puiReservedRegionLengthOut,
+			puiLog2DataPageSizeOut,
+			puiLog2ImportAlignmentOut);
+
+	VG_MARK_INITIALIZED(pszHeapNameOut, uiHeapNameBufSz);
+
+	return eError;
+}
+
+/*****************************************************************************
+ *                    Devmem heap functions                                  *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapInt(DEVMEM_HEAP *psHeap,
+		IMG_HANDLE *phDevmemHeap)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+	*phDevmemHeap = psHeap->hDevMemServerHeap;
+	return PVRSRV_OK;
+}
+
+/* See devicemem.h for important notes regarding the arguments
+   to this function */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemCreateHeap(DEVMEM_CONTEXT *psCtx,
+		IMG_DEV_VIRTADDR sBaseAddress,
+		IMG_DEVMEM_SIZE_T uiLength,
+		IMG_DEVMEM_SIZE_T uiReservedRegionLength,
+		IMG_UINT32 ui32Log2Quantum,
+		IMG_UINT32 ui32Log2ImportAlignment,
+		const IMG_CHAR *pszName,
+		DEVMEM_HEAPCFGID uiHeapBlueprintID,
+		IMG_UINT32 uiHeapIndex,
+		DEVMEM_HEAP **ppsHeapPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+	DEVMEM_HEAP *psHeap;
+	/* handle to the server-side counterpart of the device memory heap
+	   (specifically, for handling mapping to device MMU) */
+	IMG_HANDLE hDevMemServerHeap;
+	IMG_UINT32 ui32Policy = RA_POLICY_DEFAULT, ui32PolicyVMRA;
+
+	IMG_CHAR aszBuf[100];
+	IMG_CHAR *pszStr;
+	IMG_UINT32 ui32pszStrSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsHeapPtr, "ppsHeapPtr");
+
+	/* Reserved VA space of a heap must always be multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY.
+	 * Granularity has been chosen to support the max possible practically used OS page size. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uiReservedRegionLength % DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY) == 0, "uiReservedRegionLength");
+
+	ui32PolicyVMRA = RA_POLICY_DEFAULT;
+
+	PVR_ASSERT(uiReservedRegionLength + DEVMEM_HEAP_MINIMUM_SIZE <= uiLength);
+
+	psHeap = OSAllocMem(sizeof(*psHeap));
+	PVR_GOTO_IF_NOMEM(psHeap, eError, HeapAllocError);
+
+	/* Need to keep local copy of heap name, so caller may free theirs */
+	ui32pszStrSize = OSStringLength(pszName) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	PVR_GOTO_IF_NOMEM(pszStr, eError, SubAllocRANameCopyError);
+	OSStringSafeCopy(pszStr, pszName, ui32pszStrSize);
+	psHeap->pszName = pszStr;
+
+	psHeap->uiSize = uiLength;
+	psHeap->uiReservedRegionSize = uiReservedRegionLength;
+	psHeap->sBaseAddress = sBaseAddress;
+	psHeap->bPremapped = IMG_FALSE;
+	OSAtomicWrite(&psHeap->hImportCount, 0);
+
+	OSSNPrintf(aszBuf, sizeof(aszBuf),
+			"NDM heap '%s' (suballocs) ctx:%p",
+			pszName, psCtx);
+	ui32pszStrSize = OSStringLength(aszBuf) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	PVR_GOTO_IF_NOMEM(pszStr, eError, SubAllocRANameError);
+	OSStringSafeCopy(pszStr, aszBuf, ui32pszStrSize);
+	psHeap->pszSubAllocRAName = pszStr;
+
+#if defined(__KERNEL__)
+	if (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW)
+	{
+		void *pvAppHintState = NULL;
+		IMG_UINT32 ui32FirmwarePolicydefault = PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY;
+		IMG_UINT32 ui32FirmwarePolicy = PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY;
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, DevMemFWHeapPolicy,
+						&ui32FirmwarePolicydefault, &ui32FirmwarePolicy);
+		ui32PolicyVMRA = ui32Policy = ui32FirmwarePolicy;
+		OSFreeAppHintState(pvAppHintState);
+
+		/* Flag the change from default setting */
+		if (ui32FirmwarePolicy != ui32FirmwarePolicydefault)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: %s: DevMemFWHeapPolicy set to %u, default %u",
+			         __func__, pszStr,
+			         ui32FirmwarePolicy, ui32FirmwarePolicydefault));
+		}
+	}
+#endif
+
+#if defined(PDUMP)
+	/* The META heap is shared globally so a single physical memory import
+	 * may be used to satisfy allocations of different processes.
+	 * This is problematic when PDumping because the physical memory
+	 * import used to satisfy a new allocation may actually have been
+	 * imported (and thus the PDump MALLOC generated) before the PDump
+	 * client was started, leading to the MALLOC being missing.
+	 *
+	 * This is solved by disabling splitting of imports for the META physmem
+	 * RA, meaning that every firmware allocation gets its own import, thus
+	 * ensuring the MALLOC is present for every allocation made within the
+	 * pdump capture range
+	 */
+	if (uiHeapBlueprintID == DEVMEM_HEAPCFG_FORFW)
+	{
+		ui32Policy |= RA_POLICY_NO_SPLIT;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiHeapBlueprintID);
+#endif
+
+	psHeap->psSubAllocRA = RA_Create(psHeap->pszSubAllocRAName,
+			/* Subsequent imports: */
+			ui32Log2Quantum,
+			RA_LOCKCLASS_3,
+			SubAllocImportAlloc,
+			SubAllocImportFree,
+			(RA_PERARENA_HANDLE) psHeap,
+			ui32Policy);
+	if (psHeap->psSubAllocRA == NULL)
+	{
+		eError = PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA;
+		goto SubAllocRACreateError;
+	}
+
+	psHeap->uiLog2ImportAlignment = ui32Log2ImportAlignment;
+	psHeap->uiLog2Quantum = ui32Log2Quantum;
+
+	if (!OSStringNCompare(pszName, RGX_GENERAL_SVM_HEAP_IDENT, sizeof(RGX_GENERAL_SVM_HEAP_IDENT)))
+	{
+		/* The SVM heap normally starts out as this type though
+		   it may transition to DEVMEM_HEAP_MANAGER_USER
+		   on platforms with more processor virtual address
+		   bits than device virtual address bits */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_KERNEL;
+	}
+	else if (uiReservedRegionLength != 0)
+	{
+		/* Heaps which specify reserved VA space range are dual managed:
+		 * - sBaseAddress to (sBaseAddress+uiReservedRegionLength-1): User managed
+		 * - (sBaseAddress+uiReservedRegionLength) to uiLength: RA managed
+		 */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_DUAL_USER_RA;
+	}
+	else
+	{
+		/* Otherwise, heap manager is decided (USER or RA) at first map */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_UNKNOWN;
+	}
+
+	/* Mark the heap to be managed by RA */
+	if (!OSStringNCompare(pszName, RGX_VK_CAPT_REPLAY_HEAP_IDENT,
+	                      sizeof(RGX_VK_CAPT_REPLAY_HEAP_IDENT)))
+	{
+		psHeap->ui32HeapManagerFlags |= DEVMEM_HEAP_MANAGER_RA;
+	}
+
+	OSSNPrintf(aszBuf, sizeof(aszBuf),
+			"NDM heap '%s' (QVM) ctx:%p",
+			pszName, psCtx);
+	ui32pszStrSize = OSStringLength(aszBuf) + 1;
+	pszStr = OSAllocMem(ui32pszStrSize);
+	if (pszStr == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto VMRANameError;
+	}
+	OSStringSafeCopy(pszStr, aszBuf, ui32pszStrSize);
+	psHeap->pszQuantizedVMRAName = pszStr;
+
+	psHeap->psQuantizedVMRA = RA_Create(psHeap->pszQuantizedVMRAName,
+			/* Subsequent import: */
+			0, RA_LOCKCLASS_1, NULL, NULL,
+			(RA_PERARENA_HANDLE) psHeap,
+			ui32PolicyVMRA);
+	if (psHeap->psQuantizedVMRA == NULL)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA, VMRACreateError);
+	}
+
+	if (!RA_Add(psHeap->psQuantizedVMRA,
+			/* Make sure the VMRA doesn't allocate from reserved VAs */
+			(RA_BASE_T)sBaseAddress.uiAddr + uiReservedRegionLength,
+			(RA_LENGTH_T)uiLength,
+			(RA_FLAGS_T)0, /* This RA doesn't use or need flags */
+			NULL /* per ispan handle */))
+	{
+		RA_Delete(psHeap->psQuantizedVMRA);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA, VMRACreateError);
+	}
+
+	psHeap->psCtx = psCtx;
+
+
+	/* Create server-side counterpart of Device Memory heap */
+	eError = BridgeDevmemIntHeapCreate(GetBridgeHandle(psCtx->hDevConnection),
+	                                   psCtx->hDevMemServerContext,
+	                                   uiHeapBlueprintID,
+	                                   uiHeapIndex,
+	                                   &hDevMemServerHeap);
+	PVR_GOTO_IF_ERROR(eError, ServerHeapCreateError);
+
+	psHeap->hDevMemServerHeap = hDevMemServerHeap;
+
+	eError = OSLockCreate(&psHeap->hLock);
+	PVR_GOTO_IF_ERROR(eError, LockCreateErr);
+
+	psHeap->psCtx->uiNumHeaps++;
+	*ppsHeapPtr = psHeap;
+
+	return PVRSRV_OK;
+
+	/* error exit paths */
+LockCreateErr:
+	eError2 = BridgeDevmemIntHeapDestroy(GetBridgeHandle(psCtx->hDevConnection),
+			psHeap->hDevMemServerHeap);
+	PVR_ASSERT (eError2 == PVRSRV_OK);
+ServerHeapCreateError:
+	RA_Delete(psHeap->psQuantizedVMRA);
+VMRACreateError:
+	OSFreeMem(psHeap->pszQuantizedVMRAName);
+VMRANameError:
+	RA_Delete(psHeap->psSubAllocRA);
+SubAllocRACreateError:
+	OSFreeMem(psHeap->pszSubAllocRAName);
+SubAllocRANameError:
+	OSFreeMem(psHeap->pszName);
+SubAllocRANameCopyError:
+	OSFreeMem(psHeap);
+HeapAllocError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapBaseDevVAddr(struct DEVMEM_HEAP_TAG *psHeap,
+		IMG_DEV_VIRTADDR *pDevVAddr)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+
+	*pDevVAddr = psHeap->sBaseAddress;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL DEVMEM_SIZE_T
+DevmemGetHeapSize(struct DEVMEM_HEAP_TAG *psHeap)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psHeap != NULL);
+
+	return psHeap->uiSize;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum,
+		IMG_DEVMEM_SIZE_T *puiSize,
+		IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	IMG_DEVMEM_SIZE_T uiSize = *puiSize;
+	IMG_DEVMEM_ALIGN_T uiAlign = *puiAlign;
+
+	/* Just in case someone changes definition of IMG_DEVMEM_ALIGN_T. */
+	static_assert(sizeof(unsigned long long) == sizeof(uiAlign),
+	              "invalid uiAlign size");
+	/* This value is used for shifting so it cannot be greater than number
+	 * of bits in unsigned long long (sizeof(1ULL)). Using greater value is
+	 * undefined behaviour. */
+	if (uiLog2Quantum >= sizeof(unsigned long long) * 8)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((1ULL << uiLog2Quantum) > uiAlign)
+	{
+		uiAlign = 1ULL << uiLog2Quantum;
+	}
+	uiSize = PVR_ALIGN(uiSize, uiAlign);
+
+	*puiSize = uiSize;
+	*puiAlign = uiAlign;
+
+	return PVRSRV_OK;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemDestroyHeap(DEVMEM_HEAP *psHeap)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT uiImportCount;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__)
+	if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+#endif
+
+	PVR_RETURN_IF_INVALID_PARAM(psHeap);
+
+	uiImportCount = OSAtomicRead(&psHeap->hImportCount);
+	if (uiImportCount > 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%d(%s) leaks remain", uiImportCount, psHeap->pszName));
+#if defined(__KERNEL__)
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		PVR_DPF((PVR_DBG_ERROR, "Details of remaining allocated device memory (for all processes):"));
+		RIDumpAllKM();
+#else
+		PVR_DPF((PVR_DBG_ERROR, "Compile with PVRSRV_ENABLE_GPU_MEMORY_INFO=1 to get a full "
+				"list of all driver allocations."));
+#endif
+#endif
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		if (bDoCheck)
+#endif
+		{
+			return PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP;
+		}
+	}
+
+	eError = DestroyServerResource(psHeap->psCtx->hDevConnection,
+	                               NULL,
+	                               BridgeDevmemIntHeapDestroy,
+	                               psHeap->hDevMemServerHeap);
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (bDoCheck)
+#endif
+	{
+		PVR_LOG_RETURN_IF_ERROR(eError, "BridgeDevmemIntHeapDestroy");
+	}
+
+	PVR_ASSERT(psHeap->psCtx->uiNumHeaps > 0);
+	psHeap->psCtx->uiNumHeaps--;
+
+	OSLockDestroy(psHeap->hLock);
+
+	if (psHeap->psQuantizedVMRA)
+	{
+		RA_Delete(psHeap->psQuantizedVMRA);
+	}
+	if (psHeap->pszQuantizedVMRAName)
+	{
+		OSFreeMem(psHeap->pszQuantizedVMRAName);
+	}
+
+	RA_Delete(psHeap->psSubAllocRA);
+	OSFreeMem(psHeap->pszSubAllocRAName);
+
+	OSFreeMem(psHeap->pszName);
+
+	OSCachedMemSet(psHeap, 0, sizeof(*psHeap));
+	OSFreeMem(psHeap);
+
+	return PVRSRV_OK;
+}
+
+/*****************************************************************************
+ *                Devmem allocation/free functions                           *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemSubAllocateAndMap(IMG_UINT8 uiPreAllocMultiplier,
+			DEVMEM_HEAP *psHeap,
+			IMG_DEVMEM_SIZE_T uiSize,
+			IMG_DEVMEM_ALIGN_T uiAlign,
+			PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			const IMG_CHAR *pszText,
+			DEVMEM_MEMDESC **ppsMemDescPtr,
+			IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemSubAllocate(uiPreAllocMultiplier,
+				psHeap,
+				uiSize,
+				uiAlign,
+				uiFlags,
+				pszText,
+				ppsMemDescPtr);
+	PVR_GOTO_IF_ERROR(eError, fail_alloc);
+
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+				psHeap,
+				psDevVirtAddr);
+	PVR_GOTO_IF_ERROR(eError, fail_map);
+
+	return PVRSRV_OK;
+
+fail_map:
+	DevmemFree(*ppsMemDescPtr);
+fail_alloc:
+	*ppsMemDescPtr = NULL;
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	RA_BASE_T uiAllocatedAddr = 0;
+	RA_LENGTH_T uiAllocatedSize;
+	RA_PERISPAN_HANDLE hImport; /* the "import" from which this sub-allocation came */
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	IMG_DEVMEM_OFFSET_T uiOffset = 0;
+	DEVMEM_IMPORT *psImport;
+	IMG_UINT32 ui32CPUCacheLineSize;
+	void *pvAddr = NULL;
+
+	IMG_BOOL bImportClean;
+	IMG_BOOL bCPUCleanFlag = PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags);
+	IMG_BOOL bZero = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags);
+	IMG_BOOL bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags);
+	IMG_BOOL bCPUCached = (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+			PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags));
+	IMG_BOOL bGPUCached = (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) ||
+			PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags));
+	IMG_BOOL bAlign = ! (PVRSRV_CHECK_NO_CACHE_LINE_ALIGN(uiFlags));
+	PVRSRV_CACHE_OP eOp = PVRSRV_CACHE_OP_INVALIDATE;
+	IMG_UINT32	ui32CacheLineSize = 0;
+	DEVMEM_PROPERTIES_T uiProperties;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+		bPoisonOnAlloc = IMG_TRUE;
+	}
+#endif
+
+	if (uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		/* Deferred Allocation not supported on SubAllocs*/
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failParams);
+	}
+
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap->psCtx, eError, failParams);
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(psHeap->psCtx->hDevConnection, uiFlags);
+
+#if defined(__KERNEL__)
+	{
+		/* The hDevConnection holds two different types of pointers depending on the
+		 * address space in which it is used.
+		 * In this instance the variable points to the device node in server */
+		PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psHeap->psCtx->hDevConnection;
+		ui32CacheLineSize = GET_ROGUE_CACHE_LINE_SIZE(PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, SLC_CACHE_LINE_SIZE_BITS));
+	}
+#else
+	ui32CacheLineSize = ROGUE_CACHE_LINE_SIZE;
+#endif
+
+	/* The following logic makes sure that any cached memory is aligned to both the CPU and GPU.
+	 * To be aligned on both you have to take the Lowest Common Multiple (LCM) of the cache line sizes of each.
+	 * As the possibilities are all powers of 2 then simply the largest number can be picked as the LCM.
+	 * Therefore this algorithm just picks the highest from the CPU, GPU and given alignments.
+	 */
+	ui32CPUCacheLineSize = psHeap->psCtx->ui32CPUCacheLineSize;
+	/* If the CPU cache line size is larger than the alignment given then it is the lowest common multiple
+	 * Also checking if the allocation is going to be cached on the CPU
+	 * Currently there is no check for the validity of the cache coherent option.
+	 * In this case, the alignment could be applied but the mode could still fall back to uncached.
+	 */
+	if (bAlign && ui32CPUCacheLineSize > uiAlign && bCPUCached)
+	{
+		uiAlign = ui32CPUCacheLineSize;
+	}
+
+	/* If the GPU cache line size is larger than the alignment given then it is the lowest common multiple
+	 * Also checking if the allocation is going to be cached on the GPU via checking for any of the cached options.
+	 * Currently there is no check for the validity of the cache coherent option.
+	 * In this case, the alignment could be applied but the mode could still fall back to uncached.
+	 */
+	if (bAlign && ui32CacheLineSize > uiAlign && bGPUCached)
+	{
+		uiAlign = ui32CacheLineSize;
+	}
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	/* No request for exportable memory so use the RA */
+	eError = RA_Alloc(psHeap->psSubAllocRA,
+			uiSize,
+			uiPreAllocMultiplier,
+			uiFlags,
+			uiAlign,
+			pszText,
+			&uiAllocatedAddr,
+			&uiAllocatedSize,
+			&hImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	psImport = hImport;
+
+	/* This assignment is assuming the RA returns an hImport where suballocations
+	 * can be made from if uiSize is NOT a page multiple of the passed heap.
+	 *
+	 * So we check if uiSize is a page multiple and mark it as exportable
+	 * if it is not.
+	 * */
+	OSLockAcquire(psImport->hLock);
+	if (!(uiSize & (IMG_PAGE2BYTES64(psHeap->uiLog2Quantum) - 1)) &&
+	    (uiPreAllocMultiplier == RA_NO_IMPORT_MULTIPLIER))
+	{
+		psImport->uiProperties |= DEVMEM_PROPERTIES_EXPORTABLE;
+	}
+	psImport->uiProperties |= DEVMEM_PROPERTIES_SUBALLOCATABLE;
+	uiProperties = psImport->uiProperties;
+	OSLockRelease(psImport->hLock);
+
+	uiOffset = uiAllocatedAddr - psImport->sDeviceImport.sDevVAddr.uiAddr;
+
+#if defined(PDUMP) && defined(DEBUG)
+#if defined(__KERNEL__)
+	PDUMPCOMMENTWITHFLAGS(PMR_DeviceNode((PMR*)psImport->hPMR), PDUMP_CONT,
+			"Suballocated %u Byte for \"%s\" from PMR with handle ID: 0x%p (PID %u)",
+			(IMG_UINT32) uiSize, pszText, psImport->hPMR, OSGetCurrentProcessID());
+#else
+	PDUMPCOMMENTF(psHeap->psCtx->hDevConnection, PDUMP_FLAGS_CONTINUOUS,
+			"Suballocated %u Byte for \"%s\" from PMR with handle ID: %p (PID %u)",
+			(IMG_UINT32) uiSize,
+			pszText,
+			psImport->hPMR,
+			OSGetCurrentProcessID());
+#endif
+#endif
+
+	DevmemMemDescInit(psMemDesc,
+			uiOffset,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	bImportClean = ((uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_CLEAN) != 0);
+
+	/* Zero the memory */
+	if (bZero)
+	{
+		/* Has the import been zeroed on allocation and were no suballocations returned to it so far? */
+		bImportClean = bImportClean && ((uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_ZEROED) != 0);
+
+		if (!bImportClean)
+		{
+			eOp = PVRSRV_CACHE_OP_FLUSH;
+
+			eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
+			PVR_GOTO_IF_ERROR(eError, failMaintenance);
+
+			/* uiSize is a 64-bit quantity whereas the 3rd argument
+			 * to OSDeviceMemSet is a 32-bit quantity on 32-bit systems
+			 * hence a compiler warning of implicit cast and loss of data.
+			 * Added explicit cast and assert to remove warning.
+			 */
+			PVR_ASSERT(uiSize < IMG_UINT32_MAX);
+
+			DevmemCPUMemSet(pvAddr, 0, uiSize, uiFlags);
+
+#if defined(PDUMP)
+			DevmemPDumpLoadZeroMem(psMemDesc, 0, uiSize, PDUMP_FLAGS_CONTINUOUS);
+#endif
+		}
+	}
+	else if (bPoisonOnAlloc)
+	{
+		/* Has the import been poisoned on allocation and were no suballocations returned to it so far? */
+		bPoisonOnAlloc = (uiProperties & DEVMEM_PROPERTIES_IMPORT_IS_POISONED) != 0;
+
+		if (!bPoisonOnAlloc)
+		{
+			eOp = PVRSRV_CACHE_OP_FLUSH;
+
+			eError = DevmemAcquireCpuVirtAddr(psMemDesc, &pvAddr);
+			PVR_GOTO_IF_ERROR(eError, failMaintenance);
+
+			DevmemCPUMemSet(pvAddr, PVRSRV_POISON_ON_ALLOC_VALUE, uiSize, uiFlags);
+
+			bPoisonOnAlloc = IMG_TRUE;
+		}
+	}
+
+	/* Flush or invalidate */
+	if (bCPUCached && !bImportClean && (bZero || bCPUCleanFlag || bPoisonOnAlloc))
+	{
+		eError = BridgeCacheOpExec (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				(IMG_UINT64)(uintptr_t)
+				pvAddr - psMemDesc->uiOffset,
+				psMemDesc->uiOffset,
+				psMemDesc->uiAllocSize,
+				eOp);
+		PVR_GOTO_IF_ERROR(eError, failMaintenance);
+	}
+
+	if (pvAddr)
+	{
+		DevmemReleaseCpuVirtAddr(psMemDesc);
+		pvAddr = NULL;
+	}
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				OSStringNLength(psMemDesc->szText, DEVMEM_ANNOTATION_MAX_LEN),
+				psMemDesc->szText,
+				psMemDesc->uiOffset,
+				uiAllocatedSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_SUBALLOC,
+				&(psMemDesc->hRIHandle));
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+			psMemDesc->hRIHandle = NULL;
+		}
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	*ppsMemDescPtr = psMemDesc;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failMaintenance:
+	if (pvAddr)
+	{
+		DevmemReleaseCpuVirtAddr(psMemDesc);
+		pvAddr = NULL;
+	}
+	DevmemMemDescRelease(psMemDesc);
+	psMemDesc = NULL;	/* Make sure we don't do a discard after the release */
+failDeviceMemAlloc:
+	if (psMemDesc)
+	{
+		DevmemMemDescDiscard(psMemDesc);
+	}
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		IMG_UINT32 uiLog2HeapPageSize,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_UINT32 ui32MappingTable = 0;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+	}
+#endif
+
+	eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(hDevConnection, uiFlags);
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = AllocateDeviceMemory(hDevConnection,
+			uiLog2HeapPageSize,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			uiAlign,
+			uiFlags,
+			IMG_TRUE,
+			pszText,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	*ppsMemDescPtr = psMemDesc;
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				uiSize,
+				uiFlags,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failDeviceMemAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_UINT32 ui32NumPhysChunks,
+		IMG_UINT32 ui32NumVirtChunks,
+		IMG_UINT32 *pui32MappingTable,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		IMG_UINT32 uiLog2HeapPageSize,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		const IMG_CHAR *pszText,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+
+/* On nohw PDump builds, we try to minimise the amount
+ * of uninitialised data in captures.
+ */
+#if defined(PDUMP) && defined(NO_HARDWARE)
+	if (PVRSRV_CHECK_CPU_WRITEABLE(uiFlags) && !PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC;
+	}
+#endif
+
+	eError = DevmemExportalignAdjustSizeAndAlign(uiLog2HeapPageSize,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	uiFlags = DevmemOverrideFlagsOrPassThrough(hDevConnection, uiFlags);
+
+	eError = DevmemValidateParams(uiSize,
+			uiAlign,
+			&uiFlags);
+	PVR_GOTO_IF_ERROR(eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = AllocateDeviceMemory(hDevConnection,
+			uiLog2HeapPageSize,
+			uiSize,
+			ui32NumPhysChunks,
+			ui32NumVirtChunks,
+			pui32MappingTable,
+			uiAlign,
+			uiFlags,
+			IMG_TRUE,
+			pszText,
+			&psImport);
+	PVR_GOTO_IF_ERROR(eError, failDeviceMemAlloc);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+#if defined(DEBUG)
+	DevmemMemDescSetPoF(psMemDesc, uiFlags);
+#endif
+
+	/* copy the allocation descriptive name and size so it can be passed to DevicememHistory when
+	 * the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszText);
+	OSStringSafeCopy(psMemDesc->szText, pszText, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		eError = BridgeRIWritePMREntry (GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWritePMREntry");
+
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				uiSize,
+				uiFlags,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#else /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+	PVR_UNREFERENCED_PARAMETER (pszText);
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	*ppsMemDescPtr = psMemDesc;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failDeviceMemAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed! Error is %s. Allocation size: " IMG_DEVMEM_SIZE_FMTSPEC,
+			__func__,
+			PVRSRVGETERRORSTRING(eError),
+			uiSize));
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hServerHandle,
+		IMG_HANDLE *hLocalImportHandle)
+{
+	return BridgePMRMakeLocalImportHandle(GetBridgeHandle(hDevConnection),
+			hServerHandle,
+			hLocalImportHandle);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hLocalImportHandle)
+{
+	return DestroyServerResource(hDevConnection,
+	                             NULL,
+	                             BridgePMRUnmakeLocalImportHandle,
+	                             hLocalImportHandle);
+}
+
+/*****************************************************************************
+ *                Devmem unsecure export functions                           *
+ *****************************************************************************/
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static PVRSRV_ERROR
+_Mapping_Export(DEVMEM_IMPORT *psImport,
+		DEVMEM_EXPORTHANDLE *phPMRExportHandlePtr,
+		DEVMEM_EXPORTKEY *puiExportKeyPtr,
+		DEVMEM_SIZE_T *puiSize,
+		DEVMEM_LOG2ALIGN_T *puiLog2Contig)
+{
+	/* Gets an export handle and key for the PMR used for this mapping */
+	/* Can only be done if there are no suballocations for this mapping */
+
+	PVRSRV_ERROR eError;
+	DEVMEM_EXPORTHANDLE hPMRExportHandle;
+	DEVMEM_EXPORTKEY uiExportKey;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig;
+
+	PVR_GOTO_IF_INVALID_PARAM(psImport, eError, failParams);
+
+	if ((GetImportProperties(psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION, failParams);
+	}
+
+	eError = BridgePMRExportPMR(GetBridgeHandle(psImport->hDevConnection),
+			psImport->hPMR,
+			&hPMRExportHandle,
+			&uiSize,
+			&uiLog2Contig,
+			&uiExportKey);
+	PVR_GOTO_IF_ERROR(eError, failExport);
+
+	PVR_ASSERT(uiSize == psImport->uiSize);
+
+	*phPMRExportHandlePtr = hPMRExportHandle;
+	*puiExportKeyPtr = uiExportKey;
+	*puiSize = uiSize;
+	*puiLog2Contig = uiLog2Contig;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failExport:
+failParams:
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static void
+_Mapping_Unexport(DEVMEM_IMPORT *psImport,
+		DEVMEM_EXPORTHANDLE hPMRExportHandle)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT (psImport != NULL);
+
+	eError = DestroyServerResource(psImport->hDevConnection,
+	                               NULL,
+	                               BridgePMRUnexportPMR,
+	                               hPMRExportHandle);
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExport(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_EXPORTCOOKIE *psExportCookie)
+{
+	/* Caller to provide storage for export cookie struct */
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hPMRExportHandle = 0;
+	IMG_UINT64 uiPMRExportPassword = 0;
+	IMG_DEVMEM_SIZE_T uiSize = 0;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Contig = 0;
+
+	PVR_GOTO_IF_INVALID_PARAM(psMemDesc, eError, e0);
+	PVR_GOTO_IF_INVALID_PARAM(psExportCookie, eError, e0);
+
+	if (DEVMEM_PROPERTIES_EXPORTABLE !=
+			(GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This Memory (0x%p) cannot be exported!...",
+				__func__, psMemDesc));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_REQUEST, e0);
+	}
+
+	eError = _Mapping_Export(psMemDesc->psImport,
+			&hPMRExportHandle,
+			&uiPMRExportPassword,
+			&uiSize,
+			&uiLog2Contig);
+	if (eError != PVRSRV_OK)
+	{
+		psExportCookie->uiSize = 0;
+		goto e0;
+	}
+
+	psExportCookie->hPMRExportHandle = hPMRExportHandle;
+	psExportCookie->uiPMRExportPassword = uiPMRExportPassword;
+	psExportCookie->uiSize = uiSize;
+	psExportCookie->uiLog2ContiguityGuarantee = uiLog2Contig;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_EXPORTCOOKIE *psExportCookie)
+{
+	_Mapping_Unexport(psMemDesc->psImport,
+			psExportCookie->hPMRExportHandle);
+
+	psExportCookie->uiSize = 0;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemImport(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_EXPORTCOOKIE *psCookie,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, failImportAlloc);
+	}
+
+	/* Get a handle to the PMR (inc refcount) */
+	eError = BridgePMRImportPMR(GetBridgeHandle(hDevConnection),
+			psCookie->hPMRExportHandle,
+			psCookie->uiPMRExportPassword,
+			psCookie->uiSize, /* not trusted - just for validation */
+			psCookie->uiLog2ContiguityGuarantee, /* not trusted - just for validation */
+			&hPMR);
+	PVR_GOTO_IF_ERROR(eError, failImport);
+
+	DevmemImportStructInit(psImport,
+			psCookie->uiSize,
+			1ULL << psCookie->uiLog2ContiguityGuarantee,
+			uiFlags,
+			hPMR,
+			DEVMEM_PROPERTIES_IMPORTED |
+			DEVMEM_PROPERTIES_EXPORTABLE);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			psImport->uiSize);
+
+	*ppsMemDescPtr = psMemDesc;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				psMemDesc->psImport->uiSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_IMPORT | PVRSRV_MEMALLOCFLAG_RI_SUBALLOC,
+				&psMemDesc->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+
+failImport:
+	DevmemImportDiscard(psImport);
+failImportAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+/*****************************************************************************
+ *                   Common MemDesc functions                                *
+ *****************************************************************************/
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetSize(DEVMEM_MEMDESC *psMemDesc, IMG_DEVMEM_SIZE_T* puiSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	*puiSize = psMemDesc->uiAllocSize;
+
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemGetAnnotation(DEVMEM_MEMDESC *psMemDesc, IMG_CHAR **pszAnnotation)
+{
+	/*
+	 * It is expected that psMemDesc->szText is a valid NUL-terminated string,
+	 * since DevmemMemDescAlloc uses OSAllocZMem to create the memdesc.
+	 */
+	*pszAnnotation = psMemDesc->szText;
+}
+
+/*
+	This function is called for freeing any class of memory
+ */
+IMG_INTERNAL IMG_BOOL
+DevmemFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_SECURE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Please use methods dedicated to secure buffers.",
+				__func__));
+		return IMG_FALSE;
+	}
+
+	return DevmemMemDescRelease(psMemDesc);
+}
+
+IMG_INTERNAL IMG_BOOL
+DevmemReleaseDevAddrAndFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	DevmemReleaseDevVirtAddr(psMemDesc);
+	return DevmemFree(psMemDesc);
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	DEVMEM_IMPORT *psImport;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bMap = IMG_TRUE;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+	IMG_UINT64 ui64OptionalMapAddress = DEVICEMEM_UTILS_NO_ADDRESS;
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED, failCheck);
+	}
+
+	/* Don't map memory for deferred allocations */
+	if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		PVR_ASSERT(uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
+		bMap = IMG_FALSE;
+	}
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	psImport = psMemDesc->psImport;
+	DevmemMemDescAcquire(psMemDesc);
+
+#if defined(__KERNEL__)
+	if (psHeap->bPremapped)
+	{
+		ui64OptionalMapAddress = _GetPremappedVA(psImport->hPMR, psHeap->psCtx->hDevConnection);
+	}
+#endif
+
+	eError = DevmemImportStructDevMap(psHeap,
+			bMap,
+			psImport,
+			ui64OptionalMapAddress);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	sDevVAddr.uiAddr += psMemDesc->uiOffset;
+	psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
+
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				psMemDesc->uiOffset,
+				psMemDesc->sDeviceMemDesc.sDevVAddr,
+				psMemDesc->uiAllocSize,
+				psMemDesc->szText,
+				DevmemGetHeapLog2PageSize(psHeap),
+				psMemDesc->ui32AllocationIndex,
+				&psMemDesc->ui32AllocationIndex);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		if (psMemDesc->hRIHandle)
+		{
+			eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection),
+					psMemDesc->hRIHandle,
+					psImport->sDeviceImport.sDevVAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIUpdateMEMDESCAddr");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failMap:
+	bDestroyed = DevmemMemDescRelease(psMemDesc);
+failCheck:
+failParams:
+	if (!bDestroyed)
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc,
+		DEVMEM_HEAP *psHeap,
+		IMG_DEV_VIRTADDR sDevVirtAddr)
+{
+	DEVMEM_IMPORT *psImport;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bMap = IMG_TRUE;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_INVALID_PARAM(psHeap, eError, failParams);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount != 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED, failCheck);
+	}
+
+	/* Don't map memory for deferred allocations */
+	if (psMemDesc->psImport->uiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC)
+	{
+		PVR_ASSERT(uiProperties & DEVMEM_PROPERTIES_EXPORTABLE);
+		bMap = IMG_FALSE;
+	}
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	psImport = psMemDesc->psImport;
+	DevmemMemDescAcquire(psMemDesc);
+
+	eError = DevmemImportStructDevMap(psHeap,
+			bMap,
+			psImport,
+			sDevVirtAddr.uiAddr);
+	PVR_GOTO_IF_ERROR(eError, failMap);
+
+	sDevVAddr.uiAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	sDevVAddr.uiAddr += psMemDesc->uiOffset;
+	psMemDesc->sDeviceMemDesc.sDevVAddr = sDevVAddr;
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		BridgeDevicememHistoryMap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				psMemDesc->uiOffset,
+				psMemDesc->sDeviceMemDesc.sDevVAddr,
+				psMemDesc->uiAllocSize,
+				psMemDesc->szText,
+				DevmemGetHeapLog2PageSize(psHeap),
+				psMemDesc->ui32AllocationIndex,
+				&psMemDesc->ui32AllocationIndex);
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		if (psMemDesc->hRIHandle)
+		{
+			eError = BridgeRIUpdateMEMDESCAddr(GetBridgeHandle(psImport->hDevConnection),
+					psMemDesc->hRIHandle,
+					psImport->sDeviceImport.sDevVAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIUpdateMEMDESCAddr");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failMap:
+	bDestroyed = DevmemMemDescRelease(psMemDesc);
+failCheck:
+failParams:
+	if (!bDestroyed)
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL IMG_DEV_VIRTADDR
+DevmemGetDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_DEVICEMEM_NO_MAPPING, "DevmemGetDevVirtAddr");
+	}
+
+	PVR_ASSERT(psMemDesc->sDeviceMemDesc.sDevVAddr.uiAddr !=0 );
+
+	return psMemDesc->sDeviceMemDesc.sDevVAddr;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		IMG_DEV_VIRTADDR *psDevVirtAddr)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount+1);
+
+	if (psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_NO_MAPPING, failRelease);
+	}
+	psMemDesc->sDeviceMemDesc.ui32RefCount++;
+
+	*psDevVirtAddr = psMemDesc->sDeviceMemDesc.sDevVAddr;
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+	return PVRSRV_OK;
+
+failRelease:
+	OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+
+	OSLockAcquire(psMemDesc->sDeviceMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sDeviceMemDesc.ui32RefCount,
+			psMemDesc->sDeviceMemDesc.ui32RefCount-1);
+
+	PVR_ASSERT(psMemDesc->sDeviceMemDesc.ui32RefCount != 0);
+
+	if (--psMemDesc->sDeviceMemDesc.ui32RefCount == 0)
+	{
+		if (GetInfoPageDebugFlags(psMemDesc->psImport->hDevConnection) & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+		{
+			BridgeDevicememHistoryUnmap(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+					psMemDesc->psImport->hPMR,
+					psMemDesc->uiOffset,
+					psMemDesc->sDeviceMemDesc.sDevVAddr,
+					psMemDesc->uiAllocSize,
+					psMemDesc->szText,
+					DevmemGetHeapLog2PageSize(psMemDesc->psImport->sDeviceImport.psHeap),
+					psMemDesc->ui32AllocationIndex,
+					&psMemDesc->ui32AllocationIndex);
+		}
+
+		/* When device mapping destroyed, zero Dev VA so DevmemGetDevVirtAddr()
+		 * returns 0 */
+		if (DevmemImportStructDevUnmap(psMemDesc->psImport) == IMG_TRUE)
+		{
+			psMemDesc->sDeviceMemDesc.sDevVAddr.uiAddr = 0;
+		}
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+
+		DevmemMemDescRelease(psMemDesc);
+	}
+	else
+	{
+		OSLockRelease(psMemDesc->sDeviceMemDesc.hLock);
+	}
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		void **ppvCpuVirtAddr)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(ppvCpuVirtAddr != NULL);
+
+	eError = DevmemCPUMapCheckImportProperties(psMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemCPUMapCheckImportProperties");
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount+1);
+
+	if (psMemDesc->sCPUMemDesc.ui32RefCount++ == 0)
+	{
+		DEVMEM_IMPORT *psImport = psMemDesc->psImport;
+		IMG_UINT8 *pui8CPUVAddr;
+
+		DevmemMemDescAcquire(psMemDesc);
+		eError = DevmemImportStructCPUMap(psImport);
+		PVR_GOTO_IF_ERROR(eError, failMap);
+
+		pui8CPUVAddr = psImport->sCPUImport.pvCPUVAddr;
+		pui8CPUVAddr += psMemDesc->uiOffset;
+		psMemDesc->sCPUMemDesc.pvCPUVAddr = pui8CPUVAddr;
+	}
+	*ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
+
+	VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
+
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+
+	return PVRSRV_OK;
+
+failMap:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	psMemDesc->sCPUMemDesc.ui32RefCount--;
+
+	if (!DevmemMemDescRelease(psMemDesc))
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	}
+	return eError;
+}
+
+IMG_INTERNAL void
+DevmemReacquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+		void **ppvCpuVirtAddr)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+	PVR_ASSERT(ppvCpuVirtAddr != NULL);
+
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU UnMapping is not possible on this allocation!",
+				__func__));
+		return;
+	}
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount+1);
+
+	*ppvCpuVirtAddr = NULL;
+	if (psMemDesc->sCPUMemDesc.ui32RefCount)
+	{
+		*ppvCpuVirtAddr = psMemDesc->sCPUMemDesc.pvCPUVAddr;
+		psMemDesc->sCPUMemDesc.ui32RefCount += 1;
+	}
+
+	VG_MARK_INITIALIZED(*ppvCpuVirtAddr, psMemDesc->psImport->uiSize);
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+}
+
+IMG_INTERNAL void
+DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(psMemDesc != NULL);
+
+	if (GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU UnMapping is not possible on this allocation!",
+				__func__));
+		return;
+	}
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			psMemDesc->sCPUMemDesc.ui32RefCount,
+			psMemDesc->sCPUMemDesc.ui32RefCount-1);
+
+	PVR_ASSERT(psMemDesc->sCPUMemDesc.ui32RefCount != 0);
+
+	if (--psMemDesc->sCPUMemDesc.ui32RefCount == 0)
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+		DevmemImportStructCPUUnmap(psMemDesc->psImport);
+		DevmemMemDescRelease(psMemDesc);
+	}
+	else
+	{
+		OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	}
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *phImport)
+{
+	if ((GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION;
+	}
+
+	*phImport = psMemDesc->psImport->hPMR;
+
+	return PVRSRV_OK;
+}
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
+		IMG_UINT64 *pui64UID)
+{
+	DEVMEM_IMPORT *psImport = psMemDesc->psImport;
+	PVRSRV_ERROR eError;
+
+	if (!(GetImportProperties(psImport) & (DEVMEM_PROPERTIES_IMPORTED |
+		                        DEVMEM_PROPERTIES_EXPORTABLE)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This Memory (0x%p) doesn't support the functionality requested...",
+				__func__, psMemDesc));
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+
+	eError = BridgePMRGetUID(GetBridgeHandle(psImport->hDevConnection),
+			psImport->hPMR,
+			pui64UID);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *hReservation)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*hReservation = psImport->sDeviceImport.hReservation;
+
+	return PVRSRV_OK;
+}
+
+#endif /* !__KERNEL__ */
+
+/* Kernel usage of this function will only work with
+ * memdescs of buffers allocated in the FW memory context
+ * that is created in the Server
+ */
+void
+DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *phPMR,
+		IMG_DEVMEM_OFFSET_T *puiPMROffset)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	*puiPMROffset = psMemDesc->uiOffset;
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*phPMR = psImport->hPMR;
+}
+
+#if defined(__KERNEL__)
+IMG_INTERNAL void
+DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
+		PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	DEVMEM_IMPORT *psImport;
+
+	PVR_ASSERT(psMemDesc);
+	psImport = psMemDesc->psImport;
+
+	PVR_ASSERT(psImport);
+	*puiFlags = psImport->uiFlags;
+}
+
+IMG_INTERNAL SHARED_DEV_CONNECTION
+DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc)
+{
+	return psMemDesc->psImport->hDevConnection;
+}
+#endif /* __KERNEL__ */
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemLocalImport(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hExtHandle,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		DEVMEM_MEMDESC **ppsMemDescPtr,
+		IMG_DEVMEM_SIZE_T *puiSizePtr,
+		const IMG_CHAR *pszAnnotation)
+{
+	DEVMEM_MEMDESC *psMemDesc = NULL;
+	DEVMEM_IMPORT *psImport;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_DEVMEM_ALIGN_T uiAlign;
+	IMG_HANDLE hPMR;
+	PVRSRV_ERROR eError;
+
+	PVR_GOTO_IF_INVALID_PARAM(ppsMemDescPtr, eError, failParams);
+
+	eError = DevmemMemDescAlloc(&psMemDesc);
+	PVR_GOTO_IF_ERROR(eError, failMemDescAlloc);
+
+	eError = DevmemImportStructAlloc(hDevConnection,
+			&psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, failImportAlloc);
+	}
+
+	/* Get the PMR handle and its size from the server */
+	eError = BridgePMRLocalImportPMR(GetBridgeHandle(hDevConnection),
+			hExtHandle,
+			&hPMR,
+			&uiSize,
+			&uiAlign);
+	PVR_GOTO_IF_ERROR(eError, failImport);
+
+	DevmemImportStructInit(psImport,
+			uiSize,
+			uiAlign,
+			uiFlags,
+			hPMR,
+			DEVMEM_PROPERTIES_IMPORTED |
+			DEVMEM_PROPERTIES_EXPORTABLE);
+
+	DevmemMemDescInit(psMemDesc,
+			0,
+			psImport,
+			uiSize);
+
+	*ppsMemDescPtr = psMemDesc;
+	if (puiSizePtr)
+		*puiSizePtr = uiSize;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI))
+	{
+		/* Attach RI information.
+		 * Set backed size to 0 since this allocation has been allocated
+		 * by the same process and has been accounted for. */
+		eError = BridgeRIWriteMEMDESCEntry (GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+				psMemDesc->psImport->hPMR,
+				sizeof("^"),
+				"^",
+				psMemDesc->uiOffset,
+				psMemDesc->psImport->uiSize,
+				uiFlags | PVRSRV_MEMALLOCFLAG_RI_IMPORT,
+				&(psMemDesc->hRIHandle));
+		PVR_LOG_IF_ERROR(eError, "BridgeRIWriteMEMDESCEntry");
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+
+	/* Copy the allocation descriptive name and size so it can be passed
+	 * to DevicememHistory when the allocation gets mapped/unmapped
+	 */
+	CheckAnnotationLength(pszAnnotation);
+	OSStringSafeCopy(psMemDesc->szText, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN);
+
+	return PVRSRV_OK;
+
+failImport:
+	DevmemImportDiscard(psImport);
+failImportAlloc:
+	DevmemMemDescDiscard(psMemDesc);
+failMemDescAlloc:
+failParams:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+#if !defined(__KERNEL__)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
+		IMG_DEV_VIRTADDR sDevVAddr)
+{
+	return BridgeDevmemIsVDevAddrValid(GetBridgeHandle(psContext->hDevConnection),
+	                                   psContext->hDevMemServerContext,
+	                                   sDevVAddr);
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetFaultAddress(DEVMEM_CONTEXT *psContext,
+		IMG_DEV_VIRTADDR *psFaultAddress)
+{
+	return BridgeDevmemGetFaultAddress(GetBridgeHandle(psContext->hDevConnection),
+	                                   psContext->hDevMemServerContext,
+	                                   psFaultAddress);
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+IMG_INTERNAL PVRSRV_ERROR
+DevmemInvalidateFBSCTable(DEVMEM_CONTEXT *psContext,
+                          IMG_UINT64 ui64FBSCEntries)
+{
+	return BridgeDevmemInvalidateFBSCTable(GetBridgeHandle(psContext->hDevConnection),
+	                                       psContext->hDevMemServerContext,
+	                                       ui64FBSCEntries);
+}
+#endif
+
+#endif /* !__KERNEL__ */
+
+IMG_INTERNAL IMG_UINT32
+DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap)
+{
+	return psHeap->uiLog2Quantum;
+}
+
+IMG_INTERNAL PVRSRV_MEMALLOCFLAGS_T
+DevmemGetMemAllocFlags(DEVMEM_MEMDESC *psMemDesc)
+{
+	return psMemDesc->psImport->uiFlags;
+}
+
+IMG_INTERNAL IMG_DEVMEM_SIZE_T
+DevmemGetHeapReservedSize(DEVMEM_HEAP *psHeap)
+{
+	return psHeap->uiReservedRegionSize;
+}
+
+#if !defined(__KERNEL__)
+/**************************************************************************/ /*!
+@Function       RegisterDevMemPFNotify
+@Description    Registers that the application wants to be signaled when a page
+                fault occurs.
+
+@Input          psContext      Memory context the process that would like to
+                               be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR:  PVRSRV_OK on success. Otherwise, a PVRSRV_
+                               error code
+ */ /***************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext,
+		IMG_BOOL       bRegister)
+{
+	PVRSRV_ERROR eError;
+
+	eError = BridgeDevmemIntRegisterPFNotifyKM(GetBridgeHandle(psContext->hDevConnection),
+			psContext->hDevMemServerContext,
+			bRegister);
+	if (eError == PVRSRV_ERROR_BRIDGE_CALL_FAILED)
+	{
+		PVR_LOG_ERROR(eError, "BridgeDevmemIntRegisterPFNotifyKM");
+	}
+
+	return eError;
+}
+#endif /* !__KERNEL__ */
+
+IMG_INTERNAL void
+DevmemHeapSetPremapStatus(DEVMEM_HEAP *psHeap, IMG_BOOL IsPremapped)
+{
+	psHeap->bPremapped = IsPremapped;
+}
+
+/******************************************************************************
+ End of file (devicemem.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/devicemem.h b/drivers/gpu/drm/img-rogue/devicemem.h
new file mode 100644
index 000000000000..0a7d550bf194
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem.h
@@ -0,0 +1,694 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management core internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to core device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVCLIENT_DEVICEMEM_H
+#define SRVCLIENT_DEVICEMEM_H
+
+/******************************************************************************
+ *                                                                            *
+ *  +------------+   +------------+      +--------------+   +--------------+  *
+ *  | a   sub-   |   | a   sub-   |      |  an          |   | allocation   |  *
+ *  | allocation |   | allocation |      |  allocation  |   | also mapped  |  *
+ *  |            |   |            |      |  in proc 1   |   | into proc 2  |  *
+ *  +------------+   +------------+      +--------------+   +--------------+  *
+ *            |         |                       |                  |          *
+ *         +--------------+              +--------------+   +--------------+  *
+ *         | page   gran- |              | page   gran- |   | page   gran- |  *
+ *         | ular mapping |              | ular mapping |   | ular mapping |  *
+ *         +--------------+              +--------------+   +--------------+  *
+ *                |                                   |       |               *
+ *                |                                   |       |               *
+ *                |                                   |       |               *
+ *         +--------------+                       +--------------+            *
+ *         |              |                       |              |            *
+ *         | A  "P.M.R."  |                       | A  "P.M.R."  |            *
+ *         |              |                       |              |            *
+ *         +--------------+                       +--------------+            *
+ *                                                                            *
+ ******************************************************************************/
+
+/*
+    All device memory allocations are ultimately a view upon (not
+    necessarily the whole of) a "PMR".
+
+    A PMR is a "Physical Memory Resource", which may be a
+    "pre-faulted" lump of physical memory, or it may be a
+    representation of some physical memory that will be instantiated
+    at some future time.
+
+    PMRs always represent multiple of some power-of-2 "contiguity"
+    promised by the PMR, which will allow them to be mapped in whole
+    pages into the device MMU.  As memory allocations may be smaller
+    than a page, these mappings may be suballocated and thus shared
+    between multiple allocations in one process.  A PMR may also be
+    mapped simultaneously into multiple device memory contexts
+    (cross-process scenario), however, for security reasons, it is not
+    legal to share a PMR "both ways" at once, that is, mapped into
+    multiple processes and divided up amongst several suballocations.
+
+    This PMR terminology is introduced here for background
+    information, but is generally of little concern to the caller of
+    this API.  This API handles suballocations and mappings, and the
+    caller thus deals primarily with MEMORY DESCRIPTORS representing
+    an allocation or suballocation, HEAPS representing ranges of
+    virtual addresses in a CONTEXT.
+*/
+
+/*
+   |<---------------------------context------------------------------>|
+   |<-------heap------->|   |<-------heap------->|<-------heap------->|
+   |<-alloc->|          |   |<-alloc->|<-alloc->||   |<-alloc->|      |
+*/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "devicemem_typedefs.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "pdump.h"
+
+#include "device_connection.h"
+
+
+typedef IMG_UINT32 DEVMEM_HEAPCFGID;
+#define DEVMEM_HEAPCFG_FORCLIENTS 0
+#define DEVMEM_HEAPCFG_FORFW 1
+
+
+/*
+  In order to call the server side functions, we need a bridge handle.
+  We abstract that here, as we may wish to change its form.
+ */
+
+typedef IMG_HANDLE DEVMEM_BRIDGE_HANDLE;
+
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetHeapInt(DEVMEM_HEAP *psHeap,
+				 IMG_HANDLE *phDevmemHeap);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetSize(DEVMEM_MEMDESC *psMemDesc,
+			  IMG_DEVMEM_SIZE_T* puiSize);
+
+IMG_INTERNAL void
+DevmemGetAnnotation(DEVMEM_MEMDESC *psMemDesc,
+			        IMG_CHAR **pszAnnotation);
+
+/*
+ * DevmemCreateContext()
+ *
+ * Create a device memory context
+ *
+ * This must be called before any heap is created in this context
+ *
+ * Caller to provide bridge handle which will be recorded internally and used
+ * for all future operations on items from this memory context.  Caller also
+ * to provide devicenode handle, as this is used for MMU configuration and
+ * also to determine the heap configuration for the auto-instantiated heaps.
+ *
+ * Note that when compiled in services/server, the hBridge is not used and
+ * is thrown away by the "fake" direct bridge.  (This may change. It is
+ * recommended that NULL be passed for the handle for now.)
+ *
+ * hDeviceNode and uiHeapBlueprintID shall together dictate which heap-config
+ * to use.
+ *
+ * This will cause the server side counterpart to be created also.
+ *
+ * If you call DevmemCreateContext() (and the call succeeds) you are promising
+ * that you will later call Devmem_ContextDestroy(), except for abnormal
+ * process termination in which case it is expected it will be destroyed as
+ * part of handle clean up.
+ *
+ * Caller to provide storage for the pointer to the newly created
+ * NEWDEVMEM_CONTEXT object.
+ */
+PVRSRV_ERROR
+DevmemCreateContext(SHARED_DEV_CONNECTION hDevConnection,
+                    DEVMEM_HEAPCFGID uiHeapBlueprintID,
+                    DEVMEM_CONTEXT **ppsCtxPtr);
+
+/*
+ * DevmemAcquireDevPrivData()
+ *
+ * Acquire the device private data for this memory context
+ */
+PVRSRV_ERROR
+DevmemAcquireDevPrivData(DEVMEM_CONTEXT *psCtx,
+                         IMG_HANDLE *hPrivData);
+
+/*
+ * DevmemReleaseDevPrivData()
+ *
+ * Release the device private data for this memory context
+ */
+PVRSRV_ERROR
+DevmemReleaseDevPrivData(DEVMEM_CONTEXT *psCtx);
+
+/*
+ * DevmemDestroyContext()
+ *
+ * Undoes that done by DevmemCreateContext()
+ */
+PVRSRV_ERROR
+DevmemDestroyContext(DEVMEM_CONTEXT *psCtx);
+
+/*
+ * DevmemCreateHeap()
+ *
+ * Create a heap in the given context.
+ *
+ * N.B.  Not intended to be called directly, though it can be.
+ * Normally, heaps are instantiated at context creation time according
+ * to the specified blueprint.  See DevmemCreateContext() for details.
+ *
+ * This will cause MMU code to set up data structures for the heap,
+ * but may not cause page tables to be modified until allocations are
+ * made from the heap.
+ *
+ * uiReservedRegionLength Reserved address space for static VAs shared
+ * between clients and firmware
+ *
+ * The "Quantum" is both the device MMU page size to be configured for
+ * this heap, and the unit multiples of which "quantized" allocations
+ * are made (allocations smaller than this, known as "suballocations"
+ * will be made from a "sub alloc RA" and will "import" chunks
+ * according to this quantum)
+ *
+ * Where imported PMRs (or, for example, PMRs created by device class
+ * buffers) are mapped into this heap, it is important that the
+ * physical contiguity guarantee offered by the PMR is greater than or
+ * equal to the quantum size specified here, otherwise the attempt to
+ * map it will fail.  "Normal" allocations via Devmem_Allocate
+ * shall automatically meet this requirement, as each "import" will
+ * trigger the creation of a PMR with the desired contiguity.  The
+ * supported quantum sizes in that case shall be dictated by the OS
+ * specific implementation of PhysmemNewOSRamBackedPMR() (see)
+ */
+PVRSRV_ERROR
+DevmemCreateHeap(DEVMEM_CONTEXT *psCtxPtr,
+                 /* base and length of heap */
+                 IMG_DEV_VIRTADDR sBaseAddress,
+                 IMG_DEVMEM_SIZE_T uiLength,
+                 IMG_DEVMEM_SIZE_T uiReservedRegionLength,
+                 /* log2 of allocation quantum, i.e. "page" size.
+                    All allocations (that go to server side) are
+                    multiples of this.  We use a client-side RA to
+                    make sub-allocations from this */
+                 IMG_UINT32 ui32Log2Quantum,
+                 /* The minimum import alignment for this heap */
+                 IMG_UINT32 ui32Log2ImportAlignment,
+                 /* Name of heap for debug */
+                 /* N.B.  Okay to exist on caller's stack - this
+                    func takes a copy if it needs it. */
+                 const IMG_CHAR *pszName,
+                 DEVMEM_HEAPCFGID uiHeapBlueprintID,
+                 IMG_UINT32 uiHeapIndex,
+                 DEVMEM_HEAP **ppsHeapPtr);
+/*
+ * DevmemDestroyHeap()
+ *
+ * Reverses DevmemCreateHeap()
+ *
+ * N.B. All allocations must have been freed and all mappings must
+ * have been unmapped before invoking this call
+ */
+PVRSRV_ERROR
+DevmemDestroyHeap(DEVMEM_HEAP *psHeap);
+
+/*
+ * DevmemExportalignAdjustSizeAndAlign()
+ * Compute the Size and Align passed to avoid suballocations
+ * (used when allocation with PVRSRV_MEMALLOCFLAG_EXPORTALIGN).
+ *
+ * Returns PVRSRV_ERROR_INVALID_PARAMS if uiLog2Quantum has invalid value.
+ */
+IMG_INTERNAL PVRSRV_ERROR
+DevmemExportalignAdjustSizeAndAlign(IMG_UINT32 uiLog2Quantum,
+                                    IMG_DEVMEM_SIZE_T *puiSize,
+                                    IMG_DEVMEM_ALIGN_T *puiAlign);
+
+/*
+ * DevmemSubAllocate()
+ *
+ * Makes an allocation (possibly a "suballocation", as described
+ * below) of device virtual memory from this heap.
+ *
+ * The size and alignment of the allocation will be honoured by the RA
+ * that allocates the "suballocation".  The resulting allocation will
+ * be mapped into GPU virtual memory and the physical memory to back
+ * it will exist, by the time this call successfully completes.
+ *
+ * The size must be a positive integer multiple of the alignment.
+ * (i.e. the alignment specifies the alignment of both the start and
+ * the end of the resulting allocation.)
+ *
+ * Allocations made via this API are routed through a "suballocation
+ * RA" which is responsible for ensuring that small allocations can be
+ * made without wasting physical memory in the server.  Furthermore,
+ * such suballocations can be made entirely client side without
+ * needing to go to the server unless the allocation spills into a new
+ * page.
+ *
+ * Such suballocations cause many allocations to share the same "PMR".
+ * This happens only when the flags match exactly.
+ *
+ */
+
+PVRSRV_ERROR
+DevmemSubAllocate(IMG_UINT8 uiPreAllocMultiplier,
+                  DEVMEM_HEAP *psHeap,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_DEVMEM_ALIGN_T uiAlign,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  const IMG_CHAR *pszText,
+                  DEVMEM_MEMDESC **ppsMemDescPtr);
+
+#define DevmemAllocate(...) \
+    DevmemSubAllocate(DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER, __VA_ARGS__)
+
+PVRSRV_ERROR
+DevmemAllocateExportable(SHARED_DEV_CONNECTION hDevConnection,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         IMG_DEVMEM_ALIGN_T uiAlign,
+                         IMG_UINT32 uiLog2HeapPageSize,
+                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                         const IMG_CHAR *pszText,
+                         DEVMEM_MEMDESC **ppsMemDescPtr);
+
+PVRSRV_ERROR
+DeviceMemChangeSparse(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *paui32AllocPageIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pauiFreePageIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiFlags);
+
+PVRSRV_ERROR
+DevmemAllocateSparse(SHARED_DEV_CONNECTION hDevConnection,
+                     IMG_DEVMEM_SIZE_T uiSize,
+                     IMG_UINT32 ui32NumPhysChunks,
+                     IMG_UINT32 ui32NumVirtChunks,
+                     IMG_UINT32 *pui32MappingTable,
+                     IMG_DEVMEM_ALIGN_T uiAlign,
+                     IMG_UINT32 uiLog2HeapPageSize,
+                     PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                     const IMG_CHAR *pszText,
+                     DEVMEM_MEMDESC **ppsMemDescPtr);
+
+PVRSRV_ERROR
+DevmemSubAllocateAndMap(IMG_UINT8 uiPreAllocMultiplier,
+			DEVMEM_HEAP *psHeap,
+			IMG_DEVMEM_SIZE_T uiSize,
+			IMG_DEVMEM_ALIGN_T uiAlign,
+			PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			const IMG_CHAR *pszText,
+			DEVMEM_MEMDESC **ppsMemDescPtr,
+			IMG_DEV_VIRTADDR *psDevVirtAddr);
+
+#define DevmemAllocateAndMap(...) \
+	DevmemSubAllocateAndMap(DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER, __VA_ARGS__)
+
+/*
+ * DevmemFree()
+ *
+ * Reverses that done by DevmemSubAllocate() N.B.  The underlying
+ * mapping and server side allocation _may_ not be torn down, for
+ * example, if the allocation has been exported, or if multiple
+ * allocations were suballocated from the same mapping, but this is
+ * properly refcounted, so the caller does not have to care.
+ */
+
+IMG_BOOL
+DevmemFree(DEVMEM_MEMDESC *psMemDesc);
+
+IMG_BOOL
+DevmemReleaseDevAddrAndFree(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+	DevmemMapToDevice:
+
+	Map an allocation to the device it was allocated from.
+	This function _must_ be called before any call to
+	DevmemAcquireDevVirtAddr is made as it binds the allocation
+	to the heap.
+	DevmemReleaseDevVirtAddr is used to release the reference
+	to the device mapping this function created, but it doesn't
+	mean that the memory will actually be unmapped from the
+	device as other references to the mapping obtained via
+	DevmemAcquireDevVirtAddr could still be active.
+*/
+PVRSRV_ERROR DevmemMapToDevice(DEVMEM_MEMDESC *psMemDesc,
+							   DEVMEM_HEAP *psHeap,
+							   IMG_DEV_VIRTADDR *psDevVirtAddr);
+
+/*
+	DevmemMapToDeviceAddress:
+
+	Same as DevmemMapToDevice but the caller chooses the address
+	to map to.
+*/
+IMG_INTERNAL PVRSRV_ERROR
+DevmemMapToDeviceAddress(DEVMEM_MEMDESC *psMemDesc,
+                         DEVMEM_HEAP *psHeap,
+                         IMG_DEV_VIRTADDR sDevVirtAddr);
+
+/*
+	DevmemGetDevVirtAddr
+
+	Obtain the MemDesc's device virtual address.
+	This function _must_ be called after DevmemMapToDevice(Address)
+	and is expected to be used be functions which didn't allocate
+	the MemDesc but need to know it's address.
+	It will PVR_ASSERT if no device mapping exists and 0 is returned.
+ */
+IMG_DEV_VIRTADDR
+DevmemGetDevVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+	DevmemAcquireDevVirtAddr
+
+	Acquire the MemDesc's device virtual address.
+	This function _must_ be called after DevmemMapToDevice
+	and is expected to be used be functions which didn't allocate
+	the MemDesc but need to know it's address
+ */
+PVRSRV_ERROR DevmemAcquireDevVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                      IMG_DEV_VIRTADDR *psDevVirtAddrRet);
+
+/*
+ * DevmemReleaseDevVirtAddr()
+ *
+ * give up the licence to use the device virtual address that was
+ * acquired by "Acquire" or "MapToDevice"
+ */
+void
+DevmemReleaseDevVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+/*
+ * DevmemAcquireCpuVirtAddr()
+ *
+ * Acquires a license to use the cpu virtual address of this mapping.
+ * Note that the memory may not have been mapped into cpu virtual
+ * memory prior to this call.  On first "acquire" the memory will be
+ * mapped in (if it wasn't statically mapped in) and on last put it
+ * _may_ become unmapped.  Later calling "Acquire" again, _may_ cause
+ * the memory to be mapped at a different address.
+ */
+PVRSRV_ERROR DevmemAcquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                      void **ppvCpuVirtAddr);
+
+/*
+ * DevmemReacquireCpuVirtAddr()
+ *
+ * (Re)acquires license to use the cpu virtual address of this mapping
+ * if (and only if) there is already a pre-existing license to use the
+ * cpu virtual address for the mapping, returns NULL otherwise.
+ */
+void DevmemReacquireCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc,
+                                void **ppvCpuVirtAddr);
+
+/*
+ * DevmemReleaseDevVirtAddr()
+ *
+ * give up the licence to use the cpu virtual address that was granted
+ * with the "Get" call.
+ */
+void
+DevmemReleaseCpuVirtAddr(DEVMEM_MEMDESC *psMemDesc);
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+/*
+ * DevmemExport()
+ *
+ * Given a memory allocation allocated with DevmemAllocateExportable()
+ * create a "cookie" that can be passed intact by the caller's own choice
+ * of secure IPC to another process and used as the argument to "map"
+ * to map this memory into a heap in the target processes.  N.B.  This can
+ * also be used to map into multiple heaps in one process, though that's not
+ * the intention.
+ *
+ * Note, the caller must later call Unexport before freeing the
+ * memory.
+ */
+PVRSRV_ERROR DevmemExport(DEVMEM_MEMDESC *psMemDesc,
+                          DEVMEM_EXPORTCOOKIE *psExportCookie);
+
+
+void DevmemUnexport(DEVMEM_MEMDESC *psMemDesc,
+					DEVMEM_EXPORTCOOKIE *psExportCookie);
+
+PVRSRV_ERROR
+DevmemImport(SHARED_DEV_CONNECTION hDevConnection,
+			 DEVMEM_EXPORTCOOKIE *psCookie,
+			 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+			 DEVMEM_MEMDESC **ppsMemDescPtr);
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+/*
+ * DevmemMakeLocalImportHandle()
+ *
+ * This is a "special case" function for making a server export cookie
+ * which went through the direct bridge into an export cookie that can
+ * be passed through the client bridge.
+ */
+PVRSRV_ERROR
+DevmemMakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+                            IMG_HANDLE hServerExport,
+                            IMG_HANDLE *hClientExport);
+
+/*
+ * DevmemUnmakeLocalImportHandle()
+ *
+ * Free any resource associated with the Make operation
+ */
+PVRSRV_ERROR
+DevmemUnmakeLocalImportHandle(SHARED_DEV_CONNECTION hDevConnection,
+                              IMG_HANDLE hClientExport);
+
+/*
+ *
+ * The following set of functions is specific to the heap "blueprint"
+ * stuff, for automatic creation of heaps when a context is created
+ *
+ */
+
+
+/* Devmem_HeapConfigCount: returns the number of heap configs that
+   this device has.  Note that there is no acquire/release semantics
+   required, as this data is guaranteed to be constant for the
+   lifetime of the device node */
+PVRSRV_ERROR
+DevmemHeapConfigCount(SHARED_DEV_CONNECTION hDevConnection,
+                      IMG_UINT32 *puiNumHeapConfigsOut);
+
+/* Devmem_HeapCount: returns the number of heaps that a given heap
+   config on this device has.  Note that there is no acquire/release
+   semantics required, as this data is guaranteed to be constant for
+   the lifetime of the device node */
+PVRSRV_ERROR
+DevmemHeapCount(SHARED_DEV_CONNECTION hDevConnection,
+                IMG_UINT32 uiHeapConfigIndex,
+                IMG_UINT32 *puiNumHeapsOut);
+/* Devmem_HeapConfigName: return the name of the given heap config.
+   The caller is to provide the storage for the returned string and
+   indicate the number of bytes (including null terminator) for such
+   string in the BufSz arg.  Note that there is no acquire/release
+   semantics required, as this data is guaranteed to be constant for
+   the lifetime of the device node.
+ */
+PVRSRV_ERROR
+DevmemHeapConfigName(SHARED_DEV_CONNECTION hsDevConnection,
+                     IMG_UINT32 uiHeapConfigIndex,
+                     IMG_CHAR *pszConfigNameOut,
+                     IMG_UINT32 uiConfigNameBufSz);
+
+/* Devmem_HeapDetails: fetches all the metadata that is recorded in
+   this heap "blueprint".  Namely: heap name (caller to provide
+   storage, and indicate buffer size (including null terminator) in
+   BufSz arg), device virtual address and length, log2 of data page
+   size (will be one of 12, 14, 16, 18, 20, 21, at time of writing).
+   Note that there is no acquire/release semantics required, as this
+   data is guaranteed to be constant for the lifetime of the device
+   node. */
+PVRSRV_ERROR
+DevmemHeapDetails(SHARED_DEV_CONNECTION hDevConnection,
+                  IMG_UINT32 uiHeapConfigIndex,
+                  IMG_UINT32 uiHeapIndex,
+                  IMG_CHAR *pszHeapNameOut,
+                  IMG_UINT32 uiHeapNameBufSz,
+                  IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+                  IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+                  IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+                  IMG_UINT32 *puiLog2DataPageSize,
+                  IMG_UINT32 *puiLog2ImportAlignmentOut);
+
+/*
+ * Devmem_FindHeapByName()
+ *
+ * returns the heap handle for the named _automagic_ heap in this
+ * context.  "automagic" heaps are those that are born with the
+ * context from a blueprint
+ */
+PVRSRV_ERROR
+DevmemFindHeapByName(const DEVMEM_CONTEXT *psCtx,
+                     const IMG_CHAR *pszHeapName,
+                     DEVMEM_HEAP **ppsHeapRet);
+
+/*
+ * DevmemGetHeapBaseDevVAddr()
+ *
+ * returns the device virtual address of the base of the heap.
+ */
+PVRSRV_ERROR
+DevmemGetHeapBaseDevVAddr(DEVMEM_HEAP *psHeap,
+			  IMG_DEV_VIRTADDR *pDevVAddr);
+
+/*
+ * DevmemGetHeapSize()
+ *
+ * returns the size of the heap.
+ */
+IMG_INTERNAL DEVMEM_SIZE_T
+DevmemGetHeapSize(struct DEVMEM_HEAP_TAG *psHeap);
+
+PVRSRV_ERROR
+DevmemLocalGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+			   IMG_HANDLE *phImport);
+
+PVRSRV_ERROR
+DevmemGetImportUID(DEVMEM_MEMDESC *psMemDesc,
+		   IMG_UINT64 *pui64UID);
+
+PVRSRV_ERROR
+DevmemGetReservation(DEVMEM_MEMDESC *psMemDesc,
+		     IMG_HANDLE *hReservation);
+
+IMG_INTERNAL void
+DevmemGetPMRData(DEVMEM_MEMDESC *psMemDesc,
+		IMG_HANDLE *hPMR,
+		IMG_DEVMEM_OFFSET_T *puiPMROffset);
+
+IMG_INTERNAL void
+DevmemGetFlags(DEVMEM_MEMDESC *psMemDesc,
+				PVRSRV_MEMALLOCFLAGS_T *puiFlags);
+
+IMG_INTERNAL SHARED_DEV_CONNECTION
+DevmemGetConnection(DEVMEM_MEMDESC *psMemDesc);
+
+PVRSRV_ERROR
+DevmemLocalImport(SHARED_DEV_CONNECTION hDevConnection,
+				  IMG_HANDLE hExtHandle,
+				  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+				  DEVMEM_MEMDESC **ppsMemDescPtr,
+				  IMG_DEVMEM_SIZE_T *puiSizePtr,
+				  const IMG_CHAR *pszAnnotation);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemIsDevVirtAddrValid(DEVMEM_CONTEXT *psContext,
+                         IMG_DEV_VIRTADDR sDevVAddr);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemGetFaultAddress(DEVMEM_CONTEXT *psContext,
+                      IMG_DEV_VIRTADDR *psFaultAddress);
+
+IMG_INTERNAL PVRSRV_ERROR
+DevmemInvalidateFBSCTable(DEVMEM_CONTEXT *psContext,
+                          IMG_UINT64 ui64FBSCEntries);
+
+/* DevmemGetHeapLog2PageSize()
+ *
+ * Get the page size used for a certain heap.
+ */
+IMG_UINT32
+DevmemGetHeapLog2PageSize(DEVMEM_HEAP *psHeap);
+
+/* DevmemGetMemFlags()
+ *
+ * Get the memalloc flags for a certain memdesc.
+ */
+PVRSRV_MEMALLOCFLAGS_T
+DevmemGetMemAllocFlags(DEVMEM_MEMDESC *psMemDesc);
+
+/* DevmemGetHeapReservedSize()
+ *
+ * Get the reserved size used for a certain heap.
+ */
+IMG_DEVMEM_SIZE_T
+DevmemGetHeapReservedSize(DEVMEM_HEAP *psHeap);
+
+/*************************************************************************/ /*!
+@Function       RegisterDevMemPFNotify
+@Description    Registers that the application wants to be signaled when a page
+                fault occurs.
+
+@Input          psContext      Memory context the process that would like to
+                               be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR:  PVRSRV_OK on success. Otherwise, a PVRSRV_
+                               error code
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RegisterDevmemPFNotify(DEVMEM_CONTEXT *psContext,
+                       IMG_BOOL       bRegister);
+
+/*************************************************************************/ /*!
+@Function       DevmemHeapSetPremapStatus
+@Description    In some special cases like virtualisation, a device memory heap
+			    must be entirely backed by physical memory and mapped into the
+				device's virtual address space. This is done at context creation.
+			    When objects are allocated from such a heap, the mapping part
+			    must be skipped. The 'bPremapped' flag dictates if allocations
+			    are to be mapped or not.
+
+@Input          psHeap            Device memory heap to be updated
+@Input          IsPremapped       The premapping status to be set
+*/ /**************************************************************************/
+IMG_INTERNAL void
+DevmemHeapSetPremapStatus(DEVMEM_HEAP *psHeap, IMG_BOOL IsPremapped);
+
+#endif /* #ifndef SRVCLIENT_DEVICEMEM_H */
diff --git a/drivers/gpu/drm/img-rogue/devicemem_heapcfg.c b/drivers/gpu/drm/img-rogue/devicemem_heapcfg.c
new file mode 100644
index 000000000000..9b53d026bb5d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_heapcfg.c
@@ -0,0 +1,225 @@
+/*************************************************************************/ /*!
+@File           devicemem_heapcfg.c
+@Title          Device Heap Configuration Helper Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+/* our exported API */
+#include "devicemem_heapcfg.h"
+#include "devicemem_utils.h"
+
+#include "device.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+
+#include "connection_server.h"
+
+static INLINE void _CheckBlueprintHeapAlignment(DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint)
+{
+	IMG_UINT32 ui32OSPageSize = OSGetPageSize();
+
+	/* Any heap length should at least match OS page size at the minimum or
+	 * a multiple of OS page size */
+	if ((psHeapBlueprint->uiHeapLength < DEVMEM_HEAP_MINIMUM_SIZE) ||
+		(psHeapBlueprint->uiHeapLength & (ui32OSPageSize - 1)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC
+		         "("IMG_DEVMEM_SIZE_FMTSPEC")",
+		         __func__,
+		         psHeapBlueprint->pszName,
+		         psHeapBlueprint->uiHeapLength,
+		         psHeapBlueprint->uiHeapLength));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Heap Size should always be at least the DevMem minimum size and a "
+		         "multiple of OS Page Size:%u(0x%x)",
+		         ui32OSPageSize, ui32OSPageSize));
+		PVR_ASSERT(psHeapBlueprint->uiHeapLength >= ui32OSPageSize);
+	}
+
+
+	PVR_ASSERT(psHeapBlueprint->uiReservedRegionLength % DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY == 0);
+}
+
+void HeapCfgBlueprintInit(const IMG_CHAR        *pszName,
+	                      IMG_UINT64             ui64HeapBaseAddr,
+	                      IMG_DEVMEM_SIZE_T      uiHeapLength,
+	                      IMG_DEVMEM_SIZE_T      uiReservedRegionLength,
+	                      IMG_UINT32             ui32Log2DataPageSize,
+	                      IMG_UINT32             uiLog2ImportAlignment,
+	                      PFN_HEAP_INIT          pfnInit,
+	                      PFN_HEAP_DEINIT        pfnDeInit,
+	                      DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint)
+{
+	psHeapBlueprint->pszName                = pszName;
+	psHeapBlueprint->sHeapBaseAddr.uiAddr   = ui64HeapBaseAddr;
+	psHeapBlueprint->uiHeapLength           = uiHeapLength;
+	psHeapBlueprint->uiReservedRegionLength = uiReservedRegionLength;
+	psHeapBlueprint->uiLog2DataPageSize     = ui32Log2DataPageSize;
+	psHeapBlueprint->uiLog2ImportAlignment  = uiLog2ImportAlignment;
+	psHeapBlueprint->pfnInit                = pfnInit;
+	psHeapBlueprint->pfnDeInit              = pfnDeInit;
+
+	_CheckBlueprintHeapAlignment(psHeapBlueprint);
+}
+
+PVRSRV_ERROR
+HeapCfgHeapConfigCount(CONNECTION_DATA * psConnection,
+					   const PVRSRV_DEVICE_NODE *psDeviceNode,
+					   IMG_UINT32 *puiNumHeapConfigsOut)
+{
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	*puiNumHeapConfigsOut = psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapCount(CONNECTION_DATA * psConnection,
+				 const PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_UINT32 uiHeapConfigIndex,
+				 IMG_UINT32 *puiNumHeapsOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	*puiNumHeapsOut = psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapConfigName(CONNECTION_DATA * psConnection,
+					  const PVRSRV_DEVICE_NODE *psDeviceNode,
+					  IMG_UINT32 uiHeapConfigIndex,
+					  IMG_UINT32 uiHeapConfigNameBufSz,
+					  IMG_CHAR *pszHeapConfigNameOut)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	OSSNPrintf(pszHeapConfigNameOut, uiHeapConfigNameBufSz, "%s", psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].pszName);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgGetCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    PFN_HEAP_INIT *ppfnInit,
+                    PFN_HEAP_DEINIT *ppfnDeinit)
+{
+	DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppfnInit, "ppfnInit");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppfnDeinit, "ppfnDeinit");
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	if (uiHeapIndex >= psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+	}
+
+	psHeapBlueprint = &psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].psHeapBlueprintArray[uiHeapIndex];
+
+	*ppfnInit = psHeapBlueprint->pfnInit;
+	*ppfnDeinit = psHeapBlueprint->pfnDeInit;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+HeapCfgHeapDetails(CONNECTION_DATA * psConnection,
+				   const PVRSRV_DEVICE_NODE *psDeviceNode,
+				   IMG_UINT32 uiHeapConfigIndex,
+				   IMG_UINT32 uiHeapIndex,
+				   IMG_UINT32 uiHeapNameBufSz,
+				   IMG_CHAR *pszHeapNameOut,
+				   IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+				   IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+				   IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+				   IMG_UINT32 *puiLog2DataPageSizeOut,
+				   IMG_UINT32 *puiLog2ImportAlignmentOut)
+{
+	DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (uiHeapConfigIndex >= psDeviceNode->sDevMemoryInfo.uiNumHeapConfigs)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX;
+	}
+
+	if (uiHeapIndex >= psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].uiNumHeaps)
+	{
+		return PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX;
+	}
+
+	psHeapBlueprint = &psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeapConfigArray[uiHeapConfigIndex].psHeapBlueprintArray[uiHeapIndex];
+
+	OSSNPrintf(pszHeapNameOut, uiHeapNameBufSz, "%s", psHeapBlueprint->pszName);
+	*psDevVAddrBaseOut = psHeapBlueprint->sHeapBaseAddr;
+	*puiHeapLengthOut = psHeapBlueprint->uiHeapLength;
+	*puiReservedRegionLengthOut = psHeapBlueprint->uiReservedRegionLength;
+	*puiLog2DataPageSizeOut = psHeapBlueprint->uiLog2DataPageSize;
+	*puiLog2ImportAlignmentOut = psHeapBlueprint->uiLog2ImportAlignment;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img-rogue/devicemem_heapcfg.h b/drivers/gpu/drm/img-rogue/devicemem_heapcfg.h
new file mode 100644
index 000000000000..3b8c0aa08921
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_heapcfg.h
@@ -0,0 +1,213 @@
+/**************************************************************************/ /*!
+@File
+@Title          Device Heap Configuration Helper Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DEVICEMEMHEAPCFG_H
+#define DEVICEMEMHEAPCFG_H
+
+#include <powervr/mem_types.h>
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+struct _PVRSRV_DEVICE_NODE_;
+struct _CONNECTION_DATA_;
+struct _DEVMEMINT_HEAP_;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_HEAP_INIT
+@Description    Device heap initialisation function. Called in server devmem
+                heap create if the callback pointer in RGX_HEAP_INFO is
+                not NULL.
+@Input          psDeviceNode       The device node.
+@Input          psDevmemHeap       Server internal devmem heap.
+@Output         phPrivData         Private data handle. Allocated resources
+                                   can be freed in PFN_HEAP_DEINIT.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_HEAP_INIT)(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                                      struct _DEVMEMINT_HEAP_ *psDevmemHeap,
+                                      IMG_HANDLE *phPrivData);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_HEAP_DEINIT
+@Description    Device heap deinit function. Called in server devmem
+                heap create if the callback pointer in RGX_HEAP_INFO is
+                not NULL.
+@Input          hPrivData          Private data handle. To free any resources.
+*/ /**************************************************************************/
+typedef void (*PFN_HEAP_DEINIT)(IMG_HANDLE hPrivData);
+
+/*
+  A "heap config" is a blueprint to be used for initial setting up of heaps
+  when a device memory context is created.
+
+  We define a data structure to define this, but it's really down to the
+  caller to populate it. This is all expected to be in-kernel. We provide an
+  API that client code can use to enquire about the blueprint, such that it may
+  do the heap set-up during the context creation call on behalf of the user.
+*/
+
+/* Blueprint for a single heap */
+typedef struct _DEVMEM_HEAP_BLUEPRINT_
+{
+	/* Name of this heap - for debug purposes, and perhaps for lookup
+	by name */
+	const IMG_CHAR *pszName;
+
+	/* Virtual address of the beginning of the heap.  This _must_ be a
+	multiple of the data page size for the heap.  It is
+	_recommended_ that it be coarser than that - especially, it
+	should begin on a boundary appropriate to the MMU for the
+	device.  For Rogue, this is a Page Directory boundary, or 1GB
+	(virtual address a multiple of 0x0040000000). */
+	IMG_DEV_VIRTADDR sHeapBaseAddr;
+
+	/* Length of the heap.  Given that the END address of the heap has
+	a similar restriction to that of the _beginning_ of the heap.
+	That is the heap length _must_ be a whole number of data pages.
+	Again, the recommendation is that it ends on a 1GB boundary.
+	Again, this is not essential, but we do know that (at the time
+	of writing) the current implementation of mmu_common.c is such
+	that no two heaps may share a page directory, thus the
+	remaining virtual space would be wasted if the length were not
+	a multiple of 1GB */
+	IMG_DEVMEM_SIZE_T uiHeapLength;
+
+	/* VA space starting sHeapBaseAddr to uiReservedRegionLength-1 are reserved
+	for statically defined addresses (shared/known between clients and FW).
+	Services never maps allocations into this reserved address space _unless_
+	explicitly requested via PVRSRVMapToDeviceAddress by passing sDevVirtAddr
+	which falls within this reserved range. Since this range is completely for
+	clients to manage (where allocations are page granular), it _must_ again be
+	a whole number of data pages. Additionally, another constraint enforces this
+	to be a multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY (which evaluates to
+	max page size supported) to support varied pages sizes */
+	IMG_DEVMEM_SIZE_T uiReservedRegionLength;
+
+	/* Data page size.  This is the page size that is going to get
+	programmed into the MMU, so it needs to be a valid one for the
+	device.  Importantly, the start address and length _must_ be
+	multiples of this page size.  Note that the page size is
+	specified as the log 2 relative to 1 byte (e.g. 12 indicates
+	4kB) */
+	IMG_UINT32 uiLog2DataPageSize;
+
+	/* Import alignment.  Force imports to this heap to be
+	aligned to at least this value */
+	IMG_UINT32 uiLog2ImportAlignment;
+
+	/* Callback function for device specific heap init. */
+	PFN_HEAP_INIT pfnInit;
+
+	/* Callback function for device specific heap deinit. */
+	PFN_HEAP_DEINIT pfnDeInit;
+
+} DEVMEM_HEAP_BLUEPRINT;
+
+void HeapCfgBlueprintInit(const IMG_CHAR        *pszName,
+	                      IMG_UINT64             ui64HeapBaseAddr,
+	                      IMG_DEVMEM_SIZE_T      uiHeapLength,
+	                      IMG_DEVMEM_SIZE_T      uiReservedRegionLength,
+	                      IMG_UINT32             ui32Log2DataPageSize,
+	                      IMG_UINT32             uiLog2ImportAlignment,
+	                      PFN_HEAP_INIT          pfnInit,
+	                      PFN_HEAP_DEINIT        pfnDeInit,
+	                      DEVMEM_HEAP_BLUEPRINT *psHeapBlueprint);
+
+/* Entire named heap config */
+typedef struct _DEVMEM_HEAP_CONFIG_
+{
+    /* Name of this heap config - for debug and maybe lookup */
+    const IMG_CHAR *pszName;
+
+    /* Number of heaps in this config */
+    IMG_UINT32 uiNumHeaps;
+
+    /* Array of individual heap blueprints as defined above */
+    DEVMEM_HEAP_BLUEPRINT *psHeapBlueprintArray;
+} DEVMEM_HEAP_CONFIG;
+
+
+PVRSRV_ERROR
+HeapCfgHeapConfigCount(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 *puiNumHeapConfigsOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapCount(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 *puiNumHeapsOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapConfigName(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 uiHeapConfigNameBufSz,
+    IMG_CHAR *pszHeapConfigNameOut
+);
+
+PVRSRV_ERROR
+HeapCfgHeapDetails(struct _CONNECTION_DATA_ *psConnection,
+    const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+    IMG_UINT32 uiHeapConfigIndex,
+    IMG_UINT32 uiHeapIndex,
+    IMG_UINT32 uiHeapNameBufSz,
+    IMG_CHAR *pszHeapNameOut,
+    IMG_DEV_VIRTADDR *psDevVAddrBaseOut,
+    IMG_DEVMEM_SIZE_T *puiHeapLengthOut,
+    IMG_DEVMEM_SIZE_T *puiReservedRegionLengthOut,
+    IMG_UINT32 *puiLog2DataPageSizeOut,
+    IMG_UINT32 *puiLog2ImportAlignmentOut
+);
+
+PVRSRV_ERROR
+HeapCfgGetCallbacks(const struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    PFN_HEAP_INIT *ppfnInit,
+                    PFN_HEAP_DEINIT *ppfnDeinit);
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/devicemem_history_server.c b/drivers/gpu/drm/img-rogue/devicemem_history_server.c
new file mode 100644
index 000000000000..f28e96fa55e1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_history_server.c
@@ -0,0 +1,2305 @@
+/*************************************************************************/ /*!
+@File
+@Title          Devicemem history functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Devicemem history functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "allocmem.h"
+#include "img_defs.h"
+#include "pmr.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "pvr_debug.h"
+#include "devicemem_server.h"
+#include "lock.h"
+#include "di_server.h"
+#include "devicemem_history_server.h"
+#include "pdump_km.h"
+
+
+#if (PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES < 5000)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES is too low.
+#elif (PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES > 250000)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES is too high.
+#else
+#define ALLOCATION_LIST_NUM_ENTRIES PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES
+#endif
+
+
+/* data type to hold an allocation index.
+ * we make it 16 bits wide if possible
+ */
+#if ALLOCATION_LIST_NUM_ENTRIES <= 0xFFFF
+typedef uint16_t ALLOC_INDEX_T;
+#else
+typedef uint32_t ALLOC_INDEX_T;
+#endif
+
+/* a record describing a single allocation known to DeviceMemHistory.
+ * this is an element in a doubly linked list of allocations
+ */
+typedef struct _RECORD_ALLOCATION_
+{
+	/* time when this RECORD_ALLOCATION was created/initialised */
+	IMG_UINT64 ui64CreationTime;
+	/* serial number of the PMR relating to this allocation */
+	IMG_UINT64 ui64Serial;
+	/* base DevVAddr of this allocation */
+	IMG_DEV_VIRTADDR sDevVAddr;
+	/* size in bytes of this allocation */
+	IMG_DEVMEM_SIZE_T uiSize;
+	/* Log2 page size of this allocation's GPU pages */
+	IMG_UINT32 ui32Log2PageSize;
+	/* Process ID (PID) this allocation belongs to */
+	IMG_PID uiPID;
+	/* index of previous allocation in the list */
+	ALLOC_INDEX_T ui32Prev;
+	/* index of next allocation in the list */
+	ALLOC_INDEX_T ui32Next;
+	/* annotation/name of this allocation */
+	IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN];
+} RECORD_ALLOCATION;
+
+/* each command in the circular buffer is prefixed with an 8-bit value
+ * denoting the command type
+ */
+typedef enum _COMMAND_TYPE_
+{
+	COMMAND_TYPE_NONE,
+	COMMAND_TYPE_TIMESTAMP,
+	COMMAND_TYPE_MAP_ALL,
+	COMMAND_TYPE_UNMAP_ALL,
+	COMMAND_TYPE_MAP_RANGE,
+	COMMAND_TYPE_UNMAP_RANGE,
+	/* sentinel value */
+	COMMAND_TYPE_COUNT,
+} COMMAND_TYPE;
+
+/* Timestamp command:
+ * This command is inserted into the circular buffer to provide an updated
+ * timestamp.
+ * The nanosecond-accuracy timestamp is packed into a 56-bit integer, in order
+ * for the whole command to fit into 10 bytes.
+ */
+typedef struct __attribute__((__packed__))_COMMAND_TIMESTAMP_
+{
+	IMG_UINT8 aui8TimeNs[7];
+} COMMAND_TIMESTAMP;
+
+/* MAP_ALL command:
+ * This command denotes the allocation at the given index was wholly mapped
+ * in to the GPU MMU
+ */
+typedef struct __attribute__((__packed__))_COMMAND_MAP_ALL_
+{
+	ALLOC_INDEX_T uiAllocIndex;
+} COMMAND_MAP_ALL;
+
+/* UNMAP_ALL command:
+ * This command denotes the allocation at the given index was wholly unmapped
+ * from the GPU MMU
+ * Note: COMMAND_MAP_ALL and COMMAND_UNMAP_ALL commands have the same layout.
+ */
+typedef COMMAND_MAP_ALL COMMAND_UNMAP_ALL;
+
+// This shift allows room for 512GiB virtual memory regions at 4Kb pages.
+#define VM_RANGE_SHIFT 27
+
+/* packing attributes for the MAP_RANGE command */
+#define MAP_RANGE_MAX_START ((1 << VM_RANGE_SHIFT) - 1)
+#define MAP_RANGE_MAX_RANGE ((1 << VM_RANGE_SHIFT) - 1)
+
+/* MAP_RANGE command:
+ * Denotes a range of pages within the given allocation being mapped.
+ * The range is expressed as [Page Index] + [Page Count]
+ * This information is packed into a 72/88-bit struct, in order to make
+ * the command size 9/11 bytes.
+ */
+
+typedef struct __attribute__((__packed__))_COMMAND_MAP_RANGE_
+{
+	IMG_UINT8 aui8Data[7];
+	ALLOC_INDEX_T uiAllocIndex;
+} COMMAND_MAP_RANGE;
+
+/* UNMAP_RANGE command:
+ * Denotes a range of pages within the given allocation being unmapped.
+ * The range is expressed as [Page Index] + [Page Count]
+ * This information is packed into a 72/88-bit struct, in order to make
+ * the command size 9/11 bytes.
+ */
+typedef COMMAND_MAP_RANGE COMMAND_UNMAP_RANGE;
+
+/* wrapper structure for a command */
+typedef struct __attribute__((__packed__))_COMMAND_WRAPPER_
+{
+	IMG_UINT8 ui8Type;
+	union {
+		COMMAND_TIMESTAMP sTimeStamp;
+		COMMAND_MAP_ALL sMapAll;
+		COMMAND_UNMAP_ALL sUnmapAll;
+		COMMAND_MAP_RANGE sMapRange;
+		COMMAND_UNMAP_RANGE sUnmapRange;
+	} u;
+} COMMAND_WRAPPER;
+
+/* target size for the circular buffer of commands */
+#if (PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 < 5)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 is too low.
+#elif (PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 > 18)
+#error PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2 is too high.
+#else
+#define CIRCULAR_BUFFER_SIZE_KB (1 << PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2)
+#endif
+
+
+/* turn the circular buffer target size into a number of commands */
+#define CIRCULAR_BUFFER_NUM_COMMANDS ((CIRCULAR_BUFFER_SIZE_KB * 1024) / sizeof(COMMAND_WRAPPER))
+
+/* index value denoting the end of a list */
+#define END_OF_LIST 0xFFFFFFFF
+#define ALLOC_INDEX_TO_PTR(psDevHData, idx) (&((psDevHData)->sRecords.pasAllocations[idx]))
+#define CHECK_ALLOC_INDEX(idx) (idx < ALLOCATION_LIST_NUM_ENTRIES)
+
+/* wrapper structure for the allocation records and the commands circular buffer */
+typedef struct _RECORDS_
+{
+	RECORD_ALLOCATION *pasAllocations;
+	IMG_UINT32 ui32AllocationsListHead;
+
+	IMG_UINT32 ui32Head;
+	IMG_UINT32 ui32Tail;
+	COMMAND_WRAPPER *pasCircularBuffer;
+	/* Times the CB has wrapped back to start */
+	IMG_UINT64 ui64CBWrapCount;
+	/* Records of CB commands sent */
+	IMG_UINT64 ui64MapAllCount;//Incremented by InsertMapAllCommand()
+	IMG_UINT64 ui64UnMapAllCount;//Incremented by InsertUnmapAllCommand()
+	IMG_UINT64 ui64MapRangeCount;//Incremented by InsertMapRangeCommand()
+	IMG_UINT64 ui64UnMapRangeCount;//Incremented by InsertUnmapRangeCommand()
+	IMG_UINT64 ui64TimeStampCount;//Incremented by InsertTimeStampCommand()
+} RECORDS;
+
+typedef struct _DEVICEMEM_HISTORY_DATA_
+{
+	RECORDS sRecords;
+	POS_LOCK hLock;
+} DEVICEMEM_HISTORY_DATA;
+
+/* Maximum number of device instances supported. This should be DDK global */
+static DEVICEMEM_HISTORY_DATA *gapsDevicememHistoryData[PVRSRV_MAX_DEVICES] = { NULL };
+
+/* DevmemFindDataFromDev
+ *
+ * Return the address of the associated DEVICEMEM_HISTORY_DATA for the given
+ * device. If psDevNode associated unit is out of range we return NULL.
+ */
+static DEVICEMEM_HISTORY_DATA *DevmemFindDataFromDev(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DEVICEMEM_HISTORY_DATA *psDevmemData = NULL;
+
+	IMG_UINT32 uiUnit = psDevNode->sDevId.ui32InternalID;
+
+	PVR_ASSERT(uiUnit < PVRSRV_MAX_DEVICES);
+	if ((uiUnit < PVRSRV_MAX_DEVICES) && (gapsDevicememHistoryData[uiUnit] != NULL))
+	{
+		psDevmemData = gapsDevicememHistoryData[uiUnit];
+	}
+
+	return psDevmemData;
+}
+
+/* gsDevicememHistoryData is static, hLock is NULL unless
+ * EnablePageFaultDebug is set and DevicememHistoryInitKM()
+ * was called.
+ */
+static void DevicememHistoryLock(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	if (psDevHData->hLock)
+	{
+		OSLockAcquire(psDevHData->hLock);
+	}
+}
+
+static void DevicememHistoryUnlock(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	if (psDevHData->hLock)
+	{
+		OSLockRelease(psDevHData->hLock);
+	}
+}
+
+/* given a time stamp, calculate the age in nanoseconds */
+static IMG_UINT64 _CalculateAge(IMG_UINT64 ui64Now,
+						IMG_UINT64 ui64Then,
+						IMG_UINT64 ui64Max)
+{
+	if (ui64Now >= ui64Then)
+	{
+		/* no clock wrap */
+		return ui64Now - ui64Then;
+	}
+	else
+	{
+		/* clock has wrapped */
+		return (ui64Max - ui64Then) + ui64Now + 1;
+	}
+}
+
+/* AcquireCBSlot:
+ * Acquire the next slot in the circular buffer and
+ * move the circular buffer head along by one
+ * Returns a pointer to the acquired slot.
+ */
+static COMMAND_WRAPPER *AcquireCBSlot(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	COMMAND_WRAPPER *psSlot;
+
+	psSlot = &psDevHData->sRecords.pasCircularBuffer[psDevHData->sRecords.ui32Head];
+
+	psDevHData->sRecords.ui32Head =
+		(psDevHData->sRecords.ui32Head + 1)
+				% CIRCULAR_BUFFER_NUM_COMMANDS;
+
+	if (psDevHData->sRecords.ui32Head == 0)
+	{
+		psDevHData->sRecords.ui64CBWrapCount++;
+	}
+
+	return psSlot;
+}
+
+/* TimeStampPack:
+ * Packs the given timestamp value into the COMMAND_TIMESTAMP structure.
+ * This takes a 64-bit nanosecond timestamp and packs it in to a 56-bit
+ * integer in the COMMAND_TIMESTAMP command.
+ */
+static void TimeStampPack(COMMAND_TIMESTAMP *psTimeStamp, IMG_UINT64 ui64Now)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ARRAY_SIZE(psTimeStamp->aui8TimeNs); i++)
+	{
+		psTimeStamp->aui8TimeNs[i] = ui64Now & 0xFF;
+		ui64Now >>= 8;
+	}
+}
+
+/* packing a 64-bit nanosecond into a 7-byte integer loses the
+ * top 8 bits of data. This must be taken into account when
+ * comparing a full timestamp against an unpacked timestamp
+ */
+#define TIME_STAMP_MASK ((1LLU << 56) - 1)
+#define DO_TIME_STAMP_MASK(ns64) (ns64 & TIME_STAMP_MASK)
+
+/* TimeStampUnpack:
+ * Unpack the timestamp value from the given COMMAND_TIMESTAMP command
+ */
+static IMG_UINT64 TimeStampUnpack(COMMAND_TIMESTAMP *psTimeStamp)
+{
+	IMG_UINT64 ui64TimeNs = 0;
+	IMG_UINT32 i;
+
+	for (i = ARRAY_SIZE(psTimeStamp->aui8TimeNs); i > 0; i--)
+	{
+		ui64TimeNs <<= 8;
+		ui64TimeNs |= (IMG_UINT64) psTimeStamp->aui8TimeNs[i - 1];
+	}
+
+	return ui64TimeNs;
+}
+
+#if defined(PDUMP)
+
+static void EmitPDumpAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32AllocationIndex,
+                                RECORD_ALLOCATION *psAlloc)
+{
+	PDUMPCOMMENT(psDeviceNode,
+			"[SrvPFD] Allocation: %u"
+			" Addr: " IMG_DEV_VIRTADDR_FMTSPEC
+			" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+			" Page size: %u"
+			" PID: %u"
+			" Process: %s"
+			" Name: %s",
+			ui32AllocationIndex,
+			psAlloc->sDevVAddr.uiAddr,
+			psAlloc->uiSize,
+			1U << psAlloc->ui32Log2PageSize,
+			psAlloc->uiPID,
+			OSGetCurrentClientProcessNameKM(),
+			psAlloc->szName);
+}
+
+static void EmitPDumpMapUnmapAll(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 COMMAND_TYPE eType,
+                                 IMG_UINT32 ui32AllocationIndex)
+{
+	const IMG_CHAR *pszOpName;
+
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_ALL:
+			pszOpName = "MAP_ALL";
+			break;
+		case COMMAND_TYPE_UNMAP_ALL:
+			pszOpName = "UNMAP_ALL";
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapAll: Invalid type: %u",
+										eType));
+			return;
+
+	}
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "[SrvPFD] Op: %s Allocation: %u",
+	             pszOpName,
+	             ui32AllocationIndex);
+}
+
+static void EmitPDumpMapUnmapRange(PVRSRV_DEVICE_NODE *psDeviceNode,
+					COMMAND_TYPE eType,
+					IMG_UINT32 ui32AllocationIndex,
+					IMG_UINT32 ui32StartPage,
+					IMG_UINT32 ui32Count)
+{
+	const IMG_CHAR *pszOpName;
+
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_RANGE:
+			pszOpName = "MAP_RANGE";
+			break;
+		case COMMAND_TYPE_UNMAP_RANGE:
+			pszOpName = "UNMAP_RANGE";
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "EmitPDumpMapUnmapRange: Invalid type: %u",
+										eType));
+			return;
+	}
+
+	PDUMPCOMMENT(psDeviceNode,
+                 "[SrvPFD] Op: %s Allocation: %u Start Page: %u Count: %u",
+                 pszOpName,
+                 ui32AllocationIndex,
+                 ui32StartPage,
+                 ui32Count);
+}
+
+#endif
+
+/* InsertTimeStampCommand:
+ * Insert a timestamp command into the circular buffer.
+ */
+static void InsertTimeStampCommand(IMG_UINT64 ui64Now, PVRSRV_DEVICE_NODE *psDevNode)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDevNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_TIMESTAMP;
+	psDevHData->sRecords.ui64TimeStampCount++;
+	TimeStampPack(&psCommand->u.sTimeStamp, ui64Now);
+}
+
+/* InsertMapAllCommand:
+ * Insert a "MAP_ALL" command for the given allocation into the circular buffer
+ */
+static void InsertMapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32AllocIndex)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_MAP_ALL;
+	psCommand->u.sMapAll.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64MapAllCount++;
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_MAP_ALL, ui32AllocIndex);
+#endif
+}
+
+/* InsertUnmapAllCommand:
+ * Insert a "UNMAP_ALL" command for the given allocation into the circular buffer
+ */
+static void InsertUnmapAllCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32AllocIndex)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_UNMAP_ALL;
+	psCommand->u.sUnmapAll.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64UnMapAllCount++;
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapAll(psDeviceNode, COMMAND_TYPE_UNMAP_ALL, ui32AllocIndex);
+#endif
+}
+
+/* MapRangePack:
+ * Pack the given StartPage and Count values into the 75-bit representation
+ * in the MAP_RANGE command.
+ */
+static void MapRangePack(COMMAND_MAP_RANGE *psMapRange,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+
+{
+	IMG_UINT64 ui64Data;
+	IMG_UINT32 i;
+
+	/* we must encode the data into 54 bits:
+	 *   27 bits for the start page index
+	 *   27 bits for the range
+	*/
+
+	PVR_ASSERT(ui32StartPage <= MAP_RANGE_MAX_START);
+	PVR_ASSERT(ui32Count <= MAP_RANGE_MAX_RANGE);
+
+	ui32StartPage &= MAP_RANGE_MAX_START;
+	ui32Count &= MAP_RANGE_MAX_RANGE;
+
+	ui64Data = (((IMG_UINT64) ui32StartPage) << VM_RANGE_SHIFT) | ui32Count;
+
+	for (i = 0; i < ARRAY_SIZE(psMapRange->aui8Data); i++)
+	{
+		psMapRange->aui8Data[i] = ui64Data & 0xFF;
+		ui64Data >>= 8;
+	}
+}
+
+/* MapRangeUnpack:
+ * Unpack the StartPage and Count values from the 75-bit representation
+ * in the MAP_RANGE command.
+ */
+static void MapRangeUnpack(COMMAND_MAP_RANGE *psMapRange,
+						IMG_UINT32 *pui32StartPage,
+						IMG_UINT32 *pui32Count)
+{
+	IMG_UINT64 ui64Data = 0;
+	IMG_UINT32 i;
+
+	for (i = ARRAY_SIZE(psMapRange->aui8Data); i > 0; i--)
+	{
+		ui64Data <<= 8;
+		ui64Data |= (IMG_UINT64) psMapRange->aui8Data[i - 1];
+	}
+
+	*pui32StartPage =  (IMG_UINT32)(ui64Data >> VM_RANGE_SHIFT);
+	*pui32Count = (IMG_UINT32) ui64Data & (MAP_RANGE_MAX_RANGE);
+}
+
+/* InsertMapRangeCommand:
+ * Insert a MAP_RANGE command into the circular buffer with the given
+ * StartPage and Count values.
+ */
+static void InsertMapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32 ui32AllocIndex,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_MAP_RANGE;
+	psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64MapRangeCount++;
+
+	MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapRange(psDeviceNode,
+	                       COMMAND_TYPE_MAP_RANGE,
+	                       ui32AllocIndex,
+	                       ui32StartPage,
+	                       ui32Count);
+#endif
+}
+
+/* InsertUnmapRangeCommand:
+ * Insert a UNMAP_RANGE command into the circular buffer with the given
+ * StartPage and Count values.
+ */
+static void InsertUnmapRangeCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_UINT32 ui32AllocIndex,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32Count)
+{
+	COMMAND_WRAPPER *psCommand;
+	DEVICEMEM_HISTORY_DATA *psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	psCommand = AcquireCBSlot(psDevHData);
+
+	psCommand->ui8Type = COMMAND_TYPE_UNMAP_RANGE;
+	psCommand->u.sMapRange.uiAllocIndex = ui32AllocIndex;
+	psDevHData->sRecords.ui64UnMapRangeCount++;
+
+	MapRangePack(&psCommand->u.sMapRange, ui32StartPage, ui32Count);
+
+#if defined(PDUMP)
+	EmitPDumpMapUnmapRange(psDeviceNode,
+	                       COMMAND_TYPE_UNMAP_RANGE,
+	                       ui32AllocIndex,
+	                       ui32StartPage,
+	                       ui32Count);
+#endif
+}
+
+/* InsertAllocationToList:
+ * Helper function for the allocation list.
+ * Inserts the given allocation at the head of the list, whose current head is
+ * pointed to by pui32ListHead
+ */
+static void InsertAllocationToList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
+{
+	RECORD_ALLOCATION *psAlloc;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = IMG_CONTAINER_OF(pui32ListHead, DEVICEMEM_HISTORY_DATA,
+	                              sRecords.ui32AllocationsListHead);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	if (*pui32ListHead == END_OF_LIST)
+	{
+		/* list is currently empty, so just replace it */
+		*pui32ListHead = ui32Alloc;
+		psAlloc->ui32Next = psAlloc->ui32Prev = *pui32ListHead;
+	}
+	else
+	{
+		RECORD_ALLOCATION *psHeadAlloc;
+		RECORD_ALLOCATION *psTailAlloc;
+
+		psHeadAlloc = ALLOC_INDEX_TO_PTR(psDevHData, *pui32ListHead);
+		psTailAlloc = ALLOC_INDEX_TO_PTR(psDevHData, psHeadAlloc->ui32Prev);
+
+		/* make the new alloc point forwards to the previous head */
+		psAlloc->ui32Next = *pui32ListHead;
+		/* make the new alloc point backwards to the previous tail */
+		psAlloc->ui32Prev = psHeadAlloc->ui32Prev;
+
+		/* the head is now our new alloc */
+		*pui32ListHead = ui32Alloc;
+
+		/* the old head now points back to the new head */
+		psHeadAlloc->ui32Prev = *pui32ListHead;
+
+		/* the tail now points forward to the new head */
+		psTailAlloc->ui32Next = ui32Alloc;
+	}
+}
+
+static void InsertAllocationToBusyList(DEVICEMEM_HISTORY_DATA *psDevHData,
+                                       IMG_UINT32 ui32Alloc)
+{
+	InsertAllocationToList(&psDevHData->sRecords.ui32AllocationsListHead, ui32Alloc);
+}
+
+/* RemoveAllocationFromList:
+ * Helper function for the allocation list.
+ * Removes the given allocation from the list, whose head is
+ * pointed to by pui32ListHead
+ */
+static void RemoveAllocationFromList(IMG_UINT32 *pui32ListHead, IMG_UINT32 ui32Alloc)
+{
+	RECORD_ALLOCATION *psAlloc;
+
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = IMG_CONTAINER_OF(pui32ListHead, DEVICEMEM_HISTORY_DATA,
+	                              sRecords.ui32AllocationsListHead);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	/* if this is the only element in the list then just make the list empty */
+	if ((*pui32ListHead == ui32Alloc) && (psAlloc->ui32Next == ui32Alloc))
+	{
+		*pui32ListHead = END_OF_LIST;
+	}
+	else
+	{
+		RECORD_ALLOCATION *psPrev, *psNext;
+
+		psPrev = ALLOC_INDEX_TO_PTR(psDevHData, psAlloc->ui32Prev);
+		psNext = ALLOC_INDEX_TO_PTR(psDevHData, psAlloc->ui32Next);
+
+		/* remove the allocation from the list */
+		psPrev->ui32Next = psAlloc->ui32Next;
+		psNext->ui32Prev = psAlloc->ui32Prev;
+
+		/* if this allocation is the head then update the head */
+		if (*pui32ListHead == ui32Alloc)
+		{
+			*pui32ListHead = psAlloc->ui32Prev;
+		}
+	}
+}
+
+static void RemoveAllocationFromBusyList(DEVICEMEM_HISTORY_DATA *psDevHData, IMG_UINT32 ui32Alloc)
+{
+	RemoveAllocationFromList(&psDevHData->sRecords.ui32AllocationsListHead, ui32Alloc);
+}
+
+/* TouchBusyAllocation:
+ * Move the given allocation to the head of the list
+ */
+static void TouchBusyAllocation(DEVICEMEM_HISTORY_DATA *psDevHData, IMG_UINT32 ui32Alloc)
+{
+	RemoveAllocationFromBusyList(psDevHData, ui32Alloc);
+	InsertAllocationToBusyList(psDevHData, ui32Alloc);
+}
+
+/* GetOldestBusyAllocation:
+ * Returns the index of the oldest allocation in the MRU list
+ */
+static IMG_UINT32 GetOldestBusyAllocation(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 ui32Alloc;
+	RECORD_ALLOCATION *psAlloc;
+
+	if (psDevHData != NULL)
+	{
+		ui32Alloc = psDevHData->sRecords.ui32AllocationsListHead;
+	}
+	else
+	{
+		ui32Alloc = END_OF_LIST;	/* Default if no psDevHData */
+	}
+
+	if (ui32Alloc == END_OF_LIST)
+	{
+		return END_OF_LIST;
+	}
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	return psAlloc->ui32Prev;
+}
+
+static IMG_UINT32 GetFreeAllocation(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 ui32Alloc;
+
+	ui32Alloc = GetOldestBusyAllocation(psDevHData);
+
+	return ui32Alloc;
+}
+
+
+/* InitialiseAllocation:
+ * Initialise the given allocation structure with the given properties
+ */
+static void InitialiseAllocation(RECORD_ALLOCATION *psAlloc,
+							const IMG_CHAR *pszName,
+							IMG_UINT64 ui64Serial,
+							IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT32 ui32Log2PageSize)
+{
+	OSStringSafeCopy(psAlloc->szName, pszName, sizeof(psAlloc->szName));
+	psAlloc->ui64Serial = ui64Serial;
+	psAlloc->uiPID = uiPID;
+	psAlloc->sDevVAddr = sDevVAddr;
+	psAlloc->uiSize = uiSize;
+	psAlloc->ui32Log2PageSize = ui32Log2PageSize;
+	psAlloc->ui64CreationTime = OSClockns64();
+}
+
+/* CreateAllocation:
+ * Creates a new allocation with the given properties then outputs the
+ * index of the allocation
+ */
+static PVRSRV_ERROR CreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+							const IMG_CHAR *pszName,
+							IMG_UINT64 ui64Serial,
+							IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 *puiAllocationIndex)
+{
+	IMG_UINT32 ui32Alloc;
+	RECORD_ALLOCATION *psAlloc;
+
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	ui32Alloc = GetFreeAllocation(psDevHData);
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+	InitialiseAllocation(ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc),
+						pszName,
+						ui64Serial,
+						uiPID,
+						sDevVAddr,
+						uiSize,
+						ui32Log2PageSize);
+
+	/* put the newly initialised allocation at the front of the MRU list */
+	TouchBusyAllocation(psDevHData, ui32Alloc);
+
+	*puiAllocationIndex = ui32Alloc;
+
+#if defined(PDUMP)
+	EmitPDumpAllocation(psDeviceNode, ui32Alloc, psAlloc);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/* MatchAllocation:
+ * Tests if the allocation at the given index matches the supplied properties.
+ * Returns IMG_TRUE if it is a match, otherwise IMG_FALSE.
+ */
+static IMG_BOOL MatchAllocation(DEVICEMEM_HISTORY_DATA *psDevHData,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT64 ui64Serial,
+						IMG_DEV_VIRTADDR sDevVAddr,
+						IMG_DEVMEM_SIZE_T uiSize,
+						const IMG_CHAR *pszName,
+						IMG_UINT32 ui32Log2PageSize)
+{
+	RECORD_ALLOCATION *psAlloc;
+
+	if (psDevHData == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocationIndex);
+
+	return (psAlloc->ui64Serial == ui64Serial) &&
+	       (psAlloc->sDevVAddr.uiAddr == sDevVAddr.uiAddr) &&
+	       (psAlloc->uiSize == uiSize) &&
+	       (psAlloc->ui32Log2PageSize == ui32Log2PageSize) &&
+	       (OSStringNCompare(psAlloc->szName, pszName, DEVMEM_ANNOTATION_MAX_LEN) == 0);
+}
+
+/* FindOrCreateAllocation:
+ * Convenience function.
+ * Given a set of allocation properties (serial, DevVAddr, size, name, etc),
+ * this function will look for an existing record of this allocation and
+ * create the allocation if there is no existing record
+ */
+static PVRSRV_ERROR FindOrCreateAllocation(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_UINT32 ui32AllocationIndexHint,
+							IMG_UINT64 ui64Serial,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char *pszName,
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_PID uiPID,
+							IMG_UINT32 *pui32AllocationIndexOut,
+							IMG_BOOL *pbCreated)
+{
+	IMG_UINT32 ui32AllocationIndex;
+	PVRSRV_ERROR eError;
+
+	if (ui32AllocationIndexHint != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE)
+	{
+		IMG_BOOL bHaveAllocation;
+
+		/* first, try to match against the index given by the client.
+		 * if the caller provided a hint but the allocation record is no longer
+		 * there, it must have been purged, so go ahead and create a new allocation
+		 */
+		bHaveAllocation = MatchAllocation(DevmemFindDataFromDev(psDeviceNode),
+								ui32AllocationIndexHint,
+								ui64Serial,
+								sDevVAddr,
+								uiSize,
+								pszName,
+								ui32Log2PageSize);
+		if (bHaveAllocation)
+		{
+			*pbCreated = IMG_FALSE;
+			*pui32AllocationIndexOut = ui32AllocationIndexHint;
+			return PVRSRV_OK;
+		}
+	}
+
+	/* if there is no record of the allocation then we
+	 * create it now
+	 */
+	eError = CreateAllocation(psDeviceNode,
+					pszName,
+					ui64Serial,
+					uiPID,
+					sDevVAddr,
+					uiSize,
+					ui32Log2PageSize,
+					&ui32AllocationIndex);
+
+	if (eError == PVRSRV_OK)
+	{
+		*pui32AllocationIndexOut = ui32AllocationIndex;
+		*pbCreated = IMG_TRUE;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed to create record for allocation %s",
+								__func__,
+								pszName));
+	}
+
+	return eError;
+}
+
+/* GenerateMapUnmapCommandsForSparsePMR:
+ * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the PMR's
+ * current mapping table
+ *
+ * PMR: The PMR whose mapping table to read.
+ * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
+ * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
+ *
+ * This function goes through every page in the PMR's mapping table and looks for
+ * virtually contiguous ranges to record as being mapped or unmapped.
+ */
+static void GenerateMapUnmapCommandsForSparsePMR(PMR *psPMR,
+							IMG_UINT32 ui32AllocIndex,
+							IMG_BOOL bMap)
+{
+	PMR_MAPPING_TABLE *psMappingTable;
+	IMG_UINT32 ui32DonePages = 0;
+	IMG_UINT32 ui32NumPages;
+	IMG_UINT32 i;
+	IMG_BOOL bInARun = IMG_FALSE;
+	IMG_UINT32 ui32CurrentStart = 0;
+	IMG_UINT32 ui32RunCount = 0;
+
+	psMappingTable = PMR_GetMappingTable(psPMR);
+	ui32NumPages = psMappingTable->ui32NumPhysChunks;
+
+	if (ui32NumPages == 0)
+	{
+		/* nothing to do */
+		return;
+	}
+
+	for (i = 0; i < psMappingTable->ui32NumLogicalChunks; i++)
+	{
+		if (psMappingTable->aui32Translation[i] != TRANSLATION_INVALID)
+		{
+			if (!bInARun)
+			{
+				bInARun = IMG_TRUE;
+				ui32CurrentStart = i;
+				ui32RunCount = 1;
+			}
+			else
+			{
+				ui32RunCount++;
+			}
+		}
+
+		if (bInARun)
+		{
+			/* test if we need to end this current run and generate the command,
+			 * either because the next page is not virtually contiguous
+			 * to the current page, we have reached the maximum range,
+			 * or this is the last page in the mapping table
+			 */
+			if ((psMappingTable->aui32Translation[i] == TRANSLATION_INVALID) ||
+				(ui32RunCount == MAP_RANGE_MAX_RANGE) ||
+				(i == (psMappingTable->ui32NumLogicalChunks - 1)))
+			{
+				if (bMap)
+				{
+					InsertMapRangeCommand(PMR_DeviceNode(psPMR),
+					                      ui32AllocIndex,
+					                      ui32CurrentStart,
+					                      ui32RunCount);
+				}
+				else
+				{
+					InsertUnmapRangeCommand(PMR_DeviceNode(psPMR),
+					                        ui32AllocIndex,
+					                        ui32CurrentStart,
+					                        ui32RunCount);
+				}
+
+				ui32DonePages += ui32RunCount;
+
+				if (ui32DonePages == ui32NumPages)
+				{
+					break;
+				}
+
+				bInARun = IMG_FALSE;
+			}
+		}
+	}
+
+}
+
+/* GenerateMapUnmapCommandsForChangeList:
+ * Generate the MAP_RANGE or UNMAP_RANGE commands for the sparse PMR, using the
+ * list of page change (page map or page unmap) indices given.
+ *
+ * ui32NumPages: Number of pages which have changed.
+ * pui32PageList: List of indices of the pages which have changed.
+ * ui32AllocIndex: The allocation to attribute the MAP_RANGE/UNMAP range commands to.
+ * bMap: Set to TRUE for mapping or IMG_FALSE for unmapping
+ *
+ * This function goes through every page in the list and looks for
+ * virtually contiguous ranges to record as being mapped or unmapped.
+ */
+static void GenerateMapUnmapCommandsForChangeList(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_UINT32 ui32NumPages,
+							IMG_UINT32 *pui32PageList,
+							IMG_UINT32 ui32AllocIndex,
+							IMG_BOOL bMap)
+{
+	IMG_UINT32 i;
+	IMG_BOOL bInARun = IMG_FALSE;
+	IMG_UINT32 ui32CurrentStart = 0;
+	IMG_UINT32 ui32RunCount = 0;
+
+	for (i = 0; i < ui32NumPages; i++)
+	{
+		if (!bInARun)
+		{
+			bInARun = IMG_TRUE;
+			ui32CurrentStart = pui32PageList[i];
+		}
+
+		ui32RunCount++;
+
+		 /* we flush if:
+		 * - the next page in the list is not one greater than the current page
+		 * - this is the last page in the list
+		 * - we have reached the maximum range size
+		 */
+		if ((i == (ui32NumPages - 1)) ||
+			((pui32PageList[i] + 1) != pui32PageList[i + 1]) ||
+			(ui32RunCount == MAP_RANGE_MAX_RANGE))
+		{
+			if (bMap)
+			{
+				InsertMapRangeCommand(psDeviceNode,
+									ui32AllocIndex,
+									ui32CurrentStart,
+									ui32RunCount);
+			}
+			else
+			{
+				InsertUnmapRangeCommand(psDeviceNode,
+									ui32AllocIndex,
+									ui32CurrentStart,
+									ui32RunCount);
+			}
+
+			bInARun = IMG_FALSE;
+			ui32RunCount = 0;
+		}
+	}
+}
+
+/* DevicememHistoryMapKM:
+ * Entry point for when an allocation is mapped into the MMU GPU
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistoryMapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_BOOL bSparse = PMR_IsSparse(psPMR);
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	if (!bSparse)
+	{
+		InsertMapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
+	}
+	else
+	{
+		GenerateMapUnmapCommandsForSparsePMR(psPMR,
+								ui32AllocationIndex,
+								IMG_TRUE);
+	}
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+static void VRangeInsertMapUnmapCommands(PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_BOOL bMap,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							const IMG_CHAR *pszName)
+{
+	PVR_UNREFERENCED_PARAMETER(pszName);
+
+	while (ui32NumPages > 0)
+	{
+		IMG_UINT32 ui32PagesToAdd;
+
+		ui32PagesToAdd = MIN(ui32NumPages, MAP_RANGE_MAX_RANGE);
+
+		if (ui32StartPage > MAP_RANGE_MAX_START)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Cannot record %s range beginning at page "
+									"%u on allocation %s",
+									bMap ? "map" : "unmap",
+									ui32StartPage,
+									pszName));
+			return;
+		}
+
+		if (bMap)
+		{
+			InsertMapRangeCommand(psDeviceNode,
+								ui32AllocationIndex,
+								ui32StartPage,
+								ui32PagesToAdd);
+		}
+		else
+		{
+			InsertUnmapRangeCommand(psDeviceNode,
+								ui32AllocationIndex,
+								ui32StartPage,
+								ui32PagesToAdd);
+		}
+
+		ui32StartPage += ui32PagesToAdd;
+		ui32NumPages -= ui32PagesToAdd;
+	}
+}
+
+PVRSRV_ERROR DevicememHistoryMapVRangeKM(CONNECTION_DATA *psConnection,
+						PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_DEV_VIRTADDR sBaseDevVAddr,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32NumPages,
+						IMG_DEVMEM_SIZE_T uiAllocSize,
+						const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+						IMG_UINT32 ui32Log2PageSize,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+							ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(psDeviceNode,
+						ui32AllocationIndex,
+						0,
+						sBaseDevVAddr,
+						uiAllocSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	VRangeInsertMapUnmapCommands(psDeviceNode,
+						IMG_TRUE,
+						ui32AllocationIndex,
+						sBaseDevVAddr,
+						ui32StartPage,
+						ui32NumPages,
+						szName);
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+
+}
+
+PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(CONNECTION_DATA *psConnection,
+						PVRSRV_DEVICE_NODE *psDeviceNode,
+						IMG_DEV_VIRTADDR sBaseDevVAddr,
+						IMG_UINT32 ui32StartPage,
+						IMG_UINT32 ui32NumPages,
+						IMG_DEVMEM_SIZE_T uiAllocSize,
+						const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+						IMG_UINT32 ui32Log2PageSize,
+						IMG_UINT32 ui32AllocationIndex,
+						IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+							ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(psDeviceNode,
+						ui32AllocationIndex,
+						0,
+						sBaseDevVAddr,
+						uiAllocSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	VRangeInsertMapUnmapCommands(psDeviceNode,
+						IMG_FALSE,
+						ui32AllocationIndex,
+						sBaseDevVAddr,
+						ui32StartPage,
+						ui32NumPages,
+						szName);
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+
+
+/* DevicememHistoryUnmapKM:
+ * Entry point for when an allocation is unmapped from the MMU GPU
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistoryUnmapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_BOOL bSparse = PMR_IsSparse(psPMR);
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	if (!bSparse)
+	{
+		InsertUnmapAllCommand(PMR_DeviceNode(psPMR), ui32AllocationIndex);
+	}
+	else
+	{
+		GenerateMapUnmapCommandsForSparsePMR(psPMR,
+								ui32AllocationIndex,
+								IMG_FALSE);
+	}
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+}
+
+/* DevicememHistorySparseChangeKM:
+ * Entry point for when a sparse allocation is changed, such that some of the
+ * pages within the sparse allocation are mapped or unmapped.
+ *
+ * psPMR: The PMR to which the allocation belongs.
+ * ui32Offset: The offset within the PMR at which the allocation begins.
+ * sDevVAddr: The DevVAddr at which the allocation begins.
+ * szName: Annotation/name for the allocation.
+ * ui32Log2PageSize: Page size of the allocation, expressed in log2 form.
+ * ui32AllocPageCount: Number of pages which have been mapped.
+ * paui32AllocPageIndices: Indices of pages which have been mapped.
+ * ui32FreePageCount: Number of pages which have been unmapped.
+ * paui32FreePageIndices: Indices of pages which have been unmapped.
+ * ui32AllocationIndex: Allocation index as provided by the client.
+ *                      We will use this as a short-cut to find the allocation
+ *                      in our records.
+ * pui32AllocationIndexOut: An updated allocation index for the client.
+ *                          This may be a new value if we just created the
+ *                          allocation record.
+ */
+PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocPageCount,
+							IMG_UINT32 *paui32AllocPageIndices,
+							IMG_UINT32 ui32FreePageCount,
+							IMG_UINT32 *paui32FreePageIndices,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut)
+{
+	IMG_UINT64 ui64Serial = PMRInternalGetUID(psPMR);
+	IMG_PID uiPID = OSGetCurrentClientProcessIDKM();
+	PVRSRV_ERROR eError;
+	IMG_BOOL bCreated;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+
+	if (!PMRValidateSize((IMG_UINT64) ui32AllocPageCount << ui32Log2PageSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				 "PMR size exceeds limit #Chunks: %u ChunkSz %"IMG_UINT64_FMTSPECX"",
+				 ui32AllocPageCount,
+				 (IMG_UINT64) 1ULL << ui32Log2PageSize);
+		return PVRSRV_ERROR_PMR_TOO_LARGE;
+	}
+
+	if ((ui32AllocationIndex != DEVICEMEM_HISTORY_ALLOC_INDEX_NONE) &&
+		!CHECK_ALLOC_INDEX(ui32AllocationIndex))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid allocation index: %u",
+								__func__,
+								ui32AllocationIndex));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevHData = DevmemFindDataFromDev(PMR_DeviceNode(psPMR));
+
+	if (psDevHData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	eError = FindOrCreateAllocation(PMR_DeviceNode(psPMR),
+						ui32AllocationIndex,
+						ui64Serial,
+						sDevVAddr,
+						uiSize,
+						szName,
+						ui32Log2PageSize,
+						uiPID,
+						&ui32AllocationIndex,
+						&bCreated);
+
+	if ((eError == PVRSRV_OK) && !bCreated)
+	{
+		/* touch the allocation so it goes to the head of our MRU list */
+		TouchBusyAllocation(psDevHData, ui32AllocationIndex);
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to Find or Create allocation %s (%s)",
+									__func__,
+									szName,
+									PVRSRVGETERRORSTRING(eError)));
+		goto out_unlock;
+	}
+
+	GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
+							ui32AllocPageCount,
+							paui32AllocPageIndices,
+							ui32AllocationIndex,
+							IMG_TRUE);
+
+	GenerateMapUnmapCommandsForChangeList(PMR_DeviceNode(psPMR),
+							ui32FreePageCount,
+							paui32FreePageIndices,
+							ui32AllocationIndex,
+							IMG_FALSE);
+
+	InsertTimeStampCommand(OSClockns64(), PMR_DeviceNode(psPMR));
+
+	*pui32AllocationIndexOut = ui32AllocationIndex;
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return eError;
+
+}
+
+/* CircularBufferIterateStart:
+ * Initialise local state for iterating over the circular buffer
+ */
+static void CircularBufferIterateStart(DEVICEMEM_HISTORY_DATA *psHData, IMG_UINT32 *pui32Head, IMG_UINT32 *pui32Iter)
+{
+	*pui32Head = psHData->sRecords.ui32Head;
+
+	if (*pui32Head != 0)
+	{
+		*pui32Iter = *pui32Head - 1;
+	}
+	else
+	{
+		*pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
+	}
+}
+
+/* CircularBufferIteratePrevious:
+ * Iterate to the previous item in the circular buffer.
+ * This is called repeatedly to iterate over the whole circular buffer.
+ */
+static COMMAND_WRAPPER *CircularBufferIteratePrevious(DEVICEMEM_HISTORY_DATA *psHData,
+							IMG_UINT32 ui32Head,
+							IMG_UINT32 *pui32Iter,
+							COMMAND_TYPE *peType,
+							IMG_BOOL *pbLast)
+{
+	IMG_UINT8 *pui8Header;
+	COMMAND_WRAPPER *psOut = NULL;
+
+	psOut = psHData->sRecords.pasCircularBuffer + *pui32Iter;
+
+	pui8Header = (void *) psOut;
+
+	/* Check the command looks valid.
+	 * this condition should never happen, but check for it anyway
+	 * and try to handle it
+	 */
+	if (*pui8Header >= COMMAND_TYPE_COUNT)
+	{
+		/* invalid header detected. Circular buffer corrupted? */
+		PVR_DPF((PVR_DBG_ERROR, "CircularBufferIteratePrevious: "
+							"Invalid header: %u",
+							*pui8Header));
+		*pbLast = IMG_TRUE;
+		return NULL;
+	}
+
+	*peType = *pui8Header;
+
+	if (*pui32Iter != 0)
+	{
+		(*pui32Iter)--;
+	}
+	else
+	{
+		*pui32Iter = CIRCULAR_BUFFER_NUM_COMMANDS - 1;
+	}
+
+
+	/* inform the caller this is the last command if either we have reached
+	 * the head (where we started) or if we have reached an empty command,
+	 * which means we have covered all populated entries
+	 */
+	if ((*pui32Iter == ui32Head) || (*peType == COMMAND_TYPE_NONE))
+	{
+		/* this is the final iteration */
+		*pbLast = IMG_TRUE;
+	}
+
+	return psOut;
+}
+
+/* MapUnmapCommandGetInfo:
+ * Helper function to get the address and mapping information from a MAP_ALL, UNMAP_ALL,
+ * MAP_RANGE or UNMAP_RANGE command
+ */
+static void MapUnmapCommandGetInfo(DEVICEMEM_HISTORY_DATA *psHData,
+					COMMAND_WRAPPER *psCommand,
+					COMMAND_TYPE eType,
+					IMG_DEV_VIRTADDR *psDevVAddrStart,
+					IMG_DEV_VIRTADDR *psDevVAddrEnd,
+					IMG_BOOL *pbMap,
+					IMG_UINT32 *pui32AllocIndex)
+{
+	if ((eType == COMMAND_TYPE_MAP_ALL) || ((eType == COMMAND_TYPE_UNMAP_ALL)))
+	{
+		COMMAND_MAP_ALL *psMapAll = &psCommand->u.sMapAll;
+		RECORD_ALLOCATION *psAlloc;
+
+		*pbMap = (eType == COMMAND_TYPE_MAP_ALL);
+		*pui32AllocIndex = psMapAll->uiAllocIndex;
+
+		psAlloc = ALLOC_INDEX_TO_PTR(psHData, psMapAll->uiAllocIndex);
+
+		*psDevVAddrStart = psAlloc->sDevVAddr;
+		psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr + psAlloc->uiSize - 1;
+	}
+	else if ((eType == COMMAND_TYPE_MAP_RANGE) || ((eType == COMMAND_TYPE_UNMAP_RANGE)))
+	{
+		COMMAND_MAP_RANGE *psMapRange = &psCommand->u.sMapRange;
+		RECORD_ALLOCATION *psAlloc;
+		IMG_UINT32 ui32StartPage, ui32Count;
+
+		*pbMap = (eType == COMMAND_TYPE_MAP_RANGE);
+		*pui32AllocIndex = psMapRange->uiAllocIndex;
+
+		psAlloc = ALLOC_INDEX_TO_PTR(psHData, psMapRange->uiAllocIndex);
+
+		MapRangeUnpack(psMapRange, &ui32StartPage, &ui32Count);
+
+		psDevVAddrStart->uiAddr = psAlloc->sDevVAddr.uiAddr +
+				((1ULL << psAlloc->ui32Log2PageSize) * ui32StartPage);
+
+		psDevVAddrEnd->uiAddr = psDevVAddrStart->uiAddr +
+				((1ULL << psAlloc->ui32Log2PageSize) * ui32Count) - 1;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid command type: %u",
+								__func__,
+								eType));
+	}
+}
+
+void DevicememHistoryDumpRecordStats(PVRSRV_DEVICE_NODE *psDevNode,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile)
+{
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+	psDevHData = DevmemFindDataFromDev(psDevNode);
+
+	if (psDevHData)
+	{
+		PVR_DUMPDEBUG_LOG("    DevmemHistoryRecordStats -"
+							  " CBWC:%"IMG_UINT64_FMTSPEC
+							  " MAC:%"IMG_UINT64_FMTSPEC
+							  " UMAC:%"IMG_UINT64_FMTSPEC
+							  " MRC:%"IMG_UINT64_FMTSPEC
+							  " UMRC:%"IMG_UINT64_FMTSPEC
+							  " TSC:%"IMG_UINT64_FMTSPEC
+							  " MAX:%"IMG_UINT64_FMTSPEC
+							  " CHD:%u",
+							  psDevHData->sRecords.ui64CBWrapCount,
+							  psDevHData->sRecords.ui64MapAllCount,
+							  psDevHData->sRecords.ui64UnMapAllCount,
+							  psDevHData->sRecords.ui64MapRangeCount,
+							  psDevHData->sRecords.ui64UnMapRangeCount,
+							  psDevHData->sRecords.ui64TimeStampCount,
+							  (IMG_UINT64)CIRCULAR_BUFFER_NUM_COMMANDS,
+							  psDevHData->sRecords.ui32Head);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("    DevmemHistoryRecordStats - None");
+	}
+}
+
+/* DevicememHistoryQuery:
+ * Entry point for rgxdebug to look up addresses relating to a page fault
+ */
+IMG_BOOL DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
+                               DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+                               IMG_UINT32 ui32PageSizeBytes,
+                               IMG_BOOL bMatchAnyAllocInPage)
+{
+	IMG_UINT32 ui32Head, ui32Iter;
+	COMMAND_TYPE eType = COMMAND_TYPE_NONE;
+	COMMAND_WRAPPER *psCommand = NULL;
+	IMG_BOOL bLast = IMG_FALSE;
+	IMG_UINT64 ui64StartTime = OSClockns64();
+	IMG_UINT64 ui64TimeNs = 0;
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	/* initialise the results count for the caller */
+	psQueryOut->ui32NumResults = 0;
+	psQueryOut->ui64SearchCount = 0;
+
+	psDevHData = DevmemFindDataFromDev(psQueryIn->psDevNode);
+
+	if (psDevHData == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	DevicememHistoryLock(psDevHData);
+
+	/* if the search is constrained to a particular PID then we
+	 * first search the list of allocations to see if this
+	 * PID is known to us
+	 */
+	if (psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY)
+	{
+		IMG_UINT32 ui32Alloc;
+		ui32Alloc = psDevHData->sRecords.ui32AllocationsListHead;
+
+		while (ui32Alloc != END_OF_LIST)
+		{
+			RECORD_ALLOCATION *psAlloc;
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32Alloc);
+
+			if (psAlloc->uiPID == psQueryIn->uiPID)
+			{
+				goto found_pid;
+			}
+
+			if (ui32Alloc == psDevHData->sRecords.ui32AllocationsListHead)
+			{
+				/* gone through whole list */
+				break;
+			}
+		}
+
+		/* PID not found, so we do not have any suitable data for this
+		 * page fault
+		 */
+		goto out_unlock;
+	}
+
+found_pid:
+
+	CircularBufferIterateStart(psDevHData, &ui32Head, &ui32Iter);
+
+	while (!bLast)
+	{
+		psCommand = CircularBufferIteratePrevious(psDevHData, ui32Head, &ui32Iter, &eType, &bLast);
+		if (psCommand == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: CircularBufferIteratePrevious returned NULL psCommand",
+			         __func__));
+			return IMG_FALSE;
+		}
+
+		if (eType == COMMAND_TYPE_TIMESTAMP)
+		{
+			ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
+			continue;
+		}
+
+		if ((eType == COMMAND_TYPE_MAP_ALL) ||
+			(eType == COMMAND_TYPE_UNMAP_ALL) ||
+			(eType == COMMAND_TYPE_MAP_RANGE) ||
+			(eType == COMMAND_TYPE_UNMAP_RANGE))
+		{
+			RECORD_ALLOCATION *psAlloc;
+			IMG_DEV_VIRTADDR sAllocStartAddrOrig, sAllocEndAddrOrig;
+			IMG_DEV_VIRTADDR sAllocStartAddr, sAllocEndAddr;
+			IMG_BOOL bMap;
+			IMG_UINT32 ui32AllocIndex;
+
+			MapUnmapCommandGetInfo(psDevHData,
+			                psCommand,
+							eType,
+							&sAllocStartAddrOrig,
+							&sAllocEndAddrOrig,
+							&bMap,
+							&ui32AllocIndex);
+
+			sAllocStartAddr = sAllocStartAddrOrig;
+			sAllocEndAddr = sAllocEndAddrOrig;
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocIndex);
+
+			/* skip this command if we need to search within
+			 * a particular PID, and this allocation is not from
+			 * that PID
+			 */
+			if ((psQueryIn->uiPID != DEVICEMEM_HISTORY_PID_ANY) &&
+				(psAlloc->uiPID != psQueryIn->uiPID))
+			{
+				continue;
+			}
+
+			/* if the allocation was created after this event, then this
+			 * event must be for an old/removed allocation, so skip it
+			 */
+			if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
+			{
+				continue;
+			}
+
+			psQueryOut->ui64SearchCount++;
+
+			/* if the caller wants us to match any allocation in the
+			 * same page as the allocation then tweak the real start/end
+			 * addresses of the allocation here
+			 */
+			if (bMatchAnyAllocInPage)
+			{
+				sAllocStartAddr.uiAddr = sAllocStartAddr.uiAddr & ~(IMG_UINT64) (ui32PageSizeBytes - 1);
+				sAllocEndAddr.uiAddr = PVR_ALIGN(sAllocEndAddr.uiAddr, (IMG_UINT64)ui32PageSizeBytes);
+			}
+
+			if ((psQueryIn->sDevVAddr.uiAddr >= sAllocStartAddr.uiAddr) &&
+				(psQueryIn->sDevVAddr.uiAddr <  sAllocEndAddr.uiAddr))
+			{
+				DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult = &psQueryOut->sResults[psQueryOut->ui32NumResults];
+
+				OSStringSafeCopy(psResult->szString, psAlloc->szName, sizeof(psResult->szString));
+				psResult->sBaseDevVAddr = psAlloc->sDevVAddr;
+				psResult->uiSize = psAlloc->uiSize;
+				psResult->bMap = bMap;
+				psResult->ui64Age = _CalculateAge(ui64StartTime, ui64TimeNs, TIME_STAMP_MASK);
+				psResult->ui64When = ui64TimeNs;
+				/* write the responsible PID in the placeholder */
+				psResult->sProcessInfo.uiPID = psAlloc->uiPID;
+
+				if ((eType == COMMAND_TYPE_MAP_ALL) || (eType == COMMAND_TYPE_UNMAP_ALL))
+				{
+					psResult->bRange = IMG_FALSE;
+					psResult->bAll = IMG_TRUE;
+				}
+				else
+				{
+					psResult->bRange = IMG_TRUE;
+					MapRangeUnpack(&psCommand->u.sMapRange,
+										&psResult->ui32StartPage,
+										&psResult->ui32PageCount);
+					psResult->bAll = (psResult->ui32PageCount * (1U << psAlloc->ui32Log2PageSize))
+											== psAlloc->uiSize;
+					psResult->sMapStartAddr = sAllocStartAddrOrig;
+					psResult->sMapEndAddr = sAllocEndAddrOrig;
+				}
+
+				psQueryOut->ui32NumResults++;
+
+				if (psQueryOut->ui32NumResults == DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS)
+				{
+					break;
+				}
+			}
+		}
+	}
+
+out_unlock:
+	DevicememHistoryUnlock(psDevHData);
+
+	return psQueryOut->ui32NumResults > 0;
+}
+
+#if defined(SUPPORT_RGX)
+static void DeviceMemHistoryFmt(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN],
+							IMG_PID uiPID,
+							const IMG_CHAR *pszName,
+							const IMG_CHAR *pszAction,
+							IMG_DEV_VIRTADDR sDevVAddrStart,
+							IMG_DEV_VIRTADDR sDevVAddrEnd,
+							IMG_UINT64 ui64TimeNs)
+{
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+				/* PID NAME MAP/UNMAP MIN-MAX SIZE AbsUS AgeUS*/
+				"%04u %-40s %-10s "
+				IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC " "
+				"0x%08" IMG_UINT64_FMTSPECX " "
+				"%013" IMG_UINT64_FMTSPEC, /* 13 digits is over 2 hours of ns */
+				uiPID,
+				pszName,
+				pszAction,
+				sDevVAddrStart.uiAddr,
+				sDevVAddrEnd.uiAddr,
+				sDevVAddrEnd.uiAddr - sDevVAddrStart.uiAddr + 1,
+				ui64TimeNs);
+}
+
+static void DeviceMemHistoryFmtHeader(IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN])
+{
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN,
+				"%-4s %-40s %-6s   %10s   %10s   %8s %13s",
+				"PID",
+				"NAME",
+				"ACTION",
+				"ADDR MIN",
+				"ADDR MAX",
+				"SIZE",
+				"ABS NS");
+}
+
+static const char *CommandTypeToString(COMMAND_TYPE eType)
+{
+	switch (eType)
+	{
+		case COMMAND_TYPE_MAP_ALL:
+			return "MapAll";
+		case COMMAND_TYPE_UNMAP_ALL:
+			return "UnmapAll";
+		case COMMAND_TYPE_MAP_RANGE:
+			return "MapRange";
+		case COMMAND_TYPE_UNMAP_RANGE:
+			return "UnmapRange";
+		case COMMAND_TYPE_TIMESTAMP:
+			return "TimeStamp";
+		default:
+			return "???";
+	}
+}
+
+static void DevicememHistoryPrintAll(OSDI_IMPL_ENTRY *psEntry)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	IMG_UINT32 ui32Iter;
+	IMG_UINT32 ui32Head;
+	IMG_BOOL bLast = IMG_FALSE;
+	IMG_UINT64 ui64TimeNs = 0;
+	IMG_UINT64 ui64StartTime = OSClockns64();
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+
+	DeviceMemHistoryFmtHeader(szBuffer);
+	DIPrintf(psEntry, "%s\n", szBuffer);
+
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+
+	if (psDevHData == NULL)
+	{
+		return;
+	}
+
+	CircularBufferIterateStart(psDevHData, &ui32Head, &ui32Iter);
+
+	while (!bLast)
+	{
+		COMMAND_WRAPPER *psCommand;
+		COMMAND_TYPE eType = COMMAND_TYPE_NONE;
+
+		psCommand = CircularBufferIteratePrevious(psDevHData, ui32Head, &ui32Iter,
+		                                          &eType, &bLast);
+		if (psCommand == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: CircularBufferIteratePrevious returned NULL psCommand",
+			         __func__));
+			return;
+		}
+
+		if (eType == COMMAND_TYPE_TIMESTAMP)
+		{
+			ui64TimeNs = TimeStampUnpack(&psCommand->u.sTimeStamp);
+			continue;
+		}
+
+
+		if ((eType == COMMAND_TYPE_MAP_ALL) ||
+			(eType == COMMAND_TYPE_UNMAP_ALL) ||
+			(eType == COMMAND_TYPE_MAP_RANGE) ||
+			(eType == COMMAND_TYPE_UNMAP_RANGE))
+		{
+			RECORD_ALLOCATION *psAlloc;
+			IMG_DEV_VIRTADDR sDevVAddrStart, sDevVAddrEnd;
+			IMG_BOOL bMap;
+			IMG_UINT32 ui32AllocIndex;
+
+			MapUnmapCommandGetInfo(psDevHData,
+			                       psCommand,
+			                       eType,
+			                       &sDevVAddrStart,
+			                       &sDevVAddrEnd,
+			                       &bMap,
+			                       &ui32AllocIndex);
+
+			psAlloc = ALLOC_INDEX_TO_PTR(psDevHData, ui32AllocIndex);
+
+			if (DO_TIME_STAMP_MASK(psAlloc->ui64CreationTime) > ui64TimeNs)
+			{
+				/* if this event relates to an allocation we
+				 * are no longer tracking then do not print it
+				 */
+				continue;
+			}
+
+			DeviceMemHistoryFmt(szBuffer,
+			                    psAlloc->uiPID,
+			                    psAlloc->szName,
+			                    CommandTypeToString(eType),
+			                    sDevVAddrStart,
+			                    sDevVAddrEnd,
+			                    ui64TimeNs);
+
+			DIPrintf(psEntry, "%s\n", szBuffer);
+		}
+	}
+
+	DIPrintf(psEntry, "\nTimestamp reference: %013" IMG_UINT64_FMTSPEC "\n",
+	         ui64StartTime);
+}
+
+static int DevicememHistoryPrintAllWrapper(OSDI_IMPL_ENTRY *psEntry,
+                                           void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)DIGetPrivData(psEntry);
+	DEVICEMEM_HISTORY_DATA *psDevHData;
+
+	/* Get the backing store associated with the device. If we are
+	 * called before the device has been started (i.e. FW loaded)
+	 * then we haven't yet had this data allocated.
+	 * Return to provide a NULL data stream to the consumer.
+	 */
+	psDevHData = DevmemFindDataFromDev(psDeviceNode);
+	if (psDevHData == NULL)
+	{
+		return 0;
+	}
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	DevicememHistoryLock(psDevHData);
+	DevicememHistoryPrintAll(psEntry);
+	DevicememHistoryUnlock(psDevHData);
+
+	return 0;
+}
+#endif	/* defined(SUPPORT_RGX) */
+
+static PVRSRV_ERROR CreateRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	psDevHData->sRecords.pasAllocations =
+			OSAllocZMemNoStats(sizeof(RECORD_ALLOCATION) * ALLOCATION_LIST_NUM_ENTRIES);
+
+	PVR_RETURN_IF_NOMEM(psDevHData->sRecords.pasAllocations);
+
+	/* Allocated and initialise the circular buffer with zeros so every
+	 * command is initialised as a command of type COMMAND_TYPE_NONE. */
+	psDevHData->sRecords.pasCircularBuffer =
+			OSAllocZMemNoStats(sizeof(COMMAND_WRAPPER) * CIRCULAR_BUFFER_NUM_COMMANDS);
+
+	if (psDevHData->sRecords.pasCircularBuffer == NULL)
+	{
+		OSFreeMemNoStats(psDevHData->sRecords.pasAllocations);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void DestroyRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	OSFreeMemNoStats(psDevHData->sRecords.pasCircularBuffer);
+	OSFreeMemNoStats(psDevHData->sRecords.pasAllocations);
+}
+
+static void InitialiseRecords(DEVICEMEM_HISTORY_DATA *psDevHData)
+{
+	IMG_UINT32 i;
+
+	/* initialise the allocations list */
+
+	psDevHData->sRecords.pasAllocations[0].ui32Prev = ALLOCATION_LIST_NUM_ENTRIES - 1;
+	psDevHData->sRecords.pasAllocations[0].ui32Next = 1;
+
+	for (i = 1; i < ALLOCATION_LIST_NUM_ENTRIES; i++)
+	{
+		psDevHData->sRecords.pasAllocations[i].ui32Prev = i - 1;
+		psDevHData->sRecords.pasAllocations[i].ui32Next = i + 1;
+	}
+
+	psDevHData->sRecords.pasAllocations[ALLOCATION_LIST_NUM_ENTRIES - 1].ui32Next = 0;
+
+	psDevHData->sRecords.ui32AllocationsListHead = 0;
+}
+
+static void DevicememHistoryDevDeInitUnit(IMG_UINT32 uiUnit);
+static PVRSRV_ERROR DevicememHistoryDevInitUnit(IMG_UINT32 uiUnit);
+
+PVRSRV_ERROR DevicememHistoryInitKM(void)
+{
+	IMG_UINT32 ui;
+
+	/* Zero-fill the gapsDevicememHistoryData array entries */
+	for (ui = 0; ui < PVRSRV_MAX_DEVICES; ui++)
+	{
+		gapsDevicememHistoryData[ui] = NULL;
+	}
+
+	return PVRSRV_OK;
+}
+
+void DevicememHistoryDeInitKM(void)
+{
+	IMG_UINT32 uiUnit;
+
+	/* Iterate over all potential units and remove their data.
+	 * DI entry is removed by DevicememHistoryDeviceDestroy()
+	 */
+	for (uiUnit = 0; uiUnit < PVRSRV_MAX_DEVICES; uiUnit++)
+	{
+		DevicememHistoryDevDeInitUnit(uiUnit);
+	}
+}
+
+/* Allocate DEVICEMEM_HISTORY_DATA entry for the specified unit */
+static PVRSRV_ERROR DevicememHistoryDevInitUnit(IMG_UINT32 uiUnit)
+{
+	PVRSRV_ERROR eError;
+	DEVICEMEM_HISTORY_DATA *psDevicememHistoryData;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		PVR_LOG_RETURN_IF_FALSE(uiUnit < PVRSRV_MAX_DEVICES, "Invalid Unit",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Valid unit, try and allocate and fill all structure members */
+
+	psDevicememHistoryData = OSAllocZMemNoStats(sizeof(DEVICEMEM_HISTORY_DATA));
+	PVR_RETURN_IF_NOMEM(psDevicememHistoryData);
+
+	eError = OSLockCreate(&psDevicememHistoryData->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", err_lock);
+
+	eError = CreateRecords(psDevicememHistoryData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "CreateRecords", err_allocations);
+
+	InitialiseRecords(psDevicememHistoryData);
+
+	gapsDevicememHistoryData[uiUnit] = psDevicememHistoryData;
+
+	return PVRSRV_OK;
+
+err_allocations:
+	OSLockDestroy(psDevicememHistoryData->hLock);
+	psDevicememHistoryData->hLock = NULL;
+err_lock:
+	OSFreeMemNoStats(psDevicememHistoryData);
+	return eError;
+}
+
+/* Allocate DI entry for specified psDeviceNode */
+PVRSRV_ERROR DevicememHistoryDeviceCreate(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiUnit = psDeviceNode->sDevId.ui32InternalID;
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_DEBUG_INFO *psDevDebugInfo = &psDeviceNode->sDebugInfo;
+	DI_ITERATOR_CB sIterator = {.pfnShow = DevicememHistoryPrintAllWrapper};
+#endif
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Create the DI entry for the device's devicemem_history handle */
+
+
+#if defined(SUPPORT_RGX)
+	eError = DICreateEntry("devicemem_history", psDevDebugInfo->psGroup,
+	                       &sIterator, psDeviceNode,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psDevDebugInfo->psDevMemEntry);
+#endif	/* defined(SUPPORT_RGX) */
+
+	return eError;
+}
+
+/* Allocate the DEVICEMEM_HISTORY_DATA for specified psDeviceNode */
+PVRSRV_ERROR DevicememHistoryDeviceInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiUnit = psDeviceNode->sDevId.ui32InternalID;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = DevicememHistoryDevInitUnit(uiUnit);
+
+	return eError;
+}
+
+static void DevicememHistoryDevDeInitUnit(IMG_UINT32 uiUnit)
+{
+	DEVICEMEM_HISTORY_DATA *psDevicememHistoryData;
+
+	if (uiUnit >= PVRSRV_MAX_DEVICES)
+	{
+		return;
+	}
+
+	psDevicememHistoryData = gapsDevicememHistoryData[uiUnit];
+
+	if (psDevicememHistoryData == NULL)
+	{
+		return;
+	}
+
+	DestroyRecords(psDevicememHistoryData);
+
+	if (psDevicememHistoryData->hLock != NULL)
+	{
+		OSLockDestroy(psDevicememHistoryData->hLock);
+		psDevicememHistoryData->hLock = NULL;
+	}
+
+	OSFreeMemNoStats(psDevicememHistoryData);
+	gapsDevicememHistoryData[uiUnit] = NULL;
+}
+
+void DevicememHistoryDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_DEBUG_INFO *psDevDebugInfo = &psDeviceNode->sDebugInfo;
+
+	/* Remove the DI entry associated with this device */
+	DIDestroyEntry(psDevDebugInfo->psDevMemEntry);
+#endif	/* defined(SUPPORT_RGX) */
+
+}
diff --git a/drivers/gpu/drm/img-rogue/devicemem_history_server.h b/drivers/gpu/drm/img-rogue/devicemem_history_server.h
new file mode 100644
index 000000000000..81fe702371ac
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_history_server.h
@@ -0,0 +1,167 @@
+/*************************************************************************/ /*!
+@File           devicemem_history_server.h
+@Title          Resource Information abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Devicemem History functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_HISTORY_SERVER_H
+#define DEVICEMEM_HISTORY_SERVER_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxmem.h"
+#include "devicemem_utils.h"
+#include "connection_server.h"
+
+PVRSRV_ERROR DevicememHistoryInitKM(void);
+
+void DevicememHistoryDeInitKM(void);
+
+PVRSRV_ERROR DevicememHistoryDeviceInit(PVRSRV_DEVICE_NODE *psDevNode);
+PVRSRV_ERROR DevicememHistoryDeviceCreate(PVRSRV_DEVICE_NODE *psDevNode);
+void DevicememHistoryDeviceDestroy(PVRSRV_DEVICE_NODE *psDevNode);
+
+PVRSRV_ERROR DevicememHistoryMapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryUnmapKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryMapVRangeKM(CONNECTION_DATA *psConnection,
+							PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							IMG_DEVMEM_SIZE_T uiAllocSize,
+							const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *ui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistoryUnmapVRangeKM(CONNECTION_DATA *psConnection,
+							PVRSRV_DEVICE_NODE *psDeviceNode,
+							IMG_DEV_VIRTADDR sBaseDevVAddr,
+							IMG_UINT32 ui32StartPage,
+							IMG_UINT32 ui32NumPages,
+							IMG_DEVMEM_SIZE_T uiAllocSize,
+							const IMG_CHAR szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32Log2PageSize,
+							IMG_UINT32 ui32AllocationIndex,
+							IMG_UINT32 *ui32AllocationIndexOut);
+
+PVRSRV_ERROR DevicememHistorySparseChangeKM(PMR *psPMR,
+							IMG_UINT32 ui32Offset,
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEVMEM_SIZE_T uiSize,
+							const char szName[DEVMEM_ANNOTATION_MAX_LEN],
+							IMG_UINT32 ui32PageSize,
+							IMG_UINT32 ui32AllocPageCount,
+							IMG_UINT32 *paui32AllocPageIndices,
+							IMG_UINT32 ui32FreePageCount,
+							IMG_UINT32 *pauiFreePageIndices,
+							IMG_UINT32 AllocationIndex,
+							IMG_UINT32 *pui32AllocationIndexOut);
+
+/* used when the PID does not matter */
+#define DEVICEMEM_HISTORY_PID_ANY 0xFFFFFFFE
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_IN_
+{
+	IMG_PID uiPID;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	PVRSRV_DEVICE_NODE *psDevNode;
+} DEVICEMEM_HISTORY_QUERY_IN;
+
+/* Store up to 4 results for a lookup. In the case of the faulting page being
+ * re-mapped between the page fault occurring on HW and the page fault analysis
+ * being done, the second result entry will show the allocation being unmapped.
+ * A further 2 entries are added to cater for multiple buffers in the same page.
+ */
+#define DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS 4
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_OUT_RESULT_
+{
+	IMG_CHAR szString[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_DEV_VIRTADDR sBaseDevVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_BOOL bMap;
+	IMG_BOOL bRange;
+	IMG_BOOL bAll;
+	IMG_UINT64 ui64When;
+	IMG_UINT64 ui64Age;
+	/* info for sparse map/unmap operations (i.e. bRange=IMG_TRUE) */
+	IMG_UINT32 ui32StartPage;
+	IMG_UINT32 ui32PageCount;
+	IMG_DEV_VIRTADDR sMapStartAddr;
+	IMG_DEV_VIRTADDR sMapEndAddr;
+	RGXMEM_PROCESS_INFO sProcessInfo;
+} DEVICEMEM_HISTORY_QUERY_OUT_RESULT;
+
+typedef struct _DEVICEMEM_HISTORY_QUERY_OUT_
+{
+	IMG_UINT32 ui32NumResults;
+	IMG_UINT64 ui64SearchCount;
+	/* result 0 is the newest */
+	DEVICEMEM_HISTORY_QUERY_OUT_RESULT sResults[DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS];
+} DEVICEMEM_HISTORY_QUERY_OUT;
+
+void DevicememHistoryDumpRecordStats(PVRSRV_DEVICE_NODE *psDevNode,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile);
+
+IMG_BOOL
+DevicememHistoryQuery(DEVICEMEM_HISTORY_QUERY_IN *psQueryIn,
+                      DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+                      IMG_UINT32 ui32PageSizeBytes,
+                      IMG_BOOL bMatchAnyAllocInPage);
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/devicemem_pdump.h b/drivers/gpu/drm/img-rogue/devicemem_pdump.h
new file mode 100644
index 000000000000..09b28afe7e51
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_pdump.h
@@ -0,0 +1,363 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management PDump internal
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal interface to PDump device memory management
+                functions that are shared between client and server code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_PDUMP_H
+#define DEVICEMEM_PDUMP_H
+
+#include "devicemem.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+
+#if defined(PDUMP)
+/*
+ * DevmemPDumpLoadMem()
+ *
+ * takes a memory descriptor, offset, and size, and takes the current contents
+ * of the memory at that location and writes it to the prm pdump file, and
+ * emits a pdump LDB to load the data from that file.  The intention here is
+ * that the contents of the simulated buffer upon pdump playback will be made
+ * to be the same as they are when this command is run, enabling pdump of
+ * cases where the memory has been modified externally, i.e. by the host cpu
+ * or by a third party.
+ */
+void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpLoadZeroMem()
+ *
+ * As DevmemPDumpLoadMem() but the PDump allocation will be populated with
+ * zeros from the zero page in the parameter stream
+ */
+void
+DevmemPDumpLoadZeroMem(DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpLoadMemValue32()
+ *
+ * As above but dumps the value at a dword-aligned address in plain text to
+ * the pdump script2 file. Useful for patching a buffer at pdump playback by
+ * simply editing the script output file.
+ *
+ * (The same functionality can be achieved by the above function but the
+ *  binary PARAM file must be patched in that case.)
+ */
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpMemValue64()
+ *
+ * As above but dumps the 64bit-value at a dword-aligned address in plain text
+ * to the pdump script2 file. Useful for patching a buffer at pdump playback by
+ * simply editing the script output file.
+ *
+ * (The same functionality can be achieved by the above function but the
+ *  binary PARAM file must be patched in that case.)
+ */
+IMG_INTERNAL void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * DevmemPDumpPageCatBaseToSAddr()
+ *
+ * Returns the symbolic address of a piece of memory represented by an offset
+ * into the mem descriptor.
+ */
+PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC *psMemDesc,
+                              IMG_DEVMEM_OFFSET_T *puiMemOffset,
+                              IMG_CHAR *pszName,
+                              IMG_UINT32 ui32Size);
+
+/*
+ * DevmemPDumpSaveToFile()
+ *
+ * Emits a pdump SAB to cause the current contents of the memory to be written
+ * to the given file during playback
+ */
+void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset);
+
+/*
+ * DevmemPDumpSaveToFileVirtual()
+ *
+ * Emits a pdump SAB, just like DevmemPDumpSaveToFile(), but uses the virtual
+ * address and device MMU context to cause the pdump player to traverse the
+ * MMU page tables itself.
+ */
+void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags);
+
+/*
+ * DevmemPDumpDataDescriptor()
+ *
+ * Emits a pdump CMD:OutputData, using the virtual address and device MMU
+ * context. Provides more flexibility than a pdump SAB because metadata can
+ * be passed to an external pdump player library via the command header.
+ */
+void
+DevmemPDumpDataDescriptor(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_DEVMEM_SIZE_T uiSize,
+                          const IMG_CHAR *pszFilename,
+                          IMG_UINT32 ui32HeaderType,
+                          IMG_UINT32 ui32ElementType,
+                          IMG_UINT32 ui32ElementCount,
+                          IMG_UINT32 ui32PdumpFlags);
+
+
+/*
+ *
+ * DevmemPDumpDevmemPol32()
+ *
+ * Writes a PDump 'POL' command to wait for a masked 32-bit memory location to
+ * become the specified value.
+ */
+PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags);
+
+#if defined(__KERNEL__)
+/*
+ *
+ * DevmemPDumpDevmemCheck32()
+ *
+ * Writes a PDump 'POL' command to run a single-shot check for a masked
+ * 32-bit memory location to match the specified value.
+ */
+PVRSRV_ERROR
+DevmemPDumpDevmemCheck32(const DEVMEM_MEMDESC *psMemDesc,
+                         IMG_DEVMEM_OFFSET_T uiOffset,
+                         IMG_UINT32 ui32Value,
+                         IMG_UINT32 ui32Mask,
+                         PDUMP_POLL_OPERATOR eOperator,
+                         PDUMP_FLAGS_T ui32PDumpFlags);
+#endif
+
+/*
+ * DevmemPDumpCBP()
+ *
+ * Polls for space in circular buffer. Reads the read offset from memory and
+ * waits until there is enough space to write the packet.
+ *
+ * psMemDesc     - MemDesc which contains the read offset
+ * uiReadOffset  - Offset into MemDesc to the read offset
+ * uiWriteOffset - Current write offset
+ * uiPacketSize  - Size of packet to write
+ * uiBufferSize  - Size of circular buffer
+ */
+PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize);
+
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMem)
+#endif
+static INLINE void
+DevmemPDumpLoadMem(DEVMEM_MEMDESC *psMemDesc,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMemValue32)
+#endif
+static INLINE void
+DevmemPDumpLoadMemValue32(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpLoadMemValue64)
+#endif
+static INLINE void
+DevmemPDumpLoadMemValue64(DEVMEM_MEMDESC *psMemDesc,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT64 ui64Value,
+                          PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui64Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpPageCatBaseToSAddr)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpPageCatBaseToSAddr(DEVMEM_MEMDESC *psMemDesc,
+                              IMG_DEVMEM_OFFSET_T *puiMemOffset,
+                              IMG_CHAR *pszName,
+                              IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(puiMemOffset);
+	PVR_UNREFERENCED_PARAMETER(pszName);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpSaveToFile)
+#endif
+static INLINE void
+DevmemPDumpSaveToFile(DEVMEM_MEMDESC *psMemDesc,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_DEVMEM_SIZE_T uiSize,
+                      const IMG_CHAR *pszFilename,
+                      IMG_UINT32 uiFileOffset)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(uiFileOffset);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpSaveToFileVirtual)
+#endif
+static INLINE void
+DevmemPDumpSaveToFileVirtual(DEVMEM_MEMDESC *psMemDesc,
+                             IMG_DEVMEM_OFFSET_T uiOffset,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             const IMG_CHAR *pszFilename,
+                             IMG_UINT32 ui32FileOffset,
+                             IMG_UINT32 ui32PdumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PdumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpDevmemPol32)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpDevmemPol32(const DEVMEM_MEMDESC *psMemDesc,
+                       IMG_DEVMEM_OFFSET_T uiOffset,
+                       IMG_UINT32 ui32Value,
+                       IMG_UINT32 ui32Mask,
+                       PDUMP_POLL_OPERATOR eOperator,
+                       PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemPDumpCBP)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemPDumpCBP(const DEVMEM_MEMDESC *psMemDesc,
+               IMG_DEVMEM_OFFSET_T uiReadOffset,
+               IMG_DEVMEM_OFFSET_T uiWriteOffset,
+               IMG_DEVMEM_SIZE_T uiPacketSize,
+               IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psMemDesc);
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+#endif /* DEVICEMEM_PDUMP_H */
diff --git a/drivers/gpu/drm/img-rogue/devicemem_server.c b/drivers/gpu/drm/img-rogue/devicemem_server.c
new file mode 100644
index 000000000000..67e7c4fafbda
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_server.c
@@ -0,0 +1,2951 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server-side component of the Device Memory Management.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* our exported API */
+#include "devicemem_server.h"
+#include "devicemem_utils.h"
+#include "devicemem.h"
+
+#include "device.h" /* For device node */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "mmu_common.h"
+#include "pdump_km.h"
+#include "pmr.h"
+#include "physmem.h"
+#include "pdumpdesc.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+#include "lock.h"
+
+#include "pvrsrv.h" /* for PVRSRVGetPVRSRVData() */
+
+#define DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE (1 << 0)
+#define DEVMEMHEAP_REFCOUNT_MIN 1
+#define DEVMEMHEAP_REFCOUNT_MAX IMG_INT32_MAX
+#define DEVMEMRESERVATION_REFCOUNT_MIN 1
+#define DEVMEMRESERVATION_REFCOUNT_MAX IMG_INT32_MAX
+#define DEVMEMCTX_REFCOUNT_MIN 1
+#define DEVMEMCTX_REFCOUNT_MAX IMG_INT32_MAX
+
+struct _DEVMEMINT_CTX_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* MMU common code needs to have a context. There's a one-to-one
+	   correspondence between device memory context and MMU context,
+	   but we have the abstraction here so that we don't need to care
+	   what the MMU does with its context, and the MMU code need not
+	   know about us at all. */
+	MMU_CONTEXT *psMMUContext;
+
+	/* This handle is for devices that require notification when a new
+	   memory context is created and they need to store private data that
+	   is associated with the context. */
+	IMG_HANDLE hPrivData;
+
+	/* The following tracks UM applications that need to be notified of a
+	 * page fault.
+	 * Access to this list is protected by lock defined on a device node:
+	 * PVRSRV_DEVICE_NODE::hPageFaultNotifyLock. */
+	DLLIST_NODE sProcessNotifyListHead;
+	/* The following is a node for the list of registered devmem contexts.
+	 * Access to this list is protected by lock defined on a device node:
+	 * PVRSRV_DEVICE_NODE::hPageFaultNotifyLock. */
+	DLLIST_NODE sPageFaultNotifyListElem;
+
+	/* Device virtual address of a page fault on this context */
+	IMG_DEV_VIRTADDR sFaultAddress;
+
+	/* Bitfield stating which heaps were created on this context. */
+	IMG_UINT64 uiCreatedHeaps;
+
+	/* Context's reference count */
+	ATOMIC_T hRefCount;
+
+	/* General purpose flags */
+	IMG_UINT32 ui32Flags;
+};
+
+struct _DEVMEMINT_CTX_EXPORT_
+{
+	DEVMEMINT_CTX *psDevmemCtx;
+	PMR *psPMR;
+	ATOMIC_T hRefCount;
+	DLLIST_NODE sNode;
+};
+
+struct _DEVMEMINT_HEAP_
+{
+	struct _DEVMEMINT_CTX_ *psDevmemCtx;
+	IMG_DEV_VIRTADDR sBaseAddr;
+	IMG_DEV_VIRTADDR sLastAddr;
+
+	/* Private data for callback functions */
+	IMG_HANDLE hPrivData;
+
+	/* Callback function init */
+	PFN_HEAP_INIT pfnInit;
+
+	/* Callback function deinit */
+	PFN_HEAP_DEINIT pfnDeInit;
+
+	/* Heap's reference count */
+	ATOMIC_T uiRefCount;
+
+	/* Page shift of the heap */
+	IMG_UINT32 uiLog2PageSize;
+
+	/* Copy of the heap index from Device Heap Configuration module */
+	IMG_UINT32 uiHeapIndex;
+};
+
+struct _DEVMEMINT_RESERVATION_
+{
+	struct _DEVMEMINT_HEAP_ *psDevmemHeap;
+	IMG_DEV_VIRTADDR sBase;
+	IMG_DEVMEM_SIZE_T uiLength;
+	/* lock used to guard against potential race when freeing reservation */
+	POS_LOCK hLock;
+	IMG_INT32 i32RefCount;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;
+
+	/* We keep a reference to the single PMR associated with this reservation
+	 * once mapped. When creating the reservation this is null. Used in
+	 * ChangeSparse to validate parameters. We could have a sparse PMR with
+	 * no backing and have it mapped to a reservation.
+	 */
+	PMR *psMappedPMR;
+
+	/* Array of bitfields of size `uiNumPages / MAP_MASK_SHIFT`.
+	 * This array represents the mapping between a PMR (psMappedPMR) and
+	 * the reservation. Each bit represents an index of a physical page - a value
+	 * 1 means the page is mapped and vice versa.
+	 */
+	IMG_UINT8 *pui8Map;
+	#define MAP_MASK_SHIFT 3
+};
+
+struct _DEVMEMINT_MAPPING_
+{
+	struct _DEVMEMINT_RESERVATION_ *psReservation;
+	PMR *psPMR;
+	IMG_UINT32 uiNumPages;
+};
+
+/*! Object representing a virtual range reservation and mapping between
+ * the virtual range and a set of PMRs.
+ *
+ * The physical allocations may be mapped entirely or partially to the entire
+ * or partial virtual range. */
+struct _DEVMEMXINT_RESERVATION_
+{
+	/*! Pointer to a device memory heap this reservation is made on. */
+	struct _DEVMEMINT_HEAP_ *psDevmemHeap;
+	/*! Base device virtual address of this reservation. */
+	IMG_DEV_VIRTADDR sBase;
+	/*! Size of this reservation (in bytes). */
+	IMG_DEVMEM_SIZE_T uiLength;
+	/*! Lock for protecting concurrent operations on the mapping. */
+	POS_LOCK hLock;
+	/*! Array of PMRs of size `uiNumPages`. This array represents how the
+	 *  physical memory is mapped to the virtual range. Each entry in the array
+	 *  represents to one device page which means that one PMR may be spread
+	 *  across many indices. */
+	PMR **ppsPMR;
+};
+
+struct _DEVMEMINT_PF_NOTIFY_
+{
+	IMG_UINT32  ui32PID;
+	DLLIST_NODE sProcessNotifyListElem;
+};
+
+/** Computes division using log2 of divisor. */
+#define LOG2_DIV(x, log2) ((x) >> (log2))
+
+/** Computes modulo of a power of 2. */
+#define LOG2_MOD(x, log2) ((x) & ((1 << (log2)) - 1))
+
+static INLINE IMG_UINT32 _DevmemReservationPageCount(DEVMEMINT_RESERVATION *psRsrv);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationIsIndexMapped
+@Description    Checks whether a particular index in the reservation has been
+                mapped to a page in psMappedPMR.
+
+@Return         IMG_TRUE if mapped or IMG_FALSE if not.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntReservationIsIndexMapped(DEVMEMINT_RESERVATION *psReservation,
+                                                         IMG_UINT32 ui32Index)
+{
+	IMG_UINT32 ui32MapIndex = LOG2_DIV(ui32Index, MAP_MASK_SHIFT);
+
+	PVR_ASSERT(psReservation != NULL);
+	PVR_ASSERT(ui32Index < _DevmemReservationPageCount(psReservation));
+
+	return BIT_ISSET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationSetMappingIndex
+@Description    Sets an index of the reservation map to indicate a mapped or
+                unmapped PMR page.
+
+@Note           The reservations hLock must be acquired before calling this
+                function.
+
+@Return         None
+*/ /**************************************************************************/
+static void DevmemIntReservationSetMappingIndex(DEVMEMINT_RESERVATION *psReservation,
+                                                IMG_UINT32 ui32Index,
+                                                IMG_BOOL bMap)
+{
+	IMG_UINT32 ui32MapIndex = LOG2_DIV(ui32Index, MAP_MASK_SHIFT);
+
+	PVR_ASSERT(psReservation != NULL);
+	PVR_ASSERT(ui32Index < _DevmemReservationPageCount(psReservation));
+
+	if (bMap)
+	{
+		BIT_SET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+	}
+	else
+	{
+		BIT_UNSET(psReservation->pui8Map[ui32MapIndex], LOG2_MOD(ui32Index, MAP_MASK_SHIFT));
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxAcquire
+@Description    Acquire a reference to the provided device memory context.
+@Return         IMG_TRUE on success, IMG_FALSE on overflow
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntCtxAcquire(DEVMEMINT_CTX *psDevmemCtx)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psDevmemCtx->hRefCount, 1,
+	                                        DEVMEMCTX_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMCTX_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "context, reference count has overflowed.", __func__));
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxRelease
+@Description    Release the reference to the provided device memory context.
+                If this is the last reference which was taken then the
+                memory context will be freed.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void DevmemIntCtxRelease(DEVMEMINT_CTX *psDevmemCtx)
+{
+	if (OSAtomicDecrement(&psDevmemCtx->hRefCount) == 0)
+	{
+		/* The last reference has gone, destroy the context */
+		PVRSRV_DEVICE_NODE *psDevNode = psDevmemCtx->psDevNode;
+		DLLIST_NODE *psNode, *psNodeNext;
+
+		/* Protect removing the node from the list in case it's being accessed
+		 * by DevmemIntPFNotify(). */
+		OSWRLockAcquireWrite(psDevNode->hPageFaultNotifyLock);
+		/* If this context is in the list registered for a debugger, remove
+		 * from that list */
+		if (dllist_node_is_in_list(&psDevmemCtx->sPageFaultNotifyListElem))
+		{
+			dllist_remove_node(&psDevmemCtx->sPageFaultNotifyListElem);
+		}
+		/* It should be safe to release the lock here (as long as
+		 * DevmemIntPFNotify() protects accessing memory contexts as well). */
+		OSWRLockReleaseWrite(psDevNode->hPageFaultNotifyLock);
+
+		/* If there are any PIDs registered for page fault notification.
+		 * Loop through the registered PIDs and free each one */
+		dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+		{
+			DEVMEMINT_PF_NOTIFY *psNotifyNode =
+				IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+			dllist_remove_node(psNode);
+			OSFreeMem(psNotifyNode);
+		}
+
+		if (psDevNode->pfnUnregisterMemoryContext)
+		{
+			psDevNode->pfnUnregisterMemoryContext(psDevmemCtx->hPrivData);
+		}
+		MMU_ContextDestroy(psDevmemCtx->psMMUContext);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Freed memory context %p",
+				 __func__, psDevmemCtx));
+		OSFreeMem(psDevmemCtx);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapAcquire
+@Description    Acquire a reference to the provided device memory heap.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+static INLINE IMG_BOOL DevmemIntHeapAcquire(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	IMG_INT32 iOldValue = OSAtomicAddUnless(&psDevmemHeap->uiRefCount, 1,
+	                                        DEVMEMHEAP_REFCOUNT_MAX);
+
+	if (iOldValue == DEVMEMHEAP_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "heap, reference count has overflowed.", __func__));
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapRelease
+@Description    Release the reference to the provided device memory heap.
+                If this is the last reference which was taken then the
+                memory context will be freed.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void DevmemIntHeapRelease(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	IMG_INT32 iOldValue = OSAtomicSubtractUnless(&psDevmemHeap->uiRefCount, 1,
+	                                             DEVMEMHEAP_REFCOUNT_MIN);
+
+	if (iOldValue == DEVMEMHEAP_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "heap, reference count has underflowed.", __func__));
+	}
+}
+
+static IMG_BOOL DevmemIntReservationAcquireUnlocked(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+#if defined(DEBUG)
+	if (!OSLockIsLocked(psDevmemReservation->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Reservation is not locked", __func__));
+		return IMG_FALSE;
+	}
+#endif
+
+	if (psDevmemReservation->i32RefCount == DEVMEMRESERVATION_REFCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to acquire the device memory "
+		         "reservation, reference count has overflowed.", __func__));
+
+		return IMG_FALSE;
+	}
+
+	psDevmemReservation->i32RefCount++;
+
+	return IMG_TRUE;
+}
+
+static void DevmemIntReservationReleaseUnlocked(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+#if defined(DEBUG)
+	if (!OSLockIsLocked(psDevmemReservation->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Reservation is not locked", __func__));
+		return;
+	}
+#endif
+
+	if (psDevmemReservation->i32RefCount == DEVMEMRESERVATION_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s(): Failed to release the device memory "
+		         "reservation, reference count has underflowed.", __func__));
+
+		/* for better debugging */
+		PVR_ASSERT(psDevmemReservation->i32RefCount == DEVMEMRESERVATION_REFCOUNT_MIN);
+
+		return;
+	}
+
+	psDevmemReservation->i32RefCount--;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationAcquire
+@Description    Acquire a reference to the provided device memory reservation.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+IMG_BOOL DevmemIntReservationAcquire(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+	IMG_BOOL bSuccess;
+
+	OSLockAcquire(psDevmemReservation->hLock);
+	bSuccess = DevmemIntReservationAcquireUnlocked(psDevmemReservation);
+	OSLockRelease(psDevmemReservation->hLock);
+
+	return bSuccess;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationRelease
+@Description    Release the reference to the provided device memory reservation.
+                If this is the last reference which was taken then the
+                reservation will be freed.
+@Return         None.
+*/ /**************************************************************************/
+void DevmemIntReservationRelease(DEVMEMINT_RESERVATION *psDevmemReservation)
+{
+	OSLockAcquire(psDevmemReservation->hLock);
+	DevmemIntReservationReleaseUnlocked(psDevmemReservation);
+	OSLockRelease(psDevmemReservation->hLock);
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetImportHandle
+@Description    For given exportable memory descriptor returns PMR handle.
+@Return         Memory is exportable - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+                            IMG_HANDLE *phImport)
+{
+	PVRSRV_ERROR eError;
+
+	if ((GetImportProperties(psMemDesc->psImport) & DEVMEM_PROPERTIES_EXPORTABLE) == 0)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION, e0);
+	}
+
+	/* A new handle means a new import tracking the PMR.
+	 * Hence the source PMR memory layout should be marked fixed
+	 * to make sure the importer view of the memory is the same as
+	 * the exporter throughout its lifetime */
+	PMR_SetLayoutFixed((PMR *)psMemDesc->psImport->hPMR, IMG_TRUE);
+
+	*phImport = psMemDesc->psImport->hPMR;
+	return PVRSRV_OK;
+
+e0:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetHeapHandle
+@Description    For given reservation returns the Heap handle.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetHeapHandle(DEVMEMINT_RESERVATION *psReservation,
+                          IMG_HANDLE *phHeap)
+{
+	if (psReservation == NULL || phHeap == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*phHeap = psReservation->psDevmemHeap;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetContext
+@Description    For given heap returns the context.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetContext(DEVMEMINT_HEAP *psDevmemHeap,
+					   DEVMEMINT_CTX **ppsDevmemCtxPtr)
+{
+	if (psDevmemHeap == NULL || ppsDevmemCtxPtr == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*ppsDevmemCtxPtr = psDevmemHeap->psDevmemCtx;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemServerGetPrivData
+@Description    For given context returns the private data handle.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemServerGetPrivData(DEVMEMINT_CTX *psDevmemCtx,
+						IMG_HANDLE *phPrivData)
+{
+	if (psDevmemCtx == NULL || phPrivData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*phPrivData = psDevmemCtx->hPrivData;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxCreate
+@Description    Creates and initialises a device memory context.
+@Return         valid Device Memory context handle - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntCtxCreate(CONNECTION_DATA *psConnection,
+                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                   IMG_BOOL bKernelFWMemoryCtx,
+                   DEVMEMINT_CTX **ppsDevmemCtxPtr,
+                   IMG_HANDLE *hPrivData,
+                   IMG_UINT32 *pui32CPUCacheLineSize)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_CTX *psDevmemCtx;
+	IMG_HANDLE hPrivDataInt = NULL;
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = psDeviceNode->pfnGetMMUDeviceAttributes(psDeviceNode,
+	                                                                           bKernelFWMemoryCtx);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s", __func__));
+
+	/* Only allow request for a kernel context that comes from a direct bridge
+	 * (psConnection == NULL). Only the FW/KM Ctx is created over the direct bridge. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(!bKernelFWMemoryCtx || psConnection == NULL,
+	                                "bKernelFWMemoryCtx && psConnection");
+
+	/*
+	 * Ensure that we are safe to perform unaligned accesses on memory
+	 * we mark write-combine, as the compiler might generate
+	 * instructions operating on this memory which require this
+	 * assumption to be true.
+	 */
+	PVR_ASSERT(OSIsWriteCombineUnalignedSafe());
+
+	/* allocate a Devmem context */
+	psDevmemCtx = OSAllocMem(sizeof(*psDevmemCtx));
+	PVR_LOG_GOTO_IF_NOMEM(psDevmemCtx, eError, fail_alloc);
+
+	OSAtomicWrite(&psDevmemCtx->hRefCount, 1);
+	psDevmemCtx->psDevNode = psDeviceNode;
+
+	/* Call down to MMU context creation */
+
+	eError = MMU_ContextCreate(psConnection,
+	                           psDeviceNode,
+	                           &psDevmemCtx->psMMUContext,
+	                           psMMUDevAttrs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_ContextCreate", fail_mmucontext);
+
+	if (psDeviceNode->pfnRegisterMemoryContext)
+	{
+		eError = psDeviceNode->pfnRegisterMemoryContext(psDeviceNode, psDevmemCtx->psMMUContext, &hPrivDataInt);
+		PVR_LOG_GOTO_IF_ERROR(eError, "pfnRegisterMemoryContext", fail_register);
+	}
+
+	/* Store the private data as it is required to unregister the memory context */
+	psDevmemCtx->hPrivData = hPrivDataInt;
+	*hPrivData = hPrivDataInt;
+	*ppsDevmemCtxPtr = psDevmemCtx;
+
+	/* Pass the CPU cache line size through the bridge to the user mode as it can't be queried in user mode.*/
+	*pui32CPUCacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+
+	/* Initialise the PID notify list */
+	dllist_init(&(psDevmemCtx->sProcessNotifyListHead));
+	psDevmemCtx->sPageFaultNotifyListElem.psNextNode = NULL;
+	psDevmemCtx->sPageFaultNotifyListElem.psPrevNode = NULL;
+
+	/* Initialise page fault address */
+	psDevmemCtx->sFaultAddress.uiAddr = 0ULL;
+
+	/* Initialise flags */
+	psDevmemCtx->ui32Flags = 0;
+
+	psDevmemCtx->uiCreatedHeaps = 0;
+
+	return PVRSRV_OK;
+
+fail_register:
+	MMU_ContextDestroy(psDevmemCtx->psMMUContext);
+fail_mmucontext:
+	OSFreeMem(psDevmemCtx);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntHeapCreate
+@Description    Creates and initialises a device memory heap.
+@Return         valid Device Memory heap handle - Success
+                PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntHeapCreate(DEVMEMINT_CTX *psDevmemCtx,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    DEVMEMINT_HEAP **ppsDevmemHeapPtr)
+{
+	DEVMEMINT_HEAP *psDevmemHeap;
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sBlueprintHeapBaseAddr;
+	IMG_DEVMEM_SIZE_T uiBlueprintHeapLength;
+	IMG_DEVMEM_SIZE_T uiBlueprintResRgnLength;
+	IMG_UINT32 ui32BlueprintLog2DataPageSize;
+	IMG_UINT32 ui32BlueprintLog2ImportAlignment;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s", __func__));
+
+	if (!DevmemIntCtxAcquire(psDevmemCtx))
+	{
+		return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+	}
+
+	/* Retrieve page size and base addr match the heap blueprint */
+	eError = HeapCfgHeapDetails(NULL,
+	                            psDevmemCtx->psDevNode,
+	                            uiHeapConfigIndex,
+	                            uiHeapIndex,
+	                            0, NULL,
+	                            &sBlueprintHeapBaseAddr,
+	                            &uiBlueprintHeapLength,
+	                            &uiBlueprintResRgnLength,
+	                            &ui32BlueprintLog2DataPageSize,
+	                            &ui32BlueprintLog2ImportAlignment);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get details for HeapConfig:%d HeapIndex:%d.",
+				 __func__, uiHeapConfigIndex, uiHeapIndex));
+		goto ErrorCtxRelease;
+	}
+
+	/* uiHeapConfigIndex and uiHeapIndex are validated in HeapCfgHeapDetails()
+	 * so it should be safe to use here without additional checks. We must assert
+	 * though that the index is less than the number of bits in uiCreatedHeaps
+	 * bitfield (we assume 8 bits in a byte and bitfield width of 64). */
+	PVR_ASSERT(uiHeapIndex < sizeof(psDevmemCtx->uiCreatedHeaps) * 8);
+
+	if (BIT_ISSET(psDevmemCtx->uiCreatedHeaps, uiHeapIndex))
+	{
+		eError = PVRSRV_ERROR_ALREADY_EXISTS;
+		goto ErrorCtxRelease;
+	}
+
+	/* allocate the heap object */
+	psDevmemHeap = OSAllocMem(sizeof(*psDevmemHeap));
+	PVR_LOG_GOTO_IF_NOMEM(psDevmemHeap, eError, ErrorCtxRelease);
+
+	psDevmemHeap->psDevmemCtx = psDevmemCtx;
+	psDevmemHeap->uiLog2PageSize = ui32BlueprintLog2DataPageSize;
+	psDevmemHeap->sBaseAddr = sBlueprintHeapBaseAddr;
+	/* Store the last accessible address as our LastAddr. We can access
+	 * every address between sBlueprintHeapBaseAddr and
+	 * sBlueprintHeapBaseAddr + uiBlueprintHeapLength - 1
+	 */
+	psDevmemHeap->sLastAddr.uiAddr = sBlueprintHeapBaseAddr.uiAddr + uiBlueprintHeapLength - 1;
+	psDevmemHeap->uiHeapIndex = uiHeapIndex;
+
+	OSAtomicWrite(&psDevmemHeap->uiRefCount, 1);
+
+	eError = HeapCfgGetCallbacks(psDevmemCtx->psDevNode,
+	                             uiHeapConfigIndex,
+	                             uiHeapIndex,
+	                             &psDevmemHeap->pfnInit,
+	                             &psDevmemHeap->pfnDeInit);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get callbacks for HeapConfig:%d HeapIndex:%d.",
+				 __func__, uiHeapConfigIndex, uiHeapIndex));
+		goto ErrorFreeDevmemHeap;
+	}
+
+	if (psDevmemHeap->pfnInit != NULL)
+	{
+		eError = psDevmemHeap->pfnInit(psDevmemCtx->psDevNode,
+		                               psDevmemHeap,
+		                               &psDevmemHeap->hPrivData);
+		PVR_GOTO_IF_ERROR(eError, ErrorFreeDevmemHeap);
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: sBaseAddr = %" IMG_UINT64_FMTSPECX ", "
+	        "sLastAddr = %" IMG_UINT64_FMTSPECX, __func__,
+	        psDevmemHeap->sBaseAddr.uiAddr, psDevmemHeap->sLastAddr.uiAddr));
+
+	BIT_SET(psDevmemCtx->uiCreatedHeaps, uiHeapIndex);
+
+	*ppsDevmemHeapPtr = psDevmemHeap;
+
+	return PVRSRV_OK;
+
+ErrorFreeDevmemHeap:
+	OSFreeMem(psDevmemHeap);
+ErrorCtxRelease:
+	DevmemIntCtxRelease(psDevmemCtx);
+
+	return eError;
+}
+
+static INLINE IMG_UINT32
+_DevmemXReservationPageCount(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	return psRsrv->uiLength >> psRsrv->psDevmemHeap->uiLog2PageSize;
+}
+
+static INLINE IMG_DEV_VIRTADDR
+_DevmemXReservationPageAddress(DEVMEMXINT_RESERVATION *psRsrv, IMG_UINT32 uiVirtPageOffset)
+{
+	IMG_DEV_VIRTADDR sAddr = {
+		.uiAddr = psRsrv->sBase.uiAddr + ((IMG_UINT64)uiVirtPageOffset << psRsrv->psDevmemHeap->uiLog2PageSize)
+	};
+
+	return sAddr;
+}
+
+static INLINE PVRSRV_ERROR ReserveRangeParamValidation(DEVMEMINT_HEAP *psDevmemHeap,
+                                                       IMG_DEV_VIRTADDR sReservationVAddr,
+                                                       IMG_DEVMEM_SIZE_T uiVirtualSize)
+{
+	IMG_DEV_VIRTADDR sLastReserveAddr;
+	IMG_UINT64 ui64InvalidSizeMask = (1 << psDevmemHeap->uiLog2PageSize) - 1;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap != NULL, "psDevmemHeap");
+
+	sLastReserveAddr.uiAddr = sReservationVAddr.uiAddr + uiVirtualSize - 1;
+
+	/* Check that the requested address is not less than the base address of the heap. */
+	if (sReservationVAddr.uiAddr < psDevmemHeap->sBaseAddr.uiAddr)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"sReservationVAddr ("IMG_DEV_VIRTADDR_FMTSPEC") is invalid! "
+			"Must be greater or equal to "IMG_DEV_VIRTADDR_FMTSPEC,
+			sReservationVAddr.uiAddr,
+			psDevmemHeap->sBaseAddr.uiAddr);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check the allocation size is valid (must be page granular). */
+	if ((uiVirtualSize & ui64InvalidSizeMask) != 0 || uiVirtualSize == 0)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"uiVirtualSize ("IMG_DEVMEM_SIZE_FMTSPEC") is invalid! Must a multiple of %u and greater than 0",
+			uiVirtualSize,
+			1 << psDevmemHeap->uiLog2PageSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiVirtualSize > PMR_MAX_SUPPORTED_SIZE)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"uiVirtualSize must be less than or equal to the max PMR size ("
+			IMG_DEVMEM_SIZE_FMTSPEC")",
+			PMR_MAX_SUPPORTED_SIZE);
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Check that requested address + size fits in our heap. */
+	if (sLastReserveAddr.uiAddr > psDevmemHeap->sLastAddr.uiAddr)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+			"sReservationVAddr ("IMG_DEV_VIRTADDR_FMTSPEC") is invalid! "
+			"Must be lower than "IMG_DEV_VIRTADDR_FMTSPEC,
+			sReservationVAddr.uiAddr,
+			psDevmemHeap->sLastAddr.uiAddr - uiVirtualSize + 1);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemXIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                       IMG_DEV_VIRTADDR sReservationVAddr,
+                       IMG_DEVMEM_SIZE_T uiVirtualSize,
+                       DEVMEMXINT_RESERVATION **ppsRsrv)
+{
+	DEVMEMXINT_RESERVATION *psRsrv;
+	IMG_UINT32 uiNumPages;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(ppsRsrv != NULL);
+
+	eError = ReserveRangeParamValidation(psDevmemHeap,
+	                                     sReservationVAddr,
+	                                     uiVirtualSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "ReserveRangeParamValidation");
+
+
+	if (!DevmemIntHeapAcquire(psDevmemHeap))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, ErrorReturnError);
+	}
+
+	uiNumPages = uiVirtualSize >> psDevmemHeap->uiLog2PageSize;
+	psRsrv = OSAllocZMem(sizeof(*psRsrv->ppsPMR) * uiNumPages + sizeof(*psRsrv));
+	PVR_LOG_GOTO_IF_NOMEM(psRsrv, eError, ErrorUnreferenceHeap);
+
+	eError = OSLockCreate(&psRsrv->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeReservation);
+
+	psRsrv->sBase = sReservationVAddr;
+	psRsrv->uiLength = uiVirtualSize;
+	psRsrv->ppsPMR = IMG_OFFSET_ADDR(psRsrv, sizeof(*psRsrv));
+
+	eError = MMU_Alloc(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                   uiVirtualSize,
+	                   0, /* IMG_UINT32 uiProtFlags */
+	                   0, /* alignment is n/a since we supply devvaddr */
+	                   &sReservationVAddr,
+	                   psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrorDestroyLock);
+
+	/* since we supplied the virt addr, MMU_Alloc shouldn't have
+	   chosen a new one for us */
+	PVR_ASSERT(sReservationVAddr.uiAddr == psRsrv->sBase.uiAddr);
+
+	psRsrv->psDevmemHeap = psDevmemHeap;
+	*ppsRsrv = psRsrv;
+
+	return PVRSRV_OK;
+
+ErrorDestroyLock:
+	OSLockDestroy(psRsrv->hLock);
+ErrorFreeReservation:
+	OSFreeMem(psRsrv);
+ErrorUnreferenceHeap:
+	DevmemIntHeapRelease(psDevmemHeap);
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntUnreserveRange(DEVMEMXINT_RESERVATION *psRsrv)
+{
+	IMG_UINT32 ui32FirstMappedIdx = IMG_UINT32_MAX; // Initialise with invalid value.
+	IMG_UINT32 ui32ContigPageCount = 0;
+	IMG_UINT32 ui32PageIdx;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* No need to lock the mapping here since this is a handle destruction path which can not be
+	 * executed while there are outstanding handle lookups, i.e. other operations are performed
+	 * on the mapping. Bridge and handle framework also make sure this path can also not be executed
+	 * concurrently. */
+
+	for (ui32PageIdx = 0; ui32PageIdx < _DevmemXReservationPageCount(psRsrv); ui32PageIdx++)
+	{
+		if (psRsrv->ppsPMR[ui32PageIdx] != NULL)
+		{
+			if (ui32ContigPageCount == 0)
+			{
+#if defined(DEBUG)
+				if (ui32FirstMappedIdx == IMG_UINT32_MAX)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Reservation was not unmapped! The reservation will "
+					         "be unmapped before proceeding.",
+					         __func__));
+				}
+#endif
+				ui32FirstMappedIdx = ui32PageIdx;
+			}
+
+			ui32ContigPageCount++;
+		}
+		else
+		{
+			if (ui32ContigPageCount != 0)
+			{
+				eError = DevmemXIntUnmapPages(psRsrv,
+				                              ui32FirstMappedIdx,
+				                              ui32ContigPageCount);
+				PVR_RETURN_IF_ERROR(eError);
+			}
+
+			ui32ContigPageCount = 0;
+		}
+	}
+
+	if (ui32ContigPageCount != 0)
+	{
+		eError = DevmemXIntUnmapPages(psRsrv,
+		                              ui32FirstMappedIdx,
+		                              ui32ContigPageCount);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	MMU_Free(psRsrv->psDevmemHeap->psDevmemCtx->psMMUContext,
+	         psRsrv->sBase,
+	         psRsrv->uiLength,
+	         psRsrv->psDevmemHeap->uiLog2PageSize);
+
+	/* Don't bother with refcount on reservation, as a reservation only ever
+	 * holds one mapping, so we directly decrement the refcount on the heap
+	 * instead.
+	 * Function will print an error if the heap could not be unreferenced. */
+	DevmemIntHeapRelease(psRsrv->psDevmemHeap);
+
+	OSLockDestroy(psRsrv->hLock);
+	OSFreeMem(psRsrv);
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+DevmemValidateFlags(PMR *psPMR, PVRSRV_MEMALLOCFLAGS_T uiMapFlags)
+{
+	PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (PVRSRV_CHECK_GPU_READABLE(uiMapFlags) && !PVRSRV_CHECK_GPU_READABLE(uiPMRFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not GPU readable.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+	if (PVRSRV_CHECK_GPU_WRITEABLE(uiMapFlags) && !PVRSRV_CHECK_GPU_WRITEABLE(uiPMRFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not GPU writeable.", __func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_FLAGS, ErrorReturnError);
+	}
+
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntMapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                   PMR *psPMR,
+                   IMG_UINT32 uiPageCount,
+                   IMG_UINT32 uiPhysPageOffset,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   IMG_UINT32 uiVirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPMRMaxChunkCount = PMR_LogicalSize(psPMR) >> PMR_GetLog2Contiguity(psPMR);
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+	IMG_UINT32 uiLog2PageSize = psDevmemHeap->uiLog2PageSize;
+	IMG_UINT32 i;
+
+	/* Test uiPageCount+uiPhysPageOffset will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiPageCount + (IMG_UINT64)uiPhysPageOffset) <= (IMG_UINT64)IMG_UINT32_MAX, "uiPageCount+uiPhysPageOffset exceeds IMG_UINT32_MAX");
+	/* Test we do not exceed the PMR's maximum physical extent (in pages) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uiPageCount + uiPhysPageOffset) <= uiPMRMaxChunkCount, "uiPageCount+uiPhysPageOffset");
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	/* The range is not valid for the given virtual descriptor */
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+	PVR_LOG_RETURN_IF_FALSE((uiFlags & ~PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK) == 0,
+	                        "invalid flags", PVRSRV_ERROR_INVALID_FLAGS);
+	PVR_LOG_RETURN_IF_FALSE(!PMR_IsSparse(psPMR),
+		                    "PMR is Sparse, devmemx PMRs should be non-sparse", PVRSRV_ERROR_INVALID_FLAGS);
+	PVR_LOG_RETURN_IF_FALSE(!(PMR_Flags(psPMR) & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC),
+		                    "PMR allocation is deferred, devmemx PMRs can not be deferred", PVRSRV_ERROR_INVALID_FLAGS);
+
+	if (uiLog2PageSize > PMR_GetLog2Contiguity(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device heap and PMR have incompatible "
+		        "Log2Contiguity (%u - %u). PMR contiguity must be a multiple "
+		        "of the heap contiguity!", __func__, uiLog2PageSize,
+		        PMR_GetLog2Contiguity(psPMR)));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = DevmemValidateFlags(psPMR, uiFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemValidateFlags");
+
+	OSLockAcquire(psRsrv->hLock);
+
+	/* Increase map count to ensure page isolation/migration doesn't occur whilst
+	 * we are in process of mapping.
+	 */
+	PMRGpuMapDevPageCountIncr(psPMR, uiPageCount);
+
+	eError = MMU_MapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                      uiFlags,
+	                      _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                      psPMR,
+	                      uiPhysPageOffset,
+	                      uiPageCount,
+	                      NULL,
+	                      psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrUnlock);
+
+	for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+	{
+		PMRRefPMR2(psPMR);
+
+		if (psRsrv->ppsPMR[i] != NULL)
+		{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+			/* If PMR is allocated on demand the backing memory is freed by
+			 * pfnUnlockPhysAddresses(). */
+			if (!PVRSRV_CHECK_ON_DEMAND(PMR_Flags(psRsrv->ppsPMR[i])))
+			{
+				PMRMarkForDeferFree(psRsrv->ppsPMR[i]);
+			}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+			PMRGpuMapDevPageCountDecr(psRsrv->ppsPMR[i], 1);
+			PMRUnrefPMR2(psRsrv->ppsPMR[i]);
+		}
+
+		psRsrv->ppsPMR[i] = psPMR;
+	}
+
+	OSLockRelease(psRsrv->hLock);
+
+	return PVRSRV_OK;
+
+ErrUnlock:
+	/* Unwind map count if we failed to map. */
+	PMRGpuMapDevPageCountDecr(psPMR, uiPageCount);
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntUnmapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                     IMG_UINT32 uiVirtPageOffset,
+                     IMG_UINT32 uiPageCount)
+{
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+
+	OSLockAcquire(psRsrv->hLock);
+
+	/* Unmap the pages and mark them invalid in the MMU PTE */
+	eError = MMU_UnmapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                        0,
+	                        _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                        uiPageCount,
+	                        NULL,
+	                        psDevmemHeap->uiLog2PageSize,
+	                        0);
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+
+	for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+	{
+		if (psRsrv->ppsPMR[i] != NULL)
+		{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+			/* If PMR is allocated on demand the backing memory is freed by
+			 * pfnUnlockPhysAddresses(). */
+			if (!PVRSRV_CHECK_ON_DEMAND(PMR_Flags(psRsrv->ppsPMR[i])))
+			{
+				PMRMarkForDeferFree(psRsrv->ppsPMR[i]);
+			}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+			PMRGpuMapDevPageCountDecr(psRsrv->ppsPMR[i], 1);
+			PMRUnrefPMR2(psRsrv->ppsPMR[i]);
+			psRsrv->ppsPMR[i] = NULL;
+		}
+	}
+
+	OSLockRelease(psRsrv->hLock);
+
+	return PVRSRV_OK;
+
+ErrUnlock:
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemXIntMapVRangeToBackingPage(DEVMEMXINT_RESERVATION *psRsrv,
+                                 IMG_UINT32 uiPageCount,
+                                 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                 IMG_UINT32 uiVirtPageOffset)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeap = psRsrv->psDevmemHeap;
+	IMG_UINT32 i;
+
+	/* Test uiVirtPageOffset+uiPageCount will not exceed IMG_UINT32_MAX (and thereby wrap) */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(((IMG_UINT64)uiVirtPageOffset + (IMG_UINT64)uiPageCount) <= (IMG_UINT64)IMG_UINT32_MAX, "uiVirtPageOffset+uiPageCount exceeds IMG_UINT32_MAX");
+	/* The range is not valid for the given virtual descriptor */
+	PVR_LOG_RETURN_IF_FALSE((uiVirtPageOffset + uiPageCount) <= _DevmemXReservationPageCount(psRsrv),
+	                        "mapping offset out of range", PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE);
+	PVR_LOG_RETURN_IF_FALSE((uiFlags & ~(PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK |
+	                                     PVRSRV_MEMALLOCFLAG_ZERO_BACKING)) == 0,
+	                        "invalid flags", PVRSRV_ERROR_INVALID_FLAGS);
+
+	OSLockAcquire(psRsrv->hLock);
+
+	eError = MMUX_MapVRangeToBackingPage(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                                     uiFlags,
+	                                     _DevmemXReservationPageAddress(psRsrv, uiVirtPageOffset),
+	                                     uiPageCount,
+	                                     psDevmemHeap->uiLog2PageSize);
+	if (eError == PVRSRV_OK)
+	{
+		for (i = uiVirtPageOffset; i < (uiVirtPageOffset + uiPageCount); i++)
+		{
+			if (psRsrv->ppsPMR[i] != NULL)
+			{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+				/* If PMR is allocated on demand the backing memory is freed by
+				 * pfnUnlockPhysAddresses(). */
+				if (!PVRSRV_CHECK_ON_DEMAND(PMR_Flags(psRsrv->ppsPMR[i])))
+				{
+					PMRMarkForDeferFree(psRsrv->ppsPMR[i]);
+				}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+				PMRUnrefPMR2(psRsrv->ppsPMR[i]);
+				psRsrv->ppsPMR[i] = NULL;
+			}
+		}
+	}
+
+	OSLockRelease(psRsrv->hLock);
+
+	return eError;
+}
+
+static INLINE IMG_UINT32
+_DevmemReservationPageCount(DEVMEMINT_RESERVATION *psRsrv)
+{
+	return psRsrv->uiLength >> psRsrv->psDevmemHeap->uiLog2PageSize;
+}
+
+PVRSRV_ERROR
+DevmemIntMapPMR(DEVMEMINT_RESERVATION *psReservation, PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+	/* number of pages (device pages) that allocation spans */
+	IMG_UINT32 ui32NumDevPages;
+	/* device virtual address of start of allocation */
+	IMG_DEV_VIRTADDR sReservationVAddr;
+	/* and its length */
+	IMG_DEVMEM_SIZE_T uiVirtualSize;
+	IMG_UINT32 uiLog2HeapContiguity = psReservation->psDevmemHeap->uiLog2PageSize;
+	PVRSRV_MEMALLOCFLAGS_T uiMapFlags = psReservation->uiFlags;
+	IMG_BOOL bIsSparse = IMG_FALSE;
+	void *pvTmpBuf;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psReservation->psMappedPMR == NULL, "psReservation");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(PMR_LogicalSize(psPMR) == psReservation->uiLength, "psPMR logical size");
+
+	if (uiLog2HeapContiguity > PMR_GetLog2Contiguity(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Device heap and PMR have incompatible contiguity (%u - %u). "
+		         "Heap contiguity must be a multiple of the heap contiguity!",
+		         __func__,
+		         uiLog2HeapContiguity,
+		         PMR_GetLog2Contiguity(psPMR) ));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, ErrorReturnError);
+	}
+
+	eError = DevmemValidateFlags(psPMR, uiMapFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemValidateFlags", ErrorReturnError);
+
+	OSLockAcquire(psReservation->hLock);
+
+	if (!DevmemIntReservationAcquireUnlocked(psReservation))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, ErrorReleaseResLock);
+	}
+
+	uiVirtualSize = psReservation->uiLength;
+
+	ui32NumDevPages = 0xffffffffU & ( ( (uiVirtualSize - 1) >> uiLog2HeapContiguity) + 1);
+	PVR_ASSERT((IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity == uiVirtualSize);
+
+	/* Increase map count to ensure page isolation/migration doesn't occur whilst
+	 * we are in process of mapping. Also ensures we don't lock phys addresses causing allocation
+	 * then migrate before we increase the map count.
+	 */
+	PMRGpuMapDevPageCountIncr(psPMR, ui32NumDevPages);
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreference);
+
+	PMRLockPMR(psPMR);
+
+	/* Increase reservation association count so we know if multiple mappings have been created
+	 * on the PMR
+	 */
+	PMRGpuResCountIncr(psPMR);
+
+	sReservationVAddr = psReservation->sBase;
+
+	/*Check if the PMR that needs to be mapped is sparse */
+	bIsSparse = PMR_IsSparse(psPMR);
+	if (bIsSparse)
+	{
+		IMG_DEV_PHYADDR *psDevPAddr;
+		IMG_BOOL *pbValid;
+
+		pvTmpBuf = OSAllocMem((sizeof(IMG_DEV_PHYADDR) + sizeof(IMG_BOOL)) *
+		                      ui32NumDevPages);
+		PVR_LOG_GOTO_IF_NOMEM(pvTmpBuf, eError, ErrorUnlockPhysAddr);
+
+		psDevPAddr = IMG_OFFSET_ADDR(pvTmpBuf, 0);
+		pbValid = IMG_OFFSET_ADDR(pvTmpBuf, ui32NumDevPages * sizeof(IMG_DEV_PHYADDR));
+
+		/* N.B. We pass mapping permission flags to MMU_MapPages and let
+		 * it reject the mapping if the permissions on the PMR are not compatible. */
+		eError = MMU_MapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                      uiMapFlags,
+		                      sReservationVAddr,
+		                      psPMR,
+		                      0,
+		                      ui32NumDevPages,
+		                      NULL,
+		                      uiLog2HeapContiguity);
+		PVR_GOTO_IF_ERROR(eError, ErrorFreeSparseTmpBuf);
+
+		/* Determine which entries of the PMR are valid */
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2HeapContiguity,
+		                         ui32NumDevPages,
+		                         0,
+		                         psDevPAddr,
+		                         pbValid,
+		                         DEVICE_USE);
+		PVR_GOTO_IF_ERROR(eError, ErrorUnmapSparseMap);
+
+		for (i = 0; i < ui32NumDevPages; i++)
+		{
+			if (DevmemIntReservationIsIndexMapped(psReservation, i))
+			{
+				PMRUnrefPMR2(psReservation->psMappedPMR);
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+			}
+
+			if (pbValid[i])
+			{
+				PMRRefPMR2(psPMR);
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_TRUE);
+			}
+		}
+
+		OSFreeMem(pvTmpBuf);
+	}
+	else
+	{
+		eError = MMU_MapPMRFast(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        sReservationVAddr,
+		                        psPMR,
+		                        (IMG_DEVMEM_SIZE_T) ui32NumDevPages << uiLog2HeapContiguity,
+		                        uiMapFlags,
+		                        uiLog2HeapContiguity);
+		PVR_GOTO_IF_ERROR(eError, ErrorUnlockPhysAddr);
+	}
+
+	psReservation->psMappedPMR = psPMR;
+
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(psReservation->hLock);
+
+	return PVRSRV_OK;
+
+ErrorUnmapSparseMap:
+	(void) MMU_UnmapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+	                      0,
+	                      sReservationVAddr,
+	                      ui32NumDevPages,
+	                      NULL,
+	                      uiLog2HeapContiguity,
+	                      0);
+ErrorFreeSparseTmpBuf:
+	OSFreeMem(pvTmpBuf);
+
+ErrorUnlockPhysAddr:
+	PMRGpuResCountDecr(psPMR);
+	PMRUnlockPMR(psPMR);
+
+	{
+		PVRSRV_ERROR eError1 = PVRSRV_OK;
+		eError1 = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError1, "PMRUnlockSysPhysAddresses");
+	}
+
+ErrorUnreference:
+	/* Unwind map count if we failed to map. */
+	PMRGpuMapDevPageCountDecr(psPMR, ui32NumDevPages);
+
+	/* if fails there's not much to do (the function will print an error) */
+	DevmemIntReservationReleaseUnlocked(psReservation);
+
+ErrorReleaseResLock:
+	OSLockRelease(psReservation->hLock);
+
+ErrorReturnError:
+	PVR_ASSERT (eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntGetReservationData(DEVMEMINT_RESERVATION* psReservation, PMR** ppsPMR, IMG_DEV_VIRTADDR* psDevVAddr)
+{
+	/* Reservation might not have a PMR if a mapping was not yet performed */
+	if (psReservation->psMappedPMR == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevVAddr->uiAddr = psReservation->sBase.uiAddr;
+	*ppsPMR = psReservation->psMappedPMR;
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntReserveRangeAndMapPMR(DEVMEMINT_HEAP *psDevmemHeap,
+                               IMG_DEV_VIRTADDR sReservationVAddr,
+                               IMG_DEVMEM_SIZE_T uiVirtualSize,
+                               PMR *psPMR,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               DEVMEMINT_RESERVATION **ppsReservation)
+{
+	PVRSRV_ERROR eError, eUnreserveError;
+
+	eError = DevmemIntReserveRange(psDevmemHeap, sReservationVAddr, uiVirtualSize, uiFlags, ppsReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorReturnError);
+
+	eError = DevmemIntMapPMR(*ppsReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	eUnreserveError = DevmemIntUnreserveRange(*ppsReservation);
+	*ppsReservation = NULL;
+	PVR_LOG_IF_ERROR(eUnreserveError, "DevmemIntUnreserveRange");
+ErrorReturnError:
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnmapPMR(DEVMEMINT_RESERVATION *psReservation)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_HEAP *psDevmemHeap = psReservation->psDevmemHeap;
+	/* device virtual address of start of allocation */
+	IMG_DEV_VIRTADDR sReservationVAddr;
+	/* number of pages (device pages) that allocation spans */
+	IMG_UINT32 ui32NumDevPages;
+	IMG_BOOL bIsSparse = IMG_FALSE;
+	IMG_UINT32 i;
+
+	PVR_RETURN_IF_INVALID_PARAM(psReservation->psMappedPMR != NULL);
+
+	ui32NumDevPages = _DevmemReservationPageCount(psReservation);
+	sReservationVAddr = psReservation->sBase;
+
+	OSLockAcquire(psReservation->hLock);
+	PMRLockPMR(psReservation->psMappedPMR);
+
+	bIsSparse = PMR_IsSparse(psReservation->psMappedPMR);
+
+	if (bIsSparse)
+	{
+		eError = MMU_UnmapPages(psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        0,
+		                        sReservationVAddr,
+		                        ui32NumDevPages,
+		                        NULL,
+		                        psDevmemHeap->uiLog2PageSize,
+		                        0);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", ErrUnlock);
+
+		/* We are unmapping the whole PMR */
+		for (i = 0; i < ui32NumDevPages; i++)
+		{
+			if (DevmemIntReservationIsIndexMapped(psReservation, i))
+			{
+				/* All PMRs in the range should be the same, set local PMR
+				 * for Unlocking phys addrs later */
+				PMRUnrefPMR2(psReservation->psMappedPMR);
+				DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+			}
+		}
+	}
+	else
+	{
+		eError = MMU_UnmapPMRFast(psDevmemHeap->psDevmemCtx->psMMUContext,
+		                          sReservationVAddr,
+		                          ui32NumDevPages,
+		                          psDevmemHeap->uiLog2PageSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPMRFast", ErrUnlock);
+	}
+
+	PMRGpuMapDevPageCountDecr(psReservation->psMappedPMR, ui32NumDevPages);
+
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* If PMR is allocated on demand the backing memory is freed by
+	 * pfnUnlockPhysAddresses(). */
+	if (!PVRSRV_CHECK_ON_DEMAND(PMR_Flags(psReservation->psMappedPMR)))
+	{
+		PMRMarkForDeferFree(psReservation->psMappedPMR);
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	PMRGpuResCountDecr(psReservation->psMappedPMR);
+
+	PMRUnlockPMR(psReservation->psMappedPMR);
+
+	eError = PMRUnlockSysPhysAddresses(psReservation->psMappedPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psReservation->psMappedPMR = NULL;
+
+	DevmemIntReservationReleaseUnlocked(psReservation);
+
+	OSLockRelease(psReservation->hLock);
+
+	return PVRSRV_OK;
+
+ErrUnlock:
+	PMRUnlockPMR(psReservation->psMappedPMR);
+	OSLockRelease(psReservation->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                      IMG_DEV_VIRTADDR sReservationVAddr,
+                      IMG_DEVMEM_SIZE_T uiVirtualSize,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      DEVMEMINT_RESERVATION **ppsReservationPtr)
+{
+	PVRSRV_ERROR eError;
+	DEVMEMINT_RESERVATION *psReservation;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT64 ui64MapSize;
+
+	PVR_ASSERT(ppsReservationPtr != NULL);
+
+	eError = ReserveRangeParamValidation(psDevmemHeap,
+	                                     sReservationVAddr,
+	                                     uiVirtualSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "ReserveRangeParamValidation");
+
+	if (!DevmemIntHeapAcquire(psDevmemHeap))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW,
+		                    ErrorReturnError);
+	}
+
+	uiNumPages = uiVirtualSize >> psDevmemHeap->uiLog2PageSize;
+
+	/* allocate memory to record the reservation info */
+	ui64MapSize = sizeof(*psReservation->pui8Map) * LOG2_DIV(uiNumPages, MAP_MASK_SHIFT);
+	if (LOG2_MOD(uiNumPages, MAP_MASK_SHIFT) != 0)
+	{
+		ui64MapSize += 1;
+	}
+
+	psReservation = OSAllocZMem(sizeof(*psReservation) + ui64MapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psReservation, eError, ErrorUnreference);
+
+	/* Create lock */
+	eError = OSLockCreate(&psReservation->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeReservation);
+
+	/* Initialise refcount */
+	psReservation->i32RefCount = 1;
+
+	psReservation->uiFlags = uiFlags;
+	psReservation->sBase = sReservationVAddr;
+	psReservation->uiLength = uiVirtualSize;
+	psReservation->pui8Map = IMG_OFFSET_ADDR(psReservation, sizeof(*psReservation));
+
+	eError = MMU_Alloc(psDevmemHeap->psDevmemCtx->psMMUContext,
+	                   uiVirtualSize,
+	                   0, /* IMG_UINT32 uiProtFlags */
+	                   0, /* alignment is n/a since we supply devvaddr */
+	                   &sReservationVAddr,
+	                   psDevmemHeap->uiLog2PageSize);
+	PVR_GOTO_IF_ERROR(eError, ErrorDestroyLock);
+
+	/* since we supplied the virt addr, MMU_Alloc shouldn't have
+	   chosen a new one for us */
+	PVR_ASSERT(sReservationVAddr.uiAddr == psReservation->sBase.uiAddr);
+
+	psReservation->psDevmemHeap = psDevmemHeap;
+	*ppsReservationPtr = psReservation;
+
+	return PVRSRV_OK;
+
+	/*
+	 *  error exit paths follow
+	 */
+
+ErrorDestroyLock:
+	OSLockDestroy(psReservation->hLock);
+
+ErrorFreeReservation:
+	OSFreeMem(psReservation);
+
+ErrorUnreference:
+	/* if fails there's not much to do (the function will print an error) */
+	DevmemIntHeapRelease(psDevmemHeap);
+
+ErrorReturnError:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnreserveRange(DEVMEMINT_RESERVATION *psReservation)
+{
+	IMG_UINT32 i;
+	DEVMEMINT_HEAP *psDevmemHeap = psReservation->psDevmemHeap;
+	PVRSRV_ERROR eError;
+
+	if (psReservation->psMappedPMR != NULL)
+	{
+		/* No warning to be emitted as this is expected behaviour for the
+		 * Devmem interface. */
+		eError = DevmemIntUnmapPMR(psReservation);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevmemIntUnmapPMR");
+	}
+
+	OSLockAcquire(psReservation->hLock);
+
+	if (psReservation->i32RefCount != DEVMEMRESERVATION_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but still has existing references "
+		         "(%d), free existing reservations & mappings first.", __func__,
+		         psReservation->i32RefCount));
+
+		OSLockRelease(psReservation->hLock);
+
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	OSLockRelease(psReservation->hLock);
+
+	MMU_Free(psDevmemHeap->psDevmemCtx->psMMUContext,
+	         psReservation->sBase,
+	         psReservation->uiLength,
+	         psDevmemHeap->uiLog2PageSize);
+
+	for (i = 0; i < _DevmemReservationPageCount(psReservation); i++)
+	{
+		if (DevmemIntReservationIsIndexMapped(psReservation, i))
+		{
+			PMRUnrefPMR2(psReservation->psMappedPMR);
+			DevmemIntReservationSetMappingIndex(psReservation, i, IMG_FALSE);
+		}
+	}
+
+	OSLockDestroy(psReservation->hLock);
+	OSFreeMem(psReservation);
+
+	DevmemIntHeapRelease(psDevmemHeap);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+DevmemIntHeapDestroy(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	if (psDevmemHeap->pfnDeInit != NULL)
+	{
+		psDevmemHeap->pfnDeInit(psDevmemHeap->hPrivData);
+		psDevmemHeap->pfnDeInit = NULL;
+	}
+
+	if (OSAtomicRead(&psDevmemHeap->uiRefCount) != DEVMEMHEAP_REFCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but still has existing references "
+		         "(%d), free existing reservations & mappings first.", __func__,
+		         OSAtomicRead(&psDevmemHeap->uiRefCount)));
+
+		/*
+		 * Try again later when you've freed all the memory
+		 *
+		 * Note:
+		 * We don't expect the application to retry (after all this call would
+		 * succeed if the client had freed all the memory which it should have
+		 * done before calling this function). However, given there should be
+		 * an associated handle, when the handle base is destroyed it will free
+		 * any allocations leaked by the client and then it will retry this call,
+		 * which should then succeed.
+		 */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	PVR_ASSERT(OSAtomicRead(&psDevmemHeap->uiRefCount) == DEVMEMHEAP_REFCOUNT_MIN);
+
+	BIT_UNSET(psDevmemHeap->psDevmemCtx->uiCreatedHeaps, psDevmemHeap->uiHeapIndex);
+
+	DevmemIntCtxRelease(psDevmemHeap->psDevmemCtx);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Freed heap %p", __func__, psDevmemHeap));
+	OSFreeMem(psDevmemHeap);
+
+	return PVRSRV_OK;
+}
+
+IMG_DEV_VIRTADDR
+DevmemIntHeapGetBaseAddr(DEVMEMINT_HEAP *psDevmemHeap)
+{
+	PVR_ASSERT(psDevmemHeap != NULL);
+
+	return psDevmemHeap->sBaseAddr;
+}
+
+static PVRSRV_ERROR
+DevmemIntValidateSparsePMRIndices(IMG_UINT32 ui32PMRLogicalChunkCount,
+                                  IMG_UINT32 *paui32LogicalIndices,
+                                  IMG_UINT32 ui32LogicalIndexCount)
+{
+	IMG_UINT32 i;
+	IMG_UINT8 *paui8TrackedIndices;
+	IMG_UINT32 ui32AllocSize;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32PMRLogicalChunkCount != 0);
+	PVR_ASSERT(paui32LogicalIndices != NULL);
+	PVR_ASSERT(ui32LogicalIndexCount != 0 && ui32LogicalIndexCount <= ui32PMRLogicalChunkCount);
+
+	ui32AllocSize = LOG2_DIV(ui32PMRLogicalChunkCount, 3);
+	if (LOG2_MOD(ui32PMRLogicalChunkCount, 3) != 0)
+	{
+		++ui32AllocSize;
+	}
+
+	paui8TrackedIndices = OSAllocZMem(ui32AllocSize);
+	if (paui8TrackedIndices == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	for (i = 0; i < ui32LogicalIndexCount; i++)
+	{
+		IMG_UINT32 ui32LogicalIndex = paui32LogicalIndices[i];
+
+		if (ui32LogicalIndex >= ui32PMRLogicalChunkCount)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Reservation index %u is OOB",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+			break;
+		}
+
+		if (BIT_ISSET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3)))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Duplicate index found: %u",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+
+		BIT_SET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3));
+	}
+
+	OSFreeMem(paui8TrackedIndices);
+	return eError;
+}
+
+static PVRSRV_ERROR
+DevmemIntComputeVirtualIndicesFromLogical(DEVMEMINT_RESERVATION *psReservation,
+                                          IMG_UINT32 ui32LogicalIndexCount,
+                                          IMG_UINT32 *paui32LogicalIndices,
+                                          IMG_UINT32 *pui32VirtualIndexCount,
+                                          IMG_UINT32 **ppaui32VirtualIndices)
+{
+	IMG_UINT32 ui32OrderDiff = PMR_GetLog2Contiguity(psReservation->psMappedPMR) -
+		psReservation->psDevmemHeap->uiLog2PageSize;
+
+	if (ui32OrderDiff == 0)
+	{
+		*pui32VirtualIndexCount = ui32LogicalIndexCount;
+		*ppaui32VirtualIndices = paui32LogicalIndices;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		IMG_UINT32 ui32PagesPerOrder = 1 << ui32OrderDiff;
+		IMG_UINT32 *paui32VirtualIndices;
+		IMG_UINT32 i = 0;
+
+		paui32VirtualIndices = OSAllocMem(*pui32VirtualIndexCount * sizeof(IMG_UINT32));
+		PVR_RETURN_IF_NOMEM(paui32VirtualIndices);
+
+		for (i = 0; i < ui32LogicalIndexCount; i++)
+		{
+			IMG_UINT32 ui32LogicalIndex = paui32LogicalIndices[i];
+			IMG_UINT32 ui32PageOffset;
+
+			for (ui32PageOffset = 0; ui32PageOffset < ui32PagesPerOrder; ui32PageOffset++)
+			{
+				IMG_UINT32 ui32VirtAddr = ui32LogicalIndex * ui32PagesPerOrder + ui32PageOffset;
+				paui32VirtualIndices[i * ui32PagesPerOrder + ui32PageOffset] = ui32VirtAddr;
+			}
+		}
+
+		*ppaui32VirtualIndices = paui32VirtualIndices;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntChangeSparse(IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiSparseFlags,
+                      DEVMEMINT_RESERVATION *psReservation)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	IMG_UINT32 uiLog2PMRContiguity;
+	IMG_UINT32 uiLog2HeapContiguity;
+	PVRSRV_MEMALLOCFLAGS_T uiFlags = psReservation->uiFlags;
+
+	IMG_UINT32 *pai32MapIndices;
+	IMG_UINT32 *pai32UnmapIndices;
+	IMG_UINT32 uiMapPageCount;
+	IMG_UINT32 uiUnmapPageCount;
+
+	IMG_UINT32 ui32LogicalChunkCount;
+	PMR *psPMR = psReservation->psMappedPMR;
+
+	/* Ensure a PMR has been mapped to this reservation. */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR != NULL, "psReservation");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiSparseFlags & SPARSE_RESIZE_BOTH, "uiSparseFlags");
+
+	ui32LogicalChunkCount = PMR_GetLogicalChunkCount(psPMR);
+
+	if (!PMR_IsSparse(psPMR) || PMR_IsMemLayoutFixed(psPMR) ||
+	    PMR_IsCpuMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR cannot be changed because one or more of the following"
+		         " were true: !PMR_IsSparse() = %s, PMR_IsMemLayoutFixed() = %s,"
+		         " PMR_IsCpuMapped() = %s",
+		         __func__,
+		         !PMR_IsSparse(psPMR) ? "true" : "false",
+		         PMR_IsMemLayoutFixed(psPMR) ? "true" : "false",
+		         PMR_IsCpuMapped(psPMR) ? "true" : "false"));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	uiLog2PMRContiguity = PMR_GetLog2Contiguity(psPMR);
+	uiLog2HeapContiguity = psReservation->psDevmemHeap->uiLog2PageSize;
+
+	/* This is check is made in DevmemIntMapPMR - no need to do it again in release. */
+	PVR_ASSERT(uiLog2HeapContiguity <= uiLog2PMRContiguity);
+
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		PVR_LOG_RETURN_IF_INVALID_PARAM(ui32AllocPageCount != 0, "ui32AllocPageCount");
+		PVR_LOG_RETURN_IF_FALSE(ui32AllocPageCount <= ui32LogicalChunkCount,
+		                        "ui32AllocPageCount is invalid",
+		                        PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE);
+	}
+
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		PVR_LOG_RETURN_IF_INVALID_PARAM(ui32FreePageCount != 0, "ui32FreePageCount");
+		PVR_LOG_RETURN_IF_FALSE(ui32FreePageCount <= ui32LogicalChunkCount,
+		                        "ui32FreePageCount is invalid",
+		                        PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE);
+	}
+
+	pai32MapIndices = pai32AllocIndices;
+	pai32UnmapIndices = pai32FreeIndices;
+	uiMapPageCount = ui32AllocPageCount;
+	uiUnmapPageCount = ui32FreePageCount;
+
+	OSLockAcquire(psReservation->hLock);
+
+	PMRLockPMR(psPMR);
+
+	if (PMR_IsGpuMultiMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR cannot be changed because PMR_IsGpuMultiMapped() = true",
+		         __func__));
+		PVR_GOTO_WITH_ERROR(eError,
+		                    PVRSRV_ERROR_PMR_NOT_PERMITTED,
+		                    e0);
+	}
+
+	/* Pre check free indices against reservation given */
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		IMG_UINT32 i;
+
+		eError = DevmemIntValidateSparsePMRIndices(ui32LogicalChunkCount,
+		                                           pai32FreeIndices,
+		                                           ui32FreePageCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntValidateSparsePMRIndices", e0);
+
+		eError = DevmemIntComputeVirtualIndicesFromLogical(psReservation,
+		                                                   ui32FreePageCount,
+		                                                   pai32FreeIndices,
+		                                                   &uiUnmapPageCount,
+		                                                   &pai32UnmapIndices);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntComputeVirtualIndicesFromLogical", e1);
+
+		for (i = 0; i < uiUnmapPageCount; i++)
+		{
+			IMG_UINT32 ui32VirtIndex = pai32UnmapIndices[i];
+			IMG_BOOL bIsMapped;
+
+			bIsMapped = DevmemIntReservationIsIndexMapped(psReservation,
+			                                              ui32VirtIndex);
+			if (!bIsMapped)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Reservation index %u is not mapped into the reservation",
+				         __func__,
+				         ui32VirtIndex));
+
+				PVR_GOTO_WITH_ERROR(eError,
+				                    PVRSRV_ERROR_DEVICEMEM_NO_MAPPING,
+				                    e1);
+			}
+		}
+	}
+
+	/* Pre check alloc indices against reservation given */
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		IMG_UINT32 i;
+
+		eError = DevmemIntValidateSparsePMRIndices(ui32LogicalChunkCount,
+		                                           pai32AllocIndices,
+		                                           ui32AllocPageCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntValidateSparsePMRIndices", e1);
+
+		eError = DevmemIntComputeVirtualIndicesFromLogical(psReservation,
+		                                                   ui32AllocPageCount,
+		                                                   pai32AllocIndices,
+		                                                   &uiMapPageCount,
+		                                                   &pai32MapIndices);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemIntComputeVirtualIndicesFromLogical", e1);
+
+		for (i = 0; i < uiMapPageCount; i++)
+		{
+			IMG_UINT32 ui32VirtIndex = pai32MapIndices[i];
+			IMG_BOOL bIsMapped;
+
+			bIsMapped = DevmemIntReservationIsIndexMapped(psReservation,
+			                                              ui32VirtIndex);
+			if (bIsMapped)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Reservation index %u is mapped into the reservation",
+				         __func__,
+				         ui32VirtIndex));
+
+				PVR_GOTO_WITH_ERROR(eError,
+				                    PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED,
+				                    e1);
+			}
+		}
+	}
+
+	/* Invalidate the page table entries before freeing the physical pages. */
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		PMR_FLAGS_T uiPMRFlags;
+		IMG_UINT32 i;
+
+		/*Get the flags*/
+		uiPMRFlags = PMR_Flags(psPMR);
+
+		/* Unmap the pages and mark them invalid in the MMU PTE */
+		eError = MMU_UnmapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                        uiFlags,
+		                        psReservation->sBase,
+		                        uiUnmapPageCount,
+		                        pai32UnmapIndices,
+		                        uiLog2HeapContiguity,
+		                        uiPMRFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMU_UnmapPages", e1);
+
+		for (i = 0; i < uiUnmapPageCount; i++)
+		{
+			IMG_UINT32 uiIndex = pai32UnmapIndices[i];
+
+			if (DevmemIntReservationIsIndexMapped(psReservation, uiIndex))
+			{
+				PMRUnrefPMR2(psReservation->psMappedPMR);
+				DevmemIntReservationSetMappingIndex(psReservation,
+				                                    uiIndex,
+				                                    IMG_FALSE);
+			}
+		}
+	}
+
+	/* Do the PMR specific changes */
+	eError = PMR_ChangeSparseMemUnlocked(psPMR,
+	                                     ui32AllocPageCount,
+	                                     pai32AllocIndices,
+	                                     ui32FreePageCount,
+	                                     pai32FreeIndices,
+	                                     uiSparseFlags);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: Failed to do PMR specific changes.",
+		         __func__));
+		goto e1;
+	}
+
+	/* Wire the pages tables that got allocated */
+	if (uiSparseFlags & SPARSE_RESIZE_ALLOC)
+	{
+		IMG_UINT32 i;
+		/* Map the pages and mark them Valid in the MMU PTE */
+		eError = MMU_MapPages(psReservation->psDevmemHeap->psDevmemCtx->psMMUContext,
+		                      uiFlags,
+		                      psReservation->sBase,
+		                      psPMR,
+		                      0,
+		                      uiMapPageCount,
+		                      pai32MapIndices,
+		                      uiLog2HeapContiguity);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: Failed to map alloc indices.",
+			         __func__));
+			goto e1;
+		}
+
+		for (i = 0; i < uiMapPageCount; i++)
+		{
+			IMG_UINT32 uiIndex = pai32MapIndices[i];
+
+			if (!DevmemIntReservationIsIndexMapped(psReservation, uiIndex))
+			{
+				PMRRefPMR2(psReservation->psMappedPMR);
+				DevmemIntReservationSetMappingIndex(psReservation,
+				                                    uiIndex,
+				                                    IMG_TRUE);
+			}
+		}
+	}
+
+e1:
+	if (pai32MapIndices != pai32AllocIndices)
+	{
+		OSFreeMem(pai32MapIndices);
+	}
+	if (pai32UnmapIndices != pai32FreeIndices)
+	{
+		OSFreeMem(pai32UnmapIndices);
+	}
+e0:
+	PMRUnlockPMR(psPMR);
+	OSLockRelease(psReservation->hLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntCtxDestroy
+@Description    Destroy that created by DevmemIntCtxCreate
+@Input          psDevmemCtx   Device Memory context
+@Return         cannot fail.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntCtxDestroy(DEVMEMINT_CTX *psDevmemCtx)
+{
+	/*
+	   We can't determine if we should be freeing the context here
+	   as a refcount!=1 could be due to either the fact that heap(s)
+	   remain with allocations on them, or that this memory context
+	   has been exported.
+	   As the client couldn't do anything useful with this information
+	   anyway and the fact that the refcount will ensure we only
+	   free the context when _all_ references have been released
+	   don't bother checking and just return OK regardless.
+	   */
+	DevmemIntCtxRelease(psDevmemCtx);
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntGetVDevAddrPageSize
+@Description    Get the page size for a virtual address.
+@Input			psDevNode
+@Input			sDevAddr			Get the page size for this virtual address.
+@Output			puiLog2HeapPageSize	On success returns log2 of the page size.
+@Return         Failure code if the virtual address is outside any heap.
+*/ /**************************************************************************/
+static
+PVRSRV_ERROR DevmemIntGetVDevAddrPageSize(PVRSRV_DEVICE_NODE *psDevNode,
+										  IMG_DEV_VIRTADDR sDevAddr,
+										  IMG_PUINT32 puiLog2HeapPageSize)
+{
+	IMG_UINT32 i, j, uiLog2HeapPageSize = 0;
+	DEVICE_MEMORY_INFO *psDinfo = &psDevNode->sDevMemoryInfo;
+	DEVMEM_HEAP_CONFIG *psConfig = psDinfo->psDeviceMemoryHeapConfigArray;
+
+	IMG_BOOL bFound = IMG_FALSE;
+
+	for (i = 0;
+		 i < psDinfo->uiNumHeapConfigs && !bFound;
+		 i++)
+	{
+		for (j = 0;
+			 j < psConfig[i].uiNumHeaps  && !bFound;
+			 j++)
+		{
+			IMG_DEV_VIRTADDR uiBase =
+					psConfig[i].psHeapBlueprintArray[j].sHeapBaseAddr;
+			IMG_DEVMEM_SIZE_T uiSize =
+					psConfig[i].psHeapBlueprintArray[j].uiHeapLength;
+
+			if ((sDevAddr.uiAddr >= uiBase.uiAddr) &&
+				(sDevAddr.uiAddr < (uiBase.uiAddr + uiSize)))
+			{
+				uiLog2HeapPageSize =
+						psConfig[i].psHeapBlueprintArray[j].uiLog2DataPageSize;
+				bFound = IMG_TRUE;
+			}
+		}
+	}
+
+	if (uiLog2HeapPageSize == 0)
+	{
+		return PVRSRV_ERROR_INVALID_GPU_ADDR;
+	}
+
+	*puiLog2HeapPageSize = uiLog2HeapPageSize;
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntIsVDevAddrValid
+@Description    Checks if a virtual address is valid for access.
+@Input          psConnection
+@Input			psDevNode
+@Input			psDevmemCtx			Device Memory context
+@Input			sDevAddr			Virtual address to check.
+@Return         Failure code if the virtual address is invalid.
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntIsVDevAddrValid(CONNECTION_DATA * psConnection,
+                                      PVRSRV_DEVICE_NODE *psDevNode,
+                                      DEVMEMINT_CTX *psDevMemContext,
+                                      IMG_DEV_VIRTADDR sDevAddr)
+{
+	IMG_UINT32 uiLog2HeapPageSize = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = DevmemIntGetVDevAddrPageSize(psDevNode,
+										  sDevAddr,
+										  &uiLog2HeapPageSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	return MMU_IsVDevAddrValid(psDevMemContext->psMMUContext,
+	                           uiLog2HeapPageSize,
+	                           sDevAddr) ? PVRSRV_OK : PVRSRV_ERROR_INVALID_GPU_ADDR;
+}
+
+PVRSRV_ERROR
+DevmemIntInvalidateFBSCTable(DEVMEMINT_CTX *psDevMemContext,
+                             IMG_UINT64 ui64FBSCEntryMask)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psDevMemContext->psDevNode;
+	MMU_CONTEXT *psMMUContext = psDevMemContext->psMMUContext;
+
+	if (psDevNode->pfnInvalFBSCTable)
+	{
+		return psDevNode->pfnInvalFBSCTable(psDevNode,
+		                                    psMMUContext,
+		                                    ui64FBSCEntryMask);
+	}
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR DevmemIntGetFaultAddress(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDevNode,
+                                      DEVMEMINT_CTX *psDevMemContext,
+                                      IMG_DEV_VIRTADDR *psFaultAddress)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	if ((psDevMemContext->ui32Flags & DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE) == 0)
+	{
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	*psFaultAddress = psDevMemContext->sFaultAddress;
+	psDevMemContext->ui32Flags &= ~DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE;
+
+	return PVRSRV_OK;
+}
+
+static POSWR_LOCK g_hExportCtxListLock;
+static DLLIST_NODE g_sExportCtxList;
+
+PVRSRV_ERROR
+DevmemIntInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	dllist_init(&g_sExportCtxList);
+
+	eError = OSWRLockCreate(&g_hExportCtxListLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntDeInit(void)
+{
+	PVR_ASSERT(dllist_is_empty(&g_sExportCtxList));
+
+	OSWRLockDestroy(g_hExportCtxListLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntExportCtx(DEVMEMINT_CTX *psContext,
+                   PMR *psPMR,
+                   DEVMEMINT_CTX_EXPORT **ppsContextExport)
+{
+	DEVMEMINT_CTX_EXPORT *psCtxExport;
+	PVRSRV_ERROR eError;
+
+	psCtxExport = OSAllocMem(sizeof(DEVMEMINT_CTX_EXPORT));
+	PVR_LOG_RETURN_IF_NOMEM(psCtxExport, "psCtxExport");
+
+	if (!DevmemIntCtxAcquire(psContext))
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, ErrorFreeCtxExport);
+	}
+	PMRRefPMR(psPMR);
+	/* Now that the source PMR is exported, the layout
+	 * can't change as there could be outstanding importers
+	 * This is to make sure both exporter and importers view of
+	 * the memory is same */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+	psCtxExport->psDevmemCtx = psContext;
+	psCtxExport->psPMR = psPMR;
+	OSWRLockAcquireWrite(g_hExportCtxListLock);
+	dllist_add_to_tail(&g_sExportCtxList, &psCtxExport->sNode);
+	OSWRLockReleaseWrite(g_hExportCtxListLock);
+
+	*ppsContextExport = psCtxExport;
+
+	return PVRSRV_OK;
+
+ErrorFreeCtxExport:
+	OSFreeMem(psCtxExport);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntUnexportCtx(DEVMEMINT_CTX_EXPORT *psContextExport)
+{
+	OSWRLockAcquireWrite(g_hExportCtxListLock);
+	dllist_remove_node(&psContextExport->sNode);
+	OSWRLockReleaseWrite(g_hExportCtxListLock);
+
+	PMRUnrefPMR(psContextExport->psPMR);
+	DevmemIntCtxRelease(psContextExport->psDevmemCtx);
+
+	OSFreeMem(psContextExport);
+
+	/* Unable to find exported context, return error */
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntAcquireRemoteCtx(PMR *psPMR,
+                          DEVMEMINT_CTX **ppsContext,
+                          IMG_HANDLE *phPrivData)
+{
+	PDLLIST_NODE psListNode, psListNodeNext;
+	DEVMEMINT_CTX_EXPORT *psCtxExport;
+
+	OSWRLockAcquireRead(g_hExportCtxListLock);
+	/* Find context from list using PMR as key */
+	dllist_foreach_node(&g_sExportCtxList, psListNode, psListNodeNext)
+	{
+		psCtxExport = IMG_CONTAINER_OF(psListNode, DEVMEMINT_CTX_EXPORT, sNode);
+		if (psCtxExport->psPMR == psPMR)
+		{
+			if (!DevmemIntCtxAcquire(psCtxExport->psDevmemCtx))
+			{
+				OSWRLockReleaseRead(g_hExportCtxListLock);
+
+				return PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+			}
+			*ppsContext = psCtxExport->psDevmemCtx;
+			*phPrivData = psCtxExport->psDevmemCtx->hPrivData;
+
+			OSWRLockReleaseRead(g_hExportCtxListLock);
+
+			/* PMR should have been already exported to import it
+			 * If a PMR is exported, its immutable and the same is
+			 * checked here */
+			PVR_ASSERT(IMG_TRUE == PMR_IsMemLayoutFixed(psPMR));
+
+			return PVRSRV_OK;
+		}
+	}
+	OSWRLockReleaseRead(g_hExportCtxListLock);
+
+	/* Unable to find exported context, return error */
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Failed to acquire remote context. Could not retrieve context with given PMR",
+			__func__));
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntRegisterPFNotify
+@Description    Registers a PID to be notified when a page fault occurs on a
+                specific device memory context.
+@Input          psDevmemCtx    The context to be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntRegisterPFNotifyKM(DEVMEMINT_CTX *psDevmemCtx,
+                                         IMG_BOOL      bRegister)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	DLLIST_NODE         *psNode, *psNodeNext;
+	DEVMEMINT_PF_NOTIFY *psNotifyNode;
+	IMG_BOOL            bPresent = IMG_FALSE;
+	PVRSRV_ERROR        eError;
+	IMG_PID             ui32PID;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemCtx, "psDevmemCtx");
+
+	/* We can be certain that the memory context is valid and not freed during
+	 * the call time of this function as it's assured by the handle framework
+	 * (while the handle is looked up it cannot be freed). Therefore we can
+	 * safely retrieve the pointer to the device node and acquire the
+	 * hPageFaultNotifyLock lock.
+	 */
+	psDevNode = psDevmemCtx->psDevNode;
+
+	/* Acquire write lock to avoid resource free while
+	 * sMemoryContextPageFaultNotifyListHead and sProcessNotifyListHead
+	 * are being accessed.
+	 */
+	OSWRLockAcquireWrite(psDevNode->hPageFaultNotifyLock);
+
+	if (bRegister)
+	{
+		/* If this is the first PID in the list, the device memory context
+		 * needs to be registered for notification */
+		if (dllist_is_empty(&psDevmemCtx->sProcessNotifyListHead))
+		{
+			dllist_add_to_tail(&psDevNode->sMemoryContextPageFaultNotifyListHead,
+			                   &psDevmemCtx->sPageFaultNotifyListElem);
+		}
+	}
+
+	/* Obtain current client PID */
+	ui32PID = OSGetCurrentClientProcessIDKM();
+
+	/* Loop through the registered PIDs and check whether this one is
+	 * present */
+	dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+	{
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+
+		if (psNotifyNode->ui32PID == ui32PID)
+		{
+			bPresent = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (bRegister)
+	{
+		if (bPresent)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to register a PID that is already registered",
+			         __func__));
+			eError = PVRSRV_ERROR_PID_ALREADY_REGISTERED;
+			goto err_already_registered;
+		}
+
+		psNotifyNode = OSAllocMem(sizeof(*psNotifyNode));
+		if (psNotifyNode == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unable to allocate memory for the notify list",
+			          __func__));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto err_out_of_mem;
+		}
+		psNotifyNode->ui32PID = ui32PID;
+		/* Write lock is already held */
+		dllist_add_to_tail(&(psDevmemCtx->sProcessNotifyListHead), &(psNotifyNode->sProcessNotifyListElem));
+	}
+	else
+	{
+		if (!bPresent)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to unregister a PID that is not registered",
+			         __func__));
+			eError = PVRSRV_ERROR_PID_NOT_REGISTERED;
+			goto err_not_registered;
+		}
+		/* Write lock is already held */
+		dllist_remove_node(psNode);
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+		OSFreeMem(psNotifyNode);
+
+		/* If the last process in the list is being unregistered, then also
+		 * unregister the device memory context from the notify list. */
+		if (dllist_is_empty(&psDevmemCtx->sProcessNotifyListHead))
+		{
+			dllist_remove_node(&psDevmemCtx->sPageFaultNotifyListElem);
+		}
+	}
+	eError = PVRSRV_OK;
+
+err_already_registered:
+err_out_of_mem:
+err_not_registered:
+	OSWRLockReleaseWrite(psDevNode->hPageFaultNotifyLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       DevmemIntPFNotify
+@Description    Notifies any processes that have registered themselves to be
+                notified when a page fault happens on a specific device memory
+                context.
+@Input          *psDevNode           The device node.
+@Input          ui64FaultedPCAddress The page catalogue address that faulted.
+@Input          sFaultAddress        The address that triggered the fault.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntPFNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                               IMG_UINT64         ui64FaultedPCAddress,
+                               IMG_DEV_VIRTADDR   sFaultAddress)
+{
+	DLLIST_NODE         *psNode, *psNodeNext;
+	DEVMEMINT_PF_NOTIFY *psNotifyNode;
+	PVRSRV_ERROR        eError;
+	DEVMEMINT_CTX       *psDevmemCtx = NULL;
+	IMG_BOOL            bFailed = IMG_FALSE;
+
+	/* Protect access both to sMemoryContextPageFaultNotifyListHead and
+	 * to sProcessNotifyListHead. Those lists must be accessed atomically
+	 * in relation to each other, otherwise we risk accessing context that
+	 * might have already been destroyed. */
+	OSWRLockAcquireRead(psDevNode->hPageFaultNotifyLock);
+	if (dllist_is_empty(&(psDevNode->sMemoryContextPageFaultNotifyListHead)))
+	{
+		OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+		return PVRSRV_OK;
+	}
+
+	dllist_foreach_node(&(psDevNode->sMemoryContextPageFaultNotifyListHead), psNode, psNodeNext)
+	{
+		DEVMEMINT_CTX *psThisContext =
+			IMG_CONTAINER_OF(psNode, DEVMEMINT_CTX, sPageFaultNotifyListElem);
+		IMG_DEV_PHYADDR sPCDevPAddr;
+
+		eError = MMU_AcquireBaseAddr(psThisContext->psMMUContext, &sPCDevPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "MMU_AcquireBaseAddr");
+			OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+			return eError;
+		}
+
+		if (sPCDevPAddr.uiAddr == ui64FaultedPCAddress)
+		{
+			psDevmemCtx = psThisContext;
+			break;
+		}
+	}
+
+	if (psDevmemCtx == NULL)
+	{
+		/* Not found, just return */
+		OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+		return PVRSRV_OK;
+	}
+
+	/*
+	 * Store the first occurrence of a page fault address,
+	 * until that address is consumed by a client.
+	 */
+	if ((psDevmemCtx->ui32Flags & DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE) == 0)
+	{
+		psDevmemCtx->sFaultAddress = sFaultAddress;
+		psDevmemCtx->ui32Flags |= DEVMEMCTX_FLAGS_FAULT_ADDRESS_AVAILABLE;
+	}
+
+	/* Loop through each registered PID and send a signal to the process */
+	dllist_foreach_node(&(psDevmemCtx->sProcessNotifyListHead), psNode, psNodeNext)
+	{
+		psNotifyNode = IMG_CONTAINER_OF(psNode, DEVMEMINT_PF_NOTIFY, sProcessNotifyListElem);
+
+		eError = OSDebugSignalPID(psNotifyNode->ui32PID);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unable to signal process for PID: %u",
+			         __func__,
+			         psNotifyNode->ui32PID));
+
+			PVR_ASSERT(!"Unable to signal process");
+
+			bFailed = IMG_TRUE;
+		}
+	}
+	OSWRLockReleaseRead(psDevNode->hPageFaultNotifyLock);
+
+	if (bFailed)
+	{
+		return PVRSRV_ERROR_SIGNAL_FAILED;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+#if defined(PDUMP)
+typedef struct _DEVMEMINT_PDUMP_VALID_REGION_
+{
+	DLLIST_NODE sNode;
+	IMG_DEV_VIRTADDR sDevVAddr;
+	IMG_DEVMEM_SIZE_T uiSize;
+} DEVMEMINT_PDUMP_VALID_REGION;
+
+IMG_UINT32 DevmemIntMMUContextID(DEVMEMINT_CTX *psDevMemContext)
+{
+	IMG_UINT32 ui32MMUContextID;
+	MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32MMUContextID, PDUMP_FLAGS_CONTINUOUS);
+	return ui32MMUContextID;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize,
+                              DLLIST_NODE *psValidRegionsList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2HeapPageSize;
+	IMG_UINT32 uiHeapPageSize;
+	IMG_DEV_VIRTADDR sValidStart, sValidEnd, sCurrent, sEnd, sStartPage;
+
+	/* Get the page size for heap containing the start virtual address. */
+	eError = DevmemIntGetVDevAddrPageSize(psDeviceNode,
+	                                      sDevAddrStart,
+	                                      &uiLog2HeapPageSize);
+	PVR_RETURN_IF_ERROR(eError);
+
+	uiHeapPageSize = 1 << uiLog2HeapPageSize;
+
+	/* Iterate every page in the region to dump... */
+	sValidStart.uiAddr = sValidEnd.uiAddr = 0; /* Start/end of the current region which is valid to read. */
+	sStartPage.uiAddr = sDevAddrStart.uiAddr & ~((IMG_UINT64) uiHeapPageSize - 1); /* Page aligned start of the region to dump. */
+	sCurrent = sStartPage;
+	sEnd.uiAddr = sDevAddrStart.uiAddr + uiSize; /* End of the region to dump. */
+
+	while (IMG_TRUE)
+	{
+		IMG_BOOL bAtEnd = sCurrent.uiAddr >= sEnd.uiAddr;
+		IMG_BOOL bValid = IMG_FALSE;
+
+		if (!bAtEnd)
+		{
+			/* Check if the page starting at the current address is valid for reading. */
+			eError = DevmemIntIsVDevAddrValid(psConnection,
+			                                  psDeviceNode,
+			                                  psDevmemCtx,
+			                                  sCurrent);
+			if (eError == PVRSRV_OK)
+			{
+				/* If the current valid region is empty then set the start
+				 * to the current page. */
+				if (sValidStart.uiAddr == 0)
+				{
+					if (sCurrent.uiAddr == sStartPage.uiAddr)
+					{
+						/* Use the start of the region to dump if it doesn't
+						 * start page aligned. */
+						sValidStart = sDevAddrStart;
+					}
+					else
+					{
+						sValidStart = sCurrent;
+					}
+				}
+				/* Set the end of the valid region. */
+				sValidEnd.uiAddr = sCurrent.uiAddr + uiHeapPageSize;
+				/* Restrict to the region to dump. */
+				if (sValidEnd.uiAddr > sEnd.uiAddr)
+				{
+					sValidEnd = sEnd;
+				}
+				bValid = IMG_TRUE;
+			}
+			/* Move to the next page. */
+			sCurrent.uiAddr += uiHeapPageSize;
+		}
+
+		/* If the current page is invalid or we've reached the end of the region
+		 * to dump then pdump the current valid region. */
+		if (!bValid && sValidEnd.uiAddr > sValidStart.uiAddr)
+		{
+			DEVMEMINT_PDUMP_VALID_REGION *psRegion = OSAllocMem(sizeof(*psRegion));
+			PVR_LOG_GOTO_IF_NOMEM(psRegion, eError, ErrFreeRegions);
+
+			psRegion->sDevVAddr = sValidStart;
+			psRegion->uiSize = sValidEnd.uiAddr - sValidStart.uiAddr;
+
+			dllist_add_to_tail(psValidRegionsList, &psRegion->sNode);
+
+			sValidStart.uiAddr = sValidEnd.uiAddr = 0;
+		}
+
+		if (bAtEnd)
+		{
+			break;
+		}
+	}
+
+	return PVRSRV_OK;
+
+ErrFreeRegions:
+	DevmemIntPDumpFreeValidRegions(psValidRegionsList);
+	return eError;
+}
+
+void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psValidRegionsList)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(psValidRegionsList, psThis, psNext)
+	{
+		DEVMEMINT_PDUMP_VALID_REGION *psRegion =
+		    IMG_CONTAINER_OF(psThis, DEVMEMINT_PDUMP_VALID_REGION, sNode);
+
+		dllist_remove_node(psThis);
+		OSFreeMem(psRegion);
+	}
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+	DLLIST_NODE *psThis, *psNext;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+	                                    &uiPDumpMMUCtx,
+	                                    ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* The following SYSMEM refers to the 'MMU Context', hence it
+	 * should be the MMU context, not the PMR, that says what the PDump
+	 * MemSpace tag is?
+	 * From a PDump P.O.V. it doesn't matter which name space we use as long
+	 * as that MemSpace is used on the 'MMU Context' we're dumping from
+	 */
+
+	dllist_foreach_node(psDevAddrRegions, psThis, psNext)
+	{
+		DEVMEMINT_PDUMP_VALID_REGION *psRegion =
+		    IMG_CONTAINER_OF(psThis, DEVMEMINT_PDUMP_VALID_REGION, sNode);
+
+		eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+		                     psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+		                     uiPDumpMMUCtx,
+		                     psRegion->sDevVAddr,
+		                     psRegion->uiSize,
+		                     pszFilename,
+		                     ui32FileOffset,
+		                     ui32PDumpFlags);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		ui32FileOffset += psRegion->uiSize;
+
+		dllist_remove_node(psThis);
+		OSFreeMem(psRegion);
+	}
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 ui32ArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+	IMG_UINT32 uiLog2HeapPageSize;
+	IMG_UINT32 uiHeapPageSize;
+	IMG_DEV_VIRTADDR sValidStart, sValidEnd, sCurrent, sEnd, sStartPage;
+	IMG_UINT64 ui64PageMask;
+
+	PVR_UNREFERENCED_PARAMETER(ui32ArraySize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+			&uiPDumpMMUCtx,
+			ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/*
+	  Get the page size for heap containing the start virtual address.
+	 */
+	eError = DevmemIntGetVDevAddrPageSize(psDeviceNode,
+										  sDevAddrStart,
+										  &uiLog2HeapPageSize);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+	uiHeapPageSize = 1 << uiLog2HeapPageSize;
+	ui64PageMask = uiHeapPageSize - 1;
+
+	/*
+	  Iterate every page in the region to dump...
+	 */
+	sValidStart.uiAddr = sValidEnd.uiAddr = 0; /* Start/end of the current region which is valid to read. */
+	sStartPage.uiAddr = sDevAddrStart.uiAddr & ~ui64PageMask; /* Page aligned start of the region to dump. */
+	sCurrent = sStartPage;
+	sEnd.uiAddr = sDevAddrStart.uiAddr + uiSize; /* End of the region to dump. */
+	for (;;)
+	{
+		IMG_BOOL bAtEnd = sCurrent.uiAddr >= sEnd.uiAddr;
+		IMG_BOOL bValid = IMG_FALSE;
+
+		if (!bAtEnd)
+		{
+			/* Check if the page starting at the current address is valid for reading. */
+			eError = DevmemIntIsVDevAddrValid(psConnection,
+											  psDeviceNode,
+											  psDevmemCtx,
+											  sCurrent);
+			if (eError == PVRSRV_OK)
+			{
+				/* If the current valid region is empty then set the start to the current page. */
+				if (sValidStart.uiAddr == 0)
+				{
+					if (sCurrent.uiAddr == sStartPage.uiAddr)
+					{
+						/* Use the start of the region to dump if it doesn't start page aligned. */
+						sValidStart = sDevAddrStart;
+					}
+					else
+					{
+						sValidStart = sCurrent;
+					}
+				}
+				/* Set the end of the valid region. */
+				sValidEnd.uiAddr = sCurrent.uiAddr + uiHeapPageSize;
+				/* Restrict to the region to dump. */
+				if (sValidEnd.uiAddr > sEnd.uiAddr)
+				{
+					sValidEnd = sEnd;
+				}
+				bValid = IMG_TRUE;
+			}
+			/* Move to the next page. */
+			sCurrent.uiAddr += uiHeapPageSize;
+		}
+		/*
+		  If the current page is invalid or we've reached the end of the region to dump then pdump the current valid region.
+		 */
+		if (!bValid && sValidEnd.uiAddr > sValidStart.uiAddr)
+		{
+			IMG_DEVMEM_SIZE_T uiValidSize = sValidEnd.uiAddr - sValidStart.uiAddr;
+			eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+								 psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+								 uiPDumpMMUCtx,
+								 sValidStart,
+								 uiValidSize,
+								 pszFilename,
+								 ui32FileOffset,
+								 ui32PDumpFlags);
+			PVR_ASSERT(eError == PVRSRV_OK);
+
+			ui32FileOffset += uiValidSize;
+
+			sValidStart.uiAddr = sValidEnd.uiAddr = 0;
+		}
+
+		if (bAtEnd)
+		{
+			break;
+		}
+	}
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          DEVMEMINT_CTX *psDevmemCtx,
+                                          IMG_DEV_VIRTADDR sDevAddrStart,
+                                          IMG_DEVMEM_SIZE_T uiSize,
+                                          const IMG_CHAR *pszFilename,
+                                          IMG_UINT32 ui32FileOffset,
+                                          IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPDumpMMUCtx;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (psDevmemCtx->psDevNode->sDevId.ui32InternalID !=
+	    (PVRSRVGetPVRSRVData())->ui32PDumpBoundDevice)
+	{
+		return PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevmemCtx->psMMUContext,
+	                                    &uiPDumpMMUCtx,
+	                                    ui32PDumpFlags);
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* The following SYSMEM refers to the 'MMU Context', hence it
+	 * should be the MMU context, not the PMR, that says what the PDump
+	 * MemSpace tag is?
+	 * From a PDump P.O.V. it doesn't matter which name space we use as long
+	 * as that MemSpace is used on the 'MMU Context' we're dumping from
+	 */
+	eError = PDumpMMUSAB(psDevmemCtx->psDevNode,
+	                     psDevmemCtx->psDevNode->sDevId.pszPDumpDevName,
+	                     uiPDumpMMUCtx,
+	                     sDevAddrStart,
+	                     uiSize,
+	                     pszFilename,
+	                     ui32FileOffset,
+	                     ui32PDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	MMU_ReleasePDumpMMUContext(psDevmemCtx->psMMUContext, ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+							  PVRSRV_DEVICE_NODE *psDeviceNode,
+							  DEVMEMINT_CTX *psDevMemContext,
+							  IMG_UINT32 ui32Size,
+							  const IMG_CHAR *pszFileName,
+							  IMG_DEV_VIRTADDR sData,
+							  IMG_UINT32 ui32DataSize,
+							  IMG_UINT32 ui32LogicalWidth,
+							  IMG_UINT32 ui32LogicalHeight,
+							  IMG_UINT32 ui32PhysicalWidth,
+							  IMG_UINT32 ui32PhysicalHeight,
+							  PDUMP_PIXEL_FORMAT ePixFmt,
+							  IMG_MEMLAYOUT eMemLayout,
+							  IMG_FB_COMPRESSION eFBCompression,
+							  const IMG_UINT32 *paui32FBCClearColour,
+							  PDUMP_FBC_SWIZZLE eFBCSwizzle,
+							  IMG_DEV_VIRTADDR sHeader,
+							  IMG_UINT32 ui32HeaderSize,
+							  IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32 ui32ContextID;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	eError = MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32ContextID, ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_AcquirePDumpMMUContext");
+
+	eError = PDumpImageDescriptor(psDeviceNode,
+									ui32ContextID,
+									(IMG_CHAR *)pszFileName,
+									sData,
+									ui32DataSize,
+									ui32LogicalWidth,
+									ui32LogicalHeight,
+									ui32PhysicalWidth,
+									ui32PhysicalHeight,
+									ePixFmt,
+									eMemLayout,
+									eFBCompression,
+									paui32FBCClearColour,
+									eFBCSwizzle,
+									sHeader,
+									ui32HeaderSize,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "PDumpImageDescriptor");
+
+	/* Don't care about return value */
+	(void) MMU_ReleasePDumpMMUContext(psDevMemContext->psMMUContext, ui32PDumpFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+							 PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_CTX *psDevMemContext,
+							 IMG_UINT32 ui32Size,
+							 const IMG_CHAR *pszFileName,
+							 IMG_DEV_VIRTADDR sData,
+							 IMG_UINT32 ui32DataSize,
+							 IMG_UINT32 ui32HeaderType,
+							 IMG_UINT32 ui32ElementType,
+							 IMG_UINT32 ui32ElementCount,
+							 IMG_UINT32 ui32PDumpFlags)
+{
+	IMG_UINT32 ui32ContextID;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+
+	if ((ui32HeaderType != IBIN_HEADER_TYPE) &&
+		(ui32HeaderType != DATA_HEADER_TYPE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid header type (%u)",
+		         __func__,
+		         ui32HeaderType));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = MMU_AcquirePDumpMMUContext(psDevMemContext->psMMUContext, &ui32ContextID, ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_AcquirePDumpMMUContext");
+
+	eError = PDumpDataDescriptor(psDeviceNode,
+									ui32ContextID,
+									(IMG_CHAR *)pszFileName,
+									sData,
+									ui32DataSize,
+									ui32HeaderType,
+									ui32ElementType,
+									ui32ElementCount,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "PDumpDataDescriptor");
+
+	/* Don't care about return value */
+	(void) MMU_ReleasePDumpMMUContext(psDevMemContext->psMMUContext, ui32PDumpFlags);
+
+	return eError;
+}
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/devicemem_server.h b/drivers/gpu/drm/img-rogue/devicemem_server.h
new file mode 100644
index 000000000000..8fa2b90cbda8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_server.h
@@ -0,0 +1,748 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server side component for device memory management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_SERVER_H
+#define DEVICEMEM_SERVER_H
+
+#include "device.h" /* For device node */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "connection_server.h"
+#include "pmr.h"
+
+typedef struct _DEVMEMINT_CTX_ DEVMEMINT_CTX;
+typedef struct _DEVMEMINT_CTX_EXPORT_ DEVMEMINT_CTX_EXPORT;
+typedef struct _DEVMEMINT_HEAP_ DEVMEMINT_HEAP;
+
+typedef struct _DEVMEMINT_RESERVATION_ DEVMEMINT_RESERVATION;
+
+typedef struct _DEVMEMINT_MAPPING_ DEVMEMINT_MAPPING;
+typedef struct _DEVMEMXINT_RESERVATION_ DEVMEMXINT_RESERVATION;
+typedef struct _DEVMEMINT_PF_NOTIFY_ DEVMEMINT_PF_NOTIFY;
+
+/*
+ * DevmemServerGetImportHandle()
+ *
+ * For given exportable memory descriptor returns PMR handle
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetImportHandle(DEVMEM_MEMDESC *psMemDesc,
+                            IMG_HANDLE *phImport);
+
+/*
+ * DevmemServerGetHeapHandle()
+ *
+ * For given reservation returns the Heap handle
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetHeapHandle(DEVMEMINT_RESERVATION *psReservation,
+                          IMG_HANDLE *phHeap);
+
+/*
+ * DevmemServerGetContext()
+ *
+ * For given heap returns the context.
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetContext(DEVMEMINT_HEAP *psDevmemHeap,
+                       DEVMEMINT_CTX **ppsDevmemCtxPtr);
+
+/*
+ * DevmemServerGetPrivData()
+ *
+ * For given context returns the private data handle.
+ *
+ */
+PVRSRV_ERROR
+DevmemServerGetPrivData(DEVMEMINT_CTX *psDevmemCtx,
+                        IMG_HANDLE *phPrivData);
+
+/*
+ * DevmemIntCtxCreate()
+ *
+ * Create a Server-side Device Memory Context. This is usually the counterpart
+ * of the client side memory context, and indeed is usually created at the
+ * same time.
+ *
+ * You must have one of these before creating any heaps.
+ *
+ * All heaps must have been destroyed before calling
+ * DevmemIntCtxDestroy()
+ *
+ * If you call DevmemIntCtxCreate() (and it succeeds) you are promising to
+ * later call DevmemIntCtxDestroy()
+ *
+ * Note that this call will cause the device MMU code to do some work for
+ * creating the device memory context, but it does not guarantee that a page
+ * catalogue will have been created, as this may be deferred until the first
+ * allocation.
+ *
+ * Caller to provide storage for a pointer to the DEVMEM_CTX object that will
+ * be created by this call.
+ */
+PVRSRV_ERROR
+DevmemIntCtxCreate(CONNECTION_DATA *psConnection,
+                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                   /* devnode / perproc etc */
+                   IMG_BOOL bKernelMemoryCtx,
+                   DEVMEMINT_CTX **ppsDevmemCtxPtr,
+                   IMG_HANDLE *hPrivData,
+                   IMG_UINT32 *pui32CPUCacheLineSize);
+/*
+ * DevmemIntCtxDestroy()
+ *
+ * Undoes a prior DevmemIntCtxCreate or DevmemIntCtxImport.
+ */
+PVRSRV_ERROR
+DevmemIntCtxDestroy(DEVMEMINT_CTX *psDevmemCtx);
+
+/*
+ * DevmemIntHeapCreate()
+ *
+ * Creates a new heap in this device memory context.  This will cause a call
+ * into the MMU code to allocate various data structures for managing this
+ * heap. It will not necessarily cause any page tables to be set up, as this
+ * can be deferred until first allocation. (i.e. we shouldn't care - it's up
+ * to the MMU code)
+ *
+ * Note that the data page size must be specified (as log 2). The data page
+ * size as specified here will be communicated to the mmu module, and thus may
+ * determine the page size configured in page directory entries for subsequent
+ * allocations from this heap. It is essential that the page size here is less
+ * than or equal to the "minimum contiguity guarantee" of any PMR that you
+ * subsequently attempt to map to this heap.
+ *
+ * If you call DevmemIntHeapCreate() (and the call succeeds) you are promising
+ * that you shall subsequently call DevmemIntHeapDestroy()
+ *
+ * Caller to provide storage for a pointer to the DEVMEM_HEAP object that will
+ * be created by this call.
+ */
+PVRSRV_ERROR
+DevmemIntHeapCreate(DEVMEMINT_CTX *psDevmemCtx,
+                    IMG_UINT32 uiHeapConfigIndex,
+                    IMG_UINT32 uiHeapIndex,
+                    DEVMEMINT_HEAP **ppsDevmemHeapPtr);
+/*
+ * DevmemIntHeapDestroy()
+ *
+ * Destroys a heap previously created with DevmemIntHeapCreate()
+ *
+ * All allocations from his heap must have been freed before this
+ * call.
+ */
+PVRSRV_ERROR
+DevmemIntHeapDestroy(DEVMEMINT_HEAP *psDevmemHeap);
+
+/* DevmemIntHeapGetBaseAddr()
+ *
+ * Get heap base address pre carveouts.
+ */
+IMG_DEV_VIRTADDR
+DevmemIntHeapGetBaseAddr(DEVMEMINT_HEAP *psDevmemHeap);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntReserveRange()
+ * @Description Reserves a number of virtual addresses starting sReservationVAddr
+ *              and continuing until sReservationVAddr + uiVirtualSize - 1.
+ *
+ *              If you call DevmemIntReserveRange() (and the call succeeds)
+ *              then you are promising that you shall later call DevmemIntUnreserveRange()
+ *
+ * @Input       psDevmemHeap        The virtual heap the DevVAddr is within.
+ * @Input       sReservationVAddr   The first virtual address of the range.
+ * @Input       uiVirtualSize       The number of bytes in the virtual range.
+ * @Input       uiFlags             Mem alloc flags
+ * @Output      ppsReservationPtr   A pointer to the created reservation.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                      IMG_DEV_VIRTADDR sReservationVAddr,
+                      IMG_DEVMEM_SIZE_T uiVirtualSize,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      DEVMEMINT_RESERVATION **ppsReservationPtr);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntUnreserveRange()
+ * @Description Unreserves the specified virtual range. In the case that the
+ *              virtual range has not been unmapped, it will be unmapped.
+ *              If any references are held on the reservation PVRSRV_ERROR_RETRY
+ *              will be returned.
+ *
+ * @Input       psDevmemReservation The reservation to unreserve
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntUnreserveRange(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntMapPMR
+ *
+ * @Description Maps the given PMR to the virtual range previously reserved with
+ *              DevmemIntReserveRange(). When calling this function, the reservation
+ *              must be valid, and not mapped. Additionally, the PMRs logical
+ *              size and the reservations virtual size must be equal.
+ *
+ *              If appropriate, the PMR must have had its physical backing
+ *              committed, as this call will call into the MMU code to set
+ *              up the page tables for this allocation, which shall in turn
+ *              request the physical addresses from the PMR. Alternatively,
+ *              the PMR implementation can choose to do so off the back of
+ *              the "lock" callback, which it will receive as a result
+ *              (indirectly) of this call.
+ *
+ *              If you call DevmemIntMapPMR() (and the call succeeds) then you
+ *              are promising that you shall later call DevmemIntUnmapPMR()
+ *
+ * @Input       psReservation    The reservation the PMR will be mapped into.
+ * @Input       psPMR            The PMR to be mapped.
+ *
+ * @Return      PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntMapPMR(DEVMEMINT_RESERVATION *psReservation, PMR *psPMR);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntUnmapPMR()
+ *
+ * @Description Unmaps a previously mapped virtual range.
+ *
+ * @Input       psReservation   The virtual range to unmap.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntUnmapPMR(DEVMEMINT_RESERVATION *psReservation);
+
+
+/*************************************************************************/ /*!
+ * @Function    DevmemIntReserveRangeAndMapPMR()
+ *
+ * @Description Reserve (with DevmemIntReserveRange), and map a virtual range
+ *              to a PMR (with DevmemIntMapPMR).
+ *
+ * @Input       psDevmemHeap        The virtual heap DevVAddr is within.
+ * @Input       sReservationVAddr   The first virtual address of the range.
+ * @Input       uiVirtualSize       The number of bytes in the virtual range.
+ * @Input       psPMR               The PMR to be mapped.
+ * @Input       uiFlags             Mem alloc flags
+ * @Output      ppsReservation      A pointer to the created reservation.
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntReserveRangeAndMapPMR(DEVMEMINT_HEAP *psDevmemHeap,
+                               IMG_DEV_VIRTADDR sReservationVAddr,
+                               IMG_DEVMEM_SIZE_T uiVirtualSize,
+                               PMR *psPMR,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               DEVMEMINT_RESERVATION **ppsReservation);
+
+/*************************************************************************/ /*!
+ * @Function       DevmemIntChangeSparse
+ * @Description    Changes the sparse allocations of a PMR by allocating and freeing
+ *                 pages and changing their corresponding GPU mapping.
+ *
+ *                 Prior to calling this function DevmemIntMapPMR
+ *                 or DevmemIntReserveRangeAndMapPMR must be used.
+ *
+ * @Input          psReservation         The reservation that the PMR is mapped to.
+ * @Input          ui32AllocPageCount    Number of pages to allocate
+ * @Input          pai32AllocIndices     The logical PMR indices where pages will
+ *                                       be allocated. May be NULL.
+ * @Input          ui32FreePageCount     Number of pages to free
+ * @Input          pai32FreeIndices      The logical PMR indices where pages will
+ *                                       be freed. May be NULL.
+ * @Input          uiSparseFlags         Flags passed in to determine which kind
+ *                                       of sparse change the user wanted.
+ *                                       See devicemem_typedefs.h for details.
+ * @Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntChangeSparse(IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+                      SPARSE_MEM_RESIZE_FLAGS uiSparseFlags,
+                      DEVMEMINT_RESERVATION *psReservation);
+
+PVRSRV_ERROR
+DevmemIntGetReservationData(DEVMEMINT_RESERVATION* psReservation, PMR** ppsPMR, IMG_DEV_VIRTADDR* psDevVAddr);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntReserveRange()
+ * @Description Indicates that the specified range should be reserved from the
+ *              given heap.
+ *
+ *              In turn causes the page tables to be allocated to cover the
+ *              specified range.
+ *
+ *              If you call DevmemIntReserveRange() (and the call succeeds)
+ *              then you are promising that you shall later call
+ *              DevmemIntUnreserveRange().
+ *
+ * @Input       psDevmemHeap        Pointer to the heap the reservation is made
+ *                                  on
+ * @Input       sReservationVAddr   Virtual address of the reservation
+ * @Input       uiVirtualSize       Size of the reservation (in bytes)
+ * @Input       ppsRsrv             Return pointer to the reservation object
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntReserveRange(DEVMEMINT_HEAP *psDevmemHeap,
+                       IMG_DEV_VIRTADDR sReservationVAddr,
+                       IMG_DEVMEM_SIZE_T uiVirtualSize,
+                       DEVMEMXINT_RESERVATION **ppsRsrv);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntUnreserveRange()
+ * @Description Undoes the state change caused by DevmemXIntReserveRage()
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntUnreserveRange(DEVMEMXINT_RESERVATION *psRsrv);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationAcquire
+@Description    Acquire a reference to the provided device memory reservation.
+@Return         IMG_TRUE if referenced and IMG_FALSE in case of error
+*/ /**************************************************************************/
+IMG_BOOL
+DevmemIntReservationAcquire(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntReservationRelease
+@Description    Release the reference to the provided device memory reservation.
+            If this is the last reference which was taken then the
+                reservation will be freed.
+@Return         None.
+*/ /**************************************************************************/
+void
+DevmemIntReservationRelease(DEVMEMINT_RESERVATION *psDevmemReservation);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntMapPages()
+ * @Description Maps an arbitrary amount of pages from a PMR to a reserved range
+ *              and takes references on the PMR.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       psPMR              PMR that is mapped
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 mapped
+ * @Input       uiPhysPageOffset   Logical offset in the PMR (measured in pages)
+ * @Input       uiFlags            Mapping flags
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntMapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                   PMR *psPMR,
+                   IMG_UINT32 uiPageCount,
+                   IMG_UINT32 uiPhysPageOffset,
+                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                   IMG_UINT32 uiVirtPageOffset);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntUnmapPages()
+ * @Description Unmaps an arbitrary amount of pages from a reserved range and
+ *              releases references on associated PMRs.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 unmapped
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntUnmapPages(DEVMEMXINT_RESERVATION *psRsrv,
+                     IMG_UINT32 uiVirtPageOffset,
+                     IMG_UINT32 uiPageCount);
+
+/*************************************************************************/ /*!
+ * @Function    DevmemXIntMapVRangeToBackingPage()
+ * @Description Maps a kernel internal backing page to a reserved range.
+ *
+ * @Input       psRsrv             Reservation handle for the range
+ * @Input       uiPageCount        Number of consecutive pages that are
+ *                                 mapped
+ * @Input       uiFlags            Mapping flags
+ * @Input       uiVirtPageOffset   Offset from the reservation base to start the
+ *                                 mapping from (measured in pages)
+ *
+ * @Return      PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemXIntMapVRangeToBackingPage(DEVMEMXINT_RESERVATION *psRsrv,
+                                 IMG_UINT32 uiPageCount,
+                                 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                 IMG_UINT32 uiVirtPageOffset);
+
+/*
+ * DevmemIntInvalidateFBSCTable()
+ *
+ * Invalidate selected FBSC table indices.
+ *
+ */
+PVRSRV_ERROR
+DevmemIntInvalidateFBSCTable(DEVMEMINT_CTX *psDevmemCtx,
+                             IMG_UINT64 ui64FBSCEntryMask);
+
+PVRSRV_ERROR
+DevmemIntIsVDevAddrValid(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDevNode,
+                         DEVMEMINT_CTX *psDevMemContext,
+                         IMG_DEV_VIRTADDR sDevAddr);
+
+PVRSRV_ERROR
+DevmemIntGetFaultAddress(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDevNode,
+                         DEVMEMINT_CTX *psDevMemContext,
+                         IMG_DEV_VIRTADDR *psFaultAddress);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntRegisterPFNotifyKM
+@Description    Registers a PID to be notified when a page fault occurs on a
+                specific device memory context.
+@Input          psDevmemCtx    The context to be notified about.
+@Input          bRegister      If true, register. If false, de-register.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevmemIntRegisterPFNotifyKM(DEVMEMINT_CTX *psDevmemCtx,
+                            IMG_BOOL      bRegister);
+
+/*************************************************************************/ /*!
+@Function       DevmemIntPFNotify
+@Description    Notifies any processes that have registered themselves to be
+                notified when a page fault happens on a specific device memory
+                context.
+@Input          *psDevNode           The device node.
+@Input          ui64FaultedPCAddress The page catalogue address that faulted.
+@Input          sFaultAddress        The address that triggered the fault.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR DevmemIntPFNotify(PVRSRV_DEVICE_NODE *psDevNode,
+                               IMG_UINT64         ui64FaultedPCAddress,
+                               IMG_DEV_VIRTADDR   sFaultAddress);
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize,
+                              DLLIST_NODE *psValidRegionsList);
+
+void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psValidRegionsList);
+
+PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * DevmemIntPDumpSaveToFileVirtual()
+ *
+ * Writes out PDump "SAB" commands with the data found in memory at
+ * the given virtual address.
+ */
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 uiArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * DevmemIntPDumpSaveToFileVirtualNoValidate()
+ *
+ * Writes out PDump "SAB" commands with the data found in memory at
+ * the given virtual address. Doesn't perform address validation.
+ */
+PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtualNoValidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          DEVMEMINT_CTX *psDevmemCtx,
+                                          IMG_DEV_VIRTADDR sDevAddrStart,
+                                          IMG_DEVMEM_SIZE_T uiSize,
+                                          const IMG_CHAR *pszFilename,
+                                          IMG_UINT32 ui32FileOffset,
+                                          IMG_UINT32 ui32PDumpFlags);
+
+IMG_UINT32
+DevmemIntMMUContextID(DEVMEMINT_CTX *psDevMemContext);
+
+PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32LogicalWidth,
+                              IMG_UINT32 ui32LogicalHeight,
+                              IMG_UINT32 ui32PhysicalWidth,
+                              IMG_UINT32 ui32PhysicalHeight,
+                              PDUMP_PIXEL_FORMAT ePixFmt,
+                              IMG_MEMLAYOUT eMemLayout,
+                              IMG_FB_COMPRESSION eFBCompression,
+                              const IMG_UINT32 *paui32FBCClearColour,
+                              PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                              IMG_DEV_VIRTADDR sHeader,
+                              IMG_UINT32 ui32HeaderSize,
+                              IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             DEVMEMINT_CTX *psDevMemContext,
+                             IMG_UINT32 ui32Size,
+                             const IMG_CHAR *pszFileName,
+                             IMG_DEV_VIRTADDR sData,
+                             IMG_UINT32 ui32DataSize,
+                             IMG_UINT32 ui32HeaderType,
+                             IMG_UINT32 ui32ElementType,
+                             IMG_UINT32 ui32ElementCount,
+                             IMG_UINT32 ui32PDumpFlags);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpGetValidRegions)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpGetValidRegions(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevmemCtx,
+                              IMG_DEV_VIRTADDR sDevAddrStart,
+                              IMG_DEVMEM_SIZE_T uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(sDevAddrStart);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpFreeValidRegions)
+#endif
+static INLINE void
+DevmemIntPDumpFreeValidRegions(DLLIST_NODE *psDevAddrRegions)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevAddrRegions);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpSaveFromRegionListToFileVirtual)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpSaveFromRegionListToFileVirtual(CONNECTION_DATA * psConnection,
+                                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              DEVMEMINT_CTX *psDevmemCtx,
+                                              DLLIST_NODE *psDevAddrRegions,
+                                              const IMG_CHAR *pszFilename,
+                                              IMG_UINT32 ui32FileOffset,
+                                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(psDevAddrRegions);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpSaveToFileVirtual)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpSaveToFileVirtual(CONNECTION_DATA * psConnection,
+                                PVRSRV_DEVICE_NODE *psDeviceNode,
+                                DEVMEMINT_CTX *psDevmemCtx,
+                                IMG_DEV_VIRTADDR sDevAddrStart,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_UINT32 uiArraySize,
+                                const IMG_CHAR *pszFilename,
+                                IMG_UINT32 ui32FileOffset,
+                                IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevmemCtx);
+	PVR_UNREFERENCED_PARAMETER(sDevAddrStart);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(ui32FileOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpImageDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpImageDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32LogicalWidth,
+                              IMG_UINT32 ui32LogicalHeight,
+                              IMG_UINT32 ui32PhysicalWidth,
+                              IMG_UINT32 ui32PhysicalHeight,
+                              PDUMP_PIXEL_FORMAT ePixFmt,
+                              IMG_MEMLAYOUT eMemLayout,
+                              IMG_FB_COMPRESSION eFBCompression,
+                              const IMG_UINT32 *paui32FBCClearColour,
+                              PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                              IMG_DEV_VIRTADDR sHeader,
+                              IMG_UINT32 ui32HeaderSize,
+                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevMemContext);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	PVR_UNREFERENCED_PARAMETER(pszFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(DevmemIntPDumpDataDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+DevmemIntPDumpDataDescriptor(CONNECTION_DATA * psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEMINT_CTX *psDevMemContext,
+                              IMG_UINT32 ui32Size,
+                              const IMG_CHAR *pszFileName,
+                              IMG_DEV_VIRTADDR sData,
+                              IMG_UINT32 ui32DataSize,
+                              IMG_UINT32 ui32ElementType,
+                              IMG_UINT32 ui32ElementCount,
+                              IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psDevMemContext);
+	PVR_UNREFERENCED_PARAMETER(ui32Size);
+	PVR_UNREFERENCED_PARAMETER(pszFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#endif /* PDUMP */
+
+PVRSRV_ERROR
+DevmemIntInit(void);
+
+PVRSRV_ERROR
+DevmemIntDeInit(void);
+
+PVRSRV_ERROR
+DevmemIntExportCtx(DEVMEMINT_CTX *psContext,
+                   PMR *psPMR,
+                   DEVMEMINT_CTX_EXPORT **ppsContextExport);
+
+PVRSRV_ERROR
+DevmemIntUnexportCtx(DEVMEMINT_CTX_EXPORT *psContextExport);
+
+PVRSRV_ERROR
+DevmemIntAcquireRemoteCtx(PMR *psPMR,
+                          DEVMEMINT_CTX **ppsContext,
+                          IMG_HANDLE *phPrivData);
+
+#endif /* DEVICEMEM_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/devicemem_server_utils.h b/drivers/gpu/drm/img-rogue/devicemem_server_utils.h
new file mode 100644
index 000000000000..923bfc9a8f18
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_server_utils.h
@@ -0,0 +1,179 @@
+/**************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header file utilities that are specific to device memory functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv.h"
+
+static INLINE PVRSRV_ERROR DevmemCPUCacheMode(PVRSRV_MEMALLOCFLAGS_T ulFlags,
+											  IMG_UINT32 *pui32Ret)
+{
+	IMG_UINT32 ui32CPUCacheMode = PVRSRV_CPU_CACHE_MODE(ulFlags);
+	IMG_UINT32 ui32Ret;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32CPUCacheMode == PVRSRV_CPU_CACHE_MODE(ulFlags));
+
+	switch (ui32CPUCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_CACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT:
+
+			/*
+			 * If system has no coherency but coherency has been requested for CPU
+			 * and GPU we currently fall back to write-combine.
+			 * This avoids errors on arm64 when uncached is turned into ordered device memory
+			 * and suffers from problems with unaligned access.
+			 */
+			if (PVRSRV_GPU_CACHE_MODE(ulFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+			}
+			else
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_CACHED;
+			}
+			break;
+
+		default:
+			PVR_LOG(("DevmemCPUCacheMode: Unknown CPU cache mode 0x%08x", ui32CPUCacheMode));
+			PVR_ASSERT(0);
+			/*
+				We should never get here, but if we do then setting the mode
+				to uncached is the safest thing to do.
+			*/
+			ui32Ret = PVRSRV_MEMALLOCFLAG_CPU_UNCACHED;
+			eError = PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+			break;
+	}
+
+	*pui32Ret = ui32Ret;
+
+	return eError;
+}
+
+static INLINE PVRSRV_ERROR DevmemDeviceCacheMode(PVRSRV_MEMALLOCFLAGS_T ulFlags,
+												 IMG_UINT32 *pui32Ret)
+{
+	IMG_UINT32 ui32DeviceCacheMode = PVRSRV_GPU_CACHE_MODE(ulFlags);
+	IMG_UINT32 ui32Ret;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(ui32DeviceCacheMode == PVRSRV_GPU_CACHE_MODE(ulFlags));
+
+	switch (ui32DeviceCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT:
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_CACHED;
+			break;
+
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT:
+
+			/*
+			 * If system has no coherency but coherency has been requested for CPU
+			 * and GPU we currently fall back to write-combine.
+			 * This avoids errors on arm64 when uncached is turned into ordered device memory
+			 * and suffers from problems with unaligned access.
+			 */
+			if (PVRSRV_CPU_CACHE_MODE(ulFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC;
+			}
+			else
+			{
+				ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_CACHED;
+			}
+			break;
+
+		default:
+			PVR_LOG(("DevmemDeviceCacheMode: Unknown device cache mode 0x%08x", ui32DeviceCacheMode));
+			PVR_ASSERT(0);
+			/*
+				We should never get here, but if we do then setting the mode
+				to uncached is the safest thing to do.
+			*/
+			ui32Ret = PVRSRV_MEMALLOCFLAG_GPU_UNCACHED;
+			eError = PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+			break;
+	}
+
+	*pui32Ret = ui32Ret;
+
+	return eError;
+}
+
+static INLINE IMG_BOOL DevmemDeviceCacheCoherency(PVRSRV_DEVICE_NODE *psDeviceNode,
+												  PVRSRV_MEMALLOCFLAGS_T ulFlags)
+{
+	IMG_UINT32 ui32DeviceCacheMode = PVRSRV_GPU_CACHE_MODE(ulFlags);
+	IMG_BOOL bRet = IMG_FALSE;
+
+	PVR_ASSERT(ui32DeviceCacheMode == PVRSRV_GPU_CACHE_MODE(ulFlags));
+
+	if (ui32DeviceCacheMode == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+	{
+		bRet = PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig);
+	}
+	return bRet;
+}
diff --git a/drivers/gpu/drm/img-rogue/devicemem_typedefs.h b/drivers/gpu/drm/img-rogue/devicemem_typedefs.h
new file mode 100644
index 000000000000..d51c099eed7a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_typedefs.h
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Client side part of device memory management -- this file
+                is forked from new_devmem_allocation.h as this one has to
+                reside in the top level include so that client code is able
+                to make use of the typedefs.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_TYPEDEFS_H
+#define DEVICEMEM_TYPEDEFS_H
+
+#include <powervr/mem_types.h>
+#include "img_types.h"
+#include "pvrsrv_memallocflags.h"
+
+typedef struct DEVMEM_CONTEXT_TAG DEVMEM_CONTEXT;     /*!< Convenience typedef for struct DEVMEM_CONTEXT_TAG */
+typedef struct DEVMEM_HEAP_TAG DEVMEM_HEAP;           /*!< Convenience typedef for struct DEVMEM_HEAP_TAG */
+typedef struct DEVMEM_MEMDESC_TAG DEVMEM_MEMDESC;     /*!< Convenience typedef for struct DEVMEM_MEMDESC_TAG */
+typedef struct DEVMEM_PAGELIST_TAG DEVMEM_PAGELIST;   /*!< Convenience typedef for struct DEVMEM_PAGELIST_TAG */
+
+typedef IMG_HANDLE DEVMEM_EXPORTHANDLE;             /*!< Typedef for DeviceMem Export Handle */
+typedef IMG_UINT64 DEVMEM_EXPORTKEY;                /*!< Typedef for DeviceMem Export Key */
+typedef IMG_DEVMEM_SIZE_T DEVMEM_SIZE_T;            /*!< Typedef for DeviceMem SIZE_T */
+typedef IMG_DEVMEM_LOG2ALIGN_T DEVMEM_LOG2ALIGN_T;  /*!< Typedef for DeviceMem LOG2 Alignment */
+
+typedef struct DEVMEMX_PHYS_MEMDESC_TAG DEVMEMX_PHYSDESC;    /*!< Convenience typedef for DevmemX physical */
+typedef struct DEVMEMX_VIRT_MEMDESC_TAG DEVMEMX_VIRTDESC;    /*!< Convenience typedef for DevmemX virtual */
+
+/*! calling code needs all the info in this struct, to be able to pass it around */
+typedef struct
+{
+    /*! A handle to the PMR. */
+    IMG_HANDLE hPMRExportHandle;
+    /*! The "key" to prove we have authorisation to use this PMR */
+    IMG_UINT64 uiPMRExportPassword;
+    /*! Size and alignment properties for this PMR.  Note, these
+       numbers are not trusted in kernel, but we need to cache them
+       client-side in order to allocate from the VM arena.  The kernel
+       will know the actual alignment and size of the PMR and thus
+       would prevent client code from breaching security here.  Ditto
+       for physmem granularity (aka page size) if this is different
+       from alignment */
+    IMG_DEVMEM_SIZE_T uiSize;
+    /*! We call this "contiguity guarantee" to be more precise than
+       calling it "alignment" or "page size", terms which may seem
+       similar but have different emphasis.  The number reported here
+       is the minimum contiguity guarantee from the creator of the
+       PMR.  Now, there is no requirement to allocate that coarsely
+       from the RA.  The alignment given to the RA simply needs to be
+       at least as coarse as the device page size for the heap we
+       ultimately intend to map into.  What is important is that the
+       device MMU data page size is not greater than the minimum
+       contiguity guarantee from the PMR.  This value is reported to
+       the client in order that it can choose to make early checks and
+       perhaps decide which heap (in a variable page size scenario) it
+       would be safe to map this PMR into.  For convenience, the
+       client may choose to use this argument as the alignment of the
+       virtual range he chooses to allocate, but this is _not_
+       necessary and in many cases would be able to get away with a
+       finer alignment, should the heap into which this PMR will be
+       mapped support it. */
+    IMG_DEVMEM_LOG2ALIGN_T uiLog2ContiguityGuarantee;
+} DEVMEM_EXPORTCOOKIE;
+
+/* Enum that describes the operation associated with changing sparse memory */
+typedef IMG_UINT32 SPARSE_MEM_RESIZE_FLAGS;
+#define SPARSE_RESIZE_NONE 0U
+
+	/* This should be set to indicate the change needs allocation */
+#define SPARSE_RESIZE_ALLOC 1U
+
+	/* This should be set to indicate the change needs free */
+#define SPARSE_RESIZE_FREE 2U
+
+#define SPARSE_RESIZE_BOTH (SPARSE_RESIZE_ALLOC | SPARSE_RESIZE_FREE)
+
+/* To use with DevmemSubAllocate() as the default factor if no over-allocation
+ * is desired.
+ */
+#define DEVMEM_NO_PRE_ALLOCATE_MULTIPLIER (1U)
+
+/* Defines the max length for PMR, MemDesc, Device memory History and RI debug
+ * annotations stored in memory, including the null terminator.
+ */
+#define DEVMEM_ANNOTATION_MAX_LEN ((IMG_UINT32)PVR_ANNOTATION_MAX_LEN + 1U)
+
+
+/* Reserved VA space of a heap must always be multiple of DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+ * Granularity has been chosen to support the max possible practically used OS page size.
+ */
+#define DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY        0x10000 /* 64KB is MAX anticipated OS page size */
+
+#endif /* #ifndef DEVICEMEM_TYPEDEFS_H */
diff --git a/drivers/gpu/drm/img-rogue/devicemem_utils.c b/drivers/gpu/drm/img-rogue/devicemem_utils.c
new file mode 100644
index 000000000000..741063b8b1aa
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_utils.c
@@ -0,0 +1,1254 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management internal utility functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally by device memory management
+                code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "allocmem.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "ra.h"
+#include "devicemem_utils.h"
+#include "client_mm_bridge.h"
+#include "client_cache_bridge.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "client_ri_bridge.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#else
+#include "pvr_bridge_client.h"
+#endif
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "proc_stats.h"
+#endif
+
+#if defined(__KERNEL__)
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+/*
+	SVM heap management support functions for CPU (un)mapping
+ */
+#define DEVMEM_MAP_SVM_USER_MANAGED_RETRY				2
+
+static inline PVRSRV_ERROR
+DevmemCPUMapSVMKernelManaged(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT64 *ui64MapAddress)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64SvmMapAddr;
+	IMG_UINT64 ui64SvmMapAddrEnd;
+	IMG_UINT64 ui64SvmHeapAddrEnd;
+
+	/* SVM heap management always has XXX_MANAGER_KERNEL unless we
+	   have triggered the fall back code-path in which case we
+	   should not be calling into this code-path */
+	PVR_ASSERT(psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_KERNEL);
+
+	/* By acquiring the CPU virtual address here, it essentially
+	   means we lock-down the virtual address for the duration
+	   of the life-cycle of the allocation until a de-allocation
+	   request comes in. Thus the allocation is guaranteed not to
+	   change its virtual address on the CPU during its life-time.
+	   NOTE: Import might have already been CPU Mapped before now,
+	   normally this is not a problem, see fall back */
+	eError = DevmemImportStructCPUMap(psImport);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "DevmemImportStructCPUMap");
+		eError = PVRSRV_ERROR_DEVICEMEM_MAP_FAILED;
+		goto failSVM;
+	}
+
+	/* Supplied kernel mmap virtual address is also device virtual address;
+	   calculate the heap & kernel supplied mmap virtual address limits */
+	ui64SvmMapAddr = (IMG_UINT64)(uintptr_t)psImport->sCPUImport.pvCPUVAddr;
+	ui64SvmHeapAddrEnd = psHeap->sBaseAddress.uiAddr + psHeap->uiSize;
+	ui64SvmMapAddrEnd = ui64SvmMapAddr + psImport->uiSize;
+	PVR_ASSERT(ui64SvmMapAddr != (IMG_UINT64)0);
+
+	/* SVM limit test may fail if processor has more virtual address bits than device */
+	if ((ui64SvmMapAddr >= ui64SvmHeapAddrEnd || ui64SvmMapAddrEnd > ui64SvmHeapAddrEnd) ||
+		(ui64SvmMapAddr & ~(ui64SvmHeapAddrEnd - 1)))
+	{
+		/* Unmap incompatible SVM virtual address, this
+		   may not release address if it was elsewhere
+		   CPU Mapped before call into this function */
+		DevmemImportStructCPUUnmap(psImport);
+
+		/* Flag incompatible SVM mapping */
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto failSVM;
+	}
+
+	*ui64MapAddress = ui64SvmMapAddr;
+failSVM:
+	/* either OK, MAP_FAILED or BAD_MAPPING */
+	return eError;
+}
+
+static inline void
+DevmemCPUUnmapSVMKernelManaged(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	PVR_UNREFERENCED_PARAMETER(psHeap);
+	DevmemImportStructCPUUnmap(psImport);
+}
+
+static inline PVRSRV_ERROR
+DevmemCPUMapSVMUserManaged(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT uiAlign,
+		IMG_UINT64 *ui64MapAddress)
+{
+	RA_LENGTH_T uiAllocatedSize;
+	RA_BASE_T uiAllocatedAddr;
+	IMG_UINT64 ui64SvmMapAddr;
+	IMG_UINT uiRetry = 0;
+	PVRSRV_ERROR eError;
+
+	/* If SVM heap management has transitioned to XXX_MANAGER_USER,
+	   this is essentially a fall back approach that ensures we
+	   continue to satisfy SVM alloc. This approach is not without
+	   hazards in that we may specify a virtual address that is
+	   already in use by the user process */
+	PVR_ASSERT(psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER);
+
+	/* Normally, for SVM heap allocations, CPUMap _must_ be done
+	   before DevMap; ideally the initial CPUMap should be done by
+	   SVM functions though this is not a hard requirement as long
+	   as the prior elsewhere obtained CPUMap virtual address meets
+	   SVM address requirements. This is a fall-back code-pathway
+	   so we have to test that this assumption holds before we
+	   progress any further */
+	OSLockAcquire(psImport->sCPUImport.hLock);
+
+	if (psImport->sCPUImport.ui32RefCount)
+	{
+		/* Already CPU Mapped SVM heap allocation, this prior elsewhere
+		   obtained virtual address is responsible for the above
+		   XXX_MANAGER_KERNEL failure. As we are not responsible for
+		   this, we cannot progress any further so need to fail */
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Previously obtained CPU map address not SVM compatible"
+				, __func__));
+
+		/* Revert SVM heap to DEVMEM_HEAP_MANAGER_KERNEL */
+		psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_KERNEL;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"%s: Reverting SVM heap back to kernel managed",
+				__func__));
+
+		OSLockRelease(psImport->sCPUImport.hLock);
+
+		/* Do we need a more specific error code here */
+		eError = PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED;
+		goto failSVM;
+	}
+
+	OSLockRelease(psImport->sCPUImport.hLock);
+
+	do
+	{
+		/* Next we proceed to instruct the kernel to use the RA_Alloc supplied
+		   virtual address to map-in this SVM import suballocation; there is no
+		   guarantee that this RA_Alloc virtual address may not collide with an
+		   already in-use VMA range in the process */
+		eError = RA_Alloc(psHeap->psQuantizedVMRA,
+				psImport->uiSize,
+				RA_NO_IMPORT_MULTIPLIER,
+				0, /* flags: this RA doesn't use flags*/
+				uiAlign,
+				"SVM_Virtual_Alloc",
+				&uiAllocatedAddr,
+				&uiAllocatedSize,
+				NULL /* don't care about per-import priv data */);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "RA_Alloc");
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			if (eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+			{
+				PVRSRV_ERROR eErr;
+				eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+								  PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT,
+								  OSGetCurrentProcessID());
+				PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+			}
+#endif
+			goto failSVM;
+		}
+
+		/* No reason for allocated virtual size to be different from
+		   the PMR's size */
+		psImport->sCPUImport.pvCPUVAddr = (void*)(uintptr_t)uiAllocatedAddr;
+		PVR_ASSERT(uiAllocatedSize == psImport->uiSize);
+
+		/* Map the import or allocation using the RA_Alloc virtual address;
+		   the kernel may fail the request if the supplied virtual address
+		   is already in-use in which case we re-try using another virtual
+		   address obtained from the RA_Alloc */
+		eError = DevmemImportStructCPUMap(psImport);
+		if (eError != PVRSRV_OK)
+		{
+			/* For now we simply discard failed RA_Alloc() obtained virtual
+			   address (i.e. plenty of virtual space), this prevents us from
+			   re-using these and furthermore essentially blacklists these
+			   addresses from future SVM consideration; We exit fall-back
+			   attempt if retry exceeds the fall-back retry limit */
+			if (uiRetry++ > DEVMEM_MAP_SVM_USER_MANAGED_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Cannot find SVM compatible address, bad mapping",
+						__func__));
+				eError = PVRSRV_ERROR_BAD_MAPPING;
+				goto failSVM;
+			}
+		}
+		else
+		{
+			/* Found compatible SVM virtual address, set as device virtual address */
+			ui64SvmMapAddr = (IMG_UINT64)(uintptr_t)psImport->sCPUImport.pvCPUVAddr;
+		}
+	} while (eError != PVRSRV_OK);
+
+	*ui64MapAddress = ui64SvmMapAddr;
+failSVM:
+	return eError;
+}
+
+static inline void
+DevmemCPUUnmapSVMUserManaged(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	RA_BASE_T uiAllocatedAddr;
+
+	/* We only free SVM compatible addresses, all addresses in
+	   the blacklist are essentially excluded from future RA_Alloc */
+	uiAllocatedAddr = psImport->sDeviceImport.sDevVAddr.uiAddr;
+	RA_Free(psHeap->psQuantizedVMRA, uiAllocatedAddr);
+
+	DevmemImportStructCPUUnmap(psImport);
+}
+
+static inline PVRSRV_ERROR
+DevmemImportStructDevMapSVM(DEVMEM_HEAP *psHeap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT uiAlign,
+		IMG_UINT64 *ui64MapAddress)
+{
+	PVRSRV_ERROR eError;
+
+	switch (psHeap->ui32HeapManagerFlags)
+	{
+	case DEVMEM_HEAP_MANAGER_KERNEL:
+		eError = DevmemCPUMapSVMKernelManaged(psHeap,
+				psImport,
+				ui64MapAddress);
+		if (eError == PVRSRV_ERROR_BAD_MAPPING)
+		{
+			/* If the SVM map address is outside of SVM heap limits,
+				   change heap type to DEVMEM_HEAP_MANAGER_USER */
+			psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_USER;
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Kernel managed SVM heap is now user managed",
+					__func__));
+
+			/* Retry using user managed fall-back approach */
+			eError = DevmemCPUMapSVMUserManaged(psHeap,
+					psImport,
+					uiAlign,
+					ui64MapAddress);
+		}
+		break;
+
+	case DEVMEM_HEAP_MANAGER_USER:
+		eError = DevmemCPUMapSVMUserManaged(psHeap,
+				psImport,
+				uiAlign,
+				ui64MapAddress);
+		break;
+
+	default:
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		break;
+	}
+
+	return eError;
+}
+
+static inline void
+DevmemImportStructDevUnmapSVM(DEVMEM_HEAP *psHeap, DEVMEM_IMPORT *psImport)
+{
+	switch (psHeap->ui32HeapManagerFlags)
+	{
+	case DEVMEM_HEAP_MANAGER_KERNEL:
+		DevmemCPUUnmapSVMKernelManaged(psHeap, psImport);
+		break;
+
+	case DEVMEM_HEAP_MANAGER_USER:
+		DevmemCPUUnmapSVMUserManaged(psHeap, psImport);
+		break;
+
+	default:
+		break;
+	}
+}
+
+/*
+	The Devmem import structure is the structure we use
+	to manage memory that is "imported" (which is page
+	granular) from the server into our process, this
+	includes allocations.
+
+	This allows memory to be imported without requiring
+	any CPU or device mapping. Memory can then be mapped
+	into the device or CPU on demand, but neither is
+	required.
+ */
+
+IMG_INTERNAL
+void DevmemImportStructAcquire(DEVMEM_IMPORT *psImport)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psImport->hRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+	PVR_ASSERT(iRefCount != 1);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			iRefCount-1,
+			iRefCount);
+}
+
+IMG_INTERNAL
+IMG_BOOL DevmemImportStructRelease(DEVMEM_IMPORT *psImport)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psImport->hRefCount);
+	PVR_ASSERT(iRefCount >= 0);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			iRefCount+1,
+			iRefCount);
+
+	if (iRefCount == 0)
+	{
+		PVRSRV_ERROR eError = DestroyServerResource(psImport->hDevConnection,
+		                                            NULL,
+		                                            BridgePMRUnrefPMR,
+		                                            psImport->hPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		OSLockDestroy(psImport->sCPUImport.hLock);
+		OSLockDestroy(psImport->sDeviceImport.hLock);
+		OSLockDestroy(psImport->hLock);
+		OSFreeMem(psImport);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_INTERNAL
+void DevmemImportDiscard(DEVMEM_IMPORT *psImport)
+{
+	PVR_ASSERT(OSAtomicRead(&psImport->hRefCount) == 0);
+	OSLockDestroy(psImport->sCPUImport.hLock);
+	OSLockDestroy(psImport->sDeviceImport.hLock);
+	OSLockDestroy(psImport->hLock);
+	OSFreeMem(psImport);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR DevmemMemDescAlloc(DEVMEM_MEMDESC **ppsMemDesc)
+{
+	DEVMEM_MEMDESC *psMemDesc;
+	PVRSRV_ERROR eError;
+
+	/* Must be zeroed in case it needs to be freed before it is initialised */
+	psMemDesc = OSAllocZMem(sizeof(DEVMEM_MEMDESC));
+	PVR_GOTO_IF_NOMEM(psMemDesc, eError, failAlloc);
+
+	eError = OSLockCreate(&psMemDesc->hLock);
+	PVR_GOTO_IF_ERROR(eError, failMDLock);
+
+	eError = OSLockCreate(&psMemDesc->sDeviceMemDesc.hLock);
+	PVR_GOTO_IF_ERROR(eError, failDMDLock);
+
+	eError = OSLockCreate(&psMemDesc->sCPUMemDesc.hLock);
+	PVR_GOTO_IF_ERROR(eError, failCMDLock);
+
+	OSAtomicWrite(&psMemDesc->hRefCount, 0);
+
+	*ppsMemDesc = psMemDesc;
+
+	return PVRSRV_OK;
+
+failCMDLock:
+	OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+failDMDLock:
+	OSLockDestroy(psMemDesc->hLock);
+failMDLock:
+	OSFreeMem(psMemDesc);
+failAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	Init the MemDesc structure
+ */
+IMG_INTERNAL
+void DevmemMemDescInit(DEVMEM_MEMDESC *psMemDesc,
+		IMG_DEVMEM_OFFSET_T uiOffset,
+		DEVMEM_IMPORT *psImport,
+		IMG_DEVMEM_SIZE_T uiSize)
+{
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			0,
+			1);
+
+	psMemDesc->psImport = psImport;
+	psMemDesc->uiOffset = uiOffset;
+
+	psMemDesc->sDeviceMemDesc.ui32RefCount = 0;
+	psMemDesc->sCPUMemDesc.ui32RefCount = 0;
+	psMemDesc->uiAllocSize = uiSize;
+	psMemDesc->hPrivData = NULL;
+	psMemDesc->ui32AllocationIndex = DEVICEMEM_HISTORY_ALLOC_INDEX_NONE;
+
+#if defined(DEBUG)
+	psMemDesc->bPoisonOnFree = IMG_FALSE;
+#endif
+
+	OSAtomicWrite(&psMemDesc->hRefCount, 1);
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+void DevmemMemDescSetPoF(DEVMEM_MEMDESC *psMemDesc, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		psMemDesc->bPoisonOnFree = IMG_TRUE;
+	}
+}
+#endif
+
+IMG_INTERNAL
+void DevmemMemDescAcquire(DEVMEM_MEMDESC *psMemDesc)
+{
+	IMG_INT iRefCount = 0;
+
+	iRefCount = OSAtomicIncrement(&psMemDesc->hRefCount);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			iRefCount-1,
+			iRefCount);
+
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+#if defined(DEBUG)
+static void _DevmemPoisonOnFree(DEVMEM_MEMDESC *psMemDesc)
+{
+	void *pvAddr = NULL;
+	IMG_UINT8 *pui8CPUVAddr;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemCPUMapCheckImportProperties(psMemDesc);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "DevmemCPUMapCheckImportProperties");
+
+	OSLockAcquire(psMemDesc->sCPUMemDesc.hLock);
+	eError = DevmemImportStructCPUMap(psMemDesc->psImport);
+	OSLockRelease(psMemDesc->sCPUMemDesc.hLock);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "DevmemImportStructCPUMap");
+
+	pui8CPUVAddr = psMemDesc->psImport->sCPUImport.pvCPUVAddr;
+	pui8CPUVAddr += psMemDesc->uiOffset;
+	pvAddr = pui8CPUVAddr;
+
+	DevmemCPUMemSet(pvAddr,
+	                PVRSRV_POISON_ON_FREE_VALUE,
+	                psMemDesc->uiAllocSize,
+	                psMemDesc->psImport->uiFlags);
+
+	if (PVRSRV_CHECK_CPU_CACHE_COHERENT(psMemDesc->psImport->uiFlags) ||
+	    PVRSRV_CHECK_CPU_CACHE_INCOHERENT(psMemDesc->psImport->uiFlags))
+	{
+		eError = BridgeCacheOpExec(GetBridgeHandle(psMemDesc->psImport->hDevConnection),
+								   psMemDesc->psImport->hPMR,
+								   (IMG_UINT64) (uintptr_t)
+								   pvAddr - psMemDesc->uiOffset,
+								   psMemDesc->uiOffset,
+								   psMemDesc->uiAllocSize,
+								   PVRSRV_CACHE_OP_FLUSH);
+		PVR_LOG_IF_ERROR(eError, "BridgeCacheOpExec");
+	}
+
+	DevmemImportStructCPUUnmap(psMemDesc->psImport);
+	pvAddr = NULL;
+}
+#endif
+
+IMG_INTERNAL
+IMG_BOOL DevmemMemDescRelease(DEVMEM_MEMDESC *psMemDesc)
+{
+	IMG_INT iRefCount;
+	PVR_ASSERT(psMemDesc != NULL);
+
+	iRefCount = OSAtomicDecrement(&psMemDesc->hRefCount);
+	PVR_ASSERT(iRefCount >= 0);
+
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psMemDesc,
+			iRefCount+1,
+			iRefCount);
+
+	if (iRefCount == 0)
+	{
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		if (PVRSRVIsBridgeEnabled(GetBridgeHandle(psMemDesc->psImport->hDevConnection), PVRSRV_BRIDGE_RI) &&
+		    (psMemDesc->hRIHandle))
+		{
+			PVRSRV_ERROR eError;
+
+			eError = DestroyServerResource(psMemDesc->psImport->hDevConnection,
+			                               NULL,
+			                               BridgeRIDeleteMEMDESCEntry,
+			                               psMemDesc->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "BridgeRIDeleteMEMDESCEntry");
+		}
+#endif
+
+		OSLockAcquire(psMemDesc->psImport->hLock);
+		if (psMemDesc->psImport->uiProperties & DEVMEM_PROPERTIES_SUBALLOCATABLE)
+		{
+			/* As soon as the first sub-allocation on the psImport is freed
+			 * we might get dirty memory when reusing it.
+			 * We have to delete the ZEROED, CLEAN & POISONED flag */
+			psMemDesc->psImport->uiProperties &=
+					~(DEVMEM_PROPERTIES_IMPORT_IS_ZEROED |
+							DEVMEM_PROPERTIES_IMPORT_IS_CLEAN |
+							DEVMEM_PROPERTIES_IMPORT_IS_POISONED);
+
+			OSLockRelease(psMemDesc->psImport->hLock);
+
+#if defined(DEBUG)
+			if (psMemDesc->bPoisonOnFree)
+			{
+				_DevmemPoisonOnFree(psMemDesc);
+			}
+#endif
+
+			RA_Free(psMemDesc->psImport->sDeviceImport.psHeap->psSubAllocRA,
+					psMemDesc->psImport->sDeviceImport.sDevVAddr.uiAddr +
+					psMemDesc->uiOffset);
+		}
+		else
+		{
+			OSLockRelease(psMemDesc->psImport->hLock);
+			DevmemImportStructRelease(psMemDesc->psImport);
+		}
+
+		OSLockDestroy(psMemDesc->sCPUMemDesc.hLock);
+		OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+		OSLockDestroy(psMemDesc->hLock);
+		OSFreeMem(psMemDesc);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+IMG_INTERNAL
+void DevmemMemDescDiscard(DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_ASSERT(OSAtomicRead(&psMemDesc->hRefCount) == 0);
+
+	OSLockDestroy(psMemDesc->sCPUMemDesc.hLock);
+	OSLockDestroy(psMemDesc->sDeviceMemDesc.hLock);
+	OSLockDestroy(psMemDesc->hLock);
+	OSFreeMem(psMemDesc);
+}
+
+
+IMG_INTERNAL
+PVRSRV_ERROR DevmemValidateParams(IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &&
+			(*puiFlags & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Zero on Alloc and Poison on Alloc are mutually exclusive.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((*puiFlags & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC) &&
+			(*puiFlags & PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Defer Alloc and Alloc Now are mutually exclusive.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiAlign & (uiAlign-1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: The requested alignment is not a power of two.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Please request a non-zero size value.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* If zero or poison flags are set we have to have write access to the page. */
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(*puiFlags) ||
+	    PVRSRV_CHECK_POISON_ON_ALLOC(*puiFlags) ||
+#if defined(DEBUG)
+	    PVRSRV_CHECK_POISON_ON_FREE(*puiFlags) ||
+#endif
+	    PVRSRV_CHECK_CPU_WRITEABLE(*puiFlags))
+	{
+		(*puiFlags) |= PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	Allocate and init an import structure
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructAlloc(SHARED_DEV_CONNECTION hDevConnection,
+		DEVMEM_IMPORT **ppsImport)
+{
+	DEVMEM_IMPORT *psImport;
+	PVRSRV_ERROR eError;
+
+	psImport = OSAllocMem(sizeof(*psImport));
+	PVR_RETURN_IF_FALSE(psImport != NULL, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	/* Setup some known bad values for things we don't have yet */
+	psImport->sDeviceImport.hReservation = LACK_OF_RESERVATION_POISON;
+	psImport->sDeviceImport.hMapping = LACK_OF_MAPPING_POISON;
+	psImport->sDeviceImport.psHeap = NULL;
+	psImport->sDeviceImport.bMapped = IMG_FALSE;
+
+	eError = OSLockCreate(&psImport->sDeviceImport.hLock);
+	PVR_GOTO_IF_ERROR(eError, failDIOSLockCreate);
+
+	psImport->sCPUImport.hOSMMapData = NULL;
+	psImport->sCPUImport.pvCPUVAddr = NULL;
+
+	eError = OSLockCreate(&psImport->sCPUImport.hLock);
+	PVR_GOTO_IF_ERROR(eError, failCIOSLockCreate);
+
+	/* Set up common elements */
+	psImport->hDevConnection = hDevConnection;
+
+	/* Setup properties */
+	psImport->uiProperties = 0;
+
+	/* Setup refcounts */
+	psImport->sDeviceImport.ui32RefCount = 0;
+	psImport->sCPUImport.ui32RefCount = 0;
+	OSAtomicWrite(&psImport->hRefCount, 0);
+
+	/* Create the lock */
+	eError = OSLockCreate(&psImport->hLock);
+	PVR_GOTO_IF_ERROR(eError, failILockAlloc);
+
+	*ppsImport = psImport;
+
+	return PVRSRV_OK;
+
+failILockAlloc:
+	OSLockDestroy(psImport->sCPUImport.hLock);
+failCIOSLockCreate:
+	OSLockDestroy(psImport->sDeviceImport.hLock);
+failDIOSLockCreate:
+	OSFreeMem(psImport);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	Initialise the import structure
+ */
+IMG_INTERNAL
+void DevmemImportStructInit(DEVMEM_IMPORT *psImport,
+		IMG_DEVMEM_SIZE_T uiSize,
+		IMG_DEVMEM_ALIGN_T uiAlign,
+		PVRSRV_MEMALLOCFLAGS_T uiFlags,
+		IMG_HANDLE hPMR,
+		DEVMEM_PROPERTIES_T uiProperties)
+{
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			0,
+			1);
+
+	psImport->uiSize = uiSize;
+	psImport->uiAlign = uiAlign;
+	psImport->uiFlags = uiFlags;
+	psImport->hPMR = hPMR;
+	psImport->uiProperties = uiProperties;
+	OSAtomicWrite(&psImport->hRefCount, 1);
+}
+
+/* Allocate the requested device virtual address region
+ * from the heap */
+static PVRSRV_ERROR DevmemReserveVARange(DEVMEM_HEAP *psHeap,
+                                         DEVMEM_SIZE_T uiSize,
+                                         IMG_UINT uiAlign,
+                                         RA_LENGTH_T *puiAllocatedSize,
+                                         IMG_UINT64 ui64OptionalMapAddress)
+{
+	PVRSRV_ERROR eError;
+
+	/* Allocate space in the VM */
+	eError = RA_Alloc_Range(psHeap->psQuantizedVMRA,
+							uiSize,
+							0,
+							uiAlign,
+							ui64OptionalMapAddress,
+							puiAllocatedSize);
+
+	if (PVRSRV_OK != eError)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		if ((eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL) ||
+				(eError == PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL))
+		{
+			PVRSRV_ERROR eErr;
+			eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+											PVRSRV_DEVICE_STAT_TYPE_INVALID_VIRTMEM,
+											OSGetCurrentProcessID());
+			PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+		}
+#endif
+		return eError;
+	}
+
+	/* No reason for the allocated virtual size to be different from
+					   the PMR's size */
+	PVR_ASSERT(*puiAllocatedSize == uiSize);
+
+	return PVRSRV_OK;
+}
+
+/*
+	Map an import to the device
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructDevMap(DEVMEM_HEAP *psHeap,
+		IMG_BOOL bMap,
+		DEVMEM_IMPORT *psImport,
+		IMG_UINT64 ui64OptionalMapAddress)
+{
+	DEVMEM_DEVICE_IMPORT *psDeviceImport;
+	RA_BASE_T uiAllocatedAddr;
+	RA_LENGTH_T uiAllocatedSize;
+	IMG_DEV_VIRTADDR sBase;
+	PVRSRV_ERROR eError;
+	IMG_UINT uiAlign;
+	IMG_BOOL bDestroyed = IMG_FALSE;
+
+	/* Round the provided import alignment to the configured heap alignment */
+	uiAlign = 1ULL << psHeap->uiLog2ImportAlignment;
+	uiAlign = PVR_ALIGN(psImport->uiAlign, uiAlign);
+
+	psDeviceImport = &psImport->sDeviceImport;
+
+	OSLockAcquire(psDeviceImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psDeviceImport->ui32RefCount,
+			psDeviceImport->ui32RefCount+1);
+
+	if (psDeviceImport->ui32RefCount++ == 0)
+	{
+		DevmemImportStructAcquire(psImport);
+
+		OSAtomicIncrement(&psHeap->hImportCount);
+
+		if (PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+		{
+			/*  SVM (shared virtual memory) imports or allocations always
+				need to acquire CPU virtual address first as address is
+				used to map the allocation into the device virtual address
+				space; i.e. the virtual address of the allocation for both
+				the CPU/GPU must be identical. */
+			eError = DevmemImportStructDevMapSVM(psHeap,
+					psImport,
+					uiAlign,
+					&ui64OptionalMapAddress);
+			PVR_GOTO_IF_ERROR(eError, failVMRAAlloc);
+		}
+
+		if (ui64OptionalMapAddress == 0)
+		{
+			/* If heap is _completely_ managed by USER or KERNEL, we shouldn't
+			 * be here, as this is RA manager code-path */
+			if (psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ||
+				psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_KERNEL)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ?
+						"%s: Heap is user managed, please use PVRSRVMapToDeviceAddress().":
+						"%s: Heap is kernel managed, use right allocation flags (e.g. SVM).",
+						__func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			if (psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_UNKNOWN)
+			{
+				/* Only set the heap manager (to RA) at first map when heap manager
+				 * is unknown. It might be a dual heap (both, user and RA managed),
+				 * in which case heap manager is set at creation time */
+				psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_RA;
+			}
+
+			/* Allocate space in the VM */
+			eError = RA_Alloc(psHeap->psQuantizedVMRA,
+					psImport->uiSize,
+					RA_NO_IMPORT_MULTIPLIER,
+					0, /* flags: this RA doesn't use flags*/
+					uiAlign,
+					"Virtual_Alloc",
+					&uiAllocatedAddr,
+					&uiAllocatedSize,
+					NULL /* don't care about per-import priv data */
+			);
+			if (PVRSRV_OK != eError)
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				if (eError == PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+				{
+					PVRSRV_ERROR eErr;
+					eErr = BridgePVRSRVStatsUpdateOOMStat(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+									  PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT,
+									  OSGetCurrentProcessID());
+					PVR_LOG_IF_ERROR(eErr, "BridgePVRSRVStatsUpdateOOMStat");
+				}
+#endif
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_DEVICE_VM, failVMRAAlloc);
+			}
+
+			/* No reason for the allocated virtual size to be different from
+			   the PMR's size */
+			PVR_ASSERT(uiAllocatedSize == psImport->uiSize);
+
+			sBase.uiAddr = uiAllocatedAddr;
+
+		}
+		else
+		{
+			IMG_UINT64 ui64ValidEndAddr;
+
+			/* Ensure supplied ui64OptionalMapAddress is within heap range */
+			ui64ValidEndAddr = psHeap->sBaseAddress.uiAddr + psHeap->uiSize;
+			if ((ui64OptionalMapAddress + psImport->uiSize > ui64ValidEndAddr) ||
+					(ui64OptionalMapAddress < psHeap->sBaseAddress.uiAddr))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: ui64OptionalMapAddress %p is outside of heap limits <%p:%p>."
+						, __func__
+						, (void*)(uintptr_t)ui64OptionalMapAddress
+						, (void*)(uintptr_t)psHeap->sBaseAddress.uiAddr
+						, (void*)(uintptr_t)ui64ValidEndAddr));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			switch (psHeap->ui32HeapManagerFlags)
+			{
+				case DEVMEM_HEAP_MANAGER_UNKNOWN:
+					/* DEVMEM_HEAP_MANAGER_USER can apply to _any_ heap and can only
+					 * be determined here. This heap type transitions from
+					 * DEVMEM_HEAP_MANAGER_UNKNOWN to DEVMEM_HEAP_MANAGER_USER on
+					 * 1st alloc. */
+					psHeap->ui32HeapManagerFlags = DEVMEM_HEAP_MANAGER_USER;
+					break;
+
+				case DEVMEM_HEAP_MANAGER_USER:
+				case DEVMEM_HEAP_MANAGER_KERNEL:
+					if (! psHeap->uiSize)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								psHeap->ui32HeapManagerFlags == DEVMEM_HEAP_MANAGER_USER ?
+										"%s: Heap DEVMEM_HEAP_MANAGER_USER is disabled.":
+										"%s: Heap DEVMEM_HEAP_MANAGER_KERNEL is disabled."
+										, __func__));
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAP, failVMRAAlloc);
+					}
+					break;
+
+				case DEVMEM_HEAP_MANAGER_DUAL_USER_RA:
+					/* When the heap is dual managed, ensure supplied ui64OptionalMapAddress
+					 * and import size are within heap address space range */
+					if (ui64OptionalMapAddress + psImport->uiSize <=
+							psHeap->sBaseAddress.uiAddr + psHeap->uiReservedRegionSize)
+					{
+						break;
+					}
+					else
+					{
+						/* Allocate requested VM range */
+						eError = DevmemReserveVARange(psHeap,
+													psImport->uiSize,
+													uiAlign,
+													&uiAllocatedSize,
+													ui64OptionalMapAddress);
+						if (eError != PVRSRV_OK)
+						{
+							PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED, failVMRAAlloc);
+						}
+
+					}
+					break;
+				case DEVMEM_HEAP_MANAGER_RA:
+					/* Allocate requested VM range */
+					eError = DevmemReserveVARange(psHeap,
+												psImport->uiSize,
+												uiAlign,
+												&uiAllocatedSize,
+												ui64OptionalMapAddress);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED, failVMRAAlloc);
+					}
+					break;
+
+				default:
+					break;
+			}
+
+			if (ui64OptionalMapAddress & ((1 << psHeap->uiLog2Quantum) - 1))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid address to map to. Please provide an "
+						"address aligned to a page multiple of the heap."
+						, __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			if (psImport->uiSize & ((1 << psHeap->uiLog2Quantum) - 1))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Invalid heap to map to. "
+						"Please choose a heap that can handle smaller page sizes."
+						, __func__));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, failVMRAAlloc);
+			}
+
+			uiAllocatedAddr = ui64OptionalMapAddress;
+			uiAllocatedSize = psImport->uiSize;
+			sBase.uiAddr = uiAllocatedAddr;
+		}
+
+		if (psHeap->bPremapped)
+		{
+			/* No virtual address reservation and mapping are required for
+			 * memory that is already pre-mapped e.g. FW heaps in VZ configs */
+			psDeviceImport->hReservation = LACK_OF_RESERVATION_POISON;
+			psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+		}
+		else
+		{
+			PVRSRV_MEMALLOCFLAGS_T uiFlags;
+			uiFlags = psImport->uiFlags & PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK;
+			if (bMap)
+			{
+				eError = BridgeDevmemIntReserveRangeAndMapPMR(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+				                                              psHeap->hDevMemServerHeap,
+				                                              sBase,
+				                                              uiAllocatedSize,
+				                                              psImport->hPMR,
+				                                              uiFlags,
+				                                              &psDeviceImport->hReservation);
+
+				psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+				PVR_GOTO_IF_ERROR(eError, failReserve);
+
+				psDeviceImport->bMapped = IMG_TRUE;
+			}
+			else
+			{
+				/* Setup page tables for the allocated VM space */
+				eError = BridgeDevmemIntReserveRange(GetBridgeHandle(psHeap->psCtx->hDevConnection),
+				                                     psHeap->hDevMemServerHeap,
+				                                     sBase,
+				                                     uiAllocatedSize,
+				                                     uiFlags,
+				                                     &psDeviceImport->hReservation);
+				PVR_GOTO_IF_ERROR(eError, failReserve);
+			}
+		}
+
+		/* Setup device mapping specific parts of the mapping info */
+		psDeviceImport->sDevVAddr.uiAddr = uiAllocatedAddr;
+		psDeviceImport->psHeap = psHeap;
+	}
+	else
+	{
+		/*
+			Check that we've been asked to map it into the
+			same heap 2nd time around
+		 */
+		if (psHeap != psDeviceImport->psHeap)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAP, failParams);
+		}
+	}
+	OSLockRelease(psDeviceImport->hLock);
+
+	return PVRSRV_OK;
+
+failReserve:
+	if (ui64OptionalMapAddress == 0)
+	{
+		RA_Free(psHeap->psQuantizedVMRA,
+				uiAllocatedAddr);
+	}
+failVMRAAlloc:
+	if ((ui64OptionalMapAddress) && PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+	{
+		DevmemImportStructDevUnmapSVM(psHeap, psImport);
+	}
+	bDestroyed = DevmemImportStructRelease(psImport);
+	OSAtomicDecrement(&psHeap->hImportCount);
+failParams:
+	if (!bDestroyed)
+	{
+		psDeviceImport->ui32RefCount--;
+		OSLockRelease(psDeviceImport->hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+	Unmap an import from the Device
+ */
+IMG_INTERNAL
+IMG_BOOL DevmemImportStructDevUnmap(DEVMEM_IMPORT *psImport)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_DEVICE_IMPORT *psDeviceImport;
+
+	psDeviceImport = &psImport->sDeviceImport;
+
+	OSLockAcquire(psDeviceImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psDeviceImport->ui32RefCount,
+			psDeviceImport->ui32RefCount-1);
+
+	if (--psDeviceImport->ui32RefCount == 0)
+	{
+		DEVMEM_HEAP *psHeap = psDeviceImport->psHeap;
+
+		if (!psHeap->bPremapped)
+		{
+			eError = DestroyServerResource(psImport->hDevConnection,
+			                               NULL,
+			                               BridgeDevmemIntUnreserveRange,
+			                               psDeviceImport->hReservation);
+			PVR_ASSERT(eError == PVRSRV_OK);
+		}
+
+		psDeviceImport->bMapped = IMG_FALSE;
+		psDeviceImport->hMapping = LACK_OF_MAPPING_POISON;
+		psDeviceImport->hReservation = LACK_OF_RESERVATION_POISON;
+
+		/* DEVMEM_HEAP_MANAGER_RA can also come from a dual managed heap in which case,
+		   we need to check if the allocated VA falls within RA managed range */
+		if ((psHeap->ui32HeapManagerFlags & DEVMEM_HEAP_MANAGER_RA) &&
+		    psDeviceImport->sDevVAddr.uiAddr >= (psHeap->sBaseAddress.uiAddr + psHeap->uiReservedRegionSize) &&
+		    psDeviceImport->sDevVAddr.uiAddr < (psHeap->sBaseAddress.uiAddr + psHeap->uiSize))
+		{
+			RA_Free(psHeap->psQuantizedVMRA, psDeviceImport->sDevVAddr.uiAddr);
+		}
+
+		if (PVRSRV_CHECK_SVM_ALLOC(psImport->uiFlags))
+		{
+			DevmemImportStructDevUnmapSVM(psHeap, psImport);
+		}
+
+		OSLockRelease(psDeviceImport->hLock);
+
+		DevmemImportStructRelease(psImport);
+
+		OSAtomicDecrement(&psHeap->hImportCount);
+
+		return IMG_TRUE;
+	}
+	else
+	{
+		OSLockRelease(psDeviceImport->hLock);
+		return IMG_FALSE;
+	}
+}
+
+/*
+	Map an import into the CPU
+ */
+IMG_INTERNAL
+PVRSRV_ERROR DevmemImportStructCPUMap(DEVMEM_IMPORT *psImport)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_CPU_IMPORT *psCPUImport;
+	size_t uiMappingLength;
+
+	psCPUImport = &psImport->sCPUImport;
+
+	OSLockAcquire(psCPUImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psCPUImport->ui32RefCount,
+			psCPUImport->ui32RefCount+1);
+
+	if (psCPUImport->ui32RefCount++ == 0)
+	{
+		DevmemImportStructAcquire(psImport);
+
+		eError = OSMMapPMR(GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				psImport->uiSize,
+				psImport->uiFlags,
+				&psCPUImport->hOSMMapData,
+				&psCPUImport->pvCPUVAddr,
+				&uiMappingLength);
+		PVR_GOTO_IF_ERROR(eError, failMap);
+
+		/* MappingLength might be rounded up to page size */
+		PVR_ASSERT(uiMappingLength >= psImport->uiSize);
+	}
+	OSLockRelease(psCPUImport->hLock);
+
+	return PVRSRV_OK;
+
+failMap:
+	psCPUImport->ui32RefCount--;
+	if (!DevmemImportStructRelease(psImport))
+	{
+		OSLockRelease(psCPUImport->hLock);
+	}
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+	Unmap an import from the CPU
+ */
+IMG_INTERNAL
+void DevmemImportStructCPUUnmap(DEVMEM_IMPORT *psImport)
+{
+	DEVMEM_CPU_IMPORT *psCPUImport;
+
+	psCPUImport = &psImport->sCPUImport;
+
+	OSLockAcquire(psCPUImport->hLock);
+	DEVMEM_REFCOUNT_PRINT("%s (%p) %d->%d",
+			__func__,
+			psImport,
+			psCPUImport->ui32RefCount,
+			psCPUImport->ui32RefCount-1);
+
+	if (--psCPUImport->ui32RefCount == 0)
+	{
+		/* psImport->uiSize is a 64-bit quantity whereas the 5th
+		 * argument to OSUnmapPMR is a 32-bit quantity on 32-bit systems
+		 * hence a compiler warning of implicit cast and loss of data.
+		 * Added explicit cast and assert to remove warning.
+		 */
+#if defined(__linux__) && defined(__i386__)
+		PVR_ASSERT(psImport->uiSize<IMG_UINT32_MAX);
+#endif
+		OSMUnmapPMR(GetBridgeHandle(psImport->hDevConnection),
+				psImport->hPMR,
+				psCPUImport->hOSMMapData,
+				psCPUImport->pvCPUVAddr,
+				(size_t)psImport->uiSize);
+
+		psCPUImport->hOSMMapData = NULL;
+		psCPUImport->pvCPUVAddr = NULL;
+
+		OSLockRelease(psCPUImport->hLock);
+
+		DevmemImportStructRelease(psImport);
+	}
+	else
+	{
+		OSLockRelease(psCPUImport->hLock);
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/devicemem_utils.h b/drivers/gpu/drm/img-rogue/devicemem_utils.h
new file mode 100644
index 000000000000..9e24c1dceca7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/devicemem_utils.h
@@ -0,0 +1,582 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management internal utility functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally by device memory management
+                code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DEVICEMEM_UTILS_H
+#define DEVICEMEM_UTILS_H
+
+#include "devicemem.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "osmmap.h"
+#include "pvrsrv_memallocflags_internal.h"
+
+#define DEVMEM_HEAPNAME_MAXLENGTH 160
+
+/*
+ * VA heap size should be at least OS page size. This check is validated in the DDK.
+ */
+#define DEVMEM_HEAP_MINIMUM_SIZE                     0x10000 /* 64KB is MAX anticipated OS page size */
+
+#if defined(DEVMEM_DEBUG) && defined(REFCOUNT_DEBUG)
+#define DEVMEM_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_ERROR, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define DEVMEM_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+/* If we need a "hMapping" but we don't have a server-side mapping, we poison
+ * the entry with this value so that it's easily recognised in the debugger.
+ * Note that this is potentially a valid handle, but then so is NULL, which is
+ * no better, indeed worse, as it's not obvious in the debugger. The value
+ * doesn't matter. We _never_ use it (and because it's valid, we never assert
+ * it isn't this) but it's nice to have a value in the source code that we can
+ * grep for if things go wrong.
+ */
+#define LACK_OF_MAPPING_POISON ((IMG_HANDLE)0x6116dead)
+#define LACK_OF_RESERVATION_POISON ((IMG_HANDLE)0x7117dead)
+
+#define DEVICEMEM_HISTORY_ALLOC_INDEX_NONE 0xFFFFFFFF
+
+struct DEVMEM_CONTEXT_TAG
+{
+
+	SHARED_DEV_CONNECTION hDevConnection;
+
+	/* Number of heaps that have been created in this context
+	 * (regardless of whether they have allocations)
+	 */
+	IMG_UINT32 uiNumHeaps;
+
+	/* Each "DEVMEM_CONTEXT" has a counterpart in the server, which
+	 * is responsible for handling the mapping into device MMU.
+	 * We have a handle to that here.
+	 */
+	IMG_HANDLE hDevMemServerContext;
+
+	/* Number of automagically created heaps in this context,
+	 *  i.e. those that are born at context creation time from the
+	 * chosen "heap config" or "blueprint"
+	 */
+	IMG_UINT32 uiAutoHeapCount;
+
+	/* Pointer to array of such heaps */
+	struct DEVMEM_HEAP_TAG **ppsAutoHeapArray;
+
+	/* The cache line size for use when allocating memory,
+	 * as it is not queryable on the client side
+	 */
+	IMG_UINT32 ui32CPUCacheLineSize;
+
+	/* Private data handle for device specific data */
+	IMG_HANDLE hPrivData;
+};
+
+/* Flags that record how a heaps virtual address space is managed. */
+#define DEVMEM_HEAP_MANAGER_UNKNOWN      0
+/* Heap VAs assigned by the client of Services APIs, heap's RA not used at all. */
+#define DEVMEM_HEAP_MANAGER_USER         (1U << 0)
+/* Heap VAs managed by the OSs kernel, VA from CPU mapping call used */
+#define DEVMEM_HEAP_MANAGER_KERNEL       (1U << 1)
+/* Heap VAs managed by the heap's own RA */
+#define DEVMEM_HEAP_MANAGER_RA           (1U << 2)
+/* Heap VAs managed jointly by Services and the client of Services.
+ * The reserved region of the heap is managed explicitly by the client of Services
+ * The non-reserved region of the heap is managed by the heap's own RA */
+#define DEVMEM_HEAP_MANAGER_DUAL_USER_RA (DEVMEM_HEAP_MANAGER_USER | DEVMEM_HEAP_MANAGER_RA)
+
+struct DEVMEM_HEAP_TAG
+{
+	/* Name of heap - for debug and lookup purposes. */
+	IMG_CHAR *pszName;
+
+	/* Number of live imports in the heap */
+	ATOMIC_T hImportCount;
+
+	/* Base address and size of heap, required by clients due to some
+	 * requesters not being full range
+	 */
+	IMG_DEV_VIRTADDR sBaseAddress;
+	DEVMEM_SIZE_T uiSize;
+
+	DEVMEM_SIZE_T uiReservedRegionSize; /* uiReservedRegionLength in DEVMEM_HEAP_BLUEPRINT */
+
+	/* The heap manager, describing if the space is managed by the user, an RA,
+	 * kernel or combination */
+	IMG_UINT32 ui32HeapManagerFlags;
+
+	/* This RA is for managing sub-allocations within the imports (PMRs)
+	 * within the heap's virtual space. RA only used in DevmemSubAllocate()
+	 * to track sub-allocated buffers.
+	 *
+	 * Resource Span - a PMR import added when the RA calls the
+	 *                 imp_alloc CB (SubAllocImportAlloc) which returns the
+	 *                 PMR import and size (span length).
+	 * Resource - an allocation/buffer i.e. a MemDesc. Resource size represents
+	 *            the size of the sub-allocation.
+	 */
+	RA_ARENA *psSubAllocRA;
+	IMG_CHAR *pszSubAllocRAName;
+
+	/* The psQuantizedVMRA is for the coarse allocation (PMRs) of virtual
+	 * space from the heap.
+	 *
+	 * Resource Span - the heap's VM space from base to base+length,
+	 *                 only one is added at heap creation.
+	 * Resource - a PMR import associated with the heap. Dynamic number
+	 *            as memory is allocated/freed from or mapped/unmapped to
+	 *            the heap. Resource size follows PMR logical size.
+	 */
+	RA_ARENA *psQuantizedVMRA;
+	IMG_CHAR *pszQuantizedVMRAName;
+
+	/* We also need to store a copy of the quantum size in order to feed
+	 * this down to the server.
+	 */
+	IMG_UINT32 uiLog2Quantum;
+
+	/* Store a copy of the minimum import alignment */
+	IMG_UINT32 uiLog2ImportAlignment;
+
+	/* The parent memory context for this heap */
+	struct DEVMEM_CONTEXT_TAG *psCtx;
+
+	/* Lock to protect this structure */
+	POS_LOCK hLock;
+
+	/* Each "DEVMEM_HEAP" has a counterpart in the server, which is
+	 * responsible for handling the mapping into device MMU.
+	 * We have a handle to that here.
+	 */
+	IMG_HANDLE hDevMemServerHeap;
+
+	/* This heap is fully allocated and premapped into the device address space.
+	 * Used in virtualisation for firmware heaps of Guest and optionally Host
+	 * drivers. */
+	IMG_BOOL bPremapped;
+};
+
+typedef IMG_UINT32 DEVMEM_PROPERTIES_T;                  /*!< Typedef for Devicemem properties */
+#define DEVMEM_PROPERTIES_EXPORTABLE         (1UL<<0)    /*!< Is it exportable? */
+#define DEVMEM_PROPERTIES_IMPORTED           (1UL<<1)    /*!< Is it imported from another process? */
+#define DEVMEM_PROPERTIES_SUBALLOCATABLE     (1UL<<2)    /*!< Is it suballocatable? */
+#define DEVMEM_PROPERTIES_IMPORT_IS_ZEROED   (1UL<<4)    /*!< Is the memory fully zeroed? */
+#define DEVMEM_PROPERTIES_IMPORT_IS_CLEAN    (1UL<<5)    /*!< Is the memory clean, i.e. not been used before? */
+#define DEVMEM_PROPERTIES_SECURE             (1UL<<6)    /*!< Is it a special secure buffer? No CPU maps allowed! */
+#define DEVMEM_PROPERTIES_IMPORT_IS_POISONED (1UL<<7)    /*!< Is the memory fully poisoned? */
+#define DEVMEM_PROPERTIES_NO_CPU_MAPPING     (1UL<<8)    /* No CPU Mapping is allowed, RW attributes
+                                                            are further derived from allocation memory flags */
+#define DEVMEM_PROPERTIES_NO_LAYOUT_CHANGE	 (1UL<<9)    /* No sparse resizing allowed, once a memory
+                                                            layout is chosen, no change allowed later */
+
+
+typedef struct DEVMEM_DEVICE_IMPORT_TAG
+{
+	DEVMEM_HEAP *psHeap;            /*!< Heap this import is bound to */
+	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the import */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
+	IMG_HANDLE hReservation;        /*!< Device memory reservation handle */
+	IMG_HANDLE hMapping;            /*!< Device mapping handle */
+	IMG_BOOL bMapped;               /*!< This is import mapped? */
+	POS_LOCK hLock;                 /*!< Lock to protect the device import */
+} DEVMEM_DEVICE_IMPORT;
+
+typedef struct DEVMEM_CPU_IMPORT_TAG
+{
+	void *pvCPUVAddr;               /*!< CPU virtual address of the import */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the CPU virtual address */
+	IMG_HANDLE hOSMMapData;         /*!< CPU mapping handle */
+	POS_LOCK hLock;                 /*!< Lock to protect the CPU import */
+} DEVMEM_CPU_IMPORT;
+
+typedef struct DEVMEM_IMPORT_TAG
+{
+	SHARED_DEV_CONNECTION hDevConnection;
+	IMG_DEVMEM_ALIGN_T uiAlign;         /*!< Alignment of the PMR */
+	DEVMEM_SIZE_T uiSize;               /*!< Size of import */
+	ATOMIC_T hRefCount;                 /*!< Refcount for this import */
+	DEVMEM_PROPERTIES_T uiProperties;   /*!< Stores properties of an import like if
+	                                         it is exportable, pinned or suballocatable */
+	IMG_HANDLE hPMR;                    /*!< Handle to the PMR */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;     /*!< Flags for this import */
+	POS_LOCK hLock;                     /*!< Lock to protect the import */
+
+	DEVMEM_DEVICE_IMPORT sDeviceImport; /*!< Device specifics of the import */
+	DEVMEM_CPU_IMPORT sCPUImport;       /*!< CPU specifics of the import */
+} DEVMEM_IMPORT;
+
+typedef struct DEVMEM_DEVICE_MEMDESC_TAG
+{
+	IMG_DEV_VIRTADDR sDevVAddr;     /*!< Device virtual address of the allocation */
+	IMG_UINT32 ui32RefCount;        /*!< Refcount of the device virtual address */
+	POS_LOCK hLock;                 /*!< Lock to protect device memdesc */
+} DEVMEM_DEVICE_MEMDESC;
+
+typedef struct DEVMEM_CPU_MEMDESC_TAG
+{
+	void *pvCPUVAddr;           /*!< CPU virtual address of the import */
+	IMG_UINT32 ui32RefCount;    /*!< Refcount of the device CPU address */
+	POS_LOCK hLock;             /*!< Lock to protect CPU memdesc */
+} DEVMEM_CPU_MEMDESC;
+
+struct DEVMEM_MEMDESC_TAG
+{
+	DEVMEM_IMPORT *psImport;                /*!< Import this memdesc is on */
+	IMG_DEVMEM_OFFSET_T uiOffset;           /*!< Offset into import where our allocation starts */
+	IMG_DEVMEM_SIZE_T uiAllocSize;          /*!< Size of the allocation */
+	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
+	POS_LOCK hLock;                         /*!< Lock to protect memdesc */
+	IMG_HANDLE hPrivData;
+
+	DEVMEM_DEVICE_MEMDESC sDeviceMemDesc;   /*!< Device specifics of the memdesc */
+	DEVMEM_CPU_MEMDESC sCPUMemDesc;         /*!< CPU specifics of the memdesc */
+
+	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this memdesc */
+
+	IMG_UINT32 ui32AllocationIndex;
+
+#if defined(DEBUG)
+	IMG_BOOL bPoisonOnFree;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
+#endif
+};
+
+/* The physical descriptor used to store handles and information of device
+ * physical allocations.
+ */
+struct DEVMEMX_PHYS_MEMDESC_TAG
+{
+	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
+	IMG_UINT32 uiLog2PageSize;              /*!< Page size */
+	ATOMIC_T hRefCount;                     /*!< Refcount of the memdesc */
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;         /*!< Flags for this import */
+	IMG_HANDLE hPMR;                        /*!< Handle to the PMR */
+	DEVMEM_CPU_IMPORT sCPUImport;           /*!< CPU specifics of the memdesc */
+	SHARED_DEV_CONNECTION hConnection;      /*!< Services connection for the server */
+	void *pvUserData;						/*!< User data */
+};
+
+/* The virtual descriptor used to store handles and information of a device
+ * virtual range and the mappings to it.
+ */
+struct DEVMEMX_VIRT_MEMDESC_TAG
+{
+	IMG_UINT32 uiNumPages;                  /*!< Number of pages that the import has*/
+	PVRSRV_MEMALLOCFLAGS_T uiFlags;         /*!< Flags for this import */
+	DEVMEMX_PHYSDESC **apsPhysDescTable;    /*!< Table to store links to physical descs */
+	DEVMEM_DEVICE_IMPORT sDeviceImport;     /*!< Device specifics of the memdesc */
+
+	IMG_CHAR szText[DEVMEM_ANNOTATION_MAX_LEN]; /*!< Annotation for this virt memdesc */
+	IMG_UINT32 ui32AllocationIndex;         /*!< To track mappings in this range */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	IMG_HANDLE hRIHandle;                   /*!< Handle to RI information */
+#endif
+};
+
+#define DEVICEMEM_UTILS_NO_ADDRESS 0
+
+/******************************************************************************
+@Function       DevmemValidateParams
+@Description    Check if flags are conflicting and if align is a size multiple.
+
+@Input          uiSize      Size of the import.
+@Input          uiAlign     Alignment of the import.
+@Input          puiFlags    Pointer to the flags for the import.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemValidateParams(IMG_DEVMEM_SIZE_T uiSize,
+                                   IMG_DEVMEM_ALIGN_T uiAlign,
+                                   PVRSRV_MEMALLOCFLAGS_T *puiFlags);
+
+/******************************************************************************
+@Function       DevmemImportStructAlloc
+@Description    Allocates memory for an import struct. Does not allocate a PMR!
+                Create locks for CPU and Devmem mappings.
+
+@Input          hDevConnection  Connection to use for calls from the import.
+@Input          ppsImport       The import to allocate.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructAlloc(SHARED_DEV_CONNECTION hDevConnection,
+                                      DEVMEM_IMPORT **ppsImport);
+
+/******************************************************************************
+@Function       DevmemImportStructInit
+@Description    Initialises the import struct with the given parameters.
+                Set it's refcount to 1!
+
+@Input          psImport     The import to initialise.
+@Input          uiSize       Size of the import.
+@Input          uiAlign      Alignment of allocations in the import.
+@Input          uiMapFlags
+@Input          hPMR         Reference to the PMR of this import struct.
+@Input          uiProperties Properties of the import. Is it exportable,
+                              imported, suballocatable?
+******************************************************************************/
+void DevmemImportStructInit(DEVMEM_IMPORT *psImport,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                             IMG_DEVMEM_ALIGN_T uiAlign,
+                             PVRSRV_MEMALLOCFLAGS_T uiMapFlags,
+                             IMG_HANDLE hPMR,
+                             DEVMEM_PROPERTIES_T uiProperties);
+
+/******************************************************************************
+@Function       DevmemImportStructDevMap
+@Description    NEVER call after the last DevmemMemDescRelease()
+                Maps the PMR referenced by the import struct to the device's
+                virtual address space.
+                Does nothing but increase the cpu mapping refcount if the
+                import struct was already mapped.
+
+@Input          psHeap    The heap to map to.
+@Input          bMap      Caller can choose if the import should be really
+                          mapped in the page tables or if just a virtual range
+                          should be reserved and the refcounts increased.
+@Input          psImport  The import we want to map.
+@Input          uiOptionalMapAddress  An optional address to map to.
+                                      Pass DEVICEMEM_UTILS_NOADDRESS if not used.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructDevMap(DEVMEM_HEAP *psHeap,
+                                       IMG_BOOL bMap,
+                                       DEVMEM_IMPORT *psImport,
+                                       IMG_UINT64 uiOptionalMapAddress);
+
+/******************************************************************************
+@Function       DevmemImportStructDevUnmap
+@Description    Unmaps the PMR referenced by the import struct from the
+                device's virtual address space.
+                If this was not the last remaining CPU mapping on the import
+                struct only the cpu mapping refcount is decreased.
+@return         A boolean to signify if the import was unmapped.
+******************************************************************************/
+IMG_BOOL DevmemImportStructDevUnmap(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructCPUMap
+@Description    NEVER call after the last DevmemMemDescRelease()
+                Maps the PMR referenced by the import struct to the CPU's
+                virtual address space.
+                Does nothing but increase the cpu mapping refcount if the
+                import struct was already mapped.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemImportStructCPUMap(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructCPUUnmap
+@Description    Unmaps the PMR referenced by the import struct from the CPU's
+                virtual address space.
+                If this was not the last remaining CPU mapping on the import
+                struct only the cpu mapping refcount is decreased.
+******************************************************************************/
+void DevmemImportStructCPUUnmap(DEVMEM_IMPORT *psImport);
+
+
+/******************************************************************************
+@Function       DevmemImportStructAcquire
+@Description    Acquire an import struct by increasing it's refcount.
+******************************************************************************/
+void DevmemImportStructAcquire(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportStructRelease
+@Description    Reduces the refcount of the import struct.
+                Destroys the import in the case it was the last reference.
+                Destroys underlying PMR if this import was the last reference
+                to it.
+@return         A boolean to signal if the import was destroyed. True = yes.
+******************************************************************************/
+IMG_BOOL DevmemImportStructRelease(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemImportDiscard
+@Description    Discard a created, but uninitialised import structure.
+                This must only be called before DevmemImportStructInit
+                after which DevmemImportStructRelease must be used to
+                "free" the import structure.
+******************************************************************************/
+void DevmemImportDiscard(DEVMEM_IMPORT *psImport);
+
+/******************************************************************************
+@Function       DevmemMemDescAlloc
+@Description    Allocates a MemDesc and create it's various locks.
+                Zero the allocated memory.
+@return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR DevmemMemDescAlloc(DEVMEM_MEMDESC **ppsMemDesc);
+
+#if defined(DEBUG)
+/******************************************************************************
+@Function       DevmemMemDescSetPoF
+@Description    Sets the Poison on Free flag to true for this MemDesc if the
+                given MemAllocFlags have the Poison on Free bit set.
+                Poison on Free is a debug only feature.
+******************************************************************************/
+void DevmemMemDescSetPoF(DEVMEM_MEMDESC *psMemDesc, PVRSRV_MEMALLOCFLAGS_T uiFlags);
+#endif
+
+/******************************************************************************
+@Function       DevmemMemDescInit
+@Description    Sets the given offset and import struct fields in the MemDesc.
+                Initialises refcount to 1 and other values to 0.
+
+@Input          psMemDesc    MemDesc to initialise.
+@Input          uiOffset     Offset in the import structure.
+@Input          psImport     Import the MemDesc is on.
+@Input          uiAllocSize  Size of the allocation
+******************************************************************************/
+void DevmemMemDescInit(DEVMEM_MEMDESC *psMemDesc,
+						IMG_DEVMEM_OFFSET_T uiOffset,
+						DEVMEM_IMPORT *psImport,
+						IMG_DEVMEM_SIZE_T uiAllocSize);
+
+/******************************************************************************
+@Function       DevmemMemDescAcquire
+@Description    Acquires the MemDesc by increasing it's refcount.
+******************************************************************************/
+void DevmemMemDescAcquire(DEVMEM_MEMDESC *psMemDesc);
+
+/******************************************************************************
+@Function       DevmemMemDescRelease
+@Description    Releases the MemDesc by reducing it's refcount.
+                Destroy the MemDesc if it's recount is 0.
+                Destroy the import struct the MemDesc is on if that was the
+                last MemDesc on the import, probably following the destruction
+                of the underlying PMR.
+@return         A boolean to signal if the MemDesc was destroyed. True = yes.
+******************************************************************************/
+IMG_BOOL DevmemMemDescRelease(DEVMEM_MEMDESC *psMemDesc);
+
+/******************************************************************************
+@Function       DevmemMemDescDiscard
+@Description    Discard a created, but uninitialised MemDesc structure.
+                This must only be called before DevmemMemDescInit after
+                which DevmemMemDescRelease must be used to "free" the
+                MemDesc structure.
+******************************************************************************/
+void DevmemMemDescDiscard(DEVMEM_MEMDESC *psMemDesc);
+
+
+/******************************************************************************
+@Function       GetImportProperties
+@Description    Atomically read psImport->uiProperties
+                It's possible that another thread modifies uiProperties
+                immediately after this function returns, making its result
+                stale. So, it's recommended to use this function only to
+                check if certain non-volatile flags were set.
+******************************************************************************/
+static INLINE DEVMEM_PROPERTIES_T GetImportProperties(DEVMEM_IMPORT *psImport)
+{
+	DEVMEM_PROPERTIES_T uiProperties;
+
+	OSLockAcquire(psImport->hLock);
+	uiProperties = psImport->uiProperties;
+	OSLockRelease(psImport->hLock);
+	return uiProperties;
+}
+
+/******************************************************************************
+@Function       DevmemCPUMemSet
+@Description    Given a CPU Mapped Devmem address, set the memory at that
+                range (address, address + size) to the uiPattern provided.
+                Flags determine the OS abstracted MemSet method to use.
+******************************************************************************/
+static INLINE void DevmemCPUMemSet(void *pvMem,
+                                   IMG_UINT8 uiPattern,
+                                   IMG_DEVMEM_SIZE_T uiSize,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags))
+	{
+		OSDeviceMemSet(pvMem, uiPattern, uiSize);
+	}
+	else
+	{
+		/* it's safe to use OSCachedMemSet() for cached and wc memory */
+		OSCachedMemSet(pvMem, uiPattern, uiSize);
+	}
+}
+
+/******************************************************************************
+@Function       DevmemCPUMapCheckImportProperties
+@Description    Given a MemDesc check that the import properties are correct
+                to allow for mapping the MemDesc to the CPU.
+                Returns PVRSRV_OK on success.
+******************************************************************************/
+static INLINE PVRSRV_ERROR DevmemCPUMapCheckImportProperties(DEVMEM_MEMDESC *psMemDesc)
+{
+	DEVMEM_PROPERTIES_T uiProperties = GetImportProperties(psMemDesc->psImport);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiProperties & DEVMEM_PROPERTIES_SECURE)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+					"%s: Allocation is a secure buffer. "
+					"It should not be possible to map to CPU, but for security "
+					"validation this will be allowed for testing purposes, "
+					"as long as the buffer is pinned.",
+					__func__));
+	}
+#endif
+
+	if (uiProperties & DEVMEM_PROPERTIES_NO_CPU_MAPPING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: CPU Mapping is not possible on this allocation!",
+				__func__));
+		return PVRSRV_ERROR_INVALID_MAP_REQUEST;
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* DEVICEMEM_UTILS_H */
diff --git a/drivers/gpu/drm/img-rogue/di_common.h b/drivers/gpu/drm/img-rogue/di_common.h
new file mode 100644
index 000000000000..a10178708a75
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/di_common.h
@@ -0,0 +1,236 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common types for Debug Info framework.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DI_COMMON_H
+#define DI_COMMON_H
+
+#include "img_types.h"
+
+/* Token that signals that a header should be printed. */
+#define DI_START_TOKEN ((void *) 1)
+
+/* This is a public handle to an entry. */
+#ifndef DI_GROUP_DEFINED
+#define DI_GROUP_DEFINED
+typedef struct DI_GROUP DI_GROUP;
+#endif
+#ifndef DI_ENTRY_DEFINED
+#define DI_ENTRY_DEFINED
+typedef struct DI_ENTRY DI_ENTRY;
+#endif
+typedef struct OSDI_IMPL_ENTRY OSDI_IMPL_ENTRY;
+
+/*! Debug Info entries types. */
+typedef enum DI_ENTRY_TYPE
+{
+    DI_ENTRY_TYPE_GENERIC,          /*!< generic entry type, implements
+                                         start/stop/next/show iterator
+                                         interface */
+    DI_ENTRY_TYPE_RANDOM_ACCESS,    /*!< random access entry, implements
+                                         seek/read iterator interface */
+} DI_ENTRY_TYPE;
+
+/*! @Function DI_PFN_START
+ *
+ * @Description
+ * Start operation returns first entry and passes it to Show operation.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @InOut pui64Pos current data position in the entry
+ *
+ * @Return pointer to data that will be passed to the other iterator
+ *         functions in pvData argument
+ */
+typedef void *(*DI_PFN_START)(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos);
+
+/*! @Function DI_PFN_STOP
+ *
+ * @Description
+ * Stop operations is called after iterator reaches end of data.
+ *
+ * If pvData was allocated in pfnStart it should be freed here.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ */
+typedef void (*DI_PFN_STOP)(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*! @Function DI_PFN_NEXT
+ *
+ * @Description
+ * Next returns next data entry and passes it to Show operation.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ * @InOut pui64Pos current data position in the entry
+ */
+typedef void *(*DI_PFN_NEXT)(OSDI_IMPL_ENTRY *psEntry, void *pvData,
+                             IMG_UINT64 *pui64Pos);
+
+/*! @Function DI_PFN_SHOW
+ *
+ * @Description
+ * Outputs the data element.
+ *
+ * @Input psEntry pointer to the implementation entry
+ * @Input pvData pointer to data returned from pfnStart/pfnNext
+ */
+typedef int (*DI_PFN_SHOW)(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*! @Function DI_PFN_SEEK
+ *
+ * @Description
+ * Changes position of the entry data pointer
+ *
+ * @Input uiOffset new entry offset (absolute)
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_SEEK)(IMG_UINT64 ui64Offset, void *pvData);
+
+/*! @Function DI_PFN_READ
+ *
+ * @Description
+ * Retrieves data from the entry from position previously set by Seek.
+ *
+ * @Input pszBuffer output buffer
+ * @Input ui64Count length of the output buffer
+ * @InOut pui64Pos pointer to the current position in the entry
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_READ)(IMG_CHAR *pszBuffer, IMG_UINT64 ui64Count,
+                                 IMG_UINT64 *pui64Pos, void *pvData);
+
+/*! @Function DI_PFN_WRITE
+ *
+ * @Description
+ * Handle writes operation to the entry.
+ *
+ * @Input pszBuffer NUL-terminated buffer containing written data
+ * @Input ui64Count length of the data in pszBuffer (length of the buffer)
+ * @InOut pui64Pos pointer to the current position in the entry
+ * @Input pvData private data provided during entry creation
+ */
+typedef IMG_INT64 (*DI_PFN_WRITE)(const IMG_CHAR *pszBuffer,
+                                  IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                                  void *pvData);
+
+/*! Debug info entry iterator.
+ *
+ * This covers all entry types: GENERIC and RANDOM_ACCESS.
+ *
+ * The GENERIC entry type
+ *
+ * The GENERIC type should implement either a full set of following callbacks:
+ * pfnStart, pfnStop, pfnNext and pfnShow, or pfnShow only. If only pfnShow
+ * callback is given the framework will use default handlers in place of the
+ * other ones.
+ *
+ * e.g. for generic entry:
+ *
+ *   struct sIter = {
+ *     .pfnStart = StartCb, .pfnStop = StopCb, pfnNext = NextCb,
+ *     .pfnShow = ShowCb
+ *   };
+ *
+ * The use case for implementing pfnShow only is if the data for the given
+ * entry is short and can be printed in one go because the pfnShow callback
+ * will be called only once.
+ *
+ * e.g. for one-shot print generic entry:
+ *
+ *   struct sIter = {
+ *     .pfnShow = SingleShowCb
+ *   };
+ *
+ * The DICreateEntry() function will return error if DI_ENTRY_TYPE_GENERIC
+ * type is used and invalid combination of callbacks is given.
+ *
+ * The RANDOM_ACCESS entry
+ *
+ * The RANDOM_ACCESS type should implement either both pfnSeek and pfnRead
+ * or pfnRead only callbacks.
+ *
+ * e.g. of seekable and readable random access entry:
+ *
+ *   struct sIter = {
+ *     .pfnSeek = SeekCb, .pfnRead = ReadCb
+ *   };
+ *
+ * The DICreateEntry() function will return error if DI_ENTRY_TYPE_RANDOM_ACCESS
+ * type is used and invalid combination of callbacks is given.
+ *
+ * Writing to file (optional)
+ *
+ * The iterator allows also to pass a pfnWrite callback that allows implementing
+ * write operation on the entry. The write operation is entry type agnostic
+ * which means that it can be defined for both GENERIC and RANDOM_ACCESS
+ * entries.
+ *
+ * e.g. for writable one-shot print generic entry
+ *
+ *   struct sIter = {
+ *     .pfnShow = SingleShowCb, .pfnWrite = WriteCb
+ *   };
+ */
+typedef struct DI_ITERATOR_CB
+{
+    /* Generic entry interface. */
+
+    DI_PFN_START pfnStart; /*!< Starts iteration and returns first element
+                                of entry's data. */
+    DI_PFN_STOP pfnStop;   /*!< Stops iteration. */
+    DI_PFN_NEXT pfnNext;   /*!< Returns next element of entry's data. */
+    DI_PFN_SHOW pfnShow;   /*!< Shows current data element of an entry. */
+
+    /* Optional random access entry interface. */
+
+    DI_PFN_SEEK pfnSeek;   /*!< Sets data pointer in an entry. */
+    DI_PFN_READ pfnRead;   /*!< Reads data from an entry. */
+
+    /* Optional writing to entry interface. Null terminated. */
+
+    DI_PFN_WRITE pfnWrite; /*!< Performs write operation on an entry. */
+    IMG_UINT32   ui32WriteLenMax; /*!< Maximum char length of entry
+                                       accepted for write. Includes \0 */
+} DI_ITERATOR_CB;
+
+#endif /* DI_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/di_impl_brg.c b/drivers/gpu/drm/img-rogue/di_impl_brg.c
new file mode 100644
index 000000000000..811e13b018e0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/di_impl_brg.c
@@ -0,0 +1,908 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via pvrdebug.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "allocmem.h"
+#include "hash.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "lock.h"
+#include "osfunc_common.h"
+#include "osfunc.h" /* for thread */
+#include "tlstream.h"
+#include "dllist.h"
+
+#include "osdi_impl.h"
+#include "di_impl_brg.h"
+#include "di_impl_brg_intern.h"
+#include "pvr_dicommon.h"
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+#include "pvrsrv.h"
+#endif
+
+#define ENTRIES_TABLE_INIT_SIZE 64
+#define STREAM_BUFFER_SIZE 0x4000 /* 16KB */
+#define STREAM_LINE_LENGTH 512
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define WRITER_THREAD_SLEEP_TIMEOUT 0ULL
+#else
+#define WRITER_THREAD_SLEEP_TIMEOUT 28800000000ULL
+#endif
+#define WRITER_THREAD_DESTROY_TIMEOUT 100000ULL
+#define WRITER_THREAD_DESTROY_RETRIES 10U
+
+#define WRITE_RETRY_COUNT 10      /* retry a write to a TL buffer 10 times */
+#define WRITE_RETRY_WAIT_TIME 100 /* wait 10ms between write retries */
+
+typedef enum THREAD_STATE
+{
+	THREAD_STATE_NULL,
+	THREAD_STATE_ALIVE,
+	THREAD_STATE_TERMINATED,
+} THREAD_STATE;
+
+static struct DIIB_IMPL
+{
+	HASH_TABLE *psEntriesTable;    /*!< Table of entries. */
+	POSWR_LOCK psEntriesLock;      /*!< Protects psEntriesTable. */
+	IMG_HANDLE hWriterThread;
+	IMG_HANDLE hWriterEventObject;
+	ATOMIC_T eThreadState;
+
+	DLLIST_NODE sWriterQueue;
+	POS_LOCK psWriterLock;         /*!< Protects sWriterQueue. */
+} *_g_psImpl;
+
+struct DIIB_GROUP
+{
+	const IMG_CHAR *pszName;
+	struct DIIB_GROUP *psParentGroup;
+};
+
+struct DIIB_ENTRY
+{
+	struct DIIB_GROUP *psParentGroup;
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+	DI_ENTRY_TYPE eType;
+	IMG_CHAR pszFullPath[DI_IMPL_BRG_PATH_LEN];
+	void *pvPrivData;
+
+	POS_LOCK hLock; /*!< Protects access to entry's iterator. */
+};
+
+struct DI_CONTEXT_TAG
+{
+	IMG_HANDLE hStream;
+	ATOMIC_T iRefCnt;
+	IMG_BOOL bClientConnected; /*!< Indicated that the client is or is not
+	                                connected to the DI. */
+};
+
+struct DIIB_WORK_ITEM
+{
+	DI_CONTEXT *psContext;
+	DIIB_ENTRY *psEntry;
+	IMG_UINT64 ui64Size;
+	IMG_UINT64 ui64Offset;
+
+	DLLIST_NODE sQueueElement;
+};
+
+/* Declaring function here to avoid dependencies that are introduced by
+ * including osfunc.h. */
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                           size_t uiSize);
+
+/* djb2 hash function is public domain */
+static IMG_UINT32 _Hash(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_CHAR *pszStr = pKey;
+	IMG_UINT32 ui32Hash = 5381, ui32Char;
+
+	PVR_UNREFERENCED_PARAMETER(uKeySize);
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	while ((ui32Char = *pszStr++) != '\0')
+	{
+		ui32Hash = ((ui32Hash << 5) + ui32Hash) + ui32Char; /* hash * 33 + c */
+	}
+
+	return ui32Hash;
+}
+
+static IMG_BOOL _Compare(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_CHAR *pszKey1 = pKey1, *pszKey2 = pKey2;
+
+	return OSStringNCompare(pszKey1, pszKey2, uKeySize) == 0;
+}
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteWithRetires(void *pvNativeHandle, const IMG_CHAR *pszStr,
+                              IMG_UINT uiLen)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iRetry = 0;
+	IMG_UINT32 ui32Flags = TL_FLAG_NO_WRITE_FAILED;
+
+	do
+	{
+		/* Try to write to the buffer but don't inject MOST_RECENT_WRITE_FAILED
+		 * packet in case of failure because we're going to retry. */
+		eError = TLStreamWriteRetFlags(pvNativeHandle, (IMG_UINT8 *) pszStr,
+		                               uiLen, &ui32Flags);
+		if (eError == PVRSRV_ERROR_STREAM_FULL)
+		{
+			// wait to give the client a change to read
+			OSSleepms(WRITE_RETRY_WAIT_TIME);
+		}
+	}
+	while (eError == PVRSRV_ERROR_STREAM_FULL && iRetry++ < WRITE_RETRY_COUNT);
+
+	/* One last try to write to the buffer. In this case upon failure
+	 * a MOST_RECENT_WRITE_FAILED packet will be inject to the buffer to
+	 * indicate data loss. */
+	if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		eError = TLStreamWrite(pvNativeHandle, (IMG_UINT8 *) pszStr, uiLen);
+	}
+
+	PVR_LOG_IF_ERROR(eError, "TLStreamWrite");
+}
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	_WriteWithRetires(pvNativeHandle, pvData, uiSize);
+}
+
+__printf(2, 0)
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	IMG_CHAR pcBuffer[STREAM_LINE_LENGTH];
+	IMG_UINT uiLen = OSVSNPrintf(pcBuffer, sizeof(pcBuffer) - 1, pszFmt, pArgs);
+	pcBuffer[uiLen] = '\0';
+
+	_WriteWithRetires(pvNativeHandle, pcBuffer, uiLen + 1);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	_WriteWithRetires(pvNativeHandle, pszStr, OSStringLength(pszStr) + 1);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvNativeHandle);
+	return IMG_FALSE;
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- entry operations --------------------------------------------------- */
+
+static PVRSRV_ERROR _ContextUnrefAndMaybeDestroy(DI_CONTEXT *psContext)
+{
+	if (OSAtomicDecrement(&psContext->iRefCnt) == 0)
+	{
+		TLStreamClose(psContext->hStream);
+		OSFreeMem(psContext);
+	}
+
+	return PVRSRV_OK;
+}
+
+static IMG_INT64 _ReadGeneric(const DI_CONTEXT *psContext, DIIB_ENTRY *psEntry)
+{
+	IMG_INT64 iRet = 0;
+	IMG_UINT64 ui64Pos = 0;
+	DI_ITERATOR_CB *psIter = &psEntry->sIterCb;
+	OSDI_IMPL_ENTRY *psImplEntry = &psEntry->sImplEntry;
+	PVRSRV_ERROR eError;
+
+	if (psIter->pfnStart != NULL)
+	{
+		/* this is a full sequence of the operation */
+		void *pvData = psIter->pfnStart(psImplEntry, &ui64Pos);
+
+		while (pvData != NULL && psContext->bClientConnected)
+		{
+			iRet = psIter->pfnShow(psImplEntry, pvData);
+			if (iRet < 0)
+			{
+				break;
+			}
+
+			pvData = psIter->pfnNext(psImplEntry, pvData, &ui64Pos);
+		}
+
+		psIter->pfnStop(psImplEntry, pvData);
+	}
+	else if (psIter->pfnShow != NULL)
+	{
+		/* this is a simplified sequence of the operation */
+		iRet = psIter->pfnShow(psImplEntry, NULL);
+	}
+
+	eError = TLStreamMarkEOS(psImplEntry->pvNative, IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamMarkEOS", return_error_);
+
+	return iRet;
+
+return_error_:
+	return -1;
+}
+
+static IMG_INT64 _ReadRndAccess(DIIB_ENTRY *psEntry, IMG_UINT64 ui64Count,
+                                IMG_UINT64 *pui64Pos, void *pvData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT8 *pui8Buffer;
+	IMG_HANDLE hStream = psEntry->sImplEntry.pvNative;
+
+	if (psEntry->sIterCb.pfnRead == NULL)
+	{
+		return -1;
+	}
+
+	eError = TLStreamReserve(hStream, &pui8Buffer, ui64Count);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamReserve", return_error_);
+
+	psEntry->sIterCb.pfnRead((IMG_CHAR *) pui8Buffer, ui64Count, pui64Pos,
+	                         pvData);
+
+	eError = TLStreamCommit(hStream, ui64Count);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCommit", return_error_);
+
+	eError = TLStreamMarkEOS(psEntry->sImplEntry.pvNative, IMG_FALSE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamMarkEOS", return_error_);
+
+	return 0;
+
+return_error_:
+	return -1;
+}
+
+static void _WriterThread(void *pvArg)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	DLLIST_NODE *psNode;
+
+	PVR_UNREFERENCED_PARAMETER(pvArg);
+
+	eError = OSEventObjectOpen(_g_psImpl->hWriterEventObject, &hEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#ifdef PVRSRV_FORCE_UNLOAD_IF_BAD_STATE
+	while (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK &&
+	       OSAtomicRead(&_g_psImpl->eThreadState) == THREAD_STATE_ALIVE)
+#else
+	while (OSAtomicRead(&_g_psImpl->eThreadState) == THREAD_STATE_ALIVE)
+#endif
+	{
+		struct DIIB_WORK_ITEM *psItem = NULL;
+
+		OSLockAcquire(_g_psImpl->psWriterLock);
+		/* Get element from list tail so that we always get the oldest element
+		 * (elements are added to head). */
+		while ((psNode = dllist_get_prev_node(&_g_psImpl->sWriterQueue)) != NULL)
+		{
+			IMG_INT64 i64Ret;
+			DIIB_ENTRY *psEntry;
+			OSDI_IMPL_ENTRY *psImplEntry;
+
+			dllist_remove_node(psNode);
+			OSLockRelease(_g_psImpl->psWriterLock);
+
+			psItem = IMG_CONTAINER_OF(psNode, struct DIIB_WORK_ITEM,
+			                          sQueueElement);
+
+			psEntry = psItem->psEntry;
+			psImplEntry = &psItem->psEntry->sImplEntry;
+
+			/* if client has already disconnected we can just drop this item */
+			if (psItem->psContext->bClientConnected)
+			{
+
+				PVR_ASSERT(psItem->psContext->hStream != NULL);
+
+				psImplEntry->pvNative = psItem->psContext->hStream;
+
+				if (psEntry->eType == DI_ENTRY_TYPE_GENERIC)
+				{
+					i64Ret = _ReadGeneric(psItem->psContext, psEntry);
+					PVR_LOG_IF_FALSE(i64Ret >= 0, "generic access read operation "
+					                 "failed");
+				}
+				else if (psEntry->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+				{
+					IMG_UINT64 ui64Pos = psItem->ui64Offset;
+
+					i64Ret = _ReadRndAccess(psEntry, psItem->ui64Size, &ui64Pos,
+					                        psEntry->pvPrivData);
+					PVR_LOG_IF_FALSE(i64Ret >= 0, "random access read operation "
+					                 "failed");
+				}
+				else
+				{
+					PVR_ASSERT(psEntry->eType == DI_ENTRY_TYPE_GENERIC ||
+					           psEntry->eType == DI_ENTRY_TYPE_RANDOM_ACCESS);
+				}
+
+				psImplEntry->pvNative = NULL;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "client reading entry \"%s\" has "
+				        "disconnected", psEntry->pszFullPath));
+			}
+
+			_ContextUnrefAndMaybeDestroy(psItem->psContext);
+			OSFreeMemNoStats(psItem);
+
+			OSLockAcquire(_g_psImpl->psWriterLock);
+		}
+		OSLockRelease(_g_psImpl->psWriterLock);
+
+		eError = OSEventObjectWaitKernel(hEvent, WRITER_THREAD_SLEEP_TIMEOUT);
+		if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_LOG_ERROR(eError, "OSEventObjectWaitKernel");
+		}
+	}
+
+	OSLockAcquire(_g_psImpl->psWriterLock);
+	/* clear the queue if there are any items pending */
+	while ((psNode = dllist_get_prev_node(&_g_psImpl->sWriterQueue)) != NULL)
+	{
+		struct DIIB_WORK_ITEM *psItem = IMG_CONTAINER_OF(psNode,
+		                                                 struct DIIB_WORK_ITEM,
+		                                                 sQueueElement);
+
+		dllist_remove_node(psNode);
+		_ContextUnrefAndMaybeDestroy(psItem->psContext);
+		OSFreeMem(psItem);
+	}
+	OSLockRelease(_g_psImpl->psWriterLock);
+
+	eError = OSEventObjectClose(hEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_TERMINATED);
+}
+
+/* ----- DI internal API ---------------------------------------------------- */
+
+DIIB_ENTRY *DIImplBrgFind(const IMG_CHAR *pszPath)
+{
+	DIIB_ENTRY *psEntry;
+
+	OSWRLockAcquireRead(_g_psImpl->psEntriesLock);
+	psEntry = (void *) HASH_Retrieve_Extended(_g_psImpl->psEntriesTable,
+	                                          (IMG_CHAR *) pszPath);
+	OSWRLockReleaseRead(_g_psImpl->psEntriesLock);
+
+	return psEntry;
+}
+
+/* ----- DI bridge interface ------------------------------------------------ */
+
+static PVRSRV_ERROR _CreateStream(IMG_CHAR *pszStreamName, IMG_HANDLE *phStream)
+{
+	IMG_UINT32 iRet;
+	IMG_HANDLE hStream;
+	PVRSRV_ERROR eError;
+
+	/* for now only one stream can be created. Should we be able to create
+	 * per context stream? */
+	iRet = OSSNPrintf(pszStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+	                  "di_stream_%x", OSGetCurrentClientProcessIDKM());
+	if (iRet >= PRVSRVTL_MAX_STREAM_NAME_SIZE)
+	{
+		/* this check is superfluous because it can never happen but in case
+		 * someone changes the definition of PRVSRVTL_MAX_STREAM_NAME_SIZE
+		 * handle this case */
+		pszStreamName[0] = '\0';
+		return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+
+	eError = TLStreamCreate(&hStream, pszStreamName, STREAM_BUFFER_SIZE,
+	                        TL_OPMODE_DROP_NEWER, NULL, NULL, NULL, NULL,
+	                        NULL, NULL);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*phStream = hStream;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR DICreateContextKM(IMG_CHAR *pszStreamName, DI_CONTEXT **ppsContext)
+{
+	PVRSRV_ERROR eError;
+	DI_CONTEXT *psContext;
+	IMG_HANDLE hStream = NULL;
+	THREAD_STATE eTState;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsContext != NULL, "ppsContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszStreamName != NULL, "pszStreamName");
+
+	psContext = OSAllocMem(sizeof(*psContext));
+	PVR_LOG_GOTO_IF_NOMEM(psContext, eError, return_);
+
+	eError = _CreateStream(pszStreamName, &hStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CreateStream", free_desc_);
+
+	psContext->hStream = hStream;
+	/* indicated to the write thread if the client is still connected and
+	 * waiting for the data */
+	psContext->bClientConnected = IMG_TRUE;
+	OSAtomicWrite(&psContext->iRefCnt, 1);
+
+	eTState = OSAtomicCompareExchange(&_g_psImpl->eThreadState,
+	                                  THREAD_STATE_NULL,
+	                                  THREAD_STATE_ALIVE);
+
+	/* if the thread has not been started yet do it */
+	if (eTState == THREAD_STATE_NULL)
+	{
+		PVR_ASSERT(_g_psImpl->hWriterThread == NULL);
+
+		eError = OSThreadCreate(&_g_psImpl->hWriterThread, "di_writer",
+		                        _WriterThread, NULL, IMG_FALSE, NULL);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreate", free_close_stream_);
+	}
+
+	*ppsContext = psContext;
+
+	return PVRSRV_OK;
+
+free_close_stream_:
+	TLStreamClose(psContext->hStream);
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_TERMINATED);
+free_desc_:
+	OSFreeMem(psContext);
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIDestroyContextKM(DI_CONTEXT *psContext)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+
+	/* pass the information to the write thread that the client has
+	 * disconnected */
+	psContext->bClientConnected = IMG_FALSE;
+
+	return _ContextUnrefAndMaybeDestroy(psContext);
+}
+
+PVRSRV_ERROR DIReadEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT64 ui64Offset, IMG_UINT64 ui64Size)
+{
+	PVRSRV_ERROR eError;
+	struct DIIB_WORK_ITEM *psItem;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszEntryPath != NULL, "pszEntryPath");
+
+	/* 'no stats' to avoid acquiring the process stats locks */
+	psItem = OSAllocMemNoStats(sizeof(*psItem));
+	PVR_LOG_GOTO_IF_NOMEM(psItem, eError, return_);
+
+	psItem->psContext = psContext;
+	psItem->psEntry = DIImplBrgFind(pszEntryPath);
+	PVR_LOG_GOTO_IF_FALSE_VA(psItem->psEntry != NULL, free_item_,
+	                         "entry %s does not exist", pszEntryPath);
+	psItem->ui64Size = ui64Size;
+	psItem->ui64Offset = ui64Offset;
+
+	/* increment ref count on the context so that it doesn't get freed
+	 * before it gets processed by the writer thread. */
+	if (OSAtomicAddUnless(&psContext->iRefCnt, 1, IMG_INT32_MAX) == IMG_INT32_MAX)
+	{
+		/* Job is not scheduled to the writer queue; there are too many waiting. */
+		PVR_LOG_GOTO_WITH_ERROR("OSAtomicAddUnless", eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, overflow_);
+	}
+
+	OSLockAcquire(_g_psImpl->psWriterLock);
+	dllist_add_to_head(&_g_psImpl->sWriterQueue, &psItem->sQueueElement);
+	OSLockRelease(_g_psImpl->psWriterLock);
+
+	eError = OSEventObjectSignal(_g_psImpl->hWriterEventObject);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+
+	return PVRSRV_OK;
+
+free_item_:
+	eError = PVRSRV_ERROR_NOT_FOUND;
+overflow_:
+	OSFreeMemNoStats(psItem);
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIWriteEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT32 ui32ValueSize, const IMG_CHAR *pszValue)
+{
+	DIIB_ENTRY *psEntry;
+	DI_PFN_WRITE pfnEntryPuts;
+	IMG_INT64 i64Length = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszEntryPath != NULL, "pszEntryPath");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszValue != NULL, "pszValue");
+
+	psEntry = DIImplBrgFind(pszEntryPath);
+	PVR_LOG_RETURN_IF_FALSE_VA(psEntry != NULL, PVRSRV_ERROR_NOT_FOUND,
+	                         "entry %s does not exist", pszEntryPath);
+
+	pfnEntryPuts = psEntry->sIterCb.pfnWrite;
+	if (pfnEntryPuts != NULL)
+	{
+		i64Length = pfnEntryPuts(pszValue, ui32ValueSize, (IMG_UINT64*)&i64Length, psEntry->pvPrivData);
+
+		/* To deal with -EINVAL being returned */
+		PVR_LOG_RETURN_IF_INVALID_PARAM(i64Length >= 0, pszValue);
+	}
+	else
+	{
+		PVR_LOG_MSG(PVR_DBG_WARNING, "Unable to write to Entry. Write callback not enabled");
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _listName(uintptr_t k,
+                               uintptr_t v,
+                               void* hStream)
+{
+	PVRSRV_ERROR eError;
+	DIIB_ENTRY *psEntry;
+	IMG_UINT32 ui32Size;
+	IMG_CHAR aszName[DI_IMPL_BRG_PATH_LEN];
+
+	psEntry = (DIIB_ENTRY*) v;
+	PVR_ASSERT(psEntry != NULL);
+	PVR_UNREFERENCED_PARAMETER(k);
+
+	ui32Size = OSSNPrintf(aszName, DI_IMPL_BRG_PATH_LEN, "%s\n", psEntry->pszFullPath);
+	PVR_LOG_IF_FALSE(ui32Size > 5, "ui32Size too small, Error suspected!");
+	eError = TLStreamWrite(hStream, (IMG_UINT8 *)aszName, ui32Size+1);
+
+	return eError;
+}
+
+
+PVRSRV_ERROR DIListAllEntriesKM(DI_CONTEXT *psContext)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psContext != NULL, "psContext");
+
+	OSWRLockAcquireRead(_g_psImpl->psEntriesLock);
+	eError = HASH_Iterate(_g_psImpl->psEntriesTable, _listName, psContext->hStream);
+	OSWRLockReleaseRead(_g_psImpl->psEntriesLock);
+	PVR_LOG_IF_ERROR(eError, "HASH_Iterate_Extended");
+
+	eError = TLStreamMarkEOS(psContext->hStream, IMG_FALSE);
+	return eError;
+}
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	PVRSRV_ERROR eError;
+
+	_g_psImpl = OSAllocMem(sizeof(*_g_psImpl));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psImpl, eError, return_);
+
+	_g_psImpl->psEntriesTable = HASH_Create_Extended(ENTRIES_TABLE_INIT_SIZE,
+	                                                 DI_IMPL_BRG_PATH_LEN,
+	                                                 _Hash, _Compare);
+	PVR_LOG_GOTO_IF_NOMEM(_g_psImpl->psEntriesTable, eError, free_impl_);
+
+	eError = OSWRLockCreate(&_g_psImpl->psEntriesLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSCreateLock", free_table_);
+
+	eError = OSLockCreate(&_g_psImpl->psWriterLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSCreateLock", free_entries_lock_);
+
+	eError = OSEventObjectCreate("DI_WRITER_EO",
+	                             &_g_psImpl->hWriterEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", free_writer_lock_);
+
+	_g_psImpl->hWriterThread = NULL;
+	OSAtomicWrite(&_g_psImpl->eThreadState, THREAD_STATE_NULL);
+
+	dllist_init(&_g_psImpl->sWriterQueue);
+
+	return PVRSRV_OK;
+
+free_writer_lock_:
+	OSLockDestroy(_g_psImpl->psWriterLock);
+free_entries_lock_:
+	OSWRLockDestroy(_g_psImpl->psEntriesLock);
+free_table_:
+	HASH_Delete_Extended(_g_psImpl->psEntriesTable, IMG_FALSE);
+free_impl_:
+	OSFreeMem(_g_psImpl);
+	_g_psImpl = NULL;
+return_:
+	return eError;
+}
+
+static void _DeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	THREAD_STATE eTState;
+
+	eTState = OSAtomicCompareExchange(&_g_psImpl->eThreadState,
+	                                  THREAD_STATE_ALIVE,
+	                                  THREAD_STATE_TERMINATED);
+
+	if (eTState == THREAD_STATE_ALIVE)
+	{
+		if (_g_psImpl->hWriterEventObject != NULL)
+		{
+			eError = OSEventObjectSignal(_g_psImpl->hWriterEventObject);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(WRITER_THREAD_DESTROY_TIMEOUT)
+		{
+			eError = OSThreadDestroy(_g_psImpl->hWriterThread);
+			if (eError == PVRSRV_OK)
+			{
+				break;
+			}
+			OSWaitus(WRITER_THREAD_DESTROY_TIMEOUT/WRITER_THREAD_DESTROY_RETRIES);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (_g_psImpl->hWriterEventObject != NULL)
+	{
+		eError = OSEventObjectDestroy(_g_psImpl->hWriterEventObject);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	HASH_Delete_Extended(_g_psImpl->psEntriesTable, IMG_FALSE);
+	OSLockDestroy(_g_psImpl->psWriterLock);
+	OSWRLockDestroy(_g_psImpl->psEntriesLock);
+	OSFreeMem(_g_psImpl);
+	_g_psImpl = NULL;
+}
+
+/* Recursively traverses the ancestors list up to the root group and
+ * appends their names preceded by "/" to the path in reverse order
+ * (root group's name first and psGroup group's name last).
+ * Returns current offset in the path (the current path length without the
+ * NUL character). If there is no more space in the path returns -1
+ * to indicate an error (the path is too long to fit into the buffer). */
+static ssize_t _BuildGroupPath(IMG_CHAR *pszPath, const DIIB_GROUP *psGroup)
+{
+	ssize_t iOff;
+	ssize_t iCopiedCnt;
+	size_t  uFreeCnt;
+
+	PVR_RETURN_IF_FALSE(psGroup != NULL, 0);
+
+	PVR_ASSERT(pszPath != NULL);
+
+	iOff = _BuildGroupPath(pszPath, psGroup->psParentGroup);
+	PVR_RETURN_IF_FALSE(iOff != -1, -1);
+
+	uFreeCnt = DI_IMPL_BRG_PATH_LEN - iOff;
+	PVR_RETURN_IF_FALSE(uFreeCnt > 1, -1);
+
+	pszPath[iOff++] = '/';
+	--uFreeCnt;
+
+	iCopiedCnt = OSStringSafeCopy(&pszPath[iOff], psGroup->pszName, uFreeCnt);
+	PVR_RETURN_IF_FALSE(iCopiedCnt >= 0, -1);
+	iOff += iCopiedCnt;
+
+	return iOff;
+}
+
+static PVRSRV_ERROR _BuildEntryPath(IMG_CHAR *pszPath, const IMG_CHAR *pszName,
+                                    const DIIB_GROUP *psGroup)
+{
+	ssize_t iOff = _BuildGroupPath(pszPath, psGroup);
+	ssize_t iCopiedCnt;
+	size_t  uFreeCnt;
+
+	PVR_RETURN_IF_FALSE(iOff != -1, PVRSRV_ERROR_INVALID_OFFSET);
+
+	uFreeCnt = DI_IMPL_BRG_PATH_LEN - iOff;
+	PVR_RETURN_IF_FALSE(uFreeCnt > 1, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	pszPath[iOff++] = '/';
+	--uFreeCnt;
+
+	iCopiedCnt = OSStringSafeCopy(&pszPath[iOff], pszName, uFreeCnt);
+	PVR_RETURN_IF_FALSE(iCopiedCnt >= 0, PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _CreateEntry(const IMG_CHAR *pszName,
+                                 DI_ENTRY_TYPE eType,
+                                 const DI_ITERATOR_CB *psIterCb,
+                                 void *pvPrivData,
+                                 void *pvParentGroup,
+                                 void **pvEntry)
+{
+	DIIB_GROUP *psParentGroup = pvParentGroup;
+	DIIB_ENTRY *psEntry;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvEntry != NULL, "pvEntry");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentGroup != NULL, "pvParentGroup");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, return_);
+	}
+
+	psEntry = OSAllocMem(sizeof(*psEntry));
+	PVR_LOG_GOTO_IF_NOMEM(psEntry, eError, return_);
+
+	eError = OSLockCreate(&psEntry->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", free_entry_);
+
+	psEntry->eType = eType;
+	psEntry->sIterCb = *psIterCb;
+	psEntry->pvPrivData = pvPrivData;
+	psEntry->psParentGroup = psParentGroup;
+	psEntry->pszFullPath[0] = '\0';
+
+	psEntry->sImplEntry.pvPrivData = pvPrivData;
+	psEntry->sImplEntry.pvNative = NULL;
+	psEntry->sImplEntry.psCb = &_g_sEntryCallbacks;
+
+	eError = _BuildEntryPath(psEntry->pszFullPath, pszName,
+	                         psEntry->psParentGroup);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s() failed in _BuildEntryPath() for \"%s\" "
+		        "entry", __func__, pszName));
+		goto destroy_lock_;
+	}
+
+	OSWRLockAcquireWrite(_g_psImpl->psEntriesLock);
+	eError = HASH_Insert_Extended(_g_psImpl->psEntriesTable,
+	                              psEntry->pszFullPath,
+	                              (uintptr_t) psEntry) ?
+	         PVRSRV_OK : PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+	OSWRLockReleaseWrite(_g_psImpl->psEntriesLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HASH_Insert_Extended failed", destroy_lock_);
+
+	*pvEntry = psEntry;
+
+	return PVRSRV_OK;
+
+destroy_lock_:
+	OSLockDestroy(psEntry->hLock);
+free_entry_:
+	OSFreeMem(psEntry);
+return_:
+	return eError;
+}
+
+static void _DestroyEntry(void *pvEntry)
+{
+	DIIB_ENTRY *psEntry = pvEntry;
+	PVR_ASSERT(psEntry != NULL);
+
+	OSWRLockAcquireWrite(_g_psImpl->psEntriesLock);
+	HASH_Remove_Extended(_g_psImpl->psEntriesTable, psEntry->pszFullPath);
+	OSWRLockReleaseWrite(_g_psImpl->psEntriesLock);
+
+	OSLockDestroy(psEntry->hLock);
+	OSFreeMem(psEntry);
+}
+
+static PVRSRV_ERROR _CreateGroup(const IMG_CHAR *pszName,
+                                 void *pvParentGroup,
+                                 void **ppvGroup)
+{
+	DIIB_GROUP *psNewGroup;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvGroup != NULL, "ppvGroup");
+
+	psNewGroup = OSAllocMem(sizeof(*psNewGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psNewGroup, "OSAllocMem");
+
+	psNewGroup->pszName = pszName;
+	psNewGroup->psParentGroup = pvParentGroup;
+
+	*ppvGroup = psNewGroup;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyGroup(void *pvGroup)
+{
+	DIIB_GROUP *psGroup = pvGroup;
+	PVR_ASSERT(psGroup != NULL);
+
+	OSFreeMem(psGroup);
+}
+
+PVRSRV_ERROR PVRDIImplBrgRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateEntry,
+		.pfnDestroyEntry = _DestroyEntry,
+		.pfnCreateGroup = _CreateGroup,
+		.pfnDestroyGroup = _DestroyGroup
+	};
+
+	return DIRegisterImplementation("impl_brg", &sImplCb);
+}
+
+/******************************************************************************
+ End of file (di_impl_brg.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/di_impl_brg.h b/drivers/gpu/drm/img-rogue/di_impl_brg.h
new file mode 100644
index 000000000000..b8cff25d94ec
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/di_impl_brg.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_IMPL_BRG_H
+#define PVR_IMPL_BRG_H
+
+#include "pvrsrv_error.h"
+#include "lock_types.h"
+
+typedef struct DI_CONTEXT_TAG DI_CONTEXT;
+typedef struct DI_ENTRY_DESC DI_ENTRY_DESC;
+
+PVRSRV_ERROR PVRDIImplBrgRegister(void);
+
+/*! @Function DICreateContextKM
+ *
+ * @Description
+ * Creates DI context which among others also creates a TL stream for reading
+ * entries.
+ *
+ * @Output pszStreamName: name of the TL stream created in this context
+ * @Output ppsContext: pointer to the new context
+ *
+ * @Return PVRSRV_ERROR error code
+ *         PVRSRV_OK in case of a success
+ *         PVRSRV_ERROR_INVALID_PARAMS if any of the parameters is invalid
+ *         PVRSRV_ERROR_OUT_OF_MEMORY if any of the memory allocations failed
+ *         error codes returned by TLStreamCreate()
+ */
+PVRSRV_ERROR DICreateContextKM(IMG_CHAR *pszStreamName,
+                               DI_CONTEXT **ppsContext);
+
+/*! @Function DIDestroyContextKM
+ *
+ * @Description
+ * Destroy the DI context and all underlying dependencies.
+ *
+ * @Input psContext: pointer to the context
+ *
+ * @Return PVRSRV_ERROR error code
+ *         PVRSRV_OK in case of a success
+ *         PVRSRV_ERROR_INVALID_PARAMS if invalid context pointer given
+ */
+PVRSRV_ERROR DIDestroyContextKM(DI_CONTEXT *psContext);
+
+PVRSRV_ERROR DIReadEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                           IMG_UINT64 ui64Offset, IMG_UINT64 ui64Size);
+
+PVRSRV_ERROR DIWriteEntryKM(DI_CONTEXT *psContext, const IMG_CHAR *pszEntryPath,
+                            IMG_UINT32 ui32ValueSize, const IMG_CHAR *pszValue);
+
+PVRSRV_ERROR DIListAllEntriesKM(DI_CONTEXT *psContext);
+
+
+#endif /* PVR_IMPL_BRG_H */
diff --git a/drivers/gpu/drm/img-rogue/di_impl_brg_intern.h b/drivers/gpu/drm/img-rogue/di_impl_brg_intern.h
new file mode 100644
index 000000000000..5e11cac59854
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/di_impl_brg_intern.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS agnostic implementation of Debug Info internal interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_IMPL_BRG_INTERN_H
+#define PVR_IMPL_BRG_INTERN_H
+
+typedef struct DIIB_GROUP DIIB_GROUP;
+typedef struct DIIB_ENTRY DIIB_ENTRY;
+
+/*! @Function DIImplBrgFind
+ *
+ * @Description
+ * Retrieves an entry based on a given path.
+ *
+ * @Input pszPath: Full entry path in form of
+ *                 /rootGroup/.../parentGroup/entryName.
+ *
+ * @Return Returns entry object if exists or NULL otherwise.
+ */
+DIIB_ENTRY *DIImplBrgFind(const IMG_CHAR *pszPath);
+
+#endif /* PVR_IMPL_BRG_INTERN_H */
diff --git a/drivers/gpu/drm/img-rogue/di_server.c b/drivers/gpu/drm/img-rogue/di_server.c
new file mode 100644
index 000000000000..47046d7ec3a2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/di_server.c
@@ -0,0 +1,800 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Info framework functions and types.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "di_server.h"
+#include "osdi_impl.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "lock.h"
+#include "allocmem.h"
+#include "osfunc.h"
+
+#define ROOT_GROUP_NAME PVR_DRM_NAME
+
+/*! Implementation object. */
+typedef struct DI_IMPL
+{
+	const IMG_CHAR *pszName;       /*<! name of the implementation */
+	OSDI_IMPL_CB sCb;              /*<! implementation callbacks */
+	IMG_BOOL bInitialised;         /*<! set to IMG_TRUE after implementation
+	                                    is initialised */
+
+	DLLIST_NODE sListNode;         /*<! node element of the global list of all
+	                                    implementations */
+} DI_IMPL;
+
+/*! Wrapper object for objects originating from derivative implementations.
+ * This wraps both entries and groups native implementation objects. */
+typedef struct DI_NATIVE_HANDLE
+{
+	void *pvHandle;                /*!< opaque handle to the native object */
+	DI_IMPL *psDiImpl;             /*!< implementation pvHandle is associated
+	                                    with */
+	DLLIST_NODE sListNode;         /*!< node element of native handles list */
+} DI_NATIVE_HANDLE;
+
+/*! Debug Info entry object.
+ *
+ * Depending on the implementation this can be represented differently. For
+ * example for the DebugFS this translates to a file.
+ */
+struct DI_ENTRY
+{
+	const IMG_CHAR *pszName;       /*!< name of the entry */
+	void *pvPrivData;              /*! handle to entry's private data */
+	DI_ENTRY_TYPE eType;           /*! entry type */
+	DI_ITERATOR_CB sIterCb;        /*!< iterator interface for the entry */
+
+	DLLIST_NODE sListNode;         /*!< node element of group's entry list */
+	DLLIST_NODE sNativeHandleList; /*!< list of native handles belonging to this
+	                                    entry */
+};
+
+/*! Debug Info group object.
+ *
+ * Depending on the implementation this can be represented differently. For
+ * example for the DebugFS this translates to a directory.
+ */
+struct DI_GROUP
+{
+	IMG_CHAR *pszName;               /*!< name of the group */
+	const struct DI_GROUP *psParent; /*!< parent groups */
+
+	DLLIST_NODE sListNode;           /*!< node element of group's group list */
+	DLLIST_NODE sGroupList;          /*!< list of groups (children) that belong
+	                                      to this group */
+	DLLIST_NODE sEntryList;          /*!< list of entries (children) that belong
+	                                      to this group */
+	DLLIST_NODE sNativeHandleList;   /*!< list of native handles belonging to
+	                                      this group */
+};
+
+/* List of all registered implementations. */
+static DECLARE_DLLIST(_g_sImpls);
+
+/* Root group for the DI entries and groups. This group is used as a root
+ * group for all other groups and entries if during creation a parent groups
+ * is not given. */
+static DI_GROUP *_g_psRootGroup;
+
+/* Protects access to _g_sImpls and _g_psRootGroup */
+static POS_LOCK _g_hLock;
+
+PVRSRV_ERROR DIInit(void)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(__linux__) && defined(__KERNEL__)
+	eError = OSLockCreateNoStats(&_g_hLock);
+#else
+	eError = OSLockCreate(&_g_hLock);
+#endif
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", return_);
+
+	_g_psRootGroup = OSAllocMemNoStats(sizeof(*_g_psRootGroup));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psRootGroup, eError, destroy_lock_);
+
+	_g_psRootGroup->pszName = OSAllocMemNoStats(sizeof(ROOT_GROUP_NAME));
+	PVR_LOG_GOTO_IF_NOMEM(_g_psRootGroup->pszName, eError, cleanup_name_);
+	OSStringSafeCopy(_g_psRootGroup->pszName, ROOT_GROUP_NAME,
+				  sizeof(ROOT_GROUP_NAME));
+
+	dllist_init(&_g_psRootGroup->sListNode);
+	dllist_init(&_g_psRootGroup->sGroupList);
+	dllist_init(&_g_psRootGroup->sEntryList);
+	dllist_init(&_g_psRootGroup->sNativeHandleList);
+
+	return PVRSRV_OK;
+
+cleanup_name_:
+	OSFreeMemNoStats(_g_psRootGroup);
+destroy_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(_g_hLock);
+#else
+	OSLockDestroy(_g_hLock);
+#endif
+return_:
+	return eError;
+}
+
+/* Destroys the whole tree of group and entries for a given group as a root. */
+static void _DeInitGroupRecursively(DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+		DIDestroyEntry(psThisEntry);
+	}
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		_DeInitGroupRecursively(psThisGroup);
+	}
+
+	DIDestroyGroup(psGroup);
+}
+
+void DIDeInit(void)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	OSLockAcquire(_g_hLock);
+
+	if (!dllist_is_empty(&_g_psRootGroup->sGroupList) ||
+	    !dllist_is_empty(&_g_psRootGroup->sEntryList))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: entries or groups still exist during "
+		        "de-initialisation process, destroying all", __func__));
+	}
+
+	_DeInitGroupRecursively(_g_psRootGroup);
+	_g_psRootGroup = NULL;
+
+	/* Remove all of the implementations. */
+	dllist_foreach_node(&_g_sImpls, psThis, psNext)
+	{
+		DI_IMPL *psDiImpl = IMG_CONTAINER_OF(psThis, DI_IMPL, sListNode);
+
+		if (psDiImpl->bInitialised)
+		{
+			psDiImpl->sCb.pfnDeInit();
+			psDiImpl->bInitialised = IMG_FALSE;
+		}
+
+		dllist_remove_node(&psDiImpl->sListNode);
+		OSFreeMem(psDiImpl);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	/* all resources freed so free the lock itself too */
+
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(_g_hLock);
+#else
+	OSLockDestroy(_g_hLock);
+#endif
+}
+
+static IMG_BOOL _ValidateIteratorCb(const DI_ITERATOR_CB *psIterCb,
+                                    DI_ENTRY_TYPE eType)
+{
+	IMG_UINT32 uiFlags = 0;
+
+	if (psIterCb == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	if (eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		uiFlags |= psIterCb->pfnShow != NULL ? BIT(0) : 0;
+		uiFlags |= psIterCb->pfnStart != NULL ? BIT(1) : 0;
+		uiFlags |= psIterCb->pfnStop != NULL ? BIT(2) : 0;
+		uiFlags |= psIterCb->pfnNext != NULL ? BIT(3) : 0;
+
+		/* either only pfnShow or all callbacks need to be set */
+		if (uiFlags != BIT(0) && !BITMASK_HAS(uiFlags, 0x0f))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else if (eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		uiFlags |= psIterCb->pfnRead != NULL ? BIT(0) : 0;
+		uiFlags |= psIterCb->pfnSeek != NULL ? BIT(1) : 0;
+
+		/* either only pfnRead or all callbacks need to be set */
+		if (uiFlags != BIT(0) && !BITMASK_HAS(uiFlags, 0x03))
+		{
+			return IMG_FALSE;
+		}
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static PVRSRV_ERROR _CreateNativeEntry(DI_ENTRY *psEntry,
+                                       const DI_NATIVE_HANDLE *psNativeParent)
+{
+	PVRSRV_ERROR eError;
+	DI_IMPL *psImpl = psNativeParent->psDiImpl;
+
+	DI_NATIVE_HANDLE *psNativeEntry = OSAllocMem(sizeof(*psNativeEntry));
+	PVR_LOG_GOTO_IF_NOMEM(psNativeEntry, eError, return_);
+
+	eError = psImpl->sCb.pfnCreateEntry(psEntry->pszName,
+	                                    psEntry->eType,
+	                                    &psEntry->sIterCb,
+	                                    psEntry->pvPrivData,
+	                                    psNativeParent->pvHandle,
+	                                    &psNativeEntry->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateEntry", free_memory_);
+
+	psNativeEntry->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psEntry->sNativeHandleList, &psNativeEntry->sListNode);
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeEntry);
+return_:
+	return eError;
+}
+
+static void _DestroyNativeEntry(DI_NATIVE_HANDLE *psNativeEntry)
+{
+	dllist_remove_node(&psNativeEntry->sListNode);
+	OSFreeMem(psNativeEntry);
+}
+
+PVRSRV_ERROR DICreateEntry(const IMG_CHAR *pszName,
+                           DI_GROUP *psGroup,
+                           const DI_ITERATOR_CB *psIterCb,
+                           void *pvPriv,
+                           DI_ENTRY_TYPE eType,
+                           DI_ENTRY **ppsEntry)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_ENTRY *psEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(_ValidateIteratorCb(psIterCb, eType),
+	                                "psIterCb");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsEntry != NULL, "psEntry");
+
+	psEntry = OSAllocMem(sizeof(*psEntry));
+	PVR_LOG_RETURN_IF_NOMEM(psEntry, "OSAllocMem");
+
+	if (psGroup == NULL)
+	{
+		psGroup = _g_psRootGroup;
+	}
+
+	psEntry->pszName = pszName;
+	psEntry->pvPrivData = pvPriv;
+	psEntry->eType = eType;
+	psEntry->sIterCb = *psIterCb;
+	dllist_init(&psEntry->sNativeHandleList);
+
+	OSLockAcquire(_g_hLock);
+
+	dllist_add_to_tail(&psGroup->sEntryList, &psEntry->sListNode);
+
+	/* Iterate over all of the native handles of parent group to create
+	 * the entry for every registered implementation. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		eError = _CreateNativeEntry(psEntry, psNativeGroup);
+		PVR_GOTO_IF_ERROR(eError, cleanup_);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	*ppsEntry = psEntry;
+
+	return PVRSRV_OK;
+
+cleanup_:
+	OSLockRelease(_g_hLock);
+
+	/* Something went wrong so if there were any native entries created remove
+	 * them from the list, free them and free the DI entry itself. */
+	dllist_foreach_node(&psEntry->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeEntry =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		_DestroyNativeEntry(psNativeEntry);
+	}
+
+	OSFreeMem(psEntry);
+
+	return eError;
+}
+
+void DIDestroyEntry(DI_ENTRY *psEntry)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psEntry != NULL,
+	                             "psEntry invalid in DIDestroyEntry()");
+
+	/* Iterate through all of the native entries of the DI entry, remove
+	 * them from the list and then destroy them. After that, destroy the
+	 * DI entry itself. */
+	dllist_foreach_node(&psEntry->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNative = IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE,
+		                                              sListNode);
+
+		/* The implementation must ensure that entry is not removed if any
+		 * operations are being executed on the entry. If this is the case
+		 * the implementation should block until all of them are finished
+		 * and prevent any further operations.
+		 * This will guarantee proper synchronisation between the DI framework
+		 * and underlying implementations and prevent destruction/access
+		 * races. */
+		psNative->psDiImpl->sCb.pfnDestroyEntry(psNative->pvHandle);
+		dllist_remove_node(&psNative->sListNode);
+		OSFreeMem(psNative);
+	}
+
+	dllist_remove_node(&psEntry->sListNode);
+
+	OSFreeMem(psEntry);
+}
+
+static PVRSRV_ERROR _CreateNativeGroup(DI_GROUP *psGroup,
+                                       const DI_NATIVE_HANDLE *psNativeParent,
+                                       DI_NATIVE_HANDLE **ppsNativeGroup)
+{
+	PVRSRV_ERROR eError;
+	DI_IMPL *psImpl = psNativeParent->psDiImpl;
+
+	DI_NATIVE_HANDLE *psNativeGroup = OSAllocMem(sizeof(*psNativeGroup));
+	PVR_LOG_GOTO_IF_NOMEM(psNativeGroup, eError, return_);
+
+	eError = psImpl->sCb.pfnCreateGroup(psGroup->pszName,
+	                                    psNativeParent->pvHandle,
+	                                    &psNativeGroup->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateGroup", free_memory_);
+
+	psNativeGroup->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psGroup->sNativeHandleList, &psNativeGroup->sListNode);
+
+	*ppsNativeGroup = psNativeGroup;
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeGroup);
+return_:
+	return eError;
+}
+
+static void _DestroyNativeGroup(DI_NATIVE_HANDLE *psNativeEntry)
+{
+	dllist_remove_node(&psNativeEntry->sListNode);
+	OSFreeMem(psNativeEntry);
+}
+
+PVRSRV_ERROR DICreateGroup(const IMG_CHAR *pszName,
+                           DI_GROUP *psParent,
+                           DI_GROUP **ppsGroup)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_GROUP *psGroup;
+	size_t uSize;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsGroup != NULL, "ppsDiGroup");
+
+	psGroup = OSAllocMem(sizeof(*psGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psGroup, "OSAllocMem");
+
+	if (psParent == NULL)
+	{
+		psParent = _g_psRootGroup;
+	}
+
+	uSize = OSStringLength(pszName) + 1;
+	psGroup->pszName = OSAllocMem(uSize * sizeof(*psGroup->pszName));
+	PVR_LOG_GOTO_IF_NOMEM(psGroup->pszName, eError, cleanup_name_);
+	OSStringSafeCopy(psGroup->pszName, pszName, uSize);
+
+	psGroup->psParent = psParent;
+	dllist_init(&psGroup->sGroupList);
+	dllist_init(&psGroup->sEntryList);
+	dllist_init(&psGroup->sNativeHandleList);
+
+	OSLockAcquire(_g_hLock);
+
+	dllist_add_to_tail(&psParent->sGroupList, &psGroup->sListNode);
+
+	/* Iterate over all of the native handles of parent group to create
+	 * the group for every registered implementation. */
+	dllist_foreach_node(&psParent->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup = NULL, *psNativeParent =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		eError = _CreateNativeGroup(psGroup, psNativeParent, &psNativeGroup);
+		PVR_GOTO_IF_ERROR(eError, cleanup_);
+	}
+
+	OSLockRelease(_g_hLock);
+
+	*ppsGroup = psGroup;
+
+	return PVRSRV_OK;
+
+cleanup_:
+	OSLockRelease(_g_hLock);
+
+	/* Something went wrong so if there were any native groups created remove
+	 * them from the list, free them and free the DI group itself. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		dllist_remove_node(&psNativeGroup->sListNode);
+		OSFreeMem(psNativeGroup);
+	}
+
+	OSFreeMem(psGroup->pszName);
+cleanup_name_:
+	OSFreeMem(psGroup);
+
+	return eError;
+}
+
+void DIDestroyGroup(DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psGroup != NULL,
+	                             "psGroup invalid in DIDestroyGroup()");
+
+	/* Iterate through all of the native groups of the DI group, remove
+	 * them from the list and then destroy them. After that destroy the
+	 * DI group itself. */
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNative = IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE,
+		                                              sListNode);
+
+		psNative->psDiImpl->sCb.pfnDestroyGroup(psNative->pvHandle);
+		dllist_remove_node(&psNative->sListNode);
+		OSFreeMem(psNative);
+	}
+
+	dllist_remove_node(&psGroup->sListNode);
+
+	if (psGroup == _g_psRootGroup)
+	{
+		OSFreeMemNoStats(psGroup->pszName);
+		OSFreeMemNoStats(psGroup);
+	}
+	else
+	{
+		OSFreeMem(psGroup->pszName);
+		OSFreeMem(psGroup);
+	}
+}
+
+void *DIGetPrivData(const OSDI_IMPL_ENTRY *psEntry)
+{
+	PVR_ASSERT(psEntry != NULL);
+
+	return psEntry->pvPrivData;
+}
+
+void DIWrite(const OSDI_IMPL_ENTRY *psEntry, const void *pvData,
+             IMG_UINT32 uiSize)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnWrite != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnWrite(psEntry->pvNative, pvData, uiSize);
+}
+
+void DIPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt, ...)
+{
+	va_list args;
+
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnVPrintf != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	va_start(args, pszFmt);
+	psEntry->psCb->pfnVPrintf(psEntry->pvNative, pszFmt, args);
+	va_end(args);
+}
+
+void DIVPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt,
+               va_list pArgs)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnVPrintf != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnVPrintf(psEntry->pvNative, pszFmt, pArgs);
+}
+
+void DIPuts(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszStr)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnPuts != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	psEntry->psCb->pfnPuts(psEntry->pvNative, pszStr);
+}
+
+IMG_BOOL DIHasOverflowed(const OSDI_IMPL_ENTRY *psEntry)
+{
+	PVR_ASSERT(psEntry != NULL);
+	PVR_ASSERT(psEntry->psCb != NULL);
+	PVR_ASSERT(psEntry->psCb->pfnHasOverflowed != NULL);
+	PVR_ASSERT(psEntry->pvNative != NULL);
+
+	return psEntry->psCb->pfnHasOverflowed(psEntry->pvNative);
+}
+
+/* ---- OS implementation API ---------------------------------------------- */
+
+static IMG_BOOL _ValidateImplCb(const OSDI_IMPL_CB *psImplCb)
+{
+	PVR_GOTO_IF_FALSE(psImplCb->pfnInit != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDeInit != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnCreateGroup != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDestroyGroup != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnCreateEntry != NULL, failed_);
+	PVR_GOTO_IF_FALSE(psImplCb->pfnDestroyEntry != NULL, failed_);
+
+	return IMG_TRUE;
+
+failed_:
+	return IMG_FALSE;
+}
+
+/* Walks the tree of groups and entries and create all of the native handles
+ * for the given implementation for all of the already existing groups and
+ * entries. */
+static PVRSRV_ERROR _InitNativeHandlesRecursively(DI_IMPL *psImpl,
+                                               DI_GROUP *psGroup,
+                                               DI_NATIVE_HANDLE *psNativeParent)
+{
+	PVRSRV_ERROR eError;
+	DLLIST_NODE *psThis, *psNext;
+	DI_NATIVE_HANDLE *psNativeGroup;
+
+	psNativeGroup = OSAllocMem(sizeof(*psNativeGroup));
+	PVR_LOG_RETURN_IF_NOMEM(psNativeGroup, "OSAllocMem");
+
+	eError = psImpl->sCb.pfnCreateGroup(psGroup->pszName,
+	                           psNativeParent ? psNativeParent->pvHandle : NULL,
+	                           &psNativeGroup->pvHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->sCb.pfnCreateGroup", free_memory_);
+
+	psNativeGroup->psDiImpl = psImpl;
+
+	dllist_add_to_head(&psGroup->sNativeHandleList,
+	                   &psNativeGroup->sListNode);
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		// and then walk the new group
+		eError = _InitNativeHandlesRecursively(psImpl, psThisGroup,
+		                                       psNativeGroup);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_InitNativeHandlesRecursively");
+	}
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+
+		eError = _CreateNativeEntry(psThisEntry, psNativeGroup);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_CreateNativeEntry");
+	}
+
+	return PVRSRV_OK;
+
+free_memory_:
+	OSFreeMem(psNativeGroup);
+
+	return eError;
+}
+
+/* Walks the tree of groups and entries and destroys all of the native handles
+ * for the given implementation. */
+static void _DeInitNativeHandlesRecursively(DI_IMPL *psImpl, DI_GROUP *psGroup)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	dllist_foreach_node(&psGroup->sEntryList, psThis, psNext)
+	{
+		DI_ENTRY *psThisEntry = IMG_CONTAINER_OF(psThis, DI_ENTRY, sListNode);
+
+		// free all of the native entries that belong to this implementation
+		dllist_foreach_node(&psThisEntry->sNativeHandleList, psThis, psNext)
+		{
+			DI_NATIVE_HANDLE *psNativeEntry =
+			        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+			if (psNativeEntry->psDiImpl == psImpl)
+			{
+				_DestroyNativeEntry(psNativeEntry);
+				// there can be only one entry on the list for a given
+				// implementation
+				break;
+			}
+		}
+	}
+
+	dllist_foreach_node(&psGroup->sGroupList, psThis, psNext)
+	{
+		DI_GROUP *psThisGroup = IMG_CONTAINER_OF(psThis, DI_GROUP, sListNode);
+
+		// and then walk the new group
+		_DeInitNativeHandlesRecursively(psImpl, psThisGroup);
+	}
+
+	// free all of the native entries that belong to this implementation
+	dllist_foreach_node(&psGroup->sNativeHandleList, psThis, psNext)
+	{
+		DI_NATIVE_HANDLE *psNativeGroup =
+		        IMG_CONTAINER_OF(psThis, DI_NATIVE_HANDLE, sListNode);
+
+		if (psNativeGroup->psDiImpl == psImpl)
+		{
+			_DestroyNativeGroup(psNativeGroup);
+			// there can be only one entry on the list for a given
+			// implementation
+			break;
+		}
+	}
+}
+
+static PVRSRV_ERROR _InitImpl(DI_IMPL *psImpl)
+{
+	PVRSRV_ERROR eError;
+	// DI_NATIVE_HANDLE *psNativeGroup;
+
+	eError = psImpl->sCb.pfnInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "psImpl->pfnInit()", return_);
+
+	/* if the implementation is being created after any groups or entries
+	 * have been created we need to walk the current tree and create
+	 * native groups and entries for all of the existing ones */
+	eError = _InitNativeHandlesRecursively(psImpl, _g_psRootGroup, NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_InitNativeHandlesRecursively",
+	                      free_native_handles_and_deinit_);
+
+	psImpl->bInitialised = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+free_native_handles_and_deinit_:
+	/* something went wrong so we need to walk the tree and remove all of the
+	 * native entries and groups that we've created before we can destroy
+	 * the implementation */
+	_DeInitNativeHandlesRecursively(psImpl, _g_psRootGroup);
+	psImpl->sCb.pfnDeInit();
+return_:
+	return eError;
+}
+
+PVRSRV_ERROR DIRegisterImplementation(const IMG_CHAR *pszName,
+                                      const OSDI_IMPL_CB *psImplCb)
+{
+	DI_IMPL *psImpl;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(_ValidateImplCb(psImplCb), "psImplCb");
+	/* if root group does not exist it can mean 2 things:
+	 * - DIInit() was not called so initialisation order is incorrect and needs
+	 *   to be fixed
+	 * - DIInit() failed but if that happens we should never make it here */
+	PVR_ASSERT(_g_psRootGroup != NULL);
+
+	psImpl = OSAllocMem(sizeof(*psImpl));
+	PVR_LOG_RETURN_IF_NOMEM(psImpl, "OSAllocMem");
+
+	psImpl->pszName = pszName;
+	psImpl->sCb = *psImplCb;
+
+	OSLockAcquire(_g_hLock);
+
+	eError = _InitImpl(psImpl);
+	if (eError != PVRSRV_OK)
+	{
+		/* implementation could not be initialised so remove it from the
+		 * list, free the memory and forget about it */
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: could not initialise \"%s\" debug "
+		        "info implementation, discarding", __func__,
+		        psImpl->pszName));
+
+		goto free_impl_;
+	}
+
+	psImpl->bInitialised = IMG_TRUE;
+
+	dllist_add_to_tail(&_g_sImpls, &psImpl->sListNode);
+
+	OSLockRelease(_g_hLock);
+
+	return PVRSRV_OK;
+
+free_impl_:
+	OSLockRelease(_g_hLock);
+
+	OSFreeMem(psImpl);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/di_server.h b/drivers/gpu/drm/img-rogue/di_server.h
new file mode 100644
index 000000000000..a797890c069a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/di_server.h
@@ -0,0 +1,220 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for creating Debug Info groups and entries.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DI_SERVER_H
+#define DI_SERVER_H
+
+#if defined(__KERNEL__) && defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __KERNEL__ && __linux__ */
+
+#include "di_common.h"
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+/*! @Function DIInit
+ *
+ * @Description
+ * Initialises Debug Info framework. This function will create common resources
+ * for the framework.
+ *
+ * Note: This function must be called before first call to
+ *       DIRegisterImplementation() all of the implementations.
+ */
+PVRSRV_ERROR DIInit(void);
+
+/*! @Function DIDeInit
+ *
+ * @Description
+ * De-initialises Debug Info framework. This function will call pfnDeInit()
+ * on each implementation and clean up common resources.
+ *
+ * In case some of the entries and groups have not been cleaned up this function
+ * will also perform recursive sweep and remove all entries and group for
+ * all implementations.
+ */
+void DIDeInit(void);
+
+/*! @Function DICreateEntry
+ *
+ * @Description
+ * Creates debug info entry. Depending on different implementations the entry
+ * might be for example a DebugFS file or something totally different.
+ *
+ * The entry will belong to a parent group if provided or to the root group
+ * if not.
+ *
+ * @Input pszName: name of the new entry
+ * @Input psDiGroup: parent group, if NULL entry will belong to the root group
+ * @Input psIterCb: implementation of the iterator for the entry
+ * @Input psPriv: private data that will be passed to the iterator operations
+ * @Input eType: type of the entry
+ *
+ * @Output ppsEntry: handle to the newly created entry
+ *
+ * @Return   PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DICreateEntry(const IMG_CHAR *pszName,
+                           DI_GROUP *psGroup,
+                           const DI_ITERATOR_CB *psIterCb,
+                           void *psPriv,
+                           DI_ENTRY_TYPE eType,
+                           DI_ENTRY **ppsEntry);
+
+/*! @Function DIDestroyEntry
+ *
+ * @Description
+ * Destroys debug info entry.
+ *
+ * @Input psEntry: handle to the entry
+ */
+void DIDestroyEntry(DI_ENTRY *psEntry);
+
+/*! @Function DICreateGroup
+ *
+ * @Description
+ * Creates debug info group. Depending on different implementations the group
+ * might be for example a DebugFS directory or something totally different.
+ *
+ * The group will belong to a parent group if provided or to the root group
+ * if not.
+ *
+ * @Input pszName: name of the new entry
+ * @Input psParent: parent group, if NULL entry will belong to the root group
+ *
+ * @Output ppsGroup: handle to the newly created entry
+ *
+ * @Return   PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DICreateGroup(const IMG_CHAR *pszName,
+                           DI_GROUP *psParent,
+                           DI_GROUP **ppsGroup);
+
+/*! @Function DIDestroyGroup
+ *
+ * @Description
+ * Destroys debug info group.
+ *
+ * @Input psGroup: handle to the group
+ */
+void DIDestroyGroup(DI_GROUP *psGroup);
+
+/*! @Function DIGetPrivData
+ *
+ * @Description
+ * Retrieves private data from psEntry. The data is either passed during
+ * entry creation via psPriv parameter of DICreateEntry() function
+ * or by explicitly setting it with DIGetPrivData() function.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ *
+ * @Returns pointer to the private data (can be NULL if private data
+ *          has not been specified)
+ */
+void *DIGetPrivData(const OSDI_IMPL_ENTRY *psEntry);
+
+/*! @Function DIWrite
+ *
+ * @Description
+ * Writes the binary data of the DI entry to the output sync, whatever that may
+ * be for the DI implementation.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pvData data
+ * @Input uiSize pvData length
+ */
+void DIWrite(const OSDI_IMPL_ENTRY *psEntry, const void *pvData,
+             IMG_UINT32 uiSize);
+
+/*! @Function DIPrintf
+ *
+ * @Description
+ * Prints formatted string to the DI entry.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated format string
+ */
+__printf(2, 3)
+void DIPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt, ...);
+
+/*! @Function DIVPrintf
+ *
+ * @Description
+ * Prints formatted string to the DI entry. Equivalent to DIPrintf but takes
+ * va_list instead of a variable number of arguments.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated format string
+ * @Input pArgs vs_list object
+ */
+__printf(2, 0)
+void DIVPrintf(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszFmt,
+               va_list pArgs);
+
+/*! @Function DIPrintf
+ *
+ * @Description
+ * Prints a string to the DI entry.
+ *
+ * @Input psEntry pointer to OSDI_IMPL_ENTRY object
+ * @Input pszFmt NUL-terminated string
+ */
+void DIPuts(const OSDI_IMPL_ENTRY *psEntry, const IMG_CHAR *pszStr);
+
+/*! @Function DIHasOverflowed
+ *
+ * @Description
+ * Checks if the DI buffer has overflowed.
+ *
+ * @Return IMG_TRUE if buffer overflowed
+ */
+IMG_BOOL DIHasOverflowed(const OSDI_IMPL_ENTRY *psEntry);
+
+#endif /* DI_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/dkf_server.c b/drivers/gpu/drm/img-rogue/dkf_server.c
new file mode 100644
index 000000000000..6136b627d9ae
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/dkf_server.c
@@ -0,0 +1,409 @@
+/*************************************************************************/ /*!
+@File         dkf_server.c
+@Title        DRM Key Framework support routines.
+@Copyright    Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "dkf_server.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvrsrv_error.h"
+#include "dkp_impl.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#if defined(SUPPORT_LINUX_FDINFO)
+typedef struct DKF_REGISTERED_DKP_TAG
+{
+	IMG_HANDLE              hPrivHandle;    /*!< Private data - can be NULL */
+	const IMG_CHAR          *pszDKPName;    /*!< DKP provider name */
+	IMG_PID                 pid;            /*!< Process-ID - can be 0 */
+	IMG_UINT32              uiReserved1;    /*!< Reserved field - padding */
+	DKP_PFN_SHOW            *psDKPShowPfn;  /*!< DKP Callback function */
+	DLLIST_NODE             sDKFEntryNode;  /*!< List of DKP entries */
+
+	DKP_CONNECTION_FLAGS    ui32Filter;     /*!< The types of connection to output to */
+} DKF_REGISTERED_DKP;
+
+/* Global sentinel to track all allocated DKP callback key/value pairs */
+typedef struct DKF_TAG
+{
+	POS_LOCK            hDKFListLock;   /*!< Lock for accessing sDKFListNode */
+	IMG_UINT32          ui32NumEntries; /*!< Number of registered DKP entries */
+	IMG_UINT32          uiReserved1;    /*!< Reserved field - padding */
+	DLLIST_NODE         sDKFListNode;   /*!< Head of the DKF_REGISTERED_DKP linked list */
+	DKF_VPRINTF_FUNC    *pfnPrint;  /*!< Default printf-style output fn */
+	void                *pvPrintArg1; /*!< First arg for pfnPrint */
+} DKF;
+
+static DKF *gpsDKF;
+
+static_assert(DKF_CONNECTION_FLAG_INVALID == DKP_CONNECTION_FLAG_INVALID, "DKF and DKP INVALID connection flags do not match");
+static_assert(DKF_CONNECTION_FLAG_SYNC == DKP_CONNECTION_FLAG_SYNC, "DKF and DKP SYNC connection flags do not match");
+static_assert(DKF_CONNECTION_FLAG_SERVICES == DKP_CONNECTION_FLAG_SERVICES, "DKF and DKP SERVICES connection flags do not match");
+
+PVRSRV_ERROR PVRDKFInit(void)
+{
+	DKF *psDKF;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	if (gpsDKF != NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: gpsDKF = %p, NULL expected",
+		        __func__, gpsDKF));
+
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s called", __func__));
+
+	psDKF = OSAllocZMemNoStats(sizeof(*psDKF));
+	PVR_GOTO_IF_NOMEM(psDKF, eError, Error);
+
+	dllist_init(&psDKF->sDKFListNode);
+	eError = OSLockCreate(&psDKF->hDKFListLock);
+	PVR_GOTO_IF_ERROR(eError, ErrorFree);
+
+	gpsDKF = psDKF;
+
+	return PVRSRV_OK;
+
+ErrorFree:
+	OSFreeMemNoStats(psDKF);
+	/* fallthrough */
+
+Error:
+	PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+	return eError;
+}
+
+
+void PVRDKFDeInit(void)
+{
+	IMG_UINT32 uiNumFreed = 0U;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s called", __func__));
+
+	if (gpsDKF == NULL)
+	{
+		return;
+	}
+
+	/* Ensure we leave no data allocated. The DKP instances should have
+	 * been cleaned up by their module deInit processing. Handle badly
+	 * behaved clients here.
+	 */
+	if (gpsDKF->ui32NumEntries > 0)
+	{
+		DLLIST_NODE *psThis, *psNext;
+
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Have %u un-freed allocations remaining", __func__,
+		         gpsDKF->ui32NumEntries));
+
+		OSLockAcquire(gpsDKF->hDKFListLock);
+
+		dllist_foreach_node(&gpsDKF->sDKFListNode, psThis, psNext)
+		{
+			DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(psThis,
+			                                             DKF_REGISTERED_DKP,
+			                                             sDKFEntryNode);
+
+			dllist_remove_node(&psEntry->sDKFEntryNode);
+
+			uiNumFreed++;
+
+			OSFreeMemNoStats(psEntry);
+		}
+
+		if (uiNumFreed != gpsDKF->ui32NumEntries)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could only free %u out of %u",
+			         uiNumFreed, gpsDKF->ui32NumEntries));
+		}
+
+		dllist_remove_node(&gpsDKF->sDKFListNode);
+
+		OSLockRelease(gpsDKF->hDKFListLock);
+	}
+
+	OSLockDestroy(gpsDKF->hDKFListLock);
+
+	OSFreeMemNoStats(gpsDKF);
+
+	gpsDKF = NULL;
+}
+
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    DKF_CONNECTION_FLAGS ui32ConnectionType)
+{
+	PDLLIST_NODE pNext, pNode;
+
+	PVR_ASSERT(gpsDKF != NULL);
+	PVR_ASSERT(pfnPrint != NULL);
+	PVR_ASSERT(ui32ConnectionType != DKF_CONNECTION_FLAG_INVALID);
+
+	OSLockAcquire(gpsDKF->hDKFListLock);
+
+	gpsDKF->pfnPrint = pfnPrint;
+	gpsDKF->pvPrintArg1 = pvArg;
+
+	if (dllist_is_empty(&gpsDKF->sDKFListNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: No DKPs registered", __func__));
+	}
+	else
+	{
+		dllist_foreach_node(&gpsDKF->sDKFListNode, pNode, pNext)
+		{
+			DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(pNode,
+			                                               DKF_REGISTERED_DKP,
+			                                               sDKFEntryNode);
+
+			if (psEntry->psDKPShowPfn != NULL &&
+			    BITMASK_ANY(psEntry->ui32Filter, ui32ConnectionType))
+			{
+				psEntry->psDKPShowPfn(psDevNode, pid,
+				                      psEntry->hPrivHandle);
+			}
+		}
+	}
+
+	OSLockRelease(gpsDKF->hDKFListLock);
+}
+
+/*
+ * Wrapper function to display data using preconfigured DKF-specific output
+ * function.
+ */
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...)
+{
+	DKF_REGISTERED_DKP *psDKFEntry = (DKF_REGISTERED_DKP *)hPrivData;
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list Arglist;
+	DLLIST_NODE *pNode, *pNext;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (psDKFEntry == NULL || gpsDKF == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: NULL DKF entry found (%p, %p)",
+		        __func__, psDKFEntry, gpsDKF));
+		return;
+	}
+
+	PVR_ASSERT(gpsDKF->pfnPrint != NULL);
+
+	/* validate that this is a legitimate output function reference */
+
+	dllist_foreach_node(&gpsDKF->sDKFListNode, pNode, pNext)
+	{
+		DKF_REGISTERED_DKP *psEntry = IMG_CONTAINER_OF(pNode,
+		                                               DKF_REGISTERED_DKP,
+		                                               sDKFEntryNode);
+
+		if (psEntry == psDKFEntry)
+		{
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!bFound)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Handle %p not found.", __func__,
+		         hPrivData));
+		return;
+	}
+
+	OSSNPrintf(szBuffer, PVR_MAX_DEBUG_MESSAGE_LEN, "%s", fmt);
+
+	va_start(Arglist, fmt);
+	gpsDKF->pfnPrint(gpsDKF->pvPrintArg1, szBuffer, &Arglist);
+	va_end(Arglist);
+}
+
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                               const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	DKF_REGISTERED_DKP *psDKFEntry; /* New entry for this DKP */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+
+	/* Check for a NULL argument. Nothing to allocate if we are not provided
+	 * a location to store the DkpHandle reference.
+	 */
+	if (phDkpHandle == NULL || ui32Filter == DKF_CONNECTION_FLAG_INVALID)
+	{
+
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_WARNING, "%s(%p, %s, %p, %p) Called", __func__,
+		         hPrivData, pszDKPName, psShowPfn,
+		         phDkpHandle));
+#endif
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDKFEntry = OSAllocZMemNoStats(sizeof(*psDKFEntry));
+	PVR_GOTO_IF_NOMEM(psDKFEntry, eError, Error);
+
+	OSLockAcquire(gpsDKF->hDKFListLock);
+
+	psDKFEntry->hPrivHandle = hPrivData;
+	psDKFEntry->pszDKPName = pszDKPName;
+	psDKFEntry->psDKPShowPfn = psShowPfn;
+	psDKFEntry->ui32Filter = ui32Filter;
+	dllist_init(&psDKFEntry->sDKFEntryNode);
+
+	/*
+	 * Append the new entry to the end of the gpsDKF list-head. Return a
+	 * reference to the new entry to the caller.
+	 */
+	dllist_add_to_tail(&gpsDKF->sDKFListNode, &psDKFEntry->sDKFEntryNode);
+
+	gpsDKF->ui32NumEntries++;
+
+	OSLockRelease(gpsDKF->hDKFListLock);
+
+	*phDkpHandle = psDKFEntry;
+
+	return PVRSRV_OK;
+
+
+Error:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Error: '%s'", __func__,
+	         PVRSRVGetErrorString(eError)));
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData, PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	DKF_REGISTERED_DKP *psDKFEntry = (DKF_REGISTERED_DKP *)hDkpHandle;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDKFEntry)
+	{
+#if defined(DEBUG)
+		if (psDKFEntry->hPrivHandle == hPrivData)
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Matched %p private handle",
+			         __func__, hDkpHandle));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_VERBOSE,
+			         "%s: Did not find match (%p. vs %p), freeing anyway",
+			         __func__, hPrivData, psDKFEntry->hPrivHandle));
+		}
+#endif	/* DEBUG */
+
+		OSLockAcquire(gpsDKF->hDKFListLock);
+
+		dllist_remove_node(&psDKFEntry->sDKFEntryNode);
+
+		gpsDKF->ui32NumEntries--;
+
+		OSLockRelease(gpsDKF->hDKFListLock);
+
+		OSFreeMemNoStats(psDKFEntry);
+
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+#else
+
+/* Stub routines for earlier kernel versions */
+
+PVRSRV_ERROR PVRDKFInit(void)
+{
+	return PVRSRV_OK;
+}
+
+void PVRDKFDeInit(void)
+{
+}
+
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    IMG_UINT32 ui32ConnectionType)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(pid);
+}
+
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(fmt);
+}
+
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData, const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pszDKPName);
+	PVR_UNREFERENCED_PARAMETER(psShowPfn);
+	PVR_UNREFERENCED_PARAMETER(phDkpHandle);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData, PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(hDkpHandle);
+
+	return PVRSRV_OK;
+}
+#endif	/* SUPPORT_LINUX_FDINFO */
diff --git a/drivers/gpu/drm/img-rogue/dkf_server.h b/drivers/gpu/drm/img-rogue/dkf_server.h
new file mode 100644
index 000000000000..304d9bc71b60
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/dkf_server.h
@@ -0,0 +1,103 @@
+/**************************************************************************/ /*!
+@File           dkf_server.h
+@Title          Functions for supporting the DRM Key Framework
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(DKF_SERVER_H)
+#define DKF_SERVER_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(SUPPORT_LINUX_FDINFO)
+
+#include <drm/drm_print.h>
+typedef void (DKF_VPRINTF_FUNC)(struct drm_printer *p, const char *fmt, va_list *va) __printf(2, 0);
+
+#else /* !defined(SUPPORT_LINUX_FDINFO) */
+typedef void (DKF_VPRINTF_FUNC)(void *p, const char *fmt, ...) __printf(2, 3);
+
+#endif	/* defined(SUPPORT_LINUX_FDINFO) */
+struct _PVRSRV_DEVICE_NODE_;
+
+typedef IMG_UINT32 DKF_CONNECTION_FLAGS;
+
+#define DKF_CONNECTION_FLAG_SYNC        BIT(0)
+#define DKF_CONNECTION_FLAG_SERVICES    BIT(1)
+
+#define DKF_CONNECTION_FLAG_INVALID     IMG_UINT32_C(0)
+
+/*! @Function PVRDKFTraverse
+ *
+ * @Description
+ * Outputs the DKP data associated with the given device node's
+ * framework entries.
+ *
+ * @Input   pfnPrint            The print function callback to be used to output.
+ * @Input   pvArg               Print function first argument.
+ * @Input   psDevNode           Device node associated with fdinfo owner.
+ * @Input   pid                 Process ID of the process owning the FD the fdinfo
+ *                              file relates to.
+ * @Input   ui32ConnectionType  A value indicating the PVR connection type
+ *                              (sync or services).
+ */
+void PVRDKFTraverse(DKF_VPRINTF_FUNC *pfnPrint,
+                    void *pvArg,
+                    struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                    IMG_PID pid,
+                    DKF_CONNECTION_FLAGS ui32ConnectionType);
+
+/* @Function PVRDKFInit
+ *
+ * @Description
+ * Initialises the DKF infrastructure for subsequent usage by the PVR system.
+ *
+ * @Returns  PVRSRV_ERROR.
+ */
+PVRSRV_ERROR PVRDKFInit(void);
+
+/* @Function PVRDKFDeInit
+ *
+ * @Description
+ * Removes and frees all associated system-specific DKF meta-data.
+ */
+void PVRDKFDeInit(void);
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/dkp_impl.h b/drivers/gpu/drm/img-rogue/dkp_impl.h
new file mode 100644
index 000000000000..b5f47f8910a2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/dkp_impl.h
@@ -0,0 +1,163 @@
+/**************************************************************************/ /*!
+@File           dkp_impl.h
+@Title          Functions for supporting the DRM Key Provider
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef DKP_IMPL_H
+#define DKP_IMPL_H
+
+typedef IMG_HANDLE PVRDKF_DKP_HANDLE;
+typedef PVRDKF_DKP_HANDLE * PPVRDKF_DKP_HANDLE;
+
+/*! @Function DKP_PFN_SHOW
+ *
+ * @Description
+ *
+ * Describes the function called by the DKF infrastructure to request the DKP
+ * outputs all the mandatory and optional keys it supports. Each
+ * key / value pair should be output one per line using the provided
+ * PVRDPFOutput routine.
+ *
+ * The function should check that the passed 'psDevNode' references the
+ * same device as that associated with the module-specific 'hPrivData' handle.
+ * If it doesn't, no data should be displayed.
+ *
+ * @Input psDevNode   Pointer to device node for which data must be shown.
+ * @Input pid         Process-ID for which data must be shown.
+ * @Input hPrivData   Private data passed to DKF from DKP_Register. The DKP
+ *                     must use this within calls to the DKP_PFN_SHOW to limit
+ *                     the data key/value pairs to only those that are relevant
+ *                     to this instance.
+ */
+typedef void (DKP_PFN_SHOW)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, int pid,
+                            IMG_HANDLE hPrivData);
+
+typedef IMG_UINT32 DKP_CONNECTION_FLAGS;
+
+#define DKP_CONNECTION_FLAG_SYNC        BIT(0)
+#define DKP_CONNECTION_FLAG_SERVICES    BIT(1)
+
+#define DKP_CONNECTION_FLAG_INVALID     IMG_UINT32_C(0)
+#define DKP_CONNECTION_FLAG_ALL (DKP_CONNECTION_FLAG_SYNC | DKP_CONNECTION_FLAG_SERVICES)
+
+/* @Function PVRSRVRegisterDKP
+ *
+ * @Description
+ * Registers a DKP with the specified module's DKF entry.
+ *
+ * @Input   hPrivData               DKP Private data handle that the DKF will pass into
+ *                                  the ShowPFN when called. Can be NULL if there is no
+ *                                  DKP instance private data.
+ * @Input   pszDKPName              Provider name associated with the caller.
+ * @Input   psShowPfn               Function to be used when the fdinfo statistics
+ *                                  are queried.
+ * @Input   ui32Filter              The connection types this DKP should be output
+ *                                  on.
+ * @Output  phDkpHandle             Location to store generated handle created by this
+ *                                  function.
+ *
+ * @Returns  PVRSRV_ERROR
+ */
+PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                               const char *pszDKPName,
+                               DKP_PFN_SHOW *psShowPfn,
+                               DKP_CONNECTION_FLAGS ui32Filter,
+                               PPVRDKF_DKP_HANDLE phDkpHandle);
+
+/* @Function PVRSRVUnRegisterDKP
+ *
+ * @Description
+ * Removes a previously registered key from the device's DKF entry.
+ *
+ * @Input  hPrivData      Private data handle.
+ * @Input  hDkpHandle     Handle to the DKP's registered entry.
+ *                         Obtained from earlier PVRSRV_DKP_Register call.
+ *
+ * @Returns  PVRSRV_ERROR
+ */
+PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData,
+                                 PVRDKF_DKP_HANDLE hDkpHandle);
+
+
+/* @Function PVRDKPOutput
+ * Wrapper function which passes the printf-style arguments to the registered
+ * DKF output function associated with the registered DKP handle.
+ *
+ * @Input   hPrivData   Handle for associated DKP instance producing the output.
+ * @Input   fmt         printf-style format string
+ *
+ */
+void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...) __printf(2, 3);
+
+
+#if !defined(__linux__) || defined(INTEGRITY_OS) || defined(__QNXNTO__)
+
+/* Stub routines follow */
+static inline PVRSRV_ERROR PVRSRVRegisterDKP(IMG_HANDLE hPrivData,
+                                             const char *pszDKPName,
+                                             DKP_PFN_SHOW *psShowPfn,
+                                             DKP_CONNECTION_FLAGS ui32Filter,
+                                             PPVRDKF_DKP_HANDLE phDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pszDKPName);
+	PVR_UNREFERENCED_PARAMETER(psShowPfn);
+	PVR_UNREFERENCED_PARAMETER(phDkpHandle);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+static inline PVRSRV_ERROR PVRSRVUnRegisterDKP(IMG_HANDLE hPrivData,
+                                               PVRDKF_DKP_HANDLE hDkpHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(hDkpHandle);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+static inline void PVRDKPOutput(IMG_HANDLE hPrivData, const char *fmt, ...) __printf(2, 3)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(fmt);
+}
+#endif /* !__linux__ || INTEGRITY_OS || __QNXNTO__ */
+
+#endif /* DKP_IMPL_H */
diff --git a/drivers/gpu/drm/img-rogue/dllist.h b/drivers/gpu/drm/img-rogue/dllist.h
new file mode 100644
index 000000000000..2c237fb8edef
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/dllist.h
@@ -0,0 +1,407 @@
+/*************************************************************************/ /*!
+@File
+@Title          Double linked list header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Double linked list interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DLLIST_H
+#define DLLIST_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+/*!
+	Pointer to a linked list node
+*/
+typedef struct DLLIST_NODE_TAG	*PDLLIST_NODE;
+
+
+/*!
+	Node in a linked list
+*/
+/*
+ * Note: the following structure's size is architecture-dependent and clients
+ * may need to create a mirror of the structure definition if it needs to be
+ * used in a structure shared between host and device.
+ * Consider such clients if any changes are made to this structure.
+ */
+typedef struct DLLIST_NODE_TAG
+{
+	struct DLLIST_NODE_TAG	*psPrevNode;
+	struct DLLIST_NODE_TAG	*psNextNode;
+} DLLIST_NODE;
+
+
+/*!
+	Static initialiser
+*/
+#define DECLARE_DLLIST(n) \
+DLLIST_NODE (n) = {&(n), &(n)}
+
+/*************************************************************************/ /*!
+@Function       dllist_foreach_node
+
+@Description    Walk through all the nodes on the list.
+                Safe against removal of (node).
+
+@Input          list_head              List node to start the operation
+@Input          node                   Current list node
+@Input          next                   Node after the current one
+
+*/
+/*****************************************************************************/
+#define dllist_foreach_node(list_head, node, next)						\
+	for ((node) = (list_head)->psNextNode, (next) = (node)->psNextNode;		\
+		 (node) != (list_head);											\
+		 (node) = (next), (next) = (node)->psNextNode)
+
+#define dllist_foreach_node_backwards(list_head, node, prev)			\
+	for ((node) = (list_head)->psPrevNode, (prev) = (node)->psPrevNode;		\
+		 (node) != (list_head);											\
+		 (node) = (prev), (prev) = (node)->psPrevNode)
+
+
+/*************************************************************************/ /*!
+@Function       dllist_foreach
+
+@Description    Simplification of dllist_foreach_node.
+                Walk through all the nodes on the list.
+                Safe against removal of currently-iterated node.
+
+                Adds utility-macro dllist_cur() to typecast the current node.
+
+@Input          list_head              List node to start the operation
+
+*/
+/*****************************************************************************/
+#define dllist_foreach(list_head)	\
+	for (DLLIST_NODE *DllCurNode = (list_head).psNextNode, *DllNextNode = DllCurNode->psNextNode;		\
+		 DllCurNode != &(list_head);																\
+		 DllCurNode = DllNextNode, DllNextNode = DllCurNode->psNextNode)
+
+#define dllist_foreach_backwards(list_head)	\
+	for (DLLIST_NODE *DllCurNode = (list_head).psPrevNode, *DllPrevNode = DllCurNode->psPrevNode;		\
+		 DllCurNode != &(list_head);																\
+		 DllCurNode = DllPrevNode, DllPrevNode = DllCurNode->psPrevNode)
+
+#define dllist_cur(type, member)	IMG_CONTAINER_OF(DllCurNode, type, member)
+
+/*************************************************************************/ /*!
+@Function       dllist_init
+
+@Description    Initialize a new double linked list
+
+@Input          psListHead             List head Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_init(PDLLIST_NODE psListHead)
+{
+	psListHead->psPrevNode = psListHead;
+	psListHead->psNextNode = psListHead;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_is_empty
+
+@Description    Returns whether the list is empty
+
+@Input          psListHead             List head Node
+
+*/
+/*****************************************************************************/
+static INLINE
+bool dllist_is_empty(const DLLIST_NODE *const psListHead)
+{
+	return (psListHead->psPrevNode == psListHead);
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_add_to_head
+
+@Description    Add psNewNode to head of list psListHead
+
+@Input          psListHead             Head Node
+@Input          psNewNode              New Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_add_to_head(PDLLIST_NODE psListHead, PDLLIST_NODE psNewNode)
+{
+	PDLLIST_NODE psTmp;
+
+	psTmp = psListHead->psNextNode;
+
+	psListHead->psNextNode = psNewNode;
+	psNewNode->psNextNode = psTmp;
+
+	psTmp->psPrevNode = psNewNode;
+	psNewNode->psPrevNode = psListHead;
+}
+
+
+/*************************************************************************/ /*!
+@Function       dllist_add_to_tail
+
+@Description    Add psNewNode to tail of list psListHead
+
+@Input          psListHead             Head Node
+@Input          psNewNode              New Node
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_add_to_tail(PDLLIST_NODE psListHead, PDLLIST_NODE psNewNode)
+{
+	PDLLIST_NODE psTmp;
+
+	psTmp = psListHead->psPrevNode;
+
+	psListHead->psPrevNode = psNewNode;
+	psNewNode->psPrevNode = psTmp;
+
+	psTmp->psNextNode = psNewNode;
+	psNewNode->psNextNode = psListHead;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_node_is_in_list
+
+@Description    Returns true if psNode is in a list
+
+@Input          psNode                 List node
+
+*/
+/*****************************************************************************/
+static INLINE
+bool dllist_node_is_in_list(const DLLIST_NODE *const psNode)
+{
+	return (psNode->psNextNode != NULL);
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_get_next_node
+
+@Description    Returns the list node after psListHead or NULL psListHead is
+                the only element in the list.
+
+@Input          psListHead             List node to start the operation
+
+*/
+/*****************************************************************************/
+static INLINE
+PDLLIST_NODE dllist_get_next_node(PDLLIST_NODE psListHead)
+{
+	if (psListHead->psNextNode == psListHead)
+	{
+		return NULL;
+	}
+	else
+	{
+		return psListHead->psNextNode;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_get_prev_node
+
+@Description    Returns the list node preceding psListHead or NULL if
+                psListHead is the only element in the list.
+
+@Input          psListHead             List node to start the operation
+
+*/
+/*****************************************************************************/
+static INLINE
+PDLLIST_NODE dllist_get_prev_node(PDLLIST_NODE psListHead)
+{
+	if (psListHead->psPrevNode == psListHead)
+	{
+		return NULL;
+	}
+	else
+	{
+		return psListHead->psPrevNode;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_remove_node
+
+@Description    Removes psListNode from the list where it currently belongs
+
+@Input          psListNode             List node to be removed
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_remove_node(PDLLIST_NODE psListNode)
+{
+	psListNode->psNextNode->psPrevNode = psListNode->psPrevNode;
+	psListNode->psPrevNode->psNextNode = psListNode->psNextNode;
+
+	/* Clear the node to show it's not in a list */
+	psListNode->psPrevNode = NULL;
+	psListNode->psNextNode = NULL;
+}
+
+/*************************************************************************/ /*!
+@Function       dllist_replace_head
+
+@Description    Moves the list from psOldHead to psNewHead
+
+@Input          psOldHead              List node to be replaced. Will become a
+                                       head node of an empty list.
+@Input          psNewHead              List node to be inserted. Must be an
+                                       empty list head.
+
+*/
+/*****************************************************************************/
+static INLINE
+void dllist_replace_head(PDLLIST_NODE psOldHead, PDLLIST_NODE psNewHead)
+{
+	if (dllist_is_empty(psOldHead))
+	{
+		psNewHead->psNextNode = psNewHead;
+		psNewHead->psPrevNode = psNewHead;
+	}
+	else
+	{
+		/* Change the neighbouring nodes */
+		psOldHead->psNextNode->psPrevNode = psNewHead;
+		psOldHead->psPrevNode->psNextNode = psNewHead;
+
+		/* Copy the old data to the new node */
+		psNewHead->psNextNode = psOldHead->psNextNode;
+		psNewHead->psPrevNode = psOldHead->psPrevNode;
+
+		/* Remove links to the previous list */
+		psOldHead->psNextNode = psOldHead;
+		psOldHead->psPrevNode = psOldHead;
+	}
+}
+
+/**************************************************************************/ /*!
+@Function       dllist_insert_list_at_head
+
+@Description    Inserts psInHead list into the head of the psOutHead list.
+                After this operation psOutHead will contain psInHead at the
+                head of the list and the remaining elements that were
+                already in psOutHead will be places after the psInList (so
+                at a tail of the original list).
+
+@Input          psOutHead       List node psInHead will be inserted to.
+@Input          psInHead        List node to be inserted to psOutHead.
+                                After this operation this becomes an empty list.
+*/ /***************************************************************************/
+static INLINE
+void dllist_insert_list_at_head(PDLLIST_NODE psOutHead, PDLLIST_NODE psInHead)
+{
+	PDLLIST_NODE psInHeadNextNode = psInHead->psNextNode;
+	PDLLIST_NODE psOutHeadNextNode = psOutHead->psNextNode;
+
+	if (!dllist_is_empty(psInHead))
+	{
+		psOutHead->psNextNode = psInHeadNextNode;
+		psInHeadNextNode->psPrevNode = psOutHead;
+
+		psInHead->psPrevNode->psNextNode = psOutHeadNextNode;
+		psOutHeadNextNode->psPrevNode = psInHead->psPrevNode;
+
+		dllist_init(psInHead);
+	}
+ }
+
+/*************************************************************************/ /*!
+@Description    Pointer to a dllist comparison callback function.
+@Input          psNode  Pointer to a node in a dllist.
+@Input          psNext  Pointer to psNode's next neighbour.
+*/ /**************************************************************************/
+typedef bool (*DLLIST_CMP_CB)(const DLLIST_NODE *psNode, const DLLIST_NODE *psNext);
+
+/*************************************************************************/ /*!
+@Function       dllist_sort
+
+@Description    Insert-sorts the List in place
+                The cmpr function passes the current and next node,
+                From which the user writes the function responsible
+                for choosing to swap order or not.
+                The function returns true if a swap is required
+
+@Input          psListHead              List Head to be sorted.
+
+@Input          cmpr                    Function pointer to use for sorting
+
+*/
+/*****************************************************************************/
+static INLINE void dllist_sort(PDLLIST_NODE psListHead,
+                        DLLIST_CMP_CB cmpr)
+{
+	DLLIST_NODE *node, *next;
+	DLLIST_NODE sTempHead;
+
+	dllist_init(&sTempHead);
+
+	dllist_foreach_node(psListHead, node, next)
+	{
+		dllist_remove_node(node);
+		dllist_add_to_head(&sTempHead, node);
+	}
+
+	while (!dllist_is_empty(&sTempHead))
+	{
+		DLLIST_NODE *psSmallestNode = NULL;
+
+		dllist_foreach_node(&sTempHead, node, next)
+		{
+			if (!psSmallestNode || cmpr(psSmallestNode, node))
+			{
+				psSmallestNode = node;
+			}
+		}
+
+		dllist_remove_node(psSmallestNode);
+		dllist_add_to_tail(psListHead, psSmallestNode);
+	}
+}
+
+#endif /* DLLIST_H */
diff --git a/drivers/gpu/drm/img-rogue/dma_km.h b/drivers/gpu/drm/img-rogue/dma_km.h
new file mode 100644
index 000000000000..9bf633dfd98f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/dma_km.h
@@ -0,0 +1,85 @@
+/*************************************************************************/ /*!
+@File           dma_km.h
+@Title          DMA transfer module header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_KM_H
+#define DMA_KM_H
+
+#if defined(__linux__)
+#include <linux/version.h>
+#else
+#define KERNEL_VERSION
+#endif
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "pmr.h"
+#include "pvrsrv_sync_km.h"
+#include "connection_server.h"
+
+PVRSRV_ERROR DmaDeviceParams(CONNECTION_DATA *psConnection,
+							 PVRSRV_DEVICE_NODE *psDevNode,
+							 IMG_UINT32 *ui32DmaBuffAlign,
+							 IMG_UINT32 *ui32DmaTransferMult);
+
+PVRSRV_ERROR DmaSparseMappingTable(PMR *psPMR,
+								   IMG_DEVMEM_OFFSET_T uiOffset,
+								   IMG_UINT32 ui32SizeInPages,
+								   IMG_BOOL *pbTable);
+
+PVRSRV_ERROR DmaTransfer(CONNECTION_DATA *psConnection,
+		    PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 uiNumDMAs,
+			PMR** ppsPMR,
+			IMG_UINT64 *puiAddress,
+			IMG_DEVMEM_OFFSET_T *puiOffset,
+			IMG_DEVMEM_SIZE_T *puiSize,
+			IMG_UINT32 uiFlags,
+			PVRSRV_TIMELINE iUpdateTimeline);
+
+PVRSRV_ERROR PVRSRVInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode,
+								 CONNECTION_DATA *psConnectionData);
+void PVRSRVDeInitialiseDMA(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   CONNECTION_DATA *psConnectionData);
+
+#endif /* DMA_KM_H */
diff --git a/drivers/gpu/drm/img-rogue/dma_support.c b/drivers/gpu/drm/img-rogue/dma_support.c
new file mode 100644
index 000000000000..c561d0622d5f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/dma_support.c
@@ -0,0 +1,523 @@
+/*************************************************************************/ /*!
+@File           dma_support.c
+@Title          System DMA support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a contiguous memory allocator (i.e. DMA allocator);
+                APIs are used for allocation/ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/mm.h>
+#include <asm/page.h>
+#include <linux/device.h>
+#include <linux/highmem.h>
+#include <linux/vmalloc.h>
+#include <linux/dma-mapping.h>
+#include <asm-generic/getorder.h>
+
+#include "allocmem.h"
+#include "dma_support.h"
+#include "pvr_vmap.h"
+#include "kernel_compatibility.h"
+
+#define DMA_MAX_IOREMAP_ENTRIES 2
+static IMG_BOOL gbEnableDmaIoRemapping = IMG_FALSE;
+static DMA_ALLOC gsDmaIoRemapArray[DMA_MAX_IOREMAP_ENTRIES] = {{0}};
+
+static void*
+SysDmaAcquireKernelAddress(struct page *psPage, IMG_UINT64 ui64Size, DMA_ALLOC *psDmaAlloc)
+{
+	IMG_BOOL bPageByPage = IMG_TRUE;
+	IMG_UINT32 uiIdx;
+	void *pvVirtAddr = NULL;
+	IMG_UINT32 ui32PgCount = (IMG_UINT32)(ui64Size >> OSGetPageShift());
+	PVRSRV_DEVICE_NODE *psDevNode = OSAllocZMemNoStats(sizeof(*psDevNode));
+	PVRSRV_DEVICE_CONFIG *psDevConfig = OSAllocZMemNoStats(sizeof(*psDevConfig));
+	struct page **pagearray = OSAllocZMemNoStats(ui32PgCount * sizeof(struct page *));
+	void *pvOSDevice = psDmaAlloc->pvOSDevice;
+#if defined(CONFIG_ARM64)
+	pgprot_t prot = pgprot_writecombine(PAGE_KERNEL);
+#else
+	pgprot_t prot = pgprot_noncached(PAGE_KERNEL);
+#endif
+
+	/* Validate all required dynamic tmp buffer allocations */
+	if (psDevNode == NULL || psDevConfig == NULL || pagearray == NULL)
+	{
+		if (psDevNode)
+		{
+			OSFreeMem(psDevNode);
+		}
+
+		if (psDevConfig)
+		{
+			OSFreeMem(psDevConfig);
+		}
+
+		if (pagearray)
+		{
+			OSFreeMem(pagearray);
+		}
+
+		goto e0;
+	}
+
+	/* Fake psDevNode->psDevConfig->pvOSDevice */
+	psDevConfig->pvOSDevice = pvOSDevice;
+	psDevNode->psDevConfig = psDevConfig;
+
+	/* Evict any page data contents from d-cache */
+	for (uiIdx = 0; uiIdx < ui32PgCount; uiIdx++)
+	{
+		void *pvVirtStart, *pvVirtEnd;
+		IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+
+		/* Prepare array required for vmap */
+		pagearray[uiIdx] = &psPage[uiIdx];
+
+		if (bPageByPage)
+		{
+#if defined(CONFIG_64BIT)
+			bPageByPage = IMG_FALSE;
+
+			pvVirtStart = kmap(&psPage[uiIdx]);
+			pvVirtEnd = pvVirtStart + ui64Size;
+
+			sCPUPhysStart.uiAddr = page_to_phys(&psPage[uiIdx]);
+			sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + ui64Size;
+			/* all pages have a kernel linear address, flush entire range */
+#else
+			pvVirtStart = kmap(&psPage[uiIdx]);
+			pvVirtEnd = pvVirtStart + PAGE_SIZE;
+
+			sCPUPhysStart.uiAddr = page_to_phys(&psPage[uiIdx]);
+			sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+			/* pages might be from HIGHMEM, need to kmap/flush per page */
+#endif
+
+			/* Fallback to range-based d-cache flush */
+			OSCPUCacheInvalidateRangeKM(psDevNode,
+										pvVirtStart, pvVirtEnd,
+										sCPUPhysStart, sCPUPhysEnd);
+
+			kunmap(&psPage[uiIdx]);
+		}
+	}
+
+	/* Remap pages into VMALLOC space */
+	pvVirtAddr = pvr_vmap(pagearray, ui32PgCount, VM_MAP, prot);
+	psDmaAlloc->PageProps = prot;
+
+	/* Clean-up tmp buffers */
+	OSFreeMem(psDevConfig);
+	OSFreeMem(psDevNode);
+	OSFreeMem(pagearray);
+
+e0:
+	return pvVirtAddr;
+}
+
+static void SysDmaReleaseKernelAddress(void *pvVirtAddr, IMG_UINT64 ui64Size, pgprot_t pgprot)
+{
+	pvr_vunmap(pvVirtAddr, ui64Size >> OSGetPageShift(), pgprot);
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaAllocMem
+
+ @Description		Allocates physically contiguous memory
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct device *psDev;
+	struct page *psPage;
+	size_t uiSize;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->hHandle ||
+		psDmaAlloc->pvVirtAddr ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->sBusAddr.uiAddr ||
+		psDmaAlloc->pvOSDevice == NULL)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+	psDev = (struct device *)psDmaAlloc->pvOSDevice;
+
+	psDmaAlloc->hHandle = dma_alloc_coherent(psDev, uiSize, (dma_addr_t *)&psDmaAlloc->sBusAddr.uiAddr, GFP_KERNEL);
+
+	if (psDmaAlloc->hHandle)
+	{
+		psDmaAlloc->pvVirtAddr = psDmaAlloc->hHandle;
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"Allocated DMA buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
+				psDmaAlloc->pvVirtAddr,
+				psDmaAlloc->sBusAddr.uiAddr,
+				uiSize));
+	}
+	else if ((psPage = alloc_pages(GFP_KERNEL, get_order(uiSize))))
+	{
+		psDmaAlloc->sBusAddr.uiAddr = dma_map_page(psDev, psPage, 0, uiSize, DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(psDev, psDmaAlloc->sBusAddr.uiAddr))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"dma_map_page() failed, page 0x%p order %d",
+					psPage,
+					get_order(uiSize)));
+			__free_pages(psPage, get_order(uiSize));
+			goto e0;
+		}
+		psDmaAlloc->psPage = psPage;
+
+		psDmaAlloc->pvVirtAddr = SysDmaAcquireKernelAddress(psPage, uiSize, psDmaAlloc);
+		if (! psDmaAlloc->pvVirtAddr)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"SysDmaAcquireKernelAddress() failed, page 0x%p order %d",
+					psPage,
+					get_order(uiSize)));
+			dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
+			__free_pages(psPage, get_order(uiSize));
+			goto e0;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"Allocated contiguous buffer V:0x%p P:0x%llx S:0x"IMG_SIZE_FMTSPECX,
+				psDmaAlloc->pvVirtAddr,
+				psDmaAlloc->sBusAddr.uiAddr,
+				uiSize));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,	"Unable to allocate contiguous buffer, size: 0x"IMG_SIZE_FMTSPECX, uiSize));
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+	}
+
+e0:
+	PVR_LOG_RETURN_IF_FALSE((psDmaAlloc->pvVirtAddr), "DMA/CMA allocation failed", PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES);
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaFreeMem
+
+ @Description		Free physically contiguous memory
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psDmaAlloc)
+{
+	size_t uiSize;
+	struct device *psDev;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+	psDev = (struct device *)psDmaAlloc->pvOSDevice;
+
+	if (psDmaAlloc->pvVirtAddr != psDmaAlloc->hHandle)
+	{
+		SysDmaReleaseKernelAddress(psDmaAlloc->pvVirtAddr, uiSize, psDmaAlloc->PageProps);
+	}
+
+	if (! psDmaAlloc->hHandle)
+	{
+		struct page *psPage;
+		dma_unmap_page(psDev, psDmaAlloc->sBusAddr.uiAddr, uiSize, DMA_BIDIRECTIONAL);
+		psPage = psDmaAlloc->psPage;
+		__free_pages(psPage, get_order(uiSize));
+		return;
+	}
+
+	dma_free_coherent(psDev, uiSize, psDmaAlloc->hHandle, (dma_addr_t )psDmaAlloc->sBusAddr.uiAddr);
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaRegisterForIoRemapping
+
+ @Description		Registers DMA_ALLOC for manual I/O remapping
+
+ @Return			PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+ ******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	size_t uiSize;
+	IMG_UINT32 ui32Idx;
+	IMG_BOOL bTabEntryFound = IMG_TRUE;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TOO_FEW_BUFFERS;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		/* Check if an I/O remap entry exists for remapping */
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == NULL)
+		{
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == 0);
+			PVR_ASSERT(gsDmaIoRemapArray[ui32Idx].ui64Size == 0);
+			break;
+		}
+	}
+
+	if (ui32Idx >= DMA_MAX_IOREMAP_ENTRIES)
+	{
+		bTabEntryFound = IMG_FALSE;
+	}
+
+	if (bTabEntryFound)
+	{
+		IMG_BOOL bSameVAddr, bSamePAddr, bSameSize;
+
+		bSamePAddr = gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr == psDmaAlloc->sBusAddr.uiAddr;
+		bSameVAddr = gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr;
+		bSameSize = gsDmaIoRemapArray[ui32Idx].ui64Size == uiSize;
+
+		if (bSameVAddr)
+		{
+			if (bSamePAddr && bSameSize)
+			{
+				eError = PVRSRV_OK;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_ALREADY_EXISTS;
+			}
+		}
+		else
+		{
+			PVR_ASSERT(bSamePAddr == IMG_FALSE);
+
+			gsDmaIoRemapArray[ui32Idx].ui64Size = uiSize;
+			gsDmaIoRemapArray[ui32Idx].sBusAddr = psDmaAlloc->sBusAddr;
+			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = psDmaAlloc->pvVirtAddr;
+
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: register I/O remap: "
+					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
+					psDmaAlloc->pvVirtAddr,
+					psDmaAlloc->sBusAddr.uiAddr,
+					uiSize));
+
+			gbEnableDmaIoRemapping = IMG_TRUE;
+			eError = PVRSRV_OK;
+		}
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDeregisterForIoRemapping
+
+ @Description		Deregisters DMA_ALLOC from manual I/O remapping
+
+ @Return			void
+ ******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psDmaAlloc)
+{
+	__maybe_unused size_t uiSize;
+	IMG_UINT32 ui32Idx;
+
+	if (psDmaAlloc == NULL ||
+		psDmaAlloc->ui64Size == 0 ||
+		psDmaAlloc->pvOSDevice == NULL ||
+		psDmaAlloc->pvVirtAddr == NULL ||
+		psDmaAlloc->sBusAddr.uiAddr == 0)
+	{
+		PVR_LOG_IF_FALSE((IMG_FALSE), "Invalid parameter");
+		return;
+	}
+
+	uiSize = PVR_ALIGN(psDmaAlloc->ui64Size, PAGE_SIZE);
+
+	/* Remove specified entries from list of I/O remap entries */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr == psDmaAlloc->pvVirtAddr)
+		{
+			gsDmaIoRemapArray[ui32Idx].sBusAddr.uiAddr = 0;
+			gsDmaIoRemapArray[ui32Idx].pvVirtAddr = NULL;
+			gsDmaIoRemapArray[ui32Idx].ui64Size = 0;
+
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: deregister I/O remap: "
+					"VA: 0x%p, PA: 0x%llx, Size: 0x"IMG_SIZE_FMTSPECX,
+					psDmaAlloc->pvVirtAddr,
+					psDmaAlloc->sBusAddr.uiAddr,
+					uiSize));
+
+			break;
+		}
+	}
+
+	/* Check if no other I/O remap entries exists for remapping */
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		if (gsDmaIoRemapArray[ui32Idx].pvVirtAddr != NULL)
+		{
+			break;
+		}
+	}
+
+	if (ui32Idx == DMA_MAX_IOREMAP_ENTRIES)
+	{
+		/* No entries found so disable remapping */
+		gbEnableDmaIoRemapping = IMG_FALSE;
+	}
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaDevPAddrToCpuVAddr
+
+ @Description		Maps a DMA_ALLOC physical address to CPU virtual address
+
+ @Return			IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+ ******************************************************************************/
+IMG_CPU_VIRTADDR SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size)
+{
+	IMG_CPU_VIRTADDR pvDMAVirtAddr = NULL;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return pvDMAVirtAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->sBusAddr.uiAddr && uiAddr >= psHeapDmaAlloc->sBusAddr.uiAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = uiAddr - psHeapDmaAlloc->sBusAddr.uiAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				PVR_ASSERT((uiOffset+ui64Size-1) < uiSpan);
+				pvDMAVirtAddr = psHeapDmaAlloc->pvVirtAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: PA: 0x%llx => VA: 0x%p",
+					uiAddr, pvDMAVirtAddr));
+
+				break;
+			}
+		}
+	}
+
+	return pvDMAVirtAddr;
+}
+
+/*!
+******************************************************************************
+ @Function			SysDmaCpuVAddrToDevPAddr
+
+ @Description		Maps a DMA_ALLOC CPU virtual address to physical address
+
+ @Return			Non-zero value on success. Otherwise, a 0
+ ******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr)
+{
+	IMG_UINT64 uiAddr = 0;
+	DMA_ALLOC *psHeapDmaAlloc;
+	IMG_UINT32 ui32Idx;
+
+	if (gbEnableDmaIoRemapping == IMG_FALSE)
+	{
+		return uiAddr;
+	}
+
+	for (ui32Idx = 0; ui32Idx < DMA_MAX_IOREMAP_ENTRIES; ++ui32Idx)
+	{
+		psHeapDmaAlloc = &gsDmaIoRemapArray[ui32Idx];
+		if (psHeapDmaAlloc->pvVirtAddr && pvDMAVirtAddr >= psHeapDmaAlloc->pvVirtAddr)
+		{
+			IMG_UINT64 uiSpan = psHeapDmaAlloc->ui64Size;
+			IMG_UINT64 uiOffset = pvDMAVirtAddr - psHeapDmaAlloc->pvVirtAddr;
+
+			if (uiOffset < uiSpan)
+			{
+				uiAddr = psHeapDmaAlloc->sBusAddr.uiAddr + uiOffset;
+
+				PVR_DPF((PVR_DBG_MESSAGE,
+					"DMA: remap: VA: 0x%p => PA: 0x%llx",
+					pvDMAVirtAddr, uiAddr));
+
+				break;
+			}
+		}
+	}
+
+	return uiAddr;
+}
+
+/******************************************************************************
+ End of file (dma_support.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/dma_support.h b/drivers/gpu/drm/img-rogue/dma_support.h
new file mode 100644
index 000000000000..c1d22bddf523
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/dma_support.h
@@ -0,0 +1,117 @@
+/*************************************************************************/ /*!
+@File           dma_support.h
+@Title          Device contiguous memory allocator and I/O re-mapper
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides a contiguous memory allocator API; mainly
+                used for allocating / ioremapping (DMA/PA <-> CPU/VA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef DMA_SUPPORT_H
+#define DMA_SUPPORT_H
+
+#include "osfunc.h"
+#include "pvrsrv.h"
+
+typedef struct _DMA_ALLOC_
+{
+	IMG_UINT64       ui64Size;
+	IMG_CPU_VIRTADDR pvVirtAddr;
+	IMG_DEV_PHYADDR  sBusAddr;
+	IMG_HANDLE       hHandle;
+#if defined(__linux__)
+	struct page      *psPage;
+	pgprot_t         PageProps;
+#endif
+	void             *pvOSDevice;
+} DMA_ALLOC;
+
+/*!
+*******************************************************************************
+ @Function      SysDmaAllocMem
+ @Description   Allocates physically contiguous memory
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaAllocMem(DMA_ALLOC *psDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaFreeMem
+ @Description   Free physically contiguous memory
+ @Return        void
+******************************************************************************/
+void SysDmaFreeMem(DMA_ALLOC *psCmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaRegisterForIoRemapping
+ @Description   Registers DMA_ALLOC for manual I/O remapping
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR SysDmaRegisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDeregisterForIoRemapping
+ @Description   Deregisters DMA_ALLOC from manual I/O remapping
+ @Return        void
+******************************************************************************/
+void SysDmaDeregisterForIoRemapping(DMA_ALLOC *psPhysHeapDmaAlloc);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaDevPAddrToCpuVAddr
+ @Description   Maps a DMA_ALLOC physical address to CPU virtual address
+ @Return        IMG_CPU_VIRTADDR on success. Otherwise, a NULL
+******************************************************************************/
+IMG_CPU_VIRTADDR
+SysDmaDevPAddrToCpuVAddr(IMG_UINT64 uiAddr, IMG_UINT64 ui64Size);
+
+/*!
+*******************************************************************************
+ @Function      SysDmaCpuVAddrToDevPAddr
+ @Description   Maps a DMA_ALLOC CPU virtual address to physical address
+ @Return        Non-zero value on success. Otherwise, a 0
+******************************************************************************/
+IMG_UINT64 SysDmaCpuVAddrToDevPAddr(IMG_CPU_VIRTADDR pvDMAVirtAddr);
+
+#endif /* DMA_SUPPORT_H */
+
+/******************************************************************************
+ End of file (dma_support.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/env_connection.h b/drivers/gpu/drm/img-rogue/env_connection.h
new file mode 100644
index 000000000000..d2e4f9743370
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/env_connection.h
@@ -0,0 +1,96 @@
+/*************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux specific server side connection management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(ENV_CONNECTION_H)
+#define ENV_CONNECTION_H
+
+#include <linux/version.h>
+#include <linux/list.h>
+#include <linux/types.h>
+
+#include "handle.h"
+#include "pvr_debug.h"
+#include "device.h"
+
+#if defined(SUPPORT_ION)
+#include PVR_ANDROID_ION_HEADER
+#include "ion_sys.h"
+#include "allocmem.h"
+#endif
+
+struct drm_file;
+
+typedef struct _ENV_CONNECTION_PRIVATE_DATA_
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	struct drm_file *psDRMFile;
+} ENV_CONNECTION_PRIVATE_DATA;
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#define ION_CLIENT_NAME_SIZE	50
+
+typedef struct _ENV_ION_CONNECTION_DATA_
+{
+	IMG_CHAR azIonClientName[ION_CLIENT_NAME_SIZE];
+	struct ion_device *psIonDev;
+	struct ion_client *psIonClient;
+} ENV_ION_CONNECTION_DATA;
+#endif
+
+typedef struct _ENV_CONNECTION_DATA_
+{
+	pid_t owner;
+
+	PVRSRV_DEVICE_NODE *psDevNode;
+	struct drm_file *psDRMFile;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	void *pvPvrSyncPrivateData;
+#endif
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	ENV_ION_CONNECTION_DATA *psIonData;
+#endif
+} ENV_CONNECTION_DATA;
+
+#endif /* !defined(ENV_CONNECTION_H) */
diff --git a/drivers/gpu/drm/img-rogue/event.c b/drivers/gpu/drm/img-rogue/event.c
new file mode 100644
index 000000000000..98b029c926df
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/event.c
@@ -0,0 +1,516 @@
+/*************************************************************************/ /*!
+@File
+@Title          Event Object
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+
+#include <linux/version.h>
+#include <linux/string.h>
+#include <linux/sched.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,11,0)
+#include <linux/sched/signal.h>
+#endif
+#include <linux/interrupt.h>
+#include <asm/hardirq.h>
+#include <linux/timer.h>
+#include <linux/capability.h>
+#include <linux/freezer.h>
+#include <linux/uaccess.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "event.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "pvr_bridge_k.h"
+
+#include "osfunc.h"
+
+/* Uncomment to enable event object stats that are useful for debugging.
+ * The stats can be gotten at any time (during lifetime of event object)
+ * using OSEventObjectDumpdebugInfo API */
+// #define LINUX_EVENT_OBJECT_STATS
+
+
+typedef struct PVRSRV_LINUX_EVENT_OBJECT_LIST_TAG
+{
+	rwlock_t sLock;
+	/* Counts how many times event object was signalled i.e. how many times
+	 * LinuxEventObjectSignal() was called on a given event object.
+	 * Used for detecting pending signals.
+	 * Note that this is in no way related to OS signals. */
+	atomic_t sEventSignalCount;
+	struct list_head sList;
+} PVRSRV_LINUX_EVENT_OBJECT_LIST;
+
+
+typedef struct PVRSRV_LINUX_EVENT_OBJECT_TAG
+{
+	IMG_UINT32 ui32EventSignalCountPrevious;
+#if defined(DEBUG)
+	IMG_UINT ui32Stats;
+#endif
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	POS_LOCK hLock;
+	IMG_UINT32 ui32ScheduleAvoided;
+	IMG_UINT32 ui32ScheduleCalled;
+	IMG_UINT32 ui32ScheduleSleptFully;
+	IMG_UINT32 ui32ScheduleSleptPartially;
+	IMG_UINT32 ui32ScheduleReturnedImmediately;
+#endif
+	wait_queue_head_t sWait;
+	struct list_head sList;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList;
+} PVRSRV_LINUX_EVENT_OBJECT;
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectListCreate
+
+ @Description
+
+ Linux wait object list creation
+
+ @Output    hOSEventKM : Pointer to the event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectListCreate(IMG_HANDLE *phEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psEvenObjectList;
+
+	psEvenObjectList = OSAllocMem(sizeof(*psEvenObjectList));
+	if (psEvenObjectList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectCreate: failed to allocate memory for event list"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_LIST_HEAD(&psEvenObjectList->sList);
+
+	rwlock_init(&psEvenObjectList->sLock);
+	atomic_set(&psEvenObjectList->sEventSignalCount, 0);
+
+	*phEventObjectList = (IMG_HANDLE *) psEvenObjectList;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectListDestroy
+
+ @Description
+
+ Linux wait object list destruction
+
+ @Input    hOSEventKM : Event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectListDestroy(IMG_HANDLE hEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psEvenObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST *) hEventObjectList;
+
+	if (psEvenObjectList)
+	{
+		if (!list_empty(&psEvenObjectList->sList))
+		{
+			 PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectListDestroy: Event List is not empty"));
+			 return PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+		}
+		OSFreeMem(psEvenObjectList);
+		/*not nulling pointer, copy on stack*/
+	}
+	return PVRSRV_OK;
+}
+
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectDelete
+
+ @Description
+
+ Linux wait object removal
+
+ @Input    hOSEventObject : Event object handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectDelete(IMG_HANDLE hOSEventObject)
+{
+	if (hOSEventObject)
+	{
+		PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)hOSEventObject;
+		PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = psLinuxEventObject->psLinuxEventObjectList;
+
+		write_lock_bh(&psLinuxEventObjectList->sLock);
+		list_del(&psLinuxEventObject->sList);
+		write_unlock_bh(&psLinuxEventObjectList->sLock);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+		OSLockDestroy(psLinuxEventObject->hLock);
+#endif
+
+#if defined(DEBUG)
+//		PVR_DPF((PVR_DBG_MESSAGE, "LinuxEventObjectDelete: Event object waits: %u", psLinuxEventObject->ui32Stats));
+#endif
+
+		OSFreeMem(psLinuxEventObject);
+		/*not nulling pointer, copy on stack*/
+
+		return PVRSRV_OK;
+	}
+	return PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectAdd
+
+ @Description
+
+ Linux wait object addition
+
+ @Input    hOSEventObjectList : Event object list handle
+ @Output   phOSEventObject : Pointer to the event object handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectAdd(IMG_HANDLE hOSEventObjectList, IMG_HANDLE *phOSEventObject)
+ {
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST*)hOSEventObjectList;
+
+	/* allocate completion variable */
+	psLinuxEventObject = OSAllocMem(sizeof(*psLinuxEventObject));
+	if (psLinuxEventObject == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "LinuxEventObjectAdd: failed to allocate memory"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	INIT_LIST_HEAD(&psLinuxEventObject->sList);
+
+	/* Start with the timestamp at which event object was added to the list */
+	psLinuxEventObject->ui32EventSignalCountPrevious = atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&psLinuxEventObject->hLock), "OSLockCreate");
+	psLinuxEventObject->ui32ScheduleAvoided = 0;
+	psLinuxEventObject->ui32ScheduleCalled = 0;
+	psLinuxEventObject->ui32ScheduleSleptFully = 0;
+	psLinuxEventObject->ui32ScheduleSleptPartially = 0;
+	psLinuxEventObject->ui32ScheduleReturnedImmediately = 0;
+#endif
+
+#if defined(DEBUG)
+	psLinuxEventObject->ui32Stats = 0;
+#endif
+	init_waitqueue_head(&psLinuxEventObject->sWait);
+
+	psLinuxEventObject->psLinuxEventObjectList = psLinuxEventObjectList;
+
+	write_lock_bh(&psLinuxEventObjectList->sLock);
+	list_add(&psLinuxEventObject->sList, &psLinuxEventObjectList->sList);
+	write_unlock_bh(&psLinuxEventObjectList->sLock);
+
+	*phOSEventObject = psLinuxEventObject;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectSignal
+
+ @Description
+
+ Linux wait object signaling function
+
+ @Input    hOSEventObjectList : Event object list handle
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectSignal(IMG_HANDLE hOSEventObjectList)
+{
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = (PVRSRV_LINUX_EVENT_OBJECT_LIST*)hOSEventObjectList;
+	struct list_head *psListEntry, *psListEntryTemp, *psList;
+	psList = &psLinuxEventObjectList->sList;
+
+	/* Move the timestamp ahead for this call, so a potential "Wait" from any
+	 * EventObject/s doesn't wait for the signal to occur before returning. Early
+	 * setting/incrementing of timestamp reduces the window where a concurrent
+	 * "Wait" call might block while "this" Signal call is being processed */
+	atomic_inc(&psLinuxEventObjectList->sEventSignalCount);
+
+	read_lock_bh(&psLinuxEventObjectList->sLock);
+	list_for_each_safe(psListEntry, psListEntryTemp, psList)
+	{
+		psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)list_entry(psListEntry, PVRSRV_LINUX_EVENT_OBJECT, sList);
+		wake_up_interruptible(&psLinuxEventObject->sWait);
+	}
+	read_unlock_bh(&psLinuxEventObjectList->sLock);
+
+	return PVRSRV_OK;
+}
+
+static void _TryToFreeze(void)
+{
+	/* if we reach zero it means that all of the threads called try_to_freeze */
+	LinuxBridgeNumActiveKernelThreadsDecrement();
+
+	/* Returns true if the thread was frozen, should we do anything with this
+	* information? What do we return? Which one is the error case? */
+	try_to_freeze();
+
+	LinuxBridgeNumActiveKernelThreadsIncrement();
+}
+
+void LinuxEventObjectDumpDebugInfo(IMG_HANDLE hOSEventObject)
+{
+#ifdef LINUX_EVENT_OBJECT_STATS
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT *)hOSEventObject;
+
+	OSLockAcquire(psLinuxEventObject->hLock);
+	PVR_LOG(("%s: EvObj(%p) schedule: Avoided(%u) Called(%u) ReturnedImmediately(%u) SleptFully(%u) SleptPartially(%u)",
+	         __func__, psLinuxEventObject, psLinuxEventObject->ui32ScheduleAvoided,
+			 psLinuxEventObject->ui32ScheduleCalled, psLinuxEventObject->ui32ScheduleReturnedImmediately,
+			 psLinuxEventObject->ui32ScheduleSleptFully, psLinuxEventObject->ui32ScheduleSleptPartially));
+	OSLockRelease(psLinuxEventObject->hLock);
+#else
+	PVR_LOG(("%s: LINUX_EVENT_OBJECT_STATS disabled!", __func__));
+#endif
+}
+
+/*!
+******************************************************************************
+
+ @Function	LinuxEventObjectWait
+
+ @Description
+
+ Linux wait object routine
+
+ @Input    hOSEventObject : Event object handle
+
+ @Input   ui64Timeoutus : Time out value in usec
+
+ @Return   PVRSRV_ERROR  :  Error code
+
+******************************************************************************/
+PVRSRV_ERROR LinuxEventObjectWait(IMG_HANDLE hOSEventObject,
+                                  IMG_UINT64 ui64Timeoutus,
+                                  IMG_BOOL bFreezable)
+{
+	IMG_UINT32 ui32EventSignalCount;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32Remainder;
+	long timeOutJiffies;
+#ifdef LINUX_EVENT_OBJECT_STATS
+	long totalTimeoutJiffies;
+	IMG_BOOL bScheduleCalled = IMG_FALSE;
+#endif
+
+	DEFINE_WAIT(sWait);
+
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject = (PVRSRV_LINUX_EVENT_OBJECT*)hOSEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList = psLinuxEventObject->psLinuxEventObjectList;
+
+	PVR_ASSERT(psLinuxEventObjectList != NULL);
+
+	/* Check if the driver is good shape */
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* usecs_to_jiffies only takes an uint. So if our timeout is bigger than an
+	 * uint use the msec version. With such a long timeout we really don't need
+	 * the high resolution of usecs. */
+	if (ui64Timeoutus > 0xffffffffULL)
+		timeOutJiffies = msecs_to_jiffies(OSDivide64(ui64Timeoutus, 1000, &ui32Remainder));
+	else
+		timeOutJiffies = usecs_to_jiffies(ui64Timeoutus);
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	totalTimeoutJiffies = timeOutJiffies;
+#endif
+
+	do
+	{
+		prepare_to_wait(&psLinuxEventObject->sWait, &sWait, TASK_INTERRUPTIBLE);
+		ui32EventSignalCount = (IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+		if (psLinuxEventObject->ui32EventSignalCountPrevious != ui32EventSignalCount)
+		{
+			/* There is a pending event signal i.e. LinuxEventObjectSignal()
+			 * was called on the event object since the last time we checked.
+			 * Return without waiting. */
+			break;
+		}
+
+		if (signal_pending(current))
+		{
+			/* There is an OS signal pending so return.
+			 * This allows to kill/interrupt user space processes which
+			 * are waiting on this event object. */
+			break;
+		}
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+		bScheduleCalled = IMG_TRUE;
+#endif
+		timeOutJiffies = schedule_timeout(timeOutJiffies);
+
+		if (bFreezable)
+		{
+			_TryToFreeze();
+		}
+
+#if defined(DEBUG)
+		psLinuxEventObject->ui32Stats++;
+#endif
+
+
+	} while (timeOutJiffies);
+
+	finish_wait(&psLinuxEventObject->sWait, &sWait);
+
+	psLinuxEventObject->ui32EventSignalCountPrevious = ui32EventSignalCount;
+
+#ifdef LINUX_EVENT_OBJECT_STATS
+	OSLockAcquire(psLinuxEventObject->hLock);
+	if (bScheduleCalled)
+	{
+		psLinuxEventObject->ui32ScheduleCalled++;
+		if (totalTimeoutJiffies == timeOutJiffies)
+		{
+			psLinuxEventObject->ui32ScheduleReturnedImmediately++;
+		}
+		else if (timeOutJiffies == 0)
+		{
+			psLinuxEventObject->ui32ScheduleSleptFully++;
+		}
+		else
+		{
+			psLinuxEventObject->ui32ScheduleSleptPartially++;
+		}
+	}
+	else
+	{
+		psLinuxEventObject->ui32ScheduleAvoided++;
+	}
+	OSLockRelease(psLinuxEventObject->hLock);
+#endif
+
+	if (signal_pending(current) && test_tsk_thread_flag(current, TIF_SIGPENDING))
+	{
+		return PVRSRV_ERROR_INTERRUPTED;
+	}
+	else
+	{
+		return timeOutJiffies ? PVRSRV_OK : PVRSRV_ERROR_TIMEOUT;
+	}
+}
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+PVRSRV_ERROR LinuxEventObjectWaitUntilSignalled(IMG_HANDLE hOSEventObject)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	DEFINE_WAIT(sWait);
+
+	PVRSRV_LINUX_EVENT_OBJECT *psLinuxEventObject =
+			(PVRSRV_LINUX_EVENT_OBJECT *) hOSEventObject;
+	PVRSRV_LINUX_EVENT_OBJECT_LIST *psLinuxEventObjectList =
+			psLinuxEventObject->psLinuxEventObjectList;
+
+	/* Check if the driver is in good shape */
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	prepare_to_wait(&psLinuxEventObject->sWait, &sWait, TASK_INTERRUPTIBLE);
+
+	if (psLinuxEventObject->ui32EventSignalCountPrevious !=
+	    (IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount))
+	{
+		/* There is a pending signal, so return without waiting */
+		goto finish;
+	}
+
+	schedule();
+
+	_TryToFreeze();
+
+finish:
+	finish_wait(&psLinuxEventObject->sWait, &sWait);
+
+	psLinuxEventObject->ui32EventSignalCountPrevious =
+			(IMG_UINT32) atomic_read(&psLinuxEventObjectList->sEventSignalCount);
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
diff --git a/drivers/gpu/drm/img-rogue/event.h b/drivers/gpu/drm/img-rogue/event.h
new file mode 100644
index 000000000000..bb378cb9220f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/event.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@File
+@Title          Event Object
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR LinuxEventObjectListCreate(IMG_HANDLE *phEventObjectList);
+PVRSRV_ERROR LinuxEventObjectListDestroy(IMG_HANDLE hEventObjectList);
+PVRSRV_ERROR LinuxEventObjectAdd(IMG_HANDLE hOSEventObjectList, IMG_HANDLE *phOSEventObject);
+PVRSRV_ERROR LinuxEventObjectDelete(IMG_HANDLE hOSEventObject);
+PVRSRV_ERROR LinuxEventObjectSignal(IMG_HANDLE hOSEventObjectList);
+PVRSRV_ERROR LinuxEventObjectWait(IMG_HANDLE hOSEventObject,
+                                  IMG_UINT64 ui64Timeoutus,
+                                  IMG_BOOL bFreezable);
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+PVRSRV_ERROR LinuxEventObjectWaitUntilSignalled(IMG_HANDLE hOSEventObject);
+#endif
+void LinuxEventObjectDumpDebugInfo(IMG_HANDLE hOSEventObject);
diff --git a/drivers/gpu/drm/img-rogue/fwload.c b/drivers/gpu/drm/img-rogue/fwload.c
new file mode 100644
index 000000000000..8c597b3c2e28
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/fwload.c
@@ -0,0 +1,259 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services firmware load and access routines for Linux
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/firmware.h>
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/err.h>
+
+#include "device.h"
+#include "module_common.h"
+#include "fwload.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+
+#if defined(RGX_FW_SIGNED)
+
+#include <linux/verification.h>
+#include <linux/module.h>
+#include <crypto/public_key.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <linux/module_signature.h>
+#else
+#define PKEY_ID_PKCS7 2
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) */
+
+#include "signfw.h"
+#endif /* RGX_FW_SIGNED */
+
+struct OS_FW_IMAGE_t
+{
+	const struct firmware *psFW;
+	size_t                 uSignatureSize;
+};
+
+#if defined(RGX_FW_SIGNED)
+
+static int OSCheckSignature(const struct FirmwareSignatureHeader *psHeader, size_t uSize)
+{
+	if (be32_to_cpu(psHeader->ui32SignatureLen) >= uSize - sizeof(*psHeader))
+	{
+		return -EBADMSG;
+	}
+
+	if (psHeader->ui8IDType != PKEY_ID_PKCS7)
+	{
+		return -ENOPKG;
+	}
+
+	if (psHeader->ui8Algo != 0 || psHeader->ui8HashAlgo != 0 ||
+	    psHeader->ui8SignerLen != 0 || psHeader->ui8KeyIDLen != 0 ||
+	    psHeader->__ui8Padding[0] != 0 || psHeader->__ui8Padding[1] != 0 ||
+	    psHeader->__ui8Padding[2] != 0)
+	{
+		return -EBADMSG;
+	}
+
+	return 0;
+}
+
+bool OSVerifyFirmware(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW        = psFWImage->psFW;
+	const u8              *pui8FWData  = psFW->data;
+	size_t                uFWSize      = psFW->size;
+	uint32_t              ui32MagicLen = sizeof(MODULE_SIG_STRING) - 1;
+	struct FirmwareSignatureHeader sHeader;
+	int                            err;
+
+	if (uFWSize <= ui32MagicLen)
+	{
+		return false;
+	}
+
+	/*
+	 * Linux Kernel's sign-file utility is primarily intended for signing
+	 * modules, and so appends the MODULE_SIG_STRING magic at the end of
+	 * the signature. Only proceed with verification if this magic is found.
+	 */
+	if (memcmp(pui8FWData + uFWSize - ui32MagicLen, MODULE_SIG_STRING, ui32MagicLen) != 0)
+	{
+		return false;
+	}
+
+	uFWSize -= ui32MagicLen;
+	if (uFWSize <= sizeof(sHeader))
+	{
+		return false;
+	}
+
+	/*
+	 * After the magic, a header is placed which informs about the digest /
+	 * crypto algorithm etc. Copy that header and ensure that it has valid
+	 * contents (We only support RSA Crypto, SHA Hash, X509 certificate and
+	 * PKCS#7 signature).
+	 */
+	memcpy(&sHeader, pui8FWData + (uFWSize - sizeof(sHeader)), sizeof(sHeader));
+	if (OSCheckSignature(&sHeader, uFWSize) != 0)
+	{
+		return false;
+	}
+
+	/*
+	 * As all information is now extracted, we can go ahead and ask PKCS
+	 * module to verify the sign.
+	 */
+	uFWSize -= be32_to_cpu(sHeader.ui32SignatureLen) + sizeof(sHeader);
+	err = verify_pkcs7_signature(pui8FWData, uFWSize, pui8FWData + uFWSize,
+				     be32_to_cpu(sHeader.ui32SignatureLen), NULL,
+				     VERIFYING_UNSPECIFIED_SIGNATURE, NULL, NULL);
+	if (err == 0)
+	{
+		psFWImage->uSignatureSize = psFW->size - uFWSize;
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware Successfully Verified",
+						__func__));
+		return true;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Verification Failed (%d)",
+					__func__, err));
+	return false;
+}
+
+#else /* defined(RGX_FW_SIGNED) */
+
+inline bool OSVerifyFirmware(OS_FW_IMAGE *psFWImage)
+{
+	return true;
+}
+
+#endif /* defined(RGX_FW_SIGNED) */
+
+PVRSRV_ERROR
+OSLoadFirmware(PVRSRV_DEVICE_NODE *psDeviceNode, const IMG_CHAR *pszBVNCString,
+               bool (*pfnVerifyFirmware)(OS_FW_IMAGE*), OS_FW_IMAGE **ppsFWImage)
+{
+	const struct firmware *psFW = NULL;
+	OS_FW_IMAGE *psFWImage;
+	IMG_INT32    res;
+	PVRSRV_ERROR eError;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0))
+	res = firmware_request_nowarn(&psFW, pszBVNCString, psDeviceNode->psDevConfig->pvOSDevice);
+#else
+	res = request_firmware(&psFW, pszBVNCString, psDeviceNode->psDevConfig->pvOSDevice);
+#endif
+	if (res != 0)
+	{
+		release_firmware(psFW);
+		if (res == -ENOENT)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: requested firmware('%s') not found (%d)",
+							__func__, pszBVNCString, res));
+			eError = PVRSRV_ERROR_NOT_FOUND;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: requested firmware('%s') not ready (%d)",
+							__func__, pszBVNCString, res));
+			eError = PVRSRV_ERROR_NOT_READY;
+		}
+		goto err_exit;
+	}
+
+	psFWImage = OSAllocZMem(sizeof(*psFWImage));
+	if (psFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: OSAllocZMem('%s') failed.",
+						__func__, pszBVNCString));
+
+		release_firmware(psFW);
+		eError =  PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_exit;
+	}
+
+	psFWImage->psFW = psFW;
+	if (pfnVerifyFirmware != NULL && !pfnVerifyFirmware(psFWImage))
+	{
+		release_firmware(psFW);
+		OSFreeMem(psFWImage);
+		eError = PVRSRV_ERROR_NOT_AUTHENTICATED;
+		goto err_exit;
+	}
+
+	*ppsFWImage = psFWImage;
+	return PVRSRV_OK;
+
+err_exit:
+	*ppsFWImage = NULL;
+	return eError;
+}
+
+void
+OSUnloadFirmware(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+
+	release_firmware(psFW);
+	OSFreeMem(psFWImage);
+}
+
+size_t
+OSFirmwareSize(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+	return psFW->size - psFWImage->uSignatureSize;
+}
+
+const void *
+OSFirmwareData(OS_FW_IMAGE *psFWImage)
+{
+	const struct firmware *psFW = psFWImage->psFW;
+
+	return psFW->data;
+}
+
+/******************************************************************************
+ End of file (fwload.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/fwload.h b/drivers/gpu/drm/img-rogue/fwload.h
new file mode 100644
index 000000000000..08e7f533b666
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/fwload.h
@@ -0,0 +1,158 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services RGX OS Interface for loading the firmware
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines the OS interface through which the RGX
+                device initialisation code in the kernel/server will obtain
+                the RGX firmware binary image. The API is used during the
+                initialisation of an RGX device via the
+                PVRSRVCommonDeviceInitialise()
+                call sequence.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef FWLOAD_H
+#define FWLOAD_H
+
+#include "img_defs.h"
+#include "device_connection.h"
+#include "device.h"
+
+/*! Opaque type handle defined and known to the OS layer implementation of this
+ * fwload.h OS API. This private data is allocated in the implementation of
+ * OSLoadFirmware() and contains whatever data and information needed to be
+ * able to acquire and return the firmware binary image to the Services
+ * kernel/server during initialisation.
+ * It is no longer required and may be freed when OSUnloadFirmware() is called.
+ */
+typedef struct OS_FW_IMAGE_t OS_FW_IMAGE;
+
+#if defined(__linux__)
+
+bool OSVerifyFirmware(OS_FW_IMAGE* psFWImage);
+
+#endif
+
+/*************************************************************************/ /*!
+@Function     OSLoadFirmware
+@Description  The OS implementation must load or acquire the firmware (FW)
+              image binary needed by the driver stack.
+              A handle to the common layer device node is given to identify
+              which device instance in the system is being initialised. The
+              BVNC string is also supplied so that the implementation knows
+              which FW image to retrieve since each FW image only supports one
+              GPU type/revision.
+              The calling server code supports multiple GPU types and revisions
+              and will detect the specific GPU type and revision before calling
+              this API. It will also have runtime configuration of the VZ mode,
+              hence this API must be able to retrieve different FW binary
+              images based on the pszBVNCString given. The purpose of the end
+              platform/system is key to understand which FW images must be
+              available to the kernel server.
+              On exit the implementation must return a pointer to some private
+              data it uses to hold the FW image information and data. It will
+              be passed onto later API calls by the kernel server code.
+              NULL should be returned if the FW image could not be retrieved.
+              The format of the BVNC string is as follows ([x] denotes
+              optional field):
+                "rgx.fw[.signed].B.V[p].N.C[.vz]"
+              The implementation must first try to load the FW identified
+              by the pszBVpNCString parameter. If this is not available then it
+              should drop back to retrieving the FW identified by the
+              pszBVNCString parameter. The fields in the string are:
+                B, V, N, C are all unsigned integer identifying type/revision.
+                [.signed] is present when RGX_FW_SIGNED=1 is defined in the
+                  server build.
+                [p] denotes a provisional (pre-silicon) GPU configuration.
+                [.vz] is present when the kernel server is loaded on the HOST
+                  of a virtualised platform. See the DriverMode server
+                  AppHint for details.
+
+@Input        psDeviceNode       Device instance identifier.
+@Input        pszBVNCString      Identifier string of the FW image to
+                                 be loaded/acquired in production driver.
+@Input        pfnVerifyFirmware  Callback which checks validity of FW image.
+@Output       ppsFWImage         Ptr to private data on success,
+                                 NULL otherwise.
+@Return       PVRSRV_ERROR       PVRSRV_OK on success,
+                                 PVRSRV_ERROR_NOT_READY if filesystem is not
+                                                        ready/initialised,
+                                 PVRSRV_ERROR_NOT_FOUND if no suitable FW
+                                                        image could be found
+                                 PVRSRV_ERROR_OUT_OF_MEMORY if unable to alloc
+                                                        memory for FW image
+                                 PVRSRV_ERROR_NOT_AUTHENTICATED if FW image
+                                                        cannot be verified.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSLoadFirmware(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            const IMG_CHAR *pszBVNCString,
+                            bool (*pfnVerifyFirmware)(OS_FW_IMAGE*),
+                            OS_FW_IMAGE **ppsFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSFirmwareData
+@Description  This function returns a pointer to the start of the FW image
+              binary data held in memory. It must remain valid until
+              OSUnloadFirmware() is called.
+@Input        psFWImage  Private data opaque handle
+@Return       void*      Ptr to FW binary image to start on GPU.
+*/ /**************************************************************************/
+const void* OSFirmwareData(OS_FW_IMAGE *psFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSFirmwareSize
+@Description  This function returns the size of the FW image binary data.
+@Input        psFWImage  Private data opaque handle
+@Return       size_t     Size in bytes of the firmware binary image
+*/ /**************************************************************************/
+size_t OSFirmwareSize(OS_FW_IMAGE *psFWImage);
+
+/*************************************************************************/ /*!
+@Function     OSUnloadFirmware
+@Description  This is called when the server has completed firmware
+              initialisation and no longer needs the private data, possibly
+              allocated by OSLoadFirmware().
+@Input        psFWImage  Private data opaque handle
+*/ /**************************************************************************/
+void OSUnloadFirmware(OS_FW_IMAGE *psFWImage);
+
+#endif /* FWLOAD_H */
+
+/******************************************************************************
+ End of file (fwload.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/fwtrace_string.h b/drivers/gpu/drm/img-rogue/fwtrace_string.h
new file mode 100644
index 000000000000..a2ab95c3a815
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/fwtrace_string.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File           fwtrace_string.h
+@Title          RGX Firmware trace strings for KM
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       Generic
+@Description    This file defines SFs tuple.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef KM_TRACE_STRING_H
+#define KM_TRACE_STRING_H
+
+#include "rgx_fwif_sf.h"
+
+extern const RGXKM_STID_FMT SFs[];
+extern const IMG_UINT32 g_ui32SFsCount;
+
+#endif /* KM_TRACE_STRING_H */
diff --git a/drivers/gpu/drm/img-rogue/handle.c b/drivers/gpu/drm/img-rogue/handle.c
new file mode 100644
index 000000000000..a7fec73f7d21
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/handle.c
@@ -0,0 +1,2501 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Handle Manager
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide resource handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+/* See handle.h for a description of the handle API. */
+
+/*
+ * The implementation supports movable handle structures, allowing the address
+ * of a handle structure to change without having to fix up pointers in
+ * any of the handle structures. For example, the linked list mechanism
+ * used to link subhandles together uses handle array indices rather than
+ * pointers to the structures themselves.
+ */
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "handle.h"
+#include "handle_impl.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "lock.h"
+#include "connection_server.h"
+
+#define	HANDLE_HASH_TAB_INIT_SIZE 32
+#define HANDLE_PROC_HANDLE_HASH_INIT_SIZE 10
+
+#define	TEST_FLAG(v, f) BITMASK_HAS(v, f)
+#define	TEST_ALLOC_FLAG(psHandleData, f) BITMASK_HAS((psHandleData)->eFlag, f)
+
+
+/* Linked list structure. Used for both the list head and list items */
+typedef struct _HANDLE_LIST_
+{
+	IMG_HANDLE hPrev;
+	IMG_HANDLE hNext;
+	IMG_HANDLE hParent;
+} HANDLE_LIST;
+
+typedef struct _HANDLE_DATA_
+{
+	/* The handle that represents this structure */
+	IMG_HANDLE hHandle;
+
+	/* Handle type */
+	PVRSRV_HANDLE_TYPE eType;
+
+	/* Flags specified when the handle was allocated */
+	PVRSRV_HANDLE_ALLOC_FLAG eFlag;
+
+	/* Pointer to the data that the handle represents */
+	void *pvData;
+
+	/*
+	 * Callback specified at handle allocation time to
+	 * release/destroy/free the data represented by the
+	 * handle when it's reference count reaches 0. This
+	 * should always be NULL for subhandles.
+	 */
+	PFN_HANDLE_RELEASE pfnReleaseData;
+
+	/* List head for subhandles of this handle */
+	HANDLE_LIST sChildren;
+
+	/* List entry for sibling subhandles */
+	HANDLE_LIST sSiblings;
+
+	/* Reference count of lookups made. It helps track which resources are in
+	 * use in concurrent bridge calls. */
+	IMG_INT32 iLookupCount;
+	/* State of a handle. If the handle was already destroyed this is false.
+	 * If this is false and iLookupCount is 0 the pfnReleaseData callback is
+	 * called on the handle. */
+	IMG_BOOL bCanLookup;
+
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	/* Store the handle base used for this handle, so we
+	 * can later access the handle base lock (or check if
+	 * it has been already acquired)
+	 */
+	PVRSRV_HANDLE_BASE *psBase;
+#endif
+
+} HANDLE_DATA;
+
+struct _HANDLE_BASE_
+{
+	/* Pointer to a handle implementations base structure */
+	HANDLE_IMPL_BASE *psImplBase;
+
+	/*
+	 * Pointer to handle hash table.
+	 * The hash table is used to do reverse lookups, converting data
+	 * pointers to handles.
+	 */
+	HASH_TABLE *psHashTab;
+
+	/* Type specific (connection/global/process) Lock handle */
+	POS_LOCK hLock;
+
+	/* Can be connection, process, global */
+	PVRSRV_HANDLE_BASE_TYPE eType;
+};
+
+/*
+ * The key for the handle hash table is an array of three elements, the
+ * pointer to the resource, the resource type and the parent handle (or
+ * NULL if there is no parent). The eHandKey enumeration gives the
+ * array indices of the elements making up the key.
+ */
+enum eHandKey
+{
+	HAND_KEY_DATA = 0,
+	HAND_KEY_TYPE,
+	HAND_KEY_PARENT,
+	HAND_KEY_LEN		/* Must be last item in list */
+};
+
+/* HAND_KEY is the type of the hash table key */
+typedef uintptr_t HAND_KEY[HAND_KEY_LEN];
+
+typedef struct FREE_HANDLE_DATA_TAG
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	PVRSRV_HANDLE_TYPE eHandleFreeType;
+	/* timing data (ns) to release bridge lock upon the deadline */
+	IMG_UINT64 ui64TimeStart;
+	IMG_UINT64 ui64MaxBridgeTime;
+} FREE_HANDLE_DATA;
+
+typedef struct FREE_KERNEL_HANDLE_DATA_TAG
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	HANDLE_DATA *psProcessHandleData;
+	IMG_HANDLE hKernelHandle;
+} FREE_KERNEL_HANDLE_DATA;
+
+/* Stores a pointer to the function table of the handle back-end in use */
+static HANDLE_IMPL_FUNCTAB const *gpsHandleFuncs;
+
+static POS_LOCK gKernelHandleLock;
+static IMG_BOOL gbLockInitialised = IMG_FALSE;
+/* Pointer to process handle base currently being freed */
+static PVRSRV_HANDLE_BASE *g_psProcessHandleBaseBeingFreed;
+/* Lock for the process handle base table */
+static POS_LOCK g_hProcessHandleBaseLock;
+/* Hash table with process handle bases */
+static HASH_TABLE *g_psProcessHandleBaseTable;
+
+void LockHandle(PVRSRV_HANDLE_BASE *psBase)
+{
+	OSLockAcquire(psBase->hLock);
+}
+
+void UnlockHandle(PVRSRV_HANDLE_BASE *psBase)
+{
+	OSLockRelease(psBase->hLock);
+}
+
+/*
+ * Kernel handle base structure. This is used for handles that are not
+ * allocated on behalf of a particular process.
+ */
+PVRSRV_HANDLE_BASE *gpsKernelHandleBase = NULL;
+
+/* Increase the lookup reference count on the given handle.
+ * The handle lock must already be acquired.
+ * Returns: the reference count after the increment
+ */
+static inline IMG_UINT32 HandleGet(HANDLE_DATA *psHandleData)
+{
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	if (!OSLockIsLocked(psHandleData->psBase->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle lock is not locked", __func__));
+		OSDumpStack();
+	}
+#endif
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = %u, iLookupCount %d -> %d",
+	        __func__, psHandleData->bCanLookup, psHandleData->iLookupCount,
+	        psHandleData->iLookupCount + 1));
+#endif /* DEBUG_REFCNT */
+
+	PVR_ASSERT(psHandleData->bCanLookup);
+
+	return ++psHandleData->iLookupCount;
+}
+
+/* Decrease the lookup reference count on the given handle.
+ * The handle lock must already be acquired.
+ * Returns: the reference count after the decrement
+ */
+static inline IMG_UINT32 HandlePut(HANDLE_DATA *psHandleData)
+{
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	if (!OSLockIsLocked(psHandleData->psBase->hLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle lock is not locked", __func__));
+		OSDumpStack();
+	}
+#endif
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = %u, iLookupCount %d -> %d",
+	        __func__, psHandleData->bCanLookup, psHandleData->iLookupCount,
+	        psHandleData->iLookupCount - 1));
+#endif /* DEBUG_REFCNT */
+
+	/* psHandleData->bCanLookup can be false at this point */
+	PVR_ASSERT(psHandleData->iLookupCount > 0);
+
+	return --psHandleData->iLookupCount;
+}
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+static const IMG_CHAR *HandleTypeToString(PVRSRV_HANDLE_TYPE eType)
+{
+	#define HANDLETYPE(x) \
+			case PVRSRV_HANDLE_TYPE_##x: \
+				return #x;
+	switch (eType)
+	{
+		#include "handle_types.h"
+		#undef HANDLETYPE
+
+		default:
+			return "INVALID";
+	}
+}
+
+static const IMG_CHAR *HandleBaseTypeToString(PVRSRV_HANDLE_BASE_TYPE eType)
+{
+	#define HANDLEBASETYPE(x) \
+			case PVRSRV_HANDLE_BASE_TYPE_##x: \
+				return #x;
+	switch (eType)
+	{
+		HANDLEBASETYPE(CONNECTION);
+		HANDLEBASETYPE(PROCESS);
+		HANDLEBASETYPE(GLOBAL);
+		#undef HANDLEBASETYPE
+
+		default:
+			return "INVALID";
+	}
+}
+#endif
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFree(PVRSRV_HANDLE_BASE *psBase,
+                                                   HANDLE_DATA *psHandleData,
+                                                   IMG_HANDLE hHandle,
+                                                   PVRSRV_HANDLE_TYPE eType);
+
+static PVRSRV_ERROR HandleFreePrivData(PVRSRV_HANDLE_BASE *psBase,
+                                       HANDLE_DATA *psHandleData,
+                                       IMG_HANDLE hHandle,
+                                       PVRSRV_HANDLE_TYPE eType);
+
+static PVRSRV_ERROR HandleFreeDestroy(PVRSRV_HANDLE_BASE *psBase,
+                                      HANDLE_DATA *psHandleData,
+                                      IMG_HANDLE hHandle,
+                                      PVRSRV_HANDLE_TYPE eType);
+
+/*!
+*******************************************************************************
+ @Function      GetHandleData
+ @Description   Get the handle data structure for a given handle
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type or PVRSRV_HANDLE_TYPE_NONE if the handle
+                        type is not to be checked.
+ @Output        ppsHandleData - pointer to a pointer to the handle data struct
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(GetHandleData)
+#endif
+static INLINE
+PVRSRV_ERROR GetHandleData(PVRSRV_HANDLE_BASE *psBase,
+			   HANDLE_DATA **ppsHandleData,
+			   IMG_HANDLE hHandle,
+			   PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError;
+
+	eError = gpsHandleFuncs->pfnGetHandleData(psBase->psImplBase,
+						  hHandle,
+						  (void **)&psHandleData);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/*
+	 * Unless PVRSRV_HANDLE_TYPE_NONE was passed in to this function,
+	 * check handle is of the correct type.
+	 */
+	if (unlikely(eType != PVRSRV_HANDLE_TYPE_NONE && eType != psHandleData->eType))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "GetHandleData: Type mismatch. Lookup request: Handle %p, type: %s (%u) but stored handle is type %s (%u)",
+			 hHandle,
+			 HandleTypeToString(eType),
+			 eType,
+			 HandleTypeToString(psHandleData->eType),
+			 psHandleData->eType));
+		return PVRSRV_ERROR_HANDLE_TYPE_MISMATCH;
+	}
+
+	/* Return the handle structure */
+	*ppsHandleData = psHandleData;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListInit
+ @Description   Initialise a linked list structure embedded in a handle
+                structure.
+ @Input         hHandle - handle containing the linked list structure
+                psList - pointer to linked list structure
+                hParent - parent handle or NULL
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListInit)
+#endif
+static INLINE
+void HandleListInit(IMG_HANDLE hHandle, HANDLE_LIST *psList, IMG_HANDLE hParent)
+{
+	psList->hPrev = hHandle;
+	psList->hNext = hHandle;
+	psList->hParent = hParent;
+}
+
+/*!
+*******************************************************************************
+ @Function      InitParentList
+ @Description   Initialise the children list head in a handle structure.
+                The children are the subhandles of this handle.
+ @Input         psHandleData - pointer to handle data structure
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitParentList)
+#endif
+static INLINE
+void InitParentList(HANDLE_DATA *psHandleData)
+{
+	IMG_HANDLE hParent = psHandleData->hHandle;
+
+	HandleListInit(hParent, &psHandleData->sChildren, hParent);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitChildEntry
+ @Description   Initialise the child list entry in a handle structure. The list
+                entry is used to link together subhandles of a given handle.
+ @Input         psHandleData - pointer to handle data structure
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitChildEntry)
+#endif
+static INLINE
+void InitChildEntry(HANDLE_DATA *psHandleData)
+{
+	HandleListInit(psHandleData->hHandle, &psHandleData->sSiblings, NULL);
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListIsEmpty
+ @Description   Determine whether a given linked list is empty.
+ @Input         hHandle - handle containing the list head
+                psList - pointer to the list head
+ @Return        IMG_TRUE if the list is empty, IMG_FALSE if it isn't.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListIsEmpty)
+#endif
+static INLINE
+IMG_BOOL HandleListIsEmpty(IMG_HANDLE hHandle, HANDLE_LIST *psList) /* Instead of passing in the handle can we not just do (psList->hPrev == psList->hNext) ? IMG_TRUE : IMG_FALSE ??? */
+{
+	IMG_BOOL bIsEmpty;
+
+	bIsEmpty = (IMG_BOOL)(psList->hNext == hHandle);
+
+#ifdef DEBUG
+	{
+		IMG_BOOL bIsEmpty2;
+
+		bIsEmpty2 = (IMG_BOOL)(psList->hPrev == hHandle);
+		PVR_ASSERT(bIsEmpty == bIsEmpty2);
+	}
+#endif
+
+	return bIsEmpty;
+}
+
+#ifdef DEBUG
+/*!
+*******************************************************************************
+ @Function      NoChildren
+ @Description   Determine whether a handle has any subhandles
+ @Input         psHandleData - pointer to handle data structure
+ @Return        IMG_TRUE if the handle has no subhandles, IMG_FALSE if it does.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(NoChildren)
+#endif
+static INLINE
+IMG_BOOL NoChildren(HANDLE_DATA *psHandleData)
+{
+	PVR_ASSERT(psHandleData->sChildren.hParent == psHandleData->hHandle);
+
+	return HandleListIsEmpty(psHandleData->hHandle, &psHandleData->sChildren);
+}
+
+/*!
+*******************************************************************************
+ @Function      NoParent
+ @Description   Determine whether a handle is a subhandle
+ @Input         psHandleData - pointer to handle data structure
+ @Return        IMG_TRUE if the handle is not a subhandle, IMG_FALSE if it is.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(NoParent)
+#endif
+static INLINE
+IMG_BOOL NoParent(HANDLE_DATA *psHandleData)
+{
+	if (HandleListIsEmpty(psHandleData->hHandle, &psHandleData->sSiblings))
+	{
+		PVR_ASSERT(psHandleData->sSiblings.hParent == NULL);
+
+		return IMG_TRUE;
+	}
+
+	PVR_ASSERT(psHandleData->sSiblings.hParent != NULL);
+	return IMG_FALSE;
+}
+#endif /*DEBUG*/
+
+/*!
+*******************************************************************************
+ @Function      ParentHandle
+ @Description   Determine the parent of a handle
+ @Input         psHandleData - pointer to handle data structure
+ @Return        Parent handle, or NULL if the handle is not a subhandle.
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(ParentHandle)
+#endif
+static INLINE
+IMG_HANDLE ParentHandle(HANDLE_DATA *psHandleData)
+{
+	return psHandleData->sSiblings.hParent;
+}
+
+/*
+ * GetHandleListFromHandleAndOffset is used to generate either a
+ * pointer to the subhandle list head, or a pointer to the linked list
+ * structure of an item on a subhandle list.
+ * The list head is itself on the list, but is at a different offset
+ * in the handle structure to the linked list structure for items on
+ * the list. The two linked list structures are differentiated by
+ * the third parameter, containing the parent handle. The parent field
+ * in the list head structure references the handle structure that contains
+ * it. For items on the list, the parent field in the linked list structure
+ * references the parent handle, which will be different from the handle
+ * containing the linked list structure.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(GetHandleListFromHandleAndOffset)
+#endif
+static INLINE
+HANDLE_LIST *GetHandleListFromHandleAndOffset(PVRSRV_HANDLE_BASE *psBase,
+					      IMG_HANDLE hEntry,
+					      IMG_HANDLE hParent,
+					      size_t uiParentOffset,
+					      size_t uiEntryOffset)
+{
+	HANDLE_DATA *psHandleData = NULL;
+
+	PVR_ASSERT(psBase != NULL);
+
+	if (GetHandleData(psBase, &psHandleData, hEntry,
+	                  PVRSRV_HANDLE_TYPE_NONE) != PVRSRV_OK)
+	{
+		return NULL;
+	}
+
+	if (hEntry == hParent)
+	{
+		return (HANDLE_LIST *)IMG_OFFSET_ADDR(psHandleData, uiParentOffset);
+	}
+	else
+	{
+		return (HANDLE_LIST *)IMG_OFFSET_ADDR(psHandleData, uiEntryOffset);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListInsertBefore
+ @Description   Insert a handle before a handle currently on the list.
+ @Input         hEntry - handle to be inserted after
+                psEntry - pointer to handle structure to be inserted after
+                uiParentOffset - offset to list head struct in handle structure
+                hNewEntry - handle to be inserted
+                psNewEntry - pointer to handle structure of item to be inserted
+                uiEntryOffset - offset of list item struct in handle structure
+                hParent - parent handle of hNewEntry
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListInsertBefore)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListInsertBefore(PVRSRV_HANDLE_BASE *psBase,
+				    IMG_HANDLE hEntry,
+				    HANDLE_LIST *psEntry,
+				    size_t uiParentOffset,
+				    IMG_HANDLE hNewEntry,
+				    HANDLE_LIST *psNewEntry,
+				    size_t uiEntryOffset,
+				    IMG_HANDLE hParent)
+{
+	HANDLE_LIST *psPrevEntry;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psEntry != NULL, "psEntry");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNewEntry != NULL, "psNewEntry");
+
+	psPrevEntry = GetHandleListFromHandleAndOffset(psBase,
+						       psEntry->hPrev,
+						       hParent,
+						       uiParentOffset,
+						       uiEntryOffset);
+	if (psPrevEntry == NULL)
+	{
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+
+	PVR_ASSERT(psNewEntry->hParent == NULL);
+	PVR_ASSERT(hEntry == psPrevEntry->hNext);
+
+#if defined(DEBUG)
+	{
+		HANDLE_LIST *psParentList;
+
+		psParentList = GetHandleListFromHandleAndOffset(psBase,
+								hParent,
+								hParent,
+								uiParentOffset,
+								uiParentOffset);
+		PVR_ASSERT(psParentList && psParentList->hParent == hParent);
+	}
+#endif /* defined(DEBUG) */
+
+	psNewEntry->hPrev = psEntry->hPrev;
+	psEntry->hPrev = hNewEntry;
+
+	psNewEntry->hNext = hEntry;
+	psPrevEntry->hNext = hNewEntry;
+
+	psNewEntry->hParent = hParent;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      AdoptChild
+ @Description   Assign a subhandle to a handle
+ @Input         psParentData - pointer to handle structure of parent handle
+                psChildData - pointer to handle structure of child subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(AdoptChild)
+#endif
+static INLINE
+PVRSRV_ERROR AdoptChild(PVRSRV_HANDLE_BASE *psBase,
+			HANDLE_DATA *psParentData,
+			HANDLE_DATA *psChildData)
+{
+	IMG_HANDLE hParent = psParentData->sChildren.hParent;
+
+	PVR_ASSERT(hParent == psParentData->hHandle);
+
+	return HandleListInsertBefore(psBase,
+				      hParent,
+				      &psParentData->sChildren,
+				      offsetof(HANDLE_DATA, sChildren),
+				      psChildData->hHandle,
+				      &psChildData->sSiblings,
+				      offsetof(HANDLE_DATA, sSiblings),
+				      hParent);
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListRemove
+ @Description   Remove a handle from a list
+ @Input         hEntry - handle to be removed
+                psEntry - pointer to handle structure of item to be removed
+                uiEntryOffset - offset of list item struct in handle structure
+                uiParentOffset - offset to list head struct in handle structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListRemove)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListRemove(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE hEntry,
+			      HANDLE_LIST *psEntry,
+			      size_t uiEntryOffset,
+			      size_t uiParentOffset)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psEntry != NULL, "psEntry");
+
+	if (!HandleListIsEmpty(hEntry, psEntry))
+	{
+		HANDLE_LIST *psPrev;
+		HANDLE_LIST *psNext;
+
+		psPrev = GetHandleListFromHandleAndOffset(psBase,
+							  psEntry->hPrev,
+							  psEntry->hParent,
+							  uiParentOffset,
+							  uiEntryOffset);
+		if (psPrev == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		psNext = GetHandleListFromHandleAndOffset(psBase,
+							  psEntry->hNext,
+							  psEntry->hParent,
+							  uiParentOffset,
+							  uiEntryOffset);
+		if (psNext == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		/*
+		 * The list head is on the list, and we don't want to
+		 * remove it.
+		 */
+		PVR_ASSERT(psEntry->hParent != NULL);
+
+		psPrev->hNext = psEntry->hNext;
+		psNext->hPrev = psEntry->hPrev;
+
+		HandleListInit(hEntry, psEntry, NULL);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      UnlinkFromParent
+ @Description   Remove a subhandle from its parents list
+ @Input         psHandleData - pointer to handle data structure of child
+                               subhandle.
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(UnlinkFromParent)
+#endif
+static INLINE
+PVRSRV_ERROR UnlinkFromParent(PVRSRV_HANDLE_BASE *psBase,
+			      HANDLE_DATA *psHandleData)
+{
+	return HandleListRemove(psBase,
+				psHandleData->hHandle,
+				&psHandleData->sSiblings,
+				offsetof(HANDLE_DATA, sSiblings),
+				offsetof(HANDLE_DATA, sChildren));
+}
+
+/*!
+*******************************************************************************
+ @Function      HandleListIterate
+ @Description   Iterate over the items in a list
+ @Input         psHead - pointer to list head
+                uiParentOffset - offset to list head struct in handle structure
+                uiEntryOffset - offset of list item struct in handle structure
+                pfnIterFunc - function to be called for each handle in the list
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(HandleListIterate)
+#endif
+static INLINE
+PVRSRV_ERROR HandleListIterate(PVRSRV_HANDLE_BASE *psBase,
+			       HANDLE_LIST *psHead,
+			       size_t uiParentOffset,
+			       size_t uiEntryOffset,
+			       PVRSRV_ERROR (*pfnIterFunc)(PVRSRV_HANDLE_BASE *, IMG_HANDLE))
+{
+	IMG_HANDLE hHandle = psHead->hNext;
+	IMG_HANDLE hParent = psHead->hParent;
+	IMG_HANDLE hNext;
+
+	PVR_ASSERT(psHead->hParent != NULL);
+
+	/*
+	 * Follow the next chain from the list head until we reach
+	 * the list head again, which signifies the end of the list.
+	 */
+	while (hHandle != hParent)
+	{
+		HANDLE_LIST *psEntry;
+		PVRSRV_ERROR eError;
+
+		psEntry = GetHandleListFromHandleAndOffset(psBase,
+							   hHandle,
+							   hParent,
+							   uiParentOffset,
+							   uiEntryOffset);
+		if (psEntry == NULL)
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+
+		PVR_ASSERT(psEntry->hParent == psHead->hParent);
+
+		/*
+		 * Get the next index now, in case the list item is
+		 * modified by the iteration function.
+		 */
+		hNext = psEntry->hNext;
+
+		eError = (*pfnIterFunc)(psBase, hHandle);
+		PVR_RETURN_IF_ERROR(eError);
+
+		hHandle = hNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      IterateOverChildren
+ @Description   Iterate over the subhandles of a parent handle
+ @Input         psParentData - pointer to parent handle structure
+                pfnIterFunc - function to be called for each subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(IterateOverChildren)
+#endif
+static INLINE
+PVRSRV_ERROR IterateOverChildren(PVRSRV_HANDLE_BASE *psBase,
+				 HANDLE_DATA *psParentData,
+				 PVRSRV_ERROR (*pfnIterFunc)(PVRSRV_HANDLE_BASE *, IMG_HANDLE))
+{
+	 return HandleListIterate(psBase,
+				  &psParentData->sChildren,
+				  offsetof(HANDLE_DATA, sChildren),
+				  offsetof(HANDLE_DATA, sSiblings),
+				  pfnIterFunc);
+}
+
+/*!
+*******************************************************************************
+ @Function      ParentIfPrivate
+ @Description   Return the parent handle if the handle was allocated with
+                PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE, else return NULL.
+ @Input         psHandleData - pointer to handle data structure
+ @Return        Parent handle or NULL
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(ParentIfPrivate)
+#endif
+static INLINE
+IMG_HANDLE ParentIfPrivate(HANDLE_DATA *psHandleData)
+{
+	return TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE) ?
+			ParentHandle(psHandleData) : NULL;
+}
+
+/*!
+*******************************************************************************
+ @Function      InitKey
+ @Description   Initialise a hash table key for the current process
+ @Input         aKey - pointer to key
+                psBase - pointer to handle base structure
+                pvData - pointer to the resource the handle represents
+                eType - type of resource
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(InitKey)
+#endif
+static INLINE
+void InitKey(HAND_KEY aKey,
+	     PVRSRV_HANDLE_BASE *psBase,
+	     void *pvData,
+	     PVRSRV_HANDLE_TYPE eType,
+	     IMG_HANDLE hParent)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+
+	aKey[HAND_KEY_DATA] = (uintptr_t)pvData;
+	aKey[HAND_KEY_TYPE] = (uintptr_t)eType;
+	aKey[HAND_KEY_PARENT] = (uintptr_t)hParent;
+}
+
+/*!
+*******************************************************************************
+ @Function      FindHandle
+ @Description   Find handle corresponding to a resource pointer
+ @Input         psBase - pointer to handle base structure
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Return        The handle, or NULL if not found
+******************************************************************************/
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(FindHandle)
+#endif
+static INLINE
+IMG_HANDLE FindHandle(PVRSRV_HANDLE_BASE *psBase,
+		      void *pvData,
+		      PVRSRV_HANDLE_TYPE eType,
+		      IMG_HANDLE hParent)
+{
+	HAND_KEY aKey;
+
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+
+	InitKey(aKey, psBase, pvData, eType, hParent);
+
+	return (IMG_HANDLE) HASH_Retrieve_Extended(psBase->psHashTab, aKey);
+}
+
+/*!
+*******************************************************************************
+ @Function      AllocHandle
+ @Description   Allocate a new handle
+ @Input         phHandle - location for new handle
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                hParent - parent handle or NULL
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+static PVRSRV_ERROR AllocHandle(PVRSRV_HANDLE_BASE *psBase,
+				IMG_HANDLE *phHandle,
+				void *pvData,
+				PVRSRV_HANDLE_TYPE eType,
+				PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				IMG_HANDLE hParent,
+				PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	HANDLE_DATA *psNewHandleData;
+	IMG_HANDLE hHandle;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(psBase != NULL && psBase->psHashTab != NULL);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	if (!TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		/* Handle must not already exist */
+		PVR_ASSERT(FindHandle(psBase, pvData, eType, hParent) == NULL);
+	}
+
+	psNewHandleData = OSAllocZMem(sizeof(*psNewHandleData));
+	PVR_LOG_RETURN_IF_NOMEM(psNewHandleData, "OSAllocZMem");
+
+	eError = gpsHandleFuncs->pfnAcquireHandle(psBase->psImplBase, &hHandle,
+	                                          psNewHandleData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnAcquireHandle",
+	                  ErrorFreeHandleData);
+
+	/*
+	 * If a data pointer can be associated with multiple handles, we
+	 * don't put the handle in the hash table, as the data pointer
+	 * may not map to a unique handle
+	 */
+	if (!TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+
+		/* Initialise hash key */
+		InitKey(aKey, psBase, pvData, eType, hParent);
+
+		/* Put the new handle in the hash table */
+		eError = HASH_Insert_Extended(psBase->psHashTab, aKey, (uintptr_t)hHandle) ?
+		        PVRSRV_OK : PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "couldn't add handle to hash table",
+		                  ErrorReleaseHandle);
+	}
+
+	psNewHandleData->hHandle = hHandle;
+	psNewHandleData->eType = eType;
+	psNewHandleData->eFlag = eFlag;
+	psNewHandleData->pvData = pvData;
+	psNewHandleData->pfnReleaseData = pfnReleaseData;
+	psNewHandleData->iLookupCount = 0;
+	psNewHandleData->bCanLookup = IMG_TRUE;
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = true", __func__));
+#endif /* DEBUG_REFCNT */
+
+	InitParentList(psNewHandleData);
+#if defined(DEBUG)
+	PVR_ASSERT(NoChildren(psNewHandleData));
+#endif
+
+	InitChildEntry(psNewHandleData);
+#if defined(DEBUG)
+	PVR_ASSERT(NoParent(psNewHandleData));
+#endif
+
+#if defined(PVRSRV_DEBUG_HANDLE_LOCK)
+	psNewHandleData->psBase = psBase;
+#endif
+
+	/* Return the new handle to the client */
+	*phHandle = psNewHandleData->hHandle;
+
+	return PVRSRV_OK;
+
+ErrorReleaseHandle:
+	(void)gpsHandleFuncs->pfnReleaseHandle(psBase->psImplBase, hHandle, NULL);
+
+ErrorFreeHandleData:
+	OSFreeMem(psNewHandleData);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandle
+ @Description   Allocate a handle
+ @Input         psBase - pointer to handle base structure
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase,
+			       IMG_HANDLE *phHandle,
+			       void *pvData,
+			       PVRSRV_HANDLE_TYPE eType,
+			       PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+			       PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVAllocHandleUnlocked(psBase, phHandle, pvData, eType, eFlag, pfnReleaseData);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandleUnlocked
+ @Description   Allocate a handle without acquiring/releasing the handle lock.
+                The function assumes you hold the lock when called.
+ @Input         phHandle - location for new handle
+                pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+                pfnReleaseData - Function to release resource at handle release
+                                 time
+ @Output        phHandle - points to new handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+			       IMG_HANDLE *phHandle,
+			       void *pvData,
+			       PVRSRV_HANDLE_TYPE eType,
+			       PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+			       PFN_HANDLE_RELEASE pfnReleaseData)
+{
+	*phHandle = NULL;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnReleaseData != NULL, "pfnReleaseData");
+
+	return AllocHandle(psBase, phHandle, pvData, eType, eFlag, NULL, pfnReleaseData);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocSubHandle
+ @Description   Allocate a subhandle
+ @Input         pvData - pointer to resource to be associated with the subhandle
+                eType - the type of resource
+                hParent - parent handle
+ @Output        phHandle - points to new subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData,
+				  PVRSRV_HANDLE_TYPE eType,
+				  PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				  IMG_HANDLE hParent)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVAllocSubHandleUnlocked(psBase, phHandle, pvData, eType, eFlag, hParent);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocSubHandleUnlocked
+ @Description   Allocate a subhandle without acquiring/releasing the handle
+                lock. The function assumes you hold the lock when called.
+ @Input         pvData - pointer to resource to be associated with the subhandle
+                eType - the type of resource
+                hParent - parent handle
+ @Output        phHandle - points to new subhandle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocSubHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData,
+				  PVRSRV_HANDLE_TYPE eType,
+				  PVRSRV_HANDLE_ALLOC_FLAG eFlag,
+				  IMG_HANDLE hParent)
+{
+	HANDLE_DATA *psPHandleData = NULL;
+	HANDLE_DATA *psCHandleData = NULL;
+	IMG_HANDLE hParentKey;
+	IMG_HANDLE hHandle;
+	PVRSRV_ERROR eError;
+
+	*phHandle = NULL;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psBase, eError, Exit);
+
+	hParentKey = TEST_FLAG(eFlag, PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE) ? hParent : NULL;
+
+	/* Lookup the parent handle */
+	eError = GetHandleData(psBase, &psPHandleData, hParent, PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to get parent handle structure",
+	                  Exit);
+
+	eError = AllocHandle(psBase, &hHandle, pvData, eType, eFlag, hParentKey, NULL);
+	PVR_GOTO_IF_ERROR(eError, Exit);
+
+	eError = GetHandleData(psBase, &psCHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+	/* If we were able to allocate the handle then there should be no reason why we
+	 * can't also get it's handle structure. Otherwise something has gone badly wrong.
+	 */
+	PVR_ASSERT(eError == PVRSRV_OK);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "Failed to get parent handle structure",
+	                  ExitFreeHandle);
+
+	/*
+	 * Get the parent handle structure again, in case the handle
+	 * structure has moved (depending on the implementation
+	 * of AllocHandle).
+	 */
+	eError = GetHandleData(psBase, &psPHandleData, hParent, PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to get parent handle structure",
+	                  ExitFreeHandle);
+
+	eError = AdoptChild(psBase, psPHandleData, psCHandleData);
+	PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "parent handle failed to adopt subhandle",
+	                  ExitFreeHandle);
+
+	*phHandle = hHandle;
+
+	return PVRSRV_OK;
+
+ExitFreeHandle:
+	PVRSRVDestroyHandleUnlocked(psBase, hHandle, eType);
+Exit:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFindHandle
+ @Description   Find handle corresponding to a resource pointer
+ @Input         pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Output        phHandle - points to returned handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE *phHandle,
+			      void *pvData,
+			      PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVFindHandleUnlocked(psBase, phHandle, pvData, eType);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFindHandleUnlocked
+ @Description   Find handle corresponding to a resource pointer without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         pvData - pointer to resource to be associated with the handle
+                eType - the type of resource
+ @Output        phHandle - points to the returned handle
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFindHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+			      IMG_HANDLE *phHandle,
+			      void *pvData,
+			      PVRSRV_HANDLE_TYPE eType)
+{
+	IMG_HANDLE hHandle;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	/* See if there is a handle for this data pointer */
+	hHandle = FindHandle(psBase, pvData, eType, NULL);
+	if (hHandle == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error finding handle. Type %u",
+			 __func__,
+			 eType));
+
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	*phHandle = hHandle;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupHandle
+ @Description   Lookup the data pointer corresponding to a handle
+ @Input         hHandle - handle from client
+                eType - handle type
+                bRef - If TRUE, a reference will be added on the handle if the
+                       lookup is successful.
+ @Output        ppvData - points to the return data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase,
+				void **ppvData,
+				IMG_HANDLE hHandle,
+				PVRSRV_HANDLE_TYPE eType,
+				IMG_BOOL bRef)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = PVRSRVLookupHandleUnlocked(psBase, ppvData, hHandle, eType, bRef);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupHandleUnlocked
+ @Description   Lookup the data pointer corresponding to a handle without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         hHandle - handle from client
+                eType - handle type
+                bRef - If TRUE, a reference will be added on the handle if the
+                       lookup is successful.
+ @Output        ppvData - points to the returned data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+				void **ppvData,
+				IMG_HANDLE hHandle,
+				PVRSRV_HANDLE_TYPE eType,
+				IMG_BOOL bRef)
+{
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error looking up handle (%s) for base %p of type %s. Handle %p, type %s",
+			 __func__,
+			 PVRSRVGetErrorString(eError),
+			 psBase,
+			 HandleBaseTypeToString(psBase->eType),
+			 (void*) hHandle,
+			 HandleTypeToString(eType)));
+#if defined(DEBUG) || defined(PVRSRV_NEED_PVR_DPF)
+		OSDumpStack();
+#endif
+		return eError;
+	}
+
+	/* If bCanLookup is false it means that a destroy operation was already
+	 * called on this handle; therefore it can no longer be looked up. */
+	if (!psHandleData->bCanLookup)
+	{
+		return PVRSRV_ERROR_HANDLE_NOT_ALLOCATED;
+	}
+
+	if (bRef)
+	{
+		HandleGet(psHandleData);
+	}
+
+	*ppvData = psHandleData->pvData;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVLookupSubHandle
+ @Description   Lookup the data pointer corresponding to a subhandle
+ @Input         hHandle - handle from client
+                eType - handle type
+                hAncestor - ancestor handle
+ @Output        ppvData - points to the returned data pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase,
+				   void **ppvData,
+				   IMG_HANDLE hHandle,
+				   PVRSRV_HANDLE_TYPE eType,
+				   IMG_HANDLE hAncestor)
+{
+	HANDLE_DATA *psPHandleData = NULL;
+	HANDLE_DATA *psCHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	LockHandle(psBase);
+
+	eError = GetHandleData(psBase, &psCHandleData, hHandle, eType);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Error looking up subhandle (%s). Handle %p, type %u",
+			 __func__,
+			 PVRSRVGetErrorString(eError),
+			 (void*) hHandle,
+			 eType));
+		OSDumpStack();
+		goto ExitUnlock;
+	}
+
+	/* Look for hAncestor among the handle's ancestors */
+	for (psPHandleData = psCHandleData; ParentHandle(psPHandleData) != hAncestor; )
+	{
+		eError = GetHandleData(psBase, &psPHandleData, ParentHandle(psPHandleData), PVRSRV_HANDLE_TYPE_NONE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "GetHandleData");
+			eError = PVRSRV_ERROR_INVALID_SUBHANDLE;
+			goto ExitUnlock;
+		}
+	}
+
+	*ppvData = psCHandleData->pvData;
+
+	eError = PVRSRV_OK;
+
+ExitUnlock:
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseHandle
+ @Description   Release a handle that is no longer needed
+ @Input         hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+void PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase,
+                         IMG_HANDLE hHandle,
+                         PVRSRV_HANDLE_TYPE eType)
+{
+	LockHandle(psBase);
+	PVRSRVReleaseHandleUnlocked(psBase, hHandle, eType);
+	UnlockHandle(psBase);
+}
+
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseHandleUnlocked
+ @Description   Release a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+ @Input         hHandle - handle from client
+                eType - handle type
+******************************************************************************/
+void PVRSRVReleaseHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                 IMG_HANDLE hHandle,
+                                 PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	/* PVRSRV_HANDLE_TYPE_NONE is reserved for internal use */
+	PVR_ASSERT(psBase != NULL);
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psBase != NULL, "invalid psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error (%s) looking up handle %p of type %s "
+		        "for base %p of type %s.", __func__, PVRSRVGetErrorString(eError),
+		        (void*) hHandle, HandleTypeToString(eType), psBase,
+		        HandleBaseTypeToString(psBase->eType)));
+
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		return;
+	}
+
+	PVR_ASSERT(psHandleData->bCanLookup);
+	PVR_ASSERT(psHandleData->iLookupCount > 0);
+
+	/* If there are still outstanding lookups for this handle or the handle
+	 * has not been destroyed yet, return early */
+	HandlePut(psHandleData);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVPurgeHandles
+ @Description   Purge handles for a given handle base
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPurgeHandles(PVRSRV_HANDLE_BASE *psBase)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	LockHandle(psBase);
+	eError = gpsHandleFuncs->pfnPurgeHandles(psBase->psImplBase);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFreeWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                                          IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleUnrefAndMaybeMarkForFree(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleUnrefAndMaybeMarkForFree(PVRSRV_HANDLE_BASE *psBase,
+                                                   HANDLE_DATA *psHandleData,
+                                                   IMG_HANDLE hHandle,
+                                                   PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	/* If bCanLookup is false it means that the destructor was called more than
+	 * once on this handle. */
+	if (!psHandleData->bCanLookup)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle %p of type %s already freed.",
+		        __func__, psHandleData->hHandle,
+		        HandleTypeToString(psHandleData->eType)));
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	if (psHandleData->iLookupCount > 0)
+	{
+		return PVRSRV_ERROR_OBJECT_STILL_REFERENCED;
+	}
+
+	/* Mark this handle as freed only if it's no longer referenced by any
+	 * lookup. The user space should retry freeing this handle once there are
+	 * no outstanding lookups. */
+	psHandleData->bCanLookup = IMG_FALSE;
+
+#ifdef DEBUG_REFCNT
+	PVR_DPF((PVR_DBG_ERROR, "%s: bCanLookup = false, iLookupCount = %d", __func__,
+	        psHandleData->iLookupCount));
+#endif /* DEBUG_REFCNT */
+
+	/* Prepare children for destruction */
+	eError = IterateOverChildren(psBase, psHandleData,
+	                             HandleUnrefAndMaybeMarkForFreeWrapper);
+	PVR_LOG_RETURN_IF_ERROR(eError, "HandleUnrefAndMaybeMarkForFreeWrapper");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HandleFreePrivDataWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                              IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleFreePrivData(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleFreePrivData(PVRSRV_HANDLE_BASE *psBase,
+                                       HANDLE_DATA *psHandleData,
+                                       IMG_HANDLE hHandle,
+                                       PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	/* Call the release data callback for each reference on the handle */
+	if (psHandleData->pfnReleaseData != NULL)
+	{
+		eError = psHandleData->pfnReleaseData(psHandleData->pvData);
+		if (eError != PVRSRV_OK)
+		{
+			if (PVRSRVIsRetryError(eError))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Got retry while calling release "
+						"data callback for handle %p of type = %s", __func__,
+						hHandle, HandleTypeToString(psHandleData->eType)));
+			}
+			else
+			{
+				PVR_LOG_ERROR(eError, "pfnReleaseData");
+			}
+
+			return eError;
+		}
+
+		/* we don't need this so make sure it's not called on
+		 * the pvData for the second time
+		 */
+		psHandleData->pfnReleaseData = NULL;
+	}
+
+	/* Free children's data */
+	eError = IterateOverChildren(psBase, psHandleData,
+	                             HandleFreePrivDataWrapper);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IterateOverChildren->HandleFreePrivData");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR HandleFreeDestroyWrapper(PVRSRV_HANDLE_BASE *psBase,
+                                             IMG_HANDLE hHandle)
+{
+	HANDLE_DATA *psHandleData;
+	PVRSRV_ERROR eError = GetHandleData(psBase, &psHandleData, hHandle,
+	                                    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return HandleFreeDestroy(psBase, psHandleData, hHandle, PVRSRV_HANDLE_TYPE_NONE);
+}
+
+static PVRSRV_ERROR HandleFreeDestroy(PVRSRV_HANDLE_BASE *psBase,
+                                      HANDLE_DATA *psHandleData,
+                                      IMG_HANDLE hHandle,
+                                      PVRSRV_HANDLE_TYPE eType)
+{
+	HANDLE_DATA *psReleasedHandleData;
+	PVRSRV_ERROR eError;
+
+	eError = UnlinkFromParent(psBase, psHandleData);
+	PVR_LOG_RETURN_IF_ERROR(eError, "UnlinkFromParent");
+
+	if (!TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+		IMG_HANDLE hRemovedHandle;
+
+		InitKey(aKey, psBase, psHandleData->pvData, psHandleData->eType,
+		        ParentIfPrivate(psHandleData));
+
+		hRemovedHandle = (IMG_HANDLE) HASH_Remove_Extended(psBase->psHashTab,
+		                                                   aKey);
+
+		PVR_ASSERT(hRemovedHandle != NULL);
+		PVR_ASSERT(hRemovedHandle == psHandleData->hHandle);
+		PVR_UNREFERENCED_PARAMETER(hRemovedHandle);
+	}
+
+	/* Free children */
+	eError = IterateOverChildren(psBase, psHandleData, HandleFreeDestroyWrapper);
+	PVR_LOG_RETURN_IF_ERROR(eError, "IterateOverChildren->HandleFreeDestroy");
+
+	eError = gpsHandleFuncs->pfnReleaseHandle(psBase->psImplBase,
+	                                          psHandleData->hHandle,
+	                                          (void **)&psReleasedHandleData);
+	OSFreeMem(psHandleData);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnReleaseHandle");
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR DestroyHandle(PVRSRV_HANDLE_BASE *psBase,
+                                  IMG_HANDLE hHandle,
+                                  PVRSRV_HANDLE_TYPE eType,
+                                  IMG_BOOL bReleaseLock)
+{
+	PVRSRV_ERROR eError;
+	HANDLE_DATA *psHandleData = NULL;
+
+	PVR_ASSERT(eType != PVRSRV_HANDLE_TYPE_NONE);
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase, &psHandleData, hHandle, eType);
+	PVR_RETURN_IF_ERROR(eError);
+
+	eError = HandleUnrefAndMaybeMarkForFree(psBase, psHandleData, hHandle, eType);
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (bReleaseLock)
+	{
+		UnlockHandle(psBase);
+	}
+
+	eError = HandleFreePrivData(psBase, psHandleData, hHandle, eType);
+	if (eError != PVRSRV_OK)
+	{
+		if (bReleaseLock)
+		{
+			LockHandle(psBase);
+		}
+
+		/* If the data could not be freed due to a temporary condition the
+		 * handle must be kept alive so that the next destroy call can try again */
+		if (PVRSRVIsRetryError(eError))
+		{
+			psHandleData->bCanLookup = IMG_TRUE;
+		}
+
+		return eError;
+	}
+
+	if (bReleaseLock)
+	{
+		LockHandle(psBase);
+	}
+
+	return HandleFreeDestroy(psBase, psHandleData, hHandle, eType);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandle
+ @Description   Destroys a handle that is no longer needed. Will
+                acquiring the handle lock for duration of the call.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandle(PVRSRV_HANDLE_BASE *psBase,
+                                 IMG_HANDLE hHandle,
+                                 PVRSRV_HANDLE_TYPE eType)
+{
+	PVRSRV_ERROR eError;
+
+	LockHandle(psBase);
+	eError = DestroyHandle(psBase, hHandle, eType, IMG_FALSE);
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandleUnlocked
+ @Description   Destroys a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandleUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                         IMG_HANDLE hHandle,
+                                         PVRSRV_HANDLE_TYPE eType)
+{
+	return DestroyHandle(psBase, hHandle, eType, IMG_FALSE);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVDestroyHandleStagedUnlocked
+ @Description   Destroys a handle that is no longer needed without
+                acquiring/releasing the handle lock. The function assumes you
+                hold the lock when called. This function, unlike
+                PVRSRVDestroyHandleUnlocked(), releases the handle lock while
+                destroying handle private data. This is done to open the
+                bridge for other bridge calls.
+                Can return RETRY or KERNEL_CCB_FULL if resource could not be
+                destroyed, caller should retry sometime later.
+ @Input         psBase - pointer to handle base structure
+                hHandle - handle from client
+                eType - handle type
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHandleStagedUnlocked(PVRSRV_HANDLE_BASE *psBase,
+                                               IMG_HANDLE hHandle,
+                                               PVRSRV_HANDLE_TYPE eType)
+{
+	return DestroyHandle(psBase, hHandle, eType, IMG_TRUE);
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAllocHandleBase
+ @Description   Allocate a handle base structure for a process
+ @Input         eType - handle type
+ @Output        ppsBase - points to handle base structure pointer
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAllocHandleBase(PVRSRV_HANDLE_BASE **ppsBase,
+                                   PVRSRV_HANDLE_BASE_TYPE eType)
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(gpsHandleFuncs != NULL, "handle management not initialised",
+	                  PVRSRV_ERROR_NOT_READY);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsBase != NULL, "ppsBase");
+
+	psBase = OSAllocZMem(sizeof(*psBase));
+	PVR_LOG_RETURN_IF_NOMEM(psBase, "psBase");
+
+	eError = OSLockCreate(&psBase->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", ErrorFreeHandleBase);
+
+	psBase->eType = eType;
+
+	LockHandle(psBase);
+
+	eError = gpsHandleFuncs->pfnCreateHandleBase(&psBase->psImplBase);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnlock);
+
+	psBase->psHashTab = HASH_Create_Extended(HANDLE_HASH_TAB_INIT_SIZE,
+						 sizeof(HAND_KEY),
+						 HASH_Func_Default,
+						 HASH_Key_Comp_Default);
+	PVR_LOG_GOTO_IF_FALSE(psBase->psHashTab != NULL, "couldn't create data pointer"
+	                  " hash table", ErrorDestroyHandleBase);
+
+	*ppsBase = psBase;
+
+	UnlockHandle(psBase);
+
+	return PVRSRV_OK;
+
+ErrorDestroyHandleBase:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+	(void)gpsHandleFuncs->pfnDestroyHandleBase(psBase->psImplBase);
+
+ErrorUnlock:
+	UnlockHandle(psBase);
+	OSLockDestroy(psBase->hLock);
+
+ErrorFreeHandleBase:
+	OSFreeMem(psBase);
+
+	return eError;
+}
+
+#if defined(DEBUG)
+typedef struct _COUNT_HANDLE_DATA_
+{
+	PVRSRV_HANDLE_BASE *psBase;
+	IMG_UINT32 uiHandleDataCount;
+} COUNT_HANDLE_DATA;
+
+/* Used to count the number of handles that have data associated with them */
+static PVRSRV_ERROR CountHandleDataWrapper(IMG_HANDLE hHandle, void *pvData)
+{
+	COUNT_HANDLE_DATA *psData = (COUNT_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData != NULL, "psData");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->psBase != NULL, "psData->psBase");
+
+	eError = GetHandleData(psData->psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData != NULL)
+	{
+		psData->uiHandleDataCount++;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Print a handle in the handle base. Used with the iterator callback. */
+static PVRSRV_ERROR ListHandlesInBase(IMG_HANDLE hHandle, void *pvData)
+{
+	PVRSRV_HANDLE_BASE *psBase = (PVRSRV_HANDLE_BASE*) pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBase != NULL, "psBase");
+
+	eError = GetHandleData(psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData != NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "    Handle: %6u, CanLookup: %u, LookupCount: %3u, Type: %s (%u), pvData<%p>",
+		       (IMG_UINT32) (uintptr_t) psHandleData->hHandle, psHandleData->bCanLookup,
+		       psHandleData->iLookupCount, HandleTypeToString(psHandleData->eType),
+		       psHandleData->eType, psHandleData->pvData));
+	}
+
+	return PVRSRV_OK;
+}
+
+#endif /* defined(DEBUG) */
+
+static INLINE IMG_BOOL _CheckIfMaxTimeExpired(IMG_UINT64 ui64TimeStart, IMG_UINT64 ui64MaxBridgeTime)
+{
+	/* unsigned arithmetic is well defined so this will wrap around correctly */
+	return (IMG_BOOL)((OSClockns64() - ui64TimeStart) >= ui64MaxBridgeTime);
+}
+
+static PVRSRV_ERROR FreeKernelHandlesWrapperIterKernel(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_KERNEL_HANDLE_DATA *psData = (FREE_KERNEL_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psKernelHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	/* Get kernel handle data. */
+	eError = GetHandleData(KERNEL_HANDLE_BASE,
+			    &psKernelHandleData,
+			    hHandle,
+			    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psKernelHandleData->pvData == psData->psProcessHandleData->pvData)
+	{
+		/* This kernel handle belongs to our process handle. */
+		psData->hKernelHandle = hHandle;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FreeKernelHandlesWrapperIterProcess(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_KERNEL_HANDLE_DATA *psData = (FREE_KERNEL_HANDLE_DATA *)pvData;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	/* Get process handle data. */
+	eError = GetHandleData(psData->psBase,
+			    &psData->psProcessHandleData,
+			    hHandle,
+			    PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psData->psProcessHandleData->eFlag == PVRSRV_HANDLE_ALLOC_FLAG_MULTI
+#if defined(SUPPORT_INSECURE_EXPORT)
+		|| psData->psProcessHandleData->eType == PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT
+#endif
+		)
+	{
+		/* Only multi alloc process handles might be in kernel handle base. */
+		psData->hKernelHandle = NULL;
+		/* Iterate over kernel handles. */
+		eError = gpsHandleFuncs->pfnIterateOverHandles(KERNEL_HANDLE_BASE->psImplBase,
+									&FreeKernelHandlesWrapperIterKernel,
+									(void *)psData);
+		PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "failed to iterate over kernel handles",
+		                  eError);
+
+		if (psData->hKernelHandle)
+		{
+			/* Release kernel handle which belongs to our process handle. */
+			eError = gpsHandleFuncs->pfnReleaseHandle(KERNEL_HANDLE_BASE->psImplBase,
+						psData->hKernelHandle,
+						NULL);
+			PVR_LOG_RETURN_IF_FALSE(eError == PVRSRV_OK, "couldn't release kernel handle",
+			                  eError);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR FreeHandleDataWrapper(IMG_HANDLE hHandle, void *pvData)
+{
+	FREE_HANDLE_DATA *psData = (FREE_HANDLE_DATA *)pvData;
+	HANDLE_DATA *psHandleData = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData != NULL, "psData");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->psBase != NULL, "psData->psBase");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psData->eHandleFreeType != PVRSRV_HANDLE_TYPE_NONE,
+	                          "psData->eHandleFreeType");
+
+	eError = GetHandleData(psData->psBase,
+			       &psHandleData,
+			       hHandle,
+			       PVRSRV_HANDLE_TYPE_NONE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "GetHandleData");
+
+	if (psHandleData == NULL || psHandleData->eType != psData->eHandleFreeType)
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(psHandleData->bCanLookup && psHandleData->iLookupCount == 0);
+
+	if (psHandleData->bCanLookup)
+	{
+		if (psHandleData->pfnReleaseData != NULL)
+		{
+			eError = psHandleData->pfnReleaseData(psHandleData->pvData);
+			if (PVRSRVIsRetryError(eError))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Got retry while calling release "
+				        "data callback for handle %p of type = %s", __func__,
+				        hHandle, HandleTypeToString(psHandleData->eType)));
+
+				return eError;
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected error from pfnReleaseData (%s)",
+						 __func__,
+						 PVRSRVGetErrorString(eError)));
+			}
+		}
+
+		psHandleData->bCanLookup = IMG_FALSE;
+	}
+
+	if (!TEST_ALLOC_FLAG(psHandleData, PVRSRV_HANDLE_ALLOC_FLAG_MULTI))
+	{
+		HAND_KEY aKey;
+		IMG_HANDLE hRemovedHandle;
+
+		InitKey(aKey,
+			psData->psBase,
+			psHandleData->pvData,
+			psHandleData->eType,
+			ParentIfPrivate(psHandleData));
+
+		hRemovedHandle = (IMG_HANDLE)HASH_Remove_Extended(psData->psBase->psHashTab, aKey);
+
+		PVR_ASSERT(hRemovedHandle != NULL);
+		PVR_ASSERT(hRemovedHandle == psHandleData->hHandle);
+		PVR_UNREFERENCED_PARAMETER(hRemovedHandle);
+	}
+
+	eError = gpsHandleFuncs->pfnSetHandleData(psData->psBase->psImplBase, hHandle, NULL);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSFreeMem(psHandleData);
+
+	/* If we reach the end of the time slice release we can release the global
+	 * lock, invoke the scheduler and reacquire the lock */
+	if ((psData->ui64MaxBridgeTime != 0) && _CheckIfMaxTimeExpired(psData->ui64TimeStart, psData->ui64MaxBridgeTime))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+			 "%s: Lock timeout (timeout: %" IMG_UINT64_FMTSPEC")",
+			 __func__,
+			 psData->ui64MaxBridgeTime));
+		UnlockHandle(psData->psBase);
+		/* Invoke the scheduler to check if other processes are waiting for the lock */
+		OSReleaseThreadQuanta();
+		LockHandle(psData->psBase);
+		/* Set again lock timeout and reset the counter */
+		psData->ui64TimeStart = OSClockns64();
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Lock acquired again", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/* The Ordered Array of PVRSRV_HANDLE_TYPE Enum Entries.
+ *
+ *   Some handles must be destroyed prior to other handles,
+ *   such relationships are established with respect to handle types.
+ *   Therefore elements of this array have to maintain specific order,
+ *   e.g. the PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET must be placed
+ *   before PVRSRV_HANDLE_TYPE_RGX_FREELIST.
+ *
+ *   If ordering is incorrect driver may fail on the ground of cleanup
+ *   routines. Unfortunately, we can mainly rely on the actual definition of
+ *   the array, there is no explicit information about all relationships
+ *   between handle types. These relationships do not necessarily come from
+ *   bridge-specified handle attributes such as 'sub handle' and 'parent
+ *   handle'. They may come from internal/private ref-counters contained by
+ *   objects referenced by our kernel handles.
+ *
+ *   For example, at the bridge level, PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET
+ *   and PVRSRV_HANDLE_TYPE_RGX_FREELIST have no explicit relationship, meaning
+ *   none of them is a sub-handle for the other.
+ *   However the freelist contains internal ref-count that is decremented by
+ *   the destroy routine for KM_HW_RT_DATASET.
+ *
+ *   BE CAREFUL when adding/deleting/moving handle types.
+ */
+static const PVRSRV_HANDLE_TYPE g_aeOrderedFreeList[] =
+{
+	PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+	PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+	PVRSRV_HANDLE_TYPE_RGX_FW_MEMDESC,
+	PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+	PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+	PVRSRV_HANDLE_TYPE_RGX_MEMORY_BLOCK,
+	PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+	PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_RAY_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+	PVRSRV_HANDLE_TYPE_RI_HANDLE,
+	PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+	PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+	PVRSRV_HANDLE_TYPE_PVRSRV_TIMELINE_SERVER,
+	PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_EXPORT,
+	PVRSRV_HANDLE_TYPE_PVRSRV_FENCE_SERVER,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_MAPPING,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+	PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+	PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+	PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_PAGELIST,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_SECURE_EXPORT,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+	PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+	PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+	PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+	PVRSRV_HANDLE_TYPE_DC_PIN_HANDLE,
+	PVRSRV_HANDLE_TYPE_DC_BUFFER,
+	PVRSRV_HANDLE_TYPE_DC_DISPLAY_CONTEXT,
+	PVRSRV_HANDLE_TYPE_DC_DEVICE,
+	PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+	PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+	PVRSRV_HANDLE_TYPE_MM_PLAT_CLEANUP
+};
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFreeKernelHandles
+ @Description   Free kernel handles which belongs to process handles
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFreeKernelHandles(PVRSRV_HANDLE_BASE *psBase)
+{
+	FREE_KERNEL_HANDLE_DATA sHandleData = {NULL};
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	LockHandle(psBase);
+
+	sHandleData.psBase = psBase;
+	/* Iterate over process handles. */
+	eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+								&FreeKernelHandlesWrapperIterProcess,
+								(void *)&sHandleData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnIterateOverHandles", ExitUnlock);
+
+	eError = PVRSRV_OK;
+
+ExitUnlock:
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVRetrieveProcessHandleBase
+ @Description   Returns a pointer to the process handle base for the current
+                process. If the current process is the cleanup thread, then the
+                process handle base for the process currently being cleaned up
+                is returned
+ @Return        Pointer to the process handle base, or NULL if not found.
+******************************************************************************/
+PVRSRV_HANDLE_BASE *PVRSRVRetrieveProcessHandleBase(void)
+{
+	PVRSRV_HANDLE_BASE *psHandleBase = NULL;
+	PROCESS_HANDLE_BASE *psProcHandleBase = NULL;
+	IMG_PID ui32PurgePid = PVRSRVGetPurgeConnectionPid();
+	IMG_PID uiCleanupPid = PVRSRVCleanupThreadGetPid();
+	uintptr_t uiCleanupTid = PVRSRVCleanupThreadGetTid();
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	/* Check to see if we're being called from the cleanup thread... */
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+	    (OSGetCurrentThreadID() == uiCleanupTid) &&
+	    (ui32PurgePid > 0))
+	{
+		/* Check to see if the cleanup thread has already removed the
+		 * process handle base from the HASH table.
+		 */
+		psHandleBase = g_psProcessHandleBaseBeingFreed;
+		/* psHandleBase shouldn't be null, as cleanup thread
+		 * should be removing this from the HASH table before
+		 * we get here, so assert if not.
+		 */
+		PVR_ASSERT(psHandleBase);
+	}
+	else
+	{
+		/* Not being called from the cleanup thread, so return the process
+		 * handle base for the current process.
+		 */
+		psProcHandleBase = (PROCESS_HANDLE_BASE *)
+		    HASH_Retrieve(g_psProcessHandleBaseTable, OSGetCurrentClientProcessIDKM());
+	}
+
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	if (psHandleBase == NULL && psProcHandleBase != NULL)
+	{
+		psHandleBase = psProcHandleBase->psHandleBase;
+	}
+	return psHandleBase;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVAcquireProcessHandleBase
+ @Description   Increments reference count on a process handle base identified
+                by uiPid and returns pointer to the base. If the handle base
+                does not exist it will be allocated.
+ @Inout         uiPid - PID of a process
+ @Output        ppsBase - pointer to a handle base for the process identified by
+                          uiPid
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVAcquireProcessHandleBase(IMG_PID uiPid, PROCESS_HANDLE_BASE **ppsBase)
+{
+	PROCESS_HANDLE_BASE *psBase;
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	psBase = (PROCESS_HANDLE_BASE*) HASH_Retrieve(g_psProcessHandleBaseTable, uiPid);
+
+	/* In case there is none we are going to allocate one */
+	if (psBase == NULL)
+	{
+		IMG_BOOL bSuccess;
+
+		psBase = OSAllocZMem(sizeof(*psBase));
+		PVR_LOG_GOTO_IF_NOMEM(psBase, eError, ErrorUnlock);
+
+		/* Allocate handle base for this process */
+		eError = PVRSRVAllocHandleBase(&psBase->psHandleBase, PVRSRV_HANDLE_BASE_TYPE_PROCESS);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", ErrorFreeProcessHandleBase);
+
+		/* Insert the handle base into the global hash table */
+		bSuccess = HASH_Insert(g_psProcessHandleBaseTable, uiPid, (uintptr_t) psBase);
+		PVR_LOG_GOTO_IF_FALSE(bSuccess, "HASH_Insert failed", ErrorFreeHandleBase);
+	}
+
+	OSAtomicIncrement(&psBase->iRefCount);
+
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	*ppsBase = psBase;
+
+	return PVRSRV_OK;
+
+ErrorFreeHandleBase:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+	PVRSRVFreeHandleBase(psBase->psHandleBase, 0);
+ErrorFreeProcessHandleBase:
+	OSFreeMem(psBase);
+ErrorUnlock:
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVReleaseProcessHandleBase
+ @Description   Decrements reference count on a process handle base psBase
+                for a process identified by uiPid. If the reference count
+                reaches 0 the handle base will be freed..
+ @Input         psBase - pointer to a process handle base
+ @Inout         uiPid - PID of a process
+ @Inout         ui64MaxBridgeTime - maximum time a handle destroy operation
+                                    can hold the handle base lock (after that
+                                    time a lock will be release and reacquired
+                                    for another time slice)
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVReleaseProcessHandleBase(PROCESS_HANDLE_BASE *psBase, IMG_PID uiPid,
+                                            IMG_UINT64 ui64MaxBridgeTime)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT iRefCount;
+	uintptr_t uiHashValue;
+
+	OSLockAcquire(g_hProcessHandleBaseLock);
+
+	iRefCount = OSAtomicDecrement(&psBase->iRefCount);
+
+	if (iRefCount != 0)
+	{
+		OSLockRelease(g_hProcessHandleBaseLock);
+		return PVRSRV_OK;
+	}
+
+	/* in case the refcount becomes 0 we can remove the process handle base
+	 * and all related objects */
+
+	uiHashValue = HASH_Remove(g_psProcessHandleBaseTable, uiPid);
+	OSLockRelease(g_hProcessHandleBaseLock);
+
+	PVR_LOG_RETURN_IF_FALSE(uiHashValue != 0, "HASH_Remove failed",
+	                        PVRSRV_ERROR_UNABLE_TO_REMOVE_HASH_VALUE);
+
+	eError = PVRSRVFreeKernelHandles(psBase->psHandleBase);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeKernelHandles");
+
+	eError = PVRSRVFreeHandleBase(psBase->psHandleBase, ui64MaxBridgeTime);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVFreeHandleBase");
+
+	OSFreeMem(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVFreeHandleBase
+ @Description   Free a handle base structure
+ @Input         psBase - pointer to handle base structure
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVFreeHandleBase(PVRSRV_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime)
+{
+#if defined(DEBUG)
+	COUNT_HANDLE_DATA sCountData = {NULL};
+#endif
+	FREE_HANDLE_DATA sHandleData = {NULL};
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_PID uiCleanupPid = PVRSRVCleanupThreadGetPid();
+	uintptr_t uiCleanupTid = PVRSRVCleanupThreadGetTid();
+	IMG_UINT32 ui32ErrorCount = 0;
+
+	PVR_ASSERT(gpsHandleFuncs);
+
+	LockHandle(psBase);
+
+	/* If this is a process handle base being freed by the cleanup
+	 * thread, store this in g_psProcessHandleBaseBeingFreed
+	 */
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+	    (OSGetCurrentThreadID() == uiCleanupTid) &&
+	    (psBase->eType == PVRSRV_HANDLE_BASE_TYPE_PROCESS))
+	{
+		g_psProcessHandleBaseBeingFreed = psBase;
+	}
+
+	sHandleData.psBase = psBase;
+	sHandleData.ui64TimeStart = OSClockns64();
+	sHandleData.ui64MaxBridgeTime = ui64MaxBridgeTime;
+
+
+#if defined(DEBUG)
+
+	sCountData.psBase = psBase;
+
+	eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+						       &CountHandleDataWrapper,
+						       (void *)&sCountData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnIterateOverHandles", ExitUnlock);
+
+	if (sCountData.uiHandleDataCount != 0)
+	{
+		IMG_BOOL bList = (IMG_BOOL)(sCountData.uiHandleDataCount < HANDLE_DEBUG_LISTING_MAX_NUM);
+
+		PVR_DPF((PVR_DBG_WARNING,
+			 "%s: %u remaining handles in handle base 0x%p "
+			 "(PVRSRV_HANDLE_BASE_TYPE %u).%s",
+			 __func__,
+			 sCountData.uiHandleDataCount,
+			 psBase,
+			 psBase->eType,
+			 bList ? "": " Skipping details, too many items..."));
+
+		if (bList)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "-------- Listing Handles --------"));
+			(void) gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+			                                             &ListHandlesInBase,
+			                                             psBase);
+			PVR_DPF((PVR_DBG_WARNING, "-------- Done Listing    --------"));
+		}
+	}
+
+#endif /* defined(DEBUG) */
+
+	/*
+	 * As we're freeing handles based on type, make sure all
+	 * handles have actually had their data freed to avoid
+	 * resources being leaked
+	 */
+	for (i = 0; i < ARRAY_SIZE(g_aeOrderedFreeList); i++)
+	{
+		sHandleData.eHandleFreeType = g_aeOrderedFreeList[i];
+
+		/* Make sure all handles have been freed before destroying the handle base */
+		eError = gpsHandleFuncs->pfnIterateOverHandles(psBase->psImplBase,
+							       &FreeHandleDataWrapper,
+							       (void *)&sHandleData);
+
+		/* On retry error return without destroying handle base. Caller may retry. */
+		if (PVRSRVIsRetryError(eError))
+		{
+			PVR_GOTO_IF_ERROR(eError, ExitUnlock);
+		}
+
+		/* Retry is not possible. Continue freeing remaining handles. */
+		if (eError != PVRSRV_OK)
+		{
+			ui32ErrorCount++;
+		}
+	}
+
+	if (ui32ErrorCount > 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error freeing %d handles.", __func__, ui32ErrorCount));
+	}
+
+	if (psBase->psHashTab != NULL)
+	{
+		HASH_Delete(psBase->psHashTab);
+	}
+
+	eError = gpsHandleFuncs->pfnDestroyHandleBase(psBase->psImplBase);
+	PVR_GOTO_IF_ERROR(eError, ExitUnlock);
+
+	UnlockHandle(psBase);
+	OSLockDestroy(psBase->hLock);
+	OSFreeMem(psBase);
+
+	return eError;
+
+ExitUnlock:
+	if ((OSGetCurrentProcessID() == uiCleanupPid) &&
+		(OSGetCurrentThreadID() == uiCleanupTid))
+	{
+		g_psProcessHandleBaseBeingFreed = NULL;
+	}
+	UnlockHandle(psBase);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVHandleInit
+ @Description   Initialise handle management
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVHandleInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(gpsKernelHandleBase == NULL);
+	PVR_ASSERT(gpsHandleFuncs == NULL);
+	PVR_ASSERT(g_hProcessHandleBaseLock == NULL);
+	PVR_ASSERT(g_psProcessHandleBaseTable == NULL);
+	PVR_ASSERT(!gbLockInitialised);
+
+	eError = OSLockCreate(&gKernelHandleLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate:1");
+
+	eError = OSLockCreate(&g_hProcessHandleBaseLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate:2", ErrorHandleDeinit);
+
+	gbLockInitialised = IMG_TRUE;
+
+	eError = PVRSRVHandleGetFuncTable(&gpsHandleFuncs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVHandleGetFuncTable", ErrorHandleDeinit);
+
+	eError = PVRSRVAllocHandleBase(&gpsKernelHandleBase,
+	                               PVRSRV_HANDLE_BASE_TYPE_GLOBAL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAllocHandleBase", ErrorHandleDeinit);
+
+	g_psProcessHandleBaseTable = HASH_Create(HANDLE_PROC_HANDLE_HASH_INIT_SIZE);
+	PVR_LOG_GOTO_IF_NOMEM(g_psProcessHandleBaseTable, eError, ErrorHandleDeinit);
+
+	eError = gpsHandleFuncs->pfnEnableHandlePurging(gpsKernelHandleBase->psImplBase);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnEnableHandlePurging", ErrorHandleDeinit);
+
+	return PVRSRV_OK;
+
+ErrorHandleDeinit:
+	(void) PVRSRVHandleDeInit();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function      PVRSRVHandleDeInit
+ @Description   De-initialise handle management
+ @Return        Error code or PVRSRV_OK
+******************************************************************************/
+PVRSRV_ERROR PVRSRVHandleDeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (gpsHandleFuncs != NULL)
+	{
+		if (gpsKernelHandleBase != NULL)
+		{
+			eError = PVRSRVFreeHandleBase(gpsKernelHandleBase, 0 /* do not release bridge lock */);
+			if (eError == PVRSRV_OK)
+			{
+				gpsKernelHandleBase = NULL;
+			}
+			else
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVFreeHandleBase");
+			}
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+			gpsHandleFuncs = NULL;
+		}
+	}
+	else
+	{
+		/* If we don't have a handle function table we shouldn't have a handle base either */
+		PVR_ASSERT(gpsKernelHandleBase == NULL);
+	}
+
+	if (g_psProcessHandleBaseTable != NULL)
+	{
+		HASH_Delete(g_psProcessHandleBaseTable);
+		g_psProcessHandleBaseTable = NULL;
+	}
+
+	if (g_hProcessHandleBaseLock != NULL)
+	{
+		OSLockDestroy(g_hProcessHandleBaseLock);
+		g_hProcessHandleBaseLock = NULL;
+	}
+
+	if (gKernelHandleLock != NULL)
+	{
+		OSLockDestroy(gKernelHandleLock);
+		gbLockInitialised = IMG_FALSE;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (handle.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/handle.h b/drivers/gpu/drm/img-rogue/handle.h
new file mode 100644
index 000000000000..27fc7d4fc658
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/handle.h
@@ -0,0 +1,207 @@
+/**************************************************************************/ /*!
+@File
+@Title          Handle Manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(HANDLE_API_H)
+#define HANDLE_API_H
+
+#include "lock_types.h"
+
+/*
+ * Handle API
+ * ----------
+ * The handle API is intended to provide handles for kernel resources, which
+ * can then be passed back to user space processes.
+ *
+ * The following functions comprise the API. Each function takes a pointer to
+ * a PVRSRV_HANDLE_BASE structure, one of which is allocated for each process,
+ * and stored in the per-process data area. Use KERNEL_HANDLE_BASE for handles
+ * not allocated for a particular process, or for handles that need to be
+ * allocated before the PVRSRV_HANDLE_BASE structure for the process is
+ * available.
+ *
+ * PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType,
+ *      PVRSRV_HANDLE_ALLOC_FLAG eFlag);
+ *
+ * Allocate a handle phHandle, for the resource of type eType pointed to by
+ * pvData.
+ *
+ * For handles that have a definite lifetime, where the corresponding resource
+ * is explicitly created and destroyed, eFlag should be zero.
+ *
+ * If a particular resource may be referenced multiple times by a given
+ * process, setting eFlag to PVRSRV_HANDLE_ALLOC_FLAG_MULTI will allow multiple
+ * handles to be allocated for the resource. Such handles cannot be found with
+ * PVRSRVFindHandle.
+ *
+ * PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType,
+ *      PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+ *
+ * This function is similar to PVRSRVAllocHandle, except that the allocated
+ * handles are associated with a parent handle, hParent, that has been
+ * allocated previously. Subhandles are automatically deallocated when their
+ * parent handle is deallocated.
+ * Subhandles can be treated as ordinary handles. For example, they may have
+ * subhandles of their own, and may be explicitly deallocated using
+ * PVRSRVReleaseHandle (see below).
+ *
+ * PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Find the handle previously allocated for the resource pointed to by pvData,
+ * of type eType. Handles allocated with the flag
+ * PVRSRV_HANDLE_ALLOC_FLAG_MULTI cannot be found using this function.
+ *
+ * PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Given a handle for a resource of type eType, return the pointer to the
+ * resource.
+ *
+ * PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType,
+ *      IMH_HANDLE hAncestor);
+ *
+ * Similar to PVRSRVLookupHandle, but checks the handle is a descendant
+ * of hAncestor.
+ *
+ * PVRSRV_ERROR PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase,
+ *      IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+ *
+ * Deallocate a handle of given type.
+ *
+ * Return the parent of a handle in *phParent, or NULL if the handle has
+ * no parent.
+ */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "hash.h"
+
+typedef enum
+{
+	#define HANDLETYPE(x) PVRSRV_HANDLE_TYPE_##x,
+	#include "handle_types.h"
+	#undef HANDLETYPE
+} PVRSRV_HANDLE_TYPE;
+
+static_assert(PVRSRV_HANDLE_TYPE_NONE == 0, "PVRSRV_HANDLE_TYPE_NONE must be zero");
+
+typedef enum
+{
+	PVRSRV_HANDLE_BASE_TYPE_CONNECTION,
+	PVRSRV_HANDLE_BASE_TYPE_PROCESS,
+	PVRSRV_HANDLE_BASE_TYPE_GLOBAL
+} PVRSRV_HANDLE_BASE_TYPE;
+
+
+typedef enum
+{
+	/* No flags */
+	PVRSRV_HANDLE_ALLOC_FLAG_NONE =    0,
+	/* Multiple handles can point at the given data pointer */
+	PVRSRV_HANDLE_ALLOC_FLAG_MULTI =   0x01,
+	/* Subhandles are allocated in a private handle space */
+	PVRSRV_HANDLE_ALLOC_FLAG_PRIVATE = 0x02
+} PVRSRV_HANDLE_ALLOC_FLAG;
+
+typedef struct _HANDLE_BASE_ PVRSRV_HANDLE_BASE;
+
+typedef struct _PROCESS_HANDLE_BASE_
+{
+	PVRSRV_HANDLE_BASE *psHandleBase;
+	ATOMIC_T iRefCount;
+} PROCESS_HANDLE_BASE;
+
+extern PVRSRV_HANDLE_BASE *gpsKernelHandleBase;
+#define	KERNEL_HANDLE_BASE (gpsKernelHandleBase)
+
+#define HANDLE_DEBUG_LISTING_MAX_NUM 20
+
+typedef PVRSRV_ERROR (*PFN_HANDLE_RELEASE)(void *pvData);
+
+PVRSRV_ERROR PVRSRVAllocHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, PFN_HANDLE_RELEASE pfnReleaseData);
+PVRSRV_ERROR PVRSRVAllocHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, PFN_HANDLE_RELEASE pfnReleaseData);
+
+PVRSRV_ERROR PVRSRVAllocSubHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+PVRSRV_ERROR PVRSRVAllocSubHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType, PVRSRV_HANDLE_ALLOC_FLAG eFlag, IMG_HANDLE hParent);
+
+PVRSRV_ERROR PVRSRVFindHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVFindHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE *phHandle, void *pvData, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVLookupHandle(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_BOOL bRef);
+PVRSRV_ERROR PVRSRVLookupHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_BOOL bRef);
+
+PVRSRV_ERROR PVRSRVLookupSubHandle(PVRSRV_HANDLE_BASE *psBase, void **ppvData, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType, IMG_HANDLE hAncestor);
+
+void PVRSRVReleaseHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+void PVRSRVReleaseHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVDestroyHandle(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVDestroyHandleUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+PVRSRV_ERROR PVRSRVDestroyHandleStagedUnlocked(PVRSRV_HANDLE_BASE *psBase, IMG_HANDLE hHandle, PVRSRV_HANDLE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVPurgeHandles(PVRSRV_HANDLE_BASE *psBase);
+
+PVRSRV_ERROR PVRSRVAllocHandleBase(PVRSRV_HANDLE_BASE **ppsBase,
+                                   PVRSRV_HANDLE_BASE_TYPE eType);
+
+PVRSRV_ERROR PVRSRVFreeHandleBase(PVRSRV_HANDLE_BASE *psBase, IMG_UINT64 ui64MaxBridgeTime);
+
+PVRSRV_ERROR PVRSRVFreeKernelHandles(PVRSRV_HANDLE_BASE *psBase);
+
+PVRSRV_ERROR PVRSRVHandleInit(void);
+
+PVRSRV_ERROR PVRSRVHandleDeInit(void);
+
+PVRSRV_HANDLE_BASE *PVRSRVRetrieveProcessHandleBase(void);
+
+PVRSRV_ERROR PVRSRVAcquireProcessHandleBase(IMG_PID uiPid, PROCESS_HANDLE_BASE **ppsBase);
+PVRSRV_ERROR PVRSRVReleaseProcessHandleBase(PROCESS_HANDLE_BASE *psBase, IMG_PID uiPid, IMG_UINT64 ui64MaxBridgeTime);
+
+void LockHandle(PVRSRV_HANDLE_BASE *psBase);
+void UnlockHandle(PVRSRV_HANDLE_BASE *psBase);
+
+
+#endif /* !defined(HANDLE_API_H) */
diff --git a/drivers/gpu/drm/img-rogue/handle_idr.c b/drivers/gpu/drm/img-rogue/handle_idr.c
new file mode 100644
index 000000000000..cb448213d6fa
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/handle_idr.c
@@ -0,0 +1,417 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Handle Manager - IDR Back-end
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide IDR based resource handle management back-end
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <linux/gfp.h>
+#include <linux/idr.h>
+
+#include "handle_impl.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#define ID_VALUE_MIN	1
+#define ID_VALUE_MAX	INT_MAX
+
+#define	ID_TO_HANDLE(i) ((IMG_HANDLE)(uintptr_t)(i))
+#define	HANDLE_TO_ID(h) ((IMG_INT)(uintptr_t)(h))
+
+struct _HANDLE_IMPL_BASE_
+{
+	struct idr sIdr;
+
+	IMG_UINT32 ui32MaxHandleValue;
+
+	IMG_UINT32 ui32TotalHandCount;
+};
+
+typedef struct _HANDLE_ITER_DATA_WRAPPER_
+{
+	PFN_HANDLE_ITER pfnHandleIter;
+	void *pvHandleIterData;
+} HANDLE_ITER_DATA_WRAPPER;
+
+
+static int HandleIterFuncWrapper(int id, void *data, void *iter_data)
+{
+	HANDLE_ITER_DATA_WRAPPER *psIterData = (HANDLE_ITER_DATA_WRAPPER *)iter_data;
+
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	return (int)psIterData->pfnHandleIter(ID_TO_HANDLE(id), psIterData->pvHandleIterData);
+}
+
+/*!
+******************************************************************************
+
+ @Function	AcquireHandle
+
+ @Description	Acquire a new handle
+
+ @Input		psBase - Pointer to handle base structure
+		phHandle - Points to a handle pointer
+		pvData - Pointer to resource to be associated with the handle
+
+ @Output	phHandle - Points to a handle pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR AcquireHandle(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE *phHandle,
+				  void *pvData)
+{
+	int id;
+	int result;
+
+	PVR_ASSERT(psBase != NULL);
+	PVR_ASSERT(phHandle != NULL);
+	PVR_ASSERT(pvData != NULL);
+
+	idr_preload(GFP_KERNEL);
+	id = idr_alloc(&psBase->sIdr, pvData, ID_VALUE_MIN, psBase->ui32MaxHandleValue + 1, 0);
+	idr_preload_end();
+
+	result = id;
+
+	if (result < 0)
+	{
+		if (result == -ENOSPC)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Limit of %u handles reached",
+				 __func__, psBase->ui32MaxHandleValue));
+
+			return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		}
+
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psBase->ui32TotalHandCount++;
+
+	*phHandle = ID_TO_HANDLE(id);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	ReleaseHandle
+
+ @Description	Release a handle that is no longer needed.
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle to release
+		ppvData - Points to a void data pointer
+
+ @Output	ppvData - Points to a void data pointer
+
+ @Return	PVRSRV_OK or PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR ReleaseHandle(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void **ppvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvData;
+
+	PVR_ASSERT(psBase);
+
+	/* Get the data associated with the handle. If we get back NULL then
+	   it's an invalid handle */
+
+	pvData = idr_find(&psBase->sIdr, id);
+	if (likely(pvData))
+	{
+		idr_remove(&psBase->sIdr, id);
+		psBase->ui32TotalHandCount--;
+	}
+
+	if (unlikely(pvData == NULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Handle out of range (%u > %u)",
+			 __func__, id, psBase->ui32TotalHandCount));
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+
+	if (ppvData)
+	{
+		*ppvData = pvData;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	GetHandleData
+
+ @Description	Get the data associated with the given handle
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle from which data should be retrieved
+		ppvData - Points to a void data pointer
+
+ @Output	ppvData - Points to a void data pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR GetHandleData(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void **ppvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvData;
+
+	PVR_ASSERT(psBase);
+	PVR_ASSERT(ppvData);
+
+	pvData = idr_find(&psBase->sIdr, id);
+	if (likely(pvData))
+	{
+		*ppvData = pvData;
+
+		return PVRSRV_OK;
+	}
+	else
+	{
+		return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+	}
+}
+
+/*!
+******************************************************************************
+
+ @Function	SetHandleData
+
+ @Description	Set the data associated with the given handle
+
+ @Input		psBase - Pointer to handle base structure
+		hHandle - Handle for which data should be changed
+		pvData - Pointer to new data to be associated with the handle
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR SetHandleData(HANDLE_IMPL_BASE *psBase,
+				  IMG_HANDLE hHandle,
+				  void *pvData)
+{
+	int id = HANDLE_TO_ID(hHandle);
+	void *pvOldData;
+
+	PVR_ASSERT(psBase);
+
+	pvOldData = idr_replace(&psBase->sIdr, pvData, id);
+	if (IS_ERR(pvOldData))
+	{
+		if (PTR_ERR(pvOldData) == -ENOENT)
+		{
+			return PVRSRV_ERROR_HANDLE_NOT_ALLOCATED;
+		}
+		else
+		{
+			return PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR IterateOverHandles(HANDLE_IMPL_BASE *psBase, PFN_HANDLE_ITER pfnHandleIter, void *pvHandleIterData)
+{
+	HANDLE_ITER_DATA_WRAPPER sIterData;
+
+	PVR_ASSERT(psBase);
+	PVR_ASSERT(pfnHandleIter);
+
+	sIterData.pfnHandleIter = pfnHandleIter;
+	sIterData.pvHandleIterData = pvHandleIterData;
+
+	return (PVRSRV_ERROR)idr_for_each(&psBase->sIdr, HandleIterFuncWrapper, &sIterData);
+}
+
+/*!
+******************************************************************************
+
+ @Function	EnableHandlePurging
+
+ @Description	Enable purging for a given handle base
+
+ @Input		psBase - pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR EnableHandlePurging(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+	PVR_ASSERT(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PurgeHandles
+
+ @Description	Purge handles for a given handle base
+
+ @Input		psBase - Pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR PurgeHandles(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psBase);
+	PVR_ASSERT(psBase);
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	CreateHandleBase
+
+ @Description	Create a handle base structure
+
+ @Input		ppsBase - pointer to handle base structure pointer
+
+ @Output	ppsBase - points to handle base structure pointer
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR CreateHandleBase(HANDLE_IMPL_BASE **ppsBase)
+{
+	HANDLE_IMPL_BASE *psBase;
+
+	PVR_ASSERT(ppsBase);
+
+	psBase = OSAllocZMem(sizeof(*psBase));
+	if (psBase == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Couldn't allocate generic handle base",
+				 __func__));
+
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	idr_init(&psBase->sIdr);
+
+	psBase->ui32MaxHandleValue = ID_VALUE_MAX;
+	psBase->ui32TotalHandCount = 0;
+
+	*ppsBase = psBase;
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	DestroyHandleBase
+
+ @Description	Destroy a handle base structure
+
+ @Input		psBase - pointer to handle base structure
+
+ @Return	Error code or PVRSRV_OK
+
+******************************************************************************/
+static PVRSRV_ERROR DestroyHandleBase(HANDLE_IMPL_BASE *psBase)
+{
+	PVR_ASSERT(psBase);
+
+	/* Finally destroy the idr */
+	idr_destroy(&psBase->sIdr);
+
+	OSFreeMem(psBase);
+
+	return PVRSRV_OK;
+}
+
+
+static const HANDLE_IMPL_FUNCTAB g_sHandleFuncTab =
+{
+	.pfnAcquireHandle = AcquireHandle,
+	.pfnReleaseHandle = ReleaseHandle,
+	.pfnGetHandleData = GetHandleData,
+	.pfnSetHandleData = SetHandleData,
+	.pfnIterateOverHandles = IterateOverHandles,
+	.pfnEnableHandlePurging = EnableHandlePurging,
+	.pfnPurgeHandles = PurgeHandles,
+	.pfnCreateHandleBase = CreateHandleBase,
+	.pfnDestroyHandleBase = DestroyHandleBase
+};
+
+PVRSRV_ERROR PVRSRVHandleGetFuncTable(HANDLE_IMPL_FUNCTAB const **ppsFuncs)
+{
+	static IMG_BOOL bAcquired = IMG_FALSE;
+
+	if (bAcquired)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Function table already acquired",
+			 __func__));
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	if (ppsFuncs == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*ppsFuncs = &g_sHandleFuncTab;
+
+	bAcquired = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img-rogue/handle_impl.h b/drivers/gpu/drm/img-rogue/handle_impl.h
new file mode 100644
index 000000000000..94305979d130
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/handle_impl.h
@@ -0,0 +1,89 @@
+/**************************************************************************/ /*!
+@File
+@Title          Implementation Callbacks for Handle Manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the handle manager API. This file is for declarations
+                and definitions that are private/internal to the handle manager
+                API but need to be shared between the generic handle manager
+                code and the various handle manager backends, i.e. the code that
+                implements the various callbacks.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(HANDLE_IMPL_H)
+#define HANDLE_IMPL_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+typedef struct _HANDLE_IMPL_BASE_ HANDLE_IMPL_BASE;
+
+typedef PVRSRV_ERROR (*PFN_HANDLE_ITER)(IMG_HANDLE hHandle, void *pvData);
+
+typedef struct _HANDLE_IMPL_FUNCTAB_
+{
+	/* Acquire a new handle which is associated with the given data */
+	PVRSRV_ERROR (*pfnAcquireHandle)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE *phHandle, void *pvData);
+
+	/* Release the given handle (optionally returning the data associated with it) */
+	PVRSRV_ERROR (*pfnReleaseHandle)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void **ppvData);
+
+	/* Get the data associated with the given handle */
+	PVRSRV_ERROR (*pfnGetHandleData)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void **ppvData);
+
+	/* Set the data associated with the given handle */
+	PVRSRV_ERROR (*pfnSetHandleData)(HANDLE_IMPL_BASE *psHandleBase, IMG_HANDLE hHandle, void *pvData);
+
+	PVRSRV_ERROR (*pfnIterateOverHandles)(HANDLE_IMPL_BASE *psHandleBase, PFN_HANDLE_ITER pfnHandleIter, void *pvHandleIterData);
+
+	/* Enable handle purging on the given handle base */
+	PVRSRV_ERROR (*pfnEnableHandlePurging)(HANDLE_IMPL_BASE *psHandleBase);
+
+	/* Purge handles on the given handle base */
+	PVRSRV_ERROR (*pfnPurgeHandles)(HANDLE_IMPL_BASE *psHandleBase);
+
+	/* Create handle base */
+	PVRSRV_ERROR (*pfnCreateHandleBase)(HANDLE_IMPL_BASE **psHandleBase);
+
+	/* Destroy handle base */
+	PVRSRV_ERROR (*pfnDestroyHandleBase)(HANDLE_IMPL_BASE *psHandleBase);
+} HANDLE_IMPL_FUNCTAB;
+
+PVRSRV_ERROR PVRSRVHandleGetFuncTable(HANDLE_IMPL_FUNCTAB const **ppsFuncs);
+
+#endif /* !defined(HANDLE_IMPL_H) */
diff --git a/drivers/gpu/drm/img-rogue/handle_types.h b/drivers/gpu/drm/img-rogue/handle_types.h
new file mode 100644
index 000000000000..75ac5c5fa6a0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/handle_types.h
@@ -0,0 +1,86 @@
+/**************************************************************************/ /*!
+@File
+@Title          Handle Manager handle types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provide handle management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+/* NOTE: Do not add include guards to this file */
+
+HANDLETYPE(NONE)
+HANDLETYPE(SHARED_EVENT_OBJECT)
+HANDLETYPE(EVENT_OBJECT_CONNECT)
+HANDLETYPE(PMR_LOCAL_EXPORT_HANDLE)
+HANDLETYPE(PHYSMEM_PMR)
+HANDLETYPE(PHYSMEM_PMR_EXPORT)
+HANDLETYPE(PHYSMEM_PMR_SECURE_EXPORT)
+HANDLETYPE(DEVMEMINT_CTX)
+HANDLETYPE(DEVMEMINT_CTX_EXPORT)
+HANDLETYPE(DEVMEMINT_HEAP)
+HANDLETYPE(DEVMEMINT_RESERVATION)
+HANDLETYPE(DEVMEMXINT_RESERVATION)
+HANDLETYPE(DEVMEMINT_MAPPING)
+HANDLETYPE(RGX_FW_MEMDESC)
+HANDLETYPE(RGX_FREELIST)
+HANDLETYPE(RGX_MEMORY_BLOCK)
+HANDLETYPE(RGX_SERVER_RENDER_CONTEXT)
+HANDLETYPE(RGX_SERVER_TQ_CONTEXT)
+HANDLETYPE(RGX_SERVER_TQ_TDM_CONTEXT)
+HANDLETYPE(RGX_SERVER_COMPUTE_CONTEXT)
+HANDLETYPE(RGX_SERVER_RAY_CONTEXT)
+HANDLETYPE(RGX_SERVER_KICKSYNC_CONTEXT)
+HANDLETYPE(SYNC_PRIMITIVE_BLOCK)
+HANDLETYPE(SYNC_RECORD_HANDLE)
+HANDLETYPE(PVRSRV_TIMELINE_SERVER)
+HANDLETYPE(PVRSRV_FENCE_SERVER)
+HANDLETYPE(PVRSRV_FENCE_EXPORT)
+HANDLETYPE(RGX_KM_HW_RT_DATASET)
+HANDLETYPE(RGX_FWIF_ZSBUFFER)
+HANDLETYPE(RGX_POPULATION)
+HANDLETYPE(DC_DEVICE)
+HANDLETYPE(DC_DISPLAY_CONTEXT)
+HANDLETYPE(DC_BUFFER)
+HANDLETYPE(DC_PIN_HANDLE)
+HANDLETYPE(DEVMEM_MEM_IMPORT)
+HANDLETYPE(PHYSMEM_PMR_PAGELIST)
+HANDLETYPE(PVR_TL_SD)
+HANDLETYPE(RI_HANDLE)
+HANDLETYPE(DEV_PRIV_DATA)
+HANDLETYPE(MM_PLAT_CLEANUP)
+HANDLETYPE(WORKEST_RETURN_DATA)
+HANDLETYPE(DI_CONTEXT)
diff --git a/drivers/gpu/drm/img-rogue/hash.c b/drivers/gpu/drm/img-rogue/hash.c
new file mode 100644
index 000000000000..434c6e967bc4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/hash.c
@@ -0,0 +1,747 @@
+/*************************************************************************/ /*!
+@File
+@Title          Self scaling hash tables.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+   Implements simple self scaling hash tables. Hash collisions are handled by
+   chaining entries together. Hash tables are increased in size when they
+   become more than (50%?) full and decreased in size when less than (25%?)
+   full. Hash tables are never decreased below their initial size.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* include/ */
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+/* services/shared/include/ */
+#include "hash.h"
+
+/* services/client/include/ or services/server/include/ */
+#include "osfunc_common.h"
+#include "allocmem.h"
+
+//#define PERF_DBG_RESIZE
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+#include <sys/time.h>
+#endif
+
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#endif
+
+#define	KEY_TO_INDEX(pHash, key, uSize) \
+	((pHash)->pfnHashFunc((pHash)->uKeySize, (key), (uSize)) % (uSize))
+
+#define	KEY_COMPARE(pHash, pKey1, pKey2) \
+	((pHash)->pfnKeyComp((pHash)->uKeySize, (pKey1), (pKey2)))
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define _AllocMem OSAllocMemNoStats
+#define _AllocZMem OSAllocZMemNoStats
+#define _FreeMem OSFreeMemNoStats
+#else
+#define _AllocMem OSAllocMem
+#define _AllocZMem OSAllocZMem
+#define _FreeMem OSFreeMem
+#endif
+
+#define NO_SHRINK 0
+
+/* Each entry in a hash table is placed into a bucket */
+typedef struct _BUCKET_
+{
+	struct _BUCKET_ *pNext; /*!< the next bucket on the same chain */
+	uintptr_t v;            /*!< entry value */
+	uintptr_t k[];          /* PRQA S 0642 */
+	                        /* override dynamic array declaration warning */
+} BUCKET;
+
+struct _HASH_TABLE_
+{
+	IMG_UINT32 uSize;            /*!< current size of the hash table */
+	IMG_UINT32 uCount;           /*!< number of entries currently in the hash table */
+	IMG_UINT32 uMinimumSize;     /*!< the minimum size that the hash table should be re-sized to */
+	IMG_UINT32 uKeySize;         /*!< size of key in bytes */
+	IMG_UINT32 uShrinkThreshold; /*!< The threshold at which to trigger a shrink */
+	IMG_UINT32 uGrowThreshold;   /*!< The threshold at which to trigger a grow */
+	HASH_FUNC*     pfnHashFunc;  /*!< hash function */
+	HASH_KEY_COMP* pfnKeyComp;   /*!< key comparison function */
+	BUCKET**   ppBucketTable;    /*!< the hash table array */
+#if defined(DEBUG)
+	const char*      pszFile;
+	unsigned int     ui32LineNum;
+#endif
+};
+
+/*************************************************************************/ /*!
+@Function       HASH_Func_Default
+@Description    Hash function intended for hashing keys composed of uintptr_t
+                arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey         A pointer to the key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_UINT32
+HASH_Func_Default(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	uintptr_t *p = (uintptr_t *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(uintptr_t);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	PVR_ASSERT((uKeySize % sizeof(uintptr_t)) == 0);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = (IMG_UINT32)*p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares keys composed of uintptr_t arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey1        Pointer to first hash key to compare.
+@Input          pKey2        Pointer to second hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Key_Comp_Default(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	uintptr_t *p1 = (uintptr_t *)pKey1;
+	uintptr_t *p2 = (uintptr_t *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(uintptr_t);
+	IMG_UINT32 ui;
+
+	PVR_ASSERT((uKeySize % sizeof(uintptr_t)) == 0);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _ChainInsert
+@Description    Insert a bucket into the appropriate hash table chain.
+@Input          pBucket       The bucket
+@Input          ppBucketTable The hash table
+@Input          uSize         The size of the hash table
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static void
+_ChainInsert(HASH_TABLE *pHash, BUCKET *pBucket, BUCKET **ppBucketTable, IMG_UINT32 uSize)
+{
+	IMG_UINT32 uIndex;
+
+	/* We assume that all parameters passed by the caller are valid. */
+	PVR_ASSERT(pBucket != NULL);
+	PVR_ASSERT(ppBucketTable != NULL);
+	PVR_ASSERT(uSize != 0);
+
+	uIndex = KEY_TO_INDEX(pHash, pBucket->k, uSize);	/* PRQA S 0432,0541 */ /* ignore dynamic array warning */
+	pBucket->pNext = ppBucketTable[uIndex];
+	ppBucketTable[uIndex] = pBucket;
+}
+
+/*************************************************************************/ /*!
+@Function       _Rehash
+@Description    Iterate over every entry in an old hash table and rehash into
+                the new table.
+@Input          ppOldTable   The old hash table
+@Input          uOldSize     The size of the old hash table
+@Input          ppNewTable   The new hash table
+@Input          uNewSize     The size of the new hash table
+@Return         None
+*/ /**************************************************************************/
+static void
+_Rehash(HASH_TABLE *pHash,
+		 BUCKET **ppOldTable, IMG_UINT32 uOldSize,
+		 BUCKET **ppNewTable, IMG_UINT32 uNewSize)
+{
+	IMG_UINT32 uIndex;
+	for (uIndex=0; uIndex< uOldSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		pBucket = ppOldTable[uIndex];
+		while (pBucket != NULL)
+		{
+			BUCKET *pNextBucket = pBucket->pNext;
+			_ChainInsert(pHash, pBucket, ppNewTable, uNewSize);
+			pBucket = pNextBucket;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       _Resize
+@Description    Attempt to resize a hash table, failure to allocate a new
+                larger hash table is not considered a hard failure. We simply
+                continue and allow the table to fill up, the effect is to
+                allow hash chains to become longer.
+@Input          pHash        Hash table to resize.
+@Input          uNewSize     Required table size.
+@Return         IMG_TRUE Success
+                IMG_FALSE Failed
+*/ /**************************************************************************/
+static IMG_BOOL
+_Resize(HASH_TABLE *pHash, IMG_UINT32 uNewSize)
+{
+	BUCKET **ppNewTable;
+	IMG_UINT32 uiThreshold = uNewSize >> 2;
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	struct timeval start, end;
+#endif
+
+	if (uNewSize == pHash->uSize)
+	{
+		return IMG_TRUE;
+	}
+
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	gettimeofday(&start, NULL);
+#endif
+
+	ppNewTable = _AllocZMem(sizeof(BUCKET *) * uNewSize);
+	if (ppNewTable == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	_Rehash(pHash, pHash->ppBucketTable, pHash->uSize, ppNewTable, uNewSize);
+
+	_FreeMem(pHash->ppBucketTable);
+
+#if !defined(__KERNEL__) && defined(PERF_DBG_RESIZE)
+	gettimeofday(&end, NULL);
+	if (start.tv_usec > end.tv_usec)
+	{
+		end.tv_usec = 1000000 - start.tv_usec + end.tv_usec;
+	}
+	else
+	{
+		end.tv_usec -= start.tv_usec;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: H:%p O:%d N:%d C:%d G:%d S:%d T:%06luus", __func__, pHash, pHash->uSize, uNewSize, pHash->uCount, pHash->uGrowThreshold, pHash->uShrinkThreshold, end.tv_usec));
+#endif
+
+	/*not nulling pointer, being reassigned just below*/
+	pHash->ppBucketTable = ppNewTable;
+	pHash->uSize = uNewSize;
+
+	pHash->uGrowThreshold = uiThreshold * 3;
+	pHash->uShrinkThreshold = (uNewSize <= pHash->uMinimumSize) ? NO_SHRINK : uiThreshold;
+
+	return IMG_TRUE;
+}
+
+
+/*************************************************************************/ /*!
+@Function       HASH_Create_Extended
+@Description    Create a self scaling hash table, using the supplied key size,
+                and the supplied hash and key comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Input          uKeySize     The size of the key, in bytes.
+@Input          pfnHashFunc  Pointer to hash function.
+@Input          pfnKeyComp   Pointer to key comparison function.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Extended_Int (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp)
+{
+	HASH_TABLE *pHash;
+
+	if (uInitialLen == 0 || uKeySize == 0 || pfnHashFunc == NULL || pfnKeyComp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid input parameters", __func__));
+		return NULL;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: InitialSize=0x%x", __func__, uInitialLen));
+
+	pHash = _AllocMem(sizeof(HASH_TABLE));
+	if (pHash == NULL)
+	{
+		return NULL;
+	}
+
+	pHash->uCount = 0;
+	pHash->uSize = uInitialLen;
+	pHash->uMinimumSize = uInitialLen;
+	pHash->uKeySize = uKeySize;
+	pHash->uGrowThreshold = (uInitialLen >> 2) * 3;
+	pHash->uShrinkThreshold = NO_SHRINK;
+	pHash->pfnHashFunc = pfnHashFunc;
+	pHash->pfnKeyComp = pfnKeyComp;
+
+	pHash->ppBucketTable = _AllocZMem(sizeof(BUCKET *) * pHash->uSize);
+	if (pHash->ppBucketTable == NULL)
+	{
+		_FreeMem(pHash);
+		/*not nulling pointer, out of scope*/
+		return NULL;
+	}
+
+	return pHash;
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Extended_Debug (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp,
+										 const char *file, const unsigned int line)
+{
+	HASH_TABLE *hash;
+	hash = HASH_Create_Extended_Int(uInitialLen, uKeySize,
+									pfnHashFunc, pfnKeyComp);
+	if (hash)
+	{
+		hash->pszFile = file;
+		hash->ui32LineNum = line;
+	}
+	return hash;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Create
+@Description    Create a self scaling hash table with a key consisting of a
+                single uintptr_t, and using the default hash and key
+                comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Int (IMG_UINT32 uInitialLen)
+{
+	return HASH_Create_Extended_Int(uInitialLen, sizeof(uintptr_t),
+									&HASH_Func_Default, &HASH_Key_Comp_Default);
+}
+
+#if defined(DEBUG)
+IMG_INTERNAL
+HASH_TABLE * HASH_Create_Debug(IMG_UINT32 uInitialLen, const char *file, const unsigned int line)
+{
+	HASH_TABLE *hash;
+	hash = HASH_Create_Extended_Int(uInitialLen, sizeof(uintptr_t),
+									&HASH_Func_Default, &HASH_Key_Comp_Default);
+	if (hash)
+	{
+		hash->pszFile = file;
+		hash->ui32LineNum = line;
+	}
+	return hash;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete_Extended
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table should have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Input          bWarn        Set false to suppress warnings in the case of
+                             deletion with active entries.
+*/ /**************************************************************************/
+IMG_INTERNAL void
+HASH_Delete_Extended(HASH_TABLE *pHash, IMG_BOOL bWarn)
+{
+	IMG_BOOL bDoCheck = IMG_TRUE;
+#if defined(__KERNEL__) && !defined(__QNXNTO__)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData != NULL)
+	{
+		if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			bDoCheck = IMG_FALSE;
+		}
+	}
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	else
+	{
+		bDoCheck = IMG_FALSE;
+	}
+#endif
+#endif
+	if (pHash != NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HASH_Delete"));
+
+		if (bDoCheck)
+		{
+			PVR_ASSERT(pHash->uCount==0);
+		}
+		if (pHash->uCount != 0)
+		{
+			IMG_UINT32 i;
+			if (bWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Leak detected in hash table!", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Removing remaining %u hash entries.", __func__, pHash->uCount));
+#if defined(DEBUG)
+				PVR_DPF ((PVR_DBG_ERROR, "%s: Hash %p created at %s:%u.", __func__, (uintptr_t*)pHash, pHash->pszFile, pHash->ui32LineNum));
+#endif
+			}
+
+			for (i = 0; i < pHash->uSize; i++)
+			{
+				BUCKET *pBucket = pHash->ppBucketTable[i];
+				while (pBucket != NULL)
+				{
+					BUCKET *pNextBucket = pBucket->pNext;
+					_FreeMem(pBucket);
+					pBucket = pNextBucket;
+				}
+			}
+
+		}
+		_FreeMem(pHash->ppBucketTable);
+		pHash->ppBucketTable = NULL;
+		_FreeMem(pHash);
+		/*not nulling pointer, copy on stack*/
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table must have been removed
+                before calling this function.
+@Input          pHash        Hash table
+*/ /**************************************************************************/
+IMG_INTERNAL void
+HASH_Delete(HASH_TABLE *pHash)
+{
+	HASH_Delete_Extended(pHash, IMG_TRUE);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert_Extended
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to the key.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Insert_Extended(HASH_TABLE *pHash, void *pKey, uintptr_t v)
+{
+	BUCKET *pBucket;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter", __func__));
+		return IMG_FALSE;
+	}
+
+	pBucket = _AllocMem(sizeof(BUCKET) + pHash->uKeySize);
+	if (pBucket == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pBucket->v = v;
+	/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k (linux)*/
+	OSCachedMemCopy(pBucket->k, pKey, pHash->uKeySize);
+
+	_ChainInsert(pHash, pBucket, pHash->ppBucketTable, pHash->uSize);
+
+	pHash->uCount++;
+
+	/* check if we need to think about re-balancing */
+	if (pHash->uCount > pHash->uGrowThreshold)
+	{
+		/* Ignore the return code from _Resize because the hash table is
+		   still in a valid state and although not ideally sized, it is still
+		   functional */
+		_Resize(pHash, pHash->uSize << 1);
+	}
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+HASH_Insert(HASH_TABLE *pHash, uintptr_t k, uintptr_t v)
+{
+	return HASH_Insert_Extended(pHash, &k, v);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove_Extended
+@Description    Remove a key from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Remove_Extended(HASH_TABLE *pHash, void *pKey)
+{
+	BUCKET **ppBucket;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Null hash table", __func__));
+		return 0;
+	}
+
+	uIndex = KEY_TO_INDEX(pHash, pKey, pHash->uSize);
+
+	for (ppBucket = &(pHash->ppBucketTable[uIndex]); *ppBucket != NULL; ppBucket = &((*ppBucket)->pNext))
+	{
+		/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k */
+		if (KEY_COMPARE(pHash, (*ppBucket)->k, pKey))
+		{
+			BUCKET *pBucket = *ppBucket;
+			uintptr_t v = pBucket->v;
+			(*ppBucket) = pBucket->pNext;
+
+			_FreeMem(pBucket);
+			/*not nulling original pointer, already overwritten*/
+
+			pHash->uCount--;
+
+			/* check if we need to think about re-balancing, when the shrink
+			 * threshold is 0 we are at the minimum size, no further shrink */
+			if (pHash->uCount < pHash->uShrinkThreshold)
+			{
+				/* Ignore the return code from _Resize because the
+				   hash table is still in a valid state and although
+				   not ideally sized, it is still functional */
+				_Resize(pHash, MAX(pHash->uSize >> 1, pHash->uMinimumSize));
+			}
+
+			return v;
+		}
+	}
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove
+@Description    Remove a key value pair from a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Remove(HASH_TABLE *pHash, uintptr_t k)
+{
+	return HASH_Remove_Extended(pHash, &k);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve_Extended
+@Description    Retrieve a value from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Retrieve_Extended(HASH_TABLE *pHash, void *pKey)
+{
+	BUCKET **ppBucket;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pHash != NULL);
+
+	if (pHash == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Null hash table", __func__));
+		return 0;
+	}
+
+	uIndex = KEY_TO_INDEX(pHash, pKey, pHash->uSize);
+
+	for (ppBucket = &(pHash->ppBucketTable[uIndex]); *ppBucket != NULL; ppBucket = &((*ppBucket)->pNext))
+	{
+		/* PRQA S 0432,0541 1 */ /* ignore warning about dynamic array k */
+		if (KEY_COMPARE(pHash, (*ppBucket)->k, pKey))
+		{
+			BUCKET *pBucket = *ppBucket;
+			uintptr_t v = pBucket->v;
+
+			return v;
+		}
+	}
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve
+@Description    Retrieve a value from a hash table created with HASH_Create.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+IMG_INTERNAL uintptr_t
+HASH_Retrieve(HASH_TABLE *pHash, uintptr_t k)
+{
+	return HASH_Retrieve_Extended(pHash, &k);
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Iterate
+@Description    Iterate over every entry in the hash table.
+@Input          pHash        Hash table to iterate.
+@Input          pfnCallback  Callback to call with the key and data for each
+.                            entry in the hash table
+@Return         Callback error if any, otherwise PVRSRV_OK
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HASH_Iterate(HASH_TABLE *pHash, HASH_pfnCallback pfnCallback, void* args)
+{
+	IMG_UINT32 uIndex;
+	for (uIndex=0; uIndex < pHash->uSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		pBucket = pHash->ppBucketTable[uIndex];
+		while (pBucket != NULL)
+		{
+			PVRSRV_ERROR eError;
+			BUCKET *pNextBucket = pBucket->pNext;
+
+			eError = pfnCallback((uintptr_t) ((void *) *(pBucket->k)), pBucket->v, args);
+
+			/* The callback might want us to break out early */
+			if (eError != PVRSRV_OK)
+				return eError;
+
+			pBucket = pNextBucket;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       HASH_Count
+@Description    Retrieve the number of entries in the hash table.
+@Input          pHash        The hash table.
+@Return         The number of entries.
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_UINT32
+HASH_Count(HASH_TABLE *pHash)
+{
+	return pHash->uCount;
+}
+
+
+#ifdef HASH_TRACE
+/*************************************************************************/ /*!
+@Function       HASH_Dump
+@Description    Dump out some information about a hash table.
+@Input          pHash         The hash table.
+*/ /**************************************************************************/
+void
+HASH_Dump(HASH_TABLE *pHash)
+{
+	IMG_UINT32 uIndex;
+	IMG_UINT32 uMaxLength=0;
+	IMG_UINT32 uEmptyCount=0;
+
+	PVR_ASSERT(pHash != NULL);
+	for (uIndex=0; uIndex<pHash->uSize; uIndex++)
+	{
+		BUCKET *pBucket;
+		IMG_UINT32 uLength = 0;
+		if (pHash->ppBucketTable[uIndex] == NULL)
+		{
+			uEmptyCount++;
+		}
+		for (pBucket=pHash->ppBucketTable[uIndex];
+				pBucket != NULL;
+				pBucket = pBucket->pNext)
+		{
+			uLength++;
+		}
+		uMaxLength = MAX(uMaxLength, uLength);
+	}
+
+	PVR_TRACE(("hash table: uMinimumSize=%d  size=%d  count=%d",
+			   pHash->uMinimumSize, pHash->uSize, pHash->uCount));
+	PVR_TRACE(("  empty=%d  max=%d", uEmptyCount, uMaxLength));
+}
+#endif
diff --git a/drivers/gpu/drm/img-rogue/hash.h b/drivers/gpu/drm/img-rogue/hash.h
new file mode 100644
index 000000000000..feb879c1489a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/hash.h
@@ -0,0 +1,255 @@
+/*************************************************************************/ /*!
+@File
+@Title          Self scaling hash tables
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements simple self scaling hash tables.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HASH_H
+#define HASH_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * Keys passed to the comparison function are only guaranteed to be aligned on
+ * an uintptr_t boundary.
+ */
+typedef IMG_UINT32 HASH_FUNC(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+typedef IMG_BOOL HASH_KEY_COMP(size_t uKeySize, void *pKey1, void *pKey2);
+
+typedef struct _HASH_TABLE_ HASH_TABLE;
+
+typedef PVRSRV_ERROR (*HASH_pfnCallback) (
+	uintptr_t k,
+	uintptr_t v,
+	void* pvPriv
+);
+
+#if defined(DEBUG)
+#else
+#define HASH_CREATE(LEN)		HASH_Create(LEN)
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Func_Default
+@Description    Hash function intended for hashing keys composed of uintptr_t
+                arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey         A pointer to the key to hash.
+@Input          uHashTabLen  The length of the hash table.
+@Return         The hash value.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Func_Default(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+/*************************************************************************/ /*!
+@Function       HASH_Key_Comp_Default
+@Description    Compares keys composed of uintptr_t arrays.
+@Input          uKeySize     The size of the hash key, in bytes.
+@Input          pKey1        Pointer to first hash key to compare.
+@Input          pKey2        Pointer to second hash key to compare.
+@Return         IMG_TRUE  - The keys match.
+                IMG_FALSE - The keys don't match.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Key_Comp_Default(size_t uKeySize, void *pKey1, void *pKey2);
+
+/*************************************************************************/ /*!
+@Function       HASH_Create_Extended
+@Description    Create a self scaling hash table, using the supplied key size,
+                and the supplied hash and key comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Input          uKeySize     The size of the key, in bytes.
+@Input          pfnHashFunc  Pointer to hash function.
+@Input          pfnKeyComp   Pointer to key comparison function.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+HASH_TABLE * HASH_Create_Extended_Int(IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp);
+#if defined(DEBUG)
+#define HASH_Create_Extended(LEN, KS, FUN, CMP)		HASH_Create_Extended_Debug(LEN, KS, FUN, CMP, __FILE__, __LINE__)
+HASH_TABLE * HASH_Create_Extended_Debug (IMG_UINT32 uInitialLen, size_t uKeySize, HASH_FUNC *pfnHashFunc, HASH_KEY_COMP *pfnKeyComp,
+										 const char *file, const unsigned int line);
+#else
+#define HASH_Create_Extended	HASH_Create_Extended_Int
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Create
+@Description    Create a self scaling hash table with a key consisting of a
+                single uintptr_t, and using the default hash and key
+                comparison functions.
+@Input          uInitialLen  Initial and minimum length of the hash table,
+                             where the length refers to the number of entries
+                             in the hash table, not its size in bytes.
+@Return         NULL or hash table handle.
+*/ /**************************************************************************/
+HASH_TABLE * HASH_Create_Int(IMG_UINT32 uInitialLen);
+#if defined(DEBUG)
+#define HASH_Create(LEN)		HASH_Create_Debug(LEN, __FILE__, __LINE__)
+HASH_TABLE * HASH_Create_Debug (IMG_UINT32 uInitialLen, const char *file, const unsigned int line);
+#else
+#define HASH_Create				HASH_Create_Int
+#endif
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete_Extended
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table should have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Input          bWarn        Set false to suppress warnings in the case of
+                             deletion with active entries.
+@Return         None
+*/ /**************************************************************************/
+void HASH_Delete_Extended(HASH_TABLE *pHash, IMG_BOOL bWarn);
+
+/*************************************************************************/ /*!
+@Function       HASH_Delete
+@Description    Delete a hash table created by HASH_Create_Extended or
+                HASH_Create. All entries in the table must have been removed
+                before calling this function.
+@Input          pHash        Hash table
+@Return         None
+*/ /**************************************************************************/
+void HASH_Delete(HASH_TABLE *pHash);
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert_Extended
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to the key.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Insert_Extended(HASH_TABLE *pHash, void *pKey, uintptr_t v);
+
+/*************************************************************************/ /*!
+@Function       HASH_Insert
+@Description    Insert a key value pair into a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Input          v            The value associated with the key.
+@Return         IMG_TRUE - success.
+                IMG_FALSE - failure.
+*/ /**************************************************************************/
+IMG_BOOL HASH_Insert(HASH_TABLE *pHash, uintptr_t k, uintptr_t v);
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove_Extended
+@Description    Remove a key from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Remove_Extended(HASH_TABLE *pHash, void *pKey);
+
+/*************************************************************************/ /*!
+@Function       HASH_Remove
+@Description    Remove a key value pair from a hash table created with
+                HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Remove(HASH_TABLE *pHash, uintptr_t k);
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve_Extended
+@Description    Retrieve a value from a hash table created with
+                HASH_Create_Extended.
+@Input          pHash        The hash table.
+@Input          pKey         Pointer to key.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Retrieve_Extended(HASH_TABLE *pHash, void *pKey);
+
+/*************************************************************************/ /*!
+@Function       HASH_Retrieve
+@Description    Retrieve a value from a hash table created with HASH_Create.
+@Input          pHash        The hash table.
+@Input          k            The key value.
+@Return         0 if the key is missing, or the value associated with the key.
+*/ /**************************************************************************/
+uintptr_t HASH_Retrieve(HASH_TABLE *pHash, uintptr_t k);
+
+/*************************************************************************/ /*!
+@Function       HASH_Iterate
+@Description    Iterate over every entry in the hash table.
+@Input          pHash        Hash table to iterate.
+@Input          pfnCallback  Callback to call with the key and data for each
+.                            entry in the hash table
+@Return         Callback error if any, otherwise PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR HASH_Iterate(HASH_TABLE *pHash, HASH_pfnCallback pfnCallback, void* args);
+
+/*************************************************************************/ /*!
+@Function       HASH_Count
+@Description    Retrieve the number of entries in the hash table.
+@Input          pHash        The hash table.
+@Return         The number of entries.
+*/ /**************************************************************************/
+IMG_UINT32 HASH_Count(HASH_TABLE *pHash);
+
+#ifdef HASH_TRACE
+/*************************************************************************/ /*!
+@Function       HASH_Dump
+@Description    Dump out some information about a hash table.
+@Input          pHash         The hash table.
+*/ /**************************************************************************/
+void HASH_Dump(HASH_TABLE *pHash);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HASH_H */
+
+/******************************************************************************
+ End of file (hash.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/htb_debug.c b/drivers/gpu/drm/img-rogue/htb_debug.c
new file mode 100644
index 000000000000..6290fbd30b31
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htb_debug.c
@@ -0,0 +1,1218 @@
+/*************************************************************************/ /*!
+@File           htb_debug.c
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides kernel side debugFS Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxdevice.h"
+#include "htbserver.h"
+#include "htbuffer_types.h"
+#include "tlstream.h"
+#include "tlclient.h"
+#include "pvrsrv_tlcommon.h"
+#include "di_server.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "htb_debug.h"
+
+// Global data handles for buffer manipulation and processing
+
+typedef struct {
+	IMG_PBYTE	pBuf;		/* Raw data buffer from TL stream */
+	IMG_UINT32	uiBufLen;	/* Amount of data to process from 'pBuf' */
+	IMG_UINT32	uiTotal;	/* Total bytes processed */
+	IMG_UINT32	uiMsgLen;	/* Length of HTB message to be processed */
+	IMG_PBYTE	pCurr;		/* pointer to current message to be decoded */
+	IMG_CHAR	szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];	/* Output string */
+} HTB_Sentinel_t;
+
+typedef struct
+{
+	DI_ENTRY *psDumpHostDiEntry;  /* debug info entry */
+	HTB_Sentinel_t sSentinel;     /* private control structure for HTB DI
+	                                 operations */
+	IMG_HANDLE hStream;           /* stream handle for debugFS use */
+} HTB_DBG_INFO;
+
+static HTB_DBG_INFO g_sHTBData;
+
+// Comment out for extra debug level
+// #define HTB_CHATTY_PRINT(x) PVR_DPF(x)
+#define HTB_CHATTY_PRINT(x)
+
+typedef void (DI_PRINTF)(const OSDI_IMPL_ENTRY *, const IMG_CHAR *, ...);
+
+/******************************************************************************
+ * debugFS display routines
+ *****************************************************************************/
+static int HTBDumpBuffer(DI_PRINTF, OSDI_IMPL_ENTRY *, void *);
+
+static int _DebugHBTraceDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	int	retVal;
+
+	PVR_ASSERT(psEntry != NULL);
+
+	/* psEntry should never be NULL */
+	if (psEntry == NULL)
+	{
+		return -1;
+	}
+
+	/* Ensure that we have a valid address to use to dump info from. If NULL we
+	 * return a failure code to terminate the DI read call. pvData is either
+	 * DI_START_TOKEN (for the initial call) or an HTB buffer address for
+	 * subsequent calls [returned from the NEXT function]. */
+	if (pvData == NULL)
+	{
+		return -1;
+	}
+
+	retVal = HTBDumpBuffer(DIPrintf, psEntry, pvData);
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %d", __func__, retVal));
+
+	return retVal;
+}
+
+static IMG_UINT32 idToLogIdx(IMG_UINT32);	/* Forward declaration */
+
+/*
+ * HTB_GetNextMessage
+ *
+ * Get next non-empty message block from the buffer held in pSentinel->pBuf
+ * If we exhaust the data buffer we refill it (after releasing the previous
+ * message(s) [only one non-NULL message, but PAD messages will get released
+ * as we traverse them].
+ *
+ * Input:
+ *	pSentinel		references the already acquired data buffer
+ *
+ * Output:
+ *	pSentinel
+ *		-> uiMsglen updated to the size of the non-NULL message
+ *
+ * Returns:
+ *	Address of first non-NULL message in the buffer (if any)
+ *	NULL if there is no further data available from the stream and the buffer
+ *	contents have been drained.
+ */
+static IMG_PBYTE HTB_GetNextMessage(HTB_Sentinel_t *pSentinel)
+{
+	void	*pNext, *pLast, *pStart, *pData = NULL;
+	void	*pCurrent;		/* Current processing point within buffer */
+	PVRSRVTL_PPACKETHDR	ppHdr;	/* Current packet header */
+	IMG_UINT32	uiHdrType;		/* Packet header type */
+	IMG_UINT32	uiMsgSize;		/* Message size of current packet (bytes) */
+	IMG_BOOL	bUnrecognizedErrorPrinted = IMG_FALSE;
+	IMG_UINT32	ui32Data;
+	IMG_UINT32	ui32LogIdx;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(pSentinel != NULL);
+
+	pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+
+	pStart = pSentinel->pBuf;
+
+	pNext = pStart;
+	pSentinel->uiMsgLen = 0;	// Reset count for this message
+	uiMsgSize = 0;				// nothing processed so far
+	ui32LogIdx = HTB_SF_LAST;	// Loop terminator condition
+
+	do
+	{
+		/*
+		 * If we've drained the buffer we must RELEASE and ACQUIRE some more.
+		 */
+		if (pNext >= pLast)
+		{
+			eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream);
+			PVR_ASSERT(eError == PVRSRV_OK);
+
+			eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+				g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+			if (PVRSRV_OK != eError)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'", __func__,
+					"TLClientAcquireData", PVRSRVGETERRORSTRING(eError)));
+				return NULL;
+			}
+
+			// Reset our limits - if we've returned an empty buffer we're done.
+			pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+			pStart = pSentinel->pBuf;
+			pNext = pStart;
+
+			if (pStart == NULL || pLast == NULL)
+			{
+				return NULL;
+			}
+		}
+
+		/*
+		 * We should have a header followed by data block(s) in the stream.
+		 */
+
+		pCurrent = pNext;
+		ppHdr = GET_PACKET_HDR(pCurrent);
+
+		if (ppHdr == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Unexpected NULL packet in Host Trace buffer",
+			         __func__));
+			pSentinel->uiMsgLen += uiMsgSize;
+			return NULL;		// This should never happen
+		}
+
+		/*
+		 * This should *NEVER* fire. If it does it means we have got some
+		 * dubious packet header back from the HTB stream. In this case
+		 * the sensible thing is to abort processing and return to
+		 * the caller
+		 */
+		uiHdrType = GET_PACKET_TYPE(ppHdr);
+
+		PVR_ASSERT(uiHdrType < PVRSRVTL_PACKETTYPE_LAST &&
+			uiHdrType > PVRSRVTL_PACKETTYPE_UNDEF);
+
+		if (uiHdrType < PVRSRVTL_PACKETTYPE_LAST &&
+			uiHdrType > PVRSRVTL_PACKETTYPE_UNDEF)
+		{
+			/*
+			 * We have a (potentially) valid data header. We should see if
+			 * the associated packet header matches one of our expected
+			 * types.
+			 */
+			pNext = GET_NEXT_PACKET_ADDR(ppHdr);
+
+			PVR_ASSERT(pNext != NULL);
+
+			uiMsgSize = (IMG_UINT32)((size_t)pNext - (size_t)ppHdr);
+
+			pSentinel->uiMsgLen += uiMsgSize;
+
+			pData = GET_PACKET_DATA_PTR(ppHdr);
+
+			/*
+			 * Handle non-DATA packet types. These include PAD fields which
+			 * may have data associated and other types. We simply discard
+			 * these as they have no decodable information within them.
+			 */
+			if (uiHdrType != PVRSRVTL_PACKETTYPE_DATA)
+			{
+				/*
+				 * Now release the current non-data packet and proceed to the
+				 * next entry (if any).
+				 */
+				eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+				    g_sHTBData.hStream, uiMsgSize);
+
+				HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Packet Type %x "
+				                 "Length %u", __func__, uiHdrType, uiMsgSize));
+
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - '%s' message"
+						" size %u", __func__, "TLClientReleaseDataLess",
+						PVRSRVGETERRORSTRING(eError), uiMsgSize));
+				}
+
+				eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+					g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+				if (PVRSRV_OK != eError)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - %s Giving up",
+						__func__, "TLClientAcquireData",
+						PVRSRVGETERRORSTRING(eError)));
+
+					return NULL;
+				}
+				pSentinel->uiMsgLen = 0;
+				// Reset our limits - if we've returned an empty buffer we're done.
+				pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+				pStart = pSentinel->pBuf;
+				pNext = pStart;
+
+				if (pStart == NULL || pLast == NULL)
+				{
+					return NULL;
+				}
+				continue;
+			}
+			if (pData == NULL || pData >= pLast)
+			{
+				continue;
+			}
+			ui32Data = *(IMG_UINT32 *)pData;
+			ui32LogIdx = idToLogIdx(ui32Data);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Unexpected Header @%p value %x",
+				ppHdr, uiHdrType));
+
+			return NULL;
+		}
+
+		/*
+		 * Check if the unrecognized ID is valid and therefore, tracebuf
+		 * needs updating.
+		 */
+		if (HTB_SF_LAST == ui32LogIdx && HTB_LOG_VALIDID(ui32Data)
+			&& IMG_FALSE == bUnrecognizedErrorPrinted)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			    "%s: Unrecognised LOG value '%x' GID %x Params %d ID %x @ '%p'",
+			    __func__, ui32Data, HTB_SF_GID(ui32Data),
+			    HTB_SF_PARAMNUM(ui32Data), ui32Data & 0xfff, pData));
+			bUnrecognizedErrorPrinted = IMG_TRUE;
+		}
+
+	} while (HTB_SF_LAST == ui32LogIdx);
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning data @ %p Log value '%x'",
+	                 __func__, pCurrent, ui32Data));
+
+	return pCurrent;
+}
+
+/*
+ * HTB_GetFirstMessage
+ *
+ * Called from START to obtain the buffer address of the first message within
+ * pSentinel->pBuf. Will ACQUIRE data if the buffer is empty.
+ *
+ * Input:
+ *	pSentinel
+ *	pui64Pos			Offset within the debugFS file
+ *
+ * Output:
+ *	pSentinel->pCurr	Set to reference the first valid non-NULL message within
+ *						the buffer. If no valid message is found set to NULL.
+ *	pSentinel
+ *		->pBuf		if unset on entry
+ *		->uiBufLen	if pBuf unset on entry
+ *
+ * Side-effects:
+ *	HTB TL stream will be updated to bypass any zero-length PAD messages before
+ *	the first non-NULL message (if any).
+ */
+static void HTB_GetFirstMessage(HTB_Sentinel_t *pSentinel, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_ERROR	eError;
+
+	PVR_UNREFERENCED_PARAMETER(pui64Pos);
+
+	if (pSentinel == NULL)
+		return;
+
+	if (pSentinel->pBuf == NULL)
+	{
+		/* Acquire data */
+		pSentinel->uiMsgLen = 0;
+
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		    g_sHTBData.hStream, &pSentinel->pBuf, &pSentinel->uiBufLen);
+
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'",
+			    __func__, "TLClientAcquireData", PVRSRVGETERRORSTRING(eError)));
+
+			pSentinel->pBuf = NULL;
+			pSentinel->pCurr = NULL;
+		}
+		else
+		{
+			/*
+			 * If there is no data available we set pSentinel->pCurr to NULL
+			 * and return. This is expected behaviour if we've drained the
+			 * data and nothing else has yet been produced.
+			 */
+			if (pSentinel->uiBufLen == 0 || pSentinel->pBuf == NULL)
+			{
+				HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Empty Buffer @ %p",
+				                 __func__, pSentinel->pBuf));
+
+				pSentinel->pCurr = NULL;
+				return;
+			}
+		}
+	}
+
+	/* Locate next message within buffer. NULL => no more data to process */
+	pSentinel->pCurr = HTB_GetNextMessage(pSentinel);
+}
+
+/*
+ * _DebugHBTraceDIStart:
+ *
+ * Returns the address to use for subsequent 'Show', 'Next', 'Stop' file ops.
+ * Return DI_START_TOKEN for the very first call and allocate a sentinel for
+ * use by the 'Show' routine and its helpers.
+ * This is stored in the psEntry's private hook field.
+ *
+ * We obtain access to the TLstream associated with the HTB. If this doesn't
+ * exist (because no pvrdebug capture trace has been set) we simply return with
+ * a NULL value which will stop the DI traversal.
+ */
+static void *_DebugHBTraceDIStart(OSDI_IMPL_ENTRY *psEntry,
+                                  IMG_UINT64 *pui64Pos)
+{
+	HTB_Sentinel_t	*pSentinel = DIGetPrivData(psEntry);
+	PVRSRV_ERROR	eError;
+	IMG_UINT32		uiTLMode;
+	void			*retVal;
+	IMG_HANDLE		hStream;
+
+	/* The sentinel object should have been allocated during the creation
+	 * of the DI entry. If it's not there it means that something went
+	 * wrong. Return NULL in such case. */
+	if (pSentinel == NULL)
+	{
+		return NULL;
+	}
+
+	/* Check to see if the HTB stream has been configured yet. If not, there is
+	 * nothing to display so we just return NULL to stop the stream access.
+	 */
+	if (!HTBIsConfigured())
+	{
+		return NULL;
+	}
+
+	/* Open the stream in non-blocking mode so that we can determine if there
+	 * is no data to consume. Also disable the producer callback (if any) and
+	 * the open callback so that we do not generate spurious trace data when
+	 * accessing the stream.
+	 */
+	uiTLMode = PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING|
+			   PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK|
+			   PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK;
+
+	/* If two or more processes try to read from this file at the same time
+	 * the TLClientOpenStream() function will handle this by allowing only
+	 * one of them to actually open the stream. The other process will get
+	 * an error stating that the stream is already open. The open function
+	 * is threads safe. */
+	eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE, HTB_STREAM_NAME, uiTLMode,
+	                            &hStream);
+
+	if (eError == PVRSRV_ERROR_ALREADY_OPEN)
+	{
+		/* Stream allows only one reader so return error if it's already
+		 * opened. */
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Stream handle %p already "
+		                 "exists for %s", __func__, g_sHTBData.hStream,
+		                 HTB_STREAM_NAME));
+		return NULL;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		/*
+		 * No stream available so nothing to report
+		 */
+		return NULL;
+	}
+
+	/* There is a window where hStream can be NULL but the stream is already
+	 * opened. This shouldn't matter since the TLClientOpenStream() will make
+	 * sure that only one stream can be opened and only one process can reach
+	 * this place at a time. Also the .stop function will be always called
+	 * after this function returns so there should be no risk of stream
+	 * not being closed. */
+	PVR_ASSERT(g_sHTBData.hStream == NULL);
+	g_sHTBData.hStream = hStream;
+
+	/* We're starting the read operation so ensure we properly zero the
+	 * sentinel object. */
+	memset(pSentinel, 0, sizeof(*pSentinel));
+
+	/*
+	 * Find the first message location within pSentinel->pBuf
+	 * => for DI_START_TOKEN we must issue our first ACQUIRE, also for the
+	 * subsequent re-START calls (if any).
+	 */
+
+	HTB_GetFirstMessage(pSentinel, pui64Pos);
+
+	retVal = *pui64Pos == 0 ? DI_START_TOKEN : pSentinel->pCurr;
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %p, Stream %s @ %p",
+	                 __func__, retVal, HTB_STREAM_NAME, g_sHTBData.hStream));
+
+	return retVal;
+}
+
+/*
+ * _DebugTBTraceDIStop:
+ *
+ * Stop processing data collection and release any previously allocated private
+ * data structure if we have exhausted the previously filled data buffers.
+ */
+static void _DebugHBTraceDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+	IMG_UINT32 uiMsgLen;
+	PVRSRV_ERROR eError;
+
+	if (pSentinel == NULL)
+	{
+		return;
+	}
+
+	uiMsgLen = pSentinel->uiMsgLen;
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: MsgLen = %d", __func__, uiMsgLen));
+
+	/* If we get here the handle should never be NULL because
+	 * _DebugHBTraceDIStart() shouldn't allow that. */
+	if (g_sHTBData.hStream == NULL)
+	{
+		return;
+	}
+
+	if (uiMsgLen != 0)
+	{
+		eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+		                                 g_sHTBData.hStream, uiMsgLen);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED - %s, nBytes %u",
+			        __func__, "TLClientReleaseDataLess",
+			        PVRSRVGETERRORSTRING(eError), uiMsgLen));
+		}
+	}
+
+	eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()",
+		        "TLClientCloseStream", PVRSRVGETERRORSTRING(eError),
+		        __func__));
+	}
+
+	g_sHTBData.hStream = NULL;
+}
+
+
+/*
+ * _DebugHBTraceDINext:
+ *
+ * This is where we release any acquired data which has been processed by the
+ * DIShow routine. If we have encountered a DI entry overflow we stop
+ * processing and return NULL. Otherwise we release the message that we
+ * previously processed and simply update our position pointer to the next
+ * valid HTB message (if any)
+ */
+static void *_DebugHBTraceDINext(OSDI_IMPL_ENTRY *psEntry, void *pvPriv,
+                                 IMG_UINT64 *pui64Pos)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+	IMG_UINT64 ui64CurPos;
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+	if (pui64Pos)
+	{
+		ui64CurPos = *pui64Pos;
+		*pui64Pos = ui64CurPos + 1;
+	}
+
+	/* Determine if we've had an overflow on the previous 'Show' call. If so
+	 * we leave the previously acquired data in the queue (by releasing 0 bytes)
+	 * and return NULL to end this DI entry iteration.
+	 * If we have not overflowed we simply get the next HTB message and use that
+	 * for our display purposes. */
+
+	if (DIHasOverflowed(psEntry))
+	{
+		(void) TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE, g_sHTBData.hStream,
+		                               0);
+
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: OVERFLOW - returning NULL",
+		                 __func__));
+
+		return NULL;
+	}
+	else
+	{
+		eError = TLClientReleaseDataLess(DIRECT_BRIDGE_HANDLE,
+		                                 g_sHTBData.hStream,
+		                                 pSentinel->uiMsgLen);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s' @ %p Length %d",
+			        __func__, "TLClientReleaseDataLess",
+			        PVRSRVGETERRORSTRING(eError), pSentinel->pCurr,
+			        pSentinel->uiMsgLen));
+			PVR_DPF((PVR_DBG_WARNING, "%s: Buffer @ %p..%p", __func__,
+			        pSentinel->pBuf,
+			        (IMG_PBYTE) (pSentinel->pBuf + pSentinel->uiBufLen)));
+
+		}
+
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		                             g_sHTBData.hStream, &pSentinel->pBuf,
+		                             &pSentinel->uiBufLen);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: %s FAILED '%s'\nPrev message len %d",
+			        __func__, "TLClientAcquireData",
+			        PVRSRVGETERRORSTRING(eError), pSentinel->uiMsgLen));
+			pSentinel->pBuf = NULL;
+		}
+
+		pSentinel->uiMsgLen = 0; /* We don't (yet) know the message size */
+	}
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Returning %p Msglen %d", __func__,
+	                 pSentinel->pBuf, pSentinel->uiMsgLen));
+
+	if (pSentinel->pBuf == NULL || pSentinel->uiBufLen == 0)
+	{
+		return NULL;
+	}
+
+	pSentinel->pCurr = HTB_GetNextMessage(pSentinel);
+
+	return pSentinel->pCurr;
+}
+
+/******************************************************************************
+ * HTB Dumping routines and definitions
+ *****************************************************************************/
+#define IS_VALID_FMT_STRING(FMT) (strchr(FMT, '%') != NULL)
+#define MAX_STRING_SIZE (128)
+
+typedef enum
+{
+	TRACEBUF_ARG_TYPE_INT,
+	TRACEBUF_ARG_TYPE_ERR,
+	TRACEBUF_ARG_TYPE_NONE
+} TRACEBUF_ARG_TYPE;
+
+/*
+ * Array of all Host Trace log IDs used to convert the tracebuf data
+ */
+typedef struct _HTB_TRACEBUF_LOG_ {
+	HTB_LOG_SFids eSFId;
+	IMG_CHAR      *pszName;
+	IMG_CHAR      *pszFmt;
+	IMG_UINT32    ui32ArgNum;
+	IMG_UINT32    ui32StrArgIdx; /* This is 1 indexed, if 0, there is not a string arg. */
+} HTB_TRACEBUF_LOG;
+
+static const HTB_TRACEBUF_LOG aLogs[] = {
+#define X(a, b, c, d, e, f) {HTB_LOG_CREATESFID(a,b,e,f), #c, d, e, f},
+	HTB_LOG_SFIDLIST
+#undef X
+};
+
+static const IMG_CHAR *aGroups[] = {
+#define X(A,B) #B,
+	HTB_LOG_SFGROUPLIST
+#undef X
+};
+static const IMG_UINT32 uiMax_aGroups = ARRAY_SIZE(aGroups) - 1;
+
+static TRACEBUF_ARG_TYPE ExtractOneArgFmt(IMG_CHAR **, IMG_CHAR *);
+/*
+ * ExtractOneArgFmt
+ *
+ * Scan the input 'printf-like' string *ppszFmt and return the next
+ * value string to be displayed. If there is no '%' format field in the
+ * string we return 'TRACEBUF_ARG_TYPE_NONE' and leave the input string
+ * untouched.
+ *
+ * Input
+ *	ppszFmt          reference to format string to be decoded
+ *	pszOneArgFmt     single field format from *ppszFmt
+ *
+ * Returns
+ *	TRACEBUF_ARG_TYPE_ERR       unrecognised argument
+ *	TRACEBUF_ARG_TYPE_INT       variable is of numeric type
+ *	TRACEBUF_ARG_TYPE_NONE      no variable reference in *ppszFmt
+ *
+ * Side-effect
+ *	*ppszFmt is updated to reference the next part of the format string
+ *	to be scanned
+ */
+static TRACEBUF_ARG_TYPE ExtractOneArgFmt(
+	IMG_CHAR **ppszFmt,
+	IMG_CHAR *pszOneArgFmt)
+{
+	IMG_CHAR          *pszFmt;
+	IMG_CHAR          *psT;
+	IMG_UINT32        ui32Count = MAX_STRING_SIZE;
+	IMG_UINT32        ui32OneArgSize;
+	TRACEBUF_ARG_TYPE eRet = TRACEBUF_ARG_TYPE_ERR;
+
+	if (NULL == ppszFmt)
+		return TRACEBUF_ARG_TYPE_ERR;
+
+	pszFmt = *ppszFmt;
+	if (NULL == pszFmt)
+		return TRACEBUF_ARG_TYPE_ERR;
+
+	/*
+	 * Find the first '%'
+	 * NOTE: we can be passed a simple string to display which will have no
+	 * parameters embedded within it. In this case we simply return
+	 * TRACEBUF_ARG_TYPE_NONE and the string contents will be the full pszFmt
+	 */
+	psT = strchr(pszFmt, '%');
+	if (psT == NULL)
+	{
+		return TRACEBUF_ARG_TYPE_NONE;
+	}
+
+	/* Find next conversion identifier after the initial '%' */
+	while ((*psT++) && (ui32Count-- > 0))
+	{
+		switch (*psT)
+		{
+			case 'd':
+			case 'i':
+			case 'o':
+			case 'u':
+			case 'x':
+			case 'X':
+			{
+				eRet = TRACEBUF_ARG_TYPE_INT;
+				goto _found_arg;
+			}
+			case 's':
+			{
+				eRet = TRACEBUF_ARG_TYPE_ERR;
+				goto _found_arg;
+			}
+		}
+	}
+
+	if ((psT == NULL) || (ui32Count == 0)) return TRACEBUF_ARG_TYPE_ERR;
+
+_found_arg:
+	ui32OneArgSize = psT - pszFmt + 1;
+	OSCachedMemCopy(pszOneArgFmt, pszFmt, ui32OneArgSize);
+	pszOneArgFmt[ui32OneArgSize] = '\0';
+
+	*ppszFmt = psT + 1;
+
+	return eRet;
+}
+
+static IMG_UINT32 idToLogIdx(IMG_UINT32 ui32CheckData)
+{
+	IMG_UINT32	i = 0;
+	for (i = 0; aLogs[i].eSFId != HTB_SF_LAST; i++)
+	{
+		if ( ui32CheckData == aLogs[i].eSFId )
+			return i;
+	}
+	/* Nothing found, return max value */
+	return HTB_SF_LAST;
+}
+
+/*
+ * DecodeHTB
+ *
+ * Decode the data buffer message located at pBuf. This should be a valid
+ * HTB message as we are provided with the start of the buffer. If empty there
+ * is no message to process. We update the uiMsgLen field with the size of the
+ * HTB message that we have processed so that it can be returned to the system
+ * on successful logging of the message to the output file.
+ *
+ *	Input
+ *		pSentinel reference to newly read data and pending completion data
+ *		          from a previous invocation [handle DI entry buffer overflow]
+ *		 -> pBuf         reference to raw data that we are to parse
+ *		 -> uiBufLen     total number of bytes of data available
+ *		 -> pCurr        start of message to decode
+ *
+ *		pvDumpDebugFile     output file
+ *		pfnDumpDebugPrintf  output generating routine
+ *
+ * Output
+ *		pSentinel
+ *		 -> uiMsgLen	length of the decoded message which will be freed to
+ *						the system on successful completion of the DI entry
+ *						update via _DebugHBTraceDINext(),
+ * Return Value
+ *		0				successful decode
+ *		-1				unsuccessful decode
+ */
+static int
+DecodeHTB(HTB_Sentinel_t *pSentinel, OSDI_IMPL_ENTRY *pvDumpDebugFile,
+          DI_PRINTF pfnDumpDebugPrintf)
+{
+	IMG_UINT32	ui32Data, ui32LogIdx, ui32ArgsCur;
+	IMG_CHAR	*pszFmt = NULL;
+	IMG_CHAR	aszOneArgFmt[MAX_STRING_SIZE];
+	IMG_BOOL	bUnrecognizedErrorPrinted = IMG_FALSE;
+
+	size_t	nPrinted;
+
+	void	*pNext, *pLast, *pStart, *pData = NULL;
+	PVRSRVTL_PPACKETHDR	ppHdr;	/* Current packet header */
+	IMG_UINT32	uiHdrType;		/* Packet header type */
+	IMG_UINT32	uiMsgSize;		/* Message size of current packet (bytes) */
+	IMG_BOOL	bPacketsDropped;
+
+	pLast = pSentinel->pBuf + pSentinel->uiBufLen;
+	pStart = pSentinel->pCurr;
+
+	pSentinel->uiMsgLen = 0;	// Reset count for this message
+
+	HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: Buf @ %p..%p, Length = %d",
+	                 __func__, pStart, pLast, pSentinel->uiBufLen));
+
+	/*
+	 * We should have a DATA header with the necessary information following
+	 */
+	ppHdr = GET_PACKET_HDR(pStart);
+
+	if (ppHdr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Unexpected NULL packet in Host Trace buffer", __func__));
+		return -1;
+	}
+
+	uiHdrType = GET_PACKET_TYPE(ppHdr);
+	PVR_ASSERT(uiHdrType == PVRSRVTL_PACKETTYPE_DATA);
+
+	pNext = GET_NEXT_PACKET_ADDR(ppHdr);
+
+	PVR_ASSERT(pNext != NULL);
+
+	uiMsgSize = (IMG_UINT32)((size_t)pNext - (size_t)ppHdr);
+
+	pSentinel->uiMsgLen += uiMsgSize;
+
+	pData = GET_PACKET_DATA_PTR(ppHdr);
+
+	if (pData == NULL || pData >= pLast)
+	{
+		HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: pData = %p, pLast = %p "
+		                 "Returning 0", __func__, pData, pLast));
+		return 0;
+	}
+
+	ui32Data = *(IMG_UINT32 *)pData;
+	ui32LogIdx = idToLogIdx(ui32Data);
+
+	/*
+	 * Check if the unrecognised ID is valid and therefore, tracebuf
+	 * needs updating.
+	 */
+	if (ui32LogIdx == HTB_SF_LAST)
+	{
+		if (HTB_LOG_VALIDID(ui32Data))
+		{
+			if (!bUnrecognizedErrorPrinted)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				    "%s: Unrecognised LOG value '%x' GID %x Params %d ID %x @ '%p'",
+				    __func__, ui32Data, HTB_SF_GID(ui32Data),
+				    HTB_SF_PARAMNUM(ui32Data), ui32Data & 0xfff, pData));
+				bUnrecognizedErrorPrinted = IMG_TRUE;
+			}
+
+			return 0;
+		}
+
+		PVR_DPF((PVR_DBG_ERROR,
+		    "%s: Unrecognised and invalid LOG value detected '%x'",
+		    __func__, ui32Data));
+
+		return -1;
+	}
+
+	/* The string format we are going to display */
+	/*
+	 * The display will show the header (log-ID, group-ID, number of params)
+	 * The maximum parameter list length = 15 (only 4bits used to encode)
+	 * so we need HEADER + 15 * sizeof(UINT32) and the displayed string
+	 * describing the event. We use a buffer in the per-process pSentinel
+	 * structure to hold the data.
+	 */
+	pszFmt = aLogs[ui32LogIdx].pszFmt;
+
+	/* add the message payload size to the running count */
+	ui32ArgsCur = HTB_SF_PARAMNUM(ui32Data);
+
+	/* Determine if we've over-filled the buffer and had to drop packets */
+	bPacketsDropped = CHECK_PACKETS_DROPPED(ppHdr);
+	if (bPacketsDropped ||
+		(uiHdrType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED))
+	{
+		/* Flag this as it is useful to know ... */
+
+		PVR_DUMPDEBUG_LOG("\n<========================== *** PACKETS DROPPED *** ======================>\n");
+	}
+
+	{
+		IMG_UINT32 ui32Timestampns, ui32PID, ui32TID;
+		IMG_UINT64 ui64Timestamp, ui64TimestampSec;
+		IMG_CHAR	*szBuffer = pSentinel->szBuffer;	// Buffer start
+		IMG_CHAR	*pszBuffer = pSentinel->szBuffer;	// Current place in buf
+		size_t		uBufBytesAvailable = sizeof(pSentinel->szBuffer);
+		IMG_UINT32	*pui32Data = (IMG_UINT32 *)pData;
+		IMG_UINT32	ui_aGroupIdx;
+
+		// Get PID field from data stream
+		pui32Data++;
+		ui32PID = *pui32Data;
+		// Get TID field from data stream
+		pui32Data++;
+		ui32TID = *pui32Data;
+		// Get Timestamp part 1 from data stream
+		pui32Data++;
+		ui64Timestamp = (IMG_UINT64) *pui32Data << 32;
+		// Get Timestamp part 2 from data stream
+		pui32Data++;
+		ui64Timestamp |= (IMG_UINT64) *pui32Data;
+		// Move to start of message contents data
+		pui32Data++;
+
+		/*
+		 * We need to snprintf the data to a local in-kernel buffer
+		 * and then PVR_DUMPDEBUG_LOG() that in one shot
+		 */
+		ui_aGroupIdx = MIN(HTB_SF_GID(ui32Data), uiMax_aGroups);
+
+		/* Divide by 1B to get seconds & mod using output var (nanosecond resolution)*/
+		ui64TimestampSec = OSDivide64r64(ui64Timestamp, 1000000000, &ui32Timestampns);
+
+		nPrinted = OSSNPrintf(szBuffer, uBufBytesAvailable, "%010"IMG_UINT64_FMTSPEC".%09u:%-5u-%-5u-%s> ",
+			ui64TimestampSec, ui32Timestampns, ui32PID, ui32TID, aGroups[ui_aGroupIdx]);
+		if (nPrinted >= uBufBytesAvailable)
+		{
+			PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+				" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+				uBufBytesAvailable);
+
+			nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+		}
+
+		PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+		/* Update where our next 'output' point in the buffer is */
+		pszBuffer += nPrinted;
+		uBufBytesAvailable -= nPrinted;
+
+		/*
+		 * Print one argument at a time as this simplifies handling variable
+		 * number of arguments. Special case handling for no arguments.
+		 * This is the case for simple format strings such as
+		 * HTB_SF_MAIN_KICK_UNCOUNTED.
+		 */
+		if (ui32ArgsCur == 0)
+		{
+			if (pszFmt)
+			{
+				const ssize_t iCopiedCnt =
+					OSStringSafeCopy(pszBuffer, pszFmt, uBufBytesAvailable);
+				if (iCopiedCnt < 0)
+				{
+					PVR_DUMPDEBUG_LOG("Buffer overrun - %zu required,"
+					                  " max space %zu\n",
+					                   OSStringLength(pszFmt),
+					                   uBufBytesAvailable);
+
+					/* Ensure we don't overflow buffer */
+					nPrinted = uBufBytesAvailable;
+				}
+				else
+				{
+					nPrinted = iCopiedCnt;
+				}
+				PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+				pszBuffer += nPrinted;
+				/* Don't update the uBufBytesAvailable as we have finished this
+				 * message decode. pszBuffer - szBuffer is the total amount of
+				 * data we have decoded.
+				 */
+			}
+		}
+		else
+		{
+			if (HTB_SF_GID(ui32Data) == HTB_GID_CTRL && HTB_SF_ID(ui32Data) == HTB_ID_MARK_SCALE)
+			{
+				IMG_UINT32 i;
+				IMG_UINT32 ui32ArgArray[HTB_MARK_SCALE_ARG_ARRAY_SIZE];
+				IMG_UINT64 ui64OSTS = 0;
+				IMG_UINT32 ui32OSTSRem = 0;
+				IMG_UINT64 ui64CRTS = 0;
+
+				/* Retrieve 6 args to an array */
+				for (i = 0; i < ARRAY_SIZE(ui32ArgArray); i++)
+				{
+					ui32ArgArray[i] = *pui32Data;
+					pui32Data++;
+					--ui32ArgsCur;
+				}
+
+				ui64OSTS = (IMG_UINT64) ui32ArgArray[HTB_ARG_OSTS_PT1] << 32 | ui32ArgArray[HTB_ARG_OSTS_PT2];
+				ui64CRTS = (IMG_UINT64) ui32ArgArray[HTB_ARG_CRTS_PT1] << 32 | ui32ArgArray[HTB_ARG_CRTS_PT2];
+
+				/* Divide by 1B to get seconds, remainder in nano seconds*/
+				ui64OSTS = OSDivide64r64(ui64OSTS, 1000000000, &ui32OSTSRem);
+
+				nPrinted = OSSNPrintf(pszBuffer,
+						              uBufBytesAvailable,
+						              "HTBFWMkSync Mark=%u OSTS=%010" IMG_UINT64_FMTSPEC ".%09u CRTS=%" IMG_UINT64_FMTSPEC " CalcClkSpd=%u\n",
+						              ui32ArgArray[HTB_ARG_SYNCMARK],
+						              ui64OSTS,
+						              ui32OSTSRem,
+						              ui64CRTS,
+						              ui32ArgArray[HTB_ARG_CLKSPD]);
+
+				if (nPrinted >= uBufBytesAvailable)
+				{
+					PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+						" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+						uBufBytesAvailable);
+					nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+				}
+
+				PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+				pszBuffer += nPrinted;
+				uBufBytesAvailable -= nPrinted;
+			}
+			else
+			{
+				while (IS_VALID_FMT_STRING(pszFmt) && (uBufBytesAvailable > 0))
+				{
+					IMG_UINT32 ui32TmpArg = *pui32Data;
+					TRACEBUF_ARG_TYPE eArgType;
+
+					eArgType = ExtractOneArgFmt(&pszFmt, aszOneArgFmt);
+
+					pui32Data++;
+					ui32ArgsCur--;
+
+					switch (eArgType)
+					{
+						case TRACEBUF_ARG_TYPE_INT:
+							nPrinted = OSSNPrintf(pszBuffer, uBufBytesAvailable,
+								aszOneArgFmt, ui32TmpArg);
+							break;
+
+						case TRACEBUF_ARG_TYPE_NONE:
+							{
+								const ssize_t iCopiedCnt =
+									OSStringSafeCopy(pszBuffer,
+									                 pszFmt,
+									                 uBufBytesAvailable);
+								if (iCopiedCnt < 0)
+								{
+									nPrinted = OSStringLength(pszFmt);
+								}
+								else
+								{
+									nPrinted = iCopiedCnt;
+								}
+							}
+							break;
+
+						default:
+							nPrinted = OSSNPrintf(pszBuffer, uBufBytesAvailable,
+								"Error processing arguments, type not "
+								"recognized (fmt: %s)", aszOneArgFmt);
+							break;
+					}
+					if (nPrinted >= uBufBytesAvailable)
+					{
+						PVR_DUMPDEBUG_LOG("Buffer overrun - "IMG_SIZE_FMTSPEC" printed,"
+							" max space "IMG_SIZE_FMTSPEC"\n", nPrinted,
+							uBufBytesAvailable);
+						nPrinted = uBufBytesAvailable;	/* Ensure we don't overflow buffer */
+					}
+					PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+					pszBuffer += nPrinted;
+					uBufBytesAvailable -= nPrinted;
+				}
+				/* Display any remaining text in pszFmt string */
+				if (pszFmt)
+				{
+					const ssize_t iCopiedCnt =
+						OSStringSafeCopy(pszBuffer, pszFmt, uBufBytesAvailable);
+					if (iCopiedCnt < 0)
+					{
+						PVR_DUMPDEBUG_LOG("Buffer overrun - %zu required,"
+						                  " max space %zu\n",
+						                   OSStringLength(pszFmt),
+						                   uBufBytesAvailable);
+
+						/* Ensure we don't overflow buffer */
+						nPrinted = uBufBytesAvailable;
+					}
+					else
+					{
+						nPrinted = iCopiedCnt;
+					}
+					PVR_DUMPDEBUG_LOG("%s", pszBuffer);
+					pszBuffer += nPrinted;
+					/* Don't update the uBufBytesAvailable as we have finished this
+					 * message decode. pszBuffer - szBuffer is the total amount of
+					 * data we have decoded.
+					 */
+				}
+			}
+		}
+
+		/* Update total bytes processed */
+		pSentinel->uiTotal += (pszBuffer - szBuffer);
+	}
+	return 0;
+}
+
+/*
+ * HTBDumpBuffer: Dump the Host Trace Buffer using the TLClient API
+ *
+ * This routine just parses *one* message from the buffer.
+ * The stream will be opened by the Start() routine, closed by the Stop() and
+ * updated for data consumed by this routine once we have DebugPrintf'd it.
+ * We use the new TLReleaseDataLess() routine which enables us to update the
+ * HTB contents with just the amount of data we have successfully processed.
+ * If we need to leave the data available we can call this with a 0 count.
+ * This will happen in the case of a buffer overflow so that we can reprocess
+ * any data which wasn't handled before.
+ *
+ * In case of overflow or an error we return -1 otherwise 0
+ *
+ * Input:
+ *  pfnPrintf           output routine to display data
+ *  psEntry             handle to debug frontend
+ *  pvData              data address to start dumping from
+ *                      (set by Start() / Next())
+ */
+static int HTBDumpBuffer(DI_PRINTF pfnPrintf, OSDI_IMPL_ENTRY *psEntry,
+                         void *pvData)
+{
+	HTB_Sentinel_t *pSentinel = DIGetPrivData(psEntry);
+
+	PVR_ASSERT(pvData != NULL);
+
+	if (pvData == DI_START_TOKEN)
+	{
+		if (pSentinel->pCurr == NULL)
+		{
+			HTB_CHATTY_PRINT((PVR_DBG_WARNING, "%s: DI_START_TOKEN, "
+			                 "Empty buffer", __func__));
+			return 0;
+		}
+		PVR_ASSERT(pSentinel->pCurr != NULL);
+
+		/* Display a Header as we have data to process */
+		pfnPrintf(psEntry, "%-20s:%-5s-%-5s-%s  %s\n", "Timestamp", "PID", "TID", "Group>",
+		         "Log Entry");
+	}
+	else
+	{
+		if (pvData != NULL)
+		{
+			PVR_ASSERT(pSentinel->pCurr == pvData);
+		}
+	}
+
+	return DecodeHTB(pSentinel, psEntry, pfnPrintf);
+}
+
+
+/******************************************************************************
+ * External Entry Point routines ...
+ *****************************************************************************/
+/*************************************************************************/ /*!
+ @Function     HTB_CreateDIEntry
+
+ @Description  Create the debugFS entry-point for the host-trace-buffer
+
+ @Returns      eError          internal error code, PVRSRV_OK on success
+
+ */ /*************************************************************************/
+PVRSRV_ERROR HTB_CreateDIEntry_Impl(void)
+{
+	PVRSRV_ERROR eError;
+
+	DI_ITERATOR_CB sIterator = {
+		.pfnStart = _DebugHBTraceDIStart,
+		.pfnStop  = _DebugHBTraceDIStop,
+		.pfnNext  = _DebugHBTraceDINext,
+		.pfnShow  = _DebugHBTraceDIShow,
+	};
+
+	eError = DICreateEntry("host_trace", NULL, &sIterator,
+	                       &g_sHTBData.sSentinel,
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &g_sHTBData.psDumpHostDiEntry);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DICreateEntry");
+
+	return PVRSRV_OK;
+}
+
+
+/*************************************************************************/ /*!
+ @Function     HTB_DestroyDIEntry
+
+ @Description  Destroy the debugFS entry-point created by earlier
+               HTB_CreateDIEntry() call.
+*/ /**************************************************************************/
+void HTB_DestroyDIEntry_Impl(void)
+{
+	if (g_sHTBData.psDumpHostDiEntry != NULL)
+	{
+		DIDestroyEntry(g_sHTBData.psDumpHostDiEntry);
+		g_sHTBData.psDumpHostDiEntry = NULL;
+	}
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img-rogue/htb_debug.h b/drivers/gpu/drm/img-rogue/htb_debug.h
new file mode 100644
index 000000000000..cf18bbb2ac5d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htb_debug.h
@@ -0,0 +1,72 @@
+/*************************************************************************/ /*!
+@File           htb_debug.h
+@Title          Linux debugFS routine setup header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HTB_DEBUG_H
+#define HTB_DEBUG_H
+
+/**************************************************************************/ /*!
+ @Function     HTB_CreateDIEntry
+
+ @Description  Create the debugFS entry-point for the host-trace-buffer
+
+ @Returns      eError          internal error code, PVRSRV_OK on success
+
+ */ /**************************************************************************/
+PVRSRV_ERROR HTB_CreateDIEntry_Impl(void);
+
+/**************************************************************************/ /*!
+ @Function     HTB_DestroyFSEntry
+
+ @Description  Destroy the debugFS entry-point created by earlier
+               HTB_CreateDIEntry() call.
+*/ /**************************************************************************/
+void HTB_DestroyDIEntry_Impl(void);
+
+#if defined(PVRSRV_ENABLE_HTB)
+#define HTB_CreateDIEntry() HTB_CreateDIEntry_Impl()
+#define HTB_DestroyDIEntry() HTB_DestroyDIEntry_Impl()
+#else /* !PVRSRV_ENABLE_HTB */
+#define HTB_CreateDIEntry() PVRSRV_OK
+#define HTB_DestroyDIEntry()
+#endif /* PVRSRV_ENABLE_HTB */
+
+#endif /* HTB_DEBUG_H */
diff --git a/drivers/gpu/drm/img-rogue/htbserver.c b/drivers/gpu/drm/img-rogue/htbserver.c
new file mode 100644
index 000000000000..9ed3283f52b2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htbserver.c
@@ -0,0 +1,998 @@
+/*************************************************************************/ /*!
+@File           htbserver.c
+@Title          Host Trace Buffer server implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "htbserver.h"
+#include "htbuffer_types.h"
+#include "tlstream.h"
+#include "pvrsrv_tlcommon.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+
+/* size of circular buffer controlling the maximum number of concurrent PIDs logged */
+#define HTB_MAX_NUM_PID 8
+
+/* number of times to try rewriting a log entry */
+#define HTB_LOG_RETRY_COUNT 5
+
+#if defined(__linux__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+/*************************************************************************/ /*!
+  Host Trace Buffer control information structure
+*/ /**************************************************************************/
+typedef struct
+{
+	IMG_UINT32 ui32BufferSize;      /*!< Requested buffer size in bytes
+                                         Once set this may not be changed */
+
+	HTB_OPMODE_CTRL eOpMode;        /*!< Control what trace data is dropped if
+                                         the buffer is full.
+                                         Once set this may not be changed */
+
+/*	IMG_UINT32 ui32GroupEnable; */  /*!< Flags word controlling groups to be
+                                         logged */
+
+	IMG_UINT32 ui32LogLevel;        /*!< Log level to control messages logged */
+
+	IMG_UINT32 aui32EnablePID[HTB_MAX_NUM_PID]; /*!< PIDs to enable logging for
+                                                     a specific set of processes */
+
+	IMG_UINT32 ui32PIDCount;        /*!< Current number of PIDs being logged */
+
+	IMG_UINT32 ui32PIDHead;         /*!< Head of the PID circular buffer */
+
+	HTB_LOGMODE_CTRL eLogMode;      /*!< Logging mode control */
+
+	IMG_BOOL bLogDropSignalled;     /*!< Flag indicating if a log message has
+                                         been signalled as dropped */
+
+	/* synchronisation parameters */
+	IMG_UINT64 ui64SyncOSTS;
+	IMG_UINT64 ui64SyncCRTS;
+	IMG_UINT32 ui32SyncCalcClkSpd;
+	IMG_UINT32 ui32SyncMarker;
+
+	IMG_BOOL bInitDone;             /* Set by HTBInit, reset by HTBDeInit */
+
+	POS_SPINLOCK hRepeatMarkerLock;     /*!< Spinlock used in HTBLogKM to protect global variables
+	                                     (ByteCount, OSTS, CRTS ClkSpeed)
+	                                     from becoming inconsistent due to calls from
+	                                     both KM and UM */
+
+	IMG_UINT32 ui32ByteCount; /* Byte count used for triggering repeat sync point */
+	/* static variables containing details of previous sync point */
+	IMG_UINT64 ui64OSTS;
+	IMG_UINT64 ui64CRTS;
+	IMG_UINT32 ui32ClkSpeed;
+
+} HTB_CTRL_INFO;
+
+
+/*************************************************************************/ /*!
+*/ /**************************************************************************/
+static const IMG_UINT32 MapFlags[] =
+{
+	0,                    /* HTB_OPMODE_UNDEF = 0 */
+	TL_OPMODE_DROP_NEWER, /* HTB_OPMODE_DROPLATEST */
+	TL_OPMODE_DROP_OLDEST,/* HTB_OPMODE_DROPOLDEST */
+	TL_OPMODE_BLOCK       /* HTB_OPMODE_BLOCK */
+};
+
+static_assert(0 == HTB_OPMODE_UNDEF,      "Unexpected value for HTB_OPMODE_UNDEF");
+static_assert(1 == HTB_OPMODE_DROPLATEST, "Unexpected value for HTB_OPMODE_DROPLATEST");
+static_assert(2 == HTB_OPMODE_DROPOLDEST, "Unexpected value for HTB_OPMODE_DROPOLDEST");
+static_assert(3 == HTB_OPMODE_BLOCK,      "Unexpected value for HTB_OPMODE_BLOCK");
+
+static_assert(1 == TL_OPMODE_DROP_NEWER,  "Unexpected value for TL_OPMODE_DROP_NEWER");
+static_assert(2 == TL_OPMODE_DROP_OLDEST, "Unexpected value for TL_OPMODE_DROP_OLDEST");
+static_assert(3 == TL_OPMODE_BLOCK,       "Unexpected value for TL_OPMODE_BLOCK");
+
+static const IMG_UINT32 g_ui32TLBaseFlags; //TL_FLAG_NO_SIGNAL_ON_COMMIT
+
+/* Minimum TL buffer size.
+ * Large enough for around 60 worst case messages or 200 average messages
+ */
+#define HTB_TL_BUFFER_SIZE_MIN	(0x10000)
+
+/* Minimum concentration of HTB packets in a TL Stream is 60%
+ * If we just put the HTB header in the TL stream (12 bytes), the TL overhead
+ * is 8 bytes for its own header, so for the smallest possible (and most
+ * inefficient) packet we have 3/5 of the buffer used for actual HTB data.
+ * This shift is used as a guaranteed estimation on when to produce a repeat
+ * packet. By shifting the size of the buffer by 1 we effectively /2 this
+ * under the 60% boundary chance we may have overwritten the marker and thus
+ * guaranteed to always have a marker in the stream */
+#define HTB_MARKER_PREDICTION_THRESHOLD(val) (val >> 1)
+
+static HTB_CTRL_INFO g_sCtrl;
+static IMG_BOOL g_bConfigured = IMG_FALSE;
+static IMG_HANDLE g_hTLStream;
+
+static IMG_HANDLE hHtbDbgReqNotify;
+
+/************************************************************************/ /*!
+ @Function      _LookupFlags
+ @Description   Convert HTBuffer Operation mode to TLStream flags
+
+ @Input         eModeHTBuffer   Operation Mode
+
+ @Return        IMG_UINT32      TLStream FLags
+*/ /**************************************************************************/
+static IMG_UINT32
+_LookupFlags( HTB_OPMODE_CTRL eMode )
+{
+	return (eMode < ARRAY_SIZE(MapFlags)) ? MapFlags[eMode] : 0;
+}
+
+
+/************************************************************************/ /*!
+ @Function      _HTBLogDebugInfo
+ @Description   Debug dump handler used to dump the state of the HTB module.
+                Called for each verbosity level during a debug dump. Function
+                only prints state when called for High verbosity.
+
+ @Input         hDebugRequestHandle See PFN_DBGREQ_NOTIFY
+
+ @Input         ui32VerbLevel       See PFN_DBGREQ_NOTIFY
+
+ @Input         pfnDumpDebugPrintf  See PFN_DBGREQ_NOTIFY
+
+ @Input         pvDumpDebugFile     See PFN_DBGREQ_NOTIFY
+
+*/ /**************************************************************************/
+static void _HTBLogDebugInfo(
+		PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+		IMG_UINT32 ui32VerbLevel,
+		DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+		void *pvDumpDebugFile
+)
+{
+	PVR_UNREFERENCED_PARAMETER(hDebugRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+
+		if (g_bConfigured)
+		{
+			IMG_INT i;
+
+			PVR_DUMPDEBUG_LOG("------[ HTB Log state: On ]------");
+
+			PVR_DUMPDEBUG_LOG("HTB Log mode: %d", g_sCtrl.eLogMode);
+			PVR_DUMPDEBUG_LOG("HTB Log level: %d", g_sCtrl.ui32LogLevel);
+			PVR_DUMPDEBUG_LOG("HTB Buffer Opmode: %d", g_sCtrl.eOpMode);
+
+			for (i=0; i < HTB_FLAG_NUM_EL; i++)
+			{
+				PVR_DUMPDEBUG_LOG("HTB Log group %d: %x", i, g_auiHTBGroupEnable[i]);
+			}
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("------[ HTB Log state: Off ]------");
+		}
+	}
+}
+
+static IMG_UINT32 g_ui32HTBufferSize = HTB_TL_BUFFER_SIZE_MIN;
+
+/*
+ * AppHint access routine forward definitions
+ */
+static PVRSRV_ERROR _HTBSetLogGroup(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32);
+static PVRSRV_ERROR _HTBReadLogGroup(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32 *);
+
+static PVRSRV_ERROR	_HTBSetOpMode(const PVRSRV_DEVICE_NODE *, const void *,
+                                   IMG_UINT32);
+static PVRSRV_ERROR _HTBReadOpMode(const PVRSRV_DEVICE_NODE *, const void *,
+                                    IMG_UINT32 *);
+
+static void _OnTLReaderOpenCallback(void *);
+
+/************************************************************************/ /*!
+ @Function      HTBInit
+ @Description   Allocate and initialise the Host Trace Buffer
+                The buffer size may be changed by specifying
+                HTBufferSizeInKB=xxxx
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBInit_Impl(void)
+{
+	void			*pvAppHintState = NULL;
+	IMG_UINT32		ui32AppHintDefault;
+	IMG_UINT32		ui32BufBytes;
+	PVRSRV_ERROR	eError;
+
+	if (g_sCtrl.bInitDone)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "HTBInit: Driver already initialised"));
+		return PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+
+	/*
+	 * Buffer Size can be configured by specifying a value in the AppHint
+	 * This will only take effect at module load time so there is no query
+	 * or setting mechanism available.
+	 */
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HTBufferSizeInKB,
+										NULL,
+										NULL,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableHTBLogGroup,
+	                                    _HTBReadLogGroup,
+	                                    _HTBSetLogGroup,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HTBOperationMode,
+	                                    _HTBReadOpMode,
+	                                    _HTBSetOpMode,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    NULL);
+
+	/*
+	 * Now get whatever values have been configured for our AppHints
+	 */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = HTB_TL_BUFFER_SIZE_MIN / 1024;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+						 &ui32AppHintDefault, &g_ui32HTBufferSize);
+	OSFreeAppHintState(pvAppHintState);
+
+	ui32BufBytes = g_ui32HTBufferSize * 1024;
+
+	/* initialise rest of state */
+	g_sCtrl.ui32BufferSize =
+		(ui32BufBytes < HTB_TL_BUFFER_SIZE_MIN)
+		? HTB_TL_BUFFER_SIZE_MIN
+		: ui32BufBytes;
+	g_sCtrl.eOpMode = HTB_OPMODE_DROPOLDEST;
+	g_sCtrl.ui32LogLevel = 0;
+	g_sCtrl.ui32PIDCount = 0;
+	g_sCtrl.ui32PIDHead = 0;
+	g_sCtrl.eLogMode = HTB_LOGMODE_ALLPID;
+	g_sCtrl.bLogDropSignalled = IMG_FALSE;
+
+	eError = OSSpinLockCreate(&g_sCtrl.hRepeatMarkerLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSSpinLockCreate");
+
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&hHtbDbgReqNotify,
+			 _HTBLogDebugInfo, DEBUG_REQUEST_HTB, NULL);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify");
+
+	g_sCtrl.bInitDone = IMG_TRUE;
+
+	/* Log the current driver parameter setting for the HTBufferSizeInKB.
+	 * We do this here as there is no other infrastructure for obtaining
+	 * the value.
+	 */
+	if (g_ui32HTBufferSize != ui32AppHintDefault)
+	{
+		PVR_LOG(("Increasing HTBufferSize to %uKB", g_ui32HTBufferSize));
+	}
+
+
+	return PVRSRV_OK;
+}
+
+/************************************************************************/ /*!
+ @Function      HTBDeInit
+ @Description   Close the Host Trace Buffer and free all resources. Must
+                perform a no-op if already de-initialised.
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBDeInit_Impl( void )
+{
+	if (!g_sCtrl.bInitDone)
+		return PVRSRV_OK;
+
+	if (hHtbDbgReqNotify)
+	{
+		/* Not much we can do if it fails, driver unloading */
+		(void)PVRSRVUnregisterDriverDbgRequestNotify(hHtbDbgReqNotify);
+		hHtbDbgReqNotify = NULL;
+	}
+
+	if (g_hTLStream)
+	{
+		TLStreamClose( g_hTLStream );
+		g_hTLStream = NULL;
+	}
+
+	if (g_sCtrl.hRepeatMarkerLock != NULL)
+	{
+		OSSpinLockDestroy(g_sCtrl.hRepeatMarkerLock);
+		g_sCtrl.hRepeatMarkerLock = NULL;
+	}
+
+	g_sCtrl.bInitDone = IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+
+/*************************************************************************/ /*!
+ AppHint interface functions
+*/ /**************************************************************************/
+static
+PVRSRV_ERROR _HTBSetLogGroup(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const void *psPrivate,
+                             IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return HTBControlKM(1, &ui32Value, 0, 0,
+	                    HTB_LOGMODE_UNDEF, HTB_OPMODE_UNDEF);
+}
+
+static
+PVRSRV_ERROR _HTBReadLogGroup(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 *pui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value = g_auiHTBGroupEnable[0];
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR _HTBSetOpMode(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                           const void *psPrivate,
+                           IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return HTBControlKM(0, NULL, 0, 0, HTB_LOGMODE_UNDEF, ui32Value);
+}
+
+static
+PVRSRV_ERROR _HTBReadOpMode(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                            const void *psPrivate,
+                            IMG_UINT32 *pui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value = (IMG_UINT32)g_sCtrl.eOpMode;
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_HTB)
+static IMG_BOOL
+_ValidPID( IMG_UINT32 PID )
+{
+	IMG_UINT32 i;
+	for (i = 0; i < g_sCtrl.ui32PIDCount; i++)
+	{
+		if ( g_sCtrl.aui32EnablePID[i] == PID )
+		{
+			return IMG_TRUE;
+		}
+	}
+	return IMG_FALSE;
+}
+#endif	/* PVRSRV_ENABLE_HTB */
+
+/*************************************************************************/ /*!
+ @Function      HTBLogKM
+ @Description   Record a Host Trace Buffer log event
+
+ @Input         PID             The PID of the process the event is associated
+                                with. This is provided as an argument rather
+                                than querying internally so that events associated
+                                with a particular process, but performed by
+                                another can be logged correctly.
+
+ @Input         ui64TimeStamp   The timestamp to be associated with this log event
+
+ @Input         SF              The log event ID
+
+ @Input         ...             Log parameters
+
+ @Return        PVRSRV_OK       Success.
+
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+HTBLogKM(IMG_UINT32 PID,
+		IMG_UINT32 TID,
+		IMG_UINT64 ui64TimeStamp,
+		HTB_LOG_SFids SF,
+		va_list args
+)
+{
+#if defined(PVRSRV_ENABLE_HTB)
+
+	OS_SPINLOCK_FLAGS uiSpinLockFlags;
+	IMG_UINT32 ui32ReturnFlags = 0;
+	IMG_UINT32 ui32CurrentArg = 0;
+
+	/* Local snapshot variables of global counters */
+	IMG_UINT64 ui64OSTSSnap;
+	IMG_UINT64 ui64CRTSSnap;
+	IMG_UINT32 ui32ClkSpeedSnap;
+
+	/* format of messages is: SF:PID:TID:TIMEPT1:TIMEPT2:[PARn]*
+	 * Buffer is on the stack so we don't need a semaphore to guard it
+	 */
+	IMG_UINT32 aui32MessageBuffer[HTB_LOG_HEADER_SIZE+HTB_LOG_MAX_PARAMS];
+	IMG_UINT32 aui32Args[HTB_LOG_MAX_PARAMS + 1] = {0};
+
+	/* Min HTB size is HTB_TL_BUFFER_SIZE_MIN : 10000 bytes and Max message/
+	 * packet size is 4*(HTB_LOG_HEADER_SIZE+HTB_LOG_MAX_PARAMS) = 80 bytes,
+	 * hence with these constraints this design is unlikely to get
+	 * PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED error
+	 */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_ENABLED;
+	IMG_UINT32 ui32RetryCount = HTB_LOG_RETRY_COUNT;
+	IMG_UINT32 * pui32Message = aui32MessageBuffer;
+	IMG_UINT32 ui32NumArgs = HTB_SF_PARAMNUM(SF);
+	IMG_UINT32 ui32StrArg = HTB_SF_STRNUM(SF);
+	IMG_UINT32 ui32CurrentStringArg;
+
+	IMG_UINT32 ui32MessageSize = 4 * (HTB_LOG_HEADER_SIZE+ui32NumArgs);
+
+	if (ui32StrArg > 0)
+	{
+		ui32MessageSize += HTB_LOG_STR_ARG_SIZE - sizeof(IMG_UINT32);
+	}
+
+	PVR_ASSERT(ui32NumArgs <= HTB_LOG_MAX_PARAMS);
+	ui32NumArgs = (ui32NumArgs>HTB_LOG_MAX_PARAMS) ?
+			HTB_LOG_MAX_PARAMS : ui32NumArgs;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32NumArgs == HTB_SF_PARAMNUM(SF), eError, ReturnError);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32NumArgs <= HTB_LOG_MAX_PARAMS, eError, ReturnError);
+
+	/* Needs to be set up here because it's accessed from both `if` blocks below
+	 * and it needs to be pre-populated for both of them (pui32Message case and
+	 * HTB_SF_CTRL_FWSYNC_MARK_SCALE case). */
+
+	while (ui32CurrentArg < ui32NumArgs)
+	{
+		if (ui32StrArg != 0 && ui32CurrentArg == ui32StrArg - 1)
+		{
+			IMG_CHAR* strArg;
+			strArg = va_arg(args, IMG_CHAR*);
+
+			/* if a string is present, it will need more than one UINT32 words. */
+			ui32NumArgs += HTB_LOG_STR_ARG_NUM_WORDS - 1;
+
+			/* Ignore if filename is not supported. */
+			if (strcmp(strArg, "n/a") == 0)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "N/A\n"));
+				return PVRSRV_OK;
+			}
+
+			/* looping through the created string to get the substrings to encode. */
+			for (ui32CurrentStringArg = 0; ui32CurrentStringArg < HTB_LOG_STR_ARG_NUM_WORDS; ui32CurrentStringArg++)
+			{
+				IMG_UINT32 encodedString = 0;
+
+				if (*strArg == '\0')
+				{
+					aui32Args[ui32CurrentArg] = encodedString;
+					ui32CurrentArg++;
+				}
+				else
+				{
+					IMG_UINT32 currentSubstring;
+
+					for (currentSubstring = 0; currentSubstring < sizeof(IMG_UINT32); currentSubstring++)
+					{
+						IMG_UINT32 bitPos = currentSubstring * 8;
+
+						if (*strArg == '\0')
+						{
+							break;
+						}
+						else
+						{
+							encodedString |= (IMG_UINT32) *strArg << bitPos;
+							strArg++;
+						}
+					}
+
+					aui32Args[ui32CurrentArg] = encodedString;
+					ui32CurrentArg++;
+				}
+			}
+		}
+		else
+		{
+			aui32Args[ui32CurrentArg] = va_arg(args, IMG_UINT32);
+			ui32CurrentArg++;
+		}
+	}
+
+	if ( g_hTLStream
+			&& ( 0 == PID || ~0 == PID || HTB_LOGMODE_ALLPID == g_sCtrl.eLogMode || _ValidPID(PID) )
+/*			&& ( g_sCtrl.ui32GroupEnable & (0x1 << HTB_SF_GID(SF)) ) */
+/*			&& ( g_sCtrl.ui32LogLevel >= HTB_SF_LVL(SF) ) */
+			)
+	{
+		*pui32Message++ = SF;
+		*pui32Message++ = PID;
+		*pui32Message++ = TID;
+		*pui32Message++ = ((IMG_UINT32)((ui64TimeStamp>>32)&0xffffffff));
+		*pui32Message++ = ((IMG_UINT32)(ui64TimeStamp&0xffffffff));
+		for (ui32CurrentArg = 0; ui32CurrentArg < ui32NumArgs; ui32CurrentArg++)
+		{
+			pui32Message[ui32CurrentArg] = aui32Args[ui32CurrentArg];
+		}
+
+		eError = TLStreamWriteRetFlags( g_hTLStream, (IMG_UINT8*)aui32MessageBuffer, ui32MessageSize, &ui32ReturnFlags );
+
+		while ( PVRSRV_ERROR_NOT_READY == eError && ui32RetryCount-- )
+		{
+			OSReleaseThreadQuanta();
+			eError = TLStreamWriteRetFlags( g_hTLStream, (IMG_UINT8*)aui32MessageBuffer, ui32MessageSize, &ui32ReturnFlags );
+		}
+
+		if ( PVRSRV_OK == eError )
+		{
+			g_sCtrl.bLogDropSignalled = IMG_FALSE;
+		}
+		else if ( PVRSRV_ERROR_STREAM_FULL != eError || !g_sCtrl.bLogDropSignalled )
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()", "TLStreamWrite", PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		if ( PVRSRV_ERROR_STREAM_FULL == eError )
+		{
+			g_sCtrl.bLogDropSignalled = IMG_TRUE;
+		}
+
+	}
+
+	if (SF == HTB_SF_CTRL_FWSYNC_MARK_SCALE)
+	{
+		OSSpinLockAcquire(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+
+		/* If a marker is being placed reset byte count from last marker */
+		g_sCtrl.ui32ByteCount = 0;
+		g_sCtrl.ui64OSTS = (IMG_UINT64)aui32Args[HTB_ARG_OSTS_PT1] << 32 | aui32Args[HTB_ARG_OSTS_PT2];
+		g_sCtrl.ui64CRTS = (IMG_UINT64)aui32Args[HTB_ARG_CRTS_PT1] << 32 | aui32Args[HTB_ARG_CRTS_PT2];
+		g_sCtrl.ui32ClkSpeed = aui32Args[HTB_ARG_CLKSPD];
+
+		OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+	}
+	else
+	{
+		OSSpinLockAcquire(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+		/* Increase global count */
+		g_sCtrl.ui32ByteCount += ui32MessageSize;
+
+		/* Check if packet has overwritten last marker/rpt &&
+		   If the packet count is over half the size of the buffer */
+		if (ui32ReturnFlags & TL_FLAG_OVERWRITE_DETECTED &&
+				 g_sCtrl.ui32ByteCount > HTB_MARKER_PREDICTION_THRESHOLD(g_sCtrl.ui32BufferSize))
+		{
+			/* Take snapshot of global variables */
+			ui64OSTSSnap = g_sCtrl.ui64OSTS;
+			ui64CRTSSnap = g_sCtrl.ui64CRTS;
+			ui32ClkSpeedSnap = g_sCtrl.ui32ClkSpeed;
+			/* Reset global variable counter */
+			g_sCtrl.ui32ByteCount = 0;
+			OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+
+			/* Produce a repeat marker */
+			HTBSyncPartitionMarkerRepeat(g_sCtrl.ui32SyncMarker, ui64OSTSSnap, ui64CRTSSnap, ui32ClkSpeedSnap);
+		}
+		else
+		{
+			OSSpinLockRelease(g_sCtrl.hRepeatMarkerLock, uiSpinLockFlags);
+		}
+	}
+
+ReturnError:
+	return eError;
+
+#else
+	/* HTB support is disabled. Just return PVRSRV_OK and do nothing. */
+	PVR_UNREFERENCED_PARAMETER(PID);
+	PVR_UNREFERENCED_PARAMETER(TID);
+	PVR_UNREFERENCED_PARAMETER(ui64TimeStamp);
+	PVR_UNREFERENCED_PARAMETER(SF);
+	PVR_UNREFERENCED_PARAMETER(args);
+	return PVRSRV_OK;
+#endif
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBLog
+ @Description   Record a Host Trace Buffer log event
+ @Input         PID     The PID of the process the event is associated
+                        with. This is provided as an argument rather
+                        than querying internally so that events
+                        associated with a particular process, but
+                        performed by another can be logged correctly.
+ @Input         ui64TimeStamp   The timestamp to be associated with this
+                                log event
+ @Input         SF              The log event ID
+ @Input         ...             Log parameters
+ @Return        PVRSRV_OK       Success.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+HTBLog(IMG_UINT32 PID, IMG_UINT32 TID, IMG_UINT64 ui64TimeStamp,
+		IMG_UINT32 SF, ...)
+{
+	PVRSRV_ERROR eError;
+
+	va_list args;
+	va_start(args, SF);
+	eError = HTBLogKM(PID, TID, ui64TimeStamp, SF, args);
+	va_end(args);
+	return eError;
+}
+
+static void
+_OnTLReaderOpenCallback( void *pvArg )
+{
+	if ( g_hTLStream )
+	{
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+		(void) HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+		              g_sCtrl.ui32SyncMarker,
+		              ((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)),
+		              ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+		              ((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)),
+		              ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+		              g_sCtrl.ui32SyncCalcClkSpd);
+	}
+
+	PVR_UNREFERENCED_PARAMETER(pvArg);
+}
+
+
+/*************************************************************************/ /*!
+ @Function      HTBControlKM
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control the behaviour of the data buffer
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBControlKM_Impl(
+	const IMG_UINT32 ui32NumFlagGroups,
+	const IMG_UINT32 * aui32GroupEnable,
+	const IMG_UINT32 ui32LogLevel,
+	const IMG_UINT32 ui32EnablePID,
+	const HTB_LOGMODE_CTRL eLogMode,
+	const HTB_OPMODE_CTRL eOpMode
+)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32RetryCount = HTB_LOG_RETRY_COUNT;
+	IMG_UINT32 i;
+	IMG_UINT64 ui64Time;
+	OSClockMonotonicns64(&ui64Time);
+
+	if ( !g_bConfigured && ui32NumFlagGroups )
+	{
+		eError = TLStreamCreate(
+				&g_hTLStream,
+				HTB_STREAM_NAME,
+				g_sCtrl.ui32BufferSize,
+				_LookupFlags(HTB_OPMODE_DROPOLDEST) | g_ui32TLBaseFlags,
+				_OnTLReaderOpenCallback, NULL, NULL, NULL, NULL, NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamCreate");
+		g_bConfigured = IMG_TRUE;
+	}
+
+	if (HTB_OPMODE_UNDEF != eOpMode && g_sCtrl.eOpMode != eOpMode)
+	{
+		g_sCtrl.eOpMode = eOpMode;
+		eError = TLStreamReconfigure(g_hTLStream, _LookupFlags(g_sCtrl.eOpMode | g_ui32TLBaseFlags));
+		while ( PVRSRV_ERROR_NOT_READY == eError && ui32RetryCount-- )
+		{
+			OSReleaseThreadQuanta();
+			eError = TLStreamReconfigure(g_hTLStream, _LookupFlags(g_sCtrl.eOpMode | g_ui32TLBaseFlags));
+		}
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamReconfigure");
+	}
+
+	if ( ui32EnablePID )
+	{
+		g_sCtrl.aui32EnablePID[g_sCtrl.ui32PIDHead] = ui32EnablePID;
+		g_sCtrl.ui32PIDHead++;
+		g_sCtrl.ui32PIDHead %= HTB_MAX_NUM_PID;
+		g_sCtrl.ui32PIDCount++;
+		if ( g_sCtrl.ui32PIDCount > HTB_MAX_NUM_PID )
+		{
+			g_sCtrl.ui32PIDCount = HTB_MAX_NUM_PID;
+		}
+	}
+
+	/* HTB_LOGMODE_ALLPID overrides ui32EnablePID */
+	if ( HTB_LOGMODE_ALLPID == eLogMode )
+	{
+		OSCachedMemSet(g_sCtrl.aui32EnablePID, 0, sizeof(g_sCtrl.aui32EnablePID));
+		g_sCtrl.ui32PIDCount = 0;
+		g_sCtrl.ui32PIDHead = 0;
+	}
+	if ( HTB_LOGMODE_UNDEF != eLogMode )
+	{
+		g_sCtrl.eLogMode = eLogMode;
+	}
+
+	if ( ui32NumFlagGroups )
+	{
+		for (i = 0; i < HTB_FLAG_NUM_EL && i < ui32NumFlagGroups; i++)
+		{
+			g_auiHTBGroupEnable[i] = aui32GroupEnable[i];
+		}
+		for (; i < HTB_FLAG_NUM_EL; i++)
+		{
+			g_auiHTBGroupEnable[i] = 0;
+		}
+	}
+
+	if ( ui32LogLevel )
+	{
+		g_sCtrl.ui32LogLevel = ui32LogLevel;
+	}
+
+	/* Dump the current configuration state */
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_OPMODE, g_sCtrl.eOpMode);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_ENABLE_GROUP, g_auiHTBGroupEnable[0]);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_LOG_LEVEL, g_sCtrl.ui32LogLevel);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_LOGMODE, g_sCtrl.eLogMode);
+	PVR_LOG_IF_ERROR(eError, "HTBLog");
+	for (i = 0; i < g_sCtrl.ui32PIDCount; i++)
+	{
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_ENABLE_PID, g_sCtrl.aui32EnablePID[i]);
+		PVR_LOG_IF_ERROR(eError, "HTBLog");
+	}
+	/* Else should never be hit as we set the spd when the power state is updated */
+	if (0 != g_sCtrl.ui32SyncMarker && 0 != g_sCtrl.ui32SyncCalcClkSpd)
+	{
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+				g_sCtrl.ui32SyncMarker,
+				((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+				((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+				g_sCtrl.ui32SyncCalcClkSpd);
+		PVR_LOG_IF_ERROR(eError, "HTBLog");
+	}
+
+	return eError;
+}
+
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarker
+ @Description   Write an HTB sync partition marker to the HTB log
+
+ @Input         ui33Marker      Marker value
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarker_Impl(
+	const IMG_UINT32 ui32Marker
+)
+{
+	g_sCtrl.ui32SyncMarker = ui32Marker;
+	if ( g_hTLStream )
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+
+		/* Else should never be hit as we set the spd when the power state is updated */
+		if (0 != g_sCtrl.ui32SyncCalcClkSpd)
+		{
+			eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+					ui32Marker,
+					((IMG_UINT32)((g_sCtrl.ui64SyncOSTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncOSTS&0xffffffff)),
+					((IMG_UINT32)((g_sCtrl.ui64SyncCRTS>>32)&0xffffffff)), ((IMG_UINT32)(g_sCtrl.ui64SyncCRTS&0xffffffff)),
+					g_sCtrl.ui32SyncCalcClkSpd);
+			PVR_WARN_IF_ERROR(eError, "HTBLog");
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarkerRepeat
+ @Description   Write a HTB sync partition marker to the HTB log, given
+                the previous values to repeat.
+
+ @Input         ui33Marker      Marker value
+ @Input         ui64SyncOSTS    previous OSTS
+ @Input         ui64SyncCRTS    previous CRTS
+ @Input         ui32ClkSpeed    previous Clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarkerRepeat_Impl(
+	const IMG_UINT32 ui32Marker,
+	const IMG_UINT64 ui64SyncOSTS,
+	const IMG_UINT64 ui64SyncCRTS,
+	const IMG_UINT32 ui32ClkSpeed
+)
+{
+	if ( g_hTLStream )
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+
+		/* Else should never be hit as we set the spd when the power state is updated */
+		if (0 != ui32ClkSpeed)
+		{
+			eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+					ui32Marker,
+					((IMG_UINT32)((ui64SyncOSTS>>32)&0xffffffffU)), ((IMG_UINT32)(ui64SyncOSTS&0xffffffffU)),
+					((IMG_UINT32)((ui64SyncCRTS>>32)&0xffffffffU)), ((IMG_UINT32)(ui64SyncCRTS&0xffffffffU)),
+					ui32ClkSpeed);
+			PVR_WARN_IF_ERROR(eError, "HTBLog");
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncScale
+ @Description   Write FW-Host synchronisation data to the HTB log when clocks
+                change or are re-calibrated
+
+ @Input         bLogValues      IMG_TRUE if value should be immediately written
+                                out to the log
+
+ @Input         ui32OSTS        OS Timestamp
+
+ @Input         ui32CRTS        Rogue timestamp
+
+ @Input         ui32CalcClkSpd  Calculated clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncScale_Impl(
+	const IMG_BOOL bLogValues,
+	const IMG_UINT64 ui64OSTS,
+	const IMG_UINT64 ui64CRTS,
+	const IMG_UINT32 ui32CalcClkSpd
+)
+{
+	g_sCtrl.ui64SyncOSTS = ui64OSTS;
+	g_sCtrl.ui64SyncCRTS = ui64CRTS;
+	g_sCtrl.ui32SyncCalcClkSpd = ui32CalcClkSpd;
+	if (g_hTLStream && bLogValues)
+	{
+		PVRSRV_ERROR eError;
+		IMG_UINT64 ui64Time;
+		OSClockMonotonicns64(&ui64Time);
+		eError = HTBLog(0, 0, ui64Time, HTB_SF_CTRL_FWSYNC_MARK_SCALE,
+				g_sCtrl.ui32SyncMarker,
+				((IMG_UINT32)((ui64OSTS>>32)&0xffffffff)), ((IMG_UINT32)(ui64OSTS&0xffffffff)),
+				((IMG_UINT32)((ui64CRTS>>32)&0xffffffff)), ((IMG_UINT32)(ui64CRTS&0xffffffff)),
+				ui32CalcClkSpd);
+		/*
+		 * Don't spam the log with non-failure cases
+		 */
+		PVR_WARN_IF_ERROR(eError, "HTBLog");
+	}
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBLogSimple
+ @Description   Record a Host Trace Buffer log event with implicit PID and
+                Timestamp
+ @Input         SF              The log event ID
+ @Input         ...             Log parameters
+ @Return        PVRSRV_OK       Success.
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBLogSimple_Impl(IMG_UINT32 SF, ...)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64TimeStamp;
+	va_list args;
+	va_start(args, SF);
+	OSClockMonotonicns64(&ui64TimeStamp);
+	eError = HTBLogKM(OSGetCurrentProcessID(), OSGetCurrentThreadID(), ui64TimeStamp,
+			SF, args);
+	va_end(args);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ @Function      HTBIsConfigured
+ @Description   Determine if HTB stream has been configured
+
+ @Input         none
+
+ @Return        IMG_FALSE       Stream has not been configured
+                IMG_TRUE        Stream has been configured
+
+*/ /**************************************************************************/
+IMG_BOOL
+HTBIsConfigured(void)
+{
+	return g_bConfigured;
+}
+/* EOF */
diff --git a/drivers/gpu/drm/img-rogue/htbserver.h b/drivers/gpu/drm/img-rogue/htbserver.h
new file mode 100644
index 000000000000..4aaa3dd35e08
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htbserver.h
@@ -0,0 +1,240 @@
+/*************************************************************************/ /*!
+@File           htbserver.h
+@Title          Host Trace Buffer server implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+
+                A Host Trace can be merged with a corresponding Firmware Trace.
+                This is achieved by inserting synchronisation data into both
+                traces and post processing to merge them.
+
+                The FW Trace will contain a "Sync Partition Marker". This is
+                updated every time the RGX is brought out of reset (RGX clock
+                timestamps reset at this point) and is repeated when the FW
+                Trace buffer wraps to ensure there is always at least 1
+                partition marker in the Firmware Trace buffer whenever it is
+                read.
+
+                The Host Trace will contain corresponding "Sync Partition
+                Markers" - #HTBSyncPartitionMarker(). Each partition is then
+                subdivided into "Sync Scale" sections - #HTBSyncScale(). The
+                "Sync Scale" data allows the timestamps from the two traces to
+                be correlated. The "Sync Scale" data is updated as part of the
+                standard RGX time correlation code (rgxtimecorr.c) and is
+                updated periodically including on power and clock changes.
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef HTBSERVER_H
+#define HTBSERVER_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "htbuffer_types.h"
+
+#define HTBLOGK(SF, args...) do { if (HTB_GROUP_ENABLED(SF)) { (void)HTBLogSimple(SF, ## args); } } while (0)
+
+/* macros to cast 64 or 32-bit pointers into 32-bit integer components for Host Trace */
+#define HTBLOG_PTR_BITS_HIGH(p) ((IMG_UINT32)((((IMG_UINT64)((uintptr_t)p))>>32)&0xffffffff))
+#define HTBLOG_PTR_BITS_LOW(p)  ((IMG_UINT32)(((IMG_UINT64)((uintptr_t)p))&0xffffffff))
+
+/* macros to cast 64-bit integers into 32-bit integer components for Host Trace */
+#define HTBLOG_U64_BITS_HIGH(u) ((IMG_UINT32)((u>>32)&0xffffffff))
+#define HTBLOG_U64_BITS_LOW(u)  ((IMG_UINT32)(u&0xffffffff))
+
+/* Host Trace Buffer name */
+#define HTB_STREAM_NAME	"PVRHTBuffer"
+
+/************************************************************************/ /*!
+ @Function      HTBInit
+ @Description   Initialise the Host Trace Buffer and allocate all resources
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBInit_Impl(void);
+
+/************************************************************************/ /*!
+ @Function      HTBDeInit
+ @Description   Close the Host Trace Buffer and free all resources
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBDeInit_Impl(void);
+
+/*************************************************************************/ /*!
+ @Function      HTBControlKM
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control the behaviour of the data buffer
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+PVRSRV_ERROR
+HTBControlKM_Impl(const IMG_UINT32 ui32NumFlagGroups,
+			 const IMG_UINT32 *aui32GroupEnable,
+			 const IMG_UINT32 ui32LogLevel,
+			 const IMG_UINT32 ui32EnablePID,
+			 const HTB_LOGMODE_CTRL eLogMode,
+			 const HTB_OPMODE_CTRL eOpMode);
+
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarker
+ @Description   Write an HTB sync partition marker to the HTB log
+
+ @Input         ui32Marker      Marker value
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarker_Impl(const IMG_UINT32 ui32Marker);
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncPartitionMarkerRpt
+ @Description   Write a HTB sync partition marker to the HTB log, given
+                the previous values to repeat.
+
+ @Input         ui32Marker      Marker value
+ @Input         ui64SyncOSTS    previous OSTS
+ @Input         ui64SyncCRTS    previous CRTS
+ @Input         ui32ClkSpeed    previous Clockspeed
+
+*/ /**************************************************************************/
+void
+HTBSyncPartitionMarkerRepeat_Impl(const IMG_UINT32 ui32Marker,
+							 const IMG_UINT64 ui64SyncOSTS,
+							 const IMG_UINT64 ui64SyncCRTS,
+							 const IMG_UINT32 ui32ClkSpeed);
+
+/*************************************************************************/ /*!
+ @Function      HTBSyncScale
+ @Description   Write FW-Host synchronisation data to the HTB log when clocks
+                change or are re-calibrated
+
+ @Input         bLogValues      IMG_TRUE if value should be immediately written
+                                out to the log
+
+ @Input         ui64OSTS        OS Timestamp
+
+ @Input         ui64CRTS        Rogue timestamp
+
+ @Input         ui32CalcClkSpd  Calculated clock speed
+
+*/ /**************************************************************************/
+void
+HTBSyncScale_Impl(const IMG_BOOL bLogValues, const IMG_UINT64 ui64OSTS,
+			 const IMG_UINT64 ui64CRTS, const IMG_UINT32 ui32CalcClkSpd);
+
+/*************************************************************************/ /*!
+ @Function      HTBLogSimple
+ @Description   Record a Host Trace Buffer log event with implicit PID and Timestamp
+
+ @Input         SF              The log event ID
+
+ @Input         ...             Log parameters
+
+ @Return        PVRSRV_OK       Success.
+
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBLogSimple_Impl(IMG_UINT32 SF, ...);
+
+/*  DEBUG log group enable */
+#if !defined(HTB_DEBUG_LOG_GROUP)
+#undef HTB_LOG_TYPE_DBG    /* No trace statements in this log group should be checked in */
+#define HTB_LOG_TYPE_DBG    __BUILDERROR__
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+/*************************************************************************/ /*!
+ @Function      HTBIsConfigured
+ @Description   Determine if HTB stream has been configured
+
+ @Input         none
+
+ @Return        IMG_FALSE       Stream has not been configured
+                IMG_TRUE        Stream has been configured
+
+*/ /**************************************************************************/
+IMG_BOOL
+HTBIsConfigured_Impl(void);
+
+#define HTBIsConfigured HTBIsConfigured_Impl
+#define HTBLogSimple HTBLogSimple_Impl
+#define HTBSyncScale(bLogValues, ui64OSTS, ui64CRTS, ui32CalcClkSpd) \
+  HTBSyncScale_Impl((bLogValues), (ui64OSTS), (ui64CRTS), (ui32CalcClkSpd))
+#define HTBSyncPartitionMarkerRepeat(ui32Marker, ui64SyncOSTS, ui64SyncCRTS, ui32ClkSpeed) \
+  HTBSyncPartitionMarkerRepeat_Impl((ui32Marker), (ui64SyncOSTS), (ui64SyncCRTS), (ui32ClkSpeed))
+#define HTBSyncPartitionMarker(a) HTBSyncPartitionMarker_Impl((a))
+#define HTBControlKM(ui32NumFlagGroups, aui32GroupEnable, ui32LogLevel, ui32EnablePID, eLogMode, eOpMode) \
+  HTBControlKM_Impl((ui32NumFlagGroups), (aui32GroupEnable), (ui32LogLevel), (ui32EnablePID), (eLogMode), (eOpMode))
+#define HTBInit() HTBInit_Impl()
+#define HTBDeInit() HTBDeInit_Impl()
+#else	/* !PVRSRV_ENABLE_HTB) */
+#define HTBIsConfigured()  IMG_FALSE
+#define HTBLogSimple(SF, args...)  PVRSRV_OK
+#define HTBSyncScale(a, b, c, d)
+#define HTBSyncPartitionMarkerRepeat(a, b, c, d)
+#define HTBSyncPartitionMarker(a)
+#define HTBControlKM(a, b, c, d, e, f) PVRSRV_OK
+#define HTBDeInit() PVRSRV_OK
+#define HTBInit() PVRSRV_OK
+#endif	/* PVRSRV_ENABLE_HTB */
+#endif /* HTBSERVER_H */
+
+/* EOF */
diff --git a/drivers/gpu/drm/img-rogue/htbuffer.c b/drivers/gpu/drm/img-rogue/htbuffer.c
new file mode 100644
index 000000000000..5f83127a5462
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htbuffer.c
@@ -0,0 +1,106 @@
+/*************************************************************************/ /*!
+@File           htbuffer.c
+@Title          Host Trace Buffer shared API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "htbuffer.h"
+#include "osfunc.h"
+#include "client_htbuffer_bridge.h"
+
+/* The group flags array of ints large enough to store all the group flags
+ * NB: This will only work while all logging is in the kernel
+ */
+IMG_INTERNAL HTB_FLAG_EL_T g_auiHTBGroupEnable[HTB_FLAG_NUM_EL] = {0};
+
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+ @Input         hSrvHandle      Server Handle
+ @Input         ui32NumFlagGroups Number of group enable flags words
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+ @Input         ui32LogLevel    Log level to record
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+ @Input         eLogPidMode     Enable logging for all or specific processes,
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogPidMode,
+	HTB_OPMODE_CTRL eOpMode
+)
+{
+	return BridgeHTBControl(
+			hSrvHandle,
+			ui32NumFlagGroups,
+			aui32GroupEnable,
+			ui32LogLevel,
+			ui32EnablePID,
+			eLogPidMode,
+			eOpMode
+			);
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img-rogue/htbuffer.h b/drivers/gpu/drm/img-rogue/htbuffer.h
new file mode 100644
index 000000000000..96daa9727c47
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htbuffer.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@File           htbuffer.h
+@Title          Host Trace Buffer shared API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_H
+#define HTBUFFER_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "htbuffer_sf.h"
+#include "htbuffer_types.h"
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+ @Input         hSrvHandle      Server Handle
+ @Input         ui32NumFlagGroups Number of group enable flags words
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+ @Input         ui32LogLevel    Log level to record
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+ @Input         eLogPidMode     Enable logging for all or specific processes,
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogPidMode,
+	HTB_OPMODE_CTRL eOpMode);
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_H */
+/*****************************************************************************
+ End of file (htbuffer.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/htbuffer_init.h b/drivers/gpu/drm/img-rogue/htbuffer_init.h
new file mode 100644
index 000000000000..9a93b59d8628
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htbuffer_init.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@File           htbuffer_init.h
+@Title          Host Trace Buffer functions needed for Services initialisation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_INIT_H
+#define HTBUFFER_INIT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+/*************************************************************************/ /*!
+ @Function      HTBControl
+ @Description   Update the configuration of the Host Trace Buffer
+
+ @Input         hSrvHandle      Server Handle
+
+ @Input         ui32NumFlagGroups Number of group enable flags words
+
+ @Input         aui32GroupEnable  Flags words controlling groups to be logged
+
+ @Input         ui32LogLevel    Log level to record
+
+ @Input         ui32EnablePID   PID to enable logging for a specific process
+
+ @Input         eLogMode        Enable logging for all or specific processes,
+
+ @Input         eOpMode         Control what trace data is dropped if the TL
+                                buffer is full
+
+ @Return        eError          Internal services call returned eError error
+                                number
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+HTBControl(
+	IMG_HANDLE hSrvHandle,
+	IMG_UINT32 ui32NumFlagGroups,
+	IMG_UINT32 * aui32GroupEnable,
+	IMG_UINT32 ui32LogLevel,
+	IMG_UINT32 ui32EnablePID,
+	HTB_LOGMODE_CTRL eLogMode,
+	HTB_OPMODE_CTRL eOpMode
+);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_INIT_H */
+/*****************************************************************************
+ End of file (htbuffer_init.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/htbuffer_sf.h b/drivers/gpu/drm/img-rogue/htbuffer_sf.h
new file mode 100644
index 000000000000..83699a2d9547
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htbuffer_sf.h
@@ -0,0 +1,272 @@
+/*************************************************************************/ /*!
+@File           htbuffer_sf.h
+@Title          Host Trace Buffer interface string format specifiers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the Host Trace Buffer logging messages. The following
+                list are the messages the host driver prints. Changing anything
+                but the first column or spelling mistakes in the strings will
+                break compatibility with log files created with older/newer
+                driver versions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_SF_H
+#define HTBUFFER_SF_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+
+/******************************************************************************
+ * *DO*NOT* rearrange or delete lines in SFIDLIST or SFGROUPLIST or you
+ *          WILL BREAK host tracing message compatibility with previous
+ *          driver versions. Only add new ones, if so required.
+ *****************************************************************************/
+
+
+/* String used in pvrdebug -h output */
+#define HTB_LOG_GROUPS_STRING_LIST   "ctrl,mmu,sync,main,brg"
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define HTB_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s"
+
+/* Available log groups - Master template
+ *
+ * Group usage is as follows:
+ *    CTRL  - Internal Host Trace information and synchronisation data
+ *    MMU   - MMU page mapping information
+ *    SYNC  - Synchronisation debug
+ *    MAIN  - Data master kicks, etc. tying in with the MAIN group in FWTrace
+ *    DBG   - Temporary debugging group, logs not to be left in the driver
+ *
+ */
+#define HTB_LOG_SFGROUPLIST                               \
+	X( HTB_GROUP_NONE,     NONE  )                        \
+/*     gid,                group flag / apphint name */   \
+	X( HTB_GROUP_CTRL,     CTRL  )                        \
+	X( HTB_GROUP_MMU,      MMU   )                        \
+	X( HTB_GROUP_SYNC,     SYNC  )                        \
+	X( HTB_GROUP_MAIN,     MAIN  )                        \
+	X( HTB_GROUP_BRG,      BRG  )                         \
+/* Debug group HTB_GROUP_DBG must always be last */       \
+	X( HTB_GROUP_DBG,      DBG   )
+
+
+/* Table of String Format specifiers, the group they belong and the number of
+ * arguments each expects. Xmacro styled macros are used to generate what is
+ * needed without requiring hand editing.
+ *
+ * id		: unique id within a group
+ * gid		: group id as defined above
+ * sym name	: symbolic name of enumerations used to identify message strings
+ * string	: Actual string
+ * #args	: number of arguments the string format requires
+ */
+#define HTB_LOG_SFIDLIST \
+/*id,  gid,             sym name,                       string,                           # arguments */ \
+X( 0,  HTB_GROUP_NONE,  HTB_SF_FIRST,                   "You should not use this string", 0, 0) \
+\
+X( 1,  HTB_GROUP_CTRL,  HTB_SF_CTRL_LOGMODE,            "HTB log mode set to %d (1- all PID, 2 - restricted PID)\n", 1, 0) \
+X( 2,  HTB_GROUP_CTRL,  HTB_SF_CTRL_ENABLE_PID,         "HTB enable logging for PID %d\n", 1, 0) \
+X( 3,  HTB_GROUP_CTRL,  HTB_SF_CTRL_ENABLE_GROUP,       "HTB enable logging groups 0x%08x\n", 1, 0) \
+X( 4,  HTB_GROUP_CTRL,  HTB_SF_CTRL_LOG_LEVEL,          "HTB log level set to %d\n", 1, 0) \
+X( 5,  HTB_GROUP_CTRL,  HTB_SF_CTRL_OPMODE,             "HTB operating mode set to %d (1 - droplatest, 2 - drop oldest, 3 - block)\n", 1, 0) \
+X( 6,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_SCALE,       "HTBFWSync OSTS=%08x%08x CRTS=%08x%08x CalcClkSpd=%d\n", 5, 0) \
+X( 7,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_SCALE_RPT,   "FW Sync scale info OSTS=%08x%08x CRTS=%08x%08x CalcClkSpd=%d\n", 5, 0) \
+X( 8,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK,        "FW Sync Partition marker: %d\n", 1, 0) \
+X( 9,  HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK_RPT,    "FW Sync Partition repeat: %d\n", 1, 0) \
+X( 10, HTB_GROUP_CTRL,  HTB_SF_CTRL_FWSYNC_MARK_SCALE,  "Text not used", 6, 0)\
+\
+X( 1,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_TABLE,       "MMU page op table entry page_id=%08x%08x index=%d level=%d val=%08x%08x map=%d\n", 7, 0) \
+X( 2,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_ALLOC,       "MMU allocating DevVAddr from %08x%08x to %08x%08x\n", 4, 0) \
+X( 3,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_FREE,        "MMU freeing DevVAddr from %08x%08x to %08x%08x\n", 4, 0) \
+X( 4,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_MAP,         "MMU mapping DevVAddr %08x%08x to DevPAddr %08x%08x\n", 4, 0) \
+X( 5,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_PMRMAP,      "MMU mapping PMR DevVAddr %08x%08x to DevPAddr %08x%08x\n", 4, 0)  \
+X( 6,  HTB_GROUP_MMU,   HTB_SF_MMU_PAGE_OP_UNMAP,       "MMU unmapping DevVAddr %08x%08x\n", 2, 0) \
+\
+X( 1,  HTB_GROUP_SYNC,  HTB_SF_SYNC_SERVER_ALLOC,       "Server sync allocation [%08X]\n", 1, 0) \
+X( 2,  HTB_GROUP_SYNC,  HTB_SF_SYNC_SERVER_UNREF,       "Server sync unreferenced [%08X]\n", 1, 0) \
+X( 3,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_CREATE,     "Sync OP create 0x%08x, block count=%d, server syncs=%d, client syncs=%d\n", 4, 0) \
+X( 4,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_TAKE,       "Sync OP take 0x%08x server syncs=%d, client syncs=%d\n", 3, 0) \
+X( 5,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_COMPLETE,   "Sync OP complete 0x%08x\n", 1, 0) \
+X( 6,  HTB_GROUP_SYNC,  HTB_SF_SYNC_PRIM_OP_DESTROY,    "Sync OP destroy 0x%08x\n", 1, 0) \
+\
+X( 1,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_TA_DEPRECATED, "Kick TA: FWCtx %08X @ %d\n", 2, 0) \
+X( 2,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_3D_DEPRECATED, "Kick 3D: FWCtx %08X @ %d\n", 2, 0) \
+X( 3,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_CDM_DEPRECATED,"Kick CDM: FWCtx %08X @ %d\n", 2, 0) \
+X( 4,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_RTU,           "Kick RTU: FWCtx %08X @ %d\n", 2, 0) \
+X( 5,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_SHG,           "Kick SHG: FWCtx %08X @ %d\n", 2, 0) \
+X( 6,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_2D_DEPRECATED, "Kick 2D: FWCtx %08X @ %d\n", 2, 0) \
+X( 7,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_UNCOUNTED,     "Kick (uncounted) for all DMs\n", 0, 0) \
+X( 8,  HTB_GROUP_MAIN,  HTB_SF_MAIN_FWCCB_CMD,          "FW CCB Cmd: %d\n", 1, 0) \
+X( 9,  HTB_GROUP_MAIN,  HTB_SF_MAIN_PRE_POWER,          "Pre-power duration @ phase [%d] (0-shutdown,1-startup) RGX: %llu ns SYS: %llu ns\n", 3, 0) \
+X(10,  HTB_GROUP_MAIN,  HTB_SF_MAIN_POST_POWER,         "Post-power duration @ phase [%d] (0-shutdown,1-startup) SYS: %llu ns RGX: %llu ns\n", 3, 0) \
+X(11,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_TA,            "Kick TA: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(12,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_3D,            "Kick 3D: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(13,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_CDM,           "Kick CDM: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(14,  HTB_GROUP_MAIN,  HTB_SF_MAIN_KICK_2D,            "Kick 2D: FWCtx %08x @ %d (frame:%d, ext:0x%08x, int:0x%08x)\n", 5, 0) \
+X(15,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_ERROR,          "Error: (%u) in file: %s:%u\n", 3, 2) \
+X(16,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_MSGLVL_ERROR,   "Log error in file: %s:%u\n", 2, 1) \
+X(17,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_COND_ERROR_T,   "Error: (%u) Conditional is unexpectedly true in file: %s:%u\n", 3, 2) \
+X(18,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_COND_ERROR_F,   "Error: (%u) Conditional is unexpectedly false in file: %s:%u\n", 3, 2) \
+X(19,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_WARNING,        "Warning: (%u) in file: %s @ line: %u\n", 3, 2) \
+X(20,  HTB_GROUP_MAIN,  HTB_SF_MAIN_DBG_MSGLVL_WARN,    "Log warning in file: %s:%u\n", 2, 1) \
+\
+X( 1,  HTB_GROUP_BRG,   HTB_SF_BRG_BRIDGE_CALL,         "Bridge call: start: %010u: bid %03d fid %d\n", 3, 0) \
+X( 2,  HTB_GROUP_BRG,   HTB_SF_BRG_BRIDGE_CALL_ERR,     "Bridge call: start: %010u: bid %03d fid %d error %d\n", 4, 0) \
+\
+X( 1,  HTB_GROUP_DBG,   HTB_SF_DBG_INTPAIR,             "0x%8.8x 0x%8.8x\n", 2, 0) \
+\
+X( 65535, HTB_GROUP_NONE, HTB_SF_LAST,                  "You should not use this string\n", 15, 0)
+
+
+
+/* gid - Group numbers */
+typedef enum _HTB_LOG_SFGROUPS {
+#define X(A,B) A,
+	HTB_LOG_SFGROUPLIST
+#undef X
+} HTB_LOG_SFGROUPS;
+
+
+/* Group flags are stored in an array of elements.
+ * Each of which have a certain number of bits.
+ */
+#define HTB_FLAG_EL_T                   IMG_UINT32
+#define HTB_FLAG_NUM_BITS_IN_EL         (sizeof(HTB_FLAG_EL_T) * 8)
+
+#define HTB_LOG_GROUP_FLAG_GROUP(gid)   ((gid-1) / HTB_FLAG_NUM_BITS_IN_EL)
+#define HTB_LOG_GROUP_FLAG(gid)         (gid ? (0x1 << ((gid-1)%HTB_FLAG_NUM_BITS_IN_EL)) : 0)
+#define HTB_LOG_GROUP_FLAG_NAME(gid)    HTB_LOG_TYPE_ ## gid
+
+/* Group enable flags */
+typedef enum _HTB_LOG_TYPE {
+#define X(a, b) HTB_LOG_GROUP_FLAG_NAME(b) = HTB_LOG_GROUP_FLAG(a),
+	HTB_LOG_SFGROUPLIST
+#undef X
+} HTB_LOG_TYPE;
+
+
+
+/* The symbolic names found in the table above are assigned an ui32 value of
+ * the following format:
+ * 31 30 28 27       20   19  16    15  12      11            0   bits
+ * -   ---   ---- ----     ----      ----        ---- ---- ----
+ *    0-11: id number
+ *   12-15: group id number
+ *   16-19: number of parameters
+ *   20-24: If the format contains a string argument, argument
+ *          index is passed. If there isn't a string argument, 0 is passed
+ *   25-27: unused
+ *   28-30: active: identify SF packet, otherwise regular int32
+ *      31: reserved for signed/unsigned compatibility
+ *
+ * The following macro assigns those values to the enum generated SF ids list.
+ */
+#define HTB_LOG_IDMARKER            (0x70000000)
+#define HTB_LOG_CREATESFID(a,b,e,f)   (((a) | (b << 12) | (e << 16)) | (f << 20) | HTB_LOG_IDMARKER)
+
+#define HTB_LOG_IDMASK              (0xFFF00000)
+#define HTB_LOG_VALIDID(I)          ( ((I) & HTB_LOG_IDMASK) == HTB_LOG_IDMARKER )
+
+typedef enum HTB_LOG_SFids {
+#define X(a, b, c, d, e, f) c = HTB_LOG_CREATESFID(a,b,e,f),
+	HTB_LOG_SFIDLIST
+#undef X
+} HTB_LOG_SFids;
+
+/* Return the group id that the given (enum generated) id belongs to */
+#define HTB_SF_GID(x) (((x)>>12) & 0xf)
+/* Future improvement to support log levels */
+#define HTB_SF_LVL(x) (0)
+/* Returns how many arguments the SF(string format) for the given
+ * (enum generated) id requires.
+ */
+#define HTB_SF_PARAMNUM(x) (((x)>>16) & 0xf)
+/* Returns the index of the argument that contains a string, 1 indexed so 0 means no string argument.
+ */
+#define HTB_SF_STRNUM(x) (((x)>>20) & 0xf)
+/* Returns the id of given enum */
+#define HTB_SF_ID(x) (x & 0xfff)
+
+/* Format of messages is: SF:PID:TID:TIMEPT1:TIMEPT2:[PARn]*
+ */
+#define HTB_LOG_HEADER_SIZE         5
+#define HTB_LOG_MAX_PARAMS          15
+
+/* HTB supports string lengths of up to 23 characters + \0 */
+#define HTB_LOG_STR_ARG_NUM_WORDS         (6)
+#define HTB_LOG_STR_ARG_SIZE              (HTB_LOG_STR_ARG_NUM_WORDS * sizeof(IMG_UINT32))
+
+#if defined(__cplusplus)
+}
+#endif
+
+#if defined(__GNUC__)
+#if GCC_VERSION_AT_LEAST(12, 1)
+#define HTB_FILE_NAME __FILE_NAME__
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+#elif defined(__clang__)
+#if CLANG_VERSION_AT_LEAST(10)
+#define HTB_FILE_NAME __FILE_NAME__
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+#else
+#define HTB_FILE_NAME "n/a"
+#endif
+
+/* Defines for handling MARK_SCALE special case */
+#define HTB_GID_CTRL 1
+#define HTB_ID_MARK_SCALE 10
+#define HTB_MARK_SCALE_ARG_ARRAY_SIZE 6
+
+/* Defines for extracting args from array for special case MARK_SCALE */
+#define HTB_ARG_SYNCMARK 0
+#define HTB_ARG_OSTS_PT1 1
+#define HTB_ARG_OSTS_PT2 2
+#define HTB_ARG_CRTS_PT1 3
+#define HTB_ARG_CRTS_PT2 4
+#define HTB_ARG_CLKSPD   5
+
+#endif /* HTBUFFER_SF_H */
+/*****************************************************************************
+ End of file (htbuffer_sf.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/htbuffer_types.h b/drivers/gpu/drm/img-rogue/htbuffer_types.h
new file mode 100644
index 000000000000..fd061eb428d7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/htbuffer_types.h
@@ -0,0 +1,122 @@
+/*************************************************************************/ /*!
+@File           htbuffer_types.h
+@Title          Host Trace Buffer types.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Host Trace Buffer provides a mechanism to log Host events to a
+                buffer in a similar way to the Firmware Trace mechanism.
+                Host Trace Buffer logs data using a Transport Layer buffer.
+                The Transport Layer and pvrtld tool provides the mechanism to
+                retrieve the trace data.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef HTBUFFER_TYPES_H
+#define HTBUFFER_TYPES_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+#include "htbuffer_sf.h"
+
+/* The group flags array of ints large enough to store all the group flags */
+#if defined(PVRSRV_ENABLE_HTB)
+#define HTB_FLAG_NUM_EL (((HTB_GROUP_DBG-1) / HTB_FLAG_NUM_BITS_IN_EL) + 1)
+extern IMG_INTERNAL HTB_FLAG_EL_T g_auiHTBGroupEnable[HTB_FLAG_NUM_EL];
+
+#define HTB_GROUP_ENABLED(SF) (g_auiHTBGroupEnable[HTB_LOG_GROUP_FLAG_GROUP(HTB_SF_GID(SF))] & HTB_LOG_GROUP_FLAG(HTB_SF_GID(SF)))
+#else
+#define HTB_GROUP_ENABLED(SF) IMG_FALSE
+#endif
+
+/*************************************************************************/ /*!
+ Host Trace Buffer operation mode
+ Care must be taken if changing this enum to ensure the MapFlags[] array
+ in htbserver.c is kept in-step.
+*/ /**************************************************************************/
+typedef enum
+{
+	/*! Undefined operation mode */
+	HTB_OPMODE_UNDEF = 0,
+
+	/*! Drop latest, intended for continuous logging to a UM daemon.
+	 *  If the daemon does not keep up, the most recent log data
+	 *  will be dropped
+	 */
+	HTB_OPMODE_DROPLATEST,
+
+	/*! Drop oldest, intended for crash logging.
+	 *  Data will be continuously written to a circular buffer.
+	 *  After a crash the buffer will contain events leading up to the crash
+	 */
+	HTB_OPMODE_DROPOLDEST,
+
+	/*! Block write if buffer is full */
+	HTB_OPMODE_BLOCK,
+
+	HTB_OPMODE_LAST = HTB_OPMODE_BLOCK
+} HTB_OPMODE_CTRL;
+
+
+/*************************************************************************/ /*!
+ Host Trace Buffer log mode control
+*/ /**************************************************************************/
+typedef enum
+{
+	/*! Undefined log mode, used if update is not applied */
+	HTB_LOGMODE_UNDEF = 0,
+
+	/*! Log trace messages for all PIDs. */
+	HTB_LOGMODE_ALLPID,
+
+	/*! Log trace messages for specific PIDs only. */
+	HTB_LOGMODE_RESTRICTEDPID,
+
+	HTB_LOGMODE_LAST = HTB_LOGMODE_RESTRICTEDPID
+} HTB_LOGMODE_CTRL;
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* HTBUFFER_TYPES_H */
+
+/******************************************************************************
+ End of file (htbuffer_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/img_3dtypes.h b/drivers/gpu/drm/img-rogue/img_3dtypes.h
new file mode 100644
index 000000000000..102c6eeda959
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/img_3dtypes.h
@@ -0,0 +1,248 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global 3D types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines 3D types for use by IMG APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_3DTYPES_H
+#define IMG_3DTYPES_H
+
+#include <powervr/buffer_attribs.h>
+#include "img_types.h"
+#include "img_defs.h"
+
+/**
+ * Comparison functions
+ * This comparison function is defined as:
+ * A {CmpFunc} B
+ * A is a reference value, e.g., incoming depth etc.
+ * B is the sample value, e.g., value in depth buffer.
+ */
+typedef enum _IMG_COMPFUNC_
+{
+	IMG_COMPFUNC_NEVER,			/**< The comparison never succeeds */
+	IMG_COMPFUNC_LESS,			/**< The comparison is a less-than operation */
+	IMG_COMPFUNC_EQUAL,			/**< The comparison is an equal-to operation */
+	IMG_COMPFUNC_LESS_EQUAL,	/**< The comparison is a less-than or equal-to
+									 operation */
+	IMG_COMPFUNC_GREATER,		/**< The comparison is a greater-than operation
+								*/
+	IMG_COMPFUNC_NOT_EQUAL,		/**< The comparison is a no-equal-to operation
+								*/
+	IMG_COMPFUNC_GREATER_EQUAL,	/**< The comparison is a greater-than or
+									 equal-to operation */
+	IMG_COMPFUNC_ALWAYS,		/**< The comparison always succeeds */
+} IMG_COMPFUNC;
+
+/**
+ * Stencil op functions
+ */
+typedef enum _IMG_STENCILOP_
+{
+	IMG_STENCILOP_KEEP,		/**< Keep original value */
+	IMG_STENCILOP_ZERO,		/**< Set stencil to 0 */
+	IMG_STENCILOP_REPLACE,	/**< Replace stencil entry */
+	IMG_STENCILOP_INCR_SAT,	/**< Increment stencil entry, clamping to max */
+	IMG_STENCILOP_DECR_SAT,	/**< Decrement stencil entry, clamping to zero */
+	IMG_STENCILOP_INVERT,	/**< Invert bits in stencil entry */
+	IMG_STENCILOP_INCR,		/**< Increment stencil entry,
+								 wrapping if necessary */
+	IMG_STENCILOP_DECR,		/**< Decrement stencil entry,
+								 wrapping if necessary */
+} IMG_STENCILOP;
+
+/**
+ * Alpha blending allows colours and textures on one surface
+ * to be blended with transparency onto another surface.
+ * These definitions apply to both source and destination blending
+ * states
+ */
+typedef enum _IMG_BLEND_
+{
+	IMG_BLEND_ZERO = 0,        /**< Blend factor is (0,0,0,0) */
+	IMG_BLEND_ONE,             /**< Blend factor is (1,1,1,1) */
+	IMG_BLEND_SRC_COLOUR,      /**< Blend factor is the source colour */
+	IMG_BLEND_INV_SRC_COLOUR,  /**< Blend factor is the inverted source colour
+									(i.e. 1-src_col) */
+	IMG_BLEND_SRC_ALPHA,       /**< Blend factor is the source alpha */
+	IMG_BLEND_INV_SRC_ALPHA,   /**< Blend factor is the inverted source alpha
+									(i.e. 1-src_alpha) */
+	IMG_BLEND_DEST_ALPHA,      /**< Blend factor is the destination alpha */
+	IMG_BLEND_INV_DEST_ALPHA,  /**< Blend factor is the inverted destination
+									alpha */
+	IMG_BLEND_DEST_COLOUR,     /**< Blend factor is the destination colour */
+	IMG_BLEND_INV_DEST_COLOUR, /**< Blend factor is the inverted destination
+									colour */
+	IMG_BLEND_SRC_ALPHASAT,    /**< Blend factor is the alpha saturation (the
+									minimum of (Src alpha,
+									1 - destination alpha)) */
+	IMG_BLEND_BLEND_FACTOR,    /**< Blend factor is a constant */
+	IMG_BLEND_INVBLEND_FACTOR, /**< Blend factor is a constant (inverted)*/
+	IMG_BLEND_SRC1_COLOUR,     /**< Blend factor is the colour outputted from
+									the pixel shader */
+	IMG_BLEND_INV_SRC1_COLOUR, /**< Blend factor is the inverted colour
+									outputted from the pixel shader */
+	IMG_BLEND_SRC1_ALPHA,      /**< Blend factor is the alpha outputted from
+									the pixel shader */
+	IMG_BLEND_INV_SRC1_ALPHA   /**< Blend factor is the inverted alpha
+									outputted from the pixel shader */
+} IMG_BLEND;
+
+/**
+ * The arithmetic operation to perform when blending
+ */
+typedef enum _IMG_BLENDOP_
+{
+	IMG_BLENDOP_ADD = 0,      /**< Result = (Source + Destination) */
+	IMG_BLENDOP_SUBTRACT,     /**< Result = (Source - Destination) */
+	IMG_BLENDOP_REV_SUBTRACT, /**< Result = (Destination - Source) */
+	IMG_BLENDOP_MIN,          /**< Result = min (Source, Destination) */
+	IMG_BLENDOP_MAX           /**< Result = max (Source, Destination) */
+} IMG_BLENDOP;
+
+/**
+ * Logical operation to perform when logic ops are enabled
+ */
+typedef enum _IMG_LOGICOP_
+{
+	IMG_LOGICOP_CLEAR = 0,     /**< Result = 0 */
+	IMG_LOGICOP_SET,           /**< Result = -1 */
+	IMG_LOGICOP_COPY,          /**< Result = Source */
+	IMG_LOGICOP_COPY_INVERTED, /**< Result = ~Source */
+	IMG_LOGICOP_NOOP,          /**< Result = Destination */
+	IMG_LOGICOP_INVERT,        /**< Result = ~Destination */
+	IMG_LOGICOP_AND,           /**< Result = Source & Destination */
+	IMG_LOGICOP_NAND,          /**< Result = ~(Source & Destination) */
+	IMG_LOGICOP_OR,            /**< Result = Source | Destination */
+	IMG_LOGICOP_NOR,           /**< Result = ~(Source | Destination) */
+	IMG_LOGICOP_XOR,           /**< Result = Source ^ Destination */
+	IMG_LOGICOP_EQUIV,         /**< Result = ~(Source ^ Destination) */
+	IMG_LOGICOP_AND_REVERSE,   /**< Result = Source & ~Destination */
+	IMG_LOGICOP_AND_INVERTED,  /**< Result = ~Source & Destination */
+	IMG_LOGICOP_OR_REVERSE,    /**< Result = Source | ~Destination */
+	IMG_LOGICOP_OR_INVERTED    /**< Result = ~Source | Destination */
+} IMG_LOGICOP;
+
+/**
+ * Type of fog blending supported
+ */
+typedef enum _IMG_FOGMODE_
+{
+	IMG_FOGMODE_NONE, /**< No fog blending - fog calculations are
+					   *   based on the value output from the vertex phase */
+	IMG_FOGMODE_LINEAR, /**< Linear interpolation */
+	IMG_FOGMODE_EXP, /**< Exponential */
+	IMG_FOGMODE_EXP2, /**< Exponential squaring */
+} IMG_FOGMODE;
+
+/**
+ * Types of filtering
+ */
+typedef enum _IMG_FILTER_
+{
+	IMG_FILTER_DONTCARE,	/**< Any filtering mode is acceptable */
+	IMG_FILTER_POINT,		/**< Point filtering */
+	IMG_FILTER_LINEAR,		/**< Bi-linear filtering */
+	IMG_FILTER_BICUBIC,		/**< Bi-cubic filtering */
+} IMG_FILTER;
+
+/**
+ * Addressing modes for textures
+ */
+typedef enum _IMG_ADDRESSMODE_
+{
+	IMG_ADDRESSMODE_DONTCARE,
+	IMG_ADDRESSMODE_REPEAT,	/**< Texture repeats continuously */
+	IMG_ADDRESSMODE_FLIP, /**< Texture flips on odd integer part */
+	IMG_ADDRESSMODE_CLAMP, /**< Texture clamped at 0 or 1 */
+	IMG_ADDRESSMODE_FLIPCLAMP, /**< Flipped once, then clamp */
+	IMG_ADDRESSMODE_CLAMPBORDER,
+	IMG_ADDRESSMODE_OGL_CLAMP,
+	IMG_ADDRESSMODE_OVG_TILEFILL,
+} IMG_ADDRESSMODE;
+
+/**
+ * Culling based on winding order of triangle.
+ */
+typedef enum _IMG_CULLMODE_
+{
+	IMG_CULLMODE_NONE,			/**< Don't cull */
+	IMG_CULLMODE_FRONTFACING,	/**< Front facing triangles */
+	IMG_CULLMODE_BACKFACING,	/**< Back facing triangles */
+} IMG_CULLMODE;
+
+/**
+ * Colour for clearing surfaces.
+ *  The four elements of the 4 x 32 bit array will map to colour
+ *  R,G,B,A components, in order.
+ *  For YUV colour space the order is Y,U,V.
+ *  For Depth and Stencil formats D maps to R and S maps to G.
+ */
+typedef union IMG_CLEAR_COLOUR_TAG {
+	IMG_UINT32        aui32[4];
+	IMG_INT32         ai32[4];
+	IMG_FLOAT         af32[4];
+} IMG_CLEAR_COLOUR;
+
+static_assert(sizeof(IMG_FLOAT) == sizeof(IMG_INT32), "Size of IMG_FLOAT is not 32 bits.");
+
+/*! ************************************************************************//**
+@brief          Specifies the MSAA resolve operation.
+*/ /**************************************************************************/
+typedef enum _IMG_RESOLVE_OP_
+{
+	IMG_RESOLVE_BLEND   = 0,          /*!< box filter on the samples */
+	IMG_RESOLVE_MIN     = 1,          /*!< minimum of the samples */
+	IMG_RESOLVE_MAX     = 2,          /*!< maximum of the samples */
+	IMG_RESOLVE_SAMPLE0 = 3,          /*!< choose sample 0 */
+	IMG_RESOLVE_SAMPLE1 = 4,          /*!< choose sample 1 */
+	IMG_RESOLVE_SAMPLE2 = 5,          /*!< choose sample 2 */
+	IMG_RESOLVE_SAMPLE3 = 6,          /*!< choose sample 3 */
+	IMG_RESOLVE_SAMPLE4 = 7,          /*!< choose sample 4 */
+	IMG_RESOLVE_SAMPLE5 = 8,          /*!< choose sample 5 */
+	IMG_RESOLVE_SAMPLE6 = 9,          /*!< choose sample 6 */
+	IMG_RESOLVE_SAMPLE7 = 10,         /*!< choose sample 7 */
+} IMG_RESOLVE_OP;
+
+
+#endif /* IMG_3DTYPES_H */
+/******************************************************************************
+ End of file (img_3dtypes.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/img_defs.h b/drivers/gpu/drm/img-rogue/img_defs.h
new file mode 100644
index 000000000000..b0992b7188c9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/img_defs.h
@@ -0,0 +1,629 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common header containing type definitions for portability
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains variable and structure definitions. Any platform
+                specific types should be defined in this file.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DEFS_H
+#define IMG_DEFS_H
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include <linux/types.h>
+#else
+#include <stddef.h>
+#endif
+#if !(defined(__linux__) && defined(__KERNEL__))
+#if defined(__riscv)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wundef"
+#endif
+#include <assert.h>
+#if defined(__riscv)
+#pragma GCC diagnostic pop
+#endif
+#endif
+
+#include "img_types.h"
+
+#if defined(NO_INLINE_FUNCS)
+	#define	INLINE
+	#define	FORCE_INLINE
+#else
+#if defined(__cplusplus) || defined(INTEGRITY_OS)
+	#if	!defined(INLINE)
+		#define INLINE				inline
+	#endif
+	#define	FORCE_INLINE			static inline
+#else
+#if	!defined(INLINE)
+	#define	INLINE					__inline
+#endif
+#if (defined(UNDER_WDDM) || defined(WINDOWS_WDF)) && defined(_X86_)
+	#define	FORCE_INLINE			__forceinline
+#else
+	#define	FORCE_INLINE			static __inline
+#endif
+#endif
+#endif
+
+/* True if the GCC version is at least the given version. False for older
+ * versions of GCC, or other compilers.
+ */
+#if defined(__GNUC__)
+#define GCC_VERSION_AT_LEAST(major, minor) \
+	(__GNUC__ > (major) || \
+	(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#define GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if defined(__clang__)
+#define CLANG_VERSION_AT_LEAST(major) \
+	(__clang_major__ >= (major))
+#else
+#define CLANG_VERSION_AT_LEAST(major) 0
+#endif
+
+/* Use Clang's __has_extension and __has_builtin macros if available. */
+#if defined(__has_extension)
+#define has_clang_extension(e) __has_extension(e)
+#else
+#define has_clang_extension(e) 0
+#endif
+
+#if defined(__has_builtin)
+#define has_clang_builtin(e) __has_builtin(e)
+#else
+#define has_clang_builtin(e) 0
+#endif
+
+/* Use this in any file, or use attributes under GCC - see below */
+#ifndef PVR_UNREFERENCED_PARAMETER
+#define	PVR_UNREFERENCED_PARAMETER(param) ((void)(param))
+#endif
+
+/* static_assert(condition, "message to print if it fails");
+ *
+ * Assert something at compile time. If the assertion fails, try to print
+ * the message, otherwise do nothing. static_assert is available if:
+ *
+ * - It's already defined as a macro (e.g. by <assert.h> in C11)
+ * - We're using MSVC which exposes static_assert unconditionally
+ * - We're using a C++ compiler that supports C++11
+ * - We're using GCC 4.6 and up in C mode (in which case it's available as
+ *   _Static_assert)
+ *
+ * In all other cases, fall back to an equivalent that makes an invalid
+ * declaration.
+ */
+#if !defined(static_assert) && !defined(_MSC_VER) && \
+		(!defined(__cplusplus) || __cplusplus < 201103L) || defined(__KLOCWORK__)
+	/* static_assert isn't already available */
+	#if !defined(__cplusplus) && (GCC_VERSION_AT_LEAST(4, 6) || \
+								  (defined(__clang__) && has_clang_extension(c_static_assert)))
+		#define static_assert _Static_assert
+	#else
+		#define static_assert(expr, message) \
+			extern int static_assert_failed[(expr) ? 1 : -1] __attribute__((unused))
+	#endif
+#endif
+
+/*
+ * unreachable("explanation") can be used to indicate to the compiler that
+ * some parts of the code can never be reached, like the default branch
+ * of a switch that covers all real-world possibilities, even though there
+ * are other ints that exist for instance.
+ *
+ * The message will be printed as an assert() when debugging.
+ *
+ * Note: there is no need to add a 'return' or any error handling after
+ * calling unreachable(), as this call will never return.
+ */
+#if defined(__linux__) && defined(__KERNEL__)
+/* Kernel has its own unreachable(), which is a simple infinite loop */
+#elif GCC_VERSION_AT_LEAST(4, 5) || has_clang_builtin(__builtin_unreachable)
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			__builtin_unreachable(); \
+		} while (false)
+#elif defined(_MSC_VER)
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			__assume(0); \
+		} while (false)
+#else
+	#define unreachable(msg) \
+		do { \
+			assert(!(msg)); \
+			while (1); \
+		} while (false)
+#endif
+
+/*
+ * assume(x > 2 && x <= 7) works like an assert(), except it hints to the
+ * compiler what it can assume to optimise the code, like a limited range
+ * of parameter values.
+ */
+#if has_clang_builtin(__builtin_assume)
+	#define assume(expr) \
+		do { \
+			assert(expr); \
+			__builtin_assume(expr); \
+		} while (false)
+#elif defined(_MSC_VER)
+	#define assume(expr) \
+		do { \
+			assert(expr); \
+			__assume(expr); \
+		} while (false)
+#elif defined(__linux__) && defined(__KERNEL__)
+	#define assume(expr) ((void)(expr))
+#elif GCC_VERSION_AT_LEAST(4, 5) || has_clang_builtin(__builtin_unreachable)
+	#define assume(expr) \
+		do { \
+			if (unlikely(!(expr))) \
+				unreachable("Assumption isn't true: " # expr); \
+		} while (false)
+#else
+	#define assume(expr) assert(expr)
+#endif
+
+/*! Macro to calculate the n-byte aligned value from that supplied rounding up.
+ * n must be a power of two.
+ */
+#define PVR_ALIGN(_x, _n)	((((_x)+((_n)-1U))|((_n)-1U))^((_n)-1U))
+
+#if defined(_WIN32)
+
+#if defined(WINDOWS_WDF)
+
+	/*
+	 * For WINDOWS_WDF drivers we don't want these defines to overwrite calling conventions propagated through the build system.
+	 * This 'empty' choice helps to resolve all the calling conv issues.
+	 *
+	 */
+	#define IMG_CALLCONV
+	#define C_CALLCONV
+
+	#define IMG_INTERNAL
+	#define IMG_RESTRICT __restrict
+
+	/*
+	 * The proper way of dll linking under MS compilers is made of two things:
+	 * - decorate implementation with __declspec(dllexport)
+	 *   this decoration helps compiler with making the so called
+	 *   'export library'
+	 * - decorate forward-declaration (in a source dependent on a dll) with
+	 *   __declspec(dllimport), this decoration helps the compiler to make
+	 *   faster and smaller code in terms of calling dll-imported functions
+	 *
+	 * Usually these decorations are performed by having a single macro define
+	 * making that expands to a proper __declspec() depending on the
+	 * translation unit, dllexport inside the dll source and dllimport outside
+	 * the dll source. Having IMG_EXPORT and IMG_IMPORT resolving to the same
+	 * __declspec() makes no sense, but at least works.
+	 */
+	#define IMG_IMPORT __declspec(dllexport)
+	#define IMG_EXPORT __declspec(dllexport)
+
+#else
+
+	#define IMG_CALLCONV __stdcall
+	#define IMG_INTERNAL
+	#define	IMG_EXPORT	__declspec(dllexport)
+	#define IMG_RESTRICT __restrict
+	#define C_CALLCONV	__cdecl
+
+	/*
+	 * IMG_IMPORT is defined as IMG_EXPORT so that headers and implementations
+	 * match. Some compilers require the header to be declared IMPORT, while
+	 * the implementation is declared EXPORT.
+	 */
+	#define	IMG_IMPORT	IMG_EXPORT
+
+#endif
+
+#if defined(UNDER_WDDM)
+	#ifndef	_INC_STDLIB
+		#if defined(__mips)
+			/* do nothing */
+		#elif defined(UNDER_MSBUILD)
+			/* do nothing */
+		#else
+			_CRTIMP void __cdecl abort(void);
+		#endif
+	#endif
+#endif /* UNDER_WDDM */
+#else
+	#if (defined(__linux__) || defined(__QNXNTO__)) && defined(__KERNEL__)
+		#define IMG_INTERNAL
+		#define IMG_EXPORT
+		#define IMG_CALLCONV
+	#elif defined(__linux__) || defined(__METAG) || defined(__mips) || defined(__QNXNTO__) || defined(__riscv) || defined(__APPLE__) || defined(TEE_DDK)
+		#define IMG_CALLCONV
+		#define C_CALLCONV
+
+		#if defined(__METAG)
+			#define IMG_INTERNAL
+		#else
+			#define IMG_INTERNAL    __attribute__((visibility("hidden")))
+		#endif
+
+		#define IMG_EXPORT      __attribute__((visibility("default")))
+		#define IMG_RESTRICT    __restrict__
+	#elif defined(INTEGRITY_OS)
+		#define IMG_CALLCONV
+		#define IMG_INTERNAL
+		#define IMG_EXPORT
+		#define IMG_RESTRICT
+		#define C_CALLCONV
+		#define __cdecl
+
+		#ifndef USE_CODE
+			#define IMG_ABORT() printf("IMG_ABORT was called.\n")
+		#endif
+	#else
+		#error("define an OS")
+	#endif
+
+#endif
+
+/* Use default definition if not overridden */
+#ifndef IMG_ABORT
+	#if defined(EXIT_ON_ABORT)
+		#define IMG_ABORT()	exit(1)
+	#else
+		#define IMG_ABORT()	abort()
+	#endif
+#endif
+
+/* The best way to suppress unused parameter warnings using GCC is to use a
+ * variable attribute. Place the __maybe_unused between the type and name of an
+ * unused parameter in a function parameter list e.g. 'int __maybe_unused var'.
+ * This should only be used in GCC build environments, for example, in files
+ * that compile only on Linux.
+ * Other files should use PVR_UNREFERENCED_PARAMETER
+ */
+
+/* Kernel macros for compiler attributes */
+/* Note: param positions start at 1 */
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/compiler.h>
+
+	#if !defined(__fallthrough)
+		#if GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+			#define __fallthrough __attribute__((__fallthrough__))
+		#else
+			#define __fallthrough
+		#endif
+	#endif
+#elif defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES)
+	#define __must_check       __attribute__((warn_unused_result))
+	#define __maybe_unused     __attribute__((unused))
+	#define __malloc           __attribute__((malloc))
+
+	/* Bionic's <sys/cdefs.h> might have defined these already */
+	/* See https://android.googlesource.com/platform/bionic.git/+/master/libc/include/sys/cdefs.h */
+	#if !defined(__packed)
+		#define __packed           __attribute__((packed))
+	#endif
+	#if !defined(__aligned)
+		#define __aligned(n)       __attribute__((aligned(n)))
+	#endif
+	#if !defined(__noreturn)
+		#define __noreturn         __attribute__((noreturn))
+	#endif
+
+	/* That one compiler that supports attributes but doesn't support
+	 * the printf attribute... */
+	#if defined(__GNUC__)
+		#if defined(__MINGW32__)
+		    #define __printf(fmt, va)  __attribute__((format(gnu_printf, (fmt), (va))))
+		#else
+			#define __printf(fmt, va)  __attribute__((format(printf, (fmt), (va))))
+		#endif
+	#else
+		#define __printf(fmt, va)
+	#endif /* defined(__GNUC__) */
+
+	#if defined(__cplusplus) && (__cplusplus >= 201703L)
+		#define __fallthrough [[fallthrough]]
+	#elif GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+		#define __fallthrough __attribute__((__fallthrough__))
+	#else
+		#define __fallthrough
+	#endif
+
+	#define __user
+	#define __force
+	#define __iomem
+#else
+	/* Silently ignore those attributes */
+	#define __printf(fmt, va)
+	#define __packed
+	#define __aligned(n)
+	#define __must_check
+	#define __maybe_unused
+	#define __malloc
+
+	#if defined(_MSC_VER) || defined(CC_ARM)
+		#define __noreturn __declspec(noreturn)
+	#else
+		#define __noreturn
+	#endif
+
+	/* This may already been defined, e.g. by SAL (Source Annotation Language) */
+	#if !defined(__fallthrough)
+		#define __fallthrough
+	#endif
+
+	#define __user
+	#define __force
+	#define __iomem
+#endif
+
+
+/* Other attributes, following the same style */
+#if defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES)
+	#define __const_function      __attribute__((const))
+#else
+	#define __const_function
+#endif
+
+
+/* GCC builtins */
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/compiler.h>
+#elif defined(__GNUC__) || defined(INTEGRITY_OS)
+
+/* Klocwork does not support __builtin_expect, which makes the actual condition
+ * expressions hidden during analysis, affecting it negatively. */
+#if !defined(__KLOCWORK__) && !defined(INTEGRITY_OS) && !defined(DEBUG)
+	#define likely(x)   __builtin_expect(!!(x), 1)
+	#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+
+	/* Compiler memory barrier to prevent reordering */
+	#define barrier() __asm__ __volatile__("": : :"memory")
+#else
+	#define barrier() static_assert(0, "barrier() isn't supported by your compiler");
+#endif
+
+/* That one OS that defines one but not the other... */
+#ifndef likely
+	#define likely(x)   (x)
+#endif
+#ifndef unlikely
+	#define unlikely(x) (x)
+#endif
+
+#if !defined(BITS_PER_BYTE)
+#define BITS_PER_BYTE (8)
+#endif /* BITS_PER_BYTE */
+
+/* These two macros are also provided by the kernel */
+#ifndef BIT
+#define BIT(b) (1UL << (b))
+#endif
+
+#ifndef BIT_ULL
+#define BIT_ULL(b) (1ULL << (b))
+#endif
+
+#define BIT_SET(f, b)     BITMASK_SET((f),    BIT(b))
+#define BIT_UNSET(f, b)   BITMASK_UNSET((f),  BIT(b))
+#define BIT_TOGGLE(f, b)  BITMASK_TOGGLE((f), BIT(b))
+#define BIT_ISSET(f, b)   BITMASK_HAS((f),    BIT(b))
+
+#define BITMASK_SET(f, m)     do { ((f) |= (m)); } while (false)
+#define BITMASK_UNSET(f, m)   do { ((f) &= ~(m)); } while (false)
+#define BITMASK_TOGGLE(f, m)  do { ((f) ^= (m)); } while (false)
+#define BITMASK_HAS(f, m)     (((f) & (m)) == (m)) /* the bits from the mask are all set */
+#define BITMASK_ANY(f, m)     (((f) & (m)) != 0U)  /* any bit from the mask is set */
+
+#ifndef MAX
+#define MAX(a ,b)	(((a) > (b)) ? (a) : (b))
+#endif
+
+#ifndef MIN
+#define MIN(a, b)	(((a) < (b)) ? (a) : (b))
+#endif
+
+#ifndef CLAMP
+#define CLAMP(min, max, n)  ((n) < (min) ? (min) : ((n) > (max) ? (max) : (n)))
+#endif
+
+#define SWAP(X, Y) (X) ^= (Y); (Y) ^= (X); (X) ^= (Y);
+
+#if defined(__linux__) && defined(__KERNEL__)
+	#include <linux/kernel.h>
+	#include <linux/bug.h>
+#endif
+
+/* Get a structure's address from the address of a member */
+#define IMG_CONTAINER_OF(ptr, type, member) \
+	(type *) ((uintptr_t) (ptr) - offsetof(type, member))
+
+/* Get a new pointer with an offset (in bytes) from a base address, useful
+ * when traversing byte buffers and accessing data in buffers through struct
+ * pointers.
+ * Note, this macro is not equivalent to or replacing offsetof() */
+#define IMG_OFFSET_ADDR(addr, offset_in_bytes) \
+	(void*)&(((IMG_UINT8*)(void*)(addr))[offset_in_bytes])
+
+/* Get a new pointer with an offset (in bytes) from a base address, version
+ * for volatile memory.
+ */
+#define IMG_OFFSET_ADDR_VOLATILE(addr, offset_in_bytes) \
+	(volatile void*)&(((volatile IMG_UINT8*)(volatile void*)(addr))[offset_in_bytes])
+
+/* Get a new pointer with an offset (in dwords) from a base address, useful
+ * when traversing byte buffers and accessing data in buffers through struct
+ * pointers.
+ * Note, this macro is not equivalent to or replacing offsetof() */
+#define IMG_OFFSET_ADDR_DW(addr, offset_in_dwords) \
+	(void*)(((IMG_UINT32*)(void*)(addr)) + (offset_in_dwords))
+
+/* The number of elements in a fixed-sized array */
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(ARR) (sizeof(ARR) / sizeof((ARR)[0]))
+#endif
+
+/* To guarantee that __func__ can be used, define it as a macro here if it
+   isn't already provided by the compiler. */
+#if defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus < 201103L)
+#define __func__ __FUNCTION__
+#endif
+
+#if defined(__cplusplus)
+/* C++ Specific:
+ * Disallow use of copy and assignment operator within a class.
+ * Should be placed under private. */
+#define IMG_DISALLOW_COPY_AND_ASSIGN(C) \
+	C(const C&); \
+	void operator=(const C&)
+#endif
+
+#if defined(SUPPORT_PVR_VALGRIND) && !defined(__METAG) && !defined(__mips) && !defined(__riscv)
+	#include "/usr/include/valgrind/memcheck.h"
+
+	#define VG_MARK_INITIALIZED(pvData,ui32Size) VALGRIND_MAKE_MEM_DEFINED(pvData,ui32Size)
+	#define VG_MARK_NOACCESS(pvData,ui32Size) VALGRIND_MAKE_MEM_NOACCESS(pvData,ui32Size)
+	#define VG_MARK_ACCESS(pvData,ui32Size) VALGRIND_MAKE_MEM_UNDEFINED(pvData,ui32Size)
+	#define VG_ASSERT_DEFINED(pvData,ui32Size) VALGRIND_CHECK_MEM_IS_DEFINED(pvData,ui32Size)
+#else
+	#if defined(_MSC_VER)
+	#	define PVR_MSC_SUPPRESS_4127 __pragma(warning(suppress:4127))
+	#else
+	#	define PVR_MSC_SUPPRESS_4127
+	#endif
+
+	#define VG_MARK_INITIALIZED(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_MARK_NOACCESS(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_MARK_ACCESS(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+	#define VG_ASSERT_DEFINED(pvData,ui32Size) PVR_MSC_SUPPRESS_4127 do { } while (false)
+#endif
+
+#define IMG_STRINGIFY_IMPL(x) # x
+#define IMG_STRINGIFY(x) IMG_STRINGIFY_IMPL(x)
+
+#define IMG_CONCATENATE_IMPL(x,y) x ## y
+#define IMG_CONCATENATE(x,y) IMG_CONCATENATE_IMPL(x,y)
+
+#if defined(DEBUG) && !defined(INTEGRITY_OS)
+#define IMG_PAGESLOG2BYTES(_tcast, _npages, _log2) \
+	({ \
+		PVR_ASSERT( ((IMG_UINT64)(1ULL) << (sizeof(_tcast)*8UL)) >= ((IMG_UINT64)(_npages) << (_log2)) ); \
+		(_tcast)(_npages) << (_log2); \
+	})
+#else
+#define IMG_PAGESLOG2BYTES(_tcast, _npages, _log2) ((_npages) << (_log2))
+#endif
+
+#define IMG_PAGE2BYTES32(logsize) IMG_PAGESLOG2BYTES(IMG_UINT32,IMG_UINT32_C(1),logsize)
+#define IMG_PAGE2BYTES64(logsize) ((IMG_UINT64)IMG_UINT64_C(1) << (logsize))
+
+#define IMG_PAGES2BYTES32(pages,logsize) IMG_PAGESLOG2BYTES(IMG_UINT32,pages,logsize)
+#define IMG_PAGES2BYTES64(pages,logsize) ((IMG_UINT64)(pages) << (logsize))
+
+#define IMG_PAGE_SHIFT_4KB   12U
+#define IMG_PAGE_SHIFT_16KB  14U
+#define IMG_PAGE_SHIFT_64KB  16U
+#define IMG_PAGE_SHIFT_256KB 18U
+#define IMG_PAGE_SHIFT_1MB   20U
+#define IMG_PAGE_SHIFT_2MB   21U
+
+#if defined(INTEGRITY_OS)
+	/* Definitions not present in INTEGRITY. */
+	#define PATH_MAX	200
+#endif
+
+#if defined(__clang__) || defined(__GNUC__)
+	/* __SIZEOF_POINTER__ is defined already by these compilers */
+#elif defined(INTEGRITY_OS)
+	#if defined(__Ptr_Is_64)
+		#define __SIZEOF_POINTER__ 8
+	#else
+		#define __SIZEOF_POINTER__ 4
+	#endif
+#elif defined(_WIN32)
+	#define __SIZEOF_POINTER__ sizeof(char *)
+#else
+	#warning Unknown OS - using default method to determine whether CPU arch is 64-bit.
+	#define __SIZEOF_POINTER__ sizeof(char *)
+#endif
+
+/* RDI8567: gcc/clang/llvm load/store optimisations may cause issues with
+ * uncached device memory allocations. Some pointers are made 'volatile'
+ * to prevent those optimisations being applied to writes through those
+ * pointers.
+ */
+#if (GCC_VERSION_AT_LEAST(7, 0) || defined(__clang__)) && (defined(__arm64__) || defined(__aarch64__))
+#define NOLDSTOPT volatile
+/* after applying 'volatile' to a pointer, we may need to cast it to 'void *'
+ * to keep it compatible with its existing uses.
+ */
+#define NOLDSTOPT_VOID (void *)
+
+#define NOLDSTOPT_REQUIRED 1
+#else
+#define NOLDSTOPT
+#define NOLDSTOPT_VOID
+#endif
+
+#define PVR_PRE_DPF (void) printf
+
+/* C STD >= C99 */
+#if !defined(INTEGRITY_OS) && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+#define IMG_FLEX_ARRAY_MEMBER
+#define IMG_FLEX_ARRAY_SIZE(size, count) ((size) * (count))
+#else
+/* In C STD prior to C99 flexible array members are an extension feature and syntax requires alternative approach */
+#define IMG_FLEX_ARRAY_MEMBER (1)
+#define IMG_FLEX_ARRAY_SIZE(size, count) ((size) * ((count) - 1))
+#endif
+
+#endif /* IMG_DEFS_H */
+/*****************************************************************************
+ End of file (img_defs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/img_elf.h b/drivers/gpu/drm/img-rogue/img_elf.h
new file mode 100644
index 000000000000..8837d9592599
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/img_elf.h
@@ -0,0 +1,111 @@
+/*************************************************************************/ /*!
+@File           img_elf.h
+@Title          IMG ELF file definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    Definitions for ELF file structures used in the DDK.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(IMG_ELF_H)
+#define IMG_ELF_H
+
+#include "img_types.h"
+
+/* ELF format defines */
+#define ELF_PT_LOAD     (0x1U)   /* Program header identifier as Load */
+#define ELF_SHT_SYMTAB  (0x2U)   /* Section identifier as Symbol Table */
+#define ELF_SHT_STRTAB  (0x3U)   /* Section identifier as String Table */
+#define MAX_STRTAB_NUM  (0x8U)   /* Maximum number of string table in the ELF file */
+
+/* Redefined structs of ELF format */
+typedef struct
+{
+	IMG_UINT8    ui32Eident[16];
+	IMG_UINT16   ui32Etype;
+	IMG_UINT16   ui32Emachine;
+	IMG_UINT32   ui32Eversion;
+	IMG_UINT32   ui32Eentry;
+	IMG_UINT32   ui32Ephoff;
+	IMG_UINT32   ui32Eshoff;
+	IMG_UINT32   ui32Eflags;
+	IMG_UINT16   ui32Eehsize;
+	IMG_UINT16   ui32Ephentsize;
+	IMG_UINT16   ui32Ephnum;
+	IMG_UINT16   ui32Eshentsize;
+	IMG_UINT16   ui32Eshnum;
+	IMG_UINT16   ui32Eshtrndx;
+} IMG_ELF_HDR;
+
+typedef struct
+{
+	IMG_UINT32   ui32Stname;
+	IMG_UINT32   ui32Stvalue;
+	IMG_UINT32   ui32Stsize;
+	IMG_UINT8    ui32Stinfo;
+	IMG_UINT8    ui32Stother;
+	IMG_UINT16   ui32Stshndx;
+} IMG_ELF_SYM;
+
+typedef struct
+{
+	IMG_UINT32   ui32Shname;
+	IMG_UINT32   ui32Shtype;
+	IMG_UINT32   ui32Shflags;
+	IMG_UINT32   ui32Shaddr;
+	IMG_UINT32   ui32Shoffset;
+	IMG_UINT32   ui32Shsize;
+	IMG_UINT32   ui32Shlink;
+	IMG_UINT32   ui32Shinfo;
+	IMG_UINT32   ui32Shaddralign;
+	IMG_UINT32   ui32Shentsize;
+} IMG_ELF_SHDR;
+
+typedef struct
+{
+	IMG_UINT32   ui32Ptype;
+	IMG_UINT32   ui32Poffset;
+	IMG_UINT32   ui32Pvaddr;
+	IMG_UINT32   ui32Ppaddr;
+	IMG_UINT32   ui32Pfilesz;
+	IMG_UINT32   ui32Pmemsz;
+	IMG_UINT32   ui32Pflags;
+	IMG_UINT32   ui32Palign;
+} IMG_ELF_PROGRAM_HDR;
+
+#endif /* IMG_ELF_H */
diff --git a/drivers/gpu/drm/img-rogue/img_types.h b/drivers/gpu/drm/img-rogue/img_types.h
new file mode 100644
index 000000000000..1ceebc5d5552
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/img_types.h
@@ -0,0 +1,331 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines type aliases for use by IMG APIs.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_TYPES_H
+#define IMG_TYPES_H
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* To use C99 types and definitions, there are two special cases we need to
+ * cater for:
+ *
+ * - Visual Studio: in VS2010 or later, some standard headers are available,
+ *   and MSVC has its own built-in sized types. We can define the C99 types
+ *   in terms of these.
+ *
+ * - Linux kernel code: C99 sized types are defined in <linux/types.h>, but
+ *   some other features (like macros for constants or printf format
+ *   strings) are missing, so we need to fill in the gaps ourselves.
+ *
+ * For other cases (userspace code under Linux, Android or Neutrino, or
+ * firmware code), we can include the standard headers.
+ */
+#if defined(_MSC_VER)
+	#include <stdbool.h>		/* bool */
+	#include "msvc_types.h"
+#elif defined(__linux__) && defined(__KERNEL__)
+	#include <linux/version.h>
+	#include <linux/types.h>
+	#include "kernel_types.h"
+#elif defined(__linux__) || defined(__METAG) || defined(__MINGW32__) || \
+	defined(__QNXNTO__) || defined(INTEGRITY_OS) || defined(__riscv) || \
+	defined(__APPLE__) || defined(TEE_DDK)
+	#include <stddef.h>			/* NULL */
+	#include <stdint.h>
+#if defined(__riscv)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wundef"
+#endif
+	#include <inttypes.h>		/* intX_t/uintX_t, format specifiers */
+	#include <limits.h>			/* INT_MIN, etc */
+	#include <sys/types.h>		/* ssize_t */
+#if defined(__riscv)
+#pragma GCC diagnostic pop
+#endif
+	#include <stdbool.h>		/* bool */
+#elif defined(__mips)
+	#include <stddef.h>			/* NULL */
+	#include <inttypes.h>		/* intX_t/uintX_t, format specifiers */
+	#include <stdbool.h>		/* bool */
+#else
+	#error C99 support not set up for this build
+#endif
+
+/*
+ * Due to a Klocwork bug, 'true'/'false' constants are not recognized to be of
+ * boolean type. This results in large number of false-positives being reported
+ * (MISRA.ETYPE.ASSIGN.2012: "An expression value of essential type 'signed char'
+ * is assigned to an object of essential type 'bool'"). Work around this by
+ * redefining those constants with cast to bool added.
+ */
+#if defined(__KLOCWORK__) && !defined(__cplusplus)
+#undef true
+#undef false
+#define true ((bool) 1)
+#define false ((bool) 0)
+#endif
+
+typedef unsigned int	IMG_UINT;
+typedef int				IMG_INT;
+
+typedef uint8_t			IMG_UINT8,	*IMG_PUINT8;
+typedef uint8_t			IMG_BYTE,	*IMG_PBYTE;
+typedef int8_t			IMG_INT8;
+typedef char			IMG_CHAR,	*IMG_PCHAR;
+
+typedef uint16_t		IMG_UINT16,	*IMG_PUINT16;
+typedef int16_t			IMG_INT16;
+typedef uint32_t		IMG_UINT32,	*IMG_PUINT32;
+typedef int32_t			IMG_INT32,	*IMG_PINT32;
+#if defined(INTEGRITY_OS)
+#if __INT_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## U))
+#elif __LONG_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## UL))
+#elif defined(__LLONG_BIT) && __LLONG_BIT >= 32U
+#define IMG_UINT32_C(n) ((IMG_UINT32)(n ## ULL))
+#endif
+#else /* defined(INTEGRITY_OS) */
+#define IMG_UINT32_C(c) ((IMG_UINT32)UINT32_C(c))
+#endif /* defined(INTEGRITY_OS) */
+
+typedef uint64_t		IMG_UINT64,	*IMG_PUINT64;
+typedef int64_t			IMG_INT64;
+#define IMG_INT64_C(c)	INT64_C(c)
+#if defined(INTEGRITY_OS)
+#if __INT_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## U)
+#elif defined(__LONG_BIT) && __LONG_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## UL)
+#elif defined(__LLONG_BIT) && __LLONG_BIT >= 64U
+#define IMG_UINT64_C(n)	(n ## ULL)
+#endif
+#else /* defined(INTEGRITY_OS) */
+#define IMG_UINT64_C(c)	UINT64_C(c)
+#endif /* defined(INTEGRITY_OS) */
+#define IMG_UINT16_C(c)	UINT16_C(c)
+#define IMG_UINT64_FMTSPEC PRIu64
+#define IMG_UINT64_FMTSPECX PRIX64
+#define IMG_UINT64_FMTSPECx PRIx64
+#define IMG_UINT64_FMTSPECo PRIo64
+#define IMG_INT64_FMTSPECd PRId64
+
+#define IMG_UINT8_MAX	UINT8_MAX
+#define IMG_UINT16_MAX	UINT16_MAX
+#define IMG_UINT32_MAX	UINT32_MAX
+#define IMG_UINT64_MAX	UINT64_MAX
+
+#define IMG_INT8_MAX	INT8_MAX
+#define IMG_INT16_MAX	INT16_MAX
+#define IMG_INT32_MAX	INT32_MAX
+#define IMG_INT64_MAX	INT64_MAX
+
+/* Linux kernel mode does not use floating point */
+typedef float			IMG_FLOAT,	*IMG_PFLOAT;
+typedef double			IMG_DOUBLE;
+
+typedef union
+{
+	IMG_UINT32 ui32;
+	IMG_FLOAT f;
+} IMG_UINT32_FLOAT;
+
+typedef int				IMG_SECURE_TYPE;
+
+typedef bool      IMG_BOOL;
+typedef bool*     IMG_PBOOL;
+#define IMG_FALSE ((bool) 0)
+#define IMG_TRUE  ((bool) 1)
+
+typedef IMG_CHAR const* IMG_PCCHAR;
+
+/* Format specifiers for 'size_t' type */
+#if defined(_MSC_VER)
+#define IMG_SIZE_FMTSPEC  "%Iu"
+#define IMG_SIZE_FMTSPECX "%Ix"
+#else
+#define IMG_SIZE_FMTSPEC  "%zu"
+#define IMG_SIZE_FMTSPECX "%zx"
+#endif
+
+#if defined(__linux__) && defined(__KERNEL__)
+/* prints the function name when used with printk */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#define IMG_PFN_FMTSPEC "%ps"
+#else
+#define IMG_PFN_FMTSPEC "%pf"
+#endif
+#else
+#define IMG_PFN_FMTSPEC "%p"
+#endif
+
+typedef void           *IMG_HANDLE;
+
+/* Process IDs */
+typedef IMG_UINT32      IMG_PID;
+
+/* OS connection type */
+typedef int             IMG_OS_CONNECTION;
+
+
+/*
+ * Address types.
+ * All types used to refer to a block of memory are wrapped in structures
+ * to enforce some degree of type safety, i.e. a IMG_DEV_VIRTADDR cannot
+ * be assigned to a variable of type IMG_DEV_PHYADDR because they are not the
+ * same thing.
+ *
+ * There is an assumption that the system contains at most one non-cpu mmu,
+ * and a memory block is only mapped by the MMU once.
+ *
+ * Different devices could have offset views of the physical address space.
+ *
+ */
+
+
+/*
+ *
+ * +------------+    +------------+      +------------+        +------------+
+ * |    CPU     |    |    DEV     |      |    DEV     |        |    DEV     |
+ * +------------+    +------------+      +------------+        +------------+
+ *       |                 |                   |                     |
+ *       | void *          |IMG_DEV_VIRTADDR   |IMG_DEV_VIRTADDR     |
+ *       |                 \-------------------/                     |
+ *       |                          |                                |
+ * +------------+             +------------+                         |
+ * |    MMU     |             |    MMU     |                         |
+ * +------------+             +------------+                         |
+ *       |                          |                                |
+ *       |                          |                                |
+ *       |                          |                                |
+ *   +--------+                +---------+                      +--------+
+ *   | Offset |                | (Offset)|                      | Offset |
+ *   +--------+                +---------+                      +--------+
+ *       |                          |                IMG_DEV_PHYADDR |
+ *       |                          |                                |
+ *       |                          | IMG_DEV_PHYADDR                |
+ * +---------------------------------------------------------------------+
+ * |                         System Address bus                          |
+ * +---------------------------------------------------------------------+
+ *
+ */
+
+#define IMG_DEV_VIRTADDR_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_SIZE_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_ALIGN_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+#define IMG_DEVMEM_OFFSET_FMTSPEC "0x%010" IMG_UINT64_FMTSPECX
+
+/* cpu physical address */
+typedef struct
+{
+#if defined(UNDER_WDDM) || defined(WINDOWS_WDF)
+	uintptr_t uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(uintptr_t)(var)
+#define CPUPHYADDR_FMTARG(var)				(IMG_UINT64)(var)
+#define CPUPHYADDR_UINT_FMTSPEC "0x%016" IMG_UINT64_FMTSPECx
+#elif defined(__linux__) && defined(__KERNEL__)
+	phys_addr_t uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(phys_addr_t)(var)
+#define CPUPHYADDR_FMTARG(var)				(&var)
+#define CPUPHYADDR_UINT_FMTSPEC "%pa"
+#else
+	IMG_UINT64 uiAddr;
+#define IMG_CAST_TO_CPUPHYADDR_UINT(var)	(IMG_UINT64)(var)
+#define CPUPHYADDR_FMTARG(var)				(var)
+#define CPUPHYADDR_UINT_FMTSPEC "0x%016" IMG_UINT64_FMTSPECx
+#endif
+} IMG_CPU_PHYADDR;
+
+/* device physical address */
+typedef struct
+{
+	IMG_UINT64 uiAddr;
+} IMG_DEV_PHYADDR;
+
+/* dma address */
+typedef struct
+{
+	IMG_UINT64 uiAddr;
+} IMG_DMA_ADDR;
+
+/*
+	rectangle structure
+*/
+typedef struct
+{
+	IMG_INT32	x0;
+	IMG_INT32	y0;
+	IMG_INT32	x1;
+	IMG_INT32	y1;
+} IMG_RECT;
+
+typedef struct
+{
+	IMG_INT16	x0;
+	IMG_INT16	y0;
+	IMG_INT16	x1;
+	IMG_INT16	y1;
+} IMG_RECT_16;
+
+/*
+ * box structure
+ */
+typedef struct
+{
+	IMG_INT32	x0;
+	IMG_INT32	y0;
+	IMG_INT32	z0;
+	IMG_INT32	x1;
+	IMG_INT32	y1;
+	IMG_INT32	z1;
+} IMG_BOX;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* IMG_TYPES_H */
+/******************************************************************************
+ End of file (img_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/img_types_check.h b/drivers/gpu/drm/img-rogue/img_types_check.h
new file mode 100644
index 000000000000..ce73789877f6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/img_types_check.h
@@ -0,0 +1,58 @@
+/*************************************************************************/ /*!
+@File
+@Title          Global types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Performs size checks on some of the IMG types.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_TYPES_CHECK_H
+#define IMG_TYPES_CHECK_H
+
+#ifndef __KERNEL__
+#include <assert.h>
+#endif /* __KERNEL__ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+static_assert(sizeof(IMG_BOOL) == 1, "invalid size of IMG_BOOL");
+static_assert(sizeof(IMG_INT) == 4, "invalid size of IMG_INT");
+static_assert(sizeof(IMG_UINT) == 4, "invalid size of IMG_UINT");
+static_assert(sizeof(PVRSRV_ERROR) == 4, "invalid size of PVRSRV_ERROR");
+
+#endif /* IMG_TYPES_CHECK_H */
diff --git a/drivers/gpu/drm/img-rogue/info_page.h b/drivers/gpu/drm/img-rogue/info_page.h
new file mode 100644
index 000000000000..5816125d742f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/info_page.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose memory shared between kernel driver and user
+                mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_KM_H
+#define INFO_PAGE_KM_H
+
+#include "pvrsrv_error.h"
+
+#include "pmr.h"
+#include "pvrsrv.h"
+#include "info_page_defs.h"
+
+/**
+ * @Function InfoPageCreate
+ * @Description Allocates resources for global information page.
+ * @Input psData pointer to PVRSRV data
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+PVRSRV_ERROR InfoPageCreate(PVRSRV_DATA *psData);
+
+/**
+ * @Function InfoPageDestroy
+ * @Description Frees all of the resource of global information page.
+ * @Input psData pointer to PVRSRV data
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+void InfoPageDestroy(PVRSRV_DATA *psData);
+
+/**
+ * @Function PVRSRVAcquireInfoPageKM()
+ * @Description This interface is used for obtaining the global information page
+ *              which acts as a general purpose shared memory between KM and UM.
+ *              The use of this information page outside of services is _not_
+ *              recommended.
+ * @Output ppsPMR handle to exported PMR
+ * @Return
+ */
+PVRSRV_ERROR PVRSRVAcquireInfoPageKM(PMR **ppsPMR);
+
+/**
+ * @Function PVRSRVReleaseInfoPageKM()
+ * @Description This function matches PVRSRVAcquireInfoPageKM().
+ * @Input psPMR handle to exported PMR
+ * @Return PVRSRV_OK on success and other PVRSRV_ERROR code on error.
+ */
+PVRSRV_ERROR PVRSRVReleaseInfoPageKM(PMR *psPMR);
+
+/**
+ * @Function GetInfoPageDebugFlagsKM()
+ * @Description Return info page debug flags
+ * @Return info page debug flags
+ */
+static INLINE IMG_UINT32 GetInfoPageDebugFlagsKM(void)
+{
+	return (PVRSRVGetPVRSRVData())->pui32InfoPage[DEBUG_FEATURE_FLAGS];
+}
+
+#endif /* INFO_PAGE_KM_H */
diff --git a/drivers/gpu/drm/img-rogue/info_page_client.h b/drivers/gpu/drm/img-rogue/info_page_client.h
new file mode 100644
index 000000000000..56d394dd99a3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/info_page_client.h
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose shared memory (i.e. information page) mapped by
+                kernel space driver and user space clients. All info page
+                entries are sizeof(IMG_UINT32) on both 32/64-bit environments.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_CLIENT_H
+#define INFO_PAGE_CLIENT_H
+
+#include "device_connection.h"
+#include "info_page_defs.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#endif
+
+/*************************************************************************/ /*!
+@Function      GetInfoPage
+
+@Description   Return Info Page address
+
+@Input         hDevConnection - Services device connection
+
+@Return        Info Page address
+*/
+/*****************************************************************************/
+static INLINE IMG_PUINT32 GetInfoPage(SHARED_DEV_CONNECTION hDevConnection)
+{
+#if defined(__KERNEL__)
+	PVR_UNREFERENCED_PARAMETER(hDevConnection);
+
+	return (PVRSRVGetPVRSRVData())->pui32InfoPage;
+#else
+	return hDevConnection->pui32InfoPage;
+#endif
+}
+
+/*************************************************************************/ /*!
+@Function      GetInfoPageDebugFlags
+
+@Description   Return Info Page debug flags
+
+@Input         hDevConnection - Services device connection
+
+@Return        Info Page debug flags
+*/
+/*****************************************************************************/
+static INLINE IMG_UINT32 GetInfoPageDebugFlags(SHARED_DEV_CONNECTION hDevConnection)
+{
+	return GetInfoPage(hDevConnection)[DEBUG_FEATURE_FLAGS];
+}
+
+#endif /* INFO_PAGE_CLIENT_H */
diff --git a/drivers/gpu/drm/img-rogue/info_page_defs.h b/drivers/gpu/drm/img-rogue/info_page_defs.h
new file mode 100644
index 000000000000..949afcfd869d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/info_page_defs.h
@@ -0,0 +1,133 @@
+/*************************************************************************/ /*!
+@File
+@Title          Kernel/User mode general purpose shared memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose shared memory (i.e. information page) mapped by
+                kernel space driver and user space clients. All information page
+                entries are sizeof(IMG_UINT32) on both 32/64-bit environments.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef INFO_PAGE_DEFS_H
+#define INFO_PAGE_DEFS_H
+
+/* Info page is divided in "blocks" of size INFO_PAGE_CHUNK_SIZE. Each block
+ * should start with the INFO_PAGE_[NAME]_BLOCK_START macro which takes the
+ * value of previous block (except for the first block which starts from 0).
+ *
+ * Each last value of the block (INFO_PAGE_[NAME]_BLOCK_END) should be unused
+ * within that block since it's a first value of the next block. This value
+ * should be a multiple of INFO_PAGE_CHUNK_SIZE.
+ *
+ * Care must be taken to not go over allowed number of elements in each block
+ * which is marked with the INFO_PAGE_[NAME]_BLOCK_END macro.
+ *
+ * Blocks consist of entries that are defined with the INFO_PAGE_ENTRY() macro.
+ * Each entry must define a unique index within the block and as mentioned
+ * can't go over the INFO_PAGE_[NAME]_BLOCK_END limit.
+ *
+ * Always add blocks to the end of the existing list and update
+ * INFO_PAGE_TOTAL_SIZE after.
+ *
+ * See current usage of the Info Page below for examples.
+ */
+
+#define INFO_PAGE_CHUNK_SIZE                        8
+#define INFO_PAGE_BLOCK_END(start,size)             ((start) + (size) * INFO_PAGE_CHUNK_SIZE)
+#define INFO_PAGE_ENTRY(start,index)                ((start) + (index))
+#define INFO_PAGE_SIZE_IN_BYTES(end)                ((end) * sizeof(IMG_UINT32))
+
+#define INFO_PAGE_CACHEOP_BLOCK_START               0
+#define INFO_PAGE_CACHEOP_BLOCK_END                 INFO_PAGE_BLOCK_END(INFO_PAGE_CACHEOP_BLOCK_START, 1)
+#define INFO_PAGE_HWPERF_BLOCK_START                INFO_PAGE_CACHEOP_BLOCK_END
+#define INFO_PAGE_HWPERF_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_HWPERF_BLOCK_START, 1)
+#define INFO_PAGE_TIMEOUT_BLOCK_START               INFO_PAGE_HWPERF_BLOCK_END
+#define INFO_PAGE_TIMEOUT_BLOCK_END                 INFO_PAGE_BLOCK_END(INFO_PAGE_TIMEOUT_BLOCK_START, 2)
+#define INFO_PAGE_BRIDGE_BLOCK_START                INFO_PAGE_TIMEOUT_BLOCK_END
+#define INFO_PAGE_BRIDGE_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_BRIDGE_BLOCK_START, 1)
+#define INFO_PAGE_DEBUG_BLOCK_START                 INFO_PAGE_BRIDGE_BLOCK_END
+#define INFO_PAGE_DEBUG_BLOCK_END                   INFO_PAGE_BLOCK_END(INFO_PAGE_DEBUG_BLOCK_START, 1)
+#define INFO_PAGE_DEVMEM_BLOCK_START                INFO_PAGE_DEBUG_BLOCK_END
+#define INFO_PAGE_DEVMEM_BLOCK_END                  INFO_PAGE_BLOCK_END(INFO_PAGE_DEVMEM_BLOCK_START, 1)
+
+/* IMPORTANT: Make sure this always uses the last INFO_PAGE_[NAME]_BLOCK_END definition.*/
+#define INFO_PAGE_TOTAL_SIZE                        INFO_PAGE_SIZE_IN_BYTES(INFO_PAGE_DEVMEM_BLOCK_END)
+
+/* CacheOp information page entries */
+
+#define CACHEOP_INFO_UMKMTHRESHLD                   INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 0) /*!< UM=>KM routing threshold in bytes */
+#define CACHEOP_INFO_KMDFTHRESHLD                   INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 1) /*!< KM/DF threshold in bytes */
+#define CACHEOP_INFO_LINESIZE                       INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 2) /*!< CPU data cache line size */
+#define CACHEOP_INFO_PGSIZE                         INFO_PAGE_ENTRY(INFO_PAGE_CACHEOP_BLOCK_START, 3) /*!< CPU MMU page size */
+
+/* HWPerf information page entries */
+
+#define HWPERF_FILTER_SERVICES_IDX                  INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 0)
+#define HWPERF_FILTER_EGL_IDX                       INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 1)
+#define HWPERF_FILTER_OPENGLES_IDX                  INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 2)
+#define HWPERF_FILTER_OPENCL_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 3)
+#define HWPERF_FILTER_VULKAN_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 4)
+#define HWPERF_FILTER_OPENGL_IDX                    INFO_PAGE_ENTRY(INFO_PAGE_HWPERF_BLOCK_START, 5)
+
+/* Timeout values */
+
+#define TIMEOUT_INFO_VALUE_RETRIES                  INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 0)
+#define TIMEOUT_INFO_VALUE_TIMEOUT_MS               INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 1)
+#define TIMEOUT_INFO_CONDITION_RETRIES              INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 2)
+#define TIMEOUT_INFO_CONDITION_TIMEOUT_MS           INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 3)
+#define TIMEOUT_INFO_TASK_QUEUE_RETRIES             INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 4)
+#define TIMEOUT_INFO_TASK_QUEUE_FLUSH_TIMEOUT_MS    INFO_PAGE_ENTRY(INFO_PAGE_TIMEOUT_BLOCK_START, 5)
+
+/* Bridge Info */
+
+#define BRIDGE_INFO_RGX_BRIDGES                     INFO_PAGE_ENTRY(INFO_PAGE_BRIDGE_BLOCK_START, 0)
+#define BRIDGE_INFO_PVR_BRIDGES                     INFO_PAGE_ENTRY(INFO_PAGE_BRIDGE_BLOCK_START, 1)
+
+/* Debug features */
+
+#define DEBUG_FEATURE_FLAGS                         INFO_PAGE_ENTRY(INFO_PAGE_DEBUG_BLOCK_START, 0)
+
+#define DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED    0x1 /* flag - not part of info page */
+#define DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED      0x2 /* flag - not part of info page */
+
+/* Device memory related information */
+
+/* This value is 64-bits wide, next value should have index larger by 2 */
+#define DEVMEM_INFO_PHYS_BUF_MAX_SIZE               INFO_PAGE_ENTRY(INFO_PAGE_DEVMEM_BLOCK_START, 0)
+
+#endif /* INFO_PAGE_DEFS_H */
diff --git a/drivers/gpu/drm/img-rogue/info_page_km.c b/drivers/gpu/drm/img-rogue/info_page_km.c
new file mode 100644
index 000000000000..79b03fe7e658
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/info_page_km.c
@@ -0,0 +1,142 @@
+/*************************************************************************/ /*!
+@File           info_page_km.c
+@Title          Kernel/User space shared memory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements general purpose shared memory between kernel driver
+                and user mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "info_page_defs.h"
+#include "info_page.h"
+#include "pvrsrv.h"
+#include "devicemem.h"
+#include "pmr.h"
+
+PVRSRV_ERROR InfoPageCreate(PVRSRV_DATA *psData)
+{
+    const PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+                                              PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+                                              PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+                                              PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL);
+    PVRSRV_ERROR eError;
+
+    PVR_ASSERT(psData != NULL);
+
+    /* Allocate single page of memory for driver information page */
+    eError = DevmemAllocateExportable(psData->psHostMemDeviceNode,
+                                      PVR_ALIGN(INFO_PAGE_TOTAL_SIZE, OSGetPageSize()),
+                                      OSGetPageSize(),
+                                      OSGetPageShift(),
+                                      uiMemFlags,
+                                      "PVRSRVInfoPage",
+                                      &psData->psInfoPageMemDesc);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+    eError =  DevmemAcquireCpuVirtAddr(psData->psInfoPageMemDesc,
+                                       (void **) &psData->pui32InfoPage);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+    /* Look-up the memory descriptor PMR handle */
+    eError = DevmemLocalGetImportHandle(psData->psInfoPageMemDesc,
+                                        (void **) &psData->psInfoPagePMR);
+    PVR_LOG_GOTO_IF_ERROR(eError, "DevmemLocalGetImportHandle", e0);
+
+    eError = OSLockCreate(&psData->hInfoPageLock);
+    PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+    /* Because the memory is allocated read only we need to explicitly set it to
+     * 0. The reason for this is that if the memory is allocated with
+     * ZERO_ON_ALLOC a WRITEABLE attribute is implicitly added to the flags (see
+     * DevmemValidateParams()). */
+    OSCachedMemSet(psData->pui32InfoPage, 0, OSGetPageSize());
+
+    return PVRSRV_OK;
+
+e0:
+    InfoPageDestroy(psData);
+    return eError;
+}
+
+void InfoPageDestroy(PVRSRV_DATA *psData)
+{
+    if (psData->psInfoPageMemDesc)
+    {
+        if (psData->pui32InfoPage != NULL)
+        {
+            DevmemReleaseCpuVirtAddr(psData->psInfoPageMemDesc);
+            psData->pui32InfoPage = NULL;
+        }
+
+        DevmemFree(psData->psInfoPageMemDesc);
+        psData->psInfoPageMemDesc = NULL;
+    }
+
+    if (psData->hInfoPageLock)
+    {
+        OSLockDestroy(psData->hInfoPageLock);
+        psData->hInfoPageLock = NULL;
+    }
+}
+
+PVRSRV_ERROR PVRSRVAcquireInfoPageKM(PMR **ppsPMR)
+{
+    PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+
+    PVR_LOG_RETURN_IF_FALSE(psData->psInfoPageMemDesc != NULL, "invalid MEMDESC"
+                      " handle", PVRSRV_ERROR_INVALID_PARAMS);
+    PVR_LOG_RETURN_IF_FALSE(psData->psInfoPagePMR != NULL, "invalid PMR handle",
+                      PVRSRV_ERROR_INVALID_PARAMS);
+
+    /* Copy the PMR import handle back */
+    *ppsPMR = psData->psInfoPagePMR;
+
+    /* Mark the PMR such that no layout changes can happen
+     * This is a fixed layout created during early stages of
+     * driver loading and shouldn't change later */
+    PMR_SetLayoutFixed(psData->psInfoPagePMR, IMG_TRUE);
+
+    return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVReleaseInfoPageKM(PMR *ppsPMR)
+{
+    /* Nothing to do here as PMR is singleton */
+    PVR_UNREFERENCED_PARAMETER(ppsPMR);
+    return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img-rogue/interrupt_support.c b/drivers/gpu/drm/img-rogue/interrupt_support.c
new file mode 100644
index 000000000000..c67d45352050
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/interrupt_support.c
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/interrupt.h>
+
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "interrupt_support.h"
+
+typedef struct LISR_DATA_TAG
+{
+	IMG_UINT32	ui32IRQ;
+	PFN_SYS_LISR	pfnLISR;
+	void		*pvData;
+} LISR_DATA;
+
+static irqreturn_t SystemISRWrapper(int irq, void *dev_id)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *)dev_id;
+
+	PVR_UNREFERENCED_PARAMETER(irq);
+
+	if (psLISRData)
+	{
+		if (psLISRData->pfnLISR(psLISRData->pvData))
+		{
+			return IRQ_HANDLED;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Missing interrupt data", __func__));
+	}
+
+	return IRQ_NONE;
+}
+
+PVRSRV_ERROR OSInstallSystemLISR(IMG_HANDLE *phLISR,
+				 IMG_UINT32 ui32IRQ,
+				 const IMG_CHAR *pszDevName,
+				 PFN_SYS_LISR pfnLISR,
+				 void *pvData,
+				 IMG_UINT32 ui32Flags)
+{
+	LISR_DATA *psLISRData;
+	unsigned long ulIRQFlags = 0;
+
+	if (pfnLISR == NULL || pvData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32Flags & ~SYS_IRQ_FLAG_MASK)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	switch (ui32Flags & SYS_IRQ_FLAG_TRIGGER_MASK)
+	{
+		case SYS_IRQ_FLAG_TRIGGER_DEFAULT:
+			break;
+		case SYS_IRQ_FLAG_TRIGGER_LOW:
+			ulIRQFlags |= IRQF_TRIGGER_LOW;
+			break;
+		case SYS_IRQ_FLAG_TRIGGER_HIGH:
+			ulIRQFlags |= IRQF_TRIGGER_HIGH;
+			break;
+		default:
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32Flags & SYS_IRQ_FLAG_SHARED)
+	{
+		ulIRQFlags |= IRQF_SHARED;
+	}
+
+	psLISRData = OSAllocMem(sizeof(*psLISRData));
+	if (psLISRData == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psLISRData->ui32IRQ = ui32IRQ;
+	psLISRData->pfnLISR = pfnLISR;
+	psLISRData->pvData = pvData;
+
+	if (request_irq(ui32IRQ, SystemISRWrapper, ulIRQFlags, pszDevName, psLISRData))
+	{
+		OSFreeMem(psLISRData);
+
+		return PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER;
+	}
+
+	*phLISR = (IMG_HANDLE)psLISRData;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSUninstallSystemLISR(IMG_HANDLE hLISR)
+{
+	LISR_DATA *psLISRData = (LISR_DATA *)hLISR;
+
+	if (psLISRData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	free_irq(psLISRData->ui32IRQ, psLISRData);
+
+	OSFreeMem(psLISRData);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img-rogue/interrupt_support.h b/drivers/gpu/drm/img-rogue/interrupt_support.h
new file mode 100644
index 000000000000..b87772d223cb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/interrupt_support.h
@@ -0,0 +1,103 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(INTERRUPT_SUPPORT_H)
+#define INTERRUPT_SUPPORT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_device.h"
+
+/*! Default trigger type for the interrupt line. */
+#define SYS_IRQ_FLAG_TRIGGER_DEFAULT (0x0 << 0)
+/*! Interrupt triggered when interrupt line is low. */
+#define SYS_IRQ_FLAG_TRIGGER_LOW     (0x1 << 0)
+/*! Interrupt triggered when interrupt line is high. */
+#define SYS_IRQ_FLAG_TRIGGER_HIGH    (0x2 << 0)
+/*! Interrupt trigger mask. */
+#define SYS_IRQ_FLAG_TRIGGER_MASK    (SYS_IRQ_FLAG_TRIGGER_DEFAULT | \
+                                      SYS_IRQ_FLAG_TRIGGER_LOW | \
+                                      SYS_IRQ_FLAG_TRIGGER_HIGH)
+/*! The irq is allowed to be shared among several devices. */
+#define SYS_IRQ_FLAG_SHARED          (0x1 << 8)
+
+/*! Interrupt flags mask. */
+#define SYS_IRQ_FLAG_MASK            (SYS_IRQ_FLAG_TRIGGER_MASK | \
+                                      SYS_IRQ_FLAG_SHARED)
+
+/*************************************************************************/ /*!
+@Description    Pointer to a system Low-level Interrupt Service Routine (LISR).
+@Input  pvData  Private data provided to the LISR.
+@Return         IMG_TRUE if interrupt handled, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_SYS_LISR)(void *pvData);
+
+/*************************************************************************/ /*!
+@Function       OSInstallSystemLISR
+@Description    Installs a system low-level interrupt handler
+@Output         phLISR                  On return, contains a handle to the
+                                        installed LISR
+@Input          ui32IRQ                 The IRQ number for which the
+                                        interrupt handler should be installed
+@Input          pszDevName              Name of the device for which the handler
+                                        is being installed
+@Input          pfnLISR                 A pointer to an interrupt handler
+                                        function
+@Input          pvData                  A pointer to data that should be passed
+                                        to pfnLISR when it is called
+@Input          ui32Flags               Interrupt flags
+@Return         PVRSRV_OK on success, a failure code otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInstallSystemLISR(IMG_HANDLE *phLISR,
+				 IMG_UINT32 ui32IRQ,
+				 const IMG_CHAR *pszDevName,
+				 PFN_SYS_LISR pfnLISR,
+				 void *pvData,
+				 IMG_UINT32 ui32Flags);
+
+/*************************************************************************/ /*!
+@Function       OSUninstallSystemLISR
+@Description    Uninstalls a system low-level interrupt handler
+@Input          hLISRData              The handle to the LISR to uninstall
+@Return         PVRSRV_OK on success, a failure code otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR OSUninstallSystemLISR(IMG_HANDLE hLISRData);
+#endif /* !defined(INTERRUPT_SUPPORT_H) */
diff --git a/drivers/gpu/drm/img-rogue/kernel_compatibility.h b/drivers/gpu/drm/img-rogue/kernel_compatibility.h
new file mode 100644
index 000000000000..048f4449ae68
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/kernel_compatibility.h
@@ -0,0 +1,404 @@
+/*************************************************************************/ /*!
+@Title          Kernel versions compatibility macros
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_COMPATIBILITY_H__
+#define __KERNEL_COMPATIBILITY_H__
+
+#include <linux/version.h>
+#include <linux/compiler.h>
+
+/* Explicitly error out if DDK is built against out-of-support Linux kernel */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0))
+#error Linux kernels older than 4.9.0 are not supported
+#endif
+
+/*
+ * Stop supporting an old kernel? Remove the top block.
+ * New incompatible kernel?       Append a new block at the bottom.
+ *
+ * Please write your version test as `VERSION < X.Y`, and use the earliest
+ * possible version :)
+ *
+ * If including this header file in other files, this should always be the
+ * last file included, as it can affect definitions/declarations in files
+ * included after it.
+ */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+#define refcount_read(r) atomic_read(r)
+#define drm_mm_insert_node(mm, node, size) drm_mm_insert_node(mm, node, size, 0, DRM_MM_SEARCH_DEFAULT)
+
+#define drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd) drm_helper_mode_fill_fb_struct(fb, mode_cmd)
+
+/*
+ * In Linux Kernels >= 4.12 for x86 another level of page tables has been
+ * added. The added level (p4d) sits between pgd and pud, so when it
+ * doesn`t exist, pud_offset function takes pgd as a parameter instead
+ * of p4d.
+ */
+#define p4d_t pgd_t
+#define p4d_offset(pgd, address) (pgd)
+#define p4d_none(p4d) (0)
+#define p4d_bad(p4d) (0)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+
+#define drm_mode_object_get(obj)          drm_mode_object_reference(obj)
+#define drm_mode_object_put(obj)          drm_mode_object_unreference(obj)
+#define drm_connector_get(obj)            drm_connector_reference(obj)
+#define drm_connector_put(obj)            drm_connector_unreference(obj)
+#define drm_framebuffer_get(obj)          drm_framebuffer_reference(obj)
+#define drm_framebuffer_put(obj)          drm_framebuffer_unreference(obj)
+#define drm_gem_object_get(obj)           drm_gem_object_reference(obj)
+#define drm_gem_object_put_locked(obj)    drm_gem_object_unreference(obj)
+#define __drm_gem_object_put(obj)         __drm_gem_object_unreference(obj)
+#define drm_property_blob_get(obj)        drm_property_reference_blob(obj)
+#define drm_property_blob_put(obj)        drm_property_unreference_blob(obj)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+
+#define drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, format_modifiers, type, name, ...) \
+	({ (void) format_modifiers; drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, type, name, ##__VA_ARGS__); })
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0))
+
+#define drm_dev_put(dev) drm_dev_unref(dev)
+
+#define drm_mode_object_find(dev, file_priv, id, type) drm_mode_object_find(dev, id, type)
+#define drm_encoder_find(dev, file_priv, id) drm_encoder_find(dev, id)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_plane_helper_check_state(plane_state, \
+												 &__clip, \
+												 min_scale, max_scale, \
+												 can_position, \
+												 can_update_disabled); \
+		__ret; \
+	})
+
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_atomic_helper_check_plane_state(plane_state, \
+														crtc_state, \
+														&__clip, \
+														min_scale, max_scale, \
+														can_position, \
+														can_update_disabled); \
+		__ret; \
+	})
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+
+#define drm_connector_attach_encoder(connector, encoder) \
+	drm_mode_connector_attach_encoder(connector, encoder)
+
+#define drm_connector_update_edid_property(connector, edid) \
+	drm_mode_connector_update_edid_property(connector, edid)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+
+/*
+ * Work around architectures, e.g. MIPS, that define copy_from_user and
+ * copy_to_user as macros that call access_ok, as this gets redefined below.
+ * As of kernel 4.12, these functions are no longer defined per-architecture
+ * so this work around isn't needed.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#if defined(copy_from_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_from_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+	return copy_from_user(to, from, n);
+}
+
+#undef copy_from_user
+#define copy_from_user(to, from, n) __copy_from_user(to, from, n)
+#endif
+
+#if defined(copy_to_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_to_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+	return copy_to_user(to, from, n);
+}
+
+#undef copy_to_user
+#define copy_to_user(to, from, n) __copy_to_user(to, from, n)
+#endif
+#endif
+
+/*
+ * Linux 5.0 dropped the type argument.
+ *
+ * This is unused in at least Linux 3.4 and above for all architectures other
+ * than 'um' (User Mode Linux), which stopped using it in 4.2.
+ */
+#if defined(access_ok)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around access_ok being defined as a macro.
+  */
+static inline int
+__pvr_access_ok_compat(int type, const void __user * addr, unsigned long size)
+{
+	return access_ok(type, addr, size);
+}
+
+#undef access_ok
+#define access_ok(addr, size) __pvr_access_ok_compat(0, addr, size)
+#else
+#define access_ok(addr, size) access_ok(0, addr, size)
+#endif
+
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+#define MODULE_IMPORT_NS(ns)
+#endif
+
+/*
+ * Before v5.8, the "struct mm" has a semaphore named "mmap_sem" which is
+ * renamed to "mmap_lock" in v5.8. Moreover, new APIs are provided to
+ * access this lock starting from v5.8.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+
+#define mmap_write_lock(mm)   down_write(&mm->mmap_sem)
+#define mmap_write_unlock(mm) up_write(&mm->mmap_sem)
+
+#define mmap_read_lock(mm)    down_read(&mm->mmap_sem)
+#define mmap_read_unlock(mm)  up_read(&mm->mmap_sem)
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#define drm_gem_object_put(obj) drm_gem_object_unreference_unlocked(obj)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+#define drm_gem_object_put(obj) drm_gem_object_put_unlocked(obj)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+
+#define drm_prime_pages_to_sg(dev, pages, nr_pages) \
+	drm_prime_pages_to_sg(pages, nr_pages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+
+struct dma_buf_map {
+	void *vaddr;
+};
+
+#define dma_buf_vmap(dmabuf, map) \
+	({ \
+		(map)->vaddr = dma_buf_vmap(dmabuf); \
+		(map)->vaddr ? 0 : ((dmabuf) && (dmabuf)->ops->vmap) ? -ENOMEM : -EINVAL; \
+	})
+
+#define dma_buf_vunmap(dmabuf, map) \
+	({ \
+		dma_buf_vunmap(dmabuf, (map)->vaddr); \
+		(map)->vaddr = NULL; \
+	})
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0))
+
+#define drm_prime_sg_to_page_array(sgt, pages, npages) \
+	drm_prime_sg_to_page_addr_arrays(sgt, pages, NULL, npages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0))
+
+#define drm_gem_plane_helper_prepare_fb drm_gem_fb_prepare_fb
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0)) */
+
+/*
+ * Linux 5.11 renames the privileged uaccess routines for arm64 and Android
+ * kernel v5.10 merges the change as well. These routines are only used for
+ * arm64 so CONFIG_ARM64 testing can be ignored.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) || \
+	((LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) && !defined(ANDROID))
+#define uaccess_enable_privileged() uaccess_enable()
+#define uaccess_disable_privileged() uaccess_disable()
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0))
+#define pde_data PDE_DATA
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+#define kthread_complete_and_exit(comp, ret) complete_and_exit(comp, ret);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0))
+#define iosys_map dma_buf_map
+#define iosys_map_set_vaddr dma_buf_map_set_vaddr
+#define iosys_map_set_vaddr_iomem dma_buf_map_set_vaddr_iomem
+#define iosys_map_clear dma_buf_map_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+
+#define register_shrinker(shrinker, name) \
+	register_shrinker(shrinker)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0))
+#define DRM_PLANE_NO_SCALING DRM_PLANE_HELPER_NO_SCALING
+#define drm_plane_helper_destroy drm_primary_helper_destroy
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0))
+#define genl_split_ops genl_ops
+#define COMPAT_FB_INFO fbdev
+#define drm_fb_helper_alloc_info drm_fb_helper_alloc_fbi
+#define drm_fb_helper_unregister_info drm_fb_helper_unregister_fbi
+#else
+#define COMPAT_FB_INFO info
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) || \
+        ((LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) && !defined(ANDROID))
+static inline void pvr_vm_flags_set(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags |= flags;
+}
+static inline void pvr_vm_flags_init(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags = flags;
+}
+static inline void pvr_vm_flags_clear(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags &= ~flags;
+}
+#else
+#define pvr_vm_flags_set  vm_flags_set
+#define pvr_vm_flags_init vm_flags_init
+#define pvr_vm_flags_clear vm_flags_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0))
+#define pvr_class_create(name) class_create(THIS_MODULE, name)
+#else
+#define pvr_class_create(name) class_create(name)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+#define thermal_tripless_zone_device_register(type, devdata, ops, tzp) \
+	thermal_zone_device_register((type), 0, 0, (devdata), (ops), (tzp), 0, 0)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0)) */
+
+#if defined(__GNUC__)
+#define GCC_VERSION_AT_LEAST(major, minor) \
+	(__GNUC__ > (major) || \
+	(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#define GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if defined(__clang__)
+#define CLANG_VERSION_AT_LEAST(major) \
+	(__clang_major__ >= (major))
+#else
+#define CLANG_VERSION_AT_LEAST(major) 0
+#endif
+
+#if !defined(__fallthrough)
+	#if GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+		#define __fallthrough __attribute__((__fallthrough__))
+	#else
+		#define __fallthrough
+	#endif
+#endif
+
+#endif /* __KERNEL_COMPATIBILITY_H__ */
diff --git a/drivers/gpu/drm/img-rogue/kernel_config_compatibility.h b/drivers/gpu/drm/img-rogue/kernel_config_compatibility.h
new file mode 100644
index 000000000000..63effd65ecf9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/kernel_config_compatibility.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@Title          Kernel config compatibility define options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file is exclusively for Linux config kernel options.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_CONFIG_COMPATIBILITY_H__
+#define __KERNEL_CONFIG_COMPATIBILITY_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0))
+#ifdef SUPPORT_DRM_FBDEV_EMULATION
+#define CONFIG_DRM_FBDEV_EMULATION
+#endif
+#endif
+
+#endif /* __KERNEL_CONFIG_COMPATIBILITY_H__ */
diff --git a/drivers/gpu/drm/img-rogue/kernel_nospec.h b/drivers/gpu/drm/img-rogue/kernel_nospec.h
new file mode 100644
index 000000000000..ef6fee368327
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/kernel_nospec.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@Title          Macro to limit CPU speculative execution in kernel code
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_NOSPEC_H__
+#define __KERNEL_NOSPEC_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 2) ||			\
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) &&		\
+	 LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 18)) ||		\
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) &&		\
+	 LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 81)))
+#include <linux/kernel.h>
+#include <linux/bug.h>
+#include <linux/nospec.h>
+#else
+#define array_index_nospec(index, size) (index)
+#endif
+
+/*
+ * For Ubuntu kernels, the features available for a given Linux version code
+ * may not match those in upstream kernels. This is the case for the
+ * availability of the array_index_nospec macro.
+ */
+#if !defined(array_index_nospec)
+#define array_index_nospec(index, size) (index)
+#endif
+
+#endif /* __KERNEL_NOSPEC_H__ */
diff --git a/drivers/gpu/drm/img-rogue/kernel_types.h b/drivers/gpu/drm/img-rogue/kernel_types.h
new file mode 100644
index 000000000000..c3305102fc20
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/kernel_types.h
@@ -0,0 +1,137 @@
+/*************************************************************************/ /*!
+@Title          C99-compatible types and definitions for Linux kernel code
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/kernel.h>
+
+/* Limits of specified-width integer types */
+
+/* S8_MIN, etc were added in kernel version 3.14. The other versions are for
+ * earlier kernels. They can be removed once older kernels don't need to be
+ * supported.
+ */
+#ifdef S8_MIN
+	#define INT8_MIN	S8_MIN
+#else
+	#define INT8_MIN	(-128)
+#endif
+
+#ifdef S8_MAX
+	#define INT8_MAX	S8_MAX
+#else
+	#define INT8_MAX	127
+#endif
+
+#ifdef U8_MAX
+	#define UINT8_MAX	U8_MAX
+#else
+	#define UINT8_MAX	0xFF
+#endif
+
+#ifdef S16_MIN
+	#define INT16_MIN	S16_MIN
+#else
+	#define INT16_MIN	(-32768)
+#endif
+
+#ifdef S16_MAX
+	#define INT16_MAX	S16_MAX
+#else
+	#define INT16_MAX	32767
+#endif
+
+#ifdef U16_MAX
+	#define UINT16_MAX	U16_MAX
+#else
+	#define UINT16_MAX	0xFFFF
+#endif
+
+#ifdef S32_MIN
+	#define INT32_MIN	S32_MIN
+#else
+	#define INT32_MIN	(-2147483647 - 1)
+#endif
+
+#ifdef S32_MAX
+	#define INT32_MAX	S32_MAX
+#else
+	#define INT32_MAX	2147483647
+#endif
+
+#ifdef U32_MAX
+	#define UINT32_MAX	U32_MAX
+#else
+	#define UINT32_MAX	0xFFFFFFFF
+#endif
+
+#ifdef S64_MIN
+	#define INT64_MIN	S64_MIN
+#else
+	#define INT64_MIN	(-9223372036854775807LL)
+#endif
+
+#ifdef S64_MAX
+	#define INT64_MAX	S64_MAX
+#else
+	#define INT64_MAX	9223372036854775807LL
+#endif
+
+#ifdef U64_MAX
+	#define UINT64_MAX	U64_MAX
+#else
+	#define UINT64_MAX	0xFFFFFFFFFFFFFFFFULL
+#endif
+
+/* Macros for integer constants */
+#define INT8_C			S8_C
+#define UINT8_C			U8_C
+#define INT16_C			S16_C
+#define UINT16_C		U16_C
+#define INT32_C			S32_C
+#define UINT32_C		U32_C
+#define INT64_C			S64_C
+#define UINT64_C		U64_C
+
+/* Format conversion of integer types <inttypes.h> */
+
+#define PRIX64		"llX"
+#define PRIx64		"llx"
+#define PRIu64		"llu"
+#define PRId64		"lld"
diff --git a/drivers/gpu/drm/img-rogue/km/rgx_bvnc_defs_km.h b/drivers/gpu/drm/img-rogue/km/rgx_bvnc_defs_km.h
new file mode 100644
index 000000000000..73c44b77f498
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km/rgx_bvnc_defs_km.h
@@ -0,0 +1,341 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_defs_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_DEFS_KM_H
+#define RGX_BVNC_DEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#if defined(RGX_BVNC_DEFS_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_defs_um.h"
+#endif
+
+#define BVNC_FIELD_WIDTH (16U)
+
+#define PVR_ARCH_NAME "rogue"
+
+
+/******************************************************************************
+ * Mask and bit-position macros for features without values
+ *****************************************************************************/
+
+#define RGX_FEATURE_AXI_ACELITE_POS                                     (0U)
+#define RGX_FEATURE_AXI_ACELITE_BIT_MASK                                (IMG_UINT64_C(0x0000000000000001))
+
+#define RGX_FEATURE_CLUSTER_GROUPING_POS                                (1U)
+#define RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK                           (IMG_UINT64_C(0x0000000000000002))
+
+#define RGX_FEATURE_COMPUTE_POS                                         (2U)
+#define RGX_FEATURE_COMPUTE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000000004))
+
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_POS                          (3U)
+#define RGX_FEATURE_COMPUTE_MORTON_CAPABLE_BIT_MASK                     (IMG_UINT64_C(0x0000000000000008))
+
+#define RGX_FEATURE_COMPUTE_OVERLAP_POS                                 (4U)
+#define RGX_FEATURE_COMPUTE_OVERLAP_BIT_MASK                            (IMG_UINT64_C(0x0000000000000010))
+
+#define RGX_FEATURE_COREID_PER_OS_POS                                   (5U)
+#define RGX_FEATURE_COREID_PER_OS_BIT_MASK                              (IMG_UINT64_C(0x0000000000000020))
+
+#define RGX_FEATURE_DYNAMIC_DUST_POWER_POS                              (6U)
+#define RGX_FEATURE_DYNAMIC_DUST_POWER_BIT_MASK                         (IMG_UINT64_C(0x0000000000000040))
+
+#define RGX_FEATURE_FASTRENDER_DM_POS                                   (7U)
+#define RGX_FEATURE_FASTRENDER_DM_BIT_MASK                              (IMG_UINT64_C(0x0000000000000080))
+
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_POS                           (8U)
+#define RGX_FEATURE_GPU_MULTICORE_SUPPORT_BIT_MASK                      (IMG_UINT64_C(0x0000000000000100))
+
+#define RGX_FEATURE_GPU_VIRTUALISATION_POS                              (9U)
+#define RGX_FEATURE_GPU_VIRTUALISATION_BIT_MASK                         (IMG_UINT64_C(0x0000000000000200))
+
+#define RGX_FEATURE_GS_RTA_SUPPORT_POS                                  (10U)
+#define RGX_FEATURE_GS_RTA_SUPPORT_BIT_MASK                             (IMG_UINT64_C(0x0000000000000400))
+
+#define RGX_FEATURE_IRQ_PER_OS_POS                                      (11U)
+#define RGX_FEATURE_IRQ_PER_OS_BIT_MASK                                 (IMG_UINT64_C(0x0000000000000800))
+
+#define RGX_FEATURE_MIPS_POS                                            (12U)
+#define RGX_FEATURE_MIPS_BIT_MASK                                       (IMG_UINT64_C(0x0000000000001000))
+
+#define RGX_FEATURE_PBE2_IN_XE_POS                                      (13U)
+#define RGX_FEATURE_PBE2_IN_XE_BIT_MASK                                 (IMG_UINT64_C(0x0000000000002000))
+
+#define RGX_FEATURE_PBE_CHECKSUM_2D_POS                                 (14U)
+#define RGX_FEATURE_PBE_CHECKSUM_2D_BIT_MASK                            (IMG_UINT64_C(0x0000000000004000))
+
+#define RGX_FEATURE_PBVNC_COREID_REG_POS                                (15U)
+#define RGX_FEATURE_PBVNC_COREID_REG_BIT_MASK                           (IMG_UINT64_C(0x0000000000008000))
+
+#define RGX_FEATURE_PERFBUS_POS                                         (16U)
+#define RGX_FEATURE_PERFBUS_BIT_MASK                                    (IMG_UINT64_C(0x0000000000010000))
+
+#define RGX_FEATURE_PERF_COUNTER_BATCH_POS                              (17U)
+#define RGX_FEATURE_PERF_COUNTER_BATCH_BIT_MASK                         (IMG_UINT64_C(0x0000000000020000))
+
+#define RGX_FEATURE_PM_MMU_VFP_POS                                      (18U)
+#define RGX_FEATURE_PM_MMU_VFP_BIT_MASK                                 (IMG_UINT64_C(0x0000000000040000))
+
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_POS                              (19U)
+#define RGX_FEATURE_RISCV_FW_PROCESSOR_BIT_MASK                         (IMG_UINT64_C(0x0000000000080000))
+
+#define RGX_FEATURE_ROGUEXE_POS                                         (20U)
+#define RGX_FEATURE_ROGUEXE_BIT_MASK                                    (IMG_UINT64_C(0x0000000000100000))
+
+#define RGX_FEATURE_SAFETY_IRQ_POS                                      (21U)
+#define RGX_FEATURE_SAFETY_IRQ_BIT_MASK                                 (IMG_UINT64_C(0x0000000000200000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_POS                (22U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_BIT_MASK           (IMG_UINT64_C(0x0000000000400000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1_POS             (23U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V1_BIT_MASK        (IMG_UINT64_C(0x0000000000800000))
+
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2_POS             (24U)
+#define RGX_FEATURE_SIMPLE_INTERNAL_PARAMETER_FORMAT_V2_BIT_MASK        (IMG_UINT64_C(0x0000000001000000))
+
+#define RGX_FEATURE_SINGLE_BIF_POS                                      (25U)
+#define RGX_FEATURE_SINGLE_BIF_BIT_MASK                                 (IMG_UINT64_C(0x0000000002000000))
+
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE_POS                           (26U)
+#define RGX_FEATURE_SLC_SIZE_CONFIGURABLE_BIT_MASK                      (IMG_UINT64_C(0x0000000004000000))
+
+#define RGX_FEATURE_SOC_TIMER_POS                                       (27U)
+#define RGX_FEATURE_SOC_TIMER_BIT_MASK                                  (IMG_UINT64_C(0x0000000008000000))
+
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_POS                            (28U)
+#define RGX_FEATURE_SYS_BUS_SECURE_RESET_BIT_MASK                       (IMG_UINT64_C(0x0000000010000000))
+
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_POS                                (29U)
+#define RGX_FEATURE_TDM_PDS_CHECKSUM_BIT_MASK                           (IMG_UINT64_C(0x0000000020000000))
+
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION_POS                          (30U)
+#define RGX_FEATURE_TFBC_DELTA_CORRELATION_BIT_MASK                     (IMG_UINT64_C(0x0000000040000000))
+
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT_POS                           (31U)
+#define RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK                      (IMG_UINT64_C(0x0000000080000000))
+
+#define RGX_FEATURE_TFBC_NATIVE_YUV10_POS                               (32U)
+#define RGX_FEATURE_TFBC_NATIVE_YUV10_BIT_MASK                          (IMG_UINT64_C(0x0000000100000000))
+
+#define RGX_FEATURE_TILE_REGION_PROTECTION_POS                          (33U)
+#define RGX_FEATURE_TILE_REGION_PROTECTION_BIT_MASK                     (IMG_UINT64_C(0x0000000200000000))
+
+#define RGX_FEATURE_TLA_POS                                             (34U)
+#define RGX_FEATURE_TLA_BIT_MASK                                        (IMG_UINT64_C(0x0000000400000000))
+
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_POS             (35U)
+#define RGX_FEATURE_TPU_CEM_DATAMASTER_GLOBAL_REGISTERS_BIT_MASK        (IMG_UINT64_C(0x0000000800000000))
+
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_POS                         (36U)
+#define RGX_FEATURE_TPU_DM_GLOBAL_REGISTERS_BIT_MASK                    (IMG_UINT64_C(0x0000001000000000))
+
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_POS                      (37U)
+#define RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK                 (IMG_UINT64_C(0x0000002000000000))
+
+#define RGX_FEATURE_VOLCANIC_TB_POS                                     (38U)
+#define RGX_FEATURE_VOLCANIC_TB_BIT_MASK                                (IMG_UINT64_C(0x0000004000000000))
+
+#define RGX_FEATURE_WATCHDOG_TIMER_POS                                  (39U)
+#define RGX_FEATURE_WATCHDOG_TIMER_BIT_MASK                             (IMG_UINT64_C(0x0000008000000000))
+
+#define RGX_FEATURE_WORKGROUP_PROTECTION_POS                            (40U)
+#define RGX_FEATURE_WORKGROUP_PROTECTION_BIT_MASK                       (IMG_UINT64_C(0x0000010000000000))
+
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY_POS                             (41U)
+#define RGX_FEATURE_XE_MEMORY_HIERARCHY_BIT_MASK                        (IMG_UINT64_C(0x0000020000000000))
+
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE_POS                           (42U)
+#define RGX_FEATURE_XT_TOP_INFRASTRUCTURE_BIT_MASK                      (IMG_UINT64_C(0x0000040000000000))
+
+
+/******************************************************************************
+ * Defines for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+#define RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX             (2U)
+#define RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX                              (3U)
+#define RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX                  (2U)
+#define RGX_FEATURE_FBCDC_MAX_VALUE_IDX                                 (4U)
+#define RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX                       (6U)
+#define RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX                    (4U)
+#define RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX           (2U)
+#define RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX             (2U)
+#define RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_META_MAX_VALUE_IDX                                  (4U)
+#define RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX                     (3U)
+#define RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX                          (5U)
+#define RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX                     (9U)
+#define RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX                            (2U)
+#define RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX                             (3U)
+#define RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX                      (3U)
+#define RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX                        (4U)
+#define RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX       (3U)
+#define RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX                             (4U)
+#define RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX              (2U)
+#define RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX                 (6U)
+#define RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX                          (2U)
+#define RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX                          (4U)
+#define RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX                           (3U)
+#define RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX            (2U)
+#define RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX                       (2U)
+#define RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX           (2U)
+#define RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX                        (2U)
+#define RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX                (2U)
+
+/******************************************************************************
+ * Features with values indexes
+ *****************************************************************************/
+
+typedef enum _RGX_FEATURE_WITH_VALUE_INDEX_ {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_IDX,
+	RGX_FEATURE_ECC_RAMS_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_IDX,
+	RGX_FEATURE_FBCDC_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_IDX,
+	RGX_FEATURE_LAYOUT_MARS_IDX,
+	RGX_FEATURE_META_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX,
+	RGX_FEATURE_NUM_MEMBUS_IDX,
+	RGX_FEATURE_NUM_OSIDS_IDX,
+	RGX_FEATURE_NUM_RASTER_PIPES_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_IDX,
+	RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_IDX,
+	RGX_FEATURE_SLC_BANKS_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_IDX,
+	RGX_FEATURE_TFBC_VERSION_IDX,
+	RGX_FEATURE_TILE_SIZE_X_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_IDX,
+	RGX_FEATURE_XE_ARCHITECTURE_IDX,
+	RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_IDX,
+	RGX_FEATURE_XPU_MAX_SLAVES_IDX,
+	RGX_FEATURE_XPU_REGISTER_BROADCAST_IDX,
+	RGX_FEATURE_WITH_VALUES_MAX_IDX,
+} RGX_FEATURE_WITH_VALUE_INDEX;
+
+
+/******************************************************************************
+ * Mask and bit-position macros for ERNs and BRNs
+ *****************************************************************************/
+
+#define FIX_HW_BRN_38344_POS                                            (0U)
+#define FIX_HW_BRN_38344_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000001))
+
+#define HW_ERN_42290_POS                                                (1U)
+#define HW_ERN_42290_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000002))
+
+#define FIX_HW_BRN_42321_POS                                            (2U)
+#define FIX_HW_BRN_42321_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000004))
+
+#define HW_ERN_42606_POS                                                (3U)
+#define HW_ERN_42606_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000008))
+
+#define HW_ERN_47025_POS                                                (4U)
+#define HW_ERN_47025_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000010))
+
+#define FIX_HW_BRN_50767_POS                                            (5U)
+#define FIX_HW_BRN_50767_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000020))
+
+#define HW_ERN_57596_POS                                                (6U)
+#define HW_ERN_57596_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000040))
+
+#define FIX_HW_BRN_60084_POS                                            (7U)
+#define FIX_HW_BRN_60084_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000080))
+
+#define HW_ERN_61389_POS                                                (8U)
+#define HW_ERN_61389_BIT_MASK                                           (IMG_UINT64_C(0x0000000000000100))
+
+#define FIX_HW_BRN_61450_POS                                            (9U)
+#define FIX_HW_BRN_61450_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000200))
+
+#define FIX_HW_BRN_63142_POS                                            (10U)
+#define FIX_HW_BRN_63142_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000400))
+
+#define FIX_HW_BRN_63553_POS                                            (11U)
+#define FIX_HW_BRN_63553_BIT_MASK                                       (IMG_UINT64_C(0x0000000000000800))
+
+#define FIX_HW_BRN_64502_POS                                            (12U)
+#define FIX_HW_BRN_64502_BIT_MASK                                       (IMG_UINT64_C(0x0000000000001000))
+
+#define FIX_HW_BRN_65101_POS                                            (13U)
+#define FIX_HW_BRN_65101_BIT_MASK                                       (IMG_UINT64_C(0x0000000000002000))
+
+#define FIX_HW_BRN_65273_POS                                            (14U)
+#define FIX_HW_BRN_65273_BIT_MASK                                       (IMG_UINT64_C(0x0000000000004000))
+
+#define HW_ERN_66622_POS                                                (15U)
+#define HW_ERN_66622_BIT_MASK                                           (IMG_UINT64_C(0x0000000000008000))
+
+#define FIX_HW_BRN_66927_POS                                            (16U)
+#define FIX_HW_BRN_66927_BIT_MASK                                       (IMG_UINT64_C(0x0000000000010000))
+
+#define FIX_HW_BRN_68186_POS                                            (17U)
+#define FIX_HW_BRN_68186_BIT_MASK                                       (IMG_UINT64_C(0x0000000000020000))
+
+#define FIX_HW_BRN_71242_POS                                            (18U)
+#define FIX_HW_BRN_71242_BIT_MASK                                       (IMG_UINT64_C(0x0000000000040000))
+
+#define FIX_HW_BRN_71317_POS                                            (19U)
+#define FIX_HW_BRN_71317_BIT_MASK                                       (IMG_UINT64_C(0x0000000000080000))
+
+/* Macro used for padding the unavailable values for features with values */
+#define RGX_FEATURE_VALUE_INVALID	(0xFFFFFFFEU)
+
+/* Macro used for marking a feature with value as disabled for a specific bvnc */
+#define RGX_FEATURE_VALUE_DISABLED	(0xFFFFFFFFU)
+
+#endif /* RGX_BVNC_DEFS_KM_H */
diff --git a/drivers/gpu/drm/img-rogue/km/rgx_bvnc_table_km.h b/drivers/gpu/drm/img-rogue/km/rgx_bvnc_table_km.h
new file mode 100644
index 000000000000..d10a0a1ab22d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km/rgx_bvnc_table_km.h
@@ -0,0 +1,458 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_bvnc_table_km.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/******************************************************************************
+ *                 Auto generated file by rgxbvnc_tablegen.py                 *
+ *                  This file should not be edited manually                   *
+ *****************************************************************************/
+
+#ifndef RGX_BVNC_TABLE_KM_H
+#define RGX_BVNC_TABLE_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgx_bvnc_defs_km.h"
+
+#ifndef RGXBVNC_C
+#error "This file should only be included from rgxbvnc.c"
+#endif
+
+#if defined(RGX_BVNC_TABLE_UM_H)
+#error "This file should not be included in conjunction with rgx_bvnc_table_um.h"
+#endif
+
+
+/******************************************************************************
+ * Arrays for each feature with values used
+ * for handling the corresponding values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values[RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_ECC_RAMS_values[RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values[RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_values[RGX_FEATURE_FBCDC_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, 4, 50, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ALGORITHM_values[RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 50, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values[RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 7, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values[RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_LAYOUT_MARS_values[RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_values[RGX_FEATURE_META_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, LTP217, LTP218, MTP218, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_META_COREMEM_SIZE_values[RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 0, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_CLUSTERS_values[RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, 6, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values[RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 3, 4, 6, 7, 8, 12, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_MEMBUS_values[RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_OSIDS_values[RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 8, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_NUM_RASTER_PIPES_values[RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values[RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 32, 36, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_values[RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_BANKS_values[RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, 2, 4, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values[RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 512, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values[RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 2, 8, 16, 64, 128, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TB_GPU_COUNT_values[RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TFBC_VERSION_values[RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 10, 11, 20, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_X_values[RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_TILE_SIZE_Y_values[RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 16, 32, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values[RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 40, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XE_ARCHITECTURE_values[RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_values[RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 19, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_MAX_SLAVES_values[RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 3, };
+
+static const IMG_UINT16 aui16_RGX_FEATURE_XPU_REGISTER_BROADCAST_values[RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX] = {(IMG_UINT16)RGX_FEATURE_VALUE_DISABLED, 1, };
+
+
+/******************************************************************************
+ * Table contains pointers to each feature value array for features that have
+ * values.
+ * Indexed using enum RGX_FEATURE_WITH_VALUE_INDEX from rgx_bvnc_defs_km.h
+ *****************************************************************************/
+
+static const void * const gaFeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX] = {
+	aui16_RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_values,
+	aui16_RGX_FEATURE_ECC_RAMS_values,
+	aui16_RGX_FEATURE_FAULT_DECODE_VERSION_values,
+	aui16_RGX_FEATURE_FBCDC_values,
+	aui16_RGX_FEATURE_FBCDC_ALGORITHM_values,
+	aui16_RGX_FEATURE_FBCDC_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_values,
+	aui16_RGX_FEATURE_LAYOUT_MARS_values,
+	aui16_RGX_FEATURE_META_values,
+	aui16_RGX_FEATURE_META_COREMEM_SIZE_values,
+	aui16_RGX_FEATURE_NUM_CLUSTERS_values,
+	aui16_RGX_FEATURE_NUM_ISP_IPP_PIPES_values,
+	aui16_RGX_FEATURE_NUM_MEMBUS_values,
+	aui16_RGX_FEATURE_NUM_OSIDS_values,
+	aui16_RGX_FEATURE_NUM_RASTER_PIPES_values,
+	aui16_RGX_FEATURE_PHYS_BUS_WIDTH_values,
+	aui16_RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_values,
+	aui16_RGX_FEATURE_SLC_BANKS_values,
+	aui16_RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_values,
+	aui16_RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_values,
+	aui16_RGX_FEATURE_TB_GPU_COUNT_values,
+	aui16_RGX_FEATURE_TFBC_VERSION_values,
+	aui16_RGX_FEATURE_TILE_SIZE_X_values,
+	aui16_RGX_FEATURE_TILE_SIZE_Y_values,
+	aui16_RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_values,
+	aui16_RGX_FEATURE_XE_ARCHITECTURE_values,
+	aui16_RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_values,
+	aui16_RGX_FEATURE_XPU_MAX_SLAVES_values,
+	aui16_RGX_FEATURE_XPU_REGISTER_BROADCAST_values,
+};
+
+
+/******************************************************************************
+ * Array containing the lengths of the arrays containing the values.
+ * Used for indexing the aui16_<FEATURE>_values defined upwards
+ *****************************************************************************/
+
+
+static const IMG_UINT16 gaFeaturesValuesMaxIndexes[] = {
+	RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_MAX_VALUE_IDX,
+	RGX_FEATURE_ECC_RAMS_MAX_VALUE_IDX,
+	RGX_FEATURE_FAULT_DECODE_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ALGORITHM_MAX_VALUE_IDX,
+	RGX_FEATURE_FBCDC_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_MAX_VALUE_IDX,
+	RGX_FEATURE_LAYOUT_MARS_MAX_VALUE_IDX,
+	RGX_FEATURE_META_MAX_VALUE_IDX,
+	RGX_FEATURE_META_COREMEM_SIZE_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_CLUSTERS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_ISP_IPP_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_MEMBUS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_OSIDS_MAX_VALUE_IDX,
+	RGX_FEATURE_NUM_RASTER_PIPES_MAX_VALUE_IDX,
+	RGX_FEATURE_PHYS_BUS_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_BANKS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_MAX_VALUE_IDX,
+	RGX_FEATURE_TB_GPU_COUNT_MAX_VALUE_IDX,
+	RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_X_MAX_VALUE_IDX,
+	RGX_FEATURE_TILE_SIZE_Y_MAX_VALUE_IDX,
+	RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_MAX_VALUE_IDX,
+	RGX_FEATURE_XE_ARCHITECTURE_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_MAX_SLAVES_MAX_VALUE_IDX,
+	RGX_FEATURE_XPU_REGISTER_BROADCAST_MAX_VALUE_IDX,
+};
+
+#define RGX_FEATURE_VALUE_TYPE_UINT16 (0x0000U)
+#define RGX_FEATURE_VALUE_TYPE_UINT32 (0x8000U)
+#define RGX_FEATURE_TYPE_BIT_SHIFT 14
+
+/******************************************************************************
+ * Bit-positions for features with values
+ *****************************************************************************/
+
+static const IMG_UINT16 aui16FeaturesWithValuesBitPositions[] = {
+	(0U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_POS */
+	(2U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_ECC_RAMS_POS */
+	(4U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FAULT_DECODE_VERSION_POS */
+	(6U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_POS */
+	(9U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ALGORITHM_POS */
+	(12U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBCDC_ARCHITECTURE_POS */
+	(15U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_POS */
+	(17U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_POS */
+	(19U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_LAYOUT_MARS_POS */
+	(21U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_POS */
+	(24U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_META_COREMEM_SIZE_POS */
+	(26U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_CLUSTERS_POS */
+	(29U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_ISP_IPP_PIPES_POS */
+	(33U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_MEMBUS_POS */
+	(35U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_OSIDS_POS */
+	(37U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_NUM_RASTER_PIPES_POS */
+	(39U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_PHYS_BUS_WIDTH_POS */
+	(42U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_POS */
+	(44U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_BANKS_POS */
+	(47U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_POS */
+	(49U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_POS */
+	(52U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TB_GPU_COUNT_POS */
+	(54U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TFBC_VERSION_POS */
+	(57U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_X_POS */
+	(59U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_TILE_SIZE_Y_POS */
+	(61U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_POS */
+	(64U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XE_ARCHITECTURE_POS */
+	(66U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_POS */
+	(68U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_MAX_SLAVES_POS */
+	(70U) | RGX_FEATURE_VALUE_TYPE_UINT16, /* RGX_FEATURE_XPU_REGISTER_BROADCAST_POS */
+};
+
+
+/******************************************************************************
+ * Bit-masks for features with values
+ *****************************************************************************/
+
+static const IMG_UINT64 aui64FeaturesWithValuesBitMasks[] = {
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_CDM_CONTROL_STREAM_FORMAT_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_ECC_RAMS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_FAULT_DECODE_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000001C0)), /* RGX_FEATURE_FBCDC_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000E00)), /* RGX_FEATURE_FBCDC_ALGORITHM_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000007000)), /* RGX_FEATURE_FBCDC_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000018000)), /* RGX_FEATURE_FBC_MAX_DEFAULT_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000060000)), /* RGX_FEATURE_FBC_MAX_LARGE_DESCRIPTORS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000180000)), /* RGX_FEATURE_LAYOUT_MARS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000E00000)), /* RGX_FEATURE_META_BIT_MASK */
+	(IMG_UINT64_C(0x0000000003000000)), /* RGX_FEATURE_META_COREMEM_SIZE_BIT_MASK */
+	(IMG_UINT64_C(0x000000001C000000)), /* RGX_FEATURE_NUM_CLUSTERS_BIT_MASK */
+	(IMG_UINT64_C(0x00000001E0000000)), /* RGX_FEATURE_NUM_ISP_IPP_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000000600000000)), /* RGX_FEATURE_NUM_MEMBUS_BIT_MASK */
+	(IMG_UINT64_C(0x0000001800000000)), /* RGX_FEATURE_NUM_OSIDS_BIT_MASK */
+	(IMG_UINT64_C(0x0000006000000000)), /* RGX_FEATURE_NUM_RASTER_PIPES_BIT_MASK */
+	(IMG_UINT64_C(0x0000038000000000)), /* RGX_FEATURE_PHYS_BUS_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x00000C0000000000)), /* RGX_FEATURE_SIMPLE_PARAMETER_FORMAT_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0000700000000000)), /* RGX_FEATURE_SLC_BANKS_BIT_MASK */
+	(IMG_UINT64_C(0x0001800000000000)), /* RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x000E000000000000)), /* RGX_FEATURE_SLC_SIZE_IN_KILOBYTES_BIT_MASK */
+	(IMG_UINT64_C(0x0030000000000000)), /* RGX_FEATURE_TB_GPU_COUNT_BIT_MASK */
+	(IMG_UINT64_C(0x01C0000000000000)), /* RGX_FEATURE_TFBC_VERSION_BIT_MASK */
+	(IMG_UINT64_C(0x0600000000000000)), /* RGX_FEATURE_TILE_SIZE_X_BIT_MASK */
+	(IMG_UINT64_C(0x1800000000000000)), /* RGX_FEATURE_TILE_SIZE_Y_BIT_MASK */
+	(IMG_UINT64_C(0x6000000000000000)), /* RGX_FEATURE_VIRTUAL_ADDRESS_SPACE_BITS_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000003)), /* RGX_FEATURE_XE_ARCHITECTURE_BIT_MASK */
+	(IMG_UINT64_C(0x000000000000000C)), /* RGX_FEATURE_XPU_MAX_REGBANKS_ADDR_WIDTH_BIT_MASK */
+	(IMG_UINT64_C(0x0000000000000030)), /* RGX_FEATURE_XPU_MAX_SLAVES_BIT_MASK */
+	(IMG_UINT64_C(0x00000000000000C0)), /* RGX_FEATURE_XPU_REGISTER_BROADCAST_BIT_MASK */
+};
+
+
+/******************************************************************************
+ * Table mapping bitmasks for features and features with values
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaFeatures[][4]=
+{
+	{ IMG_UINT64_C(0x0001000000040005), IMG_UINT64_C(0x0000000400010414), IMG_UINT64_C(0x340a81a06d6a9211), IMG_UINT64_C(0x0000000000000000) },	/* 1.0.4.5 */
+	{ IMG_UINT64_C(0x0001000000040013), IMG_UINT64_C(0x0000000400010415), IMG_UINT64_C(0x340a81a06d6a9211), IMG_UINT64_C(0x0000000000000000) },	/* 1.0.4.19 */
+	{ IMG_UINT64_C(0x0004000000020033), IMG_UINT64_C(0x0000042c0803045f), IMG_UINT64_C(0x340a81a0ea4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.2.51 */
+	{ IMG_UINT64_C(0x000400000002003a), IMG_UINT64_C(0x0000042c0803065f), IMG_UINT64_C(0x340a81b0ea4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.2.58 */
+	{ IMG_UINT64_C(0x0004000000040037), IMG_UINT64_C(0x0000042c0803045e), IMG_UINT64_C(0x340a81a0ee4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.4.55 */
+	{ IMG_UINT64_C(0x000400000006003e), IMG_UINT64_C(0x0000042c0803065f), IMG_UINT64_C(0x340ab1d0f24aa411), IMG_UINT64_C(0x0000000000000000) },	/* 4.0.6.62 */
+	{ IMG_UINT64_C(0x000500000001002e), IMG_UINT64_C(0x0000000008110445), IMG_UINT64_C(0x340691a0252a9411), IMG_UINT64_C(0x0000000000000001) },	/* 5.0.1.46 */
+	{ IMG_UINT64_C(0x0006000000040023), IMG_UINT64_C(0x0000042c0803045f), IMG_UINT64_C(0x340a81a0ee4aa411), IMG_UINT64_C(0x0000000000000000) },	/* 6.0.4.35 */
+	{ IMG_UINT64_C(0x000f000000010040), IMG_UINT64_C(0x0000000008110645), IMG_UINT64_C(0x340891b0452a9411), IMG_UINT64_C(0x0000000000000001) },	/* 15.0.1.64 */
+	{ IMG_UINT64_C(0x0016000000150010), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0494b044080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.21.16 */
+	{ IMG_UINT64_C(0x0016000000360019), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b064080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.25 */
+	{ IMG_UINT64_C(0x001600000036001e), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b084080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.30 */
+	{ IMG_UINT64_C(0x0016000000360026), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a08953084080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.38 */
+	{ IMG_UINT64_C(0x001600000036014a), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a0894b0840a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.54.330 */
+	{ IMG_UINT64_C(0x0016000000680012), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a089530c4080011), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.104.18 */
+	{ IMG_UINT64_C(0x00160000006800da), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a089530c40a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.104.218 */
+	{ IMG_UINT64_C(0x0016000000d0013e), IMG_UINT64_C(0x000000001ad1b615), IMG_UINT64_C(0x2a08a551080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 22.0.208.318 */
+	{ IMG_UINT64_C(0x00180000003600cc), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a089930840a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.54.204 */
+	{ IMG_UINT64_C(0x00180000006801f8), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a089930a40a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.104.504 */
+	{ IMG_UINT64_C(0x0018000000d001f8), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a0aa951080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.208.504 */
+	{ IMG_UINT64_C(0x0018000000d001f9), IMG_UINT64_C(0x000002003951fe95), IMG_UINT64_C(0x2a0aa951080a9651), IMG_UINT64_C(0x0000000000000001) },	/* 24.0.208.505 */
+	{ IMG_UINT64_C(0x001d0000003400ca), IMG_UINT64_C(0x000002183951feb5), IMG_UINT64_C(0x2a069930440a9891), IMG_UINT64_C(0x0000000000000001) },	/* 29.0.52.202 */
+	{ IMG_UINT64_C(0x001d0000006c00d0), IMG_UINT64_C(0x000002183951feb5), IMG_UINT64_C(0x2a0aa950e80a9891), IMG_UINT64_C(0x0000000000000001) },	/* 29.0.108.208 */
+	{ IMG_UINT64_C(0x00210000000b0003), IMG_UINT64_C(0x000002001551ba25), IMG_UINT64_C(0x2a02992824080011), IMG_UINT64_C(0x0000000000000001) },	/* 33.0.11.3 */
+	{ IMG_UINT64_C(0x0021000000160001), IMG_UINT64_C(0x000002181955be35), IMG_UINT64_C(0x2a06993044080011), IMG_UINT64_C(0x0000000000000001) },	/* 33.0.22.1 */
+	{ IMG_UINT64_C(0x00240000003400b6), IMG_UINT64_C(0x000002001d51bf35), IMG_UINT64_C(0x2a4699304412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.52.182 */
+	{ IMG_UINT64_C(0x0024000000360067), IMG_UINT64_C(0x000002181d51b735), IMG_UINT64_C(0x2a469930840abad1), IMG_UINT64_C(0x0000000000000001) },	/* 36.0.54.103 */
+	{ IMG_UINT64_C(0x00240000003600b6), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a4699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.182 */
+	{ IMG_UINT64_C(0x00240000003600b7), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a4699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.183 */
+	{ IMG_UINT64_C(0x0024000000360118), IMG_UINT64_C(0x00000219dd51bf35), IMG_UINT64_C(0x2a8699308412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.54.280 */
+	{ IMG_UINT64_C(0x00240000006800b6), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a469930a412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.182 */
+	{ IMG_UINT64_C(0x00240000006800b7), IMG_UINT64_C(0x000002181d51bf35), IMG_UINT64_C(0x2a469930a412bad1), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.183 */
+	{ IMG_UINT64_C(0x002400000068031c), IMG_UINT64_C(0x0000039a1d79af35), IMG_UINT64_C(0x2a469930a412bad9), IMG_UINT64_C(0x0000000000000055) },	/* 36.0.104.796 */
+	{ IMG_UINT64_C(0x002e000000660185), IMG_UINT64_C(0x00000241dd51bf35), IMG_UINT64_C(0x2ad899324412bad5), IMG_UINT64_C(0x0000000000000055) },	/* 46.0.102.389 */
+	{ IMG_UINT64_C(0x002e000000cc0186), IMG_UINT64_C(0x00000259dd51bf35), IMG_UINT64_C(0x2ad89932c412bad5), IMG_UINT64_C(0x0000000000000055) },	/* 46.0.204.390 */
+};
+
+/******************************************************************************
+ * Table mapping bitmasks for ERNs/BRNs
+ *****************************************************************************/
+
+
+static const IMG_UINT64 gaErnsBrns[][2]=
+{
+	{ IMG_UINT64_C(0x0001002700040013), IMG_UINT64_C(0x0000000000000005) },	/* 1.39.4.19 */
+	{ IMG_UINT64_C(0x0001005200040005), IMG_UINT64_C(0x0000000000000000) },	/* 1.82.4.5 */
+	{ IMG_UINT64_C(0x0004001f00040037), IMG_UINT64_C(0x000000000000042a) },	/* 4.31.4.55 */
+	{ IMG_UINT64_C(0x0004002800020033), IMG_UINT64_C(0x000000000000042a) },	/* 4.40.2.51 */
+	{ IMG_UINT64_C(0x0004002b0006003e), IMG_UINT64_C(0x000000000000142a) },	/* 4.43.6.62 */
+	{ IMG_UINT64_C(0x0004002d0002003a), IMG_UINT64_C(0x000000000000140a) },	/* 4.45.2.58 */
+	{ IMG_UINT64_C(0x0004002e0006003e), IMG_UINT64_C(0x000000000000142a) },	/* 4.46.6.62 */
+	{ IMG_UINT64_C(0x000500090001002e), IMG_UINT64_C(0x0000000000000001) },	/* 5.9.1.46 */
+	{ IMG_UINT64_C(0x0006002200040023), IMG_UINT64_C(0x000000000000040a) },	/* 6.34.4.35 */
+	{ IMG_UINT64_C(0x000f000500010040), IMG_UINT64_C(0x0000000000001008) },	/* 15.5.1.64 */
+	{ IMG_UINT64_C(0x0016001e00360019), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.30.54.25 */
+	{ IMG_UINT64_C(0x001600280036001e), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.40.54.30 */
+	{ IMG_UINT64_C(0x0016002e0036014a), IMG_UINT64_C(0x0000000000087a8a) },	/* 22.46.54.330 */
+	{ IMG_UINT64_C(0x0016003100150010), IMG_UINT64_C(0x0000000000085ac8) },	/* 22.49.21.16 */
+	{ IMG_UINT64_C(0x001600430036001e), IMG_UINT64_C(0x00000000000859c8) },	/* 22.67.54.30 */
+	{ IMG_UINT64_C(0x001600440036001e), IMG_UINT64_C(0x0000000000085948) },	/* 22.68.54.30 */
+	{ IMG_UINT64_C(0x00160056006800da), IMG_UINT64_C(0x0000000000083908) },	/* 22.86.104.218 */
+	{ IMG_UINT64_C(0x0016005700680012), IMG_UINT64_C(0x0000000000081948) },	/* 22.87.104.18 */
+	{ IMG_UINT64_C(0x0016006600360026), IMG_UINT64_C(0x0000000000081948) },	/* 22.102.54.38 */
+	{ IMG_UINT64_C(0x0016006800d0013e), IMG_UINT64_C(0x000000000008390a) },	/* 22.104.208.318 */
+	{ IMG_UINT64_C(0x0018003200d001f8), IMG_UINT64_C(0x000000000008884a) },	/* 24.50.208.504 */
+	{ IMG_UINT64_C(0x0018003800d001f9), IMG_UINT64_C(0x000000000008884a) },	/* 24.56.208.505 */
+	{ IMG_UINT64_C(0x00180042003600cc), IMG_UINT64_C(0x000000000008884a) },	/* 24.66.54.204 */
+	{ IMG_UINT64_C(0x00180043006801f8), IMG_UINT64_C(0x000000000008884a) },	/* 24.67.104.504 */
+	{ IMG_UINT64_C(0x001d000e006c00d0), IMG_UINT64_C(0x00000000000a885a) },	/* 29.14.108.208 */
+	{ IMG_UINT64_C(0x001d0013003400ca), IMG_UINT64_C(0x00000000000a885a) },	/* 29.19.52.202 */
+	{ IMG_UINT64_C(0x0021000800160001), IMG_UINT64_C(0x000000000008085a) },	/* 33.8.22.1 */
+	{ IMG_UINT64_C(0x0021000f000b0003), IMG_UINT64_C(0x00000000000c085a) },	/* 33.15.11.3 */
+	{ IMG_UINT64_C(0x0024001d003400b6), IMG_UINT64_C(0x000000000008085a) },	/* 36.29.52.182 */
+	{ IMG_UINT64_C(0x00240032003600b6), IMG_UINT64_C(0x00000000000c085a) },	/* 36.50.54.182 */
+	{ IMG_UINT64_C(0x00240034006800b6), IMG_UINT64_C(0x00000000000c085a) },	/* 36.52.104.182 */
+	{ IMG_UINT64_C(0x002400350068031c), IMG_UINT64_C(0x000000000008005a) },	/* 36.53.104.796 */
+	{ IMG_UINT64_C(0x00240036003600b7), IMG_UINT64_C(0x000000000008085a) },	/* 36.54.54.183 */
+	{ IMG_UINT64_C(0x0024003700360067), IMG_UINT64_C(0x000000000008085a) },	/* 36.55.54.103 */
+	{ IMG_UINT64_C(0x00240038006800b7), IMG_UINT64_C(0x000000000008085a) },	/* 36.56.104.183 */
+	{ IMG_UINT64_C(0x0024003c00360118), IMG_UINT64_C(0x000000000008085a) },	/* 36.60.54.280 */
+	{ IMG_UINT64_C(0x002e004800660185), IMG_UINT64_C(0x000000000009085a) },	/* 46.72.102.389 */
+	{ IMG_UINT64_C(0x002e005e00cc0186), IMG_UINT64_C(0x000000000009085a) },	/* 46.94.204.390 */
+};
+
+#if defined(DEBUG)
+
+#define FEATURE_NO_VALUES_NAMES_MAX_IDX (43U)
+
+static const IMG_CHAR * const gaszFeaturesNoValuesNames[FEATURE_NO_VALUES_NAMES_MAX_IDX] =
+{
+	"AXI_ACELITE",
+	"CLUSTER_GROUPING",
+	"COMPUTE",
+	"COMPUTE_MORTON_CAPABLE",
+	"COMPUTE_OVERLAP",
+	"COREID_PER_OS",
+	"DYNAMIC_DUST_POWER",
+	"FASTRENDER_DM",
+	"GPU_MULTICORE_SUPPORT",
+	"GPU_VIRTUALISATION",
+	"GS_RTA_SUPPORT",
+	"IRQ_PER_OS",
+	"MIPS",
+	"PBE2_IN_XE",
+	"PBE_CHECKSUM_2D",
+	"PBVNC_COREID_REG",
+	"PERFBUS",
+	"PERF_COUNTER_BATCH",
+	"PM_MMU_VFP",
+	"RISCV_FW_PROCESSOR",
+	"ROGUEXE",
+	"SAFETY_IRQ",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT_V1",
+	"SIMPLE_INTERNAL_PARAMETER_FORMAT_V2",
+	"SINGLE_BIF",
+	"SLC_SIZE_CONFIGURABLE",
+	"SOC_TIMER",
+	"SYS_BUS_SECURE_RESET",
+	"TDM_PDS_CHECKSUM",
+	"TFBC_DELTA_CORRELATION",
+	"TFBC_LOSSY_37_PERCENT",
+	"TFBC_NATIVE_YUV10",
+	"TILE_REGION_PROTECTION",
+	"TLA",
+	"TPU_CEM_DATAMASTER_GLOBAL_REGISTERS",
+	"TPU_DM_GLOBAL_REGISTERS",
+	"TPU_FILTERING_MODE_CONTROL",
+	"VOLCANIC_TB",
+	"WATCHDOG_TIMER",
+	"WORKGROUP_PROTECTION",
+	"XE_MEMORY_HIERARCHY",
+	"XT_TOP_INFRASTRUCTURE",
+};
+
+#define ERNSBRNS_IDS_MAX_IDX (20U)
+
+static const IMG_UINT32 gaui64ErnsBrnsIDs[ERNSBRNS_IDS_MAX_IDX] =
+{
+	38344,
+	42290,
+	42321,
+	42606,
+	47025,
+	50767,
+	57596,
+	60084,
+	61389,
+	61450,
+	63142,
+	63553,
+	64502,
+	65101,
+	65273,
+	66622,
+	66927,
+	68186,
+	71242,
+	71317,
+};
+
+#endif /* defined(DEBUG) */
+#endif /* RGX_BVNC_TABLE_KM_H */
diff --git a/drivers/gpu/drm/img-rogue/km/rgx_cr_defs_km.h b/drivers/gpu/drm/img-rogue/km/rgx_cr_defs_km.h
new file mode 100644
index 000000000000..1326877b50be
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km/rgx_cr_defs_km.h
@@ -0,0 +1,8472 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgx_cr_defs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ */
+
+
+#ifndef RGX_CR_DEFS_KM_H
+#define RGX_CR_DEFS_KM_H
+
+#if !defined(IMG_EXPLICIT_INCLUDE_HWDEFS)
+#error This file may only be included if explicitly defined
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGX_CR_DEFS_KM_REVISION 1
+
+/*
+    Register RGX_CR_RASTERISATION_INDIRECT
+*/
+#define RGX_CR_RASTERISATION_INDIRECT                                               (0x8238U)
+#define RGX_CR_RASTERISATION_INDIRECT_MASKFULL                                      (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_RASTERISATION_INDIRECT_ADDRESS_SHIFT                                 (0U)
+#define RGX_CR_RASTERISATION_INDIRECT_ADDRESS_CLRMSK                                (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_USC_INDIRECT
+*/
+#define RGX_CR_USC_INDIRECT                                                         (0x8000U)
+#define RGX_CR_USC_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_USC_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_USC_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_PBE_INDIRECT
+*/
+#define RGX_CR_PBE_INDIRECT                                                         (0x83E0U)
+#define RGX_CR_PBE_INDIRECT_MASKFULL                                                (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_PBE_INDIRECT_ADDRESS_SHIFT                                           (0U)
+#define RGX_CR_PBE_INDIRECT_ADDRESS_CLRMSK                                          (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_PBE_PERF_INDIRECT
+*/
+#define RGX_CR_PBE_PERF_INDIRECT                                                    (0x83D8U)
+#define RGX_CR_PBE_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_PBE_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_PBE_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_TPU_PERF_INDIRECT
+*/
+#define RGX_CR_TPU_PERF_INDIRECT                                                    (0x83F0U)
+#define RGX_CR_TPU_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TPU_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_TPU_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_INDIRECT
+*/
+#define RGX_CR_RASTERISATION_PERF_INDIRECT                                          (0x8318U)
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_MASKFULL                                 (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_ADDRESS_SHIFT                            (0U)
+#define RGX_CR_RASTERISATION_PERF_INDIRECT_ADDRESS_CLRMSK                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_INDIRECT
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT                                             (0x8028U)
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_ADDRESS_SHIFT                               (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_INDIRECT_ADDRESS_CLRMSK                              (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_USC_PERF_INDIRECT
+*/
+#define RGX_CR_USC_PERF_INDIRECT                                                    (0x8030U)
+#define RGX_CR_USC_PERF_INDIRECT_MASKFULL                                           (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_USC_PERF_INDIRECT_ADDRESS_SHIFT                                      (0U)
+#define RGX_CR_USC_PERF_INDIRECT_ADDRESS_CLRMSK                                     (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_INDIRECT
+*/
+#define RGX_CR_BLACKPEARL_INDIRECT                                                  (0x8388U)
+#define RGX_CR_BLACKPEARL_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BLACKPEARL_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BLACKPEARL_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_INDIRECT
+*/
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT                                             (0x83F8U)
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_ADDRESS_SHIFT                               (0U)
+#define RGX_CR_BLACKPEARL_PERF_INDIRECT_ADDRESS_CLRMSK                              (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_TEXAS3_PERF_INDIRECT
+*/
+#define RGX_CR_TEXAS3_PERF_INDIRECT                                                 (0x83D0U)
+#define RGX_CR_TEXAS3_PERF_INDIRECT_MASKFULL                                        (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_TEXAS3_PERF_INDIRECT_ADDRESS_SHIFT                                   (0U)
+#define RGX_CR_TEXAS3_PERF_INDIRECT_ADDRESS_CLRMSK                                  (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_INDIRECT
+*/
+#define RGX_CR_TEXAS_PERF_INDIRECT                                                  (0x8288U)
+#define RGX_CR_TEXAS_PERF_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_TEXAS_PERF_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_TEXAS_PERF_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_BX_TU_PERF_INDIRECT
+*/
+#define RGX_CR_BX_TU_PERF_INDIRECT                                                  (0xC900U)
+#define RGX_CR_BX_TU_PERF_INDIRECT_MASKFULL                                         (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BX_TU_PERF_INDIRECT_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BX_TU_PERF_INDIRECT_ADDRESS_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_CLK_CTRL
+*/
+#define RGX_CR_CLK_CTRL                                                             (0x0000U)
+#define RGX_CR_CLK_CTRL__PBE2_XE__MASKFULL                                          (IMG_UINT64_C(0xFFFFFF003F3FFFFF))
+#define RGX_CR_CLK_CTRL__S7_INFRA__MASKFULL                                         (IMG_UINT64_C(0xCFCF03000F3F3F0F))
+#define RGX_CR_CLK_CTRL_MASKFULL                                                    (IMG_UINT64_C(0xFFFFFF003F3FFFFF))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_SHIFT                                             (62U)
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_CLRMSK                                            (IMG_UINT64_C(0x3FFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_ON                                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS_AUTO                                              (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__ON                                     (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__S7_INFRA__AUTO                                   (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__OFF                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__ON                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_TEXAS__PBE2_XE__AUTO                                    (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_SHIFT                                                   (60U)
+#define RGX_CR_CLK_CTRL_IPP_CLRMSK                                                  (IMG_UINT64_C(0xCFFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_IPP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_ON                                                      (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP_AUTO                                                    (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__ON                                           (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__ON                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_CLK_CTRL_IPP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_CLK_CTRL_FBC_SHIFT                                                   (58U)
+#define RGX_CR_CLK_CTRL_FBC_CLRMSK                                                  (IMG_UINT64_C(0xF3FFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FBC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC_ON                                                      (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC_AUTO                                                    (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_CLK_CTRL_FBC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_SHIFT                                                  (56U)
+#define RGX_CR_CLK_CTRL_FBDC_CLRMSK                                                 (IMG_UINT64_C(0xFCFFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FBDC_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_ON                                                     (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC_AUTO                                                   (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__ON                                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__S7_INFRA__AUTO                                        (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__ON                                           (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_CLK_CTRL_FBDC__PBE2_XE__AUTO                                         (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_SHIFT                                            (54U)
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_CLRMSK                                           (IMG_UINT64_C(0xFF3FFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_ON                                               (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE_AUTO                                             (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__ON                                    (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__S7_INFRA__AUTO                                  (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__OFF                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__ON                                     (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_CLK_CTRL_FB_TLCACHE__PBE2_XE__AUTO                                   (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_CLK_CTRL_USCS_SHIFT                                                  (52U)
+#define RGX_CR_CLK_CTRL_USCS_CLRMSK                                                 (IMG_UINT64_C(0xFFCFFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_USCS_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS_ON                                                     (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS_AUTO                                                   (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__ON                                          (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS__S7_INFRA__AUTO                                        (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__ON                                           (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_CLK_CTRL_USCS__PBE2_XE__AUTO                                         (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_CLK_CTRL_PBE_SHIFT                                                   (50U)
+#define RGX_CR_CLK_CTRL_PBE_CLRMSK                                                  (IMG_UINT64_C(0xFFF3FFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_PBE_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE_ON                                                      (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE_AUTO                                                    (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__ON                                           (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__ON                                            (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_CLK_CTRL_PBE__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_SHIFT                                                (48U)
+#define RGX_CR_CLK_CTRL_MCU_L1_CLRMSK                                               (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_MCU_L1_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_ON                                                   (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1_AUTO                                                 (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__ON                                        (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__S7_INFRA__AUTO                                      (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__ON                                         (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L1__PBE2_XE__AUTO                                       (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_CLK_CTRL_CDM_SHIFT                                                   (46U)
+#define RGX_CR_CLK_CTRL_CDM_CLRMSK                                                  (IMG_UINT64_C(0xFFFF3FFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_CDM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM_ON                                                      (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM_AUTO                                                    (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_CLK_CTRL_CDM__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_SHIFT                                              (44U)
+#define RGX_CR_CLK_CTRL_SIDEKICK_CLRMSK                                             (IMG_UINT64_C(0xFFFFCFFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_SIDEKICK_OFF                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_ON                                                 (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK_AUTO                                               (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__OFF                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__ON                                      (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__S7_INFRA__AUTO                                    (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__OFF                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__ON                                       (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_CLK_CTRL_SIDEKICK__PBE2_XE__AUTO                                     (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_SHIFT                                          (42U)
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_CLRMSK                                         (IMG_UINT64_C(0xFFFFF3FFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_ON                                             (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK_AUTO                                           (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__ON                                  (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__S7_INFRA__AUTO                                (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__OFF                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__ON                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_CLK_CTRL_BIF_SIDEKICK__PBE2_XE__AUTO                                 (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_CLK_CTRL_BIF_SHIFT                                                   (40U)
+#define RGX_CR_CLK_CTRL_BIF_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_CLK_CTRL_BIF_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF_ON                                                      (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF_AUTO                                                    (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_CLK_CTRL_BIF__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_SHIFT                                         (28U)
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_ON                                            (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX_AUTO                                          (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__OFF                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__ON                                 (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__S7_INFRA__AUTO                               (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__OFF                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__ON                                  (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_CTRL_TPU_MCU_DEMUX__PBE2_XE__AUTO                                (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_SHIFT                                                (26U)
+#define RGX_CR_CLK_CTRL_MCU_L0_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_CTRL_MCU_L0_OFF                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_ON                                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0_AUTO                                                 (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__ON                                        (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__S7_INFRA__AUTO                                      (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__OFF                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__ON                                         (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_CTRL_MCU_L0__PBE2_XE__AUTO                                       (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_CTRL_TPU_SHIFT                                                   (24U)
+#define RGX_CR_CLK_CTRL_TPU_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_CTRL_TPU_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU_ON                                                      (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU_AUTO                                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_CTRL_TPU__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_CTRL_USC_SHIFT                                                   (20U)
+#define RGX_CR_CLK_CTRL_USC_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_CTRL_USC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC_ON                                                      (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC_AUTO                                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_CTRL_USC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_CTRL_TLA_SHIFT                                                   (18U)
+#define RGX_CR_CLK_CTRL_TLA_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_CTRL_TLA_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA_ON                                                      (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA_AUTO                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_CTRL_TLA__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_CTRL_SLC_SHIFT                                                   (16U)
+#define RGX_CR_CLK_CTRL_SLC_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_CTRL_SLC_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC_ON                                                      (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC_AUTO                                                    (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_CTRL_SLC__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_CLK_CTRL_UVS_SHIFT                                                   (14U)
+#define RGX_CR_CLK_CTRL_UVS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_CLK_CTRL_UVS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS_ON                                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS_AUTO                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_CTRL_UVS__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_CLK_CTRL_PDS_SHIFT                                                   (12U)
+#define RGX_CR_CLK_CTRL_PDS_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_CLK_CTRL_PDS_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS_ON                                                      (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS_AUTO                                                    (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_CTRL_PDS__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_CTRL_VDM_SHIFT                                                   (10U)
+#define RGX_CR_CLK_CTRL_VDM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL_VDM_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM_ON                                                      (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM_AUTO                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL_VDM__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL_PM_SHIFT                                                    (8U)
+#define RGX_CR_CLK_CTRL_PM_CLRMSK                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL_PM_OFF                                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM_ON                                                       (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM_AUTO                                                     (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__ON                                            (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM__S7_INFRA__AUTO                                          (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__ON                                             (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL_PM__PBE2_XE__AUTO                                           (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL_GPP_SHIFT                                                   (6U)
+#define RGX_CR_CLK_CTRL_GPP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFF3F))
+#define RGX_CR_CLK_CTRL_GPP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP_ON                                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP_AUTO                                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_CTRL_GPP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_CTRL_TE_SHIFT                                                    (4U)
+#define RGX_CR_CLK_CTRL_TE_CLRMSK                                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL_TE_OFF                                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE_ON                                                       (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE_AUTO                                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__ON                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE__S7_INFRA__AUTO                                          (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__ON                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL_TE__PBE2_XE__AUTO                                           (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL_TSP_SHIFT                                                   (2U)
+#define RGX_CR_CLK_CTRL_TSP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF3))
+#define RGX_CR_CLK_CTRL_TSP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP_ON                                                      (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP_AUTO                                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_CTRL_TSP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_CTRL_ISP_SHIFT                                                   (0U)
+#define RGX_CR_CLK_CTRL_ISP_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL_ISP_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP_ON                                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP_AUTO                                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__ON                                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP__S7_INFRA__AUTO                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__ON                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL_ISP__PBE2_XE__AUTO                                          (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS
+*/
+#define RGX_CR_CLK_STATUS                                                           (0x0008U)
+#define RGX_CR_CLK_STATUS__PBE2_XE__MASKFULL                                        (IMG_UINT64_C(0x00000001FFF077FF))
+#define RGX_CR_CLK_STATUS__S7_INFRA__MASKFULL                                       (IMG_UINT64_C(0x00000001B3101773))
+#define RGX_CR_CLK_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x00000001FFF077FF))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_SHIFT                                            (32U)
+#define RGX_CR_CLK_STATUS_MCU_FBTC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC_RUNNING                                          (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__S7_INFRA__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__S7_INFRA__RUNNING                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__PBE2_XE__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_FBTC__PBE2_XE__RUNNING                                (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_SHIFT                                           (31U)
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_GATED                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS_RUNNING                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__S7_INFRA__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__S7_INFRA__RUNNING                              (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__PBE2_XE__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_TEXAS__PBE2_XE__RUNNING                               (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_STATUS_IPP_SHIFT                                                 (30U)
+#define RGX_CR_CLK_STATUS_IPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_CLK_STATUS_IPP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP_RUNNING                                               (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_IPP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_IPP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_IPP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_STATUS_FBC_SHIFT                                                 (29U)
+#define RGX_CR_CLK_STATUS_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_CLK_STATUS_FBC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC_RUNNING                                               (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_STATUS_FBDC_SHIFT                                                (28U)
+#define RGX_CR_CLK_STATUS_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_CLK_STATUS_FBDC_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC_RUNNING                                              (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FBDC__S7_INFRA__GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC__S7_INFRA__RUNNING                                   (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FBDC__PBE2_XE__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FBDC__PBE2_XE__RUNNING                                    (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_SHIFT                                          (27U)
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE_RUNNING                                        (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__S7_INFRA__GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__S7_INFRA__RUNNING                             (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__PBE2_XE__GATED                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_FB_TLCACHE__PBE2_XE__RUNNING                              (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_STATUS_USCS_SHIFT                                                (26U)
+#define RGX_CR_CLK_STATUS_USCS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_CLK_STATUS_USCS_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS_RUNNING                                              (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_USCS__S7_INFRA__GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS__S7_INFRA__RUNNING                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_USCS__PBE2_XE__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USCS__PBE2_XE__RUNNING                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_STATUS_PBE_SHIFT                                                 (25U)
+#define RGX_CR_CLK_STATUS_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_CLK_STATUS_PBE_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE_RUNNING                                               (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_PBE__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_PBE__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PBE__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_STATUS_MCU_L1_SHIFT                                              (24U)
+#define RGX_CR_CLK_STATUS_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_CLK_STATUS_MCU_L1_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1_RUNNING                                            (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__S7_INFRA__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__S7_INFRA__RUNNING                                 (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__PBE2_XE__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L1__PBE2_XE__RUNNING                                  (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_STATUS_CDM_SHIFT                                                 (23U)
+#define RGX_CR_CLK_STATUS_CDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_CLK_STATUS_CDM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM_RUNNING                                               (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_CDM__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_CDM__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_CDM__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_STATUS_SIDEKICK_SHIFT                                            (22U)
+#define RGX_CR_CLK_STATUS_SIDEKICK_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_CLK_STATUS_SIDEKICK_GATED                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK_RUNNING                                          (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__S7_INFRA__GATED                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__S7_INFRA__RUNNING                               (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__PBE2_XE__GATED                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SIDEKICK__PBE2_XE__RUNNING                                (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_SHIFT                                        (21U)
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK_RUNNING                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__S7_INFRA__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__S7_INFRA__RUNNING                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__PBE2_XE__GATED                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_SIDEKICK__PBE2_XE__RUNNING                            (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_STATUS_BIF_SHIFT                                                 (20U)
+#define RGX_CR_CLK_STATUS_BIF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_CLK_STATUS_BIF_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF_RUNNING                                               (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_BIF__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_BIF__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_BIF__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_SHIFT                                       (14U)
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX_RUNNING                                     (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__S7_INFRA__GATED                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__S7_INFRA__RUNNING                          (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__PBE2_XE__GATED                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_MCU_DEMUX__PBE2_XE__RUNNING                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_CLK_STATUS_MCU_L0_SHIFT                                              (13U)
+#define RGX_CR_CLK_STATUS_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_CLK_STATUS_MCU_L0_GATED                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0_RUNNING                                            (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_MCU_L0__S7_INFRA__GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0__S7_INFRA__RUNNING                                 (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_MCU_L0__PBE2_XE__GATED                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_MCU_L0__PBE2_XE__RUNNING                                  (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_CLK_STATUS_TPU_SHIFT                                                 (12U)
+#define RGX_CR_CLK_STATUS_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_CLK_STATUS_TPU_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU_RUNNING                                               (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_TPU__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_TPU__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TPU__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_CLK_STATUS_USC_SHIFT                                                 (10U)
+#define RGX_CR_CLK_STATUS_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_STATUS_USC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC_RUNNING                                               (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_USC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_USC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_USC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_STATUS_TLA_SHIFT                                                 (9U)
+#define RGX_CR_CLK_STATUS_TLA_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_STATUS_TLA_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA_RUNNING                                               (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_TLA__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_TLA__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TLA__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_STATUS_SLC_SHIFT                                                 (8U)
+#define RGX_CR_CLK_STATUS_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_STATUS_SLC_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC_RUNNING                                               (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_SLC__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_SLC__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_SLC__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_STATUS_UVS_SHIFT                                                 (7U)
+#define RGX_CR_CLK_STATUS_UVS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_STATUS_UVS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS_RUNNING                                               (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_UVS__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_UVS__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_UVS__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_STATUS_PDS_SHIFT                                                 (6U)
+#define RGX_CR_CLK_STATUS_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_STATUS_PDS_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS_RUNNING                                               (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_PDS__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_PDS__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PDS__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_STATUS_VDM_SHIFT                                                 (5U)
+#define RGX_CR_CLK_STATUS_VDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_STATUS_VDM_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM_RUNNING                                               (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_VDM__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_VDM__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_VDM__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_STATUS_PM_SHIFT                                                  (4U)
+#define RGX_CR_CLK_STATUS_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS_PM_GATED                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM_RUNNING                                                (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_PM__S7_INFRA__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM__S7_INFRA__RUNNING                                     (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_PM__PBE2_XE__GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_PM__PBE2_XE__RUNNING                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS_GPP_SHIFT                                                 (3U)
+#define RGX_CR_CLK_STATUS_GPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_STATUS_GPP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP_RUNNING                                               (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_GPP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_GPP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_GPP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_STATUS_TE_SHIFT                                                  (2U)
+#define RGX_CR_CLK_STATUS_TE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS_TE_GATED                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE_RUNNING                                                (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TE__S7_INFRA__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE__S7_INFRA__RUNNING                                     (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TE__PBE2_XE__GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TE__PBE2_XE__RUNNING                                      (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS_TSP_SHIFT                                                 (1U)
+#define RGX_CR_CLK_STATUS_TSP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_STATUS_TSP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP_RUNNING                                               (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_TSP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_TSP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_TSP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_STATUS_ISP_SHIFT                                                 (0U)
+#define RGX_CR_CLK_STATUS_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS_ISP_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP_RUNNING                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS_ISP__S7_INFRA__GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP__S7_INFRA__RUNNING                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_STATUS_ISP__PBE2_XE__GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS_ISP__PBE2_XE__RUNNING                                     (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID__PBVNC                                                       (0x0020U)
+#define RGX_CR_CORE_ID__PBVNC__MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__BRANCH_ID_SHIFT                                      (48U)
+#define RGX_CR_CORE_ID__PBVNC__BRANCH_ID_CLRMSK                                     (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__VERSION_ID_SHIFT                                     (32U)
+#define RGX_CR_CORE_ID__PBVNC__VERSION_ID_CLRMSK                                    (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_SHIFT                       (16U)
+#define RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_CORE_ID__PBVNC__CONFIG_ID_SHIFT                                      (0U)
+#define RGX_CR_CORE_ID__PBVNC__CONFIG_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_CORE_ID
+*/
+#define RGX_CR_CORE_ID                                                              (0x0018U)
+#define RGX_CR_CORE_ID_MASKFULL                                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CORE_ID_ID_SHIFT                                                     (16U)
+#define RGX_CR_CORE_ID_ID_CLRMSK                                                    (0x0000FFFFU)
+#define RGX_CR_CORE_ID_CONFIG_SHIFT                                                 (0U)
+#define RGX_CR_CORE_ID_CONFIG_CLRMSK                                                (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_CORE_REVISION
+*/
+#define RGX_CR_CORE_REVISION                                                        (0x0020U)
+#define RGX_CR_CORE_REVISION_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CORE_REVISION_DESIGNER_SHIFT                                         (24U)
+#define RGX_CR_CORE_REVISION_DESIGNER_CLRMSK                                        (0x00FFFFFFU)
+#define RGX_CR_CORE_REVISION_MAJOR_SHIFT                                            (16U)
+#define RGX_CR_CORE_REVISION_MAJOR_CLRMSK                                           (0xFF00FFFFU)
+#define RGX_CR_CORE_REVISION_MINOR_SHIFT                                            (8U)
+#define RGX_CR_CORE_REVISION_MINOR_CLRMSK                                           (0xFFFF00FFU)
+#define RGX_CR_CORE_REVISION_MAINTENANCE_SHIFT                                      (0U)
+#define RGX_CR_CORE_REVISION_MAINTENANCE_CLRMSK                                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_DESIGNER_REV_FIELD1
+*/
+#define RGX_CR_DESIGNER_REV_FIELD1                                                  (0x0028U)
+#define RGX_CR_DESIGNER_REV_FIELD1_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_DESIGNER_REV_FIELD1_DESIGNER_REV_FIELD1_SHIFT                        (0U)
+#define RGX_CR_DESIGNER_REV_FIELD1_DESIGNER_REV_FIELD1_CLRMSK                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_DESIGNER_REV_FIELD2
+*/
+#define RGX_CR_DESIGNER_REV_FIELD2                                                  (0x0030U)
+#define RGX_CR_DESIGNER_REV_FIELD2_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_DESIGNER_REV_FIELD2_DESIGNER_REV_FIELD2_SHIFT                        (0U)
+#define RGX_CR_DESIGNER_REV_FIELD2_DESIGNER_REV_FIELD2_CLRMSK                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_CHANGESET_NUMBER
+*/
+#define RGX_CR_CHANGESET_NUMBER                                                     (0x0040U)
+#define RGX_CR_CHANGESET_NUMBER_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_CHANGESET_NUMBER_CHANGESET_NUMBER_SHIFT                              (0U)
+#define RGX_CR_CHANGESET_NUMBER_CHANGESET_NUMBER_CLRMSK                             (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_GRAY
+*/
+#define RGX_CR_SOC_TIMER_GRAY                                                       (0x00E0U)
+#define RGX_CR_SOC_TIMER_GRAY_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_SHIFT                                           (0U)
+#define RGX_CR_SOC_TIMER_GRAY_VALUE_CLRMSK                                          (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SOC_TIMER_BINARY
+*/
+#define RGX_CR_SOC_TIMER_BINARY                                                     (0x00E8U)
+#define RGX_CR_SOC_TIMER_BINARY_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SOC_TIMER_BINARY_VALUE_CLRMSK                                        (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_CLK_XTPLUS_CTRL
+*/
+#define RGX_CR_CLK_XTPLUS_CTRL                                                      (0x0080U)
+#define RGX_CR_CLK_XTPLUS_CTRL_MASKFULL                                             (IMG_UINT64_C(0x0000003FFFFF0000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_SHIFT                                            (36U)
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFCFFFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_ON                                               (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_TDM_AUTO                                             (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_SHIFT                                           (34U)
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFF3FFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_OFF                                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_ON                                              (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_ASTC_AUTO                                            (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_SHIFT                                            (32U)
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFCFFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_OFF                                              (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_ON                                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_IPF_AUTO                                             (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_SHIFT                                        (30U)
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFF3FFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_OFF                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_ON                                           (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_COMPUTE_AUTO                                         (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_SHIFT                                          (28U)
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFCFFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_ON                                             (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PIXEL_AUTO                                           (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_SHIFT                                         (26U)
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF3FFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_OFF                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_ON                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_VERTEX_AUTO                                          (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_SHIFT                                          (24U)
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_OFF                                            (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_ON                                             (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USCPS_AUTO                                           (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_SHIFT                                     (22U)
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFF3FFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_ON                                        (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_CLK_XTPLUS_CTRL_PDS_SHARED_AUTO                                      (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_SHIFT                                 (20U)
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_OFF                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_ON                                    (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_CLK_XTPLUS_CTRL_BIF_BLACKPEARL_AUTO                                  (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_SHIFT                                     (18U)
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_OFF                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_ON                                        (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_CLK_XTPLUS_CTRL_USC_SHARED_AUTO                                      (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_SHIFT                                       (16U)
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_OFF                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_ON                                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_CLK_XTPLUS_CTRL_GEOMETRY_AUTO                                        (IMG_UINT64_C(0x0000000000020000))
+
+
+/*
+    Register RGX_CR_CLK_XTPLUS_STATUS
+*/
+#define RGX_CR_CLK_XTPLUS_STATUS                                                    (0x0088U)
+#define RGX_CR_CLK_XTPLUS_STATUS_MASKFULL                                           (IMG_UINT64_C(0x00000000000007FF))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_SHIFT                                          (10U)
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_TDM_RUNNING                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_SHIFT                                          (9U)
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_GATED                                          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_IPF_RUNNING                                        (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_SHIFT                                      (8U)
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_GATED                                      (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_COMPUTE_RUNNING                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_SHIFT                                         (7U)
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_GATED                                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_ASTC_RUNNING                                       (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_SHIFT                                        (6U)
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_PIXEL_RUNNING                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_SHIFT                                       (5U)
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_GATED                                       (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_VERTEX_RUNNING                                     (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_SHIFT                                        (4U)
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_GATED                                        (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_USCPS_RUNNING                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_SHIFT                                   (3U)
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_PDS_SHARED_RUNNING                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_SHIFT                               (2U)
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_GATED                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_BIF_BLACKPEARL_RUNNING                             (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_SHIFT                                   (1U)
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_GATED                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_USC_SHARED_RUNNING                                 (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_SHIFT                                     (0U)
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_GATED                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_XTPLUS_STATUS_GEOMETRY_RUNNING                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET
+*/
+#define RGX_CR_SOFT_RESET                                                           (0x0100U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL                                        (IMG_UINT64_C(0xFFEFFFFFFFFFFC3D))
+#define RGX_CR_SOFT_RESET_MASKFULL                                                  (IMG_UINT64_C(0x00E7FFFFFFFFFC3D))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_SHIFT                             (63U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_CLRMSK                            (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM3_CORE_EN                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_SHIFT                             (62U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__PHANTOM2_CORE_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_SHIFT                             (61U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_CLRMSK                            (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__BERNADO2_CORE_EN                                (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_SHIFT                                (60U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_CLRMSK                               (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__JONES_CORE_EN                                   (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_SHIFT                               (59U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_CLRMSK                              (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TILING_CORE_EN                                  (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_SHIFT                                       (58U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_CLRMSK                                      (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__TE3_EN                                          (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_SHIFT                                       (57U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_CLRMSK                                      (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VCE_EN                                          (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_SHIFT                                       (56U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_CLRMSK                                      (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__VBS_EN                                          (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SOFT_RESET_DPX1_CORE_SHIFT                                           (55U)
+#define RGX_CR_SOFT_RESET_DPX1_CORE_CLRMSK                                          (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DPX1_CORE_EN                                              (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SOFT_RESET_DPX0_CORE_SHIFT                                           (54U)
+#define RGX_CR_SOFT_RESET_DPX0_CORE_CLRMSK                                          (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DPX0_CORE_EN                                              (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SOFT_RESET_FBA_SHIFT                                                 (53U)
+#define RGX_CR_SOFT_RESET_FBA_CLRMSK                                                (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_FBA_EN                                                    (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_SHIFT                                    (51U)
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_CLRMSK                                   (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET__PBE2_XE__FB_CDC_EN                                       (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SOFT_RESET_SH_SHIFT                                                  (50U)
+#define RGX_CR_SOFT_RESET_SH_CLRMSK                                                 (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_SH_EN                                                     (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SOFT_RESET_VRDM_SHIFT                                                (49U)
+#define RGX_CR_SOFT_RESET_VRDM_CLRMSK                                               (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_VRDM_EN                                                   (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SOFT_RESET_MCU_FBTC_SHIFT                                            (48U)
+#define RGX_CR_SOFT_RESET_MCU_FBTC_CLRMSK                                           (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_MCU_FBTC_EN                                               (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_SHIFT                                       (47U)
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PHANTOM1_CORE_EN                                          (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_SHIFT                                       (46U)
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_PHANTOM0_CORE_EN                                          (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_SHIFT                                       (45U)
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BERNADO1_CORE_EN                                          (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_SHIFT                                       (44U)
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_CLRMSK                                      (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BERNADO0_CORE_EN                                          (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SOFT_RESET_IPP_SHIFT                                                 (43U)
+#define RGX_CR_SOFT_RESET_IPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_IPP_EN                                                    (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_SHIFT                                           (42U)
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_CLRMSK                                          (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF_TEXAS_EN                                              (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_SHIFT                                        (41U)
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_TORNADO_CORE_EN                                           (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_SHIFT                                         (40U)
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_H_CORE_EN                                            (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_SHIFT                                         (39U)
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_G_CORE_EN                                            (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_SHIFT                                         (38U)
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_F_CORE_EN                                            (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_SHIFT                                         (37U)
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_E_CORE_EN                                            (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_SHIFT                                         (36U)
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_D_CORE_EN                                            (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_SHIFT                                         (35U)
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_C_CORE_EN                                            (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SOFT_RESET_MMU_SHIFT                                                 (34U)
+#define RGX_CR_SOFT_RESET_MMU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SOFT_RESET_MMU_EN                                                    (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SOFT_RESET_BIF1_SHIFT                                                (33U)
+#define RGX_CR_SOFT_RESET_BIF1_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SOFT_RESET_BIF1_EN                                                   (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SOFT_RESET_GARTEN_SHIFT                                              (32U)
+#define RGX_CR_SOFT_RESET_GARTEN_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SOFT_RESET_GARTEN_EN                                                 (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SOFT_RESET_CPU_SHIFT                                                 (32U)
+#define RGX_CR_SOFT_RESET_CPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SOFT_RESET_CPU_EN                                                    (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_SHIFT                                         (31U)
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SOFT_RESET_RASCAL_CORE_EN                                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_SHIFT                                         (30U)
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_B_CORE_EN                                            (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_SHIFT                                         (29U)
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SOFT_RESET_DUST_A_CORE_EN                                            (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_SHIFT                                          (28U)
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SOFT_RESET_FB_TLCACHE_EN                                             (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SOFT_RESET_SLC_SHIFT                                                 (27U)
+#define RGX_CR_SOFT_RESET_SLC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SOFT_RESET_SLC_EN                                                    (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SOFT_RESET_TLA_SHIFT                                                 (26U)
+#define RGX_CR_SOFT_RESET_TLA_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SOFT_RESET_TLA_EN                                                    (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SOFT_RESET_UVS_SHIFT                                                 (25U)
+#define RGX_CR_SOFT_RESET_UVS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SOFT_RESET_UVS_EN                                                    (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SOFT_RESET_TE_SHIFT                                                  (24U)
+#define RGX_CR_SOFT_RESET_TE_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SOFT_RESET_TE_EN                                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SOFT_RESET_GPP_SHIFT                                                 (23U)
+#define RGX_CR_SOFT_RESET_GPP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SOFT_RESET_GPP_EN                                                    (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SOFT_RESET_FBDC_SHIFT                                                (22U)
+#define RGX_CR_SOFT_RESET_FBDC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SOFT_RESET_FBDC_EN                                                   (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SOFT_RESET_FBC_SHIFT                                                 (21U)
+#define RGX_CR_SOFT_RESET_FBC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SOFT_RESET_FBC_EN                                                    (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SOFT_RESET_PM_SHIFT                                                  (20U)
+#define RGX_CR_SOFT_RESET_PM_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SOFT_RESET_PM_EN                                                     (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SOFT_RESET_PBE_SHIFT                                                 (19U)
+#define RGX_CR_SOFT_RESET_PBE_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SOFT_RESET_PBE_EN                                                    (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SOFT_RESET_USC_SHARED_SHIFT                                          (18U)
+#define RGX_CR_SOFT_RESET_USC_SHARED_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SOFT_RESET_USC_SHARED_EN                                             (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SOFT_RESET_MCU_L1_SHIFT                                              (17U)
+#define RGX_CR_SOFT_RESET_MCU_L1_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SOFT_RESET_MCU_L1_EN                                                 (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SOFT_RESET_BIF_SHIFT                                                 (16U)
+#define RGX_CR_SOFT_RESET_BIF_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SOFT_RESET_BIF_EN                                                    (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SOFT_RESET_CDM_SHIFT                                                 (15U)
+#define RGX_CR_SOFT_RESET_CDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SOFT_RESET_CDM_EN                                                    (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SOFT_RESET_VDM_SHIFT                                                 (14U)
+#define RGX_CR_SOFT_RESET_VDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SOFT_RESET_VDM_EN                                                    (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SOFT_RESET_TESS_SHIFT                                                (13U)
+#define RGX_CR_SOFT_RESET_TESS_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SOFT_RESET_TESS_EN                                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SOFT_RESET_PDS_SHIFT                                                 (12U)
+#define RGX_CR_SOFT_RESET_PDS_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SOFT_RESET_PDS_EN                                                    (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SOFT_RESET_ISP_SHIFT                                                 (11U)
+#define RGX_CR_SOFT_RESET_ISP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SOFT_RESET_ISP_EN                                                    (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SOFT_RESET_TSP_SHIFT                                                 (10U)
+#define RGX_CR_SOFT_RESET_TSP_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SOFT_RESET_TSP_EN                                                    (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SOFT_RESET_SYSARB_SHIFT                                              (5U)
+#define RGX_CR_SOFT_RESET_SYSARB_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SOFT_RESET_SYSARB_EN                                                 (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_SHIFT                                       (4U)
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SOFT_RESET_TPU_MCU_DEMUX_EN                                          (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SOFT_RESET_MCU_L0_SHIFT                                              (3U)
+#define RGX_CR_SOFT_RESET_MCU_L0_CLRMSK                                             (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SOFT_RESET_MCU_L0_EN                                                 (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SOFT_RESET_TPU_SHIFT                                                 (2U)
+#define RGX_CR_SOFT_RESET_TPU_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SOFT_RESET_TPU_EN                                                    (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SOFT_RESET_USC_SHIFT                                                 (0U)
+#define RGX_CR_SOFT_RESET_USC_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SOFT_RESET_USC_EN                                                    (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SOFT_RESET2
+*/
+#define RGX_CR_SOFT_RESET2                                                          (0x0108U)
+#define RGX_CR_SOFT_RESET2_MASKFULL                                                 (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_SOFT_RESET2_SPFILTER_SHIFT                                           (12U)
+#define RGX_CR_SOFT_RESET2_SPFILTER_CLRMSK                                          (0xFFE00FFFU)
+#define RGX_CR_SOFT_RESET2_TDM_SHIFT                                                (11U)
+#define RGX_CR_SOFT_RESET2_TDM_CLRMSK                                               (0xFFFFF7FFU)
+#define RGX_CR_SOFT_RESET2_TDM_EN                                                   (0x00000800U)
+#define RGX_CR_SOFT_RESET2_ASTC_SHIFT                                               (10U)
+#define RGX_CR_SOFT_RESET2_ASTC_CLRMSK                                              (0xFFFFFBFFU)
+#define RGX_CR_SOFT_RESET2_ASTC_EN                                                  (0x00000400U)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_SHIFT                                         (9U)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_CLRMSK                                        (0xFFFFFDFFU)
+#define RGX_CR_SOFT_RESET2_BLACKPEARL_EN                                            (0x00000200U)
+#define RGX_CR_SOFT_RESET2_USCPS_SHIFT                                              (8U)
+#define RGX_CR_SOFT_RESET2_USCPS_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_SOFT_RESET2_USCPS_EN                                                 (0x00000100U)
+#define RGX_CR_SOFT_RESET2_IPF_SHIFT                                                (7U)
+#define RGX_CR_SOFT_RESET2_IPF_CLRMSK                                               (0xFFFFFF7FU)
+#define RGX_CR_SOFT_RESET2_IPF_EN                                                   (0x00000080U)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_SHIFT                                           (6U)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SOFT_RESET2_GEOMETRY_EN                                              (0x00000040U)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_SHIFT                                         (5U)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_SOFT_RESET2_USC_SHARED_EN                                            (0x00000020U)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_SHIFT                                         (4U)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_CR_SOFT_RESET2_PDS_SHARED_EN                                            (0x00000010U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_SHIFT                                     (3U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_SOFT_RESET2_BIF_BLACKPEARL_EN                                        (0x00000008U)
+#define RGX_CR_SOFT_RESET2_PIXEL_SHIFT                                              (2U)
+#define RGX_CR_SOFT_RESET2_PIXEL_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_SOFT_RESET2_PIXEL_EN                                                 (0x00000004U)
+#define RGX_CR_SOFT_RESET2_CDM_SHIFT                                                (1U)
+#define RGX_CR_SOFT_RESET2_CDM_CLRMSK                                               (0xFFFFFFFDU)
+#define RGX_CR_SOFT_RESET2_CDM_EN                                                   (0x00000002U)
+#define RGX_CR_SOFT_RESET2_VERTEX_SHIFT                                             (0U)
+#define RGX_CR_SOFT_RESET2_VERTEX_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_CR_SOFT_RESET2_VERTEX_EN                                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_ENABLE
+*/
+#define RGX_CR_EVENT_ENABLE                                                         (0x0128U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__MASKFULL                                      (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_ENABLE__SIGNALS__MASKFULL                                      (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_ENABLE_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_SHIFT                                 (28U)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_DPX_OUT_OF_MEMORY_EN                                    (0x10000000U)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_SHIFT                                (27U)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_CLRMSK                               (0xF7FFFFFFU)
+#define RGX_CR_EVENT_ENABLE_DPX_MMU_PAGE_FAULT_EN                                   (0x08000000U)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_SHIFT                                 (26U)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RPM_OUT_OF_MEMORY_EN                                    (0x04000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_SHIFT                                  (25U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_CLRMSK                                 (0xFDFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC3_FINISHED_EN                                     (0x02000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_SHIFT                                  (24U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_CLRMSK                                 (0xFEFFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC2_FINISHED_EN                                     (0x01000000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_SHIFT                                  (23U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_CLRMSK                                 (0xFF7FFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC1_FINISHED_EN                                     (0x00800000U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_SHIFT                                  (22U)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_EVENT_ENABLE_FBA_FC0_FINISHED_EN                                     (0x00400000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_SHIFT                                  (21U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_CLRMSK                                 (0xFFDFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC3_FINISHED_EN                                     (0x00200000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_SHIFT                                  (20U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC2_FINISHED_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_SHIFT                                  (20U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SAFETY_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_SHIFT                                  (19U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_CLRMSK                                 (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC1_FINISHED_EN                                     (0x00080000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_SHIFT                               (19U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__SLAVE_REQ_EN                                  (0x00080000U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_SHIFT                                  (18U)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_CLRMSK                                 (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE_RDM_FC0_FINISHED_EN                                     (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_SHIFT                                      (17U)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_CLRMSK                                     (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE_SHG_FINISHED_EN                                         (0x00020000U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_EVENT_ENABLE__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_ENABLE_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_SHIFT                                      (14U)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_CLRMSK                                     (0xFFFFBFFFU)
+#define RGX_CR_EVENT_ENABLE_ZLS_FINISHED_EN                                         (0x00004000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_ENABLE_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_ENABLE_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_ENABLE_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_ENABLE_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_SHIFT                                    (8U)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_EVENT_ENABLE_PM_3D_MEM_FREE_EN                                       (0x00000100U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_ENABLE_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_ENABLE_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_ENABLE_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_ENABLE_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_SHIFT                                (3U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_EVENT_ENABLE_PIXELBE_END_RENDER_EN                                   (0x00000008U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_ENABLE_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_ENABLE_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_SHIFT                                      (0U)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_EVENT_ENABLE_TLA_COMPLETE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_STATUS
+*/
+#define RGX_CR_EVENT_STATUS                                                         (0x0130U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__MASKFULL                                      (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_STATUS__SIGNALS__MASKFULL                                      (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_STATUS_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_SHIFT                                (31U)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_CLRMSK                               (0x7FFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_FENCE_FINISHED_EN                                   (0x80000000U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_SHIFT                                  (30U)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_CLRMSK                                 (0xBFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_TDM_BUFFER_STALL_EN                                     (0x40000000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_SHIFT                            (29U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_CLRMSK                           (0xDFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_SIGNAL_FAILURE_EN                               (0x20000000U)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_SHIFT                                 (28U)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_EVENT_STATUS_DPX_OUT_OF_MEMORY_EN                                    (0x10000000U)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_SHIFT                                (27U)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_CLRMSK                               (0xF7FFFFFFU)
+#define RGX_CR_EVENT_STATUS_DPX_MMU_PAGE_FAULT_EN                                   (0x08000000U)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_SHIFT                                 (26U)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_EVENT_STATUS_RPM_OUT_OF_MEMORY_EN                                    (0x04000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_SHIFT                                  (25U)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_CLRMSK                                 (0xFDFFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC3_FINISHED_EN                                     (0x02000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_SHIFT                                  (24U)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_CLRMSK                                 (0xFEFFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC2_FINISHED_EN                                     (0x01000000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_SHIFT                                  (23U)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_CLRMSK                                 (0xFF7FFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC1_FINISHED_EN                                     (0x00800000U)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_SHIFT                                  (22U)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_EVENT_STATUS_FBA_FC0_FINISHED_EN                                     (0x00400000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_SHIFT                                  (21U)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_CLRMSK                                 (0xFFDFFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC3_FINISHED_EN                                     (0x00200000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_SHIFT                                  (20U)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC2_FINISHED_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_SHIFT                                  (20U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_CLRMSK                                 (0xFFEFFFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_EN                                     (0x00100000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_SHIFT                                  (19U)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_CLRMSK                                 (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC1_FINISHED_EN                                     (0x00080000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_SHIFT                               (19U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__SLAVE_REQ_EN                                  (0x00080000U)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_SHIFT                                  (18U)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_CLRMSK                                 (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS_RDM_FC0_FINISHED_EN                                     (0x00040000U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT              (18U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK             (0xFFFBFFFFU)
+#define RGX_CR_EVENT_STATUS__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                 (0x00040000U)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_SHIFT                                      (17U)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_CLRMSK                                     (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS_SHG_FINISHED_EN                                         (0x00020000U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                  (17U)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                 (0xFFFDFFFFU)
+#define RGX_CR_EVENT_STATUS__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                     (0x00020000U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_SHIFT                              (16U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_CLRMSK                             (0xFFFEFFFFU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_BUFFER_STALL_EN                                 (0x00010000U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_SHIFT                                       (15U)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_CLRMSK                                      (0xFFFF7FFFU)
+#define RGX_CR_EVENT_STATUS_USC_TRIGGER_EN                                          (0x00008000U)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_SHIFT                                      (14U)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_CLRMSK                                     (0xFFFFBFFFU)
+#define RGX_CR_EVENT_STATUS_ZLS_FINISHED_EN                                         (0x00004000U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_SHIFT                                          (13U)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_ACK_EN                                             (0x00002000U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_SHIFT                                          (12U)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_CLRMSK                                         (0xFFFFEFFFU)
+#define RGX_CR_EVENT_STATUS_GPIO_REQ_EN                                             (0x00001000U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_SHIFT                                       (11U)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_CLRMSK                                      (0xFFFFF7FFU)
+#define RGX_CR_EVENT_STATUS_POWER_ABORT_EN                                          (0x00000800U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_SHIFT                                    (10U)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_CLRMSK                                   (0xFFFFFBFFU)
+#define RGX_CR_EVENT_STATUS_POWER_COMPLETE_EN                                       (0x00000400U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_SHIFT                                    (9U)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT_EN                                       (0x00000200U)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_SHIFT                                    (8U)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_EVENT_STATUS_PM_3D_MEM_FREE_EN                                       (0x00000100U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_SHIFT                                  (7U)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_CLRMSK                                 (0xFFFFFF7FU)
+#define RGX_CR_EVENT_STATUS_PM_OUT_OF_MEMORY_EN                                     (0x00000080U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_SHIFT                                      (6U)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_CLRMSK                                     (0xFFFFFFBFU)
+#define RGX_CR_EVENT_STATUS_TA_TERMINATE_EN                                         (0x00000040U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_SHIFT                                       (5U)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_CLRMSK                                      (0xFFFFFFDFU)
+#define RGX_CR_EVENT_STATUS_TA_FINISHED_EN                                          (0x00000020U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_SHIFT                                 (4U)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_EVENT_STATUS_ISP_END_MACROTILE_EN                                    (0x00000010U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_SHIFT                                (3U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_EVENT_STATUS_PIXELBE_END_RENDER_EN                                   (0x00000008U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_SHIFT                                  (2U)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_EVENT_STATUS_COMPUTE_FINISHED_EN                                     (0x00000004U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_SHIFT                                   (1U)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_EVENT_STATUS_KERNEL_FINISHED_EN                                      (0x00000002U)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_SHIFT                                      (0U)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_EVENT_STATUS_TLA_COMPLETE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_EVENT_CLEAR
+*/
+#define RGX_CR_EVENT_CLEAR                                                          (0x0138U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__MASKFULL                                       (IMG_UINT64_C(0x00000000E01DFFFF))
+#define RGX_CR_EVENT_CLEAR__SIGNALS__MASKFULL                                       (IMG_UINT64_C(0x00000000E007FFFF))
+#define RGX_CR_EVENT_CLEAR_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_SHIFT                                 (31U)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_CLRMSK                                (0x7FFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_FENCE_FINISHED_EN                                    (0x80000000U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_SHIFT                                   (30U)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_CLRMSK                                  (0xBFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_TDM_BUFFER_STALL_EN                                      (0x40000000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_SHIFT                             (29U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_CLRMSK                            (0xDFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_SIGNAL_FAILURE_EN                                (0x20000000U)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_SHIFT                                  (28U)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_CLRMSK                                 (0xEFFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_DPX_OUT_OF_MEMORY_EN                                     (0x10000000U)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_SHIFT                                 (27U)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_CLRMSK                                (0xF7FFFFFFU)
+#define RGX_CR_EVENT_CLEAR_DPX_MMU_PAGE_FAULT_EN                                    (0x08000000U)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_SHIFT                                  (26U)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_CLRMSK                                 (0xFBFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RPM_OUT_OF_MEMORY_EN                                     (0x04000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_SHIFT                                   (25U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_CLRMSK                                  (0xFDFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC3_FINISHED_EN                                      (0x02000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_SHIFT                                   (24U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_CLRMSK                                  (0xFEFFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC2_FINISHED_EN                                      (0x01000000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_SHIFT                                   (23U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_CLRMSK                                  (0xFF7FFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC1_FINISHED_EN                                      (0x00800000U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_SHIFT                                   (22U)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_CLRMSK                                  (0xFFBFFFFFU)
+#define RGX_CR_EVENT_CLEAR_FBA_FC0_FINISHED_EN                                      (0x00400000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_SHIFT                                   (21U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_CLRMSK                                  (0xFFDFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC3_FINISHED_EN                                      (0x00200000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_SHIFT                                   (20U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC2_FINISHED_EN                                      (0x00100000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_SHIFT                                   (20U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_EN                                      (0x00100000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_SHIFT                                   (19U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_CLRMSK                                  (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC1_FINISHED_EN                                      (0x00080000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_SHIFT                                (19U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_CLRMSK                               (0xFFF7FFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__SLAVE_REQ_EN                                   (0x00080000U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_SHIFT                                   (18U)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_CLRMSK                                  (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR_RDM_FC0_FINISHED_EN                                      (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_SHIFT               (18U)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_CLRMSK              (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__ROGUEXE__TDM_CONTEXT_STORE_FINISHED_EN                  (0x00040000U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_SHIFT               (18U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_CLRMSK              (0xFFFBFFFFU)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__TDM_CONTEXT_STORE_FINISHED_EN                  (0x00040000U)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_SHIFT                                       (17U)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_CLRMSK                                      (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR_SHG_FINISHED_EN                                          (0x00020000U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_SHIFT                   (17U)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_CLRMSK                  (0xFFFDFFFFU)
+#define RGX_CR_EVENT_CLEAR__SIGNALS__SPFILTER_SIGNAL_UPDATE_EN                      (0x00020000U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_SHIFT                               (16U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_BUFFER_STALL_EN                                  (0x00010000U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_SHIFT                                        (15U)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_CLRMSK                                       (0xFFFF7FFFU)
+#define RGX_CR_EVENT_CLEAR_USC_TRIGGER_EN                                           (0x00008000U)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_SHIFT                                       (14U)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_CLRMSK                                      (0xFFFFBFFFU)
+#define RGX_CR_EVENT_CLEAR_ZLS_FINISHED_EN                                          (0x00004000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_SHIFT                                           (13U)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_CLRMSK                                          (0xFFFFDFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_ACK_EN                                              (0x00002000U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_SHIFT                                           (12U)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_CLRMSK                                          (0xFFFFEFFFU)
+#define RGX_CR_EVENT_CLEAR_GPIO_REQ_EN                                              (0x00001000U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_SHIFT                                        (11U)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_CLRMSK                                       (0xFFFFF7FFU)
+#define RGX_CR_EVENT_CLEAR_POWER_ABORT_EN                                           (0x00000800U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_SHIFT                                     (10U)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_CLRMSK                                    (0xFFFFFBFFU)
+#define RGX_CR_EVENT_CLEAR_POWER_COMPLETE_EN                                        (0x00000400U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_SHIFT                                     (9U)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_CLRMSK                                    (0xFFFFFDFFU)
+#define RGX_CR_EVENT_CLEAR_MMU_PAGE_FAULT_EN                                        (0x00000200U)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_SHIFT                                     (8U)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_CLRMSK                                    (0xFFFFFEFFU)
+#define RGX_CR_EVENT_CLEAR_PM_3D_MEM_FREE_EN                                        (0x00000100U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_SHIFT                                   (7U)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_CLRMSK                                  (0xFFFFFF7FU)
+#define RGX_CR_EVENT_CLEAR_PM_OUT_OF_MEMORY_EN                                      (0x00000080U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_SHIFT                                       (6U)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_CLRMSK                                      (0xFFFFFFBFU)
+#define RGX_CR_EVENT_CLEAR_TA_TERMINATE_EN                                          (0x00000040U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_SHIFT                                        (5U)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_EVENT_CLEAR_TA_FINISHED_EN                                           (0x00000020U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_SHIFT                                  (4U)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_CLRMSK                                 (0xFFFFFFEFU)
+#define RGX_CR_EVENT_CLEAR_ISP_END_MACROTILE_EN                                     (0x00000010U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_SHIFT                                 (3U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_EVENT_CLEAR_PIXELBE_END_RENDER_EN                                    (0x00000008U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_SHIFT                                   (2U)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_EVENT_CLEAR_COMPUTE_FINISHED_EN                                      (0x00000004U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_SHIFT                                    (1U)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_CLRMSK                                   (0xFFFFFFFDU)
+#define RGX_CR_EVENT_CLEAR_KERNEL_FINISHED_EN                                       (0x00000002U)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_SHIFT                                       (0U)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_EVENT_CLEAR_TLA_COMPLETE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_TIMER
+*/
+#define RGX_CR_TIMER                                                                (0x0160U)
+#define RGX_CR_TIMER_MASKFULL                                                       (IMG_UINT64_C(0x8000FFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_SHIFT                                                    (63U)
+#define RGX_CR_TIMER_BIT31_CLRMSK                                                   (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_TIMER_BIT31_EN                                                       (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_TIMER_VALUE_SHIFT                                                    (0U)
+#define RGX_CR_TIMER_VALUE_CLRMSK                                                   (IMG_UINT64_C(0xFFFF000000000000))
+
+
+/*
+    Register RGX_CR_TLA_STATUS
+*/
+#define RGX_CR_TLA_STATUS                                                           (0x0178U)
+#define RGX_CR_TLA_STATUS_MASKFULL                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_TLA_STATUS_BLIT_COUNT_SHIFT                                          (39U)
+#define RGX_CR_TLA_STATUS_BLIT_COUNT_CLRMSK                                         (IMG_UINT64_C(0x0000007FFFFFFFFF))
+#define RGX_CR_TLA_STATUS_REQUEST_SHIFT                                             (7U)
+#define RGX_CR_TLA_STATUS_REQUEST_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF800000007F))
+#define RGX_CR_TLA_STATUS_FIFO_FULLNESS_SHIFT                                       (1U)
+#define RGX_CR_TLA_STATUS_FIFO_FULLNESS_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFF81))
+#define RGX_CR_TLA_STATUS_BUSY_SHIFT                                                (0U)
+#define RGX_CR_TLA_STATUS_BUSY_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_TLA_STATUS_BUSY_EN                                                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_PM_PARTIAL_RENDER_ENABLE
+*/
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE                                             (0x0338U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_SHIFT                                    (0U)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_PM_PARTIAL_RENDER_ENABLE_OP_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_SIDEKICK_IDLE
+*/
+#define RGX_CR_SIDEKICK_IDLE                                                        (0x03C8U)
+#define RGX_CR_SIDEKICK_IDLE_MASKFULL                                               (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_SHIFT                                           (6U)
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SIDEKICK_IDLE_FB_CDC_EN                                              (0x00000040U)
+#define RGX_CR_SIDEKICK_IDLE_MMU_SHIFT                                              (5U)
+#define RGX_CR_SIDEKICK_IDLE_MMU_CLRMSK                                             (0xFFFFFFDFU)
+#define RGX_CR_SIDEKICK_IDLE_MMU_EN                                                 (0x00000020U)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_SHIFT                                           (4U)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_CLRMSK                                          (0xFFFFFFEFU)
+#define RGX_CR_SIDEKICK_IDLE_BIF128_EN                                              (0x00000010U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_SHIFT                                              (3U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_CLRMSK                                             (0xFFFFFFF7U)
+#define RGX_CR_SIDEKICK_IDLE_TLA_EN                                                 (0x00000008U)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_SHIFT                                           (2U)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_SIDEKICK_IDLE_GARTEN_EN                                              (0x00000004U)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_SHIFT                                           (1U)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_SIDEKICK_IDLE_HOSTIF_EN                                              (0x00000002U)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_SHIFT                                            (0U)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SIDEKICK_IDLE_SOCIF_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_MARS_IDLE
+*/
+#define RGX_CR_MARS_IDLE                                                            (0x08F8U)
+#define RGX_CR_MARS_IDLE_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_SHIFT                                           (2U)
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_MARS_IDLE_MH_SYSARB0_EN                                              (0x00000004U)
+#define RGX_CR_MARS_IDLE_CPU_SHIFT                                                  (1U)
+#define RGX_CR_MARS_IDLE_CPU_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_MARS_IDLE_CPU_EN                                                     (0x00000002U)
+#define RGX_CR_MARS_IDLE_SOCIF_SHIFT                                                (0U)
+#define RGX_CR_MARS_IDLE_SOCIF_CLRMSK                                               (0xFFFFFFFEU)
+#define RGX_CR_MARS_IDLE_SOCIF_EN                                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS                                             (0x0430U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_MASKFULL                                    (IMG_UINT64_C(0x00000000000000F3))
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_LAST_PIPE_SHIFT                             (4U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_LAST_PIPE_CLRMSK                            (0xFFFFFF0FU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_SHIFT                           (1U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_CLRMSK                          (0xFFFFFFFDU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_NEED_RESUME_EN                              (0x00000002U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_SHIFT                              (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_VDM_CONTEXT_STORE_STATUS_COMPLETE_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK0
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0                                              (0x0438U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_MASKFULL                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE1_SHIFT                             (32U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE1_CLRMSK                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE0_SHIFT                             (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK0_PDS_STATE0_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK1
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1                                              (0x0440U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_PDS_STATE2_SHIFT                             (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK1_PDS_STATE2_CLRMSK                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_TASK2
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2                                              (0x0448U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_MASKFULL                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT2_SHIFT                            (32U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT2_CLRMSK                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT1_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_TASK2_STREAM_OUT1_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK0
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0                                             (0x0450U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_MASKFULL                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE1_SHIFT                            (32U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE1_CLRMSK                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE0_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK0_PDS_STATE0_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK1
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1                                             (0x0458U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_PDS_STATE2_SHIFT                            (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK1_PDS_STATE2_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_RESUME_TASK2
+*/
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2                                             (0x0460U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_MASKFULL                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT2_SHIFT                           (32U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT2_CLRMSK                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT1_SHIFT                           (0U)
+#define RGX_CR_VDM_CONTEXT_RESUME_TASK2_STREAM_OUT1_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_STORE_STATUS
+*/
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS                                             (0x04A0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_MASKFULL                                    (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_SHIFT                           (1U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_CLRMSK                          (0xFFFFFFFDU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_NEED_RESUME_EN                              (0x00000002U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_SHIFT                              (0U)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_STORE_STATUS_COMPLETE_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_PDS0                                                     (0x04A8U)
+#define RGX_CR_CDM_CONTEXT_PDS0_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_SHIFT                                     (36U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_CLRMSK                                    (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_DATA_ADDR_ALIGNSIZE                                 (16U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_SHIFT                                     (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_PDS0_CODE_ADDR_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_PDS1                                                     (0x04B0U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__MASKFULL                                (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1_MASKFULL                                            (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                       (30U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                      (0xBFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                          (0x40000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_SHIFT                                   (29U)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_CLRMSK                                  (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_PDS_SEQ_DEP_EN                                      (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                       (29U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                      (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                          (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_SHIFT                                   (28U)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_CLRMSK                                  (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_USC_SEQ_DEP_EN                                      (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_SHIFT                            (28U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_CLRMSK                           (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TARGET_EN                               (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_SHIFT                                        (27U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_CLRMSK                                       (0xF7FFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_TARGET_EN                                           (0x08000000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                      (22U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                     (0xF03FFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_SHIFT                                  (21U)
+#define RGX_CR_CDM_CONTEXT_PDS1_UNIFIED_SIZE_CLRMSK                                 (0xF81FFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                     (21U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK                    (0xFFDFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SHARED_EN                        (0x00200000U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_SHIFT                                 (20U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_CLRMSK                                (0xFFEFFFFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SHARED_EN                                    (0x00100000U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                       (12U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                      (0xFFE00FFFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SIZE_SHIFT                                   (11U)
+#define RGX_CR_CDM_CONTEXT_PDS1_COMMON_SIZE_CLRMSK                                  (0xFFF007FFU)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_SHIFT                                     (7U)
+#define RGX_CR_CDM_CONTEXT_PDS1_TEMP_SIZE_CLRMSK                                    (0xFFFFF87FU)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                         (7U)
+#define RGX_CR_CDM_CONTEXT_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                        (0xFFFFF07FU)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_SHIFT                                     (1U)
+#define RGX_CR_CDM_CONTEXT_PDS1_DATA_SIZE_CLRMSK                                    (0xFFFFFF81U)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_SHIFT                                         (0U)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_PDS1_FENCE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS
+*/
+#define RGX_CR_CDM_TERMINATE_PDS                                                    (0x04B8U)
+#define RGX_CR_CDM_TERMINATE_PDS_MASKFULL                                           (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_SHIFT                                    (36U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_CLRMSK                                   (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_DATA_ADDR_ALIGNSIZE                                (16U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_SHIFT                                    (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSHIFT                               (4U)
+#define RGX_CR_CDM_TERMINATE_PDS_CODE_ADDR_ALIGNSIZE                                (16U)
+
+
+/*
+    Register RGX_CR_CDM_TERMINATE_PDS1
+*/
+#define RGX_CR_CDM_TERMINATE_PDS1                                                   (0x04C0U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__MASKFULL                              (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1_MASKFULL                                          (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                     (30U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                    (0xBFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                        (0x40000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_SHIFT                                 (29U)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_CLRMSK                                (0xDFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_PDS_SEQ_DEP_EN                                    (0x20000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                     (29U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                    (0xDFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                        (0x20000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_SHIFT                                 (28U)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_CLRMSK                                (0xEFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_USC_SEQ_DEP_EN                                    (0x10000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_SHIFT                          (28U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_CLRMSK                         (0xEFFFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TARGET_EN                             (0x10000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_SHIFT                                      (27U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_CLRMSK                                     (0xF7FFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_TARGET_EN                                         (0x08000000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                    (22U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                   (0xF03FFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_SHIFT                                (21U)
+#define RGX_CR_CDM_TERMINATE_PDS1_UNIFIED_SIZE_CLRMSK                               (0xF81FFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                   (21U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK                  (0xFFDFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SHARED_EN                      (0x00200000U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_SHIFT                               (20U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_CLRMSK                              (0xFFEFFFFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SHARED_EN                                  (0x00100000U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                     (12U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                    (0xFFE00FFFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SIZE_SHIFT                                 (11U)
+#define RGX_CR_CDM_TERMINATE_PDS1_COMMON_SIZE_CLRMSK                                (0xFFF007FFU)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_SHIFT                                   (7U)
+#define RGX_CR_CDM_TERMINATE_PDS1_TEMP_SIZE_CLRMSK                                  (0xFFFFF87FU)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                       (7U)
+#define RGX_CR_CDM_TERMINATE_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                      (0xFFFFF07FU)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_SHIFT                                   (1U)
+#define RGX_CR_CDM_TERMINATE_PDS1_DATA_SIZE_CLRMSK                                  (0xFFFFFF81U)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_SHIFT                                       (0U)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_CDM_TERMINATE_PDS1_FENCE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS0
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0                                                (0x04D8U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFF0FFFFFFF0))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_SHIFT                                (36U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_CLRMSK                               (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_DATA_ADDR_ALIGNSIZE                            (16U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_SHIFT                                (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF0000000F))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSHIFT                           (4U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS0_CODE_ADDR_ALIGNSIZE                            (16U)
+
+
+/*
+    Register RGX_CR_CDM_CONTEXT_LOAD_PDS1
+*/
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1                                                (0x04E0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__MASKFULL                           (IMG_UINT64_C(0x000000007FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_MASKFULL                                       (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_SHIFT                  (30U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_CLRMSK                 (0xBFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__PDS_SEQ_DEP_EN                     (0x40000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_SHIFT                              (29U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_PDS_SEQ_DEP_EN                                 (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_SHIFT                  (29U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_CLRMSK                 (0xDFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__USC_SEQ_DEP_EN                     (0x20000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_SHIFT                              (28U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_USC_SEQ_DEP_EN                                 (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_SHIFT                       (28U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_CLRMSK                      (0xEFFFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TARGET_EN                          (0x10000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_SHIFT                                   (27U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_CLRMSK                                  (0xF7FFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TARGET_EN                                      (0x08000000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__UNIFIED_SIZE_SHIFT                 (22U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__UNIFIED_SIZE_CLRMSK                (0xF03FFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_SHIFT                             (21U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_UNIFIED_SIZE_CLRMSK                            (0xF81FFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_SHIFT                (21U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_CLRMSK               (0xFFDFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SHARED_EN                   (0x00200000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_SHIFT                            (20U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_CLRMSK                           (0xFFEFFFFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SHARED_EN                               (0x00100000U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SIZE_SHIFT                  (12U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__COMMON_SIZE_CLRMSK                 (0xFFE00FFFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SIZE_SHIFT                              (11U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_COMMON_SIZE_CLRMSK                             (0xFFF007FFU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_SHIFT                                (7U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_TEMP_SIZE_CLRMSK                               (0xFFFFF87FU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TEMP_SIZE_SHIFT                    (7U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1__TEMPSIZE8__TEMP_SIZE_CLRMSK                   (0xFFFFF07FU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_SHIFT                                (1U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_DATA_SIZE_CLRMSK                               (0xFFFFFF81U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_SHIFT                                    (0U)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_CDM_CONTEXT_LOAD_PDS1_FENCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_CONFIG
+*/
+#define RGX_CR_MIPS_WRAPPER_CONFIG                                                  (0x0810U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_MASKFULL                                         (IMG_UINT64_C(0x000001030F01FFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_SHIFT                             (40U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_CLRMSK                            (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_FW_IDLE_ENABLE_EN                                (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_SHIFT                               (33U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_DISABLE_BOOT_EN                                  (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_SHIFT                               (32U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_L2_CACHE_OFF_EN                                  (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_OS_ID_SHIFT                                      (25U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_OS_ID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFF1FFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_SHIFT                                    (24U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_TRUSTED_EN                                       (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_SHIFT                              (16U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MIPS32                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS                          (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_MIPS_WRAPPER_CONFIG_REGBANK_BASE_ADDR_SHIFT                          (0U)
+#define RGX_CR_MIPS_WRAPPER_CONFIG_REGBANK_BASE_ADDR_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP1_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1                                             (0x0818U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP1_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2                                             (0x0820U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP2_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1                                             (0x0828U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP2_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2                                             (0x0830U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP3_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1                                             (0x0838U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP3_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2                                             (0x0840U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP4_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1                                             (0x0848U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP4_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2                                             (0x0850U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP5_CONFIG1
+*/
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1                                             (0x0858U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFF001))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_BASE_ADDR_IN_SHIFT                          (12U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_BASE_ADDR_IN_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP5_CONFIG2
+*/
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2                                             (0x0860U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_MASKFULL                                    (IMG_UINT64_C(0x000000FFFFFFF1FF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_SHIFT                              (12U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_OS_ID_SHIFT                                 (6U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_OS_ID_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFE3F))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_TRUSTED_EN                                  (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_REGION_SIZE_POW2_SHIFT                      (0U)
+#define RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_REGION_SIZE_POW2_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFE0))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS                                      (0x0868U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_MASKFULL                             (IMG_UINT64_C(0x00000001FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_SHIFT                          (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_EVENT_EN                             (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_ADDRESS_SHIFT                        (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_STATUS_ADDRESS_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR                                       (0x0870U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_MASKFULL                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_SHIFT                           (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_CLRMSK                          (0xFFFFFFFEU)
+#define RGX_CR_MIPS_ADDR_REMAP_UNMAPPED_CLEAR_EVENT_EN                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG                                         (0x0878U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MASKFULL                                (IMG_UINT64_C(0xFFFFFFF7FFFFFFBF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ADDR_OUT_SHIFT                          (36U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ADDR_OUT_CLRMSK                         (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_OS_ID_SHIFT                             (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_OS_ID_CLRMSK                            (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_BASE_ADDR_IN_SHIFT                      (12U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_BASE_ADDR_IN_CLRMSK                     (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_SHIFT                           (11U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_TRUSTED_EN                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_SHIFT                       (7U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFF87F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4KB                         (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_16KB                        (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64KB                        (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256KB                       (IMG_UINT64_C(0x0000000000000180))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_1MB                         (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4MB                         (IMG_UINT64_C(0x0000000000000280))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_16MB                        (IMG_UINT64_C(0x0000000000000300))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64MB                        (IMG_UINT64_C(0x0000000000000380))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256MB                       (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ENTRY_SHIFT                             (1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_ENTRY_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFC1))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_MODE_ENABLE_EN                          (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_READ
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ                                           (0x0880U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_MASKFULL                                  (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_ENTRY_SHIFT                               (1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_ENTRY_CLRMSK                              (0xFFFFFFC1U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_SHIFT                             (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_READ_REQUEST_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA
+*/
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA                                           (0x0888U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFF7FFFFFF81))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_SHIFT                            (36U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_CLRMSK                           (IMG_UINT64_C(0x0000000FFFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_OS_ID_SHIFT                               (32U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_OS_ID_CLRMSK                              (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_SHIFT                        (12U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFF00000FFF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_SHIFT                             (11U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_TRUSTED_EN                                (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_SHIFT                         (7U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFF87F))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_SHIFT                         (0U)
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_MODE_ENABLE_EN                            (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_ENABLE
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE                                              (0x08A0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_ENABLE_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_STATUS
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS                                              (0x08A8U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_IRQ_CLEAR
+*/
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR                                               (0x08B0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_MASKFULL                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_SHIFT                                   (0U)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_NMI_ENABLE
+*/
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE                                              (0x08B8U)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_MASKFULL                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_SHIFT                                  (0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT                                               (0x08C0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_MASKFULL                                      (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_SHIFT                                 (0U)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_DEBUG_CONFIG
+*/
+#define RGX_CR_MIPS_DEBUG_CONFIG                                                    (0x08C8U)
+#define RGX_CR_MIPS_DEBUG_CONFIG_MASKFULL                                           (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_SHIFT                          (0U)
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_CLRMSK                         (0xFFFFFFFEU)
+#define RGX_CR_MIPS_DEBUG_CONFIG_DISABLE_PROBE_DEBUG_EN                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_EXCEPTION_STATUS
+*/
+#define RGX_CR_MIPS_EXCEPTION_STATUS                                                (0x08D0U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000003F))
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_SHIFT                                 (5U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_CLRMSK                                (0xFFFFFFDFU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_SLEEP_EN                                    (0x00000020U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_SHIFT                             (4U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_CLRMSK                            (0xFFFFFFEFU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN                                (0x00000010U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_SHIFT                              (3U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_CLRMSK                             (0xFFFFFFF7U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_EXL_EN                                 (0x00000008U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_SHIFT                              (2U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_NEST_ERL_EN                                 (0x00000004U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_SHIFT                                   (1U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_EXL_EN                                      (0x00000002U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_SHIFT                                   (0U)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_CLRMSK                                  (0xFFFFFFFEU)
+#define RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN                                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_MIPS_WRAPPER_STATUS
+*/
+#define RGX_CR_MIPS_WRAPPER_STATUS                                                  (0x08E8U)
+#define RGX_CR_MIPS_WRAPPER_STATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MIPS_WRAPPER_STATUS_OUTSTANDING_REQUESTS_SHIFT                       (0U)
+#define RGX_CR_MIPS_WRAPPER_STATUS_OUTSTANDING_REQUESTS_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_XPU_BROADCAST
+*/
+#define RGX_CR_XPU_BROADCAST                                                        (0x0890U)
+#define RGX_CR_XPU_BROADCAST_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_XPU_BROADCAST_MASK_SHIFT                                             (0U)
+#define RGX_CR_XPU_BROADCAST_MASK_CLRMSK                                            (0xFFFFFE00U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAX
+*/
+#define RGX_CR_META_SP_MSLVDATAX                                                    (0x0A00U)
+#define RGX_CR_META_SP_MSLVDATAX_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAX_MSLVDATAX_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVDATAT
+*/
+#define RGX_CR_META_SP_MSLVDATAT                                                    (0x0A08U)
+#define RGX_CR_META_SP_MSLVDATAT_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVDATAT_MSLVDATAT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL0
+*/
+#define RGX_CR_META_SP_MSLVCTRL0                                                    (0x0A10U)
+#define RGX_CR_META_SP_MSLVCTRL0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_SHIFT                                         (2U)
+#define RGX_CR_META_SP_MSLVCTRL0_ADDR_CLRMSK                                        (0x00000003U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_SHIFT                                     (1U)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN                                        (0x00000002U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_SHIFT                                           (0U)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVCTRL0_RD_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVCTRL1
+*/
+#define RGX_CR_META_SP_MSLVCTRL1                                                    (0x0A18U)
+#define RGX_CR_META_SP_MSLVCTRL1_MASKFULL                                           (IMG_UINT64_C(0x00000000F7F4003F))
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_SHIFT                                 (30U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRTHREAD_CLRMSK                                (0x3FFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_SHIFT                              (29U)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_CLRMSK                             (0xDFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_LOCK2_INTERLOCK_EN                                 (0x20000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_SHIFT                             (28U)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_CLRMSK                            (0xEFFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_ATOMIC_INTERLOCK_EN                                (0x10000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_SHIFT                                 (26U)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_CLRMSK                                (0xFBFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN                                    (0x04000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_SHIFT                                 (25U)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_CLRMSK                                (0xFDFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_COREMEM_IDLE_EN                                    (0x02000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_SHIFT                                        (24U)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_CLRMSK                                       (0xFEFFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_READY_EN                                           (0x01000000U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_SHIFT                                     (21U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERRID_CLRMSK                                    (0xFF1FFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_SHIFT                                       (20U)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_CLRMSK                                      (0xFFEFFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_DEFERR_EN                                          (0x00100000U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_SHIFT                                    (18U)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_CLRMSK                                   (0xFFFBFFFFU)
+#define RGX_CR_META_SP_MSLVCTRL1_WR_ACTIVE_EN                                       (0x00040000U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_SHIFT                                       (4U)
+#define RGX_CR_META_SP_MSLVCTRL1_THREAD_CLRMSK                                      (0xFFFFFFCFU)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVCTRL1_TRANS_SIZE_CLRMSK                                  (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_SHIFT                                   (0U)
+#define RGX_CR_META_SP_MSLVCTRL1_BYTE_ROUND_CLRMSK                                  (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVHANDSHKE
+*/
+#define RGX_CR_META_SP_MSLVHANDSHKE                                                 (0x0A50U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_MASKFULL                                        (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_SHIFT                                     (2U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_INPUT_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_SHIFT                                    (0U)
+#define RGX_CR_META_SP_MSLVHANDSHKE_OUTPUT_CLRMSK                                   (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICK
+*/
+#define RGX_CR_META_SP_MSLVT0KICK                                                   (0x0A80U)
+#define RGX_CR_META_SP_MSLVT0KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT0KICK_MSLVT0KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT0KICKI
+*/
+#define RGX_CR_META_SP_MSLVT0KICKI                                                  (0x0A88U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT0KICKI_MSLVT0KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICK
+*/
+#define RGX_CR_META_SP_MSLVT1KICK                                                   (0x0A90U)
+#define RGX_CR_META_SP_MSLVT1KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT1KICK_MSLVT1KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT1KICKI
+*/
+#define RGX_CR_META_SP_MSLVT1KICKI                                                  (0x0A98U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT1KICKI_MSLVT1KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICK
+*/
+#define RGX_CR_META_SP_MSLVT2KICK                                                   (0x0AA0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT2KICK_MSLVT2KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT2KICKI
+*/
+#define RGX_CR_META_SP_MSLVT2KICKI                                                  (0x0AA8U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT2KICKI_MSLVT2KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICK
+*/
+#define RGX_CR_META_SP_MSLVT3KICK                                                   (0x0AB0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_SHIFT                                  (0U)
+#define RGX_CR_META_SP_MSLVT3KICK_MSLVT3KICK_CLRMSK                                 (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVT3KICKI
+*/
+#define RGX_CR_META_SP_MSLVT3KICKI                                                  (0x0AB8U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MASKFULL                                         (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_SHIFT                                (0U)
+#define RGX_CR_META_SP_MSLVT3KICKI_MSLVT3KICKI_CLRMSK                               (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVRST
+*/
+#define RGX_CR_META_SP_MSLVRST                                                      (0x0AC0U)
+#define RGX_CR_META_SP_MSLVRST_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVRST_SOFTRESET_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQSTATUS
+*/
+#define RGX_CR_META_SP_MSLVIRQSTATUS                                                (0x0AC8U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_SHIFT                                (3U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT3_EN                                   (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_SHIFT                                (2U)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN                                   (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQENABLE
+*/
+#define RGX_CR_META_SP_MSLVIRQENABLE                                                (0x0AD0U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_MASKFULL                                       (IMG_UINT64_C(0x000000000000000C))
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_SHIFT                                   (3U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_CLRMSK                                  (0xFFFFFFF7U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT1_EN                                      (0x00000008U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_SHIFT                                   (2U)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_META_SP_MSLVIRQENABLE_EVENT0_EN                                      (0x00000004U)
+
+
+/*
+    Register RGX_CR_META_SP_MSLVIRQLEVEL
+*/
+#define RGX_CR_META_SP_MSLVIRQLEVEL                                                 (0x0AD8U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_SHIFT                                      (0U)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_META_SP_MSLVIRQLEVEL_MODE_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE
+*/
+#define RGX_CR_MTS_SCHEDULE                                                         (0x0B00U)
+#define RGX_CR_MTS_SCHEDULE_MASKFULL                                                (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE_HOST_SHIFT                                              (8U)
+#define RGX_CR_MTS_SCHEDULE_HOST_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE_HOST_BG_TIMER                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_HOST_HOST                                               (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_SHIFT                                          (6U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_CLRMSK                                         (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT0                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT1                                           (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT2                                           (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE_PRIORITY_PRT3                                           (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_SHIFT                                           (5U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_CLRMSK                                          (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_BGCTX                                           (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_CONTEXT_INTCTX                                          (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE_TASK_SHIFT                                              (4U)
+#define RGX_CR_MTS_SCHEDULE_TASK_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED                                        (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_TASK_COUNTED                                            (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE_DM_SHIFT                                                (0U)
+#define RGX_CR_MTS_SCHEDULE_DM_CLRMSK                                               (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM0                                                  (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM1                                                  (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM2                                                  (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM3                                                  (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM4                                                  (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM5                                                  (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM6                                                  (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM7                                                  (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE_DM_DM_ALL                                               (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE1
+*/
+#define RGX_CR_MTS_SCHEDULE1                                                        (0x10B00U)
+#define RGX_CR_MTS_SCHEDULE1_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE1_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE1_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE1_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE1_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE1_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE1_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE2
+*/
+#define RGX_CR_MTS_SCHEDULE2                                                        (0x20B00U)
+#define RGX_CR_MTS_SCHEDULE2_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE2_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE2_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE2_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE2_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE2_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE2_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE3
+*/
+#define RGX_CR_MTS_SCHEDULE3                                                        (0x30B00U)
+#define RGX_CR_MTS_SCHEDULE3_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE3_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE3_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE3_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE3_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE3_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE3_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE4
+*/
+#define RGX_CR_MTS_SCHEDULE4                                                        (0x40B00U)
+#define RGX_CR_MTS_SCHEDULE4_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE4_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE4_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE4_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE4_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE4_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE4_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE5
+*/
+#define RGX_CR_MTS_SCHEDULE5                                                        (0x50B00U)
+#define RGX_CR_MTS_SCHEDULE5_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE5_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE5_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE5_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE5_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE5_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE5_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE6
+*/
+#define RGX_CR_MTS_SCHEDULE6                                                        (0x60B00U)
+#define RGX_CR_MTS_SCHEDULE6_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE6_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE6_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE6_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE6_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE6_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE6_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE7
+*/
+#define RGX_CR_MTS_SCHEDULE7                                                        (0x70B00U)
+#define RGX_CR_MTS_SCHEDULE7_MASKFULL                                               (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_MTS_SCHEDULE7_HOST_SHIFT                                             (8U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_CLRMSK                                            (0xFFFFFEFFU)
+#define RGX_CR_MTS_SCHEDULE7_HOST_BG_TIMER                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_HOST_HOST                                              (0x00000100U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_SHIFT                                         (6U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_CLRMSK                                        (0xFFFFFF3FU)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT0                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT1                                          (0x00000040U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT2                                          (0x00000080U)
+#define RGX_CR_MTS_SCHEDULE7_PRIORITY_PRT3                                          (0x000000C0U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_SHIFT                                          (5U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_BGCTX                                          (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_CONTEXT_INTCTX                                         (0x00000020U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_SHIFT                                             (4U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SCHEDULE7_TASK_NON_COUNTED                                       (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_TASK_COUNTED                                           (0x00000010U)
+#define RGX_CR_MTS_SCHEDULE7_DM_SHIFT                                               (0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_CLRMSK                                              (0xFFFFFFF0U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM0                                                 (0x00000000U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM1                                                 (0x00000001U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM2                                                 (0x00000002U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM3                                                 (0x00000003U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM4                                                 (0x00000004U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM5                                                 (0x00000005U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM6                                                 (0x00000006U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM7                                                 (0x00000007U)
+#define RGX_CR_MTS_SCHEDULE7_DM_DM_ALL                                              (0x0000000FU)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC                                           (0x0B30U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC                                           (0x0B38U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                            (0U)
+#define RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC                                          (0x0B40U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC
+*/
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC                                          (0x0B48U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL                                 (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_SHIFT                           (0U)
+#define RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK                          (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_MTS_GARTEN_WRAPPER_CONFIG
+*/
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG                                            (0x0B50U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__MASKFULL                        (IMG_UINT64_C(0x000FF0FFFFFFF701))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_MASKFULL                                   (IMG_UINT64_C(0x0000FFFFFFFFF001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT                        (44U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK                       (IMG_UINT64_C(0xFFFF0FFFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT             (44U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_CLRMSK            (IMG_UINT64_C(0xFFF00FFFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT                             (40U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_CLRMSK                            (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_ADDR_SHIFT                           (12U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_ADDR_CLRMSK                          (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PERSISTENCE_SHIFT         (9U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PERSISTENCE_CLRMSK        (IMG_UINT64_C(0xFFFFFFFFFFFFF9FF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_SHIFT        (8U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_CLRMSK       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_SLC_COHERENT_EN           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_SHIFT                            (0U)
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META                             (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_MTS                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S7_INFRA__META                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL__S7_INFRA__MTS                   (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MTS_DM0_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE                                             (0x0B58U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM0_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM1_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE                                             (0x0B60U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM1_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM2_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE                                             (0x0B68U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM2_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM3_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE                                             (0x0B70U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM3_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM4_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE                                             (0x0B78U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM4_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_DM5_INTERRUPT_ENABLE
+*/
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE                                             (0x0B80U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_MASKFULL                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_SHIFT                            (0U)
+#define RGX_CR_MTS_DM5_INTERRUPT_ENABLE_INT_ENABLE_CLRMSK                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_INTCTX
+*/
+#define RGX_CR_MTS_INTCTX                                                           (0x0B98U)
+#define RGX_CR_MTS_INTCTX_MASKFULL                                                  (IMG_UINT64_C(0x000000003FFFFFFF))
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_SHIFT                                    (22U)
+#define RGX_CR_MTS_INTCTX_DM_HOST_SCHEDULE_CLRMSK                                   (0xC03FFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_PTR_SHIFT                                              (18U)
+#define RGX_CR_MTS_INTCTX_DM_PTR_CLRMSK                                             (0xFFC3FFFFU)
+#define RGX_CR_MTS_INTCTX_THREAD_ACTIVE_SHIFT                                       (16U)
+#define RGX_CR_MTS_INTCTX_THREAD_ACTIVE_CLRMSK                                      (0xFFFCFFFFU)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_SHIFT                                   (8U)
+#define RGX_CR_MTS_INTCTX_DM_TIMER_SCHEDULE_CLRMSK                                  (0xFFFF00FFU)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_INTCTX_DM_INTERRUPT_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX
+*/
+#define RGX_CR_MTS_BGCTX                                                            (0x0BA0U)
+#define RGX_CR_MTS_BGCTX_MASKFULL                                                   (IMG_UINT64_C(0x0000000000003FFF))
+#define RGX_CR_MTS_BGCTX_DM_PTR_SHIFT                                               (10U)
+#define RGX_CR_MTS_BGCTX_DM_PTR_CLRMSK                                              (0xFFFFC3FFU)
+#define RGX_CR_MTS_BGCTX_THREAD_ACTIVE_SHIFT                                        (8U)
+#define RGX_CR_MTS_BGCTX_THREAD_ACTIVE_CLRMSK                                       (0xFFFFFCFFU)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_SHIFT                               (0U)
+#define RGX_CR_MTS_BGCTX_DM_NONCOUNTED_SCHEDULE_CLRMSK                              (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE
+*/
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE                                           (0x0BA8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_MASKFULL                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_SHIFT                                 (56U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM7_CLRMSK                                (IMG_UINT64_C(0x00FFFFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_SHIFT                                 (48U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM6_CLRMSK                                (IMG_UINT64_C(0xFF00FFFFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_SHIFT                                 (40U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM5_CLRMSK                                (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_SHIFT                                 (32U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM4_CLRMSK                                (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_SHIFT                                 (24U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM3_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_SHIFT                                 (16U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM2_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_SHIFT                                 (8U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM1_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_SHIFT                                 (0U)
+#define RGX_CR_MTS_BGCTX_COUNTED_SCHEDULE_DM0_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MTS_GPU_INT_STATUS
+*/
+#define RGX_CR_MTS_GPU_INT_STATUS                                                   (0x0BB0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_SHIFT                                      (0U)
+#define RGX_CR_MTS_GPU_INT_STATUS_STATUS_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_MTS_SCHEDULE_ENABLE
+*/
+#define RGX_CR_MTS_SCHEDULE_ENABLE                                                  (0x0BC8U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASKFULL                                         (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_SHIFT                                       (0U)
+#define RGX_CR_MTS_SCHEDULE_ENABLE_MASK_CLRMSK                                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS0_EVENT_STATUS                                                 (0x0BD8U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS0_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR                                                  (0x0BE8U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS1_EVENT_STATUS                                                 (0x10BD8U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS1_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS1_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR                                                  (0x10BE8U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS1_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS2_EVENT_STATUS                                                 (0x20BD8U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS2_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS2_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR                                                  (0x20BE8U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS2_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS3_EVENT_STATUS                                                 (0x30BD8U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS3_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS3_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR                                                  (0x30BE8U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS3_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS4_EVENT_STATUS                                                 (0x40BD8U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS4_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS4_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR                                                  (0x40BE8U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS4_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS5_EVENT_STATUS                                                 (0x50BD8U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS5_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS5_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR                                                  (0x50BE8U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS5_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS6_EVENT_STATUS                                                 (0x60BD8U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS6_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS6_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR                                                  (0x60BE8U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS6_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_STATUS
+*/
+#define RGX_CR_IRQ_OS7_EVENT_STATUS                                                 (0x70BD8U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_SHIFT                                    (0U)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS7_EVENT_STATUS_SOURCE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_IRQ_OS7_EVENT_CLEAR
+*/
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR                                                  (0x70BE8U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_SHIFT                                     (0U)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_IRQ_OS7_EVENT_CLEAR_SOURCE_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_META_BOOT
+*/
+#define RGX_CR_META_BOOT                                                            (0x0BF8U)
+#define RGX_CR_META_BOOT_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_META_BOOT_MODE_SHIFT                                                 (0U)
+#define RGX_CR_META_BOOT_MODE_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_META_BOOT_MODE_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_GARTEN_SLC
+*/
+#define RGX_CR_GARTEN_SLC                                                           (0x0BB8U)
+#define RGX_CR_GARTEN_SLC_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_SHIFT                                     (0U)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_GARTEN_SLC_FORCE_COHERENCY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_PPP
+*/
+#define RGX_CR_PPP                                                                  (0x0CD0U)
+#define RGX_CR_PPP_MASKFULL                                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CHECKSUM_SHIFT                                                   (0U)
+#define RGX_CR_PPP_CHECKSUM_CLRMSK                                                  (0x00000000U)
+
+
+#define RGX_CR_ISP_RENDER_DIR_TYPE_MASK                                             (0x00000003U)
+/*
+Top-left to bottom-right */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_TL2BR                                            (0x00000000U)
+/*
+Top-right to bottom-left */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_TR2BL                                            (0x00000001U)
+/*
+Bottom-left to top-right */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_BL2TR                                            (0x00000002U)
+/*
+Bottom-right to top-left */
+#define RGX_CR_ISP_RENDER_DIR_TYPE_BR2TL                                            (0x00000003U)
+
+
+#define RGX_CR_ISP_RENDER_MODE_TYPE_MASK                                            (0x00000003U)
+/*
+Normal render     */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_NORM                                            (0x00000000U)
+/*
+Fast 2D render    */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_FAST_2D                                         (0x00000002U)
+/*
+Fast scale render */
+#define RGX_CR_ISP_RENDER_MODE_TYPE_FAST_SCALE                                      (0x00000003U)
+
+
+/*
+    Register RGX_CR_ISP_RENDER
+*/
+#define RGX_CR_ISP_RENDER                                                           (0x0F08U)
+#define RGX_CR_ISP_RENDER_MASKFULL                                                  (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_SHIFT                           (8U)
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_CLRMSK                          (0xFFFFFEFFU)
+#define RGX_CR_ISP_RENDER_FAST_RENDER_FORCE_PROTECT_EN                              (0x00000100U)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_SHIFT                             (7U)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_CLRMSK                            (0xFFFFFF7FU)
+#define RGX_CR_ISP_RENDER_PROCESS_PROTECTED_TILES_EN                                (0x00000080U)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_SHIFT                           (6U)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_CLRMSK                          (0xFFFFFFBFU)
+#define RGX_CR_ISP_RENDER_PROCESS_UNPROTECTED_TILES_EN                              (0x00000040U)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_SHIFT                                        (5U)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_ISP_RENDER_DISABLE_EOMT_EN                                           (0x00000020U)
+#define RGX_CR_ISP_RENDER_RESUME_SHIFT                                              (4U)
+#define RGX_CR_ISP_RENDER_RESUME_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_ISP_RENDER_RESUME_EN                                                 (0x00000010U)
+#define RGX_CR_ISP_RENDER_DIR_SHIFT                                                 (2U)
+#define RGX_CR_ISP_RENDER_DIR_CLRMSK                                                (0xFFFFFFF3U)
+#define RGX_CR_ISP_RENDER_DIR_TL2BR                                                 (0x00000000U)
+#define RGX_CR_ISP_RENDER_DIR_TR2BL                                                 (0x00000004U)
+#define RGX_CR_ISP_RENDER_DIR_BL2TR                                                 (0x00000008U)
+#define RGX_CR_ISP_RENDER_DIR_BR2TL                                                 (0x0000000CU)
+#define RGX_CR_ISP_RENDER_MODE_SHIFT                                                (0U)
+#define RGX_CR_ISP_RENDER_MODE_CLRMSK                                               (0xFFFFFFFCU)
+#define RGX_CR_ISP_RENDER_MODE_NORM                                                 (0x00000000U)
+#define RGX_CR_ISP_RENDER_MODE_FAST_2D                                              (0x00000002U)
+#define RGX_CR_ISP_RENDER_MODE_FAST_SCALE                                           (0x00000003U)
+
+
+/*
+    Register RGX_CR_ISP_CTL
+*/
+#define RGX_CR_ISP_CTL                                                              (0x0F38U)
+#define RGX_CR_ISP_CTL_MASKFULL                                                     (IMG_UINT64_C(0x00000000FFFFF3FF))
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_SHIFT                                         (31U)
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_CLRMSK                                        (0x7FFFFFFFU)
+#define RGX_CR_ISP_CTL_SKIP_INIT_HDRS_EN                                            (0x80000000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_SHIFT                                             (30U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_CLRMSK                                            (0xBFFFFFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_EN                                                (0x40000000U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_SHIFT                                         (29U)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_CLRMSK                                        (0xDFFFFFFFU)
+#define RGX_CR_ISP_CTL_LINE_STYLE_PIX_EN                                            (0x20000000U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_SHIFT                                        (28U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_CLRMSK                                       (0xEFFFFFFFU)
+#define RGX_CR_ISP_CTL_PAIR_TILES_VERT_EN                                           (0x10000000U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_SHIFT                                             (27U)
+#define RGX_CR_ISP_CTL_PAIR_TILES_CLRMSK                                            (0xF7FFFFFFU)
+#define RGX_CR_ISP_CTL_PAIR_TILES_EN                                                (0x08000000U)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_SHIFT                                            (26U)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_CLRMSK                                           (0xFBFFFFFFU)
+#define RGX_CR_ISP_CTL_CREQ_BUF_EN_EN                                               (0x04000000U)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_SHIFT                                            (25U)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_CLRMSK                                           (0xFDFFFFFFU)
+#define RGX_CR_ISP_CTL_TILE_AGE_EN_EN                                               (0x02000000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_SHIFT                                    (23U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_CLRMSK                                   (0xFE7FFFFFU)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_DX9                                      (0x00000000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_DX10                                     (0x00800000U)
+#define RGX_CR_ISP_CTL_ISP_SAMPLE_POS_MODE_OGL                                      (0x01000000U)
+#define RGX_CR_ISP_CTL_NUM_TILES_PER_USC_SHIFT                                      (21U)
+#define RGX_CR_ISP_CTL_NUM_TILES_PER_USC_CLRMSK                                     (0xFF9FFFFFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_SHIFT                                           (20U)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_CLRMSK                                          (0xFFEFFFFFU)
+#define RGX_CR_ISP_CTL_DBIAS_IS_INT_EN                                              (0x00100000U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_SHIFT                                     (19U)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_CLRMSK                                    (0xFFF7FFFFU)
+#define RGX_CR_ISP_CTL_OVERLAP_CHECK_MODE_EN                                        (0x00080000U)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_SHIFT                               (18U)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_ISP_CTL_PT_UPFRONT_DEPTH_DISABLE_EN                                  (0x00040000U)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_SHIFT                                    (17U)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_CLRMSK                                   (0xFFFDFFFFU)
+#define RGX_CR_ISP_CTL_PROCESS_EMPTY_TILES_EN                                       (0x00020000U)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_SHIFT                                             (16U)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_CLRMSK                                            (0xFFFEFFFFU)
+#define RGX_CR_ISP_CTL_SAMPLE_POS_EN                                                (0x00010000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_SHIFT                                            (12U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_CLRMSK                                           (0xFFFF0FFFU)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_ONE                                         (0x00000000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TWO                                         (0x00001000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_THREE                                       (0x00002000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FOUR                                        (0x00003000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FIVE                                        (0x00004000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SIX                                         (0x00005000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SEVEN                                       (0x00006000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_EIGHT                                       (0x00007000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_NINE                                        (0x00008000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TEN                                         (0x00009000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_ELEVEN                                      (0x0000A000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_TWELVE                                      (0x0000B000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_THIRTEEN                                    (0x0000C000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FOURTEEN                                    (0x0000D000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_FIFTEEN                                     (0x0000E000U)
+#define RGX_CR_ISP_CTL_PIPE_ENABLE_PIPE_SIXTEEN                                     (0x0000F000U)
+#define RGX_CR_ISP_CTL_VALID_ID_SHIFT                                               (4U)
+#define RGX_CR_ISP_CTL_VALID_ID_CLRMSK                                              (0xFFFFFC0FU)
+#define RGX_CR_ISP_CTL_UPASS_START_SHIFT                                            (0U)
+#define RGX_CR_ISP_CTL_UPASS_START_CLRMSK                                           (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_ISP_STATUS
+*/
+#define RGX_CR_ISP_STATUS                                                           (0x1038U)
+#define RGX_CR_ISP_STATUS_MASKFULL                                                  (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_SHIFT                                           (2U)
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_CLRMSK                                          (0xFFFFFFFBU)
+#define RGX_CR_ISP_STATUS_SPLIT_MAX_EN                                              (0x00000004U)
+#define RGX_CR_ISP_STATUS_ACTIVE_SHIFT                                              (1U)
+#define RGX_CR_ISP_STATUS_ACTIVE_CLRMSK                                             (0xFFFFFFFDU)
+#define RGX_CR_ISP_STATUS_ACTIVE_EN                                                 (0x00000002U)
+#define RGX_CR_ISP_STATUS_EOR_SHIFT                                                 (0U)
+#define RGX_CR_ISP_STATUS_EOR_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_ISP_STATUS_EOR_EN                                                    (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_ISP_XTP_RESUME, with 64 repeats
+*/
+#define RGX_CR_ISP_XTP_RESUME_REPEATCOUNT                                           (64U)
+/*
+    Register RGX_CR_ISP_XTP_RESUME0
+*/
+#define RGX_CR_ISP_XTP_RESUME0                                                      (0x3A00U)
+#define RGX_CR_ISP_XTP_RESUME0_MASKFULL                                             (IMG_UINT64_C(0x00000000003FF3FF))
+#define RGX_CR_ISP_XTP_RESUME0_TILE_X_SHIFT                                         (12U)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_X_CLRMSK                                        (0xFFC00FFFU)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_Y_SHIFT                                         (0U)
+#define RGX_CR_ISP_XTP_RESUME0_TILE_Y_CLRMSK                                        (0xFFFFFC00U)
+
+
+/*
+    Register group: RGX_CR_ISP_XTP_STORE, with 32 repeats
+*/
+#define RGX_CR_ISP_XTP_STORE_REPEATCOUNT                                            (32U)
+/*
+    Register RGX_CR_ISP_XTP_STORE0
+*/
+#define RGX_CR_ISP_XTP_STORE0                                                       (0x3C00U)
+#define RGX_CR_ISP_XTP_STORE0_MASKFULL                                              (IMG_UINT64_C(0x000000007F3FF3FF))
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_SHIFT                                          (30U)
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_CLRMSK                                         (0xBFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_ACTIVE_EN                                             (0x40000000U)
+#define RGX_CR_ISP_XTP_STORE0_EOR_SHIFT                                             (29U)
+#define RGX_CR_ISP_XTP_STORE0_EOR_CLRMSK                                            (0xDFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_EOR_EN                                                (0x20000000U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_SHIFT                                       (28U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_CLRMSK                                      (0xEFFFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_LAST_EN                                          (0x10000000U)
+#define RGX_CR_ISP_XTP_STORE0_MT_SHIFT                                              (24U)
+#define RGX_CR_ISP_XTP_STORE0_MT_CLRMSK                                             (0xF0FFFFFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_X_SHIFT                                          (12U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_X_CLRMSK                                         (0xFFC00FFFU)
+#define RGX_CR_ISP_XTP_STORE0_TILE_Y_SHIFT                                          (0U)
+#define RGX_CR_ISP_XTP_STORE0_TILE_Y_CLRMSK                                         (0xFFFFFC00U)
+
+
+/*
+    Register group: RGX_CR_BIF_CAT_BASE, with 8 repeats
+*/
+#define RGX_CR_BIF_CAT_BASE_REPEATCOUNT                                             (8U)
+/*
+    Register RGX_CR_BIF_CAT_BASE0
+*/
+#define RGX_CR_BIF_CAT_BASE0                                                        (0x1200U)
+#define RGX_CR_BIF_CAT_BASE0_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE1
+*/
+#define RGX_CR_BIF_CAT_BASE1                                                        (0x1208U)
+#define RGX_CR_BIF_CAT_BASE1_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE1_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE1_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE1_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE1_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE2
+*/
+#define RGX_CR_BIF_CAT_BASE2                                                        (0x1210U)
+#define RGX_CR_BIF_CAT_BASE2_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE2_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE2_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE2_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE2_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE3
+*/
+#define RGX_CR_BIF_CAT_BASE3                                                        (0x1218U)
+#define RGX_CR_BIF_CAT_BASE3_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE3_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE3_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE3_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE3_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE4
+*/
+#define RGX_CR_BIF_CAT_BASE4                                                        (0x1220U)
+#define RGX_CR_BIF_CAT_BASE4_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE4_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE4_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE4_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE4_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE5
+*/
+#define RGX_CR_BIF_CAT_BASE5                                                        (0x1228U)
+#define RGX_CR_BIF_CAT_BASE5_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE5_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE5_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE5_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE5_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE6
+*/
+#define RGX_CR_BIF_CAT_BASE6                                                        (0x1230U)
+#define RGX_CR_BIF_CAT_BASE6_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE6_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE6_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE6_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE6_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE7
+*/
+#define RGX_CR_BIF_CAT_BASE7                                                        (0x1238U)
+#define RGX_CR_BIF_CAT_BASE7_MASKFULL                                               (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_BIF_CAT_BASE7_ADDR_SHIFT                                             (12U)
+#define RGX_CR_BIF_CAT_BASE7_ADDR_CLRMSK                                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_CAT_BASE7_ADDR_ALIGNSHIFT                                        (12U)
+#define RGX_CR_BIF_CAT_BASE7_ADDR_ALIGNSIZE                                         (4096U)
+
+
+/*
+    Register RGX_CR_BIF_CAT_BASE_INDEX
+*/
+#define RGX_CR_BIF_CAT_BASE_INDEX                                                   (0x1240U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_MASKFULL                                          (IMG_UINT64_C(0x00070707073F0707))
+#define RGX_CR_BIF_CAT_BASE_INDEX_RVTX_SHIFT                                        (48U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_RVTX_CLRMSK                                       (IMG_UINT64_C(0xFFF8FFFFFFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_RAY_SHIFT                                         (40U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_RAY_CLRMSK                                        (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_HOST_SHIFT                                        (32U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_HOST_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFF8FFFFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TLA_SHIFT                                         (24U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TLA_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFF8FFFFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TDM_SHIFT                                         (19U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TDM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFC7FFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_CDM_SHIFT                                         (16U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_CDM_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFF8FFFF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_PIXEL_SHIFT                                       (8U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_PIXEL_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFF8FF))
+#define RGX_CR_BIF_CAT_BASE_INDEX_TA_SHIFT                                          (0U)
+#define RGX_CR_BIF_CAT_BASE_INDEX_TA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF8))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_VCE0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0                                                 (0x1248U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_MASKFULL                                        (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_INIT_PAGE_SHIFT                                 (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_INIT_PAGE_CLRMSK                                (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_ADDR_SHIFT                                      (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_SHIFT                                      (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_WRAP_EN                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_SHIFT                                     (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE0_VALID_EN                                        (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_TE0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_TE0                                                  (0x1250U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_MASKFULL                                         (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_INIT_PAGE_SHIFT                                  (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_INIT_PAGE_CLRMSK                                 (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_ADDR_SHIFT                                       (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_ADDR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_SHIFT                                       (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_WRAP_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_SHIFT                                      (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_TE0_VALID_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_ALIST0
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0                                               (0x1260U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_MASKFULL                                      (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_INIT_PAGE_SHIFT                               (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_INIT_PAGE_CLRMSK                              (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_ADDR_SHIFT                                    (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_SHIFT                                    (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_WRAP_EN                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_SHIFT                                   (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST0_VALID_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_VCE1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1                                                 (0x1268U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_MASKFULL                                        (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_INIT_PAGE_SHIFT                                 (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_INIT_PAGE_CLRMSK                                (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_ADDR_SHIFT                                      (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_SHIFT                                      (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_WRAP_EN                                         (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_SHIFT                                     (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_VCE1_VALID_EN                                        (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_TE1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_TE1                                                  (0x1270U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_MASKFULL                                         (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_INIT_PAGE_SHIFT                                  (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_INIT_PAGE_CLRMSK                                 (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_ADDR_SHIFT                                       (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_ADDR_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_SHIFT                                       (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_WRAP_EN                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_SHIFT                                      (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_TE1_VALID_EN                                         (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_PM_CAT_BASE_ALIST1
+*/
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1                                               (0x1280U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_MASKFULL                                      (IMG_UINT64_C(0x0FFFFFFFFFFFF003))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_INIT_PAGE_SHIFT                               (40U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_INIT_PAGE_CLRMSK                              (IMG_UINT64_C(0xF00000FFFFFFFFFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_ADDR_SHIFT                                    (12U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_ADDR_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_SHIFT                                    (1U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_WRAP_EN                                       (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_SHIFT                                   (0U)
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_BIF_PM_CAT_BASE_ALIST1_VALID_EN                                      (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_BIF_MMU_ENTRY_STATUS
+*/
+#define RGX_CR_BIF_MMU_ENTRY_STATUS                                                 (0x1288U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF0F3))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_SHIFT                                   (12U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_CLRMSK                                  (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_SHIFT                                  (4U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFF0F))
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_DATA_TYPE_SHIFT                                 (0U)
+#define RGX_CR_BIF_MMU_ENTRY_STATUS_DATA_TYPE_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+/*
+    Register RGX_CR_BIF_MMU_ENTRY
+*/
+#define RGX_CR_BIF_MMU_ENTRY                                                        (0x1290U)
+#define RGX_CR_BIF_MMU_ENTRY_MASKFULL                                               (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_SHIFT                                           (1U)
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_CLRMSK                                          (0xFFFFFFFDU)
+#define RGX_CR_BIF_MMU_ENTRY_ENABLE_EN                                              (0x00000002U)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_SHIFT                                          (0U)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_BIF_MMU_ENTRY_PENDING_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_CTRL_INVAL
+*/
+#define RGX_CR_BIF_CTRL_INVAL                                                       (0x12A0U)
+#define RGX_CR_BIF_CTRL_INVAL_MASKFULL                                              (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_SHIFT                                            (3U)
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_BIF_CTRL_INVAL_TLB1_EN                                               (0x00000008U)
+#define RGX_CR_BIF_CTRL_INVAL_PC_SHIFT                                              (2U)
+#define RGX_CR_BIF_CTRL_INVAL_PC_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_BIF_CTRL_INVAL_PC_EN                                                 (0x00000004U)
+#define RGX_CR_BIF_CTRL_INVAL_PD_SHIFT                                              (1U)
+#define RGX_CR_BIF_CTRL_INVAL_PD_CLRMSK                                             (0xFFFFFFFDU)
+#define RGX_CR_BIF_CTRL_INVAL_PD_EN                                                 (0x00000002U)
+#define RGX_CR_BIF_CTRL_INVAL_PT_SHIFT                                              (0U)
+#define RGX_CR_BIF_CTRL_INVAL_PT_CLRMSK                                             (0xFFFFFFFEU)
+#define RGX_CR_BIF_CTRL_INVAL_PT_EN                                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_CTRL
+*/
+#define RGX_CR_BIF_CTRL                                                             (0x12A8U)
+#define RGX_CR_BIF_CTRL__XE_MEM__MASKFULL                                           (IMG_UINT64_C(0x000000000000033F))
+#define RGX_CR_BIF_CTRL_MASKFULL                                                    (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_SHIFT                                (9U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_CLRMSK                               (0xFFFFFDFFU)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_CPU_EN                                   (0x00000200U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_SHIFT                               (8U)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_CLRMSK                              (0xFFFFFEFFU)
+#define RGX_CR_BIF_CTRL__XE_MEM__PAUSE_MMU_BIF4_EN                                  (0x00000100U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_SHIFT                               (7U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_CLRMSK                              (0xFFFFFF7FU)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_QUEUE_BYPASS_EN                                  (0x00000080U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_SHIFT                              (6U)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_BIF_CTRL_ENABLE_MMU_AUTO_PREFETCH_EN                                 (0x00000040U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_SHIFT                                        (5U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_CLRMSK                                       (0xFFFFFFDFU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF3_EN                                           (0x00000020U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_SHIFT                                        (4U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_CLRMSK                                       (0xFFFFFFEFU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF2_EN                                           (0x00000010U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_SHIFT                                            (3U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_BIF_CTRL_PAUSE_BIF1_EN                                               (0x00000008U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_SHIFT                                          (2U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_PM_EN                                             (0x00000004U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_SHIFT                                        (1U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_CLRMSK                                       (0xFFFFFFFDU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF1_EN                                           (0x00000002U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_SHIFT                                        (0U)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_BIF_CTRL_PAUSE_MMU_BIF0_EN                                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK0_MMU_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS                                           (0x12B0U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_MASKFULL                                  (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT                            (12U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK                           (0xFFFF0FFFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT                           (8U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK                          (0xFFFFF8FFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT                           (5U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK                          (0xFFFFFF9FU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_SHIFT                            (4U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN                               (0x00000010U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_SHIFT                    (2U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN                       (0x00000004U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_SHIFT                               (0U)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK0_REQ_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS                                           (0x12B8U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__MASKFULL                         (IMG_UINT64_C(0x001FFFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_MASKFULL                                  (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_SHIFT                        (52U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_CLRMSK                       (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN                           (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_SHIFT                                 (50U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_CLRMSK                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT                     (46U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK                    (IMG_UINT64_C(0xFFF03FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT                              (44U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK                             (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT                              (40U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT                     (40U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK                    (IMG_UINT64_C(0xFFFFC0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT                             (4U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK                            (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT                        (4U)
+#define RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSIZE                         (16U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK1_MMU_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS                                           (0x12C0U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_MASKFULL                                  (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_CAT_BASE_SHIFT                            (12U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_CAT_BASE_CLRMSK                           (0xFFFF0FFFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_PAGE_SIZE_SHIFT                           (8U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_PAGE_SIZE_CLRMSK                          (0xFFFFF8FFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_DATA_TYPE_SHIFT                           (5U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_DATA_TYPE_CLRMSK                          (0xFFFFFF9FU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_SHIFT                            (4U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_RO_EN                               (0x00000010U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_SHIFT                    (2U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                   (0xFFFFFFFBU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_PM_META_RO_EN                       (0x00000004U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_SHIFT                               (0U)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_BIF_FAULT_BANK1_MMU_STATUS_FAULT_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_BANK1_REQ_STATUS
+*/
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS                                           (0x12C8U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_MASKFULL                                  (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_SHIFT                                 (50U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_CLRMSK                                (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_RNW_EN                                    (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_SB_SHIFT                              (44U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_SB_CLRMSK                             (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_ID_SHIFT                              (40U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_TAG_ID_CLRMSK                             (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_SHIFT                             (4U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_CLRMSK                            (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_ALIGNSHIFT                        (4U)
+#define RGX_CR_BIF_FAULT_BANK1_REQ_STATUS_ADDRESS_ALIGNSIZE                         (16U)
+
+
+/*
+    Register RGX_CR_BIF_MMU_STATUS
+*/
+#define RGX_CR_BIF_MMU_STATUS                                                       (0x12D0U)
+#define RGX_CR_BIF_MMU_STATUS__XE_MEM__MASKFULL                                     (IMG_UINT64_C(0x000000001FFFFFF7))
+#define RGX_CR_BIF_MMU_STATUS_MASKFULL                                              (IMG_UINT64_C(0x000000001FFFFFF7))
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_SHIFT                                        (28U)
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_CLRMSK                                       (0xEFFFFFFFU)
+#define RGX_CR_BIF_MMU_STATUS_PM_FAULT_EN                                           (0x10000000U)
+#define RGX_CR_BIF_MMU_STATUS_PC_DATA_SHIFT                                         (20U)
+#define RGX_CR_BIF_MMU_STATUS_PC_DATA_CLRMSK                                        (0xF00FFFFFU)
+#define RGX_CR_BIF_MMU_STATUS_PD_DATA_SHIFT                                         (12U)
+#define RGX_CR_BIF_MMU_STATUS_PD_DATA_CLRMSK                                        (0xFFF00FFFU)
+#define RGX_CR_BIF_MMU_STATUS_PT_DATA_SHIFT                                         (4U)
+#define RGX_CR_BIF_MMU_STATUS_PT_DATA_CLRMSK                                        (0xFFFFF00FU)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_SHIFT                                         (2U)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_CLRMSK                                        (0xFFFFFFFBU)
+#define RGX_CR_BIF_MMU_STATUS_STALLED_EN                                            (0x00000004U)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_SHIFT                                          (1U)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_BIF_MMU_STATUS_PAUSED_EN                                             (0x00000002U)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_SHIFT                                            (0U)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_BIF_MMU_STATUS_BUSY_EN                                               (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_BIF_TILING_CFG, with 8 repeats
+*/
+#define RGX_CR_BIF_TILING_CFG_REPEATCOUNT                                           (8U)
+/*
+    Register RGX_CR_BIF_TILING_CFG0
+*/
+#define RGX_CR_BIF_TILING_CFG0                                                      (0x12D8U)
+#define RGX_CR_BIF_TILING_CFG0_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG0_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG1
+*/
+#define RGX_CR_BIF_TILING_CFG1                                                      (0x12E0U)
+#define RGX_CR_BIF_TILING_CFG1_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG1_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG2
+*/
+#define RGX_CR_BIF_TILING_CFG2                                                      (0x12E8U)
+#define RGX_CR_BIF_TILING_CFG2_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG2_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG3
+*/
+#define RGX_CR_BIF_TILING_CFG3                                                      (0x12F0U)
+#define RGX_CR_BIF_TILING_CFG3_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG3_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG4
+*/
+#define RGX_CR_BIF_TILING_CFG4                                                      (0x12F8U)
+#define RGX_CR_BIF_TILING_CFG4_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG4_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG5
+*/
+#define RGX_CR_BIF_TILING_CFG5                                                      (0x1300U)
+#define RGX_CR_BIF_TILING_CFG5_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG5_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG6
+*/
+#define RGX_CR_BIF_TILING_CFG6                                                      (0x1308U)
+#define RGX_CR_BIF_TILING_CFG6_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG6_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_TILING_CFG7
+*/
+#define RGX_CR_BIF_TILING_CFG7                                                      (0x1310U)
+#define RGX_CR_BIF_TILING_CFG7_MASKFULL                                             (IMG_UINT64_C(0xFFFFFFFF0FFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_XSTRIDE_SHIFT                                        (61U)
+#define RGX_CR_BIF_TILING_CFG7_XSTRIDE_CLRMSK                                       (IMG_UINT64_C(0x1FFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_SHIFT                                         (60U)
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_CLRMSK                                        (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_ENABLE_EN                                            (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_SHIFT                                    (32U)
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xF0000000FFFFFFFF))
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSIZE                                (4096U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_SHIFT                                    (0U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFF0000000))
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSHIFT                               (12U)
+#define RGX_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_BIF_READS_EXT_STATUS
+*/
+#define RGX_CR_BIF_READS_EXT_STATUS                                                 (0x1320U)
+#define RGX_CR_BIF_READS_EXT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_BIF_READS_EXT_STATUS_MMU_SHIFT                                       (16U)
+#define RGX_CR_BIF_READS_EXT_STATUS_MMU_CLRMSK                                      (0xF000FFFFU)
+#define RGX_CR_BIF_READS_EXT_STATUS_BANK1_SHIFT                                     (0U)
+#define RGX_CR_BIF_READS_EXT_STATUS_BANK1_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_READS_INT_STATUS
+*/
+#define RGX_CR_BIF_READS_INT_STATUS                                                 (0x1328U)
+#define RGX_CR_BIF_READS_INT_STATUS_MASKFULL                                        (IMG_UINT64_C(0x0000000007FFFFFF))
+#define RGX_CR_BIF_READS_INT_STATUS_MMU_SHIFT                                       (16U)
+#define RGX_CR_BIF_READS_INT_STATUS_MMU_CLRMSK                                      (0xF800FFFFU)
+#define RGX_CR_BIF_READS_INT_STATUS_BANK1_SHIFT                                     (0U)
+#define RGX_CR_BIF_READS_INT_STATUS_BANK1_CLRMSK                                    (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_READS_INT_STATUS
+*/
+#define RGX_CR_BIFPM_READS_INT_STATUS                                               (0x1330U)
+#define RGX_CR_BIFPM_READS_INT_STATUS_MASKFULL                                      (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIFPM_READS_INT_STATUS_BANK0_SHIFT                                   (0U)
+#define RGX_CR_BIFPM_READS_INT_STATUS_BANK0_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_READS_EXT_STATUS
+*/
+#define RGX_CR_BIFPM_READS_EXT_STATUS                                               (0x1338U)
+#define RGX_CR_BIFPM_READS_EXT_STATUS_MASKFULL                                      (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIFPM_READS_EXT_STATUS_BANK0_SHIFT                                   (0U)
+#define RGX_CR_BIFPM_READS_EXT_STATUS_BANK0_CLRMSK                                  (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIFPM_STATUS_MMU
+*/
+#define RGX_CR_BIFPM_STATUS_MMU                                                     (0x1350U)
+#define RGX_CR_BIFPM_STATUS_MMU_MASKFULL                                            (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIFPM_STATUS_MMU_REQUESTS_SHIFT                                      (0U)
+#define RGX_CR_BIFPM_STATUS_MMU_REQUESTS_CLRMSK                                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_STATUS_MMU
+*/
+#define RGX_CR_BIF_STATUS_MMU                                                       (0x1358U)
+#define RGX_CR_BIF_STATUS_MMU_MASKFULL                                              (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_BIF_STATUS_MMU_REQUESTS_SHIFT                                        (0U)
+#define RGX_CR_BIF_STATUS_MMU_REQUESTS_CLRMSK                                       (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_FAULT_READ
+*/
+#define RGX_CR_BIF_FAULT_READ                                                       (0x13E0U)
+#define RGX_CR_BIF_FAULT_READ_MASKFULL                                              (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_SHIFT                                         (4U)
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_ALIGNSHIFT                                    (4U)
+#define RGX_CR_BIF_FAULT_READ_ADDRESS_ALIGNSIZE                                     (16U)
+
+
+/*
+    Register RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS
+*/
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS                                     (0x1430U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_MASKFULL                            (IMG_UINT64_C(0x000000000000F775))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT                      (12U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK                     (0xFFFF0FFFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT                     (8U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK                    (0xFFFFF8FFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT                     (5U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK                    (0xFFFFFF9FU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_SHIFT                      (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_CLRMSK                     (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN                         (0x00000010U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_SHIFT              (2U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_CLRMSK             (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN                 (0x00000004U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_SHIFT                         (0U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS_FAULT_EN                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS
+*/
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS                                     (0x1438U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_MASKFULL                            (IMG_UINT64_C(0x0007FFFFFFFFFFF0))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_SHIFT                           (50U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_CLRMSK                          (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN                              (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT                        (44U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK                       (IMG_UINT64_C(0xFFFC0FFFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT                        (40U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK                       (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT                       (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK                      (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT                  (4U)
+#define RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSIZE                   (16U)
+
+
+/*
+    Register RGX_CR_TFBC_COMPRESSION_CONTROL
+*/
+#define RGX_CR_TFBC_COMPRESSION_CONTROL                                             (0x14A0U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_MASKFULL                                    (IMG_UINT64_C(0x00000000000001FF))
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_SHIFT            (8U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_CLRMSK           (0xFFFFFEFFU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN               (0x00000100U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_SHIFT                        (7U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN                           (0x00000080U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_SHIFT_SHIFT                         (4U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_SHIFT_CLRMSK                        (0xFFFFFF8FU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_SHIFT                        (3U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_CLRMSK                       (0xFFFFFFF7U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_QUALITY_ENABLE_EN                           (0x00000008U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT                                (1U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK                               (0xFFFFFFF9U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_DEFAULT                              (0x00000000U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD_AND_CORRELATION  (0x00000002U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_TFBC_DELTA_STANDARD                  (0x00000004U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_RESERVED                             (0x00000006U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT                         (0U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_0                       (0x00000000U)
+#define RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_GROUP_1                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_MCU_FENCE
+*/
+#define RGX_CR_MCU_FENCE                                                            (0x1740U)
+#define RGX_CR_MCU_FENCE_MASKFULL                                                   (IMG_UINT64_C(0x000007FFFFFFFFE0))
+#define RGX_CR_MCU_FENCE_DM_SHIFT                                                   (40U)
+#define RGX_CR_MCU_FENCE_DM_CLRMSK                                                  (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_MCU_FENCE_DM_VERTEX                                                  (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_MCU_FENCE_DM_PIXEL                                                   (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_MCU_FENCE_DM_COMPUTE                                                 (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_MCU_FENCE_DM_RAY_VERTEX                                              (IMG_UINT64_C(0x0000030000000000))
+#define RGX_CR_MCU_FENCE_DM_RAY                                                     (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_MCU_FENCE_DM_FASTRENDER                                              (IMG_UINT64_C(0x0000050000000000))
+#define RGX_CR_MCU_FENCE_ADDR_SHIFT                                                 (5U)
+#define RGX_CR_MCU_FENCE_ADDR_CLRMSK                                                (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_CR_MCU_FENCE_ADDR_ALIGNSHIFT                                            (5U)
+#define RGX_CR_MCU_FENCE_ADDR_ALIGNSIZE                                             (32U)
+
+
+/*
+    Register group: RGX_CR_SCRATCH, with 16 repeats
+*/
+#define RGX_CR_SCRATCH_REPEATCOUNT                                                  (16U)
+/*
+    Register RGX_CR_SCRATCH0
+*/
+#define RGX_CR_SCRATCH0                                                             (0x1A00U)
+#define RGX_CR_SCRATCH0_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH0_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH0_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH1
+*/
+#define RGX_CR_SCRATCH1                                                             (0x1A08U)
+#define RGX_CR_SCRATCH1_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH1_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH1_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH2
+*/
+#define RGX_CR_SCRATCH2                                                             (0x1A10U)
+#define RGX_CR_SCRATCH2_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH2_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH2_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH3
+*/
+#define RGX_CR_SCRATCH3                                                             (0x1A18U)
+#define RGX_CR_SCRATCH3_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH3_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH3_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH4
+*/
+#define RGX_CR_SCRATCH4                                                             (0x1A20U)
+#define RGX_CR_SCRATCH4_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH4_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH4_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH5
+*/
+#define RGX_CR_SCRATCH5                                                             (0x1A28U)
+#define RGX_CR_SCRATCH5_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH5_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH5_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH6
+*/
+#define RGX_CR_SCRATCH6                                                             (0x1A30U)
+#define RGX_CR_SCRATCH6_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH6_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH6_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH7
+*/
+#define RGX_CR_SCRATCH7                                                             (0x1A38U)
+#define RGX_CR_SCRATCH7_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH7_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH7_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH8
+*/
+#define RGX_CR_SCRATCH8                                                             (0x1A40U)
+#define RGX_CR_SCRATCH8_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH8_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH8_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH9
+*/
+#define RGX_CR_SCRATCH9                                                             (0x1A48U)
+#define RGX_CR_SCRATCH9_MASKFULL                                                    (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH9_DATA_SHIFT                                                  (0U)
+#define RGX_CR_SCRATCH9_DATA_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH10
+*/
+#define RGX_CR_SCRATCH10                                                            (0x1A50U)
+#define RGX_CR_SCRATCH10_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH10_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH10_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH11
+*/
+#define RGX_CR_SCRATCH11                                                            (0x1A58U)
+#define RGX_CR_SCRATCH11_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH11_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH11_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH12
+*/
+#define RGX_CR_SCRATCH12                                                            (0x1A60U)
+#define RGX_CR_SCRATCH12_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH12_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH12_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH13
+*/
+#define RGX_CR_SCRATCH13                                                            (0x1A68U)
+#define RGX_CR_SCRATCH13_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH13_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH13_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH14
+*/
+#define RGX_CR_SCRATCH14                                                            (0x1A70U)
+#define RGX_CR_SCRATCH14_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH14_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH14_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register RGX_CR_SCRATCH15
+*/
+#define RGX_CR_SCRATCH15                                                            (0x1A78U)
+#define RGX_CR_SCRATCH15_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SCRATCH15_DATA_SHIFT                                                 (0U)
+#define RGX_CR_SCRATCH15_DATA_CLRMSK                                                (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_OS0_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS0_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS0_SCRATCH0
+*/
+#define RGX_CR_OS0_SCRATCH0                                                         (0x1A80U)
+#define RGX_CR_OS0_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH1
+*/
+#define RGX_CR_OS0_SCRATCH1                                                         (0x1A88U)
+#define RGX_CR_OS0_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS0_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH2
+*/
+#define RGX_CR_OS0_SCRATCH2                                                         (0x1A90U)
+#define RGX_CR_OS0_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS0_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS0_SCRATCH3
+*/
+#define RGX_CR_OS0_SCRATCH3                                                         (0x1A98U)
+#define RGX_CR_OS0_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS0_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS0_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS1_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS1_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS1_SCRATCH0
+*/
+#define RGX_CR_OS1_SCRATCH0                                                         (0x11A80U)
+#define RGX_CR_OS1_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH1
+*/
+#define RGX_CR_OS1_SCRATCH1                                                         (0x11A88U)
+#define RGX_CR_OS1_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS1_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH2
+*/
+#define RGX_CR_OS1_SCRATCH2                                                         (0x11A90U)
+#define RGX_CR_OS1_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS1_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS1_SCRATCH3
+*/
+#define RGX_CR_OS1_SCRATCH3                                                         (0x11A98U)
+#define RGX_CR_OS1_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS1_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS1_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS2_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS2_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS2_SCRATCH0
+*/
+#define RGX_CR_OS2_SCRATCH0                                                         (0x21A80U)
+#define RGX_CR_OS2_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH1
+*/
+#define RGX_CR_OS2_SCRATCH1                                                         (0x21A88U)
+#define RGX_CR_OS2_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS2_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH2
+*/
+#define RGX_CR_OS2_SCRATCH2                                                         (0x21A90U)
+#define RGX_CR_OS2_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS2_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS2_SCRATCH3
+*/
+#define RGX_CR_OS2_SCRATCH3                                                         (0x21A98U)
+#define RGX_CR_OS2_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS2_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS2_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS3_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS3_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS3_SCRATCH0
+*/
+#define RGX_CR_OS3_SCRATCH0                                                         (0x31A80U)
+#define RGX_CR_OS3_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH1
+*/
+#define RGX_CR_OS3_SCRATCH1                                                         (0x31A88U)
+#define RGX_CR_OS3_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS3_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH2
+*/
+#define RGX_CR_OS3_SCRATCH2                                                         (0x31A90U)
+#define RGX_CR_OS3_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS3_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS3_SCRATCH3
+*/
+#define RGX_CR_OS3_SCRATCH3                                                         (0x31A98U)
+#define RGX_CR_OS3_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS3_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS3_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS4_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS4_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS4_SCRATCH0
+*/
+#define RGX_CR_OS4_SCRATCH0                                                         (0x41A80U)
+#define RGX_CR_OS4_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH1
+*/
+#define RGX_CR_OS4_SCRATCH1                                                         (0x41A88U)
+#define RGX_CR_OS4_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS4_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH2
+*/
+#define RGX_CR_OS4_SCRATCH2                                                         (0x41A90U)
+#define RGX_CR_OS4_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS4_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS4_SCRATCH3
+*/
+#define RGX_CR_OS4_SCRATCH3                                                         (0x41A98U)
+#define RGX_CR_OS4_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS4_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS4_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS5_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS5_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS5_SCRATCH0
+*/
+#define RGX_CR_OS5_SCRATCH0                                                         (0x51A80U)
+#define RGX_CR_OS5_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH1
+*/
+#define RGX_CR_OS5_SCRATCH1                                                         (0x51A88U)
+#define RGX_CR_OS5_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS5_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH2
+*/
+#define RGX_CR_OS5_SCRATCH2                                                         (0x51A90U)
+#define RGX_CR_OS5_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS5_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS5_SCRATCH3
+*/
+#define RGX_CR_OS5_SCRATCH3                                                         (0x51A98U)
+#define RGX_CR_OS5_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS5_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS5_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS6_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS6_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS6_SCRATCH0
+*/
+#define RGX_CR_OS6_SCRATCH0                                                         (0x61A80U)
+#define RGX_CR_OS6_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH1
+*/
+#define RGX_CR_OS6_SCRATCH1                                                         (0x61A88U)
+#define RGX_CR_OS6_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS6_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH2
+*/
+#define RGX_CR_OS6_SCRATCH2                                                         (0x61A90U)
+#define RGX_CR_OS6_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS6_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS6_SCRATCH3
+*/
+#define RGX_CR_OS6_SCRATCH3                                                         (0x61A98U)
+#define RGX_CR_OS6_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS6_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS6_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register group: RGX_CR_OS7_SCRATCH, with 2 repeats
+*/
+#define RGX_CR_OS7_SCRATCH_REPEATCOUNT                                              (2U)
+/*
+    Register RGX_CR_OS7_SCRATCH0
+*/
+#define RGX_CR_OS7_SCRATCH0                                                         (0x71A80U)
+#define RGX_CR_OS7_SCRATCH0_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH0_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH0_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH1
+*/
+#define RGX_CR_OS7_SCRATCH1                                                         (0x71A88U)
+#define RGX_CR_OS7_SCRATCH1_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_OS7_SCRATCH1_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH1_DATA_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH2
+*/
+#define RGX_CR_OS7_SCRATCH2                                                         (0x71A90U)
+#define RGX_CR_OS7_SCRATCH2_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS7_SCRATCH2_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH2_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_OS7_SCRATCH3
+*/
+#define RGX_CR_OS7_SCRATCH3                                                         (0x71A98U)
+#define RGX_CR_OS7_SCRATCH3_MASKFULL                                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_OS7_SCRATCH3_DATA_SHIFT                                              (0U)
+#define RGX_CR_OS7_SCRATCH3_DATA_CLRMSK                                             (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_SPFILTER_SIGNAL_DESCR
+*/
+#define RGX_CR_SPFILTER_SIGNAL_DESCR                                                (0x2700U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MASKFULL                                       (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_SHIFT                                     (0U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_CLRMSK                                    (0xFFFF0000U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_ALIGNSHIFT                                (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_SIZE_ALIGNSIZE                                 (16U)
+
+
+/*
+    Register RGX_CR_SPFILTER_SIGNAL_DESCR_MIN
+*/
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN                                            (0x2708U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_MASKFULL                                   (IMG_UINT64_C(0x000000FFFFFFFFF0))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_SHIFT                                 (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_CLRMSK                                (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_ALIGNSHIFT                            (4U)
+#define RGX_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_ALIGNSIZE                             (16U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_ADDR_REMAP_CONFIG, with 16 repeats
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG_REPEATCOUNT                                 (16U)
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG0
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0                                            (0x3000U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG1
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1                                            (0x3008U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG2
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2                                            (0x3010U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG3
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3                                            (0x3018U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG4
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4                                            (0x3020U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG5
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5                                            (0x3028U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG6
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6                                            (0x3030U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG7
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7                                            (0x3038U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG8
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8                                            (0x3040U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG9
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9                                            (0x3048U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_MASKFULL                                   (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_SHIFT                              (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_CLRMSK                             (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_EN                                 (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_SHIFT                        (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_CLRMSK                       (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_EN                           (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_SHIFT                             (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_CLRMSK                            (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_EN                                (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_SHIFT                                 (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_CLRMSK                                (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_SHIFT                                (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_CLRMSK                               (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_SHIFT                             (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_CLRMSK                            (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSHIFT                        (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSIZE                         (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG10
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10                                           (0x3050U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG11
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11                                           (0x3058U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG12
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12                                           (0x3060U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG13
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13                                           (0x3068U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG14
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14                                           (0x3070U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_ADDR_REMAP_CONFIG15
+*/
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15                                           (0x3078U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_MASKFULL                                  (IMG_UINT64_C(0x7FFFF7FFFFFFF000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_SHIFT                             (62U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_CLRMSK                            (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_EN                                (IMG_UINT64_C(0x4000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_SHIFT                       (61U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_CLRMSK                      (IMG_UINT64_C(0xDFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_EN                          (IMG_UINT64_C(0x2000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_SHIFT                            (60U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_CLRMSK                           (IMG_UINT64_C(0xEFFFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_EN                               (IMG_UINT64_C(0x1000000000000000))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_SHIFT                                (44U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_CLRMSK                               (IMG_UINT64_C(0xF0000FFFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_SHIFT                               (40U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_CLRMSK                              (IMG_UINT64_C(0xFFFFF8FFFFFFFFFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_SHIFT                            (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_CLRMSK                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSHIFT                       (12U)
+#define RGX_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSIZE                        (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_BOOT
+*/
+#define RGX_CR_FWCORE_BOOT                                                          (0x3090U)
+#define RGX_CR_FWCORE_BOOT_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_BOOT_ENABLE_SHIFT                                             (0U)
+#define RGX_CR_FWCORE_BOOT_ENABLE_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_BOOT_ENABLE_EN                                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_RESET_ADDR
+*/
+#define RGX_CR_FWCORE_RESET_ADDR                                                    (0x3098U)
+#define RGX_CR_FWCORE_RESET_ADDR_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_SHIFT                                         (1U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_CLRMSK                                        (0x00000001U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_ALIGNSHIFT                                    (1U)
+#define RGX_CR_FWCORE_RESET_ADDR_ADDR_ALIGNSIZE                                     (2U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_ADDR
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR                                              (0x30A0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFE))
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_SHIFT                                   (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_CLRMSK                                  (0x00000001U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_ALIGNSHIFT                              (1U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_ADDR_ADDR_ALIGNSIZE                               (2U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_NMI_EVENT
+*/
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT                                             (0x30A8U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_MASKFULL                                    (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_SHIFT                            (0U)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_NMI_EVENT_TRIGGER_EN_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS                                          (0x30B0U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_MASKFULL                                 (IMG_UINT64_C(0x000000000000F771))
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_CAT_BASE_SHIFT                           (12U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_CAT_BASE_CLRMSK                          (0xFFFF0FFFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_SHIFT                          (8U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_CLRMSK                         (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_DATA_TYPE_SHIFT                          (5U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_DATA_TYPE_CLRMSK                         (0xFFFFFF9FU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_SHIFT                           (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_CLRMSK                          (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_RO_EN                              (0x00000010U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_SHIFT                              (0U)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_CLRMSK                             (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_FAULT_EN                                 (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS                                          (0x30B8U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_MASKFULL                                 (IMG_UINT64_C(0x001FFFFFFFFFFFF0))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_SHIFT                                (52U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_CLRMSK                               (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN                                   (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT                             (46U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK                            (IMG_UINT64_C(0xFFF03FFFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT                             (40U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK                            (IMG_UINT64_C(0xFFFFC0FFFFFFFFFF))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT                            (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK                           (IMG_UINT64_C(0xFFFFFF000000000F))
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT                       (4U)
+#define RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSIZE                        (16U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CTRL_INVAL
+*/
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL                                                (0x30C0U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_MASKFULL                                       (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_SHIFT                                      (3U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_TLB_EN                                         (0x00000008U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_SHIFT                                       (2U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_CLRMSK                                      (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PC_EN                                          (0x00000004U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_SHIFT                                       (1U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PD_EN                                          (0x00000002U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_SHIFT                                       (0U)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_CTRL_INVAL_PT_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_MMU_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_MMU_STATUS                                                (0x30C8U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_MASKFULL                                       (IMG_UINT64_C(0x000000000FFFFFF7))
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PC_DATA_SHIFT                                  (20U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PC_DATA_CLRMSK                                 (0xF00FFFFFU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PD_DATA_SHIFT                                  (12U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PD_DATA_CLRMSK                                 (0xFFF00FFFU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PT_DATA_SHIFT                                  (4U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PT_DATA_CLRMSK                                 (0xFFFFF00FU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_SHIFT                                  (2U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_STALLED_EN                                     (0x00000004U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_SHIFT                                   (1U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_CLRMSK                                  (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_PAUSED_EN                                      (0x00000002U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_MEM_MMU_STATUS_BUSY_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_READS_EXT_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS                                          (0x30D8U)
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MMU_SHIFT                                (0U)
+#define RGX_CR_FWCORE_MEM_READS_EXT_STATUS_MMU_CLRMSK                               (0xFFFFF000U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_READS_INT_STATUS
+*/
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS                                          (0x30E0U)
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MASKFULL                                 (IMG_UINT64_C(0x00000000000007FF))
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MMU_SHIFT                                (0U)
+#define RGX_CR_FWCORE_MEM_READS_INT_STATUS_MMU_CLRMSK                               (0xFFFFF800U)
+
+
+/*
+    Register RGX_CR_FWCORE_WRAPPER_FENCE
+*/
+#define RGX_CR_FWCORE_WRAPPER_FENCE                                                 (0x30E8U)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_MASKFULL                                        (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_CLRMSK                                       (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WRAPPER_FENCE_ID_EN                                           (0x00000001U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_MEM_CAT_BASE, with 8 repeats
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE_REPEATCOUNT                                      (8U)
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE0
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE0                                                 (0x30F0U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE1
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE1                                                 (0x30F8U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE2
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE2                                                 (0x3100U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE3
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE3                                                 (0x3108U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE4
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE4                                                 (0x3110U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE5
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE5                                                 (0x3118U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE6
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE6                                                 (0x3120U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_MEM_CAT_BASE7
+*/
+#define RGX_CR_FWCORE_MEM_CAT_BASE7                                                 (0x3128U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_MASKFULL                                        (IMG_UINT64_C(0x000000FFFFFFF000))
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_SHIFT                                      (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_CLRMSK                                     (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSHIFT                                 (12U)
+#define RGX_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSIZE                                  (4096U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_RESET
+*/
+#define RGX_CR_FWCORE_WDT_RESET                                                     (0x3130U)
+#define RGX_CR_FWCORE_WDT_RESET_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_FWCORE_WDT_RESET_EN_SHIFT                                            (0U)
+#define RGX_CR_FWCORE_WDT_RESET_EN_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_RESET_EN_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_CTRL
+*/
+#define RGX_CR_FWCORE_WDT_CTRL                                                      (0x3138U)
+#define RGX_CR_FWCORE_WDT_CTRL_MASKFULL                                             (IMG_UINT64_C(0x00000000FFFF1F01))
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_SHIFT                                           (16U)
+#define RGX_CR_FWCORE_WDT_CTRL_PROT_CLRMSK                                          (0x0000FFFFU)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_SHIFT                                      (8U)
+#define RGX_CR_FWCORE_WDT_CTRL_THRESHOLD_CLRMSK                                     (0xFFFFE0FFU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_WDT_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_WDT_COUNT
+*/
+#define RGX_CR_FWCORE_WDT_COUNT                                                     (0x3140U)
+#define RGX_CR_FWCORE_WDT_COUNT_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FWCORE_WDT_COUNT_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED0, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED0_REPEATCOUNT                                     (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED00
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED00                                                (0x3400U)
+#define RGX_CR_FWCORE_DMI_RESERVED00_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED01
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED01                                                (0x3408U)
+#define RGX_CR_FWCORE_DMI_RESERVED01_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED02
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED02                                                (0x3410U)
+#define RGX_CR_FWCORE_DMI_RESERVED02_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED03
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED03                                                (0x3418U)
+#define RGX_CR_FWCORE_DMI_RESERVED03_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA0
+*/
+#define RGX_CR_FWCORE_DMI_DATA0                                                     (0x3420U)
+#define RGX_CR_FWCORE_DMI_DATA0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA0_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DATA1
+*/
+#define RGX_CR_FWCORE_DMI_DATA1                                                     (0x3428U)
+#define RGX_CR_FWCORE_DMI_DATA1_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_SHIFT                                           (0U)
+#define RGX_CR_FWCORE_DMI_DATA1_VAL_CLRMSK                                          (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED1, with 10 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED1_REPEATCOUNT                                     (10U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED10
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED10                                                (0x3430U)
+#define RGX_CR_FWCORE_DMI_RESERVED10_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED11
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED11                                                (0x3438U)
+#define RGX_CR_FWCORE_DMI_RESERVED11_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED12
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED12                                                (0x3440U)
+#define RGX_CR_FWCORE_DMI_RESERVED12_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED13
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED13                                                (0x3448U)
+#define RGX_CR_FWCORE_DMI_RESERVED13_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED14
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED14                                                (0x3450U)
+#define RGX_CR_FWCORE_DMI_RESERVED14_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMCONTROL
+*/
+#define RGX_CR_FWCORE_DMI_DMCONTROL                                                 (0x3480U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_MASKFULL                                        (IMG_UINT64_C(0x00000000D0000003))
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_SHIFT                                   (31U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_CLRMSK                                  (0x7FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN                                      (0x80000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_SHIFT                                 (30U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_CLRMSK                                (0xBFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN                                    (0x40000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_SHIFT                              (28U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_CLRMSK                             (0xEFFFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_ACKHAVERESET_EN                                 (0x10000000U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_SHIFT                                  (1U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_CLRMSK                                 (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_NDMRESET_EN                                     (0x00000002U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_CLRMSK                                 (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_DMSTATUS
+*/
+#define RGX_CR_FWCORE_DMI_DMSTATUS                                                  (0x3488U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_MASKFULL                                         (IMG_UINT64_C(0x00000000004FFFFF))
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_SHIFT                                  (22U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_CLRMSK                                 (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_IMPEBREAK_EN                                     (0x00400000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_SHIFT                               (19U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_CLRMSK                              (0xFFF7FFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHAVERESET_EN                                  (0x00080000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_SHIFT                               (18U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_CLRMSK                              (0xFFFBFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHAVERESET_EN                                  (0x00040000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_SHIFT                               (17U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_CLRMSK                              (0xFFFDFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN                                  (0x00020000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_SHIFT                               (16U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_CLRMSK                              (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRESUMEACK_EN                                  (0x00010000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_SHIFT                             (15U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_CLRMSK                            (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLNONEXISTENT_EN                                (0x00008000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_SHIFT                             (14U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_CLRMSK                            (0xFFFFBFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYNONEXISTENT_EN                                (0x00004000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_SHIFT                                 (13U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_CLRMSK                                (0xFFFFDFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLUNAVAIL_EN                                    (0x00002000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_SHIFT                                 (12U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_CLRMSK                                (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYUNAVAIL_EN                                    (0x00001000U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_SHIFT                                 (11U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_CLRMSK                                (0xFFFFF7FFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLRUNNING_EN                                    (0x00000800U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_SHIFT                                 (10U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_CLRMSK                                (0xFFFFFBFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYRUNNING_EN                                    (0x00000400U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_SHIFT                                  (9U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_CLRMSK                                 (0xFFFFFDFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN                                     (0x00000200U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_SHIFT                                  (8U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_ANYHALTED_EN                                     (0x00000100U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_SHIFT                              (7U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_CLRMSK                             (0xFFFFFF7FU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHENTICATED_EN                                 (0x00000080U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_SHIFT                                   (6U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_CLRMSK                                  (0xFFFFFFBFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_AUTHBUSY_EN                                      (0x00000040U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_SHIFT                            (5U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_CLRMSK                           (0xFFFFFFDFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_HASRESETHALTREQ_EN                               (0x00000020U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_SHIFT                            (4U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_CLRMSK                           (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_CONFSTRPTRVALID_EN                               (0x00000010U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_SHIFT                                    (0U)
+#define RGX_CR_FWCORE_DMI_DMSTATUS_VERSION_CLRMSK                                   (0xFFFFFFF0U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED2, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED2_REPEATCOUNT                                     (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED20
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED20                                                (0x3490U)
+#define RGX_CR_FWCORE_DMI_RESERVED20_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED21
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED21                                                (0x3498U)
+#define RGX_CR_FWCORE_DMI_RESERVED21_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED22
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED22                                                (0x34A0U)
+#define RGX_CR_FWCORE_DMI_RESERVED22_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED23
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED23                                                (0x34A8U)
+#define RGX_CR_FWCORE_DMI_RESERVED23_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_ABSTRACTCS
+*/
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS                                                (0x34B0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_MASKFULL                                       (IMG_UINT64_C(0x000000001F00170F))
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_SHIFT                              (24U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_PROGBUFSIZE_CLRMSK                             (0xE0FFFFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_SHIFT                                     (12U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_CLRMSK                                    (0xFFFFEFFFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN                                        (0x00001000U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT                                   (8U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK                                  (0xFFFFF8FFU)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_SHIFT                                (0U)
+#define RGX_CR_FWCORE_DMI_ABSTRACTCS_DATACOUNT_CLRMSK                               (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_COMMAND
+*/
+#define RGX_CR_FWCORE_DMI_COMMAND                                                   (0x34B8U)
+#define RGX_CR_FWCORE_DMI_COMMAND_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT                                     (24U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_CLRMSK                                    (0x00FFFFFFU)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_SHIFT                                     (0U)
+#define RGX_CR_FWCORE_DMI_COMMAND_CONTROL_CLRMSK                                    (0xFF000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_RESERVED3, with 32 repeats
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED3_REPEATCOUNT                                     (32U)
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED30
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED30                                                (0x34C0U)
+#define RGX_CR_FWCORE_DMI_RESERVED30_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_RESERVED31
+*/
+#define RGX_CR_FWCORE_DMI_RESERVED31                                                (0x34C8U)
+#define RGX_CR_FWCORE_DMI_RESERVED31_MASKFULL                                       (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBCS
+*/
+#define RGX_CR_FWCORE_DMI_SBCS                                                      (0x35C0U)
+#define RGX_CR_FWCORE_DMI_SBCS_MASKFULL                                             (IMG_UINT64_C(0x00000000E07FFFFF))
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_SHIFT                                      (29U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBVERSION_CLRMSK                                     (0x1FFFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_SHIFT                                    (22U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_CLRMSK                                   (0xFFBFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSYERROR_EN                                       (0x00400000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_SHIFT                                         (21U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_CLRMSK                                        (0xFFDFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN                                            (0x00200000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_SHIFT                                   (20U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_CLRMSK                                  (0xFFEFFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN                                      (0x00100000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT                                       (17U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS_CLRMSK                                      (0xFFF1FFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_SHIFT                                (16U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_CLRMSK                               (0xFFFEFFFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBAUTOINCREMENT_EN                                   (0x00010000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_SHIFT                                   (15U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_CLRMSK                                  (0xFFFF7FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBREADONDATA_EN                                      (0x00008000U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT                                        (12U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK                                       (0xFFFF8FFFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_SHIFT                                        (5U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBASIZE_CLRMSK                                       (0xFFFFF01FU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_SHIFT                                    (4U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS128_EN                                       (0x00000010U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_SHIFT                                     (3U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_CLRMSK                                    (0xFFFFFFF7U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS64_EN                                        (0x00000008U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_SHIFT                                     (2U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS32_EN                                        (0x00000004U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_SHIFT                                     (1U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS16_EN                                        (0x00000002U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_SHIFT                                      (0U)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FWCORE_DMI_SBCS_SBACCESS8_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBADDRESS0
+*/
+#define RGX_CR_FWCORE_DMI_SBADDRESS0                                                (0x35C8U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_SHIFT                                  (0U)
+#define RGX_CR_FWCORE_DMI_SBADDRESS0_ADDRESS_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register group: RGX_CR_FWCORE_DMI_SBDATA, with 4 repeats
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA_REPEATCOUNT                                        (4U)
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA0
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA0                                                   (0x35E0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA0_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA0_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA1
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA1                                                   (0x35E8U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA1_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA1_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA2
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA2                                                   (0x35F0U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA2_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA2_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_SBDATA3
+*/
+#define RGX_CR_FWCORE_DMI_SBDATA3                                                   (0x35F8U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_SBDATA3_DATA_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_SBDATA3_DATA_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_FWCORE_DMI_HALTSUM0
+*/
+#define RGX_CR_FWCORE_DMI_HALTSUM0                                                  (0x3600U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_SHIFT                                        (0U)
+#define RGX_CR_FWCORE_DMI_HALTSUM0_VAL_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_SLC_CTRL_MISC
+*/
+#define RGX_CR_SLC_CTRL_MISC                                                        (0x3800U)
+#define RGX_CR_SLC_CTRL_MISC_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFF03FF010F))
+#define RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS_SHIFT                                    (32U)
+#define RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS_CLRMSK                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_SHIFT                             (25U)
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_TAG_ID_LIMIT_CONTROL_EN                                (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_SHIFT                                  (24U)
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_EN                                     (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SHIFT                                 (16U)
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_CLRMSK                                (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_INTERLEAVED_64_BYTE                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_INTERLEAVED_128_BYTE                  (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SIMPLE_HASH1                          (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_SIMPLE_HASH2                          (IMG_UINT64_C(0x0000000000110000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH1                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH2_SCRAMBLE                    (IMG_UINT64_C(0x0000000000210000))
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_SHIFT                                            (8U)
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SLC_CTRL_MISC_PAUSE_EN                                               (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_SHIFT                                    (3U)
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SLC_CTRL_MISC_RESP_PRIORITY_EN                                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_SHIFT                            (2U)
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_LINE_USE_LIMIT_EN                               (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_SHIFT                          (1U)
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_SHIFT                            (0U)
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_EN                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC_CTRL_FLUSH_INVAL
+*/
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL                                                 (0x3818U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_MASKFULL                                        (IMG_UINT64_C(0x0000000080000FFF))
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_SHIFT                                      (31U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_CLRMSK                                     (0x7FFFFFFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_LAZY_EN                                         (0x80000000U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_SHIFT                             (11U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_CLRMSK                            (0xFFFFF7FFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FASTRENDER_EN                                (0x00000800U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_SHIFT                             (10U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_CLRMSK                            (0xFFFFFBFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_VERTEX_EN                                (0x00000400U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_SHIFT                                    (9U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_CLRMSK                                   (0xFFFFFDFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_RAY_EN                                       (0x00000200U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_SHIFT                                    (8U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_CLRMSK                                   (0xFFFFFEFFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_FRC_EN                                       (0x00000100U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_SHIFT                                    (7U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_CLRMSK                                   (0xFFFFFF7FU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXE_EN                                       (0x00000080U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_SHIFT                                    (6U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_CLRMSK                                   (0xFFFFFFBFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_VXD_EN                                       (0x00000040U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_SHIFT                              (5U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_HOST_META_EN                                 (0x00000020U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_SHIFT                                    (4U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_CLRMSK                                   (0xFFFFFFEFU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_MMU_EN                                       (0x00000010U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_SHIFT                                (3U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_CLRMSK                               (0xFFFFFFF7U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_COMPUTE_EN                                   (0x00000008U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_SHIFT                                  (2U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_CLRMSK                                 (0xFFFFFFFBU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_PIXEL_EN                                     (0x00000004U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_SHIFT                                     (1U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_DM_TA_EN                                        (0x00000002U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_SHIFT                                       (0U)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS0
+*/
+#define RGX_CR_SLC_STATUS0                                                          (0x3820U)
+#define RGX_CR_SLC_STATUS0_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000007))
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_SHIFT                                (2U)
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_CLRMSK                               (0xFFFFFFFBU)
+#define RGX_CR_SLC_STATUS0_FLUSH_INVAL_PENDING_EN                                   (0x00000004U)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_SHIFT                                      (1U)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_SLC_STATUS0_INVAL_PENDING_EN                                         (0x00000002U)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_SHIFT                                      (0U)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_SLC_STATUS0_FLUSH_PENDING_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_CTRL_BYPASS
+*/
+#define RGX_CR_SLC_CTRL_BYPASS                                                      (0x3828U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__MASKFULL                                    (IMG_UINT64_C(0x0FFFFFFFFFFF7FFF))
+#define RGX_CR_SLC_CTRL_BYPASS_MASKFULL                                             (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_SHIFT                     (59U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_CLRMSK                    (IMG_UINT64_C(0xF7FFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_ZLS_EN                        (IMG_UINT64_C(0x0800000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_SHIFT            (58U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_CLRMSK           (IMG_UINT64_C(0xFBFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_HEADER_EN               (IMG_UINT64_C(0x0400000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_SHIFT            (57U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_CLRMSK           (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_HEADER_EN               (IMG_UINT64_C(0x0200000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_SHIFT              (56U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_CLRMSK             (IMG_UINT64_C(0xFEFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_ZLS_DATA_EN                 (IMG_UINT64_C(0x0100000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_SHIFT              (55U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_CLRMSK             (IMG_UINT64_C(0xFF7FFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_DECOMP_TCU_DATA_EN                 (IMG_UINT64_C(0x0080000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_SHIFT                     (54U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_CLRMSK                    (IMG_UINT64_C(0xFFBFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TFBC_COMP_PBE_EN                        (IMG_UINT64_C(0x0040000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_SHIFT                    (53U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_CLRMSK                   (IMG_UINT64_C(0xFFDFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_DM_COMPUTE_EN                       (IMG_UINT64_C(0x0020000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_SHIFT                      (52U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_CLRMSK                     (IMG_UINT64_C(0xFFEFFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PDSRW_NOLINEFILL_EN                         (IMG_UINT64_C(0x0010000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_SHIFT                        (51U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_CLRMSK                       (IMG_UINT64_C(0xFFF7FFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__PBE_NOLINEFILL_EN                           (IMG_UINT64_C(0x0008000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_SHIFT                               (50U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_CLRMSK                              (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_FBC_EN                                  (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_SHIFT                          (49U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_CLRMSK                         (IMG_UINT64_C(0xFFFDFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_RREQ_EN                             (IMG_UINT64_C(0x0002000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_SHIFT                          (48U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_CLRMSK                         (IMG_UINT64_C(0xFFFEFFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_CREQ_EN                             (IMG_UINT64_C(0x0001000000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_SHIFT                          (47U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_CLRMSK                         (IMG_UINT64_C(0xFFFF7FFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_PREQ_EN                             (IMG_UINT64_C(0x0000800000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_SHIFT                          (46U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_CLRMSK                         (IMG_UINT64_C(0xFFFFBFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_IPF_DBSC_EN                             (IMG_UINT64_C(0x0000400000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_SHIFT                               (45U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_CLRMSK                              (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TCU_EN                                  (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_SHIFT                               (44U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_CLRMSK                              (IMG_UINT64_C(0xFFFFEFFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PBE_EN                                  (IMG_UINT64_C(0x0000100000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_SHIFT                               (43U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_CLRMSK                              (IMG_UINT64_C(0xFFFFF7FFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_ISP_EN                                  (IMG_UINT64_C(0x0000080000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_SHIFT                                (42U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_CLRMSK                               (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_PM_EN                                   (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_SHIFT                               (41U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFDFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TDM_EN                                  (IMG_UINT64_C(0x0000020000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_SHIFT                               (40U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_CDM_EN                                  (IMG_UINT64_C(0x0000010000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_SHIFT                    (39U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_CLRMSK                   (IMG_UINT64_C(0xFFFFFF7FFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_PDS_STATE_EN                       (IMG_UINT64_C(0x0000008000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_SHIFT                           (38U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_CLRMSK                          (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_DB_EN                              (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_SHIFT                      (37U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_CLRMSK                     (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TSPF_VTX_VAR_EN                         (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_SHIFT                               (36U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_CLRMSK                              (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_VDM_EN                                  (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_SHIFT                     (35U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_CLRMSK                    (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_STREAM_EN                        (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_SHIFT                     (34U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_CLRMSK                    (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PSG_REGION_EN                        (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_SHIFT                            (33U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_VCE_EN                               (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_SHIFT                            (32U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_CLRMSK                           (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__REQ_TA_PPP_EN                               (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_SHIFT                         (31U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_FASTRENDER_EN                            (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_SHIFT                           (30U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PM_ALIST_EN                              (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_SHIFT                              (29U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_TE_EN                                 (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_SHIFT                             (28U)
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFEFFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS__XE_MEM__DM_PB_VCE_EN                                (IMG_UINT64_C(0x0000000010000000))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_SHIFT                                  (27U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF7FFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_VERTEX_EN                                     (IMG_UINT64_C(0x0000000008000000))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_SHIFT                                         (26U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_RAY_EN                                            (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_SHIFT                                    (25U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFDFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_EN                                       (IMG_UINT64_C(0x0000000002000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_SHIFT                                        (24U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFEFFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_EN                                           (IMG_UINT64_C(0x0000000001000000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_SHIFT                                       (23U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_FBDC_EN                                          (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_SHIFT                                        (22U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TLA_EN                                           (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_SHIFT                                       (21U)
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_N_EN                                          (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_SHIFT                                         (20U)
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFEFFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_BYP_CC_EN                                            (IMG_UINT64_C(0x0000000000100000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_SHIFT                                        (19U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MCU_EN                                           (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_SHIFT                                        (18U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_PDS_EN                                           (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_SHIFT                                        (17U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TPF_EN                                           (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_SHIFT                                     (16U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_TA_TPC_EN                                        (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_SHIFT                                    (15U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN                                       (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_SHIFT                                        (14U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_USC_EN                                           (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_SHIFT                                       (13U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_META_EN                                          (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_SHIFT                                       (12U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_HOST_EN                                          (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_SHIFT                                     (11U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PT_EN                                        (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_SHIFT                                     (10U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PD_EN                                        (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_SHIFT                                     (9U)
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_SLC_CTRL_BYPASS_REQ_MMU_PC_EN                                        (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_SHIFT                                         (8U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_FRC_EN                                            (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_SHIFT                                         (7U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXE_EN                                            (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_SHIFT                                         (6U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_VXD_EN                                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_SHIFT                                   (5U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_HOST_META_EN                                      (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_SHIFT                                         (4U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_MMU_EN                                            (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_SHIFT                                     (3U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_COMPUTE_EN                                        (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_SHIFT                                       (2U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_PIXEL_EN                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_SHIFT                                          (1U)
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_SLC_CTRL_BYPASS_DM_TA_EN                                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_SHIFT                                            (0U)
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_SLC_CTRL_BYPASS_ALL_EN                                               (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC_STATUS1
+*/
+#define RGX_CR_SLC_STATUS1                                                          (0x3870U)
+#define RGX_CR_SLC_STATUS1_MASKFULL                                                 (IMG_UINT64_C(0x800003FF03FFFFFF))
+#define RGX_CR_SLC_STATUS1_PAUSED_SHIFT                                             (63U)
+#define RGX_CR_SLC_STATUS1_PAUSED_CLRMSK                                            (IMG_UINT64_C(0x7FFFFFFFFFFFFFFF))
+#define RGX_CR_SLC_STATUS1_PAUSED_EN                                                (IMG_UINT64_C(0x8000000000000000))
+#define RGX_CR_SLC_STATUS1_READS1_SHIFT                                             (32U)
+#define RGX_CR_SLC_STATUS1_READS1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_CR_SLC_STATUS1_READS0_SHIFT                                             (16U)
+#define RGX_CR_SLC_STATUS1_READS0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_CR_SLC_STATUS1_READS1_EXT_SHIFT                                         (8U)
+#define RGX_CR_SLC_STATUS1_READS1_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_SLC_STATUS1_READS0_EXT_SHIFT                                         (0U)
+#define RGX_CR_SLC_STATUS1_READS0_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_SLC_IDLE
+*/
+#define RGX_CR_SLC_IDLE                                                             (0x3898U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MASKFULL                                           (IMG_UINT64_C(0x00000000000003FF))
+#define RGX_CR_SLC_IDLE_MASKFULL                                                    (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_SHIFT                                   (9U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_CLRMSK                                  (0xFFFFFDFFU)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB1_EN                                      (0x00000200U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_SHIFT                                   (8U)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_CLRMSK                                  (0xFFFFFEFFU)
+#define RGX_CR_SLC_IDLE__XE_MEM__MH_SYSARB0_EN                                      (0x00000100U)
+#define RGX_CR_SLC_IDLE_IMGBV4_SHIFT                                                (7U)
+#define RGX_CR_SLC_IDLE_IMGBV4_CLRMSK                                               (0xFFFFFF7FU)
+#define RGX_CR_SLC_IDLE_IMGBV4_EN                                                   (0x00000080U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_SHIFT                                           (6U)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_CLRMSK                                          (0xFFFFFFBFU)
+#define RGX_CR_SLC_IDLE_CACHE_BANKS_EN                                              (0x00000040U)
+#define RGX_CR_SLC_IDLE_RBOFIFO_SHIFT                                               (5U)
+#define RGX_CR_SLC_IDLE_RBOFIFO_CLRMSK                                              (0xFFFFFFDFU)
+#define RGX_CR_SLC_IDLE_RBOFIFO_EN                                                  (0x00000020U)
+#define RGX_CR_SLC_IDLE_FRC_CONV_SHIFT                                              (4U)
+#define RGX_CR_SLC_IDLE_FRC_CONV_CLRMSK                                             (0xFFFFFFEFU)
+#define RGX_CR_SLC_IDLE_FRC_CONV_EN                                                 (0x00000010U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_SHIFT                                              (3U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_CLRMSK                                             (0xFFFFFFF7U)
+#define RGX_CR_SLC_IDLE_VXE_CONV_EN                                                 (0x00000008U)
+#define RGX_CR_SLC_IDLE_VXD_CONV_SHIFT                                              (2U)
+#define RGX_CR_SLC_IDLE_VXD_CONV_CLRMSK                                             (0xFFFFFFFBU)
+#define RGX_CR_SLC_IDLE_VXD_CONV_EN                                                 (0x00000004U)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_SHIFT                                             (1U)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_CLRMSK                                            (0xFFFFFFFDU)
+#define RGX_CR_SLC_IDLE_BIF1_CONV_EN                                                (0x00000002U)
+#define RGX_CR_SLC_IDLE_CBAR_SHIFT                                                  (0U)
+#define RGX_CR_SLC_IDLE_CBAR_CLRMSK                                                 (0xFFFFFFFEU)
+#define RGX_CR_SLC_IDLE_CBAR_EN                                                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_STATUS2
+*/
+#define RGX_CR_SLC_STATUS2                                                          (0x3908U)
+#define RGX_CR_SLC_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x000003FF03FFFFFF))
+#define RGX_CR_SLC_STATUS2_READS3_SHIFT                                             (32U)
+#define RGX_CR_SLC_STATUS2_READS3_CLRMSK                                            (IMG_UINT64_C(0xFFFFFC00FFFFFFFF))
+#define RGX_CR_SLC_STATUS2_READS2_SHIFT                                             (16U)
+#define RGX_CR_SLC_STATUS2_READS2_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFC00FFFF))
+#define RGX_CR_SLC_STATUS2_READS3_EXT_SHIFT                                         (8U)
+#define RGX_CR_SLC_STATUS2_READS3_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_SLC_STATUS2_READS2_EXT_SHIFT                                         (0U)
+#define RGX_CR_SLC_STATUS2_READS2_EXT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_SLC_CTRL_MISC2
+*/
+#define RGX_CR_SLC_CTRL_MISC2                                                       (0x3930U)
+#define RGX_CR_SLC_CTRL_MISC2_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SLC_CTRL_MISC2_SCRAMBLE_BITS_SHIFT                                   (0U)
+#define RGX_CR_SLC_CTRL_MISC2_SCRAMBLE_BITS_CLRMSK                                  (0x00000000U)
+
+
+/*
+    Register RGX_CR_SLC_CROSSBAR_LOAD_BALANCE
+*/
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE                                            (0x3938U)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_MASKFULL                                   (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_SHIFT                               (0U)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_CLRMSK                              (0xFFFFFFFEU)
+#define RGX_CR_SLC_CROSSBAR_LOAD_BALANCE_BYPASS_EN                                  (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC_SIZE_IN_KB
+*/
+#define RGX_CR_SLC_SIZE_IN_KB                                                       (0x3970U)
+#define RGX_CR_SLC_SIZE_IN_KB_MASKFULL                                              (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_SLC_SIZE_IN_KB_SIZE_SHIFT                                            (0U)
+#define RGX_CR_SLC_SIZE_IN_KB_SIZE_CLRMSK                                           (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_USC_TIMER
+*/
+#define RGX_CR_USC_TIMER                                                            (0x46C8U)
+#define RGX_CR_USC_TIMER_MASKFULL                                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_TIMER_CNT_SHIFT                                                  (0U)
+#define RGX_CR_USC_TIMER_CNT_CLRMSK                                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_TIMER_CNT
+*/
+#define RGX_CR_USC_TIMER_CNT                                                        (0x46D0U)
+#define RGX_CR_USC_TIMER_CNT_MASKFULL                                               (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_USC_TIMER_CNT_RESET_SHIFT                                            (0U)
+#define RGX_CR_USC_TIMER_CNT_RESET_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_USC_TIMER_CNT_RESET_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_USC_UVS0_CHECKSUM
+*/
+#define RGX_CR_USC_UVS0_CHECKSUM                                                    (0x5000U)
+#define RGX_CR_USC_UVS0_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS0_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS0_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS1_CHECKSUM
+*/
+#define RGX_CR_USC_UVS1_CHECKSUM                                                    (0x5008U)
+#define RGX_CR_USC_UVS1_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS1_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS1_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS2_CHECKSUM
+*/
+#define RGX_CR_USC_UVS2_CHECKSUM                                                    (0x5010U)
+#define RGX_CR_USC_UVS2_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS2_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS2_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS3_CHECKSUM
+*/
+#define RGX_CR_USC_UVS3_CHECKSUM                                                    (0x5018U)
+#define RGX_CR_USC_UVS3_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS3_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS3_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_SIGNATURE
+*/
+#define RGX_CR_PPP_SIGNATURE                                                        (0x5020U)
+#define RGX_CR_PPP_SIGNATURE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_SIGNATURE_VALUE_SHIFT                                            (0U)
+#define RGX_CR_PPP_SIGNATURE_VALUE_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_SIGNATURE
+*/
+#define RGX_CR_TE_SIGNATURE                                                         (0x5028U)
+#define RGX_CR_TE_SIGNATURE_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_SIGNATURE_VALUE_SHIFT                                             (0U)
+#define RGX_CR_TE_SIGNATURE_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_TE_CHECKSUM
+*/
+#define RGX_CR_TE_CHECKSUM                                                          (0x5110U)
+#define RGX_CR_TE_CHECKSUM_MASKFULL                                                 (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TE_CHECKSUM_VALUE_SHIFT                                              (0U)
+#define RGX_CR_TE_CHECKSUM_VALUE_CLRMSK                                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVB_CHECKSUM
+*/
+#define RGX_CR_USC_UVB_CHECKSUM                                                     (0x5118U)
+#define RGX_CR_USC_UVB_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_USC_UVB_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_VCE_CHECKSUM
+*/
+#define RGX_CR_VCE_CHECKSUM                                                         (0x5030U)
+#define RGX_CR_VCE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_VCE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_VCE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_PDS_CHECKSUM
+*/
+#define RGX_CR_ISP_PDS_CHECKSUM                                                     (0x5038U)
+#define RGX_CR_ISP_PDS_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_PDS_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_ISP_TPF_CHECKSUM
+*/
+#define RGX_CR_ISP_TPF_CHECKSUM                                                     (0x5040U)
+#define RGX_CR_ISP_TPF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_ISP_TPF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_PLANE0_CHECKSUM
+*/
+#define RGX_CR_TFPU_PLANE0_CHECKSUM                                                 (0x5048U)
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_VALUE_SHIFT                                     (0U)
+#define RGX_CR_TFPU_PLANE0_CHECKSUM_VALUE_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_TFPU_PLANE1_CHECKSUM
+*/
+#define RGX_CR_TFPU_PLANE1_CHECKSUM                                                 (0x5050U)
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_VALUE_SHIFT                                     (0U)
+#define RGX_CR_TFPU_PLANE1_CHECKSUM_VALUE_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_CHECKSUM
+*/
+#define RGX_CR_PBE_CHECKSUM                                                         (0x5058U)
+#define RGX_CR_PBE_CHECKSUM_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_CHECKSUM_VALUE_SHIFT                                             (0U)
+#define RGX_CR_PBE_CHECKSUM_VALUE_CLRMSK                                            (0x00000000U)
+
+
+/*
+    Register RGX_CR_PDS_DOUTM_STM_SIGNATURE
+*/
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE                                              (0x5060U)
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_VALUE_SHIFT                                  (0U)
+#define RGX_CR_PDS_DOUTM_STM_SIGNATURE_VALUE_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_IFPU_ISP_CHECKSUM
+*/
+#define RGX_CR_IFPU_ISP_CHECKSUM                                                    (0x5068U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_IFPU_ISP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS4_CHECKSUM
+*/
+#define RGX_CR_USC_UVS4_CHECKSUM                                                    (0x5100U)
+#define RGX_CR_USC_UVS4_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS4_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS4_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_UVS5_CHECKSUM
+*/
+#define RGX_CR_USC_UVS5_CHECKSUM                                                    (0x5108U)
+#define RGX_CR_USC_UVS5_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_UVS5_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_USC_UVS5_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PPP_CLIP_CHECKSUM
+*/
+#define RGX_CR_PPP_CLIP_CHECKSUM                                                    (0x5120U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_SHIFT                                        (0U)
+#define RGX_CR_PPP_CLIP_CHECKSUM_VALUE_CLRMSK                                       (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_PHASE
+*/
+#define RGX_CR_PERF_TA_PHASE                                                        (0x6008U)
+#define RGX_CR_PERF_TA_PHASE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_PHASE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_TA_PHASE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_PHASE
+*/
+#define RGX_CR_PERF_3D_PHASE                                                        (0x6010U)
+#define RGX_CR_PERF_3D_PHASE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_PHASE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_3D_PHASE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_COMPUTE_PHASE
+*/
+#define RGX_CR_PERF_COMPUTE_PHASE                                                   (0x6018U)
+#define RGX_CR_PERF_COMPUTE_PHASE_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_COMPUTE_PHASE_COUNT_SHIFT                                       (0U)
+#define RGX_CR_PERF_COMPUTE_PHASE_COUNT_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_CYCLE
+*/
+#define RGX_CR_PERF_TA_CYCLE                                                        (0x6020U)
+#define RGX_CR_PERF_TA_CYCLE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_CYCLE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_TA_CYCLE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_CYCLE
+*/
+#define RGX_CR_PERF_3D_CYCLE                                                        (0x6028U)
+#define RGX_CR_PERF_3D_CYCLE_MASKFULL                                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_CYCLE_COUNT_SHIFT                                            (0U)
+#define RGX_CR_PERF_3D_CYCLE_COUNT_CLRMSK                                           (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_COMPUTE_CYCLE
+*/
+#define RGX_CR_PERF_COMPUTE_CYCLE                                                   (0x6030U)
+#define RGX_CR_PERF_COMPUTE_CYCLE_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_COMPUTE_CYCLE_COUNT_SHIFT                                       (0U)
+#define RGX_CR_PERF_COMPUTE_CYCLE_COUNT_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_TA_OR_3D_CYCLE
+*/
+#define RGX_CR_PERF_TA_OR_3D_CYCLE                                                  (0x6038U)
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_MASKFULL                                         (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_COUNT_SHIFT                                      (0U)
+#define RGX_CR_PERF_TA_OR_3D_CYCLE_COUNT_CLRMSK                                     (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_INITIAL_TA_CYCLE
+*/
+#define RGX_CR_PERF_INITIAL_TA_CYCLE                                                (0x6040U)
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_INITIAL_TA_CYCLE_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_READ_STALL
+*/
+#define RGX_CR_PERF_SLC0_READ_STALL                                                 (0x60B8U)
+#define RGX_CR_PERF_SLC0_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC0_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC0_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC0_WRITE_STALL                                                (0x60C0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC0_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_READ_STALL
+*/
+#define RGX_CR_PERF_SLC1_READ_STALL                                                 (0x60E0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC1_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC1_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC1_WRITE_STALL                                                (0x60E8U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC1_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_READ_STALL
+*/
+#define RGX_CR_PERF_SLC2_READ_STALL                                                 (0x6158U)
+#define RGX_CR_PERF_SLC2_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC2_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC2_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC2_WRITE_STALL                                                (0x6160U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC2_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_READ_STALL
+*/
+#define RGX_CR_PERF_SLC3_READ_STALL                                                 (0x6180U)
+#define RGX_CR_PERF_SLC3_READ_STALL_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_SHIFT                                     (0U)
+#define RGX_CR_PERF_SLC3_READ_STALL_COUNT_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_SLC3_WRITE_STALL
+*/
+#define RGX_CR_PERF_SLC3_WRITE_STALL                                                (0x6188U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_SHIFT                                    (0U)
+#define RGX_CR_PERF_SLC3_WRITE_STALL_COUNT_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_PERF_3D_SPINUP
+*/
+#define RGX_CR_PERF_3D_SPINUP                                                       (0x6220U)
+#define RGX_CR_PERF_3D_SPINUP_MASKFULL                                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PERF_3D_SPINUP_CYCLES_SHIFT                                          (0U)
+#define RGX_CR_PERF_3D_SPINUP_CYCLES_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_AXI_ACE_LITE_CONFIGURATION
+*/
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION                                           (0x38C0U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_MASKFULL                                  (IMG_UINT64_C(0x00003FFFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT                    (45U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_CLRMSK                   (IMG_UINT64_C(0xFFFFDFFFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_EN                       (IMG_UINT64_C(0x0000200000000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_OSID_SECURITY_SHIFT                       (37U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_OSID_SECURITY_CLRMSK                      (IMG_UINT64_C(0xFFFFE01FFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT    (36U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_CLRMSK   (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_EN       (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_SHIFT              (35U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_CLRMSK             (IMG_UINT64_C(0xFFFFFFF7FFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITE_EN                 (IMG_UINT64_C(0x0000000800000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_SHIFT               (34U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_CLRMSK              (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_READ_EN                  (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT           (30U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_CLRMSK          (IMG_UINT64_C(0xFFFFFFFC3FFFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT                    (26U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFC3FFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT                    (22U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFC3FFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_BARRIER_SHIFT                    (20U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_BARRIER_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFCFFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_BARRIER_SHIFT                    (18U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_BARRIER_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFF3FFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT          (16U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_CLRMSK         (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT                   (14U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFF3FFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT                   (12U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_CLRMSK                  (IMG_UINT64_C(0xFFFFFFFFFFFFCFFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT               (10U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT               (8U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_CLRMSK              (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_NON_SNOOPING_SHIFT                (4U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_NON_SNOOPING_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFF0F))
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_NON_SNOOPING_SHIFT                (0U)
+#define RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_NON_SNOOPING_CLRMSK               (IMG_UINT64_C(0xFFFFFFFFFFFFFFF0))
+
+
+/*
+    Register RGX_CR_POWER_ESTIMATE_RESULT
+*/
+#define RGX_CR_POWER_ESTIMATE_RESULT                                                (0x6328U)
+#define RGX_CR_POWER_ESTIMATE_RESULT_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_POWER_ESTIMATE_RESULT_VALUE_SHIFT                                    (0U)
+#define RGX_CR_POWER_ESTIMATE_RESULT_VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF
+*/
+#define RGX_CR_TA_PERF                                                              (0x7600U)
+#define RGX_CR_TA_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TA_PERF_CLR_3_SHIFT                                                  (4U)
+#define RGX_CR_TA_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define RGX_CR_TA_PERF_CLR_3_EN                                                     (0x00000010U)
+#define RGX_CR_TA_PERF_CLR_2_SHIFT                                                  (3U)
+#define RGX_CR_TA_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_TA_PERF_CLR_2_EN                                                     (0x00000008U)
+#define RGX_CR_TA_PERF_CLR_1_SHIFT                                                  (2U)
+#define RGX_CR_TA_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define RGX_CR_TA_PERF_CLR_1_EN                                                     (0x00000004U)
+#define RGX_CR_TA_PERF_CLR_0_SHIFT                                                  (1U)
+#define RGX_CR_TA_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_TA_PERF_CLR_0_EN                                                     (0x00000002U)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_TA_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT0
+*/
+#define RGX_CR_TA_PERF_SELECT0                                                      (0x7608U)
+#define RGX_CR_TA_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT1
+*/
+#define RGX_CR_TA_PERF_SELECT1                                                      (0x7610U)
+#define RGX_CR_TA_PERF_SELECT1_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT1_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT1_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT1_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT1_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT1_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT1_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT1_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT2
+*/
+#define RGX_CR_TA_PERF_SELECT2                                                      (0x7618U)
+#define RGX_CR_TA_PERF_SELECT2_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT2_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT2_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT2_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT2_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT2_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT2_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT2_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECT3
+*/
+#define RGX_CR_TA_PERF_SELECT3                                                      (0x7620U)
+#define RGX_CR_TA_PERF_SELECT3_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_TA_PERF_SELECT3_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_MODE_SHIFT                                           (21U)
+#define RGX_CR_TA_PERF_SELECT3_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TA_PERF_SELECT3_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TA_PERF_SELECT3_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_TA_PERF_SELECT3_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TA_PERF_SELECT3_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECT3_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_SELECTED_BITS
+*/
+#define RGX_CR_TA_PERF_SELECTED_BITS                                                (0x7648U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG3_SHIFT                                     (48U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG3_CLRMSK                                    (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG2_SHIFT                                     (32U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG2_CLRMSK                                    (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG1_SHIFT                                     (16U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG1_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG0_SHIFT                                     (0U)
+#define RGX_CR_TA_PERF_SELECTED_BITS_REG0_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_0
+*/
+#define RGX_CR_TA_PERF_COUNTER_0                                                    (0x7650U)
+#define RGX_CR_TA_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_1
+*/
+#define RGX_CR_TA_PERF_COUNTER_1                                                    (0x7658U)
+#define RGX_CR_TA_PERF_COUNTER_1_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_1_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_1_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_2
+*/
+#define RGX_CR_TA_PERF_COUNTER_2                                                    (0x7660U)
+#define RGX_CR_TA_PERF_COUNTER_2_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_2_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_2_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_TA_PERF_COUNTER_3
+*/
+#define RGX_CR_TA_PERF_COUNTER_3                                                    (0x7668U)
+#define RGX_CR_TA_PERF_COUNTER_3_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TA_PERF_COUNTER_3_REG_SHIFT                                          (0U)
+#define RGX_CR_TA_PERF_COUNTER_3_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF
+*/
+#define RGX_CR_RASTERISATION_PERF                                                   (0x7700U)
+#define RGX_CR_RASTERISATION_PERF_MASKFULL                                          (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_RASTERISATION_PERF_CLR_3_SHIFT                                       (4U)
+#define RGX_CR_RASTERISATION_PERF_CLR_3_CLRMSK                                      (0xFFFFFFEFU)
+#define RGX_CR_RASTERISATION_PERF_CLR_3_EN                                          (0x00000010U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_SHIFT                                       (3U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_CLRMSK                                      (0xFFFFFFF7U)
+#define RGX_CR_RASTERISATION_PERF_CLR_2_EN                                          (0x00000008U)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_SHIFT                                       (2U)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_CLRMSK                                      (0xFFFFFFFBU)
+#define RGX_CR_RASTERISATION_PERF_CLR_1_EN                                          (0x00000004U)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_SHIFT                                       (1U)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_RASTERISATION_PERF_CLR_0_EN                                          (0x00000002U)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_SHIFT                                 (0U)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_RASTERISATION_PERF_CTRL_ENABLE_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_SELECT0
+*/
+#define RGX_CR_RASTERISATION_PERF_SELECT0                                           (0x7708U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MASKFULL                                  (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MAX_SHIFT                           (48U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MAX_CLRMSK                          (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MIN_SHIFT                           (32U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BATCH_MIN_CLRMSK                          (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_SHIFT                                (21U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_MODE_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_GROUP_SELECT_SHIFT                        (16U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_GROUP_SELECT_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BIT_SELECT_SHIFT                          (0U)
+#define RGX_CR_RASTERISATION_PERF_SELECT0_BIT_SELECT_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_RASTERISATION_PERF_COUNTER_0
+*/
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0                                         (0x7750U)
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_MASKFULL                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_REG_SHIFT                               (0U)
+#define RGX_CR_RASTERISATION_PERF_COUNTER_0_REG_CLRMSK                              (0x00000000U)
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF                                                  (0x7800U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_MASKFULL                                         (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_SHIFT                                      (4U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_CLRMSK                                     (0xFFFFFFEFU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_3_EN                                         (0x00000010U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_SHIFT                                      (3U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_CLRMSK                                     (0xFFFFFFF7U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_2_EN                                         (0x00000008U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_SHIFT                                      (2U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_CLRMSK                                     (0xFFFFFFFBU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_1_EN                                         (0x00000004U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_SHIFT                                      (1U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_CLRMSK                                     (0xFFFFFFFDU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CLR_0_EN                                         (0x00000002U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_SHIFT                                (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_CTRL_ENABLE_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0                                          (0x7808U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MASKFULL                                 (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MAX_SHIFT                          (48U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MAX_CLRMSK                         (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MIN_SHIFT                          (32U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BATCH_MIN_CLRMSK                         (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_SHIFT                               (21U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_CLRMSK                              (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_MODE_EN                                  (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_GROUP_SELECT_SHIFT                       (16U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_GROUP_SELECT_CLRMSK                      (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BIT_SELECT_SHIFT                         (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0_BIT_SELECT_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0
+*/
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0                                        (0x7850U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_REG_SHIFT                              (0U)
+#define RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0_REG_CLRMSK                             (0x00000000U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF
+*/
+#define RGX_CR_TPU_MCU_L0_PERF                                                      (0x7900U)
+#define RGX_CR_TPU_MCU_L0_PERF_MASKFULL                                             (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_SHIFT                                          (4U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_3_EN                                             (0x00000010U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_SHIFT                                          (3U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_2_EN                                             (0x00000008U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_SHIFT                                          (2U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_1_EN                                             (0x00000004U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_SHIFT                                          (1U)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_TPU_MCU_L0_PERF_CLR_0_EN                                             (0x00000002U)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_SHIFT                                    (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_TPU_MCU_L0_PERF_CTRL_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_SELECT0
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0                                              (0x7908U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MASKFULL                                     (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MAX_SHIFT                              (48U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MAX_CLRMSK                             (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MIN_SHIFT                              (32U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BATCH_MIN_CLRMSK                             (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_SHIFT                                   (21U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_MODE_EN                                      (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_GROUP_SELECT_SHIFT                           (16U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_GROUP_SELECT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BIT_SELECT_SHIFT                             (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_SELECT0_BIT_SELECT_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TPU_MCU_L0_PERF_COUNTER_0
+*/
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0                                            (0x7950U)
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_REG_SHIFT                                  (0U)
+#define RGX_CR_TPU_MCU_L0_PERF_COUNTER_0_REG_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_USC_PERF
+*/
+#define RGX_CR_USC_PERF                                                             (0x8100U)
+#define RGX_CR_USC_PERF_MASKFULL                                                    (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_USC_PERF_CLR_3_SHIFT                                                 (4U)
+#define RGX_CR_USC_PERF_CLR_3_CLRMSK                                                (0xFFFFFFEFU)
+#define RGX_CR_USC_PERF_CLR_3_EN                                                    (0x00000010U)
+#define RGX_CR_USC_PERF_CLR_2_SHIFT                                                 (3U)
+#define RGX_CR_USC_PERF_CLR_2_CLRMSK                                                (0xFFFFFFF7U)
+#define RGX_CR_USC_PERF_CLR_2_EN                                                    (0x00000008U)
+#define RGX_CR_USC_PERF_CLR_1_SHIFT                                                 (2U)
+#define RGX_CR_USC_PERF_CLR_1_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_USC_PERF_CLR_1_EN                                                    (0x00000004U)
+#define RGX_CR_USC_PERF_CLR_0_SHIFT                                                 (1U)
+#define RGX_CR_USC_PERF_CLR_0_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_USC_PERF_CLR_0_EN                                                    (0x00000002U)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_SHIFT                                           (0U)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_USC_PERF_CTRL_ENABLE_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_USC_PERF_SELECT0
+*/
+#define RGX_CR_USC_PERF_SELECT0                                                     (0x8108U)
+#define RGX_CR_USC_PERF_SELECT0_MASKFULL                                            (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MAX_SHIFT                                     (48U)
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MAX_CLRMSK                                    (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MIN_SHIFT                                     (32U)
+#define RGX_CR_USC_PERF_SELECT0_BATCH_MIN_CLRMSK                                    (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_MODE_SHIFT                                          (21U)
+#define RGX_CR_USC_PERF_SELECT0_MODE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_USC_PERF_SELECT0_MODE_EN                                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_USC_PERF_SELECT0_GROUP_SELECT_SHIFT                                  (16U)
+#define RGX_CR_USC_PERF_SELECT0_GROUP_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_USC_PERF_SELECT0_BIT_SELECT_SHIFT                                    (0U)
+#define RGX_CR_USC_PERF_SELECT0_BIT_SELECT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_USC_PERF_COUNTER_0
+*/
+#define RGX_CR_USC_PERF_COUNTER_0                                                   (0x8150U)
+#define RGX_CR_USC_PERF_COUNTER_0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_USC_PERF_COUNTER_0_REG_SHIFT                                         (0U)
+#define RGX_CR_USC_PERF_COUNTER_0_REG_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_JONES_IDLE
+*/
+#define RGX_CR_JONES_IDLE                                                           (0x8328U)
+#define RGX_CR_JONES_IDLE_MASKFULL                                                  (IMG_UINT64_C(0x0000000000007FFF))
+#define RGX_CR_JONES_IDLE_TDM_SHIFT                                                 (14U)
+#define RGX_CR_JONES_IDLE_TDM_CLRMSK                                                (0xFFFFBFFFU)
+#define RGX_CR_JONES_IDLE_TDM_EN                                                    (0x00004000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_SHIFT                                          (13U)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_CLRMSK                                         (0xFFFFDFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_TLA_EN                                             (0x00002000U)
+#define RGX_CR_JONES_IDLE_FB_CDC_SHIFT                                              (12U)
+#define RGX_CR_JONES_IDLE_FB_CDC_CLRMSK                                             (0xFFFFEFFFU)
+#define RGX_CR_JONES_IDLE_FB_CDC_EN                                                 (0x00001000U)
+#define RGX_CR_JONES_IDLE_MMU_SHIFT                                                 (11U)
+#define RGX_CR_JONES_IDLE_MMU_CLRMSK                                                (0xFFFFF7FFU)
+#define RGX_CR_JONES_IDLE_MMU_EN                                                    (0x00000800U)
+#define RGX_CR_JONES_IDLE_TLA_SHIFT                                                 (10U)
+#define RGX_CR_JONES_IDLE_TLA_CLRMSK                                                (0xFFFFFBFFU)
+#define RGX_CR_JONES_IDLE_TLA_EN                                                    (0x00000400U)
+#define RGX_CR_JONES_IDLE_GARTEN_SHIFT                                              (9U)
+#define RGX_CR_JONES_IDLE_GARTEN_CLRMSK                                             (0xFFFFFDFFU)
+#define RGX_CR_JONES_IDLE_GARTEN_EN                                                 (0x00000200U)
+#define RGX_CR_JONES_IDLE_HOSTIF_SHIFT                                              (8U)
+#define RGX_CR_JONES_IDLE_HOSTIF_CLRMSK                                             (0xFFFFFEFFU)
+#define RGX_CR_JONES_IDLE_HOSTIF_EN                                                 (0x00000100U)
+#define RGX_CR_JONES_IDLE_SOCIF_SHIFT                                               (7U)
+#define RGX_CR_JONES_IDLE_SOCIF_CLRMSK                                              (0xFFFFFF7FU)
+#define RGX_CR_JONES_IDLE_SOCIF_EN                                                  (0x00000080U)
+#define RGX_CR_JONES_IDLE_TILING_SHIFT                                              (6U)
+#define RGX_CR_JONES_IDLE_TILING_CLRMSK                                             (0xFFFFFFBFU)
+#define RGX_CR_JONES_IDLE_TILING_EN                                                 (0x00000040U)
+#define RGX_CR_JONES_IDLE_IPP_SHIFT                                                 (5U)
+#define RGX_CR_JONES_IDLE_IPP_CLRMSK                                                (0xFFFFFFDFU)
+#define RGX_CR_JONES_IDLE_IPP_EN                                                    (0x00000020U)
+#define RGX_CR_JONES_IDLE_USCS_SHIFT                                                (4U)
+#define RGX_CR_JONES_IDLE_USCS_CLRMSK                                               (0xFFFFFFEFU)
+#define RGX_CR_JONES_IDLE_USCS_EN                                                   (0x00000010U)
+#define RGX_CR_JONES_IDLE_PM_SHIFT                                                  (3U)
+#define RGX_CR_JONES_IDLE_PM_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_JONES_IDLE_PM_EN                                                     (0x00000008U)
+#define RGX_CR_JONES_IDLE_CDM_SHIFT                                                 (2U)
+#define RGX_CR_JONES_IDLE_CDM_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_JONES_IDLE_CDM_EN                                                    (0x00000004U)
+#define RGX_CR_JONES_IDLE_VDM_SHIFT                                                 (1U)
+#define RGX_CR_JONES_IDLE_VDM_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_JONES_IDLE_VDM_EN                                                    (0x00000002U)
+#define RGX_CR_JONES_IDLE_BIF_SHIFT                                                 (0U)
+#define RGX_CR_JONES_IDLE_BIF_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_JONES_IDLE_BIF_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_TORNADO_PERF
+*/
+#define RGX_CR_TORNADO_PERF                                                         (0x8228U)
+#define RGX_CR_TORNADO_PERF_MASKFULL                                                (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_TORNADO_PERF_CLR_3_SHIFT                                             (4U)
+#define RGX_CR_TORNADO_PERF_CLR_3_CLRMSK                                            (0xFFFFFFEFU)
+#define RGX_CR_TORNADO_PERF_CLR_3_EN                                                (0x00000010U)
+#define RGX_CR_TORNADO_PERF_CLR_2_SHIFT                                             (3U)
+#define RGX_CR_TORNADO_PERF_CLR_2_CLRMSK                                            (0xFFFFFFF7U)
+#define RGX_CR_TORNADO_PERF_CLR_2_EN                                                (0x00000008U)
+#define RGX_CR_TORNADO_PERF_CLR_1_SHIFT                                             (2U)
+#define RGX_CR_TORNADO_PERF_CLR_1_CLRMSK                                            (0xFFFFFFFBU)
+#define RGX_CR_TORNADO_PERF_CLR_1_EN                                                (0x00000004U)
+#define RGX_CR_TORNADO_PERF_CLR_0_SHIFT                                             (1U)
+#define RGX_CR_TORNADO_PERF_CLR_0_CLRMSK                                            (0xFFFFFFFDU)
+#define RGX_CR_TORNADO_PERF_CLR_0_EN                                                (0x00000002U)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_SHIFT                                       (0U)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_CLRMSK                                      (0xFFFFFFFEU)
+#define RGX_CR_TORNADO_PERF_CTRL_ENABLE_EN                                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_TORNADO_PERF_SELECT0
+*/
+#define RGX_CR_TORNADO_PERF_SELECT0                                                 (0x8230U)
+#define RGX_CR_TORNADO_PERF_SELECT0_MASKFULL                                        (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MAX_SHIFT                                 (48U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MAX_CLRMSK                                (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MIN_SHIFT                                 (32U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BATCH_MIN_CLRMSK                                (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_SHIFT                                      (21U)
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_MODE_EN                                         (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_TORNADO_PERF_SELECT0_GROUP_SELECT_SHIFT                              (16U)
+#define RGX_CR_TORNADO_PERF_SELECT0_GROUP_SELECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_TORNADO_PERF_SELECT0_BIT_SELECT_SHIFT                                (0U)
+#define RGX_CR_TORNADO_PERF_SELECT0_BIT_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TORNADO_PERF_COUNTER_0
+*/
+#define RGX_CR_TORNADO_PERF_COUNTER_0                                               (0x8268U)
+#define RGX_CR_TORNADO_PERF_COUNTER_0_MASKFULL                                      (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TORNADO_PERF_COUNTER_0_REG_SHIFT                                     (0U)
+#define RGX_CR_TORNADO_PERF_COUNTER_0_REG_CLRMSK                                    (0x00000000U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF
+*/
+#define RGX_CR_TEXAS_PERF                                                           (0x8290U)
+#define RGX_CR_TEXAS_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_TEXAS_PERF_CLR_5_SHIFT                                               (6U)
+#define RGX_CR_TEXAS_PERF_CLR_5_CLRMSK                                              (0xFFFFFFBFU)
+#define RGX_CR_TEXAS_PERF_CLR_5_EN                                                  (0x00000040U)
+#define RGX_CR_TEXAS_PERF_CLR_4_SHIFT                                               (5U)
+#define RGX_CR_TEXAS_PERF_CLR_4_CLRMSK                                              (0xFFFFFFDFU)
+#define RGX_CR_TEXAS_PERF_CLR_4_EN                                                  (0x00000020U)
+#define RGX_CR_TEXAS_PERF_CLR_3_SHIFT                                               (4U)
+#define RGX_CR_TEXAS_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define RGX_CR_TEXAS_PERF_CLR_3_EN                                                  (0x00000010U)
+#define RGX_CR_TEXAS_PERF_CLR_2_SHIFT                                               (3U)
+#define RGX_CR_TEXAS_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define RGX_CR_TEXAS_PERF_CLR_2_EN                                                  (0x00000008U)
+#define RGX_CR_TEXAS_PERF_CLR_1_SHIFT                                               (2U)
+#define RGX_CR_TEXAS_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define RGX_CR_TEXAS_PERF_CLR_1_EN                                                  (0x00000004U)
+#define RGX_CR_TEXAS_PERF_CLR_0_SHIFT                                               (1U)
+#define RGX_CR_TEXAS_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_TEXAS_PERF_CLR_0_EN                                                  (0x00000002U)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_TEXAS_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_SELECT0
+*/
+#define RGX_CR_TEXAS_PERF_SELECT0                                                   (0x8298U)
+#define RGX_CR_TEXAS_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF803FFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_SHIFT                                        (31U)
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_TEXAS_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define RGX_CR_TEXAS_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFC0FFFF))
+#define RGX_CR_TEXAS_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define RGX_CR_TEXAS_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_TEXAS_PERF_COUNTER_0
+*/
+#define RGX_CR_TEXAS_PERF_COUNTER_0                                                 (0x82D8U)
+#define RGX_CR_TEXAS_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_TEXAS_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define RGX_CR_TEXAS_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_JONES_PERF
+*/
+#define RGX_CR_JONES_PERF                                                           (0x8330U)
+#define RGX_CR_JONES_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_JONES_PERF_CLR_3_SHIFT                                               (4U)
+#define RGX_CR_JONES_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define RGX_CR_JONES_PERF_CLR_3_EN                                                  (0x00000010U)
+#define RGX_CR_JONES_PERF_CLR_2_SHIFT                                               (3U)
+#define RGX_CR_JONES_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define RGX_CR_JONES_PERF_CLR_2_EN                                                  (0x00000008U)
+#define RGX_CR_JONES_PERF_CLR_1_SHIFT                                               (2U)
+#define RGX_CR_JONES_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define RGX_CR_JONES_PERF_CLR_1_EN                                                  (0x00000004U)
+#define RGX_CR_JONES_PERF_CLR_0_SHIFT                                               (1U)
+#define RGX_CR_JONES_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define RGX_CR_JONES_PERF_CLR_0_EN                                                  (0x00000002U)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_JONES_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_JONES_PERF_SELECT0
+*/
+#define RGX_CR_JONES_PERF_SELECT0                                                   (0x8338U)
+#define RGX_CR_JONES_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define RGX_CR_JONES_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_MODE_SHIFT                                        (21U)
+#define RGX_CR_JONES_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_JONES_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_JONES_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define RGX_CR_JONES_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_JONES_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define RGX_CR_JONES_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_JONES_PERF_COUNTER_0
+*/
+#define RGX_CR_JONES_PERF_COUNTER_0                                                 (0x8368U)
+#define RGX_CR_JONES_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_JONES_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define RGX_CR_JONES_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF
+*/
+#define RGX_CR_BLACKPEARL_PERF                                                      (0x8400U)
+#define RGX_CR_BLACKPEARL_PERF_MASKFULL                                             (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_SHIFT                                          (6U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_CLRMSK                                         (0xFFFFFFBFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_5_EN                                             (0x00000040U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_SHIFT                                          (5U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_CLRMSK                                         (0xFFFFFFDFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_4_EN                                             (0x00000020U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_SHIFT                                          (4U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_CLRMSK                                         (0xFFFFFFEFU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_3_EN                                             (0x00000010U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_SHIFT                                          (3U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_2_EN                                             (0x00000008U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_SHIFT                                          (2U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_CLRMSK                                         (0xFFFFFFFBU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_1_EN                                             (0x00000004U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_SHIFT                                          (1U)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_CLRMSK                                         (0xFFFFFFFDU)
+#define RGX_CR_BLACKPEARL_PERF_CLR_0_EN                                             (0x00000002U)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_SHIFT                                    (0U)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_CLRMSK                                   (0xFFFFFFFEU)
+#define RGX_CR_BLACKPEARL_PERF_CTRL_ENABLE_EN                                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_SELECT0
+*/
+#define RGX_CR_BLACKPEARL_PERF_SELECT0                                              (0x8408U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MASKFULL                                     (IMG_UINT64_C(0x3FFF3FFF803FFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MAX_SHIFT                              (48U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MAX_CLRMSK                             (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MIN_SHIFT                              (32U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BATCH_MIN_CLRMSK                             (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_SHIFT                                   (31U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_MODE_EN                                      (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_GROUP_SELECT_SHIFT                           (16U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_GROUP_SELECT_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFFFFC0FFFF))
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BIT_SELECT_SHIFT                             (0U)
+#define RGX_CR_BLACKPEARL_PERF_SELECT0_BIT_SELECT_CLRMSK                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_BLACKPEARL_PERF_COUNTER_0
+*/
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0                                            (0x8448U)
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_MASKFULL                                   (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_REG_SHIFT                                  (0U)
+#define RGX_CR_BLACKPEARL_PERF_COUNTER_0_REG_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_PBE_PERF
+*/
+#define RGX_CR_PBE_PERF                                                             (0x8478U)
+#define RGX_CR_PBE_PERF_MASKFULL                                                    (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_PBE_PERF_CLR_3_SHIFT                                                 (4U)
+#define RGX_CR_PBE_PERF_CLR_3_CLRMSK                                                (0xFFFFFFEFU)
+#define RGX_CR_PBE_PERF_CLR_3_EN                                                    (0x00000010U)
+#define RGX_CR_PBE_PERF_CLR_2_SHIFT                                                 (3U)
+#define RGX_CR_PBE_PERF_CLR_2_CLRMSK                                                (0xFFFFFFF7U)
+#define RGX_CR_PBE_PERF_CLR_2_EN                                                    (0x00000008U)
+#define RGX_CR_PBE_PERF_CLR_1_SHIFT                                                 (2U)
+#define RGX_CR_PBE_PERF_CLR_1_CLRMSK                                                (0xFFFFFFFBU)
+#define RGX_CR_PBE_PERF_CLR_1_EN                                                    (0x00000004U)
+#define RGX_CR_PBE_PERF_CLR_0_SHIFT                                                 (1U)
+#define RGX_CR_PBE_PERF_CLR_0_CLRMSK                                                (0xFFFFFFFDU)
+#define RGX_CR_PBE_PERF_CLR_0_EN                                                    (0x00000002U)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_SHIFT                                           (0U)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_PBE_PERF_CTRL_ENABLE_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_PBE_PERF_SELECT0
+*/
+#define RGX_CR_PBE_PERF_SELECT0                                                     (0x8480U)
+#define RGX_CR_PBE_PERF_SELECT0_MASKFULL                                            (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MAX_SHIFT                                     (48U)
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MAX_CLRMSK                                    (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MIN_SHIFT                                     (32U)
+#define RGX_CR_PBE_PERF_SELECT0_BATCH_MIN_CLRMSK                                    (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_MODE_SHIFT                                          (21U)
+#define RGX_CR_PBE_PERF_SELECT0_MODE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_PBE_PERF_SELECT0_MODE_EN                                             (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_PBE_PERF_SELECT0_GROUP_SELECT_SHIFT                                  (16U)
+#define RGX_CR_PBE_PERF_SELECT0_GROUP_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_PBE_PERF_SELECT0_BIT_SELECT_SHIFT                                    (0U)
+#define RGX_CR_PBE_PERF_SELECT0_BIT_SELECT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_PBE_PERF_COUNTER_0
+*/
+#define RGX_CR_PBE_PERF_COUNTER_0                                                   (0x84B0U)
+#define RGX_CR_PBE_PERF_COUNTER_0_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_PBE_PERF_COUNTER_0_REG_SHIFT                                         (0U)
+#define RGX_CR_PBE_PERF_COUNTER_0_REG_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_OCP_REVINFO
+*/
+#define RGX_CR_OCP_REVINFO                                                          (0x9000U)
+#define RGX_CR_OCP_REVINFO_MASKFULL                                                 (IMG_UINT64_C(0x00000007FFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_SYSBUS_SHIFT                                      (33U)
+#define RGX_CR_OCP_REVINFO_HWINFO_SYSBUS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFF9FFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_SHIFT                                      (32U)
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_OCP_REVINFO_HWINFO_MEMBUS_EN                                         (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_OCP_REVINFO_REVISION_SHIFT                                           (0U)
+#define RGX_CR_OCP_REVINFO_REVISION_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_OCP_SYSCONFIG
+*/
+#define RGX_CR_OCP_SYSCONFIG                                                        (0x9010U)
+#define RGX_CR_OCP_SYSCONFIG_MASKFULL                                               (IMG_UINT64_C(0x0000000000000FFF))
+#define RGX_CR_OCP_SYSCONFIG_DUST2_STANDBY_MODE_SHIFT                               (10U)
+#define RGX_CR_OCP_SYSCONFIG_DUST2_STANDBY_MODE_CLRMSK                              (0xFFFFF3FFU)
+#define RGX_CR_OCP_SYSCONFIG_DUST1_STANDBY_MODE_SHIFT                               (8U)
+#define RGX_CR_OCP_SYSCONFIG_DUST1_STANDBY_MODE_CLRMSK                              (0xFFFFFCFFU)
+#define RGX_CR_OCP_SYSCONFIG_DUST0_STANDBY_MODE_SHIFT                               (6U)
+#define RGX_CR_OCP_SYSCONFIG_DUST0_STANDBY_MODE_CLRMSK                              (0xFFFFFF3FU)
+#define RGX_CR_OCP_SYSCONFIG_RASCAL_STANDBYMODE_SHIFT                               (4U)
+#define RGX_CR_OCP_SYSCONFIG_RASCAL_STANDBYMODE_CLRMSK                              (0xFFFFFFCFU)
+#define RGX_CR_OCP_SYSCONFIG_STANDBY_MODE_SHIFT                                     (2U)
+#define RGX_CR_OCP_SYSCONFIG_STANDBY_MODE_CLRMSK                                    (0xFFFFFFF3U)
+#define RGX_CR_OCP_SYSCONFIG_IDLE_MODE_SHIFT                                        (0U)
+#define RGX_CR_OCP_SYSCONFIG_IDLE_MODE_CLRMSK                                       (0xFFFFFFFCU)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_0
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_0                                                  (0x9020U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_SHIFT                        (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_CLRMSK                       (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_0_INIT_MINTERRUPT_RAW_EN                           (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_1
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_1                                                  (0x9028U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_SHIFT                      (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_CLRMSK                     (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_1_TARGET_SINTERRUPT_RAW_EN                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_RAW_2
+*/
+#define RGX_CR_OCP_IRQSTATUS_RAW_2                                                  (0x9030U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_SHIFT                                (0U)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_RAW_2_RGX_IRQ_RAW_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_0
+*/
+#define RGX_CR_OCP_IRQSTATUS_0                                                      (0x9038U)
+#define RGX_CR_OCP_IRQSTATUS_0_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_SHIFT                         (0U)
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_CLRMSK                        (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_0_INIT_MINTERRUPT_STATUS_EN                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_1
+*/
+#define RGX_CR_OCP_IRQSTATUS_1                                                      (0x9040U)
+#define RGX_CR_OCP_IRQSTATUS_1_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_SHIFT                       (0U)
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_CLRMSK                      (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_1_TARGET_SINTERRUPT_STATUS_EN                          (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQSTATUS_2
+*/
+#define RGX_CR_OCP_IRQSTATUS_2                                                      (0x9048U)
+#define RGX_CR_OCP_IRQSTATUS_2_MASKFULL                                             (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_SHIFT                                 (0U)
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_CLRMSK                                (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQSTATUS_2_RGX_IRQ_STATUS_EN                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_0
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_0                                                  (0x9050U)
+#define RGX_CR_OCP_IRQENABLE_SET_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_SHIFT                     (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_CLRMSK                    (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_0_INIT_MINTERRUPT_ENABLE_EN                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_1
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_1                                                  (0x9058U)
+#define RGX_CR_OCP_IRQENABLE_SET_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_SHIFT                   (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_1_TARGET_SINTERRUPT_ENABLE_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_SET_2
+*/
+#define RGX_CR_OCP_IRQENABLE_SET_2                                                  (0x9060U)
+#define RGX_CR_OCP_IRQENABLE_SET_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_SHIFT                             (0U)
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_CLRMSK                            (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_SET_2_RGX_IRQ_ENABLE_EN                                (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_0
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_0                                                  (0x9068U)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_SHIFT                    (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_CLRMSK                   (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_0_INIT_MINTERRUPT_DISABLE_EN                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_1
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_1                                                  (0x9070U)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_SHIFT                  (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_CLRMSK                 (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_1_TARGET_SINTERRUPT_DISABLE_EN                     (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQENABLE_CLR_2
+*/
+#define RGX_CR_OCP_IRQENABLE_CLR_2                                                  (0x9078U)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_MASKFULL                                         (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_SHIFT                            (0U)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_CLRMSK                           (0xFFFFFFFEU)
+#define RGX_CR_OCP_IRQENABLE_CLR_2_RGX_IRQ_DISABLE_EN                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_IRQ_EVENT
+*/
+#define RGX_CR_OCP_IRQ_EVENT                                                        (0x9080U)
+#define RGX_CR_OCP_IRQ_EVENT_MASKFULL                                               (IMG_UINT64_C(0x00000000000FFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_SHIFT                    (19U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFF7FFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNEXPECTED_RDATA_EN                       (IMG_UINT64_C(0x0000000000080000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_SHIFT                    (18U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETH_RCVD_UNSUPPORTED_MCMD_EN                       (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_SHIFT                    (17U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFDFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNEXPECTED_RDATA_EN                       (IMG_UINT64_C(0x0000000000020000))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_SHIFT                    (16U)
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFEFFFF))
+#define RGX_CR_OCP_IRQ_EVENT_TARGETS_RCVD_UNSUPPORTED_MCMD_EN                       (IMG_UINT64_C(0x0000000000010000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (15U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_SHIFT                         (14U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_SHIFT                          (13U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_SHIFT                        (12U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFEFFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT3_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000001000))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (11U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFF7FF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_SHIFT                         (10U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_SHIFT                          (9U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFDFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_SHIFT                        (8U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFEFF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT2_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (7U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_SHIFT                         (6U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_SHIFT                          (5U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_SHIFT                        (4U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_OCP_IRQ_EVENT_INIT1_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_SHIFT                    (3U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_CLRMSK                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_IMG_PAGE_BOUNDARY_CROSS_EN                       (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_SHIFT                         (2U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_RESP_ERR_FAIL_EN                            (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_SHIFT                          (1U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_CLRMSK                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RCVD_UNUSED_TAGID_EN                             (IMG_UINT64_C(0x0000000000000002))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_SHIFT                        (0U)
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_CLRMSK                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_OCP_IRQ_EVENT_INIT0_RDATA_FIFO_OVERFILL_EN                           (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_OCP_DEBUG_CONFIG
+*/
+#define RGX_CR_OCP_DEBUG_CONFIG                                                     (0x9088U)
+#define RGX_CR_OCP_DEBUG_CONFIG_MASKFULL                                            (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_SHIFT                                           (0U)
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_OCP_DEBUG_CONFIG_REG_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_OCP_DEBUG_STATUS
+*/
+#define RGX_CR_OCP_DEBUG_STATUS                                                     (0x9090U)
+#define RGX_CR_OCP_DEBUG_STATUS_MASKFULL                                            (IMG_UINT64_C(0x001F1F77FFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SDISCACK_SHIFT                              (51U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SDISCACK_CLRMSK                             (IMG_UINT64_C(0xFFE7FFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_SHIFT                              (50U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFBFFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SCONNECT_EN                                 (IMG_UINT64_C(0x0004000000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_MCONNECT_SHIFT                              (48U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_MCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFCFFFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SDISCACK_SHIFT                              (43U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SDISCACK_CLRMSK                             (IMG_UINT64_C(0xFFFFE7FFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_SHIFT                              (42U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFBFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SCONNECT_EN                                 (IMG_UINT64_C(0x0000040000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_MCONNECT_SHIFT                              (40U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_MCONNECT_CLRMSK                             (IMG_UINT64_C(0xFFFFFCFFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_SHIFT                                  (38U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFBFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_BUSY_EN                                     (IMG_UINT64_C(0x0000004000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_SHIFT                         (37U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFDFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_CMD_FIFO_FULL_EN                            (IMG_UINT64_C(0x0000002000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_SHIFT                           (36U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_CLRMSK                          (IMG_UINT64_C(0xFFFFFFEFFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETH_SRESP_ERROR_EN                              (IMG_UINT64_C(0x0000001000000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_SHIFT                                  (34U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFBFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_BUSY_EN                                     (IMG_UINT64_C(0x0000000400000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_SHIFT                         (33U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_CLRMSK                        (IMG_UINT64_C(0xFFFFFFFDFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_CMD_FIFO_FULL_EN                            (IMG_UINT64_C(0x0000000200000000))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_SHIFT                           (32U)
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_CLRMSK                          (IMG_UINT64_C(0xFFFFFFFEFFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_TARGETS_SRESP_ERROR_EN                              (IMG_UINT64_C(0x0000000100000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_SHIFT                                (31U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFF7FFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_RESERVED_EN                                   (IMG_UINT64_C(0x0000000080000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_SHIFT                                   (30U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFBFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SWAIT_EN                                      (IMG_UINT64_C(0x0000000040000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_SHIFT                                (29U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFDFFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000020000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCACK_SHIFT                                (27U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFE7FFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_SHIFT                                (26U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFBFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000004000000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MCONNECT_SHIFT                                (24U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT3_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFCFFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_SHIFT                                (23U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFF7FFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000800000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_SHIFT                                   (22U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFBFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000400000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_SHIFT                                (21U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCACK_SHIFT                                (19U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE7FFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_SHIFT                                (18U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFBFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000040000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MCONNECT_SHIFT                                (16U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT2_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFCFFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_SHIFT                                (15U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFF7FFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000008000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_SHIFT                                   (14U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFBFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000004000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_SHIFT                                (13U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFDFFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000002000))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCACK_SHIFT                                (11U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFE7FF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_SHIFT                                (10U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFBFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MCONNECT_SHIFT                                (8U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT1_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_SHIFT                                (7U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFF7F))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_RESERVED_EN                                   (IMG_UINT64_C(0x0000000000000080))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_SHIFT                                   (6U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFBF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SWAIT_EN                                      (IMG_UINT64_C(0x0000000000000040))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_SHIFT                                (5U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCREQ_EN                                   (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCACK_SHIFT                                (3U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MDISCACK_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFE7))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_SHIFT                                (2U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_SCONNECT_EN                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MCONNECT_SHIFT                                (0U)
+#define RGX_CR_OCP_DEBUG_STATUS_INIT0_MCONNECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+
+
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_SHIFT                                     (6U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_CLRMSK                                    (0xFFFFFFBFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PM_ALIST_EN                                        (0x00000040U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_SHIFT                                         (5U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_CLRMSK                                        (0xFFFFFFDFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_HOST_EN                                            (0x00000020U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_SHIFT                                         (4U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_CLRMSK                                        (0xFFFFFFEFU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_META_EN                                            (0x00000010U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_SHIFT                                       (3U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_CLRMSK                                      (0xFFFFFFF7U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_ZLS_EN                                          (0x00000008U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_SHIFT                                        (2U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_CLRMSK                                       (0xFFFFFFFBU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_TE_EN                                           (0x00000004U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_SHIFT                                       (1U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_PB_VCE_EN                                          (0x00000002U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_SHIFT                                          (0U)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_BIF_TRUST_DM_TYPE_TLA_EN                                             (0x00000001U)
+
+
+#define RGX_CR_BIF_TRUST_DM_MASK                                                    (0x0000007FU)
+
+
+/*
+    Register RGX_CR_BIF_TRUST
+*/
+#define RGX_CR_BIF_TRUST                                                            (0xA000U)
+#define RGX_CR_BIF_TRUST_MASKFULL                                                   (IMG_UINT64_C(0x00000000001FFFFF))
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_SHIFT                          (20U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_CLRMSK                         (0xFFEFFFFFU)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_VERTEX_DM_TRUSTED_EN                             (0x00100000U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_SHIFT                            (19U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_CLRMSK                           (0xFFF7FFFFU)
+#define RGX_CR_BIF_TRUST_MCU_RAY_VERTEX_DM_TRUSTED_EN                               (0x00080000U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_SHIFT                                 (18U)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_CLRMSK                                (0xFFFBFFFFU)
+#define RGX_CR_BIF_TRUST_OTHER_RAY_DM_TRUSTED_EN                                    (0x00040000U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_SHIFT                                   (17U)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_CLRMSK                                  (0xFFFDFFFFU)
+#define RGX_CR_BIF_TRUST_MCU_RAY_DM_TRUSTED_EN                                      (0x00020000U)
+#define RGX_CR_BIF_TRUST_ENABLE_SHIFT                                               (16U)
+#define RGX_CR_BIF_TRUST_ENABLE_CLRMSK                                              (0xFFFEFFFFU)
+#define RGX_CR_BIF_TRUST_ENABLE_EN                                                  (0x00010000U)
+#define RGX_CR_BIF_TRUST_DM_TRUSTED_SHIFT                                           (9U)
+#define RGX_CR_BIF_TRUST_DM_TRUSTED_CLRMSK                                          (0xFFFF01FFU)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_SHIFT                             (8U)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_CLRMSK                            (0xFFFFFEFFU)
+#define RGX_CR_BIF_TRUST_OTHER_COMPUTE_DM_TRUSTED_EN                                (0x00000100U)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_SHIFT                               (7U)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_CLRMSK                              (0xFFFFFF7FU)
+#define RGX_CR_BIF_TRUST_MCU_COMPUTE_DM_TRUSTED_EN                                  (0x00000080U)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_SHIFT                               (6U)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_CLRMSK                              (0xFFFFFFBFU)
+#define RGX_CR_BIF_TRUST_PBE_COMPUTE_DM_TRUSTED_EN                                  (0x00000040U)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_SHIFT                               (5U)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_CLRMSK                              (0xFFFFFFDFU)
+#define RGX_CR_BIF_TRUST_OTHER_PIXEL_DM_TRUSTED_EN                                  (0x00000020U)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_SHIFT                                 (4U)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_BIF_TRUST_MCU_PIXEL_DM_TRUSTED_EN                                    (0x00000010U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_SHIFT                                 (3U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_CLRMSK                                (0xFFFFFFF7U)
+#define RGX_CR_BIF_TRUST_PBE_PIXEL_DM_TRUSTED_EN                                    (0x00000008U)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_SHIFT                              (2U)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_CLRMSK                             (0xFFFFFFFBU)
+#define RGX_CR_BIF_TRUST_OTHER_VERTEX_DM_TRUSTED_EN                                 (0x00000004U)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_SHIFT                                (1U)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_CLRMSK                               (0xFFFFFFFDU)
+#define RGX_CR_BIF_TRUST_MCU_VERTEX_DM_TRUSTED_EN                                   (0x00000002U)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_SHIFT                                (0U)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_CLRMSK                               (0xFFFFFFFEU)
+#define RGX_CR_BIF_TRUST_PBE_VERTEX_DM_TRUSTED_EN                                   (0x00000001U)
+
+
+/*
+    Register RGX_CR_SYS_BUS_SECURE
+*/
+#define RGX_CR_SYS_BUS_SECURE                                                       (0xA100U)
+#define RGX_CR_SYS_BUS_SECURE__SECR__MASKFULL                                       (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_MASKFULL                                              (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_SHIFT                                          (0U)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_CLRMSK                                         (0xFFFFFFFEU)
+#define RGX_CR_SYS_BUS_SECURE_ENABLE_EN                                             (0x00000001U)
+
+
+/*
+    Register RGX_CR_FBA_FC0_CHECKSUM
+*/
+#define RGX_CR_FBA_FC0_CHECKSUM                                                     (0xD170U)
+#define RGX_CR_FBA_FC0_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC0_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC0_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC1_CHECKSUM
+*/
+#define RGX_CR_FBA_FC1_CHECKSUM                                                     (0xD178U)
+#define RGX_CR_FBA_FC1_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC1_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC1_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC2_CHECKSUM
+*/
+#define RGX_CR_FBA_FC2_CHECKSUM                                                     (0xD180U)
+#define RGX_CR_FBA_FC2_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC2_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC2_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_FBA_FC3_CHECKSUM
+*/
+#define RGX_CR_FBA_FC3_CHECKSUM                                                     (0xD188U)
+#define RGX_CR_FBA_FC3_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_FBA_FC3_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_FBA_FC3_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_CLK_CTRL2
+*/
+#define RGX_CR_CLK_CTRL2                                                            (0xD200U)
+#define RGX_CR_CLK_CTRL2_MASKFULL                                                   (IMG_UINT64_C(0x0000000000000F33))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_SHIFT                                             (10U)
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFF3FF))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_OFF                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_ON                                                (IMG_UINT64_C(0x0000000000000400))
+#define RGX_CR_CLK_CTRL2_MCU_FBTC_AUTO                                              (IMG_UINT64_C(0x0000000000000800))
+#define RGX_CR_CLK_CTRL2_VRDM_SHIFT                                                 (8U)
+#define RGX_CR_CLK_CTRL2_VRDM_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFCFF))
+#define RGX_CR_CLK_CTRL2_VRDM_OFF                                                   (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_VRDM_ON                                                    (IMG_UINT64_C(0x0000000000000100))
+#define RGX_CR_CLK_CTRL2_VRDM_AUTO                                                  (IMG_UINT64_C(0x0000000000000200))
+#define RGX_CR_CLK_CTRL2_SH_SHIFT                                                   (4U)
+#define RGX_CR_CLK_CTRL2_SH_CLRMSK                                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_CLK_CTRL2_SH_OFF                                                     (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_SH_ON                                                      (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_CTRL2_SH_AUTO                                                    (IMG_UINT64_C(0x0000000000000020))
+#define RGX_CR_CLK_CTRL2_FBA_SHIFT                                                  (0U)
+#define RGX_CR_CLK_CTRL2_FBA_CLRMSK                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFC))
+#define RGX_CR_CLK_CTRL2_FBA_OFF                                                    (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_CTRL2_FBA_ON                                                     (IMG_UINT64_C(0x0000000000000001))
+#define RGX_CR_CLK_CTRL2_FBA_AUTO                                                   (IMG_UINT64_C(0x0000000000000002))
+
+
+/*
+    Register RGX_CR_CLK_STATUS2
+*/
+#define RGX_CR_CLK_STATUS2                                                          (0xD208U)
+#define RGX_CR_CLK_STATUS2_MASKFULL                                                 (IMG_UINT64_C(0x0000000000000015))
+#define RGX_CR_CLK_STATUS2_VRDM_SHIFT                                               (4U)
+#define RGX_CR_CLK_STATUS2_VRDM_CLRMSK                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_CR_CLK_STATUS2_VRDM_GATED                                               (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_VRDM_RUNNING                                             (IMG_UINT64_C(0x0000000000000010))
+#define RGX_CR_CLK_STATUS2_SH_SHIFT                                                 (2U)
+#define RGX_CR_CLK_STATUS2_SH_CLRMSK                                                (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_CR_CLK_STATUS2_SH_GATED                                                 (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_SH_RUNNING                                               (IMG_UINT64_C(0x0000000000000004))
+#define RGX_CR_CLK_STATUS2_FBA_SHIFT                                                (0U)
+#define RGX_CR_CLK_STATUS2_FBA_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_CLK_STATUS2_FBA_GATED                                                (IMG_UINT64_C(0x0000000000000000))
+#define RGX_CR_CLK_STATUS2_FBA_RUNNING                                              (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL
+*/
+#define RGX_CR_RPM_SHF_FPL                                                          (0xD520U)
+#define RGX_CR_RPM_SHF_FPL_MASKFULL                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFC))
+#define RGX_CR_RPM_SHF_FPL_SIZE_SHIFT                                               (40U)
+#define RGX_CR_RPM_SHF_FPL_SIZE_CLRMSK                                              (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_CR_RPM_SHF_FPL_BASE_SHIFT                                               (2U)
+#define RGX_CR_RPM_SHF_FPL_BASE_CLRMSK                                              (IMG_UINT64_C(0xFFFFFF0000000003))
+#define RGX_CR_RPM_SHF_FPL_BASE_ALIGNSHIFT                                          (2U)
+#define RGX_CR_RPM_SHF_FPL_BASE_ALIGNSIZE                                           (4U)
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL_READ
+*/
+#define RGX_CR_RPM_SHF_FPL_READ                                                     (0xD528U)
+#define RGX_CR_RPM_SHF_FPL_READ_MASKFULL                                            (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_SHIFT                                        (22U)
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_CLRMSK                                       (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHF_FPL_READ_TOGGLE_EN                                           (0x00400000U)
+#define RGX_CR_RPM_SHF_FPL_READ_OFFSET_SHIFT                                        (0U)
+#define RGX_CR_RPM_SHF_FPL_READ_OFFSET_CLRMSK                                       (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHF_FPL_WRITE
+*/
+#define RGX_CR_RPM_SHF_FPL_WRITE                                                    (0xD530U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_MASKFULL                                           (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_SHIFT                                       (22U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHF_FPL_WRITE_TOGGLE_EN                                          (0x00400000U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_OFFSET_SHIFT                                       (0U)
+#define RGX_CR_RPM_SHF_FPL_WRITE_OFFSET_CLRMSK                                      (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL
+*/
+#define RGX_CR_RPM_SHG_FPL                                                          (0xD538U)
+#define RGX_CR_RPM_SHG_FPL_MASKFULL                                                 (IMG_UINT64_C(0x3FFFFFFFFFFFFFFC))
+#define RGX_CR_RPM_SHG_FPL_SIZE_SHIFT                                               (40U)
+#define RGX_CR_RPM_SHG_FPL_SIZE_CLRMSK                                              (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_CR_RPM_SHG_FPL_BASE_SHIFT                                               (2U)
+#define RGX_CR_RPM_SHG_FPL_BASE_CLRMSK                                              (IMG_UINT64_C(0xFFFFFF0000000003))
+#define RGX_CR_RPM_SHG_FPL_BASE_ALIGNSHIFT                                          (2U)
+#define RGX_CR_RPM_SHG_FPL_BASE_ALIGNSIZE                                           (4U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL_READ
+*/
+#define RGX_CR_RPM_SHG_FPL_READ                                                     (0xD540U)
+#define RGX_CR_RPM_SHG_FPL_READ_MASKFULL                                            (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_SHIFT                                        (22U)
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_CLRMSK                                       (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHG_FPL_READ_TOGGLE_EN                                           (0x00400000U)
+#define RGX_CR_RPM_SHG_FPL_READ_OFFSET_SHIFT                                        (0U)
+#define RGX_CR_RPM_SHG_FPL_READ_OFFSET_CLRMSK                                       (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_RPM_SHG_FPL_WRITE
+*/
+#define RGX_CR_RPM_SHG_FPL_WRITE                                                    (0xD548U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_MASKFULL                                           (IMG_UINT64_C(0x00000000007FFFFF))
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_SHIFT                                       (22U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_CLRMSK                                      (0xFFBFFFFFU)
+#define RGX_CR_RPM_SHG_FPL_WRITE_TOGGLE_EN                                          (0x00400000U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_OFFSET_SHIFT                                       (0U)
+#define RGX_CR_RPM_SHG_FPL_WRITE_OFFSET_CLRMSK                                      (0xFFC00000U)
+
+
+/*
+    Register RGX_CR_SH_PERF
+*/
+#define RGX_CR_SH_PERF                                                              (0xD5F8U)
+#define RGX_CR_SH_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_SH_PERF_CLR_3_SHIFT                                                  (4U)
+#define RGX_CR_SH_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define RGX_CR_SH_PERF_CLR_3_EN                                                     (0x00000010U)
+#define RGX_CR_SH_PERF_CLR_2_SHIFT                                                  (3U)
+#define RGX_CR_SH_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define RGX_CR_SH_PERF_CLR_2_EN                                                     (0x00000008U)
+#define RGX_CR_SH_PERF_CLR_1_SHIFT                                                  (2U)
+#define RGX_CR_SH_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define RGX_CR_SH_PERF_CLR_1_EN                                                     (0x00000004U)
+#define RGX_CR_SH_PERF_CLR_0_SHIFT                                                  (1U)
+#define RGX_CR_SH_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define RGX_CR_SH_PERF_CLR_0_EN                                                     (0x00000002U)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define RGX_CR_SH_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register RGX_CR_SH_PERF_SELECT0
+*/
+#define RGX_CR_SH_PERF_SELECT0                                                      (0xD600U)
+#define RGX_CR_SH_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define RGX_CR_SH_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define RGX_CR_SH_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define RGX_CR_SH_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define RGX_CR_SH_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define RGX_CR_SH_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define RGX_CR_SH_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define RGX_CR_SH_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SH_PERF_COUNTER_0
+*/
+#define RGX_CR_SH_PERF_COUNTER_0                                                    (0xD628U)
+#define RGX_CR_SH_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SH_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define RGX_CR_SH_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_SHG_CHECKSUM
+*/
+#define RGX_CR_SHF_SHG_CHECKSUM                                                     (0xD1C0U)
+#define RGX_CR_SHF_SHG_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_SHG_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SHF_SHG_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_VERTEX_BIF_CHECKSUM
+*/
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM                                              (0xD1C8U)
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_MASKFULL                                     (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_VALUE_SHIFT                                  (0U)
+#define RGX_CR_SHF_VERTEX_BIF_CHECKSUM_VALUE_CLRMSK                                 (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHF_VARY_BIF_CHECKSUM
+*/
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM                                                (0xD1D0U)
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_MASKFULL                                       (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_VALUE_SHIFT                                    (0U)
+#define RGX_CR_SHF_VARY_BIF_CHECKSUM_VALUE_CLRMSK                                   (0x00000000U)
+
+
+/*
+    Register RGX_CR_RPM_BIF_CHECKSUM
+*/
+#define RGX_CR_RPM_BIF_CHECKSUM                                                     (0xD1D8U)
+#define RGX_CR_RPM_BIF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_RPM_BIF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_RPM_BIF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHG_BIF_CHECKSUM
+*/
+#define RGX_CR_SHG_BIF_CHECKSUM                                                     (0xD1E0U)
+#define RGX_CR_SHG_BIF_CHECKSUM_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHG_BIF_CHECKSUM_VALUE_SHIFT                                         (0U)
+#define RGX_CR_SHG_BIF_CHECKSUM_VALUE_CLRMSK                                        (0x00000000U)
+
+
+/*
+    Register RGX_CR_SHG_FE_BE_CHECKSUM
+*/
+#define RGX_CR_SHG_FE_BE_CHECKSUM                                                   (0xD1E8U)
+#define RGX_CR_SHG_FE_BE_CHECKSUM_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_SHG_FE_BE_CHECKSUM_VALUE_SHIFT                                       (0U)
+#define RGX_CR_SHG_FE_BE_CHECKSUM_VALUE_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register DPX_CR_BF_PERF
+*/
+#define DPX_CR_BF_PERF                                                              (0xC458U)
+#define DPX_CR_BF_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BF_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_BF_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_BF_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_BF_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_BF_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_BF_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_BF_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_BF_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_BF_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_BF_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_BF_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_BF_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BF_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_BF_PERF_SELECT0
+*/
+#define DPX_CR_BF_PERF_SELECT0                                                      (0xC460U)
+#define DPX_CR_BF_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_BF_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_BF_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BF_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BF_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_BF_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BF_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_BF_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BF_PERF_COUNTER_0
+*/
+#define DPX_CR_BF_PERF_COUNTER_0                                                    (0xC488U)
+#define DPX_CR_BF_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BF_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_BF_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_BT_PERF
+*/
+#define DPX_CR_BT_PERF                                                              (0xC3D0U)
+#define DPX_CR_BT_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BT_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_BT_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_BT_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_BT_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_BT_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_BT_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_BT_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_BT_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_BT_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_BT_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_BT_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_BT_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BT_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_BT_PERF_SELECT0
+*/
+#define DPX_CR_BT_PERF_SELECT0                                                      (0xC3D8U)
+#define DPX_CR_BT_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_BT_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_BT_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BT_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BT_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_BT_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BT_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_BT_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BT_PERF_COUNTER_0
+*/
+#define DPX_CR_BT_PERF_COUNTER_0                                                    (0xC420U)
+#define DPX_CR_BT_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BT_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_BT_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_RQ_USC_DEBUG
+*/
+#define DPX_CR_RQ_USC_DEBUG                                                         (0xC110U)
+#define DPX_CR_RQ_USC_DEBUG_MASKFULL                                                (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RQ_USC_DEBUG_CHECKSUM_SHIFT                                          (0U)
+#define DPX_CR_RQ_USC_DEBUG_CHECKSUM_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register DPX_CR_BIF_FAULT_BANK_MMU_STATUS
+*/
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS                                            (0xC5C8U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_MASKFULL                                   (IMG_UINT64_C(0x000000000000F775))
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_CAT_BASE_SHIFT                             (12U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_CAT_BASE_CLRMSK                            (0xFFFF0FFFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_PAGE_SIZE_SHIFT                            (8U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_PAGE_SIZE_CLRMSK                           (0xFFFFF8FFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_DATA_TYPE_SHIFT                            (5U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_DATA_TYPE_CLRMSK                           (0xFFFFFF9FU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_SHIFT                             (4U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_CLRMSK                            (0xFFFFFFEFU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_RO_EN                                (0x00000010U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_SHIFT                     (2U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_CLRMSK                    (0xFFFFFFFBU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_PM_META_RO_EN                        (0x00000004U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_SHIFT                                (0U)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_CLRMSK                               (0xFFFFFFFEU)
+#define DPX_CR_BIF_FAULT_BANK_MMU_STATUS_FAULT_EN                                   (0x00000001U)
+
+
+/*
+    Register DPX_CR_BIF_FAULT_BANK_REQ_STATUS
+*/
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS                                            (0xC5D0U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_MASKFULL                                   (IMG_UINT64_C(0x03FFFFFFFFFFFFF0))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_SHIFT                                  (57U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_CLRMSK                                 (IMG_UINT64_C(0xFDFFFFFFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_RNW_EN                                     (IMG_UINT64_C(0x0200000000000000))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_SB_SHIFT                               (44U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_SB_CLRMSK                              (IMG_UINT64_C(0xFE000FFFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_ID_SHIFT                               (40U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_TAG_ID_CLRMSK                              (IMG_UINT64_C(0xFFFFF0FFFFFFFFFF))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_SHIFT                              (4U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_CLRMSK                             (IMG_UINT64_C(0xFFFFFF000000000F))
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_ALIGNSHIFT                         (4U)
+#define DPX_CR_BIF_FAULT_BANK_REQ_STATUS_ADDRESS_ALIGNSIZE                          (16U)
+
+
+/*
+    Register DPX_CR_BIF_MMU_STATUS
+*/
+#define DPX_CR_BIF_MMU_STATUS                                                       (0xC5D8U)
+#define DPX_CR_BIF_MMU_STATUS_MASKFULL                                              (IMG_UINT64_C(0x000000000FFFFFF7))
+#define DPX_CR_BIF_MMU_STATUS_PC_DATA_SHIFT                                         (20U)
+#define DPX_CR_BIF_MMU_STATUS_PC_DATA_CLRMSK                                        (0xF00FFFFFU)
+#define DPX_CR_BIF_MMU_STATUS_PD_DATA_SHIFT                                         (12U)
+#define DPX_CR_BIF_MMU_STATUS_PD_DATA_CLRMSK                                        (0xFFF00FFFU)
+#define DPX_CR_BIF_MMU_STATUS_PT_DATA_SHIFT                                         (4U)
+#define DPX_CR_BIF_MMU_STATUS_PT_DATA_CLRMSK                                        (0xFFFFF00FU)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_SHIFT                                         (2U)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_CLRMSK                                        (0xFFFFFFFBU)
+#define DPX_CR_BIF_MMU_STATUS_STALLED_EN                                            (0x00000004U)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_SHIFT                                          (1U)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_CLRMSK                                         (0xFFFFFFFDU)
+#define DPX_CR_BIF_MMU_STATUS_PAUSED_EN                                             (0x00000002U)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_SHIFT                                            (0U)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_BIF_MMU_STATUS_BUSY_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_RT_PERF
+*/
+#define DPX_CR_RT_PERF                                                              (0xC700U)
+#define DPX_CR_RT_PERF_MASKFULL                                                     (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_RT_PERF_CLR_3_SHIFT                                                  (4U)
+#define DPX_CR_RT_PERF_CLR_3_CLRMSK                                                 (0xFFFFFFEFU)
+#define DPX_CR_RT_PERF_CLR_3_EN                                                     (0x00000010U)
+#define DPX_CR_RT_PERF_CLR_2_SHIFT                                                  (3U)
+#define DPX_CR_RT_PERF_CLR_2_CLRMSK                                                 (0xFFFFFFF7U)
+#define DPX_CR_RT_PERF_CLR_2_EN                                                     (0x00000008U)
+#define DPX_CR_RT_PERF_CLR_1_SHIFT                                                  (2U)
+#define DPX_CR_RT_PERF_CLR_1_CLRMSK                                                 (0xFFFFFFFBU)
+#define DPX_CR_RT_PERF_CLR_1_EN                                                     (0x00000004U)
+#define DPX_CR_RT_PERF_CLR_0_SHIFT                                                  (1U)
+#define DPX_CR_RT_PERF_CLR_0_CLRMSK                                                 (0xFFFFFFFDU)
+#define DPX_CR_RT_PERF_CLR_0_EN                                                     (0x00000002U)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_SHIFT                                            (0U)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_CLRMSK                                           (0xFFFFFFFEU)
+#define DPX_CR_RT_PERF_CTRL_ENABLE_EN                                               (0x00000001U)
+
+
+/*
+    Register DPX_CR_RT_PERF_SELECT0
+*/
+#define DPX_CR_RT_PERF_SELECT0                                                      (0xC708U)
+#define DPX_CR_RT_PERF_SELECT0_MASKFULL                                             (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MAX_SHIFT                                      (48U)
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MAX_CLRMSK                                     (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MIN_SHIFT                                      (32U)
+#define DPX_CR_RT_PERF_SELECT0_BATCH_MIN_CLRMSK                                     (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_MODE_SHIFT                                           (21U)
+#define DPX_CR_RT_PERF_SELECT0_MODE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_RT_PERF_SELECT0_MODE_EN                                              (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_RT_PERF_SELECT0_GROUP_SELECT_SHIFT                                   (16U)
+#define DPX_CR_RT_PERF_SELECT0_GROUP_SELECT_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_RT_PERF_SELECT0_BIT_SELECT_SHIFT                                     (0U)
+#define DPX_CR_RT_PERF_SELECT0_BIT_SELECT_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_RT_PERF_COUNTER_0
+*/
+#define DPX_CR_RT_PERF_COUNTER_0                                                    (0xC730U)
+#define DPX_CR_RT_PERF_COUNTER_0_MASKFULL                                           (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RT_PERF_COUNTER_0_REG_SHIFT                                          (0U)
+#define DPX_CR_RT_PERF_COUNTER_0_REG_CLRMSK                                         (0x00000000U)
+
+
+/*
+    Register DPX_CR_BX_TU_PERF
+*/
+#define DPX_CR_BX_TU_PERF                                                           (0xC908U)
+#define DPX_CR_BX_TU_PERF_MASKFULL                                                  (IMG_UINT64_C(0x000000000000001F))
+#define DPX_CR_BX_TU_PERF_CLR_3_SHIFT                                               (4U)
+#define DPX_CR_BX_TU_PERF_CLR_3_CLRMSK                                              (0xFFFFFFEFU)
+#define DPX_CR_BX_TU_PERF_CLR_3_EN                                                  (0x00000010U)
+#define DPX_CR_BX_TU_PERF_CLR_2_SHIFT                                               (3U)
+#define DPX_CR_BX_TU_PERF_CLR_2_CLRMSK                                              (0xFFFFFFF7U)
+#define DPX_CR_BX_TU_PERF_CLR_2_EN                                                  (0x00000008U)
+#define DPX_CR_BX_TU_PERF_CLR_1_SHIFT                                               (2U)
+#define DPX_CR_BX_TU_PERF_CLR_1_CLRMSK                                              (0xFFFFFFFBU)
+#define DPX_CR_BX_TU_PERF_CLR_1_EN                                                  (0x00000004U)
+#define DPX_CR_BX_TU_PERF_CLR_0_SHIFT                                               (1U)
+#define DPX_CR_BX_TU_PERF_CLR_0_CLRMSK                                              (0xFFFFFFFDU)
+#define DPX_CR_BX_TU_PERF_CLR_0_EN                                                  (0x00000002U)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_SHIFT                                         (0U)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_CLRMSK                                        (0xFFFFFFFEU)
+#define DPX_CR_BX_TU_PERF_CTRL_ENABLE_EN                                            (0x00000001U)
+
+
+/*
+    Register DPX_CR_BX_TU_PERF_SELECT0
+*/
+#define DPX_CR_BX_TU_PERF_SELECT0                                                   (0xC910U)
+#define DPX_CR_BX_TU_PERF_SELECT0_MASKFULL                                          (IMG_UINT64_C(0x3FFF3FFF003FFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MAX_SHIFT                                   (48U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MAX_CLRMSK                                  (IMG_UINT64_C(0xC000FFFFFFFFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MIN_SHIFT                                   (32U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BATCH_MIN_CLRMSK                                  (IMG_UINT64_C(0xFFFFC000FFFFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_SHIFT                                        (21U)
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFDFFFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_MODE_EN                                           (IMG_UINT64_C(0x0000000000200000))
+#define DPX_CR_BX_TU_PERF_SELECT0_GROUP_SELECT_SHIFT                                (16U)
+#define DPX_CR_BX_TU_PERF_SELECT0_GROUP_SELECT_CLRMSK                               (IMG_UINT64_C(0xFFFFFFFFFFE0FFFF))
+#define DPX_CR_BX_TU_PERF_SELECT0_BIT_SELECT_SHIFT                                  (0U)
+#define DPX_CR_BX_TU_PERF_SELECT0_BIT_SELECT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register DPX_CR_BX_TU_PERF_COUNTER_0
+*/
+#define DPX_CR_BX_TU_PERF_COUNTER_0                                                 (0xC938U)
+#define DPX_CR_BX_TU_PERF_COUNTER_0_MASKFULL                                        (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_BX_TU_PERF_COUNTER_0_REG_SHIFT                                       (0U)
+#define DPX_CR_BX_TU_PERF_COUNTER_0_REG_CLRMSK                                      (0x00000000U)
+
+
+/*
+    Register DPX_CR_RS_PDS_RR_CHECKSUM
+*/
+#define DPX_CR_RS_PDS_RR_CHECKSUM                                                   (0xC0F0U)
+#define DPX_CR_RS_PDS_RR_CHECKSUM_MASKFULL                                          (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define DPX_CR_RS_PDS_RR_CHECKSUM_VALUE_SHIFT                                       (0U)
+#define DPX_CR_RS_PDS_RR_CHECKSUM_VALUE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFF00000000))
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING_CONTEXT
+*/
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT                                            (0xE140U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_MASKFULL                                   (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_SHIFT                                   (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_CONTEXT_ID_CLRMSK                                  (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MMU_CBASE_MAPPING
+*/
+#define RGX_CR_MMU_CBASE_MAPPING                                                    (0xE148U)
+#define RGX_CR_MMU_CBASE_MAPPING_MASKFULL                                           (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT                                    (0U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK                                   (0xF0000000U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT                               (12U)
+#define RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSIZE                                (4096U)
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS
+*/
+#define RGX_CR_MMU_FAULT_STATUS                                                     (0xE150U)
+#define RGX_CR_MMU_FAULT_STATUS_MASKFULL                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_ADDRESS_SHIFT                                       (28U)
+#define RGX_CR_MMU_FAULT_STATUS_ADDRESS_CLRMSK                                      (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_CONTEXT_SHIFT                                       (20U)
+#define RGX_CR_MMU_FAULT_STATUS_CONTEXT_CLRMSK                                      (IMG_UINT64_C(0xFFFFFFFFF00FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_TAG_SB_SHIFT                                        (12U)
+#define RGX_CR_MMU_FAULT_STATUS_TAG_SB_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFF00FFF))
+#define RGX_CR_MMU_FAULT_STATUS_REQ_ID_SHIFT                                        (6U)
+#define RGX_CR_MMU_FAULT_STATUS_REQ_ID_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_CR_MMU_FAULT_STATUS_LEVEL_SHIFT                                         (4U)
+#define RGX_CR_MMU_FAULT_STATUS_LEVEL_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_MMU_FAULT_STATUS_RNW_SHIFT                                           (3U)
+#define RGX_CR_MMU_FAULT_STATUS_RNW_CLRMSK                                          (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_RNW_EN                                              (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT                                          (1U)
+#define RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_SHIFT                                         (0U)
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_FAULT_EN                                            (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_MMU_FAULT_STATUS_META
+*/
+#define RGX_CR_MMU_FAULT_STATUS_META                                                (0xE158U)
+#define RGX_CR_MMU_FAULT_STATUS_META_MASKFULL                                       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT                                  (28U)
+#define RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK                                 (IMG_UINT64_C(0x000000000FFFFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT                                  (20U)
+#define RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFF00FFFFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_SHIFT                                   (12U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFF00FFF))
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT                                   (6U)
+#define RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT                                    (4U)
+#define RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFCF))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_SHIFT                                      (3U)
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_CLRMSK                                     (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_CR_MMU_FAULT_STATUS_META_RNW_EN                                         (IMG_UINT64_C(0x0000000000000008))
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT                                     (1U)
+#define RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFF9))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_SHIFT                                    (0U)
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN                                       (IMG_UINT64_C(0x0000000000000001))
+
+
+/*
+    Register RGX_CR_SLC3_CTRL_MISC
+*/
+#define RGX_CR_SLC3_CTRL_MISC                                                       (0xE200U)
+#define RGX_CR_SLC3_CTRL_MISC_MASKFULL                                              (IMG_UINT64_C(0x0000000000000107))
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_SHIFT                                  (8U)
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_CLRMSK                                 (0xFFFFFEFFU)
+#define RGX_CR_SLC3_CTRL_MISC_WRITE_COMBINER_EN                                     (0x00000100U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_SHIFT                                (0U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_CLRMSK                               (0xFFFFFFF8U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_LINEAR                               (0x00000000U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_IN_PAGE_HASH                         (0x00000001U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_FIXED_PVR_HASH                       (0x00000002U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_SCRAMBLE_PVR_HASH                    (0x00000003U)
+#define RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_WEAVED_HASH                          (0x00000004U)
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE
+*/
+#define RGX_CR_SLC3_SCRAMBLE                                                        (0xE208U)
+#define RGX_CR_SLC3_SCRAMBLE_MASKFULL                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE_BITS_SHIFT                                             (0U)
+#define RGX_CR_SLC3_SCRAMBLE_BITS_CLRMSK                                            (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE2
+*/
+#define RGX_CR_SLC3_SCRAMBLE2                                                       (0xE210U)
+#define RGX_CR_SLC3_SCRAMBLE2_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE2_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE2_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE3
+*/
+#define RGX_CR_SLC3_SCRAMBLE3                                                       (0xE218U)
+#define RGX_CR_SLC3_SCRAMBLE3_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE3_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE3_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_SCRAMBLE4
+*/
+#define RGX_CR_SLC3_SCRAMBLE4                                                       (0xE260U)
+#define RGX_CR_SLC3_SCRAMBLE4_MASKFULL                                              (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_SCRAMBLE4_BITS_SHIFT                                            (0U)
+#define RGX_CR_SLC3_SCRAMBLE4_BITS_CLRMSK                                           (IMG_UINT64_C(0x0000000000000000))
+
+
+/*
+    Register RGX_CR_SLC3_STATUS
+*/
+#define RGX_CR_SLC3_STATUS                                                          (0xE220U)
+#define RGX_CR_SLC3_STATUS_MASKFULL                                                 (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+#define RGX_CR_SLC3_STATUS_WRITES1_SHIFT                                            (48U)
+#define RGX_CR_SLC3_STATUS_WRITES1_CLRMSK                                           (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_SLC3_STATUS_WRITES0_SHIFT                                            (32U)
+#define RGX_CR_SLC3_STATUS_WRITES0_CLRMSK                                           (IMG_UINT64_C(0xFFFF0000FFFFFFFF))
+#define RGX_CR_SLC3_STATUS_READS1_SHIFT                                             (16U)
+#define RGX_CR_SLC3_STATUS_READS1_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFF0000FFFF))
+#define RGX_CR_SLC3_STATUS_READS0_SHIFT                                             (0U)
+#define RGX_CR_SLC3_STATUS_READS0_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFF0000))
+
+
+/*
+    Register RGX_CR_SLC3_IDLE
+*/
+#define RGX_CR_SLC3_IDLE                                                            (0xE228U)
+#define RGX_CR_SLC3_IDLE_MASKFULL                                                   (IMG_UINT64_C(0x00000000000FFFFF))
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST2_SHIFT                                         (18U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST2_CLRMSK                                        (0xFFF3FFFFU)
+#define RGX_CR_SLC3_IDLE_MMU_SHIFT                                                  (17U)
+#define RGX_CR_SLC3_IDLE_MMU_CLRMSK                                                 (0xFFFDFFFFU)
+#define RGX_CR_SLC3_IDLE_MMU_EN                                                     (0x00020000U)
+#define RGX_CR_SLC3_IDLE_RDI_SHIFT                                                  (16U)
+#define RGX_CR_SLC3_IDLE_RDI_CLRMSK                                                 (0xFFFEFFFFU)
+#define RGX_CR_SLC3_IDLE_RDI_EN                                                     (0x00010000U)
+#define RGX_CR_SLC3_IDLE_IMGBV4_SHIFT                                               (12U)
+#define RGX_CR_SLC3_IDLE_IMGBV4_CLRMSK                                              (0xFFFF0FFFU)
+#define RGX_CR_SLC3_IDLE_CACHE_BANKS_SHIFT                                          (4U)
+#define RGX_CR_SLC3_IDLE_CACHE_BANKS_CLRMSK                                         (0xFFFFF00FU)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST_SHIFT                                          (2U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_DUST_CLRMSK                                         (0xFFFFFFF3U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_SHIFT                                         (1U)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_SLC3_IDLE_ORDERQ_JONES_EN                                            (0x00000002U)
+#define RGX_CR_SLC3_IDLE_XBAR_SHIFT                                                 (0U)
+#define RGX_CR_SLC3_IDLE_XBAR_CLRMSK                                                (0xFFFFFFFEU)
+#define RGX_CR_SLC3_IDLE_XBAR_EN                                                    (0x00000001U)
+
+
+/*
+    Register RGX_CR_SLC3_FAULT_STOP_STATUS
+*/
+#define RGX_CR_SLC3_FAULT_STOP_STATUS                                               (0xE248U)
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_MASKFULL                                      (IMG_UINT64_C(0x0000000000001FFF))
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_BIF_SHIFT                                     (0U)
+#define RGX_CR_SLC3_FAULT_STOP_STATUS_BIF_CLRMSK                                    (0xFFFFE000U)
+
+
+/*
+    Register RGX_CR_VDM_CONTEXT_STORE_MODE
+*/
+#define RGX_CR_VDM_CONTEXT_STORE_MODE                                               (0xF048U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MASKFULL                                      (IMG_UINT64_C(0x0000000000000003))
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_SHIFT                                    (0U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_CLRMSK                                   (0xFFFFFFFCU)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_INDEX                                    (0x00000000U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_INSTANCE                                 (0x00000001U)
+#define RGX_CR_VDM_CONTEXT_STORE_MODE_MODE_LIST                                     (0x00000002U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING0
+*/
+#define RGX_CR_CONTEXT_MAPPING0                                                     (0xF078U)
+#define RGX_CR_CONTEXT_MAPPING0_MASKFULL                                            (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING0_2D_SHIFT                                            (24U)
+#define RGX_CR_CONTEXT_MAPPING0_2D_CLRMSK                                           (0x00FFFFFFU)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_SHIFT                                           (16U)
+#define RGX_CR_CONTEXT_MAPPING0_CDM_CLRMSK                                          (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING0_3D_SHIFT                                            (8U)
+#define RGX_CR_CONTEXT_MAPPING0_3D_CLRMSK                                           (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING0_TA_SHIFT                                            (0U)
+#define RGX_CR_CONTEXT_MAPPING0_TA_CLRMSK                                           (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING1
+*/
+#define RGX_CR_CONTEXT_MAPPING1                                                     (0xF080U)
+#define RGX_CR_CONTEXT_MAPPING1_MASKFULL                                            (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_CONTEXT_MAPPING1_HOST_SHIFT                                          (8U)
+#define RGX_CR_CONTEXT_MAPPING1_HOST_CLRMSK                                         (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING1_TLA_SHIFT                                           (0U)
+#define RGX_CR_CONTEXT_MAPPING1_TLA_CLRMSK                                          (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING2
+*/
+#define RGX_CR_CONTEXT_MAPPING2                                                     (0xF088U)
+#define RGX_CR_CONTEXT_MAPPING2_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING2_ALIST0_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING2_TE0_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING2_VCE0_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING3
+*/
+#define RGX_CR_CONTEXT_MAPPING3                                                     (0xF090U)
+#define RGX_CR_CONTEXT_MAPPING3_MASKFULL                                            (IMG_UINT64_C(0x0000000000FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_SHIFT                                        (16U)
+#define RGX_CR_CONTEXT_MAPPING3_ALIST1_CLRMSK                                       (0xFF00FFFFU)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_SHIFT                                           (8U)
+#define RGX_CR_CONTEXT_MAPPING3_TE1_CLRMSK                                          (0xFFFF00FFU)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_SHIFT                                          (0U)
+#define RGX_CR_CONTEXT_MAPPING3_VCE1_CLRMSK                                         (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_BIF_JONES_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ                                           (0xF098U)
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_MASKFULL                                  (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_COUNTER_SHIFT                             (0U)
+#define RGX_CR_BIF_JONES_OUTSTANDING_READ_COUNTER_CLRMSK                            (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ                                      (0xF0A0U)
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_MASKFULL                             (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_COUNTER_SHIFT                        (0U)
+#define RGX_CR_BIF_BLACKPEARL_OUTSTANDING_READ_COUNTER_CLRMSK                       (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_BIF_DUST_OUTSTANDING_READ
+*/
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ                                            (0xF0A8U)
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_MASKFULL                                   (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_COUNTER_SHIFT                              (0U)
+#define RGX_CR_BIF_DUST_OUTSTANDING_READ_COUNTER_CLRMSK                             (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_JONES_FIX
+*/
+#define RGX_CR_JONES_FIX                                                            (0xF0C0U)
+#define RGX_CR_JONES_FIX__ROGUE3__MASKFULL                                          (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX_MASKFULL                                                   (IMG_UINT64_C(0x000000000000FFFF))
+#define RGX_CR_JONES_FIX_DISABLE_SHIFT                                              (0U)
+#define RGX_CR_JONES_FIX_DISABLE_CLRMSK                                             (0xFFFF0000U)
+
+
+/*
+    Register RGX_CR_CONTEXT_MAPPING4
+*/
+#define RGX_CR_CONTEXT_MAPPING4                                                     (0xF210U)
+#define RGX_CR_CONTEXT_MAPPING4_MASKFULL                                            (IMG_UINT64_C(0x0000FFFFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_SHIFT                                  (40U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFF00FFFFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_SHIFT                                    (32U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFF00FFFFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_SHIFT                                     (24U)
+#define RGX_CR_CONTEXT_MAPPING4_3D_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFF00FFFFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_SHIFT                                  (16U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_MMU_STACK_CLRMSK                                 (IMG_UINT64_C(0xFFFFFFFFFF00FFFF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_SHIFT                                    (8U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_UFSTACK_CLRMSK                                   (IMG_UINT64_C(0xFFFFFFFFFFFF00FF))
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_SHIFT                                     (0U)
+#define RGX_CR_CONTEXT_MAPPING4_TA_FSTACK_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFF00))
+
+
+/*
+    Register RGX_CR_MULTICORE_GPU
+*/
+#define RGX_CR_MULTICORE_GPU                                                        (0xF300U)
+#define RGX_CR_MULTICORE_GPU_MASKFULL                                               (IMG_UINT64_C(0x000000000000007F))
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_SHIFT                              (6U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_CLRMSK                             (0xFFFFFFBFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_EN                                 (0x00000040U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_SHIFT                              (5U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_CLRMSK                             (0xFFFFFFDFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_EN                                 (0x00000020U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_SHIFT                               (4U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_CLRMSK                              (0xFFFFFFEFU)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_EN                                  (0x00000010U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_SHIFT                               (3U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_CLRMSK                              (0xFFFFFFF7U)
+#define RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN                                  (0x00000008U)
+#define RGX_CR_MULTICORE_GPU_ID_SHIFT                                               (0U)
+#define RGX_CR_MULTICORE_GPU_ID_CLRMSK                                              (0xFFFFFFF8U)
+
+
+/*
+    Register RGX_CR_MULTICORE_SYSTEM
+*/
+#define RGX_CR_MULTICORE_SYSTEM                                                     (0xF308U)
+#define RGX_CR_MULTICORE_SYSTEM_MASKFULL                                            (IMG_UINT64_C(0x000000000000000F))
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT                                     (0U)
+#define RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_CLRMSK                                    (0xFFFFFFF0U)
+
+
+/*
+    Register RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON                                       (0xF310U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT          (8U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK         (0xC00000FFU)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_FRAGMENT_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON                                       (0xF320U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_MASKFULL                              (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                   (30U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                  (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT          (8U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK         (0xC00000FFU)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_GPU_ENABLE_SHIFT                      (0U)
+#define RGX_CR_MULTICORE_GEOMETRY_CTRL_COMMON_GPU_ENABLE_CLRMSK                     (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON
+*/
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON                                        (0xF330U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_MASKFULL                               (IMG_UINT64_C(0x00000000FFFFFFFF))
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_SHIFT                    (30U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_TYPE_CLRMSK                   (0x3FFFFFFFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_SHIFT           (8U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_WORKLOAD_EXECUTE_COUNT_CLRMSK          (0xC00000FFU)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_SHIFT                       (0U)
+#define RGX_CR_MULTICORE_COMPUTE_CTRL_COMMON_GPU_ENABLE_CLRMSK                      (0xFFFFFF00U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_ERR_INJ
+*/
+#define RGX_CR_ECC_RAM_ERR_INJ                                                      (0xF340U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MASKFULL                                             (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_SHIFT                                   (4U)
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_CLRMSK                                  (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_ERR_INJ_SLC_SIDEKICK_EN                                      (0x00000010U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_SHIFT                                            (3U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_CLRMSK                                           (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_ERR_INJ_USC_EN                                               (0x00000008U)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_SHIFT                                     (2U)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_CLRMSK                                    (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_ERR_INJ_TPU_MCU_L0_EN                                        (0x00000004U)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_SHIFT                                         (1U)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_CLRMSK                                        (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_ERR_INJ_RASCAL_EN                                            (0x00000002U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_SHIFT                                           (0U)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_CLRMSK                                          (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_ERR_INJ_MARS_EN                                              (0x00000001U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_INIT_KICK
+*/
+#define RGX_CR_ECC_RAM_INIT_KICK                                                    (0xF348U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MASKFULL                                           (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_SHIFT                                 (4U)
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_INIT_KICK_SLC_SIDEKICK_EN                                    (0x00000010U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_SHIFT                                          (3U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_INIT_KICK_USC_EN                                             (0x00000008U)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_SHIFT                                   (2U)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_INIT_KICK_TPU_MCU_L0_EN                                      (0x00000004U)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_SHIFT                                       (1U)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_INIT_KICK_RASCAL_EN                                          (0x00000002U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_SHIFT                                         (0U)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_INIT_KICK_MARS_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_ECC_RAM_INIT_DONE
+*/
+#define RGX_CR_ECC_RAM_INIT_DONE                                                    (0xF350U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MASKFULL                                           (IMG_UINT64_C(0x000000000000001F))
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_SHIFT                                 (4U)
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_CLRMSK                                (0xFFFFFFEFU)
+#define RGX_CR_ECC_RAM_INIT_DONE_SLC_SIDEKICK_EN                                    (0x00000010U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_SHIFT                                          (3U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_CLRMSK                                         (0xFFFFFFF7U)
+#define RGX_CR_ECC_RAM_INIT_DONE_USC_EN                                             (0x00000008U)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_SHIFT                                   (2U)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_CLRMSK                                  (0xFFFFFFFBU)
+#define RGX_CR_ECC_RAM_INIT_DONE_TPU_MCU_L0_EN                                      (0x00000004U)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_SHIFT                                       (1U)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_CLRMSK                                      (0xFFFFFFFDU)
+#define RGX_CR_ECC_RAM_INIT_DONE_RASCAL_EN                                          (0x00000002U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_SHIFT                                         (0U)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_CLRMSK                                        (0xFFFFFFFEU)
+#define RGX_CR_ECC_RAM_INIT_DONE_MARS_EN                                            (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_ENABLE
+*/
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE                                         (0xF390U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL                               (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_SHIFT                       (7U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_CLRMSK                      (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_EN                          (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_SHIFT                   (6U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_EN                      (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                (5U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK               (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_EN                   (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                 (4U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_EN                    (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_SHIFT                         (3U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_CLRMSK                        (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_EN                            (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_SHIFT                         (2U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_EN                            (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_SHIFT                        (1U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_EN                           (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_SHIFT                   (0U)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_STATUS
+*/
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE                                         (0xF398U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__MASKFULL                               (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_SHIFT                       (7U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_CLRMSK                      (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_LOCKUP_EN                          (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_SHIFT                   (6U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__CPU_PAGE_FAULT_EN                      (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                (5U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK               (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__SAFE_COMPUTE_FAIL_EN                   (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                 (4U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_EN                    (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_SHIFT                         (3U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_CLRMSK                        (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__TRP_FAIL_EN                            (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_SHIFT                         (2U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_CLRMSK                        (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_EN                            (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_SHIFT                        (1U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_CLRMSK                       (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_GPU_EN                           (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_SHIFT                   (0U)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                  (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__GPU_PAGE_FAULT_EN                      (0x00000001U)
+
+
+/*
+    Register RGX_CR_SAFETY_EVENT_CLEAR
+*/
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE                                          (0xF3A0U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__MASKFULL                                (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_SHIFT                        (7U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_CLRMSK                       (0xFFFFFF7FU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_LOCKUP_EN                           (0x00000080U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_SHIFT                    (6U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_CLRMSK                   (0xFFFFFFBFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__CPU_PAGE_FAULT_EN                       (0x00000040U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT                 (5U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK                (0xFFFFFFDFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__SAFE_COMPUTE_FAIL_EN                    (0x00000020U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT                  (4U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK                 (0xFFFFFFEFU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__WATCHDOG_TIMEOUT_EN                     (0x00000010U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_SHIFT                          (3U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_CLRMSK                         (0xFFFFFFF7U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__TRP_FAIL_EN                             (0x00000008U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_SHIFT                          (2U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_CLRMSK                         (0xFFFFFFFBU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_FW_EN                             (0x00000004U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_SHIFT                         (1U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_CLRMSK                        (0xFFFFFFFDU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__FAULT_GPU_EN                            (0x00000002U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_SHIFT                    (0U)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_CLRMSK                   (0xFFFFFFFEU)
+#define RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE__GPU_PAGE_FAULT_EN                       (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_STATUS
+*/
+#define RGX_CR_FAULT_FW_STATUS                                                      (0xF3B0U)
+#define RGX_CR_FAULT_FW_STATUS_MASKFULL                                             (IMG_UINT64_C(0x0000000000010001))
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_SHIFT                                    (16U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_CLRMSK                                   (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_EN                                       (0x00010000U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_SHIFT                                     (0U)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_CLRMSK                                    (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_STATUS_CPU_DETECT_EN                                        (0x00000001U)
+
+
+/*
+    Register RGX_CR_FAULT_FW_CLEAR
+*/
+#define RGX_CR_FAULT_FW_CLEAR                                                       (0xF3B8U)
+#define RGX_CR_FAULT_FW_CLEAR_MASKFULL                                              (IMG_UINT64_C(0x0000000000010001))
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_SHIFT                                     (16U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_CLRMSK                                    (0xFFFEFFFFU)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_CORRECT_EN                                        (0x00010000U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_SHIFT                                      (0U)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_CLRMSK                                     (0xFFFFFFFEU)
+#define RGX_CR_FAULT_FW_CLEAR_CPU_DETECT_EN                                         (0x00000001U)
+
+
+/*
+    Register RGX_CR_MTS_SAFETY_EVENT_ENABLE
+*/
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE                                     (0xF3D8U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL                           (IMG_UINT64_C(0x00000000000000FF))
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_SHIFT                   (7U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_CLRMSK                  (0xFFFFFF7FU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_LOCKUP_EN                      (0x00000080U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_SHIFT               (6U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_CLRMSK              (0xFFFFFFBFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__CPU_PAGE_FAULT_EN                  (0x00000040U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_SHIFT            (5U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_CLRMSK           (0xFFFFFFDFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__SAFE_COMPUTE_FAIL_EN               (0x00000020U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT             (4U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_CLRMSK            (0xFFFFFFEFU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__WATCHDOG_TIMEOUT_EN                (0x00000010U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_SHIFT                     (3U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_CLRMSK                    (0xFFFFFFF7U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__TRP_FAIL_EN                        (0x00000008U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_SHIFT                     (2U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_CLRMSK                    (0xFFFFFFFBU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_FW_EN                        (0x00000004U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_SHIFT                    (1U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_CLRMSK                   (0xFFFFFFFDU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__FAULT_GPU_EN                       (0x00000002U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_SHIFT               (0U)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_CLRMSK              (0xFFFFFFFEU)
+#define RGX_CR_MTS_SAFETY_EVENT_ENABLE__ROGUEXE__GPU_PAGE_FAULT_EN                  (0x00000001U)
+
+
+#endif /* RGX_CR_DEFS_KM_H */
+/*****************************************************************************
+ End of file (rgx_cr_defs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/km/rgxdefs_km.h b/drivers/gpu/drm/img-rogue/km/rgxdefs_km.h
new file mode 100644
index 000000000000..dbaaa5faaf97
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km/rgxdefs_km.h
@@ -0,0 +1,377 @@
+/*************************************************************************/ /*!
+@Title          Rogue hw definitions (kernel mode)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXDEFS_KM_H
+#define RGXDEFS_KM_H
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+#include RGX_BVNC_CORE_KM_HEADER
+#include RGX_BNC_CONFIG_KM_HEADER
+#endif
+
+#define IMG_EXPLICIT_INCLUDE_HWDEFS
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#include "rgx_cr_defs_km.h"
+#endif
+#undef IMG_EXPLICIT_INCLUDE_HWDEFS
+
+#include "rgx_heap_firmware.h"
+
+/* The following Macros are picked up through BVNC headers for no hardware
+ * operations to be compatible with old build infrastructure.
+ */
+#if defined(NO_HARDWARE)
+/******************************************************************************
+ * Check for valid B.X.N.C
+ *****************************************************************************/
+#if !defined(RGX_BVNC_KM_B) || !defined(RGX_BVNC_KM_V) || !defined(RGX_BVNC_KM_N) || !defined(RGX_BVNC_KM_C)
+#error "Need to specify BVNC (RGX_BVNC_KM_B, RGX_BVNC_KM_V, RGX_BVNC_KM_N and RGX_BVNC_C)"
+#endif
+
+/* Check core/config compatibility */
+#if (RGX_BVNC_KM_B != RGX_BNC_KM_B) || (RGX_BVNC_KM_N != RGX_BNC_KM_N) || (RGX_BVNC_KM_C != RGX_BNC_KM_C)
+#error "BVNC headers are mismatching (KM core/config)"
+#endif
+#endif
+
+/******************************************************************************
+ * RGX Version name
+ *****************************************************************************/
+#define RGX_BVNC_KM_ST2(S)	#S
+#define RGX_BVNC_KM_ST(S)	RGX_BVNC_KM_ST2(S)
+#define RGX_BVNC_KM			RGX_BVNC_KM_ST(RGX_BVNC_KM_B) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_V) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_N) "." RGX_BVNC_KM_ST(RGX_BVNC_KM_C)
+#define RGX_BVNC_KM_V_ST	RGX_BVNC_KM_ST(RGX_BVNC_KM_V)
+
+/* Maximum string size is [bb.vvvp.nnnn.cccc\0], includes null char */
+#define RGX_BVNC_STR_SIZE_MAX (2U+1U+4U+1U+4U+1U+4U+1U)
+#define RGX_BVNC_STR_FMTSPEC  "%u.%u.%u.%u"
+#define RGX_BVNC_STRP_FMTSPEC "%u.%up.%u.%u"
+
+
+/******************************************************************************
+ * RGX Defines
+ *****************************************************************************/
+
+#define BVNC_FIELD_MASK     ((1UL << BVNC_FIELD_WIDTH) - 1U)
+#define C_POSITION          (0U)
+#define N_POSITION          ((C_POSITION) + (BVNC_FIELD_WIDTH))
+#define V_POSITION          ((N_POSITION) + (BVNC_FIELD_WIDTH))
+#define B_POSITION          ((V_POSITION) + (BVNC_FIELD_WIDTH))
+
+#define B_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (B_POSITION)))
+#define V_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (V_POSITION)))
+#define N_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (N_POSITION)))
+#define C_POSTION_MASK      (((IMG_UINT64)(BVNC_FIELD_MASK) << (C_POSITION)))
+
+#define GET_B(x)            (((x) & (B_POSTION_MASK)) >> (B_POSITION))
+#define GET_V(x)            (((x) & (V_POSTION_MASK)) >> (V_POSITION))
+#define GET_N(x)            (((x) & (N_POSTION_MASK)) >> (N_POSITION))
+#define GET_C(x)            (((x) & (C_POSTION_MASK)) >> (C_POSITION))
+
+#define BVNC_PACK(B,V,N,C)  (((((IMG_UINT64)(B))) << (B_POSITION)) | \
+                             ((((IMG_UINT64)(V))) << (V_POSITION)) | \
+                             ((((IMG_UINT64)(N))) << (N_POSITION)) | \
+                             ((((IMG_UINT64)(C))) << (C_POSITION)) \
+                            )
+
+#define RGX_CR_CORE_ID_CONFIG_N_SHIFT    (8U)
+#define RGX_CR_CORE_ID_CONFIG_C_SHIFT    (0U)
+
+#define RGX_CR_CORE_ID_CONFIG_N_CLRMSK   (0XFFFF00FFU)
+#define RGX_CR_CORE_ID_CONFIG_C_CLRMSK   (0XFFFFFF00U)
+
+#if defined(RGX_CR_CORE_ID__PBVNC)
+#define GET_PBVNC_B(X)	((X & ~RGX_CR_CORE_ID__PBVNC__BRANCH_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__BRANCH_ID_SHIFT)
+#define GET_PBVNC_V(X)	((X & ~RGX_CR_CORE_ID__PBVNC__VERSION_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__VERSION_ID_SHIFT)
+#define GET_PBVNC_N(X)	((X & ~RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__NUMBER_OF_SCALABLE_UNITS_SHIFT)
+#define GET_PBVNC_C(X)	((X & ~RGX_CR_CORE_ID__PBVNC__CONFIG_ID_CLRMSK) >> RGX_CR_CORE_ID__PBVNC__CONFIG_ID_SHIFT)
+#endif
+
+#if defined(RGX_FEATURE_NUM_OSIDS)
+#define RGXFW_MAX_NUM_OSIDS                               (RGX_FEATURE_NUM_OSIDS)
+#else
+#define RGXFW_MAX_NUM_OSIDS                               (8U)
+#endif
+
+#define RGXFW_HOST_DRIVER_ID                              (0U)
+#define RGXFW_GUEST_DRIVER_ID_START                       (RGXFW_HOST_DRIVER_ID + 1U)
+
+#define RGXFW_THREAD_0                                    (0U)
+#define RGXFW_THREAD_1                                    (1U)
+
+/* META cores (required for the RGX_FEATURE_META) */
+#define MTP218   (1U)
+#define MTP219   (2U)
+#define LTP218   (3U)
+#define LTP217   (4U)
+
+/* META Core memory feature depending on META variants */
+#define RGX_META_COREMEM_32K      (32*1024)
+#define RGX_META_COREMEM_48K      (48*1024)
+#define RGX_META_COREMEM_64K      (64*1024)
+#define RGX_META_COREMEM_96K      (96*1024)
+#define RGX_META_COREMEM_128K     (128*1024)
+#define RGX_META_COREMEM_256K     (256*1024)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+#if (!defined(SUPPORT_TRUSTED_DEVICE) || defined(RGX_FEATURE_META_DMA)) && \
+    (defined(RGX_FEATURE_META_COREMEM_SIZE) && RGX_FEATURE_META_COREMEM_SIZE != 0)
+#define RGX_META_COREMEM_SIZE     (RGX_FEATURE_META_COREMEM_SIZE*1024U)
+#define RGX_META_COREMEM          (1)
+#define RGX_META_COREMEM_CODE     (1)
+#if !defined(FIX_HW_BRN_50767) && (!defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1))
+#define RGX_META_COREMEM_DATA     (1)
+#endif
+#else
+#undef SUPPORT_META_COREMEM
+#undef RGX_FEATURE_META_COREMEM_SIZE
+#undef RGX_FEATURE_META_DMA
+#define RGX_FEATURE_META_COREMEM_SIZE (0)
+#define RGX_META_COREMEM_SIZE         (0)
+#endif
+#endif
+
+#define GET_ROGUE_CACHE_LINE_SIZE(x)    ((((IMG_UINT32)(x)) > 0U) ? ((IMG_UINT32)(x)/8U) : (0U))
+
+#if defined(RGX_FEATURE_META_DMA)
+#define RGX_META_DMA_BLOCK_SIZE (32U)
+#else
+#define RGX_META_DMA_BLOCK_SIZE (0U)
+#endif
+
+#if defined(SUPPORT_AGP)
+#define MAX_HW_TA3DCONTEXTS	3U
+#else
+#define MAX_HW_TA3DCONTEXTS	2U
+#endif
+
+#define RGX_CR_CLK_CTRL_ALL_ON          (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL_MASKFULL)
+#define RGX_CR_CLK_CTRL_ALL_AUTO        (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_ON         (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_CTRL2_ALL_AUTO       (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_CTRL2_MASKFULL)
+#define RGX_CR_CLK_XTPLUS_CTRL_ALL_ON   (IMG_UINT64_C(0x5555555555555555)&RGX_CR_CLK_XTPLUS_CTRL_MASKFULL)
+#define RGX_CR_CLK_XTPLUS_CTRL_ALL_AUTO (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&RGX_CR_CLK_XTPLUS_CTRL_MASKFULL)
+#define DPX_CR_DPX_CLK_CTRL_ALL_ON      (IMG_UINT64_C(0x5555555555555555)&DPX_CR_DPX_CLK_CTRL_MASKFULL)
+#define DPX_CR_DPX_CLK_CTRL_ALL_AUTO    (IMG_UINT64_C(0xaaaaaaaaaaaaaaaa)&DPX_CR_DPX_CLK_CTRL_MASKFULL)
+
+#define RGX_CR_SOFT_RESET_DUST_n_CORE_EN	(RGX_CR_SOFT_RESET_DUST_A_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_B_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_C_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_D_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_E_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_F_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_G_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_H_CORE_EN)
+
+/* SOFT_RESET Rascal and DUSTs bits */
+#define RGX_CR_SOFT_RESET_RASCALDUSTS_EN	(RGX_CR_SOFT_RESET_RASCAL_CORE_EN | \
+											 RGX_CR_SOFT_RESET_DUST_n_CORE_EN)
+
+
+
+
+/* SOFT_RESET steps as defined in the TRM */
+#define RGX_S7_SOFT_RESET_DUSTS (RGX_CR_SOFT_RESET_DUST_n_CORE_EN)
+
+#define RGX_S7_SOFT_RESET_JONES (RGX_CR_SOFT_RESET_PM_EN  | \
+                                 RGX_CR_SOFT_RESET_VDM_EN | \
+                                 RGX_CR_SOFT_RESET_ISP_EN)
+
+#define RGX_S7_SOFT_RESET_JONES_ALL (RGX_S7_SOFT_RESET_JONES  | \
+                                     RGX_CR_SOFT_RESET_BIF_EN | \
+                                     RGX_CR_SOFT_RESET_SLC_EN | \
+                                     RGX_CR_SOFT_RESET_GARTEN_EN)
+
+#define RGX_S7_SOFT_RESET2 (RGX_CR_SOFT_RESET2_BLACKPEARL_EN | \
+                            RGX_CR_SOFT_RESET2_PIXEL_EN | \
+                            RGX_CR_SOFT_RESET2_CDM_EN | \
+                            RGX_CR_SOFT_RESET2_VERTEX_EN)
+
+
+/* PM interactive mode enabled default for all rogue cores */
+#define PM_INTERACTIVE_MODE
+
+#define RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT		(12U)
+#define RGX_BIF_PM_PHYSICAL_PAGE_SIZE			(1UL << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT		(14U)
+#define RGX_BIF_PM_VIRTUAL_PAGE_SIZE			(1UL << RGX_BIF_PM_VIRTUAL_PAGE_ALIGNSHIFT)
+
+#define RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE	(16U)
+
+/* To get the number of required Dusts, divide the number of
+ * clusters by 2 and round up
+ */
+#define RGX_REQ_NUM_DUSTS(CLUSTERS)    (((CLUSTERS) + 1U) / 2U)
+
+/* To get the number of required Bernado/Phantom(s), divide
+ * the number of clusters by 4 and round up
+ */
+#define RGX_REQ_NUM_PHANTOMS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+#define RGX_REQ_NUM_BERNADOS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+#define RGX_REQ_NUM_BLACKPEARLS(CLUSTERS) (((CLUSTERS) + 3U) / 4U)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+# define RGX_NUM_PHANTOMS (RGX_REQ_NUM_PHANTOMS(RGX_FEATURE_NUM_CLUSTERS))
+#endif
+
+
+/* META second thread feature depending on META variants and
+ * available CoreMem
+ */
+#if defined(RGX_FEATURE_META) && (RGX_FEATURE_META == MTP218 || RGX_FEATURE_META == MTP219) && defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && (RGX_FEATURE_META_COREMEM_SIZE == 256)
+#define RGXFW_META_SUPPORT_2ND_THREAD
+#endif
+
+/*
+ * Utility macros to calculate CAT_BASE register addresses
+ */
+#define BIF_CAT_BASEx(n) \
+	(RGX_CR_BIF_CAT_BASE0 + ((n) * (RGX_CR_BIF_CAT_BASE1 - RGX_CR_BIF_CAT_BASE0)))
+
+#define FWCORE_MEM_CAT_BASEx(n) \
+	(RGX_CR_FWCORE_MEM_CAT_BASE0 + ((n) * (RGX_CR_FWCORE_MEM_CAT_BASE1 - RGX_CR_FWCORE_MEM_CAT_BASE0)))
+
+/*
+ * FWCORE wrapper register defines
+ */
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT   RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT
+#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK  RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK
+#define FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT     (12U)
+
+/******************************************************************************
+ * WA HWBRNs
+ *****************************************************************************/
+
+#if defined(RGX_CR_JONES_IDLE_MASKFULL)
+/* Workaround for HW BRN 57289 */
+#if (RGX_CR_JONES_IDLE_MASKFULL != 0x0000000000007FFF)
+#error This WA must be updated if RGX_CR_JONES_IDLE is expanded!!!
+#endif
+#undef RGX_CR_JONES_IDLE_MASKFULL
+#undef RGX_CR_JONES_IDLE_TDM_SHIFT
+#undef RGX_CR_JONES_IDLE_TDM_CLRMSK
+#undef RGX_CR_JONES_IDLE_TDM_EN
+#define RGX_CR_JONES_IDLE_MASKFULL                        (IMG_UINT64_C(0x0000000000003FFF))
+#endif
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK))
+
+#if defined(RGX_FEATURE_ROGUEXE)
+#define RGX_NUM_RASTERISATION_MODULES	RGX_FEATURE_NUM_CLUSTERS
+#else
+#define RGX_NUM_RASTERISATION_MODULES	RGX_NUM_PHANTOMS
+#endif
+
+#endif /* !(defined(__KERNEL__) || defined(TEE_DDK)) */
+
+/* GPU CR timer tick in GPU cycles */
+#define RGX_CRTIME_TICK_IN_CYCLES					(256U)
+#define RGX_CRTIME_TICK_IN_CYCLES_SHIFT				(8U)
+
+/* for nohw multicore return max cores possible to client */
+#define RGX_MULTICORE_MAX_NOHW_CORES               (4U)
+
+/*
+  If the size of the SLC is less than this value then the TPU bypasses the SLC.
+ */
+#define RGX_TPU_CACHED_SLC_SIZE_THRESHOLD_KB			(128U)
+
+/*
+ * If the size of the SLC is bigger than this value then the TCU must not be bypassed in the SLC.
+ * In XE_MEMORY_HIERARCHY cores, the TCU is bypassed by default.
+ */
+#define RGX_TCU_CACHED_SLC_SIZE_THRESHOLD_KB			(32U)
+
+/*
+ * Register used by the FW to track the current boot stage (not used in MIPS)
+ */
+#define RGX_FW_BOOT_STAGE_REGISTER     (RGX_CR_POWER_ESTIMATE_RESULT)
+
+/*
+ * Virtualisation definitions
+ */
+#define RGX_VIRTUALISATION_REG_SIZE_PER_OS (RGX_CR_MTS_SCHEDULE1 - RGX_CR_MTS_SCHEDULE)
+
+/*
+ * Renaming MTS sideband bitfields to emphasize that the Register Bank number
+ * of the MTS register used identifies a specific Driver/VM rather than the OSID tag
+ * emitted on bus memory transactions.
+ */
+#define RGX_MTS_SBDATA_DRIVERID_CLRMSK RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_CLRMSK
+#define RGX_MTS_SBDATA_DRIVERID_SHIFT RGX_CR_MTS_BGCTX_SBDATA0_OS_ID_SHIFT
+
+/*
+ * Register Bank containing registers secured against host access
+ */
+#define RGX_HOST_SECURE_REGBANK_OFFSET				(0xF0000U)
+#define RGX_HOST_SECURE_REGBANK_SIZE				(0x10000U)
+
+/*
+	Maximum number of render targets in array
+*/
+#define RGX_MAX_TA_RENDER_TARGETS					(2048U)
+
+/*
+ * Macro used to indicate which version of HWPerf is active
+ */
+#define RGX_FEATURE_HWPERF_ROGUE
+
+/*
+ * Maximum number of cores supported by TRP
+ */
+#define RGX_TRP_MAX_NUM_CORES                           (4U)
+
+/*
+ * Maximum number of cores supported by WGP
+ */
+#define RGX_WGP_MAX_NUM_CORES                           (8U)
+
+#if defined(RGX_FEATURE_VOLCANIC_TB)
+#define SUPPORT_VOLCANIC_TB
+#endif
+#define RGX_FEATURE_SECURITY_ROGUE
+
+/* Typically the PCI bus returns this value on error */
+#define RGX_PCI_ERROR_VALUE_BYTE (0xFFU)
+#define RGX_PCI_ERROR_VALUE_DWORD (0xFFFFFFFFU)
+
+#endif /* RGXDEFS_KM_H */
diff --git a/drivers/gpu/drm/img-rogue/km/rgxmhdefs_km.h b/drivers/gpu/drm/img-rogue/km/rgxmhdefs_km.h
new file mode 100644
index 000000000000..663001dd9ebc
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km/rgxmhdefs_km.h
@@ -0,0 +1,286 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmhdefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_mh.def
+ */
+
+
+#ifndef RGXMHDEFS_KM_H
+#define RGXMHDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMHDEFS_KM_REVISION 0
+
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_FENCE                            (0x00000000U)
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_CONTEXT                          (0x00000001U)
+#define RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_QUEUE                            (0x00000002U)
+
+
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTL_STREAM                       (0x00000000U)
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTX_BUFFER                       (0x00000001U)
+#define RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_QUEUE_CTL                        (0x00000002U)
+
+
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAFSTACK                              (0x00000008U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMLIST                               (0x00000009U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DFSTACK                              (0x0000000aU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMLIST                               (0x0000000bU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX0                                (0x0000000cU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX1                                (0x0000002dU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_UFSTACK                               (0x0000000fU)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMMUSTACK                            (0x00000012U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMMUSTACK                            (0x00000013U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAUFSTACK                             (0x00000016U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DUFSTACK                             (0x00000017U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DVFP                                 (0x00000019U)
+#define RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAVFP                                 (0x0000001aU)
+
+
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAFSTACK                              (0x00000000U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMLIST                               (0x00000001U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DFSTACK                              (0x00000002U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMLIST                               (0x00000003U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX0                                (0x00000004U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX1                                (0x00000025U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_MAVP                                  (0x00000006U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_UFSTACK                               (0x00000007U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMMUSTACK                            (0x00000008U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMMUSTACK                            (0x00000009U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAUFSTACK                             (0x00000014U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DUFSTACK                             (0x00000015U)
+#define RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAVFP                                 (0x00000018U)
+
+
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PPP                                        (0x00000008U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_VCERTC                                     (0x00000007U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_TEACRTC                                    (0x00000006U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGRTC                                     (0x00000005U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGR                                       (0x00000004U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_PSGS                                       (0x00000003U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_TPC                                        (0x00000002U)
+#define RGX_MH_TAG_SB_TA_ENCODING_TA_TAG_VCE                                        (0x00000001U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_PREQ00                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_PREQ01                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_RREQ                     (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_DBSC                     (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_CPF                      (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_2_ENCODING_IPF_TAG_DELTA                    (0x00000007U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ02                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CREQ03                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ00                   (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ01                   (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ02                   (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_PREQ03                   (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_RREQ                     (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_DBSC                     (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_CPF                      (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_4_ENCODING_IPF_TAG_DELTA                    (0x0000000bU)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ00                   (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ01                   (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ02                   (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ03                   (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ04                   (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ05                   (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CREQ06                   (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ00                   (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ01                   (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ02                   (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ03                   (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ04                   (0x0000000bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ05                   (0x0000000cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_PREQ06                   (0x0000000dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_RREQ                     (0x0000000eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_DBSC                     (0x0000000fU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_CPF                      (0x00000010U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_7_ENCODING_IPF_TAG_DELTA                    (0x00000011U)
+
+
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ00                  (0x00000000U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ01                  (0x00000001U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ02                  (0x00000002U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ03                  (0x00000003U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ04                  (0x00000004U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ05                  (0x00000005U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ06                  (0x00000006U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ07                  (0x00000007U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ08                  (0x00000008U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ09                  (0x00000009U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ10                  (0x0000000aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ11                  (0x0000000bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ12                  (0x0000000cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CREQ13                  (0x0000000dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ00                  (0x0000000eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ01                  (0x0000000fU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ02                  (0x00000010U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ03                  (0x00000011U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ04                  (0x00000012U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ05                  (0x00000013U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ06                  (0x00000014U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ07                  (0x00000015U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ08                  (0x00000016U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ09                  (0x00000017U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ10                  (0x00000018U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ11                  (0x00000019U)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ12                  (0x0000001aU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_PREQ13                  (0x0000001bU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_RREQ                    (0x0000001cU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_DBSC                    (0x0000001dU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_CPF                     (0x0000001eU)
+#define RGX_MH_TAG_SB_IPF_IPF_NUM_PIPES_14_ENCODING_IPF_TAG_DELTA                   (0x0000001fU)
+
+
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_PDS_STATE                                (0x00000000U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DEPTH_BIAS                               (0x00000001U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_FLOOR_OFFSET_DATA                        (0x00000002U)
+#define RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DELTA_DATA                               (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_ISP_ENCODING_ISP_TAG_ZLS                                      (0x00000000U)
+#define RGX_MH_TAG_SB_ISP_ENCODING_ISP_TAG_DS                                       (0x00000001U)
+
+
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTROL                                  (0x00000000U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STATE                                    (0x00000001U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_INDEX                                    (0x00000002U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STACK                                    (0x00000004U)
+#define RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTEXT                                  (0x00000008U)
+
+
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTROL_STREAM                           (0x00000000U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_INDIRECT_DATA                            (0x00000001U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_EVENT_DATA                               (0x00000002U)
+#define RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTEXT_STATE                            (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_OPCODE_FETCH                           (0x00000002U)
+#define RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_DATA_ACCESS                            (0x00000003U)
+
+
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PT_REQUEST                               (0x00000000U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PD_REQUEST                               (0x00000001U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PC_REQUEST                               (0x00000002U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PT_REQUEST                            (0x00000003U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_REQUEST                            (0x00000004U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_REQUEST                            (0x00000005U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_WREQUEST                           (0x00000006U)
+#define RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_WREQUEST                           (0x00000007U)
+
+
+#define RGX_MH_TAG_ENCODING_MH_TAG_MMU                                              (0x00000000U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU                                          (0x00000001U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU                                          (0x00000002U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU                                          (0x00000003U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MIPS                                             (0x00000004U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0                                         (0x00000005U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1                                         (0x00000006U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2                                         (0x00000007U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3                                         (0x00000008U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0                                         (0x00000009U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1                                         (0x0000000aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2                                         (0x0000000bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3                                         (0x0000000cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4                                         (0x0000000dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PDS_0                                            (0x0000000eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PDS_1                                            (0x0000000fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCA                                         (0x00000010U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCB                                         (0x00000011U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCC                                         (0x00000012U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCD                                         (0x00000013U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCA                                     (0x00000014U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCB                                     (0x00000015U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCC                                     (0x00000016U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCD                                     (0x00000017U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDSRW                                        (0x00000018U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TCU_0                                            (0x00000019U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TCU_1                                            (0x0000001aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_0                                          (0x0000001bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_1                                          (0x0000001cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_2                                          (0x0000001dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_3                                          (0x0000001eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_USC                                              (0x0000001fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_ISP_ZLS                                          (0x00000020U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_ISP_DS                                           (0x00000021U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF                                              (0x00000022U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS                                     (0x00000023U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF                                          (0x00000024U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ                                         (0x00000025U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS                                       (0x00000026U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG5                                         (0x00000027U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PPP                                           (0x00000028U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPWRTC                                        (0x00000029U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TEACRTC                                       (0x0000002aU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGRTC                                        (0x0000002bU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGREGION                                     (0x0000002cU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGSTREAM                                     (0x0000002dU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPW                                           (0x0000002eU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TA_TPC                                           (0x0000002fU)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PM_ALLOC                                         (0x00000030U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PM_DEALLOC                                       (0x00000031U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TDM_DMA                                          (0x00000032U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_TDM_CTL                                          (0x00000033U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE0                                             (0x00000034U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE1                                             (0x00000035U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE2                                             (0x00000036U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_PBE3                                             (0x00000037U)
+
+
+#endif /* RGXMHDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmhdefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/km/rgxmmudefs_km.h b/drivers/gpu/drm/img-rogue/km/rgxmmudefs_km.h
new file mode 100644
index 000000000000..14a6a73c84bc
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km/rgxmmudefs_km.h
@@ -0,0 +1,216 @@
+/*************************************************************************/ /*!
+@Title          Hardware definition file rgxmmudefs_km.h
+@Brief          The file contains auto-generated hardware definitions without
+                BVNC-specific compile time conditionals.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/*               ****   Autogenerated C -- do not edit    ****               */
+
+/*
+ *      rogue_bif.def
+ */
+
+
+#ifndef RGXMMUDEFS_KM_H
+#define RGXMMUDEFS_KM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+
+
+#define RGXMMUDEFS_KM_REVISION 0
+
+#define RGX_BIF_DM_ENCODING_VERTEX                                                  (0x00000000U)
+#define RGX_BIF_DM_ENCODING_PIXEL                                                   (0x00000001U)
+#define RGX_BIF_DM_ENCODING_COMPUTE                                                 (0x00000002U)
+#define RGX_BIF_DM_ENCODING_TLA                                                     (0x00000003U)
+#define RGX_BIF_DM_ENCODING_PB_VCE                                                  (0x00000004U)
+#define RGX_BIF_DM_ENCODING_PB_TE                                                   (0x00000005U)
+#define RGX_BIF_DM_ENCODING_META                                                    (0x00000007U)
+#define RGX_BIF_DM_ENCODING_HOST                                                    (0x00000008U)
+#define RGX_BIF_DM_ENCODING_PM_ALIST                                                (0x00000009U)
+
+
+#define RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT                                            (30U)
+#define RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF003FFFFFFF))
+#define RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT                                            (21U)
+#define RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFC01FFFFF))
+#define RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT                                            (12U)
+#define RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFE00FFF))
+
+
+#define RGX_MMUCTRL_ENTRIES_PC_VALUE                                                (0x00000400U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PD_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRIES_PT_VALUE                                                (0x00000200U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PC_VALUE                                             (0x00000020U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PD_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_ENTRY_SIZE_PT_VALUE                                             (0x00000040U)
+
+
+#define RGX_MMUCTRL_PAGE_SIZE_MASK                                                  (0x00000007U)
+#define RGX_MMUCTRL_PAGE_SIZE_4KB                                                   (0x00000000U)
+#define RGX_MMUCTRL_PAGE_SIZE_16KB                                                  (0x00000001U)
+#define RGX_MMUCTRL_PAGE_SIZE_64KB                                                  (0x00000002U)
+#define RGX_MMUCTRL_PAGE_SIZE_256KB                                                 (0x00000003U)
+#define RGX_MMUCTRL_PAGE_SIZE_1MB                                                   (0x00000004U)
+#define RGX_MMUCTRL_PAGE_SIZE_2MB                                                   (0x00000005U)
+
+
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT                                            (12U)
+#define RGX_MMUCTRL_PAGE_4KB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_SHIFT                                           (14U)
+#define RGX_MMUCTRL_PAGE_16KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF0000003FFF))
+
+
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_SHIFT                                           (16U)
+#define RGX_MMUCTRL_PAGE_64KB_RANGE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_SHIFT                                          (18U)
+#define RGX_MMUCTRL_PAGE_256KB_RANGE_CLRMSK                                         (IMG_UINT64_C(0xFFFFFF000003FFFF))
+
+
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_SHIFT                                            (20U)
+#define RGX_MMUCTRL_PAGE_1MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00000FFFFF))
+
+
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_SHIFT                                            (21U)
+#define RGX_MMUCTRL_PAGE_2MB_RANGE_CLRMSK                                           (IMG_UINT64_C(0xFFFFFF00001FFFFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_SHIFT                                         (12U)
+#define RGX_MMUCTRL_PT_BASE_4KB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF0000000FFF))
+
+
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_SHIFT                                        (10U)
+#define RGX_MMUCTRL_PT_BASE_16KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000003FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_SHIFT                                        (8U)
+#define RGX_MMUCTRL_PT_BASE_64KB_RANGE_CLRMSK                                       (IMG_UINT64_C(0xFFFFFF00000000FF))
+
+
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_SHIFT                                       (6U)
+#define RGX_MMUCTRL_PT_BASE_256KB_RANGE_CLRMSK                                      (IMG_UINT64_C(0xFFFFFF000000003F))
+
+
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_1MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_SHIFT                                         (5U)
+#define RGX_MMUCTRL_PT_BASE_2MB_RANGE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFF000000001F))
+
+
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_SHIFT                                   (62U)
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_CLRMSK                                  (IMG_UINT64_C(0xBFFFFFFFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN                                      (IMG_UINT64_C(0x4000000000000000))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_SHIFT                                        (40U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_HI_CLRMSK                                       (IMG_UINT64_C(0xC00000FFFFFFFFFF))
+#define RGX_MMUCTRL_PT_DATA_PAGE_SHIFT                                              (12U)
+#define RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK                                             (IMG_UINT64_C(0xFFFFFF0000000FFF))
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_SHIFT                                        (6U)
+#define RGX_MMUCTRL_PT_DATA_VP_PAGE_LO_CLRMSK                                       (IMG_UINT64_C(0xFFFFFFFFFFFFF03F))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_SHIFT                                     (5U)
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFFFFFFFFFFDF))
+#define RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000000000000020))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_SHIFT                                            (4U)
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_CLRMSK                                           (IMG_UINT64_C(0xFFFFFFFFFFFFFFEF))
+#define RGX_MMUCTRL_PT_DATA_PM_SRC_EN                                               (IMG_UINT64_C(0x0000000000000010))
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_SHIFT                                   (3U)
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_CLRMSK                                  (IMG_UINT64_C(0xFFFFFFFFFFFFFFF7))
+#define RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN                                      (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PT_DATA_CC_SHIFT                                                (2U)
+#define RGX_MMUCTRL_PT_DATA_CC_CLRMSK                                               (IMG_UINT64_C(0xFFFFFFFFFFFFFFFB))
+#define RGX_MMUCTRL_PT_DATA_CC_EN                                                   (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFFD))
+#define RGX_MMUCTRL_PT_DATA_READ_ONLY_EN                                            (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PT_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PT_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PT_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_SHIFT                                     (40U)
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_CLRMSK                                    (IMG_UINT64_C(0xFFFFFEFFFFFFFFFF))
+#define RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN                                        (IMG_UINT64_C(0x0000010000000000))
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_SHIFT                                           (5U)
+#define RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK                                          (IMG_UINT64_C(0xFFFFFF000000001F))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_SHIFT                                         (1U)
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK                                        (IMG_UINT64_C(0xFFFFFFFFFFFFFFF1))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB                                           (IMG_UINT64_C(0x0000000000000000))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB                                          (IMG_UINT64_C(0x0000000000000002))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB                                          (IMG_UINT64_C(0x0000000000000004))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB                                         (IMG_UINT64_C(0x0000000000000006))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB                                           (IMG_UINT64_C(0x0000000000000008))
+#define RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB                                           (IMG_UINT64_C(0x000000000000000a))
+#define RGX_MMUCTRL_PD_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PD_DATA_VALID_CLRMSK                                            (IMG_UINT64_C(0xFFFFFFFFFFFFFFFE))
+#define RGX_MMUCTRL_PD_DATA_VALID_EN                                                (IMG_UINT64_C(0x0000000000000001))
+
+
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT                                           (4U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK                                          (0x0000000FU)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT                                      (12U)
+#define RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSIZE                                       (4096U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_SHIFT                                     (1U)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_CLRMSK                                    (0xFFFFFFFDU)
+#define RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN                                        (0x00000002U)
+#define RGX_MMUCTRL_PC_DATA_VALID_SHIFT                                             (0U)
+#define RGX_MMUCTRL_PC_DATA_VALID_CLRMSK                                            (0xFFFFFFFEU)
+#define RGX_MMUCTRL_PC_DATA_VALID_EN                                                (0x00000001U)
+
+
+#endif /* RGXMMUDEFS_KM_H */
+/*****************************************************************************
+ End of file (rgxmmudefs_km.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/km_apphint.c b/drivers/gpu/drm/img-rogue/km_apphint.c
new file mode 100644
index 000000000000..6e9bfd6f07d4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km_apphint.c
@@ -0,0 +1,1767 @@
+/*************************************************************************/ /*!
+@File           km_apphint.c
+@Title          Apphint routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "di_server.h"
+#include "pvr_uaccess.h"
+#include <linux/moduleparam.h>
+#include <linux/workqueue.h>
+#include <linux/string.h>
+
+/* Common and SO layer */
+#include "img_defs.h"
+#include "sofunc_pvr.h"
+
+/* for action device access */
+#include "pvrsrv.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+#include "rgxutils.h"
+#include "rgxapi_km.h"
+
+
+/* defines for default values */
+#include "rgx_fwif_km.h"
+#include "htbuffer_types.h"
+
+#include "pvr_notifier.h"
+
+#include "km_apphint_defs.h"
+#include "km_apphint.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#include "pdump_km.h"
+#endif
+
+/* Size of temporary buffers used to read and write AppHint data.
+ * Must be large enough to contain any strings read or written but no larger
+ * than 4096: which is the buffer size for the kernel_param_ops .get
+ * function. And less than 1024 to keep the stack frame size within bounds.
+ */
+#define APPHINT_BUFFER_SIZE 512
+
+/* Apphint Debug output level */
+#define APPHINT_DPF_LEVEL PVR_DBG_VERBOSE
+
+/*
+*******************************************************************************
+ * AppHint mnemonic data type helper tables
+******************************************************************************/
+struct apphint_lookup {
+	const char *name;
+	int value;
+};
+
+static const struct apphint_lookup fwt_logtype_tbl[] = {
+	{ "trace", 0},
+	{ "none", 0}
+#if defined(SUPPORT_TBI_INTERFACE)
+	, { "tbi", 1}
+#endif
+};
+
+static const struct apphint_lookup fwt_loggroup_tbl[] = {
+	RGXFWIF_LOG_GROUP_NAME_VALUE_MAP
+};
+
+static const struct apphint_lookup htb_loggroup_tbl[] = {
+#define X(a, b) { #b, HTB_LOG_GROUP_FLAG(a) },
+	HTB_LOG_SFGROUPLIST
+#undef X
+};
+
+static const struct apphint_lookup htb_opmode_tbl[] = {
+	{ "droplatest", HTB_OPMODE_DROPLATEST},
+	{ "dropoldest", HTB_OPMODE_DROPOLDEST},
+	{ "block", HTB_OPMODE_BLOCK}
+};
+
+__maybe_unused
+static const struct apphint_lookup htb_logmode_tbl[] = {
+	{ "all", HTB_LOGMODE_ALLPID},
+	{ "restricted", HTB_LOGMODE_RESTRICTEDPID}
+};
+
+__maybe_unused
+static const struct apphint_lookup timecorr_clk_tbl[] = {
+	{ "mono", 0 },
+	{ "mono_raw", 1 },
+	{ "sched", 2 }
+};
+
+/*
+*******************************************************************************
+ Data types
+******************************************************************************/
+union apphint_value {
+	IMG_UINT64 UINT64;
+	IMG_UINT32 UINT32;
+	IMG_BOOL BOOL;
+	IMG_CHAR *STRING;
+};
+
+union apphint_query_action {
+	PVRSRV_ERROR (*UINT64)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT64 *value);
+	PVRSRV_ERROR (*UINT32)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT32 *value);
+	PVRSRV_ERROR (*BOOL)(const PVRSRV_DEVICE_NODE *device,
+	                     const void *private_data, IMG_BOOL *value);
+	PVRSRV_ERROR (*STRING)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_CHAR **value);
+};
+
+union apphint_set_action {
+	PVRSRV_ERROR (*UINT64)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT64 value);
+	PVRSRV_ERROR (*UINT32)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_UINT32 value);
+	PVRSRV_ERROR (*BOOL)(const PVRSRV_DEVICE_NODE *device,
+	                     const void *private_data, IMG_BOOL value);
+	PVRSRV_ERROR (*STRING)(const PVRSRV_DEVICE_NODE *device,
+	                       const void *private_data, IMG_CHAR *value);
+};
+
+struct apphint_action {
+	union apphint_query_action query; /*!< Query callbacks. */
+	union apphint_set_action set;     /*!< Set callbacks. */
+	const PVRSRV_DEVICE_NODE *device; /*!< Pointer to the device node.*/
+	const void *private_data;         /*!< Opaque data passed to `query` and
+	                                       `set` callbacks. */
+	union apphint_value stored;       /*!< Value of the AppHint. */
+	bool free;                        /*!< Flag indicating that memory has been
+	                                       allocated for this AppHint and it
+	                                       needs to be freed on deinit. */
+	bool initialised;                 /*!< Flag indicating if the AppHint has
+	                                       been already initialised. */
+};
+
+struct apphint_param {
+	IMG_UINT32 id;
+	APPHINT_DATA_TYPE data_type;
+	const void *data_type_helper;
+	IMG_UINT32 helper_size;
+};
+
+struct apphint_init_data {
+	IMG_UINT32 id;          /* index into AppHint Table */
+	APPHINT_CLASS class;
+	const IMG_CHAR *name;
+	union apphint_value default_value;
+	APPHINT_RT_CLASS guest; /* ALWAYS => present on GUEST,
+	                           NEVER => not present on GUEST */
+};
+
+struct apphint_init_data_mapping {
+	IMG_UINT32 device_apphint_id;
+	IMG_UINT32 modparam_apphint_id;
+};
+
+struct apphint_class_state {
+	APPHINT_CLASS class;
+	IMG_BOOL enabled;
+};
+
+struct apphint_work {
+	struct work_struct work;
+	union apphint_value new_value;
+	struct apphint_action *action;
+};
+
+/*
+*******************************************************************************
+ Initialization / configuration table data
+******************************************************************************/
+#define UINT32Bitfield UINT32
+#define UINT32List UINT32
+
+static const struct apphint_init_data init_data_buildvar[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+};
+
+static const struct apphint_init_data init_data_modparam[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+};
+
+static const struct apphint_init_data init_data_debuginfo[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+};
+
+static const struct apphint_init_data init_data_debuginfo_device[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_CLASS_ ## c, #a, {.b=d}, APPHINT_RT_CLASS_ ## f },
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+};
+
+static const struct apphint_init_data_mapping init_data_debuginfo_device_to_modparams[] = {
+#define X(a, b) \
+	{APPHINT_ID_ ## a, APPHINT_ID_ ## b},
+	APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT_COMMON
+	APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT
+#undef X
+};
+
+#undef UINT32Bitfield
+#undef UINT32List
+
+__maybe_unused static const char NO_PARAM_TABLE[] = {};
+
+static const struct apphint_param param_lookup[] = {
+#define X(a, b, c, d, e, f) \
+	{APPHINT_ID_ ## a, APPHINT_DATA_TYPE_ ## b, e, ARRAY_SIZE(e) },
+	APPHINT_LIST_ALL
+#undef X
+};
+
+static const struct apphint_class_state class_state[] = {
+#define X(a) {APPHINT_CLASS_ ## a, APPHINT_ENABLED_CLASS_ ## a},
+	APPHINT_CLASS_LIST
+#undef X
+};
+
+/*
+*******************************************************************************
+ Global state
+******************************************************************************/
+/* If the union apphint_value becomes such that it is not possible to read
+ * and write atomically, a mutex may be desirable to prevent a read returning
+ * a partially written state.
+ * This would require a statically initialized mutex outside of the
+ * struct apphint_state to prevent use of an uninitialized mutex when
+ * module_params are provided on the command line.
+ *     static DEFINE_MUTEX(apphint_mutex);
+ */
+static struct apphint_state
+{
+	struct workqueue_struct *workqueue;
+	DI_GROUP *debuginfo_device_rootdir[PVRSRV_MAX_DEVICES];
+	DI_ENTRY *debuginfo_device_entry[PVRSRV_MAX_DEVICES][APPHINT_DEBUGINFO_DEVICE_ID_MAX];
+	DI_GROUP *debuginfo_rootdir;
+	DI_ENTRY *debuginfo_entry[APPHINT_DEBUGINFO_ID_MAX];
+	DI_GROUP *buildvar_rootdir;
+	DI_ENTRY *buildvar_entry[APPHINT_BUILDVAR_ID_MAX];
+
+	unsigned int num_devices;
+	PVRSRV_DEVICE_NODE *devices[PVRSRV_MAX_DEVICES];
+	unsigned int initialized;
+
+	/* Array contains value space for 1 copy of all apphint values defined
+	 * (for device 1) and N copies of device specific apphint values for
+	 * multi-device platforms.
+	 */
+	struct apphint_action val[APPHINT_ID_MAX + ((PVRSRV_MAX_DEVICES-1)*APPHINT_DEBUGINFO_DEVICE_ID_MAX)];
+
+} apphint = {
+/* statically initialise default values to ensure that any module_params
+ * provided on the command line are not overwritten by defaults.
+ */
+	.val = {
+#define UINT32Bitfield UINT32
+#define UINT32List UINT32
+#define X(a, b, c, d, e, f) \
+	{ {NULL}, {NULL}, NULL, NULL, {.b=d}, NULL },
+	APPHINT_LIST_ALL
+#undef X
+#undef UINT32Bitfield
+#undef UINT32List
+	},
+	.initialized = 0,
+	.num_devices = 0
+};
+
+#define APPHINT_DEBUGINFO_DEVICE_ID_OFFSET (APPHINT_ID_MAX-APPHINT_DEBUGINFO_DEVICE_ID_MAX)
+
+static inline void
+get_apphint_id_from_action_addr(const struct apphint_action * const addr,
+                                APPHINT_ID * const id)
+{
+	*id = (APPHINT_ID)(addr - apphint.val);
+	if (*id >= APPHINT_ID_MAX) {
+		*id -= APPHINT_DEBUGINFO_DEVICE_ID_OFFSET;
+		*id %= APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+		*id += APPHINT_DEBUGINFO_DEVICE_ID_OFFSET;
+	}
+}
+
+static inline void
+get_value_offset_from_device(const PVRSRV_DEVICE_NODE * const device,
+                             int * const offset,
+                             APPHINT_ID id)
+{
+	int i;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	/* No device offset if not a device specific apphint */
+	if (APPHINT_OF_DRIVER_NO_DEVICE == device) {
+		*offset = 0;
+		return;
+	}
+
+	/* Check that the specified ID is a device-specific one. If not we
+	 * set the offset to 0 for the global MODPARAM / BUILDVAR etc. AppHint
+	 */
+	for (i = 0; i < ARRAY_SIZE(init_data_debuginfo_device); i++)
+	{
+		const struct apphint_init_data *device_init = &init_data_debuginfo_device[i];
+
+		if ((IMG_UINT32)id == device_init->id) {
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!bFound) {
+		*offset = 0;
+		return;
+	}
+
+	for (i = 0; device && i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i] == device)
+			break;
+	}
+	if (PVRSRV_MAX_DEVICES == i) {
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unregistered device", __func__));
+		i = 0;
+	}
+	*offset = i * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+}
+
+/**
+ * apphint_action_worker - perform an action after an AppHint update has been
+ *                    requested by a UM process
+ *                    And update the record of the current active value
+ */
+static void apphint_action_worker(struct work_struct *work)
+{
+	struct apphint_work *work_pkt = container_of(work,
+	                                             struct apphint_work,
+	                                             work);
+	struct apphint_action *a = work_pkt->action;
+	union apphint_value value = work_pkt->new_value;
+	APPHINT_ID id;
+	PVRSRV_ERROR result = PVRSRV_OK;
+
+	get_apphint_id_from_action_addr(a, &id);
+
+	if (a->set.UINT64) {
+		switch (param_lookup[id].data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			result = a->set.UINT64(a->device,
+			                       a->private_data,
+			                       value.UINT64);
+			break;
+
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+		case APPHINT_DATA_TYPE_UINT32List:
+			result = a->set.UINT32(a->device,
+			                       a->private_data,
+			                       value.UINT32);
+			break;
+
+		case APPHINT_DATA_TYPE_BOOL:
+			result = a->set.BOOL(a->device,
+			                     a->private_data,
+			                     value.BOOL);
+			break;
+
+		case APPHINT_DATA_TYPE_STRING:
+			result = a->set.STRING(a->device,
+								   a->private_data,
+								   value.STRING);
+			kfree(value.STRING);
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: unrecognised data type (%d), index (%d)",
+			         __func__, param_lookup[id].data_type, id));
+		}
+
+		if (PVRSRV_OK != result) {
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: failed (%s)",
+			         __func__, PVRSRVGetErrorString(result)));
+		}
+	} else {
+		if (a->free) {
+			kfree(a->stored.STRING);
+		}
+		a->stored = value;
+		if (param_lookup[id].data_type == APPHINT_DATA_TYPE_STRING) {
+			a->free = true;
+		}
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "%s: AppHint value updated before handler is registered, ID(%d)",
+		         __func__, id));
+	}
+	kfree((void *)work_pkt);
+}
+
+static void apphint_action(union apphint_value new_value,
+                           struct apphint_action *action)
+{
+	struct apphint_work *work_pkt = kmalloc(sizeof(*work_pkt), GFP_KERNEL);
+
+	/* queue apphint update on a serialized workqueue to avoid races */
+	if (work_pkt) {
+		work_pkt->new_value = new_value;
+		work_pkt->action = action;
+		INIT_WORK(&work_pkt->work, apphint_action_worker);
+		if (0 == queue_work(apphint.workqueue, &work_pkt->work)) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to queue apphint change request",
+				__func__));
+			goto err_exit;
+		}
+	} else {
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: failed to alloc memory for apphint change request",
+			__func__));
+		goto err_exit;
+	}
+	return;
+err_exit:
+	kfree(new_value.STRING);
+}
+
+/**
+ * apphint_read - read the different AppHint data types
+ * return -errno or the buffer size
+ */
+static int apphint_read(char *buffer, size_t count, APPHINT_ID ue,
+			 union apphint_value *value)
+{
+	APPHINT_DATA_TYPE data_type = param_lookup[ue].data_type;
+	int result = 0;
+
+	switch (data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		if (kstrtou64(buffer, 0, &value->UINT64) < 0) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid UINT64 input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32:
+		if (kstrtou32(buffer, 0, &value->UINT32) < 0) {
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid UINT32 input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_BOOL:
+		switch (buffer[0]) {
+		case '0':
+		case 'n':
+		case 'N':
+		case 'f':
+		case 'F':
+			value->BOOL = IMG_FALSE;
+			break;
+		case '1':
+		case 'y':
+		case 'Y':
+		case 't':
+		case 'T':
+			value->BOOL = IMG_TRUE;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid BOOL input data for id %d: %s",
+				__func__, ue, buffer));
+			result = -EINVAL;
+			goto err_exit;
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32List:
+	{
+		int i;
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *)
+			param_lookup[ue].data_type_helper;
+		int size = param_lookup[ue].helper_size;
+		/* buffer may include '\n', remove it */
+		char *arg = strsep(&buffer, "\n");
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < size; i++) {
+			if (strcasecmp(lookup[i].name, arg) == 0) {
+				value->UINT32 = lookup[i].value;
+				break;
+			}
+		}
+		if (i == size) {
+			if (OSStringLength(arg) == 0) {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: No value set for AppHint",
+					__func__));
+			} else {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unrecognised AppHint value (%s)",
+					__func__, arg));
+			}
+			result = -EINVAL;
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	{
+		int i;
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *)
+			param_lookup[ue].data_type_helper;
+		int size = param_lookup[ue].helper_size;
+		/* buffer may include '\n', remove it */
+		char *string = strsep(&buffer, "\n");
+		char *token = strsep(&string, ",");
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		value->UINT32 = 0;
+		/* empty string is valid to clear the bitfield */
+		while (token && *token) {
+			for (i = 0; i < size; i++) {
+				if (strcasecmp(lookup[i].name, token) == 0) {
+					value->UINT32 |= lookup[i].value;
+					break;
+				}
+			}
+			if (i == size) {
+				PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unrecognised AppHint value (%s)",
+					__func__, token));
+				result = -EINVAL;
+				goto err_exit;
+			}
+			token = strsep(&string, ",");
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_STRING:
+	{
+		/* buffer may include '\n', remove it */
+		char *string = strsep(&buffer, "\n");
+		size_t len = OSStringLength(string);
+
+		if (!len) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		++len;
+
+		value->STRING = kmalloc(len , GFP_KERNEL);
+		if (!value->STRING) {
+			result = -ENOMEM;
+			goto err_exit;
+		}
+
+		OSStringSafeCopy(value->STRING, string, len);
+		break;
+	}
+	default:
+		result = -EINVAL;
+		goto err_exit;
+	}
+
+err_exit:
+	return (result < 0) ? result : count;
+}
+
+static PVRSRV_ERROR get_apphint_value_from_action(const struct apphint_action * const action,
+                                                  union apphint_value * const value,
+                                                  const PVRSRV_DEVICE_NODE * const psDevNode)
+{
+	APPHINT_ID id;
+	APPHINT_DATA_TYPE data_type;
+	PVRSRV_ERROR result = PVRSRV_OK;
+	const PVRSRV_DEVICE_NODE *psDevice;
+
+	get_apphint_id_from_action_addr(action, &id);
+	data_type = param_lookup[id].data_type;
+
+	/* If we've got an entry that is APPHINT_OF_DRIVER_NO_DEVICE we should use
+	 * the higher-level psDevNode value instead. This is the device-node that is
+	 * associated with the original debug_dump request.
+	 * Note: if we're called with psDevNode == APPHINT_OF_DRIVER_NO_DEVICE
+	 * we attempt to use the first registered apphint.devices[0] (if any
+	 * devices have been presented). If we have no devices hooked into the
+	 * apphint mechanism we just return the default value for the AppHint.
+	 */
+	if (psDevNode == APPHINT_OF_DRIVER_NO_DEVICE) {
+		if (action->device == APPHINT_OF_DRIVER_NO_DEVICE) {
+			if (apphint.num_devices > 0) {
+				psDevice = apphint.devices[0];
+			} else {
+				PVR_DPF((PVR_DBG_ERROR,
+				        "Uninitialised AppHint device for AppHint index (%d)",
+				        id));
+				return PVRSRV_ERROR_RETRY;
+			}
+		} else {
+			psDevice = action->device;
+		}
+	} else {
+		if (action->device == APPHINT_OF_DRIVER_NO_DEVICE) {
+			if (psDevNode != action->device) {
+				return PVRSRV_ERROR_INVALID_DEVICE;
+			}
+			psDevice = psDevNode;
+		} else {
+			psDevice = action->device;
+		}
+	}
+
+	if (action->query.UINT64) {
+		switch (data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			result = action->query.UINT64(psDevice,
+			                              action->private_data,
+			                              &value->UINT64);
+			break;
+
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+		case APPHINT_DATA_TYPE_UINT32List:
+			result = action->query.UINT32(psDevice,
+			                              action->private_data,
+			                              &value->UINT32);
+			break;
+
+		case APPHINT_DATA_TYPE_BOOL:
+			result = action->query.BOOL(psDevice,
+			                            action->private_data,
+			                            &value->BOOL);
+			break;
+
+		case APPHINT_DATA_TYPE_STRING:
+			result = action->query.STRING(psDevice,
+			                              action->private_data,
+			                              &value->STRING);
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: unrecognised data type (%d), index (%d)",
+			         __func__, data_type, id));
+		}
+	} else {
+		*value = action->stored;
+	}
+
+	if (PVRSRV_OK != result) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed (%d), index (%d)", __func__, result, id));
+	}
+
+	return result;
+}
+
+/*
+ * apphint_write - write the current AppHint data to a buffer
+ *
+ * Returns length written or -errno
+ */
+static int apphint_write(char *buffer, const size_t size,
+                         const struct apphint_action *a)
+{
+	const struct apphint_param *hint;
+	int result = 0;
+	APPHINT_ID id;
+	union apphint_value value;
+
+	get_apphint_id_from_action_addr(a, &id);
+	hint = &param_lookup[id];
+
+	result = get_apphint_value_from_action(a, &value, a->device);
+
+	switch (hint->data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		result += snprintf(buffer + result, size - result,
+				"0x%016llx",
+				value.UINT64);
+		break;
+	case APPHINT_DATA_TYPE_UINT32:
+		result += snprintf(buffer + result, size - result,
+				"0x%08x",
+				value.UINT32);
+		break;
+	case APPHINT_DATA_TYPE_BOOL:
+		result += snprintf(buffer + result, size - result,
+			"%s",
+			value.BOOL ? "Y" : "N");
+		break;
+	case APPHINT_DATA_TYPE_STRING:
+		if (value.STRING) {
+			result += snprintf(buffer + result, size - result,
+				"%s",
+				*value.STRING ? value.STRING : "(none)");
+		} else {
+			result += snprintf(buffer + result, size - result,
+			"(none)");
+		}
+		break;
+	case APPHINT_DATA_TYPE_UINT32List:
+	{
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *) hint->data_type_helper;
+		IMG_UINT32 i;
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < hint->helper_size; i++) {
+			if (lookup[i].value == value.UINT32) {
+				result += snprintf(buffer + result,
+						size - result,
+						"%s",
+						lookup[i].name);
+				break;
+			}
+		}
+		break;
+	}
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	{
+		struct apphint_lookup *lookup =
+			(struct apphint_lookup *) hint->data_type_helper;
+		IMG_UINT32 i;
+
+		if (lookup == (struct apphint_lookup *)NO_PARAM_TABLE) {
+			result = -EINVAL;
+			goto err_exit;
+		}
+
+		for (i = 0; i < hint->helper_size; i++) {
+			if (lookup[i].value & value.UINT32) {
+				result += snprintf(buffer + result,
+						size - result,
+						"%s,",
+						lookup[i].name);
+			}
+		}
+		if (result) {
+			/* remove any trailing ',' */
+			--result;
+			*(buffer + result) = '\0';
+		} else {
+			result += snprintf(buffer + result,
+					size - result, "none");
+		}
+		break;
+	}
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: unrecognised data type (%d), index (%d)",
+			 __func__, hint->data_type, id));
+		result = -EINVAL;
+	}
+
+err_exit:
+	return result;
+}
+
+/*
+*******************************************************************************
+ Module parameters initialization - different from debuginfo
+******************************************************************************/
+/*
+ * apphint_kparam_set - Handle an update of a module parameter
+ *
+ * Returns 0, or -errno.  arg is in kp->arg.
+ */
+static int apphint_kparam_set(const char *val, const struct kernel_param *kp)
+{
+	char val_copy[APPHINT_BUFFER_SIZE];
+	APPHINT_ID id;
+	union apphint_value value;
+	ssize_t result = OSStringSafeCopy(val_copy, val, APPHINT_BUFFER_SIZE);
+
+	/* Document the assumption - we can safely store the result of
+	 * apphint_read() in a ssize_t variable, no need for a separate one */
+#ifndef SSIZE_MAX
+#define SSIZE_MAX ((~(size_t)0) >> 1)
+#endif
+	BUILD_BUG_ON(INT_MAX > SSIZE_MAX);
+
+	if (result < 0) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: String too long", __func__));
+		return (int)result;
+	}
+
+	get_apphint_id_from_action_addr(kp->arg, &id);
+	result = apphint_read(val_copy, result, id, &value);
+	if (result < 0) {
+		return (int)result;
+	}
+
+	((struct apphint_action *)kp->arg)->stored = value;
+	((struct apphint_action *)kp->arg)->initialised = true;
+	if (param_lookup[id].data_type == APPHINT_DATA_TYPE_STRING) {
+		((struct apphint_action *)kp->arg)->free = true;
+	}
+
+	return 0;
+}
+
+/*
+ * apphint_kparam_get - handle a read of a module parameter
+ *
+ * Returns length written or -errno.  Buffer is 4k (ie. be short!)
+ */
+static int apphint_kparam_get(char *buffer, const struct kernel_param *kp)
+{
+	return apphint_write(buffer, PAGE_SIZE, kp->arg);
+}
+
+__maybe_unused
+static const struct kernel_param_ops apphint_kparam_fops = {
+	.set = apphint_kparam_set,
+	.get = apphint_kparam_get,
+};
+
+/*
+ * call module_param_cb() for all AppHints listed in APPHINT_LIST_MODPARAM_COMMON + APPHINT_LIST_MODPARAM
+ * apphint_modparam_class_ ## resolves to apphint_modparam_enable() except for
+ * AppHint classes that have been disabled.
+ */
+
+#define apphint_modparam_enable(name, number, perm) \
+	module_param_cb(name, &apphint_kparam_fops, &apphint.val[number], perm);
+
+#define X(a, b, c, d, e, f) \
+	apphint_modparam_class_ ##c(a, APPHINT_ID_ ## a, 0444)
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+
+/*
+*******************************************************************************
+ Debug Info get (seq file) operations - supporting functions
+******************************************************************************/
+static void *apphint_di_start(OSDI_IMPL_ENTRY *s, IMG_UINT64 *pos)
+{
+	if (*pos == 0) {
+		/* We want only one entry in the sequence, one call to show() */
+		return (void *) 1;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(s);
+
+	return NULL;
+}
+
+static void apphint_di_stop(OSDI_IMPL_ENTRY *s, void *v)
+{
+	PVR_UNREFERENCED_PARAMETER(s);
+	PVR_UNREFERENCED_PARAMETER(v);
+}
+
+static void *apphint_di_next(OSDI_IMPL_ENTRY *s, void *v, IMG_UINT64 *pos)
+{
+	PVR_UNREFERENCED_PARAMETER(s);
+	PVR_UNREFERENCED_PARAMETER(v);
+	(*pos)++;
+	return NULL;
+}
+
+static int apphint_di_show(OSDI_IMPL_ENTRY *s, void *v)
+{
+	IMG_CHAR km_buffer[APPHINT_BUFFER_SIZE];
+	int result;
+	void *private = DIGetPrivData(s);
+
+	PVR_UNREFERENCED_PARAMETER(v);
+
+	result = apphint_write(km_buffer, APPHINT_BUFFER_SIZE, private);
+	if (result < 0) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: failure", __func__));
+	} else {
+		/* debuginfo requires a trailing \n, module_params don't */
+		result += snprintf(km_buffer + result,
+				APPHINT_BUFFER_SIZE - result,
+				"\n");
+		DIPuts(s, km_buffer);
+	}
+
+	/* have to return 0 to see output */
+	return (result < 0) ? result : 0;
+}
+
+/*
+*******************************************************************************
+ Debug Info supporting functions
+******************************************************************************/
+
+/*
+ * apphint_set - Handle a DI value update
+ */
+static IMG_INT64 apphint_set(const IMG_CHAR *buffer, IMG_UINT64 count,
+                             IMG_UINT64 *ppos, void *data)
+{
+	APPHINT_ID id;
+	union apphint_value value;
+	struct apphint_action *action = data;
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	int result = 0;
+
+	if (ppos == NULL)
+		return -EIO;
+
+	if (count >= APPHINT_BUFFER_SIZE) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: String too long (%" IMG_INT64_FMTSPECd ")",
+			__func__, count));
+		result = -EINVAL;
+		goto err_exit;
+	}
+
+	/* apphint_read() modifies the buffer so we need to copy it */
+	memcpy(km_buffer, buffer, count);
+	/* count is larger than real buffer by 1 because DI framework appends
+	 * a '\0' character at the end, but here we're ignoring this */
+	count -= 1;
+	km_buffer[count] = '\0';
+
+	get_apphint_id_from_action_addr(action, &id);
+	result = apphint_read(km_buffer, count, id, &value);
+	if (result >= 0)
+		apphint_action(value, action);
+
+	*ppos += count;
+err_exit:
+	return result;
+}
+
+/*
+ * apphint_debuginfo_init - Create the specified debuginfo entries
+ */
+static int apphint_debuginfo_init(const char *sub_dir,
+		unsigned int device_num,
+		unsigned int init_data_size,
+		const struct apphint_init_data *init_data,
+		DI_GROUP *parentdir,
+		DI_GROUP **rootdir,
+		DI_ENTRY *entry[])
+{
+	PVRSRV_ERROR result;
+	unsigned int i;
+	unsigned int device_value_offset = device_num * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	const DI_ITERATOR_CB iterator = {
+		.pfnStart = apphint_di_start, .pfnStop = apphint_di_stop,
+		.pfnNext  = apphint_di_next,  .pfnShow = apphint_di_show,
+		.pfnWrite = apphint_set,      .ui32WriteLenMax = APPHINT_BUFFER_SIZE
+	};
+
+	/* Determine if we're booted as a GUEST VZ OS */
+	IMG_BOOL bIsGUEST = PVRSRV_VZ_MODE_IS(GUEST, DEVID, device_num);
+
+	if (*rootdir) {
+		PVR_DPF((PVR_DBG_WARNING,
+			"AppHint DebugFS already created, skipping"));
+		result = -EEXIST;
+		goto err_exit;
+	}
+
+	result = DICreateGroup(sub_dir, parentdir, rootdir);
+	if (result != PVRSRV_OK) {
+		PVR_DPF((PVR_DBG_WARNING,
+			"Failed to create \"%s\" DebugFS directory.", sub_dir));
+		goto err_exit;
+	}
+
+	for (i = 0; i < init_data_size; i++) {
+		if (!class_state[init_data[i].class].enabled)
+			continue;
+
+		/* Check to see if this AppHint should appear in a GUEST OS.
+		 * This will have a value in the init_data[i].guest field of ALWAYS
+		 * and if we don't have this set (and we're in GUEST mode) we must
+		 * not present this AppHint to the OS.
+		 */
+		if (bIsGUEST && (init_data[i].guest != APPHINT_RT_CLASS_ALWAYS))
+			continue;
+
+		result = DICreateEntry(init_data[i].name,
+				*rootdir,
+				&iterator,
+				(void *) &apphint.val[init_data[i].id + device_value_offset],
+				DI_ENTRY_TYPE_GENERIC,
+				&entry[i]);
+		if (result != PVRSRV_OK) {
+			PVR_DPF((PVR_DBG_WARNING,
+				"Failed to create \"%s/%s\" DebugFS entry.",
+				sub_dir, init_data[i].name));
+		}
+	}
+
+	return 0;
+
+err_exit:
+	return result;
+}
+
+/*
+ * apphint_debuginfo_deinit- destroy the debuginfo entries
+ */
+static void apphint_debuginfo_deinit(unsigned int num_entries,
+		DI_GROUP **rootdir,
+		DI_ENTRY *entry[])
+{
+	unsigned int i;
+
+	for (i = 0; i < num_entries; i++) {
+		if (entry[i]) {
+			DIDestroyEntry(entry[i]);
+		}
+	}
+
+	if (*rootdir) {
+		DIDestroyGroup(*rootdir);
+		*rootdir = NULL;
+	}
+}
+
+/*
+*******************************************************************************
+ AppHint status dump implementation
+******************************************************************************/
+#if defined(PDUMP)
+static void apphint_pdump_values(void *pvDeviceNode,
+                                 const IMG_CHAR *format, ...)
+{
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	IMG_UINT32 ui32Flags = PDUMP_FLAGS_CONTINUOUS;
+	va_list ap;
+
+	va_start(ap, format);
+	(void)vsnprintf(km_buffer, APPHINT_BUFFER_SIZE, format, ap);
+	va_end(ap);
+
+	/* ui32CommentSize set to 0 here as function does not make use of the value. */
+	PDumpCommentKM(NULL, (PVRSRV_DEVICE_NODE*)pvDeviceNode, 0, km_buffer, ui32Flags);
+}
+#endif
+
+static IMG_BOOL is_apphint_value_equal(const APPHINT_DATA_TYPE data_type,
+									const union apphint_value * const left,
+									const union apphint_value * const right)
+{
+		switch (data_type) {
+		case APPHINT_DATA_TYPE_UINT64:
+			return left->UINT64 == right->UINT64;
+		case APPHINT_DATA_TYPE_UINT32:
+		case APPHINT_DATA_TYPE_UINT32List:
+		case APPHINT_DATA_TYPE_UINT32Bitfield:
+			return left->UINT32 == right->UINT32;
+		case APPHINT_DATA_TYPE_BOOL:
+			return left->BOOL == right->BOOL;
+		case APPHINT_DATA_TYPE_STRING:
+			return (OSStringNCompare(left->STRING, right->STRING, OSStringLength(right->STRING) + 1) == 0 ? IMG_TRUE : IMG_FALSE);
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "%s: unhandled data type (%d)", __func__, data_type));
+			return IMG_FALSE;
+		}
+}
+
+static void apphint_dump_values(const char *group_name,
+			int device_num,
+			const struct apphint_init_data *group_data,
+			int group_size,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile,
+			bool list_all,
+			PVRSRV_DEVICE_NODE *psDevNode)
+{
+	int i, result;
+	int device_value_offset = device_num * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	char km_buffer[APPHINT_BUFFER_SIZE];
+	char count = 0;
+
+	PVR_DUMPDEBUG_LOG("  %s", group_name);
+	for (i = 0; i < group_size; i++)
+	{
+		IMG_UINT32 id = group_data[i].id;
+		APPHINT_DATA_TYPE data_type = param_lookup[id].data_type;
+		const struct apphint_action *action = &apphint.val[id + device_value_offset];
+		union apphint_value value;
+
+		result = get_apphint_value_from_action(action, &value, psDevNode);
+
+		if (PVRSRV_OK != result) {
+			continue;
+		}
+
+		/* List only apphints with non-default values */
+		if (!list_all &&
+			is_apphint_value_equal(data_type, &value, &group_data[i].default_value)) {
+			continue;
+		}
+
+		result = apphint_write(km_buffer, APPHINT_BUFFER_SIZE, action);
+		count++;
+
+		if (result <= 0) {
+			PVR_DUMPDEBUG_LOG("    %s: <Error>",
+				group_data[i].name);
+		} else {
+			PVR_DUMPDEBUG_LOG("    %s: %s",
+				group_data[i].name, km_buffer);
+		}
+	}
+
+	if (count == 0) {
+		PVR_DUMPDEBUG_LOG("    none");
+	}
+}
+
+/*
+ * Callback for debug dump
+ */
+static void apphint_dump_state(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+			IMG_UINT32 ui32VerbLevel,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	int i, result;
+	char description_buffer[50];
+	PVRSRV_DEVICE_NODE *device = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH)) {
+		PVR_DUMPDEBUG_LOG("------[ AppHint Settings ]------");
+
+		apphint_dump_values("Build Vars", 0,
+			init_data_buildvar, ARRAY_SIZE(init_data_buildvar),
+			pfnDumpDebugPrintf, pvDumpDebugFile, true, device);
+
+		apphint_dump_values("Module Params", 0,
+			init_data_modparam, ARRAY_SIZE(init_data_modparam),
+			pfnDumpDebugPrintf, pvDumpDebugFile, false, device);
+
+		apphint_dump_values("Debug Info Params", 0,
+			init_data_debuginfo, ARRAY_SIZE(init_data_debuginfo),
+			pfnDumpDebugPrintf, pvDumpDebugFile, false, device);
+
+		for (i = 0; i < PVRSRV_MAX_DEVICES; i++) {
+			if (!apphint.devices[i]
+			    || (device && device != apphint.devices[i]))
+				continue;
+
+			result = snprintf(description_buffer,
+					  sizeof(description_buffer),
+					  "Debug Info Params Device ID: %d",
+					  i);
+			if (0 > result)
+				continue;
+
+			apphint_dump_values(description_buffer, i,
+					    init_data_debuginfo_device,
+					    ARRAY_SIZE(init_data_debuginfo_device),
+					    pfnDumpDebugPrintf,
+					    pvDumpDebugFile,
+						false, device);
+		}
+	}
+}
+
+/*
+*******************************************************************************
+ Public interface
+******************************************************************************/
+int pvr_apphint_init(void)
+{
+	int result, i;
+
+	if (apphint.initialized) {
+		result = -EEXIST;
+		goto err_out;
+	}
+
+	for (i = 0; i < PVRSRV_MAX_DEVICES; i++)
+		apphint.devices[i] = NULL;
+
+	/* create workqueue with strict execution ordering to ensure no
+	 * race conditions when setting/updating apphints from different
+	 * contexts
+	 */
+	apphint.workqueue = alloc_workqueue("apphint_workqueue",
+	                                    WQ_UNBOUND | WQ_FREEZABLE, 1);
+	if (!apphint.workqueue) {
+		result = -ENOMEM;
+		goto err_out;
+	}
+
+	result = apphint_debuginfo_init("apphint", 0,
+		ARRAY_SIZE(init_data_debuginfo), init_data_debuginfo,
+		NULL,
+		&apphint.debuginfo_rootdir, apphint.debuginfo_entry);
+	if (0 != result)
+		goto err_out;
+
+	result = apphint_debuginfo_init("buildvar", 0,
+		ARRAY_SIZE(init_data_buildvar), init_data_buildvar,
+		NULL,
+		&apphint.buildvar_rootdir, apphint.buildvar_entry);
+
+	apphint.initialized = 1;
+
+err_out:
+	return result;
+}
+
+int pvr_apphint_device_register(PVRSRV_DEVICE_NODE *device)
+{
+	int result, i;
+	unsigned int device_value_offset;
+
+	if (!apphint.initialized) {
+		result = -EAGAIN;
+		goto err_out;
+	}
+
+	if (apphint.num_devices+1 > PVRSRV_MAX_DEVICES) {
+		result = -EMFILE;
+		goto err_out;
+	}
+
+	/* Set the default values for the new device */
+	device_value_offset = apphint.num_devices * APPHINT_DEBUGINFO_DEVICE_ID_MAX;
+	for (i = 0; i < APPHINT_DEBUGINFO_DEVICE_ID_MAX; i++) {
+		apphint.val[init_data_debuginfo_device[i].id + device_value_offset].stored
+			= init_data_debuginfo_device[i].default_value;
+	}
+
+	/* Set value of an apphint if mapping to module param exists for it
+	 * and this module parameter has been initialised */
+	for (i = 0; i < ARRAY_SIZE(init_data_debuginfo_device_to_modparams); i++) {
+		const struct apphint_init_data_mapping *mapping =
+			&init_data_debuginfo_device_to_modparams[i];
+		const struct apphint_action *modparam_action =
+			&apphint.val[mapping->modparam_apphint_id];
+		struct apphint_action *device_action =
+			&apphint.val[mapping->device_apphint_id + device_value_offset];
+
+		/* Set only if the module parameter was explicitly set during the module
+		 * load. */
+		if (modparam_action->initialised) {
+			device_action->stored = modparam_action->stored;
+		}
+	}
+
+	result = apphint_debuginfo_init("apphint", device->sDevId.ui32InternalID,
+	                              ARRAY_SIZE(init_data_debuginfo_device),
+	                              init_data_debuginfo_device,
+	                              device->sDebugInfo.psGroup,
+	                              &apphint.debuginfo_device_rootdir[device->sDevId.ui32InternalID],
+	                              apphint.debuginfo_device_entry[device->sDevId.ui32InternalID]);
+	if (0 != result)
+		goto err_out;
+
+	apphint.devices[device->sDevId.ui32InternalID] = device;
+	apphint.num_devices++;
+
+	(void)SOPvrDbgRequestNotifyRegister(
+			&device->hAppHintDbgReqNotify,
+			device,
+			apphint_dump_state,
+			DEBUG_REQUEST_APPHINT,
+			device);
+
+err_out:
+	return result;
+}
+
+void pvr_apphint_device_unregister(PVRSRV_DEVICE_NODE *device)
+{
+	int i;
+
+	if (!apphint.initialized)
+		return;
+
+	/* find the device */
+	for (i = 0; i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i] == device)
+			break;
+	}
+
+	if (PVRSRV_MAX_DEVICES == i)
+		return;
+
+	if (device->hAppHintDbgReqNotify) {
+		(void)SOPvrDbgRequestNotifyUnregister(
+			device->hAppHintDbgReqNotify);
+		device->hAppHintDbgReqNotify = NULL;
+	}
+
+	apphint_debuginfo_deinit(APPHINT_DEBUGINFO_DEVICE_ID_MAX,
+	                       &apphint.debuginfo_device_rootdir[i],
+	                       apphint.debuginfo_device_entry[i]);
+
+	apphint.devices[i] = NULL;
+
+	WARN_ON(apphint.num_devices==0);
+	apphint.num_devices--;
+}
+
+void pvr_apphint_deinit(void)
+{
+	int i;
+
+	if (!apphint.initialized)
+		return;
+
+	/* remove any remaining device data */
+	for (i = 0; apphint.num_devices && i < PVRSRV_MAX_DEVICES; i++) {
+		if (apphint.devices[i])
+			pvr_apphint_device_unregister(apphint.devices[i]);
+	}
+
+	/* free all alloc'd string apphints and set to NULL */
+	for (i = 0; i < ARRAY_SIZE(apphint.val); i++) {
+		if (apphint.val[i].free && apphint.val[i].stored.STRING) {
+			kfree(apphint.val[i].stored.STRING);
+			apphint.val[i].stored.STRING = NULL;
+			apphint.val[i].free = false;
+		}
+	}
+
+	apphint_debuginfo_deinit(APPHINT_DEBUGINFO_ID_MAX,
+			&apphint.debuginfo_rootdir, apphint.debuginfo_entry);
+	apphint_debuginfo_deinit(APPHINT_BUILDVAR_ID_MAX,
+			&apphint.buildvar_rootdir, apphint.buildvar_entry);
+
+	destroy_workqueue(apphint.workqueue);
+
+	apphint.initialized = 0;
+}
+
+void pvr_apphint_dump_state(PVRSRV_DEVICE_NODE *device)
+{
+#if defined(PDUMP)
+	/* NB. apphint_pdump_values() is the pfnDumpDebugPrintf
+	 * function used when PDUMP is defined.
+	 * apphintpdump_values() calls PDumpCommentKM(), which
+	 * requires the device but as it is only called as a
+	 * DUMPDEBUG_PRINTF_FUNC it is only passed pvDumpDebugFile
+	 * (which happens to be the 4th parameter in the call to
+	 * apphint_dump_state() below).
+	 * Hence, we also need to pass device in the 4th parameter.
+	 */
+	apphint_dump_state(device, DEBUG_REQUEST_VERBOSITY_HIGH,
+	                   apphint_pdump_values, device);
+#endif
+	apphint_dump_state(device, DEBUG_REQUEST_VERBOSITY_HIGH,
+	                   NULL, NULL);
+}
+
+
+int pvr_apphint_get_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.UINT64;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.UINT64;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.UINT32;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.UINT32;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL *pVal)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			*pVal = apphint.val[ue + device_offset].stored.BOOL;
+			error = 0;
+		}
+		else
+		{
+			*pVal = apphint.val[ue].stored.BOOL;
+			error = 0;
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_get_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if (ue < APPHINT_ID_MAX && apphint.val[ue].stored.STRING) {
+		if ((int)ue > APPHINT_DEBUGINFO_DEVICE_ID_OFFSET) // From this point, we're in the device apphints
+		{
+			if (OSStringSafeCopy(pBuffer, apphint.val[ue + device_offset].stored.STRING, size) >= 0) {
+				error = 0;
+			}
+		}
+		else
+		{
+			if (OSStringSafeCopy(pBuffer, apphint.val[ue].stored.STRING, size) >= 0) {
+				error = 0;
+			}
+		}
+	}
+	return error;
+}
+
+int pvr_apphint_set_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_UINT64)) {
+
+		if (apphint.val[ue + device_offset].set.UINT64) {
+			apphint.val[ue + device_offset].set.UINT64(apphint.val[ue + device_offset].device,
+													 apphint.val[ue + device_offset].private_data,
+													 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.UINT64 = Val;
+		}
+		apphint.val[ue].device = device;
+		error = 0;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_UINT32)) {
+
+		if (apphint.val[ue + device_offset].set.UINT32) {
+			apphint.val[ue + device_offset].set.UINT32(apphint.val[ue + device_offset].device,
+													 apphint.val[ue + device_offset].private_data,
+													 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.UINT32 = Val;
+		}
+		apphint.val[ue].device = device;
+		error = 0;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL Val)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		(param_lookup[ue].data_type == APPHINT_DATA_TYPE_BOOL)) {
+
+		error = 0;
+		if (apphint.val[ue + device_offset].set.BOOL) {
+			apphint.val[ue + device_offset].set.BOOL(apphint.val[ue + device_offset].device,
+												 apphint.val[ue + device_offset].private_data,
+												 Val);
+		} else {
+			apphint.val[ue + device_offset].stored.BOOL = Val;
+		}
+		apphint.val[ue].device = device;
+	}
+
+	return error;
+}
+
+int pvr_apphint_set_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size)
+{
+	int error = -ERANGE;
+	int device_offset = (device != NULL) ? device->sDevId.ui32InternalID * APPHINT_DEBUGINFO_DEVICE_ID_MAX : 0;
+
+	if ((ue < APPHINT_ID_MAX) &&
+		((param_lookup[ue].data_type == APPHINT_DATA_TYPE_STRING) &&
+		apphint.val[ue + device_offset].stored.STRING)) {
+
+		if (apphint.val[ue + device_offset].set.STRING) {
+			error = apphint.val[ue + device_offset].set.STRING(apphint.val[ue + device_offset].device,
+															 apphint.val[ue + device_offset].private_data,
+															 pBuffer);
+		} else if (OSStringSafeCopy(apphint.val[ue + device_offset].stored.STRING, pBuffer, size) >= 0) {
+			error = 0;
+		}
+		apphint.val[ue].device = device;
+	}
+
+	return error;
+}
+
+void pvr_apphint_register_handlers_uint64(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_UINT64:
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.UINT64 = query,
+		.set.UINT64 = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_uint32(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_UINT32:
+	case APPHINT_DATA_TYPE_UINT32Bitfield:
+	case APPHINT_DATA_TYPE_UINT32List:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.UINT32 = query,
+		.set.UINT32 = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_bool(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_BOOL:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.BOOL = query,
+		.set.BOOL = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+void pvr_apphint_register_handlers_string(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR **value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR *value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data)
+{
+	int device_value_offset;
+
+	PVR_DPF((APPHINT_DPF_LEVEL, "%s(%d, %p, %p, %p, %p)",
+	         __func__, id, query, set, device, private_data));
+
+	if (id >= APPHINT_ID_MAX) {
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: AppHint ID (%d) is out of range, max (%d)",
+		         __func__, id, APPHINT_ID_MAX-1));
+		return;
+	}
+
+	get_value_offset_from_device(device, &device_value_offset, id);
+
+	switch (param_lookup[id].data_type) {
+	case APPHINT_DATA_TYPE_STRING:
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Does not match AppHint data type for ID (%d)",
+		         __func__, id));
+		return;
+	}
+
+	apphint.val[id + device_value_offset] = (struct apphint_action){
+		.query.STRING = query,
+		.set.STRING = set,
+		.device = device,
+		.private_data = private_data,
+		.stored = apphint.val[id + device_value_offset].stored
+	};
+}
+
+/* EOF */
diff --git a/drivers/gpu/drm/img-rogue/km_apphint.h b/drivers/gpu/drm/img-rogue/km_apphint.h
new file mode 100644
index 000000000000..71e2ce94000a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km_apphint.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File           km_apphint.h
+@Title          Apphint internal header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux kernel AppHint control
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef KM_APPHINT_H
+#define KM_APPHINT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "pvrsrv_apphint.h"
+#include "km_apphint_defs.h"
+#include "device.h"
+
+int pvr_apphint_init(void);
+void pvr_apphint_deinit(void);
+int pvr_apphint_device_register(PVRSRV_DEVICE_NODE *device);
+void pvr_apphint_device_unregister(PVRSRV_DEVICE_NODE *device);
+void pvr_apphint_dump_state(PVRSRV_DEVICE_NODE *device);
+
+int pvr_apphint_get_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 *pVal);
+int pvr_apphint_get_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 *pVal);
+int pvr_apphint_get_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL *pVal);
+int pvr_apphint_get_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size);
+
+int pvr_apphint_set_uint64(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT64 Val);
+int pvr_apphint_set_uint32(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_UINT32 Val);
+int pvr_apphint_set_bool(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_BOOL Val);
+int pvr_apphint_set_string(PVRSRV_DEVICE_NODE *device, APPHINT_ID ue, IMG_CHAR *pBuffer, size_t size);
+
+void pvr_apphint_register_handlers_uint64(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT64 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void * private_data);
+void pvr_apphint_register_handlers_uint32(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_UINT32 value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+void pvr_apphint_register_handlers_bool(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL *value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_BOOL value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+void pvr_apphint_register_handlers_string(APPHINT_ID id,
+	PVRSRV_ERROR (*query)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR **value),
+	PVRSRV_ERROR (*set)(const PVRSRV_DEVICE_NODE *device, const void *private_data, IMG_CHAR *value),
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data);
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* KM_APPHINT_H */
+
+/******************************************************************************
+ End of file (km_apphint.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/km_apphint_defs.h b/drivers/gpu/drm/img-rogue/km_apphint_defs.h
new file mode 100644
index 000000000000..f2ffad0030cd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km_apphint_defs.h
@@ -0,0 +1,162 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "km_apphint_defs_common.h"
+
+#ifndef KM_APPHINT_DEFS_H
+#define KM_APPHINT_DEFS_H
+
+/* NB: The 'DEVICE' AppHints must be last in this list as they will be
+ * duplicated in the case of a driver supporting multiple devices
+ */
+#define APPHINT_LIST_ALL \
+	APPHINT_LIST_BUILDVAR_COMMON \
+	APPHINT_LIST_BUILDVAR \
+	APPHINT_LIST_MODPARAM_COMMON \
+	APPHINT_LIST_MODPARAM \
+	APPHINT_LIST_DEBUGINFO_COMMON \
+	APPHINT_LIST_DEBUGINFO \
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+	APPHINT_LIST_DEBUGINFO_DEVICE
+
+
+/*
+*******************************************************************************
+ Build variables (rogue-specific)
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR
+
+/*
+*******************************************************************************
+ Module parameters (rogue-specific)
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+X(EnableCDMKillingRandMode,         BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLECDMKILLINGRANDMODE,         NO_PARAM_TABLE,  ALWAYS   ) \
+\
+X(HWPerfDisableCustomCounterFilter, BOOL,           VALIDATION,  PVRSRV_APPHINT_HWPERFDISABLECUSTOMCOUNTERFILTER, NO_PARAM_TABLE,  ALWAYS   ) \
+X(ValidateSOCUSCTimer,              BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATESOCUSCTIMERS,             NO_PARAM_TABLE,  ALWAYS   ) \
+X(ECCRAMErrInj,                     UINT32,         VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+\
+X(TFBCCompressionControlGroup,      UINT32,         VALIDATION,  PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP,      NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlScheme,     UINT32,         VALIDATION,  PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME,     NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlYUVFormat,  BOOL,           VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCCompressionControlLossyMinChannel, BOOL,      VALIDATION,  0,                                               NO_PARAM_TABLE,  ALWAYS   ) \
+X(TFBCVersionDowngrade,             UINT32,         ALWAYS,      PVRSRV_APPHINT_TFBCVERSION,                      NO_PARAM_TABLE,  ALWAYS   ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters (rogue-specific) - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+
+/*
+*******************************************************************************
+ Debugfs parameters (rogue-specific) - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE \
+/* name,                            type,           class,       default,                                         helper,          guest,  */ \
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters. Each entry in this table will provide a default value to all
+ devices (i.e. if there is more than one device each device's value will
+ be initialised).
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT \
+/* debuginfo device apphint name  modparam name */
+
+/*
+*******************************************************************************
+
+ Table generated enums
+
+******************************************************************************/
+/* Unique ID for all AppHints */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_ID_ ## a,
+	APPHINT_LIST_ALL
+#undef X
+	APPHINT_ID_MAX
+} APPHINT_ID;
+
+/* ID for build variable Apphints - used for build variable only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_BUILDVAR_ID_ ## a,
+	APPHINT_LIST_BUILDVAR_COMMON
+	APPHINT_LIST_BUILDVAR
+#undef X
+	APPHINT_BUILDVAR_ID_MAX
+} APPHINT_BUILDVAR_ID;
+
+/* ID for Modparam Apphints - used for modparam only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_MODPARAM_ID_ ## a,
+	APPHINT_LIST_MODPARAM_COMMON
+	APPHINT_LIST_MODPARAM
+#undef X
+	APPHINT_MODPARAM_ID_MAX
+} APPHINT_MODPARAM_ID;
+
+/* ID for Debugfs Apphints - used for debugfs only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_COMMON
+	APPHINT_LIST_DEBUGINFO
+#undef X
+	APPHINT_DEBUGINFO_ID_MAX
+} APPHINT_DEBUGINFO_ID;
+
+/* ID for Debugfs Device Apphints - used for debugfs device only structures */
+typedef enum {
+#define X(a, b, c, d, e, f) APPHINT_DEBUGINFO_DEVICE_ID_ ## a,
+	APPHINT_LIST_DEBUGINFO_DEVICE_COMMON
+	APPHINT_LIST_DEBUGINFO_DEVICE
+#undef X
+	APPHINT_DEBUGINFO_DEVICE_ID_MAX
+} APPHINT_DEBUGINFO_DEVICE_ID;
+
+#endif /* KM_APPHINT_DEFS_H */
diff --git a/drivers/gpu/drm/img-rogue/km_apphint_defs_common.h b/drivers/gpu/drm/img-rogue/km_apphint_defs_common.h
new file mode 100644
index 000000000000..b3087e79d73a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/km_apphint_defs_common.h
@@ -0,0 +1,308 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services AppHint definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef KM_APPHINT_DEFS_COMMON_H
+#define KM_APPHINT_DEFS_COMMON_H
+
+/*
+*******************************************************************************
+ Build variables
+ All of these should be configurable only through the 'default' value
+******************************************************************************/
+#define APPHINT_LIST_BUILDVAR_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(EnableTrustedDeviceAceConfig,     BOOL,           GPUVIRT_VAL, PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG,  NO_PARAM_TABLE,   ALWAYS   ) \
+X(CleanupThreadPriority,            UINT32,         NEVER,       PVRSRV_APPHINT_CLEANUPTHREADPRIORITY,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(WatchdogThreadPriority,           UINT32,         NEVER,       PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientBufferSize,           UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevmemHistoryBufSizeLog2,         UINT32,         ALWAYS,      PVRSRV_APPHINT_DEVMEM_HISTORY_BUFSIZE_LOG2,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevmemHistoryMaxEntries,          UINT32,         ALWAYS,      PVRSRV_APPHINT_DEVMEM_HISTORY_MAX_ENTRIES,    NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Module parameters
+******************************************************************************/
+#define APPHINT_LIST_MODPARAM_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(GeneralNon4KHeapPageSize,         UINT32,         ALWAYS,      PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE,      NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableSignatureChecks,            BOOL,           PDUMP,       PVRSRV_APPHINT_ENABLESIGNATURECHECKS,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(SignatureChecksBufSize,           UINT32,         PDUMP,       PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DisableClockGating,               BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLECLOCKGATING,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisableDMOverlap,                 BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLEDMOVERLAP,              NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableRandomContextSwitch,        BOOL,           VALIDATION,  PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableSoftResetContextSwitch,     BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH,  NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFWContextSwitch,            UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(FWContextSwitchProfile,           UINT32,         VALIDATION,  PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableRDPowerIsland,              UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLERDPOWERISLAND,           NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DriverMode,                       STRING,         ALWAYS,      PVRSRV_APPHINT_DRIVERMODE,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(AutoVzGPUPowerdown,               BOOL,           ALWAYS,      PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(GuestFWHeapStride,                UINT64,         ALWAYS,      PVRSRV_APPHINT_GUESTFWHEAPSTRIDE,             NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FirmwarePerf,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_FIRMWAREPERF,                  NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(HWPerfFWBufSizeInKB,              UINT32,         PDUMP,       PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB,           NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostBufSizeInKB,            UINT32,         VALIDATION,  PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostThreadTimeoutInMS,      UINT32,         VALIDATION,  PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS,   NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(NewFilteringMode,                 BOOL,           VALIDATION,  PVRSRV_APPHINT_NEWFILTERINGMODE,              NO_PARAM_TABLE,   ALWAYS   ) \
+X(TruncateMode,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_TRUNCATEMODE,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(EmuMaxFreq,                       UINT32,         ALWAYS,      PVRSRV_APPHINT_EMUMAXFREQ,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(GPIOValidationMode,               UINT32,         VALIDATION,  PVRSRV_APPHINT_GPIOVALIDATIONMODE,            NO_PARAM_TABLE,   ALWAYS   ) \
+X(RGXBVNC,                          STRING,         ALWAYS,      PVRSRV_APPHINT_RGXBVNC,                       NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FWContextSwitchCrossDM,           UINT32,         ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(ValidateIrq,                      BOOL,           VALIDATION,  PVRSRV_APPHINT_VALIDATEIRQ,                   NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(TPUTrilinearFracMaskPDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskVDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskCDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskTDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TPUTrilinearFracMaskRDM,          UINT32,         VALIDATION,  0x7,                                          NO_PARAM_TABLE,   ALWAYS   ) \
+X(HTBufferSizeInKB,                 UINT32,         ALWAYS,      PVRSRV_APPHINT_HTBUFFERSIZE,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(FWTraceBufSizeInDWords,           UINT32,         ALWAYS,      PVRSRV_APPHINT_FWTRACEBUFSIZEINDWORDS,        NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnablePageFaultDebug,             BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFullSyncTracking,           BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(IgnoreHWReportedBVNC,             BOOL,           ALWAYS,      PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC,          NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(PhysMemTestPasses,                UINT32,         ALWAYS,      PVRSRV_APPHINT_PHYSMEMTESTPASSES,             NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FBCDCVersionOverride,             UINT32,         VALIDATION,  PVRSRV_APPHINT_FBCDCVERSIONOVERRIDE,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(TestSLRInterval,                  UINT32,         VALIDATION,  PVRSRV_APPHINT_TESTSLRINTERVAL,               NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnablePollOnChecksumErrorStatus,  UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(RiscvDmiTest,                     BOOL,           VALIDATION,  PVRSRV_APPHINT_RISCVDMITEST,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(DevMemFWHeapPolicy,               UINT32,         ALWAYS,      PVRSRV_APPHINT_FIRMWARE_HEAP_POLICY,          NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableAPMAll,                     UINT32,         VALIDATION,  PVRSRV_APPHINT_ENABLEAPM,                     NO_PARAM_TABLE,   ALWAYS   ) \
+X(KernelCCBSizeLog2,                UINT32,         VALIDATION,  PVRSRV_APPHINT_KCCB_SIZE_LOG2,                NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(SyncCheckpointPoolMaxLog2,        UINT32,         ALWAYS,      PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2,         NO_PARAM_TABLE,   ALWAYS   ) \
+X(SyncCheckpointPoolInitLog2,       UINT32,         ALWAYS,      PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(PhysHeapMinMemOnConnection,       UINT32,         ALWAYS,      PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION,    NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(RestrictGpuLocalPhysHeapSizeMB,   UINT32,         VALIDATION,  0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+X(PhysHeapHybridDefault2CpuLocal,   BOOL,           ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(DebugDumpFWTLogType,              UINT32,         ALWAYS,      PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE,           NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(EnableIdleCycleStealing,          UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLEIDLECYCLESTEALING,       NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FaultDetectionTimeInterval,       UINT32,         ALWAYS,      PVRSRV_APPHINT_FAULTDETECTIONTIMEINTERVAL_USEC, NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(ICSTimeIntervalThreshold,         UINT32,         ALWAYS,      PVRSRV_APPHINT_ICSTIMEINTERVAL_THRESHOLD,     NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(ICSTestModeOn,                    BOOL,           ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   ) \
+\
+X(FaultInjection,                   UINT32,         ALWAYS,      0,                                            NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Debugfs parameters - driver configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+X(EnableHTBLogGroup,                UINT32Bitfield, ALWAYS,      PVRSRV_APPHINT_ENABLEHTBLOGGROUP,             htb_loggroup_tbl, ALWAYS   ) \
+X(HTBOperationMode,                 UINT32List,     ALWAYS,      PVRSRV_APPHINT_HTBOPERATIONMODE,              htb_opmode_tbl,   ALWAYS   ) \
+X(EnableFTraceGPU,                  BOOL,           ALWAYS,      PVRSRV_APPHINT_ENABLEFTRACEGPU,               NO_PARAM_TABLE,   NEVER    ) \
+X(HWPerfClientFilter_Services,      UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_SERVICES,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_EGL,           UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_EGL,        NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenGLES,      UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGLES,   NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenCL,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENCL,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_Vulkan,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_VULKAN,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfClientFilter_OpenGL,        UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFCLIENTFILTER_OPENGL,     NO_PARAM_TABLE,   ALWAYS   ) \
+X(CacheOpConfig,                    UINT32,         ALWAYS,      PVRSRV_APPHINT_CACHEOPCONFIG,                 NO_PARAM_TABLE,   ALWAYS   ) \
+X(CacheOpUMKMThresholdSize,         UINT32,         ALWAYS,      PVRSRV_APPHINT_CACHEOPUMKMHRESHOLDSIZE,       NO_PARAM_TABLE,   ALWAYS   ) \
+
+/*
+*******************************************************************************
+ Debugfs parameters - device configuration
+******************************************************************************/
+#define APPHINT_LIST_DEBUGINFO_DEVICE_COMMON \
+/* name,                            type,           class,       default,                                      helper,           guest,  */ \
+/* Device Firmware config */\
+X(AssertOnHWRTrigger,               BOOL,           ALWAYS,      APPHNT_BLDVAR_ASSERTONHWRTRIGGER,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(AssertOutOfMemory,                BOOL,           ALWAYS,      PVRSRV_APPHINT_ASSERTOUTOFMEMORY,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(CheckMList,                       BOOL,           ALWAYS,      PVRSRV_APPHINT_CHECKMLIST,                    NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableLogGroup,                   UINT32Bitfield, ALWAYS,      PVRSRV_APPHINT_ENABLELOGGROUP,                fwt_loggroup_tbl, ALWAYS   ) \
+X(FirmwareLogType,                  UINT32List,     ALWAYS,      PVRSRV_APPHINT_FIRMWARELOGTYPE,               fwt_logtype_tbl,  ALWAYS   ) \
+X(HWRDebugDumpLimit,                UINT32,         ALWAYS,      PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(TimeCorrClock,                    UINT32List,     ALWAYS,      PVRSRV_APPHINT_TIMECORRCLOCK,                 timecorr_clk_tbl, ALWAYS   ) \
+X(HWPerfFWFilter,                   UINT64,         ALWAYS,      PVRSRV_APPHINT_HWPERFFWFILTER,                NO_PARAM_TABLE,   ALWAYS   ) \
+/* Device host config */ \
+X(EnableAPM,                        UINT32,         ALWAYS,      PVRSRV_APPHINT_ENABLEAPM,                     NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisableFEDLogging,                BOOL,           ALWAYS,      PVRSRV_APPHINT_DISABLEFEDLOGGING,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(ZeroFreelist,                     BOOL,           ALWAYS,      PVRSRV_APPHINT_ZEROFREELIST,                  NO_PARAM_TABLE,   ALWAYS   ) \
+X(DisablePDumpPanic,                BOOL,           PDUMP,       PVRSRV_APPHINT_DISABLEPDUMPPANIC,             NO_PARAM_TABLE,   ALWAYS   ) \
+X(EnableFWPoisonOnFree,             BOOL,           DEBUG,       PVRSRV_APPHINT_ENABLEFWPOISONONFREE,          NO_PARAM_TABLE,   ALWAYS   ) \
+X(GPUUnitsPowerChange,              BOOL,           VALIDATION,  PVRSRV_APPHINT_GPUUNITSPOWERCHANGE,           NO_PARAM_TABLE,   ALWAYS   ) \
+X(HWPerfHostFilter,                 UINT32,         ALWAYS,      PVRSRV_APPHINT_HWPERFHOSTFILTER,              NO_PARAM_TABLE,   ALWAYS   )
+
+/*
+*******************************************************************************
+ Mapping between debugfs parameters and module parameters.
+ This mapping is used to initialise device specific apphints from module
+ parameters.
+******************************************************************************/
+#define APPHINT_LIST_DEBUIGINFO_DEVICE_X_MODPARAM_INIT_COMMON \
+/* debuginfo device apphint name  modparam name */ \
+X(EnableAPM,                      EnableAPMAll)
+
+/*
+*******************************************************************************
+ * Types used in the APPHINT_LIST_<GROUP> lists must be defined here.
+ * New types require specific handling code to be added
+******************************************************************************/
+#define APPHINT_DATA_TYPE_LIST \
+X(BOOL) \
+X(UINT64) \
+X(UINT32) \
+X(UINT32Bitfield) \
+X(UINT32List) \
+X(STRING)
+
+#define APPHINT_CLASS_LIST \
+X(ALWAYS) \
+X(NEVER) \
+X(DEBUG) \
+X(PDUMP) \
+X(VALIDATION) \
+X(GPUVIRT_VAL)
+
+#define APPHINT_RT_CLASS_LIST \
+X(ALWAYS) \
+X(NEVER)
+
+/*
+*******************************************************************************
+ Visibility control for module parameters
+ These bind build variables to AppHint Visibility Groups.
+******************************************************************************/
+#define APPHINT_ENABLED_CLASS_ALWAYS IMG_TRUE
+#define APPHINT_ENABLED_CLASS_NEVER IMG_FALSE
+#define apphint_modparam_class_ALWAYS(a, b, c) apphint_modparam_enable(a, b, c)
+#if defined(DEBUG)
+	#define APPHINT_ENABLED_CLASS_DEBUG IMG_TRUE
+	#define apphint_modparam_class_DEBUG(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_DEBUG IMG_FALSE
+	#define apphint_modparam_class_DEBUG(a, b, c)
+#endif
+#if defined(PDUMP)
+	#define APPHINT_ENABLED_CLASS_PDUMP IMG_TRUE
+	#define apphint_modparam_class_PDUMP(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_PDUMP IMG_FALSE
+	#define apphint_modparam_class_PDUMP(a, b, c)
+#endif
+	#define APPHINT_ENABLED_CLASS_VALIDATION IMG_FALSE
+	#define apphint_modparam_class_VALIDATION(a, b, c)
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	#define APPHINT_ENABLED_CLASS_GPUVIRT_VAL IMG_TRUE
+	#define apphint_modparam_class_GPUVIRT_VAL(a, b, c) apphint_modparam_enable(a, b, c)
+#else
+	#define APPHINT_ENABLED_CLASS_GPUVIRT_VAL IMG_FALSE
+	#define apphint_modparam_class_GPUVIRT_VAL(a, b, c)
+#endif
+
+/*
+*******************************************************************************
+ AppHint defaults based on other build parameters
+******************************************************************************/
+#if defined(ASSERTONHWRTRIGGER_DEFAULT_ENABLED)
+	#define APPHNT_BLDVAR_ASSERTONHWRTRIGGER      1
+#else
+	#define APPHNT_BLDVAR_ASSERTONHWRTRIGGER      0
+#endif
+#if defined(DEBUG)
+	#define APPHNT_BLDVAR_DEBUG             1
+	#define APPHNT_BLDVAR_DBGDUMPLIMIT      RGXFWIF_HWR_DEBUG_DUMP_ALL
+#else
+	#define APPHNT_BLDVAR_DEBUG             0
+	#define APPHNT_BLDVAR_DBGDUMPLIMIT      1
+#endif
+#if defined(PDUMP)
+#define APPHNT_BLDVAR_ENABLESIGNATURECHECKS     IMG_TRUE
+#else
+#define APPHNT_BLDVAR_ENABLESIGNATURECHECKS     IMG_FALSE
+#endif
+#if defined(DEBUG)
+#define APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG      IMG_TRUE
+#else
+#define APPHNT_BLDVAR_ENABLEPAGEFAULTDEBUG      IMG_FALSE
+#endif
+
+#if defined(DEBUG)
+	#define APPHNT_PHYSMEMTEST_ENABLE             1
+#else
+	#define APPHNT_PHYSMEMTEST_ENABLE             0
+#endif
+
+/* Data types and actions */
+typedef enum {
+	APPHINT_DATA_TYPE_INVALID = 0,
+#define X(a) APPHINT_DATA_TYPE_ ## a,
+	APPHINT_DATA_TYPE_LIST
+#undef X
+	APPHINT_DATA_TYPE_MAX
+} APPHINT_DATA_TYPE;
+
+typedef enum {
+#define X(a) APPHINT_CLASS_ ## a,
+	APPHINT_CLASS_LIST
+#undef X
+	APPHINT_CLASS_MAX
+} APPHINT_CLASS;
+
+typedef enum {
+#define X(a) APPHINT_RT_CLASS_ ## a,
+    APPHINT_RT_CLASS_LIST
+#undef X
+    APPHINT_RT_CLASS_MAX
+} APPHINT_RT_CLASS;
+#endif /* KM_APPHINT_DEFS_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/linkage.h b/drivers/gpu/drm/img-rogue/linkage.h
new file mode 100644
index 000000000000..3f24dc68eff6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/linkage.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux specific Services code internal interfaces
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Interfaces between various parts of the Linux specific
+                Services code, that don't have any other obvious
+                header file to go into.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(LINKAGE_H)
+#define LINKAGE_H
+
+PVRSRV_ERROR PVROSFuncInit(void);
+void PVROSFuncDeInit(void);
+
+#endif /* !defined(LINKAGE_H) */
diff --git a/drivers/gpu/drm/img-rogue/linux_sw_sync.h b/drivers/gpu/drm/img-rogue/linux_sw_sync.h
new file mode 100644
index 000000000000..c12c650294a2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/linux_sw_sync.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef _UAPI_LINUX_PVR_SW_SYNC_H
+#define _UAPI_LINUX_PVR_SW_SYNC_H
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+
+#include <linux/types.h>
+
+#include "pvrsrv_sync_km.h"
+#include "pvr_drm.h"
+
+#endif /* defined(SUPPORT_NATIVE_FENCE_SYNC) */
+#endif
diff --git a/drivers/gpu/drm/img-rogue/lists.c b/drivers/gpu/drm/img-rogue/lists.c
new file mode 100644
index 000000000000..e8e7088a3296
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/lists.c
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linked list shared functions implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implementation of the list iterators for types shared among
+                more than one file in the services code.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "lists.h"
+
+/*===================================================================
+  LIST ITERATOR FUNCTIONS USED IN MORE THAN ONE FILE (those used just
+  once are implemented locally).
+  ===================================================================*/
+
+IMPLEMENT_LIST_ANY(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_ANY_2(PVRSRV_DEVICE_NODE, IMG_BOOL, IMG_FALSE)
+IMPLEMENT_LIST_ANY_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK)
+IMPLEMENT_LIST_ANY_VA(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_ANY_VA_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK)
+IMPLEMENT_LIST_FOR_EACH(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_FOR_EACH_VA(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_INSERT_TAIL(PVRSRV_DEVICE_NODE)
+IMPLEMENT_LIST_REMOVE(PVRSRV_DEVICE_NODE)
diff --git a/drivers/gpu/drm/img-rogue/lists.h b/drivers/gpu/drm/img-rogue/lists.h
new file mode 100644
index 000000000000..2e2c29a0b7b0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/lists.h
@@ -0,0 +1,367 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linked list shared functions templates.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Definition of the linked list function templates.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LISTS_UTILS_H
+#define LISTS_UTILS_H
+
+/* instruct QAC to ignore warnings about the following custom formatted macros */
+/* PRQA S 0881,3410 ++ */
+
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#include "img_types.h"
+#include "device.h"
+#include "power.h"
+
+/*
+ - USAGE -
+
+ The list functions work with any structure that provides the fields psNext and
+ ppsThis. In order to make a function available for a given type, it is required
+ to use the function template macro that creates the actual code.
+
+ There are 5 main types of functions:
+ - INSERT      : given a pointer to the head pointer of the list and a pointer
+                 to the node, inserts it as the new head.
+ - INSERT TAIL : given a pointer to the head pointer of the list and a pointer
+                 to the node, inserts the node at the tail of the list.
+ - REMOVE      : given a pointer to a node, removes it from its list.
+ - FOR EACH    : apply a function over all the elements of a list.
+ - ANY         : apply a function over the elements of a list, until one of them
+                 return a non null value, and then returns it.
+
+ The two last functions can have a variable argument form, with allows to pass
+ additional parameters to the callback function. In order to do this, the
+ callback function must take two arguments, the first is the current node and
+ the second is a list of variable arguments (va_list).
+
+ The ANY functions have also another for which specifies the return type of the
+ callback function and the default value returned by the callback function.
+
+*/
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_ForEach
+@Description    Apply a callback function to all the elements of a list.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+*/ /**************************************************************************/
+#define DECLARE_LIST_FOR_EACH(TYPE) \
+void List_##TYPE##_ForEach(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_FOR_EACH(TYPE) \
+void List_##TYPE##_ForEach(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))\
+{\
+	while (psHead)\
+	{\
+		pfnCallBack(psHead);\
+		psHead = psHead->psNext;\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_ForEachSafe
+@Description    Apply a callback function to all the elements of a list. Do it
+                in a safe way that handles the fact that a node might remove
+                itself from the list during the iteration.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+*/ /**************************************************************************/
+#define DECLARE_LIST_FOR_EACH_SAFE(TYPE) \
+void List_##TYPE##_ForEachSafe(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_FOR_EACH_SAFE(TYPE) \
+void List_##TYPE##_ForEachSafe(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode))\
+{\
+	TYPE *psNext;\
+\
+	while (psHead)\
+	{\
+		psNext = psHead->psNext; \
+		pfnCallBack(psHead);\
+		psHead = psNext;\
+	}\
+}
+
+
+#define DECLARE_LIST_FOR_EACH_VA(TYPE) \
+void List_##TYPE##_ForEach_va(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_FOR_EACH_VA(TYPE) \
+void List_##TYPE##_ForEach_va(TYPE *psHead, void(*pfnCallBack)(TYPE* psNode, va_list va), ...) \
+{\
+	va_list ap;\
+	while (psHead)\
+	{\
+		va_start(ap, pfnCallBack);\
+		pfnCallBack(psHead, ap);\
+		psHead = psHead->psNext;\
+		va_end(ap);\
+	}\
+}
+
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Any
+@Description    Applies a callback function to the elements of a list until
+                the function returns a non null value, then returns it.
+@Input          psHead        The head of the list to be processed.
+@Input          pfnCallBack   The function to be applied to each element of the list.
+@Return         The first non null value returned by the callback function.
+*/ /**************************************************************************/
+#define DECLARE_LIST_ANY(TYPE) \
+void* List_##TYPE##_Any(TYPE *psHead, void* (*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_ANY(TYPE) \
+void* List_##TYPE##_Any(TYPE *psHead, void* (*pfnCallBack)(TYPE* psNode))\
+{ \
+	void *pResult;\
+	TYPE *psNextNode;\
+	pResult = NULL;\
+	psNextNode = psHead;\
+	while (psHead && !pResult)\
+	{\
+		psNextNode = psNextNode->psNext;\
+		pResult = pfnCallBack(psHead);\
+		psHead = psNextNode;\
+	}\
+	return pResult;\
+}
+
+
+/*with variable arguments, that will be passed as a va_list to the callback function*/
+
+#define DECLARE_LIST_ANY_VA(TYPE) \
+void* List_##TYPE##_Any_va(TYPE *psHead, void*(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_ANY_VA(TYPE) \
+void* List_##TYPE##_Any_va(TYPE *psHead, void*(*pfnCallBack)(TYPE* psNode, va_list va), ...)\
+{\
+	va_list ap;\
+	TYPE *psNextNode;\
+	void* pResult = NULL;\
+	while (psHead && !pResult)\
+	{\
+		psNextNode = psHead->psNext;\
+		va_start(ap, pfnCallBack);\
+		pResult = pfnCallBack(psHead, ap);\
+		va_end(ap);\
+		psHead = psNextNode;\
+	}\
+	return pResult;\
+}
+
+/*those ones are for extra type safety, so there's no need to use castings for the results*/
+
+#define DECLARE_LIST_ANY_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any(TYPE *psHead, RTYPE (*pfnCallBack)(TYPE* psNode))
+
+#define IMPLEMENT_LIST_ANY_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any(TYPE *psHead, RTYPE (*pfnCallBack)(TYPE* psNode))\
+{ \
+	RTYPE result;\
+	TYPE *psNextNode;\
+	result = CONTINUE;\
+	psNextNode = psHead;\
+	while (psHead && result == CONTINUE)\
+	{\
+		psNextNode = psNextNode->psNext;\
+		result = pfnCallBack(psHead);\
+		psHead = psNextNode;\
+	}\
+	return result;\
+}
+
+
+#define DECLARE_LIST_ANY_VA_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any_va(TYPE *psHead, RTYPE(*pfnCallBack)(TYPE* psNode, va_list va), ...)
+
+#define IMPLEMENT_LIST_ANY_VA_2(TYPE, RTYPE, CONTINUE) \
+RTYPE List_##TYPE##_##RTYPE##_Any_va(TYPE *psHead, RTYPE(*pfnCallBack)(TYPE* psNode, va_list va), ...)\
+{\
+	va_list ap;\
+	TYPE *psNextNode;\
+	RTYPE result = CONTINUE;\
+	while (psHead && result == CONTINUE)\
+	{\
+		psNextNode = psHead->psNext;\
+		va_start(ap, pfnCallBack);\
+		result = pfnCallBack(psHead, ap);\
+		va_end(ap);\
+		psHead = psNextNode;\
+	}\
+	return result;\
+}
+
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Remove
+@Description    Removes a given node from the list.
+@Input          psNode      The pointer to the node to be removed.
+*/ /**************************************************************************/
+#define DECLARE_LIST_REMOVE(TYPE) \
+void List_##TYPE##_Remove(TYPE *psNode)
+
+#define IMPLEMENT_LIST_REMOVE(TYPE) \
+void List_##TYPE##_Remove(TYPE *psNode)\
+{\
+	(*psNode->ppsThis)=psNode->psNext;\
+	if (psNode->psNext)\
+	{\
+		psNode->psNext->ppsThis = psNode->ppsThis;\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Insert
+@Description    Inserts a given node at the beginning of the list.
+@Input          psHead   The pointer to the pointer to the head node.
+@Input          psNode   The pointer to the node to be inserted.
+*/ /**************************************************************************/
+#define DECLARE_LIST_INSERT(TYPE) \
+void List_##TYPE##_Insert(TYPE **ppsHead, TYPE *psNewNode)
+
+#define IMPLEMENT_LIST_INSERT(TYPE) \
+void List_##TYPE##_Insert(TYPE **ppsHead, TYPE *psNewNode)\
+{\
+	psNewNode->ppsThis = ppsHead;\
+	psNewNode->psNext = *ppsHead;\
+	*ppsHead = psNewNode;\
+	if (psNewNode->psNext)\
+	{\
+		psNewNode->psNext->ppsThis = &(psNewNode->psNext);\
+	}\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_InsertTail
+@Description    Inserts a given node at the end of the list.
+@Input          psHead   The pointer to the pointer to the head node.
+@Input          psNode   The pointer to the node to be inserted.
+*/ /**************************************************************************/
+#define DECLARE_LIST_INSERT_TAIL(TYPE) \
+void List_##TYPE##_InsertTail(TYPE **ppsHead, TYPE *psNewNode)
+
+#define IMPLEMENT_LIST_INSERT_TAIL(TYPE) \
+void List_##TYPE##_InsertTail(TYPE **ppsHead, TYPE *psNewNode)\
+{\
+	TYPE *psTempNode = *ppsHead;\
+	if (psTempNode != NULL)\
+	{\
+		while (psTempNode->psNext)\
+			psTempNode = psTempNode->psNext;\
+		ppsHead = &psTempNode->psNext;\
+	}\
+	psNewNode->ppsThis = ppsHead;\
+	psNewNode->psNext = NULL;\
+	*ppsHead = psNewNode;\
+}
+
+/*************************************************************************/ /*!
+@Function       List_##TYPE##_Reverse
+@Description    Reverse a list in place
+@Input          ppsHead    The pointer to the pointer to the head node.
+*/ /**************************************************************************/
+#define DECLARE_LIST_REVERSE(TYPE) \
+void List_##TYPE##_Reverse(TYPE **ppsHead)
+
+#define IMPLEMENT_LIST_REVERSE(TYPE) \
+void List_##TYPE##_Reverse(TYPE **ppsHead)\
+{\
+	TYPE *psTmpNode1; \
+	TYPE *psTmpNode2; \
+	TYPE *psCurNode; \
+	psTmpNode1 = NULL; \
+	psCurNode = *ppsHead; \
+	while (psCurNode) { \
+		psTmpNode2 = psCurNode->psNext; \
+		psCurNode->psNext = psTmpNode1; \
+		psTmpNode1 = psCurNode; \
+		psCurNode = psTmpNode2; \
+		if (psCurNode) \
+		{ \
+			psTmpNode1->ppsThis = &(psCurNode->psNext); \
+		} \
+		else \
+		{ \
+			psTmpNode1->ppsThis = ppsHead; \
+		} \
+	} \
+	*ppsHead = psTmpNode1; \
+}
+
+#define IS_LAST_ELEMENT(x) ((x)->psNext == NULL)
+
+
+DECLARE_LIST_ANY(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_ANY_2(PVRSRV_DEVICE_NODE, IMG_BOOL, IMG_FALSE);
+DECLARE_LIST_ANY_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK);
+DECLARE_LIST_ANY_VA(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_ANY_VA_2(PVRSRV_DEVICE_NODE, PVRSRV_ERROR, PVRSRV_OK);
+DECLARE_LIST_FOR_EACH(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_FOR_EACH_VA(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_INSERT_TAIL(PVRSRV_DEVICE_NODE);
+DECLARE_LIST_REMOVE(PVRSRV_DEVICE_NODE);
+
+#undef DECLARE_LIST_ANY_2
+#undef DECLARE_LIST_ANY_VA
+#undef DECLARE_LIST_ANY_VA_2
+#undef DECLARE_LIST_FOR_EACH
+#undef DECLARE_LIST_FOR_EACH_VA
+#undef DECLARE_LIST_INSERT
+#undef DECLARE_LIST_REMOVE
+
+#endif
+
+/* re-enable warnings */
+/* PRQA S 0881,3410 -- */
diff --git a/drivers/gpu/drm/img-rogue/lock.h b/drivers/gpu/drm/img-rogue/lock.h
new file mode 100644
index 000000000000..72997df1107e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/lock.h
@@ -0,0 +1,436 @@
+/*************************************************************************/ /*!
+@File           lock.h
+@Title          Locking interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services internal locking interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOCK_H
+#define LOCK_H
+
+/* In Linux kernel mode we are using the kernel mutex implementation directly
+ * with macros. This allows us to use the kernel lockdep feature for lock
+ * debugging. */
+#include "lock_types.h"
+
+#if defined(__linux__) && defined(__KERNEL__)
+
+#include "allocmem.h"
+#include <linux/atomic.h>
+#include <linux/version.h>
+
+#define OSLockCreateNoStats(phLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(phLock) = OSAllocMemNoStats(sizeof(struct mutex)); \
+	if (*(phLock)) { mutex_init(*(phLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSLockCreate(phLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(phLock) = OSAllocMem(sizeof(struct mutex)); \
+	if (*(phLock)) { mutex_init(*(phLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSLockDestroy(hLock) ({mutex_destroy((hLock)); OSFreeMem((hLock));})
+#define OSLockDestroyNoStats(hLock) ({mutex_destroy((hLock)); OSFreeMemNoStats((hLock));})
+
+#define OSLockAcquire(hLock) ({mutex_lock((hLock));})
+#define OSLockAcquireNested(hLock, subclass) ({mutex_lock_nested((hLock), (subclass));})
+#define OSLockRelease(hLock) ({mutex_unlock((hLock));})
+
+#define OSLockIsLocked(hLock) ((mutex_is_locked((hLock)) == 1) ? IMG_TRUE : IMG_FALSE)
+#define OSTryLockAcquire(hLock) ((mutex_trylock(hLock) == 1) ? IMG_TRUE : IMG_FALSE)
+
+#define OSSpinLockCreate(_ppsLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(_ppsLock) = OSAllocMem(sizeof(spinlock_t)); \
+	if (*(_ppsLock)) {spin_lock_init(*(_ppsLock)); e = PVRSRV_OK;} \
+	e;})
+#define OSSpinLockDestroy(_psLock) ({OSFreeMem(_psLock);})
+
+typedef unsigned long OS_SPINLOCK_FLAGS;
+#define OSSpinLockAcquire(_pLock, _flags) spin_lock_irqsave(_pLock, _flags)
+#define OSSpinLockRelease(_pLock, _flags) spin_unlock_irqrestore(_pLock, _flags)
+
+/* These _may_ be reordered or optimized away entirely by the compiler/hw */
+#define OSAtomicRead(pCounter)	atomic_read(pCounter)
+#define OSAtomicWrite(pCounter, i)	atomic_set(pCounter, i)
+
+/* The following atomic operations, in addition to being SMP-safe, also
+   imply a memory barrier around the operation */
+#define OSAtomicIncrement(pCounter) atomic_inc_return(pCounter)
+#define OSAtomicDecrement(pCounter) atomic_dec_return(pCounter)
+#define OSAtomicCompareExchange(pCounter, oldv, newv) atomic_cmpxchg(pCounter,oldv,newv)
+#define OSAtomicExchange(pCounter, iNewVal) atomic_xchg(pCounter, iNewVal)
+
+static inline IMG_INT OSAtomicOr(ATOMIC_T *pCounter, IMG_INT iVal)
+{
+	IMG_INT iOldVal, iLastVal, iNewVal;
+
+	iLastVal = OSAtomicRead(pCounter);
+	do
+	{
+		iOldVal = iLastVal;
+		iNewVal = iOldVal | iVal;
+
+		iLastVal = OSAtomicCompareExchange(pCounter, iOldVal, iNewVal);
+	}
+	while (iOldVal != iLastVal);
+
+	return iNewVal;
+}
+
+#define OSAtomicAdd(pCounter, incr) atomic_add_return(incr,pCounter)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)
+#define OSAtomicAddUnless(pCounter, incr, test) atomic_fetch_add_unless(pCounter, (incr), (test))
+#else
+#define OSAtomicAddUnless(pCounter, incr, test) __atomic_add_unless(pCounter,incr,test)
+#endif
+
+#define OSAtomicSubtract(pCounter, incr) atomic_add_return(-(incr),pCounter)
+#define OSAtomicSubtractUnless(pCounter, incr, test) OSAtomicAddUnless(pCounter, -(incr), (test))
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+/**************************************************************************/ /*!
+@Function       OSLockCreate
+@Description    Creates an operating system lock object.
+@Output         phLock           The created lock.
+@Return         PVRSRV_OK on success. PVRSRV_ERROR_OUT_OF_MEMORY if the driver
+                cannot allocate CPU memory needed for the lock.
+                PVRSRV_ERROR_INIT_FAILURE if the Operating System fails to
+                allocate the lock.
+ */ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR OSLockCreate(POS_LOCK *phLock);
+#if defined(INTEGRITY_OS) || defined(__QNXNTO__)
+#define OSLockCreateNoStats OSLockCreate
+#endif
+
+/**************************************************************************/ /*!
+@Function       OSLockDestroy
+@Description    Destroys an operating system lock object.
+@Input          hLock            The lock to be destroyed.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockDestroy(POS_LOCK hLock);
+
+#if defined(INTEGRITY_OS) || defined(__QNXNTO__)
+#define OSLockDestroyNoStats OSLockDestroy
+#endif
+/**************************************************************************/ /*!
+@Function       OSLockAcquire
+@Description    Acquires an operating system lock.
+                NB. This function must not return until the lock is acquired
+                (meaning the implementation should not timeout or return with
+                an error, as the caller will assume they have the lock).
+@Input          hLock            The lock to be acquired.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockAcquire(POS_LOCK hLock);
+
+/**************************************************************************/ /*!
+@Function       OSTryLockAcquire
+@Description    Try to acquire an operating system lock.
+                NB. If lock is acquired successfully in the first attempt,
+                then the function returns true and else it will return false.
+@Input          hLock            The lock to be acquired.
+@Return         IMG_TRUE if lock acquired successfully,
+                IMG_FALSE otherwise.
+ */ /**************************************************************************/
+IMG_INTERNAL
+IMG_BOOL OSTryLockAcquire(POS_LOCK hLock);
+
+/* Nested notation isn't used in UM or other OS's */
+/**************************************************************************/ /*!
+@Function       OSLockAcquireNested
+@Description    For operating systems other than Linux, this equates to an
+                OSLockAcquire() call. On Linux, this function wraps a call
+                to mutex_lock_nested(). This recognises the scenario where
+                there may be multiple subclasses within a particular class
+                of lock. In such cases, the order in which the locks belonging
+                these various subclasses are acquired is important and must be
+                validated.
+@Input          hLock            The lock to be acquired.
+@Input          subclass         The subclass of the lock.
+@Return         None.
+ */ /**************************************************************************/
+#define OSLockAcquireNested(hLock, subclass) OSLockAcquire((hLock))
+
+/**************************************************************************/ /*!
+@Function       OSLockRelease
+@Description    Releases an operating system lock.
+@Input          hLock            The lock to be released.
+@Return         None.
+ */ /**************************************************************************/
+IMG_INTERNAL
+void OSLockRelease(POS_LOCK hLock);
+
+/**************************************************************************/ /*!
+@Function       OSLockIsLocked
+@Description    Tests whether or not an operating system lock is currently
+                locked.
+@Input          hLock            The lock to be tested.
+@Return         IMG_TRUE if locked, IMG_FALSE if not locked.
+ */ /**************************************************************************/
+IMG_INTERNAL
+IMG_BOOL OSLockIsLocked(POS_LOCK hLock);
+
+#if defined(__linux__)
+
+/* Use GCC intrinsics (read/write semantics consistent with kernel-side implementation) */
+#define OSAtomicRead(pCounter) (*(volatile IMG_INT32 *)&(pCounter)->counter)
+#define OSAtomicWrite(pCounter, i) ((pCounter)->counter = (IMG_INT32) i)
+#define OSAtomicIncrement(pCounter) __sync_add_and_fetch((&(pCounter)->counter), 1)
+#define OSAtomicDecrement(pCounter) __sync_sub_and_fetch((&(pCounter)->counter), 1)
+#define OSAtomicCompareExchange(pCounter, oldv, newv) \
+	__sync_val_compare_and_swap((&(pCounter)->counter), oldv, newv)
+#define OSAtomicOr(pCounter, iVal) __sync_or_and_fetch((&(pCounter)->counter), iVal)
+
+static inline IMG_UINT32 OSAtomicExchange(ATOMIC_T *pCounter, IMG_UINT32 iNewVal)
+{
+	IMG_UINT32 iOldVal;
+	IMG_UINT32 iLastVal;
+
+	iLastVal = OSAtomicRead(pCounter);
+	do
+	{
+		iOldVal = iLastVal;
+		iLastVal = OSAtomicCompareExchange(pCounter, iOldVal, iNewVal);
+	}
+	while (iOldVal != iLastVal);
+
+	return iOldVal;
+}
+
+#define OSAtomicAdd(pCounter, incr) __sync_add_and_fetch((&(pCounter)->counter), incr)
+#define OSAtomicAddUnless(pCounter, incr, test) ({ \
+	IMG_INT32 c; IMG_INT32 old; \
+	c = OSAtomicRead(pCounter); \
+	while (1) { \
+		if (c == (test)) break; \
+		old = OSAtomicCompareExchange(pCounter, c, c+(incr)); \
+		if (old == c) break; \
+		c = old; \
+	} c; })
+
+#define OSAtomicSubtract(pCounter, incr) OSAtomicAdd(pCounter, -(incr))
+#define OSAtomicSubtractUnless(pCounter, incr, test) OSAtomicAddUnless(pCounter, -(incr), test)
+
+#else
+
+/*************************************************************************/ /*!
+@Function       OSAtomicRead
+@Description    Read the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to read
+@Return         The value of the atomic variable
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicRead(const ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicWrite
+@Description    Write the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be written to
+@Input          v               The value to write
+@Return         None
+*/ /**************************************************************************/
+IMG_INTERNAL
+void OSAtomicWrite(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/* For the following atomic operations, in addition to being SMP-safe,
+   should also  have a memory barrier around each operation */
+/*************************************************************************/ /*!
+@Function       OSAtomicIncrement
+@Description    Increment the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be incremented
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicIncrement(ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicDecrement
+@Description    Decrement the value of a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be decremented
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicDecrement(ATOMIC_T *pCounter);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicAdd
+@Description    Add a specified value to a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to add the value to
+@Input          v               The value to be added
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicAdd(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicAddUnless
+@Description    Add a specified value to a variable atomically unless it
+                already equals a particular value.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to add the value to
+@Input          v               The value to be added to 'pCounter'
+@Input          t               The test value. If 'pCounter' equals this,
+                                its value will not be adjusted
+@Return         The old value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicAddUnless(ATOMIC_T *pCounter, IMG_INT32 v, IMG_INT32 t);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicSubtract
+@Description    Subtract a specified value to a variable atomically.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to subtract the value from
+@Input          v               The value to be subtracted
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicSubtract(ATOMIC_T *pCounter, IMG_INT32 v);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicSubtractUnless
+@Description    Subtract a specified value from a variable atomically unless
+                it already equals a particular value.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to subtract the value from
+@Input          v               The value to be subtracted from 'pCounter'
+@Input          t               The test value. If 'pCounter' equals this,
+                                its value will not be adjusted
+@Return         The old value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicSubtractUnless(ATOMIC_T *pCounter, IMG_INT32 v, IMG_INT32 t);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicCompareExchange
+@Description    Set a variable to a given value only if it is currently
+                equal to a specified value. The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be checked and
+                                possibly updated
+@Input          oldv            The value the atomic variable must have in
+                                order to be modified
+@Input          newv            The value to write to the atomic variable if
+                                it equals 'oldv'
+@Return         The old value of *pCounter
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicCompareExchange(ATOMIC_T *pCounter, IMG_INT32 oldv, IMG_INT32 newv);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicExchange
+@Description    Set a variable to a given value and retrieve previous value.
+                The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be updated
+@Input          iNewVal         The value to write to the atomic variable
+@Return         The previous value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicExchange(ATOMIC_T *pCounter, IMG_INT32 iNewVal);
+
+/*************************************************************************/ /*!
+@Function       OSAtomicOr
+@Description    Set a variable to the bitwise or of its current value and the
+                specified value. Equivalent to *pCounter |= iVal.
+                The whole operation must be atomic.
+                Atomic functions must be implemented in a manner that
+                is both symmetric multiprocessor (SMP) safe and has a memory
+                barrier around each operation.
+@Input          pCounter        The atomic variable to be updated
+@Input          iVal            The value to bitwise or against
+@Return         The new value of *pCounter.
+*/ /**************************************************************************/
+IMG_INTERNAL
+IMG_INT32 OSAtomicOr(ATOMIC_T *pCounter, IMG_INT32 iVal);
+
+/* For now, spin-locks are required on Linux only, so other platforms fake
+ * spinlocks with normal mutex locks */
+/*! Type definitions for OS_SPINLOCK accessor and creation / deletion */
+typedef unsigned long OS_SPINLOCK_FLAGS;
+/*! Pointer to an OS Spinlock */
+#define POS_SPINLOCK POS_LOCK
+/*! Wrapper for OSLockCreate() */
+#define OSSpinLockCreate(ppLock) OSLockCreate(ppLock)
+/*! Wrapper for OSLockDestroy() */
+#define OSSpinLockDestroy(pLock) OSLockDestroy(pLock)
+/*! Wrapper for OSLockAcquire() */
+#define OSSpinLockAcquire(pLock, flags) {flags = 0; OSLockAcquire(pLock);}
+/*! Wrapper for OSLockRelease() */
+#define OSSpinLockRelease(pLock, flags) {flags = 0; OSLockRelease(pLock);}
+
+#endif /* defined(__linux__) */
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#endif /* LOCK_H */
diff --git a/drivers/gpu/drm/img-rogue/lock_types.h b/drivers/gpu/drm/img-rogue/lock_types.h
new file mode 100644
index 000000000000..bf7167b554f6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/lock_types.h
@@ -0,0 +1,92 @@
+/*************************************************************************/ /*!
+@File           lock_types.h
+@Title          Locking types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Locking specific enums, defines and structures
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOCK_TYPES_H
+#define LOCK_TYPES_H
+
+/* In Linux kernel mode we are using the kernel mutex implementation directly
+ * with macros. This allows us to use the kernel lockdep feature for lock
+ * debugging. */
+#if defined(__linux__) && defined(__KERNEL__)
+
+#include <linux/types.h>
+#include <linux/mutex.h>
+/* The mutex is defined as a pointer to be compatible with the other code. This
+ * isn't ideal and usually you wouldn't do that in kernel code. */
+typedef struct mutex *POS_LOCK;
+typedef struct rw_semaphore *POSWR_LOCK;
+typedef spinlock_t *POS_SPINLOCK;
+typedef atomic_t ATOMIC_T;
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+#include "img_types.h" /* needed for IMG_INT */
+typedef struct OS_LOCK_TAG *POS_LOCK;
+
+#if defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+typedef struct OSWR_LOCK_TAG *POSWR_LOCK;
+#else /* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+typedef struct OSWR_LOCK_TAG {
+	IMG_UINT32 ui32Unused;
+} *POSWR_LOCK;
+#endif /* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+
+#if defined(__linux__) || defined(__APPLE__)
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(__QNXNTO__)
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(_WIN32)
+	/*
+	 * Placeholder definition. WDDM doesn't use Services, but some headers
+	 * still have to be shared. This is one such case.
+	 */
+	typedef struct OS_ATOMIC_TAG {IMG_INT32 counter;} ATOMIC_T;
+#elif defined(INTEGRITY_OS)
+	/* Only lower 32bits are used in OS ATOMIC APIs to have consistent behaviour across all OS */
+	typedef struct OS_ATOMIC_TAG {IMG_INT64 counter;} ATOMIC_T;
+#else
+	#error "Please type-define an atomic lock for this environment"
+#endif
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#endif /* LOCK_TYPES_H */
diff --git a/drivers/gpu/drm/img-rogue/log2.h b/drivers/gpu/drm/img-rogue/log2.h
new file mode 100644
index 000000000000..795660400850
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/log2.h
@@ -0,0 +1,417 @@
+/*************************************************************************/ /*!
+@Title          Integer log2 and related functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef LOG2_H
+#define LOG2_H
+
+#include "img_defs.h"
+
+/*************************************************************************/ /*!
+@Description    Determine if a number is a power of two.
+@Input          n
+@Return         True if n is a power of 2, false otherwise. True if n == 0.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL __const_function IsPower2(uint32_t n)
+{
+	/* C++ needs this cast. */
+	return (IMG_BOOL)((n & (n - 1U)) == 0U);
+}
+
+/*************************************************************************/ /*!
+@Description    Determine if a number is a power of two.
+@Input          n
+@Return         True if n is a power of 2, false otherwise. True if n == 0.
+*/ /**************************************************************************/
+static INLINE IMG_BOOL __const_function IsPower2_64(uint64_t n)
+{
+	/* C++ needs this cast. */
+	return (IMG_BOOL)((n & (n - 1U)) == 0U);
+}
+
+/* Code using GNU GCC intrinsics */
+#if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER))
+
+/* CHAR_BIT is typically found in <limits.h>. For all the platforms where
+ * CHAR_BIT is not available, defined it here with the assumption that there
+ * are 8 bits in a byte */
+#ifndef CHAR_BIT
+#define CHAR_BIT 8U
+#endif
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2(uint32_t n)
+{
+	if (unlikely(n == 0U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+		return uNumBits - (uint32_t)__builtin_clz(n) - 1U;
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2_64(uint64_t n)
+{
+	if (unlikely(n == 0U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+		return uNumBits - (uint32_t)__builtin_clzll(n) - 1U;
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2(uint32_t n)
+{
+	if (unlikely(n == 0U || n == 1U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+
+		n--; /* Handle powers of 2 */
+		return uNumBits - (uint32_t)__builtin_clz(n);
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2_64(uint64_t n)
+{
+	if (unlikely(n == 0U || n == 1U))
+	{
+		return 0;
+	}
+	else
+	{
+		uint32_t uNumBits = (uint32_t)CHAR_BIT * (uint32_t)sizeof(n);
+
+		n--; /* Handle powers of 2 */
+		return uNumBits - (uint32_t)__builtin_clzll(n);
+	}
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2(uint32_t n)
+{
+	return (uint32_t)CHAR_BIT * (uint32_t)sizeof(n) - (uint32_t)__builtin_clz(n) - 1U;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2_64(uint64_t n)
+{
+	return (uint32_t)CHAR_BIT * (uint32_t)sizeof(n) - (uint32_t)__builtin_clzll(n) - 1U;
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function RoundUpToNextPowerOfTwo(uint32_t n)
+{
+	/* Cases with n greater than 2^31 needs separate handling
+	 * as result of (1<<32) is undefined. */
+	if (unlikely( n == 0U || n > (uint32_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - 1U)))
+	{
+		return 0;
+	}
+
+	/* Return n if it is already a power of 2 */
+	if ((IMG_BOOL)((n & (n - 1U)) == 0U))
+	{
+		return n;
+	}
+
+	return (uint32_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - (uint32_t)__builtin_clz(n));
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint64_t __const_function RoundUpToNextPowerOfTwo_64(uint64_t n)
+{
+	/* Cases with n greater than 2^63 needs separate handling
+	 * as result of (1<<64) is undefined. */
+	if (unlikely( n == 0U || n > (uint64_t)1 << ((uint32_t)CHAR_BIT * sizeof(n) - 1U)))
+	{
+		return 0;
+	}
+
+	/* Return n if it is already a power of 2 */
+	if ((IMG_BOOL)((n & (n - 1U)) == 0U))
+	{
+		return n;
+	}
+
+	return (uint64_t)1 << ((uint64_t)CHAR_BIT * sizeof(n) - (uint64_t)__builtin_clzll(n));
+}
+
+#else /* #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER)) */
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function RoundUpToNextPowerOfTwo(uint32_t n)
+{
+	n--;
+	n |= n >> 1;  /* handle  2 bit numbers */
+	n |= n >> 2;  /* handle  4 bit numbers */
+	n |= n >> 4;  /* handle  8 bit numbers */
+	n |= n >> 8;  /* handle 16 bit numbers */
+	n |= n >> 16; /* handle 32 bit numbers */
+	n++;
+
+	return n;
+}
+
+/*************************************************************************/ /*!
+@Description    Round a non-power-of-two number up to the next power of two.
+@Input          n
+@Return         n rounded up to the next power of two. If n is zero or
+                already a power of two, return n unmodified.
+*/ /**************************************************************************/
+static INLINE uint64_t __const_function RoundUpToNextPowerOfTwo_64(uint64_t n)
+{
+	n--;
+	n |= n >> 1;  /* handle  2 bit numbers */
+	n |= n >> 2;  /* handle  4 bit numbers */
+	n |= n >> 4;  /* handle  8 bit numbers */
+	n |= n >> 8;  /* handle 16 bit numbers */
+	n |= n >> 16; /* handle 32 bit numbers */
+	n |= n >> 32; /* handle 64 bit numbers */
+	n++;
+
+	return n;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2(uint32_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	for (n >>= 1; n != 0U; n >>= 1)
+	{
+		ui32log2++;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(n))
+@Input          n
+@Return         log2(n) rounded down to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function FloorLog2_64(uint64_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	for (n >>= 1; n != 0U; n >>= 1)
+	{
+		ui32log2++;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2(uint32_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	if (n == 0U)
+	{
+		return 0;
+	}
+
+	n--; /* Handle powers of 2 */
+
+	while (n != 0U)
+	{
+		ui32log2++;
+		n >>= 1;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute ceil(log2(n))
+@Input          n
+@Return         log2(n) rounded up to the nearest integer. Returns 0 if n == 0
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function CeilLog2_64(uint64_t n)
+{
+	uint32_t ui32log2 = 0;
+
+	if (n == 0U)
+	{
+		return 0;
+	}
+
+	n--; /* Handle powers of 2 */
+
+	while (n != 0U)
+	{
+		ui32log2++;
+		n >>= 1;
+	}
+
+	return ui32log2;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2(uint32_t n)
+{
+	static const uint32_t b[] =
+		{0xAAAAAAAAU, 0xCCCCCCCCU, 0xF0F0F0F0U, 0xFF00FF00U, 0xFFFF0000U};
+	uint32_t r = (((n & b[0]) != 0U) ? 1U : 0U);
+
+	r |= (uint32_t) (((n & b[4]) != 0U) ? (1U << 4) : 0U);
+	r |= (uint32_t) (((n & b[3]) != 0U) ? (1U << 3) : 0U);
+	r |= (uint32_t) (((n & b[2]) != 0U) ? (1U << 2) : 0U);
+	r |= (uint32_t) (((n & b[1]) != 0U) ? (1U << 1) : 0U);
+
+	return r;
+}
+
+/*************************************************************************/ /*!
+@Description    Compute log2(n) for exact powers of two only
+@Input          n                   Must be a power of two
+@Return         log2(n)
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function ExactLog2_64(uint64_t n)
+{
+	static const uint64_t b[] =
+		{0xAAAAAAAAAAAAAAAAUL, 0xCCCCCCCCCCCCCCCCUL,
+		 0xF0F0F0F0F0F0F0F0UL, 0xFF00FF00FF00FF00UL,
+		 0xFFFF0000FFFF0000UL, 0xFFFFFFFF00000000UL};
+	uint32_t r = (((n & b[0]) != 0U) ? 1U : 0U);
+
+	r |= (uint32_t) (((n & b[5]) != 0U) ? (1U << 5) : 0U);
+	r |= (uint32_t) (((n & b[4]) != 0U) ? (1U << 4) : 0U);
+	r |= (uint32_t) (((n & b[3]) != 0U) ? (1U << 3) : 0U);
+	r |= (uint32_t) (((n & b[2]) != 0U) ? (1U << 2) : 0U);
+	r |= (uint32_t) (((n & b[1]) != 0U) ? (1U << 1) : 0U);
+
+	return r;
+}
+
+#endif /* #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__) || defined(__INTEL_COMPILER)) */
+
+/*************************************************************************/ /*!
+@Description    Compute floor(log2(size)) , where size is the max of 3 sizes
+				This is almost always the ONLY EVER valid use of FloorLog2.
+				Usually CeilLog2() should be used instead.
+				For a 5x5x1 texture, the 3 miplevels are:
+					0:  5x5x1
+					1:	2x2x1
+					2:	1x1x1
+
+				For an 8x8x1 texture, the 4 miplevels are:
+					0:  8x8x1
+					1:	4x4x1
+					2:  2x2x1
+					3:  1x1x1
+
+
+@Input          sizeX, sizeY, sizeZ
+@Return         Count of mipmap levels for given dimensions
+*/ /**************************************************************************/
+static INLINE uint32_t __const_function NumMipLevels(uint32_t sizeX, uint32_t sizeY, uint32_t sizeZ)
+{
+
+	uint32_t maxSize = MAX(MAX(sizeX, sizeY), sizeZ);
+	return FloorLog2(maxSize) + 1U;
+}
+
+#endif /* LOG2_H */
diff --git a/drivers/gpu/drm/img-rogue/mem_utils.c b/drivers/gpu/drm/img-rogue/mem_utils.c
new file mode 100644
index 000000000000..59f808cfcb20
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/mem_utils.c
@@ -0,0 +1,445 @@
+/*************************************************************************/ /*!
+@File
+@Title          Memory manipulation functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory related functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "osfunc_common.h"
+#include "img_defs.h"
+
+/* This workaround is only *required* on ARM64. Avoid building or including
+ * it by default on other architectures, unless the 'safe memcpy' test flag
+ * is enabled. (The code should work on other architectures.)
+ */
+
+
+
+/* NOTE: This C file is compiled with -ffreestanding to avoid pattern matching
+ *       by the compiler to stdlib functions, and it must only use the below
+ *       headers. Do not include any IMG or services headers in this file.
+ */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/* The attribute "vector_size" will generate floating point instructions
+ * and use FPU registers. In kernel OS, the FPU registers might be corrupted
+ * when CPU is doing context switch because FPU registers are not expected to
+ * be stored.
+ * GCC enables compiler option, -mgeneral-regs-only, by default.
+ * This option restricts the generated code to use general registers only
+ * so that we don't have issues on that.
+ */
+#if defined(__KERNEL__) && defined(__clang__)
+
+#define DEVICE_MEMSETCPY_NON_VECTOR_KM
+/* Loading or storing 16 or 32 bytes is only supported on 64-bit machines. */
+#if DEVICE_MEMSETCPY_ALIGN_IN_BYTES > 8
+typedef __uint128_t uint128_t;
+
+typedef struct
+{
+	uint128_t ui128DataFields[2];
+}
+uint256_t;
+#endif
+
+#endif
+
+/* This file is only intended to be used on platforms which use GCC or Clang,
+ * due to its requirement on __attribute__((vector_size(n))), typeof() and
+ * __SIZEOF__ macros.
+ */
+
+#if defined(__GNUC__)
+
+#ifndef MIN
+#define MIN(a, b) \
+ ({__typeof(a) _a = (a); __typeof(b) _b = (b); _a > _b ? _b : _a;})
+#endif
+
+#if !defined(DEVICE_MEMSETCPY_ALIGN_IN_BYTES)
+#define DEVICE_MEMSETCPY_ALIGN_IN_BYTES __SIZEOF_LONG__
+#endif
+#if (DEVICE_MEMSETCPY_ALIGN_IN_BYTES & (DEVICE_MEMSETCPY_ALIGN_IN_BYTES - 1)) != 0
+#error "DEVICE_MEMSETCPY_ALIGN_IN_BYTES must be a power of 2"
+#endif
+#if DEVICE_MEMSETCPY_ALIGN_IN_BYTES < 4
+#error "DEVICE_MEMSETCPY_ALIGN_IN_BYTES must be equal or greater than 4"
+#endif
+
+#if __SIZEOF_POINTER__ != __SIZEOF_LONG__
+#error No support for architectures where void* and long are sized differently
+#endif
+
+#if __SIZEOF_LONG__ > DEVICE_MEMSETCPY_ALIGN_IN_BYTES
+/* Meaningless, and harder to do correctly */
+# error Cannot handle DEVICE_MEMSETCPY_ALIGN_IN_BYTES < sizeof(long)
+typedef unsigned long block_t;
+#elif __SIZEOF_LONG__ <= DEVICE_MEMSETCPY_ALIGN_IN_BYTES
+# if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+#  if DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+    typedef uint64_t block_t;
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+    typedef uint128_t block_t;
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+    typedef uint256_t block_t;
+#  endif
+# else
+typedef unsigned int block_t
+	__attribute__((vector_size(DEVICE_MEMSETCPY_ALIGN_IN_BYTES)));
+# endif
+# if defined(__arm64__) || defined(__aarch64__)
+#  if   DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+#   define DEVICE_MEMSETCPY_ARM64
+#   define REGSZ "w"
+#   define REGCL "w"
+#   define BVCLB "r"
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+#   define DEVICE_MEMSETCPY_ARM64
+#   define REGSZ "x"
+#   define REGCL "x"
+#   define BVCLB "r"
+#  elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+#   if defined(__ARM_NEON_FP)
+#    define DEVICE_MEMSETCPY_ARM64
+#    define REGSZ "q"
+#    define REGCL "v"
+#    define BVCLB "w"
+#   endif
+#  endif
+#  if defined(DEVICE_MEMSETCPY_ARM64)
+#   if defined(DEVICE_MEMSETCPY_ARM64_NON_TEMPORAL)
+#    define NSHLD() __asm__ ("dmb nshld")
+#    define NSHST() __asm__ ("dmb nshst")
+#    define LDP "ldnp"
+#    define STP "stnp"
+#   else
+#    define NSHLD()
+#    define NSHST()
+#    define LDP "ldp"
+#    define STP "stp"
+#   endif
+#   if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+#    if DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 8
+typedef uint32_t block_half_t;
+#    elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 16
+typedef uint64_t block_half_t;
+#    elif DEVICE_MEMSETCPY_ALIGN_IN_BYTES == 32
+typedef uint128_t block_half_t;
+#    endif
+#   else
+ typedef unsigned int block_half_t
+	__attribute__((vector_size(DEVICE_MEMSETCPY_ALIGN_IN_BYTES / 2)));
+#   endif
+#  endif
+# endif
+#endif
+
+__attribute__((visibility("hidden")))
+void DeviceMemCopy(void *pvDst, const void *pvSrc, size_t uSize)
+{
+	volatile const char *pcSrc = pvSrc;
+	volatile char *pcDst = pvDst;
+	size_t uPreambleBytes;
+	int bBlockCopy = 0;
+
+	size_t uSrcUnaligned = (size_t)pcSrc % sizeof(block_t);
+	size_t uDstUnaligned = (size_t)pcDst % sizeof(block_t);
+
+	if (!uSrcUnaligned && !uDstUnaligned)
+	{
+		/* Neither pointer is unaligned. Optimal case. */
+		bBlockCopy = 1;
+	}
+	else
+	{
+		if (uSrcUnaligned == uDstUnaligned)
+		{
+			/* Neither pointer is usefully aligned, but they are misaligned in
+			 * the same way, so we can copy a preamble in a slow way, then
+			 * optimize the rest.
+			 */
+			uPreambleBytes = MIN(sizeof(block_t) - uDstUnaligned, uSize);
+			uSize -= uPreambleBytes;
+			while (uPreambleBytes)
+			{
+				*pcDst++ = *pcSrc++;
+				uPreambleBytes--;
+			}
+
+			bBlockCopy = 1;
+		}
+		else if ((uSrcUnaligned | uDstUnaligned) % sizeof(int) == 0)
+		{
+			/* Both pointers are at least 32-bit aligned, and we assume that
+			 * the processor must handle all kinds of 32-bit load-stores.
+			 * NOTE: Could we optimize this with a non-temporal version?
+			 */
+			if (uSize >= sizeof(int))
+			{
+				volatile int *piSrc = (int *)((void *)pcSrc);
+				volatile int *piDst = (int *)((void *)pcDst);
+
+				while (uSize >= sizeof(int))
+				{
+					*piDst++ = *piSrc++;
+					uSize -= sizeof(int);
+				}
+
+				pcSrc = (char *)((void *)piSrc);
+				pcDst = (char *)((void *)piDst);
+			}
+		}
+	}
+
+	if (bBlockCopy && uSize >= sizeof(block_t))
+	{
+		volatile block_t *pSrc = (block_t *)((void *)pcSrc);
+		volatile block_t *pDst = (block_t *)((void *)pcDst);
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHLD();
+#endif
+
+		while (uSize >= sizeof(block_t))
+		{
+#if defined(DEVICE_MEMSETCPY_ARM64)
+			__asm__ (LDP " " REGSZ "0, " REGSZ "1, [%[pSrc]]\n\t"
+			         STP " " REGSZ "0, " REGSZ "1, [%[pDst]]"
+						:
+						: [pSrc] "r" (pSrc), [pDst] "r" (pDst)
+						: "memory", REGCL "0", REGCL "1");
+#else
+			*pDst = *pSrc;
+#endif
+			pDst++;
+			pSrc++;
+			uSize -= sizeof(block_t);
+		}
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHST();
+#endif
+
+		pcSrc = (char *)((void *)pSrc);
+		pcDst = (char *)((void *)pDst);
+	}
+
+	while (uSize)
+	{
+		*pcDst++ = *pcSrc++;
+		uSize--;
+	}
+}
+
+__attribute__((visibility("hidden")))
+void DeviceMemSet(void *pvDst, unsigned char ui8Value, size_t uSize)
+{
+	volatile char *pcDst = pvDst;
+	size_t uPreambleBytes;
+
+	size_t uDstUnaligned = (size_t)pcDst % sizeof(block_t);
+
+	if (uDstUnaligned)
+	{
+		uPreambleBytes = MIN(sizeof(block_t) - uDstUnaligned, uSize);
+		uSize -= uPreambleBytes;
+		while (uPreambleBytes)
+		{
+			*pcDst++ = ui8Value;
+			uPreambleBytes--;
+		}
+	}
+
+	if (uSize >= sizeof(block_t))
+	{
+		volatile block_t *pDst = (block_t *)((void *)pcDst);
+		size_t i, uBlockSize;
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		typedef block_half_t BLK_t;
+#else
+		typedef block_t BLK_t;
+#endif /* defined(DEVICE_MEMSETCPY_ARM64) */
+
+#if defined(DEVICE_MEMSETCPY_NON_VECTOR_KM)
+		BLK_t bValue = 0;
+
+		uBlockSize = sizeof(BLK_t) / sizeof(ui8Value);
+
+		for (i = 0; i < uBlockSize; i++)
+		{
+			bValue |= (BLK_t)ui8Value << ((uBlockSize - i - 1) * BITS_PER_BYTE);
+		}
+#else
+		BLK_t bValue = {0};
+
+		uBlockSize = sizeof(bValue) / sizeof(unsigned int);
+		for (i = 0; i < uBlockSize; i++)
+			bValue[i] = ui8Value << 24U |
+			            ui8Value << 16U |
+			            ui8Value <<  8U |
+			            ui8Value;
+#endif /* defined(DEVICE_MEMSETCPY_NON_VECTOR_KM) */
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHLD();
+#endif
+
+		while (uSize >= sizeof(block_t))
+		{
+#if defined(DEVICE_MEMSETCPY_ARM64)
+			__asm__ (STP " %" REGSZ "[bValue], %" REGSZ "[bValue], [%[pDst]]"
+						:
+						: [bValue] BVCLB (bValue), [pDst] "r" (pDst)
+						: "memory");
+#else
+			*pDst = bValue;
+#endif
+			pDst++;
+			uSize -= sizeof(block_t);
+		}
+
+#if defined(DEVICE_MEMSETCPY_ARM64)
+		NSHST();
+#endif
+
+		pcDst = (char *)((void *)pDst);
+	}
+
+	while (uSize)
+	{
+		*pcDst++ = ui8Value;
+		uSize--;
+	}
+}
+
+#endif /* defined(__GNUC__) */
+
+/* Potentially very slow (but safe) fallbacks for non-GNU C compilers */
+IMG_INTERNAL
+void DeviceMemCopyBytes(void *pvDst, const void *pvSrc, size_t uSize)
+{
+	volatile const char *pcSrc = pvSrc;
+	volatile char *pcDst = pvDst;
+
+	while (uSize)
+	{
+		*pcDst++ = *pcSrc++;
+		uSize--;
+	}
+}
+
+IMG_INTERNAL
+void DeviceMemSetBytes(void *pvDst, unsigned char ui8Value, size_t uSize)
+{
+	volatile char *pcDst = pvDst;
+
+	while (uSize)
+	{
+		*pcDst++ = ui8Value;
+		uSize--;
+	}
+}
+
+#if defined(__linux__) && defined(__KERNEL__)
+#if defined(DEBUG)
+/*
+ * In case of Linux kernel-mode in a debug build, choose the variant of
+ * OSStringSafeCopy that uses strscpy and logs truncation via a stack dump. For
+ * Linux kernel-mode in a release build, strscpy alone is used.
+ */
+IMG_INTERNAL
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	/*
+	 * Let strscpy handle any truncation cases correctly.
+	 * We will definitely get a NUL-terminated string set in pszDest
+	 */
+	ssize_t sCopiedCnt = strscpy(pszDest, pszSrc, uDataSize);
+
+#if defined(PVR_DEBUG_STRLCPY)
+	/* Handle truncation by dumping calling stack if debug allows */
+	if (sCopiedCnt < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+			"%s: String truncated Src = '<%s>' %zu bytes, Dest = '%s'",
+			__func__, pszSrc, uDataSize, pszDest));
+		OSDumpStack();
+	}
+#endif /* defined(PVR_DEBUG_STRLCPY) && defined(DEBUG) */
+
+	return sCopiedCnt;
+}
+#endif /* defined(DEBUG) */
+#else /* defined(__linux__) && defined(__KERNEL__) */
+/*
+ * For every other platform, make use of the strnlen and strncpy implementation
+ * of OSStringSafeCopy.
+ *
+ * NOTE: It is crucial to avoid memcpy as this has a hidden side-effect of
+ * dragging in whatever the build-environment flavour of GLIBC is which can
+ * cause unexpected failures for host-side command execution.
+ */
+IMG_INTERNAL
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	/* Match the specification of Linux strscpy - if destination size is 0,
+	 * return negative value (-E2BIG to be precise, but it's probably best
+	 * not to use Linux-specific return values, so -1 will do). */
+	if (uDataSize && (uDataSize <= SSIZE_MAX))
+	{
+		size_t uSrcSize = strnlen(pszSrc, uDataSize);
+
+		(void)strncpy(pszDest, pszSrc, MIN(uSrcSize + 1, uDataSize));
+		if (uSrcSize == uDataSize)
+		{
+			pszDest[uSrcSize-1] = '\0';
+			return -1;
+		}
+
+		return (ssize_t)uSrcSize;
+	}
+
+	return -1;
+}
+#endif /* defined(__linux__) && defined(__KERNEL__) */
diff --git a/drivers/gpu/drm/img-rogue/mmu_common.c b/drivers/gpu/drm/img-rogue/mmu_common.c
new file mode 100644
index 000000000000..3c402ec25c8e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/mmu_common.c
@@ -0,0 +1,5173 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "dllist.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "htbserver.h"
+#include "pvr_ricommon.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+# include "process_stats.h"
+# include "proc_stats.h"
+#endif
+
+#if defined(PDUMP)
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#endif
+
+#include "physmem.h"
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "physmem_lma.h"
+#endif
+
+/*
+Major Interfaces to other modules:
+
+Let's keep this graph up-to-date:
+
+   +-----------+
+   | devicemem |
+   +-----------+
+         |
+   +============+
+   | mmu_common |
+   +============+
+         |
+         +-----------------+
+         |                 |
+    +---------+      +----------+
+    |   pmr   |      |  device  |
+    +---------+      +----------+
+ */
+
+#include "mmu_common.h"
+#include "pmr.h"
+#include "devicemem_server_utils.h"
+
+/* #define MMU_OBJECT_REFCOUNT_DEBUGING 1 */
+#if defined(MMU_OBJECT_REFCOUNT_DEBUGING)
+#define MMU_OBJ_DBG(x)	PVR_DPF(x)
+#else
+#define MMU_OBJ_DBG(x)
+#endif
+
+#define SCRATCH_PAGE 1
+#define DEV_ZERO_PAGE 2
+#if defined(PDUMP)
+#define SCRATCH_PAGE_STR "SCRATCH_PAGE"
+#define DEV_ZERO_PAGE_STR "DEV_ZERO_PAGE"
+#endif
+#define PVR_SCRATCH_PAGE_INIT_VALUE 0
+#define PVR_ZERO_PAGE_INIT_VALUE  0
+
+/*!
+ * Refcounted structure that is shared between the context and
+ * the cleanup thread items.
+ * It is used to keep track of all cleanup items and whether the creating
+ * MMU context has been destroyed and therefore is not allowed to be
+ * accessed any more.
+ *
+ * The cleanup thread is used to defer the freeing of the page tables
+ * because we have to make sure that the MMU cache has been invalidated.
+ * If we don't take care of this the MMU might partially access cached
+ * and uncached tables which might lead to inconsistencies and in the
+ * worst case to MMU pending faults on random memory.
+ */
+typedef struct _MMU_CTX_CLEANUP_DATA_
+{
+	/*! Refcount to know when this structure can be destroyed */
+	ATOMIC_T iRef;
+	/*! Protect items in this structure, especially the refcount */
+	POS_LOCK hCleanupLock;
+	/*! List of all cleanup items currently in flight */
+	DLLIST_NODE sMMUCtxCleanupItemsHead;
+	/*! Was the MMU context destroyed and should not be accessed any more? */
+	IMG_BOOL bMMUContextExists;
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/*! Associated OSid for this context */
+	IMG_UINT32 ui32OSid;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+} MMU_CTX_CLEANUP_DATA;
+
+
+/*!
+ * Structure holding one or more page tables that need to be
+ * freed after the MMU cache has been flushed which is signalled when
+ * the stored sync has a value that is <= the required value.
+ */
+typedef struct _MMU_CLEANUP_ITEM_
+{
+	/*! Cleanup thread data */
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+	/*! List to hold all the MMU_MEMORY_MAPPINGs, i.e. page tables */
+	DLLIST_NODE sMMUMappingHead;
+	/*! Node of the cleanup item list for the context */
+	DLLIST_NODE sMMUCtxCleanupItem;
+	/* Pointer to the cleanup meta data */
+	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData;
+	/* Sync to query if the MMU cache was flushed */
+	PVRSRV_CLIENT_SYNC_PRIM *psSync;
+	/*! The update value of the sync to signal that the cache was flushed */
+	IMG_UINT32 uiRequiredSyncVal;
+	/*! The update value of the power off counter */
+	IMG_UINT32 uiRequiredPowerOffCounter;
+	/*! The device node needed to free the page tables */
+	PVRSRV_DEVICE_NODE *psDevNode;
+} MMU_CLEANUP_ITEM;
+
+/*!
+	All physical allocations and frees are relative to this context, so
+	we would get all the allocations of PCs, PDs, and PTs from the same
+	RA.
+
+	We have one per MMU context in case we have mixed UMA/LMA devices
+	within the same system.
+ */
+typedef struct _MMU_PHYSMEM_CONTEXT_
+{
+	/*! Associated MMU_CONTEXT */
+	struct _MMU_CONTEXT_ *psMMUContext;
+
+	/*! Parent device node */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/*! Refcount so we know when to free up the arena */
+	IMG_UINT32 uiNumAllocations;
+
+	/*! Arena from which physical memory is derived */
+	RA_ARENA *psPhysMemRA;
+	/*! Arena name */
+	IMG_CHAR *pszPhysMemRAName;
+	/*! Size of arena name string */
+	size_t uiPhysMemRANameAllocSize;
+
+	/*! Meta data for deferred cleanup */
+	MMU_CTX_CLEANUP_DATA *psCleanupData;
+	/*! Temporary list of all deferred MMU_MEMORY_MAPPINGs. */
+	DLLIST_NODE sTmpMMUMappingHead;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	IMG_UINT32 ui32OSid;
+	IMG_UINT32 ui32OSidReg;
+	IMG_BOOL   bOSidAxiProt;
+#endif
+
+} MMU_PHYSMEM_CONTEXT;
+
+/*!
+	Mapping structure for MMU memory allocation
+ */
+typedef struct _MMU_MEMORY_MAPPING_
+{
+	/*! Physmem context to allocate from */
+	MMU_PHYSMEM_CONTEXT		*psContext;
+	/*! OS/system Handle for this allocation */
+	PG_HANDLE				sMemHandle;
+	/*! CPU virtual address of this allocation */
+	void					*pvCpuVAddr;
+	/*! Device physical address of this allocation */
+	IMG_DEV_PHYADDR			sDevPAddr;
+	/*! Size of this allocation */
+	size_t					uiSize;
+	/*! Number of current mappings of this allocation */
+	IMG_UINT32				uiCpuVAddrRefCount;
+	/*! Node for the defer free list */
+	DLLIST_NODE				sMMUMappingItem;
+} MMU_MEMORY_MAPPING;
+
+/*!
+	Memory descriptor for MMU objects. There can be more than one memory
+	descriptor per MMU memory allocation.
+ */
+typedef struct _MMU_MEMORY_DESC_
+{
+	/* NB: bValid is set if this descriptor describes physical
+	   memory.  This allows "empty" descriptors to exist, such that we
+	   can allocate them in batches. */
+	/*! Does this MMU object have physical backing */
+	IMG_BOOL				bValid;
+	/*! Device Physical address of physical backing */
+	IMG_DEV_PHYADDR			sDevPAddr;
+	/*! CPU virtual address of physical backing */
+	void					*pvCpuVAddr;
+	/*! Mapping data for this MMU object */
+	MMU_MEMORY_MAPPING		*psMapping;
+	/*! Memdesc offset into the psMapping */
+	IMG_UINT32 uiOffset;
+	/*! Size of the Memdesc */
+	IMG_UINT32 uiSize;
+} MMU_MEMORY_DESC;
+
+/*!
+	MMU levelx structure. This is generic and is used
+	for all levels (PC, PD, PT).
+ */
+typedef struct _MMU_Levelx_INFO_
+{
+	/*! The Number of entries in this level */
+	IMG_UINT32 ui32NumOfEntries;
+
+	/*! Number of times this level has been reference. Note: For Level1 (PTE)
+	    we still take/drop the reference when setting up the page tables rather
+	    then at map/unmap time as this simplifies things */
+	IMG_UINT32 ui32RefCount;
+
+	/*! MemDesc for this level */
+	MMU_MEMORY_DESC sMemDesc;
+
+	/*! Array of infos for the next level. Must be last member in structure */
+	struct _MMU_Levelx_INFO_ *apsNextLevel[IMG_FLEX_ARRAY_MEMBER];
+} MMU_Levelx_INFO;
+
+/*!
+	MMU context structure
+ */
+struct _MMU_CONTEXT_
+{
+	/*! Originating Connection */
+	CONNECTION_DATA *psConnection;
+
+	/*! Device MMU attribute descriptions */
+	MMU_DEVICEATTRIBS *psDevAttrs;
+
+	/*! For allocation and deallocation of the physical memory where
+	    the pagetables live */
+	struct _MMU_PHYSMEM_CONTEXT_ *psPhysMemCtx;
+
+#if defined(PDUMP)
+	/*! PDump context ID (required for PDump commands with virtual addresses) */
+	IMG_UINT32 uiPDumpContextID;
+
+	/*! The refcount of the PDump context ID */
+	IMG_UINT32 ui32PDumpContextIDRefCount;
+#endif
+
+	/*! MMU cache invalidation flags. For Rogue, only one set of flags are
+	 * maintained (in the KernelMMUCtx) and used for all context page table
+	 * updates, while in Volcanic each context maintains its own flags
+	 * for the PTs it contains. This is due to the different memory hierarchy
+	 * designs in use between the architectures, See SLC_VIVT feature. */
+	ATOMIC_T sCacheFlags;
+
+	/*! Lock to ensure exclusive access when manipulating the MMU context or
+	 * reading and using its content
+	 */
+	POS_LOCK hLock;
+
+	/*! Base level info structure. Must be last member in structure */
+	MMU_Levelx_INFO sBaseLevelInfo;
+	/* NO OTHER MEMBERS AFTER THIS STRUCTURE ! */
+};
+
+/*
+ * Only the kernel created (direct bridge) firmware memory context will
+ * have a NULL connection and all other application memory context get a
+ * valid connection object.
+ */
+#define _MMU_IS_FWKM_CTX(_ctx) ((_ctx)->psConnection == NULL)
+
+/*
+ * Used to determine if the MMU Ctx provided is the FWKM memory context
+ * and if it belongs to the VZ Guest.
+ */
+#define _MMU_IS_FWKM_CTX_VZGUEST(_ctx) (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, _ctx->psPhysMemCtx->psDevNode) && _MMU_IS_FWKM_CTX(_ctx))
+
+static const IMG_DEV_PHYADDR gsBadDevPhyAddr = {MMU_BAD_PHYS_ADDR};
+
+#if defined(DEBUG)
+#include "log2.h"
+#endif
+
+
+static PVRSRV_ERROR
+MMU_UnmapPagesUnlocked(MMU_CONTEXT *psMMUContext,
+                       PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                       IMG_DEV_VIRTADDR sDevVAddrBase,
+                       IMG_UINT32 ui32PageCount,
+                       IMG_UINT32 *pai32FreeIndices,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags);
+
+static PVRSRV_ERROR
+MMU_UnmapPMRFastUnlocked(MMU_CONTEXT *psMMUContext,
+                         IMG_DEV_VIRTADDR sDevVAddrBase,
+                         IMG_UINT32 ui32PageCount,
+                         IMG_UINT32 uiLog2PageSize);
+
+/*****************************************************************************
+ *                          Utility functions                                *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _FreeMMUMapping
+
+@Description    Free a given dllist of MMU_MEMORY_MAPPINGs and the page tables
+                they represent.
+
+@Input          psDevNode           Device node
+
+@Input          psTmpMMUMappingHead List of MMU_MEMORY_MAPPINGs to free
+ */
+/*****************************************************************************/
+static void
+_FreeMMUMapping(PVRSRV_DEVICE_NODE *psDevNode,
+                PDLLIST_NODE psTmpMMUMappingHead)
+{
+	PDLLIST_NODE psNode, psNextNode;
+
+	/* Free the current list unconditionally */
+	dllist_foreach_node(psTmpMMUMappingHead,
+	                    psNode,
+	                    psNextNode)
+	{
+		MMU_MEMORY_MAPPING *psMapping = IMG_CONTAINER_OF(psNode,
+		                                                 MMU_MEMORY_MAPPING,
+		                                                 sMMUMappingItem);
+
+		PhysHeapPagesFree(psDevNode->psMMUPhysHeap, &psMapping->sMemHandle);
+		dllist_remove_node(psNode);
+		OSFreeMem(psMapping);
+	}
+}
+
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+/*************************************************************************/ /*!
+@Function       _CleanupThread_FreeMMUMapping
+
+@Description    Function to be executed by the cleanup thread to free
+                MMU_MEMORY_MAPPINGs after the MMU cache has been invalidated.
+
+                This function will request a MMU cache invalidate once and
+                retry to free the MMU_MEMORY_MAPPINGs until the invalidate
+                has been executed.
+
+                If the memory context that created this cleanup item has been
+                destroyed in the meantime this function will directly free the
+                MMU_MEMORY_MAPPINGs without waiting for any MMU cache
+                invalidation.
+
+@Input          pvData           Cleanup data in form of a MMU_CLEANUP_ITEM
+
+@Return         PVRSRV_OK if successful otherwise PVRSRV_ERROR_RETRY
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR
+_CleanupThread_FreeMMUMapping(void* pvData)
+{
+	PVRSRV_ERROR eError;
+	MMU_CLEANUP_ITEM *psCleanup = (MMU_CLEANUP_ITEM *)pvData;
+	MMU_CTX_CLEANUP_DATA *psMMUCtxCleanupData = psCleanup->psMMUCtxCleanupData;
+	PVRSRV_DEVICE_NODE *psDevNode = psCleanup->psDevNode;
+	IMG_BOOL bFreeNow;
+
+	OSLockAcquire(psMMUCtxCleanupData->hCleanupLock);
+
+	/* Don't attempt to free anything when the context has been destroyed.
+	 * Especially don't access any device specific structures any more!*/
+	if (!psMMUCtxCleanupData->bMMUContextExists)
+	{
+		OSFreeMem(psCleanup);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_OK, e0);
+	}
+
+	if (psCleanup->psSync == NULL)
+	{
+		/* Kick to invalidate the MMU caches and get sync info */
+		eError = psDevNode->pfnMMUCacheInvalidateKick(psDevNode,
+		                                     &psCleanup->uiRequiredSyncVal);
+		if (eError != PVRSRV_OK)
+		{
+			OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+			return PVRSRV_ERROR_RETRY;
+		}
+		psCleanup->psSync = psDevNode->psMMUCacheSyncPrim;
+		/* If we have no sync reference we treat this as a simple FREE.
+		 * We cannot retry as there will never be a sync-prim to kick / invalidate.
+		 */
+		if (psCleanup->psSync == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device %u has no MMUCacheSyncPrim",
+			         __func__, psDevNode->sDevId.ui32InternalID));
+			bFreeNow = IMG_TRUE;
+			goto freeNow;
+		}
+	}
+
+	/* Has the invalidate executed (sync is updated when the Firmware performs
+	 * the invalidate)? */
+	bFreeNow = PVRSRVHasCounter32Advanced(OSReadDeviceMem32(psCleanup->psSync->pui32LinAddr),
+	                                      psCleanup->uiRequiredSyncVal);
+
+	/* Has there been a power off? */
+	bFreeNow = bFreeNow || PVRSRVHasCounter32Advanced(psDevNode->uiPowerOffCounter,
+	                                                  psCleanup->uiRequiredPowerOffCounter);
+
+#if defined(NO_HARDWARE)
+	/* In NOHW the syncs will never be updated so just free the tables */
+	bFreeNow = IMG_TRUE;
+#endif
+	/* If the Invalidate operation is not completed, check if the operation timed out */
+	if (!bFreeNow)
+	{
+		IMG_UINT32 uiTimeStart = psCleanup->sCleanupThreadFn.ui32TimeStart;
+		IMG_UINT32 uiTimeEnd = psCleanup->sCleanupThreadFn.ui32TimeEnd;
+
+		/* If the time left for the completion of invalidate operation is
+		 * within 500ms of time-out, consider the operation as timed out */
+		if ((uiTimeEnd - uiTimeStart - 500) <= (OSClockms() - uiTimeStart))
+		{
+			/* Consider the operation is timed out */
+			bFreeNow = IMG_TRUE;
+		}
+	}
+
+freeNow:
+	/* Free if the invalidate operation completed or the operation itself timed out */
+	if (bFreeNow)
+	{
+		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);
+
+		dllist_remove_node(&psCleanup->sMMUCtxCleanupItem);
+
+		OSFreeMem(psCleanup);
+
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_RETRY;
+	}
+
+e0:
+
+	/* If this cleanup task has been successfully executed we can
+	 * decrease the context cleanup data refcount. Successfully
+	 * means here that the MMU_MEMORY_MAPPINGs have been freed by
+	 * either this cleanup task of when the MMU context has been
+	 * destroyed. */
+	if (eError == PVRSRV_OK)
+	{
+		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+
+		if (OSAtomicDecrement(&psMMUCtxCleanupData->iRef) == 0)
+		{
+			OSLockDestroy(psMMUCtxCleanupData->hCleanupLock);
+			OSFreeMem(psMMUCtxCleanupData);
+		}
+	}
+	else
+	{
+		OSLockRelease(psMMUCtxCleanupData->hCleanupLock);
+	}
+
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _SetupCleanup_FreeMMUMapping
+
+@Description    Setup a cleanup item for the cleanup thread that will
+                kick off a MMU invalidate request and free the associated
+                MMU_MEMORY_MAPPINGs when the invalidate was successful.
+
+@Input          psPhysMemCtx        The current MMU physmem context
+ */
+/*****************************************************************************/
+static void
+_SetupCleanup_FreeMMUMapping(MMU_PHYSMEM_CONTEXT *psPhysMemCtx)
+{
+
+	MMU_CLEANUP_ITEM *psCleanupItem;
+	MMU_CTX_CLEANUP_DATA *psCleanupData = psPhysMemCtx->psCleanupData;
+	PVRSRV_DEVICE_NODE *psDevNode = psPhysMemCtx->psDevNode;
+
+	if (dllist_is_empty(&psPhysMemCtx->sTmpMMUMappingHead))
+	{
+		goto e0;
+	}
+
+#if defined(PDUMP)
+	/* Free the page tables immediately in case of pdump, which avoids
+	 * changing script files due to the additional invalidation kick */
+	goto e1;
+#endif
+
+	/* Don't defer the freeing if we are currently unloading the driver
+	 * or if the sync has been destroyed */
+	if (PVRSRVGetPVRSRVData()->bUnload ||
+			psDevNode->psMMUCacheSyncPrim == NULL)
+	{
+		goto e1;
+	}
+
+	/* Allocate a cleanup item */
+	psCleanupItem = OSAllocMem(sizeof(*psCleanupItem));
+	if (!psCleanupItem)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to get memory for deferred page table cleanup. "
+				"Freeing tables immediately",
+				__func__));
+		goto e1;
+	}
+
+	/* Set sync to NULL to indicate we did not interact with
+	 * the FW yet. Kicking off an MMU cache invalidate should
+	 * be done in the cleanup thread to not waste time here. */
+	psCleanupItem->psSync = NULL;
+	psCleanupItem->uiRequiredSyncVal = 0;
+	psCleanupItem->uiRequiredPowerOffCounter = psDevNode->uiPowerOffCounterNext;
+	psCleanupItem->psDevNode = psDevNode;
+	psCleanupItem->psMMUCtxCleanupData = psCleanupData;
+
+	OSAtomicIncrement(&psCleanupData->iRef);
+
+	/* Move the page tables to free to the cleanup item */
+	dllist_replace_head(&psPhysMemCtx->sTmpMMUMappingHead,
+	                    &psCleanupItem->sMMUMappingHead);
+
+	/* Add the cleanup item itself to the context list */
+	dllist_add_to_tail(&psCleanupData->sMMUCtxCleanupItemsHead,
+	                   &psCleanupItem->sMMUCtxCleanupItem);
+
+	/* Setup the cleanup thread data and add the work item */
+	psCleanupItem->sCleanupThreadFn.pfnFree = _CleanupThread_FreeMMUMapping;
+	psCleanupItem->sCleanupThreadFn.pvData = psCleanupItem;
+	psCleanupItem->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+	psCleanupItem->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_MMU;
+	CLEANUP_THREAD_SET_RETRY_TIMEOUT(&psCleanupItem->sCleanupThreadFn,
+	                                 CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT);
+
+	PVRSRVCleanupThreadAddWork(psDevNode, &psCleanupItem->sCleanupThreadFn);
+
+	return;
+
+e1:
+	/* Free the page tables now */
+	_FreeMMUMapping(psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+e0:
+	return;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _CalcPCEIdx
+
+@Description    Calculate the page catalogue index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page catalogue index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPCEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPCIndexMask)
+			>> psDevVAddrConfig->uiPCIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _CalcPDEIdx
+
+@Description    Calculate the page directory index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page directory index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPDEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPDIndexMask)
+			>> psDevVAddrConfig->uiPDIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _CalcPTEIdx
+
+@Description    Calculate the page entry index
+
+@Input          sDevVAddr           Device virtual address
+
+@Input          psDevVAddrConfig    Configuration of the virtual address
+
+@Input          bRoundUp            Round up the index
+
+@Return         The page entry index
+ */
+/*****************************************************************************/
+static IMG_UINT32 _CalcPTEIdx(IMG_DEV_VIRTADDR sDevVAddr,
+                              const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                              IMG_BOOL bRoundUp)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	IMG_UINT32 ui32RetVal;
+
+	sTmpDevVAddr = sDevVAddr;
+	sTmpDevVAddr.uiAddr -= psDevVAddrConfig->uiOffsetInBytes;
+	if (bRoundUp)
+	{
+		sTmpDevVAddr.uiAddr--;
+	}
+	ui32RetVal = (IMG_UINT32) ((sTmpDevVAddr.uiAddr & psDevVAddrConfig->uiPTIndexMask)
+			>> psDevVAddrConfig->uiPTIndexShift);
+
+	if (bRoundUp)
+	{
+		ui32RetVal++;
+	}
+
+	return ui32RetVal;
+}
+
+/*************************************************************************/ /*!
+@Function       _GetParityBit
+
+@Description    Calculate parity bit
+
+@Input          uiSrc           Value to calculate the parity bit from.
+
+@Return         The parity bit
+ */
+/*****************************************************************************/
+static inline IMG_UINT64 _GetParityBit(IMG_UINT64 uiSrc)
+{
+	uiSrc ^= uiSrc >> 32U;
+	uiSrc ^= uiSrc >> 16U;
+	uiSrc ^= uiSrc >> 8U;
+	uiSrc ^= uiSrc >> 4U;
+	uiSrc ^= uiSrc >> 2U;
+	uiSrc ^= uiSrc >> 1U;
+
+	return (uiSrc & 1U);
+}
+
+/*****************************************************************************
+ *         MMU memory allocation/management functions (mem desc)             *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMem_RAImportAlloc
+
+@Description    Imports MMU Px memory into the RA. This is where the
+                actual allocation of physical memory happens.
+
+@Input          hArenaHandle    Handle that was passed in during the
+                                creation of the RA
+
+@Input          uiSize          Size of the memory to import
+
+@Input          uiFlags         Flags that where passed in the allocation.
+
+@Input          uBaseAlignment  Alignment for the base returned, not used
+                                in this context.
+
+@Output         puiBase         The address of where to insert this import
+
+@Output         puiActualSize   The actual size of the import
+
+@Output         phPriv          Handle which will be passed back when
+                                this import is freed
+
+@Return         PVRSRV_OK if import alloc was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _MMU_PhysMem_RAImportAlloc(RA_PERARENA_HANDLE hArenaHandle,
+                                               RA_LENGTH_T uiSize,
+                                               RA_FLAGS_T uiFlags,
+                                               RA_LENGTH_T uBaseAlignment,
+                                               const IMG_CHAR *pszAnnotation,
+                                               RA_IMPORT *psImport)
+{
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx = (MMU_PHYSMEM_CONTEXT *)hArenaHandle;
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psPhysMemCtx->psDevNode;
+	MMU_MEMORY_MAPPING *psMapping;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPid = 0;
+
+	PVR_UNREFERENCED_PARAMETER(pszAnnotation);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	PVR_ASSERT(psDevNode != NULL);
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode, eError, e0);
+
+	psMapping = OSAllocMem(sizeof(MMU_MEMORY_MAPPING));
+	PVR_GOTO_IF_NOMEM(psMapping, eError, e0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	uiPid = psDevNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ?
+	        PVR_SYS_ALLOC_PID : OSGetCurrentClientProcessIDKM();
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/*
+	 * Store the OSid in the PG_HANDLE.uiOSid field for use by the
+	 * pfnDevPxFree() routine.
+	 */
+	psMapping->sMemHandle.uiOSid = psPhysMemCtx->ui32OSid;
+	eError = PhysHeapPagesAllocGPV(psDevNode->psMMUPhysHeap,
+	                               TRUNCATE_64BITS_TO_SIZE_T(uiSize),
+	                               &psMapping->sMemHandle,
+	                               &psMapping->sDevPAddr,
+	                               psPhysMemCtx->ui32OSid,
+	                               uiPid);
+#else
+	eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+	                            TRUNCATE_64BITS_TO_SIZE_T(uiSize),
+	                            &psMapping->sMemHandle,
+	                            &psMapping->sDevPAddr,
+	                            uiPid);
+#endif
+	if (eError != PVRSRV_OK)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		PVRSRVStatsUpdateOOMStat(NULL, psDevNode, PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+					  OSGetCurrentClientProcessIDKM());
+#endif
+		goto e1;
+	}
+
+	psMapping->psContext = psPhysMemCtx;
+	psMapping->uiSize = TRUNCATE_64BITS_TO_SIZE_T(uiSize);
+
+	psMapping->uiCpuVAddrRefCount = 0;
+
+	psImport->hPriv = (RA_PERISPAN_HANDLE) psMapping;
+
+	/* Note: This assumes this memory never gets paged out */
+	psImport->base = (RA_BASE_T)psMapping->sDevPAddr.uiAddr;
+	psImport->uSize = uiSize;
+
+	return PVRSRV_OK;
+
+e1:
+	OSFreeMem(psMapping);
+e0:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMem_RAImportFree
+
+@Description    Imports MMU Px memory into the RA. This is where the
+                actual free of physical memory happens.
+
+@Input          hArenaHandle    Handle that was passed in during the
+                                creation of the RA
+
+@Input          puiBase         The address of where to insert this import
+
+@Output         phPriv          Private data that the import alloc provided
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _MMU_PhysMem_RAImportFree(RA_PERARENA_HANDLE hArenaHandle,
+                                      RA_BASE_T uiBase,
+                                      RA_PERISPAN_HANDLE hPriv)
+{
+	MMU_MEMORY_MAPPING *psMapping = (MMU_MEMORY_MAPPING *)hPriv;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx = (MMU_PHYSMEM_CONTEXT *)hArenaHandle;
+
+	PVR_UNREFERENCED_PARAMETER(uiBase);
+
+	/* Check we have dropped all CPU mappings */
+	PVR_ASSERT(psMapping->uiCpuVAddrRefCount == 0);
+
+	/* Add mapping to defer free list */
+	psMapping->psContext = NULL;
+	dllist_add_to_tail(&psPhysMemCtx->sTmpMMUMappingHead, &psMapping->sMMUMappingItem);
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMemAlloc
+
+@Description    Allocates physical memory for MMU objects
+
+@Input          psPhysMemCtx    Physmem context to do the allocation from
+
+@Output         psMemDesc       Allocation description
+
+@Input          uiBytes         Size of the allocation in bytes
+
+@Input          uiAlignment     Alignment requirement of this allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+
+static PVRSRV_ERROR _MMU_PhysMemAlloc(MMU_PHYSMEM_CONTEXT *psPhysMemCtx,
+                                      MMU_MEMORY_DESC *psMemDesc,
+                                      size_t uiBytes,
+                                      size_t uiAlignment)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiPhysAddr;
+
+	PVR_RETURN_IF_INVALID_PARAM(psMemDesc);
+	PVR_RETURN_IF_INVALID_PARAM(!psMemDesc->bValid);
+
+	eError = RA_Alloc(psPhysMemCtx->psPhysMemRA,
+	                  uiBytes,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0, /* flags */
+	                  uiAlignment,
+	                  "",
+	                  &uiPhysAddr,
+	                  NULL,
+	                  (RA_PERISPAN_HANDLE *)&psMemDesc->psMapping);
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "RA_Alloc");
+
+	psMemDesc->bValid = IMG_TRUE;
+	psMemDesc->pvCpuVAddr = NULL;
+	psMemDesc->sDevPAddr.uiAddr = (IMG_UINT64) uiPhysAddr;
+
+	if (psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
+	{
+		eError = PhysHeapPagesMap(psPhysMemCtx->psDevNode->psMMUPhysHeap,
+		                          &psMemDesc->psMapping->sMemHandle,
+		                          psMemDesc->psMapping->uiSize,
+		                          &psMemDesc->psMapping->sDevPAddr,
+		                          &psMemDesc->psMapping->pvCpuVAddr);
+		if (eError != PVRSRV_OK)
+		{
+			RA_Free(psPhysMemCtx->psPhysMemRA, psMemDesc->sDevPAddr.uiAddr);
+			return eError;
+		}
+	}
+
+	psMemDesc->psMapping->uiCpuVAddrRefCount++;
+	psMemDesc->uiOffset = (psMemDesc->sDevPAddr.uiAddr - psMemDesc->psMapping->sDevPAddr.uiAddr);
+	psMemDesc->pvCpuVAddr = (IMG_UINT8 *)psMemDesc->psMapping->pvCpuVAddr + psMemDesc->uiOffset;
+	psMemDesc->uiSize = uiBytes;
+	PVR_ASSERT(psMemDesc->pvCpuVAddr != NULL);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PhysMemFree
+
+@Description    Allocates physical memory for MMU objects
+
+@Input          psPhysMemCtx    Physmem context to do the free on
+
+@Input          psMemDesc       Allocation description
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _MMU_PhysMemFree(MMU_PHYSMEM_CONTEXT *psPhysMemCtx,
+                             MMU_MEMORY_DESC *psMemDesc)
+{
+	RA_BASE_T uiPhysAddr;
+
+	PVR_ASSERT(psMemDesc->bValid);
+
+	if (--psMemDesc->psMapping->uiCpuVAddrRefCount == 0)
+	{
+		PhysHeapPagesUnMap(psPhysMemCtx->psDevNode->psMMUPhysHeap,
+		                   &psMemDesc->psMapping->sMemHandle,
+		                   psMemDesc->psMapping->pvCpuVAddr);
+	}
+
+	psMemDesc->pvCpuVAddr = NULL;
+
+	uiPhysAddr = psMemDesc->sDevPAddr.uiAddr;
+	RA_Free(psPhysMemCtx->psPhysMemRA, uiPhysAddr);
+
+	psMemDesc->bValid = IMG_FALSE;
+}
+
+
+/*****************************************************************************
+ *              MMU object allocation/management functions                   *
+ *****************************************************************************/
+
+static INLINE PVRSRV_ERROR _MMU_ConvertDevMemFlags(IMG_BOOL bInvalidate,
+                                                   PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                                                   MMU_PROTFLAGS_T *puiMMUProtFlags,
+                                                   MMU_CONTEXT *psMMUContext)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiGPUCacheMode;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Do flag conversion between devmem flags and MMU generic flags */
+	if (bInvalidate == IMG_TRUE)
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_INVALID;
+		return eError; /* OK */
+	}
+
+	/* Convert to valid flags for valid mapping */
+	*puiMMUProtFlags |= ((uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK)
+			>> PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET)
+			<< MMU_PROTFLAGS_DEVICE_OFFSET;
+
+	if (PVRSRV_CHECK_GPU_READABLE(uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+	}
+	if (PVRSRV_CHECK_GPU_WRITEABLE(uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_WRITEABLE;
+	}
+
+	eError = DevmemDeviceCacheMode(uiMappingFlags, &uiGPUCacheMode);
+	PVR_RETURN_IF_ERROR(eError);
+
+	switch (uiGPUCacheMode)
+	{
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED:
+		case PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC:
+			break;
+		case PVRSRV_MEMALLOCFLAG_GPU_CACHED:
+			*puiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Wrong parameters",
+					__func__));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (DevmemDeviceCacheCoherency(psDevNode, uiMappingFlags))
+	{
+		*puiMMUProtFlags |= MMU_PROTFLAGS_CACHE_COHERENT;
+	}
+
+	if (psDevNode->pfnMMUTweakProtFlags)
+	{
+		psDevNode->pfnMMUTweakProtFlags(psDevNode, psMMUContext->psDevAttrs, uiMappingFlags, puiMMUProtFlags);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _PxMemAlloc
+
+@Description    Allocates physical memory for MMU objects, initialises
+                and PDumps it.
+
+@Input          psMMUContext    MMU context
+
+@Input          uiNumEntries    Number of entries to allocate
+
+@Input          psConfig        MMU Px config
+
+@Input          eMMULevel       MMU level that allocation is for
+
+@Output         psMemDesc       Description of allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _PxMemAlloc(MMU_CONTEXT *psMMUContext,
+                                IMG_UINT32 uiNumEntries,
+                                const MMU_PxE_CONFIG *psConfig,
+                                MMU_LEVEL eMMULevel,
+                                MMU_MEMORY_DESC *psMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+                                IMG_DEV_VIRTADDR* psRunningDevVAddr,
+                                IMG_UINT32 uiLog2DataPageSize,
+#endif
+                                IMG_UINT32 uiLog2Align)
+{
+	PVRSRV_ERROR eError;
+	size_t uiBytes;
+	size_t uiAlign;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	PVR_ASSERT(psConfig->uiBytesPerEntry != 0);
+
+	uiBytes = uiNumEntries * psConfig->uiBytesPerEntry;
+	/* We need here the alignment of the previous level because that is the entry for we generate here */
+	uiAlign = 1 << uiLog2Align;
+
+	/*
+	 * If the hardware specifies an alignment requirement for a page table then
+	 * it also requires that all memory up to the next aligned address is
+	 * zeroed.
+	 *
+	 * Failing to do this can result in uninitialised data outside of the actual
+	 * page table range being read by the MMU and treated as valid, e.g. the
+	 * pending flag.
+	 *
+	 * Typically this will affect 1MiB, 2MiB PT pages which have a size of 16
+	 * and 8 bytes respectively but an alignment requirement of 64 bytes each.
+	 */
+	uiBytes = PVR_ALIGN(uiBytes, uiAlign);
+
+	/* allocate the object */
+	eError = _MMU_PhysMemAlloc(psMMUContext->psPhysMemCtx,
+	                           psMemDesc, uiBytes, uiAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("_MMU_PhysMemAlloc", eError, PVRSRV_ERROR_OUT_OF_MEMORY, e0);
+	}
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	/* If parity bit is needed, set it accordingly to avoid parity errors along with page-faults. */
+	if (psConfig->uiParityBitMask && eMMULevel == MMU_LEVEL_1 && psRunningDevVAddr != NULL)
+	{
+		IMG_UINT32 uiParity = (IMG_UINT32)_GetParityBit(psRunningDevVAddr->uiAddr ^ 0ULL);
+
+		/* Only calculate parity for first address in the PT.
+		   This tells us which of the two patterns to use for the rest of the PT. */
+		OSCachedMemCopy(psMemDesc->pvCpuVAddr, psMMUContext->psDevAttrs->pui64PrecomputedAllocParity[uiParity], uiBytes);
+
+		/* Increment running dev virtual address */
+		psRunningDevVAddr->uiAddr += (1 << uiLog2DataPageSize) * uiNumEntries;
+	}
+	else
+#endif
+	{
+		/*
+			Clear the object
+			Note: if any MMUs are cleared with non-zero values then will need a
+			custom clear function
+			Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
+			unlikely
+		*/
+		OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, uiBytes);
+	}
+
+	eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+	                            &psMemDesc->psMapping->sMemHandle,
+	                            psMemDesc->uiOffset,
+	                            psMemDesc->uiSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Alloc MMU object");
+
+	PDumpMMUMalloc(psDevNode,
+	               psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	               eMMULevel,
+	               &psMemDesc->sDevPAddr,
+	               uiBytes,
+	               uiAlign,
+	               psMMUContext->psDevAttrs->eMMUType);
+
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      eMMULevel,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      0,
+	                      uiNumEntries,
+	                      NULL, NULL, 0, /* pdump symbolic info is irrelevant here */
+	                      psConfig->uiBytesPerEntry,
+	                      uiLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      0, /* Unused - Parity bit values are taken directly from PTE memory when PTE is 0-initialised or invalid */
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      psMMUContext->psDevAttrs->eMMUType);
+#else
+	PVR_UNREFERENCED_PARAMETER(eMMULevel);
+#endif
+
+	return PVRSRV_OK;
+e1:
+	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx,
+	                 psMemDesc);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _PxMemFree
+
+@Description    Frees physical memory for MMU objects, de-initialises
+                and PDumps it.
+
+@Input          psMemDesc       Description of allocation
+
+@Return         PVRSRV_OK if allocation was successful
+ */
+/*****************************************************************************/
+
+static void _PxMemFree(MMU_CONTEXT *psMMUContext,
+                       MMU_MEMORY_DESC *psMemDesc, MMU_LEVEL eMMULevel)
+{
+#if defined(MMU_CLEARMEM_ON_FREE)
+	/*
+		Clear the MMU object
+		Note: if any MMUs are cleared with non-zero values then will need a
+		custom clear function
+		Note: 'Cached' is wrong for the LMA + ARM64 combination, but this is
+		unlikely
+	 */
+	OSCachedMemSet(psMemDesc->pvCpuVAddr, 0, psMemDesc->ui32Bytes);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Clear MMU object before freeing it");
+#endif
+#endif/* MMU_CLEARMEM_ON_FREE */
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Free MMU object");
+	PDumpMMUFree(psMMUContext->psPhysMemCtx->psDevNode,
+	             psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	             eMMULevel,
+	             &psMemDesc->sDevPAddr,
+	             psMMUContext->psDevAttrs->eMMUType);
+#else
+	PVR_UNREFERENCED_PARAMETER(eMMULevel);
+#endif
+	/* free the PC */
+	_MMU_PhysMemFree(psMMUContext->psPhysMemCtx, psMemDesc);
+}
+
+static INLINE PVRSRV_ERROR _SetupPTE(MMU_CONTEXT *psMMUContext,
+                                     MMU_Levelx_INFO *psLevel,
+                                     IMG_UINT32 uiIndex,
+                                     const MMU_PxE_CONFIG *psConfig,
+                                     const IMG_DEV_PHYADDR *psDevPAddr,
+                                     const IMG_DEV_VIRTADDR *psDevVAddr,
+                                     IMG_BOOL bUnmap,
+#if defined(PDUMP)
+                                     const IMG_CHAR *pszMemspaceName,
+                                     const IMG_CHAR *pszSymbolicAddr,
+                                     IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
+#endif
+                                     IMG_UINT64 uiProtFlags)
+{
+	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;
+	IMG_UINT64 ui64PxE64;
+	IMG_UINT64 uiAddr = psDevPAddr->uiAddr;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_BOOL bParity = psConfig->uiParityBitMask;
+#else
+	IMG_BOOL bParity = psConfig->uiParityBitMask && !bUnmap;
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(bParity);
+#endif
+#endif
+
+	if (psDevNode->pfnValidateOrTweakPhysAddrs)
+	{
+		PVRSRV_ERROR eErr = psDevNode->pfnValidateOrTweakPhysAddrs(psDevNode,
+		                                                           psMMUContext->psDevAttrs,
+		                                                           &uiAddr);
+		/* return if error */
+		PVR_LOG_RETURN_IF_ERROR(eErr, "_SetupPTE");
+	}
+
+	/* Calculate Entry */
+	ui64PxE64 =    uiAddr /* Calculate the offset to that base */
+			>> psConfig->uiAddrLog2Align /* Shift away the useless bits, because the alignment is very coarse and we address by alignment */
+			<< psConfig->uiAddrShift /* Shift back to fit address in the Px entry */
+			& psConfig->uiAddrMask; /* Delete unused bits */
+
+	ui64PxE64 |= uiProtFlags;
+
+	/* Add parity */
+	if (bParity)
+	{
+		ui64PxE64 |= _GetParityBit(psDevVAddr->uiAddr ^ psDevPAddr->uiAddr) << psConfig->uiParityBitShift;
+	}
+
+	/* Set the entry */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+		pui64Px[uiIndex] = ui64PxE64;
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+		/* assert that the result fits into 32 bits before writing
+		   it into the 32-bit array with a cast */
+		PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+		PVR_ASSERT(!bParity);
+
+		pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
+	}
+	else
+	{
+		return PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
+	}
+
+	/* Log modification */
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+	        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+	        uiIndex, MMU_LEVEL_1,
+	        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+	        !bUnmap);
+
+	PVR_UNREFERENCED_PARAMETER(bUnmap); /* when HTBLOGK is disabled */
+
+#if defined(PDUMP)
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      MMU_LEVEL_1,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      uiIndex,
+	                      1,
+	                      pszMemspaceName,
+	                      pszSymbolicAddr,
+	                      uiSymbolicAddrOffset,
+	                      psConfig->uiBytesPerEntry,
+	                      psConfig->uiAddrLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      bParity ? _GetParityBit(psDevVAddr->uiAddr ^ 0) : 0,
+	                      psConfig->uiParityBitShift,
+	                      bParity ? psConfig->uiParityBitMask : 0,
+	                      psMMUContext->psDevAttrs->eMMUType);
+#endif /*PDUMP*/
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _SetupPxE
+
+@Description    Sets up an entry of an MMU object to point to the
+                provided address
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel         Level info for MMU object
+
+@Input          uiIndex         Index into the MMU object to setup
+
+@Input          psConfig        MMU Px config
+
+@Input          eMMULevel       Level of MMU object
+
+@Input          psDevPAddr      Address to setup the MMU object to point to
+
+@Input          pszMemspaceName Name of the PDump memory space that the entry
+                                will point to
+
+@Input          pszSymbolicAddr PDump symbolic address that the entry will
+                                point to
+
+@Input          uiProtFlags     MMU protection flags
+
+@Return         PVRSRV_OK if the setup was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _SetupPxE(MMU_CONTEXT *psMMUContext,
+                              MMU_Levelx_INFO *psLevel,
+                              IMG_UINT32 uiIndex,
+                              const MMU_PxE_CONFIG *psConfig,
+                              MMU_LEVEL eMMULevel,
+                              const IMG_DEV_PHYADDR *psDevPAddr,
+#if defined(PDUMP)
+                              const IMG_CHAR *pszMemspaceName,
+                              const IMG_CHAR *pszSymbolicAddr,
+                              IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset,
+#endif
+                              MMU_PROTFLAGS_T uiProtFlags,
+                              IMG_UINT32 uiLog2DataPageSize)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+	MMU_MEMORY_DESC *psMemDesc = &psLevel->sMemDesc;
+
+	IMG_UINT32 (*pfnDerivePxEProt4)(IMG_UINT32);
+	IMG_UINT64 (*pfnDerivePxEProt8)(IMG_UINT32, IMG_UINT32);
+
+	if (!psDevPAddr)
+	{
+		/* Invalidate entry */
+		if (~uiProtFlags & MMU_PROTFLAGS_INVALID)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Error, no physical address specified, but not invalidating entry"));
+			uiProtFlags |= MMU_PROTFLAGS_INVALID;
+		}
+		psDevPAddr = &gsBadDevPhyAddr;
+	}
+	else
+	{
+		if (uiProtFlags & MMU_PROTFLAGS_INVALID)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "A physical address was specified when requesting invalidation of entry"));
+			uiProtFlags |= MMU_PROTFLAGS_INVALID;
+		}
+	}
+
+	switch (eMMULevel)
+	{
+		case MMU_LEVEL_3:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePCEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePCEProt8;
+			break;
+
+		case MMU_LEVEL_2:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePDEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePDEProt8;
+			break;
+
+		case MMU_LEVEL_1:
+			pfnDerivePxEProt4 = psMMUContext->psDevAttrs->pfnDerivePTEProt4;
+			pfnDerivePxEProt8 = psMMUContext->psDevAttrs->pfnDerivePTEProt8;
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid MMU level", __func__));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* How big is a PxE in bytes? */
+	/* Filling the actual Px entry with an address */
+	switch (psConfig->uiBytesPerEntry)
+	{
+		case 4:
+		{
+			IMG_UINT32 *pui32Px;
+			IMG_UINT64 ui64PxE64;
+
+			pui32Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+
+			ui64PxE64 = psDevPAddr->uiAddr               /* Calculate the offset to that base */
+					>> psConfig->uiAddrLog2Align /* Shift away the unnecessary bits of the address */
+					<< psConfig->uiAddrShift     /* Shift back to fit address in the Px entry */
+					& psConfig->uiAddrMask;      /* Delete unused higher bits */
+
+			ui64PxE64 |= (IMG_UINT64)pfnDerivePxEProt4(uiProtFlags);
+			/* assert that the result fits into 32 bits before writing
+			   it into the 32-bit array with a cast */
+			PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+
+			/* We should never invalidate an invalid page */
+			if (uiProtFlags & MMU_PROTFLAGS_INVALID)
+			{
+				PVR_ASSERT(pui32Px[uiIndex] != ui64PxE64);
+			}
+			pui32Px[uiIndex] = (IMG_UINT32) ui64PxE64;
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+			        uiIndex, eMMULevel,
+			        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
+			break;
+		}
+		case 8:
+		{
+			IMG_UINT64 *pui64Px = psMemDesc->pvCpuVAddr; /* Give the virtual base address of Px */
+			IMG_UINT64 ui64PxE64;
+
+			ui64PxE64 = psDevPAddr->uiAddr             /* Calculate the offset to that base */
+					>> psConfig->uiAddrLog2Align  /* Shift away the unnecessary bits of the address */
+					<< psConfig->uiAddrShift      /* Shift back to fit address in the Px entry */
+					& psConfig->uiAddrMask;       /* Delete unused higher bits */
+			ui64PxE64 |= pfnDerivePxEProt8(uiProtFlags, uiLog2DataPageSize);
+
+			pui64Px[uiIndex] = ui64PxE64;
+
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+			        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+			        uiIndex, eMMULevel,
+			        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+			        (uiProtFlags & MMU_PROTFLAGS_INVALID)? 0: 1);
+			break;
+		}
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: PxE size not supported (%d) for level %d",
+					__func__, psConfig->uiBytesPerEntry, eMMULevel));
+
+			return PVRSRV_ERROR_MMU_CONFIG_IS_WRONG;
+	}
+
+#if defined(PDUMP)
+	PDumpMMUDumpPxEntries(psDevNode,
+	                      eMMULevel,
+	                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                      psMemDesc->pvCpuVAddr,
+	                      psMemDesc->sDevPAddr,
+	                      uiIndex,
+	                      1,
+	                      pszMemspaceName,
+	                      pszSymbolicAddr,
+	                      uiSymbolicAddrOffset,
+	                      psConfig->uiBytesPerEntry,
+	                      psConfig->uiAddrLog2Align,
+	                      psConfig->uiAddrShift,
+	                      psConfig->uiAddrMask,
+	                      psConfig->uiProtMask,
+	                      psConfig->uiValidEnMask,
+	                      0,
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      0, /* Unused */
+	                      psMMUContext->psDevAttrs->eMMUType);
+#endif
+
+	psDevNode->pfnMMUCacheInvalidate(psDevNode, psMMUContext,
+	                                 eMMULevel,
+	                                 uiProtFlags & MMU_PROTFLAGS_INVALID);
+
+	return PVRSRV_OK;
+}
+
+/*****************************************************************************
+ *                   MMU host control functions (Level Info)                 *
+ *****************************************************************************/
+
+
+/*************************************************************************/ /*!
+@Function       _MMU_FreeLevel
+
+@Description    Recursively frees the specified range of Px entries. If any
+                level has its last reference dropped then the MMU object
+                memory and the MMU_Levelx_Info will be freed.
+
+				At each level we might be crossing a boundary from one Px to
+				another. The values for auiStartArray should be by used for
+				the first call into each level and the values in auiEndArray
+				should only be used in the last call for each level.
+				In order to determine if this is the first/last call we pass
+				in bFirst and bLast.
+				When one level calls down to the next only if bFirst/bLast is set
+				and it's the first/last iteration of the loop at its level will
+				bFirst/bLast set for the next recursion.
+				This means that each iteration has the knowledge of the previous
+				level which is required.
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel                 Level info on which to free the
+                                        specified range
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          auiEntriesPerPxArray    Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          pui32CurrentLevel       Pointer to a variable which is set to our
+                                        current level
+
+@Input          uiStartIndex            Start index of the range to free
+
+@Input          uiEndIndex              End index of the range to free
+
+@Input			bFirst                  This is the first call for this level
+
+@Input			bLast                   This is the last call for this level
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static IMG_BOOL _MMU_FreeLevel(MMU_CONTEXT *psMMUContext,
+                               MMU_Levelx_INFO *psLevel,
+                               IMG_UINT32 auiStartArray[],
+                               IMG_UINT32 auiEndArray[],
+                               IMG_UINT32 auiEntriesPerPxArray[],
+                               const MMU_PxE_CONFIG *apsConfig[],
+                               MMU_LEVEL aeMMULevel[],
+                               IMG_UINT32 *pui32CurrentLevel,
+                               IMG_UINT32 uiStartIndex,
+                               IMG_UINT32 uiEndIndex,
+                               IMG_BOOL bFirst,
+                               IMG_BOOL bLast,
+                               IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 uiThisLevel = *pui32CurrentLevel;
+	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel];
+	IMG_UINT32 i;
+	IMG_BOOL bFreed = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Call before parameter check for debugging purposes. */
+	MMU_OBJ_DBG((PVR_DBG_ERROR,
+	             "_MMU_FreeLevel: level = %u, range %u - %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], uiStartIndex, uiEndIndex,
+	             (psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX)));
+
+	/* Parameter checks */
+	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);
+
+	if (psLevel == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid MMU level data", __func__));
+		goto ErrReturn;
+	}
+
+	for (i = uiStartIndex; i < uiEndIndex; i++)
+	{
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			MMU_Levelx_INFO *psNextLevel = psLevel->apsNextLevel[i];
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			/* Recurse into the next level */
+			(*pui32CurrentLevel)++;
+			if (_MMU_FreeLevel(psMMUContext, psNextLevel, auiStartArray,
+			                   auiEndArray, auiEntriesPerPxArray,
+			                   apsConfig, aeMMULevel, pui32CurrentLevel,
+			                   uiNextStartIndex, uiNextEndIndex,
+			                   bNextFirst, bNextLast, uiLog2DataPageSize))
+			{
+				PVRSRV_ERROR eError;
+
+				/* Un-wire the entry */
+				eError = _SetupPxE(psMMUContext,
+				                   psLevel,
+				                   i,
+				                   psConfig,
+				                   aeMMULevel[uiThisLevel],
+				                   NULL,
+#if defined(PDUMP)
+				                   NULL,	/* Only required for data page */
+				                   NULL,	/* Only required for data page */
+				                   0,      /* Only required for data page */
+#endif
+				                   MMU_PROTFLAGS_INVALID,
+				                   uiLog2DataPageSize);
+
+				PVR_ASSERT(eError == PVRSRV_OK);
+
+				/* Free table of the level below, pointed to by this table entry.
+				 * We don't destroy the table inside the above _MMU_FreeLevel call because we
+				 * first have to set the table entry of the level above to invalid. */
+				_PxMemFree(psMMUContext, &psNextLevel->sMemDesc, aeMMULevel[*pui32CurrentLevel]);
+				OSFreeMem(psNextLevel);
+
+				/* The level below us is empty, drop the refcount and clear the pointer */
+				psLevel->ui32RefCount--;
+				psLevel->apsNextLevel[i] = NULL;
+
+				/* Check we haven't wrapped around */
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+			}
+			(*pui32CurrentLevel)--;
+		}
+		else
+		{
+			psLevel->ui32RefCount--;
+		}
+
+		/*
+		   Free this level if it is no longer referenced, unless it's the base
+		   level in which case it's part of the MMU context and should be freed
+		   when the MMU context is freed
+		 */
+		if ((psLevel->ui32RefCount == 0) && (psLevel != &psMMUContext->sBaseLevelInfo))
+		{
+			bFreed = IMG_TRUE;
+		}
+	}
+
+	/* Level one flushing is done when we actually write the table entries */
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		PVRSRV_ERROR eError;
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                   &psLevel->sMemDesc.psMapping->sMemHandle,
+		                   uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                   (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+ErrReturn:
+	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_FreeLevel end: level = %u, refcount = %u",
+	             aeMMULevel[uiThisLevel],
+	             bFreed ? 0 : (psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX)));
+
+	return bFreed;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_AllocLevel
+
+@Description    Recursively allocates the specified range of Px entries. If any
+                level has its last reference dropped then the MMU object
+                memory and the MMU_Levelx_Info will be freed.
+
+				At each level we might be crossing a boundary from one Px to
+				another. The values for auiStartArray should be by used for
+				the first call into each level and the values in auiEndArray
+				should only be used in the last call for each level.
+				In order to determine if this is the first/last call we pass
+				in bFirst and bLast.
+				When one level calls down to the next only if bFirst/bLast is set
+				and it's the first/last iteration of the loop at its level will
+				bFirst/bLast set for the next recursion.
+				This means that each iteration has the knowledge of the previous
+				level which is required.
+
+@Input          psMMUContext    MMU context to operate on
+
+@Input          psLevel                 Level info on which to free the
+                                        specified range
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          auiEntriesPerPxArray    Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          pui32CurrentLevel       Pointer to a variable which is set to our
+                                        current level
+
+@Input          uiStartIndex            Start index of the range to free
+
+@Input          uiEndIndex              End index of the range to free
+
+@Input			bFirst                  This is the first call for this level
+
+@Input			bLast                   This is the last call for this level
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR _MMU_AllocLevel(MMU_CONTEXT *psMMUContext,
+                                    MMU_Levelx_INFO *psLevel,
+                                    IMG_UINT32 auiStartArray[],
+                                    IMG_UINT32 auiEndArray[],
+                                    IMG_UINT32 auiEntriesPerPxArray[],
+                                    const MMU_PxE_CONFIG *apsConfig[],
+                                    MMU_LEVEL aeMMULevel[],
+                                    IMG_UINT32 *pui32CurrentLevel,
+                                    IMG_UINT32 uiStartIndex,
+                                    IMG_UINT32 uiEndIndex,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+                                    IMG_DEV_VIRTADDR* psRunningDevVAddr,
+#endif
+                                    IMG_BOOL bFirst,
+                                    IMG_BOOL bLast,
+                                    IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 uiThisLevel = *pui32CurrentLevel; /* Starting with 0 */
+	const MMU_PxE_CONFIG *psConfig = apsConfig[uiThisLevel]; /* The table config for the current level */
+	PVRSRV_ERROR eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	IMG_UINT32 uiAllocState = 99; /* Debug info to check what progress was made in the function. Updated during this function. */
+	IMG_UINT32 i;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Parameter check */
+	PVR_ASSERT(*pui32CurrentLevel < MMU_MAX_LEVEL);
+
+	MMU_OBJ_DBG((PVR_DBG_ERROR,
+	             "_MMU_AllocLevel: level = %u, range %u - %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], uiStartIndex, uiEndIndex,
+	             psLevel != NULL ? psLevel->ui32RefCount : IMG_UINT32_MAX));
+
+	/* Go from uiStartIndex to uiEndIndex through the Px */
+	for (i = uiStartIndex;i < uiEndIndex;i++)
+	{
+		/* Only try an allocation if this is not the last level */
+		/*Because a PT allocation is already done while setting the entry in PD */
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If there is already a next Px level existing, do not allocate it */
+			if (!psLevel->apsNextLevel[i])
+			{
+				MMU_Levelx_INFO *psNextLevel;
+				IMG_UINT32 ui32AllocSize;
+				IMG_UINT32 uiNextEntries;
+
+				/* Allocate and setup the next level */
+				uiNextEntries = auiEntriesPerPxArray[uiThisLevel + 1];
+				ui32AllocSize = sizeof(MMU_Levelx_INFO);
+				if (aeMMULevel[uiThisLevel + 1] != MMU_LEVEL_1)
+				{
+					ui32AllocSize += IMG_FLEX_ARRAY_SIZE(sizeof(MMU_Levelx_INFO *), uiNextEntries);
+				}
+				psNextLevel = OSAllocZMem(ui32AllocSize);
+				if (psNextLevel == NULL)
+				{
+					uiAllocState = 0;
+					goto e0;
+				}
+
+				/* Hook in this level for next time */
+				psLevel->apsNextLevel[i] = psNextLevel;
+
+				psNextLevel->ui32NumOfEntries = uiNextEntries;
+				psNextLevel->ui32RefCount = 0;
+				/* Allocate Px memory for a sub level*/
+				eError = _PxMemAlloc(psMMUContext, uiNextEntries, apsConfig[uiThisLevel + 1],
+				                     aeMMULevel[uiThisLevel + 1],
+				                     &psNextLevel->sMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+				                     psRunningDevVAddr,
+				                     uiLog2DataPageSize,
+#endif
+				                     psConfig->uiAddrLog2Align
+);
+				if (eError != PVRSRV_OK)
+				{
+					uiAllocState = 1;
+					goto e0;
+				}
+
+				/* Wire up the entry */
+				eError = _SetupPxE(psMMUContext,
+				                   psLevel,
+				                   i,
+				                   psConfig,
+				                   aeMMULevel[uiThisLevel],
+				                   &psNextLevel->sMemDesc.sDevPAddr,
+#if defined(PDUMP)
+				                   NULL, /* Only required for data page */
+				                   NULL, /* Only required for data page */
+				                   0,    /* Only required for data page */
+#endif
+				                   0,
+				                   uiLog2DataPageSize);
+
+				if (eError != PVRSRV_OK)
+				{
+					uiAllocState = 2;
+					goto e0;
+				}
+
+				psLevel->ui32RefCount++;
+			}
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+			else
+			{
+				/* The level structure already exists, increment running device virtual address
+				   This is necessary for correct parity bit calculation on further allocated page tables */
+				if (apsConfig[aeMMULevel[MMU_LEVEL_1]]->uiParityBitMask && aeMMULevel[uiThisLevel+1] == MMU_LEVEL_1)
+				{
+					psRunningDevVAddr->uiAddr += psLevel->apsNextLevel[i]->ui32NumOfEntries * (1 << uiLog2DataPageSize);
+				}
+			}
+#endif
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			/* Recurse into the next level */
+			(*pui32CurrentLevel)++;
+			eError = _MMU_AllocLevel(psMMUContext, psLevel->apsNextLevel[i],
+			                         auiStartArray,
+			                         auiEndArray,
+			                         auiEntriesPerPxArray,
+			                         apsConfig,
+			                         aeMMULevel,
+			                         pui32CurrentLevel,
+			                         uiNextStartIndex,
+			                         uiNextEndIndex,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+
+			                         psRunningDevVAddr,
+#endif
+			                         bNextFirst,
+			                         bNextLast,
+			                         uiLog2DataPageSize);
+			(*pui32CurrentLevel)--;
+			if (eError != PVRSRV_OK)
+			{
+				uiAllocState = 3;
+				goto e0;
+			}
+		}
+		else
+		{
+			/* All we need to do for level 1 is bump the refcount */
+			psLevel->ui32RefCount++;
+		}
+
+		if (psLevel->ui32RefCount > psLevel->ui32NumOfEntries)
+		{
+			/* Given how the reference counting is implemented for MMU_LEVEL_2
+			 * and MMU_LEVEL_3 this should never happen for those levels. Only
+			 * in case of MMU_LEVEL_1 this should be possible (e.g. when someone
+			 * takes multiple reservations of the same range). In such case
+			 * return error to prevent reference count to rollover. */
+			uiAllocState = 4;
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, e0);
+		}
+	}
+
+	/* Level one flushing is done when we actually write the table entries */
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiStartIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiEndIndex - uiStartIndex) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, e1);
+	}
+
+	MMU_OBJ_DBG((PVR_DBG_ERROR, "_MMU_AllocLevel end: level = %u, refcount = %u",
+	             aeMMULevel[uiThisLevel], psLevel->ui32RefCount));
+	return PVRSRV_OK;
+
+e0:
+	/* Confirm that we've not come down this route unexpectedly */
+	PVR_ASSERT(uiAllocState!=99);
+	PVR_DPF((PVR_DBG_ERROR, "_MMU_AllocLevel: Error %d allocating Px for level %d in stage %d"
+			,eError, aeMMULevel[uiThisLevel], uiAllocState));
+
+	/* In case of an error process current `i` in special way without
+	 * recursively calling `_MMU_FreeLevel()`. */
+
+	if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+	{
+		/* If this is not a PT (so it's a PC or PD) it means that an error
+		 * happened either during PT or PD allocation/setup.
+		 *
+		 * - If the error happened during PT allocation/setup it means that
+		 *   we're at the PD level now and we already run below `else` clause
+		 *   for the failing PT and the `for` loop below for the other PTs. This
+		 *   means that all of the PTs that have been referenced during this
+		 *   operation have been dereferenced. So we just need to free that
+		 *   failing PT here and the rest of them in the loop below.
+		 * - If the error happened during PD allocation/setup it means that
+		 *   we're at the PC level now and we already run below `else` clause
+		 *   for the failing PD and the `for` loop below for the other PDs. This
+		 *   means that all of the PTs that have been allocated during this
+		 *   operation for the failing PD have been freed. So we just need to
+		 *   free that failing PD and the rest of them in a recursive manner
+		 *   in the loop below. */
+		if (uiAllocState >= 3)
+		{
+			if (psLevel->apsNextLevel[i] != NULL &&
+			    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+			{
+				psLevel->ui32RefCount--;
+			}
+		}
+		if (uiAllocState >= 2)
+		{
+			if (psLevel->apsNextLevel[i] != NULL &&
+			    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+			{
+				_PxMemFree(psMMUContext, &psLevel->apsNextLevel[i]->sMemDesc,
+				           aeMMULevel[uiThisLevel + 1]);
+			}
+		}
+		if (psLevel->apsNextLevel[i] != NULL &&
+		    psLevel->apsNextLevel[i]->ui32RefCount == 0)
+		{
+			OSFreeMem(psLevel->apsNextLevel[i]);
+			psLevel->apsNextLevel[i] = NULL;
+		}
+	}
+	else
+	{
+		/* This is a PT which means that we just need to dereference it. It's
+		 * going to be freed on the PD level in this error path in the `if`
+		 * clause above. */
+		psLevel->ui32RefCount--;
+	}
+
+	/* Check we haven't wrapped around */
+	PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+
+e1:
+	i--;
+
+	/* The start value of index variable i is not initialised on purpose.
+	 * This clean-up loop deinitialises what was already initialised in
+	 * reverse order, so the i index already has the correct value.
+	 */
+	for (/* i already set */; i >= uiStartIndex && i < uiEndIndex; i--)
+	{
+		if (aeMMULevel[uiThisLevel] != MMU_LEVEL_1)
+		{
+			IMG_UINT32 uiNextStartIndex;
+			IMG_UINT32 uiNextEndIndex;
+			IMG_BOOL bNextFirst;
+			IMG_BOOL bNextLast;
+
+			/* If we're crossing a Px then the start index changes */
+			if (bFirst && (i == uiStartIndex))
+			{
+				uiNextStartIndex = auiStartArray[uiThisLevel + 1];
+				bNextFirst = IMG_TRUE;
+			}
+			else
+			{
+				uiNextStartIndex = 0;
+				bNextFirst = IMG_FALSE;
+			}
+
+			/* If we're crossing a Px then the end index changes */
+			if (bLast && (i == (uiEndIndex - 1)))
+			{
+				uiNextEndIndex = auiEndArray[uiThisLevel + 1];
+				bNextLast = IMG_TRUE;
+			}
+			else
+			{
+				uiNextEndIndex = auiEntriesPerPxArray[uiThisLevel + 1];
+				bNextLast = IMG_FALSE;
+			}
+
+			(*pui32CurrentLevel)++;
+			if (_MMU_FreeLevel(psMMUContext, psLevel->apsNextLevel[i],
+			                   auiStartArray, auiEndArray,
+			                   auiEntriesPerPxArray, apsConfig,
+			                   aeMMULevel, pui32CurrentLevel,
+			                   uiNextStartIndex, uiNextEndIndex,
+			                   bNextFirst, bNextLast, uiLog2DataPageSize))
+			{
+				_PxMemFree(psMMUContext, &psLevel->apsNextLevel[i]->sMemDesc,
+				           aeMMULevel[uiThisLevel + 1]);
+				OSFreeMem(psLevel->apsNextLevel[i]);
+				psLevel->apsNextLevel[i] = NULL;
+
+				psLevel->ui32RefCount--;
+
+				/* Check we haven't wrapped around */
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+			}
+			(*pui32CurrentLevel)--;
+		}
+		else
+		{
+			/* We should never come down this path, but it's here
+			   for completeness */
+			psLevel->ui32RefCount--;
+
+			/* Check we haven't wrapped around */
+			PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+		}
+	}
+
+	return eError;
+}
+
+/*****************************************************************************
+ *                   MMU page table functions                                *
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetLevelData
+
+@Description    Get the all the level data and calculates the indexes for the
+                specified address range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Log2 of the page size to use
+
+@Input          auiStartArray           Array of start indexes (one for each level)
+
+@Input          auiEndArray             Array of end indexes (one for each level)
+
+@Input          uiEntriesPerPxArray     Array of number of entries for the Px
+                                        (one for each level)
+
+@Input          apsConfig               Array of PxE configs (one for each level)
+
+@Input          aeMMULevel              Array of MMU levels (one for each level)
+
+@Input          ppsMMUDevVAddrConfig    Device virtual address config
+
+@Input			phPriv					Private data of page size config
+
+@Return         IMG_TRUE if the last reference to psLevel was dropped
+ */
+/*****************************************************************************/
+static void _MMU_GetLevelData(MMU_CONTEXT *psMMUContext,
+                              IMG_DEV_VIRTADDR sDevVAddrStart,
+                              IMG_DEV_VIRTADDR sDevVAddrEnd,
+                              IMG_UINT32 uiLog2DataPageSize,
+                              IMG_UINT32 auiStartArray[],
+                              IMG_UINT32 auiEndArray[],
+                              IMG_UINT32 auiEntriesPerPx[],
+                              const MMU_PxE_CONFIG *apsConfig[],
+                              MMU_LEVEL aeMMULevel[],
+                              const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+                              IMG_HANDLE *phPriv)
+{
+	const MMU_PxE_CONFIG *psMMUPDEConfig;
+	const MMU_PxE_CONFIG *psMMUPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i = 0;
+
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
+	                                                 &psMMUPDEConfig,
+	                                                 &psMMUPTEConfig,
+	                                                 ppsMMUDevVAddrConfig,
+	                                                 phPriv);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psDevVAddrConfig = *ppsMMUDevVAddrConfig;
+
+	if (psDevVAddrConfig->uiPCIndexMask != 0)
+	{
+		auiStartArray[i] = _CalcPCEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+		auiEndArray[i] = _CalcPCEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPC;
+		apsConfig[i] = psDevAttrs->psBaseConfig;
+		aeMMULevel[i] = MMU_LEVEL_3;
+		i++;
+	}
+
+	if (psDevVAddrConfig->uiPDIndexMask != 0)
+	{
+		auiStartArray[i] = _CalcPDEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+		auiEndArray[i] = _CalcPDEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+		auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPD;
+		if (i == 0)
+		{
+			apsConfig[i] = psDevAttrs->psBaseConfig;
+		}
+		else
+		{
+			apsConfig[i] = psMMUPDEConfig;
+		}
+		aeMMULevel[i] = MMU_LEVEL_2;
+		i++;
+	}
+
+	/*
+		There is always a PTE entry so we have a slightly different behaviour than above.
+		E.g. for 2 MB pages the uiPTIndexMask is 0x0000000000 but still there
+		is a PT with one entry.
+
+	 */
+	auiStartArray[i] = _CalcPTEIdx(sDevVAddrStart, psDevVAddrConfig, IMG_FALSE);
+	if (psDevVAddrConfig->uiPTIndexMask !=0)
+	{
+		auiEndArray[i] = _CalcPTEIdx(sDevVAddrEnd, psDevVAddrConfig, IMG_TRUE);
+	}
+	else
+	{
+		/*
+			If the PTE mask is zero it means there is only 1 PTE and thus, as an
+			an exclusive bound, the end array index is equal to the start index + 1.
+		 */
+
+		auiEndArray[i] = auiStartArray[i] + 1;
+	}
+
+	auiEntriesPerPx[i] = psDevVAddrConfig->uiNumEntriesPT;
+
+	if (i == 0)
+	{
+		apsConfig[i] = psDevAttrs->psBaseConfig;
+	}
+	else
+	{
+		apsConfig[i] = psMMUPTEConfig;
+	}
+	aeMMULevel[i] = MMU_LEVEL_1;
+}
+
+static void _MMU_PutLevelData(MMU_CONTEXT *psMMUContext, IMG_HANDLE hPriv)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+}
+
+/*************************************************************************/ /*!
+@Function       _AllocPageTables
+
+@Description    Allocate page tables and any higher level MMU objects required
+                for the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Page size of the data pages
+
+@Return         PVRSRV_OK if the allocation was successful
+ */
+/*****************************************************************************/
+static PVRSRV_ERROR
+_AllocPageTables(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrStart,
+                 IMG_DEV_VIRTADDR sDevVAddrEnd,
+                 IMG_UINT32 uiLog2DataPageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
+	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
+	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
+	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+	IMG_UINT32 ui32CurrentLevel = 0;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_DEV_VIRTADDR sRunningDevVAddrStart;
+	IMG_BOOL bSetParity;
+#endif
+
+	PVR_DPF((PVR_DBG_ALLOC,
+			"_AllocPageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+			sDevVAddrStart.uiAddr,
+			sDevVAddrEnd.uiAddr
+	));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode,
+	             "Allocating page tables for %"IMG_UINT64_FMTSPEC" bytes virtual range: "
+	             IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+	             (IMG_UINT64)sDevVAddrEnd.uiAddr - (IMG_UINT64)sDevVAddrStart.uiAddr,
+	             (IMG_UINT64)sDevVAddrStart.uiAddr,
+	             (IMG_UINT64)sDevVAddrEnd.uiAddr);
+#endif
+
+	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
+	                  (IMG_UINT32) uiLog2DataPageSize, auiStartArray, auiEndArray,
+	                  auiEntriesPerPx, apsConfig, aeMMULevel,
+	                  &psDevVAddrConfig, &hPriv);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	bSetParity = apsConfig[aeMMULevel[MMU_LEVEL_1]]->uiParityBitMask != 0;
+	if (bSetParity)
+	{
+		/* If parity bit needs to be written for PTEs save the first VA of the PT */
+		sRunningDevVAddrStart.uiAddr = sDevVAddrStart.uiAddr & ~((IMG_UINT64)(auiEntriesPerPx[MMU_LEVEL_1] * (1 << uiLog2DataPageSize)) - 1);
+	}
+
+#endif
+
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_ALLOC,
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));
+
+	eError = _MMU_AllocLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
+	                         auiStartArray, auiEndArray, auiEntriesPerPx,
+	                         apsConfig, aeMMULevel, &ui32CurrentLevel,
+	                         auiStartArray[0], auiEndArray[0],
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	                         &sRunningDevVAddrStart,
+#endif
+	                         IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP) && defined(DEBUG)
+	if (bSetParity)
+	{
+		PVR_ASSERT(sRunningDevVAddrStart.uiAddr >= sDevVAddrEnd.uiAddr);
+	}
+#endif
+
+	_MMU_PutLevelData(psMMUContext, hPriv);
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreePageTables
+
+@Description    Free page tables and any higher level MMU objects at are no
+                longer referenced for the specified virtual range.
+                This will fill the temporary free list of the MMU context which
+                needs cleanup after the call.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrStart          Start device virtual address
+
+@Input          sDevVAddrEnd            End device virtual address
+
+@Input          uiLog2DataPageSize      Page size of the data pages
+
+@Return         None
+ */
+/*****************************************************************************/
+static void _FreePageTables(MMU_CONTEXT *psMMUContext,
+                            IMG_DEV_VIRTADDR sDevVAddrStart,
+                            IMG_DEV_VIRTADDR sDevVAddrEnd,
+                            IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT32 auiStartArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEndArray[MMU_MAX_LEVEL];
+	IMG_UINT32 auiEntriesPerPx[MMU_MAX_LEVEL];
+	MMU_LEVEL aeMMULevel[MMU_MAX_LEVEL];
+	const MMU_PxE_CONFIG *apsConfig[MMU_MAX_LEVEL];
+	const MMU_DEVVADDR_CONFIG	*psDevVAddrConfig;
+	IMG_UINT32 ui32CurrentLevel = 0;
+	IMG_HANDLE hPriv;
+
+	PVR_DPF((PVR_DBG_ALLOC,
+			"_FreePageTables: vaddr range: "IMG_DEV_VIRTADDR_FMTSPEC":"IMG_DEV_VIRTADDR_FMTSPEC,
+			sDevVAddrStart.uiAddr,
+			sDevVAddrEnd.uiAddr
+	));
+
+	_MMU_GetLevelData(psMMUContext, sDevVAddrStart, sDevVAddrEnd,
+	                  uiLog2DataPageSize, auiStartArray, auiEndArray,
+	                  auiEntriesPerPx, apsConfig, aeMMULevel,
+	                  &psDevVAddrConfig, &hPriv);
+
+	HTBLOGK(HTB_SF_MMU_PAGE_OP_FREE,
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrStart.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrStart.uiAddr),
+	        HTBLOG_U64_BITS_HIGH(sDevVAddrEnd.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrEnd.uiAddr));
+
+	/* ignoring return code, in this case there should be no references
+	 * to the level anymore, and at this stage there is nothing to do with
+	 * the return status */
+	(void) _MMU_FreeLevel(psMMUContext, &psMMUContext->sBaseLevelInfo,
+	                      auiStartArray, auiEndArray, auiEntriesPerPx,
+	                      apsConfig, aeMMULevel, &ui32CurrentLevel,
+	                      auiStartArray[0], auiEndArray[0],
+	                      IMG_TRUE, IMG_TRUE, uiLog2DataPageSize);
+
+	_MMU_PutLevelData(psMMUContext, hPriv);
+}
+
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetPTInfo
+
+@Description    Get the PT level information and PT entry index for the specified
+                virtual address
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          psDevVAddr              Device virtual address to get the PTE info
+                                        from.
+
+@Input          psDevVAddrConfig        The current virtual address config obtained
+                                        by another function call before.
+
+@Output         psLevel                 Level info of the PT
+
+@Output         pui32PTEIndex           Index into the PT the address corresponds to
+
+@Return         IMG_TRUE if the operation was successful and IMG_FALSE otherwise
+ */
+/*****************************************************************************/
+static INLINE IMG_BOOL _MMU_GetPTInfo(MMU_CONTEXT                *psMMUContext,
+                                      IMG_DEV_VIRTADDR            sDevVAddr,
+                                      const MMU_DEVVADDR_CONFIG  *psDevVAddrConfig,
+                                      MMU_Levelx_INFO           **ppsLevel,
+                                      IMG_UINT32                 *pui32PTEIndex)
+{
+	MMU_Levelx_INFO *psLocalLevel = NULL;
+	MMU_LEVEL eMMULevel = psMMUContext->psDevAttrs->psBaseConfig->ePxLevel;
+	const MMU_LEVEL eMMUBaseLevel = eMMULevel;
+	IMG_UINT32 uiPCEIndex;
+	IMG_UINT32 uiPDEIndex;
+
+	if ((eMMULevel <= MMU_LEVEL_0) || (eMMULevel >= MMU_LEVEL_LAST))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTEInfo: Invalid MMU level"));
+		PVR_ASSERT(0);
+	}
+
+	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
+	{
+		if (eMMULevel == MMU_LEVEL_3)
+		{
+			/* find the page directory pointed by the PCE */
+			uiPCEIndex = _CalcPCEIdx (sDevVAddr, psDevVAddrConfig,
+			                          IMG_FALSE);
+			psLocalLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiPCEIndex];
+		}
+
+		if (eMMULevel == MMU_LEVEL_2)
+		{
+			/* find the page table pointed by the PDE */
+			uiPDEIndex = _CalcPDEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+			if (psLocalLevel == NULL)
+			{
+				return IMG_FALSE;
+			}
+
+			psLocalLevel = psLocalLevel->apsNextLevel[uiPDEIndex];
+		}
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			/* find PTE index into page table */
+			*pui32PTEIndex = _CalcPTEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLocalLevel == NULL)
+			{
+				if (eMMUBaseLevel == eMMULevel)
+				{
+					/* if the MMU only supports one level return the base level */
+					psLocalLevel = &psMMUContext->sBaseLevelInfo;
+				}
+				else
+				{
+					return IMG_FALSE;
+				}
+			}
+		}
+	}
+
+	*ppsLevel = psLocalLevel;
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_GetPTConfig
+
+@Description    Get the level config. Call _MMU_PutPTConfig after use!
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiLog2DataPageSize      Log 2 of the page size
+
+@Output         ppsConfig               Config of the PTE
+
+@Output         phPriv                  Private data handle to be passed back
+                                        when the info is put
+
+@Output         ppsDevVAddrConfig       Config of the device virtual addresses
+
+@Return         None
+ */
+/*****************************************************************************/
+static INLINE void _MMU_GetPTConfig(MMU_CONTEXT               *psMMUContext,
+                                    IMG_UINT32                  uiLog2DataPageSize,
+                                    const MMU_PxE_CONFIG      **ppsConfig,
+                                    IMG_HANDLE                 *phPriv,
+                                    const MMU_DEVVADDR_CONFIG **ppsDevVAddrConfig)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+
+	if (psDevAttrs->pfnGetPageSizeConfiguration(uiLog2DataPageSize,
+	                                            &psPDEConfig,
+	                                            &psPTEConfig,
+	                                            &psDevVAddrConfig,
+	                                            phPriv) != PVRSRV_OK)
+	{
+		/*
+		   There should be no way we got here unless uiLog2DataPageSize
+		   has changed after the MMU_Alloc call (in which case it's a bug in
+		   the MM code)
+		 */
+		PVR_DPF((PVR_DBG_ERROR, "_MMU_GetPTConfig: Could not get valid page size config"));
+		PVR_ASSERT(0);
+	}
+
+	*ppsConfig = psPTEConfig;
+	*ppsDevVAddrConfig = psDevVAddrConfig;
+}
+
+/*************************************************************************/ /*!
+@Function       _MMU_PutPTConfig
+
+@Description    Put the level info. Has to be called after _MMU_GetPTConfig to
+                ensure correct refcounting.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          phPriv                  Private data handle created by
+                                        _MMU_GetPTConfig.
+
+@Return         None
+ */
+/*****************************************************************************/
+static INLINE void _MMU_PutPTConfig(MMU_CONTEXT *psMMUContext,
+                                    IMG_HANDLE hPriv)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	if (psDevAttrs->pfnPutPageSizeConfiguration(hPriv) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Could not put page size config",
+				__func__));
+		PVR_ASSERT(0);
+	}
+}
+
+/* scratch / zero pages */
+
+static INLINE PVRSRV_ERROR _MMU_GetDefPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                           IMG_INT uiDefPage,
+                                           PVRSRV_DEF_PAGE **ppsDefPage,
+                                           IMG_CHAR **ppcDefPageName)
+{
+	switch (uiDefPage)
+	{
+		case SCRATCH_PAGE:
+		{
+			*ppsDefPage = &psDevNode->sScratchPage;
+#if defined(PDUMP)
+			*ppcDefPageName = SCRATCH_PAGE_STR;
+#endif
+			break;
+		}
+		case DEV_ZERO_PAGE:
+		{
+			*ppsDefPage = &psDevNode->sDevZeroPage;
+#if defined(PDUMP)
+			*ppcDefPageName = DEV_ZERO_PAGE_STR;
+#endif
+			break;
+		}
+		default:
+		{
+			/* Invalid pcDefPageName */
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MMU_AllocBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                          IMG_INT uiInitValue,
+                                          IMG_INT uiDefPage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR	sDevPAddr = {0};
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+
+	eError = _MMU_GetDefPage(psDevNode,
+	                         uiDefPage,
+	                         &psDefPage,
+	                         &pcDefPageName);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr != MMU_BAD_PHYS_ADDR)
+	{
+		goto UnlockAndReturn;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Alloc %s page object", pcDefPageName);
+#endif
+
+	/* Allocate the scratch / zero page required for physical backing
+	 * of virtual ranges
+	 */
+	eError = DevPhysMemAlloc(psDevNode,
+	                         (1 << psDefPage->ui32Log2PgSize),
+	                         0,
+	                         uiInitValue,
+	                         IMG_TRUE,
+#if defined(PDUMP)
+	                         psDevNode->psMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+	                         pcDefPageName,
+	                         &psDefPage->hPdumpPg,
+#endif
+	                         PVR_SYS_ALLOC_PID,
+	                         &psDefPage->sPageHandle,
+	                         &sDevPAddr);
+
+	psDefPage->ui64PgPhysAddr = sDevPAddr.uiAddr;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _MMU_GetBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                        IMG_UINT64 *pui64PgPhysAddr,
+                                        IMG_INT uiDefPage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+
+	eError = _MMU_GetDefPage(psDevNode,
+	                         uiDefPage,
+	                         &psDefPage,
+	                         &pcDefPageName);
+	PVR_RETURN_IF_ERROR(eError);
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr == MMU_BAD_PHYS_ADDR)
+	{
+		eError = PVRSRV_ERROR_NOT_FOUND;
+		PVR_DPF((PVR_DBG_ERROR,"%s: %s (uiDefPage=%d)",
+		         __func__, PVRSRVGetErrorString(eError), uiDefPage));
+		goto UnlockAndReturn;
+	}
+
+	if (pui64PgPhysAddr)
+		*pui64PgPhysAddr = psDefPage->ui64PgPhysAddr;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+
+	return eError;
+}
+
+static void _MMU_FreeBackingPage(PVRSRV_DEVICE_NODE *psDevNode,
+                                 IMG_INT uiDefPage)
+{
+	PVRSRV_DEF_PAGE *psDefPage;
+	IMG_CHAR *pcDefPageName;
+	PVRSRV_ERROR eErr;
+
+	eErr = _MMU_GetDefPage(psDevNode,
+	                       uiDefPage,
+	                       &psDefPage,
+	                       &pcDefPageName);
+	if (eErr != PVRSRV_OK)
+		return;
+
+	OSLockAcquire(psDefPage->psPgLock);
+
+	if (psDefPage->ui64PgPhysAddr == MMU_BAD_PHYS_ADDR)
+	{
+		goto UnlockAndReturn;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Free %s page object", pcDefPageName);
+#endif
+
+	DevPhysMemFree(psDevNode,
+#if defined(PDUMP)
+	               psDefPage->hPdumpPg,
+#endif
+	               &psDefPage->sPageHandle);
+
+#if defined(PDUMP)
+	psDefPage->hPdumpPg = NULL;
+#endif
+	psDefPage->ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+UnlockAndReturn:
+	OSLockRelease(psDefPage->psPgLock);
+}
+
+
+/*****************************************************************************
+ *                     Public interface functions                            *
+ *****************************************************************************/
+
+/*
+	MMU_InitDevice
+*/
+PVRSRV_ERROR MMU_InitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Set the order to 0 */
+	psDevNode->sScratchPage.sPageHandle.uiOrder = 0;
+	psDevNode->sDevZeroPage.sPageHandle.uiOrder = 0;
+
+	/* Set the size of the Scratch and Zero pages to largest page size */
+	if (psDevNode->ui32Non4KPageSizeLog2 != 0)
+	{
+		psDevNode->sScratchPage.ui32Log2PgSize = psDevNode->ui32Non4KPageSizeLog2;
+		psDevNode->sDevZeroPage.ui32Log2PgSize = psDevNode->ui32Non4KPageSizeLog2;
+	}
+	else
+	{
+		psDevNode->sScratchPage.ui32Log2PgSize = OSGetPageShift();
+		psDevNode->sDevZeroPage.ui32Log2PgSize = OSGetPageShift();
+	}
+
+	/* Set the Scratch page phys addr */
+	psDevNode->sScratchPage.ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+	/* Set the Zero page phys addr */
+	psDevNode->sDevZeroPage.ui64PgPhysAddr = MMU_BAD_PHYS_ADDR;
+
+	/* The lock can be acquired from MISR (Z-buffer) path */
+	eError = OSLockCreate(&psDevNode->sScratchPage.psPgLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.Scratch", ErrReturnError);
+
+	/* Create the lock for zero page */
+	eError = OSLockCreate(&psDevNode->sDevZeroPage.psPgLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.Zero", ErrFreeScratchPageLock);
+
+#ifdef PDUMP
+	psDevNode->sScratchPage.hPdumpPg = NULL;
+	psDevNode->sDevZeroPage.hPdumpPg = NULL;
+#endif /* PDUMP */
+
+	eError = _MMU_AllocBackingPage(psDevNode,
+	                               PVR_SCRATCH_PAGE_INIT_VALUE,
+	                               SCRATCH_PAGE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_AllocBackingPage.Scratch", ErrFreeZeroPageLock);
+
+	eError = _MMU_AllocBackingPage(psDevNode,
+	                               PVR_ZERO_PAGE_INIT_VALUE,
+	                               DEV_ZERO_PAGE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_AllocBackingPage.Zero", ErrFreeScratchPage);
+
+	return PVRSRV_OK;
+
+ErrFreeScratchPage:
+	_MMU_FreeBackingPage(psDevNode, SCRATCH_PAGE);
+ErrFreeZeroPageLock:
+	OSLockDestroy(psDevNode->sDevZeroPage.psPgLock);
+	psDevNode->sDevZeroPage.psPgLock = NULL;
+ErrFreeScratchPageLock:
+	OSLockDestroy(psDevNode->sScratchPage.psPgLock);
+	psDevNode->sScratchPage.psPgLock = NULL;
+ErrReturnError:
+	return eError;
+}
+
+/*
+	MMU_DeInitDevice
+*/
+void MMU_DeInitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode)
+{
+	if (psDevNode->sScratchPage.psPgLock != NULL)
+	{
+		_MMU_FreeBackingPage(psDevNode, SCRATCH_PAGE);
+
+		OSLockDestroy(psDevNode->sScratchPage.psPgLock);
+		psDevNode->sScratchPage.psPgLock = NULL;
+	}
+
+	if (psDevNode->sDevZeroPage.psPgLock)
+	{
+		_MMU_FreeBackingPage(psDevNode, DEV_ZERO_PAGE);
+
+
+		OSLockDestroy(psDevNode->sDevZeroPage.psPgLock);
+		psDevNode->sDevZeroPage.psPgLock = NULL;
+	}
+}
+
+/*
+	MMU_ContextCreate
+ */
+PVRSRV_ERROR
+MMU_ContextCreate(CONNECTION_DATA *psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  MMU_CONTEXT **ppsMMUContext,
+                  MMU_DEVICEATTRIBS *psDevAttrs)
+{
+	MMU_CONTEXT *psMMUContext;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	const MMU_PxE_CONFIG *psConfig;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx;
+	IMG_UINT32 ui32BaseObjects;
+	IMG_UINT32 ui32Size;
+	IMG_CHAR sBuf[40];
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "MMU context create");
+#endif
+
+	psConfig = psDevAttrs->psBaseConfig;
+	psDevVAddrConfig = psDevAttrs->psTopLevelDevVAddrConfig;
+
+	switch (psDevAttrs->psBaseConfig->ePxLevel)
+	{
+		case MMU_LEVEL_3:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPC;
+			break;
+
+		case MMU_LEVEL_2:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPD;
+			break;
+
+		case MMU_LEVEL_1:
+			ui32BaseObjects = psDevVAddrConfig->uiNumEntriesPT;
+			break;
+
+		default:
+			PVR_LOG_GOTO_WITH_ERROR("psDevAttrs->psBaseConfig->ePxLevel", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+	}
+
+	/* Allocate the MMU context with the Level 1 Px info's */
+	ui32Size = sizeof(MMU_CONTEXT) +
+			((ui32BaseObjects - 1) * sizeof(MMU_Levelx_INFO *));
+
+	psMMUContext = OSAllocZMem(ui32Size);
+	PVR_LOG_GOTO_IF_NOMEM(psMMUContext, eError, e0);
+
+#if defined(PDUMP)
+	/* Clear the refcount */
+	psMMUContext->ui32PDumpContextIDRefCount = 0;
+#endif
+	/* Record Device specific attributes in the context for subsequent use */
+	psMMUContext->psDevAttrs = psDevAttrs;
+
+	/*
+	  Allocate physmem context and set it up
+	 */
+	psPhysMemCtx = OSAllocZMem(sizeof(MMU_PHYSMEM_CONTEXT));
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx, eError, e1);
+
+	psMMUContext->psPhysMemCtx = psPhysMemCtx;
+	psMMUContext->psConnection = psConnection;
+
+	psPhysMemCtx->psDevNode = psDevNode;		/* Needed for Direct Bridge case */
+	psPhysMemCtx->psMMUContext = psMMUContext;	/* Back-link to self */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/* Save the app-specific values for external reference via MMU_GetOSids. */
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+	{
+		psPhysMemCtx->ui32OSid     = psConnection->ui32OSid;
+		psPhysMemCtx->ui32OSidReg  = psConnection->ui32OSidReg;
+		psPhysMemCtx->bOSidAxiProt = psConnection->bOSidAxiProtReg;
+	}
+	else
+	{
+		/* Direct Bridge calling sequence e.g. Firmware */
+		psPhysMemCtx->ui32OSid     = 0;
+		psPhysMemCtx->ui32OSidReg  = 0;
+		psPhysMemCtx->bOSidAxiProt = IMG_FALSE;
+	}
+#endif
+
+	OSSNPrintf(sBuf, sizeof(sBuf), "pgtables %p", psPhysMemCtx);
+	psPhysMemCtx->uiPhysMemRANameAllocSize = OSStringLength(sBuf)+1;
+	psPhysMemCtx->pszPhysMemRAName = OSAllocMem(psPhysMemCtx->uiPhysMemRANameAllocSize);
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx->pszPhysMemRAName, eError, e2);
+
+	OSStringSafeCopy(psPhysMemCtx->pszPhysMemRAName, sBuf, psPhysMemCtx->uiPhysMemRANameAllocSize);
+
+	psPhysMemCtx->psPhysMemRA = RA_Create(psPhysMemCtx->pszPhysMemRAName,
+	                                      /* subsequent import */
+	                                      PhysHeapGetPageShift(psDevNode->psMMUPhysHeap),
+	                                      RA_LOCKCLASS_2,
+	                                      _MMU_PhysMem_RAImportAlloc,
+	                                      _MMU_PhysMem_RAImportFree,
+	                                      psPhysMemCtx, /* priv */
+	                                      RA_POLICY_DEFAULT);
+	if (psPhysMemCtx->psPhysMemRA == NULL)
+	{
+		OSFreeMem(psPhysMemCtx->pszPhysMemRAName);
+		psPhysMemCtx->pszPhysMemRAName = NULL;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, e3);
+	}
+
+	/* Setup cleanup meta data to check if a MMU context
+	 * has been destroyed and should not be accessed anymore */
+	psPhysMemCtx->psCleanupData = OSAllocMem(sizeof(*(psPhysMemCtx->psCleanupData)));
+	PVR_LOG_GOTO_IF_NOMEM(psPhysMemCtx->psCleanupData, eError, e4);
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+	/* Record the originating OSid for all allocation / free for this context */
+	psPhysMemCtx->psCleanupData->ui32OSid = psPhysMemCtx->ui32OSid;
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+	OSLockCreate(&psPhysMemCtx->psCleanupData->hCleanupLock);
+	psPhysMemCtx->psCleanupData->bMMUContextExists = IMG_TRUE;
+	dllist_init(&psPhysMemCtx->psCleanupData->sMMUCtxCleanupItemsHead);
+	OSAtomicWrite(&psPhysMemCtx->psCleanupData->iRef, 1);
+
+	/* allocate the base level object */
+	/*
+	   Note: Although this is not required by the this file until
+	         the 1st allocation is made, a device specific callback
+	         might request the base object address so we allocate
+	         it up front.
+	         In VZ we only need to allocate the FW/KM ctx on the
+	         Host Driver as all Guest tables are allocated and
+	         pre-mapped into the host FW Memory Context.
+	         Drivers with security support and premapped heaps
+	         rely on the TEE to manage the Firmware mappings.
+	 */
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#else
+	if (!_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+#endif
+	{
+		if (_PxMemAlloc(psMMUContext,
+						ui32BaseObjects,
+						psConfig,
+						psDevAttrs->psBaseConfig->ePxLevel,
+						&psMMUContext->sBaseLevelInfo.sMemDesc,
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+						NULL,
+						0U,
+#endif
+						psDevAttrs->ui32BaseAlign))
+		{
+			PVR_LOG_GOTO_WITH_ERROR("_PxMemAlloc", eError, PVRSRV_ERROR_OUT_OF_MEMORY, e5);
+		}
+	}
+
+	dllist_init(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);
+
+	psMMUContext->sBaseLevelInfo.ui32NumOfEntries = ui32BaseObjects;
+	psMMUContext->sBaseLevelInfo.ui32RefCount = 0;
+
+	/* Support cores that need to adjust the base level Px */
+	if (psDevNode->pfnMMUTopLevelPxWorkarounds != NULL)
+	{
+		PVRSRV_ERROR eError;
+		MMU_MEMORY_DESC  *psMemDesc  = &psMMUContext->sBaseLevelInfo.sMemDesc;
+
+		psDevNode->pfnMMUTopLevelPxWorkarounds(psConnection,
+		                                     psDevNode,
+		                                     psMemDesc->sDevPAddr,
+		                                     psMemDesc->pvCpuVAddr);
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psMemDesc->psMapping->sMemHandle,
+		                            psMemDesc->uiOffset,
+		                            psMemDesc->uiSize);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+	eError = OSLockCreate(&psMMUContext->hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e6);
+
+	/* return context */
+	*ppsMMUContext = psMMUContext;
+
+	return PVRSRV_OK;
+
+e6:
+	_PxMemFree(psMMUContext, &psMMUContext->sBaseLevelInfo.sMemDesc, psDevAttrs->psBaseConfig->ePxLevel);
+e5:
+	OSFreeMem(psPhysMemCtx->psCleanupData);
+e4:
+	RA_Delete(psPhysMemCtx->psPhysMemRA);
+e3:
+	OSFreeMem(psPhysMemCtx->pszPhysMemRAName);
+e2:
+	OSFreeMem(psPhysMemCtx);
+e1:
+	OSFreeMem(psMMUContext);
+e0:
+	return eError;
+}
+
+/*
+	MMU_ContextDestroy
+ */
+void
+MMU_ContextDestroy (MMU_CONTEXT *psMMUContext)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PDLLIST_NODE psNode, psNextNode;
+
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)psMMUContext->psPhysMemCtx->psDevNode;
+	MMU_CTX_CLEANUP_DATA *psCleanupData = psMMUContext->psPhysMemCtx->psCleanupData;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Enter", __func__));
+
+	if (psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK)
+	{
+		/* There should be no way to get here with live pages unless
+		   there is a bug in this module or the MM code */
+		PVR_ASSERT(psMMUContext->sBaseLevelInfo.ui32RefCount == 0);
+	}
+
+	/* Cleanup lock must be acquired before MMUContext lock. Reverse order
+	 * may lead to a deadlock and is reported by lockdep. */
+	OSLockAcquire(psCleanupData->hCleanupLock);
+	OSLockAcquire(psMMUContext->hLock);
+
+	/* Free the top level MMU object - will be put on defer free list.
+	 * This has to be done before the step below that will empty the
+	 * defer-free list. */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#else
+	if (!_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+#endif
+	{
+		_PxMemFree(psMMUContext,
+		           &psMMUContext->sBaseLevelInfo.sMemDesc,
+		           psMMUContext->psDevAttrs->psBaseConfig->ePxLevel);
+	}
+
+	/* Empty the temporary defer-free list of Px */
+	_FreeMMUMapping(psDevNode, &psMMUContext->psPhysMemCtx->sTmpMMUMappingHead);
+	PVR_ASSERT(dllist_is_empty(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead));
+
+	/* Empty the defer free list so the cleanup thread will
+	 * not have to access any MMU context related structures anymore */
+	dllist_foreach_node(&psCleanupData->sMMUCtxCleanupItemsHead,
+	                    psNode,
+	                    psNextNode)
+	{
+		MMU_CLEANUP_ITEM *psCleanup = IMG_CONTAINER_OF(psNode,
+		                                               MMU_CLEANUP_ITEM,
+		                                               sMMUCtxCleanupItem);
+
+		_FreeMMUMapping(psDevNode, &psCleanup->sMMUMappingHead);
+
+		dllist_remove_node(psNode);
+	}
+	PVR_ASSERT(dllist_is_empty(&psCleanupData->sMMUCtxCleanupItemsHead));
+
+	psCleanupData->bMMUContextExists = IMG_FALSE;
+
+	/* Free physmem context */
+	RA_Delete(psMMUContext->psPhysMemCtx->psPhysMemRA);
+	psMMUContext->psPhysMemCtx->psPhysMemRA = NULL;
+	OSFreeMem(psMMUContext->psPhysMemCtx->pszPhysMemRAName);
+	psMMUContext->psPhysMemCtx->pszPhysMemRAName = NULL;
+
+	OSFreeMem(psMMUContext->psPhysMemCtx);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	OSLockRelease(psCleanupData->hCleanupLock);
+
+	if (OSAtomicDecrement(&psCleanupData->iRef) == 0)
+	{
+		OSLockDestroy(psCleanupData->hCleanupLock);
+		OSFreeMem(psCleanupData);
+	}
+
+	OSLockDestroy(psMMUContext->hLock);
+
+	/* free the context itself. */
+	OSFreeMem(psMMUContext);
+	/*not nulling pointer, copy on stack*/
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Exit", __func__));
+}
+
+/*
+	MMU_Alloc
+ */
+PVRSRV_ERROR
+MMU_Alloc (MMU_CONTEXT *psMMUContext,
+           IMG_DEVMEM_SIZE_T uSize,
+           IMG_UINT32 uiProtFlags,
+           IMG_DEVMEM_SIZE_T uDevVAddrAlignment,
+           IMG_DEV_VIRTADDR *psDevVAddr,
+           IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_VIRTADDR sDevVAddrEnd;
+
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	MMU_DEVICEATTRIBS *psDevAttrs;
+	IMG_HANDLE hPriv;
+
+#if !defined(DEBUG)
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uDevVAddrAlignment);
+#endif
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: uSize=" IMG_DEVMEM_SIZE_FMTSPEC
+			", uiProtFlags=0x%x, align="IMG_DEVMEM_ALIGN_FMTSPEC,
+			__func__, uSize, uiProtFlags, uDevVAddrAlignment));
+
+	/* check params */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevVAddr, "psDevVAddr");
+
+	psDevAttrs = psMMUContext->psDevAttrs;
+
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(uiLog2PageSize,
+	                                                 &psPDEConfig,
+	                                                 &psPTEConfig,
+	                                                 &psDevVAddrConfig,
+	                                                 &hPriv);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnGetPageSizeConfiguration");
+
+	/* size and alignment must be datapage granular */
+	if (((psDevVAddr->uiAddr & psDevVAddrConfig->uiPageOffsetMask) != 0)
+			|| ((uSize & psDevVAddrConfig->uiPageOffsetMask) != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: invalid address or size granularity",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sDevVAddrEnd = *psDevVAddr;
+	sDevVAddrEnd.uiAddr += uSize;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (_MMU_IS_FWKM_CTX(psMMUContext))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver relying on firmware mappings created by the TEE.", __func__));
+	}
+	else
+#endif
+	{
+		eError = _AllocPageTables(psMMUContext, *psDevVAddr, sDevVAddrEnd, uiLog2PageSize);
+	}
+	OSLockRelease(psMMUContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "_AllocPageTables");
+		return PVRSRV_ERROR_MMU_FAILED_TO_ALLOCATE_PAGETABLES;
+	}
+
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_Free
+ */
+void
+MMU_Free (MMU_CONTEXT *psMMUContext,
+          IMG_DEV_VIRTADDR sDevVAddr,
+          IMG_DEVMEM_SIZE_T uiSize,
+          IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_DEV_VIRTADDR sDevVAddrEnd;
+	MMU_PHYSMEM_CONTEXT *psPhysMemCtx;
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+#endif
+
+
+	PVR_ASSERT(psMMUContext != NULL);
+	PVR_LOG_RETURN_VOID_IF_FALSE(psMMUContext != NULL, "psMMUContext");
+
+	psPhysMemCtx = psMMUContext->psPhysMemCtx;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Freeing DevVAddr " IMG_DEV_VIRTADDR_FMTSPEC,
+			__func__, sDevVAddr.uiAddr));
+
+	/* ensure the address range to free is inside the heap */
+	sDevVAddrEnd = sDevVAddr;
+	sDevVAddrEnd.uiAddr += uiSize;
+
+	/* The Cleanup lock has to be taken before the MMUContext hLock to
+	 * prevent deadlock scenarios. It is necessary only for parts of
+	 * _SetupCleanup_FreeMMUMapping though.*/
+	OSLockAcquire(psPhysMemCtx->psCleanupData->hCleanupLock);
+
+	OSLockAcquire(psMMUContext->hLock);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	if (!_MMU_IS_FWKM_CTX(psMMUContext))
+#endif
+	{
+		_FreePageTables(psMMUContext,
+		                sDevVAddr,
+		                sDevVAddrEnd,
+		                uiLog2DataPageSize);
+	}
+
+#if defined(SUPPORT_MMU_DEFERRED_FREE)
+	eError = PVRSRVGetDevicePowerState(psPhysMemCtx->psDevNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		/* Treat unknown power state as ON. */
+		ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+	}
+
+	if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF)
+	{
+		_FreeMMUMapping(psPhysMemCtx->psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+	}
+	else
+	{
+		_SetupCleanup_FreeMMUMapping(psMMUContext->psPhysMemCtx);
+	}
+#else
+	if (!dllist_is_empty(&psMMUContext->psPhysMemCtx->sTmpMMUMappingHead))
+	{
+		_FreeMMUMapping(psPhysMemCtx->psDevNode, &psPhysMemCtx->sTmpMMUMappingHead);
+	}
+#endif
+	OSLockRelease(psMMUContext->hLock);
+
+	OSLockRelease(psMMUContext->psPhysMemCtx->psCleanupData->hCleanupLock);
+}
+
+PVRSRV_ERROR
+MMU_MapPages(MMU_CONTEXT *psMMUContext,
+             PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+             IMG_DEV_VIRTADDR sDevVAddrBase,
+             PMR *psPMR,
+             IMG_UINT32 ui32PhysPgOffset,
+             IMG_UINT32 ui32MapPageCount,
+             IMG_UINT32 *paui32MapIndices,
+             IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hPriv;
+
+	MMU_Levelx_INFO *psLevel = NULL;
+
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+
+	IMG_UINT32 uiPTEIndex = 0;
+	IMG_UINT32 uiPageSize = (1 << uiLog2HeapPageSize);
+	IMG_UINT32 uiLoop = 0;
+	IMG_UINT32 ui32MappedCount = 0;
+	IMG_DEVMEM_OFFSET_T uiPgOffset = 0;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0, uiDefProtFlags=0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *psDevPAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+	IMG_BOOL *pbValid;
+	IMG_BOOL bValid;
+	IMG_BOOL bScratchBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
+	IMG_BOOL bNeedBacking = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+#if defined(PDUMP)
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
+
+	PDUMPCOMMENT(psMMUContext->psPhysMemCtx->psDevNode, "Wire up Page Table entries to point to the Data Pages (%"IMG_INT64_FMTSPECd" bytes)",
+	             (IMG_UINT64)(ui32MapPageCount * uiPageSize));
+#endif /*PDUMP*/
+
+	/* Validate the most essential parameters */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psMMUContext != NULL, "psMMUContext");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR != NULL, "psPMR");
+
+	psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/* Allocate memory for page-frame-numbers and validity states,
+	   N.B. assert could be triggered by an illegal uiSizeBytes */
+	if (ui32MapPageCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psDevPAddr = OSAllocMem(ui32MapPageCount * sizeof(IMG_DEV_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(psDevPAddr, eError, ErrReturnError);
+
+		pbValid = OSAllocMem(ui32MapPageCount * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrFreePAddrMappingArray);
+	}
+	else
+	{
+		psDevPAddr = asDevPAddr;
+		pbValid	= abValid;
+	}
+
+	/* Get the Device physical addresses of the pages we are trying to map
+	 * In the case of non indexed mapping we can get all addresses at once */
+	if (NULL == paui32MapIndices)
+	{
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2HeapPageSize,
+		                         ui32MapPageCount,
+		                         ((IMG_DEVMEM_OFFSET_T) ui32PhysPgOffset << uiLog2HeapPageSize),
+		                         psDevPAddr,
+		                         pbValid,
+		                         DEVICE_USE);
+		PVR_GOTO_IF_ERROR(eError, ErrFreeValidArray);
+	}
+
+	/*Get the Page table level configuration */
+	_MMU_GetPTConfig(psMMUContext,
+	                 (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig,
+	                 &hPriv,
+	                 &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, ErrPutPTConfig);
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE),
+		                                                                  uiLog2HeapPageSize);
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE));
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrPutPTConfig);
+	}
+
+	if (PMR_IsSparse(psPMR))
+	{
+		/* We know there will not be 4G number of PMR's */
+		bScratchBacking = PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(PMR_Flags(psPMR));
+		if (bScratchBacking)
+		{
+			bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(PMR_Flags(psPMR));
+		}
+	}
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	for (uiLoop = 0; uiLoop < ui32MapPageCount; uiLoop++)
+	{
+
+#if defined(PDUMP)
+		IMG_DEVMEM_OFFSET_T uiNextSymName;
+#endif /*PDUMP*/
+
+		if (NULL != paui32MapIndices)
+		{
+			uiPgOffset = paui32MapIndices[uiLoop];
+
+			/*Calculate the Device Virtual Address of the page */
+			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr + (uiPgOffset * uiPageSize);
+
+			/* Get the physical address to map */
+			eError = PMR_DevPhysAddr(psPMR,
+			                         uiLog2HeapPageSize,
+			                         1,
+			                         uiPgOffset * uiPageSize,
+			                         &sDevPAddr,
+			                         &bValid,
+			                         DEVICE_USE);
+			PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+		}
+		else
+		{
+			uiPgOffset = uiLoop + ui32PhysPgOffset;
+			sDevPAddr = psDevPAddr[uiLoop];
+			bValid = pbValid[uiLoop];
+		}
+
+		uiDefProtFlags = uiProtFlags;
+		/*
+			The default value of the entry is invalid so we don't need to mark
+			it as such if the page wasn't valid, we just advance pass that address
+		 */
+		if (bValid || bScratchBacking)
+		{
+			if (!bValid)
+			{
+				if (bZeroBacking)
+				{
+					eError = _MMU_GetBackingPage(psDevNode,
+					                             &sDevPAddr.uiAddr,
+					                             DEV_ZERO_PAGE);
+					PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+					                      ErrUnlockAndUnmapPages);
+
+					/* Ensure the zero back page PTE is read only */
+					uiDefProtFlags = uiProtFlagsReadOnly;
+				}
+				else
+				{
+					eError = _MMU_GetBackingPage(psDevNode,
+					                             &sDevPAddr.uiAddr,
+					                             SCRATCH_PAGE);
+					PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+					                      ErrUnlockAndUnmapPages);
+				}
+			}
+			else
+			{
+				/* check the physical alignment of the memory to map */
+				PVR_ASSERT((sDevPAddr.uiAddr & (uiPageSize-1)) == 0);
+			}
+
+#if defined(DEBUG)
+			{
+				IMG_INT32	i32FeatureVal = 0;
+				IMG_UINT32 ui32BitLength = FloorLog2(sDevPAddr.uiAddr);
+
+				i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+				do {
+					/* i32FeatureVal can be negative for cases where this feature is undefined
+					 * In that situation we need to bail out than go ahead with debug comparison */
+					if (0 > i32FeatureVal)
+						break;
+
+					if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"%s Failed. The physical address bitlength (%u)"
+								" is greater than the chip can handle (%d).",
+								__func__, ui32BitLength, i32FeatureVal));
+
+						PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto ErrUnlockAndUnmapPages;
+					}
+				} while (0);
+			}
+#endif /*DEBUG*/
+
+#if defined(PDUMP)
+			if (bValid)
+			{
+				eError = PMR_PDumpSymbolicAddr(psPMR, uiPgOffset * uiPageSize,
+				                               sizeof(aszMemspaceName), &aszMemspaceName[0],
+				                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
+				                               &uiSymbolicAddrOffset,
+				                               &uiNextSymName);
+				PVR_ASSERT(eError == PVRSRV_OK);
+			}
+#endif /*PDUMP*/
+
+			psPrevLevel = psLevel;
+			/* Calculate PT index and get new table descriptor */
+			if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+			                    &psLevel, &uiPTEIndex))
+			{
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+				                    ErrUnlockAndUnmapPages);
+			}
+
+			if (psPrevLevel == psLevel)
+			{
+				/*
+				 * Sparse allocations may have page offsets which
+				 * decrement as well as increment, so make sure we
+				 * update the range we will flush correctly.
+				 */
+				if (uiPTEIndex > uiFlushEnd)
+					uiFlushEnd = uiPTEIndex;
+				else if (uiPTEIndex < uiFlushStart)
+					uiFlushStart = uiPTEIndex;
+			}
+			else
+			{
+				/* Flush if we moved to another psLevel, i.e. page table */
+				if (psPrevLevel != NULL)
+				{
+					eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+					                            &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+					                            uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+					                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+					PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+				}
+
+				uiFlushStart = uiPTEIndex;
+				uiFlushEnd = uiFlushStart;
+			}
+
+			HTBLOGK(HTB_SF_MMU_PAGE_OP_MAP,
+			        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+			        HTBLOG_U64_BITS_HIGH(sDevPAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevPAddr.uiAddr));
+
+			/* Set the PT entry with the specified address and protection flags */
+			eError = _SetupPTE(psMMUContext,
+			                   psLevel,
+			                   uiPTEIndex,
+			                   psConfig,
+			                   &sDevPAddr,
+			                   &sDevVAddr,
+			                   IMG_FALSE,
+#if defined(PDUMP)
+			                   (bValid)?aszMemspaceName:(psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName),
+			                          ((bValid)?aszSymbolicAddress:((bZeroBacking)?DEV_ZERO_PAGE_STR:SCRATCH_PAGE_STR)),
+			                          (bValid)?uiSymbolicAddrOffset:0,
+#endif /*PDUMP*/
+			                    uiDefProtFlags);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockAndUnmapPages);
+
+			if (bValid)
+			{
+				PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+				PVR_DPF ((PVR_DBG_MESSAGE,
+						"%s: devVAddr=" IMG_DEV_VIRTADDR_FMTSPEC ", "
+						"size=" IMG_DEVMEM_OFFSET_FMTSPEC,
+						__func__,
+						sDevVAddr.uiAddr,
+						uiPgOffset * uiPageSize));
+
+				ui32MappedCount++;
+			}
+		}
+
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(pbValid);
+		OSFreeMem(psDevPAddr);
+	}
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Wired up %d Page Table entries (out of %d)", ui32MappedCount, ui32MapPageCount);
+#endif /*PDUMP*/
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	if (PMR_IsSparse(psPMR) && PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiMappingFlags))
+	{
+		bNeedBacking = IMG_TRUE;
+	}
+
+	(void) MMU_UnmapPagesUnlocked(psMMUContext,
+	                              (bNeedBacking) ? uiMappingFlags : 0,
+	                              sDevVAddrBase,
+	                              uiLoop,
+	                              paui32MapIndices,
+	                              uiLog2HeapPageSize,
+	                              uiMappingFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+ErrFreeValidArray:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(pbValid);
+	}
+ErrFreePAddrMappingArray:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+ErrReturnError:
+	return eError;
+}
+
+static PVRSRV_ERROR
+MMU_UnmapPagesUnlocked(MMU_CONTEXT *psMMUContext,
+                       PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                       IMG_DEV_VIRTADDR sDevVAddrBase,
+                       IMG_UINT32 ui32PageCount,
+                       IMG_UINT32 *pai32FreeIndices,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiPTEIndex = 0, ui32Loop=0;
+	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+	IMG_HANDLE hPriv;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0, uiMMUReadOnlyProtFlags = 0;
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_DEV_PHYADDR sBackingPgDevPhysAddr;
+	IMG_BOOL bUnmap = IMG_TRUE, bScratchBacking = IMG_FALSE, bZeroBacking = IMG_FALSE;
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+#if defined(PDUMP)
+	const IMG_CHAR *pcBackingPageName = NULL;
+
+	PDUMPCOMMENT(psDevNode,
+	             "Invalidate %d entries in page tables for virtual range: 0x%010"IMG_UINT64_FMTSPECX" to 0x%010"IMG_UINT64_FMTSPECX,
+	             ui32PageCount,
+	             (IMG_UINT64)sDevVAddr.uiAddr,
+	             ((IMG_UINT64)sDevVAddr.uiAddr) + (uiPageSize*ui32PageCount)-1);
+#endif
+
+	PVR_ASSERT(OSLockIsLocked(psMMUContext->hLock));
+
+	bScratchBacking = PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiMemAllocFlags);
+	bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(uiMemAllocFlags);
+
+	if (bZeroBacking)
+	{
+		/* Ensure the zero backing page has been created */
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sBackingPgDevPhysAddr.uiAddr,
+		                             DEV_ZERO_PAGE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_MMU_GetBackingPage (zero)");
+	}
+	else
+	{
+		/* Ensure the scratch backing page has been created */
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sBackingPgDevPhysAddr.uiAddr,
+		                             SCRATCH_PAGE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_MMU_GetBackingPage (scratch)");
+	}
+
+#if defined(PDUMP)
+	if (bScratchBacking)
+	{
+		pcBackingPageName = bZeroBacking ? DEV_ZERO_PAGE_STR : SCRATCH_PAGE_STR;
+	}
+#endif
+
+	bUnmap = (uiMappingFlags)? !bScratchBacking : IMG_TRUE;
+	/* Get PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(bUnmap,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+	uiMMUReadOnlyProtFlags = (uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE) | MMU_PROTFLAGS_READABLE;
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUReadOnlyProtFlags);
+	}
+	else if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUReadOnlyProtFlags, uiLog2PageSize);
+	}
+
+	/* Unmap page by page */
+	while (ui32Loop < ui32PageCount)
+	{
+		if (NULL != pai32FreeIndices)
+		{
+			/*Calculate the Device Virtual Address of the page */
+			sDevVAddr.uiAddr = sDevVAddrBase.uiAddr +
+					pai32FreeIndices[ui32Loop] * (IMG_UINT64) uiPageSize;
+		}
+
+		psPrevLevel = psLevel;
+		/* Calculate PT index and get new table descriptor */
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND, e0);
+		}
+
+		if (psPrevLevel == psLevel)
+		{
+			/*
+			 * Sparse allocations may have page offsets which
+			 * decrement as well as increment, so make sure we
+			 * update the range we will flush correctly.
+			 */
+			if (uiPTEIndex > uiFlushEnd)
+				uiFlushEnd = uiPTEIndex;
+			else if (uiPTEIndex < uiFlushStart)
+				uiFlushStart = uiPTEIndex;
+		}
+		else
+		{
+			/* Flush if we moved to another psLevel, i.e. page table */
+			if (psPrevLevel != NULL)
+			{
+				PVRSRV_ERROR eError;
+				eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+				                   &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+				                   uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+				                   (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+				PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+			}
+
+			uiFlushStart = uiPTEIndex;
+			uiFlushEnd = uiFlushStart;
+		}
+
+		HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
+		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr));
+
+		/* Set the PT entry to invalid and poison it with a bad address */
+		eError = _SetupPTE(psMMUContext,
+		                   psLevel,
+		                   uiPTEIndex,
+		                   psConfig,
+		                   (bScratchBacking)? &sBackingPgDevPhysAddr : &gsBadDevPhyAddr,
+		                   &sDevVAddr,
+		                   bUnmap,
+#if defined(PDUMP)
+		                   (bScratchBacking)? (psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName): NULL,
+		                   pcBackingPageName,
+		                   0U,
+#endif
+		                   (bZeroBacking)? uiProtFlagsReadOnly: uiProtFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Check we haven't wrapped around */
+		PVR_ASSERT(psLevel->ui32RefCount <= psLevel->ui32NumOfEntries);
+		ui32Loop++;
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		PVRSRV_ERROR eError;
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                   &psLevel->sMemDesc.psMapping->sMemHandle,
+		                   uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                   (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+	}
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+e0:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+	PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map/unmap page table "
+	         "with error %u", __func__, eError));
+
+	return eError;
+}
+
+/*
+	MMU_UnmapPages
+ */
+PVRSRV_ERROR
+MMU_UnmapPages(MMU_CONTEXT *psMMUContext,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_DEV_VIRTADDR sDevVAddrBase,
+               IMG_UINT32 ui32PageCount,
+               IMG_UINT32 *pai32FreeIndices,
+               IMG_UINT32 uiLog2PageSize,
+               PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = MMU_UnmapPagesUnlocked(psMMUContext,
+	                                uiMappingFlags,
+	                                sDevVAddrBase,
+	                                ui32PageCount,
+	                                pai32FreeIndices,
+	                                uiLog2PageSize,
+	                                uiMemAllocFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+MMUX_MapVRangeToBackingPage(MMU_CONTEXT *psMMUContext,
+                          PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                          IMG_DEV_VIRTADDR sDevVAddrBase,
+                          IMG_UINT32 ui32MapPageCount,
+                          IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError;
+
+	IMG_UINT32 uiPageSize = (1 << uiLog2HeapPageSize);
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_UINT32 uiFlushEnd = 0, uiFlushStart = 0;
+	IMG_UINT32 uiPTEIndex = 0;
+
+	MMU_Levelx_INFO *psLevel = NULL;
+	MMU_Levelx_INFO *psPrevLevel = NULL;
+
+	IMG_UINT32 uiLoop = 0;
+
+	IMG_UINT64 uiProtFlags = 0, uiProtFlagsReadOnly = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	IMG_BOOL bZeroBacking = PVRSRV_IS_ZERO_BACKING_REQUIRED(uiMappingFlags);
+
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+
+	IMG_DEV_PHYADDR sDevPAddr;
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	/*Get the Page table level configuration */
+	_MMU_GetPTConfig(psMMUContext,
+	                 (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig,
+	                 &hPriv,
+	                 &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, ErrPutPTConfig);
+
+	/* Callback to get device specific protection flags */
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt8((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE),
+		                                                                  uiLog2HeapPageSize);
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+		uiMMUProtFlags |= MMU_PROTFLAGS_READABLE;
+		uiProtFlagsReadOnly = psMMUContext->psDevAttrs->pfnDerivePTEProt4((uiMMUProtFlags & ~MMU_PROTFLAGS_WRITEABLE));
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_INVALID_PARAMS, ErrPutPTConfig);
+	}
+
+	if (bZeroBacking)
+	{
+		eError = _MMU_GetBackingPage(psDevNode,
+		                             &sDevPAddr.uiAddr,
+		                             DEV_ZERO_PAGE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+		                      ErrPutPTConfig);
+	}
+	else
+	{
+		eError = _MMU_GetBackingPage(psDevNode,
+									 &sDevPAddr.uiAddr,
+									 SCRATCH_PAGE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_MMU_GetBackingPage",
+							  ErrPutPTConfig);
+	}
+
+#if defined(DEBUG)
+	{
+		IMG_INT32	i32FeatureVal = 0;
+		IMG_UINT32 ui32BitLength = FloorLog2(sDevPAddr.uiAddr);
+
+		i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+		do {
+			/* i32FeatureVal can be negative for cases where this feature is undefined
+			 * In that situation we need to bail out than go ahead with debug comparison */
+			if (0 > i32FeatureVal)
+				break;
+
+			if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s Failed. The physical address bitlength (%u)"
+						" is greater than the chip can handle (%d).",
+						__func__, ui32BitLength, i32FeatureVal));
+
+				PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto ErrPutPTConfig;
+			}
+		} while (0);
+	}
+#endif /*DEBUG*/
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	for (uiLoop = 0; uiLoop < ui32MapPageCount; uiLoop++)
+	{
+		psPrevLevel = psLevel;
+		/* Calculate PT index and get new table descriptor */
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+			                    ErrUnlockAndUnmapPages);
+		}
+
+		if (psPrevLevel == psLevel)
+		{
+			/*
+			 * Sparse allocations may have page offsets which
+			 * decrement as well as increment, so make sure we
+			 * update the range we will flush correctly.
+			 */
+			if (uiPTEIndex > uiFlushEnd)
+				uiFlushEnd = uiPTEIndex;
+			else if (uiPTEIndex < uiFlushStart)
+				uiFlushStart = uiPTEIndex;
+		}
+		else
+		{
+			/* Flush if we moved to another psLevel, i.e. page table */
+			if (psPrevLevel != NULL)
+			{
+				eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+				                            &psPrevLevel->sMemDesc.psMapping->sMemHandle,
+				                            uiFlushStart * psConfig->uiBytesPerEntry + psPrevLevel->sMemDesc.uiOffset,
+				                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+				PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+			}
+
+			uiFlushStart = uiPTEIndex;
+			uiFlushEnd = uiFlushStart;
+		}
+
+		HTBLOGK(HTB_SF_MMU_PAGE_OP_MAP,
+		        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+		        HTBLOG_U64_BITS_HIGH(sDevPAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevPAddr.uiAddr));
+
+		/* Set the PT entry with the specified address and protection flags */
+		eError = _SetupPTE(psMMUContext,
+		                   psLevel,
+		                   uiPTEIndex,
+		                   psConfig,
+		                   &sDevPAddr,
+		                   &sDevVAddr,
+		                   IMG_FALSE,
+#if defined(PDUMP)
+		                   psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+		                   bZeroBacking ? DEV_ZERO_PAGE_STR : SCRATCH_PAGE_STR,
+		                   0,
+#endif /*PDUMP*/
+		                   bZeroBacking ? uiProtFlagsReadOnly : uiProtFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_SetupPTE", ErrUnlockAndUnmapPages);
+
+		sDevVAddr.uiAddr += uiPageSize;
+	}
+
+	/* Flush the last level we touched */
+	if (psLevel != NULL)
+	{
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiFlushStart * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiFlushEnd+1 - uiFlushStart) * psConfig->uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, ErrUnlockAndUnmapPages);
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	return PVRSRV_OK;
+
+ErrUnlockAndUnmapPages:
+	(void) MMU_UnmapPagesUnlocked(psMMUContext,
+	                              0,
+	                              sDevVAddrBase,
+	                              uiLoop,
+	                              NULL,
+	                              uiLog2HeapPageSize,
+	                              uiMappingFlags);
+
+	OSLockRelease(psMMUContext->hLock);
+ErrPutPTConfig:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+	return eError;
+}
+
+PVRSRV_ERROR
+MMU_MapPMRFast(MMU_CONTEXT *psMMUContext,
+               IMG_DEV_VIRTADDR sDevVAddrBase,
+               const PMR *psPMR,
+               IMG_DEVMEM_SIZE_T uiSizeBytes,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_UINT32 uiLog2HeapPageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_HANDLE hPriv;
+
+	IMG_UINT32 i, uiChunkStart, uiLastPTEIndex, uiNumEntriesToWrite;
+	IMG_UINT32 ui32PagesDone=0, uiPTEIndex=0;
+
+	IMG_UINT8 uiAddrLog2Align, uiAddrShift, uiParityShift;
+	IMG_UINT64 uiAddrMask, uiProtFlags;
+	IMG_UINT32 uiBytesPerEntry;
+	IMG_UINT64 uiParityBit = 0;
+	IMG_BOOL bSetParity = IMG_FALSE;
+	IMG_DEV_VIRTADDR sDevVAddrRunning, sDevVAddrBaseCopy = sDevVAddrBase;
+
+	IMG_UINT64* pui64LevelBase;
+	IMG_UINT32* pui32LevelBase;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	MMU_Levelx_INFO *psLevel = NULL;
+
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 uiNumPages = uiSizeBytes >> uiLog2HeapPageSize;
+
+
+#if defined(PVRSRV_ENABLE_HTB)
+	IMG_BOOL bHTBLog =
+	    HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_PMRMAP) ||
+	    HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_TABLE);
+#endif
+
+	IMG_BOOL bValidateOrTweak = psDevNode->pfnValidateOrTweakPhysAddrs ? IMG_TRUE : IMG_FALSE;
+
+#if defined(PDUMP)
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicAddress[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiSymbolicAddrOffset;
+	IMG_UINT32 ui32MappedCount = 0;
+	IMG_DEV_VIRTADDR sDevVAddrRunningPdump;
+	PDUMPCOMMENT(psDevNode, "Wire up Page Table entries to point to the Data Pages (%d bytes)", uiNumPages << uiLog2HeapPageSize);
+#endif /*PDUMP*/
+
+	PVR_ASSERT (psMMUContext != NULL);
+	PVR_ASSERT (psPMR != NULL);
+	PVR_ASSERT((IMG_DEVMEM_OFFSET_T)uiNumPages << uiLog2HeapPageSize == uiSizeBytes);
+
+	/* Get general PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2HeapPageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_FALSE,
+	                                 uiMappingFlags,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_GOTO_IF_ERROR(eError, put_mmu_context);
+
+	uiAddrLog2Align = psConfig->uiAddrLog2Align;
+	uiAddrShift = psConfig->uiAddrShift;
+	uiAddrMask = psConfig->uiAddrMask;
+	uiBytesPerEntry = psConfig->uiBytesPerEntry;
+
+	bSetParity = psConfig->uiParityBitMask;
+	uiParityShift = psConfig->uiParityBitShift;
+
+	sDevVAddrRunning.uiAddr = sDevVAddrBase.uiAddr;
+#if defined(PDUMP)
+	sDevVAddrRunningPdump.uiAddr = sDevVAddrBase.uiAddr;
+#endif
+	if (uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2HeapPageSize);
+	}
+	else if (uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psConfig->uiBytesPerEntry", eError, PVRSRV_ERROR_MMU_CONFIG_IS_WRONG, put_mmu_context);
+	}
+
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	do
+	{
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddrBase, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND,
+			                    unlock_mmu_context);
+		}
+
+		pui64LevelBase = (IMG_UINT64*)psLevel->sMemDesc.pvCpuVAddr;
+		pui32LevelBase = (IMG_UINT32*)psLevel->sMemDesc.pvCpuVAddr;
+
+		uiLastPTEIndex = MIN(uiPTEIndex + uiNumPages - ui32PagesDone, psDevVAddrConfig->uiNumEntriesPT);
+		uiNumEntriesToWrite = uiLastPTEIndex - uiPTEIndex;
+
+		for (uiChunkStart = 0; uiChunkStart < uiNumEntriesToWrite; uiChunkStart += PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			IMG_UINT32 uiNumPagesInBlock = MIN(uiNumEntriesToWrite - uiChunkStart, PMR_MAX_TRANSLATION_STACK_ALLOC);
+
+			eError = PMR_DevPhysAddr(psPMR,
+			                         uiLog2HeapPageSize,
+			                         uiNumPagesInBlock,
+			                         (IMG_UINT64) (ui32PagesDone + uiChunkStart) << uiLog2HeapPageSize,
+			                         asDevPAddr,
+			                         abValid,
+			                         DEVICE_USE);
+			PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+
+			if (bValidateOrTweak)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					PVRSRV_ERROR eError = psDevNode->pfnValidateOrTweakPhysAddrs(psDevNode,
+					                                                             psMMUContext->psDevAttrs,
+					                                                             &asDevPAddr[i].uiAddr);
+					PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+				}
+			}
+
+#if defined(DEBUG)
+			{
+				IMG_INT32 i32FeatureVal = PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevNode, PHYS_BUS_WIDTH);
+
+				if (i32FeatureVal >= 0)
+				{
+					for (i=0; i<uiNumPagesInBlock; i++)
+					{
+						IMG_UINT32 ui32BitLength = FloorLog2(asDevPAddr[i].uiAddr);
+
+						if (ui32BitLength > (IMG_UINT32) i32FeatureVal)
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+							        "%s Failed. The physical address bitlength (%u)"
+							        " is greater than the chip can handle (%d).",
+							        __func__, ui32BitLength, i32FeatureVal));
+
+							PVR_ASSERT(ui32BitLength <= (IMG_UINT32) i32FeatureVal);
+							PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, unlock_mmu_context);
+						}
+					}
+				}
+			}
+#endif /*DEBUG*/
+
+			if (uiBytesPerEntry == 8)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					if (bSetParity)
+					{
+						uiParityBit = _GetParityBit(sDevVAddrRunning.uiAddr ^ asDevPAddr[i].uiAddr);
+						uiParityBit <<= uiParityShift;
+
+						sDevVAddrRunning.uiAddr += (1 << uiLog2HeapPageSize);
+					}
+
+					pui64LevelBase[uiPTEIndex + uiChunkStart + i] =
+						(((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags | uiParityBit;
+				}
+			}
+			else if (uiBytesPerEntry == 4)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+					/* assert that the result fits into 32 bits before writing
+					it into the 32-bit array with a cast */
+					IMG_UINT64 ui64PxE64 =
+					    (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+					PVR_ASSERT(ui64PxE64 == (ui64PxE64 & 0xffffffffU));
+					PVR_ASSERT(!bSetParity);
+#endif
+
+					pui32LevelBase[uiPTEIndex + uiChunkStart + i] =
+					    (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+				}
+			}
+
+#if defined(PDUMP)
+			for (i=0; i<uiNumPagesInBlock; i++)
+			{
+				IMG_DEVMEM_OFFSET_T uiNextSymName;
+				IMG_UINT64 uiParityBit = psConfig->uiParityBitMask ? _GetParityBit(sDevVAddrRunningPdump.uiAddr ^ 0ULL) : 0;
+
+				eError = PMR_PDumpSymbolicAddr(psPMR, (ui32PagesDone + uiChunkStart + i) << uiLog2HeapPageSize,
+				                               sizeof(aszMemspaceName), &aszMemspaceName[0],
+				                               sizeof(aszSymbolicAddress), &aszSymbolicAddress[0],
+				                               &uiSymbolicAddrOffset,
+				                               &uiNextSymName);
+				PVR_ASSERT(eError == PVRSRV_OK);
+				ui32MappedCount++;
+
+				PDumpMMUDumpPxEntries(psDevNode,
+				                      MMU_LEVEL_1,
+				                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+				                      psLevel->sMemDesc.pvCpuVAddr,
+				                      psLevel->sMemDesc.sDevPAddr,
+				                      uiPTEIndex + uiChunkStart + i,
+				                      1,
+				                      aszMemspaceName,
+				                      aszSymbolicAddress,
+				                      uiSymbolicAddrOffset,
+				                      uiBytesPerEntry,
+				                      uiAddrLog2Align,
+				                      uiAddrShift,
+				                      uiAddrMask,
+				                      psConfig->uiProtMask,
+				                      psConfig->uiValidEnMask,
+				                      0,
+				                      uiParityBit,
+				                      psConfig->uiParityBitShift,
+				                      psConfig->uiParityBitMask,
+				                      psMMUContext->psDevAttrs->eMMUType);
+
+				sDevVAddrRunningPdump.uiAddr += (1 << uiLog2HeapPageSize);
+			}
+#endif /*PDUMP*/
+
+#if defined(PVRSRV_ENABLE_HTB)
+			if (bHTBLog)
+			{
+				for (i=0; i<uiNumPagesInBlock; i++)
+				{
+					IMG_DEV_VIRTADDR sDevVAddr;
+					IMG_UINT64 ui64PxE64 = (((asDevPAddr[i].uiAddr >> uiAddrLog2Align) << uiAddrShift) & uiAddrMask) | uiProtFlags;
+					sDevVAddr.uiAddr = sDevVAddrBase.uiAddr + (ui32PagesDone + uiChunkStart + i)  * (1 << uiLog2HeapPageSize);
+
+					if (bSetParity)
+					{
+						ui64PxE64 |= _GetParityBit(sDevVAddr.uiAddr ^ asDevPAddr[i].uiAddr) << uiParityShift;
+					}
+
+					HTBLOGK(HTB_SF_MMU_PAGE_OP_PMRMAP,
+					        HTBLOG_U64_BITS_HIGH(sDevVAddr.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddr.uiAddr),
+					        HTBLOG_U64_BITS_HIGH(asDevPAddr[i].uiAddr), HTBLOG_U64_BITS_LOW(asDevPAddr[i].uiAddr));
+
+					HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+					        HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+					        uiPTEIndex + uiChunkStart + i, MMU_LEVEL_1,
+					        HTBLOG_U64_BITS_HIGH(ui64PxE64), HTBLOG_U64_BITS_LOW(ui64PxE64),
+					        IMG_FALSE);
+				}
+			}
+#endif
+		}
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            &psLevel->sMemDesc.psMapping->sMemHandle,
+		                            uiPTEIndex * uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+		                            (uiNumEntriesToWrite) * uiBytesPerEntry);
+		PVR_GOTO_IF_ERROR(eError, unlock_mmu_context);
+
+		sDevVAddrBase.uiAddr += uiNumEntriesToWrite * (1 << uiLog2HeapPageSize);
+		ui32PagesDone += uiNumEntriesToWrite;
+
+	} while (ui32PagesDone < uiNumPages);
+
+	OSLockRelease(psMMUContext->hLock);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevNode, "Wired up %d Page Table entries (out of %d)", ui32MappedCount, uiNumPages);
+#endif /*PDUMP*/
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_FALSE);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return PVRSRV_OK;
+
+unlock_mmu_context:
+	/* Unmap starting from the address passed as an argument. */
+	(void) MMU_UnmapPMRFastUnlocked(psMMUContext,
+	                                sDevVAddrBaseCopy,
+	                                uiNumPages,
+	                                uiLog2HeapPageSize);
+	OSLockRelease(psMMUContext->hLock);
+put_mmu_context:
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+MMU_UnmapPMRFastUnlocked(MMU_CONTEXT *psMMUContext,
+                         IMG_DEV_VIRTADDR sDevVAddrBase,
+                         IMG_UINT32 ui32PageCount,
+                         IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiPTEIndex = 0, uiLastPTEIndex = 0, ui32PagesDone=0, i, uiNumEntriesToWrite;
+	IMG_UINT32 uiPageSize = 1 << uiLog2PageSize;
+	MMU_Levelx_INFO *psLevel = NULL;
+	IMG_HANDLE hPriv;
+	void* pvPTStart;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_DEV_VIRTADDR sDevVAddr = sDevVAddrBase;
+	IMG_UINT64 uiProtFlags = 0;
+	MMU_PROTFLAGS_T uiMMUProtFlags = 0;
+	IMG_UINT64 uiEntry = 0;
+
+	IMG_UINT64 uiParityBit = 0;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_DEV_VIRTADDR sDevVAddrStartOfTable;
+	IMG_UINT32 uiParityPatternIdx;
+	IMG_BOOL bSetParity = IMG_FALSE;
+	IMG_DEV_VIRTADDR sDevVAddrRunning;
+	IMG_UINT64 ui64BadPhysAddrParity = 0;
+#endif
+	IMG_UINT64 ui64BadPhysAddr = 0;
+
+#if defined(PVRSRV_ENABLE_HTB)
+	IMG_BOOL bLog;
+#endif
+
+	PVRSRV_DEVICE_NODE *psDevNode = psMMUContext->psPhysMemCtx->psDevNode;
+
+	PVR_ASSERT(OSLockIsLocked(psMMUContext->hLock));
+
+	/* Get PT and address configs */
+	_MMU_GetPTConfig(psMMUContext, (IMG_UINT32) uiLog2PageSize,
+	                 &psConfig, &hPriv, &psDevVAddrConfig);
+
+	eError = _MMU_ConvertDevMemFlags(IMG_TRUE,
+	                                 0,
+	                                 &uiMMUProtFlags,
+	                                 psMMUContext);
+	PVR_RETURN_IF_ERROR(eError);
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	bSetParity = psConfig->uiParityBitMask != 0;
+	sDevVAddrRunning.uiAddr = sDevVAddrBase.uiAddr;
+#endif
+
+	/* Callback to get device specific protection flags */
+
+	if (psConfig->uiBytesPerEntry == 8)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt8(uiMMUProtFlags , uiLog2PageSize);
+		ui64BadPhysAddr = gsBadDevPhyAddr.uiAddr & ~(psConfig->uiProtMask | psConfig->uiParityBitMask);
+		/* Fill the entry with a bad address but leave space for protection flags and parity bit */
+		uiEntry = ui64BadPhysAddr | uiProtFlags;
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		ui64BadPhysAddrParity = _GetParityBit(ui64BadPhysAddr) << psConfig->uiParityBitShift;
+#endif
+	}
+	else if (psConfig->uiBytesPerEntry == 4)
+	{
+		uiProtFlags = psMMUContext->psDevAttrs->pfnDerivePTEProt4(uiMMUProtFlags);
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		PVR_ASSERT(!bSetParity);
+#endif
+		/* Fill the entry with a bad address but leave space for protection flags */
+		uiEntry = (((IMG_UINT32) gsBadDevPhyAddr.uiAddr) & ~psConfig->uiProtMask) | (IMG_UINT32) uiProtFlags;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: The page table entry byte length is not supported",
+				__func__));
+		goto e0;
+	}
+
+#if defined(PVRSRV_ENABLE_HTB)
+	bLog = HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_UNMAP) ||
+		   HTB_GROUP_ENABLED(HTB_SF_MMU_PAGE_OP_TABLE);
+#endif
+
+	do
+	{
+		if (!_MMU_GetPTInfo(psMMUContext, sDevVAddr, psDevVAddrConfig,
+		                    &psLevel, &uiPTEIndex))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_MAPPING_NOT_FOUND, e0);
+		}
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+		sDevVAddrStartOfTable.uiAddr = sDevVAddr.uiAddr - uiPTEIndex * (1 << uiLog2PageSize);
+		uiParityPatternIdx = _GetParityBit(sDevVAddrStartOfTable.uiAddr);
+#endif
+		pvPTStart = psLevel->sMemDesc.pvCpuVAddr;
+
+		uiLastPTEIndex = MIN(uiPTEIndex + ui32PageCount - ui32PagesDone, psDevVAddrConfig->uiNumEntriesPT);
+		uiNumEntriesToWrite = uiLastPTEIndex - uiPTEIndex;
+
+		if (psConfig->uiBytesPerEntry == 8)
+		{
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+				if (bSetParity)
+				{
+					uiParityBit = psMMUContext->psDevAttrs->pui64PrecomputedAllocParity[uiParityPatternIdx][i] ^ ui64BadPhysAddrParity;
+					sDevVAddrRunning.uiAddr += (1 << uiLog2PageSize);
+				}
+#endif
+				((IMG_UINT64*)pvPTStart)[i] = uiEntry | uiParityBit;
+			}
+		}
+		else
+		{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+			PVR_ASSERT(psConfig->uiBytesPerEntry == 4);
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+			PVR_ASSERT(!bSetParity);
+#endif
+#endif
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+				((IMG_UINT32*)pvPTStart)[i] = (IMG_UINT32) uiEntry;
+			}
+		}
+
+#if defined(PVRSRV_ENABLE_HTB)
+		/* Log separately to avoid polluting the fast path */
+		if (bLog)
+		{
+			for (i=uiPTEIndex; i<uiLastPTEIndex; i++)
+			{
+				IMG_DEV_VIRTADDR sDevVAddrToLog;
+				sDevVAddrToLog.uiAddr = sDevVAddr.uiAddr + (i - uiPTEIndex) * uiPageSize;
+
+				/* Log modifications */
+				HTBLOGK(HTB_SF_MMU_PAGE_OP_UNMAP,
+						HTBLOG_U64_BITS_HIGH(sDevVAddrToLog.uiAddr), HTBLOG_U64_BITS_LOW(sDevVAddrToLog.uiAddr));
+
+				HTBLOGK(HTB_SF_MMU_PAGE_OP_TABLE,
+						HTBLOG_PTR_BITS_HIGH(psLevel), HTBLOG_PTR_BITS_LOW(psLevel),
+						uiPTEIndex, MMU_LEVEL_1,
+						HTBLOG_U64_BITS_HIGH(uiEntry), HTBLOG_U64_BITS_LOW(uiEntry),
+						IMG_FALSE);
+			}
+		}
+#endif
+
+#if defined(PDUMP)
+		PDumpMMUDumpPxEntries(psDevNode,
+		                      MMU_LEVEL_1,
+		                      psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName,
+		                      psLevel->sMemDesc.pvCpuVAddr,
+		                      psLevel->sMemDesc.sDevPAddr,
+		                      uiPTEIndex,
+		                      uiNumEntriesToWrite,
+		                      NULL,
+		                      NULL,
+		                      0,
+		                      psConfig->uiBytesPerEntry,
+		                      psConfig->uiAddrLog2Align,
+		                      psConfig->uiAddrShift,
+		                      psConfig->uiAddrMask,
+		                      psConfig->uiProtMask,
+		                      psConfig->uiValidEnMask,
+		                      0,
+		                      0, /* Unused - Parity bit values are taken directly from PTE memory when bad phys addr is used */
+		                      0, /* Unused */
+		                      0, /* Unused */
+		                      psMMUContext->psDevAttrs->eMMUType);
+#endif /*PDUMP*/
+
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+						&psLevel->sMemDesc.psMapping->sMemHandle,
+						uiPTEIndex * psConfig->uiBytesPerEntry + psLevel->sMemDesc.uiOffset,
+						(uiNumEntriesToWrite) * psConfig->uiBytesPerEntry);
+		PVR_LOG_IF_ERROR(eError, "PhysHeapPagesClean");
+
+		sDevVAddr.uiAddr += uiNumEntriesToWrite * uiPageSize;
+		ui32PagesDone += uiNumEntriesToWrite;
+
+	} while (ui32PagesDone < ui32PageCount);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	/* Flush TLB for PTs*/
+	psDevNode->pfnMMUCacheInvalidate(psDevNode,
+	                                 psMMUContext,
+	                                 MMU_LEVEL_1,
+	                                 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+e0:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map/unmap page table with error %u",
+	         __func__, eError));
+
+	return eError;
+}
+
+/*
+    MMU_UnmapPMRFast
+ */
+PVRSRV_ERROR
+MMU_UnmapPMRFast(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrBase,
+                 IMG_UINT32 ui32PageCount,
+                 IMG_UINT32 uiLog2PageSize)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	eError = MMU_UnmapPMRFastUnlocked(psMMUContext,
+                                      sDevVAddrBase,
+                                      ui32PageCount,
+                                      uiLog2PageSize);
+
+	OSLockRelease(psMMUContext->hLock);
+
+	return eError;
+}
+
+/*
+	MMU_AcquireBaseAddr
+ */
+PVRSRV_ERROR
+MMU_AcquireBaseAddr(MMU_CONTEXT *psMMUContext, IMG_DEV_PHYADDR *psPhysAddr)
+{
+	if (!psMMUContext)
+	{
+		psPhysAddr->uiAddr = 0;
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_NOT_SUPPORTED,
+		                        "MMU_AcquireBaseAddr");
+	}
+
+	*psPhysAddr = psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_AcquireCPUBaseAddr
+ */
+PVRSRV_ERROR
+MMU_AcquireCPUBaseAddr(MMU_CONTEXT *psMMUContext, void **ppvCPUVAddr)
+{
+	if (!psMMUContext)
+	{
+		*ppvCPUVAddr = NULL;
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (_MMU_IS_FWKM_CTX_VZGUEST(psMMUContext))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_NOT_SUPPORTED,
+		                        "MMU_AcquireCPUBaseAddr");
+	}
+
+	*ppvCPUVAddr = psMMUContext->sBaseLevelInfo.sMemDesc.pvCpuVAddr;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_ReleaseBaseAddr
+ */
+void
+MMU_ReleaseBaseAddr(MMU_CONTEXT *psMMUContext)
+{
+	PVR_UNREFERENCED_PARAMETER(psMMUContext);
+}
+
+/*
+	MMU_AppendCacheFlags
+*/
+
+void MMU_AppendCacheFlags(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32AppendFlags)
+{
+	PVR_ASSERT(psMMUContext != NULL);
+
+	if (psMMUContext == NULL)
+	{
+		return;
+	}
+
+	OSAtomicOr(&psMMUContext->sCacheFlags, (IMG_INT)ui32AppendFlags);
+}
+
+/*
+	MMU_GetAndResetCacheFlags
+*/
+IMG_UINT32 MMU_GetAndResetCacheFlags(MMU_CONTEXT *psMMUContext)
+{
+	IMG_UINT32 uiFlags;
+
+	PVR_ASSERT(psMMUContext != NULL);
+	if (psMMUContext == NULL)
+	{
+		return 0;
+	}
+
+	uiFlags = (IMG_UINT32) OSAtomicExchange(&psMMUContext->sCacheFlags, 0);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* kick cleanup thread to free all zombie PMRs residing in the device's
+	 * zombie list */
+	if (PMRQueueZombiesForCleanup(psMMUContext->psPhysMemCtx->psDevNode))
+	{
+		BITMASK_SET(uiFlags, RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL | RGXFWIF_MMUCACHEDATA_FLAGS_PT);
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	return uiFlags;
+}
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+/*
+    MMU_GetOSids
+ */
+
+void MMU_GetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 *pui32OSid, IMG_UINT32 *pui32OSidReg, IMG_BOOL *pbOSidAxiProt)
+{
+	*pui32OSid     = psMMUContext->psPhysMemCtx->ui32OSid;
+	*pui32OSidReg  = psMMUContext->psPhysMemCtx->ui32OSidReg;
+	*pbOSidAxiProt = psMMUContext->psPhysMemCtx->bOSidAxiProt;
+}
+
+#endif
+
+static IMG_BOOL _MMUGetPxEFaultLevelData(const MMU_CONTEXT *psMMUContext,
+		                                 IMG_DEV_VIRTADDR *psDevVAddr,
+		                                 const MMU_PxE_CONFIG *psConfig,
+		                                 void *pvCpuVAddr,
+		                                 IMG_UINT32 ui32PxIndex,
+		                                 MMU_LEVEL_DATA *psMMULevelDataOut,
+		                                 IMG_UINT32 *ui32Log2PageSizeOut)
+{
+	static const IMG_CHAR *apszMMUValidStr[4] =  {
+			/*--*/ "not valid",
+			/*-V*/ "valid",
+			/*P-*/ "pending",
+			/*PV*/ "inconsistent (pending and valid)"  };
+	#define _MMU_VALID_STR(_cfg, _entry) \
+		(apszMMUValidStr[((((_entry)&(_cfg->uiPendingEnMask))!=0) << 1)| \
+		                 ((((_entry)&(_cfg->uiValidEnMask))!=0) << 0)])
+	#define _MMU_MASK_VALID_FOR_32BITS(_cfg) \
+		((_cfg->uiPendingEnMask | _cfg->uiValidEnMask) <= 0xFFFFFFFF)
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+
+	if (psConfig->uiBytesPerEntry == 4)
+	{
+		IMG_UINT32 *pui32Ptr = pvCpuVAddr;
+
+		psMMULevelDataOut->ui64Address = pui32Ptr[ui32PxIndex];
+
+		/* Check if we are dealing with a PxE entry where these bits are in the first 32 bits */
+		/* But if we know it is a 4 byte entry, why check this? */
+		if (_MMU_MASK_VALID_FOR_32BITS(psConfig))
+		{
+			psMMULevelDataOut->psDebugStr = _MMU_VALID_STR(psConfig, pui32Ptr[ui32PxIndex] & psConfig->uiProtMask);
+		}
+		else
+		{
+			psMMULevelDataOut->psDebugStr = "";
+			PVR_DPF((PVR_DBG_ERROR, "Invalid %sE masks for 32-bit entry", psConfig->pszPxLevelStr));
+			return IMG_FALSE;
+		}
+
+		if (ui32Log2PageSizeOut != NULL)
+		{
+			if (psDevAttrs->pfnGetPageSizeFromPDE4(pui32Ptr[ui32PxIndex], ui32Log2PageSizeOut) != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from the %sE", psConfig->pszPxLevelStr));
+				return IMG_FALSE;
+			}
+		}
+	}
+	else
+	{
+		IMG_UINT64 *pui64Ptr = pvCpuVAddr;
+
+		psMMULevelDataOut->ui64Address = pui64Ptr[ui32PxIndex];
+		psMMULevelDataOut->psDebugStr  = _MMU_VALID_STR(psConfig, pui64Ptr[ui32PxIndex] & psConfig->uiProtMask);
+
+		if (ui32Log2PageSizeOut != NULL)
+		{
+			/* MMU_VERSION >= 4 */
+			if (psDevAttrs->pfnGetPageSizeFromVirtAddr != NULL)
+			{
+				if (psDevAttrs->pfnGetPageSizeFromVirtAddr(psMMUContext->psPhysMemCtx->psDevNode, *psDevVAddr, ui32Log2PageSizeOut) != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from device VA"));
+					return IMG_FALSE;
+				}
+			}
+			/* MMU_VERSION < 4 */
+			else if (psDevAttrs->pfnGetPageSizeFromPDE8(pui64Ptr[ui32PxIndex], ui32Log2PageSizeOut) != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to get the page size from the %sE", psConfig->pszPxLevelStr));
+				return IMG_FALSE;
+			}
+		}
+	}
+
+	return IMG_TRUE;
+}
+
+/*
+	MMU_CheckFaultAddress
+ */
+void MMU_CheckFaultAddress(MMU_CONTEXT *psMMUContext,
+                           IMG_DEV_VIRTADDR *psDevVAddr,
+                           MMU_FAULT_DATA *psOutFaultData)
+{
+	MMU_DEVICEATTRIBS *psDevAttrs = psMMUContext->psDevAttrs;
+	MMU_LEVEL	eMMULevel = psDevAttrs->psBaseConfig->ePxLevel;
+	const MMU_PxE_CONFIG *psConfig;
+	const MMU_PxE_CONFIG *psMMUPDEConfig;
+	const MMU_PxE_CONFIG *psMMUPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psMMUDevVAddrConfig;
+	IMG_HANDLE hPriv;
+	MMU_Levelx_INFO *psLevel = NULL;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 uiIndex;
+	IMG_UINT32 ui32PCIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32PDIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32PTIndex = 0xFFFFFFFF;
+	IMG_UINT32 ui32Log2PageSize;
+	MMU_FAULT_DATA sMMUFaultData = {0};
+	MMU_LEVEL_DATA *psMMULevelData;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	/*
+		At this point we don't know the page size so assume it's 4K.
+		When we get the PD level (MMU_LEVEL_2) we can check to see
+		if this assumption is correct.
+	 */
+	eError = psDevAttrs->pfnGetPageSizeConfiguration(12,
+	                                                 &psMMUPDEConfig,
+	                                                 &psMMUPTEConfig,
+	                                                 &psMMUDevVAddrConfig,
+	                                                 &hPriv);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("Failed to get the page size info for log2 page sizeof 12"));
+	}
+
+	psLevel = &psMMUContext->sBaseLevelInfo;
+	psConfig = psDevAttrs->psBaseConfig;
+
+	sMMUFaultData.eTopLevel = psDevAttrs->psBaseConfig->ePxLevel;
+	sMMUFaultData.eType = MMU_FAULT_TYPE_NON_PM;
+
+
+	for (; eMMULevel > MMU_LEVEL_0; eMMULevel--)
+	{
+		if (eMMULevel == MMU_LEVEL_3)
+		{
+			/* Determine the PC index */
+			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexMask;
+			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPCIndexShift;
+			ui32PCIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PCIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_3];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PCIndex;
+
+			if (ui32PCIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PCIndex,
+					                        psMMULevelData, NULL);
+
+			psLevel = psLevel->apsNextLevel[ui32PCIndex];
+			if (!psLevel)
+			{
+				break;
+			}
+			psConfig = psMMUPDEConfig;
+			continue; /* continue to the next level */
+		}
+
+
+		if (eMMULevel == MMU_LEVEL_2)
+		{
+			/* Determine the PD index */
+			uiIndex = psDevVAddr->uiAddr & psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexMask;
+			uiIndex = uiIndex >> psDevAttrs->psTopLevelDevVAddrConfig->uiPDIndexShift;
+			ui32PDIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PDIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_2];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PDIndex;
+
+			if (ui32PDIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PDIndex,
+					                        psMMULevelData, &ui32Log2PageSize);
+
+			/*
+				We assumed the page size was 4K, now we have the actual size
+				from the PDE we can confirm if our assumption was correct.
+				Until now it hasn't mattered as the PC and PD are the same
+				regardless of the page size
+			 */
+			if (ui32Log2PageSize != 12)
+			{
+				/* Put the 4K page size data */
+				psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+
+				/* Get the correct size data */
+				eError = psDevAttrs->pfnGetPageSizeConfiguration(ui32Log2PageSize,
+				                                                 &psMMUPDEConfig,
+				                                                 &psMMUPTEConfig,
+				                                                 &psMMUDevVAddrConfig,
+				                                                 &hPriv);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_LOG(("Failed to get the page size info for log2 page sizeof %d", ui32Log2PageSize));
+					break;
+				}
+			}
+
+			psLevel = psLevel->apsNextLevel[ui32PDIndex];
+			if (!psLevel)
+			{
+				break;
+			}
+			psConfig = psMMUPTEConfig;
+			continue; /* continue to the next level */
+		}
+
+
+		if (eMMULevel == MMU_LEVEL_1)
+		{
+			/* Determine the PT index */
+			uiIndex = psDevVAddr->uiAddr & psMMUDevVAddrConfig->uiPTIndexMask;
+			uiIndex = uiIndex >> psMMUDevVAddrConfig->uiPTIndexShift;
+			ui32PTIndex = (IMG_UINT32) uiIndex;
+			PVR_ASSERT(uiIndex == ((IMG_UINT64) ui32PTIndex));
+
+			psMMULevelData = &sMMUFaultData.sLevelData[MMU_LEVEL_1];
+			psMMULevelData->uiBytesPerEntry = psConfig->uiBytesPerEntry;
+			psMMULevelData->ui32Index = ui32PTIndex;
+
+			if (ui32PTIndex >= psLevel->ui32NumOfEntries)
+			{
+				psMMULevelData->ui32NumOfEntries = psLevel->ui32NumOfEntries;
+				break;
+			}
+
+			(void) _MMUGetPxEFaultLevelData(psMMUContext, psDevVAddr, psConfig,
+					                        psLevel->sMemDesc.pvCpuVAddr,
+					                        ui32PTIndex,
+					                        psMMULevelData, NULL);
+			break;
+		}
+
+		PVR_LOG(("Unsupported MMU setup: %d", eMMULevel));
+		break;
+	}
+
+	/* Put the page size data back */
+	psDevAttrs->pfnPutPageSizeConfiguration(hPriv);
+	OSLockRelease(psMMUContext->hLock);
+
+	*psOutFaultData = sMMUFaultData;
+}
+
+static IMG_UINT64 MMU_GetVDevAddrPTE(MMU_CONTEXT *psMMUContext,
+                                     const MMU_PxE_CONFIG *psConfig,
+                                     const MMU_DEVVADDR_CONFIG *psDevVAddrConfig,
+                                     IMG_DEV_VIRTADDR sDevVAddr,
+                                     IMG_BOOL *pbStatusOut)
+{
+	MMU_Levelx_INFO *psLevel = NULL;
+	IMG_UINT32 uiIndex = 0;
+	IMG_BOOL bStatus = IMG_FALSE;
+	IMG_UINT64 ui64Entry = 0;
+
+	OSLockAcquire(psMMUContext->hLock);
+
+	switch (psMMUContext->psDevAttrs->psBaseConfig->ePxLevel)
+	{
+		case MMU_LEVEL_3:
+			uiIndex = _CalcPCEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+			psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];
+			if (psLevel == NULL)
+				break;
+
+			__fallthrough;
+		case MMU_LEVEL_2:
+			uiIndex = _CalcPDEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLevel != NULL)
+				psLevel = psLevel->apsNextLevel[uiIndex];
+			else
+				psLevel = psMMUContext->sBaseLevelInfo.apsNextLevel[uiIndex];
+
+			if (psLevel == NULL)
+				break;
+
+			__fallthrough;
+		case MMU_LEVEL_1:
+			uiIndex = _CalcPTEIdx(sDevVAddr, psDevVAddrConfig, IMG_FALSE);
+
+			if (psLevel == NULL)
+				psLevel = &psMMUContext->sBaseLevelInfo;
+
+			ui64Entry = ((IMG_UINT64 *)psLevel->sMemDesc.pvCpuVAddr)[uiIndex];
+			bStatus = ui64Entry & psConfig->uiValidEnMask;
+
+			break;
+		default:
+			PVR_LOG(("MMU_IsVDevAddrValid: Unsupported MMU setup"));
+			break;
+	}
+
+	OSLockRelease(psMMUContext->hLock);
+
+	*pbStatusOut = bStatus;
+
+	return ui64Entry;
+}
+
+IMG_BOOL MMU_IsVDevAddrValid(MMU_CONTEXT *psMMUContext,
+                             IMG_UINT32 uiLog2PageSize,
+                             IMG_DEV_VIRTADDR sDevVAddr)
+{
+	IMG_BOOL bStatus;
+	const MMU_PxE_CONFIG *psConfig;
+	IMG_HANDLE hPriv;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+
+	_MMU_GetPTConfig(psMMUContext, uiLog2PageSize, &psConfig, &hPriv, &psDevVAddrConfig);
+
+	MMU_GetVDevAddrPTE(psMMUContext,
+	                   psConfig,
+	                   psDevVAddrConfig,
+	                   sDevVAddr,
+	                   &bStatus);
+
+	_MMU_PutPTConfig(psMMUContext, hPriv);
+
+	return bStatus;
+}
+
+#if defined(PDUMP)
+/*
+	MMU_ContextDerivePCPDumpSymAddr
+ */
+PVRSRV_ERROR MMU_ContextDerivePCPDumpSymAddr(MMU_CONTEXT *psMMUContext,
+                                             IMG_CHAR *pszPDumpSymbolicNameBuffer,
+                                             size_t uiPDumpSymbolicNameBufferSize)
+{
+	size_t uiCount;
+	IMG_UINT64 ui64PhysAddr;
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	if (!psMMUContext->sBaseLevelInfo.sMemDesc.bValid)
+	{
+		/* We don't have any allocations.  You're not allowed to ask
+		 * for the page catalogue base address until you've made at
+		 * least one allocation.
+		 */
+		return PVRSRV_ERROR_MMU_API_PROTOCOL_ERROR;
+	}
+
+	ui64PhysAddr = (IMG_UINT64)psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr.uiAddr;
+
+	PVR_ASSERT(uiPDumpSymbolicNameBufferSize >= (IMG_UINT32)(21 + OSStringLength(psDevId->pszPDumpDevName)));
+
+	/* Page table Symbolic Name is formed from page table phys addr
+	   prefixed with MMUPT_. */
+	uiCount = OSSNPrintf(pszPDumpSymbolicNameBuffer,
+	                     uiPDumpSymbolicNameBufferSize,
+	                     ":%s:%s%016"IMG_UINT64_FMTSPECX,
+	                     psDevId->pszPDumpDevName,
+	                     psMMUContext->sBaseLevelInfo.sMemDesc.bValid?"MMUPC_":"XXX",
+	                     ui64PhysAddr);
+
+	if (uiCount + 1 > uiPDumpSymbolicNameBufferSize)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_PDumpWritePageCatBase
+ */
+PVRSRV_ERROR
+MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                          const IMG_CHAR *pszSpaceName,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32WordSize,
+                          IMG_UINT32 ui32AlignShift,
+                          IMG_UINT32 ui32Shift,
+                          PDUMP_FLAGS_T uiPdumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszPageCatBaseSymbolicAddr[100];
+	const IMG_CHAR *pszPDumpDevName = psMMUContext->psDevAttrs->pszMMUPxPDumpMemSpaceName;
+
+	eError = MMU_ContextDerivePCPDumpSymAddr(psMMUContext,
+	                                         &aszPageCatBaseSymbolicAddr[0],
+	                                         sizeof(aszPageCatBaseSymbolicAddr));
+	if (eError == PVRSRV_OK)
+	{
+		eError = PDumpWriteSymbAddress(psMMUContext->psPhysMemCtx->psDevNode,
+		                               pszSpaceName,
+		                               uiOffset,
+		                               aszPageCatBaseSymbolicAddr,
+		                               0, /* offset -- Could be non-zero for var. pgsz */
+		                               pszPDumpDevName,
+		                               ui32WordSize,
+		                               ui32AlignShift,
+		                               ui32Shift,
+		                               uiPdumpFlags | PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	return eError;
+}
+
+/*
+	MMU_AcquirePDumpMMUContext
+ */
+PVRSRV_ERROR MMU_AcquirePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                                        IMG_UINT32 *pui32PDumpMMUContextID,
+                                        IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	if (!psMMUContext->ui32PDumpContextIDRefCount)
+	{
+		PDUMP_MMU_ALLOC_MMUCONTEXT(psMMUContext->psPhysMemCtx->psDevNode,
+		                           psDevId->pszPDumpDevName,
+		                           psMMUContext->sBaseLevelInfo.sMemDesc.sDevPAddr,
+		                           psMMUContext->psDevAttrs->eMMUType,
+		                           &psMMUContext->uiPDumpContextID,
+		                           ui32PDumpFlags);
+	}
+
+	psMMUContext->ui32PDumpContextIDRefCount++;
+	*pui32PDumpMMUContextID = psMMUContext->uiPDumpContextID;
+
+	return PVRSRV_OK;
+}
+
+/*
+	MMU_ReleasePDumpMMUContext
+ */
+PVRSRV_ERROR MMU_ReleasePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+	                                        IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_DEVICE_IDENTIFIER *psDevId = &psMMUContext->psPhysMemCtx->psDevNode->sDevId;
+
+	PVR_ASSERT(psMMUContext->ui32PDumpContextIDRefCount != 0);
+	psMMUContext->ui32PDumpContextIDRefCount--;
+
+	if (psMMUContext->ui32PDumpContextIDRefCount == 0)
+	{
+		PDUMP_MMU_FREE_MMUCONTEXT(psMMUContext->psPhysMemCtx->psDevNode,
+		                          psDevId->pszPDumpDevName,
+		                          psMMUContext->uiPDumpContextID,
+		                          ui32PDumpFlags);
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+PVRSRV_ERROR MMU_CacheInvalidateKick(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                     IMG_UINT32 *puiRequiredSyncValue)
+{
+	IMG_UINT32 uiRequiredSyncValue;
+	PVRSRV_ERROR eError;
+
+	eError = psDeviceNode->pfnMMUCacheInvalidateKick(psDeviceNode, &uiRequiredSyncValue);
+
+	if (puiRequiredSyncValue != NULL)
+	{
+		*puiRequiredSyncValue = uiRequiredSyncValue;
+	}
+
+	return eError;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/******************************************************************************
+ End of file (mmu_common.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/mmu_common.h b/drivers/gpu/drm/img-rogue/mmu_common.h
new file mode 100644
index 000000000000..f88d80a5dda4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/mmu_common.h
@@ -0,0 +1,830 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef MMU_COMMON_H
+#define MMU_COMMON_H
+
+/*
+	The Memory Management Unit (MMU) performs device virtual to physical
+	translation.
+
+	Terminology:
+	 - page catalogue, PC	(optional, 3 tier MMU)
+	 - page directory, PD
+	 - page table, PT (can be variable sized)
+	 - data page, DP (can be variable sized)
+	Note: PD and PC are fixed size and can't be larger than the native
+	      physical (CPU) page size
+	Shifts and AlignShift variables:
+	 - 'xxxShift' represent the number of bits a bitfield is shifted left from bit0
+	 - 'xxxAlignShift' is used to convert a bitfield (based at bit0) into byte units
+	   by applying a bit shift left by 'xxxAlignShift' bits
+*/
+
+/*
+	Device Virtual Address Config:
+
+	Incoming Device Virtual Address is deconstructed into up to 4
+	fields, where the virtual address is up to 64bits:
+	MSB-----------------------------------------------LSB
+	| PC Index:   | PD Index:  | PT Index: | DP offset: |
+	| d bits      | c bits     | b-v bits  |  a+v bits  |
+	-----------------------------------------------------
+	where v is the variable page table modifier, e.g.
+			v == 0 -> 4KB DP
+			v == 2 -> 16KB DP
+			v == 4 -> 64KB DP
+			v == 6 -> 256KB DP
+			v == 8 -> 1MB DP
+			v == 10 -> 4MB DP
+*/
+
+/* services/server/include/ */
+#include "pmr.h"
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv_error.h"
+#include "servicesext.h"
+#include "sync_prim_internal.h"
+
+/*!
+	The level of the MMU
+*/
+typedef enum
+{
+	MMU_LEVEL_0 = 0,    /* Level 0 = Page */
+
+	MMU_LEVEL_1,        /* Level 1 = PT */
+	MMU_LEVEL_2,        /* Level 2 = PD */
+	MMU_LEVEL_3,        /* Level 3 = PC */
+	MMU_LEVEL_LAST
+} MMU_LEVEL;
+
+/* moved after declaration of MMU_LEVEL, as pdump_mmu.h references it */
+#include "pdump_mmu.h"
+
+#define MMU_MAX_LEVEL 3
+
+
+typedef struct _MMU_CONTEXT_ MMU_CONTEXT;
+
+
+/*
+	P(C/D/T) Entry Config:
+
+	MSB-----------------------------------------------LSB
+	| PT Addr:   | variable PT ctrl | protection flags: |
+	| bits c+v   | b bits           | a bits            |
+	-----------------------------------------------------
+	where v is the variable page table modifier and is optional
+*/
+/*!
+	Generic MMU entry description. This is used to describe PC, PD and PT entries.
+*/
+typedef struct _MMU_PxE_CONFIG_
+{
+	MMU_LEVEL		ePxLevel;        /*! MMU Level this config describes */
+	const IMG_CHAR	*pszPxLevelStr;  /*! Px string for this level */
+	IMG_UINT8		uiBytesPerEntry; /*! Size of an entry in bytes */
+
+	IMG_UINT64	uiAddrMask;      /*! Physical address mask */
+	IMG_UINT8	uiAddrShift;     /*! Physical address shift */
+	IMG_UINT8	uiAddrLog2Align; /*! Physical address Log 2 alignment */
+
+	IMG_UINT64	uiVarCtrlMask;   /*! Variable control mask */
+	IMG_UINT8	uiVarCtrlShift;  /*! Variable control shift */
+
+	IMG_UINT64	uiProtMask;      /*! Protection flags mask */
+	IMG_UINT8	uiProtShift;     /*! Protection flags shift */
+	IMG_UINT64	uiPendingEnMask; /*! Entry pending bit mask */
+	IMG_UINT64	uiValidEnMask;   /*! Entry valid bit mask */
+	IMG_UINT8	uiValidEnShift;  /*! Entry valid bit shift */
+	IMG_UINT64	uiParityBitMask;   /*! Entry parity bit mask */
+	IMG_UINT8	uiParityBitShift;  /*! Entry parity bit shift */
+} MMU_PxE_CONFIG;
+
+/*!
+	MMU virtual address split
+*/
+typedef struct _MMU_DEVVADDR_CONFIG_
+{
+	/*! Page catalogue index mask */
+	IMG_UINT64	uiPCIndexMask;
+	/*! Page catalogue index shift */
+	IMG_UINT8	uiPCIndexShift;
+	/*! Total number of PC entries */
+	IMG_UINT32	uiNumEntriesPC;
+
+	/*! Page directory mask */
+	IMG_UINT64	uiPDIndexMask;
+	/*! Page directory shift */
+	IMG_UINT8	uiPDIndexShift;
+	/*! Total number of PD entries */
+	IMG_UINT32	uiNumEntriesPD;
+
+	/*! Page table mask */
+	IMG_UINT64	uiPTIndexMask;
+	/*! Page index shift */
+	IMG_UINT8	uiPTIndexShift;
+	/*! Total number of PT entries */
+	IMG_UINT32	uiNumEntriesPT;
+
+	/*! Page offset mask */
+	IMG_UINT64	uiPageOffsetMask;
+	/*! Page offset shift */
+	IMG_UINT8	uiPageOffsetShift;
+
+	/*! First virtual address mappable for this config */
+	IMG_UINT64	uiOffsetInBytes;
+
+} MMU_DEVVADDR_CONFIG;
+
+/*!
+	MMU device attributes. This structure is the interface between the generic
+	MMU code and the device specific MMU code.
+*/
+typedef struct _MMU_DEVICEATTRIBS_
+{
+	/*! Page and address type */
+	PDUMP_MMU_TYPE eMMUType;
+
+	/*! Name string of the PDUMP memory space to use */
+	IMG_CHAR *pszMMUPxPDumpMemSpaceName;
+
+	/*! Alignment requirement of the top/base object */
+	IMG_UINT32 ui32BaseAlign;
+
+	/*! HW config of the top/base object */
+	struct _MMU_PxE_CONFIG_ *psBaseConfig;
+
+	/*! Address split for the base object */
+	const struct _MMU_DEVVADDR_CONFIG_ *psTopLevelDevVAddrConfig;
+
+	/*! Supported page sizes validation mask */
+	IMG_UINT32 ui32ValidPageSizeMask;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+	IMG_UINT64* pui64PrecomputedAllocParity[2];
+#endif
+
+	/*! Callback for creating protection bits for the page catalogue entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePCEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page catalogue entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePCEProt4)(IMG_UINT32 uiProtFlags);
+	/*! Callback for creating protection bits for the page directory entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePDEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page directory entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePDEProt4)(IMG_UINT32 uiProtFlags);
+	/*! Callback for creating protection bits for the page table entry with 8 byte entry */
+	IMG_UINT64 (*pfnDerivePTEProt8)(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+	/*! Callback for creating protection bits for the page table entry with 4 byte entry */
+	IMG_UINT32 (*pfnDerivePTEProt4)(IMG_UINT32 uiProtFlags);
+
+	/*! Callback for getting the MMU configuration based on the specified page size */
+	PVRSRV_ERROR (*pfnGetPageSizeConfiguration)(IMG_UINT32 ui32DataPageSize,
+												const struct _MMU_PxE_CONFIG_ **ppsMMUPDEConfig,
+												const struct _MMU_PxE_CONFIG_ **ppsMMUPTEConfig,
+												const struct _MMU_DEVVADDR_CONFIG_ **ppsMMUDevVAddrConfig,
+												IMG_HANDLE *phPriv2);
+	/*! Callback for putting the MMU configuration obtained from pfnGetPageSizeConfiguration */
+	PVRSRV_ERROR (*pfnPutPageSizeConfiguration)(IMG_HANDLE hPriv);
+
+	/*! Callback for getting the page size from the PDE for the page table entry with 4 byte entry */
+	PVRSRV_ERROR (*pfnGetPageSizeFromPDE4)(IMG_UINT32, IMG_UINT32 *);
+	/*! Callback for getting the page size from the PDE for the page table entry with 8 byte entry */
+	PVRSRV_ERROR (*pfnGetPageSizeFromPDE8)(IMG_UINT64, IMG_UINT32 *);
+	/*! Callback for getting the page size directly from the address. Supported on MMU4 */
+	PVRSRV_ERROR (*pfnGetPageSizeFromVirtAddr)(struct _PVRSRV_DEVICE_NODE_ *psDevNode, IMG_DEV_VIRTADDR, IMG_UINT32 *);
+
+	/*! Private data handle */
+	IMG_HANDLE hGetPageSizeFnPriv;
+} MMU_DEVICEATTRIBS;
+
+
+/* MMU Protection flags */
+
+/* These are specified generically and in a h/w independent way, and
+   are interpreted at each level (PC/PD/PT) separately. */
+
+/* The following flags are for internal use only, and should not
+   traverse the API */
+#define MMU_PROTFLAGS_INVALID 0x80000000U
+
+typedef IMG_UINT32 MMU_PROTFLAGS_T;
+
+/* The following flags should be supplied by the caller: */
+#define MMU_PROTFLAGS_READABLE					(1U<<0)
+#define MMU_PROTFLAGS_WRITEABLE					(1U<<1)
+#define MMU_PROTFLAGS_CACHE_COHERENT			(1U<<2)
+#define MMU_PROTFLAGS_CACHED					(1U<<3)
+
+/* Device specific flags*/
+#define MMU_PROTFLAGS_DEVICE_OFFSET		16
+#define MMU_PROTFLAGS_DEVICE_MASK		0x000f0000UL
+#define MMU_PROTFLAGS_DEVICE(n)	\
+			(((n) << MMU_PROTFLAGS_DEVICE_OFFSET) & \
+			MMU_PROTFLAGS_DEVICE_MASK)
+
+
+struct _PVRSRV_DEVICE_NODE_;
+
+struct _CONNECTION_DATA_;
+
+typedef struct _MMU_PAGESIZECONFIG_
+{
+	const MMU_PxE_CONFIG *psPDEConfig;
+	const MMU_PxE_CONFIG *psPTEConfig;
+	const MMU_DEVVADDR_CONFIG *psDevVAddrConfig;
+	IMG_UINT32 uiRefCount;
+	IMG_UINT32 uiMaxRefCount;
+} MMU_PAGESIZECONFIG;
+
+/*************************************************************************/ /*!
+@Function       MMU_InitDevice
+
+@Description    Creates MMU device specific resources.
+
+@Input          psDevNode               Device node of the device to create the
+                                        MMU context for
+
+@Return         PVRSRV_OK if the initialisation process was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_InitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       MMU_DeInitDevice
+
+@Description    Clean-up MMU device specific resources.
+
+@Input          psDevNode               Device node of the device
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_DeInitDevice(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextCreate
+
+@Description    Create a new MMU context
+
+@Input          psConnection            Connection requesting the MMU context
+                                        creation. Can be NULL for kernel/FW
+                                        memory context.
+@Input          psDevNode               Device node of the device to create the
+                                        MMU context for
+@Output         ppsMMUContext           The created MMU context
+
+@Return         PVRSRV_OK if the MMU context was successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ContextCreate(struct _CONNECTION_DATA_ *psConnection,
+                  struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                  MMU_CONTEXT **ppsMMUContext,
+                  MMU_DEVICEATTRIBS *psDevAttrs);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextDestroy
+
+@Description    Destroy a MMU context
+
+@Input          psMMUContext            MMU context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+MMU_ContextDestroy(MMU_CONTEXT *psMMUContext);
+
+/*************************************************************************/ /*!
+@Function       MMU_Alloc
+
+@Description    Allocate the page tables required for the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uSize                   The size of the allocation
+
+@Input          uiProtFlags             Generic MMU protection flags
+
+@Input          uDevVAddrAlignment      Alignment requirement of the virtual
+                                        allocation
+
+@Input          psDevVAddr              Virtual address to start the allocation
+                                        from
+
+@Return         PVRSRV_OK if the allocation of the page tables was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_Alloc(MMU_CONTEXT *psMMUContext,
+          IMG_DEVMEM_SIZE_T uSize,
+          IMG_UINT32 uiProtFlags,
+          IMG_DEVMEM_SIZE_T uDevVAddrAlignment,
+          IMG_DEV_VIRTADDR *psDevVAddr,
+          IMG_UINT32 uiLog2PageSize);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_Free
+
+@Description    Free the page tables of the specified virtual range
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddr               Virtual address to start the free
+                                        from
+
+@Input          uiSize                  The size of the allocation
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+MMU_Free(MMU_CONTEXT *psMMUContext,
+         IMG_DEV_VIRTADDR sDevVAddr,
+         IMG_DEVMEM_SIZE_T uiSize,
+         IMG_UINT32 uiLog2DataPageSize);
+
+
+/*************************************************************************/ /*!
+@Function       MMU_MapPages
+
+@Description    Map pages to the MMU.
+                Two modes of operation: One requires a list of physical page
+                indices that are going to be mapped, the other just takes
+                the PMR and a possible offset to map parts of it.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+
+@Input          psPMR                   PMR to map
+
+@Input          ui32PhysPgOffset        Physical offset into the PMR
+
+@Input          ui32MapPageCount        Number of pages to map
+
+@Input          paui32MapIndices        List of page indices to map,
+                                         can be NULL
+
+@Input          uiLog2PageSize          Log2 page size of the pages to map
+
+@Return         PVRSRV_OK if the mapping was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_MapPages(MMU_CONTEXT *psMMUContext,
+             PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+             IMG_DEV_VIRTADDR sDevVAddrBase,
+             PMR *psPMR,
+             IMG_UINT32 ui32PhysPgOffset,
+             IMG_UINT32 ui32MapPageCount,
+             IMG_UINT32 *paui32MapIndices,
+             IMG_UINT32 uiLog2PageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_UnmapPages
+
+@Description    Unmap pages from the MMU.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Input          sDevVAddr               Device virtual address of the 1st page
+
+@Input          ui32PageCount           Number of pages to unmap
+
+@Input          pai32UnmapIndicies      Array of page indices to be unmapped
+
+@Input          uiLog2PageSize          log2 size of the page
+
+
+@Input          uiMemAllocFlags         Indicates if the unmapped regions need
+                                        to be backed by dummy or zero page
+
+@Return         PVRSRV_OK if the unmap operation was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_UnmapPages(MMU_CONTEXT *psMMUContext,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_DEV_VIRTADDR sDevVAddr,
+               IMG_UINT32 ui32PageCount,
+               IMG_UINT32 *pai32UnmapIndicies,
+               IMG_UINT32 uiLog2PageSize,
+               PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags);
+
+/*************************************************************************/ /*!
+@Function       MMUX_MapVRangeToBackingPage
+
+@Description    Map virtual range to a backing page in the MMU.
+                Used in DevmemX calls which don't tie a PMR to a virtual
+                range implicitly.
+
+@Input          psMMUContext            MMU context to operate on
+@Input          uiMappingFlags          Memalloc flags for the mapping
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+@Input          ui32MapPageCount        Number of pages to map
+@Input          uiLog2PageSize          Log2 page size of the pages to map
+
+@Return         PVRSRV_OK if the mapping was successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMUX_MapVRangeToBackingPage(MMU_CONTEXT *psMMUContext,
+                            PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+                            IMG_DEV_VIRTADDR sDevVAddrBase,
+                            IMG_UINT32 ui32MapPageCount,
+                            IMG_UINT32 uiLog2HeapPageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_MapPMRFast
+
+@Description    Map a PMR into the MMU. Must be not sparse.
+                This is supposed to cover most mappings and, as the name suggests,
+                should be as fast as possible.
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddr               Device virtual address to map the PMR
+                                        into
+
+@Input          psPMR                   PMR to map
+
+@Input          uiSizeBytes             Size in bytes to map
+
+@Input          uiMappingFlags          Memalloc flags for the mapping
+
+@Return         PVRSRV_OK if the PMR was successfully mapped
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_MapPMRFast(MMU_CONTEXT *psMMUContext,
+               IMG_DEV_VIRTADDR sDevVAddr,
+               const PMR *psPMR,
+               IMG_DEVMEM_SIZE_T uiSizeBytes,
+               PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+               IMG_UINT32 uiLog2PageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_UnmapPMRFast
+
+@Description    Unmap pages from the MMU as fast as possible.
+                PMR must be non-sparse!
+
+@Input          psMMUContext            MMU context to operate on
+
+@Input          sDevVAddrBase           Device virtual address of the 1st page
+
+@Input          ui32PageCount           Number of pages to unmap
+
+@Input          uiLog2PageSize          log2 size of the page
+
+@Return         PVRSRV_OK if the PMR was successfully unmapped
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_UnmapPMRFast(MMU_CONTEXT *psMMUContext,
+                 IMG_DEV_VIRTADDR sDevVAddrBase,
+                 IMG_UINT32 ui32PageCount,
+                 IMG_UINT32 uiLog2PageSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquireBaseAddr
+
+@Description    Acquire the device physical address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         psPhysAddr              Device physical address of the base level
+                                        MMU object
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquireBaseAddr(MMU_CONTEXT *psMMUContext, IMG_DEV_PHYADDR *psPhysAddr);
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquireCPUBaseAddr
+
+@Description    Acquire the CPU Virtual Address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         ppvCPUVAddr             CPU Virtual Address of the base level
+                                        MMU object
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquireCPUBaseAddr(MMU_CONTEXT *psMMUContext, void **ppvCPUVAddr);
+
+/*************************************************************************/ /*!
+@Function       MMU_ReleaseBaseAddr
+
+@Description    Release the device physical address of the base level MMU object
+
+@Input          psMMUContext            MMU context to operate on
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+void
+MMU_ReleaseBaseAddr(MMU_CONTEXT *psMMUContext);
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+/***********************************************************************************/ /*!
+@Function       MMU_SetOSid
+
+@Description    Set the OSid associated with the application (and the MMU Context)
+
+@Input          psMMUContext            MMU context to store the OSid on
+
+@Input          ui32OSid                the OSid in question
+
+@Input          ui32OSidReg             The value that the firmware will assign to the
+                                        registers.
+
+@Input          bOSidAxiProt            Toggles whether the AXI prot bit will be set or
+                                        not.
+@Return None
+*/
+/***********************************************************************************/
+
+void MMU_SetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32OSid,
+                  IMG_UINT32 ui32OSidReg, IMG_BOOL bOSidAxiProt);
+
+/***********************************************************************************/ /*!
+@Function       MMU_GetOSid
+
+@Description    Retrieve the OSid associated with the MMU context.
+
+@Input          psMMUContext            MMU context in which the OSid is stored
+
+@Output         pui32OSid               The OSid in question
+
+@Output         pui32OSidReg            The OSid that the firmware will assign to the
+                                        registers.
+
+@Output         pbOSidAxiProt           Toggles whether the AXI prot bit will be set or
+                                        not.
+@Return None
+*/
+/***********************************************************************************/
+
+void MMU_GetOSids(MMU_CONTEXT *psMMUContext, IMG_UINT32 * pui32OSid,
+                  IMG_UINT32 * pui32OSidReg, IMG_BOOL *pbOSidAxiProt);
+#endif
+
+/*************************************************************************/ /*!
+@Function       MMU_AppendCacheFlags
+
+@Description    Set the cache flags to the bitwise or of themselves and the
+                specified input flags, i.e. ui32CacheFlags |= ui32NewCacheFlags,
+                atomically.
+
+@Input          psMMUContext            MMU context
+
+@Input          ui32NewCacheFlags       Cache flags to append.
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_AppendCacheFlags(MMU_CONTEXT *psMMUContext, IMG_UINT32 ui32NewCacheFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_GetAndResetCacheFlags
+
+@Description    Clears MMU context flags, atomically.
+
+@Input          psMMUContext            MMU context
+
+@Return         Previous MMU context cache flags.
+*/
+/*****************************************************************************/
+IMG_UINT32 MMU_GetAndResetCacheFlags(MMU_CONTEXT *psMMUContext);
+
+typedef struct _MMU_LEVEL_DATA_
+{
+	IMG_UINT32	ui32Index;
+	IMG_UINT32	ui32NumOfEntries;
+	const IMG_CHAR *psDebugStr;
+	IMG_UINT8	uiBytesPerEntry;
+	IMG_UINT64	ui64Address;
+} MMU_LEVEL_DATA;
+
+typedef enum _MMU_FAULT_TYPE_
+{
+	MMU_FAULT_TYPE_UNKNOWN = 0, /* If fault is not analysed by Host */
+	MMU_FAULT_TYPE_PM,
+	MMU_FAULT_TYPE_NON_PM,
+} MMU_FAULT_TYPE;
+
+typedef struct _MMU_FAULT_DATA_
+{
+	MMU_LEVEL	eTopLevel;
+	MMU_FAULT_TYPE	eType;
+	MMU_LEVEL_DATA	sLevelData[MMU_LEVEL_LAST];
+} MMU_FAULT_DATA;
+
+/*************************************************************************/ /*!
+@Function       MMU_CheckFaultAddress
+
+@Description    Check the specified MMU context to see if the provided address
+                should be valid
+
+@Input          psMMUContext            MMU context to store the data on
+
+@Input          psDevVAddr              Address to check
+
+@Output          psOutFaultData          To store fault details after checking
+
+@Return         None
+*/
+/*****************************************************************************/
+void MMU_CheckFaultAddress(MMU_CONTEXT *psMMUContext,
+                           IMG_DEV_VIRTADDR *psDevVAddr,
+                           MMU_FAULT_DATA *psOutFaultData);
+
+/*************************************************************************/ /*!
+@Function       MMU_IsVDevAddrValid
+@Description    Checks if given address is valid.
+@Input          psMMUContext MMU context to store the data on
+@Input          uiLog2PageSize page size
+@Input          sDevVAddr Address to check
+@Return         IMG_TRUE of address is valid
+*/ /**************************************************************************/
+IMG_BOOL MMU_IsVDevAddrValid(MMU_CONTEXT *psMMUContext,
+                             IMG_UINT32 uiLog2PageSize,
+                             IMG_DEV_VIRTADDR sDevVAddr);
+
+#if defined(PDUMP)
+
+/*************************************************************************/ /*!
+@Function       MMU_ContextDerivePCPDumpSymAddr
+
+@Description    Derives a PDump Symbolic address for the top level MMU object
+
+@Input          psMMUContext                    MMU context to operate on
+
+@Input          pszPDumpSymbolicNameBuffer      Buffer to write the PDump symbolic
+                                                address to
+
+@Input          uiPDumpSymbolicNameBufferSize   Size of the buffer
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ContextDerivePCPDumpSymAddr(MMU_CONTEXT *psMMUContext,
+                                IMG_CHAR *pszPDumpSymbolicNameBuffer,
+                                size_t uiPDumpSymbolicNameBufferSize);
+
+/*************************************************************************/ /*!
+@Function       MMU_PDumpWritePageCatBase
+
+@Description    PDump write of the top level MMU object to a device register
+
+@Input          psMMUContext        MMU context to operate on
+
+@Input          pszSpaceName        PDump name of the mem/reg space
+
+@Input          uiOffset            Offset to write the address to
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                                       const IMG_CHAR *pszSpaceName,
+                                       IMG_DEVMEM_OFFSET_T uiOffset,
+                                       IMG_UINT32 ui32WordSize,
+                                       IMG_UINT32 ui32AlignShift,
+                                       IMG_UINT32 ui32Shift,
+                                       PDUMP_FLAGS_T uiPdumpFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_AcquirePDumpMMUContext
+
+@Description    Acquire a reference to the PDump MMU context for this MMU
+                context
+
+@Input          psMMUContext            MMU context to operate on
+
+@Output         pui32PDumpMMUContextID  PDump MMU context ID
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_AcquirePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                           IMG_UINT32 *pui32PDumpMMUContextID,
+                           IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       MMU_ReleasePDumpMMUContext
+
+@Description    Release a reference to the PDump MMU context for this MMU context
+
+@Input          psMMUContext            MMU context to operate on
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+MMU_ReleasePDumpMMUContext(MMU_CONTEXT *psMMUContext,
+                           IMG_UINT32 ui32PDumpFlags);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(MMU_PDumpWritePageCatBase)
+#endif
+static INLINE void
+MMU_PDumpWritePageCatBase(MMU_CONTEXT *psMMUContext,
+                          const IMG_CHAR *pszSpaceName,
+                          IMG_DEVMEM_OFFSET_T uiOffset,
+                          IMG_UINT32 ui32WordSize,
+                          IMG_UINT32 ui32AlignShift,
+                          IMG_UINT32 ui32Shift,
+                          PDUMP_FLAGS_T uiPdumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psMMUContext);
+	PVR_UNREFERENCED_PARAMETER(pszSpaceName);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32WordSize);
+	PVR_UNREFERENCED_PARAMETER(ui32AlignShift);
+	PVR_UNREFERENCED_PARAMETER(ui32Shift);
+	PVR_UNREFERENCED_PARAMETER(uiPdumpFlags);
+}
+#endif /* PDUMP */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*************************************************************************/ /*!
+@Function       MMU_CacheInvalidateKick
+
+@Description    Kicks the Firmware to invalidate caches
+
+@Input          psDeviceNode            Pointer to the device node
+@Output         puiRequiredSyncValue    Value the associated sync prim will be
+                                        updated to after the kick is finished
+                                        (parameter ignored if NULL)
+
+@Return         PVRSRV_OK if successful
+*/
+/*****************************************************************************/
+PVRSRV_ERROR MMU_CacheInvalidateKick(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                     IMG_UINT32 *puiRequiredSyncValue);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#endif /* #ifdef MMU_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/module_common.c b/drivers/gpu/drm/img-rogue/module_common.c
new file mode 100644
index 000000000000..8df0593db286
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/module_common.c
@@ -0,0 +1,787 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common Linux module setup
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+
+#if defined(CONFIG_DEBUG_FS)
+#include "pvr_debugfs.h"
+#endif /* defined(CONFIG_DEBUG_FS) */
+#if defined(CONFIG_PROC_FS)
+#include "pvr_procfs.h"
+#endif /* defined(CONFIG_PROC_FS) */
+#include "di_server.h"
+#include "private_data.h"
+#include "linkage.h"
+#include "power.h"
+#include "env_connection.h"
+#include "process_stats.h"
+#include "module_common.h"
+#include "pvrsrv.h"
+#include "srvcore.h"
+#if defined(SUPPORT_RGX)
+#include "rgxdevice.h"
+#endif
+#include "pvrsrv_error.h"
+#include "pvr_drv.h"
+#include "pvr_bridge_k.h"
+
+#include "pvr_fence.h"
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+#include "pvr_sync.h"
+#if !defined(USE_PVRSYNC_DEVNODE)
+#include "pvr_sync_ioctl_drm.h"
+#endif
+#endif
+
+#include "ospvr_gputrace.h"
+
+#include "km_apphint.h"
+#include "srvinit.h"
+
+#include "pvr_ion_stats.h"
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+#include "physmem_osmem_linux.h"
+#endif
+#endif
+
+#include "dkf_server.h"
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+/* Display class interface */
+#include "kerneldisplay.h"
+EXPORT_SYMBOL(DCRegisterDevice);
+EXPORT_SYMBOL(DCUnregisterDevice);
+EXPORT_SYMBOL(DCDisplayConfigurationRetired);
+EXPORT_SYMBOL(DCDisplayHasPendingCommand);
+EXPORT_SYMBOL(DCImportBufferAcquire);
+EXPORT_SYMBOL(DCImportBufferRelease);
+
+EXPORT_SYMBOL(PVRSRVGetDriverStatus);
+EXPORT_SYMBOL(PVRSRVSystemInstallDeviceLISR);
+EXPORT_SYMBOL(PVRSRVSystemUninstallDeviceLISR);
+
+#include "pvr_notifier.h"
+EXPORT_SYMBOL(PVRSRVCheckStatus);
+#endif /* defined(SUPPORT_DISPLAY_CLASS) */
+
+#if defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+/*
+ * Physmem interface.
+ * Required by LMA DC drivers, and some non-DC LMA display drivers.
+ */
+#include "physheap.h"
+EXPORT_SYMBOL(PhysHeapAcquireByID);
+EXPORT_SYMBOL(PhysHeapRelease);
+EXPORT_SYMBOL(PhysHeapGetType);
+EXPORT_SYMBOL(PhysHeapGetCpuPAddr);
+EXPORT_SYMBOL(PhysHeapGetSize);
+EXPORT_SYMBOL(PhysHeapCpuPAddrToDevPAddr);
+
+#include "pvr_debug.h"
+EXPORT_SYMBOL(PVRSRVGetErrorString);
+EXPORT_SYMBOL(PVRSRVGetDeviceInstance);
+#endif
+
+#if defined(SUPPORT_RGX)
+#include "rgxapi_km.h"
+EXPORT_SYMBOL(RGXHWPerfConnect);
+EXPORT_SYMBOL(RGXHWPerfDisconnect);
+EXPORT_SYMBOL(RGXHWPerfControl);
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+EXPORT_SYMBOL(RGXHWPerfConfigureCounters);
+#else
+EXPORT_SYMBOL(RGXHWPerfConfigMuxCounters);
+EXPORT_SYMBOL(RGXHWPerfConfigureAndEnableCustomCounters);
+#endif
+EXPORT_SYMBOL(RGXHWPerfDisableCounters);
+EXPORT_SYMBOL(RGXHWPerfAcquireEvents);
+EXPORT_SYMBOL(RGXHWPerfReleaseEvents);
+EXPORT_SYMBOL(RGXHWPerfConvertCRTimeStamp);
+#if defined(SUPPORT_KERNEL_HWPERF_TEST)
+EXPORT_SYMBOL(OSAddTimer);
+EXPORT_SYMBOL(OSEnableTimer);
+EXPORT_SYMBOL(OSDisableTimer);
+EXPORT_SYMBOL(OSRemoveTimer);
+#endif
+#endif
+
+static int PVRSRVDeviceSyncOpen(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                                struct drm_file *psDRMFile);
+
+CONNECTION_DATA *LinuxServicesConnectionFromFile(struct file *pFile)
+{
+	if (pFile)
+	{
+		struct drm_file *psDRMFile;
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+
+		psDRMFile = pFile->private_data;
+		PVR_LOG_RETURN_IF_FALSE(psDRMFile != NULL, "psDRMFile is NULL", NULL);
+
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+		PVR_LOG_RETURN_IF_FALSE(psConnectionPriv != NULL, "psConnectionPriv is NULL", NULL);
+
+		return (CONNECTION_DATA*)psConnectionPriv->pvConnectionData;
+	}
+
+	return NULL;
+}
+
+CONNECTION_DATA *LinuxSyncConnectionFromFile(struct file *pFile)
+{
+	if (pFile)
+	{
+		struct drm_file *psDRMFile = pFile->private_data;
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+		return (CONNECTION_DATA*)psConnectionPriv->pvConnectionData;
+#else
+		return (CONNECTION_DATA*)psConnectionPriv->pvSyncConnectionData;
+#endif
+	}
+
+	return NULL;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDriverInit
+@Description  Common one time driver initialisation
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDriverInit(void)
+{
+	PVRSRV_ERROR error;
+	int os_err;
+
+	error = PVROSFuncInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	error = PVRDKFInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+	error = PVRSRVCommonDriverInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	error = pvr_sync_register_functions();
+	if (error != PVRSRV_OK)
+	{
+		return -EPERM;
+	}
+
+	os_err = pvr_sync_init();
+	if (os_err != 0)
+	{
+		return os_err;
+	}
+#endif
+
+	os_err = pvr_apphint_init();
+	if (os_err != 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed AppHint setup(%d)", __func__,
+			     os_err));
+	}
+
+#if defined(SUPPORT_RGX)
+	error = PVRGpuTraceSupportInit();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif
+
+#if defined(ANDROID)
+#if defined(CONFIG_PROC_FS)
+	error = PVRProcFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#elif defined(CONFIG_DEBUG_FS)
+	error = PVRDebugFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif /* defined(CONFIG_PROC_FS) || defined(CONFIG_DEBUG_FS) */
+#else
+#if defined(CONFIG_DEBUG_FS)
+	error = PVRDebugFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#elif defined(CONFIG_PROC_FS)
+	error = PVRProcFsRegister();
+	if (error != PVRSRV_OK)
+	{
+		return -ENOMEM;
+	}
+#endif /* defined(CONFIG_DEBUG_FS) || defined(CONFIG_PROC_FS) */
+#endif /* defined(ANDROID) */
+
+	error = PVRSRVIonStatsInitialise();
+	if (error != PVRSRV_OK)
+	{
+		return -ENODEV;
+	}
+
+#if defined(SUPPORT_RGX)
+	/* calling here because we need to handle input from the file even
+	 * before the devices are initialised
+	 * note: we're not passing a device node because apphint callbacks don't
+	 * need it */
+	PVRGpuTraceInitAppHintCallbacks(NULL);
+#endif
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDriverDeinit
+@Description  Common one time driver de-initialisation
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDriverDeinit(void)
+{
+	pvr_apphint_deinit();
+
+	PVRSRVIonStatsDestroy();
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	pvr_sync_deinit();
+#endif
+
+#if defined(SUPPORT_RGX)
+	PVRGpuTraceSupportDeInit();
+#endif
+
+	PVRSRVCommonDriverDeInit();
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	PVRDKFDeInit();
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+	PVROSFuncDeInit();
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceInit
+@Description  Common device related initialisation.
+@Input        psDeviceNode  The device node for which initialisation should be
+                            performed
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	{
+		PVRSRV_ERROR eError = pvr_sync_device_init(psDeviceNode->psDevConfig->pvOSDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: unable to create sync (%d)",
+					 __func__, eError));
+			return -EBUSY;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_RGX)
+	{
+		int error = PVRGpuTraceInitDevice(psDeviceNode);
+		if (error != 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+				 "%s: failed to initialise PVR GPU Tracing on device%d (%d)",
+				 __func__, psDeviceNode->sDevId.i32KernelDeviceID, error));
+		}
+	}
+#endif
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceDeinit
+@Description  Common device related de-initialisation.
+@Input        psDeviceNode  The device node for which de-initialisation should
+                            be performed
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDeviceDeinit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(SUPPORT_RGX)
+	PVRGpuTraceDeInitDevice(psDeviceNode);
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	pvr_sync_device_deinit(psDeviceNode->psDevConfig->pvOSDevice);
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	pvr_fence_cleanup();
+#endif
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceShutdown
+@Description  Common device shutdown.
+@Input        psDev  The device node representing the device that should
+                     be shutdown
+@Return       void
+*/ /***************************************************************************/
+
+void PVRSRVDeviceShutdown(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	/* Since this is a shutdown request ignore the returned error and try to
+	 * to power off the device. This is done because there is no way of
+	 * signalling the OS that this call failed. */
+	(void) LinuxBridgeBlockClientsAccess(psDevPriv, IMG_TRUE);
+
+	/* Passing PVRSRV_POWER_FLAGS_NONE as there are no special actions required
+	 * from the shutdown call beside the regular device power off. */
+	(void) PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+	                                       PVRSRV_SYS_POWER_STATE_OFF,
+	                                       PVRSRV_POWER_FLAGS_NONE);
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceSuspend
+@Description  Common device suspend.
+@Input        psDev  The device node representing the device that should
+                     be suspended
+@Return       int    0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceSuspend(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+	PVRSRV_ERROR eError;
+
+	/* LinuxBridgeBlockClientsAccess prevents processes from using the driver
+	 * while it's suspended (this is needed for Android). */
+	eError = LinuxBridgeBlockClientsAccess(psDevPriv, IMG_FALSE);
+	PVR_LOG_RETURN_IF_FALSE_VA(eError == PVRSRV_OK, -EFAULT,
+	                           "LinuxBridgeBlockClientsAccess() failed with error %u",
+	                           eError);
+
+#if defined(SUPPORT_AUTOVZ)
+	/* To allow the driver to power down the GPU under AutoVz, the firmware must
+	 * declared as offline, otherwise all power requests will be ignored. */
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_OFF,
+										PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ) != PVRSRV_OK)
+	{
+		/* Ignore return error as we're already returning an error here. */
+		(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceResume
+@Description  Common device resume.
+@Input        psDev  The device node representing the device that should
+                     be resumed
+@Return       int    0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceResume(struct drm_device *psDev)
+{
+	struct pvr_drm_private *psDevPriv = psDev->dev_private;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevPriv->dev_node;
+
+	if (PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+										PVRSRV_SYS_POWER_STATE_ON,
+										PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ) != PVRSRV_OK)
+	{
+		return -EINVAL;
+	}
+
+	/* Ignore return error. We should proceed even if this fails. */
+	(void) LinuxBridgeUnblockClientsAccess(psDevPriv);
+
+	/*
+	 * Reprocess the device queues in case commands were blocked during
+	 * suspend.
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVRSRVCheckStatus(NULL);
+	}
+
+	return 0;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceServicesOpen
+@Description  Services device open.
+@Input        psDeviceNode  The device node representing the device being
+                            opened by a user mode process
+@Input        psDRMFile     The DRM file data that backs the file handle
+                            returned to the user mode process
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+int PVRSRVDeviceServicesOpen(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             struct drm_file *psDRMFile)
+{
+	static DEFINE_MUTEX(sDeviceInitMutex);
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	ENV_CONNECTION_PRIVATE_DATA sPrivData;
+	PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+	PVRSRV_ERROR eError;
+	int iErr = 0;
+
+	if (!psPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device data", __func__));
+		iErr = -ENODEV;
+		goto out;
+	}
+
+	mutex_lock(&sDeviceInitMutex);
+	/*
+	 * If the first attempt already set the state to bad,
+	 * there is no point in going the second time, so get out
+	 */
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_BAD)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Driver already in bad state. Device open failed.",
+				 __func__));
+		iErr = -ENODEV;
+		mutex_unlock(&sDeviceInitMutex);
+		goto out;
+	}
+
+	if (psDRMFile->driver_priv == NULL)
+	{
+		/* Allocate psConnectionPriv (stores private data and release pfn under driver_priv) */
+		psConnectionPriv = kzalloc(sizeof(*psConnectionPriv), GFP_KERNEL);
+		if (!psConnectionPriv)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate driver_priv data", __func__));
+			iErr = -ENOMEM;
+			mutex_unlock(&sDeviceInitMutex);
+			goto fail_alloc_connection_priv;
+		}
+		psConnectionPriv->ui32Type = DKF_CONNECTION_FLAG_SERVICES;
+	}
+	else
+	{
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+	}
+
+	if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_CREATED)
+	{
+		eError = PVRSRVCommonDeviceInitialise(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to initialise device (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			iErr = -ENODEV;
+			mutex_unlock(&sDeviceInitMutex);
+			goto fail_device_init;
+		}
+
+#if defined(SUPPORT_RGX)
+		PVRGpuTraceInitIfEnabled(psDeviceNode);
+#endif
+	}
+	mutex_unlock(&sDeviceInitMutex);
+
+	sPrivData.psDevNode = psDeviceNode;
+	sPrivData.psDRMFile = psDRMFile;
+
+	/*
+	 * Here we pass the file pointer which will passed through to our
+	 * OSConnectionPrivateDataInit function where we can save it so
+	 * we can back reference the file structure from its connection
+	 */
+	eError = PVRSRVCommonConnectionConnect(&psConnectionPriv->pvConnectionData,
+	                                       (void *)&sPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		iErr = -ENOMEM;
+		goto fail_connect;
+	}
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pfDeviceRelease = PVRSRVCommonConnectionDisconnect;
+#endif
+	psDRMFile->driver_priv = (void*)psConnectionPriv;
+
+	goto out;
+
+fail_connect:
+fail_device_init:
+	kfree(psConnectionPriv);
+fail_alloc_connection_priv:
+out:
+	return iErr;
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceSyncOpen
+@Description  Sync device open.
+@Input        psDeviceNode  The device node representing the device being
+                            opened by a user mode process
+@Input        psDRMFile     The DRM file data that backs the file handle
+                            returned to the user mode process
+@Return       int           0 on success and a Linux error code otherwise
+*/ /***************************************************************************/
+static int PVRSRVDeviceSyncOpen(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                struct drm_file *psDRMFile)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	CONNECTION_DATA *psConnection = NULL;
+	ENV_CONNECTION_PRIVATE_DATA sPrivData;
+	PVRSRV_CONNECTION_PRIV *psConnectionPriv;
+	PVRSRV_ERROR eError;
+	int iErr = 0;
+
+	if (!psPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No device data", __func__));
+		iErr = -ENODEV;
+		goto out;
+	}
+
+	if (psDRMFile->driver_priv == NULL)
+	{
+		/* Allocate psConnectionPriv (stores private data and release pfn under driver_priv) */
+		psConnectionPriv = kzalloc(sizeof(*psConnectionPriv), GFP_KERNEL);
+		if (!psConnectionPriv)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate driver_priv data", __func__));
+			iErr = -ENOMEM;
+			goto out;
+		}
+	}
+	else
+	{
+		psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+	}
+
+	/* Allocate connection data area, no stats since process not registered yet */
+	psConnection = kzalloc(sizeof(*psConnection), GFP_KERNEL);
+	if (!psConnection)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: No memory to allocate connection data", __func__));
+		iErr = -ENOMEM;
+		goto fail_alloc_connection;
+	}
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pvConnectionData = (void*)psConnection;
+#else
+	psConnectionPriv->pvSyncConnectionData = (void*)psConnection;
+#endif
+
+	psConnectionPriv->ui32Type = DKF_CONNECTION_FLAG_SYNC;
+
+	sPrivData.psDevNode = psDeviceNode;
+	sPrivData.psDRMFile = psDRMFile;
+
+	/* Call environment specific connection data init function */
+	eError = OSConnectionPrivateDataInit(&psConnection->hOsPrivateData, &sPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSConnectionPrivateDataInit() failed (%s)",
+		        __func__, PVRSRVGetErrorString(eError)));
+		goto fail_private_data_init;
+	}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	iErr = pvr_sync_open(psConnectionPriv->pvConnectionData, psDRMFile);
+#else
+	iErr = pvr_sync_open(psConnectionPriv->pvSyncConnectionData, psDRMFile);
+#endif
+	if (iErr)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: pvr_sync_open() failed(%d)",
+				__func__, iErr));
+		goto fail_pvr_sync_open;
+	}
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	psConnectionPriv->pfDeviceRelease = pvr_sync_close;
+#endif
+#endif
+	psDRMFile->driver_priv = psConnectionPriv;
+	goto out;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+fail_pvr_sync_open:
+	OSConnectionPrivateDataDeInit(psConnection->hOsPrivateData);
+#endif
+fail_private_data_init:
+	kfree(psConnection);
+fail_alloc_connection:
+	kfree(psConnectionPriv);
+out:
+	return iErr;
+#else /* if defined(SUPPORT_NATIVE_FENCE_SYNC) */
+	/* Ioctl should not be being called (no DDK support for native sync) */
+	PVR_DPF((PVR_DBG_ERROR, "%s: only supported when SUPPORT_NATIVE_FENCE_SYNC=1", __func__));
+	return -ENOTTY;
+#endif /* if defined(SUPPORT_NATIVE_FENCE_SYNC) */
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceRelease
+@Description  Common device release.
+@Input        psDeviceNode  The device node for the device that the given file
+                            represents
+@Input        psDRMFile     The DRM file data that's being released
+@Return       void
+*/ /***************************************************************************/
+void PVRSRVDeviceRelease(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         struct drm_file *psDRMFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	if (psDRMFile->driver_priv)
+	{
+		PVRSRV_CONNECTION_PRIV *psConnectionPriv = (PVRSRV_CONNECTION_PRIV*)psDRMFile->driver_priv;
+
+		if (psConnectionPriv->pvConnectionData)
+		{
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+			if (psConnectionPriv->pfDeviceRelease)
+			{
+				psConnectionPriv->pfDeviceRelease(psConnectionPriv->pvConnectionData);
+			}
+#else
+			if (psConnectionPriv->pvConnectionData)
+				PVRSRVCommonConnectionDisconnect(psConnectionPriv->pvConnectionData);
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+			if (psConnectionPriv->pvSyncConnectionData)
+				pvr_sync_close(psConnectionPriv->pvSyncConnectionData);
+#endif
+#endif
+		}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+		LinuxDeregisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+
+		kfree(psDRMFile->driver_priv);
+		psDRMFile->driver_priv = NULL;
+	}
+}
+
+int
+drm_pvr_srvkm_init(struct drm_device *dev, void *arg, struct drm_file *psDRMFile)
+{
+	struct drm_pvr_srvkm_init_data *data = arg;
+	struct pvr_drm_private *priv = dev->dev_private;
+	int iErr = 0;
+
+	switch (data->init_module)
+	{
+		case PVR_SRVKM_SYNC_INIT:
+		{
+			iErr = PVRSRVDeviceSyncOpen(priv->dev_node, psDRMFile);
+			break;
+		}
+		case PVR_SRVKM_SERVICES_INIT:
+		{
+			iErr = PVRSRVDeviceServicesOpen(priv->dev_node, psDRMFile);
+			if (iErr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDeviceServicesOpen() failed(%d)",
+						__func__, iErr));
+				break;
+			}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+			iErr = LinuxRegisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+			break;
+		}
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		case PVR_SRVKM_SERVICES_PAGE_MIGRATE_INIT:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+			iErr = LinuxRegisterMigrateCallbacks(psDRMFile->filp);
+#endif
+#endif
+			break;
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid init_module (%d)",
+			        __func__, data->init_module));
+			iErr = -EINVAL;
+		}
+	}
+
+	return iErr;
+}
diff --git a/drivers/gpu/drm/img-rogue/module_common.h b/drivers/gpu/drm/img-rogue/module_common.h
new file mode 100644
index 000000000000..35f33770e2e5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/module_common.h
@@ -0,0 +1,107 @@
+/*************************************************************************/ /*!
+@File           module_common.h
+@Title          Common linux module setup header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef MODULE_COMMON_H
+#define MODULE_COMMON_H
+
+#include "pvr_drm.h"
+
+/* DRVNAME is the name we use to register our driver. */
+#define DRVNAME PVR_LDM_DRIVER_REGISTRATION_NAME
+
+struct _PVRSRV_DEVICE_NODE_;
+struct drm_file;
+struct drm_device;
+
+/* psDRMFile->driver_priv will point to a PVRSV_CONNECTION_PRIV
+ * struct, which will contain a ptr to the CONNECTION_DATA and
+ * a pfn to the release function (which will differ depending
+ * on whether the connection is to Sync or Services).
+ */
+typedef void (*PFN_PVRSRV_DEV_RELEASE)(void *pvData);
+typedef struct
+{
+	/* pvConnectionData is used to hold Services connection data
+	 * for all PVRSRV_DEVICE_INIT_MODE options.
+	 */
+	void *pvConnectionData;
+
+	/* pfDeviceRelease is used to indicate the release function
+	 * to be called when PVRSRV_DEVICE_INIT_MODE is PVRSRV_LINUX_DEV_INIT_ON_CONNECT,
+	 * as we can then have one connections made (either for Services or Sync) per
+	 * psDRMFile, and need to know which type of connection is being released
+	 * (as the ioctl release call is common for both).
+	 */
+	PFN_PVRSRV_DEV_RELEASE pfDeviceRelease;
+
+	/* pvSyncConnectionData is used to hold Sync connection data
+	 * when PVRSRV_DEVICE_INIT_MODE is not PVRSRV_LINUX_DEV_INIT_ON_CONNECT,
+	 * as we can then have two connections made (for Services and Sync) to
+	 * the same psDRMFile.
+	 */
+	void *pvSyncConnectionData;
+
+	/* An integer representing the type of connection (indicated by the
+	 * DKF_CONNECTION_FLAGS).
+	 */
+	uint32_t ui32Type;
+
+} PVRSRV_CONNECTION_PRIV;
+
+int PVRSRVDriverInit(void);
+void PVRSRVDriverDeinit(void);
+
+int PVRSRVDeviceInit(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+void PVRSRVDeviceDeinit(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+void PVRSRVDeviceShutdown(struct drm_device *psDev);
+int PVRSRVDeviceSuspend(struct drm_device *psDev);
+int PVRSRVDeviceResume(struct drm_device *psDev);
+
+int PVRSRVDeviceServicesOpen(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                             struct drm_file *psDRMFile);
+void PVRSRVDeviceRelease(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode,
+                         struct drm_file *psDRMFile);
+int drm_pvr_srvkm_init(struct drm_device *dev,
+                       void *arg, struct drm_file *psDRMFile);
+
+#endif /* MODULE_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/multicore_defs.h b/drivers/gpu/drm/img-rogue/multicore_defs.h
new file mode 100644
index 000000000000..b410f57075f4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/multicore_defs.h
@@ -0,0 +1,53 @@
+/**************************************************************************/ /*!
+@File
+@Title          RGX Multicore Information flags
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_MULTICORE_DEFS_H
+#define RGX_MULTICORE_DEFS_H
+
+/* Capability bits returned to client in RGXGetMultiCoreInfo */
+#define RGX_MULTICORE_CAPABILITY_FRAGMENT_EN    (0x00000040U)
+#define RGX_MULTICORE_CAPABILITY_GEOMETRY_EN    (0x00000020U)
+#define RGX_MULTICORE_CAPABILITY_COMPUTE_EN     (0x00000010U)
+#define RGX_MULTICORE_CAPABILITY_PRIMARY_EN     (0x00000008U)
+#define RGX_MULTICORE_ID_CLRMSK                 (0xFFFFFFF8U)
+
+#endif /* RGX_MULTICORE_DEFS_H */
diff --git a/drivers/gpu/drm/img-rogue/opaque_types.h b/drivers/gpu/drm/img-rogue/opaque_types.h
new file mode 100644
index 000000000000..766bc22ea418
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/opaque_types.h
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File
+@Title          Opaque Types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines opaque types for various services types
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef SERVICES_OPAQUE_TYPES_H
+#define SERVICES_OPAQUE_TYPES_H
+
+#include "img_defs.h"
+#include "img_types.h"
+
+typedef struct _PVRSRV_DEVICE_NODE_ *PPVRSRV_DEVICE_NODE;
+typedef const struct _PVRSRV_DEVICE_NODE_ *PCPVRSRV_DEVICE_NODE;
+
+#endif /* SERVICES_OPAQUE_TYPES_H */
+
+/******************************************************************************
+ End of file (opaque_types.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/os_apphint.h b/drivers/gpu/drm/img-rogue/os_apphint.h
new file mode 100644
index 000000000000..93c9272498e5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/os_apphint.h
@@ -0,0 +1,186 @@
+/*************************************************************************/ /*!
+@File           os_apphint.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS-independent interface for retrieving apphints
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_defs.h"
+#if defined(__linux__)
+#include "km_apphint.h"
+#include "device.h"
+#else
+#include "services_client_porting.h"
+#endif
+#if !defined(OS_APPHINT_H)
+#define OS_APPHINT_H
+
+/*! Supplied to os_get_apphint_XXX() functions when the param/AppHint is
+ * applicable to all devices and not a specific device. Typically used
+ * for server-wide build and module AppHints.
+ */
+#define APPHINT_NO_DEVICE (NULL)
+
+#if defined(__linux__) && !defined(DOXYGEN)
+static INLINE IMG_UINT os_get_apphint_UINT32(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT32 *pAppHintDefault, IMG_UINT32 *pVal) {
+	return !pvr_apphint_get_uint32(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_UINT64(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT64 *pAppHintDefault, IMG_UINT64 *pVal) {
+	return !pvr_apphint_get_uint64(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_BOOL(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_BOOL *pAppHintDefault, IMG_BOOL *pVal) {
+	return !pvr_apphint_get_bool(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_apphint_STRING(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_CHAR *pAppHintDefault, IMG_CHAR *buffer, size_t size) {
+	return !pvr_apphint_get_string(device, id, buffer, size);
+}
+
+#define OSGetAppHintUINT32(device, state, name, appHintDefault, value) \
+	os_get_apphint_UINT32(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintUINT64(device, state, name, appHintDefault, value) \
+	os_get_apphint_UINT64(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintBOOL(device, state, name, appHintDefault, value) \
+	os_get_apphint_BOOL(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	os_get_apphint_STRING(device, state, APPHINT_ID_ ## name, appHintDefault, buffer, size)
+
+
+#define OSCreateAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#define OSFreeAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#else /* defined(__linux__) && !defined(DOXYGEN) */
+
+/**************************************************************************/ /*!
+@def OSGetAppHintUINT32(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint32 server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintUINT32(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintUINT64(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint64 server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintUINT64(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintBOOL(state, name, appHintDefault, value)
+@Description    Interface for retrieval of IMG_BOOL server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetAppHintBOOL(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_BOOL_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetAppHintSTRING(state, name, appHintDefault, buffer, size)
+@Description    Interface for retrieval of string server app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         buffer            Buffer used to return app hint string.
+@Input			size			  Size of the buffer.
+ */ /**************************************************************************/
+#define OSGetAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	(PVR_UNREFERENCED_PARAMETER(size), PVRSRVGetAppHint(state, # name, IMG_STRING_TYPE, appHintDefault, buffer))
+
+/**************************************************************************/ /*!
+@def OSCreateAppHintState(state)
+@Description    Creates the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSCreateAppHintState(state) \
+	PVRSRVCreateAppHintState(IMG_SRV_UM, 0, state)
+
+/**************************************************************************/ /*!
+@def OSFreeAppHintState
+@Description    Free the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSFreeAppHintState(state) \
+	PVRSRVFreeAppHintState(IMG_SRV_UM, state)
+
+#endif /* defined(__linux__) */
+
+#endif /* OS_APPHINT_H */
+
+/******************************************************************************
+ End of file (os_apphint.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/os_cpu_cache.h b/drivers/gpu/drm/img-rogue/os_cpu_cache.h
new file mode 100644
index 000000000000..56f92036ff49
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/os_cpu_cache.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS and CPU d-cache maintenance mechanisms
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines for cache management which are visible internally only
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OS_CPU_CACHE_H
+#define OS_CPU_CACHE_H
+
+#include "info_page_defs.h"
+
+#define PVRSRV_CACHE_OP_TIMELINE			0x8 /*!< Request SW_SYNC timeline notification when executed */
+#define PVRSRV_CACHE_OP_FORCE_SYNCHRONOUS	0x10 /*!< Force all batch members to be executed synchronously */
+
+#define CACHEFLUSH_ISA_X86					0x1	/*!< x86/x64 specific UM range-based cache flush */
+#define CACHEFLUSH_ISA_ARM64				0x2	/*!< Aarch64 specific UM range-based cache flush */
+#define CACHEFLUSH_ISA_GENERIC				0x3	/*!< Other ISA's without UM range-based cache flush */
+#ifndef CACHEFLUSH_ISA_TYPE
+	#if defined(__i386__) || defined(__x86_64__)
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_X86
+	#elif defined(__arm64__) || defined(__aarch64__)
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_ARM64
+	#else
+		#define CACHEFLUSH_ISA_TYPE CACHEFLUSH_ISA_GENERIC
+	#endif
+#endif
+
+#if (CACHEFLUSH_ISA_TYPE == CACHEFLUSH_ISA_X86) || (CACHEFLUSH_ISA_TYPE == CACHEFLUSH_ISA_ARM64)
+#define CACHEFLUSH_ISA_SUPPORTS_UM_FLUSH		/*!< x86/x86_64/ARM64 supports user-mode d-cache flush */
+#endif
+
+#endif	/* OS_CPU_CACHE_H */
diff --git a/drivers/gpu/drm/img-rogue/os_srvinit_param.h b/drivers/gpu/drm/img-rogue/os_srvinit_param.h
new file mode 100644
index 000000000000..a4d77e381ff7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/os_srvinit_param.h
@@ -0,0 +1,328 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services initialisation parameters header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services initialisation parameter support for the Linux kernel.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OS_SRVINIT_PARAM_H
+#define OS_SRVINIT_PARAM_H
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include "km_apphint.h"
+#include "km_apphint_defs.h"
+
+/* Supplied to SrvInitParamGetXXX() functions when the param/AppHint is
+ * applicable to all devices and not a specific device. Typically used
+ * for server-wide build and module AppHints.
+ */
+#define INITPARAM_NO_DEVICE (NULL)
+
+#define SrvInitParamOpen() NULL
+#define SrvInitParamClose(pvState) ((void)(pvState))
+
+#define SrvInitParamGetBOOL(device, state, name, value) \
+	((void) pvr_apphint_get_bool(device, APPHINT_ID_ ## name, &value))
+
+#define SrvInitParamGetUINT32(device, state, name, value) \
+	((void) pvr_apphint_get_uint32(device, APPHINT_ID_ ## name, &value))
+
+#define SrvInitParamGetUINT64(device, state, name, value) \
+	((void) pvr_apphint_get_uint64(device, APPHINT_ID_ ## name, &value))
+
+#define SrvInitParamGetSTRING(device, state, name, buffer, size) \
+	((void) pvr_apphint_get_string(device, APPHINT_ID_ ## name, buffer, size))
+
+#define SrvInitParamGetUINT32BitField(device, state, name, value) \
+	((void) pvr_apphint_get_uint32(device, APPHINT_ID_ ## name, &value))
+
+#define SrvInitParamGetUINT32List(device, state, name, value) \
+	((void) pvr_apphint_get_uint32(device, APPHINT_ID_ ## name, &value))
+
+#else	/* defined(__linux__) && defined(__KERNEL__) */
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+#include "img_types.h"
+
+/*! Lookup item. */
+typedef struct
+{
+	const IMG_CHAR *pszValue;       /*!< looked up name */
+	IMG_UINT32 ui32Value;           /*!< looked up value */
+} SRV_INIT_PARAM_UINT32_LOOKUP;
+
+/*************************************************************************/ /*!
+@Brief          SrvInitParamOpen
+
+@Description    Establish a connection to the Parameter resource store which is
+                used to hold configuration information associated with the
+                server instance.
+
+@Return         (void *) Handle to Parameter resource store to be used for
+                subsequent parameter value queries
+
+*/ /**************************************************************************/
+void *SrvInitParamOpen(void);
+
+/*************************************************************************/ /*!
+@Brief          SrvInitParamClose
+
+@Description    Remove a pre-existing connection to the Parameter resource store
+                given by 'pvState' and release any temporary storage associated
+                with the 'pvState' mapping handle
+
+@Input          pvState             Handle to Parameter resource store
+
+*/ /**************************************************************************/
+void SrvInitParamClose(void *pvState);
+
+/*************************************************************************/ /*!
+@Brief          _SrvInitParamGetBOOL
+
+@Description    Get the current BOOL value for parameter 'pszName' from the
+                Parameter resource store attached to 'pvState'
+
+@Input          pvState             Handle to Parameter resource store
+
+@Input          pszName             Name of parameter to look-up
+
+@Input          pbDefault           Value to return if parameter not found
+
+@Output         pbValue             Value of parameter 'pszName' or 'pbDefault'
+                                    if not found
+
+*/ /**************************************************************************/
+void _SrvInitParamGetBOOL(
+	void *pvState,
+	const IMG_CHAR *pszName,
+	const IMG_BOOL *pbDefault,
+	IMG_BOOL *pbValue
+);
+
+/*! Get the BOOL value for parameter 'name' from the parameter resource store
+ *  attached to 'state'. */
+#define SrvInitParamGetBOOL(device, state, name, value) \
+		_SrvInitParamGetBOOL(state, # name, & __SrvInitParam_ ## name, &(value))
+
+/*! Initialise FLAG type parameter identified by 'name'. */
+#define SrvInitParamInitFLAG(name, defval, unused) \
+	static const IMG_BOOL __SrvInitParam_ ## name = defval;
+
+/*! Initialise BOOL type parameter identified by 'name'. */
+#define SrvInitParamInitBOOL(name, defval, unused) \
+	static const IMG_BOOL __SrvInitParam_ ## name = defval;
+
+/*************************************************************************/ /*!
+@Brief          _SrvInitParamGetUINT32
+
+@Description    Get the current IMG_UINT32 value for parameter 'pszName'
+                from the Parameter resource store attached to 'pvState'
+
+@Input          pvState             Handle to Parameter resource store
+
+@Input          pszName             Name of parameter to look-up
+
+@Input          pui32Default        Value to return if parameter not found
+
+@Output         pui32Value            Value of parameter 'pszName' or
+                                    'pui32Default' if not found
+
+*/ /**************************************************************************/
+void _SrvInitParamGetUINT32(
+	void *pvState,
+	const IMG_CHAR *pszName,
+	const IMG_UINT32 *pui32Default,
+	IMG_UINT32 *pui32Value
+);
+
+/*! Get the UINT32 value for parameter 'name' from the parameter resource store
+ *  attached to 'state'. */
+#define SrvInitParamGetUINT32(device, state, name, value) \
+		_SrvInitParamGetUINT32(state, # name, & __SrvInitParam_ ## name, &(value))
+
+/*! Initialise UINT32 type parameter identified by 'name'. */
+#define SrvInitParamInitUINT32(name, defval, unused) \
+	static const IMG_UINT32 __SrvInitParam_ ## name = defval;
+
+/*! Initialise UINT64 type parameter identified by 'name'. */
+#define SrvInitParamInitUINT64(name, defval, unused) \
+	static const IMG_UINT64 __SrvInitParam_ ## name = defval;
+
+/*! @cond Doxygen_Suppress */
+#define SrvInitParamUnreferenced(name) \
+		PVR_UNREFERENCED_PARAMETER( __SrvInitParam_ ## name )
+/*! @endcond */
+
+/*************************************************************************/ /*!
+@Brief          _SrvInitParamGetUINT32BitField
+
+@Description    Get the current IMG_UINT32 bitfield value for parameter
+                'pszBasename' from the Parameter resource store
+                attached to 'pvState'
+
+@Input          pvState             Handle to Parameter resource store
+
+@Input          pszBaseName         Bitfield parameter name to search for
+
+@Input          uiDefault           Default return value if parameter not found
+
+@Input          psLookup            Bitfield array to traverse
+
+@Input          uiSize              number of elements in 'psLookup'
+
+@Output         puiValue            Value of bitfield or 'uiDefault' if
+                                    parameter not found
+*/ /**************************************************************************/
+void _SrvInitParamGetUINT32BitField(
+	void *pvState,
+	const IMG_CHAR *pszBaseName,
+	IMG_UINT32 uiDefault,
+	const SRV_INIT_PARAM_UINT32_LOOKUP *psLookup,
+	IMG_UINT32 uiSize,
+	IMG_UINT32 *puiValue
+);
+
+/*! Initialise UINT32 bitfield type parameter identified by 'name' with
+ *  'inival' value and 'lookup' look up array. */
+#define SrvInitParamInitUINT32Bitfield(name, inival, lookup) \
+	static IMG_UINT32 __SrvInitParam_ ## name = inival; \
+	static SRV_INIT_PARAM_UINT32_LOOKUP * \
+		__SrvInitParamLookup_ ## name = &lookup[0]; \
+	static const IMG_UINT32 __SrvInitParamSize_ ## name = \
+					ARRAY_SIZE(lookup);
+
+/*! Get the UINT32 bitfield value for parameter 'name' from the parameter
+ *  resource store attached to 'state'. */
+#define SrvInitParamGetUINT32BitField(device, state, name, value) \
+		_SrvInitParamGetUINT32BitField(state, # name, __SrvInitParam_ ## name, __SrvInitParamLookup_ ## name, __SrvInitParamSize_ ## name, &(value))
+
+/*************************************************************************/ /*!
+@Brief          _SrvInitParamGetUINT32List
+
+@Description    Get the current IMG_UINT32 list value for the specified
+                parameter 'pszName' from the Parameter resource store
+                attached to 'pvState'
+
+@Input          pvState             Handle to Parameter resource store
+
+@Input          pszName             Parameter list name to search for
+
+@Input          uiDefault           Default value to return if 'pszName' is
+                                    not set within 'pvState'
+
+@Input          psLookup            parameter list to traverse
+
+@Input          uiSize              number of elements in 'psLookup' list
+
+@Output         puiValue            value of located list element or
+                                    'uiDefault' if parameter not found
+
+*/ /**************************************************************************/
+void _SrvInitParamGetUINT32List(
+	void *pvState,
+	const IMG_CHAR *pszName,
+	IMG_UINT32 uiDefault,
+	const SRV_INIT_PARAM_UINT32_LOOKUP *psLookup,
+	IMG_UINT32 uiSize,
+	IMG_UINT32 *puiValue
+);
+
+/*! Get the UINT32 list value for parameter 'name' from the parameter
+ *  resource store attached to 'state'. */
+#define SrvInitParamGetUINT32List(device, state, name, value) \
+		_SrvInitParamGetUINT32List(state, # name, __SrvInitParam_ ## name, __SrvInitParamLookup_ ## name, __SrvInitParamSize_ ## name, &(value))
+
+/*! Initialise UINT32 list type parameter identified by 'name' with
+ *  'defval' default value and 'lookup' look up list. */
+#define SrvInitParamInitUINT32List(name, defval, lookup) \
+	static IMG_UINT32 __SrvInitParam_ ## name = defval; \
+	static SRV_INIT_PARAM_UINT32_LOOKUP * \
+		__SrvInitParamLookup_ ## name = &lookup[0]; \
+	static const IMG_UINT32 __SrvInitParamSize_ ## name = \
+					ARRAY_SIZE(lookup);
+
+/*************************************************************************/ /*!
+@Brief          _SrvInitParamGetSTRING
+
+@Description    Get the contents of the specified parameter string 'pszName'
+                from the Parameter resource store attached to 'pvState'
+
+@Input          pvState             Handle to Parameter resource store
+
+@Input          pszName             Parameter string name to search for
+
+@Input          psDefault           Default string to return if 'pszName' is
+                                    not set within 'pvState'
+
+@Input          size                Size of output 'pBuffer'
+
+@Output         pBuffer             Output copy of 'pszName' contents or
+                                    copy of 'psDefault' if 'pszName' is not
+                                    set within 'pvState'
+
+*/ /**************************************************************************/
+void _SrvInitParamGetSTRING(
+	void *pvState,
+	const IMG_CHAR *pszName,
+	const IMG_CHAR *psDefault,
+	IMG_CHAR *pBuffer,
+	size_t size
+);
+
+/*! Initialise STRING type parameter identified by 'name' with 'defval' default
+ *  value. */
+#define SrvInitParamInitSTRING(name, defval, unused) \
+	static const IMG_CHAR *__SrvInitParam_ ## name = defval;
+
+/*! Get the STRING value for parameter 'name' from the parameter resource store
+ *  attached to 'state'. */
+#define SrvInitParamGetSTRING(device, state, name, buffer, size) \
+		_SrvInitParamGetSTRING(state, # name,  __SrvInitParam_ ## name, buffer, size)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#endif /* OS_SRVINIT_PARAM_H */
diff --git a/drivers/gpu/drm/img-rogue/osconnection_server.c b/drivers/gpu/drm/img-rogue/osconnection_server.c
new file mode 100644
index 000000000000..543b4d8ddabf
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osconnection_server.c
@@ -0,0 +1,168 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux specific per process data functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "connection_server.h"
+#include "osconnection_server.h"
+
+#include "env_connection.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+
+#include <linux/sched.h>
+
+#if defined(SUPPORT_ION)
+#include <linux/err.h>
+#include PVR_ANDROID_ION_HEADER
+
+/*
+	The ion device (the base object for all requests)
+	gets created by the system and we acquire it via
+	Linux specific functions provided by the system layer
+*/
+#include "ion_sys.h"
+#endif
+
+PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData)
+{
+	ENV_CONNECTION_PRIVATE_DATA *psPrivData = pvOSData;
+	ENV_CONNECTION_DATA *psEnvConnection;
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	ENV_ION_CONNECTION_DATA *psIonConnection;
+#endif
+
+	*phOsPrivateData = OSAllocZMem(sizeof(ENV_CONNECTION_DATA));
+
+	if (*phOsPrivateData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem failed", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psEnvConnection = (ENV_CONNECTION_DATA *)*phOsPrivateData;
+
+	psEnvConnection->owner = current->tgid;
+
+	psEnvConnection->psDevNode = psPrivData->psDevNode;
+	psEnvConnection->psDRMFile = psPrivData->psDRMFile;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	psEnvConnection->pvPvrSyncPrivateData = NULL;
+#endif
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	psIonConnection = (ENV_ION_CONNECTION_DATA *)OSAllocZMem(sizeof(ENV_ION_CONNECTION_DATA));
+	if (psIonConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem failed", __func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psEnvConnection->psIonData = psIonConnection;
+	/*
+		We can have more than one connection per process, so we need
+		more than the PID to have a unique name.
+	*/
+	psEnvConnection->psIonData->psIonDev = IonDevAcquire();
+	OSSNPrintf(psEnvConnection->psIonData->azIonClientName, ION_CLIENT_NAME_SIZE, "pvr_ion_client-%p-%d", *phOsPrivateData, OSGetCurrentClientProcessIDKM());
+	psEnvConnection->psIonData->psIonClient =
+		ion_client_create(psEnvConnection->psIonData->psIonDev,
+						  psEnvConnection->psIonData->azIonClientName);
+
+	if (IS_ERR_OR_NULL(psEnvConnection->psIonData->psIonClient))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSConnectionPrivateDataInit: Couldn't create "
+								"ion client for per connection data"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+#endif /* SUPPORT_ION && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData)
+{
+	if (hOsPrivateData == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_ION) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	{
+		ENV_CONNECTION_DATA *psEnvConnection = hOsPrivateData;
+
+		PVR_ASSERT(psEnvConnection->psIonData != NULL);
+
+		PVR_ASSERT(psEnvConnection->psIonData->psIonClient != NULL);
+		ion_client_destroy(psEnvConnection->psIonData->psIonClient);
+
+		IonDevRelease(psEnvConnection->psIonData->psIonDev);
+		OSFreeMem(psEnvConnection->psIonData);
+	}
+#endif
+
+	OSFreeMem(hOsPrivateData);
+	/*not nulling pointer, copy on stack*/
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_DEVICE_NODE *OSGetDevNode(CONNECTION_DATA *psConnection)
+{
+	ENV_CONNECTION_DATA *psEnvConnection;
+
+	psEnvConnection = PVRSRVConnectionPrivateData(psConnection);
+	PVR_ASSERT(psEnvConnection);
+
+	return psEnvConnection->psDevNode;
+}
+
+struct drm_file *OSGetDRMFile(CONNECTION_DATA *psConnection)
+{
+	ENV_CONNECTION_DATA *psEnvConnection;
+
+	psEnvConnection = PVRSRVConnectionPrivateData(psConnection);
+	PVR_ASSERT(psEnvConnection);
+
+	return psEnvConnection->psDRMFile;
+}
diff --git a/drivers/gpu/drm/img-rogue/osconnection_server.h b/drivers/gpu/drm/img-rogue/osconnection_server.h
new file mode 100644
index 000000000000..88f7aab29dca
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osconnection_server.h
@@ -0,0 +1,137 @@
+/**************************************************************************/ /*!
+@File
+@Title          Server side connection management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    API for OS specific callbacks from server side connection
+                management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#ifndef OSCONNECTION_SERVER_H
+#define OSCONNECTION_SERVER_H
+
+#include "handle.h"
+#include "osfunc.h"
+
+/*! Function not implemented definition. */
+#define OSCONNECTION_SERVER_NOT_IMPLEMENTED 0
+/*! Assert used for OSCONNECTION_SERVER_NOT_IMPLEMENTED. */
+#define OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT() PVR_ASSERT(OSCONNECTION_SERVER_NOT_IMPLEMENTED)
+
+#if defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData);
+PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData);
+
+PVRSRV_ERROR OSConnectionSetHandleOptions(PVRSRV_HANDLE_BASE *psHandleBase);
+
+PVRSRV_DEVICE_NODE* OSGetDevNode(CONNECTION_DATA *psConnection);
+
+struct drm_file;
+
+struct drm_file* OSGetDRMFile(CONNECTION_DATA *psConnection);
+
+#else	/* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionPrivateDataInit)
+#endif
+/*************************************************************************/ /*!
+@Function       OSConnectionPrivateDataInit
+@Description    Allocates and initialises any OS-specific private data
+                relating to a connection.
+                Called from PVRSRVCommonConnectionConnect().
+@Input          pvOSData            pointer to any OS private data
+@Output         phOsPrivateData     handle to the created connection
+                                    private data
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSConnectionPrivateDataInit(IMG_HANDLE *phOsPrivateData, void *pvOSData)
+{
+	PVR_UNREFERENCED_PARAMETER(phOsPrivateData);
+	PVR_UNREFERENCED_PARAMETER(pvOSData);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionPrivateDataDeInit)
+#endif
+/*************************************************************************/ /*!
+@Function       OSConnectionPrivateDataDeInit
+@Description    Frees previously allocated OS-specific private data
+                relating to a connection.
+@Input          hOsPrivateData      handle to the connection private data
+                                    to be freed
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSConnectionPrivateDataDeInit(IMG_HANDLE hOsPrivateData)
+{
+	PVR_UNREFERENCED_PARAMETER(hOsPrivateData);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSConnectionSetHandleOptions)
+#endif
+static INLINE PVRSRV_ERROR OSConnectionSetHandleOptions(PVRSRV_HANDLE_BASE *psHandleBase)
+{
+	PVR_UNREFERENCED_PARAMETER(psHandleBase);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(OSGetDevNode)
+#endif
+static INLINE PVRSRV_DEVICE_NODE* OSGetDevNode(CONNECTION_DATA *psConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSCONNECTION_SERVER_NOT_IMPLEMENTED_ASSERT();
+
+	return NULL;
+}
+#endif	/* defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS) */
+
+
+#endif /* OSCONNECTION_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/osdi_impl.h b/drivers/gpu/drm/img-rogue/osdi_impl.h
new file mode 100644
index 000000000000..5be78d6c2b62
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osdi_impl.h
@@ -0,0 +1,213 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions and types for creating Debug Info implementations.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSDI_IMPL_H
+#define OSDI_IMPL_H
+
+#if defined(__KERNEL__) && defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __KERNEL__ && __linux__ */
+
+#include "di_common.h"
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+/*! Implementation callbacks. Those operations are performed on native
+ * implementation handles. */
+typedef struct OSDI_IMPL_ENTRY_CB
+{
+    /*! @Function pfnWrite
+     *
+     * @Description
+     * Writes the binary data of the DI entry to the output sync, whatever that
+     * may be for the DI implementation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pvData data
+     * @Input uiSize pvData length
+     */
+    void (*pfnWrite)(void  *pvNativeHandle, const void *pvData,
+                     IMG_UINT32 uiSize);
+
+    /*! @Function pfnVPrintf
+     *
+     * @Description
+     * Implementation of the 'vprintf' operation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pszFmt NUL-terminated format string
+     * @Input va_list variable length argument list
+     */
+    __printf(2, 0)
+    void (*pfnVPrintf)(void *pvNativeHandle, const IMG_CHAR *pszFmt, va_list pArgs);
+
+    /*! @Function pfnPuts
+     *
+     * @Description
+     * Implementation of the 'puts' operation.
+     *
+     * @Input pvNativeHandle native implementation handle
+     * @Input pszStr NUL-terminated string
+     */
+    void (*pfnPuts)(void *pvNativeHandle, const IMG_CHAR *pszStr);
+
+    /*! @Function pfnHasOverflowed
+     *
+     * @Description
+     * Checks if the native implementation's buffer has overflowed.
+     *
+     * @Input pvNativeHandle native implementation handle
+     */
+    IMG_BOOL (*pfnHasOverflowed)(void *pvNativeHandle);
+} OSDI_IMPL_ENTRY_CB;
+
+/*! Debug Info entry specialisation. */
+struct OSDI_IMPL_ENTRY
+{
+    /*! Pointer to the private data. The data originates from DICreateEntry()
+     *  function. */
+    void *pvPrivData;
+    /*! Pointer to the implementation native handle. */
+    void *pvNative;
+    /*! Implementation entry callbacks. */
+    OSDI_IMPL_ENTRY_CB *psCb;
+}; /* OSDI_IMPL_ENTRY is already typedef-ed in di_common.h */
+
+/*! Debug Info implementation callbacks. */
+typedef struct OSDI_IMPL_CB
+{
+    /*! Initialise implementation callback.
+     */
+    PVRSRV_ERROR (*pfnInit)(void);
+
+    /*! De-initialise implementation callback.
+     */
+    void (*pfnDeInit)(void);
+
+    /*! @Function pfnCreateEntry
+     *
+     * @Description
+     * Creates entry of eType type with pszName in the pvNativeGroup parent
+     * group. The entry is an abstract term which depends on the implementation,
+     * e.g.: a file in DebugFS.
+     *
+     * @Input pszName: name of the entry
+     * @Input eType: type of the entry
+     * @Input psIterCb: iterator implementation for the entry
+     * @Input pvPrivData: data that will be passed to the iterator callbacks
+     *                    in OSDI_IMPL_ENTRY - it can be retrieved by calling
+     *                    DIGetPrivData() function
+     * @Input pvNativeGroup: implementation specific handle to the parent group
+     *
+     * @Output pvNativeEntry: implementation specific handle to the entry
+     *
+     * return PVRSRV_ERROR error code
+     */
+    PVRSRV_ERROR (*pfnCreateEntry)(const IMG_CHAR *pszName,
+                                   DI_ENTRY_TYPE eType,
+                                   const DI_ITERATOR_CB *psIterCb,
+                                   void *pvPrivData,
+                                   void *pvNativeGroup,
+                                   void **pvNativeEntry);
+
+    /*! @Function pfnDestroyEntry
+     *
+     * @Description
+     * Destroys native entry.
+     *
+     * @Input psNativeEntry: handle to the entry
+     */
+    void (*pfnDestroyEntry)(void *psNativeEntry);
+
+    /*! @Function pfnCreateGroup
+     *
+     * @Description
+     * Creates group with pszName in the psNativeParentGroup parent group.
+     * The group is an abstract term which depends on the implementation,
+     * e.g.: a directory in DebugFS.
+     *
+     * @Input pszName: name of the entry
+     * @Input psNativeParentGroup: implementation specific handle to the parent
+     *                             group
+     *
+     * @Output psNativeGroup: implementation specific handle to the group
+     *
+     * return PVRSRV_ERROR error code
+     */
+    PVRSRV_ERROR (*pfnCreateGroup)(const IMG_CHAR *pszName,
+                                   void *psNativeParentGroup,
+                                   void **psNativeGroup);
+
+    /*! @Function pfnDestroyGroup
+     *
+     * @Description
+     * Destroys native group.
+     *
+     * @Input psNativeGroup: handle to the group
+     */
+    void (*pfnDestroyGroup)(void *psNativeGroup);
+} OSDI_IMPL_CB;
+
+/*! @Function DIRegisterImplementation
+ *
+ * @Description
+ * Registers Debug Info implementations with the framework. The framework takes
+ * the ownership of the implementation and will clean up the resources when
+ * it's de-initialised.
+ *
+ * @Input pszName: name of the implementation
+ * @Input psImplCb: implementation callbacks
+ *
+ * @Return PVRSRV_ERROR error code
+ */
+PVRSRV_ERROR DIRegisterImplementation(const IMG_CHAR *pszName,
+                                      const OSDI_IMPL_CB *psImplCb);
+
+#endif /* OSDI_IMPL_H */
diff --git a/drivers/gpu/drm/img-rogue/osfunc.c b/drivers/gpu/drm/img-rogue/osfunc.c
new file mode 100644
index 000000000000..a3e08336eccd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osfunc.c
@@ -0,0 +1,2726 @@
+/*************************************************************************/ /*!
+@File
+@Title          Environment related functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <asm/page.h>
+#include <asm/div64.h>
+#include <linux/atomic.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/hugetlb.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/delay.h>
+#include <linux/genalloc.h>
+#include <linux/string.h>
+#include <linux/freezer.h>
+#include <asm/hardirq.h>
+#include <asm/tlbflush.h>
+#include <linux/timer.h>
+#include <linux/capability.h>
+#include <linux/uaccess.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-direction.h>
+#include <linux/kthread.h>
+#include <linux/utsname.h>
+#include <linux/scatterlist.h>
+#include <linux/interrupt.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+#include <linux/pfn_t.h>
+#endif
+#include <linux/pfn.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+#include <linux/sched/clock.h>
+#include <linux/sched/signal.h>
+#else
+#include <linux/sched.h>
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) */
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+#include <linux/dma-heap.h>
+#include "physmem_dmabuf.h"
+#else
+#include "physmem.h"
+#endif
+#endif
+
+#include "log2.h"
+#include "sysinfo.h"
+#include "osfunc.h"
+#include "cache_km.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "devicemem_server_utils.h"
+#include "event.h"
+#include "linkage.h"
+#include "pvr_uaccess.h"
+#include "pvr_debug.h"
+#include "pvr_bridge_k.h"
+#include "pvrsrv_memallocflags.h"
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+#include "physmem_osmem_linux.h"
+#include "dma_support.h"
+
+#include "pvrsrv_sync_server.h"
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#include "pvr_ricommon.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+#if !defined(EVENT_OBJECT_TIMEOUT_US)
+#error EVENT_OBJECT_TIMEOUT_US should be defined sysinfo.h
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+#define PVR_FOLL_LONGTERM (0x0U)
+#else
+#define PVR_FOLL_LONGTERM FOLL_LONGTERM
+#endif
+
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(5, 6, 0))
+
+#define pvr_pin_user_pages_for_dma(puiAddress, num_pages, bWrite, pages)  get_user_pages_fast( \
+    (unsigned long)puiAddress, \
+    (int)num_pages, \
+    (int) (bWrite ? FOLL_WRITE : 0) | PVR_FOLL_LONGTERM, \
+    pages)
+#define pvr_unpin_user_page_for_dma(p) put_page(p)
+
+#else
+
+#define pvr_pin_user_pages_for_dma(puiAddress, num_pages, bWrite, pages) pin_user_pages_fast( \
+    (unsigned long)puiAddress, \
+    (int) num_pages, \
+    (int) (bWrite ? FOLL_WRITE : 0) | PVR_FOLL_LONGTERM, \
+    pages)
+#define pvr_unpin_user_page_for_dma(p) unpin_user_page(p)
+
+#endif
+
+#define _FREEZABLE IMG_TRUE
+#define _NON_FREEZABLE IMG_FALSE
+
+typedef struct {
+	struct task_struct *kthread;
+	PFN_THREAD pfnThread;
+	void *hData;
+	IMG_CHAR *pszThreadName;
+	IMG_BOOL   bIsThreadRunning;
+	IMG_BOOL   bIsSupportingThread;
+	PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB;
+	DLLIST_NODE sNode;
+} OSThreadData;
+
+void OSSuspendTaskInterruptible(void)
+{
+	set_current_state(TASK_INTERRUPTIBLE);
+	schedule();
+}
+
+static DLLIST_NODE gsThreadListHead;
+
+static void _ThreadListAddEntry(OSThreadData *psThreadListNode)
+{
+	dllist_add_to_tail(&gsThreadListHead, &(psThreadListNode->sNode));
+}
+
+static void _ThreadListRemoveEntry(OSThreadData *psThreadListNode)
+{
+	dllist_remove_node(&(psThreadListNode->sNode));
+}
+
+static void _ThreadSetStopped(OSThreadData *psOSThreadData)
+{
+	psOSThreadData->bIsThreadRunning = IMG_FALSE;
+}
+
+static void _OSInitThreadList(void)
+{
+	dllist_init(&gsThreadListHead);
+}
+
+void OSThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile)
+{
+	PDLLIST_NODE psNodeCurr, psNodeNext;
+	LINUX_THREAD_ACTIVITY_STATS sThreadStats;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	dllist_foreach_node(&gsThreadListHead, psNodeCurr, psNodeNext)
+	{
+		OSThreadData *psThreadListNode;
+		psThreadListNode = IMG_CONTAINER_OF(psNodeCurr, OSThreadData, sNode);
+
+		PVR_DUMPDEBUG_LOG("  %s : %s",
+				  psThreadListNode->pszThreadName,
+				  (psThreadListNode->bIsThreadRunning) ? "Running" : "Stopped");
+
+		if (psThreadListNode->pfnDebugDumpCB)
+		{
+			psThreadListNode->pfnDebugDumpCB(pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+	}
+
+	eError = LinuxGetThreadActivityStats(&sThreadStats);
+	if (eError == PVRSRV_OK)
+	{
+		PVR_DUMPDEBUG_LOG("Active Threads (UM/KM): %d / %d",
+		                  sThreadStats.i32DriverThreadCount,
+		                  sThreadStats.i32KernelThreadCount);
+
+		PVR_DUMPDEBUG_LOG("Suspended Threads: %d",
+		                  sThreadStats.i32SuspendedThreadCount);
+	}
+	else
+	{
+		PVR_LOG_ERROR(eError, "LinuxGetThreadActivityStats");
+	}
+}
+
+PVRSRV_ERROR OSPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+							PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+							IMG_PID uiPid)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+	IMG_CPU_PHYADDR sCpuPAddr;
+	struct page *psPage;
+	IMG_UINT32	ui32Order=0;
+	gfp_t gfp_flags;
+
+	PVR_ASSERT(uiSize != 0);
+	/*Align the size to the page granularity */
+	uiSize = PAGE_ALIGN(uiSize);
+
+	/*Get the order to be used with the allocation */
+	ui32Order = get_order(uiSize);
+
+	gfp_flags = GFP_KERNEL;
+
+#if !defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+	if (psDev)
+	{
+		if (*psDev->dma_mask == DMA_BIT_MASK(32))
+		{
+			/* Limit to 32 bit.
+			 * Achieved by setting __GFP_DMA32 for 64 bit systems */
+			gfp_flags |= __GFP_DMA32;
+		}
+		else if (*psDev->dma_mask < DMA_BIT_MASK(32))
+		{
+			/* Limit to whatever the size of DMA zone is. */
+			gfp_flags |= __GFP_DMA;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDev);
+#endif
+
+	/*allocate the pages */
+	psPage = alloc_pages(gfp_flags, ui32Order);
+	if (psPage == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	uiSize = (1 << ui32Order) * PAGE_SIZE;
+
+	psMemHandle->u.pvHandle = psPage;
+	psMemHandle->uiOrder = ui32Order;
+	sCpuPAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(psPage));
+
+	/*
+	 * Even when more pages are allocated as base MMU object we still need one single physical address because
+	 * they are physically contiguous.
+	 */
+	PhysHeapCpuPAddrToDevPAddr(psPhysHeap, 1, psDevPAddr, &sCpuPAddr);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                    uiSize,
+	                                    (IMG_UINT64)(uintptr_t) psPage,
+	                                    uiPid);
+#else
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                             psPage,
+	                             sCpuPAddr,
+	                             uiSize,
+	                             uiPid
+	                             DEBUG_MEMSTATS_VALUES);
+#endif
+#else
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif
+
+	return PVRSRV_OK;
+}
+
+void OSPhyContigPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle)
+{
+	struct page *psPage = (struct page*) psMemHandle->u.pvHandle;
+	IMG_UINT32 ui32Order;
+
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+	ui32Order = psMemHandle->uiOrder;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                      (IMG_UINT64)(uintptr_t) psPage);
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,
+	                                (IMG_UINT64)(uintptr_t) psPage,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	__free_pages(psPage, ui32Order);
+	psMemHandle->uiOrder = 0;
+}
+
+PVRSRV_ERROR OSPhyContigPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle,
+						size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+						void **pvPtr)
+{
+	size_t actualSize = 1 << (PAGE_SHIFT + psMemHandle->uiOrder);
+	*pvPtr = kmap((struct page*)psMemHandle->u.pvHandle);
+
+	PVR_UNREFERENCED_PARAMETER(psDevPAddr);
+
+	PVR_UNREFERENCED_PARAMETER(actualSize); /* If we don't take an #ifdef path */
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA, actualSize, OSGetCurrentClientProcessIDKM());
+#else
+	{
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = 0;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+									 *pvPtr,
+									 sCpuPAddr,
+									 actualSize,
+									 OSGetCurrentClientProcessIDKM()
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+}
+
+void OSPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	/* Mapping is done a page at a time */
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+	                            (1 << (PAGE_SHIFT + psMemHandle->uiOrder)),
+	                            OSGetCurrentClientProcessIDKM());
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,
+	                                (IMG_UINT64)(uintptr_t)pvPtr,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+	PVR_UNREFERENCED_PARAMETER(pvPtr);
+
+	kunmap((struct page*) psMemHandle->u.pvHandle);
+}
+
+PVRSRV_ERROR OSPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_UINT32 uiOffset,
+                                   IMG_UINT32 uiLength)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	struct page* psPage = (struct page*) psMemHandle->u.pvHandle;
+
+	void* pvVirtAddrStart = kmap(psPage) + uiOffset;
+	IMG_CPU_PHYADDR sPhysStart, sPhysEnd;
+
+	IMG_UINT32 ui32Order;
+
+	if (uiLength == 0)
+	{
+		goto e0;
+	}
+
+	ui32Order = psMemHandle->uiOrder;
+	if ((uiOffset + uiLength) > ((1 << ui32Order) * PAGE_SIZE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid size params, uiOffset %u, uiLength %u",
+				__func__,
+				uiOffset,
+				uiLength));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	sPhysStart.uiAddr = page_to_phys(psPage) + uiOffset;
+	sPhysEnd.uiAddr = sPhysStart.uiAddr + uiLength;
+
+	CacheOpExec(psDevNode,
+				pvVirtAddrStart,
+				pvVirtAddrStart + uiLength,
+				sPhysStart,
+				sPhysEnd,
+				PVRSRV_CACHE_OP_CLEAN);
+
+e0:
+	kunmap(psPage);
+
+	return eError;
+}
+
+#if defined(__GNUC__)
+#define PVRSRV_MEM_ALIGN __attribute__ ((aligned (0x8)))
+#define PVRSRV_MEM_ALIGN_MASK (0x7)
+#else
+#error "PVRSRV Alignment macros need to be defined for this compiler"
+#endif
+
+IMG_UINT32 OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE eCacheAttribute)
+{
+	IMG_UINT32 uiSize = 0;
+
+	switch (eCacheAttribute)
+	{
+		case OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE:
+			uiSize = cache_line_size();
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid cache attribute type %d",
+					__func__, (IMG_UINT32)eCacheAttribute));
+			PVR_ASSERT(0);
+			break;
+	}
+
+	return uiSize;
+}
+
+IMG_INT OSMemCmp(void *pvBufA, void *pvBufB, size_t uiLen)
+{
+	return (IMG_INT)memcmp(pvBufA, pvBufB, uiLen);
+}
+
+size_t OSStringLCat(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDstSize)
+{
+	/*
+	 * Let strlcat handle any truncation cases correctly.
+	 * We will definitely get a NUL-terminated string set in pszDest
+	 */
+	size_t uSrcSize = strlcat(pszDest, pszSrc, uDstSize);
+
+#if defined(PVR_DEBUG_STRLCPY) && defined(DEBUG)
+	/* Handle truncation by dumping calling stack if debug allows */
+	if (uSrcSize >= uDstSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+			"%s: String truncated Src = '<%s>' %ld bytes, Dest = '%s'",
+			__func__, pszSrc, (long)uDstSize, pszDest));
+		OSDumpStack();
+	}
+#endif /* defined(PVR_DEBUG_STRLCPY) && defined(DEBUG) */
+
+	return uSrcSize;
+}
+
+IMG_INT32 OSSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR *pszFormat, ...)
+{
+	va_list argList;
+	IMG_INT32 iCount;
+
+	va_start(argList, pszFormat);
+	iCount = vsnprintf(pStr, (size_t)ui32Size, pszFormat, argList);
+	va_end(argList);
+
+	return iCount;
+}
+
+IMG_INT32 OSVSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR* pszFormat, va_list vaArgs)
+{
+	return vsnprintf(pStr, ui32Size, pszFormat, vaArgs);
+}
+
+size_t OSStringLength(const IMG_CHAR *pStr)
+{
+	return strlen(pStr);
+}
+
+size_t OSStringNLength(const IMG_CHAR *pStr, size_t uiCount)
+{
+	return strnlen(pStr, uiCount);
+}
+
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                          size_t uiSize)
+{
+#if defined(DEBUG)
+	/* Double-check that we are not passing NULL parameters in. If we are we
+	 * return -1 (for arg1 == NULL, arg2 != NULL)
+	 * 0 (for arg1 == NULL, arg2 == NULL
+	 * +1 (for arg1 != NULL, arg2 == NULL)
+	 * strncmp(arg1, arg2, size) otherwise
+	 */
+	if (pStr1 == NULL)
+	{
+		if (pStr2 == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): Both args NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return 0;	/* Both NULL */
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): arg1 NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return -1;	/* NULL < non-NULL */
+		}
+	}
+	else
+	{
+		if (pStr2 == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s(%p, %p, %d): arg2 NULL",
+				 __func__, pStr1, pStr2, (int)uiSize));
+			OSDumpStack();
+			return +1;	/* non-NULL > NULL */
+		}
+		else
+		{
+			return strncmp(pStr1, pStr2, uiSize);
+		}
+	}
+#else
+	return strncmp(pStr1, pStr2, uiSize);
+#endif
+}
+
+PVRSRV_ERROR OSStringToUINT32(const IMG_CHAR *pStr, IMG_UINT32 ui32Base,
+                              IMG_UINT32 *ui32Result)
+{
+	if (kstrtou32(pStr, ui32Base, ui32Result) != 0)
+		return PVRSRV_ERROR_CONVERSION_FAILED;
+
+	return PVRSRV_OK;
+}
+
+IMG_UINT32 OSStringUINT32ToStr(IMG_CHAR *pszBuf, size_t uSize,
+							IMG_UINT32 ui32Num)
+{
+	IMG_UINT32 ui32i, ui32Len = 0, ui32NumCopy = ui32Num;
+
+	/* calculate string length required to hold the number string */
+	do
+	{
+		ui32Len++;
+		ui32NumCopy /= 10;
+	} while (ui32NumCopy != 0);
+
+	if (unlikely(ui32Len >= uSize))
+	{
+		/* insufficient buffer */
+		return 0;
+	}
+
+	for (ui32i = 0; ui32i < ui32Len; ui32i++)
+	{
+		pszBuf[ui32Len - (ui32i + 1)] = '0' + ui32Num % 10;
+		ui32Num = ui32Num / 10;
+	}
+
+	pszBuf[ui32Len] = '\0';
+	return ui32Len;
+}
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+static struct workqueue_struct *gpFenceStatusWq;
+static struct workqueue_struct *gpFenceCtxDestroyWq;
+
+static PVRSRV_ERROR _NativeSyncInit(void)
+{
+	gpFenceStatusWq = create_freezable_workqueue("pvr_fence_status");
+	if (!gpFenceStatusWq)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create foreign fence status workqueue",
+				 __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	gpFenceCtxDestroyWq = create_freezable_workqueue("pvr_fence_context_destroy");
+	if (!gpFenceCtxDestroyWq)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create foreign fence context destruction workqueue",
+				 __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static void _NativeSyncDeinit(void)
+{
+	destroy_workqueue(gpFenceStatusWq);
+	destroy_workqueue(gpFenceCtxDestroyWq);
+}
+
+struct workqueue_struct *NativeSyncGetFenceStatusWq(void)
+{
+	if (!gpFenceStatusWq)
+	{
+#if defined(DEBUG)
+		PVR_ASSERT(gpFenceStatusWq);
+#endif
+		return NULL;
+	}
+
+	return gpFenceStatusWq;
+}
+
+struct workqueue_struct *NativeSyncGetFenceCtxDestroyWq(void)
+{
+	if (!gpFenceCtxDestroyWq)
+	{
+#if defined(DEBUG)
+		PVR_ASSERT(gpFenceCtxDestroyWq);
+#endif
+		return NULL;
+	}
+
+	return gpFenceCtxDestroyWq;
+}
+#endif
+
+PVRSRV_ERROR OSInitEnvData(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = LinuxInitPhysmem();
+	PVR_GOTO_IF_ERROR(eError, error_out);
+
+	_OSInitThreadList();
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	eError = _NativeSyncInit();
+#endif
+
+error_out:
+	return eError;
+}
+
+void OSDeInitEnvData(void)
+{
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+	_NativeSyncDeinit();
+#endif
+
+	LinuxDeinitPhysmem();
+}
+
+void OSReleaseThreadQuanta(void)
+{
+	schedule();
+}
+
+void OSMemoryBarrier(volatile void *hReadback)
+{
+	mb();
+
+	if (hReadback)
+	{
+		/* Force a read-back to memory to avoid posted writes on certain buses
+		 * e.g. PCI(E)
+		 */
+		(void) OSReadDeviceMem32(hReadback);
+	}
+}
+
+void OSWriteMemoryBarrier(volatile void *hReadback)
+{
+	wmb();
+
+	if (hReadback)
+	{
+		/* Force a read-back to memory to avoid posted writes on certain buses
+		 * e.g. PCI(E)
+		 */
+		(void) OSReadDeviceMem32(hReadback);
+	}
+}
+
+/* Not matching/aligning this API to the Clockus() API above to avoid necessary
+ * multiplication/division operations in calling code.
+ */
+static inline IMG_UINT64 Clockns64(void)
+{
+	IMG_UINT64 timenow;
+
+	/* Kernel thread preempt protection. Some architecture implementations
+	 * (ARM) of sched_clock are not preempt safe when the kernel is configured
+	 * as such e.g. CONFIG_PREEMPT and others.
+	 */
+	preempt_disable();
+
+	/* Using sched_clock instead of ktime_get since we need a time stamp that
+	 * correlates with that shown in kernel logs and trace data not one that
+	 * is a bit behind. */
+	timenow = sched_clock();
+
+	preempt_enable();
+
+	return timenow;
+}
+
+IMG_UINT64 OSClockns64(void)
+{
+	return Clockns64();
+}
+
+IMG_UINT64 OSClockus64(void)
+{
+	IMG_UINT64 timenow = Clockns64();
+	IMG_UINT32 remainder;
+
+	return OSDivide64r64(timenow, 1000, &remainder);
+}
+
+IMG_UINT32 OSClockus(void)
+{
+	return (IMG_UINT32) OSClockus64();
+}
+
+IMG_UINT32 OSClockms(void)
+{
+	IMG_UINT64 timenow = Clockns64();
+	IMG_UINT32 remainder;
+
+	return OSDivide64(timenow, 1000000, &remainder);
+}
+
+static inline IMG_UINT64 KClockns64(void)
+{
+	ktime_t sTime = ktime_get();
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return sTime;
+#else
+	return sTime.tv64;
+#endif
+}
+
+PVRSRV_ERROR OSClockMonotonicns64(IMG_UINT64 *pui64Time)
+{
+	*pui64Time = KClockns64();
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSClockMonotonicus64(IMG_UINT64 *pui64Time)
+{
+	IMG_UINT64 timenow = KClockns64();
+	IMG_UINT32 remainder;
+
+	*pui64Time = OSDivide64r64(timenow, 1000, &remainder);
+	return PVRSRV_OK;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+IMG_UINT64 OSClockMonotonicRawns64(void)
+{
+	struct timespec64 ts;
+
+	ktime_get_raw_ts64(&ts);
+	return ts.tv_sec * 1000000000 + ts.tv_nsec;
+}
+#else
+IMG_UINT64 OSClockMonotonicRawns64(void)
+{
+	struct timespec ts;
+
+	getrawmonotonic(&ts);
+	return (IMG_UINT64) ts.tv_sec * 1000000000 + ts.tv_nsec;
+}
+#endif
+
+IMG_UINT64 OSClockMonotonicRawus64(void)
+{
+	IMG_UINT32 rem;
+	return OSDivide64r64(OSClockMonotonicRawns64(), 1000, &rem);
+}
+
+/*
+	OSWaitus
+*/
+void OSWaitus(IMG_UINT32 ui32Timeus)
+{
+	udelay(ui32Timeus);
+}
+
+
+/*
+	OSSleepms
+*/
+void OSSleepms(IMG_UINT32 ui32Timems)
+{
+	msleep(ui32Timems);
+}
+
+
+INLINE IMG_UINT64 OSGetCurrentProcessVASpaceSize(void)
+{
+	return (IMG_UINT64)TASK_SIZE;
+}
+
+INLINE IMG_PID OSGetCurrentProcessID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return (IMG_PID)task_tgid_nr(current);
+}
+
+INLINE IMG_PID OSGetCurrentVirtualProcessID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return (IMG_PID)task_tgid_vnr(current);
+}
+
+INLINE IMG_CHAR *OSGetCurrentProcessName(void)
+{
+	return current->comm;
+}
+
+INLINE uintptr_t OSGetCurrentThreadID(void)
+{
+	if (in_interrupt())
+	{
+		return KERNEL_ID;
+	}
+
+	return current->pid;
+}
+
+IMG_PID OSGetCurrentClientProcessIDKM(void)
+{
+	return OSGetCurrentProcessID();
+}
+
+IMG_CHAR *OSGetCurrentClientProcessNameKM(void)
+{
+	return OSGetCurrentProcessName();
+}
+
+uintptr_t OSGetCurrentClientThreadIDKM(void)
+{
+	return OSGetCurrentThreadID();
+}
+
+size_t OSGetPageSize(void)
+{
+	return PAGE_SIZE;
+}
+
+size_t OSGetPageShift(void)
+{
+	return PAGE_SHIFT;
+}
+
+size_t OSGetPageMask(void)
+{
+	return (OSGetPageSize()-1);
+}
+
+size_t OSGetOrder(size_t uSize)
+{
+	return get_order(PAGE_ALIGN(uSize));
+}
+
+IMG_UINT64 OSGetRAMSize(void)
+{
+	struct sysinfo SI;
+	si_meminfo(&SI);
+
+	return (PAGE_SIZE * SI.totalram);
+}
+
+typedef struct
+{
+	int os_error;
+	PVRSRV_ERROR pvr_error;
+} error_map_t;
+
+#define PVRSRV_ERROR_TO_OS_ERROR \
+	X(PVRSRV_OK, 0) \
+	X(PVRSRV_ERROR_BRIDGE_EFAULT, EFAULT) \
+	X(PVRSRV_ERROR_BRIDGE_EINVAL, EINVAL) \
+	X(PVRSRV_ERROR_BRIDGE_ENOMEM, ENOMEM) \
+	X(PVRSRV_ERROR_BRIDGE_ERANGE, ERANGE) \
+	X(PVRSRV_ERROR_BRIDGE_EPERM, EPERM) \
+	X(PVRSRV_ERROR_BRIDGE_ENOTTY, ENOTTY) \
+	X(PVRSRV_ERROR_BRIDGE_CALL_FAILED, ENOTTY) \
+	X(PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL, ERANGE) \
+	X(PVRSRV_ERROR_OUT_OF_MEMORY, ENOMEM) \
+	X(PVRSRV_ERROR_PMR_NOT_PERMITTED, EACCES) \
+	X(PVRSRV_ERROR_INVALID_PARAMS, EINVAL) \
+	X(PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING, EPERM) \
+	X(PVRSRV_ERROR_NOT_IMPLEMENTED, ENOSYS) \
+	X(PVRSRV_ERROR_BAD_MAPPING, EINVAL)
+
+/* return -ve versions of POSIX errors as they are used in this form */
+int PVRSRVToNativeError(PVRSRV_ERROR eError)
+{
+	switch (eError)
+	{
+#define X(_PVRSRV_ERROR, _OS_ERROR) \
+	case (_PVRSRV_ERROR): return -(_OS_ERROR);
+
+		PVRSRV_ERROR_TO_OS_ERROR
+
+#undef X
+		default:
+			return -EFAULT;
+	}
+}
+
+typedef struct  _MISR_DATA_ {
+	struct workqueue_struct *psWorkQueue;
+	struct work_struct sMISRWork;
+	const IMG_CHAR* pszName;
+	PFN_MISR pfnMISR;
+	void *hData;
+} MISR_DATA;
+
+/*
+	MISRWrapper
+*/
+static void MISRWrapper(struct work_struct *data)
+{
+	MISR_DATA *psMISRData = container_of(data, MISR_DATA, sMISRWork);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Waking up '%s' MISR %p", psMISRData->pszName, psMISRData));
+
+	psMISRData->pfnMISR(psMISRData->hData);
+}
+
+/*
+	OSInstallMISR
+*/
+PVRSRV_ERROR OSInstallMISR(IMG_HANDLE *hMISRData, PFN_MISR pfnMISR,
+							void *hData, const IMG_CHAR *pszMisrName)
+{
+	MISR_DATA *psMISRData;
+
+	psMISRData = OSAllocMem(sizeof(*psMISRData));
+	PVR_LOG_RETURN_IF_NOMEM(psMISRData, "psMISRData");
+
+	psMISRData->hData = hData;
+	psMISRData->pfnMISR = pfnMISR;
+	psMISRData->pszName = pszMisrName;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Installing MISR with cookie %p", psMISRData));
+
+	psMISRData->psWorkQueue = create_singlethread_workqueue("pvr_misr");
+
+	if (psMISRData->psWorkQueue == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSInstallMISR: create_singlethreaded_workqueue failed"));
+		OSFreeMem(psMISRData);
+		return PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD;
+	}
+
+	INIT_WORK(&psMISRData->sMISRWork, MISRWrapper);
+
+	*hMISRData = (IMG_HANDLE) psMISRData;
+
+	return PVRSRV_OK;
+}
+
+/*
+	OSUninstallMISR
+*/
+PVRSRV_ERROR OSUninstallMISR(IMG_HANDLE hMISRData)
+{
+	MISR_DATA *psMISRData = (MISR_DATA *) hMISRData;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Uninstalling MISR with cookie %p", psMISRData));
+
+	destroy_workqueue(psMISRData->psWorkQueue);
+	OSFreeMem(psMISRData);
+
+	return PVRSRV_OK;
+}
+
+/*
+	OSScheduleMISR
+*/
+PVRSRV_ERROR OSScheduleMISR(IMG_HANDLE hMISRData)
+{
+	MISR_DATA *psMISRData = (MISR_DATA *) hMISRData;
+
+	/*
+		Note:
+
+		In the case of NO_HARDWARE we want the driver to be synchronous so
+		that we don't have to worry about waiting for previous operations
+		to complete
+	*/
+#if defined(NO_HARDWARE)
+	psMISRData->pfnMISR(psMISRData->hData);
+	return PVRSRV_OK;
+#else
+	{
+		bool rc = queue_work(psMISRData->psWorkQueue, &psMISRData->sMISRWork);
+		return rc ? PVRSRV_OK : PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+#endif
+}
+
+void OSSyncIRQ(IMG_UINT32 ui32IRQ)
+{
+	synchronize_irq(ui32IRQ);
+}
+
+/* OS specific values for thread priority */
+static const IMG_INT32 ai32OSPriorityValues[OS_THREAD_LAST_PRIORITY] =
+{
+	  0, /* OS_THREAD_NOSET_PRIORITY */
+	-20, /* OS_THREAD_HIGHEST_PRIORITY */
+	-10, /* OS_THREAD_HIGH_PRIORITY */
+	  0, /* OS_THREAD_NORMAL_PRIORITY */
+	  9, /* OS_THREAD_LOW_PRIORITY */
+	 19, /* OS_THREAD_LOWEST_PRIORITY */
+};
+
+static int OSThreadRun(void *data)
+{
+	OSThreadData *psOSThreadData = data;
+
+	/* count freezable threads */
+	LinuxBridgeNumActiveKernelThreadsIncrement();
+
+	/* Returns true if the thread was frozen, should we do anything with this
+	 * information? What do we return? Which one is the error case? */
+	set_freezable();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Starting Thread '%s'...", psOSThreadData->pszThreadName));
+
+	/* Call the client's kernel thread with the client's data pointer */
+	psOSThreadData->pfnThread(psOSThreadData->hData);
+
+	if (psOSThreadData->bIsSupportingThread)
+	{
+		_ThreadSetStopped(psOSThreadData);
+	}
+
+	/* Wait for OSThreadDestroy() to call kthread_stop() */
+	while (!kthread_freezable_should_stop(NULL))
+	{
+		schedule();
+	}
+
+	LinuxBridgeNumActiveKernelThreadsDecrement();
+
+	return 0;
+}
+
+PVRSRV_ERROR OSThreadCreate(IMG_HANDLE *phThread,
+                            IMG_CHAR *pszThreadName,
+                            PFN_THREAD pfnThread,
+                            PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                            IMG_BOOL bIsSupportingThread,
+                            void *hData)
+{
+	return OSThreadCreatePriority(phThread, pszThreadName, pfnThread,
+	                              pfnDebugDumpCB, bIsSupportingThread, hData,
+	                              OS_THREAD_NOSET_PRIORITY);
+}
+
+PVRSRV_ERROR OSThreadCreatePriority(IMG_HANDLE *phThread,
+                                    IMG_CHAR *pszThreadName,
+                                    PFN_THREAD pfnThread,
+                                    PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                                    IMG_BOOL bIsSupportingThread,
+                                    void *hData,
+                                    OS_THREAD_LEVEL eThreadPriority)
+{
+	OSThreadData *psOSThreadData;
+	PVRSRV_ERROR eError;
+
+	psOSThreadData = OSAllocZMem(sizeof(*psOSThreadData));
+	PVR_LOG_GOTO_IF_NOMEM(psOSThreadData, eError, fail_alloc);
+
+	psOSThreadData->pfnThread = pfnThread;
+	psOSThreadData->hData = hData;
+	psOSThreadData->pszThreadName = pszThreadName;
+	psOSThreadData->kthread = kthread_run(OSThreadRun, psOSThreadData, "%s", pszThreadName);
+
+	if (IS_ERR(psOSThreadData->kthread))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_kthread;
+	}
+
+	if (bIsSupportingThread)
+	{
+		psOSThreadData->pfnDebugDumpCB = pfnDebugDumpCB;
+		psOSThreadData->bIsThreadRunning = IMG_TRUE;
+		psOSThreadData->bIsSupportingThread = IMG_TRUE;
+
+		_ThreadListAddEntry(psOSThreadData);
+	}
+
+	if (eThreadPriority != OS_THREAD_NOSET_PRIORITY &&
+	    eThreadPriority < OS_THREAD_LAST_PRIORITY)
+	{
+		set_user_nice(psOSThreadData->kthread,
+		              ai32OSPriorityValues[eThreadPriority]);
+	}
+
+	*phThread = psOSThreadData;
+
+	return PVRSRV_OK;
+
+fail_kthread:
+	OSFreeMem(psOSThreadData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR OSThreadDestroy(IMG_HANDLE hThread)
+{
+	OSThreadData *psOSThreadData = hThread;
+	int ret;
+
+	/* Let the thread know we are ready for it to end and wait for it. */
+	ret = kthread_stop(psOSThreadData->kthread);
+	if (0 != ret)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "kthread_stop failed(%d)", ret));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	if (psOSThreadData->bIsSupportingThread)
+	{
+		_ThreadListRemoveEntry(psOSThreadData);
+	}
+
+	OSFreeMem(psOSThreadData);
+
+	return PVRSRV_OK;
+}
+
+void OSPanic(void)
+{
+	BUG();
+
+#if defined(__KLOCWORK__)
+	/* Klocwork does not understand that BUG is terminal... */
+	abort();
+#endif
+}
+
+IMG_BOOL OSIsMapPhysNonContigSupported(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) && !defined(CONFIG_VMAP_PFN)
+	return IMG_FALSE;
+#else
+	return IMG_TRUE;
+#endif
+}
+
+void OSUnMapPhysArrayToLin(void *pvLinAddr, void *pvPrivData)
+{
+	if (is_vmalloc_addr(pvLinAddr))
+	{
+#if defined(CONFIG_VMAP_PFN)
+		PVR_UNREFERENCED_PARAMETER(pvPrivData);
+		vunmap(pvLinAddr);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+		unmap_kernel_range((unsigned long) (uintptr_t) pvLinAddr,
+                           get_vm_area_size(pvPrivData));
+		free_vm_area(pvPrivData);
+#else
+		PVR_DPF((PVR_DBG_ERROR,"%s: Cannot map into kernel, no method supported.", __func__));
+		PVR_ASSERT(0);
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Given kernel address is not a vmalloc addr", __func__));
+	}
+}
+
+#define PagePAToPFN(PageAddr) (PageAddr >> PAGE_SHIFT)
+
+PVRSRV_ERROR OSMapPhysArrayToLin(IMG_CPU_PHYADDR pPagePA[],
+                                 IMG_UINT32 uiPagesCount,
+                                 void **ppvLinAddr,
+                                 void **ppvPrivData)
+{
+	if (ppvLinAddr == NULL || ppvPrivData == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if defined(CONFIG_VMAP_PFN)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < uiPagesCount; i++)
+		{
+			pPagePA[i].uiAddr = PagePAToPFN(pPagePA[i].uiAddr);
+		}
+
+		*ppvLinAddr = vmap_pfn((unsigned long *)pPagePA,
+							   (unsigned int)uiPagesCount,
+							   pgprot_device(PAGE_KERNEL));
+		if (NULL == *ppvLinAddr)
+		{
+			return PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+		}
+		*ppvPrivData = NULL;
+		return PVRSRV_OK;
+	}
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	{
+		pte_t *pte[32], **pte_array;
+		struct vm_struct *psVMA;
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_UINT32 i = 0;
+
+		pte_array = &pte[0];
+		if (sizeof(pte) < (sizeof(pte[0]) * uiPagesCount))
+		{
+			pte_array = kzalloc(uiPagesCount * sizeof(*pte_array), GFP_KERNEL);
+			if (NULL == pte_array)
+			{
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+		}
+
+		psVMA = alloc_vm_area((size_t)(uiPagesCount << PAGE_SHIFT), pte_array);
+		if (NULL == psVMA)
+		{
+			eError = PVRSRV_ERROR_FAILED_TO_ALLOC_VIRT_MEMORY;
+			goto FreePTEArray;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Allocated VM: %s VMA: %p Addr: %p Size: 0x%lx count: %d", __func__,
+				psVMA, psVMA->addr, psVMA->size, psVMA->nr_pages));
+
+		for (i = 0; i < uiPagesCount; i++)
+		{
+			*(pte_array[i]) = pte_mkspecial(pfn_pte((unsigned long) PagePAToPFN(pPagePA[i].uiAddr),
+													pgprot_device(PAGE_KERNEL)));
+		}
+
+		OSWriteMemoryBarrier(psVMA->addr);
+
+		*ppvLinAddr = psVMA->addr;
+		*ppvPrivData = psVMA;
+
+FreePTEArray:
+		if (pte_array != pte)
+		{
+			kfree(pte_array);
+		}
+
+		return eError;
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR,"%s: Cannot map into kernel, no method supported.", __func__));
+	PVR_ASSERT(0);
+	*ppvLinAddr = NULL;
+	return PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+#endif
+}
+
+void *
+OSMapPhysToLin(IMG_CPU_PHYADDR BasePAddr,
+			   size_t ui32Bytes,
+			   PVRSRV_MEMALLOCFLAGS_T uiMappingFlags)
+{
+	void __iomem *pvLinAddr;
+
+	if (uiMappingFlags & ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK))
+	{
+		PVR_ASSERT(!"Found non-cpu cache mode flag when mapping to the cpu");
+		return NULL;
+	}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/*
+	  This is required to support DMA physheaps for GPU virtualization.
+	  Unfortunately, if a region of kernel managed memory is turned into
+	  a DMA buffer, conflicting mappings can come about easily on Linux
+	  as the original memory is mapped by the kernel as normal cached
+	  memory whilst DMA buffers are mapped mostly as uncached device or
+	  cache-coherent device memory. In both cases the system will have
+	  two conflicting mappings for the same memory region and will have
+	  "undefined behaviour" for most processors notably ARMv6 onwards
+	  and some x86 micro-architectures. As a result, perform ioremapping
+	  manually for DMA physheap allocations by translating from CPU/VA
+	  to BUS/PA thereby preventing the creation of conflicting mappings.
+	*/
+	pvLinAddr = (void __iomem *) SysDmaDevPAddrToCpuVAddr(BasePAddr.uiAddr, ui32Bytes);
+	if (pvLinAddr != NULL)
+	{
+		return (void __force *) pvLinAddr;
+	}
+#endif
+
+	switch (uiMappingFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+#if defined(CONFIG_X86) || defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+			pvLinAddr = (void __iomem *)ioremap_wc(BasePAddr.uiAddr, ui32Bytes);
+#else
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+#endif
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+#if defined(CONFIG_X86) || defined(CONFIG_ARM)
+			pvLinAddr = (void __iomem *)ioremap_cache(BasePAddr.uiAddr, ui32Bytes);
+#else
+			pvLinAddr = (void __iomem *)ioremap(BasePAddr.uiAddr, ui32Bytes);
+#endif
+			break;
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT:
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT:
+			PVR_ASSERT(!"Unexpected cpu cache mode");
+			pvLinAddr = NULL;
+			break;
+		default:
+			PVR_ASSERT(!"Unsupported cpu cache mode");
+			pvLinAddr = NULL;
+			break;
+	}
+
+	return (void __force *) pvLinAddr;
+}
+
+
+IMG_BOOL
+OSUnMapPhysToLin(void *pvLinAddr, size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32Bytes);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (SysDmaCpuVAddrToDevPAddr(pvLinAddr))
+	{
+		return IMG_TRUE;
+	}
+#endif
+
+	iounmap((void __iomem *) pvLinAddr);
+
+	return IMG_TRUE;
+}
+
+#define OS_MAX_TIMERS	8
+
+/* Timer callback structure used by OSAddTimer */
+typedef struct TIMER_CALLBACK_DATA_TAG
+{
+	IMG_BOOL			bInUse;
+	PFN_TIMER_FUNC		pfnTimerFunc;
+	void				*pvData;
+	struct timer_list	sTimer;
+	IMG_UINT32			ui32Delay;
+	IMG_BOOL			bActive;
+	struct work_struct	sWork;
+}TIMER_CALLBACK_DATA;
+
+static struct workqueue_struct *psTimerWorkQueue;
+
+static TIMER_CALLBACK_DATA sTimers[OS_MAX_TIMERS];
+
+static DEFINE_MUTEX(sTimerStructLock);
+
+static void OSTimerCallbackBody(TIMER_CALLBACK_DATA *psTimerCBData)
+{
+	if (!psTimerCBData->bActive)
+		return;
+
+	/* call timer callback */
+	psTimerCBData->pfnTimerFunc(psTimerCBData->pvData);
+
+	/* reset timer */
+	mod_timer(&psTimerCBData->sTimer, psTimerCBData->sTimer.expires + psTimerCBData->ui32Delay);
+}
+
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+/*************************************************************************/ /*!
+@Function       OSTimerCallbackWrapper
+@Description    OS specific timer callback wrapper function
+@Input          psTimer    Timer list structure
+*/ /**************************************************************************/
+static void OSTimerCallbackWrapper(struct timer_list *psTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = timer_container_of(psTimerCBData, psTimer, sTimer);
+#else
+/*************************************************************************/ /*!
+@Function       OSTimerCallbackWrapper
+@Description    OS specific timer callback wrapper function
+@Input          uData    Timer callback data
+*/ /**************************************************************************/
+static void OSTimerCallbackWrapper(uintptr_t uData)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = (TIMER_CALLBACK_DATA*)uData;
+#endif
+	int res;
+
+	res = queue_work(psTimerWorkQueue, &psTimerCBData->sWork);
+	if (res == 0)
+	{
+		PVR_LOG(("OSTimerCallbackWrapper: work already queued"));
+	}
+}
+
+
+static void OSTimerWorkQueueCallBack(struct work_struct *psWork)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = container_of(psWork, TIMER_CALLBACK_DATA, sWork);
+
+	OSTimerCallbackBody(psTimerCBData);
+}
+
+IMG_HANDLE OSAddTimer(PFN_TIMER_FUNC pfnTimerFunc, void *pvData, IMG_UINT32 ui32MsTimeout)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData;
+	IMG_UINT32		ui32i;
+
+	/* check callback */
+	if (!pfnTimerFunc)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAddTimer: passed invalid callback"));
+		return NULL;
+	}
+
+	/* Allocate timer callback data structure */
+	mutex_lock(&sTimerStructLock);
+	for (ui32i = 0; ui32i < OS_MAX_TIMERS; ui32i++)
+	{
+		psTimerCBData = &sTimers[ui32i];
+		if (!psTimerCBData->bInUse)
+		{
+			psTimerCBData->bInUse = IMG_TRUE;
+			break;
+		}
+	}
+	mutex_unlock(&sTimerStructLock);
+	if (ui32i >= OS_MAX_TIMERS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAddTimer: all timers are in use"));
+		return NULL;
+	}
+
+	psTimerCBData->pfnTimerFunc = pfnTimerFunc;
+	psTimerCBData->pvData = pvData;
+	psTimerCBData->bActive = IMG_FALSE;
+
+	/*
+		HZ = ticks per second
+		ui32MsTimeout = required ms delay
+		ticks = (Hz * ui32MsTimeout) / 1000
+	*/
+	psTimerCBData->ui32Delay = ((HZ * ui32MsTimeout) < 1000)
+								?	1
+								:	((HZ * ui32MsTimeout) / 1000);
+
+	/* initialise object */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0))
+	timer_setup(&psTimerCBData->sTimer, OSTimerCallbackWrapper, 0);
+#else
+	init_timer(&psTimerCBData->sTimer);
+
+	/* setup timer object */
+	psTimerCBData->sTimer.function = (void *)OSTimerCallbackWrapper;
+	psTimerCBData->sTimer.data = (uintptr_t)psTimerCBData;
+#endif
+
+	return (IMG_HANDLE)(uintptr_t)(ui32i + 1);
+}
+
+
+static inline TIMER_CALLBACK_DATA *GetTimerStructure(IMG_HANDLE hTimer)
+{
+	IMG_UINT32 ui32i = (IMG_UINT32)((uintptr_t)hTimer) - 1;
+
+	PVR_ASSERT(ui32i < OS_MAX_TIMERS);
+
+	return &sTimers[ui32i];
+}
+
+PVRSRV_ERROR OSRemoveTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(!psTimerCBData->bActive);
+
+	/* free timer callback data struct */
+	psTimerCBData->bInUse = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSEnableTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(!psTimerCBData->bActive);
+
+	/* Start timer arming */
+	psTimerCBData->bActive = IMG_TRUE;
+
+	/* set the expire time */
+	psTimerCBData->sTimer.expires = psTimerCBData->ui32Delay + jiffies;
+
+	/* Add the timer to the list */
+	add_timer(&psTimerCBData->sTimer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR OSDisableTimer (IMG_HANDLE hTimer)
+{
+	TIMER_CALLBACK_DATA *psTimerCBData = GetTimerStructure(hTimer);
+
+	PVR_ASSERT(psTimerCBData->bInUse);
+	PVR_ASSERT(psTimerCBData->bActive);
+
+	/* Stop timer from arming */
+	psTimerCBData->bActive = IMG_FALSE;
+	smp_mb();
+
+	flush_workqueue(psTimerWorkQueue);
+
+	/* remove timer */
+	timer_delete_sync(&psTimerCBData->sTimer);
+
+	/*
+	 * This second flush is to catch the case where the timer ran
+	 * before we managed to delete it, in which case, it will have
+	 * queued more work for the workqueue. Since the bActive flag
+	 * has been cleared, this second flush won't result in the
+	 * timer being rearmed.
+	 */
+	flush_workqueue(psTimerWorkQueue);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR OSEventObjectCreate(const IMG_CHAR *pszName, IMG_HANDLE *hEventObject)
+{
+	PVR_UNREFERENCED_PARAMETER(pszName);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectListCreate(hEventObject);
+}
+
+
+PVRSRV_ERROR OSEventObjectDestroy(IMG_HANDLE hEventObject)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectListDestroy(hEventObject);
+}
+
+PVRSRV_ERROR OSEventObjectWaitTimeout(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus)
+{
+	PVRSRV_ERROR eError;
+
+	if (hOSEventKM && uiTimeoutus > 0)
+	{
+		eError = LinuxEventObjectWait(hOSEventKM, uiTimeoutus, _NON_FREEZABLE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSEventObjectWait: invalid arguments %p, %lld", hOSEventKM, uiTimeoutus));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR OSEventObjectWait(IMG_HANDLE hOSEventKM)
+{
+	return OSEventObjectWaitTimeout(hOSEventKM, EVENT_OBJECT_TIMEOUT_US);
+}
+
+PVRSRV_ERROR OSEventObjectWaitKernel(IMG_HANDLE hOSEventKM,
+                                     IMG_UINT64 uiTimeoutus)
+{
+	PVRSRV_ERROR eError;
+
+	if (hOSEventKM != NULL && uiTimeoutus > 0)
+	{
+		eError = LinuxEventObjectWait(hOSEventKM, uiTimeoutus, _FREEZABLE);
+	}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	else if (hOSEventKM != NULL)
+	{
+		eError = LinuxEventObjectWaitUntilSignalled(hOSEventKM);
+	}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSEventObjectWaitKernel: invalid arguments %p",
+		        hOSEventKM));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eError;
+}
+
+void OSEventObjectDumpDebugInfo(IMG_HANDLE hOSEventKM)
+{
+	LinuxEventObjectDumpDebugInfo(hOSEventKM);
+}
+
+PVRSRV_ERROR OSEventObjectOpen(IMG_HANDLE hEventObject, IMG_HANDLE *phOSEvent)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phOSEvent, "phOSEvent");
+	PVR_LOG_GOTO_IF_INVALID_PARAM(hEventObject, eError, error);
+
+	eError = LinuxEventObjectAdd(hEventObject, phOSEvent);
+	PVR_LOG_GOTO_IF_ERROR(eError, "LinuxEventObjectAdd", error);
+
+	return PVRSRV_OK;
+
+error:
+	*phOSEvent = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR OSEventObjectClose(IMG_HANDLE hOSEventKM)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hOSEventKM, "hOSEventKM");
+
+	return LinuxEventObjectDelete(hOSEventKM);
+}
+
+PVRSRV_ERROR OSEventObjectSignal(IMG_HANDLE hEventObject)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hEventObject, "hEventObject");
+
+	return LinuxEventObjectSignal(hEventObject);
+}
+
+PVRSRV_ERROR OSCopyToUser(void *pvProcess,
+						  void __user *pvDest,
+						  const void *pvSrc,
+						  size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(pvProcess);
+
+	if (pvr_copy_to_user(pvDest, pvSrc, ui32Bytes)==0)
+		return PVRSRV_OK;
+	else
+		return PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY;
+}
+
+PVRSRV_ERROR OSCopyFromUser(void *pvProcess,
+							void *pvDest,
+							const void __user *pvSrc,
+							size_t ui32Bytes)
+{
+	PVR_UNREFERENCED_PARAMETER(pvProcess);
+
+	if (likely(pvr_copy_from_user(pvDest, pvSrc, ui32Bytes)==0))
+		return PVRSRV_OK;
+	else
+		return PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY;
+}
+
+IMG_UINT64 OSDivide64r64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder)
+{
+	*pui32Remainder = do_div(ui64Divident, ui32Divisor);
+
+	return ui64Divident;
+}
+
+IMG_UINT32 OSDivide64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder)
+{
+	*pui32Remainder = do_div(ui64Divident, ui32Divisor);
+
+	return (IMG_UINT32) ui64Divident;
+}
+
+/* One time osfunc initialisation */
+PVRSRV_ERROR PVROSFuncInit(void)
+{
+	{
+		PVR_ASSERT(!psTimerWorkQueue);
+
+		psTimerWorkQueue = create_freezable_workqueue("pvr_timer");
+		if (psTimerWorkQueue == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: couldn't create timer workqueue",
+					 __func__));
+			return PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD;
+		}
+	}
+
+	{
+		IMG_UINT32 ui32i;
+
+		for (ui32i = 0; ui32i < OS_MAX_TIMERS; ui32i++)
+		{
+			TIMER_CALLBACK_DATA *psTimerCBData = &sTimers[ui32i];
+
+			INIT_WORK(&psTimerCBData->sWork, OSTimerWorkQueueCallBack);
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*
+ * Osfunc deinitialisation.
+ * Note that PVROSFuncInit may not have been called
+ */
+void PVROSFuncDeInit(void)
+{
+	if (psTimerWorkQueue != NULL)
+	{
+		destroy_workqueue(psTimerWorkQueue);
+		psTimerWorkQueue = NULL;
+	}
+}
+
+void OSDumpStack(void)
+{
+	dump_stack();
+}
+
+/*************************************************************************/ /*!
+@Function       OSDebugSignalPID
+@Description    Sends a SIGTRAP signal to a specific PID in user mode for
+                debugging purposes. The user mode process can register a handler
+                against this signal.
+                This is necessary to support the Rogue debugger. If the Rogue
+                debugger is not used then this function may be implemented as
+                a stub.
+@Input          ui32PID    The PID for the signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDebugSignalPID(IMG_UINT32 ui32PID)
+{
+	int err;
+	struct pid *psPID;
+
+	psPID = find_vpid(ui32PID);
+	if (psPID == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get PID struct.", __func__));
+		return PVRSRV_ERROR_NOT_FOUND;
+	}
+
+	err = kill_pid(psPID, SIGTRAP, 0);
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Signal Failure %d", __func__, err));
+		return PVRSRV_ERROR_SIGNAL_FAILED;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSIsKernelThread
+@Description    This API determines if the current running thread is a kernel
+                thread (i.e. one not associated with any userland process,
+                typically an MISR handler.)
+@Return         IMG_TRUE if it is a kernel thread, otherwise IMG_FALSE.
+*/ /**************************************************************************/
+IMG_BOOL OSIsKernelThread(void)
+{
+	/*
+	 * Kernel threads have a NULL memory descriptor.
+	 *
+	 * See https://www.kernel.org/doc/Documentation/vm/active_mm.txt
+	 */
+	return current->mm == NULL;
+}
+
+void OSDumpVersionInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					   void *pvDumpDebugFile)
+{
+	PVR_DUMPDEBUG_LOG("OS kernel info: %s %s %s %s",
+					utsname()->sysname,
+					utsname()->release,
+					utsname()->version,
+					utsname()->machine);
+}
+#if defined(SUPPORT_DMA_TRANSFER)
+
+typedef struct _OS_CLEANUP_DATA_
+{
+	IMG_BOOL bSucceed;
+	IMG_BOOL bAdvanceTimeline;
+	IMG_UINT uiRefCount;
+	IMG_UINT uiNumDMA;
+	IMG_UINT uiCount;
+
+	struct dma_async_tx_descriptor** ppsDescriptors;
+
+
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PFN_SERVER_CLEANUP pfnServerCleanup;
+	void* pvServerCleanupData;
+
+	enum dma_data_direction eDirection;
+	struct sg_table **ppsSg;
+	struct page ***pages;
+	IMG_UINT32* puiNumPages;
+	spinlock_t spinlock;
+
+	struct completion start_cleanup;
+	struct completion *sync_completion;
+
+	/* Sparse PMR transfer information */
+	IMG_BOOL *pbIsSparse;
+	IMG_UINT *uiNumValidPages;
+	struct sg_table ***ppsSgSparse;
+	struct dma_async_tx_descriptor*** ppsDescriptorsSparse;
+
+} OS_CLEANUP_DATA;
+
+static int cleanup_thread(void *pvData)
+{
+	IMG_UINT32 i, j;
+	struct completion *sync_completion = NULL;
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvData;
+	IMG_BOOL bSucceed = psOSCleanup->bSucceed;
+
+	sync_completion = psOSCleanup->sync_completion;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Cleanup thread waiting (%p) on completion", pvData));
+#endif
+
+	wait_for_completion(&psOSCleanup->start_cleanup);
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "Cleanup thread notified (%p)", pvData));
+#endif
+	/* Free resources */
+	for (i=0; i<psOSCleanup->uiCount; i++)
+	{
+		if (!psOSCleanup->pbIsSparse[i])
+		{
+			dma_sync_sg_for_cpu(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+								psOSCleanup->ppsSg[i]->sgl,
+								psOSCleanup->ppsSg[i]->nents,
+								psOSCleanup->eDirection);
+
+			dma_unmap_sg(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+						 psOSCleanup->ppsSg[i]->sgl,
+						 psOSCleanup->ppsSg[i]->nents,
+						 psOSCleanup->eDirection);
+
+			sg_free_table(psOSCleanup->ppsSg[i]);
+
+			OSFreeMem(psOSCleanup->ppsSg[i]);
+
+			/* Unpin pages */
+			for (j=0; j<psOSCleanup->puiNumPages[i]; j++)
+			{
+				/*
+				 * using DMA_FROM_DEVICE from enum dma_data_direction instead of DMA_DEV_TO_MEM
+				 * from enum dma_transfer_direction to avoid casting explicitly
+				 */
+				if (psOSCleanup->eDirection == DMA_FROM_DEVICE)
+				{
+					set_page_dirty_lock(psOSCleanup->pages[i][j]);
+				}
+
+				pvr_unpin_user_page_for_dma(psOSCleanup->pages[i][j]);
+			}
+		}
+		else
+		{
+			for (j = 0; j < psOSCleanup->puiNumPages[i]; j++)
+			{
+				if (psOSCleanup->ppsSgSparse[i][j]) {
+					dma_sync_sg_for_cpu(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+										psOSCleanup->ppsSgSparse[i][j]->sgl,
+										psOSCleanup->ppsSgSparse[i][j]->nents,
+										psOSCleanup->eDirection);
+
+
+					dma_unmap_sg(psOSCleanup->psDevNode->psDevConfig->pvOSDevice,
+								psOSCleanup->ppsSgSparse[i][j]->sgl,
+								psOSCleanup->ppsSgSparse[i][j]->nents,
+								psOSCleanup->eDirection);
+
+					sg_free_table(psOSCleanup->ppsSgSparse[i][j]);
+
+					OSFreeMem(psOSCleanup->ppsSgSparse[i][j]);
+
+				}
+			}
+
+			OSFreeMem(psOSCleanup->ppsSgSparse[i]);
+			OSFreeMem(psOSCleanup->ppsDescriptorsSparse[i]);
+
+			/* Unpin pages */
+			for (j=0; j<psOSCleanup->puiNumPages[i]*2; j++)
+			{
+				/*
+				 * Some pages might've been pinned twice
+				 * Others may have not been pinned at all
+				 */
+				if (psOSCleanup->pages[i][j])
+				{
+					/*
+					 * using DMA_FROM_DEVICE from enum dma_data_direction instead of DMA_DEV_TO_MEM
+					 * from enum dma_transfer_direction to avoid casting explicitly
+					 */
+					if (psOSCleanup->eDirection == DMA_FROM_DEVICE)
+					{
+						set_page_dirty_lock(psOSCleanup->pages[i][j]);
+					}
+
+					pvr_unpin_user_page_for_dma(psOSCleanup->pages[i][j]);
+				}
+			}
+		}
+
+		OSFreeMem(psOSCleanup->pages[i]);
+	}
+
+	psOSCleanup->pfnServerCleanup(psOSCleanup->pvServerCleanupData,
+								  psOSCleanup->bAdvanceTimeline);
+
+	OSFreeMem(psOSCleanup->ppsSg);
+	OSFreeMem(psOSCleanup->pages);
+	OSFreeMem(psOSCleanup->puiNumPages);
+	OSFreeMem(psOSCleanup->ppsSgSparse);
+	OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+	OSFreeMem(psOSCleanup->ppsDescriptors);
+	OSFreeMem(psOSCleanup->pbIsSparse);
+	OSFreeMem(psOSCleanup->uiNumValidPages);
+	OSFreeMem(psOSCleanup);
+
+	if (bSucceed)
+	{
+		kthread_complete_and_exit(sync_completion, 0);
+	}
+	else
+	{
+		kthread_complete_and_exit(NULL, 0);
+	}
+	return 0;
+}
+
+static void dma_callback(void *pvOSCleanup)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvOSCleanup;
+	unsigned long flags;
+
+#if defined(DMA_VERBOSE)
+	PVR_DPF((PVR_DBG_ERROR, "dma_callback (%p) refcount decreased to %d", psOSCleanup, psOSCleanup->uiRefCount - 1));
+#endif
+	spin_lock_irqsave(&psOSCleanup->spinlock, flags);
+
+	psOSCleanup->uiRefCount--;
+
+	if (psOSCleanup->uiRefCount==0)
+	{
+		/* Notify the cleanup thread */
+		spin_unlock_irqrestore(&psOSCleanup->spinlock, flags);
+		complete(&psOSCleanup->start_cleanup);
+		return;
+	}
+
+	spin_unlock_irqrestore(&psOSCleanup->spinlock, flags);
+}
+
+#if defined(PVRSRV_DEBUG_DMA)
+static void
+DMADumpPhysicalAddresses(struct page **ppsHostMemPages,
+						 IMG_UINT32 uiNumPages,
+						 IMG_DMA_ADDR *sDmaAddr,
+						 IMG_UINT64 ui64Offset)
+{
+	IMG_CPU_PHYADDR sPagePhysAddr;
+	IMG_UINT32 uiIdx;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "DMA Transfer Address Dump"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Hostmem phys addresses:"));
+
+	for (uiIdx = 0; uiIdx < uiNumPages; uiIdx++)
+	{
+		sPagePhysAddr.uiAddr = page_to_phys(ppsHostMemPages[uiIdx]);
+		if (uiIdx == 0)
+		{
+			sPagePhysAddr.uiAddr += ui64Offset;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "\tHost mem start at 0x%" IMG_UINT64_FMTSPECx,
+					 (IMG_UINT64)sPagePhysAddr.uiAddr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "\tHost Mem Page %d at 0x%" IMG_UINT64_FMTSPECx,
+					 uiIdx, (IMG_UINT64)sPagePhysAddr.uiAddr));
+		}
+	}
+	PVR_DPF((PVR_DBG_MESSAGE,
+			 "Devmem CPU phys address: 0x%" IMG_UINT64_FMTSPECx,
+			 (IMG_UINT64)sDmaAddr->uiAddr));
+}
+#endif
+
+PVRSRV_ERROR OSDmaSubmitTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *pvOSData,
+				    void *pvChan, IMG_BOOL bSynchronous)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA*)pvOSData;
+	struct completion* sync_completion = NULL;
+
+	psOSCleanup->bSucceed = IMG_TRUE;
+	psOSCleanup->bAdvanceTimeline = IMG_TRUE;
+
+	if (bSynchronous)
+	{
+		sync_completion = OSAllocZMem(sizeof(struct completion));
+		init_completion(sync_completion);
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	/* Wait only on number of ops scheduled. This might be different to NumDMAs
+	in certain error conditions */
+	psOSCleanup->uiRefCount = psOSCleanup->uiCount;
+	psOSCleanup->sync_completion = sync_completion;
+
+	{
+		IMG_UINT32 i,j;
+		for (i=0; i<psOSCleanup->uiCount; i++)
+		{
+			if (psOSCleanup->pbIsSparse[i])
+			{
+				for (j=0; j<psOSCleanup->puiNumPages[i]; j++)
+				{
+					if (psOSCleanup->ppsDescriptorsSparse[i][j])
+						dmaengine_submit(psOSCleanup->ppsDescriptorsSparse[i][j]);
+				}
+			}
+			else
+			{
+				dmaengine_submit(psOSCleanup->ppsDescriptors[i]);
+			}
+		}
+	}
+
+	dma_async_issue_pending(pvChan);
+
+	if (bSynchronous)
+	{
+		wait_for_completion(sync_completion);
+		OSFreeMem(sync_completion);
+	}
+
+	return PVRSRV_OK;
+}
+
+void OSDmaForceCleanup(PVRSRV_DEVICE_NODE *psDevNode, void *pvChan,
+					   void *pvOSData, void *pvServerCleanupParam,
+					   PFN_SERVER_CLEANUP pfnServerCleanup)
+{
+	OS_CLEANUP_DATA *psOSCleanup = (OS_CLEANUP_DATA *)pvOSData;
+	IMG_UINT ui32Retries;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	psOSCleanup->bSucceed = IMG_FALSE;
+	psOSCleanup->bAdvanceTimeline = IMG_TRUE;
+
+	/* Need to wait for outstanding DMA Engine ops before advancing the
+	   user-supplied timeline in case of error. dmaengine_terminate_sync
+	   cannot be called from within atomic context, so cannot invoke it
+	   from inside the cleanup kernel thread. */
+	for (ui32Retries = 0; ui32Retries < DMA_ERROR_SYNC_RETRIES; ui32Retries++)
+	{
+		if (dmaengine_terminate_sync(pvChan) == 0)
+		{
+			break;
+		}
+	}
+	if (ui32Retries == DMA_ERROR_SYNC_RETRIES)
+	{
+		/* We cannot guarantee all outstanding DMAs were terminated
+		 * so we let the UM fence time out as a fallback mechanism */
+		psOSCleanup->bAdvanceTimeline = IMG_FALSE;
+	}
+
+	if (psOSCleanup->uiCount > 0)
+	{
+		complete(&psOSCleanup->start_cleanup);
+	}
+	else
+	{
+		/* Cleanup kthread never run, need to manually wind down */
+		pfnServerCleanup(pvServerCleanupParam, psOSCleanup->bAdvanceTimeline);
+
+		OSFreeMem(psOSCleanup->ppsSg);
+		OSFreeMem(psOSCleanup->pages);
+		OSFreeMem(psOSCleanup->puiNumPages);
+		OSFreeMem(psOSCleanup->ppsSgSparse);
+		OSFreeMem(psOSCleanup->pbIsSparse);
+		OSFreeMem(psOSCleanup->uiNumValidPages);
+		OSFreeMem(psOSCleanup->ppsDescriptors);
+		OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+
+		OSFreeMem(psOSCleanup);
+	}
+}
+
+PVRSRV_ERROR OSDmaAllocData(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 uiNumDMA, void **pvOutData)
+{
+	PVRSRV_ERROR eError;
+	OS_CLEANUP_DATA *psOSCleanup = OSAllocZMem(sizeof(OS_CLEANUP_DATA));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup, eError, e0);
+
+	psOSCleanup->uiNumDMA = uiNumDMA;
+	psOSCleanup->psDevNode = psDevNode;
+
+	spin_lock_init(&psOSCleanup->spinlock);
+
+	init_completion(&psOSCleanup->start_cleanup);
+
+	psOSCleanup->ppsDescriptors = OSAllocZMem(uiNumDMA * sizeof(struct dma_async_tx_descriptor*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsDescriptors, eError, e0);
+
+	psOSCleanup->ppsDescriptorsSparse = OSAllocZMem(uiNumDMA * sizeof(struct dma_async_tx_descriptor*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsDescriptorsSparse, eError, e11);
+
+	psOSCleanup->ppsSg = OSAllocZMem(uiNumDMA * sizeof(struct sg_table*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsSg, eError, e1);
+
+	psOSCleanup->ppsSgSparse = OSAllocZMem(uiNumDMA * sizeof(struct sg_table*));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->ppsSgSparse, eError, e12);
+
+	psOSCleanup->pbIsSparse = OSAllocZMem(uiNumDMA * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->pbIsSparse, eError, e13);
+
+	psOSCleanup->uiNumValidPages = OSAllocZMem(uiNumDMA * sizeof(IMG_UINT));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->uiNumValidPages, eError, e14);
+
+	psOSCleanup->pages = OSAllocZMem(uiNumDMA * sizeof(struct page **));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->pages, eError, e2);
+
+	psOSCleanup->puiNumPages = OSAllocZMem(uiNumDMA * sizeof(IMG_UINT32));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanup->puiNumPages, eError, e3);
+
+	*pvOutData = psOSCleanup;
+
+	return PVRSRV_OK;
+
+e3:
+	OSFreeMem(psOSCleanup->pages);
+e2:
+	OSFreeMem(psOSCleanup->uiNumValidPages);
+e14:
+	OSFreeMem(psOSCleanup->pbIsSparse);
+e13:
+	OSFreeMem(psOSCleanup->ppsSgSparse);
+e12:
+	OSFreeMem(psOSCleanup->ppsSg);
+e1:
+	OSFreeMem(psOSCleanup->ppsDescriptorsSparse);
+e11:
+	OSFreeMem(psOSCleanup->ppsDescriptors);
+e0:
+	OSFreeMem(psOSCleanup);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       OSDmaTransfer
+@Description    This API is used to ask OS to perform a DMA transfer operation
+@Return
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDmaPrepareTransfer(PVRSRV_DEVICE_NODE *psDevNode,
+							   void* pvChan,
+							   IMG_DMA_ADDR* psDmaAddr, IMG_UINT64* puiAddress,
+							   IMG_UINT64 uiSize, IMG_BOOL bMemToDev,
+							   void* pvOSData,
+							   void* pvServerCleanupParam, PFN_SERVER_CLEANUP pfnServerCleanup, IMG_BOOL bFirst)
+{
+
+	IMG_INT iRet;
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+	OS_CLEANUP_DATA* psOSCleanupData = pvOSData;
+
+	enum dma_data_direction eDataDirection = bMemToDev ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct dma_slave_config sConfig = {0};
+	struct dma_async_tx_descriptor *psDesc;
+
+	unsigned long offset = (unsigned long)puiAddress & ((1 << PAGE_SHIFT) - 1);
+	unsigned int num_pages = (uiSize + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	int num_pinned_pages = 0;
+
+	struct sg_table *psSg = OSAllocZMem(sizeof(struct sg_table));
+	PVR_LOG_GOTO_IF_NOMEM(psSg, eError, e0);
+
+	psOSCleanupData->pages[psOSCleanupData->uiCount] = OSAllocZMem(num_pages * sizeof(struct page *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->pages[psOSCleanupData->uiCount], eError, e1);
+
+	num_pinned_pages = pvr_pin_user_pages_for_dma(
+		puiAddress,
+		num_pages,
+		!bMemToDev,
+		psOSCleanupData->pages[psOSCleanupData->uiCount]);
+
+	psOSCleanupData->puiNumPages[psOSCleanupData->uiCount] = num_pinned_pages;
+	if (num_pinned_pages != num_pages)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "get_user_pages_fast failed: (%d - %u)", num_pinned_pages, num_pages));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e2; /* Unpin what was pinned and return error */
+	}
+
+#if defined(PVRSRV_DEBUG_DMA)
+	DMADumpPhysicalAddresses(psOSCleanupData->pages[psOSCleanupData->uiCount],
+							 num_pages, psDmaAddr, offset);
+#endif
+
+	if (sg_alloc_table_from_pages(psSg, psOSCleanupData->pages[psOSCleanupData->uiCount], num_pages, offset, uiSize, GFP_KERNEL) != 0)
+	{
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table_from_pages failed"));
+		goto e2;
+	}
+
+	if (bMemToDev)
+	{
+		sConfig.direction = DMA_MEM_TO_DEV;
+		sConfig.src_addr = 0;
+		sConfig.dst_addr = psDmaAddr->uiAddr;
+	}
+	else
+	{
+		sConfig.direction = DMA_DEV_TO_MEM;
+		sConfig.src_addr = psDmaAddr->uiAddr;
+		sConfig.dst_addr = 0;
+	}
+	dmaengine_slave_config(pvChan, &sConfig);
+
+	iRet = dma_map_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+	if (!iRet)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error mapping SG list", __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e3;
+	}
+
+	dma_sync_sg_for_device(psDevConfig->pvOSDevice, psSg->sgl,(unsigned int)iRet, eDataDirection);
+
+	psDesc = dmaengine_prep_slave_sg(pvChan, psSg->sgl, (unsigned int)iRet, sConfig.direction, 0);
+	if (!psDesc)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: dmaengine_prep_slave_sg failed", __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e4;
+	}
+
+	psOSCleanupData->eDirection = eDataDirection;
+	psOSCleanupData->ppsSg[psOSCleanupData->uiCount] = psSg;
+	psOSCleanupData->pfnServerCleanup = pfnServerCleanup;
+	psOSCleanupData->pvServerCleanupData = pvServerCleanupParam;
+
+	psDesc->callback_param = psOSCleanupData;
+	psDesc->callback = dma_callback;
+
+	if (bFirst)
+		kthread_run(cleanup_thread, psOSCleanupData, "dma-cleanup-thread");
+	psOSCleanupData->ppsDescriptors[psOSCleanupData->uiCount] = psDesc;
+
+	psOSCleanupData->uiCount++;
+
+	return PVRSRV_OK;
+
+e4:
+	dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+e3:
+	sg_free_table(psSg);
+e2:
+	{
+		IMG_UINT32 i;
+		/* Unpin pages */
+		for (i=0; i<psOSCleanupData->puiNumPages[psOSCleanupData->uiCount]; i++)
+		{
+			pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][i]);
+		}
+	}
+	OSFreeMem(psOSCleanupData->pages[psOSCleanupData->uiCount]);
+e1:
+	OSFreeMem(psSg);
+e0:
+	return eError;
+}
+
+static IMG_UINT32
+CalculateValidPages(IMG_BOOL *pbValid,
+					IMG_UINT32 ui32SizeInPages)
+{
+	IMG_UINT32 ui32nValid;
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0, ui32nValid = 0; ui32Idx < ui32SizeInPages; ui32Idx++)
+	{
+		ui32nValid += pbValid[ui32Idx] ? 1 : 0;
+	}
+
+	return ui32nValid;
+}
+
+PVRSRV_ERROR OSDmaPrepareTransferSparse(PVRSRV_DEVICE_NODE *psDevNode,
+										void* pvChan,
+										IMG_DMA_ADDR* psDmaAddr,
+										IMG_BOOL *pbValid,
+										IMG_UINT64* puiAddress,
+										IMG_UINT64 uiSize,
+										IMG_UINT32 uiOffsetInFirstPMRPage,
+										IMG_UINT32 ui32SizeInPages,
+										IMG_BOOL bMemToDev,
+										void* pvOSData,
+										void* pvServerCleanupParam,
+										PFN_SERVER_CLEANUP pfnServerCleanup,
+										IMG_BOOL bFirst)
+{
+
+	IMG_INT iRet;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+	OS_CLEANUP_DATA* psOSCleanupData = pvOSData;
+	IMG_UINT32 ui32PageSize = OSGetPageSize();
+	void *pvNextAddress = puiAddress;
+	IMG_UINT32 ui32Idx;
+	IMG_INT32 i32Rwd;
+
+	enum dma_data_direction eDataDirection = bMemToDev ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct dma_slave_config sConfig = {0};
+	struct dma_async_tx_descriptor *psDesc;
+
+	unsigned long offset = (unsigned long)puiAddress & ((1 << PAGE_SHIFT) - 1);
+	unsigned int num_pages = (uiSize + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	unsigned int num_valid_pages = CalculateValidPages(pbValid, ui32SizeInPages);
+	unsigned int num_pinned_pages = 0;
+	unsigned int valid_idx;
+	size_t transfer_size;
+	struct page ** next_pages;
+	struct sg_table *psSg;
+
+	psOSCleanupData->uiNumValidPages[psOSCleanupData->uiCount] = num_valid_pages;
+	psOSCleanupData->pbIsSparse[psOSCleanupData->uiCount] = IMG_TRUE;
+
+	/*
+	 * If an SG transfer from virtual memory to card memory goes over a page boundary in
+	 * main memory, it'll span two different pages - therefore, total number of pages to
+	 * keep track of should be twice as many as for a simple transfer. This twice-as-big
+	 * allocation is also necessary because the same virtual memory page might be present
+	 * in more than one SG DMA transfer, because of differences in first-page offset between
+	 * the sparse device PMR and the virtual memory buffer.
+	 */
+	psOSCleanupData->pages[psOSCleanupData->uiCount] = OSAllocZMem(2*num_valid_pages * sizeof(struct page *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->pages[psOSCleanupData->uiCount], eError, e0);
+
+	psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount] = OSAllocZMem(num_valid_pages * sizeof(struct sg_table *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount], eError, e1);
+
+	psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount] = OSAllocZMem(num_valid_pages * sizeof(struct dma_async_tx_descriptor *));
+	PVR_LOG_GOTO_IF_NOMEM(psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount], eError, e11);
+
+	for (ui32Idx = 0, valid_idx = 0; ui32Idx < ui32SizeInPages; ui32Idx++)
+	{
+		if (valid_idx == num_valid_pages)
+		{
+			break;
+		}
+		if (!pbValid[ui32Idx])
+		{
+			pvNextAddress += (ui32Idx == 0) ? ui32PageSize - uiOffsetInFirstPMRPage : ui32PageSize;
+			continue;
+		}
+
+		/* Pick transfer size */
+		if (ui32Idx == 0)
+		{
+			if (uiOffsetInFirstPMRPage + uiSize <= ui32PageSize)
+			{
+				PVR_ASSERT(num_valid_pages == 1);
+				transfer_size = uiSize;
+			}
+			else
+			{
+				transfer_size = ui32PageSize - uiOffsetInFirstPMRPage;
+			}
+		}
+		else
+		{
+			/* Last valid LMA page */
+			if (valid_idx == num_valid_pages - 1)
+			{
+				transfer_size = ((uiOffsetInFirstPMRPage + uiSize - 1) % ui32PageSize) + 1;
+			}
+			else
+			{
+				transfer_size = ui32PageSize;
+			}
+		}
+
+		if (((unsigned long long)pvNextAddress & (ui32PageSize - 1)) + transfer_size > ui32PageSize)
+		{
+			num_pages = 2;
+		}
+		else
+		{
+			num_pages = 1;
+		}
+
+		next_pages = psOSCleanupData->pages[psOSCleanupData->uiCount] + (valid_idx * 2);
+
+		num_pinned_pages = pvr_pin_user_pages_for_dma(
+			pvNextAddress,
+			num_pages,
+			!bMemToDev,
+			next_pages);
+
+		if (num_pinned_pages != num_pages)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "get_user_pages_fast for sparse failed: (%d - %u)", num_pinned_pages, num_pages));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e2;
+		}
+
+#if defined(PVRSRV_DEBUG_DMA)
+		DMADumpPhysicalAddresses(next_pages, num_pages,
+								 &psDmaAddr[ui32Idx],
+								 (unsigned long)pvNextAddress & (ui32PageSize - 1));
+#endif
+
+		psSg = OSAllocZMem(sizeof(struct sg_table));
+		PVR_LOG_GOTO_IF_NOMEM(psSg, eError, e3);
+
+		if (sg_alloc_table_from_pages(psSg, next_pages, num_pages,
+									  (unsigned long)pvNextAddress & (ui32PageSize - 1),
+									  transfer_size,
+									  GFP_KERNEL) != 0)
+		{
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table_from_pages failed"));
+			goto e4;
+		}
+
+		pvNextAddress += transfer_size;
+
+		if (bMemToDev)
+		{
+			sConfig.direction = DMA_MEM_TO_DEV;
+			sConfig.src_addr = 0;
+			sConfig.dst_addr = psDmaAddr[ui32Idx].uiAddr;
+		}
+		else
+		{
+			sConfig.direction = DMA_DEV_TO_MEM;
+			sConfig.src_addr = psDmaAddr[ui32Idx].uiAddr;
+			sConfig.dst_addr = 0;
+		}
+		dmaengine_slave_config(pvChan, &sConfig);
+
+		iRet = dma_map_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+		if (!iRet)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Error mapping SG list", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e5;
+		}
+		dma_sync_sg_for_device(psDevConfig->pvOSDevice, psSg->sgl,(unsigned int)iRet, eDataDirection);
+
+		psDesc = dmaengine_prep_slave_sg(pvChan, psSg->sgl, (unsigned int)iRet, sConfig.direction, 0);
+		if (!psDesc)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: dmaengine_prep_slave_sg failed", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+		goto e6;
+		}
+
+		psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount][valid_idx] = psSg;
+		psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount][valid_idx] = psDesc;
+		psOSCleanupData->puiNumPages[psOSCleanupData->uiCount] = ++valid_idx;
+
+		if (valid_idx == num_valid_pages)
+		{
+			psDesc->callback_param = psOSCleanupData;
+			psDesc->callback = dma_callback;
+
+			if (bFirst)
+			{
+				psOSCleanupData->eDirection = eDataDirection;
+				psOSCleanupData->pfnServerCleanup = pfnServerCleanup;
+				psOSCleanupData->pvServerCleanupData = pvServerCleanupParam;
+
+				kthread_run(cleanup_thread, psOSCleanupData, "dma-cleanup-thread");
+			}
+
+			psOSCleanupData->uiCount++;
+		}
+
+	}
+
+	return PVRSRV_OK;
+
+e6:
+	dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+e5:
+	sg_free_table(psSg);
+e4:
+	OSFreeMem(psSg);
+e3:
+	/* Unpin last */
+	pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx]);
+
+	if (psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx+1])
+	{
+		pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][valid_idx+1]);
+	}
+e2:
+	/* rewind */
+	for (i32Rwd=valid_idx-1; i32Rwd >= 0; i32Rwd--)
+	{
+		IMG_UINT32 i;
+
+		psSg = psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount][i32Rwd];
+		dma_unmap_sg(psDevConfig->pvOSDevice, psSg->sgl, psSg->nents, eDataDirection);
+		sg_free_table(psSg);
+
+		/* Unpin pages */
+		for (i=0; i < psOSCleanupData->puiNumPages[psOSCleanupData->uiCount]*2; i++)
+		{
+			if (psOSCleanupData->pages[psOSCleanupData->uiCount][i])
+			{
+				pvr_unpin_user_page_for_dma(psOSCleanupData->pages[psOSCleanupData->uiCount][i]);
+			}
+		}
+	}
+	OSFreeMem(psOSCleanupData->ppsDescriptorsSparse[psOSCleanupData->uiCount]);
+e11:
+	OSFreeMem(psOSCleanupData->ppsSgSparse[psOSCleanupData->uiCount]);
+e1:
+	OSFreeMem(psOSCleanupData->pages[psOSCleanupData->uiCount]);
+e0:
+	return eError;
+}
+
+#endif /* SUPPORT_DMA_TRANSFER */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+IMG_INTERNAL PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR)
+{
+	struct dma_heap *heap;
+	struct dma_buf *buf;
+	struct device *dev;
+	struct dma_buf_attachment *buf_attachment;
+
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_ERROR eError;
+	IMG_CHAR *pszHeapName;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode->psDevConfig->pszSecureDMAHeapName, "pszSecureDMAHeapName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM((OSStringLength(psDeviceNode->psDevConfig->pszSecureDMAHeapName) > 0), "pszSecureDMAHeapName length");
+
+	pszHeapName = psDeviceNode->psDevConfig->pszSecureDMAHeapName;
+	dev = (struct device*)psDeviceNode->psDevConfig->pvOSDevice;
+
+	heap = dma_heap_find(pszHeapName);
+	PVR_LOG_GOTO_IF_NOMEM(heap, eError, ErrorExit);
+
+	buf = dma_heap_buffer_alloc(heap, uiSize, 0, 0);
+	PVR_LOG_GOTO_IF_NOMEM(buf, eError, ErrorBufPut);
+
+	if (buf->size < uiSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: buffer size (%ld) is less than requested (%lld).",
+				 __func__, buf->size, uiSize));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorBufFree;
+	}
+
+	buf_attachment = dma_buf_attach(buf, dev);
+	PVR_LOG_GOTO_IF_NOMEM(buf_attachment, eError, ErrorBufFree);
+
+	eError = PhysmemCreateNewDmaBufBackedPMR(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_EXTERNAL],
+											 buf_attachment,
+											 NULL,
+											 PVRSRV_MEMALLOCFLAG_GPU_READABLE
+											 | PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE,
+											 buf->size,
+											 1,
+											 1,
+											 &ui32MappingTable,
+											 OSStringLength(pszName),
+											 pszName,
+											 ppsPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateNewDmaBufBackedPMR", ErrorBufDetach);
+
+	return PVRSRV_OK;
+
+ErrorBufDetach:
+	dma_buf_detach(buf, buf_attachment);
+ErrorBufFree:
+	dma_heap_buffer_free(buf);
+ErrorBufPut:
+	dma_buf_put(buf);
+ErrorExit:
+
+	return eError;
+}
+
+IMG_INTERNAL void
+OSFreeSecBuf(PMR *psPMR)
+{
+	struct dma_buf *buf = PhysmemGetDmaBuf(psPMR);
+	dma_buf_put(buf);
+	dma_heap_buffer_free(buf);
+
+	PMRUnrefPMR(psPMR);
+}
+#else /* PVR_ANDROID_HAS_DMA_HEAP_FIND */
+IMG_INTERNAL PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR)
+{
+	IMG_UINT32 ui32MappingTable = 0;
+	PVRSRV_ERROR eError;
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiSize,
+									1,
+									1,
+									&ui32MappingTable,
+									ExactLog2(OSGetPageSize()),
+									PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_SECURE)
+									| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE,
+									OSStringLength(pszName),
+									pszName,
+									OSGetCurrentClientProcessIDKM(),
+									ppsPMR,
+									PDUMP_NONE,
+									NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemNewRamBackedPMR", ErrorExit);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(*ppsPMR, PVR_SYS_ALLOC_PID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RIWritePMREntryWithOwnerKM", ErrorUnrefPMR);
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+ErrorUnrefPMR:
+	PMRUnrefPMR(*ppsPMR);
+#endif
+ErrorExit:
+	return eError;
+}
+
+IMG_INTERNAL void
+OSFreeSecBuf(PMR *psPMR)
+{
+	PMRUnrefPMR(psPMR);
+}
+#endif
+#endif /* SUPPORT_SECURE_ALLOC_KM */
diff --git a/drivers/gpu/drm/img-rogue/osfunc.h b/drivers/gpu/drm/img-rogue/osfunc.h
new file mode 100644
index 000000000000..15ce5e6728b0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osfunc.h
@@ -0,0 +1,1745 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS functions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifdef DEBUG_RELEASE_BUILD
+#pragma optimize( "", off )
+#define DEBUG		1
+#endif
+
+#ifndef OSFUNC_H
+/*! @cond Doxygen_Suppress */
+#define OSFUNC_H
+/*! @endcond */
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include "kernel_nospec.h"
+#if !defined(NO_HARDWARE)
+#include <linux/io.h>
+
+#endif
+#endif
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+
+#if defined(__QNXNTO__)
+#include <stdio.h>
+#include <string.h>
+#endif
+
+#if defined(INTEGRITY_OS)
+#include <stdio.h>
+#include <string.h>
+#endif
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "device.h"
+#include "pvrsrv_device.h"
+#include "cache_ops.h"
+#include "osfunc_common.h"
+#if defined(SUPPORT_DMA_TRANSFER)
+#include "dma_km.h"
+#include "pmr.h"
+#endif
+
+/******************************************************************************
+ * Static defines
+ *****************************************************************************/
+/*!
+ * Returned by OSGetCurrentProcessID() and OSGetCurrentThreadID() if the OS
+ * is currently operating in the interrupt context.
+ */
+#define KERNEL_ID			0xffffffffL
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define OSConfineArrayIndexNoSpeculation(index, size) array_index_nospec((index), (size))
+#elif defined(__QNXNTO__)
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#elif defined(INTEGRITY_OS)
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#else
+/*************************************************************************/ /*!
+@Function       OSConfineArrayIndexNoSpeculation
+@Description    This macro aims to avoid code exposure to Cache Timing
+                Side-Channel Mechanisms which rely on speculative code
+                execution (Variant 1). It does so by ensuring a value to be
+                used as an array index will be set to zero if outside of the
+                bounds of the array, meaning any speculative execution of code
+                which uses this suitably adjusted index value will not then
+                attempt to load data from memory outside of the array bounds.
+                Code calling this macro must still first verify that the
+                original unmodified index value is within the bounds of the
+                array, and should then only use the modified value returned
+                by this function when accessing the array itself.
+                NB. If no OS-specific implementation of this macro is
+                defined, the original index is returned unmodified and no
+                protection against the potential exploit is provided.
+@Input          index    The original array index value that would be used to
+                         access the array.
+@Input          size     The number of elements in the array being accessed.
+@Return         The value to use for the array index, modified so that it
+                remains within array bounds.
+*/ /**************************************************************************/
+#define OSConfineArrayIndexNoSpeculation(index, size) (index)
+#if !defined(DOXYGEN)
+#define PVRSRV_MISSING_NO_SPEC_IMPL
+#endif
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSClockns64
+@Description    This function returns the number of ticks since system boot
+                expressed in nanoseconds. Unlike OSClockns, OSClockns64 has
+                a near 64-bit range.
+@Return         The 64-bit clock value, in nanoseconds.
+*/ /**************************************************************************/
+IMG_UINT64 OSClockns64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockus64
+@Description    This function returns the number of ticks since system boot
+                expressed in microseconds. Unlike OSClockus, OSClockus64 has
+                a near 64-bit range.
+@Return         The 64-bit clock value, in microseconds.
+*/ /**************************************************************************/
+IMG_UINT64 OSClockus64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockus
+@Description    This function returns the number of ticks since system boot
+                in microseconds.
+@Return         The 32-bit clock value, in microseconds.
+*/ /**************************************************************************/
+IMG_UINT32 OSClockus(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockms
+@Description    This function returns the number of ticks since system boot
+                in milliseconds.
+@Return         The 32-bit clock value, in milliseconds.
+*/ /**************************************************************************/
+IMG_UINT32 OSClockms(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicns64
+@Description    This function returns a clock value based on the system
+                monotonic clock.
+@Output         pui64Time     The 64-bit clock value, in nanoseconds.
+@Return         Error Code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSClockMonotonicns64(IMG_UINT64 *pui64Time);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicus64
+@Description    This function returns a clock value based on the system
+                monotonic clock.
+@Output         pui64Time     The 64-bit clock value, in microseconds.
+@Return         Error Code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSClockMonotonicus64(IMG_UINT64 *pui64Time);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicRawns64
+@Description    This function returns a clock value based on the system
+                monotonic raw clock.
+@Return         64bit ns timestamp
+*/ /**************************************************************************/
+IMG_UINT64 OSClockMonotonicRawns64(void);
+
+/*************************************************************************/ /*!
+@Function       OSClockMonotonicRawus64
+@Description    This function returns a clock value based on the system
+                monotonic raw clock.
+@Return         64bit us timestamp
+*/ /**************************************************************************/
+IMG_UINT64 OSClockMonotonicRawus64(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageSize
+@Description    This function returns the page size.
+                If the OS is not using memory mappings it should return a
+                default value of 4096.
+@Return         The size of a page, in bytes.
+*/ /**************************************************************************/
+size_t OSGetPageSize(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageShift
+@Description    This function returns the page size expressed as a power of
+                two. A number of pages, left-shifted by this value, gives the
+                equivalent size in bytes.
+                If the OS is not using memory mappings it should return a
+                default value of 12.
+@Return         The page size expressed as a power of two.
+*/ /**************************************************************************/
+size_t OSGetPageShift(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetPageMask
+@Description    This function returns a bitmask that may be applied to an
+                address to mask off the least-significant bits so as to
+                leave the start address of the page containing that address.
+@Return         The page mask.
+*/ /**************************************************************************/
+size_t OSGetPageMask(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetOrder
+@Description    This function returns the order of power of two for a given
+                size. Eg. for a uSize of 4096 bytes the function would
+                return 12 (4096 = 2^12).
+@Input          uSize     The size in bytes.
+@Return         The order of power of two.
+*/ /**************************************************************************/
+size_t OSGetOrder(size_t uSize);
+
+/*************************************************************************/ /*!
+@Function       OSGetRAMSize
+@Description    This function returns the total amount of GPU-addressable
+                memory provided by the system. In other words, after loading
+                the driver this would be the largest allocation an
+                application would reasonably expect to be able to make.
+                Note that this is function is not expected to return the
+                current available memory but the amount which would be
+                available on startup.
+@Return         Total GPU-addressable memory size, in bytes.
+*/ /**************************************************************************/
+IMG_UINT64 OSGetRAMSize(void);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a Mid-level Interrupt Service Routine (MISR).
+@Input  pvData  Pointer to MISR specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_MISR)(void *pvData);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a thread entry point function.
+@Input  pvData  Pointer to thread specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_THREAD)(void *pvData);
+
+/*************************************************************************/ /*!
+@Function       OSInstallMISR
+@Description    Installs a Mid-level Interrupt Service Routine (MISR)
+                which handles higher-level processing of interrupts from
+                the device (GPU).
+                An MISR runs outside of interrupt context, and so may be
+                descheduled. This means it can contain code that would
+                not be permitted in the LISR.
+                An MISR is invoked when OSScheduleMISR() is called. This
+                call should be made by installed LISR once it has completed
+                its interrupt processing.
+                Multiple MISRs may be installed by the driver to handle
+                different causes of interrupt.
+@Input          pfnMISR       pointer to the function to be installed
+                              as the MISR
+@Input          hData         private data provided to the MISR
+@Input          pszMisrName   Name describing purpose of MISR worker thread
+                              (Must be a string literal).
+@Output         hMISRData     handle to the installed MISR (to be used
+                              for a subsequent uninstall)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInstallMISR(IMG_HANDLE *hMISRData,
+							PFN_MISR pfnMISR,
+							void *hData,
+							const IMG_CHAR *pszMisrName);
+
+/*************************************************************************/ /*!
+@Function       OSUninstallMISR
+@Description    Uninstalls a Mid-level Interrupt Service Routine (MISR).
+@Input          hMISRData     handle to the installed MISR
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSUninstallMISR(IMG_HANDLE hMISRData);
+
+/*************************************************************************/ /*!
+@Function       OSScheduleMISR
+@Description    Schedules a Mid-level Interrupt Service Routine (MISR) to be
+                executed. An MISR should be executed outside of interrupt
+                context, for example in a work queue.
+@Input          hMISRData     handle to the installed MISR
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSScheduleMISR(IMG_HANDLE hMISRData);
+
+/*************************************************************************/ /*!
+@Function       OSSyncIRQ
+@Description    Wait for LISR to complete. If you use this function while
+                holding a resource which the IRQ handler also requires,
+                you will deadlock.
+@Input          ui32IRQ     IRQ number
+*/ /**************************************************************************/
+void OSSyncIRQ(IMG_UINT32 ui32IRQ);
+
+/*************************************************************************/ /*!
+@Description    Pointer to a function implementing debug dump of thread-specific
+                data.
+@Input          pfnDumpDebugPrintf      Used to specify the print function used
+                                        to dump any debug information. If this
+                                        argument is NULL then a default print
+                                        function will be used.
+@Input          pvDumpDebugFile         File identifier to be passed to the
+                                        print function if specified.
+*/ /**************************************************************************/
+
+typedef void (*PFN_THREAD_DEBUG_DUMP)(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                                      void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSThreadCreate
+@Description    Creates a kernel thread and starts it running. The caller
+                is responsible for informing the thread that it must finish
+                and return from the pfnThread function. It is not possible
+                to kill or terminate it. The new thread runs with the default
+                priority provided by the Operating System.
+                Note: Kernel threads are freezable which means that they
+                can be frozen by the kernel on for example driver suspend.
+                Because of that only OSEventObjectWaitKernel() function should
+                be used to put kernel threads in waiting state.
+@Output         phThread            Returned handle to the thread.
+@Input          pszThreadName       Name to assign to the thread.
+@Input          pfnThread           Thread entry point function.
+@Input          pfnDebugDumpCB      Used to dump info of the created thread
+@Input          bIsSupportingThread Set, if summary of this thread needs to
+                                    be dumped in debug_dump
+@Input          hData               Thread specific data pointer for pfnThread().
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR OSThreadCreate(IMG_HANDLE *phThread,
+                            IMG_CHAR *pszThreadName,
+                            PFN_THREAD pfnThread,
+                            PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                            IMG_BOOL bIsSupportingThread,
+                            void *hData);
+
+/*! Available priority levels for the creation of a new Kernel Thread. */
+typedef enum priority_levels
+{
+	OS_THREAD_NOSET_PRIORITY = 0,   /* With this option the priority level is the default for the given OS */
+	OS_THREAD_HIGHEST_PRIORITY,
+	OS_THREAD_HIGH_PRIORITY,
+	OS_THREAD_NORMAL_PRIORITY,
+	OS_THREAD_LOW_PRIORITY,
+	OS_THREAD_LOWEST_PRIORITY,
+	OS_THREAD_LAST_PRIORITY     /* This must be always the last entry */
+} OS_THREAD_LEVEL;
+
+/*************************************************************************/ /*!
+@Function       OSThreadCreatePriority
+@Description    As OSThreadCreate, this function creates a kernel thread and
+                starts it running. The difference is that with this function
+                is possible to specify the priority used to schedule the new
+                thread.
+
+@Output         phThread            Returned handle to the thread.
+@Input          pszThreadName       Name to assign to the thread.
+@Input          pfnThread           Thread entry point function.
+@Input          pfnDebugDumpCB      Used to dump info of the created thread
+@Input          bIsSupportingThread Set, if summary of this thread needs to
+                                    be dumped in debug_dump
+@Input          hData               Thread specific data pointer for pfnThread().
+@Input          eThreadPriority     Priority level to assign to the new thread.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSThreadCreatePriority(IMG_HANDLE *phThread,
+                                    IMG_CHAR *pszThreadName,
+                                    PFN_THREAD pfnThread,
+                                    PFN_THREAD_DEBUG_DUMP pfnDebugDumpCB,
+                                    IMG_BOOL bIsSupportingThread,
+                                    void *hData,
+                                    OS_THREAD_LEVEL eThreadPriority);
+
+/*************************************************************************/ /*!
+@Function       OSThreadDestroy
+@Description    Waits for the thread to end and then destroys the thread
+                handle memory. This function will block and wait for the
+                thread to finish successfully, thereby providing a sync point
+                for the thread completing its work. No attempt is made to kill
+                or otherwise terminate the thread.
+@Input          hThread   The thread handle returned by OSThreadCreate().
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSThreadDestroy(IMG_HANDLE hThread);
+
+/*************************************************************************/ /*!
+@Function       OSIsMapPhysNonContigSupported
+@Description    Determine if the kernel mapping of physically non-contiguous
+                pages is supported in the OS layer
+
+                Note: For this function to return IMG_TRUE a full implementation
+                of the following functions is required:
+                OSMapPhysArrayToLin
+                OSUnMapPhysArrayToLin
+
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+IMG_BOOL OSIsMapPhysNonContigSupported(void);
+
+/*************************************************************************/ /*!
+@Function       OSUnMapPhysArrayToLin
+@Description    UnMap a kernel virtual address that was produced by mapping
+                a number of Pages in OSMapPhysArrayToLin.
+
+                Note: This function is only required if the Non contiguous
+                allocation feature is required, in this case
+                OSIsMapPhysNonContigSupported should return IMG_TRUE.
+                If not required this function should return
+                PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED
+
+@Input          pvLinAddr       The linear mapping to be unmapped
+@Input          pvPrivData      Optional implementation specific data.
+
+@Return         None
+*/ /**************************************************************************/
+void OSUnMapPhysArrayToLin(void *pvLinAddr, void *pvPrivData);
+
+/*************************************************************************/ /*!
+@Function       OSMapPhysArrayToLin
+@Description    Given an array of OS page physical addresses and a count
+                of said Pages, this function will map those pages into a
+                virtually contiguous range, this allows for non physically
+                contiguous allocations to be mapped into the kernel.
+                Page size is assumed to be OS page size.
+
+                Note: This function is only required if the Non contiguous
+                allocation feature is required, in this case
+                OSIsMapPhysNonContigSupported should return IMG_TRUE.
+                If not required this function should return
+                PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED
+
+@Input          pPagePA          Array of Pages
+@Input          uiPageCount      Page count of pulPages
+@Output         ppvLinAddr       Pointer to a virtual kernel address of the
+                                 mapped Pages.
+@Output         ppvPrivData      Optional implementation specific data.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSMapPhysArrayToLin(IMG_CPU_PHYADDR pPagePA[],
+                                 IMG_UINT32 uiPageCount,
+                                 void **ppvLinAddr,
+                                 void **ppvPrivData);
+
+/*************************************************************************/ /*!
+@Function       OSMapPhysToLin
+@Description    Maps physical memory into a linear address range.
+@Input          BasePAddr    physical CPU address
+@Input          ui32Bytes    number of bytes to be mapped
+@Input          uiFlags      flags denoting the caching mode to be employed
+                             for the mapping (uncached/write-combined,
+                             cached coherent or cached incoherent).
+                             See pvrsrv_memallocflags.h for full flag bit
+                             definitions.
+@Return         Pointer to the new mapping if successful, NULL otherwise.
+*/ /**************************************************************************/
+void *OSMapPhysToLin(IMG_CPU_PHYADDR BasePAddr, size_t ui32Bytes, PVRSRV_MEMALLOCFLAGS_T uiFlags);
+
+/*************************************************************************/ /*!
+@Function       OSUnMapPhysToLin
+@Description    Unmaps physical memory previously mapped by OSMapPhysToLin().
+@Input          pvLinAddr    the linear mapping to be unmapped
+@Input          ui32Bytes    number of bytes to be unmapped
+@Return         IMG_TRUE if unmapping was successful, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSUnMapPhysToLin(void *pvLinAddr, size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheFlushRangeKM
+@Description    Clean and invalidate the CPU cache for the specified
+                address range.
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              flushed
+@Input          pvVirtEnd     virtual end address of the range to be
+                              flushed
+@Input          sCPUPhysStart physical start address of the range to be
+                              flushed
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              flushed
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheCleanRangeKM
+@Description    Clean the CPU cache for the specified address range.
+                This writes out the contents of the cache and clears the
+                'dirty' bit (which indicates the physical memory is
+                consistent with the cache contents).
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              cleaned
+@Input          pvVirtEnd     virtual end address of the range to be
+                              cleaned
+@Input          sCPUPhysStart physical start address of the range to be
+                              cleaned
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              cleaned
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheInvalidateRangeKM
+@Description    Invalidate the CPU cache for the specified address range.
+                The cache must reload data from those addresses if they
+                are accessed.
+@Input          psDevNode     device on which the allocation was made
+@Input          pvVirtStart   virtual start address of the range to be
+                              invalidated
+@Input          pvVirtEnd     virtual end address of the range to be
+                              invalidated
+@Input          sCPUPhysStart physical start address of the range to be
+                              invalidated
+@Input          sCPUPhysEnd   physical end address of the range to be
+                              invalidated
+@Return         None
+*/ /**************************************************************************/
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd);
+
+/*! CPU Cache operations address domain type */
+typedef enum
+{
+	OS_CACHE_OP_ADDR_TYPE_VIRTUAL,    /*!< Operation requires CPU virtual address only */
+	OS_CACHE_OP_ADDR_TYPE_PHYSICAL,   /*!< Operation requires CPU physical address only */
+	OS_CACHE_OP_ADDR_TYPE_BOTH        /*!< Operation requires both CPU virtual & physical addresses */
+} OS_CACHE_OP_ADDR_TYPE;
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheOpAddressType
+@Description    Returns the address type (i.e. virtual/physical/both) the CPU
+                architecture performs cache maintenance operations under.
+                This is used to infer whether the virtual or physical address
+                supplied to the OSCPUCacheXXXRangeKM functions can be omitted
+                when called.
+@Input          psDevNode   device on which the allocation was made
+@Return         OS_CACHE_OP_ADDR_TYPE
+*/ /**************************************************************************/
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*! CPU Cache attributes available for retrieval, DCache unless specified */
+typedef enum _OS_CPU_CACHE_ATTRIBUTE_
+{
+	OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE, /*!< The cache line size */
+	OS_CPU_CACHE_ATTRIBUTE_COUNT      /*!< The number of attributes (must be last) */
+} OS_CPU_CACHE_ATTRIBUTE;
+
+/*************************************************************************/ /*!
+@Function       OSCPUCacheAttributeSize
+@Description    Returns the size of a given cache attribute.
+                Typically this function is used to return the cache line
+                size, but may be extended to return the size of other
+                cache attributes.
+@Input          eCacheAttribute   the cache attribute whose size should
+                                  be returned.
+@Return         The size of the specified cache attribute, in bytes.
+*/ /**************************************************************************/
+IMG_UINT32 OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE eCacheAttribute);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessID
+@Description    Returns ID of current process (thread group)
+@Return         ID of current process
+*****************************************************************************/
+IMG_PID OSGetCurrentProcessID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentVirtualProcessID
+@Description    Returns ID of current process (thread group of current
+                PID namespace)
+@Return         ID of current process in PID namespace
+*****************************************************************************/
+IMG_PID OSGetCurrentVirtualProcessID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessName
+@Description    Gets the name of current process
+@Return         Process name
+*****************************************************************************/
+IMG_CHAR *OSGetCurrentProcessName(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentProcessVASpaceSize
+@Description    Returns the CPU virtual address space size of current process
+@Return         Process VA space size
+*/ /**************************************************************************/
+IMG_UINT64 OSGetCurrentProcessVASpaceSize(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentThreadID
+@Description    Returns ID for current thread
+@Return         ID of current thread
+*****************************************************************************/
+uintptr_t OSGetCurrentThreadID(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientProcessIDKM
+@Description    Returns ID of current client process (thread group) which
+                has made a bridge call into the server.
+                For some operating systems, this may simply be the current
+                process id. For others, it may be that a dedicated thread
+                is used to handle the processing of bridge calls and that
+                some additional processing is required to obtain the ID of
+                the client process making the bridge call.
+@Return         ID of current client process
+*****************************************************************************/
+IMG_PID OSGetCurrentClientProcessIDKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientProcessNameKM
+@Description    Gets the name of current client process
+@Return         Client process name
+*****************************************************************************/
+IMG_CHAR *OSGetCurrentClientProcessNameKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSGetCurrentClientThreadIDKM
+@Description    Returns ID for current client thread
+                For some operating systems, this may simply be the current
+                thread id. For others, it may be that a dedicated thread
+                is used to handle the processing of bridge calls and that
+                some additional processing is require to obtain the ID of
+                the client thread making the bridge call.
+@Return         ID of current client thread
+*****************************************************************************/
+uintptr_t OSGetCurrentClientThreadIDKM(void);
+
+/*************************************************************************/ /*!
+@Function       OSMemCmp
+@Description    Compares two blocks of memory for equality.
+@Input          pvBufA      Pointer to the first block of memory
+@Input          pvBufB      Pointer to the second block of memory
+@Input          uiLen       The number of bytes to be compared
+@Return         Value < 0 if pvBufA is less than pvBufB.
+                Value > 0 if pvBufB is less than pvBufA.
+                Value = 0 if pvBufA is equal to pvBufB.
+*****************************************************************************/
+IMG_INT OSMemCmp(void *pvBufA, void *pvBufB, size_t uiLen);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesAlloc
+@Description    Allocates a number of contiguous physical pages.
+                If allocations made by this function are CPU cached then
+                OSPhyContigPagesClean has to be implemented to write the
+                cached data to memory.
+@Input          psPhysHeap    the heap from which to allocate
+@Input          uiSize        the size of the required allocation (in bytes)
+@Output         psMemHandle   a returned handle to be used to refer to this
+                              allocation
+@Output         psDevPAddr    the physical address of the allocation
+@Input          uiPid         the process ID that this allocation should
+                              be associated with
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+							PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+							IMG_PID uiPid);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesFree
+@Description    Frees a previous allocation of contiguous physical pages
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be freed
+@Return         None.
+*****************************************************************************/
+void OSPhyContigPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesMap
+@Description    Maps the specified allocation of contiguous physical pages
+                to a kernel virtual address
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be mapped
+@Input          uiSize        the size of the allocation (in bytes)
+@Input          psDevPAddr    the physical address of the allocation
+@Output         pvPtr         the virtual kernel address to which the
+                              allocation is now mapped
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle,
+						size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+						void **pvPtr);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesUnmap
+@Description    Unmaps the kernel mapping for the specified allocation of
+                contiguous physical pages
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be unmapped
+@Input          pvPtr         the virtual kernel address to which the
+                              allocation is currently mapped
+@Return         None.
+*****************************************************************************/
+void OSPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr);
+
+/*************************************************************************/ /*!
+@Function       OSPhyContigPagesClean
+@Description    Write the content of the specified allocation from CPU cache to
+                memory from (start + uiOffset) to (start + uiOffset + uiLength)
+                It is expected to be implemented as a cache clean operation but
+                it is allowed to fall back to a cache clean + invalidate
+                (i.e. flush).
+                If allocations returned by OSPhyContigPagesAlloc are always
+                uncached this can be implemented as nop.
+@Input          psPhysHeap    the heap from which to allocate
+@Input          psMemHandle   the handle of the allocation to be flushed
+@Input          uiOffset      the offset in bytes from the start of the
+                              allocation from where to start flushing
+@Input          uiLength      the amount to flush from the offset in bytes
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*****************************************************************************/
+PVRSRV_ERROR OSPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_UINT32 uiOffset,
+                                   IMG_UINT32 uiLength);
+
+
+/*************************************************************************/ /*!
+@Function       OSInitEnvData
+@Description    Called to initialise any environment-specific data. This
+                could include initialising the bridge calling infrastructure
+                or device memory management infrastructure.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSInitEnvData(void);
+
+/*************************************************************************/ /*!
+@Function       OSDeInitEnvData
+@Description    The counterpart to OSInitEnvData(). Called to free any
+                resources which may have been allocated by OSInitEnvData().
+@Return         None.
+*/ /**************************************************************************/
+void OSDeInitEnvData(void);
+
+/*************************************************************************/ /*!
+@Function       OSStringLCat
+@Description    OS function to support the BSD C strlcat() function.
+*/ /**************************************************************************/
+size_t OSStringLCat(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDstSize);
+
+/*************************************************************************/ /*!
+@Function       OSSNPrintf
+@Description    OS function to support the standard C snprintf() function.
+@Output         pStr        char array to print into
+@Input          ui32Size    maximum size of data to write (chars)
+@Input          pszFormat   format string
+*/ /**************************************************************************/
+IMG_INT32 OSSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR *pszFormat, ...) __printf(3, 4);
+
+/*************************************************************************/ /*!
+@Function       OSVSNPrintf
+@Description    Printf to IMG string using variable args (see stdarg.h).
+                This is necessary because the '...' notation does not
+                support nested function calls.
+@Input          ui32Size           maximum size of data to write (chars)
+@Input          pszFormat          format string
+@Input          vaArgs             variable args structure (from stdarg.h)
+@Output         pStr               char array to print into
+@Return         Number of character written in buffer if successful other wise -1 on error
+*/ /**************************************************************************/
+IMG_INT32 OSVSNPrintf(IMG_CHAR *pStr, size_t ui32Size, const IMG_CHAR* pszFormat, va_list vaArgs) __printf(3, 0);
+
+/*************************************************************************/ /*!
+@Function       OSStringLength
+@Description    OS function to support the standard C strlen() function.
+*/ /**************************************************************************/
+size_t OSStringLength(const IMG_CHAR *pStr);
+
+/*************************************************************************/ /*!
+@Function       OSStringNLength
+@Description    Return the length of a string, excluding the terminating null
+                byte ('\0'), but return at most 'uiCount' bytes. Only the first
+                'uiCount' bytes of 'pStr' are interrogated.
+@Input          pStr     pointer to the string
+@Input          uiCount  the maximum length to return
+@Return         Length of the string if less than 'uiCount' bytes, otherwise
+                'uiCount'.
+*/ /**************************************************************************/
+size_t OSStringNLength(const IMG_CHAR *pStr, size_t uiCount);
+
+/*************************************************************************/ /*!
+@Function       OSStringNCompare
+@Description    OS function to support the standard C strncmp() function.
+*/ /**************************************************************************/
+IMG_INT32 OSStringNCompare(const IMG_CHAR *pStr1, const IMG_CHAR *pStr2,
+                           size_t uiSize);
+
+/*************************************************************************/ /*!
+@Function       OSStringToUINT32
+@Description    Changes string to IMG_UINT32.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSStringToUINT32(const IMG_CHAR *pStr, IMG_UINT32 ui32Base,
+                              IMG_UINT32 *ui32Result);
+
+/*************************************************************************/ /*!
+@Function       OSStringUINT32ToStr
+@Description    Changes IMG_UINT32 to string
+@Input          pszBuf         Buffer to write output number string
+@Input          uSize          Size of buffer provided, i.e. size of pszBuf
+@Input          ui32Num        Number to convert to string
+@Return         Returns 0 if buffer is not sufficient to hold the number string,
+                else returns length of number string
+*/ /**************************************************************************/
+IMG_UINT32 OSStringUINT32ToStr(IMG_CHAR *pszBuf, size_t uSize, IMG_UINT32 ui32Num);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectCreate
+@Description    Create an event object.
+@Input          pszName         name to assign to the new event object.
+@Output         EventObject     the created event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectCreate(const IMG_CHAR *pszName,
+								 IMG_HANDLE *EventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectDestroy
+@Description    Destroy an event object.
+@Input          hEventObject    the event object to destroy.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectDestroy(IMG_HANDLE hEventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectSignal
+@Description    Signal an event object. Any thread waiting on that event
+                object will be woken.
+@Input          hEventObject    the event object to signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectSignal(IMG_HANDLE hEventObject);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWait
+@Description    Wait for an event object to signal. The function is passed
+                an OS event object handle (which allows the OS to have the
+                calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after a default timeout
+                period (defined in EVENT_OBJECT_TIMEOUT_MS), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+
+
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectWait(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWaitKernel
+@Description    Wait for an event object to signal. The function is passed
+                an OS event object handle (which allows the OS to have the
+                calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after a default timeout
+                period (defined in EVENT_OBJECT_TIMEOUT_MS), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+
+                Note: This function should be used only by kernel thread.
+                This is because all kernel threads are freezable and
+                this function allows the kernel to freeze the threads
+                when waiting.
+
+                See OSEventObjectWait() for more details.
+
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__)
+PVRSRV_ERROR OSEventObjectWaitKernel(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus);
+#else
+#define OSEventObjectWaitKernel OSEventObjectWaitTimeout
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSSuspendTaskInterruptible
+@Description    Suspend the current task into interruptible state.
+@Return         none.
+*/ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__)
+void OSSuspendTaskInterruptible(void);
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectWaitTimeout
+@Description    Wait for an event object to signal or timeout. The function
+                is passed an OS event object handle (which allows the OS to
+                have the calling thread wait on the associated event object).
+                The calling thread will be rescheduled when the associated
+                event object signals.
+                If the event object has not signalled after the specified
+                timeout period (passed in 'uiTimeoutus'), the function
+                will return with the result code PVRSRV_ERROR_TIMEOUT.
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Input          uiTimeoutus   the timeout period (in usecs)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectWaitTimeout(IMG_HANDLE hOSEventKM, IMG_UINT64 uiTimeoutus);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectDumpDebugInfo
+@Description    Emits debug counters/stats related to the event object passed
+@Input          hOSEventKM    the OS event object handle associated with
+                              the event object.
+@Return         None.
+*/ /**************************************************************************/
+void OSEventObjectDumpDebugInfo(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectOpen
+@Description    Open an OS handle on the specified event object.
+                This OS handle may then be used to make a thread wait for
+                that event object to signal.
+@Input          hEventObject    Event object handle.
+@Output         phOSEvent       OS handle to the returned event object.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectOpen(IMG_HANDLE hEventObject,
+											IMG_HANDLE *phOSEvent);
+
+/*************************************************************************/ /*!
+@Function       OSEventObjectClose
+@Description    Close an OS handle previously opened for an event object.
+@Input          hOSEventKM      OS event object handle to close.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEventObjectClose(IMG_HANDLE hOSEventKM);
+
+/*************************************************************************/ /*!
+@Function      OSWaitus
+@Description   Implements a busy wait of the specified number of microseconds.
+               This function does NOT release thread quanta.
+@Input         ui32Timeus     The duration of the wait period (in us)
+@Return        None.
+*/ /**************************************************************************/
+void OSWaitus(IMG_UINT32 ui32Timeus);
+
+/*************************************************************************/ /*!
+@Function       OSSleepms
+@Description    Implements a sleep of the specified number of milliseconds.
+                This function may allow pre-emption, meaning the thread
+                may potentially not be rescheduled for a longer period.
+@Input          ui32Timems    The duration of the sleep (in ms)
+@Return         None.
+*/ /**************************************************************************/
+void OSSleepms(IMG_UINT32 ui32Timems);
+
+/*************************************************************************/ /*!
+@Function       OSReleaseThreadQuanta
+@Description    Relinquishes the current thread's execution time-slice,
+                permitting the OS scheduler to schedule another thread.
+@Return         None.
+*/ /**************************************************************************/
+void OSReleaseThreadQuanta(void);
+
+#if defined(__linux__) && defined(__KERNEL__)
+#define OSReadMemoryBarrier() rmb()
+#else
+/*************************************************************************/ /*!
+@Function       OSReadMemoryBarrier
+@Description    Insert a read memory barrier.
+                The read memory barrier guarantees that all load (read)
+                operations specified before the barrier will appear to happen
+                before all of the load operations specified after the barrier.
+*/ /**************************************************************************/
+void OSReadMemoryBarrier(void);
+#endif
+/*************************************************************************/ /*!
+@Function       OSMemoryBarrier
+@Description    Insert a read/write memory barrier.
+                The read and write memory barrier guarantees that all load
+                (read) and all store (write) operations specified before the
+                barrier will appear to happen before all of the load/store
+                operations specified after the barrier.
+@Input          hReadback     Optional pointer to memory to read back, can be
+                useful for flushing queues in bus interconnects to RAM before
+                device (GPU) access the shared memory.
+@Return         None.
+*/ /**************************************************************************/
+void OSMemoryBarrier(volatile void *hReadback);
+/*************************************************************************/ /*!
+@Function       OSWriteMemoryBarrier
+@Description    Insert a write memory barrier.
+                The write memory barrier guarantees that all store operations
+                (writes) specified before the barrier will appear to happen
+                before all of the store operations specified after the barrier.
+@Input          hReadback     Optional pointer to memory to read back, can be
+                useful for flushing queues in bus interconnects to RAM before
+                device (GPU) access the shared memory.
+@Return         None.
+*/ /**************************************************************************/
+void OSWriteMemoryBarrier(volatile void *hReadback);
+
+/*************************************************************************/ /*!
+*/ /**************************************************************************/
+
+/* The access method is dependent on the location of the physical memory that
+ * makes up the PhyHeaps defined for the system and the CPU architecture. These
+ * macros may change in future to accommodate different access requirements.
+ */
+/*! Performs a 32 bit word read from the device memory. */
+#define OSReadDeviceMem32(addr)        (*((volatile IMG_UINT32 __force *)((void*)addr)))
+/*! Performs a 32 bit word write to the device memory. */
+#define OSWriteDeviceMem32(addr, val)  (*((volatile IMG_UINT32 __force *)((void*)addr)) = (IMG_UINT32)(val))
+/*! Performs a 32 bit word write to the device memory and issues a write memory barrier */
+#define OSWriteDeviceMem32WithWMB(addr, val) \
+	do { \
+		*((volatile IMG_UINT32 __force *)((void*)addr)) = (IMG_UINT32)(val); \
+		OSWriteMemoryBarrier(addr); \
+	} while (0)
+
+#if defined(NO_HARDWARE)
+	/* OSReadHWReg and OSWriteHWReg operations are skipped to no-op in nohw builds */
+	#define OSReadUncheckedHWReg32(addr, off) ((void)(addr), (void)(off), 0x30f73a4eU)
+#if defined(__QNXNTO__) && __SIZEOF_LONG__ == 8
+	/* This is needed for 64-bit QNX builds where the size of a long is 64 bits */
+	#define OSReadUncheckedHWReg64(addr, off) ((void)(addr), (void)(off), 0x5b376c9d30f73a4eUL)
+#else
+	#define OSReadUncheckedHWReg64(addr, off) ((void)(addr), (void)(off), 0x5b376c9d30f73a4eULL)
+#endif
+
+	#define OSWriteUncheckedHWReg32(addr, off, val) ((void)(addr), (void)(off), (void)(val))
+	#define OSWriteUncheckedHWReg64(addr, off, val) ((void)(addr), (void)(off), (void)(val))
+
+	#define OSReadHWReg32(addr, off) OSReadUncheckedHWReg32(addr, off)
+	#define OSReadHWReg64(addr, off) OSReadUncheckedHWReg64(addr, off)
+
+	#define OSWriteHWReg32(addr, off, val) OSWriteUncheckedHWReg32(addr, off, val)
+	#define OSWriteHWReg64(addr, off, val) OSWriteUncheckedHWReg64(addr, off, val)
+
+#else
+
+#if defined(__linux__) && defined(__KERNEL__)
+	#define OSReadUncheckedHWReg32(addr, off) ((IMG_UINT32)readl((IMG_BYTE __iomem *)(addr) + (off)))
+
+	/* Little endian support only */
+	#define OSReadUncheckedHWReg64(addr, off) \
+			({ \
+				__typeof__(addr) _addr = addr; \
+				__typeof__(off) _off = off; \
+				(IMG_UINT64) \
+				( \
+					( (IMG_UINT64)(readl((IMG_BYTE __iomem *)(_addr) + (_off) + 4)) << 32) \
+					| readl((IMG_BYTE __iomem *)(_addr) + (_off)) \
+				); \
+			})
+
+	#define OSWriteUncheckedHWReg32(addr, off, val) writel((IMG_UINT32)(val), (IMG_BYTE __iomem *)(addr) + (off))
+	/* Little endian support only */
+	#define OSWriteUncheckedHWReg64(addr, off, val) do \
+			{ \
+				__typeof__(addr) _addr = addr; \
+				__typeof__(off) _off = off; \
+				__typeof__(val) _val = val; \
+				writel((IMG_UINT32)((_val) & 0xffffffff), (IMG_BYTE __iomem *)(_addr) + (_off)); \
+				writel((IMG_UINT32)(((IMG_UINT64)(_val) >> 32) & 0xffffffff), (IMG_BYTE __iomem *)(_addr) + (_off) + 4); \
+			} while (0)
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+/*************************************************************************/ /*!
+@Function       OSReadUncheckedHWReg32
+@Description    Read from a 32-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to read from a location
+                but instead returns a constant value.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be read.
+@Return         The long word read.
+*/ /**************************************************************************/
+	IMG_UINT32 OSReadUncheckedHWReg32(volatile void *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset);
+
+/*************************************************************************/ /*!
+@Function       OSReadUncheckedHWReg64
+@Description    Read from a 64-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to read from a location
+                but instead returns a constant value.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be read.
+@Return         The long long word read.
+*/ /**************************************************************************/
+	IMG_UINT64 OSReadUncheckedHWReg64(volatile void *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset);
+/*************************************************************************/ /*!
+@Function       OSWriteUncheckedHWReg32
+@Description    Write to a 32-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to write to a location.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be written to.
+@Input          ui32Value          The long word to be written to the register.
+@Return         None.
+*/ /**************************************************************************/
+	void OSWriteUncheckedHWReg32(volatile void *pvLinRegBaseAddr,
+								 IMG_UINT32 ui32Offset,
+								 IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+@Function       OSWriteUncheckedHWReg64
+@Description    Write to a 64-bit memory-mapped device register.
+                The implementation should not permit the compiler to
+                reorder the I/O sequence.
+                The implementation should ensure that for a NO_HARDWARE
+                build the code does not attempt to write to a location.
+@Input          pvLinRegBaseAddr   The virtual base address of the register
+                                   block.
+@Input          ui32Offset         The byte offset from the base address of
+                                   the register to be written to.
+@Input          ui64Value          The long long word to be written to the
+                                   register.
+@Return         None.
+*/ /**************************************************************************/
+	void OSWriteUncheckedHWReg64(volatile void *pvLinRegBaseAddr,
+								 IMG_UINT32 ui32Offset,
+								 IMG_UINT64 ui64Value);
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+
+#if !defined(DOXYGEN)
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(DEBUG)
+static INLINE bool _NonSecureRegister(IMG_UINT32 ui32Offset)
+{
+	const IMG_UINT32 ui32PerCoreRegBankSize = RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE;
+	const IMG_UINT32 ui32RegOffsetInCoreBank = ui32Offset % ui32PerCoreRegBankSize;
+
+	if (ui32RegOffsetInCoreBank < RGX_HOST_SECURE_REGBANK_OFFSET)
+	{
+		return true;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Secure register (0x%X) accessed incorrectly. "
+								"Call OS<Read/Write>UncheckedHWReg<Width> instead with "
+								"psDevInfo->pvSecureRegsBaseKM as a register base.",
+								ui32RegOffsetInCoreBank));
+		return false;
+	}
+
+}
+#else
+#define _NonSecureRegister(ui32Offset) (true)
+#endif
+
+	/* systems using real hardware must check that regular register
+	 * operations don't attempt to access secure registers */
+	static INLINE IMG_UINT32 OSReadHWReg32(volatile void __iomem *pvLinRegBaseAddr,
+										   IMG_UINT32 ui32Offset)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		return OSReadUncheckedHWReg32(pvLinRegBaseAddr, ui32Offset);
+	}
+
+	static INLINE IMG_UINT64 OSReadHWReg64(volatile void __iomem *pvLinRegBaseAddr,
+										   IMG_UINT32 ui32Offset)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		return OSReadUncheckedHWReg64(pvLinRegBaseAddr, ui32Offset);
+	}
+
+	static INLINE void OSWriteHWReg32(volatile void __iomem *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset,
+									  IMG_UINT32 ui32Value)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		OSWriteUncheckedHWReg32(pvLinRegBaseAddr, ui32Offset, ui32Value);
+	}
+
+	static INLINE void OSWriteHWReg64(volatile void __iomem *pvLinRegBaseAddr,
+									  IMG_UINT32 ui32Offset,
+									  IMG_UINT64 ui64Value)
+	{
+		PVR_ASSERT(_NonSecureRegister(ui32Offset));
+		OSWriteUncheckedHWReg64(pvLinRegBaseAddr, ui32Offset, ui64Value);
+	}
+
+#endif /* !defined(DOXYGEN) */
+#endif /* defined(NO_HARDWARE) */
+
+/*************************************************************************/ /*!
+@Description    Pointer to a timer callback function.
+@Input          pvData  Pointer to timer specific data.
+*/ /**************************************************************************/
+typedef void (*PFN_TIMER_FUNC)(void* pvData);
+
+/*************************************************************************/ /*!
+@Function       OSAddTimer
+@Description    OS specific function to install a timer callback. The
+                timer will then need to be enabled, as it is disabled by
+                default.
+                When enabled, the callback will be invoked once the specified
+                timeout has elapsed.
+@Input          pfnTimerFunc    Timer callback
+@Input          *pvData         Callback data
+@Input          ui32MsTimeout   Callback period
+@Return         Valid handle on success, NULL if a failure
+*/ /**************************************************************************/
+IMG_HANDLE OSAddTimer(PFN_TIMER_FUNC pfnTimerFunc, void *pvData, IMG_UINT32 ui32MsTimeout);
+
+/*************************************************************************/ /*!
+@Function       OSRemoveTimer
+@Description    Removes the specified timer. The handle becomes invalid and
+                should no longer be used.
+@Input          hTimer          handle of the timer to be removed
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSRemoveTimer(IMG_HANDLE hTimer);
+
+/*************************************************************************/ /*!
+@Function       OSEnableTimer
+@Description    Enable the specified timer. after enabling, the timer will
+                invoke the associated callback at an interval determined by
+                the configured timeout period until disabled.
+@Input          hTimer          handle of the timer to be enabled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSEnableTimer(IMG_HANDLE hTimer);
+
+/*************************************************************************/ /*!
+@Function       OSDisableTimer
+@Description    Disable the specified timer
+@Input          hTimer          handle of the timer to be disabled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDisableTimer(IMG_HANDLE hTimer);
+
+
+/*************************************************************************/ /*!
+ @Function      OSPanic
+ @Description   Take action in response to an unrecoverable driver error
+ @Return        None
+*/ /**************************************************************************/
+void __noreturn OSPanic(void);
+
+/*************************************************************************/ /*!
+@Function       OSCopyToUser
+@Description    Copy data to user-addressable memory from kernel-addressable
+                memory.
+                Note that pvDest may be an invalid address or NULL and the
+                function should return an error in this case.
+                For operating systems that do not have a user/kernel space
+                distinction, this function should be implemented as a stub
+                which simply returns PVRSRV_ERROR_NOT_SUPPORTED.
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination User memory
+@Input          pvSrc            pointer to the source Kernel memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSCopyToUser(void *pvProcess, void __user *pvDest, const void *pvSrc, size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSCopyFromUser
+@Description    Copy data from user-addressable memory to kernel-addressable
+                memory.
+                Note that pvSrc may be an invalid address or NULL and the
+                function should return an error in this case.
+                For operating systems that do not have a user/kernel space
+                distinction, this function should be implemented as a stub
+                which simply returns PVRSRV_ERROR_NOT_SUPPORTED.
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination Kernel memory
+@Input          pvSrc            pointer to the source User memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSCopyFromUser(void *pvProcess, void *pvDest, const void __user *pvSrc, size_t ui32Bytes);
+
+#if defined(__linux__) || defined(INTEGRITY_OS)
+#define OSBridgeCopyFromUser OSCopyFromUser
+#define OSBridgeCopyToUser OSCopyToUser
+#else
+/*************************************************************************/ /*!
+@Function       OSBridgeCopyFromUser
+@Description    Copy data from user-addressable memory into kernel-addressable
+                memory as part of a bridge call operation.
+                For operating systems that do not have a user/kernel space
+                distinction, this function will require whatever implementation
+                is needed to pass data for making the bridge function call.
+                For operating systems which do have a user/kernel space
+                distinction (such as Linux) this function may be defined so
+                as to equate to a call to OSCopyFromUser().
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination Kernel memory
+@Input          pvSrc            pointer to the source User memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSBridgeCopyFromUser (void *pvProcess,
+						void *pvDest,
+						const void *pvSrc,
+						size_t ui32Bytes);
+
+/*************************************************************************/ /*!
+@Function       OSBridgeCopyToUser
+@Description    Copy data to user-addressable memory from kernel-addressable
+                memory as part of a bridge call operation.
+                For operating systems that do not have a user/kernel space
+                distinction, this function will require whatever implementation
+                is needed to pass data for making the bridge function call.
+                For operating systems which do have a user/kernel space
+                distinction (such as Linux) this function may be defined so
+                as to equate to a call to OSCopyToUser().
+@Input          pvProcess        handle of the connection
+@Input          pvDest           pointer to the destination User memory
+@Input          pvSrc            pointer to the source Kernel memory
+@Input          ui32Bytes        size of the data to be copied
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSBridgeCopyToUser (void *pvProcess,
+						void *pvDest,
+						const void *pvSrc,
+						size_t ui32Bytes);
+#endif
+
+/* To be increased if required in future */
+#define PVRSRV_MAX_BRIDGE_IN_SIZE      0x2000    /*!< Size of the memory block used to hold data passed in to a bridge call */
+#define PVRSRV_MAX_BRIDGE_OUT_SIZE     0x1000    /*!< Size of the memory block used to hold data returned from a bridge call */
+
+/*************************************************************************/ /*!
+@Function       OSPlatformBridgeInit
+@Description    Called during device creation to allow the OS port to register
+                other bridge modules and related resources that it requires.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPlatformBridgeInit(void);
+
+/*************************************************************************/ /*!
+@Function       OSPlatformBridgeDeInit
+@Description    Called during device destruction to allow the OS port to
+                deregister its OS specific bridges and clean up other
+                related resources.
+*/ /**************************************************************************/
+void OSPlatformBridgeDeInit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVToNativeError
+@Description    Returns the OS-specific equivalent error number/code for
+                the specified PVRSRV_ERROR value.
+                If there is no equivalent, or the PVRSRV_ERROR value is
+                PVRSRV_OK (no error), 0 is returned.
+@Return         The OS equivalent error code.
+*/ /**************************************************************************/
+int PVRSRVToNativeError(PVRSRV_ERROR e);
+/** See PVRSRVToNativeError(). */
+#define OSPVRSRVToNativeError(e) ( (PVRSRV_OK == e)? 0: PVRSRVToNativeError(e) )
+
+
+#if defined(__linux__) && defined(__KERNEL__)
+
+/* Provide LockDep friendly definitions for Services RW locks */
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include "allocmem.h"
+
+#define OSWRLockCreate(ppsLock) ({ \
+	PVRSRV_ERROR e = PVRSRV_ERROR_OUT_OF_MEMORY; \
+	*(ppsLock) = OSAllocMem(sizeof(struct rw_semaphore)); \
+	if (*(ppsLock)) { init_rwsem(*(ppsLock)); e = PVRSRV_OK; }; \
+	e;})
+#define OSWRLockDestroy(psLock) ({OSFreeMem(psLock); PVRSRV_OK;})
+
+#define OSWRLockAcquireRead(psLock) ({down_read(psLock); PVRSRV_OK;})
+#define OSWRLockAcquireReadNested(psLock, subclass) ({down_read_nested((psLock), (subclass)); PVRSRV_OK;})
+#define OSWRLockReleaseRead(psLock) ({up_read(psLock); PVRSRV_OK;})
+#define OSWRLockAcquireWrite(psLock) ({down_write(psLock); PVRSRV_OK;})
+#define OSWRLockReleaseWrite(psLock) ({up_write(psLock); PVRSRV_OK;})
+
+#elif defined(__linux__) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+/* User-mode unit tests use these definitions on Linux */
+
+PVRSRV_ERROR OSWRLockCreate(POSWR_LOCK *ppsLock);
+void OSWRLockDestroy(POSWR_LOCK psLock);
+void OSWRLockAcquireRead(POSWR_LOCK psLock);
+#define OSWRLockAcquireReadNested(psLock, subclass) OSWRLockAcquireRead((psLock))
+void OSWRLockReleaseRead(POSWR_LOCK psLock);
+void OSWRLockAcquireWrite(POSWR_LOCK psLock);
+void OSWRLockReleaseWrite(POSWR_LOCK psLock);
+
+#else
+
+#if !defined(DOXYGEN)
+#define OSFUNC_NOT_IMPLEMENTED 0
+#define OSFUNC_NOT_IMPLEMENTED_ASSERT() PVR_ASSERT(OSFUNC_NOT_IMPLEMENTED)
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSWRLockCreate
+@Description    Create a writer/reader lock.
+                This type of lock allows multiple concurrent readers but
+                only a single writer, allowing for optimized performance.
+@Output         ppsLock     A handle to the created WR lock.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+static INLINE PVRSRV_ERROR OSWRLockCreate(POSWR_LOCK *ppsLock)
+{
+	PVR_UNREFERENCED_PARAMETER(ppsLock);
+
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockDestroy
+@Description    Destroys a writer/reader lock.
+@Input          psLock     The handle of the WR lock to be destroyed.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockDestroy(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireRead
+@Description    Acquire a writer/reader read lock.
+                If the write lock is already acquired, the caller will
+                block until it is released.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           reading.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireRead(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireReadNested
+@Description    Acquire a nested writer/reader read lock.
+                If the write lock is already acquired, the caller will
+                block until it is released.
+                For operating systems other than Linux, this equates to an
+                OSWRLockAcquireRead() call. On Linux, this function wraps a call
+                to down_read_nested(). This recognises the scenario where
+                there may be multiple subclasses within a particular class
+                of lock. In such cases, the order in which the locks belonging
+                these various subclasses are acquired is important and must be
+                validated.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           reading.
+@Input          iSubclass  The subclass of the lock.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireReadNested(POSWR_LOCK psLock, IMG_INT iSubclass)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	PVR_UNREFERENCED_PARAMETER(iSubclass);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockReleaseRead
+@Description    Release a writer/reader read lock.
+@Input          psLock     The handle of the WR lock whose read lock is to
+                           be released.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockReleaseRead(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockAcquireWrite
+@Description    Acquire a writer/reader write lock.
+                If the write lock or any read lock are already acquired,
+                the caller will block until all are released.
+@Input          psLock     The handle of the WR lock to be acquired for
+                           writing.
+@Return         None.
+*/ /**************************************************************************/
+static INLINE void OSWRLockAcquireWrite(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+
+/*************************************************************************/ /*!
+@Function       OSWRLockReleaseWrite
+@Description    Release a writer/reader write lock.
+@Input          psLock     The handle of the WR lock whose write lock is to
+                           be released.
+@Return         None
+*/ /**************************************************************************/
+static INLINE void OSWRLockReleaseWrite(POSWR_LOCK psLock)
+{
+	PVR_UNREFERENCED_PARAMETER(psLock);
+	OSFUNC_NOT_IMPLEMENTED_ASSERT();
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSDivide64r64
+@Description    Divide a 64-bit value by a 32-bit value. Return the 64-bit
+                quotient.
+                The remainder is also returned in 'pui32Remainder'.
+@Input          ui64Divident        The number to be divided.
+@Input          ui32Divisor         The 32-bit value 'ui64Divident' is to
+                                    be divided by.
+@Output         pui32Remainder      The remainder of the division.
+@Return         The 64-bit quotient (result of the division).
+*/ /**************************************************************************/
+IMG_UINT64 OSDivide64r64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder);
+
+/*************************************************************************/ /*!
+@Function       OSDivide64
+@Description    Divide a 64-bit value by a 32-bit value. Return a 32-bit
+                quotient.
+                The remainder is also returned in 'pui32Remainder'.
+                This function allows for a more optimal implementation
+                of a 64-bit division when the result is known to be
+                representable in 32-bits.
+@Input          ui64Divident        The number to be divided.
+@Input          ui32Divisor         The 32-bit value 'ui64Divident' is to
+                                    be divided by.
+@Output         pui32Remainder      The remainder of the division.
+@Return         The 32-bit quotient (result of the division).
+*/ /**************************************************************************/
+IMG_UINT32 OSDivide64(IMG_UINT64 ui64Divident, IMG_UINT32 ui32Divisor, IMG_UINT32 *pui32Remainder);
+
+/*************************************************************************/ /*!
+@Function       OSDumpStack
+@Description    Dump the current task information and its stack trace.
+@Return         None
+*/ /**************************************************************************/
+void OSDumpStack(void);
+
+/*************************************************************************/ /*!
+@Function       OSUserModeAccessToPerfCountersEn
+@Description    Permit User-mode access to CPU performance counter
+                registers.
+                This function is called during device initialisation.
+                Certain CPU architectures may need to explicitly permit
+                User mode access to performance counters - if this is
+                required, the necessary code should be implemented inside
+                this function.
+@Return         None.
+*/ /**************************************************************************/
+void OSUserModeAccessToPerfCountersEn(void);
+
+/*************************************************************************/ /*!
+@Function       OSDebugSignalPID
+@Description    Sends a SIGTRAP signal to a specific PID in user mode for
+                debugging purposes. The user mode process can register a handler
+                against this signal.
+                This is necessary to support the Rogue debugger. If the Rogue
+                debugger is not used then this function may be implemented as
+                a stub.
+@Input          ui32PID    The PID for the signal.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR OSDebugSignalPID(IMG_UINT32 ui32PID);
+
+#if defined(__linux__) && defined(__KERNEL__) && !defined(DOXYGEN)
+#define OSWarnOn(a) WARN_ON(a)
+#else
+/*************************************************************************/ /*!
+@Function       OSWarnOn
+@Description    This API allows the driver to emit a special token and stack
+                dump to the server log when an issue is detected that needs the
+                OS to be notified. The token or call may be used to trigger
+                log collection by the OS environment.
+                PVR_DPF log messages will have been emitted prior to this call.
+@Input          a    Expression to evaluate, if true trigger Warn signal
+@Return         None
+*/ /**************************************************************************/
+#define OSWarnOn(a) do { if ((a)) { OSDumpStack(); } } while (0)
+#endif
+
+/*************************************************************************/ /*!
+@Function       OSIsKernelThread
+@Description    This API determines if the current running thread is a kernel
+                thread (i.e. one not associated with any userland process,
+                typically an MISR handler.)
+@Return         IMG_TRUE if it is a kernel thread, otherwise IMG_FALSE.
+*/ /**************************************************************************/
+IMG_BOOL OSIsKernelThread(void);
+
+/*************************************************************************/ /*!
+@Function       OSThreadDumpInfo
+@Description    Traverse the thread list and call each of the stored
+                callbacks to dump the info in debug_dump.
+@Input          pfnDumpDebugPrintf  The 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+*/ /**************************************************************************/
+void OSThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSDumpVersionInfo
+@Description    Store OS version information in debug dump.
+@Input          pfnDumpDebugPrintf  The 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+*/ /**************************************************************************/
+void OSDumpVersionInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                       void *pvDumpDebugFile);
+
+/*************************************************************************/ /*!
+@Function       OSIsWriteCombineUnalignedSafe
+@Description    Determine if unaligned accesses to write-combine memory are
+                safe to perform, i.e. whether we are safe from a CPU fault
+                occurring. This test is specifically aimed at ARM64 platforms
+                which cannot provide this guarantee if the memory is 'device'
+                memory rather than 'normal' under the ARM memory architecture.
+@Return         IMG_TRUE if safe, IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void);
+
+/*************************************************************************/ /*!
+@Function       OSDebugLevel
+@Description    Returns current value of the debug level.
+@Return         Debug level.
+*/ /**************************************************************************/
+IMG_UINT32 OSDebugLevel(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVSetDebugLevel
+@Description    Sets the current value of the debug level to ui32DebugLevel.
+@Input          ui32DebugLevel New debug level value.
+*/ /**************************************************************************/
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVIsDebugLevel
+@Description    Tests if a given debug level is enabled.
+@Input          ui32DebugLevel IMG_TRUE if debug level is enabled
+                and IMG_FALSE otherwise.
+*/ /**************************************************************************/
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel);
+
+#if defined(SUPPORT_DMA_TRANSFER)
+
+typedef void (*PFN_SERVER_CLEANUP)(void *pvData, IMG_BOOL bAdvanceTimeline);
+
+#define DMA_COMPLETION_TIMEOUT_MS 60000
+#define DMA_ERROR_SYNC_RETRIES 100
+
+PVRSRV_ERROR OSDmaPrepareTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *psChan,
+							   IMG_DMA_ADDR* psDmaAddr, IMG_UINT64* puiAddress,
+							   IMG_UINT64 uiSize, IMG_BOOL bMemToDev,
+							   IMG_HANDLE pvOSData,
+							   IMG_HANDLE pvServerCleanupParam,PFN_SERVER_CLEANUP pfnServerCleanup, IMG_BOOL bFirst);
+
+PVRSRV_ERROR OSDmaPrepareTransferSparse(PVRSRV_DEVICE_NODE *psDevNode, IMG_HANDLE pvChan,
+										IMG_DMA_ADDR* psDmaAddr, IMG_BOOL *pbValid,
+										IMG_UINT64* puiAddress, IMG_UINT64 uiSize,
+										IMG_UINT32 uiOffsetInPage,
+										IMG_UINT32 ui32SizeInPages,
+										IMG_BOOL bMemToDev,
+										IMG_HANDLE pvOSData,
+										IMG_HANDLE pvServerCleanupParam, PFN_SERVER_CLEANUP pfnServerCleanup,
+										IMG_BOOL bFirst);
+
+PVRSRV_ERROR OSDmaAllocData(PVRSRV_DEVICE_NODE *psDevNode,IMG_UINT32 uiNumDMA, void **pvAllocedData);
+PVRSRV_ERROR OSDmaSubmitTransfer(PVRSRV_DEVICE_NODE *psDevNode, void *pvOSData, void *psChan, IMG_BOOL bSynchronous);
+void OSDmaForceCleanup(PVRSRV_DEVICE_NODE *psDevNode, void *pvChan,
+					   void *pvOSData, IMG_HANDLE pvServerCleanupParam,
+					   PFN_SERVER_CLEANUP pfnServerCleanup);
+#endif
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+PVRSRV_ERROR
+OSAllocateSecBuf(PVRSRV_DEVICE_NODE *psDeviceNode,
+				 IMG_DEVMEM_SIZE_T uiSize,
+				 const IMG_CHAR *pszName,
+				 PMR **ppsPMR);
+
+void
+OSFreeSecBuf(PMR *psPMR);
+#endif
+#endif /* OSFUNC_H */
+
+/******************************************************************************
+ End of file (osfunc.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/osfunc_arm.c b/drivers/gpu/drm/img-rogue/osfunc_arm.c
new file mode 100644
index 000000000000..d3d3ab9f83fa
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osfunc_arm.c
@@ -0,0 +1,155 @@
+/*************************************************************************/ /*!
+@File
+@Title          arm specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+ #include <linux/dma-map-ops.h>
+#else
+ #include <linux/dma-mapping.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) */
+#include <asm/cacheflush.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+
+static inline size_t pvr_dmac_range_len(const void *pvStart, const void *pvEnd)
+{
+	return (size_t)((char *)pvEnd - (char *)pvStart);
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+		                           sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                           DMA_TO_DEVICE);
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+		                        sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                        DMA_FROM_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_device(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_TO_DEVICE);
+	arm_dma_ops.sync_single_for_cpu(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_FROM_DEVICE);
+#endif
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+		                           sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                           DMA_TO_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_device(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_TO_DEVICE);
+#endif
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+	struct device *dev = psDevNode->psDevConfig->pvOSDevice;
+	if (dev)
+	{
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+		                        sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+		                        DMA_FROM_DEVICE);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	arm_dma_ops.sync_single_for_cpu(psDevNode->psDevConfig->pvOSDevice, sCPUPhysStart.uiAddr, sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr, DMA_FROM_DEVICE);
+#endif
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+}
+
+/* User Enable Register */
+#define PMUSERENR_EN      0x00000001 /* enable user access to the counters */
+
+static void per_cpu_perf_counter_user_access_en(void *data)
+{
+	PVR_UNREFERENCED_PARAMETER(data);
+	/* Enable user-mode access to counters. */
+	asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(PMUSERENR_EN));
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+	on_each_cpu(per_cpu_perf_counter_user_access_en, NULL, 1);
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	/*
+	 * The kernel looks to have always used normal memory under ARM32.
+	 * See osfunc_arm64.c implementation for more details.
+	 */
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img-rogue/osfunc_arm64.c b/drivers/gpu/drm/img-rogue/osfunc_arm64.c
new file mode 100644
index 000000000000..4722e3d3b34b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osfunc_arm64.c
@@ -0,0 +1,261 @@
+/*************************************************************************/ /*!
+@File
+@Title          arm64 specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#include <linux/cpumask.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+ #include <linux/dma-map-ops.h>
+#else
+ #include <linux/dma-mapping.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) */
+#include <asm/cacheflush.h>
+#include <linux/uaccess.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+#include "kernel_compatibility.h"
+
+#if defined(CONFIG_OUTER_CACHE)
+  /* If you encounter a 64-bit ARM system with an outer cache, you'll need
+   * to add the necessary code to manage that cache. See osfunc_arm.c
+   * for an example of how to do so.
+   */
+	#error "CONFIG_OUTER_CACHE not supported on arm64."
+#endif
+
+static inline void begin_user_mode_access(void)
+{
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	uaccess_enable_privileged();
+#endif
+}
+
+static inline void end_user_mode_access(void)
+{
+#if defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	uaccess_disable_privileged();
+#endif
+}
+
+static inline void FlushRange(void *pvRangeAddrStart,
+							  void *pvRangeAddrEnd,
+							  PVRSRV_CACHE_OP eCacheOp)
+{
+	IMG_UINT32 ui32CacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+	IMG_BYTE *pbStart = pvRangeAddrStart;
+	IMG_BYTE *pbEnd = pvRangeAddrEnd;
+	IMG_BYTE *pbBase;
+
+	/*
+	  On arm64, the TRM states in D5.8.1 (data and unified caches) that if cache
+	  maintenance is performed on a memory location using a VA, the effect of
+	  that cache maintenance is visible to all VA aliases of the physical memory
+	  location. So here it's quicker to issue the machine cache maintenance
+	  instruction directly without going via the Linux kernel DMA framework as
+	  this is sufficient to maintain the CPU d-caches on arm64.
+	 */
+
+	begin_user_mode_access();
+
+	pbEnd = (IMG_BYTE *) PVR_ALIGN((uintptr_t)pbEnd, (uintptr_t)ui32CacheLineSize);
+
+	/* Memory-barrier */
+	asm volatile("dsb sy" : : : "memory");
+
+	switch (eCacheOp)
+	{
+		case PVRSRV_CACHE_OP_CLEAN:
+			for (pbBase = pbStart; pbBase < pbEnd; pbBase += ui32CacheLineSize)
+			{
+				asm volatile ("dc cvac, %0" :: "r" (pbBase));
+			}
+			break;
+		case PVRSRV_CACHE_OP_INVALIDATE:
+		case PVRSRV_CACHE_OP_FLUSH:
+			for (pbBase = pbStart; pbBase < pbEnd; pbBase += ui32CacheLineSize)
+			{
+				asm volatile ("dc civac, %0" :: "r" (pbBase));
+			}
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Cache maintenance operation type %d is invalid",
+					__func__, eCacheOp));
+			break;
+	}
+
+	/* Memory-barrier */
+	asm volatile("dsb sy" : : : "memory");
+
+	end_user_mode_access();
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+	PVR_UNREFERENCED_PARAMETER(dev);
+
+	if (pvVirtStart == NULL)
+	{
+		/*
+		 * Converting the physical addresses to virtual addresses allows us to
+		 * utilize the assembly instruction that makes the Flush + Invalidate
+		 * cache operation atomic.
+		 * There is no state in-between the flush and the invalidate operation.
+		 */
+		pvVirtStart = phys_to_virt(sCPUPhysStart.uiAddr);
+		pvVirtEnd = phys_to_virt(sCPUPhysEnd.uiAddr);
+	}
+
+	FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_FLUSH);
+	return;
+
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+
+	if (pvVirtStart)
+	{
+		FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_CLEAN);
+		return;
+	}
+
+	dev = psDevNode->psDevConfig->pvOSDevice;
+
+	if (dev)
+	{
+		dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+								   sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+								   DMA_TO_DEVICE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cache operation cannot be completed!"));
+	}
+
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+								 void *pvVirtStart,
+								 void *pvVirtEnd,
+								 IMG_CPU_PHYADDR sCPUPhysStart,
+								 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+
+	if (pvVirtStart)
+	{
+		FlushRange(pvVirtStart, pvVirtEnd, PVRSRV_CACHE_OP_INVALIDATE);
+		return;
+	}
+
+	dev = psDevNode->psDevConfig->pvOSDevice;
+
+	if (dev)
+	{
+		dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+								sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+								DMA_FROM_DEVICE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Cache operation cannot be completed!"));
+	}
+}
+
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	if (!psDevNode->psDevConfig->pvOSDevice)
+	{
+		/* Host Mem device node doesn't have an associated Linux dev ptr.
+		   Use virtual addr ops instead of asking kernel to do physical
+		   maintenance */
+		return OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+	}
+
+	return OS_CACHE_OP_ADDR_TYPE_PHYSICAL;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	/*
+	 * Under ARM64 there is the concept of 'device' [0] and 'normal' [1] memory.
+	 * Unaligned access on device memory is explicitly disallowed [2]:
+	 *
+	 * 'Further, unaligned accesses are only allowed to regions marked as Normal
+	 *  memory type.
+	 *  ...
+	 *  Attempts to perform unaligned accesses when not allowed will cause an
+	 *  alignment fault (data abort).'
+	 *
+	 * Write-combine on ARM64 can be implemented as either normal non-cached
+	 * memory (NORMAL_NC) or as device memory with gathering enabled
+	 * (DEVICE_GRE.) Kernel 3.13 changed this from the latter to the former.
+	 *
+	 * [0]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/CHDBDIDF.html
+	 * [1]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.den0024a/ch13s01s01.html
+	 * [2]:http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html
+	 */
+
+	pgprot_t pgprot = pgprot_writecombine(PAGE_KERNEL);
+
+	return (pgprot_val(pgprot) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_NC);
+}
diff --git a/drivers/gpu/drm/img-rogue/osfunc_common.h b/drivers/gpu/drm/img-rogue/osfunc_common.h
new file mode 100644
index 000000000000..d93cb6d61aba
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osfunc_common.h
@@ -0,0 +1,274 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS functions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific API definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSFUNC_COMMON_H
+/*! @cond Doxygen_Suppress */
+#define OSFUNC_COMMON_H
+/*! @endcond */
+
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/string.h>
+#else
+#include <string.h>
+#endif
+
+#include "img_types.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/**************************************************************************/ /*!
+@Function       DeviceMemSet
+@Description    Set memory, whose mapping may be uncached, to a given value.
+                Safe implementation for all architectures for uncached mapping,
+                optimised for speed where supported by tool chains.
+                In such cases, OSDeviceMemSet() is defined as a call to this
+                function.
+@Input          pvDest     void pointer to the memory to be set
+@Input          ui8Value   byte containing the value to be set
+@Input          ui32Size   the number of bytes to be set to the given value
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemSet(void *pvDest, IMG_UINT8 ui8Value, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemCopy
+@Description    Copy values from one area of memory. Safe implementation for
+                all architectures for uncached mapping, of either the source
+                or destination, optimised for speed where supported by tool
+                chains. In such cases, OSDeviceMemCopy() is defined as a call
+                to this function.
+@Input          pvDst      void pointer to the destination memory
+@Input          pvSrc      void pointer to the source memory
+@Input          ui32Size   the number of bytes to be copied
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemCopy(void *pvDst, const void *pvSrc, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemSetBytes
+@Description    Potentially very slow (but safe) memset fallback for non-GNU C
+                compilers for arm64/aarch64
+@Input          pvDest     void pointer to the memory to be set
+@Input          ui8Value   byte containing the value to be set
+@Input          ui32Size   the number of bytes to be set to the given value
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemSetBytes(void *pvDest, IMG_UINT8 ui8Value, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       DeviceMemCopyBytes
+@Description    Potentially very slow (but safe) memcpy fallback for non-GNU C
+                compilers for arm64/aarch64
+@Input          pvDst      void pointer to the destination memory
+@Input          pvSrc      void pointer to the source memory
+@Input          ui32Size   the number of bytes to be copied
+@Return         None
+ */ /**************************************************************************/
+void DeviceMemCopyBytes(void *pvDst, const void *pvSrc, size_t ui32Size);
+
+/**************************************************************************/ /*!
+@Function       OSStringSafeCopy
+@Description    Copy at most uDataSize-1 bytes from pszSrc to pszDest. pszDest
+                is always null-terminated. If no null byte ('\0') is contained
+                within the first uDataSize-1 characters of the source string,
+                the destination will contain the truncated source string,
+                otherwise pszDest will contain the entire source string.
+@Input          pszDest    char pointer to the destination string
+@Input          pszSrc     const char pointer to the source string
+@Input          uDataSize  the maximum number of bytes to be copied; this
+                           should normally be the size of the destination
+                           buffer
+@Return         Number of bytes copied if uDataSize > strlen(pszSrc), negative
+                value otherwise
+ */ /**************************************************************************/
+#if defined(__linux__) && defined(__KERNEL__) && !defined(DEBUG)
+static inline ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize)
+{
+	return strscpy(pszDest, pszSrc, uDataSize);
+}
+#else
+ssize_t OSStringSafeCopy(IMG_CHAR *pszDest, const IMG_CHAR *pszSrc, size_t uDataSize);
+#endif
+
+#if defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)
+#if defined(__GNUC__)
+/* Workarounds for assumptions made that memory will not be mapped uncached
+ * in kernel or user address spaces on arm64 platforms (or other testing).
+ */
+
+#define OSDeviceMemSet(a,b,c)  DeviceMemSet((a), (b), (c))
+#define OSDeviceMemCopy(a,b,c) DeviceMemCopy((a), (b), (c))
+
+#else /* defined __GNUC__ */
+
+#define OSDeviceMemSet(a,b,c)  DeviceMemSetBytes((a), (b), (c))
+#define OSDeviceMemCopy(a,b,c) DeviceMemCopyBytes((a), (b), (c))
+
+#endif /* defined __GNUC__ */
+
+#else /* (defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)) */
+
+/* Everything else */
+
+/**************************************************************************/ /*!
+@Function       OSDeviceMemSet
+@Description    Set memory, whose mapping may be uncached, to a given value.
+                On some architectures, additional processing may be needed
+                if the mapping is uncached.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSDeviceMemSet(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memset((a), (b), (c)); \
+			(void) *(volatile IMG_UINT32*)((void*)(a)); \
+		} \
+	} while (false)
+
+/**************************************************************************/ /*!
+@Function       OSDeviceMemCopy
+@Description    Copy values from one area of memory, to another, when one
+                or both mappings may be uncached.
+                On some architectures, additional processing may be needed
+                if mappings are uncached.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSDeviceMemCopy(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			memcpy((a), (b), (c)); \
+			(void) *(volatile IMG_UINT32*)((void*)(a)); \
+		} \
+	} while (false)
+
+#endif /* (defined(__arm64__) || defined(__aarch64__) || defined(PVRSRV_DEVMEM_TEST_SAFE_MEMSETCPY)) */
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemSet
+@Description    Set memory, where the mapping is known to be cached, to a
+                given value. This function exists to allow an optimal memset
+                to be performed when memory is known to be cached.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemSet(a,b,c)  (void) memset((a), (b), (c))
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemCopy
+@Description    Copy values from one area of memory, to another, when both
+                mappings are known to be cached.
+                This function exists to allow an optimal memcpy to be
+                performed when memory is known to be cached.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemCopy(a,b,c) memcpy((a), (b), (c))
+
+#if defined(__KERNEL__)
+
+/**************************************************************************/ /*!
+@Function       OSCachedMemSetWMB
+@Description    Set memory, where the mapping is known to be cached or
+                write-combine, to a given value and issue a write memory barrier
+                after. This
+                function exists to allow an optimal memset to be performed when
+                memory is known to be cached or write-combine.
+@Input          a     void pointer to the memory to be set
+@Input          b     byte containing the value to be set
+@Input          c     the number of bytes to be set to the given value
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemSetWMB(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memset((a), (b), (c)); \
+			OSWriteMemoryBarrier(a); \
+		} \
+	} while (false)
+/**************************************************************************/ /*!
+@Function       OSCachedMemCopy
+@Description    Copy values from one area of memory, to another, when both
+                mappings are known to be cached or write-combine and issue
+                a write memory barrier after.
+                This function exists to allow an optimal memcpy to be
+                performed when memory is known to be cached or write-combine.
+@Input          a     void pointer to the destination memory
+@Input          b     void pointer to the source memory
+@Input          c     the number of bytes to be copied
+@Return         Pointer to the destination memory.
+ */ /**************************************************************************/
+#define OSCachedMemCopyWMB(a,b,c) \
+	do { \
+		if ((c) != 0U) \
+		{ \
+			(void) memcpy((a), (b), (c)); \
+			OSWriteMemoryBarrier(a); \
+		} \
+	} while (false)
+#endif /* defined(__KERNEL__) */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OSFUNC_COMMON_H */
+
+/******************************************************************************
+ End of file (osfunc_common.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/osfunc_riscv.c b/drivers/gpu/drm/img-rogue/osfunc_riscv.c
new file mode 100644
index 000000000000..ff3c71680053
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osfunc_riscv.c
@@ -0,0 +1,209 @@
+/*************************************************************************/ /*!
+@File
+@Title          RISC-V specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "cache_ops.h"
+#include <linux/dma-mapping.h>
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_FLUSH,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		dev = psDevNode->psDevConfig->pvOSDevice;
+		if (dev)
+		{
+			dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+						   sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+						   DMA_TO_DEVICE);
+			dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+						sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+						DMA_FROM_DEVICE);
+		}
+		//PVR_DPF((PVR_DBG_WARNING,
+		//         "%s: System doesn't implement cache maintenance. Skipping!",
+		//         __func__));
+	}
+#endif
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+							void *pvVirtStart,
+							void *pvVirtEnd,
+							IMG_CPU_PHYADDR sCPUPhysStart,
+							IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_CLEAN,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		dev = psDevNode->psDevConfig->pvOSDevice;
+		if (dev)
+		{
+			dma_sync_single_for_device(dev, sCPUPhysStart.uiAddr,
+						   sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+						   DMA_TO_DEVICE);
+		}
+		//PVR_DPF((PVR_DBG_WARNING,
+		//         "%s: System doesn't implement cache maintenance. Skipping!",
+		//         __func__));
+	}
+#endif
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+								 void *pvVirtStart,
+								 void *pvVirtEnd,
+								 IMG_CPU_PHYADDR sCPUPhysStart,
+								 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	struct device *dev;
+	/*
+	 * RISC-V cache maintenance mechanism is not part of the core spec.
+	 * This leaves the actual mechanism of action to an implementer.
+	 * Here we let the system layer decide how maintenance is done.
+	 */
+	if (psDevNode->psDevConfig->pfnHostCacheMaintenance)
+	{
+		psDevNode->psDevConfig->pfnHostCacheMaintenance(
+				psDevNode->psDevConfig->hSysData,
+				PVRSRV_CACHE_OP_INVALIDATE,
+				pvVirtStart,
+				pvVirtEnd,
+				sCPUPhysStart,
+				sCPUPhysEnd);
+
+	}
+#if !defined(NO_HARDWARE)
+	else
+	{
+		dev = psDevNode->psDevConfig->pvOSDevice;
+
+		if (dev)
+		{
+			dma_sync_single_for_cpu(dev, sCPUPhysStart.uiAddr,
+								sCPUPhysEnd.uiAddr - sCPUPhysStart.uiAddr,
+								DMA_FROM_DEVICE);
+		}
+		//PVR_DPF((PVR_DBG_WARNING,
+		//         "%s: System doesn't implement cache maintenance. Skipping!",
+		//         __func__));
+	}
+#endif
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	/*
+	 * Need to obtain psDevNode here and do the following:
+	 *
+	 * OS_CACHE_OP_ADDR_TYPE eOpAddrType =
+	 *	psDevNode->psDevConfig->bHasPhysicalCacheMaintenance ?
+	 *		OS_CACHE_OP_ADDR_TYPE_PHYSICAL : OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+	 *
+	 * Return BOTH for now on.
+	 *
+	 */
+	return OS_CACHE_OP_ADDR_TYPE_BOTH;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+#if !defined(NO_HARDWARE)
+	PVR_DPF((PVR_DBG_WARNING, "%s: Not implemented!", __func__));
+	//PVR_ASSERT(0);
+#endif
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+#if !defined(NO_HARDWARE)
+	//PVR_DPF((PVR_DBG_WARNING,
+	//         "%s: Not implemented (assuming false)!",
+	//         __func__));
+	//PVR_ASSERT(0);	
+	return IMG_TRUE;
+	//return IMG_FALSE;
+#else
+	return IMG_TRUE;
+#endif
+}
diff --git a/drivers/gpu/drm/img-rogue/osfunc_x86.c b/drivers/gpu/drm/img-rogue/osfunc_x86.c
new file mode 100644
index 000000000000..2c5c145e268f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osfunc_x86.c
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File
+@Title          x86 specific OS functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Processor specific OS functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/smp.h>
+#include <linux/uaccess.h>
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+
+static void x86_flush_cache_range(const void *pvStart, const void *pvEnd)
+{
+	IMG_BYTE *pbStart = (IMG_BYTE *)pvStart;
+	IMG_BYTE *pbEnd = (IMG_BYTE *)pvEnd;
+	IMG_BYTE *pbBase;
+
+	pbEnd = (IMG_BYTE *)PVR_ALIGN((uintptr_t)pbEnd,
+	                              (uintptr_t)boot_cpu_data.x86_clflush_size);
+
+	mb();
+
+	__uaccess_begin();
+
+	for (pbBase = pbStart; pbBase < pbEnd; pbBase += boot_cpu_data.x86_clflush_size)
+	{
+		clflush(pbBase);
+	}
+
+	__uaccess_end();
+
+	mb();
+}
+
+void OSCPUCacheFlushRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+void OSCPUCacheCleanRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            void *pvVirtStart,
+                            void *pvVirtEnd,
+                            IMG_CPU_PHYADDR sCPUPhysStart,
+                            IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	/* No clean feature on x86 */
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+void OSCPUCacheInvalidateRangeKM(PVRSRV_DEVICE_NODE *psDevNode,
+                                 void *pvVirtStart,
+                                 void *pvVirtEnd,
+                                 IMG_CPU_PHYADDR sCPUPhysStart,
+                                 IMG_CPU_PHYADDR sCPUPhysEnd)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysStart);
+	PVR_UNREFERENCED_PARAMETER(sCPUPhysEnd);
+
+	/* No invalidate-only support */
+	x86_flush_cache_range(pvVirtStart, pvVirtEnd);
+}
+
+OS_CACHE_OP_ADDR_TYPE OSCPUCacheOpAddressType(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	return OS_CACHE_OP_ADDR_TYPE_VIRTUAL;
+}
+
+void OSUserModeAccessToPerfCountersEn(void)
+{
+	/* Not applicable to x86 architecture. */
+}
+
+IMG_BOOL OSIsWriteCombineUnalignedSafe(void)
+{
+	return IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img-rogue/oskm_apphint.h b/drivers/gpu/drm/img-rogue/oskm_apphint.h
new file mode 100644
index 000000000000..8740a6a597e1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/oskm_apphint.h
@@ -0,0 +1,186 @@
+/*************************************************************************/ /*!
+@File           oskm_apphint.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS-independent interface for retrieving KM apphints
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_defs.h"
+#if defined(__linux__)
+#include "km_apphint.h"
+#include "device.h"
+#else
+#include "services_client_porting.h"
+#endif
+#if !defined(OSKM_APPHINT_H)
+#define OSKM_APPHINT_H
+
+/*! Supplied to os_get_km_apphint_XXX() functions when the param/AppHint is
+ * applicable to all devices and not a specific device. Typically used
+ * for server-wide build and module AppHints.
+ */
+#define APPHINT_NO_DEVICE (NULL)
+
+#if defined(__linux__) && !defined(DOXYGEN)
+static INLINE IMG_UINT os_get_km_apphint_UINT32(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT32 *pAppHintDefault, IMG_UINT32 *pVal) {
+	return !pvr_apphint_get_uint32(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_km_apphint_UINT64(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_UINT64 *pAppHintDefault, IMG_UINT64 *pVal) {
+	return !pvr_apphint_get_uint64(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_km_apphint_BOOL(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_BOOL *pAppHintDefault, IMG_BOOL *pVal) {
+	return !pvr_apphint_get_bool(device, id, pVal);
+}
+static INLINE IMG_UINT os_get_km_apphint_STRING(PVRSRV_DEVICE_NODE *device, void *state, APPHINT_ID id, const IMG_CHAR *pAppHintDefault, IMG_CHAR *buffer, size_t size) {
+	return !pvr_apphint_get_string(device, id, buffer, size);
+}
+
+#define OSGetKMAppHintUINT32(device, state, name, appHintDefault, value) \
+	os_get_km_apphint_UINT32(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetKMAppHintUINT64(device, state, name, appHintDefault, value) \
+	os_get_km_apphint_UINT64(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetKMAppHintBOOL(device, state, name, appHintDefault, value) \
+	os_get_km_apphint_BOOL(device, state, APPHINT_ID_ ## name, appHintDefault, value)
+
+#define OSGetKMAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	os_get_km_apphint_STRING(device, state, APPHINT_ID_ ## name, appHintDefault, buffer, size)
+
+
+#define OSCreateKMAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#define OSFreeKMAppHintState(state) \
+	PVR_UNREFERENCED_PARAMETER(state)
+
+#else /* defined(__linux__) && !defined(DOXYGEN) */
+
+/**************************************************************************/ /*!
+@def OSGetKMAppHintUINT32(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint32 km app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetKMAppHintUINT32(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetKMAppHintUINT64(state, name, appHintDefault, value)
+@Description    Interface for retrieval of uint64 km app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetKMAppHintUINT64(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_UINT_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetKMAppHintBOOL(state, name, appHintDefault, value)
+@Description    Interface for retrieval of IMG_BOOL km app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         value             Pointer to returned app hint value.
+ */ /**************************************************************************/
+#define OSGetKMAppHintBOOL(device, state, name, appHintDefault, value) \
+	PVRSRVGetAppHint(state, # name, IMG_BOOL_TYPE, appHintDefault, value)
+
+/**************************************************************************/ /*!
+@def OSGetKMAppHintSTRING(state, name, appHintDefault, buffer, size)
+@Description    Interface for retrieval of string km app hint.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVGetAppHint() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Input          device            Device node
+@Input          state             App hint state
+@Input          name              Name used to identify app hint
+@Input          appHintDefault    Default value to be returned if no
+								  app hint is found.
+@Output         buffer            Buffer used to return app hint string.
+@Input			size			  Size of the buffer.
+ */ /**************************************************************************/
+#define OSGetKMAppHintSTRING(device, state, name, appHintDefault, buffer, size) \
+	(PVR_UNREFERENCED_PARAMETER(size), PVRSRVGetAppHint(state, # name, IMG_STRING_TYPE, appHintDefault, buffer))
+
+/**************************************************************************/ /*!
+@def OSCreateKMAppHintState(state)
+@Description    Creates the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSCreateKMAppHintState(state) \
+	PVRSRVCreateAppHintState(IMG_SRV_UM, 0, state)
+
+/**************************************************************************/ /*!
+@def OSFreeKMAppHintState
+@Description    Free the app hint state.
+				For non-linux operating systems, this macro implements a call
+				from server code to PVRSRVCreateAppHintState() declared in
+				services_client_porting.h, effectively making it 'shared' code.
+@Output          state             App hint state
+ */ /**************************************************************************/
+#define OSFreeKMAppHintState(state) \
+	PVRSRVFreeAppHintState(IMG_SRV_UM, state)
+
+#endif /* defined(__linux__) */
+
+#endif /* OSKM_APPHINT_H */
+
+/******************************************************************************
+ End of file (oskm_apphint.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/osmmap.h b/drivers/gpu/drm/img-rogue/osmmap.h
new file mode 100644
index 000000000000..40a509d19440
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osmmap.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS Interface for mapping PMRs into CPU space.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS abstraction for the mmap2 interface for mapping PMRs into
+                User Mode memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef OSMMAP_H
+#define OSMMAP_H
+
+#include <powervr/mem_types.h>
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+/*************************************************************************/ /*!
+@Function       OSMMapPMR
+@Description    Maps the specified PMR into CPU memory so that it may be
+                accessed by the user process.
+                Whether the memory is mapped read only, read/write, or not at
+                all, is dependent on the PMR itself.
+                The PMR handle is opaque to the user, and lower levels of this
+                stack ensure that the handle is private to this process, such
+                that this API cannot be abused to gain access to other people's
+                PMRs. The OS implementation of this function should return the
+                virtual address and length for the User to use. The "PrivData"
+                is to be stored opaquely by the caller (N.B. he should make no
+                assumptions, in particular, NULL is a valid handle) and given
+                back to the call to OSMUnmapPMR.
+                The OS implementation is free to use the PrivData handle for
+                any purpose it sees fit.
+@Input          hBridge              The bridge handle.
+@Input          hPMR                 The handle of the PMR to be mapped.
+@Input          uiPMRLength          The size of the PMR.
+@Input          uiFlags              Flags indicating how the mapping should
+                                     be done (read-only, etc). These may not
+                                     be honoured if the PMR does not permit
+                                     them.
+@Output         phOSMMapPrivDataOut  Returned private data.
+@Output         ppvMappingAddressOut The returned mapping.
+@Output         puiMappingLengthOut  The size of the returned mapping.
+@Return         PVRSRV_OK on success, failure code otherwise.
+ */ /*************************************************************************/
+PVRSRV_ERROR
+OSMMapPMR(IMG_HANDLE hBridge,
+          IMG_HANDLE hPMR,
+          IMG_DEVMEM_SIZE_T uiPMRLength,
+          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+          IMG_HANDLE *phOSMMapPrivDataOut,
+          void **ppvMappingAddressOut,
+          size_t *puiMappingLengthOut);
+
+/*************************************************************************/ /*!
+@Function       OSMUnmapPMR
+@Description    Unmaps the specified PMR from CPU memory.
+                This function is the counterpart to OSMMapPMR.
+                The caller is required to pass the PMR handle back in along
+                with the same 3-tuple of information that was returned by the
+                call to OSMMapPMR in phOSMMapPrivDataOut.
+                It is possible to unmap only part of the original mapping
+                with this call, by specifying only the address range to be
+                unmapped in pvMappingAddress and uiMappingLength.
+@Input          hBridge              The bridge handle.
+@Input          hPMR                 The handle of the PMR to be unmapped.
+@Input          hOSMMapPrivData      The OS private data of the mapping.
+@Input          pvMappingAddress     The address to be unmapped.
+@Input          uiMappingLength      The size to be unmapped.
+@Return         PVRSRV_OK on success, failure code otherwise.
+ */ /*************************************************************************/
+void
+OSMUnmapPMR(IMG_HANDLE hBridge,
+            IMG_HANDLE hPMR,
+            IMG_HANDLE hOSMMapPrivData,
+            void *pvMappingAddress,
+            size_t uiMappingLength);
+
+#endif /* OSMMAP_H */
diff --git a/drivers/gpu/drm/img-rogue/osmmap_stub.c b/drivers/gpu/drm/img-rogue/osmmap_stub.c
new file mode 100644
index 000000000000..74bad7073c58
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/osmmap_stub.c
@@ -0,0 +1,146 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS abstraction for the mmap2 interface for mapping PMRs into
+                User Mode memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* our exported API */
+#include "osmmap.h"
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+/* services/include/ */
+
+/* services/include/srvhelper/ */
+#include "ra.h"
+
+#include "pmr.h"
+
+PVRSRV_ERROR
+OSMMapPMR(IMG_HANDLE hBridge,
+          IMG_HANDLE hPMR,
+          IMG_DEVMEM_SIZE_T uiPMRSize,
+          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+          IMG_HANDLE *phOSMMapPrivDataOut,
+          void **ppvMappingAddressOut,
+          size_t *puiMappingLengthOut)
+{
+    PVRSRV_ERROR eError;
+    PMR *psPMR;
+    void *pvKernelAddress;
+    size_t uiLength;
+    IMG_HANDLE hPriv;
+
+    PVR_UNREFERENCED_PARAMETER(hBridge);
+    PVR_UNREFERENCED_PARAMETER(uiFlags);
+
+    /*
+      Normally this function would mmap a PMR into the memory space of
+      user process, but in this case we're taking a PMR and mapping it
+      into kernel virtual space.  We keep the same function name for
+      symmetry as this allows the higher layers of the software stack
+      to not care whether they are user mode or kernel
+    */
+
+    psPMR = hPMR;
+
+    if (PMR_IsSparse(psPMR))
+    {
+        eError = PMRAcquireSparseKernelMappingData(psPMR,
+                                            0,
+                                            0,
+                                            &pvKernelAddress,
+                                            &uiLength,
+                                            &hPriv);
+    }
+    else
+    {
+        eError = PMRAcquireKernelMappingData(psPMR,
+                                            0,
+                                            0,
+                                            &pvKernelAddress,
+                                            &uiLength,
+                                            &hPriv);
+    }
+    if (eError != PVRSRV_OK)
+    {
+        goto e0;
+    }
+
+    *phOSMMapPrivDataOut = hPriv;
+    *ppvMappingAddressOut = pvKernelAddress;
+    *puiMappingLengthOut = uiLength;
+
+    /* MappingLength might be rounded up to page size */
+    PVR_ASSERT(*puiMappingLengthOut >= uiPMRSize);
+
+    return PVRSRV_OK;
+
+    /*
+      error exit paths follow
+    */
+
+e0:
+    PVR_ASSERT(eError != PVRSRV_OK);
+    return eError;
+}
+
+void
+OSMUnmapPMR(IMG_HANDLE hBridge,
+            IMG_HANDLE hPMR,
+            IMG_HANDLE hOSMMapPrivData,
+            void *pvMappingAddress,
+            size_t uiMappingLength)
+{
+    PMR *psPMR;
+
+    PVR_UNREFERENCED_PARAMETER(hBridge);
+    PVR_UNREFERENCED_PARAMETER(pvMappingAddress);
+    PVR_UNREFERENCED_PARAMETER(uiMappingLength);
+
+    psPMR = hPMR;
+    PMRReleaseKernelMappingData(psPMR,
+                                hOSMMapPrivData);
+}
diff --git a/drivers/gpu/drm/img-rogue/ospvr_gputrace.h b/drivers/gpu/drm/img-rogue/ospvr_gputrace.h
new file mode 100644
index 000000000000..6bafc4d6042f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/ospvr_gputrace.h
@@ -0,0 +1,185 @@
+/*************************************************************************/ /*!
+@File           ospvr_gputrace.h
+@Title          PVR GPU Trace module common environment interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_GPUTRACE_H_
+#define PVR_GPUTRACE_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_hwperf.h"
+#include "rgxdevice.h"
+#include "device.h"
+
+#if defined(__linux__) && defined(CONFIG_EVENT_TRACING) && \
+	(defined(PVRSRV_TRACE_ROGUE_EVENTS) || \
+	 defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD))
+
+/* Early initialisation of GPU Trace events logic.
+ * This function is called on *driver* initialisation. */
+PVRSRV_ERROR PVRGpuTraceSupportInit(void);
+
+/* GPU Trace resources final cleanup.
+ * This function is called on driver de-initialisation. */
+void PVRGpuTraceSupportDeInit(void);
+
+/* Initialisation for AppHints callbacks.
+ * This function is called during the late stage of driver initialisation but
+ * before the device initialisation but after the debugfs sub-system has been
+ * initialised. */
+void PVRGpuTraceInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* Per-device initialisation of the GPU Trace resources */
+PVRSRV_ERROR PVRGpuTraceInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* TL Initialisation of FTrace */
+PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/* Per-device cleanup for the GPU Trace resources. */
+void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* Enables the gpu trace sub-system for a given device. */
+PVRSRV_ERROR PVRGpuTraceSetEnabled(
+		PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_BOOL bNewValue);
+
+/* Performs some initialisation steps if the feature was enabled
+ * on driver startup. */
+void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* FTrace events callbacks interface */
+
+void PVRGpuTraceEnableUfoCallback(void);
+void PVRGpuTraceDisableUfoCallback(void);
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void);
+void PVRGpuTraceDisableFirmwareActivityCallback(void);
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+PVRSRV_ERROR
+PVRSRVGpuTraceWorkPeriodEventStatsRegister(IMG_HANDLE*
+		phGpuWorkPeriodEventStats);
+void PVRSRVGpuTraceWorkPeriodEventStatsUnregister(
+		IMG_HANDLE hGpuWorkPeriodEventStats);
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#else /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) */
+
+static inline PVRSRV_ERROR PVRGpuTraceSupportInit(void) {
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceSupportDeInit(void) {}
+
+static inline void PVRGpuTraceInitAppHintCallbacks(
+		const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceInitDevice(
+		PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline PVRSRV_ERROR PVRGpuTraceSetEnabled(
+		PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_BOOL bNewValue)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(bNewValue);
+	return PVRSRV_OK;
+}
+
+static inline void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+static inline void PVRGpuTraceEnableUfoCallback(void) {}
+static inline void PVRGpuTraceDisableUfoCallback(void) {}
+
+static inline void PVRGpuTraceEnableFirmwareActivityCallback(void) {}
+static inline void PVRGpuTraceDisableFirmwareActivityCallback(void) {}
+
+#endif /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) */
+
+#if defined(__linux__) && defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExternalJobRef,
+		IMG_UINT32 ui32InternalJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType);
+
+#else
+
+static inline void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExternalJobRef,
+		IMG_UINT32 ui32InternalJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ui32FirmwareCtx);
+	PVR_UNREFERENCED_PARAMETER(ui32ExternalJobRef);
+	PVR_UNREFERENCED_PARAMETER(ui32InternalJobRef);
+	PVR_UNREFERENCED_PARAMETER(eKickType);
+}
+
+#endif /* defined(__linux__) && defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#endif /* PVR_GPUTRACE_H_ */
diff --git a/drivers/gpu/drm/img-rogue/pci_support.c b/drivers/gpu/drm/img-rogue/pci_support.c
new file mode 100644
index 000000000000..a8321a493e3e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pci_support.c
@@ -0,0 +1,643 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/pci.h>
+
+#if defined(CONFIG_MTRR)
+#include <asm/mtrr.h>
+#endif
+
+#include "pci_support.h"
+#include "allocmem.h"
+
+typedef	struct _PVR_PCI_DEV_TAG
+{
+	struct pci_dev		*psPCIDev;
+	HOST_PCI_INIT_FLAGS	ePCIFlags;
+	IMG_BOOL		abPCIResourceInUse[DEVICE_COUNT_RESOURCE];
+	int			iMTRR[DEVICE_COUNT_RESOURCE];
+} PVR_PCI_DEV;
+
+/*************************************************************************/ /*!
+@Function       OSPCISetDev
+@Description    Set a PCI device for subsequent use.
+@Input          pvPCICookie             Pointer to OS specific PCI structure
+@Input          eFlags                  Flags
+@Return		PVRSRV_PCI_DEV_HANDLE   Pointer to PCI device handle
+*/ /**************************************************************************/
+PVRSRV_PCI_DEV_HANDLE OSPCISetDev(void *pvPCICookie, HOST_PCI_INIT_FLAGS eFlags)
+{
+	int err;
+	IMG_UINT32 i;
+	PVR_PCI_DEV *psPVRPCI;
+
+	psPVRPCI = OSAllocMem(sizeof(*psPVRPCI));
+	if (psPVRPCI == NULL)
+	{
+		printk(KERN_ERR "OSPCISetDev: Couldn't allocate PVR PCI structure\n");
+		return NULL;
+	}
+
+	psPVRPCI->psPCIDev = (struct pci_dev *)pvPCICookie;
+	psPVRPCI->ePCIFlags = eFlags;
+
+	err = pci_enable_device(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCISetDev: Couldn't enable device (%d)\n", err);
+		OSFreeMem(psPVRPCI);
+		return NULL;
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_set_master(psPVRPCI->psPCIDev);
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_MSI)		/* PRQA S 3358 */ /* misuse of enums */
+	{
+#if defined(CONFIG_PCI_MSI)
+		err = pci_enable_msi(psPVRPCI->psPCIDev);
+		if (err != 0)
+		{
+			printk(KERN_ERR "OSPCISetDev: Couldn't enable MSI (%d)", err);
+			psPVRPCI->ePCIFlags &= ~HOST_PCI_INIT_FLAG_MSI;	/* PRQA S 1474,3358,4130 */ /* misuse of enums */
+		}
+#else
+		printk(KERN_ERR "OSPCISetDev: MSI support not enabled in the kernel");
+#endif
+	}
+
+	/* Initialise the PCI resource and MTRR tracking array */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		psPVRPCI->abPCIResourceInUse[i] = IMG_FALSE;
+		psPVRPCI->iMTRR[i] = -1;
+	}
+
+	return (PVRSRV_PCI_DEV_HANDLE)psPVRPCI;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAcquireDev
+@Description    Acquire a PCI device for subsequent use.
+@Input          ui16VendorID            Vendor PCI ID
+@Input          ui16DeviceID            Device PCI ID
+@Input          eFlags                  Flags
+@Return		PVRSRV_PCI_DEV_HANDLE   Pointer to PCI device handle
+*/ /**************************************************************************/
+PVRSRV_PCI_DEV_HANDLE OSPCIAcquireDev(IMG_UINT16 ui16VendorID,
+				      IMG_UINT16 ui16DeviceID,
+				      HOST_PCI_INIT_FLAGS eFlags)
+{
+	struct pci_dev *psPCIDev;
+
+	psPCIDev = pci_get_device(ui16VendorID, ui16DeviceID, NULL);
+	if (psPCIDev == NULL)
+	{
+		return NULL;
+	}
+
+	return OSPCISetDev((void *)psPCIDev, eFlags);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIIRQ
+@Description    Get the interrupt number for the device.
+@Input          hPVRPCI                 PCI device handle
+@Output         pui16DeviceID           Pointer to where the interrupt number
+                                        should be returned
+@Return		PVRSRV_ERROR            Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIIRQ(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 *pui32IRQ)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (pui32IRQ == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui32IRQ = psPVRPCI->psPCIDev->irq;
+
+	return PVRSRV_OK;
+}
+
+/* Functions supported by OSPCIAddrRangeFunc */
+enum HOST_PCI_ADDR_RANGE_FUNC
+{
+	HOST_PCI_ADDR_RANGE_FUNC_LEN,
+	HOST_PCI_ADDR_RANGE_FUNC_START,
+	HOST_PCI_ADDR_RANGE_FUNC_END,
+	HOST_PCI_ADDR_RANGE_FUNC_REQUEST,
+	HOST_PCI_ADDR_RANGE_FUNC_RELEASE
+};
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeFunc
+@Description    Internal support function for various address range related
+                functions
+@Input          eFunc                   Function to perform
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Function dependent value
+*/ /**************************************************************************/
+static IMG_UINT64 OSPCIAddrRangeFunc(enum HOST_PCI_ADDR_RANGE_FUNC eFunc,
+										 PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+										 IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (ui32Index >= DEVICE_COUNT_RESOURCE)
+	{
+		printk(KERN_ERR "OSPCIAddrRangeFunc: Index out of range");
+		return 0;
+	}
+
+	switch (eFunc)
+	{
+		case HOST_PCI_ADDR_RANGE_FUNC_LEN:
+		{
+			return pci_resource_len(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_START:
+		{
+			return pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_END:
+		{
+			return pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_REQUEST:
+		{
+			int err = pci_request_region(psPVRPCI->psPCIDev, (IMG_INT)ui32Index, PVRSRV_MODNAME);
+			if (err != 0)
+			{
+				printk(KERN_ERR "OSPCIAddrRangeFunc: pci_request_region_failed (%d)", err);
+				return 0;
+			}
+			psPVRPCI->abPCIResourceInUse[ui32Index] = IMG_TRUE;
+			return 1;
+		}
+		case HOST_PCI_ADDR_RANGE_FUNC_RELEASE:
+		{
+			if (psPVRPCI->abPCIResourceInUse[ui32Index])
+			{
+				pci_release_region(psPVRPCI->psPCIDev, (IMG_INT)ui32Index);
+				psPVRPCI->abPCIResourceInUse[ui32Index] = IMG_FALSE;
+			}
+			return 1;
+		}
+		default:
+		{
+			printk(KERN_ERR "OSPCIAddrRangeFunc: Unknown function");
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeLen
+@Description    Returns length of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Length of address range or 0 if no
+                                        such range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeLen(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_LEN, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeStart
+@Description    Returns the start of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              Start of address range or 0 if no
+                                        such range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeStart(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_START, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIAddrRangeEnd
+@Description    Returns the end of a given address range
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return		IMG_UINT32              End of address range or 0 if no such
+                                        range
+*/ /**************************************************************************/
+IMG_UINT64 OSPCIAddrRangeEnd(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	return OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_END, hPVRPCI, ui32Index);
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIRequestAddrRange
+@Description    Request a given address range index for subsequent use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIRequestAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+								   IMG_UINT32 ui32Index)
+{
+	if (OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_REQUEST, hPVRPCI, ui32Index) == 0)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseAddrRange
+@Description    Release a given address range that is no longer being used
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	if (OSPCIAddrRangeFunc(HOST_PCI_ADDR_RANGE_FUNC_RELEASE, hPVRPCI, ui32Index) == 0)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIRequestAddrRegion
+@Description    Request a given region from an address range for subsequent use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Input          uiOffset              Offset into the address range that forms
+                                        the start of the region
+@Input          uiLength              Length of the region
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIRequestAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+									IMG_UINT32 ui32Index,
+									IMG_UINT64 uiOffset,
+									IMG_UINT64 uiLength)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start;
+	resource_size_t end;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+
+	/* Check that the requested region is valid */
+	if ((start + uiOffset + uiLength - 1) > end)
+	{
+		return PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH;
+	}
+
+	if (pci_resource_flags(psPVRPCI->psPCIDev, ui32Index) & IORESOURCE_IO)
+	{
+		if (request_region(start + uiOffset, uiLength, PVRSRV_MODNAME) == NULL)
+		{
+			return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		}
+	}
+	else
+	{
+		if (request_mem_region(start + uiOffset, uiLength, PVRSRV_MODNAME) == NULL)
+		{
+			return PVRSRV_ERROR_PCI_REGION_UNAVAILABLE;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseAddrRegion
+@Description    Release a given region, from an address range, that is no
+                longer in use
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Input          ui32Offset              Offset into the address range that forms
+                                        the start of the region
+@Input          ui32Length              Length of the region
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+									IMG_UINT32 ui32Index,
+									IMG_UINT64 uiOffset,
+									IMG_UINT64 uiLength)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start;
+	resource_size_t end;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index);
+
+	/* Check that the region is valid */
+	if ((start + uiOffset + uiLength - 1) > end)
+	{
+		return PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH;
+	}
+
+	if (pci_resource_flags(psPVRPCI->psPCIDev, ui32Index) & IORESOURCE_IO)
+	{
+		release_region(start + uiOffset, uiLength);
+	}
+	else
+	{
+		release_mem_region(start + uiOffset, uiLength);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseDev
+@Description    Release a PCI device that is no longer being used
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIReleaseDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int i;
+
+	/* Release all PCI regions that are currently in use */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			pci_release_region(psPVRPCI->psPCIDev, i);
+			psPVRPCI->abPCIResourceInUse[i] = IMG_FALSE;
+		}
+	}
+
+#if defined(CONFIG_PCI_MSI)
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_MSI)		/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_disable_msi(psPVRPCI->psPCIDev);
+	}
+#endif
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+	{
+		pci_clear_master(psPVRPCI->psPCIDev);
+	}
+
+	pci_disable_device(psPVRPCI->psPCIDev);
+
+	OSFreeMem(psPVRPCI);
+	/*not nulling pointer, copy on stack*/
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCISuspendDev
+@Description    Prepare PCI device to be turned off by power management
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCISuspendDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int i;
+	int err;
+
+	/* Release all PCI regions that are currently in use */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			pci_release_region(psPVRPCI->psPCIDev, i);
+		}
+	}
+
+	err = pci_save_state(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCISuspendDev: pci_save_state_failed (%d)", err);
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+
+	pci_disable_device(psPVRPCI->psPCIDev);
+
+	err = pci_set_power_state(psPVRPCI->psPCIDev, pci_choose_state(psPVRPCI->psPCIDev, PMSG_SUSPEND));
+	switch (err)
+	{
+		case 0:
+			break;
+		case -EIO:
+			printk(KERN_ERR "OSPCISuspendDev: device doesn't support PCI PM");
+			break;
+		case -EINVAL:
+			printk(KERN_ERR "OSPCISuspendDev: can't enter requested power state");
+			break;
+		default:
+			printk(KERN_ERR "OSPCISuspendDev: pci_set_power_state failed (%d)", err);
+			break;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIResumeDev
+@Description    Prepare a PCI device to be resumed by power management
+@Input          hPVRPCI                 PCI device handle
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIResumeDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	int err;
+	int i;
+
+	err = pci_set_power_state(psPVRPCI->psPCIDev, pci_choose_state(psPVRPCI->psPCIDev, PMSG_ON));
+	switch (err)
+	{
+		case 0:
+			break;
+		case -EIO:
+			printk(KERN_ERR "OSPCIResumeDev: device doesn't support PCI PM");
+			break;
+		case -EINVAL:
+			printk(KERN_ERR "OSPCIResumeDev: can't enter requested power state");
+			return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+		default:
+			printk(KERN_ERR "OSPCIResumeDev: pci_set_power_state failed (%d)", err);
+			return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	pci_restore_state(psPVRPCI->psPCIDev);
+
+	err = pci_enable_device(psPVRPCI->psPCIDev);
+	if (err != 0)
+	{
+		printk(KERN_ERR "OSPCIResumeDev: Couldn't enable device (%d)", err);
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+
+	if (psPVRPCI->ePCIFlags & HOST_PCI_INIT_FLAG_BUS_MASTER)	/* PRQA S 3358 */ /* misuse of enums */
+		pci_set_master(psPVRPCI->psPCIDev);
+
+	/* Restore the PCI resource tracking array */
+	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
+	{
+		if (psPVRPCI->abPCIResourceInUse[i])
+		{
+			err = pci_request_region(psPVRPCI->psPCIDev, i, PVRSRV_MODNAME);
+			if (err != 0)
+			{
+				printk(KERN_ERR "OSPCIResumeDev: pci_request_region_failed (region %d, error %d)", i, err);
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIGetVendorDeviceIDs
+@Description    Retrieve PCI vendor ID and device ID.
+@Input          hPVRPCI                 PCI device handle
+@Output         pui16VendorID           Vendor ID
+@Output         pui16DeviceID           Device ID
+@Return         PVRSRV_ERROR            Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIGetVendorDeviceIDs(PVRSRV_PCI_DEV_HANDLE hPVRPCI,
+                                     IMG_UINT16 *pui16VendorID,
+                                     IMG_UINT16 *pui16DeviceID)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	struct pci_dev *psPCIDev;
+
+	if (psPVRPCI == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psPCIDev = psPVRPCI->psPCIDev;
+	if (psPCIDev == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui16VendorID = psPCIDev->vendor;
+	*pui16DeviceID = psPCIDev->device;
+
+	return PVRSRV_OK;
+}
+
+#if defined(CONFIG_MTRR)
+
+/*************************************************************************/ /*!
+@Function       OSPCIClearResourceMTRRs
+@Description    Clear any BIOS-configured MTRRs for a PCI memory region
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+@Return	        PVRSRV_ERROR	        Services error code
+*/ /**************************************************************************/
+PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+	resource_size_t start, end;
+	int res;
+
+	start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+	end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index) + 1;
+
+	res = arch_io_reserve_memtype_wc(start, end - start);
+	if (res)
+	{
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	res = arch_phys_wc_add(start, end - start);
+	if (res < 0)
+	{
+		arch_io_free_memtype_wc(start, end - start);
+
+		return PVRSRV_ERROR_PCI_CALL_FAILED;
+	}
+	psPVRPCI->iMTRR[ui32Index] = res;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       OSPCIReleaseResourceMTRRs
+@Description    Release resources allocated by OSPCIClearResourceMTRRs
+@Input          hPVRPCI                 PCI device handle
+@Input          ui32Index               Address range index
+*/ /**************************************************************************/
+void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_PCI_DEV *psPVRPCI = (PVR_PCI_DEV *)hPVRPCI;
+
+	if (psPVRPCI->iMTRR[ui32Index] >= 0)
+	{
+		arch_phys_wc_del(psPVRPCI->iMTRR[ui32Index]);
+		psPVRPCI->iMTRR[ui32Index] = -1;
+
+		{
+			resource_size_t start, end;
+
+			start = pci_resource_start(psPVRPCI->psPCIDev, ui32Index);
+			end = pci_resource_end(psPVRPCI->psPCIDev, ui32Index) + 1;
+
+			arch_io_free_memtype_wc(start, end - start);
+		}
+	}
+}
+#endif /* defined(CONFIG_MTRR) */
diff --git a/drivers/gpu/drm/img-rogue/pci_support.h b/drivers/gpu/drm/img-rogue/pci_support.h
new file mode 100644
index 000000000000..29746c672047
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pci_support.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PCI_SUPPORT_H
+#define PCI_SUPPORT_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#if defined(__linux__)
+#include <linux/pci.h>
+#define TO_PCI_COOKIE(dev) to_pci_dev((struct device *)(dev))
+#else
+#define TO_PCI_COOKIE(dev) (dev)
+#endif
+
+typedef enum _HOST_PCI_INIT_FLAGS_
+{
+	HOST_PCI_INIT_FLAG_BUS_MASTER	= 0x00000001,
+	HOST_PCI_INIT_FLAG_MSI		= 0x00000002,
+	HOST_PCI_INIT_FLAG_FORCE_I32	= 0x7fffffff
+} HOST_PCI_INIT_FLAGS;
+
+struct _PVRSRV_PCI_DEV_OPAQUE_STRUCT_;
+typedef struct _PVRSRV_PCI_DEV_OPAQUE_STRUCT_ *PVRSRV_PCI_DEV_HANDLE;
+
+PVRSRV_PCI_DEV_HANDLE OSPCIAcquireDev(IMG_UINT16 ui16VendorID, IMG_UINT16 ui16DeviceID, HOST_PCI_INIT_FLAGS eFlags);
+PVRSRV_PCI_DEV_HANDLE OSPCISetDev(void *pvPCICookie, HOST_PCI_INIT_FLAGS eFlags);
+PVRSRV_ERROR OSPCIReleaseDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIIRQ(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 *pui32IRQ);
+IMG_UINT64 OSPCIAddrRangeLen(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+IMG_UINT64 OSPCIAddrRangeStart(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+IMG_UINT64 OSPCIAddrRangeEnd(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIRequestAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIReleaseAddrRange(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+PVRSRV_ERROR OSPCIRequestAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index, IMG_UINT64 uiOffset, IMG_UINT64 uiLength);
+PVRSRV_ERROR OSPCIReleaseAddrRegion(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index, IMG_UINT64 uiOffset, IMG_UINT64 uiLength);
+PVRSRV_ERROR OSPCISuspendDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIResumeDev(PVRSRV_PCI_DEV_HANDLE hPVRPCI);
+PVRSRV_ERROR OSPCIGetVendorDeviceIDs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT16 *pui16VendorID, IMG_UINT16 *pui16DeviceID);
+
+#if defined(CONFIG_MTRR)
+PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index);
+#else
+static inline PVRSRV_ERROR OSPCIClearResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRPCI);
+	PVR_UNREFERENCED_PARAMETER(ui32Index);
+	return PVRSRV_OK;
+}
+
+static inline void OSPCIReleaseResourceMTRRs(PVRSRV_PCI_DEV_HANDLE hPVRPCI, IMG_UINT32 ui32Index)
+{
+	PVR_UNREFERENCED_PARAMETER(hPVRPCI);
+	PVR_UNREFERENCED_PARAMETER(ui32Index);
+}
+#endif
+
+#endif /* PCI_SUPPORT_H */
diff --git a/drivers/gpu/drm/img-rogue/pdp/drm_pdp.mk b/drivers/gpu/drm/img-rogue/pdp/drm_pdp.mk
new file mode 100644
index 000000000000..f4ac53c4a7d2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdp/drm_pdp.mk
@@ -0,0 +1,13 @@
+drm_pdp-y += \
+ ../apollo/drm_pdp_crtc.o \
+ ../apollo/drm_pdp_debugfs.o \
+ ../apollo/drm_pdp_drv.o \
+ ../apollo/drm_pdp_dvi.o \
+ ../apollo/drm_pdp_fb.o \
+ ../apollo/drm_pdp_gem.o \
+ ../apollo/drm_pdp_modeset.o \
+ ../apollo/drm_pdp_plane.o \
+ ../apollo/drm_pdp_tmds.o \
+ ../apollo/pdp_apollo.o \
+ ../apollo/pdp_odin.o \
+ ../apollo/pdp_plato.o
diff --git a/drivers/gpu/drm/img-rogue/pdp2_mmu_regs.h b/drivers/gpu/drm/img-rogue/pdp2_mmu_regs.h
new file mode 100644
index 000000000000..6164c5814140
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdp2_mmu_regs.h
@@ -0,0 +1,764 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef _PDP2_MMU_REGS_H
+#define _PDP2_MMU_REGS_H
+
+/* Hardware register definitions */
+
+#define PDP_BIF_DIR_BASE_ADDR_OFFSET		(0x0020)
+#define PDP_BIF_DIR_BASE_ADDR_STRIDE		(4)
+#define PDP_BIF_DIR_BASE_ADDR_NO_ENTRIES		(4)
+
+/* PDP_BIF, DIR_BASE_ADDR, MMU_DIR_BASE_ADDR
+Base address in physical memory for MMU Directory n Entries. When MMU_ENABLE_EXT_ADDRESSING is '1', the bits 31:0 are assigned to the address 31+EXT_ADDR_RANGE:0+EXT_ADDR_RANGE, but then any address offset within a page is forced to 0. When MMU_ENABLE_EXT_ADDRESSING is '0', bits 31:12 are assigned to address 31:12
+*/
+#define PDP_BIF_DIR_BASE_ADDR_MMU_DIR_BASE_ADDR_MASK		(0xFFFFFFFF)
+#define PDP_BIF_DIR_BASE_ADDR_MMU_DIR_BASE_ADDR_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_DIR_BASE_ADDR_MMU_DIR_BASE_ADDR_SHIFT		(0)
+#define PDP_BIF_DIR_BASE_ADDR_MMU_DIR_BASE_ADDR_LENGTH		(32)
+#define PDP_BIF_DIR_BASE_ADDR_MMU_DIR_BASE_ADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_TILE_CFG_OFFSET		(0x0040)
+#define PDP_BIF_TILE_CFG_STRIDE		(4)
+#define PDP_BIF_TILE_CFG_NO_ENTRIES		(4)
+
+/* PDP_BIF, TILE_CFG, TILE_128INTERLEAVE
+*/
+#define PDP_BIF_TILE_CFG_TILE_128INTERLEAVE_MASK		(0x00000010)
+#define PDP_BIF_TILE_CFG_TILE_128INTERLEAVE_LSBMASK		(0x00000001)
+#define PDP_BIF_TILE_CFG_TILE_128INTERLEAVE_SHIFT		(4)
+#define PDP_BIF_TILE_CFG_TILE_128INTERLEAVE_LENGTH		(1)
+#define PDP_BIF_TILE_CFG_TILE_128INTERLEAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, TILE_CFG, TILE_ENABLE
+*/
+#define PDP_BIF_TILE_CFG_TILE_ENABLE_MASK		(0x00000008)
+#define PDP_BIF_TILE_CFG_TILE_ENABLE_LSBMASK		(0x00000001)
+#define PDP_BIF_TILE_CFG_TILE_ENABLE_SHIFT		(3)
+#define PDP_BIF_TILE_CFG_TILE_ENABLE_LENGTH		(1)
+#define PDP_BIF_TILE_CFG_TILE_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, TILE_CFG, TILE_STRIDE
+*/
+#define PDP_BIF_TILE_CFG_TILE_STRIDE_MASK		(0x00000007)
+#define PDP_BIF_TILE_CFG_TILE_STRIDE_LSBMASK		(0x00000007)
+#define PDP_BIF_TILE_CFG_TILE_STRIDE_SHIFT		(0)
+#define PDP_BIF_TILE_CFG_TILE_STRIDE_LENGTH		(3)
+#define PDP_BIF_TILE_CFG_TILE_STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_TILE_MIN_ADDR_OFFSET		(0x0050)
+#define PDP_BIF_TILE_MIN_ADDR_STRIDE		(4)
+#define PDP_BIF_TILE_MIN_ADDR_NO_ENTRIES		(4)
+
+/* PDP_BIF, TILE_MIN_ADDR, TILE_MIN_ADDR
+*/
+#define PDP_BIF_TILE_MIN_ADDR_TILE_MIN_ADDR_MASK		(0xFFFFFFFF)
+#define PDP_BIF_TILE_MIN_ADDR_TILE_MIN_ADDR_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_TILE_MIN_ADDR_TILE_MIN_ADDR_SHIFT		(0)
+#define PDP_BIF_TILE_MIN_ADDR_TILE_MIN_ADDR_LENGTH		(32)
+#define PDP_BIF_TILE_MIN_ADDR_TILE_MIN_ADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_TILE_MAX_ADDR_OFFSET		(0x0060)
+#define PDP_BIF_TILE_MAX_ADDR_STRIDE		(4)
+#define PDP_BIF_TILE_MAX_ADDR_NO_ENTRIES		(4)
+
+/* PDP_BIF, TILE_MAX_ADDR, TILE_MAX_ADDR
+*/
+#define PDP_BIF_TILE_MAX_ADDR_TILE_MAX_ADDR_MASK		(0xFFFFFFFF)
+#define PDP_BIF_TILE_MAX_ADDR_TILE_MAX_ADDR_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_TILE_MAX_ADDR_TILE_MAX_ADDR_SHIFT		(0)
+#define PDP_BIF_TILE_MAX_ADDR_TILE_MAX_ADDR_LENGTH		(32)
+#define PDP_BIF_TILE_MAX_ADDR_TILE_MAX_ADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_CONTROL0_OFFSET		(0x0000)
+
+/* PDP_BIF, CONTROL0, MMU_TILING_SCHEME
+*/
+#define PDP_BIF_CONTROL0_MMU_TILING_SCHEME_MASK		(0x00000001)
+#define PDP_BIF_CONTROL0_MMU_TILING_SCHEME_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL0_MMU_TILING_SCHEME_SHIFT		(0)
+#define PDP_BIF_CONTROL0_MMU_TILING_SCHEME_LENGTH		(1)
+#define PDP_BIF_CONTROL0_MMU_TILING_SCHEME_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL0, MMU_CACHE_POLICY
+*/
+#define PDP_BIF_CONTROL0_MMU_CACHE_POLICY_MASK		(0x00000100)
+#define PDP_BIF_CONTROL0_MMU_CACHE_POLICY_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL0_MMU_CACHE_POLICY_SHIFT		(8)
+#define PDP_BIF_CONTROL0_MMU_CACHE_POLICY_LENGTH		(1)
+#define PDP_BIF_CONTROL0_MMU_CACHE_POLICY_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL0, FORCE_CACHE_POLICY_BYPASS
+*/
+#define PDP_BIF_CONTROL0_FORCE_CACHE_POLICY_BYPASS_MASK		(0x00000200)
+#define PDP_BIF_CONTROL0_FORCE_CACHE_POLICY_BYPASS_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL0_FORCE_CACHE_POLICY_BYPASS_SHIFT		(9)
+#define PDP_BIF_CONTROL0_FORCE_CACHE_POLICY_BYPASS_LENGTH		(1)
+#define PDP_BIF_CONTROL0_FORCE_CACHE_POLICY_BYPASS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL0, STALL_ON_PROTOCOL_FAULT
+*/
+#define PDP_BIF_CONTROL0_STALL_ON_PROTOCOL_FAULT_MASK		(0x00001000)
+#define PDP_BIF_CONTROL0_STALL_ON_PROTOCOL_FAULT_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL0_STALL_ON_PROTOCOL_FAULT_SHIFT		(12)
+#define PDP_BIF_CONTROL0_STALL_ON_PROTOCOL_FAULT_LENGTH		(1)
+#define PDP_BIF_CONTROL0_STALL_ON_PROTOCOL_FAULT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_CONTROL1_OFFSET		(0x0008)
+
+/* PDP_BIF, CONTROL1, MMU_FLUSH0
+*/
+#define PDP_BIF_CONTROL1_MMU_FLUSH0_MASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_FLUSH0_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_FLUSH0_SHIFT		(0)
+#define PDP_BIF_CONTROL1_MMU_FLUSH0_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_FLUSH0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_FLUSH1
+*/
+#define PDP_BIF_CONTROL1_MMU_FLUSH1_MASK		(0x00000002)
+#define PDP_BIF_CONTROL1_MMU_FLUSH1_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_FLUSH1_SHIFT		(1)
+#define PDP_BIF_CONTROL1_MMU_FLUSH1_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_FLUSH1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_FLUSH2
+*/
+#define PDP_BIF_CONTROL1_MMU_FLUSH2_MASK		(0x00000004)
+#define PDP_BIF_CONTROL1_MMU_FLUSH2_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_FLUSH2_SHIFT		(2)
+#define PDP_BIF_CONTROL1_MMU_FLUSH2_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_FLUSH2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_FLUSH3
+*/
+#define PDP_BIF_CONTROL1_MMU_FLUSH3_MASK		(0x00000008)
+#define PDP_BIF_CONTROL1_MMU_FLUSH3_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_FLUSH3_SHIFT		(3)
+#define PDP_BIF_CONTROL1_MMU_FLUSH3_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_FLUSH3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_INVALDC0
+*/
+#define PDP_BIF_CONTROL1_MMU_INVALDC0_MASK		(0x00000100)
+#define PDP_BIF_CONTROL1_MMU_INVALDC0_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_INVALDC0_SHIFT		(8)
+#define PDP_BIF_CONTROL1_MMU_INVALDC0_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_INVALDC0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_INVALDC1
+*/
+#define PDP_BIF_CONTROL1_MMU_INVALDC1_MASK		(0x00000200)
+#define PDP_BIF_CONTROL1_MMU_INVALDC1_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_INVALDC1_SHIFT		(9)
+#define PDP_BIF_CONTROL1_MMU_INVALDC1_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_INVALDC1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_INVALDC2
+*/
+#define PDP_BIF_CONTROL1_MMU_INVALDC2_MASK		(0x00000400)
+#define PDP_BIF_CONTROL1_MMU_INVALDC2_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_INVALDC2_SHIFT		(10)
+#define PDP_BIF_CONTROL1_MMU_INVALDC2_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_INVALDC2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_INVALDC3
+*/
+#define PDP_BIF_CONTROL1_MMU_INVALDC3_MASK		(0x00000800)
+#define PDP_BIF_CONTROL1_MMU_INVALDC3_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_INVALDC3_SHIFT		(11)
+#define PDP_BIF_CONTROL1_MMU_INVALDC3_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_INVALDC3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_FAULT_CLEAR
+*/
+#define PDP_BIF_CONTROL1_MMU_FAULT_CLEAR_MASK		(0x00010000)
+#define PDP_BIF_CONTROL1_MMU_FAULT_CLEAR_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_FAULT_CLEAR_SHIFT		(16)
+#define PDP_BIF_CONTROL1_MMU_FAULT_CLEAR_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_FAULT_CLEAR_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, PROTOCOL_FAULT_CLEAR
+*/
+#define PDP_BIF_CONTROL1_PROTOCOL_FAULT_CLEAR_MASK		(0x00100000)
+#define PDP_BIF_CONTROL1_PROTOCOL_FAULT_CLEAR_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_PROTOCOL_FAULT_CLEAR_SHIFT		(20)
+#define PDP_BIF_CONTROL1_PROTOCOL_FAULT_CLEAR_LENGTH		(1)
+#define PDP_BIF_CONTROL1_PROTOCOL_FAULT_CLEAR_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_PAUSE_SET
+*/
+#define PDP_BIF_CONTROL1_MMU_PAUSE_SET_MASK		(0x01000000)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_SET_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_SET_SHIFT		(24)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_SET_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_SET_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_PAUSE_CLEAR
+*/
+#define PDP_BIF_CONTROL1_MMU_PAUSE_CLEAR_MASK		(0x02000000)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_CLEAR_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_CLEAR_SHIFT		(25)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_CLEAR_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_PAUSE_CLEAR_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONTROL1, MMU_SOFT_RESET
+*/
+#define PDP_BIF_CONTROL1_MMU_SOFT_RESET_MASK		(0x10000000)
+#define PDP_BIF_CONTROL1_MMU_SOFT_RESET_LSBMASK		(0x00000001)
+#define PDP_BIF_CONTROL1_MMU_SOFT_RESET_SHIFT		(28)
+#define PDP_BIF_CONTROL1_MMU_SOFT_RESET_LENGTH		(1)
+#define PDP_BIF_CONTROL1_MMU_SOFT_RESET_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_BANK_INDEX_OFFSET		(0x0010)
+
+/* PDP_BIF, BANK_INDEX, MMU_BANK_INDEX
+*/
+#define PDP_BIF_BANK_INDEX_MMU_BANK_INDEX_MASK		(0xC0000000)
+#define PDP_BIF_BANK_INDEX_MMU_BANK_INDEX_LSBMASK		(0x00000003)
+#define PDP_BIF_BANK_INDEX_MMU_BANK_INDEX_SHIFT		(30)
+#define PDP_BIF_BANK_INDEX_MMU_BANK_INDEX_LENGTH		(2)
+#define PDP_BIF_BANK_INDEX_MMU_BANK_INDEX_SIGNED_FIELD	IMG_FALSE
+#define PDP_BIF_BANK_INDEX_MMU_BANK_INDEX_NO_REPS		(16)
+#define PDP_BIF_BANK_INDEX_MMU_BANK_INDEX_SIZE		(2)
+
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_OFFSET		(0x0018)
+
+/* PDP_BIF, REQUEST_PRIORITY_ENABLE, CMD_PRIORITY_ENABLE
+*/
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_PRIORITY_ENABLE_MASK		(0x00008000)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_PRIORITY_ENABLE_LSBMASK		(0x00000001)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_PRIORITY_ENABLE_SHIFT		(15)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_PRIORITY_ENABLE_LENGTH		(1)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_PRIORITY_ENABLE_SIGNED_FIELD	IMG_FALSE
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_PRIORITY_ENABLE_NO_REPS		(16)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_PRIORITY_ENABLE_SIZE		(1)
+
+/* PDP_BIF, REQUEST_PRIORITY_ENABLE, CMD_MMU_PRIORITY_ENABLE
+*/
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_MMU_PRIORITY_ENABLE_MASK		(0x00010000)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_MMU_PRIORITY_ENABLE_LSBMASK		(0x00000001)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_MMU_PRIORITY_ENABLE_SHIFT		(16)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_MMU_PRIORITY_ENABLE_LENGTH		(1)
+#define PDP_BIF_REQUEST_PRIORITY_ENABLE_CMD_MMU_PRIORITY_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_OFFSET		(0x001C)
+
+/* PDP_BIF, REQUEST_LIMITED_THROUGHPUT, LIMITED_WORDS
+*/
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_LIMITED_WORDS_MASK		(0x000003FF)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_LIMITED_WORDS_LSBMASK		(0x000003FF)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_LIMITED_WORDS_SHIFT		(0)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_LIMITED_WORDS_LENGTH		(10)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_LIMITED_WORDS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, REQUEST_LIMITED_THROUGHPUT, REQUEST_GAP
+*/
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_REQUEST_GAP_MASK		(0x0FFF0000)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_REQUEST_GAP_LSBMASK		(0x00000FFF)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_REQUEST_GAP_SHIFT		(16)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_REQUEST_GAP_LENGTH		(12)
+#define PDP_BIF_REQUEST_LIMITED_THROUGHPUT_REQUEST_GAP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_ADDRESS_CONTROL_OFFSET		(0x0070)
+
+/* PDP_BIF, ADDRESS_CONTROL, MMU_BYPASS
+*/
+#define PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS_MASK		(0x00000001)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS_LSBMASK		(0x00000001)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS_SHIFT		(0)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS_LENGTH		(1)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_BYPASS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, ADDRESS_CONTROL, MMU_ENABLE_EXT_ADDRESSING
+*/
+#define PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING_MASK		(0x00000010)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING_LSBMASK		(0x00000001)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING_SHIFT		(4)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING_LENGTH		(1)
+#define PDP_BIF_ADDRESS_CONTROL_MMU_ENABLE_EXT_ADDRESSING_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, ADDRESS_CONTROL, UPPER_ADDRESS_FIXED
+*/
+#define PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED_MASK		(0x00FF0000)
+#define PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED_LSBMASK		(0x000000FF)
+#define PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED_SHIFT		(16)
+#define PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED_LENGTH		(8)
+#define PDP_BIF_ADDRESS_CONTROL_UPPER_ADDRESS_FIXED_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_CONFIG0_OFFSET		(0x0080)
+
+/* PDP_BIF, CONFIG0, NUM_REQUESTORS
+*/
+#define PDP_BIF_CONFIG0_NUM_REQUESTORS_MASK		(0x0000000F)
+#define PDP_BIF_CONFIG0_NUM_REQUESTORS_LSBMASK		(0x0000000F)
+#define PDP_BIF_CONFIG0_NUM_REQUESTORS_SHIFT		(0)
+#define PDP_BIF_CONFIG0_NUM_REQUESTORS_LENGTH		(4)
+#define PDP_BIF_CONFIG0_NUM_REQUESTORS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG0, EXTENDED_ADDR_RANGE
+*/
+#define PDP_BIF_CONFIG0_EXTENDED_ADDR_RANGE_MASK		(0x000000F0)
+#define PDP_BIF_CONFIG0_EXTENDED_ADDR_RANGE_LSBMASK		(0x0000000F)
+#define PDP_BIF_CONFIG0_EXTENDED_ADDR_RANGE_SHIFT		(4)
+#define PDP_BIF_CONFIG0_EXTENDED_ADDR_RANGE_LENGTH		(4)
+#define PDP_BIF_CONFIG0_EXTENDED_ADDR_RANGE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG0, GROUP_OVERRIDE_SIZE
+*/
+#define PDP_BIF_CONFIG0_GROUP_OVERRIDE_SIZE_MASK		(0x00000700)
+#define PDP_BIF_CONFIG0_GROUP_OVERRIDE_SIZE_LSBMASK		(0x00000007)
+#define PDP_BIF_CONFIG0_GROUP_OVERRIDE_SIZE_SHIFT		(8)
+#define PDP_BIF_CONFIG0_GROUP_OVERRIDE_SIZE_LENGTH		(3)
+#define PDP_BIF_CONFIG0_GROUP_OVERRIDE_SIZE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG0, ADDR_COHERENCY_SUPPORTED
+*/
+#define PDP_BIF_CONFIG0_ADDR_COHERENCY_SUPPORTED_MASK		(0x00001000)
+#define PDP_BIF_CONFIG0_ADDR_COHERENCY_SUPPORTED_LSBMASK		(0x00000001)
+#define PDP_BIF_CONFIG0_ADDR_COHERENCY_SUPPORTED_SHIFT		(12)
+#define PDP_BIF_CONFIG0_ADDR_COHERENCY_SUPPORTED_LENGTH		(1)
+#define PDP_BIF_CONFIG0_ADDR_COHERENCY_SUPPORTED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG0, MMU_SUPPORTED
+*/
+#define PDP_BIF_CONFIG0_MMU_SUPPORTED_MASK		(0x00002000)
+#define PDP_BIF_CONFIG0_MMU_SUPPORTED_LSBMASK		(0x00000001)
+#define PDP_BIF_CONFIG0_MMU_SUPPORTED_SHIFT		(13)
+#define PDP_BIF_CONFIG0_MMU_SUPPORTED_LENGTH		(1)
+#define PDP_BIF_CONFIG0_MMU_SUPPORTED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG0, TILE_ADDR_GRANULARITY
+*/
+#define PDP_BIF_CONFIG0_TILE_ADDR_GRANULARITY_MASK		(0x001F0000)
+#define PDP_BIF_CONFIG0_TILE_ADDR_GRANULARITY_LSBMASK		(0x0000001F)
+#define PDP_BIF_CONFIG0_TILE_ADDR_GRANULARITY_SHIFT		(16)
+#define PDP_BIF_CONFIG0_TILE_ADDR_GRANULARITY_LENGTH		(5)
+#define PDP_BIF_CONFIG0_TILE_ADDR_GRANULARITY_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG0, NO_READ_REORDER
+*/
+#define PDP_BIF_CONFIG0_NO_READ_REORDER_MASK		(0x00200000)
+#define PDP_BIF_CONFIG0_NO_READ_REORDER_LSBMASK		(0x00000001)
+#define PDP_BIF_CONFIG0_NO_READ_REORDER_SHIFT		(21)
+#define PDP_BIF_CONFIG0_NO_READ_REORDER_LENGTH		(1)
+#define PDP_BIF_CONFIG0_NO_READ_REORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG0, TAGS_SUPPORTED
+*/
+#define PDP_BIF_CONFIG0_TAGS_SUPPORTED_MASK		(0xFFC00000)
+#define PDP_BIF_CONFIG0_TAGS_SUPPORTED_LSBMASK		(0x000003FF)
+#define PDP_BIF_CONFIG0_TAGS_SUPPORTED_SHIFT		(22)
+#define PDP_BIF_CONFIG0_TAGS_SUPPORTED_LENGTH		(10)
+#define PDP_BIF_CONFIG0_TAGS_SUPPORTED_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_CONFIG1_OFFSET		(0x0084)
+
+/* PDP_BIF, CONFIG1, PAGE_SIZE
+*/
+#define PDP_BIF_CONFIG1_PAGE_SIZE_MASK		(0x0000000F)
+#define PDP_BIF_CONFIG1_PAGE_SIZE_LSBMASK		(0x0000000F)
+#define PDP_BIF_CONFIG1_PAGE_SIZE_SHIFT		(0)
+#define PDP_BIF_CONFIG1_PAGE_SIZE_LENGTH		(4)
+#define PDP_BIF_CONFIG1_PAGE_SIZE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG1, PAGE_CACHE_ENTRIES
+*/
+#define PDP_BIF_CONFIG1_PAGE_CACHE_ENTRIES_MASK		(0x0000FF00)
+#define PDP_BIF_CONFIG1_PAGE_CACHE_ENTRIES_LSBMASK		(0x000000FF)
+#define PDP_BIF_CONFIG1_PAGE_CACHE_ENTRIES_SHIFT		(8)
+#define PDP_BIF_CONFIG1_PAGE_CACHE_ENTRIES_LENGTH		(8)
+#define PDP_BIF_CONFIG1_PAGE_CACHE_ENTRIES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG1, DIR_CACHE_ENTRIES
+*/
+#define PDP_BIF_CONFIG1_DIR_CACHE_ENTRIES_MASK		(0x001F0000)
+#define PDP_BIF_CONFIG1_DIR_CACHE_ENTRIES_LSBMASK		(0x0000001F)
+#define PDP_BIF_CONFIG1_DIR_CACHE_ENTRIES_SHIFT		(16)
+#define PDP_BIF_CONFIG1_DIR_CACHE_ENTRIES_LENGTH		(5)
+#define PDP_BIF_CONFIG1_DIR_CACHE_ENTRIES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG1, BANDWIDTH_COUNT_SUPPORTED
+*/
+#define PDP_BIF_CONFIG1_BANDWIDTH_COUNT_SUPPORTED_MASK		(0x01000000)
+#define PDP_BIF_CONFIG1_BANDWIDTH_COUNT_SUPPORTED_LSBMASK		(0x00000001)
+#define PDP_BIF_CONFIG1_BANDWIDTH_COUNT_SUPPORTED_SHIFT		(24)
+#define PDP_BIF_CONFIG1_BANDWIDTH_COUNT_SUPPORTED_LENGTH		(1)
+#define PDP_BIF_CONFIG1_BANDWIDTH_COUNT_SUPPORTED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG1, STALL_COUNT_SUPPORTED
+*/
+#define PDP_BIF_CONFIG1_STALL_COUNT_SUPPORTED_MASK		(0x02000000)
+#define PDP_BIF_CONFIG1_STALL_COUNT_SUPPORTED_LSBMASK		(0x00000001)
+#define PDP_BIF_CONFIG1_STALL_COUNT_SUPPORTED_SHIFT		(25)
+#define PDP_BIF_CONFIG1_STALL_COUNT_SUPPORTED_LENGTH		(1)
+#define PDP_BIF_CONFIG1_STALL_COUNT_SUPPORTED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG1, LATENCY_COUNT_SUPPORTED
+*/
+#define PDP_BIF_CONFIG1_LATENCY_COUNT_SUPPORTED_MASK		(0x04000000)
+#define PDP_BIF_CONFIG1_LATENCY_COUNT_SUPPORTED_LSBMASK		(0x00000001)
+#define PDP_BIF_CONFIG1_LATENCY_COUNT_SUPPORTED_SHIFT		(26)
+#define PDP_BIF_CONFIG1_LATENCY_COUNT_SUPPORTED_LENGTH		(1)
+#define PDP_BIF_CONFIG1_LATENCY_COUNT_SUPPORTED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, CONFIG1, SUPPORT_READ_INTERLEAVE
+*/
+#define PDP_BIF_CONFIG1_SUPPORT_READ_INTERLEAVE_MASK		(0x10000000)
+#define PDP_BIF_CONFIG1_SUPPORT_READ_INTERLEAVE_LSBMASK		(0x00000001)
+#define PDP_BIF_CONFIG1_SUPPORT_READ_INTERLEAVE_SHIFT		(28)
+#define PDP_BIF_CONFIG1_SUPPORT_READ_INTERLEAVE_LENGTH		(1)
+#define PDP_BIF_CONFIG1_SUPPORT_READ_INTERLEAVE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_STATUS0_OFFSET		(0x0088)
+
+/* PDP_BIF, STATUS0, MMU_PF_N_RW
+*/
+#define PDP_BIF_STATUS0_MMU_PF_N_RW_MASK		(0x00000001)
+#define PDP_BIF_STATUS0_MMU_PF_N_RW_LSBMASK		(0x00000001)
+#define PDP_BIF_STATUS0_MMU_PF_N_RW_SHIFT		(0)
+#define PDP_BIF_STATUS0_MMU_PF_N_RW_LENGTH		(1)
+#define PDP_BIF_STATUS0_MMU_PF_N_RW_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, STATUS0, MMU_FAULT_ADDR
+*/
+#define PDP_BIF_STATUS0_MMU_FAULT_ADDR_MASK		(0xFFFFF000)
+#define PDP_BIF_STATUS0_MMU_FAULT_ADDR_LSBMASK		(0x000FFFFF)
+#define PDP_BIF_STATUS0_MMU_FAULT_ADDR_SHIFT		(12)
+#define PDP_BIF_STATUS0_MMU_FAULT_ADDR_LENGTH		(20)
+#define PDP_BIF_STATUS0_MMU_FAULT_ADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_STATUS1_OFFSET		(0x008C)
+
+/* PDP_BIF, STATUS1, MMU_FAULT_REQ_STAT
+*/
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_STAT_MASK		(0x0000FFFF)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_STAT_LSBMASK		(0x0000FFFF)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_STAT_SHIFT		(0)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_STAT_LENGTH		(16)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_STAT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, STATUS1, MMU_FAULT_REQ_ID
+*/
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_ID_MASK		(0x000F0000)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_ID_LSBMASK		(0x0000000F)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_ID_SHIFT		(16)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_ID_LENGTH		(4)
+#define PDP_BIF_STATUS1_MMU_FAULT_REQ_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, STATUS1, MMU_FAULT_INDEX
+*/
+#define PDP_BIF_STATUS1_MMU_FAULT_INDEX_MASK		(0x03000000)
+#define PDP_BIF_STATUS1_MMU_FAULT_INDEX_LSBMASK		(0x00000003)
+#define PDP_BIF_STATUS1_MMU_FAULT_INDEX_SHIFT		(24)
+#define PDP_BIF_STATUS1_MMU_FAULT_INDEX_LENGTH		(2)
+#define PDP_BIF_STATUS1_MMU_FAULT_INDEX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, STATUS1, MMU_FAULT_RNW
+*/
+#define PDP_BIF_STATUS1_MMU_FAULT_RNW_MASK		(0x10000000)
+#define PDP_BIF_STATUS1_MMU_FAULT_RNW_LSBMASK		(0x00000001)
+#define PDP_BIF_STATUS1_MMU_FAULT_RNW_SHIFT		(28)
+#define PDP_BIF_STATUS1_MMU_FAULT_RNW_LENGTH		(1)
+#define PDP_BIF_STATUS1_MMU_FAULT_RNW_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_MEM_REQ_OFFSET		(0x0090)
+
+/* PDP_BIF, MEM_REQ, TAG_OUTSTANDING
+*/
+#define PDP_BIF_MEM_REQ_TAG_OUTSTANDING_MASK		(0x000003FF)
+#define PDP_BIF_MEM_REQ_TAG_OUTSTANDING_LSBMASK		(0x000003FF)
+#define PDP_BIF_MEM_REQ_TAG_OUTSTANDING_SHIFT		(0)
+#define PDP_BIF_MEM_REQ_TAG_OUTSTANDING_LENGTH		(10)
+#define PDP_BIF_MEM_REQ_TAG_OUTSTANDING_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, MEM_REQ, EXT_WRRESP_FAULT
+*/
+#define PDP_BIF_MEM_REQ_EXT_WRRESP_FAULT_MASK		(0x00001000)
+#define PDP_BIF_MEM_REQ_EXT_WRRESP_FAULT_LSBMASK		(0x00000001)
+#define PDP_BIF_MEM_REQ_EXT_WRRESP_FAULT_SHIFT		(12)
+#define PDP_BIF_MEM_REQ_EXT_WRRESP_FAULT_LENGTH		(1)
+#define PDP_BIF_MEM_REQ_EXT_WRRESP_FAULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, MEM_REQ, EXT_RDRESP_FAULT
+*/
+#define PDP_BIF_MEM_REQ_EXT_RDRESP_FAULT_MASK		(0x00002000)
+#define PDP_BIF_MEM_REQ_EXT_RDRESP_FAULT_LSBMASK		(0x00000001)
+#define PDP_BIF_MEM_REQ_EXT_RDRESP_FAULT_SHIFT		(13)
+#define PDP_BIF_MEM_REQ_EXT_RDRESP_FAULT_LENGTH		(1)
+#define PDP_BIF_MEM_REQ_EXT_RDRESP_FAULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, MEM_REQ, EXT_READ_BURST_FAULT
+*/
+#define PDP_BIF_MEM_REQ_EXT_READ_BURST_FAULT_MASK		(0x00004000)
+#define PDP_BIF_MEM_REQ_EXT_READ_BURST_FAULT_LSBMASK		(0x00000001)
+#define PDP_BIF_MEM_REQ_EXT_READ_BURST_FAULT_SHIFT		(14)
+#define PDP_BIF_MEM_REQ_EXT_READ_BURST_FAULT_LENGTH		(1)
+#define PDP_BIF_MEM_REQ_EXT_READ_BURST_FAULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, MEM_REQ, INT_PROTOCOL_FAULT
+*/
+#define PDP_BIF_MEM_REQ_INT_PROTOCOL_FAULT_MASK		(0x80000000)
+#define PDP_BIF_MEM_REQ_INT_PROTOCOL_FAULT_LSBMASK		(0x00000001)
+#define PDP_BIF_MEM_REQ_INT_PROTOCOL_FAULT_SHIFT		(31)
+#define PDP_BIF_MEM_REQ_INT_PROTOCOL_FAULT_LENGTH		(1)
+#define PDP_BIF_MEM_REQ_INT_PROTOCOL_FAULT_SIGNED_FIELD	IMG_FALSE
+#define PDP_BIF_MEM_REQ_INT_PROTOCOL_FAULT_NO_REPS		(16)
+#define PDP_BIF_MEM_REQ_INT_PROTOCOL_FAULT_SIZE		(1)
+
+#define PDP_BIF_MEM_EXT_OUTSTANDING_OFFSET		(0x0094)
+
+/* PDP_BIF, MEM_EXT_OUTSTANDING, READ_WORDS_OUTSTANDING
+*/
+#define PDP_BIF_MEM_EXT_OUTSTANDING_READ_WORDS_OUTSTANDING_MASK		(0x0000FFFF)
+#define PDP_BIF_MEM_EXT_OUTSTANDING_READ_WORDS_OUTSTANDING_LSBMASK		(0x0000FFFF)
+#define PDP_BIF_MEM_EXT_OUTSTANDING_READ_WORDS_OUTSTANDING_SHIFT		(0)
+#define PDP_BIF_MEM_EXT_OUTSTANDING_READ_WORDS_OUTSTANDING_LENGTH		(16)
+#define PDP_BIF_MEM_EXT_OUTSTANDING_READ_WORDS_OUTSTANDING_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_FAULT_SELECT_OFFSET		(0x00A0)
+
+/* PDP_BIF, FAULT_SELECT, MMU_FAULT_SELECT
+*/
+#define PDP_BIF_FAULT_SELECT_MMU_FAULT_SELECT_MASK		(0x0000000F)
+#define PDP_BIF_FAULT_SELECT_MMU_FAULT_SELECT_LSBMASK		(0x0000000F)
+#define PDP_BIF_FAULT_SELECT_MMU_FAULT_SELECT_SHIFT		(0)
+#define PDP_BIF_FAULT_SELECT_MMU_FAULT_SELECT_LENGTH		(4)
+#define PDP_BIF_FAULT_SELECT_MMU_FAULT_SELECT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_PROTOCOL_FAULT_OFFSET		(0x00A8)
+
+/* PDP_BIF, PROTOCOL_FAULT, FAULT_PAGE_BREAK
+*/
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_PAGE_BREAK_MASK		(0x00000001)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_PAGE_BREAK_LSBMASK		(0x00000001)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_PAGE_BREAK_SHIFT		(0)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_PAGE_BREAK_LENGTH		(1)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_PAGE_BREAK_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, PROTOCOL_FAULT, FAULT_WRITE
+*/
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_WRITE_MASK		(0x00000010)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_WRITE_LSBMASK		(0x00000001)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_WRITE_SHIFT		(4)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_WRITE_LENGTH		(1)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_WRITE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, PROTOCOL_FAULT, FAULT_READ
+*/
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_READ_MASK		(0x00000020)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_READ_LSBMASK		(0x00000001)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_READ_SHIFT		(5)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_READ_LENGTH		(1)
+#define PDP_BIF_PROTOCOL_FAULT_FAULT_READ_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_TOTAL_READ_REQ_OFFSET		(0x0100)
+
+/* PDP_BIF, TOTAL_READ_REQ, TOTAL_READ_REQ
+*/
+#define PDP_BIF_TOTAL_READ_REQ_TOTAL_READ_REQ_MASK		(0xFFFFFFFF)
+#define PDP_BIF_TOTAL_READ_REQ_TOTAL_READ_REQ_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_TOTAL_READ_REQ_TOTAL_READ_REQ_SHIFT		(0)
+#define PDP_BIF_TOTAL_READ_REQ_TOTAL_READ_REQ_LENGTH		(32)
+#define PDP_BIF_TOTAL_READ_REQ_TOTAL_READ_REQ_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_TOTAL_WRITE_REQ_OFFSET		(0x0104)
+
+/* PDP_BIF, TOTAL_WRITE_REQ, TOTAL_WRITE_REQ
+*/
+#define PDP_BIF_TOTAL_WRITE_REQ_TOTAL_WRITE_REQ_MASK		(0xFFFFFFFF)
+#define PDP_BIF_TOTAL_WRITE_REQ_TOTAL_WRITE_REQ_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_TOTAL_WRITE_REQ_TOTAL_WRITE_REQ_SHIFT		(0)
+#define PDP_BIF_TOTAL_WRITE_REQ_TOTAL_WRITE_REQ_LENGTH		(32)
+#define PDP_BIF_TOTAL_WRITE_REQ_TOTAL_WRITE_REQ_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_READS_LESS_64_REQ_OFFSET		(0x0108)
+
+/* PDP_BIF, READS_LESS_64_REQ, READS_LESS_64_REQ
+*/
+#define PDP_BIF_READS_LESS_64_REQ_READS_LESS_64_REQ_MASK		(0xFFFFFFFF)
+#define PDP_BIF_READS_LESS_64_REQ_READS_LESS_64_REQ_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_READS_LESS_64_REQ_READS_LESS_64_REQ_SHIFT		(0)
+#define PDP_BIF_READS_LESS_64_REQ_READS_LESS_64_REQ_LENGTH		(32)
+#define PDP_BIF_READS_LESS_64_REQ_READS_LESS_64_REQ_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_WRITES_LESS_64_REQ_OFFSET		(0x010C)
+
+/* PDP_BIF, WRITES_LESS_64_REQ, WRITES_LESS_64_REQ
+*/
+#define PDP_BIF_WRITES_LESS_64_REQ_WRITES_LESS_64_REQ_MASK		(0xFFFFFFFF)
+#define PDP_BIF_WRITES_LESS_64_REQ_WRITES_LESS_64_REQ_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_WRITES_LESS_64_REQ_WRITES_LESS_64_REQ_SHIFT		(0)
+#define PDP_BIF_WRITES_LESS_64_REQ_WRITES_LESS_64_REQ_LENGTH		(32)
+#define PDP_BIF_WRITES_LESS_64_REQ_WRITES_LESS_64_REQ_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_EXT_CMD_STALL_OFFSET		(0x0120)
+
+/* PDP_BIF, EXT_CMD_STALL, EXT_CMD_STALL
+*/
+#define PDP_BIF_EXT_CMD_STALL_EXT_CMD_STALL_MASK		(0xFFFFFFFF)
+#define PDP_BIF_EXT_CMD_STALL_EXT_CMD_STALL_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_EXT_CMD_STALL_EXT_CMD_STALL_SHIFT		(0)
+#define PDP_BIF_EXT_CMD_STALL_EXT_CMD_STALL_LENGTH		(32)
+#define PDP_BIF_EXT_CMD_STALL_EXT_CMD_STALL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_WRITE_REQ_STALL_OFFSET		(0x0124)
+
+/* PDP_BIF, WRITE_REQ_STALL, WRITE_REQ_STALL
+*/
+#define PDP_BIF_WRITE_REQ_STALL_WRITE_REQ_STALL_MASK		(0xFFFFFFFF)
+#define PDP_BIF_WRITE_REQ_STALL_WRITE_REQ_STALL_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_WRITE_REQ_STALL_WRITE_REQ_STALL_SHIFT		(0)
+#define PDP_BIF_WRITE_REQ_STALL_WRITE_REQ_STALL_LENGTH		(32)
+#define PDP_BIF_WRITE_REQ_STALL_WRITE_REQ_STALL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_MISS_STALL_OFFSET		(0x0128)
+
+/* PDP_BIF, MISS_STALL, MMU_MISS_STALL
+*/
+#define PDP_BIF_MISS_STALL_MMU_MISS_STALL_MASK		(0xFFFFFFFF)
+#define PDP_BIF_MISS_STALL_MMU_MISS_STALL_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_MISS_STALL_MMU_MISS_STALL_SHIFT		(0)
+#define PDP_BIF_MISS_STALL_MMU_MISS_STALL_LENGTH		(32)
+#define PDP_BIF_MISS_STALL_MMU_MISS_STALL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_ADDRESS_STALL_OFFSET		(0x012C)
+
+/* PDP_BIF, ADDRESS_STALL, ADDRESS_STALL
+*/
+#define PDP_BIF_ADDRESS_STALL_ADDRESS_STALL_MASK		(0xFFFFFFFF)
+#define PDP_BIF_ADDRESS_STALL_ADDRESS_STALL_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_ADDRESS_STALL_ADDRESS_STALL_SHIFT		(0)
+#define PDP_BIF_ADDRESS_STALL_ADDRESS_STALL_LENGTH		(32)
+#define PDP_BIF_ADDRESS_STALL_ADDRESS_STALL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_TAG_STALL_OFFSET		(0x0130)
+
+/* PDP_BIF, TAG_STALL, TAG_STALL
+*/
+#define PDP_BIF_TAG_STALL_TAG_STALL_MASK		(0xFFFFFFFF)
+#define PDP_BIF_TAG_STALL_TAG_STALL_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_TAG_STALL_TAG_STALL_SHIFT		(0)
+#define PDP_BIF_TAG_STALL_TAG_STALL_LENGTH		(32)
+#define PDP_BIF_TAG_STALL_TAG_STALL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_PEAK_READ_OUTSTANDING_OFFSET		(0x0140)
+
+/* PDP_BIF, PEAK_READ_OUTSTANDING, PEAK_TAG_OUTSTANDING
+*/
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_TAG_OUTSTANDING_MASK		(0x000003FF)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_TAG_OUTSTANDING_LSBMASK		(0x000003FF)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_TAG_OUTSTANDING_SHIFT		(0)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_TAG_OUTSTANDING_LENGTH		(10)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_TAG_OUTSTANDING_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, PEAK_READ_OUTSTANDING, PEAK_READ_LATENCY
+*/
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_READ_LATENCY_MASK		(0xFFFF0000)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_READ_LATENCY_LSBMASK		(0x0000FFFF)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_READ_LATENCY_SHIFT		(16)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_READ_LATENCY_LENGTH		(16)
+#define PDP_BIF_PEAK_READ_OUTSTANDING_PEAK_READ_LATENCY_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_AVERAGE_READ_LATENCY_OFFSET		(0x0144)
+
+/* PDP_BIF, AVERAGE_READ_LATENCY, AVERAGE_READ_LATENCY
+*/
+#define PDP_BIF_AVERAGE_READ_LATENCY_AVERAGE_READ_LATENCY_MASK		(0xFFFFFFFF)
+#define PDP_BIF_AVERAGE_READ_LATENCY_AVERAGE_READ_LATENCY_LSBMASK		(0xFFFFFFFF)
+#define PDP_BIF_AVERAGE_READ_LATENCY_AVERAGE_READ_LATENCY_SHIFT		(0)
+#define PDP_BIF_AVERAGE_READ_LATENCY_AVERAGE_READ_LATENCY_LENGTH		(32)
+#define PDP_BIF_AVERAGE_READ_LATENCY_AVERAGE_READ_LATENCY_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_STATISTICS_CONTROL_OFFSET		(0x0160)
+
+/* PDP_BIF, STATISTICS_CONTROL, BANDWIDTH_STATS_INIT
+*/
+#define PDP_BIF_STATISTICS_CONTROL_BANDWIDTH_STATS_INIT_MASK		(0x00000001)
+#define PDP_BIF_STATISTICS_CONTROL_BANDWIDTH_STATS_INIT_LSBMASK		(0x00000001)
+#define PDP_BIF_STATISTICS_CONTROL_BANDWIDTH_STATS_INIT_SHIFT		(0)
+#define PDP_BIF_STATISTICS_CONTROL_BANDWIDTH_STATS_INIT_LENGTH		(1)
+#define PDP_BIF_STATISTICS_CONTROL_BANDWIDTH_STATS_INIT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, STATISTICS_CONTROL, STALL_STATS_INIT
+*/
+#define PDP_BIF_STATISTICS_CONTROL_STALL_STATS_INIT_MASK		(0x00000002)
+#define PDP_BIF_STATISTICS_CONTROL_STALL_STATS_INIT_LSBMASK		(0x00000001)
+#define PDP_BIF_STATISTICS_CONTROL_STALL_STATS_INIT_SHIFT		(1)
+#define PDP_BIF_STATISTICS_CONTROL_STALL_STATS_INIT_LENGTH		(1)
+#define PDP_BIF_STATISTICS_CONTROL_STALL_STATS_INIT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, STATISTICS_CONTROL, LATENCY_STATS_INIT
+*/
+#define PDP_BIF_STATISTICS_CONTROL_LATENCY_STATS_INIT_MASK		(0x00000004)
+#define PDP_BIF_STATISTICS_CONTROL_LATENCY_STATS_INIT_LSBMASK		(0x00000001)
+#define PDP_BIF_STATISTICS_CONTROL_LATENCY_STATS_INIT_SHIFT		(2)
+#define PDP_BIF_STATISTICS_CONTROL_LATENCY_STATS_INIT_LENGTH		(1)
+#define PDP_BIF_STATISTICS_CONTROL_LATENCY_STATS_INIT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BIF_VERSION_OFFSET		(0x01D0)
+
+/* PDP_BIF, VERSION, MMU_MAJOR_REV
+*/
+#define PDP_BIF_VERSION_MMU_MAJOR_REV_MASK		(0x00FF0000)
+#define PDP_BIF_VERSION_MMU_MAJOR_REV_LSBMASK		(0x000000FF)
+#define PDP_BIF_VERSION_MMU_MAJOR_REV_SHIFT		(16)
+#define PDP_BIF_VERSION_MMU_MAJOR_REV_LENGTH		(8)
+#define PDP_BIF_VERSION_MMU_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, VERSION, MMU_MINOR_REV
+*/
+#define PDP_BIF_VERSION_MMU_MINOR_REV_MASK		(0x0000FF00)
+#define PDP_BIF_VERSION_MMU_MINOR_REV_LSBMASK		(0x000000FF)
+#define PDP_BIF_VERSION_MMU_MINOR_REV_SHIFT		(8)
+#define PDP_BIF_VERSION_MMU_MINOR_REV_LENGTH		(8)
+#define PDP_BIF_VERSION_MMU_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP_BIF, VERSION, MMU_MAINT_REV
+*/
+#define PDP_BIF_VERSION_MMU_MAINT_REV_MASK		(0x000000FF)
+#define PDP_BIF_VERSION_MMU_MAINT_REV_LSBMASK		(0x000000FF)
+#define PDP_BIF_VERSION_MMU_MAINT_REV_SHIFT		(0)
+#define PDP_BIF_VERSION_MMU_MAINT_REV_LENGTH		(8)
+#define PDP_BIF_VERSION_MMU_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#endif /* _PDP2_MMU_REGS_H */
diff --git a/drivers/gpu/drm/img-rogue/pdp2_regs.h b/drivers/gpu/drm/img-rogue/pdp2_regs.h
new file mode 100644
index 000000000000..bf85386b1df5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdp2_regs.h
@@ -0,0 +1,8565 @@
+/*************************************************************************/ /*!
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef _PDP2_REGS_H
+#define _PDP2_REGS_H
+
+/*
+ * Bitfield operations
+ * For each argument field, the following preprocessor macros must exist
+ * field##_MASK - the number of bits in the bit field
+ * field##_SHIFT - offset from the first bit
+ */
+#define PLACE_FIELD(field, val) \
+	(((u32)(val) << (field##_SHIFT)) & (field##_MASK))
+
+#define ADJ_FIELD(x, field, val) \
+	(((x) & ~(field##_MASK)) \
+	| PLACE_FIELD(field, val))
+
+#define SET_FIELD(x, field, val) \
+	(x) = ADJ_FIELD(x, field, val)
+
+#define GET_FIELD(x, field) \
+	(((x) & (field##_MASK)) >> (field##_SHIFT))
+
+/* Keeps most significant bits */
+#define MOVE_FIELD(x, o1, l1, o2, l2) \
+	(((x) >> ((o1) + (l1) - (l2))) << (o2))
+
+#define MAX_FIELD_VALUE(field) \
+	((field##_MASK) >> (field##_SHIFT))
+
+/* Hardware register definitions */
+
+#define PDP_GRPH1SURF_OFFSET		(0x0000)
+
+/* PDP, GRPH1SURF, GRPH1PIXFMT
+*/
+#define PDP_GRPH1SURF_GRPH1PIXFMT_MASK		(0xF8000000)
+#define PDP_GRPH1SURF_GRPH1PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_GRPH1SURF_GRPH1PIXFMT_SHIFT		(27)
+#define PDP_GRPH1SURF_GRPH1PIXFMT_LENGTH		(5)
+#define PDP_GRPH1SURF_GRPH1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USEGAMMA
+*/
+#define PDP_GRPH1SURF_GRPH1USEGAMMA_MASK		(0x04000000)
+#define PDP_GRPH1SURF_GRPH1USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_GRPH1SURF_GRPH1USEGAMMA_SHIFT		(26)
+#define PDP_GRPH1SURF_GRPH1USEGAMMA_LENGTH		(1)
+#define PDP_GRPH1SURF_GRPH1USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USECSC
+*/
+#define PDP_GRPH1SURF_GRPH1USECSC_MASK		(0x02000000)
+#define PDP_GRPH1SURF_GRPH1USECSC_LSBMASK		(0x00000001)
+#define PDP_GRPH1SURF_GRPH1USECSC_SHIFT		(25)
+#define PDP_GRPH1SURF_GRPH1USECSC_LENGTH		(1)
+#define PDP_GRPH1SURF_GRPH1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1LUTRWCHOICE
+*/
+#define PDP_GRPH1SURF_GRPH1LUTRWCHOICE_MASK		(0x01000000)
+#define PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LSBMASK		(0x00000001)
+#define PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SHIFT		(24)
+#define PDP_GRPH1SURF_GRPH1LUTRWCHOICE_LENGTH		(1)
+#define PDP_GRPH1SURF_GRPH1LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SURF, GRPH1USELUT
+*/
+#define PDP_GRPH1SURF_GRPH1USELUT_MASK		(0x00800000)
+#define PDP_GRPH1SURF_GRPH1USELUT_LSBMASK		(0x00000001)
+#define PDP_GRPH1SURF_GRPH1USELUT_SHIFT		(23)
+#define PDP_GRPH1SURF_GRPH1USELUT_LENGTH		(1)
+#define PDP_GRPH1SURF_GRPH1USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2SURF_OFFSET		(0x0004)
+
+/* PDP, GRPH2SURF, GRPH2PIXFMT
+*/
+#define PDP_GRPH2SURF_GRPH2PIXFMT_MASK		(0xF8000000)
+#define PDP_GRPH2SURF_GRPH2PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_GRPH2SURF_GRPH2PIXFMT_SHIFT		(27)
+#define PDP_GRPH2SURF_GRPH2PIXFMT_LENGTH		(5)
+#define PDP_GRPH2SURF_GRPH2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USEGAMMA
+*/
+#define PDP_GRPH2SURF_GRPH2USEGAMMA_MASK		(0x04000000)
+#define PDP_GRPH2SURF_GRPH2USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_GRPH2SURF_GRPH2USEGAMMA_SHIFT		(26)
+#define PDP_GRPH2SURF_GRPH2USEGAMMA_LENGTH		(1)
+#define PDP_GRPH2SURF_GRPH2USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USECSC
+*/
+#define PDP_GRPH2SURF_GRPH2USECSC_MASK		(0x02000000)
+#define PDP_GRPH2SURF_GRPH2USECSC_LSBMASK		(0x00000001)
+#define PDP_GRPH2SURF_GRPH2USECSC_SHIFT		(25)
+#define PDP_GRPH2SURF_GRPH2USECSC_LENGTH		(1)
+#define PDP_GRPH2SURF_GRPH2USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2LUTRWCHOICE
+*/
+#define PDP_GRPH2SURF_GRPH2LUTRWCHOICE_MASK		(0x01000000)
+#define PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LSBMASK		(0x00000001)
+#define PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SHIFT		(24)
+#define PDP_GRPH2SURF_GRPH2LUTRWCHOICE_LENGTH		(1)
+#define PDP_GRPH2SURF_GRPH2LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SURF, GRPH2USELUT
+*/
+#define PDP_GRPH2SURF_GRPH2USELUT_MASK		(0x00800000)
+#define PDP_GRPH2SURF_GRPH2USELUT_LSBMASK		(0x00000001)
+#define PDP_GRPH2SURF_GRPH2USELUT_SHIFT		(23)
+#define PDP_GRPH2SURF_GRPH2USELUT_LENGTH		(1)
+#define PDP_GRPH2SURF_GRPH2USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3SURF_OFFSET		(0x0008)
+
+/* PDP, GRPH3SURF, GRPH3PIXFMT
+*/
+#define PDP_GRPH3SURF_GRPH3PIXFMT_MASK		(0xF8000000)
+#define PDP_GRPH3SURF_GRPH3PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_GRPH3SURF_GRPH3PIXFMT_SHIFT		(27)
+#define PDP_GRPH3SURF_GRPH3PIXFMT_LENGTH		(5)
+#define PDP_GRPH3SURF_GRPH3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USEGAMMA
+*/
+#define PDP_GRPH3SURF_GRPH3USEGAMMA_MASK		(0x04000000)
+#define PDP_GRPH3SURF_GRPH3USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_GRPH3SURF_GRPH3USEGAMMA_SHIFT		(26)
+#define PDP_GRPH3SURF_GRPH3USEGAMMA_LENGTH		(1)
+#define PDP_GRPH3SURF_GRPH3USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USECSC
+*/
+#define PDP_GRPH3SURF_GRPH3USECSC_MASK		(0x02000000)
+#define PDP_GRPH3SURF_GRPH3USECSC_LSBMASK		(0x00000001)
+#define PDP_GRPH3SURF_GRPH3USECSC_SHIFT		(25)
+#define PDP_GRPH3SURF_GRPH3USECSC_LENGTH		(1)
+#define PDP_GRPH3SURF_GRPH3USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3LUTRWCHOICE
+*/
+#define PDP_GRPH3SURF_GRPH3LUTRWCHOICE_MASK		(0x01000000)
+#define PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LSBMASK		(0x00000001)
+#define PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SHIFT		(24)
+#define PDP_GRPH3SURF_GRPH3LUTRWCHOICE_LENGTH		(1)
+#define PDP_GRPH3SURF_GRPH3LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SURF, GRPH3USELUT
+*/
+#define PDP_GRPH3SURF_GRPH3USELUT_MASK		(0x00800000)
+#define PDP_GRPH3SURF_GRPH3USELUT_LSBMASK		(0x00000001)
+#define PDP_GRPH3SURF_GRPH3USELUT_SHIFT		(23)
+#define PDP_GRPH3SURF_GRPH3USELUT_LENGTH		(1)
+#define PDP_GRPH3SURF_GRPH3USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4SURF_OFFSET		(0x000C)
+
+/* PDP, GRPH4SURF, GRPH4PIXFMT
+*/
+#define PDP_GRPH4SURF_GRPH4PIXFMT_MASK		(0xF8000000)
+#define PDP_GRPH4SURF_GRPH4PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_GRPH4SURF_GRPH4PIXFMT_SHIFT		(27)
+#define PDP_GRPH4SURF_GRPH4PIXFMT_LENGTH		(5)
+#define PDP_GRPH4SURF_GRPH4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USEGAMMA
+*/
+#define PDP_GRPH4SURF_GRPH4USEGAMMA_MASK		(0x04000000)
+#define PDP_GRPH4SURF_GRPH4USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_GRPH4SURF_GRPH4USEGAMMA_SHIFT		(26)
+#define PDP_GRPH4SURF_GRPH4USEGAMMA_LENGTH		(1)
+#define PDP_GRPH4SURF_GRPH4USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USECSC
+*/
+#define PDP_GRPH4SURF_GRPH4USECSC_MASK		(0x02000000)
+#define PDP_GRPH4SURF_GRPH4USECSC_LSBMASK		(0x00000001)
+#define PDP_GRPH4SURF_GRPH4USECSC_SHIFT		(25)
+#define PDP_GRPH4SURF_GRPH4USECSC_LENGTH		(1)
+#define PDP_GRPH4SURF_GRPH4USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4LUTRWCHOICE
+*/
+#define PDP_GRPH4SURF_GRPH4LUTRWCHOICE_MASK		(0x01000000)
+#define PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LSBMASK		(0x00000001)
+#define PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SHIFT		(24)
+#define PDP_GRPH4SURF_GRPH4LUTRWCHOICE_LENGTH		(1)
+#define PDP_GRPH4SURF_GRPH4LUTRWCHOICE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SURF, GRPH4USELUT
+*/
+#define PDP_GRPH4SURF_GRPH4USELUT_MASK		(0x00800000)
+#define PDP_GRPH4SURF_GRPH4USELUT_LSBMASK		(0x00000001)
+#define PDP_GRPH4SURF_GRPH4USELUT_SHIFT		(23)
+#define PDP_GRPH4SURF_GRPH4USELUT_LENGTH		(1)
+#define PDP_GRPH4SURF_GRPH4USELUT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1SURF_OFFSET		(0x0010)
+
+/* PDP, VID1SURF, VID1PIXFMT
+*/
+#define PDP_VID1SURF_VID1PIXFMT_MASK		(0xF8000000)
+#define PDP_VID1SURF_VID1PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_VID1SURF_VID1PIXFMT_SHIFT		(27)
+#define PDP_VID1SURF_VID1PIXFMT_LENGTH		(5)
+#define PDP_VID1SURF_VID1PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEGAMMA
+*/
+#define PDP_VID1SURF_VID1USEGAMMA_MASK		(0x04000000)
+#define PDP_VID1SURF_VID1USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_VID1SURF_VID1USEGAMMA_SHIFT		(26)
+#define PDP_VID1SURF_VID1USEGAMMA_LENGTH		(1)
+#define PDP_VID1SURF_VID1USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USECSC
+*/
+#define PDP_VID1SURF_VID1USECSC_MASK		(0x02000000)
+#define PDP_VID1SURF_VID1USECSC_LSBMASK		(0x00000001)
+#define PDP_VID1SURF_VID1USECSC_SHIFT		(25)
+#define PDP_VID1SURF_VID1USECSC_LENGTH		(1)
+#define PDP_VID1SURF_VID1USECSC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEI2P
+*/
+#define PDP_VID1SURF_VID1USEI2P_MASK		(0x01000000)
+#define PDP_VID1SURF_VID1USEI2P_LSBMASK		(0x00000001)
+#define PDP_VID1SURF_VID1USEI2P_SHIFT		(24)
+#define PDP_VID1SURF_VID1USEI2P_LENGTH		(1)
+#define PDP_VID1SURF_VID1USEI2P_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1COSITED
+*/
+#define PDP_VID1SURF_VID1COSITED_MASK		(0x00800000)
+#define PDP_VID1SURF_VID1COSITED_LSBMASK		(0x00000001)
+#define PDP_VID1SURF_VID1COSITED_SHIFT		(23)
+#define PDP_VID1SURF_VID1COSITED_LENGTH		(1)
+#define PDP_VID1SURF_VID1COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEHQCD
+*/
+#define PDP_VID1SURF_VID1USEHQCD_MASK		(0x00400000)
+#define PDP_VID1SURF_VID1USEHQCD_LSBMASK		(0x00000001)
+#define PDP_VID1SURF_VID1USEHQCD_SHIFT		(22)
+#define PDP_VID1SURF_VID1USEHQCD_LENGTH		(1)
+#define PDP_VID1SURF_VID1USEHQCD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SURF, VID1USEINSTREAM
+*/
+#define PDP_VID1SURF_VID1USEINSTREAM_MASK		(0x00200000)
+#define PDP_VID1SURF_VID1USEINSTREAM_LSBMASK		(0x00000001)
+#define PDP_VID1SURF_VID1USEINSTREAM_SHIFT		(21)
+#define PDP_VID1SURF_VID1USEINSTREAM_LENGTH		(1)
+#define PDP_VID1SURF_VID1USEINSTREAM_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2SURF_OFFSET		(0x0014)
+
+/* PDP, VID2SURF, VID2PIXFMT
+*/
+#define PDP_VID2SURF_VID2PIXFMT_MASK		(0xF8000000)
+#define PDP_VID2SURF_VID2PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_VID2SURF_VID2PIXFMT_SHIFT		(27)
+#define PDP_VID2SURF_VID2PIXFMT_LENGTH		(5)
+#define PDP_VID2SURF_VID2PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2COSITED
+*/
+#define PDP_VID2SURF_VID2COSITED_MASK		(0x00800000)
+#define PDP_VID2SURF_VID2COSITED_LSBMASK		(0x00000001)
+#define PDP_VID2SURF_VID2COSITED_SHIFT		(23)
+#define PDP_VID2SURF_VID2COSITED_LENGTH		(1)
+#define PDP_VID2SURF_VID2COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USEGAMMA
+*/
+#define PDP_VID2SURF_VID2USEGAMMA_MASK		(0x04000000)
+#define PDP_VID2SURF_VID2USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_VID2SURF_VID2USEGAMMA_SHIFT		(26)
+#define PDP_VID2SURF_VID2USEGAMMA_LENGTH		(1)
+#define PDP_VID2SURF_VID2USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SURF, VID2USECSC
+*/
+#define PDP_VID2SURF_VID2USECSC_MASK		(0x02000000)
+#define PDP_VID2SURF_VID2USECSC_LSBMASK		(0x00000001)
+#define PDP_VID2SURF_VID2USECSC_SHIFT		(25)
+#define PDP_VID2SURF_VID2USECSC_LENGTH		(1)
+#define PDP_VID2SURF_VID2USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3SURF_OFFSET		(0x0018)
+
+/* PDP, VID3SURF, VID3PIXFMT
+*/
+#define PDP_VID3SURF_VID3PIXFMT_MASK		(0xF8000000)
+#define PDP_VID3SURF_VID3PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_VID3SURF_VID3PIXFMT_SHIFT		(27)
+#define PDP_VID3SURF_VID3PIXFMT_LENGTH		(5)
+#define PDP_VID3SURF_VID3PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3COSITED
+*/
+#define PDP_VID3SURF_VID3COSITED_MASK		(0x00800000)
+#define PDP_VID3SURF_VID3COSITED_LSBMASK		(0x00000001)
+#define PDP_VID3SURF_VID3COSITED_SHIFT		(23)
+#define PDP_VID3SURF_VID3COSITED_LENGTH		(1)
+#define PDP_VID3SURF_VID3COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USEGAMMA
+*/
+#define PDP_VID3SURF_VID3USEGAMMA_MASK		(0x04000000)
+#define PDP_VID3SURF_VID3USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_VID3SURF_VID3USEGAMMA_SHIFT		(26)
+#define PDP_VID3SURF_VID3USEGAMMA_LENGTH		(1)
+#define PDP_VID3SURF_VID3USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SURF, VID3USECSC
+*/
+#define PDP_VID3SURF_VID3USECSC_MASK		(0x02000000)
+#define PDP_VID3SURF_VID3USECSC_LSBMASK		(0x00000001)
+#define PDP_VID3SURF_VID3USECSC_SHIFT		(25)
+#define PDP_VID3SURF_VID3USECSC_LENGTH		(1)
+#define PDP_VID3SURF_VID3USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4SURF_OFFSET		(0x001C)
+
+/* PDP, VID4SURF, VID4PIXFMT
+*/
+#define PDP_VID4SURF_VID4PIXFMT_MASK		(0xF8000000)
+#define PDP_VID4SURF_VID4PIXFMT_LSBMASK		(0x0000001F)
+#define PDP_VID4SURF_VID4PIXFMT_SHIFT		(27)
+#define PDP_VID4SURF_VID4PIXFMT_LENGTH		(5)
+#define PDP_VID4SURF_VID4PIXFMT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4COSITED
+*/
+#define PDP_VID4SURF_VID4COSITED_MASK		(0x00800000)
+#define PDP_VID4SURF_VID4COSITED_LSBMASK		(0x00000001)
+#define PDP_VID4SURF_VID4COSITED_SHIFT		(23)
+#define PDP_VID4SURF_VID4COSITED_LENGTH		(1)
+#define PDP_VID4SURF_VID4COSITED_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USEGAMMA
+*/
+#define PDP_VID4SURF_VID4USEGAMMA_MASK		(0x04000000)
+#define PDP_VID4SURF_VID4USEGAMMA_LSBMASK		(0x00000001)
+#define PDP_VID4SURF_VID4USEGAMMA_SHIFT		(26)
+#define PDP_VID4SURF_VID4USEGAMMA_LENGTH		(1)
+#define PDP_VID4SURF_VID4USEGAMMA_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SURF, VID4USECSC
+*/
+#define PDP_VID4SURF_VID4USECSC_MASK		(0x02000000)
+#define PDP_VID4SURF_VID4USECSC_LSBMASK		(0x00000001)
+#define PDP_VID4SURF_VID4USECSC_SHIFT		(25)
+#define PDP_VID4SURF_VID4USECSC_LENGTH		(1)
+#define PDP_VID4SURF_VID4USECSC_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1CTRL_OFFSET		(0x0020)
+
+/* PDP, GRPH1CTRL, GRPH1STREN
+*/
+#define PDP_GRPH1CTRL_GRPH1STREN_MASK		(0x80000000)
+#define PDP_GRPH1CTRL_GRPH1STREN_LSBMASK		(0x00000001)
+#define PDP_GRPH1CTRL_GRPH1STREN_SHIFT		(31)
+#define PDP_GRPH1CTRL_GRPH1STREN_LENGTH		(1)
+#define PDP_GRPH1CTRL_GRPH1STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYEN
+*/
+#define PDP_GRPH1CTRL_GRPH1CKEYEN_MASK		(0x40000000)
+#define PDP_GRPH1CTRL_GRPH1CKEYEN_LSBMASK		(0x00000001)
+#define PDP_GRPH1CTRL_GRPH1CKEYEN_SHIFT		(30)
+#define PDP_GRPH1CTRL_GRPH1CKEYEN_LENGTH		(1)
+#define PDP_GRPH1CTRL_GRPH1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1CKEYSRC
+*/
+#define PDP_GRPH1CTRL_GRPH1CKEYSRC_MASK		(0x20000000)
+#define PDP_GRPH1CTRL_GRPH1CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_GRPH1CTRL_GRPH1CKEYSRC_SHIFT		(29)
+#define PDP_GRPH1CTRL_GRPH1CKEYSRC_LENGTH		(1)
+#define PDP_GRPH1CTRL_GRPH1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLEND
+*/
+#define PDP_GRPH1CTRL_GRPH1BLEND_MASK		(0x18000000)
+#define PDP_GRPH1CTRL_GRPH1BLEND_LSBMASK		(0x00000003)
+#define PDP_GRPH1CTRL_GRPH1BLEND_SHIFT		(27)
+#define PDP_GRPH1CTRL_GRPH1BLEND_LENGTH		(2)
+#define PDP_GRPH1CTRL_GRPH1BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1BLENDPOS
+*/
+#define PDP_GRPH1CTRL_GRPH1BLENDPOS_MASK		(0x07000000)
+#define PDP_GRPH1CTRL_GRPH1BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_GRPH1CTRL_GRPH1BLENDPOS_SHIFT		(24)
+#define PDP_GRPH1CTRL_GRPH1BLENDPOS_LENGTH		(3)
+#define PDP_GRPH1CTRL_GRPH1BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CTRL, GRPH1DITHEREN
+*/
+#define PDP_GRPH1CTRL_GRPH1DITHEREN_MASK		(0x00800000)
+#define PDP_GRPH1CTRL_GRPH1DITHEREN_LSBMASK		(0x00000001)
+#define PDP_GRPH1CTRL_GRPH1DITHEREN_SHIFT		(23)
+#define PDP_GRPH1CTRL_GRPH1DITHEREN_LENGTH		(1)
+#define PDP_GRPH1CTRL_GRPH1DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2CTRL_OFFSET		(0x0024)
+
+/* PDP, GRPH2CTRL, GRPH2STREN
+*/
+#define PDP_GRPH2CTRL_GRPH2STREN_MASK		(0x80000000)
+#define PDP_GRPH2CTRL_GRPH2STREN_LSBMASK		(0x00000001)
+#define PDP_GRPH2CTRL_GRPH2STREN_SHIFT		(31)
+#define PDP_GRPH2CTRL_GRPH2STREN_LENGTH		(1)
+#define PDP_GRPH2CTRL_GRPH2STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYEN
+*/
+#define PDP_GRPH2CTRL_GRPH2CKEYEN_MASK		(0x40000000)
+#define PDP_GRPH2CTRL_GRPH2CKEYEN_LSBMASK		(0x00000001)
+#define PDP_GRPH2CTRL_GRPH2CKEYEN_SHIFT		(30)
+#define PDP_GRPH2CTRL_GRPH2CKEYEN_LENGTH		(1)
+#define PDP_GRPH2CTRL_GRPH2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2CKEYSRC
+*/
+#define PDP_GRPH2CTRL_GRPH2CKEYSRC_MASK		(0x20000000)
+#define PDP_GRPH2CTRL_GRPH2CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_GRPH2CTRL_GRPH2CKEYSRC_SHIFT		(29)
+#define PDP_GRPH2CTRL_GRPH2CKEYSRC_LENGTH		(1)
+#define PDP_GRPH2CTRL_GRPH2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLEND
+*/
+#define PDP_GRPH2CTRL_GRPH2BLEND_MASK		(0x18000000)
+#define PDP_GRPH2CTRL_GRPH2BLEND_LSBMASK		(0x00000003)
+#define PDP_GRPH2CTRL_GRPH2BLEND_SHIFT		(27)
+#define PDP_GRPH2CTRL_GRPH2BLEND_LENGTH		(2)
+#define PDP_GRPH2CTRL_GRPH2BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2BLENDPOS
+*/
+#define PDP_GRPH2CTRL_GRPH2BLENDPOS_MASK		(0x07000000)
+#define PDP_GRPH2CTRL_GRPH2BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_GRPH2CTRL_GRPH2BLENDPOS_SHIFT		(24)
+#define PDP_GRPH2CTRL_GRPH2BLENDPOS_LENGTH		(3)
+#define PDP_GRPH2CTRL_GRPH2BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CTRL, GRPH2DITHEREN
+*/
+#define PDP_GRPH2CTRL_GRPH2DITHEREN_MASK		(0x00800000)
+#define PDP_GRPH2CTRL_GRPH2DITHEREN_LSBMASK		(0x00000001)
+#define PDP_GRPH2CTRL_GRPH2DITHEREN_SHIFT		(23)
+#define PDP_GRPH2CTRL_GRPH2DITHEREN_LENGTH		(1)
+#define PDP_GRPH2CTRL_GRPH2DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3CTRL_OFFSET		(0x0028)
+
+/* PDP, GRPH3CTRL, GRPH3STREN
+*/
+#define PDP_GRPH3CTRL_GRPH3STREN_MASK		(0x80000000)
+#define PDP_GRPH3CTRL_GRPH3STREN_LSBMASK		(0x00000001)
+#define PDP_GRPH3CTRL_GRPH3STREN_SHIFT		(31)
+#define PDP_GRPH3CTRL_GRPH3STREN_LENGTH		(1)
+#define PDP_GRPH3CTRL_GRPH3STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYEN
+*/
+#define PDP_GRPH3CTRL_GRPH3CKEYEN_MASK		(0x40000000)
+#define PDP_GRPH3CTRL_GRPH3CKEYEN_LSBMASK		(0x00000001)
+#define PDP_GRPH3CTRL_GRPH3CKEYEN_SHIFT		(30)
+#define PDP_GRPH3CTRL_GRPH3CKEYEN_LENGTH		(1)
+#define PDP_GRPH3CTRL_GRPH3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3CKEYSRC
+*/
+#define PDP_GRPH3CTRL_GRPH3CKEYSRC_MASK		(0x20000000)
+#define PDP_GRPH3CTRL_GRPH3CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_GRPH3CTRL_GRPH3CKEYSRC_SHIFT		(29)
+#define PDP_GRPH3CTRL_GRPH3CKEYSRC_LENGTH		(1)
+#define PDP_GRPH3CTRL_GRPH3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLEND
+*/
+#define PDP_GRPH3CTRL_GRPH3BLEND_MASK		(0x18000000)
+#define PDP_GRPH3CTRL_GRPH3BLEND_LSBMASK		(0x00000003)
+#define PDP_GRPH3CTRL_GRPH3BLEND_SHIFT		(27)
+#define PDP_GRPH3CTRL_GRPH3BLEND_LENGTH		(2)
+#define PDP_GRPH3CTRL_GRPH3BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3BLENDPOS
+*/
+#define PDP_GRPH3CTRL_GRPH3BLENDPOS_MASK		(0x07000000)
+#define PDP_GRPH3CTRL_GRPH3BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_GRPH3CTRL_GRPH3BLENDPOS_SHIFT		(24)
+#define PDP_GRPH3CTRL_GRPH3BLENDPOS_LENGTH		(3)
+#define PDP_GRPH3CTRL_GRPH3BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CTRL, GRPH3DITHEREN
+*/
+#define PDP_GRPH3CTRL_GRPH3DITHEREN_MASK		(0x00800000)
+#define PDP_GRPH3CTRL_GRPH3DITHEREN_LSBMASK		(0x00000001)
+#define PDP_GRPH3CTRL_GRPH3DITHEREN_SHIFT		(23)
+#define PDP_GRPH3CTRL_GRPH3DITHEREN_LENGTH		(1)
+#define PDP_GRPH3CTRL_GRPH3DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4CTRL_OFFSET		(0x002C)
+
+/* PDP, GRPH4CTRL, GRPH4STREN
+*/
+#define PDP_GRPH4CTRL_GRPH4STREN_MASK		(0x80000000)
+#define PDP_GRPH4CTRL_GRPH4STREN_LSBMASK		(0x00000001)
+#define PDP_GRPH4CTRL_GRPH4STREN_SHIFT		(31)
+#define PDP_GRPH4CTRL_GRPH4STREN_LENGTH		(1)
+#define PDP_GRPH4CTRL_GRPH4STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYEN
+*/
+#define PDP_GRPH4CTRL_GRPH4CKEYEN_MASK		(0x40000000)
+#define PDP_GRPH4CTRL_GRPH4CKEYEN_LSBMASK		(0x00000001)
+#define PDP_GRPH4CTRL_GRPH4CKEYEN_SHIFT		(30)
+#define PDP_GRPH4CTRL_GRPH4CKEYEN_LENGTH		(1)
+#define PDP_GRPH4CTRL_GRPH4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4CKEYSRC
+*/
+#define PDP_GRPH4CTRL_GRPH4CKEYSRC_MASK		(0x20000000)
+#define PDP_GRPH4CTRL_GRPH4CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_GRPH4CTRL_GRPH4CKEYSRC_SHIFT		(29)
+#define PDP_GRPH4CTRL_GRPH4CKEYSRC_LENGTH		(1)
+#define PDP_GRPH4CTRL_GRPH4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLEND
+*/
+#define PDP_GRPH4CTRL_GRPH4BLEND_MASK		(0x18000000)
+#define PDP_GRPH4CTRL_GRPH4BLEND_LSBMASK		(0x00000003)
+#define PDP_GRPH4CTRL_GRPH4BLEND_SHIFT		(27)
+#define PDP_GRPH4CTRL_GRPH4BLEND_LENGTH		(2)
+#define PDP_GRPH4CTRL_GRPH4BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4BLENDPOS
+*/
+#define PDP_GRPH4CTRL_GRPH4BLENDPOS_MASK		(0x07000000)
+#define PDP_GRPH4CTRL_GRPH4BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_GRPH4CTRL_GRPH4BLENDPOS_SHIFT		(24)
+#define PDP_GRPH4CTRL_GRPH4BLENDPOS_LENGTH		(3)
+#define PDP_GRPH4CTRL_GRPH4BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CTRL, GRPH4DITHEREN
+*/
+#define PDP_GRPH4CTRL_GRPH4DITHEREN_MASK		(0x00800000)
+#define PDP_GRPH4CTRL_GRPH4DITHEREN_LSBMASK		(0x00000001)
+#define PDP_GRPH4CTRL_GRPH4DITHEREN_SHIFT		(23)
+#define PDP_GRPH4CTRL_GRPH4DITHEREN_LENGTH		(1)
+#define PDP_GRPH4CTRL_GRPH4DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1CTRL_OFFSET		(0x0030)
+
+/* PDP, VID1CTRL, VID1STREN
+*/
+#define PDP_VID1CTRL_VID1STREN_MASK		(0x80000000)
+#define PDP_VID1CTRL_VID1STREN_LSBMASK		(0x00000001)
+#define PDP_VID1CTRL_VID1STREN_SHIFT		(31)
+#define PDP_VID1CTRL_VID1STREN_LENGTH		(1)
+#define PDP_VID1CTRL_VID1STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYEN
+*/
+#define PDP_VID1CTRL_VID1CKEYEN_MASK		(0x40000000)
+#define PDP_VID1CTRL_VID1CKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID1CTRL_VID1CKEYEN_SHIFT		(30)
+#define PDP_VID1CTRL_VID1CKEYEN_LENGTH		(1)
+#define PDP_VID1CTRL_VID1CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1CKEYSRC
+*/
+#define PDP_VID1CTRL_VID1CKEYSRC_MASK		(0x20000000)
+#define PDP_VID1CTRL_VID1CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_VID1CTRL_VID1CKEYSRC_SHIFT		(29)
+#define PDP_VID1CTRL_VID1CKEYSRC_LENGTH		(1)
+#define PDP_VID1CTRL_VID1CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLEND
+*/
+#define PDP_VID1CTRL_VID1BLEND_MASK		(0x18000000)
+#define PDP_VID1CTRL_VID1BLEND_LSBMASK		(0x00000003)
+#define PDP_VID1CTRL_VID1BLEND_SHIFT		(27)
+#define PDP_VID1CTRL_VID1BLEND_LENGTH		(2)
+#define PDP_VID1CTRL_VID1BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1BLENDPOS
+*/
+#define PDP_VID1CTRL_VID1BLENDPOS_MASK		(0x07000000)
+#define PDP_VID1CTRL_VID1BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_VID1CTRL_VID1BLENDPOS_SHIFT		(24)
+#define PDP_VID1CTRL_VID1BLENDPOS_LENGTH		(3)
+#define PDP_VID1CTRL_VID1BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CTRL, VID1DITHEREN
+*/
+#define PDP_VID1CTRL_VID1DITHEREN_MASK		(0x00800000)
+#define PDP_VID1CTRL_VID1DITHEREN_LSBMASK		(0x00000001)
+#define PDP_VID1CTRL_VID1DITHEREN_SHIFT		(23)
+#define PDP_VID1CTRL_VID1DITHEREN_LENGTH		(1)
+#define PDP_VID1CTRL_VID1DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2CTRL_OFFSET		(0x0034)
+
+/* PDP, VID2CTRL, VID2STREN
+*/
+#define PDP_VID2CTRL_VID2STREN_MASK		(0x80000000)
+#define PDP_VID2CTRL_VID2STREN_LSBMASK		(0x00000001)
+#define PDP_VID2CTRL_VID2STREN_SHIFT		(31)
+#define PDP_VID2CTRL_VID2STREN_LENGTH		(1)
+#define PDP_VID2CTRL_VID2STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYEN
+*/
+#define PDP_VID2CTRL_VID2CKEYEN_MASK		(0x40000000)
+#define PDP_VID2CTRL_VID2CKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID2CTRL_VID2CKEYEN_SHIFT		(30)
+#define PDP_VID2CTRL_VID2CKEYEN_LENGTH		(1)
+#define PDP_VID2CTRL_VID2CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2CKEYSRC
+*/
+#define PDP_VID2CTRL_VID2CKEYSRC_MASK		(0x20000000)
+#define PDP_VID2CTRL_VID2CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_VID2CTRL_VID2CKEYSRC_SHIFT		(29)
+#define PDP_VID2CTRL_VID2CKEYSRC_LENGTH		(1)
+#define PDP_VID2CTRL_VID2CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLEND
+*/
+#define PDP_VID2CTRL_VID2BLEND_MASK		(0x18000000)
+#define PDP_VID2CTRL_VID2BLEND_LSBMASK		(0x00000003)
+#define PDP_VID2CTRL_VID2BLEND_SHIFT		(27)
+#define PDP_VID2CTRL_VID2BLEND_LENGTH		(2)
+#define PDP_VID2CTRL_VID2BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2BLENDPOS
+*/
+#define PDP_VID2CTRL_VID2BLENDPOS_MASK		(0x07000000)
+#define PDP_VID2CTRL_VID2BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_VID2CTRL_VID2BLENDPOS_SHIFT		(24)
+#define PDP_VID2CTRL_VID2BLENDPOS_LENGTH		(3)
+#define PDP_VID2CTRL_VID2BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CTRL, VID2DITHEREN
+*/
+#define PDP_VID2CTRL_VID2DITHEREN_MASK		(0x00800000)
+#define PDP_VID2CTRL_VID2DITHEREN_LSBMASK		(0x00000001)
+#define PDP_VID2CTRL_VID2DITHEREN_SHIFT		(23)
+#define PDP_VID2CTRL_VID2DITHEREN_LENGTH		(1)
+#define PDP_VID2CTRL_VID2DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3CTRL_OFFSET		(0x0038)
+
+/* PDP, VID3CTRL, VID3STREN
+*/
+#define PDP_VID3CTRL_VID3STREN_MASK		(0x80000000)
+#define PDP_VID3CTRL_VID3STREN_LSBMASK		(0x00000001)
+#define PDP_VID3CTRL_VID3STREN_SHIFT		(31)
+#define PDP_VID3CTRL_VID3STREN_LENGTH		(1)
+#define PDP_VID3CTRL_VID3STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYEN
+*/
+#define PDP_VID3CTRL_VID3CKEYEN_MASK		(0x40000000)
+#define PDP_VID3CTRL_VID3CKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID3CTRL_VID3CKEYEN_SHIFT		(30)
+#define PDP_VID3CTRL_VID3CKEYEN_LENGTH		(1)
+#define PDP_VID3CTRL_VID3CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3CKEYSRC
+*/
+#define PDP_VID3CTRL_VID3CKEYSRC_MASK		(0x20000000)
+#define PDP_VID3CTRL_VID3CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_VID3CTRL_VID3CKEYSRC_SHIFT		(29)
+#define PDP_VID3CTRL_VID3CKEYSRC_LENGTH		(1)
+#define PDP_VID3CTRL_VID3CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLEND
+*/
+#define PDP_VID3CTRL_VID3BLEND_MASK		(0x18000000)
+#define PDP_VID3CTRL_VID3BLEND_LSBMASK		(0x00000003)
+#define PDP_VID3CTRL_VID3BLEND_SHIFT		(27)
+#define PDP_VID3CTRL_VID3BLEND_LENGTH		(2)
+#define PDP_VID3CTRL_VID3BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3BLENDPOS
+*/
+#define PDP_VID3CTRL_VID3BLENDPOS_MASK		(0x07000000)
+#define PDP_VID3CTRL_VID3BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_VID3CTRL_VID3BLENDPOS_SHIFT		(24)
+#define PDP_VID3CTRL_VID3BLENDPOS_LENGTH		(3)
+#define PDP_VID3CTRL_VID3BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CTRL, VID3DITHEREN
+*/
+#define PDP_VID3CTRL_VID3DITHEREN_MASK		(0x00800000)
+#define PDP_VID3CTRL_VID3DITHEREN_LSBMASK		(0x00000001)
+#define PDP_VID3CTRL_VID3DITHEREN_SHIFT		(23)
+#define PDP_VID3CTRL_VID3DITHEREN_LENGTH		(1)
+#define PDP_VID3CTRL_VID3DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4CTRL_OFFSET		(0x003C)
+
+/* PDP, VID4CTRL, VID4STREN
+*/
+#define PDP_VID4CTRL_VID4STREN_MASK		(0x80000000)
+#define PDP_VID4CTRL_VID4STREN_LSBMASK		(0x00000001)
+#define PDP_VID4CTRL_VID4STREN_SHIFT		(31)
+#define PDP_VID4CTRL_VID4STREN_LENGTH		(1)
+#define PDP_VID4CTRL_VID4STREN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYEN
+*/
+#define PDP_VID4CTRL_VID4CKEYEN_MASK		(0x40000000)
+#define PDP_VID4CTRL_VID4CKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID4CTRL_VID4CKEYEN_SHIFT		(30)
+#define PDP_VID4CTRL_VID4CKEYEN_LENGTH		(1)
+#define PDP_VID4CTRL_VID4CKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4CKEYSRC
+*/
+#define PDP_VID4CTRL_VID4CKEYSRC_MASK		(0x20000000)
+#define PDP_VID4CTRL_VID4CKEYSRC_LSBMASK		(0x00000001)
+#define PDP_VID4CTRL_VID4CKEYSRC_SHIFT		(29)
+#define PDP_VID4CTRL_VID4CKEYSRC_LENGTH		(1)
+#define PDP_VID4CTRL_VID4CKEYSRC_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLEND
+*/
+#define PDP_VID4CTRL_VID4BLEND_MASK		(0x18000000)
+#define PDP_VID4CTRL_VID4BLEND_LSBMASK		(0x00000003)
+#define PDP_VID4CTRL_VID4BLEND_SHIFT		(27)
+#define PDP_VID4CTRL_VID4BLEND_LENGTH		(2)
+#define PDP_VID4CTRL_VID4BLEND_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4BLENDPOS
+*/
+#define PDP_VID4CTRL_VID4BLENDPOS_MASK		(0x07000000)
+#define PDP_VID4CTRL_VID4BLENDPOS_LSBMASK		(0x00000007)
+#define PDP_VID4CTRL_VID4BLENDPOS_SHIFT		(24)
+#define PDP_VID4CTRL_VID4BLENDPOS_LENGTH		(3)
+#define PDP_VID4CTRL_VID4BLENDPOS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CTRL, VID4DITHEREN
+*/
+#define PDP_VID4CTRL_VID4DITHEREN_MASK		(0x00800000)
+#define PDP_VID4CTRL_VID4DITHEREN_LSBMASK		(0x00000001)
+#define PDP_VID4CTRL_VID4DITHEREN_SHIFT		(23)
+#define PDP_VID4CTRL_VID4DITHEREN_LENGTH		(1)
+#define PDP_VID4CTRL_VID4DITHEREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1UCTRL_OFFSET		(0x0050)
+
+/* PDP, VID1UCTRL, VID1UVHALFSTR
+*/
+#define PDP_VID1UCTRL_VID1UVHALFSTR_MASK		(0xC0000000)
+#define PDP_VID1UCTRL_VID1UVHALFSTR_LSBMASK		(0x00000003)
+#define PDP_VID1UCTRL_VID1UVHALFSTR_SHIFT		(30)
+#define PDP_VID1UCTRL_VID1UVHALFSTR_LENGTH		(2)
+#define PDP_VID1UCTRL_VID1UVHALFSTR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2UCTRL_OFFSET		(0x0054)
+
+/* PDP, VID2UCTRL, VID2UVHALFSTR
+*/
+#define PDP_VID2UCTRL_VID2UVHALFSTR_MASK		(0xC0000000)
+#define PDP_VID2UCTRL_VID2UVHALFSTR_LSBMASK		(0x00000003)
+#define PDP_VID2UCTRL_VID2UVHALFSTR_SHIFT		(30)
+#define PDP_VID2UCTRL_VID2UVHALFSTR_LENGTH		(2)
+#define PDP_VID2UCTRL_VID2UVHALFSTR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3UCTRL_OFFSET		(0x0058)
+
+/* PDP, VID3UCTRL, VID3UVHALFSTR
+*/
+#define PDP_VID3UCTRL_VID3UVHALFSTR_MASK		(0xC0000000)
+#define PDP_VID3UCTRL_VID3UVHALFSTR_LSBMASK		(0x00000003)
+#define PDP_VID3UCTRL_VID3UVHALFSTR_SHIFT		(30)
+#define PDP_VID3UCTRL_VID3UVHALFSTR_LENGTH		(2)
+#define PDP_VID3UCTRL_VID3UVHALFSTR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4UCTRL_OFFSET		(0x005C)
+
+/* PDP, VID4UCTRL, VID4UVHALFSTR
+*/
+#define PDP_VID4UCTRL_VID4UVHALFSTR_MASK		(0xC0000000)
+#define PDP_VID4UCTRL_VID4UVHALFSTR_LSBMASK		(0x00000003)
+#define PDP_VID4UCTRL_VID4UVHALFSTR_SHIFT		(30)
+#define PDP_VID4UCTRL_VID4UVHALFSTR_LENGTH		(2)
+#define PDP_VID4UCTRL_VID4UVHALFSTR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1STRIDE_OFFSET		(0x0060)
+
+/* PDP, GRPH1STRIDE, GRPH1STRIDE
+*/
+#define PDP_GRPH1STRIDE_GRPH1STRIDE_MASK		(0xFFC00000)
+#define PDP_GRPH1STRIDE_GRPH1STRIDE_LSBMASK		(0x000003FF)
+#define PDP_GRPH1STRIDE_GRPH1STRIDE_SHIFT		(22)
+#define PDP_GRPH1STRIDE_GRPH1STRIDE_LENGTH		(10)
+#define PDP_GRPH1STRIDE_GRPH1STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2STRIDE_OFFSET		(0x0064)
+
+/* PDP, GRPH2STRIDE, GRPH2STRIDE
+*/
+#define PDP_GRPH2STRIDE_GRPH2STRIDE_MASK		(0xFFC00000)
+#define PDP_GRPH2STRIDE_GRPH2STRIDE_LSBMASK		(0x000003FF)
+#define PDP_GRPH2STRIDE_GRPH2STRIDE_SHIFT		(22)
+#define PDP_GRPH2STRIDE_GRPH2STRIDE_LENGTH		(10)
+#define PDP_GRPH2STRIDE_GRPH2STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3STRIDE_OFFSET		(0x0068)
+
+/* PDP, GRPH3STRIDE, GRPH3STRIDE
+*/
+#define PDP_GRPH3STRIDE_GRPH3STRIDE_MASK		(0xFFC00000)
+#define PDP_GRPH3STRIDE_GRPH3STRIDE_LSBMASK		(0x000003FF)
+#define PDP_GRPH3STRIDE_GRPH3STRIDE_SHIFT		(22)
+#define PDP_GRPH3STRIDE_GRPH3STRIDE_LENGTH		(10)
+#define PDP_GRPH3STRIDE_GRPH3STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4STRIDE_OFFSET		(0x006C)
+
+/* PDP, GRPH4STRIDE, GRPH4STRIDE
+*/
+#define PDP_GRPH4STRIDE_GRPH4STRIDE_MASK		(0xFFC00000)
+#define PDP_GRPH4STRIDE_GRPH4STRIDE_LSBMASK		(0x000003FF)
+#define PDP_GRPH4STRIDE_GRPH4STRIDE_SHIFT		(22)
+#define PDP_GRPH4STRIDE_GRPH4STRIDE_LENGTH		(10)
+#define PDP_GRPH4STRIDE_GRPH4STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1STRIDE_OFFSET		(0x0070)
+
+/* PDP, VID1STRIDE, VID1STRIDE
+*/
+#define PDP_VID1STRIDE_VID1STRIDE_MASK		(0xFFC00000)
+#define PDP_VID1STRIDE_VID1STRIDE_LSBMASK		(0x000003FF)
+#define PDP_VID1STRIDE_VID1STRIDE_SHIFT		(22)
+#define PDP_VID1STRIDE_VID1STRIDE_LENGTH		(10)
+#define PDP_VID1STRIDE_VID1STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2STRIDE_OFFSET		(0x0074)
+
+/* PDP, VID2STRIDE, VID2STRIDE
+*/
+#define PDP_VID2STRIDE_VID2STRIDE_MASK		(0xFFC00000)
+#define PDP_VID2STRIDE_VID2STRIDE_LSBMASK		(0x000003FF)
+#define PDP_VID2STRIDE_VID2STRIDE_SHIFT		(22)
+#define PDP_VID2STRIDE_VID2STRIDE_LENGTH		(10)
+#define PDP_VID2STRIDE_VID2STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3STRIDE_OFFSET		(0x0078)
+
+/* PDP, VID3STRIDE, VID3STRIDE
+*/
+#define PDP_VID3STRIDE_VID3STRIDE_MASK		(0xFFC00000)
+#define PDP_VID3STRIDE_VID3STRIDE_LSBMASK		(0x000003FF)
+#define PDP_VID3STRIDE_VID3STRIDE_SHIFT		(22)
+#define PDP_VID3STRIDE_VID3STRIDE_LENGTH		(10)
+#define PDP_VID3STRIDE_VID3STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4STRIDE_OFFSET		(0x007C)
+
+/* PDP, VID4STRIDE, VID4STRIDE
+*/
+#define PDP_VID4STRIDE_VID4STRIDE_MASK		(0xFFC00000)
+#define PDP_VID4STRIDE_VID4STRIDE_LSBMASK		(0x000003FF)
+#define PDP_VID4STRIDE_VID4STRIDE_SHIFT		(22)
+#define PDP_VID4STRIDE_VID4STRIDE_LENGTH		(10)
+#define PDP_VID4STRIDE_VID4STRIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1SIZE_OFFSET		(0x0080)
+
+/* PDP, GRPH1SIZE, GRPH1WIDTH
+*/
+#define PDP_GRPH1SIZE_GRPH1WIDTH_MASK		(0x0FFF0000)
+#define PDP_GRPH1SIZE_GRPH1WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_GRPH1SIZE_GRPH1WIDTH_SHIFT		(16)
+#define PDP_GRPH1SIZE_GRPH1WIDTH_LENGTH		(12)
+#define PDP_GRPH1SIZE_GRPH1WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1SIZE, GRPH1HEIGHT
+*/
+#define PDP_GRPH1SIZE_GRPH1HEIGHT_MASK		(0x00000FFF)
+#define PDP_GRPH1SIZE_GRPH1HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_GRPH1SIZE_GRPH1HEIGHT_SHIFT		(0)
+#define PDP_GRPH1SIZE_GRPH1HEIGHT_LENGTH		(12)
+#define PDP_GRPH1SIZE_GRPH1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2SIZE_OFFSET		(0x0084)
+
+/* PDP, GRPH2SIZE, GRPH2WIDTH
+*/
+#define PDP_GRPH2SIZE_GRPH2WIDTH_MASK		(0x0FFF0000)
+#define PDP_GRPH2SIZE_GRPH2WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_GRPH2SIZE_GRPH2WIDTH_SHIFT		(16)
+#define PDP_GRPH2SIZE_GRPH2WIDTH_LENGTH		(12)
+#define PDP_GRPH2SIZE_GRPH2WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2SIZE, GRPH2HEIGHT
+*/
+#define PDP_GRPH2SIZE_GRPH2HEIGHT_MASK		(0x00000FFF)
+#define PDP_GRPH2SIZE_GRPH2HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_GRPH2SIZE_GRPH2HEIGHT_SHIFT		(0)
+#define PDP_GRPH2SIZE_GRPH2HEIGHT_LENGTH		(12)
+#define PDP_GRPH2SIZE_GRPH2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3SIZE_OFFSET		(0x0088)
+
+/* PDP, GRPH3SIZE, GRPH3WIDTH
+*/
+#define PDP_GRPH3SIZE_GRPH3WIDTH_MASK		(0x0FFF0000)
+#define PDP_GRPH3SIZE_GRPH3WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_GRPH3SIZE_GRPH3WIDTH_SHIFT		(16)
+#define PDP_GRPH3SIZE_GRPH3WIDTH_LENGTH		(12)
+#define PDP_GRPH3SIZE_GRPH3WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3SIZE, GRPH3HEIGHT
+*/
+#define PDP_GRPH3SIZE_GRPH3HEIGHT_MASK		(0x00000FFF)
+#define PDP_GRPH3SIZE_GRPH3HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_GRPH3SIZE_GRPH3HEIGHT_SHIFT		(0)
+#define PDP_GRPH3SIZE_GRPH3HEIGHT_LENGTH		(12)
+#define PDP_GRPH3SIZE_GRPH3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4SIZE_OFFSET		(0x008C)
+
+/* PDP, GRPH4SIZE, GRPH4WIDTH
+*/
+#define PDP_GRPH4SIZE_GRPH4WIDTH_MASK		(0x0FFF0000)
+#define PDP_GRPH4SIZE_GRPH4WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_GRPH4SIZE_GRPH4WIDTH_SHIFT		(16)
+#define PDP_GRPH4SIZE_GRPH4WIDTH_LENGTH		(12)
+#define PDP_GRPH4SIZE_GRPH4WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4SIZE, GRPH4HEIGHT
+*/
+#define PDP_GRPH4SIZE_GRPH4HEIGHT_MASK		(0x00000FFF)
+#define PDP_GRPH4SIZE_GRPH4HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_GRPH4SIZE_GRPH4HEIGHT_SHIFT		(0)
+#define PDP_GRPH4SIZE_GRPH4HEIGHT_LENGTH		(12)
+#define PDP_GRPH4SIZE_GRPH4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1SIZE_OFFSET		(0x0090)
+
+/* PDP, VID1SIZE, VID1WIDTH
+*/
+#define PDP_VID1SIZE_VID1WIDTH_MASK		(0x0FFF0000)
+#define PDP_VID1SIZE_VID1WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID1SIZE_VID1WIDTH_SHIFT		(16)
+#define PDP_VID1SIZE_VID1WIDTH_LENGTH		(12)
+#define PDP_VID1SIZE_VID1WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SIZE, VID1HEIGHT
+*/
+#define PDP_VID1SIZE_VID1HEIGHT_MASK		(0x00000FFF)
+#define PDP_VID1SIZE_VID1HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID1SIZE_VID1HEIGHT_SHIFT		(0)
+#define PDP_VID1SIZE_VID1HEIGHT_LENGTH		(12)
+#define PDP_VID1SIZE_VID1HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2SIZE_OFFSET		(0x0094)
+
+/* PDP, VID2SIZE, VID2WIDTH
+*/
+#define PDP_VID2SIZE_VID2WIDTH_MASK		(0x0FFF0000)
+#define PDP_VID2SIZE_VID2WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID2SIZE_VID2WIDTH_SHIFT		(16)
+#define PDP_VID2SIZE_VID2WIDTH_LENGTH		(12)
+#define PDP_VID2SIZE_VID2WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SIZE, VID2HEIGHT
+*/
+#define PDP_VID2SIZE_VID2HEIGHT_MASK		(0x00000FFF)
+#define PDP_VID2SIZE_VID2HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID2SIZE_VID2HEIGHT_SHIFT		(0)
+#define PDP_VID2SIZE_VID2HEIGHT_LENGTH		(12)
+#define PDP_VID2SIZE_VID2HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3SIZE_OFFSET		(0x0098)
+
+/* PDP, VID3SIZE, VID3WIDTH
+*/
+#define PDP_VID3SIZE_VID3WIDTH_MASK		(0x0FFF0000)
+#define PDP_VID3SIZE_VID3WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID3SIZE_VID3WIDTH_SHIFT		(16)
+#define PDP_VID3SIZE_VID3WIDTH_LENGTH		(12)
+#define PDP_VID3SIZE_VID3WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SIZE, VID3HEIGHT
+*/
+#define PDP_VID3SIZE_VID3HEIGHT_MASK		(0x00000FFF)
+#define PDP_VID3SIZE_VID3HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID3SIZE_VID3HEIGHT_SHIFT		(0)
+#define PDP_VID3SIZE_VID3HEIGHT_LENGTH		(12)
+#define PDP_VID3SIZE_VID3HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4SIZE_OFFSET		(0x009C)
+
+/* PDP, VID4SIZE, VID4WIDTH
+*/
+#define PDP_VID4SIZE_VID4WIDTH_MASK		(0x0FFF0000)
+#define PDP_VID4SIZE_VID4WIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID4SIZE_VID4WIDTH_SHIFT		(16)
+#define PDP_VID4SIZE_VID4WIDTH_LENGTH		(12)
+#define PDP_VID4SIZE_VID4WIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SIZE, VID4HEIGHT
+*/
+#define PDP_VID4SIZE_VID4HEIGHT_MASK		(0x00000FFF)
+#define PDP_VID4SIZE_VID4HEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID4SIZE_VID4HEIGHT_SHIFT		(0)
+#define PDP_VID4SIZE_VID4HEIGHT_LENGTH		(12)
+#define PDP_VID4SIZE_VID4HEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1POSN_OFFSET		(0x00A0)
+
+/* PDP, GRPH1POSN, GRPH1XSTART
+*/
+#define PDP_GRPH1POSN_GRPH1XSTART_MASK		(0x0FFF0000)
+#define PDP_GRPH1POSN_GRPH1XSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH1POSN_GRPH1XSTART_SHIFT		(16)
+#define PDP_GRPH1POSN_GRPH1XSTART_LENGTH		(12)
+#define PDP_GRPH1POSN_GRPH1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1POSN, GRPH1YSTART
+*/
+#define PDP_GRPH1POSN_GRPH1YSTART_MASK		(0x00000FFF)
+#define PDP_GRPH1POSN_GRPH1YSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH1POSN_GRPH1YSTART_SHIFT		(0)
+#define PDP_GRPH1POSN_GRPH1YSTART_LENGTH		(12)
+#define PDP_GRPH1POSN_GRPH1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2POSN_OFFSET		(0x00A4)
+
+/* PDP, GRPH2POSN, GRPH2XSTART
+*/
+#define PDP_GRPH2POSN_GRPH2XSTART_MASK		(0x0FFF0000)
+#define PDP_GRPH2POSN_GRPH2XSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH2POSN_GRPH2XSTART_SHIFT		(16)
+#define PDP_GRPH2POSN_GRPH2XSTART_LENGTH		(12)
+#define PDP_GRPH2POSN_GRPH2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2POSN, GRPH2YSTART
+*/
+#define PDP_GRPH2POSN_GRPH2YSTART_MASK		(0x00000FFF)
+#define PDP_GRPH2POSN_GRPH2YSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH2POSN_GRPH2YSTART_SHIFT		(0)
+#define PDP_GRPH2POSN_GRPH2YSTART_LENGTH		(12)
+#define PDP_GRPH2POSN_GRPH2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3POSN_OFFSET		(0x00A8)
+
+/* PDP, GRPH3POSN, GRPH3XSTART
+*/
+#define PDP_GRPH3POSN_GRPH3XSTART_MASK		(0x0FFF0000)
+#define PDP_GRPH3POSN_GRPH3XSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH3POSN_GRPH3XSTART_SHIFT		(16)
+#define PDP_GRPH3POSN_GRPH3XSTART_LENGTH		(12)
+#define PDP_GRPH3POSN_GRPH3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3POSN, GRPH3YSTART
+*/
+#define PDP_GRPH3POSN_GRPH3YSTART_MASK		(0x00000FFF)
+#define PDP_GRPH3POSN_GRPH3YSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH3POSN_GRPH3YSTART_SHIFT		(0)
+#define PDP_GRPH3POSN_GRPH3YSTART_LENGTH		(12)
+#define PDP_GRPH3POSN_GRPH3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4POSN_OFFSET		(0x00AC)
+
+/* PDP, GRPH4POSN, GRPH4XSTART
+*/
+#define PDP_GRPH4POSN_GRPH4XSTART_MASK		(0x0FFF0000)
+#define PDP_GRPH4POSN_GRPH4XSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH4POSN_GRPH4XSTART_SHIFT		(16)
+#define PDP_GRPH4POSN_GRPH4XSTART_LENGTH		(12)
+#define PDP_GRPH4POSN_GRPH4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4POSN, GRPH4YSTART
+*/
+#define PDP_GRPH4POSN_GRPH4YSTART_MASK		(0x00000FFF)
+#define PDP_GRPH4POSN_GRPH4YSTART_LSBMASK		(0x00000FFF)
+#define PDP_GRPH4POSN_GRPH4YSTART_SHIFT		(0)
+#define PDP_GRPH4POSN_GRPH4YSTART_LENGTH		(12)
+#define PDP_GRPH4POSN_GRPH4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1POSN_OFFSET		(0x00B0)
+
+/* PDP, VID1POSN, VID1XSTART
+*/
+#define PDP_VID1POSN_VID1XSTART_MASK		(0x0FFF0000)
+#define PDP_VID1POSN_VID1XSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID1POSN_VID1XSTART_SHIFT		(16)
+#define PDP_VID1POSN_VID1XSTART_LENGTH		(12)
+#define PDP_VID1POSN_VID1XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSN, VID1YSTART
+*/
+#define PDP_VID1POSN_VID1YSTART_MASK		(0x00000FFF)
+#define PDP_VID1POSN_VID1YSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID1POSN_VID1YSTART_SHIFT		(0)
+#define PDP_VID1POSN_VID1YSTART_LENGTH		(12)
+#define PDP_VID1POSN_VID1YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2POSN_OFFSET		(0x00B4)
+
+/* PDP, VID2POSN, VID2XSTART
+*/
+#define PDP_VID2POSN_VID2XSTART_MASK		(0x0FFF0000)
+#define PDP_VID2POSN_VID2XSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID2POSN_VID2XSTART_SHIFT		(16)
+#define PDP_VID2POSN_VID2XSTART_LENGTH		(12)
+#define PDP_VID2POSN_VID2XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSN, VID2YSTART
+*/
+#define PDP_VID2POSN_VID2YSTART_MASK		(0x00000FFF)
+#define PDP_VID2POSN_VID2YSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID2POSN_VID2YSTART_SHIFT		(0)
+#define PDP_VID2POSN_VID2YSTART_LENGTH		(12)
+#define PDP_VID2POSN_VID2YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3POSN_OFFSET		(0x00B8)
+
+/* PDP, VID3POSN, VID3XSTART
+*/
+#define PDP_VID3POSN_VID3XSTART_MASK		(0x0FFF0000)
+#define PDP_VID3POSN_VID3XSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID3POSN_VID3XSTART_SHIFT		(16)
+#define PDP_VID3POSN_VID3XSTART_LENGTH		(12)
+#define PDP_VID3POSN_VID3XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSN, VID3YSTART
+*/
+#define PDP_VID3POSN_VID3YSTART_MASK		(0x00000FFF)
+#define PDP_VID3POSN_VID3YSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID3POSN_VID3YSTART_SHIFT		(0)
+#define PDP_VID3POSN_VID3YSTART_LENGTH		(12)
+#define PDP_VID3POSN_VID3YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4POSN_OFFSET		(0x00BC)
+
+/* PDP, VID4POSN, VID4XSTART
+*/
+#define PDP_VID4POSN_VID4XSTART_MASK		(0x0FFF0000)
+#define PDP_VID4POSN_VID4XSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID4POSN_VID4XSTART_SHIFT		(16)
+#define PDP_VID4POSN_VID4XSTART_LENGTH		(12)
+#define PDP_VID4POSN_VID4XSTART_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSN, VID4YSTART
+*/
+#define PDP_VID4POSN_VID4YSTART_MASK		(0x00000FFF)
+#define PDP_VID4POSN_VID4YSTART_LSBMASK		(0x00000FFF)
+#define PDP_VID4POSN_VID4YSTART_SHIFT		(0)
+#define PDP_VID4POSN_VID4YSTART_LENGTH		(12)
+#define PDP_VID4POSN_VID4YSTART_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1GALPHA_OFFSET		(0x00C0)
+
+/* PDP, GRPH1GALPHA, GRPH1GALPHA
+*/
+#define PDP_GRPH1GALPHA_GRPH1GALPHA_MASK		(0x000003FF)
+#define PDP_GRPH1GALPHA_GRPH1GALPHA_LSBMASK		(0x000003FF)
+#define PDP_GRPH1GALPHA_GRPH1GALPHA_SHIFT		(0)
+#define PDP_GRPH1GALPHA_GRPH1GALPHA_LENGTH		(10)
+#define PDP_GRPH1GALPHA_GRPH1GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2GALPHA_OFFSET		(0x00C4)
+
+/* PDP, GRPH2GALPHA, GRPH2GALPHA
+*/
+#define PDP_GRPH2GALPHA_GRPH2GALPHA_MASK		(0x000003FF)
+#define PDP_GRPH2GALPHA_GRPH2GALPHA_LSBMASK		(0x000003FF)
+#define PDP_GRPH2GALPHA_GRPH2GALPHA_SHIFT		(0)
+#define PDP_GRPH2GALPHA_GRPH2GALPHA_LENGTH		(10)
+#define PDP_GRPH2GALPHA_GRPH2GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3GALPHA_OFFSET		(0x00C8)
+
+/* PDP, GRPH3GALPHA, GRPH3GALPHA
+*/
+#define PDP_GRPH3GALPHA_GRPH3GALPHA_MASK		(0x000003FF)
+#define PDP_GRPH3GALPHA_GRPH3GALPHA_LSBMASK		(0x000003FF)
+#define PDP_GRPH3GALPHA_GRPH3GALPHA_SHIFT		(0)
+#define PDP_GRPH3GALPHA_GRPH3GALPHA_LENGTH		(10)
+#define PDP_GRPH3GALPHA_GRPH3GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4GALPHA_OFFSET		(0x00CC)
+
+/* PDP, GRPH4GALPHA, GRPH4GALPHA
+*/
+#define PDP_GRPH4GALPHA_GRPH4GALPHA_MASK		(0x000003FF)
+#define PDP_GRPH4GALPHA_GRPH4GALPHA_LSBMASK		(0x000003FF)
+#define PDP_GRPH4GALPHA_GRPH4GALPHA_SHIFT		(0)
+#define PDP_GRPH4GALPHA_GRPH4GALPHA_LENGTH		(10)
+#define PDP_GRPH4GALPHA_GRPH4GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1GALPHA_OFFSET		(0x00D0)
+
+/* PDP, VID1GALPHA, VID1GALPHA
+*/
+#define PDP_VID1GALPHA_VID1GALPHA_MASK		(0x000003FF)
+#define PDP_VID1GALPHA_VID1GALPHA_LSBMASK		(0x000003FF)
+#define PDP_VID1GALPHA_VID1GALPHA_SHIFT		(0)
+#define PDP_VID1GALPHA_VID1GALPHA_LENGTH		(10)
+#define PDP_VID1GALPHA_VID1GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2GALPHA_OFFSET		(0x00D4)
+
+/* PDP, VID2GALPHA, VID2GALPHA
+*/
+#define PDP_VID2GALPHA_VID2GALPHA_MASK		(0x000003FF)
+#define PDP_VID2GALPHA_VID2GALPHA_LSBMASK		(0x000003FF)
+#define PDP_VID2GALPHA_VID2GALPHA_SHIFT		(0)
+#define PDP_VID2GALPHA_VID2GALPHA_LENGTH		(10)
+#define PDP_VID2GALPHA_VID2GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3GALPHA_OFFSET		(0x00D8)
+
+/* PDP, VID3GALPHA, VID3GALPHA
+*/
+#define PDP_VID3GALPHA_VID3GALPHA_MASK		(0x000003FF)
+#define PDP_VID3GALPHA_VID3GALPHA_LSBMASK		(0x000003FF)
+#define PDP_VID3GALPHA_VID3GALPHA_SHIFT		(0)
+#define PDP_VID3GALPHA_VID3GALPHA_LENGTH		(10)
+#define PDP_VID3GALPHA_VID3GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4GALPHA_OFFSET		(0x00DC)
+
+/* PDP, VID4GALPHA, VID4GALPHA
+*/
+#define PDP_VID4GALPHA_VID4GALPHA_MASK		(0x000003FF)
+#define PDP_VID4GALPHA_VID4GALPHA_LSBMASK		(0x000003FF)
+#define PDP_VID4GALPHA_VID4GALPHA_SHIFT		(0)
+#define PDP_VID4GALPHA_VID4GALPHA_LENGTH		(10)
+#define PDP_VID4GALPHA_VID4GALPHA_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1CKEY_R_OFFSET		(0x00E0)
+
+/* PDP, GRPH1CKEY_R, GRPH1CKEY_R
+*/
+#define PDP_GRPH1CKEY_R_GRPH1CKEY_R_MASK		(0x000003FF)
+#define PDP_GRPH1CKEY_R_GRPH1CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH1CKEY_R_GRPH1CKEY_R_SHIFT		(0)
+#define PDP_GRPH1CKEY_R_GRPH1CKEY_R_LENGTH		(10)
+#define PDP_GRPH1CKEY_R_GRPH1CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1CKEY_GB_OFFSET		(0x00E4)
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_G
+*/
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_G_MASK		(0x03FF0000)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SHIFT		(16)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_G_LENGTH		(10)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1CKEY_GB, GRPH1CKEY_B
+*/
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_B_MASK		(0x000003FF)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SHIFT		(0)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_B_LENGTH		(10)
+#define PDP_GRPH1CKEY_GB_GRPH1CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2CKEY_R_OFFSET		(0x00E8)
+
+/* PDP, GRPH2CKEY_R, GRPH2CKEY_R
+*/
+#define PDP_GRPH2CKEY_R_GRPH2CKEY_R_MASK		(0x000003FF)
+#define PDP_GRPH2CKEY_R_GRPH2CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH2CKEY_R_GRPH2CKEY_R_SHIFT		(0)
+#define PDP_GRPH2CKEY_R_GRPH2CKEY_R_LENGTH		(10)
+#define PDP_GRPH2CKEY_R_GRPH2CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2CKEY_GB_OFFSET		(0x00EC)
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_G
+*/
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_G_MASK		(0x03FF0000)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SHIFT		(16)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_G_LENGTH		(10)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2CKEY_GB, GRPH2CKEY_B
+*/
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_B_MASK		(0x000003FF)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SHIFT		(0)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_B_LENGTH		(10)
+#define PDP_GRPH2CKEY_GB_GRPH2CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3CKEY_R_OFFSET		(0x00F0)
+
+/* PDP, GRPH3CKEY_R, GRPH3CKEY_R
+*/
+#define PDP_GRPH3CKEY_R_GRPH3CKEY_R_MASK		(0x000003FF)
+#define PDP_GRPH3CKEY_R_GRPH3CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH3CKEY_R_GRPH3CKEY_R_SHIFT		(0)
+#define PDP_GRPH3CKEY_R_GRPH3CKEY_R_LENGTH		(10)
+#define PDP_GRPH3CKEY_R_GRPH3CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3CKEY_GB_OFFSET		(0x00F4)
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_G
+*/
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_G_MASK		(0x03FF0000)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SHIFT		(16)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_G_LENGTH		(10)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3CKEY_GB, GRPH3CKEY_B
+*/
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_B_MASK		(0x000003FF)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SHIFT		(0)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_B_LENGTH		(10)
+#define PDP_GRPH3CKEY_GB_GRPH3CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4CKEY_R_OFFSET		(0x00F8)
+
+/* PDP, GRPH4CKEY_R, GRPH4CKEY_R
+*/
+#define PDP_GRPH4CKEY_R_GRPH4CKEY_R_MASK		(0x000003FF)
+#define PDP_GRPH4CKEY_R_GRPH4CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH4CKEY_R_GRPH4CKEY_R_SHIFT		(0)
+#define PDP_GRPH4CKEY_R_GRPH4CKEY_R_LENGTH		(10)
+#define PDP_GRPH4CKEY_R_GRPH4CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4CKEY_GB_OFFSET		(0x00FC)
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_G
+*/
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_G_MASK		(0x03FF0000)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SHIFT		(16)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_G_LENGTH		(10)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4CKEY_GB, GRPH4CKEY_B
+*/
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_B_MASK		(0x000003FF)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SHIFT		(0)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_B_LENGTH		(10)
+#define PDP_GRPH4CKEY_GB_GRPH4CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1CKEY_R_OFFSET		(0x0100)
+
+/* PDP, VID1CKEY_R, VID1CKEY_R
+*/
+#define PDP_VID1CKEY_R_VID1CKEY_R_MASK		(0x000003FF)
+#define PDP_VID1CKEY_R_VID1CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_VID1CKEY_R_VID1CKEY_R_SHIFT		(0)
+#define PDP_VID1CKEY_R_VID1CKEY_R_LENGTH		(10)
+#define PDP_VID1CKEY_R_VID1CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1CKEY_GB_OFFSET		(0x0104)
+
+/* PDP, VID1CKEY_GB, VID1CKEY_G
+*/
+#define PDP_VID1CKEY_GB_VID1CKEY_G_MASK		(0x03FF0000)
+#define PDP_VID1CKEY_GB_VID1CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_VID1CKEY_GB_VID1CKEY_G_SHIFT		(16)
+#define PDP_VID1CKEY_GB_VID1CKEY_G_LENGTH		(10)
+#define PDP_VID1CKEY_GB_VID1CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1CKEY_GB, VID1CKEY_B
+*/
+#define PDP_VID1CKEY_GB_VID1CKEY_B_MASK		(0x000003FF)
+#define PDP_VID1CKEY_GB_VID1CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_VID1CKEY_GB_VID1CKEY_B_SHIFT		(0)
+#define PDP_VID1CKEY_GB_VID1CKEY_B_LENGTH		(10)
+#define PDP_VID1CKEY_GB_VID1CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2CKEY_R_OFFSET		(0x0108)
+
+/* PDP, VID2CKEY_R, VID2CKEY_R
+*/
+#define PDP_VID2CKEY_R_VID2CKEY_R_MASK		(0x000003FF)
+#define PDP_VID2CKEY_R_VID2CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_VID2CKEY_R_VID2CKEY_R_SHIFT		(0)
+#define PDP_VID2CKEY_R_VID2CKEY_R_LENGTH		(10)
+#define PDP_VID2CKEY_R_VID2CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2CKEY_GB_OFFSET		(0x010C)
+
+/* PDP, VID2CKEY_GB, VID2CKEY_G
+*/
+#define PDP_VID2CKEY_GB_VID2CKEY_G_MASK		(0x03FF0000)
+#define PDP_VID2CKEY_GB_VID2CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_VID2CKEY_GB_VID2CKEY_G_SHIFT		(16)
+#define PDP_VID2CKEY_GB_VID2CKEY_G_LENGTH		(10)
+#define PDP_VID2CKEY_GB_VID2CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2CKEY_GB, VID2CKEY_B
+*/
+#define PDP_VID2CKEY_GB_VID2CKEY_B_MASK		(0x000003FF)
+#define PDP_VID2CKEY_GB_VID2CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_VID2CKEY_GB_VID2CKEY_B_SHIFT		(0)
+#define PDP_VID2CKEY_GB_VID2CKEY_B_LENGTH		(10)
+#define PDP_VID2CKEY_GB_VID2CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3CKEY_R_OFFSET		(0x0110)
+
+/* PDP, VID3CKEY_R, VID3CKEY_R
+*/
+#define PDP_VID3CKEY_R_VID3CKEY_R_MASK		(0x000003FF)
+#define PDP_VID3CKEY_R_VID3CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_VID3CKEY_R_VID3CKEY_R_SHIFT		(0)
+#define PDP_VID3CKEY_R_VID3CKEY_R_LENGTH		(10)
+#define PDP_VID3CKEY_R_VID3CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3CKEY_GB_OFFSET		(0x0114)
+
+/* PDP, VID3CKEY_GB, VID3CKEY_G
+*/
+#define PDP_VID3CKEY_GB_VID3CKEY_G_MASK		(0x03FF0000)
+#define PDP_VID3CKEY_GB_VID3CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_VID3CKEY_GB_VID3CKEY_G_SHIFT		(16)
+#define PDP_VID3CKEY_GB_VID3CKEY_G_LENGTH		(10)
+#define PDP_VID3CKEY_GB_VID3CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3CKEY_GB, VID3CKEY_B
+*/
+#define PDP_VID3CKEY_GB_VID3CKEY_B_MASK		(0x000003FF)
+#define PDP_VID3CKEY_GB_VID3CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_VID3CKEY_GB_VID3CKEY_B_SHIFT		(0)
+#define PDP_VID3CKEY_GB_VID3CKEY_B_LENGTH		(10)
+#define PDP_VID3CKEY_GB_VID3CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4CKEY_R_OFFSET		(0x0118)
+
+/* PDP, VID4CKEY_R, VID4CKEY_R
+*/
+#define PDP_VID4CKEY_R_VID4CKEY_R_MASK		(0x000003FF)
+#define PDP_VID4CKEY_R_VID4CKEY_R_LSBMASK		(0x000003FF)
+#define PDP_VID4CKEY_R_VID4CKEY_R_SHIFT		(0)
+#define PDP_VID4CKEY_R_VID4CKEY_R_LENGTH		(10)
+#define PDP_VID4CKEY_R_VID4CKEY_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4CKEY_GB_OFFSET		(0x011C)
+
+/* PDP, VID4CKEY_GB, VID4CKEY_G
+*/
+#define PDP_VID4CKEY_GB_VID4CKEY_G_MASK		(0x03FF0000)
+#define PDP_VID4CKEY_GB_VID4CKEY_G_LSBMASK		(0x000003FF)
+#define PDP_VID4CKEY_GB_VID4CKEY_G_SHIFT		(16)
+#define PDP_VID4CKEY_GB_VID4CKEY_G_LENGTH		(10)
+#define PDP_VID4CKEY_GB_VID4CKEY_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4CKEY_GB, VID4CKEY_B
+*/
+#define PDP_VID4CKEY_GB_VID4CKEY_B_MASK		(0x000003FF)
+#define PDP_VID4CKEY_GB_VID4CKEY_B_LSBMASK		(0x000003FF)
+#define PDP_VID4CKEY_GB_VID4CKEY_B_SHIFT		(0)
+#define PDP_VID4CKEY_GB_VID4CKEY_B_LENGTH		(10)
+#define PDP_VID4CKEY_GB_VID4CKEY_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1BLND2_R_OFFSET		(0x0120)
+
+/* PDP, GRPH1BLND2_R, GRPH1PIXDBL
+*/
+#define PDP_GRPH1BLND2_R_GRPH1PIXDBL_MASK		(0x80000000)
+#define PDP_GRPH1BLND2_R_GRPH1PIXDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH1BLND2_R_GRPH1PIXDBL_SHIFT		(31)
+#define PDP_GRPH1BLND2_R_GRPH1PIXDBL_LENGTH		(1)
+#define PDP_GRPH1BLND2_R_GRPH1PIXDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1LINDBL
+*/
+#define PDP_GRPH1BLND2_R_GRPH1LINDBL_MASK		(0x20000000)
+#define PDP_GRPH1BLND2_R_GRPH1LINDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH1BLND2_R_GRPH1LINDBL_SHIFT		(29)
+#define PDP_GRPH1BLND2_R_GRPH1LINDBL_LENGTH		(1)
+#define PDP_GRPH1BLND2_R_GRPH1LINDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1BLND2_R, GRPH1CKEYMASK_R
+*/
+#define PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SHIFT		(0)
+#define PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_LENGTH		(10)
+#define PDP_GRPH1BLND2_R_GRPH1CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1BLND2_GB_OFFSET		(0x0124)
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_G
+*/
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SHIFT		(16)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_LENGTH		(10)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1BLND2_GB, GRPH1CKEYMASK_B
+*/
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SHIFT		(0)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_LENGTH		(10)
+#define PDP_GRPH1BLND2_GB_GRPH1CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2BLND2_R_OFFSET		(0x0128)
+
+/* PDP, GRPH2BLND2_R, GRPH2PIXDBL
+*/
+#define PDP_GRPH2BLND2_R_GRPH2PIXDBL_MASK		(0x80000000)
+#define PDP_GRPH2BLND2_R_GRPH2PIXDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH2BLND2_R_GRPH2PIXDBL_SHIFT		(31)
+#define PDP_GRPH2BLND2_R_GRPH2PIXDBL_LENGTH		(1)
+#define PDP_GRPH2BLND2_R_GRPH2PIXDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2LINDBL
+*/
+#define PDP_GRPH2BLND2_R_GRPH2LINDBL_MASK		(0x20000000)
+#define PDP_GRPH2BLND2_R_GRPH2LINDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH2BLND2_R_GRPH2LINDBL_SHIFT		(29)
+#define PDP_GRPH2BLND2_R_GRPH2LINDBL_LENGTH		(1)
+#define PDP_GRPH2BLND2_R_GRPH2LINDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2BLND2_R, GRPH2CKEYMASK_R
+*/
+#define PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SHIFT		(0)
+#define PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_LENGTH		(10)
+#define PDP_GRPH2BLND2_R_GRPH2CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2BLND2_GB_OFFSET		(0x012C)
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_G
+*/
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SHIFT		(16)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_LENGTH		(10)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2BLND2_GB, GRPH2CKEYMASK_B
+*/
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SHIFT		(0)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_LENGTH		(10)
+#define PDP_GRPH2BLND2_GB_GRPH2CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3BLND2_R_OFFSET		(0x0130)
+
+/* PDP, GRPH3BLND2_R, GRPH3PIXDBL
+*/
+#define PDP_GRPH3BLND2_R_GRPH3PIXDBL_MASK		(0x80000000)
+#define PDP_GRPH3BLND2_R_GRPH3PIXDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH3BLND2_R_GRPH3PIXDBL_SHIFT		(31)
+#define PDP_GRPH3BLND2_R_GRPH3PIXDBL_LENGTH		(1)
+#define PDP_GRPH3BLND2_R_GRPH3PIXDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3LINDBL
+*/
+#define PDP_GRPH3BLND2_R_GRPH3LINDBL_MASK		(0x20000000)
+#define PDP_GRPH3BLND2_R_GRPH3LINDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH3BLND2_R_GRPH3LINDBL_SHIFT		(29)
+#define PDP_GRPH3BLND2_R_GRPH3LINDBL_LENGTH		(1)
+#define PDP_GRPH3BLND2_R_GRPH3LINDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3BLND2_R, GRPH3CKEYMASK_R
+*/
+#define PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SHIFT		(0)
+#define PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_LENGTH		(10)
+#define PDP_GRPH3BLND2_R_GRPH3CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3BLND2_GB_OFFSET		(0x0134)
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_G
+*/
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SHIFT		(16)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_LENGTH		(10)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3BLND2_GB, GRPH3CKEYMASK_B
+*/
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SHIFT		(0)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_LENGTH		(10)
+#define PDP_GRPH3BLND2_GB_GRPH3CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4BLND2_R_OFFSET		(0x0138)
+
+/* PDP, GRPH4BLND2_R, GRPH4PIXDBL
+*/
+#define PDP_GRPH4BLND2_R_GRPH4PIXDBL_MASK		(0x80000000)
+#define PDP_GRPH4BLND2_R_GRPH4PIXDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH4BLND2_R_GRPH4PIXDBL_SHIFT		(31)
+#define PDP_GRPH4BLND2_R_GRPH4PIXDBL_LENGTH		(1)
+#define PDP_GRPH4BLND2_R_GRPH4PIXDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4LINDBL
+*/
+#define PDP_GRPH4BLND2_R_GRPH4LINDBL_MASK		(0x20000000)
+#define PDP_GRPH4BLND2_R_GRPH4LINDBL_LSBMASK		(0x00000001)
+#define PDP_GRPH4BLND2_R_GRPH4LINDBL_SHIFT		(29)
+#define PDP_GRPH4BLND2_R_GRPH4LINDBL_LENGTH		(1)
+#define PDP_GRPH4BLND2_R_GRPH4LINDBL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4BLND2_R, GRPH4CKEYMASK_R
+*/
+#define PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SHIFT		(0)
+#define PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_LENGTH		(10)
+#define PDP_GRPH4BLND2_R_GRPH4CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4BLND2_GB_OFFSET		(0x013C)
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_G
+*/
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SHIFT		(16)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_LENGTH		(10)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4BLND2_GB, GRPH4CKEYMASK_B
+*/
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SHIFT		(0)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_LENGTH		(10)
+#define PDP_GRPH4BLND2_GB_GRPH4CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1BLND2_R_OFFSET		(0x0140)
+
+/* PDP, VID1BLND2_R, VID1CKEYMASK_R
+*/
+#define PDP_VID1BLND2_R_VID1CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_VID1BLND2_R_VID1CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_VID1BLND2_R_VID1CKEYMASK_R_SHIFT		(0)
+#define PDP_VID1BLND2_R_VID1CKEYMASK_R_LENGTH		(10)
+#define PDP_VID1BLND2_R_VID1CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1BLND2_GB_OFFSET		(0x0144)
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_G
+*/
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_G_SHIFT		(16)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_G_LENGTH		(10)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1BLND2_GB, VID1CKEYMASK_B
+*/
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_B_SHIFT		(0)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_B_LENGTH		(10)
+#define PDP_VID1BLND2_GB_VID1CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2BLND2_R_OFFSET		(0x0148)
+
+/* PDP, VID2BLND2_R, VID2CKEYMASK_R
+*/
+#define PDP_VID2BLND2_R_VID2CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_VID2BLND2_R_VID2CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_VID2BLND2_R_VID2CKEYMASK_R_SHIFT		(0)
+#define PDP_VID2BLND2_R_VID2CKEYMASK_R_LENGTH		(10)
+#define PDP_VID2BLND2_R_VID2CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2BLND2_GB_OFFSET		(0x014C)
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_G
+*/
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_G_SHIFT		(16)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_G_LENGTH		(10)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2BLND2_GB, VID2CKEYMASK_B
+*/
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_B_SHIFT		(0)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_B_LENGTH		(10)
+#define PDP_VID2BLND2_GB_VID2CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3BLND2_R_OFFSET		(0x0150)
+
+/* PDP, VID3BLND2_R, VID3CKEYMASK_R
+*/
+#define PDP_VID3BLND2_R_VID3CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_VID3BLND2_R_VID3CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_VID3BLND2_R_VID3CKEYMASK_R_SHIFT		(0)
+#define PDP_VID3BLND2_R_VID3CKEYMASK_R_LENGTH		(10)
+#define PDP_VID3BLND2_R_VID3CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3BLND2_GB_OFFSET		(0x0154)
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_G
+*/
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_G_SHIFT		(16)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_G_LENGTH		(10)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3BLND2_GB, VID3CKEYMASK_B
+*/
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_B_SHIFT		(0)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_B_LENGTH		(10)
+#define PDP_VID3BLND2_GB_VID3CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4BLND2_R_OFFSET		(0x0158)
+
+/* PDP, VID4BLND2_R, VID4CKEYMASK_R
+*/
+#define PDP_VID4BLND2_R_VID4CKEYMASK_R_MASK		(0x000003FF)
+#define PDP_VID4BLND2_R_VID4CKEYMASK_R_LSBMASK		(0x000003FF)
+#define PDP_VID4BLND2_R_VID4CKEYMASK_R_SHIFT		(0)
+#define PDP_VID4BLND2_R_VID4CKEYMASK_R_LENGTH		(10)
+#define PDP_VID4BLND2_R_VID4CKEYMASK_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4BLND2_GB_OFFSET		(0x015C)
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_G
+*/
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_G_MASK		(0x03FF0000)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_G_LSBMASK		(0x000003FF)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_G_SHIFT		(16)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_G_LENGTH		(10)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4BLND2_GB, VID4CKEYMASK_B
+*/
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_B_MASK		(0x000003FF)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_B_LSBMASK		(0x000003FF)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_B_SHIFT		(0)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_B_LENGTH		(10)
+#define PDP_VID4BLND2_GB_VID4CKEYMASK_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1INTERLEAVE_CTRL_OFFSET		(0x0160)
+
+/* PDP, GRPH1INTERLEAVE_CTRL, GRPH1INTFIELD
+*/
+#define PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_MASK		(0x00000001)
+#define PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LSBMASK		(0x00000001)
+#define PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SHIFT		(0)
+#define PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_LENGTH		(1)
+#define PDP_GRPH1INTERLEAVE_CTRL_GRPH1INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2INTERLEAVE_CTRL_OFFSET		(0x0164)
+
+/* PDP, GRPH2INTERLEAVE_CTRL, GRPH2INTFIELD
+*/
+#define PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_MASK		(0x00000001)
+#define PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LSBMASK		(0x00000001)
+#define PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SHIFT		(0)
+#define PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_LENGTH		(1)
+#define PDP_GRPH2INTERLEAVE_CTRL_GRPH2INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3INTERLEAVE_CTRL_OFFSET		(0x0168)
+
+/* PDP, GRPH3INTERLEAVE_CTRL, GRPH3INTFIELD
+*/
+#define PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_MASK		(0x00000001)
+#define PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LSBMASK		(0x00000001)
+#define PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SHIFT		(0)
+#define PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_LENGTH		(1)
+#define PDP_GRPH3INTERLEAVE_CTRL_GRPH3INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4INTERLEAVE_CTRL_OFFSET		(0x016C)
+
+/* PDP, GRPH4INTERLEAVE_CTRL, GRPH4INTFIELD
+*/
+#define PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_MASK		(0x00000001)
+#define PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LSBMASK		(0x00000001)
+#define PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SHIFT		(0)
+#define PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_LENGTH		(1)
+#define PDP_GRPH4INTERLEAVE_CTRL_GRPH4INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1INTERLEAVE_CTRL_OFFSET		(0x0170)
+
+/* PDP, VID1INTERLEAVE_CTRL, VID1INTFIELD
+*/
+#define PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_MASK		(0x00000001)
+#define PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LSBMASK		(0x00000001)
+#define PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SHIFT		(0)
+#define PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_LENGTH		(1)
+#define PDP_VID1INTERLEAVE_CTRL_VID1INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2INTERLEAVE_CTRL_OFFSET		(0x0174)
+
+/* PDP, VID2INTERLEAVE_CTRL, VID2INTFIELD
+*/
+#define PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_MASK		(0x00000001)
+#define PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LSBMASK		(0x00000001)
+#define PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SHIFT		(0)
+#define PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_LENGTH		(1)
+#define PDP_VID2INTERLEAVE_CTRL_VID2INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3INTERLEAVE_CTRL_OFFSET		(0x0178)
+
+/* PDP, VID3INTERLEAVE_CTRL, VID3INTFIELD
+*/
+#define PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_MASK		(0x00000001)
+#define PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LSBMASK		(0x00000001)
+#define PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SHIFT		(0)
+#define PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_LENGTH		(1)
+#define PDP_VID3INTERLEAVE_CTRL_VID3INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4INTERLEAVE_CTRL_OFFSET		(0x017C)
+
+/* PDP, VID4INTERLEAVE_CTRL, VID4INTFIELD
+*/
+#define PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_MASK		(0x00000001)
+#define PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LSBMASK		(0x00000001)
+#define PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SHIFT		(0)
+#define PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_LENGTH		(1)
+#define PDP_VID4INTERLEAVE_CTRL_VID4INTFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1BASEADDR_OFFSET		(0x0180)
+
+/* PDP, GRPH1BASEADDR, GRPH1BASEADDR
+*/
+#define PDP_GRPH1BASEADDR_GRPH1BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_GRPH1BASEADDR_GRPH1BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_GRPH1BASEADDR_GRPH1BASEADDR_SHIFT		(5)
+#define PDP_GRPH1BASEADDR_GRPH1BASEADDR_LENGTH		(27)
+#define PDP_GRPH1BASEADDR_GRPH1BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2BASEADDR_OFFSET		(0x0184)
+
+/* PDP, GRPH2BASEADDR, GRPH2BASEADDR
+*/
+#define PDP_GRPH2BASEADDR_GRPH2BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_GRPH2BASEADDR_GRPH2BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_GRPH2BASEADDR_GRPH2BASEADDR_SHIFT		(5)
+#define PDP_GRPH2BASEADDR_GRPH2BASEADDR_LENGTH		(27)
+#define PDP_GRPH2BASEADDR_GRPH2BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3BASEADDR_OFFSET		(0x0188)
+
+/* PDP, GRPH3BASEADDR, GRPH3BASEADDR
+*/
+#define PDP_GRPH3BASEADDR_GRPH3BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_GRPH3BASEADDR_GRPH3BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_GRPH3BASEADDR_GRPH3BASEADDR_SHIFT		(5)
+#define PDP_GRPH3BASEADDR_GRPH3BASEADDR_LENGTH		(27)
+#define PDP_GRPH3BASEADDR_GRPH3BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4BASEADDR_OFFSET		(0x018C)
+
+/* PDP, GRPH4BASEADDR, GRPH4BASEADDR
+*/
+#define PDP_GRPH4BASEADDR_GRPH4BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_GRPH4BASEADDR_GRPH4BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_GRPH4BASEADDR_GRPH4BASEADDR_SHIFT		(5)
+#define PDP_GRPH4BASEADDR_GRPH4BASEADDR_LENGTH		(27)
+#define PDP_GRPH4BASEADDR_GRPH4BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1BASEADDR_OFFSET		(0x0190)
+
+/* PDP, VID1BASEADDR, VID1BASEADDR
+*/
+#define PDP_VID1BASEADDR_VID1BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID1BASEADDR_VID1BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID1BASEADDR_VID1BASEADDR_SHIFT		(5)
+#define PDP_VID1BASEADDR_VID1BASEADDR_LENGTH		(27)
+#define PDP_VID1BASEADDR_VID1BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2BASEADDR_OFFSET		(0x0194)
+
+/* PDP, VID2BASEADDR, VID2BASEADDR
+*/
+#define PDP_VID2BASEADDR_VID2BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID2BASEADDR_VID2BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID2BASEADDR_VID2BASEADDR_SHIFT		(5)
+#define PDP_VID2BASEADDR_VID2BASEADDR_LENGTH		(27)
+#define PDP_VID2BASEADDR_VID2BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3BASEADDR_OFFSET		(0x0198)
+
+/* PDP, VID3BASEADDR, VID3BASEADDR
+*/
+#define PDP_VID3BASEADDR_VID3BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID3BASEADDR_VID3BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID3BASEADDR_VID3BASEADDR_SHIFT		(5)
+#define PDP_VID3BASEADDR_VID3BASEADDR_LENGTH		(27)
+#define PDP_VID3BASEADDR_VID3BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4BASEADDR_OFFSET		(0x019C)
+
+/* PDP, VID4BASEADDR, VID4BASEADDR
+*/
+#define PDP_VID4BASEADDR_VID4BASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID4BASEADDR_VID4BASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID4BASEADDR_VID4BASEADDR_SHIFT		(5)
+#define PDP_VID4BASEADDR_VID4BASEADDR_LENGTH		(27)
+#define PDP_VID4BASEADDR_VID4BASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1UBASEADDR_OFFSET		(0x01B0)
+
+/* PDP, VID1UBASEADDR, VID1UBASEADDR
+*/
+#define PDP_VID1UBASEADDR_VID1UBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID1UBASEADDR_VID1UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID1UBASEADDR_VID1UBASEADDR_SHIFT		(5)
+#define PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH		(27)
+#define PDP_VID1UBASEADDR_VID1UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2UBASEADDR_OFFSET		(0x01B4)
+
+/* PDP, VID2UBASEADDR, VID2UBASEADDR
+*/
+#define PDP_VID2UBASEADDR_VID2UBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID2UBASEADDR_VID2UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID2UBASEADDR_VID2UBASEADDR_SHIFT		(5)
+#define PDP_VID2UBASEADDR_VID2UBASEADDR_LENGTH		(27)
+#define PDP_VID2UBASEADDR_VID2UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3UBASEADDR_OFFSET		(0x01B8)
+
+/* PDP, VID3UBASEADDR, VID3UBASEADDR
+*/
+#define PDP_VID3UBASEADDR_VID3UBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID3UBASEADDR_VID3UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID3UBASEADDR_VID3UBASEADDR_SHIFT		(5)
+#define PDP_VID3UBASEADDR_VID3UBASEADDR_LENGTH		(27)
+#define PDP_VID3UBASEADDR_VID3UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4UBASEADDR_OFFSET		(0x01BC)
+
+/* PDP, VID4UBASEADDR, VID4UBASEADDR
+*/
+#define PDP_VID4UBASEADDR_VID4UBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID4UBASEADDR_VID4UBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID4UBASEADDR_VID4UBASEADDR_SHIFT		(5)
+#define PDP_VID4UBASEADDR_VID4UBASEADDR_LENGTH		(27)
+#define PDP_VID4UBASEADDR_VID4UBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VBASEADDR_OFFSET		(0x01D0)
+
+/* PDP, VID1VBASEADDR, VID1VBASEADDR
+*/
+#define PDP_VID1VBASEADDR_VID1VBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID1VBASEADDR_VID1VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID1VBASEADDR_VID1VBASEADDR_SHIFT		(5)
+#define PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH		(27)
+#define PDP_VID1VBASEADDR_VID1VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VBASEADDR_OFFSET		(0x01D4)
+
+/* PDP, VID2VBASEADDR, VID2VBASEADDR
+*/
+#define PDP_VID2VBASEADDR_VID2VBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID2VBASEADDR_VID2VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID2VBASEADDR_VID2VBASEADDR_SHIFT		(5)
+#define PDP_VID2VBASEADDR_VID2VBASEADDR_LENGTH		(27)
+#define PDP_VID2VBASEADDR_VID2VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VBASEADDR_OFFSET		(0x01D8)
+
+/* PDP, VID3VBASEADDR, VID3VBASEADDR
+*/
+#define PDP_VID3VBASEADDR_VID3VBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID3VBASEADDR_VID3VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID3VBASEADDR_VID3VBASEADDR_SHIFT		(5)
+#define PDP_VID3VBASEADDR_VID3VBASEADDR_LENGTH		(27)
+#define PDP_VID3VBASEADDR_VID3VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VBASEADDR_OFFSET		(0x01DC)
+
+/* PDP, VID4VBASEADDR, VID4VBASEADDR
+*/
+#define PDP_VID4VBASEADDR_VID4VBASEADDR_MASK		(0xFFFFFFE0)
+#define PDP_VID4VBASEADDR_VID4VBASEADDR_LSBMASK		(0x07FFFFFF)
+#define PDP_VID4VBASEADDR_VID4VBASEADDR_SHIFT		(5)
+#define PDP_VID4VBASEADDR_VID4VBASEADDR_LENGTH		(27)
+#define PDP_VID4VBASEADDR_VID4VBASEADDR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1POSTSKIPCTRL_OFFSET		(0x0230)
+
+/* PDP, VID1POSTSKIPCTRL, VID1HPOSTCLIP
+*/
+#define PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_MASK		(0x007F0000)
+#define PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LSBMASK		(0x0000007F)
+#define PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SHIFT		(16)
+#define PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_LENGTH		(7)
+#define PDP_VID1POSTSKIPCTRL_VID1HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1POSTSKIPCTRL, VID1VPOSTCLIP
+*/
+#define PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_MASK		(0x0000003F)
+#define PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LSBMASK		(0x0000003F)
+#define PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SHIFT		(0)
+#define PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_LENGTH		(6)
+#define PDP_VID1POSTSKIPCTRL_VID1VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2POSTSKIPCTRL_OFFSET		(0x0234)
+
+/* PDP, VID2POSTSKIPCTRL, VID2HPOSTCLIP
+*/
+#define PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_MASK		(0x007F0000)
+#define PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LSBMASK		(0x0000007F)
+#define PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SHIFT		(16)
+#define PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_LENGTH		(7)
+#define PDP_VID2POSTSKIPCTRL_VID2HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2POSTSKIPCTRL, VID2VPOSTCLIP
+*/
+#define PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_MASK		(0x0000003F)
+#define PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LSBMASK		(0x0000003F)
+#define PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SHIFT		(0)
+#define PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_LENGTH		(6)
+#define PDP_VID2POSTSKIPCTRL_VID2VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3POSTSKIPCTRL_OFFSET		(0x0238)
+
+/* PDP, VID3POSTSKIPCTRL, VID3HPOSTCLIP
+*/
+#define PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_MASK		(0x007F0000)
+#define PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LSBMASK		(0x0000007F)
+#define PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SHIFT		(16)
+#define PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_LENGTH		(7)
+#define PDP_VID3POSTSKIPCTRL_VID3HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3POSTSKIPCTRL, VID3VPOSTCLIP
+*/
+#define PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_MASK		(0x0000003F)
+#define PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LSBMASK		(0x0000003F)
+#define PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SHIFT		(0)
+#define PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_LENGTH		(6)
+#define PDP_VID3POSTSKIPCTRL_VID3VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4POSTSKIPCTRL_OFFSET		(0x023C)
+
+/* PDP, VID4POSTSKIPCTRL, VID4HPOSTCLIP
+*/
+#define PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_MASK		(0x007F0000)
+#define PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LSBMASK		(0x0000007F)
+#define PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SHIFT		(16)
+#define PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_LENGTH		(7)
+#define PDP_VID4POSTSKIPCTRL_VID4HPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4POSTSKIPCTRL, VID4VPOSTCLIP
+*/
+#define PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_MASK		(0x0000003F)
+#define PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LSBMASK		(0x0000003F)
+#define PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SHIFT		(0)
+#define PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_LENGTH		(6)
+#define PDP_VID4POSTSKIPCTRL_VID4VPOSTCLIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1DECIMATE_CTRL_OFFSET		(0x0240)
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_PIXEL_HALVE
+*/
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1DECIMATE_CTRL, GRPH1DECIMATE_EN
+*/
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_MASK		(0x00000001)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SHIFT		(0)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_LENGTH		(1)
+#define PDP_GRPH1DECIMATE_CTRL_GRPH1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2DECIMATE_CTRL_OFFSET		(0x0244)
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_PIXEL_HALVE
+*/
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2DECIMATE_CTRL, GRPH2DECIMATE_EN
+*/
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_MASK		(0x00000001)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SHIFT		(0)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_LENGTH		(1)
+#define PDP_GRPH2DECIMATE_CTRL_GRPH2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3DECIMATE_CTRL_OFFSET		(0x0248)
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_PIXEL_HALVE
+*/
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3DECIMATE_CTRL, GRPH3DECIMATE_EN
+*/
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_MASK		(0x00000001)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SHIFT		(0)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_LENGTH		(1)
+#define PDP_GRPH3DECIMATE_CTRL_GRPH3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4DECIMATE_CTRL_OFFSET		(0x024C)
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_PIXEL_HALVE
+*/
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4DECIMATE_CTRL, GRPH4DECIMATE_EN
+*/
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_MASK		(0x00000001)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SHIFT		(0)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_LENGTH		(1)
+#define PDP_GRPH4DECIMATE_CTRL_GRPH4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1DECIMATE_CTRL_OFFSET		(0x0250)
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_PIXEL_HALVE
+*/
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1DECIMATE_CTRL, VID1DECIMATE_EN
+*/
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_MASK		(0x00000001)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SHIFT		(0)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_LENGTH		(1)
+#define PDP_VID1DECIMATE_CTRL_VID1DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2DECIMATE_CTRL_OFFSET		(0x0254)
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_PIXEL_HALVE
+*/
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2DECIMATE_CTRL, VID2DECIMATE_EN
+*/
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_MASK		(0x00000001)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SHIFT		(0)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_LENGTH		(1)
+#define PDP_VID2DECIMATE_CTRL_VID2DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3DECIMATE_CTRL_OFFSET		(0x0258)
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_PIXEL_HALVE
+*/
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3DECIMATE_CTRL, VID3DECIMATE_EN
+*/
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_MASK		(0x00000001)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SHIFT		(0)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_LENGTH		(1)
+#define PDP_VID3DECIMATE_CTRL_VID3DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4DECIMATE_CTRL_OFFSET		(0x025C)
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_COUNT
+*/
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_MASK		(0x000000F0)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LSBMASK		(0x0000000F)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SHIFT		(4)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_LENGTH		(4)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_COUNT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_LINE_DISCARD_MODE
+*/
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_MASK		(0x00000008)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LSBMASK		(0x00000001)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SHIFT		(3)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_LENGTH		(1)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_LINE_DISCARD_MODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_PIXEL_HALVE
+*/
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_MASK		(0x00000004)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LSBMASK		(0x00000001)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SHIFT		(2)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_LENGTH		(1)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_PIXEL_HALVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4DECIMATE_CTRL, VID4DECIMATE_EN
+*/
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_MASK		(0x00000001)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LSBMASK		(0x00000001)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SHIFT		(0)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_LENGTH		(1)
+#define PDP_VID4DECIMATE_CTRL_VID4DECIMATE_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1SKIPCTRL_OFFSET		(0x0270)
+
+/* PDP, VID1SKIPCTRL, VID1HSKIP
+*/
+#define PDP_VID1SKIPCTRL_VID1HSKIP_MASK		(0x0FFF0000)
+#define PDP_VID1SKIPCTRL_VID1HSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID1SKIPCTRL_VID1HSKIP_SHIFT		(16)
+#define PDP_VID1SKIPCTRL_VID1HSKIP_LENGTH		(12)
+#define PDP_VID1SKIPCTRL_VID1HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SKIPCTRL, VID1VSKIP
+*/
+#define PDP_VID1SKIPCTRL_VID1VSKIP_MASK		(0x00000FFF)
+#define PDP_VID1SKIPCTRL_VID1VSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID1SKIPCTRL_VID1VSKIP_SHIFT		(0)
+#define PDP_VID1SKIPCTRL_VID1VSKIP_LENGTH		(12)
+#define PDP_VID1SKIPCTRL_VID1VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2SKIPCTRL_OFFSET		(0x0274)
+
+/* PDP, VID2SKIPCTRL, VID2HSKIP
+*/
+#define PDP_VID2SKIPCTRL_VID2HSKIP_MASK		(0x0FFF0000)
+#define PDP_VID2SKIPCTRL_VID2HSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID2SKIPCTRL_VID2HSKIP_SHIFT		(16)
+#define PDP_VID2SKIPCTRL_VID2HSKIP_LENGTH		(12)
+#define PDP_VID2SKIPCTRL_VID2HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SKIPCTRL, VID2VSKIP
+*/
+#define PDP_VID2SKIPCTRL_VID2VSKIP_MASK		(0x00000FFF)
+#define PDP_VID2SKIPCTRL_VID2VSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID2SKIPCTRL_VID2VSKIP_SHIFT		(0)
+#define PDP_VID2SKIPCTRL_VID2VSKIP_LENGTH		(12)
+#define PDP_VID2SKIPCTRL_VID2VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3SKIPCTRL_OFFSET		(0x0278)
+
+/* PDP, VID3SKIPCTRL, VID3HSKIP
+*/
+#define PDP_VID3SKIPCTRL_VID3HSKIP_MASK		(0x0FFF0000)
+#define PDP_VID3SKIPCTRL_VID3HSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID3SKIPCTRL_VID3HSKIP_SHIFT		(16)
+#define PDP_VID3SKIPCTRL_VID3HSKIP_LENGTH		(12)
+#define PDP_VID3SKIPCTRL_VID3HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SKIPCTRL, VID3VSKIP
+*/
+#define PDP_VID3SKIPCTRL_VID3VSKIP_MASK		(0x00000FFF)
+#define PDP_VID3SKIPCTRL_VID3VSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID3SKIPCTRL_VID3VSKIP_SHIFT		(0)
+#define PDP_VID3SKIPCTRL_VID3VSKIP_LENGTH		(12)
+#define PDP_VID3SKIPCTRL_VID3VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4SKIPCTRL_OFFSET		(0x027C)
+
+/* PDP, VID4SKIPCTRL, VID4HSKIP
+*/
+#define PDP_VID4SKIPCTRL_VID4HSKIP_MASK		(0x0FFF0000)
+#define PDP_VID4SKIPCTRL_VID4HSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID4SKIPCTRL_VID4HSKIP_SHIFT		(16)
+#define PDP_VID4SKIPCTRL_VID4HSKIP_LENGTH		(12)
+#define PDP_VID4SKIPCTRL_VID4HSKIP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SKIPCTRL, VID4VSKIP
+*/
+#define PDP_VID4SKIPCTRL_VID4VSKIP_MASK		(0x00000FFF)
+#define PDP_VID4SKIPCTRL_VID4VSKIP_LSBMASK		(0x00000FFF)
+#define PDP_VID4SKIPCTRL_VID4VSKIP_SHIFT		(0)
+#define PDP_VID4SKIPCTRL_VID4VSKIP_LENGTH		(12)
+#define PDP_VID4SKIPCTRL_VID4VSKIP_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1SCALECTRL_OFFSET		(0x0460)
+
+/* PDP, VID1SCALECTRL, VID1HSCALEBP
+*/
+#define PDP_VID1SCALECTRL_VID1HSCALEBP_MASK		(0x80000000)
+#define PDP_VID1SCALECTRL_VID1HSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID1SCALECTRL_VID1HSCALEBP_SHIFT		(31)
+#define PDP_VID1SCALECTRL_VID1HSCALEBP_LENGTH		(1)
+#define PDP_VID1SCALECTRL_VID1HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSCALEBP
+*/
+#define PDP_VID1SCALECTRL_VID1VSCALEBP_MASK		(0x40000000)
+#define PDP_VID1SCALECTRL_VID1VSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID1SCALECTRL_VID1VSCALEBP_SHIFT		(30)
+#define PDP_VID1SCALECTRL_VID1VSCALEBP_LENGTH		(1)
+#define PDP_VID1SCALECTRL_VID1VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1HSBEFOREVS
+*/
+#define PDP_VID1SCALECTRL_VID1HSBEFOREVS_MASK		(0x20000000)
+#define PDP_VID1SCALECTRL_VID1HSBEFOREVS_LSBMASK		(0x00000001)
+#define PDP_VID1SCALECTRL_VID1HSBEFOREVS_SHIFT		(29)
+#define PDP_VID1SCALECTRL_VID1HSBEFOREVS_LENGTH		(1)
+#define PDP_VID1SCALECTRL_VID1HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VSURUNCTRL
+*/
+#define PDP_VID1SCALECTRL_VID1VSURUNCTRL_MASK		(0x08000000)
+#define PDP_VID1SCALECTRL_VID1VSURUNCTRL_LSBMASK		(0x00000001)
+#define PDP_VID1SCALECTRL_VID1VSURUNCTRL_SHIFT		(27)
+#define PDP_VID1SCALECTRL_VID1VSURUNCTRL_LENGTH		(1)
+#define PDP_VID1SCALECTRL_VID1VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1PAN_EN
+*/
+#define PDP_VID1SCALECTRL_VID1PAN_EN_MASK		(0x00040000)
+#define PDP_VID1SCALECTRL_VID1PAN_EN_LSBMASK		(0x00000001)
+#define PDP_VID1SCALECTRL_VID1PAN_EN_SHIFT		(18)
+#define PDP_VID1SCALECTRL_VID1PAN_EN_LENGTH		(1)
+#define PDP_VID1SCALECTRL_VID1PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VORDER
+*/
+#define PDP_VID1SCALECTRL_VID1VORDER_MASK		(0x00030000)
+#define PDP_VID1SCALECTRL_VID1VORDER_LSBMASK		(0x00000003)
+#define PDP_VID1SCALECTRL_VID1VORDER_SHIFT		(16)
+#define PDP_VID1SCALECTRL_VID1VORDER_LENGTH		(2)
+#define PDP_VID1SCALECTRL_VID1VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALECTRL, VID1VPITCH
+*/
+#define PDP_VID1SCALECTRL_VID1VPITCH_MASK		(0x0000FFFF)
+#define PDP_VID1SCALECTRL_VID1VPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID1SCALECTRL_VID1VPITCH_SHIFT		(0)
+#define PDP_VID1SCALECTRL_VID1VPITCH_LENGTH		(16)
+#define PDP_VID1SCALECTRL_VID1VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VSINIT_OFFSET		(0x0464)
+
+/* PDP, VID1VSINIT, VID1VINITIAL1
+*/
+#define PDP_VID1VSINIT_VID1VINITIAL1_MASK		(0xFFFF0000)
+#define PDP_VID1VSINIT_VID1VINITIAL1_LSBMASK		(0x0000FFFF)
+#define PDP_VID1VSINIT_VID1VINITIAL1_SHIFT		(16)
+#define PDP_VID1VSINIT_VID1VINITIAL1_LENGTH		(16)
+#define PDP_VID1VSINIT_VID1VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1VSINIT, VID1VINITIAL0
+*/
+#define PDP_VID1VSINIT_VID1VINITIAL0_MASK		(0x0000FFFF)
+#define PDP_VID1VSINIT_VID1VINITIAL0_LSBMASK		(0x0000FFFF)
+#define PDP_VID1VSINIT_VID1VINITIAL0_SHIFT		(0)
+#define PDP_VID1VSINIT_VID1VINITIAL0_LENGTH		(16)
+#define PDP_VID1VSINIT_VID1VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF0_OFFSET		(0x0468)
+
+/* PDP, VID1VCOEFF0, VID1VCOEFF0
+*/
+#define PDP_VID1VCOEFF0_VID1VCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF0_VID1VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF0_VID1VCOEFF0_SHIFT		(0)
+#define PDP_VID1VCOEFF0_VID1VCOEFF0_LENGTH		(32)
+#define PDP_VID1VCOEFF0_VID1VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF1_OFFSET		(0x046C)
+
+/* PDP, VID1VCOEFF1, VID1VCOEFF1
+*/
+#define PDP_VID1VCOEFF1_VID1VCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF1_VID1VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF1_VID1VCOEFF1_SHIFT		(0)
+#define PDP_VID1VCOEFF1_VID1VCOEFF1_LENGTH		(32)
+#define PDP_VID1VCOEFF1_VID1VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF2_OFFSET		(0x0470)
+
+/* PDP, VID1VCOEFF2, VID1VCOEFF2
+*/
+#define PDP_VID1VCOEFF2_VID1VCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF2_VID1VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF2_VID1VCOEFF2_SHIFT		(0)
+#define PDP_VID1VCOEFF2_VID1VCOEFF2_LENGTH		(32)
+#define PDP_VID1VCOEFF2_VID1VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF3_OFFSET		(0x0474)
+
+/* PDP, VID1VCOEFF3, VID1VCOEFF3
+*/
+#define PDP_VID1VCOEFF3_VID1VCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF3_VID1VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF3_VID1VCOEFF3_SHIFT		(0)
+#define PDP_VID1VCOEFF3_VID1VCOEFF3_LENGTH		(32)
+#define PDP_VID1VCOEFF3_VID1VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF4_OFFSET		(0x0478)
+
+/* PDP, VID1VCOEFF4, VID1VCOEFF4
+*/
+#define PDP_VID1VCOEFF4_VID1VCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF4_VID1VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF4_VID1VCOEFF4_SHIFT		(0)
+#define PDP_VID1VCOEFF4_VID1VCOEFF4_LENGTH		(32)
+#define PDP_VID1VCOEFF4_VID1VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF5_OFFSET		(0x047C)
+
+/* PDP, VID1VCOEFF5, VID1VCOEFF5
+*/
+#define PDP_VID1VCOEFF5_VID1VCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF5_VID1VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF5_VID1VCOEFF5_SHIFT		(0)
+#define PDP_VID1VCOEFF5_VID1VCOEFF5_LENGTH		(32)
+#define PDP_VID1VCOEFF5_VID1VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF6_OFFSET		(0x0480)
+
+/* PDP, VID1VCOEFF6, VID1VCOEFF6
+*/
+#define PDP_VID1VCOEFF6_VID1VCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF6_VID1VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF6_VID1VCOEFF6_SHIFT		(0)
+#define PDP_VID1VCOEFF6_VID1VCOEFF6_LENGTH		(32)
+#define PDP_VID1VCOEFF6_VID1VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF7_OFFSET		(0x0484)
+
+/* PDP, VID1VCOEFF7, VID1VCOEFF7
+*/
+#define PDP_VID1VCOEFF7_VID1VCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF7_VID1VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1VCOEFF7_VID1VCOEFF7_SHIFT		(0)
+#define PDP_VID1VCOEFF7_VID1VCOEFF7_LENGTH		(32)
+#define PDP_VID1VCOEFF7_VID1VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1VCOEFF8_OFFSET		(0x0488)
+
+/* PDP, VID1VCOEFF8, VID1VCOEFF8
+*/
+#define PDP_VID1VCOEFF8_VID1VCOEFF8_MASK		(0x000000FF)
+#define PDP_VID1VCOEFF8_VID1VCOEFF8_LSBMASK		(0x000000FF)
+#define PDP_VID1VCOEFF8_VID1VCOEFF8_SHIFT		(0)
+#define PDP_VID1VCOEFF8_VID1VCOEFF8_LENGTH		(8)
+#define PDP_VID1VCOEFF8_VID1VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HSINIT_OFFSET		(0x048C)
+
+/* PDP, VID1HSINIT, VID1HINITIAL
+*/
+#define PDP_VID1HSINIT_VID1HINITIAL_MASK		(0xFFFF0000)
+#define PDP_VID1HSINIT_VID1HINITIAL_LSBMASK		(0x0000FFFF)
+#define PDP_VID1HSINIT_VID1HINITIAL_SHIFT		(16)
+#define PDP_VID1HSINIT_VID1HINITIAL_LENGTH		(16)
+#define PDP_VID1HSINIT_VID1HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1HSINIT, VID1HPITCH
+*/
+#define PDP_VID1HSINIT_VID1HPITCH_MASK		(0x0000FFFF)
+#define PDP_VID1HSINIT_VID1HPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID1HSINIT_VID1HPITCH_SHIFT		(0)
+#define PDP_VID1HSINIT_VID1HPITCH_LENGTH		(16)
+#define PDP_VID1HSINIT_VID1HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF0_OFFSET		(0x0490)
+
+/* PDP, VID1HCOEFF0, VID1HCOEFF0
+*/
+#define PDP_VID1HCOEFF0_VID1HCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF0_VID1HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF0_VID1HCOEFF0_SHIFT		(0)
+#define PDP_VID1HCOEFF0_VID1HCOEFF0_LENGTH		(32)
+#define PDP_VID1HCOEFF0_VID1HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF1_OFFSET		(0x0494)
+
+/* PDP, VID1HCOEFF1, VID1HCOEFF1
+*/
+#define PDP_VID1HCOEFF1_VID1HCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF1_VID1HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF1_VID1HCOEFF1_SHIFT		(0)
+#define PDP_VID1HCOEFF1_VID1HCOEFF1_LENGTH		(32)
+#define PDP_VID1HCOEFF1_VID1HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF2_OFFSET		(0x0498)
+
+/* PDP, VID1HCOEFF2, VID1HCOEFF2
+*/
+#define PDP_VID1HCOEFF2_VID1HCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF2_VID1HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF2_VID1HCOEFF2_SHIFT		(0)
+#define PDP_VID1HCOEFF2_VID1HCOEFF2_LENGTH		(32)
+#define PDP_VID1HCOEFF2_VID1HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF3_OFFSET		(0x049C)
+
+/* PDP, VID1HCOEFF3, VID1HCOEFF3
+*/
+#define PDP_VID1HCOEFF3_VID1HCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF3_VID1HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF3_VID1HCOEFF3_SHIFT		(0)
+#define PDP_VID1HCOEFF3_VID1HCOEFF3_LENGTH		(32)
+#define PDP_VID1HCOEFF3_VID1HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF4_OFFSET		(0x04A0)
+
+/* PDP, VID1HCOEFF4, VID1HCOEFF4
+*/
+#define PDP_VID1HCOEFF4_VID1HCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF4_VID1HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF4_VID1HCOEFF4_SHIFT		(0)
+#define PDP_VID1HCOEFF4_VID1HCOEFF4_LENGTH		(32)
+#define PDP_VID1HCOEFF4_VID1HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF5_OFFSET		(0x04A4)
+
+/* PDP, VID1HCOEFF5, VID1HCOEFF5
+*/
+#define PDP_VID1HCOEFF5_VID1HCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF5_VID1HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF5_VID1HCOEFF5_SHIFT		(0)
+#define PDP_VID1HCOEFF5_VID1HCOEFF5_LENGTH		(32)
+#define PDP_VID1HCOEFF5_VID1HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF6_OFFSET		(0x04A8)
+
+/* PDP, VID1HCOEFF6, VID1HCOEFF6
+*/
+#define PDP_VID1HCOEFF6_VID1HCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF6_VID1HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF6_VID1HCOEFF6_SHIFT		(0)
+#define PDP_VID1HCOEFF6_VID1HCOEFF6_LENGTH		(32)
+#define PDP_VID1HCOEFF6_VID1HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF7_OFFSET		(0x04AC)
+
+/* PDP, VID1HCOEFF7, VID1HCOEFF7
+*/
+#define PDP_VID1HCOEFF7_VID1HCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF7_VID1HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF7_VID1HCOEFF7_SHIFT		(0)
+#define PDP_VID1HCOEFF7_VID1HCOEFF7_LENGTH		(32)
+#define PDP_VID1HCOEFF7_VID1HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF8_OFFSET		(0x04B0)
+
+/* PDP, VID1HCOEFF8, VID1HCOEFF8
+*/
+#define PDP_VID1HCOEFF8_VID1HCOEFF8_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF8_VID1HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF8_VID1HCOEFF8_SHIFT		(0)
+#define PDP_VID1HCOEFF8_VID1HCOEFF8_LENGTH		(32)
+#define PDP_VID1HCOEFF8_VID1HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF9_OFFSET		(0x04B4)
+
+/* PDP, VID1HCOEFF9, VID1HCOEFF9
+*/
+#define PDP_VID1HCOEFF9_VID1HCOEFF9_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF9_VID1HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF9_VID1HCOEFF9_SHIFT		(0)
+#define PDP_VID1HCOEFF9_VID1HCOEFF9_LENGTH		(32)
+#define PDP_VID1HCOEFF9_VID1HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF10_OFFSET		(0x04B8)
+
+/* PDP, VID1HCOEFF10, VID1HCOEFF10
+*/
+#define PDP_VID1HCOEFF10_VID1HCOEFF10_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF10_VID1HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF10_VID1HCOEFF10_SHIFT		(0)
+#define PDP_VID1HCOEFF10_VID1HCOEFF10_LENGTH		(32)
+#define PDP_VID1HCOEFF10_VID1HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF11_OFFSET		(0x04BC)
+
+/* PDP, VID1HCOEFF11, VID1HCOEFF11
+*/
+#define PDP_VID1HCOEFF11_VID1HCOEFF11_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF11_VID1HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF11_VID1HCOEFF11_SHIFT		(0)
+#define PDP_VID1HCOEFF11_VID1HCOEFF11_LENGTH		(32)
+#define PDP_VID1HCOEFF11_VID1HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF12_OFFSET		(0x04C0)
+
+/* PDP, VID1HCOEFF12, VID1HCOEFF12
+*/
+#define PDP_VID1HCOEFF12_VID1HCOEFF12_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF12_VID1HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF12_VID1HCOEFF12_SHIFT		(0)
+#define PDP_VID1HCOEFF12_VID1HCOEFF12_LENGTH		(32)
+#define PDP_VID1HCOEFF12_VID1HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF13_OFFSET		(0x04C4)
+
+/* PDP, VID1HCOEFF13, VID1HCOEFF13
+*/
+#define PDP_VID1HCOEFF13_VID1HCOEFF13_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF13_VID1HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF13_VID1HCOEFF13_SHIFT		(0)
+#define PDP_VID1HCOEFF13_VID1HCOEFF13_LENGTH		(32)
+#define PDP_VID1HCOEFF13_VID1HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF14_OFFSET		(0x04C8)
+
+/* PDP, VID1HCOEFF14, VID1HCOEFF14
+*/
+#define PDP_VID1HCOEFF14_VID1HCOEFF14_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF14_VID1HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF14_VID1HCOEFF14_SHIFT		(0)
+#define PDP_VID1HCOEFF14_VID1HCOEFF14_LENGTH		(32)
+#define PDP_VID1HCOEFF14_VID1HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF15_OFFSET		(0x04CC)
+
+/* PDP, VID1HCOEFF15, VID1HCOEFF15
+*/
+#define PDP_VID1HCOEFF15_VID1HCOEFF15_MASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF15_VID1HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID1HCOEFF15_VID1HCOEFF15_SHIFT		(0)
+#define PDP_VID1HCOEFF15_VID1HCOEFF15_LENGTH		(32)
+#define PDP_VID1HCOEFF15_VID1HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1HCOEFF16_OFFSET		(0x04D0)
+
+/* PDP, VID1HCOEFF16, VID1HCOEFF16
+*/
+#define PDP_VID1HCOEFF16_VID1HCOEFF16_MASK		(0x000000FF)
+#define PDP_VID1HCOEFF16_VID1HCOEFF16_LSBMASK		(0x000000FF)
+#define PDP_VID1HCOEFF16_VID1HCOEFF16_SHIFT		(0)
+#define PDP_VID1HCOEFF16_VID1HCOEFF16_LENGTH		(8)
+#define PDP_VID1HCOEFF16_VID1HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1SCALESIZE_OFFSET		(0x04D4)
+
+/* PDP, VID1SCALESIZE, VID1SCALEWIDTH
+*/
+#define PDP_VID1SCALESIZE_VID1SCALEWIDTH_MASK		(0x0FFF0000)
+#define PDP_VID1SCALESIZE_VID1SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID1SCALESIZE_VID1SCALEWIDTH_SHIFT		(16)
+#define PDP_VID1SCALESIZE_VID1SCALEWIDTH_LENGTH		(12)
+#define PDP_VID1SCALESIZE_VID1SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1SCALESIZE, VID1SCALEHEIGHT
+*/
+#define PDP_VID1SCALESIZE_VID1SCALEHEIGHT_MASK		(0x00000FFF)
+#define PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SHIFT		(0)
+#define PDP_VID1SCALESIZE_VID1SCALEHEIGHT_LENGTH		(12)
+#define PDP_VID1SCALESIZE_VID1SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CORE_ID_OFFSET		(0x04E0)
+
+/* PDP, PVR_PDP_CORE_ID, GROUP_ID
+*/
+#define PDP_CORE_ID_GROUP_ID_MASK		(0xFF000000)
+#define PDP_CORE_ID_GROUP_ID_LSBMASK		(0x000000FF)
+#define PDP_CORE_ID_GROUP_ID_SHIFT		(24)
+#define PDP_CORE_ID_GROUP_ID_LENGTH		(8)
+#define PDP_CORE_ID_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_PDP_CORE_ID, CORE_ID
+*/
+#define PDP_CORE_ID_CORE_ID_MASK		(0x00FF0000)
+#define PDP_CORE_ID_CORE_ID_LSBMASK		(0x000000FF)
+#define PDP_CORE_ID_CORE_ID_SHIFT		(16)
+#define PDP_CORE_ID_CORE_ID_LENGTH		(8)
+#define PDP_CORE_ID_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_PDP_CORE_ID, CONFIG_ID
+*/
+#define PDP_CORE_ID_CONFIG_ID_MASK		(0x0000FFFF)
+#define PDP_CORE_ID_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define PDP_CORE_ID_CONFIG_ID_SHIFT		(0)
+#define PDP_CORE_ID_CONFIG_ID_LENGTH		(16)
+#define PDP_CORE_ID_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CORE_REV_OFFSET		(0x04F0)
+
+/* PDP, PVR_PDP_CORE_REV, MAJOR_REV
+*/
+#define PDP_CORE_REV_MAJOR_REV_MASK		(0x00FF0000)
+#define PDP_CORE_REV_MAJOR_REV_LSBMASK		(0x000000FF)
+#define PDP_CORE_REV_MAJOR_REV_SHIFT		(16)
+#define PDP_CORE_REV_MAJOR_REV_LENGTH		(8)
+#define PDP_CORE_REV_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_PDP_CORE_REV, MINOR_REV
+*/
+#define PDP_CORE_REV_MINOR_REV_MASK		(0x0000FF00)
+#define PDP_CORE_REV_MINOR_REV_LSBMASK		(0x000000FF)
+#define PDP_CORE_REV_MINOR_REV_SHIFT		(8)
+#define PDP_CORE_REV_MINOR_REV_LENGTH		(8)
+#define PDP_CORE_REV_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_PDP_CORE_REV, MAINT_REV
+*/
+#define PDP_CORE_REV_MAINT_REV_MASK		(0x000000FF)
+#define PDP_CORE_REV_MAINT_REV_LSBMASK		(0x000000FF)
+#define PDP_CORE_REV_MAINT_REV_SHIFT		(0)
+#define PDP_CORE_REV_MAINT_REV_LENGTH		(8)
+#define PDP_CORE_REV_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2SCALECTRL_OFFSET		(0x0500)
+
+/* PDP, VID2SCALECTRL, VID2HSCALEBP
+*/
+#define PDP_VID2SCALECTRL_VID2HSCALEBP_MASK		(0x80000000)
+#define PDP_VID2SCALECTRL_VID2HSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID2SCALECTRL_VID2HSCALEBP_SHIFT		(31)
+#define PDP_VID2SCALECTRL_VID2HSCALEBP_LENGTH		(1)
+#define PDP_VID2SCALECTRL_VID2HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSCALEBP
+*/
+#define PDP_VID2SCALECTRL_VID2VSCALEBP_MASK		(0x40000000)
+#define PDP_VID2SCALECTRL_VID2VSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID2SCALECTRL_VID2VSCALEBP_SHIFT		(30)
+#define PDP_VID2SCALECTRL_VID2VSCALEBP_LENGTH		(1)
+#define PDP_VID2SCALECTRL_VID2VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2HSBEFOREVS
+*/
+#define PDP_VID2SCALECTRL_VID2HSBEFOREVS_MASK		(0x20000000)
+#define PDP_VID2SCALECTRL_VID2HSBEFOREVS_LSBMASK		(0x00000001)
+#define PDP_VID2SCALECTRL_VID2HSBEFOREVS_SHIFT		(29)
+#define PDP_VID2SCALECTRL_VID2HSBEFOREVS_LENGTH		(1)
+#define PDP_VID2SCALECTRL_VID2HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VSURUNCTRL
+*/
+#define PDP_VID2SCALECTRL_VID2VSURUNCTRL_MASK		(0x08000000)
+#define PDP_VID2SCALECTRL_VID2VSURUNCTRL_LSBMASK		(0x00000001)
+#define PDP_VID2SCALECTRL_VID2VSURUNCTRL_SHIFT		(27)
+#define PDP_VID2SCALECTRL_VID2VSURUNCTRL_LENGTH		(1)
+#define PDP_VID2SCALECTRL_VID2VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2PAN_EN
+*/
+#define PDP_VID2SCALECTRL_VID2PAN_EN_MASK		(0x00040000)
+#define PDP_VID2SCALECTRL_VID2PAN_EN_LSBMASK		(0x00000001)
+#define PDP_VID2SCALECTRL_VID2PAN_EN_SHIFT		(18)
+#define PDP_VID2SCALECTRL_VID2PAN_EN_LENGTH		(1)
+#define PDP_VID2SCALECTRL_VID2PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VORDER
+*/
+#define PDP_VID2SCALECTRL_VID2VORDER_MASK		(0x00030000)
+#define PDP_VID2SCALECTRL_VID2VORDER_LSBMASK		(0x00000003)
+#define PDP_VID2SCALECTRL_VID2VORDER_SHIFT		(16)
+#define PDP_VID2SCALECTRL_VID2VORDER_LENGTH		(2)
+#define PDP_VID2SCALECTRL_VID2VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALECTRL, VID2VPITCH
+*/
+#define PDP_VID2SCALECTRL_VID2VPITCH_MASK		(0x0000FFFF)
+#define PDP_VID2SCALECTRL_VID2VPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID2SCALECTRL_VID2VPITCH_SHIFT		(0)
+#define PDP_VID2SCALECTRL_VID2VPITCH_LENGTH		(16)
+#define PDP_VID2SCALECTRL_VID2VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VSINIT_OFFSET		(0x0504)
+
+/* PDP, VID2VSINIT, VID2VINITIAL1
+*/
+#define PDP_VID2VSINIT_VID2VINITIAL1_MASK		(0xFFFF0000)
+#define PDP_VID2VSINIT_VID2VINITIAL1_LSBMASK		(0x0000FFFF)
+#define PDP_VID2VSINIT_VID2VINITIAL1_SHIFT		(16)
+#define PDP_VID2VSINIT_VID2VINITIAL1_LENGTH		(16)
+#define PDP_VID2VSINIT_VID2VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2VSINIT, VID2VINITIAL0
+*/
+#define PDP_VID2VSINIT_VID2VINITIAL0_MASK		(0x0000FFFF)
+#define PDP_VID2VSINIT_VID2VINITIAL0_LSBMASK		(0x0000FFFF)
+#define PDP_VID2VSINIT_VID2VINITIAL0_SHIFT		(0)
+#define PDP_VID2VSINIT_VID2VINITIAL0_LENGTH		(16)
+#define PDP_VID2VSINIT_VID2VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF0_OFFSET		(0x0508)
+
+/* PDP, VID2VCOEFF0, VID2VCOEFF0
+*/
+#define PDP_VID2VCOEFF0_VID2VCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF0_VID2VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF0_VID2VCOEFF0_SHIFT		(0)
+#define PDP_VID2VCOEFF0_VID2VCOEFF0_LENGTH		(32)
+#define PDP_VID2VCOEFF0_VID2VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF1_OFFSET		(0x050C)
+
+/* PDP, VID2VCOEFF1, VID2VCOEFF1
+*/
+#define PDP_VID2VCOEFF1_VID2VCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF1_VID2VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF1_VID2VCOEFF1_SHIFT		(0)
+#define PDP_VID2VCOEFF1_VID2VCOEFF1_LENGTH		(32)
+#define PDP_VID2VCOEFF1_VID2VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF2_OFFSET		(0x0510)
+
+/* PDP, VID2VCOEFF2, VID2VCOEFF2
+*/
+#define PDP_VID2VCOEFF2_VID2VCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF2_VID2VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF2_VID2VCOEFF2_SHIFT		(0)
+#define PDP_VID2VCOEFF2_VID2VCOEFF2_LENGTH		(32)
+#define PDP_VID2VCOEFF2_VID2VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF3_OFFSET		(0x0514)
+
+/* PDP, VID2VCOEFF3, VID2VCOEFF3
+*/
+#define PDP_VID2VCOEFF3_VID2VCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF3_VID2VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF3_VID2VCOEFF3_SHIFT		(0)
+#define PDP_VID2VCOEFF3_VID2VCOEFF3_LENGTH		(32)
+#define PDP_VID2VCOEFF3_VID2VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF4_OFFSET		(0x0518)
+
+/* PDP, VID2VCOEFF4, VID2VCOEFF4
+*/
+#define PDP_VID2VCOEFF4_VID2VCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF4_VID2VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF4_VID2VCOEFF4_SHIFT		(0)
+#define PDP_VID2VCOEFF4_VID2VCOEFF4_LENGTH		(32)
+#define PDP_VID2VCOEFF4_VID2VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF5_OFFSET		(0x051C)
+
+/* PDP, VID2VCOEFF5, VID2VCOEFF5
+*/
+#define PDP_VID2VCOEFF5_VID2VCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF5_VID2VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF5_VID2VCOEFF5_SHIFT		(0)
+#define PDP_VID2VCOEFF5_VID2VCOEFF5_LENGTH		(32)
+#define PDP_VID2VCOEFF5_VID2VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF6_OFFSET		(0x0520)
+
+/* PDP, VID2VCOEFF6, VID2VCOEFF6
+*/
+#define PDP_VID2VCOEFF6_VID2VCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF6_VID2VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF6_VID2VCOEFF6_SHIFT		(0)
+#define PDP_VID2VCOEFF6_VID2VCOEFF6_LENGTH		(32)
+#define PDP_VID2VCOEFF6_VID2VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF7_OFFSET		(0x0524)
+
+/* PDP, VID2VCOEFF7, VID2VCOEFF7
+*/
+#define PDP_VID2VCOEFF7_VID2VCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF7_VID2VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2VCOEFF7_VID2VCOEFF7_SHIFT		(0)
+#define PDP_VID2VCOEFF7_VID2VCOEFF7_LENGTH		(32)
+#define PDP_VID2VCOEFF7_VID2VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2VCOEFF8_OFFSET		(0x0528)
+
+/* PDP, VID2VCOEFF8, VID2VCOEFF8
+*/
+#define PDP_VID2VCOEFF8_VID2VCOEFF8_MASK		(0x000000FF)
+#define PDP_VID2VCOEFF8_VID2VCOEFF8_LSBMASK		(0x000000FF)
+#define PDP_VID2VCOEFF8_VID2VCOEFF8_SHIFT		(0)
+#define PDP_VID2VCOEFF8_VID2VCOEFF8_LENGTH		(8)
+#define PDP_VID2VCOEFF8_VID2VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HSINIT_OFFSET		(0x052C)
+
+/* PDP, VID2HSINIT, VID2HINITIAL
+*/
+#define PDP_VID2HSINIT_VID2HINITIAL_MASK		(0xFFFF0000)
+#define PDP_VID2HSINIT_VID2HINITIAL_LSBMASK		(0x0000FFFF)
+#define PDP_VID2HSINIT_VID2HINITIAL_SHIFT		(16)
+#define PDP_VID2HSINIT_VID2HINITIAL_LENGTH		(16)
+#define PDP_VID2HSINIT_VID2HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2HSINIT, VID2HPITCH
+*/
+#define PDP_VID2HSINIT_VID2HPITCH_MASK		(0x0000FFFF)
+#define PDP_VID2HSINIT_VID2HPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID2HSINIT_VID2HPITCH_SHIFT		(0)
+#define PDP_VID2HSINIT_VID2HPITCH_LENGTH		(16)
+#define PDP_VID2HSINIT_VID2HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF0_OFFSET		(0x0530)
+
+/* PDP, VID2HCOEFF0, VID2HCOEFF0
+*/
+#define PDP_VID2HCOEFF0_VID2HCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF0_VID2HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF0_VID2HCOEFF0_SHIFT		(0)
+#define PDP_VID2HCOEFF0_VID2HCOEFF0_LENGTH		(32)
+#define PDP_VID2HCOEFF0_VID2HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF1_OFFSET		(0x0534)
+
+/* PDP, VID2HCOEFF1, VID2HCOEFF1
+*/
+#define PDP_VID2HCOEFF1_VID2HCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF1_VID2HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF1_VID2HCOEFF1_SHIFT		(0)
+#define PDP_VID2HCOEFF1_VID2HCOEFF1_LENGTH		(32)
+#define PDP_VID2HCOEFF1_VID2HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF2_OFFSET		(0x0538)
+
+/* PDP, VID2HCOEFF2, VID2HCOEFF2
+*/
+#define PDP_VID2HCOEFF2_VID2HCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF2_VID2HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF2_VID2HCOEFF2_SHIFT		(0)
+#define PDP_VID2HCOEFF2_VID2HCOEFF2_LENGTH		(32)
+#define PDP_VID2HCOEFF2_VID2HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF3_OFFSET		(0x053C)
+
+/* PDP, VID2HCOEFF3, VID2HCOEFF3
+*/
+#define PDP_VID2HCOEFF3_VID2HCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF3_VID2HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF3_VID2HCOEFF3_SHIFT		(0)
+#define PDP_VID2HCOEFF3_VID2HCOEFF3_LENGTH		(32)
+#define PDP_VID2HCOEFF3_VID2HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF4_OFFSET		(0x0540)
+
+/* PDP, VID2HCOEFF4, VID2HCOEFF4
+*/
+#define PDP_VID2HCOEFF4_VID2HCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF4_VID2HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF4_VID2HCOEFF4_SHIFT		(0)
+#define PDP_VID2HCOEFF4_VID2HCOEFF4_LENGTH		(32)
+#define PDP_VID2HCOEFF4_VID2HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF5_OFFSET		(0x0544)
+
+/* PDP, VID2HCOEFF5, VID2HCOEFF5
+*/
+#define PDP_VID2HCOEFF5_VID2HCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF5_VID2HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF5_VID2HCOEFF5_SHIFT		(0)
+#define PDP_VID2HCOEFF5_VID2HCOEFF5_LENGTH		(32)
+#define PDP_VID2HCOEFF5_VID2HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF6_OFFSET		(0x0548)
+
+/* PDP, VID2HCOEFF6, VID2HCOEFF6
+*/
+#define PDP_VID2HCOEFF6_VID2HCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF6_VID2HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF6_VID2HCOEFF6_SHIFT		(0)
+#define PDP_VID2HCOEFF6_VID2HCOEFF6_LENGTH		(32)
+#define PDP_VID2HCOEFF6_VID2HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF7_OFFSET		(0x054C)
+
+/* PDP, VID2HCOEFF7, VID2HCOEFF7
+*/
+#define PDP_VID2HCOEFF7_VID2HCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF7_VID2HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF7_VID2HCOEFF7_SHIFT		(0)
+#define PDP_VID2HCOEFF7_VID2HCOEFF7_LENGTH		(32)
+#define PDP_VID2HCOEFF7_VID2HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF8_OFFSET		(0x0550)
+
+/* PDP, VID2HCOEFF8, VID2HCOEFF8
+*/
+#define PDP_VID2HCOEFF8_VID2HCOEFF8_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF8_VID2HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF8_VID2HCOEFF8_SHIFT		(0)
+#define PDP_VID2HCOEFF8_VID2HCOEFF8_LENGTH		(32)
+#define PDP_VID2HCOEFF8_VID2HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF9_OFFSET		(0x0554)
+
+/* PDP, VID2HCOEFF9, VID2HCOEFF9
+*/
+#define PDP_VID2HCOEFF9_VID2HCOEFF9_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF9_VID2HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF9_VID2HCOEFF9_SHIFT		(0)
+#define PDP_VID2HCOEFF9_VID2HCOEFF9_LENGTH		(32)
+#define PDP_VID2HCOEFF9_VID2HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF10_OFFSET		(0x0558)
+
+/* PDP, VID2HCOEFF10, VID2HCOEFF10
+*/
+#define PDP_VID2HCOEFF10_VID2HCOEFF10_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF10_VID2HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF10_VID2HCOEFF10_SHIFT		(0)
+#define PDP_VID2HCOEFF10_VID2HCOEFF10_LENGTH		(32)
+#define PDP_VID2HCOEFF10_VID2HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF11_OFFSET		(0x055C)
+
+/* PDP, VID2HCOEFF11, VID2HCOEFF11
+*/
+#define PDP_VID2HCOEFF11_VID2HCOEFF11_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF11_VID2HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF11_VID2HCOEFF11_SHIFT		(0)
+#define PDP_VID2HCOEFF11_VID2HCOEFF11_LENGTH		(32)
+#define PDP_VID2HCOEFF11_VID2HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF12_OFFSET		(0x0560)
+
+/* PDP, VID2HCOEFF12, VID2HCOEFF12
+*/
+#define PDP_VID2HCOEFF12_VID2HCOEFF12_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF12_VID2HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF12_VID2HCOEFF12_SHIFT		(0)
+#define PDP_VID2HCOEFF12_VID2HCOEFF12_LENGTH		(32)
+#define PDP_VID2HCOEFF12_VID2HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF13_OFFSET		(0x0564)
+
+/* PDP, VID2HCOEFF13, VID2HCOEFF13
+*/
+#define PDP_VID2HCOEFF13_VID2HCOEFF13_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF13_VID2HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF13_VID2HCOEFF13_SHIFT		(0)
+#define PDP_VID2HCOEFF13_VID2HCOEFF13_LENGTH		(32)
+#define PDP_VID2HCOEFF13_VID2HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF14_OFFSET		(0x0568)
+
+/* PDP, VID2HCOEFF14, VID2HCOEFF14
+*/
+#define PDP_VID2HCOEFF14_VID2HCOEFF14_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF14_VID2HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF14_VID2HCOEFF14_SHIFT		(0)
+#define PDP_VID2HCOEFF14_VID2HCOEFF14_LENGTH		(32)
+#define PDP_VID2HCOEFF14_VID2HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF15_OFFSET		(0x056C)
+
+/* PDP, VID2HCOEFF15, VID2HCOEFF15
+*/
+#define PDP_VID2HCOEFF15_VID2HCOEFF15_MASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF15_VID2HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID2HCOEFF15_VID2HCOEFF15_SHIFT		(0)
+#define PDP_VID2HCOEFF15_VID2HCOEFF15_LENGTH		(32)
+#define PDP_VID2HCOEFF15_VID2HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2HCOEFF16_OFFSET		(0x0570)
+
+/* PDP, VID2HCOEFF16, VID2HCOEFF16
+*/
+#define PDP_VID2HCOEFF16_VID2HCOEFF16_MASK		(0x000000FF)
+#define PDP_VID2HCOEFF16_VID2HCOEFF16_LSBMASK		(0x000000FF)
+#define PDP_VID2HCOEFF16_VID2HCOEFF16_SHIFT		(0)
+#define PDP_VID2HCOEFF16_VID2HCOEFF16_LENGTH		(8)
+#define PDP_VID2HCOEFF16_VID2HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2SCALESIZE_OFFSET		(0x0574)
+
+/* PDP, VID2SCALESIZE, VID2SCALEWIDTH
+*/
+#define PDP_VID2SCALESIZE_VID2SCALEWIDTH_MASK		(0x0FFF0000)
+#define PDP_VID2SCALESIZE_VID2SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID2SCALESIZE_VID2SCALEWIDTH_SHIFT		(16)
+#define PDP_VID2SCALESIZE_VID2SCALEWIDTH_LENGTH		(12)
+#define PDP_VID2SCALESIZE_VID2SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2SCALESIZE, VID2SCALEHEIGHT
+*/
+#define PDP_VID2SCALESIZE_VID2SCALEHEIGHT_MASK		(0x00000FFF)
+#define PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SHIFT		(0)
+#define PDP_VID2SCALESIZE_VID2SCALEHEIGHT_LENGTH		(12)
+#define PDP_VID2SCALESIZE_VID2SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3SCALECTRL_OFFSET		(0x0578)
+
+/* PDP, VID3SCALECTRL, VID3HSCALEBP
+*/
+#define PDP_VID3SCALECTRL_VID3HSCALEBP_MASK		(0x80000000)
+#define PDP_VID3SCALECTRL_VID3HSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID3SCALECTRL_VID3HSCALEBP_SHIFT		(31)
+#define PDP_VID3SCALECTRL_VID3HSCALEBP_LENGTH		(1)
+#define PDP_VID3SCALECTRL_VID3HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSCALEBP
+*/
+#define PDP_VID3SCALECTRL_VID3VSCALEBP_MASK		(0x40000000)
+#define PDP_VID3SCALECTRL_VID3VSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID3SCALECTRL_VID3VSCALEBP_SHIFT		(30)
+#define PDP_VID3SCALECTRL_VID3VSCALEBP_LENGTH		(1)
+#define PDP_VID3SCALECTRL_VID3VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3HSBEFOREVS
+*/
+#define PDP_VID3SCALECTRL_VID3HSBEFOREVS_MASK		(0x20000000)
+#define PDP_VID3SCALECTRL_VID3HSBEFOREVS_LSBMASK		(0x00000001)
+#define PDP_VID3SCALECTRL_VID3HSBEFOREVS_SHIFT		(29)
+#define PDP_VID3SCALECTRL_VID3HSBEFOREVS_LENGTH		(1)
+#define PDP_VID3SCALECTRL_VID3HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VSURUNCTRL
+*/
+#define PDP_VID3SCALECTRL_VID3VSURUNCTRL_MASK		(0x08000000)
+#define PDP_VID3SCALECTRL_VID3VSURUNCTRL_LSBMASK		(0x00000001)
+#define PDP_VID3SCALECTRL_VID3VSURUNCTRL_SHIFT		(27)
+#define PDP_VID3SCALECTRL_VID3VSURUNCTRL_LENGTH		(1)
+#define PDP_VID3SCALECTRL_VID3VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3PAN_EN
+*/
+#define PDP_VID3SCALECTRL_VID3PAN_EN_MASK		(0x00040000)
+#define PDP_VID3SCALECTRL_VID3PAN_EN_LSBMASK		(0x00000001)
+#define PDP_VID3SCALECTRL_VID3PAN_EN_SHIFT		(18)
+#define PDP_VID3SCALECTRL_VID3PAN_EN_LENGTH		(1)
+#define PDP_VID3SCALECTRL_VID3PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VORDER
+*/
+#define PDP_VID3SCALECTRL_VID3VORDER_MASK		(0x00030000)
+#define PDP_VID3SCALECTRL_VID3VORDER_LSBMASK		(0x00000003)
+#define PDP_VID3SCALECTRL_VID3VORDER_SHIFT		(16)
+#define PDP_VID3SCALECTRL_VID3VORDER_LENGTH		(2)
+#define PDP_VID3SCALECTRL_VID3VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALECTRL, VID3VPITCH
+*/
+#define PDP_VID3SCALECTRL_VID3VPITCH_MASK		(0x0000FFFF)
+#define PDP_VID3SCALECTRL_VID3VPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID3SCALECTRL_VID3VPITCH_SHIFT		(0)
+#define PDP_VID3SCALECTRL_VID3VPITCH_LENGTH		(16)
+#define PDP_VID3SCALECTRL_VID3VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VSINIT_OFFSET		(0x057C)
+
+/* PDP, VID3VSINIT, VID3VINITIAL1
+*/
+#define PDP_VID3VSINIT_VID3VINITIAL1_MASK		(0xFFFF0000)
+#define PDP_VID3VSINIT_VID3VINITIAL1_LSBMASK		(0x0000FFFF)
+#define PDP_VID3VSINIT_VID3VINITIAL1_SHIFT		(16)
+#define PDP_VID3VSINIT_VID3VINITIAL1_LENGTH		(16)
+#define PDP_VID3VSINIT_VID3VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3VSINIT, VID3VINITIAL0
+*/
+#define PDP_VID3VSINIT_VID3VINITIAL0_MASK		(0x0000FFFF)
+#define PDP_VID3VSINIT_VID3VINITIAL0_LSBMASK		(0x0000FFFF)
+#define PDP_VID3VSINIT_VID3VINITIAL0_SHIFT		(0)
+#define PDP_VID3VSINIT_VID3VINITIAL0_LENGTH		(16)
+#define PDP_VID3VSINIT_VID3VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF0_OFFSET		(0x0580)
+
+/* PDP, VID3VCOEFF0, VID3VCOEFF0
+*/
+#define PDP_VID3VCOEFF0_VID3VCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF0_VID3VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF0_VID3VCOEFF0_SHIFT		(0)
+#define PDP_VID3VCOEFF0_VID3VCOEFF0_LENGTH		(32)
+#define PDP_VID3VCOEFF0_VID3VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF1_OFFSET		(0x0584)
+
+/* PDP, VID3VCOEFF1, VID3VCOEFF1
+*/
+#define PDP_VID3VCOEFF1_VID3VCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF1_VID3VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF1_VID3VCOEFF1_SHIFT		(0)
+#define PDP_VID3VCOEFF1_VID3VCOEFF1_LENGTH		(32)
+#define PDP_VID3VCOEFF1_VID3VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF2_OFFSET		(0x0588)
+
+/* PDP, VID3VCOEFF2, VID3VCOEFF2
+*/
+#define PDP_VID3VCOEFF2_VID3VCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF2_VID3VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF2_VID3VCOEFF2_SHIFT		(0)
+#define PDP_VID3VCOEFF2_VID3VCOEFF2_LENGTH		(32)
+#define PDP_VID3VCOEFF2_VID3VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF3_OFFSET		(0x058C)
+
+/* PDP, VID3VCOEFF3, VID3VCOEFF3
+*/
+#define PDP_VID3VCOEFF3_VID3VCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF3_VID3VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF3_VID3VCOEFF3_SHIFT		(0)
+#define PDP_VID3VCOEFF3_VID3VCOEFF3_LENGTH		(32)
+#define PDP_VID3VCOEFF3_VID3VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF4_OFFSET		(0x0590)
+
+/* PDP, VID3VCOEFF4, VID3VCOEFF4
+*/
+#define PDP_VID3VCOEFF4_VID3VCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF4_VID3VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF4_VID3VCOEFF4_SHIFT		(0)
+#define PDP_VID3VCOEFF4_VID3VCOEFF4_LENGTH		(32)
+#define PDP_VID3VCOEFF4_VID3VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF5_OFFSET		(0x0594)
+
+/* PDP, VID3VCOEFF5, VID3VCOEFF5
+*/
+#define PDP_VID3VCOEFF5_VID3VCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF5_VID3VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF5_VID3VCOEFF5_SHIFT		(0)
+#define PDP_VID3VCOEFF5_VID3VCOEFF5_LENGTH		(32)
+#define PDP_VID3VCOEFF5_VID3VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF6_OFFSET		(0x0598)
+
+/* PDP, VID3VCOEFF6, VID3VCOEFF6
+*/
+#define PDP_VID3VCOEFF6_VID3VCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF6_VID3VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF6_VID3VCOEFF6_SHIFT		(0)
+#define PDP_VID3VCOEFF6_VID3VCOEFF6_LENGTH		(32)
+#define PDP_VID3VCOEFF6_VID3VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF7_OFFSET		(0x059C)
+
+/* PDP, VID3VCOEFF7, VID3VCOEFF7
+*/
+#define PDP_VID3VCOEFF7_VID3VCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF7_VID3VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3VCOEFF7_VID3VCOEFF7_SHIFT		(0)
+#define PDP_VID3VCOEFF7_VID3VCOEFF7_LENGTH		(32)
+#define PDP_VID3VCOEFF7_VID3VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3VCOEFF8_OFFSET		(0x05A0)
+
+/* PDP, VID3VCOEFF8, VID3VCOEFF8
+*/
+#define PDP_VID3VCOEFF8_VID3VCOEFF8_MASK		(0x000000FF)
+#define PDP_VID3VCOEFF8_VID3VCOEFF8_LSBMASK		(0x000000FF)
+#define PDP_VID3VCOEFF8_VID3VCOEFF8_SHIFT		(0)
+#define PDP_VID3VCOEFF8_VID3VCOEFF8_LENGTH		(8)
+#define PDP_VID3VCOEFF8_VID3VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HSINIT_OFFSET		(0x05A4)
+
+/* PDP, VID3HSINIT, VID3HINITIAL
+*/
+#define PDP_VID3HSINIT_VID3HINITIAL_MASK		(0xFFFF0000)
+#define PDP_VID3HSINIT_VID3HINITIAL_LSBMASK		(0x0000FFFF)
+#define PDP_VID3HSINIT_VID3HINITIAL_SHIFT		(16)
+#define PDP_VID3HSINIT_VID3HINITIAL_LENGTH		(16)
+#define PDP_VID3HSINIT_VID3HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3HSINIT, VID3HPITCH
+*/
+#define PDP_VID3HSINIT_VID3HPITCH_MASK		(0x0000FFFF)
+#define PDP_VID3HSINIT_VID3HPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID3HSINIT_VID3HPITCH_SHIFT		(0)
+#define PDP_VID3HSINIT_VID3HPITCH_LENGTH		(16)
+#define PDP_VID3HSINIT_VID3HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF0_OFFSET		(0x05A8)
+
+/* PDP, VID3HCOEFF0, VID3HCOEFF0
+*/
+#define PDP_VID3HCOEFF0_VID3HCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF0_VID3HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF0_VID3HCOEFF0_SHIFT		(0)
+#define PDP_VID3HCOEFF0_VID3HCOEFF0_LENGTH		(32)
+#define PDP_VID3HCOEFF0_VID3HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF1_OFFSET		(0x05AC)
+
+/* PDP, VID3HCOEFF1, VID3HCOEFF1
+*/
+#define PDP_VID3HCOEFF1_VID3HCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF1_VID3HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF1_VID3HCOEFF1_SHIFT		(0)
+#define PDP_VID3HCOEFF1_VID3HCOEFF1_LENGTH		(32)
+#define PDP_VID3HCOEFF1_VID3HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF2_OFFSET		(0x05B0)
+
+/* PDP, VID3HCOEFF2, VID3HCOEFF2
+*/
+#define PDP_VID3HCOEFF2_VID3HCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF2_VID3HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF2_VID3HCOEFF2_SHIFT		(0)
+#define PDP_VID3HCOEFF2_VID3HCOEFF2_LENGTH		(32)
+#define PDP_VID3HCOEFF2_VID3HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF3_OFFSET		(0x05B4)
+
+/* PDP, VID3HCOEFF3, VID3HCOEFF3
+*/
+#define PDP_VID3HCOEFF3_VID3HCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF3_VID3HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF3_VID3HCOEFF3_SHIFT		(0)
+#define PDP_VID3HCOEFF3_VID3HCOEFF3_LENGTH		(32)
+#define PDP_VID3HCOEFF3_VID3HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF4_OFFSET		(0x05B8)
+
+/* PDP, VID3HCOEFF4, VID3HCOEFF4
+*/
+#define PDP_VID3HCOEFF4_VID3HCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF4_VID3HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF4_VID3HCOEFF4_SHIFT		(0)
+#define PDP_VID3HCOEFF4_VID3HCOEFF4_LENGTH		(32)
+#define PDP_VID3HCOEFF4_VID3HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF5_OFFSET		(0x05BC)
+
+/* PDP, VID3HCOEFF5, VID3HCOEFF5
+*/
+#define PDP_VID3HCOEFF5_VID3HCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF5_VID3HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF5_VID3HCOEFF5_SHIFT		(0)
+#define PDP_VID3HCOEFF5_VID3HCOEFF5_LENGTH		(32)
+#define PDP_VID3HCOEFF5_VID3HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF6_OFFSET		(0x05C0)
+
+/* PDP, VID3HCOEFF6, VID3HCOEFF6
+*/
+#define PDP_VID3HCOEFF6_VID3HCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF6_VID3HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF6_VID3HCOEFF6_SHIFT		(0)
+#define PDP_VID3HCOEFF6_VID3HCOEFF6_LENGTH		(32)
+#define PDP_VID3HCOEFF6_VID3HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF7_OFFSET		(0x05C4)
+
+/* PDP, VID3HCOEFF7, VID3HCOEFF7
+*/
+#define PDP_VID3HCOEFF7_VID3HCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF7_VID3HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF7_VID3HCOEFF7_SHIFT		(0)
+#define PDP_VID3HCOEFF7_VID3HCOEFF7_LENGTH		(32)
+#define PDP_VID3HCOEFF7_VID3HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF8_OFFSET		(0x05C8)
+
+/* PDP, VID3HCOEFF8, VID3HCOEFF8
+*/
+#define PDP_VID3HCOEFF8_VID3HCOEFF8_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF8_VID3HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF8_VID3HCOEFF8_SHIFT		(0)
+#define PDP_VID3HCOEFF8_VID3HCOEFF8_LENGTH		(32)
+#define PDP_VID3HCOEFF8_VID3HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF9_OFFSET		(0x05CC)
+
+/* PDP, VID3HCOEFF9, VID3HCOEFF9
+*/
+#define PDP_VID3HCOEFF9_VID3HCOEFF9_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF9_VID3HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF9_VID3HCOEFF9_SHIFT		(0)
+#define PDP_VID3HCOEFF9_VID3HCOEFF9_LENGTH		(32)
+#define PDP_VID3HCOEFF9_VID3HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF10_OFFSET		(0x05D0)
+
+/* PDP, VID3HCOEFF10, VID3HCOEFF10
+*/
+#define PDP_VID3HCOEFF10_VID3HCOEFF10_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF10_VID3HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF10_VID3HCOEFF10_SHIFT		(0)
+#define PDP_VID3HCOEFF10_VID3HCOEFF10_LENGTH		(32)
+#define PDP_VID3HCOEFF10_VID3HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF11_OFFSET		(0x05D4)
+
+/* PDP, VID3HCOEFF11, VID3HCOEFF11
+*/
+#define PDP_VID3HCOEFF11_VID3HCOEFF11_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF11_VID3HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF11_VID3HCOEFF11_SHIFT		(0)
+#define PDP_VID3HCOEFF11_VID3HCOEFF11_LENGTH		(32)
+#define PDP_VID3HCOEFF11_VID3HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF12_OFFSET		(0x05D8)
+
+/* PDP, VID3HCOEFF12, VID3HCOEFF12
+*/
+#define PDP_VID3HCOEFF12_VID3HCOEFF12_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF12_VID3HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF12_VID3HCOEFF12_SHIFT		(0)
+#define PDP_VID3HCOEFF12_VID3HCOEFF12_LENGTH		(32)
+#define PDP_VID3HCOEFF12_VID3HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF13_OFFSET		(0x05DC)
+
+/* PDP, VID3HCOEFF13, VID3HCOEFF13
+*/
+#define PDP_VID3HCOEFF13_VID3HCOEFF13_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF13_VID3HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF13_VID3HCOEFF13_SHIFT		(0)
+#define PDP_VID3HCOEFF13_VID3HCOEFF13_LENGTH		(32)
+#define PDP_VID3HCOEFF13_VID3HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF14_OFFSET		(0x05E0)
+
+/* PDP, VID3HCOEFF14, VID3HCOEFF14
+*/
+#define PDP_VID3HCOEFF14_VID3HCOEFF14_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF14_VID3HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF14_VID3HCOEFF14_SHIFT		(0)
+#define PDP_VID3HCOEFF14_VID3HCOEFF14_LENGTH		(32)
+#define PDP_VID3HCOEFF14_VID3HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF15_OFFSET		(0x05E4)
+
+/* PDP, VID3HCOEFF15, VID3HCOEFF15
+*/
+#define PDP_VID3HCOEFF15_VID3HCOEFF15_MASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF15_VID3HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID3HCOEFF15_VID3HCOEFF15_SHIFT		(0)
+#define PDP_VID3HCOEFF15_VID3HCOEFF15_LENGTH		(32)
+#define PDP_VID3HCOEFF15_VID3HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3HCOEFF16_OFFSET		(0x05E8)
+
+/* PDP, VID3HCOEFF16, VID3HCOEFF16
+*/
+#define PDP_VID3HCOEFF16_VID3HCOEFF16_MASK		(0x000000FF)
+#define PDP_VID3HCOEFF16_VID3HCOEFF16_LSBMASK		(0x000000FF)
+#define PDP_VID3HCOEFF16_VID3HCOEFF16_SHIFT		(0)
+#define PDP_VID3HCOEFF16_VID3HCOEFF16_LENGTH		(8)
+#define PDP_VID3HCOEFF16_VID3HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3SCALESIZE_OFFSET		(0x05EC)
+
+/* PDP, VID3SCALESIZE, VID3SCALEWIDTH
+*/
+#define PDP_VID3SCALESIZE_VID3SCALEWIDTH_MASK		(0x0FFF0000)
+#define PDP_VID3SCALESIZE_VID3SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID3SCALESIZE_VID3SCALEWIDTH_SHIFT		(16)
+#define PDP_VID3SCALESIZE_VID3SCALEWIDTH_LENGTH		(12)
+#define PDP_VID3SCALESIZE_VID3SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3SCALESIZE, VID3SCALEHEIGHT
+*/
+#define PDP_VID3SCALESIZE_VID3SCALEHEIGHT_MASK		(0x00000FFF)
+#define PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SHIFT		(0)
+#define PDP_VID3SCALESIZE_VID3SCALEHEIGHT_LENGTH		(12)
+#define PDP_VID3SCALESIZE_VID3SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4SCALECTRL_OFFSET		(0x05F0)
+
+/* PDP, VID4SCALECTRL, VID4HSCALEBP
+*/
+#define PDP_VID4SCALECTRL_VID4HSCALEBP_MASK		(0x80000000)
+#define PDP_VID4SCALECTRL_VID4HSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID4SCALECTRL_VID4HSCALEBP_SHIFT		(31)
+#define PDP_VID4SCALECTRL_VID4HSCALEBP_LENGTH		(1)
+#define PDP_VID4SCALECTRL_VID4HSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSCALEBP
+*/
+#define PDP_VID4SCALECTRL_VID4VSCALEBP_MASK		(0x40000000)
+#define PDP_VID4SCALECTRL_VID4VSCALEBP_LSBMASK		(0x00000001)
+#define PDP_VID4SCALECTRL_VID4VSCALEBP_SHIFT		(30)
+#define PDP_VID4SCALECTRL_VID4VSCALEBP_LENGTH		(1)
+#define PDP_VID4SCALECTRL_VID4VSCALEBP_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4HSBEFOREVS
+*/
+#define PDP_VID4SCALECTRL_VID4HSBEFOREVS_MASK		(0x20000000)
+#define PDP_VID4SCALECTRL_VID4HSBEFOREVS_LSBMASK		(0x00000001)
+#define PDP_VID4SCALECTRL_VID4HSBEFOREVS_SHIFT		(29)
+#define PDP_VID4SCALECTRL_VID4HSBEFOREVS_LENGTH		(1)
+#define PDP_VID4SCALECTRL_VID4HSBEFOREVS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VSURUNCTRL
+*/
+#define PDP_VID4SCALECTRL_VID4VSURUNCTRL_MASK		(0x08000000)
+#define PDP_VID4SCALECTRL_VID4VSURUNCTRL_LSBMASK		(0x00000001)
+#define PDP_VID4SCALECTRL_VID4VSURUNCTRL_SHIFT		(27)
+#define PDP_VID4SCALECTRL_VID4VSURUNCTRL_LENGTH		(1)
+#define PDP_VID4SCALECTRL_VID4VSURUNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4PAN_EN
+*/
+#define PDP_VID4SCALECTRL_VID4PAN_EN_MASK		(0x00040000)
+#define PDP_VID4SCALECTRL_VID4PAN_EN_LSBMASK		(0x00000001)
+#define PDP_VID4SCALECTRL_VID4PAN_EN_SHIFT		(18)
+#define PDP_VID4SCALECTRL_VID4PAN_EN_LENGTH		(1)
+#define PDP_VID4SCALECTRL_VID4PAN_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VORDER
+*/
+#define PDP_VID4SCALECTRL_VID4VORDER_MASK		(0x00030000)
+#define PDP_VID4SCALECTRL_VID4VORDER_LSBMASK		(0x00000003)
+#define PDP_VID4SCALECTRL_VID4VORDER_SHIFT		(16)
+#define PDP_VID4SCALECTRL_VID4VORDER_LENGTH		(2)
+#define PDP_VID4SCALECTRL_VID4VORDER_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALECTRL, VID4VPITCH
+*/
+#define PDP_VID4SCALECTRL_VID4VPITCH_MASK		(0x0000FFFF)
+#define PDP_VID4SCALECTRL_VID4VPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID4SCALECTRL_VID4VPITCH_SHIFT		(0)
+#define PDP_VID4SCALECTRL_VID4VPITCH_LENGTH		(16)
+#define PDP_VID4SCALECTRL_VID4VPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VSINIT_OFFSET		(0x05F4)
+
+/* PDP, VID4VSINIT, VID4VINITIAL1
+*/
+#define PDP_VID4VSINIT_VID4VINITIAL1_MASK		(0xFFFF0000)
+#define PDP_VID4VSINIT_VID4VINITIAL1_LSBMASK		(0x0000FFFF)
+#define PDP_VID4VSINIT_VID4VINITIAL1_SHIFT		(16)
+#define PDP_VID4VSINIT_VID4VINITIAL1_LENGTH		(16)
+#define PDP_VID4VSINIT_VID4VINITIAL1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4VSINIT, VID4VINITIAL0
+*/
+#define PDP_VID4VSINIT_VID4VINITIAL0_MASK		(0x0000FFFF)
+#define PDP_VID4VSINIT_VID4VINITIAL0_LSBMASK		(0x0000FFFF)
+#define PDP_VID4VSINIT_VID4VINITIAL0_SHIFT		(0)
+#define PDP_VID4VSINIT_VID4VINITIAL0_LENGTH		(16)
+#define PDP_VID4VSINIT_VID4VINITIAL0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF0_OFFSET		(0x05F8)
+
+/* PDP, VID4VCOEFF0, VID4VCOEFF0
+*/
+#define PDP_VID4VCOEFF0_VID4VCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF0_VID4VCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF0_VID4VCOEFF0_SHIFT		(0)
+#define PDP_VID4VCOEFF0_VID4VCOEFF0_LENGTH		(32)
+#define PDP_VID4VCOEFF0_VID4VCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF1_OFFSET		(0x05FC)
+
+/* PDP, VID4VCOEFF1, VID4VCOEFF1
+*/
+#define PDP_VID4VCOEFF1_VID4VCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF1_VID4VCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF1_VID4VCOEFF1_SHIFT		(0)
+#define PDP_VID4VCOEFF1_VID4VCOEFF1_LENGTH		(32)
+#define PDP_VID4VCOEFF1_VID4VCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF2_OFFSET		(0x0600)
+
+/* PDP, VID4VCOEFF2, VID4VCOEFF2
+*/
+#define PDP_VID4VCOEFF2_VID4VCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF2_VID4VCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF2_VID4VCOEFF2_SHIFT		(0)
+#define PDP_VID4VCOEFF2_VID4VCOEFF2_LENGTH		(32)
+#define PDP_VID4VCOEFF2_VID4VCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF3_OFFSET		(0x0604)
+
+/* PDP, VID4VCOEFF3, VID4VCOEFF3
+*/
+#define PDP_VID4VCOEFF3_VID4VCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF3_VID4VCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF3_VID4VCOEFF3_SHIFT		(0)
+#define PDP_VID4VCOEFF3_VID4VCOEFF3_LENGTH		(32)
+#define PDP_VID4VCOEFF3_VID4VCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF4_OFFSET		(0x0608)
+
+/* PDP, VID4VCOEFF4, VID4VCOEFF4
+*/
+#define PDP_VID4VCOEFF4_VID4VCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF4_VID4VCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF4_VID4VCOEFF4_SHIFT		(0)
+#define PDP_VID4VCOEFF4_VID4VCOEFF4_LENGTH		(32)
+#define PDP_VID4VCOEFF4_VID4VCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF5_OFFSET		(0x060C)
+
+/* PDP, VID4VCOEFF5, VID4VCOEFF5
+*/
+#define PDP_VID4VCOEFF5_VID4VCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF5_VID4VCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF5_VID4VCOEFF5_SHIFT		(0)
+#define PDP_VID4VCOEFF5_VID4VCOEFF5_LENGTH		(32)
+#define PDP_VID4VCOEFF5_VID4VCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF6_OFFSET		(0x0610)
+
+/* PDP, VID4VCOEFF6, VID4VCOEFF6
+*/
+#define PDP_VID4VCOEFF6_VID4VCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF6_VID4VCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF6_VID4VCOEFF6_SHIFT		(0)
+#define PDP_VID4VCOEFF6_VID4VCOEFF6_LENGTH		(32)
+#define PDP_VID4VCOEFF6_VID4VCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF7_OFFSET		(0x0614)
+
+/* PDP, VID4VCOEFF7, VID4VCOEFF7
+*/
+#define PDP_VID4VCOEFF7_VID4VCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF7_VID4VCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4VCOEFF7_VID4VCOEFF7_SHIFT		(0)
+#define PDP_VID4VCOEFF7_VID4VCOEFF7_LENGTH		(32)
+#define PDP_VID4VCOEFF7_VID4VCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4VCOEFF8_OFFSET		(0x0618)
+
+/* PDP, VID4VCOEFF8, VID4VCOEFF8
+*/
+#define PDP_VID4VCOEFF8_VID4VCOEFF8_MASK		(0x000000FF)
+#define PDP_VID4VCOEFF8_VID4VCOEFF8_LSBMASK		(0x000000FF)
+#define PDP_VID4VCOEFF8_VID4VCOEFF8_SHIFT		(0)
+#define PDP_VID4VCOEFF8_VID4VCOEFF8_LENGTH		(8)
+#define PDP_VID4VCOEFF8_VID4VCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HSINIT_OFFSET		(0x061C)
+
+/* PDP, VID4HSINIT, VID4HINITIAL
+*/
+#define PDP_VID4HSINIT_VID4HINITIAL_MASK		(0xFFFF0000)
+#define PDP_VID4HSINIT_VID4HINITIAL_LSBMASK		(0x0000FFFF)
+#define PDP_VID4HSINIT_VID4HINITIAL_SHIFT		(16)
+#define PDP_VID4HSINIT_VID4HINITIAL_LENGTH		(16)
+#define PDP_VID4HSINIT_VID4HINITIAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4HSINIT, VID4HPITCH
+*/
+#define PDP_VID4HSINIT_VID4HPITCH_MASK		(0x0000FFFF)
+#define PDP_VID4HSINIT_VID4HPITCH_LSBMASK		(0x0000FFFF)
+#define PDP_VID4HSINIT_VID4HPITCH_SHIFT		(0)
+#define PDP_VID4HSINIT_VID4HPITCH_LENGTH		(16)
+#define PDP_VID4HSINIT_VID4HPITCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF0_OFFSET		(0x0620)
+
+/* PDP, VID4HCOEFF0, VID4HCOEFF0
+*/
+#define PDP_VID4HCOEFF0_VID4HCOEFF0_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF0_VID4HCOEFF0_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF0_VID4HCOEFF0_SHIFT		(0)
+#define PDP_VID4HCOEFF0_VID4HCOEFF0_LENGTH		(32)
+#define PDP_VID4HCOEFF0_VID4HCOEFF0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF1_OFFSET		(0x0624)
+
+/* PDP, VID4HCOEFF1, VID4HCOEFF1
+*/
+#define PDP_VID4HCOEFF1_VID4HCOEFF1_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF1_VID4HCOEFF1_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF1_VID4HCOEFF1_SHIFT		(0)
+#define PDP_VID4HCOEFF1_VID4HCOEFF1_LENGTH		(32)
+#define PDP_VID4HCOEFF1_VID4HCOEFF1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF2_OFFSET		(0x0628)
+
+/* PDP, VID4HCOEFF2, VID4HCOEFF2
+*/
+#define PDP_VID4HCOEFF2_VID4HCOEFF2_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF2_VID4HCOEFF2_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF2_VID4HCOEFF2_SHIFT		(0)
+#define PDP_VID4HCOEFF2_VID4HCOEFF2_LENGTH		(32)
+#define PDP_VID4HCOEFF2_VID4HCOEFF2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF3_OFFSET		(0x062C)
+
+/* PDP, VID4HCOEFF3, VID4HCOEFF3
+*/
+#define PDP_VID4HCOEFF3_VID4HCOEFF3_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF3_VID4HCOEFF3_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF3_VID4HCOEFF3_SHIFT		(0)
+#define PDP_VID4HCOEFF3_VID4HCOEFF3_LENGTH		(32)
+#define PDP_VID4HCOEFF3_VID4HCOEFF3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF4_OFFSET		(0x0630)
+
+/* PDP, VID4HCOEFF4, VID4HCOEFF4
+*/
+#define PDP_VID4HCOEFF4_VID4HCOEFF4_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF4_VID4HCOEFF4_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF4_VID4HCOEFF4_SHIFT		(0)
+#define PDP_VID4HCOEFF4_VID4HCOEFF4_LENGTH		(32)
+#define PDP_VID4HCOEFF4_VID4HCOEFF4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF5_OFFSET		(0x0634)
+
+/* PDP, VID4HCOEFF5, VID4HCOEFF5
+*/
+#define PDP_VID4HCOEFF5_VID4HCOEFF5_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF5_VID4HCOEFF5_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF5_VID4HCOEFF5_SHIFT		(0)
+#define PDP_VID4HCOEFF5_VID4HCOEFF5_LENGTH		(32)
+#define PDP_VID4HCOEFF5_VID4HCOEFF5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF6_OFFSET		(0x0638)
+
+/* PDP, VID4HCOEFF6, VID4HCOEFF6
+*/
+#define PDP_VID4HCOEFF6_VID4HCOEFF6_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF6_VID4HCOEFF6_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF6_VID4HCOEFF6_SHIFT		(0)
+#define PDP_VID4HCOEFF6_VID4HCOEFF6_LENGTH		(32)
+#define PDP_VID4HCOEFF6_VID4HCOEFF6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF7_OFFSET		(0x063C)
+
+/* PDP, VID4HCOEFF7, VID4HCOEFF7
+*/
+#define PDP_VID4HCOEFF7_VID4HCOEFF7_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF7_VID4HCOEFF7_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF7_VID4HCOEFF7_SHIFT		(0)
+#define PDP_VID4HCOEFF7_VID4HCOEFF7_LENGTH		(32)
+#define PDP_VID4HCOEFF7_VID4HCOEFF7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF8_OFFSET		(0x0640)
+
+/* PDP, VID4HCOEFF8, VID4HCOEFF8
+*/
+#define PDP_VID4HCOEFF8_VID4HCOEFF8_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF8_VID4HCOEFF8_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF8_VID4HCOEFF8_SHIFT		(0)
+#define PDP_VID4HCOEFF8_VID4HCOEFF8_LENGTH		(32)
+#define PDP_VID4HCOEFF8_VID4HCOEFF8_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF9_OFFSET		(0x0644)
+
+/* PDP, VID4HCOEFF9, VID4HCOEFF9
+*/
+#define PDP_VID4HCOEFF9_VID4HCOEFF9_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF9_VID4HCOEFF9_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF9_VID4HCOEFF9_SHIFT		(0)
+#define PDP_VID4HCOEFF9_VID4HCOEFF9_LENGTH		(32)
+#define PDP_VID4HCOEFF9_VID4HCOEFF9_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF10_OFFSET		(0x0648)
+
+/* PDP, VID4HCOEFF10, VID4HCOEFF10
+*/
+#define PDP_VID4HCOEFF10_VID4HCOEFF10_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF10_VID4HCOEFF10_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF10_VID4HCOEFF10_SHIFT		(0)
+#define PDP_VID4HCOEFF10_VID4HCOEFF10_LENGTH		(32)
+#define PDP_VID4HCOEFF10_VID4HCOEFF10_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF11_OFFSET		(0x064C)
+
+/* PDP, VID4HCOEFF11, VID4HCOEFF11
+*/
+#define PDP_VID4HCOEFF11_VID4HCOEFF11_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF11_VID4HCOEFF11_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF11_VID4HCOEFF11_SHIFT		(0)
+#define PDP_VID4HCOEFF11_VID4HCOEFF11_LENGTH		(32)
+#define PDP_VID4HCOEFF11_VID4HCOEFF11_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF12_OFFSET		(0x0650)
+
+/* PDP, VID4HCOEFF12, VID4HCOEFF12
+*/
+#define PDP_VID4HCOEFF12_VID4HCOEFF12_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF12_VID4HCOEFF12_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF12_VID4HCOEFF12_SHIFT		(0)
+#define PDP_VID4HCOEFF12_VID4HCOEFF12_LENGTH		(32)
+#define PDP_VID4HCOEFF12_VID4HCOEFF12_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF13_OFFSET		(0x0654)
+
+/* PDP, VID4HCOEFF13, VID4HCOEFF13
+*/
+#define PDP_VID4HCOEFF13_VID4HCOEFF13_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF13_VID4HCOEFF13_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF13_VID4HCOEFF13_SHIFT		(0)
+#define PDP_VID4HCOEFF13_VID4HCOEFF13_LENGTH		(32)
+#define PDP_VID4HCOEFF13_VID4HCOEFF13_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF14_OFFSET		(0x0658)
+
+/* PDP, VID4HCOEFF14, VID4HCOEFF14
+*/
+#define PDP_VID4HCOEFF14_VID4HCOEFF14_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF14_VID4HCOEFF14_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF14_VID4HCOEFF14_SHIFT		(0)
+#define PDP_VID4HCOEFF14_VID4HCOEFF14_LENGTH		(32)
+#define PDP_VID4HCOEFF14_VID4HCOEFF14_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF15_OFFSET		(0x065C)
+
+/* PDP, VID4HCOEFF15, VID4HCOEFF15
+*/
+#define PDP_VID4HCOEFF15_VID4HCOEFF15_MASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF15_VID4HCOEFF15_LSBMASK		(0xFFFFFFFF)
+#define PDP_VID4HCOEFF15_VID4HCOEFF15_SHIFT		(0)
+#define PDP_VID4HCOEFF15_VID4HCOEFF15_LENGTH		(32)
+#define PDP_VID4HCOEFF15_VID4HCOEFF15_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4HCOEFF16_OFFSET		(0x0660)
+
+/* PDP, VID4HCOEFF16, VID4HCOEFF16
+*/
+#define PDP_VID4HCOEFF16_VID4HCOEFF16_MASK		(0x000000FF)
+#define PDP_VID4HCOEFF16_VID4HCOEFF16_LSBMASK		(0x000000FF)
+#define PDP_VID4HCOEFF16_VID4HCOEFF16_SHIFT		(0)
+#define PDP_VID4HCOEFF16_VID4HCOEFF16_LENGTH		(8)
+#define PDP_VID4HCOEFF16_VID4HCOEFF16_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4SCALESIZE_OFFSET		(0x0664)
+
+/* PDP, VID4SCALESIZE, VID4SCALEWIDTH
+*/
+#define PDP_VID4SCALESIZE_VID4SCALEWIDTH_MASK		(0x0FFF0000)
+#define PDP_VID4SCALESIZE_VID4SCALEWIDTH_LSBMASK		(0x00000FFF)
+#define PDP_VID4SCALESIZE_VID4SCALEWIDTH_SHIFT		(16)
+#define PDP_VID4SCALESIZE_VID4SCALEWIDTH_LENGTH		(12)
+#define PDP_VID4SCALESIZE_VID4SCALEWIDTH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4SCALESIZE, VID4SCALEHEIGHT
+*/
+#define PDP_VID4SCALESIZE_VID4SCALEHEIGHT_MASK		(0x00000FFF)
+#define PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LSBMASK		(0x00000FFF)
+#define PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SHIFT		(0)
+#define PDP_VID4SCALESIZE_VID4SCALEHEIGHT_LENGTH		(12)
+#define PDP_VID4SCALESIZE_VID4SCALEHEIGHT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND0_OFFSET		(0x0668)
+
+/* PDP, PORTER_BLND0, BLND0BLENDTYPE
+*/
+#define PDP_PORTER_BLND0_BLND0BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND0_BLND0BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND0_BLND0BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND0_BLND0BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND0_BLND0BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND0, BLND0PORTERMODE
+*/
+#define PDP_PORTER_BLND0_BLND0PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND0_BLND0PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND0_BLND0PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND0_BLND0PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND0_BLND0PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND1_OFFSET		(0x066C)
+
+/* PDP, PORTER_BLND1, BLND1BLENDTYPE
+*/
+#define PDP_PORTER_BLND1_BLND1BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND1_BLND1BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND1_BLND1BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND1_BLND1BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND1_BLND1BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND1, BLND1PORTERMODE
+*/
+#define PDP_PORTER_BLND1_BLND1PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND1_BLND1PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND1_BLND1PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND1_BLND1PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND1_BLND1PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND2_OFFSET		(0x0670)
+
+/* PDP, PORTER_BLND2, BLND2BLENDTYPE
+*/
+#define PDP_PORTER_BLND2_BLND2BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND2_BLND2BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND2_BLND2BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND2_BLND2BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND2_BLND2BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND2, BLND2PORTERMODE
+*/
+#define PDP_PORTER_BLND2_BLND2PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND2_BLND2PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND2_BLND2PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND2_BLND2PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND2_BLND2PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND3_OFFSET		(0x0674)
+
+/* PDP, PORTER_BLND3, BLND3BLENDTYPE
+*/
+#define PDP_PORTER_BLND3_BLND3BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND3_BLND3BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND3_BLND3BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND3_BLND3BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND3_BLND3BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND3, BLND3PORTERMODE
+*/
+#define PDP_PORTER_BLND3_BLND3PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND3_BLND3PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND3_BLND3PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND3_BLND3PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND3_BLND3PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND4_OFFSET		(0x0678)
+
+/* PDP, PORTER_BLND4, BLND4BLENDTYPE
+*/
+#define PDP_PORTER_BLND4_BLND4BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND4_BLND4BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND4_BLND4BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND4_BLND4BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND4_BLND4BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND4, BLND4PORTERMODE
+*/
+#define PDP_PORTER_BLND4_BLND4PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND4_BLND4PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND4_BLND4PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND4_BLND4PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND4_BLND4PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND5_OFFSET		(0x067C)
+
+/* PDP, PORTER_BLND5, BLND5BLENDTYPE
+*/
+#define PDP_PORTER_BLND5_BLND5BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND5_BLND5BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND5_BLND5BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND5_BLND5BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND5_BLND5BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND5, BLND5PORTERMODE
+*/
+#define PDP_PORTER_BLND5_BLND5PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND5_BLND5PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND5_BLND5PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND5_BLND5PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND5_BLND5PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND6_OFFSET		(0x0680)
+
+/* PDP, PORTER_BLND6, BLND6BLENDTYPE
+*/
+#define PDP_PORTER_BLND6_BLND6BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND6_BLND6BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND6_BLND6BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND6_BLND6BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND6_BLND6BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND6, BLND6PORTERMODE
+*/
+#define PDP_PORTER_BLND6_BLND6PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND6_BLND6PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND6_BLND6PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND6_BLND6PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND6_BLND6PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PORTER_BLND7_OFFSET		(0x0684)
+
+/* PDP, PORTER_BLND7, BLND7BLENDTYPE
+*/
+#define PDP_PORTER_BLND7_BLND7BLENDTYPE_MASK		(0x00000010)
+#define PDP_PORTER_BLND7_BLND7BLENDTYPE_LSBMASK		(0x00000001)
+#define PDP_PORTER_BLND7_BLND7BLENDTYPE_SHIFT		(4)
+#define PDP_PORTER_BLND7_BLND7BLENDTYPE_LENGTH		(1)
+#define PDP_PORTER_BLND7_BLND7BLENDTYPE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PORTER_BLND7, BLND7PORTERMODE
+*/
+#define PDP_PORTER_BLND7_BLND7PORTERMODE_MASK		(0x0000000F)
+#define PDP_PORTER_BLND7_BLND7PORTERMODE_LSBMASK		(0x0000000F)
+#define PDP_PORTER_BLND7_BLND7PORTERMODE_SHIFT		(0)
+#define PDP_PORTER_BLND7_BLND7PORTERMODE_LENGTH		(4)
+#define PDP_PORTER_BLND7_BLND7PORTERMODE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06C8)
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_TRANS
+*/
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_ALPHA_TRANS_OPAQUE, VID1LUMAKEYALPHA_OPAQUE
+*/
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define PDP_VID1LUMAKEY_ALPHA_TRANS_OPAQUE_VID1LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06CC)
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMAX
+*/
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_MASK		(0x03FF0000)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LSBMASK		(0x000003FF)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SHIFT		(16)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_LENGTH		(10)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_LUMA_MAX_MIN, VID1LUMAKEYYMIN
+*/
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_MASK		(0x000003FF)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LSBMASK		(0x000003FF)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SHIFT		(0)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_LENGTH		(10)
+#define PDP_VID1LUMAKEY_LUMA_MAX_MIN_VID1LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1LUMAKEY_C_RG_OFFSET		(0x06D0)
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_R
+*/
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_MASK		(0x0FFF0000)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SHIFT		(16)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_LENGTH		(12)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_RG, VID1LUMAKEYC_G
+*/
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_MASK		(0x00000FFF)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SHIFT		(0)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_LENGTH		(12)
+#define PDP_VID1LUMAKEY_C_RG_VID1LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1LUMAKEY_C_B_OFFSET		(0x06D4)
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYALPHAMULT
+*/
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SHIFT		(29)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_LENGTH		(1)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYEN
+*/
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_MASK		(0x10000000)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SHIFT		(28)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_LENGTH		(1)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYOUTOFF
+*/
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LSBMASK		(0x000003FF)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SHIFT		(16)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_LENGTH		(10)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1LUMAKEY_C_B, VID1LUMAKEYC_B
+*/
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_MASK		(0x00000FFF)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SHIFT		(0)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_LENGTH		(12)
+#define PDP_VID1LUMAKEY_C_B_VID1LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06D8)
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_TRANS
+*/
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_ALPHA_TRANS_OPAQUE, VID2LUMAKEYALPHA_OPAQUE
+*/
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define PDP_VID2LUMAKEY_ALPHA_TRANS_OPAQUE_VID2LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06DC)
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMAX
+*/
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_MASK		(0x03FF0000)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LSBMASK		(0x000003FF)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SHIFT		(16)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_LENGTH		(10)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_LUMA_MAX_MIN, VID2LUMAKEYYMIN
+*/
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_MASK		(0x000003FF)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LSBMASK		(0x000003FF)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SHIFT		(0)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_LENGTH		(10)
+#define PDP_VID2LUMAKEY_LUMA_MAX_MIN_VID2LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2LUMAKEY_C_RG_OFFSET		(0x06E0)
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_R
+*/
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_MASK		(0x0FFF0000)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SHIFT		(16)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_LENGTH		(12)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_RG, VID2LUMAKEYC_G
+*/
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_MASK		(0x00000FFF)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SHIFT		(0)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_LENGTH		(12)
+#define PDP_VID2LUMAKEY_C_RG_VID2LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2LUMAKEY_C_B_OFFSET		(0x06E4)
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYALPHAMULT
+*/
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SHIFT		(29)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_LENGTH		(1)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYEN
+*/
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_MASK		(0x10000000)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SHIFT		(28)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_LENGTH		(1)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYOUTOFF
+*/
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LSBMASK		(0x000003FF)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SHIFT		(16)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_LENGTH		(10)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2LUMAKEY_C_B, VID2LUMAKEYC_B
+*/
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_MASK		(0x00000FFF)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SHIFT		(0)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_LENGTH		(12)
+#define PDP_VID2LUMAKEY_C_B_VID2LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06E8)
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_TRANS
+*/
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_ALPHA_TRANS_OPAQUE, VID3LUMAKEYALPHA_OPAQUE
+*/
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define PDP_VID3LUMAKEY_ALPHA_TRANS_OPAQUE_VID3LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06EC)
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMAX
+*/
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_MASK		(0x03FF0000)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LSBMASK		(0x000003FF)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SHIFT		(16)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_LENGTH		(10)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_LUMA_MAX_MIN, VID3LUMAKEYYMIN
+*/
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_MASK		(0x000003FF)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LSBMASK		(0x000003FF)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SHIFT		(0)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_LENGTH		(10)
+#define PDP_VID3LUMAKEY_LUMA_MAX_MIN_VID3LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3LUMAKEY_C_RG_OFFSET		(0x06F0)
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_R
+*/
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_MASK		(0x0FFF0000)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SHIFT		(16)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_LENGTH		(12)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_RG, VID3LUMAKEYC_G
+*/
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_MASK		(0x00000FFF)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SHIFT		(0)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_LENGTH		(12)
+#define PDP_VID3LUMAKEY_C_RG_VID3LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3LUMAKEY_C_B_OFFSET		(0x06F4)
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYALPHAMULT
+*/
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SHIFT		(29)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_LENGTH		(1)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYEN
+*/
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_MASK		(0x10000000)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SHIFT		(28)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_LENGTH		(1)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYOUTOFF
+*/
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LSBMASK		(0x000003FF)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SHIFT		(16)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_LENGTH		(10)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3LUMAKEY_C_B, VID3LUMAKEYC_B
+*/
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_MASK		(0x00000FFF)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SHIFT		(0)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_LENGTH		(12)
+#define PDP_VID3LUMAKEY_C_B_VID3LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_OFFSET		(0x06F8)
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_TRANS
+*/
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_MASK		(0x03FF0000)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LSBMASK		(0x000003FF)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SHIFT		(16)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_LENGTH		(10)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_TRANS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_ALPHA_TRANS_OPAQUE, VID4LUMAKEYALPHA_OPAQUE
+*/
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_MASK		(0x000003FF)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LSBMASK		(0x000003FF)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SHIFT		(0)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_LENGTH		(10)
+#define PDP_VID4LUMAKEY_ALPHA_TRANS_OPAQUE_VID4LUMAKEYALPHA_OPAQUE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_OFFSET		(0x06FC)
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMAX
+*/
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_MASK		(0x03FF0000)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LSBMASK		(0x000003FF)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SHIFT		(16)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_LENGTH		(10)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_LUMA_MAX_MIN, VID4LUMAKEYYMIN
+*/
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_MASK		(0x000003FF)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LSBMASK		(0x000003FF)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SHIFT		(0)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_LENGTH		(10)
+#define PDP_VID4LUMAKEY_LUMA_MAX_MIN_VID4LUMAKEYYMIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4LUMAKEY_C_RG_OFFSET		(0x0700)
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_R
+*/
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_MASK		(0x0FFF0000)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LSBMASK		(0x00000FFF)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SHIFT		(16)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_LENGTH		(12)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_R_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_RG, VID4LUMAKEYC_G
+*/
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_MASK		(0x00000FFF)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LSBMASK		(0x00000FFF)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SHIFT		(0)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_LENGTH		(12)
+#define PDP_VID4LUMAKEY_C_RG_VID4LUMAKEYC_G_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4LUMAKEY_C_B_OFFSET		(0x0704)
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYALPHAMULT
+*/
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_MASK		(0x20000000)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LSBMASK		(0x00000001)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SHIFT		(29)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_LENGTH		(1)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYALPHAMULT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYEN
+*/
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_MASK		(0x10000000)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LSBMASK		(0x00000001)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SHIFT		(28)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_LENGTH		(1)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYOUTOFF
+*/
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_MASK		(0x03FF0000)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LSBMASK		(0x000003FF)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SHIFT		(16)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_LENGTH		(10)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYOUTOFF_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4LUMAKEY_C_B, VID4LUMAKEYC_B
+*/
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_MASK		(0x00000FFF)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LSBMASK		(0x00000FFF)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SHIFT		(0)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_LENGTH		(12)
+#define PDP_VID4LUMAKEY_C_B_VID4LUMAKEYC_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CSCCOEFF0_OFFSET		(0x0708)
+
+/* PDP, CSCCOEFF0, CSCCOEFFRU
+*/
+#define PDP_CSCCOEFF0_CSCCOEFFRU_MASK		(0x003FF800)
+#define PDP_CSCCOEFF0_CSCCOEFFRU_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF0_CSCCOEFFRU_SHIFT		(11)
+#define PDP_CSCCOEFF0_CSCCOEFFRU_LENGTH		(11)
+#define PDP_CSCCOEFF0_CSCCOEFFRU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF0, CSCCOEFFRY
+*/
+#define PDP_CSCCOEFF0_CSCCOEFFRY_MASK		(0x000007FF)
+#define PDP_CSCCOEFF0_CSCCOEFFRY_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF0_CSCCOEFFRY_SHIFT		(0)
+#define PDP_CSCCOEFF0_CSCCOEFFRY_LENGTH		(11)
+#define PDP_CSCCOEFF0_CSCCOEFFRY_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CSCCOEFF1_OFFSET		(0x070C)
+
+/* PDP, CSCCOEFF1, CSCCOEFFGY
+*/
+#define PDP_CSCCOEFF1_CSCCOEFFGY_MASK		(0x003FF800)
+#define PDP_CSCCOEFF1_CSCCOEFFGY_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF1_CSCCOEFFGY_SHIFT		(11)
+#define PDP_CSCCOEFF1_CSCCOEFFGY_LENGTH		(11)
+#define PDP_CSCCOEFF1_CSCCOEFFGY_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF1, CSCCOEFFRV
+*/
+#define PDP_CSCCOEFF1_CSCCOEFFRV_MASK		(0x000007FF)
+#define PDP_CSCCOEFF1_CSCCOEFFRV_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF1_CSCCOEFFRV_SHIFT		(0)
+#define PDP_CSCCOEFF1_CSCCOEFFRV_LENGTH		(11)
+#define PDP_CSCCOEFF1_CSCCOEFFRV_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CSCCOEFF2_OFFSET		(0x0710)
+
+/* PDP, CSCCOEFF2, CSCCOEFFGV
+*/
+#define PDP_CSCCOEFF2_CSCCOEFFGV_MASK		(0x003FF800)
+#define PDP_CSCCOEFF2_CSCCOEFFGV_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF2_CSCCOEFFGV_SHIFT		(11)
+#define PDP_CSCCOEFF2_CSCCOEFFGV_LENGTH		(11)
+#define PDP_CSCCOEFF2_CSCCOEFFGV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF2, CSCCOEFFGU
+*/
+#define PDP_CSCCOEFF2_CSCCOEFFGU_MASK		(0x000007FF)
+#define PDP_CSCCOEFF2_CSCCOEFFGU_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF2_CSCCOEFFGU_SHIFT		(0)
+#define PDP_CSCCOEFF2_CSCCOEFFGU_LENGTH		(11)
+#define PDP_CSCCOEFF2_CSCCOEFFGU_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CSCCOEFF3_OFFSET		(0x0714)
+
+/* PDP, CSCCOEFF3, CSCCOEFFBU
+*/
+#define PDP_CSCCOEFF3_CSCCOEFFBU_MASK		(0x003FF800)
+#define PDP_CSCCOEFF3_CSCCOEFFBU_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF3_CSCCOEFFBU_SHIFT		(11)
+#define PDP_CSCCOEFF3_CSCCOEFFBU_LENGTH		(11)
+#define PDP_CSCCOEFF3_CSCCOEFFBU_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CSCCOEFF3, CSCCOEFFBY
+*/
+#define PDP_CSCCOEFF3_CSCCOEFFBY_MASK		(0x000007FF)
+#define PDP_CSCCOEFF3_CSCCOEFFBY_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF3_CSCCOEFFBY_SHIFT		(0)
+#define PDP_CSCCOEFF3_CSCCOEFFBY_LENGTH		(11)
+#define PDP_CSCCOEFF3_CSCCOEFFBY_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CSCCOEFF4_OFFSET		(0x0718)
+
+/* PDP, CSCCOEFF4, CSCCOEFFBV
+*/
+#define PDP_CSCCOEFF4_CSCCOEFFBV_MASK		(0x000007FF)
+#define PDP_CSCCOEFF4_CSCCOEFFBV_LSBMASK		(0x000007FF)
+#define PDP_CSCCOEFF4_CSCCOEFFBV_SHIFT		(0)
+#define PDP_CSCCOEFF4_CSCCOEFFBV_LENGTH		(11)
+#define PDP_CSCCOEFF4_CSCCOEFFBV_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BGNDCOL_AR_OFFSET		(0x071C)
+
+/* PDP, BGNDCOL_AR, BGNDCOL_A
+*/
+#define PDP_BGNDCOL_AR_BGNDCOL_A_MASK		(0x03FF0000)
+#define PDP_BGNDCOL_AR_BGNDCOL_A_LSBMASK		(0x000003FF)
+#define PDP_BGNDCOL_AR_BGNDCOL_A_SHIFT		(16)
+#define PDP_BGNDCOL_AR_BGNDCOL_A_LENGTH		(10)
+#define PDP_BGNDCOL_AR_BGNDCOL_A_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_AR, BGNDCOL_R
+*/
+#define PDP_BGNDCOL_AR_BGNDCOL_R_MASK		(0x000003FF)
+#define PDP_BGNDCOL_AR_BGNDCOL_R_LSBMASK		(0x000003FF)
+#define PDP_BGNDCOL_AR_BGNDCOL_R_SHIFT		(0)
+#define PDP_BGNDCOL_AR_BGNDCOL_R_LENGTH		(10)
+#define PDP_BGNDCOL_AR_BGNDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BGNDCOL_GB_OFFSET		(0x0720)
+
+/* PDP, BGNDCOL_GB, BGNDCOL_G
+*/
+#define PDP_BGNDCOL_GB_BGNDCOL_G_MASK		(0x03FF0000)
+#define PDP_BGNDCOL_GB_BGNDCOL_G_LSBMASK		(0x000003FF)
+#define PDP_BGNDCOL_GB_BGNDCOL_G_SHIFT		(16)
+#define PDP_BGNDCOL_GB_BGNDCOL_G_LENGTH		(10)
+#define PDP_BGNDCOL_GB_BGNDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BGNDCOL_GB, BGNDCOL_B
+*/
+#define PDP_BGNDCOL_GB_BGNDCOL_B_MASK		(0x000003FF)
+#define PDP_BGNDCOL_GB_BGNDCOL_B_LSBMASK		(0x000003FF)
+#define PDP_BGNDCOL_GB_BGNDCOL_B_SHIFT		(0)
+#define PDP_BGNDCOL_GB_BGNDCOL_B_LENGTH		(10)
+#define PDP_BGNDCOL_GB_BGNDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BORDCOL_R_OFFSET		(0x0724)
+
+/* PDP, BORDCOL_R, BORDCOL_R
+*/
+#define PDP_BORDCOL_R_BORDCOL_R_MASK		(0x000003FF)
+#define PDP_BORDCOL_R_BORDCOL_R_LSBMASK		(0x000003FF)
+#define PDP_BORDCOL_R_BORDCOL_R_SHIFT		(0)
+#define PDP_BORDCOL_R_BORDCOL_R_LENGTH		(10)
+#define PDP_BORDCOL_R_BORDCOL_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BORDCOL_GB_OFFSET		(0x0728)
+
+/* PDP, BORDCOL_GB, BORDCOL_G
+*/
+#define PDP_BORDCOL_GB_BORDCOL_G_MASK		(0x03FF0000)
+#define PDP_BORDCOL_GB_BORDCOL_G_LSBMASK		(0x000003FF)
+#define PDP_BORDCOL_GB_BORDCOL_G_SHIFT		(16)
+#define PDP_BORDCOL_GB_BORDCOL_G_LENGTH		(10)
+#define PDP_BORDCOL_GB_BORDCOL_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, BORDCOL_GB, BORDCOL_B
+*/
+#define PDP_BORDCOL_GB_BORDCOL_B_MASK		(0x000003FF)
+#define PDP_BORDCOL_GB_BORDCOL_B_LSBMASK		(0x000003FF)
+#define PDP_BORDCOL_GB_BORDCOL_B_SHIFT		(0)
+#define PDP_BORDCOL_GB_BORDCOL_B_LENGTH		(10)
+#define PDP_BORDCOL_GB_BORDCOL_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_LINESTAT_OFFSET		(0x0734)
+
+/* PDP, LINESTAT, LINENO
+*/
+#define PDP_LINESTAT_LINENO_MASK		(0x00001FFF)
+#define PDP_LINESTAT_LINENO_LSBMASK		(0x00001FFF)
+#define PDP_LINESTAT_LINENO_SHIFT		(0)
+#define PDP_LINESTAT_LINENO_LENGTH		(13)
+#define PDP_LINESTAT_LINENO_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_C11C12_OFFSET		(0x0738)
+
+/* PDP, CR_PDP_PROCAMP_C11C12, CR_PROCAMP_C12
+*/
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_MASK		(0x3FFF0000)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SHIFT		(16)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C12_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_C11C12, CR_PROCAMP_C11
+*/
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_MASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C11C12_CR_PROCAMP_C11_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_C13C21_OFFSET		(0x073C)
+
+/* PDP, CR_PDP_PROCAMP_C13C21, CR_PROCAMP_C21
+*/
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_MASK		(0x3FFF0000)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SHIFT		(16)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C21_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_C13C21, CR_PROCAMP_C13
+*/
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_MASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C13C21_CR_PROCAMP_C13_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_C22C23_OFFSET		(0x0740)
+
+/* PDP, CR_PDP_PROCAMP_C22C23, CR_PROCAMP_C23
+*/
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_MASK		(0x3FFF0000)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SHIFT		(16)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C23_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_C22C23, CR_PROCAMP_C22
+*/
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_MASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C22C23_CR_PROCAMP_C22_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_C31C32_OFFSET		(0x0744)
+
+/* PDP, CR_PDP_PROCAMP_C31C32, CR_PROCAMP_C32
+*/
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_MASK		(0x3FFF0000)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SHIFT		(16)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C32_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_C31C32, CR_PROCAMP_C31
+*/
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_MASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C31C32_CR_PROCAMP_C31_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_C33_OFFSET		(0x0748)
+
+/* PDP, CR_PDP_PROCAMP_C33, CR_PROCAMP_C33
+*/
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_C33_MASK		(0x3FFF0000)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_C33_LSBMASK		(0x00003FFF)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_C33_SHIFT		(16)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_C33_LENGTH		(14)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_C33_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_C33, CR_PROCAMP_RANGE
+*/
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_MASK		(0x00000030)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LSBMASK		(0x00000003)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SHIFT		(4)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_LENGTH		(2)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_RANGE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_C33, CR_PROCAMP_EN
+*/
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_EN_MASK		(0x00000001)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_EN_LSBMASK		(0x00000001)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_EN_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_EN_LENGTH		(1)
+#define PDP_CR_PDP_PROCAMP_C33_CR_PROCAMP_EN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_OFFSET		(0x074C)
+
+/* PDP, CR_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_G
+*/
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_MASK		(0x0FFF0000)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LSBMASK		(0x00000FFF)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SHIFT		(16)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_LENGTH		(12)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_OUTOFFSET_BG, CR_PROCAMP_OUTOFF_B
+*/
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_MASK		(0x00000FFF)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LSBMASK		(0x00000FFF)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_LENGTH		(12)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_BG_CR_PROCAMP_OUTOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_R_OFFSET		(0x0750)
+
+/* PDP, CR_PDP_PROCAMP_OUTOFFSET_R, CR_PROCAMP_OUTOFF_R
+*/
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_MASK		(0x00000FFF)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LSBMASK		(0x00000FFF)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_LENGTH		(12)
+#define PDP_CR_PDP_PROCAMP_OUTOFFSET_R_CR_PROCAMP_OUTOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_OFFSET		(0x0754)
+
+/* PDP, CR_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_G
+*/
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_MASK		(0x03FF0000)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LSBMASK		(0x000003FF)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SHIFT		(16)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_LENGTH		(10)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, CR_PDP_PROCAMP_INOFFSET_BG, CR_PROCAMP_INOFF_B
+*/
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_MASK		(0x000003FF)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LSBMASK		(0x000003FF)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_LENGTH		(10)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_BG_CR_PROCAMP_INOFF_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_CR_PDP_PROCAMP_INOFFSET_R_OFFSET		(0x0758)
+
+/* PDP, CR_PDP_PROCAMP_INOFFSET_R, CR_PROCAMP_INOFF_R
+*/
+#define PDP_CR_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_MASK		(0x000003FF)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LSBMASK		(0x000003FF)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SHIFT		(0)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_LENGTH		(10)
+#define PDP_CR_PDP_PROCAMP_INOFFSET_R_CR_PROCAMP_INOFF_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_SIGNAT_R_OFFSET		(0x075C)
+
+/* PDP, SIGNAT_R, SIGNATURE_R
+*/
+#define PDP_SIGNAT_R_SIGNATURE_R_MASK		(0x000003FF)
+#define PDP_SIGNAT_R_SIGNATURE_R_LSBMASK		(0x000003FF)
+#define PDP_SIGNAT_R_SIGNATURE_R_SHIFT		(0)
+#define PDP_SIGNAT_R_SIGNATURE_R_LENGTH		(10)
+#define PDP_SIGNAT_R_SIGNATURE_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_SIGNAT_GB_OFFSET		(0x0760)
+
+/* PDP, SIGNAT_GB, SIGNATURE_G
+*/
+#define PDP_SIGNAT_GB_SIGNATURE_G_MASK		(0x03FF0000)
+#define PDP_SIGNAT_GB_SIGNATURE_G_LSBMASK		(0x000003FF)
+#define PDP_SIGNAT_GB_SIGNATURE_G_SHIFT		(16)
+#define PDP_SIGNAT_GB_SIGNATURE_G_LENGTH		(10)
+#define PDP_SIGNAT_GB_SIGNATURE_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SIGNAT_GB, SIGNATURE_B
+*/
+#define PDP_SIGNAT_GB_SIGNATURE_B_MASK		(0x000003FF)
+#define PDP_SIGNAT_GB_SIGNATURE_B_LSBMASK		(0x000003FF)
+#define PDP_SIGNAT_GB_SIGNATURE_B_SHIFT		(0)
+#define PDP_SIGNAT_GB_SIGNATURE_B_LENGTH		(10)
+#define PDP_SIGNAT_GB_SIGNATURE_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_REGISTER_UPDATE_CTRL_OFFSET		(0x0764)
+
+/* PDP, REGISTER_UPDATE_CTRL, BYPASS_DOUBLE_BUFFERING
+*/
+#define PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_MASK		(0x00000004)
+#define PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LSBMASK		(0x00000001)
+#define PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SHIFT		(2)
+#define PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_LENGTH		(1)
+#define PDP_REGISTER_UPDATE_CTRL_BYPASS_DOUBLE_BUFFERING_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, REGISTERS_VALID
+*/
+#define PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_MASK		(0x00000002)
+#define PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LSBMASK		(0x00000001)
+#define PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SHIFT		(1)
+#define PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_LENGTH		(1)
+#define PDP_REGISTER_UPDATE_CTRL_REGISTERS_VALID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGISTER_UPDATE_CTRL, USE_VBLANK
+*/
+#define PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_MASK		(0x00000001)
+#define PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LSBMASK		(0x00000001)
+#define PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SHIFT		(0)
+#define PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_LENGTH		(1)
+#define PDP_REGISTER_UPDATE_CTRL_USE_VBLANK_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_REGISTER_UPDATE_STATUS_OFFSET		(0x0768)
+
+/* PDP, REGISTER_UPDATE_STATUS, REGISTERS_UPDATED
+*/
+#define PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_MASK		(0x00000002)
+#define PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LSBMASK		(0x00000001)
+#define PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SHIFT		(1)
+#define PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_LENGTH		(1)
+#define PDP_REGISTER_UPDATE_STATUS_REGISTERS_UPDATED_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DBGCTRL_OFFSET		(0x076C)
+
+/* PDP, DBGCTRL, DBG_READ
+*/
+#define PDP_DBGCTRL_DBG_READ_MASK		(0x00000002)
+#define PDP_DBGCTRL_DBG_READ_LSBMASK		(0x00000001)
+#define PDP_DBGCTRL_DBG_READ_SHIFT		(1)
+#define PDP_DBGCTRL_DBG_READ_LENGTH		(1)
+#define PDP_DBGCTRL_DBG_READ_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGCTRL, DBG_ENAB
+*/
+#define PDP_DBGCTRL_DBG_ENAB_MASK		(0x00000001)
+#define PDP_DBGCTRL_DBG_ENAB_LSBMASK		(0x00000001)
+#define PDP_DBGCTRL_DBG_ENAB_SHIFT		(0)
+#define PDP_DBGCTRL_DBG_ENAB_LENGTH		(1)
+#define PDP_DBGCTRL_DBG_ENAB_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DBGDATA_R_OFFSET		(0x0770)
+
+/* PDP, DBGDATA_R, DBG_DATA_R
+*/
+#define PDP_DBGDATA_R_DBG_DATA_R_MASK		(0x000003FF)
+#define PDP_DBGDATA_R_DBG_DATA_R_LSBMASK		(0x000003FF)
+#define PDP_DBGDATA_R_DBG_DATA_R_SHIFT		(0)
+#define PDP_DBGDATA_R_DBG_DATA_R_LENGTH		(10)
+#define PDP_DBGDATA_R_DBG_DATA_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DBGDATA_GB_OFFSET		(0x0774)
+
+/* PDP, DBGDATA_GB, DBG_DATA_G
+*/
+#define PDP_DBGDATA_GB_DBG_DATA_G_MASK		(0x03FF0000)
+#define PDP_DBGDATA_GB_DBG_DATA_G_LSBMASK		(0x000003FF)
+#define PDP_DBGDATA_GB_DBG_DATA_G_SHIFT		(16)
+#define PDP_DBGDATA_GB_DBG_DATA_G_LENGTH		(10)
+#define PDP_DBGDATA_GB_DBG_DATA_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGDATA_GB, DBG_DATA_B
+*/
+#define PDP_DBGDATA_GB_DBG_DATA_B_MASK		(0x000003FF)
+#define PDP_DBGDATA_GB_DBG_DATA_B_LSBMASK		(0x000003FF)
+#define PDP_DBGDATA_GB_DBG_DATA_B_SHIFT		(0)
+#define PDP_DBGDATA_GB_DBG_DATA_B_LENGTH		(10)
+#define PDP_DBGDATA_GB_DBG_DATA_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DBGSIDE_OFFSET		(0x0778)
+
+/* PDP, DBGSIDE, DBG_VAL
+*/
+#define PDP_DBGSIDE_DBG_VAL_MASK		(0x00000008)
+#define PDP_DBGSIDE_DBG_VAL_LSBMASK		(0x00000001)
+#define PDP_DBGSIDE_DBG_VAL_SHIFT		(3)
+#define PDP_DBGSIDE_DBG_VAL_LENGTH		(1)
+#define PDP_DBGSIDE_DBG_VAL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DBGSIDE, DBG_SIDE
+*/
+#define PDP_DBGSIDE_DBG_SIDE_MASK		(0x00000007)
+#define PDP_DBGSIDE_DBG_SIDE_LSBMASK		(0x00000007)
+#define PDP_DBGSIDE_DBG_SIDE_SHIFT		(0)
+#define PDP_DBGSIDE_DBG_SIDE_LENGTH		(3)
+#define PDP_DBGSIDE_DBG_SIDE_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_OUTPUT_OFFSET		(0x077C)
+
+/* PDP, OUTPUT, EIGHT_BIT_OUTPUT
+*/
+#define PDP_OUTPUT_EIGHT_BIT_OUTPUT_MASK		(0x00000002)
+#define PDP_OUTPUT_EIGHT_BIT_OUTPUT_LSBMASK		(0x00000001)
+#define PDP_OUTPUT_EIGHT_BIT_OUTPUT_SHIFT		(1)
+#define PDP_OUTPUT_EIGHT_BIT_OUTPUT_LENGTH		(1)
+#define PDP_OUTPUT_EIGHT_BIT_OUTPUT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OUTPUT, OUTPUT_CONFIG
+*/
+#define PDP_OUTPUT_OUTPUT_CONFIG_MASK		(0x00000001)
+#define PDP_OUTPUT_OUTPUT_CONFIG_LSBMASK		(0x00000001)
+#define PDP_OUTPUT_OUTPUT_CONFIG_SHIFT		(0)
+#define PDP_OUTPUT_OUTPUT_CONFIG_LENGTH		(1)
+#define PDP_OUTPUT_OUTPUT_CONFIG_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_SYNCCTRL_OFFSET		(0x0780)
+
+/* PDP, SYNCCTRL, SYNCACTIVE
+*/
+#define PDP_SYNCCTRL_SYNCACTIVE_MASK		(0x80000000)
+#define PDP_SYNCCTRL_SYNCACTIVE_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_SYNCACTIVE_SHIFT		(31)
+#define PDP_SYNCCTRL_SYNCACTIVE_LENGTH		(1)
+#define PDP_SYNCCTRL_SYNCACTIVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, PDP_RST
+*/
+#define PDP_SYNCCTRL_PDP_RST_MASK		(0x20000000)
+#define PDP_SYNCCTRL_PDP_RST_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_PDP_RST_SHIFT		(29)
+#define PDP_SYNCCTRL_PDP_RST_LENGTH		(1)
+#define PDP_SYNCCTRL_PDP_RST_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, POWERDN
+*/
+#define PDP_SYNCCTRL_POWERDN_MASK		(0x10000000)
+#define PDP_SYNCCTRL_POWERDN_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_POWERDN_SHIFT		(28)
+#define PDP_SYNCCTRL_POWERDN_LENGTH		(1)
+#define PDP_SYNCCTRL_POWERDN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, LOWPWRMODE
+*/
+#define PDP_SYNCCTRL_LOWPWRMODE_MASK		(0x08000000)
+#define PDP_SYNCCTRL_LOWPWRMODE_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_LOWPWRMODE_SHIFT		(27)
+#define PDP_SYNCCTRL_LOWPWRMODE_LENGTH		(1)
+#define PDP_SYNCCTRL_LOWPWRMODE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDSYNCTRL
+*/
+#define PDP_SYNCCTRL_UPDSYNCTRL_MASK		(0x04000000)
+#define PDP_SYNCCTRL_UPDSYNCTRL_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_UPDSYNCTRL_SHIFT		(26)
+#define PDP_SYNCCTRL_UPDSYNCTRL_LENGTH		(1)
+#define PDP_SYNCCTRL_UPDSYNCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDINTCTRL
+*/
+#define PDP_SYNCCTRL_UPDINTCTRL_MASK		(0x02000000)
+#define PDP_SYNCCTRL_UPDINTCTRL_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_UPDINTCTRL_SHIFT		(25)
+#define PDP_SYNCCTRL_UPDINTCTRL_LENGTH		(1)
+#define PDP_SYNCCTRL_UPDINTCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDCTRL
+*/
+#define PDP_SYNCCTRL_UPDCTRL_MASK		(0x01000000)
+#define PDP_SYNCCTRL_UPDCTRL_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_UPDCTRL_SHIFT		(24)
+#define PDP_SYNCCTRL_UPDCTRL_LENGTH		(1)
+#define PDP_SYNCCTRL_UPDCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, UPDWAIT
+*/
+#define PDP_SYNCCTRL_UPDWAIT_MASK		(0x000F0000)
+#define PDP_SYNCCTRL_UPDWAIT_LSBMASK		(0x0000000F)
+#define PDP_SYNCCTRL_UPDWAIT_SHIFT		(16)
+#define PDP_SYNCCTRL_UPDWAIT_LENGTH		(4)
+#define PDP_SYNCCTRL_UPDWAIT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, FIELD_EN
+*/
+#define PDP_SYNCCTRL_FIELD_EN_MASK		(0x00002000)
+#define PDP_SYNCCTRL_FIELD_EN_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_FIELD_EN_SHIFT		(13)
+#define PDP_SYNCCTRL_FIELD_EN_LENGTH		(1)
+#define PDP_SYNCCTRL_FIELD_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, CSYNC_EN
+*/
+#define PDP_SYNCCTRL_CSYNC_EN_MASK		(0x00001000)
+#define PDP_SYNCCTRL_CSYNC_EN_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_CSYNC_EN_SHIFT		(12)
+#define PDP_SYNCCTRL_CSYNC_EN_LENGTH		(1)
+#define PDP_SYNCCTRL_CSYNC_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, CLKPOL
+*/
+#define PDP_SYNCCTRL_CLKPOL_MASK		(0x00000800)
+#define PDP_SYNCCTRL_CLKPOL_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_CLKPOL_SHIFT		(11)
+#define PDP_SYNCCTRL_CLKPOL_LENGTH		(1)
+#define PDP_SYNCCTRL_CLKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VS_SLAVE
+*/
+#define PDP_SYNCCTRL_VS_SLAVE_MASK		(0x00000080)
+#define PDP_SYNCCTRL_VS_SLAVE_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_VS_SLAVE_SHIFT		(7)
+#define PDP_SYNCCTRL_VS_SLAVE_LENGTH		(1)
+#define PDP_SYNCCTRL_VS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HS_SLAVE
+*/
+#define PDP_SYNCCTRL_HS_SLAVE_MASK		(0x00000040)
+#define PDP_SYNCCTRL_HS_SLAVE_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_HS_SLAVE_SHIFT		(6)
+#define PDP_SYNCCTRL_HS_SLAVE_LENGTH		(1)
+#define PDP_SYNCCTRL_HS_SLAVE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKPOL
+*/
+#define PDP_SYNCCTRL_BLNKPOL_MASK		(0x00000020)
+#define PDP_SYNCCTRL_BLNKPOL_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_BLNKPOL_SHIFT		(5)
+#define PDP_SYNCCTRL_BLNKPOL_LENGTH		(1)
+#define PDP_SYNCCTRL_BLNKPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, BLNKDIS
+*/
+#define PDP_SYNCCTRL_BLNKDIS_MASK		(0x00000010)
+#define PDP_SYNCCTRL_BLNKDIS_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_BLNKDIS_SHIFT		(4)
+#define PDP_SYNCCTRL_BLNKDIS_LENGTH		(1)
+#define PDP_SYNCCTRL_BLNKDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSPOL
+*/
+#define PDP_SYNCCTRL_VSPOL_MASK		(0x00000008)
+#define PDP_SYNCCTRL_VSPOL_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_VSPOL_SHIFT		(3)
+#define PDP_SYNCCTRL_VSPOL_LENGTH		(1)
+#define PDP_SYNCCTRL_VSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, VSDIS
+*/
+#define PDP_SYNCCTRL_VSDIS_MASK		(0x00000004)
+#define PDP_SYNCCTRL_VSDIS_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_VSDIS_SHIFT		(2)
+#define PDP_SYNCCTRL_VSDIS_LENGTH		(1)
+#define PDP_SYNCCTRL_VSDIS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSPOL
+*/
+#define PDP_SYNCCTRL_HSPOL_MASK		(0x00000002)
+#define PDP_SYNCCTRL_HSPOL_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_HSPOL_SHIFT		(1)
+#define PDP_SYNCCTRL_HSPOL_LENGTH		(1)
+#define PDP_SYNCCTRL_HSPOL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, SYNCCTRL, HSDIS
+*/
+#define PDP_SYNCCTRL_HSDIS_MASK		(0x00000001)
+#define PDP_SYNCCTRL_HSDIS_LSBMASK		(0x00000001)
+#define PDP_SYNCCTRL_HSDIS_SHIFT		(0)
+#define PDP_SYNCCTRL_HSDIS_LENGTH		(1)
+#define PDP_SYNCCTRL_HSDIS_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_HSYNC1_OFFSET		(0x0784)
+
+/* PDP, HSYNC1, HBPS
+*/
+#define PDP_HSYNC1_HBPS_MASK		(0x1FFF0000)
+#define PDP_HSYNC1_HBPS_LSBMASK		(0x00001FFF)
+#define PDP_HSYNC1_HBPS_SHIFT		(16)
+#define PDP_HSYNC1_HBPS_LENGTH		(13)
+#define PDP_HSYNC1_HBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC1, HT
+*/
+#define PDP_HSYNC1_HT_MASK		(0x00001FFF)
+#define PDP_HSYNC1_HT_LSBMASK		(0x00001FFF)
+#define PDP_HSYNC1_HT_SHIFT		(0)
+#define PDP_HSYNC1_HT_LENGTH		(13)
+#define PDP_HSYNC1_HT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_HSYNC2_OFFSET		(0x0788)
+
+/* PDP, HSYNC2, HAS
+*/
+#define PDP_HSYNC2_HAS_MASK		(0x1FFF0000)
+#define PDP_HSYNC2_HAS_LSBMASK		(0x00001FFF)
+#define PDP_HSYNC2_HAS_SHIFT		(16)
+#define PDP_HSYNC2_HAS_LENGTH		(13)
+#define PDP_HSYNC2_HAS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC2, HLBS
+*/
+#define PDP_HSYNC2_HLBS_MASK		(0x00001FFF)
+#define PDP_HSYNC2_HLBS_LSBMASK		(0x00001FFF)
+#define PDP_HSYNC2_HLBS_SHIFT		(0)
+#define PDP_HSYNC2_HLBS_LENGTH		(13)
+#define PDP_HSYNC2_HLBS_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_HSYNC3_OFFSET		(0x078C)
+
+/* PDP, HSYNC3, HFPS
+*/
+#define PDP_HSYNC3_HFPS_MASK		(0x1FFF0000)
+#define PDP_HSYNC3_HFPS_LSBMASK		(0x00001FFF)
+#define PDP_HSYNC3_HFPS_SHIFT		(16)
+#define PDP_HSYNC3_HFPS_LENGTH		(13)
+#define PDP_HSYNC3_HFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HSYNC3, HRBS
+*/
+#define PDP_HSYNC3_HRBS_MASK		(0x00001FFF)
+#define PDP_HSYNC3_HRBS_LSBMASK		(0x00001FFF)
+#define PDP_HSYNC3_HRBS_SHIFT		(0)
+#define PDP_HSYNC3_HRBS_LENGTH		(13)
+#define PDP_HSYNC3_HRBS_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VSYNC1_OFFSET		(0x0790)
+
+/* PDP, VSYNC1, VBPS
+*/
+#define PDP_VSYNC1_VBPS_MASK		(0x1FFF0000)
+#define PDP_VSYNC1_VBPS_LSBMASK		(0x00001FFF)
+#define PDP_VSYNC1_VBPS_SHIFT		(16)
+#define PDP_VSYNC1_VBPS_LENGTH		(13)
+#define PDP_VSYNC1_VBPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC1, VT
+*/
+#define PDP_VSYNC1_VT_MASK		(0x00001FFF)
+#define PDP_VSYNC1_VT_LSBMASK		(0x00001FFF)
+#define PDP_VSYNC1_VT_SHIFT		(0)
+#define PDP_VSYNC1_VT_LENGTH		(13)
+#define PDP_VSYNC1_VT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VSYNC2_OFFSET		(0x0794)
+
+/* PDP, VSYNC2, VAS
+*/
+#define PDP_VSYNC2_VAS_MASK		(0x1FFF0000)
+#define PDP_VSYNC2_VAS_LSBMASK		(0x00001FFF)
+#define PDP_VSYNC2_VAS_SHIFT		(16)
+#define PDP_VSYNC2_VAS_LENGTH		(13)
+#define PDP_VSYNC2_VAS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC2, VTBS
+*/
+#define PDP_VSYNC2_VTBS_MASK		(0x00001FFF)
+#define PDP_VSYNC2_VTBS_LSBMASK		(0x00001FFF)
+#define PDP_VSYNC2_VTBS_SHIFT		(0)
+#define PDP_VSYNC2_VTBS_LENGTH		(13)
+#define PDP_VSYNC2_VTBS_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VSYNC3_OFFSET		(0x0798)
+
+/* PDP, VSYNC3, VFPS
+*/
+#define PDP_VSYNC3_VFPS_MASK		(0x1FFF0000)
+#define PDP_VSYNC3_VFPS_LSBMASK		(0x00001FFF)
+#define PDP_VSYNC3_VFPS_SHIFT		(16)
+#define PDP_VSYNC3_VFPS_LENGTH		(13)
+#define PDP_VSYNC3_VFPS_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VSYNC3, VBBS
+*/
+#define PDP_VSYNC3_VBBS_MASK		(0x00001FFF)
+#define PDP_VSYNC3_VBBS_LSBMASK		(0x00001FFF)
+#define PDP_VSYNC3_VBBS_SHIFT		(0)
+#define PDP_VSYNC3_VBBS_LENGTH		(13)
+#define PDP_VSYNC3_VBBS_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_INTSTAT_OFFSET		(0x079C)
+
+/* PDP, INTSTAT, INTS_VID4ORUN
+*/
+#define PDP_INTSTAT_INTS_VID4ORUN_MASK		(0x00080000)
+#define PDP_INTSTAT_INTS_VID4ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID4ORUN_SHIFT		(19)
+#define PDP_INTSTAT_INTS_VID4ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3ORUN
+*/
+#define PDP_INTSTAT_INTS_VID3ORUN_MASK		(0x00040000)
+#define PDP_INTSTAT_INTS_VID3ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID3ORUN_SHIFT		(18)
+#define PDP_INTSTAT_INTS_VID3ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2ORUN
+*/
+#define PDP_INTSTAT_INTS_VID2ORUN_MASK		(0x00020000)
+#define PDP_INTSTAT_INTS_VID2ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID2ORUN_SHIFT		(17)
+#define PDP_INTSTAT_INTS_VID2ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1ORUN
+*/
+#define PDP_INTSTAT_INTS_VID1ORUN_MASK		(0x00010000)
+#define PDP_INTSTAT_INTS_VID1ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID1ORUN_SHIFT		(16)
+#define PDP_INTSTAT_INTS_VID1ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4ORUN
+*/
+#define PDP_INTSTAT_INTS_GRPH4ORUN_MASK		(0x00008000)
+#define PDP_INTSTAT_INTS_GRPH4ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH4ORUN_SHIFT		(15)
+#define PDP_INTSTAT_INTS_GRPH4ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3ORUN
+*/
+#define PDP_INTSTAT_INTS_GRPH3ORUN_MASK		(0x00004000)
+#define PDP_INTSTAT_INTS_GRPH3ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH3ORUN_SHIFT		(14)
+#define PDP_INTSTAT_INTS_GRPH3ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2ORUN
+*/
+#define PDP_INTSTAT_INTS_GRPH2ORUN_MASK		(0x00002000)
+#define PDP_INTSTAT_INTS_GRPH2ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH2ORUN_SHIFT		(13)
+#define PDP_INTSTAT_INTS_GRPH2ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1ORUN
+*/
+#define PDP_INTSTAT_INTS_GRPH1ORUN_MASK		(0x00001000)
+#define PDP_INTSTAT_INTS_GRPH1ORUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH1ORUN_SHIFT		(12)
+#define PDP_INTSTAT_INTS_GRPH1ORUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID4URUN
+*/
+#define PDP_INTSTAT_INTS_VID4URUN_MASK		(0x00000800)
+#define PDP_INTSTAT_INTS_VID4URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID4URUN_SHIFT		(11)
+#define PDP_INTSTAT_INTS_VID4URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID3URUN
+*/
+#define PDP_INTSTAT_INTS_VID3URUN_MASK		(0x00000400)
+#define PDP_INTSTAT_INTS_VID3URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID3URUN_SHIFT		(10)
+#define PDP_INTSTAT_INTS_VID3URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID2URUN
+*/
+#define PDP_INTSTAT_INTS_VID2URUN_MASK		(0x00000200)
+#define PDP_INTSTAT_INTS_VID2URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID2URUN_SHIFT		(9)
+#define PDP_INTSTAT_INTS_VID2URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VID1URUN
+*/
+#define PDP_INTSTAT_INTS_VID1URUN_MASK		(0x00000100)
+#define PDP_INTSTAT_INTS_VID1URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VID1URUN_SHIFT		(8)
+#define PDP_INTSTAT_INTS_VID1URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH4URUN
+*/
+#define PDP_INTSTAT_INTS_GRPH4URUN_MASK		(0x00000080)
+#define PDP_INTSTAT_INTS_GRPH4URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH4URUN_SHIFT		(7)
+#define PDP_INTSTAT_INTS_GRPH4URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH3URUN
+*/
+#define PDP_INTSTAT_INTS_GRPH3URUN_MASK		(0x00000040)
+#define PDP_INTSTAT_INTS_GRPH3URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH3URUN_SHIFT		(6)
+#define PDP_INTSTAT_INTS_GRPH3URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH2URUN
+*/
+#define PDP_INTSTAT_INTS_GRPH2URUN_MASK		(0x00000020)
+#define PDP_INTSTAT_INTS_GRPH2URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH2URUN_SHIFT		(5)
+#define PDP_INTSTAT_INTS_GRPH2URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_GRPH1URUN
+*/
+#define PDP_INTSTAT_INTS_GRPH1URUN_MASK		(0x00000010)
+#define PDP_INTSTAT_INTS_GRPH1URUN_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_GRPH1URUN_SHIFT		(4)
+#define PDP_INTSTAT_INTS_GRPH1URUN_LENGTH		(1)
+#define PDP_INTSTAT_INTS_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK1
+*/
+#define PDP_INTSTAT_INTS_VBLNK1_MASK		(0x00000008)
+#define PDP_INTSTAT_INTS_VBLNK1_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VBLNK1_SHIFT		(3)
+#define PDP_INTSTAT_INTS_VBLNK1_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_VBLNK0
+*/
+#define PDP_INTSTAT_INTS_VBLNK0_MASK		(0x00000004)
+#define PDP_INTSTAT_INTS_VBLNK0_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_VBLNK0_SHIFT		(2)
+#define PDP_INTSTAT_INTS_VBLNK0_LENGTH		(1)
+#define PDP_INTSTAT_INTS_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK1
+*/
+#define PDP_INTSTAT_INTS_HBLNK1_MASK		(0x00000002)
+#define PDP_INTSTAT_INTS_HBLNK1_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_HBLNK1_SHIFT		(1)
+#define PDP_INTSTAT_INTS_HBLNK1_LENGTH		(1)
+#define PDP_INTSTAT_INTS_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTSTAT, INTS_HBLNK0
+*/
+#define PDP_INTSTAT_INTS_HBLNK0_MASK		(0x00000001)
+#define PDP_INTSTAT_INTS_HBLNK0_LSBMASK		(0x00000001)
+#define PDP_INTSTAT_INTS_HBLNK0_SHIFT		(0)
+#define PDP_INTSTAT_INTS_HBLNK0_LENGTH		(1)
+#define PDP_INTSTAT_INTS_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_INTENAB_OFFSET		(0x07A0)
+
+/* PDP, INTENAB, INTEN_VID4ORUN
+*/
+#define PDP_INTENAB_INTEN_VID4ORUN_MASK		(0x00080000)
+#define PDP_INTENAB_INTEN_VID4ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID4ORUN_SHIFT		(19)
+#define PDP_INTENAB_INTEN_VID4ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3ORUN
+*/
+#define PDP_INTENAB_INTEN_VID3ORUN_MASK		(0x00040000)
+#define PDP_INTENAB_INTEN_VID3ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID3ORUN_SHIFT		(18)
+#define PDP_INTENAB_INTEN_VID3ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2ORUN
+*/
+#define PDP_INTENAB_INTEN_VID2ORUN_MASK		(0x00020000)
+#define PDP_INTENAB_INTEN_VID2ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID2ORUN_SHIFT		(17)
+#define PDP_INTENAB_INTEN_VID2ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1ORUN
+*/
+#define PDP_INTENAB_INTEN_VID1ORUN_MASK		(0x00010000)
+#define PDP_INTENAB_INTEN_VID1ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID1ORUN_SHIFT		(16)
+#define PDP_INTENAB_INTEN_VID1ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4ORUN
+*/
+#define PDP_INTENAB_INTEN_GRPH4ORUN_MASK		(0x00008000)
+#define PDP_INTENAB_INTEN_GRPH4ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH4ORUN_SHIFT		(15)
+#define PDP_INTENAB_INTEN_GRPH4ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3ORUN
+*/
+#define PDP_INTENAB_INTEN_GRPH3ORUN_MASK		(0x00004000)
+#define PDP_INTENAB_INTEN_GRPH3ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH3ORUN_SHIFT		(14)
+#define PDP_INTENAB_INTEN_GRPH3ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2ORUN
+*/
+#define PDP_INTENAB_INTEN_GRPH2ORUN_MASK		(0x00002000)
+#define PDP_INTENAB_INTEN_GRPH2ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH2ORUN_SHIFT		(13)
+#define PDP_INTENAB_INTEN_GRPH2ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1ORUN
+*/
+#define PDP_INTENAB_INTEN_GRPH1ORUN_MASK		(0x00001000)
+#define PDP_INTENAB_INTEN_GRPH1ORUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH1ORUN_SHIFT		(12)
+#define PDP_INTENAB_INTEN_GRPH1ORUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID4URUN
+*/
+#define PDP_INTENAB_INTEN_VID4URUN_MASK		(0x00000800)
+#define PDP_INTENAB_INTEN_VID4URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID4URUN_SHIFT		(11)
+#define PDP_INTENAB_INTEN_VID4URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID3URUN
+*/
+#define PDP_INTENAB_INTEN_VID3URUN_MASK		(0x00000400)
+#define PDP_INTENAB_INTEN_VID3URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID3URUN_SHIFT		(10)
+#define PDP_INTENAB_INTEN_VID3URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID2URUN
+*/
+#define PDP_INTENAB_INTEN_VID2URUN_MASK		(0x00000200)
+#define PDP_INTENAB_INTEN_VID2URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID2URUN_SHIFT		(9)
+#define PDP_INTENAB_INTEN_VID2URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VID1URUN
+*/
+#define PDP_INTENAB_INTEN_VID1URUN_MASK		(0x00000100)
+#define PDP_INTENAB_INTEN_VID1URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VID1URUN_SHIFT		(8)
+#define PDP_INTENAB_INTEN_VID1URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH4URUN
+*/
+#define PDP_INTENAB_INTEN_GRPH4URUN_MASK		(0x00000080)
+#define PDP_INTENAB_INTEN_GRPH4URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH4URUN_SHIFT		(7)
+#define PDP_INTENAB_INTEN_GRPH4URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH3URUN
+*/
+#define PDP_INTENAB_INTEN_GRPH3URUN_MASK		(0x00000040)
+#define PDP_INTENAB_INTEN_GRPH3URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH3URUN_SHIFT		(6)
+#define PDP_INTENAB_INTEN_GRPH3URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH2URUN
+*/
+#define PDP_INTENAB_INTEN_GRPH2URUN_MASK		(0x00000020)
+#define PDP_INTENAB_INTEN_GRPH2URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH2URUN_SHIFT		(5)
+#define PDP_INTENAB_INTEN_GRPH2URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_GRPH1URUN
+*/
+#define PDP_INTENAB_INTEN_GRPH1URUN_MASK		(0x00000010)
+#define PDP_INTENAB_INTEN_GRPH1URUN_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_GRPH1URUN_SHIFT		(4)
+#define PDP_INTENAB_INTEN_GRPH1URUN_LENGTH		(1)
+#define PDP_INTENAB_INTEN_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK1
+*/
+#define PDP_INTENAB_INTEN_VBLNK1_MASK		(0x00000008)
+#define PDP_INTENAB_INTEN_VBLNK1_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VBLNK1_SHIFT		(3)
+#define PDP_INTENAB_INTEN_VBLNK1_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_VBLNK0
+*/
+#define PDP_INTENAB_INTEN_VBLNK0_MASK		(0x00000004)
+#define PDP_INTENAB_INTEN_VBLNK0_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_VBLNK0_SHIFT		(2)
+#define PDP_INTENAB_INTEN_VBLNK0_LENGTH		(1)
+#define PDP_INTENAB_INTEN_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK1
+*/
+#define PDP_INTENAB_INTEN_HBLNK1_MASK		(0x00000002)
+#define PDP_INTENAB_INTEN_HBLNK1_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_HBLNK1_SHIFT		(1)
+#define PDP_INTENAB_INTEN_HBLNK1_LENGTH		(1)
+#define PDP_INTENAB_INTEN_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTENAB, INTEN_HBLNK0
+*/
+#define PDP_INTENAB_INTEN_HBLNK0_MASK		(0x00000001)
+#define PDP_INTENAB_INTEN_HBLNK0_LSBMASK		(0x00000001)
+#define PDP_INTENAB_INTEN_HBLNK0_SHIFT		(0)
+#define PDP_INTENAB_INTEN_HBLNK0_LENGTH		(1)
+#define PDP_INTENAB_INTEN_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_INTCLR_OFFSET		(0x07A4)
+
+/* PDP, INTCLR, INTCLR_VID4ORUN
+*/
+#define PDP_INTCLR_INTCLR_VID4ORUN_MASK		(0x00080000)
+#define PDP_INTCLR_INTCLR_VID4ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID4ORUN_SHIFT		(19)
+#define PDP_INTCLR_INTCLR_VID4ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3ORUN
+*/
+#define PDP_INTCLR_INTCLR_VID3ORUN_MASK		(0x00040000)
+#define PDP_INTCLR_INTCLR_VID3ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID3ORUN_SHIFT		(18)
+#define PDP_INTCLR_INTCLR_VID3ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2ORUN
+*/
+#define PDP_INTCLR_INTCLR_VID2ORUN_MASK		(0x00020000)
+#define PDP_INTCLR_INTCLR_VID2ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID2ORUN_SHIFT		(17)
+#define PDP_INTCLR_INTCLR_VID2ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1ORUN
+*/
+#define PDP_INTCLR_INTCLR_VID1ORUN_MASK		(0x00010000)
+#define PDP_INTCLR_INTCLR_VID1ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID1ORUN_SHIFT		(16)
+#define PDP_INTCLR_INTCLR_VID1ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4ORUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH4ORUN_MASK		(0x00008000)
+#define PDP_INTCLR_INTCLR_GRPH4ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH4ORUN_SHIFT		(15)
+#define PDP_INTCLR_INTCLR_GRPH4ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH4ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3ORUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH3ORUN_MASK		(0x00004000)
+#define PDP_INTCLR_INTCLR_GRPH3ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH3ORUN_SHIFT		(14)
+#define PDP_INTCLR_INTCLR_GRPH3ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH3ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2ORUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH2ORUN_MASK		(0x00002000)
+#define PDP_INTCLR_INTCLR_GRPH2ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH2ORUN_SHIFT		(13)
+#define PDP_INTCLR_INTCLR_GRPH2ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH2ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1ORUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH1ORUN_MASK		(0x00001000)
+#define PDP_INTCLR_INTCLR_GRPH1ORUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH1ORUN_SHIFT		(12)
+#define PDP_INTCLR_INTCLR_GRPH1ORUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH1ORUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID4URUN
+*/
+#define PDP_INTCLR_INTCLR_VID4URUN_MASK		(0x00000800)
+#define PDP_INTCLR_INTCLR_VID4URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID4URUN_SHIFT		(11)
+#define PDP_INTCLR_INTCLR_VID4URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID3URUN
+*/
+#define PDP_INTCLR_INTCLR_VID3URUN_MASK		(0x00000400)
+#define PDP_INTCLR_INTCLR_VID3URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID3URUN_SHIFT		(10)
+#define PDP_INTCLR_INTCLR_VID3URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID2URUN
+*/
+#define PDP_INTCLR_INTCLR_VID2URUN_MASK		(0x00000200)
+#define PDP_INTCLR_INTCLR_VID2URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID2URUN_SHIFT		(9)
+#define PDP_INTCLR_INTCLR_VID2URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VID1URUN
+*/
+#define PDP_INTCLR_INTCLR_VID1URUN_MASK		(0x00000100)
+#define PDP_INTCLR_INTCLR_VID1URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VID1URUN_SHIFT		(8)
+#define PDP_INTCLR_INTCLR_VID1URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VID1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH4URUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH4URUN_MASK		(0x00000080)
+#define PDP_INTCLR_INTCLR_GRPH4URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH4URUN_SHIFT		(7)
+#define PDP_INTCLR_INTCLR_GRPH4URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH4URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH3URUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH3URUN_MASK		(0x00000040)
+#define PDP_INTCLR_INTCLR_GRPH3URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH3URUN_SHIFT		(6)
+#define PDP_INTCLR_INTCLR_GRPH3URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH3URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH2URUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH2URUN_MASK		(0x00000020)
+#define PDP_INTCLR_INTCLR_GRPH2URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH2URUN_SHIFT		(5)
+#define PDP_INTCLR_INTCLR_GRPH2URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH2URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_GRPH1URUN
+*/
+#define PDP_INTCLR_INTCLR_GRPH1URUN_MASK		(0x00000010)
+#define PDP_INTCLR_INTCLR_GRPH1URUN_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_GRPH1URUN_SHIFT		(4)
+#define PDP_INTCLR_INTCLR_GRPH1URUN_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_GRPH1URUN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK1
+*/
+#define PDP_INTCLR_INTCLR_VBLNK1_MASK		(0x00000008)
+#define PDP_INTCLR_INTCLR_VBLNK1_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VBLNK1_SHIFT		(3)
+#define PDP_INTCLR_INTCLR_VBLNK1_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_VBLNK0
+*/
+#define PDP_INTCLR_INTCLR_VBLNK0_MASK		(0x00000004)
+#define PDP_INTCLR_INTCLR_VBLNK0_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_VBLNK0_SHIFT		(2)
+#define PDP_INTCLR_INTCLR_VBLNK0_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_VBLNK0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK1
+*/
+#define PDP_INTCLR_INTCLR_HBLNK1_MASK		(0x00000002)
+#define PDP_INTCLR_INTCLR_HBLNK1_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_HBLNK1_SHIFT		(1)
+#define PDP_INTCLR_INTCLR_HBLNK1_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_HBLNK1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, INTCLR, INTCLR_HBLNK0
+*/
+#define PDP_INTCLR_INTCLR_HBLNK0_MASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_HBLNK0_LSBMASK		(0x00000001)
+#define PDP_INTCLR_INTCLR_HBLNK0_SHIFT		(0)
+#define PDP_INTCLR_INTCLR_HBLNK0_LENGTH		(1)
+#define PDP_INTCLR_INTCLR_HBLNK0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_MEMCTRL_OFFSET		(0x07A8)
+
+/* PDP, MEMCTRL, MEMREFRESH
+*/
+#define PDP_MEMCTRL_MEMREFRESH_MASK		(0xC0000000)
+#define PDP_MEMCTRL_MEMREFRESH_LSBMASK		(0x00000003)
+#define PDP_MEMCTRL_MEMREFRESH_SHIFT		(30)
+#define PDP_MEMCTRL_MEMREFRESH_LENGTH		(2)
+#define PDP_MEMCTRL_MEMREFRESH_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEMCTRL, BURSTLEN
+*/
+#define PDP_MEMCTRL_BURSTLEN_MASK		(0x000000FF)
+#define PDP_MEMCTRL_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_MEMCTRL_BURSTLEN_SHIFT		(0)
+#define PDP_MEMCTRL_BURSTLEN_LENGTH		(8)
+#define PDP_MEMCTRL_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_MEM_THRESH_OFFSET		(0x07AC)
+
+/* PDP, MEM_THRESH, UVTHRESHOLD
+*/
+#define PDP_MEM_THRESH_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_MEM_THRESH_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_MEM_THRESH_UVTHRESHOLD_SHIFT		(24)
+#define PDP_MEM_THRESH_UVTHRESHOLD_LENGTH		(8)
+#define PDP_MEM_THRESH_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, YTHRESHOLD
+*/
+#define PDP_MEM_THRESH_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_MEM_THRESH_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_MEM_THRESH_YTHRESHOLD_SHIFT		(12)
+#define PDP_MEM_THRESH_YTHRESHOLD_LENGTH		(9)
+#define PDP_MEM_THRESH_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, MEM_THRESH, THRESHOLD
+*/
+#define PDP_MEM_THRESH_THRESHOLD_MASK		(0x000001FF)
+#define PDP_MEM_THRESH_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_MEM_THRESH_THRESHOLD_SHIFT		(0)
+#define PDP_MEM_THRESH_THRESHOLD_LENGTH		(9)
+#define PDP_MEM_THRESH_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_ALTERNATE_3D_CTRL_OFFSET		(0x07B0)
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_ON
+*/
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_ON_MASK		(0x00000010)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LSBMASK		(0x00000001)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SHIFT		(4)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_ON_LENGTH		(1)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_ON_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, ALTERNATE_3D_CTRL, ALT3D_BLENDSEL
+*/
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_MASK		(0x00000007)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LSBMASK		(0x00000007)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SHIFT		(0)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_LENGTH		(3)
+#define PDP_ALTERNATE_3D_CTRL_ALT3D_BLENDSEL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA0_R_OFFSET		(0x07B4)
+
+/* PDP, GAMMA0_R, GAMMA0_R
+*/
+#define PDP_GAMMA0_R_GAMMA0_R_MASK		(0x000003FF)
+#define PDP_GAMMA0_R_GAMMA0_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA0_R_GAMMA0_R_SHIFT		(0)
+#define PDP_GAMMA0_R_GAMMA0_R_LENGTH		(10)
+#define PDP_GAMMA0_R_GAMMA0_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA0_GB_OFFSET		(0x07B8)
+
+/* PDP, GAMMA0_GB, GAMMA0_G
+*/
+#define PDP_GAMMA0_GB_GAMMA0_G_MASK		(0x03FF0000)
+#define PDP_GAMMA0_GB_GAMMA0_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA0_GB_GAMMA0_G_SHIFT		(16)
+#define PDP_GAMMA0_GB_GAMMA0_G_LENGTH		(10)
+#define PDP_GAMMA0_GB_GAMMA0_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA0_GB, GAMMA0_B
+*/
+#define PDP_GAMMA0_GB_GAMMA0_B_MASK		(0x000003FF)
+#define PDP_GAMMA0_GB_GAMMA0_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA0_GB_GAMMA0_B_SHIFT		(0)
+#define PDP_GAMMA0_GB_GAMMA0_B_LENGTH		(10)
+#define PDP_GAMMA0_GB_GAMMA0_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA1_R_OFFSET		(0x07BC)
+
+/* PDP, GAMMA1_R, GAMMA1_R
+*/
+#define PDP_GAMMA1_R_GAMMA1_R_MASK		(0x000003FF)
+#define PDP_GAMMA1_R_GAMMA1_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA1_R_GAMMA1_R_SHIFT		(0)
+#define PDP_GAMMA1_R_GAMMA1_R_LENGTH		(10)
+#define PDP_GAMMA1_R_GAMMA1_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA1_GB_OFFSET		(0x07C0)
+
+/* PDP, GAMMA1_GB, GAMMA1_G
+*/
+#define PDP_GAMMA1_GB_GAMMA1_G_MASK		(0x03FF0000)
+#define PDP_GAMMA1_GB_GAMMA1_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA1_GB_GAMMA1_G_SHIFT		(16)
+#define PDP_GAMMA1_GB_GAMMA1_G_LENGTH		(10)
+#define PDP_GAMMA1_GB_GAMMA1_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA1_GB, GAMMA1_B
+*/
+#define PDP_GAMMA1_GB_GAMMA1_B_MASK		(0x000003FF)
+#define PDP_GAMMA1_GB_GAMMA1_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA1_GB_GAMMA1_B_SHIFT		(0)
+#define PDP_GAMMA1_GB_GAMMA1_B_LENGTH		(10)
+#define PDP_GAMMA1_GB_GAMMA1_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA2_R_OFFSET		(0x07C4)
+
+/* PDP, GAMMA2_R, GAMMA2_R
+*/
+#define PDP_GAMMA2_R_GAMMA2_R_MASK		(0x000003FF)
+#define PDP_GAMMA2_R_GAMMA2_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA2_R_GAMMA2_R_SHIFT		(0)
+#define PDP_GAMMA2_R_GAMMA2_R_LENGTH		(10)
+#define PDP_GAMMA2_R_GAMMA2_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA2_GB_OFFSET		(0x07C8)
+
+/* PDP, GAMMA2_GB, GAMMA2_G
+*/
+#define PDP_GAMMA2_GB_GAMMA2_G_MASK		(0x03FF0000)
+#define PDP_GAMMA2_GB_GAMMA2_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA2_GB_GAMMA2_G_SHIFT		(16)
+#define PDP_GAMMA2_GB_GAMMA2_G_LENGTH		(10)
+#define PDP_GAMMA2_GB_GAMMA2_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA2_GB, GAMMA2_B
+*/
+#define PDP_GAMMA2_GB_GAMMA2_B_MASK		(0x000003FF)
+#define PDP_GAMMA2_GB_GAMMA2_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA2_GB_GAMMA2_B_SHIFT		(0)
+#define PDP_GAMMA2_GB_GAMMA2_B_LENGTH		(10)
+#define PDP_GAMMA2_GB_GAMMA2_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA3_R_OFFSET		(0x07CC)
+
+/* PDP, GAMMA3_R, GAMMA3_R
+*/
+#define PDP_GAMMA3_R_GAMMA3_R_MASK		(0x000003FF)
+#define PDP_GAMMA3_R_GAMMA3_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA3_R_GAMMA3_R_SHIFT		(0)
+#define PDP_GAMMA3_R_GAMMA3_R_LENGTH		(10)
+#define PDP_GAMMA3_R_GAMMA3_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA3_GB_OFFSET		(0x07D0)
+
+/* PDP, GAMMA3_GB, GAMMA3_G
+*/
+#define PDP_GAMMA3_GB_GAMMA3_G_MASK		(0x03FF0000)
+#define PDP_GAMMA3_GB_GAMMA3_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA3_GB_GAMMA3_G_SHIFT		(16)
+#define PDP_GAMMA3_GB_GAMMA3_G_LENGTH		(10)
+#define PDP_GAMMA3_GB_GAMMA3_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA3_GB, GAMMA3_B
+*/
+#define PDP_GAMMA3_GB_GAMMA3_B_MASK		(0x000003FF)
+#define PDP_GAMMA3_GB_GAMMA3_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA3_GB_GAMMA3_B_SHIFT		(0)
+#define PDP_GAMMA3_GB_GAMMA3_B_LENGTH		(10)
+#define PDP_GAMMA3_GB_GAMMA3_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA4_R_OFFSET		(0x07D4)
+
+/* PDP, GAMMA4_R, GAMMA4_R
+*/
+#define PDP_GAMMA4_R_GAMMA4_R_MASK		(0x000003FF)
+#define PDP_GAMMA4_R_GAMMA4_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA4_R_GAMMA4_R_SHIFT		(0)
+#define PDP_GAMMA4_R_GAMMA4_R_LENGTH		(10)
+#define PDP_GAMMA4_R_GAMMA4_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA4_GB_OFFSET		(0x07D8)
+
+/* PDP, GAMMA4_GB, GAMMA4_G
+*/
+#define PDP_GAMMA4_GB_GAMMA4_G_MASK		(0x03FF0000)
+#define PDP_GAMMA4_GB_GAMMA4_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA4_GB_GAMMA4_G_SHIFT		(16)
+#define PDP_GAMMA4_GB_GAMMA4_G_LENGTH		(10)
+#define PDP_GAMMA4_GB_GAMMA4_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA4_GB, GAMMA4_B
+*/
+#define PDP_GAMMA4_GB_GAMMA4_B_MASK		(0x000003FF)
+#define PDP_GAMMA4_GB_GAMMA4_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA4_GB_GAMMA4_B_SHIFT		(0)
+#define PDP_GAMMA4_GB_GAMMA4_B_LENGTH		(10)
+#define PDP_GAMMA4_GB_GAMMA4_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA5_R_OFFSET		(0x07DC)
+
+/* PDP, GAMMA5_R, GAMMA5_R
+*/
+#define PDP_GAMMA5_R_GAMMA5_R_MASK		(0x000003FF)
+#define PDP_GAMMA5_R_GAMMA5_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA5_R_GAMMA5_R_SHIFT		(0)
+#define PDP_GAMMA5_R_GAMMA5_R_LENGTH		(10)
+#define PDP_GAMMA5_R_GAMMA5_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA5_GB_OFFSET		(0x07E0)
+
+/* PDP, GAMMA5_GB, GAMMA5_G
+*/
+#define PDP_GAMMA5_GB_GAMMA5_G_MASK		(0x03FF0000)
+#define PDP_GAMMA5_GB_GAMMA5_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA5_GB_GAMMA5_G_SHIFT		(16)
+#define PDP_GAMMA5_GB_GAMMA5_G_LENGTH		(10)
+#define PDP_GAMMA5_GB_GAMMA5_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA5_GB, GAMMA5_B
+*/
+#define PDP_GAMMA5_GB_GAMMA5_B_MASK		(0x000003FF)
+#define PDP_GAMMA5_GB_GAMMA5_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA5_GB_GAMMA5_B_SHIFT		(0)
+#define PDP_GAMMA5_GB_GAMMA5_B_LENGTH		(10)
+#define PDP_GAMMA5_GB_GAMMA5_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA6_R_OFFSET		(0x07E4)
+
+/* PDP, GAMMA6_R, GAMMA6_R
+*/
+#define PDP_GAMMA6_R_GAMMA6_R_MASK		(0x000003FF)
+#define PDP_GAMMA6_R_GAMMA6_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA6_R_GAMMA6_R_SHIFT		(0)
+#define PDP_GAMMA6_R_GAMMA6_R_LENGTH		(10)
+#define PDP_GAMMA6_R_GAMMA6_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA6_GB_OFFSET		(0x07E8)
+
+/* PDP, GAMMA6_GB, GAMMA6_G
+*/
+#define PDP_GAMMA6_GB_GAMMA6_G_MASK		(0x03FF0000)
+#define PDP_GAMMA6_GB_GAMMA6_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA6_GB_GAMMA6_G_SHIFT		(16)
+#define PDP_GAMMA6_GB_GAMMA6_G_LENGTH		(10)
+#define PDP_GAMMA6_GB_GAMMA6_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA6_GB, GAMMA6_B
+*/
+#define PDP_GAMMA6_GB_GAMMA6_B_MASK		(0x000003FF)
+#define PDP_GAMMA6_GB_GAMMA6_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA6_GB_GAMMA6_B_SHIFT		(0)
+#define PDP_GAMMA6_GB_GAMMA6_B_LENGTH		(10)
+#define PDP_GAMMA6_GB_GAMMA6_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA7_R_OFFSET		(0x07EC)
+
+/* PDP, GAMMA7_R, GAMMA7_R
+*/
+#define PDP_GAMMA7_R_GAMMA7_R_MASK		(0x000003FF)
+#define PDP_GAMMA7_R_GAMMA7_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA7_R_GAMMA7_R_SHIFT		(0)
+#define PDP_GAMMA7_R_GAMMA7_R_LENGTH		(10)
+#define PDP_GAMMA7_R_GAMMA7_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA7_GB_OFFSET		(0x07F0)
+
+/* PDP, GAMMA7_GB, GAMMA7_G
+*/
+#define PDP_GAMMA7_GB_GAMMA7_G_MASK		(0x03FF0000)
+#define PDP_GAMMA7_GB_GAMMA7_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA7_GB_GAMMA7_G_SHIFT		(16)
+#define PDP_GAMMA7_GB_GAMMA7_G_LENGTH		(10)
+#define PDP_GAMMA7_GB_GAMMA7_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA7_GB, GAMMA7_B
+*/
+#define PDP_GAMMA7_GB_GAMMA7_B_MASK		(0x000003FF)
+#define PDP_GAMMA7_GB_GAMMA7_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA7_GB_GAMMA7_B_SHIFT		(0)
+#define PDP_GAMMA7_GB_GAMMA7_B_LENGTH		(10)
+#define PDP_GAMMA7_GB_GAMMA7_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA8_R_OFFSET		(0x07F4)
+
+/* PDP, GAMMA8_R, GAMMA8_R
+*/
+#define PDP_GAMMA8_R_GAMMA8_R_MASK		(0x000003FF)
+#define PDP_GAMMA8_R_GAMMA8_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA8_R_GAMMA8_R_SHIFT		(0)
+#define PDP_GAMMA8_R_GAMMA8_R_LENGTH		(10)
+#define PDP_GAMMA8_R_GAMMA8_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA8_GB_OFFSET		(0x07F8)
+
+/* PDP, GAMMA8_GB, GAMMA8_G
+*/
+#define PDP_GAMMA8_GB_GAMMA8_G_MASK		(0x03FF0000)
+#define PDP_GAMMA8_GB_GAMMA8_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA8_GB_GAMMA8_G_SHIFT		(16)
+#define PDP_GAMMA8_GB_GAMMA8_G_LENGTH		(10)
+#define PDP_GAMMA8_GB_GAMMA8_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA8_GB, GAMMA8_B
+*/
+#define PDP_GAMMA8_GB_GAMMA8_B_MASK		(0x000003FF)
+#define PDP_GAMMA8_GB_GAMMA8_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA8_GB_GAMMA8_B_SHIFT		(0)
+#define PDP_GAMMA8_GB_GAMMA8_B_LENGTH		(10)
+#define PDP_GAMMA8_GB_GAMMA8_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA9_R_OFFSET		(0x07FC)
+
+/* PDP, GAMMA9_R, GAMMA9_R
+*/
+#define PDP_GAMMA9_R_GAMMA9_R_MASK		(0x000003FF)
+#define PDP_GAMMA9_R_GAMMA9_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA9_R_GAMMA9_R_SHIFT		(0)
+#define PDP_GAMMA9_R_GAMMA9_R_LENGTH		(10)
+#define PDP_GAMMA9_R_GAMMA9_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA9_GB_OFFSET		(0x0800)
+
+/* PDP, GAMMA9_GB, GAMMA9_G
+*/
+#define PDP_GAMMA9_GB_GAMMA9_G_MASK		(0x03FF0000)
+#define PDP_GAMMA9_GB_GAMMA9_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA9_GB_GAMMA9_G_SHIFT		(16)
+#define PDP_GAMMA9_GB_GAMMA9_G_LENGTH		(10)
+#define PDP_GAMMA9_GB_GAMMA9_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA9_GB, GAMMA9_B
+*/
+#define PDP_GAMMA9_GB_GAMMA9_B_MASK		(0x000003FF)
+#define PDP_GAMMA9_GB_GAMMA9_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA9_GB_GAMMA9_B_SHIFT		(0)
+#define PDP_GAMMA9_GB_GAMMA9_B_LENGTH		(10)
+#define PDP_GAMMA9_GB_GAMMA9_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA10_R_OFFSET		(0x0804)
+
+/* PDP, GAMMA10_R, GAMMA10_R
+*/
+#define PDP_GAMMA10_R_GAMMA10_R_MASK		(0x000003FF)
+#define PDP_GAMMA10_R_GAMMA10_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA10_R_GAMMA10_R_SHIFT		(0)
+#define PDP_GAMMA10_R_GAMMA10_R_LENGTH		(10)
+#define PDP_GAMMA10_R_GAMMA10_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA10_GB_OFFSET		(0x0808)
+
+/* PDP, GAMMA10_GB, GAMMA10_G
+*/
+#define PDP_GAMMA10_GB_GAMMA10_G_MASK		(0x03FF0000)
+#define PDP_GAMMA10_GB_GAMMA10_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA10_GB_GAMMA10_G_SHIFT		(16)
+#define PDP_GAMMA10_GB_GAMMA10_G_LENGTH		(10)
+#define PDP_GAMMA10_GB_GAMMA10_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA10_GB, GAMMA10_B
+*/
+#define PDP_GAMMA10_GB_GAMMA10_B_MASK		(0x000003FF)
+#define PDP_GAMMA10_GB_GAMMA10_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA10_GB_GAMMA10_B_SHIFT		(0)
+#define PDP_GAMMA10_GB_GAMMA10_B_LENGTH		(10)
+#define PDP_GAMMA10_GB_GAMMA10_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA11_R_OFFSET		(0x080C)
+
+/* PDP, GAMMA11_R, GAMMA11_R
+*/
+#define PDP_GAMMA11_R_GAMMA11_R_MASK		(0x000003FF)
+#define PDP_GAMMA11_R_GAMMA11_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA11_R_GAMMA11_R_SHIFT		(0)
+#define PDP_GAMMA11_R_GAMMA11_R_LENGTH		(10)
+#define PDP_GAMMA11_R_GAMMA11_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA11_GB_OFFSET		(0x0810)
+
+/* PDP, GAMMA11_GB, GAMMA11_G
+*/
+#define PDP_GAMMA11_GB_GAMMA11_G_MASK		(0x03FF0000)
+#define PDP_GAMMA11_GB_GAMMA11_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA11_GB_GAMMA11_G_SHIFT		(16)
+#define PDP_GAMMA11_GB_GAMMA11_G_LENGTH		(10)
+#define PDP_GAMMA11_GB_GAMMA11_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA11_GB, GAMMA11_B
+*/
+#define PDP_GAMMA11_GB_GAMMA11_B_MASK		(0x000003FF)
+#define PDP_GAMMA11_GB_GAMMA11_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA11_GB_GAMMA11_B_SHIFT		(0)
+#define PDP_GAMMA11_GB_GAMMA11_B_LENGTH		(10)
+#define PDP_GAMMA11_GB_GAMMA11_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA12_R_OFFSET		(0x0814)
+
+/* PDP, GAMMA12_R, GAMMA12_R
+*/
+#define PDP_GAMMA12_R_GAMMA12_R_MASK		(0x000003FF)
+#define PDP_GAMMA12_R_GAMMA12_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA12_R_GAMMA12_R_SHIFT		(0)
+#define PDP_GAMMA12_R_GAMMA12_R_LENGTH		(10)
+#define PDP_GAMMA12_R_GAMMA12_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA12_GB_OFFSET		(0x0818)
+
+/* PDP, GAMMA12_GB, GAMMA12_G
+*/
+#define PDP_GAMMA12_GB_GAMMA12_G_MASK		(0x03FF0000)
+#define PDP_GAMMA12_GB_GAMMA12_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA12_GB_GAMMA12_G_SHIFT		(16)
+#define PDP_GAMMA12_GB_GAMMA12_G_LENGTH		(10)
+#define PDP_GAMMA12_GB_GAMMA12_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA12_GB, GAMMA12_B
+*/
+#define PDP_GAMMA12_GB_GAMMA12_B_MASK		(0x000003FF)
+#define PDP_GAMMA12_GB_GAMMA12_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA12_GB_GAMMA12_B_SHIFT		(0)
+#define PDP_GAMMA12_GB_GAMMA12_B_LENGTH		(10)
+#define PDP_GAMMA12_GB_GAMMA12_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA13_R_OFFSET		(0x081C)
+
+/* PDP, GAMMA13_R, GAMMA13_R
+*/
+#define PDP_GAMMA13_R_GAMMA13_R_MASK		(0x000003FF)
+#define PDP_GAMMA13_R_GAMMA13_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA13_R_GAMMA13_R_SHIFT		(0)
+#define PDP_GAMMA13_R_GAMMA13_R_LENGTH		(10)
+#define PDP_GAMMA13_R_GAMMA13_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA13_GB_OFFSET		(0x0820)
+
+/* PDP, GAMMA13_GB, GAMMA13_G
+*/
+#define PDP_GAMMA13_GB_GAMMA13_G_MASK		(0x03FF0000)
+#define PDP_GAMMA13_GB_GAMMA13_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA13_GB_GAMMA13_G_SHIFT		(16)
+#define PDP_GAMMA13_GB_GAMMA13_G_LENGTH		(10)
+#define PDP_GAMMA13_GB_GAMMA13_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA13_GB, GAMMA13_B
+*/
+#define PDP_GAMMA13_GB_GAMMA13_B_MASK		(0x000003FF)
+#define PDP_GAMMA13_GB_GAMMA13_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA13_GB_GAMMA13_B_SHIFT		(0)
+#define PDP_GAMMA13_GB_GAMMA13_B_LENGTH		(10)
+#define PDP_GAMMA13_GB_GAMMA13_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA14_R_OFFSET		(0x0824)
+
+/* PDP, GAMMA14_R, GAMMA14_R
+*/
+#define PDP_GAMMA14_R_GAMMA14_R_MASK		(0x000003FF)
+#define PDP_GAMMA14_R_GAMMA14_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA14_R_GAMMA14_R_SHIFT		(0)
+#define PDP_GAMMA14_R_GAMMA14_R_LENGTH		(10)
+#define PDP_GAMMA14_R_GAMMA14_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA14_GB_OFFSET		(0x0828)
+
+/* PDP, GAMMA14_GB, GAMMA14_G
+*/
+#define PDP_GAMMA14_GB_GAMMA14_G_MASK		(0x03FF0000)
+#define PDP_GAMMA14_GB_GAMMA14_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA14_GB_GAMMA14_G_SHIFT		(16)
+#define PDP_GAMMA14_GB_GAMMA14_G_LENGTH		(10)
+#define PDP_GAMMA14_GB_GAMMA14_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA14_GB, GAMMA14_B
+*/
+#define PDP_GAMMA14_GB_GAMMA14_B_MASK		(0x000003FF)
+#define PDP_GAMMA14_GB_GAMMA14_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA14_GB_GAMMA14_B_SHIFT		(0)
+#define PDP_GAMMA14_GB_GAMMA14_B_LENGTH		(10)
+#define PDP_GAMMA14_GB_GAMMA14_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA15_R_OFFSET		(0x082C)
+
+/* PDP, GAMMA15_R, GAMMA15_R
+*/
+#define PDP_GAMMA15_R_GAMMA15_R_MASK		(0x000003FF)
+#define PDP_GAMMA15_R_GAMMA15_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA15_R_GAMMA15_R_SHIFT		(0)
+#define PDP_GAMMA15_R_GAMMA15_R_LENGTH		(10)
+#define PDP_GAMMA15_R_GAMMA15_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA15_GB_OFFSET		(0x0830)
+
+/* PDP, GAMMA15_GB, GAMMA15_G
+*/
+#define PDP_GAMMA15_GB_GAMMA15_G_MASK		(0x03FF0000)
+#define PDP_GAMMA15_GB_GAMMA15_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA15_GB_GAMMA15_G_SHIFT		(16)
+#define PDP_GAMMA15_GB_GAMMA15_G_LENGTH		(10)
+#define PDP_GAMMA15_GB_GAMMA15_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA15_GB, GAMMA15_B
+*/
+#define PDP_GAMMA15_GB_GAMMA15_B_MASK		(0x000003FF)
+#define PDP_GAMMA15_GB_GAMMA15_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA15_GB_GAMMA15_B_SHIFT		(0)
+#define PDP_GAMMA15_GB_GAMMA15_B_LENGTH		(10)
+#define PDP_GAMMA15_GB_GAMMA15_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA16_R_OFFSET		(0x0834)
+
+/* PDP, GAMMA16_R, GAMMA16_R
+*/
+#define PDP_GAMMA16_R_GAMMA16_R_MASK		(0x000003FF)
+#define PDP_GAMMA16_R_GAMMA16_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA16_R_GAMMA16_R_SHIFT		(0)
+#define PDP_GAMMA16_R_GAMMA16_R_LENGTH		(10)
+#define PDP_GAMMA16_R_GAMMA16_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA16_GB_OFFSET		(0x0838)
+
+/* PDP, GAMMA16_GB, GAMMA16_G
+*/
+#define PDP_GAMMA16_GB_GAMMA16_G_MASK		(0x03FF0000)
+#define PDP_GAMMA16_GB_GAMMA16_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA16_GB_GAMMA16_G_SHIFT		(16)
+#define PDP_GAMMA16_GB_GAMMA16_G_LENGTH		(10)
+#define PDP_GAMMA16_GB_GAMMA16_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA16_GB, GAMMA16_B
+*/
+#define PDP_GAMMA16_GB_GAMMA16_B_MASK		(0x000003FF)
+#define PDP_GAMMA16_GB_GAMMA16_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA16_GB_GAMMA16_B_SHIFT		(0)
+#define PDP_GAMMA16_GB_GAMMA16_B_LENGTH		(10)
+#define PDP_GAMMA16_GB_GAMMA16_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA17_R_OFFSET		(0x083C)
+
+/* PDP, GAMMA17_R, GAMMA17_R
+*/
+#define PDP_GAMMA17_R_GAMMA17_R_MASK		(0x000003FF)
+#define PDP_GAMMA17_R_GAMMA17_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA17_R_GAMMA17_R_SHIFT		(0)
+#define PDP_GAMMA17_R_GAMMA17_R_LENGTH		(10)
+#define PDP_GAMMA17_R_GAMMA17_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA17_GB_OFFSET		(0x0840)
+
+/* PDP, GAMMA17_GB, GAMMA17_G
+*/
+#define PDP_GAMMA17_GB_GAMMA17_G_MASK		(0x03FF0000)
+#define PDP_GAMMA17_GB_GAMMA17_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA17_GB_GAMMA17_G_SHIFT		(16)
+#define PDP_GAMMA17_GB_GAMMA17_G_LENGTH		(10)
+#define PDP_GAMMA17_GB_GAMMA17_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA17_GB, GAMMA17_B
+*/
+#define PDP_GAMMA17_GB_GAMMA17_B_MASK		(0x000003FF)
+#define PDP_GAMMA17_GB_GAMMA17_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA17_GB_GAMMA17_B_SHIFT		(0)
+#define PDP_GAMMA17_GB_GAMMA17_B_LENGTH		(10)
+#define PDP_GAMMA17_GB_GAMMA17_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA18_R_OFFSET		(0x0844)
+
+/* PDP, GAMMA18_R, GAMMA18_R
+*/
+#define PDP_GAMMA18_R_GAMMA18_R_MASK		(0x000003FF)
+#define PDP_GAMMA18_R_GAMMA18_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA18_R_GAMMA18_R_SHIFT		(0)
+#define PDP_GAMMA18_R_GAMMA18_R_LENGTH		(10)
+#define PDP_GAMMA18_R_GAMMA18_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA18_GB_OFFSET		(0x0848)
+
+/* PDP, GAMMA18_GB, GAMMA18_G
+*/
+#define PDP_GAMMA18_GB_GAMMA18_G_MASK		(0x03FF0000)
+#define PDP_GAMMA18_GB_GAMMA18_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA18_GB_GAMMA18_G_SHIFT		(16)
+#define PDP_GAMMA18_GB_GAMMA18_G_LENGTH		(10)
+#define PDP_GAMMA18_GB_GAMMA18_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA18_GB, GAMMA18_B
+*/
+#define PDP_GAMMA18_GB_GAMMA18_B_MASK		(0x000003FF)
+#define PDP_GAMMA18_GB_GAMMA18_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA18_GB_GAMMA18_B_SHIFT		(0)
+#define PDP_GAMMA18_GB_GAMMA18_B_LENGTH		(10)
+#define PDP_GAMMA18_GB_GAMMA18_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA19_R_OFFSET		(0x084C)
+
+/* PDP, GAMMA19_R, GAMMA19_R
+*/
+#define PDP_GAMMA19_R_GAMMA19_R_MASK		(0x000003FF)
+#define PDP_GAMMA19_R_GAMMA19_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA19_R_GAMMA19_R_SHIFT		(0)
+#define PDP_GAMMA19_R_GAMMA19_R_LENGTH		(10)
+#define PDP_GAMMA19_R_GAMMA19_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA19_GB_OFFSET		(0x0850)
+
+/* PDP, GAMMA19_GB, GAMMA19_G
+*/
+#define PDP_GAMMA19_GB_GAMMA19_G_MASK		(0x03FF0000)
+#define PDP_GAMMA19_GB_GAMMA19_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA19_GB_GAMMA19_G_SHIFT		(16)
+#define PDP_GAMMA19_GB_GAMMA19_G_LENGTH		(10)
+#define PDP_GAMMA19_GB_GAMMA19_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA19_GB, GAMMA19_B
+*/
+#define PDP_GAMMA19_GB_GAMMA19_B_MASK		(0x000003FF)
+#define PDP_GAMMA19_GB_GAMMA19_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA19_GB_GAMMA19_B_SHIFT		(0)
+#define PDP_GAMMA19_GB_GAMMA19_B_LENGTH		(10)
+#define PDP_GAMMA19_GB_GAMMA19_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA20_R_OFFSET		(0x0854)
+
+/* PDP, GAMMA20_R, GAMMA20_R
+*/
+#define PDP_GAMMA20_R_GAMMA20_R_MASK		(0x000003FF)
+#define PDP_GAMMA20_R_GAMMA20_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA20_R_GAMMA20_R_SHIFT		(0)
+#define PDP_GAMMA20_R_GAMMA20_R_LENGTH		(10)
+#define PDP_GAMMA20_R_GAMMA20_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA20_GB_OFFSET		(0x0858)
+
+/* PDP, GAMMA20_GB, GAMMA20_G
+*/
+#define PDP_GAMMA20_GB_GAMMA20_G_MASK		(0x03FF0000)
+#define PDP_GAMMA20_GB_GAMMA20_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA20_GB_GAMMA20_G_SHIFT		(16)
+#define PDP_GAMMA20_GB_GAMMA20_G_LENGTH		(10)
+#define PDP_GAMMA20_GB_GAMMA20_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA20_GB, GAMMA20_B
+*/
+#define PDP_GAMMA20_GB_GAMMA20_B_MASK		(0x000003FF)
+#define PDP_GAMMA20_GB_GAMMA20_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA20_GB_GAMMA20_B_SHIFT		(0)
+#define PDP_GAMMA20_GB_GAMMA20_B_LENGTH		(10)
+#define PDP_GAMMA20_GB_GAMMA20_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA21_R_OFFSET		(0x085C)
+
+/* PDP, GAMMA21_R, GAMMA21_R
+*/
+#define PDP_GAMMA21_R_GAMMA21_R_MASK		(0x000003FF)
+#define PDP_GAMMA21_R_GAMMA21_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA21_R_GAMMA21_R_SHIFT		(0)
+#define PDP_GAMMA21_R_GAMMA21_R_LENGTH		(10)
+#define PDP_GAMMA21_R_GAMMA21_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA21_GB_OFFSET		(0x0860)
+
+/* PDP, GAMMA21_GB, GAMMA21_G
+*/
+#define PDP_GAMMA21_GB_GAMMA21_G_MASK		(0x03FF0000)
+#define PDP_GAMMA21_GB_GAMMA21_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA21_GB_GAMMA21_G_SHIFT		(16)
+#define PDP_GAMMA21_GB_GAMMA21_G_LENGTH		(10)
+#define PDP_GAMMA21_GB_GAMMA21_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA21_GB, GAMMA21_B
+*/
+#define PDP_GAMMA21_GB_GAMMA21_B_MASK		(0x000003FF)
+#define PDP_GAMMA21_GB_GAMMA21_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA21_GB_GAMMA21_B_SHIFT		(0)
+#define PDP_GAMMA21_GB_GAMMA21_B_LENGTH		(10)
+#define PDP_GAMMA21_GB_GAMMA21_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA22_R_OFFSET		(0x0864)
+
+/* PDP, GAMMA22_R, GAMMA22_R
+*/
+#define PDP_GAMMA22_R_GAMMA22_R_MASK		(0x000003FF)
+#define PDP_GAMMA22_R_GAMMA22_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA22_R_GAMMA22_R_SHIFT		(0)
+#define PDP_GAMMA22_R_GAMMA22_R_LENGTH		(10)
+#define PDP_GAMMA22_R_GAMMA22_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA22_GB_OFFSET		(0x0868)
+
+/* PDP, GAMMA22_GB, GAMMA22_G
+*/
+#define PDP_GAMMA22_GB_GAMMA22_G_MASK		(0x03FF0000)
+#define PDP_GAMMA22_GB_GAMMA22_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA22_GB_GAMMA22_G_SHIFT		(16)
+#define PDP_GAMMA22_GB_GAMMA22_G_LENGTH		(10)
+#define PDP_GAMMA22_GB_GAMMA22_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA22_GB, GAMMA22_B
+*/
+#define PDP_GAMMA22_GB_GAMMA22_B_MASK		(0x000003FF)
+#define PDP_GAMMA22_GB_GAMMA22_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA22_GB_GAMMA22_B_SHIFT		(0)
+#define PDP_GAMMA22_GB_GAMMA22_B_LENGTH		(10)
+#define PDP_GAMMA22_GB_GAMMA22_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA23_R_OFFSET		(0x086C)
+
+/* PDP, GAMMA23_R, GAMMA23_R
+*/
+#define PDP_GAMMA23_R_GAMMA23_R_MASK		(0x000003FF)
+#define PDP_GAMMA23_R_GAMMA23_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA23_R_GAMMA23_R_SHIFT		(0)
+#define PDP_GAMMA23_R_GAMMA23_R_LENGTH		(10)
+#define PDP_GAMMA23_R_GAMMA23_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA23_GB_OFFSET		(0x0870)
+
+/* PDP, GAMMA23_GB, GAMMA23_G
+*/
+#define PDP_GAMMA23_GB_GAMMA23_G_MASK		(0x03FF0000)
+#define PDP_GAMMA23_GB_GAMMA23_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA23_GB_GAMMA23_G_SHIFT		(16)
+#define PDP_GAMMA23_GB_GAMMA23_G_LENGTH		(10)
+#define PDP_GAMMA23_GB_GAMMA23_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA23_GB, GAMMA23_B
+*/
+#define PDP_GAMMA23_GB_GAMMA23_B_MASK		(0x000003FF)
+#define PDP_GAMMA23_GB_GAMMA23_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA23_GB_GAMMA23_B_SHIFT		(0)
+#define PDP_GAMMA23_GB_GAMMA23_B_LENGTH		(10)
+#define PDP_GAMMA23_GB_GAMMA23_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA24_R_OFFSET		(0x0874)
+
+/* PDP, GAMMA24_R, GAMMA24_R
+*/
+#define PDP_GAMMA24_R_GAMMA24_R_MASK		(0x000003FF)
+#define PDP_GAMMA24_R_GAMMA24_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA24_R_GAMMA24_R_SHIFT		(0)
+#define PDP_GAMMA24_R_GAMMA24_R_LENGTH		(10)
+#define PDP_GAMMA24_R_GAMMA24_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA24_GB_OFFSET		(0x0878)
+
+/* PDP, GAMMA24_GB, GAMMA24_G
+*/
+#define PDP_GAMMA24_GB_GAMMA24_G_MASK		(0x03FF0000)
+#define PDP_GAMMA24_GB_GAMMA24_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA24_GB_GAMMA24_G_SHIFT		(16)
+#define PDP_GAMMA24_GB_GAMMA24_G_LENGTH		(10)
+#define PDP_GAMMA24_GB_GAMMA24_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA24_GB, GAMMA24_B
+*/
+#define PDP_GAMMA24_GB_GAMMA24_B_MASK		(0x000003FF)
+#define PDP_GAMMA24_GB_GAMMA24_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA24_GB_GAMMA24_B_SHIFT		(0)
+#define PDP_GAMMA24_GB_GAMMA24_B_LENGTH		(10)
+#define PDP_GAMMA24_GB_GAMMA24_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA25_R_OFFSET		(0x087C)
+
+/* PDP, GAMMA25_R, GAMMA25_R
+*/
+#define PDP_GAMMA25_R_GAMMA25_R_MASK		(0x000003FF)
+#define PDP_GAMMA25_R_GAMMA25_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA25_R_GAMMA25_R_SHIFT		(0)
+#define PDP_GAMMA25_R_GAMMA25_R_LENGTH		(10)
+#define PDP_GAMMA25_R_GAMMA25_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA25_GB_OFFSET		(0x0880)
+
+/* PDP, GAMMA25_GB, GAMMA25_G
+*/
+#define PDP_GAMMA25_GB_GAMMA25_G_MASK		(0x03FF0000)
+#define PDP_GAMMA25_GB_GAMMA25_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA25_GB_GAMMA25_G_SHIFT		(16)
+#define PDP_GAMMA25_GB_GAMMA25_G_LENGTH		(10)
+#define PDP_GAMMA25_GB_GAMMA25_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA25_GB, GAMMA25_B
+*/
+#define PDP_GAMMA25_GB_GAMMA25_B_MASK		(0x000003FF)
+#define PDP_GAMMA25_GB_GAMMA25_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA25_GB_GAMMA25_B_SHIFT		(0)
+#define PDP_GAMMA25_GB_GAMMA25_B_LENGTH		(10)
+#define PDP_GAMMA25_GB_GAMMA25_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA26_R_OFFSET		(0x0884)
+
+/* PDP, GAMMA26_R, GAMMA26_R
+*/
+#define PDP_GAMMA26_R_GAMMA26_R_MASK		(0x000003FF)
+#define PDP_GAMMA26_R_GAMMA26_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA26_R_GAMMA26_R_SHIFT		(0)
+#define PDP_GAMMA26_R_GAMMA26_R_LENGTH		(10)
+#define PDP_GAMMA26_R_GAMMA26_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA26_GB_OFFSET		(0x0888)
+
+/* PDP, GAMMA26_GB, GAMMA26_G
+*/
+#define PDP_GAMMA26_GB_GAMMA26_G_MASK		(0x03FF0000)
+#define PDP_GAMMA26_GB_GAMMA26_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA26_GB_GAMMA26_G_SHIFT		(16)
+#define PDP_GAMMA26_GB_GAMMA26_G_LENGTH		(10)
+#define PDP_GAMMA26_GB_GAMMA26_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA26_GB, GAMMA26_B
+*/
+#define PDP_GAMMA26_GB_GAMMA26_B_MASK		(0x000003FF)
+#define PDP_GAMMA26_GB_GAMMA26_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA26_GB_GAMMA26_B_SHIFT		(0)
+#define PDP_GAMMA26_GB_GAMMA26_B_LENGTH		(10)
+#define PDP_GAMMA26_GB_GAMMA26_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA27_R_OFFSET		(0x088C)
+
+/* PDP, GAMMA27_R, GAMMA27_R
+*/
+#define PDP_GAMMA27_R_GAMMA27_R_MASK		(0x000003FF)
+#define PDP_GAMMA27_R_GAMMA27_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA27_R_GAMMA27_R_SHIFT		(0)
+#define PDP_GAMMA27_R_GAMMA27_R_LENGTH		(10)
+#define PDP_GAMMA27_R_GAMMA27_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA27_GB_OFFSET		(0x0890)
+
+/* PDP, GAMMA27_GB, GAMMA27_G
+*/
+#define PDP_GAMMA27_GB_GAMMA27_G_MASK		(0x03FF0000)
+#define PDP_GAMMA27_GB_GAMMA27_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA27_GB_GAMMA27_G_SHIFT		(16)
+#define PDP_GAMMA27_GB_GAMMA27_G_LENGTH		(10)
+#define PDP_GAMMA27_GB_GAMMA27_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA27_GB, GAMMA27_B
+*/
+#define PDP_GAMMA27_GB_GAMMA27_B_MASK		(0x000003FF)
+#define PDP_GAMMA27_GB_GAMMA27_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA27_GB_GAMMA27_B_SHIFT		(0)
+#define PDP_GAMMA27_GB_GAMMA27_B_LENGTH		(10)
+#define PDP_GAMMA27_GB_GAMMA27_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA28_R_OFFSET		(0x0894)
+
+/* PDP, GAMMA28_R, GAMMA28_R
+*/
+#define PDP_GAMMA28_R_GAMMA28_R_MASK		(0x000003FF)
+#define PDP_GAMMA28_R_GAMMA28_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA28_R_GAMMA28_R_SHIFT		(0)
+#define PDP_GAMMA28_R_GAMMA28_R_LENGTH		(10)
+#define PDP_GAMMA28_R_GAMMA28_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA28_GB_OFFSET		(0x0898)
+
+/* PDP, GAMMA28_GB, GAMMA28_G
+*/
+#define PDP_GAMMA28_GB_GAMMA28_G_MASK		(0x03FF0000)
+#define PDP_GAMMA28_GB_GAMMA28_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA28_GB_GAMMA28_G_SHIFT		(16)
+#define PDP_GAMMA28_GB_GAMMA28_G_LENGTH		(10)
+#define PDP_GAMMA28_GB_GAMMA28_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA28_GB, GAMMA28_B
+*/
+#define PDP_GAMMA28_GB_GAMMA28_B_MASK		(0x000003FF)
+#define PDP_GAMMA28_GB_GAMMA28_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA28_GB_GAMMA28_B_SHIFT		(0)
+#define PDP_GAMMA28_GB_GAMMA28_B_LENGTH		(10)
+#define PDP_GAMMA28_GB_GAMMA28_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA29_R_OFFSET		(0x089C)
+
+/* PDP, GAMMA29_R, GAMMA29_R
+*/
+#define PDP_GAMMA29_R_GAMMA29_R_MASK		(0x000003FF)
+#define PDP_GAMMA29_R_GAMMA29_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA29_R_GAMMA29_R_SHIFT		(0)
+#define PDP_GAMMA29_R_GAMMA29_R_LENGTH		(10)
+#define PDP_GAMMA29_R_GAMMA29_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA29_GB_OFFSET		(0x08A0)
+
+/* PDP, GAMMA29_GB, GAMMA29_G
+*/
+#define PDP_GAMMA29_GB_GAMMA29_G_MASK		(0x03FF0000)
+#define PDP_GAMMA29_GB_GAMMA29_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA29_GB_GAMMA29_G_SHIFT		(16)
+#define PDP_GAMMA29_GB_GAMMA29_G_LENGTH		(10)
+#define PDP_GAMMA29_GB_GAMMA29_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA29_GB, GAMMA29_B
+*/
+#define PDP_GAMMA29_GB_GAMMA29_B_MASK		(0x000003FF)
+#define PDP_GAMMA29_GB_GAMMA29_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA29_GB_GAMMA29_B_SHIFT		(0)
+#define PDP_GAMMA29_GB_GAMMA29_B_LENGTH		(10)
+#define PDP_GAMMA29_GB_GAMMA29_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA30_R_OFFSET		(0x08A4)
+
+/* PDP, GAMMA30_R, GAMMA30_R
+*/
+#define PDP_GAMMA30_R_GAMMA30_R_MASK		(0x000003FF)
+#define PDP_GAMMA30_R_GAMMA30_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA30_R_GAMMA30_R_SHIFT		(0)
+#define PDP_GAMMA30_R_GAMMA30_R_LENGTH		(10)
+#define PDP_GAMMA30_R_GAMMA30_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA30_GB_OFFSET		(0x08A8)
+
+/* PDP, GAMMA30_GB, GAMMA30_G
+*/
+#define PDP_GAMMA30_GB_GAMMA30_G_MASK		(0x03FF0000)
+#define PDP_GAMMA30_GB_GAMMA30_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA30_GB_GAMMA30_G_SHIFT		(16)
+#define PDP_GAMMA30_GB_GAMMA30_G_LENGTH		(10)
+#define PDP_GAMMA30_GB_GAMMA30_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA30_GB, GAMMA30_B
+*/
+#define PDP_GAMMA30_GB_GAMMA30_B_MASK		(0x000003FF)
+#define PDP_GAMMA30_GB_GAMMA30_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA30_GB_GAMMA30_B_SHIFT		(0)
+#define PDP_GAMMA30_GB_GAMMA30_B_LENGTH		(10)
+#define PDP_GAMMA30_GB_GAMMA30_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA31_R_OFFSET		(0x08AC)
+
+/* PDP, GAMMA31_R, GAMMA31_R
+*/
+#define PDP_GAMMA31_R_GAMMA31_R_MASK		(0x000003FF)
+#define PDP_GAMMA31_R_GAMMA31_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA31_R_GAMMA31_R_SHIFT		(0)
+#define PDP_GAMMA31_R_GAMMA31_R_LENGTH		(10)
+#define PDP_GAMMA31_R_GAMMA31_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA31_GB_OFFSET		(0x08B0)
+
+/* PDP, GAMMA31_GB, GAMMA31_G
+*/
+#define PDP_GAMMA31_GB_GAMMA31_G_MASK		(0x03FF0000)
+#define PDP_GAMMA31_GB_GAMMA31_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA31_GB_GAMMA31_G_SHIFT		(16)
+#define PDP_GAMMA31_GB_GAMMA31_G_LENGTH		(10)
+#define PDP_GAMMA31_GB_GAMMA31_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA31_GB, GAMMA31_B
+*/
+#define PDP_GAMMA31_GB_GAMMA31_B_MASK		(0x000003FF)
+#define PDP_GAMMA31_GB_GAMMA31_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA31_GB_GAMMA31_B_SHIFT		(0)
+#define PDP_GAMMA31_GB_GAMMA31_B_LENGTH		(10)
+#define PDP_GAMMA31_GB_GAMMA31_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA32_R_OFFSET		(0x08B4)
+
+/* PDP, GAMMA32_R, GAMMA32_R
+*/
+#define PDP_GAMMA32_R_GAMMA32_R_MASK		(0x000003FF)
+#define PDP_GAMMA32_R_GAMMA32_R_LSBMASK		(0x000003FF)
+#define PDP_GAMMA32_R_GAMMA32_R_SHIFT		(0)
+#define PDP_GAMMA32_R_GAMMA32_R_LENGTH		(10)
+#define PDP_GAMMA32_R_GAMMA32_R_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GAMMA32_GB_OFFSET		(0x08B8)
+
+/* PDP, GAMMA32_GB, GAMMA32_G
+*/
+#define PDP_GAMMA32_GB_GAMMA32_G_MASK		(0x03FF0000)
+#define PDP_GAMMA32_GB_GAMMA32_G_LSBMASK		(0x000003FF)
+#define PDP_GAMMA32_GB_GAMMA32_G_SHIFT		(16)
+#define PDP_GAMMA32_GB_GAMMA32_G_LENGTH		(10)
+#define PDP_GAMMA32_GB_GAMMA32_G_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GAMMA32_GB, GAMMA32_B
+*/
+#define PDP_GAMMA32_GB_GAMMA32_B_MASK		(0x000003FF)
+#define PDP_GAMMA32_GB_GAMMA32_B_LSBMASK		(0x000003FF)
+#define PDP_GAMMA32_GB_GAMMA32_B_SHIFT		(0)
+#define PDP_GAMMA32_GB_GAMMA32_B_LENGTH		(10)
+#define PDP_GAMMA32_GB_GAMMA32_B_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VEVENT_OFFSET		(0x08BC)
+
+/* PDP, VEVENT, VEVENT
+*/
+#define PDP_VEVENT_VEVENT_MASK		(0x1FFF0000)
+#define PDP_VEVENT_VEVENT_LSBMASK		(0x00001FFF)
+#define PDP_VEVENT_VEVENT_SHIFT		(16)
+#define PDP_VEVENT_VEVENT_LENGTH		(13)
+#define PDP_VEVENT_VEVENT_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VEVENT, VFETCH
+*/
+#define PDP_VEVENT_VFETCH_MASK		(0x00001FFF)
+#define PDP_VEVENT_VFETCH_LSBMASK		(0x00001FFF)
+#define PDP_VEVENT_VFETCH_SHIFT		(0)
+#define PDP_VEVENT_VFETCH_LENGTH		(13)
+#define PDP_VEVENT_VFETCH_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_HDECTRL_OFFSET		(0x08C0)
+
+/* PDP, HDECTRL, HDES
+*/
+#define PDP_HDECTRL_HDES_MASK		(0x1FFF0000)
+#define PDP_HDECTRL_HDES_LSBMASK		(0x00001FFF)
+#define PDP_HDECTRL_HDES_SHIFT		(16)
+#define PDP_HDECTRL_HDES_LENGTH		(13)
+#define PDP_HDECTRL_HDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, HDECTRL, HDEF
+*/
+#define PDP_HDECTRL_HDEF_MASK		(0x00001FFF)
+#define PDP_HDECTRL_HDEF_LSBMASK		(0x00001FFF)
+#define PDP_HDECTRL_HDEF_SHIFT		(0)
+#define PDP_HDECTRL_HDEF_LENGTH		(13)
+#define PDP_HDECTRL_HDEF_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VDECTRL_OFFSET		(0x08C4)
+
+/* PDP, VDECTRL, VDES
+*/
+#define PDP_VDECTRL_VDES_MASK		(0x1FFF0000)
+#define PDP_VDECTRL_VDES_LSBMASK		(0x00001FFF)
+#define PDP_VDECTRL_VDES_SHIFT		(16)
+#define PDP_VDECTRL_VDES_LENGTH		(13)
+#define PDP_VDECTRL_VDES_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VDECTRL, VDEF
+*/
+#define PDP_VDECTRL_VDEF_MASK		(0x00001FFF)
+#define PDP_VDECTRL_VDEF_LSBMASK		(0x00001FFF)
+#define PDP_VDECTRL_VDEF_SHIFT		(0)
+#define PDP_VDECTRL_VDEF_LENGTH		(13)
+#define PDP_VDECTRL_VDEF_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_OPMASK_R_OFFSET		(0x08C8)
+
+/* PDP, OPMASK_R, MASKLEVEL
+*/
+#define PDP_OPMASK_R_MASKLEVEL_MASK		(0x80000000)
+#define PDP_OPMASK_R_MASKLEVEL_LSBMASK		(0x00000001)
+#define PDP_OPMASK_R_MASKLEVEL_SHIFT		(31)
+#define PDP_OPMASK_R_MASKLEVEL_LENGTH		(1)
+#define PDP_OPMASK_R_MASKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, BLANKLEVEL
+*/
+#define PDP_OPMASK_R_BLANKLEVEL_MASK		(0x40000000)
+#define PDP_OPMASK_R_BLANKLEVEL_LSBMASK		(0x00000001)
+#define PDP_OPMASK_R_BLANKLEVEL_SHIFT		(30)
+#define PDP_OPMASK_R_BLANKLEVEL_LENGTH		(1)
+#define PDP_OPMASK_R_BLANKLEVEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_R, MASKR
+*/
+#define PDP_OPMASK_R_MASKR_MASK		(0x000003FF)
+#define PDP_OPMASK_R_MASKR_LSBMASK		(0x000003FF)
+#define PDP_OPMASK_R_MASKR_SHIFT		(0)
+#define PDP_OPMASK_R_MASKR_LENGTH		(10)
+#define PDP_OPMASK_R_MASKR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_OPMASK_GB_OFFSET		(0x08CC)
+
+/* PDP, OPMASK_GB, MASKG
+*/
+#define PDP_OPMASK_GB_MASKG_MASK		(0x03FF0000)
+#define PDP_OPMASK_GB_MASKG_LSBMASK		(0x000003FF)
+#define PDP_OPMASK_GB_MASKG_SHIFT		(16)
+#define PDP_OPMASK_GB_MASKG_LENGTH		(10)
+#define PDP_OPMASK_GB_MASKG_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, OPMASK_GB, MASKB
+*/
+#define PDP_OPMASK_GB_MASKB_MASK		(0x000003FF)
+#define PDP_OPMASK_GB_MASKB_LSBMASK		(0x000003FF)
+#define PDP_OPMASK_GB_MASKB_SHIFT		(0)
+#define PDP_OPMASK_GB_MASKB_LENGTH		(10)
+#define PDP_OPMASK_GB_MASKB_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_REGLD_ADDR_CTRL_OFFSET		(0x08D0)
+
+/* PDP, REGLD_ADDR_CTRL, REGLD_ADDRIN
+*/
+#define PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_MASK		(0xFFFFFFF0)
+#define PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LSBMASK		(0x0FFFFFFF)
+#define PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SHIFT		(4)
+#define PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_LENGTH		(28)
+#define PDP_REGLD_ADDR_CTRL_REGLD_ADDRIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_REGLD_ADDR_STAT_OFFSET		(0x08D4)
+
+/* PDP, REGLD_ADDR_STAT, REGLD_ADDROUT
+*/
+#define PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_MASK		(0xFFFFFFF0)
+#define PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LSBMASK		(0x0FFFFFFF)
+#define PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SHIFT		(4)
+#define PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_LENGTH		(28)
+#define PDP_REGLD_ADDR_STAT_REGLD_ADDROUT_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_REGLD_STAT_OFFSET		(0x08D8)
+
+/* PDP, REGLD_STAT, REGLD_ADDREN
+*/
+#define PDP_REGLD_STAT_REGLD_ADDREN_MASK		(0x00800000)
+#define PDP_REGLD_STAT_REGLD_ADDREN_LSBMASK		(0x00000001)
+#define PDP_REGLD_STAT_REGLD_ADDREN_SHIFT		(23)
+#define PDP_REGLD_STAT_REGLD_ADDREN_LENGTH		(1)
+#define PDP_REGLD_STAT_REGLD_ADDREN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_REGLD_CTRL_OFFSET		(0x08DC)
+
+/* PDP, REGLD_CTRL, REGLD_ADDRLEN
+*/
+#define PDP_REGLD_CTRL_REGLD_ADDRLEN_MASK		(0xFF000000)
+#define PDP_REGLD_CTRL_REGLD_ADDRLEN_LSBMASK		(0x000000FF)
+#define PDP_REGLD_CTRL_REGLD_ADDRLEN_SHIFT		(24)
+#define PDP_REGLD_CTRL_REGLD_ADDRLEN_LENGTH		(8)
+#define PDP_REGLD_CTRL_REGLD_ADDRLEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, REGLD_CTRL, REGLD_VAL
+*/
+#define PDP_REGLD_CTRL_REGLD_VAL_MASK		(0x00800000)
+#define PDP_REGLD_CTRL_REGLD_VAL_LSBMASK		(0x00000001)
+#define PDP_REGLD_CTRL_REGLD_VAL_SHIFT		(23)
+#define PDP_REGLD_CTRL_REGLD_VAL_LENGTH		(1)
+#define PDP_REGLD_CTRL_REGLD_VAL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_UPDCTRL_OFFSET		(0x08E0)
+
+/* PDP, UPDCTRL, UPDFIELD
+*/
+#define PDP_UPDCTRL_UPDFIELD_MASK		(0x00000001)
+#define PDP_UPDCTRL_UPDFIELD_LSBMASK		(0x00000001)
+#define PDP_UPDCTRL_UPDFIELD_SHIFT		(0)
+#define PDP_UPDCTRL_UPDFIELD_LENGTH		(1)
+#define PDP_UPDCTRL_UPDFIELD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_INTCTRL_OFFSET		(0x08E4)
+
+/* PDP, PVR_PDP_INTCTRL, HBLNK_LINE
+*/
+#define PDP_INTCTRL_HBLNK_LINE_MASK		(0x00010000)
+#define PDP_INTCTRL_HBLNK_LINE_LSBMASK		(0x00000001)
+#define PDP_INTCTRL_HBLNK_LINE_SHIFT		(16)
+#define PDP_INTCTRL_HBLNK_LINE_LENGTH		(1)
+#define PDP_INTCTRL_HBLNK_LINE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PVR_PDP_INTCTRL, HBLNK_LINENO
+*/
+#define PDP_INTCTRL_HBLNK_LINENO_MASK		(0x00001FFF)
+#define PDP_INTCTRL_HBLNK_LINENO_LSBMASK		(0x00001FFF)
+#define PDP_INTCTRL_HBLNK_LINENO_SHIFT		(0)
+#define PDP_INTCTRL_HBLNK_LINENO_LENGTH		(13)
+#define PDP_INTCTRL_HBLNK_LINENO_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PDISETUP_OFFSET		(0x0900)
+
+/* PDP, PDISETUP, PDI_BLNKLVL
+*/
+#define PDP_PDISETUP_PDI_BLNKLVL_MASK		(0x00000040)
+#define PDP_PDISETUP_PDI_BLNKLVL_LSBMASK		(0x00000001)
+#define PDP_PDISETUP_PDI_BLNKLVL_SHIFT		(6)
+#define PDP_PDISETUP_PDI_BLNKLVL_LENGTH		(1)
+#define PDP_PDISETUP_PDI_BLNKLVL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_BLNK
+*/
+#define PDP_PDISETUP_PDI_BLNK_MASK		(0x00000020)
+#define PDP_PDISETUP_PDI_BLNK_LSBMASK		(0x00000001)
+#define PDP_PDISETUP_PDI_BLNK_SHIFT		(5)
+#define PDP_PDISETUP_PDI_BLNK_LENGTH		(1)
+#define PDP_PDISETUP_PDI_BLNK_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_PWR
+*/
+#define PDP_PDISETUP_PDI_PWR_MASK		(0x00000010)
+#define PDP_PDISETUP_PDI_PWR_LSBMASK		(0x00000001)
+#define PDP_PDISETUP_PDI_PWR_SHIFT		(4)
+#define PDP_PDISETUP_PDI_PWR_LENGTH		(1)
+#define PDP_PDISETUP_PDI_PWR_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_EN
+*/
+#define PDP_PDISETUP_PDI_EN_MASK		(0x00000008)
+#define PDP_PDISETUP_PDI_EN_LSBMASK		(0x00000001)
+#define PDP_PDISETUP_PDI_EN_SHIFT		(3)
+#define PDP_PDISETUP_PDI_EN_LENGTH		(1)
+#define PDP_PDISETUP_PDI_EN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_GDEN
+*/
+#define PDP_PDISETUP_PDI_GDEN_MASK		(0x00000004)
+#define PDP_PDISETUP_PDI_GDEN_LSBMASK		(0x00000001)
+#define PDP_PDISETUP_PDI_GDEN_SHIFT		(2)
+#define PDP_PDISETUP_PDI_GDEN_LENGTH		(1)
+#define PDP_PDISETUP_PDI_GDEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_NFEN
+*/
+#define PDP_PDISETUP_PDI_NFEN_MASK		(0x00000002)
+#define PDP_PDISETUP_PDI_NFEN_LSBMASK		(0x00000001)
+#define PDP_PDISETUP_PDI_NFEN_SHIFT		(1)
+#define PDP_PDISETUP_PDI_NFEN_LENGTH		(1)
+#define PDP_PDISETUP_PDI_NFEN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDISETUP, PDI_CR
+*/
+#define PDP_PDISETUP_PDI_CR_MASK		(0x00000001)
+#define PDP_PDISETUP_PDI_CR_LSBMASK		(0x00000001)
+#define PDP_PDISETUP_PDI_CR_SHIFT		(0)
+#define PDP_PDISETUP_PDI_CR_LENGTH		(1)
+#define PDP_PDISETUP_PDI_CR_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PDITIMING0_OFFSET		(0x0904)
+
+/* PDP, PDITIMING0, PDI_PWRSVGD
+*/
+#define PDP_PDITIMING0_PDI_PWRSVGD_MASK		(0x0F000000)
+#define PDP_PDITIMING0_PDI_PWRSVGD_LSBMASK		(0x0000000F)
+#define PDP_PDITIMING0_PDI_PWRSVGD_SHIFT		(24)
+#define PDP_PDITIMING0_PDI_PWRSVGD_LENGTH		(4)
+#define PDP_PDITIMING0_PDI_PWRSVGD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_LSDEL
+*/
+#define PDP_PDITIMING0_PDI_LSDEL_MASK		(0x007F0000)
+#define PDP_PDITIMING0_PDI_LSDEL_LSBMASK		(0x0000007F)
+#define PDP_PDITIMING0_PDI_LSDEL_SHIFT		(16)
+#define PDP_PDITIMING0_PDI_LSDEL_LENGTH		(7)
+#define PDP_PDITIMING0_PDI_LSDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING0, PDI_PWRSV2GD2
+*/
+#define PDP_PDITIMING0_PDI_PWRSV2GD2_MASK		(0x000003FF)
+#define PDP_PDITIMING0_PDI_PWRSV2GD2_LSBMASK		(0x000003FF)
+#define PDP_PDITIMING0_PDI_PWRSV2GD2_SHIFT		(0)
+#define PDP_PDITIMING0_PDI_PWRSV2GD2_LENGTH		(10)
+#define PDP_PDITIMING0_PDI_PWRSV2GD2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PDITIMING1_OFFSET		(0x0908)
+
+/* PDP, PDITIMING1, PDI_NLDEL
+*/
+#define PDP_PDITIMING1_PDI_NLDEL_MASK		(0x000F0000)
+#define PDP_PDITIMING1_PDI_NLDEL_LSBMASK		(0x0000000F)
+#define PDP_PDITIMING1_PDI_NLDEL_SHIFT		(16)
+#define PDP_PDITIMING1_PDI_NLDEL_LENGTH		(4)
+#define PDP_PDITIMING1_PDI_NLDEL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDITIMING1, PDI_ACBDEL
+*/
+#define PDP_PDITIMING1_PDI_ACBDEL_MASK		(0x000003FF)
+#define PDP_PDITIMING1_PDI_ACBDEL_LSBMASK		(0x000003FF)
+#define PDP_PDITIMING1_PDI_ACBDEL_SHIFT		(0)
+#define PDP_PDITIMING1_PDI_ACBDEL_LENGTH		(10)
+#define PDP_PDITIMING1_PDI_ACBDEL_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PDICOREID_OFFSET		(0x090C)
+
+/* PDP, PDICOREID, PDI_GROUP_ID
+*/
+#define PDP_PDICOREID_PDI_GROUP_ID_MASK		(0xFF000000)
+#define PDP_PDICOREID_PDI_GROUP_ID_LSBMASK		(0x000000FF)
+#define PDP_PDICOREID_PDI_GROUP_ID_SHIFT		(24)
+#define PDP_PDICOREID_PDI_GROUP_ID_LENGTH		(8)
+#define PDP_PDICOREID_PDI_GROUP_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CORE_ID
+*/
+#define PDP_PDICOREID_PDI_CORE_ID_MASK		(0x00FF0000)
+#define PDP_PDICOREID_PDI_CORE_ID_LSBMASK		(0x000000FF)
+#define PDP_PDICOREID_PDI_CORE_ID_SHIFT		(16)
+#define PDP_PDICOREID_PDI_CORE_ID_LENGTH		(8)
+#define PDP_PDICOREID_PDI_CORE_ID_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREID, PDI_CONFIG_ID
+*/
+#define PDP_PDICOREID_PDI_CONFIG_ID_MASK		(0x0000FFFF)
+#define PDP_PDICOREID_PDI_CONFIG_ID_LSBMASK		(0x0000FFFF)
+#define PDP_PDICOREID_PDI_CONFIG_ID_SHIFT		(0)
+#define PDP_PDICOREID_PDI_CONFIG_ID_LENGTH		(16)
+#define PDP_PDICOREID_PDI_CONFIG_ID_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_PDICOREREV_OFFSET		(0x0910)
+
+/* PDP, PDICOREREV, PDI_MAJOR_REV
+*/
+#define PDP_PDICOREREV_PDI_MAJOR_REV_MASK		(0x00FF0000)
+#define PDP_PDICOREREV_PDI_MAJOR_REV_LSBMASK		(0x000000FF)
+#define PDP_PDICOREREV_PDI_MAJOR_REV_SHIFT		(16)
+#define PDP_PDICOREREV_PDI_MAJOR_REV_LENGTH		(8)
+#define PDP_PDICOREREV_PDI_MAJOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MINOR_REV
+*/
+#define PDP_PDICOREREV_PDI_MINOR_REV_MASK		(0x0000FF00)
+#define PDP_PDICOREREV_PDI_MINOR_REV_LSBMASK		(0x000000FF)
+#define PDP_PDICOREREV_PDI_MINOR_REV_SHIFT		(8)
+#define PDP_PDICOREREV_PDI_MINOR_REV_LENGTH		(8)
+#define PDP_PDICOREREV_PDI_MINOR_REV_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, PDICOREREV, PDI_MAINT_REV
+*/
+#define PDP_PDICOREREV_PDI_MAINT_REV_MASK		(0x000000FF)
+#define PDP_PDICOREREV_PDI_MAINT_REV_LSBMASK		(0x000000FF)
+#define PDP_PDICOREREV_PDI_MAINT_REV_SHIFT		(0)
+#define PDP_PDICOREREV_PDI_MAINT_REV_LENGTH		(8)
+#define PDP_PDICOREREV_PDI_MAINT_REV_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX2_OFFSET		(0x0920)
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y1
+*/
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_MASK		(0x000000C0)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LSBMASK		(0x00000003)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SHIFT		(6)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_LENGTH		(2)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y1
+*/
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_MASK		(0x00000030)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LSBMASK		(0x00000003)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SHIFT		(4)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_LENGTH		(2)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X1Y0
+*/
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_MASK		(0x0000000C)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LSBMASK		(0x00000003)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SHIFT		(2)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_LENGTH		(2)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX2, DITHERMATRIX2X0Y0
+*/
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_MASK		(0x00000003)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LSBMASK		(0x00000003)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SHIFT		(0)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_LENGTH		(2)
+#define PDP_DITHERMATRIX2_DITHERMATRIX2X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX4_0_OFFSET		(0x0924)
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y1
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_MASK		(0xF0000000)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SHIFT		(28)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y1
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_MASK		(0x0F000000)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SHIFT		(24)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y1
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_MASK		(0x00F00000)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SHIFT		(20)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y1
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_MASK		(0x000F0000)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SHIFT		(16)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X3Y0
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_MASK		(0x0000F000)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SHIFT		(12)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X2Y0
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_MASK		(0x00000F00)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SHIFT		(8)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X1Y0
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_MASK		(0x000000F0)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SHIFT		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_0, DITHERMATRIX4X0Y0
+*/
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_MASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SHIFT		(0)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_LENGTH		(4)
+#define PDP_DITHERMATRIX4_0_DITHERMATRIX4X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX4_1_OFFSET		(0x0928)
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y3
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_MASK		(0xF0000000)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SHIFT		(28)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y3
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_MASK		(0x0F000000)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SHIFT		(24)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y3
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_MASK		(0x00F00000)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SHIFT		(20)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y3
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_MASK		(0x000F0000)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SHIFT		(16)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X3Y2
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_MASK		(0x0000F000)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SHIFT		(12)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X2Y2
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_MASK		(0x00000F00)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SHIFT		(8)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X1Y2
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_MASK		(0x000000F0)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SHIFT		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX4_1, DITHERMATRIX4X0Y2
+*/
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_MASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LSBMASK		(0x0000000F)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SHIFT		(0)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_LENGTH		(4)
+#define PDP_DITHERMATRIX4_1_DITHERMATRIX4X0Y2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_0_OFFSET		(0x092C)
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X4Y0
+*/
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SHIFT		(24)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X4Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X3Y0
+*/
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SHIFT		(18)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X3Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X2Y0
+*/
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SHIFT		(12)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X2Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X1Y0
+*/
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SHIFT		(6)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X1Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_0, DITHERMATRIX8X0Y0
+*/
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SHIFT		(0)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_0_DITHERMATRIX8X0Y0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_1_OFFSET		(0x0930)
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X1Y1
+*/
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SHIFT		(24)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X1Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X0Y1
+*/
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SHIFT		(18)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X0Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X7Y0
+*/
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SHIFT		(12)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X7Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X6Y0
+*/
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SHIFT		(6)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X6Y0_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_1, DITHERMATRIX8X5Y0
+*/
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SHIFT		(0)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_LENGTH		(6)
+#define PDP_DITHERMATRIX8_1_DITHERMATRIX8X5Y0_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_2_OFFSET		(0x0934)
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X6Y1
+*/
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SHIFT		(24)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X6Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X5Y1
+*/
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SHIFT		(18)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X5Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X4Y1
+*/
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SHIFT		(12)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X4Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X3Y1
+*/
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SHIFT		(6)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X3Y1_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_2, DITHERMATRIX8X2Y1
+*/
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SHIFT		(0)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_2_DITHERMATRIX8X2Y1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_3_OFFSET		(0x0938)
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X3Y2
+*/
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SHIFT		(24)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X3Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X2Y2
+*/
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SHIFT		(18)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X2Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X1Y2
+*/
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SHIFT		(12)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X1Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X0Y2
+*/
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SHIFT		(6)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X0Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_3, DITHERMATRIX8X7Y1
+*/
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SHIFT		(0)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_LENGTH		(6)
+#define PDP_DITHERMATRIX8_3_DITHERMATRIX8X7Y1_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_4_OFFSET		(0x093C)
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X0Y3
+*/
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SHIFT		(24)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X0Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X7Y2
+*/
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SHIFT		(18)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X7Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X6Y2
+*/
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SHIFT		(12)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X6Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X5Y2
+*/
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SHIFT		(6)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X5Y2_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_4, DITHERMATRIX8X4Y2
+*/
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SHIFT		(0)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_LENGTH		(6)
+#define PDP_DITHERMATRIX8_4_DITHERMATRIX8X4Y2_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_5_OFFSET		(0x0940)
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X5Y3
+*/
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SHIFT		(24)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X5Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X4Y3
+*/
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SHIFT		(18)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X4Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X3Y3
+*/
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SHIFT		(12)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X3Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X2Y3
+*/
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SHIFT		(6)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X2Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_5, DITHERMATRIX8X1Y3
+*/
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SHIFT		(0)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_5_DITHERMATRIX8X1Y3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_6_OFFSET		(0x0944)
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X2Y4
+*/
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SHIFT		(24)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X2Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X1Y4
+*/
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SHIFT		(18)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X1Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X0Y4
+*/
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SHIFT		(12)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X0Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X7Y3
+*/
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SHIFT		(6)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X7Y3_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_6, DITHERMATRIX8X6Y3
+*/
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SHIFT		(0)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_LENGTH		(6)
+#define PDP_DITHERMATRIX8_6_DITHERMATRIX8X6Y3_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_7_OFFSET		(0x0948)
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X7Y4
+*/
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SHIFT		(24)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X7Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X6Y4
+*/
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SHIFT		(18)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X6Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X5Y4
+*/
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SHIFT		(12)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X5Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X4Y4
+*/
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SHIFT		(6)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X4Y4_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_7, DITHERMATRIX8X3Y4
+*/
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SHIFT		(0)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_LENGTH		(6)
+#define PDP_DITHERMATRIX8_7_DITHERMATRIX8X3Y4_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_8_OFFSET		(0x094C)
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X4Y5
+*/
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SHIFT		(24)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X4Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X3Y5
+*/
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SHIFT		(18)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X3Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X2Y5
+*/
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SHIFT		(12)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X2Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X1Y5
+*/
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SHIFT		(6)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X1Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_8, DITHERMATRIX8X0Y5
+*/
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SHIFT		(0)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_8_DITHERMATRIX8X0Y5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_9_OFFSET		(0x0950)
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X1Y6
+*/
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SHIFT		(24)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X1Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X0Y6
+*/
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SHIFT		(18)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X0Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X7Y5
+*/
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SHIFT		(12)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X7Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X6Y5
+*/
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SHIFT		(6)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X6Y5_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_9, DITHERMATRIX8X5Y5
+*/
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SHIFT		(0)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_LENGTH		(6)
+#define PDP_DITHERMATRIX8_9_DITHERMATRIX8X5Y5_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_10_OFFSET		(0x0954)
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X6Y6
+*/
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SHIFT		(24)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X6Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X5Y6
+*/
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SHIFT		(18)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X5Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X4Y6
+*/
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SHIFT		(12)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X4Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X3Y6
+*/
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SHIFT		(6)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X3Y6_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_10, DITHERMATRIX8X2Y6
+*/
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SHIFT		(0)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_10_DITHERMATRIX8X2Y6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_11_OFFSET		(0x0958)
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X3Y7
+*/
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_MASK		(0x3F000000)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SHIFT		(24)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X3Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X2Y7
+*/
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SHIFT		(18)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X2Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X1Y7
+*/
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SHIFT		(12)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X1Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X0Y7
+*/
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SHIFT		(6)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X0Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_11, DITHERMATRIX8X7Y6
+*/
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SHIFT		(0)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_LENGTH		(6)
+#define PDP_DITHERMATRIX8_11_DITHERMATRIX8X7Y6_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_DITHERMATRIX8_12_OFFSET		(0x095C)
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X7Y7
+*/
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_MASK		(0x00FC0000)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SHIFT		(18)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X7Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X6Y7
+*/
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_MASK		(0x0003F000)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SHIFT		(12)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X6Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X5Y7
+*/
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_MASK		(0x00000FC0)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SHIFT		(6)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X5Y7_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, DITHERMATRIX8_12, DITHERMATRIX8X4Y7
+*/
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_MASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LSBMASK		(0x0000003F)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SHIFT		(0)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_LENGTH		(6)
+#define PDP_DITHERMATRIX8_12_DITHERMATRIX8X4Y7_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1_MEMCTRL_OFFSET		(0x0960)
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_GRPH1_MEMCTRL_GRPH1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEMCTRL, GRPH1_BURSTLEN
+*/
+#define PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_MASK		(0x000000FF)
+#define PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SHIFT		(0)
+#define PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_LENGTH		(8)
+#define PDP_GRPH1_MEMCTRL_GRPH1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1_MEM_THRESH_OFFSET		(0x0964)
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_UVTHRESHOLD
+*/
+#define PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SHIFT		(24)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_LENGTH		(8)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_YTHRESHOLD
+*/
+#define PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SHIFT		(12)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_LENGTH		(9)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_MEM_THRESH, GRPH1_THRESHOLD
+*/
+#define PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_MASK		(0x000001FF)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SHIFT		(0)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_LENGTH		(9)
+#define PDP_GRPH1_MEM_THRESH_GRPH1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2_MEMCTRL_OFFSET		(0x0968)
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_GRPH2_MEMCTRL_GRPH2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEMCTRL, GRPH2_BURSTLEN
+*/
+#define PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_MASK		(0x000000FF)
+#define PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SHIFT		(0)
+#define PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_LENGTH		(8)
+#define PDP_GRPH2_MEMCTRL_GRPH2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2_MEM_THRESH_OFFSET		(0x096C)
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_UVTHRESHOLD
+*/
+#define PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SHIFT		(24)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_LENGTH		(8)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_YTHRESHOLD
+*/
+#define PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SHIFT		(12)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_LENGTH		(9)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_MEM_THRESH, GRPH2_THRESHOLD
+*/
+#define PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_MASK		(0x000001FF)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SHIFT		(0)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_LENGTH		(9)
+#define PDP_GRPH2_MEM_THRESH_GRPH2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3_MEMCTRL_OFFSET		(0x0970)
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_GRPH3_MEMCTRL_GRPH3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEMCTRL, GRPH3_BURSTLEN
+*/
+#define PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_MASK		(0x000000FF)
+#define PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SHIFT		(0)
+#define PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_LENGTH		(8)
+#define PDP_GRPH3_MEMCTRL_GRPH3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3_MEM_THRESH_OFFSET		(0x0974)
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_UVTHRESHOLD
+*/
+#define PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SHIFT		(24)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_LENGTH		(8)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_YTHRESHOLD
+*/
+#define PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SHIFT		(12)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_LENGTH		(9)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_MEM_THRESH, GRPH3_THRESHOLD
+*/
+#define PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_MASK		(0x000001FF)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SHIFT		(0)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_LENGTH		(9)
+#define PDP_GRPH3_MEM_THRESH_GRPH3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4_MEMCTRL_OFFSET		(0x0978)
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_GRPH4_MEMCTRL_GRPH4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEMCTRL, GRPH4_BURSTLEN
+*/
+#define PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_MASK		(0x000000FF)
+#define PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SHIFT		(0)
+#define PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_LENGTH		(8)
+#define PDP_GRPH4_MEMCTRL_GRPH4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4_MEM_THRESH_OFFSET		(0x097C)
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_UVTHRESHOLD
+*/
+#define PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SHIFT		(24)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_LENGTH		(8)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_YTHRESHOLD
+*/
+#define PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SHIFT		(12)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_LENGTH		(9)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_MEM_THRESH, GRPH4_THRESHOLD
+*/
+#define PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_MASK		(0x000001FF)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SHIFT		(0)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_LENGTH		(9)
+#define PDP_GRPH4_MEM_THRESH_GRPH4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1_MEMCTRL_OFFSET		(0x0980)
+
+/* PDP, VID1_MEMCTRL, VID1_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_VID1_MEMCTRL_VID1_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEMCTRL, VID1_BURSTLEN
+*/
+#define PDP_VID1_MEMCTRL_VID1_BURSTLEN_MASK		(0x000000FF)
+#define PDP_VID1_MEMCTRL_VID1_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_VID1_MEMCTRL_VID1_BURSTLEN_SHIFT		(0)
+#define PDP_VID1_MEMCTRL_VID1_BURSTLEN_LENGTH		(8)
+#define PDP_VID1_MEMCTRL_VID1_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1_MEM_THRESH_OFFSET		(0x0984)
+
+/* PDP, VID1_MEM_THRESH, VID1_UVTHRESHOLD
+*/
+#define PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SHIFT		(24)
+#define PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_LENGTH		(8)
+#define PDP_VID1_MEM_THRESH_VID1_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_YTHRESHOLD
+*/
+#define PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SHIFT		(12)
+#define PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_LENGTH		(9)
+#define PDP_VID1_MEM_THRESH_VID1_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_MEM_THRESH, VID1_THRESHOLD
+*/
+#define PDP_VID1_MEM_THRESH_VID1_THRESHOLD_MASK		(0x000001FF)
+#define PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SHIFT		(0)
+#define PDP_VID1_MEM_THRESH_VID1_THRESHOLD_LENGTH		(9)
+#define PDP_VID1_MEM_THRESH_VID1_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2_MEMCTRL_OFFSET		(0x0988)
+
+/* PDP, VID2_MEMCTRL, VID2_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_VID2_MEMCTRL_VID2_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEMCTRL, VID2_BURSTLEN
+*/
+#define PDP_VID2_MEMCTRL_VID2_BURSTLEN_MASK		(0x000000FF)
+#define PDP_VID2_MEMCTRL_VID2_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_VID2_MEMCTRL_VID2_BURSTLEN_SHIFT		(0)
+#define PDP_VID2_MEMCTRL_VID2_BURSTLEN_LENGTH		(8)
+#define PDP_VID2_MEMCTRL_VID2_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2_MEM_THRESH_OFFSET		(0x098C)
+
+/* PDP, VID2_MEM_THRESH, VID2_UVTHRESHOLD
+*/
+#define PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SHIFT		(24)
+#define PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_LENGTH		(8)
+#define PDP_VID2_MEM_THRESH_VID2_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_YTHRESHOLD
+*/
+#define PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SHIFT		(12)
+#define PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_LENGTH		(9)
+#define PDP_VID2_MEM_THRESH_VID2_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_MEM_THRESH, VID2_THRESHOLD
+*/
+#define PDP_VID2_MEM_THRESH_VID2_THRESHOLD_MASK		(0x000001FF)
+#define PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SHIFT		(0)
+#define PDP_VID2_MEM_THRESH_VID2_THRESHOLD_LENGTH		(9)
+#define PDP_VID2_MEM_THRESH_VID2_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3_MEMCTRL_OFFSET		(0x0990)
+
+/* PDP, VID3_MEMCTRL, VID3_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_VID3_MEMCTRL_VID3_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEMCTRL, VID3_BURSTLEN
+*/
+#define PDP_VID3_MEMCTRL_VID3_BURSTLEN_MASK		(0x000000FF)
+#define PDP_VID3_MEMCTRL_VID3_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_VID3_MEMCTRL_VID3_BURSTLEN_SHIFT		(0)
+#define PDP_VID3_MEMCTRL_VID3_BURSTLEN_LENGTH		(8)
+#define PDP_VID3_MEMCTRL_VID3_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3_MEM_THRESH_OFFSET		(0x0994)
+
+/* PDP, VID3_MEM_THRESH, VID3_UVTHRESHOLD
+*/
+#define PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SHIFT		(24)
+#define PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_LENGTH		(8)
+#define PDP_VID3_MEM_THRESH_VID3_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_YTHRESHOLD
+*/
+#define PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SHIFT		(12)
+#define PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_LENGTH		(9)
+#define PDP_VID3_MEM_THRESH_VID3_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_MEM_THRESH, VID3_THRESHOLD
+*/
+#define PDP_VID3_MEM_THRESH_VID3_THRESHOLD_MASK		(0x000001FF)
+#define PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SHIFT		(0)
+#define PDP_VID3_MEM_THRESH_VID3_THRESHOLD_LENGTH		(9)
+#define PDP_VID3_MEM_THRESH_VID3_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4_MEMCTRL_OFFSET		(0x0998)
+
+/* PDP, VID4_MEMCTRL, VID4_LOCAL_GLOBAL_MEMCTRL
+*/
+#define PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_MASK		(0x80000000)
+#define PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LSBMASK		(0x00000001)
+#define PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SHIFT		(31)
+#define PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_LENGTH		(1)
+#define PDP_VID4_MEMCTRL_VID4_LOCAL_GLOBAL_MEMCTRL_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEMCTRL, VID4_BURSTLEN
+*/
+#define PDP_VID4_MEMCTRL_VID4_BURSTLEN_MASK		(0x000000FF)
+#define PDP_VID4_MEMCTRL_VID4_BURSTLEN_LSBMASK		(0x000000FF)
+#define PDP_VID4_MEMCTRL_VID4_BURSTLEN_SHIFT		(0)
+#define PDP_VID4_MEMCTRL_VID4_BURSTLEN_LENGTH		(8)
+#define PDP_VID4_MEMCTRL_VID4_BURSTLEN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4_MEM_THRESH_OFFSET		(0x099C)
+
+/* PDP, VID4_MEM_THRESH, VID4_UVTHRESHOLD
+*/
+#define PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_MASK		(0xFF000000)
+#define PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LSBMASK		(0x000000FF)
+#define PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SHIFT		(24)
+#define PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_LENGTH		(8)
+#define PDP_VID4_MEM_THRESH_VID4_UVTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_YTHRESHOLD
+*/
+#define PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_MASK		(0x001FF000)
+#define PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SHIFT		(12)
+#define PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_LENGTH		(9)
+#define PDP_VID4_MEM_THRESH_VID4_YTHRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_MEM_THRESH, VID4_THRESHOLD
+*/
+#define PDP_VID4_MEM_THRESH_VID4_THRESHOLD_MASK		(0x000001FF)
+#define PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LSBMASK		(0x000001FF)
+#define PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SHIFT		(0)
+#define PDP_VID4_MEM_THRESH_VID4_THRESHOLD_LENGTH		(9)
+#define PDP_VID4_MEM_THRESH_VID4_THRESHOLD_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH1_PANIC_THRESH_OFFSET		(0x09A0)
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_ENABLE
+*/
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_ENABLE
+*/
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH1_PANIC_THRESH, GRPH1_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_GRPH1_PANIC_THRESH_GRPH1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH2_PANIC_THRESH_OFFSET		(0x09A4)
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_ENABLE
+*/
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_ENABLE
+*/
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH2_PANIC_THRESH, GRPH2_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_GRPH2_PANIC_THRESH_GRPH2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH3_PANIC_THRESH_OFFSET		(0x09A8)
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_ENABLE
+*/
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_ENABLE
+*/
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH3_PANIC_THRESH, GRPH3_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_GRPH3_PANIC_THRESH_GRPH3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_GRPH4_PANIC_THRESH_OFFSET		(0x09AC)
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_ENABLE
+*/
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_ENABLE
+*/
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, GRPH4_PANIC_THRESH, GRPH4_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_GRPH4_PANIC_THRESH_GRPH4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID1_PANIC_THRESH_OFFSET		(0x09B0)
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_ENABLE
+*/
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_ENABLE
+*/
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID1_PANIC_THRESH, VID1_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_VID1_PANIC_THRESH_VID1_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID2_PANIC_THRESH_OFFSET		(0x09B4)
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_ENABLE
+*/
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_ENABLE
+*/
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID2_PANIC_THRESH, VID2_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_VID2_PANIC_THRESH_VID2_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID3_PANIC_THRESH_OFFSET		(0x09B8)
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_ENABLE
+*/
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_ENABLE
+*/
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID3_PANIC_THRESH, VID3_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_VID3_PANIC_THRESH_VID3_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_VID4_PANIC_THRESH_OFFSET		(0x09BC)
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_ENABLE
+*/
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_MASK		(0x80000000)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SHIFT		(31)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_LENGTH		(1)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_ENABLE
+*/
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_MASK		(0x40000000)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LSBMASK		(0x00000001)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SHIFT		(30)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_LENGTH		(1)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_ENABLE_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MAX
+*/
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_MASK		(0x3F800000)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LSBMASK		(0x0000007F)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SHIFT		(23)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_LENGTH		(7)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_UV_THRESHOLD_MIN
+*/
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_MASK		(0x007F0000)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LSBMASK		(0x0000007F)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SHIFT		(16)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_LENGTH		(7)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_UV_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MAX
+*/
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_MASK		(0x0000FF00)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LSBMASK		(0x000000FF)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SHIFT		(8)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_LENGTH		(8)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MAX_SIGNED_FIELD	IMG_FALSE
+
+/* PDP, VID4_PANIC_THRESH, VID4_ALERT_Y_THRESHOLD_MIN
+*/
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_MASK		(0x000000FF)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LSBMASK		(0x000000FF)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SHIFT		(0)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_LENGTH		(8)
+#define PDP_VID4_PANIC_THRESH_VID4_ALERT_Y_THRESHOLD_MIN_SIGNED_FIELD	IMG_FALSE
+
+#define PDP_BURST_BOUNDARY_OFFSET		(0x09C0)
+
+/* PDP, BURST_BOUNDARY, BURST_BOUNDARY
+*/
+#define PDP_BURST_BOUNDARY_BURST_BOUNDARY_MASK		(0x0000003F)
+#define PDP_BURST_BOUNDARY_BURST_BOUNDARY_LSBMASK		(0x0000003F)
+#define PDP_BURST_BOUNDARY_BURST_BOUNDARY_SHIFT		(0)
+#define PDP_BURST_BOUNDARY_BURST_BOUNDARY_LENGTH		(6)
+#define PDP_BURST_BOUNDARY_BURST_BOUNDARY_SIGNED_FIELD	IMG_FALSE
+
+
+/* ------------------------ End of register definitions ------------------------ */
+
+/*
+// NUMREG defines the extent of register address space.
+*/
+
+#define		PDP_NUMREG	   ((0x09C0 >> 2)+1)
+
+/* Info about video plane addresses */
+#define PDP_YADDR_BITS		(PDP_VID1BASEADDR_VID1BASEADDR_LENGTH)
+#define PDP_YADDR_ALIGN		5
+#define PDP_UADDR_BITS		(PDP_VID1UBASEADDR_VID1UBASEADDR_LENGTH)
+#define PDP_UADDR_ALIGN		5
+#define PDP_VADDR_BITS		(PDP_VID1VBASEADDR_VID1VBASEADDR_LENGTH)
+#define PDP_VADDR_ALIGN		5
+
+#define PDP_YSTRIDE_BITS	(PDP_VID1STRIDE_VID1STRIDE_LENGTH)
+#define PDP_YSTRIDE_ALIGN	5
+
+#define PDP_MAX_INPUT_WIDTH (PDP_VID1SIZE_VID1WIDTH_LSBMASK + 1)
+#define PDP_MAX_INPUT_HEIGHT (PDP_VID1SIZE_VID1HEIGHT_LSBMASK + 1)
+
+/* Maximum 6 bytes per pixel for RGB161616 */
+#define PDP_MAX_IMAGE_BYTES (PDP_MAX_INPUT_WIDTH * PDP_MAX_INPUT_HEIGHT * 6)
+
+/* Round up */
+#define PDP_MAX_IMAGE_PAGES ((PDP_MAX_IMAGE_BYTES+PAGE_SIZE-1)/PAGE_SIZE)
+
+#define PDP_YADDR_MAX		(((1 << PDP_YADDR_BITS) - 1) << PDP_YADDR_ALIGN)
+#define PDP_UADDR_MAX		(((1 << PDP_UADDR_BITS) - 1) << PDP_UADDR_ALIGN)
+#define PDP_VADDR_MAX		(((1 << PDP_VADDR_BITS) - 1) << PDP_VADDR_ALIGN)
+#define PDP_YSTRIDE_MAX		((1 << PDP_YSTRIDE_BITS) << PDP_YSTRIDE_ALIGN)
+#define PDP_YADDR_ALIGNMASK	((1 << PDP_YADDR_ALIGN) - 1)
+#define PDP_UADDR_ALIGNMASK	((1 << PDP_UADDR_ALIGN) - 1)
+#define PDP_VADDR_ALIGNMASK	((1 << PDP_VADDR_ALIGN) - 1)
+#define PDP_YSTRIDE_ALIGNMASK	((1 << PDP_YSTRIDE_ALIGN) - 1)
+
+/* Field Values */
+#define PDP_SURF_PIXFMT_RGB332					  0x3
+#define PDP_SURF_PIXFMT_ARGB4444				  0x4
+#define PDP_SURF_PIXFMT_ARGB1555				  0x5
+#define PDP_SURF_PIXFMT_RGB888					  0x6
+#define PDP_SURF_PIXFMT_RGB565					  0x7
+#define PDP_SURF_PIXFMT_ARGB8888				  0x8
+#define PDP_SURF_PIXFMT_420_PL8					  0x9
+#define PDP_SURF_PIXFMT_420_PL8IVU				  0xA
+#define PDP_SURF_PIXFMT_420_PL8IUV				  0xB
+#define PDP_SURF_PIXFMT_422_UY0VY1_8888			  0xC
+#define PDP_SURF_PIXFMT_422_VY0UY1_8888			  0xD
+#define PDP_SURF_PIXFMT_422_Y0UY1V_8888			  0xE
+#define PDP_SURF_PIXFMT_422_Y0VY1U_8888			  0xF
+#define PDP_SURF_PIXFMT_AYUV8888				  0x10
+#define PDP_SURF_PIXFMT_YUV101010				  0x15
+#define PDP_SURF_PIXFMT_RGB101010				  0x17
+#define PDP_SURF_PIXFMT_420_PL10IUV				  0x18
+#define PDP_SURF_PIXFMT_420_PL10IVU				  0x19
+#define PDP_SURF_PIXFMT_422_PL10IUV				  0x1A
+#define PDP_SURF_PIXFMT_422_PL10IVU				  0x1B
+#define PDP_SURF_PIXFMT_RGB121212				  0x1E
+#define PDP_SURF_PIXFMT_RGB161616				  0x1F
+
+#define PDP_CTRL_CKEYSRC_PREV					  0x0
+#define PDP_CTRL_CKEYSRC_CUR					  0x1
+
+#define PDP_MEMCTRL_MEMREFRESH_ALWAYS			  0x0
+#define PDP_MEMCTRL_MEMREFRESH_HBLNK			  0x1
+#define PDP_MEMCTRL_MEMREFRESH_VBLNK			  0x2
+#define PDP_MEMCTRL_MEMREFRESH_BOTH				  0x3
+
+#define PDP_3D_CTRL_BLENDSEL_BGND_WITH_POS0		  0x0
+#define PDP_3D_CTRL_BLENDSEL_POS0_WITH_POS1		  0x1
+#define PDP_3D_CTRL_BLENDSEL_POS1_WITH_POS2		  0x2
+#define PDP_3D_CTRL_BLENDSEL_POS2_WITH_POS3		  0x3
+#define PDP_3D_CTRL_BLENDSEL_POS3_WITH_POS4		  0x4
+#define PDP_3D_CTRL_BLENDSEL_POS4_WITH_POS5		  0x5
+#define PDP_3D_CTRL_BLENDSEL_POS5_WITH_POS6		  0x6
+#define PDP_3D_CTRL_BLENDSEL_POS6_WITH_POS7		  0x7
+
+#define PDP_UADDR_UV_STRIDE_EQUAL_TO_Y_STRIDE		  0x0
+#define PDP_UADDR_UV_STRIDE_EQUAL_TO_DOUBLE_Y_STRIDE  0x1
+#define PDP_UADDR_UV_STRIDE_EQUAL_TO_HALF_Y_STRIDE	  0x2
+
+#define PDP_PROCAMP_OUTPUT_OFFSET_FRACTIONAL_BITS 1
+#define PDP_PROCAMP_COEFFICIENT_FRACTIONAL_BITS	  10
+
+/*-------------------------------------------------------------------------------*/
+
+#endif /* _PDP2_REGS_H */
diff --git a/drivers/gpu/drm/img-rogue/pdp_drm.h b/drivers/gpu/drm/img-rogue/pdp_drm.h
new file mode 100644
index 000000000000..f5d747d3ad10
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdp_drm.h
@@ -0,0 +1,105 @@
+/*
+ * @File
+ * @Title       PDP DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_DRM_H__)
+#define __PDP_DRM_H__
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+struct drm_pdp_gem_create {
+	__u64 size;	/* in */
+	__u32 flags;	/* in */
+	__u32 handle;	/* out */
+};
+
+struct drm_pdp_gem_mmap {
+	__u32 handle;	/* in */
+	__u32 pad;
+	__u64 offset;	/* out */
+};
+
+#define PDP_GEM_CPU_PREP_READ	(1 << 0)
+#define PDP_GEM_CPU_PREP_WRITE	(1 << 1)
+#define PDP_GEM_CPU_PREP_NOWAIT	(1 << 2)
+
+struct drm_pdp_gem_cpu_prep {
+	__u32 handle;	/* in */
+	__u32 flags;	/* in */
+};
+
+struct drm_pdp_gem_cpu_fini {
+	__u32 handle;	/* in */
+	__u32 pad;
+};
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+#define DRM_PDP_GEM_CREATE		0x00
+#define DRM_PDP_GEM_MMAP		0x01
+#define DRM_PDP_GEM_CPU_PREP		0x02
+#define DRM_PDP_GEM_CPU_FINI		0x03
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define DRM_IOCTL_PDP_GEM_CREATE \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_CREATE, \
+		 struct drm_pdp_gem_create)
+
+#define DRM_IOCTL_PDP_GEM_MMAP\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_MMAP, \
+		 struct drm_pdp_gem_mmap)
+
+#define DRM_IOCTL_PDP_GEM_CPU_PREP \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_PREP, \
+		struct drm_pdp_gem_cpu_prep)
+
+#define DRM_IOCTL_PDP_GEM_CPU_FINI \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_FINI, \
+		struct drm_pdp_gem_cpu_fini)
+
+#endif /* defined(__PDP_DRM_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pdump.h b/drivers/gpu/drm/img-rogue/pdump.h
new file mode 100644
index 000000000000..3ef71846446e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdump.h
@@ -0,0 +1,238 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef SERVICES_PDUMP_H
+#define SERVICES_PDUMP_H
+
+#include "img_types.h"
+#include "services_km.h"
+
+
+/* A PDump out2.txt script is made up of 3 sections from three buffers:
+ * *
+ *  - Init phase buffer    - holds PDump data written during driver
+ *                            initialisation, non-volatile.
+ *  - Main phase buffer   - holds PDump data written after driver init,
+ *                            volatile.
+ *  - Deinit phase buffer - holds PDump data  needed to shutdown HW/play back,
+ *                            written only during driver initialisation using
+ *                            the DEINIT flag.
+ *
+ * Volatile in this sense means that the buffer is drained and cleared when
+ * the pdump capture application connects and transfers the data to file.
+ *
+ * The PDump sub-system uses the driver state (init/post-init), whether
+ * the pdump capture application is connected or not (capture range set/unset)
+ * and, if pdump connected whether the frame is in the range set, to decide
+ * which of the 3 buffers to write the PDump data. Hence there are several
+ * key time periods in the lifetime of the kernel driver that is enabled
+ * with PDUMP=1 (flag XX labels below time line):
+ *
+ * Events:load              init        pdump       enter          exit         pdump
+ *       driver             done       connects     range          range     disconnects
+ *         |__________________|____________|__________|______________|____________|______ . . .
+ * State:  |   init phase     | no capture | <- capture client connected ->       | no capture
+ *         |                  |            |                                      |
+ *         |__________________|____________|______________________________________|_____ . . .
+ * Flag:   | CT,DI            | NONE,CT,PR | NONE,CT,PR                           | See no
+ *         | Never NONE or PR | Never DI   | Never DI                             |   capture
+ *         |__________________|____________|______________________________________|_____ . . .
+ * Write   | NONE -undef      | -No write  | -No write | -Main buf    | -No write | See no
+ * buffer  | CT -Init buf     | -Main buf  | -Main buf | -Main buf    | -Main buf |   capture
+ *         | PR -undef        | -Init buf  | -undef    | -Init & Main | -undef    |
+ *         | DI -Deinit buf   | -undef     | -undef    | -undef       | -undef    |
+ *         |__________________|____________|___________|______________|___________|_____ . . .
+ *
+ * Note: The time line could repeat if the pdump capture application is
+ * disconnected and reconnected without unloading the driver module.
+ *
+ * The DEINIT (DI) | CONTINUOUS (CT) | PERSISTENT (PR) flags must never
+ * be OR'd together and given to a PDump call since undefined behaviour may
+ * result and produce an invalid PDump which does not play back cleanly.
+ *
+ * The decision on which flag to use comes down to which time period the
+ * client or server driver makes the PDump write call AND the nature/purpose
+ * of the data.
+ *
+ * Note: This is a simplified time line, not all conditions represented.
+ *
+ */
+
+typedef IMG_UINT32 PDUMP_FLAGS_T;
+
+#define PDUMP_FLAGS_NONE            PDUMP_NONE     /*<! Output this entry with no special treatment i.e. output
+                                                          only if in frame range. */
+#define PDUMP_FLAGS_BLKDATA         PDUMP_BLKDATA  /*<! This flag indicates block-mode PDump data to be recorded
+                                                          in Block script stream in addition to Main script stream,
+                                                          if capture mode is set to BLOCKED */
+
+#define PDUMP_FLAGS_DEINIT          0x20000000U    /*<! Output this entry to the de-initialisation section, must
+                                                          only be used by the initialisation code in the Server. */
+
+#define PDUMP_FLAGS_POWER           0x08000000U    /*<! Output this entry even when a power transition is ongoing,
+                                                          as directed by other PDUMP flags. */
+
+#define PDUMP_FLAGS_CONTINUOUS      PDUMP_CONT     /*<! Output this entry always regardless of framed capture range,
+                                                          used by client applications being dumped.
+                                                          During init phase of driver such data carrying this flag
+                                                          will be recorded and present for all PDump client
+                                                          connections.
+                                                          Never combine with the PERSIST flag. */
+
+#define PDUMP_FLAGS_PERSISTENT      PDUMP_PERSIST  /*<! Output this entry always regardless of app and range,
+                                                          used by persistent resources created *after* driver
+                                                          initialisation that must appear in all PDump captures
+                                                          (i.e. current capture regardless of frame range (CONT)
+                                                          and all future PDump captures) for that driver
+                                                          instantiation/session.
+                                                          Effectively this is data that is not forgotten
+                                                          for the second and subsequent PDump client connections.
+                                                          Never combine with the CONTINUOUS flag. */
+
+#define PDUMP_FLAGS_INTERVAL        0x04000000U    /*<! Output this entry even when the capture is on a
+                                                          "no capture interval frame" (see pdump -sr option).
+                                                          Useful for commands that have a resource that was written
+                                                          out in a frame that was captured. For example,
+                                                          used by RGXScheduleCleanupCommand. */
+
+#define PDUMP_FLAGS_DEBUG           0x00010000U    /*<! For internal debugging use */
+
+#define PDUMP_FLAGS_NOHW            0x00000001U    /* For internal use: Skip sending instructions to the hardware
+                                                        when NO_HARDWARE=0 AND PDUMP=1 */
+
+#define PDUMP_FLAGS_FORCESPLIT      0x00000002U	   /* Forces Main and Block script streams to split - Internal
+                                                        flag used in Block mode of PDump */
+
+#define PDUMP_FLAGS_PDUMP_LOCK_HELD 0x00000004U    /* This flags denotes that PDUMP_LOCK is held already so
+                                                        further calls to PDUMP_LOCK  with this flag set will not
+                                                        try to take pdump lock. If PDUMP_LOCK is called without
+                                                        this flag by some other thread then it will try to take
+                                                        PDUMP_LOCK which would make that thread sleep. This flag
+                                                        introduced to enforce the order of pdumping after bridge
+                                                        lock removal */
+
+#define PDUMP_FILEOFFSET_FMTSPEC    "0x%08X"
+typedef IMG_UINT32 PDUMP_FILEOFFSET_T;
+
+/* PDump stream macros*/
+
+/* Parameter stream */
+#define PDUMP_PARAM_INIT_STREAM_NAME       "paramInit"
+#define PDUMP_PARAM_MAIN_STREAM_NAME       "paramMain"
+#define PDUMP_PARAM_DEINIT_STREAM_NAME     "paramDeinit"
+#define PDUMP_PARAM_BLOCK_STREAM_NAME      "paramBlock"
+
+/*** Parameter stream sizes ***/
+
+/* Parameter Init Stream */
+
+/* The streams are made tunable in core.mk, wherein the size of these buffers can be specified.
+ * In case nothing specified in core.mk, then the default size is taken
+ */
+#if defined(PDUMP_PARAM_INIT_STREAM_SIZE)
+	#if (PDUMP_PARAM_INIT_STREAM_SIZE < (700 * 1024))
+		#error PDUMP_PARAM_INIT_STREAM_SIZE must be at least 700 KB
+	#endif
+#endif
+
+/* Parameter Main Stream */
+#if defined(PDUMP_PARAM_MAIN_STREAM_SIZE)
+	#if (PDUMP_PARAM_MAIN_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_PARAM_MAIN_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+/* Parameter Deinit Stream */
+#if defined(PDUMP_PARAM_DEINIT_STREAM_SIZE)
+	#if (PDUMP_PARAM_DEINIT_STREAM_SIZE < (64 * 1024))
+		#error PDUMP_PARAM_DEINIT_STREAM_SIZE must be at least 64 KB
+	#endif
+#endif
+
+/* Parameter Block Stream */
+/* There is no separate parameter Block stream as the Block script stream is
+ * just a filtered Main script stream. Hence it will refer to the Main stream
+ * parameters themselves.
+ */
+
+/*Script stream */
+#define PDUMP_SCRIPT_INIT_STREAM_NAME      "scriptInit"
+#define PDUMP_SCRIPT_MAIN_STREAM_NAME      "scriptMain"
+#define PDUMP_SCRIPT_DEINIT_STREAM_NAME    "scriptDeinit"
+#define PDUMP_SCRIPT_BLOCK_STREAM_NAME     "scriptBlock"
+
+/*** Script stream sizes ***/
+
+/* Script Init Stream */
+#if defined(PDUMP_SCRIPT_INIT_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_INIT_STREAM_SIZE < (256 * 1024))
+		#error PDUMP_SCRIPT_INIT_STREAM_SIZE must be at least 256 KB
+	#endif
+#endif
+
+/* Script Main Stream */
+#if defined(PDUMP_SCRIPT_MAIN_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_MAIN_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_SCRIPT_MAIN_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+/* Script Deinit Stream */
+#if defined(PDUMP_SCRIPT_DEINIT_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_DEINIT_STREAM_SIZE < (64 * 1024))
+		#error PDUMP_SCRIPT_DEINIT_STREAM_SIZE must be at least 64 KB
+	#endif
+#endif
+
+/* Script Block Stream */
+#if defined(PDUMP_SCRIPT_BLOCK_STREAM_SIZE)
+	#if (PDUMP_SCRIPT_BLOCK_STREAM_SIZE < (2 * 1024 * 1024))
+		#error PDUMP_SCRIPT_BLOCK_STREAM_SIZE must be at least 2 MB
+	#endif
+#endif
+
+
+#define PDUMP_PARAM_0_FILE_NAME     "%%0%%.prm"      /*!< Initial Param filename used in PDump capture */
+#define PDUMP_PARAM_N_FILE_NAME     "%%0%%_%02u.prm" /*!< Param filename used when PRM file split */
+#define PDUMP_PARAM_MAX_FILE_NAME   32               /*!< Max Size of parameter name used in out2.txt */
+
+#define PDUMP_IS_CONTINUOUS(flags) ((flags & PDUMP_FLAGS_CONTINUOUS) != 0)
+
+#endif /* SERVICES_PDUMP_H */
diff --git a/drivers/gpu/drm/img-rogue/pdump_km.h b/drivers/gpu/drm/img-rogue/pdump_km.h
new file mode 100644
index 000000000000..58713cdfa17d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdump_km.h
@@ -0,0 +1,1159 @@
+/*************************************************************************/ /*!
+@File
+@Title          pdump functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for pdump functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PDUMP_KM_H
+#define PDUMP_KM_H
+
+#if defined(PDUMP)
+ #if defined(__linux__)
+  #include <linux/version.h>
+  #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+   #include <linux/stdarg.h>
+  #else
+   #include <stdarg.h>
+  #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+ #else
+  #include <stdarg.h>
+ #endif /* __linux__ */
+#endif /* PDUMP */
+
+/* services/srvkm/include/ */
+#include "device.h"
+
+/* include/ */
+#include "pvrsrv_error.h"
+
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#include "connection_server.h"
+/* Pull in pdump flags from services include */
+#include "pdump.h"
+#include "pdumpdefs.h"
+
+/* Define this to enable the PDUMP_HERE trace in the server */
+#undef PDUMP_TRACE
+
+#if defined(PDUMP_TRACE)
+#define PDUMP_HERE_VAR  __maybe_unused IMG_UINT32 here = 0;
+#define PDUMP_HERE(a)	{ here = (a); if (ui32Flags & PDUMP_FLAGS_DEBUG) PVR_DPF((PVR_DBG_WARNING, "HERE %d", (a))); }
+#define PDUMP_HEREA(a)	{ here = (a); PVR_DPF((PVR_DBG_WARNING, "HERE ALWAYS %d", (a))); }
+#else
+#define PDUMP_HERE_VAR  __maybe_unused IMG_UINT32 here = 0;
+#define PDUMP_HERE(a)	here = (a);
+#define PDUMP_HEREA(a)	here = (a);
+#endif
+
+#define PDUMP_PD_UNIQUETAG	(IMG_HANDLE)0
+#define PDUMP_PT_UNIQUETAG	(IMG_HANDLE)0
+
+/* Defines for CMD:SetParity's state word */
+#define PDUMP_SET_PARITY_STATE_WORD_VERSION_SHIFT		(0U)
+#define PDUMP_SET_PARITY_STATE_WORD_VERSION_MASK		(0x00000007U)
+
+#define PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_SHIFT	(3U)
+#define PDUMP_SET_PARITY_STATE_WORD_PARITY_SHIFT_MASK	(0x000001F8U)
+
+#define PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_SHIFT		(9U)
+#define PDUMP_SET_PARITY_STATE_WORD_VA_PARITY_MASK		(0x00000200U)
+
+/* Invalid value for PDump block number */
+#define PDUMP_BLOCKNUM_INVALID      IMG_UINT32_MAX
+
+typedef struct _PDUMP_CONNECTION_DATA_ PDUMP_CONNECTION_DATA;
+
+/* PDump transition events */
+typedef enum _PDUMP_TRANSITION_EVENT_
+{
+	PDUMP_TRANSITION_EVENT_NONE,              /* No event */
+	PDUMP_TRANSITION_EVENT_BLOCK_FINISHED,    /* Block mode event, current PDump-block has finished */
+	PDUMP_TRANSITION_EVENT_BLOCK_STARTED,     /* Block mode event, new PDump-block has started */
+	PDUMP_TRANSITION_EVENT_RANGE_ENTERED,     /* Transition into capture range */
+	PDUMP_TRANSITION_EVENT_RANGE_APPEND,      /* Append to an already entered capture range */
+	PDUMP_TRANSITION_EVENT_RANGE_EXITED,      /* Transition out of capture range */
+} PDUMP_TRANSITION_EVENT;
+
+typedef PVRSRV_ERROR (*PFN_PDUMP_TRANSITION)(void *pvData, void *pvDevice, PDUMP_TRANSITION_EVENT eEvent, IMG_UINT32 ui32PDumpFlags);
+typedef void (*PFN_PDUMP_SYNCBLOCKS)(PVRSRV_DEVICE_NODE *psDevNode, void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+
+typedef PVRSRV_ERROR (*PFN_PDUMP_TRANSITION_FENCE_SYNC)(void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+
+#ifdef PDUMP
+
+/*! Macro used to record a panic in the PDump script stream */
+#define PDUMP_PANIC(_dev, _id, _msg) do \
+		{ PVRSRV_ERROR _eE;\
+			_eE = PDumpPanic((_dev), ((RGX_PDUMP_PANIC_ ## _id) & 0xFFFF), _msg, __func__, __LINE__);	\
+			PVR_LOG_IF_ERROR(_eE, "PDumpPanic");\
+		MSC_SUPPRESS_4127\
+		} while (0)
+
+/*! Macro used to record a driver error in the PDump script stream to invalidate the capture */
+#define PDUMP_ERROR(_dev, _err, _msg) \
+	(void)PDumpCaptureError((_dev), _err, _msg, __func__, __LINE__)
+
+#define SZ_MSG_SIZE_MAX			PVRSRV_PDUMP_MAX_COMMENT_SIZE
+#define SZ_SCRIPT_SIZE_MAX		PVRSRV_PDUMP_MAX_COMMENT_SIZE
+#define SZ_FILENAME_SIZE_MAX	(PVRSRV_PDUMP_MAX_FILENAME_SIZE+sizeof(PDUMP_PARAM_N_FILE_NAME))
+
+#define PDUMP_GET_SCRIPT_STRING()																			\
+	IMG_HANDLE hScript;																						\
+	void *pvScriptAlloc;																					\
+	IMG_UINT32 ui32MaxLen = SZ_SCRIPT_SIZE_MAX-1;															\
+	pvScriptAlloc = OSAllocMem( SZ_SCRIPT_SIZE_MAX );														\
+	if (!pvScriptAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_STRING() failed to allocate memory for script buffer"));	\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+																											\
+	hScript = (IMG_HANDLE) pvScriptAlloc;
+
+#define PDUMP_GET_MSG_STRING()																				\
+	IMG_CHAR *pszMsg;																						\
+	void *pvMsgAlloc;																						\
+	IMG_UINT32 ui32MaxLen = SZ_MSG_SIZE_MAX-1;																\
+	pvMsgAlloc = OSAllocMem( SZ_MSG_SIZE_MAX );																\
+	if (!pvMsgAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_MSG_STRING() failed to allocate memory for message buffer"));	\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+	pszMsg = (IMG_CHAR *)pvMsgAlloc;
+
+#define PDUMP_GET_SCRIPT_AND_FILE_STRING()																	\
+	IMG_HANDLE hScript;																						\
+	IMG_CHAR *pszFileName;																					\
+	IMG_UINT32 ui32MaxLenScript = SZ_SCRIPT_SIZE_MAX-1;														\
+	void *pvScriptAlloc;																					\
+	void *pvFileAlloc;																						\
+	pvScriptAlloc = OSAllocMem( SZ_SCRIPT_SIZE_MAX );														\
+	if (!pvScriptAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_AND_FILE_STRING() failed to allocate memory for script buffer"));		\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+																											\
+	hScript = (IMG_HANDLE) pvScriptAlloc;																	\
+	pvFileAlloc = OSAllocMem( SZ_FILENAME_SIZE_MAX );														\
+	if (!pvFileAlloc)																						\
+	{																										\
+		PVR_DPF((PVR_DBG_ERROR, "PDUMP_GET_SCRIPT_AND_FILE_STRING() failed to allocate memory for filename buffer"));	\
+		OSFreeMem(pvScriptAlloc);																			\
+		return PVRSRV_ERROR_OUT_OF_MEMORY;																	\
+	}																										\
+	pszFileName = (IMG_CHAR *)pvFileAlloc;
+
+#define PDUMP_RELEASE_SCRIPT_STRING()																		\
+	if (pvScriptAlloc)																						\
+	{																										\
+		OSFreeMem(pvScriptAlloc);																			\
+		pvScriptAlloc = NULL;																				\
+	}
+
+#define PDUMP_RELEASE_MSG_STRING()																			\
+	if (pvMsgAlloc)																							\
+	{																										\
+		OSFreeMem(pvMsgAlloc);																				\
+		pvMsgAlloc = NULL;																					\
+	}
+
+#define PDUMP_RELEASE_FILE_STRING()																			\
+	if (pvFileAlloc)																						\
+	{																										\
+		OSFreeMem(pvFileAlloc);																				\
+		pvFileAlloc = NULL;																					\
+	}
+
+#define PDUMP_RELEASE_SCRIPT_AND_FILE_STRING()																\
+	if (pvScriptAlloc)																						\
+	{																										\
+		OSFreeMem(pvScriptAlloc);																			\
+		pvScriptAlloc = NULL;																				\
+	}																										\
+	if (pvFileAlloc)																						\
+	{																										\
+		OSFreeMem(pvFileAlloc);																				\
+		pvFileAlloc = NULL;																					\
+	}
+
+
+/* Shared across pdump_x files */
+PVRSRV_ERROR PDumpInitCommon(void);
+void PDumpDeInitCommon(void);
+PVRSRV_ERROR PDumpValidateUMFlags(PDUMP_FLAGS_T uiFlags);
+PVRSRV_ERROR PDumpReady(void);
+void PDumpGetParameterZeroPageInfo(PDUMP_FILEOFFSET_T *puiZeroPageOffset,
+                                   size_t *puiZeroPageSize,
+                                   const IMG_CHAR **ppszZeroPageFilename);
+
+void PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode);
+void PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32Frame);
+PVRSRV_ERROR PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                             PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 *pui32Frame);
+PVRSRV_ERROR PDumpCommentKM(CONNECTION_DATA *psConnection,
+                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_UINT32 ui32CommentSize,
+                            IMG_CHAR *pszComment,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                                            PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_UINT32 ui32Mode,
+                                            IMG_UINT32 ui32Start,
+                                            IMG_UINT32 ui32End,
+                                            IMG_UINT32 ui32Interval,
+                                            IMG_UINT32 ui32MaxParamFileSize,
+                                            IMG_UINT32 ui32AutoTermTimeout);
+
+
+PVRSRV_ERROR PDumpReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_CHAR *pszPDumpRegName,
+                        IMG_UINT32 ui32RegAddr,
+                        IMG_UINT32 ui32RegValue,
+                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_CHAR *pszPDumpRegName,
+                        IMG_UINT32 ui32RegAddr,
+                        IMG_UINT64 ui64RegValue,
+                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32RegDst,
+                                  IMG_UINT32 ui32RegSrc,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpPhysHandleToInternalVar64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_CHAR *pszInternalVar,
+                                            IMG_HANDLE hPdumpPages,
+                                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToInternalVar64(IMG_CHAR *pszInternalVar,
+                                          PMR *psPMR,
+                                          IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                          IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpInternalVarToMemLabel(PMR *psPMR,
+                                        IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpWriteVarORValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const IMG_CHAR *pszInternalVariable,
+                                    const IMG_UINT64 ui64Value,
+                                    const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarANDValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarSHRValueOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const IMG_CHAR *pszInternalVariable,
+                                     const IMG_UINT64 ui64Value,
+                                     const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarORVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const IMG_CHAR *pszInternalVar,
+                                  const IMG_CHAR *pszInternalVar2,
+                                  const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpWriteVarANDVarOp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const IMG_CHAR *pszInternalVar,
+                                   const IMG_CHAR *pszInternalVar2,
+                                   const IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR PDumpInternalVarToReg32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpInternalVarToReg64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_CHAR *pszPDumpRegName,
+                                     IMG_UINT32 ui32Reg,
+                                     IMG_CHAR *pszInternalVar,
+                                     IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToMem32(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpMemLabelToMem64(PMR *psPMRSource,
+                                  PMR *psPMRDest,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToMem32(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToMem64(IMG_CHAR *pszPDumpRegName,
+                                  IMG_UINT32 ui32Reg,
+                                  PMR *psPMR,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegLabelToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR *pszPDumpRegName,
+                                        IMG_UINT32 ui32Reg,
+                                        IMG_CHAR *pszInternalVar,
+                                        IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpSAW(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR      *pszDevSpaceName,
+                      IMG_UINT32    ui32HPOffsetBytes,
+                      IMG_UINT32    ui32NumSaveBytes,
+                      IMG_CHAR      *pszOutfileName,
+                      IMG_UINT32    ui32OutfileOffsetByte,
+                      PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR PDumpRegPolKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           IMG_CHAR            *pszPDumpRegName,
+                           IMG_UINT32          ui32RegAddr,
+                           IMG_UINT32          ui32RegValue,
+                           IMG_UINT32          ui32Mask,
+                           IMG_UINT32          ui32Flags,
+                           PDUMP_POLL_OPERATOR eOperator);
+
+
+/**************************************************************************/ /*!
+@Function       PDumpImageDescriptor
+@Description    PDumps image data out as an IMGBv2 data section
+@Input          psDeviceNode         Pointer to device node.
+@Input          ui32MMUContextID     PDUMP MMU context ID.
+@Input          pszSABFileName       Pointer to string containing file name of
+                                     Image being SABed
+@Input          sData                GPU virtual address of this surface.
+@Input          ui32DataSize         Image data size
+@Input          ui32LogicalWidth     Image logical width
+@Input          ui32LogicalHeight    Image logical height
+@Input          ui32PhysicalWidth    Image physical width
+@Input          ui32PhysicalHeight   Image physical height
+@Input          ePixFmt              Image pixel format
+@Input          eFBCompression       FB compression mode
+@Input          paui32FBCClearColour FB clear colour (Only applicable to FBC surfaces)
+@Input          eFBCSwizzle          FBC channel swizzle (Only applicable to FBC surfaces)
+@Input          sHeader              GPU virtual address of the headers of this
+                                     surface (Only applicable to FBC surfaces)
+@Input          ui32HeaderSize       Header size (Only applicable to FBC surfaces)
+@Input          ui32PDumpFlags       PDUMP flags
+@Return         PVRSRV_ERROR:        PVRSRV_OK on success. Otherwise, a PVRSRV_
+                                     error code
+*/ /***************************************************************************/
+PVRSRV_ERROR PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  IMG_UINT32 ui32MMUContextID,
+                                  IMG_CHAR *pszSABFileName,
+                                  IMG_DEV_VIRTADDR sData,
+                                  IMG_UINT32 ui32DataSize,
+                                  IMG_UINT32 ui32LogicalWidth,
+                                  IMG_UINT32 ui32LogicalHeight,
+                                  IMG_UINT32 ui32PhysicalWidth,
+                                  IMG_UINT32 ui32PhysicalHeight,
+                                  PDUMP_PIXEL_FORMAT ePixFmt,
+                                  IMG_MEMLAYOUT eMemLayout,
+                                  IMG_FB_COMPRESSION eFBCompression,
+                                  const IMG_UINT32 *paui32FBCClearColour,
+                                  PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                                  IMG_DEV_VIRTADDR sHeader,
+                                  IMG_UINT32 ui32HeaderSize,
+                                  IMG_UINT32 ui32PDumpFlags);
+
+/**************************************************************************/ /*!
+@Function       PDumpDataDescriptor
+@Description    PDumps non-image data out as an IMGCv1 data section
+@Input          psDeviceNode         Pointer to device node.
+@Input          ui32MMUContextID     PDUMP MMU context ID.
+@Input          pszSABFileName       Pointer to string containing file name of
+                                     Data being SABed
+@Input          sData                GPU virtual address of this data.
+@Input          ui32DataSize         Data size
+@Input          ui32HeaderType       Header type
+@Input          ui32ElementType      Data element type
+@Input          ui32ElementCount     Number of data elements
+@Input          ui32PDumpFlags       PDUMP flags
+@Return         PVRSRV_ERROR:        PVRSRV_OK on success. Otherwise, a PVRSRV_
+                                     error code
+*/ /***************************************************************************/
+PVRSRV_ERROR PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32MMUContextID,
+                                 IMG_CHAR *pszSABFileName,
+                                 IMG_DEV_VIRTADDR sData,
+                                 IMG_UINT32 ui32DataSize,
+                                 IMG_UINT32 ui32HeaderType,
+                                 IMG_UINT32 ui32ElementType,
+                                 IMG_UINT32 ui32ElementCount,
+                                 IMG_UINT32 ui32PDumpFlags);
+
+
+PVRSRV_ERROR PDumpReadRegKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            IMG_CHAR *pszFileName,
+                            IMG_UINT32 ui32FileOffset,
+                            IMG_UINT32 ui32Address,
+                            IMG_UINT32 ui32Size,
+                            IMG_UINT32 ui32PDumpFlags);
+
+__printf(3, 4)
+PVRSRV_ERROR PDumpCommentWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_UINT32 ui32Flags,
+                                   IMG_CHAR* pszFormat,
+                                   ...);
+
+PVRSRV_ERROR PDumpCommentWithFlagsVA(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32Flags,
+                                     const IMG_CHAR * pszFormat,
+                                     va_list args);
+
+PVRSRV_ERROR PDumpPanic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT32      ui32PanicNo,
+                        IMG_CHAR*       pszPanicMsg,
+                        const IMG_CHAR* pszPPFunc,
+                        IMG_UINT32      ui32PPline);
+
+PVRSRV_ERROR PDumpCaptureError(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               PVRSRV_ERROR    ui32ErrorNo,
+                               IMG_CHAR*       pszErrorMsg,
+                               const IMG_CHAR* pszPPFunc,
+                               IMG_UINT32      ui32PPline);
+
+PVRSRV_ERROR PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame);
+
+PVRSRV_ERROR PDumpGetStateKM(IMG_UINT64 *ui64State);
+
+PVRSRV_ERROR PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PDumpRegRead32ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_CHAR *pszPDumpRegName,
+                                         IMG_UINT32 ui32RegOffset,
+                                         IMG_CHAR *pszInternalVar,
+                                         IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead32(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            const IMG_UINT32 dwRegOffset,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead64(PVRSRV_DEVICE_NODE *psDeviceNode,
+                            IMG_CHAR *pszPDumpRegName,
+                            const IMG_UINT32 dwRegOffset,
+                            IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpRegRead64ToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         IMG_CHAR *pszPDumpRegName,
+                                         IMG_CHAR *pszInternalVar,
+                                         const IMG_UINT32 dwRegOffset,
+                                         IMG_UINT32	ui32Flags);
+
+PVRSRV_ERROR PDumpIDLWithFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Clocks,
+                               IMG_UINT32 ui32Flags);
+PVRSRV_ERROR PDumpIDL(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32Clocks);
+
+PVRSRV_ERROR PDumpRegBasedCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_CHAR   *pszPDumpRegName,
+                              IMG_UINT32 ui32RegOffset,
+                              IMG_UINT32 ui32WPosVal,
+                              IMG_UINT32 ui32PacketSize,
+                              IMG_UINT32 ui32BufferSize,
+                              IMG_UINT32 ui32Flags);
+
+PVRSRV_ERROR PDumpTRG(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_CHAR *pszMemSpace,
+                      IMG_UINT32 ui32MMUCtxID,
+                      IMG_UINT32 ui32RegionID,
+                      IMG_BOOL bEnable,
+                      IMG_UINT64 ui64VAddr,
+                      IMG_UINT64 ui64LenBytes,
+                      IMG_UINT32 ui32XStride,
+                      IMG_UINT32 ui32Flags);
+
+void PDumpLock(void);
+void PDumpUnlock(void);
+
+PVRSRV_ERROR PDumpRegCondStr(IMG_CHAR            **ppszPDumpCond,
+                             IMG_CHAR            *pszPDumpRegName,
+                             IMG_UINT32          ui32RegAddr,
+                             IMG_UINT32          ui32RegValue,
+                             IMG_UINT32          ui32Mask,
+                             IMG_UINT32          ui32Flags,
+                             PDUMP_POLL_OPERATOR eOperator);
+
+PVRSRV_ERROR PDumpInternalValCondStr(IMG_CHAR            **ppszPDumpCond,
+                                     IMG_CHAR            *pszInternalVar,
+                                     IMG_UINT32          ui32RegValue,
+                                     IMG_UINT32          ui32Mask,
+                                     IMG_UINT32          ui32Flags,
+                                     PDUMP_POLL_OPERATOR eOperator);
+
+PVRSRV_ERROR PDumpIfKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpElseKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpFiKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                       IMG_CHAR *pszPDumpCond, IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpStartDoLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpEndDoWhileLoopKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_CHAR *pszPDumpWhileCond,
+                                   IMG_UINT32 ui32PDumpFlags);
+PVRSRV_ERROR PDumpCOMCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             IMG_UINT32 ui32PDumpFlags,
+                             const IMG_CHAR *pszPDump);
+
+void PDumpPowerTransitionStart(PVRSRV_DEVICE_NODE *psDeviceNode);
+void PDumpPowerTransitionEnd(PVRSRV_DEVICE_NODE *psDeviceNode);
+IMG_BOOL PDumpCheckFlagsWrite(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              IMG_UINT32 ui32Flags);
+
+/*!
+ * @name	PDumpWriteParameter
+ * @brief	General function for writing to PDump stream. Used
+ *          mainly for memory dumps to parameter stream.
+ *          Usually more convenient to use PDumpWriteScript below
+ *          for the script stream.
+ * @param	psDeviceNode - device PDump pertains to
+ * @param	psui8Data - data to write
+ * @param	ui32Size - size of write
+ * @param	ui32Flags - PDump flags
+ * @param   pui32FileOffset - on return contains the file offset to
+ *                            the start of the parameter data
+ * @param   aszFilenameStr - pointer to at least a 20 char buffer to
+ *                           return the parameter filename
+ * @return	error
+ */
+PVRSRV_ERROR PDumpWriteParameter(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT8 *psui8Data, IMG_UINT32 ui32Size,
+                                 IMG_UINT32 ui32Flags, IMG_UINT32* pui32FileOffset,
+                                 IMG_CHAR* aszFilenameStr);
+
+/*!
+ * @name	PDumpWriteScript
+ * @brief	Write an PDumpOS created string to the "script" output stream
+ * @param	psDeviceNode - device PDump pertains to
+ * @param	hString - PDump OS layer handle of string buffer to write
+ * @param	ui32Flags - PDump flags
+ * @return	IMG_TRUE on success.
+ */
+IMG_BOOL PDumpWriteScript(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          IMG_HANDLE hString, IMG_UINT32 ui32Flags);
+
+/**************************************************************************/ /*!
+@Function       PDumpSNPrintf
+@Description    Printf to OS-specific PDump state buffer. This function is
+                only called if PDUMP is defined.
+@Input          hBuf               handle of buffer to write into
+@Input          ui32ScriptSizeMax  maximum size of data to write (chars)
+@Input          pszFormat          format string
+@Return         None
+*/ /**************************************************************************/
+__printf(3, 4)
+PVRSRV_ERROR PDumpSNPrintf(IMG_HANDLE hBuf, IMG_UINT32 ui32ScriptSizeMax, IMG_CHAR* pszFormat, ...);
+
+
+/*
+	PDumpWriteShiftedMaskedValue():
+
+	loads the "reference" address into an internal PDump register,
+	optionally shifts it right,
+	optionally shifts it left,
+	optionally masks it
+	then finally writes the computed value to the given destination address
+
+	i.e. it emits pdump language equivalent to this expression:
+
+	 dest = ((&ref) >> SHRamount << SHLamount) & MASK
+*/
+PVRSRV_ERROR
+PDumpWriteShiftedMaskedValue(PVRSRV_DEVICE_NODE *psDeviceNode,
+                             const IMG_CHAR *pszDestRegspaceName,
+                             const IMG_CHAR *pszDestSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiDestOffset,
+                             const IMG_CHAR *pszRefRegspaceName,
+                             const IMG_CHAR *pszRefSymbolicName,
+                             IMG_DEVMEM_OFFSET_T uiRefOffset,
+                             IMG_UINT32 uiSHRAmount,
+                             IMG_UINT32 uiSHLAmount,
+                             IMG_UINT32 uiMask,
+                             IMG_DEVMEM_SIZE_T uiWordSize,
+                             IMG_UINT32 uiPDumpFlags);
+
+/*
+	PDumpWriteSymbAddress():
+	writes the address of the "reference" to the offset given
+*/
+PVRSRV_ERROR
+PDumpWriteSymbAddress(PVRSRV_DEVICE_NODE *psDeviceNode,
+                      const IMG_CHAR *pszDestSpaceName,
+                      IMG_DEVMEM_OFFSET_T uiDestOffset,
+                      const IMG_CHAR *pszRefSymbolicName,
+                      IMG_DEVMEM_OFFSET_T uiRefOffset,
+                      const IMG_CHAR *pszPDumpDevName,
+                      IMG_UINT32 ui32WordSize,
+                      IMG_UINT32 ui32AlignShift,
+                      IMG_UINT32 ui32Shift,
+                      IMG_UINT32 uiPDumpFlags);
+
+/* Register the connection with the PDump subsystem */
+PVRSRV_ERROR
+PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        void *hSyncPrivData,
+                        PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                        PDUMP_CONNECTION_DATA **ppsPDumpConnectionData);
+
+/* Unregister the connection with the PDump subsystem */
+void
+PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          PDUMP_CONNECTION_DATA *psPDumpConnectionData);
+
+/* Register for notification of PDump Transition into/out of capture range */
+PVRSRV_ERROR
+PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                PFN_PDUMP_TRANSITION pfnCallback,
+                                void *hPrivData,
+                                void *pvDevice,
+                                void **ppvHandle);
+
+/* Unregister notification of PDump Transition */
+void
+PDumpUnregisterTransitionCallback(void *pvHandle);
+
+PVRSRV_ERROR
+PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+                                         PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback,
+                                         void **ppvHandle);
+
+void
+PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle);
+
+/* Notify PDump of a Transition into/out of capture range */
+PVRSRV_ERROR
+PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                PDUMP_TRANSITION_EVENT eEvent,
+                IMG_UINT32 ui32PDumpFlags);
+
+/* Check if writing to a PDump file is permitted for the given device */
+IMG_BOOL PDumpIsDevicePermitted(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_LOCK(_ui32PDumpFlags) do \
+	{ if ((_ui32PDumpFlags & PDUMP_FLAGS_PDUMP_LOCK_HELD) == 0)\
+		{\
+			PDumpLock();\
+		}\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_UNLOCK(_ui32PDumpFlags) do \
+	{ if ((_ui32PDumpFlags & PDUMP_FLAGS_PDUMP_LOCK_HELD) == 0)\
+		{\
+			PDumpUnlock();\
+		}\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#define PDUMPINIT				PDumpInitCommon
+#define PDUMPDEINIT				PDumpDeInitCommon
+#define PDUMPREG32				PDumpReg32
+#define PDUMPREG64				PDumpReg64
+#define PDUMPREGREAD32			PDumpRegRead32
+#define PDUMPREGREAD64			PDumpRegRead64
+#define PDUMPCOMMENT(d, ...)	PDumpCommentWithFlags(d, PDUMP_FLAGS_CONTINUOUS, __VA_ARGS__)
+#define PDUMPCOMMENTWITHFLAGS	PDumpCommentWithFlags
+#define PDUMPREGPOL				PDumpRegPolKM
+#define PDUMPREGBASEDCBP		PDumpRegBasedCBP
+#define PDUMPENDINITPHASE		PDumpStopInitPhase
+#define PDUMPIDLWITHFLAGS		PDumpIDLWithFlags
+#define PDUMPIDL				PDumpIDL
+#define PDUMPPOWCMDSTART		PDumpPowerTransitionStart
+#define PDUMPPOWCMDEND			PDumpPowerTransitionEnd
+#define PDUMPCOM				PDumpCOMCommand
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_BLKSTART(_ui32PDumpFlags) do \
+	{ PDUMP_LOCK(_ui32PDumpFlags);\
+	_ui32PDumpFlags |= PDUMP_FLAGS_PDUMP_LOCK_HELD;\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMP_BLKEND(_ui32PDumpFlags) do \
+	{ _ui32PDumpFlags &= ~PDUMP_FLAGS_PDUMP_LOCK_HELD;\
+	PDUMP_UNLOCK(_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMPIF(_dev,_msg,_ui32PDumpFlags) do \
+	{PDUMP_BLKSTART(_ui32PDumpFlags);\
+	PDumpIfKM(_dev,_msg,_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#define PDUMPELSE				PDumpElseKM
+
+/* _ui32PDumpFlags must be a variable in the local scope */
+#define PDUMPFI(_dev,_msg,_ui32PDumpFlags) do \
+	{ PDumpFiKM(_dev,_msg,_ui32PDumpFlags);\
+	PDUMP_BLKEND(_ui32PDumpFlags);\
+	MSC_SUPPRESS_4127\
+	} while (0)
+
+#else
+/*
+	We should be clearer about which functions can be called
+	across the bridge as this looks rather unbalanced
+*/
+
+/*! Macro used to record a panic in the PDump script stream */
+#define PDUMP_PANIC(_dev, _id, _msg)  ((void)0)
+
+/*! Macro used to record a driver error in the PDump script stream to invalidate the capture */
+#define PDUMP_ERROR(_dev, _err, _msg) ((void)0)
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpInitCommon)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpInitCommon(void)
+{
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpConnectionNotify)
+#endif
+static INLINE void
+PDumpConnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpDisconnectionNotify)
+#endif
+static INLINE void
+PDumpDisconnectionNotify(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpLock)
+#endif
+static INLINE void
+PDumpLock(void)
+{
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnlock)
+#endif
+static INLINE void
+PDumpUnlock(void)
+{
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpStopInitPhase)
+#endif
+static INLINE void
+PDumpStopInitPhase(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpSetFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpSetFrameKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE *psDeviceNode,
+                IMG_UINT32 ui32Frame)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32Frame);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetFrameKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE *psDeviceNode,
+                IMG_UINT32 *pui32Frame)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pui32Frame);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpCommentKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpCommentKM(CONNECTION_DATA *psConnection,
+               PVRSRV_DEVICE_NODE *psDeviceNode,
+               IMG_UINT32 ui32CommentSize,
+               IMG_CHAR *pszComment,
+               IMG_UINT32 ui32Flags)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+    PVR_UNREFERENCED_PARAMETER(ui32CommentSize);
+	PVR_UNREFERENCED_PARAMETER(pszComment);
+	PVR_UNREFERENCED_PARAMETER(ui32Flags);
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpSetDefaultCaptureParamsKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpSetDefaultCaptureParamsKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32Mode,
+                               IMG_UINT32 ui32Start,
+                               IMG_UINT32 ui32End,
+                               IMG_UINT32 ui32Interval,
+                               IMG_UINT32 ui32MaxParamFileSize,
+                               IMG_UINT32 ui32AutoTermTimeout)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32Mode);
+	PVR_UNREFERENCED_PARAMETER(ui32Start);
+	PVR_UNREFERENCED_PARAMETER(ui32End);
+	PVR_UNREFERENCED_PARAMETER(ui32Interval);
+	PVR_UNREFERENCED_PARAMETER(ui32MaxParamFileSize);
+	PVR_UNREFERENCED_PARAMETER(ui32AutoTermTimeout);
+
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpPanic)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpPanic(IMG_UINT32      ui32PanicNo,
+           IMG_CHAR*       pszPanicMsg,
+           const IMG_CHAR* pszPPFunc,
+           IMG_UINT32      ui32PPline)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PanicNo);
+	PVR_UNREFERENCED_PARAMETER(pszPanicMsg);
+	PVR_UNREFERENCED_PARAMETER(pszPPFunc);
+	PVR_UNREFERENCED_PARAMETER(ui32PPline);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpCaptureError)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpCaptureError(PVRSRV_ERROR    ui32ErrorNo,
+                  IMG_CHAR*       pszErrorMsg,
+                  const IMG_CHAR* pszPPFunc,
+                  IMG_UINT32      ui32PPline)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32ErrorNo);
+	PVR_UNREFERENCED_PARAMETER(pszErrorMsg);
+	PVR_UNREFERENCED_PARAMETER(pszPPFunc);
+	PVR_UNREFERENCED_PARAMETER(ui32PPline);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpIsLastCaptureFrameKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpIsLastCaptureFrameKM(IMG_BOOL *pbIsLastCaptureFrame)
+{
+	*pbIsLastCaptureFrame = IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetStateKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetStateKM(IMG_UINT64 *ui64State)
+{
+	*ui64State = 0;
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpForceCaptureStopKM)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpForceCaptureStopKM(CONNECTION_DATA *psConnection,
+                        PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpImageDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpImageDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT32 ui32MMUContextID,
+                     IMG_CHAR *pszSABFileName,
+                     IMG_DEV_VIRTADDR sData,
+                     IMG_UINT32 ui32DataSize,
+                     IMG_UINT32 ui32LogicalWidth,
+                     IMG_UINT32 ui32LogicalHeight,
+                     IMG_UINT32 ui32PhysicalWidth,
+                     IMG_UINT32 ui32PhysicalHeight,
+                     PDUMP_PIXEL_FORMAT ePixFmt,
+                     IMG_MEMLAYOUT eMemLayout,
+                     IMG_FB_COMPRESSION eFBCompression,
+                     const IMG_UINT32 *paui32FBCClearColour,
+                     PDUMP_FBC_SWIZZLE eFBCSwizzle,
+                     IMG_DEV_VIRTADDR sHeader,
+                     IMG_UINT32 ui32HeaderSize,
+                     IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32LogicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalWidth);
+	PVR_UNREFERENCED_PARAMETER(ui32PhysicalHeight);
+	PVR_UNREFERENCED_PARAMETER(ePixFmt);
+	PVR_UNREFERENCED_PARAMETER(eMemLayout);
+	PVR_UNREFERENCED_PARAMETER(eFBCompression);
+	PVR_UNREFERENCED_PARAMETER(paui32FBCClearColour);
+	PVR_UNREFERENCED_PARAMETER(eFBCSwizzle);
+	PVR_UNREFERENCED_PARAMETER(sHeader);
+	PVR_UNREFERENCED_PARAMETER(ui32HeaderSize);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpDataDescriptor)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpDataDescriptor(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    IMG_UINT32 ui32MMUContextID,
+                    IMG_CHAR *pszSABFileName,
+                    IMG_DEV_VIRTADDR sData,
+                    IMG_UINT32 ui32DataSize,
+                    IMG_UINT32 ui32ElementType,
+                    IMG_UINT32 ui32ElementCount,
+                    IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32MMUContextID);
+	PVR_UNREFERENCED_PARAMETER(pszSABFileName);
+	PVR_UNREFERENCED_PARAMETER(sData);
+	PVR_UNREFERENCED_PARAMETER(ui32DataSize);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementType);
+	PVR_UNREFERENCED_PARAMETER(ui32ElementCount);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterConnection)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        void *hSyncPrivData,
+                        PFN_PDUMP_SYNCBLOCKS pfnPDumpSyncBlocks,
+                        PDUMP_CONNECTION_DATA **ppsPDumpConnectionData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hSyncPrivData);
+	PVR_UNREFERENCED_PARAMETER(pfnPDumpSyncBlocks);
+	PVR_UNREFERENCED_PARAMETER(ppsPDumpConnectionData);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterConnection)
+#endif
+static INLINE void
+PDumpUnregisterConnection(PVRSRV_DEVICE_NODE *psDeviceNode,
+                          PDUMP_CONNECTION_DATA *psPDumpConnectionData)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterTransitionCallback)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterTransitionCallback(PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                                PFN_PDUMP_TRANSITION pfnCallback,
+                                void *hPrivData,
+                                void *pvDevice,
+                                void **ppvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+	PVR_UNREFERENCED_PARAMETER(pfnCallback);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(pvDevice);
+	PVR_UNREFERENCED_PARAMETER(ppvHandle);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterTransitionCallback)
+#endif
+static INLINE void
+PDumpUnregisterTransitionCallback(void *pvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvHandle);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpRegisterTransitionCallback)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpRegisterTransitionCallbackFenceSync(void *hPrivData,
+                                         PFN_PDUMP_TRANSITION_FENCE_SYNC pfnCallback,
+                                         void **ppvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pfnCallback);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	PVR_UNREFERENCED_PARAMETER(ppvHandle);
+
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpUnregisterTransitionCallbackFenceSync)
+#endif
+static INLINE void
+PDumpUnregisterTransitionCallbackFenceSync(void *pvHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(pvHandle);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpTransition)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpTransition(PVRSRV_DEVICE_NODE *psDeviceNode,
+                PDUMP_CONNECTION_DATA *psPDumpConnectionData,
+                PDUMP_TRANSITION_EVENT eEvent,
+                IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psPDumpConnectionData);
+	PVR_UNREFERENCED_PARAMETER(eEvent);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+#if defined(__linux__) || defined(GCC_IA32) || defined(GCC_ARM) || defined(__QNXNTO__) || defined(INTEGRITY_OS)
+	#define PDUMPINIT	PDumpInitCommon
+	#define PDUMPDEINIT(args...)
+	#define PDUMPREG32(args...)
+	#define PDUMPREG64(args...)
+	#define PDUMPREGREAD32(args...)
+	#define PDUMPREGREAD64(args...)
+	#define PDUMPCOMMENT(args...)
+	#define PDUMPREGPOL(args...)
+	#define PDUMPSYNC(args...)
+	#define PDUMPCOPYTOMEM(args...)
+	#define PDUMPWRITE(args...)
+	#define PDUMPREGBASEDCBP(args...)
+	#define PDUMPCOMMENTWITHFLAGS(args...)
+	#define PDUMPENDINITPHASE(args...)
+	#define PDUMPIDLWITHFLAGS(args...)
+	#define PDUMPIDL(args...)
+	#define PDUMPPOWCMDSTART(args...)
+	#define PDUMPPOWCMDEND(args...)
+	#define PDUMP_LOCK(args...)
+	#define PDUMP_UNLOCK(args...)
+	#define PDUMPIF(args...)
+	#define PDUMPFI(args...)
+	#define PDUMPCOM(args...)
+#else
+	#error Compiler not specified
+#endif
+
+#endif /* PDUMP */
+
+#endif /* PDUMP_KM_H */
+
+/******************************************************************************
+ End of file (pdump_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pdump_mmu.h b/drivers/gpu/drm/img-rogue/pdump_mmu.h
new file mode 100644
index 000000000000..5d03e430febd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdump_mmu.h
@@ -0,0 +1,150 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common MMU Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_PDUMP_MMU_H
+#define SRVKM_PDUMP_MMU_H
+
+/* services/server/include/ */
+#include "pdump_symbolicaddr.h"
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "mmu_common.h"
+
+#include "opaque_types.h"
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PDumpMMUMalloc(PPVRSRV_DEVICE_NODE psDeviceNode,
+               const IMG_CHAR *pszPDumpDevName,
+               MMU_LEVEL eMMULevel,
+               IMG_DEV_PHYADDR *psDevPAddr,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32Align,
+               PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpMMUFree(PPVRSRV_DEVICE_NODE psDeviceNode,
+             const IMG_CHAR *pszPDumpDevName,
+             MMU_LEVEL eMMULevel,
+             IMG_DEV_PHYADDR *psDevPAddr,
+             PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpPTBaseObjectToMem64(const IMG_CHAR *pszPDumpDevName,
+                         PMR *psPMRDest,
+                         IMG_DEVMEM_OFFSET_T uiLogicalOffsetSource,
+                         IMG_DEVMEM_OFFSET_T uiLogicalOffsetDest,
+                         IMG_UINT32 ui32Flags,
+                         MMU_LEVEL eMMULevel,
+                         IMG_UINT64 ui64PxSymAddr,
+                         IMG_UINT64 ui64PxOffset);
+
+PVRSRV_ERROR
+PDumpMMUDumpPxEntries(PPVRSRV_DEVICE_NODE psDeviceNode,
+                      MMU_LEVEL eMMULevel,
+                      const IMG_CHAR *pszPDumpDevName,
+                      void *pvPxMem,
+                      IMG_DEV_PHYADDR sPxDevPAddr,
+                      IMG_UINT32 uiFirstEntry,
+                      IMG_UINT32 uiNumEntries,
+                      const IMG_CHAR *pszMemspaceName,
+                      const IMG_CHAR *pszSymbolicAddr,
+                      IMG_UINT64 uiSymbolicAddrOffset,
+                      IMG_UINT32 uiBytesPerEntry,
+                      IMG_UINT32 uiLog2Align,
+                      IMG_UINT32 uiAddrShift,
+                      IMG_UINT64 uiAddrMask,
+                      IMG_UINT64 uiPxEProtMask,
+                      IMG_UINT64 uiDataValidEnable,
+                      IMG_UINT32 ui32Flags,
+                      IMG_UINT32 ui32VAParity,
+                      IMG_UINT32 ui32ParityShift,
+                      IMG_UINT64 ui64ParityMask,
+                      PDUMP_MMU_TYPE eMMUType);
+
+PVRSRV_ERROR
+PDumpMMUAllocMMUContext(PPVRSRV_DEVICE_NODE psDeviceNode,
+                        const IMG_CHAR *pszPDumpMemSpaceName,
+                        IMG_DEV_PHYADDR sPCDevPAddr,
+                        PDUMP_MMU_TYPE eMMUType,
+                        IMG_UINT32 *pui32MMUContextID,
+                        IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMMUFreeMMUContext(PPVRSRV_DEVICE_NODE psDeviceNode,
+                       const IMG_CHAR *pszPDumpMemSpaceName,
+                       IMG_UINT32 ui32MMUContextID,
+                       IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMMUSAB(PPVRSRV_DEVICE_NODE psDeviceNode,
+            const IMG_CHAR *pszPDumpMemNamespace,
+            IMG_UINT32 uiPDumpMMUCtx,
+            IMG_DEV_VIRTADDR sDevAddrStart,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            IMG_UINT32 ui32PDumpFlags);
+
+#define PDUMP_MMU_ALLOC_MMUCONTEXT(psDevNode, pszPDumpMemDevName, sPCDevPAddr, eMMUType, puiPDumpCtxID, ui32PDumpFlags) \
+        PDumpMMUAllocMMUContext(psDevNode,                              \
+                                pszPDumpMemDevName,                     \
+                                sPCDevPAddr,                            \
+                                eMMUType,                               \
+                                puiPDumpCtxID,                          \
+                                ui32PDumpFlags)
+
+#define PDUMP_MMU_FREE_MMUCONTEXT(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags) \
+        PDumpMMUFreeMMUContext(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags)
+#else /* PDUMP */
+
+#define PDUMP_MMU_ALLOC_MMUCONTEXT(pszPDumpMemDevName, sPCDevPAddr, eMMUType, puiPDumpCtxID, ui32PDumpFlags) \
+        ((void)0)
+#define PDUMP_MMU_FREE_MMUCONTEXT(psDevNode, pszPDumpMemDevName, uiPDumpCtxID, ui32PDumpFlags) \
+        ((void)0)
+
+#endif /* PDUMP */
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/pdump_physmem.h b/drivers/gpu/drm/img-rogue/pdump_physmem.h
new file mode 100644
index 000000000000..bf38df94f3fa
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdump_physmem.h
@@ -0,0 +1,300 @@
+/**************************************************************************/ /*!
+@File
+@Title          pdump functions to assist with physmem allocations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements basic low level control of MMU.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PDUMP_PHYSMEM_H
+#define SRVSRV_PDUMP_PHYSMEM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pmr.h"
+#include "device.h" /* For device node */
+
+#define PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH 40
+#define PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH 60
+#define PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH (PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH + PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH)
+
+typedef struct _PDUMP_PHYSMEM_INFO_T_ PDUMP_PHYSMEM_INFO_T;
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                     IMG_CHAR **ppszSymbolicAddress);
+
+PVRSRV_ERROR
+PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicAddress,
+            IMG_UINT64 ui64Size,
+            /* alignment is alignment of start of buffer _and_
+               minimum contiguity - i.e. smallest allowable
+               page-size. */
+            IMG_DEVMEM_ALIGN_T uiAlign,
+            IMG_BOOL bInitialise,
+            IMG_UINT8 ui8InitValue,
+            IMG_HANDLE *phHandlePtr,
+            IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    const IMG_CHAR *pszDevSpace,
+                    const IMG_CHAR *pszSymbolicAddress,
+                    IMG_UINT64 ui64Size,
+                    /* alignment is alignment of start of buffer _and_
+                       minimum contiguity - i.e. smallest allowable
+                       page-size. */
+                    IMG_DEVMEM_ALIGN_T uiAlign,
+                    IMG_BOOL bInitialise,
+                    IMG_UINT8 ui8InitValue,
+                    IMG_HANDLE *phHandlePtr,
+                    IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR
+PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+          IMG_HANDLE hPDumpAllocationInfoHandle);
+
+PVRSRV_ERROR
+PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                  IMG_HANDLE hPDumpAllocationInfoHandle);
+
+void
+PDumpMakeStringValid(IMG_CHAR *pszString,
+                     IMG_UINT32 ui32StrLen);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PDumpGetSymbolicAddr)
+#endif
+static INLINE PVRSRV_ERROR
+PDumpGetSymbolicAddr(const IMG_HANDLE hPhysmemPDumpHandle,
+                     IMG_CHAR **ppszSymbolicAddress)
+{
+	PVR_UNREFERENCED_PARAMETER(hPhysmemPDumpHandle);
+	PVR_UNREFERENCED_PARAMETER(ppszSymbolicAddress);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpMalloc(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicAddress,
+            IMG_UINT64 ui64Size,
+            IMG_DEVMEM_ALIGN_T uiAlign,
+            IMG_BOOL bInitialise,
+            IMG_UINT8 ui8InitValue,
+            IMG_HANDLE *phHandlePtr,
+            IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pszDevSpace);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(uiAlign);
+	PVR_UNREFERENCED_PARAMETER(bInitialise);
+	PVR_UNREFERENCED_PARAMETER(ui8InitValue);
+	PVR_UNREFERENCED_PARAMETER(phHandlePtr);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpMallocUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                    const IMG_CHAR *pszDevSpace,
+                    const IMG_CHAR *pszSymbolicAddress,
+                    IMG_UINT64 ui64Size,
+                    IMG_DEVMEM_ALIGN_T uiAlign,
+                    IMG_BOOL bInitialise,
+                    IMG_UINT8 ui8InitValue,
+                    IMG_HANDLE *phHandlePtr,
+                    IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(pszDevSpace);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddress);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(uiAlign);
+	PVR_UNREFERENCED_PARAMETER(bInitialise);
+	PVR_UNREFERENCED_PARAMETER(ui8InitValue);
+	PVR_UNREFERENCED_PARAMETER(phHandlePtr);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+          IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hPDumpAllocationInfoHandle);
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+PDumpFreeUnlocked(PVRSRV_DEVICE_NODE *psDeviceNode,
+                  IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(hPDumpAllocationInfoHandle);
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+#define PMR_DEFAULT_PREFIX "PMR"
+#define PMR_SPARSE_PREFIX "SPMR"
+#define PMR_SYMBOLICADDR_FMTSPEC "%s%"IMG_UINT64_FMTSPEC"_%"IMG_UINT64_FMTSPEC"_%s"
+#define PMR_MEMSPACE_FMTSPEC "%s"
+#define PMR_MEMSPACE_CACHE_COHERENT_FMTSPEC "CC_%s"
+
+PVRSRV_ERROR
+PDumpPMRWRW32(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT32 ui32Value,
+              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW32InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRRDW32MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW64(PVRSRV_DEVICE_NODE *psDeviceNode,
+              const IMG_CHAR *pszDevSpace,
+              const IMG_CHAR *pszSymbolicName,
+              IMG_DEVMEM_OFFSET_T uiOffset,
+              IMG_UINT64 ui64Value,
+              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRWRW64InternalVarToMem(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              const IMG_CHAR *pszInternalVar,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRRDW64MemToInternalVar(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const IMG_CHAR *pszInternalVar,
+                              const IMG_CHAR *pszDevSpace,
+                              const IMG_CHAR *pszSymbolicName,
+                              IMG_DEVMEM_OFFSET_T uiOffset,
+                              PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRLDB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFilename,
+            IMG_UINT32 uiFileOffset,
+            PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRSAB(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszDevSpace,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_DEVMEM_SIZE_T uiSize,
+            const IMG_CHAR *pszFileName,
+            IMG_UINT32 uiFileOffset);
+
+/*
+  PDumpPMRPOL()
+
+  Emits a POL to the PDUMP.
+*/
+PVRSRV_ERROR
+PDumpPMRPOL(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMempaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiOffset,
+            IMG_UINT32 ui32Value,
+            IMG_UINT32 ui32Mask,
+            PDUMP_POLL_OPERATOR eOperator,
+            IMG_UINT32 uiCount,
+            IMG_UINT32 uiDelay,
+            PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PDumpPMRCBP(PVRSRV_DEVICE_NODE *psDeviceNode,
+            const IMG_CHAR *pszMemspaceName,
+            const IMG_CHAR *pszSymbolicName,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize);
+
+/*
+ * PDumpWriteParameterBlob()
+ *
+ * Writes a binary blob to the pdump param stream containing the current
+ * contents of the memory, and returns the filename and offset of where
+ * that blob is located (for use in a subsequent LDB, for example).
+ *
+ * Caller to provide buffer to receive filename, and declare the size of
+ * that buffer.
+ */
+PVRSRV_ERROR
+PDumpWriteParameterBlob(PVRSRV_DEVICE_NODE *psDeviceNode,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiNumBytes,
+                        PDUMP_FLAGS_T uiPDumpFlags,
+                        IMG_CHAR *pszFilenameOut,
+                        size_t uiFilenameBufSz,
+                        PDUMP_FILEOFFSET_T *puiOffsetOut);
+
+#endif /* #ifndef SRVSRV_PDUMP_PHYSMEM_H */
diff --git a/drivers/gpu/drm/img-rogue/pdump_symbolicaddr.h b/drivers/gpu/drm/img-rogue/pdump_symbolicaddr.h
new file mode 100644
index 000000000000..ed912a5096c9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdump_symbolicaddr.h
@@ -0,0 +1,55 @@
+/**************************************************************************/ /*!
+@File
+@Title          Abstraction of PDUMP symbolic address derivation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Allows pdump functions to derive symbolic addresses on-the-fly
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_PDUMP_SYMBOLICADDR_H
+#define SRVKM_PDUMP_SYMBOLICADDR_H
+
+#include "img_types.h"
+
+#include "pvrsrv_error.h"
+
+/* pdump symbolic addresses are generated on-the-fly with a callback */
+
+typedef PVRSRV_ERROR (*PVRSRV_SYMADDRFUNCPTR)(IMG_HANDLE hPriv, IMG_UINT32 uiOffset, IMG_CHAR *pszSymbolicAddr, IMG_UINT32 ui32SymbolicAddrLen, IMG_UINT32 *pui32NewOffset);
+
+#endif /* #ifndef SRVKM_PDUMP_SYMBOLICADDR_H */
diff --git a/drivers/gpu/drm/img-rogue/pdumpdefs.h b/drivers/gpu/drm/img-rogue/pdumpdefs.h
new file mode 100644
index 000000000000..7cea362a4152
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdumpdefs.h
@@ -0,0 +1,268 @@
+/*************************************************************************/ /*!
+@File
+@Title          PDUMP definitions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    PDUMP definitions header
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PDUMPDEFS_H
+#define PDUMPDEFS_H
+
+/*! PDump Pixel Format Enumeration */
+typedef enum _PDUMP_PIXEL_FORMAT_
+{
+	PVRSRV_PDUMP_PIXEL_FORMAT_UNSUPPORTED = 0,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB8 = 1,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB332 = 2,
+	PVRSRV_PDUMP_PIXEL_FORMAT_KRGB555 = 3,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB565 = 4,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB4444 = 5,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB1555 = 6,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB888 = 7,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB8888 = 8,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV8 = 9,
+/*	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV4444 = 10, */
+	PVRSRV_PDUMP_PIXEL_FORMAT_VY0UY1_8888 = 11,
+	PVRSRV_PDUMP_PIXEL_FORMAT_UY0VY1_8888 = 12,
+	PVRSRV_PDUMP_PIXEL_FORMAT_Y0UY1V_8888 = 13,
+	PVRSRV_PDUMP_PIXEL_FORMAT_Y0VY1U_8888 = 14,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV888 = 15,
+	PVRSRV_PDUMP_PIXEL_FORMAT_UYVY10101010 = 16,
+	PVRSRV_PDUMP_PIXEL_FORMAT_VYAUYA8888 = 17,
+	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV8888 = 18,
+	PVRSRV_PDUMP_PIXEL_FORMAT_AYUV2101010 = 19,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV101010 = 20,
+	PVRSRV_PDUMP_PIXEL_FORMAT_PL12Y8 = 21,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_IMC2 = 22,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV12 = 23,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_PL8 = 24,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_PL12 = 25,
+	PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV8 = 26,
+	PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV8 = 27,
+	PVRSRV_PDUMP_PIXEL_FORMAT_PL12Y10 = 28,
+	PVRSRV_PDUMP_PIXEL_FORMAT_422PL12YUV10 = 29,
+	PVRSRV_PDUMP_PIXEL_FORMAT_420PL12YUV10 = 30,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR8888 = 31,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA8888 = 32,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB8332 = 33,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGB555 = 34,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16 = 35,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32 = 36,
+	PVRSRV_PDUMP_PIXEL_FORMAT_L16 = 37,
+	PVRSRV_PDUMP_PIXEL_FORMAT_L32 = 38,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA8888 = 39,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR4444 = 40,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA4444 = 41,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA4444 = 42,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR1555 = 43,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA5551 = 44,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA5551 = 45,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGR565 = 46,
+	PVRSRV_PDUMP_PIXEL_FORMAT_A8 = 47,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16F16F16 = 49,
+	PVRSRV_PDUMP_PIXEL_FORMAT_A4 = 50,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB2101010 = 51,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RSGSBS888 = 52,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32F32F32 = 53,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16 = 54,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32 = 55,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F16F16F16 = 56,
+	PVRSRV_PDUMP_PIXEL_FORMAT_F32F32F32 = 57,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U8 = 58,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U8U8 = 59,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16 = 60,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16U16 = 61,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U16U16U16U16 = 62,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32 = 63,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32U32 = 64,
+	PVRSRV_PDUMP_PIXEL_FORMAT_U32U32U32U32 = 65,
+	PVRSRV_PDUMP_PIXEL_FORMAT_YUV_YV32 = 66,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ABGR10_PACK16 = 67,
+	PVRSRV_PDUMP_PIXEL_FORMAT_ARGB10_PACK16 = 68,
+	PVRSRV_PDUMP_PIXEL_FORMAT_BGRA10_PACK16 = 69,
+	PVRSRV_PDUMP_PIXEL_FORMAT_RGBA10_PACK16 = 70,
+
+	PVRSRV_PDUMP_PIXEL_FORMAT_FORCE_I32 = 0x7fffffff
+
+} PDUMP_PIXEL_FORMAT;
+
+typedef enum _PDUMP_FBC_SWIZZLE_
+{
+	PVRSRV_PDUMP_FBC_SWIZZLE_ARGB = 0x0,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ARBG = 0x1,
+	PVRSRV_PDUMP_FBC_SWIZZLE_AGRB = 0x2,
+	PVRSRV_PDUMP_FBC_SWIZZLE_AGBR = 0x3,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ABGR = 0x4,
+	PVRSRV_PDUMP_FBC_SWIZZLE_ABRG = 0x5,
+	PVRSRV_PDUMP_FBC_SWIZZLE_RGBA = 0x8,
+	PVRSRV_PDUMP_FBC_SWIZZLE_RBGA = 0x9,
+	PVRSRV_PDUMP_FBC_SWIZZLE_GRBA = 0xA,
+	PVRSRV_PDUMP_FBC_SWIZZLE_GBRA = 0xB,
+	PVRSRV_PDUMP_FBC_SWIZZLE_BGRA = 0xC,
+	PVRSRV_PDUMP_FBC_SWIZZLE_BRGA = 0xD,
+} PDUMP_FBC_SWIZZLE;
+
+/*! PDump addrmode */
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT			0
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_MASK			0x000000FF
+
+#define PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_SHIFT			8
+#define PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_NEGATIVE		(1U << PVRSRV_PDUMP_ADDRMODE_STRIDESENSE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT		11
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_MASK		0x00000800
+
+#define PVRSRV_PDUMP_ADDRMODE_BIFTILE_MODE_SHIFT		12
+#define PVRSRV_PDUMP_ADDRMODE_BIFTILE_MODE_MASK			0x000FF000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT				20
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_MASK				0x00F00000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCDECOR_SHIFT			24
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_SHIFT			25
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT		26
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_MASK		0x0C000000
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT			28
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_MASK			0xF0000000
+
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_STRIDE			(0U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE1 (1U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE2 (2U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE3 (3U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE4 (4U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE5 (5U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE6 (6U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_BIFTILE_STRIDE7 (7U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_TWIDDLED		(9U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_PAGETILED		(11U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_ZTWIDDLED		(12U << PVRSRV_PDUMP_ADDRMODE_MEMFORMAT_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_25_50_75	(0U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_25_37_50	(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_GROUP_CONTROL_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_NONE				(0U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_DIRECT		(1U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_DIRECT		(2U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_32X2_DIRECT		(3U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_INDIRECT		(4U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_INDIRECT		(5U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_8X8_INDIRECT_4TILE	(6U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCMODE_16X4_INDIRECT_4TILE	(7U << PVRSRV_PDUMP_ADDRMODE_FBCMODE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_NONE		(0U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_75		(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_37		(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_50		(2U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_25		(3U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_TFBC_MODE_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBC_DECOR					(1U << PVRSRV_PDUMP_ADDRMODE_FBCDECOR_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBC_LOSSY					(1U << PVRSRV_PDUMP_ADDRMODE_FBCLOSSY_SHIFT)
+
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_BASE			(1U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_ENHANCED		(2U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V2				(3U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_SURFACE		(4U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_RESOURCE		(5U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_1_SURFACE	(6U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V3_1_RESOURCE	(7U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V4				(8U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_V4PLUS			(9U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+#define PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_TFBCDC			(10U << PVRSRV_PDUMP_ADDRMODE_FBCCOMPAT_SHIFT)
+
+/*! PDump Poll Operator */
+typedef enum _PDUMP_POLL_OPERATOR
+{
+	PDUMP_POLL_OPERATOR_EQUAL = 0,
+	PDUMP_POLL_OPERATOR_LESS = 1,
+	PDUMP_POLL_OPERATOR_LESSEQUAL = 2,
+	PDUMP_POLL_OPERATOR_GREATER = 3,
+	PDUMP_POLL_OPERATOR_GREATEREQUAL = 4,
+	PDUMP_POLL_OPERATOR_NOTEQUAL = 5,
+} PDUMP_POLL_OPERATOR;
+
+
+#define PVRSRV_PDUMP_MAX_FILENAME_SIZE			75  /*!< Max length of a pdump log file name */
+#define PVRSRV_PDUMP_MAX_COMMENT_SIZE			350 /*!< Max length of a pdump comment */
+
+/*!
+	PDump MMU type
+*/
+typedef enum
+{
+	PDUMP_MMU_TYPE_4KPAGE_32BIT_STDTILE  = 1,
+	PDUMP_MMU_TYPE_VARPAGE_32BIT_STDTILE = 2,
+	PDUMP_MMU_TYPE_4KPAGE_36BIT_EXTTILE  = 3,
+	PDUMP_MMU_TYPE_4KPAGE_32BIT_EXTTILE  = 4,
+	PDUMP_MMU_TYPE_4KPAGE_36BIT_STDTILE  = 5,
+	PDUMP_MMU_TYPE_VARPAGE_40BIT         = 6,
+	PDUMP_MMU_TYPE_VIDEO_40BIT_STDTILE   = 7,
+	PDUMP_MMU_TYPE_VIDEO_40BIT_EXTTILE   = 8,
+	PDUMP_MMU_TYPE_MIPS_MICROAPTIV       = 9,
+	PDUMP_MMU_TYPE_LAST
+} PDUMP_MMU_TYPE;
+
+/*!
+	PDump states
+	These values are used by the bridge call PVRSRVPDumpGetState
+*/
+#define PDUMP_STATE_CAPTURE_FRAME		(1U)		/*!< Flag represents the PDump being in capture range or not*/
+#define PDUMP_STATE_CONNECTED			(2U)		/*!< Flag represents the PDump Client App being connected on not */
+#define PDUMP_STATE_SUSPENDED			(4U)		/*!< Flag represents the PDump being suspended or not */
+#define PDUMP_STATE_CAPTURE_IN_INTERVAL	(8U)		/*!< Flag represents the PDump being in a capture range interval */
+#define PDUMP_STATE_APP_TERMINATED		(16U)		/*!< Flag represents the PDump captured app has been terminated */
+
+/*!
+	PDump Capture modes
+	Values used with calls to PVRSRVPDumpSetDefaultCaptureParams
+*/
+#define PDUMP_CAPMODE_UNSET                     0x00000000UL
+#define PDUMP_CAPMODE_FRAMED                    0x00000001UL
+#define PDUMP_CAPMODE_CONTINUOUS                0x00000002UL
+#define PDUMP_CAPMODE_BLOCKED                   0x00000003UL
+
+#define PDUMP_CAPMODE_MAX                       PDUMP_CAPMODE_BLOCKED
+
+#endif /* PDUMPDEFS_H */
+
+/*****************************************************************************
+ End of file (pdumpdefs.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pdumpdesc.h b/drivers/gpu/drm/img-rogue/pdumpdesc.h
new file mode 100644
index 000000000000..273cde910e7f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pdumpdesc.h
@@ -0,0 +1,230 @@
+/*************************************************************************/ /*!
+@File           pdumpdesc.h
+@Title          PDump Descriptor format
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Describes PDump descriptors that may be passed to the
+                extraction routines (SAB).
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PDUMPDESC_H)
+#define PDUMPDESC_H
+
+#include "pdumpdefs.h"
+
+/*
+ * Common fields
+ */
+#define HEADER_WORD0_TYPE_SHIFT				(0)
+#define HEADER_WORD0_TYPE_CLRMSK			(0xFFFFFFFFU)
+
+#define HEADER_WORD1_SIZE_SHIFT				(0)
+#define HEADER_WORD1_SIZE_CLRMSK			(0x0000FFFFU)
+#define HEADER_WORD1_VERSION_SHIFT			(16)
+#define HEADER_WORD1_VERSION_CLRMSK			(0xFFFF0000U)
+
+#define HEADER_WORD2_DATA_SIZE_SHIFT		(0)
+#define HEADER_WORD2_DATA_SIZE_CLRMSK		(0xFFFFFFFFU)
+
+
+/*
+ * The image type descriptor
+ */
+
+/*
+ * Header type (IMGBv3) - 'IMGB' in hex + VERSION 3
+ * Header size - 64 bytes
+ */
+#define IMAGE_HEADER_TYPE					(0x42474D49)
+#define IMAGE_HEADER_SIZE					(64)
+#define IMAGE_HEADER_VERSION				(3)
+
+/*
+ * Image type-specific fields
+ */
+#define IMAGE_HEADER_WORD3_LOGICAL_WIDTH_SHIFT		(0)
+#define IMAGE_HEADER_WORD3_LOGICAL_WIDTH_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_SHIFT		(0)
+#define IMAGE_HEADER_WORD4_LOGICAL_HEIGHT_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD5_FORMAT_SHIFT				(0)
+#define IMAGE_HEADER_WORD5_FORMAT_CLRMSK			(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_SHIFT		(0)
+#define IMAGE_HEADER_WORD6_PHYSICAL_WIDTH_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_SHIFT	(0)
+#define IMAGE_HEADER_WORD7_PHYSICAL_HEIGHT_CLRMSK	(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD8_TWIDDLING_SHIFT			(0)
+#define IMAGE_HEADER_WORD8_TWIDDLING_CLRMSK			(0x000000FFU)
+#define IMAGE_HEADER_WORD8_TWIDDLING_STRIDED		(0 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+#define IMAGE_HEADER_WORD8_TWIDDLING_NTWIDDLE		(9 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+#define IMAGE_HEADER_WORD8_TWIDDLING_ZTWIDDLE		(12 << IMAGE_HEADER_WORD8_TWIDDLING_SHIFT)
+
+
+#define IMAGE_HEADER_WORD8_STRIDE_SHIFT				(8)
+#define IMAGE_HEADER_WORD8_STRIDE_CLRMSK			(0x0000FF00U)
+#define IMAGE_HEADER_WORD8_STRIDE_POSITIVE			(0 << IMAGE_HEADER_WORD8_STRIDE_SHIFT)
+#define IMAGE_HEADER_WORD8_STRIDE_NEGATIVE			(1 << IMAGE_HEADER_WORD8_STRIDE_SHIFT)
+
+#define IMAGE_HEADER_WORD8_BIFTYPE_SHIFT			(16)
+#define IMAGE_HEADER_WORD8_BIFTYPE_CLRMSK			(0x00FF0000U)
+#define IMAGE_HEADER_WORD8_BIFTYPE_NONE				(0 << IMAGE_HEADER_WORD8_BIFTYPE_SHIFT)
+
+#define IMAGE_HEADER_WORD8_FBCTYPE_SHIFT			(24)
+#define IMAGE_HEADER_WORD8_FBCTYPE_CLRMSK			(0xFF000000U)
+#define IMAGE_HEADER_WORD8_FBCTYPE_8X8				(1 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+#define IMAGE_HEADER_WORD8_FBCTYPE_16x4				(2 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+#define IMAGE_HEADER_WORD8_FBCTYPE_32x2				(3 << IMAGE_HEADER_WORD8_FBCTYPE_SHIFT)
+
+#define IMAGE_HEADER_WORD9_FBCDECOR_SHIFT			(0)
+#define IMAGE_HEADER_WORD9_FBCDECOR_CLRMSK			(0x000000FFU)
+#define IMAGE_HEADER_WORD9_FBCDECOR_ENABLE			(1 << IMAGE_HEADER_WORD9_FBCDECOR_SHIFT)
+
+/* Align with fbcomp_export_c.h in pdump_tools branch */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT			(8)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_CLRMSK			(0x0000FF00U)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_SAME_AS_GPU	(0 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_BASE			(1 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_TWIDDLED_EN	(2 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* TWIDDLED_ENHANCED */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V2				(3 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT1   (4 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_0_LAYOUT2   (5 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* V30_WITH_HEADER_REMAP */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT1   (6 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V3_1_LAYOUT2   (7 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT) /* V31_WITH_HEADER_REMAP */
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V4				(8 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_V4_PLUS		(9 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+#define IMAGE_HEADER_WORD9_FBCCOMPAT_TFBC			(10 << IMAGE_HEADER_WORD9_FBCCOMPAT_SHIFT)
+
+#define IMAGE_HEADER_WORD9_LOSSY_SHIFT				(16)
+#define IMAGE_HEADER_WORD9_LOSSY_CLRMSK				(0x00FF0000U)
+/* Non-TFBC */
+#define IMAGE_HEADER_WORD9_LOSSY_ON					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+
+/* TFBC */
+#define IMAGE_HEADER_WORD9_LOSSY_75					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_37					(1 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_50					(2 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_25					(3 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+#define IMAGE_HEADER_WORD9_LOSSY_OFF				(0 << IMAGE_HEADER_WORD9_LOSSY_SHIFT)
+
+#define IMAGE_HEADER_WORD9_SWIZZLE_SHIFT			(24)
+#define IMAGE_HEADER_WORD9_SWIZZLE_CLRMSK			(0xFF000000U)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ARGB		(0x0 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ARBG		(0x1 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_AGRB		(0x2 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_AGBR		(0x3 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ABGR		(0x4 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_ABRG		(0x5 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_RGBA		(0x8 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_RBGA		(0x9 << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_GRBA		(0xA << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_GBRA		(0xB << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_BGRA		(0xC << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+#define IMAGE_HEADER_WORD9_SWIZZLE_MODE_BRGA		(0xD << IMAGE_HEADER_WORD9_SWIZZLE_SHIFT)
+
+#define IMAGE_HEADER_WORD10_FBCCLEAR_CH0_SHIFT		(0)
+#define IMAGE_HEADER_WORD10_FBCCLEAR_CH0_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD11_FBCCLEAR_CH1_SHIFT		(0)
+#define IMAGE_HEADER_WORD11_FBCCLEAR_CH1_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD12_FBCCLEAR_CH2_SHIFT		(0)
+#define IMAGE_HEADER_WORD12_FBCCLEAR_CH2_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD13_FBCCLEAR_CH3_SHIFT		(0)
+#define IMAGE_HEADER_WORD13_FBCCLEAR_CH3_CLRMSK		(0xFFFFFFFFU)
+
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT		(0)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_CLRMSK		(0x000000FFU)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_25_50_75		(0 << IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT)
+#define IMAGE_HEADER_WORD14_TFBC_GROUP_25_37_50		(1 << IMAGE_HEADER_WORD14_TFBC_GROUP_SHIFT)
+
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT		(8)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_CLRMSK		(0x0000FF00U)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_ALL			(0 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_CORR	(1 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_D_STD_ONLY	(2 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+#define IMAGE_HEADER_WORD14_COMP_SCHEME_PTC_ONLY	(3 << IMAGE_HEADER_WORD14_COMP_SCHEME_SHIFT)
+
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_SHIFT	(16)
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_CLRMSK	(0x00FF0000U)
+#define IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_EN		(1 << IMAGE_HEADER_WORD14_YUV10_OPTIMAL_FMT_8_SHIFT) /* Treat YUV10 optimal formats as 8 bits */
+
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_SHIFT	(24)
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_CLRMSK	(0xFF000000U)
+#define IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_EN		(1 << IMAGE_HEADER_WORD14_LOSSY_MIN_CHANNEL_SHIFT) /* Override lossy min channel setting */
+
+/* IMAGE_HEADER_WORD15_RESERVED2 */
+
+/*
+ * The data type descriptor
+ */
+
+/*
+ * Header type (IMGCv1) - 'IMGC' in hex + VERSION 0
+ * Header size - 20 bytes (5 x 32 bit WORDS)
+ */
+#define DATA_HEADER_TYPE                    (0x43474D49)
+#define DATA_HEADER_SIZE                    (20)
+#define DATA_HEADER_VERSION                 (0)
+
+/*
+ * The IBIN type descriptor
+ */
+
+/*
+ * Header type (IBIN) - 'IBIN' in hex + VERSION 0
+ * Header size - 12 bytes (3 x 32 bit WORDS)
+ */
+#define IBIN_HEADER_TYPE                    (0x4e494249)
+#define IBIN_HEADER_SIZE                    (12)
+#define IBIN_HEADER_VERSION                 (0)
+
+/*
+ * Data type-specific fields
+ */
+#define DATA_HEADER_WORD3_ELEMENT_TYPE_SHIFT        (0)
+#define DATA_HEADER_WORD3_ELEMENT_TYPE_CLRMSK       (0xFFFFFFFFU)
+
+#define DATA_HEADER_WORD4_ELEMENT_COUNT_SHIFT       (0)
+#define DATA_HEADER_WORD4_ELEMENT_COUNT_CLRMSK      (0xFFFFFFFFU)
+
+#endif /* PDUMPDESC_H */
diff --git a/drivers/gpu/drm/img-rogue/physheap.c b/drivers/gpu/drm/img-rogue/physheap.c
new file mode 100644
index 000000000000..999f469bfcf3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physheap.c
@@ -0,0 +1,2203 @@
+/*************************************************************************/ /*!
+@File           physheap.c
+@Title          Physical heap management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Management functions for the physical heap(s). A heap contains
+                all the information required by services when using memory from
+                that heap (such as CPU <> Device physical address translation).
+                A system must register one heap but can have more then one which
+                is why a heap must register with a (system) unique ID.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "physheap.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "physmem.h"
+#include "physmem_hostmem.h"
+#include "physmem_lma.h"
+#include "physmem_dlm.h"
+#include "physmem_ima.h"
+#include "physmem_osmem.h"
+#include "debug_common.h"
+
+struct _PHYS_HEAP_
+{
+	/*! The type of this heap */
+	PHYS_HEAP_TYPE			eType;
+
+	/*! The allocation policy for this heap */
+	PHYS_HEAP_POLICY uiPolicy;
+
+	/* Config flags */
+	PHYS_HEAP_USAGE_FLAGS		ui32UsageFlags;
+
+	/* OOM Detection state */
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	ATOMIC_T					sOOMDetected;
+#endif
+
+	/*! Pointer to device node struct */
+	PPVRSRV_DEVICE_NODE         psDevNode;
+	/*! PDump name of this physical memory heap */
+	IMG_CHAR					*pszPDumpMemspaceName;
+	/*! Physheap name of this physical memory heap */
+	IMG_CHAR					aszName[PHYS_HEAP_NAME_SIZE];
+	/*! Private data for the translate routines */
+	IMG_HANDLE					hPrivData;
+	/*! Function callbacks */
+	PHYS_HEAP_FUNCTIONS			*psMemFuncs;
+
+	/*! Refcount */
+	IMG_UINT32					ui32RefCount;
+
+	/*! Implementation specific */
+	PHEAP_IMPL_DATA             pvImplData;
+	PHEAP_IMPL_FUNCS            *psImplFuncs;
+
+	/*! Pointer to next physical heap */
+	struct _PHYS_HEAP_		*psNext;
+
+#if defined(SUPPORT_STATIC_IPA)
+	/*! IPA Policy value from Heap Config */
+	IMG_UINT32                  ui32IPAPolicyValue;
+
+	/*! IPA Clear Mask value from Heap Config */
+	IMG_UINT32                  ui32IPAClearMask;
+
+	/*! IPA Bit Shift value from Heap Config */
+	IMG_UINT32                  ui32IPAShift;
+#endif /* defined(SUPPORT_STATIC_IPA) */
+};
+
+#if defined(REFCOUNT_DEBUG)
+#define PHYSHEAP_REFCOUNT_PRINT(fmt, ...)	\
+	PVRSRVDebugPrintf(PVR_DBG_WARNING,	\
+			  __FILE__,		\
+			  __LINE__,		\
+			  fmt,			\
+			  __VA_ARGS__)
+#else
+#define PHYSHEAP_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+#define IsOOMError(err) ((err == PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES) | \
+						 (err == PVRSRV_ERROR_OUT_OF_MEMORY) | \
+						 (err == PVRSRV_ERROR_PMR_TOO_LARGE))
+
+typedef enum _PVR_LAYER_HEAP_ACTION_
+{
+	PVR_LAYER_HEAP_ACTION_IGNORE,          /* skip heap during heap init */
+	PVR_LAYER_HEAP_ACTION_INSTANTIATE,     /* instantiate heap but don't acquire */
+	PVR_LAYER_HEAP_ACTION_INITIALISE       /* instantiate and acquire */
+
+} PVR_LAYER_HEAP_ACTION;
+
+typedef struct PHYS_HEAP_PROPERTIES_TAG
+{
+	PVRSRV_PHYS_HEAP eFallbackHeap;
+	PVR_LAYER_HEAP_ACTION ePVRLayerAction;
+	IMG_BOOL bUserModeAlloc;
+} PHYS_HEAP_PROPERTIES;
+
+/* NOTE: Table entries and order must match enum PVRSRV_PHYS_HEAP to ensure
+ * correct operation of PhysHeapCreatePMR().
+ */
+static PHYS_HEAP_PROPERTIES gasHeapProperties[PVRSRV_PHYS_HEAP_LAST] =
+{
+	/* eFallbackHeap,               ePVRLayerAction,                   bUserModeAlloc */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* DEFAULT */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* CPU_LOCAL */
+    {  PVRSRV_PHYS_HEAP_DEFAULT,    PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_LOCAL */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_PRIVATE */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_MAIN */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_FALSE }, /* EXTERNAL */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_FALSE }, /* GPU_COHERENT */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_INITIALISE,  IMG_TRUE  }, /* GPU_SECURE */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_CONFIG */
+    {  PVRSRV_PHYS_HEAP_FW_MAIN,    PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_CODE */
+    {  PVRSRV_PHYS_HEAP_FW_MAIN,    PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PRIV_DATA */
+    {  PVRSRV_PHYS_HEAP_GPU_LOCAL,  PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP_PT */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP0, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP0 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP1, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP1 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP2, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP2 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP3, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP3 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP4, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP4 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP5, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP5 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP6, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP6 */
+    {  PVRSRV_PHYS_HEAP_FW_PREMAP7, PVR_LAYER_HEAP_ACTION_IGNORE,      IMG_FALSE }, /* FW_PREMAP7 */
+    {  PVRSRV_PHYS_HEAP_WRAP,       PVR_LAYER_HEAP_ACTION_INSTANTIATE, IMG_FALSE }, /* WRAP */
+    {  PVRSRV_PHYS_HEAP_DISPLAY,    PVR_LAYER_HEAP_ACTION_INSTANTIATE, IMG_FALSE }, /* DISPLAY */
+};
+
+static_assert((ARRAY_SIZE(gasHeapProperties) == PVRSRV_PHYS_HEAP_LAST),
+	"Size or order of gasHeapProperties entries incorrect for PVRSRV_PHYS_HEAP enum");
+
+static IMG_BOOL PhysHeapCreatedByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+static IMG_BOOL PhysHeapAcquiredByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+
+/**
+ * ! IMPORTANT !
+ * Do not change this string array unless the usage flag definitions in
+ * physheap_config.h have changed.
+ *
+ * NOTE: Use DebugCommonFlagStrings or GetPhysHeapUsageString to get
+ * usage flags string.
+ */
+static const IMG_FLAGS2DESC g_asPhysHeapUsageFlagStrings[] =
+{
+	{PHYS_HEAP_USAGE_CPU_LOCAL,		"CPU_LOCAL"},
+	{PHYS_HEAP_USAGE_GPU_LOCAL,		"GPU_LOCAL"},
+	{PHYS_HEAP_USAGE_GPU_PRIVATE,	"GPU_PRIVATE"},
+	{PHYS_HEAP_USAGE_EXTERNAL,		"EXTERNAL"},
+	{PHYS_HEAP_USAGE_GPU_COHERENT,	"GPU_COHERENT"},
+	{PHYS_HEAP_USAGE_GPU_SECURE,	"GPU_SECURE"},
+	{PHYS_HEAP_USAGE_FW_SHARED,		"FW_SHARED"},
+	{PHYS_HEAP_USAGE_FW_PRIVATE,	"FW_PRIVATE"},
+	{PHYS_HEAP_USAGE_FW_CODE,		"FW_CODE"},
+	{PHYS_HEAP_USAGE_FW_PRIV_DATA,	"FW_PRIV_DATA"},
+	{PHYS_HEAP_USAGE_FW_PREMAP_PT,	"FW_PREMAP_PT"},
+	{PHYS_HEAP_USAGE_FW_PREMAP,		"FW_PREMAP"},
+	{PHYS_HEAP_USAGE_WRAP,			"WRAP"},
+	{PHYS_HEAP_USAGE_DISPLAY,		"DISPLAY"},
+	{PHYS_HEAP_USAGE_DLM,			"DLM"}
+};
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCheckValidUsageFlags
+@Description    Checks if any bits were set outside of the valid ones within
+                PHYS_HEAP_USAGE_FLAGS.
+
+@Input          ui32PhysHeapUsage  The value of the usage flag.
+
+@Return         True or False depending on whether there were only valid bits set.
+*/ /**************************************************************************/
+static inline IMG_BOOL PhysHeapCheckValidUsageFlags(PHYS_HEAP_USAGE_FLAGS ui32PhysHeapUsage)
+{
+	return !(ui32PhysHeapUsage & ~PHYS_HEAP_USAGE_MASK);
+}
+
+/*************************************************************************/ /*!
+@Function       GetPhysHeapUsageString
+@Description    This function is used to create a comma separated string of all
+                usage flags passed in as a bitfield.
+
+@Input          ui32UsageFlags     The bitfield of usage flags.
+@Input          ui32Size           The size of the memory pointed to by
+                                   pszUsageString.
+@Output         pszUsageString     A pointer to memory where the created string
+                                   will be stored.
+
+@Return         If successful PVRSRV_OK, else a PVRSRV_ERROR.
+*/ /**************************************************************************/
+static PVRSRV_ERROR GetPhysHeapUsageString(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags,
+                                          IMG_UINT32 ui32Size,
+                                          IMG_CHAR *const pszUsageString)
+{
+	IMG_UINT32 i;
+	IMG_BOOL bFirst = IMG_TRUE;
+	size_t uiSize = 0;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszUsageString != NULL, "pszUsageString");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui32Size > 0, "ui32Size");
+
+	/* Initialise the string to be null terminated at the beginning */
+	pszUsageString[0] = '\0';
+
+	if (ui32UsageFlags == 0)
+	{
+		const ssize_t iCopiedCnt = OSStringSafeCopy(pszUsageString, "NONE", (size_t)ui32Size);
+		PVR_LOG_RETURN_IF_FALSE((iCopiedCnt >= 0), "OSStringSafeCopy", PVRSRV_ERROR_OUT_OF_MEMORY);
+
+		return PVRSRV_OK;
+	}
+
+	/* Process from left to right. */
+	for (i = (sizeof(PHYS_HEAP_USAGE_FLAGS) * BITS_PER_BYTE - 1); i > 0; i--)
+	{
+		IMG_UINT32 ui32Flag = BIT(i);
+
+		if (BITMASK_HAS(ui32UsageFlags, ui32Flag))
+		{
+			IMG_CHAR pszString[32] = "\0";
+
+			if (PhysHeapCheckValidUsageFlags(ui32Flag))
+			{
+				DebugCommonFlagStrings(pszString,
+				                   sizeof(pszString),
+				                   g_asPhysHeapUsageFlagStrings,
+				                   ARRAY_SIZE(g_asPhysHeapUsageFlagStrings),
+				                   ui32Flag);
+			}
+			else
+			{
+				uiSize = OSStringLCat(pszString,
+				                      "INVALID",
+				                      sizeof(pszString));
+				PVR_LOG_RETURN_IF_FALSE((uiSize < sizeof(pszString)), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+			}
+
+			if (!bFirst)
+			{
+				uiSize = OSStringLCat(pszUsageString,
+				                      ", ",
+				                      (size_t)ui32Size);
+				PVR_LOG_RETURN_IF_FALSE((uiSize < ui32Size), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+			}
+			else
+			{
+				bFirst = IMG_FALSE;
+			}
+
+			uiSize = OSStringLCat(pszUsageString,
+			                      pszString,
+			                      (size_t)ui32Size);
+			PVR_LOG_RETURN_IF_FALSE((uiSize < ui32Size), "OSStringLCat", PVRSRV_ERROR_OUT_OF_MEMORY);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapPrintHeapProperties
+@Description    This function is used to print properties
+                of the specified physheap.
+
+@Input          psPhysHeap          The physheap to create the string from.
+@Input          pfnDumpDebugPrintf  The specified print function that should be
+                                    used to dump any debug information
+                                    (see PVRSRVDebugRequest).
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+
+@Return         If successful PVRSRV_OK, else a PVRSRV_ERROR.
+*/ /**************************************************************************/
+static PVRSRV_ERROR PhysHeapPrintHeapProperties(PHYS_HEAP *psPhysHeap,
+                                                IMG_BOOL bDefaultHeap,
+                                                DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                                void *pvDumpDebugFile)
+{
+	static const IMG_CHAR *const pszTypeStrings[] = {
+	#define X(_name) #_name,
+		PHYS_HEAP_TYPE_LIST
+	#undef X
+	};
+
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	IMG_CHAR pszUsageString[127] = "\0";
+	PVRSRV_ERROR eError;
+
+	if (psPhysHeap->eType >= ARRAY_SIZE(pszTypeStrings))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PhysHeap at address %p eType is not a PHYS_HEAP_TYPE",
+		         psPhysHeap));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_HEAPINFO, failure);
+	}
+
+	psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+	                                               &ui64TotalSize,
+	                                               &ui64FreeSize);
+
+	eError = GetPhysHeapUsageString(psPhysHeap->ui32UsageFlags,
+	                                sizeof(pszUsageString),
+	                                pszUsageString);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GetPhysHeapUsageString", failure);
+
+	if ((psPhysHeap->eType == PHYS_HEAP_TYPE_LMA) ||
+	    (psPhysHeap->eType == PHYS_HEAP_TYPE_DLM))
+	{
+		IMG_CPU_PHYADDR sCPUPAddr;
+		IMG_DEV_PHYADDR sGPUPAddr;
+
+		PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetCPUPAddr != NULL);
+		PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetDevPAddr != NULL);
+
+		eError = psPhysHeap->psImplFuncs->pfnGetCPUPAddr(psPhysHeap->pvImplData,
+		                                                 &sCPUPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "pfnGetCPUPAddr");
+			sCPUPAddr.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(IMG_UINT64_MAX);
+		}
+
+		eError = psPhysHeap->psImplFuncs->pfnGetDevPAddr(psPhysHeap->pvImplData,
+		                                                 &sGPUPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "pfnGetDevPAddr");
+			sGPUPAddr.uiAddr = IMG_UINT64_MAX;
+		}
+
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "CPU PA Base: " CPUPHYADDR_UINT_FMTSPEC", "
+		                  "GPU PA Base: 0x%08"IMG_UINT64_FMTSPECx", "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B",
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  CPUPHYADDR_FMTARG(sCPUPAddr.uiAddr),
+		                  sGPUPAddr.uiAddr,
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize);
+	}
+	else if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+	{
+		IMG_CHAR pszSpanString[128] = "\0";
+		void *pvIterHandle = NULL;
+
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B Spans:",
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize);
+
+		while (psPhysHeap->psImplFuncs->pfnGetHeapSpansStringIter(psPhysHeap->pvImplData,
+		                                                                   pszSpanString,
+		                                                                   sizeof(pszSpanString),
+		                                                                   &pvIterHandle))
+		{
+			PVR_DUMPDEBUG_LOG("%s", pszSpanString);
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("0x%p -> PdMs: %s, Type: %s, %s, "
+		                  "Usage Flags: 0x%08x (%s), Refs: %d, "
+		                  "Free Size: %"IMG_UINT64_FMTSPEC"B, "
+		                  "Total Size: %"IMG_UINT64_FMTSPEC"B",
+		                  psPhysHeap,
+		                  psPhysHeap->pszPDumpMemspaceName,
+		                  pszTypeStrings[psPhysHeap->eType],
+		                  bDefaultHeap ? "default" : "-",
+		                  psPhysHeap->ui32UsageFlags,
+		                  pszUsageString,
+		                  psPhysHeap->ui32RefCount,
+		                  ui64FreeSize,
+		                  ui64TotalSize);
+	}
+
+	return PVRSRV_OK;
+
+failure:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDebugRequest
+@Description    This function is used to output debug information for a given
+                device's PhysHeaps.
+@Input          pfnDbgRequestHandle Data required by this function that is
+                                    passed through the RegisterDeviceDbgRequestNotify
+                                    function.
+@Input          ui32VerbLevel       The maximum verbosity of the debug request.
+@Input          pfnDumpDebugPrintf  The specified print function that should be
+                                    used to dump any debug information
+                                    (see PVRSRVDebugRequest).
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+@Return         void
+*/ /**************************************************************************/
+static void PhysHeapDebugRequest(PVRSRV_DBGREQ_HANDLE pfnDbgRequestHandle,
+                                  IMG_UINT32 ui32VerbLevel,
+                                  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                  void *pvDumpDebugFile)
+{
+	PPVRSRV_DEVICE_NODE psDeviceNode = (PPVRSRV_DEVICE_NODE)pfnDbgRequestHandle;
+	PHYS_HEAP *psPhysHeap;
+
+	PVR_UNREFERENCED_PARAMETER(ui32VerbLevel);
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDeviceNode != NULL,
+	                             "Phys Heap debug request failed. psDeviceNode was NULL");
+
+	PVR_DUMPDEBUG_LOG("------[ Device ID: %d - Phys Heaps ]------",
+	                  psDeviceNode->sDevId.i32KernelDeviceID);
+
+	for (psPhysHeap = psDeviceNode->psPhysHeapList; psPhysHeap != NULL; psPhysHeap = psPhysHeap->psNext)
+	{
+		PVRSRV_ERROR eError = PVRSRV_OK;
+		IMG_BOOL bDefaultHeap = psPhysHeap == psDeviceNode->apsPhysHeap[psDeviceNode->psDevConfig->eDefaultHeap];
+
+		eError = PhysHeapPrintHeapProperties(psPhysHeap,
+		                                     bDefaultHeap,
+		                                     pfnDumpDebugPrintf,
+		                                     pvDumpDebugFile);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "PhysHeapCreateProperties");
+			continue;
+		}
+	}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	OSLockAcquire(psDeviceNode->hPMRZombieListLock);
+	PVR_DUMPDEBUG_LOG("PMR Zombie Count: %u, PMR Zombie Count In Cleanup: %u",
+	                  psDeviceNode->uiPMRZombieCount,
+	                  psDeviceNode->uiPMRZombieCountInCleanup);
+	OSLockRelease(psDeviceNode->hPMRZombieListLock);
+#endif
+	PVR_DUMPDEBUG_LOG("PMR Live Count: %d", PMRGetLiveCount());
+}
+
+/*************************************************************************/ /*!
+@Function       HeapCfgUsedByPVRLayer
+@Description    Checks if a physheap config must be handled by the PVR Layer
+@Input          psConfig PhysHeapConfig
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+static IMG_BOOL HeapCfgUsedByPVRLayer(PHYS_HEAP_CONFIG *psConfig)
+{
+	PVRSRV_PHYS_HEAP eHeap;
+	IMG_BOOL bPVRHeap = IMG_FALSE;
+
+	/* Heaps are triaged for initialisation by either
+	 * the PVR Layer or the device-specific heap handler. */
+	for (eHeap = PVRSRV_PHYS_HEAP_DEFAULT;
+		 eHeap < PVRSRV_PHYS_HEAP_LAST;
+		 eHeap++)
+	{
+		if ((BIT_ISSET(psConfig->ui32UsageFlags, eHeap) &&
+		     PhysHeapCreatedByPVRLayer(eHeap)))
+		{
+			bPVRHeap = IMG_TRUE;
+			break;
+		}
+	}
+
+	return bPVRHeap;
+}
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateDLMIMAHeapsFromConfig
+@Description    Create new heaps for a device from DLM and IMA configs.
+                This function will both create and construct the link
+                between them.
+@Input          psDevNode      Pointer to device node struct
+@Input          pasConfigs     Pointer to array of Heap configurations.
+@Input          ui32NumConfigs Number of configurations in array.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PhysHeapCreateDLMIMAHeapsFromConfig(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PHYS_HEAP_CONFIG *pasConfigs,
+                                    IMG_UINT32 ui32NumConfigs)
+{
+	/* The DLM heaps must be created before IMA heaps and the config order is not well-defined.
+	 * So for each DLM heap create it and then create the IMA heaps associated. */
+
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiDLMIdx, uiIMAIdx;
+	PHYS_HEAP_POLICY uiIMAPolicy = OSIsMapPhysNonContigSupported() ? PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG : PHYS_HEAP_POLICY_DEFAULT;
+	PHYS_HEAP *psDLMHeap;
+
+	/* Iterate and create the DLM heaps */
+	for (uiDLMIdx = 0; uiDLMIdx < ui32NumConfigs; uiDLMIdx++)
+	{
+		if (pasConfigs[uiDLMIdx].eType == PHYS_HEAP_TYPE_DLM)
+		{
+			IMG_UINT32 uiHeapCount = 0;
+			eError = PhysmemCreateHeapDLM(psDevNode,
+			                               PHYS_HEAP_POLICY_DEFAULT,
+			                               &pasConfigs[uiDLMIdx],
+			                               pasConfigs[uiDLMIdx].uConfig.sDLM.pszHeapName,
+			                               &psDLMHeap);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeapDLM");
+
+			/* Then iterate and create the IMA heaps linked to this DLM heap */
+			for (uiIMAIdx = 0; uiIMAIdx < ui32NumConfigs; uiIMAIdx++)
+			{
+				if (pasConfigs[uiIMAIdx].eType == PHYS_HEAP_TYPE_IMA &&
+				    pasConfigs[uiIMAIdx].uConfig.sIMA.uiDLMHeapIdx == uiDLMIdx)
+				{
+					PhysmemCreateHeapIMA(psDevNode,
+					                     uiIMAPolicy,
+					                     &pasConfigs[uiIMAIdx],
+					                     pasConfigs[uiIMAIdx].uConfig.sIMA.pszHeapName,
+					                     psDLMHeap,
+					                     pasConfigs[uiDLMIdx].uConfig.sDLM.ui32Log2PMBSize,
+					                     NULL);
+					PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeapIMA");
+					uiHeapCount++;
+				}
+			}
+
+			if (uiHeapCount == 0)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "DLM phys heap config %d: No connected IMA heaps. Phys heap will go unused.", uiDLMIdx));
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+PhysHeapDestroyDLMIMAHeaps(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PHYS_HEAP *psNode = psDevNode->psPhysHeapList;
+
+	/* We need to ensure IMA heaps are destroyed before DLM so that
+	 * we don't cause RA leaks by freeing the DLM first.
+	 */
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+
+		if (psTmp->eType == PHYS_HEAP_TYPE_IMA)
+		{
+			PhysHeapDestroy(psTmp);
+		}
+	}
+
+	/* Reset the loop */
+	psNode = psDevNode->psPhysHeapList;
+
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+
+		if (psTmp->eType == PHYS_HEAP_TYPE_DLM)
+		{
+			PhysHeapDestroy(psTmp);
+		}
+	}
+}
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateDeviceHeapsFromConfigs
+@Description    Create new heaps for a device from configs.
+@Input          psDevNode      Pointer to device node struct
+@Input          pasConfigs     Pointer to array of Heap configurations.
+@Input          ui32NumConfigs Number of configurations in array.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PhysHeapCreateDeviceHeapsFromConfigs(PPVRSRV_DEVICE_NODE psDevNode,
+                                     PHYS_HEAP_CONFIG *pasConfigs,
+                                     IMG_UINT32 ui32NumConfigs)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	psDevNode->psPhysHeapList = NULL;
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+	/* DLM/IMA heaps must be constructed in a specific order */
+	eError = PhysHeapCreateDLMIMAHeapsFromConfig(psDevNode,
+	                                             pasConfigs,
+	                                             ui32NumConfigs);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateDLMIMAHeapsFromConfig");
+#endif
+
+
+	for (i = 0; i < ui32NumConfigs; i++)
+	{
+		/* A PhysHeapConfig can have multiple usage flags. If any flag in a
+		 * heap's set points to a heap type that is handled by the PVR Layer
+		 * then we assume that a single heap is shared between multiple
+		 * allocators and it is safe to instantiate it here. If the heap
+		 * is not marked to be initialised by the PVR Layer, leave it
+		 * to the device specific handler.
+		 * DLM Heaps have usage flags but don't have element in the
+		 * fallback table, this check will prevent instantiate them twice.
+		 */
+		if (HeapCfgUsedByPVRLayer(&pasConfigs[i]))
+		{
+			eError = PhysHeapCreateHeapFromConfig(psDevNode, &pasConfigs[i], NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+		}
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST)
+	/* For a temporary device node there will never be a debug dump
+	 * request targeting it */
+	if (psDevNode->hDebugTable != NULL)
+#endif
+	{
+		eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hPhysHeapDbgReqNotify,
+		                                              psDevNode,
+		                                              PhysHeapDebugRequest,
+		                                              DEBUG_REQUEST_SYS,
+		                                              psDevNode);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify");
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PhysHeapCreateHeapFromConfig(PVRSRV_DEVICE_NODE *psDevNode,
+							 PHYS_HEAP_CONFIG *psConfig,
+							 PHYS_HEAP **ppsPhysHeap)
+{
+	PVRSRV_ERROR eResult;
+
+	if (psConfig->eType == PHYS_HEAP_TYPE_UMA
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+		|| psConfig->eType == PHYS_HEAP_TYPE_WRAP
+#endif
+		)
+	{
+		eResult = PhysmemCreateHeapOSMEM(psDevNode,
+		                                 PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG,
+		                                 psConfig,
+		                                 ppsPhysHeap);
+	}
+	else if ((psConfig->eType == PHYS_HEAP_TYPE_LMA) ||
+	         (psConfig->eType == PHYS_HEAP_TYPE_DMA))
+	{
+		PHYS_HEAP_POLICY uiHeapPolicy;
+
+		if (psDevNode->pfnPhysHeapGetLMAPolicy != NULL)
+		{
+			uiHeapPolicy = psDevNode->pfnPhysHeapGetLMAPolicy(psConfig->ui32UsageFlags, psDevNode);
+		}
+		else
+		{
+			uiHeapPolicy = OSIsMapPhysNonContigSupported() ?
+			               PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG :
+			               PHYS_HEAP_POLICY_DEFAULT;
+		}
+
+		eResult = PhysmemCreateHeapLMA(psDevNode,
+		                               uiHeapPolicy,
+		                               psConfig,
+		                               (psConfig->eType == PHYS_HEAP_TYPE_LMA) ?
+		                                "GPU LMA (Sys)" :
+		                                "GPU LMA DMA (Sys)",
+		                               ppsPhysHeap);
+	}
+	else if ((psConfig->eType == PHYS_HEAP_TYPE_DLM) ||
+	         (psConfig->eType == PHYS_HEAP_TYPE_IMA))
+	{
+		/* These heaps have already been instantiated in
+		 * PhysHeapCreateDLMIMAHeapsFromConfig
+		 */
+		eResult = PVRSRV_OK;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s Invalid phys heap type: %d",
+				 __func__, psConfig->eType));
+		eResult = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return eResult;
+}
+
+#define PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE (IMG_UINT64_C(0x100000) * IMG_UINT64_C(32)) /* 32MB */
+
+static PVRSRV_ERROR PVRSRVValidatePhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	IMG_UINT32 ui32FlagsAccumulate = 0;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_FALSE(psDevConfig->ui32PhysHeapCount > 0,
+							"Device config must specify at least one phys heap config.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		/* Flags that may be used by multiple heaps. */
+		const PHYS_HEAP_USAGE_FLAGS uiDuplicateFlags = PHYS_HEAP_USAGE_DLM;
+		PHYS_HEAP_CONFIG *psHeapConf = &psDevConfig->pasPhysHeaps[i];
+
+		PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->ui32UsageFlags != 0,
+		                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+		                           "Phys heap config %d: must specify usage flags.", i);
+
+		PVR_LOG_RETURN_IF_FALSE_VA(((ui32FlagsAccumulate & ~uiDuplicateFlags) & psHeapConf->ui32UsageFlags) == 0,
+		                             PVRSRV_ERROR_PHYSHEAP_CONFIG,
+		                             "Phys heap config %d: duplicate usage flags.", i);
+
+		ui32FlagsAccumulate |= psHeapConf->ui32UsageFlags;
+
+		if (BITMASK_ANY((1U << psDevConfig->eDefaultHeap), PHYS_HEAP_USAGE_MASK) &&
+		    BITMASK_ANY((1U << psDevConfig->eDefaultHeap), psHeapConf->ui32UsageFlags))
+		{
+			switch (psHeapConf->eType)
+			{
+#if defined(__KERNEL__)
+				case PHYS_HEAP_TYPE_DMA:
+#endif
+				case PHYS_HEAP_TYPE_LMA:
+				{
+					if (psHeapConf->uConfig.sLMA.uiSize < PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE)
+					{
+						/* Output message if default heap is LMA and smaller than recommended minimum. */
+						PVR_DPF((PVR_DBG_ERROR, "%s: Size of default heap is 0x%" IMG_UINT64_FMTSPECX
+						         " (recommended minimum heap size is 0x%" IMG_UINT64_FMTSPECX ")",
+						         __func__, psHeapConf->uConfig.sLMA.uiSize,
+						         PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE));
+					}
+					break;
+				}
+				case PHYS_HEAP_TYPE_IMA:
+				{
+					/* Check if chained DLM has more than default LMA size.
+					 * This is not perfect as other connected IMAs could reserve all the memory. */
+					IMG_UINT64 uiSize = psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].uConfig.sDLM.uiSize;
+
+					if (uiSize < PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Default heap has access to 0x%" IMG_UINT64_FMTSPECX
+						         " (recommended minimum heap size is 0x%" IMG_UINT64_FMTSPECX ")",
+						         __func__, uiSize,
+						         PVRSRV_MIN_DEFAULT_LMA_PHYS_HEAP_SIZE));
+					}
+					break;
+				}
+				case PHYS_HEAP_TYPE_DLM:
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Phys heap config %d: "
+					         "A DLM heap cannot be the default heap.",
+					         __func__, i));
+					return PVRSRV_ERROR_PHYSHEAP_CONFIG;
+				}
+				default:
+					break;
+			}
+		}
+
+#if !defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+		if ((psHeapConf->eType == PHYS_HEAP_TYPE_IMA) || (psHeapConf->eType == PHYS_HEAP_TYPE_DLM))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Phys heap config %d: "
+			         "Dynamic phys heaps are not supported! "
+			         "Please enable PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS",
+			         __func__, i));
+			return PVRSRV_ERROR_PHYSHEAP_CONFIG;
+		}
+#else /* PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS */
+		if (psHeapConf->eType == PHYS_HEAP_TYPE_DLM)
+		{
+			PVR_LOG_RETURN_IF_FALSE_VA((psHeapConf->ui32UsageFlags & PHYS_HEAP_USAGE_DLM) == PHYS_HEAP_USAGE_DLM,
+			                            PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                            "Phys heap config %d: DLM heap must specify the DLM usage flag.", i);
+
+			/* If a DLM heap has less space than a PMB, it cannot export a single PMB. The PMB size should be lowered. */
+			PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->uConfig.sDLM.uiSize >= psHeapConf->uConfig.sDLM.ui32Log2PMBSize,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: Size of DLM heap is 0x%" IMG_UINT64_FMTSPECX
+			                           " but the PMB size is 0x%" IMG_UINT64_FMTSPECX
+			                           ". The total size must be greater than or equal to the PMB size.",
+			                           i, psHeapConf->uConfig.sDLM.uiSize,
+			                           IMG_UINT64_C(1) << psHeapConf->uConfig.sDLM.ui32Log2PMBSize);
+		}
+
+		/* IMA heaps must point to a DLM heap. */
+		if (psHeapConf->eType == PHYS_HEAP_TYPE_IMA)
+		{
+			PVR_LOG_RETURN_IF_FALSE_VA(psHeapConf->uConfig.sIMA.uiDLMHeapIdx < psDevConfig->ui32PhysHeapCount,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: IMA heap is trying to link to a DLM heap out of bounds. "
+			                           "Requested Heap Index: %d, Heap Array Count: %d",
+			                           i, psHeapConf->uConfig.sIMA.uiDLMHeapIdx, psDevConfig->ui32PhysHeapCount);
+			PVR_LOG_RETURN_IF_FALSE_VA(psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].eType == PHYS_HEAP_TYPE_DLM,
+			                           PVRSRV_ERROR_PHYSHEAP_CONFIG,
+			                           "Phys heap config %d: IMA heap is trying to link to a NON-DLM heap type. "
+			                           "Requested Heap Idx: %d, Heap Type: %d",
+			                           i, psHeapConf->uConfig.sIMA.uiDLMHeapIdx,
+			                           psDevConfig->pasPhysHeaps[psHeapConf->uConfig.sIMA.uiDLMHeapIdx].eType);
+		}
+#endif /* PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS */
+
+	}
+
+
+	if (psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_GPU_LOCAL)
+	{
+		PVR_LOG_RETURN_IF_FALSE(((ui32FlagsAccumulate & PHYS_HEAP_USAGE_GPU_LOCAL) != 0) ,
+							"Device config must specify GPU local phys heap config.",
+							PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+	else if (psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_CPU_LOCAL)
+	{
+		PVR_LOG_RETURN_IF_FALSE(((ui32FlagsAccumulate & PHYS_HEAP_USAGE_CPU_LOCAL) != 0) ,
+						"Device config must specify CPU local phys heap config.",
+						PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+static void DestroyGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       CreateGpuVirtValArenas
+@Description    Create virtualization validation arenas
+@Input          psDeviceNode The device node
+@Return         PVRSRV_ERROR PVRSRV_OK on success
+*/ /**************************************************************************/
+static PVRSRV_ERROR CreateGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* aui64OSidMin and aui64OSidMax are what we program into HW registers.
+	   The values are different from base/size of arenas. */
+	IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS];
+	IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS];
+	PHYS_HEAP_CONFIG *psGPULocalHeap = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig, PHYS_HEAP_USAGE_GPU_LOCAL);
+	IMG_DEV_PHYADDR sGPULocalCardBase = PhysHeapConfigGetCardBase(psGPULocalHeap);
+	IMG_UINT64 uiGPULocalSize = PhysHeapConfigGetSize(psGPULocalHeap);
+	PHYS_HEAP_CONFIG *psDisplayHeap = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig, PHYS_HEAP_USAGE_DISPLAY);
+
+	IMG_UINT64 uPrivateRABase;
+	IMG_UINT64 uPrivateRASize;
+	IMG_UINT64 uSharedRABase;
+	IMG_UINT64 uSharedRASize;
+	IMG_UINT64 uSharedRegionBase;
+	IMG_UINT64 uSharedRegionSize;
+	IMG_UINT32 i;
+
+	//PVR_LOG_RETURN_IF_FALSE((psDeviceNode->psOSSharedArena == NULL), "Validation RAs already created.", PVRSRV_OK);
+
+	/* Shared region is fixed size, the remaining space is divided amongst OSes */
+	uPrivateRASize = uiGPULocalSize - PVR_ALIGN(GPUVIRT_SIZEOF_SHARED, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+	uPrivateRASize /= GPUVIRT_VALIDATION_NUM_OS;
+	uPrivateRASize = uPrivateRASize & ~((IMG_UINT64)OSGetPageSize() - 1ULL); /* Align, round down */
+	uSharedRASize = uiGPULocalSize - uPrivateRASize * GPUVIRT_VALIDATION_NUM_OS;
+
+	PVR_LOG(("GPUVIRT_VALIDATION: GPU_LOCAL heap base (base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ")"
+			 " split into %u private regions of size 0x%" IMG_UINT64_FMTSPECX " and one shared region of size 0x%" IMG_UINT64_FMTSPECX " ",
+			 sGPULocalCardBase.uiAddr, uiGPULocalSize, GPUVIRT_VALIDATION_NUM_OS, uPrivateRASize, uSharedRASize));
+
+	/* If a display heap config exists, include the display heap in the non-secure regions */
+	if (psDisplayHeap)
+	{
+		IMG_DEV_PHYADDR sGPUDisplayCardBase = PhysHeapConfigGetCardBase(psDisplayHeap);
+		IMG_UINT64 uiGPUDisplaySize = PhysHeapConfigGetSize(psDisplayHeap);
+
+		PVR_LOG(("GPUVIRT_VALIDATION: DISPLAY heap (base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ") merged into shared region.",
+		         sGPUDisplayCardBase.uiAddr,
+		         uiGPUDisplaySize));
+
+		if (sGPUDisplayCardBase.uiAddr > sGPULocalCardBase.uiAddr)
+		{
+			/*
+			 * ---------------------------------------------------------------------------------------------------
+			 * |        .        .        .     GPU LOCAL HEAP       .        .        .        |  DISPLAY HEAP  |
+			 * ---------------------------------------------------------------------------------------------------
+			 *  \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \_______________/
+			 *   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7   SHARED      DISPLAY
+			 *                                                                            \____SHARED REGION____/
+			 *  OSID 0 has full access over everything
+			 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+			 */
+			if ((sGPULocalCardBase.uiAddr + uiGPULocalSize) != sGPUDisplayCardBase.uiAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: GPU Local heap and display heap must be adjacent in physical memory.", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			uPrivateRABase = sGPULocalCardBase.uiAddr;
+			uSharedRABase = sGPULocalCardBase.uiAddr + uiGPULocalSize - uSharedRASize;
+			uSharedRegionBase = uSharedRABase;
+		}
+		else
+		{
+			/*
+			 * ---------------------------------------------------------------------------------------------------
+			 * |  DISPLAY HEAP   |        .        .        .     GPU LOCAL HEAP       .        .        .       |
+			 * ---------------------------------------------------------------------------------------------------
+			 *  \_______________/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/
+			 *        DISPLAY      SHARED   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7
+			 *   \____SHARED REGION____/
+			 *
+			 *  OSID 0 has full access over everything
+			 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+			 */
+			if ((sGPUDisplayCardBase.uiAddr + uiGPUDisplaySize) != sGPULocalCardBase.uiAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: GPU Local heap and display heap must be adjacent in physical memory.", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			uSharedRABase = sGPULocalCardBase.uiAddr;
+			uPrivateRABase = sGPULocalCardBase.uiAddr + uSharedRASize;
+			uSharedRegionBase = sGPUDisplayCardBase.uiAddr;
+		}
+
+		uSharedRegionSize = uSharedRASize + uiGPUDisplaySize;
+	}
+	else
+	{
+		/*
+		 * ----------------------------------------------------------------------------------
+		 * |        .        .        .     GPU LOCAL HEAP       .        .        .        |
+		 * ----------------------------------------------------------------------------------
+		 *  \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/ \______/
+		 *   OSID 0   OSID 1   OSID 2   OSID 3   OSID 4   OSID 5   OSID 6   OSID 7   SHARED
+		 *
+		 *  OSID 0 has full access over everything
+		 *  Shared RA of the GPU Local heap is accessible to all OSIDs for special purpose allocations.
+		 */
+		uPrivateRABase = sGPULocalCardBase.uiAddr;
+		uSharedRABase = uPrivateRABase + uPrivateRASize * GPUVIRT_VALIDATION_NUM_OS;
+		uSharedRASize = uiGPULocalSize - (uSharedRABase - uPrivateRABase);
+		uSharedRegionSize = uSharedRASize;
+		uSharedRegionBase = uSharedRABase;
+	}
+
+	if (uPrivateRASize < GPUVIRT_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Per-OSID private memory regions are too small (current: 0x%" IMG_UINT64_FMTSPECX ""
+								" required: 0x%" IMG_UINT64_FMTSPECX "). Increase GPU Local heap size.",
+								__func__, uPrivateRASize, GPUVIRT_MIN_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uSharedRASize < GPUVIRT_SIZEOF_SHARED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Memory region shared between all OSIDs is too small (current: 0x%" IMG_UINT64_FMTSPECX ""
+								" required: 0x%" IMG_UINT64_FMTSPECX "). Increase GPU Local heap size.",
+								__func__, uSharedRASize, GPUVIRT_SIZEOF_SHARED));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	FOREACH_VALIDATION_OSID(i)
+	{
+		IMG_CHAR aszOSRAName[RA_MAX_NAME_LENGTH];
+
+		PVR_LOG(("GPUVIRT_VALIDATION: create arena Private OSID: %u, base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ".", i, uPrivateRABase, uPrivateRASize));
+
+		OSSNPrintf(aszOSRAName, RA_MAX_NAME_LENGTH, "GPUVIRT_OS%u", i);
+
+		psDeviceNode->psOSidSubArena[i] = RA_Create_With_Span(aszOSRAName,
+		                                                      OSGetPageShift(),
+		                                                      0,
+		                                                      uPrivateRABase,
+		                                                      uPrivateRASize,
+		                                                      RA_POLICY_DEFAULT);
+		PVR_GOTO_IF_FALSE(psDeviceNode->psOSidSubArena[i] != NULL, FreeArenas);
+		aui64OSidMin[GPUVIRT_VAL_REGION_SECURE][i] = uPrivateRABase;
+
+		if (i == 0)
+		{
+			/* Host OSID0 has access to all regions */
+			aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i] = uiGPULocalSize - 1ULL;
+		}
+		else
+		{
+			/* Guest OSIDs are limited to their private range */
+			aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i] = uPrivateRABase + uPrivateRASize - 1ULL;
+		}
+
+		/* uSharedRegionSize includes display heap */
+		aui64OSidMin[GPUVIRT_VAL_REGION_SHARED][i] = uSharedRegionBase;
+		aui64OSidMax[GPUVIRT_VAL_REGION_SHARED][i] = uSharedRegionBase + uSharedRegionSize - 1ULL;
+
+		uPrivateRABase += uPrivateRASize;
+	}
+
+	PVR_LOG(("GPUVIRT_VALIDATION: create arena Shared, base: 0x%" IMG_UINT64_FMTSPECX ", size: 0x%" IMG_UINT64_FMTSPECX ".", uSharedRABase, uSharedRASize));
+
+	/* uSharedRASize does not include display heap */
+	psDeviceNode->psOSSharedArena = RA_Create_With_Span("GPUVIRT_SHARED",
+	                                                    OSGetPageShift(),
+	                                                    0,
+	                                                    uSharedRABase,
+	                                                    uSharedRASize,
+	                                                    RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_FALSE(psDeviceNode->psOSSharedArena != NULL, FreeArenas);
+
+	FOREACH_VALIDATION_OSID(i)
+	{
+		PVR_LOG(("GPUVIRT_VALIDATION: HW regions OSID %u: Private Region = [0x%" IMG_UINT64_FMTSPECX "..0x%" IMG_UINT64_FMTSPECX "]"
+														" Shared Region = [0x%" IMG_UINT64_FMTSPECX "..0x%" IMG_UINT64_FMTSPECX "]",
+				 i,
+				 aui64OSidMin[GPUVIRT_VAL_REGION_SECURE][i],
+				 aui64OSidMax[GPUVIRT_VAL_REGION_SECURE][i],
+				 aui64OSidMin[GPUVIRT_VAL_REGION_SHARED][i],
+				 aui64OSidMax[GPUVIRT_VAL_REGION_SHARED][i]));
+	}
+
+	if (psDeviceNode->psDevConfig->pfnSysInitFirewall != NULL)
+	{
+		psDeviceNode->psDevConfig->pfnSysInitFirewall(psDeviceNode->psDevConfig->hSysData, aui64OSidMin, aui64OSidMax);
+	}
+
+	return PVRSRV_OK;
+
+FreeArenas:
+	DestroyGpuVirtValArenas(psDeviceNode);
+
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+}
+
+/*
+ * Counter-part to CreateGpuVirtValArenas.
+ */
+static void DestroyGpuVirtValArenas(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32	uiCounter = 0;
+
+	FOREACH_VALIDATION_OSID(uiCounter)
+	{
+		if (uiCounter == RGXFW_HOST_DRIVER_ID)
+		{
+			/*
+			 * NOTE: We overload psOSidSubArena[0] into the psLocalMemArena so we must
+			 * not free it here as it gets cleared later.
+			 */
+			continue;
+		}
+
+		if (psDeviceNode->psOSidSubArena[uiCounter] == NULL)
+		{
+			continue;
+		}
+		RA_Delete(psDeviceNode->psOSidSubArena[uiCounter]);
+	}
+
+	if (psDeviceNode->psOSSharedArena != NULL)
+	{
+		RA_Delete(psDeviceNode->psOSSharedArena);
+	}
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       PhysHeapMMUPxSetup
+@Description    Setup MMU Px allocation function pointers.
+@Input          psDeviceNode Pointer to device node struct
+@Return         PVRSRV_ERROR PVRSRV_OK on success.
+*/ /**************************************************************************/
+static PVRSRV_ERROR PhysHeapMMUPxSetup(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PHYS_HEAP_TYPE eHeapType;
+	PVRSRV_ERROR eError;
+
+	eError = PhysHeapAcquireByID(psDeviceNode->psDevConfig->eDefaultHeap,
+	                                      psDeviceNode, &psDeviceNode->psMMUPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorDeinit);
+
+	eHeapType = PhysHeapGetType(psDeviceNode->psMMUPhysHeap);
+
+	if (eHeapType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: GPU physical heap uses OS System memory (UMA)", __func__));
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		PVR_DPF((PVR_DBG_ERROR, "%s: Virtualisation Validation builds are currently only"
+								 " supported on systems with local memory (LMA).", __func__));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto ErrorDeinit;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: GPU physical heap uses local memory managed by the driver (LMA)", __func__));
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		eError = CreateGpuVirtValArenas(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "CreateGpuVirtValArenas", ErrorDeinit);
+#endif
+	}
+
+	return PVRSRV_OK;
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       PhysHeapMMUPxDeInit
+@Description    Deinit after PhysHeapMMUPxSetup.
+@Input          psDeviceNode Pointer to device node struct
+*/ /**************************************************************************/
+static void PhysHeapMMUPxDeInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/* Remove local LMA subarenas */
+	DestroyGpuVirtValArenas(psDeviceNode);
+#endif	/* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+	if (psDeviceNode->psMMUPhysHeap != NULL)
+	{
+		PhysHeapRelease(psDeviceNode->psMMUPhysHeap);
+		psDeviceNode->psMMUPhysHeap = NULL;
+	}
+}
+
+PVRSRV_ERROR PhysHeapInitDeviceHeaps(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PHYS_HEAP ePhysHeap;
+
+	eError = OSLockCreate(&psDeviceNode->hPhysHeapLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	eError = PVRSRVValidatePhysHeapConfig(psDevConfig);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVValidatePhysHeapConfig");
+
+	eError = PhysHeapCreateDeviceHeapsFromConfigs(psDeviceNode,
+	                                              psDevConfig->pasPhysHeaps,
+	                                              psDevConfig->ui32PhysHeapCount);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreateDeviceHeapsFromConfigs", ErrorDeinit);
+
+	/* Must loop from the 2nd heap to the last */
+	PVR_ASSERT(PVRSRV_PHYS_HEAP_DEFAULT == 0);
+	for (ePhysHeap = (PVRSRV_PHYS_HEAP)(PVRSRV_PHYS_HEAP_DEFAULT+1); ePhysHeap < PVRSRV_PHYS_HEAP_LAST; ePhysHeap++)
+	{
+		if (PhysHeapAcquiredByPVRLayer(ePhysHeap))
+		{
+			eError = PhysHeapAcquireByID(ePhysHeap, psDeviceNode, &psDeviceNode->apsPhysHeap[ePhysHeap]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorDeinit);
+		}
+	}
+
+	if (PhysHeapValidateDefaultHeapExists(psDeviceNode))
+	{
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapValidateDefaultHeapExists", ErrorDeinit);
+	}
+
+	eError = PhysHeapMMUPxSetup(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapMMUPxSetup", ErrorDeinit);
+
+	return PVRSRV_OK;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+
+	return eError;
+}
+
+void PhysHeapDeInitDeviceHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_PHYS_HEAP ePhysHeapIdx;
+	IMG_UINT32 i;
+
+	PhysHeapMMUPxDeInit(psDeviceNode);
+
+	/* Release heaps */
+	for (ePhysHeapIdx = PVRSRV_PHYS_HEAP_DEFAULT;
+		 ePhysHeapIdx < ARRAY_SIZE(psDeviceNode->apsPhysHeap);
+		 ePhysHeapIdx++)
+	{
+		if (psDeviceNode->apsPhysHeap[ePhysHeapIdx])
+		{
+			PhysHeapRelease(psDeviceNode->apsPhysHeap[ePhysHeapIdx]);
+		}
+	}
+
+	FOREACH_SUPPORTED_DRIVER(i)
+	{
+		if (psDeviceNode->apsFWPremapPhysHeap[i])
+		{
+			PhysHeapDestroy(psDeviceNode->apsFWPremapPhysHeap[i]);
+			psDeviceNode->apsFWPremapPhysHeap[i] = NULL;
+		}
+	}
+
+	PhysHeapDestroyDeviceHeaps(psDeviceNode);
+
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+}
+
+PVRSRV_ERROR PhysHeapCreate(PPVRSRV_DEVICE_NODE psDevNode,
+							  PHYS_HEAP_CONFIG *psConfig,
+							  PHYS_HEAP_POLICY uiPolicy,
+							  PHEAP_IMPL_DATA pvImplData,
+							  PHEAP_IMPL_FUNCS *psImplFuncs,
+							  PHYS_HEAP **ppsPhysHeap)
+{
+	PHYS_HEAP *psNew;
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	if (psConfig->eType == PHYS_HEAP_TYPE_UNKNOWN)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psImplFuncs != NULL, "psImplFuncs");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psImplFuncs->pfnCreatePMR != NULL ||
+	                                psImplFuncs->pfnCreatePMB,
+	                                "psImplFuncs->pfnCreatePMR || psImplFuncs->pfnCreatePMB");
+
+	psNew = OSAllocMem(sizeof(PHYS_HEAP));
+	PVR_RETURN_IF_NOMEM(psNew);
+
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		psNew->psMemFuncs = psConfig->uConfig.sLMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sLMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sLMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sLMA.pszHeapName) ? psConfig->uConfig.sLMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_IMA:
+		psNew->psMemFuncs = psConfig->uConfig.sIMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sIMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sIMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sIMA.pszHeapName) ? psConfig->uConfig.sIMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_DMA:
+		psNew->psMemFuncs = psConfig->uConfig.sDMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sDMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sDMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sDMA.pszHeapName) ? psConfig->uConfig.sDMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_UMA:
+		psNew->psMemFuncs = psConfig->uConfig.sUMA.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sUMA.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sUMA.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sUMA.pszHeapName) ? psConfig->uConfig.sUMA.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+	case PHYS_HEAP_TYPE_DLM:
+		psNew->psMemFuncs = psConfig->uConfig.sDLM.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sDLM.hPrivData;
+		psNew->pszPDumpMemspaceName = "None";
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sDLM.pszHeapName) ? psConfig->uConfig.sDLM.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+	case PHYS_HEAP_TYPE_WRAP:
+		psNew->psMemFuncs = psConfig->uConfig.sWRAP.psMemFuncs;
+		psNew->hPrivData = psConfig->uConfig.sWRAP.hPrivData;
+		psNew->pszPDumpMemspaceName = psConfig->uConfig.sWRAP.pszPDumpMemspaceName;
+		OSStringSafeCopy(psNew->aszName,
+		                 (psConfig->uConfig.sWRAP.pszHeapName) ? psConfig->uConfig.sWRAP.pszHeapName :
+		                                                        "Unknown PhysHeap",
+		                 PHYS_HEAP_NAME_SIZE);
+		break;
+#endif
+	default:
+		PVR_LOG_ERROR(PVRSRV_ERROR_NOT_IMPLEMENTED, "psConfig->eType not implemented");
+	}
+
+	psNew->eType = psConfig->eType;
+	psNew->ui32UsageFlags = psConfig->ui32UsageFlags;
+	psNew->uiPolicy = uiPolicy;
+	psNew->ui32RefCount = 0;
+	psNew->psDevNode = psDevNode;
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	OSAtomicWrite(&psNew->sOOMDetected, IMG_FALSE);
+#endif
+	psNew->pvImplData = pvImplData;
+	psNew->psImplFuncs = psImplFuncs;
+
+#if defined(SUPPORT_STATIC_IPA)
+	{
+		IMG_UINT8  ui8Val;
+
+		/* Ensure we do not cause an address fault by accessing beyond
+		 * the end of the psConfig->sIPAConfig structure.
+		 */
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyDefault;
+		psNew->ui32IPAPolicyValue = (IMG_UINT32)ui8Val;
+
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyMask;
+		psNew->ui32IPAClearMask = (IMG_UINT32)ui8Val;
+
+		ui8Val = psConfig->sIPAConfig.ui8IPAPolicyShift;
+		psNew->ui32IPAShift = (IMG_UINT32)ui8Val;
+		PVR_LOG_VA(PVR_DBG_MESSAGE, "%s: Physheap <%p> ['%s'] Config @ <%p> IPA = [0x%x, 0x%x, 0x%x]",
+		           __func__, psNew, psNew->aszName,
+		           psConfig, psNew->ui32IPAPolicyValue,
+		           psNew->ui32IPAClearMask, psNew->ui32IPAShift);
+	}
+#endif
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psNew;
+	}
+
+	psNew->psNext = psDevNode->psPhysHeapList;
+	psDevNode->psPhysHeapList = psNew;
+
+	PVR_DPF_RETURN_RC1(PVRSRV_OK, psNew);
+}
+
+void PhysHeapDestroyDeviceHeaps(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psNode;
+
+	if (psDevNode->hPhysHeapDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hPhysHeapDbgReqNotify);
+	}
+
+#if defined(PVRSRV_ENABLE_DYNAMIC_PHYSHEAPS)
+	/* DLM/IMA heaps must be destroyed in a specific order */
+	PhysHeapDestroyDLMIMAHeaps(psDevNode);
+#endif
+
+	psNode = psDevNode->psPhysHeapList;
+
+	while (psNode)
+	{
+		PHYS_HEAP *psTmp = psNode;
+
+		psNode = psNode->psNext;
+		PhysHeapDestroy(psTmp);
+	}
+}
+
+void PhysHeapDestroy(PHYS_HEAP *psPhysHeap)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PPVRSRV_DEVICE_NODE psDevNode = psPhysHeap->psDevNode;
+
+	PVR_DPF_ENTERED1(psPhysHeap);
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	if (PVRSRVGetPVRSRVData()->eServicesState == PVRSRV_SERVICES_STATE_OK)
+#endif
+	{
+		PVR_ASSERT(psPhysHeap->ui32RefCount == 0);
+	}
+
+	if (psDevNode->psPhysHeapList == psPhysHeap)
+	{
+		psDevNode->psPhysHeapList = psPhysHeap->psNext;
+	}
+	else
+	{
+		PHYS_HEAP *psTmp = psDevNode->psPhysHeapList;
+
+		while (psTmp->psNext != psPhysHeap)
+		{
+			psTmp = psTmp->psNext;
+		}
+		psTmp->psNext = psPhysHeap->psNext;
+	}
+
+	if (psImplFuncs->pfnDestroyData != NULL)
+	{
+		psImplFuncs->pfnDestroyData(psPhysHeap->pvImplData);
+	}
+
+	OSFreeMem(psPhysHeap);
+
+	PVR_DPF_RETURN;
+}
+
+static void _PhysHeapCountUserModeHeaps(PPVRSRV_DEVICE_NODE psDevNode,
+										PHYS_HEAP_USAGE_FLAGS ui32UsageFlags)
+{
+	PVRSRV_PHYS_HEAP eHeap;
+
+	for (eHeap = PVRSRV_PHYS_HEAP_DEFAULT;
+		 eHeap <= PVRSRV_PHYS_HEAP_LAST;
+		 eHeap++)
+	{
+		if (BIT_ISSET(ui32UsageFlags, eHeap) &&
+			PhysHeapUserModeAlloc(eHeap))
+		{
+			psDevNode->ui32UserAllocHeapCount++;
+			break;
+		}
+	}
+}
+
+PVRSRV_ERROR PhysHeapAcquire(PHYS_HEAP *psPhysHeap)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+
+	psPhysHeap->ui32RefCount++;
+
+	/* When acquiring a heap for the 1st time, perform a check and
+	 * calculate the total number of user accessible physical heaps */
+	if (psPhysHeap->ui32RefCount == 1)
+	{
+		_PhysHeapCountUserModeHeaps(psPhysHeap->psDevNode,
+									psPhysHeap->ui32UsageFlags);
+	}
+
+	return PVRSRV_OK;
+}
+
+static PHYS_HEAP * _PhysHeapFindHeapOrFallback(PVRSRV_PHYS_HEAP ePhysHeap,
+								   PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psPhysHeapNode = psDevNode->psPhysHeapList;
+	PVRSRV_PHYS_HEAP eFallback;
+
+	/* Swap the default heap alias for the system's real default PhysHeap */
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	/* Check cache of PhysHeaps to see if it has been resolved before */
+	if (psDevNode->apsPhysHeap[ePhysHeap] != NULL)
+	{
+		return psDevNode->apsPhysHeap[ePhysHeap];
+	}
+
+	/* Cache not ready, carry out search with fallback */
+	while (psPhysHeapNode)
+	{
+		if (BIT_ISSET(psPhysHeapNode->ui32UsageFlags, ePhysHeap))
+		{
+			return psPhysHeapNode;
+		}
+
+		psPhysHeapNode = psPhysHeapNode->psNext;
+	}
+
+	/* Find fallback PhysHeap */
+	eFallback = gasHeapProperties[ePhysHeap].eFallbackHeap;
+	if (ePhysHeap == eFallback)
+	{
+		return NULL;
+	}
+	else
+	{
+		return _PhysHeapFindHeapOrFallback(eFallback, psDevNode);
+	}
+}
+
+/*
+ * Acquire heap, no fallback, no recursion: single loop acquisition
+ */
+static PHYS_HEAP* _PhysHeapFindRealHeapNoFallback(PVRSRV_PHYS_HEAP ePhysHeap,
+                                          PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PHYS_HEAP *psPhysHeapNode = psDevNode->psPhysHeapList;
+
+	/* Swap the default heap alias for the system's real default PhysHeap */
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	/* Check cache of PhysHeaps to see if it has been resolved before.
+	 * The physheap must initialised to access its cache. */
+	if (psDevNode->apsPhysHeap[ePhysHeap] != NULL &&
+	    BIT_ISSET(psDevNode->apsPhysHeap[ePhysHeap]->ui32UsageFlags, ePhysHeap))
+	{
+		return psDevNode->apsPhysHeap[ePhysHeap];
+	}
+
+	/* Cache not ready, carry out search for real PhysHeap, no fallback */
+	while (psPhysHeapNode)
+	{
+		if (BIT_ISSET(psPhysHeapNode->ui32UsageFlags, ePhysHeap))
+		{
+			return psPhysHeapNode;
+		}
+
+		psPhysHeapNode = psPhysHeapNode->psNext;
+	}
+	return NULL;
+}
+
+PVRSRV_ERROR PhysHeapAcquireByID(PVRSRV_PHYS_HEAP eDevPhysHeap,
+								 PPVRSRV_DEVICE_NODE psDevNode,
+								 PHYS_HEAP **ppsPhysHeap)
+{
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(eDevPhysHeap < PVRSRV_PHYS_HEAP_LAST, "eDevPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	PVR_DPF_ENTERED1(ui32Flags);
+
+	OSLockAcquire(psDevNode->hPhysHeapLock);
+
+	psPhysHeap = _PhysHeapFindHeapOrFallback(eDevPhysHeap, psDevNode);
+
+	if (psPhysHeap != NULL)
+	{
+		psPhysHeap->ui32RefCount++;
+		PHYSHEAP_REFCOUNT_PRINT("%s: Heap %p, refcount = %d",
+								__func__, psPhysHeap, psPhysHeap->ui32RefCount);
+
+		/* When acquiring a heap for the 1st time, perform a check and
+		 * calculate the total number of user accessible physical heaps */
+		if (psPhysHeap->ui32RefCount == 1)
+		{
+			_PhysHeapCountUserModeHeaps(psDevNode, BIT(eDevPhysHeap));
+		}
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_PHYSHEAP_ID_INVALID;
+	}
+
+	OSLockRelease(psDevNode->hPhysHeapLock);
+
+	*ppsPhysHeap = psPhysHeap;
+	PVR_DPF_RETURN_RC1(eError, *ppsPhysHeap);
+}
+
+void PhysHeapRelease(PHYS_HEAP *psPhysHeap)
+{
+	PVR_DPF_ENTERED1(psPhysHeap);
+
+	OSLockAcquire(psPhysHeap->psDevNode->hPhysHeapLock);
+	psPhysHeap->ui32RefCount--;
+	PHYSHEAP_REFCOUNT_PRINT("%s: Heap %p, refcount = %d",
+							__func__, psPhysHeap, psPhysHeap->ui32RefCount);
+	OSLockRelease(psPhysHeap->psDevNode->hPhysHeapLock);
+
+	PVR_DPF_RETURN;
+}
+
+PHEAP_IMPL_DATA PhysHeapGetImplData(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->pvImplData;
+}
+
+PHYS_HEAP_TYPE PhysHeapGetType(PHYS_HEAP *psPhysHeap)
+{
+	PVR_ASSERT(psPhysHeap->eType != PHYS_HEAP_TYPE_UNKNOWN);
+	return psPhysHeap->eType;
+}
+
+PHYS_HEAP_POLICY PhysHeapGetPolicy(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->uiPolicy;
+}
+
+PHYS_HEAP_USAGE_FLAGS PhysHeapGetFlags(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32UsageFlags;
+}
+
+const IMG_CHAR *PhysHeapName(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->aszName;
+}
+
+IMG_BOOL PhysHeapValidateDefaultHeapExists(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_PHYS_HEAP eDefaultHeap = psDevNode->psDevConfig->eDefaultHeap;
+
+	return ((psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_DEFAULT] != NULL) &&
+			((psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_DEFAULT] ==
+			  psDevNode->apsPhysHeap[eDefaultHeap])));
+}
+
+#if defined(SUPPORT_STATIC_IPA)
+IMG_UINT32 PhysHeapGetIPAValue(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAPolicyValue;
+}
+
+IMG_UINT32 PhysHeapGetIPAMask(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAClearMask;
+}
+
+IMG_UINT32 PhysHeapGetIPAShift(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->ui32IPAShift;
+}
+#endif
+
+/*
+ * This function will set the psDevPAddr to whatever the system layer
+ * has set it for the referenced region.
+ * It will not fail if the psDevPAddr is invalid.
+ */
+PVRSRV_ERROR PhysHeapGetDevPAddr(PHYS_HEAP *psPhysHeap,
+									   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetDevPAddr != NULL)
+	{
+		eResult = psImplFuncs->pfnGetDevPAddr(psPhysHeap->pvImplData,
+											  psDevPAddr);
+	}
+
+	return eResult;
+}
+
+/*
+ * This function will set the psCpuPAddr to whatever the system layer
+ * has set it for the referenced region.
+ * It will not fail if the psCpuPAddr is invalid.
+ */
+PVRSRV_ERROR PhysHeapGetCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetCPUPAddr != NULL)
+	{
+		eResult = psImplFuncs->pfnGetCPUPAddr(psPhysHeap->pvImplData,
+											  psCpuPAddr);
+	}
+
+	return eResult;
+}
+
+PVRSRV_ERROR PhysHeapGetSize(PHYS_HEAP *psPhysHeap,
+								   IMG_UINT64 *puiSize)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnGetSize != NULL)
+	{
+		eResult = psImplFuncs->pfnGetSize(psPhysHeap->pvImplData,
+										  puiSize);
+	}
+
+	return eResult;
+}
+
+/* Used to add the total and free sizes of an IMA heap and the backing
+ * DLM heap
+ */
+static void PhysHeapIMAGetMemInfo(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 *puiTotalSize,
+                                  IMG_UINT64 *puiFreeSize)
+{
+	PHYS_HEAP *psDLMHeap;
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	IMG_UINT64 ui64DLMTotalSize;
+	IMG_UINT64 ui64DLMFreeSize;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(
+		psPhysHeap->eType == PHYS_HEAP_TYPE_IMA,
+		"Physheap type not IMA");
+
+	/* Obtain the DLM heap chained to this IMA heap */
+	PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetHeapDLMBacking);
+	psPhysHeap->psImplFuncs->pfnGetHeapDLMBacking(psPhysHeap->pvImplData,
+	                                              &psDLMHeap);
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDLMHeap != NULL, "pfnGetHeapDLMBacking");
+
+	/* Obtain the memstats for the current IMA heap */
+	PVR_ASSERT(psPhysHeap->psImplFuncs->pfnGetFactoryMemStats);
+	psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+	                                               &ui64TotalSize,
+	                                               &ui64FreeSize);
+
+	/* Obtain the memstats for the chained DLM heap */
+	PVR_ASSERT(psDLMHeap->psImplFuncs->pfnGetFactoryMemStats);
+	psDLMHeap->psImplFuncs->pfnGetFactoryMemStats(psDLMHeap->pvImplData,
+	                                              &ui64DLMTotalSize,
+	                                              &ui64DLMFreeSize);
+
+	/* Total size is just the DLM heap size as the IMA heap imports a
+	 * subset of that total size.
+	 * Free size is a combination of the two because DLM heap may
+	 * have spare PMBs to provide and the IMA heap may have space
+	 * in an imported PMB.
+	 */
+	*puiTotalSize = ui64DLMTotalSize;
+	*puiFreeSize = ui64FreeSize + ui64DLMFreeSize;
+}
+
+PVRSRV_ERROR
+PhysHeapGetMemInfo(PVRSRV_DEVICE_NODE *psDevNode,
+                   IMG_UINT32 ui32PhysHeapCount,
+                   PVRSRV_PHYS_HEAP *paePhysHeapID,
+                   PHYS_HEAP_MEM_STATS_PTR paPhysHeapMemStats)
+{
+	IMG_UINT32 i = 0;
+	PHYS_HEAP *psPhysHeap;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui32PhysHeapCount <= MAX_USER_MODE_ALLOC_PHYS_HEAPS, "ui32PhysHeapCount invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(paePhysHeapID != NULL, "paePhysHeapID invalid");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(paPhysHeapMemStats != NULL, "paPhysHeapMemStats invalid");
+
+	for (i = 0; i < ui32PhysHeapCount; i++)
+	{
+		if (paePhysHeapID[i] >= PVRSRV_PHYS_HEAP_LAST)
+		{
+			return PVRSRV_ERROR_PHYSHEAP_ID_INVALID;
+		}
+
+		psPhysHeap = _PhysHeapFindRealHeapNoFallback(paePhysHeapID[i], psDevNode);
+
+		paPhysHeapMemStats[i].ui32PhysHeapFlags = 0;
+
+		if (psPhysHeap && PhysHeapUserModeAlloc(paePhysHeapID[i])
+				&& psPhysHeap->psImplFuncs->pfnGetFactoryMemStats)
+		{
+			if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+			{
+				PhysHeapIMAGetMemInfo(psPhysHeap,
+				                      &paPhysHeapMemStats[i].ui64TotalSize,
+				                      &paPhysHeapMemStats[i].ui64FreeSize);
+			}
+			else
+			{
+				psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+				                                               &paPhysHeapMemStats[i].ui64TotalSize,
+				                                               &paPhysHeapMemStats[i].ui64FreeSize);
+			}
+
+			if (paePhysHeapID[i] == psDevNode->psDevConfig->eDefaultHeap)
+			{
+				paPhysHeapMemStats[i].ui32PhysHeapFlags |= PVRSRV_PHYS_HEAP_FLAGS_IS_DEFAULT;
+			}
+
+			paPhysHeapMemStats[i].ePhysHeapType = psPhysHeap->eType;
+		}
+		else
+		{
+			paPhysHeapMemStats[i].ui64TotalSize = 0;
+			paPhysHeapMemStats[i].ui64FreeSize = 0;
+			paPhysHeapMemStats[i].ePhysHeapType = PHYS_HEAP_TYPE_UNKNOWN;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+void PhysHeapCpuPAddrToDevPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	psPhysHeap->psMemFuncs->pfnCpuPAddrToDevPAddr(psPhysHeap->hPrivData,
+												 ui32NumOfAddr,
+												 psDevPAddr,
+												 psCpuPAddr);
+}
+
+void PhysHeapDevPAddrToCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr,
+								IMG_DEV_PHYADDR *psDevPAddr)
+{
+	psPhysHeap->psMemFuncs->pfnDevPAddrToCpuPAddr(psPhysHeap->hPrivData,
+												 ui32NumOfAddr,
+												 psCpuPAddr,
+												 psDevPAddr);
+}
+
+IMG_CHAR *PhysHeapPDumpMemspaceName(PHYS_HEAP *psPhysHeap)
+{
+	return psPhysHeap->pszPDumpMemspaceName;
+}
+
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+static inline void _LogOOMDetection(IMG_BOOL isOOMDetected, PHYS_HEAP *psPhysHeap, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	IMG_BOOL bExistingVal = OSAtomicExchange(&psPhysHeap->sOOMDetected, isOOMDetected);
+	PVRSRV_PHYS_HEAP ePhysIdx = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	if (bExistingVal != isOOMDetected)
+	{
+		PVR_LOG(("Device: %d Physheap: %s OOM: %s",
+				(psPhysHeap->psDevNode->sDevId.ui32InternalID),
+				g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel,
+				(isOOMDetected) ? "Detected" : "Resolved"));
+	}
+}
+#endif
+
+PVRSRV_ERROR PhysHeapCreatePMR(PHYS_HEAP *psPhysHeap,
+							   struct _CONNECTION_DATA_ *psConnection,
+							   IMG_DEVMEM_SIZE_T uiSize,
+							   IMG_UINT32 ui32NumPhysChunks,
+							   IMG_UINT32 ui32NumVirtChunks,
+							   IMG_UINT32 *pui32MappingTable,
+							   IMG_UINT32 uiLog2PageSize,
+							   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							   const IMG_CHAR *pszAnnotation,
+							   IMG_PID uiPid,
+							   PMR **ppsPMRPtr,
+							   IMG_UINT32 ui32PDumpFlags,
+							   PVRSRV_MEMALLOCFLAGS_T *puiOutFlags)
+{
+	PVRSRV_ERROR eError;
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	IMG_UINT64 uiFreeBytes;
+	PVRSRV_PHYS_HEAP eDemotionPhysIdx;
+	PVRSRV_MEMALLOCFLAGS_T uiDemotionFlags = uiFlags;
+	PVRSRV_PHYS_HEAP ePhysIdx = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+	PHYS_HEAP *psDemotionHeap = NULL;
+#endif
+	eError = psImplFuncs->pfnCreatePMR(psPhysHeap,
+									 psConnection,
+									 uiSize,
+									 ui32NumPhysChunks,
+									 ui32NumVirtChunks,
+									 pui32MappingTable,
+									 uiLog2PageSize,
+									 uiFlags,
+									 pszAnnotation,
+									 uiPid,
+									 ppsPMRPtr,
+									 ui32PDumpFlags);
+
+#if !defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+	/* Check for OOM error, return if otherwise */
+	_LogOOMDetection(((IsOOMError(eError)) ? IMG_TRUE : IMG_FALSE), psPhysHeap, uiFlags);
+	if (eError == PVRSRV_OK)
+	{
+		if (puiOutFlags)
+		{
+			*puiOutFlags = uiFlags;
+		}
+		return eError;
+	}
+	PVR_LOG_RETURN_IF_FALSE((IsOOMError(eError)), "Failed to allocate PMR", eError);
+
+	/* Skip logic and return if mandate flag is set */
+	if (PVRSRV_CHECK_MANDATED_PHYSHEAP(uiFlags))
+	{
+		return eError;
+	}
+
+	/* Demotion only occurs on CPU_LOCAL,GPU_LOCAL,GPU_PRIVATE */
+	if (ePhysIdx > PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+	{
+		return eError;
+	}
+
+	eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+	for (eDemotionPhysIdx = (PVRSRV_PHYS_HEAP)(ePhysIdx-1); eDemotionPhysIdx != PVRSRV_PHYS_HEAP_DEFAULT; eDemotionPhysIdx--)
+	{
+		PVRSRV_CHANGE_PHYS_HEAP_HINT(eDemotionPhysIdx, uiDemotionFlags);
+		PVR_LOG_IF_FALSE_VA(PVR_DBG_MESSAGE, (ePhysIdx-eDemotionPhysIdx < 2), "Demoted from %s to CPU_LOCAL. "
+						"Expect Performance to be affected!", g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel);
+		psDemotionHeap = _PhysHeapFindRealHeapNoFallback(eDemotionPhysIdx, psPhysHeap->psDevNode);
+
+		/* Either no alternative available, or allocation already failed on selected heap */
+		if (psDemotionHeap == NULL || psPhysHeap == psDemotionHeap)
+		{
+			continue;
+		}
+
+		if (PhysHeapFreeMemCheck(psDemotionHeap, uiSize, &uiFreeBytes) != PVRSRV_OK)
+		{
+			_LogOOMDetection(IMG_TRUE, psDemotionHeap, uiDemotionFlags);
+			continue;
+		}
+
+		psImplFuncs = psDemotionHeap->psImplFuncs;
+		eError = psImplFuncs->pfnCreatePMR(psDemotionHeap,
+						 psConnection,
+						 uiSize,
+						 ui32NumPhysChunks,
+						 ui32NumVirtChunks,
+						 pui32MappingTable,
+						 uiLog2PageSize,
+						 uiDemotionFlags,
+						 pszAnnotation,
+						 uiPid,
+						 ppsPMRPtr,
+						 ui32PDumpFlags);
+		_LogOOMDetection(((IsOOMError(eError)) ? IMG_TRUE : IMG_FALSE), psDemotionHeap, uiDemotionFlags);
+
+		if (eError == PVRSRV_OK)
+		{
+			if (puiOutFlags)
+			{
+				*puiOutFlags = uiDemotionFlags;
+			}
+			break;
+		}
+	}
+	if (eError == PVRSRV_OK)
+	{
+		/* Success demotion worked error Ok - emit warning. */
+		PVR_LOG_VA(PVR_DBG_WARNING, "PhysHeap(%s) failed to allocate PMR. Demoted to %s" ,
+						g_asPhysHeapUsageFlagStrings[ePhysIdx-1].pszLabel,
+						g_asPhysHeapUsageFlagStrings[eDemotionPhysIdx-1].pszLabel);
+	}
+	else
+	{
+		/* Unable to create PMR (Heap not found or CreatePMR failed) - emit Error */
+		PVR_LOG_VA(PVR_DBG_ERROR, "Error raised %s : Unable to %s." ,
+		                          PVRSRVGETERRORSTRING(eError),
+		                          (psDemotionHeap == NULL) ?  "find heaps for demotion" :
+		                                                      "allocate PMR via Demotion heap");
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		{
+			PPVRSRV_DEVICE_NODE psDevNode = PhysHeapDeviceNode(psPhysHeap);
+			OSLockAcquire(psDevNode->hPMRZombieListLock);
+			PVR_LOG_VA(PVR_DBG_ERROR, "PMR Zombie Count: %u, PMR Zombie Count In Cleanup: %u",
+			                          psDevNode->uiPMRZombieCount,
+			                          psDevNode->uiPMRZombieCountInCleanup);
+			OSLockRelease(psDevNode->hPMRZombieListLock);
+		}
+#endif
+	}
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PhysHeapCreatePMB(PHYS_HEAP *psPhysHeap,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               const IMG_CHAR *pszAnnotation,
+                               PMB **ppsPMRPtr,
+                               RA_BASE_T *puiBase,
+                               RA_LENGTH_T *puiSize)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_LOG_RETURN_IF_FALSE(
+		psPhysHeap->eType == PHYS_HEAP_TYPE_DLM,
+		"Physheap type not DLM",
+		PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT(psImplFuncs->pfnCreatePMB);
+
+	return psImplFuncs->pfnCreatePMB(psPhysHeap,
+	                                 uiSize,
+	                                 pszAnnotation,
+	                                 ppsPMRPtr,
+	                                 puiBase,
+	                                 puiSize);
+}
+
+PPVRSRV_DEVICE_NODE PhysHeapDeviceNode(PHYS_HEAP *psPhysHeap)
+{
+	PVR_ASSERT(psPhysHeap != NULL);
+
+	return psPhysHeap->psDevNode;
+}
+
+static IMG_BOOL PhysHeapCreatedByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return (gasHeapProperties[ePhysHeap].ePVRLayerAction != PVR_LAYER_HEAP_ACTION_IGNORE);
+}
+
+static IMG_BOOL PhysHeapAcquiredByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return (gasHeapProperties[ePhysHeap].ePVRLayerAction == PVR_LAYER_HEAP_ACTION_INITIALISE);
+}
+
+IMG_BOOL PhysHeapUserModeAlloc(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	PVR_ASSERT(ePhysHeap < PVRSRV_PHYS_HEAP_LAST);
+
+	return gasHeapProperties[ePhysHeap].bUserModeAlloc;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+PVRSRV_ERROR PhysHeapPagesAllocGPV(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_UINT32 ui32OSid, IMG_PID uiPid)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesAllocGPV != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesAllocGPV(psPhysHeap,
+		                                        uiSize, psMemHandle, psDevPAddr, ui32OSid, uiPid);
+	}
+
+	return eResult;
+}
+#endif
+
+PVRSRV_ERROR PhysHeapPagesAlloc(PHYS_HEAP *psPhysHeap, size_t uiSize,
+								PG_HANDLE *psMemHandle,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_PID uiPid)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesAlloc != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesAlloc(psPhysHeap,
+		                                       uiSize, psMemHandle, psDevPAddr, uiPid);
+	}
+
+	return eResult;
+}
+
+void PhysHeapPagesFree(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_ASSERT(psImplFuncs->pfnPagesFree != NULL);
+
+	if (psImplFuncs->pfnPagesFree != NULL)
+	{
+		psImplFuncs->pfnPagesFree(psPhysHeap,
+		                          psMemHandle);
+	}
+}
+
+PVRSRV_ERROR PhysHeapPagesMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle, size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+							  void **pvPtr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesMap != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesMap(psPhysHeap,
+		                                   pshMemHandle, uiSize, psDevPAddr, pvPtr);
+	}
+
+	return eResult;
+}
+
+void PhysHeapPagesUnMap(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle, void *pvPtr)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+
+	PVR_ASSERT(psImplFuncs->pfnPagesUnMap != NULL);
+
+	if (psImplFuncs->pfnPagesUnMap != NULL)
+	{
+		psImplFuncs->pfnPagesUnMap(psPhysHeap,
+		                           psMemHandle, pvPtr);
+	}
+}
+
+PVRSRV_ERROR PhysHeapPagesClean(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle,
+							  IMG_UINT32 uiOffset,
+							  IMG_UINT32 uiLength)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	PVRSRV_ERROR eResult = PVRSRV_ERROR_NOT_IMPLEMENTED;
+
+	if (psImplFuncs->pfnPagesClean != NULL)
+	{
+		eResult = psImplFuncs->pfnPagesClean(psPhysHeap,
+		                                     pshMemHandle, uiOffset, uiLength);
+	}
+
+	return eResult;
+}
+
+IMG_UINT32 PhysHeapGetPageShift(PHYS_HEAP *psPhysHeap)
+{
+	PHEAP_IMPL_FUNCS *psImplFuncs = psPhysHeap->psImplFuncs;
+	IMG_UINT32 ui32PageShift = 0;
+
+	PVR_ASSERT(psImplFuncs->pfnGetPageShift != NULL);
+
+	if (psImplFuncs->pfnGetPageShift != NULL)
+	{
+		ui32PageShift = psImplFuncs->pfnGetPageShift();
+	}
+
+	return ui32PageShift;
+}
+
+PVRSRV_ERROR PhysHeapFreeMemCheck(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 ui64MinRequiredMem,
+                                  IMG_UINT64 *pui64FreeMem)
+{
+	IMG_UINT64 ui64TotalSize;
+	IMG_UINT64 ui64FreeSize;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPhysHeap != NULL, "psPhysHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui64FreeMem != NULL, "pui64FreeMem");
+
+
+	if (psPhysHeap->eType == PHYS_HEAP_TYPE_IMA)
+	{
+		PhysHeapIMAGetMemInfo(psPhysHeap->pvImplData,
+		                            &ui64TotalSize,
+		                            &ui64FreeSize);
+	}
+	else
+	{
+		psPhysHeap->psImplFuncs->pfnGetFactoryMemStats(psPhysHeap->pvImplData,
+		                                               &ui64TotalSize,
+		                                               &ui64FreeSize);
+	}
+
+	*pui64FreeMem = ui64FreeSize;
+	if (ui64MinRequiredMem >= *pui64FreeMem)
+	{
+		eError = PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (physheap.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/physheap.h b/drivers/gpu/drm/img-rogue/physheap.h
new file mode 100644
index 000000000000..f5e03c655278
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physheap.h
@@ -0,0 +1,551 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physical heap management header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the interface for the physical heap management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"
+#include "opaque_types.h"
+#include "pmr_impl.h"
+#include "physheap_config.h"
+#include "pvrsrv_device.h"
+#include "ra.h"
+
+#ifndef PHYSHEAP_H
+#define PHYSHEAP_H
+
+#define B2KB(x) ((x) >> 10)
+#define B2MB(x) ((x) >> 20)
+
+static inline IMG_UINT64 KB2B(IMG_UINT64 ui64Kilobytes) { return ui64Kilobytes << 10; }
+static inline IMG_UINT64 MB2B(IMG_UINT64 ui64Megabytes) { return ui64Megabytes << 20; }
+
+typedef struct _PHYS_HEAP_ PHYS_HEAP;
+#define INVALID_PHYS_HEAP 0xDEADDEAD
+
+/* PMB (Physical Memory Block) */
+typedef struct _PMB_ PMB;
+
+typedef IMG_UINT32 PHYS_HEAP_POLICY;
+
+/* Heap has default allocation policy and does not require
+ * any additional OS Functionality. Physically contiguous
+ * allocations are required for this physheap.
+ */
+#define PHYS_HEAP_POLICY_DEFAULT (0U)
+
+/*
+ * Heap has allocation strategy that may produce non
+ * physically contiguous allocations, additional OS functionality
+ * is required to map these allocations into the kernel.
+ */
+#define PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG      (1U)
+#define PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK (1U)
+
+struct _CONNECTION_DATA_;
+
+typedef struct _PG_HANDLE_
+{
+	union
+	{
+		void *pvHandle;
+		IMG_UINT64 ui64Handle;
+	}u;
+	/* The allocation order is log2 value of the number of pages to allocate.
+	 * As such this is a correspondingly small value. E.g, for order 4 we
+	 * are talking 2^4 * PAGE_SIZE contiguous allocation.
+	 * DevPxAlloc API does not need to support orders higher than 4.
+	 */
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	IMG_BYTE    uiOrder;    /* Order of the corresponding allocation */
+	IMG_BYTE    uiOSid;     /* OSid to use for allocation arena.
+	                         * Connection-specific. */
+	IMG_BYTE    uiPad1,
+	            uiPad2;     /* Spare */
+#else
+	IMG_BYTE    uiOrder;    /* Order of the corresponding allocation */
+	IMG_BYTE    uiPad1,
+	            uiPad2,
+	            uiPad3;     /* Spare */
+#endif
+} PG_HANDLE;
+
+/*! Pointer to private implementation specific data */
+typedef void *PHEAP_IMPL_DATA;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_DESTROY_DATA
+@Description    Destroy private implementation specific data.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+*/ /**************************************************************************/
+typedef void (*PFN_DESTROY_DATA)(PHEAP_IMPL_DATA);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_DEV_PADDR
+@Description    Get heap device physical address.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_DEV_PHYADDR    Device physical address.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_DEV_PADDR)(PHEAP_IMPL_DATA, IMG_DEV_PHYADDR*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_CPU_PADDR
+@Description    Get heap CPU physical address.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_CPU_PHYADDR    CPU physical address.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_CPU_PADDR)(PHEAP_IMPL_DATA, IMG_CPU_PHYADDR*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_SIZE
+@Description    Get size of heap.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_UINT64         Size of heap.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_GET_SIZE)(PHEAP_IMPL_DATA, IMG_UINT64*);
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_PAGE_SHIFT
+@Description    Get heap log2 page shift.
+@Return         IMG_UINT32         Log2 page shift
+*/ /**************************************************************************/
+typedef IMG_UINT32 (*PFN_GET_PAGE_SHIFT)(void);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_MEM_STATS
+@Description    Get total and free memory size of the physical heap managed by
+                the PMR Factory.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@Output         IMG_UINT64         total Size of heap.
+@Output         IMG_UINT64         free Size available in a heap.
+@Return         none
+*/ /**************************************************************************/
+typedef void (*PFN_GET_MEM_STATS)(PHEAP_IMPL_DATA, IMG_UINT64 *, IMG_UINT64 *);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_HEAP_STATS_STR_ITER
+@Description    Get string of heap memory spans constituting the heap. This
+                function can be iterated on to print sequential lines of the
+                heap data. Iterate until IMG_FALSE is returned
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@InOut          IMG_CHAR           Pointer to string buffer to be populated
+                                   with sequential heap data.
+@Input          IMG_UINT32         Size of the string buffer.
+@InOut          void**             Iter handle.
+@Return         IMG_BOOL
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_GET_HEAP_SPANS_STR_ITER)(PHEAP_IMPL_DATA, IMG_CHAR *, IMG_UINT32, void **);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_GET_HEAP_DLM_BACKING
+@Description    Extract reference to DLM heap backing the current IMA heap.
+@Input          PHEAP_IMPL_DATA    Pointer to implementation data.
+@InOut          PHYS_HEAP**        Pointer to DLM backing heap.
+@Return         void
+*/ /**************************************************************************/
+typedef void (*PFN_GET_HEAP_DLM_BACKING)(PHEAP_IMPL_DATA, PHYS_HEAP **);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef PVRSRV_ERROR (*PFN_PAGES_ALLOC_GPV)(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                            PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+                                            IMG_UINT32 ui32OSid, IMG_PID uiPid);
+#endif
+typedef PVRSRV_ERROR (*PFN_PAGES_ALLOC)(PHYS_HEAP *psPhysHeap, size_t uiSize,
+                                        PG_HANDLE *psMemHandle, IMG_DEV_PHYADDR *psDevPAddr,
+                                        IMG_PID uiPid);
+
+typedef void (*PFN_PAGES_FREE)(PHYS_HEAP *psPhysHeap, PG_HANDLE *psMemHandle);
+
+typedef PVRSRV_ERROR (*PFN_PAGES_MAP)(PHYS_HEAP *psPhysHeap, PG_HANDLE *pshMemHandle,
+                                      size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                                      void **pvPtr);
+
+typedef void (*PFN_PAGES_UNMAP)(PHYS_HEAP *psPhysHeap,
+                                PG_HANDLE *psMemHandle, void *pvPtr);
+
+typedef PVRSRV_ERROR (*PFN_PAGES_CLEAN)(PHYS_HEAP *psPhysHeap,
+                                        PG_HANDLE *pshMemHandle,
+                                        IMG_UINT32 uiOffset,
+                                        IMG_UINT32 uiLength);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_CREATE_PMR
+@Description    Create a PMR physical allocation and back with RAM on creation,
+                if required. The RAM page comes either directly from
+                the Phys Heap's associated pool of memory or from an OS API.
+@Input          psPhysHeap         Pointer to Phys Heap.
+@Input          psConnection       Pointer to device connection.
+@Input          uiSize             Allocation size.
+@Input          ui32NumPhysChunks  Physical chunk count.
+@Input          ui32NumVirtChunks  Virtual chunk count.
+@Input          pui32MappingTable  Mapping Table.
+@Input          uiLog2PageSize     Page size.
+@Input          uiFlags            Memalloc flags.
+@Input          pszAnnotation      Annotation.
+@Input          uiPid              Process ID.
+@Output         ppsPMRPtr          Pointer to PMR.
+@Input          ui32PDumpFlag      PDump flags.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CREATE_PMR)(PHYS_HEAP *psPhysHeap,
+									   struct _CONNECTION_DATA_ *psConnection,
+									   IMG_DEVMEM_SIZE_T uiSize,
+									   IMG_UINT32 ui32NumPhysChunks,
+									   IMG_UINT32 ui32NumVirtChunks,
+									   IMG_UINT32 *pui32MappingTable,
+									   IMG_UINT32 uiLog2PageSize,
+									   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+									   const IMG_CHAR *pszAnnotation,
+									   IMG_PID uiPid,
+									   PMR **ppsPMRPtr,
+									   IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_CREATE_PMB
+@Description    Create a PMB physical allocation and back with Card memory
+                on creation, if required. The card memory comes
+                directly from the DLM Phys Heap's associated pool of memory.
+@Input          psPhysHeap         Pointer to Phys Heap to create the PMB on,
+                                   physheap should be DLM type.
+@Input          uiSize             Allocation size.
+@Input          pszAnnotation      Annotation.
+@Output         ppsPMBPtr          Pointer to PMB created.
+@Output         puiBase            Out pointer to RA Base of PMB.
+@Output         puiSize            Out pointer to size of PMB
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CREATE_PMB)(PHYS_HEAP *psPhysHeap,
+                                       IMG_DEVMEM_SIZE_T uiSize,
+                                       const IMG_CHAR *pszAnnotation,
+                                       PMB **ppsPMBPtr,
+                                       RA_BASE_T *puiBase,
+                                       RA_LENGTH_T *puiSize);
+
+/*! Implementation specific function table */
+typedef struct PHEAP_IMPL_FUNCS_TAG
+{
+	PFN_DESTROY_DATA pfnDestroyData;
+	PFN_GET_DEV_PADDR pfnGetDevPAddr;
+	PFN_GET_CPU_PADDR pfnGetCPUPAddr;
+	PFN_GET_SIZE pfnGetSize;
+	PFN_GET_PAGE_SHIFT pfnGetPageShift;
+	PFN_GET_MEM_STATS pfnGetFactoryMemStats;
+	PFN_GET_HEAP_SPANS_STR_ITER pfnGetHeapSpansStringIter;
+	PFN_GET_HEAP_DLM_BACKING pfnGetHeapDLMBacking;
+	PFN_CREATE_PMR pfnCreatePMR;
+	PFN_CREATE_PMB pfnCreatePMB;
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	PFN_PAGES_ALLOC_GPV pfnPagesAllocGPV;
+#endif
+	PFN_PAGES_ALLOC pfnPagesAlloc;
+	PFN_PAGES_FREE pfnPagesFree;
+	PFN_PAGES_MAP pfnPagesMap;
+	PFN_PAGES_UNMAP pfnPagesUnMap;
+	PFN_PAGES_CLEAN pfnPagesClean;
+} PHEAP_IMPL_FUNCS;
+
+/*************************************************************************/ /*!
+@Function       PhysHeapInitDeviceHeaps
+@Description    Registers and acquires physical memory heaps
+@Input          psDeviceNode pointer to device node
+@Input          psDevConfig  pointer to device config
+@Return         PVRSRV_ERROR PVRSRV_OK on success, or a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapInitDeviceHeaps(PPVRSRV_DEVICE_NODE psDeviceNode, PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDeInitDeviceHeaps
+@Description    Releases and unregisters physical memory heaps
+@Input          psDeviceNode pointer to device node
+@Return         PVRSRV_ERROR PVRSRV_OK on success, or a PVRSRV_ error code
+*/ /**************************************************************************/
+void PhysHeapDeInitDeviceHeaps(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreateHeapFromConfig
+@Description    Create a new heap. Calls specific heap API depending
+                on heap type.
+@Input          psDevNode    Pointer to device node struct.
+@Input          psConfig     Heap configuration.
+@Output         ppsPhysHeap  Optional pointer to the created heap. Can be NULL
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysHeapCreateHeapFromConfig(PPVRSRV_DEVICE_NODE psDevNode,
+							 PHYS_HEAP_CONFIG *psConfig,
+							 PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreate
+@Description    Create a new heap. Allocated and stored internally.
+                Destroy with PhysHeapDestroy when no longer required.
+@Input          psDevNode    Pointer to device node struct
+@Input          psConfig     Heap configuration.
+@Input          uiPolicy     Phys heap allocation policy.
+@Input          pvImplData   Implementation specific data. Can be NULL.
+@Input          psImplFuncs  Implementation specific function table. Must be
+                             a valid pointer.
+@Output         ppsPhysHeap  Optional pointer to the created heap. Can be NULL
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreate(PPVRSRV_DEVICE_NODE psDevNode,
+							PHYS_HEAP_CONFIG *psConfig,
+							PHYS_HEAP_POLICY uiPolicy,
+							PHEAP_IMPL_DATA pvImplData,
+							PHEAP_IMPL_FUNCS *psImplFuncs,
+							PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDestroyDeviceHeaps
+@Description    Destroys all heaps referenced by a device.
+@Input          psDevNode Pointer to a device node struct.
+@Return         void
+*/ /**************************************************************************/
+void PhysHeapDestroyDeviceHeaps(PPVRSRV_DEVICE_NODE psDevNode);
+
+void PhysHeapDestroy(PHYS_HEAP *psPhysHeap);
+
+PVRSRV_ERROR PhysHeapAcquire(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapAcquireByID
+@Description    Acquire PhysHeap by DevPhysHeap.
+@Input          eDevPhysHeap Device Phys Heap.
+@Input          psDevNode    Pointer to device node struct
+@Output         ppsPhysHeap  PhysHeap if found.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapAcquireByID(PVRSRV_PHYS_HEAP eDevPhysHeap,
+								 PPVRSRV_DEVICE_NODE psDevNode,
+								 PHYS_HEAP **ppsPhysHeap);
+
+void PhysHeapRelease(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetImplData
+@Description    Get physical heap implementation specific data.
+@Input          psPhysHeap   Pointer to physical heap.
+@Input          psConfig     Heap configuration.
+@Return         pvImplData   Implementation specific data. Can be NULL.
+*/ /**************************************************************************/
+PHEAP_IMPL_DATA PhysHeapGetImplData(PHYS_HEAP *psPhysHeap);
+
+PHYS_HEAP_TYPE PhysHeapGetType(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetPolicy
+@Description    Get phys heap allocation policy flags.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         PHYS_HEAP_POLICY Phys heap policy flags.
+*/ /**************************************************************************/
+PHYS_HEAP_POLICY PhysHeapGetPolicy(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetFlags
+@Description    Get phys heap usage flags.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         PHYS_HEAP_USAGE_FLAGS Phys heap usage flags.
+*/ /**************************************************************************/
+PHYS_HEAP_USAGE_FLAGS PhysHeapGetFlags(PHYS_HEAP *psPhysHeap);
+
+IMG_BOOL PhysHeapValidateDefaultHeapExists(PPVRSRV_DEVICE_NODE psDevNode);
+
+#if defined(SUPPORT_STATIC_IPA)
+IMG_UINT32 PhysHeapGetIPAValue(PHYS_HEAP *psPhysHeap);
+
+IMG_UINT32 PhysHeapGetIPAMask(PHYS_HEAP *psPhysHeap);
+
+IMG_UINT32 PhysHeapGetIPAShift(PHYS_HEAP *psPhysHeap);
+#endif
+
+PVRSRV_ERROR PhysHeapGetCpuPAddr(PHYS_HEAP *psPhysHeap,
+									   IMG_CPU_PHYADDR *psCpuPAddr);
+
+
+PVRSRV_ERROR PhysHeapGetSize(PHYS_HEAP *psPhysHeap,
+								   IMG_UINT64 *puiSize);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetMemInfo
+@Description    Get phys heap memory statistics for a given physical heap ID.
+@Input          psDevNode          Pointer to device node struct
+@Input          ui32PhysHeapCount  Physical heap count
+@Input          paePhysHeapID      Physical heap ID
+@Output         paPhysHeapMemStats Buffer that holds the memory statistics
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysHeapGetMemInfo(PPVRSRV_DEVICE_NODE psDevNode,
+                   IMG_UINT32 ui32PhysHeapCount,
+                   PVRSRV_PHYS_HEAP *paePhysHeapID,
+                   PHYS_HEAP_MEM_STATS_PTR paPhysHeapMemStats);
+
+PVRSRV_ERROR PhysHeapGetDevPAddr(PHYS_HEAP *psPhysHeap,
+								 IMG_DEV_PHYADDR *psDevPAddr);
+
+void PhysHeapCpuPAddrToDevPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_DEV_PHYADDR *psDevPAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr);
+
+void PhysHeapDevPAddrToCpuPAddr(PHYS_HEAP *psPhysHeap,
+								IMG_UINT32 ui32NumOfAddr,
+								IMG_CPU_PHYADDR *psCpuPAddr,
+								IMG_DEV_PHYADDR *psDevPAddr);
+
+IMG_CHAR *PhysHeapPDumpMemspaceName(PHYS_HEAP *psPhysHeap);
+
+const IMG_CHAR *PhysHeapName(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreatePMR
+@Description    Function calls an implementation-specific function pointer.
+                See function pointer for details.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreatePMR(PHYS_HEAP *psPhysHeap,
+							   struct _CONNECTION_DATA_ *psConnection,
+							   IMG_DEVMEM_SIZE_T uiSize,
+							   IMG_UINT32 ui32NumPhysChunks,
+							   IMG_UINT32 ui32NumVirtChunks,
+							   IMG_UINT32 *pui32MappingTable,
+							   IMG_UINT32 uiLog2PageSize,
+							   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							   const IMG_CHAR *pszAnnotation,
+							   IMG_PID uiPid,
+							   PMR **ppsPMRPtr,
+							   IMG_UINT32 ui32PDumpFlags,
+							   PVRSRV_MEMALLOCFLAGS_T *uiOutFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapCreatePMB
+@Description    Function calls an implementation-specific function pointer.
+                See @Ref PFN_CREATE_PMB "PFN_CREATE_PMB" for details.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapCreatePMB(PHYS_HEAP *psPhysHeap,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               const IMG_CHAR *pszAnnotation,
+                               PMB **ppsPMRPtr,
+                               RA_BASE_T *puiBase,
+                               RA_LENGTH_T *puiSize);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapDeviceNode
+@Description    Get pointer to the device node this heap belongs to.
+@Input          psPhysHeap          Pointer to physical heap.
+@Return         PPVRSRV_DEVICE_NODE Pointer to device node.
+*/ /**************************************************************************/
+PPVRSRV_DEVICE_NODE PhysHeapDeviceNode(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapInitByPVRLayer
+@Description    Is phys heap to be initialised in PVR layer?
+@Input          ePhysHeap           phys heap
+@Return         IMG_BOOL            return IMG_TRUE if yes
+*/ /**************************************************************************/
+IMG_BOOL PhysHeapInitByPVRLayer(PVRSRV_PHYS_HEAP ePhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapUserModeAlloc
+@Description    Is allocation from UM allowed?
+@Input          ePhysHeap           phys heap
+@Return         IMG_BOOL            return IMG_TRUE if yes
+*/ /**************************************************************************/
+IMG_BOOL PhysHeapUserModeAlloc(PVRSRV_PHYS_HEAP ePhysHeap);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+PVRSRV_ERROR PhysHeapPagesAllocGPV(PHYS_HEAP *psPhysHeap,
+                                   size_t uiSize,
+                                   PG_HANDLE *psMemHandle,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_UINT32 ui32OSid, IMG_PID uiPid);
+#endif
+
+PVRSRV_ERROR PhysHeapPagesAlloc(PHYS_HEAP *psPhysHeap,
+                                size_t uiSize,
+                                PG_HANDLE *psMemHandle,
+                                IMG_DEV_PHYADDR *psDevPAddr,
+                                IMG_PID uiPid);
+
+void PhysHeapPagesFree(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle);
+
+PVRSRV_ERROR PhysHeapPagesMap(PHYS_HEAP *psPhysHeap,
+                              PG_HANDLE *pshMemHandle,
+                              size_t uiSize,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              void **pvPtr);
+
+void PhysHeapPagesUnMap(PHYS_HEAP *psPhysHeap,
+                        PG_HANDLE *psMemHandle,
+                        void *pvPtr);
+
+PVRSRV_ERROR PhysHeapPagesClean(PHYS_HEAP *psPhysHeap,
+                                PG_HANDLE *pshMemHandle,
+                                IMG_UINT32 uiOffset,
+                                IMG_UINT32 uiLength);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapGetPageShift
+@Description    Get phys heap page shift.
+@Input          psPhysHeap   Pointer to physical heap.
+@Return         IMG_UINT32   Log2 page shift
+*/ /**************************************************************************/
+IMG_UINT32 PhysHeapGetPageShift(PHYS_HEAP *psPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysHeapFreeMemCheck
+@Description    Check a physheap has the required amount of free memory.
+
+@Input          psPhysHeap          Pointer to physical heap.
+@Input          ui64MinRequiredMem  The minimum free memory for success (bytes).
+@Output         pui64FreeMem        The free memory in the physical heap (bytes).
+
+@Return         PVRSRV_ERROR    If successful PVRSRV_OK else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+PVRSRV_ERROR PhysHeapFreeMemCheck(PHYS_HEAP *psPhysHeap,
+                                  IMG_UINT64 ui64MinRequiredMem,
+                                  IMG_UINT64 *pui64FreeMem);
+
+
+#endif /* PHYSHEAP_H */
diff --git a/drivers/gpu/drm/img-rogue/physheap_config.h b/drivers/gpu/drm/img-rogue/physheap_config.h
new file mode 100644
index 000000000000..eb1de937a9c6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physheap_config.h
@@ -0,0 +1,299 @@
+/*************************************************************************/ /*!
+@File           physheap_config.h
+@Title          Physical heap Config API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Physical heap configs are created in the system layer and
+                stored against each device node for use in the Services Server
+                common layer.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSHEAP_CONFIG_H
+#define PHYSHEAP_CONFIG_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv_memalloc_physheap.h"
+
+typedef IMG_UINT32 PHYS_HEAP_USAGE_FLAGS;
+
+/**
+ * ! IMPORTANT !
+ * If you update the PHYS_HEAP_USAGE_FLAGS definitions, you must update the
+ * g_asPhysHeapUsageFlagStrings structure within physheap.c and the
+ * PHYS_HEAP_USAGE_MASK.
+ */
+#define PHYS_HEAP_USAGE_CPU_LOCAL      (1U <<PVRSRV_PHYS_HEAP_CPU_LOCAL)
+#define PHYS_HEAP_USAGE_GPU_LOCAL      (1U <<PVRSRV_PHYS_HEAP_GPU_LOCAL)
+#define PHYS_HEAP_USAGE_GPU_PRIVATE    (1U <<PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+#define PHYS_HEAP_USAGE_EXTERNAL       (1U <<PVRSRV_PHYS_HEAP_EXTERNAL)
+#define PHYS_HEAP_USAGE_GPU_COHERENT   (1U <<PVRSRV_PHYS_HEAP_GPU_COHERENT)
+#define PHYS_HEAP_USAGE_GPU_SECURE     (1U <<PVRSRV_PHYS_HEAP_GPU_SECURE)
+#define PHYS_HEAP_USAGE_FW_CODE        (1U <<PVRSRV_PHYS_HEAP_FW_CODE)
+#define PHYS_HEAP_USAGE_FW_PRIV_DATA   (1U <<PVRSRV_PHYS_HEAP_FW_PRIV_DATA)
+#define PHYS_HEAP_USAGE_FW_PREMAP_PT   (1U <<PVRSRV_PHYS_HEAP_FW_PREMAP_PT)
+#define PHYS_HEAP_USAGE_FW_PREMAP      (1U <<PVRSRV_PHYS_HEAP_FW_PREMAP0)
+#define PHYS_HEAP_USAGE_WRAP           (1U <<PVRSRV_PHYS_HEAP_WRAP)
+#define PHYS_HEAP_USAGE_DISPLAY        (1U <<PVRSRV_PHYS_HEAP_DISPLAY)
+#define PHYS_HEAP_USAGE_DLM            (1U <<29)
+#define PHYS_HEAP_USAGE_FW_SHARED      (1U <<30)
+#define PHYS_HEAP_USAGE_FW_PRIVATE     (1U <<31)
+
+#define PHYS_HEAP_USAGE_MASK (PHYS_HEAP_USAGE_CPU_LOCAL    | \
+                              PHYS_HEAP_USAGE_GPU_LOCAL    | \
+                              PHYS_HEAP_USAGE_GPU_PRIVATE  | \
+                              PHYS_HEAP_USAGE_EXTERNAL     | \
+                              PHYS_HEAP_USAGE_GPU_COHERENT | \
+                              PHYS_HEAP_USAGE_GPU_SECURE   | \
+                              PHYS_HEAP_USAGE_FW_CODE      | \
+                              PHYS_HEAP_USAGE_FW_PRIV_DATA | \
+                              PHYS_HEAP_USAGE_FW_SHARED    | \
+                              PHYS_HEAP_USAGE_FW_PRIVATE   | \
+                              PHYS_HEAP_USAGE_FW_PREMAP    | \
+                              PHYS_HEAP_USAGE_FW_PREMAP_PT | \
+                              PHYS_HEAP_USAGE_WRAP         | \
+                              PHYS_HEAP_USAGE_DISPLAY      | \
+                              PHYS_HEAP_USAGE_DLM)
+
+#define FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM    PVRSRV_PHYS_HEAP_FW_CODE
+
+typedef void (*CpuPAddrToDevPAddr)(IMG_HANDLE hPrivData,
+                                   IMG_UINT32 ui32NumOfAddr,
+                                   IMG_DEV_PHYADDR *psDevPAddr,
+                                   IMG_CPU_PHYADDR *psCpuPAddr);
+
+typedef void (*DevPAddrToCpuPAddr)(IMG_HANDLE hPrivData,
+                                   IMG_UINT32 ui32NumOfAddr,
+                                   IMG_CPU_PHYADDR *psCpuPAddr,
+                                   IMG_DEV_PHYADDR *psDevPAddr);
+
+/*! Structure used to hold function pointers used for run-time physical address
+ * translation by Services. Gives flexibility to allow the CPU and GPU to see
+ * the same pool of physical RAM and different physical bus addresses.
+ * Both fields must be valid functions even if the conversion is simple.
+ */
+typedef struct _PHYS_HEAP_FUNCTIONS_
+{
+	/*! Translate CPU physical address to device physical address */
+	CpuPAddrToDevPAddr	pfnCpuPAddrToDevPAddr;
+	/*! Translate device physical address to CPU physical address */
+	DevPAddrToCpuPAddr	pfnDevPAddrToCpuPAddr;
+} PHYS_HEAP_FUNCTIONS;
+
+/*! Structure used to describe the IPA settings for a physical heap.
+ */
+typedef struct _IPA_CONFIG_
+{
+	/*! IPA Policy Default (0-7) */
+	IMG_UINT8             ui8IPAPolicyDefault;
+	/*! IPA Policy Mask (3-bits for this use-case - 0x7) */
+	IMG_UINT8             ui8IPAPolicyMask;
+	/*! IPA Policy mask shift (LSB position - policy value will be << by this amount */
+	IMG_UINT8             ui8IPAPolicyShift;
+	/*! IPA Policy reserved byte for alignment and slop reduction */
+	IMG_UINT8             ui8IPAPolicyReserved;
+} IPA_CONFIG;
+
+typedef struct _PHYS_HEAP_CONFIG_LMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+													 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_CPU_PHYADDR       sStartAddr;           /*!< CPU Physical base address of memory region */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Device physical base address of memory
+													 region as seen from the PoV of the GPU */
+	IMG_UINT64            uiSize;               /*!< Size of memory region in bytes */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+													 psMemFuncs */
+} PHYS_HEAP_CONFIG_LMA;
+
+typedef struct _PHYS_HEAP_CONFIG_UMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+													 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Optional Device physical base address of memory
+													 region as seen from the PoV of the GPU */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+													 psMemFuncs */
+} PHYS_HEAP_CONFIG_UMA;
+
+typedef struct _PHYS_HEAP_CONFIG_DLM_
+{
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_UINT32            ui32Log2PMBSize;      /*!< PMB (Physical Memory Block) Log 2 size in bytes
+	                                                 for DLM heap. */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CPU_PHYADDR       sStartAddr;           /*!< CPU Physical base address of memory region */
+	IMG_DEV_PHYADDR       sCardBase;            /*!< Device physical base address of memory
+	                                                 region as seen from the PoV of the GPU */
+	IMG_UINT64            uiSize;               /*!< Size of memory region in bytes */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+                                                     psMemFuncs */
+} PHYS_HEAP_CONFIG_DLM;
+
+typedef struct _PHYS_HEAP_CONFIG_IMA_
+{
+	IMG_CHAR*             pszPDumpMemspaceName; /*!< Name given to the heap's symbolic memory
+	                                                 space in a PDUMP enabled driver */
+	PHYS_HEAP_FUNCTIONS*  psMemFuncs;           /*!< Physical address translation functions */
+	IMG_CHAR*             pszHeapName;          /*!< Name given to the heap */
+	IMG_HANDLE            hPrivData;            /*!< System layer private data shared with
+	                                                 psMemFuncs */
+	IMG_UINT32            ui32PMBStartingMultiple; /*!< Multiple of PMB size defined in DLM heap
+	                                                    to be allocated on creation */
+	IMG_UINT32            uiDLMHeapIdx;         /*!< The index in the array of physheaps to a
+	                                                 DLM heap to import PMBs from. */
+} PHYS_HEAP_CONFIG_IMA;
+
+/*! Structure used to describe a physical Heap supported by a system. A
+ * system layer module can declare multiple physical heaps for different
+ * purposes. At a minimum a system must provide one physical heap tagged for
+ * PHYS_HEAP_USAGE_GPU_LOCAL use.
+ * A heap represents a discrete pool of physical memory and how it is managed,
+ * as well as associating other properties and address translation logic.
+ * The structure fields sStartAddr, sCardBase and uiSize must be given valid
+ * values for LMA and DMA physical heaps types.
+ */
+typedef struct _PHYS_HEAP_CONFIG_
+{
+	PHYS_HEAP_TYPE        eType;                /*!< Class of heap and PMR factory used */
+	PHYS_HEAP_USAGE_FLAGS ui32UsageFlags;       /*!< Supported uses flags, conveys the type of
+	                                                 buffers the physical heap can be used for */
+	IPA_CONFIG            sIPAConfig;           /*!< IPA configuration to be applied to all
+	                                                 requested physical addresses when physically
+	                                                 backed */
+	union {
+		PHYS_HEAP_CONFIG_LMA sLMA;
+		PHYS_HEAP_CONFIG_LMA sDMA;
+		PHYS_HEAP_CONFIG_UMA sUMA;
+		PHYS_HEAP_CONFIG_DLM sDLM;
+		PHYS_HEAP_CONFIG_IMA sIMA;
+#if defined(SUPPORT_WRAP_EXTMEMOBJECT)
+		PHYS_HEAP_CONFIG_UMA sWRAP;
+#endif
+	} uConfig;
+} PHYS_HEAP_CONFIG;
+
+/* Code throughout the driver sometimes relies on the fact that the backing for
+ * LMA and DMA heaps are the same struct, this check is not comprehensive as the structs
+ * could have changed but kept the same overall size, although this is unlikely.
+ */
+static_assert(sizeof(((PHYS_HEAP_CONFIG*)0)->uConfig.sLMA) == sizeof(((PHYS_HEAP_CONFIG*)0)->uConfig.sDMA),
+              "PHYS_HEAP_CONFIG sizeof sLMA != sDMA, Type has potentially changed");
+
+static INLINE IMG_UINT64 PhysHeapConfigGetSize(PHYS_HEAP_CONFIG *psConfig)
+{
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.uiSize;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no size member");
+		return 0;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.uiSize;
+#endif
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.uiSize;
+	case PHYS_HEAP_TYPE_UMA:
+		PVR_ASSERT(!"UMA Config has no size member");
+		return 0;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return 0;
+	}
+}
+
+static INLINE IMG_CPU_PHYADDR PhysHeapConfigGetStartAddr(PHYS_HEAP_CONFIG *psConfig)
+{
+	IMG_CPU_PHYADDR sUnsupportedPhyAddr = {0};
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.sStartAddr;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no StartAddr member");
+		return sUnsupportedPhyAddr;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.sStartAddr;
+#endif
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.sStartAddr;
+	case PHYS_HEAP_TYPE_UMA:
+		PVR_ASSERT(!"UMA Config has no StartAddr member");
+		return sUnsupportedPhyAddr;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return sUnsupportedPhyAddr;
+	}
+}
+
+static INLINE IMG_DEV_PHYADDR PhysHeapConfigGetCardBase(PHYS_HEAP_CONFIG *psConfig)
+{
+	IMG_DEV_PHYADDR sUnsupportedPhyAddr = {0};
+	switch (psConfig->eType)
+	{
+	case PHYS_HEAP_TYPE_LMA:
+		return psConfig->uConfig.sLMA.sCardBase;
+	case PHYS_HEAP_TYPE_IMA:
+		PVR_ASSERT(!"IMA Config has no CardBase member");
+		return sUnsupportedPhyAddr;
+#if defined(__KERNEL__)
+	case PHYS_HEAP_TYPE_DMA:
+		return psConfig->uConfig.sDMA.sCardBase;
+#endif
+	case PHYS_HEAP_TYPE_UMA:
+		return psConfig->uConfig.sUMA.sCardBase;
+	case PHYS_HEAP_TYPE_DLM:
+		return psConfig->uConfig.sDLM.sCardBase;
+	default:
+		PVR_ASSERT(!"Not Implemented for Config Type");
+		return sUnsupportedPhyAddr;
+	}
+}
+
+#define PHYS_HEAP_NAME_SIZE 24
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/physmem.c b/drivers/gpu/drm/img-rogue/physmem.c
new file mode 100644
index 000000000000..b04ced3638ad
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem.c
@@ -0,0 +1,843 @@
+/*************************************************************************/ /*!
+@File           physmem.c
+@Title          Physmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common entry point for creation of RAM backed PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "device.h"
+#include "physmem.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "pdump_physmem.h"
+#include "pdump_km.h"
+#include "pvr_ricommon.h"
+#include "allocmem.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#if defined(DEBUG)
+static IMG_UINT32 PMRAllocFail;
+
+#if defined(__linux__)
+#include <linux/moduleparam.h>
+
+module_param(PMRAllocFail, uint, 0644);
+MODULE_PARM_DESC(PMRAllocFail, "When number of PMR allocs reaches "
+				 "this value, it will fail (default value is 0 which "
+				 "means that alloc function will behave normally).");
+#endif /* defined(__linux__) */
+#endif /* defined(DEBUG) */
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#include "proc_stats.h"
+#endif
+
+/** Computes division using log2 of divisor. */
+#define LOG2_DIV(x, log2) ((x) >> (log2))
+
+/** Computes modulo of a power of 2. */
+#define LOG2_MOD(x, log2) ((x) & ((1 << (log2)) - 1))
+
+PVRSRV_ERROR DevPhysMemAlloc(PVRSRV_DEVICE_NODE	*psDevNode,
+                             IMG_UINT32 ui32MemSize,
+                             IMG_UINT32 ui32Log2Align,
+                             const IMG_UINT8 u8Value,
+                             IMG_BOOL bInitPage,
+#if defined(PDUMP)
+                             const IMG_CHAR *pszDevSpace,
+                             const IMG_CHAR *pszSymbolicAddress,
+                             IMG_HANDLE *phHandlePtr,
+#endif
+                             IMG_PID uiPid,
+                             IMG_HANDLE hMemHandle,
+                             IMG_DEV_PHYADDR *psDevPhysAddr)
+{
+	void *pvCpuVAddr;
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	IMG_CHAR szFilenameOut[PDUMP_PARAM_MAX_FILE_NAME];
+	PDUMP_FILEOFFSET_T uiOffsetOut;
+	IMG_UINT32 ui32PageSize;
+	IMG_UINT32 ui32PDumpMemSize = ui32MemSize;
+	PVRSRV_ERROR ePDumpError;
+#endif
+	PG_HANDLE *psMemHandle;
+	IMG_UINT64 uiMask;
+	IMG_DEV_PHYADDR sDevPhysAddr_int;
+
+	psMemHandle = hMemHandle;
+
+	/* Allocate the pages */
+	eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+	                            TRUNCATE_64BITS_TO_SIZE_T(ui32MemSize),
+	                            psMemHandle,
+	                            &sDevPhysAddr_int,
+	                            uiPid);
+	PVR_LOG_RETURN_IF_ERROR(eError, "pfnDevPxAlloc:1");
+
+	/* Check to see if the page allocator returned pages with our desired
+	 * alignment, which is not unlikely
+	 */
+	uiMask = IMG_PAGE2BYTES32(ui32Log2Align) - 1;
+	if (ui32Log2Align && (sDevPhysAddr_int.uiAddr & uiMask))
+	{
+		/* use over allocation instead */
+		PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+
+		ui32MemSize += (IMG_UINT32) uiMask;
+		eError = PhysHeapPagesAlloc(psDevNode->psMMUPhysHeap,
+		                            TRUNCATE_64BITS_TO_SIZE_T(ui32MemSize),
+		                            psMemHandle,
+		                            &sDevPhysAddr_int,
+		                            uiPid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "pfnDevPxAlloc:2");
+
+		sDevPhysAddr_int.uiAddr += uiMask;
+		sDevPhysAddr_int.uiAddr &= ~uiMask;
+	}
+	*psDevPhysAddr = sDevPhysAddr_int;
+
+#if defined(PDUMP)
+	ui32PageSize = ui32Log2Align? IMG_PAGE2BYTES32(ui32Log2Align) : OSGetPageSize();
+	eError = PDumpMalloc(psDevNode,
+	                     pszDevSpace,
+	                     pszSymbolicAddress,
+	                     ui32PDumpMemSize,
+	                     ui32PageSize,
+	                     IMG_FALSE,
+	                     0,
+	                     phHandlePtr,
+	                     PDUMP_NONE);
+	if (PVRSRV_OK != eError)
+	{
+		PDUMPCOMMENT(psDevNode, "Allocating pages failed");
+		*phHandlePtr = NULL;
+	}
+	ePDumpError = eError;
+#endif
+
+	if (bInitPage)
+	{
+		/*Map the page to the CPU VA space */
+		eError = PhysHeapPagesMap(psDevNode->psMMUPhysHeap,
+		                          psMemHandle,
+		                          ui32MemSize,
+		                          &sDevPhysAddr_int,
+		                          &pvCpuVAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_LOG_ERROR(eError, "DevPxMap");
+			PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+			return eError;
+		}
+
+		/*Fill the memory with given content */
+		OSDeviceMemSet(pvCpuVAddr, u8Value, ui32MemSize);
+
+		/*Map the page to the CPU VA space */
+		eError = PhysHeapPagesClean(psDevNode->psMMUPhysHeap,
+		                            psMemHandle,
+		                            0,
+		                            ui32MemSize);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_LOG_ERROR(eError, "DevPxClean");
+			PhysHeapPagesUnMap(psDevNode->psMMUPhysHeap, psMemHandle, pvCpuVAddr);
+			PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+			return eError;
+		}
+
+#if defined(PDUMP)
+		if (ePDumpError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+		{
+			/* PDumping of the page contents can be done in two ways
+			 * 1. Store the single byte init value to the .prm file
+			 *    and load the same value to the entire dummy page buffer
+			 *    This method requires lot of LDB's inserted into the out2.txt
+			 *
+			 * 2. Store the entire contents of the buffer to the .prm file
+			 *    and load them back.
+			 *    This only needs a single LDB instruction in the .prm file
+			 *    and chosen this method
+			 *    size of .prm file might go up but that's not huge at least
+			 *    for this allocation
+			 */
+			/* Write the buffer contents to the prm file */
+			eError = PDumpWriteParameterBlob(psDevNode,
+											 pvCpuVAddr,
+											 ui32PDumpMemSize,
+											 PDUMP_FLAGS_CONTINUOUS,
+											 szFilenameOut,
+											 sizeof(szFilenameOut),
+											 &uiOffsetOut);
+			if (PVRSRV_OK == eError)
+			{
+				/* Load the buffer back to the allocated memory when playing the pdump */
+				eError = PDumpPMRLDB(psDevNode,
+									 pszDevSpace,
+									 pszSymbolicAddress,
+									 0,
+									 ui32PDumpMemSize,
+									 szFilenameOut,
+									 uiOffsetOut,
+									 PDUMP_FLAGS_CONTINUOUS);
+				if (PVRSRV_OK != eError)
+				{
+					PDUMP_ERROR(psDevNode, eError, "Failed to write LDB statement to script file");
+					PVR_LOG_ERROR(eError, "PDumpPMRLDB");
+				}
+			}
+			else if (eError != PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+			{
+				PDUMP_ERROR(psDevNode, eError, "Failed to write device allocation to parameter file");
+				PVR_LOG_ERROR(eError, "PDumpWriteParameterBlob");
+			}
+			else
+			{
+				/* Else write to parameter file prevented under the flags and
+				 * current state of the driver so skip write to script and error IF.
+				 * This is expected e.g., if not in the capture range.
+				 */
+				eError = PVRSRV_OK;
+			}
+		}
+#endif
+
+		/* Unmap the page */
+		PhysHeapPagesUnMap(psDevNode->psMMUPhysHeap,
+		                   psMemHandle,
+		                   pvCpuVAddr);
+	}
+
+	return PVRSRV_OK;
+}
+
+void DevPhysMemFree(PVRSRV_DEVICE_NODE *psDevNode,
+#if defined(PDUMP)
+							IMG_HANDLE hPDUMPMemHandle,
+#endif
+							IMG_HANDLE	hMemHandle)
+{
+	PG_HANDLE *psMemHandle;
+
+	psMemHandle = hMemHandle;
+	PhysHeapPagesFree(psDevNode->psMMUPhysHeap, psMemHandle);
+#if defined(PDUMP)
+	if (NULL != hPDUMPMemHandle)
+	{
+		PDumpFree(psDevNode, hPDUMPMemHandle);
+	}
+#endif
+
+}
+
+PVRSRV_ERROR PhysMemValidateMappingTable(IMG_UINT32 ui32TotalNumVirtChunks,
+                                         IMG_UINT32 ui32IndexCount,
+                                         const IMG_UINT32 *pui32MappingTable)
+{
+	IMG_UINT8 *paui8TrackedIndices;
+	IMG_UINT32 ui32BytesToTrackIndicies;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Allocate memory for a bitmask to track indices.
+	 * We allocate 'n' bytes with 1 bit representing each index, to allow
+	 * us to check for any repeated entries in pui32MappingTable.
+	 */
+	ui32BytesToTrackIndicies = LOG2_DIV(ui32TotalNumVirtChunks, 3);
+	if (LOG2_MOD(ui32TotalNumVirtChunks, 3) != 0)
+	{
+		++ui32BytesToTrackIndicies;
+	}
+	paui8TrackedIndices = OSAllocZMem(ui32BytesToTrackIndicies);
+	if (paui8TrackedIndices == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	for (i = 0; i < ui32IndexCount; i++)
+	{
+		IMG_UINT32 ui32LogicalIndex = pui32MappingTable[i];
+
+		/* Check that index is within the bounds of the allocation */
+		if (ui32LogicalIndex >= ui32TotalNumVirtChunks)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Index %u is OOB",
+			         __func__,
+			         ui32LogicalIndex));
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+
+		/* Check that index is not repeated */
+		if (BIT_ISSET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3)))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Duplicate index found: %u",
+			         __func__,
+			         ui32LogicalIndex));
+
+			eError = PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY;
+			break;
+		}
+		BIT_SET(paui8TrackedIndices[LOG2_DIV(ui32LogicalIndex, 3)], LOG2_MOD(ui32LogicalIndex, 3));
+	}
+
+	OSFreeMem(paui8TrackedIndices);
+
+	return eError;
+}
+
+/* Checks the input parameters and adjusts them if possible and necessary */
+PVRSRV_ERROR PhysMemValidateParams(PVRSRV_DEVICE_NODE *psDevNode,
+                                   IMG_UINT32 ui32NumPhysChunks,
+                                   IMG_UINT32 ui32NumVirtChunks,
+                                   IMG_UINT32 *pui32MappingTable,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   IMG_UINT32 *puiLog2AllocPageSize,
+                                   IMG_DEVMEM_SIZE_T *puiSize)
+{
+	IMG_UINT32 uiLog2AllocPageSize = *puiLog2AllocPageSize;
+	IMG_UINT32 ui32PageSize = IMG_PAGE2BYTES32(uiLog2AllocPageSize);
+	IMG_DEVMEM_SIZE_T uiSize = *puiSize;
+	/* Sparse if we have different number of virtual and physical chunks plus
+	 * in general all allocations with more than one virtual chunk */
+	IMG_BOOL bIsSparse = (ui32NumVirtChunks != ui32NumPhysChunks ||
+			ui32NumVirtChunks > 1) ? IMG_TRUE : IMG_FALSE;
+
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags) &&
+	    PVRSRV_CHECK_PHYS_ALLOC_NOW(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid to specify both ON_DEMAND and NOW phys alloc flags: 0x%" IMG_UINT64_FMTSPECX, __func__, uiFlags));
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+
+	/* Sparse allocations must be backed immediately as the requested
+	 * pui32MappingTable is not retained in any structure if not immediately
+	 * actioned on allocation.
+	 */
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags) && bIsSparse)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid to specify ON_DEMAND for a sparse allocation: 0x%" IMG_UINT64_FMTSPECX, __func__, uiFlags));
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+
+	if (ui32NumVirtChunks == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of virtual chunks cannot be 0",
+		         __func__));
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Protect against invalid page sizes */
+	if ((ui32PageSize & psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask) == 0)
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR, "Page size of %u is invalid", ui32PageSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Range check of the alloc size */
+	if (!PMRValidateSize(uiSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				   "PMR size exceeds limit #Chunks: %u ChunkSz 0x%08X",
+				   ui32NumVirtChunks,
+				   IMG_PAGE2BYTES32(uiLog2AllocPageSize));
+		return PVRSRV_ERROR_PMR_TOO_LARGE;
+	}
+
+	/* Fail if requesting coherency on one side but uncached on the other */
+	if (PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) &&
+	    (PVRSRV_CHECK_GPU_UNCACHED(uiFlags) || PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for CPU coherency but specifying GPU uncached "
+				"Please use GPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+	    (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for GPU coherency but specifying CPU uncached "
+				"Please use CPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags) && PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Zero on Alloc and Poison on Alloc are mutually exclusive.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (bIsSparse)
+	{
+		/* For sparse we need correct parameters like a suitable page size....  */
+		if (OSGetPageShift() > uiLog2AllocPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid log2-contiguity for sparse allocation. "
+					"Requested %u, required minimum %zd",
+					__func__,
+					uiLog2AllocPageSize,
+					OSGetPageShift() ));
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		if (IMG_PAGES2BYTES64(ui32NumVirtChunks, uiLog2AllocPageSize) != uiSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Total alloc size (%#" IMG_UINT64_FMTSPECx ") "
+					 "is not equal to virtual chunks * chunk size "
+					 "(%#" IMG_UINT64_FMTSPECx ")",
+					 __func__,
+					 uiSize,
+					 IMG_PAGES2BYTES64(ui32NumVirtChunks, uiLog2AllocPageSize)));
+
+			return PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE;
+		}
+
+		if (ui32NumPhysChunks > ui32NumVirtChunks)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Number of physical chunks (%u) must not be greater "
+					"than number of virtual chunks (%u)",
+					__func__,
+					ui32NumPhysChunks,
+					ui32NumVirtChunks));
+
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+	else
+	{
+		/*
+		 * Silently round up alignment/pagesize if request was less that PAGE_SHIFT
+		 * because it would never be harmful for memory to be _more_ contiguous that
+		 * was desired.
+		 */
+		uiLog2AllocPageSize = OSGetPageShift() > uiLog2AllocPageSize ?
+				OSGetPageShift() : uiLog2AllocPageSize;
+
+		/* Same for total size */
+		uiSize = PVR_ALIGN(uiSize, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+	}
+
+	if ((uiSize & (IMG_PAGE2BYTES32(uiLog2AllocPageSize) - 1)) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Total size (%#" IMG_UINT64_FMTSPECx ") "
+		        "must be a multiple of the requested contiguity (%u)",
+		        __func__,
+		        uiSize,
+		        IMG_PAGE2BYTES32(uiLog2AllocPageSize)));
+		return PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE;
+	}
+
+	/* Parameter validation - Mapping table entries */
+	{
+		PVRSRV_ERROR eErr = PhysMemValidateMappingTable(ui32NumVirtChunks,
+		                                                ui32NumPhysChunks,
+		                                                pui32MappingTable);
+		PVR_RETURN_IF_ERROR(eErr);
+	}
+
+	*puiLog2AllocPageSize = uiLog2AllocPageSize;
+	*puiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _DevPhysHeapFromFlags(PVRSRV_MEMALLOCFLAGS_T uiFlags,
+										  PVRSRV_PHYS_HEAP *peDevPhysHeap,
+										  PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_PHYS_HEAP eHeap = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+
+	switch (eHeap)
+	{
+		case PVRSRV_PHYS_HEAP_FW_PREMAP0:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP1:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP2:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP3:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP4:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP5:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP6:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP7:
+		{
+			/* keep heap (with check) */
+			PVR_RETURN_IF_INVALID_PARAM(!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode));
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_LAST:
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+		default:
+		{
+			break;
+		}
+	}
+
+	*peDevPhysHeap = eHeap;
+
+	return PVRSRV_OK;
+}
+
+static INLINE void _PromoteToCpuCached(PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)) == 0)
+	{
+		/* We don't need to upgrade if we don't map into the CPU */
+		return;
+	}
+
+	/* Clear the existing CPU cache flags */
+	*puiFlags &= ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK);
+
+	/* Add CPU cached flags */
+	*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT;
+}
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR_direct(CONNECTION_DATA *psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2AllocPageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMRPtr,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	PVRSRV_PHYS_HEAP ePhysHeapIdx;
+	PVRSRV_MEMALLOCFLAGS_T uiPMRFlags = uiFlags;
+	uiPid = (psConnection != NULL) ? OSGetCurrentClientProcessIDKM() : uiPid;
+
+	/* This is where we would expect to validate the uiAnnotationLength parameter
+	   (to confirm it is sufficient to store the string in pszAnnotation plus a
+	   terminating NULL). However, we do not make reference to this value when
+	   we copy the string in PMRCreatePMR() - instead there we use strlcpy()
+	   to copy at most chars and ensure whatever is copied is null-terminated.
+	   The parameter is only used by the generated bridge code.
+	 */
+	PVR_UNREFERENCED_PARAMETER(uiAnnotationLength);
+
+	if (PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY)
+	{
+		_PromoteToCpuCached(&uiPMRFlags);
+	}
+
+	eError = PhysMemValidateParams(psDevNode,
+	                               ui32NumPhysChunks,
+	                               ui32NumVirtChunks,
+	                               pui32MappingTable,
+	                               uiFlags,
+	                               &uiLog2AllocPageSize,
+	                               &uiSize);
+	PVR_RETURN_IF_ERROR(eError);
+
+#if defined(SUPPORT_STATIC_IPA)
+#if !defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	/* Do not permit IPA PMR allocation flags to be passed through to the
+	 * new PMR.
+	 */
+	uiPMRFlags &= ~PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+#endif
+#endif
+
+#if defined(PDUMP)
+	eError = PDumpValidateUMFlags(ui32PDumpFlags);
+	PVR_RETURN_IF_ERROR(eError);
+#endif
+
+	for (i = 0; i < ui32NumPhysChunks; i++)
+	{
+		PVR_LOG_RETURN_IF_FALSE(pui32MappingTable[i] < ui32NumVirtChunks,
+		                        "Mapping table value exceeds ui32NumVirtChunks",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = _DevPhysHeapFromFlags(uiFlags, &ePhysHeapIdx, psDevNode);
+	PVR_RETURN_IF_ERROR(eError);
+
+	if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeapIdx = psDevNode->psDevConfig->eDefaultHeap;
+		PVRSRV_CHANGE_PHYS_HEAP_HINT(ePhysHeapIdx, uiPMRFlags);
+	}
+
+	if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_GPU_LOCAL)
+	{
+		if ((uiFlags & PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK) == 0)
+		{
+			ePhysHeapIdx = PVRSRV_PHYS_HEAP_GPU_PRIVATE;
+			PVRSRV_SET_PHYS_HEAP_HINT(GPU_PRIVATE, uiPMRFlags);
+			PVR_DPF((PVR_DBG_VERBOSE, "%s: Consider explicit use of GPU_PRIVATE for PMR %s."
+			        " Implicit conversion to GPU PRIVATE performed",
+			        __func__, pszAnnotation));
+		}
+		else if (PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+				 PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig))
+		{
+			ePhysHeapIdx = PVRSRV_PHYS_HEAP_GPU_COHERENT;
+			PVRSRV_SET_PHYS_HEAP_HINT(GPU_COHERENT, uiPMRFlags);
+		}
+	}
+	else if (ePhysHeapIdx == PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+	{
+		if (uiFlags & PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid flags for PMR %s!"
+			        " Client requested GPU_PRIVATE physical heap with CPU access flags.",
+			        __func__, pszAnnotation));
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+	}
+
+	if (NULL == psDevNode->apsPhysHeap[ePhysHeapIdx])
+	{
+		/* In case a heap hasn't been acquired for this type, return invalid heap error */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Requested allocation on device node (%p) from "
+		        "an invalid heap (HeapIndex=%d)",
+		        __func__, psDevNode, ePhysHeapIdx));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+#if defined(DEBUG)
+	if (PMRAllocFail > 0)
+	{
+		static IMG_UINT32 ui32AllocCount = 1;
+
+		if (ui32AllocCount < PMRAllocFail)
+		{
+			ui32AllocCount++;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s failed on %d allocation.",
+			         __func__, ui32AllocCount));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+#endif /* defined(DEBUG) */
+
+	/* If the driver is in an 'init' state all of the allocated memory
+	 * should be attributed to the driver (PID 1) rather than to the
+	 * process those allocations are made under. Same applies to the memory
+	 * allocated for the Firmware. */
+	if (psDevNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+	    PVRSRV_CHECK_FW_MAIN(uiFlags))
+	{
+		uiPid = PVR_SYS_ALLOC_PID;
+	}
+
+	/* ePhysHeapIdx and PhysHeap hint in uiPMRFlags match and provide the
+	 * intended PhysHeap to use at this point, but systems vary so the next
+	 * call may fallback (apsPhysHeap[]) or demote (OutOfMem) and not be the
+	 * heap that was intended, e.g. GPU_PRIVATE index may fallback to GPU_LOCAL,
+	 * GPU_LOCAL may demote to CPU_LOCAL.
+	 * On output uiPMRFlags show the PhysHeap finally used.
+	 */
+	eError = PhysHeapCreatePMR(psDevNode->apsPhysHeap[ePhysHeapIdx],
+							   psConnection,
+							   uiSize,
+							   ui32NumPhysChunks,
+							   ui32NumVirtChunks,
+							   pui32MappingTable,
+							   uiLog2AllocPageSize,
+							   uiPMRFlags,
+							   pszAnnotation,
+							   uiPid,
+							   ppsPMRPtr,
+							   ui32PDumpFlags,
+							   &uiPMRFlags);
+
+	if (puiPMRFlags != NULL)
+	{
+		*puiPMRFlags = uiPMRFlags;
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	if (eError != PVRSRV_OK)
+	{
+		PVRSRVStatsUpdateOOMStat(psConnection,
+		                          psDevNode,
+		                          PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+		                          OSGetCurrentClientProcessIDKM());
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR(CONNECTION_DATA *psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2AllocPageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMRPtr,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags)
+{
+	PVRSRV_PHYS_HEAP ePhysHeap = PVRSRV_GET_PHYS_HEAP_HINT(uiFlags);
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ePhysHeap < PVRSRV_PHYS_HEAP_LAST, "uiFlags");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(uiAnnotationLength != 0, "uiAnnotationLength");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+
+	if (ePhysHeap == PVRSRV_PHYS_HEAP_DEFAULT)
+	{
+		ePhysHeap = psDevNode->psDevConfig->eDefaultHeap;
+	}
+
+	if (!PhysHeapUserModeAlloc(ePhysHeap))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid phys heap hint: %d.", __func__, ePhysHeap));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = PhysmemNewRamBackedPMR_direct(psConnection,
+										  psDevNode,
+										  uiSize,
+										  ui32NumPhysChunks,
+										  ui32NumVirtChunks,
+										  pui32MappingTable,
+										  uiLog2AllocPageSize,
+										  uiFlags,
+										  uiAnnotationLength,
+										  pszAnnotation,
+										  uiPid,
+										  ppsPMRPtr,
+										  ui32PDumpFlags,
+										  puiPMRFlags);
+	if (eError == PVRSRV_OK)
+	{
+		/* Lock phys addresses if backing was allocated */
+		if (PVRSRV_CHECK_PHYS_ALLOC_NOW(uiFlags))
+		{
+			eError = PMRLockSysPhysAddresses(*ppsPMRPtr);
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVGetDefaultPhysicalHeapKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDevNode,
+                               PVRSRV_PHYS_HEAP *peHeap)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	*peHeap = psDevNode->psDevConfig->eDefaultHeap;
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfoKM(CONNECTION_DATA *psConnection,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           IMG_UINT32 ui32PhysHeapCount,
+                           PVRSRV_PHYS_HEAP *paePhysHeapID,
+                           PHYS_HEAP_MEM_STATS *paPhysHeapMemStats)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	return PhysHeapGetMemInfo(psDevNode,
+	                          ui32PhysHeapCount,
+	                          paePhysHeapID,
+	                          paPhysHeapMemStats);
+}
+
+/* 'Wrapper' function to call PMRImportPMR(), which first checks the PMR is
+ * for the current device. This avoids the need to do this in pmr.c, which
+ * would then need PVRSRV_DEVICE_NODE (defining this type in pmr.h causes a
+ * typedef redefinition issue).
+ */
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR
+PhysmemImportPMR(CONNECTION_DATA *psConnection,
+             PVRSRV_DEVICE_NODE *psDevNode,
+             PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (PMRGetExportDeviceNode(psPMRExport) != psDevNode)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR invalid for this device", __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	return PMRImportPMR(psPMRExport,
+	                    uiPassword,
+	                    uiSize,
+	                    uiLog2Contig,
+	                    ppsPMR);
+}
+#endif /* if defined(SUPPORT_INSECURE_EXPORT) */
diff --git a/drivers/gpu/drm/img-rogue/physmem.h b/drivers/gpu/drm/img-rogue/physmem.h
new file mode 100644
index 000000000000..9730aad07684
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem.h
@@ -0,0 +1,286 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for common entry point for creation of RAM backed PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_H
+#define SRVSRV_PHYSMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/* Valid values for TC_MEMORY_CONFIG configuration option */
+#define TC_MEMORY_LOCAL			(1)
+#define TC_MEMORY_HOST			(2)
+#define TC_MEMORY_HYBRID		(3)
+
+/* Valid values for the PLATO_MEMORY_CONFIG configuration option */
+#define PLATO_MEMORY_LOCAL		(1)
+#define PLATO_MEMORY_HOST		(2)
+#define PLATO_MEMORY_HYBRID		(3)
+
+/*************************************************************************/ /*!
+@Function       DevPhysMemAlloc
+@Description    Allocate memory from device specific heaps directly.
+@Input          psDevNode             device node to operate on
+@Input          ui32MemSize           Size of the memory to be allocated
+@Input          u8Value               Value to be initialised to.
+@Input          bInitPage             Flag to control initialisation
+@Input          pszDevSpace           PDUMP memory space in which the
+                                        allocation is to be done
+@Input          pszSymbolicAddress    Symbolic name of the allocation
+@Input          phHandlePtr           PDUMP handle to the allocation
+@Input          uiPid                 PID of the process owning the allocation
+                                        (or PVR_SYS_ALLOC_PID if the allocation
+                                        belongs to the driver)
+@Output         hMemHandle            Handle to the allocated memory
+@Output         psDevPhysAddr         Device Physical address of allocated
+                                        page
+@Return         PVRSRV_OK if the allocation is successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+DevPhysMemAlloc(PVRSRV_DEVICE_NODE *psDevNode,
+                IMG_UINT32 ui32MemSize,
+                IMG_UINT32 ui32Log2Align,
+                const IMG_UINT8 u8Value,
+                IMG_BOOL bInitPage,
+#if defined(PDUMP)
+                const IMG_CHAR *pszDevSpace,
+                const IMG_CHAR *pszSymbolicAddress,
+                IMG_HANDLE *phHandlePtr,
+#endif
+                IMG_PID uiPid,
+                IMG_HANDLE hMemHandle,
+                IMG_DEV_PHYADDR *psDevPhysAddr);
+
+/*************************************************************************/ /*!
+@Function       DevPhysMemFree
+@Description    Free memory to device specific heaps directly.
+@Input          psDevNode             device node to operate on
+@Input          hPDUMPMemHandle       Pdump handle to allocated memory
+@Input          hMemHandle            Devmem handle to allocated memory
+@Return         None
+*/ /**************************************************************************/
+void
+DevPhysMemFree(PVRSRV_DEVICE_NODE *psDevNode,
+#if defined(PDUMP)
+               IMG_HANDLE hPDUMPMemHandle,
+#endif
+               IMG_HANDLE hMemHandle);
+
+/*
+ * PhysmemNewRamBackedPMR
+ *
+ * This function will create a RAM backed PMR using the device specific
+ * callback, this allows control at a per-devicenode level to select the
+ * memory source thus supporting mixed UMA/LMA systems.
+ *
+ * The size must be a multiple of page size. The page size is specified in
+ * log2. It should be regarded as a minimum contiguity of which the
+ * resulting memory must be a multiple. It may be that this should be a fixed
+ * number. It may be that the allocation size needs to be a multiple of some
+ * coarser "page size" than that specified in the page size argument.
+ * For example, take an OS whose page granularity is a fixed 16kB, but the
+ * caller requests memory in page sizes of 4kB. The request can be satisfied
+ * if and only if the SIZE requested is a multiple of 16kB. If the arguments
+ * supplied are such that this OS cannot grant the request,
+ * PVRSRV_ERROR_INVALID_PARAMS will be returned.
+ *
+ * The caller should supply storage of a pointer. Upon successful return a
+ * PMR object will have been created and a pointer to it returned in the
+ * PMROut argument.
+ *
+ * A PMR successfully created should be destroyed with PhysmemUnrefPMR.
+ *
+ * Note that this function may cause memory allocations and on some operating
+ * systems this may cause scheduling events, so it is important that this
+ * function be called with interrupts enabled and in a context where
+ * scheduling events and memory allocations are permitted.
+ *
+ * The flags may be used by the implementation to change its behaviour if
+ * required. The flags will also be stored in the PMR as immutable metadata
+ * and returned to mmu_common when it asks for it.
+ *
+ * The PID specified is used to tie this allocation to the process context
+ * that the allocation is made on behalf of.
+ */
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR(CONNECTION_DATA * psConnection,
+                       PVRSRV_DEVICE_NODE *psDevNode,
+                       IMG_DEVMEM_SIZE_T uiSize,
+                       IMG_UINT32 ui32NumPhysChunks,
+                       IMG_UINT32 ui32NumVirtChunks,
+                       IMG_UINT32 *pui32MappingTable,
+                       IMG_UINT32 uiLog2PageSize,
+                       PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                       IMG_UINT32 uiAnnotationLength,
+                       const IMG_CHAR *pszAnnotation,
+                       IMG_PID uiPid,
+                       PMR **ppsPMROut,
+                       IMG_UINT32 ui32PDumpFlags,
+                       PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags);
+
+PVRSRV_ERROR
+PhysmemNewRamBackedPMR_direct(CONNECTION_DATA * psConnection,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  IMG_DEVMEM_SIZE_T uiSize,
+							  IMG_UINT32 ui32NumPhysChunks,
+							  IMG_UINT32 ui32NumVirtChunks,
+							  IMG_UINT32 *pui32MappingTable,
+							  IMG_UINT32 uiLog2PageSize,
+							  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+							  IMG_UINT32 uiAnnotationLength,
+							  const IMG_CHAR *pszAnnotation,
+							  IMG_PID uiPid,
+							  PMR **ppsPMROut,
+							  IMG_UINT32 ui32PDumpFlags,
+							  PVRSRV_MEMALLOCFLAGS_T *puiPMRFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysmemImportPMR
+@Description    Import PMR a previously exported PMR
+@Input          psPMRExport           The exported PMR token
+@Input          uiPassword            Authorisation password
+                                      for the PMR being imported
+@Input          uiSize                Size of the PMR being imported
+                                      (for verification)
+@Input          uiLog2Contig          Log2 continuity of the PMR being
+                                      imported (for verification)
+@Output         ppsPMR                The imported PMR
+@Return         PVRSRV_ERROR_PMR_NOT_PERMITTED if not for the same device
+                PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR if password incorrect
+                PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES if size or contiguity incorrect
+                PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemImportPMR(CONNECTION_DATA *psConnection,
+                 PVRSRV_DEVICE_NODE *psDevNode,
+                 PMR_EXPORT *psPMRExport,
+                 PMR_PASSWORD_T uiPassword,
+                 PMR_SIZE_T uiSize,
+                 PMR_LOG2ALIGN_T uiLog2Contig,
+                 PMR **ppsPMR);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDefaultPhysicalHeapKM
+@Description    For the specified device, get the physical heap used for
+                allocations when the PVRSRV_PHYS_HEAP_DEFAULT
+                physical heap hint is set in memalloc flags.
+@Output         peHeap                 Default Heap return value
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVGetDefaultPhysicalHeapKM(CONNECTION_DATA *psConnection,
+                               PVRSRV_DEVICE_NODE *psDevNode,
+                               PVRSRV_PHYS_HEAP *peHeap);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVPhysHeapGetMemInfoKM
+@Description    Get the memory usage statistics for a given physical heap ID
+@Input          ui32PhysHeapCount      Physical Heap count
+@Input          paePhysHeapID          Array of Physical Heap ID's
+@Output         paPhysHeapMemStats     Buffer to hold the memory statistics
+@Return         PVRSRV_OK if successful
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVPhysHeapGetMemInfoKM(CONNECTION_DATA *psConnection,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           IMG_UINT32 ui32PhysHeapCount,
+                           PVRSRV_PHYS_HEAP *paePhysHeapID,
+                           PHYS_HEAP_MEM_STATS *paPhysHeapMemStats);
+
+/*************************************************************************/ /*!
+@Function       PhysMemValidateMappingTable
+@Description    Checks the PMR mapping table provided is valid (ie has
+                no entries with an index value outside the valid range
+                of indices for the allocation and has no repeated indices)
+
+@Input          ui32TotalNumVirtChunks Total number of virtual chunks
+                                       the allocation has.
+@Input          ui32IndexCount         Number of entries in the mapping
+                                       table.
+@Input          pui32MappingTable      Mapping Table.
+@Return         PVRSRV_OK              if parameters are valid.
+                PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY if mapping table
+                                       contains an index out of range or a
+                                       repeated index
+                PVRSRV_ERROR_OUT_OF_MEMORY if unable to allocate memory
+                                       for the index tracking table
+                                       (used internally in this function)
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysMemValidateMappingTable(IMG_UINT32 ui32TotalNumVirtChunks,
+                            IMG_UINT32 ui32IndexCount,
+                            const IMG_UINT32 *pui32MappingTable);
+
+/*************************************************************************/ /*!
+@Function       PhysMemValidateParams
+@Description    Checks the PMR creation parameters and adjusts them
+                if possible and necessary
+
+@Input          psDevNode              The associated device node.
+@Input          ui32NumPhysChunks      Number of physical chunks.
+@Input          ui32NumVirtChunks      Number of virtual chunks.
+@Input          pui32MappingTable      Mapping Table.
+@Input          uiFlags                Allocation flags.
+@Inout          puiLog2AllocPageSize   Log2 of allocation page size.
+                                       May be adjusted.
+@Inout          puiSize                Size of the allocation.
+                                       May be adjusted.
+@Return         PVRSRV_OK if parameters are valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysMemValidateParams(PVRSRV_DEVICE_NODE *psDevNode,
+                      IMG_UINT32 ui32NumPhysChunks,
+                      IMG_UINT32 ui32NumVirtChunks,
+                      IMG_UINT32 *pui32MappingTable,
+                      PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                      IMG_UINT32 *puiLog2AllocPageSize,
+                      IMG_DEVMEM_SIZE_T *puiSize);
+
+#endif /* SRVSRV_PHYSMEM_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_dlm.h b/drivers/gpu/drm/img-rogue/physmem_dlm.h
new file mode 100644
index 000000000000..7d18fd781f2f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_dlm.h
@@ -0,0 +1,88 @@
+/*************************************************************************/ /*!
+@File           physmem_dlm.h
+@Title          Dedicated Local Memory allocator for Physical Memory Blocks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for dedicated local memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_DLM_H
+#define PHYSMEM_DLM_H
+
+#include "img_types.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+
+/* PMBDestroy
+ *
+ * Destroys a given PMB used to represent a block of memory
+ * obtained from a DLM heap.
+ */
+void
+PMBDestroy(PMB *psPMB);
+
+/* PMBGetAnnotation
+ *
+ * Obtain the annotation of the allocation represented
+ * by this PMB
+ */
+const IMG_CHAR *
+PMBGetAnnotation(PMB *psPMB);
+
+/* DLM */
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapDLM
+@Description    Create and register new DLM heap.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Input          pszLabel     Debug identifier label
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapDLM(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* #ifndef PHYSMEM_DLM_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_dmabuf.c b/drivers/gpu/drm/img-rogue/physmem_dmabuf.c
new file mode 100644
index 000000000000..e40094b8b4e0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_dmabuf.c
@@ -0,0 +1,2206 @@
+/*************************************************************************/ /*!
+@File           physmem_dmabuf.c
+@Title          dmabuf memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for dmabuf memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include "physmem_dmabuf.h"
+#include "physmem_dmabuf_internal.h"
+#include "physmem.h"
+#include "pvrsrv.h"
+#include "pmr.h"
+
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_prime.h>
+#else
+#include <drm/drmP.h>
+#endif
+#include <drm/drm_gem.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pmr_impl.h"
+#include "pmr_env.h"
+#include "hash.h"
+#include "private_data.h"
+#include "module_common.h"
+#include "pvr_ion_stats.h"
+#include "cache_km.h"
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+typedef struct _PMR_DMA_BUF_WRAPPER_
+{
+	PMR *psPMR;
+
+	/*
+	 * Set if the PMR has been exported via GEM. This field should only
+	 * be accessed using smp_store_release and smp_load_acquire.
+	 * In general, the field is not safe to access, as the wrapper doesn't
+	 * hold a reference on the GEM object.
+	 */
+	struct drm_gem_object *psObj;
+
+	struct dma_resv sDmaResv;
+
+	/* kernel mapping */
+	IMG_UINT32 uiKernelMappingRefCnt;
+	IMG_HANDLE hKernelMappingHandle;
+	void *pvKernelMappingAddr;
+	PMR_SIZE_T uiKernelMappingLen;
+	POS_LOCK hKernelMappingLock;
+
+	/* device mapping */
+	IMG_UINT32 uiDeviceMappingRefCnt;
+	POS_LOCK hDeviceMappingLock;
+	struct sg_table *psTable;
+} PMR_DMA_BUF_WRAPPER;
+
+typedef struct _PMR_DMA_BUF_GEM_OBJ {
+	struct drm_gem_object sBase;
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+} PMR_DMA_BUF_GEM_OBJ;
+
+#define TO_PMR_DMA_BUF_GEM_OBJ(psObj) IMG_CONTAINER_OF((psObj), PMR_DMA_BUF_GEM_OBJ, sBase)
+
+static PVRSRV_ERROR _PMRKMap(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                             size_t uiSize,
+                             void **pvAddr)
+{
+	IMG_HANDLE hKernelMapping;
+	void *pvAddrOut = NULL;
+	size_t uiLengthOut = 0;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt++ == 0)
+	{
+		PVR_ASSERT(psPMRWrapper->hKernelMappingHandle == NULL);
+		PVR_ASSERT(psPMRWrapper->pvKernelMappingAddr == NULL);
+
+		if (PMR_IsSparse(psPMRWrapper->psPMR))
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMRWrapper->psPMR,
+			                                           0,
+			                                           uiSize,
+			                                           &pvAddrOut,
+			                                           &uiLengthOut,
+			                                           &hKernelMapping);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireSparseKernelMappingData",
+			                      ErrReleaseLock);
+		}
+		else
+		{
+			eError = PMRAcquireKernelMappingData(psPMRWrapper->psPMR,
+			                                     0,
+			                                     uiSize,
+			                                     &pvAddrOut,
+			                                     &uiLengthOut,
+			                                     &hKernelMapping);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData",
+			                      ErrReleaseLock);
+		}
+
+		psPMRWrapper->hKernelMappingHandle = hKernelMapping;
+		psPMRWrapper->pvKernelMappingAddr = pvAddrOut;
+		psPMRWrapper->uiKernelMappingLen = uiLengthOut;
+	}
+
+	*pvAddr = psPMRWrapper->pvKernelMappingAddr;
+	goto ExitUnlock;
+
+ErrReleaseLock:
+	psPMRWrapper->uiKernelMappingRefCnt--;
+
+ExitUnlock:
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+	return eError;
+}
+
+static void _PMRKUnmap(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (--psPMRWrapper->uiKernelMappingRefCnt == 0)
+	{
+		PVRSRV_ERROR eError;
+
+		PVR_ASSERT(psPMRWrapper->hKernelMappingHandle != NULL);
+		PVR_ASSERT(psPMRWrapper->pvKernelMappingAddr != NULL);
+
+		eError = PMRReleaseKernelMappingData(psPMRWrapper->psPMR,
+		                                     psPMRWrapper->hKernelMappingHandle);
+		PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+		psPMRWrapper->hKernelMappingHandle = NULL;
+		psPMRWrapper->pvKernelMappingAddr = NULL;
+		psPMRWrapper->uiKernelMappingLen = 0;
+	}
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+}
+
+static PVRSRV_ERROR _PMRInvalidateCache(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt == 0) {
+		OSLockRelease(psPMRWrapper->hKernelMappingLock);
+		return PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+	}
+
+	eError = CacheOpValExec(psPMRWrapper->psPMR,
+	                        (IMG_UINT64) (uintptr_t) psPMRWrapper->pvKernelMappingAddr,
+	                        0,
+	                        psPMRWrapper->uiKernelMappingLen,
+	                        PVRSRV_CACHE_OP_INVALIDATE);
+	PVR_LOG_IF_ERROR(eError, "CacheOpValExec");
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR _PMRCleanCache(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hKernelMappingLock);
+
+	if (psPMRWrapper->uiKernelMappingRefCnt == 0) {
+		OSLockRelease(psPMRWrapper->hKernelMappingLock);
+		return PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+	}
+
+	eError = CacheOpValExec(psPMRWrapper->psPMR,
+	                        (IMG_UINT64) (uintptr_t) psPMRWrapper->pvKernelMappingAddr,
+	                        0,
+	                        psPMRWrapper->uiKernelMappingLen,
+	                        PVRSRV_CACHE_OP_FLUSH);
+	PVR_LOG_IF_ERROR(eError, "CacheOpValExec");
+
+	OSLockRelease(psPMRWrapper->hKernelMappingLock);
+
+	return eError;
+}
+
+/*
+ * dma_buf_ops common code
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ * The following common functions are used by both the dma_buf and GEM
+ * callbacks.
+ */
+
+static int PVRDmaBufOpsAttachCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+							  struct dma_buf_attachment *psAttachment)
+{
+	PMR *psPMR = psPMRWrapper->psPMR;
+
+	if (PMR_GetType(psPMR) == PMR_TYPE_DMABUF)
+	{
+		// don't support exporting PMRs that are itself created from imported
+		// DmaBufs
+		PVR_DPF((PVR_DBG_ERROR, "exporting PMRs of type DMABUF not supported"));
+
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static struct sg_table *PVRDmaBufOpsMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                              struct dma_buf_attachment *psAttachment,
+                                              enum dma_data_direction eDirection)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMR = psPMRWrapper->psPMR;
+	IMG_UINT uiNents;
+	IMG_DEVMEM_SIZE_T uiPhysSize, uiVirtSize;
+	IMG_UINT32 uiNumVirtPages;
+	void *pvPAddrData = NULL;
+	IMG_DEV_PHYADDR asPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC], *psPAddr = asPAddr;
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC], *pbValid = abValid;
+	IMG_UINT32 i;
+	IMG_DEV_PHYADDR sPAddrPrev, sPAddrCurr;
+	struct sg_table *psTable;
+	struct scatterlist *psSg;
+	int iRet = 0;
+	IMG_UINT32 uiDevPageShift, uiDevPageSize;
+
+	OSLockAcquire(psPMRWrapper->hDeviceMappingLock);
+
+	psPMRWrapper->uiDeviceMappingRefCnt++;
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt > 1)
+	{
+		goto OkUnlock;
+	}
+
+	PVR_ASSERT(psPMRWrapper->psTable == NULL);
+
+	uiDevPageShift = PMR_GetLog2Contiguity(psPMR);
+	uiDevPageSize = 1u << uiDevPageShift;
+	uiVirtSize = PMR_LogicalSize(psPMR);
+	uiPhysSize = PMR_PhysicalSize(psPMR);
+	if (uiPhysSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "invalid PMR size"));
+		iRet = PVRSRVToNativeError(PVRSRV_ERROR_BAD_MAPPING);
+		goto ErrUnlockMapping;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): mapping pmr: 0x%p@0x%" IMG_UINT64_FMTSPECx
+	         " from heap: \"%s\" with page size: 0x%x", __func__, psPMR,
+	         uiPhysSize, PhysHeapName(PMR_PhysHeap(psPMR)), uiDevPageSize));
+
+	uiNumVirtPages = uiVirtSize >> uiDevPageShift;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMRLockSysPhysAddresses");
+		iRet = PVRSRVToNativeError(eError);
+		goto ErrUnlockMapping;
+	}
+
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	psTable = OSAllocZMem(sizeof(*psTable));
+	if (psTable == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OSAllocMem.1() failed"));
+		iRet = -ENOMEM;
+		goto ErrUnlockPhysAddresses;
+	}
+
+	if (uiNumVirtPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		pvPAddrData = OSAllocMem(uiNumVirtPages * (sizeof(*psPAddr) + sizeof(*pbValid)));
+		if (pvPAddrData == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "OSAllocMem.2() failed"));
+			iRet = -ENOMEM;
+			goto ErrFreeTable;
+		}
+
+		psPAddr = IMG_OFFSET_ADDR(pvPAddrData, 0);
+		pbValid = IMG_OFFSET_ADDR(pvPAddrData, uiNumVirtPages * sizeof(*psPAddr));
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+	                         uiDevPageShift,
+	                         uiNumVirtPages,
+	                         0,
+	                         psPAddr,
+	                         pbValid,
+	                         DEVICE_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "PMR_DevPhysAddr");
+		iRet = PVRSRVToNativeError(eError);
+		goto ErrFreePAddrData;
+	}
+
+	/* Calculate how many contiguous regions there are in the PMR. This
+	 * value will be used to allocate one scatter list for every region. */
+
+	/* Find first valid physical address. */
+	for (i = 0; i < uiNumVirtPages && !pbValid[i]; i++);
+
+	sPAddrPrev = psPAddr[i];
+	uiNents = 1;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): %03u paddr: 0x%" IMG_UINT64_FMTSPECx,
+	         __func__, i, sPAddrPrev.uiAddr));
+
+	/* Find the rest of the addresses. */
+	for (i = i + 1; i < uiNumVirtPages; i++)
+	{
+		if (!pbValid[i])
+		{
+			continue;
+		}
+
+		sPAddrCurr = psPAddr[i];
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s(): %03u paddr: 0x%" IMG_UINT64_FMTSPECx
+		         ", pprev: 0x%" IMG_UINT64_FMTSPECx ", valid: %u", __func__,
+		         i, sPAddrCurr.uiAddr, sPAddrPrev.uiAddr, pbValid[i]));
+
+		if (sPAddrCurr.uiAddr != (sPAddrPrev.uiAddr + uiDevPageSize))
+		{
+			uiNents++;
+		}
+
+		sPAddrPrev = sPAddrCurr;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): found %u contiguous regions", __func__,
+	         uiNents));
+
+	/* Allocate scatter lists from all of the contiguous regions. */
+
+	iRet = sg_alloc_table(psTable, uiNents, GFP_KERNEL);
+	if (iRet != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "sg_alloc_table() failed with error %d", iRet));
+		goto ErrFreePAddrData;
+	}
+
+	/* Fill in all of the physical addresses and sizes for all of the
+	 * contiguous regions that were calculated. */
+
+	for (i = 0; i < uiNumVirtPages && !pbValid[i]; i++);
+
+	psSg = psTable->sgl;
+	sPAddrPrev = psPAddr[i];
+
+	sg_dma_address(psSg) = sPAddrPrev.uiAddr;
+	sg_dma_len(psSg) = uiDevPageSize;
+
+	for (i = i + 1; i < uiNumVirtPages; i++)
+	{
+		if (!pbValid[i])
+		{
+			continue;
+		}
+
+		sPAddrCurr = psPAddr[i];
+
+		if (sPAddrCurr.uiAddr != (sPAddrPrev.uiAddr + uiDevPageSize))
+		{
+			psSg = sg_next(psSg);
+			PVR_ASSERT(psSg != NULL);
+
+			sg_dma_address(psSg) = sPAddrCurr.uiAddr;
+			sg_dma_len(psSg) = uiDevPageSize;
+		}
+		else
+		{
+			sg_dma_len(psSg) += uiDevPageSize;
+		}
+
+		sPAddrPrev = sPAddrCurr;
+	}
+
+	if (pvPAddrData != NULL)
+	{
+		OSFreeMem(pvPAddrData);
+	}
+
+	psPMRWrapper->psTable = psTable;
+
+OkUnlock:
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+
+	return psPMRWrapper->psTable;
+
+ErrFreePAddrData:
+	if (pvPAddrData != NULL)
+	{
+		OSFreeMem(pvPAddrData);
+	}
+ErrFreeTable:
+	OSFreeMem(psTable);
+ErrUnlockPhysAddresses:
+	{
+		PVRSRV_ERROR eError2 = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_LOG_IF_ERROR(eError2, "PMRUnlockSysPhysAddresses");
+	}
+ErrUnlockMapping:
+	psPMRWrapper->uiDeviceMappingRefCnt--;
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+
+	return ERR_PTR(iRet);
+}
+
+static void PVRDmaBufOpsUnmapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                              struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psPMRWrapper->hDeviceMappingLock);
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "reference count on mapping already 0"));
+		goto ErrUnlock;
+	}
+
+	psPMRWrapper->uiDeviceMappingRefCnt--;
+
+	if (psPMRWrapper->uiDeviceMappingRefCnt > 0)
+	{
+		goto ErrUnlock;
+	}
+
+	dma_unmap_sg(psAttachment->dev, psTable->sgl, psTable->nents, eDirection);
+	sg_free_table(psTable);
+
+	eError = PMRUnlockSysPhysAddresses(psPMRWrapper->psPMR);
+	PVR_LOG_IF_ERROR(eError, "PMRUnlockSysPhysAddresses");
+
+	OSFreeMem(psPMRWrapper->psTable);
+	psPMRWrapper->psTable = NULL;
+
+ErrUnlock:
+	OSLockRelease(psPMRWrapper->hDeviceMappingLock);
+}
+
+static int PVRDmaBufOpsBeginCpuAccessCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                      enum dma_data_direction eDirection)
+{
+	if (PVRSRV_CHECK_CPU_CACHED(PMR_Flags(psPMRWrapper->psPMR)))
+	{
+		PVRSRV_ERROR eError = _PMRInvalidateCache(psPMRWrapper);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PMRInvalidateCache");
+			return OSPVRSRVToNativeError(eError);
+		}
+	}
+
+	return 0;
+}
+
+static int PVRDmaBufOpsEndCpuAccessCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper,
+                                    enum dma_data_direction eDirection)
+{
+	if (PVRSRV_CHECK_CPU_CACHED(PMR_Flags(psPMRWrapper->psPMR)))
+	{
+		PVRSRV_ERROR eError = _PMRCleanCache(psPMRWrapper);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PMRCleanCache");
+			return OSPVRSRVToNativeError(eError);
+		}
+	}
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, unsigned long uiPageNum)
+{
+	return NULL;
+}
+
+static void PVRDmaBufOpsKUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, unsigned long uiPageNum, void *pvMem)
+{
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	void *pvAddrOut = NULL;
+	PVRSRV_ERROR eError;
+
+	eError = _PMRKMap(psPMRWrapper, 0, &pvAddrOut);
+
+	return eError == PVRSRV_OK ? pvAddrOut : NULL;
+}
+#else
+static int PVRDmaBufOpsVMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct iosys_map *psMap)
+{
+	void *pvAddrOut = NULL;
+	PVRSRV_ERROR eError;
+
+	eError = _PMRKMap(psPMRWrapper, 0, &pvAddrOut);
+
+	if (eError != PVRSRV_OK)
+	{
+		return OSPVRSRVToNativeError(eError);
+	}
+
+	if (PhysHeapGetType(PMR_PhysHeap(psPMRWrapper->psPMR)) == PHYS_HEAP_TYPE_UMA)
+	{
+		iosys_map_set_vaddr(psMap, pvAddrOut);
+	}
+	else
+	{
+		iosys_map_set_vaddr_iomem(psMap, pvAddrOut);
+	}
+
+	return 0;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, void *pvAddr)
+{
+	_PMRKUnmap(psPMRWrapper);
+}
+#else
+static void PVRDmaBufOpsVUnMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct iosys_map *psMap)
+{
+	_PMRKUnmap(psPMRWrapper);
+
+	iosys_map_clear(psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMapCommon(PMR_DMA_BUF_WRAPPER *psPMRWrapper, struct vm_area_struct *psVMA)
+{
+	PMR *psPMR = psPMRWrapper->psPMR;
+	PVRSRV_MEMALLOCFLAGS_T uiProtFlags =
+	    (BITMASK_HAS(psVMA->vm_flags, VM_READ) ? PVRSRV_MEMALLOCFLAG_CPU_READABLE : 0) |
+	    (BITMASK_HAS(psVMA->vm_flags, VM_WRITE) ? PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0);
+	PVRSRV_ERROR eError;
+
+	/* Forcibly clear the VM_MAYWRITE flag as this is inherited from the
+	 * kernel mmap code and we do not want to produce a potentially writable
+	 * mapping from a read-only mapping.
+	 */
+	if (!BITMASK_HAS(psVMA->vm_flags, VM_WRITE))
+	{
+		pvr_vm_flags_clear(psVMA, VM_MAYWRITE);
+	}
+
+	eError = PMRMMapPMR(psPMR, psVMA, uiProtFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_IF_ERROR(eError, "PMRMMapPMR");
+		return OSPVRSRVToNativeError(eError);
+	}
+
+	return 0;
+}
+
+/* end of dma_buf_ops common code*/
+
+/*
+ * dma_buf_ops (non-GEM)
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ */
+
+static int PVRDmaBufOpsAttach(struct dma_buf *psDmaBuf,
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+							  struct device *psDev,
+#endif
+							  struct dma_buf_attachment *psAttachment)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsAttachCommon(psPMRWrapper, psAttachment);
+}
+
+static struct sg_table *PVRDmaBufOpsMap(struct dma_buf_attachment *psAttachment,
+                                        enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psAttachment->dmabuf->priv;
+
+	return PVRDmaBufOpsMapCommon(psPMRWrapper, psAttachment, eDirection);
+}
+
+static void PVRDmaBufOpsUnmap(struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psAttachment->dmabuf->priv;
+
+	PVRDmaBufOpsUnmapCommon(psPMRWrapper, psAttachment, psTable, eDirection);
+}
+
+static void PVRDmaBufOpsRelease(struct dma_buf *psDmaBuf)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+	PMR *psPMR = psPMRWrapper->psPMR;
+
+	PMRUnrefPMR(psPMR);
+}
+
+static int PVRDmaBufOpsBeginCpuAccess(struct dma_buf *psDmaBuf,
+                                      enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsBeginCpuAccessCommon(psPMRWrapper, eDirection);
+}
+
+static int PVRDmaBufOpsEndCpuAccess(struct dma_buf *psDmaBuf,
+                                    enum dma_data_direction eDirection)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+	int iErr;
+
+	iErr = PVRDmaBufOpsEndCpuAccessCommon(psPMRWrapper, eDirection);
+
+	return iErr;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMap(struct dma_buf *psDmaBuf, unsigned long uiPageNum)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsKMapCommon(psPMRWrapper, uiPageNum);
+}
+
+static void PVRDmaBufOpsKUnMap(struct dma_buf *psDmaBuf, unsigned long uiPageNum, void *pvMem)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsKUnMapCommon(psPMRWrapper, uiPageNum, pvMem);
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMap(struct dma_buf *psDmaBuf)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper);
+}
+#else
+static int PVRDmaBufOpsVMap(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMap(struct dma_buf *psDmaBuf, void *pvAddr)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, pvAddr);
+}
+#else
+static void PVRDmaBufOpsVUnMap(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMap(struct dma_buf *psDmaBuf, struct vm_area_struct *psVMA)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): psDmaBuf = %px, psPMR = %px", __func__,
+	         psDmaBuf, psPMRWrapper->psPMR));
+
+	return PVRDmaBufOpsMMapCommon(psPMRWrapper, psVMA);
+}
+
+static const struct dma_buf_ops sPVRDmaBufOps =
+{
+	.attach        = PVRDmaBufOpsAttach,
+	.map_dma_buf   = PVRDmaBufOpsMap,
+	.unmap_dma_buf = PVRDmaBufOpsUnmap,
+	.release       = PVRDmaBufOpsRelease,
+	.begin_cpu_access = PVRDmaBufOpsBeginCpuAccess,
+	.end_cpu_access   = PVRDmaBufOpsEndCpuAccess,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+	.map_atomic    = PVRDmaBufOpsKMap,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	.map           = PVRDmaBufOpsKMap,
+	.unmap         = PVRDmaBufOpsKUnMap,
+#endif
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.kmap_atomic   = PVRDmaBufOpsKMap,
+	.kmap          = PVRDmaBufOpsKMap,
+	.kunmap        = PVRDmaBufOpsKUnMap,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.mmap          = PVRDmaBufOpsMMap,
+
+	.vmap          = PVRDmaBufOpsVMap,
+	.vunmap        = PVRDmaBufOpsVUnMap,
+};
+
+/* end of dma_buf_ops (non-GEM) */
+
+/*
+ * dma_buf_ops (GEM)
+ *
+ * Implementation of below callbacks adds the ability to export DmaBufs to other
+ * drivers.
+ */
+
+static int PVRDmaBufOpsAttachGEM(struct dma_buf *psDmaBuf,
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+							  struct device *psDev,
+#endif
+							  struct dma_buf_attachment *psAttachment)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsAttachCommon(psPMRWrapper, psAttachment);
+}
+
+static struct sg_table *PVRDmaBufOpsMapGEM(struct dma_buf_attachment *psAttachment,
+                                        enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psAttachment->dmabuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsMapCommon(psPMRWrapper, psAttachment, eDirection);
+}
+
+static void PVRDmaBufOpsUnmapGEM(struct dma_buf_attachment *psAttachment,
+                              struct sg_table *psTable,
+                              enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psAttachment->dmabuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsUnmapCommon(psPMRWrapper, psAttachment, psTable, eDirection);
+}
+
+static int PVRDmaBufOpsBeginCpuAccessGEM(struct dma_buf *psDmaBuf,
+                                      enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsBeginCpuAccessCommon(psPMRWrapper, eDirection);
+}
+
+static int PVRDmaBufOpsEndCpuAccessGEM(struct dma_buf *psDmaBuf,
+                                    enum dma_data_direction eDirection)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	int iErr;
+
+	iErr = PVRDmaBufOpsEndCpuAccessCommon(psPMRWrapper, eDirection);
+
+	return iErr;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+static void *PVRDmaBufOpsKMapGEM(struct dma_buf *psDmaBuf, unsigned long uiPageNum)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsKMapCommon(psPMRWrapper, uiPageNum);
+}
+
+static void PVRDmaBufOpsKUnMapGEM(struct dma_buf *psDmaBuf, unsigned long uiPageNum, void *pvMem)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsKUnMapCommon(psPMRWrapper, uiPageNum, pvMem);
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void *PVRDmaBufOpsVMapGEM(struct dma_buf *psDmaBuf)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper);
+}
+#else
+static int PVRDmaBufOpsVMapGEM(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	return PVRDmaBufOpsVMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void PVRDmaBufOpsVUnMapGEM(struct dma_buf *psDmaBuf, void *pvAddr)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, pvAddr);
+}
+#else
+static void PVRDmaBufOpsVUnMapGEM(struct dma_buf *psDmaBuf, struct iosys_map *psMap)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVRDmaBufOpsVUnMapCommon(psPMRWrapper, psMap);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+static int PVRDmaBufOpsMMapGEM(struct dma_buf *psDmaBuf, struct vm_area_struct *psVMA)
+{
+	struct drm_gem_object *psObj = psDmaBuf->priv;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s(): psDmaBuf = %px, psPMR = %px", __func__,
+	         psDmaBuf, psPMRWrapper->psPMR));
+
+	return PVRDmaBufOpsMMapCommon(psPMRWrapper, psVMA);
+}
+
+static const struct dma_buf_ops sPVRDmaBufOpsGEM =
+{
+	.attach        = PVRDmaBufOpsAttachGEM,
+	.map_dma_buf   = PVRDmaBufOpsMapGEM,
+	.unmap_dma_buf = PVRDmaBufOpsUnmapGEM,
+	.release       = drm_gem_dmabuf_release,
+	.begin_cpu_access = PVRDmaBufOpsBeginCpuAccessGEM,
+	.end_cpu_access   = PVRDmaBufOpsEndCpuAccessGEM,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \
+	!((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) && (defined(CHROMIUMOS_KERNEL))))
+	.map_atomic    = PVRDmaBufOpsKMapGEM,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	.map           = PVRDmaBufOpsKMapGEM,
+	.unmap         = PVRDmaBufOpsKUnMapGEM,
+#endif
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.kmap_atomic   = PVRDmaBufOpsKMapGEM,
+	.kmap          = PVRDmaBufOpsKMapGEM,
+	.kunmap        = PVRDmaBufOpsKUnMapGEM,
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) */
+	.mmap          = PVRDmaBufOpsMMapGEM,
+
+	.vmap          = PVRDmaBufOpsVMapGEM,
+	.vunmap        = PVRDmaBufOpsVUnMapGEM,
+};
+
+/* end of dma_buf_ops (GEM) */
+
+
+typedef struct _PMR_DMA_BUF_DATA_
+{
+	/* Filled in at PMR create time */
+	PHYS_HEAP *psPhysHeap;
+	struct dma_buf_attachment *psAttachment;
+	PFN_DESTROY_DMABUF_PMR pfnDestroy;
+	IMG_BOOL bPoisonOnFree;
+	IMG_PID uiOriginPID;
+
+	/* Mapping information. */
+	struct iosys_map sMap;
+
+	/* Modified by PMR lock/unlock */
+	struct sg_table *psSgTable;
+	IMG_DEV_PHYADDR *pasDevPhysAddr;
+	IMG_UINT32 ui32PhysPageCount;
+	IMG_UINT32 ui32VirtPageCount;
+
+	IMG_BOOL bZombie;
+} PMR_DMA_BUF_DATA;
+
+/* Start size of the g_psDmaBufHash hash table */
+#define DMA_BUF_HASH_SIZE 20
+
+static DEFINE_MUTEX(g_FactoryLock);
+
+static HASH_TABLE *g_psDmaBufHash;
+static IMG_UINT32 g_ui32HashRefCount;
+
+#if defined(PVR_ANDROID_ION_USE_SG_LENGTH)
+#define pvr_sg_length(sg) ((sg)->length)
+#else
+#define pvr_sg_length(sg) sg_dma_len(sg)
+#endif
+
+static int
+DmaBufSetValue(struct dma_buf *psDmaBuf, int iValue, const char *szFunc)
+{
+	struct iosys_map sMap;
+	int err, err_end_access;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0))
+	int i;
+#endif
+
+	err = dma_buf_begin_cpu_access(psDmaBuf, DMA_FROM_DEVICE);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to begin cpu access (err=%d)",
+		                        szFunc, err));
+		goto err_out;
+	}
+
+	err = dma_buf_vmap(psDmaBuf, &sMap);
+	if (err)
+	{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map page (err=%d)",
+		                        szFunc, err));
+		goto exit_end_access;
+#else
+		for (i = 0; i < psDmaBuf->size / PAGE_SIZE; i++)
+		{
+			void *pvKernAddr;
+
+			pvKernAddr = dma_buf_kmap(psDmaBuf, i);
+			if (IS_ERR_OR_NULL(pvKernAddr))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map page (err=%ld)",
+							szFunc,
+							pvKernAddr ? PTR_ERR(pvKernAddr) : -ENOMEM));
+				err = !pvKernAddr ? -ENOMEM : -EINVAL;
+
+				goto exit_end_access;
+			}
+
+			memset(pvKernAddr, iValue, PAGE_SIZE);
+
+			dma_buf_kunmap(psDmaBuf, i, pvKernAddr);
+		}
+#endif
+	}
+	else
+	{
+		memset(sMap.vaddr, iValue, psDmaBuf->size);
+
+		dma_buf_vunmap(psDmaBuf, &sMap);
+	}
+
+	err = 0;
+
+exit_end_access:
+	do {
+		err_end_access = dma_buf_end_cpu_access(psDmaBuf, DMA_TO_DEVICE);
+	} while (err_end_access == -EAGAIN || err_end_access == -EINTR);
+
+	if (err_end_access)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to end cpu access (err=%d)",
+		                        szFunc, err_end_access));
+		if (!err)
+		{
+			err = err_end_access;
+		}
+	}
+
+err_out:
+	return err;
+}
+
+/*****************************************************************************
+ *                          PMR callback functions                           *
+ *****************************************************************************/
+
+/* This function is protected by the pfn(Get/Release)PMRFactoryLock() lock
+ * acquired/released in _UnrefAndMaybeDestroy() in pmr.c. */
+static void PMRFinalizeDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+	struct sg_table *psSgTable = psPrivData->psSgTable;
+
+	if (psDmaBuf->ops != &sPVRDmaBufOps && psDmaBuf->ops != &sPVRDmaBufOpsGEM)
+	{
+		if (g_psDmaBufHash)
+		{
+			/* We have a hash table so check if we've seen this dmabuf before */
+			if (HASH_Remove(g_psDmaBufHash, (uintptr_t) psDmaBuf) != 0U)
+			{
+				g_ui32HashRefCount--;
+
+				if (g_ui32HashRefCount == 0)
+				{
+					HASH_Delete(g_psDmaBufHash);
+					g_psDmaBufHash = NULL;
+				}
+			}
+
+			PVRSRVIonRemoveMemAllocRecord(psDmaBuf);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (psPrivData->bZombie)
+	{
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+		                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+		                            psPrivData->uiOriginPID);
+	}
+	else
+#endif
+	{
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+		                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+		                            psPrivData->uiOriginPID);
+	}
+#endif
+
+	psPrivData->ui32PhysPageCount = 0;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 36))
+	dma_buf_unmap_attachment_unlocked(psAttachment, psSgTable, DMA_BIDIRECTIONAL);
+#else
+	dma_buf_unmap_attachment(psAttachment, psSgTable, DMA_BIDIRECTIONAL);
+#endif
+
+	if (psPrivData->bPoisonOnFree)
+	{
+		int err = DmaBufSetValue(psDmaBuf, PVRSRV_POISON_ON_FREE_VALUE,
+		                         __func__);
+		PVR_LOG_IF_FALSE(err != 0, "Failed to poison allocation before free");
+
+		PVR_ASSERT(err != 0);
+	}
+
+	if (psPrivData->pfnDestroy)
+	{
+		psPrivData->pfnDestroy(psPrivData->psPhysHeap, psPrivData->psAttachment);
+	}
+
+	OSFreeMem(psPrivData->pasDevPhysAddr);
+	OSFreeMem(psPrivData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static PVRSRV_ERROR PMRZombifyDmaBufMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf_attachment *psAttachment = psPrivData->psAttachment;
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	psPrivData->bZombie = IMG_TRUE;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+#else
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+#endif
+
+	PVRSRVIonZombifyMemAllocRecord(psDmaBuf);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static PVRSRV_ERROR PMRLockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+{
+	/* The imported memory is assumed to be backed by a permanent allocation.
+	 * The PMR does not need to be (un)locked */
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRUnlockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+                                                 PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR PMRUnlockPhysAddressesDmaBuf(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	/* The imported memory is assumed to be backed by a permanent allocation.
+	 * The PMR does not need to be (un)locked */
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+	return PVRSRV_OK;
+}
+
+static void PMRFactoryLock(void)
+{
+	mutex_lock(&g_FactoryLock);
+}
+
+static void PMRFactoryUnlock(void)
+{
+	mutex_unlock(&g_FactoryLock);
+}
+
+static PVRSRV_ERROR PMRDevPhysAddrDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+					 IMG_UINT32 ui32Log2PageSize,
+					 IMG_UINT32 ui32NumOfPages,
+					 IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					 IMG_UINT64 ui64IPAPolicyValue,
+					 IMG_UINT64 ui64IPAClearMask,
+#endif
+					 IMG_BOOL *pbValid,
+					 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	IMG_UINT32 ui32PageIndex;
+	IMG_UINT32 idx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (ui32Log2PageSize != PAGE_SHIFT)
+	{
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (idx=0; idx < ui32NumOfPages; idx++)
+	{
+		if (pbValid[idx])
+		{
+			IMG_UINT32 ui32InPageOffset;
+
+			ui32PageIndex = puiOffset[idx] >> PAGE_SHIFT;
+			ui32InPageOffset = puiOffset[idx] - ((IMG_DEVMEM_OFFSET_T)ui32PageIndex << PAGE_SHIFT);
+
+			PVR_LOG_RETURN_IF_FALSE(ui32PageIndex < psPrivData->ui32VirtPageCount,
+			                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+			PVR_ASSERT(ui32InPageOffset < PAGE_SIZE);
+			psDevPAddr[idx].uiAddr = psPrivData->pasDevPhysAddr[ui32PageIndex].uiAddr + ui32InPageOffset;
+#if defined(SUPPORT_STATIC_IPA)
+			/* Modify the physical address with the associated IPA values */
+			psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+			psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+		}
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+				  size_t uiOffset,
+				  size_t uiSize,
+				  void **ppvKernelAddressOut,
+				  IMG_HANDLE *phHandleOut,
+				  PMR_FLAGS_T ulFlags)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	PVRSRV_ERROR eError;
+	int err;
+
+	if (psPrivData->ui32PhysPageCount != psPrivData->ui32VirtPageCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Kernel mappings for sparse DMABufs "
+				"are not allowed!", __func__));
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail;
+	}
+
+	err = dma_buf_begin_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	if (err)
+	{
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail;
+	}
+
+	err = dma_buf_vmap(psDmaBuf, &psPrivData->sMap);
+	if (err != 0 || psPrivData->sMap.vaddr == NULL)
+	{
+		eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+		goto fail_kmap;
+	}
+
+	*ppvKernelAddressOut = psPrivData->sMap.vaddr + uiOffset;
+	*phHandleOut = psPrivData->sMap.vaddr;
+
+	return PVRSRV_OK;
+
+fail_kmap:
+	do {
+		err = dma_buf_end_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	} while (err == -EAGAIN || err == -EINTR);
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+					      IMG_HANDLE hHandle)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	int err;
+
+	dma_buf_vunmap(psDmaBuf, &psPrivData->sMap);
+
+	do {
+		err = dma_buf_end_cpu_access(psDmaBuf, DMA_BIDIRECTIONAL);
+	} while (err == -EAGAIN || err == -EINTR);
+}
+
+static PVRSRV_ERROR PMRMMapDmaBuf(PMR_IMPL_PRIVDATA pvPriv,
+                                  PMR *psPMR,
+                                  PMR_MMAP_DATA pOSMMapData)
+{
+	PMR_DMA_BUF_DATA *psPrivData = pvPriv;
+	struct dma_buf *psDmaBuf = psPrivData->psAttachment->dmabuf;
+	struct vm_area_struct *psVma = pOSMMapData;
+	int err;
+
+	if (psPrivData->ui32PhysPageCount != psPrivData->ui32VirtPageCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Not possible to MMAP sparse DMABufs",
+				__func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	err = dma_buf_mmap(psDmaBuf, psVma, 0);
+	if (err)
+	{
+		return (err == -EINVAL) ? PVRSRV_ERROR_NOT_SUPPORTED : PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	MMapStatsAddOrUpdatePMR(psPMR, psVma->vm_end - psVma->vm_start);
+#endif
+
+	return PVRSRV_OK;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRDmaBufFuncTab =
+{
+	.pfnLockPhysAddresses = PMRLockPhysAddressesDmaBuf,
+	.pfnUnlockPhysAddresses = PMRUnlockPhysAddressesDmaBuf,
+	.pfnDevPhysAddr = PMRDevPhysAddrDmaBuf,
+	.pfnAcquireKernelMappingData = PMRAcquireKernelMappingDataDmaBuf,
+	.pfnReleaseKernelMappingData = PMRReleaseKernelMappingDataDmaBuf,
+	.pfnMMap = PMRMMapDmaBuf,
+	.pfnFinalize = PMRFinalizeDmaBuf,
+	.pfnGetPMRFactoryLock = PMRFactoryLock,
+	.pfnReleasePMRFactoryLock = PMRFactoryUnlock,
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = PMRZombifyDmaBufMem,
+#endif
+};
+
+/*****************************************************************************
+ *                          Public facing interface                          *
+ *****************************************************************************/
+
+PVRSRV_ERROR
+PhysmemCreateNewDmaBufBackedPMR(PHYS_HEAP *psHeap,
+                                struct dma_buf_attachment *psAttachment,
+                                PFN_DESTROY_DMABUF_PMR pfnDestroy,
+                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                IMG_DEVMEM_SIZE_T uiChunkSize,
+                                IMG_UINT32 ui32NumPhysChunks,
+                                IMG_UINT32 ui32NumVirtChunks,
+                                IMG_UINT32 *pui32MappingTable,
+                                IMG_UINT32 ui32NameSize,
+                                const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                                PMR **ppsPMRPtr)
+{
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+	PMR_DMA_BUF_DATA *psPrivData;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_BOOL bZeroOnAlloc;
+	IMG_BOOL bPoisonOnAlloc;
+	IMG_BOOL bPoisonOnFree;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i, j;
+	IMG_UINT32 uiPagesPerChunk = uiChunkSize >> PAGE_SHIFT;
+	IMG_UINT32 ui32PageCount = 0;
+	struct scatterlist *sg;
+	struct sg_table *table;
+	IMG_UINT32 uiSglOffset;
+	IMG_CHAR pszAnnotation[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_UINT32 ui32ActualDmaBufPageCount;
+
+	bZeroOnAlloc = PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags);
+	bPoisonOnAlloc = PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags);
+#if defined(DEBUG)
+	bPoisonOnFree = PVRSRV_CHECK_POISON_ON_FREE(uiFlags);
+#else
+	bPoisonOnFree = IMG_FALSE;
+#endif
+	if (bZeroOnAlloc && bPoisonOnFree)
+	{
+		/* Zero on Alloc and Poison on Alloc are mutually exclusive */
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errReturn;
+	}
+
+	if (!PMRValidateSize((IMG_UINT64) ui32NumVirtChunks * uiChunkSize))
+	{
+		PVR_LOG_VA(PVR_DBG_ERROR,
+				 "PMR size exceeds limit #Chunks: %u ChunkSz %"IMG_UINT64_FMTSPECX"",
+				 ui32NumVirtChunks,
+				 uiChunkSize);
+		eError = PVRSRV_ERROR_PMR_TOO_LARGE;
+		goto errReturn;
+	}
+
+	psPrivData = OSAllocZMem(sizeof(*psPrivData));
+	if (psPrivData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto errReturn;
+	}
+
+	psPrivData->psPhysHeap = psHeap;
+	psPrivData->psAttachment = psAttachment;
+	psPrivData->pfnDestroy = pfnDestroy;
+	psPrivData->bPoisonOnFree = bPoisonOnFree;
+	psPrivData->uiOriginPID = OSGetCurrentClientProcessIDKM();
+	psPrivData->ui32VirtPageCount =
+			(ui32NumVirtChunks * uiChunkSize) >> PAGE_SHIFT;
+
+	psPrivData->pasDevPhysAddr =
+			OSAllocZMem(sizeof(*(psPrivData->pasDevPhysAddr)) *
+			            psPrivData->ui32VirtPageCount);
+	if (!psPrivData->pasDevPhysAddr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate buffer for physical addresses (oom)",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto errFreePrivData;
+	}
+
+	if (bZeroOnAlloc || bPoisonOnAlloc)
+	{
+		int iValue = bZeroOnAlloc ? 0 : PVRSRV_POISON_ON_ALLOC_VALUE;
+		int err;
+
+		err = DmaBufSetValue(psDmaBuf, iValue, __func__);
+		if (err)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to map buffer for %s",
+			                        __func__,
+			                        bZeroOnAlloc ? "zeroing" : "poisoning"));
+
+			eError = PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING;
+			goto errFreePhysAddr;
+		}
+	}
+
+	table = dma_buf_map_attachment_unlocked(psAttachment, DMA_BIDIRECTIONAL);
+	if (IS_ERR_OR_NULL(table))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errFreePhysAddr;
+	}
+
+	/*
+	 * We do a two pass process: first work out how many pages there
+	 * are and second, fill in the data.
+	 */
+	for_each_sg(table->sgl, sg, table->nents, i)
+	{
+		ui32PageCount += PAGE_ALIGN(pvr_sg_length(sg)) / PAGE_SIZE;
+	}
+
+	if (WARN_ON(!ui32PageCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of phys. pages must not be zero",
+				 __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errUnmap;
+	}
+
+	/* Obtain actual page count of dma buf */
+	ui32ActualDmaBufPageCount = psAttachment->dmabuf->size / PAGE_SIZE;
+
+	if (WARN_ON(ui32ActualDmaBufPageCount < ui32NumPhysChunks * uiPagesPerChunk))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Requested physical chunks greater than "
+				"number of physical dma buf pages",
+				 __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errUnmap;
+	}
+
+	psPrivData->ui32PhysPageCount = ui32ActualDmaBufPageCount;
+	psPrivData->psSgTable = table;
+	ui32PageCount = 0;
+	sg = table->sgl;
+	uiSglOffset = 0;
+
+
+	/* Fill physical address array */
+	for (i = 0; i < ui32NumPhysChunks; i++)
+	{
+		for (j = 0; j < uiPagesPerChunk; j++)
+		{
+			IMG_UINT32 uiIdx = pui32MappingTable[i] * uiPagesPerChunk + j;
+
+			psPrivData->pasDevPhysAddr[uiIdx].uiAddr =
+					sg_dma_address(sg) + uiSglOffset;
+
+			/* Get the next offset for the current sgl or the next sgl */
+			uiSglOffset += PAGE_SIZE;
+			if (uiSglOffset >= pvr_sg_length(sg))
+			{
+				sg = sg_next(sg);
+				uiSglOffset = 0;
+
+				/* Check that we haven't looped */
+				if (WARN_ON(sg == table->sgl))
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to fill phys. address "
+							"array",
+							 __func__));
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					goto errUnmap;
+				}
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,
+	                            psPrivData->ui32PhysPageCount << PAGE_SHIFT,
+	                            psPrivData->uiOriginPID);
+#endif
+
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+
+	/*
+	 * Check no significant bits were lost in cast due to different
+	 * bit widths for flags
+	 */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (OSSNPrintf((IMG_CHAR *)pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN, "ImpDmaBuf:%s", (IMG_CHAR *)pszName) < 0)
+	{
+		pszAnnotation[0] = '\0';
+	}
+	else
+	{
+		pszAnnotation[DEVMEM_ANNOTATION_MAX_LEN-1] = '\0';
+	}
+
+	eError = PMRCreatePMR(psHeap,
+			      ui32NumVirtChunks * uiChunkSize,
+			      ui32NumPhysChunks,
+			      ui32NumVirtChunks,
+			      pui32MappingTable,
+			      PAGE_SHIFT,
+			      uiPMRFlags,
+			      pszAnnotation,
+			      &_sPMRDmaBufFuncTab,
+			      psPrivData,
+			      PMR_TYPE_DMABUF,
+			      ppsPMRPtr,
+			      PDUMP_NONE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create PMR (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		goto errFreePhysAddr;
+	}
+
+	return PVRSRV_OK;
+
+errUnmap:
+	dma_buf_unmap_attachment_unlocked(psAttachment, table, DMA_BIDIRECTIONAL);
+errFreePhysAddr:
+	OSFreeMem(psPrivData->pasDevPhysAddr);
+errFreePrivData:
+	OSFreeMem(psPrivData);
+errReturn:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PhysmemDestroyDmaBuf(PHYS_HEAP *psHeap,
+                                 struct dma_buf_attachment *psAttachment)
+{
+	struct dma_buf *psDmaBuf = psAttachment->dmabuf;
+
+	PVR_UNREFERENCED_PARAMETER(psHeap);
+
+	/* PMRUnlockSysPhysAddresses(psPMR) is redundant.
+	 * See PMRUnlockPhysAddressesDmaBuf */
+
+	dma_buf_detach(psDmaBuf, psAttachment);
+	dma_buf_put(psDmaBuf);
+}
+
+struct dma_buf *
+PhysmemGetDmaBuf(PMR *psPMR)
+{
+	PMR_DMA_BUF_DATA *psPrivData;
+
+	psPrivData = PMRGetPrivateData(psPMR, &_sPMRDmaBufFuncTab);
+	if (psPrivData)
+	{
+		return psPrivData->psAttachment->dmabuf;
+	}
+
+	return NULL;
+}
+
+struct dma_resv *
+PhysmemGetDmaResv(PMR *psPMR)
+{
+	struct dma_buf *psDmaBuf = PhysmemGetDmaBuf(psPMR);
+
+	if (psDmaBuf)
+	{
+		return psDmaBuf->resv;
+	}
+	else
+	{
+		PMR_DMA_BUF_WRAPPER *psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+
+		if (psPMRWrapper)
+		{
+			return &psPMRWrapper->sDmaResv;
+
+		}
+
+		return NULL;
+	}
+}
+
+static void
+PhysmemDestroyPMRWrapper(PMR_DMA_BUF_WRAPPER *psPMRWrapper)
+{
+	dma_resv_fini(&psPMRWrapper->sDmaResv);
+
+	OSLockDestroy(psPMRWrapper->hKernelMappingLock);
+	OSLockDestroy(psPMRWrapper->hDeviceMappingLock);
+	OSFreeMem(psPMRWrapper);
+}
+
+void
+PhysmemDmaBufExportFinalize(void *pvDmaBufExportData)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = pvDmaBufExportData;
+
+	PhysmemDestroyPMRWrapper(psPMRWrapper);
+}
+
+static PVRSRV_ERROR
+PhysmemCreatePMRWrapper(PMR *psPMR, PMR_DMA_BUF_WRAPPER **ppsPMRWrapper)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	psPMRWrapper = OSAllocZMem(sizeof(*psPMRWrapper));
+	PVR_LOG_GOTO_IF_NOMEM(psPMRWrapper, eError, fail_alloc_mem);
+
+	psPMRWrapper->psPMR = psPMR;
+
+	eError = OSLockCreate(&psPMRWrapper->hKernelMappingLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.1",
+	                      fail_kernel_mapping_lock_create);
+
+	eError = OSLockCreate(&psPMRWrapper->hDeviceMappingLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate.2",
+	                      fail_device_mapping_lock_create);
+
+	dma_resv_init(&psPMRWrapper->sDmaResv);
+
+	*ppsPMRWrapper = psPMRWrapper;
+
+	return PVRSRV_OK;
+fail_device_mapping_lock_create:
+	OSLockDestroy(psPMRWrapper->hKernelMappingLock);
+fail_kernel_mapping_lock_create:
+	OSFreeMem(psPMRWrapper);
+fail_alloc_mem:
+	return eError;
+}
+
+static PVRSRV_ERROR
+PhysmemGetOrCreatePMRWrapper(PMR *psPMR, PMR_DMA_BUF_WRAPPER **ppsPMRWrapper)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PVRSRV_ERROR eError;
+
+	psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+	if (!psPMRWrapper)
+	{
+		PMRFactoryLock();
+
+		/* Check again with the factory lock held */
+		psPMRWrapper = PMREnvDmaBufGetExportData(psPMR);
+		if (!psPMRWrapper)
+		{
+			eError = PhysmemCreatePMRWrapper(psPMR, &psPMRWrapper);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreatePMRWrapper", fail_create_pmr_wrapper);
+
+			/*
+			 * A PMR memory layout can't change once exported.
+			 * This makes sure the exported and imported parties
+			 * see the same layout of the memory.
+			 */
+			PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+			PMREnvDmaBufSetExportData(psPMR, psPMRWrapper);
+		}
+
+		PMRFactoryUnlock();
+	}
+
+	*ppsPMRWrapper = psPMRWrapper;
+
+	return PVRSRV_OK;
+
+fail_create_pmr_wrapper:
+	PMRFactoryUnlock();
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemExportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    PMR *psPMR,
+                    IMG_INT *piFd)
+{
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+	IMG_INT iFd;
+
+	eError = PhysmemGetOrCreatePMRWrapper(psPMR, &psPMRWrapper);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemExportDmaBuf", fail_get_pmr_wrapper);
+
+	PMRRefPMR(psPMR);
+
+	{
+		DEFINE_DMA_BUF_EXPORT_INFO(sDmaBufExportInfo);
+
+		sDmaBufExportInfo.priv  = psPMRWrapper;
+		sDmaBufExportInfo.ops   = &sPVRDmaBufOps;
+		sDmaBufExportInfo.size  = PMR_LogicalSize(psPMR);
+		sDmaBufExportInfo.flags = O_RDWR;
+		sDmaBufExportInfo.resv  = &psPMRWrapper->sDmaResv;
+
+		psDmaBuf = dma_buf_export(&sDmaBufExportInfo);
+	}
+
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to export buffer (err=%ld)",
+		         __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_export;
+	}
+
+	iFd = dma_buf_fd(psDmaBuf, O_RDWR);
+	if (iFd < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf fd (err=%d)",
+		         __func__, iFd));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_dma_buf;
+	}
+
+	*piFd = iFd;
+
+	return PVRSRV_OK;
+
+fail_dma_buf:
+	dma_buf_put(psDmaBuf);
+	return eError;
+
+fail_export:
+	PMRUnrefPMR(psPMR);
+
+fail_get_pmr_wrapper:
+	return eError;
+}
+
+struct dma_buf *
+PhysmemGEMPrimeExport(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+			struct drm_device *psDev,
+#endif
+
+			struct drm_gem_object *psObj,
+			int iFlags)
+{
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+	struct dma_buf *psDmaBuf = NULL;
+	DEFINE_DMA_BUF_EXPORT_INFO(sDmaBufExportInfo);
+
+
+	sDmaBufExportInfo.priv  = psObj;
+	sDmaBufExportInfo.ops   = &sPVRDmaBufOpsGEM;
+	sDmaBufExportInfo.size  = PMR_LogicalSize(psPMRWrapper->psPMR);
+	sDmaBufExportInfo.flags = iFlags;
+	sDmaBufExportInfo.resv  = &psPMRWrapper->sDmaResv;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	psDmaBuf = drm_gem_dmabuf_export(psObj->dev, &sDmaBufExportInfo);
+#else
+	psDmaBuf = drm_gem_dmabuf_export(psDev, &sDmaBufExportInfo);
+#endif
+	if (IS_ERR(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to export buffer (err=%ld)",
+		         __func__, PTR_ERR(psDmaBuf)));
+	}
+
+	return psDmaBuf;
+}
+
+void
+PhysmemGEMObjectFree(struct drm_gem_object *psObj)
+{
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+	drm_gem_object_release(psObj);
+
+	if (smp_load_acquire(&psPMRWrapper->psObj) == psObj)
+	{
+		smp_store_release(&psPMRWrapper->psObj, NULL);
+	}
+
+	PMRUnrefPMR(psPMRWrapper->psPMR);
+
+	OSFreeMem(psGEMObj);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static const struct drm_gem_object_funcs sPhysmemGEMObjFuncs = {
+	.export = PhysmemGEMPrimeExport,
+	.free = PhysmemGEMObjectFree,
+};
+#endif
+
+PVRSRV_ERROR
+PhysmemExportGemHandle(CONNECTION_DATA *psConnection,
+		       PVRSRV_DEVICE_NODE *psDevNode,
+		       PMR *psPMR,
+		       IMG_UINT32 *puHandle)
+{
+	struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+	struct drm_device *psDRMDev = dev_get_drvdata(psDev);
+	struct drm_file *psDRMFile = OSGetDRMFile(psConnection);
+	PMR_DMA_BUF_WRAPPER *psPMRWrapper;
+	PMR_DMA_BUF_GEM_OBJ *psGEMObj;
+	bool bAlreadyExported;
+	PVRSRV_ERROR eError;
+	int iErr;
+
+	eError = PhysmemGetOrCreatePMRWrapper(psPMR, &psPMRWrapper);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemExportGemHandle", fail_get_pmr_wrapper);
+
+	psGEMObj = OSAllocZMem(sizeof(*psGEMObj));
+	PVR_LOG_GOTO_IF_NOMEM(psGEMObj, eError, fail_alloc_mem);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	psGEMObj->sBase.funcs = &sPhysmemGEMObjFuncs;
+#endif
+	psGEMObj->psPMRWrapper = psPMRWrapper;
+
+	PMRRefPMR(psPMR);
+
+	drm_gem_private_object_init(psDRMDev, &psGEMObj->sBase,
+	                            PMR_LogicalSize(psPMR));
+
+	PMRFactoryLock();
+	bAlreadyExported = smp_load_acquire(&psPMRWrapper->psObj) != NULL;
+	if (!bAlreadyExported)
+	{
+		smp_store_release(&psPMRWrapper->psObj, &psGEMObj->sBase);
+	}
+	PMRFactoryUnlock();
+
+	if (bAlreadyExported)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_STILL_REFERENCED,
+					    fail_export_check);
+	}
+
+	iErr = drm_gem_handle_create(psDRMFile, &psGEMObj->sBase, puHandle);
+	if (iErr)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY,
+				    fail_handle_create);
+	}
+
+	/* The handle holds a reference on the object, so drop ours */
+	drm_gem_object_put(&psGEMObj->sBase);
+
+	return PVRSRV_OK;
+
+fail_export_check:
+fail_handle_create:
+	drm_gem_object_put(&psGEMObj->sBase);
+	return eError;
+
+fail_alloc_mem:
+fail_get_pmr_wrapper:
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemImportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_INT fd,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    IMG_UINT32 ui32NameSize,
+                    const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                    PMR **ppsPMRPtr,
+                    IMG_DEVMEM_SIZE_T *puiSize,
+                    IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32 ui32MappingTable = 0;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+
+	/* Get the buffer handle */
+	psDmaBuf = dma_buf_get(fd);
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf from fd (err=%ld)",
+				 __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		return PVRSRV_ERROR_BAD_MAPPING;
+
+	}
+
+	uiSize = psDmaBuf->size;
+
+	eError = PhysmemImportSparseDmaBuf(psConnection,
+	                                 psDevNode,
+	                                 fd,
+	                                 uiFlags,
+	                                 uiSize,
+	                                 1,
+	                                 1,
+	                                 &ui32MappingTable,
+	                                 ui32NameSize,
+	                                 pszName,
+	                                 ppsPMRPtr,
+	                                 puiSize,
+	                                 puiAlign);
+
+	dma_buf_put(psDmaBuf);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysmemImportSparseDmaBuf(CONNECTION_DATA *psConnection,
+                          PVRSRV_DEVICE_NODE *psDevNode,
+                          IMG_INT fd,
+                          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                          IMG_DEVMEM_SIZE_T uiChunkSize,
+                          IMG_UINT32 ui32NumPhysChunks,
+                          IMG_UINT32 ui32NumVirtChunks,
+                          IMG_UINT32 *pui32MappingTable,
+                          IMG_UINT32 ui32NameSize,
+                          const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                          PMR **ppsPMRPtr,
+                          IMG_DEVMEM_SIZE_T *puiSize,
+                          IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PMR *psPMR = NULL;
+	struct dma_buf_attachment *psAttachment;
+	struct dma_buf *psDmaBuf;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bHashTableCreated = IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_GOTO_IF_INVALID_PARAM(psDevNode != NULL, eError, errReturn);
+
+	/* Terminate string from bridge to prevent corrupt annotations in RI */
+	if (pszName != NULL)
+	{
+		IMG_CHAR* pszName0 = (IMG_CHAR*) pszName;
+		pszName0[ui32NameSize-1] = '\0';
+	}
+
+	PMRFactoryLock();
+
+	/* Get the buffer handle */
+	psDmaBuf = dma_buf_get(fd);
+	if (IS_ERR_OR_NULL(psDmaBuf))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get dma-buf from fd (err=%ld)",
+		        __func__, psDmaBuf ? PTR_ERR(psDmaBuf) : -ENOMEM));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, errUnlockReturn);
+	}
+
+	if (psDmaBuf->ops == &sPVRDmaBufOps || psDmaBuf->ops == &sPVRDmaBufOpsGEM)
+	{
+		/* We exported this dma_buf, so we can just get its PMR. */
+		if (psDmaBuf->ops == &sPVRDmaBufOps)
+		{
+			PMR_DMA_BUF_WRAPPER *psPMRWrapper = psDmaBuf->priv;
+
+			psPMR = psPMRWrapper->psPMR;
+		}
+		else
+		{
+			struct drm_gem_object *psObj = psDmaBuf->priv;
+			PMR_DMA_BUF_GEM_OBJ *psGEMObj = TO_PMR_DMA_BUF_GEM_OBJ(psObj);
+			PMR_DMA_BUF_WRAPPER *psPMRWrapper = psGEMObj->psPMRWrapper;
+
+			psPMR = psPMRWrapper->psPMR;
+		}
+
+		/* However, we can't import it if it belongs to a different device. */
+		if (PMR_DeviceNode(psPMR) != psDevNode)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PMR invalid for this device",
+			        __func__));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PERMITTED,
+			                    errUnlockAndDMAPut);
+		}
+	}
+	else if (g_psDmaBufHash != NULL)
+	{
+		/* We have a hash table so check if we've seen this dmabuf
+		 * before. */
+		psPMR = (PMR *) HASH_Retrieve(g_psDmaBufHash, (uintptr_t) psDmaBuf);
+	}
+	else
+	{
+		/* As different processes may import the same dmabuf we need to
+		 * create a hash table so we don't generate a duplicate PMR but
+		 * rather just take a reference on an existing one. */
+		g_psDmaBufHash = HASH_Create(DMA_BUF_HASH_SIZE);
+		PVR_GOTO_IF_NOMEM(g_psDmaBufHash, eError, errUnlockAndDMAPut);
+
+		bHashTableCreated = IMG_TRUE;
+	}
+
+	if (psPMR != NULL)
+	{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		if (PMR_IsZombie(psPMR))
+		{
+			PMRDequeueZombieAndRef(psPMR);
+		}
+		else
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+		{
+			/* Reuse the PMR we already created */
+			PMRRefPMR(psPMR);
+		}
+
+		*ppsPMRPtr = psPMR;
+		*puiSize = PMR_LogicalSize(psPMR);
+		*puiAlign = PAGE_SIZE;
+
+		PMRFactoryUnlock();
+		dma_buf_put(psDmaBuf);
+
+		/* We expect a PMR to be immutable at this point.
+		 * But its explicitly set here to cover a corner case
+		 * where a PMR created through non-DMA interface could be
+		 * imported back again through DMA interface. */
+		PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+		return PVRSRV_OK;
+	}
+
+	/* Do we want this to be a sparse PMR? */
+	if (ui32NumVirtChunks > 1)
+	{
+		/* Parameter validation */
+		if (psDmaBuf->size < (uiChunkSize * ui32NumPhysChunks) ||
+		    uiChunkSize != PAGE_SIZE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Requesting sparse buffer: "
+					"uiChunkSize ("IMG_DEVMEM_SIZE_FMTSPEC") must be equal to "
+					"OS page size (%lu). uiChunkSize * ui32NumPhysChunks "
+					"("IMG_DEVMEM_SIZE_FMTSPEC") must"
+					" not be greater than the buffer size ("IMG_SIZE_FMTSPEC").",
+					 __func__,
+					uiChunkSize,
+					PAGE_SIZE,
+					uiChunkSize * ui32NumPhysChunks,
+					psDmaBuf->size));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS,
+			                    errUnlockAndDMAPut);
+		}
+	}
+	else
+	{
+		/* if ui32NumPhysChunks == 0 then pui32MappingTable == NULL
+		 * this is handled by the generated bridge code.
+		 * Because ui32NumPhysChunks is set to 1 below, we don't allow NULL array */
+		if (pui32MappingTable == NULL)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS,
+			                    errUnlockAndDMAPut);
+		}
+
+		/* Make sure parameters are valid for non-sparse allocations as well */
+		uiChunkSize = psDmaBuf->size;
+		ui32NumPhysChunks = 1;
+		ui32NumVirtChunks = 1;
+	}
+
+	{
+		IMG_DEVMEM_SIZE_T uiSize = ui32NumVirtChunks * uiChunkSize;
+		IMG_UINT32 uiLog2PageSize = PAGE_SHIFT; /* log2(uiChunkSize) */
+		eError = PhysMemValidateParams(psDevNode,
+		                               ui32NumPhysChunks,
+		                               ui32NumVirtChunks,
+		                               pui32MappingTable,
+		                               uiFlags,
+		                               &uiLog2PageSize,
+		                               &uiSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysMemValidateParams", errUnlockAndDMAPut);
+	}
+
+	psAttachment = dma_buf_attach(psDmaBuf, psDevNode->psDevConfig->pvOSDevice);
+	if (IS_ERR_OR_NULL(psAttachment))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to attach to dma-buf (err=%ld)",
+				 __func__, psAttachment? PTR_ERR(psAttachment) : -ENOMEM));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING,
+		                    errUnlockAndDMAPut);
+	}
+
+	/*
+	 * Note:
+	 * While we have no way to determine the type of the buffer we just
+	 * assume that all dmabufs are from the same physical heap.
+	 */
+	eError = PhysmemCreateNewDmaBufBackedPMR(psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_EXTERNAL],
+	                                         psAttachment,
+	                                         PhysmemDestroyDmaBuf,
+	                                         uiFlags,
+	                                         uiChunkSize,
+	                                         ui32NumPhysChunks,
+	                                         ui32NumVirtChunks,
+	                                         pui32MappingTable,
+	                                         ui32NameSize,
+	                                         pszName,
+	                                         &psPMR);
+	PVR_GOTO_IF_ERROR(eError, errDMADetach);
+
+	/* The imported memory is assumed to be backed by a permanent allocation. */
+	/* PMRLockSysPhysAddresses(psPMR) is redundant.
+	 * See PMRLockPhysAddressesDmaBuf */
+
+	/* First time we've seen this dmabuf so store it in the hash table */
+	HASH_Insert(g_psDmaBufHash, (uintptr_t) psDmaBuf, (uintptr_t) psPMR);
+	g_ui32HashRefCount++;
+
+	PMRFactoryUnlock();
+
+	PVRSRVIonAddMemAllocRecord(psDmaBuf);
+
+	*ppsPMRPtr = psPMR;
+	*puiSize = ui32NumVirtChunks * uiChunkSize;
+	*puiAlign = PAGE_SIZE;
+
+	/* The memory that's just imported is owned by some other entity.
+	 * Hence the memory layout cannot be changed through our API */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	return PVRSRV_OK;
+
+errDMADetach:
+	dma_buf_detach(psDmaBuf, psAttachment);
+
+errUnlockAndDMAPut:
+	if (bHashTableCreated)
+	{
+		HASH_Delete(g_psDmaBufHash);
+		g_psDmaBufHash = NULL;
+	}
+	dma_buf_put(psDmaBuf);
+
+errUnlockReturn:
+	PMRFactoryUnlock();
+
+errReturn:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_dmabuf.h b/drivers/gpu/drm/img-rogue/physmem_dmabuf.h
new file mode 100644
index 000000000000..ccb8c087eac6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_dmabuf.h
@@ -0,0 +1,124 @@
+/**************************************************************************/ /*!
+@File           physmem_dmabuf.h
+@Title          Header for dmabuf PMR factory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks importing Ion allocations
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PHYSMEM_DMABUF_H)
+#define PHYSMEM_DMABUF_H
+
+#include <linux/dma-buf.h>
+
+#include "pvr_dma_resv.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+#include "pmr.h"
+
+typedef void (*PFN_DESTROY_DMABUF_PMR)(PHYS_HEAP *psHeap,
+                                       struct dma_buf_attachment *psAttachment);
+
+PVRSRV_ERROR
+PhysmemCreateNewDmaBufBackedPMR(PHYS_HEAP *psHeap,
+                                struct dma_buf_attachment *psAttachment,
+                                PFN_DESTROY_DMABUF_PMR pfnDestroy,
+                                PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                IMG_DEVMEM_SIZE_T uiChunkSize,
+                                IMG_UINT32 ui32NumPhysChunks,
+                                IMG_UINT32 ui32NumVirtChunks,
+                                IMG_UINT32 *pui32MappingTable,
+		                        IMG_UINT32 ui32NameSize,
+		                        const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                                PMR **ppsPMRPtr);
+
+struct dma_buf *
+PhysmemGetDmaBuf(PMR *psPMR);
+
+struct dma_resv *
+PhysmemGetDmaResv(PMR *psPMR);
+
+PVRSRV_ERROR
+PhysmemExportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    PMR *psPMR,
+                    IMG_INT *piFd);
+
+PVRSRV_ERROR
+PhysmemExportGemHandle(CONNECTION_DATA *psConnection,
+		       PVRSRV_DEVICE_NODE *psDevNode,
+		       PMR *psPMR,
+		       IMG_UINT32 *puHandle);
+
+PVRSRV_ERROR
+PhysmemImportDmaBuf(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_INT fd,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    IMG_UINT32 ui32NameSize,
+                    const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                    PMR **ppsPMRPtr,
+                    IMG_DEVMEM_SIZE_T *puiSize,
+                    IMG_DEVMEM_ALIGN_T *puiAlign);
+
+PVRSRV_ERROR
+PhysmemImportSparseDmaBuf(CONNECTION_DATA *psConnection,
+                          PVRSRV_DEVICE_NODE *psDevNode,
+                          IMG_INT fd,
+                          PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                          IMG_DEVMEM_SIZE_T uiChunkSize,
+                          IMG_UINT32 ui32NumPhysChunks,
+                          IMG_UINT32 ui32NumVirtChunks,
+                          IMG_UINT32 *pui32MappingTable,
+                          IMG_UINT32 ui32NameSize,
+                          const IMG_CHAR pszName[DEVMEM_ANNOTATION_MAX_LEN],
+                          PMR **ppsPMRPtr,
+                          IMG_DEVMEM_SIZE_T *puiSize,
+                          IMG_DEVMEM_ALIGN_T *puiAlign);
+
+#endif /* !defined(PHYSMEM_DMABUF_H) */
diff --git a/drivers/gpu/drm/img-rogue/physmem_dmabuf_internal.h b/drivers/gpu/drm/img-rogue/physmem_dmabuf_internal.h
new file mode 100644
index 000000000000..50a8ab37f442
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_dmabuf_internal.h
@@ -0,0 +1,69 @@
+/**************************************************************************/ /*!
+@File           physmem_dmabuf_internal.h
+@Title          Internal header for dmabuf PMR factory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks importing Ion allocations
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PHYSMEM_DMABUF_INTERNAL_H)
+#define PHYSMEM_DMABUF_INTERNAL_H
+
+#include <linux/version.h>
+
+struct dma_buf;
+struct drm_device;
+struct drm_gem_object;
+
+struct dma_buf *
+PhysmemGEMPrimeExport(
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+			struct drm_device *psDev,
+#endif
+
+			struct drm_gem_object *psObj,
+			int iFlags);
+
+void
+PhysmemGEMObjectFree(struct drm_gem_object *psObj);
+
+void
+PhysmemDmaBufExportFinalize(void *pvDmaBufExportData);
+
+#endif /* !defined(PHYSMEM_DMABUF_INTERNAL_H) */
diff --git a/drivers/gpu/drm/img-rogue/physmem_extmem.c b/drivers/gpu/drm/img-rogue/physmem_extmem.c
new file mode 100644
index 000000000000..4ca2c56903c0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_extmem.c
@@ -0,0 +1,71 @@
+/*************************************************************************/ /*!
+@File           physmem_extmem.c
+@Title          Physmem - External Memory Import Support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common entry point for Importing Externally allocated Memory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "pvrsrv.h"
+#include "physmem_extmem.h"
+
+#if 0
+#if defined(LMA)
+#pragma message "WARNING! You are compiling SUPPORT_WRAP_EXTMEM on a platform with LMA. " \
+	"Make sure the memory you wrap is actually accessible by the GPU!"
+#endif
+#endif
+
+PVRSRV_ERROR
+PhysmemWrapExtMem(CONNECTION_DATA * psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT64 pvCpuVAddr,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  PMR **ppsPMRPtr)
+{
+	return PhysmemWrapExtMemOS(psConnection,
+	                           psDevNode,
+	                           uiSize,
+	                           (IMG_CPU_VIRTADDR)(uintptr_t)pvCpuVAddr,
+	                           uiFlags,
+	                           ppsPMRPtr);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_extmem.h b/drivers/gpu/drm/img-rogue/physmem_extmem.h
new file mode 100644
index 000000000000..cc7d23297d43
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_extmem.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem_extmem header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for common entry point for creation of wrapped memory PMR's
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_EXTMEM_H
+#define SRVSRV_PHYSMEM_EXTMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "connection_server.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/*
+ * Based on PhysmemNewRamBackedPMR but uses a passed in virtual address.
+ */
+PVRSRV_ERROR
+PhysmemWrapExtMemOS(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    IMG_CPU_VIRTADDR pvCpuVAddr,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    PMR **ppsPMRPtr);
+
+PVRSRV_ERROR
+PhysmemWrapExtMem(CONNECTION_DATA *psConnection,
+                  PVRSRV_DEVICE_NODE *psDevNode,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT64 pvCpuVAddr,
+                  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                  PMR **ppsPMROut);
+
+#endif /* SRVSRV_PHYSMEM_EXTMEM_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_extmem_linux.c b/drivers/gpu/drm/img-rogue/physmem_extmem_linux.c
new file mode 100644
index 000000000000..66265c266f22
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_extmem_linux.c
@@ -0,0 +1,1072 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR functions to wrap non-services allocated
+                memory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                implementing the function callbacks for physical memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/uaccess.h>
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "devicemem_server_utils.h"
+
+#include "physmem_extmem.h"
+#include "physmem_extmem_wrap.h"
+#include "pdump_physmem.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "physmem.h"
+#include "cache_km.h"
+
+#include "kernel_compatibility.h"
+
+typedef struct _WRAP_KERNEL_MAP_DATA_
+{
+	void *pvKernLinAddr;
+	IMG_BOOL bVMAP;
+} WRAP_KERNEL_MAP_DATA;
+
+/* Free the PMR private data */
+static void _FreeWrapData(PMR_WRAP_DATA *psPrivData)
+{
+	OSFreeMem(psPrivData->ppsPageArray);
+	OSFreeMem(psPrivData->ppvPhysAddr);
+	psPrivData->psVMArea = NULL;
+	OSFreeMem(psPrivData);
+}
+
+
+/* Allocate the PMR private data */
+static PVRSRV_ERROR _AllocWrapData(PMR_WRAP_DATA **ppsPrivData,
+                            PVRSRV_DEVICE_NODE *psDevNode,
+                             IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_CPU_VIRTADDR pvCpuVAddr,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_WRAP_DATA *psPrivData;
+	struct vm_area_struct *psVMArea;
+	IMG_UINT32 ui32CPUCacheMode;
+
+	/* Obtain a reader lock on the process mmap lock while we are using
+	 * the psVMArea.
+	 */
+
+	mmap_read_lock(current->mm);
+
+	/* Find the VMA */
+	psVMArea = find_vma(current->mm, (uintptr_t)pvCpuVAddr);
+	if (psVMArea == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Couldn't find memory region containing start address %p",
+				__func__,
+				(void*) pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto eUnlockReturn;
+	}
+
+	/* If requested size is larger than actual allocation
+	 * return error. Can never request more memory to be imported
+	 * than its original allocation */
+	/* Now check the end address is in range */
+	if (((uintptr_t)pvCpuVAddr + uiSize) > psVMArea->vm_end)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: End address %p is outside of the region returned by find_vma",
+				__func__,
+				(void*) (uintptr_t)((uintptr_t)pvCpuVAddr + uiSize)));
+		eError = PVRSRV_ERROR_BAD_PARAM_SIZE;
+		goto eUnlockReturn;
+	}
+
+	/* Find_vma locates a region with an end point past a given
+	 * virtual address. So check the address is actually in the region. */
+	if ((uintptr_t)pvCpuVAddr < psVMArea->vm_start)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Start address %p is outside of the region returned by find_vma",
+				__func__,
+				(void*) pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto eUnlockReturn;
+	}
+
+	eError = DevmemCPUCacheMode(uiFlags, &ui32CPUCacheMode);
+	if (eError != PVRSRV_OK)
+	{
+		goto eUnlockReturn;
+	}
+
+	/* Allocate and initialise private factory data */
+	psPrivData = OSAllocZMem(sizeof(*psPrivData));
+	if (psPrivData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	psPrivData->ui32CPUCacheFlags = ui32CPUCacheMode;
+
+	/* Track the VMA area structure so that it can be checked later */
+	psPrivData->psVMArea = psVMArea;
+
+	psPrivData->psDevNode = psDevNode;
+	psPrivData->uiTotalNumPages = uiSize >> PAGE_SHIFT;
+
+	/* Allocate page and phys address arrays */
+	psPrivData->ppsPageArray = OSAllocZMem(sizeof(*(psPrivData->ppsPageArray)) * psPrivData->uiTotalNumPages);
+	if (psPrivData == NULL)
+	{
+		OSFreeMem(psPrivData);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	psPrivData->ppvPhysAddr = OSAllocZMem(sizeof(*(psPrivData->ppvPhysAddr)) * psPrivData->uiTotalNumPages);
+	if (psPrivData->ppvPhysAddr == NULL)
+	{
+		OSFreeMem(psPrivData->ppsPageArray);
+		OSFreeMem(psPrivData);
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto eUnlockReturn;
+	}
+
+	if (uiFlags & (PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+					PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED))
+	{
+		psPrivData->bWrite = IMG_TRUE;
+	}
+
+	*ppsPrivData = psPrivData;
+
+	eError = PVRSRV_OK;
+eUnlockReturn:
+	mmap_read_unlock(current->mm);
+
+	return eError;
+}
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+/* Free the pages we got via _TryFindVMA() */
+static void _FreeFindVMAPages(PMR_WRAP_DATA *psPrivData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i] != NULL)
+		{
+			/* Release the page */
+			put_page(psPrivData->ppsPageArray[i]);
+		}
+	}
+}
+#endif
+
+/* Release pages got via get_user_pages().
+ * As the function comment for GUP states we have
+ * to set the page dirty if it has been written to and
+ * we have to put the page to decrease its refcount. */
+static void _FreeGetUserPages(PMR_WRAP_DATA *psPrivData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i])
+		{
+			if (IMG_TRUE == psPrivData->bWrite)
+			{
+				/* Write data back to fs if necessary */
+				set_page_dirty_lock(psPrivData->ppsPageArray[i]);
+			}
+
+			/* Release the page */
+			put_page(psPrivData->ppsPageArray[i]);
+		}
+	}
+}
+
+/* Get the page structures and physical addresses mapped to
+ * a CPU virtual range via get_user_pages() */
+static PVRSRV_ERROR _TryGetUserPages(PVRSRV_DEVICE_NODE *psDevNode,
+                                IMG_DEVMEM_SIZE_T uiSize,
+                                IMG_CPU_VIRTADDR pvCpuVAddr,
+                                PMR_WRAP_DATA *psPrivData)
+{
+	IMG_INT32 iMappedPages, i;
+
+	IMG_UINT64 ui64DmaMask = dma_get_mask(psDevNode->psDevConfig->pvOSDevice);
+
+	/* Do the actual call */
+	iMappedPages = get_user_pages_fast((uintptr_t) pvCpuVAddr, uiSize >> PAGE_SHIFT,
+	                                    psPrivData->bWrite, psPrivData->ppsPageArray);
+	if (iMappedPages < 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "get_user_pages_fast() failed, got back %d, expected num pages %d",
+		         iMappedPages,
+		         psPrivData->uiTotalNumPages));
+
+		return PVRSRV_ERROR_FAILED_TO_ACQUIRE_PAGES;
+	}
+
+	/* Fill the physical address array */
+	for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+	{
+		if (psPrivData->ppsPageArray[i])
+		{
+			psPrivData->ppvPhysAddr[i].uiAddr = page_to_phys(psPrivData->ppsPageArray[i]);
+			psPrivData->uiNumBackedPages += 1;
+		}
+		else
+		{
+			psPrivData->ppvPhysAddr[i].uiAddr = 0;
+		}
+
+
+		/* Check the data transfer capability of the DMA.
+		 *
+		 * For instance:
+		 * APOLLO test chips TCF5 or ES2 can only access 4G maximum memory from the card.
+		 * Hence pages with a physical address beyond 4G range cannot be accessed by the
+		 * device. An error is reported in such a case. */
+		if (psPrivData->ppvPhysAddr[i].uiAddr & ~ui64DmaMask)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Backed page @pos:%d Physical Address: %pa exceeds GPU "
+					"accessible range(mask): 0x%0llx",
+					i, &psPrivData->ppvPhysAddr[i].uiAddr, ui64DmaMask));
+
+			psPrivData->bWrite = IMG_FALSE;
+
+			/* Free the acquired pages */
+			_FreeGetUserPages(psPrivData);
+
+			return PVRSRV_ERROR_INVALID_PHYS_ADDR;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Release all the pages we got via _TryGetUserPages or _TryFindVMA */
+static PVRSRV_ERROR _WrapExtMemReleasePages(PMR_WRAP_DATA *psPrivData)
+{
+	switch (psPrivData->eWrapType)
+	{
+		case WRAP_TYPE_GET_USER_PAGES:
+			_FreeGetUserPages(psPrivData);
+			break;
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+		case WRAP_TYPE_FIND_VMA:
+			_FreeFindVMAPages(psPrivData);
+			break;
+#endif
+		case WRAP_TYPE_NULL:
+			/* fall through */
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Wrong wrap type, cannot release pages",
+					__func__));
+			return PVRSRV_ERROR_INVALID_WRAP_TYPE;
+	}
+
+	_FreeWrapData(psPrivData);
+
+	return PVRSRV_OK;
+}
+
+/* Try to get pages and physical addresses for a CPU virtual address.
+ * Will try both methods:
+ * get_user_pages or find_vma + page table walk if the first one fails */
+static PVRSRV_ERROR _WrapExtMemAcquirePages(PVRSRV_DEVICE_NODE *psDevNode,
+                                            IMG_DEVMEM_SIZE_T uiSize,
+                                            IMG_CPU_VIRTADDR pvCpuVAddr,
+                                            PMR_WRAP_DATA *psPrivData)
+{
+	PVRSRV_ERROR eError;
+
+	eError = _TryGetUserPages(psDevNode,
+	                            uiSize,
+	                            pvCpuVAddr,
+	                            psPrivData);
+	if (eError == PVRSRV_OK)
+	{
+		psPrivData->eWrapType = WRAP_TYPE_GET_USER_PAGES;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"%s: Used GetUserPages",
+				__func__));
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+	if (PVRSRV_ERROR_INVALID_PHYS_ADDR != eError)
+	{
+		PVRSRV_ERROR eError2;
+		eError2 = _TryFindVMA(uiSize,
+		                      (uintptr_t) pvCpuVAddr,
+		                      psPrivData);
+		if (eError2 == PVRSRV_OK)
+		{
+			psPrivData->eWrapType = WRAP_TYPE_FIND_VMA;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Used FindVMA",
+					__func__));
+			return PVRSRV_OK;
+		}
+
+		PVR_WARN_IF_ERROR(eError, "_TryGetUserPages");
+		PVR_LOG_ERROR(eError2, "_TryFindVMA");
+	}
+	else
+	{
+		PVR_LOG_ERROR(eError, "_TryGetUserPages");
+	}
+#else
+	PVR_LOG_ERROR(eError, "_TryGetUserPages");
+#endif
+
+	psPrivData->eWrapType = WRAP_TYPE_NULL;
+	return eError;
+}
+
+static PVRSRV_ERROR
+PMRSysPhysAddrExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                     IMG_UINT32 ui32Log2PageSize,
+                     IMG_UINT32 ui32NumOfPages,
+                     IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+                     IMG_UINT64 ui64IPAPolicyValue,
+                     IMG_UINT64 ui64IPAClearMask,
+#endif
+                     IMG_BOOL *pbValid,
+                     IMG_DEV_PHYADDR *psDevPAddr)
+{
+	const PMR_WRAP_DATA *psWrapData = pvPriv;
+	IMG_UINT32 uiPageSize = 1U << PAGE_SHIFT;
+	IMG_UINT32 uiInPageOffset;
+	IMG_UINT32 uiPageIndex;
+	IMG_UINT32 uiIdx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+
+	if (PAGE_SHIFT != ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Requested ui32Log2PageSize %u is different from "
+				"OS page shift %u. Not supported.",
+				__func__,
+				ui32Log2PageSize,
+				PAGE_SHIFT));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++)
+	{
+		uiPageIndex = puiOffset[uiIdx] >> PAGE_SHIFT;
+		uiInPageOffset = puiOffset[uiIdx] - ((IMG_DEVMEM_OFFSET_T)uiPageIndex << PAGE_SHIFT);
+
+		PVR_LOG_RETURN_IF_FALSE(uiPageIndex < psWrapData->uiTotalNumPages,
+		                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+		PVR_ASSERT(uiInPageOffset < uiPageSize);
+
+		/* The ExtMem is only enabled in UMA mode, in that mode, the device physical
+		 * address translation will be handled by the PRM factory after this call.
+		 * Here we just copy the physical address like other callback implementations. */
+		psDevPAddr[uiIdx].uiAddr = psWrapData->ppvPhysAddr[uiPageIndex].uiAddr;
+
+		pbValid[uiIdx] = (psDevPAddr[uiIdx].uiAddr)? IMG_TRUE:IMG_FALSE;
+
+		psDevPAddr[uiIdx].uiAddr += uiInPageOffset;
+#if defined(SUPPORT_STATIC_IPA)
+		psDevPAddr[uiIdx].uiAddr &= ~ui64IPAClearMask;
+		psDevPAddr[uiIdx].uiAddr |= ui64IPAPolicyValue;
+#endif	/* SUPPORT_STATIC_IPA */
+	}
+
+	return PVRSRV_OK;
+}
+
+
+static void
+PMRFinalizeExtMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PMR_WRAP_DATA *psWrapData = pvPriv;
+
+	PVRSRV_ERROR eError = _WrapExtMemReleasePages(psWrapData);
+	PVR_LOG_IF_ERROR(eError, "_WrapExtMemReleasePages");
+}
+
+
+static void _UnmapPage(PMR_WRAP_DATA *psWrapData,
+                       WRAP_KERNEL_MAP_DATA *psMapData)
+{
+	IMG_BOOL bSuccess;
+
+	if (psMapData->bVMAP)
+	{
+		vunmap(psMapData->pvKernLinAddr);
+	}
+	else
+	{
+		if (psWrapData->ui32CPUCacheFlags & ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unable to unmap wrapped extmem "
+			        "page - wrong cached mode flags passed. This may leak "
+			        "memory.", __func__));
+			PVR_ASSERT(!"Found non-cpu cache mode flag when unmapping from "
+			           "the cpu");
+		}
+		else
+		{
+			bSuccess = OSUnMapPhysToLin(psMapData->pvKernLinAddr, PAGE_SIZE);
+			if (!bSuccess)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Unable to unmap wrapped extmem "
+				        "page. This may leak memory.", __func__));
+			}
+		}
+	}
+}
+
+static void _MapPage(PMR_WRAP_DATA *psWrapData,
+                     IMG_UINT32 uiPageIdx,
+                     WRAP_KERNEL_MAP_DATA *psMapData)
+{
+	IMG_UINT8 *puiLinCPUAddr;
+
+	if (psWrapData->ppsPageArray[uiPageIdx])
+	{
+		pgprot_t prot = PAGE_KERNEL;
+
+		switch (PVRSRV_CPU_CACHE_MODE(psWrapData->ui32CPUCacheFlags))
+		{
+			case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				prot = pgprot_noncached(prot);
+				break;
+
+			case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				prot = pgprot_writecombine(prot);
+				break;
+
+			case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+				break;
+
+			default:
+				break;
+		}
+
+		puiLinCPUAddr = vmap(&psWrapData->ppsPageArray[uiPageIdx], 1, VM_MAP, prot);
+
+		psMapData->bVMAP = IMG_TRUE;
+	}
+	else
+	{
+		puiLinCPUAddr = (IMG_UINT8*) OSMapPhysToLin(psWrapData->ppvPhysAddr[uiPageIdx],
+		                                              PAGE_SIZE,
+		                                              psWrapData->ui32CPUCacheFlags);
+
+		psMapData->bVMAP = IMG_FALSE;
+	}
+
+	psMapData->pvKernLinAddr = puiLinCPUAddr;
+
+}
+
+static PVRSRV_ERROR _CopyBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                     IMG_DEVMEM_OFFSET_T uiOffset,
+                                     IMG_UINT8 *pcBuffer,
+                                     size_t uiBufSz,
+                                     size_t *puiNumBytes,
+                                     IMG_BOOL bWrite)
+{
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	size_t uiBytesToCopy = uiBufSz;
+	IMG_UINT32 uiPageIdx = uiOffset >> PAGE_SHIFT;
+	IMG_BOOL bFirst = IMG_TRUE;
+	WRAP_KERNEL_MAP_DATA sKernMapData;
+
+
+	if ((uiBufSz + uiOffset) > (psWrapData->uiTotalNumPages << PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Trying to read out of bounds of PMR.",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Copy the pages */
+	while (uiBytesToCopy != 0)
+	{
+		size_t uiBytesPerLoop;
+		IMG_DEVMEM_OFFSET_T uiCopyOffset;
+
+		if (bFirst)
+		{
+			bFirst = IMG_FALSE;
+			uiCopyOffset = (uiOffset & ~PAGE_MASK);
+			uiBytesPerLoop = (uiBufSz >= (PAGE_SIZE - uiCopyOffset)) ?
+					PAGE_SIZE - uiCopyOffset : uiBufSz;
+		}
+		else
+		{
+			uiCopyOffset = 0;
+			uiBytesPerLoop = (uiBytesToCopy > PAGE_SIZE) ? PAGE_SIZE : uiBytesToCopy;
+		}
+
+		_MapPage(psWrapData,
+		         uiPageIdx,
+		         &sKernMapData);
+
+		if (sKernMapData.pvKernLinAddr == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unable to map wrapped extmem page.",
+					__func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+
+		/* Use 'DeviceMemCopy' because we need to be conservative. We can't
+		 * know whether the wrapped memory was originally imported as cached.
+		 */
+
+		if (bWrite)
+		{
+			OSDeviceMemCopy(sKernMapData.pvKernLinAddr + uiCopyOffset,
+			          pcBuffer,
+			          uiBytesPerLoop);
+		}
+		else
+		{
+			OSDeviceMemCopy(pcBuffer,
+			          sKernMapData.pvKernLinAddr + uiCopyOffset,
+			          uiBytesPerLoop);
+		}
+
+		_UnmapPage(psWrapData,
+		           &sKernMapData);
+
+		uiBytesToCopy -= uiBytesPerLoop;
+		uiPageIdx++;
+	}
+
+	*puiNumBytes = uiBufSz;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRReadBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                   IMG_DEVMEM_OFFSET_T uiOffset,
+                   IMG_UINT8 *pcBuffer,
+                   size_t uiBufSz,
+                   size_t *puiNumBytes)
+{
+
+	return _CopyBytesExtMem(pvPriv,
+	                        uiOffset,
+	                        pcBuffer,
+	                        uiBufSz,
+	                        puiNumBytes,
+	                        IMG_FALSE);
+}
+
+static PVRSRV_ERROR
+PMRWriteBytesExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                    IMG_DEVMEM_OFFSET_T uiOffset,
+                    IMG_UINT8 *pcBuffer,
+                    size_t uiBufSz,
+                    size_t *puiNumBytes)
+{
+	return _CopyBytesExtMem(pvPriv,
+	                        uiOffset,
+	                        pcBuffer,
+	                        uiBufSz,
+	                        puiNumBytes,
+	                        IMG_TRUE);
+}
+
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                  size_t uiOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  IMG_HANDLE *phHandleOut,
+                                  PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	WRAP_KERNEL_MAP_DATA *psKernMapData;
+	IMG_UINT32 ui32PageIndex = uiOffset >> PAGE_SHIFT;
+	IMG_UINT32 ui32PageOffset = (IMG_UINT32)(uiOffset & ~PAGE_MASK);
+
+	/* Offset was out of bounds */
+	if (ui32PageIndex > psWrapData->uiTotalNumPages)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, offset out of PMR bounds.",
+				__func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	/* We can not map in more than one page with ioremap.
+	 * Only possible with physically contiguous pages */
+	if ((ui32PageOffset + uiSize) > PAGE_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, cannot map more than one page for wrapped extmem.",
+				__func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	psKernMapData = OSAllocMem(sizeof(*psKernMapData));
+	if (psKernMapData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error, unable to allocate memory for kernel mapping data.",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	_MapPage(psWrapData,
+	         ui32PageIndex,
+	         psKernMapData);
+
+	if (psKernMapData->pvKernLinAddr == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Unable to map wrapped extmem page.",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	*ppvKernelAddressOut = ((IMG_CHAR *) psKernMapData->pvKernLinAddr) + ui32PageOffset;
+	*phHandleOut = psKernMapData;
+
+	return PVRSRV_OK;
+
+	/* error exit paths follow */
+e1:
+	OSFreeMem(psKernMapData);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataExtMem(PMR_IMPL_PRIVDATA pvPriv,
+                                              IMG_HANDLE hHandle)
+{
+	PMR_WRAP_DATA *psWrapData = (PMR_WRAP_DATA*) pvPriv;
+	WRAP_KERNEL_MAP_DATA *psKernMapData = (void*) hHandle;
+
+	_UnmapPage(psWrapData,
+	           psKernMapData);
+
+	OSFreeMem(psKernMapData);
+}
+
+static inline void begin_user_mode_access(IMG_UINT *uiState)
+{
+#if defined(CONFIG_ARM) && defined(CONFIG_CPU_SW_DOMAIN_PAN)
+	*uiState = uaccess_save_and_enable();
+#elif defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	uaccess_enable_privileged();
+#elif defined(CONFIG_X86)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	__uaccess_begin();
+#else
+	PVR_UNREFERENCED_PARAMETER(uiState);
+#endif
+
+}
+
+static inline void end_user_mode_access(IMG_UINT uiState)
+{
+#if defined(CONFIG_ARM) && defined(CONFIG_CPU_SW_DOMAIN_PAN)
+	uaccess_restore(uiState);
+#elif defined(CONFIG_ARM64) && defined(CONFIG_ARM64_SW_TTBR0_PAN)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	uaccess_disable_privileged();
+#elif defined(CONFIG_X86)
+	PVR_UNREFERENCED_PARAMETER(uiState);
+	__uaccess_end();
+#else
+	PVR_UNREFERENCED_PARAMETER(uiState);
+#endif
+}
+
+static PVRSRV_ERROR _FlushUMVirtualRange(PVRSRV_DEVICE_NODE *psDevNode,
+							PMR_WRAP_DATA *psPrivData,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_CPU_VIRTADDR pvCpuVAddr)
+{
+	struct vm_area_struct *psVMArea;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT	uiUserAccessState=0;
+
+	mmap_read_lock(current->mm);
+
+	/* Check that the recorded psVMArea matches the one associated with
+	 * this request. If not, fail the request.
+	 */
+	psVMArea = find_vma(current->mm, (uintptr_t)pvCpuVAddr);
+	if ((psVMArea != psPrivData->psVMArea) || (psVMArea == NULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Couldn't find memory region containing start address %p",
+		         __func__, (void *)pvCpuVAddr));
+		eError = PVRSRV_ERROR_INVALID_CPU_ADDR;
+		goto UMFlushUnlockReturn;
+	}
+
+
+	/*
+	 * Latest kernels enable "Privileged access never" feature in the kernel
+	 * This features leads to a privilege access failure fault (oops) whenever
+	 * the kernel tries to access any user mode address.
+	 * .
+	 * This is an additional security feature that allows kernel
+	 * to be protected against possible software attacks.
+	 * In this case, as we need to flush the cache lines associated
+	 * with the user mode virtual address, we need to explicitly disable
+	 * the feature for the duration of access and re-enable it once done.
+	 * */
+	begin_user_mode_access(&uiUserAccessState);
+	{
+		if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart = {0};
+
+			eError = CacheOpExec(psDevNode,
+								pvCpuVAddr,
+								((IMG_UINT8 *)pvCpuVAddr + uiSize),
+								sCPUPhysStart,
+								sCPUPhysStart,
+								PVRSRV_CACHE_OP_FLUSH);
+		}
+		else if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_PHYSICAL)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+			IMG_UINT i = 0;
+
+			for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+			{
+				if (NULL != psPrivData->ppsPageArray[i])
+				{
+					sCPUPhysStart.uiAddr = psPrivData->ppvPhysAddr[i].uiAddr;
+					sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+
+					eError = CacheOpExec(psDevNode,
+										NULL,
+										NULL,
+										sCPUPhysStart,
+										sCPUPhysEnd,
+										PVRSRV_CACHE_OP_FLUSH);
+					if (eError != PVRSRV_OK)
+					{
+						break;
+					}
+				}
+			}
+		}
+		else if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_BOTH)
+		{
+			IMG_CPU_PHYADDR sCPUPhysStart, sCPUPhysEnd;
+			void *pvVirtStart, *pvVirtEnd;
+			IMG_UINT i = 0;
+
+			for (i = 0; i < psPrivData->uiTotalNumPages; i++)
+			{
+				if (NULL != psPrivData->ppsPageArray[i])
+				{
+					pvVirtStart = pvCpuVAddr + (i * PAGE_SIZE);
+					pvVirtEnd = pvVirtStart + PAGE_SIZE;
+					sCPUPhysStart.uiAddr = psPrivData->ppvPhysAddr[i].uiAddr;
+					sCPUPhysEnd.uiAddr = sCPUPhysStart.uiAddr + PAGE_SIZE;
+
+					eError = CacheOpExec(psDevNode,
+										pvVirtStart,
+										pvVirtEnd,
+										sCPUPhysStart,
+										sCPUPhysEnd,
+										PVRSRV_CACHE_OP_FLUSH);
+					if (eError != PVRSRV_OK)
+					{
+						break;
+					}
+				}
+			}
+		}
+
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to clean the virtual region cache %p",
+					__func__,
+					pvCpuVAddr));
+			goto UMFlushFailed;
+		}
+	}
+
+UMFlushFailed:
+	end_user_mode_access(uiUserAccessState);
+
+UMFlushUnlockReturn:
+	mmap_read_unlock(current->mm);
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRWrapPFuncTab = {
+    .pfnLockPhysAddresses = NULL,
+    .pfnUnlockPhysAddresses = NULL,
+    .pfnDevPhysAddr = &PMRSysPhysAddrExtMem,
+    .pfnAcquireKernelMappingData = PMRAcquireKernelMappingDataExtMem,
+    .pfnReleaseKernelMappingData = PMRReleaseKernelMappingDataExtMem,
+    .pfnReadBytes = PMRReadBytesExtMem,
+    .pfnWriteBytes = PMRWriteBytesExtMem,
+    .pfnChangeSparseMem = NULL,
+    .pfnFinalize = &PMRFinalizeExtMem,
+};
+
+static inline PVRSRV_ERROR PhysmemValidateParam( IMG_DEVMEM_SIZE_T uiSize,
+                                                 IMG_CPU_VIRTADDR pvCpuVAddr,
+                                                 PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	if (!access_ok(pvCpuVAddr, uiSize))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid User mode CPU virtual address"));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* Fail if requesting coherency on one side but uncached on the other */
+	if ((PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) &&
+			(PVRSRV_CHECK_GPU_UNCACHED(uiFlags) || PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for CPU coherency but specifying GPU uncached "
+				"Please use GPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+	if ((PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags) &&
+			(PVRSRV_CHECK_CPU_UNCACHED(uiFlags) || PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Request for GPU coherency but specifying CPU uncached "
+				"Please use CPU cached flags for coherency."));
+		return PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE;
+	}
+
+#if !defined(PVRSRV_WRAP_EXTMEM_WRITE_ATTRIB_ENABLE)
+	if (uiFlags & (PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+			PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED |
+			PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED |
+			PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+			PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC
+#if defined(DEBUG)
+			| PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+#endif
+		))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Write Attribute not supported. Passed Flags: 0x%"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+				__func__, uiFlags));
+		return PVRSRV_ERROR_INVALID_FLAGS;
+	}
+#endif
+
+	/* Check address and size alignment */
+	if (uiSize & (PAGE_SIZE - 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Given size %llu is not multiple of OS page size (%lu)",
+				__func__,
+				uiSize,
+				PAGE_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (((uintptr_t) pvCpuVAddr) & (PAGE_SIZE - 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Given address %p is not aligned to OS page size (%lu)",
+				__func__,
+				pvCpuVAddr,
+				PAGE_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PhysmemWrapExtMemOS(CONNECTION_DATA * psConnection,
+                    PVRSRV_DEVICE_NODE *psDevNode,
+                    IMG_DEVMEM_SIZE_T uiSize,
+                    IMG_CPU_VIRTADDR pvCpuVAddr,
+                    PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                    PMR **ppsPMRPtr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32	*pui32MappingTable = NULL;
+	PMR_WRAP_DATA *psPrivData;
+	PMR *psPMR;
+	IMG_UINT uiTotalNumPages = (uiSize >> PAGE_SHIFT);
+	IMG_UINT i = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0))
+	/* Ignore the most significant byte. */
+	pvCpuVAddr = (IMG_CPU_VIRTADDR)untagged_addr((uintptr_t)pvCpuVAddr);
+#endif
+
+	eError = PhysmemValidateParam(uiSize,
+	                              pvCpuVAddr,
+	                              uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Allocate private factory data */
+	eError = _AllocWrapData(&psPrivData,
+	                        psDevNode,
+	                        uiSize,
+	                        pvCpuVAddr,
+	                        uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	/* Actually find and acquire the pages and physical addresses */
+	eError = _WrapExtMemAcquirePages(psDevNode,
+	                                 uiSize,
+	                                 pvCpuVAddr,
+	                                 psPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		_FreeWrapData(psPrivData);
+		goto e0;
+	}
+
+
+	pui32MappingTable = OSAllocMem(sizeof(*pui32MappingTable) * uiTotalNumPages);
+	if (pui32MappingTable == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	for (i = 0; i < uiTotalNumPages; i++)
+	{
+		pui32MappingTable[i] = i;
+	}
+
+	/* Avoid creating scratch page or zero page when the entire
+	 * allocation is backed and pinned */
+	if (psPrivData->uiNumBackedPages == psPrivData->uiTotalNumPages)
+	{
+		uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING;
+	}
+
+	/* Create a suitable PMR */
+	eError = PMRCreatePMR(psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL],
+	                      uiSize,    /* PMR_SIZE_T uiLogicalSize                  */
+	                      uiTotalNumPages,    /* IMG_UINT32 ui32NumPhysChunks              */
+	                      uiTotalNumPages,    /* IMG_UINT32 ui32NumVirtChunks              */
+	                      pui32MappingTable,
+	                      PAGE_SHIFT,           /* PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee */
+	                      (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK), /* PMR_FLAGS_T uiFlags */
+	                      "WrappedExtMem",      /* const IMG_CHAR *pszAnnotation             */
+	                      &_sPMRWrapPFuncTab,   /* const PMR_IMPL_FUNCTAB *psFuncTab         */
+	                      psPrivData,           /* PMR_IMPL_PRIVDATA pvPrivData              */
+	                      PMR_TYPE_EXTMEM,
+	                      &psPMR,               /* PMR **ppsPMRPtr                           */
+	                      PDUMP_NONE);          /* IMG_UINT32 ui32PDumpFlags                 */
+	if (eError != PVRSRV_OK)
+	{
+		goto e2;
+	}
+
+	if (PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags))
+	{
+		eError = _FlushUMVirtualRange(psDevNode,
+									psPrivData,
+									uiSize,
+									pvCpuVAddr);
+		if (eError != PVRSRV_OK)
+		{
+			goto e3;
+		}
+	}
+
+	/* Mark the PMR such that no layout changes can happen.
+	 * The memory is allocated in the CPU domain and hence
+	 * no changes can be made through any of our API */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	*ppsPMRPtr = psPMR;
+
+	OSFreeMem(pui32MappingTable);
+
+	return PVRSRV_OK;
+e3:
+	PMRUnrefPMR(psPMR);
+e2:
+	OSFreeMem(pui32MappingTable);
+e1:
+	_WrapExtMemReleasePages(psPrivData);
+e0:
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_extmem_wrap.h b/drivers/gpu/drm/img-rogue/physmem_extmem_wrap.h
new file mode 100644
index 000000000000..015360736d03
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_extmem_wrap.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File           physmem_extmem_wrap.h
+@Title          Header for wrapping non-services allocated memory PMR factory.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the support for wrapping non services allocated
+                memory into GPU space.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_EXTEMEM_WRAP_H
+#define SRVSRV_PHYSMEM_EXTEMEM_WRAP_H
+
+#include <linux/version.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+#include <linux/mm_types.h>
+#include <linux/vmalloc.h>
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <linux/atomic.h>
+
+#include "img_types.h"
+#include "device.h"
+
+typedef enum _WRAP_EXT_MEM_TYPE_
+{
+	WRAP_TYPE_NULL = 0,
+	WRAP_TYPE_GET_USER_PAGES,
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+	WRAP_TYPE_FIND_VMA
+#endif
+} WRAP_EXT_MEM_TYPE;
+
+typedef struct _PMR_WRAP_DATA_
+{
+	/* Device for which this allocation has been made */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	/* Total Number of pages in the allocation */
+	IMG_UINT32 uiTotalNumPages;
+
+	/* Total number of actual physically backed pages */
+	IMG_UINT32 uiNumBackedPages;
+
+	/* This is only filled if we have page mappings,
+	 * for pfn mappings this stays empty */
+	struct page **ppsPageArray;
+
+	/* VM Area structure reference */
+	struct vm_area_struct *psVMArea;
+
+	/* This should always be filled and hold the physical addresses */
+	IMG_CPU_PHYADDR *ppvPhysAddr;
+
+	/* Did we find the pages via get_user_pages()
+	 * or via FindVMA and a page table walk? */
+	WRAP_EXT_MEM_TYPE eWrapType;
+
+	/* CPU caching modes for the PMR mapping */
+	IMG_UINT32 ui32CPUCacheFlags;
+
+	/* Write permitted ?*/
+	IMG_BOOL bWrite;
+
+} PMR_WRAP_DATA;
+
+#if defined(SUPPORT_LINUX_WRAP_EXTMEM_PAGE_TABLE_WALK)
+/* Find the VMA to a given CPU virtual address and do a
+ * page table walk to find the corresponding pfns */
+PVRSRV_ERROR _TryFindVMA(IMG_DEVMEM_SIZE_T uiSize,
+                            uintptr_t pvCpuVAddr,
+                            PMR_WRAP_DATA *psPrivData);
+#endif
+
+#endif /* SRVSRV_PHYSMEM_EXTEMEM_WRAP_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_hostmem.c b/drivers/gpu/drm/img-rogue/physmem_hostmem.c
new file mode 100644
index 000000000000..88af3c546e4c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_hostmem.c
@@ -0,0 +1,212 @@
+/*************************************************************************/ /*!
+@File           physmem_hostmem.c
+@Title          Host memory device node functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Functions relevant to device memory allocations made from host
+                mem device node.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "physmem_hostmem.h"
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "pvrsrv_device.h"
+#include "physheap.h"
+#include "physmem_osmem.h"
+
+static void HostMemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr);
+
+static void HostMemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr);
+
+/* heap callbacks for host driver's device's heap */
+static PHYS_HEAP_FUNCTIONS gsHostMemDevPhysHeapFuncs =
+{
+	.pfnCpuPAddrToDevPAddr = HostMemCpuPAddrToDevPAddr,
+	.pfnDevPAddrToCpuPAddr = HostMemDevPAddrToCpuPAddr,
+};
+
+static PVRSRV_DEVICE_CONFIG gsHostMemDevConfig[];
+
+/* heap configuration for host driver's device */
+static PHYS_HEAP_CONFIG gsPhysHeapConfigHostMemDevice[] =
+{
+	{
+		.eType = PHYS_HEAP_TYPE_UMA,
+		.ui32UsageFlags = PHYS_HEAP_USAGE_CPU_LOCAL,
+		.uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM",
+		.uConfig.sUMA.psMemFuncs = &gsHostMemDevPhysHeapFuncs,
+		.uConfig.sUMA.pszHeapName = "uma_cpu_local",
+		.uConfig.sUMA.hPrivData = (IMG_HANDLE)&gsHostMemDevConfig[0]
+	}
+};
+
+/* device configuration for host driver's device */
+static PVRSRV_DEVICE_CONFIG gsHostMemDevConfig[] =
+{
+	{
+		.pszName = "HostMemDevice",
+		.eCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE,
+		.pasPhysHeaps = &gsPhysHeapConfigHostMemDevice[0],
+		.ui32PhysHeapCount = ARRAY_SIZE(gsPhysHeapConfigHostMemDevice),
+	}
+};
+
+static void HostMemCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+static void HostMemDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+                                      IMG_UINT32 ui32NumOfAddr,
+                                      IMG_CPU_PHYADDR *psCpuPAddr,
+                                      IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[0].uiAddr);
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr =
+				IMG_CAST_TO_CPUPHYADDR_UINT(psDevPAddr[ui32Idx].uiAddr);
+		}
+	}
+}
+
+PVRSRV_ERROR HostMemDeviceCreate(PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = &gsHostMemDevConfig[0];
+
+	/* Assert ensures HostMemory device isn't already created and
+	 * that data is initialised */
+	PVR_ASSERT(*ppsDeviceNode == NULL);
+
+	/* for now, we only know a single heap (UMA) config for host device */
+	PVR_ASSERT(psDevConfig->ui32PhysHeapCount == 1 &&
+				psDevConfig->pasPhysHeaps[0].eType == PHYS_HEAP_TYPE_UMA);
+
+	/* N.B.- In case of any failures in this function, we just return error to
+	   the caller, as clean-up is taken care by _HostMemDeviceDestroy function */
+
+	psDeviceNode = OSAllocZMem(sizeof(*psDeviceNode) + sizeof(*psDeviceNode->psMMUDevAttrs));
+	PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "OSAllocZMem");
+
+	/* early save return pointer to aid clean-up */
+	*ppsDeviceNode = psDeviceNode;
+
+	psDeviceNode->sDevId.ui32InternalID = PVRSRV_HOST_DEVICE_ID;
+	psDeviceNode->psDevConfig = psDevConfig;
+	psDeviceNode->psPhysHeapList = NULL;
+
+	eError = OSLockCreate(&psDeviceNode->hPhysHeapLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", LockCreateErr);
+
+	eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+										  &psDevConfig->pasPhysHeaps[0],
+										  NULL);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig", PhysHeapCreateErr);
+
+	/* Only CPU local heap is valid on host-mem DevNode, so enable minimal callbacks */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_CPU_LOCAL,
+								 psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire", AcquirePhysHeapErr);
+
+	psDeviceNode->psMMUDevAttrs = (MMU_DEVICEATTRIBS*)(psDeviceNode + 1);
+	psDeviceNode->psMMUDevAttrs->ui32ValidPageSizeMask = OSGetPageSize();
+
+	dllist_init(&psDeviceNode->sCleanupThreadWorkList);
+
+	return PVRSRV_OK;
+
+AcquirePhysHeapErr:
+	PhysHeapDestroyDeviceHeaps(psDeviceNode);
+PhysHeapCreateErr:
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+LockCreateErr:
+	OSFreeMem(psDeviceNode);
+
+	return eError;
+}
+
+void HostMemDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (!psDeviceNode)
+	{
+		return;
+	}
+	else
+	{
+		if (psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL])
+		{
+			PhysHeapRelease(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL]);
+		}
+
+		PhysHeapDestroyDeviceHeaps(psDeviceNode);
+	}
+
+	OSLockDestroy(psDeviceNode->hPhysHeapLock);
+
+	OSFreeMem(psDeviceNode);
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_hostmem.h b/drivers/gpu/drm/img-rogue/physmem_hostmem.h
new file mode 100644
index 000000000000..cfa453de343a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_hostmem.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File           physmem_hostmem.h
+@Title          Host memory device node header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PHYSMEM_HOSTMEM_H)
+#define PHYSMEM_HOSTMEM_H
+
+#include "pvrsrv_device.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       HostMemDeviceCreate
+@Description    Allocate memory for and create host memory device node.
+@Output         ppsDeviceNode Pointer to device node pointer.
+@Return         PVRSRV_ERROR  PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR HostMemDeviceCreate(PVRSRV_DEVICE_NODE **ppsDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       HostMemDeviceDestroy
+@Description    Destroy host memory device node.
+@Input          psDeviceNode  Pointer to device node.
+*/ /**************************************************************************/
+void HostMemDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* !defined(PHYSMEM_HOSTMEM_H) */
diff --git a/drivers/gpu/drm/img-rogue/physmem_ima.h b/drivers/gpu/drm/img-rogue/physmem_ima.h
new file mode 100644
index 000000000000..4744dacacb31
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_ima.h
@@ -0,0 +1,77 @@
+/**************************************************************************/ /*!
+@File
+@Title          Header import memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of memory management. This module is responsible for
+                implementing the function callbacks for local card memory when
+                used under a shared heap system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PHYSMEM_IMA_H
+#define PHYSMEM_IMA_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapIMA
+@Description    Create and register new IMA heap with IMA specific details and
+                a DLM heap backing.
+@Input          psDevNode     Pointer to device node struct.
+@Input          uiPolicy      Heap allocation policy flags
+@Input          psConfig      Heap configuration.
+@Input          pszLabel      Debug identifier label
+@Input          psDLMHeap     DLM heap backing this LMA heap.
+@Input          uiLog2PMBSize Log 2 of PMB Size in bytes supported by DLM heap.
+@Output         ppsPhysHeap   Pointer to the created heap.
+@Return         PVRSRV_ERROR  PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapIMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP *psDLMHeap,
+                     IMG_UINT32 uiLog2PMBSize,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* PHYSMEM_IMA_H_ */
diff --git a/drivers/gpu/drm/img-rogue/physmem_lma.c b/drivers/gpu/drm/img-rogue/physmem_lma.c
new file mode 100644
index 000000000000..6a67f3673978
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_lma.c
@@ -0,0 +1,523 @@
+/*************************************************************************/ /*!
+@File           physmem_lma.c
+@Title          Local card memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for local card memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "physmem_lma.h"
+#include "pvr_debug.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "physheap.h"
+#include "physheap_config.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "device.h"
+#include "osfunc.h"
+#include "physmem_ramem.h"
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "rgxutils.h"
+#endif
+
+typedef struct PHYSMEM_LMA_DATA_TAG {
+	RA_ARENA           *psRA;
+	IMG_CPU_PHYADDR    sStartAddr;
+	IMG_DEV_PHYADDR    sCardBase;
+	IMG_UINT64         uiSize;
+} PHYSMEM_LMA_DATA;
+
+/*
+ * This function will set the psDevPAddr to whatever the system layer
+ * has set it for the referenced heap.
+ * It will not fail if the psDevPAddr is invalid.
+ */
+static PVRSRV_ERROR
+_GetDevPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*psDevPAddr = psLMAData->sCardBase;
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This function will set the psCpuPAddr to whatever the system layer
+ * has set it for the referenced heap.
+ * It will not fail if the psCpuPAddr is invalid.
+ */
+static PVRSRV_ERROR
+_GetCPUPAddr(PHEAP_IMPL_DATA pvImplData,
+			 IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*psCpuPAddr = psLMAData->sStartAddr;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_GetSize(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *puiSize)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	*puiSize = psLMAData->uiSize;
+
+	return PVRSRV_OK;
+}
+
+static void PhysmemGetRAMemRamMemStats(PHEAP_IMPL_DATA pvImplData,
+		 IMG_UINT64 *pui64TotalSize,
+		 IMG_UINT64 *pui64FreeSize)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+	RA_USAGE_STATS sRAUsageStats;
+
+	RA_Get_Usage_Stats(psLMAData->psRA, &sRAUsageStats);
+
+	*pui64TotalSize = sRAUsageStats.ui64TotalArenaSize;
+	*pui64FreeSize = sRAUsageStats.ui64FreeArenaSize;
+}
+
+static PVRSRV_ERROR
+PhysmemGetArenaLMA(PHYS_HEAP *psPhysHeap,
+				   RA_ARENA **ppsArena)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)PhysHeapGetImplData(psPhysHeap);
+
+	PVR_LOG_RETURN_IF_FALSE(psLMAData != NULL, "psLMAData", PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	*ppsArena = psLMAData->psRA;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_CreateArenas(PHEAP_IMPL_DATA pvImplData, IMG_CHAR *pszLabel, PHYS_HEAP_POLICY uiPolicy)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	IMG_UINT32 ui32RAPolicy =
+	    ((uiPolicy & PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	    ? RA_POLICY_ALLOC_ALLOW_NONCONTIG : RA_POLICY_DEFAULT;
+
+	psLMAData->psRA = RA_Create_With_Span(pszLabel,
+	                             OSGetPageShift(),
+	                             psLMAData->sStartAddr.uiAddr,
+	                             psLMAData->sCardBase.uiAddr,
+	                             psLMAData->uiSize,
+	                             ui32RAPolicy);
+	PVR_LOG_RETURN_IF_NOMEM(psLMAData->psRA, "RA_Create_With_Span");
+
+	return PVRSRV_OK;
+}
+
+static void
+_DestroyArenas(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	/* Remove RAs and RA names for local card memory */
+	if (psLMAData->psRA)
+	{
+		RA_Delete(psLMAData->psRA);
+		psLMAData->psRA = NULL;
+	}
+}
+
+static void
+_DestroyImplData(PHEAP_IMPL_DATA pvImplData)
+{
+	PHYSMEM_LMA_DATA *psLMAData = (PHYSMEM_LMA_DATA*)pvImplData;
+
+	_DestroyArenas(pvImplData);
+
+	OSFreeMem(psLMAData);
+}
+
+struct _PHYS_HEAP_ITERATOR_ {
+	PHYS_HEAP *psPhysHeap;
+	RA_ARENA_ITERATOR *psRAIter;
+
+	IMG_UINT64 uiTotalSize;
+	IMG_UINT64 uiInUseSize;
+};
+
+PVRSRV_ERROR LMA_HeapIteratorCreate(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_PHYS_HEAP ePhysHeap,
+                                    PHYS_HEAP_ITERATOR **ppsIter)
+{
+	PVRSRV_ERROR eError;
+	PHYSMEM_LMA_DATA *psLMAData;
+	PHYS_HEAP_ITERATOR *psHeapIter;
+	PHYS_HEAP *psPhysHeap = NULL;
+	RA_USAGE_STATS sStats;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppsIter != NULL, "ppsIter");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode != NULL, "psDevNode");
+
+	eError = PhysHeapAcquireByID(ePhysHeap, psDevNode, &psPhysHeap);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquireByID");
+
+	PVR_LOG_GOTO_IF_FALSE(PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA,
+	                      "PhysHeap must be of LMA type", release_heap);
+
+	psLMAData = (PHYSMEM_LMA_DATA *) PhysHeapGetImplData(psPhysHeap);
+
+	psHeapIter = OSAllocMem(sizeof(*psHeapIter));
+	PVR_LOG_GOTO_IF_NOMEM(psHeapIter, eError, release_heap);
+
+	psHeapIter->psPhysHeap = psPhysHeap;
+	psHeapIter->psRAIter = RA_IteratorAcquire(psLMAData->psRA, IMG_FALSE);
+	PVR_LOG_GOTO_IF_NOMEM(psHeapIter->psRAIter, eError, free_heap_iter);
+
+	/* get heap usage */
+	RA_Get_Usage_Stats(psLMAData->psRA, &sStats);
+
+	psHeapIter->uiTotalSize = sStats.ui64TotalArenaSize;
+	psHeapIter->uiInUseSize = sStats.ui64TotalArenaSize - sStats.ui64FreeArenaSize;
+
+	*ppsIter = psHeapIter;
+
+	return PVRSRV_OK;
+
+free_heap_iter:
+	OSFreeMem(psHeapIter);
+
+release_heap:
+	PhysHeapRelease(psPhysHeap);
+
+	return eError;
+}
+
+void LMA_HeapIteratorDestroy(PHYS_HEAP_ITERATOR *psIter)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psHeapIter != NULL, "psHeapIter is NULL");
+
+	PhysHeapRelease(psHeapIter->psPhysHeap);
+	RA_IteratorRelease(psHeapIter->psRAIter);
+	OSFreeMem(psHeapIter);
+}
+
+PVRSRV_ERROR LMA_HeapIteratorReset(PHYS_HEAP_ITERATOR *psIter)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeapIter != NULL, "ppsIter");
+
+	RA_IteratorReset(psHeapIter->psRAIter);
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL LMA_HeapIteratorNext(PHYS_HEAP_ITERATOR *psIter,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              IMG_UINT64 *puiSize)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+	RA_ITERATOR_DATA sData = {0};
+
+	if (psHeapIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "psHeapIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	if (!RA_IteratorNext(psHeapIter->psRAIter, &sData))
+	{
+		return IMG_FALSE;
+	}
+
+	PVR_ASSERT(sData.uiSize != 0);
+
+	psDevPAddr->uiAddr = sData.uiAddr;
+	*puiSize = sData.uiSize;
+
+	return IMG_TRUE;
+}
+
+PVRSRV_ERROR LMA_HeapIteratorGetHeapStats(PHYS_HEAP_ITERATOR *psIter,
+                                          IMG_UINT64 *puiTotalSize,
+                                          IMG_UINT64 *puiInUseSize)
+{
+	PHYS_HEAP_ITERATOR *psHeapIter = psIter;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psHeapIter != NULL, "psHeapIter");
+
+	*puiTotalSize = psHeapIter->uiTotalSize;
+	*puiInUseSize = psHeapIter->uiInUseSize;
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+static PVRSRV_ERROR
+LMA_PhyContigPagesAllocGPV(PHYS_HEAP *psPhysHeap,
+                           size_t uiSize,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_UINT32 ui32OSid,
+                           IMG_PID uiPid)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	PVR_ASSERT(ui32OSid < GPUVIRT_VALIDATION_NUM_OS);
+	if (ui32OSid >= GPUVIRT_VALIDATION_NUM_OS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Arena index %u defaulting to 0",
+		        __func__, ui32OSid));
+		ui32OSid = 0;
+	}
+
+	pArena = psDevNode->psOSidSubArena[ui32OSid];
+
+	if (psMemHandle->uiOSid != ui32OSid)
+	{
+		PVR_LOG(("%s: Unexpected OSid value %u - expecting %u", __func__,
+		        psMemHandle->uiOSid, ui32OSid));
+	}
+
+	psMemHandle->uiOSid = ui32OSid;		/* For Free() use */
+
+	eError =  RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                     psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "RAMemDoPhyContigPagesAlloc");
+
+	return eError;
+}
+#endif
+
+static PVRSRV_ERROR
+LMA_PhyContigPagesAlloc(PHYS_HEAP *psPhysHeap,
+                         size_t uiSize,
+                         PG_HANDLE *psMemHandle,
+                         IMG_DEV_PHYADDR *psDevPAddr,
+                         IMG_PID uiPid)
+{
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	IMG_UINT32 ui32OSid = 0;
+	return LMA_PhyContigPagesAllocGPV(psPhysHeap, uiSize, psMemHandle, psDevPAddr,
+									  ui32OSid, uiPid);
+#else
+	PVRSRV_ERROR eError;
+
+	RA_ARENA *pArena;
+	IMG_UINT32 ui32Log2NumPages = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+
+	eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLMA");
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	eError = RAMemDoPhyContigPagesAlloc(pArena, uiSize, psDevNode, psMemHandle,
+	                                    psDevPAddr, uiPid);
+	PVR_LOG_IF_ERROR(eError, "RAMemDoPhyContigPagesAlloc");
+
+	return eError;
+#endif
+}
+
+static void
+LMA_PhyContigPagesFree(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle)
+{
+	RA_ARENA	*pArena;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	RA_BASE_T uiCardAddr = (RA_BASE_T) psMemHandle->u.ui64Handle;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	IMG_UINT32	ui32OSid = psMemHandle->uiOSid;
+
+	/*
+	 * The Arena ID is set by the originating allocation, and maintained via
+	 * the call stacks into this function. We have a limited range of IDs
+	 * and if the passed value falls outside this we simply treat it as a
+	 * 'global' arena ID of 0. This is where all default OS-specific allocations
+	 * are created.
+	 */
+	PVR_ASSERT(ui32OSid < GPUVIRT_VALIDATION_NUM_OS);
+	if (ui32OSid >= GPUVIRT_VALIDATION_NUM_OS)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid Arena index %u PhysAddr 0x%"
+		         IMG_UINT64_FMTSPECx " Reverting to Arena 0", __func__,
+		         ui32OSid, uiCardAddr));
+		/*
+		 * No way of determining what we're trying to free so default to the
+		 * global default arena index 0.
+		 */
+		ui32OSid = 0;
+	}
+
+	pArena = psDevNode->psOSidSubArena[ui32OSid];
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: (GPU Virtualisation) Freeing 0x%"
+	        IMG_UINT64_FMTSPECx ", Arena %u", __func__,
+	        uiCardAddr, ui32OSid));
+
+#else
+	PhysmemGetArenaLMA(psPhysHeap, &pArena);
+#endif
+
+	RAMemDoPhyContigPagesFree(pArena,
+	                          psMemHandle);
+}
+
+static PVRSRV_ERROR
+LMAPhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                               CONNECTION_DATA *psConnection,
+                               IMG_DEVMEM_SIZE_T uiSize,
+                               IMG_UINT32 ui32NumPhysChunks,
+                               IMG_UINT32 ui32NumVirtChunks,
+                               IMG_UINT32 *pui32MappingTable,
+                               IMG_UINT32 uiLog2AllocPageSize,
+                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                               const IMG_CHAR *pszAnnotation,
+                               IMG_PID uiPid,
+                               PMR **ppsPMRPtr,
+                               IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	RA_ARENA *pArena;
+
+	eError = PhysmemGetArenaLMA(psPhysHeap, &pArena);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemGetArenaLMA");
+
+	eError = PhysmemNewRAMemRamBackedPMR(psPhysHeap,
+	                                     pArena,
+	                                     psConnection,
+	                                     uiSize,
+	                                     ui32NumPhysChunks,
+	                                     ui32NumVirtChunks,
+	                                     pui32MappingTable,
+	                                     uiLog2AllocPageSize,
+	                                     uiFlags,
+	                                     pszAnnotation,
+	                                     uiPid,
+	                                     ppsPMRPtr,
+	                                     ui32PDumpFlags);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewLocalRamBackedPMR");
+
+	return PVRSRV_OK;
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = &_DestroyImplData,
+	.pfnGetDevPAddr = &_GetDevPAddr,
+	.pfnGetCPUPAddr = &_GetCPUPAddr,
+	.pfnGetSize = &_GetSize,
+	.pfnGetPageShift = &RAMemGetPageShift,
+	.pfnGetFactoryMemStats = &PhysmemGetRAMemRamMemStats,
+	.pfnCreatePMR = &LMAPhysmemNewRAMemRamBackedPMR,
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	.pfnPagesAllocGPV = &LMA_PhyContigPagesAllocGPV,
+#endif
+	.pfnPagesAlloc = &LMA_PhyContigPagesAlloc,
+	.pfnPagesFree = &LMA_PhyContigPagesFree,
+	.pfnPagesMap = &RAMemPhyContigPagesMap,
+	.pfnPagesUnMap = &RAMemPhyContigPagesUnmap,
+	.pfnPagesClean = &RAMemPhyContigPagesClean,
+};
+
+PVRSRV_ERROR
+PhysmemCreateHeapLMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap)
+{
+	PHYSMEM_LMA_DATA *psLMAData;
+	PHYS_HEAP *psPhysHeap;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszLabel != NULL, "pszLabel");
+
+	PVR_ASSERT(psConfig->eType == PHYS_HEAP_TYPE_LMA ||
+	           psConfig->eType == PHYS_HEAP_TYPE_DMA);
+
+	psLMAData = OSAllocMem(sizeof(*psLMAData));
+	PVR_LOG_RETURN_IF_NOMEM(psLMAData, "OSAllocMem");
+
+	psLMAData->sStartAddr = PhysHeapConfigGetStartAddr(psConfig);
+	psLMAData->sCardBase = PhysHeapConfigGetCardBase(psConfig);
+	psLMAData->uiSize = PhysHeapConfigGetSize(psConfig);
+
+	eError = PhysHeapCreate(psDevNode,
+							psConfig,
+							uiPolicy,
+							(PHEAP_IMPL_DATA)psLMAData,
+							&_sPHEAPImplFuncs,
+							&psPhysHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapCreate", err_free_lma_data);
+
+	eError = _CreateArenas(psLMAData, pszLabel, uiPolicy);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CreateArenas", err_free_physheap);
+
+	if (ppsPhysHeap != NULL)
+	{
+		*ppsPhysHeap = psPhysHeap;
+	}
+
+	return PVRSRV_OK;
+
+err_free_physheap:
+	PhysHeapDestroy(psPhysHeap);
+err_free_lma_data:
+	OSFreeMem(psLMAData);
+	return eError;
+
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_lma.h b/drivers/gpu/drm/img-rogue/physmem_lma.h
new file mode 100644
index 000000000000..af42d8826dac
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_lma.h
@@ -0,0 +1,75 @@
+/**************************************************************************/ /*!
+@File
+@Title          Header for local card memory allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This module is responsible for
+                implementing the function callbacks for local card memory.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_LMA_H
+#define SRVSRV_PHYSMEM_LMA_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "device.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapLMA
+@Description    Create and register new LMA heap with LMA specific details.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Input          pszLabel     Debug identifier label
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapLMA(PVRSRV_DEVICE_NODE *psDevNode,
+                     PHYS_HEAP_POLICY uiPolicy,
+                     PHYS_HEAP_CONFIG *psConfig,
+                     IMG_CHAR *pszLabel,
+                     PHYS_HEAP **ppsPhysHeap);
+
+#endif /* #ifndef SRVSRV_PHYSMEM_LMA_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_osmem.c b/drivers/gpu/drm/img-rogue/physmem_osmem.c
new file mode 100644
index 000000000000..cef1ea620821
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_osmem.c
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File           physmem_osmem.c
+@Title          OS Memory PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                OS memory PMR factory API that must be defined so that the
+                common & device layer code in the Services Server can allocate
+                new PMRs back with pages from the OS page allocator. Applicable
+                for UMA based platforms, such platforms must implement this API
+                in the OS Porting layer, in the "env" directory for that
+                system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/* services/server/include/ */
+#include "physheap.h"
+#include "osfunc.h"
+#include "physmem_osmem.h"
+
+
+static IMG_UINT32 PhysHeapOSGetPageShift(void)
+{
+	return (IMG_UINT32)OSGetPageShift();
+}
+
+static PHEAP_IMPL_FUNCS _sPHEAPImplFuncs =
+{
+	.pfnDestroyData = NULL,
+	.pfnGetFactoryMemStats = PhysmemGetOSRamMemStats,
+	.pfnCreatePMR = PhysmemNewOSRamBackedPMR,
+	.pfnPagesAlloc = &OSPhyContigPagesAlloc,
+	.pfnPagesFree = &OSPhyContigPagesFree,
+	.pfnPagesMap = &OSPhyContigPagesMap,
+	.pfnPagesUnMap = &OSPhyContigPagesUnmap,
+	.pfnPagesClean = &OSPhyContigPagesClean,
+	.pfnGetPageShift = &PhysHeapOSGetPageShift,
+};
+
+
+PVRSRV_ERROR
+PhysmemCreateHeapOSMEM(PVRSRV_DEVICE_NODE *psDevNode,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       PHYS_HEAP_CONFIG *psConfig,
+                       PHYS_HEAP **ppsPhysHeap)
+{
+	return PhysHeapCreate(psDevNode,
+	                      psConfig,
+	                      uiPolicy,
+	                      NULL,
+	                      &_sPHEAPImplFuncs,
+	                      ppsPhysHeap);
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_osmem.h b/drivers/gpu/drm/img-rogue/physmem_osmem.h
new file mode 100644
index 000000000000..5e76513477b6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_osmem.h
@@ -0,0 +1,151 @@
+/*************************************************************************/ /*!
+@File           physmem_osmem.h
+@Title          OS memory PMR factory API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                OS memory PMR factory API that must be defined so that the
+                common & device layer code in the Services Server can allocate
+                new PMRs back with pages from the OS page allocator. Applicable
+                for UMA based platforms, such platforms must implement this API
+                in the OS Porting layer, in the "env" directory for that
+                system.
+
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_OSMEM_H
+#define PHYSMEM_OSMEM_H
+
+/* include/ */
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+
+/* services/server/include/ */
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "connection_server.h"
+#include "physheap.h"
+
+/*************************************************************************/ /*!
+@Function       PhysmemCreateHeapOSMEM
+@Description    Create and register new OSMEM heap with OSMEM specific details.
+@Input          psDevNode    Pointer to device node struct.
+@Input          uiPolicy     Heap allocation policy flags
+@Input          psConfig     Heap configuration.
+@Output         ppsPhysHeap  Pointer to the created heap.
+@Return         PVRSRV_ERROR PVRSRV_OK or error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemCreateHeapOSMEM(PVRSRV_DEVICE_NODE *psDevNode,
+                       PHYS_HEAP_POLICY uiPolicy,
+                       PHYS_HEAP_CONFIG *psConfig,
+                       PHYS_HEAP **ppsPhysHeap);
+
+/*************************************************************************/ /*!
+@Function       PhysmemNewOSRamBackedPMR
+@Description    Rogue Services will call this function to allocate GPU device
+                memory from the PMR factory supported by the OS DDK port. This
+                factory typically obtains physical memory from the kernel/OS
+                API that allocates memory from the default heap of shared
+                system memory available on the platform. The allocated memory
+                must be page-aligned and be a whole number of pages.
+                After allocating the required memory, the implementation must
+                then call PMRCreatePMR() to obtain the PMR structure that
+                describes this allocation to the upper layers of the Services.
+                memory management sub-system.
+                NB. Implementation of this function is mandatory. If shared
+                system memory is not to be used in the OS port then the
+                implementation must return PVRSRV_ERROR_NOT_SUPPORTED.
+
+@Input          psPhysHeap       The phys heap
+@Input          psConnection     The connection to the originator process
+@Input          uiSize           The size of the allocation
+                                 (must be a multiple of page size)
+@Input          ui32NumPhysChunks     When sparse allocations are requested,
+                                      this is the number of physical chunks
+                                      to be allocated.
+                                      For regular allocations, this will be 1.
+@Input          ui32NumLogicalChunks  When sparse allocations are requested,
+                                      this is the number of logical chunks
+                                      covering the sparse allocation.
+                                      For regular allocations, this will be 1.
+@Input          pui32MappingTable  When sparse allocations are requested,
+                                   this is the list of the indices of
+                                   each physically-backed virtual chunk
+                                   For regular allocations, this will
+                                   be NULL.
+@Input          uiLog2DevPageSize  The physical dev pagesize in log2(bytes).
+@Input          uiFlags            The allocation flags.
+@Input          pszAnnotation      String describing the PMR (for debug).
+                                   This should be passed into the function
+                                   PMRCreatePMR().
+@Input          uiPid              The process ID that this allocation should
+                                   be associated with.
+@Output         ppsPMROut          Pointer to the PMR created for the
+                                   new allocation
+@Input          ui32PDumpFlags     The pdump flags.
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PhysmemNewOSRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                         CONNECTION_DATA *psConnection,
+                         IMG_DEVMEM_SIZE_T uiSize,
+                         IMG_UINT32 ui32NumPhysChunks,
+                         IMG_UINT32 ui32NumLogicalChunks,
+                         IMG_UINT32 *pui32MappingTable,
+                         IMG_UINT32 uiLog2DevPageSize,
+                         PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                         const IMG_CHAR *pszAnnotation,
+                         IMG_PID uiPid,
+                         PMR **ppsPMROut,
+                         IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PhysmemGetOSRamMemStats
+@Description    Function that gets the OS memory usage statistics
+@Input          pvImplData     Physical heap private data.
+@Output         pui64TotalSize Buffer that holds the total OS memory size
+@Output         pui64FreeSize  Buffer that holds the free OS memory size
+@Return         None.
+*/ /**************************************************************************/
+void PhysmemGetOSRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                            IMG_UINT64 *pui64TotalSize,
+                            IMG_UINT64 *pui64FreeSize);
+
+#endif /* PHYSMEM_OSMEM_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_osmem_linux.c b/drivers/gpu/drm/img-rogue/physmem_osmem_linux.c
new file mode 100644
index 000000000000..834282760fef
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_osmem_linux.c
@@ -0,0 +1,4655 @@
+/*************************************************************************/ /*!
+@File
+@Title          Implementation of PMR functions for OS managed memory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                implementing the function callbacks for physical memory borrowed
+                from that normally managed by the operating system.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include <linux/version.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+#include <linux/mm_types.h>
+#include <linux/vmalloc.h>
+#include <linux/gfp.h>
+#include <linux/sched.h>
+#include <linux/atomic.h>
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#include <linux/migrate.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+#include <drm/drm_file.h>
+#endif
+#endif
+
+#if defined(CONFIG_X86)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0))
+#include <asm/set_memory.h>
+#else
+#include <asm/cacheflush.h>
+#endif
+#endif
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "rgx_pdump_panics.h"
+/* services/server/include/ */
+#include "allocmem.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "cache_km.h"
+#include "devicemem_server_utils.h"
+#include "pvr_vmap.h"
+#include "physheap.h"
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+#include "physmem_cpumap_history.h"
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+/* services/server/env/linux */
+#include "env_connection.h"
+#endif
+#endif
+
+/* ourselves */
+#include "physmem_osmem.h"
+#include "physmem_osmem_linux.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+#include "hash.h"
+#endif
+#endif
+
+#include "kernel_compatibility.h"
+
+
+static IMG_UINT32 g_uiMaxOrder = PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM;
+
+
+/* Defines how many pages should be mapped at once to the kernel */
+#define PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES 1024 /* 4 MB */
+
+/*
+	These are used to get/set/mask lower-order bits in a dma_addr_t
+	to provide side-band information associated with that address.
+	These includes whether the address was obtained via alloc_page
+	or dma_alloc and if address came allocated pre-aligned or an
+	adjustment was made manually to aligned it. They can also indicate
+	a page entry in the array has been generated by us for ease of mapping
+	and is part of a higher order page, this is known as a CMA Ghost.
+*/
+#define DMA_SET_CMA_GHOST(x)			((x) | ((dma_addr_t)0x03))
+#define DMA_IS_CMA_GHOST(x)				(((x) & ((dma_addr_t)0x03)) == 0x03)
+#define DMA_SET_ADJUSTED_ADDR(x)		((x) | ((dma_addr_t)0x02))
+#define DMA_IS_ADDR_ADJUSTED(x)			(((x) & ((dma_addr_t)0x02)) == 0x02)
+#define DMA_SET_ALLOCPG_ADDR(x)			((x) | ((dma_addr_t)0x01))
+#define DMA_IS_ALLOCPG_ADDR(x)			(((x) & ((dma_addr_t)0x01)) == 0x01)
+#define DMA_GET_ALIGN_ADJUSTMENT(x)		((x>>2) & ((dma_addr_t)0x3ff))
+#define DMA_SET_ALIGN_ADJUSTMENT(x,y)	((x) | (((dma_addr_t)y)<<0x02))
+#define DMA_GET_ADDR(x)					(((dma_addr_t)x) & ((dma_addr_t)~0xfff))
+#define DMA_VADDR_NOT_IN_USE			0xCAFEF00DDEADBEEFULL
+
+#define PVRSRV_ZERO_VALUE 0
+
+typedef struct _PMR_OSPAGEARRAY_DATA_ {
+	/* Device for which this allocation has been made */
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+
+	/* DRM Node associated with connection that made the allocation */
+	struct drm_file *psDRMFile;
+#endif
+	/* How many pages in the PMR are currently in process of migration */
+	ATOMIC_T iNumOSPagesUnderMigration;
+#endif
+
+	/* The pid that made this allocation */
+	IMG_PID uiPid;
+
+	/*
+	 * iNumOSPagesAllocated:
+	 * Number of pages allocated in this PMR so far.
+	 * This allows for up to (2^31 - 1) pages. With 4KB pages, that's 8TB of memory for each PMR.
+	 */
+	IMG_INT32 iNumOSPagesAllocated;
+
+	/*
+	 * uiTotalNumOSPages:
+	 * Total number of pages supported by this PMR. (Fixed as of now due the fixed Page table array size)
+	 *  number of "pages" (a.k.a. macro pages, compound pages, higher order pages, etc...)
+	 */
+	IMG_UINT32 uiTotalNumOSPages;
+
+	/*
+	  uiLog2DevPageSize;
+
+	  size of each requested device "page" -- this would normally be the same as
+	  PAGE_SHIFT, but we support the idea that we may allocate pages
+	  in larger chunks for better contiguity, using order>0 in the
+	  call to alloc_pages()
+	*/
+	IMG_UINT32 uiLog2DevPageSize;
+
+	/*
+	  ui64DmaMask;
+	*/
+	IMG_UINT64 ui64DmaMask;
+
+	/*
+	  For non DMA/CMA allocation, pagearray references the pages
+	  thus allocated; one entry per compound page when compound
+	  pages are used. In addition, for DMA/CMA allocations, we
+	  track the returned cpu virtual and device bus address.
+	*/
+	struct page **pagearray;
+	dma_addr_t *dmaphysarray;
+	void **dmavirtarray;
+
+
+#define FLAG_ZERO              (0U)
+#define FLAG_POISON_ON_FREE    (1U)
+#define FLAG_POISON_ON_ALLOC   (2U)
+#define FLAG_ONDEMAND          (3U)
+
+/* IS CMA ensures the allocation will go through the dma_alloc_coherent path */
+#define FLAG_DMA_CMA           (5U)
+#define FLAG_UNSET_MEMORY_TYPE (6U)
+
+#define FLAG_IS_ZOMBIE         (7U)
+/* Pref CMA appends a flag (if exists) to hint alloc_pages should source from CMA region */
+#define FLAG_PREF_CMA          (8U)
+#define FLAG_IS_MOVABLE        (9U)
+
+	/*
+	 * Allocation flags related to the pages:
+	 * Zero              - Should we Zero memory on alloc
+	 * Poison on free    - Should we Poison the memory on free.
+	 * Poison on alloc   - Should we Poison the memory on alloc.
+	 * On demand         - Is the allocation on Demand i.e Do we defer allocation to time of use.
+	 * CMA               - Is CMA memory allocated via DMA framework
+	 * Unset Memory Type - Upon free do we need to revert the cache type before return to OS
+	 * Zombie            - Pages are part of a zombie PMR
+	 * */
+	IMG_UINT32 ui32AllocFlags;
+
+	/*
+	  The cache mode of the PMR. Additionally carrying the CPU-Cache-Clean
+	  flag, advising us to do cache maintenance on behalf of the caller.
+	  Boolean used to track if we need to revert the cache attributes
+	  of the pages used in this allocation. Depends on OS/architecture.
+	*/
+	IMG_UINT32 ui32CPUCacheFlags;
+	/*
+	 * In CMA allocation path, algorithm can allocate double the size of
+	 * requested allocation size to satisfy the alignment. In this case
+	 * the additional pages allocated are tracked through this additional
+	 * variable and are accounted for in the memory statistics */
+	IMG_UINT32 ui32CMAAdjustedPageCount;
+
+	/*
+	  Handle on the parent PMR
+	 */
+	PMR *hPMR;
+
+} PMR_OSPAGEARRAY_DATA;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+typedef struct _OSMEM_PAGE_PRIVDATA_
+{
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	/* Store index of entry to speed up migration */
+	IMG_UINT32 uiPMRArrIdx;
+} OSMEM_PAGE_PRIVDATA;
+#endif
+
+/***********************************
+ * Page pooling for uncached pages *
+ ***********************************/
+
+static INLINE void
+_FreeOSPage_CMA(struct device *dev,
+				size_t alloc_size,
+				IMG_UINT32 uiOrder,
+				void *virt_addr,
+				dma_addr_t dev_addr,
+				struct page *psPage);
+
+static void
+_FreeOSPage(IMG_UINT32 uiOrder,
+			IMG_BOOL bUnsetMemoryType,
+			PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			struct page *psPage);
+
+static PVRSRV_ERROR
+_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			IMG_UINT32 *pai32FreeIndices,
+			IMG_UINT32 ui32FreePageCount);
+
+static PVRSRV_ERROR
+_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree,
+						   IMG_UINT32 *puiPagesFreed);
+
+static inline PVRSRV_ERROR
+_ApplyOSPagesAttribute(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages,
+					   IMG_BOOL bFlush,
+					   IMG_UINT32 ui32CPUCacheFlags);
+
+/* A struct for our page pool holding an array of zeroed (!) pages.
+ * We always put units of page arrays to the pool but are
+ * able to take individual pages */
+typedef	struct
+{
+	/* Linkage for page pool LRU list */
+	struct list_head sPagePoolItem;
+
+	/* How many items are still in the page array */
+	IMG_UINT32 uiItemsRemaining;
+	/* Array of the actual pages */
+	struct page **ppsPageArray;
+
+} LinuxPagePoolEntry;
+
+/* CleanupThread structure to put allocation in page pool */
+typedef struct
+{
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupWork;
+	IMG_UINT32 ui32CPUCacheMode;
+	LinuxPagePoolEntry *psPoolEntry;
+} LinuxCleanupData;
+
+
+/* Caches to hold page pool and page array structures */
+static struct kmem_cache *g_psLinuxPagePoolCache;
+static struct kmem_cache *g_psLinuxPageArray;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+static struct kmem_cache *g_psLinuxPagePrivateData;
+#endif
+
+/* Track what is live, all protected by pool lock.
+ * x86 needs two page pools because we have to change the memory attributes
+ * of the pages which is expensive due to an implicit flush.
+ * See set_pages_array_uc/wc/wb. */
+static IMG_UINT32 g_ui32PagePoolUCCount;
+#if defined(CONFIG_X86)
+static IMG_UINT32 g_ui32PagePoolWCCount;
+#endif
+/* Tracks asynchronous tasks currently accessing the page pool.
+ * It is incremented if a defer free task
+ * is created. Both will decrement the value when they finished the work.
+ * The atomic prevents piling up of deferred work in case the deferred thread
+ * cannot keep up with the application.*/
+static ATOMIC_T g_iPoolCleanTasks;
+/* We don't want too many asynchronous threads trying to access the page pool
+ * at the same time */
+#define PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS 128
+
+/* Defines how many pages the page cache should hold. */
+#if defined(PVR_LINUX_PHYSMEM_MAX_POOL_PAGES)
+static const IMG_UINT32 g_ui32PagePoolMaxEntries = PVR_LINUX_PHYSMEM_MAX_POOL_PAGES;
+#else
+static const IMG_UINT32 g_ui32PagePoolMaxEntries;
+#endif
+
+/*	We double check if we would exceed this limit if we are below MAX_POOL_PAGES
+	and want to add an allocation to the pool.
+	This prevents big allocations being given back to the OS just because they
+	exceed the MAX_POOL_PAGES limit even though the pool is currently empty. */
+#if defined(PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES)
+static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries = PVR_LINUX_PHYSMEM_MAX_EXCESS_POOL_PAGES;
+#else
+static const IMG_UINT32 g_ui32PagePoolMaxExcessEntries;
+#endif
+
+#if defined(CONFIG_X86)
+#define PHYSMEM_OSMEM_NUM_OF_POOLS 2
+static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = {
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED,
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC
+};
+#else
+#define PHYSMEM_OSMEM_NUM_OF_POOLS 1
+static const IMG_UINT32 g_aui32CPUCacheFlags[PHYSMEM_OSMEM_NUM_OF_POOLS] = {
+	PVRSRV_MEMALLOCFLAG_CPU_UNCACHED
+};
+#endif
+
+/* Global structures we use to manage the page pool */
+static DEFINE_MUTEX(g_sPagePoolMutex);
+
+/* List holding the page array pointers: */
+static LIST_HEAD(g_sPagePoolList_WC);
+static LIST_HEAD(g_sPagePoolList_UC);
+
+
+static IMG_BOOL g_bInitialisedOnAlloc = IMG_FALSE;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+static bool OSMemPageIsolate(struct page *psPage, isolate_mode_t uIsolateMode);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static int OSMemPageMigrate(struct address_space *psAddrSpace,
+		struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode);
+#else
+static int OSMemPageMigrate(struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode);
+#endif
+static void OSMemPagePutback(struct page *psPage);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION_DEBUG)
+#define MIGRATE_DBG_LOG(x) PVR_DPF(x);
+#else
+#define MIGRATE_DBG_LOG(...)
+#endif
+
+static INLINE OSMEM_PAGE_PRIVDATA* GetPrivateDataFromPage(struct page* psPage)
+{
+	return (OSMEM_PAGE_PRIVDATA*) page_private(psPage);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static const struct address_space_operations movable_callbacks =
+{
+	.isolate_page = OSMemPageIsolate,
+	.migratepage = OSMemPageMigrate,
+	.putback_page = OSMemPagePutback
+
+};
+#else
+static const struct movable_operations movable_callbacks =
+{
+	.isolate_page = OSMemPageIsolate,
+	.migrate_page = OSMemPageMigrate,
+	.putback_page = OSMemPagePutback
+};
+#endif
+
+static bool OSMemPageIsolate(struct page *psPage, isolate_mode_t uIsolateMode)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData = GetPrivateDataFromPage(psPage);
+
+
+	PVR_GOTO_IF_FALSE(psPrivData != NULL, condition_failed);
+
+	/* Attempt to migrate pages before PMR wrapper has been setup,
+	 * avoid migration at this time.
+	 */
+	PVR_GOTO_IF_FALSE(psPrivData->psPMRData->hPMR, condition_failed);
+
+	OSAtomicAdd(&psPrivData->psPMRData->iNumOSPagesUnderMigration, 1);
+	PVR_GOTO_IF_FALSE(!PMR_IsGpuMapped(psPrivData->psPMRData->hPMR), condition_failed);
+	PVR_GOTO_IF_FALSE(!PMR_IsCpuMapped(psPrivData->psPMRData->hPMR), condition_failed);
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVR_GOTO_IF_FALSE(!PMR_IsZombie(psPrivData->psPMRData->hPMR), condition_failed);
+#endif
+	/* Try take a ref on PMR, this ensures destruction will not occur while migrating */
+	PVR_GOTO_IF_ERROR(PMRTryRefPMR(psPrivData->psPMRData->hPMR), condition_failed);
+
+
+	psPrivData->psPMRData->pagearray[psPrivData->uiPMRArrIdx] = NULL;
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Isolated: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, PageAddr 0x%llx",
+	                                PMRInternalGetUID(psPrivData->psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psPrivData->psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psPage));
+
+	/* Movable */
+	return true;
+
+condition_failed:
+	/* Not movable */
+	OSAtomicSubtract(&psPrivData->psPMRData->iNumOSPagesUnderMigration, 1);
+
+	return false;
+}
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+static int OSMemPageMigrate(struct address_space *psAddrSpace,
+		struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode)
+#else
+static int OSMemPageMigrate(struct page *psDstPage, struct page *psSrcPage, enum migrate_mode eMigrateMode)
+#endif
+{
+	void* vpDst;
+	void* vpSrc;
+	PMR_OSPAGEARRAY_DATA *psPMRData;
+	OSMEM_PAGE_PRIVDATA *psSrcPagePrivData = GetPrivateDataFromPage(psSrcPage);
+
+	if (!psSrcPagePrivData)
+	{
+		/* Shouldn't hit this case */
+		PVR_DPF((PVR_DBG_ERROR, "Migrate invalid, potentially leaked PMR"));
+		return -EINVAL;
+	}
+	psPMRData = psSrcPagePrivData->psPMRData;
+
+	PVR_UNREFERENCED_PARAMETER(eMigrateMode);
+
+	vpDst = kmap(psDstPage);
+	vpSrc = kmap(psSrcPage);
+
+
+	OSDeviceMemCopy(vpDst, vpSrc, PAGE_SIZE);
+
+	kunmap(vpDst);
+	kunmap(vpSrc);
+
+#if defined(CONFIG_X86)
+	/* Reset caching attrs of page we give back to OS */
+	if (!set_pages_array_wb(&psSrcPage, 1))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute PMR UID:%llu",
+		         __func__,
+		         (unsigned long long)PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR)));
+		return -EINVAL;
+	}
+#endif
+
+	psPMRData->pagearray[psSrcPagePrivData->uiPMRArrIdx] = psDstPage;
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Migrated: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, SrcPageAddr 0x%llx, DstPageAddr 0x%llx",
+	                                PMRInternalGetUID(psSrcPagePrivData->psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psSrcPagePrivData->psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psSrcPage,
+	                                (unsigned long long) psDstPage));
+
+	/* Apply caching reqs to page we receive from OS */
+	_ApplyOSPagesAttribute(psPMRData->psDevNode,
+	                       &psPMRData->pagearray[psSrcPagePrivData->uiPMRArrIdx],
+	                       1,
+	                       IMG_FALSE,
+	                       psPMRData->ui32CPUCacheFlags);
+
+	get_page(psDstPage);
+	SetPagePrivate(psDstPage);
+	set_page_private(psDstPage, (unsigned long)psSrcPagePrivData);
+
+	/* DstPage already locked prior to callback */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	__SetPageMovable(psDstPage, psPMRData->psDRMFile->filp->f_inode->i_mapping);
+#else
+	__SetPageMovable(psDstPage, &movable_callbacks);
+#endif
+
+	set_page_private(psSrcPage, 0);
+	ClearPagePrivate(psSrcPage);
+	put_page(psSrcPage);
+	__ClearPageMovable(psSrcPage);
+
+	/* Migration for this page no longer in progress */
+	OSAtomicSubtract(&psSrcPagePrivData->psPMRData->iNumOSPagesUnderMigration, 1);
+
+	/* Unref parent PMR to allow for subsequent destruction,
+	 * if PMR resource no longer required and the bridge has
+	 * dropped its handles to it, destruction will follow.
+	 */
+	PMRUnrefPMR2(psSrcPagePrivData->psPMRData->hPMR);
+
+	return MIGRATEPAGE_SUCCESS;
+}
+
+static void OSMemPagePutback(struct page *psPage)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData = GetPrivateDataFromPage(psPage);
+	if (!psPrivData)
+	{
+		/* We shouldn't hit this case */
+		PVR_DPF((PVR_DBG_ERROR, "Putback invalid, potentially leaked PMR"));
+		return;
+	}
+
+	MIGRATE_DBG_LOG((PVR_DBG_ERROR, "Putback: PMR UID %llu, PMR Size 0x%llx, "
+	                                "OSPageSize %lu, PageAddr 0x%llx",
+	                                PMRInternalGetUID(psPrivData->psPMRData->hPMR),
+	                                (unsigned long long) PMR_LogicalSize(psPrivData->psPMRData->hPMR),
+	                                OSGetPageSize(),
+	                                (unsigned long long) psPage));
+
+
+	/* Restore the page to our structures */
+	psPrivData->psPMRData->pagearray[psPrivData->uiPMRArrIdx] = psPage;
+
+	/* Migration for this page no longer in progress */
+	OSAtomicSubtract(&psPrivData->psPMRData->iNumOSPagesUnderMigration, 1);
+
+	/* Unref surrounding PMR as we no longer need the PMR to remain in case of array access.
+	 */
+	PMRUnrefPMR2(psPrivData->psPMRData->hPMR);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+int LinuxRegisterMigrateCallbacks(struct file* filp)
+{
+	if (filp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "filp NULL, cannot set movable callbacks, PID %u:%s",
+		                        OSGetCurrentClientProcessIDKM(),
+		                        OSGetCurrentClientProcessNameKM()));
+		return -EINVAL;
+	}
+
+	filp->f_inode->i_mapping->a_ops = &movable_callbacks;
+
+	return 0;
+}
+
+void LinuxDeregisterMigrateCallbacks(struct file* filp)
+{
+	if (filp == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "filp NULL, cannot unset movable callbacks"));
+		return;
+	}
+
+	filp->f_inode->i_mapping->a_ops = NULL;
+}
+#endif
+
+
+static INLINE void OSMemSetMovablePageAttr(struct page* psPage,
+                                           PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                           IMG_UINT32 uiPageIndex)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	struct drm_file *psDRMFile = psPageArrayData->psDRMFile;
+#endif
+	OSMEM_PAGE_PRIVDATA *psPrivData;
+
+	if (!BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		return;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	/* Allocations marked with movable may occur before we have a valid connection
+	 * and thus DRM file. In this case ignore the request. An example of this would be
+	 * a kernel originated request which wouldn't have a connection associated.
+	 */
+	if (!psDRMFile)
+	{
+		PVR_ASSERT(!"Attempt to make allocation movable without connection associated, "
+		            "possible server origin");
+		return;
+	}
+#endif
+
+	psPrivData = kmem_cache_alloc(g_psLinuxPagePrivateData, GFP_KERNEL);
+	if (psPrivData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: OS refused the memory allocation for the private page data. "
+		         "Page will not be marked movable. PMR UID:%llu",
+				 __func__,
+				 PMRInternalGetUID(psPrivData->psPMRData->hPMR)));
+		return;
+	}
+
+	psPrivData->psPMRData = psPageArrayData;
+	psPrivData->uiPMRArrIdx = uiPageIndex;
+
+	SetPagePrivate(psPage);
+	set_page_private(psPage, (unsigned long)psPrivData);
+
+	/* Take page lock here while we set movable as __SetPageMovable asserts it */
+	lock_page(psPage);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	__SetPageMovable(psPage, psDRMFile->filp->f_inode->i_mapping);
+#else
+	__SetPageMovable(psPage, &movable_callbacks);
+#endif
+	unlock_page(psPage);
+}
+
+static INLINE void OSMemUnsetMovablePageAttr(struct page* psPage, PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	OSMEM_PAGE_PRIVDATA *psPrivData;
+
+	if (!BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		return;
+	}
+
+	/* Private data should only be set if page is movable */
+	psPrivData = GetPrivateDataFromPage(psPage);
+	if (!psPrivData)
+	{
+		return;
+	}
+
+	lock_page(psPage);
+	__ClearPageMovable(psPage);
+	set_page_private(psPage, 0);
+	ClearPagePrivate(psPage);
+	unlock_page(psPage);
+
+	kmem_cache_free(g_psLinuxPagePrivateData, psPrivData);
+}
+#endif /* defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+static inline IMG_BOOL
+_ShouldInitMem(IMG_UINT32 ui32AllocFlags)
+{
+	return BIT_ISSET(ui32AllocFlags, FLAG_ZERO) && !g_bInitialisedOnAlloc;
+}
+
+static inline IMG_UINT32
+_PagesInPoolUnlocked(void)
+{
+	IMG_UINT32 uiCnt = g_ui32PagePoolUCCount;
+#if defined(CONFIG_X86)
+	uiCnt += g_ui32PagePoolWCCount;
+#endif
+	return uiCnt;
+}
+
+static inline void
+_PagePoolLock(void)
+{
+	mutex_lock(&g_sPagePoolMutex);
+}
+
+static inline int
+_PagePoolTrylock(void)
+{
+	return mutex_trylock(&g_sPagePoolMutex);
+}
+
+static inline void
+_PagePoolUnlock(void)
+{
+	mutex_unlock(&g_sPagePoolMutex);
+}
+
+static inline IMG_BOOL
+_GetPoolListHead(IMG_UINT32 ui32CPUCacheFlags,
+				 struct list_head **ppsPoolHead,
+				 IMG_UINT32 **ppuiCounter)
+{
+	switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+#if defined(CONFIG_X86)
+		/*
+			For x86 we need to keep different lists for uncached
+			and write-combined as we must always honour the PAT
+			setting which cares about this difference.
+		*/
+
+			*ppsPoolHead = &g_sPagePoolList_WC;
+			*ppuiCounter = &g_ui32PagePoolWCCount;
+			break;
+#endif
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+			*ppsPoolHead = &g_sPagePoolList_UC;
+			*ppuiCounter = &g_ui32PagePoolUCCount;
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unknown CPU caching mode. "
+					 "Using default UC pool.",
+					 __func__));
+			*ppsPoolHead = &g_sPagePoolList_UC;
+			*ppuiCounter = &g_ui32PagePoolUCCount;
+			PVR_ASSERT(0);
+			return IMG_FALSE;
+	}
+	return IMG_TRUE;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+static struct shrinker g_sShrinker;
+static struct shrinker *g_psShrinker = &g_sShrinker;
+#else
+static struct shrinker *g_psShrinker;
+#endif
+
+/* Returning the number of pages that still reside in the page pool. */
+static unsigned long
+_GetNumberOfPagesInPoolUnlocked(void)
+{
+	return _PagesInPoolUnlocked();
+}
+
+/* Linux shrinker function that informs the OS about how many pages we are caching and
+ * it is able to reclaim. */
+static unsigned long
+_CountObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl)
+{
+	int remain;
+
+	PVR_ASSERT(psShrinker == g_psShrinker);
+	(void)psShrinker;
+	(void)psShrinkControl;
+
+	/* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */
+	if (_PagePoolTrylock() == 0)
+		return 0;
+	remain = _GetNumberOfPagesInPoolUnlocked();
+	_PagePoolUnlock();
+
+	return remain;
+}
+
+/* Linux shrinker function to reclaim the pages from our page pool */
+static unsigned long
+_ScanObjectsInPagePool(struct shrinker *psShrinker, struct shrink_control *psShrinkControl)
+{
+	unsigned long uNumToScan = psShrinkControl->nr_to_scan;
+	IMG_UINT32 uiPagesFreed;
+
+	PVR_ASSERT(psShrinker == g_psShrinker);
+	(void)psShrinker;
+
+	/* In order to avoid possible deadlock use mutex_trylock in place of mutex_lock */
+	if (_PagePoolTrylock() == 0)
+		return SHRINK_STOP;
+
+	_FreePagesFromPoolUnlocked(uNumToScan,
+							   &uiPagesFreed);
+	uNumToScan -= uiPagesFreed;
+
+	/* Returning the number of pages freed during the scan */
+	_PagePoolUnlock();
+	return psShrinkControl->nr_to_scan - uNumToScan;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+static struct shrinker g_sShrinker =
+{
+	.count_objects = _CountObjectsInPagePool,
+	.scan_objects = _ScanObjectsInPagePool,
+	.seeks = DEFAULT_SEEKS
+};
+#endif
+
+/* Register the shrinker so Linux can reclaim cached pages */
+PVRSRV_ERROR LinuxInitPhysmem(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	g_psLinuxPagePrivateData = kmem_cache_create("pvr-ppd", sizeof(OSMEM_PAGE_PRIVDATA), 0, 0, NULL);
+	if (!g_psLinuxPagePrivateData)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPagePrivateData",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e1);
+	}
+#endif
+
+	g_psLinuxPageArray = kmem_cache_create("pvr-pa", sizeof(PMR_OSPAGEARRAY_DATA), 0, 0, NULL);
+	if (!g_psLinuxPageArray)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPageArray",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e2);
+	}
+
+	g_psLinuxPagePoolCache = kmem_cache_create("pvr-pp", sizeof(LinuxPagePoolEntry), 0, 0, NULL);
+	if (g_psLinuxPagePoolCache)
+	{
+		/* Only create the shrinker if we created the cache OK */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+		register_shrinker(&g_sShrinker, "pvr-pp");
+#else
+		g_psShrinker = shrinker_alloc(0, "pvr-pp");
+		if (g_psShrinker)
+		{
+			g_psShrinker->count_objects = _CountObjectsInPagePool;
+			g_psShrinker->scan_objects = _ScanObjectsInPagePool;
+			g_psShrinker->seeks = DEFAULT_SEEKS;
+			shrinker_register(g_psShrinker);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Unable to allocate a shrinker for page pool cache"));
+		}
+#endif
+	}
+	else
+	{
+		PVR_LOG_GOTO_WITH_ERROR("kmem_cache_create() g_psLinuxPagePoolCache",
+		                        eError, PVRSRV_ERROR_OUT_OF_MEMORY, e3);
+	}
+
+	OSAtomicWrite(&g_iPoolCleanTasks, 0);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,3,0))
+/* Check both config and modparam setting */
+#if PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC == 1
+	g_bInitialisedOnAlloc = want_init_on_alloc(0x0);
+
+/* Assume modparam setting not in use on system */
+#elif PVRSRV_USE_LINUX_CONFIG_INIT_ON_ALLOC == 2
+#   if defined(CONFIG_INIT_ON_ALLOC_DEFAULT_ON)
+	g_bInitialisedOnAlloc = IMG_TRUE;
+#   else
+	g_bInitialisedOnAlloc = IMG_FALSE;
+#   endif
+
+/* Ignore both config and modparam settings */
+#else
+	g_bInitialisedOnAlloc = IMG_FALSE;
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+e3:
+	/* Free the page array */
+	kmem_cache_destroy(g_psLinuxPageArray);
+e2:
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Free the page private data */
+	kmem_cache_destroy(g_psLinuxPagePrivateData);
+e1:
+#endif
+
+	return eError;
+}
+
+/* Unregister the shrinker and remove all pages from the pool that are still left */
+void LinuxDeinitPhysmem(void)
+{
+	IMG_UINT32 uiPagesFreed;
+
+	if (OSAtomicRead(&g_iPoolCleanTasks) > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Still deferred cleanup tasks running "
+				"while deinitialising memory subsystem."));
+	}
+
+	_PagePoolLock();
+	if (_FreePagesFromPoolUnlocked(IMG_UINT32_MAX, &uiPagesFreed) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Unable to free all pages from page pool when "
+				"deinitialising memory subsystem."));
+		PVR_ASSERT(0);
+	}
+
+	PVR_ASSERT(_PagesInPoolUnlocked() == 0);
+
+	/* Free the page cache */
+	if (g_psLinuxPagePoolCache)
+		kmem_cache_destroy(g_psLinuxPagePoolCache);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0))
+	unregister_shrinker(&g_sShrinker);
+#else
+	shrinker_free(g_psShrinker);
+#endif
+	_PagePoolUnlock();
+
+	if (g_psLinuxPageArray)
+		kmem_cache_destroy(g_psLinuxPageArray);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (g_psLinuxPagePrivateData)
+		kmem_cache_destroy(g_psLinuxPagePrivateData);
+#endif
+}
+
+static void EnableOOMKiller(void)
+{
+	current->flags &= ~PF_DUMPCORE;
+}
+
+static void DisableOOMKiller(void)
+{
+	/* PF_DUMPCORE is treated by the VM as if the OOM killer was disabled.
+	 *
+	 * As oom_killer_disable() is an inline, non-exported function, we
+	 * can't use it from a modular driver. Furthermore, the OOM killer
+	 * API doesn't look thread safe, which 'current' is.
+	 */
+	WARN_ON(current->flags & PF_DUMPCORE);
+	current->flags |= PF_DUMPCORE;
+}
+
+/* Prints out the addresses in a page array for debugging purposes
+ * Define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY locally to activate: */
+/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY 1 */
+static inline void
+_DumpPageArray(struct page **pagearray, IMG_UINT32 uiPagesToPrint)
+{
+#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_ARRAY)
+	IMG_UINT32 i;
+	if (pagearray)
+	{
+		printk("Array %p:\n", pagearray);
+		for (i = 0; i < uiPagesToPrint; i++)
+		{
+			printk("%p | ", (pagearray)[i]);
+		}
+		printk("\n");
+	}
+	else
+	{
+		printk("Array is NULL:\n");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(pagearray);
+	PVR_UNREFERENCED_PARAMETER(uiPagesToPrint);
+#endif
+}
+
+/* Debugging function that dumps out the number of pages for every
+ * page array that is currently in the page pool.
+ * Not defined by default. Define locally to activate feature: */
+/* #define PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL 1 */
+static void
+_DumpPoolStructure(void)
+{
+#if defined(PHYSMEM_OSMEM_DEBUG_DUMP_PAGE_POOL)
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 j;
+	IMG_UINT32 *puiCounter;
+
+	printk("\n");
+	/* Empty all pools */
+	for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++)
+	{
+
+		printk("pool = %u\n", j);
+
+		/* Get the correct list for this caching mode */
+		if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter))
+		{
+			break;
+		}
+
+		list_for_each_entry_safe(psPagePoolEntry,
+								 psTempPoolEntry,
+								 psPoolHead,
+								 sPagePoolItem)
+		{
+			printk("%u | ", psPagePoolEntry->uiItemsRemaining);
+		}
+		printk("\n");
+	}
+#endif
+}
+
+/* Free a certain number of pages from the page pool.
+ * Mainly used in error paths or at deinitialisation to
+ * empty the whole pool. */
+static PVRSRV_ERROR
+_FreePagesFromPoolUnlocked(IMG_UINT32 uiMaxPagesToFree,
+						   IMG_UINT32 *puiPagesFreed)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 i, j;
+	IMG_UINT32 *puiCounter;
+
+	*puiPagesFreed = uiMaxPagesToFree;
+
+	/* Empty all pools */
+	for (j = 0; j < PHYSMEM_OSMEM_NUM_OF_POOLS; j++)
+	{
+
+		/* Get the correct list for this caching mode */
+		if (!_GetPoolListHead(g_aui32CPUCacheFlags[j], &psPoolHead, &puiCounter))
+		{
+			break;
+		}
+
+		/* Free the pages and remove page arrays from the pool if they are exhausted */
+		list_for_each_entry_safe(psPagePoolEntry,
+								 psTempPoolEntry,
+								 psPoolHead,
+								 sPagePoolItem)
+		{
+			IMG_UINT32 uiItemsToFree;
+			struct page **ppsPageArray;
+
+			/* Check if we are going to free the whole page array or just parts */
+			if (psPagePoolEntry->uiItemsRemaining <= uiMaxPagesToFree)
+			{
+				uiItemsToFree = psPagePoolEntry->uiItemsRemaining;
+				ppsPageArray = psPagePoolEntry->ppsPageArray;
+			}
+			else
+			{
+				uiItemsToFree = uiMaxPagesToFree;
+				ppsPageArray = &(psPagePoolEntry->ppsPageArray[psPagePoolEntry->uiItemsRemaining - uiItemsToFree]);
+			}
+
+#if defined(CONFIG_X86)
+			/* Set the correct page caching attributes on x86 */
+			if (!PVRSRV_CHECK_CPU_CACHED(g_aui32CPUCacheFlags[j]))
+			{
+				int ret;
+				ret = set_pages_array_wb(ppsPageArray, uiItemsToFree);
+				if (ret)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s: Failed to reset page attributes",
+							 __func__));
+					eError = PVRSRV_ERROR_FAILED_TO_FREE_PAGES;
+					goto e_exit;
+				}
+			}
+#endif
+
+			/* Free the actual pages */
+			for (i = 0; i < uiItemsToFree; i++)
+			{
+				__free_pages(ppsPageArray[i], 0);
+				ppsPageArray[i] = NULL;
+			}
+
+			/* Reduce counters */
+			uiMaxPagesToFree -= uiItemsToFree;
+			*puiCounter -= uiItemsToFree;
+			psPagePoolEntry->uiItemsRemaining -= uiItemsToFree;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+			/*
+			 * MemStats usually relies on having the bridge lock held, however
+			 * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and
+			 * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so
+			 * the page pool lock is used to ensure these calls are mutually
+			 * exclusive
+			 */
+			PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * uiItemsToFree);
+#endif
+
+			/* Is this pool entry exhausted, delete it */
+			if (psPagePoolEntry->uiItemsRemaining == 0)
+			{
+				OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+				list_del(&psPagePoolEntry->sPagePoolItem);
+				kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+			}
+
+			/* Return if we have all our pages */
+			if (uiMaxPagesToFree == 0)
+			{
+				goto e_exit;
+			}
+		}
+	}
+
+e_exit:
+	*puiPagesFreed -= uiMaxPagesToFree;
+	_DumpPoolStructure();
+	return eError;
+}
+
+/* Get a certain number of pages from the page pool and
+ * copy them directly into a given page array. */
+static void
+_GetPagesFromPoolUnlocked(IMG_UINT32 ui32CPUCacheFlags,
+						  IMG_UINT32 uiMaxNumPages,
+						  struct page **ppsPageArray,
+						  IMG_UINT32 *puiNumReceivedPages)
+{
+	LinuxPagePoolEntry *psPagePoolEntry, *psTempPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 i;
+	IMG_UINT32 *puiCounter;
+
+	*puiNumReceivedPages = 0;
+
+	/* Get the correct list for this caching mode */
+	if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter))
+	{
+		return;
+	}
+
+	/* Check if there are actually items in the list */
+	if (list_empty(psPoolHead))
+	{
+		return;
+	}
+
+	PVR_ASSERT(*puiCounter > 0);
+
+	/* Receive pages from the pool */
+	list_for_each_entry_safe(psPagePoolEntry,
+							 psTempPoolEntry,
+							 psPoolHead,
+							 sPagePoolItem)
+	{
+		/* Get the pages from this pool entry */
+		for (i = psPagePoolEntry->uiItemsRemaining; i != 0 && *puiNumReceivedPages < uiMaxNumPages; i--)
+		{
+			ppsPageArray[*puiNumReceivedPages] = psPagePoolEntry->ppsPageArray[i-1];
+			(*puiNumReceivedPages)++;
+			psPagePoolEntry->uiItemsRemaining--;
+		}
+
+		/* Is this pool entry exhausted, delete it */
+		if (psPagePoolEntry->uiItemsRemaining == 0)
+		{
+			OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+			list_del(&psPagePoolEntry->sPagePoolItem);
+			kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+		}
+
+		/* Return if we have all our pages */
+		if (*puiNumReceivedPages == uiMaxNumPages)
+		{
+			goto exit_ok;
+		}
+	}
+
+exit_ok:
+
+	/* Update counters */
+	*puiCounter -= *puiNumReceivedPages;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* MemStats usually relies on having the bridge lock held, however
+	 * the page pool code may call PVRSRVStatsIncrMemAllocPoolStat and
+	 * PVRSRVStatsDecrMemAllocPoolStat without the bridge lock held, so
+	 * the page pool lock is used to ensure these calls are mutually
+	 * exclusive
+	 */
+	PVRSRVStatsDecrMemAllocPoolStat(PAGE_SIZE * (*puiNumReceivedPages));
+#endif
+
+	_DumpPoolStructure();
+}
+
+/* Same as _GetPagesFromPoolUnlocked but handles locking and
+ * checks first whether pages from the pool are a valid option. */
+static inline void
+_GetPagesFromPoolLocked(PVRSRV_DEVICE_NODE *psDevNode,
+						IMG_UINT32 ui32CPUCacheFlags,
+						IMG_UINT32 ui32AllocFlags,
+						IMG_UINT32 uiPagesToAlloc,
+						IMG_UINT32 uiOrder,
+						struct page **ppsPageArray,
+						IMG_UINT32 *puiPagesFromPool)
+{
+#if !defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	/* Don't get pages from pool as it doesn't provide zeroed pages */
+	if (BIT_ISSET(ui32AllocFlags, FLAG_ZERO))
+	{
+		return;
+	}
+#endif
+
+	/* The page pool stores only order 0 pages. If we need zeroed memory we
+	 * directly allocate from the OS because it is faster than
+	 * doing it within the driver. CMA is naturally more commonly used for higher
+	 * order pages and so we reject that also.
+	 */
+	if (uiOrder == 0 &&
+	    !PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags) &&
+	    !BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA) &&
+	    !BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+
+		_PagePoolLock();
+		_GetPagesFromPoolUnlocked(ui32CPUCacheFlags,
+								  uiPagesToAlloc,
+								  ppsPageArray,
+								  puiPagesFromPool);
+		_PagePoolUnlock();
+	}
+}
+
+/* Takes a page array and maps it into the kernel to write zeros */
+static PVRSRV_ERROR
+_MemsetPageArray(IMG_UINT32 uiNumToClean,
+                 struct page **ppsCleanArray,
+                 pgprot_t pgprot,
+                 IMG_UINT8 ui8Pattern)
+{
+	IMG_CPU_VIRTADDR pvAddr;
+	IMG_UINT32 uiMaxPagesToMap = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES,
+	                                 uiNumToClean);
+
+	/* Map and fill the pages with zeros.
+	 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+	 * at a time. */
+	while (uiNumToClean != 0)
+	{
+		IMG_UINT32 uiToClean = MIN(uiNumToClean, uiMaxPagesToMap);
+
+		pvAddr = pvr_vmap(ppsCleanArray, uiToClean, VM_MAP, pgprot);
+		if (!pvAddr)
+		{
+			if (uiMaxPagesToMap <= 1)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				        "%s: Out of vmalloc memory, unable to map pages for %s.",
+				        __func__,
+				        ui8Pattern == PVRSRV_ZERO_VALUE ? "zeroing" : "poisoning"));
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+			else
+			{
+				/* Halve the pages to map at once and try again. */
+				uiMaxPagesToMap = uiMaxPagesToMap >> 1;
+				continue;
+			}
+		}
+
+		if (pgprot_val(pgprot) == pgprot_val(pgprot_noncached(PAGE_KERNEL)))
+		{
+			/* this is most likely unnecessary as all pages must be 8-bytes
+			 * aligned so there unaligned access is impossible */
+			OSDeviceMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		else if (pgprot_val(pgprot) == pgprot_val(pgprot_writecombine(PAGE_KERNEL)))
+		{
+			OSCachedMemSetWMB(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		else
+		{
+			OSCachedMemSet(pvAddr, ui8Pattern, PAGE_SIZE * uiToClean);
+		}
+		pvr_vunmap(pvAddr, uiToClean, pgprot);
+		ppsCleanArray = &(ppsCleanArray[uiToClean]);
+		uiNumToClean -= uiToClean;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_CleanupThread_CleanPages(void *pvData)
+{
+	LinuxCleanupData *psCleanupData = (LinuxCleanupData*) pvData;
+	LinuxPagePoolEntry *psPagePoolEntry = psCleanupData->psPoolEntry;
+	struct list_head *psPoolHead = NULL;
+	IMG_UINT32 *puiCounter = NULL;
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	PVRSRV_ERROR eError;
+	pgprot_t pgprot;
+	IMG_UINT32 i;
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+
+	/* Get the correct pool for this caching mode. */
+	_GetPoolListHead(psCleanupData->ui32CPUCacheMode , &psPoolHead, &puiCounter);
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	switch (PVRSRV_CPU_CACHE_MODE(psCleanupData->ui32CPUCacheMode))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+#if defined(CONFIG_X86)
+			/* For x86 we can only map with the same attributes
+			 * as in the PAT settings*/
+			pgprot = pgprot_noncached(PAGE_KERNEL);
+			break;
+#endif
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+			pgprot = pgprot_writecombine(PAGE_KERNEL);
+			break;
+
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Unknown caching mode to set page protection flags.",
+					__func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto eExit;
+	}
+
+	/* Map and fill the pages with zeros.
+	 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+	 * at a time. */
+	eError = _MemsetPageArray(psPagePoolEntry->uiItemsRemaining,
+	                          psPagePoolEntry->ppsPageArray,
+	                          pgprot, PVRSRV_ZERO_VALUE);
+	if (eError != PVRSRV_OK)
+	{
+		goto eExit;
+	}
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+
+	/* Lock down pool and add item */
+	_PagePoolLock();
+
+	/* Pool counters were already updated so don't do it here again*/
+
+	/* The pages are all zeroed so return them to the pool. */
+	list_add_tail(&psPagePoolEntry->sPagePoolItem, psPoolHead);
+
+	_DumpPoolStructure();
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Calling PVRSRVStatsIncrMemAllocPoolStat and PVRSRVStatsDecrMemAllocPoolStat
+	 * inside page pool lock ensures that the stat reflects the state of the pool. */
+	PVRSRVStatsIncrMemAllocPoolStat(PAGE_SIZE * psPagePoolEntry->uiItemsRemaining);
+#endif
+
+	_PagePoolUnlock();
+
+	OSFreeMem(pvData);
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+
+	return PVRSRV_OK;
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+eExit:
+	/* we failed to zero the pages so return the error so we can
+	 * retry during the next spin */
+	if ((psCleanupData->sCleanupWork.ui32RetryCount - 1) > 0)
+	{
+		return eError;
+	}
+
+	/* this was the last retry, give up and free pages to OS */
+	PVR_DPF((PVR_DBG_ERROR,
+			"%s: Deferred task error, freeing pages to OS.",
+			__func__));
+	_PagePoolLock();
+
+	*puiCounter -= psPagePoolEntry->uiItemsRemaining;
+
+	_PagePoolUnlock();
+
+	for (i = 0; i < psCleanupData->psPoolEntry->uiItemsRemaining; i++)
+	{
+		_FreeOSPage(0, IMG_TRUE, NULL, psPagePoolEntry->ppsPageArray[i]);
+	}
+	OSFreeMemNoStats(psPagePoolEntry->ppsPageArray);
+	kmem_cache_free(g_psLinuxPagePoolCache, psPagePoolEntry);
+	OSFreeMem(psCleanupData);
+
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+
+	return PVRSRV_OK;
+#endif /* defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES) */
+}
+
+static bool _PagesHaveOtherRefs(struct page **ppsPageArray, IMG_UINT32 uiNumPages)
+{
+	IMG_UINT32 ui32pageIndex;
+
+	PVR_DPF_ENTERED;
+
+	if (!ppsPageArray)
+	{
+		PVR_DPF_RETURN_RC(false);
+	}
+
+	/* Iterate pages in ppsPageArray and return
+	   false if any are found to have page_count() > 1
+	*/
+	for (ui32pageIndex = 0; ui32pageIndex < uiNumPages; ui32pageIndex++)
+	{
+		struct page *psPage = ppsPageArray[ui32pageIndex];
+
+		if (page_count(psPage) > 1)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+			         "%s: page %d in page array found with page_count() of %d",
+			         __func__, ui32pageIndex, page_count(psPage)));
+			PVR_DPF_RETURN_RC(true);
+		}
+	}
+
+	PVR_DPF_RETURN_RC(false);
+}
+
+/* Put page array to the page pool.
+ * Handles locking and checks whether the pages are
+ * suitable to be stored in the pool. */
+static inline IMG_BOOL
+_PutPagesToPoolLocked(PVRSRV_DEVICE_NODE *psDevNode,
+                      IMG_UINT32 ui32CPUCacheFlags,
+                      IMG_UINT32 ui32AllocFlags,
+                      struct page **ppsPageArray,
+                      IMG_UINT32 uiOrder,
+                      IMG_UINT32 uiNumPages)
+{
+	LinuxCleanupData *psCleanupData;
+	PVRSRV_CLEANUP_THREAD_WORK *psCleanupThreadFn;
+	IMG_UINT32 uiEntries;
+	IMG_UINT32 *puiCounter;
+	struct list_head *psPoolHead;
+#if defined(SUPPORT_PHYSMEM_TEST)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+#endif
+
+	PVR_DPF_ENTERED;
+
+	/* The page pool stores only order 0 pages. CMA is naturally more
+	 * commonly used for higher order pages and so we reject that also.
+	 */
+	if (uiOrder != 0 ||
+		PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags) ||
+		BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA) ||
+		BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		goto eExitFalse;
+	}
+
+	if (_PagesHaveOtherRefs(ppsPageArray, uiNumPages))
+	{
+		/* Pages still have other references, so
+		   must not be put into our pool */
+		goto eExitFalse;
+	}
+
+	_PagePoolLock();
+
+	uiEntries = _PagesInPoolUnlocked();
+
+	/* Check for number of current page pool entries */
+	if ( (uiEntries >= g_ui32PagePoolMaxEntries) ||
+		 ((uiEntries + uiNumPages) >=
+		  (g_ui32PagePoolMaxEntries + g_ui32PagePoolMaxExcessEntries) ))
+	{
+		goto eUnlock;
+	}
+
+	/* Check for whether we have other asynchronous tasks in-flight */
+	if (OSAtomicIncrement(&g_iPoolCleanTasks) >
+			PVR_LINUX_PHYSMEM_MAX_ASYNC_CLEAN_TASKS)
+	{
+		goto eDecrement;
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST)
+	if (!psPVRSRVData->hCleanupThread)
+	{
+		goto eDecrement;
+	}
+#endif
+
+	psCleanupData = OSAllocMem(sizeof(*psCleanupData));
+
+	if (!psCleanupData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get memory for deferred page pool cleanup. "
+				 "Trying to free pages immediately",
+				 __func__));
+		goto eDecrement;
+	}
+
+	psCleanupThreadFn = &psCleanupData->sCleanupWork;
+	psCleanupData->ui32CPUCacheMode = ui32CPUCacheFlags;
+	psCleanupData->psPoolEntry = kmem_cache_alloc(g_psLinuxPagePoolCache, GFP_KERNEL);
+
+	if (!psCleanupData->psPoolEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get memory for deferred page pool cleanup. "
+				 "Trying to free pages immediately",
+				 __func__));
+		goto eFreeCleanupData;
+	}
+
+	if (!_GetPoolListHead(ui32CPUCacheFlags, &psPoolHead, &puiCounter))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to get correct page pool",
+				 __func__));
+		goto eFreePoolEntry;
+	}
+
+	/* Increase counter here to avoid deferred cleanup tasks piling up */
+	*puiCounter = *puiCounter + uiNumPages;
+
+	psCleanupData->psPoolEntry->ppsPageArray = ppsPageArray;
+	psCleanupData->psPoolEntry->uiItemsRemaining = uiNumPages;
+
+	psCleanupThreadFn->pfnFree = _CleanupThread_CleanPages;
+	psCleanupThreadFn->pvData = psCleanupData;
+	psCleanupThreadFn->bDependsOnHW = IMG_FALSE;
+	psCleanupThreadFn->eCleanupType = PVRSRV_CLEANUP_TYPE_OSMEM;
+	CLEANUP_THREAD_SET_RETRY_COUNT(psCleanupThreadFn,
+								   CLEANUP_THREAD_RETRY_COUNT_DEFAULT);
+
+	/* We must not hold the pool lock when calling AddWork because it might call us back to
+	 * free pooled pages directly when unloading the driver	 */
+	_PagePoolUnlock();
+
+	PVRSRVCleanupThreadAddWork(psDevNode, psCleanupThreadFn);
+
+	PVR_DPF_RETURN_RC(IMG_TRUE);
+
+eFreePoolEntry:
+	OSFreeMem(psCleanupData->psPoolEntry);
+eFreeCleanupData:
+	OSFreeMem(psCleanupData);
+eDecrement:
+	OSAtomicDecrement(&g_iPoolCleanTasks);
+eUnlock:
+	_PagePoolUnlock();
+eExitFalse:
+	PVR_DPF_RETURN_RC(IMG_FALSE);
+}
+
+/* Get the GFP flags that we pass to the page allocator */
+static inline gfp_t
+_GetGFPFlags(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	/* If higher order pages then potentially ask the kernel to zero the allocation
+	 * for us. If zero order we can do it ourselves later in a batch operation
+	 */
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bZero = uiOrder ? _ShouldInitMem(psPageArrayData->ui32AllocFlags) : IMG_FALSE;
+	struct device *psDev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice;
+
+	gfp_t gfp_flags = __GFP_NOWARN | __GFP_NOMEMALLOC;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_MOVABLE))
+	{
+		gfp_flags |= GFP_HIGHUSER_MOVABLE;
+	}
+	else
+#else
+	{
+		gfp_flags |= GFP_USER;
+	}
+#endif
+
+#if defined(ANDROID)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)) && \
+	(LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	/* Only Android has CMA suggestion flag, base Linux kernel uses heuristics based on memory
+	 * pressure / availability to achieve similar goal.
+	 */
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_PREF_CMA))
+	{
+		gfp_flags |= __GFP_CMA;
+	}
+#endif
+#endif
+
+#if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+	/* Force use of HIGHMEM */
+	gfp_flags |= __GFP_HIGHMEM;
+
+	PVR_UNREFERENCED_PARAMETER(psDev);
+#else
+	if (psDev)
+	{
+#if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE)
+		if (*psDev->dma_mask > DMA_BIT_MASK(32))
+		{
+			/* If our system is able to handle large addresses use highmem */
+			gfp_flags |= __GFP_HIGHMEM;
+		}
+		else if (*psDev->dma_mask == DMA_BIT_MASK(32))
+		{
+			/* Limit to 32 bit.
+			 * Achieved by setting __GFP_DMA32 for 64 bit systems */
+			gfp_flags |= __GFP_DMA32;
+		}
+		else
+		{
+			/* Limit to size of DMA zone. */
+			gfp_flags |= __GFP_DMA;
+		}
+#else
+		if (*psDev->dma_mask < DMA_BIT_MASK(32))
+		{
+			gfp_flags |= __GFP_DMA;
+		}
+		else
+		{
+			gfp_flags |= __GFP_HIGHMEM;
+		}
+#endif /* if defined(CONFIG_64BIT) || defined(CONFIG_ARM_LPAE) || defined(CONFIG_X86_PAE) */
+	}
+
+#endif /* if defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY) */
+
+	if (bZero)
+	{
+		gfp_flags |= __GFP_ZERO;
+	}
+
+	return gfp_flags;
+}
+
+/*
+ * @Function _PoisonDevicePage
+ *
+ * @Description  Poisons a device page. In normal case the device page has the
+ *               same size as the OS page and so the ui32DevPageOrder will be
+ *               equal to 0 and page argument will point to one OS page
+ *               structure. In case of Non4K pages the order will be greater
+ *               than 0 and page argument will point to an array of OS
+ *               allocated pages.
+ *
+ * @Input psDevNode          pointer to the device object
+ * @Input page               array of the pages allocated by from the OS
+ * @Input ui32DevPageOrder   order of the page (same as the one used to allocate
+ *                           the page array by alloc_pages())
+ * @Input ui32CPUCacheFlags  CPU cache flags applied to the page
+ * @Input ui8PoisonValue     value used to poison the page
+ */
+static void
+_PoisonDevicePage(PVRSRV_DEVICE_NODE *psDevNode,
+                  struct page *page,
+                  IMG_UINT32 ui32DevPageOrder,
+                  IMG_UINT32 ui32CPUCacheFlags,
+                  IMG_BYTE ui8PoisonValue)
+{
+	IMG_UINT32 ui32OsPageIdx;
+
+	for (ui32OsPageIdx = 0;
+	     ui32OsPageIdx < (1U << ui32DevPageOrder);
+	     ui32OsPageIdx++)
+	{
+		struct page *current_page = page + ui32OsPageIdx;
+		IMG_CPU_PHYADDR sCPUPhysAddrStart = {page_to_phys(current_page)};
+		IMG_CPU_PHYADDR sCPUPhysAddrEnd = {sCPUPhysAddrStart.uiAddr + PAGE_SIZE};
+
+		void *kvaddr = kmap_atomic(current_page);
+
+		/* kmap_atomic maps pages as cached so it's safe to use OSCachedMemSet
+		 * here (also pages are always 8 bytes aligned anyway) */
+		OSCachedMemSet(kvaddr, ui8PoisonValue, PAGE_SIZE);
+
+		OSCPUCacheFlushRangeKM(psDevNode, kvaddr, kvaddr + PAGE_SIZE,
+		                       sCPUPhysAddrStart, sCPUPhysAddrEnd);
+
+		kunmap_atomic(kvaddr);
+	}
+}
+
+/* Allocate and initialise the structure to hold the metadata of the allocation */
+static PVRSRV_ERROR
+_AllocOSPageArray(PVRSRV_DEVICE_NODE *psDevNode,
+				  PMR_SIZE_T uiSize,
+				  IMG_UINT32 uiLog2DevPageSize,
+				  IMG_UINT32 ui32AllocFlags,
+				  IMG_UINT32 ui32CPUCacheFlags,
+				  IMG_PID uiPid,
+				  PMR_OSPAGEARRAY_DATA **ppsPageArrayDataPtr)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiNumOSPageSizeVirtPages;
+	PMR_OSPAGEARRAY_DATA *psPageArrayData;
+	IMG_UINT64 ui64DmaMask = 0;
+
+	/* Use of cast below is justified by the assertion that follows to
+	 * prove that no significant bits have been truncated */
+	uiNumOSPageSizeVirtPages = (IMG_UINT32) (((uiSize - 1) >> PAGE_SHIFT) + 1);
+	PVR_ASSERT(((PMR_SIZE_T) uiNumOSPageSizeVirtPages << PAGE_SHIFT) == uiSize);
+
+	/* Allocate the struct to hold the metadata */
+	psPageArrayData = kmem_cache_alloc(g_psLinuxPageArray, GFP_KERNEL);
+	if (psPageArrayData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: OS refused the memory allocation for the private data.",
+				 __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e_freed_none;
+	}
+
+	/*
+	 * Allocate the page array
+	 *
+	 * We avoid tracking this memory because this structure might go into the page pool.
+	 * The OS can drain the pool asynchronously and when doing that we have to avoid
+	 * any potential deadlocks.
+	 *
+	 * In one scenario the process stats vmalloc hash table lock is held and then
+	 * the oom-killer softirq is trying to call _ScanObjectsInPagePool(), it must not
+	 * try to acquire the vmalloc hash table lock again.
+	 */
+	psPageArrayData->pagearray = OSAllocZMemNoStats(sizeof(struct page *) * uiNumOSPageSizeVirtPages);
+	if (psPageArrayData->pagearray == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e_free_kmem_cache;
+	}
+
+	if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		/* Allocate additional DMA/CMA cpu kernel virtual address & device bus address array state */
+		psPageArrayData->dmavirtarray = OSAllocZMemNoStats(sizeof(void*) * uiNumOSPageSizeVirtPages);
+		if (psPageArrayData->dmavirtarray == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e_free_pagearray;
+		}
+
+		psPageArrayData->dmaphysarray = OSAllocZMemNoStats(sizeof(dma_addr_t) * uiNumOSPageSizeVirtPages);
+		if (psPageArrayData->dmaphysarray == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e_free_cpuvirtaddrarray;
+		}
+	}
+	else
+	{
+		psPageArrayData->dmavirtarray = NULL;
+		psPageArrayData->dmaphysarray = NULL;
+	}
+
+
+	if (psDevNode->psDevConfig && psDevNode->psDevConfig->pvOSDevice)
+	{
+		struct device *psDev = psDevNode->psDevConfig->pvOSDevice;
+		ui64DmaMask = *psDev->dma_mask;
+	}
+
+	/* Init metadata */
+	psPageArrayData->hPMR = NULL;
+	psPageArrayData->psDevNode = psDevNode;
+	psPageArrayData->uiPid = uiPid;
+	psPageArrayData->iNumOSPagesAllocated = 0;
+	psPageArrayData->uiTotalNumOSPages = uiNumOSPageSizeVirtPages;
+	psPageArrayData->uiLog2DevPageSize = uiLog2DevPageSize;
+	psPageArrayData->ui64DmaMask = ui64DmaMask;
+	psPageArrayData->ui32AllocFlags = ui32AllocFlags;
+	psPageArrayData->ui32CPUCacheFlags = ui32CPUCacheFlags;
+	psPageArrayData->ui32CMAAdjustedPageCount = 0;
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	OSAtomicWrite(&psPageArrayData->iNumOSPagesUnderMigration, 0);
+#endif
+
+	*ppsPageArrayDataPtr = psPageArrayData;
+	return PVRSRV_OK;
+
+/* Error path */
+e_free_cpuvirtaddrarray:
+	OSFreeMemNoStats(psPageArrayData->dmavirtarray);
+
+e_free_pagearray:
+	OSFreeMemNoStats(psPageArrayData->pagearray);
+
+e_free_kmem_cache:
+	kmem_cache_free(g_psLinuxPageArray, psPageArrayData);
+	PVR_DPF((PVR_DBG_ERROR,
+			 "%s: OS refused the memory allocation for the page pointer table. "
+			 "Did you ask for too much?",
+			 __func__));
+
+e_freed_none:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static inline void
+_ApplyCacheMaintenance(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages)
+{
+	void * pvAddr;
+
+	if (OSCPUCacheOpAddressType(psDevNode) == OS_CACHE_OP_ADDR_TYPE_VIRTUAL)
+	{
+		pgprot_t pgprot = PAGE_KERNEL;
+
+		IMG_UINT32 uiNumToClean = uiNumPages;
+		struct page **ppsCleanArray = ppsPage;
+
+		/* Map and flush page.
+		 * For large page arrays do it PVR_LINUX_PHYSMEM_MAX_KMAP_SIZE
+		 * at a time. */
+		while (uiNumToClean != 0)
+		{
+			IMG_UINT32 uiToClean = MIN(PVR_LINUX_PHYSMEM_MAX_KMAP_PAGES,
+			                           uiNumToClean);
+			IMG_CPU_PHYADDR sUnused =
+				{ IMG_CAST_TO_CPUPHYADDR_UINT(0xCAFEF00DDEADBEEFULL) };
+
+			pvAddr = pvr_vmap(ppsCleanArray, uiToClean, VM_MAP, pgprot);
+			if (!pvAddr)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Unable to flush page cache for new allocation, skipping flush."));
+				return;
+			}
+
+			CacheOpExec(psDevNode,
+						pvAddr,
+						pvAddr + PAGE_SIZE,
+						sUnused,
+						sUnused,
+						PVRSRV_CACHE_OP_FLUSH);
+
+			pvr_vunmap(pvAddr, uiToClean, pgprot);
+			ppsCleanArray = &(ppsCleanArray[uiToClean]);
+			uiNumToClean -= uiToClean;
+		}
+	}
+	else
+	{
+		IMG_UINT32 ui32Idx;
+
+		for (ui32Idx = 0; ui32Idx < uiNumPages; ++ui32Idx)
+		{
+			IMG_CPU_PHYADDR sCPUPhysAddrStart, sCPUPhysAddrEnd;
+
+			pvAddr = kmap(ppsPage[ui32Idx]);
+			sCPUPhysAddrStart.uiAddr = page_to_phys(ppsPage[ui32Idx]);
+			sCPUPhysAddrEnd.uiAddr = sCPUPhysAddrStart.uiAddr + PAGE_SIZE;
+
+			/* If we're zeroing, we need to make sure the cleared memory is pushed out
+			 * of the cache before the cache lines are invalidated */
+			CacheOpExec(psDevNode,
+						pvAddr,
+						pvAddr + PAGE_SIZE,
+						sCPUPhysAddrStart,
+						sCPUPhysAddrEnd,
+						PVRSRV_CACHE_OP_FLUSH);
+
+			kunmap(ppsPage[ui32Idx]);
+		}
+	}
+}
+
+/* Change the caching attribute of pages on x86 systems and takes care of
+ * cache maintenance. This function is supposed to be called once for pages that
+ * came from alloc_pages(). It expects an array of OS page sized pages!
+ *
+ * Flush/Invalidate pages in case the allocation is not cached. Necessary to
+ * remove pages from the cache that might be flushed later and corrupt memory. */
+static inline PVRSRV_ERROR
+_ApplyOSPagesAttribute(PVRSRV_DEVICE_NODE *psDevNode,
+					   struct page **ppsPage,
+					   IMG_UINT32 uiNumPages,
+					   IMG_BOOL bFlush,
+					   IMG_UINT32 ui32CPUCacheFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bCPUCached = PVRSRV_CHECK_CPU_CACHED(ui32CPUCacheFlags);
+	IMG_BOOL bCPUUncached = PVRSRV_CHECK_CPU_UNCACHED(ui32CPUCacheFlags);
+	IMG_BOOL bCPUWriteCombine = PVRSRV_CHECK_CPU_WRITE_COMBINE(ui32CPUCacheFlags);
+
+	if (ppsPage != NULL && uiNumPages != 0)
+	{
+#if defined(CONFIG_X86)
+		/* On x86 we have to set page cache attributes for non-cached pages.
+		 * The call is implicitly taking care of all flushing/invalidating
+		 * and therefore we can skip the usual cache maintenance after this. */
+		if (bCPUUncached || bCPUWriteCombine)
+		{
+			/* On x86 if we already have a mapping (e.g. low memory) we need to change the mode of
+				current mapping before we map it ourselves */
+			int ret = IMG_FALSE;
+
+			switch (PVRSRV_CPU_CACHE_MODE(ui32CPUCacheFlags))
+			{
+				case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+					ret = set_pages_array_uc(ppsPage, uiNumPages);
+					if (ret)
+					{
+						eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE;
+						PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to UC failed, returned %d", ret));
+					}
+					break;
+
+				case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+					ret = set_pages_array_wc(ppsPage, uiNumPages);
+					if (ret)
+					{
+						eError = PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE;
+						PVR_DPF((PVR_DBG_ERROR, "Setting Linux page caching mode to WC failed, returned %d", ret));
+					}
+					break;
+
+				case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+					break;
+
+				default:
+					break;
+			}
+		}
+		else
+#endif
+		{
+			if ( bFlush ||
+				 bCPUUncached || bCPUWriteCombine ||
+				 (bCPUCached && PVRSRV_CHECK_CPU_CACHE_CLEAN(ui32CPUCacheFlags)) )
+			{
+				/*  We can be given pages which still remain in the cache.
+					In order to make sure that the data we write through our mappings
+					doesn't get overwritten by later cache evictions we invalidate the
+					pages that are given to us.
+
+					Note:
+					This still seems to be true if we request cold pages, it's just less
+					likely to be in the cache. */
+				_ApplyCacheMaintenance(psDevNode,
+									   ppsPage,
+									   uiNumPages);
+			}
+		}
+	}
+
+	return eError;
+}
+
+/* Same as _AllocOSPage except it uses DMA framework to perform allocation.
+ * uiPageIndex is expected to be the pagearray index where to store the higher order page. */
+static PVRSRV_ERROR
+_AllocOSPage_CMA(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+				gfp_t gfp_flags,
+				IMG_UINT32 ui32AllocOrder,
+				IMG_UINT32 ui32MinOrder,
+				IMG_UINT32 uiPageIndex)
+{
+	void *virt_addr;
+	struct page *page;
+	dma_addr_t bus_addr;
+	IMG_UINT32 uiAllocIsMisaligned;
+	size_t alloc_size = PAGE_SIZE << ui32AllocOrder;
+	struct device *dev = psPageArrayData->psDevNode->psDevConfig->pvOSDevice;
+
+	do
+	{
+		DisableOOMKiller();
+#if defined(PVR_LINUX_PHYSMEM_SUPPRESS_DMA_AC)
+		virt_addr = NULL;
+#else
+		virt_addr = dma_alloc_coherent(dev, alloc_size, &bus_addr, gfp_flags);
+#endif
+		if (virt_addr == NULL)
+		{
+			/* The idea here is primarily to support some older kernels with
+			   broken or non-functioning DMA/CMA implementations (< Linux-3.4)
+			   and to also handle DMA/CMA allocation failures by attempting a
+			   normal page allocation though we expect dma_alloc_coherent()
+			   already attempts this internally also before failing but
+			   nonetheless it does no harm to retry the allocation ourselves */
+			page = alloc_pages(gfp_flags, ui32AllocOrder);
+			if (page)
+			{
+				/* Taint bus_addr as alloc_page, needed when freeing;
+				   also acquire the low memory page address only, this
+				   prevents mapping possible high memory pages into
+				   kernel virtual address space which might exhaust
+				   the VMALLOC address space */
+				bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page));
+				virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE;
+			}
+			else
+			{
+				EnableOOMKiller();
+				return PVRSRV_ERROR_OUT_OF_MEMORY;
+			}
+		}
+		else
+		{
+#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
+			page = pfn_to_page(bus_addr >> PAGE_SHIFT);
+#else
+			/* Assumes bus address space is identical to physical address space */
+			page = phys_to_page(bus_addr);
+#endif
+		}
+		EnableOOMKiller();
+
+		/* Physical allocation alignment works/hidden behind the scene transparently,
+		   we do this here if the allocated buffer address does not meet its alignment
+		   requirement by over-allocating using the next power-2 order and reporting
+		   aligned-adjusted values back to meet the requested alignment constraint.
+		   Evidently we waste memory by doing this so should only do so if we do not
+		   initially meet the alignment constraint. */
+		uiAllocIsMisaligned = DMA_GET_ADDR(bus_addr) & ((PAGE_SIZE<<ui32MinOrder)-1);
+		if (uiAllocIsMisaligned || ui32AllocOrder > ui32MinOrder)
+		{
+			IMG_BOOL bUsedAllocPages = DMA_IS_ALLOCPG_ADDR(bus_addr);
+			if (ui32AllocOrder == ui32MinOrder)
+			{
+				if (bUsedAllocPages)
+				{
+					__free_pages(page, ui32AllocOrder);
+				}
+				else
+				{
+					dma_free_coherent(dev, alloc_size, virt_addr, bus_addr);
+				}
+
+				ui32AllocOrder = ui32AllocOrder + 1;
+				alloc_size = PAGE_SIZE << ui32AllocOrder;
+
+				PVR_ASSERT(uiAllocIsMisaligned != 0);
+			}
+			else
+			{
+				size_t align_adjust = PAGE_SIZE << ui32MinOrder;
+
+				/* Adjust virtual/bus addresses to meet alignment */
+				bus_addr = bUsedAllocPages ? page_to_phys(page) : bus_addr;
+				align_adjust = PVR_ALIGN((size_t)bus_addr, align_adjust);
+				align_adjust -= (size_t)bus_addr;
+
+				if (align_adjust)
+				{
+					if (bUsedAllocPages)
+					{
+						page += align_adjust >> PAGE_SHIFT;
+						bus_addr = DMA_SET_ALLOCPG_ADDR(page_to_phys(page));
+						virt_addr = (void*)(uintptr_t) DMA_VADDR_NOT_IN_USE;
+					}
+					else
+					{
+						bus_addr += align_adjust;
+						virt_addr += align_adjust;
+#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
+						page = pfn_to_page(bus_addr >> PAGE_SHIFT);
+#else
+						/* Assumes bus address space is identical to physical address space */
+						page = phys_to_page(bus_addr);
+#endif
+					}
+
+					/* Store adjustments in PAGE_SIZE counts */
+					align_adjust = align_adjust >> PAGE_SHIFT;
+					bus_addr = DMA_SET_ALIGN_ADJUSTMENT(bus_addr, align_adjust);
+
+					/* Taint bus_addr due to over-allocation, allows us to free
+					 * memory correctly */
+					bus_addr = DMA_SET_ADJUSTED_ADDR(bus_addr);
+				}
+				uiAllocIsMisaligned = 0;
+			}
+		}
+	} while (uiAllocIsMisaligned);
+
+	/* Convert OSPageSize-based index into DevicePageSize-based index */
+	psPageArrayData->ui32CMAAdjustedPageCount += (alloc_size - (PAGE_SIZE << ui32AllocOrder));
+	psPageArrayData->dmavirtarray[uiPageIndex] = virt_addr;
+	psPageArrayData->dmaphysarray[uiPageIndex] = bus_addr;
+	psPageArrayData->pagearray[uiPageIndex] = page;
+
+	return PVRSRV_OK;
+}
+
+/* Allocate a page of order uiAllocOrder and stores it in the page array ppsPage at
+ * position uiPageIndex.
+ *
+ * If the order is higher than 0, it splits the page into multiples and
+ * stores them at position uiPageIndex to uiPageIndex+(1<<uiAllocOrder).
+ *
+ * This function is supposed to be used for uiMinOrder == 0 only! */
+static PVRSRV_ERROR
+_AllocOSPage(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			gfp_t gfp_flags,
+			IMG_UINT32 uiAllocOrder,
+			IMG_UINT32 uiMinOrder,
+			IMG_UINT32 uiPageIndex)
+{
+	struct page *psPage;
+	IMG_UINT32 ui32Count;
+
+	/* Parameter check. If it fails we write into the wrong places in the array. */
+	PVR_ASSERT(uiMinOrder == 0);
+
+	/* Allocate the page */
+	DisableOOMKiller();
+	psPage = alloc_pages(gfp_flags, uiAllocOrder);
+	EnableOOMKiller();
+
+	if (psPage == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* In case we need to, split the higher order page;
+	   this should only be used for order-0 allocations
+	   as higher order allocations should use DMA/CMA */
+	if (uiAllocOrder != 0)
+	{
+		split_page(psPage, uiAllocOrder);
+	}
+
+	/* Store the page (or multiple split pages) in the page array */
+	for (ui32Count = 0; ui32Count < (1 << uiAllocOrder); ui32Count++)
+	{
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+		OSMemSetMovablePageAttr(&psPage[ui32Count], psPageArrayData, uiPageIndex + ui32Count);
+#endif
+		psPageArrayData->pagearray[uiPageIndex + ui32Count] = &(psPage[ui32Count]);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+
+static inline void _AddMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                               const struct page *psPage,
+                                               IMG_UINT32 uiOrder)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = { page_to_phys(psPage) };
+	IMG_UINT32 uiPageSize = 1 << psPageArrayData->uiLog2DevPageSize;
+	PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                             NULL, sCPUPhysAddr,
+	                             uiPageSize << uiOrder,
+	                             psPageArrayData->uiPid
+	                             DEBUG_MEMSTATS_VALUES);
+}
+
+static inline void _RemoveMemAllocRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                                  const struct page *psPage)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	IMG_UINT32 uiStat = BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE)
+		? PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES
+		: PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#else
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#endif
+
+	PVRSRVStatsRemoveMemAllocRecord(uiStat, (IMG_UINT64) page_to_phys(psPage),
+	                                psPageArrayData->uiPid);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static inline void _TransferToMemZombieRecord_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                                       const struct page *psPage)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = { page_to_phys(psPage) };
+
+	PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                                  PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,
+	                                  (IMG_UINT64) sCPUPhysAddr.uiAddr,
+	                                  psPageArrayData->uiPid
+	                                  DEBUG_MEMSTATS_VALUES);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+
+static inline void _IncrMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid)
+{
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                            uiSize, uiPid);
+}
+
+static inline void _DecrMemAllocStat_UmaPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+                                              size_t uiSize, IMG_PID uiPid)
+{
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES;
+	}
+#endif
+
+	PVRSRVStatsDecrMemAllocStat(uiStat, uiSize, uiPid);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static inline void _ZombifyMemAllocStat_UmaPages(size_t uiSize, IMG_PID uiPid)
+{
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,
+	                            uiSize, uiPid);
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,
+	                            uiSize, uiPid);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+/* Allocation of OS pages: We may allocate 2^N order pages at a time for two reasons.
+ *
+ * Firstly to support device pages which are larger than OS. By asking the OS for 2^N
+ * order OS pages at a time we guarantee the device page is contiguous.
+ *
+ * Secondly for performance where we may ask for 2^N order pages to reduce the number
+ * of calls to alloc_pages, and thus reduce time for huge allocations.
+ *
+ * Regardless of page order requested, we need to break them down to track _OS pages.
+ * The maximum order requested is increased if all max order allocations were successful.
+ * If any request fails we reduce the max order.
+ */
+static PVRSRV_ERROR
+_AllocOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiArrayIndex = 0;
+	/* Order of the allocation to request, can be upgraded for large allocations
+	 * to aid efficient allocation. */
+	IMG_UINT32 ui32Order;
+	/* Min order the requested allocation can be. We express this in terms of the device
+	 * page size to the OS page size. The OS page size is the minimum page size we can request
+	 * else we cannot map it. Requesters may request further size / contiguity constraints in form
+	 * of the log2AllocPageSize */
+	IMG_UINT32 ui32MinOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bIncreaseMaxOrder = IMG_TRUE;
+
+	IMG_UINT32 ui32NumPageReq;
+	IMG_UINT32 uiOSPagesToAlloc;
+	IMG_UINT32 uiDevPagesFromPool = 0;
+
+	gfp_t gfp_flags = _GetGFPFlags(psPageArrayData);
+	gfp_t ui32GfpFlags;
+	gfp_t ui32HighOrderGfpFlags = ((gfp_flags & ~__GFP_RECLAIM) | __GFP_NORETRY);
+
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	struct page **ppsPageAttributeArray = NULL;
+
+	uiOSPagesToAlloc = psPageArrayData->uiTotalNumOSPages;
+
+	/* Try to get pages from the pool since it is faster;
+	   the page pool currently only supports zero-order pages
+	   thus currently excludes all DMA/CMA allocated memory.
+	   _ShouldInitMem() must not be used for bZero argument since it only
+	   applies to new pages allocated from the kernel.  */
+	_GetPagesFromPoolLocked(psPageArrayData->psDevNode,
+							psPageArrayData->ui32CPUCacheFlags,
+							psPageArrayData->ui32AllocFlags,
+							uiOSPagesToAlloc,
+							ui32MinOrder,
+							ppsPageArray,
+							&uiDevPagesFromPool);
+
+	uiArrayIndex = uiDevPagesFromPool;
+
+	if ((uiOSPagesToAlloc - uiDevPagesFromPool) < PVR_LINUX_HIGHORDER_ALLOCATION_THRESHOLD &&
+	    !BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{	/* Small allocations: ask for one device page at a time */
+		ui32Order = ui32MinOrder;
+		bIncreaseMaxOrder = IMG_FALSE;
+	}
+	else
+	{
+		/* Large zero-order or none zero-order allocations, ask for
+		   MAX(max-order, min-order) order pages at a time; alloc
+		   failures throttles this down to ZeroOrder allocations */
+		ui32Order = MAX(g_uiMaxOrder, ui32MinOrder);
+	}
+
+	/* Only if asking for more contiguity than we actually need, let it fail */
+	ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+	ui32NumPageReq = (1 << ui32Order);
+
+	while (uiArrayIndex < uiOSPagesToAlloc)
+	{
+		IMG_UINT32 ui32PageRemain = uiOSPagesToAlloc - uiArrayIndex;
+
+		while (ui32NumPageReq > ui32PageRemain)
+		{
+			/* Pages to request is larger than that remaining
+			   so ask for less so never over allocate */
+			ui32Order = MAX(ui32Order >> 1, ui32MinOrder);
+			ui32NumPageReq = (1 << ui32Order);
+			ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+		}
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			/* As the DMA/CMA framework rounds-up request to the
+			   next power-of-two, we request multiple uiMinOrder
+			   pages to satisfy allocation request in order to
+			   minimise wasting memory */
+			eError = _AllocOSPage_CMA(psPageArrayData,
+									  gfp_flags,
+									  ui32Order,
+									  ui32MinOrder,
+									  uiArrayIndex);
+		}
+		else
+		{
+			/* Allocate uiOrder pages at uiArrayIndex */
+			eError = _AllocOSPage(psPageArrayData,
+								  ui32GfpFlags,
+								  ui32Order,
+								  ui32MinOrder,
+								  uiArrayIndex);
+		}
+
+		if (eError == PVRSRV_OK)
+		{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+			if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+			{
+				/* We encode non Ghost entries for CMA so we can record in order chunks */
+				_AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[uiArrayIndex], ui32Order);
+			}
+			else
+			{
+				/* Higher order pages get split using split_page() meaning we can't create a higher
+				 * order record in case the logic below reduces the order request part way through
+				 */
+				IMG_UINT32 i;
+				for (i = 0; i < (1 << ui32Order); i++)
+				{
+					_AddMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[uiArrayIndex + i], 0);
+				}
+			}
+#endif
+#endif
+			/* Successful request. Move onto next. */
+			uiArrayIndex += ui32NumPageReq;
+		}
+		else
+		{
+			if (ui32Order > ui32MinOrder)
+			{
+				/* Last request failed. Let's ask for less next time */
+				ui32Order = MAX(ui32Order >> 1, ui32MinOrder);
+				bIncreaseMaxOrder = IMG_FALSE;
+				ui32NumPageReq = (1 << ui32Order);
+				ui32GfpFlags = (ui32Order > ui32MinOrder) ? ui32HighOrderGfpFlags : gfp_flags;
+				g_uiMaxOrder = ui32Order;
+			}
+			else
+			{
+				/* Failed to alloc pages at required contiguity. Failed allocation */
+				PVR_DPF((PVR_DBG_ERROR, "%s: %s failed to honour request at %u of %u, flags = %x, order = %u (%s)",
+								__func__,
+								BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA) ? "dma_alloc_coherent" : "alloc_pages",
+								uiArrayIndex,
+								uiOSPagesToAlloc,
+								ui32GfpFlags,
+								ui32Order,
+								PVRSRVGetErrorString(eError)));
+				eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+				goto e_free_pages;
+			}
+		}
+	}
+
+	if (bIncreaseMaxOrder && (g_uiMaxOrder < PVR_LINUX_PHYSMEM_MAX_ALLOC_ORDER_NUM))
+	{	/* All successful allocations on max order. Let's ask for more next time */
+		g_uiMaxOrder++;
+	}
+
+	/* Construct table of page pointers to apply attributes */
+	ppsPageAttributeArray = &ppsPageArray[uiDevPagesFromPool];
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT32 i;
+		dma_addr_t *psBusAddrArray = &psPageArrayData->dmaphysarray[uiDevPagesFromPool];
+
+		/* Iterate over page array generating ghost CMA pages */
+		for (i = 0; i < uiOSPagesToAlloc;)
+		{
+			IMG_UINT32 j;
+			IMG_UINT32 ui32NumPagesPerOrder = 1;
+
+			/* Iterate and populate until we find the next real page */
+			for (j = i + 1;
+			     j < uiOSPagesToAlloc &&
+			     ppsPageAttributeArray[j] == NULL;
+			     j++, ui32NumPagesPerOrder++)
+			{
+				/* Generate ghost page */
+				ppsPageAttributeArray[j] = ppsPageAttributeArray[i];
+				ppsPageAttributeArray[j] += ui32NumPagesPerOrder;
+				/* Mark DMA addr as ghost */
+				psBusAddrArray[j] = DMA_SET_CMA_GHOST(psBusAddrArray[i]);
+			}
+
+			i += ui32NumPagesPerOrder;
+		}
+	}
+
+	if (_ShouldInitMem(psPageArrayData->ui32AllocFlags) && ui32MinOrder == 0)
+	{
+		eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool,
+		                          ppsPageAttributeArray, PAGE_KERNEL,
+		                          PVRSRV_ZERO_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to zero pages (fast)"));
+			goto e_free_pages;
+		}
+	}
+	else if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC))
+	{
+		/* need to call twice because ppsPageArray and ppsPageAttributeArray
+		 * can point to different allocations: first for pages obtained from
+		 * the pool and then the remaining pages */
+		eError = _MemsetPageArray(uiDevPagesFromPool, ppsPageArray, PAGE_KERNEL,
+		                          PVRSRV_POISON_ON_ALLOC_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)"));
+		}
+		eError = _MemsetPageArray(uiOSPagesToAlloc - uiDevPagesFromPool,
+		                          ppsPageAttributeArray, PAGE_KERNEL,
+		                          PVRSRV_POISON_ON_ALLOC_VALUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to poison pages (fast)"));
+		}
+
+		/* for poisoning need to also flush the pool pages as the 0s have
+		 * been overwritten */
+		_ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsPageArray,
+		                       uiDevPagesFromPool);
+	}
+
+	/* Do the cache management as required */
+	eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode,
+									ppsPageAttributeArray,
+									uiOSPagesToAlloc - uiDevPagesFromPool,
+									BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_ZERO) ||
+									BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_ALLOC),
+									psPageArrayData->ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes"));
+		goto e_free_pages;
+	}
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated = psPageArrayData->uiTotalNumOSPages;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	{
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+		_IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)),
+		                           psPageArrayData->uiPid);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	return PVRSRV_OK;
+
+/* Error path */
+e_free_pages:
+	{
+		IMG_UINT32 ui32PageToFree;
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			IMG_UINT32 uiDevArrayIndex = uiArrayIndex >> ui32Order;
+			IMG_UINT32 uiDevPageSize = PAGE_SIZE << ui32Order;
+			PVR_ASSERT(ui32Order == ui32MinOrder);
+
+			for (ui32PageToFree = 0; ui32PageToFree < uiDevArrayIndex; ui32PageToFree++)
+			{
+				if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[ui32PageToFree]))
+				{
+					_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+									uiDevPageSize,
+									ui32MinOrder,
+									psPageArrayData->dmavirtarray[ui32PageToFree],
+									psPageArrayData->dmaphysarray[ui32PageToFree],
+									ppsPageArray[ui32PageToFree]);
+				}
+				psPageArrayData->dmaphysarray[ui32PageToFree]= (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[ui32PageToFree] = NULL;
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+		}
+		else
+		{
+			/* Free the pages we got from the pool */
+			for (ui32PageToFree = 0; ui32PageToFree < uiDevPagesFromPool; ui32PageToFree++)
+			{
+				/* Pages added to pool are always order 0 */
+				_FreeOSPage(0,
+							BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE),
+							psPageArrayData,
+							ppsPageArray[ui32PageToFree]);
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+
+			for (ui32PageToFree = uiDevPagesFromPool; ui32PageToFree < uiArrayIndex; ui32PageToFree++)
+			{
+				/* Higher order pages from _AllocOSPage are always split into order 0 using split_page */
+				_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsPageArray[ui32PageToFree]);
+				ppsPageArray[ui32PageToFree] = NULL;
+			}
+		}
+
+		return eError;
+	}
+}
+
+static INLINE PVRSRV_ERROR
+_CheckIfIndexInRange(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, IMG_UINT32 ui32Limit)
+{
+	if (pui32Indices[ui32Index] >= ui32Limit)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Given alloc index %u at %u is larger than page array %u.",
+		        __func__, pui32Indices[ui32Index], ui32Index, ui32Limit));
+		return PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+	}
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR
+_CheckIfPageNotAllocated(IMG_UINT32 ui32Index, IMG_UINT32 *pui32Indices, IMG_UINT32 uiOrder, struct page **ppsPageArray)
+{
+	if (ppsPageArray[pui32Indices[ui32Index] << uiOrder] != NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Mapping Idx Dev:%u|OS:%u at page array index %u already exists. "
+		        "Page struct %p", __func__,
+		        pui32Indices[ui32Index],
+		        pui32Indices[ui32Index] << uiOrder,
+		        ui32Index,
+		        ppsPageArray[pui32Indices[ui32Index] << uiOrder]));
+		return PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Allocation of OS pages: This function is used for sparse allocations.
+ *
+ * Sparse allocations provide only a proportion of sparse physical backing within the total
+ * virtual range. */
+static PVRSRV_ERROR
+_AllocOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+					 IMG_UINT32 *pauiDevPageAllocIndices,
+					 IMG_UINT32 uiDevPagesToAlloc)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_UINT32 uiDevPagesFromPool = 0;
+	IMG_UINT32 uiOSPagesToAlloc = uiDevPagesToAlloc * (1 << uiOrder);
+	IMG_UINT32 uiDevPagesAllocated = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+	const IMG_UINT32 ui32AllocFlags = psPageArrayData->ui32AllocFlags;
+	gfp_t ui32GfpFlags = _GetGFPFlags(psPageArrayData);
+
+	/* We use this page array to receive pages from the pool and then reuse it afterwards to
+	 * store pages that need their cache attribute changed on x86 */
+	struct page **ppsTempPageArray;
+	IMG_UINT32 uiTempPageArrayIndex = 0;
+
+	/* Allocate the temporary page array that we need here to receive pages
+	 * from the pool and to store pages that need their caching attributes changed.
+	 * Allocate number of OS pages to be able to use the attribute function later. */
+	ppsTempPageArray = OSAllocMem(sizeof(struct page*) * uiOSPagesToAlloc);
+	PVR_LOG_GOTO_IF_NOMEM(ppsTempPageArray, eError, e_exit);
+
+	/* Check the requested number of pages if they fit in the page array */
+	if (uiDevPagesAllocated <
+	        ((psPageArrayData->iNumOSPagesAllocated >> uiOrder) + uiDevPagesToAlloc))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Trying to allocate more pages (Order %u) than this buffer can handle, "
+				 "Request + Allocated < Max! Request %u, Allocated %u, Max %u.",
+				 __func__,
+				 uiOrder,
+				 uiDevPagesToAlloc,
+				 psPageArrayData->iNumOSPagesAllocated >> uiOrder,
+				 uiDevPagesAllocated));
+		eError = PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+		goto e_free_temp_array;
+	}
+
+	/* Try to get pages from the pool since it is faster. The pages from pool are going to be
+	 * allocated only if:
+	 * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 1 && uiOrder == 0
+	 * - PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES == 0 && uiOrder == 0 &&
+	 *   !(BIT_ISSET(ui32AllocFlags, FLAG_ZERO))
+	 * - !BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA)
+	 * _ShouldInitMem() must not be used for bZero argument since it only
+	 * applies to new pages allocated from the kernel.  */
+	_GetPagesFromPoolLocked(psPageArrayData->psDevNode,
+							psPageArrayData->ui32CPUCacheFlags,
+							psPageArrayData->ui32AllocFlags,
+							uiDevPagesToAlloc,
+							uiOrder,
+							ppsTempPageArray,
+							&uiDevPagesFromPool);
+
+	/* In general device pages can have higher order than 0 but page pool always provides only 0
+	 * order pages so they can be assigned to the OS pages values (in other words if we're
+	 * allocating non-4k pages uiDevPagesFromPool will always be 0) */
+	uiTempPageArrayIndex = uiDevPagesFromPool;
+
+	/* Move pages we got from the pool to the array. */
+	for (i = 0; i < uiDevPagesFromPool; i++)
+	{
+		eError = _CheckIfIndexInRange(i, pauiDevPageAllocIndices, uiDevPagesAllocated);
+		PVR_GOTO_IF_ERROR(eError, e_free_pool_pages);
+		eError = _CheckIfPageNotAllocated(i, pauiDevPageAllocIndices, uiOrder, ppsPageArray);
+		PVR_GOTO_IF_ERROR(eError, e_free_pool_pages);
+
+		/* In this case the allocation order is the same for device and OS page size, we can use
+		 * DevPageAllocIndices here without having to scale up from device to OS page indices.
+		 */
+		PVR_ASSERT(uiOrder == 0);
+		ppsPageArray[pauiDevPageAllocIndices[i]] = ppsTempPageArray[i];
+	}
+
+	/* Allocate pages from the OS */
+	for (i = uiDevPagesFromPool; i < uiDevPagesToAlloc; i++)
+	{
+		IMG_UINT32 uiOSPageAllocIndex = pauiDevPageAllocIndices[i] << uiOrder;
+		/* We can perform range checks in terms of device pages, then we scale the indices
+		 * to OS page size for the storage array.
+		 */
+		eError = _CheckIfIndexInRange(i, pauiDevPageAllocIndices, uiDevPagesAllocated);
+		PVR_GOTO_IF_ERROR(eError, e_free_pages);
+		eError = _CheckIfPageNotAllocated(i, pauiDevPageAllocIndices, uiOrder, ppsPageArray);
+		PVR_GOTO_IF_ERROR(eError, e_free_pages);
+
+		/* Allocated pages and assign them the array. */
+		if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			IMG_UINT32 uiSubPageInOrder;
+			dma_addr_t *psBusAddrArray = &psPageArrayData->dmaphysarray[uiDevPagesFromPool];
+
+			/* As the DMA/CMA framework rounds-up request to the
+			   next power-of-two, we request multiple uiMinOrder
+			   pages to satisfy allocation request in order to
+			   minimise wasting memory */
+			eError = _AllocOSPage_CMA(psPageArrayData,
+									  ui32GfpFlags,
+									  uiOrder,
+									  uiOrder,
+									  uiOSPageAllocIndex);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to alloc CMA pages"));
+				goto e_free_pages;
+			}
+
+			for (uiSubPageInOrder = 1;
+			     uiSubPageInOrder < (1 << uiOrder);
+			     uiSubPageInOrder++)
+			{
+				/* Generate ghost page */
+				ppsPageArray[uiOSPageAllocIndex + uiSubPageInOrder] = ppsPageArray[uiOSPageAllocIndex];
+				ppsPageArray[uiOSPageAllocIndex + uiSubPageInOrder] += uiSubPageInOrder;
+				/* Mark DMA addr as ghost */
+				psBusAddrArray[uiOSPageAllocIndex + uiSubPageInOrder] = DMA_SET_CMA_GHOST(psBusAddrArray[uiOSPageAllocIndex]);
+			}
+		}
+		else
+		{
+			eError = _AllocOSPage(psPageArrayData,
+								  ui32GfpFlags,
+								  uiOrder,
+								  uiOrder,
+								  uiOSPageAllocIndex);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to alloc CMA pages"));
+				goto e_free_pages;
+			}
+		}
+
+		if (ppsPageArray[uiOSPageAllocIndex] != NULL)
+		{
+			/* Append pages to the temporary array so it's easier to process
+			 * them later on. */
+
+			if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+			{
+				IMG_UINT32 idx;
+				struct page *psPageAddr;
+
+				psPageAddr = ppsPageArray[uiOSPageAllocIndex];
+
+				/* "divide" CMA pages into OS pages if they have higher order */
+				for (idx = 0; idx < (1 << uiOrder); idx++)
+				{
+					ppsTempPageArray[uiTempPageArrayIndex + idx] = psPageAddr;
+					psPageAddr++;
+				}
+				uiTempPageArrayIndex += (1 << uiOrder);
+			}
+			else
+			{
+				ppsTempPageArray[uiTempPageArrayIndex] = ppsPageArray[uiOSPageAllocIndex];
+				uiTempPageArrayIndex++;
+			}
+		}
+		else
+		{
+			/* Failed to alloc pages at required contiguity. Failed allocation */
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: alloc_pages failed to honour request at %u of %u, flags = %x, order = %u",
+			        __func__, i, uiDevPagesToAlloc, ui32GfpFlags, uiOrder));
+			eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+			goto e_free_pages;
+		}
+	}
+
+	if (_ShouldInitMem(ui32AllocFlags) && uiOrder == 0)
+	{
+		/* At this point this array contains pages allocated from the page pool at its start
+		 * and pages allocated from the OS after that.
+		 * If there are pages from the pool here they must be zeroed already hence we don't have
+		 * to do it again. This is because if PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES is enabled pool pages
+		 * are zeroed in the cleanup thread. If it's disabled they aren't, and in that case we never
+		 * allocate pages with FLAG_ZERO from the pool. This is why those pages need to be zeroed
+		 * here.
+		 * All of the above is true for the 0 order pages. For higher order we never allocated from
+		 * the pool and those pages are allocated already zeroed from the OS.
+		 * Long story short we can always skip pages allocated from the pool because they are either
+		 * zeroed or we didn't allocate any of them. */
+		eError = _MemsetPageArray(uiTempPageArrayIndex - uiDevPagesFromPool,
+		                          &ppsTempPageArray[uiDevPagesFromPool],
+		                          PAGE_KERNEL, PVRSRV_ZERO_VALUE);
+		PVR_LOG_GOTO_IF_FALSE(eError == PVRSRV_OK, "failed to zero pages (sparse)", e_free_pages);
+	}
+	else if (BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC))
+	{
+		/* Here we need to poison all of the pages regardless if they were
+		 * allocated from the pool or from the system. */
+		eError = _MemsetPageArray(uiTempPageArrayIndex, ppsTempPageArray,
+		                          PAGE_KERNEL, PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_IF_FALSE(eError == PVRSRV_OK, "failed to poison pages (sparse)");
+
+		/* We need to flush the cache for the poisoned pool pages here. The flush for the pages
+		 * allocated from the system is done below because we also need to add appropriate cache
+		 * attributes to them. Pages allocated from the pool already come with correct caching
+		 * mode. */
+		_ApplyCacheMaintenance(psPageArrayData->psDevNode, ppsTempPageArray, uiDevPagesFromPool);
+	}
+
+	/* Do the cache management as required */
+	eError = _ApplyOSPagesAttribute(psPageArrayData->psDevNode,
+	                                &ppsTempPageArray[uiDevPagesFromPool],
+	                                uiTempPageArrayIndex - uiDevPagesFromPool,
+	                                BIT_ISSET(ui32AllocFlags, FLAG_ZERO) ||
+	                                BIT_ISSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC),
+	                                psPageArrayData->ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to set page attributes"));
+		goto e_free_pages;
+	}
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated += uiOSPagesToAlloc;
+
+	/* Free temporary page array */
+	OSFreeMem(ppsTempPageArray);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiDevPagesToAlloc; i++)
+	{
+		_AddMemAllocRecord_UmaPages(psPageArrayData,
+		                            ppsPageArray[pauiDevPageAllocIndices[i] << uiOrder],
+		                            uiOrder);
+	}
+#else
+	_IncrMemAllocStat_UmaPages(((uiOSPagesToAlloc * PAGE_SIZE)+(psPageArrayData->ui32CMAAdjustedPageCount)),
+	                           psPageArrayData->uiPid);
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+e_free_pages:
+	if (BIT_ISSET(ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder;
+
+		/* Free the pages we just allocated from the CMA */
+		for (; i > uiDevPagesFromPool; i--)
+		{
+			IMG_UINT32 idx = (i - 1) << uiOrder;
+			IMG_UINT32 uiSubPageInOrder;
+			_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+			                uiDevPageSize,
+			                uiOrder,
+			                psPageArrayData->dmavirtarray[pauiDevPageAllocIndices[idx]],
+			                psPageArrayData->dmaphysarray[pauiDevPageAllocIndices[idx]],
+			                ppsPageArray[pauiDevPageAllocIndices[idx]]);
+			psPageArrayData->dmaphysarray[pauiDevPageAllocIndices[idx]]= (dma_addr_t) 0;
+			psPageArrayData->dmavirtarray[pauiDevPageAllocIndices[idx]] = NULL;
+			ppsPageArray[pauiDevPageAllocIndices[idx]] = NULL;
+
+			for (uiSubPageInOrder = 1;
+				 uiSubPageInOrder < (1 << uiOrder);
+				 uiSubPageInOrder++)
+			{
+				ppsPageArray[idx + uiSubPageInOrder] = NULL;
+				psPageArrayData->dmaphysarray[idx + uiSubPageInOrder] = (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[idx + uiSubPageInOrder] = NULL;
+			}
+		}
+
+	}
+	else
+	{
+		/* Free the pages we just allocated from the OS */
+		for (; i > uiDevPagesFromPool; i--)
+		{
+			_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsPageArray[pauiDevPageAllocIndices[i-1]]);
+			ppsPageArray[pauiDevPageAllocIndices[i-1]] = NULL;
+		}
+	}
+
+e_free_pool_pages:
+	/* And now free all of the pages we allocated from the pool. */
+	for (i = 0; i < uiDevPagesFromPool; i++)
+	{
+		_FreeOSPage(0, BIT_ISSET(ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE),
+		            psPageArrayData,
+		            ppsTempPageArray[i]);
+
+		/* not using _CheckIfIndexInRange() to not print error message */
+		if (pauiDevPageAllocIndices[i] < uiDevPagesAllocated)
+		{
+			ppsPageArray[pauiDevPageAllocIndices[i]] = NULL;
+		}
+	}
+
+e_free_temp_array:
+	OSFreeMem(ppsTempPageArray);
+
+e_exit:
+	return eError;
+}
+
+/* Allocate pages for a given page array.
+ *
+ * The executed allocation path depends whether an array with allocation
+ * indices has been passed or not */
+static PVRSRV_ERROR
+_AllocOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			  IMG_UINT32 *puiAllocIndices,
+			  IMG_UINT32 uiPagesToAlloc)
+{
+	PVRSRV_ERROR eError;
+	struct page **ppsPageArray;
+
+	/* Parameter checks */
+	PVR_ASSERT(NULL != psPageArrayData);
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		PVR_ASSERT(psPageArrayData->dmaphysarray != NULL);
+		PVR_ASSERT(psPageArrayData->dmavirtarray != NULL);
+	}
+	PVR_ASSERT(psPageArrayData->pagearray != NULL);
+	PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated);
+
+	ppsPageArray = psPageArrayData->pagearray;
+
+	/* Go the sparse alloc path if we have an array with alloc indices.*/
+	if (puiAllocIndices != NULL)
+	{
+		eError = _AllocOSPages_Sparse(psPageArrayData,
+									  puiAllocIndices,
+									  uiPagesToAlloc);
+	}
+	else
+	{
+		eError = _AllocOSPages_Fast(psPageArrayData);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e_exit;
+	}
+
+	_DumpPageArray(ppsPageArray,
+	               psPageArrayData->uiTotalNumOSPages >>
+	               (psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT) );
+
+	PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: allocated OS memory for PMR @0x%p", psPageArrayData));
+	return PVRSRV_OK;
+
+e_exit:
+	return eError;
+}
+
+/* Same as _FreeOSPage except free memory using DMA framework */
+static INLINE void
+_FreeOSPage_CMA(struct device *dev,
+				size_t alloc_size,
+				IMG_UINT32 uiOrder,
+				void *virt_addr,
+				dma_addr_t dev_addr,
+				struct page *psPage)
+{
+	if (DMA_IS_ALLOCPG_ADDR(dev_addr))
+	{
+#if defined(CONFIG_X86)
+		void *pvPageVAddr = page_address(psPage);
+		if (pvPageVAddr)
+		{
+			int ret = set_memory_wb((unsigned long)pvPageVAddr, 1);
+			if (ret)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to reset page attribute",
+						__func__));
+			}
+		}
+#endif
+
+		if (DMA_IS_ADDR_ADJUSTED(dev_addr))
+		{
+			psPage -= DMA_GET_ALIGN_ADJUSTMENT(dev_addr);
+			uiOrder += 1;
+		}
+
+		__free_pages(psPage, uiOrder);
+	}
+	else
+	{
+		if (DMA_IS_ADDR_ADJUSTED(dev_addr))
+		{
+			size_t align_adjust;
+
+			align_adjust = DMA_GET_ALIGN_ADJUSTMENT(dev_addr);
+			alloc_size = alloc_size << 1;
+
+			dev_addr = DMA_GET_ADDR(dev_addr);
+			dev_addr -= align_adjust << PAGE_SHIFT;
+			virt_addr -= align_adjust << PAGE_SHIFT;
+		}
+
+#if defined(DEBUG)
+		/* We should never attempt to free a CMA Ghost as it is handled in layer above */
+		if (DMA_IS_CMA_GHOST(dev_addr))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Attempt to free CMA Ghost entry!"));
+			return;
+		}
+#endif
+
+		dma_free_coherent(dev, alloc_size, virt_addr, DMA_GET_ADDR(dev_addr));
+	}
+}
+
+/* Free a single page back to the OS.
+ * Make sure the cache type is set back to the default value.
+ *
+ * Note:
+ * We must _only_ check bUnsetMemoryType in the case where we need to free
+ * the page back to the OS since we may have to revert the cache properties
+ * of the page to the default as given by the OS when it was allocated. */
+static void
+_FreeOSPage(IMG_UINT32 uiOrder,
+			IMG_BOOL bUnsetMemoryType,
+			PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			struct page *psPage)
+{
+
+#if defined(CONFIG_X86)
+	void *pvPageVAddr;
+	pvPageVAddr = page_address(psPage);
+
+	if (pvPageVAddr && bUnsetMemoryType)
+	{
+		int ret;
+
+		ret = set_memory_wb((unsigned long)pvPageVAddr, 1);
+		if (ret)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attribute",
+					 __func__));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(bUnsetMemoryType);
+#endif
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* May call _FreeOSPage from clean pages for pool path, it has no
+	 * association with PMR_OSPAGEARRAY_DATA at this point
+	 */
+	if (psPageArrayData)
+	{
+		OSMemUnsetMovablePageAttr(psPage, psPageArrayData);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psPageArrayData);
+#endif
+
+	__free_pages(psPage, uiOrder);
+}
+
+/* Free the struct holding the metadata */
+static void
+_FreeOSPagesArray(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	PVR_DPF((PVR_DBG_MESSAGE, "physmem_osmem_linux.c: freed OS memory for PMR @0x%p", psPageArrayData));
+
+	psPageArrayData->hPMR = NULL;
+
+	/* Check if the page array actually still exists.
+	 * It might be the case that has been moved to the page pool */
+	if (psPageArrayData->pagearray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->pagearray);
+	}
+
+	/* Check if we need to free additional DMA/CMA cpu kernel virtual address & device bus address arrays */
+	if (psPageArrayData->dmaphysarray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->dmaphysarray);
+	}
+	if (psPageArrayData->dmavirtarray != NULL)
+	{
+		OSFreeMemNoStats(psPageArrayData->dmavirtarray);
+	}
+
+	kmem_cache_free(g_psLinuxPageArray, psPageArrayData);
+}
+
+/* Free all or some pages from a sparse page array */
+static PVRSRV_ERROR
+_FreeOSPages_Sparse(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+					IMG_UINT32 *paui32DevPageFreeIndices,
+					IMG_UINT32 uiDevPagesToFree)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+	IMG_UINT32 uiNumDevPages;
+	struct page **ppsOSPagesToFree;
+	IMG_UINT32 *puiDevIndicesToFree;
+	IMG_UINT32 uiOSPageCount;
+	IMG_UINT32 uiTransferedDevPageCount;
+	IMG_UINT32 i;
+
+	/* We really should have something to free before we call this */
+	PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0);
+
+	if (paui32DevPageFreeIndices == NULL)
+	{
+		uiNumDevPages = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+		uiOSPageCount = psPageArrayData->iNumOSPagesAllocated;
+	}
+	else
+	{
+		uiNumDevPages = uiDevPagesToFree;
+		uiOSPageCount = uiNumDevPages << uiOrder;
+	}
+
+	/* Construct the array of pages before any free operation so error recovery is possible.
+	 * OSAllocMemNoStats required because this code may be run without the bridge lock held */
+	ppsOSPagesToFree = OSAllocMemNoStats(uiOSPageCount * sizeof(*ppsOSPagesToFree));
+	PVR_LOG_GOTO_IF_NOMEM(ppsOSPagesToFree, eError, err_array);
+	puiDevIndicesToFree = OSAllocMemNoStats(uiNumDevPages * sizeof(*puiDevIndicesToFree));
+	PVR_LOG_GOTO_IF_NOMEM(puiDevIndicesToFree, eError, err_indices);
+
+	/* Count how many allocated dev pages have been found. In case we need to unwind */
+	uiTransferedDevPageCount = 0;
+	for (i = 0; i < uiNumDevPages; i++)
+	{
+		IMG_UINT32 uiSubPageInOrder;
+		IMG_UINT32 idx = paui32DevPageFreeIndices ? paui32DevPageFreeIndices[i] : i;
+		/* Scale Device page index to OS page Idx */
+		idx <<= uiOrder;
+
+		if (ppsPageArray[idx] != NULL)
+		{
+			IMG_UINT32 uiOSFreeArrayIdx = uiTransferedDevPageCount << uiOrder;
+			puiDevIndicesToFree[uiTransferedDevPageCount] = idx;
+
+			for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+			{
+				ppsOSPagesToFree[uiOSFreeArrayIdx + uiSubPageInOrder] = ppsPageArray[idx + uiSubPageInOrder];
+				ppsPageArray[idx + uiSubPageInOrder] = NULL;
+			}
+			uiTransferedDevPageCount++;
+		}
+		else if (paui32DevPageFreeIndices != NULL)
+		{
+			IMG_UINT32 j;
+			PVR_DPF((PVR_DBG_ERROR, "Invalid free index (%u), possible duplicate", idx));
+			/* Restore the ppsPageArray to its initial state.
+			 * Reverse the loop and swap the ppsTempPages back into ppsPageArray. */
+			for (j = 0; j < uiTransferedDevPageCount; j++)
+			{
+				IMG_UINT32 uiSubPageInOrder;
+
+				idx = puiDevIndicesToFree[j];
+				ppsPageArray[idx] = ppsOSPagesToFree[j];
+
+				for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+				{
+					ppsPageArray[idx + uiSubPageInOrder] = ppsOSPagesToFree[j + uiSubPageInOrder];
+				}
+			}
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK, err_indices);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiTransferedDevPageCount; i++)
+	{
+		/* Scale Device page index to OS page Idx */
+		IMG_UINT32 idx = i << uiOrder;
+
+		_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsOSPagesToFree[idx]);
+	}
+#endif /* PVRSRV_ENABLE_PROCESS_STATS && PVRSRV_ENABLE_MEMORY_STATS */
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE))
+	{
+		for (i = 0; i < uiTransferedDevPageCount; i++)
+		{
+			/* Scale Device page index to OS page Idx */
+			IMG_UINT32 idx = i << uiOrder;
+
+			_PoisonDevicePage(psPageArrayData->psDevNode,
+			                  ppsOSPagesToFree[idx],
+			                  uiOrder,
+			                  psPageArrayData->ui32CPUCacheFlags,
+			                  PVRSRV_POISON_ON_FREE_VALUE);
+		}
+	}
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		IMG_UINT64 ui64DevPageSize = IMG_PAGE2BYTES64(psPageArrayData->uiLog2DevPageSize);
+		for (i = 0; i < uiTransferedDevPageCount; i++)
+		{
+			IMG_UINT32 uiSubPageInOrder;
+			/* Scale Device page index to OS page Idx */
+			IMG_UINT32 uiFreeIdx = i << uiOrder;
+			/* Get the page's original index in ppsPageArray, already scaled to dev pages */
+			IMG_UINT32 idx = puiDevIndicesToFree[i];
+			_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+			                ui64DevPageSize,
+			                uiOrder,
+			                psPageArrayData->dmavirtarray[idx],
+			                psPageArrayData->dmaphysarray[idx],
+			                ppsOSPagesToFree[uiFreeIdx]);
+			for (uiSubPageInOrder = 0;
+			     uiSubPageInOrder < (1 << uiOrder);
+			     uiSubPageInOrder++)
+			{
+				psPageArrayData->dmaphysarray[idx + uiSubPageInOrder] = (dma_addr_t)0;
+				psPageArrayData->dmavirtarray[idx + uiSubPageInOrder] = NULL;
+			}
+		}
+	}
+	else
+	{
+		IMG_BOOL bPagesInPool;
+
+		/* Try to move the temp page array to the pool */
+		bPagesInPool = _PutPagesToPoolLocked(psPageArrayData->psDevNode,
+		                                     psPageArrayData->ui32CPUCacheFlags,
+		                                     psPageArrayData->ui32AllocFlags,
+		                                     ppsOSPagesToFree,
+		                                     0,
+		                                     uiOSPageCount);
+		if (bPagesInPool)
+		{
+			/* Do NOT free ppsPagesToFree if it moved to the pool */
+			ppsOSPagesToFree = NULL;
+		}
+		else
+		{
+			/* Free pages and reset page caching attributes on x86 */
+#if defined(CONFIG_X86)
+			if (uiOSPageCount != 0 && BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE))
+			{
+				int iError;
+				iError = set_pages_array_wb(ppsOSPagesToFree, uiOSPageCount);
+
+				if (iError)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes", __func__));
+				}
+			}
+#endif
+
+			/* Free the pages */
+			for (i = 0; i < uiOSPageCount; i++)
+			{
+				_FreeOSPage(0, IMG_FALSE, psPageArrayData, ppsOSPagesToFree[i]);
+			}
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	_DecrMemAllocStat_UmaPages(psPageArrayData,
+	                           (uiOSPageCount * PAGE_SIZE) - psPageArrayData->ui32CMAAdjustedPageCount,
+	                           psPageArrayData->uiPid);
+#endif
+
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated -= uiOSPageCount;
+	PVR_ASSERT(0 <= psPageArrayData->iNumOSPagesAllocated);
+
+err_indices:
+	OSFreeMemNoStats(puiDevIndicesToFree);
+err_array:
+	if (ppsOSPagesToFree)
+	{
+		OSFreeMemNoStats(ppsOSPagesToFree);
+	}
+	return eError;
+}
+
+/* Free all the pages in a page array */
+static PVRSRV_ERROR
+_FreeOSPages_Fast(PMR_OSPAGEARRAY_DATA *psPageArrayData)
+{
+	IMG_BOOL bSuccess;
+	IMG_UINT32 i;
+	IMG_UINT32 uiOSNumPages = psPageArrayData->uiTotalNumOSPages;
+	IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_UINT32 uiDevNumPages = uiOSNumPages >> uiOrder;
+	IMG_UINT32 uiDevPageSize = PAGE_SIZE << uiOrder;
+	struct page **ppsPageArray = psPageArrayData->pagearray;
+
+	/* We really should have something to free before we call this */
+	PVR_ASSERT(psPageArrayData->iNumOSPagesAllocated != 0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	for (i = 0; i < uiOSNumPages; i++)
+	{
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+			{
+				_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+			}
+		}
+		else
+		{
+			_RemoveMemAllocRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+		}
+	}
+#else
+	_DecrMemAllocStat_UmaPages(psPageArrayData, uiOSNumPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+	                           psPageArrayData->uiPid);
+#endif
+#endif
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_POISON_ON_FREE))
+	{
+		for (i = 0; i < uiDevNumPages; i++)
+		{
+			/* Scale Device page index to OS page idx */
+			IMG_UINT32 idx = i << uiOrder;
+
+			_PoisonDevicePage(psPageArrayData->psDevNode,
+			                  ppsPageArray[idx],
+			                  uiOrder,
+			                  psPageArrayData->ui32CPUCacheFlags,
+			                  PVRSRV_POISON_ON_FREE_VALUE);
+		}
+	}
+
+	/* Try to move the page array to the pool */
+	bSuccess = _PutPagesToPoolLocked(psPageArrayData->psDevNode,
+	                                 psPageArrayData->ui32CPUCacheFlags,
+	                                 psPageArrayData->ui32AllocFlags,
+	                                 ppsPageArray,
+	                                 uiOrder,
+	                                 uiOSNumPages);
+	if (bSuccess)
+	{
+		psPageArrayData->pagearray = NULL;
+		goto exit_ok;
+	}
+
+	if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		for (i = 0; i < uiOSNumPages; i++)
+		{
+			if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+			{
+				_FreeOSPage_CMA(psPageArrayData->psDevNode->psDevConfig->pvOSDevice,
+								uiDevPageSize,
+								uiOrder,
+								psPageArrayData->dmavirtarray[i],
+								psPageArrayData->dmaphysarray[i],
+								ppsPageArray[i]);
+
+			}
+			psPageArrayData->dmaphysarray[i] = (dma_addr_t)0;
+			psPageArrayData->dmavirtarray[i] = NULL;
+			ppsPageArray[i] = NULL;
+		}
+	}
+	else
+	{
+#if defined(CONFIG_X86)
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE))
+		{
+			int ret;
+
+			ret = set_pages_array_wb(ppsPageArray, uiOSNumPages);
+			if (ret)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to reset page attributes",
+						 __func__));
+			}
+		}
+#endif
+
+		for (i = 0; i < uiOSNumPages; i++)
+		{
+			_FreeOSPage(uiOrder, IMG_FALSE, psPageArrayData, ppsPageArray[i]);
+			ppsPageArray[i] = NULL;
+		}
+	}
+
+exit_ok:
+	/* Update metadata */
+	psPageArrayData->iNumOSPagesAllocated = 0;
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRFreeZombiePagesOSMem(PMR_IMPL_ZOMBIEPAGES pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psZombiePageArray = pvPriv;
+
+	eError = _FreeOSPages(psZombiePageArray,
+	                      NULL,
+	                      0 /* Unused */);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	_FreeOSPagesArray(psZombiePageArray);
+
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+
+/* Extracts ui32ExtractPageCount of pages referenced in pai32ExtractIndices from the psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted pages information.
+ */
+static PVRSRV_ERROR
+_ExtractPages(PMR_OSPAGEARRAY_DATA *psSrcPageArrayData,
+			  IMG_UINT32 *pai32ExtractIndices,
+			  IMG_UINT32 ui32ExtractPageCount,
+			  PMR_OSPAGEARRAY_DATA** psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i, uiSubPageInOrder, uiOrder;
+	PMR_OSPAGEARRAY_DATA* psDstPageArrayData;
+
+	/* Alloc PMR_OSPAGEARRAY_DATA for the extracted pages */
+	eError = _AllocOSPageArray(psSrcPageArrayData->psDevNode,
+	                           (IMG_UINT64)ui32ExtractPageCount << psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->ui32AllocFlags,
+	                           psSrcPageArrayData->ui32CPUCacheFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_AllocOSPageArray failed in _ExtractPages"));
+		return eError;
+	}
+
+	uiOrder = psSrcPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+
+	/* Transfer from src pagearray to dst pagearray */
+	for (i = 0; i < ui32ExtractPageCount; i++)
+	{
+		IMG_UINT32 idxSrc = pai32ExtractIndices[i] << uiOrder;
+		IMG_UINT32 idxExtracted = i << uiOrder;
+
+		if (psSrcPageArrayData->pagearray[idxSrc] != NULL)
+		{
+			for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+			{
+				psDstPageArrayData->pagearray[idxExtracted + uiSubPageInOrder] =
+				    psSrcPageArrayData->pagearray[idxSrc + uiSubPageInOrder];
+
+				psSrcPageArrayData->pagearray[idxSrc + uiSubPageInOrder] = NULL;
+			}
+		}
+	}
+
+	/* Do the same for dmaphysarray and dmavirtarray if allocated with CMA */
+	if (BIT_ISSET(psSrcPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		for (i = 0; i < ui32ExtractPageCount; i++)
+		{
+			IMG_UINT32 idxSrc = pai32ExtractIndices[i] << uiOrder;
+			IMG_UINT32 idxDst = i << uiOrder;
+
+			if (psSrcPageArrayData->dmaphysarray[idxSrc] != (dma_addr_t)0 ||
+			    psSrcPageArrayData->dmavirtarray[idxSrc] != NULL)
+			{
+				for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+				{
+					psDstPageArrayData->dmaphysarray[idxDst + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmaphysarray[idxSrc + uiSubPageInOrder];
+					psDstPageArrayData->dmavirtarray[idxDst + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmavirtarray[idxSrc + uiSubPageInOrder];
+
+					psSrcPageArrayData->dmaphysarray[idxSrc + uiSubPageInOrder] = (dma_addr_t)0;
+					psSrcPageArrayData->dmavirtarray[idxSrc + uiSubPageInOrder] = NULL;
+				}
+			}
+		}
+	}
+
+	/* Update page counts */
+	psSrcPageArrayData->iNumOSPagesAllocated -= ui32ExtractPageCount << uiOrder;
+	psDstPageArrayData->iNumOSPagesAllocated += ui32ExtractPageCount << uiOrder;
+
+	*psOutPageArrayData = psDstPageArrayData;
+	return PVRSRV_OK;
+}
+
+/* Extracts all allocated pages referenced psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted
+ * pages information.
+ */
+static PVRSRV_ERROR
+_ExtractAllPages(PMR_OSPAGEARRAY_DATA *psSrcPageArrayData,
+				 PMR_OSPAGEARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i, uiSubPageInOrder, uiOrder;
+	PMR_OSPAGEARRAY_DATA* psDstPageArrayData;
+	IMG_UINT32 uiPagesCopied = 0;
+
+	/* Alloc PMR_OSPAGEARRAY_DATA for the extracted pages */
+	eError = _AllocOSPageArray(psSrcPageArrayData->psDevNode,
+	                           (IMG_UINT64)psSrcPageArrayData->iNumOSPagesAllocated << psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->uiLog2DevPageSize,
+	                           psSrcPageArrayData->ui32AllocFlags,
+	                           psSrcPageArrayData->ui32CPUCacheFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_AllocOSPageArray failed in _ExtractPages"));
+		return eError;
+	}
+
+	uiOrder = psSrcPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+
+	/* Transfer from src pagearray to dst pagearray */
+	/* Iterate through all pages in psSrcPageArrayData but stop once
+	 * we have copied psSrcPageArrayData->iNumOSPagesAllocated pages to
+	 * psDstPageArrayData.
+	 */
+	for (i = 0; ((i < psSrcPageArrayData->uiTotalNumOSPages) &&
+	             (uiPagesCopied < psSrcPageArrayData->iNumOSPagesAllocated)); i++)
+	{
+		IMG_UINT32 idx = i << uiOrder;
+
+		if (psSrcPageArrayData->pagearray[idx] != NULL)
+		{
+
+			for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+			{
+				psDstPageArrayData->pagearray[uiPagesCopied + uiSubPageInOrder] =
+				        psSrcPageArrayData->pagearray[idx + uiSubPageInOrder];
+
+				psSrcPageArrayData->pagearray[idx + uiSubPageInOrder] = NULL;
+			}
+			uiPagesCopied++;
+		}
+	}
+
+	/* Reset uiPagesCopied */
+	uiPagesCopied = 0;
+
+	/* Do the same for dmaphysarray and dmavirtarray if allocated with CMA */
+	if (BIT_ISSET(psSrcPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+	{
+		/* Iterate through all pages in psSrcPageArrayData but stop once
+		 * we have copied psSrcPageArrayData->iNumOSPagesAllocated pages to
+		 * psDstPageArrayData.
+		 */
+		for (i = 0; ((i < psSrcPageArrayData->uiTotalNumOSPages) &&
+		             (uiPagesCopied < psSrcPageArrayData->iNumOSPagesAllocated)); i++)
+		{
+			IMG_UINT32 idx = i << uiOrder;
+
+			if (psSrcPageArrayData->dmaphysarray[i] != (dma_addr_t)0 ||
+			    psSrcPageArrayData->dmavirtarray[i] != NULL)
+			{
+				for (uiSubPageInOrder = 0; uiSubPageInOrder < (1 << uiOrder); uiSubPageInOrder++)
+				{
+					psDstPageArrayData->dmaphysarray[uiPagesCopied + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmaphysarray[idx + uiSubPageInOrder];
+					psDstPageArrayData->dmavirtarray[uiPagesCopied + uiSubPageInOrder] =
+					    psSrcPageArrayData->dmavirtarray[idx + uiSubPageInOrder];
+
+					psSrcPageArrayData->dmaphysarray[idx + uiSubPageInOrder] = (dma_addr_t)0;
+					psSrcPageArrayData->dmavirtarray[idx + uiSubPageInOrder] = NULL;
+				}
+				uiPagesCopied++;
+			}
+		}
+	}
+
+	/* Update page counts */
+	psDstPageArrayData->iNumOSPagesAllocated = psSrcPageArrayData->iNumOSPagesAllocated << uiOrder;
+	psSrcPageArrayData->iNumOSPagesAllocated = 0;
+
+	*psOutPageArrayData = psDstPageArrayData;
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+/* Free pages from a page array.
+ * Takes care of mem stats and chooses correct free path depending on parameters. */
+static PVRSRV_ERROR
+_FreeOSPages(PMR_OSPAGEARRAY_DATA *psPageArrayData,
+			 IMG_UINT32 *pai32FreeIndices,
+			 IMG_UINT32 ui32FreePageCount)
+{
+	PVRSRV_ERROR eError;
+
+	/* Go the sparse or non-sparse path */
+	if (psPageArrayData->iNumOSPagesAllocated != psPageArrayData->uiTotalNumOSPages
+		|| pai32FreeIndices != NULL)
+	{
+		eError = _FreeOSPages_Sparse(psPageArrayData,
+									 pai32FreeIndices,
+									 ui32FreePageCount);
+	}
+	else
+	{
+		eError = _FreeOSPages_Fast(psPageArrayData);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "_FreeOSPages_FreePages failed"));
+	}
+
+	_DumpPageArray(psPageArrayData->pagearray,
+	               psPageArrayData->uiTotalNumOSPages >>
+	              (psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT) );
+
+	return eError;
+}
+
+/*
+ *
+ * Implementation of callback functions
+ *
+ */
+
+/* Destruction function is called after last reference disappears,
+ * but before PMR itself is freed.
+ */
+static void
+PMRFinalizeOSMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+
+	/* We can't free pages until now. */
+	if (psOSPageArrayData->iNumOSPagesAllocated != 0)
+	{
+
+		eError = _FreeOSPages(psOSPageArrayData, NULL, 0);
+		PVR_LOG_IF_ERROR(eError, "_FreeOSPages");
+		PVR_ASSERT(eError == PVRSRV_OK); /* can we do better? */
+	}
+
+	_FreeOSPagesArray(psOSPageArrayData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static PVRSRV_ERROR PMRZombifyOSMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PMR_OSPAGEARRAY_DATA *psPageArrayData = pvPriv;
+	IMG_UINT32 uiNumOSPages = psPageArrayData->uiTotalNumOSPages;
+
+	BIT_SET(psPageArrayData->ui32AllocFlags, FLAG_IS_ZOMBIE);
+
+	/* no need to check for free indices as it's always all the memory we're
+	 * freeing */
+	if (psPageArrayData->iNumOSPagesAllocated != psPageArrayData->uiTotalNumOSPages)
+	{
+		/* _FreeOSPages_Sparse() path */
+
+		struct page *const *const ppsPageArray = psPageArrayData->pagearray;
+		const IMG_UINT32 uiFlags = psPageArrayData->ui32AllocFlags;
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		IMG_UINT32 i;
+
+		if (BIT_ISSET(uiFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (ppsPageArray[i] != NULL && !DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+		else
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (ppsPageArray[i] != NULL)
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+		const IMG_UINT32 uiOrder = psPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+		IMG_UINT32 uiNumDevPages = psPageArrayData->uiTotalNumOSPages >> uiOrder;
+		IMG_UINT32 i, j, uiAllocatedNumPages = 0;
+
+		if (BIT_ISSET(uiFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumDevPages; i++)
+			{
+				IMG_UINT32 idx = i << uiOrder;
+				if (ppsPageArray[idx] != NULL)
+				{
+					uiAllocatedNumPages++;
+				}
+			}
+			uiAllocatedNumPages <<= uiOrder;
+		}
+		else
+		{
+			for (i = 0; i < uiNumDevPages; i++)
+			{
+				IMG_UINT32 idx = i << uiOrder;
+				if (ppsPageArray[idx] != NULL)
+				{
+					for (j = 0; j < (1<<uiOrder); j++)
+					{
+						uiAllocatedNumPages++;
+					}
+				}
+			}
+		}
+
+		_ZombifyMemAllocStat_UmaPages(
+		    uiAllocatedNumPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+		    psPageArrayData->uiPid
+		);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+	else
+	{
+		/* _FreeOSPages_Fast() path */
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+		struct page *const *const ppsPageArray = psPageArrayData->pagearray;
+		IMG_UINT32 i;
+
+		if (BIT_ISSET(psPageArrayData->ui32AllocFlags, FLAG_DMA_CMA))
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				if (!DMA_IS_CMA_GHOST(psPageArrayData->dmaphysarray[i]))
+				{
+					_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+				}
+			}
+		}
+		else
+		{
+			for (i = 0; i < uiNumOSPages; i++)
+			{
+				_TransferToMemZombieRecord_UmaPages(psPageArrayData, ppsPageArray[i]);
+			}
+		}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+		_ZombifyMemAllocStat_UmaPages(
+		    uiNumOSPages * PAGE_SIZE - psPageArrayData->ui32CMAAdjustedPageCount,
+		    psPageArrayData->uiPid
+		);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	}
+#else /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Callback function for locking the system physical page addresses.
+ * This function must be called before the lookup address func. */
+static PVRSRV_ERROR
+PMRLockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+
+	if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		/* Allocate Memory for deferred allocation */
+		eError = _AllocOSPages(psOSPageArrayData, NULL, psOSPageArrayData->uiTotalNumOSPages);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	}
+
+	eError = PVRSRV_OK;
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv,
+                               PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesOSMem(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	/* Just drops the refcount. */
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_OSPAGEARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+	*ppvZombiePages = NULL;
+#endif
+
+	if (BIT_ISSET(psOSPageArrayData->ui32AllocFlags, FLAG_ONDEMAND))
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (psOSPageArrayData->iNumOSPagesAllocated == 0)
+		{
+			*ppvZombiePages = NULL;
+			return PVRSRV_OK;
+		}
+
+		eError = _ExtractAllPages(psOSPageArrayData,
+		                          &psExtractedPagesPageArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractAllPages", e0);
+
+		if (psExtractedPagesPageArray)
+		{
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyOSMem(psExtractedPagesPageArray, NULL);
+			PVR_WARN_IF_ERROR(eError, "PMRZombifyOSMem");
+		}
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		/* Free Memory for deferred allocation */
+		eError = _FreeOSPages(psOSPageArrayData,
+							  NULL,
+							  0);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+#endif
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+e0:
+#endif
+	PVR_ASSERT(eError == PVRSRV_OK);
+	return eError;
+}
+
+static INLINE IMG_BOOL IsOffsetValid(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData,
+                                     IMG_UINT32 ui32Offset)
+{
+	return (ui32Offset >> psOSPageArrayData->uiLog2DevPageSize) <
+	    psOSPageArrayData->uiTotalNumOSPages;
+}
+
+/* Determine PA for specified offset into page array. */
+static IMG_DEV_PHYADDR GetOffsetPA(const PMR_OSPAGEARRAY_DATA *psOSPageArrayData,
+                                   IMG_UINT32 ui32Offset)
+{
+	IMG_UINT32 ui32PageIndex = ui32Offset >> PAGE_SHIFT;
+	IMG_UINT32 ui32InPageOffset = ui32Offset - (ui32PageIndex << PAGE_SHIFT);
+	IMG_DEV_PHYADDR sPA;
+
+	PVR_ASSERT(ui32InPageOffset < (1U << PAGE_SHIFT));
+
+	sPA.uiAddr = page_to_phys(psOSPageArrayData->pagearray[ui32PageIndex]);
+	sPA.uiAddr += ui32InPageOffset;
+
+	return sPA;
+}
+
+/* N.B. It is assumed that PMRLockSysPhysAddressesOSMem() is called _before_ this function! */
+static PVRSRV_ERROR
+PMRSysPhysAddrOSMem(PMR_IMPL_PRIVDATA pvPriv,
+					IMG_UINT32 ui32Log2PageSize,
+					IMG_UINT32 ui32NumOfPages,
+					IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					IMG_UINT64 ui64IPAPolicyValue,
+					IMG_UINT64 ui64IPAClearMask,
+#endif
+					IMG_BOOL *pbValid,
+					IMG_DEV_PHYADDR *psDevPAddr)
+{
+	const PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+	IMG_UINT32 uiIdx;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (psOSPageArrayData->uiLog2DevPageSize < ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "for incompatible contiguity %u!",
+		         __func__,
+		         ui32Log2PageSize));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (OSAtomicRead(&psOSPageArrayData->iNumOSPagesUnderMigration))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "in process of page migration!",
+		         __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+#endif
+
+	for (uiIdx=0; uiIdx < ui32NumOfPages; uiIdx++)
+	{
+		if (pbValid[uiIdx])
+		{
+			PVR_LOG_RETURN_IF_FALSE(IsOffsetValid(psOSPageArrayData, puiOffset[uiIdx]),
+			                        "puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+			psDevPAddr[uiIdx] = GetOffsetPA(psOSPageArrayData, puiOffset[uiIdx]);
+
+#if !defined(PVR_LINUX_PHYSMEM_USE_HIGHMEM_ONLY)
+			/* this is just a precaution, normally this should be always
+			 * available */
+			if (psOSPageArrayData->ui64DmaMask)
+			{
+				if (psDevPAddr[uiIdx].uiAddr > psOSPageArrayData->ui64DmaMask)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: physical address"
+							" (%" IMG_UINT64_FMTSPECX ") out of allowable range"
+							" [0; %" IMG_UINT64_FMTSPECX "]", __func__,
+							psDevPAddr[uiIdx].uiAddr,
+							psOSPageArrayData->ui64DmaMask));
+					BUG();
+				}
+			}
+#endif
+#if defined(SUPPORT_STATIC_IPA)
+			/* Modify the physical address with the associated IPA values */
+			psDevPAddr[uiIdx].uiAddr &= ~ui64IPAClearMask;
+			psDevPAddr[uiIdx].uiAddr |= ui64IPAPolicyValue;
+#endif
+
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+typedef struct _PMR_OSPAGEARRAY_KERNMAP_DATA_ {
+	void *pvBase;
+	IMG_UINT32 ui32PageCount;
+	pgprot_t PageProps;
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	IMG_UINT32 ui32CpuCacheFlags;
+#endif
+} PMR_OSPAGEARRAY_KERNMAP_DATA;
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv,
+								 size_t uiOffset,
+								 size_t uiSize,
+								 void **ppvKernelAddressOut,
+								 IMG_HANDLE *phHandleOut,
+								 PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_OSPAGEARRAY_DATA *psOSPageArrayData = pvPriv;
+	void *pvAddress;
+	pgprot_t prot = PAGE_KERNEL;
+	IMG_UINT32 ui32PageOffset=0;
+	size_t uiMapOffset=0;
+	IMG_UINT32 ui32PageCount = 0;
+	IMG_UINT32 uiLog2DevPageSize = psOSPageArrayData->uiLog2DevPageSize;
+	struct page **pagearray;
+	PMR_OSPAGEARRAY_KERNMAP_DATA *psData;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (OSAtomicRead(&psOSPageArrayData->iNumOSPagesUnderMigration))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "in process of page migration!",
+		         __func__));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+#endif
+
+	/*
+		Zero offset and size as a special meaning which means map in the
+		whole of the PMR, this is due to fact that the places that call
+		this callback might not have access to be able to determine the
+		physical size
+	*/
+	if ((uiOffset == 0) && (uiSize == 0))
+	{
+		ui32PageOffset = 0;
+		uiMapOffset = 0;
+		/* Page count = amount of OS pages */
+		ui32PageCount = psOSPageArrayData->iNumOSPagesAllocated;
+	}
+	else
+	{
+		size_t uiEndoffset;
+
+		ui32PageOffset = uiOffset >> uiLog2DevPageSize;
+		uiMapOffset = uiOffset - (ui32PageOffset << uiLog2DevPageSize);
+		uiEndoffset = uiOffset + uiSize - 1;
+		/* Add one as we want the count, not the offset */
+		/* Page count = amount of device pages (note uiLog2DevPageSize being used) */
+		ui32PageCount = (uiEndoffset >> uiLog2DevPageSize) + 1;
+		ui32PageCount -= ui32PageOffset;
+	}
+
+	switch (PVRSRV_CPU_CACHE_MODE(psOSPageArrayData->ui32CPUCacheFlags))
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				prot = pgprot_noncached(prot);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				prot = pgprot_writecombine(prot);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+				break;
+
+		default:
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto e0;
+	}
+
+	pagearray = &psOSPageArrayData->pagearray[ui32PageOffset];
+
+	psData = OSAllocMem(sizeof(*psData));
+	if (psData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	pvAddress = pvr_vmap(pagearray, ui32PageCount, VM_MAP, prot);
+	if (pvAddress == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e1;
+	}
+
+	*ppvKernelAddressOut = pvAddress + uiMapOffset;
+	psData->pvBase = pvAddress;
+	psData->ui32PageCount = ui32PageCount;
+	psData->PageProps = prot;
+	*phHandleOut = psData;
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	{
+		IMG_CPU_PHYADDR pvAddrPhy;
+		pvAddrPhy.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(*pagearray));
+		InsertMappingRecord(PMR_GetAnnotation(psOSPageArrayData->hPMR),
+							psOSPageArrayData->uiPid,
+							pvAddress,
+							pvAddrPhy,
+							psOSPageArrayData->ui32CPUCacheFlags,
+							uiMapOffset,
+							ui32PageCount);
+
+		psData->ui32CpuCacheFlags = psOSPageArrayData->ui32CPUCacheFlags;
+	}
+#endif
+
+	return PVRSRV_OK;
+	/*
+	  error exit paths follow
+	*/
+e1:
+	OSFreeMem(psData);
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataOSMem(PMR_IMPL_PRIVDATA pvPriv,
+											 IMG_HANDLE hHandle)
+{
+	PMR_OSPAGEARRAY_KERNMAP_DATA *psData = hHandle;
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	{
+		IMG_CPU_PHYADDR pvAddrPhy;
+		pvAddrPhy.uiAddr = IMG_CAST_TO_CPUPHYADDR_UINT(page_to_phys(vmalloc_to_page(psData->pvBase)));
+		InsertUnMappingRecord(psData->pvBase,
+				pvAddrPhy,
+				psData->ui32CpuCacheFlags,
+				psData->ui32PageCount);
+	}
+#endif
+
+	pvr_vunmap(psData->pvBase, psData->ui32PageCount, psData->PageProps);
+	OSFreeMem(psData);
+}
+
+/*************************************************************************/ /*!
+@Function       PMRChangeSparseMemOSMem
+@Description    This function Changes the sparse mapping by allocating and
+                freeing of pages. It changes the GPU and CPU maps accordingly.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PMRChangeSparseMemOSMem(PMR_IMPL_PRIVDATA pPriv,
+						const PMR *psPMR,
+						IMG_UINT32 ui32AllocPageCount,
+						IMG_UINT32 *pai32AllocIndices,
+						IMG_UINT32 ui32FreePageCount,
+						IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+						PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+						IMG_UINT32 uiFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappingTable(psPMR);
+	PMR_OSPAGEARRAY_DATA *psPMRPageArrayData = (PMR_OSPAGEARRAY_DATA *)pPriv;
+	struct page **psPageArray = psPMRPageArrayData->pagearray;
+	void **psDMAVirtArray = psPMRPageArrayData->dmavirtarray;
+	dma_addr_t *psDMAPhysArray = psPMRPageArrayData->dmaphysarray;
+	IMG_UINT32 ui32AdtnlAllocPages = 0; /*<! Number of pages to alloc from the OS */
+	IMG_UINT32 ui32AdtnlFreePages = 0; /*<! Number of pages to free back to the OS */
+	IMG_UINT32 ui32CommonRequestCount = 0; /*<! Number of pages to move position in the page array */
+	IMG_UINT32 ui32Loop = 0;
+	IMG_UINT32 ui32Index = 0;
+	IMG_UINT32 uiAllocpgidx;
+	IMG_UINT32 uiFreepgidx;
+	IMG_UINT32 uiOrder = psPMRPageArrayData->uiLog2DevPageSize - PAGE_SHIFT;
+	IMG_BOOL bCMA = BIT_ISSET(psPMRPageArrayData->ui32AllocFlags, FLAG_DMA_CMA);
+
+
+	/* Check SPARSE flags and calculate pages to allocate and free */
+	if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH))
+	{
+		ui32CommonRequestCount = (ui32AllocPageCount > ui32FreePageCount) ?
+				ui32FreePageCount : ui32AllocPageCount;
+
+		PDUMP_PANIC(PMR_DeviceNode(psPMR), SPARSEMEM_SWAP, "Request to swap alloc & free pages not supported");
+	}
+
+	if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC))
+	{
+		ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequestCount;
+	}
+	else
+	{
+		ui32AllocPageCount = 0;
+	}
+
+	if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE))
+	{
+		ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequestCount;
+	}
+	else
+	{
+		ui32FreePageCount = 0;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+
+	if (0 == (ui32CommonRequestCount || ui32AdtnlAllocPages || ui32AdtnlFreePages))
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Missing parameters for number of pages to alloc/free",
+		         __func__));
+		return eError;
+	}
+
+	/* The incoming request is classified into two operations independent of
+	 * each other: alloc & free pages.
+	 * These operations can be combined with two mapping operations as well
+	 * which are GPU & CPU space mappings.
+	 *
+	 * From the alloc and free page requests, the net amount of pages to be
+	 * allocated or freed is computed. Pages that were requested to be freed
+	 * will be reused to fulfil alloc requests.
+	 *
+	 * The order of operations is:
+	 * 1. Allocate new pages from the OS
+	 * 2. Move the free pages from free request to alloc positions.
+	 * 3. Free the rest of the pages not used for alloc
+	 *
+	 * Alloc parameters are validated at the time of allocation
+	 * and any error will be handled then. */
+
+	/* Validate the free indices */
+	if (ui32FreePageCount)
+	{
+		if (NULL != pai32FreeIndices){
+
+			for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++)
+			{
+				uiFreepgidx = pai32FreeIndices[ui32Loop];
+
+				if (uiFreepgidx >= (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder))
+				{
+					eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+					goto e0;
+				}
+
+				if (NULL == (psPageArray[uiFreepgidx << uiOrder]))
+				{
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					PVR_DPF((PVR_DBG_ERROR,
+					         "%s: Trying to free non-allocated page",
+					         __func__));
+					goto e0;
+				}
+			}
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Given non-zero free count but missing indices array",
+			         __func__));
+			return eError;
+		}
+	}
+
+	/* Validate the alloc indices */
+	for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++)
+	{
+		uiAllocpgidx = pai32AllocIndices[ui32Loop];
+
+		if (uiAllocpgidx >= (psPMRPageArrayData->uiTotalNumOSPages >> uiOrder))
+		{
+			eError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+			goto e0;
+		}
+
+		if ((NULL != psPageArray[uiAllocpgidx << uiOrder]) ||
+		    (TRANSLATION_INVALID != psPMRMapTable->aui32Translation[uiAllocpgidx]))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Trying to allocate already allocated page again",
+			         __func__));
+			goto e0;
+		}
+	}
+
+	ui32Loop = 0;
+
+	/* Allocate new pages from the OS */
+	if (0 != ui32AdtnlAllocPages)
+	{
+		eError = _AllocOSPages(psPMRPageArrayData, pai32AllocIndices, ui32AdtnlAllocPages);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "%s: New Addtl Allocation of pages failed",
+					 __func__));
+			goto e0;
+		}
+
+		psPMRMapTable->ui32NumPhysChunks += ui32AdtnlAllocPages;
+		/*Mark the corresponding pages of translation table as valid */
+		for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++)
+		{
+			psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop];
+		}
+	}
+
+
+	ui32Index = ui32Loop;
+
+	/* Move the corresponding free pages to alloc request */
+	for (ui32Loop = 0; ui32Loop < ui32CommonRequestCount; ui32Loop++, ui32Index++)
+	{
+		IMG_UINT32 uiOrderIdx;
+		uiAllocpgidx = pai32AllocIndices[ui32Index] << uiOrder;
+		uiFreepgidx  = pai32FreeIndices[ui32Loop] << uiOrder;
+
+		for (uiOrderIdx = 0; uiOrderIdx < (1 << uiOrder); uiOrderIdx++)
+		{
+			psPageArray[uiAllocpgidx + uiOrderIdx] = psPageArray[uiFreepgidx + uiOrderIdx];
+			if (bCMA)
+			{
+				psDMAVirtArray[uiAllocpgidx + uiOrderIdx] = psDMAVirtArray[uiFreepgidx + uiOrderIdx];
+				psDMAPhysArray[uiAllocpgidx + uiOrderIdx] = psDMAPhysArray[uiFreepgidx + uiOrderIdx];
+				psDMAVirtArray[uiFreepgidx + uiOrderIdx] = NULL;
+				psDMAPhysArray[uiFreepgidx + uiOrderIdx] = (dma_addr_t)0;
+			}
+			psPageArray[uiFreepgidx + uiOrderIdx] = NULL;
+		}
+
+		psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID;
+		psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
+	}
+
+	/* Free or zombie the additional free pages */
+	if (0 != ui32AdtnlFreePages)
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		PMR_OSPAGEARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+		eError = _ExtractPages(psPMRPageArrayData,
+		                       &pai32FreeIndices[ui32Loop],
+		                       ui32AdtnlFreePages,
+		                       &psExtractedPagesPageArray);
+		if (eError != PVRSRV_OK)
+		{
+			goto e0;
+		}
+
+		/* Zombify pages to get proper stats */
+		eError = PMRZombifyOSMem(psExtractedPagesPageArray, NULL);
+		PVR_LOG_IF_ERROR(eError, "psExtractedPagesPageArray");
+
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		eError = _FreeOSPages(psPMRPageArrayData,
+		                      &pai32FreeIndices[ui32Loop],
+		                      ui32AdtnlFreePages);
+		if (eError != PVRSRV_OK)
+		{
+			goto e0;
+		}
+#endif /* SUPPORT_PMR_PAGES_DEFERRED_FREE */
+		psPMRMapTable->ui32NumPhysChunks -= ui32AdtnlFreePages;
+		while (ui32Loop < ui32FreePageCount)
+		{
+			psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Loop]] = TRANSLATION_INVALID;
+			ui32Loop++;
+		}
+	}
+
+	eError = PVRSRV_OK;
+
+e0:
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMROSPFuncTab = {
+	.pfnLockPhysAddresses = &PMRLockSysPhysAddressesOSMem,
+	.pfnUnlockPhysAddresses = &PMRUnlockSysPhysAddressesOSMem,
+	.pfnDevPhysAddr = &PMRSysPhysAddrOSMem,
+	.pfnAcquireKernelMappingData = &PMRAcquireKernelMappingDataOSMem,
+	.pfnReleaseKernelMappingData = &PMRReleaseKernelMappingDataOSMem,
+	.pfnReadBytes = NULL,
+	.pfnWriteBytes = NULL,
+	.pfnChangeSparseMem = &PMRChangeSparseMemOSMem,
+	.pfnFinalize = &PMRFinalizeOSMem,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	.pfnFreeZombiePages = &PMRFreeZombiePagesOSMem,
+#endif
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = &PMRZombifyOSMem,
+#endif
+};
+
+/* Wrapper around OS page allocation. */
+static PVRSRV_ERROR
+DoPageAlloc(PMR_OSPAGEARRAY_DATA *psPrivData,
+            IMG_UINT32 *puiAllocIndices,
+            IMG_UINT32 ui32NumPhysChunks,
+            IMG_UINT32 ui32NumLogicalChunks,
+            IMG_UINT32 ui32Log2AllocPageSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Do we fill the whole page array or just parts (sparse)? */
+	if (ui32NumPhysChunks == ui32NumLogicalChunks)
+	{
+		/* Allocate the physical pages */
+		eError = _AllocOSPages(psPrivData,
+		                       NULL,
+		                       psPrivData->uiTotalNumOSPages >>
+		                       (ui32Log2AllocPageSize - PAGE_SHIFT));
+	}
+	else if (ui32NumPhysChunks != 0)
+	{
+		/* Allocate the physical pages */
+		eError = _AllocOSPages(psPrivData, puiAllocIndices,
+		                       ui32NumPhysChunks);
+	}
+
+	return eError;
+}
+
+static void _EncodeAllocationFlags(IMG_UINT32 uiLog2DevPageSize,
+	                               PVRSRV_MEMALLOCFLAGS_T uiFlags,
+	                               IMG_UINT32* ui32AllocFlags)
+{
+
+	if (PVRSRV_CHECK_OS_LINUX_PREFER_CMA(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_PREF_CMA);
+	}
+	/*
+	 * Use CMA framework if order is greater than OS page size; please note
+	 * that OSMMapPMRGeneric() has the same expectation as well.
+	 */
+	else if (uiLog2DevPageSize > PAGE_SHIFT)
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_DMA_CMA);
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	if (PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_IS_MOVABLE);
+	}
+#endif
+
+	/* OnDemand? */
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_ONDEMAND);
+	}
+
+	/* Zero? */
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_ZERO);
+	}
+
+	/* Poison on alloc? */
+	if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_ALLOC);
+	}
+
+#if defined(DEBUG)
+	/* Poison on free? */
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_POISON_ON_FREE);
+	}
+#endif
+
+	/* Indicate whether this is an allocation with default caching attribute (i.e cached) or not */
+	if (PVRSRV_CHECK_CPU_UNCACHED(uiFlags) ||
+		PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags))
+	{
+		BIT_SET(*ui32AllocFlags, FLAG_UNSET_MEMORY_TYPE);
+	}
+
+}
+
+void PhysmemGetOSRamMemStats(PHEAP_IMPL_DATA pvImplData,
+                             IMG_UINT64 *pui64TotalSize,
+                             IMG_UINT64 *pui64FreeSize)
+{
+	struct sysinfo sMeminfo;
+	si_meminfo(&sMeminfo);
+
+	PVR_UNREFERENCED_PARAMETER(pvImplData);
+
+	*pui64TotalSize = sMeminfo.totalram * sMeminfo.mem_unit;
+	*pui64FreeSize = sMeminfo.freeram * sMeminfo.mem_unit;
+
+}
+
+PVRSRV_ERROR
+PhysmemNewOSRamBackedPMR(PHYS_HEAP *psPhysHeap,
+						 CONNECTION_DATA *psConnection,
+						 IMG_DEVMEM_SIZE_T uiSize,
+						 IMG_UINT32 ui32NumPhysChunks,
+						 IMG_UINT32 ui32NumLogicalChunks,
+						 IMG_UINT32 *puiAllocIndices,
+						 IMG_UINT32 uiLog2DevPageSize,
+						 PVRSRV_MEMALLOCFLAGS_T uiFlags,
+						 const IMG_CHAR *pszAnnotation,
+						 IMG_PID uiPid,
+						 PMR **ppsPMRPtr,
+						 IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	PMR *psPMR;
+	struct _PMR_OSPAGEARRAY_DATA_ *psPrivData;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_UINT32 ui32CPUCacheFlags;
+	IMG_UINT32 ui32AllocFlags = 0;
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPhysHeap);
+	IMG_UINT32 ui32ValidPageSizeMask = psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask;
+	IMG_UINT32 ui32PageSize = IMG_PAGE2BYTES32(uiLog2DevPageSize);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(ui32PageSize & ui32ValidPageSizeMask, eError, errorOnParam);
+
+	/*
+	 * The host driver (but not guest) can still use this factory for firmware
+	 * allocations
+	 */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode) && PVRSRV_CHECK_FW_MAIN(uiFlags))
+	{
+		PVR_ASSERT(0);
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto errorOnParam;
+	}
+
+	/* Select correct caching mode */
+	eError = DevmemCPUCacheMode(uiFlags, &ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnParam;
+	}
+
+	if (PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags))
+	{
+		ui32CPUCacheFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN;
+	}
+
+	_EncodeAllocationFlags(uiLog2DevPageSize, uiFlags, &ui32AllocFlags);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* Reject sparse PMR marked for migrate */
+	if (BIT_ISSET(ui32AllocFlags, FLAG_IS_MOVABLE) &&
+	   (ui32NumPhysChunks != ui32NumLogicalChunks|| ui32NumLogicalChunks > 1))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("PhysmemNewOSRamBackedPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, errorOnParam);
+	}
+#endif
+
+
+#if defined(PVR_LINUX_PHYSMEM_ZERO_ALL_PAGES)
+	/* Overwrite flags and always zero pages that could go back to UM */
+	BIT_SET(ui32AllocFlags, FLAG_ZERO);
+	BIT_UNSET(ui32AllocFlags, FLAG_POISON_ON_ALLOC);
+#endif
+
+	/* Physical allocation alignment is generally not supported except under
+	   very restrictive conditions, also there is a maximum alignment value
+	   which must not exceed the largest device page-size. If these are not
+	   met then fail the aligned-requested allocation */
+	if (uiLog2DevPageSize > PAGE_SHIFT)
+	{
+		IMG_UINT64 ui64InvalidSizeMask = ui32PageSize - 1;
+		IMG_UINT32 ui32PageSizeMask = psDevNode->psMMUDevAttrs->ui32ValidPageSizeMask;
+		IMG_UINT32 ui32MaxPageSize = 1;
+
+		while (ui32PageSizeMask > 1)
+		{
+			ui32PageSizeMask >>= 1;
+			ui32MaxPageSize <<= 1;
+		}
+
+		if ((uiSize & ui64InvalidSizeMask) || (ui32PageSize > ui32MaxPageSize))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid PA alignment: size 0x%llx, align 0x%x",
+					__func__, uiSize, ui32PageSize));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+			goto errorOnParam;
+		}
+	}
+
+	/* Create Array structure that hold the physical pages */
+	eError = _AllocOSPageArray(psDevNode,
+							   uiSize,
+							   uiLog2DevPageSize,
+							   ui32AllocFlags,
+							   ui32CPUCacheFlags,
+							   uiPid,
+							   &psPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnAllocPageArray;
+	}
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+	if (psConnection)
+	{
+		psPrivData->psDRMFile = ((ENV_CONNECTION_DATA*)psConnection->hOsPrivateData)->psDRMFile;
+	}
+	else
+	{
+		psPrivData->psDRMFile = NULL;
+	}
+#endif
+#endif
+
+	if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		eError = DoPageAlloc(psPrivData, puiAllocIndices, ui32NumPhysChunks,
+		                     ui32NumLogicalChunks, uiLog2DevPageSize);
+		if (eError != PVRSRV_OK)
+		{
+			goto errorOnAllocPages;
+		}
+	}
+
+	/*
+	 * In this instance, we simply pass flags straight through.
+	 *
+	 * Generically, uiFlags can include things that control the PMR factory, but
+	 * we don't need any such thing (at the time of writing!), and our caller
+	 * specifies all PMR flags so we don't need to meddle with what was given to
+	 * us.
+	 */
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+
+	/*
+	 * Check no significant bits were lost in cast due to different bit widths
+	 * for flags
+	 */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		PDUMPCOMMENT(PhysHeapDeviceNode(psPhysHeap), "Deferred Allocation PMR (UMA)");
+	}
+
+	eError = PMRCreatePMR(psPhysHeap,
+						  uiSize,
+						  ui32NumPhysChunks,
+						  ui32NumLogicalChunks,
+						  puiAllocIndices,
+						  uiLog2DevPageSize,
+						  uiPMRFlags,
+						  pszAnnotation,
+						  &_sPMROSPFuncTab,
+						  psPrivData,
+						  PMR_TYPE_OSMEM,
+						  &psPMR,
+						  ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto errorOnCreate;
+	}
+
+	psPrivData->hPMR = psPMR;
+	*ppsPMRPtr = psPMR;
+
+	return PVRSRV_OK;
+
+errorOnCreate:
+	if (!BIT_ISSET(ui32AllocFlags, FLAG_ONDEMAND))
+	{
+		eError2 = _FreeOSPages(psPrivData, NULL, 0);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+errorOnAllocPages:
+	_FreeOSPagesArray(psPrivData);
+
+errorOnAllocPageArray:
+errorOnParam:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_osmem_linux.h b/drivers/gpu/drm/img-rogue/physmem_osmem_linux.h
new file mode 100644
index 000000000000..c9d7e5ce5083
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_osmem_linux.h
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux OS physmem implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_OSMEM_LINUX_H
+#define PHYSMEM_OSMEM_LINUX_H
+
+PVRSRV_ERROR LinuxInitPhysmem(void);
+void LinuxDeinitPhysmem(void);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+int LinuxRegisterMigrateCallbacks(struct file* filp);
+void LinuxDeregisterMigrateCallbacks(struct file* filp);
+#endif
+#endif
+
+#endif /* PHYSMEM_OSMEM_LINUX_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_ramem.c b/drivers/gpu/drm/img-rogue/physmem_ramem.c
new file mode 100644
index 000000000000..6597042770d9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_ramem.c
@@ -0,0 +1,2609 @@
+/*************************************************************************/ /*!
+@File           physmem_ramem.c
+@Title          Resource allocator managed PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                RA managed memory PMR factory API that is shared between local
+                physheap implementations (LMA & IMA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "physmem_ramem.h"
+#include "physheap.h"
+#include "allocmem.h"
+#include "ra.h"
+#include "connection_server.h"
+#include "device.h"
+#include "devicemem_server_utils.h"
+#include "osfunc.h"
+#include "pmr.h"
+#include "pmr_impl.h"
+#include "rgx_pdump_panics.h"
+#include "pdump_km.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "rgxutils.h"
+#endif
+
+#if defined(INTEGRITY_OS)
+#include "mm.h"
+#include "integrity_memobject.h"
+#endif
+
+#include "physmem_dlm.h"
+
+
+#if defined(PVRSRV_PHYSHEAP_DISABLE_OOM_DEMOTION)
+#define PHYSHEAP_DPF_LVL PVR_DBG_ERROR
+#else
+#define PHYSHEAP_DPF_LVL PVR_DBG_WARNING
+#endif
+
+/* Common Physheap Callback implementations */
+
+IMG_UINT32
+RAMemGetPageShift(void)
+{
+	return PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT;
+}
+
+PVRSRV_ERROR
+RAMemDoPhyContigPagesAlloc(RA_ARENA *pArena,
+                           size_t uiSize,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_PID uiPid)
+{
+	RA_BASE_T uiCardAddr = 0;
+	RA_LENGTH_T uiActualSize;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Log2NumPages;
+
+#if defined(DEBUG)
+	static IMG_UINT32 ui32MaxLog2NumPagesHistory = 0;
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* Firmware heaps on VZ are usually LMA, and several 64kb allocations need to
+	 * be made on Guest drivers: FwGuardPage, FwSysInit, FwConnectionCtl, FwOsInit.
+	 * Increase the maximum number to avoid driver warnings. */
+	IMG_UINT32	ui32MaxLog2NumPages = 7;	/* 128 pages => 512KB */
+#else
+	IMG_UINT32	ui32MaxLog2NumPages = 4;	/*  16 pages =>  64KB */
+#endif
+	ui32MaxLog2NumPages = MAX(ui32MaxLog2NumPages, OSGetOrder(1 << psDevNode->ui32Non4KPageSizeLog2));
+#else /* defined(DEBUG) */
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+#endif /* defined(DEBUG) */
+
+	PVR_ASSERT(uiSize != 0);
+	ui32Log2NumPages = OSGetOrder(uiSize);
+	uiSize = IMG_PAGES2BYTES64(OSGetPageSize(),ui32Log2NumPages);
+
+	eError = RA_Alloc(pArena,
+	                  uiSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                         /* No flags */
+	                  uiSize,
+	                  "RAMemPhyContigPagesAlloc",
+	                  &uiCardAddr,
+	                  &uiActualSize,
+	                  NULL);                     /* No private handle */
+
+	if (eError != PVRSRV_OK)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(pArena, &sRAStats);
+
+		PVR_DPF((PVR_DBG_ERROR,
+				"Failed to Allocate size = 0x"IMG_SIZE_FMTSPECX", align = 0x"
+				IMG_SIZE_FMTSPECX" Arena Free Space 0x%"IMG_UINT64_FMTSPECX,
+				uiSize, uiSize,	sRAStats.ui64FreeArenaSize));
+		return eError;
+	}
+
+	PVR_ASSERT(uiSize == uiActualSize);
+
+	psMemHandle->u.ui64Handle = uiCardAddr;
+	psDevPAddr->uiAddr = (IMG_UINT64) uiCardAddr;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+	                                    uiSize,
+	                                    uiCardAddr,
+	                                    uiPid);
+#else
+	{
+		IMG_CPU_PHYADDR sCpuPAddr;
+		sCpuPAddr.uiAddr = psDevPAddr->uiAddr;
+
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+									 NULL,
+									 sCpuPAddr,
+									 uiSize,
+									 uiPid
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#else /* PVRSRV_ENABLE_PROCESS_STATS */
+	PVR_UNREFERENCED_PARAMETER(uiPid);
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: (GPU Virtualisation) Allocated 0x" IMG_SIZE_FMTSPECX " at 0x%"
+	        IMG_UINT64_FMTSPECX ", Arena ID %u",
+	        __func__, uiSize, psDevPAddr->uiAddr, psMemHandle->uiOSid));
+#endif
+
+#if defined(DEBUG)
+	if (ui32Log2NumPages > ui32MaxLog2NumPages && ui32Log2NumPages > ui32MaxLog2NumPagesHistory)
+	{
+		PVR_ASSERT((ui32Log2NumPages <= ui32MaxLog2NumPages));
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ui32MaxLog2NumPages = %u, increasing to %u", __func__,
+		        ui32MaxLog2NumPages, ui32Log2NumPages ));
+		ui32MaxLog2NumPagesHistory = ui32Log2NumPages;
+	}
+#endif /* defined(DEBUG) */
+	psMemHandle->uiOrder = ui32Log2NumPages;
+
+	return eError;
+}
+
+void
+RAMemDoPhyContigPagesFree(RA_ARENA *pArena,
+                          PG_HANDLE *psMemHandle)
+{
+	RA_BASE_T uiCardAddr = (RA_BASE_T) psMemHandle->u.ui64Handle;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+	                                      (IMG_UINT64)uiCardAddr);
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,
+									(IMG_UINT64)uiCardAddr,
+									OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	RA_Free(pArena, uiCardAddr);
+	psMemHandle->uiOrder = 0;
+}
+
+PVRSRV_ERROR
+RAMemPhyContigPagesMap(PHYS_HEAP *psPhysHeap,
+                       PG_HANDLE *psMemHandle,
+                       size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                       void **pvPtr)
+{
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_UINT32 ui32NumPages = (1 << psMemHandle->uiOrder);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+
+	PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPAddr, psDevPAddr);
+	*pvPtr = OSMapPhysToLin(sCpuPAddr,
+							ui32NumPages * OSGetPageSize(),
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC);
+	PVR_RETURN_IF_NOMEM(*pvPtr);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+	                            ui32NumPages * OSGetPageSize(),
+	                            OSGetCurrentClientProcessIDKM());
+#else
+	{
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+									 *pvPtr,
+									 sCpuPAddr,
+									 ui32NumPages * OSGetPageSize(),
+									 OSGetCurrentClientProcessIDKM()
+									 DEBUG_MEMSTATS_VALUES);
+	}
+#endif
+#endif
+	return PVRSRV_OK;
+}
+
+void
+RAMemPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap,
+                         PG_HANDLE *psMemHandle,
+                         void *pvPtr)
+{
+	IMG_UINT32 ui32NumPages = (1 << psMemHandle->uiOrder);
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+		                            ui32NumPages * OSGetPageSize(),
+		                            OSGetCurrentClientProcessIDKM());
+#else
+	PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,
+	                                (IMG_UINT64)(uintptr_t)pvPtr,
+	                                OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	OSUnMapPhysToLin(pvPtr, ui32NumPages * OSGetPageSize());
+}
+
+PVRSRV_ERROR
+RAMemPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                         PG_HANDLE *psMemHandle,
+                         IMG_UINT32 uiOffset,
+                         IMG_UINT32 uiLength)
+{
+	/* No need to flush because we map as uncached */
+	PVR_UNREFERENCED_PARAMETER(psPhysHeap);
+	PVR_UNREFERENCED_PARAMETER(psMemHandle);
+	PVR_UNREFERENCED_PARAMETER(uiOffset);
+	PVR_UNREFERENCED_PARAMETER(uiLength);
+
+	return PVRSRV_OK;
+}
+
+/* Local memory allocation routines */
+
+/* Assert that the conversions between the RA base type and the device
+ * physical address are safe.
+ */
+static_assert(sizeof(IMG_DEV_PHYADDR) == sizeof(RA_BASE_T),
+              "Size IMG_DEV_PHYADDR != RA_BASE_T");
+
+/* Since 0x0 is a valid DevPAddr, we rely on max 64-bit value to be an invalid
+ * page address */
+#define INVALID_PAGE_ADDR ~((IMG_UINT64)0x0)
+#define ZERO_PAGE_VALUE 0
+
+typedef struct _PMR_KERNEL_MAP_HANDLE_ {
+	void *vma;
+	void *pvKernelAddress;
+	/* uiSize has 2 uses:
+	 * In Physically contiguous case it is used to track size of the mapping
+	 * for free.
+	 * In Physically sparse case it is used to determine free path to use, single page
+	 * sparse mapping or multi page
+	 */
+	size_t uiSize;
+} PMR_KERNEL_MAPPING;
+
+typedef struct _PMR_LMALLOCARRAY_DATA_ {
+
+#define FLAG_ZERO              (0U)
+#define FLAG_POISON_ON_FREE    (1U)
+#define FLAG_POISON_ON_ALLOC   (2U)
+#define FLAG_ONDEMAND          (3U)
+#define FLAG_SPARSE            (4U)
+#define FLAG_PHYS_CONTIG       (5U)
+#define FLAG_ZOMBIE            (6U)
+
+	IMG_PID uiPid;
+
+	/*
+	 * N.B Chunks referenced in this struct commonly are
+	 * to OS page sized. But in reality it is dependent on
+	 * the uiLog2ChunkSize.
+	 * Chunks will always be one 1 << uiLog2ChunkSize in size.
+	 * */
+
+	/*
+	 * The number of chunks currently allocated in the PMR.
+	 */
+	IMG_INT32 iNumChunksAllocated;
+
+	/*
+	 * Total number of (Virtual) chunks supported by this PMR.
+	 */
+	IMG_UINT32 uiTotalNumChunks;
+
+	/* The number of chunks to next be allocated for the PMR.
+	 * This will initially be the number allocated at first alloc
+	 * but may be changed in later calls to change sparse.
+	 * It represents the number of chunks to next be allocated.
+	 * This is used to store this value because we have the ability to
+	 * defer allocation.
+	 */
+	IMG_UINT32 uiChunksToAlloc;
+
+	/*
+	 * Log2 representation of the chunksize.
+	 */
+	IMG_UINT32 uiLog2ChunkSize;
+
+	/* Physical heap and arena pointers for this allocation */
+	PHYS_HEAP* psPhysHeap;
+	RA_ARENA* psArena;
+	PVRSRV_MEMALLOCFLAGS_T uiAllocFlags;
+
+	/*
+	   Connection data for this requests' originating process. NULL for
+	   direct-bridge originating calls
+	 */
+	CONNECTION_DATA *psConnection;
+
+	/*
+	 * Allocation flags related to the pages:
+	 * Zero              - Should we Zero memory on alloc
+	 * Poison on free    - Should we Poison the memory on free.
+	 * Poison on alloc   - Should we Poison the memory on alloc.
+	 * On demand         - Is the allocation on Demand i.e Do we defer allocation to time of use.
+	 * Sparse            - Is the PMR sparse.
+	 * Phys Contig       - Is the alloc Physically contiguous
+	 * Zombie            - Is zombie
+	 * */
+	IMG_UINT32 ui32Flags;
+
+	RA_BASE_ARRAY_T aBaseArray; /* Array of RA Bases */
+
+} PMR_LMALLOCARRAY_DATA;
+
+static PVRSRV_ERROR
+_FreeLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+             IMG_UINT32 *pui32FreeIndices,
+             IMG_UINT32 ui32FreeChunkCount);
+
+static PVRSRV_ERROR _MapPhysicalContigAlloc(PHYS_HEAP *psPhysHeap,
+                                            RA_BASE_ARRAY_T paBaseArray,
+                                            size_t uiSize,
+                                            PMR_FLAGS_T ulFlags,
+                                            PMR_KERNEL_MAPPING *psMapping)
+{
+	IMG_UINT32 ui32CPUCacheFlags;
+	PVRSRV_ERROR eError;
+	IMG_CPU_PHYADDR sCpuPAddr;
+	IMG_DEV_PHYADDR sDevPAddr;
+	sDevPAddr.uiAddr = RA_BASE_STRIP_GHOST_BIT(*paBaseArray);
+
+	eError = DevmemCPUCacheMode(ulFlags, &ui32CPUCacheFlags);
+	PVR_RETURN_IF_ERROR(eError);
+
+	PhysHeapDevPAddrToCpuPAddr(psPhysHeap,
+	                           1,
+	                           &sCpuPAddr,
+	                           &sDevPAddr);
+
+	psMapping->pvKernelAddress = OSMapPhysToLin(sCpuPAddr, uiSize, ui32CPUCacheFlags);
+	PVR_LOG_RETURN_IF_FALSE(psMapping->pvKernelAddress,
+	                        "OSMapPhyToLin: out of VM Mem",
+	                        PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING);
+	psMapping->vma = NULL;
+	psMapping->uiSize = uiSize;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _MapPhysicalSparseAlloc(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                                            RA_BASE_ARRAY_T paBaseArray,
+                                            size_t uiSize,
+                                            PMR_KERNEL_MAPPING *psMapping)
+{
+	IMG_UINT32 uiChunkCount = uiSize >> psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_CPU_PHYADDR uiPages[PMR_MAX_TRANSLATION_STACK_ALLOC], *puiPages;
+	PVRSRV_ERROR eError;
+	size_t uiPageShift = OSGetPageShift();
+	IMG_UINT32 uiOSPagesPerChunkShift = psLMAllocArrayData->uiLog2ChunkSize - uiPageShift;
+	IMG_UINT32 uiOSPageCount = uiChunkCount << uiOSPagesPerChunkShift;
+
+	if (uiOSPageCount > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPages = OSAllocZMem(sizeof(IMG_CPU_PHYADDR) * uiOSPageCount);
+		PVR_RETURN_IF_NOMEM(puiPages);
+	}
+	else
+	{
+		puiPages = &uiPages[0];
+	}
+
+	if (uiOSPagesPerChunkShift == 0)
+	{
+		IMG_UINT32 i;
+		PhysHeapDevPAddrToCpuPAddr(psLMAllocArrayData->psPhysHeap,
+								   uiChunkCount,
+								   puiPages,
+								   (IMG_DEV_PHYADDR *)paBaseArray);
+
+		/* If the ghost bit is present then the addrs returned will be off by 1
+		 * Strip the ghost bit to correct to real page aligned addresses.
+		 * */
+		for (i = 0; i < uiChunkCount; i++)
+		{
+			puiPages[i].uiAddr = RA_BASE_STRIP_GHOST_BIT(puiPages[i].uiAddr);
+		}
+	}
+	else
+	{
+		IMG_UINT32 i = 0, j = 0, index = 0;
+		for (i = 0; i < uiChunkCount; i++)
+		{
+			IMG_UINT32 ui32OSPagesPerDeviceChunk = (1 << uiOSPagesPerChunkShift);
+			IMG_DEV_PHYADDR uiDevAddr;
+			uiDevAddr.uiAddr = RA_BASE_STRIP_GHOST_BIT(paBaseArray[i]);
+			for (j = 0; j < ui32OSPagesPerDeviceChunk; j++)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psLMAllocArrayData->psPhysHeap,
+										   1,
+										   &puiPages[index],
+										   &uiDevAddr);
+				uiDevAddr.uiAddr += (1ULL << uiPageShift);
+				index++;
+			}
+		}
+	}
+
+	eError = OSMapPhysArrayToLin(puiPages,
+	                             uiOSPageCount,
+	                             &psMapping->pvKernelAddress,
+	                             &psMapping->vma);
+	if (eError == PVRSRV_OK)
+	{
+		psMapping->uiSize = uiSize;
+	}
+
+	if (puiPages != &uiPages[0])
+	{
+		OSFreeMem(puiPages);
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR _MapPMRKernel(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                                  RA_BASE_ARRAY_T paBaseArray,
+                                  size_t uiSize,
+                                  PMR_FLAGS_T ulFlags,
+                                  PMR_KERNEL_MAPPING *psMapping)
+{
+	PVRSRV_ERROR eError;
+	PHYS_HEAP *psPhysHeap = psLMAllocArrayData->psPhysHeap;
+	if (!BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* Physically Contig */
+		if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			eError = _MapPhysicalContigAlloc(psPhysHeap,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 ulFlags,
+			                                 psMapping);
+		}
+		/* Physically Sparse */
+		else
+		{
+			eError = _MapPhysicalSparseAlloc(psLMAllocArrayData,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 psMapping);
+		}
+	}
+	else
+	{
+		/* Sparse Alloc Single Chunk */
+		if (uiSize == IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize))
+		{
+			eError = _MapPhysicalContigAlloc(psPhysHeap,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 ulFlags,
+			                                 psMapping);
+		}
+		/* Sparse Alloc Multi Chunk */
+		else
+		{
+			eError = _MapPhysicalSparseAlloc(psLMAllocArrayData,
+			                                 paBaseArray,
+			                                 uiSize,
+			                                 psMapping);
+		}
+	}
+
+	return eError;
+}
+
+static void _UnMapPhysicalContigAlloc(PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	OSUnMapPhysToLin(psKernelMapping->pvKernelAddress, psKernelMapping->uiSize);
+}
+
+static void _UnMapPhysicalSparseAlloc(PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	OSUnMapPhysArrayToLin(psKernelMapping->pvKernelAddress,
+	                   psKernelMapping->vma);
+}
+
+static void _UnMapPMRKernel(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+                            PMR_KERNEL_MAPPING *psKernelMapping)
+{
+	if (!BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* Physically Contig */
+		if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			_UnMapPhysicalContigAlloc(psKernelMapping);
+		}
+		/* Physically Sparse */
+		else
+		{
+			_UnMapPhysicalSparseAlloc(psKernelMapping);
+		}
+	}
+	else
+	{
+		/* Sparse Alloc Single Chunk */
+		if (psKernelMapping->uiSize == IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize))
+		{
+			_UnMapPhysicalContigAlloc(psKernelMapping);
+		}
+		/* Sparse Alloc Multi Chunk */
+		else
+		{
+			_UnMapPhysicalSparseAlloc(psKernelMapping);
+		}
+	}
+}
+
+static PVRSRV_ERROR
+_PhysPgMemSet(PMR_LMALLOCARRAY_DATA *psLMAllocArrayData,
+              RA_BASE_ARRAY_T paBaseArray,
+              size_t uiSize,
+              IMG_BYTE ui8SetValue)
+{
+	PVRSRV_ERROR eError;
+	PMR_KERNEL_MAPPING sKernelMapping;
+
+	eError = _MapPMRKernel(psLMAllocArrayData,
+	                       paBaseArray,
+	                       uiSize,
+	                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+	                       &sKernelMapping);
+	PVR_GOTO_IF_ERROR(eError, map_failed);
+
+	OSCachedMemSetWMB(sKernelMapping.pvKernelAddress, ui8SetValue, uiSize);
+
+	_UnMapPMRKernel(psLMAllocArrayData, &sKernelMapping);
+
+	return PVRSRV_OK;
+
+map_failed:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to poison/zero allocation"));
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPageArray(PMR_SIZE_T uiSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 uiLog2AllocPageSize,
+                  IMG_UINT32 ui32Flags,
+                  PHYS_HEAP* psPhysHeap,
+                  RA_ARENA* pArena,
+                  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+                  IMG_PID uiPid,
+                  PMR_LMALLOCARRAY_DATA **ppsPageArrayDataPtr,
+                  CONNECTION_DATA *psConnection)
+{
+	PMR_LMALLOCARRAY_DATA *psPageArrayData = NULL;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiNumPages;
+
+	PVR_ASSERT(!BIT_ISSET(ui32Flags, FLAG_ZERO) || !BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC));
+	PVR_ASSERT(OSGetPageShift() <= uiLog2AllocPageSize);
+
+	/* Use of cast below is justified by the assertion that follows to
+	prove that no significant bits have been truncated */
+	uiNumPages = (IMG_UINT32)(((uiSize - 1) >> uiLog2AllocPageSize) + 1);
+	PVR_ASSERT(((PMR_SIZE_T)uiNumPages << uiLog2AllocPageSize) == uiSize);
+
+	psPageArrayData = OSAllocMem(sizeof(PMR_LMALLOCARRAY_DATA) + IMG_FLEX_ARRAY_SIZE(sizeof(RA_BASE_T), uiNumPages));
+	PVR_GOTO_IF_NOMEM(psPageArrayData, eError, errorOnAllocArray);
+
+	if (BIT_ISSET(ui32Flags, FLAG_SPARSE))
+	{
+		/* Since no pages are allocated yet, initialise page addresses to INVALID_PAGE_ADDR */
+		OSCachedMemSet(psPageArrayData->aBaseArray,
+					   0xFF,
+					   sizeof(RA_BASE_T) *
+					   uiNumPages);
+	}
+	else
+	{
+		/* Base pointers have been allocated for the full PMR in case we require a non
+		 * physically contiguous backing for the virtually contiguous allocation but the most
+		 * common case will be contiguous and so only require the first Base to be present
+		 */
+		psPageArrayData->aBaseArray[0] = INVALID_BASE_ADDR;
+	}
+
+	psPageArrayData->uiTotalNumChunks = uiNumPages;
+	psPageArrayData->uiChunksToAlloc = BIT_ISSET(ui32Flags, FLAG_SPARSE) ? ui32NumPhysChunks : uiNumPages;
+	psPageArrayData->uiLog2ChunkSize = uiLog2AllocPageSize;
+
+	psPageArrayData->psConnection = psConnection;
+	psPageArrayData->uiPid = uiPid;
+	psPageArrayData->iNumChunksAllocated = 0;
+	psPageArrayData->ui32Flags = ui32Flags;
+	psPageArrayData->psPhysHeap = psPhysHeap;
+	psPageArrayData->psArena = pArena;
+	psPageArrayData->uiAllocFlags = uiAllocFlags;
+
+	*ppsPageArrayDataPtr = psPageArrayData;
+
+	return PVRSRV_OK;
+
+/*
+  error exit path follows:
+*/
+
+errorOnAllocArray:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPagesContig(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiChunksToAlloc, uiLog2ChunkSize);
+	IMG_BOOL bPhysContig;
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+
+	eError = RA_AllocMulti(psPageArrayData->psArena,
+	                  ui64PhysSize,
+	                  uiLog2ChunkSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                       /* No flags */
+	                  "LMA_Page_Alloc",
+	                  psPageArrayData->aBaseArray,
+	                  psPageArrayData->uiTotalNumChunks,
+	                  &bPhysContig);
+
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"Contig: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long)ui64PhysSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+	if (bPhysContig)
+	{
+		BIT_SET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+			psPageArrayData->aBaseArray[0]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoison);
+	}
+
+	if (BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       ZERO_PAGE_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnZero);
+	}
+
+	psPageArrayData->iNumChunksAllocated += psPageArrayData->uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, ui64PhysSize, psPageArrayData->uiPid);
+#else
+	if (bPhysContig)
+	{
+		IMG_CPU_PHYADDR sLocalCpuPAddr;
+		sLocalCpuPAddr.uiAddr = (IMG_UINT64) psPageArrayData->aBaseArray[0];
+		PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+								 NULL,
+								 sLocalCpuPAddr,
+								 IMG_PAGES2BYTES64(psPageArrayData->uiTotalNumChunks, uiLog2ChunkSize),
+								 psPageArrayData->uiPid
+								 DEBUG_MEMSTATS_VALUES);
+	}
+	else
+	{
+		IMG_UINT32 i, j;
+		IMG_CPU_PHYADDR sLocalCpuPAddr;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks;)
+		{
+			IMG_UINT32 ui32AllocSizeInChunks = 1;
+
+			for (j = i;
+			     j + 1 != psPageArrayData->uiTotalNumChunks &&
+			     RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+			     j++)
+			{
+				ui32AllocSizeInChunks++;
+			}
+
+			sLocalCpuPAddr.uiAddr = (IMG_UINT64) psPageArrayData->aBaseArray[i];
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 IMG_PAGES2BYTES64(ui32AllocSizeInChunks, uiLog2ChunkSize),
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+
+			i += ui32AllocSizeInChunks;
+		}
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnZero:
+errorOnPoison:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMulti(psPageArrayData->psArena,
+	              psPageArrayData->aBaseArray,
+	              psPageArrayData->uiTotalNumChunks);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+ * Fully allocated variant of sparse allocation does not take in as argument an
+ * array of indices. It is used in cases where the amount of chunks to allocate is
+ * the same as the total the PMR can represent. I.E when we want to fully populate
+ * a sparse PMR.
+ */
+static PVRSRV_ERROR
+_AllocLMPagesSparseFull(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiChunksToAlloc, uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+
+	eError = RA_AllocMultiSparse(psPageArrayData->psArena,
+	                  uiLog2ChunkSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,                       /* No flags */
+	                  "LMA_Page_Alloc",
+	                  psPageArrayData->aBaseArray,
+	                  psPageArrayData->uiTotalNumChunks,
+	                  NULL, /* No indices given meaning allocate full base array using chunk count below */
+	                  psPageArrayData->uiChunksToAlloc);
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"SparseFull: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long)ui64PhysSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+		"(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+		psPageArrayData->aBaseArray[0]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_POISON_ON_ALLOC_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoison);
+	}
+
+	if (BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+		                       psPageArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       ZERO_PAGE_VALUE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnZero);
+	}
+
+	psPageArrayData->iNumChunksAllocated += psPageArrayData->uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, ui64PhysSize, psPageArrayData->uiPid);
+#else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			IMG_CPU_PHYADDR sLocalCpuPAddr;
+			sLocalCpuPAddr.uiAddr =
+				(IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[i]);
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 1 << uiLog2ChunkSize,
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+		}
+	}
+#endif
+#endif
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnZero:
+errorOnPoison:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMulti(psPageArrayData->psArena,
+	              psPageArrayData->aBaseArray,
+	              psPageArrayData->uiTotalNumChunks);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocLMPagesSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData, IMG_UINT32 *pui32MapTable)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT32 uiChunksToAlloc = psPageArrayData->uiChunksToAlloc;
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+
+	if (!pui32MapTable)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("pui32MapTable", eError, PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY, errorOnRAAlloc);
+	}
+
+#if defined(DEBUG)
+	/*
+	 * This block performs validation of the mapping table input in the following ways:
+	 * Check that each index in the mapping table does not exceed the number of the chunks
+	 * the whole PMR supports.
+	 * Check that each index given by the mapping table is not already allocated.
+	 * Check that there are no duplicated indices given in the mapping table.
+	 */
+	{
+		IMG_UINT32 i;
+		IMG_BOOL bIssueDetected = IMG_FALSE;
+		PVRSRV_ERROR eMapCheckError;
+
+		for (i = 0; i < uiChunksToAlloc; i++)
+		{
+			if (pui32MapTable[i] >= psPageArrayData->uiTotalNumChunks)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Page alloc request Index out of bounds for PMR @0x%p",
+						__func__,
+						psPageArrayData));
+				eMapCheckError = PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+
+			if (!RA_BASE_IS_INVALID(psPageArrayData->aBaseArray[pui32MapTable[i]]))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Mapping already exists Index %u Mapping index %u",
+						__func__,
+						i,
+						pui32MapTable[i]));
+				eMapCheckError = PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+
+			if (RA_BASE_IS_SPARSE_PREP(psPageArrayData->aBaseArray[pui32MapTable[i]]))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Mapping already exists in mapping table given Index %u Mapping index %u",
+						__func__,
+						i,
+						pui32MapTable[i]));
+				eMapCheckError = PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS;
+				bIssueDetected = IMG_TRUE;
+				break;
+			}
+			else
+			{
+				/* Set the To Prep value so we can detect duplicated map indices */
+				psPageArrayData->aBaseArray[pui32MapTable[i]] = RA_BASE_SPARSE_PREP_ALLOC_ADDR;
+			}
+		}
+		/* Unwind the Alloc Prep Values */
+		if (bIssueDetected)
+		{
+			/* We don't want to affect the index of the issue seen
+			 * as it could be a valid mapping. If it is a duplicated
+			 * mapping in the given table then we will clean-up the
+			 * previous instance anyway.
+			 */
+			IMG_UINT32 uiUnwind = i;
+
+			for (i = 0; i < uiUnwind; i++)
+			{
+				psPageArrayData->aBaseArray[pui32MapTable[i]] = INVALID_BASE_ADDR;
+			}
+
+			PVR_GOTO_WITH_ERROR(eError, eMapCheckError, errorOnRAAlloc);
+		}
+	}
+#endif
+
+	eError = RA_AllocMultiSparse(psPageArrayData->psArena,
+	                              psPageArrayData->uiLog2ChunkSize,
+	                              RA_NO_IMPORT_MULTIPLIER,
+	                              0,
+	                              "LMA_Page_Alloc",
+	                              psPageArrayData->aBaseArray,
+	                              psPageArrayData->uiTotalNumChunks,
+	                              pui32MapTable,
+	                              uiChunksToAlloc);
+	if (PVRSRV_OK != eError)
+	{
+		RA_USAGE_STATS sRAStats;
+		RA_Get_Usage_Stats(psPageArrayData->psArena, &sRAStats);
+
+		PVR_DPF((PHYSHEAP_DPF_LVL,
+				"Sparse: Failed to Allocate size = 0x%llx, align = 0x%llx"
+				" Arena Free Space 0x%"IMG_UINT64_FMTSPECX""
+				" Arena Name: '%s'",
+				(unsigned long long) uiChunksToAlloc << uiLog2ChunkSize,
+				1ULL << uiLog2ChunkSize,
+				sRAStats.ui64FreeArenaSize,
+				RA_GetArenaName(psPageArrayData->psArena)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES, errorOnRAAlloc);
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "(GPU Virtualization Validation): First RealBase: %"IMG_UINT64_FMTSPECX,
+	        psPageArrayData->aBaseArray[pui32MapTable[0]]));
+}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC) || BIT_ISSET(ui32Flags, FLAG_ZERO))
+	{
+		IMG_UINT32 i, ui32Index = 0;
+		for (i = 0; i < uiChunksToAlloc; i++)
+		{
+			ui32Index = pui32MapTable[i];
+
+			eError = _PhysPgMemSet(psPageArrayData,
+			                       &psPageArrayData->aBaseArray[ui32Index],
+			                       ui64ChunkSize,
+			                       BIT_ISSET(ui32Flags, FLAG_POISON_ON_ALLOC) ? PVRSRV_POISON_ON_ALLOC_VALUE :
+			                                                                    ZERO_PAGE_VALUE);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_PhysPgMemSet", errorOnPoisonZero);
+		}
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+	                            uiChunksToAlloc << uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiChunksToAlloc; i++)
+		{
+			IMG_UINT32 ui32Index = pui32MapTable[i];
+			IMG_CPU_PHYADDR sLocalCpuPAddr;
+			sLocalCpuPAddr.uiAddr =
+				(IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[ui32Index]);
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+									 NULL,
+									 sLocalCpuPAddr,
+									 ui64ChunkSize,
+									 psPageArrayData->uiPid
+									 DEBUG_MEMSTATS_VALUES);
+		}
+	}
+#endif
+#endif
+
+	psPageArrayData->iNumChunksAllocated += uiChunksToAlloc;
+
+	/* We have alloc'd the previous request, set 0 for book keeping */
+	psPageArrayData->uiChunksToAlloc = 0;
+
+	return PVRSRV_OK;
+
+	/*
+	  error exit paths follow:
+	*/
+errorOnPoisonZero:
+	eError = PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES;
+
+	RA_FreeMultiSparse(psPageArrayData->psArena,
+	                    psPageArrayData->aBaseArray,
+	                    psPageArrayData->uiTotalNumChunks,
+	                    psPageArrayData->uiLog2ChunkSize,
+	                    pui32MapTable,
+	                    &uiChunksToAlloc);
+
+errorOnRAAlloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR
+_AllocLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData, IMG_UINT32 *pui32MapTable)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(NULL != psPageArrayData);
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	if (psPageArrayData->uiTotalNumChunks <
+			(psPageArrayData->iNumChunksAllocated + psPageArrayData->uiChunksToAlloc))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Pages requested to allocate don't fit PMR alloc Size. "
+				"Allocated: %u + Requested: %u > Total Allowed: %u",
+				psPageArrayData->iNumChunksAllocated,
+				psPageArrayData->uiChunksToAlloc,
+				psPageArrayData->uiTotalNumChunks));
+		return PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+	}
+
+	/* If we have a non-backed sparse PMR then we can just return */
+	if (psPageArrayData->uiChunksToAlloc == 0)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+							"%s: Non-Backed Sparse PMR Created: %p.",
+							__func__,
+							psPageArrayData));
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	{
+		IMG_UINT32 ui32OSid=0;
+		PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPageArrayData->psPhysHeap);
+
+		/* Obtain the OSid specific data from our connection handle */
+		if (psPageArrayData->psConnection != NULL)
+		{
+			ui32OSid = psPageArrayData->psConnection->ui32OSid;
+		}
+
+		/* Replace the given RA Arena with OS Arena */
+		if (PVRSRV_CHECK_SHARED_BUFFER(psPageArrayData->uiAllocFlags))
+		{
+			psPageArrayData->psArena = psDevNode->psOSSharedArena;
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "(GPU Virtualization Validation): Giving from shared mem"));
+		}
+		else
+		{
+			psPageArrayData->psArena = psDevNode->psOSidSubArena[ui32OSid];
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "(GPU Virtualization Validation): Giving from OS slot %d",
+					 ui32OSid));
+		}
+	}
+#endif
+
+	/*
+	 * 3 cases:
+	 * Sparse allocation populating the whole PMR.
+	 * [**********]
+	 * Sparse allocation partially populating the PMR at given indices.
+	 * [*** *** **]
+	 * Contiguous allocation.
+	 * [**********]
+	 *
+	 * Note: Separate cases are required for 1 and 3 due to memstats tracking.
+	 * In Contiguous case we can track the block as a single memstat record as we know
+	 * we will also free in that size record.
+	 * Sparse allocations require a memstat record per chunk as they can be arbitrarily
+	 * free'd.
+	 */
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		if (psPageArrayData->uiTotalNumChunks == psPageArrayData->uiChunksToAlloc &&
+		    !pui32MapTable)
+		{
+			eError = _AllocLMPagesSparseFull(psPageArrayData);
+		}
+		else
+		{
+			eError = _AllocLMPagesSparse(psPageArrayData, pui32MapTable);
+		}
+	}
+	else
+	{
+		eError = _AllocLMPagesContig(psPageArrayData);
+	}
+
+	return eError;
+}
+
+static void
+_FreeLMPageArray(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"physmem_lma.c: freed local memory array structure for PMR @0x%p",
+			psPageArrayData));
+
+	OSFreeMem(psPageArrayData);
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesContig(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	RA_ARENA *pArena = psPageArrayData->psArena;
+	IMG_UINT64 ui64PhysSize = IMG_PAGES2BYTES64(psPageArrayData->uiTotalNumChunks, psPageArrayData->uiLog2ChunkSize);
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated != 0);
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated ==
+	           psPageArrayData->uiTotalNumChunks);
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_POISON_ON_FREE))
+	{
+		eError = _PhysPgMemSet(psPageArrayData,
+							   psPageArrayData->aBaseArray,
+							   ui64PhysSize,
+							   PVRSRV_POISON_ON_FREE_VALUE);
+		PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            ui64PhysSize,
+	                            psPageArrayData->uiPid);
+#else
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		PVRSRVStatsRemoveMemAllocRecord(uiStat,
+		                                (IMG_UINT64) psPageArrayData->aBaseArray[0],
+		                                psPageArrayData->uiPid);
+	}
+	else
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+			{
+				PVRSRVStatsRemoveMemAllocRecord(uiStat,
+				                                (IMG_UINT64) psPageArrayData->aBaseArray[i],
+				                                psPageArrayData->uiPid);
+			}
+		}
+	}
+#endif
+#endif
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		eError = RA_FreeMulti(pArena,
+							  psPageArrayData->aBaseArray,
+							  1);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+	}
+	else
+	{
+		eError = RA_FreeMulti(pArena,
+							  psPageArrayData->aBaseArray,
+							  psPageArrayData->iNumChunksAllocated);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+	}
+
+	psPageArrayData->iNumChunksAllocated = 0;
+
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: freed %"IMG_UINT64_FMTSPEC" local memory for PMR @0x%p",
+			__func__,
+			ui64PhysSize,
+			psPageArrayData));
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesRemainingSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(psPageArrayData->uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+	IMG_BOOL bPoisonOnFree = (BIT_ISSET(ui32Flags, FLAG_POISON_ON_FREE));
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            psPageArrayData->iNumChunksAllocated << psPageArrayData->uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#endif
+
+	for (i = 0; i < psPageArrayData->uiTotalNumChunks;)
+	{
+		if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+		{
+			IMG_UINT32 j;
+			IMG_UINT32 ui32AccumulatedChunks = 1;
+
+			for (j = i;
+				 j + 1 != psPageArrayData->uiTotalNumChunks &&
+				 RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+				 j++)
+			{
+				ui32AccumulatedChunks++;
+			}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+			for (j = i; j < (i + ui32AccumulatedChunks); j++)
+			{
+				PVRSRVStatsRemoveMemAllocRecord(uiStat,
+				                                RA_BASE_STRIP_GHOST_BIT(psPageArrayData->aBaseArray[j]),
+				                                psPageArrayData->uiPid);
+				if (bPoisonOnFree)
+#else
+			for (j = i; j < (i + ui32AccumulatedChunks) && bPoisonOnFree; j++)
+			{
+#endif
+				{
+					eError = _PhysPgMemSet(psPageArrayData,
+										   &psPageArrayData->aBaseArray[j],
+										   ui64ChunkSize,
+										   PVRSRV_POISON_ON_FREE_VALUE);
+					PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+				}
+			}
+
+			eError = RA_FreeMulti(psPageArrayData->psArena,
+			                       &psPageArrayData->aBaseArray[i],
+			                       ui32AccumulatedChunks);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMulti");
+
+			psPageArrayData->iNumChunksAllocated -= ui32AccumulatedChunks;
+			i += ui32AccumulatedChunks;
+		}
+		else if (RA_BASE_IS_INVALID(psPageArrayData->aBaseArray[i]))
+		{
+			i++;
+		}
+	}
+
+	/* We have freed all allocations in the previous loop */
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_FreeLMPagesSparse(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+                   IMG_UINT32 *pui32FreeIndices,
+                   IMG_UINT32 ui32FreeChunkCount)
+{
+	RA_ARENA *pArena = psPageArrayData->psArena;
+	IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT32 ui32Flags = psPageArrayData->ui32Flags;
+	IMG_UINT32 uiActualFreeCount = ui32FreeChunkCount;
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_UINT32 uiStat = PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_ZOMBIE))
+	{
+		uiStat = PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES;
+	}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_ASSERT(psPageArrayData->iNumChunksAllocated != 0);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < ui32FreeChunkCount; i++)
+		{
+			IMG_UINT32 ui32Index = pui32FreeIndices[i];
+
+			PVRSRVStatsRemoveMemAllocRecord(uiStat,
+			                                (IMG_UINT64) RA_BASE_STRIP_GHOST_BIT(
+			                                psPageArrayData->aBaseArray[ui32Index]),
+			                                psPageArrayData->uiPid);
+		}
+	}
+#endif
+
+	if (BIT_ISSET(ui32Flags, FLAG_POISON_ON_FREE))
+	{
+		IMG_UINT32 i, ui32Index = 0;
+		for (i = 0; i < ui32FreeChunkCount; i++)
+		{
+			ui32Index = pui32FreeIndices[i];
+
+			eError = _PhysPgMemSet(psPageArrayData,
+								   &psPageArrayData->aBaseArray[ui32Index],
+								   ui64ChunkSize,
+								   PVRSRV_POISON_ON_FREE_VALUE);
+			PVR_LOG_IF_ERROR(eError, "_PhysPgMemSet");
+		}
+	}
+
+	eError = RA_FreeMultiSparse(pArena,
+	                             psPageArrayData->aBaseArray,
+	                             psPageArrayData->uiTotalNumChunks,
+	                             uiLog2ChunkSize,
+	                             pui32FreeIndices,
+	                             &uiActualFreeCount);
+	psPageArrayData->iNumChunksAllocated -= uiActualFreeCount;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsDecrMemAllocStat(uiStat,
+	                            uiActualFreeCount << psPageArrayData->uiLog2ChunkSize,
+	                            psPageArrayData->uiPid);
+#endif
+	if (eError == PVRSRV_ERROR_RA_FREE_INVALID_CHUNK)
+	{
+		/* Log the RA error but convert it to PMR level to match the interface,
+		 * this is important because other PMR factories may not use the RA but
+		 * still return error, returning a PMR based error
+		 * keeps the interface agnostic to implementation behaviour.
+		 */
+		PVR_LOG_IF_ERROR(eError, "RA_FreeMultiSparse");
+		return PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "RA_FreeMultiSparse");
+
+	PVR_ASSERT(0 <= psPageArrayData->iNumChunksAllocated);
+
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"%s: freed %" IMG_UINT64_FMTSPEC " local memory for PMR @0x%p",
+			__func__,
+			(uiActualFreeCount * ui64ChunkSize),
+			psPageArrayData));
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR PMRFreeZombiePagesRAMem(PMR_IMPL_ZOMBIEPAGES pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psZombiePageArray = pvPriv;
+
+	eError = _FreeLMPages(psZombiePageArray, NULL, 0);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	_FreeLMPageArray(psZombiePageArray);
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+
+/* Allocates a new PMR_LMALLOCARRAY_DATA object and fills it with
+ * pages to be extracted from psSrcPageArrayData.
+ */
+static PVRSRV_ERROR
+_ExtractPages(PMR_LMALLOCARRAY_DATA *psSrcPageArrayData,
+			  IMG_UINT32 *pai32ExtractIndices,
+			  IMG_UINT32 ui32ExtractPageCount,
+			  PMR_LMALLOCARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32ExtractPageCountSaved;
+	PMR_LMALLOCARRAY_DATA* psDstPageArrayData;
+
+	/* Alloc PMR_LMALLOCARRAY_DATA for the extracted pages */
+	eError = _AllocLMPageArray(ui32ExtractPageCount << psSrcPageArrayData->uiLog2ChunkSize,
+	                           ui32ExtractPageCount,
+	                           psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->ui32Flags,
+	                           psSrcPageArrayData->psPhysHeap,
+	                           psSrcPageArrayData->psArena,
+	                           psSrcPageArrayData->uiAllocFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData,
+	                           psSrcPageArrayData->psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_AllocLMPageArray", alloc_error);
+
+	ui32ExtractPageCountSaved = ui32ExtractPageCount;
+	/* Transfer pages from source base array to newly allocated page array */
+	eError = RA_TransferMultiSparseIndices(psSrcPageArrayData->psArena,
+	                                       psSrcPageArrayData->aBaseArray,
+	                                       psSrcPageArrayData->uiTotalNumChunks,
+	                                       psDstPageArrayData->aBaseArray,
+	                                       psDstPageArrayData->uiTotalNumChunks,
+	                                       psSrcPageArrayData->uiLog2ChunkSize,
+	                                       pai32ExtractIndices,
+	                                       &ui32ExtractPageCountSaved);
+	PVR_LOG_GOTO_IF_FALSE((eError == PVRSRV_OK) && (ui32ExtractPageCountSaved == ui32ExtractPageCount),
+	                      "RA_TransferMultiSparseIndices failed",
+	                      transfer_error);
+
+
+	/* Update page counts */
+	psSrcPageArrayData->iNumChunksAllocated -= ui32ExtractPageCount;
+	psDstPageArrayData->iNumChunksAllocated += ui32ExtractPageCount;
+
+	*psOutPageArrayData = psDstPageArrayData;
+
+	return PVRSRV_OK;
+transfer_error:
+	_FreeLMPageArray(psDstPageArrayData);
+alloc_error:
+	return eError;
+}
+
+/* Extracts all allocated pages referenced psSrcPageArrayData
+ * Allocates a new PMR_OSPAGEARRAY_DATA object and fills it with the extracted
+ * pages information.
+ */
+static PVRSRV_ERROR
+_ExtractAllPages(PMR_LMALLOCARRAY_DATA *psSrcPageArrayData,
+				 PMR_LMALLOCARRAY_DATA **psOutPageArrayData)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA* psDstPageArrayData;
+	IMG_UINT32 ui32IdxSrc, ui32IdxDst;
+
+	if (psSrcPageArrayData->iNumChunksAllocated == 0)
+	{
+		/* Do nothing if psSrcPageArrayData contains no allocated pages */
+		return PVRSRV_OK;
+	}
+
+	/* Alloc PMR_LMALLOCARRAY_DATA for the extracted pages */
+	eError = _AllocLMPageArray(psSrcPageArrayData->iNumChunksAllocated << psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->iNumChunksAllocated,
+	                           psSrcPageArrayData->uiLog2ChunkSize,
+	                           psSrcPageArrayData->ui32Flags,
+	                           psSrcPageArrayData->psPhysHeap,
+	                           psSrcPageArrayData->psArena,
+	                           psSrcPageArrayData->uiAllocFlags,
+	                           psSrcPageArrayData->uiPid,
+	                           &psDstPageArrayData,
+	                           psSrcPageArrayData->psConnection);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "_AllocLMPageArray failed in %s", __func__);
+
+	/* Now do the transfer */
+	ui32IdxDst=0;
+	for (ui32IdxSrc=0; ((ui32IdxDst<psSrcPageArrayData->iNumChunksAllocated) &&
+	                    (psDstPageArrayData->iNumChunksAllocated<psSrcPageArrayData->iNumChunksAllocated)); ui32IdxSrc++)
+	{
+		if (psSrcPageArrayData->aBaseArray[ui32IdxSrc] != INVALID_BASE_ADDR)
+		{
+			psDstPageArrayData->aBaseArray[ui32IdxDst++] = psSrcPageArrayData->aBaseArray[ui32IdxSrc];
+			psSrcPageArrayData->aBaseArray[ui32IdxSrc] = INVALID_BASE_ADDR;
+			psDstPageArrayData->iNumChunksAllocated++;
+		}
+	}
+
+	/* Update src page count */
+	psSrcPageArrayData->iNumChunksAllocated = 0;
+
+	*psOutPageArrayData = psDstPageArrayData;
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_FreeLMPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+             IMG_UINT32 *pui32FreeIndices,
+             IMG_UINT32 ui32FreeChunkCount)
+{
+	PVRSRV_ERROR eError;
+
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		if (!pui32FreeIndices)
+		{
+			eError =  _FreeLMPagesRemainingSparse(psPageArrayData);
+		}
+		else
+		{
+			eError = _FreeLMPagesSparse(psPageArrayData, pui32FreeIndices, ui32FreeChunkCount);
+		}
+	}
+	else
+	{
+		eError = _FreeLMPagesContig(psPageArrayData);
+	}
+
+	return eError;
+}
+
+/*
+ *
+ * Implementation of PMR callback functions
+ *
+ */
+
+/* destructor func is called after last reference disappears, but
+   before PMR itself is freed. */
+static void
+PMRFinalizeLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+
+	/* We can't free pages until now. */
+	if (psLMAllocArrayData->iNumChunksAllocated != 0)
+	{
+		eError = _FreeLMPages(psLMAllocArrayData, NULL, 0);
+		PVR_LOG_IF_ERROR(eError, "_FreeLMPages");
+		PVR_ASSERT (eError == PVRSRV_OK);
+	}
+
+	_FreeLMPageArray(psLMAllocArrayData);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static inline void _TransferToMemZombieRecord_LmaPages(PMR_LMALLOCARRAY_DATA *psPageArrayData,
+                                                       RA_BASE_T uiBase)
+{
+	IMG_CPU_PHYADDR sCPUPhysAddr = {
+		.uiAddr = RA_BASE_STRIP_GHOST_BIT(uiBase)
+	};
+
+	PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,
+	                                  PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES,
+	                                  sCPUPhysAddr.uiAddr,
+	                                  psPageArrayData->uiPid
+	                                  DEBUG_MEMSTATS_VALUES);
+}
+#endif
+
+static PVRSRV_ERROR PMRZombifyLocalMem(PMR_IMPL_PRIVDATA pvPriv, PMR *psPMR)
+{
+	PMR_LMALLOCARRAY_DATA *psPageArrayData = pvPriv;
+
+	BIT_SET(psPageArrayData->ui32Flags, FLAG_ZOMBIE);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_PID uiPid = psPageArrayData->uiPid;
+		IMG_UINT32 uiLog2ChunkSize = psPageArrayData->uiLog2ChunkSize;
+		IMG_UINT64 uiSize = BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE) ?
+			(IMG_UINT64) psPageArrayData->iNumChunksAllocated << uiLog2ChunkSize :
+			(IMG_UINT64) psPageArrayData->uiTotalNumChunks << uiLog2ChunkSize;
+
+		PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES, uiSize, uiPid);
+		PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES, uiSize, uiPid);
+	}
+#else /* !defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		/* _FreeLMPagesRemainingSparse path */
+
+		IMG_UINT32 i;
+
+		for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+		{
+			if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+			{
+				IMG_UINT32 j;
+				IMG_UINT32 ui32AccumulatedChunks = 1;
+
+				for (j = i;
+				     j + 1 != psPageArrayData->uiTotalNumChunks &&
+				         RA_BASE_IS_GHOST(psPageArrayData->aBaseArray[j + 1]);
+				     j++)
+				{
+					ui32AccumulatedChunks++;
+				}
+
+				for (j = i; j < (i + ui32AccumulatedChunks); j++)
+				{
+					_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+					                                    psPageArrayData->aBaseArray[j]);
+				}
+			}
+		}
+	}
+	else
+	{
+		/* _FreeLMPagesContig path */
+
+		if (BIT_ISSET(psPageArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+		{
+			_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+			                                    psPageArrayData->aBaseArray[0]);
+		}
+		else
+		{
+			IMG_UINT32 i;
+
+			for (i = 0; i < psPageArrayData->uiTotalNumChunks; i++)
+			{
+				if (RA_BASE_IS_REAL(psPageArrayData->aBaseArray[i]))
+				{
+					_TransferToMemZombieRecord_LmaPages(psPageArrayData,
+					                                    psPageArrayData->aBaseArray[i]);
+				}
+			}
+
+		}
+	}
+#endif /* !defined(PVRSRV_ENABLE_MEMORY_STATS) */
+#endif /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* callback function for locking the system physical page addresses.
+   As we are LMA there is nothing to do as we control physical memory. */
+static PVRSRV_ERROR
+PMRLockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+{
+
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData;
+
+	psLMAllocArrayData = pvPriv;
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_ONDEMAND))
+	{
+		/* Allocate Memory for deferred allocation */
+		eError = _AllocLMPages(psLMAllocArrayData, NULL);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+                                  PMR_IMPL_ZOMBIEPAGES *ppvZombiePages)
+#else
+static PVRSRV_ERROR
+PMRUnlockSysPhysAddressesLocalMem(PMR_IMPL_PRIVDATA pvPriv)
+#endif
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_LMALLOCARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+	*ppvZombiePages = NULL;
+#endif
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_ONDEMAND))
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (psLMAllocArrayData->iNumChunksAllocated == 0)
+		{
+			*ppvZombiePages = NULL;
+			return PVRSRV_OK;
+		}
+
+		eError = _ExtractAllPages(psLMAllocArrayData,
+		                          &psExtractedPagesPageArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractAllPages", e0);
+
+		if (psExtractedPagesPageArray)
+		{
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyLocalMem(psExtractedPagesPageArray, NULL);
+			PVR_WARN_IF_ERROR(eError, "PMRZombifyLocalMem");
+		}
+		*ppvZombiePages = psExtractedPagesPageArray;
+#else
+		/* Free Memory for deferred allocation */
+		eError = _FreeLMPages(psLMAllocArrayData, NULL, 0);
+		PVR_RETURN_IF_ERROR(eError);
+#endif
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+e0:
+#endif
+	PVR_ASSERT(eError == PVRSRV_OK);
+	return eError;
+}
+
+/* N.B. It is assumed that PMRLockSysPhysAddressesLocalMem() is called _before_ this function! */
+static PVRSRV_ERROR
+PMRSysPhysAddrLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+					   IMG_UINT32 ui32Log2PageSize,
+					   IMG_UINT32 ui32NumOfPages,
+					   IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+					   IMG_UINT64 ui64IPAPolicyValue,
+					   IMG_UINT64 ui64IPAClearMask,
+#endif
+					   IMG_BOOL *pbValid,
+					   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	IMG_UINT32 idx;
+	IMG_UINT32 uiLog2AllocSize;
+	IMG_UINT64 uiAllocIndex;
+	IMG_DEVMEM_OFFSET_T uiInAllocOffset;
+	IMG_UINT32 uiNumAllocs = psLMAllocArrayData->uiTotalNumChunks;
+
+#if defined(SUPPORT_STATIC_IPA)
+	PVR_UNREFERENCED_PARAMETER(ui64IPAPolicyValue);
+	PVR_UNREFERENCED_PARAMETER(ui64IPAClearMask);
+#endif
+
+	if (psLMAllocArrayData->uiLog2ChunkSize < ui32Log2PageSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Requested physical addresses from PMR "
+		         "for incompatible contiguity %u!",
+		         __func__,
+		         ui32Log2PageSize));
+		return PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY;
+	}
+
+	PVR_ASSERT(psLMAllocArrayData->uiLog2ChunkSize != 0);
+	PVR_ASSERT(ui32Log2PageSize >= RA_BASE_FLAGS_LOG2);
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_PHYS_CONTIG))
+	{
+		for (idx=0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				psDevPAddr[idx].uiAddr = psLMAllocArrayData->aBaseArray[0] + puiOffset[idx];
+#if defined(SUPPORT_STATIC_IPA)
+				/* Modify the physical address with the associated IPA values */
+				psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+				psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+			}
+		}
+	}
+	else
+	{
+		uiLog2AllocSize = psLMAllocArrayData->uiLog2ChunkSize;
+
+		for (idx=0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				uiAllocIndex = puiOffset[idx] >> uiLog2AllocSize;
+				uiInAllocOffset = puiOffset[idx] - (uiAllocIndex << uiLog2AllocSize);
+
+				PVR_LOG_RETURN_IF_FALSE(uiAllocIndex < uiNumAllocs,
+										"puiOffset out of range", PVRSRV_ERROR_OUT_OF_RANGE);
+
+				PVR_ASSERT(uiInAllocOffset < (1ULL << uiLog2AllocSize));
+
+				/* The base may or may not be a ghost base, but we don't care,
+				 * we just need the real representation of the base.
+				 */
+				psDevPAddr[idx].uiAddr = RA_BASE_STRIP_GHOST_BIT(
+					psLMAllocArrayData->aBaseArray[uiAllocIndex]) + uiInAllocOffset;
+#if defined(SUPPORT_STATIC_IPA)
+				/* Modify the physical address with the associated IPA values */
+				psDevPAddr[idx].uiAddr &= ~ui64IPAClearMask;
+				psDevPAddr[idx].uiAddr |= ui64IPAPolicyValue;
+#endif
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+PMRAcquireKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+								 size_t uiOffset,
+								 size_t uiSize,
+								 void **ppvKernelAddressOut,
+								 IMG_HANDLE *phHandleOut,
+								 PMR_FLAGS_T ulFlags)
+{
+	PVRSRV_ERROR eError;
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	PMR_KERNEL_MAPPING *psKernelMapping;
+	RA_BASE_T *paBaseArray;
+	IMG_UINT32 ui32ChunkIndex = 0;
+	size_t uiOffsetMask = uiOffset;
+
+	IMG_UINT32 uiLog2ChunkSize = psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT64 ui64PhysSize;
+
+	PVR_ASSERT(psLMAllocArrayData);
+	PVR_ASSERT(ppvKernelAddressOut);
+	PVR_ASSERT(phHandleOut);
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		IMG_UINT32 i;
+		/* Locate the desired physical chunk to map in */
+		ui32ChunkIndex = uiOffset >> psLMAllocArrayData->uiLog2ChunkSize;
+
+		if (OSIsMapPhysNonContigSupported())
+		{
+			/* If a size hasn't been supplied assume we are mapping a single page */
+			IMG_UINT32 uiNumChunksToMap;
+
+			/* This is to support OSMapPMR originated parameters */
+			if (uiOffset == 0 && uiSize == 0)
+			{
+				uiNumChunksToMap = psLMAllocArrayData->iNumChunksAllocated;
+			}
+			else
+			{
+				uiNumChunksToMap = uiSize >> psLMAllocArrayData->uiLog2ChunkSize;
+			}
+
+			/* Check we are attempting to map at least a chunk in size */
+			if (uiNumChunksToMap < 1)
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_INVALID_PARAMS, "uiNumChunksToMap < 1");
+			}
+
+			/* Check contiguous region doesn't exceed size of PMR */
+			if (ui32ChunkIndex + (uiNumChunksToMap - 1) > psLMAllocArrayData->uiTotalNumChunks)
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_INVALID_PARAMS,
+				                        "Mapping range exceeds total num chunks in PMR");
+			}
+
+			/* Check the virtually contiguous region given is physically backed */
+			for (i = ui32ChunkIndex; i < ui32ChunkIndex + uiNumChunksToMap; i++)
+			{
+				if (RA_BASE_IS_INVALID(psLMAllocArrayData->aBaseArray[i]))
+				{
+					PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY, "Sparse contiguity check");
+				}
+			}
+			/* Size of virtually contiguous sparse alloc */
+			ui64PhysSize = IMG_PAGES2BYTES64(uiNumChunksToMap, psLMAllocArrayData->uiLog2ChunkSize);
+		}
+		else
+		{
+			size_t uiStart = uiOffset;
+			size_t uiEnd = uiOffset + uiSize - 1;
+			size_t uiChunkMask = ~(IMG_PAGE2BYTES64(psLMAllocArrayData->uiLog2ChunkSize) - 1);
+
+			/* We can still map if only one chunk is required */
+			if ((uiStart & uiChunkMask) != (uiEnd & uiChunkMask))
+			{
+				PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY, "Sparse contiguity check");
+			}
+			/* Map a single chunk */
+			ui64PhysSize = ui64ChunkSize;
+		}
+
+		paBaseArray = &psLMAllocArrayData->aBaseArray[ui32ChunkIndex];
+
+		/* Offset mask to be used for address offsets within a chunk */
+		uiOffsetMask = (1U << psLMAllocArrayData->uiLog2ChunkSize) - 1;
+	}
+	else
+	{
+		paBaseArray = psLMAllocArrayData->aBaseArray;
+		ui64PhysSize = IMG_PAGES2BYTES64(psLMAllocArrayData->uiTotalNumChunks, uiLog2ChunkSize);
+	}
+
+	PVR_ASSERT(ui32ChunkIndex < psLMAllocArrayData->uiTotalNumChunks);
+
+	psKernelMapping = OSAllocMem(sizeof(*psKernelMapping));
+	PVR_RETURN_IF_NOMEM(psKernelMapping);
+
+	eError = _MapPMRKernel(psLMAllocArrayData,
+	                       paBaseArray,
+	                       ui64PhysSize,
+	                       ulFlags,
+	                       psKernelMapping);
+	if (eError == PVRSRV_OK)
+	{
+		/* uiOffset & uiOffsetMask is used to get the kernel addr within the page */
+		*ppvKernelAddressOut = ((IMG_CHAR *) psKernelMapping->pvKernelAddress) + (uiOffset & uiOffsetMask);
+		*phHandleOut = psKernelMapping;
+	}
+	else
+	{
+		OSFreeMem(psKernelMapping);
+		PVR_LOG_ERROR(eError, "_MapPMRKernel");
+	}
+
+	return eError;
+}
+
+static void PMRReleaseKernelMappingDataLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+                                                IMG_HANDLE hHandle)
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = (PMR_LMALLOCARRAY_DATA *) pvPriv;
+	PMR_KERNEL_MAPPING *psKernelMapping = (PMR_KERNEL_MAPPING *) hHandle;
+
+	PVR_ASSERT(psLMAllocArrayData);
+	PVR_ASSERT(psKernelMapping);
+
+	_UnMapPMRKernel(psLMAllocArrayData,
+	                psKernelMapping);
+
+	OSFreeMem(psKernelMapping);
+}
+
+static PVRSRV_ERROR
+CopyBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+				  IMG_DEVMEM_OFFSET_T uiOffset,
+				  IMG_UINT8 *pcBuffer,
+				  size_t uiBufSz,
+				  size_t *puiNumBytes,
+				  void (*pfnCopyBytes)(IMG_UINT8 *pcBuffer,
+									   IMG_UINT8 *pcPMR,
+									   size_t uiSize))
+{
+	PMR_LMALLOCARRAY_DATA *psLMAllocArrayData = pvPriv;
+	size_t uiBytesCopied;
+	size_t uiBytesToCopy;
+	size_t uiBytesCopyableFromAlloc;
+	PMR_KERNEL_MAPPING sMapping;
+	IMG_UINT8 *pcKernelPointer = NULL;
+	size_t uiBufferOffset;
+	IMG_UINT64 uiAllocIndex;
+	IMG_DEVMEM_OFFSET_T uiInAllocOffset;
+	IMG_UINT32 uiLog2ChunkSize = psLMAllocArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+	IMG_UINT64 ui64PhysSize;
+	PVRSRV_ERROR eError;
+
+	uiBytesCopied = 0;
+	uiBytesToCopy = uiBufSz;
+	uiBufferOffset = 0;
+
+	if (BIT_ISSET(psLMAllocArrayData->ui32Flags, FLAG_SPARSE))
+	{
+		while (uiBytesToCopy > 0)
+		{
+			/* we have to map one alloc in at a time */
+			PVR_ASSERT(psLMAllocArrayData->uiLog2ChunkSize != 0);
+			uiAllocIndex = uiOffset >> psLMAllocArrayData->uiLog2ChunkSize;
+			uiInAllocOffset = uiOffset - (uiAllocIndex << psLMAllocArrayData->uiLog2ChunkSize);
+			uiBytesCopyableFromAlloc = uiBytesToCopy;
+			if (uiBytesCopyableFromAlloc + uiInAllocOffset > (1ULL << psLMAllocArrayData->uiLog2ChunkSize))
+			{
+				uiBytesCopyableFromAlloc = TRUNCATE_64BITS_TO_SIZE_T((1ULL << psLMAllocArrayData->uiLog2ChunkSize)-uiInAllocOffset);
+			}
+			/* Mapping a single chunk at a time */
+			ui64PhysSize = ui64ChunkSize;
+
+			PVR_ASSERT(uiBytesCopyableFromAlloc != 0);
+			PVR_ASSERT(uiAllocIndex < psLMAllocArrayData->uiTotalNumChunks);
+			PVR_ASSERT(uiInAllocOffset < (1ULL << uiLog2ChunkSize));
+
+			eError = _MapPMRKernel(psLMAllocArrayData,
+			                       &psLMAllocArrayData->aBaseArray[uiAllocIndex],
+			                       ui64PhysSize,
+			                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+			                       &sMapping);
+			PVR_GOTO_IF_ERROR(eError, e0);
+			pcKernelPointer = sMapping.pvKernelAddress;
+			pfnCopyBytes(&pcBuffer[uiBufferOffset], &pcKernelPointer[uiInAllocOffset], uiBytesCopyableFromAlloc);
+
+			_UnMapPMRKernel(psLMAllocArrayData,
+			                &sMapping);
+
+			uiBufferOffset += uiBytesCopyableFromAlloc;
+			uiBytesToCopy -= uiBytesCopyableFromAlloc;
+			uiOffset += uiBytesCopyableFromAlloc;
+			uiBytesCopied += uiBytesCopyableFromAlloc;
+		}
+	}
+	else
+	{
+		ui64PhysSize = IMG_PAGES2BYTES64(psLMAllocArrayData->uiTotalNumChunks, uiLog2ChunkSize);
+		PVR_ASSERT((uiOffset + uiBufSz) <= ui64PhysSize);
+		PVR_ASSERT(ui64ChunkSize != 0);
+		eError = _MapPMRKernel(psLMAllocArrayData,
+		                       psLMAllocArrayData->aBaseArray,
+		                       ui64PhysSize,
+		                       PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC,
+		                       &sMapping);
+		PVR_GOTO_IF_ERROR(eError, e0);
+		pcKernelPointer = sMapping.pvKernelAddress;
+		pfnCopyBytes(pcBuffer, &pcKernelPointer[uiOffset], uiBufSz);
+
+		_UnMapPMRKernel(psLMAllocArrayData,
+		                &sMapping);
+
+		uiBytesCopied = uiBufSz;
+	}
+	*puiNumBytes = uiBytesCopied;
+	return PVRSRV_OK;
+e0:
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+static void ReadLocalMem(IMG_UINT8 *pcBuffer,
+						 IMG_UINT8 *pcPMR,
+						 size_t uiSize)
+{
+	/* the memory is mapped as WC (and also aligned to page size) so we can
+	 * safely call "Cached" memcpy */
+	OSCachedMemCopy(pcBuffer, pcPMR, uiSize);
+}
+
+static PVRSRV_ERROR
+PMRReadBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+				  IMG_DEVMEM_OFFSET_T uiOffset,
+				  IMG_UINT8 *pcBuffer,
+				  size_t uiBufSz,
+				  size_t *puiNumBytes)
+{
+	return CopyBytesLocalMem(pvPriv,
+							 uiOffset,
+							 pcBuffer,
+							 uiBufSz,
+							 puiNumBytes,
+							 ReadLocalMem);
+}
+
+static void WriteLocalMem(IMG_UINT8 *pcBuffer,
+						  IMG_UINT8 *pcPMR,
+						  size_t uiSize)
+{
+	/* the memory is mapped as WC (and also aligned to page size) so we can
+	 * safely call "Cached" memcpy but need to issue a write memory barrier
+	 * to flush the write buffers after */
+	OSCachedMemCopyWMB(pcPMR, pcBuffer, uiSize);
+}
+
+static PVRSRV_ERROR
+PMRWriteBytesLocalMem(PMR_IMPL_PRIVDATA pvPriv,
+					  IMG_DEVMEM_OFFSET_T uiOffset,
+					  IMG_UINT8 *pcBuffer,
+					  size_t uiBufSz,
+					  size_t *puiNumBytes)
+{
+	return CopyBytesLocalMem(pvPriv,
+							 uiOffset,
+							 pcBuffer,
+							 uiBufSz,
+							 puiNumBytes,
+							 WriteLocalMem);
+}
+
+/*************************************************************************/ /*!
+@Function       PMRChangeSparseMemLocalMem
+@Description    This function Changes the sparse mapping by allocating and
+                freeing of pages. It also changes the GPU maps accordingly.
+@Return         PVRSRV_ERROR failure code
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+PMRChangeSparseMemLocalMem(PMR_IMPL_PRIVDATA pPriv,
+                           const PMR *psPMR,
+                           IMG_UINT32 ui32AllocPageCount,
+                           IMG_UINT32 *pai32AllocIndices,
+                           IMG_UINT32 ui32FreePageCount,
+                           IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+                           PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+                           IMG_UINT32 uiFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+	IMG_UINT32 ui32AdtnlAllocPages = 0;
+	IMG_UINT32 ui32AdtnlFreePages = 0;
+	IMG_UINT32 ui32CommonRequstCount = 0;
+	IMG_UINT32 ui32Loop = 0;
+	IMG_UINT32 ui32Index = 0;
+	IMG_UINT32 uiAllocpgidx;
+	IMG_UINT32 uiFreepgidx;
+
+	PMR_LMALLOCARRAY_DATA *psPMRPageArrayData = (PMR_LMALLOCARRAY_DATA *)pPriv;
+	IMG_UINT32 uiLog2ChunkSize = psPMRPageArrayData->uiLog2ChunkSize;
+	IMG_UINT64 ui64ChunkSize = IMG_PAGE2BYTES64(uiLog2ChunkSize);
+
+#if defined(DEBUG)
+	IMG_BOOL bPoisonFail = IMG_FALSE;
+	IMG_BOOL bZeroFail = IMG_FALSE;
+#endif
+
+	/* Fetch the Page table array represented by the PMR */
+	RA_BASE_T *paBaseArray = psPMRPageArrayData->aBaseArray;
+	PMR_MAPPING_TABLE *psPMRMapTable = PMR_GetMappingTable(psPMR);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	*ppvZombiePages = NULL;
+#endif
+	/* The incoming request is classified into two operations independent of
+	 * each other: alloc & free chunks.
+	 * These operations can be combined with two mapping operations as well
+	 * which are GPU & CPU space mappings.
+	 *
+	 * From the alloc and free chunk requests, the net amount of chunks to be
+	 * allocated or freed is computed. Chunks that were requested to be freed
+	 * will be reused to fulfil alloc requests.
+	 *
+	 * The order of operations is:
+	 * 1. Allocate new Chunks.
+	 * 2. Move the free chunks from free request to alloc positions.
+	 * 3. Free the rest of the chunks not used for alloc
+	 *
+	 * Alloc parameters are validated at the time of allocation
+	 * and any error will be handled then. */
+
+	if (SPARSE_RESIZE_BOTH == (uiFlags & SPARSE_RESIZE_BOTH))
+	{
+		ui32CommonRequstCount = (ui32AllocPageCount > ui32FreePageCount) ?
+				ui32FreePageCount : ui32AllocPageCount;
+
+		PDUMP_PANIC(PMR_DeviceNode(psPMR), SPARSEMEM_SWAP, "Request to swap alloc & free chunks not supported");
+	}
+
+	if (SPARSE_RESIZE_ALLOC == (uiFlags & SPARSE_RESIZE_ALLOC))
+	{
+		ui32AdtnlAllocPages = ui32AllocPageCount - ui32CommonRequstCount;
+	}
+	else
+	{
+		ui32AllocPageCount = 0;
+	}
+
+	if (SPARSE_RESIZE_FREE == (uiFlags & SPARSE_RESIZE_FREE))
+	{
+		ui32AdtnlFreePages = ui32FreePageCount - ui32CommonRequstCount;
+	}
+	else
+	{
+		ui32FreePageCount = 0;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(
+	    (ui32CommonRequstCount | ui32AdtnlAllocPages | ui32AdtnlFreePages) != 0,
+	    "Invalid combination of parameters: ui32CommonRequstCount,"
+	    " ui32AdtnlAllocPages and ui32AdtnlFreePages.",
+	    PVRSRV_ERROR_INVALID_PARAMS
+	);
+
+	{
+		/* Validate the free page indices */
+		if (ui32FreePageCount)
+		{
+			if (pai32FreeIndices != NULL)
+			{
+				for (ui32Loop = 0; ui32Loop < ui32FreePageCount; ui32Loop++)
+				{
+					uiFreepgidx = pai32FreeIndices[ui32Loop];
+
+					if (uiFreepgidx >= psPMRPageArrayData->uiTotalNumChunks)
+					{
+						PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+					}
+
+					if (RA_BASE_IS_INVALID(paBaseArray[uiFreepgidx]))
+					{
+						PVR_LOG_GOTO_WITH_ERROR("paBaseArray[uiFreepgidx]", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Given non-zero free count but missing indices array",
+				         __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+		}
+
+		/* The following block of code verifies any issues with common alloc chunk indices */
+		for (ui32Loop = ui32AdtnlAllocPages; ui32Loop < ui32AllocPageCount; ui32Loop++)
+		{
+			uiAllocpgidx = pai32AllocIndices[ui32Loop];
+			if (uiAllocpgidx >= psPMRPageArrayData->uiTotalNumChunks)
+			{
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE, e0);
+			}
+
+			if ((!RA_BASE_IS_INVALID(paBaseArray[uiAllocpgidx])) ||
+					(psPMRMapTable->aui32Translation[uiAllocpgidx] != TRANSLATION_INVALID))
+			{
+				PVR_LOG_GOTO_WITH_ERROR("Trying to allocate already allocated page again", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+			}
+		}
+
+		ui32Loop = 0;
+
+		/* Allocate new chunks */
+		if (0 != ui32AdtnlAllocPages)
+		{
+			/* Say how many chunks to allocate */
+			psPMRPageArrayData->uiChunksToAlloc = ui32AdtnlAllocPages;
+
+			eError = _AllocLMPages(psPMRPageArrayData, pai32AllocIndices);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_AllocLMPages", e0);
+
+			/* Mark the corresponding chunks of translation table as valid */
+			for (ui32Loop = 0; ui32Loop < ui32AdtnlAllocPages; ui32Loop++)
+			{
+				psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = pai32AllocIndices[ui32Loop];
+			}
+
+			psPMRMapTable->ui32NumPhysChunks += ui32AdtnlAllocPages;
+		}
+
+		ui32Index = ui32Loop;
+		ui32Loop = 0;
+
+		/* Move the corresponding free chunks to alloc request */
+		eError = RA_SwapSparseMem(psPMRPageArrayData->psArena,
+		                           paBaseArray,
+		                           psPMRPageArrayData->uiTotalNumChunks,
+		                           psPMRPageArrayData->uiLog2ChunkSize,
+		                           &pai32AllocIndices[ui32Index],
+		                           &pai32FreeIndices[ui32Loop],
+		                           ui32CommonRequstCount);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RA_SwapSparseMem", unwind_alloc);
+
+		for (ui32Loop = 0; ui32Loop < ui32CommonRequstCount; ui32Loop++, ui32Index++)
+		{
+			uiAllocpgidx = pai32AllocIndices[ui32Index];
+			uiFreepgidx  = pai32FreeIndices[ui32Loop];
+
+			psPMRMapTable->aui32Translation[uiFreepgidx] = TRANSLATION_INVALID;
+			psPMRMapTable->aui32Translation[uiAllocpgidx] = uiAllocpgidx;
+
+			/* Be sure to honour the attributes associated with the allocation
+			 * such as zeroing, poisoning etc. */
+			if (BIT_ISSET(psPMRPageArrayData->ui32Flags, FLAG_POISON_ON_ALLOC))
+			{
+				eError = _PhysPgMemSet(psPMRPageArrayData,
+				                       &psPMRPageArrayData->aBaseArray[uiAllocpgidx],
+				                       ui64ChunkSize,
+				                       PVRSRV_POISON_ON_ALLOC_VALUE);
+
+				/* Consider this as a soft failure and go ahead but log error to kernel log */
+				if (eError != PVRSRV_OK)
+				{
+#if defined(DEBUG)
+					bPoisonFail = IMG_TRUE;
+#endif
+				}
+			}
+
+			if (BIT_ISSET(psPMRPageArrayData->ui32Flags, FLAG_ZERO))
+			{
+				eError = _PhysPgMemSet(psPMRPageArrayData,
+									   &psPMRPageArrayData->aBaseArray[uiAllocpgidx],
+									   ui64ChunkSize,
+									   ZERO_PAGE_VALUE);
+				/* Consider this as a soft failure and go ahead but log error to kernel log */
+				if (eError != PVRSRV_OK)
+				{
+#if defined(DEBUG)
+					/* Don't think we need to zero any chunks further */
+					bZeroFail = IMG_TRUE;
+#endif
+				}
+			}
+		}
+
+		/* Free or zombie the additional free chunks */
+		if (0 != ui32AdtnlFreePages)
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			PMR_LMALLOCARRAY_DATA *psExtractedPagesPageArray = NULL;
+
+			eError = _ExtractPages(psPMRPageArrayData, &pai32FreeIndices[ui32Loop], ui32AdtnlFreePages, &psExtractedPagesPageArray);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_ExtractPages", e0);
+
+			/* Zombify pages to get proper stats */
+			eError = PMRZombifyLocalMem(psExtractedPagesPageArray, NULL);
+			PVR_LOG_IF_ERROR(eError, "PMRZombifyLocalMem");
+
+			*ppvZombiePages = psExtractedPagesPageArray;
+#else
+			eError = _FreeLMPages(psPMRPageArrayData, &pai32FreeIndices[ui32Loop], ui32AdtnlFreePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "_FreeLMPages", e0);
+#endif /* SUPPORT_PMR_PAGES_DEFERRED_FREE */
+			ui32Index = ui32Loop;
+			ui32Loop = 0;
+
+			while (ui32Loop++ < ui32AdtnlFreePages)
+			{
+				/* Set the corresponding mapping table entry to invalid address */
+				psPMRMapTable->aui32Translation[pai32FreeIndices[ui32Index++]] = TRANSLATION_INVALID;
+			}
+
+			psPMRMapTable->ui32NumPhysChunks -= ui32AdtnlFreePages;
+		}
+	}
+
+#if defined(DEBUG)
+	if (IMG_TRUE == bPoisonFail)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error in poisoning the chunk", __func__));
+	}
+
+	if (IMG_TRUE == bZeroFail)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error in zeroing the chunk", __func__));
+	}
+#endif
+
+	return PVRSRV_OK;
+
+unwind_alloc:
+	_FreeLMPages(psPMRPageArrayData, pai32AllocIndices, ui32Index);
+
+	for (ui32Loop = 0; ui32Loop < ui32Index; ui32Loop++)
+	{
+		psPMRMapTable->aui32Translation[pai32AllocIndices[ui32Loop]] = TRANSLATION_INVALID;
+	}
+
+e0:
+	return eError;
+}
+
+static PMR_IMPL_FUNCTAB _sPMRLMAFuncTab = {
+	.pfnLockPhysAddresses = &PMRLockSysPhysAddressesLocalMem,
+	.pfnUnlockPhysAddresses = &PMRUnlockSysPhysAddressesLocalMem,
+	.pfnDevPhysAddr = &PMRSysPhysAddrLocalMem,
+	.pfnAcquireKernelMappingData = &PMRAcquireKernelMappingDataLocalMem,
+	.pfnReleaseKernelMappingData = &PMRReleaseKernelMappingDataLocalMem,
+	.pfnReadBytes = &PMRReadBytesLocalMem,
+	.pfnWriteBytes = &PMRWriteBytesLocalMem,
+	.pfnChangeSparseMem = &PMRChangeSparseMemLocalMem,
+	.pfnMMap = NULL,
+	.pfnFinalize = &PMRFinalizeLocalMem,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	.pfnFreeZombiePages = &PMRFreeZombiePagesRAMem,
+#endif
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	.pfnZombify = &PMRZombifyLocalMem,
+#endif
+};
+
+PVRSRV_ERROR
+PhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                            RA_ARENA *pArena,
+                            CONNECTION_DATA *psConnection,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_UINT32 ui32NumPhysChunks,
+                            IMG_UINT32 ui32NumVirtChunks,
+                            IMG_UINT32 *pui32MappingTable,
+                            IMG_UINT32 uiLog2AllocPageSize,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                            const IMG_CHAR *pszAnnotation,
+                            IMG_PID uiPid,
+                            PMR **ppsPMRPtr,
+                            IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eError2;
+	PMR *psPMR = NULL;
+	PMR_LMALLOCARRAY_DATA *psPrivData = NULL;
+	PMR_FLAGS_T uiPMRFlags;
+	IMG_UINT32 ui32LMAllocFlags = 0;
+
+	/* This path is checking for the type of PMR to create, if sparse we
+	 * have to perform additional validation as we can only map sparse ranges
+	 * if the os functionality to do so is present. We can also only map virtually
+	 * contiguous sparse regions. Non backed gaps in a range cannot be mapped.
+	 */
+	if (ui32NumPhysChunks != ui32NumVirtChunks || ui32NumVirtChunks > 1)
+	{
+		if (PVRSRV_CHECK_KERNEL_CPU_MAPPABLE(uiFlags) &&
+		   !OSIsMapPhysNonContigSupported())
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: LMA kernel mapping functions not available "
+					"for physically discontiguous memory.",
+					__func__));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, errorOnParam);
+		}
+		BIT_SET(ui32LMAllocFlags, FLAG_SPARSE);
+	}
+
+	if (PVRSRV_CHECK_ON_DEMAND(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_ONDEMAND);
+	}
+
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_ZERO);
+	}
+
+	if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_POISON_ON_ALLOC);
+	}
+
+#if defined(DEBUG)
+	if (PVRSRV_CHECK_POISON_ON_FREE(uiFlags))
+	{
+		BIT_SET(ui32LMAllocFlags, FLAG_POISON_ON_FREE);
+	}
+#endif
+
+	/* Create Array structure that holds the physical pages */
+	eError = _AllocLMPageArray(uiSize,
+	                           ui32NumPhysChunks,
+	                           uiLog2AllocPageSize,
+	                           ui32LMAllocFlags,
+	                           psPhysHeap,
+	                           pArena,
+	                           uiFlags,
+	                           uiPid,
+	                           &psPrivData,
+	                           psConnection);
+	PVR_GOTO_IF_ERROR(eError, errorOnAllocPageArray);
+
+	if (!BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND))
+	{
+		/* Allocate the physical pages */
+		eError = _AllocLMPages(psPrivData, pui32MappingTable);
+		PVR_GOTO_IF_ERROR(eError, errorOnAllocPages);
+	}
+
+	/* In this instance, we simply pass flags straight through.
+
+	   Generically, uiFlags can include things that control the PMR
+	   factory, but we don't need any such thing (at the time of
+	   writing!), and our caller specifies all PMR flags so we don't
+	   need to meddle with what was given to us.
+	*/
+	uiPMRFlags = (PMR_FLAGS_T)(uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK);
+	/* check no significant bits were lost in cast due to different
+	   bit widths for flags */
+	PVR_ASSERT(uiPMRFlags == (uiFlags & PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK));
+
+	if (BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND))
+	{
+		PDUMPCOMMENT(PhysHeapDeviceNode(psPhysHeap), "Deferred Allocation PMR (LMA)");
+	}
+
+	eError = PMRCreatePMR(psPhysHeap,
+						  uiSize,
+						  ui32NumPhysChunks,
+						  ui32NumVirtChunks,
+						  pui32MappingTable,
+						  uiLog2AllocPageSize,
+						  uiPMRFlags,
+						  pszAnnotation,
+						  &_sPMRLMAFuncTab,
+						  psPrivData,
+						  PMR_TYPE_LMA,
+						  &psPMR,
+						  ui32PDumpFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRCreatePMR", errorOnCreate);
+
+	*ppsPMRPtr = psPMR;
+	return PVRSRV_OK;
+
+errorOnCreate:
+	if (!BIT_ISSET(ui32LMAllocFlags, FLAG_ONDEMAND) && psPrivData->iNumChunksAllocated)
+	{
+		eError2 = _FreeLMPages(psPrivData, NULL, 0);
+		PVR_ASSERT(eError2 == PVRSRV_OK);
+	}
+
+errorOnAllocPages:
+	_FreeLMPageArray(psPrivData);
+
+errorOnAllocPageArray:
+errorOnParam:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_ramem.h b/drivers/gpu/drm/img-rogue/physmem_ramem.h
new file mode 100644
index 000000000000..9dbf668e21b9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_ramem.h
@@ -0,0 +1,109 @@
+/*************************************************************************/ /*!
+@File           physmem_ramem.h
+@Title          Resource allocator managed PMR Factory common definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of Services memory management.  This file defines the
+                RA managed memory PMR factory API that is shared between local
+                physheap implementations (LMA & IMA)
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PHYSMEM_RAMEM_H
+#define PHYSMEM_RAMEM_H
+
+#include "img_types.h"
+#include "pvrsrv_memallocflags.h"
+#include "physheap.h"
+#include "ra.h"
+#include "connection_server.h"
+#include "pmr_impl.h"
+
+IMG_UINT32
+RAMemGetPageShift(void);
+
+PVRSRV_ERROR
+RAMemDoPhyContigPagesAlloc(RA_ARENA *pArena,
+                           size_t uiSize,
+                           PVRSRV_DEVICE_NODE *psDevNode,
+                           PG_HANDLE *psMemHandle,
+                           IMG_DEV_PHYADDR *psDevPAddr,
+                           IMG_PID uiPid);
+
+void
+RAMemDoPhyContigPagesFree(RA_ARENA *pArena,
+                          PG_HANDLE *psMemHandle);
+
+PVRSRV_ERROR
+RAMemPhyContigPagesMap(PHYS_HEAP *psPhysHeap,
+                  PG_HANDLE *psMemHandle,
+                  size_t uiSize, IMG_DEV_PHYADDR *psDevPAddr,
+                  void **pvPtr);
+
+void
+RAMemPhyContigPagesUnmap(PHYS_HEAP *psPhysHeap,
+                    PG_HANDLE *psMemHandle,
+                    void *pvPtr);
+
+PVRSRV_ERROR
+RAMemPhyContigPagesClean(PHYS_HEAP *psPhysHeap,
+                    PG_HANDLE *psMemHandle,
+                    IMG_UINT32 uiOffset,
+                    IMG_UINT32 uiLength);
+
+/*
+ * PhysmemNewRAMemRamBackedPMR
+ *
+ * This function will create a PMR using the memory managed by
+ * the PMR factory and is OS agnostic.
+ */
+PVRSRV_ERROR
+PhysmemNewRAMemRamBackedPMR(PHYS_HEAP *psPhysHeap,
+                            RA_ARENA *pArena,
+                            CONNECTION_DATA *psConnection,
+                            IMG_DEVMEM_SIZE_T uiSize,
+                            IMG_UINT32 ui32NumPhysChunks,
+                            IMG_UINT32 ui32NumVirtChunks,
+                            IMG_UINT32 *pui32MappingTable,
+                            IMG_UINT32 uiLog2PageSize,
+                            PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                            const IMG_CHAR *pszAnnotation,
+                            IMG_PID uiPid,
+                            PMR **ppsPMRPtr,
+                            IMG_UINT32 ui32PDumpFlags);
+
+#endif /* PHYSMEM_RAMEM_H */
diff --git a/drivers/gpu/drm/img-rogue/physmem_test.c b/drivers/gpu/drm/img-rogue/physmem_test.c
new file mode 100644
index 000000000000..60c98c78c2a5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_test.c
@@ -0,0 +1,1037 @@
+/*************************************************************************/ /*!
+@Title          Physmem_test
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Single entry point for testing of page factories
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "physmem_test.h"
+#include "device.h"
+#include "syscommon.h"
+#include "pmr.h"
+#include "osfunc.h"
+#include "physmem.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+#include "pvrsrv.h"
+
+#define PHYSMEM_TEST_PAGES        2     /* Mem test pages */
+#define PHYSMEM_TEST_PASSES_MAX   1000  /* Limit number of passes to some reasonable value */
+
+
+/* Test patterns for mem test */
+
+static const IMG_UINT64 gui64Patterns[] = {
+	0,
+	0xffffffffffffffffULL,
+	0x5555555555555555ULL,
+	0xaaaaaaaaaaaaaaaaULL,
+	0x1111111111111111ULL,
+	0x2222222222222222ULL,
+	0x4444444444444444ULL,
+	0x8888888888888888ULL,
+	0x3333333333333333ULL,
+	0x6666666666666666ULL,
+	0x9999999999999999ULL,
+	0xccccccccccccccccULL,
+	0x7777777777777777ULL,
+	0xbbbbbbbbbbbbbbbbULL,
+	0xddddddddddddddddULL,
+	0xeeeeeeeeeeeeeeeeULL,
+	0x7a6c7258554e494cULL,
+};
+
+static const IMG_UINT32 gui32Patterns[] = {
+	0,
+	0xffffffffU,
+	0x55555555U,
+	0xaaaaaaaaU,
+	0x11111111U,
+	0x22222222U,
+	0x44444444U,
+	0x88888888U,
+	0x33333333U,
+	0x66666666U,
+	0x99999999U,
+	0xccccccccU,
+	0x77777777U,
+	0xbbbbbbbbU,
+	0xddddddddU,
+	0xeeeeeeeeU,
+	0x7a6c725cU,
+};
+
+static const IMG_UINT16 gui16Patterns[] = {
+	0,
+	0xffffU,
+	0x5555U,
+	0xaaaaU,
+	0x1111U,
+	0x2222U,
+	0x4444U,
+	0x8888U,
+	0x3333U,
+	0x6666U,
+	0x9999U,
+	0xccccU,
+	0x7777U,
+	0xbbbbU,
+	0xddddU,
+	0xeeeeU,
+	0x7a6cU,
+};
+
+static const IMG_UINT8 gui8Patterns[] = {
+	0,
+	0xffU,
+	0x55U,
+	0xaaU,
+	0x11U,
+	0x22U,
+	0x44U,
+	0x88U,
+	0x33U,
+	0x66U,
+	0x99U,
+	0xccU,
+	0x77U,
+	0xbbU,
+	0xddU,
+	0xeeU,
+	0x6cU,
+};
+
+
+/* Following function does minimal required initialisation for mem test using dummy device node */
+static PVRSRV_ERROR
+PhysMemTestInit(PVRSRV_DEVICE_NODE **ppsDeviceNode, PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_ERROR eError;
+
+	/* Dummy device node */
+	psDeviceNode = OSAllocZMem(sizeof(*psDeviceNode));
+	PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "OSAllocZMem");
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_CREATED);
+	psDeviceNode->psDevConfig = psDevConfig;
+	psDeviceNode->eCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_ON;
+
+	/* Initialise Phys mem heaps */
+	eError = PhysHeapInitDeviceHeaps(psDeviceNode, psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapInitDeviceHeaps", ErrorSysDevDeInit);
+
+	*ppsDeviceNode = psDeviceNode;
+
+	return PVRSRV_OK;
+
+ErrorSysDevDeInit:
+	psDevConfig->psDevNode = NULL;
+	OSFreeMem(psDeviceNode);
+	return eError;
+}
+
+/* Undo initialisation done for mem test */
+static void
+PhysMemTestDeInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* Deinitialise Phys mem heaps */
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+
+	OSFreeMem(psDeviceNode);
+}
+
+static PVRSRV_ERROR
+PMRContiguousSparseMappingTest(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError, eError1;
+	PHYS_HEAP *psHeap;
+	PHYS_HEAP_POLICY psHeapPolicy;
+
+	PMR *psPMR = NULL;
+	PMR *psSpacingPMR = NULL, *psSecondSpacingPMR = NULL;
+	IMG_UINT32 aui32MappingTableFirstAlloc[4] = {0,1,2,3};
+	IMG_UINT32 aui32MappingTableSecondAlloc[8] = {4,5,6,7,8,9,10,11};
+	IMG_UINT32 aui32MappingTableThirdAlloc[4] = {12,13,14,15};
+	IMG_UINT32 ui32NoMappingTable = 0;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_BOOL *pbValid;
+	IMG_DEV_PHYADDR *apsDevPAddr;
+	IMG_UINT32 ui32NumOfPages = 16;
+	size_t uiMappedSize, uiPageSize;
+	IMG_UINT32 i, uiAttempts;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	eError = PhysHeapAcquireByID(PVRSRV_GET_PHYS_HEAP_HINT(uiFlags),
+	                             psDeviceNode,
+	                             &psHeap);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquireByID", ErrorReturn);
+
+	psHeapPolicy = PhysHeapGetPolicy(psHeap);
+
+	PhysHeapRelease(psHeap);
+
+	/* If this is the case then it's not supported and so don't attempt the test */
+	if (psHeapPolicy != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+	{
+		return PVRSRV_OK;
+	}
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate OS memory for PMR page list */
+	apsDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+	PVR_LOG_RETURN_IF_NOMEM(apsDevPAddr, "OSAllocMem");
+
+	/* Allocate OS memory for PMR page state */
+	pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrorFreePMRPageListMem);
+
+	/* Allocate OS memory for write buffer */
+	pcWriteBuffer = OSAllocMem(uiPageSize * ui32NumOfPages);
+	PVR_LOG_GOTO_IF_NOMEM(pcWriteBuffer, eError, ErrorFreePMRPageStateMem);
+	OSCachedMemSet(pcWriteBuffer, 0xF, uiPageSize);
+
+	/* Allocate OS memory for read buffer */
+	pcReadBuffer = OSAllocMem(uiPageSize * ui32NumOfPages);
+	PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+	/* Allocate Sparse PMR with SPARSE | READ | WRITE | UNCACHED_WC attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/*
+	 * Construct a sparse PMR attempting to ensure the allocations
+	 * are physically non contiguous but sequentially placed in the mapping
+	 * table.
+	 */
+	for (uiAttempts = 3; uiAttempts > 0; uiAttempts--)
+	{
+		/* Allocate a sparse PMR from given physical heap - CPU/GPU/FW */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										4,
+										ui32NumOfPages,
+										aui32MappingTableFirstAlloc,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorFreeReadBuffer;
+		}
+
+		/* Allocate some memory from the same physheap so that we can ensure
+		 * the allocations aren't linear
+		 */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										1,
+										1,
+										&ui32NoMappingTable,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psSpacingPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorUnrefPMR;
+		}
+
+		/* Allocate 8 more physical pages on the Sparse PMR */
+		eError = PMR_ChangeSparseMem(psPMR,
+									 8,
+									 aui32MappingTableSecondAlloc,
+									 0,
+									 NULL,
+									 uiFlags | SPARSE_RESIZE_ALLOC);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_ChangeSparseMem", ErrorUnrefSpacingPMR);
+
+		/* Allocate some more memory from the same physheap so that we can ensure
+		 * the allocations aren't linear
+		 */
+		eError = PhysmemNewRamBackedPMR(NULL,
+										psDeviceNode,
+										ui32NumOfPages * uiPageSize,
+										1,
+										1,
+										&ui32NoMappingTable,
+										OSGetPageShift(),
+										uiFlags,
+										sizeof("PMRContiguousSparseMappingTest"),
+										"PMRContiguousSparseMappingTest",
+										OSGetCurrentClientProcessIDKM(),
+										&psSecondSpacingPMR,
+										PDUMP_NONE,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+			goto ErrorUnrefSpacingPMR;
+		}
+
+		/* Allocate final 4 physical pages on the Sparse PMR */
+		eError = PMR_ChangeSparseMem(psPMR,
+									 4,
+									 aui32MappingTableThirdAlloc,
+									 0,
+									 NULL,
+									 uiFlags | SPARSE_RESIZE_ALLOC);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMR_ChangeSparseMem", ErrorUnrefSecondSpacingPMR);
+
+		/*
+		 * Check we have in fact managed to obtain a PMR with non contiguous
+		 * physical pages.
+		 */
+		eError = PMRLockSysPhysAddresses(psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to lock PMR"));
+			goto ErrorUnrefSecondSpacingPMR;
+		}
+
+		/* Get the Device physical addresses of the pages */
+		eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), ui32NumOfPages, 0, apsDevPAddr, pbValid, CPU_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to map PMR pages into device physical addresses"));
+			goto ErrorUnlockPhysAddresses;
+		}
+
+		{
+			IMG_BOOL bPhysicallyContiguous = IMG_TRUE;
+			IMG_DEV_PHYADDR sPrevDevPAddr = apsDevPAddr[0];
+			for (i = 1; i < ui32NumOfPages && bPhysicallyContiguous; i++)
+			{
+				if (apsDevPAddr[i].uiAddr != sPrevDevPAddr.uiAddr + uiPageSize)
+				{
+					bPhysicallyContiguous = IMG_FALSE;
+				}
+				sPrevDevPAddr = apsDevPAddr[i];
+			}
+
+			if (bPhysicallyContiguous)
+			{
+				/* We haven't yet managed to create the mapping scenario we
+				 * require: unwind and attempt again.
+				 */
+				eError1 = PMRUnlockSysPhysAddresses(psPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+				}
+				eError1 = PMRUnrefPMR(psPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+				}
+				eError1 = PMRUnrefPMR(psSpacingPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free Spacing PMR"));
+				}
+				eError1 = PMRUnrefPMR(psSecondSpacingPMR);
+				if (eError1 != PVRSRV_OK)
+				{
+					eError = (eError == PVRSRV_OK)? eError1 : eError;
+					PVR_DPF((PVR_DBG_ERROR, "Failed to free Second Spacing PMR"));
+				}
+			} else {
+				/* We have the scenario, break out of the attempt loop */
+				break;
+			}
+		}
+	}
+
+	if (uiAttempts == 0)
+	{
+		/* We can't create the scenario, very unlikely this would happen */
+		PVR_LOG_GOTO_IF_ERROR(PVRSRV_ERROR_MEMORY_TEST_FAILED,
+		                      "Unable to create Non Contiguous PMR scenario",
+		                      ErrorFreeReadBuffer);
+	}
+
+	/* We have the PMR scenario to test, now attempt to map the whole PMR,
+	 * write and then read from it
+	 */
+	eError = PMRAcquireSparseKernelMappingData(psPMR, 0, ui32NumOfPages * uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	OSCachedMemCopyWMB(pvKernAddr, pcWriteBuffer, ui32NumOfPages * uiPageSize);
+
+	eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+	PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+	/*
+	 * Release and reacquire the mapping to exercise the mapping paths
+	 */
+	eError = PMRAcquireSparseKernelMappingData(psPMR, 0, ui32NumOfPages * uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	OSCachedMemSetWMB(pcReadBuffer, 0x0, ui32NumOfPages * uiPageSize);
+	OSCachedMemCopyWMB(pcReadBuffer, pvKernAddr, ui32NumOfPages * uiPageSize);
+
+	eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+	PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+	for (i = 0; i < ui32NumOfPages * uiPageSize; i++)
+	{
+		if (pcReadBuffer[i] != pcWriteBuffer[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)! @ %u",
+			         __func__, pcReadBuffer[i], pcWriteBuffer[i], i));
+			eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+			goto ErrorUnlockPhysAddresses;
+		}
+	}
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it */
+	eError1 = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+	}
+
+ErrorUnrefSecondSpacingPMR:
+	eError1 = PMRUnrefPMR(psSecondSpacingPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free Second Spacing PMR"));
+	}
+ErrorUnrefSpacingPMR:
+	eError1 = PMRUnrefPMR(psSpacingPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free Spacing PMR"));
+	}
+ErrorUnrefPMR:
+	eError1 = PMRUnrefPMR(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+	}
+
+ErrorFreeReadBuffer:
+	OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+	OSFreeMem(pcWriteBuffer);
+ErrorFreePMRPageStateMem:
+	OSFreeMem(pbValid);
+ErrorFreePMRPageListMem:
+	OSFreeMem(apsDevPAddr);
+ErrorReturn:
+	return eError;
+}
+
+/* Test for PMR factory validation */
+static PVRSRV_ERROR
+PMRValidationTest(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError, eError1;
+	IMG_UINT32 i = 0, j = 0, ui32Index = 0;
+	IMG_UINT32 *pui32MappingTable = NULL;
+	PMR *psPMR = NULL;
+	IMG_BOOL *pbValid;
+	IMG_DEV_PHYADDR *apsDevPAddr;
+	IMG_UINT32 ui32NumOfPages = 10, ui32NumOfPhysPages = 5;
+	size_t uiMappedSize, uiPageSize;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate OS memory for PMR page list */
+	apsDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+	PVR_LOG_RETURN_IF_NOMEM(apsDevPAddr, "OSAllocMem");
+
+	/* Allocate OS memory for PMR page state */
+	pbValid = OSAllocZMem(ui32NumOfPages * sizeof(IMG_BOOL));
+	PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrorFreePMRPageListMem);
+
+	/* Allocate OS memory for write buffer */
+	pcWriteBuffer = OSAllocMem(uiPageSize);
+	PVR_LOG_GOTO_IF_NOMEM(pcWriteBuffer, eError, ErrorFreePMRPageStateMem);
+	OSCachedMemSet(pcWriteBuffer, 0xF, uiPageSize);
+
+	/* Allocate OS memory for read buffer */
+	pcReadBuffer = OSAllocMem(uiPageSize);
+	PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+	/* Allocate OS memory for mapping table */
+	pui32MappingTable = (IMG_UINT32 *)OSAllocMem(ui32NumOfPhysPages * sizeof(*pui32MappingTable));
+	PVR_LOG_GOTO_IF_NOMEM(pui32MappingTable, eError, ErrorFreeReadBuffer);
+
+	/* Pages having even index will have physical backing in PMR */
+	for (ui32Index=0; ui32Index < ui32NumOfPages; ui32Index+=2)
+	{
+		pui32MappingTable[i++] = ui32Index;
+	}
+
+	/* Allocate Sparse PMR with SPARSE | READ | WRITE | UNCACHED_WC attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING |
+				PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+				PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/* Allocate a sparse PMR from given physical heap - CPU/GPU/FW */
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									ui32NumOfPages * uiPageSize,
+									ui32NumOfPhysPages,
+									ui32NumOfPages,
+									pui32MappingTable,
+									OSGetPageShift(),
+									uiFlags,
+									sizeof("PMR ValidationTest"),
+									"PMR ValidationTest",
+									OSGetCurrentClientProcessIDKM(),
+									&psPMR,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to allocate a PMR"));
+		goto ErrorFreeMappingTable;
+	}
+
+	/* Check whether allocated PMR can be locked and obtain physical addresses
+	 * of underlying memory pages.
+	 */
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to lock PMR"));
+		goto ErrorUnrefPMR;
+	}
+
+	/* Get the Device physical addresses of the pages */
+	eError = PMR_DevPhysAddr(psPMR, OSGetPageShift(), ui32NumOfPages, 0, apsDevPAddr, pbValid, CPU_USE);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to map PMR pages into device physical addresses"));
+		goto ErrorUnlockPhysAddresses;
+	}
+
+	/* Check whether device address of each physical page is OS PAGE_SIZE aligned */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			if ((apsDevPAddr[i].uiAddr & OSGetPageMask()) != 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Physical memory of PMR is not page aligned"));
+				eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+				goto ErrorUnlockPhysAddresses;
+			}
+		}
+	}
+
+	/* Acquire kernel virtual address of each physical page and write to it
+	 * and then release it.
+	 */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMR, (i * uiPageSize), uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+				goto ErrorUnlockPhysAddresses;
+			}
+			OSCachedMemCopyWMB(pvKernAddr, pcWriteBuffer, OSGetPageSize());
+
+			eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+			PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+		}
+	}
+
+	/* Acquire kernel virtual address of each physical page and read
+	 * from it and check where contents are intact.
+	 */
+	for (i = 0; i < ui32NumOfPages; i++)
+	{
+		if (pbValid[i])
+		{
+			eError = PMRAcquireSparseKernelMappingData(psPMR, (i * uiPageSize), uiPageSize, &pvKernAddr, &uiMappedSize, &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to Acquire Kernel Mapping of PMR"));
+				goto ErrorUnlockPhysAddresses;
+			}
+			OSCachedMemSetWMB(pcReadBuffer, 0x0, uiPageSize);
+			OSCachedMemCopyWMB(pcReadBuffer, pvKernAddr, uiMappedSize);
+
+			eError = PMRReleaseKernelMappingData(psPMR, hPrivData);
+			PVR_LOG_IF_ERROR(eError, "PMRReleaseKernelMappingData");
+
+			for (j = 0; j < uiPageSize; j++)
+			{
+				if (pcReadBuffer[j] != pcWriteBuffer[j])
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+					         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!",
+					         __func__, pcReadBuffer[j], pcWriteBuffer[j]));
+					eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+					goto ErrorUnlockPhysAddresses;
+				}
+			}
+		}
+	}
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it */
+	eError1 = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to unlock PMR"));
+	}
+
+ErrorUnrefPMR:
+	eError1 = PMRUnrefPMR(psPMR);
+	if (eError1 != PVRSRV_OK)
+	{
+		eError = (eError == PVRSRV_OK)? eError1 : eError;
+		PVR_DPF((PVR_DBG_ERROR, "Failed to free PMR"));
+	}
+ErrorFreeMappingTable:
+	OSFreeMem(pui32MappingTable);
+ErrorFreeReadBuffer:
+	OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+	OSFreeMem(pcWriteBuffer);
+ErrorFreePMRPageStateMem:
+	OSFreeMem(pbValid);
+ErrorFreePMRPageListMem:
+	OSFreeMem(apsDevPAddr);
+
+	return eError;
+}
+
+#define DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, Patterns, NumOfPatterns, Error, ptr, i) \
+	for (i = 0; i < NumOfPatterns; i++) \
+	{ \
+		/* Write pattern */ \
+		for (ptr = StartAddr; ptr < EndAddr; ptr++) \
+		{ \
+			*ptr = Patterns[i]; \
+		} \
+		\
+		/* Read back and validate pattern */ \
+		for (ptr = StartAddr; ptr < EndAddr ; ptr++) \
+		{ \
+			if (*ptr != Patterns[i]) \
+			{ \
+				Error = PVRSRV_ERROR_MEMORY_TEST_FAILED; \
+				break; \
+			} \
+		} \
+		\
+		if (Error != PVRSRV_OK) \
+		{ \
+			break; \
+		} \
+	}
+
+static PVRSRV_ERROR
+TestPatternU8(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT8 *StartAddr = (IMG_UINT8 *) pvKernAddr;
+	IMG_UINT8 *EndAddr = ((IMG_UINT8 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT8));
+	IMG_UINT8 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT8)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui8Patterns, sizeof(gui8Patterns)/sizeof(IMG_UINT8), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!",
+		         __func__, *p, gui8Patterns[i]));
+	}
+
+	return eError;
+}
+
+
+static PVRSRV_ERROR
+TestPatternU16(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT16 *StartAddr = (IMG_UINT16 *) pvKernAddr;
+	IMG_UINT16 *EndAddr = ((IMG_UINT16 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT16));
+	IMG_UINT16 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT16)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui16Patterns, sizeof(gui16Patterns)/sizeof(IMG_UINT16), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%hx), expected (0x%hx)!",
+		         __func__, *p, gui16Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestPatternU32(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT32 *StartAddr = (IMG_UINT32 *) pvKernAddr;
+	IMG_UINT32 *EndAddr = ((IMG_UINT32 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT32));
+	IMG_UINT32 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT32)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui32Patterns, sizeof(gui32Patterns)/sizeof(IMG_UINT32), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%x), expected (0x%x)!",
+		         __func__, *p, gui32Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestPatternU64(void *pvKernAddr, size_t uiMappedSize)
+{
+	IMG_UINT64 *StartAddr = (IMG_UINT64 *) pvKernAddr;
+	IMG_UINT64 *EndAddr = ((IMG_UINT64 *) pvKernAddr) + (uiMappedSize / sizeof(IMG_UINT64));
+	IMG_UINT64 *p;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT((uiMappedSize % sizeof(IMG_UINT64)) == 0);
+
+	DO_MEMTEST_FOR_PATTERNS(StartAddr, EndAddr, gui64Patterns, sizeof(gui64Patterns)/sizeof(IMG_UINT64), eError, p, i);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Test failed. Got (0x%llx), expected (0x%llx)!",
+		         __func__, *p, gui64Patterns[i]));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+TestSplitCacheline(void *pvKernAddr, size_t uiMappedSize)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	size_t uiCacheLineSize;
+	size_t uiBlockSize;
+	size_t j;
+	IMG_UINT8 *pcWriteBuffer, *pcReadBuffer;
+	IMG_UINT8 *StartAddr = (IMG_UINT8 *) pvKernAddr;
+	IMG_UINT8 *EndAddr, *p;
+
+	uiCacheLineSize = OSCPUCacheAttributeSize(OS_CPU_CACHE_ATTRIBUTE_LINE_SIZE);
+
+	if (uiCacheLineSize > 0)
+	{
+		uiBlockSize = (uiCacheLineSize * 2)/3; /* split cacheline */
+
+		pcWriteBuffer = OSAllocMem(uiBlockSize);
+		PVR_LOG_RETURN_IF_NOMEM(pcWriteBuffer, "OSAllocMem");
+
+		/* Fill the write buffer with test data, 0xAB*/
+		OSCachedMemSet(pcWriteBuffer, 0xAB, uiBlockSize);
+
+		pcReadBuffer = OSAllocMem(uiBlockSize);
+		PVR_LOG_GOTO_IF_NOMEM(pcReadBuffer, eError, ErrorFreeWriteBuffer);
+
+		/* Fit only complete blocks in uiMappedSize, ignore leftover bytes */
+		EndAddr = StartAddr + (uiBlockSize * (uiMappedSize / uiBlockSize));
+
+		/* Write blocks into the memory */
+		for (p = StartAddr; p < EndAddr; p += uiBlockSize)
+		{
+			OSCachedMemCopy(p, pcWriteBuffer, uiBlockSize);
+		}
+
+		/* Read back blocks and check */
+		for (p = StartAddr; p < EndAddr; p += uiBlockSize)
+		{
+			OSCachedMemCopy(pcReadBuffer, p, uiBlockSize);
+
+			for (j = 0; j < uiBlockSize; j++)
+			{
+				if (pcReadBuffer[j] != pcWriteBuffer[j])
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Test failed. Got (0x%hhx), expected (0x%hhx)!", __func__, pcReadBuffer[j], pcWriteBuffer[j]));
+					eError = PVRSRV_ERROR_MEMORY_TEST_FAILED;
+					goto ErrorMemTestFailed;
+				}
+			}
+		}
+
+ErrorMemTestFailed:
+		OSFreeMem(pcReadBuffer);
+ErrorFreeWriteBuffer:
+		OSFreeMem(pcWriteBuffer);
+	}
+
+	return eError;
+}
+
+/* Memory test - writes and reads back different patterns to memory and validate the same */
+static PVRSRV_ERROR
+MemTestPatterns(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32MappingTable = 0;
+	PMR *psPMR = NULL;
+	size_t uiMappedSize, uiPageSize;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+
+	uiPageSize = OSGetPageSize();
+
+	/* Allocate PMR with READ | WRITE | WRITE_COMBINE attributes */
+	uiFlags |= PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+			   PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+			   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+
+	/*Allocate a PMR from given physical heap */
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiPageSize * PHYSMEM_TEST_PAGES,
+									1,
+									1,
+									&ui32MappingTable,
+									OSGetPageShift(),
+									uiFlags,
+									sizeof("PMR PhysMemTest"),
+									"PMR PhysMemTest",
+									OSGetCurrentClientProcessIDKM(),
+									&psPMR,
+									PDUMP_NONE,
+									NULL);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemNewRamBackedPMR");
+
+	/* Check whether allocated PMR can be locked and obtain physical
+	 * addresses of underlying memory pages.
+	 */
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRLockSysPhysAddresses", ErrorUnrefPMR);
+
+	/* Map the physical page(s) into kernel space, acquire kernel mapping
+	 * for PMR.
+	 */
+	eError = PMRAcquireKernelMappingData(psPMR, 0, uiPageSize * PHYSMEM_TEST_PAGES, &pvKernAddr, &uiMappedSize, &hPrivData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", ErrorUnlockPhysAddresses);
+
+	PVR_ASSERT((uiPageSize * PHYSMEM_TEST_PAGES) == uiMappedSize);
+
+	/* Test various patterns */
+	eError = TestPatternU64(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU32(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU16(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	eError = TestPatternU8(pvKernAddr, uiMappedSize);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrorReleaseKernelMappingData;
+	}
+
+	/* Test split cachelines */
+	eError = TestSplitCacheline(pvKernAddr, uiMappedSize);
+
+ErrorReleaseKernelMappingData:
+	(void) PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+ErrorUnlockPhysAddresses:
+	/* Unlock and Unref the PMR to destroy it, ignore returned value */
+	(void) PMRUnlockSysPhysAddresses(psPMR);
+ErrorUnrefPMR:
+	(void) PMRUnrefPMR(psPMR);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+PhysMemTestRun(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_MEMALLOCFLAGS_T uiFlags, IMG_UINT32 ui32Passes)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	/* PMR validation test */
+	eError = PMRValidationTest(psDeviceNode, uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR Contiguous PhysHeap self test failed! %"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         __func__,
+		         uiFlags));
+		return eError;
+	}
+
+	eError = PMRContiguousSparseMappingTest(psDeviceNode, uiFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR Non-contiguous PhysHeap self test failed! %"PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         __func__,
+		         uiFlags));
+		return eError;
+	}
+
+
+	for (i = 0; i < ui32Passes; i++)
+	{
+		/* Mem test */
+		eError = MemTestPatterns(psDeviceNode, uiFlags);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: [Pass#%u] MemTestPatterns failed!",
+			         __func__, i));
+			break;
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PhysMemTest(void *pvDevConfig, IMG_UINT32 ui32MemTestPasses)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_DEVICE_CONFIG *psDevConfig = pvDevConfig;
+	PHYS_HEAP_CONFIG *psHeapConfig;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	/* validate memtest passes requested */
+	ui32MemTestPasses = (ui32MemTestPasses > PHYSMEM_TEST_PASSES_MAX)? PHYSMEM_TEST_PASSES_MAX : ui32MemTestPasses;
+
+	/* Do minimal initialisation before test */
+	eError = PhysMemTestInit(&psDeviceNode, psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Test failed to initialize", __func__));
+		return eError;
+	}
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		psHeapConfig = &psDevConfig->pasPhysHeaps[i];
+		if (psHeapConfig->ui32UsageFlags & PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_LOCAL))
+		{
+			/* GPU local mem (should be only up to 1 heap) */
+			eError = PhysMemTestRun(psDeviceNode, PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_LOCAL), ui32MemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "GPU local memory test failed!", ErrorPhysMemTestDeinit);
+		}
+		if (psHeapConfig->ui32UsageFlags & PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL))
+		{
+			/* CPU local mem (should be only up to 1 heap) */
+			eError = PhysMemTestRun(psDeviceNode, PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL), ui32MemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "CPU local memory test failed!", ErrorPhysMemTestDeinit);
+		}
+	}
+
+	PVR_LOG(("PhysMemTest: Passed."));
+	goto PhysMemTestPassed;
+
+ErrorPhysMemTestDeinit:
+	PVR_DPF((PVR_DBG_ERROR, "PhysMemTest: Failed."));
+PhysMemTestPassed:
+	PhysMemTestDeInit(psDeviceNode);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/physmem_test.h b/drivers/gpu/drm/img-rogue/physmem_test.h
new file mode 100644
index 000000000000..684c729d0e51
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/physmem_test.h
@@ -0,0 +1,51 @@
+/*************************************************************************/ /*!
+@Title          Physmem test header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for single entry point for testing of page factories
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PHYSMEM_TEST_H
+#define SRVSRV_PHYSMEM_TEST_H
+/*
+ * PhysMemTest
+ */
+PVRSRV_ERROR
+PhysMemTest(void *pvDevConfig, IMG_UINT32 ui32MemTestPasses);
+
+#endif /* SRVSRV_PHYSMEM_TEST_H */
diff --git a/drivers/gpu/drm/img-rogue/plato_drv.h b/drivers/gpu/drm/img-rogue/plato_drv.h
new file mode 100644
index 000000000000..cfebfb05b8f0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/plato_drv.h
@@ -0,0 +1,462 @@
+/*
+ * @File        plato_drv.h
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PLATO_DRV_H
+#define _PLATO_DRV_H
+
+/*
+ * This contains the hooks for the plato pci driver, as used by the
+ * Rogue and PDP sub-devices, and the platform data passed to each of their
+ * drivers
+ */
+
+#include <linux/platform_device.h>
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/version.h>
+#include <linux/delay.h>
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)
+#include <linux/stringify.h>
+
+#define	PLATO_MULTI_DEVICE
+#endif
+
+// Debug output:
+// Sometimes will want to always output info or error even in release mode.
+// In that case use dev_info, dev_err directly.
+#if defined(PLATO_DRM_DEBUG)
+	#define plato_dev_info(dev, fmt, ...) \
+		dev_info(dev, fmt, ##__VA_ARGS__)
+	#define plato_dev_warn(dev, fmt, ...) \
+		dev_warn(dev, fmt, ##__VA_ARGS__)
+	#define plato_dev_error(dev, fmt, ...) \
+		dev_err(dev, fmt, ##__VA_ARGS__)
+	#define PLATO_DRM_CHECKPOINT            pr_info("line %d\n", __LINE__)
+#else
+	#define plato_dev_info(dev, fmt, ...)
+	#define plato_dev_warn(dev, fmt, ...)
+	#define plato_dev_error(dev, fmt, ...)
+	#define PLATO_DRM_CHECKPOINT
+#endif
+
+#define PLATO_INIT_SUCCESS	0
+#define PLATO_INIT_FAILURE	1
+#define PLATO_INIT_RETRY	2
+
+#define PCI_VENDOR_ID_PLATO				(0x1AEE)
+#define PCI_DEVICE_ID_PLATO				(0x0003)
+
+#define PLATO_SYSTEM_NAME				"Plato"
+
+#define PLATO_MAX_CARDS					4
+
+#define	PLATO_MAX_DEVICE_NAME_LEN			32
+
+#if defined(PLATO_MULTI_DEVICE)
+#define	PLATO_MAKE_DEVICE_TEMPLATE(p)			(p "_%u")
+#define	PLATO_MAKE_DEVICE_NAME(p, i)			(p "_" __stringify(i))
+#endif
+
+/* Interrupt defines */
+enum PLATO_INTERRUPT {
+	PLATO_INTERRUPT_GPU = 0,
+	PLATO_INTERRUPT_PDP,
+	PLATO_INTERRUPT_HDMI,
+	PLATO_INTERRUPT_MAX,
+};
+
+#define PLATO_INT_SHIFT_GPU				(0)
+#define PLATO_INT_SHIFT_PDP				(8)
+#define PLATO_INT_SHIFT_HDMI			(9)
+#define PLATO_INT_SHIFT_HDMI_WAKEUP		(11)
+#define PLATO_INT_SHIFT_TEMP_A			(12)
+
+
+struct plato_region {
+	resource_size_t base;
+	resource_size_t size;
+};
+
+struct plato_io_region {
+	struct plato_region region;
+	void __iomem *registers;
+};
+
+/* The following structs are initialised and passed down by the parent plato
+ * driver to the respective sub-drivers
+ */
+
+#define PLATO_DEVICE_NAME_PDP_PREFIX		"plato_pdp"
+
+#if defined(PLATO_MULTI_DEVICE)
+#define PLATO_DEVICE_NAME_PDP_TEMPLATE		PLATO_MAKE_DEVICE_TEMPLATE( \
+						PLATO_DEVICE_NAME_PDP_PREFIX)
+
+#define	PLATO_DEVICE_NAME_PDP_PRINTF_ARGS(i)	PLATO_DEVICE_NAME_PDP_TEMPLATE, i
+
+#define	PLATO_MAKE_DEVICE_NAME_PDP(i)		PLATO_MAKE_DEVICE_NAME( \
+						PLATO_DEVICE_NAME_PDP_PREFIX, \
+						i)
+#else
+#define PLATO_DEVICE_NAME_PDP_TEMPLATE		PLATO_DEVICE_NAME_PDP_PREFIX
+#define	PLATO_DEVICE_NAME_PDP_PRINTF_ARGS(i)	PLATO_DEVICE_NAME_PDP_TEMPLATE
+#define PLATO_DEVICE_NAME_PDP			PLATO_DEVICE_NAME_PDP_PREFIX
+#endif
+
+#define PLATO_PDP_RESOURCE_REGS			"pdp-regs"
+#define PLATO_PDP_RESOURCE_BIF_REGS		"pdp-bif-regs"
+
+#define PLATO_DEVICE_NAME_HDMI			"plato_hdmi"
+#define PLATO_HDMI_RESOURCE_REGS		"hdmi-regs"
+
+struct plato_pdp_platform_data {
+	resource_size_t memory_base;
+
+	/* The following is used by the drm_pdp driver as it manages the
+	 * pdp memory
+	 */
+	resource_size_t pdp_heap_memory_base;
+	resource_size_t pdp_heap_memory_size;
+
+	/* Used to export host address instead of pdp address,
+	 * defaults to false.
+	 */
+	bool dma_map_export_host_addr;
+};
+
+struct plato_hdmi_platform_data {
+	resource_size_t plato_memory_base;
+};
+
+
+#define PLATO_DEVICE_NAME_ROGUE_PREFIX	"plato_rogue"
+
+#if defined(PLATO_MULTI_DEVICE)
+#define PLATO_DEVICE_NAME_ROGUE_TEMPLATE	PLATO_MAKE_DEVICE_TEMPLATE( \
+						PLATO_DEVICE_NAME_ROGUE_PREFIX)
+
+#define	PLATO_DEVICE_NAME_ROGUE_PRINTF_ARGS(i)	PLATO_DEVICE_NAME_ROGUE_TEMPLATE, i
+
+#define	PLATO_MAKE_DEVICE_NAME_ROGUE(i)		PLATO_MAKE_DEVICE_NAME( \
+						PLATO_DEVICE_NAME_ROGUE_PREFIX, \
+						i)
+#else
+#define PLATO_DEVICE_NAME_ROGUE_TEMPLATE	PLATO_DEVICE_NAME_ROGUE_PREFIX
+#define	PLATO_DEVICE_NAME_ROGUE_PRINTF_ARGS(i)	PLATO_DEVICE_NAME_ROGUE_TEMPLATE
+#define PLATO_DEVICE_NAME_ROGUE			PLATO_DEVICE_NAME_ROGUE_PREFIX
+#endif
+
+#define PLATO_ROGUE_RESOURCE_REGS		"rogue-regs"
+
+struct plato_rogue_platform_data {
+
+	/* The base address of the plato memory (CPU physical address) -
+	 * used to convert from CPU-Physical to device-physical addresses
+	 */
+	resource_size_t plato_memory_base;
+
+	/* The following is used to setup the services heaps */
+	int has_nonmappable;
+	struct plato_region rogue_heap_mappable;
+	resource_size_t rogue_heap_dev_addr;
+	struct plato_region rogue_heap_nonmappable;
+#if defined(SUPPORT_PLATO_DISPLAY)
+	struct plato_region pdp_heap;
+#endif
+};
+
+struct plato_interrupt_handler {
+	bool enabled;
+	void (*handler_function)(void *data);
+	void *handler_data;
+};
+
+struct plato_device {
+	struct pci_dev *pdev;
+
+	struct plato_io_region sys_io;
+	struct plato_io_region aon_regs;
+
+	spinlock_t interrupt_handler_lock;
+	spinlock_t interrupt_enable_lock;
+
+	struct plato_interrupt_handler interrupt_handlers[PLATO_INTERRUPT_MAX];
+
+	struct plato_region rogue_mem;
+	struct plato_region rogue_heap_mappable;
+	struct plato_region rogue_heap_nonmappable;
+	int has_nonmappable;
+
+	resource_size_t dev_mem_base; /* Pointer to device memory base */
+
+	struct platform_device *rogue_dev;
+
+#if defined(SUPPORT_PLATO_DISPLAY)
+	struct platform_device *pdp_dev;
+	struct plato_region pdp_heap;
+
+	struct platform_device *hdmi_dev;
+#endif
+
+	int mtrr;
+	int instance;
+};
+
+#if defined(PLATO_LOG_CHECKPOINTS)
+#define PLATO_CHECKPOINT(p) dev_info(&p->pdev->dev, \
+					"- %s: %d", __func__, __LINE__)
+#else
+#define PLATO_CHECKPOINT(p)
+#endif
+
+#define plato_write_reg32(base, offset, value) \
+	iowrite32(value, (base) + (offset))
+#define plato_read_reg32(base, offset) ioread32(base + offset)
+#define plato_sleep_ms(x) msleep(x)
+#define plato_sleep_us(x) msleep(x/1000)
+
+/* Valid values for the PLATO_MEMORY_CONFIG configuration option */
+#define PLATO_MEMORY_LOCAL			(1)
+#define PLATO_MEMORY_HOST			(2)
+#define PLATO_MEMORY_HYBRID			(3)
+
+#if defined(PLATO_MEMORY_CONFIG)
+#if (PLATO_MEMORY_CONFIG == PLATO_MEMORY_HYBRID)
+#define PVRSRV_PHYS_HEAP_CONFIG_PDP_LOCAL_ID 3
+#elif (PLATO_MEMORY_CONFIG == PLATO_MEMORY_LOCAL)
+#define PVRSRV_PHYS_HEAP_CONFIG_PDP_LOCAL_ID 2
+#endif
+#endif /* PLATO_MEMORY_CONFIG */
+
+#define DCPDP_PHYS_HEAP_ID PVRSRV_PHYS_HEAP_CONFIG_PDP_LOCAL_ID
+
+#define PLATO_PDP_MEM_SIZE			(384 * 1024 * 1024)
+
+#define SYS_PLATO_REG_PCI_BASENUM	(1)
+#define SYS_PLATO_REG_REGION_SIZE	(4 * 1024 * 1024)
+
+/*
+ * Give system region a whole span of the reg space including
+ * RGX registers. That's because there are sys register segments
+ * both before and after the RGX segment.
+ */
+#define SYS_PLATO_REG_SYS_OFFSET			(0x0)
+#define SYS_PLATO_REG_SYS_SIZE				(4 * 1024 * 1024)
+
+/* Entire Peripheral region */
+#define SYS_PLATO_REG_PERIP_OFFSET			(0x20000)
+#define SYS_PLATO_REG_PERIP_SIZE			(164 * 1024)
+
+/* Chip level registers */
+#define SYS_PLATO_REG_CHIP_LEVEL_OFFSET		(SYS_PLATO_REG_PERIP_OFFSET)
+#define SYS_PLATO_REG_CHIP_LEVEL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_TEMPA_OFFSET			(0x80000)
+#define SYS_PLATO_REG_TEMPA_SIZE			(64 * 1024)
+
+/* USB, DMA not included */
+
+#define SYS_PLATO_REG_DDR_A_CTRL_OFFSET		(0x120000)
+#define SYS_PLATO_REG_DDR_A_CTRL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_B_CTRL_OFFSET		(0x130000)
+#define SYS_PLATO_REG_DDR_B_CTRL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_A_PUBL_OFFSET		(0x140000)
+#define SYS_PLATO_REG_DDR_A_PUBL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_DDR_B_PUBL_OFFSET		(0x150000)
+#define SYS_PLATO_REG_DDR_B_PUBL_SIZE		(64 * 1024)
+
+#define SYS_PLATO_REG_NOC_OFFSET			(0x160000)
+#define SYS_PLATO_REG_NOC_SIZE		        (64 * 1024)
+
+/* Debug NOC registers */
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_CTRL_OFFSET (0x1500)
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_DATA_OFFSET (0x1580)
+#define SYS_PLATO_REG_NOC_DBG_DDR_A_PUBL_OFFSET (0x1600)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_CTRL_OFFSET (0x1680)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_DATA_OFFSET (0x1700)
+#define SYS_PLATO_REG_NOC_DBG_DDR_B_PUBL_OFFSET (0x1780)
+#define SYS_PLATO_REG_NOC_DBG_DISPLAY_S_OFFSET  (0x1800)
+#define SYS_PLATO_REG_NOC_DBG_GPIO_0_S_OFFSET   (0x1900)
+#define SYS_PLATO_REG_NOC_DBG_GPIO_1_S_OFFSET   (0x1980)
+#define SYS_PLATO_REG_NOC_DBG_GPU_S_OFFSET      (0x1A00)
+#define SYS_PLATO_REG_NOC_DBG_PCI_PHY_OFFSET    (0x1A80)
+#define SYS_PLATO_REG_NOC_DBG_PCI_REG_OFFSET    (0x1B00)
+#define SYS_PLATO_REG_NOC_DBG_PCI_S_OFFSET      (0x1B80)
+#define SYS_PLATO_REG_NOC_DBG_PERIPH_S_OFFSET   (0x1c00)
+#define SYS_PLATO_REG_NOC_DBG_RET_REG_OFFSET    (0x1D00)
+#define SYS_PLATO_REG_NOC_DBG_SERVICE_OFFSET    (0x1E00)
+
+#define SYS_PLATO_REG_RGX_OFFSET			(0x170000)
+#define SYS_PLATO_REG_RGX_SIZE				(64 * 1024)
+
+#define SYS_PLATO_REG_AON_OFFSET			(0x180000)
+#define SYS_PLATO_REG_AON_SIZE				(64 * 1024)
+
+#define SYS_PLATO_REG_PDP_OFFSET			(0x200000)
+#define SYS_PLATO_REG_PDP_SIZE				(0x1000)
+
+#define SYS_PLATO_REG_PDP_BIF_OFFSET \
+	(SYS_PLATO_REG_PDP_OFFSET + SYS_PLATO_REG_PDP_SIZE)
+#define SYS_PLATO_REG_PDP_BIF_SIZE          (0x200)
+
+#define SYS_PLATO_REG_HDMI_OFFSET \
+	(SYS_PLATO_REG_PDP_OFFSET + 0x20000)
+#define SYS_PLATO_REG_HDMI_SIZE             (128 * 1024)
+
+/* Device memory (including HP mapping) on base register 4 */
+#define SYS_DEV_MEM_PCI_BASENUM		(4)
+
+/* Device memory size */
+#define ONE_GB_IN_BYTES					(0x40000000ULL)
+#define SYS_DEV_MEM_REGION_SIZE \
+	(PLATO_MEMORY_SIZE_GIGABYTES * ONE_GB_IN_BYTES)
+
+/* Plato DDR offset in device memory map at 32GB */
+#define PLATO_DDR_DEV_PHYSICAL_BASE		(0x800000000)
+
+/* DRAM is split at 48GB */
+#define PLATO_DRAM_SPLIT_ADDR			(0xc00000000)
+
+/*
+ * Plato DDR region is aliased if less than 32GB memory is present.
+ * This defines memory base closest to the DRAM split point.
+ * If 32GB is present this is equal to PLATO_DDR_DEV_PHYSICAL_BASE
+ */
+#define PLATO_DDR_ALIASED_DEV_PHYSICAL_BASE \
+	(PLATO_DRAM_SPLIT_ADDR - (SYS_DEV_MEM_REGION_SIZE >> 1))
+
+#define PLATO_DDR_ALIASED_DEV_PHYSICAL_END \
+	(PLATO_DRAM_SPLIT_ADDR + (SYS_DEV_MEM_REGION_SIZE >> 1))
+
+#define PLATO_DDR_ALIASED_DEV_SEGMENT_SIZE \
+	((32ULL / PLATO_MEMORY_SIZE_GIGABYTES) * ONE_GB_IN_BYTES)
+
+/* Plato Host memory offset in device memory map at 512GB */
+#define PLATO_HOSTRAM_DEV_PHYSICAL_BASE (0x8000000000)
+
+/* Plato PLL, DDR/GPU, PDP and HDMI-SFR/CEC clocks */
+#define PLATO_PLL_REF_CLOCK_SPEED	(19200000)
+
+/* 600 MHz */
+#define PLATO_MEM_CLOCK_SPEED		(600000000)
+#define PLATO_MIN_MEM_CLOCK_SPEED	(600000000)
+#define PLATO_MAX_MEM_CLOCK_SPEED	(800000000)
+
+/* 396 MHz (~400 MHz) on HW, around 1MHz on the emulator */
+#if defined(EMULATOR) || defined(VIRTUAL_PLATFORM)
+#define	PLATO_RGX_CORE_CLOCK_SPEED	(1000000)
+#define	PLATO_RGX_MIN_CORE_CLOCK_SPEED	(1000000)
+#define	PLATO_RGX_MAX_CORE_CLOCK_SPEED	(1000000)
+#else
+
+#define	PLATO_RGX_CORE_CLOCK_SPEED	(396000000)
+#define	PLATO_RGX_MIN_CORE_CLOCK_SPEED	(396000000)
+#define	PLATO_RGX_MAX_CORE_CLOCK_SPEED	(742500000)
+#endif
+
+#define PLATO_MIN_PDP_CLOCK_SPEED		(165000000)
+#define PLATO_TARGET_HDMI_SFR_CLOCK_SPEED	(27000000)
+#define PLATO_TARGET_HDMI_CEC_CLOCK_SPEED	(32768)
+
+#define REG_TO_CELSIUS(reg)			(((reg) * 352/4096) - 109)
+#define CELSIUS_TO_REG(temp)		((((temp) + 109) * 4096) / 352)
+#define PLATO_MAX_TEMP_CELSIUS		(100)
+
+#define PLATO_LMA_HEAP_REGION_MAPPABLE			0
+#define PLATO_LMA_HEAP_REGION_NONMAPPABLE		1
+
+struct plato_debug_register {
+	char *description;
+	unsigned int offset;
+	unsigned int value;
+};
+
+#if defined(ENABLE_PLATO_HDMI)
+
+#if defined(HDMI_PDUMP)
+/* Hard coded video formats for pdump type run only */
+#define VIDEO_FORMAT_1280_720p          0
+#define VIDEO_FORMAT_1920_1080p         1
+#define DC_DEFAULT_VIDEO_FORMAT     (VIDEO_FORMAT_1920_1080p)
+#endif
+
+#endif /* ENABLE_PLATO_HDMI */
+
+/* Exposed APIs */
+int plato_enable(struct device *dev);
+void plato_disable(struct device *dev);
+
+int plato_enable_interrupt(struct device *dev,
+			   enum PLATO_INTERRUPT interrupt_id);
+int plato_disable_interrupt(struct device *dev,
+			   enum PLATO_INTERRUPT interrupt_id);
+
+int plato_set_interrupt_handler(struct device *dev,
+				enum PLATO_INTERRUPT interrupt_id,
+				void (*handler_function)(void *),
+				void *handler_data);
+unsigned int plato_core_clock_speed(struct device *dev);
+unsigned int plato_mem_clock_speed(struct device *dev);
+unsigned int plato_pll_clock_speed(struct device *dev,
+				   unsigned int clock_speed);
+void plato_enable_pdp_clock(struct device *dev);
+void plato_enable_pixel_clock(struct device *dev, u32 pixel_clock);
+
+int plato_debug_info(struct device *dev,
+		     struct plato_debug_register *noc_dbg_regs,
+		     struct plato_debug_register *aon_dbg_regs);
+
+/* Internal */
+int plato_memory_init(struct plato_device *plato);
+void plato_memory_deinit(struct plato_device *plato);
+int plato_cfg_init(struct plato_device *plato);
+int request_pci_io_addr(struct pci_dev *pdev, u32 index,
+			resource_size_t offset, resource_size_t length);
+void release_pci_io_addr(struct pci_dev *pdev, u32 index,
+			resource_size_t start, resource_size_t length);
+
+#endif /* _PLATO_DRV_H */
diff --git a/drivers/gpu/drm/img-rogue/pmr.c b/drivers/gpu/drm/img-rogue/pmr.c
new file mode 100644
index 000000000000..41d8a12e7a2c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pmr.c
@@ -0,0 +1,4562 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                the "PMR" abstraction.  A PMR (Physical Memory Resource)
+                represents some unit of physical memory which is
+                allocated/freed/mapped/unmapped as an indivisible unit
+                (higher software levels provide an abstraction above that
+                to deal with dividing this down into smaller manageable units).
+                Importantly, this module knows nothing of virtual memory, or
+                of MMUs etc., with one excusable exception.  We have the
+                concept of a "page size", which really means nothing in
+                physical memory, but represents a "contiguity quantum" such
+                that the higher level modules which map this memory are able
+                to verify that it matches the needs of the page size for the
+                virtual realm into which it is being mapped.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+
+#include "pdump.h"
+#include "devicemem_server_utils.h"
+
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "pmr_impl.h"
+#include "pmr_os.h"
+#include "pmr_env.h"
+#include "pvrsrv.h"
+
+#include "allocmem.h"
+#include "lock.h"
+#include "uniq_key_splay_tree.h"
+
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "secure_export.h"
+#include "ossecure_export.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+/* ourselves */
+#include "pmr.h"
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#include "proc_stats.h"
+#endif
+
+#include "pdump_km.h"
+
+#define PMR_FLAG_INTERNAL_SPARSE_ALLOC     (1 << 0)
+#define PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE (1 << 1)
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PMR_FLAG_INTERNAL_DEFER_FREE       (1 << 2)
+#define PMR_FLAG_INTERNAL_IS_ZOMBIE        (1 << 3)
+
+/* Indicates PMR should be destroyed immediately and not deferred. */
+#define PMR_NO_ZOMBIE_FENCE IMG_UINT64_MAX
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Memalloc flags can be converted into pmr, ra or psplay flags.
+ * Ensure flags types are same size.
+ */
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(PMR_FLAGS_T),
+			  "Mismatch memalloc and pmr flags type size.");
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(RA_FLAGS_T),
+			  "Mismatch memalloc and ra flags type size.");
+static_assert(sizeof(PVRSRV_MEMALLOCFLAGS_T) == sizeof(IMG_PSPLAY_FLAGS_T),
+			  "Mismatch memalloc and psplay flags type size.");
+
+/* A "context" for the physical memory block resource allocator.
+ *
+ * Context is probably the wrong word.
+ *
+ * There is almost certainly only one of these, ever, in the system.
+ * But, let's keep the notion of a context anyway, "just-in-case".
+ */
+static struct _PMR_CTX_
+{
+	/* For debugging, and PDump, etc., let's issue a forever incrementing
+	 * serial number to each allocation.
+	 */
+	IMG_UINT64 uiNextSerialNum;
+
+	/* For security, we only allow a PMR to be mapped if the caller knows
+	 * its key. We can pseudo-randomly generate keys
+	 */
+	IMG_UINT64 uiNextKey;
+
+	/* For debugging only, I guess: Number of live PMRs */
+	ATOMIC_T uiNumLivePMRs;
+
+	/* Lock for this structure */
+	POS_LOCK hLock;
+
+	/* In order to seed the uiNextKey, we enforce initialisation at driver
+	 * load time. Also, we can debug check at driver unload that the PMR
+	 * count is zero.
+	 */
+	IMG_BOOL bModuleInitialised;
+} _gsSingletonPMRContext = { 1, 0, {0}, NULL, IMG_FALSE };
+
+typedef struct _PMR_HEADER_
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* List node used to put the header on the zombie list
+	 * (psDevNode->sPMRZombieList). */
+	DLLIST_NODE sZombieNode;
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	IMG_BOOL bIsZombiePages;
+} PMR_HEADER;
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/*
+ * A structure describing zombie pages.
+ */
+typedef struct _PMR_ZOMBIE_PAGES_
+{
+	PMR_HEADER sHeader;
+	PMR_IMPL_ZOMBIEPAGES pvFactoryPages;
+	PFN_FREE_ZOMBIE_PAGES_FN pfnFactoryFreeZombies;
+} PMR_ZOMBIE_PAGES;
+#endif
+
+/* A PMR. One per physical allocation. May be "shared".
+ *
+ * "shared" is ambiguous. We need to be careful with terminology.
+ * There are two ways in which a PMR may be "shared" and we need to be sure
+ * that we are clear which we mean.
+ *
+ * i)   multiple small allocations living together inside one PMR.
+ *
+ * ii)  one single allocation filling a PMR but mapped into multiple memory
+ *      contexts.
+ *
+ * This is more important further up the stack - at this level, all we care is
+ * that the PMR is being referenced multiple times.
+ */
+struct _PMR_
+{
+	/* A Common header structure shared between PMR and PMR-like PMR_ZOMBIE_PAGES object */
+	PMR_HEADER sHeader;
+
+	/* This object is strictly refcounted. References include:
+	 * - mapping
+	 * - live handles (to this object)
+	 * - live export handles
+	 * (thus it is normal for allocated and exported memory to have a refcount of 3)
+	 * The object is destroyed when and only when the refcount reaches 0
+	 */
+
+	/* Physical address translation (device <> cpu) is done on a per device
+	 * basis which means we need the physical heap info
+	 */
+	PHYS_HEAP *psPhysHeap;
+
+	ATOMIC_T iRefCount;
+
+	/* CPU mapping count - this is the number of times the PMR has been
+	 * mapped to the CPU. It is used to determine when it is safe to permit
+	 * modification of a sparse allocation's layout.
+	 * Note that the process of mapping also increments iRefCount
+	 * independently (as that is used to determine when a PMR may safely
+	 * be destroyed).
+	 */
+	ATOMIC_T iCpuMapCount;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	/* GPU mapping count - this is the number of times a Device Page from a PMR has been
+	 * mapped to the GPU. It is used to determine when it is safe to permit
+	 * migration of constituent OS pages when SUPPORT_LINUX_OSPAGE_MIGRATION is
+	 * used.
+	 */
+	ATOMIC_T iGpuMapCount;
+#endif
+
+	/* Count of how many reservations refer to this
+	 * PMR as a part of a GPU mapping. Must be protected
+	 * by PMR lock.
+	 */
+	IMG_INT32 iAssociatedResCount;
+
+	/* Lock count - this is the number of times PMRLockSysPhysAddresses()
+	 * has been called, less the number of PMRUnlockSysPhysAddresses()
+	 * calls. This is arguably here for debug reasons only, as the refcount
+	 * is already incremented as a matter of course.
+	 * Really, this just allows us to trap protocol errors: i.e. calling
+	 * PMRSysPhysAddr(), without a lock, or calling
+	 * PMRUnlockSysPhysAddresses() too many or too few times.
+	 */
+	ATOMIC_T iLockCount;
+
+	/* Lock for this structure */
+	POS_LOCK hLock;
+
+	/* Incrementing serial number to each allocation. */
+	IMG_UINT64 uiSerialNum;
+
+	/* For security, we only allow a PMR to be mapped if the caller knows
+	 * its key. We can pseudo-randomly generate keys
+	 */
+	PMR_PASSWORD_T uiKey;
+
+	/* Callbacks for per-flavour functions */
+	const PMR_IMPL_FUNCTAB *psFuncTab;
+
+	/* Data associated with the "subtype" */
+	PMR_IMPL_PRIVDATA pvFlavourData;
+
+	/* Environment specific data */
+	DECLARE_PMR_ENV_DATA(sEnvData)
+
+	/* What kind of PMR do we have? */
+	PMR_IMPL_TYPE eFlavour;
+
+	/* And for pdump */
+	const IMG_CHAR *pszPDumpDefaultMemspaceName;
+
+	/* Allocation annotation */
+	IMG_CHAR szAnnotation[DEVMEM_ANNOTATION_MAX_LEN];
+
+#if defined(PDUMP)
+
+	IMG_HANDLE hPDumpAllocHandle;
+
+	IMG_UINT32 uiNumPDumpBlocks;
+#endif
+
+	/* Logical size of allocation. "logical", because a PMR can represent
+	 * memory that will never physically exist.  This is the amount of
+	 * virtual space that the PMR would consume when it's mapped into a
+	 * virtual allocation.
+	 */
+	PMR_SIZE_T uiLogicalSize;
+
+	/* Mapping table for the allocation.
+	 * PMR's can be sparse in which case not all the "logic" addresses in
+	 * it are valid. We need to know which addresses are and aren't valid
+	 * when mapping or reading the PMR.
+	 * The mapping table translates "logical" offsets into physical offsets
+	 * which is what we always pass to the PMR factory (so it doesn't have
+	 * to be concerned about sparseness issues)
+	 */
+	PMR_MAPPING_TABLE *psMappingTable;
+
+	/* Minimum Physical Contiguity Guarantee.  Might be called "page size",
+	 * but that would be incorrect, as page size is something meaningful
+	 * only in virtual realm. This contiguity guarantee provides an
+	 * inequality that can be verified/asserted/whatever to ensure that
+	 * this PMR conforms to the page size requirement of the place the PMR
+	 * gets mapped. (May be used to select an appropriate heap in variable
+	 * page size systems)
+	 *
+	 * The absolutely necessary condition is this:
+	 *
+	 *    device MMU page size <= actual physical contiguity.
+	 *
+	 * We go one step further in order to be able to provide an early
+	 * warning / early compatibility check and say this:
+	 *
+	 *     device MMU page size <=
+	 *         2**(uiLog2ContiguityGuarantee) <=
+	 *             actual physical contiguity.
+	 *
+	 * In this way, it is possible to make the page table reservation
+	 * in the device MMU without even knowing the granularity of the
+	 * physical memory (i.e. useful for being able to allocate virtual
+	 * before physical)
+	 */
+	PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee;
+
+	/* Flags. We store a copy of the "PMR flags" (usually a subset of the
+	 * flags given at allocation time) and return them to any caller of
+	 * PMR_Flags(). The intention of these flags is that the ones stored
+	 * here are used to represent permissions, such that no one is able
+	 * to map a PMR in a mode in which they are not allowed, e.g.,
+	 * writeable for a read-only PMR, etc.
+	 */
+	PMR_FLAGS_T uiFlags;
+
+	/* Various flags informing about PMR's state:
+	 *
+	 * SPARSE_ALLOC:
+	 *   indicates whether this PMR has been allocated as sparse.
+	 *   The condition for this variable to be set at allocation time is:
+	 *   (numVirtChunks != numPhysChunks) || (numVirtChunks > 1)
+	 *
+	 * NO_LAYOUT_CHANGE:
+	 *   Flag that conveys mutability of the PMR:
+	 *   - set:   indicates the PMR is immutable (no more memory changes)
+	 *   - unset: means the memory layout associated with the PMR is mutable
+	 *
+	 *   A PMR is always mutable by default but is marked immutable on the
+	 *   first export for the rest of its life.
+	 *
+	 *   Also, any PMRs that track the same memory through imports are
+	 *   marked immutable as well.
+	 *
+	 * DEFER_FREE:
+	 *   If present the PMR is marked to be freed by the CleanupThread.
+	 *
+	 * IS_ZOMBIE:
+	 *   Indicates if the PMR is in the zombie state (marked for free in the
+	 *   CleanupThread). */
+	IMG_UINT32 uiInternalFlags;
+
+	/* Do we really need this?
+	 * For now we'll keep it, until we know we don't.
+	 * NB: this is not the "memory context" in client terms - this is
+	 * _purely_ the "PMR" context, of which there is almost certainly only
+	 * ever one per system as a whole, but we'll keep the concept anyway,
+	 * just-in-case.
+	 */
+	struct _PMR_CTX_ *psContext;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Stored handle to PMR RI entry */
+	void		*hRIHandle;
+#endif
+};
+
+/* Do we need a struct for the export handle?
+ * I'll use one for now, but if nothing goes in it, we'll lose it
+ */
+struct _PMR_EXPORT_
+{
+	struct _PMR_ *psPMR;
+};
+
+struct _PMR_PAGELIST_
+{
+	struct _PMR_ *psReferencePMR;
+};
+
+#if defined(PDUMP)
+static INLINE IMG_BOOL _IsHostDevicePMR(const PMR *const psPMR)
+{
+	const PVRSRV_DEVICE_NODE *psDevNode = PVRSRVGetPVRSRVData()->psHostMemDeviceNode;
+	return psPMR->psPhysHeap == psDevNode->apsPhysHeap[PVRSRV_PHYS_HEAP_CPU_LOCAL];
+}
+
+static void
+PDumpPMRFreePMR(PMR *psPMR,
+                IMG_DEVMEM_SIZE_T uiSize,
+                IMG_DEVMEM_ALIGN_T uiBlockSize,
+                IMG_UINT32 uiLog2Contiguity,
+                IMG_HANDLE hPDumpAllocationInfoHandle);
+
+static void
+PDumpPMRMallocPMR(PMR *psPMR,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT32 ui32ChunkSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 ui32NumLogicalChunks,
+                  IMG_UINT32 *puiMappingTable,
+                  IMG_UINT32 uiLog2Contiguity,
+                  IMG_BOOL bInitialise,
+                  IMG_UINT8 ui8InitValue,
+                  IMG_HANDLE *phPDumpAllocInfoOut,
+                  IMG_UINT32 ui32PDumpFlags);
+
+static void
+PDumpPMRChangeSparsePMR(PMR *psPMR,
+                        IMG_UINT32 uiBlockSize,
+                        IMG_UINT32 ui32AllocPageCount,
+                        IMG_UINT32 *pai32AllocIndices,
+                        IMG_UINT32 ui32FreePageCount,
+                        IMG_UINT32 *pai32FreeIndices,
+                        IMG_BOOL bInitialise,
+                        IMG_UINT8 ui8InitValue,
+                        IMG_HANDLE *phPDumpAllocInfoOut);
+#endif /* defined PDUMP */
+
+IMG_INT32 PMRGetLiveCount(void)
+{
+	return OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs);
+}
+
+PPVRSRV_DEVICE_NODE PMRGetExportDeviceNode(PMR_EXPORT *psExportPMR)
+{
+	PPVRSRV_DEVICE_NODE psReturnedDeviceNode = NULL;
+
+	PVR_ASSERT(psExportPMR != NULL);
+	if (psExportPMR)
+	{
+		PVR_ASSERT(psExportPMR->psPMR != NULL);
+		if (psExportPMR->psPMR)
+		{
+			PVR_ASSERT(OSAtomicRead(&psExportPMR->psPMR->iRefCount) > 0);
+			if (OSAtomicRead(&psExportPMR->psPMR->iRefCount) > 0)
+			{
+				psReturnedDeviceNode = PMR_DeviceNode(psExportPMR->psPMR);
+			}
+		}
+	}
+
+	return psReturnedDeviceNode;
+}
+
+static PVRSRV_ERROR
+_PMRCreate(PMR_SIZE_T uiLogicalSize,
+           IMG_UINT32 ui32NumPhysChunks,
+           IMG_UINT32 ui32NumLogicalChunks,
+           IMG_UINT32 *pui32MappingTable,
+           PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+           PMR_FLAGS_T uiFlags,
+           PMR **ppsPMR)
+{
+	void *pvPMRLinAddr;
+	PMR *psPMR;
+	PMR_MAPPING_TABLE *psMappingTable;
+	struct _PMR_CTX_ *psContext;
+	IMG_UINT32 i, ui32Temp = 0;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bSparse = IMG_FALSE;
+	PMR_SIZE_T uiChunkSize;
+
+	psContext = &_gsSingletonPMRContext;
+
+	/* Do we have a sparse allocation? */
+	if ( (ui32NumLogicalChunks != ui32NumPhysChunks) ||
+			(ui32NumLogicalChunks > 1) )
+	{
+		bSparse = IMG_TRUE;
+		uiChunkSize = 1ULL << uiLog2ContiguityGuarantee;
+	}
+	else
+	{
+		uiChunkSize = uiLogicalSize;
+	}
+
+	/* Extra checks required for sparse PMRs */
+	if (bSparse)
+	{
+		/* Check the logical size and chunk information agree with each other */
+		if (uiLogicalSize != (uiChunkSize * ui32NumLogicalChunks))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Bad mapping size (uiLogicalSize = 0x%llx, uiChunkSize = 0x%llx, ui32NumLogicalChunks = %d)",
+					__func__, (unsigned long long)uiLogicalSize, (unsigned long long)uiChunkSize, ui32NumLogicalChunks));
+			return PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE;
+		}
+	}
+
+	pvPMRLinAddr = OSAllocMem(sizeof(*psPMR) + sizeof(*psMappingTable) + IMG_FLEX_ARRAY_SIZE(sizeof(IMG_UINT32), ui32NumLogicalChunks));
+	PVR_RETURN_IF_NOMEM(pvPMRLinAddr);
+
+	psPMR = (PMR *) pvPMRLinAddr;
+	psMappingTable = IMG_OFFSET_ADDR(pvPMRLinAddr, sizeof(*psPMR));
+
+	/* Setup the mapping table */
+	psMappingTable->uiChunkSize = uiChunkSize;
+	psMappingTable->ui32NumLogicalChunks = ui32NumLogicalChunks;
+	psMappingTable->ui32NumPhysChunks = ui32NumPhysChunks;
+	OSCachedMemSet(&psMappingTable->aui32Translation[0], 0xFF, sizeof(psMappingTable->aui32Translation[0])*
+	               ui32NumLogicalChunks);
+	for (i=0; i<ui32NumPhysChunks; i++)
+	{
+		ui32Temp = pui32MappingTable[i];
+		PVR_ASSERT(ui32Temp < ui32NumLogicalChunks);
+		psMappingTable->aui32Translation[ui32Temp] = ui32Temp;
+	}
+
+	eError = OSLockCreate(&psPMR->hLock);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psPMR);
+		return eError;
+	}
+
+	/* Setup the PMR */
+	OSAtomicWrite(&psPMR->iRefCount, 0);
+	OSAtomicWrite(&psPMR->iCpuMapCount, 0);
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	OSAtomicWrite(&psPMR->iGpuMapCount, 0);
+#endif
+	psPMR->iAssociatedResCount = 0;
+
+	/* If allocation is not made on demand, it will be backed now and
+	 * backing will not be removed until the PMR is destroyed, therefore
+	 * we can initialise the iLockCount to 1 rather than 0.
+	 */
+	OSAtomicWrite(&psPMR->iLockCount, (PVRSRV_CHECK_ON_DEMAND(uiFlags) ? 0 : 1));
+
+	psPMR->psContext = psContext;
+	psPMR->uiLogicalSize = uiLogicalSize;
+	psPMR->uiLog2ContiguityGuarantee = uiLog2ContiguityGuarantee;
+	psPMR->uiFlags = uiFlags;
+	psPMR->psMappingTable = psMappingTable;
+	psPMR->uiInternalFlags = bSparse ? PMR_FLAG_INTERNAL_SPARSE_ALLOC : 0;
+	psPMR->szAnnotation[0] = '\0';
+	psPMR->sHeader.bIsZombiePages = IMG_FALSE;
+	PMR_ENV_INITIALIZE(psPMR, sEnvData);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	dllist_init(&psPMR->sHeader.sZombieNode);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	psPMR->hRIHandle = NULL;
+#endif
+	OSLockAcquire(psContext->hLock);
+	psPMR->uiKey = psContext->uiNextKey;
+	psPMR->uiSerialNum = psContext->uiNextSerialNum;
+	psContext->uiNextKey = (0x80200003 * psContext->uiNextKey)
+								^ (0xf00f0081 * (uintptr_t)pvPMRLinAddr);
+	psContext->uiNextSerialNum++;
+	*ppsPMR = psPMR;
+	OSLockRelease(psContext->hLock);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+
+	/* Increment live PMR count */
+	OSAtomicIncrement(&psContext->uiNumLivePMRs);
+
+	return PVRSRV_OK;
+}
+
+static IMG_UINT32
+_Ref(PMR *psPMR)
+{
+	if (OSAtomicRead(&psPMR->iRefCount) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Ref Count == 0 PMR: @0x%p Annot: %s",
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+	return OSAtomicIncrement(&psPMR->iRefCount);
+}
+
+static IMG_UINT32
+_Unref(PMR *psPMR)
+{
+	if (OSAtomicRead(&psPMR->iRefCount) <= 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pmr.c: Unref Count <= 0 PMR: @0x%p Annot: %s RefCount: %d",
+		                        psPMR,
+		                        psPMR->szAnnotation,
+		                        (IMG_INT32) OSAtomicRead(&psPMR->iRefCount)));
+		OSWarnOn(1);
+	}
+	return OSAtomicDecrement(&psPMR->iRefCount);
+}
+
+void
+PMRLockPMR(PMR *psPMR)
+{
+	OSLockAcquire(psPMR->hLock);	/* Uses same lock as PhysAddresses */
+}
+
+void
+PMRUnlockPMR(PMR *psPMR)
+{
+	OSLockRelease(psPMR->hLock);	/* Uses same lock as PhysAddresses */
+}
+
+static INLINE void
+_FactoryLock(const PMR_IMPL_FUNCTAB *psFuncTable)
+{
+	if (psFuncTable->pfnGetPMRFactoryLock != NULL)
+	{
+		psFuncTable->pfnGetPMRFactoryLock();
+	}
+}
+
+static INLINE void
+_FactoryUnlock(const PMR_IMPL_FUNCTAB *psFuncTable)
+{
+	if (psFuncTable->pfnReleasePMRFactoryLock != NULL)
+	{
+		psFuncTable->pfnReleasePMRFactoryLock();
+	}
+}
+
+static void
+_PMRDestroy(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	PMR_ENV_FINALIZE(psPMR, sEnvData);
+
+	if (psPMR->psFuncTab->pfnFinalize != NULL)
+	{
+		psPMR->psFuncTab->pfnFinalize(psPMR->pvFlavourData);
+	}
+
+#if defined(PDUMP)
+	/* if allocation is done on the host node don't include it in the PDUMP */
+	if (!_IsHostDevicePMR(psPMR))
+	{
+		PDumpPMRFreePMR(psPMR,
+		                psPMR->uiLogicalSize,
+		                IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee),
+		                psPMR->uiLog2ContiguityGuarantee,
+		                psPMR->hPDumpAllocHandle);
+	}
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	/* This PMR is about to be destroyed, update its mmap stats record (if present)
+	 * to avoid dangling pointer. Additionally, this is required because mmap stats
+	 * are identified by PMRs and a new PMR down the line "might" get the same address
+	 * as the one we're about to free and we'd like 2 different entries in mmaps
+	 * stats for such cases */
+	MMapStatsRemovePMR(psPMR);
+#endif
+
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	/* If not backed on demand, iLockCount should be 1 otherwise it should be 0 */
+	PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1));
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Delete RI entry */
+	if (psPMR->hRIHandle)
+	{
+		PVRSRV_ERROR eError = RIDeletePMREntryKM(psPMR->hRIHandle);
+		PVR_LOG_IF_ERROR(eError, "RIDeletePMREntryKM");
+		/* continue destroying the PMR */
+	}
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	/* Decrement live PMR count. Probably only of interest for debugging */
+	PVR_ASSERT(OSAtomicRead(&psPMR->psContext->uiNumLivePMRs) > 0);
+	OSAtomicDecrement(&psPMR->psContext->uiNumLivePMRs);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+
+	OSLockDestroy(psPMR->hLock);
+	OSFreeMem(psPMR);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+static IMG_BOOL
+PMR_IsListNodeZombiePages(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return psPMRHeader->bIsZombiePages;
+}
+
+static PMR_ZOMBIE_PAGES*
+PMR_GetZombiePagesFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR_ZOMBIE_PAGES, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+static PMR*
+PMR_GetPMRFromNode(const DLLIST_NODE *psNode)
+{
+	PMR_HEADER *psPMRHeader = IMG_CONTAINER_OF(psNode, PMR_HEADER, sZombieNode);
+	PVR_ASSERT(psPMRHeader != NULL);
+	return IMG_CONTAINER_OF(psPMRHeader, PMR, sHeader);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static void
+_UnrefAndMaybeDestroy(PMR *psPMR)
+{
+	const PMR_IMPL_FUNCTAB *psFuncTable;
+	IMG_INT iRefCount;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_ERROR eError;
+#endif
+
+	PVR_ASSERT(psPMR != NULL);
+
+	psFuncTable = psPMR->psFuncTab;
+
+	_FactoryLock(psFuncTable);
+
+	iRefCount = _Unref(psPMR);
+
+	if (iRefCount > 0)
+	{
+		_FactoryUnlock(psFuncTable);
+		return;
+	}
+
+#if !defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* Don't defer PMR destruction in NoHW and PDUMP drivers. */
+	_PMRDestroy(psPMR);
+#else /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+	psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+
+	eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		/* Treat unknown power state as ON. */
+		ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+	}
+
+	/* PMRs that are not marked for deferred free can be freed right away.
+	 * Those are the PMRs that are not device mappable (so only CPU
+	 * readable/writeable).
+	 * All PMRs that are device mappable need to go through the defer free
+	 * path unless the power is OFF. If power is OFF the cache invalidation
+	 * comes as a given. */
+	if (!BITMASK_HAS(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_DEFER_FREE) ||
+	    ePowerState == PVRSRV_DEV_POWER_STATE_OFF ||
+	    psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+	{
+		_PMRDestroy(psPMR);
+	}
+	else
+	{
+		/* Defer freeing the PMR until the Firmware invalidates the caches. */
+		OSLockAcquire(psDevNode->hPMRZombieListLock);
+
+		BITMASK_SET(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+		dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMR->sHeader.sZombieNode);
+		psDevNode->uiPMRZombieCount++;
+
+		/* PMR pages are accounted by the driver/process stats. Those stats
+		 * are available on page level hence they need to be adjusted by
+		 * the factories. This is done by the pfnZombify callback.
+		 * Operation needs to be done while holding hPMRZombieListLock
+		 * to prevent CleanupThread from freeing pages while memory stats
+		 * accounting is ongoing. */
+		if (psPMR->psFuncTab->pfnZombify != NULL)
+		{
+			PVRSRV_ERROR eError = psPMR->psFuncTab->pfnZombify(psPMR->pvFlavourData, psPMR);
+			PVR_LOG_IF_ERROR(eError, "pfnZombify");
+		}
+
+		OSLockRelease(psDevNode->hPMRZombieListLock);
+	}
+#endif /* !defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	_FactoryUnlock(psFuncTable);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+typedef struct _PMR_ZOMBIE_CLEANUP_ITEM_
+{
+	PVRSRV_CLEANUP_THREAD_WORK sCleanupThreadFn;
+	DLLIST_NODE sZombieList;
+	PPVRSRV_DEVICE_NODE psDevNode;
+	PVRSRV_CLIENT_SYNC_PRIM *psSync;
+	IMG_UINT32 uiRequiredSyncValue;
+	IMG_UINT32 uiRequiredPowerOffCounter;
+} PMR_ZOMBIE_CLEANUP_ITEM;
+
+static INLINE void
+_ZombieListLock(PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem)
+{
+	OSLockAcquire(psCleanupItem->psDevNode->hPMRZombieListLock);
+}
+
+static INLINE void
+_ZombieListUnlock(PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem)
+{
+	OSLockRelease(psCleanupItem->psDevNode->hPMRZombieListLock);
+}
+
+static INLINE IMG_BOOL
+_CanNotFreeZombies(const PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem)
+{
+	const PVRSRV_DEVICE_NODE *psDevNode = psCleanupItem->psDevNode;
+
+	/* For a zombie PMR to be eligible to be freed either the GPU MMU caches
+	 * need to be flushed (the Firmware updates the sync) or the GPU power needs
+	 * to be off. */
+	return !PVRSRVHasCounter32Advanced(OSReadDeviceMem32(psCleanupItem->psSync->pui32LinAddr),
+	                                   psCleanupItem->uiRequiredSyncValue) &&
+	       !PVRSRVHasCounter32Advanced(psDevNode->uiPowerOffCounter,
+	                                   psCleanupItem->uiRequiredPowerOffCounter);
+}
+
+static PVRSRV_ERROR _PmrZombieCleanup(void *pvData)
+{
+	PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem = pvData;
+	DLLIST_NODE *psNode;
+	IMG_BOOL bNeedRetry = IMG_FALSE;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (_CanNotFreeZombies(psCleanupItem))
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	do
+	{
+		/* hPMRZombieListLock will prevent removing a node while the list is
+		 * processed. If the lock is already acquired by other process which
+		 * intends to remove an item from the list it'll assure the list
+		 * consistency.
+		 * If this thread acquires the lock first it's possible that another
+		 * thread might be holding PMR factory lock. */
+
+		_ZombieListLock(psCleanupItem);
+		psNode = dllist_get_next_node(&psCleanupItem->sZombieList);
+		_ZombieListUnlock(psCleanupItem);
+
+		if (psNode != NULL)
+		{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			if (PMR_IsListNodeZombiePages(psNode))
+			{
+				PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psNode);
+				eError = psZombiePages->pfnFactoryFreeZombies(psZombiePages->pvFactoryPages);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Cannot free zombie pages! Skipping object %p", psZombiePages));
+					bNeedRetry = IMG_TRUE;
+					continue;
+				}
+				_ZombieListLock(psCleanupItem);
+				dllist_remove_node(psNode);
+				psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+				_ZombieListUnlock(psCleanupItem);
+
+				OSFreeMem(psZombiePages);
+			}
+			else
+#endif
+			{
+				PMR* psPMR = PMR_GetPMRFromNode(psNode);
+				const PMR_IMPL_FUNCTAB *psFuncTable = psPMR->psFuncTab;
+
+				_FactoryLock(psFuncTable);
+				_ZombieListLock(psCleanupItem);
+
+				/* It is possible that the element might have been removed so
+				 * we have to check if the PMR is still a zombie. */
+
+				if (PMR_IsZombie(psPMR))
+				{
+					dllist_remove_node(psNode);
+					psCleanupItem->psDevNode->uiPMRZombieCountInCleanup--;
+					/* Unlock here to avoid locking dependency with the power lock.
+					 * It's okay to do it here since the factory lock is the one
+					 * that needs to be held during PMR destruction. */
+					_ZombieListUnlock(psCleanupItem);
+
+					_PMRDestroy(psPMR);
+				}
+				else
+				{
+					_ZombieListUnlock(psCleanupItem);
+				}
+				_FactoryUnlock(psFuncTable);
+			}
+		}
+	} while (psNode != NULL);
+
+	if (bNeedRetry)
+	{
+		eError = PVRSRV_ERROR_RETRY;
+	}
+	else
+	{
+		OSFreeMem(psCleanupItem);
+	}
+
+	return eError;
+}
+
+IMG_BOOL PMRQueueZombiesForCleanup(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PMR_ZOMBIE_CLEANUP_ITEM *psCleanupItem;
+
+	/* Don't defer the freeing if we are currently unloading the driver
+	 * or if the sync has been destroyed */
+	if (psPVRSRVData->bUnload || psDevNode->psMMUCacheSyncPrim == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	OSLockAcquire(psDevNode->hPMRZombieListLock);
+
+	if (dllist_is_empty(&psDevNode->sPMRZombieList))
+	{
+		OSLockRelease(psDevNode->hPMRZombieListLock);
+		return IMG_FALSE;
+	}
+
+	psCleanupItem = OSAllocMem(sizeof(*psCleanupItem));
+	if (psCleanupItem == NULL)
+	{
+		OSLockRelease(psDevNode->hPMRZombieListLock);
+		return IMG_FALSE;
+	}
+
+	psCleanupItem->sCleanupThreadFn.pfnFree = _PmrZombieCleanup;
+	psCleanupItem->sCleanupThreadFn.pvData = psCleanupItem;
+	psCleanupItem->sCleanupThreadFn.bDependsOnHW = IMG_TRUE;
+	psCleanupItem->sCleanupThreadFn.eCleanupType = PVRSRV_CLEANUP_TYPE_PMR;
+	CLEANUP_THREAD_SET_RETRY_TIMEOUT(&psCleanupItem->sCleanupThreadFn,
+	                                 CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT);
+
+	psCleanupItem->psDevNode = psDevNode;
+	psCleanupItem->psSync = psDevNode->psMMUCacheSyncPrim;
+	psCleanupItem->uiRequiredSyncValue = psDevNode->ui32NextMMUInvalidateUpdate;
+	psCleanupItem->uiRequiredPowerOffCounter = psDevNode->uiPowerOffCounterNext;
+
+	/* This moves the zombie list to the cleanup item. */
+	dllist_replace_head(&psDevNode->sPMRZombieList, &psCleanupItem->sZombieList);
+	psDevNode->uiPMRZombieCountInCleanup += psDevNode->uiPMRZombieCount;
+	psDevNode->uiPMRZombieCount = 0;
+
+	OSLockRelease(psDevNode->hPMRZombieListLock);
+
+	PVRSRVCleanupThreadAddWork(psDevNode, &psCleanupItem->sCleanupThreadFn);
+
+	return IMG_TRUE;
+}
+
+void
+PMRDequeueZombieAndRef(PMR *psPMR)
+{
+	/* If this was on a list then it's brought back to life. */
+	OSLockAcquire(PhysHeapDeviceNode(psPMR->psPhysHeap)->hPMRZombieListLock);
+
+	/* Need to reference this PMR since it was about to be destroyed and its
+	 * reference count must be 0 (can't use _Ref() due to the warning). */
+	OSAtomicIncrement(&psPMR->iRefCount);
+
+#if  defined(DEBUG)
+	PVR_LOG(("%s: 0x%p, key:0x%016" IMG_UINT64_FMTSPECX ", numLive:%d",
+			__func__, psPMR, psPMR->uiKey, OSAtomicRead(&psPMR->psContext->uiNumLivePMRs)));
+#endif
+
+	/* If we got to this point the PMR must be on a list. If it's not
+	 * it should mean a race of some sort. */
+	PVR_ASSERT(!dllist_is_empty(&psPMR->sHeader.sZombieNode));
+
+	/* Revive the PMR (remove it from the zombie list) and therefore
+	 * prevent it's destruction. */
+	dllist_remove_node(&psPMR->sHeader.sZombieNode);
+	BITMASK_UNSET(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+	PhysHeapDeviceNode(psPMR->psPhysHeap)->uiPMRZombieCountInCleanup--;
+
+	OSLockRelease(PhysHeapDeviceNode(psPMR->psPhysHeap)->hPMRZombieListLock);
+}
+
+void
+PMRMarkForDeferFree(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	BITMASK_SET(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_DEFER_FREE);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+static INLINE IMG_BOOL _PMRIsSparse(const PMR *psPMR)
+{
+	return BITMASK_HAS(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_SPARSE_ALLOC);
+}
+
+PVRSRV_ERROR
+PMRCreatePMR(PHYS_HEAP *psPhysHeap,
+             PMR_SIZE_T uiLogicalSize,
+             IMG_UINT32 ui32NumPhysChunks,
+             IMG_UINT32 ui32NumLogicalChunks,
+             IMG_UINT32 *pui32MappingTable,
+             PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+             PMR_FLAGS_T uiFlags,
+             const IMG_CHAR *pszAnnotation,
+             const PMR_IMPL_FUNCTAB *psFuncTab,
+             PMR_IMPL_PRIVDATA pvPrivData,
+             PMR_IMPL_TYPE eType,
+             PMR **ppsPMRPtr,
+             IMG_UINT32 ui32PDumpFlags)
+{
+	PMR *psPMR = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszAnnotation != NULL, "pszAnnotation");
+
+	eError = _PMRCreate(uiLogicalSize,
+	                    ui32NumPhysChunks,
+	                    ui32NumLogicalChunks,
+	                    pui32MappingTable,
+	                    uiLog2ContiguityGuarantee,
+	                    uiFlags,
+	                    &psPMR);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	psPMR->psPhysHeap = psPhysHeap;
+	psPMR->psFuncTab = psFuncTab;
+	psPMR->pszPDumpDefaultMemspaceName = PhysHeapPDumpMemspaceName(psPhysHeap);
+	psPMR->pvFlavourData = pvPrivData;
+	psPMR->eFlavour = eType;
+	OSAtomicWrite(&psPMR->iRefCount, 1);
+
+	OSStringSafeCopy(psPMR->szAnnotation, pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN);
+
+#if defined(PDUMP)
+	/* if allocation was done on the host node don't include it in the PDUMP */
+	if (!_IsHostDevicePMR(psPMR))
+	{
+		PMR_FLAGS_T uiFlags = psPMR->uiFlags;
+		IMG_BOOL bInitialise = IMG_FALSE;
+		IMG_UINT8 ui8InitValue = 0;
+
+		if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags))
+		{
+			bInitialise = IMG_TRUE;
+		}
+		else if (PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags))
+		{
+			ui8InitValue = (IMG_UINT8)PVRSRV_POISON_ON_ALLOC_VALUE;
+			bInitialise = IMG_TRUE;
+		}
+
+		PDumpPMRMallocPMR(psPMR,
+		                  uiLogicalSize,
+		                  IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee),
+		                  ui32NumPhysChunks,
+		                  ui32NumLogicalChunks,
+		                  pui32MappingTable,
+		                  uiLog2ContiguityGuarantee,
+		                  bInitialise,
+		                  ui8InitValue,
+		                  &psPMR->hPDumpAllocHandle,
+		                  ui32PDumpFlags);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	*ppsPMRPtr = psPMR;
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PMRLockSysPhysAddressesNested(PMR *psPMR,
+                                           IMG_UINT32 ui32NestingLevel)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	/* Note: taking this lock is not required to protect the PMR reference
+	 * count, because the PMR reference count is atomic. Rather, taking
+	 * the lock here guarantees that no caller will exit this function
+	 * without the underlying physical addresses being locked.
+	 */
+	OSLockAcquireNested(psPMR->hLock, ui32NestingLevel);
+	/* We also count the locks as references, so that the PMR is not freed
+	 * while someone is using a physical address.
+	 * "lock" here simply means incrementing the refcount. It means the
+	 * refcount is multipurpose, but that's okay. We only have to promise
+	 * that physical addresses are valid after this point, and remain valid
+	 * until the corresponding PMRUnlockSysPhysAddressesOSMem()
+	 */
+	_Ref(psPMR);
+
+	/* Also count locks separately from other types of references, to
+	 * allow for debug assertions
+	 */
+
+	/* Only call callback if lockcount transitions from 0 to 1 (or 1 to 2 if not backed on demand) */
+	if (OSAtomicIncrement(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 1 : 2))
+	{
+		if (psPMR->psFuncTab->pfnLockPhysAddresses != NULL)
+		{
+			/* must always have lock and unlock in pairs! */
+			PVR_ASSERT(psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL);
+
+			eError = psPMR->psFuncTab->pfnLockPhysAddresses(psPMR->pvFlavourData);
+
+			PVR_GOTO_IF_ERROR(eError, e1);
+		}
+	}
+	OSLockRelease(psPMR->hLock);
+
+	return PVRSRV_OK;
+
+e1:
+	OSAtomicDecrement(&psPMR->iLockCount);
+	_Unref(psPMR);
+	PVR_ASSERT(OSAtomicRead(&psPMR->iRefCount) != 0);
+	OSLockRelease(psPMR->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRLockSysPhysAddresses(PMR *psPMR)
+{
+	return PMRLockSysPhysAddressesNested(psPMR, 0);
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddresses(PMR *psPMR)
+{
+	return PMRUnlockSysPhysAddressesNested(psPMR, 2);
+}
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesNested(PMR *psPMR, IMG_UINT32 ui32NestingLevel)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_IMPL_ZOMBIEPAGES pvZombiePages = NULL;
+	PMR_ZOMBIE_PAGES* psPMRZombiePages = NULL;
+#endif
+
+	PVR_ASSERT(psPMR != NULL);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	/* Speculatively preallocate in order to simplify error handling later */
+	psPMRZombiePages = OSAllocZMem(sizeof(PMR_ZOMBIE_PAGES));
+	PVR_GOTO_IF_NOMEM(psPMRZombiePages, eError, e0);
+#endif
+
+	/* Acquiring the lock here, as well as during the Lock operation ensures
+	 * the lock count hitting zero and the unlocking of the phys addresses is
+	 * an atomic operation
+	 */
+	OSLockAcquireNested(psPMR->hLock, ui32NestingLevel);
+	PVR_ASSERT(OSAtomicRead(&psPMR->iLockCount) > (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1));
+
+	if (OSAtomicDecrement(&psPMR->iLockCount) == (PVRSRV_CHECK_ON_DEMAND(psPMR->uiFlags) ? 0 : 1))
+	{
+		if (psPMR->psFuncTab->pfnUnlockPhysAddresses != NULL)
+		{
+			PVR_ASSERT(psPMR->psFuncTab->pfnLockPhysAddresses != NULL);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+			eError = psPMR->psFuncTab->pfnUnlockPhysAddresses(psPMR->pvFlavourData,
+			                                                  &pvZombiePages);
+#else
+			eError = psPMR->psFuncTab->pfnUnlockPhysAddresses(psPMR->pvFlavourData);
+#endif
+
+			/* must never fail */
+			PVR_ASSERT(eError == PVRSRV_OK);
+		}
+	}
+
+	OSLockRelease(psPMR->hLock);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+		PVRSRV_DEVICE_NODE *psDevNode;
+
+		psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+		eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+		if (eError != PVRSRV_OK)
+		{
+			/* Treat unknown power state as ON. */
+			ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+		}
+
+		if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF ||
+			psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+		{
+			/* Free preallocated psPMRZombiePages as these won't be used*/
+			OSFreeMem(psPMRZombiePages);
+
+			eError = psPMR->psFuncTab->pfnFreeZombiePages(pvZombiePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "Error when trying to free zombies immediately.", e0);
+		}
+		else
+		{
+			PVR_ASSERT(psPMRZombiePages != NULL);
+			psPMRZombiePages->sHeader.bIsZombiePages = IMG_TRUE;
+			psPMRZombiePages->pfnFactoryFreeZombies = psPMR->psFuncTab->pfnFreeZombiePages;
+			psPMRZombiePages->pvFactoryPages = pvZombiePages;
+
+			OSLockAcquire(psDevNode->hPMRZombieListLock);
+			dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMRZombiePages->sHeader.sZombieNode);
+			psDevNode->uiPMRZombieCount++;
+			OSLockRelease(psDevNode->hPMRZombieListLock);
+		}
+	}
+	else
+	{
+		OSFreeMem(psPMRZombiePages);
+	}
+#endif
+
+	/* We also count the locks as references, so that the PMR is not
+	 * freed while someone is using a physical address.
+	 */
+	_UnrefAndMaybeDestroy(psPMR);
+
+	return eError;
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+e0:
+	return eError;
+#endif
+}
+
+PVRSRV_ERROR
+PMRMakeLocalImportHandle(PMR *psPMR,
+                         PMR **ppsPMR)
+{
+	PMRRefPMR(psPMR);
+	*ppsPMR = psPMR;
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRUnmakeLocalImportHandle(PMR *psPMR)
+{
+	PMRUnrefPMR(psPMR);
+	return PVRSRV_OK;
+}
+
+/*
+	Note:
+	We pass back the PMR as it was passed in as a different handle type
+	(DEVMEM_MEM_IMPORT) and it allows us to change the import structure
+	type if we should need to embed any meta data in it.
+ */
+PVRSRV_ERROR
+PMRLocalImportPMR(PMR *psPMR,
+                  PMR **ppsPMR,
+                  IMG_DEVMEM_SIZE_T *puiSize,
+                  IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	_Ref(psPMR);
+
+	/* Return the PMR */
+	*ppsPMR = psPMR;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiAlign = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+	return PVRSRV_OK;
+}
+
+inline IMG_UINT64
+PMRInternalGetUID(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiSerialNum;
+}
+
+PVRSRV_ERROR
+PMRGetUID(PMR *psPMR,
+          IMG_UINT64 *pui64UID)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	*pui64UID = PMRInternalGetUID(psPMR);
+
+	return PVRSRV_OK;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR
+PMRExportPMR(PMR *psPMR,
+             PMR_EXPORT **ppsPMRExportPtr,
+             PMR_SIZE_T *puiSize,
+             PMR_LOG2ALIGN_T *puiLog2Contig,
+             PMR_PASSWORD_T *puiPassword)
+{
+	IMG_UINT64 uiPassword;
+	PMR_EXPORT *psPMRExport;
+
+	uiPassword = psPMR->uiKey;
+
+	psPMRExport = OSAllocMem(sizeof(*psPMRExport));
+	PVR_RETURN_IF_NOMEM(psPMRExport);
+
+	psPMRExport->psPMR = psPMR;
+	_Ref(psPMR);
+	/* The layout of a PMR can't change once exported
+	 * to make sure the importers view of the memory is
+	 * the same as exporter. */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	*ppsPMRExportPtr = psPMRExport;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiLog2Contig = psPMR->uiLog2ContiguityGuarantee;
+	*puiPassword = uiPassword;
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PMRUnexportPMR(PMR_EXPORT *psPMRExport)
+{
+	PVR_ASSERT(psPMRExport != NULL);
+	PVR_ASSERT(psPMRExport->psPMR != NULL);
+	PVR_ASSERT(OSAtomicRead(&psPMRExport->psPMR->iRefCount) > 0);
+
+	_UnrefAndMaybeDestroy(psPMRExport->psPMR);
+
+	OSFreeMem(psPMRExport);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PMRImportPMR(PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR)
+{
+	PMR *psPMR;
+
+	PVR_ASSERT(OSAtomicRead(&psPMRExport->psPMR->iRefCount) > 0);
+
+	psPMR = psPMRExport->psPMR;
+
+	PVR_ASSERT(PMR_IsMemLayoutFixed(psPMR));
+
+	if (psPMR->uiKey != uiPassword)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"PMRImport: Import failed, password specified does not match the export"));
+		return PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR;
+	}
+
+	if (psPMR->uiLogicalSize != uiSize || psPMR->uiLog2ContiguityGuarantee != uiLog2Contig)
+	{
+		return PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES;
+	}
+
+	_Ref(psPMR);
+
+	*ppsPMR = psPMR;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRUnimportPMR(PMR *psPMR)
+{
+	_UnrefAndMaybeDestroy(psPMR);
+
+	return PVRSRV_OK;
+}
+
+#endif /* if defined(SUPPORT_INSECURE_EXPORT) */
+
+#if defined(SUPPORT_SECURE_EXPORT)
+PVRSRV_ERROR PMRSecureUnexportPMR(PMR *psPMR)
+{
+	_UnrefAndMaybeDestroy(psPMR);
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _ReleaseSecurePMR(void *psExport)
+{
+	return PMRSecureUnexportPMR(psExport);
+}
+
+PVRSRV_ERROR PMRSecureExportPMR(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+                                PMR *psPMR,
+                                IMG_SECURE_TYPE *phSecure,
+                                PMR **ppsPMR,
+                                CONNECTION_DATA **ppsSecureConnection)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(ppsSecureConnection);
+
+	/* We are acquiring reference to PMR here because OSSecureExport
+	 * releases bridge lock and PMR lock for a moment and we don't want PMR
+	 * to be removed by other thread in the meantime. */
+	_Ref(psPMR);
+
+	eError = OSSecureExport("secure_pmr",
+	                        _ReleaseSecurePMR,
+	                        (void *) psPMR,
+	                        phSecure);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	*ppsPMR = psPMR;
+
+	/* Mark the PMR immutable once exported
+	 * This allows the importers and exporter to have
+	 * the same view of the memory */
+	PMR_SetLayoutFixed(psPMR, IMG_TRUE);
+
+	return PVRSRV_OK;
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	_UnrefAndMaybeDestroy(psPMR);
+	return eError;
+}
+
+PVRSRV_ERROR PMRSecureImportPMR(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE *psDevNode,
+                                IMG_SECURE_TYPE hSecure,
+                                PMR **ppsPMR,
+                                IMG_DEVMEM_SIZE_T *puiSize,
+                                IMG_DEVMEM_ALIGN_T *puiAlign)
+{
+	PVRSRV_ERROR eError;
+	PMR *psPMR;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = OSSecureImport(hSecure, (void **) &psPMR);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	PVR_LOG_RETURN_IF_FALSE(PhysHeapDeviceNode(psPMR->psPhysHeap) == psDevNode,
+					"PMR invalid for this device",
+					PVRSRV_ERROR_PMR_NOT_PERMITTED);
+
+	_Ref(psPMR);
+	/* The PMR should be immutable once exported
+	 * This allows the importers and exporter to have
+	 * the same view of the memory */
+	PVR_ASSERT(PMR_IsMemLayoutFixed(psPMR));
+
+	/* Return the PMR */
+	*ppsPMR = psPMR;
+	*puiSize = psPMR->uiLogicalSize;
+	*puiAlign = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+	return PVRSRV_OK;
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PMRSecureUnimportPMR(PMR *psPMR)
+{
+	_UnrefAndMaybeDestroy(psPMR);
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR
+PMRStoreRIHandle(PMR *psPMR,
+                 void *hRIHandle)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	psPMR->hRIHandle = hRIHandle;
+	return PVRSRV_OK;
+}
+#endif
+
+static PVRSRV_ERROR
+_PMRAcquireKernelMappingData(PMR *psPMR,
+                             size_t uiLogicalOffset,
+                             size_t uiSize,
+                             void **ppvKernelAddressOut,
+                             size_t *puiLengthOut,
+                             IMG_HANDLE *phPrivOut,
+                             IMG_BOOL bMapSparse)
+{
+	PVRSRV_ERROR eError;
+	void *pvKernelAddress;
+	IMG_HANDLE hPriv;
+
+	PVR_ASSERT(psPMR != NULL);
+
+	if (_PMRIsSparse(psPMR) && !bMapSparse)
+	{
+		/* Mapping of sparse allocations must be signalled. */
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	/* Acquire/Release functions must be overridden in pairs */
+	if (psPMR->psFuncTab->pfnAcquireKernelMappingData == NULL)
+	{
+		PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData == NULL);
+
+		/* If PMR implementation does not supply this pair of
+		 * functions, it means they do not permit the PMR to be mapped
+		 * into kernel memory at all
+		 */
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PERMITTED, e0);
+	}
+	PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL);
+
+	eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+	                                                       uiLogicalOffset,
+	                                                       uiSize,
+	                                                       &pvKernelAddress,
+	                                                       &hPriv,
+	                                                       psPMR->uiFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	*ppvKernelAddressOut = pvKernelAddress;
+	if (uiSize == 0)
+	{
+		/* Zero size means map in the whole PMR ... */
+		*puiLengthOut = (size_t)psPMR->uiLogicalSize;
+	}
+	else if (uiSize > IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee))
+	{
+		/* ... map in the requested pages ... */
+		*puiLengthOut = uiSize;
+	}
+	else
+	{
+		/* ... otherwise we just map in one page */
+		*puiLengthOut = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+	}
+	*phPrivOut = hPriv;
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRCpuMapCountIncr(psPMR);
+#endif
+
+	return PVRSRV_OK;
+
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRAcquireKernelMappingData(PMR *psPMR,
+                            size_t uiLogicalOffset,
+                            size_t uiSize,
+                            void **ppvKernelAddressOut,
+                            size_t *puiLengthOut,
+                            IMG_HANDLE *phPrivOut)
+{
+	return _PMRAcquireKernelMappingData(psPMR,
+	                                    uiLogicalOffset,
+	                                    uiSize,
+	                                    ppvKernelAddressOut,
+	                                    puiLengthOut,
+	                                    phPrivOut,
+	                                    IMG_FALSE);
+}
+
+PVRSRV_ERROR
+PMRAcquireSparseKernelMappingData(PMR *psPMR,
+                                  size_t uiLogicalOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  size_t *puiLengthOut,
+                                  IMG_HANDLE *phPrivOut)
+{
+	return _PMRAcquireKernelMappingData(psPMR,
+	                                    uiLogicalOffset,
+	                                    uiSize,
+	                                    ppvKernelAddressOut,
+	                                    puiLengthOut,
+	                                    phPrivOut,
+	                                    IMG_TRUE);
+}
+
+PVRSRV_ERROR
+PMRReleaseKernelMappingData(PMR *psPMR,
+                            IMG_HANDLE hPriv)
+{
+	PVR_ASSERT (psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL);
+	PVR_ASSERT (psPMR->psFuncTab->pfnReleaseKernelMappingData != NULL);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+	PMRCpuMapCountDecr(psPMR);
+#endif
+
+	psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+	                                              hPriv);
+
+	return PVRSRV_OK;
+}
+
+/*
+	_PMRLogicalOffsetToPhysicalOffset
+
+	Translate between the "logical" offset which the upper levels
+	provide and the physical offset which is what the PMR
+	factories works on.
+
+	As well as returning the physical offset we return the number of
+	bytes remaining till the next chunk and if this chunk is valid.
+
+	For multi-page operations, upper layers communicate their
+	Log2PageSize else argument is redundant (set to zero).
+ */
+
+static PVRSRV_ERROR
+_PMRLogicalOffsetToPhysicalOffset(const PMR *psPMR,
+                                  IMG_UINT32 ui32Log2PageSize,
+                                  IMG_UINT32 ui32NumOfPages,
+                                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                                  IMG_DEVMEM_OFFSET_T *puiPhysicalOffset,
+                                  IMG_UINT32 *pui32BytesRemain,
+                                  IMG_BOOL *bValid)
+{
+	PMR_MAPPING_TABLE *psMappingTable = psPMR->psMappingTable;
+	IMG_DEVMEM_OFFSET_T uiPageSize = 1ULL << ui32Log2PageSize;
+	IMG_DEVMEM_OFFSET_T uiOffset = uiLogicalOffset;
+	IMG_UINT64 ui64ChunkIndex;
+	IMG_UINT32 ui32Remain;
+	IMG_UINT32 idx;
+
+	/* Must be translating at least a page */
+	PVR_ASSERT(ui32NumOfPages > 0);
+
+	if (psMappingTable->ui32NumPhysChunks == psMappingTable->ui32NumLogicalChunks)
+	{
+		/* Fast path the common case, as logical and physical offsets are
+		 * equal we assume the ui32NumOfPages span is also valid */
+		*pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiOffset);
+		puiPhysicalOffset[0] = uiOffset;
+		bValid[0] = IMG_TRUE;
+
+		if (ui32NumOfPages > 1)
+		{
+			/* initial offset may not be page aligned, round down */
+			uiOffset &= ~(uiPageSize-1);
+			for (idx = 1; idx < ui32NumOfPages; idx++)
+			{
+				uiOffset += uiPageSize;
+				puiPhysicalOffset[idx] = uiOffset;
+				bValid[idx] = IMG_TRUE;
+			}
+		}
+	}
+	else
+	{
+		for (idx = 0; idx < ui32NumOfPages; idx++)
+		{
+			IMG_UINT32 uiTranslation;
+			const IMG_UINT32 uiChunkSize = psMappingTable->uiChunkSize;
+
+			ui64ChunkIndex = OSDivide64r64(
+					uiOffset,
+					TRUNCATE_64BITS_TO_32BITS(uiChunkSize),
+					&ui32Remain);
+
+			/* In some cases ui32NumOfPages can come from the user space which
+			 * means that the uiOffset could go out-of-bounds when the number
+			 * of pages is invalid. */
+			if (ui64ChunkIndex >= psMappingTable->ui32NumLogicalChunks)
+			{
+				return PVRSRV_ERROR_BAD_MAPPING;
+			}
+
+			uiTranslation = psMappingTable->aui32Translation[ui64ChunkIndex];
+
+			if (uiTranslation == TRANSLATION_INVALID)
+			{
+				bValid[idx] = IMG_FALSE;
+				/* explicitly set to an invalid value */
+				puiPhysicalOffset[idx] = IMG_UINT64_C(0xffffffffffffffff);
+			}
+			else
+			{
+				bValid[idx] = IMG_TRUE;
+				puiPhysicalOffset[idx] = uiTranslation * uiChunkSize + ui32Remain;
+			}
+
+			if (idx == 0)
+			{
+				*pui32BytesRemain = TRUNCATE_64BITS_TO_32BITS(uiChunkSize - ui32Remain);
+				/* initial offset may not be page aligned, round down */
+				uiOffset &= ~(uiPageSize-1);
+			}
+
+			uiOffset += uiPageSize;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_PMR_ReadBytesPhysical(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                       IMG_UINT8 *pcBuffer,
+                       size_t uiBufSz,
+                       size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError;
+
+	if (psPMR->psFuncTab->pfnReadBytes != NULL)
+	{
+		/* defer to callback if present */
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		eError = psPMR->psFuncTab->pfnReadBytes(psPMR->pvFlavourData,
+		                                        uiPhysicalOffset,
+		                                        pcBuffer,
+		                                        uiBufSz,
+		                                        puiNumBytes);
+		PMRUnlockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+	}
+	else if (psPMR->psFuncTab->pfnAcquireKernelMappingData)
+	{
+		/* "default" handler for reading bytes */
+
+		IMG_HANDLE hKernelMappingHandle;
+		IMG_UINT8 *pcKernelAddress;
+
+		eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+		                                                       (size_t) uiPhysicalOffset,
+		                                                       uiBufSz,
+		                                                       (void **)&pcKernelAddress,
+		                                                       &hKernelMappingHandle,
+		                                                       psPMR->uiFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Use the conservative 'DeviceMemCopy' here because we can't
+		 * know if this PMR will be mapped cached.
+		 */
+
+		OSDeviceMemCopy(&pcBuffer[0], pcKernelAddress, uiBufSz);
+		*puiNumBytes = uiBufSz;
+
+		psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+		                                              hKernelMappingHandle);
+	}
+	else
+	{
+		OSPanic();
+		PVR_LOG_GOTO_WITH_ERROR("psPMR->psFuncTab", eError, PVRSRV_ERROR_INVALID_PARAMS, e0);
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*puiNumBytes = 0;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMR_ReadBytes(PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT8 *pcBuffer,
+              size_t uiBufSz,
+              size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	size_t uiBytesCopied = 0;
+
+	/* Check for integer overflow as uiLogicalOffset might come from the client */
+	if (uiLogicalOffset + uiBufSz < uiLogicalOffset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize)
+	{
+		uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset);
+	}
+	PVR_ASSERT(uiBufSz > 0);
+	PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize);
+
+	/* PMR implementations can override this. If they don't, a "default"
+	 * handler uses kernel virtual mappings.  If the kernel can't
+	 * provide a kernel virtual mapping, this function fails.
+	 */
+	PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL ||
+	           psPMR->psFuncTab->pfnReadBytes != NULL);
+
+	while (uiBytesCopied != uiBufSz)
+	{
+		IMG_UINT32 ui32Remain;
+		size_t uiBytesToCopy;
+		size_t uiRead;
+		IMG_BOOL bValid;
+
+		eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+		                                           0,
+		                                           1,
+		                                           uiLogicalOffset,
+		                                           &uiPhysicalOffset,
+		                                           &ui32Remain,
+		                                           &bValid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+		/* Copy till either then end of the chunk or end
+		 * of the buffer
+		 */
+		uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain);
+
+		if (bValid)
+		{
+			/* Read the data from the PMR */
+			eError = _PMR_ReadBytesPhysical(psPMR,
+			                                uiPhysicalOffset,
+			                                &pcBuffer[uiBytesCopied],
+			                                uiBytesToCopy,
+			                                &uiRead);
+			if ((eError != PVRSRV_OK) || (uiRead != uiBytesToCopy))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to read chunk (eError = %s, uiRead = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")",
+						__func__,
+						PVRSRVGetErrorString(eError),
+						uiRead,
+						uiBytesToCopy));
+				/* Bail out as soon as we hit an error */
+				break;
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: Invalid phys offset at logical offset (" IMG_DEVMEM_OFFSET_FMTSPEC ") logical size (" IMG_DEVMEM_OFFSET_FMTSPEC ")",
+					__func__,
+					uiLogicalOffset,
+					psPMR->uiLogicalSize));
+			/* Fill invalid chunks with 0 */
+			OSCachedMemSet(&pcBuffer[uiBytesCopied], 0, uiBytesToCopy);
+			uiRead = uiBytesToCopy;
+			eError = PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR;
+		}
+		uiLogicalOffset += uiRead;
+		uiBytesCopied += uiRead;
+	}
+
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+static PVRSRV_ERROR
+_PMR_WriteBytesPhysical(PMR *psPMR,
+                        IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                        IMG_UINT8 *pcBuffer,
+                        size_t uiBufSz,
+                        size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError;
+
+	if (psPMR->psFuncTab->pfnWriteBytes != NULL)
+	{
+		/* defer to callback if present */
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		eError = psPMR->psFuncTab->pfnWriteBytes(psPMR->pvFlavourData,
+		                                         uiPhysicalOffset,
+		                                         pcBuffer,
+		                                         uiBufSz,
+		                                         puiNumBytes);
+		PMRUnlockSysPhysAddresses(psPMR);
+		PVR_GOTO_IF_ERROR(eError, e0);
+	}
+	else if (psPMR->psFuncTab->pfnAcquireKernelMappingData)
+	{
+		/* "default" handler for reading bytes */
+
+		IMG_HANDLE hKernelMappingHandle;
+		IMG_UINT8 *pcKernelAddress;
+
+		eError = psPMR->psFuncTab->pfnAcquireKernelMappingData(psPMR->pvFlavourData,
+		                                                       (size_t) uiPhysicalOffset,
+		                                                       uiBufSz,
+		                                                       (void **)&pcKernelAddress,
+		                                                       &hKernelMappingHandle,
+		                                                       psPMR->uiFlags);
+		PVR_GOTO_IF_ERROR(eError, e0);
+
+		/* Use the conservative 'DeviceMemCopy' here because we can't know
+		 * if this PMR will be mapped cached.
+		 */
+
+		OSDeviceMemCopy(pcKernelAddress, &pcBuffer[0], uiBufSz);
+		*puiNumBytes = uiBufSz;
+
+		psPMR->psFuncTab->pfnReleaseKernelMappingData(psPMR->pvFlavourData,
+		                                              hKernelMappingHandle);
+	}
+	else
+	{
+		/* The write callback is optional as it's only required by the
+		 * debug tools
+		 */
+		OSPanic();
+		PVR_LOG_GOTO_WITH_ERROR("psPMR->psFuncTab", eError, PVRSRV_ERROR_PMR_NOT_PERMITTED, e0);
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*puiNumBytes = 0;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMR_WriteBytes(PMR *psPMR,
+               IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+               IMG_UINT8 *pcBuffer,
+               size_t uiBufSz,
+               size_t *puiNumBytes)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	size_t uiBytesCopied = 0;
+
+	/* Check for integer overflow as uiLogicalOffset might come from the client */
+	if (uiLogicalOffset + uiBufSz < uiLogicalOffset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (uiLogicalOffset + uiBufSz > psPMR->uiLogicalSize)
+	{
+		uiBufSz = TRUNCATE_64BITS_TO_32BITS(psPMR->uiLogicalSize - uiLogicalOffset);
+	}
+	PVR_ASSERT(uiBufSz > 0);
+	PVR_ASSERT(uiBufSz <= psPMR->uiLogicalSize);
+
+	/* PMR implementations can override this. If they don't, a "default"
+	 * handler uses kernel virtual mappings. If the kernel can't provide
+	 * a kernel virtual mapping, this function fails.
+	 */
+	PVR_ASSERT(psPMR->psFuncTab->pfnAcquireKernelMappingData != NULL ||
+	           psPMR->psFuncTab->pfnWriteBytes != NULL);
+
+	while (uiBytesCopied != uiBufSz)
+	{
+		IMG_UINT32 ui32Remain;
+		size_t uiBytesToCopy;
+		size_t uiWrite;
+		IMG_BOOL bValid;
+
+		eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+		                                           0,
+		                                           1,
+		                                           uiLogicalOffset,
+		                                           &uiPhysicalOffset,
+		                                           &ui32Remain,
+		                                           &bValid);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+		/* Copy till either then end of the chunk or end of the buffer
+		 */
+		uiBytesToCopy = MIN(uiBufSz - uiBytesCopied, ui32Remain);
+
+		if (bValid)
+		{
+			/* Write the data to the PMR */
+			eError = _PMR_WriteBytesPhysical(psPMR,
+			                                 uiPhysicalOffset,
+			                                 &pcBuffer[uiBytesCopied],
+			                                 uiBytesToCopy,
+			                                 &uiWrite);
+			if ((eError != PVRSRV_OK) || (uiWrite != uiBytesToCopy))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to read chunk (eError = %s, uiWrite = " IMG_SIZE_FMTSPEC " uiBytesToCopy = " IMG_SIZE_FMTSPEC ")",
+						__func__,
+						PVRSRVGetErrorString(eError),
+						uiWrite,
+						uiBytesToCopy));
+				/* Bail out as soon as we hit an error */
+				break;
+			}
+		}
+		else
+		{
+			/* Ignore writes to invalid pages */
+			uiWrite = uiBytesToCopy;
+		}
+		uiLogicalOffset += uiWrite;
+		uiBytesCopied += uiWrite;
+	}
+
+	*puiNumBytes = uiBytesCopied;
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRMMapPMR(PMR *psPMR, PMR_MMAP_DATA pOSMMapData, PVRSRV_MEMALLOCFLAGS_T uiFlags)
+{
+	/* if writeable mapping is requested on non-writeable PMR then fail */
+	PVR_RETURN_IF_FALSE(PVRSRV_CHECK_CPU_WRITEABLE(psPMR->uiFlags) ||
+	                    !PVRSRV_CHECK_CPU_WRITEABLE(uiFlags),
+	                    PVRSRV_ERROR_PMR_NOT_PERMITTED);
+
+	PVR_LOG_RETURN_IF_TRUE(PMR_PhysicalSize(psPMR) == 0, "PVRSRV_ERROR_BAD_MAPPING can not map PMR of 0 physical size", PVRSRV_ERROR_BAD_MAPPING);
+
+	if (psPMR->psFuncTab->pfnMMap)
+	{
+		return psPMR->psFuncTab->pfnMMap(psPMR->pvFlavourData, psPMR, pOSMMapData);
+	}
+
+	return OSMMapPMRGeneric(psPMR, pOSMMapData);
+}
+
+void
+PMRRefPMR(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	_Ref(psPMR);
+
+	/* Lock phys addresses if PMR backing was allocated immediately */
+	if (PVRSRV_CHECK_PHYS_ALLOC_NOW(psPMR->uiFlags))
+	{
+		PMRLockSysPhysAddresses(psPMR);
+	}
+}
+
+PVRSRV_ERROR
+PMRUnrefPMR(PMR *psPMR)
+{
+	/* Unlock phys addresses if PMR backing was allocated immediately */
+	if (PVRSRV_CHECK_PHYS_ALLOC_NOW(psPMR->uiFlags))
+	{
+		PMRUnlockSysPhysAddresses(psPMR);
+	}
+
+	_UnrefAndMaybeDestroy(psPMR);
+	return PVRSRV_OK;
+}
+
+void
+PMRRefPMR2(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	_Ref(psPMR);
+}
+
+void
+PMRUnrefPMR2(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	_UnrefAndMaybeDestroy(psPMR);
+}
+
+#define PMR_MAPCOUNT_MIN 0
+#define PMR_MAPCOUNT_MAX IMG_INT32_MAX
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+PVRSRV_ERROR PMRTryRefPMR(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	if (OSAtomicAddUnless(&psPMR->iRefCount, 1, 0) == 0)
+	{
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	return PVRSRV_OK;
+}
+
+void
+PMRGpuMapDevPageCountIncr(PMR *psPMR, IMG_UINT32 uiCount)
+{
+	if (OSAtomicAddUnless(&psPMR->iGpuMapCount, uiCount, PMR_MAPCOUNT_MAX) == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iGpuMapCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+void
+PMRGpuMapDevPageCountDecr(PMR *psPMR, IMG_UINT32 uiCount)
+{
+	if (OSAtomicSubtractUnless(&psPMR->iGpuMapCount, uiCount, PMR_MAPCOUNT_MIN) == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iGpuMapCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        (IMG_INT32) OSAtomicRead(&psPMR->iGpuMapCount),
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+IMG_BOOL
+PMR_IsGpuMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return (OSAtomicRead(&psPMR->iGpuMapCount) > 0);
+}
+#endif /* #if defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+void
+PMRCpuMapCountIncr(PMR *psPMR)
+{
+	if (OSAtomicAddUnless(&psPMR->iCpuMapCount, 1, PMR_MAPCOUNT_MAX) == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iCpuMapCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+void
+PMRCpuMapCountDecr(PMR *psPMR)
+{
+	if (OSAtomicSubtractUnless(&psPMR->iCpuMapCount, 1, PMR_MAPCOUNT_MIN) == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iCpuMapCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        (IMG_INT32) OSAtomicRead(&psPMR->iCpuMapCount),
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+	}
+}
+
+IMG_BOOL
+PMR_IsCpuMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return (OSAtomicRead(&psPMR->iCpuMapCount) > 0);
+}
+
+void
+PMRGpuResCountIncr(PMR *psPMR)
+{
+	if (psPMR->iAssociatedResCount == PMR_MAPCOUNT_MAX)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iAssociatedResCount for PMR: @0x%p (%s) has overflowed.",
+		                        __func__,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+		return;
+	}
+
+	psPMR->iAssociatedResCount++;
+}
+
+void
+PMRGpuResCountDecr(PMR *psPMR)
+{
+	if (psPMR->iAssociatedResCount == PMR_MAPCOUNT_MIN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: iAssociatedResCount (now %d) for PMR: @0x%p (%s) has underflowed.",
+		                        __func__,
+		                        psPMR->iAssociatedResCount,
+		                        psPMR,
+		                        psPMR->szAnnotation));
+		OSWarnOn(1);
+		return;
+	}
+
+	psPMR->iAssociatedResCount--;
+}
+
+IMG_BOOL
+PMR_IsGpuMultiMapped(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->iAssociatedResCount > 1;
+}
+
+PVRSRV_DEVICE_NODE *
+PMR_DeviceNode(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return PhysHeapDeviceNode(psPMR->psPhysHeap);
+}
+
+PMR_FLAGS_T
+PMR_Flags(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiFlags;
+}
+
+IMG_BOOL
+PMR_IsSparse(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return _PMRIsSparse(psPMR);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+IMG_BOOL
+PMR_IsZombie(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return BITMASK_HAS(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_IS_ZOMBIE);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/* Function that alters the mutability property
+ * of the PMR
+ * Setting it to TRUE makes sure the PMR memory layout
+ * can't be changed through future calls */
+void
+PMR_SetLayoutFixed(PMR *psPMR, IMG_BOOL bFlag)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	if (bFlag)
+	{
+		BITMASK_SET(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+	}
+	else
+	{
+		BITMASK_UNSET(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+	}
+}
+
+IMG_BOOL PMR_IsMemLayoutFixed(PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return BITMASK_HAS(psPMR->uiInternalFlags, PMR_FLAG_INTERNAL_NO_LAYOUT_CHANGE);
+}
+
+IMG_DEVMEM_SIZE_T
+PMR_PhysicalSize(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	/* iLockCount will be > 0 for any backed PMR (backed on demand or not) */
+	if (OSAtomicRead(&psPMR->iLockCount) > 0)
+	{
+		if (_PMRIsSparse(psPMR))
+		{
+			return psPMR->psMappingTable->uiChunkSize * psPMR->psMappingTable->ui32NumPhysChunks;
+		}
+		else
+		{
+			return psPMR->uiLogicalSize;
+		}
+	}
+
+	return 0;
+}
+
+PVRSRV_ERROR
+PMR_IsOffsetValid(const PMR *psPMR,
+                  IMG_UINT32 ui32Log2PageSize,
+                  IMG_UINT32 ui32NumOfPages,
+                  IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                  IMG_BOOL *pbValid)
+{
+	IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 aui32BytesRemain[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset;
+	IMG_UINT32 *pui32BytesRemain = aui32BytesRemain;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psPMR != NULL);
+	PVR_ASSERT(psPMR->uiLogicalSize >= uiLogicalOffset);
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T));
+		PVR_GOTO_IF_NOMEM(puiPhysicalOffset, eError, e0);
+
+		pui32BytesRemain = OSAllocMem(ui32NumOfPages * sizeof(IMG_UINT32));
+		PVR_GOTO_IF_NOMEM(pui32BytesRemain, eError, e0);
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           ui32Log2PageSize,
+	                                           ui32NumOfPages,
+	                                           uiLogicalOffset,
+	                                           puiPhysicalOffset,
+	                                           pui32BytesRemain,
+	                                           pbValid);
+	PVR_LOG_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+e0:
+	if (puiPhysicalOffset != auiPhysicalOffset && puiPhysicalOffset != NULL)
+	{
+		OSFreeMem(puiPhysicalOffset);
+	}
+
+	if (pui32BytesRemain != aui32BytesRemain && pui32BytesRemain != NULL)
+	{
+		OSFreeMem(pui32BytesRemain);
+	}
+
+	return eError;
+}
+
+PHYS_HEAP *
+PMR_PhysHeap(const PMR *psPMR)
+{
+	return psPMR->psPhysHeap;
+}
+
+PMR_MAPPING_TABLE *
+PMR_GetMappingTable(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->psMappingTable;
+
+}
+
+IMG_UINT32
+PMR_GetLog2Contiguity(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->uiLog2ContiguityGuarantee;
+}
+
+IMG_DEVMEM_SIZE_T
+PMR_LogicalSize(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->uiLogicalSize;
+}
+
+IMG_UINT32 PMR_GetLogicalChunkCount(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+
+	return psPMR->psMappingTable->ui32NumLogicalChunks;
+}
+
+const IMG_CHAR *
+PMR_GetAnnotation(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->szAnnotation;
+}
+
+PMR_IMPL_TYPE
+PMR_GetType(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return psPMR->eFlavour;
+}
+
+IMG_CHAR *
+PMR_GetTypeStr(const PMR *psPMR)
+{
+	static IMG_CHAR *pszFlavour[] = {
+#define X(type) #type
+		PMR_IMPL_TYPES
+#undef X
+	};
+
+	if (psPMR->eFlavour >= PMR_TYPE_LAST)
+	{
+		return "INVALID";
+	}
+
+	return pszFlavour[psPMR->eFlavour];
+}
+
+IMG_INT32
+PMR_GetRefCount(const PMR *psPMR)
+{
+	PVR_ASSERT(psPMR != NULL);
+	return OSAtomicRead(&psPMR->iRefCount);
+}
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRGetIPAPolicy(PMR *psPMR,
+                IMG_UINT8 *pui8IPAPolicy)
+{
+	IMG_UINT32 ui32FlagsIPAPolicy;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui8IPAPolicy != NULL, "pui8IPAPolicy");
+	ui32FlagsIPAPolicy = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+
+	*pui8IPAPolicy = (IMG_UINT8)ui32FlagsIPAPolicy;
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING) */
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRGetIPAInfo(PMR *psPMR, IMG_UINT32 *pui32IPAPolicy, IMG_UINT32 *pui32IPAShift,
+              IMG_UINT32 *pui32IPAMask, IMG_UINT32 *pui32IPAFlagsValue)
+{
+	IMG_UINT32 ui32IPAFlagsPolicyValue;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAPolicy != NULL, "pui32IPAPolicy");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAShift != NULL, "pui32IPAShift");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAMask != NULL, "pui32IPAMask");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui32IPAFlagsValue != NULL, "pui32IPAFlagsValue");
+
+	/* Get the underlying heap-provided default IPA settings (if any) */
+	*pui32IPAShift = PhysHeapGetIPAShift(psPMR->psPhysHeap);
+	*pui32IPAPolicy = PhysHeapGetIPAValue(psPMR->psPhysHeap);
+	*pui32IPAMask = PhysHeapGetIPAMask(psPMR->psPhysHeap);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: Shift, Policy, Mask for Heap %p = %d, %d, %d", __func__,
+	        psPMR->psPhysHeap, *pui32IPAShift, *pui32IPAPolicy, *pui32IPAMask));
+
+	/* Query the current PMR flags settings for current IPA policy */
+	ui32IPAFlagsPolicyValue = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+	*pui32IPAFlagsValue = ui32IPAFlagsPolicyValue;
+	return PVRSRV_OK;
+}
+#endif	/* PVRSRV_INTERNAL_IPA_FEATURE_TESTING */
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRModifyIPAPolicy(PMR *psPMR,
+                   IMG_UINT8 ui8NewIPAPolicy)
+{
+	IMG_UINT64 ui64IPAFlagsPolicyValue;
+	IMG_UINT32 ui32IPAPolicyMaskValue;
+
+	ui32IPAPolicyMaskValue = (IMG_UINT32)(PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK >> PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET);
+
+	/* ui8NewIPAPolicy must be between 0 .. uiMaskValue inclusive */
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ui8NewIPAPolicy <= ui32IPAPolicyMaskValue,
+	                                "ui8NewIPAPolicy");
+
+	/* Set the appropriate policy bits in the PMR */
+	ui64IPAFlagsPolicyValue = ((IMG_UINT64)(ui8NewIPAPolicy) << PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET) &
+	                           PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+
+	psPMR->uiFlags &= ~PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK;
+	psPMR->uiFlags |= ui64IPAFlagsPolicyValue;
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: ui32IPAPolicy Mask = 0x%x, Value = 0x%x",
+	         __func__, ui32IPAPolicyMaskValue, (IMG_UINT32)ui8NewIPAPolicy));
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: uiFlags = 0x%016" IMG_UINT64_FMTSPECx ", ui64IPAFlags = 0x%016" IMG_UINT64_FMTSPECx, __func__, psPMR->uiFlags, ui64IPAFlagsPolicyValue));
+
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING) */
+
+/* must have called PMRLockSysPhysAddresses() before calling this! */
+PVRSRV_ERROR
+PMR_DevPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEV_PHYADDR *psDevAddrPtr,
+                IMG_BOOL *pbValid,
+                PMR_USAGE_TYPE ePMRUsage)
+{
+	IMG_UINT32 ui32Remain;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_OFFSET_T auiPhysicalOffset[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEVMEM_OFFSET_T *puiPhysicalOffset = auiPhysicalOffset;
+#if defined(SUPPORT_STATIC_IPA)
+	IMG_UINT32 ui32FlagsIPAPolicy;      /* Local value for the IPA policy */
+	IMG_UINT32 ui32IPAHeapShift;        /* Phys-heap bit-shift value */
+	IMG_UINT32 ui32IPAHeapPolicyValue;  /* Phys-heap default policy value */
+	IMG_UINT32 ui32IPAHeapClearMask;    /* Phys-heap ClearMask bitmask */
+	IMG_UINT64 ui64IPAPolicy;           /* IPAPolicy value to be applied to physical address(es) */
+	IMG_UINT64 ui64IPAClearMask;        /* IPAClearMask to be applied to physical address(es) */
+#else
+	PVR_UNREFERENCED_PARAMETER(ePMRUsage);
+#endif
+
+	PVR_ASSERT(psPMR != NULL);
+	PVR_ASSERT(ui32NumOfPages > 0);
+	PVR_ASSERT(psPMR->psFuncTab->pfnDevPhysAddr != NULL);
+
+	if (OSAtomicRead(&psPMR->iLockCount) <= 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,"Attempt to obtain DevPhysAddr of non-backed PMR"));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		puiPhysicalOffset = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEVMEM_OFFSET_T));
+		PVR_RETURN_IF_NOMEM(puiPhysicalOffset);
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           ui32Log2PageSize,
+	                                           ui32NumOfPages,
+	                                           uiLogicalOffset,
+	                                           puiPhysicalOffset,
+	                                           &ui32Remain,
+	                                           pbValid);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset", FreeOffsetArray);
+
+#if defined(SUPPORT_STATIC_IPA)
+	/* Need to determine the values to pass into the pfnDevPhysAddr
+	 * for Intermediate Physical Address settings associated with
+	 * this PMR.
+	 * If the ui32FlagsIPAPolicy value is non-zero, the value will be used in
+	 * preference to the default value specified in the physheap config.
+	 * Whichever value is used the associated physheap configuration bit shift and
+	 * mask values are passed to the pfnDevPhysAddr PMR factory function to modify
+	 * the returned address(es).
+	 */
+
+	PVR_ASSERT(psPMR->psPhysHeap != NULL);
+
+	ui32IPAHeapShift = PhysHeapGetIPAShift(psPMR->psPhysHeap);
+	ui32IPAHeapPolicyValue = PhysHeapGetIPAValue(psPMR->psPhysHeap);
+	ui32IPAHeapClearMask = PhysHeapGetIPAMask(psPMR->psPhysHeap);
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+	ui32FlagsIPAPolicy = PVRSRV_MEMALLOCFLAG_IPA_POLICY(psPMR->uiFlags);
+	if (ui32FlagsIPAPolicy == 0U)
+	{
+		ui32FlagsIPAPolicy = ui32IPAHeapPolicyValue;
+	}
+#else
+	ui32FlagsIPAPolicy = ui32IPAHeapPolicyValue;	/* Use heap default values*/
+#endif	/* PVRSRV_INTERNAL_IPA_FEATURE_TESTING */
+	/* To handle the 'disabling' of IPAPolicy setting for some callers we
+	 * check to see if the ePMRUsage is set to DEVICE_USE.
+	 * If so, we simply use the calculated shifts and policy values determined
+	 * above. If disabled (ePMRUsage == CPU_USE) we pass 0 values to the PMR
+	 * factory which will result in no IPA modification being made to the
+	 * phys_heap physical addresses.
+	 */
+	if (unlikely(ePMRUsage == CPU_USE))
+	{
+		ui32IPAHeapClearMask = 0U;
+		ui32FlagsIPAPolicy = 0U;
+	}
+	ui64IPAPolicy = (IMG_UINT64)ui32FlagsIPAPolicy << ui32IPAHeapShift;
+	ui64IPAClearMask = (IMG_UINT64)ui32IPAHeapClearMask << ui32IPAHeapShift;
+#endif	/* SUPPORT_STATIC_IPA */
+
+	/* Sparse PMR may not always have the first page valid */
+	eError = psPMR->psFuncTab->pfnDevPhysAddr(psPMR->pvFlavourData,
+											  ui32Log2PageSize,
+											  ui32NumOfPages,
+											  puiPhysicalOffset,
+#if defined(SUPPORT_STATIC_IPA)
+											  ui64IPAPolicy,
+											  ui64IPAClearMask,
+#endif
+											  pbValid,
+											  psDevAddrPtr);
+	PVR_GOTO_IF_ERROR(eError, FreeOffsetArray);
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+	/* Currently excluded from the default build because of performance
+	 * concerns.
+	 * We do not need this part in all systems because the GPU has the same
+	 * address view of system RAM as the CPU.
+	 * Alternatively this could be implemented as part of the PMR-factories
+	 * directly */
+	if (PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_UMA ||
+		PhysHeapGetType(psPMR->psPhysHeap) == PHYS_HEAP_TYPE_DMA)
+	{
+		IMG_UINT32 i;
+		IMG_DEV_PHYADDR sDevPAddrCorrected;
+
+		/* Copy the translated addresses to the correct array */
+		for (i = 0; i < ui32NumOfPages; i++)
+		{
+			PhysHeapCpuPAddrToDevPAddr(psPMR->psPhysHeap,
+									   1,
+									   &sDevPAddrCorrected,
+									   (IMG_CPU_PHYADDR *) &psDevAddrPtr[i]);
+			psDevAddrPtr[i].uiAddr = sDevPAddrCorrected.uiAddr;
+		}
+	}
+#endif
+
+
+FreeOffsetArray:
+	if (puiPhysicalOffset != auiPhysicalOffset)
+	{
+		OSFreeMem(puiPhysicalOffset);
+	}
+
+	return eError;
+}
+
+/* must have called PMRLockSysPhysAddresses() before calling this! */
+PVRSRV_ERROR
+PMR_CpuPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_CPU_PHYADDR *psCpuAddrPtr,
+                IMG_BOOL *pbValid)
+{
+	IMG_UINT32 idx;
+	PVRSRV_ERROR eError;
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *psDevPAddr = asDevPAddr;
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psDevPAddr = OSAllocMem(ui32NumOfPages * sizeof(IMG_DEV_PHYADDR));
+		PVR_GOTO_IF_NOMEM(psDevPAddr, eError, e0);
+	}
+
+	eError = PMR_DevPhysAddr(psPMR, ui32Log2PageSize, ui32NumOfPages,
+	                         uiLogicalOffset, psDevPAddr, pbValid, CPU_USE);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	if (_PMRIsSparse(psPMR))
+	{
+		/* Loop over each page.
+		 * If Dev addr valid, populate the CPU addr from the Dev addr
+		 */
+		for (idx = 0; idx < ui32NumOfPages; idx++)
+		{
+			if (pbValid[idx])
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPMR->psPhysHeap, 1, &psCpuAddrPtr[idx], &psDevPAddr[idx]);
+			}
+		}
+	}
+	else
+	{
+		/* In this case all addrs will be valid, so we can block translate */
+		PhysHeapDevPAddrToCpuPAddr(psPMR->psPhysHeap, ui32NumOfPages, psCpuAddrPtr, psDevPAddr);
+	}
+
+	if (ui32NumOfPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+
+	return PVRSRV_OK;
+e1:
+	if (psDevPAddr != asDevPAddr)
+	{
+		OSFreeMem(psDevPAddr);
+	}
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PMR_ChangeSparseMem(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+
+	PMRLockPMR(psPMR);
+
+	eError = PMR_ChangeSparseMemUnlocked(psPMR,
+	                                     ui32AllocPageCount,
+	                                     pai32AllocIndices,
+	                                     ui32FreePageCount,
+	                                     pai32FreeIndices,
+	                                     uiSparseFlags);
+
+	PMRUnlockPMR(psPMR);
+
+	return eError;
+}
+
+PVRSRV_ERROR PMR_ChangeSparseMemUnlocked(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	PMR_IMPL_ZOMBIEPAGES pvZombiePages = NULL;
+	PMR_ZOMBIE_PAGES* psPMRZombiePages = NULL;
+#endif
+
+	if (PMR_IsMemLayoutFixed(psPMR) || PMR_IsCpuMapped(psPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This PMR layout cannot be changed - PMR_IsMemLayoutFixed()=%c, _PMR_IsMapped()=%c",
+				__func__,
+				PMR_IsMemLayoutFixed(psPMR) ? 'Y' : 'n',
+				PMR_IsCpuMapped(psPMR) ? 'Y' : 'n'));
+		return PVRSRV_ERROR_PMR_NOT_PERMITTED;
+	}
+
+	if (NULL == psPMR->psFuncTab->pfnChangeSparseMem)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: This type of sparse PMR cannot be changed.",
+				__func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		/* Speculatively preallocate in order to simplify error handling later */
+		psPMRZombiePages = OSAllocZMem(sizeof(PMR_ZOMBIE_PAGES));
+		PVR_GOTO_IF_NOMEM(psPMRZombiePages, eError, e0);
+	}
+#endif
+
+	eError = psPMR->psFuncTab->pfnChangeSparseMem(psPMR->pvFlavourData,
+	                                              psPMR,
+	                                              ui32AllocPageCount,
+	                                              pai32AllocIndices,
+	                                              ui32FreePageCount,
+	                                              pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	                                              &pvZombiePages,
+#endif
+	                                              uiSparseFlags);
+	if (eError != PVRSRV_OK)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		if (eError == PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES)
+		{
+			PVRSRVStatsUpdateOOMStat(NULL,
+									  PMR_DeviceNode(psPMR),
+									  PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT,
+									  OSGetCurrentClientProcessIDKM());
+		}
+#endif
+		goto e1;
+	}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (pvZombiePages != NULL)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+		PVRSRV_DEVICE_NODE *psDevNode;
+
+		psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+		eError = PVRSRVGetDevicePowerState(psDevNode, &ePowerState);
+		if (eError != PVRSRV_OK)
+		{
+			/* Treat unknown power state as ON. */
+			ePowerState = PVRSRV_DEV_POWER_STATE_ON;
+		}
+
+		if (ePowerState == PVRSRV_DEV_POWER_STATE_OFF ||
+			psDevNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+		{
+			/* Free preallocated psPMRZombiePages as these won't be used*/
+			OSFreeMem(psPMRZombiePages);
+
+			eError = psPMR->psFuncTab->pfnFreeZombiePages(pvZombiePages);
+			PVR_LOG_GOTO_IF_ERROR(eError, "Error when trying to free zombies immediately.", e0);
+		}
+		else
+		{
+			PVR_ASSERT(psPMRZombiePages != NULL);
+			psPMRZombiePages->sHeader.bIsZombiePages = IMG_TRUE;
+			psPMRZombiePages->pfnFactoryFreeZombies = psPMR->psFuncTab->pfnFreeZombiePages;
+			psPMRZombiePages->pvFactoryPages = pvZombiePages;
+
+			OSLockAcquire(psDevNode->hPMRZombieListLock);
+			dllist_add_to_tail(&psDevNode->sPMRZombieList, &psPMRZombiePages->sHeader.sZombieNode);
+			psDevNode->uiPMRZombieCount++;
+			OSLockRelease(psDevNode->hPMRZombieListLock);
+		}
+	}
+	else
+	{
+		/* Free psPMRZombiePages as change sparse has not produced zombie pages */
+		OSFreeMem(psPMRZombiePages);
+	}
+#endif
+
+#if defined(PDUMP)
+	{
+		IMG_BOOL bInitialise = IMG_FALSE;
+		IMG_UINT8 ui8InitValue = 0;
+
+		if (PVRSRV_CHECK_ZERO_ON_ALLOC(PMR_Flags(psPMR)))
+		{
+			bInitialise = IMG_TRUE;
+		}
+		else if (PVRSRV_CHECK_POISON_ON_ALLOC(PMR_Flags(psPMR)))
+		{
+			ui8InitValue = (IMG_UINT8)PVRSRV_POISON_ON_ALLOC_VALUE;
+			bInitialise = IMG_TRUE;
+		}
+
+		PDumpPMRChangeSparsePMR(psPMR,
+		                        IMG_PAGE2BYTES32(psPMR->uiLog2ContiguityGuarantee),
+		                        ui32AllocPageCount,
+		                        pai32AllocIndices,
+		                        ui32FreePageCount,
+		                        pai32FreeIndices,
+		                        bInitialise,
+		                        ui8InitValue,
+		                        &psPMR->hPDumpAllocHandle);
+	}
+
+#endif
+
+	return PVRSRV_OK;
+e1:
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+	if (uiSparseFlags & SPARSE_RESIZE_FREE)
+	{
+		OSFreeMem(psPMRZombiePages);
+	}
+e0:
+#endif
+	return eError;
+}
+
+#if defined(PDUMP)
+
+static PVRSRV_ERROR
+_PMR_PDumpSymbolicAddrPhysical(const PMR *psPMR,
+                               IMG_DEVMEM_OFFSET_T uiPhysicalOffset,
+                               IMG_UINT32 ui32MemspaceNameLen,
+                               IMG_CHAR *pszMemspaceName,
+                               IMG_UINT32 ui32SymbolicAddrLen,
+                               IMG_CHAR *pszSymbolicAddr,
+                               IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                               IMG_DEVMEM_OFFSET_T *puiNextSymName)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPMR->psPhysHeap);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (PVRSRV_CHECK_PHYS_HEAP(FW_CODE, psPMR->uiFlags) ||
+		PVRSRV_CHECK_PHYS_HEAP(FW_PRIV_DATA, psPMR->uiFlags) ||
+	    PVRSRV_CHECK_PHYS_HEAP(GPU_SECURE, psPMR->uiFlags))
+	{
+		OSSNPrintf(pszMemspaceName, ui32MemspaceNameLen, PMR_MEMSPACE_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+	else
+#endif
+	if (DevmemDeviceCacheCoherency(psDevNode, psPMR->uiFlags))
+	{
+		OSSNPrintf(pszMemspaceName,
+		           ui32MemspaceNameLen,
+		           PMR_MEMSPACE_CACHE_COHERENT_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+	else
+	{
+		OSSNPrintf(pszMemspaceName, ui32MemspaceNameLen, PMR_MEMSPACE_FMTSPEC,
+		           psPMR->pszPDumpDefaultMemspaceName);
+	}
+
+	OSSNPrintf(pszSymbolicAddr,
+	           ui32SymbolicAddrLen,
+	           PMR_SYMBOLICADDR_FMTSPEC,
+	           PMR_IsSparse(psPMR) ? PMR_SPARSE_PREFIX : PMR_DEFAULT_PREFIX,
+	           psPMR->uiSerialNum,
+	           uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR),
+	           psPMR->szAnnotation);
+
+	if (pszSymbolicAddr)
+	{
+		PDumpMakeStringValid(pszSymbolicAddr, OSStringLength(pszSymbolicAddr));
+	}
+
+
+	*puiNewOffset = uiPhysicalOffset & (IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR))-1);
+	*puiNextSymName = (IMG_DEVMEM_OFFSET_T) (((uiPhysicalOffset >> PMR_GetLog2Contiguity(psPMR))+1)
+			<< PMR_GetLog2Contiguity(psPMR));
+
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32MemspaceNameLen,
+                      IMG_CHAR *pszMemspaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                      IMG_DEVMEM_OFFSET_T *puiNextSymName
+)
+{
+	IMG_DEVMEM_OFFSET_T uiPhysicalOffset;
+	IMG_UINT32 ui32Remain;
+	IMG_BOOL bValid;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(uiLogicalOffset < psPMR->uiLogicalSize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = _PMRLogicalOffsetToPhysicalOffset(psPMR,
+	                                           0,
+	                                           1,
+	                                           uiLogicalOffset,
+	                                           &uiPhysicalOffset,
+	                                           &ui32Remain,
+	                                           &bValid);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_PMRLogicalOffsetToPhysicalOffset");
+
+	if (!bValid)
+	{
+		/* For sparse allocations, for a given logical address, there
+		 * may not be a physical memory backing, the virtual range can
+		 * still be valid.
+		 */
+		uiPhysicalOffset = uiLogicalOffset;
+	}
+
+	return _PMR_PDumpSymbolicAddrPhysical(psPMR,
+	                                      uiPhysicalOffset,
+	                                      ui32MemspaceNameLen,
+	                                      pszMemspaceName,
+	                                      ui32SymbolicAddrLen,
+	                                      pszSymbolicAddr,
+	                                      puiNewOffset,
+	                                      puiNextSymName);
+}
+
+/*!
+ * @brief Writes a WRW command to the script2 buffer, representing a
+ *        dword write to a physical allocation. Size is always
+ *        sizeof(IMG_UINT32).
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - offset
+ * @param ui32Value - value to write
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                       IMG_UINT32 ui32Value,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + sizeof(ui32Value) <= psPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+			<= uiPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the PMR */
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW32(PMR_DeviceNode(psPMR),
+	                       aszMemspaceName,
+	                       aszSymbolicName,
+	                       uiPDumpSymbolicOffset,
+	                       ui32Value,
+	                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a RDW followed by a WRW command to the pdump script to perform
+ *        an effective copy from memory to memory. Memory copied is of size
+ *        sizeof(IMG_UINT32)
+ *
+ * @param psDstPMR - PMR object representing allocation of destination
+ * @param uiDstLogicalOffset - destination offset
+ * @param psSrcPMR - PMR object representing allocation of source
+ * @param uiSrcLogicalOffset - source offset
+ * @param pszTmpVar - pdump temporary variable used during the copy
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem32(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	const IMG_UINT32 uiDstPMRPageSize = 1 << psDstPMR->uiLog2ContiguityGuarantee;
+	const IMG_UINT32 uiSrcPMRPageSize = 1 << psSrcPMR->uiLog2ContiguityGuarantee;
+
+	PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiSrcLogicalOffset & (uiSrcPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= uiSrcPMRPageSize));
+
+	PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiDstLogicalOffset & (uiDstPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= uiDstPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the source PMR */
+	eError = PMR_PDumpSymbolicAddr(psSrcPMR,
+	                               uiSrcLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Issue PDump read command */
+	eError = PDumpPMRRDW32MemToInternalVar(PMR_DeviceNode(psSrcPMR),
+	                                       pszTmpVar,
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+
+	eError = PMRLockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Get the symbolic address of the destination PMR */
+	eError = PMR_PDumpSymbolicAddr(psDstPMR,
+	                               uiDstLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW32InternalVarToMem(PMR_DeviceNode(psDstPMR),
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       pszTmpVar,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	eError = PMRUnlockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a WRW64 command to the script2 buffer, representing a
+ *        dword write to a physical allocation. Size is always
+ *        sizeof(IMG_UINT64).
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - offset
+ * @param ui64Value - value to write
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+                       IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                       IMG_UINT64 ui64Value,
+                       PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	PMR_SIZE_T ui64PMRPageSize = IMG_PAGE2BYTES64(psPMR->uiLog2ContiguityGuarantee);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + sizeof(ui64Value) <= psPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (ui64PMRPageSize-1)) + sizeof(ui64Value))
+			<= ui64PMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the PMR */
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW64(PMR_DeviceNode(psPMR),
+	                       aszMemspaceName,
+	                       aszSymbolicName,
+	                       uiPDumpSymbolicOffset,
+	                       ui64Value,
+	                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief Writes a RDW64 followed by a WRW64 command to the pdump script to
+ *        perform an effective copy from memory to memory. Memory copied is of
+ *        size sizeof(IMG_UINT32)
+ *
+ * @param psDstPMR - PMR object representing allocation of destination
+ * @param uiDstLogicalOffset - destination offset
+ * @param psSrcPMR - PMR object representing allocation of source
+ * @param uiSrcLogicalOffset - source offset
+ * @param pszTmpVar - pdump temporary variable used during the copy
+ * @param uiPDumpFlags - pdump flags
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem64(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpSymbolicOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	const PMR_SIZE_T ui64DstPMRPageSize = IMG_PAGE2BYTES64(psDstPMR->uiLog2ContiguityGuarantee);
+	const PMR_SIZE_T ui64SrcPMRPageSize = IMG_PAGE2BYTES64(psSrcPMR->uiLog2ContiguityGuarantee);
+
+	PVR_ASSERT(uiSrcLogicalOffset + sizeof(IMG_UINT32) <= psSrcPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiSrcLogicalOffset & (ui64SrcPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= ui64SrcPMRPageSize));
+
+	PVR_ASSERT(uiDstLogicalOffset + sizeof(IMG_UINT32) <= psDstPMR->uiLogicalSize);
+	/* Especially make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiDstLogicalOffset & (ui64DstPMRPageSize-1)) + sizeof(IMG_UINT32))
+			<= ui64DstPMRPageSize));
+
+	eError = PMRLockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Get the symbolic address of the source PMR */
+	eError = PMR_PDumpSymbolicAddr(psSrcPMR,
+	                               uiSrcLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Issue PDump read command */
+	eError = PDumpPMRRDW64MemToInternalVar(PMR_DeviceNode(psSrcPMR),
+	                                       pszTmpVar,
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	eError = PMRUnlockSysPhysAddresses(psSrcPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+
+	eError = PMRLockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Get the symbolic address of the destination PMR */
+	eError = PMR_PDumpSymbolicAddr(psDstPMR,
+	                               uiDstLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpSymbolicOffset,
+	                               &uiNextSymName);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	/* Write the WRW script command */
+	eError = PDumpPMRWRW64InternalVarToMem(PMR_DeviceNode(psDstPMR),
+	                                       aszMemspaceName,
+	                                       aszSymbolicName,
+	                                       uiPDumpSymbolicOffset,
+	                                       pszTmpVar,
+	                                       uiPDumpFlags);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+
+	eError = PMRUnlockSysPhysAddresses(psDstPMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ * @brief PDumps the contents of the given allocation.
+ * If bZero is IMG_TRUE then the zero page in the parameter stream is used
+ * as the source of data, rather than the allocation's actual backing.
+ * @param psPMR - PMR object representing allocation
+ * @param uiLogicalOffset - Offset to write at
+ * @param uiSize - Number of bytes to write
+ * @param uiPDumpFlags - PDump flags
+ * @param bZero - Use the PDump zero page as the source
+ * @return PVRSRV_ERROR
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOutOffset;
+	IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName = 0;
+	const IMG_CHAR *pszParamStreamFileName;
+	PDUMP_FILEOFFSET_T uiParamStreamFileOffset;
+
+	/* required when !bZero */
+#define PMR_MAX_PDUMP_BUFSZ (1<<21)
+	IMG_CHAR aszParamStreamFilename[PDUMP_PARAM_MAX_FILE_NAME];
+	IMG_UINT8 *pcBuffer = NULL;
+	size_t ui64BufSz;
+	IMG_BOOL bValid;
+	IMG_DEVMEM_SIZE_T uiSizeRemain = uiSize;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(psDevNode))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Forcibly initialise the name to a 'NULL' 0-length string */
+	aszParamStreamFilename[0] = '\0';
+
+	PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize);
+
+	/* Check if pdump client is connected */
+	if (!PDumpCheckFlagsWrite(psDevNode,
+		                      PDUMP_FLAGS_CONTINUOUS))
+	{
+		/* Dumping of memory in Pdump buffer will be rejected for no client connected case.
+		 * So return early and save reading of data from PMR. */
+		return PVRSRV_OK;
+	}
+
+	/* Get the correct PDump stream file name */
+	if (bZero)
+	{
+		PDumpCommentWithFlags(psDevNode,
+		                      uiPDumpFlags,
+		                      "Zeroing allocation (" IMG_DEVMEM_SIZE_FMTSPEC " bytes)",
+		                      uiSize);
+
+		/* get the zero page information. it is constant for this function */
+		PDumpGetParameterZeroPageInfo(&uiParamStreamFileOffset,
+		                              &ui64BufSz,
+		                              &pszParamStreamFileName);
+	}
+	else
+	{
+
+		ui64BufSz = IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR));
+		PVR_ASSERT(IMG_PAGE2BYTES64(PMR_GetLog2Contiguity(psPMR)) <= PMR_MAX_PDUMP_BUFSZ);
+
+		pcBuffer = OSAllocMem(ui64BufSz);
+
+		PVR_LOG_RETURN_IF_NOMEM(pcBuffer, "OSAllocMem");
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		pszParamStreamFileName = aszParamStreamFilename;
+	}
+
+	/* Loop over all touched symbolic addresses of the PMR and
+	 * emit LDBs to load the contents. */
+	while (uiCurrentOffset < (uiLogicalOffset + uiSize))
+	{
+		/* Get the correct symbolic name for the current offset */
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiCurrentOffset,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOutOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		PVR_ASSERT((uiNextSymName - uiCurrentOffset) <= ui64BufSz);
+
+		PMR_IsOffsetValid(psPMR,
+		                  0,
+		                  1,
+		                  uiCurrentOffset,
+		                  &bValid);
+
+		/* Either just LDB the zeros or read from the PMR and store that
+		 * in the pdump stream */
+		if (bValid)
+		{
+			size_t uiNumBytes;
+			IMG_BOOL bOk2Write = IMG_TRUE;
+
+			if (bZero)
+			{
+				uiNumBytes = MIN(uiSizeRemain, uiNextSymName - uiCurrentOffset);
+			}
+			else
+			{
+				IMG_DEVMEM_OFFSET_T uiReadOffset;
+				uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ?
+						uiLogicalOffset + uiSize - uiCurrentOffset :
+						uiNextSymName - uiCurrentOffset);
+
+				eError = PMR_ReadBytes(psPMR,
+				                       uiCurrentOffset,
+				                       pcBuffer,
+				                       uiReadOffset,
+				                       &uiNumBytes);
+				PVR_ASSERT(eError == PVRSRV_OK);
+
+				eError = PDumpWriteParameterBlob(psDevNode,
+				                          pcBuffer,
+				                          uiNumBytes,
+				                          uiPDumpFlags,
+				                          &aszParamStreamFilename[0],
+				                          sizeof(aszParamStreamFilename),
+				                          &uiParamStreamFileOffset);
+
+				if (eError == PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+				{
+					/* Write to parameter file prevented under the flags and
+					 * current state of the driver so skip further writes.
+					 */
+					eError = PVRSRV_OK;
+					bOk2Write = IMG_FALSE;	/* Do *NOT* write anything */
+				}
+				else if (eError != PVRSRV_OK)
+				{
+					PDUMP_ERROR(psDevNode,
+					            eError, "Failed to write PMR memory to parameter file");
+					bOk2Write = IMG_FALSE;	/* Do *NOT* write anything */
+				}
+			}
+
+			if (bOk2Write)
+			{
+				/* Emit the LDB command to the current symbolic address */
+				eError = PDumpPMRLDB(psDevNode,
+				                     aszMemspaceName,
+				                     aszSymbolicName,
+				                     uiOutOffset,
+				                     uiNumBytes,
+				                     pszParamStreamFileName,
+				                     uiParamStreamFileOffset,
+				                     uiPDumpFlags);
+			}
+			uiSizeRemain = uiSizeRemain - uiNumBytes;
+		}
+		uiCurrentOffset = uiNextSymName;
+	}
+
+	if (!bZero)
+	{
+		eError = PMRUnlockSysPhysAddresses(psPMR);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		OSFreeMem(pcBuffer);
+	}
+
+	return PVRSRV_OK;
+}
+
+
+
+PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOutOffset;
+	IMG_DEVMEM_OFFSET_T uiCurrentOffset = uiLogicalOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName = 0;
+	IMG_UINT32 uiCurrentFileOffset = uiFileOffset;
+
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_ASSERT(uiLogicalOffset + uiSize <= psPMR->uiLogicalSize);
+
+	while (uiCurrentOffset < (uiLogicalOffset + uiSize))
+	{
+		IMG_DEVMEM_OFFSET_T uiReadOffset;
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiCurrentOffset,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOutOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		PVR_ASSERT(uiNextSymName <= psPMR->uiLogicalSize);
+
+		uiReadOffset = ((uiNextSymName > (uiLogicalOffset + uiSize)) ?
+				uiLogicalOffset + uiSize - uiCurrentOffset :
+				uiNextSymName - uiCurrentOffset);
+
+		eError = PDumpPMRSAB(PMR_DeviceNode(psPMR),
+		                     aszMemspaceName,
+		                     aszSymbolicName,
+		                     uiOutOffset,
+		                     uiReadOffset,
+		                     pszFilename,
+		                     uiCurrentFileOffset);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		uiCurrentFileOffset += uiNextSymName - uiCurrentOffset;
+		uiCurrentOffset = uiNextSymName;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+			<= uiPMRPageSize));
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+#define _MEMPOLL_DELAY		(1000)
+#define _MEMPOLL_COUNT		(2000000000 / _MEMPOLL_DELAY)
+
+	eError = PDumpPMRPOL(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     ui32Value,
+	                     ui32Mask,
+	                     eOperator,
+	                     _MEMPOLL_COUNT,
+	                     _MEMPOLL_DELAY,
+	                     uiPDumpFlags);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiPMRPageSize = 1 << psPMR->uiLog2ContiguityGuarantee;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Make sure to not cross a block boundary */
+	PVR_ASSERT(( ((uiLogicalOffset & (uiPMRPageSize-1)) + sizeof(ui32Value))
+	           < uiPMRPageSize));
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiLogicalOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = PDumpPMRPOL(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     ui32Value,
+	                     ui32Mask,
+	                     eOperator,
+	                     1,
+	                     1,
+	                     uiPDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+
+	/* Confirm that the device node's ui32InternalID matches the bound
+	 * PDump device stored* in PVRSRV_DATA.
+	 */
+	if (!PDumpIsDevicePermitted(PMR_DeviceNode(psPMR)))
+	{
+		return PVRSRV_OK;
+	}
+
+	eError = PMR_PDumpSymbolicAddr(psPMR,
+	                               uiReadOffset,
+	                               sizeof(aszMemspaceName),
+	                               &aszMemspaceName[0],
+	                               sizeof(aszSymbolicName),
+	                               &aszSymbolicName[0],
+	                               &uiPDumpOffset,
+	                               &uiNextSymName);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	eError = PDumpPMRCBP(PMR_DeviceNode(psPMR),
+	                     aszMemspaceName,
+	                     aszSymbolicName,
+	                     uiPDumpOffset,
+	                     uiWriteOffset,
+	                     uiPacketSize,
+	                     uiBufferSize);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+e0:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void
+PDumpPMRChangeSparsePMR(PMR *psPMR,
+                        IMG_UINT32 uiBlockSize,
+                        IMG_UINT32 ui32AllocPageCount,
+                        IMG_UINT32 *pai32AllocIndices,
+                        IMG_UINT32 ui32FreePageCount,
+                        IMG_UINT32 *pai32FreeIndices,
+                        IMG_BOOL bInitialise,
+                        IMG_UINT8 ui8InitValue,
+                        IMG_HANDLE *phPDumpAllocInfoOut)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE *phPDumpAllocInfo = (IMG_HANDLE*) psPMR->hPDumpAllocHandle;
+
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 i, uiIndex;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+
+	/* Remove pages from the PMR */
+	for (i = 0; i < ui32FreePageCount; i++)
+	{
+		uiIndex = pai32FreeIndices[i];
+
+		eError = PDumpFree(psDevNode,
+		                   phPDumpAllocInfo[uiIndex]);
+		PVR_ASSERT(eError == PVRSRV_OK);
+		phPDumpAllocInfo[uiIndex] = NULL;
+	}
+
+	/* Add new pages to the PMR */
+	for (i = 0; i < ui32AllocPageCount; i++)
+	{
+		uiIndex = pai32AllocIndices[i];
+
+		PVR_ASSERT(phPDumpAllocInfo[uiIndex] == NULL);
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiIndex * uiBlockSize,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpMalloc(psDevNode,
+		                     aszMemspaceName,
+		                     aszSymbolicName,
+		                     uiBlockSize,
+		                     uiBlockSize,
+		                     bInitialise,
+		                     ui8InitValue,
+		                     &phPDumpAllocInfo[uiIndex],
+		                     PDUMP_NONE);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* (IMG_HANDLE) <- (IMG_HANDLE*) */
+	*phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo;
+}
+
+static void
+PDumpPMRFreePMR(PMR *psPMR,
+                IMG_DEVMEM_SIZE_T uiSize,
+                IMG_DEVMEM_ALIGN_T uiBlockSize,
+                IMG_UINT32 uiLog2Contiguity,
+                IMG_HANDLE hPDumpAllocationInfoHandle)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	/* (IMG_HANDLE*) <- (IMG_HANDLE) */
+	IMG_HANDLE *ahPDumpAllocHandleArray = (IMG_HANDLE*) hPDumpAllocationInfoHandle;
+
+	PDUMP_LOCK(PDUMP_FLAGS_NONE);
+
+	for (i = 0; i < psPMR->uiNumPDumpBlocks; i++)
+	{
+		if (ahPDumpAllocHandleArray[i] != NULL)
+		{
+			eError = PDumpFreeUnlocked(PMR_DeviceNode(psPMR),
+			                           ahPDumpAllocHandleArray[i]);
+			PVR_ASSERT(eError == PVRSRV_OK);
+			ahPDumpAllocHandleArray[i] = NULL;
+		}
+	}
+
+	PDUMP_UNLOCK(PDUMP_FLAGS_NONE);
+	OSFreeMem(ahPDumpAllocHandleArray);
+}
+
+static void
+PDumpPMRMallocPMR(PMR *psPMR,
+                  IMG_DEVMEM_SIZE_T uiSize,
+                  IMG_UINT32 ui32ChunkSize,
+                  IMG_UINT32 ui32NumPhysChunks,
+                  IMG_UINT32 ui32NumLogicalChunks,
+                  IMG_UINT32 *puiMappingTable,
+                  IMG_UINT32 uiLog2Contiguity,
+                  IMG_BOOL bInitialise,
+                  IMG_UINT8 ui8InitValue,
+                  IMG_HANDLE *phPDumpAllocInfoOut,
+                  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE *phPDumpAllocInfo;
+
+	IMG_CHAR aszMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+	IMG_UINT32 uiNumPhysBlocks;
+	IMG_UINT32 uiNumVirtBlocks;
+	IMG_UINT32 i, uiIndex;
+
+	if (PMR_IsSparse(psPMR))
+	{
+		uiNumPhysBlocks = (ui32ChunkSize * ui32NumPhysChunks) >> uiLog2Contiguity;
+		/* Make sure we did not cut off anything */
+		PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == (ui32ChunkSize * ui32NumPhysChunks));
+	}
+	else
+	{
+		uiNumPhysBlocks = uiSize >> uiLog2Contiguity;
+		/* Make sure we did not cut off anything */
+		PVR_ASSERT(uiNumPhysBlocks << uiLog2Contiguity == uiSize);
+	}
+
+	uiNumVirtBlocks = uiSize >> uiLog2Contiguity;
+	PVR_ASSERT(uiNumVirtBlocks << uiLog2Contiguity == uiSize);
+
+	psPMR->uiNumPDumpBlocks = uiNumVirtBlocks;
+
+	phPDumpAllocInfo = (IMG_HANDLE*) OSAllocZMem(uiNumVirtBlocks * sizeof(IMG_HANDLE));
+
+	PDUMP_LOCK(ui32PDumpFlags);
+
+	for (i = 0; i < uiNumPhysBlocks; i++)
+	{
+		uiIndex = PMR_IsSparse(psPMR) ? puiMappingTable[i] : i;
+
+		eError = PMR_PDumpSymbolicAddr(psPMR,
+		                               uiIndex * ui32ChunkSize,
+		                               sizeof(aszMemspaceName),
+		                               &aszMemspaceName[0],
+		                               sizeof(aszSymbolicName),
+		                               &aszSymbolicName[0],
+		                               &uiOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpMallocUnlocked(PMR_DeviceNode(psPMR),
+		                             aszMemspaceName,
+		                             aszSymbolicName,
+		                             ui32ChunkSize,
+		                             ui32ChunkSize,
+		                             bInitialise,
+		                             ui8InitValue,
+		                             &phPDumpAllocInfo[uiIndex],
+		                             ui32PDumpFlags);
+		PVR_LOG_RETURN_VOID_IF_FALSE((eError != PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE),
+		                             "PDumpPMRMalloc PDump capture bound to other device");
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	PDUMP_UNLOCK(ui32PDumpFlags);
+
+	/* (IMG_HANDLE) <- (IMG_HANDLE*) */
+	*phPDumpAllocInfoOut = (IMG_HANDLE) phPDumpAllocInfo;
+
+}
+#endif	/* PDUMP */
+
+
+void *PMRGetPrivateData(const PMR *psPMR,
+                        const PMR_IMPL_FUNCTAB *psFuncTab)
+{
+	return (psFuncTab == psPMR->psFuncTab) ? psPMR->pvFlavourData : NULL;
+}
+
+#define PMR_PM_WORD_SIZE 4
+
+PVRSRV_ERROR
+PMRWritePMPageList(/* Target PMR, offset, and length */
+		PMR *psPageListPMR,
+		IMG_DEVMEM_OFFSET_T uiTableOffset,
+		IMG_DEVMEM_SIZE_T  uiTableLength,
+		/* Referenced PMR, and "page" granularity */
+		PMR *psReferencePMR,
+		IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+		PMR_PAGELIST **ppsPageList)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEVMEM_SIZE_T uiWordSize;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	PMR_FLAGS_T uiFlags = psPageListPMR->uiFlags;
+	PMR_PAGELIST *psPageList;
+#if defined(PDUMP)
+	IMG_CHAR aszTableEntryMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszTableEntrySymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiTableEntryPDumpOffset;
+	IMG_CHAR aszPageMemspaceName[PHYSMEM_PDUMP_MEMSPACE_MAX_LENGTH];
+	IMG_CHAR aszPageSymbolicName[PHYSMEM_PDUMP_SYMNAME_MAX_LENGTH];
+	IMG_DEVMEM_OFFSET_T uiPagePDumpOffset;
+	IMG_DEVMEM_OFFSET_T uiNextSymName;
+#endif
+#if !defined(NO_HARDWARE)
+	PMR_SIZE_T ui64PageListPageSize = IMG_PAGE2BYTES64(psPageListPMR->uiLog2ContiguityGuarantee);
+	IMG_UINT64 uiPageListPMRPage = 0;
+	IMG_UINT64 uiPrevPageListPMRPage = 0;
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+	IMG_UINT32 *pui32DataPtr = NULL;
+	IMG_DEV_PHYADDR asDevPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_DEV_PHYADDR *pasDevAddrPtr;
+	IMG_BOOL *pbPageIsValid;
+#endif
+	PVRSRV_DEVICE_NODE *psDevNode = PhysHeapDeviceNode(psPageListPMR->psPhysHeap);
+	IMG_BOOL bCPUCacheSnoop =
+		(PVRSRVSystemSnoopingOfCPUCache(psDevNode->psDevConfig) &&
+		 psDevNode->pfnGetDeviceSnoopMode(psDevNode) == PVRSRV_DEVICE_SNOOP_CPU_ONLY);
+
+	uiWordSize = PMR_PM_WORD_SIZE;
+
+	/* check we're being asked to write the same number of 4-byte units as there are pages */
+	uiNumPages = (IMG_UINT32)(psReferencePMR->uiLogicalSize >> uiLog2PageSize);
+
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psReferencePMR->uiLogicalSize)
+	{
+		/* Strictly speaking, it's possible to provoke this error in two ways:
+			(i) if it's not a whole multiple of the page size; or
+			(ii) if there are more than 4 billion pages.
+			The latter is unlikely. :) but the check is required in order to justify the cast.
+		 */
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, return_error);
+	}
+	uiWordSize = (IMG_UINT32)uiTableLength / uiNumPages;
+	if (uiNumPages * uiWordSize != uiTableLength)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, return_error);
+	}
+
+	/* Check for integer overflow */
+	PVR_GOTO_IF_INVALID_PARAM(uiTableOffset + uiTableLength > uiTableOffset, eError, return_error);
+	/* Check we're not being asked to write off the end of the PMR */
+	PVR_GOTO_IF_INVALID_PARAM(uiTableOffset + uiTableLength <= psPageListPMR->uiLogicalSize, eError, return_error);
+
+	/* the PMR into which we are writing must not be user CPU mappable: */
+	if (PVRSRV_CHECK_CPU_READABLE(uiFlags) || PVRSRV_CHECK_CPU_WRITEABLE(uiFlags))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Masked flags = 0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         (PMR_FLAGS_T)(uiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE | PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE))));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Page list PMR allows CPU mapping (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	/* the PMR into which we are writing must not be user CPU cacheable: */
+	if (!bCPUCacheSnoop &&
+		(PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags) ||
+		 PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags) ||
+		 PVRSRV_CHECK_CPU_CACHED(uiFlags)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Masked flags = 0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC,
+		         (PMR_FLAGS_T)(uiFlags &  PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Page list PMR allows CPU caching (0x%" PVRSRV_MEMALLOCFLAGS_FMTSPEC ")",
+		         uiFlags));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS, return_error);
+	}
+
+	if (_PMRIsSparse(psPageListPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPageListPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	if (_PMRIsSparse(psReferencePMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psReferencePMR", eError, PVRSRV_ERROR_INVALID_PARAMS, return_error);
+	}
+
+	psPageList = OSAllocMem(sizeof(PMR_PAGELIST));
+	PVR_LOG_GOTO_IF_NOMEM(psPageList, eError, return_error);
+
+	psPageList->psReferencePMR = psReferencePMR;
+
+	/* Need to lock down the physical addresses of the reference PMR */
+	/* N.B.  This also checks that the requested "contiguity" is achievable */
+	eError = PMRLockSysPhysAddresses(psReferencePMR);
+	PVR_GOTO_IF_ERROR(eError, free_page_list);
+
+#if !defined(NO_HARDWARE)
+	if (uiNumPages > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		pasDevAddrPtr = OSAllocMem(uiNumPages * sizeof(IMG_DEV_PHYADDR));
+		PVR_LOG_GOTO_IF_NOMEM(pasDevAddrPtr, eError, unlock_phys_addrs);
+
+		pbPageIsValid = OSAllocMem(uiNumPages * sizeof(IMG_BOOL));
+		PVR_LOG_GOTO_IF_NOMEM(pbPageIsValid, eError, free_devaddr_array);
+	}
+	else
+	{
+		pasDevAddrPtr = asDevPAddr;
+		pbPageIsValid = abValid;
+	}
+
+	eError = PMR_DevPhysAddr(psReferencePMR, uiLog2PageSize, uiNumPages, 0,
+	                         pasDevAddrPtr, pbPageIsValid, DEVICE_USE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PMR_DevPhysAddr", free_valid_array);
+#endif
+
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		IMG_DEVMEM_OFFSET_T uiPMROffset = uiTableOffset + (uiWordSize * uiPageIndex);
+
+#if defined(PDUMP)
+		eError = PMR_PDumpSymbolicAddr(psPageListPMR,
+		                               uiPMROffset,
+		                               sizeof(aszTableEntryMemspaceName),
+		                               &aszTableEntryMemspaceName[0],
+		                               sizeof(aszTableEntrySymbolicName),
+		                               &aszTableEntrySymbolicName[0],
+		                               &uiTableEntryPDumpOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PMR_PDumpSymbolicAddr(psReferencePMR,
+		                               IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                               sizeof(aszPageMemspaceName),
+		                               &aszPageMemspaceName[0],
+		                               sizeof(aszPageSymbolicName),
+		                               &aszPageSymbolicName[0],
+		                               &uiPagePDumpOffset,
+		                               &uiNextSymName);
+		PVR_ASSERT(eError == PVRSRV_OK);
+
+		eError = PDumpWriteShiftedMaskedValue(PMR_DeviceNode(psReferencePMR),
+		                                      /* destination */
+		                                      aszTableEntryMemspaceName,
+		                                      aszTableEntrySymbolicName,
+		                                      uiTableEntryPDumpOffset,
+		                                      /* source */
+		                                      aszPageMemspaceName,
+		                                      aszPageSymbolicName,
+		                                      uiPagePDumpOffset,
+		                                      /* shift right */
+		                                      uiLog2PageSize,
+		                                      /* shift left */
+		                                      0,
+		                                      /* mask */
+		                                      0xffffffff,
+		                                      /* word size */
+		                                      uiWordSize,
+		                                      /* flags */
+		                                      PDUMP_FLAGS_CONTINUOUS);
+		PVR_ASSERT(eError == PVRSRV_OK);
+#else
+		PVR_UNREFERENCED_PARAMETER(uiPMROffset);
+#endif
+
+#if !defined(NO_HARDWARE)
+
+		/*
+			We check for sparse PMR's at function entry, but as we can,
+			check that every page is valid
+		 */
+		PVR_ASSERT(pbPageIsValid[uiPageIndex]);
+		PVR_ASSERT(pasDevAddrPtr[uiPageIndex].uiAddr != 0);
+		PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000LL) == 0);
+
+		uiPageListPMRPage = uiPMROffset >> psReferencePMR->uiLog2ContiguityGuarantee;
+
+		if ((pui32DataPtr == NULL) || (uiPageListPMRPage != uiPrevPageListPMRPage))
+		{
+			size_t uiMappingOffset = uiPMROffset & (~(ui64PageListPageSize - 1));
+			size_t uiMappedSize;
+
+			/* If we already had a page list mapped, we need to unmap it... */
+			if (pui32DataPtr != NULL)
+			{
+				PMRReleaseKernelMappingData(psPageListPMR, hPrivData);
+			}
+
+			eError = PMRAcquireKernelMappingData(psPageListPMR,
+			                                     uiMappingOffset,
+			                                     ui64PageListPageSize,
+			                                     &pvKernAddr,
+			                                     &uiMappedSize,
+			                                     &hPrivData);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Error mapping page list PMR page (%" IMG_UINT64_FMTSPEC ") into kernel (%d)",
+						uiPageListPMRPage, eError));
+				goto free_valid_array;
+			}
+
+			uiPrevPageListPMRPage = uiPageListPMRPage;
+			PVR_ASSERT(uiMappedSize >= ui64PageListPageSize);
+			PVR_ASSERT(pvKernAddr != NULL);
+
+			pui32DataPtr = IMG_OFFSET_ADDR(pvKernAddr, (uiPMROffset & (ui64PageListPageSize - 1)));
+		}
+
+		PVR_ASSERT(((pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize) & 0xFFFFFFFF00000000LL) == 0);
+
+		/* Write the physical page index into the page list PMR */
+		*pui32DataPtr++ = TRUNCATE_64BITS_TO_32BITS(pasDevAddrPtr[uiPageIndex].uiAddr >> uiLog2PageSize);
+
+		/* Last page so unmap */
+		if (uiPageIndex == (uiNumPages - 1))
+		{
+			PMRReleaseKernelMappingData(psPageListPMR, hPrivData);
+		}
+#endif
+	}
+
+	OSWriteMemoryBarrier(NULL);
+
+#if !defined(NO_HARDWARE)
+	if (pasDevAddrPtr != asDevPAddr)
+	{
+		OSFreeMem(pbPageIsValid);
+		OSFreeMem(pasDevAddrPtr);
+	}
+#endif
+	*ppsPageList = psPageList;
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+#if !defined(NO_HARDWARE)
+
+free_valid_array:
+	if (pbPageIsValid != abValid)
+	{
+		OSFreeMem(pbPageIsValid);
+	}
+
+free_devaddr_array:
+	if (pasDevAddrPtr != asDevPAddr)
+	{
+		OSFreeMem(pasDevAddrPtr);
+	}
+
+unlock_phys_addrs:
+	PMRUnlockSysPhysAddresses(psReferencePMR);
+#endif
+
+free_page_list:
+	OSFreeMem(psPageList);
+
+return_error:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR
+PMRUnwritePMPageList(PMR_PAGELIST *psPageList)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PMRUnlockSysPhysAddresses(psPageList->psReferencePMR);
+	PVR_ASSERT(eError == PVRSRV_OK);
+	OSFreeMem(psPageList);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PMRZeroingPMR(PMR *psPMR,
+              IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize)
+{
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	PMR_SIZE_T ui64PageSize = IMG_PAGE2BYTES64(uiLog2PageSize);
+	IMG_HANDLE hPrivData = NULL;
+	void *pvKernAddr = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	size_t uiMappedSize;
+
+	PVR_ASSERT(psPMR);
+
+	/* Calculate number of pages in this PMR */
+	uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize);
+
+	/* Verify the logical Size is a multiple or the physical page size */
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR is not a multiple of %" IMG_UINT64_FMTSPEC,
+		         __func__,
+		         ui64PageSize));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, MultiPage_Error);
+	}
+
+	if (_PMRIsSparse(psPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, Sparse_Error);
+	}
+
+	/* Scan through all pages of the PMR */
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		/* map the physical page (for a given PMR offset) into kernel space */
+		eError = PMRAcquireKernelMappingData(psPMR,
+		                                     IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                                     ui64PageSize,
+		                                     &pvKernAddr,
+		                                     &uiMappedSize,
+		                                     &hPrivData);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PMRAcquireKernelMappingData", AcquireKernelMapping_Error);
+
+		/* ensure the mapped page size is the same as the physical page size */
+		if (uiMappedSize != ui64PageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Physical Page size = 0x%08" IMG_UINT64_FMTSPECx ", Size of Mapping = 0x%016" IMG_UINT64_FMTSPECx,
+			         __func__,
+			         ui64PageSize,
+			         (IMG_UINT64)uiMappedSize));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, MappingSize_Error);
+		}
+
+		/* Use the conservative 'DeviceMemSet' here because we can't know
+		 * if this PMR will be mapped cached.
+		 */
+		OSDeviceMemSet(pvKernAddr, 0, ui64PageSize);
+
+		/* release mapping */
+		PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "%s: Zeroing PMR %p done (num pages %u, page size %" IMG_UINT64_FMTSPEC ")",
+	         __func__,
+	         psPMR,
+	         uiNumPages,
+	         ui64PageSize));
+
+	return PVRSRV_OK;
+
+
+	/* Error handling */
+
+MappingSize_Error:
+	PMRReleaseKernelMappingData(psPMR, hPrivData);
+
+AcquireKernelMapping_Error:
+Sparse_Error:
+MultiPage_Error:
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRDumpPageList(PMR *psPMR,
+                IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize)
+{
+	IMG_DEV_PHYADDR sDevAddrPtr;
+	IMG_UINT32 uiNumPages;
+	IMG_UINT32 uiPageIndex;
+	IMG_BOOL bPageIsValid;
+	IMG_UINT32 ui32Col = 16;
+	IMG_UINT32 ui32SizePerCol = 11;
+	IMG_UINT32 ui32ByteCount = 0;
+	IMG_CHAR pszBuffer[16 /* ui32Col */ * 11 /* ui32SizePerCol */ + 1];
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Get number of pages */
+	uiNumPages = (IMG_UINT32)(psPMR->uiLogicalSize >> uiLog2PageSize);
+
+	/* Verify the logical Size is a multiple or the physical page size */
+	if ((PMR_SIZE_T)uiNumPages << uiLog2PageSize != psPMR->uiLogicalSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMR is not a multiple of %" IMG_UINT64_FMTSPEC,
+		        __func__, (IMG_UINT64) (1ULL << uiLog2PageSize)));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE, MultiPage_Error);
+	}
+
+	if (_PMRIsSparse(psPMR))
+	{
+		PVR_LOG_GOTO_WITH_ERROR("psPMR", eError, PVRSRV_ERROR_INVALID_PARAMS, Sparse_Error);
+	}
+
+	PVR_LOG(("    PMR %p, Number of pages %u, Log2PageSize %d", psPMR, uiNumPages, uiLog2PageSize));
+
+	/* Print the address of the physical pages */
+	for (uiPageIndex = 0; uiPageIndex < uiNumPages; uiPageIndex++)
+	{
+		/* Get Device physical Address */
+		eError = PMR_DevPhysAddr(psPMR,
+		                         uiLog2PageSize,
+		                         1,
+		                         IMG_PAGES2BYTES64(uiPageIndex, uiLog2PageSize),
+		                         &sDevAddrPtr,
+		                         &bPageIsValid,
+		                         DEVICE_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PMR %p failed to get DevPhysAddr with error %u",
+					__func__,
+					psPMR,
+					eError));
+			goto DevPhysAddr_Error;
+		}
+
+		ui32ByteCount += OSSNPrintf(pszBuffer + ui32ByteCount, ui32SizePerCol + 1, "%08x ", (IMG_UINT32)(sDevAddrPtr.uiAddr >> uiLog2PageSize));
+		PVR_ASSERT(ui32ByteCount < ui32Col * ui32SizePerCol);
+
+		if (uiPageIndex % ui32Col == ui32Col-1)
+		{
+			PVR_LOG(("      Phys Page: %s", pszBuffer));
+			ui32ByteCount = 0;
+		}
+	}
+	if (ui32ByteCount > 0)
+	{
+		PVR_LOG(("      Phys Page: %s", pszBuffer));
+	}
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+DevPhysAddr_Error:
+Sparse_Error:
+MultiPage_Error:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+DEFINE_PMR_ENV_GET_DATA(psPMR, sEnvData)
+
+PVRSRV_ERROR
+PMRInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	/* Singleton PMR context already initialised */
+	if (_gsSingletonPMRContext.bModuleInitialised)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+	eError = OSLockCreate(&_gsSingletonPMRContext.hLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", out);
+
+	_gsSingletonPMRContext.uiNextSerialNum = 1;
+
+	_gsSingletonPMRContext.uiNextKey = 0x8300f001 * (uintptr_t)&_gsSingletonPMRContext;
+
+	_gsSingletonPMRContext.bModuleInitialised = IMG_TRUE;
+
+	OSAtomicWrite(&_gsSingletonPMRContext.uiNumLivePMRs, 0);
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	eError = MMapStatsInit();
+	PVR_LOG_GOTO_IF_ERROR(eError, "MMapStatsInit", destroy_context_lock);
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+destroy_context_lock:
+	OSLockDestroy(_gsSingletonPMRContext.hLock);
+	_gsSingletonPMRContext.hLock = NULL;
+#endif
+out:
+	return eError;
+}
+
+PVRSRV_ERROR
+PMRDeInit(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		goto out;
+	}
+
+	/* Singleton PMR context is not initialised */
+	if (!_gsSingletonPMRContext.bModuleInitialised)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	MMapStatsDeInit();
+#endif
+
+	if (OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error: %d live PMRs remain",
+				__func__,
+				OSAtomicRead(&_gsSingletonPMRContext.uiNumLivePMRs)));
+		PVR_DPF((PVR_DBG_ERROR, "%s: This is an unrecoverable error; a subsequent crash is inevitable",
+				__func__));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR, out);
+	}
+
+	if (_gsSingletonPMRContext.hLock != NULL)
+	{
+		OSLockDestroy(_gsSingletonPMRContext.hLock);
+	}
+
+	_gsSingletonPMRContext.bModuleInitialised = IMG_FALSE;
+
+out:
+	PVR_ASSERT(eError == PVRSRV_OK);
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+PVRSRV_ERROR
+PMRInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDeviceNode->hPMRZombieListLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	dllist_init(&psDeviceNode->sPMRZombieList);
+	psDeviceNode->uiPMRZombieCount = 0;
+	psDeviceNode->uiPMRZombieCountInCleanup = 0;
+
+	return PVRSRV_OK;
+}
+
+void
+PMRFreeZombies(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	DECLARE_DLLIST(sZombieList);
+	DLLIST_NODE *psThis, *psNext;
+	IMG_INT32 uiZombieCount;
+
+	OSLockAcquire(psDeviceNode->hPMRZombieListLock);
+	/* Move the zombie list to a local copy. The original list will become
+	 * an empty list. This will allow us to process the list without holding
+	 * the list lock. */
+	dllist_replace_head(&psDeviceNode->sPMRZombieList, &sZombieList);
+	uiZombieCount = psDeviceNode->uiPMRZombieCount;
+	psDeviceNode->uiPMRZombieCount = 0;
+	OSLockRelease(psDeviceNode->hPMRZombieListLock);
+
+	dllist_foreach_node(&sZombieList, psThis, psNext)
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (PMR_IsListNodeZombiePages(psThis))
+		{
+			PVRSRV_ERROR eError;
+			PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psThis);
+
+			eError = psZombiePages->pfnFactoryFreeZombies(psZombiePages->pvFactoryPages);
+			if (eError != PVRSRV_OK)
+			{
+				/* In case of failure to free zombie pages, remove it from
+				 * the sZombieList and add back to the original list. */
+				OSLockAcquire(psDeviceNode->hPMRZombieListLock);
+				dllist_remove_node(psThis);
+				dllist_add_to_tail(&psDeviceNode->sPMRZombieList, psThis);
+				psDeviceNode->uiPMRZombieCount++;
+				OSLockRelease(psDeviceNode->hPMRZombieListLock);
+
+				PVR_DPF((PVR_DBG_ERROR, "Cannot free zombie pages!"));
+				continue;
+			}
+
+			dllist_remove_node(psThis);
+			OSFreeMem(psZombiePages);
+		}
+		else
+#endif
+		{
+			PMR *psPMR = PMR_GetPMRFromNode(psThis);
+			dllist_remove_node(psThis);
+			_PMRDestroy(psPMR);
+		}
+
+		uiZombieCount--;
+	}
+
+	PVR_ASSERT(uiZombieCount == 0);
+}
+
+void
+PMRDumpZombies(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	DLLIST_NODE *psThis, *psNext;
+
+	OSLockAcquire(psDeviceNode->hPMRZombieListLock);
+
+	PVR_DPF((PVR_DBG_ERROR, "Items in zombie list: %u",
+	        psDeviceNode->uiPMRZombieCount));
+
+	dllist_foreach_node(&psDeviceNode->sPMRZombieList, psThis, psNext)
+	{
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+		if (PMR_IsListNodeZombiePages(psThis))
+		{
+			PMR_ZOMBIE_PAGES* psZombiePages = PMR_GetZombiePagesFromNode(psThis);
+			PVR_DPF((PVR_DBG_ERROR, "Zombie Pages = %p", psZombiePages));
+		}
+		else
+#endif
+		{
+			PMR *psPMR = PMR_GetPMRFromNode(psThis);
+			PVR_DPF((PVR_DBG_ERROR, "PMR = %px, Flavour = %s, Annotation: %s",
+			        psPMR, PMR_GetTypeStr(psPMR), PMR_GetAnnotation(psPMR)));
+		}
+	}
+
+	OSLockRelease(psDeviceNode->hPMRZombieListLock);
+}
+
+void
+PMRDeInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PMRFreeZombies(psDeviceNode);
+
+	OSLockDestroy(psDeviceNode->hPMRZombieListLock);
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
diff --git a/drivers/gpu/drm/img-rogue/pmr.h b/drivers/gpu/drm/img-rogue/pmr.h
new file mode 100644
index 000000000000..7cd3fb4d8929
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pmr.h
@@ -0,0 +1,1217 @@
+/*************************************************************************/ /*!
+@File
+@Title          Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management.  This module is responsible for
+                the "PMR" abstraction.  A PMR (Physical Memory Resource)
+                represents some unit of physical memory which is
+                allocated/freed/mapped/unmapped as an indivisible unit
+                (higher software levels provide an abstraction above that
+                to deal with dividing this down into smaller manageable units).
+                Importantly, this module knows nothing of virtual memory, or
+                of MMUs etc., with one excusable exception.  We have the
+                concept of a "page size", which really means nothing in
+                physical memory, but represents a "contiguity quantum" such
+                that the higher level modules which map this memory are able
+                to verify that it matches the needs of the page size for the
+                virtual realm into which it is being mapped.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVSRV_PMR_H
+#define SRVSRV_PMR_H
+
+/* include/ */
+#include "img_types.h"
+#include "img_defs.h"
+#include "pdumpdefs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memallocflags.h"
+#include "devicemem_typedefs.h"			/* Required for export DEVMEM_EXPORTCOOKIE */
+
+/* services/include */
+#include "pdump.h"
+#include "physheap.h"
+
+/* services/server/include/ */
+#include "pmr_impl.h"
+#include "opaque_types.h"
+
+#define PMR_MAX_TRANSLATION_STACK_ALLOC				(32)
+
+/* Maximum size PMR can have is 8G of memory */
+#define PMR_MAX_SUPPORTED_SIZE IMG_UINT64_C(0x200000000)
+/* Max number of pages in a PMR at 4k page size */
+#define PMR_MAX_SUPPORTED_4K_PAGE_COUNT (PMR_MAX_SUPPORTED_SIZE >> 12ULL)
+
+typedef IMG_UINT64 PMR_BASE_T;
+typedef IMG_UINT64 PMR_SIZE_T;
+#define PMR_SIZE_FMTSPEC "0x%010"IMG_UINT64_FMTSPECX
+#define PMR_VALUE32_FMTSPEC "0x%08X"
+#define PMR_VALUE64_FMTSPEC "0x%016"IMG_UINT64_FMTSPECX
+typedef IMG_UINT32 PMR_LOG2ALIGN_T;
+typedef IMG_UINT64 PMR_PASSWORD_T;
+
+struct _PMR_MAPPING_TABLE_
+{
+	PMR_SIZE_T uiChunkSize;            /*!< Size of a "chunk" */
+	IMG_UINT32 ui32NumPhysChunks;      /*!< Number of physical chunks that are valid */
+	IMG_UINT32 ui32NumLogicalChunks;   /*!< Number of logical chunks in the mapping */
+	/* Must be last */
+	IMG_UINT32 aui32Translation[IMG_FLEX_ARRAY_MEMBER];    /*!< Translation mapping for "logical" to physical */
+};
+
+#define TRANSLATION_INVALID 0xFFFFFFFFUL
+
+typedef struct _PMR_EXPORT_ PMR_EXPORT;
+
+typedef struct _PMR_PAGELIST_ PMR_PAGELIST;
+
+IMG_INT32 PMRGetLiveCount(void);
+
+/*
+ * PMRValidateSize
+ *
+ * Given a size value, check the value against the max supported
+ * PMR size of 1GB. Return IMG_FALSE if size exceeds max, IMG_TRUE
+ * otherwise.
+ */
+static inline IMG_BOOL PMRValidateSize(IMG_UINT64 uiSize)
+{
+	return (uiSize > PMR_MAX_SUPPORTED_SIZE) ? IMG_FALSE : IMG_TRUE;
+}
+
+/*
+ * PMRCreatePMR
+ *
+ * Not to be called directly, only via implementations of PMR
+ * factories, e.g. in physmem_osmem.c, deviceclass.c, etc.
+ *
+ * Creates a PMR object, with callbacks and private data as per the
+ * FuncTab/PrivData args.
+ *
+ * Note that at creation time the PMR must set in stone the "logical
+ * size" and the "contiguity guarantee"
+ *
+ * Flags are also set at this time.  (T.B.D.  flags also immutable for
+ * the life of the PMR?)
+ *
+ * Logical size is the amount of Virtual space this allocation would
+ * take up when mapped.  Note that this does not have to be the same
+ * as the actual physical size of the memory.  For example, consider
+ * the sparsely allocated non-power-of-2 texture case.  In this
+ * instance, the "logical size" would be the virtual size of the
+ * rounded-up power-of-2 texture.  That some pages of physical memory
+ * may not exist does not affect the logical size calculation.
+ *
+ * The PMR must also supply the "contiguity guarantee" which is the
+ * finest granularity of alignment and size of physical pages that the
+ * PMR will provide after LockSysPhysAddresses is called.  Note that
+ * the calling code may choose to call PMRSysPhysAddr with a finer
+ * granularity than this, for example if it were to map into a device
+ * MMU with a smaller page size, and it's also OK for the PMR to
+ * supply physical memory in larger chunks than this.  But
+ * importantly, never the other way around.
+ *
+ * More precisely, the following inequality must be maintained
+ * whenever mappings and/or physical addresses exist:
+ *
+ *       (device MMU page size) <= 2**(uiLog2ContiguityGuarantee) <= (actual contiguity of physical memory)
+ *
+ * The function table will contain the following callbacks which may
+ * be overridden by the PMR implementation:
+ *
+ * pfnLockPhysAddresses
+ *
+ *      Called when someone locks requests that Physical pages are to
+ *      be locked down via the PMRLockSysPhysAddresses() API.  Note
+ *      that if physical pages are prefaulted at PMR creation time and
+ *      therefore static, it would not be necessary to override this
+ *      function, in which case NULL may be supplied.
+ *
+ * pfnUnlockPhysAddresses
+ *
+ *      The reverse of pfnLockPhysAddresses.  Note that this should be
+ *      NULL if and only if pfnLockPhysAddresses is NULL
+ *
+ * pfnSysPhysAddr
+ *
+ *      This function is mandatory.  This is the one which returns the
+ *      system physical address for a given offset into this PMR.  The
+ *      "lock" function will have been called, if overridden, before
+ *      this function, thus the implementation should not increase any
+ *      refcount when answering this call.  Refcounting, if necessary,
+ *      should be done in the lock/unlock calls.  Refcounting would
+ *      not be necessary in the prefaulted/static scenario, as the
+ *      pmr.c abstraction will handle the refcounting for the whole
+ *      PMR.
+ *
+ * pfnFinalize
+ *
+ *      Called when the PMR's refcount reaches zero and it gets
+ *      destroyed.  This allows the implementation to free up any
+ *      resource acquired during creation time.
+ *
+ */
+PVRSRV_ERROR
+PMRCreatePMR(PHYS_HEAP *psPhysHeap,
+             PMR_SIZE_T uiLogicalSize,
+             IMG_UINT32 ui32NumPhysChunks,
+             IMG_UINT32 ui32NumLogicalChunks,
+             IMG_UINT32 *pui32MappingTable,
+             PMR_LOG2ALIGN_T uiLog2ContiguityGuarantee,
+             PMR_FLAGS_T uiFlags,
+             const IMG_CHAR *pszAnnotation,
+             const PMR_IMPL_FUNCTAB *psFuncTab,
+             PMR_IMPL_PRIVDATA pvPrivData,
+             PMR_IMPL_TYPE eType,
+             PMR **ppsPMRPtr,
+             IMG_UINT32 ui32PDumpFlags);
+
+/*
+ * PMRLockSysPhysAddresses()
+ *
+ * Calls the relevant callback to lock down the system physical addresses of
+ * the memory that makes up the whole PMR.
+ *
+ * Before this call, it is not valid to use any of the information
+ * getting APIs: PMR_Flags(), PMR_SysPhysAddr(),
+ * [ see note below about lock/unlock semantics ]
+ *
+ * The caller of this function does not have to care about how the PMR
+ * is implemented.  He only has to know that he is allowed access to
+ * the physical addresses _after_ calling this function and _until_
+ * calling PMRUnlockSysPhysAddresses().
+ *
+ *
+ * Notes to callback implementers (authors of PMR Factories):
+ *
+ * Some PMR implementations will be such that the physical memory exists for
+ * the lifetime of the PMR, with a static address, (and normally flags and
+ * symbolic address are static too) and so it is legal for a PMR
+ * implementation to not provide an implementation for the lock callback.
+ *
+ * Some PMR implementation may wish to page memory in from secondary storage
+ * on demand. The lock/unlock callbacks _may_ be the place to do this.
+ * (More likely, there would be a separate API for doing this, but this API
+ * provides a useful place to assert that it has been done)
+ */
+
+PVRSRV_ERROR
+PMRLockSysPhysAddresses(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRLockSysPhysAddressesNested(PMR *psPMR,
+                        IMG_UINT32 ui32NestingLevel);
+
+/*
+ * PMRUnlockSysPhysAddresses()
+ *
+ * the reverse of PMRLockSysPhysAddresses()
+ */
+PVRSRV_ERROR
+PMRUnlockSysPhysAddresses(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRUnlockSysPhysAddressesNested(PMR *psPMR, IMG_UINT32 ui32NestingLevel);
+
+/*
+ * PhysmemPMRExport()
+ *
+ * Given a PMR, creates a PMR "Export", which is a handle that
+ * provides sufficient data to be able to "import" this PMR elsewhere.
+ * The PMR Export is an object in its own right, whose existence
+ * implies a reference on the PMR, thus the PMR cannot be destroyed
+ * while the PMR Export exists.  The intention is that the PMR Export
+ * will be wrapped in the devicemem layer by a cross process handle,
+ * and some IPC by which to communicate the handle value and password
+ * to other processes.  The receiving process is able to unwrap this
+ * to gain access to the same PMR Export in this layer, and, via
+ * PhysmemPMRImport(), obtain a reference to the original PMR.
+ *
+ * The caller receives, along with the PMR Export object, information
+ * about the size and contiguity guarantee for the PMR, and also the
+ * PMRs secret password, in order to authenticate the subsequent
+ * import.
+ *
+ * N.B.  If you call PMRExportPMR() (and it succeeds), you are
+ * promising to later call PMRUnexportPMR()
+ */
+PVRSRV_ERROR
+PMRExportPMR(PMR *psPMR,
+             PMR_EXPORT **ppsPMRExport,
+             PMR_SIZE_T *puiSize,
+             PMR_LOG2ALIGN_T *puiLog2Contig,
+             PMR_PASSWORD_T *puiPassword);
+
+/*!
+*******************************************************************************
+
+ @Function      PMRMakeLocalImportHandle
+
+ @Description
+
+ Transform a general handle type into one that we are able to import.
+ Takes a PMR reference.
+
+ @Input   psPMR     The input PMR.
+ @Output  ppsPMR    The output PMR that is going to be transformed to the
+                    correct handle type.
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PMRMakeLocalImportHandle(PMR *psPMR,
+                         PMR **ppsPMR);
+
+/*!
+*******************************************************************************
+
+ @Function      PMRUnmakeLocalImportHandle
+
+ @Description
+
+ Take a PMR, destroy the handle and release a reference.
+ Counterpart to PMRMakeServerExportClientExport().
+
+ @Input   psPMR       PMR to destroy.
+                      Created by PMRMakeLocalImportHandle().
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PMRUnmakeLocalImportHandle(PMR *psPMR);
+
+/*
+ * PMRUnexportPMR()
+ *
+ * The reverse of PMRExportPMR().  This causes the PMR to no longer be
+ * exported.  If the PMR has already been imported, the imported PMR
+ * reference will still be valid, but no further imports will be possible.
+ */
+PVRSRV_ERROR
+PMRUnexportPMR(PMR_EXPORT *psPMRExport);
+
+/*
+ * PMRImportPMR()
+ *
+ * Takes a PMR Export object, as obtained by PMRExportPMR(), and
+ * obtains a reference to the original PMR.
+ *
+ * The password must match, and is assumed to have been (by whatever
+ * means, IPC etc.) preserved intact from the former call to
+ * PMRExportPMR()
+ *
+ * The size and contiguity arguments are entirely irrelevant for the
+ * import, however they are verified in order to trap bugs.
+ *
+ * N.B.  If you call PhysmemPMRImport() (and it succeeds), you are
+ * promising to later call PhysmemPMRUnimport()
+ */
+PVRSRV_ERROR
+PMRImportPMR(PMR_EXPORT *psPMRExport,
+             PMR_PASSWORD_T uiPassword,
+             PMR_SIZE_T uiSize,
+             PMR_LOG2ALIGN_T uiLog2Contig,
+             PMR **ppsPMR);
+
+/* Function that alters the mutability property
+ * of the PMR
+ * Setting it to TRUE makes sure the PMR memory layout
+ * can't be changed through future calls */
+void
+PMR_SetLayoutFixed(PMR *psPMR, IMG_BOOL bFlag);
+
+IMG_BOOL PMR_IsMemLayoutFixed(PMR *psPMR);
+
+/*
+ * PMRUnimportPMR()
+ *
+ * releases the reference on the PMR as obtained by PMRImportPMR()
+ */
+PVRSRV_ERROR
+PMRUnimportPMR(PMR *psPMR);
+
+PVRSRV_ERROR
+PMRLocalImportPMR(PMR *psPMR,
+                  PMR **ppsPMR,
+                  IMG_DEVMEM_SIZE_T *puiSize,
+                  IMG_DEVMEM_ALIGN_T *puiAlign);
+
+/*
+ * Equivalent mapping functions when in kernel mode.
+ */
+PVRSRV_ERROR
+PMRAcquireKernelMappingData(PMR *psPMR,
+                            size_t uiLogicalOffset,
+                            size_t uiSize,
+                            void **ppvKernelAddressOut,
+                            size_t *puiLengthOut,
+                            IMG_HANDLE *phPrivOut);
+
+PVRSRV_ERROR
+PMRAcquireSparseKernelMappingData(PMR *psPMR,
+                                  size_t uiLogicalOffset,
+                                  size_t uiSize,
+                                  void **ppvKernelAddressOut,
+                                  size_t *puiLengthOut,
+                                  IMG_HANDLE *phPrivOut);
+
+PVRSRV_ERROR
+PMRReleaseKernelMappingData(PMR *psPMR,
+                            IMG_HANDLE hPriv);
+
+/*
+ * PMR_ReadBytes()
+ *
+ * calls into the PMR implementation to read up to uiBufSz bytes,
+ * returning the actual number read in *puiNumBytes
+ *
+ * this will read up to the end of the PMR, or the next symbolic name
+ * boundary, or until the requested number of bytes is read, whichever
+ * comes first
+ *
+ * In the case of sparse PMR's the caller doesn't know what offsets are
+ * valid and which ones aren't so we will just write 0 to invalid offsets
+ */
+PVRSRV_ERROR
+PMR_ReadBytes(PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT8 *pcBuffer,
+              size_t uiBufSz,
+              size_t *puiNumBytes);
+
+/*
+ * PMR_WriteBytes()
+ *
+ * calls into the PMR implementation to write up to uiBufSz bytes,
+ * returning the actual number read in *puiNumBytes
+ *
+ * this will write up to the end of the PMR, or the next symbolic name
+ * boundary, or until the requested number of bytes is written, whichever
+ * comes first
+ *
+ * In the case of sparse PMR's the caller doesn't know what offsets are
+ * valid and which ones aren't so we will just ignore data at invalid offsets
+ */
+PVRSRV_ERROR
+PMR_WriteBytes(PMR *psPMR,
+               IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+               IMG_UINT8 *pcBuffer,
+               size_t uiBufSz,
+               size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Function       PMRMMapPMR
+@Description    Performs the necessary steps to map the PMR into a user process
+                address space. The caller does not need to call
+                PMRLockSysPhysAddresses before calling this function.
+
+@Input          psPMR            PMR to map.
+
+@Input          pOSMMapData      OS specific data needed to create a mapping.
+
+@Input          uiCpuAccessFlags Flags to indicate if the mapping request
+                                 requires read, write or both access.
+
+@Return         PVRSRV_ERROR:   PVRSRV_OK on success or an error otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PMRMMapPMR(PMR *psPMR,
+           PMR_MMAP_DATA pOSMMapData,
+           PVRSRV_MEMALLOCFLAGS_T uiCpuAccessFlags);
+
+/*
+ * PMRRefPMR()
+ *
+ * Take a reference on the passed in PMR
+ */
+void
+PMRRefPMR(PMR *psPMR);
+
+/*
+ * PMRUnrefPMR()
+ *
+ * This undoes a call to any of the PhysmemNew* family of APIs
+ * (i.e. any PMR factory "constructor")
+ *
+ * This relinquishes a reference to the PMR, and, where the refcount
+ * reaches 0, causes the PMR to be destroyed (calling the finalizer
+ * callback on the PMR, if there is one)
+ */
+PVRSRV_ERROR
+PMRUnrefPMR(PMR *psPMR);
+
+/*
+ * PMRRefPMR2()
+ *
+ * Take a reference on the passed in PMR.
+ *
+ * This function does not perform address locking as opposed to PMRRefPMR().
+ */
+void
+PMRRefPMR2(PMR *psPMR);
+
+/*
+ * PMRUnrefPMR2()
+ *
+ * This undoes a call to any of the PhysmemNew* family of APIs
+ * (i.e. any PMR factory "constructor").
+ *
+ * This relinquishes a reference to the PMR, and, where the refcount
+ * reaches 0, causes the PMR to be destroyed (calling the finalizer
+ * callback on the PMR, if there is one)
+ */
+void
+PMRUnrefPMR2(PMR *psPMR);
+
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+/*
+ * PMRTryRefPMR()
+ *
+ * This attempts to take a reference on the PMR but only succeeds if
+ * the PMR is not at refcount 0. Other Ref functions would class this
+ * attempt as a logical error. This function is free to attempt and return
+ * an error if PMR is in a free in progress state.
+ */
+PVRSRV_ERROR
+PMRTryRefPMR(PMR *psPMR);
+
+/*
+ * PMRGpuMapDevPageCountIncr()
+ *
+ * Increment count of the number of current device page GPU mappings of the PMR.
+ */
+void
+PMRGpuMapDevPageCountIncr(PMR *psPMR, IMG_UINT32 uiCount);
+
+/*
+ * PMRGpuMapDevPageCountDecr()
+ *
+ * Decrement count of the number of current device page GPU mappings of the PMR.
+ */
+void
+PMRGpuMapDevPageCountDecr(PMR *psPMR, IMG_UINT32 uiCount);
+
+IMG_BOOL
+PMR_IsGpuMapped(PMR *psPMR);
+#else
+#define PMRGpuMapDevPageCountIncr(...)
+#define PMRGpuMapDevPageCountDecr(...)
+#endif /* #if defined(SUPPORT_LINUX_OSPAGE_MIGRATION) */
+
+/*
+ * PMRCpuMapCountIncr()
+ *
+ * Increment count of the number of current CPU mappings of the PMR.
+ */
+void
+PMRCpuMapCountIncr(PMR *psPMR);
+
+/*
+ * PMRCpuMapCountDecr()
+ *
+ * Decrement count of the number of current CPU mappings of the PMR.
+ */
+void
+PMRCpuMapCountDecr(PMR *psPMR);
+
+IMG_BOOL
+PMR_IsCpuMapped(PMR *psPMR);
+
+/*
+ * PMRGpuResCountIncr()
+ *
+ * Increment count of the number of current GPU reservations associated with the PMR.
+ * Must be protected by PMR lock.
+ */
+void
+PMRGpuResCountIncr(PMR *psPMR);
+
+/*
+ * PMRGpuResCountDecr()
+ *
+ * Decrement count of the number of current GPU reservations associated with the PMR.
+ * Must be protected by PMR lock.
+ *
+ */
+void
+PMRGpuResCountDecr(PMR *psPMR);
+
+/*
+ * PMR_IsGpuMultiMapped()
+ *
+ * Must be protected by PMR lock.
+ *
+ */
+IMG_BOOL
+PMR_IsGpuMultiMapped(PMR *psPMR);
+
+PPVRSRV_DEVICE_NODE
+PMR_DeviceNode(const PMR *psPMR);
+
+/*
+ * PMR_Flags()
+ *
+ * Flags are static and guaranteed for the life of the PMR.  Thus this
+ * function is idempotent and acquire/release semantics is not required.
+ *
+ * Returns the flags as specified on the PMR.  The flags are to be
+ * interpreted as mapping permissions
+ */
+PMR_FLAGS_T
+PMR_Flags(const PMR *psPMR);
+
+IMG_BOOL
+PMR_IsSparse(const PMR *psPMR);
+
+IMG_DEVMEM_SIZE_T
+PMR_PhysicalSize(const PMR *psPMR);
+
+/*
+ * PMR_IsOffsetValid()
+ *
+ * Returns if an address offset inside a PMR has a valid
+ * physical backing.
+ */
+PVRSRV_ERROR
+PMR_IsOffsetValid(const PMR *psPMR,
+				IMG_UINT32 ui32Log2PageSize,
+				IMG_UINT32 ui32NumOfPages,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_BOOL *pbValid);
+
+PHYS_HEAP *
+PMR_PhysHeap(const PMR *psPMR);
+
+PMR_MAPPING_TABLE *
+PMR_GetMappingTable(const PMR *psPMR);
+
+IMG_UINT32
+PMR_GetLog2Contiguity(const PMR *psPMR);
+
+IMG_DEVMEM_SIZE_T
+PMR_LogicalSize(const PMR *psPMR);
+
+/*
+ * PMR_GetLogicalChunkCount
+ *
+ * Retrieve the maximum number of chunks supported by the PMR.
+ * This property is fixed at creation time.
+ */
+IMG_UINT32 PMR_GetLogicalChunkCount(const PMR *psPMR);
+
+const IMG_CHAR *
+PMR_GetAnnotation(const PMR *psPMR);
+
+PMR_IMPL_TYPE
+PMR_GetType(const PMR *psPMR);
+
+IMG_CHAR *
+PMR_GetTypeStr(const PMR *psPMR);
+
+IMG_INT32
+PMR_GetRefCount(const PMR *psPMR);
+
+/* PMR usage type for callers of PMR_DevPhysAddr() */
+typedef enum _PMR_USAGE_TYPE_
+{
+	CPU_USE = 0,
+	DEVICE_USE
+} PMR_USAGE_TYPE;
+
+/*
+ * PMR_DevPhysAddr()
+ *
+ * A note regarding Lock/Unlock semantics
+ * ======================================
+ *
+ * PMR_DevPhysAddr may only be called after PMRLockSysPhysAddresses()
+ * has been called.  The data returned may be used only until
+ * PMRUnlockSysPhysAddresses() is called after which time the licence
+ * to use the data is revoked and the information may be invalid.
+ *
+ * Given an offset, this function returns the device physical address of the
+ * corresponding page in the PMR.  It may be called multiple times
+ * until the address of all relevant pages has been determined.
+ *
+ * If caller only wants one physical address it is sufficient to pass in:
+ * ui32Log2PageSize==0 and ui32NumOfPages==1
+ */
+PVRSRV_ERROR
+PMR_DevPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEV_PHYADDR *psDevAddr,
+                IMG_BOOL *pbValid,
+                PMR_USAGE_TYPE ePMRUsage);
+
+/*
+ * PMR_CpuPhysAddr()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * Given an offset, this function returns the CPU physical address of the
+ * corresponding page in the PMR.  It may be called multiple times
+ * until the address of all relevant pages has been determined.
+ *
+ */
+PVRSRV_ERROR
+PMR_CpuPhysAddr(const PMR *psPMR,
+                IMG_UINT32 ui32Log2PageSize,
+                IMG_UINT32 ui32NumOfPages,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_CPU_PHYADDR *psCpuAddrPtr,
+                IMG_BOOL *pbValid);
+
+/* PMRGetUID()
+ *
+ * Used for bridge calls that expect a PVRSRV_ERROR returned
+ * */
+PVRSRV_ERROR
+PMRGetUID(PMR *psPMR,
+          IMG_UINT64 *pui64UID);
+
+IMG_UINT64
+PMRInternalGetUID(PMR *psPMR);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*
+ * PMR_IsZombie()
+ *
+ * Indicates if a PMR is a "zombie" PMR. This function **must** be called
+ * inside a PMR factory lock.
+ */
+IMG_BOOL
+PMR_IsZombie(const PMR *psPMR);
+
+/*
+ * PMRMarkForDeferFree
+ *
+ * Sets sync value required for this PMR to be freed.
+ */
+void
+PMRMarkForDeferFree(PMR *psPMR);
+
+/*
+ * PMRQueueZombiesForCleanup
+ *
+ * Defers cleanup of all zombie PMRs to the CleanupThread.
+ *
+ * Returns IMG_TRUE if any PMRs were queued for free and IMG_FALSE if no PMRs
+ * were queued.
+ */
+IMG_BOOL
+PMRQueueZombiesForCleanup(PPVRSRV_DEVICE_NODE psDevNode);
+
+/*
+ * PMRDequeueZombieAndRef
+ *
+ * Removed the PMR either form zombie list or cleanup item's list
+ * and references it.
+ */
+void
+PMRDequeueZombieAndRef(PMR *psPMR);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+/*
+ * PMR_ChangeSparseMemUnlocked()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * This function alters the memory map of the given PMR in device space by
+ * adding/deleting the pages as requested. PMR lock must be taken
+ * before calling this function.
+ *
+ */
+PVRSRV_ERROR PMR_ChangeSparseMemUnlocked(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32 uiSparseFlags);
+/*
+ * PMR_ChangeSparseMem()
+ *
+ * See note above about Lock/Unlock semantics.
+ *
+ * This function alters the memory map of the given PMR in device space by
+ *  adding/deleting the pages as requested.
+ *
+ */
+PVRSRV_ERROR PMR_ChangeSparseMem(PMR *psPMR,
+                                 IMG_UINT32 ui32AllocPageCount,
+                                 IMG_UINT32 *pai32AllocIndices,
+                                 IMG_UINT32 ui32FreePageCount,
+                                 IMG_UINT32 *pai32FreeIndices,
+                                 IMG_UINT32	uiSparseFlags);
+
+#if defined(PDUMP)
+
+/*
+ * PMR_PDumpSymbolicAddr()
+ *
+ * Given an offset, returns the pdump memspace name and symbolic
+ * address of the corresponding page in the PMR.
+ *
+ * Note that PDump memspace names and symbolic addresses are static
+ * and valid for the lifetime of the PMR, therefore we don't require
+ * acquire/release semantics here.
+ *
+ * Note that it is expected that the pdump "mapping" code will call
+ * this function multiple times as each page is mapped in turn
+ *
+ * Note that NextSymName is the offset from the base of the PMR to the
+ * next pdump symbolic address (or the end of the PMR if the PMR only
+ * had one PDUMPMALLOC
+ */
+PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32NamespaceNameLen,
+                      IMG_CHAR *pszNamespaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+		      IMG_DEVMEM_OFFSET_T *puiNextSymName
+                      );
+
+/*
+ * PMRPDumpLoadMemValue32()
+ *
+ * writes the current contents of a dword in PMR memory to the pdump
+ * script stream. Useful for patching a buffer by simply editing the
+ * script output file in ASCII plain text.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT32 ui32Value,
+                     PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpCopyMem32
+ *
+ * Adds in the pdump script stream a copy of a dword in one PMR memory
+ * location to another PMR memory location.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem32(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpLoadMemValue64()
+ *
+ * writes the current contents of a dword in PMR memory to the pdump
+ * script stream. Useful for patching a buffer by simply editing the
+ * script output file in ASCII plain text.
+ *
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT64 ui64Value,
+                     PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpCopyMem64
+ *
+ * Adds in the pdump script stream a copy of a quadword in one PMR memory
+ * location to another PMR memory location.
+ */
+PVRSRV_ERROR
+PMRPDumpCopyMem64(PMR *psDstPMR,
+                  IMG_DEVMEM_OFFSET_T uiDstLogicalOffset,
+                  PMR *psSrcPMR,
+                  IMG_DEVMEM_OFFSET_T uiSrcLogicalOffset,
+                  const IMG_CHAR *pszTmpVar,
+                  PDUMP_FLAGS_T uiPDumpFlags);
+
+/*
+ * PMRPDumpLoadMem()
+ *
+ * Writes the current contents of the PMR memory to the pdump PRM stream,
+ * and emits some PDump code to the script stream to LDB said bytes from
+ * said file. If bZero is IMG_TRUE then the PDump zero page is used as the
+ * source for the LDB.
+ */
+PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero);
+
+/*
+ * PMRPDumpSaveToFile()
+ *
+ * Emits some PDump that does an SAB (save bytes) using the PDump symbolic
+ * address of the PMR. Note that this is generally not the preferred way to
+ * dump the buffer contents. There is an equivalent function in
+ * devicemem_server.h which also emits SAB but using the virtual address,
+ * which is the "right" way to dump the buffer contents to a file.
+ * This function exists just to aid testing by providing a means to dump
+ * the PMR directly by symbolic address also.
+ */
+PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMR_PDumpSymbolicAddr)
+#endif
+static INLINE PVRSRV_ERROR
+PMR_PDumpSymbolicAddr(const PMR *psPMR,
+                      IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                      IMG_UINT32 ui32NamespaceNameLen,
+                      IMG_CHAR *pszNamespaceName,
+                      IMG_UINT32 ui32SymbolicAddrLen,
+                      IMG_CHAR *pszSymbolicAddr,
+                      IMG_DEVMEM_OFFSET_T *puiNewOffset,
+                      IMG_DEVMEM_OFFSET_T *puiNextSymName)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32NamespaceNameLen);
+	PVR_UNREFERENCED_PARAMETER(pszNamespaceName);
+	PVR_UNREFERENCED_PARAMETER(ui32SymbolicAddrLen);
+	PVR_UNREFERENCED_PARAMETER(pszSymbolicAddr);
+	PVR_UNREFERENCED_PARAMETER(puiNewOffset);
+	PVR_UNREFERENCED_PARAMETER(puiNextSymName);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMemValue32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMemValue32(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT32 ui32Value,
+                     PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMemValue64)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMemValue64(PMR *psPMR,
+			         IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                     IMG_UINT64 ui64Value,
+                     PDUMP_FLAGS_T uiPDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui64Value);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpLoadMem)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpLoadMem(PMR *psPMR,
+                IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                IMG_DEVMEM_SIZE_T uiSize,
+                PDUMP_FLAGS_T uiPDumpFlags,
+                IMG_BOOL bZero)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	PVR_UNREFERENCED_PARAMETER(bZero);
+	return PVRSRV_OK;
+}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpSaveToFile)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpSaveToFile(const PMR *psPMR,
+                   IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+                   IMG_DEVMEM_SIZE_T uiSize,
+                   IMG_UINT32 uiArraySize,
+                   const IMG_CHAR *pszFilename,
+                   IMG_UINT32 uiFileOffset)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(uiSize);
+	PVR_UNREFERENCED_PARAMETER(uiArraySize);
+	PVR_UNREFERENCED_PARAMETER(pszFilename);
+	PVR_UNREFERENCED_PARAMETER(uiFileOffset);
+	return PVRSRV_OK;
+}
+
+#endif	/* PDUMP */
+
+/* This function returns the private data that a pmr subtype embedded in
+ * here. We use the function table pointer as "authorisation" that this
+ * function is being called by the pmr subtype implementation. We can
+ * assume (assert) that. It would be a bug in the implementation of the
+ * pmr subtype if this assertion ever fails.
+ */
+void *
+PMRGetPrivateData(const PMR *psPMR,
+                  const PMR_IMPL_FUNCTAB *psFuncTab);
+
+PVRSRV_ERROR
+PMRZeroingPMR(PMR *psPMR,
+				IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize);
+
+PVRSRV_ERROR
+PMRDumpPageList(PMR *psReferencePMR,
+					IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize);
+
+PVRSRV_ERROR
+PMRWritePMPageList(/* Target PMR, offset, and length */
+                   PMR *psPageListPMR,
+                   IMG_DEVMEM_OFFSET_T uiTableOffset,
+                   IMG_DEVMEM_SIZE_T  uiTableLength,
+                   /* Referenced PMR, and "page" granularity */
+                   PMR *psReferencePMR,
+                   IMG_DEVMEM_LOG2ALIGN_T uiLog2PageSize,
+                   PMR_PAGELIST **ppsPageList);
+
+/* Doesn't actually erase the page list - just releases
+ * the appropriate refcounts
+ */
+PVRSRV_ERROR // should be void, surely
+PMRUnwritePMPageList(PMR_PAGELIST *psPageList);
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiFlags);
+
+PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiPDumpFlags);
+
+PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize);
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpPol32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpPol32(const PMR *psPMR,
+              IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+              IMG_UINT32 ui32Value,
+              IMG_UINT32 ui32Mask,
+              PDUMP_POLL_OPERATOR eOperator,
+              PDUMP_FLAGS_T uiFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpCheck32)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpCheck32(const PMR *psPMR,
+				IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+				IMG_UINT32 ui32Value,
+				IMG_UINT32 ui32Mask,
+				PDUMP_POLL_OPERATOR eOperator,
+				PDUMP_FLAGS_T uiFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiLogicalOffset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PMRPDumpCBP)
+#endif
+static INLINE PVRSRV_ERROR
+PMRPDumpCBP(const PMR *psPMR,
+            IMG_DEVMEM_OFFSET_T uiReadOffset,
+            IMG_DEVMEM_OFFSET_T uiWriteOffset,
+            IMG_DEVMEM_SIZE_T uiPacketSize,
+            IMG_DEVMEM_SIZE_T uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMR);
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif
+
+PPVRSRV_DEVICE_NODE PMRGetExportDeviceNode(PMR_EXPORT *psExportPMR);
+
+/*
+ * PMRInit()
+ *
+ * To be called once and only once to initialise the internal data in
+ * the PMR module (mutexes and such)
+ *
+ * Not for general use.  Only PVRSRVInit(); should be calling this.
+ */
+PVRSRV_ERROR
+PMRInit(void);
+
+/*
+ * PMRDeInit()
+ *
+ * To be called once and only once to deinitialise the internal data in
+ * the PMR module (mutexes and such) and for debug checks
+ *
+ * Not for general use.  Only PVRSRVDeInit(); should be calling this.
+ */
+PVRSRV_ERROR
+PMRDeInit(void);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+/*
+ * PMRInitDevice()
+ *
+ * Initialised device specific PMR data.
+ */
+PVRSRV_ERROR
+PMRInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRFreeZombies()
+ *
+ * Free deferred PMRs.
+ */
+void
+PMRFreeZombies(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRFreeZombies()
+ *
+ * Print all zombies to the log.
+ */
+void
+PMRDumpZombies(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*
+ * PMRDeInitDevice()
+ *
+ * Cleans up device specific PMR data.
+ */
+void
+PMRDeInitDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR
+PMRStoreRIHandle(PMR *psPMR, void *hRIHandle);
+#endif
+
+/*
+ * PMRLockPMR()
+ *
+ * To be called when the PMR must not be modified by any other call-stack.
+ * Acquires the mutex on the passed in PMR.
+ */
+void
+PMRLockPMR(PMR *psPMR);
+
+/*
+ * PMRUnlockPMR()
+ *
+ * To be called when the PMR is no longer being modified.
+ * Releases the per-PMR mutex.
+ */
+void
+PMRUnlockPMR(PMR *psPMR);
+
+#if defined(PVRSRV_INTERNAL_IPA_FEATURE_TESTING)
+PVRSRV_ERROR
+PMRModifyIPAPolicy(PMR *psPMR, IMG_UINT8 ui8NewIPAPolicy);
+
+PVRSRV_ERROR
+PMRGetIPAPolicy(PMR *psPMR, IMG_UINT8 *pui8IPAPolicy);
+
+PVRSRV_ERROR
+PMRGetIPAInfo(PMR *psPMR, IMG_UINT32 *pui32IPAPolicy, IMG_UINT32 *pui32IPAShift,
+              IMG_UINT32 *pui32IPAMask, IMG_UINT32 *pui32IPAFlagsValue);
+#endif
+
+#endif /* #ifdef SRVSRV_PMR_H */
diff --git a/drivers/gpu/drm/img-rogue/pmr_env.c b/drivers/gpu/drm/img-rogue/pmr_env.c
new file mode 100644
index 000000000000..e0c74b4b508d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pmr_env.c
@@ -0,0 +1,83 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux environment PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/string.h>
+
+#include "img_defs.h"
+
+#include "pmr_env.h"
+#include "physmem_dmabuf_internal.h"
+
+void
+PMREnvInitialize(PMR_ENV *psPMREnv)
+{
+	memset(psPMREnv, 0, sizeof(*psPMREnv));
+}
+
+void
+PMREnvFinalize(PMR_ENV *psPMREnv)
+{
+	void *pvDmaBufExportData;
+
+	pvDmaBufExportData = smp_load_acquire(&psPMREnv->pvDmaBufExportData);
+	if (pvDmaBufExportData)
+		PhysmemDmaBufExportFinalize(pvDmaBufExportData);
+}
+
+void
+PMREnvDmaBufSetExportData(PMR *psPMR, void *pvData)
+{
+	PMR_ENV *psPMREnv = PMREnvGetData(psPMR);
+
+	smp_store_release(&psPMREnv->pvDmaBufExportData, pvData);
+}
+
+void *
+PMREnvDmaBufGetExportData(PMR *psPMR)
+{
+	PMR_ENV *psPMREnv = PMREnvGetData(psPMR);
+	void *pvDmaBufExportData;
+
+	pvDmaBufExportData = smp_load_acquire(&psPMREnv->pvDmaBufExportData);
+
+	return pvDmaBufExportData;
+}
diff --git a/drivers/gpu/drm/img-rogue/pmr_env.h b/drivers/gpu/drm/img-rogue/pmr_env.h
new file mode 100644
index 000000000000..52a172e80b82
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pmr_env.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          Environment PMR functions and data
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Environment specific PMR functions and data
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PMR_ENV_H)
+#define PMR_ENV_H
+
+#include "pvrsrv_memallocflags.h"
+#include "powervr/mem_types.h"
+#include "pmr_impl.h"
+
+/* Structures, functions and definitions shared with the generic PMR code */
+struct _PMR_ENV_
+{
+	/*
+	 * The smp_store_release and smp_load_acquire functions are used
+	 * to access this field, in order to avoid taking locks, where
+	 * possible.
+	 */
+	void *pvDmaBufExportData;
+};
+
+typedef struct _PMR_ENV_ PMR_ENV;
+
+#define	DECLARE_PMR_ENV_DATA(name) PMR_ENV name;
+
+/* The generic PMR code calls these functions, via the macros that follow */
+void PMREnvInitialize(PMR_ENV *psPMREnv);
+void PMREnvFinalize(PMR_ENV *psPMREnv);
+
+#define	PMR_ENV_INITIALIZE(psPMR, sEnvData) PMREnvInitialize(&psPMR->sEnvData)
+#define	PMR_ENV_FINALIZE(psPMR, sEnvData) PMREnvFinalize(&psPMR->sEnvData)
+
+/* The generic PMR code provides this function, using the macro that follows */
+PMR_ENV *PMREnvGetData(PMR *psPMR);
+
+#define	DEFINE_PMR_ENV_GET_DATA(psPMR, sEnvData) \
+	PMR_ENV *PMREnvGetData(PMR *psPMR) { return &psPMR->sEnvData; }
+
+/*
+ * Structures, functions and definitions used by the environment specific
+ * PMR code.
+ */
+
+void
+PMREnvDmaBufSetExportData(PMR *psPMR, void *pvData);
+
+void *
+PMREnvDmaBufGetExportData(PMR *psPMR);
+
+#endif /* !defined(PMR_ENV_H) */
diff --git a/drivers/gpu/drm/img-rogue/pmr_impl.h b/drivers/gpu/drm/img-rogue/pmr_impl.h
new file mode 100644
index 000000000000..3c290ca3a6e7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pmr_impl.h
@@ -0,0 +1,605 @@
+/**************************************************************************/ /*!
+@File
+@Title          Implementation Callbacks for Physmem (PMR) abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Part of the memory management. This file is for definitions
+                that are private to the world of PMRs, but that need to be
+                shared between pmr.c itself and the modules that implement the
+                callbacks for the PMR.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVSRV_PMR_IMPL_H
+#define SRVSRV_PMR_IMPL_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+/*! Physical Memory Resource type.
+ */
+typedef struct _PMR_ PMR;
+
+/*! Per-flavour callbacks need to be shared with generic implementation
+ * (pmr.c).
+ */
+typedef void *PMR_IMPL_PRIVDATA;
+
+/*! Type for holding flags passed to the PMR factory.
+ */
+typedef PVRSRV_MEMALLOCFLAGS_T PMR_FLAGS_T;
+
+/*! Mapping table for the allocation.
+ *
+ * PMR's can be sparse in which case not all the logical addresses in it are
+ * valid. The mapping table translates logical offsets into physical offsets.
+ *
+ * This table is always passed to the PMR factory regardless if the memory is
+ * sparse or not. In case of non-sparse memory all virtual offsets are mapped
+ * to physical offsets.
+ */
+typedef struct _PMR_MAPPING_TABLE_ PMR_MAPPING_TABLE;
+
+/*! Private data passed to the ::PFN_MMAP_FN function.
+ */
+typedef void *PMR_MMAP_DATA;
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+typedef void *PMR_IMPL_ZOMBIEPAGES;
+#endif
+
+#define PMR_IMPL_TYPES \
+	X(NONE), \
+	X(OSMEM), \
+	X(LMA), \
+	X(DMABUF), \
+	X(EXTMEM), \
+	X(DC), \
+	X(TDFWMEM), \
+	X(TDSECBUF), \
+	X(LAST)
+
+/*! PMR factory type.
+ */
+typedef enum _PMR_IMPL_TYPE_
+{
+#define X(type) PMR_TYPE_##type
+	PMR_IMPL_TYPES
+#undef X
+} PMR_IMPL_TYPE;
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_LOCK_PHYS_ADDRESSES_FN
+
+@Description    Called to lock down the physical addresses for all pages
+                allocated for a PMR.
+                The default implementation is to simply increment a
+                lock-count for debugging purposes.
+                If overridden, the PFN_LOCK_PHYS_ADDRESSES_FN function will
+                be called when someone first requires a physical address,
+                and the PFN_UNLOCK_PHYS_ADDRESSES_FN counterpart will be
+                called when the last such reference is released.
+                The PMR implementation may assume that physical addresses
+                will have been "locked" in this manner before any call is
+                made to the pfnDevPhysAddr() callback
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_LOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_UNLOCK_PHYS_ADDRESSES_FN
+
+@Description    Called to release the lock taken on the physical addresses
+                for all pages allocated for a PMR.
+                The default implementation is to simply decrement a
+                lock-count for debugging purposes.
+                If overridden, the PFN_UNLOCK_PHYS_ADDRESSES_FN will be
+                called when the last reference taken on the PMR is
+                released.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Output         ppvZombiePages        Zombie pages object. If non-null is returned
+                                      caller is obligated to call pfnFreeZombiePages
+                                      at an appropriate time to prevent memory leaks.
+                                      If support for deferred freeing of pages is not
+                                      provided, the implementation must set
+                                      *ppvZombiePages to NULL.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+typedef PVRSRV_ERROR (*PFN_UNLOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                                                     PMR_IMPL_ZOMBIEPAGES *ppvZombiePages);
+#else
+typedef PVRSRV_ERROR (*PFN_UNLOCK_PHYS_ADDRESSES_FN)(PMR_IMPL_PRIVDATA pvPriv);
+#endif
+
+#if defined(SUPPORT_STATIC_IPA)
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_DEV_PHYS_ADDR_FN
+
+@Description    Called to obtain one or more physical addresses for given
+                offsets within a PMR.
+
+                The PFN_LOCK_PHYS_ADDRESSES_FN callback (if overridden) is
+                guaranteed to have been called prior to calling the
+                PFN_DEV_PHYS_ADDR_FN callback and the caller promises not to
+                rely on the physical address thus obtained after the
+                PFN_UNLOCK_PHYS_ADDRESSES_FN callback is called.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          ui32Log2PageSize      The log2 page size.
+@Input          ui32NumOfAddr         The number of addresses to be returned
+@Input          puiOffset             The offset from the start of the PMR
+                                      (in bytes) for which the physical
+                                      address is required. Where multiple
+                                      addresses are requested, this will
+                                      contain a list of offsets.
+@Input          ui64IPAPolicyValue    The Intermediate Physical Address (IPA)
+                                      Policy value to be applied to
+                                      the physical address
+@Input          ui64IPAClearMask      The IPA Clear mask to be applied to
+                                      the physical address when setting policy.
+@Output         pbValid               List of boolean flags indicating which
+                                      addresses in the returned list
+                                      (psDevAddrPtr) are valid (for sparse
+                                      allocations, not all pages may have a
+                                      physical backing)
+@Output         psDevAddrPtr          Returned list of physical addresses
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#else
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_DEV_PHYS_ADDR_FN
+
+@Description    Called to obtain one or more physical addresses for given
+                offsets within a PMR.
+
+                The PFN_LOCK_PHYS_ADDRESSES_FN callback (if overridden) is
+                guaranteed to have been called prior to calling the
+                PFN_DEV_PHYS_ADDR_FN callback and the caller promises not to
+                rely on the physical address thus obtained after the
+                PFN_UNLOCK_PHYS_ADDRESSES_FN callback is called.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          ui32Log2PageSize      The log2 page size.
+@Input          ui32NumOfAddr         The number of addresses to be returned
+@Input          puiOffset             The offset from the start of the PMR
+                                      (in bytes) for which the physical
+                                      address is required. Where multiple
+                                      addresses are requested, this will
+                                      contain a list of offsets.
+@Output         pbValid               List of boolean flags indicating which
+                                      addresses in the returned list
+                                      (psDevAddrPtr) are valid (for sparse
+                                      allocations, not all pages may have a
+                                      physical backing)
+@Output         psDevAddrPtr          Returned list of physical addresses
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+#endif
+typedef PVRSRV_ERROR (*PFN_DEV_PHYS_ADDR_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_UINT32 ui32Log2PageSize,
+                      IMG_UINT32 ui32NumOfAddr,
+                      IMG_DEVMEM_OFFSET_T *puiOffset,
+#if defined(SUPPORT_STATIC_IPA)
+                      IMG_UINT64 ui64IPAPolicyValue,
+                      IMG_UINT64 ui64IPAClearMask,
+#endif
+                      IMG_BOOL *pbValid,
+                      IMG_DEV_PHYADDR *psDevAddrPtr);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN
+
+@Description    Called to obtain a kernel-accessible address (mapped to a
+                virtual address if required) for the PMR for use internally
+                in Services.
+
+    Implementation of this function for the (default) PMR factory providing
+    OS-allocations is mandatory (the driver will expect to be able to call
+    this function for OS-provided allocations).
+    For other PMR factories, implementation of this function is only necessary
+    where an MMU mapping is required for the Kernel to be able to access the
+    allocated memory.
+    If no mapping is needed, this function can remain unimplemented and the
+    pfn may be set to NULL.
+@Input          pvPriv                Private data (which was generated by
+                                      the PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR
+                                      at which mapping is to start
+@Input          uiSize                Size of mapping (in bytes)
+@Output         ppvKernelAddressOut   Mapped kernel address
+@Output         phHandleOut           Returned handle of the new mapping
+@Input          ulFlags               Mapping flags
+
+@Return         PVRSRV_OK if the mapping was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      size_t uiOffset,
+                      size_t uiSize,
+                      void **ppvKernelAddressOut,
+                      IMG_HANDLE *phHandleOut,
+                      PMR_FLAGS_T ulFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_RELEASE_KERNEL_MAPPING_DATA_FN
+
+@Description    Called to release a mapped kernel virtual address
+
+   Implementation of this callback is mandatory if
+   PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN is provided for the PMR factory,
+   otherwise this function can remain unimplemented and the pfn may be set
+   to NULL.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          hHandle               Handle of the mapping to be released
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_RELEASE_KERNEL_MAPPING_DATA_FN)(PMR_IMPL_PRIVDATA pvPriv,
+              IMG_HANDLE hHandle);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_READ_BYTES_FN
+
+@Description    Called to read bytes from an unmapped allocation
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will use PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN to map the entire
+   PMR (if an MMU mapping is required for the Kernel to be able to access the
+   allocated memory).
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR at
+                                      which to begin reading
+@Output         pcBuffer              Buffer in which to return the read data
+@Input          uiBufSz               Number of bytes to be read
+@Output         puiNumBytes           Number of bytes actually read (may be
+                                      less than uiBufSz)
+
+@Return         PVRSRV_OK if the read was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_READ_BYTES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_UINT8 *pcBuffer,
+                      size_t uiBufSz,
+                      size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_WRITE_BYTES_FN
+
+@Description    Called to write bytes into an unmapped allocation
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will use PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN to map the entire
+   PMR (if an MMU mapping is required for the Kernel to be able to access the
+   allocated memory).
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          uiOffset              Offset from the beginning of the PMR at
+                                      which to begin writing
+@Input          pcBuffer              Buffer containing the data to be written
+@Input          uiBufSz               Number of bytes to be written
+@Output         puiNumBytes           Number of bytes actually written (may be
+                                      less than uiBufSz)
+
+@Return         PVRSRV_OK if the write was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_WRITE_BYTES_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                      IMG_DEVMEM_OFFSET_T uiOffset,
+                      IMG_UINT8 *pcBuffer,
+                      size_t uiBufSz,
+                      size_t *puiNumBytes);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_CHANGE_SPARSE_MEM_FN
+
+@Description    Called to modify the physical backing for a given sparse
+                allocation.
+                The caller provides a list of the pages within the sparse
+                allocation which should be backed with a physical allocation
+                and a list of the pages which do not require backing.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the sparse allocation to be
+                                      modified
+@Input          ui32AllocPageCount    The number of pages specified in
+                                      pai32AllocIndices
+@Input          pai32AllocIndices     The list of pages in the sparse
+                                      allocation that should be backed with a
+                                      physical allocation. Pages are
+                                      referenced by their index within the
+                                      sparse allocation (e.g. in a 10 page
+                                      allocation, pages are denoted by
+                                      indices 0 to 9)
+@Input          ui32FreePageCount     The number of pages specified in
+                                      pai32FreeIndices
+@Input          pai32FreeIndices      The list of pages in the sparse
+                                      allocation that do not require
+                                      a physical allocation.
+@Input          ui32Flags             Allocation flags
+@Output         ppvZombiePages        Zombie pages object. If non-null is returned
+                                      caller is obligated to call pfnFreeZombiePages
+                                      at an appropriate time to prevent memory leaks
+
+@Return         PVRSRV_OK if the sparse allocation physical backing was updated
+                successfully, an error code otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CHANGE_SPARSE_MEM_FN)(PMR_IMPL_PRIVDATA pPriv,
+                      const PMR *psPMR,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices,
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+                      PMR_IMPL_ZOMBIEPAGES *ppvZombiePages,
+#endif
+                      IMG_UINT32 uiFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_CHANGE_SPARSE_MEM_CPU_MAP_FN
+
+@Description    Called to modify which pages are mapped for a given sparse
+                allocation.
+                The caller provides a list of the pages within the sparse
+                allocation which should be given a CPU mapping and a list
+                of the pages which do not require a CPU mapping.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the sparse allocation to be
+                                      modified
+@Input          sCpuVAddrBase         The virtual base address of the sparse
+                                      allocation
+@Input          ui32AllocPageCount    The number of pages specified in
+                                      pai32AllocIndices
+@Input          pai32AllocIndices     The list of pages in the sparse
+                                      allocation that should be given a CPU
+                                      mapping. Pages are referenced by their
+                                      index within the sparse allocation (e.g.
+                                      in a 10 page allocation, pages are
+                                      denoted by indices 0 to 9)
+@Input          ui32FreePageCount     The number of pages specified in
+                                      pai32FreeIndices
+@Input          pai32FreeIndices      The list of pages in the sparse
+                                      allocation that do not require a CPU
+                                      mapping.
+
+@Return         PVRSRV_OK if the page mappings were updated successfully, an
+                error code otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_CHANGE_SPARSE_MEM_CPU_MAP_FN)(PMR_IMPL_PRIVDATA pPriv,
+                      const PMR *psPMR,
+                      IMG_UINT64 sCpuVAddrBase,
+                      IMG_UINT32 ui32AllocPageCount,
+                      IMG_UINT32 *pai32AllocIndices,
+                      IMG_UINT32 ui32FreePageCount,
+                      IMG_UINT32 *pai32FreeIndices);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_MMAP_FN
+
+@Description    Called to map pages in the specified PMR.
+
+   Implementation of this callback is optional.
+   Where it is provided, it will be used in place of OSMMapPMRGeneric().
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+@Input          psPMR                 The PMR of the allocation to be mapped
+@Input          pMMapData             OS-specific data to describe how mapping
+                                      should be performed
+
+@Return         PVRSRV_OK if the mapping was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_MMAP_FN)(PMR_IMPL_PRIVDATA pPriv,
+                                    PMR *psPMR,
+                                    PMR_MMAP_DATA pMMapData);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_FINALIZE_FN
+
+@Description    Called to destroy the PMR.
+                This callback will be called only when all references to
+                the PMR have been dropped.
+                The PMR was created via a call to PhysmemNewRamBackedPMR()
+                and is destroyed via this callback.
+
+   Implementation of this callback is mandatory.
+
+@Input          pvPriv                Private data (which was generated by the
+                                      PMR factory when PMR was created)
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_FINALIZE_FN)(PMR_IMPL_PRIVDATA pvPriv);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ACQUIRE_PMR_FACTORY_LOCK_FN
+
+@Description    Called to acquire the PMR factory's global lock, if it has one,
+                hence callback optional. Factories which support entry points
+                in addition to the normal bridge calls, for example, from the
+                native OS that manipulate the PMR reference count should
+                create a factory lock and implementations for these call backs.
+
+   Implementation of this callback is optional.
+
+@Return         None
+*/
+/*****************************************************************************/
+typedef void (*PFN_ACQUIRE_PMR_FACTORY_LOCK_FN)(void);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_RELEASE_PMR_FACTORY_LOCK_FN
+
+@Description    Called to release the PMR factory's global lock acquired by calling
+                pfn_acquire_pmr_factory_lock callback.
+
+   Implementation of this callback is optional.
+
+@Return         None
+*/ /**************************************************************************/
+typedef void (*PFN_RELEASE_PMR_FACTORY_LOCK_FN)(void);
+
+#ifdef SUPPORT_PMR_DEFERRED_FREE
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_ZOMBIFY_FN
+
+@Description    Called to perform factory actions necessary when PMR becomes
+                a zombie PMR.
+
+                This function should at least adjust the driver/process memory
+                stats to reflect the amount of memory is occupied by the zombie
+                PMRs and at the same time subtract the memory from the main
+                memory stat the pages are accounted under.
+
+   Implementation of this callback is required when SUPPORT_PMR_DEFERRED_FREE=1.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_ZOMBIFY_FN)(PMR_IMPL_PRIVDATA pvPriv,
+                                       PMR *psPMR);
+#endif
+
+#ifdef SUPPORT_PMR_PAGES_DEFERRED_FREE
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_FREE_ZOMBIE_PAGES_FN
+
+@Description    Called to perform factory actions to free zombie pages object
+                previously returned by PFN_CHANGE_SPARSE_MEM_FN.
+
+                This function should free the pages described in the
+                pvZombiePages parameter and do any associated actions related
+                to freeing such as poisoning or returning to the page pool.
+
+                Implementation of this callback is required when
+                SUPPORT_PMR_PAGES_DEFERRED_FREE=1.
+
+@Return         PVRSRV_OK if the operation was successful, an error code
+                otherwise. If error is returned, the PMR layer might retry.
+                On error, factory implementations should modify the contents
+                of the PMR_IMPL_ZOMBIEPAGES object reflecting any changes in
+                underlying memory as a result of the initial (failed) call.
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_FREE_ZOMBIE_PAGES_FN)(PMR_IMPL_ZOMBIEPAGES pvZombiePages);
+#endif
+
+/*! PMR factory callback table.
+ */
+struct _PMR_IMPL_FUNCTAB_ {
+    /*! Callback function pointer, see ::PFN_LOCK_PHYS_ADDRESSES_FN */
+    PFN_LOCK_PHYS_ADDRESSES_FN pfnLockPhysAddresses;
+    /*! Callback function pointer, see ::PFN_UNLOCK_PHYS_ADDRESSES_FN */
+    PFN_UNLOCK_PHYS_ADDRESSES_FN pfnUnlockPhysAddresses;
+
+    /*! Callback function pointer, see ::PFN_DEV_PHYS_ADDR_FN */
+    PFN_DEV_PHYS_ADDR_FN pfnDevPhysAddr;
+
+    /*! Callback function pointer, see ::PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN */
+    PFN_ACQUIRE_KERNEL_MAPPING_DATA_FN pfnAcquireKernelMappingData;
+    /*! Callback function pointer, see ::PFN_RELEASE_KERNEL_MAPPING_DATA_FN */
+    PFN_RELEASE_KERNEL_MAPPING_DATA_FN pfnReleaseKernelMappingData;
+
+    /*! Callback function pointer, see ::PFN_READ_BYTES_FN */
+    PFN_READ_BYTES_FN pfnReadBytes;
+    /*! Callback function pointer, see ::PFN_WRITE_BYTES_FN */
+    PFN_WRITE_BYTES_FN pfnWriteBytes;
+
+    /*! Callback function pointer, see ::PFN_CHANGE_SPARSE_MEM_FN */
+    PFN_CHANGE_SPARSE_MEM_FN pfnChangeSparseMem;
+
+#ifdef SUPPORT_PMR_PAGES_DEFERRED_FREE
+    /*! Callback function pointer, see ::PFN_FREE_ZOMBIE_PAGES_FN */
+    PFN_FREE_ZOMBIE_PAGES_FN pfnFreeZombiePages;
+#endif
+
+    /*! Callback function pointer, see ::PFN_MMAP_FN */
+    PFN_MMAP_FN pfnMMap;
+
+    /*! Callback function pointer, see ::PFN_FINALIZE_FN */
+    PFN_FINALIZE_FN pfnFinalize;
+
+    /*! Callback function pointer, see ::PFN_ACQUIRE_PMR_FACTORY_LOCK_FN */
+    PFN_ACQUIRE_PMR_FACTORY_LOCK_FN pfnGetPMRFactoryLock;
+
+    /*! Callback function pointer, see ::PFN_RELEASE_PMR_FACTORY_LOCK_FN */
+    PFN_RELEASE_PMR_FACTORY_LOCK_FN pfnReleasePMRFactoryLock;
+
+#ifdef SUPPORT_PMR_DEFERRED_FREE
+    /*! Callback function pointer, see ::PFN_ZOMBIFY_FN */
+    PFN_ZOMBIFY_FN pfnZombify;
+#endif
+};
+
+/*! PMR factory callback table.
+ */
+typedef struct _PMR_IMPL_FUNCTAB_ PMR_IMPL_FUNCTAB;
+
+#endif /* SRVSRV_PMR_IMPL_H */
diff --git a/drivers/gpu/drm/img-rogue/pmr_os.c b/drivers/gpu/drm/img-rogue/pmr_os.c
new file mode 100644
index 000000000000..32dfab3ae8ab
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pmr_os.c
@@ -0,0 +1,617 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux OS PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <asm/page.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/version.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+#include <linux/pfn_t.h>
+#endif
+#include <linux/pfn.h>
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "allocmem.h"
+#include "devicemem_server_utils.h"
+#include "pmr.h"
+#include "pmr_os.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+#include "mmap_stats.h"
+#endif
+
+#include "kernel_compatibility.h"
+
+/*
+ * x86_32:
+ * Use vm_insert_page because remap_pfn_range has issues when mapping HIGHMEM
+ * pages with default memory attributes; these HIGHMEM pages are skipped in
+ * set_pages_array_[uc,wc] during allocation; see reserve_pfn_range().
+ * Also vm_insert_page is faster.
+ *
+ * x86_64:
+ * Use vm_insert_page because it is faster.
+ *
+ * Other platforms:
+ * Use remap_pfn_range by default because it does not issue a cache flush.
+ * It is known that ARM32 benefits from this. When other platforms become
+ * available it has to be investigated if this assumption holds for them as well.
+ *
+ * Since vm_insert_page does more precise memory accounting we have the build
+ * flag PVR_MMAP_USE_VM_INSERT that forces its use. This is useful as a debug
+ * feature.
+ *
+ */
+#if defined(CONFIG_X86) || defined(PVR_MMAP_USE_VM_INSERT)
+#define PMR_OS_USE_VM_INSERT_PAGE 1
+#endif
+
+static void MMapPMROpen(struct vm_area_struct *ps_vma)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	PVRSRV_ERROR eError;
+
+	/* VM_DONTCOPY should ensure this function never gets called */
+	PVR_DPF((PVR_DBG_WARNING,
+			 "%s: Unexpected mmap open call, this is probably an application bug.",
+			 __func__));
+	PVR_DPF((PVR_DBG_WARNING,
+			 "%s: vma struct: 0x%p, vAddr: %#lX, length: %#lX, PMR pointer: 0x%p",
+			 __func__,
+			 ps_vma,
+			 ps_vma->vm_start,
+			 ps_vma->vm_end - ps_vma->vm_start,
+			 psPMR));
+
+	/* Should this entry-point be called for one of our PMRs we must increase
+	 * the refcount and lock down the physical addresses.
+	 */
+	PMRRefPMR(psPMR);
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_LOG_ERROR(eError, "PMRLockSysPhysAddresses");
+		PMRUnrefPMR(psPMR);
+	}
+	else
+	{
+		/* MMapPMROpen() is call when a process is forked, but only if
+		 * mappings are to be inherited so increment mapping count of the
+		 * PMR to prevent its layout cannot be changed (if sparse).
+		 */
+		PMRCpuMapCountIncr(psPMR);
+	}
+}
+
+static void MMapPMRClose(struct vm_area_struct *ps_vma)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		uintptr_t vAddr = ps_vma->vm_start;
+
+		while (vAddr < ps_vma->vm_end)
+		{
+			/* USER MAPPING */
+			PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+			                                (IMG_UINT64)vAddr,
+			                                OSGetCurrentClientProcessIDKM());
+			vAddr += PAGE_SIZE;
+		}
+	}
+#else
+	PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+	                            ps_vma->vm_end - ps_vma->vm_start,
+	                            OSGetCurrentClientProcessIDKM());
+#endif
+#endif
+
+	PMRUnlockSysPhysAddresses(psPMR);
+	/* Decrement the mapping count before Unref of PMR (as Unref could destroy the PMR) */
+	PMRCpuMapCountDecr(psPMR);
+	PMRUnrefPMR(psPMR);
+}
+
+/*
+ * This vma operation is used to read data from mmap regions. It is called
+ * by access_process_vm, which is called to handle PTRACE_PEEKDATA ptrace
+ * requests and reads from /proc/<pid>/mem.
+ */
+static int MMapVAccess(struct vm_area_struct *ps_vma, unsigned long addr,
+		       void *buf, int len, int write)
+{
+	PMR *psPMR = ps_vma->vm_private_data;
+	unsigned long ulOffset = addr - ps_vma->vm_start;
+	size_t uiBytesCopied;
+	PVRSRV_ERROR eError;
+	int iRetVal = -EINVAL;
+
+	if (write)
+	{
+		eError = PMR_WriteBytes(psPMR,
+					(IMG_DEVMEM_OFFSET_T) ulOffset,
+					buf,
+					len,
+					&uiBytesCopied);
+	}
+	else
+	{
+		eError = PMR_ReadBytes(psPMR,
+				       (IMG_DEVMEM_OFFSET_T) ulOffset,
+				       buf,
+				       len,
+				       &uiBytesCopied);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Error from %s (%d)",
+			 __func__,
+			 write ? "PMR_WriteBytes" : "PMR_ReadBytes",
+			 eError));
+	}
+	else
+	{
+		iRetVal = uiBytesCopied;
+	}
+
+	return iRetVal;
+}
+
+static const struct vm_operations_struct gsMMapOps =
+{
+	.open = &MMapPMROpen,
+	.close = &MMapPMRClose,
+	.access = MMapVAccess,
+};
+
+static INLINE int _OSMMapPMR(PVRSRV_DEVICE_NODE *psDevNode,
+							struct vm_area_struct *ps_vma,
+							IMG_DEVMEM_OFFSET_T uiOffset,
+							IMG_CPU_PHYADDR *psCpuPAddr,
+							IMG_UINT32 uiLog2PageSize,
+							IMG_BOOL bUseVMInsertPage,
+							IMG_BOOL bUseMixedMap)
+{
+	IMG_INT32 iStatus;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	IMG_UINT64 sPFN;
+
+	sPFN = PHYS_PFN(psCpuPAddr->uiAddr);
+#else
+	pfn_t sPFN;
+
+	sPFN = phys_to_pfn_t(psCpuPAddr->uiAddr, 0);
+#endif
+
+	/*
+	 * vm_insert_page() allows insertion of individual pages into user
+	 * VMA space _only_ if page is a order-zero allocated page
+	 */
+	if (bUseVMInsertPage)
+	{
+		if (bUseMixedMap)
+		{
+			/*
+			 * This path is just for debugging. It should be
+			 * equivalent to the remap_pfn_range() path.
+			 */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0))
+			vm_fault_t vmf;
+
+			vmf = vmf_insert_mixed(ps_vma,
+									ps_vma->vm_start + uiOffset,
+									sPFN);
+			if (vmf & VM_FAULT_ERROR)
+			{
+				iStatus = vm_fault_to_errno(vmf, 0);
+			}
+			else
+			{
+				iStatus = 0;
+			}
+#else
+			iStatus = vm_insert_mixed(ps_vma,
+									  ps_vma->vm_start + uiOffset,
+									  sPFN);
+#endif
+		}
+		else
+		{
+			/* Since kernel 3.7 this sets VM_MIXEDMAP internally */
+			iStatus = vm_insert_page(ps_vma,
+									 ps_vma->vm_start + uiOffset,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+									 pfn_to_page(sPFN));
+#else
+									 pfn_t_to_page(sPFN));
+#endif
+		}
+	}
+	else
+	{
+		/*
+		   NOTE: Regarding absence of dma_mmap_coherent() in _OSMMapPMR()
+
+		   The current services mmap model maps in a PMR's full-length size
+		   into the user VMA & applies any user specified offset to the kernel
+		   returned zero-offset based VA in services client; this essentially
+		   means services server ignores ps_vma->vm_pgoff (this houses hPMR)
+		   during a mmap call.
+
+		   Furthermore, during a DMA/CMA memory allocation, multiple order-n
+		   pages are used to satisfy an allocation request due to DMA/CMA
+		   framework rounding-up allocation size to next power-of-two which
+		   can lead to wasted memory (so we don't allocate using single call).
+
+		   The combination of the above two issues mean that we cannot use the
+		   dma_mmap_coherent() for a number of reasons outlined below:
+
+		     - Services mmap semantics does not fit with dma_mmap_coherent()
+		       which requires proper ps_vma->vm_pgoff; seeing this houses a
+		       hPMR handle value, calls into dma_mmap_coherent() fails. This
+		       could be avoided by forcing ps_vma->vm_pgoff to zero but the
+		       ps_vma->vm_pgoff is applied to DMA bus address PFN and not
+		       user VMA which is always mapped at ps_vma->vm_start.
+
+		     - As multiple order-n pages are used for DMA/CMA allocations, a
+		       single dma_mmap_coherent() call with a vma->vm_pgoff set to
+		       zero cannot (maybe) be used because there is no guarantee that
+		       all of the multiple order-n pages in the PMR are physically
+		       contiguous from the first entry to the last. Whilst this is
+		       highly likely to be the case, there is no guarantee that it
+		       will be so we cannot depend on this being the case.
+
+		   The solution is to manually mmap DMA/CMA pages into user VMA
+		   using remap_pfn_range() directly. Furthermore, accounting is
+		   always compromised for DMA/CMA allocations.
+		*/
+		size_t uiNumContiguousBytes = 1ULL << uiLog2PageSize;
+
+		iStatus = remap_pfn_range(ps_vma,
+								  ps_vma->vm_start + uiOffset,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+								  sPFN,
+#else
+								  pfn_t_to_pfn(sPFN),
+#endif
+								  uiNumContiguousBytes,
+								  ps_vma->vm_page_prot);
+	}
+
+	return iStatus;
+}
+
+PVRSRV_ERROR
+OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData)
+{
+	struct vm_area_struct *ps_vma = pOSMMapData;
+	PVRSRV_DEVICE_NODE *psDevNode = PMR_DeviceNode(psPMR);
+	PVRSRV_ERROR eError;
+	size_t uiLength;
+	IMG_INT32 iStatus;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_UINT32 ui32CPUCacheFlags;
+	pgprot_t sPageProt;
+	IMG_CPU_PHYADDR asCpuPAddr[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_BOOL abValid[PMR_MAX_TRANSLATION_STACK_ALLOC];
+	IMG_UINT32 uiOffsetIdx;
+	IMG_UINT32 uiNumOfPFNs;
+	IMG_UINT32 uiLog2PageSize;
+	IMG_CPU_PHYADDR *psCpuPAddr;
+	IMG_BOOL *pbValid;
+	IMG_BOOL bUseMixedMap = IMG_FALSE;
+	IMG_BOOL bUseVMInsertPage = IMG_FALSE;
+	vm_flags_t uVMFlags = ps_vma->vm_flags;
+
+	/* if writeable but not shared mapping is requested then fail */
+	PVR_RETURN_IF_INVALID_PARAM(((uVMFlags & VM_WRITE) == 0) ||
+	                            ((uVMFlags & VM_SHARED) != 0));
+
+	uiLength = ps_vma->vm_end - ps_vma->vm_start;
+
+	/* Check early if the requested mapping size doesn't exceed the virtual
+	 * PMR size. */
+	if (PMR_LogicalSize(psPMR) < uiLength)
+	{
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, ErrReturn);
+	}
+
+	uiLog2PageSize = PMR_GetLog2Contiguity(psPMR);
+
+	/* Check the number of PFNs to be mapped is valid. */
+	uiNumOfPFNs = uiLength >> uiLog2PageSize;
+	if (uiNumOfPFNs == 0)
+	{
+		/* print as 64-bit value to avoid Smatch warning */
+		PVR_LOG_VA(PVR_DBG_ERROR,
+		           "uiLength is invalid. Must be >= %" IMG_UINT64_FMTSPEC ".",
+		           IMG_UINT64_C(1) << uiLog2PageSize);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BAD_MAPPING, ErrReturn);
+	}
+
+	/*
+	 * Take a reference on the PMR so that it can't be freed while mapped
+	 * into the user process.
+	 */
+	PMRRefPMR(psPMR);
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrUnrefPMR;
+	}
+
+	/* Increment mapping count of the PMR so that its layout cannot be
+	 * changed (if sparse).
+	 */
+	PMRLockPMR(psPMR);
+
+	PMRCpuMapCountIncr(psPMR);
+	sPageProt = vm_get_page_prot(uVMFlags);
+
+	eError = DevmemCPUCacheMode(PMR_Flags(psPMR), &ui32CPUCacheFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrUnlockPhysAddr;
+	}
+
+	switch (ui32CPUCacheFlags)
+	{
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED:
+				sPageProt = pgprot_noncached(sPageProt);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC:
+				sPageProt = pgprot_writecombine(sPageProt);
+				break;
+
+		case PVRSRV_MEMALLOCFLAG_CPU_CACHED:
+		{
+/* Do not set to write-combine for plato */
+#if !defined(PLATO_MEMORY_CONFIG)
+				PHYS_HEAP *psPhysHeap = PMR_PhysHeap(psPMR);
+
+				if (PhysHeapGetType(psPhysHeap) == PHYS_HEAP_TYPE_LMA)
+					sPageProt = pgprot_writecombine(sPageProt);
+#endif
+				break;
+		}
+
+		default:
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				goto ErrUnlockPhysAddr;
+	}
+	ps_vma->vm_page_prot = sPageProt;
+
+	uVMFlags |= VM_IO;
+
+	/* Don't include the mapping in core dumps */
+	uVMFlags |= VM_DONTDUMP;
+
+	/*
+	 * Disable mremap because our nopage handler assumes all
+	 * page requests have already been validated.
+	 */
+	uVMFlags |= VM_DONTEXPAND;
+
+	/* Don't allow mapping to be inherited across a process fork */
+	uVMFlags |= VM_DONTCOPY;
+
+#if defined(PMR_OS_USE_VM_INSERT_PAGE)
+	/* Is this mmap targeting non order-zero pages or does it use pfn mappings?
+	 * If yes, don't use vm_insert_page */
+	bUseVMInsertPage = (uiLog2PageSize == PAGE_SHIFT) && (PMR_GetType(psPMR) != PMR_TYPE_EXTMEM);
+#endif
+
+	/* Can we use stack allocations */
+	if (uiNumOfPFNs > PMR_MAX_TRANSLATION_STACK_ALLOC)
+	{
+		psCpuPAddr = OSAllocMem(uiNumOfPFNs * sizeof(*psCpuPAddr));
+		PVR_LOG_GOTO_IF_NOMEM(psCpuPAddr, eError, ErrUnlockPhysAddr);
+
+		/* Should allocation fail, clean-up here before exiting */
+		pbValid = OSAllocMem(uiNumOfPFNs * sizeof(*pbValid));
+		PVR_LOG_GOTO_IF_NOMEM(pbValid, eError, ErrFreeCpuPAddr);
+	}
+	else
+	{
+		psCpuPAddr = asCpuPAddr;
+		pbValid = abValid;
+	}
+
+	/* Obtain map range pfns */
+	eError = PMR_CpuPhysAddr(psPMR,
+				 uiLog2PageSize,
+				 uiNumOfPFNs,
+				 0,
+				 psCpuPAddr,
+				 pbValid);
+	if (eError != PVRSRV_OK)
+	{
+		goto ErrFreeValid;
+	}
+
+	/*
+	 * Scan the map range for pfns without struct page* handling. If
+	 * we find one, this is a mixed map, and we can't use vm_insert_page()
+	 * NOTE: vm_insert_page() allows insertion of individual pages into user
+	 * VMA space _only_ if said page is an order-zero allocated page.
+	 */
+	if (bUseVMInsertPage)
+	{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		IMG_UINT64 sPFN;
+#else
+		pfn_t sPFN;
+#endif
+		for (uiOffsetIdx = 0; uiOffsetIdx < uiNumOfPFNs; ++uiOffsetIdx)
+		{
+			if (pbValid[uiOffsetIdx])
+			{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+				sPFN = PHYS_PFN(psCpuPAddr[uiOffsetIdx].uiAddr);
+
+				if (!pfn_valid(sPFN) || page_count(pfn_to_page(sPFN)) == 0)
+#else
+				sPFN = phys_to_pfn_t(psCpuPAddr[uiOffsetIdx].uiAddr, 0);
+
+				if (!pfn_t_valid(sPFN) || page_count(pfn_t_to_page(sPFN)) == 0)
+#endif
+				{
+					bUseMixedMap = IMG_TRUE;
+					break;
+				}
+			}
+		}
+
+		if (bUseMixedMap)
+		{
+			uVMFlags |= VM_MIXEDMAP;
+		}
+	}
+	else
+	{
+		uVMFlags |= VM_PFNMAP;
+	}
+
+	/* Actually initialise the flags */
+	pvr_vm_flags_init(ps_vma, uVMFlags);
+
+	/* For each PMR page-size contiguous bytes, map page(s) into user VMA */
+	for (uiOffset = 0; uiOffset < uiLength; uiOffset += 1ULL<<uiLog2PageSize)
+	{
+		uiOffsetIdx = uiOffset >> uiLog2PageSize;
+		/*
+		 * Only map in pages that are valid, any that aren't will be
+		 * picked up by the nopage handler which will return a zeroed
+		 * page for us.
+		 */
+		if (pbValid[uiOffsetIdx])
+		{
+			iStatus = _OSMMapPMR(psDevNode,
+								 ps_vma,
+								 uiOffset,
+								 &psCpuPAddr[uiOffsetIdx],
+								 uiLog2PageSize,
+								 bUseVMInsertPage,
+								 bUseMixedMap);
+			if (iStatus)
+			{
+				/* Failure error code doesn't get propagated */
+				eError = PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED;
+				PVR_ASSERT(0);
+				goto ErrFreeValid;
+			}
+		}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && defined(PVRSRV_ENABLE_MEMORY_STATS)
+#define PMR_OS_BAD_CPUADDR 0x0BAD0BAD
+		{
+			IMG_CPU_PHYADDR sPAddr;
+			sPAddr.uiAddr = pbValid[uiOffsetIdx] ?
+					psCpuPAddr[uiOffsetIdx].uiAddr :
+					IMG_CAST_TO_CPUPHYADDR_UINT(PMR_OS_BAD_CPUADDR);
+
+			PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,
+										(void*)(uintptr_t)(ps_vma->vm_start + uiOffset),
+										sPAddr,
+										1<<uiLog2PageSize,
+										OSGetCurrentClientProcessIDKM()
+										DEBUG_MEMSTATS_VALUES);
+		}
+#undef PMR_OS_BAD_CPUADDR
+#endif
+	}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS) && !defined(PVRSRV_ENABLE_MEMORY_STATS)
+	PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES, uiNumOfPFNs * PAGE_SIZE, OSGetCurrentClientProcessIDKM());
+#endif
+
+	if (psCpuPAddr != asCpuPAddr)
+	{
+		OSFreeMem(psCpuPAddr);
+		OSFreeMem(pbValid);
+	}
+
+	/* let us see the PMR so we can unlock it later */
+	ps_vma->vm_private_data = psPMR;
+
+	/* Install open and close handlers for ref-counting */
+	ps_vma->vm_ops = &gsMMapOps;
+
+#if defined(PVRSRV_ENABLE_LINUX_MMAP_STATS)
+	/* record the stats */
+	MMapStatsAddOrUpdatePMR(psPMR, uiLength);
+#endif
+
+	PMRUnlockPMR(psPMR);
+	return PVRSRV_OK;
+
+	/* Error exit paths follow */
+ErrFreeValid:
+	if (pbValid != abValid)
+	{
+		OSFreeMem(pbValid);
+	}
+ErrFreeCpuPAddr:
+	if (psCpuPAddr != asCpuPAddr)
+	{
+		OSFreeMem(psCpuPAddr);
+	}
+ErrUnlockPhysAddr:
+	PMRCpuMapCountDecr(psPMR);
+	PMRUnlockPMR(psPMR);
+	PMRUnlockSysPhysAddresses(psPMR);
+ErrUnrefPMR:
+	PMRUnrefPMR(psPMR);
+ErrReturn:
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/pmr_os.h b/drivers/gpu/drm/img-rogue/pmr_os.h
new file mode 100644
index 000000000000..7b4a2117091c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pmr_os.h
@@ -0,0 +1,62 @@
+/*************************************************************************/ /*!
+@File
+@Title          OS PMR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    OS specific PMR functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PMR_OS_H)
+#define PMR_OS_H
+
+#include "pmr_impl.h"
+
+/*************************************************************************/ /*!
+@Function       OSMMapPMRGeneric
+@Description    Implements a generic PMR mapping function, which is used
+                to CPU map a PMR where the PMR does not have a mapping
+                function defined by the creating PMR factory.
+@Input          psPMR               the PMR to be mapped
+@Output         pOSMMapData         pointer to any private data
+                                    needed by the generic mapping function
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+OSMMapPMRGeneric(PMR *psPMR, PMR_MMAP_DATA pOSMMapData);
+
+#endif /* !defined(PMR_OS_H) */
diff --git a/drivers/gpu/drm/img-rogue/power.c b/drivers/gpu/drm/img-rogue/power.c
new file mode 100644
index 000000000000..51bdcc3a7836
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/power.c
@@ -0,0 +1,1458 @@
+/*************************************************************************/ /*!
+@File           power.c
+@Title          Power management functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for power management functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if !defined(__linux__)
+#include <errno.h>
+#endif /* #if !defined(__linux__) */
+
+#include "pdump_km.h"
+#include "allocmem.h"
+#include "osfunc.h"
+
+#include "lock.h"
+#include "pvrsrv.h"
+#include "power.h"
+#include "pvr_debug.h"
+#include "htbserver.h"
+#include "di_server.h"
+
+#define OS_POWERLOCK_TIMEOUT_MS		(5000UL)
+#define OS_POWERLOCK_TIMEOUT_US		(OS_POWERLOCK_TIMEOUT_MS * 1000UL)
+#define OS_POWERLOCK_RETRY_COUNT	(50)
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+typedef struct _EXTRA_POWER_STATS_
+{
+	IMG_UINT64	ui64PreClockSpeedChangeDuration;
+	IMG_UINT64	ui64BetweenPreEndingAndPostStartingDuration;
+	IMG_UINT64	ui64PostClockSpeedChangeDuration;
+} EXTRA_POWER_STATS;
+
+/* For the power timing stats we need 16 variables to store all the
+ * combinations of forced/not forced, power-on/power-off, pre-power/post-power
+ * and device/system statistics
+ */
+#define NUM_POWER_STATS        (16)
+#define NUM_EXTRA_POWER_STATS	10
+
+typedef struct PVRSRV_POWER_STATS_TAG
+{
+	EXTRA_POWER_STATS               asClockSpeedChanges[NUM_EXTRA_POWER_STATS];
+	IMG_UINT64                      ui64PreClockSpeedChangeMark;
+	IMG_UINT64                      ui64FirmwareIdleDuration;
+	IMG_UINT32                      aui32PowerTimingStats[NUM_POWER_STATS];
+	IMG_UINT32                      ui32ClockSpeedIndexStart;
+	IMG_UINT32                      ui32ClockSpeedIndexEnd;
+	IMG_UINT32                      ui32FirmwareStartTimestamp;
+} PVRSRV_POWER_STATS;
+#endif
+
+struct _PVRSRV_POWER_DEV_TAG_
+{
+	PFN_PRE_POWER					pfnDevicePrePower;
+	PFN_POST_POWER					pfnDevicePostPower;
+	PFN_SYS_PRE_POWER				pfnSystemPrePower;
+	PFN_SYS_POST_POWER				pfnSystemPostPower;
+	PFN_PRE_CLOCKSPEED_CHANGE		pfnPreClockSpeedChange;
+	PFN_POST_CLOCKSPEED_CHANGE		pfnPostClockSpeedChange;
+	PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest;
+	PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest;
+	PFN_GPU_UNITS_POWER_CHANGE		pfnGPUUnitsPowerChange;
+	IMG_HANDLE						hSysData;
+	IMG_HANDLE						psDevNode;
+	PVRSRV_DEV_POWER_STATE			eDefaultPowerState;
+	ATOMIC_T						eCurrentPowerState;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRV_POWER_STATS				sPowerStats;
+#endif
+};
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+/*
+ * Power statistics related definitions
+ */
+
+/* For the mean time, use an exponentially weighted moving average with a
+ * 1/4 weighting for the new measurement.
+ */
+#define MEAN_TIME(A, B)     ( ((3*(A))/4) + ((1 * (B))/4) )
+
+#define UPDATE_TIME(time, newtime) \
+	((time) > 0 ? MEAN_TIME((time), (newtime)) : (newtime))
+
+/* Enum to be used as input to GET_POWER_STAT_INDEX */
+typedef enum
+{
+	DEVICE     = 0,
+	SYSTEM     = 1,
+	POST_POWER = 0,
+	PRE_POWER  = 2,
+	POWER_OFF  = 0,
+	POWER_ON   = 4,
+	NOT_FORCED = 0,
+	FORCED     = 8,
+} PVRSRV_POWER_STAT_TYPE;
+
+/* Macro used to access one of the power timing statistics inside an array */
+#define GET_POWER_STAT_INDEX(forced,powon,prepow,system) \
+	((forced) + (powon) + (prepow) + (system))
+
+void PVRSRVSetFirmwareStartTime(PVRSRV_POWER_DEV *psPowerDevice,
+						  IMG_UINT32 ui32Time)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->ui32FirmwareStartTimestamp =
+		UPDATE_TIME(psPowerStats->ui32FirmwareStartTimestamp,
+					ui32Time);
+}
+
+void PVRSRVSetFirmwareHandshakeIdleTime(PVRSRV_POWER_DEV *psPowerDevice,
+								  IMG_UINT64 ui64Duration)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->ui64FirmwareIdleDuration =
+		UPDATE_TIME(psPowerStats->ui64FirmwareIdleDuration,
+					ui64Duration);
+}
+
+static void _InsertPowerTimeStatistic(PVRSRV_POWER_DEV *psPowerDevice,
+									  IMG_UINT64 ui64SysStartTime, IMG_UINT64 ui64SysEndTime,
+									  IMG_UINT64 ui64DevStartTime, IMG_UINT64 ui64DevEndTime,
+									  IMG_BOOL bForced, IMG_BOOL bPowerOn, IMG_BOOL bPrePower)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+	IMG_UINT32 *pui32Stat;
+	IMG_UINT64 ui64DeviceDiff = ui64DevEndTime - ui64DevStartTime;
+	IMG_UINT64 ui64SystemDiff = ui64SysEndTime - ui64SysStartTime;
+	IMG_UINT32 ui32Index;
+
+	if (bPrePower)
+	{
+		HTBLOGK(HTB_SF_MAIN_PRE_POWER, bPowerOn, ui64DeviceDiff, ui64SystemDiff);
+	}
+	else
+	{
+		HTBLOGK(HTB_SF_MAIN_POST_POWER, bPowerOn, ui64SystemDiff, ui64DeviceDiff);
+	}
+
+	ui32Index = GET_POWER_STAT_INDEX(bForced ? FORCED : NOT_FORCED,
+	                                 bPowerOn ? POWER_ON : POWER_OFF,
+	                                 bPrePower ? PRE_POWER : POST_POWER,
+	                                 DEVICE);
+	pui32Stat = &psPowerStats->aui32PowerTimingStats[ui32Index];
+	*pui32Stat = UPDATE_TIME(*pui32Stat, ui64DeviceDiff);
+
+	ui32Index = GET_POWER_STAT_INDEX(bForced ? FORCED : NOT_FORCED,
+	                                 bPowerOn ? POWER_ON : POWER_OFF,
+	                                 bPrePower ? PRE_POWER : POST_POWER,
+	                                 SYSTEM);
+	pui32Stat = &psPowerStats->aui32PowerTimingStats[ui32Index];
+	*pui32Stat = UPDATE_TIME(*pui32Stat, ui64SystemDiff);
+}
+
+static void _InsertPowerTimeStatisticExtraPre(PVRSRV_POWER_DEV *psPowerDevice,
+											  IMG_UINT64 ui64StartTimer,
+											  IMG_UINT64 ui64Stoptimer)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64PreClockSpeedChangeDuration =
+		ui64Stoptimer - ui64StartTimer;
+
+	psPowerStats->ui64PreClockSpeedChangeMark = OSClockus();
+}
+
+static void _InsertPowerTimeStatisticExtraPost(PVRSRV_POWER_DEV *psPowerDevice,
+											   IMG_UINT64 ui64StartTimer,
+											   IMG_UINT64 ui64StopTimer)
+{
+	PVRSRV_POWER_STATS *psPowerStats = &psPowerDevice->sPowerStats;
+	IMG_UINT64 ui64Duration = ui64StartTimer - psPowerStats->ui64PreClockSpeedChangeMark;
+
+	PVR_ASSERT(psPowerStats->ui64PreClockSpeedChangeMark > 0);
+
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64BetweenPreEndingAndPostStartingDuration = ui64Duration;
+	psPowerStats->asClockSpeedChanges[psPowerStats->ui32ClockSpeedIndexEnd].ui64PostClockSpeedChangeDuration = ui64StopTimer - ui64StartTimer;
+
+	psPowerStats->ui32ClockSpeedIndexEnd = (psPowerStats->ui32ClockSpeedIndexEnd + 1) % NUM_EXTRA_POWER_STATS;
+
+	if (psPowerStats->ui32ClockSpeedIndexEnd == psPowerStats->ui32ClockSpeedIndexStart)
+	{
+		psPowerStats->ui32ClockSpeedIndexStart = (psPowerStats->ui32ClockSpeedIndexStart + 1) % NUM_EXTRA_POWER_STATS;
+	}
+
+	psPowerStats->ui64PreClockSpeedChangeMark = 0;
+}
+
+static INLINE void _PowerStatsPrintGroup(IMG_UINT32 *pui32Stats,
+										 OSDI_IMPL_ENTRY *psEntry,
+										 PVRSRV_POWER_STAT_TYPE eForced,
+										 PVRSRV_POWER_STAT_TYPE ePowerOn)
+{
+	IMG_UINT32 ui32Index;
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, PRE_POWER, DEVICE);
+	DIPrintf(psEntry, "  Pre-Device:  %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, PRE_POWER, SYSTEM);
+	DIPrintf(psEntry, "  Pre-System:  %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, POST_POWER, SYSTEM);
+	DIPrintf(psEntry, "  Post-System: %9u\n", pui32Stats[ui32Index]);
+
+	ui32Index = GET_POWER_STAT_INDEX(eForced, ePowerOn, POST_POWER, DEVICE);
+	DIPrintf(psEntry, "  Post-Device: %9u\n", pui32Stats[ui32Index]);
+}
+
+int PVRSRVPowerStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = DIGetPrivData(psEntry);
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	PVRSRV_POWER_STATS *psPowerStats;
+	IMG_UINT32 *pui32Stats;
+	IMG_UINT32 ui32Idx;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	if (psPowerDevice == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Device not initialised when "
+				 "reading power timing stats!"));
+		return -EIO;
+	}
+
+	psPowerStats = &psPowerDevice->sPowerStats;
+
+	pui32Stats = &psPowerStats->aui32PowerTimingStats[0];
+
+	DIPrintf(psEntry, "Forced Power-on Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, FORCED, POWER_ON);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Forced Power-off Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, FORCED, POWER_OFF);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Not Forced Power-on Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, NOT_FORCED, POWER_ON);
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Not Forced Power-off Transition (nanoseconds):\n");
+	_PowerStatsPrintGroup(pui32Stats, psEntry, NOT_FORCED, POWER_OFF);
+	DIPrintf(psEntry, "\n");
+
+
+	DIPrintf(psEntry, "FW bootup time (timer ticks): %u\n", psPowerStats->ui32FirmwareStartTimestamp);
+	DIPrintf(psEntry, "Host Acknowledge Time for FW Idle Signal (timer ticks): %u\n", (IMG_UINT32)(psPowerStats->ui64FirmwareIdleDuration));
+	DIPrintf(psEntry, "\n");
+
+	DIPrintf(psEntry, "Last %d Clock Speed Change Timers (nanoseconds):\n", NUM_EXTRA_POWER_STATS);
+	DIPrintf(psEntry, "Prepare DVFS\tDVFS Change\tPost DVFS\n");
+
+	for (ui32Idx = psPowerStats->ui32ClockSpeedIndexStart;
+		 ui32Idx != psPowerStats->ui32ClockSpeedIndexEnd;
+		 ui32Idx = (ui32Idx + 1) % NUM_EXTRA_POWER_STATS)
+	{
+		DIPrintf(psEntry, "%12" IMG_UINT64_FMTSPEC "\t%11" IMG_UINT64_FMTSPEC "\t%9" IMG_UINT64_FMTSPEC "\n",
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64PreClockSpeedChangeDuration,
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64BetweenPreEndingAndPostStartingDuration,
+				 psPowerStats->asClockSpeedChanges[ui32Idx].ui64PostClockSpeedChangeDuration);
+	}
+
+	return 0;
+}
+
+#else /* defined(PVRSRV_ENABLE_PROCESS_STATS) */
+
+static void _InsertPowerTimeStatistic(PVRSRV_POWER_DEV *psPowerDevice,
+									  IMG_UINT64 ui64SysStartTime, IMG_UINT64 ui64SysEndTime,
+									  IMG_UINT64 ui64DevStartTime, IMG_UINT64 ui64DevEndTime,
+									  IMG_BOOL bForced, IMG_BOOL bPowerOn, IMG_BOOL bPrePower)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64SysStartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64SysEndTime);
+	PVR_UNREFERENCED_PARAMETER(ui64DevStartTime);
+	PVR_UNREFERENCED_PARAMETER(ui64DevEndTime);
+	PVR_UNREFERENCED_PARAMETER(bForced);
+	PVR_UNREFERENCED_PARAMETER(bPowerOn);
+	PVR_UNREFERENCED_PARAMETER(bPrePower);
+}
+
+static void _InsertPowerTimeStatisticExtraPre(PVRSRV_POWER_DEV *psPowerDevice,
+											  IMG_UINT64 ui64StartTimer,
+											  IMG_UINT64 ui64Stoptimer)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTimer);
+	PVR_UNREFERENCED_PARAMETER(ui64Stoptimer);
+}
+
+static void _InsertPowerTimeStatisticExtraPost(PVRSRV_POWER_DEV *psPowerDevice,
+											   IMG_UINT64 ui64StartTimer,
+											   IMG_UINT64 ui64StopTimer)
+{
+	PVR_UNREFERENCED_PARAMETER(psPowerDevice);
+	PVR_UNREFERENCED_PARAMETER(ui64StartTimer);
+	PVR_UNREFERENCED_PARAMETER(ui64StopTimer);
+}
+#endif
+
+const char *PVRSRVSysPowerStateToString(PVRSRV_SYS_POWER_STATE eState)
+{
+	switch (eState) {
+#define X(name, _) \
+		case PVRSRV_SYS_POWER_STATE_##name: \
+			return #name;
+		_PVRSRV_SYS_POWER_STATES
+#undef X
+		default:
+			return "unknown";
+	}
+}
+
+const char *PVRSRVDevPowerStateToString(PVRSRV_DEV_POWER_STATE eState)
+{
+	switch (eState) {
+		case PVRSRV_DEV_POWER_STATE_DEFAULT:
+			return "DEFAULT";
+		case PVRSRV_DEV_POWER_STATE_OFF:
+			return "OFF";
+		case PVRSRV_DEV_POWER_STATE_ON:
+			return "ON";
+		default:
+			return "unknown";
+	}
+}
+
+/*!
+  Typedef for a pointer to a function that will be called for re-acquiring
+  device powerlock after releasing it temporarily for some timeout period
+  in function PVRSRVDeviceIdleRequestKM
+ */
+#if defined(DEBUG)
+typedef PVRSRV_ERROR (*PFN_POWER_LOCK_ACQUIRE) (PPVRSRV_DEVICE_NODE psDevNode,
+                                                const char *pszFile, const unsigned int ui32LineNum);
+#else
+typedef PVRSRV_ERROR (*PFN_POWER_LOCK_ACQUIRE) (PPVRSRV_DEVICE_NODE psDevNode);
+#endif
+
+static inline IMG_UINT64 PVRSRVProcessStatsGetTimeNs(void)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	return OSClockns64();
+#else
+	return 0;
+#endif
+}
+
+static inline IMG_UINT64 PVRSRVProcessStatsGetTimeUs(void)
+{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	return OSClockus();
+#else
+	return 0;
+#endif
+}
+
+/*!
+******************************************************************************
+
+ @Function	_IsSystemStatePowered
+
+ @Description	Tests whether a given system state represents powered-up.
+
+ @Input		eSystemPowerState : a system power state
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+static IMG_BOOL _IsSystemStatePowered(PVRSRV_SYS_POWER_STATE eSystemPowerState)
+{
+	return (eSystemPowerState == PVRSRV_SYS_POWER_STATE_ON);
+}
+
+/* We don't expect PID=0 to acquire device power-lock */
+#define PWR_LOCK_OWNER_PID_CLR_VAL 0
+
+PVRSRV_ERROR PVRSRVPowerLockInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDeviceNode->hPowerLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	return PVRSRV_OK;
+}
+
+void PVRSRVPowerLockDeInit(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	OSLockDestroy(psDeviceNode->hPowerLock);
+}
+
+IMG_BOOL PVRSRVPwrLockIsLockedByMe(PCPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	return OSLockIsLocked(psDeviceNode->hPowerLock) &&
+	       OSGetCurrentClientProcessIDKM() == psDeviceNode->uiPwrLockOwnerPID;
+}
+
+
+#if defined(DEBUG)
+static void _PVRSRVPowerLockOwner(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const char *pszFile, const unsigned int ui32LineNum)
+{
+	psDeviceNode->sPowerLockOwner.pszFile = pszFile;
+	psDeviceNode->sPowerLockOwner.ui32LineNum = ui32LineNum;
+	psDeviceNode->sPowerLockOwner.ui64Timestamp = OSClockus64();
+}
+#endif
+
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                   const char *pszFile, const unsigned int ui32LineNum)
+#else
+PVRSRV_ERROR PVRSRVPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+#endif
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+#if defined(DEBUG)
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+#endif
+
+	return PVRSRV_OK;
+}
+
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerTryLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      const char *pszFile, const unsigned int ui32LineNum)
+#else
+PVRSRV_ERROR PVRSRVPowerTryLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+#endif
+{
+	if (!(OSTryLockAcquire(psDeviceNode->hPowerLock)))
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+#if defined(DEBUG)
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVPowerTryLockWaitForTimeout(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Take power lock, retry if it's in use in another task. */
+	LOOP_UNTIL_TIMEOUT_US(OS_POWERLOCK_TIMEOUT_US)
+	{
+		eError = PVRSRVPowerTryLock(psDeviceNode);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			return eError;
+		}
+
+		OSSleepms(OS_POWERLOCK_TIMEOUT_MS / OS_POWERLOCK_RETRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError == PVRSRV_ERROR_RETRY)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function     _PVRSRVForcedPowerLock
+
+ @Description  Obtain the mutex for power transitions regardless of system
+               power state
+
+ @Return       Always returns PVRSRV_OK. Function prototype required same as
+               PFN_POWER_LOCK_ACQUIRE
+
+******************************************************************************/
+#if defined(DEBUG)
+#define _PVRSRVForcedPowerLock(DEV_NODE)	\
+	_PVRSRVForcedPowerLock_Debug(DEV_NODE, __FILE__, __LINE__)
+
+static PVRSRV_ERROR _PVRSRVForcedPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                 const char *pszFile, const unsigned int ui32LineNum)
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+	_PVRSRVPowerLockOwner(psDeviceNode, pszFile, ui32LineNum);
+
+	return PVRSRV_OK;
+}
+#else
+static PVRSRV_ERROR _PVRSRVForcedPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	psDeviceNode->uiPwrLockOwnerPID = OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+}
+#endif
+
+void PVRSRVPowerUnlock(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDeviceNode));
+
+	/* Reset uiPwrLockOwnerPID before releasing lock */
+	psDeviceNode->uiPwrLockOwnerPID = PWR_LOCK_OWNER_PID_CLR_VAL;
+	OSLockRelease(psDeviceNode->hPowerLock);
+}
+
+IMG_BOOL PVRSRVDeviceIsDefaultStateOFF(PVRSRV_POWER_DEV *psPowerDevice)
+{
+	return (psPowerDevice->eDefaultPowerState == PVRSRV_DEV_POWER_STATE_OFF);
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceDefaultPowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+					PVRSRV_DEV_POWER_STATE eNewPowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psPowerDevice->eDefaultPowerState = eNewPowerState;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceCurrentPowerState(PVRSRV_POWER_DEV *psPowerDevice,
+					PVRSRV_DEV_POWER_STATE eNewPowerState)
+{
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	PVRSRV_DEV_POWER_STATE eOldPowerState;
+#endif
+
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+#if !defined(SUPPORT_PMR_DEFERRED_FREE)
+	OSAtomicWrite(&psPowerDevice->eCurrentPowerState, eNewPowerState);
+#else
+	eOldPowerState = OSAtomicExchange(&psPowerDevice->eCurrentPowerState,
+	                                  eNewPowerState);
+
+	psDeviceNode = psPowerDevice->psDevNode;
+	PVR_ASSERT(psDeviceNode);
+
+	if (eNewPowerState == PVRSRV_DEV_POWER_STATE_OFF &&
+	    eNewPowerState != eOldPowerState)
+	{
+		psDeviceNode->uiPowerOffCounter = psDeviceNode->uiPowerOffCounterNext;
+
+		/* It's not really important to know if any zombies were queued. */
+		(void) PMRQueueZombiesForCleanup(psDeviceNode);
+
+		psDeviceNode->uiPowerOffCounterNext++;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ @Input       pfnPowerLockAcquire  : Function to re-acquire power-lock in-case
+                                     it was necessary to release it.
+*/
+static PVRSRV_ERROR _PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF    pfnIsDefaultStateOff,
+					IMG_BOOL                            bDeviceOffPermitted,
+					PFN_POWER_LOCK_ACQUIRE              pfnPowerLockAcquire)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+	PVRSRV_ERROR eError;
+
+	/* if pfnIsDefaultStateOff not provided or pfnIsDefaultStateOff(psPowerDev)
+	 * is true (which means that the default state is OFF) then force idle. */
+	if ((psPowerDev && psPowerDev->pfnForcedIdleRequest) &&
+	    (pfnIsDefaultStateOff == NULL || pfnIsDefaultStateOff(psPowerDev)))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = psPowerDev->pfnForcedIdleRequest(psPowerDev->psDevNode,
+			                                          bDeviceOffPermitted);
+			if (eError == PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+			{
+				PVRSRV_ERROR eErrPwrLockAcq;
+				/* FW denied idle request */
+				PVRSRVPowerUnlock(psDeviceNode);
+
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+
+#if defined(DEBUG)
+				eErrPwrLockAcq = pfnPowerLockAcquire(psDeviceNode, __FILE__, __LINE__);
+#else
+				eErrPwrLockAcq = pfnPowerLockAcquire(psDeviceNode);
+#endif
+				if (eErrPwrLockAcq != PVRSRV_OK)
+				{
+					/* We only understand PVRSRV_ERROR_RETRY, so assert on others.
+					 * Moreover, we've ended-up releasing the power-lock which was
+					 * originally "held" by caller before calling this function -
+					 * since this needs vigilant handling at call-site, we pass
+					 * back an explicit error, for caller(s) to "avoid" calling
+					 * PVRSRVPowerUnlock */
+					PVR_ASSERT(eErrPwrLockAcq == PVRSRV_ERROR_RETRY);
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to re-acquire power-lock "
+					         "(%s) after releasing it for a time-out",
+							 __func__, PVRSRVGetErrorString(eErrPwrLockAcq)));
+					return PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED;
+				}
+			}
+			else
+			{
+				/* idle request successful or some other error occurred, return */
+				break;
+			}
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+	else
+	{
+		return PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*
+ * Wrapper function helps limiting calling complexity of supplying additional
+ * PFN_POWER_LOCK_ACQUIRE argument (required by _PVRSRVDeviceIdleRequestKM)
+ */
+inline PVRSRV_ERROR PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF      pfnIsDefaultStateOff,
+					IMG_BOOL                              bDeviceOffPermitted)
+{
+	return _PVRSRVDeviceIdleRequestKM(psDeviceNode,
+	                                  pfnIsDefaultStateOff,
+	                                  bDeviceOffPermitted,
+#if defined(DEBUG)
+	                                  PVRSRVPowerLock_Debug
+#else
+	                                  PVRSRVPowerLock
+#endif
+	                                  );
+}
+
+PVRSRV_ERROR PVRSRVDeviceIdleCancelRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_POWER_DEV *psPowerDev = psDeviceNode->psPowerDev;
+
+	if (psPowerDev && psPowerDev->pfnForcedIdleCancelRequest)
+	{
+		return psPowerDev->pfnForcedIdleCancelRequest(psPowerDev->psDevNode);
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceSystemPrePowerStateKM
+
+ @Description
+
+ Perform device and system-specific processing required before a power
+ transition
+
+ @Input		psPowerDevice : Power device
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static
+PVRSRV_ERROR PVRSRVDeviceSystemPrePowerStateKM(PVRSRV_POWER_DEV			*psPowerDevice,
+											   PVRSRV_DEV_POWER_STATE	eNewPowerState,
+											   PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+	IMG_UINT64 ui64SysTimer1 = 0;
+	IMG_UINT64 ui64SysTimer2 = 0;
+	IMG_UINT64 ui64DevTimer1 = 0;
+	IMG_UINT64 ui64DevTimer2 = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(eNewPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	if (psPowerDevice->pfnDevicePrePower != NULL)
+	{
+		ui64DevTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		/* Call the device's power callback. */
+		eError = psPowerDevice->pfnDevicePrePower(psPowerDevice->psDevNode,
+												  eNewPowerState,
+												  eCurrentPowerState,
+												  ePwrFlags);
+
+		ui64DevTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	/* Do any required system-layer processing. */
+	if (psPowerDevice->pfnSystemPrePower != NULL)
+	{
+		ui64SysTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		eError = psPowerDevice->pfnSystemPrePower(psPowerDevice->hSysData,
+												  (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													PVRSRV_SYS_POWER_STATE_ON :
+													PVRSRV_SYS_POWER_STATE_OFF,
+												  (eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													PVRSRV_SYS_POWER_STATE_ON :
+													PVRSRV_SYS_POWER_STATE_OFF,
+												  ePwrFlags);
+
+		ui64SysTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_GOTO_IF_ERROR(eError, ErrRestorePowerState);
+	}
+
+	_InsertPowerTimeStatistic(psPowerDevice, ui64SysTimer1, ui64SysTimer2,
+							 ui64DevTimer1, ui64DevTimer2,
+							 BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED),
+							 eNewPowerState == PVRSRV_DEV_POWER_STATE_ON,
+							 IMG_TRUE);
+
+	return PVRSRV_OK;
+
+ErrRestorePowerState:
+	/* In a situation where pfnDevicePrePower() succeeded but pfnSystemPrePower()
+	 * failed we need to restore the device's power state from before the current
+	 * request. Otherwise it will result in an inconsistency between the device's
+	 * actual state and what the driver thinks the state is. */
+	{
+		PVRSRV_ERROR eError2 = PVRSRV_OK;
+
+		if (psPowerDevice->pfnDevicePrePower != NULL)
+		{
+			/* Call the device's power callback. */
+			eError2 = psPowerDevice->pfnDevicePrePower(psPowerDevice->psDevNode,
+			                                           eCurrentPowerState,
+			                                           eNewPowerState,
+			                                           ePwrFlags);
+			PVR_LOG_IF_ERROR(eError2, "pfnDevicePrePower");
+		}
+		if (eError2 == PVRSRV_OK && psPowerDevice->pfnDevicePostPower != NULL)
+		{
+			/* Call the device's power callback. */
+			eError2 = psPowerDevice->pfnDevicePostPower(psPowerDevice->psDevNode,
+			                                            eCurrentPowerState,
+			                                            eNewPowerState,
+			                                            ePwrFlags);
+			PVR_LOG_IF_ERROR(eError2, "pfnDevicePostPower");
+		}
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceSystemPostPowerStateKM
+
+ @Description
+
+ Perform device and system-specific processing required after a power
+ transition
+
+ @Input		psPowerDevice : Power device
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static
+PVRSRV_ERROR PVRSRVDeviceSystemPostPowerStateKM(PVRSRV_POWER_DEV		*psPowerDevice,
+												PVRSRV_DEV_POWER_STATE	eNewPowerState,
+												PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+	IMG_UINT64 ui64SysTimer1 = 0;
+	IMG_UINT64 ui64SysTimer2 = 0;
+	IMG_UINT64 ui64DevTimer1 = 0;
+	IMG_UINT64 ui64DevTimer2 = 0;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(eNewPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	/* Do any required system-layer processing. */
+	if (psPowerDevice->pfnSystemPostPower != NULL)
+	{
+		ui64SysTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		eError = psPowerDevice->pfnSystemPostPower(psPowerDevice->hSysData,
+												   (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													 PVRSRV_SYS_POWER_STATE_ON :
+													 PVRSRV_SYS_POWER_STATE_OFF,
+												   (eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) ?
+													 PVRSRV_SYS_POWER_STATE_ON :
+													 PVRSRV_SYS_POWER_STATE_OFF,
+												   ePwrFlags);
+
+		ui64SysTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psPowerDevice->pfnDevicePostPower != NULL)
+	{
+		ui64DevTimer1 = PVRSRVProcessStatsGetTimeNs();
+
+		/* Call the device's power callback. */
+		eError = psPowerDevice->pfnDevicePostPower(psPowerDevice->psDevNode,
+												   eNewPowerState,
+												   eCurrentPowerState,
+												   ePwrFlags);
+
+		ui64DevTimer2 = PVRSRVProcessStatsGetTimeNs();
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	_InsertPowerTimeStatistic(psPowerDevice, ui64SysTimer1, ui64SysTimer2,
+							 ui64DevTimer1, ui64DevTimer2,
+							 BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED),
+							 eNewPowerState == PVRSRV_DEV_POWER_STATE_ON,
+							 IMG_FALSE);
+
+	PVRSRVSetDeviceCurrentPowerState(psPowerDevice, eNewPowerState);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVSetDevicePowerStateKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE eNewPowerState,
+										 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_ERROR	eError;
+	PVRSRV_DATA*    psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (!psPowerDevice)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (eNewPowerState == PVRSRV_DEV_POWER_STATE_DEFAULT)
+	{
+		eNewPowerState = psPowerDevice->eDefaultPowerState;
+	}
+
+	/* Call power function if the state change or if this is an OS request. */
+	if (OSAtomicRead(&psPowerDevice->eCurrentPowerState) != eNewPowerState ||
+	    BITMASK_ANY(ePwrFlags, PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ | PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ))
+	{
+		eError = PVRSRVDeviceSystemPrePowerStateKM(psPowerDevice,
+												   eNewPowerState,
+												   ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		eError = PVRSRVDeviceSystemPostPowerStateKM(psPowerDevice,
+													eNewPowerState,
+													ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		psDeviceNode->eCurrentSysPowerState = eNewPowerState;
+
+		/* Signal Device Watchdog Thread about power mode change. */
+		if (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans++;
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			if (psPVRSRVData->ui32DevicesWatchdogTimeout == DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT)
+#endif
+			{
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+		}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+		else if (eNewPowerState == PVRSRV_DEV_POWER_STATE_OFF)
+		{
+			/* signal watchdog thread and give it a chance to switch to
+			 * longer / infinite wait time */
+			eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	return PVRSRV_OK;
+
+ErrorExit:
+
+	if (eError == PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Transition to %d was denied, Flags=0x%08x",
+				 __func__, eNewPowerState, ePwrFlags));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Transition to %d FAILED (%s)",
+				 __func__, eNewPowerState, PVRSRVGetErrorString(eError)));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVSetDeviceSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState,
+											 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT uiStage = 0;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+#endif
+
+	PVRSRV_DATA*    psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_POWER_DEV *psPowerDevice;
+	PVRSRV_DEV_POWER_STATE eNewDevicePowerState;
+	PFN_SYS_DEV_IS_DEFAULT_STATE_OFF pfnIsDefaultStateOff;
+
+	/* Require a proper power state */
+	if (eNewSysPowerState == PVRSRV_SYS_POWER_STATE_Unspecified)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Prevent simultaneous SetPowerStateKM calls */
+	_PVRSRVForcedPowerLock(psDeviceNode);
+	psPowerDevice = psDeviceNode->psPowerDev;
+
+	if (psPowerDevice){
+		eNewDevicePowerState = _IsSystemStatePowered(eNewSysPowerState)
+			? psPowerDevice->eDefaultPowerState : PVRSRV_DEV_POWER_STATE_OFF;
+	}
+	else{
+		eNewDevicePowerState = _IsSystemStatePowered(eNewSysPowerState)
+			? PVRSRV_DEV_POWER_STATE_DEFAULT : PVRSRV_DEV_POWER_STATE_OFF;
+	}
+
+	/* If setting devices to default state, force idle all devices whose default state is off */
+	pfnIsDefaultStateOff =
+	  (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_OFF) ? PVRSRVDeviceIsDefaultStateOFF : NULL;
+
+	/* No power transition requested, so do nothing */
+	if (eNewSysPowerState == psDeviceNode->eCurrentSysPowerState)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return PVRSRV_OK;
+	}
+
+	/* If the device is already off don't send the idle request. */
+	if ((psDeviceNode->eCurrentSysPowerState != PVRSRV_SYS_POWER_STATE_OFF) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		&& (eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_OK)
+#endif
+		)
+	{
+		eError = _PVRSRVDeviceIdleRequestKM(psDeviceNode, pfnIsDefaultStateOff,
+#if defined(DEBUG)
+		                                    IMG_TRUE, _PVRSRVForcedPowerLock_Debug
+#else
+		                                    IMG_TRUE, _PVRSRVForcedPowerLock
+#endif
+		                                    );
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "_PVRSRVDeviceIdleRequestKM");
+			uiStage = 1;
+			goto ErrorExit;
+		}
+	}
+
+	/* Call power function if the state change or if this is an OS request. */
+	if (psPowerDevice && (OSAtomicRead(&psPowerDevice->eCurrentPowerState) != eNewDevicePowerState ||
+			 BITMASK_ANY(ePwrFlags,
+			 PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ | PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ)))
+	{
+		eError = PVRSRVDeviceSystemPrePowerStateKM(psPowerDevice,
+												   eNewDevicePowerState,
+												   ePwrFlags);
+		uiStage = 2;
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		eError = PVRSRVDeviceSystemPostPowerStateKM(psPowerDevice,
+													eNewDevicePowerState,
+													ePwrFlags);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+
+		/* Signal Device Watchdog Thread about power mode change. */
+		if (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans++;
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			if (psPVRSRVData->ui32DevicesWatchdogTimeout == DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT)
+#endif
+			{
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+		}
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+		else if (eNewDevicePowerState == PVRSRV_DEV_POWER_STATE_OFF)
+		{
+			/* signal watchdog thread and give it a chance to switch to
+			 * longer / infinite wait time */
+			eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	psDeviceNode->eCurrentSysPowerState = eNewSysPowerState;
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Transition from %s to %s FAILED (%s) at stage "
+	         "%u. Dumping debug info.", __func__,
+	         PVRSRVSysPowerStateToString(psDeviceNode->eCurrentSysPowerState),
+	         PVRSRVSysPowerStateToString(eNewSysPowerState),
+	         PVRSRVGetErrorString(eError), uiStage));
+
+	PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVSetSystemPowerState(PVRSRV_DEVICE_CONFIG *psDevConfig,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDevNode = psDevConfig->psDevNode;
+	PVRSRV_SYS_POWER_STATE eCurrentSysPowerState;
+
+	PVRSRVGetSystemPowerState(psDevNode, &eCurrentSysPowerState);
+
+	/* no power transition requested, so do nothing */
+	if (eNewSysPowerState == eCurrentSysPowerState)
+	{
+		return PVRSRV_OK;
+	}
+
+	if (psDevConfig->pfnPrePowerState != NULL)
+	{
+		eError = psDevConfig->pfnPrePowerState(psDevConfig->hSysData,
+												  eNewSysPowerState,
+												  eCurrentSysPowerState,
+												  PVRSRV_POWER_FLAGS_FORCED);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psDevConfig->pfnPostPowerState != NULL)
+	{
+		eError = psDevConfig->pfnPostPowerState(psDevConfig->hSysData,
+												   eNewSysPowerState,
+												   eCurrentSysPowerState,
+												   PVRSRV_POWER_FLAGS_FORCED);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	if (psDevNode != NULL)
+	{
+		psDevNode->eCurrentSysPowerState = eNewSysPowerState;
+	}
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVSetPowerCallbacks(PPVRSRV_DEVICE_NODE				psDeviceNode,
+							 PVRSRV_POWER_DEV					*psPowerDevice,
+							 PFN_PRE_POWER						pfnDevicePrePower,
+							 PFN_POST_POWER						pfnDevicePostPower,
+							 PFN_SYS_PRE_POWER					pfnSystemPrePower,
+							 PFN_SYS_POST_POWER					pfnSystemPostPower,
+							 PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest)
+{
+	if (psPowerDevice != NULL)
+	{
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+		{
+			psPowerDevice->pfnSystemPrePower = NULL;
+			psPowerDevice->pfnSystemPostPower = NULL;
+		}
+		else
+		{
+			psPowerDevice->pfnSystemPrePower = pfnSystemPrePower;
+			psPowerDevice->pfnSystemPostPower = pfnSystemPostPower;
+		}
+
+		psPowerDevice->pfnDevicePrePower = pfnDevicePrePower;
+		psPowerDevice->pfnDevicePostPower = pfnDevicePostPower;
+		psPowerDevice->pfnForcedIdleRequest = pfnForcedIdleRequest;
+		psPowerDevice->pfnForcedIdleCancelRequest = pfnForcedIdleCancelRequest;
+	}
+}
+
+PVRSRV_ERROR PVRSRVRegisterPowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode,
+									   PFN_PRE_POWER				pfnDevicePrePower,
+									   PFN_POST_POWER				pfnDevicePostPower,
+									   PFN_SYS_PRE_POWER			pfnSystemPrePower,
+									   PFN_SYS_POST_POWER			pfnSystemPostPower,
+									   PFN_PRE_CLOCKSPEED_CHANGE	pfnPreClockSpeedChange,
+									   PFN_POST_CLOCKSPEED_CHANGE	pfnPostClockSpeedChange,
+									   PFN_FORCED_IDLE_REQUEST	pfnForcedIdleRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+									   PFN_GPU_UNITS_POWER_CHANGE	pfnGPUUnitsPowerChange,
+									   IMG_HANDLE					psDevNode,
+									   PVRSRV_DEV_POWER_STATE		eCurrentPowerState,
+									   PVRSRV_DEV_POWER_STATE		eDefaultPowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	PVR_ASSERT(!psDeviceNode->psPowerDev);
+
+	PVR_ASSERT(eCurrentPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+	PVR_ASSERT(eDefaultPowerState != PVRSRV_DEV_POWER_STATE_DEFAULT);
+
+	psPowerDevice = OSAllocMem(sizeof(PVRSRV_POWER_DEV));
+	PVR_LOG_RETURN_IF_NOMEM(psPowerDevice, "psPowerDevice");
+
+	/* setup device for power manager */
+	PVRSRVSetPowerCallbacks(psDeviceNode,
+							psPowerDevice,
+							pfnDevicePrePower,
+							pfnDevicePostPower,
+							pfnSystemPrePower,
+							pfnSystemPostPower,
+							pfnForcedIdleRequest,
+							pfnForcedIdleCancelRequest);
+
+	psPowerDevice->pfnPreClockSpeedChange = pfnPreClockSpeedChange;
+	psPowerDevice->pfnPostClockSpeedChange = pfnPostClockSpeedChange;
+	psPowerDevice->pfnGPUUnitsPowerChange = pfnGPUUnitsPowerChange;
+	psPowerDevice->hSysData = psDeviceNode->psDevConfig->hSysData;
+	psPowerDevice->psDevNode = psDevNode;
+	PVRSRVSetDeviceCurrentPowerState(psPowerDevice, eCurrentPowerState);
+	psPowerDevice->eDefaultPowerState = eDefaultPowerState;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	OSCachedMemSet(&psPowerDevice->sPowerStats, 0, sizeof(psPowerDevice->sPowerStats));
+#endif
+
+	psDeviceNode->psPowerDev = psPowerDevice;
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVRemovePowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	if (psDeviceNode->psPowerDev)
+	{
+		/* RGX device/firmware should now be powered down */
+		PVR_ASSERT(PVRSRVIsDevicePowered(psDeviceNode) == IMG_FALSE);
+		OSFreeMem(psDeviceNode->psPowerDev);
+		psDeviceNode->psPowerDev = NULL;
+	}
+}
+
+PVRSRV_ERROR PVRSRVGetDevicePowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+									   PPVRSRV_DEV_POWER_STATE pePowerState)
+{
+	PVRSRV_POWER_DEV *psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	*pePowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL PVRSRVIsDevicePowered(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_DEV_POWER_STATE ePowerState;
+
+	if (PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState) != PVRSRV_OK)
+	{
+		return IMG_FALSE;
+	}
+
+	return (ePowerState == PVRSRV_DEV_POWER_STATE_ON);
+}
+
+PVRSRV_ERROR PVRSRVGetSystemPowerState(PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                   PPVRSRV_SYS_POWER_STATE peCurrentSysPowerState)
+{
+	if (psDeviceNode != NULL)
+	{
+		if (psDeviceNode->psDevConfig->pfnGpuDomainPower)
+		{
+			*peCurrentSysPowerState = psDeviceNode->psDevConfig->pfnGpuDomainPower(psDeviceNode);
+		}
+		else
+		{
+			*peCurrentSysPowerState = psDeviceNode->eCurrentSysPowerState;
+		}
+	}
+	else
+	{
+		/* assume power is off if no device node */
+		*peCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_OFF;
+		return PVRSRV_ERROR_UNKNOWN_POWER_STATE;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_BOOL PVRSRVIsSystemPowered(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	PVRSRV_SYS_POWER_STATE eSysPowerState;
+
+	if (PVRSRVGetSystemPowerState(psDeviceNode, &eSysPowerState) != PVRSRV_OK)
+	{
+		return IMG_FALSE;
+	}
+
+	return (eSysPowerState == PVRSRV_SYS_POWER_STATE_ON);
+}
+
+PVRSRV_ERROR
+PVRSRVDevicePreClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                IMG_BOOL            bIdleDevice,
+                                void*               pvInfo)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	IMG_UINT64 ui64StartTimer, ui64StopTimer;
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+
+	PVR_UNREFERENCED_PARAMETER(pvInfo);
+
+	if (psPowerDevice == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	ui64StartTimer = PVRSRVProcessStatsGetTimeUs();
+
+	/* This mutex is released in PVRSRVDevicePostClockSpeedChange. */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	if ((eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) && bIdleDevice)
+	{
+		/* We can change the clock speed if the device is either IDLE or OFF */
+		eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_TRUE);
+
+		if (eError != PVRSRV_OK)
+		{
+			/* FW Can signal denied when busy with SPM or other work it can not idle */
+			if (eError != PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Error (%s) from %s()", __func__,
+				         PVRSRVGETERRORSTRING(eError), "PVRSRVDeviceIdleRequestKM"));
+			}
+			if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+			return eError;
+		}
+	}
+
+	eError = psPowerDevice->pfnPreClockSpeedChange(psPowerDevice->psDevNode,
+	                                               eCurrentPowerState);
+
+	ui64StopTimer = PVRSRVProcessStatsGetTimeUs();
+
+	_InsertPowerTimeStatisticExtraPre(psPowerDevice, ui64StartTimer, ui64StopTimer);
+
+	return eError;
+}
+
+void
+PVRSRVDevicePostClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                 IMG_BOOL            bIdleDevice,
+                                 void*               pvInfo)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_POWER_DEV *psPowerDevice = psDeviceNode->psPowerDev;
+	IMG_UINT64 ui64StartTimer, ui64StopTimer;
+	PVRSRV_DEV_POWER_STATE eCurrentPowerState;
+
+	PVR_UNREFERENCED_PARAMETER(pvInfo);
+
+	if (psPowerDevice == NULL)
+	{
+		return;
+	}
+
+	ui64StartTimer = PVRSRVProcessStatsGetTimeUs();
+
+	eCurrentPowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+
+	eError = psPowerDevice->pfnPostClockSpeedChange(psPowerDevice->psDevNode,
+													eCurrentPowerState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p failed (%s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+	}
+
+	if ((eCurrentPowerState == PVRSRV_DEV_POWER_STATE_ON) && bIdleDevice)
+	{
+		eError = PVRSRVDeviceIdleCancelRequestKM(psDeviceNode);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceIdleCancelRequestKM");
+	}
+
+	/* This mutex was acquired in PVRSRVDevicePreClockSpeedChange. */
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	OSAtomicIncrement(&psDeviceNode->iNumClockSpeedChanges);
+
+	ui64StopTimer = PVRSRVProcessStatsGetTimeUs();
+
+	_InsertPowerTimeStatisticExtraPost(psPowerDevice, ui64StartTimer, ui64StopTimer);
+}
+
+PVRSRV_ERROR PVRSRVDeviceGPUUnitsPowerChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                             IMG_UINT32 ui32NewValue)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_POWER_DEV	*psPowerDevice;
+
+	psPowerDevice = psDeviceNode->psPowerDev;
+	if (psPowerDevice)
+	{
+		PVRSRV_DEV_POWER_STATE eDevicePowerState;
+
+		eError = PVRSRVPowerLock(psDeviceNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+		eDevicePowerState = OSAtomicRead(&psPowerDevice->eCurrentPowerState);
+		if (eDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			/* Device must be idle to change GPU unit(s) power state */
+			eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_FALSE);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+				if (eError == PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+				{
+					goto ErrorExit;
+				}
+				goto ErrorUnlockAndExit;
+			}
+		}
+
+		if (psPowerDevice->pfnGPUUnitsPowerChange != NULL)
+		{
+			PVRSRV_ERROR eError2 = psPowerDevice->pfnGPUUnitsPowerChange(psPowerDevice->psDevNode, ui32NewValue);
+
+			if (eError2 != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Device %p failed (%s)",
+				         __func__, psDeviceNode,
+				         PVRSRVGetErrorString(eError2)));
+			}
+		}
+
+		if (eDevicePowerState == PVRSRV_DEV_POWER_STATE_ON)
+		{
+			eError = PVRSRVDeviceIdleCancelRequestKM(psDeviceNode);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDeviceIdleCancelRequestKM", ErrorUnlockAndExit);
+		}
+
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return eError;
+
+ErrorUnlockAndExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+ErrorExit:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (power.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/power.h b/drivers/gpu/drm/img-rogue/power.h
new file mode 100644
index 000000000000..57226d91f6c5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/power.h
@@ -0,0 +1,558 @@
+/*************************************************************************/ /*!
+@File
+@Title          Power Management Functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for power management functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef POWER_H
+#define POWER_H
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_error.h"
+#include "servicesext.h"
+#include "opaque_types.h"
+#include "di_common.h"
+
+/*!
+ *****************************************************************************
+ *	Power management
+ *****************************************************************************/
+
+typedef struct _PVRSRV_POWER_DEV_TAG_ PVRSRV_POWER_DEV;
+
+typedef IMG_BOOL (*PFN_SYS_DEV_IS_DEFAULT_STATE_OFF)(PVRSRV_POWER_DEV *psPowerDevice);
+
+/* Power transition handler prototypes */
+
+/*!
+  Typedef for a pointer to a Function that will be called before a transition
+  from one power state to another. See also PFN_POST_POWER.
+ */
+typedef PVRSRV_ERROR (*PFN_PRE_POWER)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      PVRSRV_DEV_POWER_STATE eNewPowerState,
+                                      PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                                      PVRSRV_POWER_FLAGS ePwrFlags);
+/*!
+  Typedef for a pointer to a Function that will be called after a transition
+  from one power state to another. See also PFN_PRE_POWER.
+ */
+typedef PVRSRV_ERROR (*PFN_POST_POWER)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                       PVRSRV_DEV_POWER_STATE eNewPowerState,
+                                       PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                                       PVRSRV_POWER_FLAGS ePwrFlags);
+
+/* Clock speed handler prototypes */
+
+/*!
+  Typedef for a pointer to a Function that will be called before a transition
+  from one clock speed to another. See also PFN_POST_CLOCKSPEED_CHANGE.
+ */
+typedef PVRSRV_ERROR (*PFN_PRE_CLOCKSPEED_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                  PVRSRV_DEV_POWER_STATE eCurrentPowerState);
+
+/*!
+  Typedef for a pointer to a Function that will be called after a transition
+  from one clock speed to another. See also PFN_PRE_CLOCKSPEED_CHANGE.
+ */
+typedef PVRSRV_ERROR (*PFN_POST_CLOCKSPEED_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                   PVRSRV_DEV_POWER_STATE eCurrentPowerState);
+
+/*!
+  Typedef for a pointer to a function that will be called to transition the
+  device to a forced idle state. Used in unison with (forced) power requests,
+  DVFS and cluster count changes.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_REQUEST)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                IMG_BOOL bDeviceOffPermitted);
+
+/*!
+  Typedef for a pointer to a function that will be called to cancel a forced
+  idle state and return the firmware back to a state where the hardware can be
+  scheduled.
+ */
+typedef PVRSRV_ERROR (*PFN_FORCED_IDLE_CANCEL_REQUEST)(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+typedef PVRSRV_ERROR (*PFN_GPU_UNITS_POWER_CHANGE)(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                                   IMG_UINT32 ui32SESPowerState);
+
+const char *PVRSRVSysPowerStateToString(PVRSRV_SYS_POWER_STATE eState);
+const char *PVRSRVDevPowerStateToString(PVRSRV_DEV_POWER_STATE eState);
+
+PVRSRV_ERROR PVRSRVPowerLockInit(PPVRSRV_DEVICE_NODE psDeviceNode);
+void PVRSRVPowerLockDeInit(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerLock
+
+ @Description	Obtain the mutex for power transitions. Only allowed when
+                system power is on.
+
+ @Return	PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or PVRSRV_OK
+
+******************************************************************************/
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                   const char *pszFile, const unsigned int ui32LineNum);
+#define PVRSRVPowerLock(DEV_NODE)	PVRSRVPowerLock_Debug(DEV_NODE, __FILE__, __LINE__)
+#else
+PVRSRV_ERROR PVRSRVPowerLock(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerUnlock
+
+ @Description	Release the mutex for power transitions
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+void PVRSRVPowerUnlock(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerTryLock
+
+ @Description	Try to obtain the mutex for power transitions. Only allowed when
+		system power is on.
+
+ @Return	PVRSRV_ERROR_RETRY or PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or
+		PVRSRV_OK
+
+******************************************************************************/
+#if defined(DEBUG)
+PVRSRV_ERROR PVRSRVPowerTryLock_Debug(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                      const char *pszFile, const unsigned int ui32LineNum);
+#define PVRSRVPowerTryLock(DEV_NODE)	PVRSRVPowerTryLock_Debug(DEV_NODE, __FILE__, __LINE__)
+#else
+PVRSRV_ERROR PVRSRVPowerTryLock(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVPowerTryLockWaitForTimeout
+
+ @Description	Try to obtain the mutex for power transitions. Only allowed when
+		system power is on. The call blocks until either the lock is acquired,
+		or the timeout is reached.
+
+		*** Debug only. DO NOT use in core GPU functions which cannot fail. ***
+		If the power lock cannot be taken the device may be powered down at
+		any time in another worker thread.
+
+ @Return	PVRSRV_ERROR_RETRY or PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF or
+		PVRSRV_OK
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPowerTryLockWaitForTimeout(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function     PVRSRVPwrLockIsLockedByMe
+
+ @Description  Determine if the calling context is holding the device power-lock
+
+ @Return       IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVPwrLockIsLockedByMe(PCPVRSRV_DEVICE_NODE psDeviceNode);
+IMG_BOOL PVRSRVDeviceIsDefaultStateOFF(PVRSRV_POWER_DEV *psPowerDevice);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVSetDevicePowerStateKM
+
+ @Description	Set the Device into a new state
+
+ @Input		psDeviceNode : Device node
+ @Input		eNewPowerState : New power state
+ @Input		ePwrFlags : Power state change flags
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDevicePowerStateKM(PPVRSRV_DEVICE_NODE	psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+										 PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVSetDeviceSystemPowerState
+@Description  Set the device into a new power state based on the systems power
+              state
+@Input        psDeviceNode       Device node
+@Input        eNewSysPowerState  New system power state
+@Input        ePwrFlags          Power state change flags
+@Return       PVRSRV_ERROR       PVRSRV_OK on success or an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState,
+											 PVRSRV_POWER_FLAGS ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetDeviceDefaultPowerState
+
+ @Description   Set the default device power state to eNewPowerState
+
+ @Input         psDeviceNode : Device node
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceDefaultPowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+					PVRSRV_DEV_POWER_STATE eNewPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetDeviceCurrentPowerState
+
+ @Description   Set the current device power state to eNewPowerState
+
+ @Input         psPowerDevice : Power device
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetDeviceCurrentPowerState(PVRSRV_POWER_DEV *psPowerDevice,
+					PVRSRV_DEV_POWER_STATE eNewPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetSystemPowerState
+
+ @Description   Set the system power state to eNewPowerState
+
+ @Input         psDeviceConfig : Device config
+ @Input         eNewPowerState : New power state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVSetSystemPowerState(PVRSRV_DEVICE_CONFIG * psDeviceConfig,
+											 PVRSRV_SYS_POWER_STATE eNewSysPowerState);
+
+/*!
+******************************************************************************
+
+ @Function      PVRSRVSetPowerCallbacks
+
+ @Description   Initialise the Power Device's function pointers
+                to the appropriate callbacks depending on driver mode and
+                system setup.
+
+ @Input         psDeviceNode : Device node
+ @Input         psPowerDevice : Power device
+ @Input         pfnDevicePrePower : regular device pre power callback
+ @Input         pfnDevicePostPower : regular device post power callback
+ @Input         pfnSystemPrePower : regular system pre power callback
+ @Input         pfnDevicePostPower : regular system post power callback
+ @Input         pfnSystemPrePower : regular device pre power callback
+ @Input         pfnSystemPostPower : regular device pre power callback
+ @Input         pfnForcedIdleRequest : forced idle request callback
+ @Input         pfnForcedIdleCancelRequest : forced idle request cancel callback
+
+******************************************************************************/
+void PVRSRVSetPowerCallbacks(PPVRSRV_DEVICE_NODE				psDeviceNode,
+							 PVRSRV_POWER_DEV					*psPowerDevice,
+							 PFN_PRE_POWER						pfnDevicePrePower,
+							 PFN_POST_POWER					    pfnDevicePostPower,
+							 PFN_SYS_PRE_POWER				    pfnSystemPrePower,
+							 PFN_SYS_POST_POWER			        pfnSystemPostPower,
+							 PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+							 PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest);
+
+/* Type PFN_DC_REGISTER_POWER */
+PVRSRV_ERROR PVRSRVRegisterPowerDevice(PPVRSRV_DEVICE_NODE				psDeviceNode,
+									   PFN_PRE_POWER					pfnDevicePrePower,
+									   PFN_POST_POWER					pfnDevicePostPower,
+									   PFN_SYS_PRE_POWER			    pfnSystemPrePower,
+									   PFN_SYS_POST_POWER			    pfnSystemPostPower,
+									   PFN_PRE_CLOCKSPEED_CHANGE		pfnPreClockSpeedChange,
+									   PFN_POST_CLOCKSPEED_CHANGE		pfnPostClockSpeedChange,
+									   PFN_FORCED_IDLE_REQUEST			pfnForcedIdleRequest,
+									   PFN_FORCED_IDLE_CANCEL_REQUEST	pfnForcedIdleCancelRequest,
+									   PFN_GPU_UNITS_POWER_CHANGE		pfnGPUUnitsPowerChange,
+									   IMG_HANDLE						hDevCookie,
+									   PVRSRV_DEV_POWER_STATE			eCurrentPowerState,
+									   PVRSRV_DEV_POWER_STATE			eDefaultPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVRemovePowerDevice
+
+ @Description
+
+ Removes device from power management register. Device is located by Device Index
+
+ @Input		psDeviceNode : Device node
+
+******************************************************************************/
+void PVRSRVRemovePowerDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVGetDevicePowerState
+
+ @Description
+
+	Return the device power state
+
+ @Input		psDeviceNode : Device node
+ @Output	pePowerState : Current power state
+
+ @Return	PVRSRV_ERROR_UNKNOWN_POWER_STATE if device could not be found.
+            PVRSRV_OK otherwise.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVGetDevicePowerState(PCPVRSRV_DEVICE_NODE psDeviceNode,
+									   PPVRSRV_DEV_POWER_STATE pePowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVIsDevicePowered
+
+ @Description
+
+	Whether the device is powered, for the purposes of lockup detection.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVIsDevicePowered(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVGetSystemPowerState
+
+ @Description
+
+	Return the system power state
+
+ @Input		psDeviceNode : Device node
+ @Output	peCurrentSysPowerState : Current power state
+
+ @Return	PVRSRV_ERROR_UNKNOWN_POWER_STATE if device could not be found.
+            PVRSRV_OK otherwise.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVGetSystemPowerState(PPVRSRV_DEVICE_NODE psDeviceNode,
+	                                   PPVRSRV_SYS_POWER_STATE peCurrentSysPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVIsSystemPowered
+
+ @Description
+
+	Whether the system layer is powered, for ensuring the RGX regbank is powered
+	during initial GPU driver configuration.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL PVRSRVIsSystemPowered(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVDevicePreClockSpeedChange
+
+@Description    This function is called before a voltage/frequency change is
+                made to the GPU HW. It informs the host driver of the intention
+                to make a DVFS change. If allows the host driver to idle
+                the GPU and begin a hold off period from starting new work
+                on the GPU.
+                When this call succeeds the caller *must* call
+                PVRSRVDevicePostClockSpeedChange() to end the hold off period
+                to allow new work to be submitted to the GPU.
+
+                Called from system layer or OS layer implementation that
+                is responsible for triggering a GPU DVFS transition.
+
+@Input          psDeviceNode pointer to the device affected by DVFS transition.
+@Input          bIdleDevice  when True, the driver will wait for the GPU to
+                             reach an idle state before the call returns.
+@Input          pvInfo       unused
+
+@Return         PVRSRV_OK    on success, power lock acquired and held on exit,
+                             GPU idle.
+                PVRSRV_ERROR on failure, power lock not held on exit, do not
+                             call PVRSRVDevicePostClockSpeedChange().
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDevicePreClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+											 IMG_BOOL	bIdleDevice,
+											 void	*pvInfo);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVDevicePostClockSpeedChange
+
+@Description    This function is called after a voltage/frequency change has
+                been made to the GPU HW following a call to
+                PVRSRVDevicePreClockSpeedChange().
+                Before calling this function the caller must ensure the system
+                data RGX_DATA->RGX_TIMING_INFORMATION->ui32CoreClockSpeed has
+                been updated with the new frequency set, measured in Hz.
+                The function informs the host driver that the DVFS change has
+                completed. The driver will end the work hold off period, cancel
+                the device idle period and update its time data records.
+                When this call returns work submissions are unblocked and
+                are submitted to the GPU as normal.
+                This function *must* not be called if the preceding call to
+                PVRSRVDevicePreClockSpeedChange() failed.
+
+                Called from system layer or OS layer implementation that
+                is responsible for triggering a GPU DVFS transition.
+
+@Input          psDeviceNode pointer to the device affected by DVFS transition.
+@Input          bIdleDevice  when True, the driver will cancel the GPU
+                             device idle state before the call returns. Value
+                             given must match that used in the call to
+                             PVRSRVDevicePreClockSpeedChange() otherwise
+                             undefined behaviour will result.
+@Input          pvInfo       unused
+
+@Return         void         power lock released, no longer held on exit.
+*/ /**************************************************************************/
+void PVRSRVDevicePostClockSpeedChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+									  IMG_BOOL		bIdleDevice,
+									  void		*pvInfo);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVDeviceIdleRequestKM
+
+ @Description Perform device-specific processing required to force the device
+              idle. The device power-lock might be temporarily released (and
+              again re-acquired) during the course of this call, hence to
+              maintain lock-ordering power-lock should be the last acquired
+              lock before calling this function
+
+ @Input       psDeviceNode         : Device node
+
+ @Input       pfnIsDefaultStateOff : When specified, the idle request is only
+                                     processed if this function passes.
+
+ @Input       bDeviceOffPermitted  : IMG_TRUE if the transition should not fail
+                                       if device off
+                                     IMG_FALSE if the transition should fail if
+                                       device off
+
+ @Return      PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED
+                                     When re-acquisition of power-lock failed.
+                                     This error NEEDS EXPLICIT HANDLING at call
+                                     site as it signifies the caller needs to
+                                     AVOID calling PVRSRVPowerUnlock, since
+                                     power-lock is no longer "possessed" by
+                                     this context.
+
+              PVRSRV_OK              When idle request succeeded.
+              PVRSRV_ERROR           Other system errors.
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode,
+					PFN_SYS_DEV_IS_DEFAULT_STATE_OFF	pfnIsDefaultStateOff,
+					IMG_BOOL				bDeviceOffPermitted);
+
+/*!
+******************************************************************************
+
+ @Function	PVRSRVDeviceIdleCancelRequestKM
+
+ @Description Perform device-specific processing required to cancel the forced idle state
+              on the device, returning to normal operation.
+
+ @Input		psDeviceNode : Device node
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceIdleCancelRequestKM(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+@Function       PVRSRVDeviceGPUUnitsPowerChange
+@Description    Request from system layer for changing power state of GPU
+                units
+@Input          psDeviceNode            RGX Device Node.
+@Input          ui32NewValue            Value indicating the new power state
+                                        of GPU units. how this is interpreted
+                                        depends upon the device-specific
+                                        function subsequently called by the
+                                        server via a pfn.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceGPUUnitsPowerChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+					IMG_UINT32	ui32NewValue);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+void PVRSRVSetFirmwareStartTime(PVRSRV_POWER_DEV *psPowerDevice, IMG_UINT32 ui32TimeStamp);
+
+void PVRSRVSetFirmwareHandshakeIdleTime(PVRSRV_POWER_DEV *psPowerDevice, IMG_UINT64 ui64Duration);
+
+int PVRSRVPowerStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+#endif
+
+#endif /* POWER_H */
+
+/******************************************************************************
+ End of file (power.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/powervr/buffer_attribs.h b/drivers/gpu/drm/img-rogue/powervr/buffer_attribs.h
new file mode 100644
index 000000000000..41eaaaecd19e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/powervr/buffer_attribs.h
@@ -0,0 +1,193 @@
+/*************************************************************************/ /*!
+@File
+@Title          3D types for use by IMG APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef POWERVR_BUFFER_ATTRIBS_H
+#define POWERVR_BUFFER_ATTRIBS_H
+
+/*!
+ * Memory layouts
+ * Defines how pixels are laid out within a surface.
+ */
+typedef enum
+{
+	IMG_MEMLAYOUT_STRIDED,       /**< Resource is strided, one row at a time */
+	IMG_MEMLAYOUT_TWIDDLED,      /**< Resource is 2D twiddled to match HW */
+	IMG_MEMLAYOUT_3DTWIDDLED,    /**< Resource is 3D twiddled, classic style */
+	IMG_MEMLAYOUT_TILED,         /**< Resource is tiled, tiling config specified elsewhere. */
+	IMG_MEMLAYOUT_PAGETILED,     /**< Resource is pagetiled */
+	IMG_MEMLAYOUT_INVNTWIDDLED,  /**< Resource is 2D twiddled !N style */
+} IMG_MEMLAYOUT;
+
+/*!
+ * Rotation types
+ */
+typedef enum
+{
+	IMG_ROTATION_0DEG = 0,
+	IMG_ROTATION_90DEG = 1,
+	IMG_ROTATION_180DEG = 2,
+	IMG_ROTATION_270DEG = 3,
+	IMG_ROTATION_FLIP_Y = 4,
+
+	IMG_ROTATION_BAD = 255,
+} IMG_ROTATION;
+
+/*!
+ * Alpha types.
+ */
+typedef enum
+{
+	IMG_COLOURSPACE_FORMAT_UNKNOWN                 =  0x0UL << 16,
+	IMG_COLOURSPACE_FORMAT_LINEAR                  =  0x1UL << 16,
+	IMG_COLOURSPACE_FORMAT_SRGB                    =  0x2UL << 16,
+	IMG_COLOURSPACE_FORMAT_SCRGB                   =  0x3UL << 16,
+	IMG_COLOURSPACE_FORMAT_SCRGB_LINEAR            =  0x4UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3_LINEAR       =  0x5UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3              =  0x6UL << 16,
+	IMG_COLOURSPACE_FORMAT_BT2020_PQ               =  0x7UL << 16,
+	IMG_COLOURSPACE_FORMAT_BT2020_LINEAR           =  0x8UL << 16,
+	IMG_COLOURSPACE_FORMAT_DISPLAY_P3_PASSTHROUGH  =  0x9UL << 16,
+	IMG_COLOURSPACE_FORMAT_MASK                    =  0xFUL << 16,
+} IMG_COLOURSPACE_FORMAT;
+
+/*!
+ * Determines if FB Compression is Lossy
+ */
+#define IS_FBCDC_LOSSY(mode)			((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_TRUE : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_TRUE : IMG_FALSE)
+
+/*!
+ * Determines if FB Compression is Packed
+ */
+#define IS_FBCDC_PACKED(mode)			((mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8) ? IMG_TRUE : IMG_FALSE)
+
+/*!
+ * Returns type of FB Compression
+ */
+#define GET_FBCDC_BLOCK_TYPE(mode)		((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8)   ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : mode)
+
+/*!
+ * Adds Packing compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_PACKING(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_PACKED_8x8 : mode)
+
+/*!
+ * Removes Packing compression setting from mode
+ */
+#define FBCDC_MODE_REMOVE_PACKING(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_PACKED_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8 : mode)
+
+/*!
+ * Adds Lossy25 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY25(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2 : mode)
+
+/*!
+ * Adds Lossy37 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY37(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2 : mode)
+
+/*!
+ * Adds Lossy50 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY50(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2 : mode)
+
+/*!
+ * Adds Lossy75 compression setting to mode if viable
+ */
+#define FBCDC_MODE_ADD_LOSSY75(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_8x8) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_16x4) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_32x2) ? IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2 : mode)
+
+/*!
+ * Removes Lossy compression setting from mode
+ */
+#define FBCDC_MODE_REMOVE_LOSSY(mode)	((mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8) ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8)  ? IMG_FB_COMPRESSION_DIRECT_8x8  : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4) ? IMG_FB_COMPRESSION_DIRECT_16x4 : \
+										(mode == IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2) ? IMG_FB_COMPRESSION_DIRECT_32x2 : mode)
+
+/*!
+ * Types of framebuffer compression
+ */
+typedef enum
+{
+	IMG_FB_COMPRESSION_NONE,
+	IMG_FB_COMPRESSION_DIRECT_8x8,
+	IMG_FB_COMPRESSION_DIRECT_16x4,
+	IMG_FB_COMPRESSION_DIRECT_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY25_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY50_32x2,
+	IMG_FB_COMPRESSION_DIRECT_PACKED_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY75_32x2,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_8x8,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_16x4,
+	IMG_FB_COMPRESSION_DIRECT_LOSSY37_32x2,
+} IMG_FB_COMPRESSION;
+
+
+#endif /* POWERVR_BUFFER_ATTRIBS_H */
diff --git a/drivers/gpu/drm/img-rogue/powervr/img_drm_fourcc.h b/drivers/gpu/drm/img-rogue/powervr/img_drm_fourcc.h
new file mode 100644
index 000000000000..d4717db1cccb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/powervr/img_drm_fourcc.h
@@ -0,0 +1,161 @@
+/*************************************************************************/ /*!
+@File
+@Title          Wrapper around drm_fourcc.h
+@Description    FourCCs and DRM framebuffer modifiers that are not in the
+                Kernel's and libdrm's drm_fourcc.h can be added here.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DRM_FOURCC_H
+#define IMG_DRM_FOURCC_H
+
+#if defined(__KERNEL__)
+#include <drm/drm_fourcc.h>
+#else
+/*
+ * Include types.h to workaround versions of libdrm older than 2.4.68
+ * not including the correct headers.
+ */
+#include <linux/types.h>
+
+#include <drm_fourcc.h>
+#endif
+
+/*
+ * Don't get too inspired by this example :)
+ * ADF doesn't support DRM modifiers, so the memory layout had to be
+ * included in the fourcc name, but the proper way to specify information
+ * additional to pixel formats is to use DRM modifiers.
+ *
+ * See upstream drm_fourcc.h for the proper naming convention.
+ */
+#ifndef DRM_FORMAT_BGRA8888_DIRECT_16x4
+#define DRM_FORMAT_BGRA8888_DIRECT_16x4 fourcc_code('I', 'M', 'G', '0')
+#endif
+
+#if !defined(__KERNEL__)
+/*
+ * A definition for the same format was added in Linux kernel 5.2 in commit
+ * 88ab9c76d191ad8645b483f31e2b394b0f3e280e. As such, this definition has been
+ * deprecated and the DRM_FORMAT_ABGR16161616F kernel define should be used
+ * instead of this one.
+ */
+#define DRM_FORMAT_ABGR16_IMG_DEPRECATED fourcc_code('I', 'M', 'G', '1')
+#endif
+
+/*
+ * Upstream does not have a packed 10 Bits Per Channel YVU format yet,
+ * so let`s make one up.
+ * Note: at the moment this format is not intended to be used with
+ *       a framebuffer, so the kernels core DRM doesn`t need to know
+ *       about this format. This means that the kernel doesn`t need
+ *       to be patched.
+ */
+#if !defined(__KERNEL__)
+#define DRM_FORMAT_YVU444_PACK10_IMG fourcc_code('I', 'M', 'G', '2')
+#define DRM_FORMAT_YUV422_2PLANE_PACK10_IMG fourcc_code('I', 'M', 'G', '3')
+#define DRM_FORMAT_YUV420_2PLANE_PACK10_IMG fourcc_code('I', 'M', 'G', '4')
+#endif
+
+/*
+ * Value chosen in the middle of 255 pool to minimise the chance of hitting
+ * the same value potentially defined by other vendors in the drm_fourcc.h
+ */
+#define DRM_FORMAT_MOD_VENDOR_PVR 0x92
+
+#ifndef DRM_FORMAT_MOD_VENDOR_NONE
+#define DRM_FORMAT_MOD_VENDOR_NONE 0
+#endif
+
+#ifndef DRM_FORMAT_RESERVED
+#define DRM_FORMAT_RESERVED ((1ULL << 56) - 1)
+#endif
+
+#define img_fourcc_mod_combine(uiModHi, uiModLo) \
+	((__u64) ((__u32) (uiModHi)) << 32 | (__u64) ((__u32) (uiModLo)))
+
+#define img_fourcc_mod_hi(ui64Mod) \
+	((__u32) ((__u64) (ui64Mod) >> 32))
+
+#define img_fourcc_mod_lo(ui64Mod) \
+	((__u32) ((__u64) (ui64Mod)) & 0xffffffff)
+
+#ifndef fourcc_mod_code
+#define fourcc_mod_code(vendor, val) \
+	((((__u64)DRM_FORMAT_MOD_VENDOR_## vendor) << 56) | (val & 0x00ffffffffffffffULL))
+#endif
+
+#ifndef DRM_FORMAT_MOD_INVALID
+#define DRM_FORMAT_MOD_INVALID fourcc_mod_code(NONE, DRM_FORMAT_RESERVED)
+#endif
+
+#ifndef DRM_FORMAT_MOD_LINEAR
+#define DRM_FORMAT_MOD_LINEAR fourcc_mod_code(NONE, 0)
+#endif
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V1      fourcc_mod_code(PVR, 3)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V1     fourcc_mod_code(PVR, 9)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V7      fourcc_mod_code(PVR, 6)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V7     fourcc_mod_code(PVR, 12)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V10     fourcc_mod_code(PVR, 21)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V10    fourcc_mod_code(PVR, 22)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V10    fourcc_mod_code(PVR, 23)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12     fourcc_mod_code(PVR, 15)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12    fourcc_mod_code(PVR, 16)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V13             fourcc_mod_code(PVR, 24)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V13     fourcc_mod_code(PVR, 25)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V13     fourcc_mod_code(PVR, 26)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V13     fourcc_mod_code(PVR, 27)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V13            fourcc_mod_code(PVR, 28)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V13    fourcc_mod_code(PVR, 29)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V13    fourcc_mod_code(PVR, 30)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V13    fourcc_mod_code(PVR, 31)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V13     fourcc_mod_code(PVR, 32)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V13    fourcc_mod_code(PVR, 33)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V14             fourcc_mod_code(PVR, 34)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V14     fourcc_mod_code(PVR, 35)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V14     fourcc_mod_code(PVR, 36)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V14     fourcc_mod_code(PVR, 37)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V14     fourcc_mod_code(PVR, 38)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V14            fourcc_mod_code(PVR, 39)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V14    fourcc_mod_code(PVR, 40)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V14    fourcc_mod_code(PVR, 41)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V14    fourcc_mod_code(PVR, 42)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V14    fourcc_mod_code(PVR, 43)
+
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V14            fourcc_mod_code(PVR, 44)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_32x2_V14    fourcc_mod_code(PVR, 45)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_32x2_V14    fourcc_mod_code(PVR, 46)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_32x2_V14    fourcc_mod_code(PVR, 47)
+#define DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_32x2_V14    fourcc_mod_code(PVR, 48)
+
+#endif /* IMG_DRM_FOURCC_H */
diff --git a/drivers/gpu/drm/img-rogue/powervr/mem_types.h b/drivers/gpu/drm/img-rogue/powervr/mem_types.h
new file mode 100644
index 000000000000..a6dce8fe9889
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/powervr/mem_types.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File
+@Title          Public types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef POWERVR_TYPES_H
+#define POWERVR_TYPES_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(_MSC_VER)
+	#include "msvc_types.h"
+#elif defined(__linux__) && defined(__KERNEL__)
+	#include <linux/types.h>
+	#include <linux/compiler.h>
+#else
+	#include <stdint.h>
+	#define __iomem
+#endif
+
+typedef void *IMG_CPU_VIRTADDR;
+
+/* device virtual address */
+typedef struct
+{
+	uint64_t uiAddr;
+#define IMG_CAST_TO_DEVVADDR_UINT(var)		(uint64_t)(var)
+
+} IMG_DEV_VIRTADDR;
+
+typedef uint64_t IMG_DEVMEM_SIZE_T;
+typedef uint64_t IMG_DEVMEM_ALIGN_T;
+typedef uint64_t IMG_DEVMEM_OFFSET_T;
+typedef uint32_t IMG_DEVMEM_LOG2ALIGN_T;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/powervr/pvrsrv_sync_ext.h b/drivers/gpu/drm/img-rogue/powervr/pvrsrv_sync_ext.h
new file mode 100644
index 000000000000..933212e03c17
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/powervr/pvrsrv_sync_ext.h
@@ -0,0 +1,74 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services external synchronisation interface header
+@Description    Defines synchronisation structures that are visible internally
+                and externally
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        MIT
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef POWERVR_SYNC_EXT_H
+#define POWERVR_SYNC_EXT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*!
+ * Number of sync prims still used internally in operations
+ */
+#define PVRSRV_MAX_SYNC_PRIMS 4U
+
+/*!
+ * Maximum number of dev var updates passed in a kick call
+ */
+#define PVRSRV_MAX_DEV_VARS 13U
+
+/*!
+ * Number of UFOs in operations
+ */
+#define	PVRSRV_MAX_SYNCS (PVRSRV_MAX_SYNC_PRIMS + PVRSRV_MAX_DEV_VARS)
+
+/*! Implementation independent types for passing fence/timeline to Services.
+ */
+typedef int32_t PVRSRV_FENCE;
+typedef int32_t PVRSRV_TIMELINE;
+typedef int32_t PVRSRV_EXP_FENCE_CTX;
+
+/*! Maximum length for an annotation name string for fence sync model objects.
+ */
+#define PVRSRV_SYNC_NAME_LENGTH 32U
+
+/* Macros for API callers using the fence sync model
+ */
+#define PVRSRV_NO_TIMELINE      ((PVRSRV_TIMELINE) -1)
+#define PVRSRV_NO_FENCE         ((PVRSRV_FENCE)    -1)
+#define PVRSRV_NO_EXP_FENCE_CTX ((PVRSRV_EXP_FENCE_CTX) -1)
+#define PVRSRV_NO_FENCE_PTR     NULL
+#define PVRSRV_NO_TIMELINE_PTR  NULL
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/private_data.h b/drivers/gpu/drm/img-rogue/private_data.h
new file mode 100644
index 000000000000..60a1fac0b970
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/private_data.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Linux private data structure
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(INCLUDED_PRIVATE_DATA_H)
+#define INCLUDED_PRIVATE_DATA_H
+
+#include <linux/fs.h>
+
+#include "connection_server.h"
+#include "pvr_drm.h"
+
+#define PVR_SRVKM_PRIV_DATA_IDX 0
+#define PVR_SYNC_PRIV_DATA_IDX  1
+
+#define PVR_NUM_PRIV_DATA_IDXS  2
+
+CONNECTION_DATA *LinuxServicesConnectionFromFile(struct file *pFile);
+CONNECTION_DATA *LinuxSyncConnectionFromFile(struct file *pFile);
+
+#endif /* !defined(INCLUDED_PRIVATE_DATA_H) */
diff --git a/drivers/gpu/drm/img-rogue/proc_stats.h b/drivers/gpu/drm/img-rogue/proc_stats.h
new file mode 100644
index 000000000000..72c14c5f8e4d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/proc_stats.h
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File
+@Title          Process and driver statistic definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PROC_STATS_H
+#define PROC_STATS_H
+
+#define DKP_HIDDEN "hidden"
+
+#define PROCESS_STAT_KMALLOC                   X(PVRSRV_PROCESS_STAT_TYPE_KMALLOC,                "MemoryUsageKMalloc",                "host_kmalloc")
+#define PROCESS_STAT_KMALLOC_MAX               X(PVRSRV_PROCESS_STAT_TYPE_KMALLOC_MAX,            "MemoryUsageKMallocMax",             DKP_HIDDEN)
+
+#define PROCESS_STAT_VMALLOC                   X(PVRSRV_PROCESS_STAT_TYPE_VMALLOC,                "MemoryUsageVMalloc",                "host_vmalloc")
+#define PROCESS_STAT_VMALLOC_MAX               X(PVRSRV_PROCESS_STAT_TYPE_VMALLOC_MAX,            "MemoryUsageVMallocMax",             DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_PAGES_PT_UMA        X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA,     "MemoryUsageAllocPTMemoryUMA",       "host_mem_dev_pt")
+#define PROCESS_STAT_ALLOC_PAGES_PT_UMA_MAX    X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA_MAX, "MemoryUsageAllocPTMemoryUMAMax",    DKP_HIDDEN)
+
+#define PROCESS_STAT_VMAP_PT_UMA               X(PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA,            "MemoryUsageVMapPTUMA",              DKP_HIDDEN)
+#define PROCESS_STAT_VMAP_PT_UMA_MAX           X(PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA_MAX,        "MemoryUsageVMapPTUMAMax",           DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_PAGES_PT_LMA        X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA,     "MemoryUsageAllocPTMemoryLMA",       "local_mem_dev_pt")
+#define PROCESS_STAT_ALLOC_PAGES_PT_LMA_MAX    X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA_MAX, "MemoryUsageAllocPTMemoryLMAMax",    DKP_HIDDEN)
+
+#define PROCESS_STAT_IOREMAP_PT_LMA            X(PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA,         "MemoryUsageIORemapPTLMA",           DKP_HIDDEN)
+#define PROCESS_STAT_IOREMAP_PT_LMA_MAX        X(PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA_MAX,     "MemoryUsageIORemapPTLMAMax",        DKP_HIDDEN)
+
+#define PROCESS_STAT_ALLOC_LMA_PAGES           X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES,        "MemoryUsageAllocGPUMemLMA",         "local_mem_dev_buf")
+#define PROCESS_STAT_ALLOC_LMA_PAGES_MAX       X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES_MAX,    "MemoryUsageAllocGPUMemLMAMax",      DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES          X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES,       "MemoryUsageZombieGPUMemLMA",        "local_mem_dev_buf_purgeable")
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX      X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES_MAX,   "MemoryUsageZombieGPUMemLMAMax",     DKP_HIDDEN)
+#else
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES
+#define PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX
+#endif
+
+#define PROCESS_STAT_ALLOC_UMA_PAGES           X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES,        "MemoryUsageAllocGPUMemUMA",         "host_mem_dev_buf")
+#define PROCESS_STAT_ALLOC_UMA_PAGES_MAX       X(PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES_MAX,    "MemoryUsageAllocGPUMemUMAMax",      DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES          X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES,       "MemoryUsageZombieGPUMemUMA",        "host_mem_dev_buf_purgeable")
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX      X(PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES_MAX,   "MemoryUsageZombieGPUMemUMAMax",     DKP_HIDDEN)
+#else
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES
+#define PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX
+#endif
+
+#define PROCESS_STAT_MAP_UMA_LMA_PAGES         X(PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES,      "MemoryUsageMappedGPUMemUMA/LMA",    DKP_HIDDEN)
+#define PROCESS_STAT_MAP_UMA_LMA_PAGES_MAX     X(PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES_MAX,  "MemoryUsageMappedGPUMemUMA/LMAMax", DKP_HIDDEN)
+
+#define PROCESS_STAT_DMA_BUF_IMPORT            X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT,         "MemoryUsageDmaBufImport",           "dma_buf_import")
+#define PROCESS_STAT_DMA_BUF_IMPORT_MAX        X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT_MAX,     "MemoryUsageDmaBufImportMax",        DKP_HIDDEN)
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+#define PROCESS_STAT_DMA_BUF_ZOMBIE            X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE,         "MemoryUsageDmaBufZombie",           "dma_buf_purgeable")
+#define PROCESS_STAT_DMA_BUF_ZOMBIE_MAX        X(PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE_MAX,     "MemoryUsageDmaBufZombieMax",        DKP_HIDDEN)
+#else
+#define PROCESS_STAT_DMA_BUF_ZOMBIE
+#define PROCESS_STAT_DMA_BUF_ZOMBIE_MAX
+#endif
+
+#define PROCESS_STAT_TOTAL                     X(PVRSRV_PROCESS_STAT_TYPE_TOTAL,                  "MemoryUsageTotal",                  DKP_HIDDEN)
+#define PROCESS_STAT_TOTAL_MAX                 X(PVRSRV_PROCESS_STAT_TYPE_TOTAL_MAX,              "MemoryUsageTotalMax",               DKP_HIDDEN)
+
+/* Process stat keys */
+#define PVRSRV_PROCESS_STAT_KEY \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_KMALLOC_MAX \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_VMALLOC_MAX \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA_MAX \
+	PROCESS_STAT_VMAP_PT_UMA \
+	PROCESS_STAT_VMAP_PT_UMA_MAX \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA_MAX \
+	PROCESS_STAT_IOREMAP_PT_LMA \
+	PROCESS_STAT_IOREMAP_PT_LMA_MAX \
+	PROCESS_STAT_ALLOC_LMA_PAGES \
+	PROCESS_STAT_ALLOC_LMA_PAGES_MAX \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES_MAX \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ALLOC_UMA_PAGES_MAX \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES_MAX \
+	PROCESS_STAT_MAP_UMA_LMA_PAGES \
+	PROCESS_STAT_MAP_UMA_LMA_PAGES_MAX \
+	PROCESS_STAT_DMA_BUF_IMPORT \
+	PROCESS_STAT_DMA_BUF_IMPORT_MAX \
+	PROCESS_STAT_DMA_BUF_ZOMBIE \
+	PROCESS_STAT_DMA_BUF_ZOMBIE_MAX \
+	PROCESS_STAT_TOTAL \
+	PROCESS_STAT_TOTAL_MAX
+
+#if defined(SUPPORT_LINUX_FDINFO)
+/* DKP process stats within the drm-memory-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_MEMORY \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES
+
+/* DKP process stats within the drm-shared-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_SHARED \
+	PROCESS_STAT_DMA_BUF_IMPORT \
+	PROCESS_STAT_DMA_BUF_ZOMBIE
+
+/* DKP process stats within the drm-total-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_TOTAL \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_LMA_PAGES
+
+/* DKP process stats within the drm-resident-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_RESIDENT \
+	PROCESS_STAT_KMALLOC \
+	PROCESS_STAT_VMALLOC \
+	PROCESS_STAT_ALLOC_PAGES_PT_UMA \
+	PROCESS_STAT_ALLOC_UMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_ALLOC_PAGES_PT_LMA \
+	PROCESS_STAT_ALLOC_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+
+/* DKP process stats within the drm-purgeable-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_PURGEABLE \
+	PROCESS_STAT_ZOMBIE_LMA_PAGES \
+	PROCESS_STAT_ZOMBIE_UMA_PAGES \
+	PROCESS_STAT_DMA_BUF_ZOMBIE
+
+/* DKP process stats within the drm-active-<region> key */
+#define PVRSRV_DKP_MEM_STAT_GROUP_ACTIVE
+
+#endif
+
+/* X-Macro for Device stat keys */
+#define PVRSRV_DEVICE_STAT_KEY \
+	X(PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS, "Connections") \
+	X(PVRSRV_DEVICE_STAT_TYPE_MAX_CONNECTIONS, "ConnectionsMax") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_OOMS, "RenderContextOutOfMemoryEvents") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_PRS, "RenderContextPartialRenders") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_GROWS, "RenderContextGrows") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_PUSH_GROWS, "RenderContextPushGrows") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_TA_STORES, "RenderContextTAStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_3D_STORES, "RenderContext3DStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_CDM_STORES, "RenderContextCDMStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_TDM_STORES, "RenderContextTDMStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_RC_RAY_STORES, "RenderContextRayStores") \
+	X(PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_APP, "ZSBufferRequestsByApp") \
+	X(PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_FW, "ZSBufferRequestsByFirmware") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_APP, "FreeListGrowRequestsByApp") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_FW, "FreeListGrowRequestsByFirmware") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_PAGES_INIT, "FreeListInitialPages") \
+	X(PVRSRV_DEVICE_STAT_TYPE_FREELIST_MAX_PAGES, "FreeListMaxPages") \
+	X(PVRSRV_DEVICE_STAT_TYPE_OOM_VIRTMEM_COUNT, "MemoryOOMCountDeviceVirtual") \
+	X(PVRSRV_DEVICE_STAT_TYPE_OOM_PHYSMEM_COUNT, "MemoryOOMCountPhysicalHeap") \
+	X(PVRSRV_DEVICE_STAT_TYPE_INVALID_VIRTMEM, "MemoryOOMCountDeviceVirtualAtAddr")
+
+/* X-Macro for Driver stat keys */
+#define PVRSRV_DRIVER_STAT_KEY \
+	X(PVRSRV_DRIVER_STAT_TYPE_KMALLOC, "MemoryUsageKMalloc") \
+	X(PVRSRV_DRIVER_STAT_TYPE_KMALLOC_MAX, "MemoryUsageKMallocMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMALLOC, "MemoryUsageVMalloc") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMALLOC_MAX, "MemoryUsageVMallocMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, "MemoryUsageAllocPTMemoryUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA_MAX, "MemoryUsageAllocPTMemoryUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, "MemoryUsageVMapPTUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA_MAX, "MemoryUsageVMapPTUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, "MemoryUsageAllocPTMemoryLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA_MAX, "MemoryUsageAllocPTMemoryLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, "MemoryUsageIORemapPTLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA_MAX, "MemoryUsageIORemapPTLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, "MemoryUsageAllocGPUMemLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA_MAX, "MemoryUsageAllocGPUMemLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, "MemoryUsageZombieGPUMemLMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA_MAX, "MemoryUsageZombieGPUMemLMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, "MemoryUsageAllocGPUMemUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_MAX, "MemoryUsageAllocGPUMemUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, "MemoryUsageZombieGPUMemUMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA_MAX, "MemoryUsageZombieGPUMemUMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, "MemoryUsageAllocGPUMemUMAPool") \
+	X(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL_MAX, "MemoryUsageAllocGPUMemUMAPoolMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, "MemoryUsageMappedGPUMemUMA/LMA") \
+	X(PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA_MAX, "MemoryUsageMappedGPUMemUMA/LMAMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, "MemoryUsageDmaBufImport") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT_MAX, "MemoryUsageDmaBufImportMax") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, "MemoryUsageDmaBufZombie") \
+	X(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE_MAX, "MemoryUsageDmaBufZombieMax")
+
+
+typedef enum {
+#define X(stat_type, stat_str, drm_str) stat_type,
+	PVRSRV_PROCESS_STAT_KEY
+#undef X
+	PVRSRV_PROCESS_STAT_TYPE_COUNT
+}PVRSRV_PROCESS_STAT_TYPE;
+
+typedef enum {
+#define X(stat_type, stat_str) stat_type,
+	PVRSRV_DEVICE_STAT_KEY
+#undef X
+	PVRSRV_DEVICE_STAT_TYPE_COUNT
+}PVRSRV_DEVICE_STAT_TYPE;
+
+typedef enum {
+#define X(stat_type, stat_str) stat_type,
+	PVRSRV_DRIVER_STAT_KEY
+#undef X
+	PVRSRV_DRIVER_STAT_TYPE_COUNT
+}PVRSRV_DRIVER_STAT_TYPE;
+
+#endif // PROC_STATS_H
diff --git a/drivers/gpu/drm/img-rogue/process_stats.c b/drivers/gpu/drm/img-rogue/process_stats.c
new file mode 100644
index 000000000000..77441c578089
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/process_stats.c
@@ -0,0 +1,3492 @@
+/*************************************************************************/ /*!
+@File
+@Title          Process based statistics
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Manages a collection of statistics based around a process
+                and referenced via OS agnostic methods.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvr_debug.h"
+#include "lock.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "process_stats.h"
+#include "ri_server.h"
+#include "hash.h"
+#include "connection_server.h"
+#include "pvrsrv.h"
+#include "proc_stats.h"
+#include "pvr_ricommon.h"
+#include "di_server.h"
+#include "dllist.h"
+#if defined(__linux__)
+#include "trace_events.h"
+#endif
+
+#if defined(SUPPORT_LINUX_FDINFO) && defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "dkp_impl.h"
+
+#define PROC_STATS_FDINFO_ENABLED
+#endif
+
+/* Enabled OS Statistics entries: DEBUGFS on Linux, undefined for other OSs */
+#if defined(__linux__) && ( \
+	defined(PVRSRV_ENABLE_PERPID_STATS) || \
+	defined(PVRSRV_ENABLE_CACHEOP_STATS) || \
+	defined(PVRSRV_ENABLE_MEMORY_STATS) || \
+	defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) )
+#define ENABLE_DEBUGFS_PIDS
+#endif
+
+/* Enable GPU memory accounting tracepoint */
+#if defined(__linux__) && ( \
+	defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) )
+#define ENABLE_GPU_MEM_TRACEPOINT
+#endif
+
+/*
+ * Maximum history of process statistics that will be kept.
+ */
+#define MAX_DEAD_LIST_PROCESSES  (10)
+
+/*
+ * Maximum size of a process name to be displayed in process_stats.
+ */
+#define MAX_PROC_NAME_LEN  (16)
+
+/*
+ * Definition of all the strings used to format process based statistics.
+ */
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+/* Array of Process stat type defined using the X-Macro */
+#define X(stat_type, stat_str, drm_str) stat_str,
+static const IMG_CHAR *const pszProcessStatType[PVRSRV_PROCESS_STAT_TYPE_COUNT] = { PVRSRV_PROCESS_STAT_KEY };
+#undef X
+#define X(stat_type, stat_str) stat_str,
+static const IMG_CHAR *const pszDeviceStatType[PVRSRV_DEVICE_STAT_TYPE_COUNT] = { PVRSRV_DEVICE_STAT_KEY };
+#undef X
+#endif
+
+/* Array of Driver stat type defined using the X-Macro */
+#define X(stat_type, stat_str) stat_str,
+static const IMG_CHAR *const pszDriverStatType[PVRSRV_DRIVER_STAT_TYPE_COUNT] = { PVRSRV_DRIVER_STAT_KEY };
+#undef X
+
+/* structure used in hash table to track statistic entries */
+typedef struct {
+	size_t	   uiSizeInBytes;
+	IMG_PID	   uiPid;
+} _PVR_STATS_TRACKING_HASH_ENTRY;
+
+/* Function used internally to decrement tracked per-process statistic entries */
+static void _StatsDecrMemTrackedStat(_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry,
+                                     PVRSRV_MEM_ALLOC_TYPE eAllocType);
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+int RawProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+#endif
+int GlobalStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/* Note: all of the accesses to the global stats should be protected
+ * by the gsGlobalStats.hGlobalStatsLock lock. This means all of the
+ * invocations of macros *_GLOBAL_STAT_VALUE. */
+
+/* Macros for fetching stat values */
+#define GET_STAT_VALUE(ptr,var) (ptr)->i64StatValue[(var)]
+#define GET_GLOBAL_STAT_VALUE(idx) gsGlobalStats.ui64StatValue[idx]
+
+#define GET_GPUMEM_GLOBAL_STAT_VALUE() \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA) + \
+	GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT)
+
+#define GET_GPUMEM_PERPID_STAT_VALUE(ptr) \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES) + \
+	GET_STAT_VALUE((ptr), PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT)
+/*
+ * Macros for updating stat values.
+ */
+#define UPDATE_MAX_VALUE(a,b)					do { if ((b) > (a)) {(a) = (b);} } while (0)
+#define INCREASE_STAT_VALUE(ptr,var,val)		do { (ptr)->i64StatValue[(var)] += (IMG_INT64)(val); if ((ptr)->i64StatValue[(var)] > (ptr)->i64StatValue[(var##_MAX)]) {(ptr)->i64StatValue[(var##_MAX)] = (ptr)->i64StatValue[(var)];} } while (0)
+#define INCREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { (var).ui64StatValue[(idx)] += (IMG_UINT64)(val); if ((var).ui64StatValue[(idx)] > (var).ui64StatValue[(idx##_MAX)]) {(var).ui64StatValue[(idx##_MAX)] = (var).ui64StatValue[(idx)];} } while (0)
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+/* Allow stats to go negative */
+#define DECREASE_STAT_VALUE(ptr,var,val)		do { (ptr)->i64StatValue[(var)] -= (val); } while (0)
+#define DECREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { (var).ui64StatValue[(idx)] -= (val); } while (0)
+#else
+#define DECREASE_STAT_VALUE(ptr,var,val)		do { if ((ptr)->i64StatValue[(var)] >= (val)) { (ptr)->i64StatValue[(var)] -= (IMG_INT64)(val); } else { (ptr)->i64StatValue[(var)] = 0; } } while (0)
+#define DECREASE_GLOBAL_STAT_VALUE(var,idx,val)		do { if ((var).ui64StatValue[(idx)] >= (val)) { (var).ui64StatValue[(idx)] -= (IMG_UINT64)(val); } else { (var).ui64StatValue[(idx)] = 0; } } while (0)
+#endif
+#define MAX_CACHEOP_STAT 16
+#define INCREMENT_CACHEOP_STAT_IDX_WRAP(x) ((x+1) >= MAX_CACHEOP_STAT ? 0 : (x+1))
+#define DECREMENT_CACHEOP_STAT_IDX_WRAP(x) ((x-1) < 0 ? (MAX_CACHEOP_STAT-1) : (x-1))
+
+/*
+ * Track the search of one process when PVRSRV_DEBUG_LINUX_MEMORY_STATS
+ * is enabled.
+ */
+typedef enum _PVRSRV_PROC_SEARCH_STATE_
+{
+	PVRSRV_PROC_NOTFOUND,
+	PVRSRV_PROC_FOUND,
+	PVRSRV_PROC_RESURRECTED,
+} PVRSRV_PROC_SEARCH_STATE;
+
+/*
+ * Structures for holding statistics...
+ */
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+typedef struct _PVRSRV_MEM_ALLOC_REC_
+{
+	PVRSRV_MEM_ALLOC_TYPE           eAllocType;
+	void*                           pvCpuVAddr;
+	IMG_CPU_PHYADDR	                sCpuPAddr;
+	size_t                          uiBytes;
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	void*                           pvAllocdFromFile;
+	IMG_UINT32                      ui32AllocdFromLine;
+#endif
+} PVRSRV_MEM_ALLOC_REC;
+
+typedef struct PVRSRV_MEM_ALLOC_PRINT_DATA_TAG
+{
+	OSDI_IMPL_ENTRY *psEntry;
+	IMG_PID	        pid;
+	IMG_UINT32      ui32NumEntries;
+} PVRSRV_MEM_ALLOC_PRINT_DATA;
+#endif
+
+typedef struct _PVRSRV_PROCESS_STATS_ {
+
+	/* Linked list pointers */
+	DLLIST_NODE                    sNode;
+
+	/* Create per process lock that need to be held
+	 * to edit of its members */
+	POS_LOCK                       hLock;
+
+	/* OS level process ID */
+	IMG_PID	                       pid;
+	IMG_CHAR                       processName[MAX_PROC_NAME_LEN];
+	IMG_UINT32                     ui32RefCount;
+
+	/* Process memory stats */
+	IMG_INT64                      i64StatValue[PVRSRV_PROCESS_STAT_TYPE_COUNT];
+	IMG_UINT32                     ui32StatAllocFlags;
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	struct _CACHEOP_STRUCT_ {
+		PVRSRV_CACHE_OP        uiCacheOp;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+		IMG_DEV_VIRTADDR       sDevVAddr;
+		IMG_DEV_PHYADDR        sDevPAddr;
+#endif
+		IMG_DEVMEM_SIZE_T      uiOffset;
+		IMG_DEVMEM_SIZE_T      uiSize;
+		IMG_UINT64             ui64ExecuteTime;
+		IMG_BOOL               bUserModeFlush;
+		IMG_BOOL               bIsFence;
+		IMG_PID                ownerPid;
+	}                              asCacheOp[MAX_CACHEOP_STAT];
+	IMG_INT32                      uiCacheOpWriteIndex;
+#endif
+
+	/* Other statistics structures */
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	HASH_TABLE* psMemoryRecords;
+#endif
+	/* Device stats */
+	IMG_UINT32                     ui32DevCount;
+	IMG_INT32                      ai32DevStats[][PVRSRV_DEVICE_STAT_TYPE_COUNT];
+} PVRSRV_PROCESS_STATS;
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+static PVRDKF_DKP_HANDLE ghDKPHandle;
+#endif
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+
+typedef struct _PVRSRV_OS_STAT_ENTRY_
+{
+	DI_GROUP *psStatsDIGroup;
+	DI_ENTRY *psProcessStatsDIEntry;
+	DI_ENTRY *psMemStatsDIEntry;
+	DI_ENTRY *psRIMemStatsDIEntry;
+	DI_ENTRY *psCacheOpStatsDIEntry;
+} PVRSRV_OS_STAT_ENTRY;
+
+static PVRSRV_OS_STAT_ENTRY gsLiveStatEntries;
+static PVRSRV_OS_STAT_ENTRY gsRetiredStatEntries;
+
+int GenericStatsPrintElementsLive(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+int GenericStatsPrintElementsRetired(OSDI_IMPL_ENTRY *psEntry, void *pvData);
+
+/*
+ * Functions for printing the information stored...
+ */
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+void ProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                               PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+void MemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                           PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+void RIMemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                             PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void CacheOpStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                               PVRSRV_PROCESS_STATS *psProcessStats);
+#endif
+
+typedef void (PVRSRV_STATS_PRINT_ELEMENTS)(OSDI_IMPL_ENTRY *psEntry,
+                                           PVRSRV_PROCESS_STATS *psProcessStats);
+
+typedef enum
+{
+	PVRSRV_STAT_TYPE_PROCESS,
+	PVRSRV_STAT_TYPE_MEMORY,
+	PVRSRV_STAT_TYPE_RIMEMORY,
+	PVRSRV_STAT_TYPE_CACHEOP,
+	PVRSRV_STAT_TYPE_LAST
+} PVRSRV_STAT_TYPE;
+
+#define SEPARATOR_STR_LEN 166
+
+typedef struct _PVRSRV_STAT_PV_DATA_ {
+
+	PVRSRV_STAT_TYPE eStatType;
+	PVRSRV_STATS_PRINT_ELEMENTS* pfnStatsPrintElements;
+	IMG_CHAR szLiveStatsHeaderStr[SEPARATOR_STR_LEN + 1];
+	IMG_CHAR szRetiredStatsHeaderStr[SEPARATOR_STR_LEN + 1];
+
+} PVRSRV_STAT_PV_DATA;
+
+static PVRSRV_STAT_PV_DATA g_StatPvDataArr[] = {
+						{ PVRSRV_STAT_TYPE_PROCESS,  NULL, " Process"               , " Process"               },
+						{ PVRSRV_STAT_TYPE_MEMORY,   NULL, " Memory Allocation"     , " Memory Allocation"     },
+						{ PVRSRV_STAT_TYPE_RIMEMORY, NULL, " Resource Allocation"   , " Resource Allocation"   },
+						{ PVRSRV_STAT_TYPE_CACHEOP,  NULL, " Cache Maintenance Ops" , " Cache Maintenance Ops" }
+					      };
+
+#define GET_STAT_ENTRY_ID(STAT_TYPE) &g_StatPvDataArr[(STAT_TYPE)]
+
+/* Generic header strings */
+static const IMG_CHAR g_szLiveHeaderStr[]    = " Statistics for LIVE Processes ";
+static const IMG_CHAR g_szRetiredHeaderStr[] = " Statistics for RETIRED Processes ";
+
+/* Separator string used for separating stats for different PIDs */
+static IMG_CHAR g_szSeparatorStr[SEPARATOR_STR_LEN + 1] = "";
+
+static inline void
+_prepareStatsHeaderString(IMG_CHAR *pszStatsSpecificStr, const IMG_CHAR* pszGenericHeaderStr)
+{
+	IMG_UINT32 ui32NumSeparators;
+	IMG_CHAR szStatsHeaderFooterStr[75];
+
+	/* Prepare text content of the header in a local string */
+	OSStringSafeCopy(szStatsHeaderFooterStr, pszStatsSpecificStr, ARRAY_SIZE(szStatsHeaderFooterStr));
+	OSStringLCat(szStatsHeaderFooterStr, pszGenericHeaderStr, ARRAY_SIZE(szStatsHeaderFooterStr));
+
+	/* Write all '-' characters to the header string */
+	memset(pszStatsSpecificStr, '-', SEPARATOR_STR_LEN);
+	pszStatsSpecificStr[SEPARATOR_STR_LEN] = '\0';
+
+	/* Find the spot for text content in the header string */
+	ui32NumSeparators = (SEPARATOR_STR_LEN - OSStringLength(szStatsHeaderFooterStr)) >> 1;
+
+	/* Finally write the text content */
+	OSSNPrintf(pszStatsSpecificStr + ui32NumSeparators,
+		   OSStringLength(szStatsHeaderFooterStr),
+		   "%s", szStatsHeaderFooterStr);
+
+	/* Overwrite the '\0' character added by OSSNPrintf() */
+	if (OSStringLength(szStatsHeaderFooterStr) > 0)
+	{
+		pszStatsSpecificStr[ui32NumSeparators + OSStringLength(szStatsHeaderFooterStr) - 1] = ' ';
+	}
+}
+
+static inline void
+_prepareSeparatorStrings(void)
+{
+	IMG_UINT32 i;
+
+	/* Prepare header strings for each stat type */
+	for (i = 0; i < PVRSRV_STAT_TYPE_LAST; ++i)
+	{
+		_prepareStatsHeaderString(g_StatPvDataArr[i].szLiveStatsHeaderStr, g_szLiveHeaderStr);
+		_prepareStatsHeaderString(g_StatPvDataArr[i].szRetiredStatsHeaderStr, g_szRetiredHeaderStr);
+	}
+
+	/* Prepare separator string to separate stats for different PIDs */
+	memset(g_szSeparatorStr, '-', SEPARATOR_STR_LEN);
+	g_szSeparatorStr[SEPARATOR_STR_LEN] = '\0';
+}
+
+static inline void
+_prepareStatsPrivateData(void)
+{
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_PROCESS].pfnStatsPrintElements = ProcessStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_MEMORY].pfnStatsPrintElements = MemStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_RIMEMORY].pfnStatsPrintElements = RIMemStatsPrintElements;
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	g_StatPvDataArr[PVRSRV_STAT_TYPE_CACHEOP].pfnStatsPrintElements = CacheOpStatsPrintElements;
+#endif
+
+	_prepareSeparatorStrings();
+}
+
+#endif
+
+/*
+ * Global Boolean to flag when the statistics are ready to monitor
+ * memory allocations.
+ */
+static IMG_BOOL bProcessStatsInitialised = IMG_FALSE;
+
+/*
+ * Linked lists for process stats. Live stats are for processes which are still running
+ * and the dead list holds those that have exited.
+ */
+static DLLIST_NODE gsLiveList;
+static DLLIST_NODE gsDeadList;
+
+static POS_LOCK g_psLinkedListLock;
+/* Lockdep feature in the kernel cannot differentiate between different instances of same lock type.
+ * This allows it to group all such instances of the same lock type under one class
+ * The consequence of this is that, if lock acquisition is nested on different instances, it generates
+ * a false warning message about the possible occurrence of deadlock due to recursive lock acquisition.
+ * Hence we create the following sub classes to explicitly appraise Lockdep of such safe lock nesting */
+#define PROCESS_LOCK_SUBCLASS_CURRENT	1
+#if defined(ENABLE_DEBUGFS_PIDS)
+/*
+ * Pointer to OS folder to hold PID folders.
+ */
+static DI_GROUP *psProcStatsDIGroup;
+#endif
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+static DI_ENTRY *psProcStatsDIEntry;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+/* Global driver PID stats registration handle */
+static IMG_HANDLE g_hDriverProcessStats;
+#endif
+
+/* Global driver-data folders */
+typedef struct _GLOBAL_STATS_
+{
+	IMG_UINT64 ui64StatValue[PVRSRV_DRIVER_STAT_TYPE_COUNT];
+	POS_LOCK   hGlobalStatsLock;
+} GLOBAL_STATS;
+
+static DI_ENTRY *psGlobalMemDIEntry;
+static GLOBAL_STATS gsGlobalStats;
+
+#define HASH_INITIAL_SIZE 5
+/* A hash table used to store the size of any vmalloc'd allocation
+ * against its address (not needed for kmallocs as we can use ksize()) */
+static HASH_TABLE* gpsSizeTrackingHashTable;
+static POS_LOCK	 gpsSizeTrackingHashTableLock;
+
+static PVRSRV_ERROR _RegisterProcess(IMG_HANDLE *phProcessStats, IMG_PID ownerPid);
+
+static void _DestroyProcessStat(PVRSRV_PROCESS_STATS* psProcessStats);
+
+static void _DecreaseProcStatValueUnlocked(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                                           PVRSRV_PROCESS_STATS* psProcessStats,
+                                           IMG_UINT64 uiBytes);
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStatsInLiveList
+@Description    Searches the Live Process List for a statistics structure that
+                matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStatsInLiveList(IMG_PID pid)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats;
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+
+		if (psProcessStats->pid == pid)
+		{
+			return psProcessStats;
+		}
+	}
+	return NULL;
+} /* _FindProcessStatsInLiveList */
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStatsInDeadList
+@Description    Searches the Dead Process List for a statistics structure that
+                matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStatsInDeadList(IMG_PID pid)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats;
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+
+		if (psProcessStats->pid == pid)
+		{
+			return psProcessStats;
+		}
+	}
+	return NULL;
+} /* _FindProcessStatsInDeadList */
+
+/*************************************************************************/ /*!
+@Function       _FindProcessStats
+@Description    Searches the Live and Dead Process Lists for a statistics
+                structure that matches the PID given.
+@Input          pid  Process to search for.
+@Return         Pointer to stats structure for the process.
+*/ /**************************************************************************/
+static PVRSRV_PROCESS_STATS*
+_FindProcessStats(IMG_PID pid)
+{
+	PVRSRV_PROCESS_STATS* psProcessStats = _FindProcessStatsInLiveList(pid);
+
+	if (psProcessStats == NULL)
+	{
+		psProcessStats = _FindProcessStatsInDeadList(pid);
+	}
+
+	return psProcessStats;
+} /* _FindProcessStats */
+
+/*************************************************************************/ /*!
+@Function       _CompressMemoryUsage
+@Description    Reduces memory usage by deleting old statistics data.
+                This function requires that the list lock is not held!
+*/ /**************************************************************************/
+static void
+_CompressMemoryUsage(void)
+{
+	IMG_INT32 i32ItemsRemaining;
+	DLLIST_NODE *psNode, *psNext;
+	DLLIST_NODE sToBeFreedHead;
+
+	/*
+	 * We hold the lock whilst checking the list, but we'll release it
+	 * before freeing memory (as that will require the lock too)!
+	 */
+	OSLockAcquire(g_psLinkedListLock);
+
+	/* Check that the dead list is not bigger than the max size... */
+	i32ItemsRemaining = MAX_DEAD_LIST_PROCESSES;
+
+	dllist_init(&sToBeFreedHead);
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		i32ItemsRemaining--;
+		if (i32ItemsRemaining < 0)
+		{
+			/* This is the last allowed process, cut the linked list here! */
+			dllist_remove_node(psNode);
+			dllist_add_to_tail(&sToBeFreedHead, psNode);
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	dllist_foreach_node(&sToBeFreedHead, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS *psProcessStatsToBeFreed;
+		psProcessStatsToBeFreed = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+		RIDeleteEntriesForPID(psProcessStatsToBeFreed->pid);
+#endif
+		_DestroyProcessStat(psProcessStatsToBeFreed);
+	}
+} /* _CompressMemoryUsage */
+
+/* These functions move the process stats from the live to the dead list.
+ * _MoveProcessToDeadList moves the entry in the global lists and
+ * it needs to be protected by g_psLinkedListLock.
+ * _MoveProcessToDeadList performs the OS calls and it
+ * shouldn't be used under g_psLinkedListLock because this could generate a
+ * lockdep warning. */
+static void
+_MoveProcessToDeadList(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	/* Take the element out of the live list and append to the dead list... */
+	PVR_ASSERT(psProcessStats != NULL);
+	dllist_remove_node(&psProcessStats->sNode);
+	dllist_add_to_head(&gsDeadList, &psProcessStats->sNode);
+} /* _MoveProcessToDeadList */
+
+/* These functions move the process stats from the dead to the live list.
+ * _MoveProcessToLiveList moves the entry in the global lists and
+ * it needs to be protected by g_psLinkedListLock.
+ * _MoveProcessToLiveList performs the OS calls and it
+ * shouldn't be used under g_psLinkedListLock because this could generate a
+ * lockdep warning. */
+static void
+_MoveProcessToLiveList(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	/* Take the element out of the live list and append to the dead list... */
+	PVR_ASSERT(psProcessStats != NULL);
+	dllist_remove_node(&psProcessStats->sNode);
+	dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+} /* _MoveProcessToLiveList */
+
+static PVRSRV_ERROR
+_AllocateProcessStats(PVRSRV_PROCESS_STATS **ppsProcessStats, IMG_PID ownerPid)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_PROCESS_STATS *psProcessStats;
+	PVRSRV_DATA	*psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32DevCount = 0;
+
+	if (psPVRSRVData != NULL)
+	{
+		ui32DevCount = psPVRSRVData->ui32RegisteredDevices;
+	}
+
+	psProcessStats = OSAllocZMemNoStats(sizeof(PVRSRV_PROCESS_STATS) +
+	                                    ui32DevCount * PVRSRV_DEVICE_STAT_TYPE_COUNT * sizeof(IMG_INT32));
+	PVR_RETURN_IF_NOMEM(psProcessStats);
+
+	psProcessStats->pid             = ownerPid;
+	psProcessStats->ui32RefCount    = 1;
+	psProcessStats->ui32DevCount    = ui32DevCount;
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	psProcessStats->psMemoryRecords = HASH_Create(HASH_INITIAL_SIZE);
+	PVR_GOTO_IF_NOMEM(psProcessStats->psMemoryRecords, eError, free_process_stats);
+#endif
+
+	eError = OSLockCreateNoStats(&psProcessStats->hLock);
+	PVR_GOTO_IF_ERROR(eError, destroy_mem_recs);
+
+	*ppsProcessStats = psProcessStats;
+	return PVRSRV_OK;
+
+destroy_mem_recs:
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	HASH_Delete(psProcessStats->psMemoryRecords);
+free_process_stats:
+#endif
+	OSFreeMemNoStats(psProcessStats);
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+}
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static PVRSRV_ERROR _FreeMemStatsEntry(uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	PVRSRV_MEM_ALLOC_REC *psRecord = (PVRSRV_MEM_ALLOC_REC *)(uintptr_t)v;
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	PVR_DPF((PVR_DBG_WARNING, "Mem Stats Record not freed: 0x%" IMG_UINT64_FMTSPECx " %p, size="IMG_SIZE_FMTSPEC", %s:%d",
+			 (IMG_UINT64)(k), psRecord, psRecord->uiBytes,
+			 (IMG_CHAR*)psRecord->pvAllocdFromFile, psRecord->ui32AllocdFromLine));
+#else
+	PVR_UNREFERENCED_PARAMETER(k);
+#endif
+	OSFreeMemNoStats(psRecord);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _DestroyProcessStat
+@Description    Frees memory and resources held by a process statistic.
+@Input          psProcessStats  Process stats to destroy.
+*/ /**************************************************************************/
+static void
+_DestroyProcessStat(PVRSRV_PROCESS_STATS* psProcessStats)
+{
+	PVR_ASSERT(psProcessStats != NULL);
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+	/* Free the memory statistics... */
+	HASH_Iterate(psProcessStats->psMemoryRecords, (HASH_pfnCallback)_FreeMemStatsEntry, NULL);
+	HASH_Delete(psProcessStats->psMemoryRecords);
+
+	OSLockRelease(psProcessStats->hLock);
+#endif
+
+	/*Destroy the lock */
+	OSLockDestroyNoStats(psProcessStats->hLock);
+
+	/* Free the memory... */
+	OSFreeMemNoStats(psProcessStats);
+} /* _DestroyProcessStat */
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+static inline void
+_createStatsFiles(PVRSRV_OS_STAT_ENTRY* psStatsEntries,
+                  DI_PFN_SHOW pfnStatsShow)
+{
+	PVRSRV_ERROR eError;
+	DI_ITERATOR_CB sIterator = {.pfnShow = pfnStatsShow};
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	eError = DICreateEntry("process_stats", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_PROCESS),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psProcessStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (1)");
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	eError = DICreateEntry("cache_ops_exec", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_CACHEOP),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psCacheOpStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (2)");
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	eError = DICreateEntry("mem_area", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_MEMORY),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psMemStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (3)");
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = DICreateEntry("gpu_mem_area", psStatsEntries->psStatsDIGroup,
+	                       &sIterator,
+	                       GET_STAT_ENTRY_ID(PVRSRV_STAT_TYPE_RIMEMORY),
+	                       DI_ENTRY_TYPE_GENERIC,
+	                       &psStatsEntries->psRIMemStatsDIEntry);
+	PVR_LOG_IF_ERROR(eError, "DICreateEntry (4)");
+#endif
+}
+
+static inline void
+_createStatisticsEntries(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DICreateGroup("proc_stats", NULL, &psProcStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (1)");
+	eError = DICreateGroup("live_pids_stats", psProcStatsDIGroup,
+                           &gsLiveStatEntries.psStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (2)");
+	eError = DICreateGroup("retired_pids_stats", psProcStatsDIGroup,
+                           &gsRetiredStatEntries.psStatsDIGroup);
+	PVR_LOG_IF_ERROR(eError, "DICreateGroup (3)");
+
+	_createStatsFiles(&gsLiveStatEntries, GenericStatsPrintElementsLive);
+	_createStatsFiles(&gsRetiredStatEntries, GenericStatsPrintElementsRetired);
+
+	_prepareStatsPrivateData();
+}
+
+static inline void
+_removeStatsFiles(PVRSRV_OS_STAT_ENTRY* psStatsEntries)
+{
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+	DIDestroyEntry(psStatsEntries->psProcessStatsDIEntry);
+	psStatsEntries->psProcessStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+	DIDestroyEntry(psStatsEntries->psCacheOpStatsDIEntry);
+    psStatsEntries->psCacheOpStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	DIDestroyEntry(psStatsEntries->psMemStatsDIEntry);
+	psStatsEntries->psMemStatsDIEntry = NULL;
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	DIDestroyEntry(psStatsEntries->psRIMemStatsDIEntry);
+	psStatsEntries->psRIMemStatsDIEntry = NULL;
+#endif
+}
+
+static inline void
+_removeStatisticsEntries(void)
+{
+	_removeStatsFiles(&gsLiveStatEntries);
+	_removeStatsFiles(&gsRetiredStatEntries);
+
+	DIDestroyGroup(gsLiveStatEntries.psStatsDIGroup);
+	gsLiveStatEntries.psStatsDIGroup = NULL;
+	DIDestroyGroup(gsRetiredStatEntries.psStatsDIGroup);
+	gsRetiredStatEntries.psStatsDIGroup = NULL;
+	DIDestroyGroup(psProcStatsDIGroup);
+	psProcStatsDIGroup = NULL;
+}
+#endif
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+
+#define GENERATE_DKP_PRINT(_handle, _key, _value)                   \
+	PVRDKPOutput((_handle),                                         \
+	             "%s:%-*s%"IMG_UINT64_FMTSPEC"\n",                  \
+	             (_key),                                            \
+	             (int)(50 - sizeof((_key))),                        \
+	              "",                                               \
+	             (IMG_UINT64)(_value))
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#define GENERATE_PROCESS_STAT_FDINFO(_dkp_handle, _stat_type, _key, _process_stats) \
+	do { \
+		IMG_UINT32 ui32Value = MAX(0, (_process_stats)->i64StatValue[(_stat_type)]); \
+		if (((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) || \
+			((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES)) \
+		{ \
+			ui32Value = RITotalAllocProcessUnlocked((_process_stats)->pid, \
+			                                        ((_stat_type) == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) ? PHYS_HEAP_TYPE_LMA : PHYS_HEAP_TYPE_UMA); \
+		} \
+		GENERATE_DKP_PRINT((_dkp_handle), (_key), ui32Value); \
+	} while (0)
+#else
+#define GENERATE_PROCESS_STAT_FDINFO(_dkp_handle, _stat_type, _key, _process_stats) \
+	GENERATE_DKP_PRINT((_dkp_handle), (_key), MAX(0, (_process_stats)->i64StatValue[(_stat_type)]));
+#endif
+
+static void _ProcessStatsDKPShow(PVRSRV_DEVICE_NODE *psDevNode,
+                                 int pid,
+                                 IMG_HANDLE hPrivData)
+{
+	PVRSRV_PROCESS_STATS *psProcessStats = _FindProcessStatsInLiveList((IMG_PID)pid);
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	if (psProcessStats == NULL)
+	{
+		/* Just return if the specified process is no longer in the process
+		 * stats list.
+		 */
+		return;
+	}
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-memory-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_MEMORY
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-shared-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_SHARED
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-total-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_TOTAL
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-resident-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_RESIDENT
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-purgeable-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_PURGEABLE
+	#undef X
+
+	#define X(_stat_type, _stat_name, _dkp_name) \
+	GENERATE_PROCESS_STAT_FDINFO(ghDKPHandle, _stat_type, "drm-active-"_dkp_name, psProcessStats);
+	PVRSRV_DKP_MEM_STAT_GROUP_ACTIVE
+	#undef X
+}
+
+#undef GENERATE_PROCESS_STAT_FDINFO
+#undef GENERATE_DKP_PRINT
+
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsInitialise
+@Description    Entry point for initialising the statistics module.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVStatsInitialise(void)
+{
+	PVRSRV_ERROR error;
+
+	PVR_ASSERT(g_psLinkedListLock == NULL);
+	PVR_ASSERT(gpsSizeTrackingHashTable == NULL);
+	PVR_ASSERT(bProcessStatsInitialised == IMG_FALSE);
+
+	/* We need a lock to protect the linked lists... */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&g_psLinkedListLock);
+#else
+	error = OSLockCreate(&g_psLinkedListLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, return_);
+
+	/* We also need a lock to protect the hash table used for size tracking. */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&gpsSizeTrackingHashTableLock);
+#else
+	error = OSLockCreate(&gpsSizeTrackingHashTableLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, destroy_linked_list_lock_);
+
+	/* We also need a lock to protect the GlobalStat counters */
+#if defined(__linux__) && defined(__KERNEL__)
+	error = OSLockCreateNoStats(&gsGlobalStats.hGlobalStatsLock);
+#else
+	error = OSLockCreate(&gsGlobalStats.hGlobalStatsLock);
+#endif
+	PVR_GOTO_IF_ERROR(error, destroy_hashtable_lock_);
+
+	/* Flag that we are ready to start monitoring memory allocations. */
+
+	gpsSizeTrackingHashTable = HASH_Create(HASH_INITIAL_SIZE);
+	PVR_GOTO_IF_NOMEM(gpsSizeTrackingHashTable, error, destroy_stats_lock_);
+
+	dllist_init(&gsLiveList);
+	dllist_init(&gsDeadList);
+
+	bProcessStatsInitialised = IMG_TRUE;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Register our 'system' PID to hold driver-wide alloc stats */
+	_RegisterProcess(&g_hDriverProcessStats, PVR_SYS_ALLOC_PID);
+#endif
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+	_createStatisticsEntries();
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = RawProcessStatsPrintElements};
+		error = DICreateEntry("memtrack_stats", NULL, &sIterator, NULL,
+		                       DI_ENTRY_TYPE_GENERIC, &psProcStatsDIEntry);
+		PVR_LOG_IF_ERROR(error, "DICreateEntry (1)");
+	}
+#endif
+
+	{
+		DI_ITERATOR_CB sIterator = {.pfnShow = GlobalStatsPrintElements};
+		error = DICreateEntry("driver_stats", NULL, &sIterator, NULL,
+		                      DI_ENTRY_TYPE_GENERIC, &psGlobalMemDIEntry);
+		PVR_LOG_IF_ERROR(error, "DICreateEntry (3)");
+	}
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+	/* We might error here, but drm usage keys related to memory are optional.
+	 * We'll notify that the DKP registration has failed but continue as if nothing
+	 * happened. */
+	{
+		error = PVRSRVRegisterDKP(NULL,
+		                          "proc-stats",
+		                          &_ProcessStatsDKPShow,
+		                          DKP_CONNECTION_FLAG_SERVICES,
+		                          &ghDKPHandle);
+		if (error != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Failed to register process stats DKP", __func__));
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+
+destroy_stats_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(gsGlobalStats.hGlobalStatsLock);
+#else
+	OSLockDestroy(gsGlobalStats.hGlobalStatsLock);
+#endif
+	gsGlobalStats.hGlobalStatsLock = NULL;
+destroy_hashtable_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(gpsSizeTrackingHashTableLock);
+#else
+	OSLockDestroy(gpsSizeTrackingHashTableLock);
+#endif
+	gpsSizeTrackingHashTableLock = NULL;
+destroy_linked_list_lock_:
+#if defined(__linux__) && defined(__KERNEL__)
+	OSLockDestroyNoStats(g_psLinkedListLock);
+#else
+	OSLockDestroy(g_psLinkedListLock);
+#endif
+	g_psLinkedListLock = NULL;
+return_:
+	return error;
+
+}
+
+static PVRSRV_ERROR _DumpAllVMallocEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+#if defined(PVRSRV_NEED_PVR_DPF) || defined(DOXYGEN)
+	_PVR_STATS_TRACKING_HASH_ENTRY *psNewTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)(uintptr_t)v;
+	IMG_UINT64 uiCpuVAddr = (IMG_UINT64)k;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: " IMG_SIZE_FMTSPEC " bytes @ 0x%" IMG_UINT64_FMTSPECx " (PID %u)", __func__,
+	         psNewTrackingHashEntry->uiSizeInBytes,
+	         uiCpuVAddr,
+	         psNewTrackingHashEntry->uiPid));
+
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsDestroy
+@Description    Method for destroying the statistics module data.
+*/ /**************************************************************************/
+void
+PVRSRVStatsDestroy(void)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_ASSERT(bProcessStatsInitialised);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	/* Deregister our 'system' PID which holds driver-wide alloc stats */
+	PVRSRVStatsDeregisterProcess(g_hDriverProcessStats);
+#endif
+
+#if defined(PROC_STATS_FDINFO_ENABLED)
+	if (ghDKPHandle != NULL)
+	{
+		PVRSRV_ERROR eError = PVRSRVUnRegisterDKP(NULL, ghDKPHandle);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to unregister process stats DKP", __func__));
+		}
+	}
+#endif
+
+	/* Stop monitoring memory allocations... */
+	bProcessStatsInitialised = IMG_FALSE;
+
+	/* Destroy the locks... */
+	if (g_psLinkedListLock != NULL)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(g_psLinkedListLock);
+#else
+		OSLockDestroy(g_psLinkedListLock);
+#endif
+		g_psLinkedListLock = NULL;
+	}
+
+	/* Free the live and dead lists... */
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		dllist_remove_node(&psProcessStats->sNode);
+		_DestroyProcessStat(psProcessStats);
+	}
+
+	dllist_foreach_node(&gsDeadList, psNode, psNext)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		dllist_remove_node(&psProcessStats->sNode);
+		_DestroyProcessStat(psProcessStats);
+	}
+
+	if (gpsSizeTrackingHashTable != NULL)
+	{
+		/* Dump all remaining entries in HASH table (list any remaining vmallocs) */
+		HASH_Iterate(gpsSizeTrackingHashTable, (HASH_pfnCallback)_DumpAllVMallocEntries, NULL);
+		HASH_Delete(gpsSizeTrackingHashTable);
+	}
+	if (gpsSizeTrackingHashTableLock != NULL)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(gpsSizeTrackingHashTableLock);
+#else
+		OSLockDestroy(gpsSizeTrackingHashTableLock);
+#endif
+		gpsSizeTrackingHashTableLock = NULL;
+	}
+
+	if (NULL != gsGlobalStats.hGlobalStatsLock)
+	{
+#if defined(__linux__) && defined(__KERNEL__)
+		OSLockDestroyNoStats(gsGlobalStats.hGlobalStatsLock);
+#else
+		OSLockDestroy(gsGlobalStats.hGlobalStatsLock);
+#endif
+		gsGlobalStats.hGlobalStatsLock = NULL;
+	}
+
+}
+
+void
+PVRSRVStatsDestroyDI(void)
+{
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+	if (psProcStatsDIEntry != NULL)
+	{
+		DIDestroyEntry(psProcStatsDIEntry);
+		psProcStatsDIEntry = NULL;
+	}
+#endif
+
+	/* Destroy the global data entry */
+	if (psGlobalMemDIEntry != NULL)
+	{
+		DIDestroyEntry(psGlobalMemDIEntry);
+		psGlobalMemDIEntry = NULL;
+	}
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+	_removeStatisticsEntries();
+#endif
+}
+
+static void _decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize;
+#endif
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	ui64InitialSize = GET_GPUMEM_GLOBAL_STAT_VALUE();
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_KMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+			DECREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			break;
+#endif
+
+		default:
+			PVR_ASSERT(0);
+			break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_GLOBAL_STAT_VALUE();
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemGlobal(0, ui64Size);
+		}
+	}
+#endif
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+}
+
+static void _increase_global_stat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize;
+#endif
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	ui64InitialSize = GET_GPUMEM_GLOBAL_STAT_VALUE();
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_KMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMALLOC, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_LMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ZOMBIE_GPUMEM_UMA, uiBytes);
+			break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_MAPPED_GPUMEM_UMA_LMA, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA_POOL, uiBytes);
+			break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+			INCREASE_GLOBAL_STAT_VALUE(gsGlobalStats, PVRSRV_DRIVER_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			break;
+#endif
+
+		default:
+			PVR_ASSERT(0);
+			break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_GLOBAL_STAT_VALUE();
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemGlobal(0, ui64Size);
+		}
+	}
+#endif
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+}
+
+static PVRSRV_ERROR
+_RegisterProcess(IMG_HANDLE *phProcessStats, IMG_PID ownerPid)
+{
+	PVRSRV_PROCESS_STATS*	psProcessStats=NULL;
+	PVRSRV_ERROR			eError;
+
+	PVR_ASSERT(phProcessStats != NULL);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Register process PID %d [%s]",
+			__func__, ownerPid, (ownerPid == PVR_SYS_ALLOC_PID)
+			? "system" : OSGetCurrentClientProcessNameKM()));
+
+	/* Check the PID has not already moved to the dead list... */
+	OSLockAcquire(g_psLinkedListLock);
+	psProcessStats = _FindProcessStatsInDeadList(ownerPid);
+	if (psProcessStats != NULL)
+	{
+		/* Move it back onto the live list! */
+		_MoveProcessToLiveList(psProcessStats);
+	}
+	else
+	{
+		/* Check the PID is not already registered in the live list... */
+		psProcessStats = _FindProcessStatsInLiveList(ownerPid);
+	}
+
+	/* If the PID is on the live list then just increment the ref count and return... */
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		psProcessStats->ui32RefCount++;
+
+		OSLockRelease(psProcessStats->hLock);
+		OSLockRelease(g_psLinkedListLock);
+
+		*phProcessStats = psProcessStats;
+
+		return PVRSRV_OK;
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	/* Allocate a new node structure and initialise it... */
+	eError = _AllocateProcessStats(&psProcessStats, ownerPid);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	OSStringSafeCopy(psProcessStats->processName, OSGetCurrentProcessName(), MAX_PROC_NAME_LEN);
+
+	/* Add it to the live list... */
+	OSLockAcquire(g_psLinkedListLock);
+	dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+	OSLockRelease(g_psLinkedListLock);
+
+	/* Done */
+	*phProcessStats = (IMG_HANDLE) psProcessStats;
+
+	return PVRSRV_OK;
+
+e0:
+	*phProcessStats = (IMG_HANDLE) NULL;
+	return PVRSRV_ERROR_OUT_OF_MEMORY;
+} /* _RegisterProcess */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsRegisterProcess
+@Description    Register a process into the list statistics list.
+@Output         phProcessStats  Handle to the process to be used to deregister.
+@Return         Standard PVRSRV_ERROR error code.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVStatsRegisterProcess(IMG_HANDLE* phProcessStats)
+{
+	return _RegisterProcess(phProcessStats, OSGetCurrentClientProcessIDKM());
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVStatsDeregisterProcess
+@Input          hProcessStats  Handle to the process returned when registered.
+@Description    Method for destroying the statistics module data.
+*/ /**************************************************************************/
+void
+PVRSRVStatsDeregisterProcess(IMG_HANDLE hProcessStats)
+{
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Deregister process entered PID %d [%s]",
+			__func__, OSGetCurrentClientProcessIDKM(),
+			OSGetCurrentProcessName()));
+
+	if (hProcessStats != (IMG_HANDLE) NULL)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStats = (PVRSRV_PROCESS_STATS*) hProcessStats;
+
+		/* Lower the reference count, if zero then move it to the dead list */
+		OSLockAcquire(g_psLinkedListLock);
+		if (psProcessStats->ui32RefCount > 0)
+		{
+			OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+			psProcessStats->ui32RefCount--;
+
+#if !defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+			if (psProcessStats->ui32RefCount == 0)
+			{
+				OSLockRelease(psProcessStats->hLock);
+				_MoveProcessToDeadList(psProcessStats);
+			}else
+#endif
+			{
+				OSLockRelease(psProcessStats->hLock);
+			}
+		}
+		OSLockRelease(g_psLinkedListLock);
+
+		/* Check if the dead list needs to be reduced */
+		_CompressMemoryUsage();
+	}
+} /* PVRSRVStatsDeregisterProcess */
+
+PVRSRV_ERROR PVRSRVStatsDeviceConnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+	IMG_PID ownerPid = OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS*	psProcessStats;
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStatsInLiveList(ownerPid);
+
+	if (psProcessStats != NULL)
+	{
+		if ((ui32DevID < psProcessStats->ui32DevCount) ||
+		    (ui32DevID == 0 && psProcessStats->ui32DevCount == 0))
+		{
+			psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]++;
+			UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_MAX_CONNECTIONS],
+			                 psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device index %d is greater than device count %d for PID %d.",
+			         __func__, ui32DevID, psProcessStats->ui32DevCount, ownerPid));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Process %d not found.",
+				 __func__, ownerPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVStatsDeviceDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+	IMG_PID	currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA* psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_PID currentPid = OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS*	psProcessStats;
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+	}
+
+	if (psProcessStats != NULL)
+	{
+		if ((ui32DevID < psProcessStats->ui32DevCount) ||
+		    (ui32DevID == 0 && psProcessStats->ui32DevCount == 0))
+		{
+			psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_CONNECTIONS]--;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Device index %d is greater than device count %d for PID %d.",
+					 __func__, ui32DevID, psProcessStats->ui32DevCount, currentPid));
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Process %d not found.",
+				 __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+/* Assumes PVRSRV_PROCESS_STATS lock (psProcessStats->hLock) has already been taken */
+static void
+_IncreaseProcStatValueUnlocked(PVRSRV_PROCESS_STATS* psProcessStats,
+                               PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                               size_t uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+#endif
+
+	/* Update the memory watermarks... */
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_KMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMALLOC, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES, uiBytes);
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+		{
+			INCREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			psProcessStats->ui32StatAllocFlags |= (IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+		}
+		break;
+#endif
+
+		default:
+		{
+			PVR_ASSERT(0);
+		}
+		break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemPerProcess(0, psProcessStats->pid,
+			                                 ui64Size);
+		}
+	}
+#endif
+}
+
+/* Assumes PVRSRV_PROCESS_STATS lock (psProcessStats->hLock) has already been taken */
+static void
+_DecreaseProcStatValueUnlocked(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                               PVRSRV_PROCESS_STATS* psProcessStats,
+                               IMG_UINT64 uiBytes)
+{
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	IMG_UINT64 ui64InitialSize = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+#endif
+
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_KMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMALLOC, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMALLOC] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMALLOC-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_VMAP_PT_UMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_IOREMAP_PT_LMA-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES, uiBytes);
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_ZOMBIE_UMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_MAP_UMA_LMA_PAGES-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:
+		{
+			DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE, uiBytes);
+			if (psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE] == 0)
+			{
+				psProcessStats->ui32StatAllocFlags &= ~(IMG_UINT32)(1 << (PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_ZOMBIE-PVRSRV_PROCESS_STAT_TYPE_KMALLOC));
+			}
+		}
+		break;
+#endif
+
+		default:
+		{
+			PVR_ASSERT(0);
+		}
+		break;
+	}
+
+#if defined(ENABLE_GPU_MEM_TRACEPOINT)
+	if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+	{
+		IMG_UINT64 ui64Size = GET_GPUMEM_PERPID_STAT_VALUE(psProcessStats);
+		if (ui64Size != ui64InitialSize)
+		{
+			TracepointUpdateGPUMemPerProcess(0, psProcessStats->pid, ui64Size);
+		}
+	}
+#endif
+}
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static INLINE IMG_BOOL
+_CheckKeyPhysAddr(PVRSRV_MEM_ALLOC_TYPE eAllocType)
+{
+	switch (eAllocType)
+	{
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:
+#endif
+			return IMG_TRUE;
+		default:
+			return IMG_FALSE;
+	}
+}
+#endif
+
+void
+PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+							 void *pvCpuVAddr,
+							 IMG_CPU_PHYADDR sCpuPAddr,
+							 size_t uiBytes,
+							 IMG_PID currentPid
+							 DEBUG_MEMSTATS_PARAMS)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_MEM_ALLOC_REC*  psRecord = NULL;
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+	__maybe_unused PVRSRV_PROC_SEARCH_STATE eProcSearch = PVRSRV_PROC_FOUND;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/*
+	 * To prevent a recursive loop, we make the memory allocations for our
+	 * memstat records via OSAllocMemNoStats(), which does not try to
+	 * create a memstat record entry.
+	 */
+
+	/* Allocate the memory record... */
+	psRecord = OSAllocZMemNoStats(sizeof(PVRSRV_MEM_ALLOC_REC));
+	if (psRecord == NULL)
+	{
+		return;
+	}
+
+	psRecord->eAllocType       = eAllocType;
+	psRecord->pvCpuVAddr       = pvCpuVAddr;
+	psRecord->sCpuPAddr.uiAddr = sCpuPAddr.uiAddr;
+	psRecord->uiBytes          = uiBytes;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+	psRecord->pvAllocdFromFile = pvAllocFromFile;
+	psRecord->ui32AllocdFromLine = ui32AllocFromLine;
+#endif
+
+	_increase_global_stat(eAllocType, uiBytes);
+	/* Lock while we find the correct process... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+			if (!psProcessStats)
+			{
+				psProcessStats = _FindProcessStatsInDeadList(currentPid);
+				eProcSearch = PVRSRV_PROC_RESURRECTED;
+			}
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		if (!psProcessStats)
+		{
+			psProcessStats = _FindProcessStatsInDeadList(currentPid);
+			eProcSearch = PVRSRV_PROC_RESURRECTED;
+		}
+	}
+
+	if (psProcessStats == NULL)
+	{
+		eProcSearch = PVRSRV_PROC_NOTFOUND;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat increment called for 'unknown' process PID(%d)",
+				 __func__, currentPid));
+
+		if (_AllocateProcessStats(&psProcessStats, currentPid) != PVRSRV_OK)
+		{
+			OSLockRelease(g_psLinkedListLock);
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s UNABLE TO CREATE process_stats entry for pid %d [%s] (" IMG_SIZE_FMTSPEC " bytes)",
+			        __func__, currentPid, OSGetCurrentProcessName(), uiBytes));
+			goto free_record;
+		}
+
+		/* Add it to the live list... */
+		dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+
+		OSLockRelease(g_psLinkedListLock);
+
+#else /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+		OSLockRelease(g_psLinkedListLock);
+		goto free_record;
+#endif /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+	}
+	else
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		if (eProcSearch == PVRSRV_PROC_RESURRECTED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat incremented on 'dead' process PID(%d)",
+				 __func__, currentPid));
+			/* Move process from dead list to live list */
+			_MoveProcessToLiveList(psProcessStats);
+		}
+#endif
+		OSLockRelease(g_psLinkedListLock);
+	}
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	{
+		IMG_UINT64 ui64Key;
+
+		if (_CheckKeyPhysAddr(eAllocType))
+		{
+			ui64Key = psRecord->sCpuPAddr.uiAddr;
+		}
+		else
+		{
+			ui64Key = (IMG_UINT64)psRecord->pvCpuVAddr;
+		}
+
+		/* Insert the memory record... */
+		if (!HASH_Insert(psProcessStats->psMemoryRecords, ui64Key, (uintptr_t)psRecord))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s UNABLE TO CREATE mem stats record for pid %d [%s] (" IMG_SIZE_FMTSPEC " bytes)",
+					 __func__, currentPid, OSGetCurrentProcessName(), uiBytes));
+		}
+	}
+#endif
+
+	/* Update the memory watermarks... */
+	_IncreaseProcStatValueUnlocked(psProcessStats,
+	                               eAllocType,
+	                               uiBytes);
+
+	OSLockRelease(psProcessStats->hLock);
+
+	return;
+
+free_record:
+	_decrease_global_stat(eAllocType, uiBytes);
+	if (psRecord != NULL)
+	{
+		OSFreeMemNoStats(psRecord);
+	}
+#else /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+	PVR_UNREFERENCED_PARAMETER(eAllocType);
+	PVR_UNREFERENCED_PARAMETER(pvCpuVAddr);
+	PVR_UNREFERENCED_PARAMETER(sCpuPAddr);
+	PVR_UNREFERENCED_PARAMETER(uiBytes);
+	PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif /* defined(PVRSRV_ENABLE_MEMORY_STATS) */
+} /* PVRSRVStatsAddMemAllocRecord */
+
+void
+PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								IMG_UINT64 ui64Key,
+								IMG_PID currentPid)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+	PVRSRV_MEM_ALLOC_REC*  psRecord		  = NULL;
+	IMG_BOOL			   bFound	      = IMG_FALSE;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/* Lock while we find the correct process and remove this record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+		OSLockRelease(psProcessStats->hLock);
+		bFound = psRecord != NULL;
+	}
+
+	/* If not found, we need to do a full search in case it was allocated to a different PID... */
+	if (!bFound)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStatsAlreadyChecked = psProcessStats;
+		DLLIST_NODE *psNode, *psNext;
+
+		/* Search all live lists first... */
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			if (psProcessStats != psProcessStatsAlreadyChecked)
+			{
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+				psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+				OSLockRelease(psProcessStats->hLock);
+				bFound = psRecord != NULL;
+			}
+
+			if (bFound)
+			{
+				break;
+			}
+		}
+
+		/* If not found, then search all dead lists next... */
+		if (!bFound)
+		{
+			dllist_foreach_node(&gsDeadList, psNode, psNext)
+			{
+				psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+				if (psProcessStats != psProcessStatsAlreadyChecked)
+				{
+					OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+					psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+					OSLockRelease(psProcessStats->hLock);
+					bFound = psRecord != NULL;
+				}
+
+				if (bFound)
+				{
+					break;
+				}
+			}
+		}
+	}
+
+	/* Update the watermark and remove this record... */
+	if (bFound)
+	{
+		_decrease_global_stat(eAllocType, psRecord->uiBytes);
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               psRecord->uiBytes);
+
+		OSLockRelease(psProcessStats->hLock);
+		OSLockRelease(g_psLinkedListLock);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		/* If all stats are now zero, remove the entry for this thread */
+		if (psProcessStats->ui32StatAllocFlags == 0)
+		{
+			OSLockAcquire(g_psLinkedListLock);
+			_MoveProcessToDeadList(psProcessStats);
+			OSLockRelease(g_psLinkedListLock);
+
+			/* Check if the dead list needs to be reduced */
+			_CompressMemoryUsage();
+		}
+#endif
+		/*
+		 * Free the record outside the lock so we don't deadlock and so we
+		 * reduce the time the lock is held.
+		 */
+		OSFreeMemNoStats(psRecord);
+	}
+	else
+	{
+		OSLockRelease(g_psLinkedListLock);
+	}
+
+#else
+	PVR_UNREFERENCED_PARAMETER(eAllocType);
+	PVR_UNREFERENCED_PARAMETER(ui64Key);
+	PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif
+} /* PVRSRVStatsRemoveMemAllocRecord */
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void
+PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eCurrentType,
+                                  PVRSRV_MEM_ALLOC_TYPE eTargetType,
+                                  IMG_UINT64 ui64Key,
+                                  IMG_PID currentPid
+                                  DEBUG_MEMSTATS_PARAMS)
+{
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+	PVRSRV_MEM_ALLOC_REC*  psRecord		  = NULL;
+	IMG_BOOL			   bFound	      = IMG_FALSE;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	if (_CheckKeyPhysAddr(eCurrentType) != _CheckKeyPhysAddr(eTargetType))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Key type used for current type must match target type, "
+		                        "Record transfer failed!"));
+		return;
+	}
+
+	/* Lock while we find the correct process and remove this record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+		OSLockRelease(psProcessStats->hLock);
+		bFound = psRecord != NULL;
+	}
+
+	/* If not found, we need to do a full search in case it was allocated to a different PID... */
+	if (!bFound)
+	{
+		PVRSRV_PROCESS_STATS* psProcessStatsAlreadyChecked = psProcessStats;
+		DLLIST_NODE *psNode, *psNext;
+
+		/* Search all live lists first... */
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			if (psProcessStats != psProcessStatsAlreadyChecked)
+			{
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+				psRecord = (PVRSRV_MEM_ALLOC_REC*)HASH_Remove(psProcessStats->psMemoryRecords, ui64Key);
+				OSLockRelease(psProcessStats->hLock);
+				bFound = psRecord != NULL;
+			}
+
+			if (bFound)
+			{
+				break;
+			}
+		}
+	}
+
+	if (bFound)
+	{
+		/* Update the current type watermark by reducing */
+		_decrease_global_stat(eCurrentType, psRecord->uiBytes);
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		_DecreaseProcStatValueUnlocked(eCurrentType,
+		                               psProcessStats,
+		                               psRecord->uiBytes);
+
+		/* Change the type of the record */
+		psRecord->eAllocType = eTargetType;
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+		/* Adjust where the record transfer originated */
+		psRecord->pvAllocdFromFile = pvAllocFromFile;
+		psRecord->ui32AllocdFromLine = ui32AllocFromLine;
+#endif
+
+		/* Update the target watermark by increasing */
+		_IncreaseProcStatValueUnlocked(psProcessStats,
+		                               eTargetType,
+		                               psRecord->uiBytes);
+		OSLockRelease(psProcessStats->hLock);
+
+		_increase_global_stat(eTargetType, psRecord->uiBytes);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+#else
+PVR_UNREFERENCED_PARAMETER(eCurrentType);
+PVR_UNREFERENCED_PARAMETER(eTargetType);
+PVR_UNREFERENCED_PARAMETER(ui64Key);
+PVR_UNREFERENCED_PARAMETER(currentPid);
+#endif
+}
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+void
+PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									size_t uiBytes,
+									IMG_UINT64 uiCpuVAddr,
+									IMG_PID uiPid)
+{
+	IMG_BOOL bRes = IMG_FALSE;
+	_PVR_STATS_TRACKING_HASH_ENTRY *psNewTrackingHashEntry = NULL;
+
+	if (!bProcessStatsInitialised || (gpsSizeTrackingHashTable == NULL))
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	/* Alloc untracked memory for the new hash table entry */
+	psNewTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)OSAllocMemNoStats(sizeof(*psNewTrackingHashEntry));
+	if (psNewTrackingHashEntry == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "*** %s : @ line %d Failed to alloc memory for psNewTrackingHashEntry!",
+				 __func__, __LINE__));
+		return;
+	}
+
+	/* Fill-in the size of the allocation and PID of the allocating process */
+	psNewTrackingHashEntry->uiSizeInBytes = uiBytes;
+	psNewTrackingHashEntry->uiPid = uiPid;
+	OSLockAcquire(gpsSizeTrackingHashTableLock);
+	/* Insert address of the new struct into the hash table */
+	bRes = HASH_Insert(gpsSizeTrackingHashTable, uiCpuVAddr, (uintptr_t)psNewTrackingHashEntry);
+	OSLockRelease(gpsSizeTrackingHashTableLock);
+	if (bRes)
+	{
+		PVRSRVStatsIncrMemAllocStat(eAllocType, uiBytes, uiPid);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "*** %s : @ line %d HASH_Insert() failed!",
+				 __func__, __LINE__));
+		/* Free the memory allocated for psNewTrackingHashEntry, as we
+		 * failed to insert it into the Hash table.
+		 */
+		OSFreeMemNoStats(psNewTrackingHashEntry);
+	}
+}
+
+void
+PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+                            size_t uiBytes,
+                            IMG_PID currentPid)
+
+{
+	IMG_PID				  currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		  psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS* psProcessStats = NULL;
+	__maybe_unused PVRSRV_PROC_SEARCH_STATE eProcSearch = PVRSRV_PROC_FOUND;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Called when process statistics module is not initialised",
+				 __func__));
+#endif
+		return;
+	}
+
+	_increase_global_stat(eAllocType, uiBytes);
+	OSLockAcquire(g_psLinkedListLock);
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStatsInLiveList(currentPid);
+			if (!psProcessStats)
+			{
+				psProcessStats = _FindProcessStatsInDeadList(currentPid);
+				eProcSearch = PVRSRV_PROC_RESURRECTED;
+			}
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStatsInLiveList(currentPid);
+		if (!psProcessStats)
+		{
+			psProcessStats = _FindProcessStatsInDeadList(currentPid);
+			eProcSearch = PVRSRV_PROC_RESURRECTED;
+		}
+	}
+
+	if (psProcessStats == NULL)
+	{
+		eProcSearch = PVRSRV_PROC_NOTFOUND;
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: Process stat increment called for 'unknown' process PID(%d)",
+				 __func__, currentPid));
+
+		if (bProcessStatsInitialised)
+		{
+			if (_AllocateProcessStats(&psProcessStats, currentPid) != PVRSRV_OK)
+			{
+				OSLockRelease(g_psLinkedListLock);
+				return;
+			}
+			/* Add it to the live list... */
+			dllist_add_to_head(&gsLiveList, &psProcessStats->sNode);
+		}
+#else
+		OSLockRelease(g_psLinkedListLock);
+#endif /* defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS) */
+
+	}
+
+	if (psProcessStats != NULL)
+	{
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		if (eProcSearch == PVRSRV_PROC_RESURRECTED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					 "%s: Process stat incremented on 'dead' process PID(%d)",
+					 __func__, currentPid));
+
+			/* Move process from dead list to live list */
+			_MoveProcessToLiveList(psProcessStats);
+		}
+#endif
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Release the list lock as soon as we acquire the process lock,
+		 * this ensures if the process is in deadlist the entry cannot be
+		 * deleted or modified
+		 */
+		OSLockRelease(g_psLinkedListLock);
+
+		/* Update the memory watermarks... */
+		_IncreaseProcStatValueUnlocked(psProcessStats,
+		                               eAllocType,
+		                               uiBytes);
+
+		OSLockRelease(psProcessStats->hLock);
+	}
+}
+
+#if defined(PVRSRV_ENABLE_MEMTRACK_STATS_FILE)
+int RawProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_PROCESS_STATS *psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	DIPrintf(psEntry,
+	         "%s,%s,%s,%s,%s,%s,%s\n",
+	         "PID",
+	         "MemoryUsageKMalloc",           // PVRSRV_PROCESS_STAT_TYPE_KMALLOC
+	         "MemoryUsageAllocPTMemoryUMA",  // PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA
+	         "MemoryUsageAllocPTMemoryLMA",  // PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA
+	         "MemoryUsageAllocGPUMemLMA",    // PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES
+	         "MemoryUsageAllocGPUMemUMA",    // PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES
+	         "MemoryUsageDmaBufImport");     // PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+		if (psProcessStats->pid != PVR_SYS_ALLOC_PID)
+		{
+			DIPrintf(psEntry,
+			         "%d,%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd","
+			         "%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd","
+			         "%"IMG_INT64_FMTSPECd",%"IMG_INT64_FMTSPECd"\n",
+			         psProcessStats->pid,
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES],
+			         psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_DMA_BUF_IMPORT]);
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+} /* RawProcessStatsPrintElements */
+#endif
+
+void
+PVRSRVStatsDecrMemKAllocStat(size_t uiBytes,
+                             IMG_PID decrPID)
+{
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE_KMALLOC, uiBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(decrPID);
+
+	if (psProcessStats != NULL)
+	{
+		/* Decrement the kmalloc memory stat... */
+		DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_KMALLOC, uiBytes);
+		DECREASE_STAT_VALUE(psProcessStats, PVRSRV_PROCESS_STAT_TYPE_TOTAL, uiBytes);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+static void
+_StatsDecrMemTrackedStat(_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry,
+                        PVRSRV_MEM_ALLOC_TYPE eAllocType)
+{
+	PVRSRV_PROCESS_STATS*  psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(eAllocType, psTrackingHashEntry->uiSizeInBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(psTrackingHashEntry->uiPid);
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Decrement the memory stat... */
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               psTrackingHashEntry->uiSizeInBytes);
+		OSLockRelease(psProcessStats->hLock);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+}
+
+void
+PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									  IMG_UINT64 uiCpuVAddr)
+{
+	_PVR_STATS_TRACKING_HASH_ENTRY *psTrackingHashEntry = NULL;
+
+	if (!bProcessStatsInitialised || (gpsSizeTrackingHashTable == NULL))
+	{
+		return;
+	}
+
+	OSLockAcquire(gpsSizeTrackingHashTableLock);
+	psTrackingHashEntry = (_PVR_STATS_TRACKING_HASH_ENTRY *)HASH_Remove(gpsSizeTrackingHashTable, uiCpuVAddr);
+	OSLockRelease(gpsSizeTrackingHashTableLock);
+	if (psTrackingHashEntry)
+	{
+		_StatsDecrMemTrackedStat(psTrackingHashEntry, eAllocType);
+		OSFreeMemNoStats(psTrackingHashEntry);
+	}
+}
+
+void
+PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+							size_t uiBytes,
+							IMG_PID currentPid)
+{
+	IMG_PID				   currentCleanupPid = PVRSRVGetPurgeConnectionPid();
+	PVRSRV_DATA*		   psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_PROCESS_STATS*  psProcessStats = NULL;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	_decrease_global_stat(eAllocType, uiBytes);
+
+	OSLockAcquire(g_psLinkedListLock);
+	if (psPVRSRVData)
+	{
+		if ((currentPid == psPVRSRVData->cleanupThreadPid) &&
+		    (currentCleanupPid != 0))
+		{
+			psProcessStats = _FindProcessStats(currentCleanupPid);
+		}
+		else
+		{
+			psProcessStats = _FindProcessStats(currentPid);
+		}
+	}
+	else
+	{
+		psProcessStats = _FindProcessStats(currentPid);
+	}
+
+
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		/* Release the list lock as soon as we acquire the process lock,
+		 * this ensures if the process is in deadlist the entry cannot be
+		 * deleted or modified
+		 */
+		OSLockRelease(g_psLinkedListLock);
+		/* Update the memory watermarks... */
+		_DecreaseProcStatValueUnlocked(eAllocType,
+		                               psProcessStats,
+		                               uiBytes);
+		OSLockRelease(psProcessStats->hLock);
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS)
+		/* If all stats are now zero, remove the entry for this thread */
+		if (psProcessStats->ui32StatAllocFlags == 0)
+		{
+			OSLockAcquire(g_psLinkedListLock);
+			_MoveProcessToDeadList(psProcessStats);
+			OSLockRelease(g_psLinkedListLock);
+
+			/* Check if the dead list needs to be reduced */
+			_CompressMemoryUsage();
+		}
+#endif
+	}else{
+		OSLockRelease(g_psLinkedListLock);
+	}
+}
+
+/* For now we do not want to expose the global stats API
+ * so we wrap it into this specific function for pooled pages.
+ * As soon as we need to modify the global stats directly somewhere else
+ * we want to replace these functions with more general ones.
+ */
+void
+PVRSRVStatsIncrMemAllocPoolStat(size_t uiBytes)
+{
+	_increase_global_stat(PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES, uiBytes);
+}
+
+void
+PVRSRVStatsDecrMemAllocPoolStat(size_t uiBytes)
+{
+	_decrease_global_stat(PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES, uiBytes);
+}
+
+PVRSRV_ERROR
+PVRSRVStatsUpdateOOMStat(CONNECTION_DATA *psConnection,
+						  PVRSRV_DEVICE_NODE *psDeviceNode,
+						  IMG_UINT32 ui32OOMStatType,
+						  IMG_PID pidOwner)
+{
+	PVRSRV_DEVICE_STAT_TYPE eOOMStatType = (PVRSRV_DEVICE_STAT_TYPE) ui32OOMStatType;
+	IMG_PID	pidCurrent = pidOwner;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32OOMStatType >= PVRSRV_DEVICE_STAT_TYPE_COUNT)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(pidCurrent);
+	if (psProcessStats != NULL)
+	{
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[psDeviceNode->sDevId.ui32InternalID][eOOMStatType]++;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PVRSRVStatsUpdateOOMStat: Process not found for Pid=%d", pidCurrent));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+} /* PVRSRVStatsUpdateOOMStat */
+
+void
+PVRSRVStatsUpdateRenderContextStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32TotalNumPartialRenders,
+									IMG_UINT32 ui32TotalNumOutOfMemory,
+									IMG_UINT32 ui32NumTAStores,
+									IMG_UINT32 ui32Num3DStores,
+									IMG_UINT32 ui32NumCDMStores,
+									IMG_UINT32 ui32NumTDMStores,
+									IMG_UINT32 ui32NumRayStores,
+									IMG_PID pidOwner)
+{
+	IMG_PID	pidCurrent = pidOwner;
+
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(pidCurrent);
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_PRS]       += ui32TotalNumPartialRenders;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_OOMS]      += ui32TotalNumOutOfMemory;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_TA_STORES] += ui32NumTAStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_3D_STORES] += ui32Num3DStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_CDM_STORES]+= ui32NumCDMStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_TDM_STORES]+= ui32NumTDMStores;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_RC_RAY_STORES]+= ui32NumRayStores;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "PVRSRVStatsUpdateRenderContextStats: Process not found for Pid=%d", pidCurrent));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateRenderContextStats */
+
+void
+PVRSRVStatsUpdateZSBufferStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+							   IMG_UINT32 ui32NumReqByApp,
+							   IMG_UINT32 ui32NumReqByFW,
+							   IMG_PID owner)
+{
+	IMG_PID				  currentPid = (owner==0)?OSGetCurrentClientProcessIDKM():owner;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_APP] += ui32NumReqByApp;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_ZSBUFFER_REQS_BY_FW] += ui32NumReqByFW;
+		OSLockRelease(psProcessStats->hLock);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Process not found for Pid=%d", __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateZSBufferStats */
+
+void
+PVRSRVStatsUpdateFreelistStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+							   IMG_UINT32 ui32NumGrowReqByApp,
+							   IMG_UINT32 ui32NumGrowReqByFW,
+							   IMG_UINT32 ui32InitFLPages,
+							   IMG_UINT32 ui32NumHighPages,
+							   IMG_PID ownerPid)
+{
+	IMG_PID				  currentPid = (ownerPid!=0)?ownerPid:OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+
+	if (psProcessStats != NULL)
+	{
+		IMG_UINT32 ui32DevID = psDeviceNode->sDevId.ui32InternalID;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_APP] += ui32NumGrowReqByApp;
+		psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_GROW_REQS_BY_FW] += ui32NumGrowReqByFW;
+
+		UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_PAGES_INIT],
+						 (IMG_INT32) ui32InitFLPages);
+
+		UPDATE_MAX_VALUE(psProcessStats->ai32DevStats[ui32DevID][PVRSRV_DEVICE_STAT_TYPE_FREELIST_MAX_PAGES],
+						 (IMG_INT32) ui32NumHighPages);
+		OSLockRelease(psProcessStats->hLock);
+
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Process not found for Pid=%d", __func__, currentPid));
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateFreelistStats */
+
+
+#if defined(ENABLE_DEBUGFS_PIDS)
+
+int
+GenericStatsPrintElementsLive(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_STAT_PV_DATA *psStatType = DIGetPrivData(psEntry);
+	PVRSRV_PROCESS_STATS* psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	PVR_ASSERT(psStatType->pfnStatsPrintElements != NULL);
+
+	DIPrintf(psEntry, "%s\n", psStatType->szLiveStatsHeaderStr);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (dllist_is_empty(&gsLiveList))
+	{
+		DIPrintf(psEntry, "No Stats to display\n%s\n", g_szSeparatorStr);
+	}
+	else
+	{
+		dllist_foreach_node(&gsLiveList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			psStatType->pfnStatsPrintElements(psEntry, psProcessStats);
+			DIPrintf(psEntry, "%s\n", g_szSeparatorStr);
+		}
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+}
+
+int
+GenericStatsPrintElementsRetired(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVRSRV_STAT_PV_DATA *psStatType = DIGetPrivData(psEntry);
+	PVRSRV_PROCESS_STATS* psProcessStats;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	PVR_ASSERT(psStatType->pfnStatsPrintElements != NULL);
+
+	DIPrintf(psEntry, "%s\n", psStatType->szRetiredStatsHeaderStr);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	if (dllist_is_empty(&gsDeadList))
+	{
+		DIPrintf(psEntry, "No Stats to display\n%s\n", g_szSeparatorStr);
+	}
+	else
+	{
+		dllist_foreach_node(&gsDeadList, psNode, psNext)
+		{
+			psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+			psStatType->pfnStatsPrintElements(psEntry, psProcessStats);
+			DIPrintf(psEntry, "%s\n", g_szSeparatorStr);
+		}
+	}
+	OSLockRelease(g_psLinkedListLock);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PERPID_STATS)
+/*************************************************************************/ /*!
+@Function       ProcessStatsPrintElements
+@Description    Prints all elements for this process statistic record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+ProcessStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                          PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_UINT32 ui32StatNumber;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RILockAcquireKM();
+#endif
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+	DIPrintf(psEntry, "PID %u - %s\n", psProcessStats->pid, psProcessStats->processName);
+
+	/* Print device stats table PVRSRV_DEVICE_STAT_TYPE */
+	if (psProcessStats->ui32DevCount > 0)
+	{
+		IMG_UINT32 i;
+
+		for (ui32StatNumber = 0;
+			 ui32StatNumber < ARRAY_SIZE(pszDeviceStatType);
+			 ui32StatNumber++)
+		{
+			if (OSStringNCompare(pszDeviceStatType[ui32StatNumber], "", 1) != 0)
+			{
+				DIPrintf(psEntry, "%-34s",
+						 pszDeviceStatType[ui32StatNumber]);
+
+				for (i = 0; i < psProcessStats->ui32DevCount; i++)
+				{
+					if (i == 0)
+					{
+						DIPrintf(psEntry, "%10d",
+								 psProcessStats->ai32DevStats[i][ui32StatNumber]);
+					}
+					else
+					{
+						DIPrintf(psEntry, ",%d",
+								 psProcessStats->ai32DevStats[i][ui32StatNumber]);
+					}
+				}
+			}
+
+			DIPrintf(psEntry, "\n");
+		}
+	}
+
+	/* Print process memory stats table PVRSRV_PROCESS_STAT_TYPE */
+	for (ui32StatNumber = 0;
+	     ui32StatNumber < ARRAY_SIZE(pszProcessStatType);
+	     ui32StatNumber++)
+	{
+		if (OSStringNCompare(pszProcessStatType[ui32StatNumber], "", 1) != 0)
+		{
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+			if ((ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES) ||
+			    (ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES))
+			{
+				/* get the stat from RI */
+				IMG_INT32 ui32Total = RITotalAllocProcessUnlocked(psProcessStats->pid,
+									    (ui32StatNumber == PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES)
+									    ? PHYS_HEAP_TYPE_LMA : PHYS_HEAP_TYPE_UMA);
+
+				DIPrintf(psEntry, "%-34s%10d %8dK\n",
+						 pszProcessStatType[ui32StatNumber], ui32Total, ui32Total>>10);
+			}
+			else
+#endif
+			{
+				if (ui32StatNumber >= PVRSRV_PROCESS_STAT_TYPE_KMALLOC &&
+					ui32StatNumber <= PVRSRV_PROCESS_STAT_TYPE_TOTAL_MAX)
+				{
+					DIPrintf(psEntry, "%-34s%10"IMG_INT64_FMTSPECd" %8"IMG_INT64_FMTSPECd"K\n",
+					         pszProcessStatType[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber] >> 10);
+				}
+				else
+				{
+					DIPrintf(psEntry, "%-34s%10"IMG_INT64_FMTSPECd"\n",
+					         pszProcessStatType[ui32StatNumber],
+					         psProcessStats->i64StatValue[ui32StatNumber]);
+				}
+			}
+		}
+	}
+
+	OSLockRelease(psProcessStats->hLock);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RILockReleaseKM();
+#endif
+} /* ProcessStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void
+PVRSRVStatsUpdateCacheOpStats(PVRSRV_CACHE_OP uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							IMG_DEV_VIRTADDR sDevVAddr,
+							IMG_DEV_PHYADDR sDevPAddr,
+#endif
+							IMG_DEVMEM_SIZE_T uiOffset,
+							IMG_DEVMEM_SIZE_T uiSize,
+							IMG_UINT64 ui64ExecuteTime,
+							IMG_BOOL bUserModeFlush,
+							IMG_PID ownerPid)
+{
+	IMG_PID				  currentPid = (ownerPid!=0)?ownerPid:OSGetCurrentClientProcessIDKM();
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	/* Don't do anything if we are not initialised or we are shutting down! */
+	if (!bProcessStatsInitialised)
+	{
+		return;
+	}
+
+	/* Lock while we find the correct process and update the record... */
+	OSLockAcquire(g_psLinkedListLock);
+
+	psProcessStats = _FindProcessStats(currentPid);
+
+	if (psProcessStats != NULL)
+	{
+		IMG_INT32 Idx;
+
+		OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+		/* Look-up next buffer write index */
+		Idx = psProcessStats->uiCacheOpWriteIndex;
+		psProcessStats->uiCacheOpWriteIndex = INCREMENT_CACHEOP_STAT_IDX_WRAP(Idx);
+
+		/* Store all CacheOp meta-data */
+		psProcessStats->asCacheOp[Idx].uiCacheOp = uiCacheOp;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+		psProcessStats->asCacheOp[Idx].sDevVAddr = sDevVAddr;
+		psProcessStats->asCacheOp[Idx].sDevPAddr = sDevPAddr;
+#endif
+		psProcessStats->asCacheOp[Idx].uiOffset = uiOffset;
+		psProcessStats->asCacheOp[Idx].uiSize = uiSize;
+		psProcessStats->asCacheOp[Idx].bUserModeFlush = bUserModeFlush;
+		psProcessStats->asCacheOp[Idx].ui64ExecuteTime = ui64ExecuteTime;
+
+		OSLockRelease(psProcessStats->hLock);
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+} /* PVRSRVStatsUpdateCacheOpStats */
+
+/*************************************************************************/ /*!
+@Function       CacheOpStatsPrintElements
+@Description    Prints all elements for this process statistic CacheOp record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+CacheOpStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                          PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_CHAR  *pszCacheOpType, *pszFlushType, *pszFlushMode;
+	IMG_INT32 i32WriteIdx, i32ReadIdx;
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+	#define CACHEOP_RI_PRINTF_HEADER \
+		"%-10s %-10s %-5s %-16s %-16s %-10s %-10s %-12s\n"
+	#define CACHEOP_RI_PRINTF		\
+		"%-10s %-10s %-5s 0x%-14llx 0x%-14llx 0x%-8llx 0x%-8llx %-12llu\n"
+#else
+	#define CACHEOP_PRINTF_HEADER	\
+		"%-10s %-10s %-5s %-10s %-10s %-12s\n"
+	#define CACHEOP_PRINTF			\
+		"%-10s %-10s %-5s 0x%-8llx 0x%-8llx %-12llu\n"
+#endif
+
+	DIPrintf(psEntry, "PID %u - %s\n", psProcessStats->pid, psProcessStats->processName);
+
+	/* File header info */
+	DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+					CACHEOP_RI_PRINTF_HEADER,
+#else
+					CACHEOP_PRINTF_HEADER,
+#endif
+					"CacheOp",
+					"Type",
+					"Mode",
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+					"DevVAddr",
+					"DevPAddr",
+#endif
+					"Offset",
+					"Size",
+					"Time (us)");
+
+	/* Take a snapshot of write index, read backwards in buffer
+	   and wrap round at boundary */
+	i32WriteIdx = psProcessStats->uiCacheOpWriteIndex;
+	for (i32ReadIdx = DECREMENT_CACHEOP_STAT_IDX_WRAP(i32WriteIdx);
+		 i32ReadIdx != i32WriteIdx;
+		 i32ReadIdx = DECREMENT_CACHEOP_STAT_IDX_WRAP(i32ReadIdx))
+	{
+		IMG_UINT64 ui64ExecuteTime = psProcessStats->asCacheOp[i32ReadIdx].ui64ExecuteTime;
+		IMG_DEVMEM_SIZE_T ui64NumOfPages = psProcessStats->asCacheOp[i32ReadIdx].uiSize >> OSGetPageShift();
+
+		if (ui64NumOfPages <= PMR_MAX_TRANSLATION_STACK_ALLOC)
+		{
+			pszFlushType = "RBF.Fast";
+		}
+		else
+		{
+			pszFlushType = "RBF.Slow";
+		}
+
+		if (psProcessStats->asCacheOp[i32ReadIdx].bUserModeFlush)
+		{
+			pszFlushMode = "UM";
+		}
+		else
+		{
+			pszFlushMode = "KM";
+		}
+
+		switch (psProcessStats->asCacheOp[i32ReadIdx].uiCacheOp)
+		{
+			case PVRSRV_CACHE_OP_NONE:
+				pszCacheOpType = "None";
+				break;
+			case PVRSRV_CACHE_OP_CLEAN:
+				pszCacheOpType = "Clean";
+				break;
+			case PVRSRV_CACHE_OP_INVALIDATE:
+				pszCacheOpType = "Invalidate";
+				break;
+			case PVRSRV_CACHE_OP_FLUSH:
+				pszCacheOpType = "Flush";
+				break;
+			default:
+				pszCacheOpType = "Unknown";
+				break;
+		}
+
+		DIPrintf(psEntry,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							CACHEOP_RI_PRINTF,
+#else
+							CACHEOP_PRINTF,
+#endif
+							pszCacheOpType,
+							pszFlushType,
+							pszFlushMode,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+							psProcessStats->asCacheOp[i32ReadIdx].sDevVAddr.uiAddr,
+							psProcessStats->asCacheOp[i32ReadIdx].sDevPAddr.uiAddr,
+#endif
+							psProcessStats->asCacheOp[i32ReadIdx].uiOffset,
+							psProcessStats->asCacheOp[i32ReadIdx].uiSize,
+							ui64ExecuteTime);
+		}
+
+} /* CacheOpStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_MEMORY_STATS)
+static PVRSRV_ERROR _PrintMemStatsEntry(uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	IMG_UINT32	ui32VAddrFields = sizeof(void*)/sizeof(IMG_UINT32);
+	IMG_UINT32	ui32PAddrFields = sizeof(IMG_CPU_PHYADDR)/sizeof(IMG_UINT32);
+	IMG_UINT32 ui32ItemNumber;
+	PVRSRV_MEM_ALLOC_REC *psRecord = (PVRSRV_MEM_ALLOC_REC *)(uintptr_t)v;
+	PVRSRV_MEM_ALLOC_PRINT_DATA *psPrintData = (PVRSRV_MEM_ALLOC_PRINT_DATA *)pvPriv;
+	OSDI_IMPL_ENTRY *psEntry = psPrintData->psEntry;
+
+	if (psRecord != NULL)
+	{
+		IMG_BOOL bPrintStat = IMG_TRUE;
+
+		DIPrintf(psEntry, "%-5d  ", psPrintData->pid);
+
+		switch (psRecord->eAllocType)
+		{
+		case PVRSRV_MEM_ALLOC_TYPE_KMALLOC:             DIPrintf(psEntry, "KMALLOC             "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_VMALLOC:             DIPrintf(psEntry, "VMALLOC             "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA:  DIPrintf(psEntry, "ALLOC_PAGES_PT_LMA  "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA:  DIPrintf(psEntry, "ALLOC_PAGES_PT_UMA  "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA:      DIPrintf(psEntry, "IOREMAP_PT_LMA      "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA:         DIPrintf(psEntry, "VMAP_PT_UMA         "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES:     DIPrintf(psEntry, "ALLOC_LMA_PAGES     "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES:    DIPrintf(psEntry, "ZOMBIE_LMA_PAGES    "); break;
+#endif
+		case PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES:     DIPrintf(psEntry, "ALLOC_UMA_PAGES     "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES:    DIPrintf(psEntry, "ZOMBIE_UMA_PAGES    "); break;
+#endif
+		case PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES:   DIPrintf(psEntry, "MAP_UMA_LMA_PAGES   "); break;
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT:      DIPrintf(psEntry, "DMA_BUF_IMPORT      "); break;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+		case PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE:      DIPrintf(psEntry, "DMA_BUF_ZOMBIE      "); break;
+#endif
+		default:                                        DIPrintf(psEntry, "INVALID             "); break;
+		}
+
+		if (bPrintStat)
+		{
+			for (ui32ItemNumber = 0; ui32ItemNumber < ui32VAddrFields; ui32ItemNumber++)
+			{
+				DIPrintf(psEntry, "%08x", *(((IMG_UINT32*) &psRecord->pvCpuVAddr) + ui32VAddrFields - ui32ItemNumber - 1));
+			}
+			DIPrintf(psEntry, "  ");
+
+			for (ui32ItemNumber = 0; ui32ItemNumber < ui32PAddrFields; ui32ItemNumber++)
+			{
+				DIPrintf(psEntry, "%08x", *(((IMG_UINT32*) &psRecord->sCpuPAddr.uiAddr) + ui32PAddrFields - ui32ItemNumber - 1));
+			}
+
+#if defined(PVRSRV_DEBUG_LINUX_MEMORY_STATS_ON)
+			DIPrintf(psEntry, "  " IMG_SIZE_FMTSPEC, psRecord->uiBytes);
+
+			DIPrintf(psEntry, "  %s", (IMG_CHAR*) psRecord->pvAllocdFromFile);
+
+			DIPrintf(psEntry, "  %d\n", psRecord->ui32AllocdFromLine);
+#else
+			DIPrintf(psEntry, "  " IMG_SIZE_FMTSPEC "\n", psRecord->uiBytes);
+#endif
+		}
+
+		psPrintData->ui32NumEntries++;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       MemStatsPrintElements
+@Description    Prints all elements for the memory statistic record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void
+MemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                      PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_UINT32	ui32VAddrFields = sizeof(void*)/sizeof(IMG_UINT32);
+	IMG_UINT32	ui32PAddrFields = sizeof(IMG_CPU_PHYADDR)/sizeof(IMG_UINT32);
+	IMG_UINT32 ui32ItemNumber;
+	PVRSRV_MEM_ALLOC_PRINT_DATA sPrintData;
+
+	sPrintData.psEntry = psEntry;
+	sPrintData.pid = psProcessStats->pid;
+	sPrintData.ui32NumEntries = 0;
+
+	/* Write the header... */
+	DIPrintf(psEntry, "PID    ");
+
+	DIPrintf(psEntry, "Type                VAddress");
+	for (ui32ItemNumber = 1;  ui32ItemNumber < ui32VAddrFields;  ui32ItemNumber++)
+	{
+		DIPrintf(psEntry, "        ");
+	}
+
+	DIPrintf(psEntry, "  PAddress");
+	for (ui32ItemNumber = 1;  ui32ItemNumber < ui32PAddrFields;  ui32ItemNumber++)
+	{
+		DIPrintf(psEntry, "        ");
+	}
+
+	DIPrintf(psEntry, "  Size(bytes)\n");
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+	HASH_Iterate(psProcessStats->psMemoryRecords, (HASH_pfnCallback)_PrintMemStatsEntry, &sPrintData);
+	OSLockRelease(psProcessStats->hLock);
+
+	if (sPrintData.ui32NumEntries == 0)
+	{
+		DIPrintf(psEntry, "%-5d\n", psProcessStats->pid);
+	}
+} /* MemStatsPrintElements */
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+/*************************************************************************/ /*!
+@Function       RIMemStatsPrintElements
+@Description    Prints all elements for the RI Memory record.
+@Input          pvStatPtr         Pointer to statistics structure.
+@Input          pfnOSStatsPrintf  Printf function to use for output.
+*/ /**************************************************************************/
+void RIMemStatsPrintElements(OSDI_IMPL_ENTRY *psEntry,
+                             PVRSRV_PROCESS_STATS *psProcessStats)
+{
+	IMG_CHAR   *pszStatFmtText  = NULL;
+	IMG_HANDLE *pRIHandle       = NULL;
+
+	/* Acquire RI lock */
+	RILockAcquireKM();
+
+	/*
+	 * Loop through the RI system to get each line of text.
+	 */
+	while (RIGetListEntryKM(psProcessStats->pid,
+							&pRIHandle,
+							&pszStatFmtText))
+	{
+		DIPrintf(psEntry, "%s", pszStatFmtText);
+	}
+
+	/* Release RI lock */
+	RILockReleaseKM();
+
+} /* RIMemStatsPrintElements */
+#endif
+
+#endif
+
+int GlobalStatsPrintElements(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	IMG_UINT32 ui32StatNumber;
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+	for (ui32StatNumber = 0;
+	     ui32StatNumber < ARRAY_SIZE(pszDriverStatType);
+	     ui32StatNumber++)
+	{
+		if (OSStringNCompare(pszDriverStatType[ui32StatNumber], "", 1) != 0)
+		{
+			DIPrintf(psEntry, "%-34s%12" IMG_UINT64_FMTSPEC "\n",
+				    pszDriverStatType[ui32StatNumber],
+				    GET_GLOBAL_STAT_VALUE(ui32StatNumber));
+		}
+	}
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVFindProcessMemStats
+@Description    Using the provided PID find memory stats for that process.
+                Memstats will be provided for live/connected processes only.
+                Memstat values provided by this API relate only to the physical
+                memory allocated by the process and does not relate to any of
+                the mapped or imported memory.
+@Input          pid                 Process to search for.
+@Input          ArraySize           Size of the array where memstat
+                                    records will be stored
+@Input          bAllProcessStats    Flag to denote if stats for
+                                    individual process are requested
+                                    stats for all processes are
+                                    requested
+@Input          MemoryStats         Handle to the memory where memstats
+                                    are stored.
+@Output         Memory statistics records for the requested pid.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVFindProcessMemStats(IMG_PID pid,
+                                       IMG_UINT32 ui32ArrSize,
+                                       IMG_BOOL bAllProcessStats,
+                                       IMG_UINT64 *pui64MemoryStats)
+{
+	IMG_INT i;
+	PVRSRV_PROCESS_STATS* psProcessStats;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pui64MemoryStats, "pui64MemoryStats");
+
+	if (bAllProcessStats)
+	{
+		PVR_LOG_RETURN_IF_FALSE(ui32ArrSize == PVRSRV_DRIVER_STAT_TYPE_COUNT,
+			"MemStats array size is incorrect",
+			PVRSRV_ERROR_INVALID_PARAMS);
+
+		OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+		for (i = 0; i < ui32ArrSize; i++)
+		{
+			pui64MemoryStats[i] = GET_GLOBAL_STAT_VALUE(i);
+		}
+
+		OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrSize == PVRSRV_PROCESS_STAT_TYPE_COUNT,
+		"MemStats array size is incorrect",
+		PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	/* Search for the given PID in the Live List */
+	psProcessStats = _FindProcessStatsInLiveList(pid);
+
+	if (psProcessStats == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Process %d not found. This process may not be live anymore.", (IMG_INT)pid));
+		OSLockRelease(g_psLinkedListLock);
+
+		return PVRSRV_ERROR_PROCESS_NOT_FOUND;
+	}
+
+	OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+	for (i = 0; i < ui32ArrSize; i++)
+	{
+		pui64MemoryStats[i] = psProcessStats->i64StatValue[i];
+	}
+	OSLockRelease(psProcessStats->hLock);
+
+	OSLockRelease(g_psLinkedListLock);
+
+	return PVRSRV_OK;
+
+} /* PVRSRVFindProcessMemStats */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetProcessMemUsage
+@Description    Calculate allocated kernel and graphics memory for all live or
+                connected processes. Memstat values provided by this API relate
+                only to the physical memory allocated by the process and does
+                not relate to any of the mapped or imported memory.
+@Output         pui64TotalMem                   Total memory usage for all live
+                                                PIDs connected to the driver.
+@Output         pui32NumberOfLivePids           Number of live pids currently
+                                                connected to the server.
+@Output         ppsPerProcessMemUsageData       Handle to an array of
+                                                PVRSRV_PER_PROCESS_MEM_USAGE,
+                                                number of elements defined by
+                                                pui32NumberOfLivePids.
+@Return         PVRSRV_OK                       Success
+                PVRSRV_ERROR_PROCESS_NOT_FOUND  No live processes.
+                PVRSRV_ERROR_OUT_OF_MEMORY      Failed to allocate memory for
+                                                ppsPerProcessMemUsageData.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVGetProcessMemUsage(IMG_UINT64 *pui64TotalMem,
+									  IMG_UINT32 *pui32NumberOfLivePids,
+									  PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsageData)
+{
+	IMG_UINT32 ui32NumberOfLivePids = 0;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_PROCESS_NOT_FOUND;
+	PVRSRV_PER_PROCESS_MEM_USAGE* psPerProcessMemUsageData = NULL;
+	DLLIST_NODE *psNode, *psNext;
+
+	OSLockAcquire(gsGlobalStats.hGlobalStatsLock);
+
+	*pui64TotalMem = GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_KMALLOC) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_VMALLOC) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_LMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_GPUMEM_UMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_UMA) +
+		GET_GLOBAL_STAT_VALUE(PVRSRV_DRIVER_STAT_TYPE_ALLOC_PT_MEMORY_LMA);
+
+	OSLockRelease(gsGlobalStats.hGlobalStatsLock);
+
+	OSLockAcquire(g_psLinkedListLock);
+
+	dllist_foreach_node(&gsLiveList, psNode, psNext)
+	{
+		ui32NumberOfLivePids++;
+	}
+
+	if (ui32NumberOfLivePids > 0)
+	{
+		/* Use OSAllocZMemNoStats to prevent deadlock. */
+		psPerProcessMemUsageData = OSAllocZMemNoStats(ui32NumberOfLivePids * sizeof(*psPerProcessMemUsageData));
+
+		if (psPerProcessMemUsageData)
+		{
+			PVRSRV_PROCESS_STATS* psProcessStats = NULL;
+			IMG_UINT32 ui32Counter = 0;
+
+			dllist_foreach_node(&gsLiveList, psNode, psNext)
+			{
+				psProcessStats = IMG_CONTAINER_OF(psNode, PVRSRV_PROCESS_STATS, sNode);
+				OSLockAcquireNested(psProcessStats->hLock, PROCESS_LOCK_SUBCLASS_CURRENT);
+
+				psPerProcessMemUsageData[ui32Counter].ui32Pid = (IMG_UINT32)psProcessStats->pid;
+
+				psPerProcessMemUsageData[ui32Counter].ui64KernelMemUsage =
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_KMALLOC] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_VMALLOC];
+
+				psPerProcessMemUsageData[ui32Counter].ui64GraphicsMemUsage =
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_UMA] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_PAGES_PT_LMA] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_LMA_PAGES] +
+					psProcessStats->i64StatValue[PVRSRV_PROCESS_STAT_TYPE_ALLOC_UMA_PAGES];
+
+				OSLockRelease(psProcessStats->hLock);
+				ui32Counter++;
+			}
+			eError = PVRSRV_OK;
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+
+	OSLockRelease(g_psLinkedListLock);
+	*pui32NumberOfLivePids = ui32NumberOfLivePids;
+	*ppsPerProcessMemUsageData = psPerProcessMemUsageData;
+
+	return eError;
+
+} /* PVRSRVGetProcessMemUsage */
diff --git a/drivers/gpu/drm/img-rogue/process_stats.h b/drivers/gpu/drm/img-rogue/process_stats.h
new file mode 100644
index 000000000000..c410736e1749
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/process_stats.h
@@ -0,0 +1,220 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for creating and reading proc filesystem entries.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PROCESS_STATS_H
+#define PROCESS_STATS_H
+
+#include <powervr/mem_types.h>
+
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "connection_server.h"
+
+/*
+ * The publishing of Process Stats is controlled by the
+ * PVRSRV_ENABLE_PROCESS_STATS build option. The recording of all Memory
+ * allocations is controlled by the PVRSRV_ENABLE_MEMORY_STATS build option.
+ *
+ * Note: There will be a performance degradation with memory allocation
+ *       recording enabled!
+ */
+
+
+/*
+ * Memory types which can be tracked...
+ */
+typedef enum {
+	PVRSRV_MEM_ALLOC_TYPE_KMALLOC,				/* memory allocated by kmalloc() */
+	PVRSRV_MEM_ALLOC_TYPE_VMALLOC,				/* memory allocated by vmalloc() */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_UMA,	/* pages allocated from UMA to hold page table information */
+	PVRSRV_MEM_ALLOC_TYPE_VMAP_PT_UMA,			/* ALLOC_PAGES_PT_UMA mapped to kernel address space */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_PAGES_PT_LMA,	/* pages allocated from LMA to hold page table information */
+	PVRSRV_MEM_ALLOC_TYPE_IOREMAP_PT_LMA,		/* ALLOC_PAGES_PT_LMA mapped to kernel address space */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_LMA_PAGES,		/* pages allocated from LMA */
+	PVRSRV_MEM_ALLOC_TYPE_ALLOC_UMA_PAGES,		/* pages allocated from UMA */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_LMA_PAGES,		/* zombie pages allocated from LMA */
+	PVRSRV_MEM_ALLOC_TYPE_ZOMBIE_UMA_PAGES,		/* zombie pages allocated from UMA */
+#endif
+	PVRSRV_MEM_ALLOC_TYPE_MAP_UMA_LMA_PAGES,	/* mapped UMA/LMA pages */
+	PVRSRV_MEM_ALLOC_TYPE_UMA_POOL_PAGES,		/* pages in the page pool */
+	PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_IMPORT,       /* dma-buf imports */
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PVRSRV_MEM_ALLOC_TYPE_DMA_BUF_ZOMBIE,       /* dma-buf zombie */
+#endif
+
+	/* Must be the last enum...*/
+	PVRSRV_MEM_ALLOC_TYPE_COUNT
+} PVRSRV_MEM_ALLOC_TYPE;
+
+/*
+ * Functions for managing the processes recorded...
+ */
+PVRSRV_ERROR PVRSRVStatsInitialise(void);
+void PVRSRVStatsDestroy(void);
+void PVRSRVStatsDestroyDI(void);
+
+PVRSRV_ERROR PVRSRVStatsRegisterProcess(IMG_HANDLE* phProcessStats);
+
+void PVRSRVStatsDeregisterProcess(IMG_HANDLE hProcessStats);
+
+PVRSRV_ERROR PVRSRVStatsDeviceConnect(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void PVRSRVStatsDeviceDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*
+ * Functions for recording the statistics...
+ */
+
+void PVRSRVStatsAddMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								  void *pvCpuVAddr,
+								  IMG_CPU_PHYADDR sCpuPAddr,
+								  size_t uiBytes,
+								  IMG_PID uiPid
+								  DEBUG_MEMSTATS_PARAMS);
+
+void PVRSRVStatsRemoveMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									 IMG_UINT64 ui64Key,
+									 IMG_PID uiPid);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void PVRSRVStatsTransferMemAllocRecord(PVRSRV_MEM_ALLOC_TYPE eCurrentType,
+                                       PVRSRV_MEM_ALLOC_TYPE eTargetType,
+                                       IMG_UINT64 ui64Key,
+                                       IMG_PID currentPid
+                                       DEBUG_MEMSTATS_PARAMS);
+#endif
+
+void PVRSRVStatsIncrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								 size_t uiBytes,
+								 IMG_PID uiPid);
+
+/*
+ * Increases the memory stat for eAllocType. Tracks the allocation size value
+ * by inserting a value into a hash table with uiCpuVAddr as key.
+ * Pair with PVRSRVStatsDecrMemAllocStatAndUntrack().
+ */
+void PVRSRVStatsIncrMemAllocStatAndTrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+										 size_t uiBytes,
+										 IMG_UINT64 uiCpuVAddr,
+										 IMG_PID uiPid);
+
+void PVRSRVStatsDecrMemAllocStat(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+								 size_t uiBytes,
+								 IMG_PID uiPid);
+
+void PVRSRVStatsDecrMemKAllocStat(size_t uiBytes,
+								  IMG_PID decrPID);
+
+/*
+ * Decrease the memory stat for eAllocType. Takes the allocation size value
+ * from the hash table with uiCpuVAddr as key.
+ * Pair with PVRSRVStatsIncrMemAllocStatAndTrack().
+ */
+void PVRSRVStatsDecrMemAllocStatAndUntrack(PVRSRV_MEM_ALLOC_TYPE eAllocType,
+									IMG_UINT64 uiCpuVAddr);
+
+void
+PVRSRVStatsIncrMemAllocPoolStat(size_t uiBytes);
+
+void
+PVRSRVStatsDecrMemAllocPoolStat(size_t uiBytes);
+
+PVRSRV_ERROR
+PVRSRVStatsUpdateOOMStat(CONNECTION_DATA *psConnection,
+						  PVRSRV_DEVICE_NODE *psDeviceNode,
+						  IMG_UINT32 ui32OOMStatType,
+						  IMG_PID pidOwner);
+
+void PVRSRVStatsUpdateRenderContextStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+										 IMG_UINT32 ui32TotalNumPartialRenders,
+										 IMG_UINT32 ui32TotalNumOutOfMemory,
+										 IMG_UINT32 ui32TotalTAStores,
+										 IMG_UINT32 ui32Total3DStores,
+										 IMG_UINT32 ui32TotalCDMStores,
+										 IMG_UINT32 ui32TotalTDMStores,
+										 IMG_UINT32 ui32NumRayStores,
+										 IMG_PID owner);
+
+void PVRSRVStatsUpdateZSBufferStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32NumReqByApp,
+									IMG_UINT32 ui32NumReqByFW,
+									IMG_PID owner);
+
+void PVRSRVStatsUpdateFreelistStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+									IMG_UINT32 ui32NumGrowReqByApp,
+									IMG_UINT32 ui32NumGrowReqByFW,
+									IMG_UINT32 ui32InitFLPages,
+									IMG_UINT32 ui32NumHighPages,
+									IMG_PID    ownerPid);
+#if defined(PVRSRV_ENABLE_CACHEOP_STATS)
+void PVRSRVStatsUpdateCacheOpStats(PVRSRV_CACHE_OP uiCacheOp,
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) && defined(DEBUG)
+								   IMG_DEV_VIRTADDR sDevVAddr,
+								   IMG_DEV_PHYADDR sDevPAddr,
+#endif
+								   IMG_DEVMEM_SIZE_T uiOffset,
+								   IMG_DEVMEM_SIZE_T uiSize,
+								   IMG_UINT64 ui64ExecuteTimeMs,
+								   IMG_BOOL bUserModeFlush,
+								   IMG_PID ownerPid);
+#endif
+
+/* Functions used for calculating the memory usage statistics of a process */
+PVRSRV_ERROR PVRSRVFindProcessMemStats(IMG_PID pid,
+                                       IMG_UINT32 ui32ArrSize,
+                                       IMG_BOOL bAllProcessStats,
+                                       IMG_UINT64 *pui64MemoryStats);
+
+typedef struct {
+	IMG_UINT32 ui32Pid;
+	IMG_UINT64 ui64KernelMemUsage;
+	IMG_UINT64 ui64GraphicsMemUsage;
+} PVRSRV_PER_PROCESS_MEM_USAGE;
+
+PVRSRV_ERROR PVRSRVGetProcessMemUsage(IMG_UINT64 *pui64TotalMem,
+                                      IMG_UINT32 *pui32NumberOfLivePids,
+                                      PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsageData);
+
+#endif /* PROCESS_STATS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_bridge.h b/drivers/gpu/drm/img-rogue/pvr_bridge.h
new file mode 100644
index 000000000000..dafc51dd57f0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_bridge.h
@@ -0,0 +1,443 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the PVR Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BRIDGE_H
+#define PVR_BRIDGE_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "pvrsrv_error.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "common_dc_bridge.h"
+#if defined(SUPPORT_DCPLAT_BRIDGE)
+#include "common_dcplat_bridge.h"
+#endif
+#endif
+#include "common_mm_bridge.h"
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+#include "common_mmplat_bridge.h"
+#endif
+#if defined(SUPPORT_WRAP_EXTMEM)
+#include "common_mmextmem_bridge.h"
+#endif
+#if !defined(EXCLUDE_CMM_BRIDGE)
+#include "common_cmm_bridge.h"
+#endif
+#if defined(__linux__)
+#include "common_dmabuf_bridge.h"
+#endif
+#if defined(PDUMP)
+#include "common_pdump_bridge.h"
+#include "common_pdumpctrl_bridge.h"
+#include "common_pdumpmm_bridge.h"
+#endif
+#include "common_cache_bridge.h"
+#if defined(SUPPORT_DMA_TRANSFER)
+#include "common_dma_bridge.h"
+#endif
+#include "common_srvcore_bridge.h"
+#include "common_sync_bridge.h"
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "common_smm_bridge.h"
+#endif
+#if defined(PVRSRV_ENABLE_HTB)
+#include "common_htbuffer_bridge.h"
+#endif
+#include "common_pvrtl_bridge.h"
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "common_ri_bridge.h"
+#endif
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+#include "common_validation_bridge.h"
+#endif
+
+
+#include "common_devicememhistory_bridge.h"
+#include "common_synctracking_bridge.h"
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#include "common_syncfallback_bridge.h"
+#endif
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+#include "common_di_bridge.h"
+#endif
+
+/*
+ * Bridge Cmd Ids
+ */
+
+
+/* Note: The pattern
+ *   #define PVRSRV_BRIDGE_FEATURE (PVRSRV_BRIDGE_PREVFEATURE + 1)
+ *   #if defined(SUPPORT_FEATURE)
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST (PVRSRV_BRIDGE_PREVFEATURE_DISPATCH_LAST + 1)
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST  (PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST + PVRSRV_BRIDGE_FEATURE_CMD_LAST)
+ *   #else
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_FIRST 0
+ *   #define PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST  (PVRSRV_BRIDGE_PREVFEATURE_DISPATCH_LAST)
+ *   #endif
+ * is used in the macro definitions below to make PVRSRV_BRIDGE_FEATURE_*
+ * take up no space in the dispatch table if SUPPORT_FEATURE is disabled.
+ *
+ * Note however that a bridge always defines PVRSRV_BRIDGE_FEATURE, even where
+ * the feature is not enabled (each bridge group retains its own ioctl number).
+ */
+
+#define PVRSRV_BRIDGE_FIRST					0UL
+
+/*   0: Default handler */
+#define PVRSRV_BRIDGE_DEFAULT				0UL
+#define PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST 0UL
+#define PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST  (PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST)
+/*   1: CORE functions */
+#define PVRSRV_BRIDGE_SRVCORE				1UL
+#define PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST (PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST+1)
+#define PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST  (PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST + PVRSRV_BRIDGE_SRVCORE_CMD_LAST)
+
+/*   2: SYNC functions */
+#define PVRSRV_BRIDGE_SYNC					2UL
+#define PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST (PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNC_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNC_CMD_LAST)
+
+/* 3,4: Reserved */
+#define PVRSRV_BRIDGE_RESERVED1				3UL
+#define PVRSRV_BRIDGE_RESERVED1_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST)
+
+#define PVRSRV_BRIDGE_RESERVED2				4UL
+#define PVRSRV_BRIDGE_RESERVED2_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RESERVED2_DISPATCH_LAST  (PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST)
+
+/*   5: PDUMP CTRL layer functions */
+#define PVRSRV_BRIDGE_PDUMPCTRL				5UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMPCTRL_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNC_DISPATCH_LAST)
+#endif
+
+/*   6: Memory Management functions */
+#define PVRSRV_BRIDGE_MM					6UL
+#define PVRSRV_BRIDGE_MM_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MM_DISPATCH_LAST  (PVRSRV_BRIDGE_MM_DISPATCH_FIRST + PVRSRV_BRIDGE_MM_CMD_LAST)
+
+/*   7: Non-Linux Memory Management functions */
+#define PVRSRV_BRIDGE_MMPLAT				7UL
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST (PVRSRV_BRIDGE_MM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST + PVRSRV_BRIDGE_MMPLAT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_MM_DISPATCH_LAST)
+#endif
+
+/*   8: Context Memory Management functions */
+#define PVRSRV_BRIDGE_CMM					8UL
+#if !defined(EXCLUDE_CMM_BRIDGE)
+#define PVRSRV_BRIDGE_CMM_DISPATCH_FIRST (PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_CMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CMM_DISPATCH_FIRST + PVRSRV_BRIDGE_CMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_CMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_CMM_DISPATCH_LAST  (PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST)
+#endif
+
+/*   9: PDUMP Memory Management functions */
+#define PVRSRV_BRIDGE_PDUMPMM				9UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST (PVRSRV_BRIDGE_CMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMPMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CMM_DISPATCH_LAST)
+#endif
+
+/*   10: PDUMP functions */
+#define PVRSRV_BRIDGE_PDUMP					10UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST + PVRSRV_BRIDGE_PDUMP_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST)
+#endif
+
+/*  11: DMABUF functions */
+#define PVRSRV_BRIDGE_DMABUF				11UL
+#if defined(__linux__)
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST (PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST  (PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST + PVRSRV_BRIDGE_DMABUF_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST  (PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST)
+#endif
+
+/*  12: Display Class functions */
+#define PVRSRV_BRIDGE_DC					12UL
+#if defined(SUPPORT_DISPLAY_CLASS)
+#define PVRSRV_BRIDGE_DC_DISPATCH_FIRST (PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DC_DISPATCH_LAST  (PVRSRV_BRIDGE_DC_DISPATCH_FIRST + PVRSRV_BRIDGE_DC_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DC_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DC_DISPATCH_LAST  (PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST)
+#endif
+
+/*  13: Cache interface functions */
+#define PVRSRV_BRIDGE_CACHE					13UL
+#define PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST (PVRSRV_BRIDGE_DC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_CACHE_DISPATCH_LAST  (PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST + PVRSRV_BRIDGE_CACHE_CMD_LAST)
+
+/*  14: Secure Memory Management functions */
+#define PVRSRV_BRIDGE_SMM					14UL
+#if defined(SUPPORT_SECURE_EXPORT)
+#define PVRSRV_BRIDGE_SMM_DISPATCH_FIRST (PVRSRV_BRIDGE_CACHE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SMM_DISPATCH_LAST  (PVRSRV_BRIDGE_SMM_DISPATCH_FIRST + PVRSRV_BRIDGE_SMM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_SMM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_SMM_DISPATCH_LAST  (PVRSRV_BRIDGE_CACHE_DISPATCH_LAST)
+#endif
+
+/*  15: Transport Layer interface functions */
+#define PVRSRV_BRIDGE_PVRTL					15UL
+#define PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST (PVRSRV_BRIDGE_SMM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST  (PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST + PVRSRV_BRIDGE_PVRTL_CMD_LAST)
+
+/*  16: Resource Information (RI) interface functions */
+#define PVRSRV_BRIDGE_RI					16UL
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#define PVRSRV_BRIDGE_RI_DISPATCH_FIRST (PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RI_DISPATCH_LAST  (PVRSRV_BRIDGE_RI_DISPATCH_FIRST + PVRSRV_BRIDGE_RI_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RI_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_RI_DISPATCH_LAST  (PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST)
+#endif
+
+/*  17: Validation interface functions */
+#define PVRSRV_BRIDGE_VALIDATION			17UL
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST (PVRSRV_BRIDGE_RI_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST  (PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST + PVRSRV_BRIDGE_VALIDATION_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST  (PVRSRV_BRIDGE_RI_DISPATCH_LAST)
+#endif
+
+/*  18: TUTILS interface functions */
+#define PVRSRV_BRIDGE_TUTILS				18UL
+#define PVRSRV_BRIDGE_TUTILS_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST  (PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST)
+
+/*  19: DevMem history interface functions */
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY		19UL
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST (PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST  (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST + PVRSRV_BRIDGE_DEVICEMEMHISTORY_CMD_LAST)
+
+/*  20: Host Trace Buffer interface functions */
+#define PVRSRV_BRIDGE_HTBUFFER				20UL
+#if defined(PVRSRV_ENABLE_HTB)
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST  (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST + PVRSRV_BRIDGE_HTBUFFER_CMD_LAST)
+#else	/* !PVRSRV_ENABLE_HTB */
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST  (PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST)
+#endif	/* PVRSRV_ENABLE_HTB */
+
+/*  21: Non-Linux Display functions */
+#define PVRSRV_BRIDGE_DCPLAT				21UL
+#if defined(SUPPORT_DISPLAY_CLASS) && defined(SUPPORT_DCPLAT_BRIDGE)
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST + PVRSRV_BRIDGE_DCPLAT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST  (PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST)
+#endif
+
+/*  22: Extmem functions */
+#define PVRSRV_BRIDGE_MMEXTMEM				22UL
+#if defined(SUPPORT_WRAP_EXTMEM)
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST (PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST  (PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST + PVRSRV_BRIDGE_MMEXTMEM_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST (PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST)
+#endif
+
+/*  23: Sync tracking functions */
+#define PVRSRV_BRIDGE_SYNCTRACKING			23UL
+#define PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST (PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNCTRACKING_CMD_LAST)
+
+/*  24: Sync fallback functions */
+#define PVRSRV_BRIDGE_SYNCFALLBACK			24UL
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST + PVRSRV_BRIDGE_SYNCFALLBACK_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST)
+#endif
+
+/*  25: Debug Information (DI) interface functions */
+#define PVRSRV_BRIDGE_DI					25UL
+#if defined(SUPPORT_DI_BRG_IMPL)
+#define PVRSRV_BRIDGE_DI_DISPATCH_FIRST (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DI_DISPATCH_LAST  (PVRSRV_BRIDGE_DI_DISPATCH_FIRST + PVRSRV_BRIDGE_DI_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DI_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DI_DISPATCH_LAST  (PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST)
+#endif
+
+/*  26: DMA transfer functions */
+
+#define PVRSRV_BRIDGE_DMA			26UL
+#if defined(SUPPORT_DMA_TRANSFER)
+#define PVRSRV_BRIDGE_DMA_DISPATCH_FIRST (PVRSRV_BRIDGE_DI_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_DMA_DISPATCH_LAST  (PVRSRV_BRIDGE_DMA_DISPATCH_FIRST + PVRSRV_BRIDGE_DMA_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_DMA_DISPATCH_FIRST 0
+#define PVRSRV_BRIDGE_DMA_DISPATCH_LAST  (PVRSRV_BRIDGE_DI_DISPATCH_LAST)
+#endif
+
+/* NB PVRSRV_BRIDGE_LAST below must be the last bridge group defined above (PVRSRV_BRIDGE_FEATURE) */
+#define PVRSRV_BRIDGE_LAST					(PVRSRV_BRIDGE_DMA)
+/* NB PVRSRV_BRIDGE_DISPATCH LAST below must be the last dispatch entry defined above (PVRSRV_BRIDGE_FEATURE_DISPATCH_LAST) */
+#define PVRSRV_BRIDGE_DISPATCH_LAST			(PVRSRV_BRIDGE_DMA_DISPATCH_LAST)
+
+/* bit mask representing the enabled PVR bridges */
+
+static const IMG_UINT32 gui32PVRBridges =
+	  (1U << (PVRSRV_BRIDGE_DEFAULT - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_SRVCORE - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_SYNC - PVRSRV_BRIDGE_FIRST))
+
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_PDUMPCTRL - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_MM - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_MMPLAT_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_MMPLAT - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_CMM)
+	| (1U << (PVRSRV_BRIDGE_CMM - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_PDUMPMM - PVRSRV_BRIDGE_FIRST))
+	| (1U << (PVRSRV_BRIDGE_PDUMP - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(__linux__)
+	| (1U << (PVRSRV_BRIDGE_DMABUF - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+	| (1U << (PVRSRV_BRIDGE_DC - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_CACHE - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_SECURE_EXPORT)
+	| (1U << (PVRSRV_BRIDGE_SMM - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_PVRTL - PVRSRV_BRIDGE_FIRST))
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	| (1U << (PVRSRV_BRIDGE_RI - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_DEVICEMEMHISTORY - PVRSRV_BRIDGE_FIRST))
+#if defined(PVRSRV_ENABLE_HTB)
+	| (1U << (PVRSRV_BRIDGE_HTBUFFER - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS) && defined(SUPPORT_DCPLAT_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_DCPLAT - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_WRAP_EXTMEM)
+	| (1U << (PVRSRV_BRIDGE_MMEXTMEM - PVRSRV_BRIDGE_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_SYNCTRACKING - PVRSRV_BRIDGE_FIRST))
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	| (1U << (PVRSRV_BRIDGE_SYNCFALLBACK - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DI_BRG_IMPL)
+	| (1U << (PVRSRV_BRIDGE_DI - PVRSRV_BRIDGE_FIRST))
+#endif
+#if defined(SUPPORT_DMA_TRANSFER)
+	| (1U << (PVRSRV_BRIDGE_DMA - PVRSRV_BRIDGE_FIRST))
+#endif
+	;
+
+/* bit field representing which PVR bridge groups may optionally not
+ * be present in the server
+ */
+#define PVR_BRIDGES_OPTIONAL \
+	( \
+		(1U << (PVRSRV_BRIDGE_RI - PVRSRV_BRIDGE_FIRST)) \
+	)
+
+/******************************************************************************
+ * Generic bridge structures
+ *****************************************************************************/
+
+
+/******************************************************************************
+ * bridge packaging structure
+ *****************************************************************************/
+typedef struct PVRSRV_BRIDGE_PACKAGE_TAG
+{
+	IMG_UINT32		ui32BridgeID;			/*!< ioctl bridge group */
+	IMG_UINT32		ui32FunctionID;			/*!< ioctl function index */
+	IMG_UINT32		ui32Size;				/*!< size of structure */
+	void __user		*pvParamIn;				/*!< input data buffer */
+	IMG_UINT32		ui32InBufferSize;		/*!< size of input data buffer */
+	void __user		*pvParamOut;			/*!< output data buffer */
+	IMG_UINT32		ui32OutBufferSize;		/*!< size of output data buffer */
+}PVRSRV_BRIDGE_PACKAGE;
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_BRIDGE_H */
+
+/******************************************************************************
+ End of file (pvr_bridge.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pvr_bridge_k.c b/drivers/gpu/drm/img-rogue/pvr_bridge_k.c
new file mode 100644
index 000000000000..77739189b0aa
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_bridge_k.c
@@ -0,0 +1,643 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Receives calls from the user portion of services and
+                despatches them to functions in the kernel portion.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+
+#include <linux/mm_types.h>
+
+#include "img_defs.h"
+#include "pvr_bridge.h"
+#include "pvr_bridge_k.h"
+#include "connection_server.h"
+#include "syscommon.h"
+#include "pvr_debug.h"
+#include "di_server.h"
+#include "private_data.h"
+#include "linkage.h"
+#include "pmr.h"
+#include "rgx_bvnc_defs_km.h"
+#include "pvrsrv_bridge_init.h"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_print.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include "pvr_drm.h"
+#include "pvr_drv.h"
+
+#include "env_connection.h"
+#include <linux/sched.h>
+#include <linux/freezer.h>
+
+/* RGX: */
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+
+#include "srvcore.h"
+#include "common_srvcore_bridge.h"
+#include "kernel_compatibility.h"
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+#if defined(MODULE_TEST)
+/************************************************************************/
+// additional includes for services testing
+/************************************************************************/
+#include "pvr_test_bridge.h"
+#include "kern_test.h"
+/************************************************************************/
+// end of additional includes
+/************************************************************************/
+#endif
+
+/* The mmap code has its own mutex, to prevent possible re-entrant issues
+ * when the same PMR is mapped from two different connections/processes.
+ */
+static DEFINE_MUTEX(g_sMMapMutex);
+
+#define _SUSPENDED 1
+#define _NOT_SUSPENDED 0
+static ATOMIC_T g_iDriverSuspendCount;
+static ATOMIC_T g_iNumActiveDriverThreads;
+static ATOMIC_T g_iNumActiveKernelThreads;
+static IMG_HANDLE g_hDriverThreadEventObject;
+
+
+#if defined(DEBUG_BRIDGE_KM)
+static DI_ENTRY *gpsDIBridgeStatsEntry;
+
+static void *BridgeStatsDIStart(OSDI_IMPL_ENTRY *psEntry, IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psDispatchTable = DIGetPrivData(psEntry);
+
+	if (psDispatchTable == NULL || *pui64Pos > BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		return NULL;
+	}
+
+	if (*pui64Pos == 0)
+	{
+		return DI_START_TOKEN;
+	}
+
+	return &(psDispatchTable[*pui64Pos - 1]);
+}
+
+static void BridgeStatsDIStop(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(psEntry);
+	PVR_UNREFERENCED_PARAMETER(pvData);
+}
+
+static void *BridgeStatsDINext(OSDI_IMPL_ENTRY *psEntry, void *pvData,
+                               IMG_UINT64 *pui64Pos)
+{
+	PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psDispatchTable = DIGetPrivData(psEntry);
+	IMG_UINT64 uiItemAskedFor = *pui64Pos; /* pui64Pos on entry is the index to return */
+
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	/* Is the item asked for (starts at 0) a valid table index? */
+	if (uiItemAskedFor < BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		(*pui64Pos)++; /* on exit it is the next DI index to ask for */
+		return &(psDispatchTable[uiItemAskedFor]);
+	}
+
+	/* Now passed the end of the table to indicate stop */
+	return NULL;
+}
+
+static int BridgeStatsDIShow(OSDI_IMPL_ENTRY *psEntry, void *pvData)
+{
+	if (pvData == DI_START_TOKEN)
+	{
+		BridgeGlobalStatsLock();
+		DIPrintf(psEntry,
+		         "Total ioctl call count = %u\n"
+		         "Total number of bytes copied via copy_from_user = %u\n"
+		         "Total number of bytes copied via copy_to_user = %u\n"
+		         "Total number of bytes copied via copy_*_user = %u\n\n"
+		         "%3s: %-60s | %-48s | %10s | %20s | %20s | %20s | %20s\n",
+		         g_BridgeGlobalStats.ui32IOCTLCount,
+		         g_BridgeGlobalStats.ui32TotalCopyFromUserBytes,
+		         g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		         g_BridgeGlobalStats.ui32TotalCopyFromUserBytes +
+		             g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		         "#",
+		         "Bridge Name",
+		         "Wrapper Function",
+		         "Call Count",
+		         "copy_from_user (B)",
+		         "copy_to_user (B)",
+		         "Total Time (us)",
+		         "Max Time (us)");
+		BridgeGlobalStatsUnlock();
+	}
+	else if (pvData != NULL)
+	{
+		PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psTableEntry = pvData;
+		IMG_UINT32 ui32Remainder;
+
+		BridgeGlobalStatsLock();
+		DIPrintf(psEntry,
+		         "%3d: %-60s   %-48s   %-10u   %-20u   %-20u   %-20" IMG_UINT64_FMTSPEC "   %-20" IMG_UINT64_FMTSPEC "\n",
+		         (IMG_UINT32)(((size_t)psTableEntry-(size_t)g_BridgeDispatchTable)/sizeof(*g_BridgeDispatchTable)),
+		         psTableEntry->pszIOCName,
+		         (psTableEntry->pfFunction != NULL) ? psTableEntry->pszFunctionName : "(null)",
+		         psTableEntry->ui32CallCount,
+		         psTableEntry->ui32CopyFromUserTotalBytes,
+		         psTableEntry->ui32CopyToUserTotalBytes,
+		         OSDivide64r64(psTableEntry->ui64TotalTimeNS, 1000, &ui32Remainder),
+		         OSDivide64r64(psTableEntry->ui64MaxTimeNS, 1000, &ui32Remainder));
+		BridgeGlobalStatsUnlock();
+	}
+
+	return 0;
+}
+
+static IMG_INT64 BridgeStatsWrite(const IMG_CHAR *pcBuffer,
+                                  IMG_UINT64 ui64Count, IMG_UINT64 *pui64Pos,
+                                  void *pvData)
+{
+	IMG_UINT32 i;
+
+	PVR_RETURN_IF_FALSE(pcBuffer != NULL, -EIO);
+	PVR_RETURN_IF_FALSE(pui64Pos != NULL && *pui64Pos == 0, -EIO);
+	PVR_RETURN_IF_FALSE(ui64Count >= 1, -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[0] == '0', -EINVAL);
+	PVR_RETURN_IF_FALSE(pcBuffer[ui64Count - 1] == '\0', -EINVAL);
+
+	/* Reset stats. */
+
+	BridgeGlobalStatsLock();
+
+	g_BridgeGlobalStats.ui32IOCTLCount = 0;
+	g_BridgeGlobalStats.ui32TotalCopyFromUserBytes = 0;
+	g_BridgeGlobalStats.ui32TotalCopyToUserBytes = 0;
+
+	for (i = 0; i < ARRAY_SIZE(g_BridgeDispatchTable); i++)
+	{
+		g_BridgeDispatchTable[i].ui32CallCount = 0;
+		g_BridgeDispatchTable[i].ui32CopyFromUserTotalBytes = 0;
+		g_BridgeDispatchTable[i].ui32CopyToUserTotalBytes = 0;
+		g_BridgeDispatchTable[i].ui64TotalTimeNS = 0;
+		g_BridgeDispatchTable[i].ui64MaxTimeNS = 0;
+	}
+
+	BridgeGlobalStatsUnlock();
+
+	return ui64Count;
+}
+
+#endif /* defined(DEBUG_BRIDGE_KM) */
+
+PVRSRV_ERROR LinuxGetThreadActivityStats(LINUX_THREAD_ACTIVITY_STATS *psThreadStats)
+{
+	PVR_RETURN_IF_FALSE(psThreadStats != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psThreadStats->i32KernelThreadCount = OSAtomicRead(&g_iNumActiveKernelThreads);
+	psThreadStats->i32DriverThreadCount = OSAtomicRead(&g_iNumActiveDriverThreads);
+	psThreadStats->i32SuspendedThreadCount = OSAtomicRead(&g_iDriverSuspendCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR OSPlatformBridgeInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = InitDMABUFBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDMABUFBridge");
+
+	OSAtomicWrite(&g_iDriverSuspendCount, 0);
+	OSAtomicWrite(&g_iNumActiveDriverThreads, 0);
+	OSAtomicWrite(&g_iNumActiveKernelThreads, 0);
+
+	eError = OSEventObjectCreate("Global driver thread event object",
+	                             &g_hDriverThreadEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", EventCreateError);
+
+#if defined(DEBUG_BRIDGE_KM)
+	{
+		DI_ITERATOR_CB sIter = {
+			.pfnStart = BridgeStatsDIStart,
+			.pfnStop = BridgeStatsDIStop,
+			.pfnNext = BridgeStatsDINext,
+			.pfnShow = BridgeStatsDIShow,
+			.pfnWrite = BridgeStatsWrite,
+
+			//Expects '0' + Null terminator
+			.ui32WriteLenMax = ((1U)+1U)
+		};
+
+		eError = DICreateEntry("bridge_stats", NULL, &sIter,
+		                       &g_BridgeDispatchTable[0],
+		                       DI_ENTRY_TYPE_GENERIC,
+		                       &gpsDIBridgeStatsEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DICreateEntry", DIEntryCreateError);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(DEBUG_BRIDGE_KM)
+DIEntryCreateError:
+#endif
+EventCreateError:
+	OSEventObjectDestroy(g_hDriverThreadEventObject);
+	g_hDriverThreadEventObject = NULL;
+
+	return eError;
+}
+
+void OSPlatformBridgeDeInit(void)
+{
+#if defined(DEBUG_BRIDGE_KM)
+	DIDestroyEntry(gpsDIBridgeStatsEntry);
+#endif
+
+	OSEventObjectDestroy(g_hDriverThreadEventObject);
+	g_hDriverThreadEventObject = NULL;
+
+	DeinitDMABUFBridge();
+}
+
+PVRSRV_ERROR LinuxBridgeBlockClientsAccess(struct pvr_drm_private *psDevPriv,
+                                           IMG_BOOL bShutdown)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+	__maybe_unused IMG_INT iSuspendCount;
+
+	eError = OSEventObjectOpen(g_hDriverThreadEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return eError;
+	}
+
+	iSuspendCount = OSAtomicIncrement(&g_iDriverSuspendCount);
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver suspended %d times.", __func__,
+	         iSuspendCount));
+
+	if (OSAtomicCompareExchange(&psDevPriv->suspended, _NOT_SUSPENDED,
+	                            _SUSPENDED) == _SUSPENDED)
+	{
+		OSAtomicDecrement(&g_iDriverSuspendCount);
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p already suspended", __func__,
+		        psDevPriv->dev_node));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto CloseEventObject;
+	}
+
+	/* now wait for any threads currently in the server to exit */
+	while (OSAtomicRead(&g_iNumActiveDriverThreads) != 0 ||
+		   (OSAtomicRead(&g_iNumActiveKernelThreads) != 0 && !bShutdown))
+	{
+		if (OSAtomicRead(&g_iNumActiveDriverThreads) != 0)
+		{
+			PVR_LOG(("%s: waiting for user threads (%d)", __func__,
+			        OSAtomicRead(&g_iNumActiveDriverThreads)));
+		}
+		if (OSAtomicRead(&g_iNumActiveKernelThreads) != 0)
+		{
+			PVR_LOG(("%s: waiting for kernel threads (%d)", __func__,
+			        OSAtomicRead(&g_iNumActiveKernelThreads)));
+		}
+		/* Regular wait is called here (and not OSEventObjectWaitKernel) because
+		 * this code is executed by the caller of .suspend/.shutdown callbacks
+		 * which is most likely PM (or other actor responsible for suspend
+		 * process). Because of that this thread shouldn't and most likely
+		 * event cannot be frozen. */
+		OSEventObjectWait(hEvent);
+	}
+
+CloseEventObject:
+	OSEventObjectClose(hEvent);
+
+	return eError;
+}
+
+PVRSRV_ERROR LinuxBridgeUnblockClientsAccess(struct pvr_drm_private *psDevPriv)
+{
+	PVRSRV_ERROR eError;
+	__maybe_unused IMG_INT iSuspendCount;
+
+	/* resume the driver and then signal so any waiting threads wake up */
+	if (OSAtomicCompareExchange(&psDevPriv->suspended, _SUSPENDED,
+	                            _NOT_SUSPENDED) == _NOT_SUSPENDED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device is not suspended", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	iSuspendCount = OSAtomicDecrement(&g_iDriverSuspendCount);
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Driver suspended %d times.", __func__,
+	         iSuspendCount));
+
+	eError = OSEventObjectSignal(g_hDriverThreadEventObject);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OSEventObjectSignal failed: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+	}
+
+	return eError;
+}
+
+static PVRSRV_ERROR LinuxBridgeSignalIfSuspended(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (OSAtomicRead(&g_iDriverSuspendCount) > 0)
+	{
+		PVRSRV_ERROR eError = OSEventObjectSignal(g_hDriverThreadEventObject);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal driver thread event"
+			        " object: %s", __func__, PVRSRVGetErrorString(eError)));
+		}
+	}
+
+	return eError;
+}
+
+void LinuxBridgeNumActiveKernelThreadsIncrement(void)
+{
+	OSAtomicIncrement(&g_iNumActiveKernelThreads);
+}
+
+void LinuxBridgeNumActiveKernelThreadsDecrement(void)
+{
+	OSAtomicDecrement(&g_iNumActiveKernelThreads);
+	PVR_ASSERT(OSAtomicRead(&g_iNumActiveKernelThreads) >= 0);
+
+	/* Signal on every decrement in case LinuxBridgeBlockClientsAccess() is
+	 * waiting for the threads to freeze.
+	 * (error is logged in called function so ignore, we can't do much with
+	 * it anyway) */
+	(void) LinuxBridgeSignalIfSuspended();
+}
+
+static PVRSRV_ERROR _WaitForDriverUnsuspend(void)
+{
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hEvent;
+
+	eError = OSEventObjectOpen(g_hDriverThreadEventObject, &hEvent);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to open event object", __func__));
+		return eError;
+	}
+
+	while (OSAtomicRead(&g_iDriverSuspendCount) == 0)
+	{
+		/* we should be able to use normal (not kernel) wait here since
+		 * we were just unfrozen and most likely we're not going to
+		 * be frozen again (?) */
+		OSEventObjectWait(hEvent);
+	}
+
+	OSEventObjectClose(hEvent);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVDriverThreadEnter(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	CONNECTION_DATA *psConnection = (CONNECTION_DATA *)pvData;
+
+	/* Block if the associated device has been placed into a FROZEN state.
+	 * In this case we must await a PVRSRVDeviceThaw() completion request.
+	 * Device is obtained from the incoming psConnection if pvData is non-NULL.
+	 */
+	if (likely(pvData != NULL))
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+		PVRSRVBlockIfFrozen(psDevNode);
+		OSAtomicIncrement(&psDevNode->iThreadsActive);
+	}
+
+	/* increment first so there is no race between this value and
+	 * g_iDriverSuspendCount in LinuxBridgeBlockClientsAccess() */
+	OSAtomicIncrement(&g_iNumActiveDriverThreads);
+
+	if (OSAtomicRead(&g_iDriverSuspendCount) > 0)
+	{
+		/* decrement here because the driver is going to be suspended and
+		 * this thread is going to be frozen so we don't want to wait for
+		 * it in LinuxBridgeBlockClientsAccess() */
+		OSAtomicDecrement(&g_iNumActiveDriverThreads);
+
+		/* during suspend procedure this will put the current thread to
+		 * the freezer but during shutdown this will just return */
+		try_to_freeze();
+
+		/* if the thread was unfrozen but the number of suspends is non-0 wait
+		 * for it
+		 * in case this is a shutdown the thread was not frozen so we'll
+		 * wait here indefinitely but this is ok (and this is in fact what
+		 * we want) because no thread should be entering the driver in such
+		 * case */
+		eError = _WaitForDriverUnsuspend();
+
+		/* increment here because that means that the thread entered the
+		 * driver */
+		OSAtomicIncrement(&g_iNumActiveDriverThreads);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to wait for driver"
+			        " unsuspend: %s", __func__,
+			        PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVDriverThreadExit(void *pvData)
+{
+	CONNECTION_DATA *psConnection = (CONNECTION_DATA *)pvData;
+	OSAtomicDecrement(&g_iNumActiveDriverThreads);
+
+	/* Decrement the number of threads active on this device if the
+	 * connection is known.
+	 */
+	if (psConnection != NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDevNode = OSGetDevNode(psConnection);
+		OSAtomicDecrement(&psDevNode->iThreadsActive);
+	}
+	/* if the driver is being suspended then we need to signal the
+	 * event object as the thread suspending the driver is waiting
+	 * for active threads to exit
+	 * error is logged in called function so ignore returned error
+	 */
+	(void) LinuxBridgeSignalIfSuspended();
+}
+
+int
+PVRSRV_BridgeDispatchKM(struct drm_device __maybe_unused *dev, void *arg, struct drm_file *pDRMFile)
+{
+	struct drm_pvr_srvkm_cmd *psSrvkmCmd = (struct drm_pvr_srvkm_cmd *) arg;
+	PVRSRV_BRIDGE_PACKAGE sBridgePackageKM = { 0 };
+	CONNECTION_DATA *psConnection = LinuxServicesConnectionFromFile(pDRMFile->filp);
+	PVRSRV_ERROR error;
+
+	if (psConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid connection data"));
+		return -EFAULT;
+	}
+
+	PVR_ASSERT(psSrvkmCmd != NULL);
+
+	DRM_DEBUG("tgid=%d, tgid_connection=%d, bridge_id=%d, func_id=%d",
+			  task_tgid_nr(current),
+			  ((ENV_CONNECTION_DATA *)PVRSRVConnectionPrivateData(psConnection))->owner,
+			  psSrvkmCmd->bridge_id,
+			  psSrvkmCmd->bridge_func_id);
+
+	error = PVRSRVDriverThreadEnter(psConnection);
+	PVR_LOG_GOTO_IF_ERROR(error, "PVRSRVDriverThreadEnter", e0);
+
+	sBridgePackageKM.ui32BridgeID = psSrvkmCmd->bridge_id;
+	sBridgePackageKM.ui32FunctionID = psSrvkmCmd->bridge_func_id;
+	sBridgePackageKM.ui32Size = sizeof(sBridgePackageKM);
+	sBridgePackageKM.pvParamIn = (void __user *)(uintptr_t)psSrvkmCmd->in_data_ptr;
+	sBridgePackageKM.ui32InBufferSize = psSrvkmCmd->in_data_size;
+	sBridgePackageKM.pvParamOut = (void __user *)(uintptr_t)psSrvkmCmd->out_data_ptr;
+	sBridgePackageKM.ui32OutBufferSize = psSrvkmCmd->out_data_size;
+
+	error = BridgedDispatchKM(psConnection, &sBridgePackageKM);
+
+e0:
+	PVRSRVDriverThreadExit(psConnection);
+
+	return OSPVRSRVToNativeError(error);
+}
+
+int
+PVRSRV_MMap(struct file *pFile, struct vm_area_struct *ps_vma)
+{
+	CONNECTION_DATA *psConnection = LinuxServicesConnectionFromFile(pFile);
+	IMG_HANDLE hSecurePMRHandle = (IMG_HANDLE)((uintptr_t)ps_vma->vm_pgoff);
+	PMR *psPMR;
+	PVRSRV_ERROR eError;
+	PVRSRV_MEMALLOCFLAGS_T uiProtFlags =
+	    (BITMASK_HAS(ps_vma->vm_flags, VM_READ) ? PVRSRV_MEMALLOCFLAG_CPU_READABLE : 0) |
+	    (BITMASK_HAS(ps_vma->vm_flags, VM_WRITE) ? PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0);
+
+	if (psConnection == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid connection data"));
+		return -ENOENT;
+	}
+
+	eError = PVRSRVDriverThreadEnter(psConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVDriverThreadEnter", e0);
+
+	/*
+	 * The bridge lock used here to protect PVRSRVLookupHandle is replaced
+	 * by a specific lock considering that the handle functions have now
+	 * their own lock. This change was necessary to solve the lockdep issues
+	 * related with the PVRSRV_MMap.
+	 */
+
+	eError = PVRSRVLookupHandle(psConnection->psHandleBase,
+								(void **)&psPMR,
+								hSecurePMRHandle,
+								PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								IMG_TRUE);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	mutex_lock(&g_sMMapMutex);
+
+	/* Forcibly clear the VM_MAYWRITE flag as this is inherited from the
+	 * kernel mmap code and we do not want to produce a potentially writable
+	 * mapping from a read-only mapping.
+	 */
+	if (!BITMASK_HAS(ps_vma->vm_flags, VM_WRITE))
+	{
+		pvr_vm_flags_clear(ps_vma, VM_MAYWRITE);
+	}
+
+	/* Note: PMRMMapPMR will take a reference on the PMR.
+	 * Unref the handle immediately, because we have now done
+	 * the required operation on the PMR (whether it succeeded or not)
+	 */
+	eError = PMRMMapPMR(psPMR, ps_vma, uiProtFlags);
+	mutex_unlock(&g_sMMapMutex);
+	PVRSRVReleaseHandle(psConnection->psHandleBase, hSecurePMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: PMRMMapPMR failed (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	PVRSRVDriverThreadExit(psConnection);
+
+	return 0;
+
+e0:
+	PVRSRVDriverThreadExit(psConnection);
+
+	PVR_DPF((PVR_DBG_ERROR, "Failed with error: %s", PVRSRVGetErrorString(eError)));
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return OSPVRSRVToNativeError(eError);
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_bridge_k.h b/drivers/gpu/drm/img-rogue/pvr_bridge_k.h
new file mode 100644
index 000000000000..2885dd465eaf
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_bridge_k.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Receives calls from the user portion of services and
+                despatches them to functions in the kernel portion.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BRIDGE_K_H
+#define PVR_BRIDGE_K_H
+
+#include "pvrsrv_error.h"
+#include "pvr_drv.h"
+
+typedef struct LINUX_THREAD_ACTIVITY_STATS
+{
+	IMG_INT32 i32KernelThreadCount;
+	IMG_INT32 i32DriverThreadCount;
+	IMG_INT32 i32SuspendedThreadCount;
+} LINUX_THREAD_ACTIVITY_STATS;
+
+/*!
+******************************************************************************
+ @Function      LinuxBridgeBlockClientsAccess
+ @Description   This function will wait for any existing threads in the Server
+                to exit and then disable access to the driver. New threads will
+                not be allowed to enter the Server until the driver is
+                unsuspended (see LinuxBridgeUnblockClientsAccess).
+ @Input         psDevPriv pointer to devices OS specific data
+ @Input         bShutdown this flag indicates that the function was called
+                          from a shutdown callback and therefore it will
+                          not wait for the kernel threads to get frozen
+                          (because this doesn't happen during shutdown
+                          procedure)
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR LinuxBridgeBlockClientsAccess(struct pvr_drm_private *psDevPriv,
+                                           IMG_BOOL bShutdown);
+
+/*!
+******************************************************************************
+ @Function      LinuxBridgeUnblockClientsAccess
+ @Description   This function will re-enable the bridge and allow any threads
+                waiting to enter the Server to continue.
+ @Input         psDevPriv pointer to devices OS specific data
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR LinuxBridgeUnblockClientsAccess(struct pvr_drm_private *psDevPriv);
+
+void LinuxBridgeNumActiveKernelThreadsIncrement(void);
+void LinuxBridgeNumActiveKernelThreadsDecrement(void);
+
+/*************************************************************************/ /*!
+ @Function      LinuxGetThreadActivityStats
+ @Description   Getter for active and suspended thread stats.
+
+ @Output        psThreadStats   Struct to be populated with thread activity
+                                stats.
+
+ @Return        PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR LinuxGetThreadActivityStats(LINUX_THREAD_ACTIVITY_STATS *psThreadStats);
+
+/*!
+******************************************************************************
+ @Function      PVRSRVDriverThreadEnter
+ @Description   Increments number of client threads currently operating
+                in the driver's context.
+                If the driver is currently being suspended this function
+                will call try_to_freeze() on behalf of the client thread.
+                When the driver is resumed the function will exit and allow
+                the thread into the driver.
+ @Input         Reference to Connection data. NULL if no associated
+                 connection / device.
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDriverThreadEnter(void *pvData);
+
+/*!
+******************************************************************************
+ @Function      PVRSRVDriverThreadExit
+ @Description   Decrements the number of client threads currently operating
+                in the driver's context to match the call to
+                PVRSRVDriverThreadEnter().
+                The function also signals the driver that a thread left the
+                driver context so if it's waiting to suspend it knows that
+                the number of threads decreased.
+ @Input         Reference to Connection data. NULL if no associated
+                 connection / device.
+******************************************************************************/
+void PVRSRVDriverThreadExit(void *pvData);
+
+#endif /* PVR_BRIDGE_K_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_buffer_sync.c b/drivers/gpu/drm/img-rogue/pvr_buffer_sync.c
new file mode 100644
index 000000000000..529cafd2bcf3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_buffer_sync.c
@@ -0,0 +1,733 @@
+/*
+ * @File
+ * @Title       Linux buffer sync interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/dma-buf.h>
+
+#include "services_kernel_client.h"
+#include "pvr_dma_resv.h"
+#include "pvr_buffer_sync.h"
+#include "pvr_buffer_sync_shared.h"
+#include "pvr_drv.h"
+#include "pvr_fence.h"
+
+struct pvr_buffer_sync_context {
+	struct mutex ctx_lock;
+	struct pvr_fence_context *fence_ctx;
+	struct ww_acquire_ctx acquire_ctx;
+};
+
+struct pvr_buffer_sync_check_data {
+	struct dma_fence_cb base;
+
+	u32 nr_fences;
+	struct pvr_fence **fences;
+};
+
+struct pvr_buffer_sync_append_data {
+	struct pvr_buffer_sync_context *ctx;
+
+	u32 nr_pmrs;
+	struct _PMR_ **pmrs;
+	u32 *pmr_flags;
+
+	struct pvr_fence *update_fence;
+	struct pvr_buffer_sync_check_data *check_data;
+};
+
+static struct dma_resv *
+pmr_reservation_object_get(struct _PMR_ *pmr)
+{
+	return PhysmemGetDmaResv(pmr);
+}
+
+static int
+pvr_buffer_sync_pmrs_lock(struct pvr_buffer_sync_context *ctx,
+			  u32 nr_pmrs,
+			  struct _PMR_ **pmrs)
+{
+	struct dma_resv *resv, *cresv = NULL, *lresv = NULL;
+	int i, err;
+	struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx;
+
+	mutex_lock(&ctx->ctx_lock);
+
+	ww_acquire_init(acquire_ctx, &reservation_ww_class);
+retry:
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (!resv) {
+			pr_err("%s: Failed to get reservation object from pmr %p\n",
+			       __func__, pmrs[i]);
+			err = -EINVAL;
+			goto fail;
+		}
+
+		if (resv != lresv) {
+			err = ww_mutex_lock_interruptible(&resv->lock,
+							  acquire_ctx);
+			if (err) {
+				cresv = (err == -EDEADLK) ? resv : NULL;
+				goto fail;
+			}
+		} else {
+			lresv = NULL;
+		}
+	}
+
+	ww_acquire_done(acquire_ctx);
+
+	return 0;
+
+fail:
+	while (i--) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+		ww_mutex_unlock(&resv->lock);
+	}
+
+	if (lresv)
+		ww_mutex_unlock(&lresv->lock);
+
+	if (cresv) {
+		err = ww_mutex_lock_slow_interruptible(&cresv->lock,
+						       acquire_ctx);
+		if (!err) {
+			lresv = cresv;
+			cresv = NULL;
+			goto retry;
+		}
+	}
+
+	ww_acquire_fini(acquire_ctx);
+
+	mutex_unlock(&ctx->ctx_lock);
+	return err;
+}
+
+static void
+pvr_buffer_sync_pmrs_unlock(struct pvr_buffer_sync_context *ctx,
+			    u32 nr_pmrs,
+			    struct _PMR_ **pmrs)
+{
+	struct dma_resv *resv;
+	int i;
+	struct ww_acquire_ctx *acquire_ctx = &ctx->acquire_ctx;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+		ww_mutex_unlock(&resv->lock);
+	}
+
+	ww_acquire_fini(acquire_ctx);
+
+	mutex_unlock(&ctx->ctx_lock);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+
+static void
+dma_resv_count_fences(struct dma_resv *resv, u32 *read_fence_count_out, u32 *write_fence_count_out)
+{
+	struct dma_resv_iter cursor;
+	u32 write_fence_count = 0;
+	u32 read_fence_count = 0;
+	struct dma_fence *fence;
+
+	dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_READ);
+	dma_resv_for_each_fence_unlocked(&cursor, fence) {
+		if (dma_resv_iter_is_restarted(&cursor)) {
+			read_fence_count = 0;
+			write_fence_count = 0;
+		}
+		if (dma_resv_iter_usage(&cursor) == DMA_RESV_USAGE_READ)
+			read_fence_count++;
+		else if (dma_resv_iter_usage(&cursor) == DMA_RESV_USAGE_WRITE)
+			write_fence_count++;
+	}
+
+	*read_fence_count_out = read_fence_count;
+	*write_fence_count_out = write_fence_count;
+}
+
+static u32
+pvr_buffer_sync_pmrs_fence_count(u32 nr_pmrs, struct _PMR_ **pmrs,
+				 u32 *pmr_flags)
+{
+	struct dma_resv *resv;
+	u32 fence_count = 0;
+	bool exclusive;
+	int i;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		u32 write_fence_count = 0;
+		u32 read_fence_count = 0;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		dma_resv_count_fences(resv, &read_fence_count, &write_fence_count);
+
+		if (!exclusive || !read_fence_count)
+			fence_count += write_fence_count;
+		if (exclusive)
+			fence_count += read_fence_count;
+	}
+
+	return fence_count;
+}
+
+static struct pvr_buffer_sync_check_data *
+pvr_buffer_sync_check_fences_create(struct pvr_fence_context *fence_ctx,
+				    PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+				    u32 nr_pmrs,
+				    struct _PMR_ **pmrs,
+				    u32 *pmr_flags)
+{
+	struct pvr_buffer_sync_check_data *data;
+	struct dma_resv *resv;
+	struct dma_fence *fence;
+	u32 fence_count;
+	bool exclusive;
+	int i;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return NULL;
+
+	fence_count = pvr_buffer_sync_pmrs_fence_count(nr_pmrs, pmrs,
+						       pmr_flags);
+	if (fence_count) {
+		data->fences = kcalloc(fence_count, sizeof(*data->fences),
+				       GFP_KERNEL);
+		if (!data->fences)
+			goto err_check_data_free;
+	}
+
+	for (i = 0; i < nr_pmrs; i++) {
+		struct dma_resv_iter cursor;
+		bool include_write_fences;
+		bool include_read_fences;
+		u32 write_fence_count = 0;
+		u32 read_fence_count = 0;
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		dma_resv_count_fences(resv, &read_fence_count, &write_fence_count);
+
+		include_write_fences = (!exclusive || !read_fence_count);
+		include_read_fences = exclusive;
+
+		dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_READ);
+		dma_resv_for_each_fence_unlocked(&cursor, fence) {
+			enum dma_resv_usage usage = dma_resv_iter_usage(&cursor);
+
+			if ((!include_write_fences && usage == DMA_RESV_USAGE_WRITE) ||
+			    (!include_read_fences && usage == DMA_RESV_USAGE_READ))
+				continue;
+
+			data->fences[data->nr_fences++] =
+				pvr_fence_create_from_fence(fence_ctx,
+							    sync_checkpoint_ctx,
+							    fence,
+							    PVRSRV_NO_FENCE,
+							    (usage == DMA_RESV_USAGE_WRITE) ?
+							    "write check fence" :
+							    "read check fence");
+			if (!data->fences[data->nr_fences - 1]) {
+				data->nr_fences--;
+				PVR_FENCE_TRACE(fence,
+						(usage == DMA_RESV_USAGE_WRITE) ?
+						"waiting on write fence" :
+						"waiting on read fence\n");
+				WARN_ON(dma_fence_wait(fence, true) <= 0);
+			}
+		}
+	}
+
+	WARN_ON(i != nr_pmrs);
+
+	return data;
+
+err_check_data_free:
+	kfree(data);
+	return NULL;
+}
+
+#else
+
+static u32
+pvr_buffer_sync_pmrs_fence_count(u32 nr_pmrs, struct _PMR_ **pmrs,
+				 u32 *pmr_flags)
+{
+	struct dma_resv *resv;
+	struct dma_resv_list *resv_list;
+	struct dma_fence *fence;
+	u32 fence_count = 0;
+	bool exclusive;
+	int i;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		resv_list = dma_resv_shared_list(resv);
+		fence = dma_resv_excl_fence(resv);
+
+		if (fence &&
+		    (!exclusive || !resv_list || !resv_list->shared_count))
+			fence_count++;
+
+		if (exclusive && resv_list)
+			fence_count += resv_list->shared_count;
+	}
+
+	return fence_count;
+}
+
+static struct pvr_buffer_sync_check_data *
+pvr_buffer_sync_check_fences_create(struct pvr_fence_context *fence_ctx,
+				    PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+				    u32 nr_pmrs,
+				    struct _PMR_ **pmrs,
+				    u32 *pmr_flags)
+{
+	struct pvr_buffer_sync_check_data *data;
+	struct dma_resv *resv;
+	struct dma_resv_list *resv_list;
+	struct dma_fence *fence;
+	u32 fence_count;
+	bool exclusive;
+	int i, j;
+	int err;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return NULL;
+
+	fence_count = pvr_buffer_sync_pmrs_fence_count(nr_pmrs, pmrs,
+						       pmr_flags);
+	if (fence_count) {
+		data->fences = kcalloc(fence_count, sizeof(*data->fences),
+				       GFP_KERNEL);
+		if (!data->fences)
+			goto err_check_data_free;
+	}
+
+	for (i = 0; i < nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+		exclusive = !!(pmr_flags[i] & PVR_BUFFER_FLAG_WRITE);
+		if (!exclusive) {
+			err = dma_resv_reserve_shared(resv
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+						      , 1
+#endif
+				);
+			if (err)
+				goto err_destroy_fences;
+		}
+
+		resv_list = dma_resv_shared_list(resv);
+		fence = dma_resv_excl_fence(resv);
+
+		if (fence &&
+		    (!exclusive || !resv_list || !resv_list->shared_count)) {
+			data->fences[data->nr_fences++] =
+				pvr_fence_create_from_fence(fence_ctx,
+							    sync_checkpoint_ctx,
+							    fence,
+							    PVRSRV_NO_FENCE,
+							    "exclusive check fence");
+			if (!data->fences[data->nr_fences - 1]) {
+				data->nr_fences--;
+				PVR_FENCE_TRACE(fence,
+						"waiting on exclusive fence\n");
+				WARN_ON(dma_fence_wait(fence, true) <= 0);
+			}
+		}
+
+		if (exclusive && resv_list) {
+			for (j = 0; j < resv_list->shared_count; j++) {
+				fence = rcu_dereference_protected(resv_list->shared[j],
+								  dma_resv_held(resv));
+				data->fences[data->nr_fences++] =
+					pvr_fence_create_from_fence(fence_ctx,
+								    sync_checkpoint_ctx,
+								    fence,
+								    PVRSRV_NO_FENCE,
+								    "check fence");
+				if (!data->fences[data->nr_fences - 1]) {
+					data->nr_fences--;
+					PVR_FENCE_TRACE(fence,
+							"waiting on non-exclusive fence\n");
+					WARN_ON(dma_fence_wait(fence, true) <= 0);
+				}
+			}
+		}
+	}
+
+	WARN_ON((i != nr_pmrs) || (data->nr_fences != fence_count));
+
+	return data;
+
+err_destroy_fences:
+	for (i = 0; i < data->nr_fences; i++)
+		pvr_fence_destroy(data->fences[i]);
+	kfree(data->fences);
+err_check_data_free:
+	kfree(data);
+	return NULL;
+}
+
+#endif
+
+static void
+pvr_buffer_sync_check_fences_destroy(struct pvr_buffer_sync_check_data *data)
+{
+	int i;
+
+	for (i = 0; i < data->nr_fences; i++)
+		pvr_fence_destroy(data->fences[i]);
+
+	kfree(data->fences);
+	kfree(data);
+}
+
+struct pvr_buffer_sync_context *
+pvr_buffer_sync_context_create(struct device *dev, const char *name)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+	struct pvr_buffer_sync_context *ctx;
+	int err;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+
+	ctx->fence_ctx = pvr_fence_context_create(priv->dev_node,
+						  NativeSyncGetFenceStatusWq(),
+						  name);
+	if (!ctx->fence_ctx) {
+		err = -ENOMEM;
+		goto err_free_ctx;
+	}
+
+	mutex_init(&ctx->ctx_lock);
+
+	return ctx;
+
+err_free_ctx:
+	kfree(ctx);
+err_exit:
+	return ERR_PTR(err);
+}
+
+void
+pvr_buffer_sync_context_destroy(struct pvr_buffer_sync_context *ctx)
+{
+	pvr_fence_context_destroy(ctx->fence_ctx);
+	kfree(ctx);
+}
+
+int
+pvr_buffer_sync_resolve_and_create_fences(struct pvr_buffer_sync_context *ctx,
+					  PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+					  u32 nr_pmrs,
+					  struct _PMR_ **pmrs,
+					  u32 *pmr_flags,
+					  u32 *nr_fence_checkpoints_out,
+					  PSYNC_CHECKPOINT **fence_checkpoints_out,
+					  PSYNC_CHECKPOINT *update_checkpoints_out,
+					  struct pvr_buffer_sync_append_data **data_out)
+{
+	struct pvr_buffer_sync_append_data *data;
+	PSYNC_CHECKPOINT *fence_checkpoints;
+	const size_t data_size = sizeof(*data);
+	const size_t pmrs_size = sizeof(*pmrs) * nr_pmrs;
+	const size_t pmr_flags_size = sizeof(*pmr_flags) * nr_pmrs;
+	int i;
+	int j;
+	int err;
+
+	if (unlikely((nr_pmrs && !(pmrs && pmr_flags)) ||
+	    !nr_fence_checkpoints_out || !fence_checkpoints_out ||
+	    !update_checkpoints_out))
+		return -EINVAL;
+
+	for (i = 0; i < nr_pmrs; i++) {
+		if (unlikely(!(pmr_flags[i] & PVR_BUFFER_FLAG_MASK))) {
+			pr_err("%s: Invalid flags %#08x for pmr %p\n",
+			       __func__, pmr_flags[i], pmrs[i]);
+			return -EINVAL;
+		}
+	}
+
+#if defined(NO_HARDWARE)
+	/*
+	 * For NO_HARDWARE there's no checking or updating of sync checkpoints
+	 * which means SW waits on our fences will cause a deadlock (since they
+	 * will never be signalled). Avoid this by not creating any fences.
+	 */
+	nr_pmrs = 0;
+#endif
+
+	if (!nr_pmrs) {
+		*nr_fence_checkpoints_out = 0;
+		*fence_checkpoints_out = NULL;
+		*update_checkpoints_out = NULL;
+		*data_out = NULL;
+
+		return 0;
+	}
+
+	data = kzalloc(data_size + pmrs_size + pmr_flags_size, GFP_KERNEL);
+	if (unlikely(!data))
+		return -ENOMEM;
+
+	data->ctx = ctx;
+	data->pmrs = (struct _PMR_ **)(void *)(data + 1);
+	data->pmr_flags = (u32 *)(void *)(data->pmrs + nr_pmrs);
+
+	/*
+	 * It's expected that user space will provide a set of unique PMRs
+	 * but, as a PMR can have multiple handles, it's still possible to
+	 * end up here with duplicates. Take this opportunity to filter out
+	 * any remaining duplicates (updating flags when necessary) before
+	 * trying to process them further.
+	 */
+	for (i = 0; i < nr_pmrs; i++) {
+		for (j = 0; j < data->nr_pmrs; j++) {
+			if (data->pmrs[j] == pmrs[i]) {
+				data->pmr_flags[j] |= pmr_flags[i];
+				break;
+			}
+		}
+
+		if (j == data->nr_pmrs) {
+			data->pmrs[j] = pmrs[i];
+			data->pmr_flags[j] = pmr_flags[i];
+			data->nr_pmrs++;
+		}
+	}
+
+	err = pvr_buffer_sync_pmrs_lock(ctx, data->nr_pmrs, data->pmrs);
+	if (unlikely(err)) {
+		/*
+		 * -EINTR is returned if a signal arrives while trying to acquire a PMR
+		 * lock. In this case the operation should be retried after the signal
+		 * has been serviced. As this is expected behaviour, don't print an
+		 * error in this case.
+		 */
+		if (err != -EINTR) {
+			pr_err("%s: failed to lock pmrs (errno=%d)\n",
+			       __func__, err);
+		}
+		goto err_free_data;
+	}
+
+	/* create the check data */
+	data->check_data = pvr_buffer_sync_check_fences_create(ctx->fence_ctx,
+							 sync_checkpoint_ctx,
+							 data->nr_pmrs,
+							 data->pmrs,
+							 data->pmr_flags);
+	if (unlikely(!data->check_data)) {
+		err = -ENOMEM;
+		goto err_pmrs_unlock;
+	}
+
+	fence_checkpoints = kcalloc(data->check_data->nr_fences,
+				    sizeof(*fence_checkpoints),
+				    GFP_KERNEL);
+	if (fence_checkpoints) {
+		pvr_fence_get_checkpoints(data->check_data->fences,
+					  data->check_data->nr_fences,
+					  fence_checkpoints);
+	} else {
+		if (unlikely(data->check_data->nr_fences)) {
+			err = -ENOMEM;
+			goto err_free_check_data;
+		}
+	}
+
+	/* create the update fence */
+	data->update_fence = pvr_fence_create(ctx->fence_ctx,
+			sync_checkpoint_ctx,
+			SYNC_CHECKPOINT_FOREIGN_CHECKPOINT, "update fence");
+	if (unlikely(!data->update_fence)) {
+		err = -ENOMEM;
+		goto err_free_fence_checkpoints;
+	}
+
+	/*
+	 * We need to clean up the fences once the HW has finished with them.
+	 * We can do this using fence callbacks. However, instead of adding a
+	 * callback to every fence, which would result in more work, we can
+	 * simply add one to the update fence since this will be the last fence
+	 * to be signalled. This callback can do all the necessary clean up.
+	 *
+	 * Note: we take an additional reference on the update fence in case
+	 * it signals before we can add it to a reservation object.
+	 */
+	PVR_FENCE_TRACE(&data->update_fence->base,
+			"create fence calling dma_fence_get\n");
+	dma_fence_get(&data->update_fence->base);
+
+	*nr_fence_checkpoints_out = data->check_data->nr_fences;
+	*fence_checkpoints_out = fence_checkpoints;
+	*update_checkpoints_out = pvr_fence_get_checkpoint(data->update_fence);
+	*data_out = data;
+
+	return 0;
+
+err_free_fence_checkpoints:
+	kfree(fence_checkpoints);
+err_free_check_data:
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+err_pmrs_unlock:
+	pvr_buffer_sync_pmrs_unlock(ctx, data->nr_pmrs, data->pmrs);
+err_free_data:
+	kfree(data);
+	return err;
+}
+
+void
+pvr_buffer_sync_kick_succeeded(struct pvr_buffer_sync_append_data *data)
+{
+	struct dma_resv *resv;
+	int i;
+
+	dma_fence_enable_sw_signaling(&data->update_fence->base);
+
+	for (i = 0; i < data->nr_pmrs; i++) {
+		resv = pmr_reservation_object_get(data->pmrs[i]);
+		if (WARN_ON_ONCE(!resv))
+			continue;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+		dma_resv_reserve_fences(resv, 1);
+#endif
+		if (data->pmr_flags[i] & PVR_BUFFER_FLAG_WRITE) {
+			PVR_FENCE_TRACE(&data->update_fence->base,
+					"added exclusive fence (%s) to resv %p\n",
+					data->update_fence->name, resv);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+			dma_resv_add_fence(resv,
+					   &data->update_fence->base,
+					   DMA_RESV_USAGE_WRITE);
+#else
+			dma_resv_add_excl_fence(resv,
+						&data->update_fence->base);
+#endif
+		} else if (data->pmr_flags[i] & PVR_BUFFER_FLAG_READ) {
+			PVR_FENCE_TRACE(&data->update_fence->base,
+					"added non-exclusive fence (%s) to resv %p\n",
+					data->update_fence->name, resv);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 0))
+			dma_resv_add_fence(resv,
+					   &data->update_fence->base,
+					   DMA_RESV_USAGE_READ);
+#else
+			dma_resv_add_shared_fence(resv,
+						  &data->update_fence->base);
+#endif
+		}
+	}
+
+	/*
+	 * Now that the fence has been added to the necessary
+	 * reservation objects we can safely drop the extra reference
+	 * we took in pvr_buffer_sync_resolve_and_create_fences().
+	 */
+	dma_fence_put(&data->update_fence->base);
+	pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs,
+					data->pmrs);
+
+	/* destroy the check fences */
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+	/* destroy the update fence */
+	pvr_fence_destroy(data->update_fence);
+
+	/* free the append data */
+	kfree(data);
+}
+
+void
+pvr_buffer_sync_kick_failed(struct pvr_buffer_sync_append_data *data)
+{
+
+	/* drop the extra reference we took on the update fence in
+	 * pvr_buffer_sync_resolve_and_create_fences().
+	 */
+	dma_fence_put(&data->update_fence->base);
+
+	if (data->nr_pmrs > 0)
+		pvr_buffer_sync_pmrs_unlock(data->ctx, data->nr_pmrs,
+					    data->pmrs);
+
+	/* destroy the check fences */
+	pvr_buffer_sync_check_fences_destroy(data->check_data);
+	/* destroy the update fence */
+	pvr_fence_destroy(data->update_fence);
+
+	/* free the append data */
+	kfree(data);
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_buffer_sync.h b/drivers/gpu/drm/img-rogue/pvr_buffer_sync.h
new file mode 100644
index 000000000000..b6aadf940479
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_buffer_sync.h
@@ -0,0 +1,125 @@
+/*
+ * @File        pvr_buffer_sync.h
+ * @Title       PowerVR Linux buffer sync interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PVR_BUFFER_SYNC_H
+#define PVR_BUFFER_SYNC_H
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/types.h>
+
+struct _PMR_;
+struct pvr_buffer_sync_context;
+struct pvr_buffer_sync_append_data;
+
+/**
+ * pvr_buffer_sync_context_create - creates a buffer sync context
+ * @dev: Linux device
+ * @name: context name (used for debugging)
+ *
+ * pvr_buffer_sync_context_destroy() should be used to clean up the buffer
+ * sync context.
+ *
+ * Return: A buffer sync context or NULL if it fails for any reason.
+ */
+struct pvr_buffer_sync_context *
+pvr_buffer_sync_context_create(struct device *dev, const char *name);
+
+/**
+ * pvr_buffer_sync_context_destroy() - frees a buffer sync context
+ * @ctx: buffer sync context
+ */
+void
+pvr_buffer_sync_context_destroy(struct pvr_buffer_sync_context *ctx);
+
+/**
+ * pvr_buffer_sync_resolve_and_create_fences() - create checkpoints from
+ *                                               buffers
+ * @ctx: buffer sync context
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @nr_pmrs: number of buffer objects (PMRs)
+ * @pmrs: buffer array
+ * @pmr_flags: internal flags
+ * @nr_fence_checkpoints_out: returned number of fence sync checkpoints
+ * @fence_checkpoints_out: returned array of fence sync checkpoints
+ * @update_checkpoint_out: returned update sync checkpoint
+ * @data_out: returned buffer sync data
+ *
+ * After this call, either pvr_buffer_sync_kick_succeeded() or
+ * pvr_buffer_sync_kick_failed() must be called.
+ *
+ * Return: 0 on success or an error code otherwise.
+ */
+int
+pvr_buffer_sync_resolve_and_create_fences(struct pvr_buffer_sync_context *ctx,
+					  PSYNC_CHECKPOINT_CONTEXT sync_checkpoint_ctx,
+					  u32 nr_pmrs,
+					  struct _PMR_ **pmrs,
+					  u32 *pmr_flags,
+					  u32 *nr_fence_checkpoints_out,
+					  PSYNC_CHECKPOINT **fence_checkpoints_out,
+					  PSYNC_CHECKPOINT *update_checkpoint_out,
+					  struct pvr_buffer_sync_append_data **data_out);
+
+/**
+ * pvr_buffer_sync_kick_succeeded() - cleans up after a successful kick
+ *                                    operation
+ * @data: buffer sync data returned by
+ *        pvr_buffer_sync_resolve_and_create_fences()
+ *
+ * Should only be called following pvr_buffer_sync_resolve_and_create_fences().
+ */
+void
+pvr_buffer_sync_kick_succeeded(struct pvr_buffer_sync_append_data *data);
+
+/**
+ * pvr_buffer_sync_kick_failed() - cleans up after a failed kick operation
+ * @data: buffer sync data returned by
+ *        pvr_buffer_sync_resolve_and_create_fences()
+ *
+ * Should only be called following pvr_buffer_sync_resolve_and_create_fences().
+ */
+void
+pvr_buffer_sync_kick_failed(struct pvr_buffer_sync_append_data *data);
+
+#endif /* PVR_BUFFER_SYNC_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_buffer_sync_shared.h b/drivers/gpu/drm/img-rogue/pvr_buffer_sync_shared.h
new file mode 100644
index 000000000000..7a110910dbd6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_buffer_sync_shared.h
@@ -0,0 +1,57 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR buffer sync shared
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared definitions between client and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_BUFFER_SYNC_SHARED_H
+#define PVR_BUFFER_SYNC_SHARED_H
+
+#define PVR_BUFFER_FLAG_READ		(1U << 0)
+#define PVR_BUFFER_FLAG_WRITE		(1U << 1)
+#define PVR_BUFFER_FLAG_MASK		(PVR_BUFFER_FLAG_READ | \
+									 PVR_BUFFER_FLAG_WRITE)
+
+/* Maximum number of PMRs passed
+ * in a kick when using buffer sync
+ */
+#define PVRSRV_MAX_BUFFERSYNC_PMRS 32
+
+#endif /* PVR_BUFFER_SYNC_SHARED_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_counting_timeline.c b/drivers/gpu/drm/img-rogue/pvr_counting_timeline.c
new file mode 100644
index 000000000000..3d2be5077c88
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_counting_timeline.c
@@ -0,0 +1,313 @@
+/*
+ * @File
+ * @Title       PowerVR Linux software "counting" timeline fence implementation
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/kref.h>
+
+#include "services_kernel_client.h"
+#include "pvr_counting_timeline.h"
+#include "pvr_sw_fence.h"
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+struct pvr_counting_fence_timeline {
+	struct pvr_sw_fence_context *context;
+
+	void *dbg_request_handle;
+
+	spinlock_t active_fences_lock;
+	u64 current_value; /* guarded by active_fences_lock */
+	u64 next_value; /* guarded by active_fences_lock */
+	struct list_head active_fences;
+
+	struct kref kref;
+};
+
+struct pvr_counting_fence {
+	u64 value;
+	struct dma_fence *fence;
+	struct list_head active_list_entry;
+};
+
+void pvr_counting_fence_timeline_dump_timeline(
+	void *data,
+	DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+	void *dump_debug_file)
+{
+
+	struct pvr_counting_fence_timeline *timeline =
+		(struct pvr_counting_fence_timeline *) data;
+	unsigned long flags;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "TL:%s SeqNum: %llu/%llu",
+					  pvr_sw_fence_context_name(
+							  timeline->context),
+					  timeline->current_value,
+					  timeline->next_value);
+
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+}
+
+static void
+pvr_counting_fence_timeline_debug_request(void *data, u32 verbosity,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	struct pvr_counting_fence_timeline *timeline =
+		(struct pvr_counting_fence_timeline *)data;
+	struct pvr_counting_fence *obj;
+	unsigned long flags;
+	char value[128];
+
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM)) {
+		spin_lock_irqsave(&timeline->active_fences_lock, flags);
+		pvr_sw_fence_context_value_str(timeline->context, value,
+					       sizeof(value));
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  "sw: %s @%s cur=%llu",
+				  pvr_sw_fence_context_name(timeline->context),
+				  value, timeline->current_value);
+		list_for_each_entry(obj, &timeline->active_fences,
+				    active_list_entry) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+			PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+					  " @%llu: val=%llu", obj->fence->seqno, obj->value);
+#else
+			obj->fence->ops->fence_value_str(obj->fence,
+							 value, sizeof(value));
+			PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+					  " @%s: val=%llu", value, obj->value);
+#endif
+		}
+		spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+	}
+}
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_create(
+	const char *name)
+{
+	PVRSRV_ERROR srv_err;
+	struct pvr_counting_fence_timeline *timeline =
+		kzalloc(sizeof(*timeline), GFP_KERNEL);
+
+	if (!timeline)
+		goto err_out;
+
+	timeline->context = pvr_sw_fence_context_create(name,
+							"pvr_sw_sync");
+	if (!timeline->context)
+		goto err_free_timeline;
+
+	timeline->current_value = 0;
+	timeline->next_value = 1;
+	kref_init(&timeline->kref);
+	spin_lock_init(&timeline->active_fences_lock);
+	INIT_LIST_HEAD(&timeline->active_fences);
+
+	srv_err = PVRSRVRegisterDriverDbgRequestNotify(
+				&timeline->dbg_request_handle,
+				pvr_counting_fence_timeline_debug_request,
+				DEBUG_REQUEST_LINUXFENCE,
+				timeline);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+			   __func__, PVRSRVGetErrorString(srv_err));
+		goto err_free_timeline_ctx;
+	}
+
+err_out:
+	return timeline;
+
+err_free_timeline_ctx:
+	pvr_sw_fence_context_destroy(timeline->context);
+
+err_free_timeline:
+	kfree(timeline);
+	timeline = NULL;
+	goto err_out;
+}
+
+void pvr_counting_fence_timeline_force_complete(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	struct list_head *entry, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+#if defined(DEBUG) && !defined(SUPPORT_AUTOVZ)
+	/* This is just a safety measure. Normally we should never see any
+	 * unsignaled sw fences when we come here. Warn if we still do!
+	 */
+	WARN_ON(!list_empty(&timeline->active_fences));
+#endif
+
+	list_for_each_safe(entry, tmp, &timeline->active_fences) {
+		struct pvr_counting_fence *fence =
+			list_entry(entry, struct pvr_counting_fence,
+			active_list_entry);
+		dma_fence_signal(fence->fence);
+		dma_fence_put(fence->fence);
+		fence->fence = NULL;
+		list_del(&fence->active_list_entry);
+		kfree(fence);
+	}
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+}
+
+static void pvr_counting_fence_timeline_destroy(
+	struct kref *kref)
+{
+	struct pvr_counting_fence_timeline *timeline =
+		container_of(kref, struct pvr_counting_fence_timeline, kref);
+
+	WARN_ON(!list_empty(&timeline->active_fences));
+
+	PVRSRVUnregisterDriverDbgRequestNotify(timeline->dbg_request_handle);
+
+	pvr_sw_fence_context_destroy(timeline->context);
+	kfree(timeline);
+}
+
+void pvr_counting_fence_timeline_put(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	kref_put(&timeline->kref, pvr_counting_fence_timeline_destroy);
+}
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_get(
+	struct pvr_counting_fence_timeline *timeline)
+{
+	if (!timeline)
+		return NULL;
+	kref_get(&timeline->kref);
+	return timeline;
+}
+
+struct dma_fence *pvr_counting_fence_create(
+	struct pvr_counting_fence_timeline *timeline, u64 *sync_pt_idx)
+{
+	unsigned long flags;
+	struct dma_fence *sw_fence;
+	struct pvr_counting_fence *fence = kmalloc(sizeof(*fence), GFP_KERNEL);
+
+	if (!fence)
+		return NULL;
+
+	sw_fence = pvr_sw_fence_create(timeline->context);
+	if (!sw_fence)
+		goto err_free_fence;
+
+	fence->fence = dma_fence_get(sw_fence);
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	fence->value = timeline->next_value++;
+	if (sync_pt_idx)
+		*sync_pt_idx = fence->value;
+
+	list_add_tail(&fence->active_list_entry, &timeline->active_fences);
+
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+
+	/* Counting fences can be signalled any time after creation */
+	dma_fence_enable_sw_signaling(sw_fence);
+
+	return sw_fence;
+
+err_free_fence:
+	kfree(fence);
+	return NULL;
+}
+
+bool pvr_counting_fence_timeline_inc(
+	struct pvr_counting_fence_timeline *timeline, u64 *sync_pt_idx)
+{
+	struct list_head *entry, *tmp;
+	unsigned long flags;
+	bool res;
+
+	spin_lock_irqsave(&timeline->active_fences_lock, flags);
+
+	if (timeline->current_value == timeline->next_value-1) {
+		res = false;
+		goto exit_unlock;
+	}
+
+	timeline->current_value++;
+
+	if (sync_pt_idx)
+		*sync_pt_idx = timeline->current_value;
+
+	list_for_each_safe(entry, tmp, &timeline->active_fences) {
+		struct pvr_counting_fence *fence =
+			list_entry(entry, struct pvr_counting_fence,
+			active_list_entry);
+		if (fence->value <= timeline->current_value) {
+			dma_fence_signal(fence->fence);
+			dma_fence_put(fence->fence);
+			fence->fence = NULL;
+			list_del(&fence->active_list_entry);
+			kfree(fence);
+		}
+	}
+
+	res = true;
+
+exit_unlock:
+	spin_unlock_irqrestore(&timeline->active_fences_lock, flags);
+
+	return res;
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_counting_timeline.h b/drivers/gpu/drm/img-rogue/pvr_counting_timeline.h
new file mode 100644
index 000000000000..2cb8db1e5c50
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_counting_timeline.h
@@ -0,0 +1,68 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_COUNTING_TIMELINE_H__)
+#define __PVR_COUNTING_TIMELINE_H__
+
+#include "pvr_linux_fence.h"
+
+struct pvr_counting_fence_timeline;
+
+void pvr_counting_fence_timeline_dump_timeline(
+	void *data,
+	DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+	void *dump_debug_file);
+
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_create(
+	const char *name);
+void pvr_counting_fence_timeline_put(
+	struct pvr_counting_fence_timeline *fence_timeline);
+struct pvr_counting_fence_timeline *pvr_counting_fence_timeline_get(
+	struct pvr_counting_fence_timeline *fence_timeline);
+struct dma_fence *pvr_counting_fence_create(
+	struct pvr_counting_fence_timeline *fence_timeline, u64 *sync_pt_idx);
+bool pvr_counting_fence_timeline_inc(
+	struct pvr_counting_fence_timeline *fence_timeline, u64 *sync_pt_idx);
+void pvr_counting_fence_timeline_force_complete(
+	struct pvr_counting_fence_timeline *fence_timeline);
+
+#endif /* !defined(__PVR_COUNTING_TIMELINE_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_debug.c b/drivers/gpu/drm/img-rogue/pvr_debug.c
new file mode 100644
index 000000000000..cf59a2d5aa3e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_debug.c
@@ -0,0 +1,487 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debug Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides kernel side Debug Functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/sched.h>
+#include <linux/pid.h>
+#include <linux/moduleparam.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "linkage.h"
+#include "pvrsrv.h"
+#include "osfunc.h"
+#include "di_server.h"
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/******** BUFFERED LOG MESSAGES ********/
+
+/* Because we don't want to have to handle CCB wrapping, each buffered
+ * message is rounded up to PVRSRV_DEBUG_CCB_MESG_MAX bytes. This means
+ * there is the same fixed number of messages that can be stored,
+ * regardless of message length.
+ */
+
+#if defined(PVRSRV_DEBUG_CCB_MAX)
+
+#define PVRSRV_DEBUG_CCB_MESG_MAX	PVR_MAX_DEBUG_MESSAGE_LEN
+
+typedef struct
+{
+	const IMG_CHAR *pszFile;
+	IMG_INT iLine;
+	IMG_UINT32 ui32TID;
+	IMG_UINT32 ui32PID;
+	IMG_CHAR pcMesg[PVRSRV_DEBUG_CCB_MESG_MAX];
+	struct timeval sTimeVal;
+}
+PVRSRV_DEBUG_CCB;
+
+static PVRSRV_DEBUG_CCB gsDebugCCB[PVRSRV_DEBUG_CCB_MAX];
+
+static IMG_UINT giOffset;
+
+/* protects access to gsDebugCCB */
+static DEFINE_SPINLOCK(gsDebugCCBLock);
+
+static void
+AddToBufferCCB(const IMG_CHAR *pszFileName, IMG_UINT32 ui32Line,
+			   const IMG_CHAR *szBuffer)
+{
+	unsigned long uiFlags;
+
+	spin_lock_irqsave(&gsDebugCCBLock, uiFlags);
+
+	gsDebugCCB[giOffset].pszFile = pszFileName;
+	gsDebugCCB[giOffset].iLine   = ui32Line;
+	gsDebugCCB[giOffset].ui32TID = current->pid;
+	gsDebugCCB[giOffset].ui32PID = current->tgid;
+
+	do_gettimeofday(&gsDebugCCB[giOffset].sTimeVal);
+
+	OSStringSafeCopy(gsDebugCCB[giOffset].pcMesg, szBuffer,
+	              PVRSRV_DEBUG_CCB_MESG_MAX);
+
+	giOffset = (giOffset + 1) % PVRSRV_DEBUG_CCB_MAX;
+
+	spin_unlock_irqrestore(&gsDebugCCBLock, uiFlags);
+}
+
+void PVRSRVDebugPrintfDumpCCB(void)
+{
+	int i;
+	unsigned long uiFlags;
+
+	spin_lock_irqsave(&gsDebugCCBLock, uiFlags);
+
+	for (i = 0; i < PVRSRV_DEBUG_CCB_MAX; i++)
+	{
+		PVRSRV_DEBUG_CCB *psDebugCCBEntry =
+			&gsDebugCCB[(giOffset + i) % PVRSRV_DEBUG_CCB_MAX];
+
+		/* Early on, we won't have PVRSRV_DEBUG_CCB_MAX messages */
+		if (!psDebugCCBEntry->pszFile)
+		{
+			continue;
+		}
+
+		printk(KERN_ERR "%s:%d: (%ld.%ld, tid=%u, pid=%u) %s\n",
+			   psDebugCCBEntry->pszFile,
+			   psDebugCCBEntry->iLine,
+			   (long)psDebugCCBEntry->sTimeVal.tv_sec,
+			   (long)psDebugCCBEntry->sTimeVal.tv_usec,
+			   psDebugCCBEntry->ui32TID,
+			   psDebugCCBEntry->ui32PID,
+			   psDebugCCBEntry->pcMesg);
+
+		/* Clear this entry so it doesn't get printed the next time again. */
+		psDebugCCBEntry->pszFile = NULL;
+	}
+
+	spin_unlock_irqrestore(&gsDebugCCBLock, uiFlags);
+}
+
+#else /* defined(PVRSRV_DEBUG_CCB_MAX) */
+
+static INLINE void
+AddToBufferCCB(const IMG_CHAR *pszFileName, IMG_UINT32 ui32Line,
+			   const IMG_CHAR *szBuffer)
+{
+	(void)pszFileName;
+	(void)szBuffer;
+	(void)ui32Line;
+}
+
+void PVRSRVDebugPrintfDumpCCB(void)
+{
+	/* Not available */
+}
+
+#endif /* defined(PVRSRV_DEBUG_CCB_MAX) */
+
+static IMG_UINT32 PVRDebugLevel =
+	(
+	 DBGPRIV_FATAL | DBGPRIV_ERROR | DBGPRIV_WARNING
+#if defined(PVRSRV_DEBUG_CCB_MAX)
+	 | DBGPRIV_BUFFERED
+#endif /* defined(PVRSRV_DEBUG_CCB_MAX) */
+#if defined(PVR_DPF_ADHOC_DEBUG_ON)
+	 | DBGPRIV_DEBUG
+#endif /* defined(PVR_DPF_ADHOC_DEBUG_ON) */
+	);
+
+module_param(PVRDebugLevel, uint, 0644);
+MODULE_PARM_DESC(PVRDebugLevel,
+                 "Sets the level of debug output (default 0x7)");
+
+IMG_UINT32 OSDebugLevel(void)
+{
+	return PVRDebugLevel;
+}
+
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVRDebugLevel = ui32DebugLevel;
+}
+
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	return (PVRDebugLevel & ui32DebugLevel) != 0;
+}
+
+#else /* defined(PVRSRV_NEED_PVR_DPF) */
+
+IMG_UINT32 OSDebugLevel(void)
+{
+	return 0;
+}
+
+void OSSetDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32DebugLevel);
+}
+
+IMG_BOOL OSIsDebugLevel(IMG_UINT32 ui32DebugLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32DebugLevel);
+	return IMG_FALSE;
+}
+
+#endif /* defined(PVRSRV_NEED_PVR_DPF) */
+
+#define	PVR_MAX_MSG_LEN PVR_MAX_DEBUG_MESSAGE_LEN
+
+/* Message buffer for messages */
+static IMG_CHAR gszBuffer[PVR_MAX_MSG_LEN + 1];
+
+/* The lock is used to control access to gszBuffer */
+static DEFINE_SPINLOCK(gsDebugLock);
+
+/*
+ * Append a string to a buffer using formatted conversion.
+ * The function takes a variable number of arguments, pointed
+ * to by the var args list.
+ */
+__printf(3, 0)
+static IMG_BOOL VBAppend(IMG_CHAR *pszBuf, IMG_UINT32 ui32BufSiz, const IMG_CHAR *pszFormat, va_list VArgs)
+{
+	IMG_UINT32 ui32Used;
+	IMG_UINT32 ui32Space;
+	IMG_INT32 i32Len;
+
+	ui32Used = OSStringLength(pszBuf);
+	BUG_ON(ui32Used >= ui32BufSiz);
+	ui32Space = ui32BufSiz - ui32Used;
+
+	i32Len = vsnprintf(&pszBuf[ui32Used], ui32Space, pszFormat, VArgs);
+	pszBuf[ui32BufSiz - 1] = 0;
+
+	/* Return true if string was truncated */
+	return i32Len < 0 || i32Len >= (IMG_INT32)ui32Space;
+}
+
+void PVRSRVReleasePrintfVArgs(const IMG_CHAR *pszFormat, va_list vaArgs)
+{
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+	IMG_INT32  result;
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	result = snprintf(pszBuf, (ui32BufSiz - 2), "PVR_K:  %u: ", current->pid);
+	PVR_ASSERT(result>0);
+	ui32BufSiz -= result;
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, vaArgs))
+	{
+		printk(KERN_INFO "%s (truncated)\n", pszBuf);
+	}
+	else
+	{
+		printk(KERN_INFO "%s\n", pszBuf);
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVReleasePrintf
+@Description    To output an important message to the user in release builds
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVReleasePrintf(const IMG_CHAR *pszFormat, ...)
+{
+	va_list vaArgs;
+
+	va_start(vaArgs, pszFormat);
+	PVRSRVReleasePrintfVArgs(pszFormat, vaArgs);
+	va_end(vaArgs);
+}
+
+#if defined(PVRSRV_NEED_PVR_TRACE)
+
+/*************************************************************************/ /*!
+@Function       PVRTrace
+@Description    To output a debug message to the user
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVTrace(const IMG_CHAR *pszFormat, ...)
+{
+	va_list VArgs;
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+	IMG_INT32  result;
+
+	va_start(VArgs, pszFormat);
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	result = snprintf(pszBuf, (ui32BufSiz - 2), "PVR: %u: ", current->pid);
+	PVR_ASSERT(result>0);
+	ui32BufSiz -= result;
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, VArgs))
+	{
+		printk(KERN_ERR "PVR_K:(Message Truncated): %s\n", pszBuf);
+	}
+	else
+	{
+		printk(KERN_ERR "%s\n", pszBuf);
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+
+	va_end(VArgs);
+}
+
+#endif /* defined(PVRSRV_NEED_PVR_TRACE) */
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/*
+ * Append a string to a buffer using formatted conversion.
+ * The function takes a variable number of arguments, calling
+ * VBAppend to do the actual work.
+ */
+__printf(3, 4)
+static IMG_BOOL BAppend(IMG_CHAR *pszBuf, IMG_UINT32 ui32BufSiz, const IMG_CHAR *pszFormat, ...)
+{
+	va_list VArgs;
+	IMG_BOOL bTrunc;
+
+	va_start (VArgs, pszFormat);
+
+	bTrunc = VBAppend(pszBuf, ui32BufSiz, pszFormat, VArgs);
+
+	va_end (VArgs);
+
+	return bTrunc;
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintf
+@Description    To output a debug message to the user
+@Input          uDebugLevel The current debug level
+@Input          pszFile     The source file generating the message
+@Input          uLine       The line of the source file
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+void PVRSRVDebugPrintf(IMG_UINT32 ui32DebugLevel,
+			   const IMG_CHAR *pszFullFileName,
+			   IMG_UINT32 ui32Line,
+			   const IMG_CHAR *pszFormat,
+			   ...)
+{
+	const IMG_CHAR *pszFileName = pszFullFileName;
+	IMG_CHAR *pszLeafName;
+	va_list vaArgs;
+	unsigned long ulLockFlags = 0;
+	IMG_CHAR *pszBuf = gszBuffer;
+	IMG_UINT32 ui32BufSiz = sizeof(gszBuffer);
+
+	if (!(PVRDebugLevel & ui32DebugLevel))
+	{
+		return;
+	}
+
+	va_start(vaArgs, pszFormat);
+
+	spin_lock_irqsave(&gsDebugLock, ulLockFlags);
+
+	switch (ui32DebugLevel)
+	{
+		case DBGPRIV_FATAL:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Fatal): ", ui32BufSiz);
+			PVRSRV_REPORT_ERROR();
+			break;
+		}
+		case DBGPRIV_ERROR:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Error): ", ui32BufSiz);
+			PVRSRV_REPORT_ERROR();
+			break;
+		}
+		case DBGPRIV_WARNING:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Warn):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_MESSAGE:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Mesg):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_VERBOSE:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Verb):  ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_DEBUG:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K:(Debug): ", ui32BufSiz);
+			break;
+		}
+		case DBGPRIV_CALLTRACE:
+		case DBGPRIV_ALLOC:
+		case DBGPRIV_BUFFERED:
+		default:
+		{
+			OSStringSafeCopy(pszBuf, "PVR_K: ", ui32BufSiz);
+			break;
+		}
+	}
+
+	if (current->pid == task_tgid_nr(current))
+	{
+		(void) BAppend(pszBuf, ui32BufSiz, "%5u: ", current->pid);
+	}
+	else
+	{
+		(void) BAppend(pszBuf, ui32BufSiz, "%5u-%5u: ", task_tgid_nr(current) /* pid id of group*/, current->pid /* task id */);
+	}
+
+	if (VBAppend(pszBuf, ui32BufSiz, pszFormat, vaArgs))
+	{
+		printk(KERN_ERR "%s (truncated)\n", pszBuf);
+	}
+	else
+	{
+		IMG_BOOL bTruncated = IMG_FALSE;
+
+#if !defined(__sh__)
+		pszLeafName = (IMG_CHAR *)strrchr (pszFileName, '/');
+
+		if (pszLeafName)
+		{
+			pszFileName = pszLeafName+1;
+		}
+#endif /* __sh__ */
+
+#if defined(DEBUG)
+		{
+			static const IMG_CHAR *lastFile;
+
+			if (lastFile == pszFileName)
+			{
+				bTruncated = BAppend(pszBuf, ui32BufSiz, " [%u]", ui32Line);
+			}
+			else
+			{
+				bTruncated = BAppend(pszBuf, ui32BufSiz, " [%s:%u]", pszFileName, ui32Line);
+				lastFile = pszFileName;
+			}
+		}
+#else
+		bTruncated = BAppend(pszBuf, ui32BufSiz, " [%u]", ui32Line);
+#endif
+
+		if (bTruncated)
+		{
+			printk(KERN_ERR "%s (truncated)\n", pszBuf);
+		}
+		else
+		{
+			if (ui32DebugLevel & DBGPRIV_BUFFERED)
+			{
+				AddToBufferCCB(pszFileName, ui32Line, pszBuf);
+			}
+			else
+			{
+				printk(KERN_ERR "%s\n", pszBuf);
+			}
+		}
+	}
+
+	spin_unlock_irqrestore(&gsDebugLock, ulLockFlags);
+
+	va_end (vaArgs);
+}
+
+#endif /* PVRSRV_NEED_PVR_DPF */
diff --git a/drivers/gpu/drm/img-rogue/pvr_debug.h b/drivers/gpu/drm/img-rogue/pvr_debug.h
new file mode 100644
index 000000000000..46033a0da47e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_debug.h
@@ -0,0 +1,1071 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Debug Declarations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides debug functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DEBUG_H
+#define PVR_DEBUG_H
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/* If the kernel pre defined macro is present, we will use the definition of HTBLOGK
+ * in htbserver.h with the format strings specified in ht_buffer_sf.h. Otherwise,
+ * HTBLOGK needs to be defined as empty since it is not used by user-mode code.
+ */
+#if defined(__KERNEL__)
+#	include "htbserver.h"
+#	include "htbuffer_sf.h"
+#else
+#	define HTBLOGK(...) ((void) 0)
+#endif
+
+/*! @cond Doxygen_Suppress */
+#if defined(_MSC_VER)
+#	define MSC_SUPPRESS_4127 __pragma(warning(suppress:4127))
+#else
+#	define MSC_SUPPRESS_4127
+#endif
+/*! @endcond */
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ && __KERNEL__*/
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define PVR_MAX_DEBUG_MESSAGE_LEN	(512)   /*!< Max length of a Debug Message */
+
+/* These are privately used by pvr_debug, use the PVR_DBG_ defines instead */
+#define DBGPRIV_FATAL     0x001U  /*!< Debug-Fatal. Privately used by pvr_debug. */
+#define DBGPRIV_ERROR     0x002U  /*!< Debug-Error. Privately used by pvr_debug. */
+#define DBGPRIV_WARNING   0x004U  /*!< Debug-Warning. Privately used by pvr_debug. */
+#define DBGPRIV_MESSAGE   0x008U  /*!< Debug-Message. Privately used by pvr_debug. */
+#define DBGPRIV_VERBOSE   0x010U  /*!< Debug-Verbose. Privately used by pvr_debug. */
+#define DBGPRIV_CALLTRACE 0x020U  /*!< Debug-CallTrace. Privately used by pvr_debug. */
+#define DBGPRIV_ALLOC     0x040U  /*!< Debug-Alloc. Privately used by pvr_debug. */
+#define DBGPRIV_BUFFERED  0x080U  /*!< Debug-Buffered. Privately used by pvr_debug. */
+#define DBGPRIV_DEBUG     0x100U  /*!< Debug-AdHoc-Debug. Never submitted. Privately used by pvr_debug. */
+#define DBGPRIV_LAST      0x100U  /*!< Always set to highest mask value. Privately used by pvr_debug. */
+
+/* Enable DPF logging for locally from some make targets */
+#if defined(PVRSRV_NEED_PVR_DPF_LOCAL)
+#undef PVRSRV_NEED_PVR_DPF
+#define PVRSRV_NEED_PVR_DPF
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_ASSERT) && defined(DEBUG)
+#define PVRSRV_NEED_PVR_ASSERT
+#endif
+
+#if defined(PVRSRV_NEED_PVR_ASSERT) && !defined(PVRSRV_NEED_PVR_DPF)
+#define PVRSRV_NEED_PVR_DPF
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_TRACE) && (defined(DEBUG) || defined(TIMING))
+#define PVRSRV_NEED_PVR_TRACE
+#endif
+
+#if !defined(DOXYGEN)
+/*************************************************************************/ /*
+PVRSRVGetErrorString
+Returns a string describing the provided PVRSRV_ERROR code
+NB No doxygen comments provided as this function does not require porting
+   for other operating systems
+*/ /**************************************************************************/
+const IMG_CHAR *PVRSRVGetErrorString(PVRSRV_ERROR eError);
+#define PVRSRVGETERRORSTRING PVRSRVGetErrorString
+#endif
+
+/* PVR_ASSERT() and PVR_DBG_BREAK handling */
+
+#if defined(__KLOCWORK__)
+/* A dummy no-return function to be used under Klocwork to mark unreachable
+   paths instead of abort() in order to avoid MISRA.STDLIB.ABORT issues. */
+__noreturn void klocwork_abort(void);
+#endif
+
+#if defined(PVRSRV_NEED_PVR_ASSERT) || defined(DOXYGEN)
+
+/* Unfortunately the Klocwork static analysis checker doesn't understand our
+ * ASSERT macros. Thus it reports lots of false positive. Defining our Assert
+ * macros in a special way when the code is analysed by Klocwork avoids
+ * them.
+ */
+#if defined(__KLOCWORK__)
+#define PVR_ASSERT(x) do { if (!(x)) {klocwork_abort();} } while (false)
+#else /* ! __KLOCWORKS__ */
+
+#if defined(_WIN32)
+#define PVR_ASSERT(expr) do										\
+	{															\
+		MSC_SUPPRESS_4127										\
+		if (unlikely(!(expr)))								\
+		{														\
+			PVRSRVDebugPrintf(DBGPRIV_FATAL, __FILE__, __LINE__,\
+					  "*** Debug assertion failed!");			\
+			__debugbreak();										\
+		}														\
+	MSC_SUPPRESS_4127											\
+	} while (false)
+
+#else
+
+#if defined(__linux__) && defined(__KERNEL__)
+#include <linux/kernel.h>
+#include <linux/bug.h>
+
+/* In Linux kernel mode, use WARN_ON() directly. This produces the
+ * correct filename and line number in the warning message.
+ */
+#define PVR_ASSERT(EXPR) do											\
+	{																\
+		if (unlikely(!(EXPR)))										\
+		{															\
+			PVRSRVDebugPrintf(DBGPRIV_FATAL, __FILE__, __LINE__,	\
+							  "Debug assertion failed!");			\
+			WARN_ON(1);												\
+		}															\
+	} while (false)
+
+#else /* defined(__linux__) && defined(__KERNEL__) */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugAssertFail
+@Description    Indicate to the user that a debug assertion has failed and
+                prevent the program from continuing.
+                Invoked from the macro PVR_ASSERT().
+@Input          pszFile       The name of the source file where the assertion failed
+@Input          ui32Line      The line number of the failed assertion
+@Input          pszAssertion  String describing the assertion
+@Return         NEVER!
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV __noreturn
+PVRSRVDebugAssertFail(const IMG_CHAR *pszFile,
+                      IMG_UINT32 ui32Line,
+                      const IMG_CHAR *pszAssertion);
+
+#define PVR_ASSERT(EXPR) do										\
+	{															\
+		if (unlikely(!(EXPR)))									\
+		{														\
+			PVRSRVDebugAssertFail(__FILE__, __LINE__, #EXPR);	\
+		}														\
+	} while (false)
+
+#endif /* defined(__linux__) && defined(__KERNEL__) */
+#endif /* defined(_WIN32) */
+#endif /* defined(__KLOCWORK__) */
+
+#if defined(__KLOCWORK__)
+	#define PVR_DBG_BREAK do { klocwork_abort(); } while (false)
+#else
+	#if defined(WIN32)
+		#define PVR_DBG_BREAK __debugbreak()   /*!< Implementation of PVR_DBG_BREAK for (non-WinCE) Win32 */
+	#else
+		#if defined(PVR_DBG_BREAK_ASSERT_FAIL)
+		/*!< Implementation of PVR_DBG_BREAK that maps onto PVRSRVDebugAssertFail */
+			#if defined(_WIN32)
+				#define PVR_DBG_BREAK	DBG_BREAK
+			#else
+				#if defined(__linux__) && defined(__KERNEL__)
+					#define PVR_DBG_BREAK BUG()
+				#else
+					#define PVR_DBG_BREAK	PVRSRVDebugAssertFail(__FILE__, __LINE__, "PVR_DBG_BREAK")
+				#endif
+			#endif
+		#else
+			/*!< Null Implementation of PVR_DBG_BREAK (does nothing) */
+			#define PVR_DBG_BREAK
+		#endif
+	#endif
+#endif
+
+
+#else /* defined(PVRSRV_NEED_PVR_ASSERT) */
+	/* Unfortunately the Klocwork static analysis checker doesn't understand our
+	* ASSERT macros. Thus it reports lots of false positive. Defining our Assert
+	* macros in a special way when the code is analysed by Klocwork avoids
+	* them.
+	*/
+	#if defined(__KLOCWORK__)
+		#define PVR_ASSERT(EXPR) do { if (!(EXPR)) {klocwork_abort();} } while (false)
+	#else
+		#define PVR_ASSERT(EXPR) (void)(EXPR) /*!< Null Implementation of PVR_ASSERT (does nothing) */
+	#endif
+
+	#define PVR_DBG_BREAK    /*!< Null Implementation of PVR_DBG_BREAK (does nothing) */
+
+#endif /* defined(PVRSRV_NEED_PVR_ASSERT) */
+
+
+/* PVR_DPF() handling */
+
+#if defined(PVRSRV_NEED_PVR_DPF) || defined(DOXYGEN)
+
+	/* New logging mechanism */
+	#define PVR_DBG_FATAL     DBGPRIV_FATAL     /*!< Debug level passed to PVRSRVDebugPrintf() for fatal errors. */
+	#define PVR_DBG_ERROR     DBGPRIV_ERROR     /*!< Debug level passed to PVRSRVDebugPrintf() for non-fatal errors. */
+	#define PVR_DBG_WARNING   DBGPRIV_WARNING   /*!< Debug level passed to PVRSRVDebugPrintf() for warnings. */
+	#define PVR_DBG_MESSAGE   DBGPRIV_MESSAGE   /*!< Debug level passed to PVRSRVDebugPrintf() for information only. */
+	#define PVR_DBG_VERBOSE   DBGPRIV_VERBOSE   /*!< Debug level passed to PVRSRVDebugPrintf() for very low-priority debug. */
+	#define PVR_DBG_CALLTRACE DBGPRIV_CALLTRACE /*!< Debug level passed to PVRSRVDebugPrintf() for function tracing purposes. */
+	#define PVR_DBG_ALLOC     DBGPRIV_ALLOC     /*!< Debug level passed to PVRSRVDebugPrintf() for tracking some of drivers memory operations. */
+	#define PVR_DBG_BUFFERED  DBGPRIV_BUFFERED  /*!< Debug level passed to PVRSRVDebugPrintf() when debug should be written to the debug circular buffer. */
+	#define PVR_DBG_DEBUG     DBGPRIV_DEBUG     /*!< Debug level passed to PVRSRVDebugPrintf() for debug messages. */
+
+	/* These levels are always on with PVRSRV_NEED_PVR_DPF */
+	/*! @cond Doxygen_Suppress */
+	#define PVR_DPF_0x001U(...) PVRSRVDebugPrintf(DBGPRIV_FATAL, __VA_ARGS__)
+	#define PVR_DPF_0x002U(...) PVRSRVDebugPrintf(DBGPRIV_ERROR, __VA_ARGS__)
+	#define PVR_DPF_0x080U(...) PVRSRVDebugPrintf(DBGPRIV_BUFFERED, __VA_ARGS__)
+
+	/*
+	 * The AdHoc-Debug level is only supported when enabled in the local
+	 * build environment and may need to be used in both debug and release
+	 * builds. An error is generated in the formal build if it is checked in.
+	 */
+#if defined(PVR_DPF_ADHOC_DEBUG_ON)
+	#define PVR_DPF_0x100U(...) PVRSRVDebugPrintf(DBGPRIV_DEBUG, __VA_ARGS__)
+#else
+	/* Use an undefined token here to stop compilation dead in the offending module */
+	#define PVR_DPF_0x100U(...) __ERROR__PVR_DBG_DEBUG_is_in_use_but_has_not_been_enabled__Note_Debug_DPF_must_not_be_checked_in__Define_PVR_DPF_ADHOC_DEBUG_ON_for_testing
+#endif
+
+	/* Some are compiled out completely in release builds */
+#if defined(DEBUG) || defined(DOXYGEN)
+	#define PVR_DPF_0x004U(...) PVRSRVDebugPrintf(DBGPRIV_WARNING, __VA_ARGS__)
+	#define PVR_DPF_0x008U(...) PVRSRVDebugPrintf(DBGPRIV_MESSAGE, __VA_ARGS__)
+	#define PVR_DPF_0x010U(...) PVRSRVDebugPrintf(DBGPRIV_VERBOSE, __VA_ARGS__)
+	#define PVR_DPF_0x020U(...) PVRSRVDebugPrintf(DBGPRIV_CALLTRACE, __VA_ARGS__)
+	#define PVR_DPF_0x040U(...) PVRSRVDebugPrintf(DBGPRIV_ALLOC, __VA_ARGS__)
+#else
+	#define PVR_DPF_0x004U(...)
+	#define PVR_DPF_0x008U(...)
+	#define PVR_DPF_0x010U(...)
+	#define PVR_DPF_0x020U(...)
+	#define PVR_DPF_0x040U(...)
+#endif
+
+	/* Translate the different log levels to separate macros
+	 * so they can each be compiled out.
+	 */
+#if defined(DEBUG)
+	#define PVR_DPF_EX(lvl, ...) PVR_DPF_ ## lvl (__FILE__, __LINE__, __VA_ARGS__)
+#else
+	#define PVR_DPF_EX(lvl, ...) PVR_DPF_ ## lvl ("", __LINE__, __VA_ARGS__)
+#endif
+	/*! @endcond */
+
+	/* Get rid of the double bracketing */
+	#define PVR_DPF(x) PVR_DPF_EX x
+
+	#define PVR_LOG_ERROR(_rc, _call) do \
+		{ \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+		} while (false)
+
+	#define PVR_LOG_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_WARN_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_WARNING, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_WARNING, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_NOMEM(_expr, _call) do \
+		{ \
+			if (unlikely(_expr == NULL)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_OUT_OF_MEMORY, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()", _call, __func__)); \
+				return PVRSRV_ERROR_OUT_OF_MEMORY; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_NOMEM(_expr, _err, _go) do \
+		{ \
+			if (unlikely(_expr == NULL)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_OUT_OF_MEMORY, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s failed (PVRSRV_ERROR_OUT_OF_MEMORY) in %s()", #_expr, __func__)); \
+				_err = PVRSRV_ERROR_OUT_OF_MEMORY; \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_ERROR(_rc, _call) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				return; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR(_rc, _call, _go) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_WITH_ERROR(_call, _err, _rc, _go) do \
+		{ \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s() failed (%s) in %s()", _call, PVRSRVGETERRORSTRING(_rc), __func__)); \
+			_err = _rc; \
+			goto _go; \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_IF_FALSE(_expr, _msg) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_FALSE(_expr, _msg, _rc) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_FALSE(_expr, _msg) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_FALSE(_expr, _msg, _go) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_F, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE(_expr, _msg, _rc) do \
+		{ \
+			if (unlikely((_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_COND_ERROR_T, PVRSRV_ERROR_UNEXPECTED_TRUE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s in %s()", _msg, __func__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_INVALID_PARAM(_expr, _param) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_INVALID_PARAMS, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, "%s invalid in %s()", _param, __func__)); \
+				return PVRSRV_ERROR_INVALID_PARAMS; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do \
+		{ if (unlikely(!(_expr))) { \
+			HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_INVALID_PARAMS, HTB_FILE_NAME, __LINE__); \
+			PVR_DPF((PVR_DBG_ERROR, "%s invalid in %s()", #_expr, __func__)); \
+			_err = PVRSRV_ERROR_INVALID_PARAMS; \
+			goto _go; } \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_MSG(_lvl, _msg) do \
+		{ \
+			if (_lvl == PVR_DBG_ERROR) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+			} \
+			else if (_lvl == PVR_DBG_WARNING) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+			} \
+			PVR_DPF((_lvl, ("In %s() "_msg), __func__)); \
+		} while (false)
+
+	#define PVR_LOG_VA(_lvl, _msg, ...) do \
+		{ \
+			if (_lvl == PVR_DBG_ERROR) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+			} \
+			else if (_lvl == PVR_DBG_WARNING) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+			} \
+			PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+		} while (false)
+
+	#define PVR_LOG_IF_ERROR_VA(_lvl, _rc, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				if (_lvl == PVR_DBG_ERROR) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				} \
+				else if (_lvl == PVR_DBG_WARNING) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_WARNING, _rc, HTB_FILE_NAME, __LINE__); \
+				} \
+				PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_IF_FALSE_VA(_lvl, _expr, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				if (_lvl == PVR_DBG_ERROR) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_ERROR, HTB_FILE_NAME, __LINE__); \
+				} \
+				else if (_lvl == PVR_DBG_WARNING) \
+				{ \
+					HTBLOGK(HTB_SF_MAIN_DBG_MSGLVL_WARN, HTB_FILE_NAME, __LINE__); \
+				} \
+				PVR_DPF((_lvl, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR_VA(_rc, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR_VA(_rc, _go, _msg, ...) do \
+		{ \
+			if (unlikely(_rc != PVRSRV_OK)) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_FALSE_VA(_expr, _rc, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("At %s: "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_GOTO_IF_FALSE_VA(_expr, _go, _msg, ...) do \
+		{ \
+			if (unlikely(!(_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("In %s() "_msg), __func__, __VA_ARGS__)); \
+				goto _go; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE_VA(_expr, _rc, _msg, ...) do \
+		{ \
+			if (unlikely((_expr))) \
+			{ \
+				HTBLOGK(HTB_SF_MAIN_DBG_ERROR, _rc, HTB_FILE_NAME, __LINE__); \
+				PVR_DPF((PVR_DBG_ERROR, ("At %s: "_msg), __func__, __VA_ARGS__)); \
+				return _rc; \
+			} \
+		MSC_SUPPRESS_4127 \
+		} while (false)
+
+#else /* defined(PVRSRV_NEED_PVR_DPF) */
+
+	#define PVR_DPF(X)  /*!< Null Implementation of PowerVR Debug Printf (does nothing) */
+
+	#define PVR_LOG_MSG(_lvl, _msg)
+	#define PVR_LOG_VA(_lvl, _msg, ...)
+	#define PVR_LOG_ERROR(_rc, _call) (void)(_rc)
+	#define PVR_LOG_IF_ERROR(_rc, _call) (void)(_rc)
+	#define PVR_WARN_IF_ERROR(_rc, _call) (void)(_rc)
+
+	#define PVR_LOG_IF_ERROR_VA(_lvl, _rc, _msg, ...) (void)(_rc)
+	#define PVR_LOG_IF_FALSE_VA(_lvl, _expr, _msg, ...) (void)(_expr)
+
+	#define PVR_LOG_RETURN_IF_NOMEM(_expr, _call) do { if (unlikely(_expr == NULL)) { return PVRSRV_ERROR_OUT_OF_MEMORY; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_NOMEM(_expr, _err, _go) do { if (unlikely(_expr == NULL)) { _err = PVRSRV_ERROR_OUT_OF_MEMORY; goto _go; } MSC_SUPPRESS_4127	} while (false)
+
+	#define PVR_LOG_RETURN_IF_ERROR(_rc, _call) do { if (unlikely(_rc != PVRSRV_OK)) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_ERROR_VA(_rc, _msg, ...) do { if (unlikely(_rc != PVRSRV_OK)) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_VOID_IF_ERROR(_rc, _call) do { if (unlikely(_rc != PVRSRV_OK)) { return; } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_GOTO_IF_ERROR(_rc, _call, _go) do { if (unlikely(_rc != PVRSRV_OK)) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_ERROR_VA(_rc, _go, _msg, ...) do { if (unlikely(_rc != PVRSRV_OK)) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_WITH_ERROR(_call, _err, _rc, _go) do { _err = _rc; goto _go; MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_IF_FALSE(_expr, _msg) (void)(_expr)
+	#define PVR_LOG_RETURN_IF_FALSE(_expr, _msg, _rc) do { if (unlikely(!(_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_FALSE_VA(_expr, _rc, _msg, ...) do { if (unlikely(!(_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_IF_TRUE(_expr, _msg, _rc) do { if (unlikely((_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_RETURN_IF_TRUE_VA(_expr, _rc, _msg, ...) do { if (unlikely((_expr))) { return (_rc); } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_VOID_IF_FALSE(_expr, _msg) do { if (unlikely(!(_expr))) { return; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_FALSE(_expr, _msg, _go) do { if (unlikely(!(_expr))) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_FALSE_VA(_expr, _go, _msg, ...) do { if (unlikely(!(_expr))) { goto _go; } MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_LOG_RETURN_IF_INVALID_PARAM(_expr, _param) do { if (unlikely(!(_expr))) { return PVRSRV_ERROR_INVALID_PARAMS; } MSC_SUPPRESS_4127 } while (false)
+	#define PVR_LOG_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do { if (unlikely(!(_expr))) { _err = PVRSRV_ERROR_INVALID_PARAMS; goto _go; } MSC_SUPPRESS_4127 } while (false)
+
+	#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#endif /* defined(PVRSRV_NEED_PVR_DPF) */
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintf
+@Description    Output a debug message to the user, using an OS-specific
+                method, to a log or console which can be read by developers
+                Invoked from the macro PVR_DPF().
+@Input          ui32DebugLevel   The debug level of the message. This can
+                                 be used to restrict the output of debug
+                                 messages based on their severity.
+                                 If this is PVR_DBG_BUFFERED, the message
+                                 should be written into a debug circular
+                                 buffer instead of being output immediately
+                                 (useful when performance would otherwise
+                                 be adversely affected).
+                                 The debug circular buffer shall only be
+                                 output when PVRSRVDebugPrintfDumpCCB() is
+                                 called.
+@Input          pszFileName      The source file containing the code that is
+                                 generating the message
+@Input          ui32Line         The line number in the source file
+@Input          pszFormat        The formatted message string
+@Input          ...              Zero or more arguments for use by the
+                                 formatted string
+@Return         None
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVDebugPrintf(IMG_UINT32 ui32DebugLevel,
+                                               const IMG_CHAR *pszFileName,
+                                               IMG_UINT32 ui32Line,
+                                               const IMG_CHAR *pszFormat,
+                                               ...) __printf(4, 5);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugPrintfDumpCCB
+@Description    When PVRSRVDebugPrintf() is called with the ui32DebugLevel
+                specified as DBGPRIV_BUFFERED, the debug shall be written to
+                the debug circular buffer instead of being output immediately.
+                (This could be used to obtain debug without incurring a
+                performance hit by printing it at that moment).
+                This function shall dump the contents of that debug circular
+                buffer to be output in an OS-specific method to a log or
+                console which can be read by developers.
+@Return         None
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVDebugPrintfDumpCCB(void);
+
+#if !defined(DOXYGEN)
+#define PVR_DPF_FUNC__(lvl, message, ...) PVR_DPF((lvl, "%s: " message, __func__, ##__VA_ARGS__))
+#define PVR_DPF_FUNC(x) PVR_DPF_FUNC__ x
+#endif
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_ERROR macro.
+ */
+#define PVR_RETURN_IF_ERROR(_rc) do \
+	{ if (unlikely(_rc != PVRSRV_OK)) { \
+		return _rc; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_FALSE macro.
+ */
+#define PVR_RETURN_IF_FALSE(_expr, _rc) do \
+	{ if (unlikely(!(_expr))) { \
+		return _rc; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_INVALID_PARAM macro.
+ */
+#define PVR_RETURN_IF_INVALID_PARAM(_expr) do \
+	{ if (unlikely(!(_expr))) { \
+		return PVRSRV_ERROR_INVALID_PARAMS; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_RETURN_IF_NOMEM macro.
+ */
+#define PVR_RETURN_IF_NOMEM(_expr) do \
+	{ if (unlikely(!(_expr))) { \
+		return PVRSRV_ERROR_OUT_OF_MEMORY; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_NOMEM macro.
+ */
+#define PVR_GOTO_IF_NOMEM(_expr, _err, _go) do \
+	{ if (unlikely(_expr == NULL)) { \
+		_err = PVRSRV_ERROR_OUT_OF_MEMORY; \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_INVALID_PARAM macro.
+ */
+#define PVR_GOTO_IF_INVALID_PARAM(_expr, _err, _go) do \
+	{ if (unlikely(!(_expr))) { \
+		_err = PVRSRV_ERROR_INVALID_PARAMS; \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_FALSE macro.
+ */
+#define PVR_GOTO_IF_FALSE(_expr, _go) do \
+	{ if (unlikely(!(_expr))) { \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_IF_ERROR macro.
+ */
+#define PVR_GOTO_IF_ERROR(_rc, _go) do \
+	{ if (unlikely(_rc != PVRSRV_OK)) { \
+		goto _go; } \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/* Note: Use only when a log message due to the error absolutely should not
+ *       be printed. Otherwise use PVR_LOG_GOTO_WITH_ERROR macro.
+ */
+#define PVR_GOTO_WITH_ERROR(_err, _rc, _go) do \
+	{ _err = _rc; goto _go; \
+	MSC_SUPPRESS_4127 \
+	} while (false)
+
+/*! @cond Doxygen_Suppress */
+#if defined(PVR_DPF_FUNCTION_TRACE_ON)
+
+	#define PVR_DPF_ENTERED \
+	PVR_DPF((PVR_DBG_CALLTRACE, "|-> %s:%d entered", __func__, __LINE__))
+
+	#define PVR_DPF_ENTERED1(p1) \
+		PVR_DPF((PVR_DBG_CALLTRACE, "|-> %s:%d entered (0x%lx)", __func__, __LINE__, ((unsigned long)p1)))
+
+	#define PVR_DPF_RETURN_RC(a) \
+	do { int _r = (a); PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned %d", __func__, __LINE__, (_r))); return (_r); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_RC1(a,p1) \
+		do { int _r = (a); PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned %d (0x%lx)", __func__, __LINE__, (_r), ((unsigned long)p1))); return (_r); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_VAL(a) \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned with value", __func__, __LINE__)); return (a); MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN_OK \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned ok", __func__, __LINE__)); return PVRSRV_OK; MSC_SUPPRESS_4127 } while (false)
+
+	#define PVR_DPF_RETURN \
+		do { PVR_DPF((PVR_DBG_CALLTRACE, "<-| %s:%d returned", __func__, __LINE__)); return; MSC_SUPPRESS_4127 } while (false)
+
+	#if !defined(DEBUG)
+	#error PVR DPF Function trace enabled in release build, rectify
+	#endif
+
+#else /* defined(PVR_DPF_FUNCTION_TRACE_ON) */
+
+	#define PVR_DPF_ENTERED
+	#define PVR_DPF_ENTERED1(p1)
+	#define PVR_DPF_RETURN_RC(a)     return (a)
+	#define PVR_DPF_RETURN_RC1(a,p1) return (a)
+	#define PVR_DPF_RETURN_VAL(a)    return (a)
+	#define PVR_DPF_RETURN_OK        return PVRSRV_OK
+	#define PVR_DPF_RETURN           return
+
+#endif /* defined(PVR_DPF_FUNCTION_TRACE_ON) */
+/*! @endcond */
+
+#if defined(__KERNEL__) || defined(DOXYGEN) || defined(__QNXNTO__)
+/*Use PVR_DPF() unless message is necessary in release build */
+#define PVR_LOG(X) PVRSRVReleasePrintf X
+
+/*************************************************************************/ /*!
+@Function       PVRSRVReleasePrintf
+@Description    Output an important message, using an OS-specific method,
+                to the Server log or console which will always be output in
+                both release and debug builds.
+                Invoked from the macro PVR_LOG(). Used in Services Server only.
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+@Return         None
+*/ /**************************************************************************/
+void IMG_CALLCONV PVRSRVReleasePrintf(const IMG_CHAR *pszFormat, ...) __printf(1, 2);
+
+ /*************************************************************************/ /*!
+ @Function       PVRSRVReleasePrintfVArgs
+ @Description    Output an important message, using an OS-specific method,
+                 to the Server log or console which will always be output in
+                 both release and debug builds.
+                 Calls to va_start and va_end should wrap this function when
+                 passing in va_list args.
+ @Input          pszFormat   The message format string
+ @Input          vaArgs      va_list arguments to print using pszFormat.
+ @Return         None
+ */ /**************************************************************************/
+void IMG_CALLCONV PVRSRVReleasePrintfVArgs(const IMG_CHAR *pszFormat, va_list vaArgs);
+#endif
+
+/* PVR_TRACE() handling */
+
+#if defined(PVRSRV_NEED_PVR_TRACE) || defined(DOXYGEN)
+
+	#define PVR_TRACE(X)	PVRSRVTrace X    /*!< PowerVR Debug Trace Macro */
+	/* Empty string implementation that is -O0 build friendly */
+	#define PVR_TRACE_EMPTY_LINE()	PVR_TRACE(("%s", ""))
+
+/*************************************************************************/ /*!
+@Function       PVRTrace
+@Description    Output a debug message to the user
+                Invoked from the macro PVR_TRACE().
+@Input          pszFormat   The message format string
+@Input          ...         Zero or more arguments for use by the format string
+*/ /**************************************************************************/
+IMG_EXPORT void IMG_CALLCONV PVRSRVTrace(const IMG_CHAR* pszFormat, ... )
+	__printf(1, 2);
+
+#else /* defined(PVRSRV_NEED_PVR_TRACE) */
+	/*! Null Implementation of PowerVR Debug Trace Macro (does nothing) */
+	#define PVR_TRACE(X)
+
+#endif /* defined(PVRSRV_NEED_PVR_TRACE) */
+
+
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_64BITS_TO_32BITS)
+#endif
+	INLINE static IMG_UINT32 TRUNCATE_64BITS_TO_32BITS(IMG_UINT64 uiInput)
+	{
+		IMG_UINT32 uiTruncated;
+
+		uiTruncated = (IMG_UINT32)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_64BITS_TO_SIZE_T)
+#endif
+	INLINE static size_t TRUNCATE_64BITS_TO_SIZE_T(IMG_UINT64 uiInput)
+	{
+		size_t uiTruncated;
+
+		uiTruncated = (size_t)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(TRUNCATE_SIZE_T_TO_32BITS)
+#endif
+	INLINE static IMG_UINT32 TRUNCATE_SIZE_T_TO_32BITS(size_t uiInput)
+	{
+		IMG_UINT32 uiTruncated;
+
+		uiTruncated = (IMG_UINT32)uiInput;
+		PVR_ASSERT(uiInput == uiTruncated);
+		return uiTruncated;
+	}
+
+
+#else /* defined(PVRSRV_NEED_PVR_ASSERT) */
+	#define TRUNCATE_64BITS_TO_32BITS(expr) ((IMG_UINT32)(expr))
+	#define TRUNCATE_64BITS_TO_SIZE_T(expr) ((size_t)(expr))
+	#define TRUNCATE_SIZE_T_TO_32BITS(expr) ((IMG_UINT32)(expr))
+#endif /* defined(PVRSRV_NEED_PVR_ASSERT) */
+
+/*! @cond Doxygen_Suppress */
+/* Macros used to trace calls */
+#if defined(DEBUG)
+	#define PVR_DBG_FILELINE , (__FILE__), (__LINE__)
+	#define PVR_DBG_FILELINE_PARAM , const IMG_CHAR *pszaFile, IMG_UINT32 ui32Line
+	#define PVR_DBG_FILELINE_ARG , pszaFile, ui32Line
+	#define PVR_DBG_FILELINE_FMT " %s:%u"
+	#define PVR_DBG_FILELINE_UNREF() do { PVR_UNREFERENCED_PARAMETER(pszaFile); \
+				PVR_UNREFERENCED_PARAMETER(ui32Line); } while (false)
+#else
+	#define PVR_DBG_FILELINE
+	#define PVR_DBG_FILELINE_PARAM
+	#define PVR_DBG_FILELINE_ARG
+	#define PVR_DBG_FILELINE_FMT
+	#define PVR_DBG_FILELINE_UNREF()
+#endif
+/*! @endcond */
+
+#if defined(__cplusplus)
+}
+#endif
+
+/*!
+    @def PVR_ASSERT
+    @brief Aborts the program if assertion fails.
+
+    The macro will be defined only when PVRSRV_NEED_PVR_ASSERT macro is
+    enabled. It's ignored otherwise.
+
+    @def PVR_DPF
+    @brief PowerVR Debug Printf logging macro used throughout the driver.
+
+    The macro allows to print logging messages to appropriate log. The
+    destination log is based on the component (user space / kernel space) and
+    operating system (Linux, Android, etc.).
+
+    The macro also supports severity levels that allow to turn on/off messages
+    based on their importance.
+
+    This macro will print messages with severity level higher that error only
+    if PVRSRV_NEED_PVR_DPF macro is defined.
+
+    @def PVR_LOG_ERROR
+    @brief Logs error.
+
+    @def PVR_LOG_IF_ERROR
+    @brief Logs error if not PVRSRV_OK.
+
+    @def PVR_WARN_IF_ERROR
+    @brief Logs warning if not PVRSRV_OK.
+
+    @def PVR_LOG_RETURN_IF_NOMEM
+    @brief Logs error if expression is NULL and returns PVRSRV_ERROR_OUT_OF_MEMORY.
+
+    @def PVR_LOG_GOTO_IF_NOMEM
+    @brief Logs error if expression is NULL and jumps to given label.
+
+    @def PVR_LOG_RETURN_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and returns the error.
+
+    @def PVR_LOG_RETURN_VOID_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and returns (used in function that return void).
+
+    @def PVR_LOG_GOTO_IF_ERROR
+    @brief Logs error if not PVRSRV_OK and jumps to label.
+
+    @def PVR_LOG_GOTO_WITH_ERROR
+    @brief Logs error, goes to a label and sets the error code.
+
+    @def PVR_LOG_IF_FALSE
+    @brief Prints error message if expression is false.
+
+    @def PVR_LOG_RETURN_IF_FALSE
+    @brief Prints error message if expression is false and returns given error.
+
+    @def PVR_LOG_RETURN_VOID_IF_FALSE
+    @brief Prints error message if expression is false and returns (used in function that return void).
+
+    @def PVR_LOG_GOTO_IF_FALSE
+    @brief Prints error message if expression is false and jumps to label.
+
+    @def PVR_LOG_RETURN_IF_TRUE
+    @brief Prints error message if expression is true and returns given error.
+
+    @def PVR_LOG_RETURN_IF_INVALID_PARAM
+    @brief Prints error message if expression is false and returns PVRSRV_ERROR_INVALID_PARAMS.
+
+    @def PVR_LOG_GOTO_IF_INVALID_PARAM
+    @brief Prints error message if expression is false and jumps to label.
+
+    @def PVR_RETURN_IF_ERROR
+    @brief Returns passed error code if it's different than PVRSRV_OK;
+
+    @def PVR_RETURN_IF_FALSE
+    @brief Returns passed error code if expression is false.
+
+    @def PVR_RETURN_IF_INVALID_PARAM
+    @brief Returns PVRSRV_ERROR_INVALID_PARAMS if expression is false.
+
+    @def PVR_RETURN_IF_NOMEM
+    @brief Returns PVRSRV_ERROR_OUT_OF_MEMORY if expression is NULL.
+
+    @def PVR_GOTO_IF_NOMEM
+    @brief Goes to a label if expression is NULL.
+
+    @def PVR_GOTO_IF_INVALID_PARAM
+    @brief Goes to a label if expression is false.
+
+    @def PVR_GOTO_IF_FALSE
+    @brief Goes to a label if expression is false.
+
+    @def PVR_GOTO_IF_ERROR
+    @brief Goes to a label if the error code is different than PVRSRV_OK;
+
+    @def PVR_GOTO_WITH_ERROR
+    @brief Goes to a label and sets the error code.
+
+    @def PVR_LOG
+    @brief Prints message to a log unconditionally.
+
+    This macro will print messages only if PVRSRV_NEED_PVR_LOG macro is defined.
+    @def PVR_LOG_MSG
+    @brief Prints message to a log with the given log-level.
+
+    @def PVR_LOG_VA
+    @brief Prints message with var-args to a log with the given log-level.
+
+    @def PVR_LOG_IF_ERROR_VA
+    @brief Prints message with var-args to a log if the error code is different than PVRSRV_OK.
+
+    @def PVR_LOG_IF_FALSE_VA
+    @brief Prints message with var-args if expression is false.
+
+    @def PVR_LOG_RETURN_IF_ERROR_VA
+    @brief Prints message with var-args to a log and returns the error code.
+
+    @def PVR_LOG_GOTO_IF_ERROR_VA
+    @brief Prints message with var-args to a log and goes to a label if the error code is different than PVRSRV_OK.
+
+    @def PVR_LOG_RETURN_IF_FALSE_VA
+    @brief Logs the error message with var-args if the expression is false and returns the error code.
+
+    @def PVR_LOG_GOTO_IF_FALSE_VA
+    @brief Logs the error message with var-args and goes to a label if the expression is false.
+
+    @def PVR_LOG_RETURN_IF_TRUE_VA
+    @brief Logs the error message with var-args if the expression is true and returns the error code.
+
+    @def PVR_TRACE_EMPTY_LINE
+    @brief Prints empty line to a log (PVRSRV_NEED_PVR_LOG must be defined).
+
+    @def TRUNCATE_64BITS_TO_32BITS
+    @brief Truncates 64 bit value to 32 bit value (with possible precision loss).
+
+    @def TRUNCATE_64BITS_TO_SIZE_T
+    @brief Truncates 64 bit value to size_t value (with possible precision loss).
+
+    @def TRUNCATE_SIZE_T_TO_32BITS
+    @brief Truncates size_t value to 32 bit value (with possible precision loss).
+ */
+
+#endif /* PVR_DEBUG_H */
+
+/******************************************************************************
+ End of file (pvr_debug.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pvr_debugfs.c b/drivers/gpu/drm/img-rogue/pvr_debugfs.c
new file mode 100644
index 000000000000..63c557bc8459
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_debugfs.c
@@ -0,0 +1,611 @@
+/*************************************************************************/ /*!
+@File
+@Title          DebugFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements osdi_impl.h API to provide access to driver's
+                debug data via DebugFS.
+
+                Note about locking in DebugFS module.
+
+                Access to DebugFS is protected against the race where any
+                file could be removed while being accessed or accessed while
+                being removed. Any calls to debugfs_remove() will block
+                until all operations are finished.
+
+                See implementation of proxy file operations (FULL_PROXY_FUNC)
+                and implementation of debugfs_file_[get|put]() in
+                fs/debugfs/file.c in Linux kernel sources for more details.
+
+                Not about locking for sequential files.
+
+                The seq_file objects have a mutex that protects access
+                to all of the file operations hence all of the sequential
+                *read* operations are protected.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvr_debugfs.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_bridge_k.h"
+#include "pvr_uaccess.h"
+#include "osdi_impl.h"
+
+#define _DRIVER_THREAD_ENTER() \
+	do { \
+		PVRSRV_ERROR eLocalError = PVRSRVDriverThreadEnter(NULL); \
+		if (eLocalError != PVRSRV_OK) \
+		{ \
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVDriverThreadEnter failed: %s", \
+				__func__, PVRSRVGetErrorString(eLocalError))); \
+			return OSPVRSRVToNativeError(eLocalError); \
+		} \
+	} while (0)
+
+#define _DRIVER_THREAD_EXIT() \
+	PVRSRVDriverThreadExit(NULL)
+
+#define PVR_DEBUGFS_PVR_DPF_LEVEL PVR_DBG_ERROR
+
+typedef struct DFS_DIR
+{
+	struct dentry *psDirEntry;
+	struct DFS_DIR *psParentDir;
+} DFS_DIR;
+
+typedef struct DFS_ENTRY
+{
+	OSDI_IMPL_ENTRY sImplEntry;
+	DI_ITERATOR_CB sIterCb;
+} DFS_ENTRY;
+
+typedef struct DFS_FILE
+{
+	struct dentry *psFileEntry;
+	struct DFS_DIR *psParentDir;
+	const struct seq_operations *psSeqOps;
+	struct DFS_ENTRY sEntry;
+	DI_ENTRY_TYPE eType;
+} DFS_FILE;
+
+/* ----- native callbacks interface ----------------------------------------- */
+
+static void _WriteData(void *pvNativeHandle, const void *pvData,
+                       IMG_UINT32 uiSize)
+{
+	seq_write(pvNativeHandle, pvData, uiSize);
+}
+
+static void _VPrintf(void *pvNativeHandle, const IMG_CHAR *pszFmt,
+                     va_list pArgs)
+{
+	seq_vprintf(pvNativeHandle, pszFmt, pArgs);
+}
+
+static void _Puts(void *pvNativeHandle, const IMG_CHAR *pszStr)
+{
+	seq_puts(pvNativeHandle, pszStr);
+}
+
+static IMG_BOOL _HasOverflowed(void *pvNativeHandle)
+{
+	struct seq_file *psSeqFile = pvNativeHandle;
+	return seq_has_overflowed(psSeqFile);
+}
+
+static OSDI_IMPL_ENTRY_CB _g_sEntryCallbacks = {
+	.pfnWrite = _WriteData,
+	.pfnVPrintf = _VPrintf,
+	.pfnPuts = _Puts,
+	.pfnHasOverflowed = _HasOverflowed,
+};
+
+/* ----- sequential file operations ----------------------------------------- */
+
+static void *_Start(struct seq_file *psSeqFile, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	void *pvRet = psEntry->sIterCb.pfnStart(&psEntry->sImplEntry, puiPos);
+
+	if (pvRet == DI_START_TOKEN)
+	{
+		return SEQ_START_TOKEN;
+	}
+
+	return pvRet;
+}
+
+static void _Stop(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	psEntry->sIterCb.pfnStop(&psEntry->sImplEntry, pvPriv);
+}
+
+static void *_Next(struct seq_file *psSeqFile, void *pvPriv, loff_t *puiPos)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	return psEntry->sIterCb.pfnNext(&psEntry->sImplEntry, pvPriv, puiPos);
+}
+
+static int _Show(struct seq_file *psSeqFile, void *pvPriv)
+{
+	DFS_ENTRY *psEntry = psSeqFile->private;
+
+	if (pvPriv == SEQ_START_TOKEN)
+	{
+		pvPriv = DI_START_TOKEN;
+	}
+
+	return psEntry->sIterCb.pfnShow(&psEntry->sImplEntry, pvPriv);
+}
+
+static struct seq_operations _g_sSeqOps = {
+	.start = _Start,
+	.stop = _Stop,
+	.next = _Next,
+	.show = _Show
+};
+
+/* ----- file operations ---------------------------------------------------- */
+
+static int _Open(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psINode != NULL && psINode->i_private != NULL,
+	                        "psDFSFile is NULL", -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	psDFSFile = psINode->i_private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_open(psFile, psDFSFile->psSeqOps);
+	}
+	else
+	{
+		/* private data is NULL as it's going to be set below */
+		iRes = single_open(psFile, _Show, NULL);
+	}
+
+	if (iRes == 0)
+	{
+		struct seq_file *psSeqFile = psFile->private_data;
+
+		DFS_ENTRY *psEntry = OSAllocMem(sizeof(*psEntry));
+		if (psEntry == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OSAllocMem() failed", __func__));
+			iRes = -ENOMEM;
+			goto return_;
+		}
+
+		*psEntry = psDFSFile->sEntry;
+		psSeqFile->private = psEntry;
+		psEntry->sImplEntry.pvNative = psSeqFile;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to seq_open psFile, returning %d",
+		        __func__, iRes));
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static int _Close(struct inode *psINode, struct file *psFile)
+{
+	DFS_FILE *psDFSFile = psINode->i_private;
+	DFS_ENTRY *psEntry;
+	int iRes;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* save pointer to DFS_ENTRY */
+	psEntry = ((struct seq_file *) psFile->private_data)->private;
+
+	if (psDFSFile->sEntry.sIterCb.pfnStart != NULL)
+	{
+		iRes = seq_release(psINode, psFile);
+	}
+	else
+	{
+		iRes = single_release(psINode, psFile);
+	}
+
+	/* free DFS_ENTRY allocated in _Open */
+	OSFreeMem(psEntry);
+
+	/* Validation check as seq_release (and single_release which calls it)
+	 * never fail */
+	if (iRes != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release psFile, returning %d",
+		        __func__, iRes));
+	}
+
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Read(struct file *psFile, char __user *pcBuffer,
+                     size_t uiCount, loff_t *puiPos)
+{
+	DFS_FILE *psDFSFile = psFile->f_path.dentry->d_inode->i_private;
+	ssize_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_read(psFile, pcBuffer, uiCount, puiPos);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file, pfnRead() "
+			        "returned %zd", __func__, iRes));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Count = uiCount;
+
+		IMG_CHAR *pcLocalBuffer = OSAllocMem(uiCount);
+		PVR_GOTO_IF_FALSE(pcLocalBuffer != NULL, return_);
+
+		iRes = psEntry->sIterCb.pfnRead(pcLocalBuffer, ui64Count, puiPos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to read from file, pfnRead() "
+			        "returned %zd", __func__, iRes));
+			OSFreeMem(pcLocalBuffer);
+			goto return_;
+		}
+
+		if (pvr_copy_to_user(pcBuffer, pcLocalBuffer, iRes) != 0)
+		{
+			iRes = -1;
+		}
+
+		OSFreeMem(pcLocalBuffer);
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static loff_t _LSeek(struct file *psFile, loff_t iOffset, int iOrigin)
+{
+	DFS_FILE *psDFSFile = psFile->f_path.dentry->d_inode->i_private;
+	loff_t iRes = -1;
+
+	_DRIVER_THREAD_ENTER();
+
+	if (psDFSFile->eType == DI_ENTRY_TYPE_GENERIC)
+	{
+		iRes = seq_lseek(psFile, iOffset, iOrigin);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position in psFile<%p> to offset "
+			        "%lld, iOrigin %d, seq_lseek() returned %lld (dentry='%s')", __func__,
+			        psFile, iOffset, iOrigin, iRes, psFile->f_path.dentry->d_name.name));
+			goto return_;
+		}
+	}
+	else if (psDFSFile->eType == DI_ENTRY_TYPE_RANDOM_ACCESS)
+	{
+		DFS_ENTRY *psEntry = &psDFSFile->sEntry;
+		IMG_UINT64 ui64Pos;
+
+		switch (iOrigin)
+		{
+			case SEEK_SET:
+				ui64Pos = psFile->f_pos + iOffset;
+				break;
+			case SEEK_CUR:
+				ui64Pos = iOffset;
+				break;
+			case SEEK_END:
+				/* not supported as we don't know the file size here */
+				/* fall through */
+			default:
+				return -1;
+		}
+
+		/* only pass the absolute position to the callback, it's up to the
+		 * implementer to determine if the position is valid */
+
+		iRes = psEntry->sIterCb.pfnSeek(ui64Pos,
+		                                psEntry->sImplEntry.pvPrivData);
+		if (iRes < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to set file position to offset "
+			        "%lld, pfnSeek() returned %lld", __func__,
+			        iOffset, iRes));
+			goto return_;
+		}
+
+		psFile->f_pos = ui64Pos;
+	}
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return iRes;
+}
+
+static ssize_t _Write(struct file *psFile, const char __user *pszBuffer,
+                      size_t uiCount, loff_t *puiPos)
+{
+	struct inode *psINode = psFile->f_path.dentry->d_inode;
+	DFS_FILE *psDFSFile = psINode->i_private;
+	DI_ITERATOR_CB *psIter = &psDFSFile->sEntry.sIterCb;
+	IMG_CHAR *pcLocalBuffer;
+	IMG_UINT64 ui64Count;
+	IMG_INT64 i64Res = -EIO;
+	IMG_UINT64 ui64Pos = *puiPos;
+
+	PVR_LOG_RETURN_IF_FALSE(psDFSFile != NULL, "psDFSFile is NULL",
+	                        -EIO);
+	PVR_LOG_RETURN_IF_FALSE(psIter->pfnWrite != NULL, "pfnWrite is NULL",
+	                        -EIO);
+
+	_DRIVER_THREAD_ENTER();
+
+	/* Make sure we allocate the smallest amount of needed memory*/
+	ui64Count = psIter->ui32WriteLenMax;
+	PVR_LOG_GOTO_IF_FALSE(uiCount <= ui64Count, "uiCount too long", return_);
+	ui64Count = MIN(uiCount + 1, ui64Count);
+
+	/* allocate buffer with one additional byte for NUL character */
+	pcLocalBuffer = OSAllocMem(ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(pcLocalBuffer != NULL, "OSAllocMem() failed",
+	                      return_);
+
+	i64Res = pvr_copy_from_user(pcLocalBuffer, pszBuffer, ui64Count);
+	PVR_LOG_GOTO_IF_FALSE(i64Res == 0, "pvr_copy_from_user() failed",
+	                      free_local_buffer_);
+
+	/* ensure that the framework user gets a NUL terminated buffer */
+	pcLocalBuffer[ui64Count - 1] = '\0';
+
+	i64Res = psIter->pfnWrite(pcLocalBuffer, ui64Count, &ui64Pos,
+	                          psDFSFile->sEntry.sImplEntry.pvPrivData);
+	PVR_LOG_GOTO_IF_FALSE(i64Res >= 0, "pfnWrite failed", free_local_buffer_);
+
+	*puiPos = ui64Pos;
+
+free_local_buffer_:
+	OSFreeMem(pcLocalBuffer);
+
+return_:
+	_DRIVER_THREAD_EXIT();
+
+	return i64Res;
+}
+
+static const struct file_operations _g_psFileOpsGen = {
+	.owner = THIS_MODULE,
+	.open = _Open,
+	.release = _Close,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+static const struct file_operations _g_psFileOpsRndAcc = {
+	.owner = THIS_MODULE,
+	.read = _Read,
+	.llseek = _LSeek,
+	.write = _Write,
+};
+
+/* ----- DI implementation interface ---------------------------------------- */
+
+static PVRSRV_ERROR _Init(void)
+{
+	return PVRSRV_OK;
+}
+
+static void _DeInit(void)
+{
+}
+
+static PVRSRV_ERROR _CreateFile(const IMG_CHAR *pszName,
+                                DI_ENTRY_TYPE eType,
+                                const DI_ITERATOR_CB *psIterCb,
+                                void *pvPrivData,
+                                void *pvParentDir,
+                                void **pvFile)
+{
+	DFS_DIR *psParentDir = pvParentDir;
+	DFS_FILE *psFile;
+	umode_t uiMode = S_IFREG;
+	struct dentry *psEntry;
+	const struct file_operations *psFileOps = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvFile != NULL, "pvFile");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pvParentDir != NULL, "pvParentDir");
+
+	switch (eType)
+	{
+		case DI_ENTRY_TYPE_GENERIC:
+			psFileOps = &_g_psFileOpsGen;
+			break;
+		case DI_ENTRY_TYPE_RANDOM_ACCESS:
+			psFileOps = &_g_psFileOpsRndAcc;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "eType invalid in %s()", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto return_;
+	}
+
+	psFile = OSAllocMem(sizeof(*psFile));
+	PVR_LOG_GOTO_IF_NOMEM(psFile, eError, return_);
+
+	uiMode |= psIterCb->pfnShow != NULL || psIterCb->pfnRead != NULL ?
+	        S_IRUGO : 0;
+	uiMode |= psIterCb->pfnWrite != NULL ? S_IWUSR : 0;
+
+	psEntry = debugfs_create_file(pszName, uiMode, psParentDir->psDirEntry,
+	                              psFile, psFileOps);
+	if (IS_ERR_OR_NULL(psEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create debugfs '%s' file",
+		        __func__, pszName));
+
+		eError = psEntry == NULL ?
+		        PVRSRV_ERROR_OUT_OF_MEMORY : PVRSRV_ERROR_INVALID_DEVICE;
+		goto free_file_;
+	}
+
+	psFile->eType = eType;
+	psFile->psSeqOps = &_g_sSeqOps;
+	psFile->sEntry.sIterCb = *psIterCb;
+	psFile->sEntry.sImplEntry.pvPrivData = pvPrivData;
+	psFile->sEntry.sImplEntry.pvNative = NULL;
+	psFile->sEntry.sImplEntry.psCb = &_g_sEntryCallbacks;
+	psFile->psParentDir = psParentDir;
+	psFile->psFileEntry = psEntry;
+
+	*pvFile = psFile;
+
+	return PVRSRV_OK;
+
+free_file_:
+	OSFreeMem(psFile);
+
+return_:
+	return eError;
+}
+
+static void _DestroyFile(void *pvFile)
+{
+	DFS_FILE *psFile = pvFile;
+
+	PVR_ASSERT(psFile != NULL);
+
+	psFile->psFileEntry->d_inode->i_private = NULL;
+
+	debugfs_remove(psFile->psFileEntry);
+	OSFreeMem(psFile);
+}
+
+static PVRSRV_ERROR _CreateDir(const IMG_CHAR *pszName,
+                               void *pvParentDir,
+                               void **ppvDir)
+{
+	DFS_DIR *psNewDir;
+	struct dentry *psDirEntry, *psParentDir = NULL;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pszName != NULL, "pszName");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(ppvDir != NULL, "ppvDir");
+
+	psNewDir = OSAllocMem(sizeof(*psNewDir));
+	PVR_LOG_RETURN_IF_NOMEM(psNewDir, "OSAllocMem");
+
+	psNewDir->psParentDir = pvParentDir;
+
+	if (pvParentDir != NULL)
+	{
+		psParentDir = psNewDir->psParentDir->psDirEntry;
+	}
+
+	psDirEntry = debugfs_create_dir(pszName, psParentDir);
+	if (IS_ERR_OR_NULL(psDirEntry))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Cannot create '%s' debugfs directory",
+		        __func__, pszName));
+		OSFreeMem(psNewDir);
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	psNewDir->psDirEntry = psDirEntry;
+	*ppvDir = psNewDir;
+
+	return PVRSRV_OK;
+}
+
+static void _DestroyDir(void *pvDir)
+{
+	DFS_DIR *psDir = pvDir;
+
+	PVR_ASSERT(psDir != NULL);
+
+	debugfs_remove(psDir->psDirEntry);
+	OSFreeMem(psDir);
+}
+
+PVRSRV_ERROR PVRDebugFsRegister(void)
+{
+	OSDI_IMPL_CB sImplCb = {
+		.pfnInit = _Init,
+		.pfnDeInit = _DeInit,
+		.pfnCreateEntry = _CreateFile,
+		.pfnDestroyEntry = _DestroyFile,
+		.pfnCreateGroup = _CreateDir,
+		.pfnDestroyGroup = _DestroyDir
+	};
+
+	return DIRegisterImplementation("debugfs", &sImplCb);
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_debugfs.h b/drivers/gpu/drm/img-rogue/pvr_debugfs.h
new file mode 100644
index 000000000000..23ae55b12069
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_debugfs.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Title          DebugFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DEBUGFS_H
+#define PVR_DEBUGFS_H
+
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR PVRDebugFsRegister(void);
+
+#endif /* PVR_DEBUGFS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_dicommon.h b/drivers/gpu/drm/img-rogue/pvr_dicommon.h
new file mode 100644
index 000000000000..c729dde2fab6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_dicommon.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Debug Information (DI) common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Debug Information (DI) common types and definitions included
+                in both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DICOMMON_H
+#define PVR_DICOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*! Maximum DI entry path length including the null byte. */
+#define DI_IMPL_BRG_PATH_LEN 64
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_DICOMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_dma_resv.h b/drivers/gpu/drm/img-rogue/pvr_dma_resv.h
new file mode 100644
index 000000000000..a51c9de84ada
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_dma_resv.h
@@ -0,0 +1,80 @@
+/*************************************************************************/ /*!
+@Title          Kernel reservation object compatibility header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __PVR_DMA_RESV_H__
+#define __PVR_DMA_RESV_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <linux/dma-resv.h>
+#else
+#include <linux/reservation.h>
+
+/* Reservation object types */
+#define dma_resv			reservation_object
+#define dma_resv_list			reservation_object_list
+
+/* Reservation object functions */
+#define dma_resv_add_excl_fence		reservation_object_add_excl_fence
+#define dma_resv_add_shared_fence	reservation_object_add_shared_fence
+#define dma_resv_fini			reservation_object_fini
+#define dma_resv_get_excl		reservation_object_get_excl
+#define dma_resv_get_list		reservation_object_get_list
+#define dma_resv_held			reservation_object_held
+#define dma_resv_init			reservation_object_init
+#define dma_resv_reserve_shared		reservation_object_reserve_shared
+#define dma_resv_test_signaled_rcu	reservation_object_test_signaled_rcu
+#define dma_resv_wait_timeout_rcu	reservation_object_wait_timeout_rcu
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0))
+
+#define dma_resv_shared_list   dma_resv_get_list
+#define dma_resv_excl_fence    dma_resv_get_excl
+#define dma_resv_wait_timeout  dma_resv_wait_timeout_rcu
+#define dma_resv_test_signaled dma_resv_test_signaled_rcu
+#define dma_resv_get_fences    dma_resv_get_fences_rcu
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)) */
+
+#endif /* __PVR_DMA_RESV_H__ */
diff --git a/drivers/gpu/drm/img-rogue/pvr_drm.c b/drivers/gpu/drm/img-rogue/pvr_drm.c
new file mode 100644
index 000000000000..98e5eaad40a1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_drm.c
@@ -0,0 +1,570 @@
+/*
+ * @File
+ * @Title       PowerVR DRM driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#include <drm/drm.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_print.h>
+#include <linux/dma-mapping.h>
+#else
+#include <drm/drmP.h> /* include before drm_crtc.h for kernels older than 3.9 */
+#endif
+
+#include <drm/drm_crtc.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+
+#include "module_common.h"
+#include "pvr_drm.h"
+#include "pvr_drv.h"
+#include "pvrversion.h"
+#include "services_kernel_client.h"
+#include "pvr_sync_ioctl_drm.h"
+#include "physmem_dmabuf_internal.h"
+
+#include "kernel_compatibility.h"
+
+#include "dkf_server.h"
+#include "dkp_impl.h"
+
+#define PVR_DRM_DRIVER_NAME PVR_DRM_NAME
+#define PVR_DRM_DRIVER_DESC "Imagination Technologies PVR DRM"
+#define	PVR_DRM_DRIVER_DATE "20170530"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#define	PVR_DRM_DRIVER_PRIME 0
+#else
+#define	PVR_DRM_DRIVER_PRIME DRIVER_PRIME
+#endif
+
+/*
+ * Protects global PVRSRV_DATA on a multi device system. i.e. this is used to
+ * protect the PVRSRVCommonDeviceXXXX() APIs in the Server common layer which
+ * are not re-entrant for device creation and initialisation.
+ */
+static DEFINE_MUTEX(g_device_mutex);
+
+/* Executed before sleep/suspend-to-RAM/S3. During this phase the content
+ * of the video memory is preserved (copied to system RAM). This step is
+ * necessary because the device can be powered off and the content of the
+ * video memory lost.
+ */
+static int pvr_pm_suspend(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("device %p\n", dev);
+
+	return PVRSRVDeviceSuspend(ddev);
+}
+
+/* Executed after the system is woken up from sleep/suspend-to-RAM/S3. This
+ * phase restores the content of the video memory from the system RAM.
+ */
+static int pvr_pm_resume(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("device %p\n", dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+/* Executed before the hibernation image is created. This callback allows to
+ * preserve the content of the video RAM into the system RAM which in turn
+ * is then stored into a disk.
+ */
+static int pvr_pm_freeze(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceSuspend(ddev);
+}
+
+/* Executed after the hibernation image is created or if the creation of the
+ * image has failed. This callback should undo whatever was done in
+ * pvr_pm_freeze to allow the device to operate in the same way as before the
+ * call to pvr_pm_freeze.
+ */
+static int pvr_pm_thaw(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+/* Executed after the hibernation image is created. This callback should not
+ * preserve the content of the video memory since this was already done
+ * in pvr_pm_freeze.
+ *
+ * Note: from the tests performed on a TestChip this callback is not executed
+ *       and driver's pvr_shutdown() is executed instead.
+ */
+static int pvr_pm_poweroff(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	PVRSRVDeviceShutdown(ddev);
+
+	return 0;
+}
+
+/* Executed after the content of the system memory is restored from the
+ * hibernation image. This callback restored video RAM from the system RAM
+ * and performs any necessary device setup required for the device to operate
+ * properly.
+ */
+static int pvr_pm_restore(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+
+	DRM_DEBUG_DRIVER("%s(): device %p\n", __func__, dev);
+
+	return PVRSRVDeviceResume(ddev);
+}
+
+const struct dev_pm_ops pvr_pm_ops = {
+	/* Sleep (suspend-to-RAM/S3) callbacks.
+	 * This mode saves the content of the video RAM to the system RAM and
+	 * powers off the device to reduce the power consumption. Because the
+	 * video RAM can be powered off, it needs to be preserved beforehand.
+	 */
+	.suspend = pvr_pm_suspend,
+	.resume = pvr_pm_resume,
+
+	/* Hibernation (suspend-to-disk/S4) callbacks.
+	 * This mode saves the content of the video RAM to the system RAM and then
+	 * dumps the system RAM to disk (swap partition or swap file). The system
+	 * then powers off. After power on the system RAM content is loaded from
+	 * the disk and then video RAM is restored from the system RAM.
+	 *
+	 * The procedure is executed in following order
+	 *
+	 * - Suspend-to-disk is triggered
+	 *   At this point the OS goes through the list of all registered devices and
+	 *   calls provided callbacks.
+	 * -- pvr_pm_freeze() is called
+	 *         The GPU is powered of and submitting new work is blocked.
+	 *         The content of the video RAM is saved to the system RAM, and
+	 *         other actions required to suspend the device are performed.
+	 * -- system RAM image is created and saved on the disk
+	 *         The disk now contains a snapshot for the DDK Driver for the
+	 *         moment when pvr_pm_freeze() was called.
+	 * -- pvr_pm_thaw() is called
+	 *         All actions taken in pvr_pm_freeze() are undone. The memory
+	 *         allocated for the video RAM is freed and all actions necessary
+	 *         to bring the device to operational state are taken.
+	 *         This makes sure that regardless if image was created successfully
+	 *         or not the device remains operational.
+	 *
+	 * - System is powered off
+	 * -- pvr_shutdown() is called
+	 *         No actions are required beside powering off the GPU.
+	 *
+	 * - System is powered up
+	 * -- system RAM image is read from the disk
+	 *         This restores the snapshot of the DDK driver along with the saved
+	 *         video RAM buffer.
+	 * -- pvr_pm_restore() is called
+	 *         Video RAM is restored from the buffer located in the system RAM.
+	 *         Actions to reset the device and bring it back to working state
+	 *         are taken. Video RAM buffer is freed.
+	 *         In summary the same procedure as in the case of pvr_pm_thaw() is
+	 *         performed.
+	 */
+	.freeze = pvr_pm_freeze,
+	.thaw = pvr_pm_thaw,
+	.poweroff = pvr_pm_poweroff,
+	.restore = pvr_pm_restore,
+};
+
+#if defined(SUPPORT_LINUX_FDINFO)
+static void pvr_drm_show_drm_info(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	int pid, void *hPrivHandle)
+{
+	struct pvr_drm_private *priv;
+	struct drm_device *pdev;
+
+	pdev = (struct drm_device *)hPrivHandle;
+
+	/* The only field possibly valid in 'priv' is the dev_node field */
+	priv = (struct pvr_drm_private *)pdev->dev_private;
+
+	if (priv->dev_node == psDevNode) {
+
+		/* For kernels post 6.5.0 the mandatory driver fields are produced
+		 * by the 'drm_show_fdinfo' routine in the kernel.
+		 * Avoid duplicating this information here.
+		 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0))
+		PVRDKPOutput(priv->hDeviceDKPRef,
+		             "drm-driver:\t%s\n",
+		             PVR_DRM_DRIVER_NAME);
+#if defined(CONFIG_PCI)
+		if (dev_is_pci(pdev->dev)) {
+			struct pci_dev *pcidev = to_pci_dev(pdev->dev);
+
+			PVRDKPOutput(priv->hDeviceDKPRef,
+			             "drm-pdev:\t%04x:%02x:%02x.%d\n",
+			             pci_domain_nr(pcidev->bus),
+			             pcidev->bus->number,
+			             PCI_SLOT(pcidev->devfn),
+			             PCI_FUNC(pcidev->devfn));
+		}
+#endif /* defined(CONFIG_PCI) */
+#endif	/* LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0) */
+	}
+}
+#endif	/* SUPPORT_LINUX_FDINFO */
+
+int pvr_drm_load(struct drm_device *ddev, unsigned long flags)
+{
+	struct pvr_drm_private *priv;
+	enum PVRSRV_ERROR_TAG srv_err;
+	int err, deviceId;
+
+	DRM_DEBUG_DRIVER("device %p\n", ddev->dev);
+
+	dev_set_drvdata(ddev->dev, ddev);
+
+	if (ddev->render)
+		deviceId = ddev->render->index;
+	else /* when render node is NULL, fallback to primary node */
+		deviceId = ddev->primary->index;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		err = -ENOMEM;
+		goto err_exit;
+	}
+	ddev->dev_private = priv;
+
+	if (!ddev->dev->dma_parms)
+		ddev->dev->dma_parms = &priv->dma_parms;
+	dma_set_max_seg_size(ddev->dev, DMA_BIT_MASK(32));
+
+	mutex_lock(&g_device_mutex);
+
+	srv_err = PVRSRVCommonDeviceCreate(ddev->dev, deviceId, &priv->dev_node);
+	if (srv_err != PVRSRV_OK) {
+		DRM_ERROR("failed to create device node for device %p (%s)\n",
+			  ddev->dev, PVRSRVGetErrorString(srv_err));
+		if (srv_err == PVRSRV_ERROR_PROBE_DEFER)
+			err = -EPROBE_DEFER;
+		else
+			err = -ENODEV;
+		goto err_unset_dma_parms;
+	}
+
+	err = PVRSRVDeviceInit(priv->dev_node);
+	if (err) {
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+			  ddev->dev, err);
+		goto err_device_destroy;
+	}
+
+	drm_mode_config_init(ddev);
+
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+	srv_err = PVRSRVCommonDeviceInitialise(priv->dev_node);
+	if (srv_err != PVRSRV_OK) {
+		err = -ENODEV;
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+			  ddev->dev, err);
+		goto err_device_deinit;
+	}
+#endif
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	srv_err = PVRSRVRegisterDKP(ddev,
+	                            "drm-pvr-drm",
+	                            pvr_drm_show_drm_info,
+	                            DKP_CONNECTION_FLAG_ALL,
+	                            &priv->hDeviceDKPRef);
+
+	if (srv_err != PVRSRV_OK) {
+		err = -ENODEV;
+		DRM_ERROR("device %p initialisation failed (err=%d)\n",
+				  ddev->dev, err);
+		goto err_device_deinit;
+	}
+#endif
+
+	mutex_unlock(&g_device_mutex);
+
+	return 0;
+
+#if defined(SUPPORT_LINUX_FDINFO) || (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+err_device_deinit:
+#endif
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+	drm_mode_config_cleanup(ddev);
+	PVRSRVDeviceDeinit(priv->dev_node);
+#endif
+err_device_destroy:
+	PVRSRVCommonDeviceDestroy(priv->dev_node);
+err_unset_dma_parms:
+	mutex_unlock(&g_device_mutex);
+	if (ddev->dev->dma_parms == &priv->dma_parms)
+		ddev->dev->dma_parms = NULL;
+	kfree(priv);
+err_exit:
+	return err;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+int pvr_drm_unload(struct drm_device *ddev)
+#else
+void pvr_drm_unload(struct drm_device *ddev)
+#endif
+{
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	DRM_DEBUG_DRIVER("device %p\n", ddev->dev);
+
+	drm_mode_config_cleanup(ddev);
+
+	PVRSRVDeviceDeinit(priv->dev_node);
+
+	mutex_lock(&g_device_mutex);
+	PVRSRVCommonDeviceDestroy(priv->dev_node);
+	mutex_unlock(&g_device_mutex);
+
+	if (ddev->dev->dma_parms == &priv->dma_parms)
+		ddev->dev->dma_parms = NULL;
+
+#if defined(SUPPORT_LINUX_FDINFO)
+	PVRSRVUnRegisterDKP(ddev, priv->hDeviceDKPRef);
+#endif
+
+	kfree(priv);
+	ddev->dev_private = NULL;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	return 0;
+#endif
+}
+
+static int pvr_drm_open(struct drm_device *ddev, struct drm_file *dfile)
+{
+#if (PVRSRV_DEVICE_INIT_MODE != PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	struct pvr_drm_private *priv = ddev->dev_private;
+	int err;
+#endif
+
+	if (!try_module_get(THIS_MODULE)) {
+		DRM_ERROR("failed to get module reference\n");
+		return -ENOENT;
+	}
+
+#if (PVRSRV_DEVICE_INIT_MODE != PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	err = PVRSRVDeviceServicesOpen(priv->dev_node, dfile);
+	if (err)
+		module_put(THIS_MODULE);
+
+	return err;
+#else
+	return 0;
+#endif
+}
+
+static void pvr_drm_release(struct drm_device *ddev, struct drm_file *dfile)
+{
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	PVRSRVDeviceRelease(priv->dev_node, dfile);
+
+	module_put(THIS_MODULE);
+}
+
+static struct drm_ioctl_desc pvr_drm_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(PVR_SRVKM_CMD, PVRSRV_BridgeDispatchKM,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SRVKM_INIT, drm_pvr_srvkm_init,
+			  DRM_RENDER_ALLOW),
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) && !defined(USE_PVRSYNC_DEVNODE)
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_RENAME_CMD, pvr_sync_rename_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_FORCE_SW_ONLY_CMD, pvr_sync_force_sw_only_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SW_SYNC_CREATE_FENCE_CMD, pvr_sw_sync_create_fence_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SW_SYNC_INC_CMD, pvr_sw_sync_inc_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_EXP_FENCE_SYNC_FORCE_CMD, pvr_sync_ioctl_force_exp_only,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(PVR_SYNC_CREATE_EXPORT_FENCE_CMD,
+			  pvr_export_fence_sync_create_fence_ioctl,
+			  DRM_RENDER_ALLOW),
+#endif
+};
+
+#if defined(CONFIG_COMPAT)
+static long pvr_compat_ioctl(struct file *file, unsigned int cmd,
+			     unsigned long arg)
+{
+	unsigned int nr = DRM_IOCTL_NR(cmd);
+
+	if (nr < DRM_COMMAND_BASE)
+		return drm_compat_ioctl(file, cmd, arg);
+
+	return drm_ioctl(file, cmd, arg);
+}
+#endif /* defined(CONFIG_COMPAT) */
+
+#if defined(SUPPORT_LINUX_FDINFO)
+/*
+ * Produce the PVR specific fdinfo (utilization figures etc.) when queried.
+ *
+ * For kernels post 6.5 there is a helper function called 'drm_show_fdinfo'
+ * which will generate the mandatory keys (drm-driver, drm-pdev, drm-client-id)
+ * so we do not need to generate these if running on a later kernel etc.
+ *
+ */
+static void pvr_show_fdinfo(struct seq_file *seq_file, struct file *file)
+{
+	struct drm_file *dfile = file->private_data;
+	struct drm_device *dev = dfile->minor->dev;
+	struct drm_printer p = drm_seq_file_printer(seq_file);
+	PVRSRV_CONNECTION_PRIV *pvr_connection = dfile->driver_priv;
+	struct pvr_drm_private *priv;
+	int my_pid;
+
+	/* Check for NULL driver_priv to avoid kernel crash */
+	if (!pvr_connection) {
+		/* Grab the PID first for logging */
+		my_pid = dfile->pid->numbers[0].nr;
+		DRM_DEBUG_DRIVER("driver_priv is NULL for PID %d, skipping PVR fdinfo\n", my_pid);
+		goto call_drm_show_fdinfo;
+	}
+
+	/* Grab the PID from the associated drm_file->pid->numbers[0].nr */
+	my_pid = dfile->pid->numbers[0].nr;
+
+	priv = (struct pvr_drm_private *)dev->dev_private;
+
+	/* Additional safety check for dev_private */
+	if (!priv) {
+		DRM_DEBUG_DRIVER("dev_private is NULL for PID %d, skipping PVR fdinfo\n", my_pid);
+		goto call_drm_show_fdinfo;
+	}
+
+	/* Generate driver-specific keys */
+	PVRDKFTraverse((DKF_VPRINTF_FUNC*)drm_vprintf,
+	               &p,
+	               priv->dev_node,
+	               my_pid,
+	               pvr_connection->ui32Type);
+
+call_drm_show_fdinfo:
+
+	/* Call into OS-specific drm_show_fdinfo if it is supported */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0))
+	drm_show_fdinfo(seq_file, file);
+#endif	/* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0) */
+}
+#endif /* SUPPORT_LINUX_FDINFO */
+
+const struct file_operations pvr_drm_fops = {
+	.owner			= THIS_MODULE,
+	.open			= drm_open,
+	.release		= drm_release,
+	.unlocked_ioctl		= drm_ioctl,
+#if defined(CONFIG_COMPAT)
+	.compat_ioctl		= pvr_compat_ioctl,
+#endif
+	.mmap			= PVRSRV_MMap,
+	.poll			= drm_poll,
+	.read			= drm_read,
+#if defined(SUPPORT_LINUX_FDINFO)
+	.show_fdinfo	= pvr_show_fdinfo,
+#endif /* SUPPORT_LINUX_FDINFO */
+	.fop_flags              = FOP_UNSIGNED_OFFSET,
+};
+
+const struct drm_driver pvr_drm_generic_driver = {
+	.driver_features	= DRIVER_MODESET | DRIVER_RENDER |
+				  DRIVER_GEM | PVR_DRM_DRIVER_PRIME,
+
+	.load			= NULL,
+	.unload			= NULL,
+	.open			= pvr_drm_open,
+	.postclose		= pvr_drm_release,
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	/* prime_fd_to_handle is not supported */
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_prime_export	= PhysmemGEMPrimeExport,
+	.gem_free_object	= PhysmemGEMObjectFree,
+#endif
+	.ioctls			= pvr_drm_ioctls,
+	.num_ioctls		= ARRAY_SIZE(pvr_drm_ioctls),
+	.fops			= &pvr_drm_fops,
+
+	.name			= PVR_DRM_DRIVER_NAME,
+	.desc			= PVR_DRM_DRIVER_DESC,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+	.date			= PVR_DRM_DRIVER_DATE,
+#endif
+	.major			= PVRVERSION_MAJ,
+	.minor			= PVRVERSION_MIN,
+	.patchlevel		= PVRVERSION_BUILD,
+};
diff --git a/drivers/gpu/drm/img-rogue/pvr_drm.h b/drivers/gpu/drm/img-rogue/pvr_drm.h
new file mode 100644
index 000000000000..bc76ddf9b597
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_drm.h
@@ -0,0 +1,164 @@
+/*
+ * @File
+ * @Title       PVR DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_DRM_H__)
+#define __PVR_DRM_H__
+
+#include <linux/types.h>
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+/*
+ * IMPORTANT:
+ * All structures below are designed to be the same size when compiled for 32
+ * and/or 64 bit architectures, i.e. there should be no compiler inserted
+ * padding. This is achieved by sticking to the following rules:
+ * 1) only use fixed width types
+ * 2) always naturally align fields by arranging them appropriately and by using
+ *    padding fields when necessary
+ *
+ * These rules should _always_ be followed when modifying or adding new
+ * structures to this file.
+ */
+
+struct drm_pvr_srvkm_cmd {
+	__u32 bridge_id;
+	__u32 bridge_func_id;
+	__u64 in_data_ptr;
+	__u64 out_data_ptr;
+	__u32 in_data_size;
+	__u32 out_data_size;
+};
+
+struct pvr_sync_rename_ioctl_data {
+	char szName[32];
+};
+
+struct pvr_sw_sync_create_fence_data {
+	char name[32];
+	__s32 fence;
+	__u32 pad;
+	__u64 sync_pt_idx;
+};
+
+typedef struct pvr_sw_sync_create_fence_data pvr_exp_sync_create_fence_data_t;
+
+struct pvr_sw_timeline_advance_data {
+	__u64 sync_pt_idx;
+};
+
+#define PVR_SRVKM_SERVICES_INIT  1
+#define PVR_SRVKM_SYNC_INIT 2
+#define PVR_SRVKM_SYNC_EXP_FENCE_INIT 3
+#if defined(SUPPORT_LINUX_OSPAGE_MIGRATION)
+#define PVR_SRVKM_SERVICES_PAGE_MIGRATE_INIT 4
+#endif
+struct drm_pvr_srvkm_init_data {
+	__u32 init_module;
+};
+
+/* Values used to configure the PVRSRV_DEVICE_INIT_MODE tunable (Linux-only) */
+#define PVRSRV_LINUX_DEV_INIT_ON_PROBE   1
+#define PVRSRV_LINUX_DEV_INIT_ON_OPEN    2
+#define PVRSRV_LINUX_DEV_INIT_ON_CONNECT 3
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+
+/* PVR Services command */
+#define DRM_PVR_SRVKM_CMD			0
+
+/* PVR Sync commands */
+#define DRM_PVR_SYNC_RENAME_CMD			1
+#define DRM_PVR_SYNC_FORCE_SW_ONLY_CMD		2
+
+/* PVR Software Sync commands */
+#define DRM_PVR_SW_SYNC_CREATE_FENCE_CMD	3
+#define DRM_PVR_SW_SYNC_INC_CMD			4
+
+/* PVR Services Render Device Init command */
+#define DRM_PVR_SRVKM_INIT		5
+
+/* PVR Services Export Fence Sync commands */
+#define DRM_PVR_EXP_FENCE_SYNC_FORCE_CMD	6
+#define DRM_PVR_SYNC_CREATE_EXPORT_FENCE_CMD	7
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define	DRM_IOCTL_PVR_SRVKM_CMD	\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SRVKM_CMD, \
+		 struct drm_pvr_srvkm_cmd)
+
+#define DRM_IOCTL_PVR_SYNC_RENAME_CMD \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PVR_SYNC_RENAME_CMD, \
+		struct pvr_sync_rename_ioctl_data)
+
+#define DRM_IOCTL_PVR_SYNC_FORCE_SW_ONLY_CMD \
+	DRM_IO(DRM_COMMAND_BASE + DRM_PVR_SYNC_FORCE_SW_ONLY_CMD)
+
+#define	DRM_IOCTL_PVR_SW_SYNC_CREATE_FENCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SW_SYNC_CREATE_FENCE_CMD, \
+		struct pvr_sw_sync_create_fence_data)
+
+#define DRM_IOCTL_PVR_SW_SYNC_INC_CMD \
+	DRM_IOR(DRM_COMMAND_BASE + DRM_PVR_SW_SYNC_INC_CMD, \
+		struct pvr_sw_timeline_advance_data)
+
+#define DRM_IOCTL_PVR_SRVKM_INIT \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PVR_SRVKM_INIT, \
+		struct drm_pvr_srvkm_init_data)
+
+#define DRM_IOCTL_PVR_EXP_FENCE_SYNC_FORCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_EXP_FENCE_SYNC_FORCE_CMD, \
+		pvr_exp_sync_create_fence_data_t)
+
+#define DRM_IOCTL_PVR_SYNC_CREATE_EXPORT_FENCE_CMD \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PVR_SYNC_CREATE_EXPORT_FENCE_CMD, \
+		pvr_exp_sync_create_fence_data_t)
+
+#endif /* defined(__PVR_DRM_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_drv.h b/drivers/gpu/drm/img-rogue/pvr_drv.h
new file mode 100644
index 000000000000..827145d60c50
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_drv.h
@@ -0,0 +1,116 @@
+/*
+ * @File
+ * @Title       PowerVR DRM driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_DRV_H__)
+#define __PVR_DRV_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#include <linux/device.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <linux/pm.h>
+
+struct file;
+struct _PVRSRV_DEVICE_NODE_;
+struct workqueue_struct;
+struct vm_area_struct;
+
+/* This structure is used to store Linux specific per-device information. */
+struct pvr_drm_private {
+	struct _PVRSRV_DEVICE_NODE_ *dev_node;
+
+	/*
+	 * This is needed for devices that don't already have their own dma
+	 * parameters structure, e.g. platform devices, and, if necessary, will
+	 * be assigned to the 'struct device' during device initialisation. It
+	 * should therefore never be accessed directly via this structure as
+	 * this may not be the version of dma parameters in use.
+	 */
+	struct device_dma_parameters dma_parms;
+
+	/* PVR Sync debug notify handle */
+	void *sync_debug_notify_handle;
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+	/* Only used in fence sync as sync_debug_notify_handle is used
+	 * to print a header only. Content is registered separately.
+	 * Used to print foreign sync debug
+	 */
+	void *sync_foreign_debug_notify_handle;
+#endif
+
+	/* Flag stating if the device was suspended/resumed. If this is 0 then
+	 * the device was either resumed or no suspend was called but if 1 then
+	 * the OS called suspend on this device.
+	 */
+	atomic_t suspended;
+
+	/* Padding for 8-byte alignment */
+	unsigned int uiPad1;
+
+	/* Handle associated with DKP for this DRM device */
+	void *hDeviceDKPRef;
+};
+
+extern const struct dev_pm_ops pvr_pm_ops;
+extern const struct drm_driver pvr_drm_generic_driver;
+extern const struct file_operations pvr_drm_fops;
+
+int pvr_drm_load(struct drm_device *ddev, unsigned long flags);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+int pvr_drm_unload(struct drm_device *ddev);
+#else
+void pvr_drm_unload(struct drm_device *ddev);
+#endif
+
+int PVRSRV_BridgeDispatchKM(struct drm_device *dev, void *arg,
+			    struct drm_file *file);
+int PVRSRV_MMap(struct file *file, struct vm_area_struct *ps_vma);
+
+#endif /* !defined(__PVR_DRV_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_dvfs.h b/drivers/gpu/drm/img-rogue/pvr_dvfs.h
new file mode 100644
index 000000000000..e09d43d77e01
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_dvfs.h
@@ -0,0 +1,148 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs.h
+@Title          System level interface for DVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_H
+#define PVR_DVFS_H
+
+#if defined(SUPPORT_LINUX_DVFS)
+ #include <linux/devfreq.h>
+ #include <linux/thermal.h>
+
+ #if defined(CONFIG_POWERVR_THERMAL)
+  #include <linux/devfreq_cooling.h>
+ #endif
+
+ #include <linux/pm_opp.h>
+#endif
+
+#include "img_types.h"
+
+typedef void (*PFN_SYS_DEV_DVFS_SET_FREQUENCY)(IMG_HANDLE hSysData, IMG_UINT32 ui32Freq);
+typedef void (*PFN_SYS_DEV_DVFS_SET_VOLTAGE)(IMG_HANDLE hSysData, IMG_UINT32 ui32Volt);
+
+typedef struct _IMG_OPP_
+{
+	IMG_UINT32			ui32Volt;
+	/*
+	 * Unit of frequency in Hz.
+	 */
+	IMG_UINT32			ui32Freq;
+} IMG_OPP;
+
+typedef struct _IMG_DVFS_DEVICE_CFG_
+{
+	const IMG_OPP  *pasOPPTable;
+	IMG_UINT32      ui32OPPTableSize;
+#if defined(SUPPORT_LINUX_DVFS)
+	IMG_UINT32      ui32PollMs;
+#endif
+	IMG_BOOL        bIdleReq;
+	IMG_BOOL        bDTConfig;
+	PFN_SYS_DEV_DVFS_SET_FREQUENCY  pfnSetFrequency;
+	PFN_SYS_DEV_DVFS_SET_VOLTAGE    pfnSetVoltage;
+
+#if defined(CONFIG_POWERVR_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+	struct devfreq_cooling_power *psPowerOps;
+#endif
+} IMG_DVFS_DEVICE_CFG;
+
+#if defined(SUPPORT_LINUX_DVFS)
+typedef struct _IMG_DVFS_GOVERNOR_
+{
+	IMG_BOOL			bEnabled;
+} IMG_DVFS_GOVERNOR;
+
+typedef struct _IMG_DVFS_GOVERNOR_CFG_
+{
+	IMG_UINT32			ui32UpThreshold;
+	IMG_UINT32			ui32DownDifferential;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* custom thresholds */
+	IMG_UINT32			uiNumMembus;
+#endif
+} IMG_DVFS_GOVERNOR_CFG;
+#endif
+
+#if defined(__linux__)
+#if defined(SUPPORT_LINUX_DVFS)
+typedef enum
+{
+	PVR_DVFS_STATE_NONE	= 0,
+	PVR_DVFS_STATE_INIT_PENDING,
+	PVR_DVFS_STATE_READY,
+	PVR_DVFS_STATE_OFF,
+	PVR_DVFS_STATE_DEINIT
+} PVR_DVFS_STATE;
+
+typedef struct _IMG_DVFS_DEVICE_
+{
+	struct dev_pm_opp		*psOPP;
+	struct devfreq			*psDevFreq;
+	PVR_DVFS_STATE		eState;
+	IMG_HANDLE			hGpuUtilUserDVFS;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	IMG_DVFS_GOVERNOR_CFG data;
+	IMG_BOOL			bGovernorReady;
+#else
+	struct devfreq_simple_ondemand_data data;
+#endif
+#if defined(CONFIG_POWERVR_THERMAL)
+	struct thermal_cooling_device	*psDevfreqCoolingDevice;
+#endif
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	struct pvr_profiling_device *psProfilingDevice;
+#endif
+} IMG_DVFS_DEVICE;
+#endif
+
+typedef struct _IMG_DVFS_
+{
+#if defined(SUPPORT_LINUX_DVFS)
+	IMG_DVFS_DEVICE			sDVFSDevice;
+	IMG_DVFS_GOVERNOR		sDVFSGovernor;
+	IMG_DVFS_GOVERNOR_CFG	sDVFSGovernorCfg;
+#endif
+	IMG_DVFS_DEVICE_CFG		sDVFSDeviceCfg;
+} PVRSRV_DVFS;
+#endif/* (__linux__) */
+
+#endif /* PVR_DVFS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_dvfs_common.c b/drivers/gpu/drm/img-rogue/pvr_dvfs_common.c
new file mode 100644
index 000000000000..30640d542b0e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_dvfs_common.c
@@ -0,0 +1,363 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_common.c
+@Title          PowerVR devfreq device common utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux DVFS and PDVFS shared code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(NO_HARDWARE)
+
+#include <linux/version.h>
+#include <linux/device.h>
+#include <drm/drm.h>
+#if defined(CONFIG_PM_OPP)
+#include <linux/pm_opp.h>
+#endif
+
+#include "pvrsrv.h"
+
+/*
+ * Common DVFS support code shared between SUPPORT_LINUX_DVFS and
+ * SUPPORT_PDVFS, primarily for OPP table support.
+ *
+ * Note that PDVFS implements the Linux/OS devfreq module in
+ * the firmware, so no devfreq API calls should be used here.
+ */
+#include "pvr_dvfs.h"
+#include "pvr_dvfs_common.h"
+
+#include "kernel_compatibility.h"
+
+
+/*************************************************************************/ /*!
+@Function       GetOPPValues
+
+@Description    Common code to store the OPP points in the freq_table,
+                for use with the IMG DVFS/devfreq implementation, or with
+                the Proactive DVFS.
+                Requires CONFIG_PM_OPP support in the kernel.
+
+@Input          dev        OS Device node
+@Output         min_freq   Min clock freq (Hz)
+@Output         min_volt   Min voltage (V)
+@Output         max_freq   Max clock freq (Hz)
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(CONFIG_PM_OPP)
+int GetOPPValues(struct device *dev,
+                 unsigned long *min_freq,
+                 unsigned long *min_volt,
+                 unsigned long *max_freq,
+                 struct pvr_opp_freq_table *pvr_freq_table)
+{
+	struct dev_pm_opp *opp;
+	int count, i, err = 0;
+	unsigned long freq;
+
+	unsigned long *freq_table;
+
+	count = dev_pm_opp_get_opp_count(dev);
+	if (count <= 0)
+	{
+		dev_err(dev, "Could not fetch OPP count, %d\n", count);
+		return -EINVAL;
+	}
+
+	dev_info(dev, "Found %d OPP points.\n", count);
+
+	freq_table = devm_kcalloc(dev, count, sizeof(*freq_table), GFP_ATOMIC);
+	if (! freq_table)
+	{
+		return -ENOMEM;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	/* Start RCU read-side critical section to map frequency to OPP */
+	rcu_read_lock();
+#endif
+
+	/* Iterate over OPP table; Iteration 0 finds "opp w/ freq >= 0 Hz". */
+	freq = 0;
+	opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+	if (IS_ERR(opp))
+	{
+		err = PTR_ERR(opp);
+		dev_err(dev, "Couldn't find lowest frequency, %d\n", err);
+		goto exit;
+	}
+
+	*min_volt = dev_pm_opp_get_voltage(opp);
+	*max_freq = *min_freq = freq_table[0] = freq;
+	dev_info(dev, "opp[%d/%d]: (%lu Hz, %lu uV)\n", 1, count, freq, *min_volt);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	dev_pm_opp_put(opp);
+#endif
+
+	/* Iteration i > 0 finds "opp w/ freq >= (opp[i-1].freq + 1)". */
+	for (i = 1; i < count; i++)
+	{
+		freq++;
+		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+		if (IS_ERR(opp))
+		{
+			err = PTR_ERR(opp);
+			dev_err(dev, "Couldn't find %dth frequency, %d\n", i, err);
+			goto exit;
+		}
+
+		freq_table[i] = freq;
+		*max_freq = freq;
+		dev_info(dev,
+				 "opp[%d/%d]: (%lu Hz, %lu uV)\n",
+				  i + 1,
+				  count,
+				  freq,
+				  dev_pm_opp_get_voltage(opp));
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+		dev_pm_opp_put(opp);
+#endif
+	}
+
+exit:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#endif
+
+	if (!err)
+	{
+		pvr_freq_table->freq_table = freq_table;
+		pvr_freq_table->num_levels = count;
+	}
+	else
+	{
+		devm_kfree(dev, freq_table);
+	}
+
+	return err;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       DVFSCopyOPPTable
+
+@Description    Copies the OPP points (voltage/frequency) to the target
+                firmware structure RGXFWIF_OPP_INFO which must be pre-allocated.
+                The source values are expected to be read into the kernel
+                Power Manager from the platform's Device Tree.
+                Requires CONFIG_OF and CONFIG_PM_OPP support in the kernel.
+
+@Input          psDeviceNode       Device node
+@Input          psOPPInfo          Target OPP data buffer
+@Input          ui32MaxOPPLevels   Maximum number of OPP levels allowed in buffer.
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+PVRSRV_ERROR DVFSCopyOPPTable(PPVRSRV_DEVICE_NODE psDeviceNode,
+							  RGXFWIF_OPP_INFO   *psOPPInfo,
+							  IMG_UINT32          ui32MaxOPPLevels)
+{
+	PVRSRV_ERROR            eError = PVRSRV_OK;
+	struct device          *psDev = NULL;
+	OPP_LEVEL              *psOPPValue;
+	struct dev_pm_opp      *opp;
+	struct pvr_opp_freq_table pvr_freq_table = {0};
+	unsigned long           min_freq = 0, max_freq = 0, min_volt = 0;
+	unsigned int            i, err;
+	unsigned long *freq_table;
+
+	if (!psDeviceNode || !psOPPInfo)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid device or argument", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+	err = GetOPPValues(psDev, &min_freq, &min_volt, &max_freq, &pvr_freq_table);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: DVFS OPP table not initialised.", __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	freq_table = pvr_freq_table.freq_table;
+
+	if (pvr_freq_table.num_levels > ui32MaxOPPLevels)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Too many OPP levels (%u), max (%u).", __func__,
+		         pvr_freq_table.num_levels, ui32MaxOPPLevels));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto exit;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	/* Start RCU read-side critical section to map frequency to OPP */
+	rcu_read_lock();
+#endif
+
+	/* Loop over the OPP/frequency levels */
+	psOPPValue = &psOPPInfo->asOPPValues[0];
+	for (i=0; i<pvr_freq_table.num_levels; i++)
+	{
+		psOPPValue->ui32Freq = freq_table[i];
+		opp = dev_pm_opp_find_freq_exact(psDev, freq_table[i], IMG_TRUE);
+		if (IS_ERR(opp))
+		{
+			err = PTR_ERR(opp);
+			dev_err(psDev, "Couldn't find %dth frequency, %d\n", i, err);
+			eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+			goto exit;
+		}
+		psOPPValue->ui32Volt = dev_pm_opp_get_voltage(opp);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+		dev_pm_opp_put(opp);
+#endif
+		psOPPValue++;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING, "%s: Copied %u OPP points to the FW processor table.", __func__,
+	         pvr_freq_table.num_levels));
+	psOPPInfo->ui32MaxOPPPoint = pvr_freq_table.num_levels - 1;
+exit:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#endif
+
+	/* Free memory allocated in GetOPPValues */
+	devm_kfree(psDev, freq_table);
+
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+                Prepares the OPP table from the devicetree, if enabled.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_PDVFS)
+PVRSRV_ERROR InitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if !(defined(CONFIG_PM_OPP) && defined(CONFIG_OF))
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_OK;
+#else
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	struct device          *psDev;
+	int                     err;
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_ASSERT(psDeviceNode->psDevConfig);
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	/* Setup the OPP table from the device tree for Proactive DVFS. */
+	err = dev_pm_opp_of_add_table(psDev);
+	if (err == 0)
+	{
+		psDVFSDeviceCfg->bDTConfig = IMG_TRUE;
+	}
+	else
+	{
+		/*
+		 * If there are no device tree or system layer provided operating points
+		 * then return an error
+		 */
+		if (psDVFSDeviceCfg->pasOPPTable)
+		{
+			psDVFSDeviceCfg->bDTConfig = IMG_FALSE;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "No system or device tree opp points found, %d", err));
+			return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		}
+	}
+	return PVRSRV_OK;
+#endif
+}
+
+/*************************************************************************/ /*!
+@Function       DeinitPDVFS
+
+@Description    De-Initialise the device for Proactive DVFS support.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+void DeinitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+#if !(defined(CONFIG_PM_OPP) && defined(CONFIG_OF))
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#else
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = NULL;
+	struct device *psDev = NULL;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	if (psDVFSDeviceCfg->bDTConfig)
+	{
+		/*
+		 * Remove OPP entries for this device; only static entries from
+		 * the device tree are present.
+		 */
+		dev_pm_opp_of_remove_table(psDev);
+	}
+#endif
+}
+#endif /* SUPPORT_PDVFS */
+
+#endif /* !NO_HARDWARE */
diff --git a/drivers/gpu/drm/img-rogue/pvr_dvfs_common.h b/drivers/gpu/drm/img-rogue/pvr_dvfs_common.h
new file mode 100644
index 000000000000..87eccce5762e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_dvfs_common.h
@@ -0,0 +1,123 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_common.h
+@Title          System level interface for DVFS and PDVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_COMMON_H
+#define PVR_DVFS_COMMON_H
+
+#include "opaque_types.h"
+#include "pvrsrv_error.h"
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+#include "rgx_fwif_km.h"
+#endif
+
+struct pvr_opp_freq_table
+{
+	unsigned long *freq_table;
+	int num_levels;
+};
+
+/*************************************************************************/ /*!
+@Function       GetOPPValues
+
+@Description    Common code to store the OPP points in the pvr_freq_table,
+                for use with the IMG DVFS/devfreq implementation, or with
+                the Proactive DVFS.
+                Requires CONFIG_PM_OPP support in the kernel.
+
+@Input          dev        OS Device node
+@Output         min_freq   Min clock freq (Hz)
+@Output         min_volt   Min voltage (V)
+@Output         max_freq   Max clock freq (Hz)
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(CONFIG_PM_OPP)
+int GetOPPValues(struct device *dev,
+                 unsigned long *min_freq,
+                 unsigned long *min_volt,
+                 unsigned long *max_freq,
+                 struct pvr_opp_freq_table *pvr_freq_table);
+#endif
+
+/*************************************************************************/ /*!
+@Function       DVFSCopyOPPTable
+
+@Description    Copies the OPP points (voltage/frequency) to the target
+                firmware structure RGXFWIF_OPP_INFO which must be pre-allocated.
+                The source values are expected to be read into the kernel
+                Power Manager from the platform's Device Tree.
+                Requires CONFIG_OF and CONFIG_PM_OPP support in the kernel.
+
+@Input          psDeviceNode       Device node
+@Input          psOPPInfo          Target OPP data buffer
+@Input          ui32MaxOPPLevels   Maximum number of OPP levels allowed in buffer.
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF) && defined(CONFIG_PM_OPP)
+PVRSRV_ERROR DVFSCopyOPPTable(PPVRSRV_DEVICE_NODE psDeviceNode,
+							  RGXFWIF_OPP_INFO   *psOPPInfo,
+							  IMG_UINT32          ui32MaxOPPLevels);
+#endif
+
+#if defined(SUPPORT_PDVFS)
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+                Prepares the OPP table from the devicetree, if enabled.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR InitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       InitPDVFS
+
+@Description    Initialise the device for Proactive DVFS support.
+
+@Input          psDeviceNode       Device node
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+void DeinitPDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+#endif
+
+#endif /* PVR_DVFS_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_dvfs_device.c b/drivers/gpu/drm/img-rogue/pvr_dvfs_device.c
new file mode 100644
index 000000000000..ef011b39a6ca
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_dvfs_device.c
@@ -0,0 +1,1426 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs_device.c
+@Title          PowerVR devfreq device implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Linux DVFS module setup
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(NO_HARDWARE)
+
+#include <linux/devfreq.h>
+#if defined(CONFIG_POWERVR_THERMAL)
+#include <linux/devfreq_cooling.h>
+#endif
+#include <linux/version.h>
+#include <linux/device.h>
+#include <drm/drm.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include "power.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+
+#include "rgxdevice.h"
+#include "rgxinit.h"
+#include "sofunc_rgx.h"
+
+#include "syscommon.h"
+
+#include "pvr_dvfs.h"
+#include "pvr_dvfs_device.h"
+#include "pvr_dvfs_common.h"
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+
+#if !defined(CONFIG_PM_DEVFREQ)
+#error "PVR DVFS governor requires kernel support for devfreq (CONFIG_PM_DEVFREQ = 1)"
+#endif
+#include "linux/governor.h"
+#include "pvr_dvfs_governor.h"
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+#include "linux/devfreq-event.h"
+#include "pvr_dvfs_events.h"
+#endif
+#endif /* SUPPORT_PVR_DVFS_GOVERNOR */
+
+#include "kernel_compatibility.h"
+
+
+/* Default constants for PVR-Balanced (PVR) governor */
+#define PVR_UPTHRESHOLD			(90)
+#define PVR_DOWNDIFFERENTIAL	(5)
+
+static int _device_get_devid(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	int deviceId;
+
+	if (ddev->render)
+		deviceId = ddev->render->index;
+	else /* when render node is NULL, fallback to primary node */
+		deviceId = ddev->primary->index;
+
+	return deviceId;
+}
+
+static IMG_INT32 devfreq_target(struct device *dev, unsigned long *requested_freq, IMG_UINT32 flags)
+{
+	int deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	RGX_DATA		*psRGXData = NULL;
+	IMG_DVFS_DEVICE		*psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG	*psDVFSDeviceCfg = NULL;
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo = NULL;
+	IMG_UINT32		ui32Freq, ui32CurFreq, ui32Volt;
+	struct dev_pm_opp *opp;
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+
+	/* Check the RGX device is initialised */
+	if (!psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+	if (psDVFSDevice->eState != PVR_DVFS_STATE_READY)
+	{
+		*requested_freq = psRGXTimingInfo->ui32CoreClockSpeed;
+		return 0;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_lock();
+#endif
+
+	opp = devfreq_recommended_opp(dev, requested_freq, flags);
+	if (IS_ERR(opp)) {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+		rcu_read_unlock();
+#endif
+		PVR_DPF((PVR_DBG_ERROR, "Invalid OPP"));
+		return PTR_ERR(opp);
+	}
+
+	ui32Freq = dev_pm_opp_get_freq(opp);
+	ui32Volt = dev_pm_opp_get_voltage(opp);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	rcu_read_unlock();
+#else
+	dev_pm_opp_put(opp);
+#endif
+
+	ui32CurFreq = psRGXTimingInfo->ui32CoreClockSpeed;
+
+	if (ui32CurFreq == ui32Freq)
+	{
+		return 0;
+	}
+
+	if (PVRSRV_OK != PVRSRVDevicePreClockSpeedChange(psDeviceNode,
+													 psDVFSDeviceCfg->bIdleReq,
+													 NULL))
+	{
+		dev_err(dev, "PVRSRVDevicePreClockSpeedChange failed\n");
+		return -EPERM;
+	}
+
+	/* Increasing frequency, change voltage first */
+	if (ui32Freq > ui32CurFreq)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, ui32Volt);
+	}
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDeviceNode->psDevConfig->hSysData, ui32Freq);
+
+	/* Decreasing frequency, change frequency first */
+	if (ui32Freq < ui32CurFreq)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, ui32Volt);
+	}
+
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32Freq;
+
+	PVRSRVDevicePostClockSpeedChange(psDeviceNode, psDVFSDeviceCfg->bIdleReq,
+									 NULL);
+
+	return 0;
+}
+
+static int devfreq_get_dev_status(struct device *dev, struct devfreq_dev_status *stat)
+{
+	int                      deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE      *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	PVRSRV_RGXDEV_INFO      *psDevInfo = NULL;
+	IMG_DVFS_DEVICE         *psDVFSDevice = NULL;
+	RGX_DATA                *psRGXData = NULL;
+	RGX_TIMING_INFORMATION  *psRGXTimingInfo = NULL;
+	RGXFWIF_GPU_UTIL_STATS  *psGpuUtilStats = NULL;
+	PVRSRV_ERROR             eError;
+
+	static unsigned long total_time_last = 0;
+	static unsigned long busy_time_last = 0;
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	struct pvr_profiling_dev_status *pvr_stat = stat->private_data;
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* Check the RGX device is initialised */
+	if (!psDevInfo || !psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+	stat->current_frequency = psRGXTimingInfo->ui32CoreClockSpeed;
+
+	if (psDevInfo->pfnGetGpuUtilStats == NULL)
+	{
+		/* Not yet ready. So set times to something sensible. */
+		stat->busy_time = 0;
+		stat->total_time = 0;
+		return 0;
+	}
+
+	psGpuUtilStats = kzalloc(sizeof(*psGpuUtilStats), GFP_KERNEL);
+
+	if (!psGpuUtilStats)
+	{
+		return -ENOMEM;
+	}
+
+	eError = psDevInfo->pfnGetGpuUtilStats(psDeviceNode,
+						psDVFSDevice->hGpuUtilUserDVFS,
+						psGpuUtilStats);
+
+	if (eError != PVRSRV_OK)
+	{
+		if (eError == 271){
+			//PVRSRV_ERROR_RESOURCE_UNAVAILABLE=271
+			psGpuUtilStats->ui64GpuStatActive = busy_time_last;
+			psGpuUtilStats->ui64GpuStatCumulative = total_time_last;
+		}
+		else{
+			kfree(psGpuUtilStats);
+			return -EAGAIN;
+		}
+	}
+
+	stat->busy_time = psGpuUtilStats->ui64GpuStatActive;
+	stat->total_time = psGpuUtilStats->ui64GpuStatCumulative;
+	// record last GPU stat
+	busy_time_last = stat->busy_time;
+	total_time_last = stat->total_time;
+
+	kfree(psGpuUtilStats);
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	err = pvr_get_dev_status_get_events(psDVFSDevice->psProfilingDevice, pvr_stat);
+	if (err)
+		return err;
+#endif
+
+	return 0;
+}
+
+static IMG_INT32 devfreq_cur_freq(struct device *dev, unsigned long *freq)
+{
+	int deviceId = _device_get_devid(dev);
+	PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+	RGX_DATA *psRGXData = NULL;
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* Check the RGX device is initialised */
+	if (!psRGXData)
+	{
+		return -ENODATA;
+	}
+
+	*freq = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return 0;
+}
+
+static struct devfreq_dev_profile img_devfreq_dev_profile =
+{
+	.polling_ms	        = 10,
+	.target             = devfreq_target,
+	.get_dev_status     = devfreq_get_dev_status,
+	.get_cur_freq       = devfreq_cur_freq,
+};
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+
+/* DEVFREQ governor name */
+#define DEVFREQ_GOV_PVR_BALANCED	"pvr_balanced"
+
+static unsigned long _get_max_freq(struct devfreq *devfreq_dev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+	return devfreq_dev->scaling_max_freq;
+#else
+	return devfreq_dev->max_freq;
+#endif
+}
+
+/*
+ * Calculate the utilisation percentage of X cycles as a proportion
+ * of Y cycles
+ */
+static inline
+unsigned long UTILISATION_PC(unsigned long X, unsigned long Y)
+{
+	PVR_ASSERT(Y > 0);
+
+/*
+	if (X < UINT_MAX / 100UL)
+		return min(100UL, X * 100UL / Y);
+	else
+		return min(100UL, X / (Y / 100UL));
+*/
+	/* Prevent overflow */
+	if (X < UINT32_MAX / 100U)
+		return (X * 100U / Y);
+	else
+		return (X / (Y / 100U));
+}
+
+static int pvr_governor_get_target(struct devfreq *devfreq_dev,
+								   unsigned long *freq)
+{
+	struct devfreq_dev_status *stat;
+	struct pvr_profiling_dev_status *pvr_stat;
+	IMG_DVFS_GOVERNOR_CFG *data = devfreq_dev->data;
+	unsigned long long a, b;
+	unsigned int pvr_upthreshold = PVR_UPTHRESHOLD;
+	unsigned int pvr_downdifferential = PVR_DOWNDIFFERENTIAL;
+	unsigned long ui32Util, ui32UtilDM, ui32UtilBus;
+	int iGeomEventID, iFragEventID, iCompEventID, iSLCReadEventID, iSLCWriteEventID;
+	int iSLCReadEventID2, iSLCWriteEventID2;
+	int err;
+
+	if (!devfreq_dev)
+	{
+		return -ENODEV;
+	}
+
+	/*
+	 *	Governor main function for setting target frequency, based on
+	 *	metrics for
+		- utilisation
+		- memory bus bandwidth
+		- ALU counters etc.
+	 */
+
+	err = devfreq_update_stats(devfreq_dev);
+	if (err)
+	{
+		dev_warn(&devfreq_dev->dev, "get_dev_status: error (%d)", err);
+		return err;
+	}
+
+	stat = &devfreq_dev->last_status;
+	pvr_stat = stat->private_data;
+	if (!pvr_stat)
+	{
+		return -EINVAL;
+	}
+
+	iGeomEventID = pvr_find_event_by_name("geom-cycle");
+	iFragEventID = pvr_find_event_by_name("3d-cycle");
+	iCompEventID = pvr_find_event_by_name("comp-cycle");
+	if (iGeomEventID < 0 || iFragEventID < 0 || iCompEventID < 0)
+	{
+		return -EINVAL;
+	}
+
+	if (pvr_stat->event_data[iGeomEventID].total_count == 0 ||
+	    pvr_stat->event_data[iFragEventID].total_count == 0 ||
+	    pvr_stat->event_data[iCompEventID].total_count == 0)
+	{
+		/* no error to avoid verbose kernel logs before the firmware/GPU
+		 * is available. */
+		return 0;
+	}
+
+	/* GEOM utilisation */
+	if (iGeomEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iGeomEventID].load_count,
+						   pvr_stat->event_data[iGeomEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	/* 3D utilisation */
+	if (iFragEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iFragEventID].load_count,
+						   pvr_stat->event_data[iFragEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	/* Compute utilisation */
+	if (iCompEventID >= 0)
+	{
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iCompEventID].load_count,
+						   pvr_stat->event_data[iCompEventID].total_count);
+		ui32UtilDM = max(ui32UtilDM, ui32Util);
+	}
+
+	if (ui32UtilDM > 1000U)
+	{
+		/* Illegal value, keep current frequency */
+		*freq = stat->current_frequency;
+		return 0;
+	}
+
+	iSLCReadEventID =   pvr_find_event_by_name("slc-read");
+	iSLCWriteEventID =  pvr_find_event_by_name("slc-write");
+	iSLCReadEventID2 =  pvr_find_event_by_name("slc-read-2");
+	iSLCWriteEventID2 = pvr_find_event_by_name("slc-write-2");
+	if (iSLCReadEventID2 >= 0 && iSLCWriteEventID2 >= 0)
+	{
+		/* Use XE-series memory bus counters */
+
+		ui32UtilBus = 0;
+
+		while (iSLCWriteEventID < ARRAY_SIZE(g_pvr_governor_events))
+		{
+			ui32Util =
+				UTILISATION_PC(pvr_stat->event_data[iSLCReadEventID].load_count,
+							   pvr_stat->event_data[iSLCReadEventID].total_count);
+			ui32UtilBus = ui32UtilBus + ui32Util;
+
+			ui32Util =
+				UTILISATION_PC(pvr_stat->event_data[iSLCWriteEventID].load_count,
+							   pvr_stat->event_data[iSLCWriteEventID].total_count);
+			ui32UtilBus = ui32UtilBus + ui32Util;
+			iSLCReadEventID += 2;
+			iSLCWriteEventID += 2;
+		}
+	}
+	else if (iSLCReadEventID >= 0 && iSLCWriteEventID >= 0)
+	{
+		/* Use DXT-series memory bus counters */
+
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iSLCReadEventID].load_count / data->uiNumMembus,
+						   pvr_stat->event_data[iSLCReadEventID].total_count);
+		ui32UtilBus = ui32Util;
+
+		ui32Util =
+			UTILISATION_PC(pvr_stat->event_data[iSLCWriteEventID].load_count / data->uiNumMembus,
+						   pvr_stat->event_data[iSLCWriteEventID].total_count);
+		ui32UtilBus = ui32UtilBus + ui32Util;
+	}
+
+	if (ui32UtilBus > 1000U)
+	{
+		/* Illegal value, use current frequency */
+		*freq = stat->current_frequency;
+		return 0;
+	}
+
+#if defined(DEBUG)
+	iSLCReadEventID =  pvr_find_event_by_name("slc-read");
+	iSLCWriteEventID = pvr_find_event_by_name("slc-write");
+	dev_info(&devfreq_dev->dev, "Geom: %10lu, 3D: %10lu, Comp: %10lu, SLC: %10lu cycles. DM %lu pc, Bus %lu pc.",
+			pvr_stat->event_data[iGeomEventID].load_count,
+			pvr_stat->event_data[iFragEventID].load_count,
+			pvr_stat->event_data[iCompEventID].load_count,
+			pvr_stat->event_data[iSLCReadEventID].load_count + pvr_stat->event_data[iSLCWriteEventID].load_count,
+			ui32UtilDM,
+			ui32UtilBus);
+#endif
+
+	if (ui32UtilDM > 2 * ui32UtilBus) {
+		// DM pipeline limited
+		ui32Util = ui32UtilDM;
+	}
+	else if (ui32UtilBus > 2 * ui32UtilDM) {
+		// Mem bandwidth limited
+		ui32Util = ui32UtilBus;
+	}
+	else {
+		// Average utilisation to smooth random fluctuations
+		ui32Util = (ui32UtilDM + ui32UtilBus) / 2;
+	}
+
+	/* Set MAX if it's busy enough */
+	if (ui32Util > pvr_upthreshold)
+	{
+		*freq = _get_max_freq(devfreq_dev);
+		return 0;
+	}
+
+	/* Set MAX if we do not know the initial frequency */
+	if (stat->current_frequency == 0) {
+		*freq = _get_max_freq(devfreq_dev);
+		return 0;
+	}
+
+	a = (unsigned long long) ui32Util * stat->current_frequency;
+	b = div_u64(a, (pvr_upthreshold - pvr_downdifferential / 2));
+	*freq = (unsigned long) b;
+	return 0;
+}
+
+static void pvr_governor_suspend(struct devfreq *devfreq_dev)
+{
+	/* Nothing to do */
+}
+
+static void pvr_governor_resume(struct devfreq *devfreq_dev)
+{
+	int deviceId;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_DVFS_DEVICE		*psDVFSDevice = NULL;
+	struct pvr_profiling_device *psProfilingDevice;
+
+	if (!devfreq_dev || !dev_get_drvdata(&devfreq_dev->dev))
+		return;
+
+	deviceId = _device_get_devid(&devfreq_dev->dev);
+	psDeviceNode = PVRSRVGetDeviceInstanceByKernelDevID(deviceId);
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psProfilingDevice = psDVFSDevice->psProfilingDevice;
+
+	/* Reset the profiling events */
+	pvr_governor_reset_events(psProfilingDevice);
+}
+
+static int pvr_governor_event_handler(struct devfreq *devfreq_dev,
+									  unsigned int event, void *data)
+{
+	if (!devfreq_dev)
+	{
+		pr_err("%s: devfreq_dev not ready.\n", __func__);
+		return -ENODEV;
+	}
+
+	/*
+	 * We cannot take the deviceId here, as the DRM device
+	 * may not be initialised. Null pointer in
+	 * struct drm_device *ddev = dev_get_drvdata(dev)
+	 */
+
+	switch (event) {
+	case DEVFREQ_GOV_START:
+		dev_info(&devfreq_dev->dev,"GOV_START event.\n");
+		devfreq_monitor_start(devfreq_dev);
+		break;
+
+	case DEVFREQ_GOV_STOP:
+		dev_info(&devfreq_dev->dev,"GOV_STOP event.\n");
+		devfreq_monitor_stop(devfreq_dev);
+		break;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	case DEVFREQ_GOV_UPDATE_INTERVAL:
+		dev_info(&devfreq_dev->dev,"GOV_UPDATE_INTERVAL event.\n");
+		devfreq_update_interval(devfreq_dev, (unsigned int *)data);
+		break;
+
+#else
+	case DEVFREQ_GOV_INTERVAL:
+		dev_info(&devfreq_dev->dev,"GOV_INTERVAL event.\n");
+		devfreq_interval_update(devfreq_dev, (unsigned int *)data);
+		break;
+#endif
+
+	case DEVFREQ_GOV_SUSPEND:
+		dev_info(&devfreq_dev->dev,"GOV_SUSPEND event.\n");
+		pvr_governor_suspend(devfreq_dev);
+		devfreq_monitor_suspend(devfreq_dev);
+		break;
+
+	case DEVFREQ_GOV_RESUME:
+		dev_info(&devfreq_dev->dev,"GOV_RESUME event.\n");
+		devfreq_monitor_resume(devfreq_dev);
+		pvr_governor_resume(devfreq_dev);
+		break;
+
+	default:
+		printk("Unknown event.\n");
+		break;
+	}
+
+	return 0;
+}
+
+static struct devfreq_governor pvr_balanced_governor = {
+	.name = DEVFREQ_GOV_PVR_BALANCED,
+	.get_target_freq = pvr_governor_get_target,
+	.event_handler   = pvr_governor_event_handler,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	.attrs = DEVFREQ_GOV_ATTR_POLLING_INTERVAL
+		| DEVFREQ_GOV_ATTR_TIMER,
+#else
+	.immutable = true,
+#endif
+};
+
+int pvr_governor_init(void)
+{
+	int ret;
+
+	ret = devfreq_add_governor(&pvr_balanced_governor);
+	if (ret)
+	{
+		pr_err("%s: failed to install governor %d\n", __func__, ret);
+	}
+
+	return ret;
+}
+
+void pvr_governor_exit(void)
+{
+	int ret;
+
+	ret = devfreq_remove_governor(&pvr_balanced_governor);
+	if (ret)
+	{
+		pr_err("Failed to remove governor (%u)\n", ret);
+	}
+}
+
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+
+static int pvr_get_dev_status_get_events(struct pvr_profiling_device *pvr_prof_dev,
+										 struct pvr_profiling_dev_status *pvr_stat)
+{
+	struct devfreq_event_data event_data;
+	int i, ret = 0;
+
+	for (i = 0; i < pvr_prof_dev->num_events; i++)
+	{
+		if (!pvr_prof_dev->edev[i])
+			continue;
+
+		ret = devfreq_event_get_event(pvr_prof_dev->edev[i], &event_data);
+		if (ret < 0)
+			return ret;
+
+		pvr_stat->event_data[i].load_count = event_data.load_count;
+		pvr_stat->event_data[i].total_count = event_data.total_count;
+	}
+
+	return ret;
+}
+
+static int pvr_governor_reset_events(struct pvr_profiling_device *pvr_prof_dev)
+{
+	int i, ret = 0;
+
+	for (i = 0; i < pvr_prof_dev->num_events; i++)
+	{
+		if (!pvr_prof_dev->edev[i])
+			continue;
+
+		ret = devfreq_event_set_event(pvr_prof_dev->edev[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int pvr_events_enable(struct devfreq_event_dev *edev)
+{
+	/* Enable counters */
+
+	dev_dbg(&edev->dev, "%s (Enable)\n", edev->desc->name);
+
+	return 0;
+}
+
+static int pvr_events_disable(struct devfreq_event_dev *edev)
+{
+	/* Disable counters */
+
+	dev_dbg(&edev->dev, "%s (Disable)\n", edev->desc->name);
+
+	return 0;
+}
+
+static int pvr_find_event_by_name(const char *name)
+{
+	int idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(g_pvr_governor_events); idx++)
+		if (strcmp(name, g_pvr_governor_events[idx].name) == 0)
+			return idx;
+
+	return -EINVAL;
+}
+
+static int pvr_governor_set_event(struct devfreq_event_dev *edev)
+{
+	int id = pvr_find_event_by_name(edev->desc->name);
+	if (id < 0)
+		return -EINVAL;
+
+	/* clear the last value */
+	g_pvr_governor_events[id].last_value = 0;
+
+	return 0;
+}
+
+static int pvr_governor_get_event(struct devfreq_event_dev *edev,
+				struct devfreq_event_data *edata)
+{
+	struct pvr_profiling_device *info = devfreq_event_get_drvdata(edev);
+	volatile unsigned long perf_ctr;
+	int id = pvr_find_event_by_name(edev->desc->name);
+	if (id < 0)
+		return -EINVAL;
+
+	/*
+	 * Read performance counter
+	 */
+	perf_ctr = OSReadHWReg32(info->pvRegsBaseKM, g_pvr_governor_events[id].cntr);
+
+	/* Calculate deltas */
+	if (perf_ctr >= g_pvr_governor_events[id].last_value)
+	{
+		edata->load_count = perf_ctr - g_pvr_governor_events[id].last_value;
+	}
+	else
+	{
+		edata->load_count = UINT32_MAX - g_pvr_governor_events[id].last_value + perf_ctr + 1;
+	}
+
+	g_pvr_governor_events[id].last_value = perf_ctr;
+
+	/*
+	 * Calc the utilisation as a fraction of the total cycles
+	 */
+	if (edev->desc->event_type & PVR_EVENT_OP_PERF_CNTR)
+	{
+		edata->total_count = info->stat->event_data[0].load_count;
+
+		/*
+		 * Timer has 256-cycle granularity
+		 * Cycle counters have 1-cycle or 256-cycle granularity depending on GPU variant
+		 * SLC read/write counters have 1 event granularity
+		 */
+		if ((g_pvr_governor_events[0].shift > 0) && (g_pvr_governor_events[id].shift == 0))
+		{
+			int timer_shift = g_pvr_governor_events[0].shift;
+			if (edata->total_count < (UINT32_MAX >> timer_shift))
+			{
+				edata->total_count <<= timer_shift;	/*!< convert total to cycles */
+			}
+			else
+			{
+				/* avoid overflow but some loss of precision will occur */
+				edata->load_count >>= timer_shift;	/*!< convert load to ticks */
+			}
+		}
+	}
+	else
+	{
+		/* timer runs continuously, effective load is 100% */
+		edata->total_count = edata->load_count;
+	}
+
+	return 0;
+}
+
+static const struct devfreq_event_ops pvr_governor_event_ops = {
+	.enable = pvr_events_enable,
+	.disable = pvr_events_disable,
+	.set_event = pvr_governor_set_event,
+	.get_event = pvr_governor_get_event,
+};
+
+int pvr_events_register(struct device *dev, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice;
+	struct pvr_profiling_device *pvr_prof_dev;
+	struct devfreq_event_dev **edev;
+	struct devfreq_event_desc *desc;
+	int size, err;
+	int idx;
+
+	if (!psDeviceNode)
+		return -ENODEV;
+
+	pvr_prof_dev = devm_kzalloc(dev, sizeof(*pvr_prof_dev), GFP_KERNEL);
+	if (!pvr_prof_dev)
+		return -ENOMEM;
+
+	pvr_prof_dev->dev = dev;
+	pvr_prof_dev->num_events = ARRAY_SIZE(g_pvr_governor_events);
+
+	/* Alloc the event descriptor table */
+	desc = devm_kcalloc(dev, pvr_prof_dev->num_events, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	for (idx = 0; idx < pvr_prof_dev->num_events; idx++)
+	{
+		if (!g_pvr_governor_events[idx].name)
+			continue;
+
+		desc[idx].name = g_pvr_governor_events[idx].name;
+		desc[idx].event_type = (idx < PERF_CNTR_OFF) ? PVR_EVENT_OP_TIMER : PVR_EVENT_OP_PERF_CNTR;
+		desc[idx].event_type |= PVR_EVENT_OP_PERF_VER;
+		desc[idx].driver_data = pvr_prof_dev;
+		desc[idx].ops = &pvr_governor_event_ops;
+	}
+	pvr_prof_dev->desc = desc;
+
+	size = sizeof(struct devfreq_event_dev *) * pvr_prof_dev->num_events;
+	pvr_prof_dev->edev = devm_kzalloc(dev, size, GFP_KERNEL);
+	if (!pvr_prof_dev->edev)
+		return -ENOMEM;
+
+	/* Populate the events */
+	for (idx = 0, edev = pvr_prof_dev->edev; idx < pvr_prof_dev->num_events; idx++)
+	{
+		edev[idx] = devm_devfreq_event_add_edev(dev, &desc[idx]);
+		if (IS_ERR(edev[idx]))
+		{
+			dev_err(dev, "failed to add devfreq-event device (idx=%d)\n", idx);
+			return PTR_ERR(edev[idx]);
+		}
+
+		/* enable */
+		err = devfreq_event_enable_edev(edev[idx]);
+		if (err)
+		{
+			dev_err(dev, "failed to enable devfreq-event device (idx=%d)\n", idx);
+			return PTR_ERR(edev[idx]);
+		}
+
+		dev_info(dev, "%s: new PVR devfreq-event device registered %s (%s)\n",
+			__func__, dev_name(dev), desc[idx].name);
+	}
+
+	/* Allocate the profiling stats, passed to governor */
+	pvr_prof_dev->stat = devm_kzalloc(dev, sizeof(struct pvr_profiling_dev_status), GFP_KERNEL);
+	if (!pvr_prof_dev->stat)
+		return -ENOMEM;
+
+	/* Map the reg bank */
+	pvr_prof_dev->reg_size = psDeviceNode->psDevConfig->ui32RegsSize;
+	pvr_prof_dev->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvr_prof_dev->pvRegsBaseKM)
+	{
+		dev_err(dev, "failed to map register bank.");
+		return -ENODEV;
+	}
+
+	dev_info(dev, "Mapped regbank at %p, size 0x%x Bytes.\n", pvr_prof_dev->pvRegsBaseKM,
+			pvr_prof_dev->reg_size);
+
+	/* Update the DVFS device */
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDevice->psProfilingDevice = pvr_prof_dev;
+
+	return 0;
+}
+
+void pvr_events_unregister(struct device *dev, IMG_DVFS_DEVICE *psDVFSDevice)
+{
+	struct pvr_profiling_device *pvr_prof_dev = psDVFSDevice->psProfilingDevice;
+	struct devfreq_event_dev **edev;
+	int idx, err;
+
+	/* Remove the events */
+	for (idx = 0, edev = pvr_prof_dev->edev; idx < pvr_prof_dev->num_events; idx++)
+	{
+		err = devfreq_event_disable_edev(edev[idx]);
+		if (err)
+		{
+			dev_warn(dev, "failed to disable devfreq-event device (idx=%d)\n", idx);
+		}
+		devm_devfreq_event_remove_edev(dev, edev[idx]);
+	}
+
+	/* Unmap the register bank */
+	if (pvr_prof_dev != NULL && pvr_prof_dev->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) pvr_prof_dev->pvRegsBaseKM,
+		                 pvr_prof_dev->reg_size);
+	}
+
+	/* devfreq_event resources are managed by the kernel */
+}
+#endif /* CONFIG_PM_DEVFREQ_EVENT */
+
+#endif /* SUPPORT_PVR_DVFS_GOVERNOR */
+
+static int FillOPPTable(struct device *dev, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	const IMG_OPP *iopp;
+	int i, err = 0;
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = NULL;
+
+	/* Check the device exists */
+	if (!dev || !psDeviceNode)
+	{
+		return -ENODEV;
+	}
+
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	if (!psDVFSDeviceCfg->pasOPPTable)
+	{
+		dev_err(dev, "No DVFS OPP table provided in system layer and no device tree support.");
+		return -ENODATA;
+	}
+
+	for (i = 0, iopp = psDVFSDeviceCfg->pasOPPTable;
+	     i < psDVFSDeviceCfg->ui32OPPTableSize;
+	     i++, iopp++)
+	{
+		err = dev_pm_opp_add(dev, iopp->ui32Freq, iopp->ui32Volt);
+		if (err) {
+			dev_err(dev, "Could not add OPP entry, %d\n", err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+
+#if defined(CONFIG_POWERVR_THERMAL)
+static int RegisterCoolingDevice(struct device *dev,
+								 IMG_DVFS_DEVICE *psDVFSDevice,
+								 struct devfreq_cooling_power *powerOps)
+{
+	struct device_node *of_node;
+	int err = 0;
+
+	if (!psDVFSDevice)
+	{
+		return -EINVAL;
+	}
+
+	if (!powerOps)
+	{
+		dev_info(dev, "Cooling: power ops not registered, not enabling cooling");
+		return 0;
+	}
+
+	of_node = of_node_get(dev->of_node);
+
+	psDVFSDevice->psDevfreqCoolingDevice = of_devfreq_cooling_register_power(
+		of_node, psDVFSDevice->psDevFreq, powerOps);
+
+	if (IS_ERR(psDVFSDevice->psDevfreqCoolingDevice))
+	{
+		err = PTR_ERR(psDVFSDevice->psDevfreqCoolingDevice);
+		dev_err(dev, "Failed to register as devfreq cooling device %d", err);
+	}
+
+	of_node_put(of_node);
+
+	return err;
+}
+#endif
+
+#define TO_IMG_ERR(err) ((err == -EPROBE_DEFER) ? PVRSRV_ERROR_PROBE_DEFER : PVRSRV_ERROR_INIT_FAILURE)
+
+PVRSRV_ERROR InitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE        *psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	struct device          *psDev;
+	PVRSRV_ERROR            eError;
+	int                     err;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if !defined(CONFIG_PM_OPP)
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_ASSERT(psDeviceNode->psDevConfig);
+
+	if (psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState == PVR_DVFS_STATE_INIT_PENDING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "DVFS initialise pending for device node %p",
+				 psDeviceNode));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState = PVR_DVFS_STATE_INIT_PENDING;
+
+#if defined(SUPPORT_SOC_TIMER)
+	if (! psDeviceNode->psDevConfig->pfnSoCTimerRead)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "System layer SoC timer callback not implemented"));
+		//return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	eError = SORgxGpuUtilStatsRegister(&psDVFSDevice->hGpuUtilUserDVFS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to register to the GPU utilisation stats, %d", eError));
+		return eError;
+	}
+
+	err = dev_pm_opp_of_add_table(psDev);
+	if (err == 0)
+	{
+		psDVFSDeviceCfg->bDTConfig = IMG_TRUE;
+	}
+	else
+	{
+		/*
+		 * If there are no device tree or system layer provided operating points
+		 * then return an error
+		 */
+		if ((err == -ENOTSUPP || err == -ENODEV) && psDVFSDeviceCfg->pasOPPTable)
+		{
+			err = FillOPPTable(psDev, psDeviceNode);
+			if (err != 0 && err != -ENODATA)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to fill OPP table with data, %d", err));
+				eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+				goto err_exit;
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to init opp table from devicetree, %d", err));
+			eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+			goto err_exit;
+		}
+	}
+
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+#if defined(CONFIG_PM_DEVFREQ_EVENT)
+	err = pvr_events_register(psDev, psDeviceNode);
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to init PVR profiling events, %d", err));
+		goto err_exit;
+	}
+#endif
+
+	err = pvr_governor_init();
+	if (err != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to init PVR governor, %d", err));
+		goto err_exit;
+	}
+	psDVFSDevice->bGovernorReady = true;
+#endif
+
+	PVR_ASSERT(psDev);
+	PVR_ASSERT(psDeviceNode);
+
+	PVR_TRACE(("PVR DVFS init pending: dev = %p, PVR device = %p",
+			   psDev, psDeviceNode));
+
+	return PVRSRV_OK;
+
+err_exit:
+	DeinitDVFS(psDeviceNode);
+	return eError;
+}
+
+PVRSRV_ERROR RegisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE        *psDVFSDevice = NULL;
+	IMG_DVFS_DEVICE_CFG    *psDVFSDeviceCfg = NULL;
+	IMG_DVFS_GOVERNOR_CFG  *psDVFSGovernorCfg = NULL;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = NULL;
+	struct device          *psDev;
+	struct pvr_opp_freq_table	pvr_freq_table = {0};
+	unsigned long           min_freq = 0, max_freq = 0, min_volt = 0;
+	PVRSRV_ERROR            eError;
+	int                     err;
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState != PVR_DVFS_STATE_INIT_PENDING)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "DVFS initialise not yet pending for device node %p",
+				 psDeviceNode));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+	psDVFSGovernorCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSGovernorCfg;
+	psRGXTimingInfo = ((RGX_DATA *)psDeviceNode->psDevConfig->hDevData)->psRGXTimingInfo;
+	psDeviceNode->psDevConfig->sDVFS.sDVFSDevice.eState = PVR_DVFS_STATE_READY;
+
+	err = GetOPPValues(psDev, &min_freq, &min_volt, &max_freq, &pvr_freq_table);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to read OPP points, %d", err));
+		eError = TO_IMG_ERR(err);
+		goto err_exit;
+	}
+
+	img_devfreq_dev_profile.freq_table = pvr_freq_table.freq_table;
+	img_devfreq_dev_profile.max_state = pvr_freq_table.num_levels;
+	img_devfreq_dev_profile.initial_freq = min_freq;
+	img_devfreq_dev_profile.polling_ms = psDVFSDeviceCfg->ui32PollMs;
+
+	psRGXTimingInfo->ui32CoreClockSpeed = min_freq;
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDeviceNode->psDevConfig->hSysData, min_freq);
+	psDVFSDeviceCfg->pfnSetVoltage(psDeviceNode->psDevConfig->hSysData, min_volt);
+
+#if !defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* Use the Linux 'simple_ondemand' governor */
+	psDVFSDevice->data.upthreshold = psDVFSGovernorCfg->ui32UpThreshold;
+	psDVFSDevice->data.downdifferential = psDVFSGovernorCfg->ui32DownDifferential;
+#endif
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	psDVFSDevice->data.ui32UpThreshold = psDVFSGovernorCfg->ui32UpThreshold;
+	psDVFSDevice->data.ui32DownDifferential = psDVFSGovernorCfg->ui32DownDifferential;
+
+#if defined(SUPPORT_RGX)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_MEMBUS))
+		{
+			psDVFSDevice->data.uiNumMembus = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_MEMBUS);
+		}
+		else
+		{
+			psDVFSDevice->data.uiNumMembus = 1;
+		}
+	}
+#endif
+#endif
+
+	psDVFSDevice->psDevFreq = devm_devfreq_add_device(psDev,
+													  &img_devfreq_dev_profile,
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+													  "pvr_balanced",
+													  &psDVFSDevice->data);
+#else
+													  "simple_ondemand",
+													  &psDVFSDevice->data);
+#endif
+
+	if (IS_ERR(psDVFSDevice->psDevFreq))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "Failed to add as devfreq device %p, %ld",
+				 psDVFSDevice->psDevFreq,
+				 PTR_ERR(psDVFSDevice->psDevFreq)));
+		eError = TO_IMG_ERR(PTR_ERR(psDVFSDevice->psDevFreq));
+		goto err_exit;
+	}
+#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 50)))
+	/* Handle Linux kernel bug where a NULL return can occur. */
+	if (psDVFSDevice->psDevFreq == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "Failed to add as devfreq device %p, NULL return",
+				 psDVFSDevice->psDevFreq));
+		eError = TO_IMG_ERR(-EINVAL);
+		goto err_exit;
+	}
+#endif
+
+	eError = SuspendDVFS(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVInit: Failed to suspend DVFS"));
+		goto err_exit;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+	psDVFSDevice->psDevFreq->scaling_min_freq = min_freq;
+	psDVFSDevice->psDevFreq->scaling_max_freq = max_freq;
+#else
+	psDVFSDevice->psDevFreq->min_freq = min_freq;
+	psDVFSDevice->psDevFreq->max_freq = max_freq;
+#endif
+#if defined(CONFIG_PM_DEVFREQ_EVENT) && defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	psDVFSDevice->psDevFreq->last_status.private_data =
+		(void *)psDVFSDevice->psProfilingDevice->stat;
+#endif
+
+	err = devfreq_register_opp_notifier(psDev, psDVFSDevice->psDevFreq);
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to register opp notifier, %d", err));
+		eError = TO_IMG_ERR(err);
+		goto err_exit;
+	}
+
+#if defined(CONFIG_POWERVR_THERMAL)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		err = RegisterCoolingDevice(psDev, psDVFSDevice, psDVFSDeviceCfg->psPowerOps);
+		if (err)
+		{
+			eError = TO_IMG_ERR(err);
+			goto err_exit;
+		}
+	}
+#endif
+
+	PVR_TRACE(("PVR DVFS activated: %lu-%lu Hz, Period: %ums",
+			   min_freq,
+			   max_freq,
+			   psDVFSDeviceCfg->ui32PollMs));
+
+	return PVRSRV_OK;
+
+err_exit:
+	UnregisterDVFSDevice(psDeviceNode);
+	return eError;
+}
+
+void UnregisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice = NULL;
+	struct device *psDev = NULL;
+	IMG_INT32 i32Error;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+	if (! psDVFSDevice)
+	{
+		return;
+	}
+
+#if defined(CONFIG_POWERVR_THERMAL)
+	if (!IS_ERR_OR_NULL(psDVFSDevice->psDevfreqCoolingDevice))
+	{
+		devfreq_cooling_unregister(psDVFSDevice->psDevfreqCoolingDevice);
+		psDVFSDevice->psDevfreqCoolingDevice = NULL;
+	}
+#endif
+
+	if (psDVFSDevice->psDevFreq)
+	{
+		i32Error = devfreq_unregister_opp_notifier(psDev, psDVFSDevice->psDevFreq);
+		if (i32Error < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to unregister OPP notifier"));
+		}
+
+		devm_devfreq_remove_device(psDev, psDVFSDevice->psDevFreq);
+		psDVFSDevice->psDevFreq = NULL;
+	}
+
+	psDVFSDevice->eState = PVR_DVFS_STATE_DEINIT;
+}
+
+void DeinitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE *psDVFSDevice = NULL;
+	struct device *psDev = NULL;
+
+	/* Check the device exists */
+	if (!psDeviceNode)
+	{
+		return;
+	}
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	psDev = psDeviceNode->psDevConfig->pvOSDevice;
+
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	if (psDVFSDevice->psProfilingDevice)
+	{
+		pvr_events_unregister(psDev, psDVFSDevice);
+	}
+	if (psDVFSDevice->bGovernorReady)
+	{
+		pvr_governor_exit();
+		psDVFSDevice->bGovernorReady = false;
+	}
+#endif
+
+	/*
+	 * Remove OPP entries for this device; both static entries from
+	 * the device tree and dynamic entries.
+	 */
+	dev_pm_opp_remove_table(psDev);
+
+	SORgxGpuUtilStatsUnregister(psDVFSDevice->hGpuUtilUserDVFS);
+	psDVFSDevice->hGpuUtilUserDVFS = NULL;
+	psDVFSDevice->eState = PVR_DVFS_STATE_NONE;
+}
+
+PVRSRV_ERROR SuspendDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE	*psDVFSDevice = NULL;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+	if (psDVFSDevice->eState == PVR_DVFS_STATE_DEINIT)
+	{
+		/* Device is shutting down, nothing to do. */
+		return PVRSRV_OK;
+	}
+	psDVFSDevice->eState = PVR_DVFS_STATE_OFF;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	/* Communicate power suspend to devfreq framework */
+	err = devfreq_suspend_device(psDVFSDevice->psDevFreq);
+	if (err < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Failed to suspend DVFS (%d)", err));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR ResumeDVFS(PPVRSRV_DEVICE_NODE psDeviceNode)
+{
+	IMG_DVFS_DEVICE	*psDVFSDevice = NULL;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	int err;
+#endif
+
+	/* Check the device is registered */
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	psDVFSDevice = &psDeviceNode->psDevConfig->sDVFS.sDVFSDevice;
+
+	/* Not supported in GuestOS drivers */
+	psDVFSDevice->eState = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? PVR_DVFS_STATE_NONE : PVR_DVFS_STATE_READY;
+#if defined(SUPPORT_PVR_DVFS_GOVERNOR)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Communicate power resume to devfreq framework */
+		err = devfreq_resume_device(psDVFSDevice->psDevFreq);
+		if (err < 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "Failed to resume DVFS (%d)", err));
+			return PVRSRV_ERROR_INVALID_DEVICE;
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+#endif /* !NO_HARDWARE */
diff --git a/drivers/gpu/drm/img-rogue/pvr_dvfs_device.h b/drivers/gpu/drm/img-rogue/pvr_dvfs_device.h
new file mode 100644
index 000000000000..a246b240d9ee
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_dvfs_device.h
@@ -0,0 +1,62 @@
+/*************************************************************************/ /*!
+@File           pvr_dvfs.c
+@Title          System level interface for DVFS
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_DVFS_DEVICE_H
+#define PVR_DVFS_DEVICE_H
+
+#include "opaque_types.h"
+#include "pvrsrv_error.h"
+
+
+PVRSRV_ERROR InitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+void DeinitDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR RegisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+void UnregisterDVFSDevice(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR SuspendDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+PVRSRV_ERROR ResumeDVFS(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+#endif /* PVR_DVFS_DEVICE_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_export_fence.c b/drivers/gpu/drm/img-rogue/pvr_export_fence.c
new file mode 100644
index 000000000000..6ca045a14e68
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_export_fence.c
@@ -0,0 +1,476 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/spinlock_types.h>
+#include <linux/atomic.h>
+#include <linux/slab.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+
+#include "pvr_export_fence.h"
+#include "osfunc_common.h"
+
+struct pvr_exp_fence_context {
+	struct kref kref;
+	unsigned int context;
+	char context_name[32];
+	char driver_name[32];
+	atomic_t seqno;
+	atomic_t fence_count;
+
+	void *cmd_complete_handle;
+
+	/* lock for signal and fence lists */
+	spinlock_t list_lock;
+	struct list_head signal_list;
+	struct list_head fence_list;
+};
+
+struct pvr_exp_fence {
+	struct dma_fence base;
+	struct pvr_exp_fence_context *fence_context;
+	PSYNC_CHECKPOINT checkpoint_handle;
+	/* dma_fence fd (needed for hwperf) */
+	int fd;
+	/* Lock for the dma fence */
+	spinlock_t lock;
+	/* fence will point to the dma_fence in base */
+	struct dma_fence *fence;
+	struct list_head fence_head;
+	struct list_head signal_head;
+};
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+static inline bool
+pvr_exp_fence_sync_is_signaled(struct pvr_exp_fence *exp_fence, u32 fence_sync_flags)
+{
+	if (exp_fence->checkpoint_handle) {
+		return SyncCheckpointIsSignalled(exp_fence->checkpoint_handle,
+						 fence_sync_flags);
+	}
+	return false;
+}
+
+const char *pvr_exp_fence_context_name(struct pvr_exp_fence_context *fctx)
+{
+	return fctx->context_name;
+}
+
+void pvr_exp_fence_context_value_str(struct pvr_exp_fence_context *fctx,
+				    char *str, int size)
+{
+	snprintf(str, size, "%d", atomic_read(&fctx->seqno));
+}
+
+static void
+pvr_exp_fence_context_fences_dump(struct pvr_exp_fence_context *fctx,
+				  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				  void *pvDumpDebugFile)
+{
+	unsigned long flags;
+	char value[128];
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	pvr_exp_fence_context_value_str(fctx, value, sizeof(value));
+	PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+			 "exp_fence_ctx: @%s", value);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+}
+
+static inline unsigned
+pvr_exp_fence_context_seqno_next(struct pvr_exp_fence_context *fence_context)
+{
+	if (fence_context)
+		return atomic_inc_return(&fence_context->seqno) - 1;
+	else
+		return 0xfeedface;
+}
+
+static void
+pvr_exp_fence_context_signal_fences(void *data)
+{
+	struct pvr_exp_fence_context *fctx = (struct pvr_exp_fence_context *)data;
+	struct pvr_exp_fence *pvr_exp_fence, *tmp;
+	unsigned long flags1;
+	int chkpt_ct = 0;
+	int chkpt_sig_ct = 0;
+
+	LIST_HEAD(signal_list);
+
+	/*
+	 * We can't call fence_signal while holding the lock as we can end up
+	 * in a situation whereby pvr_fence_foreign_signal_sync, which also
+	 * takes the list lock, ends up being called as a result of the
+	 * fence_signal below, i.e. fence_signal(fence) -> fence->callback()
+	 *  -> fence_signal(foreign_fence) -> foreign_fence->callback() where
+	 * the foreign_fence callback is pvr_fence_foreign_signal_sync.
+	 *
+	 * So extract the items we intend to signal and add them to their own
+	 * queue.
+	 */
+	spin_lock_irqsave(&fctx->list_lock, flags1);
+	list_for_each_entry_safe(pvr_exp_fence, tmp, &fctx->signal_list, signal_head) {
+		chkpt_ct++;
+		if (pvr_exp_fence_sync_is_signaled(pvr_exp_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT)) {
+			chkpt_sig_ct++;
+			list_move_tail(&pvr_exp_fence->signal_head, &signal_list);
+		}
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags1);
+
+	list_for_each_entry_safe(pvr_exp_fence, tmp, &signal_list, signal_head) {
+		spin_lock_irqsave(&pvr_exp_fence->fence_context->list_lock, flags1);
+		list_del(&pvr_exp_fence->signal_head);
+		spin_unlock_irqrestore(&pvr_exp_fence->fence_context->list_lock, flags1);
+		dma_fence_signal(pvr_exp_fence->fence);
+		dma_fence_put(pvr_exp_fence->fence);
+	}
+}
+
+static const char *pvr_exp_fence_get_driver_name(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		return pvr_exp_fence->fence_context->driver_name;
+	else
+		return "***NO_DRIVER***";
+}
+
+static const char *pvr_exp_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		return pvr_exp_fence_context_name(pvr_exp_fence->fence_context);
+	else
+		return "***NO_TIMELINE***";
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+static void pvr_exp_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	snprintf(str, size, "%llu", (u64) fence->seqno);
+}
+
+static void pvr_exp_fence_timeline_value_str(struct dma_fence *fence,
+					    char *str, int size)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+
+	if (pvr_exp_fence && pvr_exp_fence->fence_context)
+		pvr_exp_fence_context_value_str(pvr_exp_fence->fence_context, str, size);
+}
+#endif
+
+static bool pvr_exp_fence_enable_signaling(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *exp_fence = to_pvr_exp_fence(fence);
+	unsigned long flags;
+
+	if (!exp_fence)
+		return false;
+
+	if (pvr_exp_fence_sync_is_signaled(exp_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return false;
+
+	dma_fence_get(&exp_fence->base);
+
+	spin_lock_irqsave(&exp_fence->fence_context->list_lock, flags);
+	list_add_tail(&exp_fence->signal_head, &exp_fence->fence_context->signal_list);
+	spin_unlock_irqrestore(&exp_fence->fence_context->list_lock, flags);
+
+	return true;
+}
+
+static void pvr_exp_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_exp_fence_context *fence_context =
+		container_of(kref, struct pvr_exp_fence_context, kref);
+	unsigned int fence_count;
+
+	if (WARN_ON(!list_empty_careful(&fence_context->fence_list)))
+		pvr_exp_fence_context_fences_dump(fence_context, NULL, NULL);
+
+	PVRSRVUnregisterCmdCompleteNotify(fence_context->cmd_complete_handle);
+
+	fence_count = atomic_read(&fence_context->fence_count);
+	if (WARN_ON(fence_count))
+		pr_debug("%s context has %u fence(s) remaining\n",
+			 fence_context->context_name, fence_count);
+
+	kfree(fence_context);
+}
+
+static void pvr_exp_fence_release(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fence);
+	unsigned long flags;
+
+	if (pvr_exp_fence) {
+		if (pvr_exp_fence->fence_context) {
+			spin_lock_irqsave(&pvr_exp_fence->fence_context->list_lock, flags);
+			list_del(&pvr_exp_fence->fence_head);
+			atomic_dec(&pvr_exp_fence->fence_context->fence_count);
+			spin_unlock_irqrestore(&pvr_exp_fence->fence_context->list_lock, flags);
+
+			kref_put(&pvr_exp_fence->fence_context->kref,
+				 pvr_exp_fence_context_destroy_kref);
+		}
+
+		if (pvr_exp_fence->checkpoint_handle) {
+			SyncCheckpointFree(pvr_exp_fence->checkpoint_handle);
+			pvr_exp_fence->checkpoint_handle = NULL;
+		}
+
+		kfree(pvr_exp_fence);
+	}
+}
+
+static const struct dma_fence_ops pvr_exp_fence_ops = {
+	.get_driver_name = pvr_exp_fence_get_driver_name,
+	.get_timeline_name = pvr_exp_fence_get_timeline_name,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+	.fence_value_str = pvr_exp_fence_value_str,
+	.timeline_value_str = pvr_exp_fence_timeline_value_str,
+#endif
+	.enable_signaling = pvr_exp_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+	.release = pvr_exp_fence_release,
+};
+
+struct pvr_exp_fence_context *
+pvr_exp_fence_context_create(const char *context_name, const char *driver_name)
+{
+	struct pvr_exp_fence_context *fence_context;
+	PVRSRV_ERROR srv_err;
+
+	fence_context = kzalloc(sizeof(*fence_context), GFP_KERNEL);
+	if (!fence_context)
+		return NULL;
+
+	fence_context->context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fence_context->context_name, context_name,
+		sizeof(fence_context->context_name));
+	OSStringSafeCopy(fence_context->driver_name, driver_name,
+		sizeof(fence_context->driver_name));
+	atomic_set(&fence_context->seqno, 0);
+	atomic_set(&fence_context->fence_count, 0);
+	kref_init(&fence_context->kref);
+
+	spin_lock_init(&fence_context->list_lock);
+	INIT_LIST_HEAD(&fence_context->signal_list);
+	INIT_LIST_HEAD(&fence_context->fence_list);
+
+	srv_err = PVRSRVRegisterCmdCompleteNotify(&fence_context->cmd_complete_handle,
+				pvr_exp_fence_context_signal_fences,
+				fence_context);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register command complete callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+		kfree(fence_context);
+		return NULL;
+	}
+
+	return fence_context;
+}
+
+void pvr_exp_fence_context_destroy(struct pvr_exp_fence_context *fence_context)
+{
+	if (fence_context) {
+		kref_put(&fence_context->kref, pvr_exp_fence_context_destroy_kref);
+	}
+}
+
+struct dma_fence *
+pvr_exp_fence_create(struct pvr_exp_fence_context *fence_context, int fd, u64 *sync_pt_idx)
+{
+	struct pvr_exp_fence *pvr_exp_fence;
+	unsigned long flags;
+	unsigned int seqno;
+	struct pvr_exp_fence_context *pfence_context = fence_context;
+
+	if (WARN_ON(!fence_context))
+		return NULL;
+
+	pvr_exp_fence = kzalloc(sizeof(*pvr_exp_fence), GFP_KERNEL);
+	if (WARN_ON(!pvr_exp_fence))
+		return NULL;
+
+	spin_lock_init(&pvr_exp_fence->lock);
+
+	INIT_LIST_HEAD(&pvr_exp_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_exp_fence->signal_head);
+
+	pvr_exp_fence->fence_context = pfence_context;
+
+	/* No sync checkpoint is assigned until attached to a kick */
+	pvr_exp_fence->checkpoint_handle = NULL;
+
+	seqno = pvr_exp_fence_context_seqno_next(pfence_context);
+
+	pvr_exp_fence->fence = &pvr_exp_fence->base;
+
+	dma_fence_init(&pvr_exp_fence->base, &pvr_exp_fence_ops,
+		       &pvr_exp_fence->lock, pfence_context->context, seqno);
+
+	atomic_inc(&pfence_context->fence_count);
+
+	spin_lock_irqsave(&fence_context->list_lock, flags);
+	list_add_tail(&pvr_exp_fence->fence_head, &fence_context->fence_list);
+	spin_unlock_irqrestore(&fence_context->list_lock, flags);
+
+	kref_get(&pfence_context->kref);
+
+	pvr_exp_fence->fd = fd;
+
+	*sync_pt_idx = pvr_exp_fence_context_seqno_next(pfence_context);
+
+	return &pvr_exp_fence->base;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_exp_fence_assign_checkpoint(PVRSRV_FENCE fence_to_resolve,
+				struct dma_fence *fence,
+				PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+				PSYNC_CHECKPOINT *assigned_checkpoint)
+{
+	struct SYNC_CHECKPOINT_TAG *new_sync_checkpoint;
+	struct pvr_exp_fence *pvr_exp_fence;
+	PVRSRV_FENCE export_fence_fd = fence_to_resolve;
+	PVRSRV_ERROR err;
+
+	pvr_exp_fence = to_pvr_exp_fence(fence);
+	if (!pvr_exp_fence)	{
+		pr_err("%s: Invalid fence_to_resolve\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (pvr_exp_fence->checkpoint_handle) {
+		/* export fence already has a sync checkpoint assigned */
+		*assigned_checkpoint = pvr_exp_fence->checkpoint_handle;
+		return PVRSRV_OK;
+	}
+
+	/* Ensure fd assigned to export fence when it was created is passed
+	 * to SyncCheckpointAlloc() (so correct fd is used in HWPerf event)
+	 * as the export fence may only be part of the check fence.
+	 */
+	if (pvr_exp_fence->fd != PVRSRV_NO_FENCE) {
+		export_fence_fd = pvr_exp_fence->fd;
+	}
+
+	err = SyncCheckpointAlloc(checkpoint_context,
+				  PVRSRV_NO_TIMELINE, export_fence_fd,
+				  fence->ops->get_timeline_name(fence), &new_sync_checkpoint);
+	if (unlikely(err != PVRSRV_OK)) {
+		pr_err("%s: SyncCheckpointAlloc() failed (err%d)\n",
+		       __func__, err);
+		*assigned_checkpoint = NULL;
+		goto err_out;
+	}
+
+	pvr_exp_fence->checkpoint_handle = new_sync_checkpoint;
+	*assigned_checkpoint = new_sync_checkpoint;
+	return PVRSRV_OK;
+
+err_out:
+	return err;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_exp_fence_rollback(struct dma_fence *fence)
+{
+	struct pvr_exp_fence *pvr_exp_fence;
+	PVRSRV_ERROR err;
+
+	pvr_exp_fence = to_pvr_exp_fence(fence);
+	if (!pvr_exp_fence) {
+		pr_err("%s: Invalid fence\n", __func__);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (pvr_exp_fence->checkpoint_handle) {
+		/* Free the assigned sync checkpoint */
+		SyncCheckpointFree(pvr_exp_fence->checkpoint_handle);
+		pvr_exp_fence->checkpoint_handle = NULL;
+	}
+
+	return PVRSRV_OK;
+
+err_out:
+	return err;
+}
+
+bool pvr_is_exp_fence(struct dma_fence *fence)
+{
+	return (fence->ops == &pvr_exp_fence_ops);
+}
+
+struct pvr_exp_fence *to_pvr_exp_fence(struct dma_fence *fence)
+{
+	if (pvr_is_exp_fence(fence))
+		return container_of(fence, struct pvr_exp_fence, base);
+
+	return NULL;
+}
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_exp_fence_get_checkpoint(struct pvr_exp_fence *export_fence)
+{
+	if (export_fence) {
+		return export_fence->checkpoint_handle;
+	}
+	else
+		return NULL;
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_export_fence.h b/drivers/gpu/drm/img-rogue/pvr_export_fence.h
new file mode 100644
index 000000000000..0ffe8a11ce40
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_export_fence.h
@@ -0,0 +1,77 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_EXPORT_FENCES_H__)
+#define __PVR_EXPORT_FENCES_H__
+
+#include "pvr_linux_fence.h"
+#include "services_kernel_client.h"
+
+struct pvr_exp_fence_context;
+struct pvr_exp_fence;
+
+struct pvr_exp_fence_context *pvr_exp_fence_context_create(const char *name,
+				const char *driver_name);
+void pvr_exp_fence_context_destroy(struct pvr_exp_fence_context *fence_context);
+struct dma_fence *pvr_exp_fence_create(struct pvr_exp_fence_context *fence_context,
+				       int fd,
+				       u64 *sync_pt_idx);
+
+const char *pvr_exp_fence_context_name(struct pvr_exp_fence_context *fctx);
+void pvr_exp_fence_context_value_str(struct pvr_exp_fence_context *fctx,
+				    char *str, int size);
+
+enum PVRSRV_ERROR_TAG pvr_exp_fence_assign_checkpoint(PVRSRV_FENCE fence_to_resolve,
+						      struct dma_fence *fence,
+						      PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+						      PSYNC_CHECKPOINT *assigned_checkpoint);
+
+enum PVRSRV_ERROR_TAG pvr_exp_fence_rollback(struct dma_fence *fence);
+
+bool pvr_is_exp_fence(struct dma_fence *fence);
+
+struct pvr_exp_fence *to_pvr_exp_fence(struct dma_fence *fence);
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_exp_fence_get_checkpoint(struct pvr_exp_fence *export_fence);
+
+#endif /* !defined(__PVR_EXPORT_FENCES_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_fd_sync_kernel.h b/drivers/gpu/drm/img-rogue/pvr_fd_sync_kernel.h
new file mode 100644
index 000000000000..3645e29079b1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_fd_sync_kernel.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File           pvr_fd_sync_kernel.h
+@Title          Kernel/userspace interface definitions to use the kernel sync
+                driver
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+
+#ifndef _PVR_FD_SYNC_KERNEL_H_
+#define _PVR_FD_SYNC_KERNEL_H_
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+#include "pvr_drm.h"
+
+#define PVR_SYNC_MAX_QUERY_FENCE_POINTS 14
+
+struct pvr_sync_pt_info {
+	/* Output */
+	__u32 id;
+	__u32 ui32FWAddr;
+	__u32 ui32CurrOp;
+	__u32 ui32NextOp;
+	__u32 ui32TlTaken;
+} __attribute__((packed, aligned(8)));
+
+#endif /* _PVR_FD_SYNC_KERNEL_H_ */
diff --git a/drivers/gpu/drm/img-rogue/pvr_fence.c b/drivers/gpu/drm/img-rogue/pvr_fence.c
new file mode 100644
index 000000000000..e2df3c2dbc3e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_fence.c
@@ -0,0 +1,1177 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include "pvr_fence.h"
+#include "services_kernel_client.h"
+#include "sync_checkpoint_external.h"
+#include "osfunc_common.h"
+
+#define CREATE_TRACE_POINTS
+#include "pvr_fence_trace.h"
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+/* Global kmem_cache for pvr_fence object allocations */
+static struct kmem_cache *pvr_fence_cache;
+static DEFINE_MUTEX(pvr_fence_cache_mutex);
+static u32 pvr_fence_cache_refcount;
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+static inline void
+pvr_fence_sync_signal(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
+{
+	SyncCheckpointSignal(pvr_fence->sync_checkpoint, fence_sync_flags);
+}
+
+static inline bool
+pvr_fence_sync_is_signaled(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
+{
+	return SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
+					 fence_sync_flags);
+}
+
+static inline u32
+pvr_fence_sync_value(struct pvr_fence *pvr_fence)
+{
+	if (SyncCheckpointIsErrored(pvr_fence->sync_checkpoint,
+				    PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return PVRSRV_SYNC_CHECKPOINT_ERRORED;
+	else if (SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
+					   PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+	else
+		return PVRSRV_SYNC_CHECKPOINT_ACTIVE;
+}
+
+static void
+pvr_fence_context_check_status(struct work_struct *data)
+{
+	PVRSRVCheckStatus(NULL);
+}
+
+void
+pvr_context_value_str(struct pvr_fence_context *fctx, char *str, int size)
+{
+	snprintf(str, size,
+		 "%u ctx=%llu refs=%u",
+		 atomic_read(&fctx->fence_seqno),
+		 fctx->fence_context,
+		 refcount_read(&fctx->kref.refcount));
+}
+
+static void
+pvr_fence_context_fences_dump(struct pvr_fence_context *fctx,
+			      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			      void *pvDumpDebugFile)
+{
+	struct pvr_fence *pvr_fence;
+	unsigned long flags;
+	char value[128];
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	pvr_context_value_str(fctx, value, sizeof(value));
+	PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+			 "%s: @%s", fctx->name, value);
+	list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
+		struct dma_fence *fence = pvr_fence->fence;
+		const char *timeline_value_str = "unknown timeline value";
+		const char *fence_value_str = "unknown fence value";
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		pvr_fence_fence_value_str(&pvr_fence->base, value, sizeof(value));
+#else
+		pvr_fence->base.ops->fence_value_str(&pvr_fence->base, value,
+						     sizeof(value));
+#endif
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " @%s", value);
+
+		if (is_pvr_fence(fence))
+			continue;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		pvr_fence_foreign_timeline_value_str(fence, value, sizeof(value));
+		timeline_value_str = value;
+#else
+		if (fence->ops->timeline_value_str) {
+			fence->ops->timeline_value_str(fence, value, sizeof(value));
+			timeline_value_str = value;
+		}
+#endif
+
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " | %s: %s (driver: %s)",
+				  fence->ops->get_timeline_name(fence),
+				  timeline_value_str,
+				  fence->ops->get_driver_name(fence));
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+			pvr_fence_foreign_fence_value_str(fence, value, sizeof(value));
+			fence_value_str = value;
+#else
+		if (fence->ops->fence_value_str) {
+			fence->ops->fence_value_str(fence, value,
+						    sizeof(value));
+			fence_value_str = value;
+		}
+#endif
+
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  " |  @%s (foreign)", fence_value_str);
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+}
+
+static inline unsigned int
+pvr_fence_context_seqno_next(struct pvr_fence_context *fctx)
+{
+	return atomic_inc_return(&fctx->fence_seqno) - 1;
+}
+
+/* This function prepends seqno to fence name */
+static inline void
+pvr_fence_prepare_name(char *fence_name, size_t fence_name_size,
+		const char *name, unsigned int seqno)
+{
+	unsigned int len;
+
+	len = OSStringUINT32ToStr(fence_name, fence_name_size, seqno);
+	if (likely((len > 0) && (fence_name_size >= (len + 1)))) {
+		fence_name[len] = '-';
+		fence_name[len + 1] = '\0';
+	}
+	strlcat(fence_name, name, fence_name_size);
+}
+
+static void
+pvr_fence_sched_free(struct rcu_head *rcu)
+{
+	struct pvr_fence *pvr_fence = container_of(rcu, struct pvr_fence, rcu);
+
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+}
+
+static inline void
+pvr_fence_context_free_deferred(struct pvr_fence_context *fctx)
+{
+	struct pvr_fence *pvr_fence, *tmp;
+	LIST_HEAD(deferred_free_list);
+	unsigned long flags;
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_for_each_entry_safe(pvr_fence, tmp,
+				 &fctx->deferred_free_list,
+				 fence_head)
+		list_move(&pvr_fence->fence_head, &deferred_free_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+	list_for_each_entry_safe(pvr_fence, tmp,
+				 &deferred_free_list,
+				 fence_head) {
+		list_del(&pvr_fence->fence_head);
+		SyncCheckpointFree(pvr_fence->sync_checkpoint);
+		call_rcu(&pvr_fence->rcu, pvr_fence_sched_free);
+		module_put(THIS_MODULE);
+	}
+}
+
+void
+pvr_fence_context_free_deferred_callback(void *data)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+
+	/*
+	 * Free up any fence objects we have deferred freeing.
+	 */
+	pvr_fence_context_free_deferred(fctx);
+}
+
+static void
+pvr_fence_context_signal_fences(void *data)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+	struct pvr_fence *pvr_fence, *tmp;
+	unsigned long flags1;
+
+	LIST_HEAD(signal_list);
+
+	/*
+	 * We can't call fence_signal while holding the lock as we can end up
+	 * in a situation whereby pvr_fence_foreign_signal_sync, which also
+	 * takes the list lock, ends up being called as a result of the
+	 * fence_signal below, i.e. fence_signal(fence) -> fence->callback()
+	 *  -> fence_signal(foreign_fence) -> foreign_fence->callback() where
+	 * the foreign_fence callback is pvr_fence_foreign_signal_sync.
+	 *
+	 * So extract the items we intend to signal and add them to their own
+	 * queue.
+	 */
+	spin_lock_irqsave(&fctx->list_lock, flags1);
+	list_for_each_entry_safe(pvr_fence, tmp, &fctx->signal_list, signal_head) {
+		if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+			list_move_tail(&pvr_fence->signal_head, &signal_list);
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags1);
+
+	list_for_each_entry_safe(pvr_fence, tmp, &signal_list, signal_head) {
+
+		PVR_FENCE_TRACE(&pvr_fence->base, "signalled fence (%s)\n",
+				pvr_fence->name);
+		trace_pvr_fence_signal_fence(pvr_fence);
+		spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags1);
+		list_del(&pvr_fence->signal_head);
+		spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags1);
+		if (pvr_fence_sync_value(pvr_fence) == PVRSRV_SYNC_CHECKPOINT_ERRORED)
+			dma_fence_set_error(pvr_fence->fence, -EIO);
+		dma_fence_signal(pvr_fence->fence);
+		dma_fence_put(pvr_fence->fence);
+	}
+
+	/*
+	 * Take this opportunity to free up any fence objects we
+	 * have deferred freeing.
+	 */
+	pvr_fence_context_free_deferred(fctx);
+}
+
+void
+pvr_fence_context_signal_fences_nohw(void *data)
+{
+	pvr_fence_context_signal_fences(data);
+}
+
+static void
+pvr_fence_context_destroy_internal(struct pvr_fence_context *fctx)
+{
+	pvr_fence_context_free_deferred(fctx);
+
+	if (WARN_ON(!list_empty_careful(&fctx->fence_list)))
+		pvr_fence_context_fences_dump(fctx, NULL, NULL);
+
+	PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
+
+	// wait for all fences to be freed before kmem_cache_destroy() is called
+	rcu_barrier();
+
+	/* Destroy pvr_fence object cache, if no one is using it */
+	WARN_ON(pvr_fence_cache == NULL);
+	mutex_lock(&pvr_fence_cache_mutex);
+	if (--pvr_fence_cache_refcount == 0)
+		kmem_cache_destroy(pvr_fence_cache);
+	mutex_unlock(&pvr_fence_cache_mutex);
+
+	kfree(fctx);
+}
+
+static void
+pvr_fence_context_unregister_dbg(void *dbg_request_handle)
+{
+	PVRSRVUnregisterDeviceDbgRequestNotify(dbg_request_handle);
+}
+
+static void
+pvr_fence_foreign_context_destroy_work(struct work_struct *data)
+{
+	struct pvr_fence_context *fctx =
+		container_of(data, struct pvr_fence_context, destroy_work);
+
+	pvr_fence_context_destroy_internal(fctx);
+}
+
+static void
+pvr_fence_context_destroy_work(struct work_struct *data)
+{
+	struct pvr_fence_context *fctx =
+		container_of(data, struct pvr_fence_context, destroy_work);
+
+	pvr_fence_context_unregister_dbg(fctx->dbg_request_handle);
+	pvr_fence_context_destroy_internal(fctx);
+}
+
+static void
+pvr_fence_context_debug_request(void *data, u32 verbosity,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
+
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+		pvr_fence_context_fences_dump(fctx, pfnDumpDebugPrintf,
+					      pvDumpDebugFile);
+}
+
+static struct pvr_fence_context *
+pvr_fence_context_create_internal(struct workqueue_struct *fence_status_wq,
+			const char *name,
+			work_func_t destroy_callback)
+{
+	struct pvr_fence_context *fctx;
+	PVRSRV_ERROR srv_err;
+
+	fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
+	if (!fctx)
+		return NULL;
+
+	spin_lock_init(&fctx->lock);
+	atomic_set(&fctx->fence_seqno, 0);
+	INIT_WORK(&fctx->check_status_work, pvr_fence_context_check_status);
+	INIT_WORK(&fctx->destroy_work, destroy_callback);
+	spin_lock_init(&fctx->list_lock);
+	INIT_LIST_HEAD(&fctx->signal_list);
+	INIT_LIST_HEAD(&fctx->fence_list);
+	INIT_LIST_HEAD(&fctx->deferred_free_list);
+
+	fctx->fence_wq = fence_status_wq;
+
+	fctx->fence_context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fctx->name, name, sizeof(fctx->name));
+
+	srv_err = PVRSRVRegisterCmdCompleteNotify(&fctx->cmd_complete_handle,
+				pvr_fence_context_signal_fences,
+				fctx);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register command complete callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+		goto err_free_fctx;
+	}
+
+	/* Create pvr_fence object cache, if not already created */
+	mutex_lock(&pvr_fence_cache_mutex);
+	if (pvr_fence_cache_refcount == 0) {
+		pvr_fence_cache = KMEM_CACHE(pvr_fence, 0);
+		if (!pvr_fence_cache) {
+			pr_err("%s: failed to allocate pvr_fence cache\n",
+					__func__);
+			mutex_unlock(&pvr_fence_cache_mutex);
+			goto err_unregister_cmd_complete_notify;
+		}
+	}
+	pvr_fence_cache_refcount++;
+	mutex_unlock(&pvr_fence_cache_mutex);
+
+	kref_init(&fctx->kref);
+
+	PVR_FENCE_CTX_TRACE(fctx, "created fence context (%s)\n", name);
+	trace_pvr_fence_context_create(fctx);
+
+	return fctx;
+
+err_unregister_cmd_complete_notify:
+	PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
+err_free_fctx:
+	kfree(fctx);
+	return NULL;
+}
+
+/**
+ * pvr_fence_context_register_dbg - registers the debug handler for a
+ * fence context
+ *
+ * @dbg_request_handle: handle used to keep a reference for deregister
+ * @dev: device to attach the debug notifier.
+ * @pvr_fence_context: context used as data to the callback for debug
+ *
+ * Registers a debug notifier for a given context for a given device.
+ *
+ * Returns PVRSRV_OK if successful.
+ */
+PVRSRV_ERROR pvr_fence_context_register_dbg(void *dbg_request_handle,
+				void *dev,
+				struct pvr_fence_context *fctx)
+{
+	PVRSRV_ERROR srv_err;
+
+	srv_err = PVRSRVRegisterDeviceDbgRequestNotify(dbg_request_handle,
+				dev,
+				pvr_fence_context_debug_request,
+				DEBUG_REQUEST_LINUXFENCE,
+				fctx);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(srv_err));
+	}
+
+	return srv_err;
+}
+
+/**
+ * pvr_fence_foreign_context_create - creates a PVR fence context
+ * @fence_status_wq: linux workqueue used to signal foreign fences
+ * @name: context name (used for debugging)
+ *
+ * Creates a PVR foreign fence context that can be used to create PVR fences
+ * or to create PVR fences from an existing fence.
+ *
+ * pvr_fence_context_destroy should be called to clean up the fence context.
+ *
+ * Returns NULL if a context cannot be created.
+ */
+struct pvr_fence_context *
+pvr_fence_foreign_context_create(struct workqueue_struct *fence_status_wq,
+		const char *name)
+{
+	return pvr_fence_context_create_internal(fence_status_wq, name,
+							pvr_fence_foreign_context_destroy_work);
+}
+
+/**
+ * pvr_fence_context_create - creates a PVR fence context
+ * @dev_cookie: services device cookie
+ * @fence_status_wq: Status workqueue to queue fence update CBs.
+ * @name: context name (used for debugging)
+ *
+ * Creates a PVR fence context that can be used to create PVR fences or to
+ * create PVR fences from an existing fence.
+ *
+ * pvr_fence_context_destroy should be called to clean up the fence context.
+ *
+ * Returns NULL if a context cannot be created.
+ */
+struct pvr_fence_context *
+pvr_fence_context_create(void *dev_cookie,
+			 struct workqueue_struct *fence_status_wq,
+			 const char *name)
+{
+	struct pvr_fence_context *fctx;
+	PVRSRV_ERROR eError;
+
+	fctx = pvr_fence_context_create_internal(fence_status_wq, name,
+					pvr_fence_context_destroy_work);
+	if (fctx == NULL) {
+		pr_err("%s: failed to create fence context", __func__);
+		goto err_out;
+	}
+
+	fctx->dev_cookie = dev_cookie;
+
+	eError = pvr_fence_context_register_dbg(&fctx->dbg_request_handle,
+					dev_cookie,
+					fctx);
+	if (eError != PVRSRV_OK) {
+		pr_err("%s: failed to register fence context debug (%s)\n",
+		       __func__, PVRSRVGetErrorString(eError));
+		goto err_destroy_ctx;
+	}
+
+	return fctx;
+
+err_destroy_ctx:
+	pvr_fence_context_destroy(fctx);
+err_out:
+	return NULL;
+}
+
+static void pvr_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_fence_context *fctx =
+		container_of(kref, struct pvr_fence_context, kref);
+
+	PVR_FENCE_CTX_TRACE(fctx, "destroyed fence context (%s)\n", fctx->name);
+
+	trace_pvr_fence_context_destroy_kref(fctx);
+
+	queue_work(NativeSyncGetFenceCtxDestroyWq(), &fctx->destroy_work);
+}
+
+/**
+ * pvr_fence_context_destroy - destroys a context
+ * @fctx: PVR fence context to destroy
+ *
+ * Destroys a PVR fence context with the expectation that all fences have been
+ * destroyed.
+ */
+void
+pvr_fence_context_destroy(struct pvr_fence_context *fctx)
+{
+	trace_pvr_fence_context_destroy(fctx);
+
+	kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+}
+
+static const char *
+pvr_fence_get_driver_name(struct dma_fence *fence)
+{
+	return PVR_LDM_DRIVER_REGISTRATION_NAME;
+}
+
+static const char *
+pvr_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		return pvr_fence->fctx->name;
+	return NULL;
+}
+
+void pvr_fence_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (!pvr_fence)
+		return;
+
+	snprintf(str, size,
+		 "%llu: (%s%s) refs=%u fwaddr=%#08x enqueue=%u status=%-9s %s%s",
+		 (u64) pvr_fence->fence->seqno,
+		 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 refcount_read(&pvr_fence->fence->refcount.refcount),
+		 SyncCheckpointGetFirmwareAddr(
+			 pvr_fence->sync_checkpoint),
+		 SyncCheckpointGetEnqueuedCount(pvr_fence->sync_checkpoint),
+		 SyncCheckpointGetStateString(pvr_fence->sync_checkpoint),
+		 pvr_fence->name,
+		 (&pvr_fence->base != pvr_fence->fence) ?
+		 "(foreign)" : "");
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+static
+void pvr_fence_timeline_value_str(struct dma_fence *fence, char *str, int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		pvr_context_value_str(pvr_fence->fctx, str, size);
+}
+#endif
+
+static bool
+pvr_fence_enable_signaling(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (!pvr_fence)
+		return false;
+
+	WARN_ON_SMP(!spin_is_locked(&pvr_fence->fctx->lock));
+
+	if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+		return false;
+
+	dma_fence_get(&pvr_fence->base);
+
+	spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->signal_head, &pvr_fence->fctx->signal_list);
+	spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "signalling enabled (%s)\n",
+			pvr_fence->name);
+	trace_pvr_fence_enable_signaling(pvr_fence);
+
+	return true;
+}
+
+static bool
+pvr_fence_is_signaled(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		return pvr_fence_sync_is_signaled(pvr_fence,
+						  PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT);
+	return false;
+}
+
+static void
+pvr_fence_release(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (pvr_fence) {
+		struct pvr_fence_context *fctx = pvr_fence->fctx;
+
+		PVR_FENCE_TRACE(&pvr_fence->base, "released fence (%s)\n",
+				pvr_fence->name);
+		trace_pvr_fence_release(pvr_fence);
+
+		spin_lock_irqsave(&fctx->list_lock, flags);
+		list_move(&pvr_fence->fence_head,
+			  &fctx->deferred_free_list);
+		spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+		kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+	}
+}
+
+const struct dma_fence_ops pvr_fence_ops = {
+	.get_driver_name = pvr_fence_get_driver_name,
+	.get_timeline_name = pvr_fence_get_timeline_name,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+	.fence_value_str = pvr_fence_fence_value_str,
+	.timeline_value_str = pvr_fence_timeline_value_str,
+#endif
+	.enable_signaling = pvr_fence_enable_signaling,
+	.signaled = pvr_fence_is_signaled,
+	.wait = dma_fence_default_wait,
+	.release = pvr_fence_release,
+};
+
+/**
+ * pvr_fence_create - creates a PVR fence
+ * @fctx: PVR fence context on which the PVR fence should be created
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @timeline_fd: timeline on which the PVR fence should be created
+ * @name: PVR fence name (used for debugging)
+ *
+ * Creates a PVR fence.
+ *
+ * Once the fence is finished with, pvr_fence_destroy should be called.
+ *
+ * Returns NULL if a PVR fence cannot be created.
+ */
+struct pvr_fence *
+pvr_fence_create(struct pvr_fence_context *fctx,
+		struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+		int timeline_fd, const char *name)
+{
+	struct pvr_fence *pvr_fence;
+	unsigned int seqno;
+	unsigned long flags;
+	PVRSRV_ERROR srv_err;
+
+	if (!try_module_get(THIS_MODULE))
+		goto err_exit;
+
+	/* Note: As kmem_cache is used to allocate pvr_fence objects,
+	 * make sure that all members of pvr_fence struct are initialized
+	 * here
+	 */
+	pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
+	if (unlikely(!pvr_fence))
+		goto err_module_put;
+
+	srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
+				      (PVRSRV_TIMELINE) timeline_fd, PVRSRV_NO_FENCE,
+				      name, &pvr_fence->sync_checkpoint);
+	if (unlikely(srv_err != PVRSRV_OK))
+		goto err_free_fence;
+
+	INIT_LIST_HEAD(&pvr_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_fence->signal_head);
+	pvr_fence->fctx = fctx;
+	seqno = pvr_fence_context_seqno_next(fctx);
+	/* Add the seqno to the fence name for easier debugging */
+	pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
+			name, seqno);
+
+	/* Reset cb to zero */
+	memset(&pvr_fence->cb, 0, sizeof(pvr_fence->cb));
+	pvr_fence->fence = &pvr_fence->base;
+
+	dma_fence_init(&pvr_fence->base, &pvr_fence_ops, &fctx->lock,
+		       fctx->fence_context, seqno);
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+	kref_get(&fctx->kref);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "created fence (%s)\n", name);
+	trace_pvr_fence_create(pvr_fence);
+
+	return pvr_fence;
+
+err_free_fence:
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+err_module_put:
+	module_put(THIS_MODULE);
+err_exit:
+	return NULL;
+}
+
+static const char *
+pvr_fence_foreign_get_driver_name(struct dma_fence *fence)
+{
+	return PVR_LDM_DRIVER_REGISTRATION_NAME;
+}
+
+static const char *
+pvr_fence_foreign_get_timeline_name(struct dma_fence *fence)
+{
+	return "foreign";
+}
+
+void pvr_fence_foreign_fence_value_str(struct dma_fence *fence, char *str,
+				       int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	u32 sync_addr = 0;
+	u32 sync_value_next;
+
+	if (WARN_ON(!pvr_fence))
+		return;
+
+	sync_addr = SyncCheckpointGetFirmwareAddr(pvr_fence->sync_checkpoint);
+	sync_value_next = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+
+	/*
+	 * Include the fence flag bits from the foreign fence instead of our
+	 * shadow copy. This is done as the shadow fence flag bits aren't used.
+	 */
+	snprintf(str, size,
+		 "%llu: (%s%s) refs=%u fwaddr=%#08x cur=%#08x nxt=%#08x %s",
+		 (u64) fence->seqno,
+		 test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			  &pvr_fence->fence->flags) ? "+" : "-",
+		 refcount_read(&fence->refcount.refcount),
+		 sync_addr,
+		 pvr_fence_sync_value(pvr_fence),
+		 sync_value_next,
+		 pvr_fence->name);
+}
+
+void pvr_fence_foreign_timeline_value_str(struct dma_fence *fence, char *str,
+					  int size)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+	if (pvr_fence)
+		pvr_context_value_str(pvr_fence->fctx, str, size);
+}
+
+static bool
+pvr_fence_foreign_enable_signaling(struct dma_fence *fence)
+{
+	WARN_ON("cannot enable signalling on foreign fence");
+	return false;
+}
+
+static signed long
+pvr_fence_foreign_wait(struct dma_fence *fence, bool intr, signed long timeout)
+{
+	WARN_ON("cannot wait on foreign fence");
+	return 0;
+}
+
+static void
+pvr_fence_foreign_release(struct dma_fence *fence)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned long flags;
+
+	if (pvr_fence) {
+		struct pvr_fence_context *fctx = pvr_fence->fctx;
+		struct dma_fence *foreign_fence = pvr_fence->fence;
+
+		PVR_FENCE_TRACE(&pvr_fence->base,
+				"released fence for foreign fence %llu#%d (%s)\n",
+				(u64) pvr_fence->fence->context,
+				pvr_fence->fence->seqno, pvr_fence->name);
+		trace_pvr_fence_foreign_release(pvr_fence);
+
+		spin_lock_irqsave(&fctx->list_lock, flags);
+		list_move(&pvr_fence->fence_head,
+			  &fctx->deferred_free_list);
+		spin_unlock_irqrestore(&fctx->list_lock, flags);
+
+		dma_fence_put(foreign_fence);
+
+		kref_put(&fctx->kref,
+			 pvr_fence_context_destroy_kref);
+	}
+}
+
+const struct dma_fence_ops pvr_fence_foreign_ops = {
+	.get_driver_name = pvr_fence_foreign_get_driver_name,
+	.get_timeline_name = pvr_fence_foreign_get_timeline_name,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+	.fence_value_str = pvr_fence_foreign_fence_value_str,
+	.timeline_value_str = pvr_fence_foreign_timeline_value_str,
+#endif
+	.enable_signaling = pvr_fence_foreign_enable_signaling,
+	.wait = pvr_fence_foreign_wait,
+	.release = pvr_fence_foreign_release,
+};
+
+static void
+pvr_fence_foreign_signal_sync(struct dma_fence *fence, struct dma_fence_cb *cb)
+{
+	struct pvr_fence *pvr_fence = container_of(cb, struct pvr_fence, cb);
+	struct pvr_fence_context *fctx = pvr_fence->fctx;
+
+	/* Callback registered by dma_fence_add_callback can be called from an atomic ctx */
+	if (fence->error < 0)
+		SyncCheckpointError(pvr_fence->sync_checkpoint, PVRSRV_FENCE_FLAG_CTX_ATOMIC);
+	else
+		pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_CTX_ATOMIC);
+
+	trace_pvr_fence_foreign_signal(pvr_fence);
+
+	queue_work(fctx->fence_wq, &fctx->check_status_work);
+
+	PVR_FENCE_TRACE(&pvr_fence->base,
+			"foreign fence %llu#%d signalled (%s)\n",
+			(u64) pvr_fence->fence->context,
+			pvr_fence->fence->seqno, pvr_fence->name);
+
+	/* Drop the reference on the base fence */
+	dma_fence_put(&pvr_fence->base);
+}
+
+/**
+ * pvr_fence_create_from_fence - creates a PVR fence from a fence
+ * @fctx: PVR fence context on which the PVR fence should be created
+ * @sync_checkpoint_ctx: context in which to create sync checkpoints
+ * @fence: fence from which the PVR fence should be created
+ * @fence_fd: fd for the sync file to which the fence belongs. If it doesn't
+ *            belong to a sync file then PVRSRV_NO_FENCE should be given
+ *            instead.
+ * @name: PVR fence name (used for debugging)
+ *
+ * Creates a PVR fence from an existing fence. If the fence is a foreign fence,
+ * i.e. one that doesn't originate from a PVR fence context, then a new PVR
+ * fence will be created using the specified sync_checkpoint_context.
+ * Otherwise, a reference will be taken on the underlying fence and the PVR
+ * fence will be returned.
+ *
+ * Once the fence is finished with, pvr_fence_destroy should be called.
+ *
+ * Returns NULL if a PVR fence cannot be created.
+ */
+
+struct pvr_fence *
+pvr_fence_create_from_fence(struct pvr_fence_context *fctx,
+			    struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+			    struct dma_fence *fence,
+			    PVRSRV_FENCE fence_fd,
+			    const char *name)
+{
+	struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+	unsigned int seqno;
+	unsigned long flags;
+	PVRSRV_ERROR srv_err;
+	int err;
+	bool mirror_other_dev_fence = false;
+	char tempString[40] = {0};
+
+	if (pvr_fence) {
+		if ((fctx->dev_cookie == NULL) ||
+		    (fctx->dev_cookie == pvr_fence->fctx->dev_cookie)) {
+			if (WARN_ON(fence->ops == &pvr_fence_foreign_ops))
+				return NULL;
+			dma_fence_get(fence);
+
+			PVR_FENCE_TRACE(fence, "created fence from PVR fence (%s)\n",
+					name);
+			return pvr_fence;
+		} else {
+			snprintf(tempString, sizeof(tempString), "Mirror(FWAddr0x%x)",
+				 SyncCheckpointGetFirmwareAddr(pvr_fence->sync_checkpoint));
+			mirror_other_dev_fence = true;
+		}
+	}
+
+	if (!try_module_get(THIS_MODULE))
+		goto err_exit;
+
+	/* Note: As kmem_cache is used to allocate pvr_fence objects,
+	 * make sure that all members of pvr_fence struct are initialized
+	 * here
+	 */
+	pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
+	if (!pvr_fence) {
+		pr_err("%s: kmem_cache_alloc() failed",
+			   __func__);
+		goto err_module_put;
+	}
+
+	srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
+					  SYNC_CHECKPOINT_FOREIGN_CHECKPOINT,
+					  fence_fd,
+					  mirror_other_dev_fence ? tempString : name,
+					  &pvr_fence->sync_checkpoint);
+	if (srv_err != PVRSRV_OK) {
+		pr_err("%s: SyncCheckpointAlloc() failed (srv_err=%d)",
+			   __func__, srv_err);
+		goto err_free_pvr_fence;
+	}
+
+	INIT_LIST_HEAD(&pvr_fence->fence_head);
+	INIT_LIST_HEAD(&pvr_fence->signal_head);
+	pvr_fence->fctx = fctx;
+	pvr_fence->fence = dma_fence_get(fence);
+	seqno = pvr_fence_context_seqno_next(fctx);
+	/* Add the seqno to the fence name for easier debugging */
+	pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
+			name, seqno);
+
+	/*
+	 * We use the base fence to refcount the PVR fence and to do the
+	 * necessary clean up once the refcount drops to 0.
+	 */
+	dma_fence_init(&pvr_fence->base, &pvr_fence_foreign_ops, &fctx->lock,
+		       fctx->fence_context, seqno);
+
+	/*
+	 * Take an extra reference on the base fence that gets dropped when the
+	 * foreign fence is signalled.
+	 */
+	dma_fence_get(&pvr_fence->base);
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	kref_get(&fctx->kref);
+
+	PVR_FENCE_TRACE(&pvr_fence->base,
+			"created fence from foreign fence %llu#%d (%s)\n",
+			(u64) pvr_fence->fence->context,
+			pvr_fence->fence->seqno, name);
+
+	err = dma_fence_add_callback(fence, &pvr_fence->cb,
+				     pvr_fence_foreign_signal_sync);
+	if (err) {
+		if (err != -ENOENT) {
+			pr_err("%s: failed to add fence callback (err=%d)",
+			       __func__, err);
+			goto err_put_ref;
+		}
+
+
+		/*
+		 * The fence has already signalled so set the sync as signalled.
+		 * The "signalled" hwperf packet should be emitted because the
+		 * callback won't be called for already signalled fence hence,
+		 * PVRSRV_FENCE_FLAG_NONE flag.
+		 */
+		pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
+		PVR_FENCE_TRACE(&pvr_fence->base,
+				"foreign fence %llu#%d already signaled (%s)\n",
+				(u64) pvr_fence->fence->context,
+				pvr_fence->fence->seqno,
+				name);
+		dma_fence_put(&pvr_fence->base);
+	}
+
+	trace_pvr_fence_foreign_create(pvr_fence);
+
+	return pvr_fence;
+
+err_put_ref:
+	kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	list_del(&pvr_fence->fence_head);
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	SyncCheckpointFree(pvr_fence->sync_checkpoint);
+err_free_pvr_fence:
+	kmem_cache_free(pvr_fence_cache, pvr_fence);
+err_module_put:
+	module_put(THIS_MODULE);
+err_exit:
+	return NULL;
+}
+
+/**
+ * pvr_fence_destroy - destroys a PVR fence
+ * @pvr_fence: PVR fence to destroy
+ *
+ * Destroys a PVR fence. Upon return, the PVR fence may still exist if something
+ * else still references the underlying fence, e.g. a reservation object, or if
+ * software signalling has been enabled and the fence hasn't yet been signalled.
+ */
+void
+pvr_fence_destroy(struct pvr_fence *pvr_fence)
+{
+	PVR_FENCE_TRACE(&pvr_fence->base, "destroyed fence (%s)\n",
+			pvr_fence->name);
+
+	dma_fence_put(&pvr_fence->base);
+}
+
+/**
+ * pvr_fence_sw_signal - signals a PVR fence sync
+ * @pvr_fence: PVR fence to signal
+ *
+ * Sets the PVR fence sync value to signalled.
+ *
+ * Returns -EINVAL if the PVR fence represents a foreign fence.
+ */
+int
+pvr_fence_sw_signal(struct pvr_fence *pvr_fence)
+{
+	if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
+		return -EINVAL;
+
+	pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
+
+	queue_work(pvr_fence->fctx->fence_wq,
+		   &pvr_fence->fctx->check_status_work);
+
+	PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync signalled (%s)\n",
+			pvr_fence->name);
+
+	return 0;
+}
+
+/**
+ * pvr_fence_sw_error - errors the sync checkpoint backing a PVR fence
+ * @pvr_fence: PVR fence to error
+ *
+ * Sets the PVR fence sync checkpoint value to errored.
+ *
+ * Returns -EINVAL if the PVR fence represents a foreign fence.
+ */
+int
+pvr_fence_sw_error(struct pvr_fence *pvr_fence)
+{
+	if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
+		return -EINVAL;
+
+	SyncCheckpointError(pvr_fence->sync_checkpoint, PVRSRV_FENCE_FLAG_NONE);
+	dma_fence_set_error(&pvr_fence->base, -EIO);
+	dma_fence_signal(&pvr_fence->base);
+	PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync errored (%s)\n",
+			pvr_fence->name);
+
+	return 0;
+}
+
+int
+pvr_fence_get_checkpoints(struct pvr_fence **pvr_fences, u32 nr_fences,
+			  struct SYNC_CHECKPOINT_TAG **fence_checkpoints)
+{
+	struct SYNC_CHECKPOINT_TAG **next_fence_checkpoint = fence_checkpoints;
+	struct pvr_fence **next_pvr_fence = pvr_fences;
+	int fence_checkpoint_idx;
+
+	if (nr_fences > 0) {
+
+		for (fence_checkpoint_idx = 0; fence_checkpoint_idx < nr_fences;
+		     fence_checkpoint_idx++) {
+			struct pvr_fence *next_fence = *next_pvr_fence++;
+			*next_fence_checkpoint++ = next_fence->sync_checkpoint;
+			/* Take reference on sync checkpoint (will be dropped
+			 * later by kick code)
+			 */
+			SyncCheckpointTakeRef(next_fence->sync_checkpoint);
+		}
+	}
+
+	return 0;
+}
+
+struct SYNC_CHECKPOINT_TAG *
+pvr_fence_get_checkpoint(struct pvr_fence *update_fence)
+{
+	return update_fence->sync_checkpoint;
+}
+
+/**
+ * pvr_fence_dump_info_on_stalled_ufos - displays debug
+ * information on a native fence associated with any of
+ * the ufos provided. This function will be called from
+ * pvr_sync_file.c if the driver determines any GPU work
+ * is stuck waiting for a sync checkpoint representing a
+ * foreign sync to be signalled.
+ * @fctx:    fence context
+ * @nr_ufos: number of ufos in vaddrs
+ * @vaddrs:  array of FW addresses of UFOs which the
+ *           driver is waiting on.
+ *
+ * Output debug information to kernel log on linux fences
+ * which would be responsible for signalling the sync
+ * checkpoints indicated by the ufo vaddresses.
+ *
+ * Returns the number of ufos in the array which were found
+ * to be associated with foreign syncs.
+ */
+u32 pvr_fence_dump_info_on_stalled_ufos(struct pvr_fence_context *fctx,
+					u32 nr_ufos, u32 *vaddrs)
+{
+	int our_ufo_ct = 0;
+	struct pvr_fence *pvr_fence;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fctx->list_lock, flags);
+	/* dump info on any ufos in our active list */
+	list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
+		u32 *this_ufo_vaddr = vaddrs;
+		int ufo_num;
+		DUMPDEBUG_PRINTF_FUNC *pfnDummy = NULL;
+
+		for (ufo_num = 0; ufo_num < nr_ufos; ufo_num++) {
+			struct SYNC_CHECKPOINT_TAG *checkpoint =
+				pvr_fence->sync_checkpoint;
+			const u32 fence_ufo_addr =
+				SyncCheckpointGetFirmwareAddr(checkpoint);
+
+			if (fence_ufo_addr != this_ufo_vaddr[ufo_num])
+				continue;
+
+			/* Dump sync info */
+			PVR_DUMPDEBUG_LOG(pfnDummy, NULL,
+					  "\tSyncID = %d, FWAddr = 0x%08x: TLID = %d (Foreign Fence - [%p] %s)",
+					  SyncCheckpointGetId(checkpoint),
+					  fence_ufo_addr,
+					  SyncCheckpointGetTimeline(checkpoint),
+					  pvr_fence->fence,
+					  pvr_fence->name);
+			our_ufo_ct++;
+		}
+	}
+	spin_unlock_irqrestore(&fctx->list_lock, flags);
+	return our_ufo_ct;
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_fence.h b/drivers/gpu/drm/img-rogue/pvr_fence.h
new file mode 100644
index 000000000000..a8cd0d35df7b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_fence.h
@@ -0,0 +1,247 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence interface
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_FENCE_H__)
+#define __PVR_FENCE_H__
+
+#include "services_kernel_client.h"
+#include "pvr_linux_fence.h"
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+
+struct SYNC_CHECKPOINT_CONTEXT_TAG;
+struct SYNC_CHECKPOINT_TAG;
+
+/**
+ * pvr_fence_context - PVR fence context used to create and manage PVR fences
+ * @lock: protects the context and fences created on the context
+ * @name: fence context name (used for debugging)
+ * @dbg_request_handle: handle for callback used to dump debug data
+ * @fence_context: fence context with which to associate fences
+ * @fence_seqno: sequence number to use for the next fence
+ * @fence_wq: work queue for signalled fence work
+ * @check_status_work: work item used to inform services when a foreign fence
+ * has signalled
+ * @cmd_complete_handle: handle for callback used to signal fences when fence
+ * syncs are met
+ * @list_lock: protects the active and active foreign lists
+ * @signal_list: list of fences waiting to be signalled
+ * @fence_list: list of fences (used for debugging)
+ * @deferred_free_list: list of fences that we will free when we are no longer
+ * holding spinlocks.  The frees get implemented when an update fence is
+ * signalled or the context is freed.
+ */
+struct pvr_fence_context {
+	spinlock_t lock;
+	char name[32];
+	void *dbg_request_handle;
+	u64 fence_context;
+	atomic_t fence_seqno;
+
+	struct workqueue_struct *fence_wq;
+	struct work_struct check_status_work;
+
+	void *cmd_complete_handle;
+
+	spinlock_t list_lock;
+	struct list_head signal_list;
+	struct list_head fence_list;
+	struct list_head deferred_free_list;
+
+	struct kref kref;
+	struct work_struct destroy_work;
+	void *dev_cookie;
+};
+
+/**
+ * pvr_fence - PVR fence that represents both native and foreign fences
+ * @base: fence structure
+ * @fctx: fence context on which this fence was created
+ * @name: fence name (used for debugging)
+ * @fence: pointer to base fence structure or foreign fence
+ * @sync_checkpoint: services sync checkpoint used by hardware
+ * @fence_head: entry on the context fence and deferred free list
+ * @signal_head: entry on the context signal list
+ * @cb: foreign fence callback to set the sync to signalled
+ */
+struct pvr_fence {
+	struct dma_fence base;
+	struct pvr_fence_context *fctx;
+	char name[32];
+
+	struct dma_fence *fence;
+	struct SYNC_CHECKPOINT_TAG *sync_checkpoint;
+
+	struct list_head fence_head;
+	struct list_head signal_head;
+	struct dma_fence_cb cb;
+	struct rcu_head rcu;
+};
+
+extern const struct dma_fence_ops pvr_fence_ops;
+extern const struct dma_fence_ops pvr_fence_foreign_ops;
+
+static inline bool is_our_fence(struct pvr_fence_context *fctx,
+				struct dma_fence *fence)
+{
+	return (fence->context == fctx->fence_context);
+}
+
+static inline bool is_pvr_fence(struct dma_fence *fence)
+{
+	return ((fence->ops == &pvr_fence_ops) ||
+		(fence->ops == &pvr_fence_foreign_ops));
+}
+
+static inline struct pvr_fence *to_pvr_fence(struct dma_fence *fence)
+{
+	if (is_pvr_fence(fence))
+		return container_of(fence, struct pvr_fence, base);
+
+	return NULL;
+}
+
+PVRSRV_ERROR pvr_fence_context_register_dbg(void *dbg_request_handle,
+				void *dev,
+				struct pvr_fence_context *fctx);
+struct pvr_fence_context *
+pvr_fence_foreign_context_create(struct workqueue_struct *fence_status_wq,
+		const char *name);
+struct pvr_fence_context *
+pvr_fence_context_create(void *dev_cookie,
+			 struct workqueue_struct *fence_status_wq,
+			 const char *name);
+void pvr_fence_context_destroy(struct pvr_fence_context *fctx);
+void pvr_context_value_str(struct pvr_fence_context *fctx, char *str, int size);
+
+struct pvr_fence *
+pvr_fence_create(struct pvr_fence_context *fctx,
+		 struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+		 int timeline_fd, const char *name);
+struct pvr_fence *
+pvr_fence_create_from_fence(struct pvr_fence_context *fctx,
+			    struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
+			    struct dma_fence *fence,
+			    PVRSRV_FENCE fence_fd,
+			    const char *name);
+void pvr_fence_destroy(struct pvr_fence *pvr_fence);
+int pvr_fence_sw_signal(struct pvr_fence *pvr_fence);
+int pvr_fence_sw_error(struct pvr_fence *pvr_fence);
+
+int pvr_fence_get_checkpoints(struct pvr_fence **pvr_fences, u32 nr_fences,
+			      struct SYNC_CHECKPOINT_TAG **fence_checkpoints);
+struct SYNC_CHECKPOINT_TAG *
+pvr_fence_get_checkpoint(struct pvr_fence *update_fence);
+
+void pvr_fence_context_signal_fences_nohw(void *data);
+
+void pvr_fence_context_free_deferred_callback(void *data);
+
+u32 pvr_fence_dump_info_on_stalled_ufos(struct pvr_fence_context *fctx,
+					u32 nr_ufos,
+					u32 *vaddrs);
+
+void pvr_fence_fence_value_str(struct dma_fence *fence, char *str, int size);
+
+void pvr_fence_foreign_fence_value_str(struct dma_fence *fence, char *str,
+				       int size);
+
+void pvr_fence_foreign_timeline_value_str(struct dma_fence *fence, char *str,
+					  int size);
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+static inline void pvr_fence_cleanup(void)
+{
+	/*
+	 * Ensure all PVR fence contexts have been destroyed, by flushing
+	 * the context destruction workqueue.
+	 */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 36))
+	flush_workqueue(NativeSyncGetFenceCtxDestroyWq());
+#else
+	flush_scheduled_work();
+#endif
+
+}
+#endif
+
+#if defined(PVR_FENCE_DEBUG)
+#define PVR_FENCE_CTX_TRACE(c, fmt, ...)                                   \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_err("c %llu: (PVR) " fmt, (u64) __fctx->fence_context,  \
+		       ## __VA_ARGS__);                                    \
+	} while (0)
+#else
+#define PVR_FENCE_CTX_TRACE(c, fmt, ...)
+#endif
+
+#define PVR_FENCE_CTX_WARN(c, fmt, ...)                                    \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_warn("c %llu: (PVR) " fmt, (u64) __fctx->fence_context, \
+			## __VA_ARGS__);                                   \
+	} while (0)
+
+#define PVR_FENCE_CTX_ERR(c, fmt, ...)                                     \
+	do {                                                               \
+		struct pvr_fence_context *__fctx = (c);                    \
+		pr_err("c %llu: (PVR) " fmt, (u64) __fctx->fence_context,  \
+		       ## __VA_ARGS__);                                    \
+	} while (0)
+
+#if defined(PVR_FENCE_DEBUG)
+#define PVR_FENCE_TRACE(f, fmt, ...)                                       \
+	DMA_FENCE_ERR(f, "(PVR) " fmt, ## __VA_ARGS__)
+#else
+#define PVR_FENCE_TRACE(f, fmt, ...)
+#endif
+
+#define PVR_FENCE_WARN(f, fmt, ...)                                        \
+	DMA_FENCE_WARN(f, "(PVR) " fmt, ## __VA_ARGS__)
+
+#define PVR_FENCE_ERR(f, fmt, ...)                                         \
+	DMA_FENCE_ERR(f, "(PVR) " fmt, ## __VA_ARGS__)
+
+#endif /* !defined(__PVR_FENCE_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_fence_trace.h b/drivers/gpu/drm/img-rogue/pvr_fence_trace.h
new file mode 100644
index 000000000000..9ec1cabfd87e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_fence_trace.h
@@ -0,0 +1,222 @@
+/*
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM pvr_fence
+
+#if !defined(_TRACE_PVR_FENCE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_PVR_FENCE_H
+
+#include <linux/tracepoint.h>
+
+struct pvr_fence;
+struct pvr_fence_context;
+
+DECLARE_EVENT_CLASS(pvr_fence_context,
+
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx),
+
+	TP_STRUCT__entry(
+		__string(name, fctx->name)
+		__array(char, val, 128)
+	),
+
+	TP_fast_assign(
+		__assign_str(name);
+		pvr_context_value_str(fctx, __entry->val,
+			sizeof(__entry->val));
+	),
+
+	TP_printk("name=%s val=%s",
+		  __get_str(name),
+		  __entry->val
+	)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_create,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_destroy,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_destroy_kref,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DEFINE_EVENT(pvr_fence_context, pvr_fence_context_signal_fences,
+	TP_PROTO(struct pvr_fence_context *fctx),
+	TP_ARGS(fctx)
+);
+
+DECLARE_EVENT_CLASS(pvr_fence,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence),
+
+	TP_STRUCT__entry(
+		__string(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__string(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		__array(char, val, 128)
+		__field(u64, context)
+	),
+
+	TP_fast_assign(
+		__assign_str(driver);
+		__assign_str(timeline);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		snprintf(__entry->val, sizeof(__entry->val), "%llu", fence->base.seqno);
+#else
+		fence->base.ops->fence_value_str(&fence->base,
+			__entry->val, sizeof(__entry->val));
+#endif
+		__entry->context = fence->base.context;
+	),
+
+	TP_printk("driver=%s timeline=%s ctx=%llu val=%s",
+		  __get_str(driver), __get_str(timeline),
+		  __entry->context, __entry->val
+	)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_create,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_release,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_enable_signaling,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence, pvr_fence_signal_fence,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DECLARE_EVENT_CLASS(pvr_fence_foreign,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence),
+
+	TP_STRUCT__entry(
+		__string(driver,
+			fence->base.ops->get_driver_name(&fence->base))
+		__string(timeline,
+			fence->base.ops->get_timeline_name(&fence->base))
+		__array(char, val, 128)
+		__field(u64, context)
+		__string(foreign_driver,
+			fence->fence->ops->get_driver_name ?
+			fence->fence->ops->get_driver_name(fence->fence) :
+			"unknown")
+		__string(foreign_timeline,
+			fence->fence->ops->get_timeline_name ?
+			fence->fence->ops->get_timeline_name(fence->fence) :
+			"unknown")
+		__array(char, foreign_val, 128)
+		__field(u64, foreign_context)
+	),
+
+	TP_fast_assign(
+		__assign_str(driver);
+		__assign_str(timeline);
+		__entry->context = fence->base.context;
+		__assign_str(foreign_driver);
+		__assign_str(foreign_timeline);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		(void)OSStringSafeCopy(__entry->foreign_val,
+			"unknown", sizeof(__entry->foreign_val));
+#else
+		fence->fence->ops->fence_value_str ?
+			fence->fence->ops->fence_value_str(
+				fence->fence, __entry->foreign_val,
+				sizeof(__entry->foreign_val)) :
+			(void) OSStringSafeCopy(__entry->foreign_val,
+				"unknown", sizeof(__entry->foreign_val));
+#endif
+		__entry->foreign_context = fence->fence->context;
+	),
+
+	TP_printk("driver=%s timeline=%s ctx=%llu val=%s foreign: driver=%s timeline=%s ctx=%llu val=%s",
+		  __get_str(driver), __get_str(timeline), __entry->context,
+		  __entry->val, __get_str(foreign_driver),
+		  __get_str(foreign_timeline), __entry->foreign_context,
+		  __entry->foreign_val
+	)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_create,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_release,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+DEFINE_EVENT(pvr_fence_foreign, pvr_fence_foreign_signal,
+	TP_PROTO(struct pvr_fence *fence),
+	TP_ARGS(fence)
+);
+
+#endif /* _TRACE_PVR_FENCE_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+
+/* This is needed because the name of this file doesn't match TRACE_SYSTEM. */
+#define TRACE_INCLUDE_FILE pvr_fence_trace
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/img-rogue/pvr_gputrace.c b/drivers/gpu/drm/img-rogue/pvr_gputrace.c
new file mode 100644
index 000000000000..723e6d4d9082
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_gputrace.c
@@ -0,0 +1,1668 @@
+/*************************************************************************/ /*!
+@File           pvr_gputrace.c
+@Title          PVR GPU Trace module Linux implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/fs.h>
+#include <linux/trace_events.h>
+
+#include "pvrsrv_error.h"
+#include "pvrsrv_apphint.h"
+#include "pvr_debug.h"
+#include "ospvr_gputrace.h"
+#include "rgxhwperf.h"
+#include "rgxtimecorr.h"
+#include "device.h"
+#include "trace_events.h"
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "tlclient.h"
+#include "tlstream.h"
+#include "pvr_debug.h"
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#define CREATE_TRACE_POINTS
+#include "rogue_trace_events.h"
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#define CREATE_TRACE_POINTS
+#include "gpu_work.h"
+#define TRACE_FS_CLK "/sys/kernel/tracing/trace_clock"
+#else
+#define TRACE_FS_CLK "/sys/kernel/debug/tracing/trace_clock"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#if defined(SUPPORT_RGX)
+#if defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+#define TRACE_CLK     RGXTIMECORR_CLOCK_SCHED
+#define TRACE_CLK_STR "local\n"
+#else
+#define TRACE_CLK     RGXTIMECORR_CLOCK_MONO
+#define TRACE_CLK_STR "mono\n"
+#endif
+#endif
+
+/******************************************************************************
+ Module internal implementation
+******************************************************************************/
+
+typedef enum {
+	PVR_GPUTRACE_SWITCH_TYPE_UNDEF = 0,
+
+	PVR_GPUTRACE_SWITCH_TYPE_BEGIN = 1,
+	PVR_GPUTRACE_SWITCH_TYPE_END = 2,
+	PVR_GPUTRACE_SWITCH_TYPE_SINGLE = 3
+} PVR_GPUTRACE_SWITCH_TYPE;
+
+typedef struct RGX_HWPERF_FTRACE_DATA {
+	/* This lock ensures the HWPerf TL stream reading resources are not destroyed
+	 * by one thread disabling it while another is reading from it. Keeps the
+	 * state and resource create/destroy atomic and consistent. */
+	POS_LOCK    hFTraceResourceLock;
+
+	IMG_HANDLE  hGPUTraceCmdCompleteHandle;
+	IMG_HANDLE  hGPUFTraceTLStream;
+	IMG_UINT64  ui64LastSampledTimeCorrOSTimeStamp;
+	IMG_UINT32  ui32FTraceLastOrdinal;
+	IMG_BOOL    bTrackOrdinals;
+} RGX_HWPERF_FTRACE_DATA;
+
+/* This lock ensures state change of GPU_TRACING on/off is done atomically */
+static POS_LOCK ghGPUTraceStateLock;
+static IMG_BOOL gbFTraceGPUEventsEnabled = PVRSRV_APPHINT_ENABLEFTRACEGPU;
+
+#if !defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+/* This is the FTrace clock source prior to driver initialisation */
+static IMG_CHAR gszLastClockSource[32] = {0};
+#endif
+
+/* This lock ensures that the reference counting operation on the FTrace UFO
+ * events and enable/disable operation on firmware event are performed as
+ * one atomic operation. This should ensure that there are no race conditions
+ * between reference counting and firmware event state change.
+ * See below comment for guiUfoEventRef.
+ */
+static POS_LOCK ghLockFTraceEventLock;
+
+/* Multiple FTrace UFO events are reflected in the firmware as only one event. When
+ * we enable FTrace UFO event we want to also at the same time enable it in
+ * the firmware. Since there is a multiple-to-one relation between those events
+ * we count how many FTrace UFO events is enabled. If at least one event is
+ * enabled we enabled the firmware event. When all FTrace UFO events are disabled
+ * we disable firmware event. */
+static IMG_UINT guiUfoEventRef;
+
+/******************************************************************************
+ Module In-bound API
+******************************************************************************/
+
+static PVRSRV_ERROR _GpuTraceDisable(
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+	IMG_BOOL bDeInit);
+
+static void _GpuTraceCmdCompleteNotify(PVRSRV_CMDCOMP_HANDLE);
+static void _GpuTraceProcessPackets(PVRSRV_RGXDEV_INFO *psDevInfo, void *pBuffer,
+                                    IMG_UINT32 ui32ReadLen);
+
+static void _FTrace_FWOnReaderOpenCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_FTRACE] = IMG_FALSE;
+}
+
+static void _FTrace_FWOnReaderCloseCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_FTRACE] = IMG_TRUE;
+}
+
+/* Currently supported by default */
+#if defined(SUPPORT_TL_PRODUCER_CALLBACK)
+static PVRSRV_ERROR GPUTraceTLCB(IMG_HANDLE hStream,
+                                  IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser)
+{
+	/* IN DEV: Not required as the streams goal is to be a copy of HWPerf */
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(hStream);
+	PVR_UNREFERENCED_PARAMETER(ui32ReqOp);
+	PVR_UNREFERENCED_PARAMETER(ui32Resp);
+	PVR_UNREFERENCED_PARAMETER(pvUser);
+
+	return eError;
+}
+#endif /* defined(SUPPORT_TL_PRODUCER_CALLBACK) */
+
+#if defined(SUPPORT_RGX)
+#if !defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+/* Configure the FTrace clock source to use the DDK apphint clock source */
+static void PVRGpuTraceInitFTraceClockSource(void)
+{
+	int ret, i, j;
+	bool bFound = false;
+	loff_t pos = 0;
+	char str[64];
+
+	/*  Just force the value to be what the DDK says it is
+	    Note for filp_open, the mode is only used for O_CREAT
+		Hence its value doesn't matter in this context
+	 */
+	struct file *filep = filp_open(TRACE_FS_CLK, O_RDWR, 0);
+	PVR_LOG_RETURN_VOID_IF_FALSE(!IS_ERR(filep), "TraceFS not found");
+
+	PVR_LOG_VA(PVR_DBG_MESSAGE,
+	        "Writing %s to %s to enable parallel HWPerf and FTrace support",
+	        TRACE_CLK_STR, TRACE_FS_CLK);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_read(filep, str, sizeof(str)-1, &pos);
+#else
+	ret = kernel_read(filep, pos, str, sizeof(str)-1);
+#endif
+	PVR_LOG_RETURN_VOID_IF_FALSE((ret > 0), "TraceFS Read failed");
+	str[ret] = 0;
+	pos = 0;
+
+	/* Determine clock value. trace_clock value is stored like [<clock_string>]
+	   File example: [local] global counter mono mono_raw x86-tsc
+	*/
+	for (i = 0, j = 0; i < sizeof(str) && j < sizeof(gszLastClockSource); i++)
+	{
+		if (str[i] == ']')
+		{
+			break;
+		}
+		else if (str[i] == '[')
+		{
+			bFound = true;
+		}
+		else if (bFound)
+		{
+			gszLastClockSource[j] = str[i];
+			j++;
+		}
+	}
+	PVR_LOG_VA(PVR_DBG_MESSAGE, "Got %s from FTraceFS", gszLastClockSource);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_write(filep, TRACE_CLK_STR, sizeof(TRACE_CLK_STR), &pos);
+#else
+	ret = kernel_write(filep, TRACE_CLK_STR, sizeof(TRACE_CLK_STR), pos);
+#endif
+	PVR_LOG_IF_FALSE((ret > 0), "Setting FTrace clock source failed");
+
+	filp_close(filep, NULL);
+}
+
+/* Reset the FTrace clock source to the original clock source */
+static void PVRGpuTraceDeinitFTraceClockSource(void)
+{
+	int ret;
+	loff_t pos = 0;
+
+	/* Return the original value of the file */
+	struct file *filep = filp_open(TRACE_FS_CLK, O_WRONLY, 0);
+	PVR_LOG_RETURN_VOID_IF_FALSE(!IS_ERR(filep), "TraceFS not found");
+
+	/* FTraceFS write will ignore any writes to it that don't match a clock source */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	ret = kernel_write(filep, gszLastClockSource, sizeof(gszLastClockSource), &pos);
+#else
+	ret = kernel_write(filep, gszLastClockSource, sizeof(gszLastClockSource), pos);
+#endif
+	PVR_LOG_IF_FALSE((ret >= 0), "Setting FTrace clock source failed");
+
+	filp_close(filep, NULL);
+}
+#endif /* !defined(SUPPORT_ANDROID_PLATFORM) */
+#endif /* defined(SUPPORT_RGX) */
+
+PVRSRV_ERROR PVRGpuTraceSupportInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	if (ghLockFTraceEventLock != NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FTrace Support is already initialized"));
+		return PVRSRV_OK;
+	}
+
+	/* common module params initialization */
+	eError = OSLockCreate(&ghLockFTraceEventLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	eError = OSLockCreate(&ghGPUTraceStateLock);
+	PVR_LOG_RETURN_IF_ERROR (eError, "OSLockCreate");
+
+#if defined(SUPPORT_RGX)
+#if !defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+	PVRGpuTraceInitFTraceClockSource();
+#endif /* !defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK) */
+#endif /* defined(SUPPORT_RGX) */
+
+	return PVRSRV_OK;
+}
+
+void PVRGpuTraceSupportDeInit(void)
+{
+#if defined(SUPPORT_RGX)
+#if !defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+	PVRGpuTraceDeinitFTraceClockSource();
+#endif /* !defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK) */
+#endif /* defined(SUPPORT_RGX) */
+
+	if (ghGPUTraceStateLock)
+	{
+		OSLockDestroy(ghGPUTraceStateLock);
+	}
+
+	if (ghLockFTraceEventLock)
+	{
+		OSLockDestroy(ghLockFTraceEventLock);
+		ghLockFTraceEventLock = NULL;
+	}
+}
+
+/* Called from RGXHWPerfInitOnDemandL2Stream() which is alway called from
+ * a critical section protected by hHWPerfLock. */
+PVRSRV_ERROR PVRGpuTraceInitStream(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszFTraceStreamName[sizeof(PVRSRV_TL_FTRACE_RGX_FW_STREAM) + 4];
+	/* + 4 is used to allow names up to "ftrace_999", which is enough */
+
+	IMG_HANDLE hStream = NULL;
+
+	/* form the FTrace stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszFTraceStreamName, sizeof(pszFTraceStreamName), "%s%d",
+				   PVRSRV_TL_FTRACE_RGX_FW_STREAM,
+				   psDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to form FTrace stream name for device %d",
+				__func__,
+				psDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = TLStreamCreate(&hStream,
+							pszFTraceStreamName,
+							psDevInfo->ui32RGXL2HWPerfBufSize,
+							TL_OPMODE_DROP_NEWER | TL_FLAG_NO_SIGNAL_ON_COMMIT,
+							_FTrace_FWOnReaderOpenCB, psDevInfo,
+							_FTrace_FWOnReaderCloseCB, psDevInfo,
+#if !defined(SUPPORT_TL_PRODUCER_CALLBACK)
+							NULL, NULL
+#else
+							/* Not enabled by default */
+							GPUTraceTLCB, psDevInfo
+#endif
+							);
+	PVR_RETURN_IF_ERROR(eError);
+
+	psDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_FTRACE] = hStream;
+	psDevInfo->uiHWPerfStreamCount++;
+	PVR_ASSERT(psDevInfo->uiHWPerfStreamCount <= RGX_HWPERF_L2_STREAM_LAST);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRGpuTraceInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	RGX_HWPERF_FTRACE_DATA *psData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psData = OSAllocZMem(sizeof(RGX_HWPERF_FTRACE_DATA));
+	psDevInfo->pvGpuFtraceData = psData;
+	PVR_LOG_GOTO_IF_NOMEM(psData, eError, e0);
+
+	/* We initialise it only once because we want to track if any
+	 * packets were dropped. */
+	psData->ui32FTraceLastOrdinal = IMG_UINT32_MAX - 1;
+
+	eError = OSLockCreate(&psData->hFTraceResourceLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	return PVRSRV_OK;
+
+e0:
+	PVRGpuTraceDeInitDevice(psDeviceNode);
+	return eError;
+}
+
+void PVRGpuTraceDeInitDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_HWPERF_FTRACE_DATA *psData = psDevInfo->pvGpuFtraceData;
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+	if (psData)
+	{
+		/* first disable the tracing, to free up TL resources */
+		if (psData->hFTraceResourceLock)
+		{
+			OSLockAcquire(psData->hFTraceResourceLock);
+			_GpuTraceDisable(psDeviceNode->pvDevice, IMG_TRUE);
+			OSLockRelease(psData->hFTraceResourceLock);
+
+			/* now free all the FTrace resources */
+			OSLockDestroy(psData->hFTraceResourceLock);
+		}
+		OSFreeMem(psData);
+		psDevInfo->pvGpuFtraceData = NULL;
+	}
+}
+
+static INLINE IMG_BOOL PVRGpuTraceIsEnabled(void)
+{
+	return gbFTraceGPUEventsEnabled;
+}
+
+void PVRGpuTraceInitIfEnabled(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (PVRGpuTraceIsEnabled())
+	{
+		IMG_BOOL bEnable = IMG_FALSE;
+
+		PVRSRV_ERROR eError = PVRGpuTraceSetEnabled(psDeviceNode, IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to initialise GPU event tracing"
+					" (%s)", PVRSRVGetErrorString(eError)));
+		}
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		/* below functions will enable FTrace events which in turn will
+		 * execute HWPerf callbacks that set appropriate filter values
+		 * note: unfortunately the functions don't allow to pass private
+		 *       data so they enable events for all of the devices
+		 *       at once, which means that this can happen more than once
+		 *       if there is more than one device */
+
+		/* single events can be enabled by calling trace_set_clr_event()
+		 * with the event name, e.g.:
+		 * trace_set_clr_event("rogue", "rogue_ufo_update", 1) */
+		if (trace_set_clr_event("rogue", NULL, 1) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to enable \"rogue\" event"
+			        " group"));
+		}
+		else
+		{
+			PVR_LOG(("FTrace events from \"rogue\" group enabled"));
+			bEnable = IMG_TRUE;
+		}
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+		if (trace_set_clr_event("power", "gpu_work_period", 1) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to enable \"gpu_work_period\" event"));
+		}
+		else
+		{
+			PVR_LOG(("FTrace event from \"gpu_work_period\" enabled"));
+			bEnable = IMG_TRUE;
+		}
+#endif
+
+		if (bEnable)
+		{
+			/* this enables FTrace globally (if not enabled nothing will appear
+			 * in the FTrace buffer) */
+			tracing_on();
+		}
+	}
+}
+
+/* Caller must now hold hFTraceResourceLock before calling this method.
+ */
+static PVRSRV_ERROR _GpuTraceEnable(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+	PVRSRV_DEVICE_NODE *psRgxDevNode = psRgxDevInfo->psDeviceNode;
+	IMG_CHAR pszFTraceStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 4];
+	IMG_UINT64 uiFilter;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	PVR_ASSERT(OSLockIsLocked(psFtraceData->hFTraceResourceLock));
+
+	/* return if already enabled */
+	if (psFtraceData->hGPUFTraceTLStream != NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	uiFilter =
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	    RGX_HWPERF_EVENT_MASK_HW_KICKFINISH | RGX_HWPERF_EVENT_MASK_FW_SED | RGX_HWPERF_EVENT_MASK_FW_UFO |
+#endif
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD) |
+#endif
+	    0;
+
+#if defined(SUPPORT_RGX) && !defined(NO_HARDWARE)
+	/* Signal FW to enable event generation */
+	if (psRgxDevInfo->bFirmwareInitialised)
+	{
+		eError = PVRSRVRGXCtrlHWPerfFW(psRgxDevNode,
+		                               RGX_HWPERF_L2_STREAM_FTRACE,
+		                               uiFilter, HWPERF_FILTER_OPERATION_SET);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfFW", err_out);
+	}
+	else
+#endif
+	{
+		/* only set filter and exit */
+		(void) RGXHWPerfFwSetEventFilter(psRgxDevInfo, RGX_HWPERF_L2_STREAM_FTRACE, uiFilter);
+
+		return PVRSRV_OK;
+	}
+
+	/* form the FTrace stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszFTraceStreamName, sizeof(pszFTraceStreamName), "%s%d",
+					PVRSRV_TL_FTRACE_RGX_FW_STREAM, psRgxDevNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to form FTrace stream name for device %d",
+		         __func__,
+		         psRgxDevNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* In case there was any data copied from the L1 buffer to the FTrace L2
+	 * buffer in the narrow window between disabling the FTrace and MISR
+	 * running, drop this data to make the ordinal tracking correct. */
+	TLStreamReset(psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_FTRACE]);
+
+	/* Open the TL Stream for FTrace data consumption */
+	eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+								pszFTraceStreamName,
+								PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING,
+								&psFtraceData->hGPUFTraceTLStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "TLClientOpenStream", err_out);
+
+#if defined(SUPPORT_RGX)
+	if (RGXTimeCorrGetClockSource(psRgxDevNode) != TRACE_CLK)
+	{
+		/* Set clock source for timer correlation data to hwperf clock */
+		eError = RGXTimeCorrSetClockSource(psRgxDevNode, TRACE_CLK);
+		PVR_LOG_IF_ERROR(eError, "RGXTimeCorrSetClockSource");
+	}
+#endif
+
+	/* Reset the ordinal tracking flag. We should skip checking for gaps in
+	 * ordinal on the first run after FTrace is enabled in case another HWPerf
+	 * consumer was connected while FTrace was disabled. */
+	psFtraceData->bTrackOrdinals = IMG_FALSE;
+
+	/* Reset the OS timestamp coming from the timer correlation data
+	 * associated with the latest HWPerf event we processed.
+	 */
+	psFtraceData->ui64LastSampledTimeCorrOSTimeStamp = 0;
+
+	/* Register a notifier to collect HWPerf data whenever the HW completes
+	 * an operation.
+	 */
+	eError = PVRSRVRegisterCmdCompleteNotify(
+		&psFtraceData->hGPUTraceCmdCompleteHandle,
+		&_GpuTraceCmdCompleteNotify,
+		psRgxDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterCmdCompleteNotify", err_close_stream);
+
+err_out:
+	PVR_DPF_RETURN_RC(eError);
+err_close_stream:
+	TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+						psFtraceData->hGPUFTraceTLStream);
+	psFtraceData->hGPUFTraceTLStream = NULL;
+	goto err_out;
+}
+
+/* Caller must now hold hFTraceResourceLock before calling this method.
+ */
+static PVRSRV_ERROR _GpuTraceDisable(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_BOOL bDeInit)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_NODE *psRgxDevNode = psRgxDevInfo->psDeviceNode;
+#endif
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	PVR_ASSERT(OSLockIsLocked(psFtraceData->hFTraceResourceLock));
+
+	/* if FW is not yet initialised, just set filter and exit */
+	if (!psRgxDevInfo->bFirmwareInitialised)
+	{
+		(void) RGXHWPerfFwSetEventFilter(psRgxDevInfo, RGX_HWPERF_L2_STREAM_FTRACE,
+		                                 RGX_HWPERF_EVENT_MASK_NONE);
+		return PVRSRV_OK;
+	}
+
+	if (psFtraceData->hGPUFTraceTLStream == NULL)
+	{
+		/* Tracing already disabled, just return */
+		return PVRSRV_OK;
+	}
+
+#if defined(SUPPORT_RGX)
+	if (!bDeInit)
+	{
+		eError = PVRSRVRGXCtrlHWPerfFW(psRgxDevNode,
+		                               RGX_HWPERF_L2_STREAM_FTRACE,
+		                               RGX_HWPERF_EVENT_MASK_NONE,
+		                               HWPERF_FILTER_OPERATION_SET);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfFW");
+	}
+#endif
+
+	if (psFtraceData->hGPUTraceCmdCompleteHandle)
+	{
+		/* Tracing is being turned off. Unregister the notifier. */
+		eError = PVRSRVUnregisterCmdCompleteNotify(
+				psFtraceData->hGPUTraceCmdCompleteHandle);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVUnregisterCmdCompleteNotify");
+		psFtraceData->hGPUTraceCmdCompleteHandle = NULL;
+	}
+
+	/* Let close stream perform the release data on the outstanding acquired
+	 * data */
+	eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+	                             psFtraceData->hGPUFTraceTLStream);
+	PVR_LOG_IF_ERROR(eError, "TLClientCloseStream");
+
+	psFtraceData->hGPUFTraceTLStream = NULL;
+
+#if defined(SUPPORT_RGX)
+	if (psRgxDevInfo->ui32LastClockSource != TRACE_CLK)
+	{
+		RGXTimeCorrSetClockSource(psRgxDevNode, psRgxDevInfo->ui32LastClockSource);
+	}
+#endif
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                        IMG_BOOL bNewValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+	psFtraceData = psRgxDevInfo->pvGpuFtraceData;
+
+	/* About to create/destroy FTrace resources, lock critical section
+	 * to avoid HWPerf MISR thread contention.
+	 */
+	OSLockAcquire(psFtraceData->hFTraceResourceLock);
+
+	eError = (bNewValue ? _GpuTraceEnable(psRgxDevInfo)
+					   : _GpuTraceDisable(psRgxDevInfo, IMG_FALSE));
+
+	OSLockRelease(psFtraceData->hFTraceResourceLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabledForAllDevices(IMG_BOOL bNewValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	/* enable/disable GPU trace on all devices */
+	while (psDeviceNode)
+	{
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			eError = _GpuTraceSetEnabled(psRgxDevInfo, bNewValue);
+			if (eError != PVRSRV_OK)
+				break;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR PVRGpuTraceSetEnabled(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   IMG_BOOL bNewValue)
+{
+	return _GpuTraceSetEnabled(psDeviceNode->pvDevice, bNewValue);
+}
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+/* ----- HWPerf to FTrace packet processing and events injection ------------ */
+
+static const IMG_CHAR *_HWPerfKickTypeToStr(RGX_HWPERF_KICK_TYPE eKickType)
+{
+	static const IMG_CHAR *aszKickType[RGX_HWPERF_KICK_TYPE2_LAST+1] = {
+		"TA3D", /* Deprecated */
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+		/* Volcanic deprecated kick types */
+		"CDM", "RS", "SHG", "TQTDM", "SYNC", "TA", "3D", "LAST",
+
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+#else
+		/* Rogue deprecated kick types */
+		"TQ2D", "TQ3D", "CDM", "RS", "VRDM", "TQTDM", "SYNC", "TA", "3D", "LAST",
+
+		"<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>", "<UNKNOWN>",
+		"<UNKNOWN>",
+#endif
+		"TQ2D", "TQ3D", "TQTDM", "CDM", "GEOM", "3D", "SYNC", "RS", "LAST"
+	};
+
+	/* cast in case of negative value */
+	if (((IMG_UINT32) eKickType) >= RGX_HWPERF_KICK_TYPE2_LAST)
+	{
+		return "<UNKNOWN>";
+	}
+
+	return aszKickType[eKickType];
+}
+void PVRGpuTraceEnqueueEvent(
+		PVRSRV_DEVICE_NODE *psDevNode,
+		IMG_UINT32 ui32FirmwareCtx,
+		IMG_UINT32 ui32ExtJobRef,
+		IMG_UINT32 ui32IntJobRef,
+		RGX_HWPERF_KICK_TYPE eKickType)
+{
+	const IMG_CHAR *pszKickType = _HWPerfKickTypeToStr(eKickType);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "PVRGpuTraceEnqueueEvent(%s): contextId %u, "
+	        "jobId %u", pszKickType, ui32FirmwareCtx, ui32IntJobRef));
+
+	if (PVRGpuTraceIsEnabled())
+	{
+		trace_rogue_job_enqueue(psDevNode->sDevId.ui32InternalID, ui32FirmwareCtx,
+		                        ui32IntJobRef, ui32ExtJobRef, pszKickType);
+	}
+}
+
+static void _GpuTraceWorkSwitch(
+		IMG_UINT64 ui64HWTimestampInOSTime,
+		IMG_UINT32 ui32GpuId,
+		IMG_UINT32 ui32CtxId,
+		IMG_UINT32 ui32CtxPriority,
+		IMG_UINT32 ui32ExtJobRef,
+		IMG_UINT32 ui32IntJobRef,
+		const IMG_CHAR* pszWorkType,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	PVR_ASSERT(pszWorkType);
+	trace_rogue_sched_switch(pszWorkType, eSwType, ui64HWTimestampInOSTime,
+	                         ui32GpuId, ui32CtxId, 2-ui32CtxPriority, ui32IntJobRef,
+	                         ui32ExtJobRef);
+}
+
+static void _GpuTraceUfo(
+		IMG_UINT64 ui64OSTimestamp,
+		const RGX_HWPERF_UFO_EV eEvType,
+		const IMG_UINT32 ui32GpuId,
+		const IMG_UINT32 ui32CtxId,
+		const IMG_UINT32 ui32ExtJobRef,
+		const IMG_UINT32 ui32IntJobRef,
+		const IMG_UINT32 ui32UFOCount,
+		const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	switch (eEvType) {
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			trace_rogue_ufo_updates(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, ui32UFOCount, puData);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+			trace_rogue_ufo_checks_success(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_FALSE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			trace_rogue_ufo_checks_success(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_TRUE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+			trace_rogue_ufo_checks_fail(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_FALSE, ui32UFOCount,
+					puData);
+			break;
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			trace_rogue_ufo_checks_fail(ui64OSTimestamp, ui32GpuId, ui32CtxId,
+					ui32ExtJobRef, ui32IntJobRef, IMG_TRUE, ui32UFOCount,
+					puData);
+			break;
+		default:
+			break;
+	}
+}
+
+static void _GpuTraceFirmware(
+		IMG_UINT64 ui64HWTimestampInOSTime,
+		IMG_UINT32 ui32GpuId,
+		const IMG_CHAR* pszWorkType,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	trace_rogue_firmware_activity(ui64HWTimestampInOSTime, ui32GpuId, pszWorkType, eSwType);
+}
+
+static void _GpuTraceEventsLost(
+		const RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT32 ui32GpuId,
+		const IMG_UINT32 ui32LastOrdinal,
+		const IMG_UINT32 ui32CurrOrdinal)
+{
+	trace_rogue_events_lost(eStreamId, ui32GpuId, ui32LastOrdinal, ui32CurrOrdinal);
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+/* Calculate the OS timestamp given an RGX timestamp in the HWPerf event. */
+static uint64_t CalculateEventTimestamp(
+	PVRSRV_RGXDEV_INFO *psDevInfo,
+	uint32_t ui32TimeCorrIndex,
+	uint64_t ui64EventTimestamp)
+{
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData = psDevInfo->pvGpuFtraceData;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	uint64_t ui64CRTimeStamp;
+	uint64_t ui64OSTimeStamp;
+	uint64_t ui64CRDeltaToOSDeltaKNs;
+	uint64_t ui64EventOSTimestamp, deltaRgxTimer, delta_ns;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32TimeCorrIndex], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32TimeCorrIndex];
+	ui64CRTimeStamp = psTimeCorr->ui64CRTimeStamp;
+	ui64OSTimeStamp = psTimeCorr->ui64OSTimeStamp;
+	ui64CRDeltaToOSDeltaKNs = psTimeCorr->ui64CRDeltaToOSDeltaKNs;
+
+	if (psFtraceData->ui64LastSampledTimeCorrOSTimeStamp > ui64OSTimeStamp)
+	{
+		/* The previous packet had a time reference (time correlation data) more
+		 * recent than the one in the current packet, it means the timer
+		 * correlation array wrapped too quickly (buffer too small) and in the
+		 * previous call to _GpuTraceUfoEvent we read one of the
+		 * newest timer correlations rather than one of the oldest ones.
+		 */
+		PVR_DPF((PVR_DBG_ERROR, "%s: The timestamps computed so far could be "
+				 "wrong! The time correlation array size should be increased "
+				 "to avoid this.", __func__));
+	}
+
+	psFtraceData->ui64LastSampledTimeCorrOSTimeStamp = ui64OSTimeStamp;
+
+	/* RGX CR timer ticks delta */
+	deltaRgxTimer = ui64EventTimestamp - ui64CRTimeStamp;
+	/* RGX time delta in nanoseconds */
+	delta_ns = RGXFWIF_GET_DELTA_OSTIME_NS(deltaRgxTimer, ui64CRDeltaToOSDeltaKNs);
+	/* Calculate OS time of HWPerf event */
+	ui64EventOSTimestamp = ui64OSTimeStamp + delta_ns;
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: psCurrentDvfs RGX %llu, OS %llu, DVFSCLK %u",
+			 __func__, ui64CRTimeStamp, ui64OSTimeStamp,
+			 psTimeCorr->ui32CoreClockSpeed));
+
+	return ui64EventOSTimestamp;
+}
+
+static void _GpuTraceSwitchEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt, const IMG_CHAR* pszWorkName,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+{
+	IMG_UINT64 ui64Timestamp;
+	RGX_HWPERF_HW_DATA* psHWPerfPktData;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psHWPerfPkt);
+	PVR_ASSERT(pszWorkName);
+
+	psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceSwitchEvent: %s ui32ExtJobRef=%d, ui32IntJobRef=%d, eSwType=%d",
+			 pszWorkName != NULL ? pszWorkName : "?", psHWPerfPktData->ui32DMContext,
+			 psHWPerfPktData->ui32IntJobRef, eSwType));
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	_GpuTraceWorkSwitch(ui64Timestamp,
+	                    psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+	                    psHWPerfPktData->ui32DMContext,
+	                    psHWPerfPktData->ui32CtxPriority,
+	                    psHWPerfPktData->ui32ExtJobRef,
+	                    psHWPerfPktData->ui32IntJobRef,
+	                    pszWorkName,
+	                    eSwType);
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+	PVR_DPF_RETURN;
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+static void _GpuTraceUfoEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt)
+{
+	IMG_UINT64 ui64Timestamp;
+	RGX_HWPERF_UFO_DATA *psHWPerfPktData;
+	IMG_UINT32 ui32UFOCount;
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData;
+
+	psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui32UFOCount = RGX_HWPERF_GET_UFO_STREAMSIZE(psHWPerfPktData->ui32StreamInfo);
+	puData = (RGX_HWPERF_UFO_DATA_ELEMENT *) IMG_OFFSET_ADDR(psHWPerfPktData, RGX_HWPERF_GET_UFO_STREAMOFFSET(psHWPerfPktData->ui32StreamInfo));
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceUfoEvent: ui32ExtJobRef=%d, "
+	        "ui32IntJobRef=%d", psHWPerfPktData->ui32ExtJobRef,
+	        psHWPerfPktData->ui32IntJobRef));
+
+	_GpuTraceUfo(ui64Timestamp, psHWPerfPktData->eEvType,
+	             psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+	             psHWPerfPktData->ui32DMContext, psHWPerfPktData->ui32ExtJobRef,
+	             psHWPerfPktData->ui32IntJobRef, ui32UFOCount, puData);
+}
+
+static void _GpuTraceFirmwareEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt, const IMG_CHAR* pszWorkName,
+		PVR_GPUTRACE_SWITCH_TYPE eSwType)
+
+{
+	uint64_t ui64Timestamp;
+	RGX_HWPERF_FW_DATA *psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+
+	ui64Timestamp = CalculateEventTimestamp(psDevInfo, psHWPerfPktData->ui32TimeCorrIndex,
+											psHWPerfPkt->ui64Timestamp);
+
+	_GpuTraceFirmware(ui64Timestamp, psDevInfo->psDeviceNode->sDevId.ui32InternalID, pszWorkName,
+	                  eSwType);
+}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+static IMG_BOOL ValidAndEmitFTraceEvent(PVRSRV_RGXDEV_INFO *psDevInfo,
+		RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt)
+{
+	RGX_HWPERF_EVENT_TYPE eType;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData = psDevInfo->pvGpuFtraceData;
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	IMG_UINT32 ui32HwEventTypeIndex;
+	static const struct {
+		IMG_CHAR* pszName;
+		PVR_GPUTRACE_SWITCH_TYPE eSwType;
+	} aszHwEventTypeMap[] = {
+#define _T(T) PVR_GPUTRACE_SWITCH_TYPE_##T
+		{ "BG",             _T(BEGIN)  }, /* RGX_HWPERF_FW_BGSTART */
+		{ "BG",             _T(END)    }, /* RGX_HWPERF_FW_BGEND */
+		{ "IRQ",            _T(BEGIN)  }, /* RGX_HWPERF_FW_IRQSTART */
+		{ "IRQ",            _T(END)    }, /* RGX_HWPERF_FW_IRQEND */
+		{ "DBG",            _T(BEGIN)  }, /* RGX_HWPERF_FW_DBGSTART */
+		{ "DBG",            _T(END)    }, /* RGX_HWPERF_FW_DBGEND */
+		{ "PMOOM_TAPAUSE",  _T(END)    }, /* RGX_HWPERF_HW_PMOOM_TAPAUSE */
+		{ "TA",             _T(BEGIN)  }, /* RGX_HWPERF_HW_TAKICK */
+		{ "TA",             _T(END)    }, /* RGX_HWPERF_HW_TAFINISHED */
+		{ "TQ3D",           _T(BEGIN)  }, /* RGX_HWPERF_HW_3DTQKICK */
+		{ "3D",             _T(BEGIN)  }, /* RGX_HWPERF_HW_3DKICK */
+		{ "3D",             _T(END)    }, /* RGX_HWPERF_HW_3DFINISHED */
+		{ "CDM",            _T(BEGIN)  }, /* RGX_HWPERF_HW_CDMKICK */
+		{ "CDM",            _T(END)    }, /* RGX_HWPERF_HW_CDMFINISHED */
+		{ "TQ2D",           _T(BEGIN)  }, /* RGX_HWPERF_HW_TLAKICK */
+		{ "TQ2D",           _T(END)    }, /* RGX_HWPERF_HW_TLAFINISHED */
+		{ "3DSPM",          _T(BEGIN)  }, /* RGX_HWPERF_HW_3DSPMKICK */
+		{ NULL,             0          }, /* RGX_HWPERF_HW_PERIODIC (unsupported) */
+		{ "RTU",            _T(BEGIN)  }, /* RGX_HWPERF_HW_RTUKICK */
+		{ "RTU",            _T(END)    }, /* RGX_HWPERF_HW_RTUFINISHED */
+		{ "SHG",            _T(BEGIN)  }, /* RGX_HWPERF_HW_SHGKICK */
+		{ "SHG",            _T(END)    }, /* RGX_HWPERF_HW_SHGFINISHED */
+		{ "TQ3D",           _T(END)    }, /* RGX_HWPERF_HW_3DTQFINISHED */
+		{ "3DSPM",          _T(END)    }, /* RGX_HWPERF_HW_3DSPMFINISHED */
+		{ "PMOOM_TARESUME", _T(BEGIN)  }, /* RGX_HWPERF_HW_PMOOM_TARESUME */
+		{ "TDM",            _T(BEGIN)  }, /* RGX_HWPERF_HW_TDMKICK */
+		{ "TDM",            _T(END)    }, /* RGX_HWPERF_HW_TDMFINISHED */
+		{ "NULL",           _T(SINGLE) }, /* RGX_HWPERF_HW_NULLKICK */
+		{ "GPU_WORK_PERIOD",_T(SINGLE) }, /* RGX_HWPERF_HW_GPU_WORK_PERIOD */
+#undef _T
+	};
+	static_assert(RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE == RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE + 1,
+				  "FW and HW events are not contiguous in RGX_HWPERF_EVENT_TYPE");
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+
+	PVR_ASSERT(psHWPerfPkt);
+	eType = RGX_HWPERF_GET_TYPE(psHWPerfPkt);
+
+	if (psFtraceData->bTrackOrdinals)
+	{
+		if (psFtraceData->ui32FTraceLastOrdinal != psHWPerfPkt->ui32Ordinal - 1)
+		{
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+			_GpuTraceEventsLost(RGX_HWPERF_GET_STREAM_ID(psHWPerfPkt),
+			                    psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+			                    psFtraceData->ui32FTraceLastOrdinal,
+			                    psHWPerfPkt->ui32Ordinal);
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+			PVR_DPF((PVR_DBG_WARNING, "FTrace events lost (stream_id = %u, ordinal: last = %u, current = %u)",
+			         RGX_HWPERF_GET_STREAM_ID(psHWPerfPkt), psFtraceData->ui32FTraceLastOrdinal,
+			         psHWPerfPkt->ui32Ordinal));
+		}
+	}
+	else
+	{
+		psFtraceData->bTrackOrdinals = IMG_TRUE;
+	}
+
+	psFtraceData->ui32FTraceLastOrdinal = psHWPerfPkt->ui32Ordinal;
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	/* Process UFO packets */
+	if (eType == RGX_HWPERF_UFO)
+	{
+		_GpuTraceUfoEvent(psDevInfo, psHWPerfPkt);
+		return IMG_TRUE;
+	}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	if (eType <= RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE)
+	{
+		/* this ID belongs to range 0, so index directly in range 0 */
+		ui32HwEventTypeIndex = eType - RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+	}
+	else
+	{
+		/* this ID belongs to range 1, so first index in range 1 and skip number of slots used up for range 0 */
+		ui32HwEventTypeIndex = (eType - RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE) +
+		                       (RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE - RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE + 1);
+	}
+
+	if (ui32HwEventTypeIndex >= ARRAY_SIZE(aszHwEventTypeMap))
+	{
+		goto err_unsupported;
+	}
+
+	if (aszHwEventTypeMap[ui32HwEventTypeIndex].pszName == NULL)
+	{
+		/* Not supported map entry, ignore event */
+		goto err_unsupported;
+	}
+
+	if (HWPERF_PACKET_IS_HW_TYPE(eType))
+	{
+		if (aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType == PVR_GPUTRACE_SWITCH_TYPE_SINGLE)
+		{
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     PVR_GPUTRACE_SWITCH_TYPE_BEGIN);
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     PVR_GPUTRACE_SWITCH_TYPE_END);
+		}
+		else
+		{
+			_GpuTraceSwitchEvent(psDevInfo, psHWPerfPkt,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+			                     aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType);
+		}
+
+		return IMG_TRUE;
+	}
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	else if (psDevInfo->psDeviceNode->bGPUWorkPeriodFTraceEnabled &&
+		 eType == RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD &&
+		 RGX_HWPERF_GET_OSID(psHWPerfPkt) == RGXFW_HOST_DRIVER_ID)
+	{
+		RGX_HWPERF_GPU_WORK_PERIOD_DATA *psHWPerfPktData;
+		IMG_UINT64 ui64StartTimestamp, ui64EndTimestamp;
+
+		psHWPerfPktData = RGX_HWPERF_GET_PACKET_DATA_BYTES(psHWPerfPkt);
+		ui64StartTimestamp = CalculateEventTimestamp(psDevInfo,
+					psHWPerfPktData->ui32StartTimeCorrIndex,
+					psHWPerfPktData->ui64GPUWorkPeriodStartTime);
+		ui64EndTimestamp = CalculateEventTimestamp(psDevInfo,
+					psHWPerfPktData->ui32TimeCorrIndex,
+					psHWPerfPkt->ui64Timestamp);
+
+		PVR_ASSERT(ui64EndTimestamp > ui64StartTimestamp);
+
+		trace_gpu_work_period(psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+				psHWPerfPktData->ui32UID,
+				ui64StartTimestamp,
+				ui64EndTimestamp,
+				ui64EndTimestamp - ui64StartTimestamp);
+
+		return IMG_TRUE;
+	}
+#endif
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	if (HWPERF_PACKET_IS_FW_TYPE(eType))
+	{
+		_GpuTraceFirmwareEvent(psDevInfo, psHWPerfPkt,
+										aszHwEventTypeMap[ui32HwEventTypeIndex].pszName,
+										aszHwEventTypeMap[ui32HwEventTypeIndex].eSwType);
+
+		return IMG_TRUE;
+	}
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+err_unsupported:
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) || defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD) */
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Unsupported event type %d", __func__, eType));
+	return IMG_FALSE;
+}
+
+static void _GpuTraceProcessPackets(PVRSRV_RGXDEV_INFO *psDevInfo, void *pBuffer,
+                                    IMG_UINT32 ui32ReadLen)
+{
+	IMG_UINT32			ui32TlPackets = 0;
+	IMG_UINT32			ui32HWPerfPackets = 0;
+	IMG_UINT32			ui32HWPerfPacketsSent = 0;
+	void				*pBufferEnd;
+	PVRSRVTL_PPACKETHDR psHDRptr;
+	PVRSRVTL_PACKETTYPE ui16TlType;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDevInfo);
+	PVR_ASSERT(pBuffer);
+	PVR_ASSERT(ui32ReadLen);
+
+	/* Process the TL Packets
+	 */
+	pBufferEnd = IMG_OFFSET_ADDR(pBuffer, ui32ReadLen);
+	psHDRptr = GET_PACKET_HDR(pBuffer);
+	while ( psHDRptr < (PVRSRVTL_PPACKETHDR)pBufferEnd )
+	{
+		ui16TlType = GET_PACKET_TYPE(psHDRptr);
+		if (ui16TlType == PVRSRVTL_PACKETTYPE_DATA)
+		{
+			IMG_UINT16 ui16DataLen = GET_PACKET_DATA_LEN(psHDRptr);
+			if (0 == ui16DataLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "_GpuTraceProcessPackets: ZERO Data in TL data packet: %p", psHDRptr));
+			}
+			else
+			{
+				RGX_HWPERF_V2_PACKET_HDR* psHWPerfPkt;
+				RGX_HWPERF_V2_PACKET_HDR* psHWPerfEnd;
+
+				/* Check for lost hwperf data packets */
+				psHWPerfEnd = RGX_HWPERF_GET_PACKET(GET_PACKET_DATA_PTR(psHDRptr)+ui16DataLen);
+				psHWPerfPkt = RGX_HWPERF_GET_PACKET(GET_PACKET_DATA_PTR(psHDRptr));
+				do
+				{
+					if (ValidAndEmitFTraceEvent(psDevInfo, psHWPerfPkt))
+					{
+						ui32HWPerfPacketsSent++;
+					}
+					ui32HWPerfPackets++;
+					psHWPerfPkt = RGX_HWPERF_GET_NEXT_PACKET(psHWPerfPkt);
+				}
+				while (psHWPerfPkt < psHWPerfEnd);
+			}
+		}
+		else if (ui16TlType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "_GpuTraceProcessPackets: Indication that the transport buffer was full"));
+		}
+		else
+		{
+			/* else Ignore padding packet type and others */
+			PVR_DPF((PVR_DBG_MESSAGE, "_GpuTraceProcessPackets: Ignoring TL packet, type %d", ui16TlType ));
+		}
+
+		psHDRptr = GET_NEXT_PACKET_ADDR(psHDRptr);
+		ui32TlPackets++;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "_GpuTraceProcessPackets: TL "
+			"Packets processed %03d, HWPerf packets %03d, sent %03d",
+			ui32TlPackets, ui32HWPerfPackets, ui32HWPerfPacketsSent));
+
+	PVR_DPF_RETURN;
+}
+
+static void _GpuTraceCmdCompleteNotify(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_RGXDEV_INFO *psDeviceInfo = hCmdCompHandle;
+	RGX_HWPERF_FTRACE_DATA *psFtraceData;
+	PVRSRV_ERROR eError;
+	IMG_PBYTE pBuffer;
+	IMG_UINT32 ui32ReadLen;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDeviceInfo != NULL);
+
+	psFtraceData = psDeviceInfo->pvGpuFtraceData;
+
+	/* Command-complete notifiers can run concurrently. If this is happening,
+	 * just bail out and let the previous call finish.
+	 * This is ok because we can process the queued packets on the next call.
+	 */
+	if (!OSTryLockAcquire(psFtraceData->hFTraceResourceLock))
+	{
+		PVR_DPF_RETURN;
+	}
+
+	/* If this notifier is called, it means the TL resources will be valid
+	 * at-least until the end of this call, since the DeInit function will wait
+	 * on the hFTraceResourceLock to clean-up the TL resources and un-register
+	 * the notifier, so just assert here.
+	 */
+	PVR_ASSERT(psFtraceData->hGPUFTraceTLStream != NULL);
+
+	/* If we have a valid stream attempt to acquire some data */
+	eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+	                             psFtraceData->hGPUFTraceTLStream,
+	                             &pBuffer, &ui32ReadLen);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_LOG_ERROR(eError, "TLClientAcquireData");
+		}
+
+		goto unlock;
+	}
+
+	/* Process the HWPerf packets and release the data */
+	if (ui32ReadLen > 0)
+	{
+		PVR_DPF((PVR_DBG_VERBOSE, "%s: DATA AVAILABLE offset=%p, length=%d",
+		        __func__, pBuffer, ui32ReadLen));
+
+		/* Process the transport layer data for HWPerf packets... */
+		_GpuTraceProcessPackets(psDeviceInfo, pBuffer, ui32ReadLen);
+
+		eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE,
+		                             psFtraceData->hGPUFTraceTLStream);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "TLClientReleaseData");
+
+			/* Serious error, disable FTrace GPU events */
+			_GpuTraceDisable(psDeviceInfo, IMG_FALSE);
+		}
+	}
+
+unlock:
+	OSLockRelease(psFtraceData->hFTraceResourceLock);
+
+	PVR_DPF_RETURN;
+}
+
+/* ----- AppHint interface -------------------------------------------------- */
+
+static PVRSRV_ERROR _GpuTraceIsEnabledCallback(
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data,
+	IMG_BOOL *value)
+{
+	PVR_UNREFERENCED_PARAMETER(private_data);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (device && PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, device))
+	{
+		*value = IMG_FALSE;
+	}
+	else
+#endif
+	{
+		PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+		PVRSRV_DEVICE_NODE *psDeviceNode = psPVRSRVData ? psPVRSRVData->psDeviceNodeList : NULL;
+
+		while (psDeviceNode)
+		{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+			PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+			{
+				*value = gbFTraceGPUEventsEnabled;
+				return PVRSRV_OK;
+			}
+			psDeviceNode = psDeviceNode->psNext;
+		}
+		*value = IMG_FALSE;
+	}
+
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR _GpuTraceSetEnabledCallback(
+	const PVRSRV_DEVICE_NODE *device,
+	const void *private_data,
+	IMG_BOOL value)
+{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (device && PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, device))
+	{
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	/* Lock down the state to avoid concurrent writes */
+	OSLockAcquire(ghGPUTraceStateLock);
+
+	if (value != gbFTraceGPUEventsEnabled)
+	{
+#if defined(PVRSRV_NEED_PVR_DPF)
+		const IMG_CHAR *pszOperation = value ? "enable" : "disable";
+		/* in case MESSAGE level is compiled out */
+		PVR_UNREFERENCED_PARAMETER(pszOperation);
+#endif
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		if (trace_set_clr_event("rogue", NULL, (int) value) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s GPU FTrace event group",
+			        pszOperation));
+			goto err_restore_state;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "FTrace events from \"rogue\" group %sd",
+			        pszOperation));
+		}
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+		if (trace_set_clr_event("power", "gpu_work_period", (int) value) != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s gpu_work_period GPU FTrace event",
+			        pszOperation));
+			goto err_restore_state;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "FTrace event from \"gpu_work_period\" %sd",
+			        pszOperation));
+		}
+#endif
+
+		if (value)
+		{
+			/* this enables FTrace globally (if not enabled nothing will appear
+			 * in the FTrace buffer) */
+			tracing_on();
+		}
+
+		/*  The HWPerf supplier is activated here,
+		    The FTrace consumer is activated above,
+		    The consumer should be active before the supplier    */
+		if (_GpuTraceSetEnabledForAllDevices(value) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FAILED to %s GPU FTrace for all devices",
+			        pszOperation));
+			goto err_restore_state;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s GPU FTrace", value ? "ENABLED" : "DISABLED"));
+		gbFTraceGPUEventsEnabled = value;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "GPU FTrace already %s!",
+		        value ? "enabled" : "disabled"));
+	}
+
+	OSLockRelease(ghGPUTraceStateLock);
+
+	return PVRSRV_OK;
+
+err_restore_state:
+	/* On failure, partial enable/disable might have resulted. Try best to
+	 * restore to previous state. Ignore error */
+	(void) _GpuTraceSetEnabledForAllDevices(gbFTraceGPUEventsEnabled);
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	(void) trace_set_clr_event("rogue", NULL,
+	                           (int) gbFTraceGPUEventsEnabled);
+#endif
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	(void) trace_set_clr_event("power", "gpu_work_period",
+	                           (int) gbFTraceGPUEventsEnabled);
+#endif
+
+	OSLockRelease(ghGPUTraceStateLock);
+
+	return PVRSRV_ERROR_UNABLE_TO_ENABLE_EVENT;
+}
+
+void PVRGpuTraceInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFTraceGPU,
+	                                  _GpuTraceIsEnabledCallback,
+	                                  _GpuTraceSetEnabledCallback,
+	                                  psDeviceNode, NULL);
+}
+
+/* ----- FTrace event callbacks -------------------------------------------- */
+
+void PVRGpuTraceEnableUfoCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	/* Lock down events state, for consistent value of guiUfoEventRef */
+	OSLockAcquire(ghLockFTraceEventLock);
+	if (guiUfoEventRef++ == 0)
+	{
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+		/* make sure UFO events are enabled on all rogue devices */
+		while (psDeviceNode)
+		{
+#if defined(SUPPORT_RGX)
+			PVRSRV_ERROR eError;
+
+			/* Small chance exists that ui64HWPerfFilter can be changed here and
+			 * the newest filter value will be changed to the old one + UFO event.
+			 * This is not a critical problem. */
+			eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+			                               RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO),
+			                               HWPERF_FILTER_OPERATION_BIT_OR);
+			if (eError == PVRSRV_ERROR_NOT_INITIALISED)
+			{
+				/* If we land here that means that the FW is not initialised yet.
+				 * We stored the filter and it will be passed to the firmware
+				 * during its initialisation phase. So ignore. */
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Could not enable UFO HWPerf events on device %d", psDeviceNode->sDevId.i32KernelDeviceID));
+			}
+#endif
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+	OSLockRelease(ghLockFTraceEventLock);
+}
+
+void PVRGpuTraceDisableUfoCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	/* We have to check if lock is valid because on driver unload
+	 * PVRGpuTraceSupportDeInit is called before kernel disables the ftrace
+	 * events. This means that the lock will be destroyed before this callback
+	 * is called.
+	 * We can safely return if that situation happens because driver will be
+	 * unloaded so we don't care about HWPerf state anymore. */
+	if (ghLockFTraceEventLock == NULL)
+		return;
+
+	/* Lock down events state, for consistent value of guiUfoEventRef */
+	OSLockAcquire(ghLockFTraceEventLock);
+	if (--guiUfoEventRef == 0)
+	{
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+		/* make sure UFO events are disabled on all rogue devices */
+		while (psDeviceNode)
+		{
+#if defined(SUPPORT_RGX)
+			PVRSRV_ERROR eError;
+
+			/* Small chance exists that ui64HWPerfFilter can be changed here and
+			 * the newest filter value will be changed to the old one + UFO event.
+			 * This is not a critical problem. */
+			eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+			                               RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO),
+			                               HWPERF_FILTER_OPERATION_BIT_CLR);
+			if (eError == PVRSRV_ERROR_NOT_INITIALISED)
+			{
+				/* If we land here that means that the FW is not initialised yet.
+				 * We stored the filter and it will be passed to the firmware
+				 * during its initialisation phase. So ignore. */
+			}
+			else if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Could not disable UFO HWPerf events on device %d",
+				        psDeviceNode->sDevId.i32KernelDeviceID));
+			}
+#endif
+
+			psDeviceNode = psDeviceNode->psNext;
+		}
+
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	}
+	OSLockRelease(ghLockFTraceEventLock);
+}
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_UINT64 uiFilter = 0;
+	int i;
+
+	for (i = RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+		 i <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE; i++)
+	{
+		uiFilter |= RGX_HWPERF_EVENT_MASK_VALUE(i);
+	}
+
+	OSLockAcquire(ghLockFTraceEventLock);
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	/* Enable all FW events on all the devices */
+	while (psDeviceNode)
+	{
+		PVRSRV_ERROR eError;
+
+		eError = PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+		                               uiFilter, HWPERF_FILTER_OPERATION_BIT_OR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could not enable HWPerf event for firmware"
+			        " task timings (%s).", PVRSRVGetErrorString(eError)));
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	OSLockRelease(ghLockFTraceEventLock);
+#endif /* defined(SUPPORT_RGX) */
+}
+
+void PVRGpuTraceDisableFirmwareActivityCallback(void)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	IMG_UINT64 uiFilter = 0;
+	int i;
+
+	/* We have to check if lock is valid because on driver unload
+	 * PVRGpuTraceSupportDeInit is called before kernel disables the ftrace
+	 * events. This means that the lock will be destroyed before this callback
+	 * is called.
+	 * We can safely return if that situation happens because driver will be
+	 * unloaded so we don't care about HWPerf state anymore. */
+	if (ghLockFTraceEventLock == NULL)
+		return;
+
+	OSLockAcquire(ghLockFTraceEventLock);
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	for (i = RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE;
+		 i <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE; i++)
+	{
+		uiFilter |= RGX_HWPERF_EVENT_MASK_VALUE(i);
+	}
+
+	/* Disable all FW events on all the devices */
+	while (psDeviceNode)
+	{
+		if (PVRSRVRGXCtrlHWPerfFW(psDeviceNode, RGX_HWPERF_L2_STREAM_FTRACE,
+		                          uiFilter, HWPERF_FILTER_OPERATION_BIT_CLR) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Could not disable HWPerf event for firmware task timings."));
+		}
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	OSLockRelease(ghLockFTraceEventLock);
+#endif /* defined(SUPPORT_RGX) */
+}
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableWorkPeriodCallback(void)
+#else
+void PVRGpuTraceEnableWorkPeriodCallback(void)
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	/* enable/disable GPU trace on all devices */
+	while (psDeviceNode)
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			psDeviceNode->bGPUWorkPeriodFTraceEnabled = true;
+			eError = PVRSRV_OK;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	if (eError != PVRSRV_OK)
+		return -ENODEV;
+	return 0;
+#else
+	return;
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+}
+
+void PVRGpuTraceDisableWorkPeriodCallback(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	if (!psPVRSRVData)
+		return;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	while (psDeviceNode)
+	{
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		PVRSRV_RGXDEV_INFO *psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo))
+#endif
+		{
+			psDeviceNode->bGPUWorkPeriodFTraceEnabled = false;
+		}
+
+		psDeviceNode = psDeviceNode->psNext;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+}
+#endif
+/******************************************************************************
+ End of file (pvr_gputrace.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pvr_intrinsics.h b/drivers/gpu/drm/img-rogue/pvr_intrinsics.h
new file mode 100644
index 000000000000..410a2f5a50b5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_intrinsics.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File
+@Title          Intrinsics definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_INTRINSICS_H
+#define PVR_INTRINSICS_H
+
+/* PVR_CTZLL:
+ * Count the number of trailing zeroes in a long long integer
+ */
+
+#if defined(__GNUC__)
+#if defined(__x86_64__)
+
+	#define PVR_CTZLL __builtin_ctzll
+#endif
+#endif
+
+/* PVR_CLZLL:
+ * Count the number of leading zeroes in a long long integer
+ */
+
+#if defined(__GNUC__)
+#if defined(__x86_64__) || defined(__i386__) || defined(__aarch64__) || \
+					defined(__arm__) || defined(__mips)
+
+#define PVR_CLZLL __builtin_clzll
+
+#endif
+#endif
+
+#endif /* PVR_INTRINSICS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_ion_stats.h b/drivers/gpu/drm/img-rogue/pvr_ion_stats.h
new file mode 100644
index 000000000000..1c0d44acac44
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_ion_stats.h
@@ -0,0 +1,91 @@
+/*************************************************************************/ /*!
+@File
+@Title          Functions for recording ION memory stats.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_ION_STATS_H
+#define PVR_ION_STATS_H
+
+#include "pvrsrv_error.h"
+#include "img_defs.h"
+
+struct dma_buf;
+
+#if defined(PVRSRV_ENABLE_PVR_ION_STATS)
+PVRSRV_ERROR PVRSRVIonStatsInitialise(void);
+
+void PVRSRVIonStatsDestroy(void);
+
+void PVRSRVIonAddMemAllocRecord(struct dma_buf *psDmaBuf);
+
+void PVRSRVIonRemoveMemAllocRecord(struct dma_buf *psDmaBuf);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+void PVRSRVIonZombifyMemAllocRecord(const struct dma_buf *psDmaBuf);
+#endif
+#else
+static INLINE PVRSRV_ERROR PVRSRVIonStatsInitialise(void)
+{
+	return PVRSRV_OK;
+}
+
+static INLINE void PVRSRVIonStatsDestroy(void)
+{
+}
+
+static INLINE void PVRSRVIonAddMemAllocRecord(struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+
+static INLINE void PVRSRVIonRemoveMemAllocRecord(struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+static INLINE void PVRSRVIonZombifyMemAllocRecord(const struct dma_buf *psDmaBuf)
+{
+	PVR_UNREFERENCED_PARAMETER(psDmaBuf);
+}
+#endif
+#endif /* defined(PVRSRV_ENABLE_PVR_ION_STATS) */
+
+#endif /* PVR_ION_STATS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_linux_fence.h b/drivers/gpu/drm/img-rogue/pvr_linux_fence.h
new file mode 100644
index 000000000000..b9c542a26a65
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_linux_fence.h
@@ -0,0 +1,103 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence compatibility header
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_LINUX_FENCE_H__)
+#define __PVR_LINUX_FENCE_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+#include <linux/fence.h>
+#else
+#include <linux/dma-fence.h>
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+/* Structures */
+#define	dma_fence fence
+#define dma_fence_array fence_array
+#define	dma_fence_cb fence_cb
+#define	dma_fence_ops fence_ops
+
+/* Defines and Enums */
+#define DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT FENCE_FLAG_ENABLE_SIGNAL_BIT
+#define DMA_FENCE_FLAG_SIGNALED_BIT FENCE_FLAG_SIGNALED_BIT
+#define DMA_FENCE_FLAG_USER_BITS FENCE_FLAG_USER_BITS
+
+#define DMA_FENCE_ERR FENCE_ERR
+#define	DMA_FENCE_TRACE FENCE_TRACE
+#define DMA_FENCE_WARN FENCE_WARN
+
+/* Functions */
+#define dma_fence_add_callback fence_add_callback
+#define dma_fence_context_alloc fence_context_alloc
+#define dma_fence_default_wait fence_default_wait
+#define dma_fence_is_signaled fence_is_signaled
+#define dma_fence_enable_sw_signaling fence_enable_sw_signaling
+#define dma_fence_free fence_free
+#define dma_fence_get fence_get
+#define dma_fence_get_rcu fence_get_rcu
+#define dma_fence_init fence_init
+#define dma_fence_is_array fence_is_array
+#define dma_fence_put fence_put
+#define dma_fence_signal fence_signal
+#define dma_fence_wait fence_wait
+#define to_dma_fence_array to_fence_array
+
+static inline signed long
+dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
+{
+	signed long lret;
+
+	lret = fence_wait_timeout(fence, intr, timeout);
+	if (lret || timeout)
+		return lret;
+
+	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1 : 0;
+}
+
+#endif
+
+#endif /* !defined(__PVR_LINUX_FENCE_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_notifier.c b/drivers/gpu/drm/img-rogue/pvr_notifier.c
new file mode 100644
index 000000000000..50fd35358ad2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_notifier.c
@@ -0,0 +1,670 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR notifier interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "allocmem.h"
+#include "dllist.h"
+
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrversion.h"
+#include "connection_server.h"
+
+#include "osfunc.h"
+#include "sofunc_pvr.h"
+
+#define PVR_DUMP_DRIVER_INFO(x, y)														\
+	PVR_DUMPDEBUG_LOG("%s info: %d.%d @ %8d (%s) build options: 0x%08x",				\
+					   (x),																\
+					   PVRVERSION_UNPACK_MAJ((y).ui32BuildVersion),						\
+					   PVRVERSION_UNPACK_MIN((y).ui32BuildVersion),						\
+					   (y).ui32BuildRevision,											\
+					   (BUILD_TYPE_DEBUG == (y).ui32BuildType) ? "debug":"release",		\
+					   (y).ui32BuildOptions);
+
+#if !defined(WINDOW_SYSTEM)
+#define WINDOW_SYSTEM "Unknown"
+#endif
+
+#define IS_DECLARED(x) (x[0] != '\0')
+
+/*************************************************************************/ /*!
+Command Complete Notifier Interface
+*/ /**************************************************************************/
+
+typedef struct PVRSRV_CMDCOMP_NOTIFY_TAG
+{
+	PVRSRV_CMDCOMP_HANDLE	hCmdCompHandle;
+	PFN_CMDCOMP_NOTIFY		pfnCmdCompleteNotify;
+	DLLIST_NODE				sListNode;
+} PVRSRV_CMDCOMP_NOTIFY;
+
+/* Head of the list of callbacks called when command complete happens */
+static DLLIST_NODE g_sCmdCompNotifyHead;
+static POSWR_LOCK g_hCmdCompNotifyLock;
+
+PVRSRV_ERROR
+PVRSRVCmdCompleteInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSWRLockCreate(&g_hCmdCompNotifyLock);
+	PVR_RETURN_IF_ERROR(eError);
+
+	dllist_init(&g_sCmdCompNotifyHead);
+
+	return PVRSRV_OK;
+}
+
+void
+PVRSRVCmdCompleteDeinit(void)
+{
+	/* Check that all notify function have been unregistered */
+	if (!dllist_is_empty(&g_sCmdCompNotifyHead))
+	{
+		PDLLIST_NODE psNode;
+
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Command complete notify list is not empty!", __func__));
+
+		/* Clean up any stragglers */
+		psNode = dllist_get_next_node(&g_sCmdCompNotifyHead);
+		while (psNode)
+		{
+			PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+			dllist_remove_node(psNode);
+
+			psNotify = IMG_CONTAINER_OF(psNode, PVRSRV_CMDCOMP_NOTIFY, sListNode);
+			OSFreeMem(psNotify);
+
+			psNode = dllist_get_next_node(&g_sCmdCompNotifyHead);
+		}
+	}
+
+	if (g_hCmdCompNotifyLock)
+	{
+		OSWRLockDestroy(g_hCmdCompNotifyLock);
+	}
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterCmdCompleteNotify(IMG_HANDLE *phNotify,
+								PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify,
+								PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phNotify, "phNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnCmdCompleteNotify, "pfnCmdCompleteNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hCmdCompHandle, "hCmdCompHandle");
+
+	psNotify = OSAllocMem(sizeof(*psNotify));
+	PVR_LOG_RETURN_IF_NOMEM(psNotify, "psNotify");
+
+	/* Set-up the notify data */
+	psNotify->hCmdCompHandle = hCmdCompHandle;
+	psNotify->pfnCmdCompleteNotify = pfnCmdCompleteNotify;
+
+	/* Add it to the list of Notify functions */
+	OSWRLockAcquireWrite(g_hCmdCompNotifyLock);
+	dllist_add_to_tail(&g_sCmdCompNotifyHead, &psNotify->sListNode);
+	OSWRLockReleaseWrite(g_hCmdCompNotifyLock);
+
+	*phNotify = psNotify;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterCmdCompleteNotify(IMG_HANDLE hNotify)
+{
+	PVRSRV_CMDCOMP_NOTIFY *psNotify;
+
+	psNotify = (PVRSRV_CMDCOMP_NOTIFY *) hNotify;
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNotify, "hNotify");
+
+	OSWRLockAcquireWrite(g_hCmdCompNotifyLock);
+	dllist_remove_node(&psNotify->sListNode);
+	OSWRLockReleaseWrite(g_hCmdCompNotifyLock);
+
+	OSFreeMem(psNotify);
+
+	return PVRSRV_OK;
+}
+
+void
+PVRSRVNotifyCommandCompletion(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle)
+{
+#if !defined(NO_HARDWARE)
+	DLLIST_NODE *psNode, *psNext;
+
+	/* Call notify callbacks to check if blocked work items can now proceed */
+	OSWRLockAcquireRead(g_hCmdCompNotifyLock);
+	dllist_foreach_node(&g_sCmdCompNotifyHead, psNode, psNext)
+	{
+		PVRSRV_CMDCOMP_NOTIFY *psNotify =
+			IMG_CONTAINER_OF(psNode, PVRSRV_CMDCOMP_NOTIFY, sListNode);
+
+		if (hCmdCompCallerHandle != psNotify->hCmdCompHandle)
+		{
+			psNotify->pfnCmdCompleteNotify(psNotify->hCmdCompHandle);
+		}
+	}
+	OSWRLockReleaseRead(g_hCmdCompNotifyLock);
+#endif
+}
+
+inline void
+PVRSRVSignalDriverWideEO(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if (psPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectSignal(psPVRSRVData->hGlobalEventObject);
+	}
+	/* Cleanup Thread could be waiting on Cleanup event object,
+	 * signal it as well to ensure work is processed
+	 */
+	if (psPVRSRVData->hCleanupEventObject && (OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued) != 0))
+	{
+		OSEventObjectSignal(psPVRSRVData->hCleanupEventObject);
+	}
+}
+
+inline void
+PVRSRVCheckStatus(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle)
+{
+	PVRSRVNotifyCommandCompletion(hCmdCompCallerHandle);
+	PVRSRVSignalDriverWideEO();
+}
+
+/*************************************************************************/ /*!
+Debug Notifier Interface
+*/ /**************************************************************************/
+
+/* Lockdep sees both locks as the same class due to same struct used thus warns
+ * about a possible deadlock (false positive),
+ * using nested api we can supply separate Class'
+ * */
+#define DN_LOCKCLASS_DRIVER 0
+#define DN_LOCKCLASS_DEVICE 1
+
+typedef struct DEBUG_REQUEST_ENTRY_TAG
+{
+	IMG_UINT32      ui32RequesterID;
+	DLLIST_NODE     sListHead;
+} DEBUG_REQUEST_ENTRY;
+
+typedef struct DEBUG_REQUEST_TABLE_TAG
+{
+	POSWR_LOCK              hLock;
+	DEBUG_REQUEST_ENTRY     asEntry[IMG_FLEX_ARRAY_MEMBER];
+} DEBUG_REQUEST_TABLE;
+
+typedef struct DEBUG_REQUEST_NOTIFY_TAG
+{
+	IMG_HANDLE              hDebugTable;
+	PVRSRV_DBGREQ_HANDLE    hDbgRequestHandle;
+	PFN_DBGREQ_NOTIFY       pfnDbgRequestNotify;
+	IMG_UINT32              ui32RequesterID;
+	DLLIST_NODE             sListNode;
+} DEBUG_REQUEST_NOTIFY;
+
+static DEBUG_REQUEST_TABLE *g_psDriverDebugTable;
+
+static const IMG_UINT32 g_aui32DebugOrderTable[] = {
+	DEBUG_REQUEST_SRV,
+	DEBUG_REQUEST_RGX,
+	DEBUG_REQUEST_SYS,
+	DEBUG_REQUEST_APPHINT,
+	DEBUG_REQUEST_HTB,
+	DEBUG_REQUEST_DC,
+	DEBUG_REQUEST_SYNCCHECKPOINT,
+	DEBUG_REQUEST_SYNCTRACKING,
+	DEBUG_REQUEST_ANDROIDSYNC,
+	DEBUG_REQUEST_FALLBACKSYNC,
+	DEBUG_REQUEST_LINUXFENCE
+};
+static const IMG_UINT32 g_ui32DebugOrderTableReqCount = ARRAY_SIZE(g_aui32DebugOrderTable);
+
+static PVRSRV_ERROR
+_RegisterDebugTableI(DEBUG_REQUEST_TABLE **ppsDebugTable)
+{
+	DEBUG_REQUEST_TABLE *psDebugTable;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	if (*ppsDebugTable)
+	{
+		return PVRSRV_ERROR_DBGTABLE_ALREADY_REGISTERED;
+	}
+
+	psDebugTable = OSAllocMem(sizeof(DEBUG_REQUEST_TABLE) +
+	                          IMG_FLEX_ARRAY_SIZE(sizeof(DEBUG_REQUEST_ENTRY), g_ui32DebugOrderTableReqCount));
+	PVR_RETURN_IF_NOMEM(psDebugTable);
+
+	eError = OSWRLockCreate(&psDebugTable->hLock);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeDebugTable);
+
+	/* Init the list heads */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		psDebugTable->asEntry[i].ui32RequesterID = g_aui32DebugOrderTable[i];
+		dllist_init(&psDebugTable->asEntry[i].sListHead);
+	}
+
+	*ppsDebugTable = psDebugTable;
+
+	return PVRSRV_OK;
+
+ErrorFreeDebugTable:
+	OSFreeMem(psDebugTable);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgTable(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	return _RegisterDebugTableI((DEBUG_REQUEST_TABLE**)&psDevNode->hDebugTable);
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgTable(void)
+{
+	return _RegisterDebugTableI(&g_psDriverDebugTable);
+}
+
+static void _UnregisterDbgTableI(DEBUG_REQUEST_TABLE **ppsDebugTable)
+{
+	DEBUG_REQUEST_TABLE *psDebugTable;
+	IMG_UINT32 i;
+
+	PVR_ASSERT(*ppsDebugTable);
+	psDebugTable = *ppsDebugTable;
+	*ppsDebugTable = NULL;
+
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		if (!dllist_is_empty(&psDebugTable->asEntry[i].sListHead))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Found registered callback(s) on %d",
+					 __func__, i));
+		}
+	}
+
+	OSWRLockDestroy(psDebugTable->hLock);
+	psDebugTable->hLock = NULL;
+
+	OSFreeMem(psDebugTable);
+}
+
+void
+PVRSRVUnregisterDeviceDbgTable(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	_UnregisterDbgTableI((DEBUG_REQUEST_TABLE**)&psDevNode->hDebugTable);
+	PVR_ASSERT(!psDevNode->hDebugTable);
+}
+
+void
+PVRSRVUnregisterDriverDbgTable(void)
+{
+	_UnregisterDbgTableI(&g_psDriverDebugTable);
+	PVR_ASSERT(!g_psDriverDebugTable);
+}
+
+static PVRSRV_ERROR
+_RegisterDbgRequestNotifyI(IMG_HANDLE *phNotify,
+		                   DEBUG_REQUEST_TABLE *psDebugTable,
+		                   PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+		                   IMG_UINT32 ui32RequesterID,
+		                   PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	DEBUG_REQUEST_NOTIFY *psNotify;
+	PDLLIST_NODE psHead = NULL;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phNotify, "phNotify");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDebugTable, "psDebugTable");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(pfnDbgRequestNotify, "pfnDbRequestNotify");
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psNotify = OSAllocMemNoStats(sizeof(*psNotify));
+	PVR_LOG_RETURN_IF_NOMEM(psNotify, "psNotify");
+
+	/* Set-up the notify data */
+	psNotify->hDebugTable = psDebugTable;
+	psNotify->hDbgRequestHandle = hDbgRequestHandle;
+	psNotify->pfnDbgRequestNotify = pfnDbgRequestNotify;
+	psNotify->ui32RequesterID = ui32RequesterID;
+
+	/* Lock down all the lists */
+	OSWRLockAcquireWrite(psDebugTable->hLock);
+
+	/* Find which list to add it to */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		if (psDebugTable->asEntry[i].ui32RequesterID == ui32RequesterID)
+		{
+			psHead = &psDebugTable->asEntry[i].sListHead;
+		}
+	}
+
+	/* Failed to find debug requester */
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psHead, eError, ErrorReleaseLock);
+
+	/* Add it to the list of Notify functions */
+	dllist_add_to_tail(psHead, &psNotify->sListNode);
+
+	/* Unlock the lists */
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+
+	*phNotify = psNotify;
+
+	return PVRSRV_OK;
+
+ErrorReleaseLock:
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+	OSFreeMem(psNotify);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgRequestNotify(IMG_HANDLE *phNotify,
+	                                 PVRSRV_DEVICE_NODE *psDevNode,
+	                                 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	                                 IMG_UINT32 ui32RequesterID,
+	                                 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevNode, "psDevNode");
+	if (!psDevNode->hDebugTable)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: psDevNode->hDebugTable not yet initialised!",
+				 __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	return _RegisterDbgRequestNotifyI(phNotify,
+			                          (DEBUG_REQUEST_TABLE *)psDevNode->hDebugTable,
+			                          pfnDbgRequestNotify,
+			                          ui32RequesterID,
+			                          hDbgRequestHandle);
+}
+
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgRequestNotify(IMG_HANDLE *phNotify,
+	                                 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	                                 IMG_UINT32 ui32RequesterID,
+	                                 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	if (!g_psDriverDebugTable)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: g_psDriverDebugTable not yet initialised!",
+				 __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	return _RegisterDbgRequestNotifyI(phNotify,
+			                          g_psDriverDebugTable,
+			                          pfnDbgRequestNotify,
+			                          ui32RequesterID,
+			                          hDbgRequestHandle);
+}
+
+PVRSRV_ERROR
+SOPvrDbgRequestNotifyRegister(IMG_HANDLE *phNotify,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+							  IMG_UINT32 ui32RequesterID,
+							  PVRSRV_DBGREQ_HANDLE hDbgRequestHandle)
+{
+	return PVRSRVRegisterDeviceDbgRequestNotify(phNotify,
+			psDevNode,
+			pfnDbgRequestNotify,
+			ui32RequesterID,
+			hDbgRequestHandle);
+}
+
+static PVRSRV_ERROR
+_UnregisterDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	DEBUG_REQUEST_NOTIFY *psNotify = (DEBUG_REQUEST_NOTIFY *) hNotify;
+	DEBUG_REQUEST_TABLE *psDebugTable;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psNotify, "psNotify");
+
+	psDebugTable = (DEBUG_REQUEST_TABLE *) psNotify->hDebugTable;
+
+	OSWRLockAcquireWrite(psDebugTable->hLock);
+	dllist_remove_node(&psNotify->sListNode);
+	OSWRLockReleaseWrite(psDebugTable->hLock);
+
+	OSFreeMemNoStats(psNotify);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterDeviceDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+PVRSRV_ERROR
+PVRSRVUnregisterDriverDbgRequestNotify(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+PVRSRV_ERROR
+SOPvrDbgRequestNotifyUnregister(IMG_HANDLE hNotify)
+{
+	return _UnregisterDbgRequestNotify(hNotify);
+}
+
+void
+PVRSRVDebugRequest(PVRSRV_DEVICE_NODE *psDevNode,
+				   IMG_UINT32 ui32VerbLevel,
+				   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				   void *pvDumpDebugFile)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	DEBUG_REQUEST_TABLE *psDebugTable =
+		(DEBUG_REQUEST_TABLE *) psDevNode->hDebugTable;
+	DEBUG_REQUEST_TABLE *psDriverDebugTable =
+		(DEBUG_REQUEST_TABLE *) g_psDriverDebugTable;
+	static const IMG_CHAR *apszVerbosityTable[] = { "Low", "Medium", "High" };
+	const IMG_CHAR *szVerbosityLevel;
+	const IMG_CHAR *Bit32 = "32 Bit", *Bit64 = "64 Bit";
+	IMG_UINT32 i;
+
+	static_assert(ARRAY_SIZE(apszVerbosityTable) == DEBUG_REQUEST_VERBOSITY_MAX+1,
+	              "Incorrect number of verbosity levels");
+
+	PVR_ASSERT(psDebugTable);
+	PVR_ASSERT(psDriverDebugTable);
+
+	if (ui32VerbLevel < ARRAY_SIZE(apszVerbosityTable))
+	{
+		szVerbosityLevel = apszVerbosityTable[ui32VerbLevel];
+	}
+	else
+	{
+		szVerbosityLevel = "unknown";
+		PVR_ASSERT(!"Invalid verbosity level received");
+	}
+
+	PVR_DUMPDEBUG_LOG("------------[ PVR DBG: START (%s) ]------------",
+	                  szVerbosityLevel);
+
+#if defined(RGX_IRQ_HYPERV_HANDLER)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+#endif
+	{
+		OSDumpVersionInfo(pfnDumpDebugPrintf, pvDumpDebugFile);
+	}
+
+	PVR_DUMPDEBUG_LOG("DDK info: %s (%s) %s",
+	                  PVRVERSION_STRING, PVR_BUILD_TYPE, PVR_BUILD_DIR);
+
+	PVR_DUMPDEBUG_LOG("Time now: %" IMG_UINT64_FMTSPEC "us",
+	                  OSClockus64());
+
+	switch (psPVRSRVData->eServicesState)
+	{
+		case PVRSRV_SERVICES_STATE_OK:
+			PVR_DUMPDEBUG_LOG("Services State: OK");
+			break;
+		case PVRSRV_SERVICES_STATE_BAD:
+			PVR_DUMPDEBUG_LOG("Services State: BAD");
+			break;
+		case PVRSRV_SERVICES_STATE_UNDEFINED:
+			PVR_DUMPDEBUG_LOG("Services State: UNDEFINED");
+			break;
+		default:
+			PVR_DUMPDEBUG_LOG("Services State: UNKNOWN (%d)",
+			                  psPVRSRVData->eServicesState);
+			break;
+	}
+
+	PVR_DUMPDEBUG_LOG("Server Errors: %d",
+	          PVRSRV_KM_ERRORS);
+
+	PVRSRVConnectionDebugNotify(psDevNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+
+	PVR_DUMPDEBUG_LOG("------[ Driver Info ]------");
+
+	PVR_DUMPDEBUG_LOG("Comparison of UM/KM components: %s",
+	                 (psPVRSRVData->sDriverInfo.bIsNoMatch) ? "MISMATCH" : "MATCHING");
+
+	if (psPVRSRVData->sDriverInfo.ui8KMBitArch)
+	{
+		PVR_DUMPDEBUG_LOG("KM Arch: %s",(psPVRSRVData->sDriverInfo.ui8KMBitArch & BUILD_ARCH_64BIT) ? Bit64 : Bit32);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("KM Arch is undefined");
+	}
+
+	PVR_DUMPDEBUG_LOG("Driver Mode: %s",
+	                  PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDevNode) ? "Native" : (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDevNode)) ? "Host":"Guest");
+
+	if (psPVRSRVData->sDriverInfo.ui8UMSupportedArch)
+	{
+		if ((psPVRSRVData->sDriverInfo.ui8UMSupportedArch & BUILD_ARCH_BOTH) ==
+			BUILD_ARCH_BOTH)
+		{
+			PVR_DUMPDEBUG_LOG("UM Connected Clients Arch: %s and %s", Bit64, Bit32);
+
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("UM Connected Clients: %s",
+			                 (psPVRSRVData->sDriverInfo.ui8UMSupportedArch & BUILD_ARCH_64BIT) ? Bit64 : Bit32);
+		}
+	}
+
+	PVR_DUMP_DRIVER_INFO("UM", psPVRSRVData->sDriverInfo.sUMBuildInfo);
+	PVR_DUMP_DRIVER_INFO("KM", psPVRSRVData->sDriverInfo.sKMBuildInfo);
+
+	PVR_DUMPDEBUG_LOG("Window system: %s", (IS_DECLARED(WINDOW_SYSTEM)) ? (WINDOW_SYSTEM) : "Not declared");
+
+	PVR_DUMPDEBUG_LOG("Power lock status: %s", OSLockIsLocked(psDevNode->hPowerLock) ? "Locked" : "Free");
+#if defined(DEBUG)
+	if (OSLockIsLocked(psDevNode->hPowerLock))
+	{
+		/* Ensure this info is logged before the power lock is taken, in case
+		 * it's already in use. */
+		PVR_DUMPDEBUG_LOG("Power lock owner: PID = %u at timestamp %" IMG_UINT64_FMTSPEC " (%s:%u)",
+		                  psDevNode->uiPwrLockOwnerPID, psDevNode->sPowerLockOwner.ui64Timestamp,
+		                  psDevNode->sPowerLockOwner.pszFile, psDevNode->sPowerLockOwner.ui32LineNum);
+	}
+#endif
+
+	/* Driver debug table */
+	OSWRLockAcquireReadNested(psDriverDebugTable->hLock, DN_LOCKCLASS_DRIVER);
+	/* Device debug table*/
+	OSWRLockAcquireReadNested(psDebugTable->hLock, DN_LOCKCLASS_DEVICE);
+
+	/* For each requester in Driver and Device table */
+	for (i = 0; i < g_ui32DebugOrderTableReqCount; i++)
+	{
+		DLLIST_NODE *psNode;
+		DLLIST_NODE *psNext;
+
+		/* For each notifier on this requestor */
+		dllist_foreach_node(&psDriverDebugTable->asEntry[i].sListHead, psNode, psNext)
+		{
+			DEBUG_REQUEST_NOTIFY *psNotify =
+				IMG_CONTAINER_OF(psNode, DEBUG_REQUEST_NOTIFY, sListNode);
+			psNotify->pfnDbgRequestNotify(psNotify->hDbgRequestHandle, ui32VerbLevel,
+				                          pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+
+		/* For each notifier on this requestor */
+		dllist_foreach_node(&psDebugTable->asEntry[i].sListHead, psNode, psNext)
+		{
+			DEBUG_REQUEST_NOTIFY *psNotify =
+				IMG_CONTAINER_OF(psNode, DEBUG_REQUEST_NOTIFY, sListNode);
+			psNotify->pfnDbgRequestNotify(psNotify->hDbgRequestHandle, ui32VerbLevel,
+							pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+	}
+
+	OSWRLockReleaseRead(psDebugTable->hLock);
+	OSWRLockReleaseRead(psDriverDebugTable->hLock);
+
+	PVR_DUMPDEBUG_LOG("------------[ PVR DBG: END ]------------");
+
+	if (!pfnDumpDebugPrintf)
+	{
+		/* Only notify OS of an issue if the caller requested it,
+		 * using a NULL pointer in pfnDumpDebugPrintf. */
+		OSWarnOn(IMG_TRUE);
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_notifier.h b/drivers/gpu/drm/img-rogue/pvr_notifier.h
new file mode 100644
index 000000000000..b3757494e4b7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_notifier.h
@@ -0,0 +1,326 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR notifier interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVR_NOTIFIER_H)
+#define PVR_NOTIFIER_H
+
+#include "img_types.h"
+#include "pvr_debug.h"
+
+
+/*************************************************************************/ /*!
+Command Complete Notifier Interface
+*/ /**************************************************************************/
+
+typedef IMG_HANDLE PVRSRV_CMDCOMP_HANDLE;
+#ifndef CMDCOMPNOTIFY_PFN
+typedef void (*PFN_CMDCOMP_NOTIFY)(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle);
+#define CMDCOMPNOTIFY_PFN
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCmdCompleteInit
+@Description    Performs initialisation of the command complete notifier
+                interface.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVCmdCompleteInit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCmdCompleteDeinit
+@Description    Performs cleanup for the command complete notifier interface.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+void
+PVRSRVCmdCompleteDeinit(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterCmdCompleteNotify
+@Description    Register a callback function that is called when some device
+                finishes some work, which is signalled via a call to
+                PVRSRVCheckStatus.
+@Output         phNotify             On success, points to command complete
+                                     notifier handle
+@Input          pfnCmdCompleteNotify Function callback
+@Input          hPrivData            Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterCmdCompleteNotify(IMG_HANDLE *phNotify,
+                                PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify,
+                                PVRSRV_CMDCOMP_HANDLE hPrivData);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterCmdCompleteNotify
+@Description    Unregister a previously registered callback function.
+@Input          hNotify              Command complete notifier handle
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterCmdCompleteNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVCheckStatus
+@Description    Calls PVRSRVNotifyCommandCompletion() to notify registered
+                command complete handlers of work completion and then calls
+                PVRSRVSignalGlobalEO() to signal the driver wide event objects.
+@Input          hCmdCompCallerHandle Used to prevent a handler from being
+                                     notified. A NULL value results in all
+                                     handlers being notified.
+*/ /**************************************************************************/
+void
+PVRSRVCheckStatus(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVNotifyCommandCompletion
+@Description    Notify any registered command complete handlers that some work
+                has been finished (unless hCmdCompCallerHandle matches a
+                handler's hPrivData).
+@Input          hCmdCompCallerHandle Used to prevent a handler from being
+                                     notified. A NULL value results in all
+                                     handlers being notified.
+*/ /**************************************************************************/
+void
+PVRSRVNotifyCommandCompletion(PVRSRV_CMDCOMP_HANDLE hCmdCompCallerHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVSignalDriverWideEO
+@Description    Signals the driver wide event objects.
+*/ /**************************************************************************/
+void
+PVRSRVSignalDriverWideEO(void);
+
+
+/*************************************************************************/ /*!
+Debug Notifier Interface
+*/ /**************************************************************************/
+
+#define DEBUG_REQUEST_DC                0
+#define DEBUG_REQUEST_SYNCTRACKING      1
+#define DEBUG_REQUEST_SRV               2
+#define DEBUG_REQUEST_SYS               3
+#define DEBUG_REQUEST_RGX               4
+#define DEBUG_REQUEST_ANDROIDSYNC       5
+#define DEBUG_REQUEST_LINUXFENCE        6
+#define DEBUG_REQUEST_SYNCCHECKPOINT    7
+#define DEBUG_REQUEST_HTB               8
+#define DEBUG_REQUEST_APPHINT           9
+#define DEBUG_REQUEST_FALLBACKSYNC      10
+
+#define DEBUG_REQUEST_VERBOSITY_LOW     0
+#define DEBUG_REQUEST_VERBOSITY_MEDIUM  1
+#define DEBUG_REQUEST_VERBOSITY_HIGH    2
+#define DEBUG_REQUEST_VERBOSITY_MAX     DEBUG_REQUEST_VERBOSITY_HIGH
+
+#define DD_VERB_LVL_ENABLED(_verbLvl, _verbLvlChk) ((_verbLvl) >= (_verbLvlChk))
+
+/*
+ * Macro used within debug dump functions to send output either to PVR_LOG or
+ * a custom function. The custom function should be stored as a function
+ * pointer in a local variable called 'pfnDumpDebugPrintf'. 'pvDumpDebugFile'
+ * is also required as a local variable to serve as a file identifier for the
+ * printf function if required.
+ */
+#define PVR_DUMPDEBUG_LOG(...)                                \
+	do                                                        \
+	{                                                         \
+		if (pfnDumpDebugPrintf)                               \
+			pfnDumpDebugPrintf(pvDumpDebugFile, __VA_ARGS__); \
+		else                                                  \
+			PVR_LOG((__VA_ARGS__));                           \
+	} while (0)
+
+struct _PVRSRV_DEVICE_NODE_;
+
+typedef IMG_HANDLE PVRSRV_DBGREQ_HANDLE;
+#ifndef DBGNOTIFY_PFNS
+typedef void (DUMPDEBUG_PRINTF_FUNC)(void *pvDumpDebugFile,
+					const IMG_CHAR *pszFormat, ...);
+typedef void (*PFN_DBGREQ_NOTIFY)(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile);
+#define DBGNOTIFY_PFNS
+#endif
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDeviceDbgTable
+@Description    Registers a debug requester table for the given device. The
+                order in which the debug requester IDs appear in the
+                table determine the order in which a set of notifier callbacks
+                will be called. In other words, the requester ID that appears
+                first will have all of its associated debug notifier callbacks
+                called first. This will then be followed by all the callbacks
+                associated with the next requester ID in the table and so on.
+                The order table is handled internally.
+@Input          psDevNode     Device node to register requester table with
+@Return         PVRSRV_ERROR  PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgTable(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDriverDbgTable
+@Description    Registers a debug requester table for the driver. The
+                order in which the debug requester IDs appear in the
+                table determine the order in which a set of notifier callbacks
+                will be called. In other words, the requester ID that appears
+                first will have all of its associated debug notifier callbacks
+                called first. This will then be followed by all the callbacks
+                associated with the next requester ID in the table and so on.
+                The order table is handled internally.
+@Return         PVRSRV_ERROR  PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgTable(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDeviceDbgTable
+@Description    Unregisters a debug requester table.
+@Input          psDevNode     Device node for which the requester table should
+                              be unregistered
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVUnregisterDeviceDbgTable(struct _PVRSRV_DEVICE_NODE_ *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDriverDbgTable
+@Description    Unregisters the driver debug requester table.
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVUnregisterDriverDbgTable(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDeviceDbgRequestNotify
+@Description    Register a callback function on a given device that is called
+                when a debug request is made via a call PVRSRVDebugRequest.
+                There are a number of verbosity levels ranging from
+                DEBUG_REQUEST_VERBOSITY_LOW up to
+                DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+                for each level up to the highest level specified to
+                PVRSRVDebugRequest.
+@Output         phNotify             Points to debug notifier handle on success
+@Input          psDevNode            Device node for which the debug callback
+                                     should be registered
+@Input          pfnDbgRequestNotify  Function callback
+@Input          ui32RequesterID      Requester ID. This is used to determine
+                                     the order in which callbacks are called
+@Input          hDbgReqeustHandle    Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDeviceDbgRequestNotify(IMG_HANDLE *phNotify,
+                                     struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                     PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+                                     IMG_UINT32 ui32RequesterID,
+                                     PVRSRV_DBGREQ_HANDLE hDbgReqeustHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRegisterDriverDbgRequestNotify
+@Description    Register a callback function that is called when a debug request
+                is made via a call PVRSRVDebugRequest. There are a number of
+                verbosity levels ranging from DEBUG_REQUEST_VERBOSITY_LOW up to
+                DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+                for each level up to the highest level specified to
+                PVRSRVDebugRequest.
+@Output         phNotify             Points to debug notifier handle on success
+@Input          pfnDbgRequestNotify  Function callback
+@Input          ui32RequesterID      Requester ID. This is used to determine
+                                     the order in which callbacks are called
+@Input          hDbgReqeustHandle    Data to be passed back to the caller via
+                                     the callback function
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRegisterDriverDbgRequestNotify(IMG_HANDLE *phNotify,
+									 PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+									 IMG_UINT32 ui32RequesterID,
+									 PVRSRV_DBGREQ_HANDLE hDbgRequestHandle);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDeviceDbgRequestNotify
+@Description    Unregister a previously registered (device context) callback
+                function.
+@Input          hNotify              Debug notifier handle.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterDeviceDbgRequestNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVUnregisterDriverDbgRequestNotify
+@Description    Unregister a previously registered (driver context) callback
+                function.
+@Input          hNotify              Debug notifier handle.
+@Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVUnregisterDriverDbgRequestNotify(IMG_HANDLE hNotify);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDebugRequest
+@Description    Notify any registered debug request handlers that a debug
+                request has been made and at what level.
+@Input          psDevNode           Device node for which the debug request
+                                    has been made
+@Input          ui32VerbLevel       The maximum verbosity level to dump
+@Input          pfnDumpDebugPrintf  Used to specify the print function that
+                                    should be used to dump any debug
+                                    information. If this argument is NULL then
+                                    PVR_LOG() will be used as the default
+                                    print function.
+@Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the print function if required.
+@Return         void
+*/ /**************************************************************************/
+void
+PVRSRVDebugRequest(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                   IMG_UINT32 ui32VerbLevel,
+                   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                   void *pvDumpDebugFile);
+
+#endif /* !defined(PVR_NOTIFIER_H) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_platform_drv.c b/drivers/gpu/drm/img-rogue/pvr_platform_drv.c
new file mode 100644
index 000000000000..816383c28c63
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_platform_drv.c
@@ -0,0 +1,314 @@
+/*
+ * @File
+ * @Title       PowerVR DRM platform driver
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_print.h>
+#include <linux/mod_devicetable.h>
+#include <linux/dma-mapping.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "module_common.h"
+#include "pvr_drv.h"
+#include "pvrmodule.h"
+#include "sysinfo.h"
+
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+MODULE_IMPORT_NS("DMA_BUF");
+#else
+MODULE_IMPORT_NS(DMA_BUF);
+#endif
+
+static struct drm_driver pvr_drm_platform_driver;
+
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+static unsigned int pvr_num_devices = 1;
+static struct platform_device **pvr_devices;
+
+#if defined(NO_HARDWARE)
+static int pvr_num_devices_set(const char *val,
+			       const struct kernel_param *param)
+{
+	int err;
+
+	err = param_set_uint(val, param);
+	if (err)
+		return err;
+
+	if (pvr_num_devices == 0 || pvr_num_devices > PVRSRV_MAX_DEVICES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct kernel_param_ops pvr_num_devices_ops = {
+	.set = pvr_num_devices_set,
+	.get = param_get_uint,
+};
+
+#define STR(s) #s
+#define STRINGIFY(s) STR(s)
+
+module_param_cb(num_devices, &pvr_num_devices_ops, &pvr_num_devices, 0444);
+MODULE_PARM_DESC(num_devices,
+		 "Number of platform devices to register (default: 1 - max: "
+		 STRINGIFY(PVRSRV_MAX_DEVICES) ")");
+#endif /* defined(NO_HARDWARE) */
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+
+static int pvr_devices_register(void)
+{
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+	struct platform_device_info pvr_dev_info = {
+		.name = SYS_RGX_DEV_NAME,
+		.id = -2,
+#if defined(NO_HARDWARE)
+		/* Not all cores have 40 bit physical support, but this
+		 * will work unless > 32 bit address is returned on those cores.
+		 * In the future this will be fixed more correctly.
+		 */
+		.dma_mask = DMA_BIT_MASK(40),
+#else
+		.dma_mask = DMA_BIT_MASK(40),
+#endif
+	};
+	unsigned int i;
+
+	BUG_ON(pvr_num_devices == 0 || pvr_num_devices > PVRSRV_MAX_DEVICES);
+
+	pvr_devices = kmalloc_array(pvr_num_devices, sizeof(*pvr_devices),
+				    GFP_KERNEL);
+	if (!pvr_devices)
+		return -ENOMEM;
+
+	for (i = 0; i < pvr_num_devices; i++) {
+		pvr_devices[i] = platform_device_register_full(&pvr_dev_info);
+		if (IS_ERR(pvr_devices[i])) {
+			DRM_ERROR("unable to register device %u (err=%ld)\n",
+				  i, PTR_ERR(pvr_devices[i]));
+			pvr_devices[i] = NULL;
+			return -ENODEV;
+		}
+	}
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+
+	return 0;
+}
+
+static void pvr_devices_unregister(void)
+{
+#if defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED)
+	unsigned int i;
+
+	BUG_ON(!pvr_devices);
+
+	for (i = 0; i < pvr_num_devices && pvr_devices[i]; i++)
+		platform_device_unregister(pvr_devices[i]);
+
+	kfree(pvr_devices);
+	pvr_devices = NULL;
+#endif /* defined(MODULE) && !defined(PVR_LDM_PLATFORM_PRE_REGISTERED) */
+}
+
+static int pvr_probe(struct platform_device *pdev)
+{
+	struct drm_device *ddev;
+	int ret;
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+	pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
+	pdev->dev.coherent_dma_mask = DMA_BIT_MASK(40);
+	ddev = drm_dev_alloc(&pvr_drm_platform_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+
+	/*
+	 * The load callback, called from drm_dev_register, is deprecated,
+	 * because of potential race conditions. Calling the function here,
+	 * before calling drm_dev_register, avoids those potential races.
+	 */
+	BUG_ON(pvr_drm_platform_driver.load != NULL);
+	ret = pvr_drm_load(ddev, 0);
+	if (ret)
+		goto err_drm_dev_put;
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret)
+		goto err_drm_dev_unload;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+		pvr_drm_platform_driver.name,
+		pvr_drm_platform_driver.major,
+		pvr_drm_platform_driver.minor,
+		pvr_drm_platform_driver.patchlevel,
+		pvr_drm_platform_driver.date,
+		ddev->primary->index);
+#endif
+	return 0;
+
+err_drm_dev_unload:
+	pvr_drm_unload(ddev);
+err_drm_dev_put:
+	drm_dev_put(ddev);
+	return	ret;
+}
+
+static void pvr_remove(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	drm_dev_unregister(ddev);
+
+	/* The unload callback, called from drm_dev_unregister, is
+	 * deprecated. Call the unload function directly.
+	 */
+	BUG_ON(pvr_drm_platform_driver.unload != NULL);
+	pvr_drm_unload(ddev);
+
+	drm_dev_put(ddev);
+}
+
+static void pvr_shutdown(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	DRM_DEBUG_DRIVER("device %p\n", &pdev->dev);
+
+	PVRSRVDeviceShutdown(ddev);
+}
+
+static const struct of_device_id pvr_of_ids[] = {
+#if defined(SYS_RGX_OF_COMPATIBLE)
+	{ .compatible = SYS_RGX_OF_COMPATIBLE, },
+#endif
+	{},
+};
+
+#if !defined(CHROMIUMOS_KERNEL)
+MODULE_DEVICE_TABLE(of, pvr_of_ids);
+#endif
+
+static struct platform_device_id pvr_platform_ids[] = {
+#if defined(SYS_RGX_DEV_NAME)
+	{ SYS_RGX_DEV_NAME, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_0)
+	{ SYS_RGX_DEV_NAME_0, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_1)
+	{ SYS_RGX_DEV_NAME_1, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_2)
+	{ SYS_RGX_DEV_NAME_2, 0 },
+#endif
+#if defined(SYS_RGX_DEV_NAME_3)
+	{ SYS_RGX_DEV_NAME_3, 0 },
+#endif
+	{ }
+};
+
+#if !defined(CHROMIUMOS_KERNEL)
+MODULE_DEVICE_TABLE(platform, pvr_platform_ids);
+#endif
+
+static struct platform_driver pvr_platform_driver = {
+	.driver = {
+		.name		= DRVNAME,
+		.of_match_table	= of_match_ptr(pvr_of_ids),
+		.pm		= &pvr_pm_ops,
+	},
+	.id_table		= pvr_platform_ids,
+	.probe			= pvr_probe,
+	.remove			= pvr_remove,
+	.shutdown		= pvr_shutdown,
+};
+
+static int __init pvr_init(void)
+{
+	int err;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	pvr_drm_platform_driver = pvr_drm_generic_driver;
+
+	err = PVRSRVDriverInit();
+	if (err)
+		return err;
+
+	err = platform_driver_register(&pvr_platform_driver);
+	if (err)
+		return err;
+
+	return pvr_devices_register();
+}
+
+static void __exit pvr_exit(void)
+{
+	DRM_DEBUG_DRIVER("\n");
+
+	pvr_devices_unregister();
+	platform_driver_unregister(&pvr_platform_driver);
+	PVRSRVDriverDeinit();
+
+	DRM_DEBUG_DRIVER("done\n");
+}
+
+late_initcall(pvr_init);
+module_exit(pvr_exit);
diff --git a/drivers/gpu/drm/img-rogue/pvr_procfs.h b/drivers/gpu/drm/img-rogue/pvr_procfs.h
new file mode 100644
index 000000000000..61a1f0ee28d3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_procfs.h
@@ -0,0 +1,50 @@
+/*************************************************************************/ /*!
+@File
+@Title          ProcFS implementation of Debug Info interface.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVR_PROCFS_H
+#define PVR_PROCFS_H
+
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR PVRProcFsRegister(void);
+
+#endif /* PVR_PROCFS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_ricommon.h b/drivers/gpu/drm/img-rogue/pvr_ricommon.h
new file mode 100644
index 000000000000..0521aa151794
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_ricommon.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Resource Information (RI) common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) common types and definitions included
+                in both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_RICOMMON_H
+#define PVR_RICOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+/*! Maximum text string length including the null byte */
+#define PRVSRVRI_MAX_TEXT_LENGTH	20U
+
+/* PID used to hold PMR allocations which are driver-wide (i.e. have a lifetime
+ * longer than an application process)
+ */
+#define PVR_SYS_ALLOC_PID 1
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_RICOMMON_H */
+/******************************************************************************
+ End of file (pvr_ricommon.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pvr_sw_fence.c b/drivers/gpu/drm/img-rogue/pvr_sw_fence.c
new file mode 100644
index 000000000000..aef083a29901
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sw_fence.c
@@ -0,0 +1,204 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/spinlock_types.h>
+#include <linux/atomic.h>
+#include <linux/slab.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+
+#include "pvr_sw_fence.h"
+#include "osfunc_common.h"
+
+struct pvr_sw_fence_context {
+	struct kref kref;
+	unsigned int context;
+	char context_name[32];
+	char driver_name[32];
+	atomic_t seqno;
+	atomic_t fence_count;
+};
+
+struct pvr_sw_fence {
+	struct dma_fence base;
+	struct pvr_sw_fence_context *fence_context;
+	spinlock_t lock;
+};
+
+#define to_pvr_sw_fence(fence) container_of(fence, struct pvr_sw_fence, base)
+
+const char *pvr_sw_fence_context_name(struct pvr_sw_fence_context *fctx)
+{
+	return fctx->context_name;
+}
+
+void pvr_sw_fence_context_value_str(struct pvr_sw_fence_context *fctx,
+				    char *str, int size)
+{
+	snprintf(str, size, "%d", atomic_read(&fctx->seqno));
+}
+
+static inline unsigned
+pvr_sw_fence_context_seqno_next(struct pvr_sw_fence_context *fence_context)
+{
+	return atomic_inc_return(&fence_context->seqno) - 1;
+}
+
+static const char *pvr_sw_fence_get_driver_name(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	return pvr_sw_fence->fence_context->driver_name;
+}
+
+static const char *pvr_sw_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	return pvr_sw_fence_context_name(pvr_sw_fence->fence_context);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+static void pvr_sw_fence_value_str(struct dma_fence *fence, char *str, int size)
+{
+	snprintf(str, size, "%llu", (u64) fence->seqno);
+}
+
+static void pvr_sw_fence_timeline_value_str(struct dma_fence *fence,
+					    char *str, int size)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	pvr_sw_fence_context_value_str(pvr_sw_fence->fence_context, str, size);
+}
+#endif
+
+static bool pvr_sw_fence_enable_signaling(struct dma_fence *fence)
+{
+	return true;
+}
+
+static void pvr_sw_fence_context_destroy_kref(struct kref *kref)
+{
+	struct pvr_sw_fence_context *fence_context =
+		container_of(kref, struct pvr_sw_fence_context, kref);
+	unsigned int fence_count;
+
+	fence_count = atomic_read(&fence_context->fence_count);
+	if (WARN_ON(fence_count))
+		pr_debug("%s context has %u fence(s) remaining\n",
+			 fence_context->context_name, fence_count);
+
+	kfree(fence_context);
+}
+
+static void pvr_sw_fence_release(struct dma_fence *fence)
+{
+	struct pvr_sw_fence *pvr_sw_fence = to_pvr_sw_fence(fence);
+
+	atomic_dec(&pvr_sw_fence->fence_context->fence_count);
+	kref_put(&pvr_sw_fence->fence_context->kref,
+		pvr_sw_fence_context_destroy_kref);
+	kfree(pvr_sw_fence);
+}
+
+static const struct dma_fence_ops pvr_sw_fence_ops = {
+	.get_driver_name = pvr_sw_fence_get_driver_name,
+	.get_timeline_name = pvr_sw_fence_get_timeline_name,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+	.fence_value_str = pvr_sw_fence_value_str,
+	.timeline_value_str = pvr_sw_fence_timeline_value_str,
+#endif
+	.enable_signaling = pvr_sw_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+	.release = pvr_sw_fence_release,
+};
+
+struct pvr_sw_fence_context *
+pvr_sw_fence_context_create(const char *context_name, const char *driver_name)
+{
+	struct pvr_sw_fence_context *fence_context;
+
+	fence_context = kmalloc(sizeof(*fence_context), GFP_KERNEL);
+	if (!fence_context)
+		return NULL;
+
+	fence_context->context = dma_fence_context_alloc(1);
+	OSStringSafeCopy(fence_context->context_name, context_name,
+		sizeof(fence_context->context_name));
+	OSStringSafeCopy(fence_context->driver_name, driver_name,
+		sizeof(fence_context->driver_name));
+	atomic_set(&fence_context->seqno, 0);
+	atomic_set(&fence_context->fence_count, 0);
+	kref_init(&fence_context->kref);
+
+	return fence_context;
+}
+
+void pvr_sw_fence_context_destroy(struct pvr_sw_fence_context *fence_context)
+{
+	kref_put(&fence_context->kref, pvr_sw_fence_context_destroy_kref);
+}
+
+struct dma_fence *
+pvr_sw_fence_create(struct pvr_sw_fence_context *fence_context)
+{
+	struct pvr_sw_fence *pvr_sw_fence;
+	unsigned int seqno;
+
+	pvr_sw_fence = kmalloc(sizeof(*pvr_sw_fence), GFP_KERNEL);
+	if (!pvr_sw_fence)
+		return NULL;
+
+	spin_lock_init(&pvr_sw_fence->lock);
+	pvr_sw_fence->fence_context = fence_context;
+
+	seqno = pvr_sw_fence_context_seqno_next(fence_context);
+	dma_fence_init(&pvr_sw_fence->base, &pvr_sw_fence_ops,
+		       &pvr_sw_fence->lock, fence_context->context, seqno);
+
+	atomic_inc(&fence_context->fence_count);
+	kref_get(&fence_context->kref);
+
+	return &pvr_sw_fence->base;
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_sw_fence.h b/drivers/gpu/drm/img-rogue/pvr_sw_fence.h
new file mode 100644
index 000000000000..bebbcb75e23b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sw_fence.h
@@ -0,0 +1,60 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_SW_FENCES_H__)
+#define __PVR_SW_FENCES_H__
+
+#include "pvr_linux_fence.h"
+
+struct pvr_sw_fence_context;
+
+struct pvr_sw_fence_context *pvr_sw_fence_context_create(const char *name,
+				const char *driver_name);
+void pvr_sw_fence_context_destroy(struct pvr_sw_fence_context *fence_context);
+struct dma_fence *pvr_sw_fence_create(struct pvr_sw_fence_context *
+				      fence_context);
+
+const char *pvr_sw_fence_context_name(struct pvr_sw_fence_context *fctx);
+void pvr_sw_fence_context_value_str(struct pvr_sw_fence_context *fctx,
+				    char *str, int size);
+
+#endif /* !defined(__PVR_SW_FENCES_H__) */
diff --git a/drivers/gpu/drm/img-rogue/pvr_sync.h b/drivers/gpu/drm/img-rogue/pvr_sync.h
new file mode 100644
index 000000000000..a8ecd8b2caa0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sync.h
@@ -0,0 +1,120 @@
+/*
+ * @File        pvr_sync.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_H
+#define _PVR_SYNC_H
+
+#include <linux/device.h>
+
+#include "pvr_fd_sync_kernel.h"
+#include "services_kernel_client.h"
+
+
+/* Services internal interface */
+
+/**
+ * pvr_sync_register_functions()
+ *
+ * Return: PVRSRV_OK on success.
+ */
+enum PVRSRV_ERROR_TAG pvr_sync_register_functions(void);
+
+/**
+ *  pvr_sync_init - register the pvr_sync misc device
+ *
+ *  Return: error code, 0 on success.
+ */
+int pvr_sync_init(void);
+
+/**
+ * pvr_sync_deinit - unregister the pvr_sync misc device
+ */
+void pvr_sync_deinit(void);
+
+/**
+ * pvr_sync_device_init() - create an internal sync context
+ * @dev: Linux device
+ *
+ * Return: PVRSRV_OK on success.
+ */
+enum PVRSRV_ERROR_TAG pvr_sync_device_init(struct device *dev);
+
+/**
+ * pvr_sync_device_deinit() - destroy an internal sync context
+ *
+ * Drains any work items with outstanding sync fence updates/dependencies.
+ */
+void pvr_sync_device_deinit(struct device *dev);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_wait(void *fence, u32 timeout_in_ms);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_release(void *fence);
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_get(int fence_fd, void **fence_out);
+
+enum PVRSRV_ERROR_TAG
+pvr_sync_sw_timeline_fence_create(struct _PVRSRV_DEVICE_NODE_ *pvrsrv_dev_node,
+				  int timeline_fd,
+				  const char *fence_name,
+				  int *fence_fd_out,
+				  u64 *sync_pt_idx);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_advance(void *timeline,
+					       u64 *sync_pt_idx);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_release(void *timeline);
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_get(int timeline_fd,
+					   void **timeline_out);
+
+enum PVRSRV_ERROR_TAG
+sync_dump_fence(void *sw_fence_obj,
+		DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		void *dump_debug_file);
+
+enum PVRSRV_ERROR_TAG
+sync_sw_dump_timeline(void *sw_timeline_obj,
+		      DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		      void *dump_debug_file);
+
+#endif /* _PVR_SYNC_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_sync_api.h b/drivers/gpu/drm/img-rogue/pvr_sync_api.h
new file mode 100644
index 000000000000..97dd746439b9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sync_api.h
@@ -0,0 +1,65 @@
+/*
+ * @File        pvr_sync_api.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_API_H
+#define _PVR_SYNC_API_H
+
+#include "img_types.h"
+
+int pvr_sync_api_init(void *file_handle, void **api_priv);
+int pvr_sync_api_deinit(void *api_priv, bool is_sw);
+int pvr_sync_api_rename(void *api_priv, void *user_data);
+int pvr_sync_api_force_sw_only(void *api_priv, void **api_priv_new);
+int pvr_sync_api_sw_create_fence(void *api_priv, void *user_data);
+int pvr_sync_api_sw_inc(void *api_priv, void *user_data);
+int pvr_sync_api_force_exp_only(void *api_priv, void *api_data);
+int pvr_sync_api_create_export_fence(void *api_priv, void *user_data);
+
+struct file;
+
+int pvr_sync_ioctl_init(void);
+void pvr_sync_ioctl_deinit(void);
+void *pvr_sync_get_api_priv(struct file *file);
+struct file *pvr_sync_get_file_struct(void *file_handle);
+
+#endif /* _PVR_SYNC_API_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_sync_file.c b/drivers/gpu/drm/img-rogue/pvr_sync_file.c
new file mode 100644
index 000000000000..4ec81bf335cc
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sync_file.c
@@ -0,0 +1,1317 @@
+/*
+ * @File        pvr_sync_file.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "services_kernel_client.h"
+#include "pvr_drv.h"
+#include "pvr_sync.h"
+#include "pvr_fence.h"
+#include "pvr_counting_timeline.h"
+#include "pvr_export_fence.h"
+
+#include "linux_sw_sync.h"
+
+#include <linux/version.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/sync_file.h>
+#include <linux/file.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+
+#include "pvr_sync_api.h"
+#include "osfunc_common.h"
+
+/* This header must always be included last */
+#include "kernel_compatibility.h"
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)) && !defined(CHROMIUMOS_KERNEL)
+#define sync_file_user_name(s)	((s)->name)
+#else
+#define sync_file_user_name(s)	((s)->user_name)
+#endif
+
+#define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
+	do {                                                             \
+		if (pfnDumpDebugPrintf)                                  \
+			pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
+					   ## __VA_ARGS__);              \
+		else                                                     \
+			pr_err(fmt "\n", ## __VA_ARGS__);                \
+	} while (0)
+
+#define	FILE_NAME "pvr_sync_file"
+
+struct sw_sync_create_fence_data {
+	__u32 value;
+	char name[32];
+	__s32 fence;
+};
+#define SW_SYNC_IOC_MAGIC 'W'
+#define SW_SYNC_IOC_CREATE_FENCE \
+	(_IOWR(SW_SYNC_IOC_MAGIC, 0, struct sw_sync_create_fence_data))
+#define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
+
+/* Global data for the sync driver */
+static struct {
+	struct pvr_fence_context *foreign_fence_context;
+	PFN_SYNC_CHECKPOINT_STRUCT sync_checkpoint_ops;
+} pvr_sync_data;
+
+#if defined(NO_HARDWARE)
+static DEFINE_MUTEX(pvr_timeline_active_list_lock);
+static struct list_head pvr_timeline_active_list;
+#endif
+
+/* This is the actual timeline metadata. We might keep this around after the
+ * base sync driver has destroyed the pvr_sync_timeline_wrapper object.
+ */
+struct pvr_sync_timeline {
+	char name[32];
+	void *file_handle;
+	bool is_sw;
+	bool is_export;
+	/* Fence context used for hw fences */
+	struct pvr_fence_context *hw_fence_context;
+	/* Timeline and context for sw fences */
+	union {
+		struct pvr_counting_fence_timeline *sw_fence_timeline;
+		struct pvr_exp_fence_context *exp_fence_context;
+	};
+#if defined(NO_HARDWARE)
+	/* List of all timelines (used to advance all timelines in nohw builds) */
+	struct list_head list;
+#endif
+};
+
+static
+void pvr_sync_free_checkpoint_list_mem(void *mem_ptr)
+{
+	kfree(mem_ptr);
+}
+
+#if defined(NO_HARDWARE)
+/* function used to signal pvr fence in nohw builds */
+static
+void pvr_sync_nohw_signal_fence(void *fence_data_to_signal)
+{
+	struct pvr_sync_timeline *this_timeline;
+
+	mutex_lock(&pvr_timeline_active_list_lock);
+	list_for_each_entry(this_timeline, &pvr_timeline_active_list, list) {
+		pvr_fence_context_signal_fences_nohw(this_timeline->hw_fence_context);
+	}
+	mutex_unlock(&pvr_timeline_active_list_lock);
+}
+static
+void pvr_sync_nohw_signal_exp_fence(PVRSRV_FENCE fence_to_signal)
+{
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_signal);
+	if (fence && pvr_is_exp_fence(fence))
+		dma_fence_signal(fence);
+
+	dma_fence_put(fence);
+}
+#endif
+
+static struct pvr_sync_timeline *pvr_sync_timeline_fget(int fd)
+{
+	struct file *file = fget(fd);
+	struct pvr_sync_timeline *timeline;
+
+	if (!file)
+		return NULL;
+
+	timeline = pvr_sync_get_api_priv(file);
+	if (!timeline)
+		fput(file);
+
+	return timeline;
+}
+
+static void pvr_sync_timeline_fput(struct pvr_sync_timeline *timeline)
+{
+	struct file *file = pvr_sync_get_file_struct(timeline->file_handle);
+
+	if (file)
+		fput(file);
+	else
+		pr_err(FILE_NAME ": %s: Timeline incomplete\n", __func__);
+}
+
+/* ioctl and fops handling */
+
+int pvr_sync_api_init(void *file_handle, void **api_priv)
+{
+	struct pvr_sync_timeline *timeline;
+	char task_comm[TASK_COMM_LEN];
+
+	get_task_comm(task_comm, current);
+
+	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
+	if (!timeline)
+		return -ENOMEM;
+
+	OSStringSafeCopy(timeline->name, task_comm, sizeof(timeline->name));
+	timeline->file_handle = file_handle;
+	timeline->is_sw = false;
+	timeline->is_export = false;
+
+	*api_priv = (void *)timeline;
+
+	return 0;
+}
+
+int pvr_sync_api_deinit(void *api_priv, bool is_sw)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	if (!timeline)
+		return 0;
+
+	if (timeline->is_sw && timeline->sw_fence_timeline) {
+		/* This makes sure any outstanding SW syncs are marked as
+		 * complete at timeline close time. Otherwise it'll leak the
+		 * timeline (as outstanding fences hold a ref) and possibly
+		 * wedge the system if something is waiting on one of those
+		 * fences
+		 */
+		pvr_counting_fence_timeline_force_complete(
+			timeline->sw_fence_timeline);
+		pvr_counting_fence_timeline_put(timeline->sw_fence_timeline);
+	} else if (timeline->is_export && timeline->exp_fence_context) {
+		pvr_exp_fence_context_destroy(timeline->exp_fence_context);
+	}
+
+	if (timeline->hw_fence_context) {
+#if defined(NO_HARDWARE)
+		mutex_lock(&pvr_timeline_active_list_lock);
+		list_del(&timeline->list);
+		mutex_unlock(&pvr_timeline_active_list_lock);
+#endif
+		pvr_fence_context_destroy(timeline->hw_fence_context);
+	}
+
+	kfree(timeline);
+
+	return 0;
+}
+
+/*
+ * This is the function that kick code will call in order to 'finalise' a
+ * created output fence just prior to returning from the kick function.
+ * The OS native sync code needs to implement a function meeting this
+ * specification - the implementation may be a nop if the OS does not need
+ * to perform any actions at this point.
+ *
+ * Input: fence_fd            The PVRSRV_FENCE to be 'finalised'. This value
+ *                            will have been returned by an earlier call to
+ *                            pvr_sync_create_fence().
+ * Input: finalise_data       The finalise data returned by an earlier call
+ *                            to pvr_sync_create_fence().
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_finalise_fence(PVRSRV_FENCE fence_fd, void *finalise_data)
+{
+	struct sync_file *sync_file = finalise_data;
+	struct pvr_fence *pvr_fence;
+
+	if (!sync_file || (fence_fd < 0)) {
+		pr_err(FILE_NAME ": %s: Invalid input fence\n", __func__);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pvr_fence = to_pvr_fence(sync_file->fence);
+
+	if (!pvr_fence) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr fence\n", __func__);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* pvr fences can be signalled any time after creation */
+	dma_fence_enable_sw_signaling(&pvr_fence->base);
+
+	fd_install(fence_fd, sync_file->file);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain a new
+ * PVRSRV_FENCE from the OS native sync code and the PSYNC_CHECKPOINT used
+ * in that fence. The OS native sync code needs to implement a function
+ * meeting this specification.
+ *
+ * Input: device                   Device node to use in creating a hw_fence_ctx
+ * Input: fence_name               A string to annotate the fence with (for
+ *                                 debug).
+ * Input: timeline                 The timeline on which the new fence is to be
+ *                                 created.
+ * Output: new_fence               The new PVRSRV_FENCE to be returned by the
+ *                                 kick call.
+ * Output: fence_uid               Unique ID of the update fence.
+ * Output: fence_finalise_data     Pointer to data needed to finalise the fence.
+ * Output: new_checkpoint_handle   The PSYNC_CHECKPOINT used by the new fence.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_create_fence(
+		      struct _PVRSRV_DEVICE_NODE_ *device,
+		      const char *fence_name,
+		      PVRSRV_TIMELINE new_fence_timeline,
+		      PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		      PVRSRV_FENCE *new_fence, u64 *fence_uid,
+		      void **fence_finalise_data,
+		      PSYNC_CHECKPOINT *new_checkpoint_handle,
+		      void **timeline_update_sync,
+		      __u32 *timeline_update_value)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	PVRSRV_FENCE new_fence_fd = -1;
+	struct pvr_sync_timeline *timeline;
+	struct pvr_fence *pvr_fence;
+	PSYNC_CHECKPOINT checkpoint;
+	struct sync_file *sync_file;
+
+	if (new_fence_timeline < 0 || !new_fence || !new_checkpoint_handle
+		|| !fence_finalise_data) {
+		pr_err(FILE_NAME ": %s: Invalid input params\n", __func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	/* We reserve the new fence FD before taking any operations
+	 * as we do not want to fail (e.g. run out of FDs)
+	 */
+	new_fence_fd = get_unused_fd_flags(O_CLOEXEC);
+	if (new_fence_fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to get fd\n", __func__);
+		err = PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+		goto err_out;
+	}
+
+	timeline = pvr_sync_timeline_fget(new_fence_timeline);
+	if (!timeline) {
+		pr_err(FILE_NAME ": %s: Failed to open supplied timeline fd (%d)\n",
+			__func__, new_fence_timeline);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_fd;
+	}
+
+	if (timeline->is_sw) {
+		/* This should never happen! */
+		pr_err(FILE_NAME ": %s: Request to create a pvr fence on sw timeline (%d)\n",
+			__func__, new_fence_timeline);
+		err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_timeline;
+	}
+
+	if (!timeline->hw_fence_context) {
+		/* First time we use this timeline, so create a context. */
+		timeline->hw_fence_context =
+			pvr_fence_context_create(
+				device,
+				NativeSyncGetFenceStatusWq(),
+				timeline->name);
+		if (!timeline->hw_fence_context) {
+			pr_err(FILE_NAME ": %s: Failed to create fence context (%d)\n",
+			       __func__, new_fence_timeline);
+			err = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto err_put_timeline;
+		}
+#if defined(NO_HARDWARE)
+		/* Add timeline to active list */
+		INIT_LIST_HEAD(&timeline->list);
+		mutex_lock(&pvr_timeline_active_list_lock);
+		list_add_tail(&timeline->list, &pvr_timeline_active_list);
+		mutex_unlock(&pvr_timeline_active_list_lock);
+#endif
+	}
+
+	pvr_fence = pvr_fence_create(timeline->hw_fence_context,
+								 psSyncCheckpointContext,
+								 new_fence_timeline,
+								 fence_name);
+	if (!pvr_fence) {
+		pr_err(FILE_NAME ": %s: Failed to create new pvr_fence\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_timeline;
+	}
+
+	checkpoint = pvr_fence_get_checkpoint(pvr_fence);
+	if (!checkpoint) {
+		pr_err(FILE_NAME ": %s: Failed to get fence checkpoint\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_destroy_fence;
+	}
+
+	sync_file = sync_file_create(&pvr_fence->base);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create sync_file\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_destroy_fence;
+	}
+	OSStringSafeCopy(sync_file_user_name(sync_file),
+		pvr_fence->name,
+		sizeof(sync_file_user_name(sync_file)));
+	dma_fence_put(&pvr_fence->base);
+
+	*new_fence = new_fence_fd;
+	*fence_finalise_data = sync_file;
+	*new_checkpoint_handle = checkpoint;
+	*fence_uid = OSGetCurrentClientProcessIDKM();
+	*fence_uid = (*fence_uid << 32) | (new_fence_fd & U32_MAX);
+	/* not used but don't want to return dangling pointers */
+	*timeline_update_sync = NULL;
+	*timeline_update_value = 0;
+
+	pvr_sync_timeline_fput(timeline);
+err_out:
+	return err;
+
+err_destroy_fence:
+	pvr_fence_destroy(pvr_fence);
+err_put_timeline:
+	pvr_sync_timeline_fput(timeline);
+err_put_fd:
+	put_unused_fd(new_fence_fd);
+	*fence_uid = PVRSRV_NO_FENCE;
+	goto err_out;
+}
+
+/*
+ * This is the function that kick code will call in order to 'rollback' a
+ * created output fence should an error occur when submitting the kick.
+ * The OS native sync code needs to implement a function meeting this
+ * specification.
+ *
+ * Input: fence_to_rollback The PVRSRV_FENCE to be 'rolled back'. The fence
+ *                          should be destroyed and any actions taken due to
+ *                          its creation that need to be undone should be
+ *                          reverted.
+ * Input: finalise_data     The finalise data for the fence to be 'rolled back'.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_rollback_fence_data(PVRSRV_FENCE fence_to_rollback,
+			     void *fence_data_to_rollback)
+{
+	struct sync_file *sync_file = fence_data_to_rollback;
+	struct pvr_fence *pvr_fence;
+
+	if (!sync_file || fence_to_rollback < 0) {
+		pr_err(FILE_NAME ": %s: Invalid fence (%d)\n", __func__,
+			fence_to_rollback);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pvr_fence = to_pvr_fence(sync_file->fence);
+	if (!pvr_fence) {
+		pr_err(FILE_NAME
+			": %s: Non-PVR fence (%p)\n",
+			__func__, sync_file->fence);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	fput(sync_file->file);
+
+	put_unused_fd(fence_to_rollback);
+
+	return PVRSRV_OK;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain a list of
+ * the PSYNC_CHECKPOINTs for a given PVRSRV_FENCE passed to a kick function.
+ * The OS native sync code will allocate the memory to hold the returned list
+ * of PSYNC_CHECKPOINT ptrs. The caller will free this memory once it has
+ * finished referencing it.
+ *
+ * Input: fence                     The input (check) fence
+ * Output: nr_checkpoints           The number of PVRSRV_SYNC_CHECKPOINT ptrs
+ *                                  returned in the checkpoint_handles
+ *                                  parameter.
+ * Output: fence_uid                Unique ID of the check fence
+ * Input/Output: checkpoint_handles The returned list of PVRSRV_SYNC_CHECKPOINTs.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_resolve_fence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		       PVRSRV_FENCE fence_to_resolve, u32 *nr_checkpoints,
+		       PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid)
+{
+	PSYNC_CHECKPOINT *checkpoints = NULL;
+	unsigned int i, num_fences = 0, num_used_fences = 0;
+	struct dma_fence **fences = NULL;
+	struct dma_fence *fence;
+	PVRSRV_ERROR err = PVRSRV_OK;
+
+	if (!nr_checkpoints || !checkpoint_handles || !fence_uid) {
+		pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
+			__func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	*nr_checkpoints = 0;
+	*checkpoint_handles = NULL;
+	*fence_uid = 0;
+
+	if (fence_to_resolve < 0)
+		goto err_out;
+
+	fence = sync_file_get_fence(fence_to_resolve);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_resolve);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
+
+		if (array) {
+			fences = array->fences;
+			num_fences = array->num_fences;
+		}
+	} else {
+		fences = &fence;
+		num_fences = 1;
+	}
+
+	checkpoints = kmalloc_array(num_fences, sizeof(PSYNC_CHECKPOINT),
+			      GFP_KERNEL);
+	if (!checkpoints) {
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fence;
+	}
+	for (i = 0; i < num_fences; i++) {
+		/*
+		 * Only return the checkpoint if the fence is still active.
+		 * Don't checked for signalled on PDUMP drivers as we need
+		 * to make sure that all fences make it to the pdump.
+		 */
+#if !defined(PDUMP)
+		if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+			      &fences[i]->flags))
+#endif
+		{
+			struct pvr_fence *pvr_fence;
+
+			/* Check if fences[i] is an export fence */
+			if (pvr_is_exp_fence(fences[i])) {
+				/* Assign export fence a sync checkpoint if it does not already
+				 * have one.
+				 */
+				err = pvr_exp_fence_assign_checkpoint(PVRSRV_NO_FENCE,
+				                                      fences[i],
+				                                      psSyncCheckpointContext,
+				                                      &checkpoints[num_used_fences]);
+				SyncCheckpointTakeRef(checkpoints[num_used_fences]);
+				++num_used_fences;
+			}
+			else {
+				pvr_fence = pvr_fence_create_from_fence(
+				            pvr_sync_data.foreign_fence_context,
+			            psSyncCheckpointContext,
+			            fences[i],
+			            fence_to_resolve,
+			            "foreign");
+				if (!pvr_fence) {
+					pr_err(FILE_NAME ": %s: Failed to create fence\n",
+					       __func__);
+					err = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto err_free_checkpoints;
+				}
+				checkpoints[num_used_fences] =
+				        pvr_fence_get_checkpoint(pvr_fence);
+				SyncCheckpointTakeRef(checkpoints[num_used_fences]);
+				++num_used_fences;
+				dma_fence_put(&pvr_fence->base);
+			}
+		}
+	}
+	/* If we don't return any checkpoints, delete the array because
+	 * the caller will not.
+	 */
+	if (num_used_fences == 0) {
+		kfree(checkpoints);
+		checkpoints = NULL;
+	}
+
+	*checkpoint_handles = checkpoints;
+	*nr_checkpoints = num_used_fences;
+	*fence_uid = OSGetCurrentClientProcessIDKM();
+	*fence_uid = (*fence_uid << 32) | (fence_to_resolve & U32_MAX);
+
+err_put_fence:
+	dma_fence_put(fence);
+err_out:
+	return err;
+
+err_free_checkpoints:
+	for (i = 0; i < num_used_fences; i++) {
+		if (checkpoints[i])
+			SyncCheckpointDropRef(checkpoints[i]);
+	}
+	kfree(checkpoints);
+	goto err_put_fence;
+}
+
+/*
+ * This is the function that kick code will call in order to obtain the
+ * PSYNC_CHECKPOINT for a given export fence (PVRSRV_FENCE) passed to a kick
+ * function.
+ * If export_fence is not a valid export fence, (ie fops != &pvr_exp_fence_ops)
+ * this function will return (error).
+ *
+ * Input:  export_fence              The export fence to resolve
+ * Input:  checkpoint_context        The context in which to create the new
+ *                                   sync checkpoint for the export fence
+ * Output: checkpoint_handle         The returned PVRSRV_SYNC_CHECKPOINT.
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_resolve_export_fence(PVRSRV_FENCE fence_to_resolve,
+			      PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+			      PSYNC_CHECKPOINT *checkpoint_handle)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_resolve);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_resolve);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (!pvr_is_exp_fence(fence)) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr export fence\n", __func__);
+		dma_fence_put(fence);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	err = pvr_exp_fence_assign_checkpoint(fence_to_resolve,
+					      fence,
+					      checkpoint_context,
+					      checkpoint_handle);
+	if (err != PVRSRV_OK) {
+		pr_err(FILE_NAME ": %s: Failed to assign export fence a sync checkpoint\n",
+		       __func__);
+	}
+
+	dma_fence_put(fence);
+
+err_out:
+	return err;
+}
+
+/*
+ * This is the function that kick code will call in order to rollback the
+ * PSYNC_CHECKPOINT assigned to an export fence (PVRSRV_FENCE) passed to a kick
+ * function.
+ * If export_fence is not a valid export fence, (ie fops != &pvr_exp_fence_ops)
+ * this function will return (error).
+ *
+ * Input:  fence_to_rollback              The export fence to rollback
+ */
+static enum PVRSRV_ERROR_TAG
+pvr_sync_rollback_export_fence(PVRSRV_FENCE fence_to_rollback)
+{
+	PVRSRV_ERROR err = PVRSRV_OK;
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_to_rollback);
+	if (!fence) {
+		pr_err("%s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_rollback);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (!pvr_is_exp_fence(fence)) {
+		pr_err(FILE_NAME ": %s: Fence not a pvr export fence\n", __func__);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	err = pvr_exp_fence_rollback(fence);
+	if (err != PVRSRV_OK) {
+		pr_err("%s: Failed to rollback export fence\n",
+		       __func__);
+	}
+
+	dma_fence_put(fence);
+
+err_out:
+	return err;
+}
+
+/*
+ * This is the function that driver code will call in order to request the
+ * sync implementation to output debug information relating to any sync
+ * checkpoints it may have created which appear in the provided array of
+ * FW addresses of Unified Fence Objects (UFOs).
+ *
+ * Input: nr_ufos             The number of FW addresses provided in the
+ *                            vaddrs parameter.
+ * Input: vaddrs              The array of FW addresses of UFOs. The sync
+ *                            implementation should check each of these to
+ *                            see if any relate to sync checkpoints it has
+ *                            created and where they do output debug information
+ *                            pertaining to the native/fallback sync with
+ *                            which it is associated.
+ */
+static u32
+pvr_sync_dump_info_on_stalled_ufos(u32 nr_ufos, u32 *vaddrs)
+{
+	return pvr_fence_dump_info_on_stalled_ufos(pvr_sync_data.foreign_fence_context,
+						   nr_ufos,
+						   vaddrs);
+}
+
+#if defined(PDUMP)
+static enum PVRSRV_ERROR_TAG
+pvr_sync_fence_get_checkpoints(PVRSRV_FENCE fence_to_pdump, u32 *nr_checkpoints,
+				struct SYNC_CHECKPOINT_TAG ***checkpoint_handles)
+{
+	struct dma_fence **fences = NULL;
+	struct dma_fence *fence;
+	struct pvr_fence *pvr_fence;
+	struct SYNC_CHECKPOINT_TAG **checkpoints = NULL;
+	unsigned int i, num_fences, num_used_fences = 0;
+	enum PVRSRV_ERROR_TAG err;
+
+	if (fence_to_pdump < 0) {
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	if (!nr_checkpoints || !checkpoint_handles) {
+		pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
+			__func__);
+		err =  PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_out;
+	}
+
+	fence = sync_file_get_fence(fence_to_pdump);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
+			__func__, fence_to_pdump);
+		err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
+		goto err_out;
+	}
+
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *array = to_dma_fence_array(fence);
+
+		fences = array->fences;
+		num_fences = array->num_fences;
+	} else {
+		fences = &fence;
+		num_fences = 1;
+	}
+
+	checkpoints = kmalloc_array(num_fences, sizeof(*checkpoints),
+			      GFP_KERNEL);
+	if (!checkpoints) {
+		pr_err("pvr_sync_file: %s: Failed to alloc memory for returned list of sync checkpoints\n",
+			__func__);
+		err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fence;
+	}
+
+	for (i = 0; i < num_fences; i++) {
+		if (is_pvr_fence(fences[i])) {
+			pvr_fence = to_pvr_fence(fences[i]);
+			if (pvr_fence) {
+				checkpoints[num_used_fences] = pvr_fence_get_checkpoint(pvr_fence);
+				++num_used_fences;
+			}
+		} else if (pvr_is_exp_fence(fences[i])) {
+			struct pvr_exp_fence *pvr_exp_fence = to_pvr_exp_fence(fences[i]);
+
+			checkpoints[num_used_fences] = pvr_exp_fence_get_checkpoint(pvr_exp_fence);
+			if (checkpoints[num_used_fences])
+				++num_used_fences;
+		} else
+			continue;
+	}
+
+	*checkpoint_handles = checkpoints;
+	*nr_checkpoints = num_used_fences;
+	err =  PVRSRV_OK;
+
+err_put_fence:
+	dma_fence_put(fence);
+err_out:
+	return err;
+}
+#endif
+
+int pvr_sync_api_rename(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sync_rename_ioctl_data *data = user_data;
+
+	data->szName[sizeof(data->szName) - 1] = '\0';
+	OSStringSafeCopy(timeline->name, data->szName, sizeof(timeline->name));
+	if (timeline->hw_fence_context)
+		OSStringSafeCopy(timeline->hw_fence_context->name, data->szName,
+			sizeof(timeline->hw_fence_context->name));
+
+	return 0;
+}
+
+int pvr_sync_api_force_sw_only(void *api_priv, void **api_priv_new)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	/* Already in SW mode? */
+	if (timeline->sw_fence_timeline)
+		return 0;
+
+	/* Create a sw_sync timeline with the old GPU timeline's name */
+	timeline->sw_fence_timeline = pvr_counting_fence_timeline_create(
+		timeline->name);
+	if (!timeline->sw_fence_timeline)
+		return -ENOMEM;
+
+	timeline->is_sw = true;
+
+	*api_priv_new = (void *)timeline;
+
+	return 0;
+}
+
+/* We simply treat an export-fence as a SW fence and tweak
+ * the timeline structure to flag it as an 'is_export' type.
+ */
+int pvr_sync_api_force_exp_only(void *api_priv, void *api_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+
+	if (timeline->is_export) {
+		pr_err(FILE_NAME ": %s: Already marked export timeline\n", __func__);
+		return 0;
+	}
+
+	timeline->exp_fence_context = pvr_exp_fence_context_create("pvr_exp_fence_ctx", "pvr_sync");
+	if (!timeline->exp_fence_context)
+		return -ENOMEM;
+
+	timeline->is_export = true;
+
+	return 0;
+}
+
+int pvr_sync_api_create_export_fence(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_exp_fence_context *exp_fence_context = timeline->exp_fence_context;
+	pvr_exp_sync_create_fence_data_t *data = user_data;
+	struct sync_file *sync_file;
+	int fd;
+	struct dma_fence *fence;
+	int err;
+
+	if (data == NULL) {
+		pr_err(FILE_NAME ": %s: Unexpected NULL user_data\n", __func__);
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
+		       __func__, fd);
+		err = -EMFILE;
+		goto err_out;
+	}
+
+	fence = pvr_exp_fence_create(exp_fence_context, fd, &data->sync_pt_idx);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to create a dma fence (%d)\n",
+		       __func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync_file (%d)\n",
+			__func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	data->fence = fd;
+
+	fd_install(fd, sync_file->file);
+
+	return 0;
+
+err_put_fd:
+	put_unused_fd(fd);
+err_out:
+	return err;
+}
+
+int pvr_sync_api_sw_create_fence(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sw_sync_create_fence_data *data = user_data;
+	struct sync_file *sync_file;
+	int fd = get_unused_fd_flags(O_CLOEXEC);
+	struct dma_fence *fence;
+	int err;
+
+	if (fd < 0) {
+		pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
+		       __func__, fd);
+		err = -EMFILE;
+		goto err_out;
+	}
+
+	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, &data->sync_pt_idx);
+	if (!fence) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
+		       __func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
+			__func__, fd);
+		err = -ENOMEM;
+		goto err_put_fd;
+	}
+
+	data->fence = fd;
+
+	fd_install(fd, sync_file->file);
+
+	return 0;
+
+err_put_fd:
+	put_unused_fd(fd);
+err_out:
+	return err;
+}
+
+int pvr_sync_api_sw_inc(void *api_priv, void *user_data)
+{
+	struct pvr_sync_timeline *timeline = api_priv;
+	struct pvr_sw_timeline_advance_data *data = user_data;
+	bool res;
+
+	res = pvr_counting_fence_timeline_inc(timeline->sw_fence_timeline, &data->sync_pt_idx);
+
+	/* pvr_counting_fence_timeline_inc won't allow sw timeline to be
+	 * advanced beyond the last defined point
+	 */
+	if (!res) {
+		pr_err("pvr_sync_file: attempt to advance SW timeline beyond last defined point\n");
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static void
+pvr_sync_debug_request_heading(void *data, u32 verbosity,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+		PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
+				  "------[ Native Fence Sync: timelines ]------");
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_register_functions(void)
+{
+	/* Register the resolve fence and create fence functions with
+	 * sync_checkpoint.c
+	 * The pvr_fence context registers its own EventObject callback to
+	 * update sync status
+	 */
+	/* Initialise struct and register with sync_checkpoint.c */
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceResolve = pvr_sync_resolve_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceCreate = pvr_sync_create_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceDataRollback = pvr_sync_rollback_fence_data;
+	pvr_sync_data.sync_checkpoint_ops.pfnFenceFinalise = pvr_sync_finalise_fence;
+#if defined(NO_HARDWARE)
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = pvr_sync_nohw_signal_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWSignalExpFence = pvr_sync_nohw_signal_exp_fence;
+#else
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = NULL;
+	pvr_sync_data.sync_checkpoint_ops.pfnNoHWSignalExpFence = NULL;
+#endif
+	pvr_sync_data.sync_checkpoint_ops.pfnFreeCheckpointListMem =
+		pvr_sync_free_checkpoint_list_mem;
+	pvr_sync_data.sync_checkpoint_ops.pfnDumpInfoOnStalledUFOs =
+		pvr_sync_dump_info_on_stalled_ufos;
+	OSStringSafeCopy(pvr_sync_data.sync_checkpoint_ops.pszImplName, "pvr_sync_file",
+		SYNC_CHECKPOINT_IMPL_MAX_STRLEN);
+#if defined(PDUMP)
+	pvr_sync_data.sync_checkpoint_ops.pfnSyncFenceGetCheckpoints =
+		pvr_sync_fence_get_checkpoints;
+#endif
+	pvr_sync_data.sync_checkpoint_ops.pfnExportFenceResolve = pvr_sync_resolve_export_fence;
+	pvr_sync_data.sync_checkpoint_ops.pfnExportFenceRollback = pvr_sync_rollback_export_fence;
+
+	return SyncCheckpointRegisterFunctions(&pvr_sync_data.sync_checkpoint_ops);
+}
+
+int pvr_sync_init(void)
+{
+	int err;
+
+	pvr_sync_data.foreign_fence_context =
+			pvr_fence_foreign_context_create(
+					NativeSyncGetFenceStatusWq(),
+					"foreign_sync");
+	if (!pvr_sync_data.foreign_fence_context) {
+		pr_err(FILE_NAME ": %s: Failed to create foreign sync context\n",
+			__func__);
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+#if defined(NO_HARDWARE)
+	INIT_LIST_HEAD(&pvr_timeline_active_list);
+#endif
+
+	err = pvr_sync_ioctl_init();
+	if (err) {
+		pr_err(FILE_NAME ": %s: Failed to register pvr_sync device (%d)\n",
+		       __func__, err);
+		goto err_ioctl_init;
+	}
+
+	return 0;
+
+err_ioctl_init:
+	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
+	pvr_fence_cleanup();
+err_out:
+	return err;
+}
+
+void pvr_sync_deinit(void)
+{
+	pvr_sync_ioctl_deinit();
+	pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
+	pvr_fence_cleanup();
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_device_init(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+	enum PVRSRV_ERROR_TAG error;
+
+	error = PVRSRVRegisterDeviceDbgRequestNotify(
+				&priv->sync_debug_notify_handle,
+				priv->dev_node,
+				pvr_sync_debug_request_heading,
+				DEBUG_REQUEST_LINUXFENCE,
+				NULL);
+	if (error != PVRSRV_OK) {
+		pr_err("%s: failed to register debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(error));
+		goto err_out;
+	}
+
+	/* Register the foreign sync context debug notifier on each device */
+	error = pvr_fence_context_register_dbg(
+				&priv->sync_foreign_debug_notify_handle,
+				priv->dev_node,
+				pvr_sync_data.foreign_fence_context);
+	if (error != PVRSRV_OK) {
+		pr_err("%s: failed to register fence debug request callback (%s)\n",
+		       __func__, PVRSRVGetErrorString(error));
+		goto err_context_regdbg;
+	}
+
+#if defined(NO_HARDWARE)
+	INIT_LIST_HEAD(&pvr_timeline_active_list);
+#endif
+
+	return PVRSRV_OK;
+
+err_context_regdbg:
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
+err_out:
+	return error;
+}
+
+void pvr_sync_device_deinit(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct pvr_drm_private *priv = ddev->dev_private;
+
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_foreign_debug_notify_handle);
+	PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_wait(void *fence, u32 timeout_in_ms)
+{
+	long timeout = msecs_to_jiffies(timeout_in_ms);
+	int err;
+
+	err = dma_fence_wait_timeout(fence, true, timeout);
+	/*
+	 * dma_fence_wait_timeout returns:
+	 * - the remaining timeout on success
+	 * - 0 on timeout
+	 * - -ERESTARTSYS if interrupted
+	 */
+	if (err > 0)
+		return PVRSRV_OK;
+	else if (err == 0)
+		return PVRSRV_ERROR_TIMEOUT;
+
+	return PVRSRV_ERROR_FAILED_DEPENDENCIES;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_release(void *fence)
+{
+	dma_fence_put(fence);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_fence_get(int fence_fd, void **fence_out)
+{
+	struct dma_fence *fence;
+
+	fence = sync_file_get_fence(fence_fd);
+	if (fence == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	*fence_out = fence;
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG
+pvr_sync_sw_timeline_fence_create(struct _PVRSRV_DEVICE_NODE_ *pvrsrv_dev_node,
+				  int timeline_fd,
+				  const char *fence_name,
+				  int *fence_fd_out,
+				  u64 *sync_pt_idx)
+{
+	enum PVRSRV_ERROR_TAG srv_err;
+	struct pvr_sync_timeline *timeline;
+	struct dma_fence *fence = NULL;
+	struct sync_file *sync_file = NULL;
+	int fd;
+
+	(void)(pvrsrv_dev_node);
+
+	fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fd < 0)
+		return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
+
+	timeline = pvr_sync_timeline_fget(timeline_fd);
+	if (!timeline) {
+		/* unrecognised timeline */
+		srv_err = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		goto err_put_fd;
+	}
+	if (!timeline->is_sw) {
+		pvr_sync_timeline_fput(timeline);
+		srv_err = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_put_fd;
+	}
+
+	fence = pvr_counting_fence_create(timeline->sw_fence_timeline, sync_pt_idx);
+	pvr_sync_timeline_fput(timeline);
+	if (!fence) {
+		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fd;
+	}
+
+	sync_file = sync_file_create(fence);
+	dma_fence_put(fence);
+	if (!sync_file) {
+		srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_put_fd;
+	}
+
+	fd_install(fd, sync_file->file);
+
+	*fence_fd_out = fd;
+
+	return PVRSRV_OK;
+
+err_put_fd:
+	put_unused_fd(fd);
+	return srv_err;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_advance(void *timeline, u64 *sync_pt_idx)
+{
+	if (timeline == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	pvr_counting_fence_timeline_inc(timeline, sync_pt_idx);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_release(void *timeline)
+{
+	if (timeline == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	pvr_counting_fence_timeline_put(timeline);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_get(int timeline_fd,
+					   void **timeline_out)
+{
+	struct pvr_counting_fence_timeline *sw_timeline;
+	struct pvr_sync_timeline *timeline;
+
+	timeline = pvr_sync_timeline_fget(timeline_fd);
+	if (!timeline)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	sw_timeline =
+		pvr_counting_fence_timeline_get(timeline->sw_fence_timeline);
+	pvr_sync_timeline_fput(timeline);
+	if (!sw_timeline)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	*timeline_out = sw_timeline;
+
+	return PVRSRV_OK;
+}
+static void _dump_sync_point(struct dma_fence *fence,
+							  DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+							  void *dump_debug_file)
+{
+	const struct dma_fence_ops *fence_ops = fence->ops;
+	bool signaled = dma_fence_is_signaled(fence);
+	char time[16] = { '\0' };
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	if (is_pvr_fence(fence)) {
+		struct pvr_fence *pvr_fence = to_pvr_fence(fence);
+
+		pvr_context_value_str(pvr_fence->fctx, time, sizeof(time));
+	} else {
+		snprintf(time, sizeof(time), "%llu", fence->seqno);
+	}
+#else
+	fence_ops->timeline_value_str(fence, time, sizeof(time));
+#endif
+
+	PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "<%p> Seq#=%llu TS=%s State=%s TLN=%s",
+					  fence,
+					  (u64) fence->seqno,
+					  time,
+					  (signaled) ? "Signalled" : "Active",
+					  fence_ops->get_timeline_name(fence));
+}
+
+static void _dump_fence(struct dma_fence *fence,
+			DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+			void *dump_debug_file)
+{
+	if (dma_fence_is_array(fence)) {
+		struct dma_fence_array *fence_array = to_dma_fence_array(fence);
+		int i;
+
+		if (fence_array) {
+			PVR_DUMPDEBUG_LOG(dump_debug_printf,
+					  dump_debug_file,
+					  "Fence: [%p] Sync Points:\n",
+					  fence_array);
+
+			for (i = 0; i < fence_array->num_fences; i++)
+				_dump_sync_point(fence_array->fences[i],
+						 dump_debug_printf,
+						 dump_debug_file);
+		}
+
+	} else {
+		_dump_sync_point(fence, dump_debug_printf, dump_debug_file);
+	}
+}
+
+enum PVRSRV_ERROR_TAG
+sync_dump_fence(void *sw_fence_obj,
+		DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		void *dump_debug_file)
+{
+	struct dma_fence *fence = (struct dma_fence *) sw_fence_obj;
+
+	_dump_fence(fence, dump_debug_printf, dump_debug_file);
+
+	return PVRSRV_OK;
+}
+
+enum PVRSRV_ERROR_TAG
+sync_sw_dump_timeline(void *sw_timeline_obj,
+		      DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
+		      void *dump_debug_file)
+{
+	pvr_counting_fence_timeline_dump_timeline(sw_timeline_obj,
+						  dump_debug_printf,
+						  dump_debug_file);
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_common.c b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_common.c
new file mode 100644
index 000000000000..0e1facde86cb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_common.c
@@ -0,0 +1,405 @@
+/*
+ * @File        pvr_sync_ioctl_common.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/slab.h>
+
+#include "pvr_drm.h"
+#include "pvr_sync_api.h"
+#include "pvr_sync_ioctl_common.h"
+
+#include "pvr_debug.h"
+#include "pvr_uaccess.h"
+
+/*
+ * The PVR Sync API is unusual in that some operations configure the
+ * timeline for use, and are no longer allowed once the timeline is
+ * in use. A locking mechanism, such as a read/write semaphore, would
+ * be one method of helping to ensure the API rules are followed, but
+ * this would add unnecessary overhead once the timeline has been
+ * configured, as read locks would continue to have to be taken after
+ * the timeline is in use. To avoid locks, two atomic variables are used,
+ * together with memory barriers. The in_setup variable indicates a "rename"
+ * or "force software only" ioctl is in progress. At most one of these two
+ * configuration ioctls can be in progress at any one time, and they can't
+ * overlap with any other Sync ioctl. The in_use variable indicates one
+ * of the other Sync ioctls has started. Once set, in_use stays set, and
+ * prevents any further configuration ioctls. Non-configuration ioctls
+ * are allowed to overlap.
+ * It is possible for a configuration and non-configuration ioctl to race,
+ * but at most one will be allowed to proceed, and perhaps neither.
+ * Given the intended usage of the API in user space, where the timeline
+ * is fully configured before being used, the race behaviour won't be
+ * an issue.
+ */
+
+struct pvr_sync_file_data {
+	atomic_t in_setup;
+	atomic_t in_use;
+	void *api_private;
+	bool is_sw;
+	bool is_export;
+};
+
+static bool pvr_sync_set_in_use(struct pvr_sync_file_data *fdata)
+{
+	if (atomic_read(&fdata->in_use) < 2) {
+		atomic_set(&fdata->in_use, 1);
+		/* Ensure in_use change is visible before in_setup is read */
+		smp_mb();
+		if (atomic_read(&fdata->in_setup) != 0)
+			return false;
+
+		atomic_set(&fdata->in_use, 2);
+	} else {
+		/* Ensure stale private data isn't read */
+		smp_rmb();
+	}
+
+	return true;
+}
+
+static bool pvr_sync_set_in_setup(struct pvr_sync_file_data *fdata)
+{
+	int in_setup;
+
+	in_setup = atomic_inc_return(&fdata->in_setup);
+	if (in_setup > 1 || atomic_read(&fdata->in_use) != 0) {
+		atomic_dec(&fdata->in_setup);
+		return false;
+	}
+
+	return true;
+}
+
+static inline void pvr_sync_reset_in_setup(struct pvr_sync_file_data *fdata)
+{
+	/*
+	 * Ensure setup changes are visible before allowing other
+	 * operations to proceed.
+	 */
+	smp_mb__before_atomic();
+	atomic_dec(&fdata->in_setup);
+}
+
+void *pvr_sync_get_api_priv_common(struct file *file)
+{
+	if (file != NULL && pvr_sync_is_timeline(file)) {
+		struct pvr_sync_file_data *fdata = pvr_sync_get_private_data(file);
+
+		if (fdata != NULL && pvr_sync_set_in_use(fdata))
+			return fdata->api_private;
+	}
+
+	return NULL;
+}
+
+int pvr_sync_open_common(void *connection_data, void *file_handle)
+{
+	void *data = NULL;
+	struct pvr_sync_file_data *fdata;
+	int err;
+
+	fdata = kzalloc(sizeof(*fdata), GFP_KERNEL);
+	if (!fdata)
+		return -ENOMEM;
+
+	atomic_set(&fdata->in_setup, 0);
+	atomic_set(&fdata->in_use, 0);
+
+	if (!pvr_sync_set_private_data(connection_data, fdata)) {
+		kfree(fdata);
+		return -EINVAL;
+	}
+
+	err = pvr_sync_api_init(file_handle, &data);
+	if (err)
+		kfree(fdata);
+	else
+		fdata->api_private = data;
+
+	return err;
+}
+
+int pvr_sync_close_common(void *connection_data)
+{
+	struct pvr_sync_file_data *fdata;
+
+	fdata = pvr_sync_connection_private_data(connection_data);
+	if (fdata) {
+		int err;
+
+		err = pvr_sync_api_deinit(fdata->api_private, fdata->is_sw);
+
+		kfree(fdata);
+
+		return err;
+	}
+
+	return 0;
+}
+
+#define PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(name, type) \
+static inline int pvr_sync_ioctl_common_internal_ ## name(struct pvr_sync_file_data *fdata, type data)
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(rename, struct pvr_sync_rename_ioctl_data *)
+{
+	return pvr_sync_api_rename(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(force_sw_only, void*)
+{
+	void *new_api_private = fdata->api_private;
+	int err;
+
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	err = pvr_sync_api_force_sw_only(fdata->api_private, &new_api_private);
+	if (!err) {
+		if (new_api_private != fdata->api_private)
+			fdata->api_private = new_api_private;
+
+		fdata->is_sw = true;
+	}
+
+	return err;
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(force_exp_only, void*)
+{
+	int err;
+	PVR_UNREFERENCED_PARAMETER(data);
+
+	err = pvr_sync_api_force_exp_only(fdata->api_private, data);
+	if (!err)
+		fdata->is_export = true;
+
+	return err;
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(create_export_fence, void*)
+{
+	return pvr_sync_api_create_export_fence(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(sw_create_fence, struct pvr_sw_sync_create_fence_data *)
+{
+	return pvr_sync_api_sw_create_fence(fdata->api_private, data);
+}
+
+PVR_SYNC_IOCTL_DISPATCH_INTERNAL_DECL(sw_inc, struct pvr_sw_timeline_advance_data *)
+{
+	return pvr_sync_api_sw_inc(fdata->api_private, data);
+}
+
+/*
+ * enum pvr_sync_ioctl_dispatch_type
+ * @pvr_sync_ioctl_dispatch_type_setup: The command may only be used during the setup phase
+ * @pvr_sync_ioctl_dispatch_type_export: The command may only be used with an export only timeline.
+ * @pvr_sync_ioctl_dispatch_type_software: The command may only be used with a software only timeline.
+ */
+enum pvr_sync_ioctl_dispatch_type {
+	pvr_sync_ioctl_dispatch_type_setup,
+	pvr_sync_ioctl_dispatch_type_export,
+	pvr_sync_ioctl_dispatch_type_software,
+};
+
+/* Generates a function `from` which performs validation on the data before passing it to `into` */
+#define PVR_SYNC_IOCTL_DISPATCH_VALIDATE(name, structure, type)                                                                        \
+PVR_SYNC_IOCTL_DISPATCH_DECL(name)                                                                                                     \
+{                                                                                                                                      \
+	int err = -ENOTTY;                                                                                                                 \
+	structure server_data;                                                                                                             \
+                                                                                                                                       \
+	struct pvr_sync_file_data *fdata = pvr_sync_get_private_data(file);                                                                \
+	if (unlikely(!fdata))                                                                                                              \
+		return -EINVAL;                                                                                                                \
+                                                                                                                                       \
+	/* Check if the device is busy and the operation is valid for the timelines current state */                                       \
+	if (type == pvr_sync_ioctl_dispatch_type_setup)                                                                                    \
+	{                                                                                                                                  \
+		if (!pvr_sync_set_in_setup(fdata))                                                                                             \
+			return -EBUSY;                                                                                                             \
+		if (fdata->is_sw || fdata->is_export)                                                                                          \
+			goto return_;                                                                                                              \
+	}                                                                                                                                  \
+	else                                                                                                                               \
+	{                                                                                                                                  \
+		if (!pvr_sync_set_in_use(fdata))                                                                                               \
+			return -EBUSY;                                                                                                             \
+                                                                                                                                       \
+		switch (type)                                                                                                                  \
+		{                                                                                                                              \
+		case pvr_sync_ioctl_dispatch_type_software: {                                                                                  \
+			if (!fdata->is_sw) /* Not a software timeline, software operations cannot be used */                                       \
+				goto return_;                                                                                                          \
+		}                                                                                                                              \
+		break;                                                                                                                         \
+		case pvr_sync_ioctl_dispatch_type_export: {                                                                                    \
+			if (!fdata->is_export) /* Not an export timeline, export operations cannot be used */                                      \
+				goto return_;                                                                                                          \
+		}                                                                                                                              \
+		break;                                                                                                                         \
+		default:                                                                                                                       \
+			goto return_; /* Invalid Type */                                                                                           \
+		}                                                                                                                              \
+	}                                                                                                                                  \
+                                                                                                                                       \
+	/* copy_from_user */                                                                                                               \
+	err = pvr_sync_ioctl_dispatch_copy_in__##name((structure __user *)user_data, &server_data);                                        \
+	if (unlikely(err))                                                                                                                 \
+		goto return_;                                                                                                                  \
+                                                                                                                                       \
+	/* Continue into api */                                                                                                            \
+	err = pvr_sync_ioctl_common_internal_ ## name(fdata, (structure __force *) PVR_SYNC_IOCTL_DISPATCH_DATA(user_data, &server_data)); \
+                                                                                                                                       \
+	if (likely(!err))                                                                                                                  \
+	{                                                                                                                                  \
+		/* copy_to_user */                                                                                                             \
+		err = pvr_sync_ioctl_dispatch_copy_out__##name((structure __user *)user_data, &server_data);                                   \
+	}                                                                                                                                  \
+                                                                                                                                       \
+return_:                                                                                                                               \
+	if (type == pvr_sync_ioctl_dispatch_type_setup)                                                                                    \
+		pvr_sync_reset_in_setup(fdata);                                                                                                \
+	return err;                                                                                                                        \
+}
+
+#if !defined(USE_PVRSYNC_DEVNODE)
+/* drm_ioctl() already copies the data over, see comment on drm_ioctl_t */
+#define PVR_SYNC_IOCTL_DISPATCH_DATA(pUM, pKM) pUM
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(dir, name, structure, copy) \
+INLINE static int pvr_sync_ioctl_dispatch_copy_ ## dir ## __ ## name (structure __user *pUM, structure *pKM) \
+{ return 0; }
+#else /* !defined(USE_PVRSYNC_DEVNODE) */
+/* Generates a function to copy over the arguments to/from user-mode */
+#define PVR_SYNC_IOCTL_DISPATCH_DATA(pUM, pKM) pKM
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(dir, name, structure, copy)                                     \
+INLINE static int pvr_sync_ioctl_dispatch_copy_ ## dir ## __ ## name (structure __user *pUM, structure *pKM) \
+{                                                                                                            \
+	/* May be unused if there are no in/out args */                                                          \
+	PVR_UNREFERENCED_PARAMETER(pUM);                                                                         \
+	PVR_UNREFERENCED_PARAMETER(pKM);                                                                         \
+	/* Copy over the data */                                                                                 \
+	{ copy }                                                                                                 \
+	return 0;                                                                                                \
+}
+#endif /* !defined(USE_PVRSYNC_DEVNODE) */
+
+/*************************************************************************/ /*!
+@Function       PVR_SYNC_IOCTL_DISPATCH_COPY
+@Description    Generates the code to copy from/to user mode depending on @dir.
+@Input          dir  Either `from` or `to` corresponding to `copy_from_user`
+                    and `copy_to_user` respectively.
+@Input          to   Pointer for the dest.
+@Input          from Pointer for the src.
+*/ /**************************************************************************/
+#define PVR_SYNC_IOCTL_DISPATCH_COPY(dir, to, from)        \
+if (pvr_copy_##dir##_user(to, from, sizeof(*pKM)))         \
+{                                                          \
+	PVR_DPF((PVR_DBG_ERROR, "Failed copy " #dir " user")); \
+	return -EFAULT;                                        \
+}
+
+/* Copy data from user */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_IN \
+PVR_SYNC_IOCTL_DISPATCH_COPY(from, pKM, pUM)
+
+/* Copy data to user */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_OUT \
+PVR_SYNC_IOCTL_DISPATCH_COPY(to, pUM, pKM)
+
+/* Copy no data */
+#define PVR_SYNC_IOCTL_DISPATCH_COPY_NONE
+
+/*************************************************************************/ /*!
+@Function       PVR_SYNC_IOCTL_DISPATCH_FUNCTION
+@Description    Generates a dispatch function which validates the ioctl command
+                and dispatches it to the internal an function.
+                Generates a dispatch function
+                `pvr_sync_ioctl_common_##name(struct file*, void*)`
+                which validates the ioctl command and dispatches it to
+                an internal function
+                `pvr_sync_ioctl_common_internal_##name(struct pvr_sync_file_data*, structure*)`
+@Input          name      A name for the dispatch functions, must be unique.
+@Input          structure The type of the user_data.
+@Input          type      The type specifies when the ioctls are allowed or
+                          if the ioctl is allowed on the current
+                          timeline configuration.
+@Input          copy_in   Either `COPY_IN` or `COPY_NONE`
+@Input          copy_out  Either `COPY_OUT` or `COPY_NONE`
+*/ /**************************************************************************/
+#define PVR_SYNC_IOCTL_DISPATCH_FUNCTION(name, structure, type, copy_in, copy_out) \
+	PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(in, name, structure, PVR_SYNC_IOCTL_DISPATCH_ ## copy_in) \
+	PVR_SYNC_IOCTL_DISPATCH_COPY_WRAPPER(out, name, structure, PVR_SYNC_IOCTL_DISPATCH_ ## copy_out) \
+	PVR_SYNC_IOCTL_DISPATCH_VALIDATE(name, structure, pvr_sync_ioctl_dispatch_type_ ## type)
+
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(rename, struct pvr_sync_rename_ioctl_data,
+                                 setup,
+                                 COPY_IN,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(force_sw_only, void*,
+                                 setup,
+                                 COPY_NONE,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(force_exp_only, void*,
+                                 setup,
+                                 COPY_NONE,
+                                 COPY_NONE);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(sw_create_fence, struct pvr_sw_sync_create_fence_data,
+                                 software,
+                                 COPY_IN,
+                                 COPY_OUT);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(create_export_fence, pvr_exp_sync_create_fence_data_t,
+                                 export,
+                                 COPY_IN,
+                                 COPY_OUT);
+
+PVR_SYNC_IOCTL_DISPATCH_FUNCTION(sw_inc, struct pvr_sw_timeline_advance_data,
+                                 software,
+                                 COPY_NONE,
+                                 COPY_OUT);
diff --git a/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_common.h b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_common.h
new file mode 100644
index 000000000000..0959e7673164
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_common.h
@@ -0,0 +1,85 @@
+/*
+ * @File        pvr_sync_ioctl_common.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_IOCTL_COMMON_H
+#define _PVR_SYNC_IOCTL_COMMON_H
+
+struct file;
+
+/* Functions provided by pvr_sync_ioctl_common.c */
+
+int pvr_sync_open_common(void *connection_data, void *file_handle);
+int pvr_sync_close_common(void *connection_data);
+void *pvr_sync_get_api_priv_common(struct file *file);
+
+#if defined(USE_PVRSYNC_DEVNODE)
+#define PVR_SYNC_IOCTL_DISPATCH_DECL(name) \
+	int pvr_sync_ioctl_common_ ## name(struct file *file, void __user *user_data)
+#else /* defined(USE_PVRSYNC_DEVNODE) */
+#define PVR_SYNC_IOCTL_DISPATCH_DECL(name) \
+	int pvr_sync_ioctl_common_ ## name(struct file *file, void *user_data)
+#endif /* defined(USE_PVRSYNC_DEVNODE) */
+
+PVR_SYNC_IOCTL_DISPATCH_DECL(rename);
+PVR_SYNC_IOCTL_DISPATCH_DECL(force_sw_only);
+PVR_SYNC_IOCTL_DISPATCH_DECL(force_exp_only);
+PVR_SYNC_IOCTL_DISPATCH_DECL(sw_create_fence);
+PVR_SYNC_IOCTL_DISPATCH_DECL(create_export_fence);
+PVR_SYNC_IOCTL_DISPATCH_DECL(sw_inc);
+
+struct pvr_sync_file_data;
+
+/* Functions required by pvr_sync_ioctl_common.c */
+
+bool pvr_sync_set_private_data(void *connection_data,
+			       struct pvr_sync_file_data *fdata);
+
+struct pvr_sync_file_data *
+pvr_sync_connection_private_data(void *connection_data);
+
+struct pvr_sync_file_data *
+pvr_sync_get_private_data(struct file *file);
+
+bool pvr_sync_is_timeline(struct file *file);
+
+#endif /* _PVR_SYNC_IOCTL_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_drm.c b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_drm.c
new file mode 100644
index 000000000000..313393d09c1f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_drm.c
@@ -0,0 +1,184 @@
+/*
+ * @File        pvr_sync_ioctl_drm.c
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "pvr_drv.h"
+#include "pvr_drm.h"
+#include "private_data.h"
+#include "env_connection.h"
+#include "pvr_sync_api.h"
+#include "pvr_sync_ioctl_common.h"
+#include "pvr_sync_ioctl_drm.h"
+
+bool pvr_sync_set_private_data(void *connection_data,
+			       struct pvr_sync_file_data *fdata)
+{
+	if (connection_data) {
+		ENV_CONNECTION_DATA *env_data;
+
+		env_data = PVRSRVConnectionPrivateData(connection_data);
+		if (env_data) {
+			env_data->pvPvrSyncPrivateData = fdata;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+struct pvr_sync_file_data *
+pvr_sync_connection_private_data(void *connection_data)
+{
+	if (connection_data) {
+		ENV_CONNECTION_DATA *env_data;
+
+		env_data = PVRSRVConnectionPrivateData(connection_data);
+
+		if (env_data)
+			return env_data->pvPvrSyncPrivateData;
+	}
+
+	return NULL;
+}
+
+struct pvr_sync_file_data *
+pvr_sync_get_private_data(struct file *file)
+{
+	CONNECTION_DATA *connection_data = LinuxSyncConnectionFromFile(file);
+
+	return pvr_sync_connection_private_data(connection_data);
+}
+
+bool pvr_sync_is_timeline(struct file *file)
+{
+	return file->f_op == &pvr_drm_fops;
+}
+
+void *pvr_sync_get_api_priv(struct file *file)
+{
+	return pvr_sync_get_api_priv_common(file);
+}
+
+struct file *pvr_sync_get_file_struct(void *file_handle)
+{
+	if (file_handle) {
+		struct drm_file *file = file_handle;
+
+		return file->filp;
+	}
+
+	return NULL;
+}
+
+int pvr_sync_open(void *connection_data, struct drm_file *file)
+{
+	/*
+	 * The file structure pointer (file->filp) may not have been
+	 * initialised at this point, so pass down a pointer to the
+	 * drm_file structure instead.
+	 */
+	return pvr_sync_open_common(connection_data, file);
+}
+
+void pvr_sync_close(void *connection_data)
+{
+	int iErr = pvr_sync_close_common(connection_data);
+
+	if (iErr < 0)
+		pr_err("%s: ERROR (%d) returned by pvr_sync_close_common()\n",
+		       __func__, iErr);
+
+	iErr = OSConnectionPrivateDataDeInit(PVRSRVConnectionPrivateData(connection_data));
+	if (iErr != 0)
+		pr_err("%s: ERROR (%d) returned by OSConnectionPrivateDataDeInit()\n",
+		       __func__, iErr);
+
+	if (connection_data)
+		kfree(connection_data);
+}
+
+
+int pvr_sync_rename_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_rename(file->filp, arg);
+}
+
+int pvr_sync_force_sw_only_ioctl(struct drm_device __maybe_unused *dev,
+				 void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_force_sw_only(file->filp, arg);
+}
+
+int pvr_sw_sync_create_fence_ioctl(struct drm_device __maybe_unused *dev,
+				   void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_sw_create_fence(file->filp, arg);
+}
+
+int pvr_sw_sync_inc_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_sw_inc(file->filp, arg);
+}
+
+int pvr_sync_ioctl_force_exp_only(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_force_exp_only(file->filp, arg);
+}
+
+int pvr_export_fence_sync_create_fence_ioctl(struct drm_device __maybe_unused *dev,
+			  void *arg, struct drm_file *file)
+{
+	return pvr_sync_ioctl_common_create_export_fence(file->filp, arg);
+}
+
+int pvr_sync_ioctl_init(void)
+{
+	return 0;
+}
+
+void pvr_sync_ioctl_deinit(void)
+{
+}
diff --git a/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_drm.h b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_drm.h
new file mode 100644
index 000000000000..93182205ca0b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_sync_ioctl_drm.h
@@ -0,0 +1,66 @@
+/*
+ * @File        pvr_sync_ioctl_drm.h
+ * @Title       Kernel driver for Android's sync mechanism
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _PVR_SYNC_IOCTL_DRM_H
+#define _PVR_SYNC_IOCTL_DRM_H
+
+struct drm_device;
+struct drm_file;
+
+int pvr_sync_open(void *connection_data, struct drm_file *file);
+void pvr_sync_close(void *connection_data);
+
+int pvr_sync_rename_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_sync_force_sw_only_ioctl(struct drm_device *dev, void *arg,
+				 struct drm_file *file);
+int pvr_sw_sync_create_fence_ioctl(struct drm_device *dev, void *arg,
+				   struct drm_file *file);
+int pvr_sw_sync_inc_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_sync_ioctl_force_exp_only(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+int pvr_export_fence_sync_create_fence_ioctl(struct drm_device *dev, void *arg,
+			  struct drm_file *file);
+
+#endif /* _PVR_SYNC_IOCTL_DRM_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_uaccess.h b/drivers/gpu/drm/img-rogue/pvr_uaccess.h
new file mode 100644
index 000000000000..13864eab4c7a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_uaccess.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Title          Utility functions for user space access
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_UACCESS_H
+#define PVR_UACCESS_H
+
+#include <linux/uaccess.h>
+#include <linux/version.h>
+
+static inline unsigned long pvr_copy_to_user(void __user *pvTo, const void *pvFrom, unsigned long ulBytes)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+	if (access_ok(VERIFY_WRITE, pvTo, ulBytes))
+#else
+	if (access_ok(pvTo, ulBytes))
+#endif
+	{
+		return __copy_to_user(pvTo, pvFrom, ulBytes);
+	}
+
+	return ulBytes;
+}
+
+
+#if defined(__KLOCWORK__)
+	/* this part is only to tell Klocwork not to report false positive because
+	   it doesn't understand that pvr_copy_from_user will initialise the memory
+	   pointed to by pvTo */
+#include <linux/string.h> /* get the memset prototype */
+static inline unsigned long pvr_copy_from_user(void *pvTo, const void __user *pvFrom, unsigned long ulBytes)
+{
+	if (pvTo != NULL)
+	{
+		memset(pvTo, 0xAA, ulBytes);
+		return 0;
+	}
+	return 1;
+}
+
+#else /* real implementation */
+
+static inline unsigned long pvr_copy_from_user(void *pvTo, const void __user *pvFrom, unsigned long ulBytes)
+{
+	/*
+	 * The compile time correctness checking introduced for copy_from_user in
+	 * Linux 2.6.33 isn't fully compatible with our usage of the function.
+	 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+	if (access_ok(VERIFY_READ, pvFrom, ulBytes))
+#else
+	if (access_ok(pvFrom, ulBytes))
+#endif
+	{
+		return __copy_from_user(pvTo, pvFrom, ulBytes);
+	}
+
+	return ulBytes;
+}
+#endif /* klocworks */
+
+#endif /* PVR_UACCESS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvr_vmap.h b/drivers/gpu/drm/img-rogue/pvr_vmap.h
new file mode 100644
index 000000000000..6424b20e764c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvr_vmap.h
@@ -0,0 +1,83 @@
+/*
+ * @File        pvr_vmap.h
+ * @Title       Utility functions for virtual memory mapping
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PVR_VMAP_H
+#define PVR_VMAP_H
+
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+
+static inline void *pvr_vmap(struct page **pages,
+			     unsigned int count,
+			     __maybe_unused unsigned long flags,
+			     pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	return vmap(pages, count, flags, prot);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	return vm_map_ram(pages, count, -1, prot);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		return vm_map_ram(pages, count, -1);
+	else
+		return vmap(pages, count, flags, prot);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+static inline void pvr_vunmap(const void *vaddr,
+			      __maybe_unused unsigned int count,
+			      __maybe_unused pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	vunmap(vaddr);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	vm_unmap_ram(vaddr, count);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		vm_unmap_ram(vaddr, count);
+	else
+		vunmap(vaddr);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+#endif /* PVR_VMAP_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrmodule.h b/drivers/gpu/drm/img-rogue/pvrmodule.h
new file mode 100644
index 000000000000..267c7b687487
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrmodule.h
@@ -0,0 +1,48 @@
+/*************************************************************************/ /*!
+@Title          Module Author and License.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef	_PVRMODULE_H_
+#define	_PVRMODULE_H_
+
+MODULE_AUTHOR("Imagination Technologies Ltd. <gpl-support@imgtec.com>");
+MODULE_LICENSE("Dual MIT/GPL");
+
+#endif	/* _PVRMODULE_H_ */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv.c b/drivers/gpu/drm/img-rogue/pvrsrv.c
new file mode 100644
index 000000000000..51f7de420dc5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv.c
@@ -0,0 +1,3854 @@
+/*************************************************************************/ /*!
+@File
+@Title          core services functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main APIs for core services functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#define CLEANUP_TYPE_STRINGS
+
+#include "img_defs.h"
+#include "rgxdebug_common.h"
+#include "handle.h"
+
+#include "pmr.h"
+#include "connection_server.h"
+#include "osconnection_server.h"
+#include "pdump_km.h"
+#include "ra.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "srvcore.h"
+#include "services_km.h"
+#include "pvrsrv_device.h"
+#include "pvr_debug.h"
+#include "debug_common.h"
+#include "pvr_notifier.h"
+#include "sync.h"
+#include "sync_server.h"
+#include "sync_checkpoint.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint_init.h"
+#include "devicemem.h"
+#include "cache_km.h"
+#include "info_page.h"
+#include "info_page_defs.h"
+#include "pvrsrv_bridge_init.h"
+#include "devicemem_server.h"
+#include "km_apphint_defs.h"
+#include "di_server.h"
+#include "di_impl_brg.h"
+#include "htb_debug.h"
+#include "dma_km.h"
+#include "pmr.h"
+
+#include "log2.h"
+
+#include "lists.h"
+#include "dllist.h"
+#include "syscommon.h"
+#include "sysvalidation.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+#include "physmem_hostmem.h"
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+#include "physmem_cpumap_history.h"
+#endif
+
+#include "tlintern.h"
+#include "htbserver.h"
+#include "rgxmulticore.h"
+
+
+//#define MULTI_DEVICE_BRINGUP
+
+#if defined(MULTI_DEVICE_BRINGUP)
+#define MULTI_DEVICE_BRINGUP_DPF(msg, ...) PVR_DPF((PVR_DBG_MESSAGE, msg, __VA_ARGS__))
+#else
+#define MULTI_DEVICE_BRINGUP_DPF(msg, ...)
+#endif
+
+#if defined(SUPPORT_RGX)
+#include "rgxinit.h"
+#include "rgxhwperf.h"
+#include "rgxfwutils.h"
+#include "rgx_bridge_init.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+#include "ri_server.h"
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#include "vz_vmm_pvz.h"
+
+#include "devicemem_history_server.h"
+
+#if defined(SUPPORT_LINUX_DVFS)
+#include "pvr_dvfs_device.h"
+#elif defined(SUPPORT_PDVFS)
+#include "pvr_dvfs_common.h"
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+
+#include "rgx_options.h"
+#include "srvinit.h"
+#include "rgxutils.h"
+
+#include "os_apphint.h"
+#include "pvrsrv_apphint.h"
+
+#include "pvrsrv_tlstreams.h"
+#include "tlstream.h"
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+#include "physmem_test.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#if defined(__linux__)
+#include "km_apphint.h"
+#endif /* defined(__linux__) */
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define INFINITE_SLEEP_TIMEOUT 0ULL
+#endif
+
+/*! Wait 100ms before retrying deferred clean-up again */
+#define CLEANUP_THREAD_WAIT_RETRY_TIMEOUT 100000ULL
+
+/*! Wait 8hrs when no deferred clean-up required. Allows a poll several times
+ * a day to check for any missed clean-up. */
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+#define CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT INFINITE_SLEEP_TIMEOUT
+#else
+#define CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT 28800000000ULL
+#endif
+
+/*! When unloading try a few times to free everything remaining on the list */
+#define CLEANUP_THREAD_UNLOAD_RETRY 4
+
+#define PVRSRV_TL_CTRL_STREAM_SIZE 4096
+
+static PVRSRV_DATA	*gpsPVRSRVData;
+static IMG_UINT32 g_ui32InitFlags;
+
+/* mark which parts of Services were initialised */
+#define		INIT_DATA_ENABLE_PDUMPINIT	0x1U
+
+#define CLEANUP_QUEUE_TOTAL_DPF "QUEUED: %1.5d "
+#define CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE sizeof(CLEANUP_QUEUE_TOTAL_DPF)+1
+
+#define CLEANUP_STRING_SUMMARY_MAX_LEN (CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE+(PVRSRV_CLEANUP_TYPE_LAST * CLEANUP_TYPE_ITEM_DPF_MAX_SIZE))
+
+static IMG_CHAR *_ConcatCleanupString(IMG_CHAR *cleanupString)
+{
+	IMG_UINT32 uiLoop;
+	IMG_UINT32 strSize = CLEANUP_STRING_SUMMARY_MAX_LEN;
+
+	PVRSRV_DATA *psPVRSRVData =  PVRSRVGetPVRSRVData();
+
+	OSSNPrintf(cleanupString, CLEANUP_QUEUE_TOTAL_DPF_MAX_SIZE, CLEANUP_QUEUE_TOTAL_DPF,
+				OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued));
+
+	PVR_ASSERT(PVRSRV_CLEANUP_TYPE_CONNECTION == 1);
+
+	for (uiLoop=PVRSRV_CLEANUP_TYPE_CONNECTION; (strSize > CLEANUP_TYPE_ITEM_DPF_MAX_SIZE) && (uiLoop < PVRSRV_CLEANUP_TYPE_LAST); uiLoop++)
+	{
+		IMG_CHAR acTempQueued[CLEANUP_TYPE_ITEM_DPF_MAX_SIZE];
+		OSSNPrintf(acTempQueued, CLEANUP_TYPE_ITEM_DPF_MAX_SIZE, CLEANUP_TYPE_ITEM_DPF,
+				PVRSRVGetCleanupName(uiLoop),
+				OSAtomicRead(&psPVRSRVData->i32CleanupItemTypes[uiLoop]));
+		OSStringLCat(cleanupString, acTempQueued, strSize);
+	}
+
+	return cleanupString;
+}
+
+/* Callback to dump info of cleanup thread in debug_dump */
+static void CleanupThreadDumpInfo(DUMPDEBUG_PRINTF_FUNC* pfnDumpDebugPrintf,
+                                  void *pvDumpDebugFile)
+{
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_DUMPDEBUG_LOG("    Number of deferred cleanup items: %s", _ConcatCleanupString(acCleanupString));
+
+	PVR_DUMPDEBUG_LOG("    Number of deferred cleanup items dropped after "
+					  "retry limit reached : %d",
+					  OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsNotCompleted));
+}
+
+static void _CleanupThreadDecrementStats(PVRSRV_DATA *psPVRSRVData,
+                                         PVRSRV_CLEANUP_TYPE eCleanupType)
+{
+#if defined(DEBUG)
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+#endif
+
+	PVR_ASSERT(OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsQueued) >= 0);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "BEFORE REMOVING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+	OSAtomicDecrement(&psPVRSRVData->i32NumCleanupItemsQueued);
+
+	if ((eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) || ((eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item passed: %d", eCleanupType));
+	}
+	else
+	{
+		PVR_ASSERT(OSAtomicRead(&psPVRSRVData->i32CleanupItemTypes[eCleanupType]) >= 0);
+		OSAtomicDecrement(&psPVRSRVData->i32CleanupItemTypes[eCleanupType]);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "AFTER REMOVING ----- %s", _ConcatCleanupString(acCleanupString)));
+}
+
+#if defined(DEBUG)
+static void _CleanupThreadWorkListDump(PVRSRV_DATA *psPVRSRVData)
+{
+	DLLIST_NODE *psNode;
+	DLLIST_NODE *psNextNode;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+	char pszCleanupLog[128];
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* Iterate over all devices. */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		if (dllist_is_empty(&psDeviceNode->sCleanupThreadWorkList))
+		{
+			OSSNPrintf(pszCleanupLog, 128, "Dev_%p: CLEAN", psDeviceNode);
+			PVR_LOG(("%s", pszCleanupLog));
+			continue;
+		}
+
+		OSSNPrintf(pszCleanupLog, 128, "Dev_%p: TASKS", psDeviceNode);
+		PVR_LOG(("%s", pszCleanupLog));
+
+		/* Iterate over all cleanup items. */
+		dllist_foreach_node(&psDeviceNode->sCleanupThreadWorkList, psNode, psNextNode)
+		{
+			PVRSRV_CLEANUP_THREAD_WORK *psData;
+
+			psData = IMG_CONTAINER_OF(psNode, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+
+			PVR_ASSERT(psData != NULL);
+
+			if ((psData->eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) ||
+				((psData->eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item found: %d.", psData->eCleanupType));
+				continue;
+			}
+
+			OSSNPrintf(pszCleanupLog, 128, "+ %p: type %u, depends-HW %s (%s:%d)",
+					   psData,
+					   psData->eCleanupType,
+					   (psData->bDependsOnHW) ? "Yes" : " No",
+					   psData->pszFun, psData->ui32LineNum);
+			PVR_LOG(("%s", pszCleanupLog));
+		}
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+}
+#endif
+
+/* Add work to the cleanup thread work list.
+ * The work item will be executed by the cleanup thread
+ */
+#if defined(DEBUG)
+void PVRSRVCleanupThreadAddWork_Debug(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_CLEANUP_THREAD_WORK *psData,
+	const char *pszFun, const unsigned int ui32LineNum)
+#else
+void PVRSRVCleanupThreadAddWork_Int(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_CLEANUP_THREAD_WORK *psData)
+#endif
+{
+#if defined(DEBUG)
+	IMG_CHAR acCleanupString[CLEANUP_STRING_SUMMARY_MAX_LEN];
+#endif
+
+	PVRSRV_DATA *psPVRSRVData;
+	PVRSRV_ERROR eError;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(psData != NULL);
+
+	if ((psData->eCleanupType <= PVRSRV_CLEANUP_TYPE_UNDEF) ||
+		((psData->eCleanupType >= PVRSRV_CLEANUP_TYPE_LAST)))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Incorrect cleanup item passed: %d.", psData->eCleanupType));
+	}
+
+	psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "AFTER QUEUEING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+
+	/* Cleanup thread work item added during device destructing:
+	   1. bDependsOnHW: Item may use device resources. Not safe to clean.
+	   2. !bDependsOnHW: Clean item immediately to prevent memory leak. */
+	if (psPVRSRVData->bUnload
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		|| psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK
+#endif
+		|| (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING && !psData->bDependsOnHW))
+	{
+		CLEANUP_THREAD_FN pfnFree = psData->pfnFree;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Cleanup thread has already quit: doing work immediately"));
+
+		eError = pfnFree(psData->pvData);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "BEFORE ADDING ----- %s", _ConcatCleanupString(acCleanupString)));
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to free resource "
+						"(callback " IMG_PFN_FMTSPEC "). "
+						"Immediate free will not be retried.",
+						pfnFree));
+		}
+	}
+	else
+	{
+		/*
+		 * Access psData before putting it in the work list.
+		 * Cleanup thread will free psData after it is done with the cleanup.
+		 * We should consider psData potentially freed after it was put into the worklist.
+		 */
+
+		OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsQueued);
+		OSAtomicIncrement(&psPVRSRVData->i32CleanupItemTypes[psData->eCleanupType]);
+#if defined(DEBUG)
+		psData->pszFun = pszFun;
+		psData->ui32LineNum = ui32LineNum;
+#endif
+
+		/* add this work item to the list */
+		OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+		OSAtomicIncrement(&psDeviceNode->i32NumCleanupItems);
+		dllist_add_to_tail(&psDeviceNode->sCleanupThreadWorkList, &psData->sNode);
+		OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+		/* signal the cleanup thread to ensure this item gets processed */
+		eError = OSEventObjectSignal(psPVRSRVData->hCleanupEventObject);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "AFTER ADDING ----- %s", _ConcatCleanupString(acCleanupString)));
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+	}
+}
+
+void PVRSRVCleanupThreadWaitForDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(DEBUG)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+#endif
+	IMG_INT32 i32NumCleanupItems;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+
+	if (gpsPVRSRVData->hCleanupThread == NULL)
+	{
+		return;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+	{
+		PVRSRV_ERROR eError;
+
+		if (gpsPVRSRVData->hCleanupEventObject)
+		{
+			eError = OSEventObjectSignal(gpsPVRSRVData->hCleanupEventObject);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+
+		i32NumCleanupItems = OSAtomicRead(&psDeviceNode->i32NumCleanupItems);
+
+		if (i32NumCleanupItems == 0)
+		{
+#if defined(DEBUG)
+			OS_SPINLOCK_FLAGS uiFlags;
+
+			OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+			PVR_LOG_IF_FALSE(dllist_is_empty(&psDeviceNode->sCleanupThreadWorkList),
+							 "Cleanup thread work list is not empty");
+
+			OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+#endif
+			break;
+		}
+
+		OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US / OS_THREAD_DESTROY_RETRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (i32NumCleanupItems != 0)
+	{
+		PVR_LOG(("Failed to flush device cleanup queue. %d items remain.", i32NumCleanupItems));
+	}
+
+#if defined(DEBUG)
+	_CleanupThreadWorkListDump(psPVRSRVData);
+#endif
+}
+
+static INLINE DLLIST_NODE *_CleanupThreadWorkListLast(PVRSRV_DATA *psPVRSRVData)
+{
+	DLLIST_NODE *psNode = NULL;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* We treat the Host device node as the last node in the list. */
+	psNode = dllist_get_prev_node(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList);
+	if (psNode != NULL)
+	{
+		OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+		return psNode;
+	}
+
+	/* Iterate over all devices and find the last node of the last device. */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		DLLIST_NODE *psCurrNode = dllist_get_prev_node(&psDeviceNode->sCleanupThreadWorkList);
+		if (psCurrNode != NULL)
+		{
+			psNode = psCurrNode;
+		}
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psNode;
+}
+
+/* Pop an item from the head of the cleanup thread work lists.
+ *
+ * This pops an item in a round robin manner:
+ *
+ * 1. It starts from the first device and moves to next device if no cleanup
+ *    item found.
+ * 2. If no items were found in the regular devices bound lists it moves to the
+ *    Host device list.
+ * 3. In case there was nothing in the Host device items list but there was
+ *    something in the devices lists, it returns first found device bound item
+ *
+ * If `*ppsDeviceNode` is not NULL the lookup of the next cleanup item will
+ * start from the next devices after `*ppsDeviceNode`. This makes sure that
+ * we always pop a first item from each device and move to the next device.
+ * We prevent "starvation" of some devices this way.
+ */
+static INLINE DLLIST_NODE *_CleanupThreadWorkListPop(PVRSRV_DATA *psPVRSRVData,
+                                                     PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL, *psFirstDevice = NULL;
+	DLLIST_NODE *psNode = NULL, *psFirstNode = NULL;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+
+	/* always need to start from the beginning of the list in case some of the
+	 * devices were already destroyed */
+	for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+	     psDeviceNode != NULL;
+	     psDeviceNode = psDeviceNode->psNext)
+	{
+		/* not safe to run cleanup during deinit */
+		if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+		{
+			continue;
+		}
+
+		psNode = dllist_get_next_node(&psDeviceNode->sCleanupThreadWorkList);
+
+		/* remember first non-NULL node and device in case no items are found
+		 * on later devices */
+		if (psNode != NULL && psFirstNode == NULL)
+		{
+			psFirstNode = psNode;
+			psFirstDevice = psDeviceNode;
+		}
+
+		/* to ensure that further device don't get starved skip the last device
+		 * until all devices are processed and then move to non-device bound
+		 * items */
+		if (*ppsDeviceNode != NULL) {
+			if (psDeviceNode == *ppsDeviceNode)
+			{
+				*ppsDeviceNode = NULL;
+			}
+
+			/* in case this is the last device and next iteration exits the
+			 * loop */
+			psNode = NULL;
+
+			continue;
+		}
+
+		if (psNode != NULL)
+		{
+			*ppsDeviceNode = psDeviceNode;
+			dllist_remove_node(psNode);
+
+			break;
+		}
+	}
+
+	/* if no item found in the regular devices check also the Host device */
+	if (psNode == NULL)
+	{
+		psNode = dllist_get_next_node(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList);
+		if (psNode != NULL)
+		{
+			dllist_remove_node(psNode);
+		}
+		*ppsDeviceNode = psPVRSRVData->psHostMemDeviceNode;
+	}
+
+	/* if no item found for the Host device check if there was a cleanup item
+	 * in one of the previous devices, this starts processing items from the
+	 * beginning without one extra empty call to "wrap around" */
+	if (psNode == NULL && psFirstNode != NULL)
+	{
+		psNode = psFirstNode;
+		*ppsDeviceNode = psFirstDevice;
+		dllist_remove_node(psNode);
+	}
+
+	OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	return psNode;
+}
+
+/* Process the cleanup thread work list */
+static IMG_BOOL _CleanupThreadProcessWorkList(PVRSRV_DATA *psPVRSRVData,
+                                              IMG_BOOL *pbUseHWTimeout)
+{
+	DLLIST_NODE *psNodeIter, *psNodeLast;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bNeedRetry = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_CLEANUP_TYPE eCleanupType;
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+
+	/* Reset HWTimeout Flag */
+	*pbUseHWTimeout = IMG_FALSE;
+
+	psNodeLast = _CleanupThreadWorkListLast(psPVRSRVData);
+	if (psNodeLast == NULL)
+	{
+		/* no elements to clean up */
+		return IMG_FALSE;
+	}
+
+	/* any callback functions which return error will be
+	 * moved to the back of the list, and additional items can be added
+	 * to the list at any time so we ensure we only iterate from the
+	 * head of the list to the current tail (since the tail may always
+	 * be changing)
+	 */
+	do
+	{
+		PVRSRV_CLEANUP_THREAD_WORK *psData;
+		CLEANUP_THREAD_FN pfnFree;
+		IMG_BOOL bRetry = IMG_FALSE;
+
+		psNodeIter = _CleanupThreadWorkListPop(psPVRSRVData, &psDeviceNode);
+		if (psNodeIter == NULL)
+		{
+			return IMG_FALSE;
+		}
+
+		psData = IMG_CONTAINER_OF(psNodeIter, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+
+		/* get the function pointer address here so we have access to it
+		 * in order to report the error in case of failure, without having
+		 * to depend on psData not having been freed
+		 */
+		pfnFree = psData->pfnFree;
+		eCleanupType = psData->eCleanupType;
+		eError = pfnFree(psData->pvData);
+
+		if (eError != PVRSRV_OK)
+		{
+			/* Move to back of the list, if this item's
+			 * retry count hasn't hit zero.
+			 */
+			if (CLEANUP_THREAD_IS_RETRY_TIMEOUT(psData))
+			{
+				if (CLEANUP_THREAD_RETRY_TIMEOUT_NOT_REACHED(psData))
+				{
+					bNeedRetry = IMG_TRUE;
+					bRetry = IMG_TRUE;
+					/* If any items require retry and are HW dependent
+					 * use the HW timeout
+					 */
+					if (psData->bDependsOnHW)
+					{
+						*pbUseHWTimeout = psData->bDependsOnHW;
+					}
+				}
+			}
+			else
+			{
+				if (psData->ui32RetryCount > 0)
+				{
+					psData->ui32RetryCount--;
+					bNeedRetry = IMG_TRUE;
+					bRetry = IMG_TRUE;
+					/* If any items require retry and are HW dependent
+					 * use the HW timeout
+					 */
+					if (psData->bDependsOnHW)
+					{
+						*pbUseHWTimeout = psData->bDependsOnHW;
+					}
+				}
+			}
+
+			/* If the work depends on HW then we should add it to the back of the list,
+			 * the cleanup thread will sleep for longer if required and the next MISR
+			 * from the device will wake the task again in which it might be ready.
+			 */
+			if (bRetry || psData->bDependsOnHW)
+			{
+				/* If any items on the work list depend on HW
+				 * and didn't get cleaned up.
+				 */
+				OSSpinLockAcquire(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+				dllist_add_to_tail(&psDeviceNode->sCleanupThreadWorkList, psNodeIter);
+				OSSpinLockRelease(psPVRSRVData->hCleanupThreadWorkListLock, uiFlags);
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to free resource (callback " IMG_PFN_FMTSPEC "). "
+				        "Retry limit reached", pfnFree));
+				OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsNotCompleted);
+				OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+				/* Dropping item */
+				_CleanupThreadDecrementStats(psPVRSRVData, eCleanupType);
+			}
+		}
+		else
+		{
+			/* Ok returned */
+			OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+			_CleanupThreadDecrementStats(psPVRSRVData, eCleanupType);
+		}
+	} while (psNodeIter != NULL && psNodeIter != psNodeLast);
+
+	return bNeedRetry;
+}
+
+// #define CLEANUP_DPFL PVR_DBG_WARNING
+#define CLEANUP_DPFL    PVR_DBG_MESSAGE
+
+/* Create/initialise data required by the cleanup thread,
+ * before the cleanup thread is started
+ */
+static PVRSRV_ERROR _CleanupThreadPrepare(PVRSRV_DATA *psPVRSRVData)
+{
+	PVRSRV_ERROR eError;
+
+	/* Create the clean up event object */
+
+	eError = OSEventObjectCreate("PVRSRV_CLEANUP_EVENTOBJECT", &gpsPVRSRVData->hCleanupEventObject);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Exit);
+
+	/* initialise the mutex and linked list required for the cleanup thread work list */
+
+	eError = OSSpinLockCreate(&psPVRSRVData->hCleanupThreadWorkListLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", Exit);
+
+Exit:
+	return eError;
+}
+
+static void CleanupThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_BOOL     bRetryWorkList = IMG_FALSE;
+	IMG_BOOL     bUseHWTimeout = IMG_FALSE;
+	IMG_HANDLE	 hOSEvent;
+	PVRSRV_ERROR eRc;
+	IMG_UINT32 uiUnloadRetry = 0;
+	DLLIST_NODE *psNodeIter, *psNodeLast;
+
+	/* Store the process id (pid) of the clean-up thread */
+	psPVRSRVData->cleanupThreadPid = OSGetCurrentProcessID();
+	psPVRSRVData->cleanupThreadTid = OSGetCurrentThreadID();
+
+	PVR_DPF((CLEANUP_DPFL, "CleanupThread: thread starting... "));
+
+	/* Open an event on the clean up event object so we can listen on it,
+	 * abort the clean up thread and driver if this fails.
+	 */
+	eRc = OSEventObjectOpen(psPVRSRVData->hCleanupEventObject, &hOSEvent);
+	PVR_ASSERT(eRc == PVRSRV_OK);
+
+	/* While the driver is in a good state and is not being unloaded
+	 * try to free any deferred items when signalled
+	 */
+	while (psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK)
+	{
+		IMG_UINT64 ui64Timeoutus;
+		if (psPVRSRVData->bUnload)
+		{
+			if (dllist_is_empty(&psPVRSRVData->psHostMemDeviceNode->sCleanupThreadWorkList) ||
+			    uiUnloadRetry > CLEANUP_THREAD_UNLOAD_RETRY)
+			{
+				break;
+			}
+			uiUnloadRetry++;
+		}
+
+		/* Wait until signalled for deferred clean up OR wait for a
+		 * short period if the previous deferred clean up was not able
+		 * to release all the resources before trying again.
+		 * Bridge lock re-acquired on our behalf before the wait call returns.
+		 */
+
+		if (bRetryWorkList && bUseHWTimeout)
+		{
+			/* If item depends on HW we are
+			 * waiting for GPU work to finish, so
+			 * use MAX_HW_TIME_US as timeout (this
+			 * will be set appropriately when
+			 * running on systems with emulated
+			 * hardware, etc).
+			 */
+			ui64Timeoutus = MAX_HW_TIME_US;
+		}
+		else
+		{
+			/* Use the default retry timeout. */
+			ui64Timeoutus = CLEANUP_THREAD_WAIT_RETRY_TIMEOUT;
+		}
+
+		eRc = OSEventObjectWaitKernel(hOSEvent,
+				bRetryWorkList ?
+				ui64Timeoutus :
+				CLEANUP_THREAD_WAIT_SLEEP_TIMEOUT);
+		if (eRc == PVRSRV_ERROR_TIMEOUT)
+		{
+			PVR_DPF((CLEANUP_DPFL, "CleanupThread: wait timeout"));
+		}
+		else if (eRc == PVRSRV_OK)
+		{
+			PVR_DPF((CLEANUP_DPFL, "CleanupThread: wait OK, signal received"));
+		}
+		else
+		{
+			PVR_LOG_ERROR(eRc, "OSEventObjectWaitKernel");
+		}
+
+		bRetryWorkList = _CleanupThreadProcessWorkList(psPVRSRVData,
+		                                               &bUseHWTimeout);
+	}
+
+	psNodeLast = _CleanupThreadWorkListLast(psPVRSRVData);
+	if (psNodeLast != NULL)
+	{
+		do
+		{
+			PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+			PVRSRV_CLEANUP_THREAD_WORK *psData;
+			psNodeIter = _CleanupThreadWorkListPop(psPVRSRVData, &psDeviceNode);
+			if (psNodeIter == NULL)
+			{
+				break;
+			}
+
+			psData = IMG_CONTAINER_OF(psNodeIter, PVRSRV_CLEANUP_THREAD_WORK, sNode);
+			OSAtomicIncrement(&psPVRSRVData->i32NumCleanupItemsNotCompleted);
+			OSAtomicDecrement(&psDeviceNode->i32NumCleanupItems);
+			/* Dropping item */
+			_CleanupThreadDecrementStats(psPVRSRVData, psData->eCleanupType);
+		}
+		while (psNodeIter != NULL && psNodeIter != psNodeLast);
+
+		PVR_DPF((PVR_DBG_ERROR, "Cleanup Thread Failed to free %d resources", OSAtomicRead(&psPVRSRVData->i32NumCleanupItemsNotCompleted)));
+	}
+
+	OSSpinLockDestroy(psPVRSRVData->hCleanupThreadWorkListLock);
+
+	eRc = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eRc, "OSEventObjectClose");
+
+	PVR_DPF((CLEANUP_DPFL, "CleanupThread: thread ending... "));
+}
+
+IMG_PID PVRSRVCleanupThreadGetPid(void)
+{
+	return gpsPVRSRVData->cleanupThreadPid;
+}
+
+uintptr_t PVRSRVCleanupThreadGetTid(void)
+{
+	return gpsPVRSRVData->cleanupThreadTid;
+}
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*
+ * Firmware is unresponsive.
+ * The Host will initiate a recovery process during which the
+ * Firmware and GPU are reset and returned to a working state.
+ */
+static PVRSRV_ERROR HandleFwHostSideRecovery(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32CtxIdx = 0U;
+	IMG_UINT32 ui32Nodes  = 0U;
+
+	OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+	/* Get the number of nodes in a linked list */
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		++ui32Nodes;
+	}
+
+	/* Any client contexts active at the moment? */
+	if (ui32Nodes > 0U)
+	{
+		/* Free the active context buffer previously allocated */
+		if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+			psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+		}
+
+		/* Setup allocations to store the active contexts */
+		eError = RGXSetupFwAllocation(psDevInfo,
+				RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+				(ui32Nodes + 1) * sizeof(RGXFWIF_ACTIVE_CONTEXT_BUF_DATA),
+				"FwSysActiveContextBufData",
+				&psDevInfo->psRGXFWIfActiveContextBufDesc,
+				(void *)  &psDevInfo->psRGXFWIfSysInit->sActiveContextBufBase.ui32Addr,
+				(void **) &psDevInfo->psRGXFWIfActiveContextBuf,
+				RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation",Error);
+
+		/* List of contexts to be rekicked by FW powering up the device */
+		dllist_foreach_node_backwards(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+		{
+			psDevInfo->psRGXFWIfActiveContextBuf[ui32CtxIdx].psContext =
+					RGXGetFWCommonContextAddrFromServerCommonCtx(psDevInfo, psNode);
+			++ui32CtxIdx;
+		}
+		/* Null context as the terminator marker */
+		psDevInfo->psRGXFWIfActiveContextBuf[ui32CtxIdx].psContext.ui32Addr = 0;
+	}
+	OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+
+	/* Host can't expect a response on power-down request as FW is in BAD state */
+	eError = PVRSRVSetDeviceCurrentPowerState(psDeviceNode->psPowerDev, PVRSRV_DEV_POWER_STATE_OFF);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVSetDeviceCurrentPowerState OFF", Error);
+
+	/* Flag to be set to notify FW while recovering from crash */
+	psDevInfo->psRGXFWIfSysInit->bFwHostRecoveryMode = IMG_TRUE;
+
+	/* Flush here because we have setup a fw alloc addr in the structure earlier */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit, FLUSH);
+
+	/* Power-on the device resetting GPU & FW */
+	OSLockAcquire(psDeviceNode->hPowerLock);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+			PVRSRV_DEV_POWER_STATE_ON,
+			PVRSRV_POWER_FLAGS_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVSetDevicePowerStateKM ON", Error);
+	OSLockRelease(psDeviceNode->hPowerLock);
+
+Error:
+	return eError;
+}
+#endif
+
+
+void PVRSRVDeviceSetState(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_DEVICE_STATE eNewDevState)
+{
+	if (eNewDevState == psDeviceNode->eDevState)
+	{
+		return;
+	}
+
+	switch (eNewDevState)
+	{
+		case PVRSRV_DEVICE_STATE_PCI_ERROR:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetDebugDevStateString(eNewDevState)));
+			psDeviceNode->eDevState = eNewDevState;
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+			PMRFreeZombies(psDeviceNode);
+#endif
+			break;
+		}
+		case PVRSRV_DEVICE_STATE_CREATING:
+		case PVRSRV_DEVICE_STATE_CREATED:
+		case PVRSRV_DEVICE_STATE_ACTIVE:
+		case PVRSRV_DEVICE_STATE_FROZEN:
+		case PVRSRV_DEVICE_STATE_DEINIT:
+		case PVRSRV_DEVICE_STATE_DESTRUCTING:
+		case PVRSRV_DEVICE_STATE_BAD:
+		{
+			/* PCI_ERROR is a terminal state. Reload driver to recover. */
+			if (eNewDevState != PVRSRV_DEVICE_STATE_PCI_ERROR)
+			{
+				psDeviceNode->eDevState = eNewDevState;
+			}
+			break;
+		}
+		case PVRSRV_DEVICE_STATE_UNDEFINED:
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Unknown state (%d)", __func__, eNewDevState));
+			break;
+		}
+	}
+}
+
+static void DevicesWatchdogThread_ForEachVaCb(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  va_list va)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+#endif
+	PVRSRV_DEVICE_HEALTH_STATUS *pePreviousHealthStatus, eHealthStatus;
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_DEBUG_DUMP_STATUS eDebugDumpState;
+	IMG_BOOL bCheckAfterTimePassed;
+
+	pePreviousHealthStatus = va_arg(va, PVRSRV_DEVICE_HEALTH_STATUS *);
+	/* IMG_BOOL (i.e. bool) is converted to int during default argument promotion
+	 * in variadic argument list. Thus, fetch an int to get IMG_BOOL */
+	bCheckAfterTimePassed = (IMG_BOOL) va_arg(va, int);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		return;
+	}
+
+	/* Block here if needed */
+	PVRSRVBlockIfFrozen(psDeviceNode);
+
+	if (psDeviceNode->pfnUpdateHealthStatus != NULL)
+	{
+		eError = psDeviceNode->pfnUpdateHealthStatus(psDeviceNode, bCheckAfterTimePassed);
+		PVR_WARN_IF_ERROR(eError, "pfnUpdateHealthStatus");
+	}
+	eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+	if (eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		if (eHealthStatus != *pePreviousHealthStatus)
+		{
+#if defined(SUPPORT_RGX)
+			if (!(psDevInfo->ui32DeviceFlags &
+				  RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN))
+#else
+			/* In this case we don't have an RGX device */
+			if (eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED)
+#endif
+			{
+				PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: "
+						 "Device status not OK!!!"));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX,
+								   NULL, NULL);
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+				HandleFwHostSideRecovery(psDeviceNode);
+#endif
+			}
+		}
+	}
+
+	*pePreviousHealthStatus = eHealthStatus;
+
+	/* Have we received request from FW to capture debug dump(could be due to HWR) */
+	eDebugDumpState = (PVRSRV_DEVICE_DEBUG_DUMP_STATUS)OSAtomicCompareExchange(
+						&psDeviceNode->eDebugDumpRequested,
+						PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE,
+						PVRSRV_DEVICE_DEBUG_DUMP_NONE);
+	if (PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE == eDebugDumpState)
+	{
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	}
+
+}
+
+#if defined(SUPPORT_RGX)
+static void HWPerfPeriodicHostEventsThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+
+	eError = OSEventObjectOpen(psPVRSRVData->hHWPerfHostPeriodicEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload && !psPVRSRVData->bHWPerfHostThreadStop)
+#else
+	while (!psPVRSRVData->bUnload && !psPVRSRVData->bHWPerfHostThreadStop)
+#endif
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode;
+		IMG_BOOL bInfiniteSleep = IMG_TRUE;
+
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)psPVRSRVData->ui32HWPerfHostThreadTimeout * 1000);
+		if (eError == PVRSRV_OK && (psPVRSRVData->bUnload || psPVRSRVData->bHWPerfHostThreadStop))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "HWPerfPeriodicHostEventsThread: Shutdown event received."));
+			break;
+		}
+
+		for (psDeviceNode = psPVRSRVData->psDeviceNodeList;
+		     psDeviceNode != NULL;
+		     psDeviceNode = psDeviceNode->psNext)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+			/* If the psDevInfo or hHWPerfHostStream are NULL it most
+			 * likely means that this device or stream has not been
+			 * initialised yet, so just skip */
+			if (psDevInfo == NULL || psDevInfo->hHWPerfHostStream == NULL)
+			{
+				continue;
+			}
+
+			/* Check if the HWPerf host stream is open for reading before writing
+			 * a packet, this covers cases where the event filter is not zeroed
+			 * before a reader disconnects. */
+			if (TLStreamIsOpenForReading(psDevInfo->hHWPerfHostStream))
+			{
+				/* As long as any of the streams is opened don't go into
+				 * indefinite sleep. */
+				bInfiniteSleep = IMG_FALSE;
+#if defined(SUPPORT_RGX)
+				RGXSRV_HWPERF_HOST_INFO(psDevInfo, RGX_HWPERF_INFO_EV_MEM64_USAGE);
+#endif
+			}
+		}
+
+		if (bInfiniteSleep)
+		{
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+			psPVRSRVData->ui32HWPerfHostThreadTimeout = INFINITE_SLEEP_TIMEOUT;
+#else
+			/* Use an 8 hour timeout if indefinite sleep is not supported. */
+			psPVRSRVData->ui32HWPerfHostThreadTimeout = 60 * 60 * 8 * 1000;
+#endif
+		}
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+#endif
+
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+typedef enum
+{
+	DWT_ST_INIT,
+	DWT_ST_SLEEP_POWERON,
+	DWT_ST_SLEEP_POWEROFF,
+	DWT_ST_SLEEP_DEFERRED,
+	DWT_ST_FINAL
+} DWT_STATE;
+
+typedef enum
+{
+	DWT_SIG_POWERON,
+	DWT_SIG_POWEROFF,
+	DWT_SIG_TIMEOUT,
+	DWT_SIG_UNLOAD,
+	DWT_SIG_ERROR
+} DWT_SIGNAL;
+
+static inline IMG_BOOL _DwtIsPowerOn(PVRSRV_DATA *psPVRSRVData)
+{
+	return List_PVRSRV_DEVICE_NODE_IMG_BOOL_Any(psPVRSRVData->psDeviceNodeList,
+	                                            PVRSRVIsDevicePowered);
+}
+
+static inline void _DwtCheckHealthStatus(PVRSRV_DATA *psPVRSRVData,
+                                         PVRSRV_DEVICE_HEALTH_STATUS *peStatus,
+                                         IMG_BOOL bTimeOut)
+{
+	List_PVRSRV_DEVICE_NODE_ForEach_va(psPVRSRVData->psDeviceNodeList,
+	                                   DevicesWatchdogThread_ForEachVaCb,
+	                                   peStatus,
+	                                   bTimeOut);
+}
+
+static DWT_SIGNAL _DwtWait(PVRSRV_DATA *psPVRSRVData, IMG_HANDLE hOSEvent,
+                           IMG_UINT32 ui32Timeout)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64) ui32Timeout * 1000);
+
+
+	if (eError == PVRSRV_OK)
+	{
+		if (psPVRSRVData->bUnload)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Shutdown event"
+			        " received."));
+			return DWT_SIG_UNLOAD;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power state "
+			        "change event received."));
+
+			if (_DwtIsPowerOn(psPVRSRVData))
+			{
+				return DWT_SIG_POWERON;
+			}
+			else
+			{
+				return DWT_SIG_POWEROFF;
+			}
+		}
+	}
+	else if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		return DWT_SIG_TIMEOUT;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: Error (%d) when"
+	        " waiting for event!", eError));
+	return DWT_SIG_ERROR;
+}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+
+static void DevicesWatchdogThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	PVRSRV_DEVICE_HEALTH_STATUS ePreviousHealthStatus = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	DWT_STATE eState = DWT_ST_INIT;
+	const IMG_UINT32 ui32OnTimeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	const IMG_UINT32 ui32OffTimeout = INFINITE_SLEEP_TIMEOUT;
+#else
+	IMG_UINT32 ui32Timeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	/* Flag used to defer the sleep timeout change by 1 loop iteration.
+	 * This helps to ensure at least two health checks are performed before a long sleep.
+	 */
+	IMG_BOOL bDoDeferredTimeoutChange = IMG_FALSE;
+#endif
+
+	PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power off sleep time: %d.",
+			DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT));
+
+	/* Open an event on the devices watchdog event object so we can listen on it
+	   and abort the devices watchdog thread. */
+	eError = OSEventObjectOpen(psPVRSRVData->hDevicesWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+	/* Loop continuously checking the device status every few seconds. */
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload)
+#else
+	while (!psPVRSRVData->bUnload)
+#endif
+	{
+#if defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+
+		switch (eState)
+		{
+			case DWT_ST_INIT:
+			{
+				if (_DwtIsPowerOn(psPVRSRVData))
+				{
+					eState = DWT_ST_SLEEP_POWERON;
+				}
+				else
+				{
+					eState = DWT_ST_SLEEP_POWEROFF;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_POWERON:
+			{
+				DWT_SIGNAL eSignal = _DwtWait(psPVRSRVData, hOSEvent,
+				                                    ui32OnTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_POWEROFF:
+						eState = DWT_ST_SLEEP_DEFERRED;
+						break;
+					case DWT_SIG_TIMEOUT:
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_TRUE);
+						/* self-transition */
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_POWEROFF:
+			{
+				DWT_SIGNAL eSignal = _DwtWait(psPVRSRVData, hOSEvent,
+				                                    ui32OffTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						eState = DWT_ST_SLEEP_POWERON;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_POWEROFF:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_TIMEOUT:
+						/* self-transition */
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_TRUE);
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_SLEEP_DEFERRED:
+			{
+				DWT_SIGNAL eSignal =_DwtWait(psPVRSRVData, hOSEvent,
+				                                   ui32OnTimeout);
+
+				switch (eSignal) {
+					case DWT_SIG_POWERON:
+						eState = DWT_ST_SLEEP_POWERON;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_POWEROFF:
+						/* self-transition, nothing to do */
+						break;
+					case DWT_SIG_TIMEOUT:
+						eState = DWT_ST_SLEEP_POWEROFF;
+						_DwtCheckHealthStatus(psPVRSRVData,
+						                      &ePreviousHealthStatus,
+						                      IMG_FALSE);
+						break;
+					case DWT_SIG_UNLOAD:
+						eState = DWT_ST_FINAL;
+						break;
+					case DWT_SIG_ERROR:
+						/* deliberately ignored */
+						break;
+				}
+
+				break;
+			}
+			case DWT_ST_FINAL:
+				/* the loop should terminate on next spin if this state is
+				 * reached so nothing to do here. */
+				break;
+		}
+
+#else /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+		IMG_BOOL bPwrIsOn = IMG_FALSE;
+		IMG_BOOL bTimeOut = IMG_FALSE;
+
+		/* Wait time between polls (done at the start of the loop to allow devices
+		   to initialise) or for the event signal (shutdown or power on). */
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+
+		if (eError == PVRSRV_OK)
+		{
+			if (psPVRSRVData->bUnload)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Shutdown event received."));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "DevicesWatchdogThread: Power state change event received."));
+			}
+		}
+		else if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			/* If timeout do nothing otherwise print warning message. */
+			PVR_DPF((PVR_DBG_ERROR, "DevicesWatchdogThread: "
+					"Error (%d) when waiting for event!", eError));
+		}
+		else
+		{
+			bTimeOut = IMG_TRUE;
+		}
+
+		OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+		List_PVRSRV_DEVICE_NODE_ForEach_va(psPVRSRVData->psDeviceNodeList,
+		                                   DevicesWatchdogThread_ForEachVaCb,
+		                                   &ePreviousHealthStatus,
+		                                   bTimeOut);
+		bPwrIsOn = List_PVRSRV_DEVICE_NODE_IMG_BOOL_Any(psPVRSRVData->psDeviceNodeList,
+														PVRSRVIsDevicePowered);
+		OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+		if (bPwrIsOn || psPVRSRVData->ui32DevicesWatchdogPwrTrans)
+		{
+			psPVRSRVData->ui32DevicesWatchdogPwrTrans = 0;
+			ui32Timeout = psPVRSRVData->ui32DevicesWatchdogTimeout = DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+			bDoDeferredTimeoutChange = IMG_FALSE;
+		}
+		else
+		{
+			/* First, check if the previous loop iteration signalled a need to change the timeout period */
+			if (bDoDeferredTimeoutChange)
+			{
+				ui32Timeout = psPVRSRVData->ui32DevicesWatchdogTimeout = DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT;
+				bDoDeferredTimeoutChange = IMG_FALSE;
+			}
+			else
+			{
+				/* Signal that we need to change the sleep timeout in the next loop iteration
+				 * to allow the device health check code a further iteration at the current
+				 * sleep timeout in order to determine bad health (e.g. stalled cCCB) by
+				 * comparing past and current state snapshots */
+				bDoDeferredTimeoutChange = IMG_TRUE;
+			}
+		}
+
+#endif /* defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP) */
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+#if defined(SUPPORT_AUTOVZ)
+static void AutoVzWatchdogThread_ForEachCb(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	if (psDeviceNode->pfnUpdateAutoVzWatchdog != NULL)
+	{
+		psDeviceNode->pfnUpdateAutoVzWatchdog(psDeviceNode);
+	}
+}
+
+static void AutoVzWatchdogThread(void *pvData)
+{
+	PVRSRV_DATA *psPVRSRVData = pvData;
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32Timeout = PVR_AUTOVZ_WDG_PERIOD_MS / 3;
+
+	/* Open an event on the devices watchdog event object so we can listen on it
+	   and abort the devices watchdog thread. */
+	eError = OSEventObjectOpen(psPVRSRVData->hAutoVzWatchdogEvObj, &hOSEvent);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	while ((psPVRSRVData->eServicesState == PVRSRV_SERVICES_STATE_OK) &&
+			!psPVRSRVData->bUnload)
+#else
+	while (!psPVRSRVData->bUnload)
+#endif
+	{
+		/* Wait time between polls (done at the start of the loop to allow devices
+		   to initialise) or for the event signal (shutdown or power on). */
+		eError = OSEventObjectWaitKernel(hOSEvent, (IMG_UINT64)ui32Timeout * 1000);
+
+		List_PVRSRV_DEVICE_NODE_ForEach(psPVRSRVData->psDeviceNodeList,
+		                                AutoVzWatchdogThread_ForEachCb);
+	}
+
+	eError = OSEventObjectClose(hOSEvent);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_DATA *PVRSRVGetPVRSRVData(void)
+{
+	return gpsPVRSRVData;
+}
+
+static PVRSRV_ERROR InitialiseInfoPageTimeouts(PVRSRV_DATA *psPVRSRVData)
+{
+	if (NULL == psPVRSRVData)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_VALUE_RETRIES] = WAIT_TRY_COUNT;
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_VALUE_TIMEOUT_MS] =
+		((MAX_HW_TIME_US / 10000) + 1000);
+		/* TIMEOUT_INFO_VALUE_TIMEOUT_MS resolves to...
+			vp       : 2000  + 1000
+			emu      : 2000  + 1000
+			rgx_nohw : 50    + 1000
+			plato    : 30000 + 1000 (VIRTUAL_PLATFORM or EMULATOR)
+			           50    + 1000 (otherwise)
+		*/
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_CONDITION_RETRIES] = 5;
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_CONDITION_TIMEOUT_MS] =
+		((MAX_HW_TIME_US / 10000) + 100);
+		/* TIMEOUT_INFO_CONDITION_TIMEOUT_MS resolves to...
+			vp       : 2000  + 100
+			emu      : 2000  + 100
+			rgx_nohw : 50    + 100
+			plato    : 30000 + 100 (VIRTUAL_PLATFORM or EMULATOR)
+			           50    + 100 (otherwise)
+		*/
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_TASK_QUEUE_RETRIES] = 10;
+
+	psPVRSRVData->pui32InfoPage[TIMEOUT_INFO_TASK_QUEUE_FLUSH_TIMEOUT_MS] =
+		MAX_HW_TIME_US / 1000U;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR PopulateInfoPageBridges(PVRSRV_DATA *psPVRSRVData)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psPVRSRVData);
+
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_PVR_BRIDGES] = gui32PVRBridges;
+
+#if defined(SUPPORT_RGX)
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_RGX_BRIDGES] = gui32RGXBridges;
+#else
+	psPVRSRVData->pui32InfoPage[BRIDGE_INFO_RGX_BRIDGES] = 0;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static void _ThreadsDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDbgRequestHandle,
+                                       IMG_UINT32 ui32VerbLevel,
+                                       DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                       void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(hDbgRequestHandle);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+		PVR_DUMPDEBUG_LOG("------[ Server Thread Summary ]------");
+		OSThreadDumpInfo(pfnDumpDebugPrintf, pvDumpDebugFile);
+	}
+}
+
+#define PVRVZMODE_STR_SIZE_MAX (10U)
+#define PVRVZMODE_BUFFER_SIZE (((PVRSRV_MAX_DEVICES)*(PVRVZMODE_STR_SIZE_MAX))+1)
+
+static const IMG_CHAR *const asModeStrings[] =
+{
+	"native",
+	"host",
+	"guest",
+	"default"
+};
+
+static void _InitDefaultVzDriverMode(PVRSRV_DATA *psPVRSRVData, void *pvAppHintState)
+{
+	char *pszMode, pszModeAppHint[PVRVZMODE_BUFFER_SIZE];
+	static const IMG_CHAR *pszDefault = "";
+	PVRSRV_DRIVER_MODE aeInitModes[PVRSRV_MAX_DEVICES];
+	IMG_BOOL bRet;
+	IMG_UINT ui32Dev;
+
+	for (ui32Dev = 0; ui32Dev < PVRSRV_MAX_DEVICES; ui32Dev++)
+	{
+		aeInitModes[ui32Dev] = DRIVER_MODE_DEFAULT;
+	}
+
+	pszMode = pszModeAppHint;
+	pszModeAppHint[0] = '\0';
+
+	bRet = (IMG_BOOL) OSGetAppHintSTRING(APPHINT_NO_DEVICE,
+						pvAppHintState,
+						DriverMode,
+						pszDefault,
+						pszModeAppHint,
+						PVRVZMODE_BUFFER_SIZE);
+	PVR_GOTO_IF_FALSE(bRet, uninit_default);
+
+	ui32Dev = 0;
+
+	while (*pszMode && (ui32Dev < PVRSRV_MAX_DEVICES))
+	{
+		PVRSRV_DRIVER_MODE eMode;
+
+		while (*pszMode == ',')
+		{
+			/* skip commas */
+			pszMode++;
+		}
+
+		for (eMode = DRIVER_MODE_NATIVE; eMode <= DRIVER_MODE_DEFAULT; eMode++)
+		{
+			/* if no matching strings found, device's mode remains DEFAULT */
+			if (OSStringNCompare(pszMode,
+								 asModeStrings[eMode],
+								 OSStringLength(asModeStrings[eMode])) == 0)
+			{
+				aeInitModes[ui32Dev] = eMode;
+			}
+		}
+
+		while ((*pszMode != '\0') && (*pszMode != ','))
+		{
+			/* advance until the next command or string end */
+			pszMode++;
+		}
+
+		ui32Dev++;
+	}
+
+uninit_default:
+	for (ui32Dev = 0; ui32Dev < PVRSRV_MAX_DEVICES; ui32Dev++)
+	{
+		psPVRSRVData->aeModuleParamDriverMode[ui32Dev] = aeInitModes[ui32Dev];
+	}
+}
+
+PVRSRV_ERROR
+PVRSRVCommonDriverInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVRSRV_DATA	*psPVRSRVData = NULL;
+
+	IMG_UINT32 ui32AppHintCleanupThreadPriority;
+	IMG_UINT32 ui32AppHintWatchdogThreadPriority;
+	IMG_BOOL bEnablePageFaultDebug;
+	IMG_BOOL bEnableFullSyncTracking;
+
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+
+	/*
+	 * As this function performs one time driver initialisation, use the
+	 * Services global device-independent data to determine whether or not
+	 * this function has already been called.
+	 */
+	if (gpsPVRSRVData)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Driver already initialised", __func__));
+		return PVRSRV_ERROR_ALREADY_EXISTS;
+	}
+
+#if defined(SUPPORT_RGX)
+	RGXBridgeDriverInit();
+#endif
+
+	eError = DIInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	eError = PVRSRVStatsInitialise();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	eError = PVRDIImplBrgRegister();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	eError = HTB_CreateDIEntry();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/*
+	 * Allocate the device-independent data. Do NOT track this allocation.
+	 */
+	psPVRSRVData = OSAllocZMemNoStats(sizeof(*gpsPVRSRVData));
+	PVR_GOTO_IF_NOMEM(psPVRSRVData, eError, Error);
+
+	/* Now it is set up, point gpsPVRSRVData to the actual data */
+	gpsPVRSRVData = psPVRSRVData;
+
+	eError = OSWRLockCreate(&gpsPVRSRVData->hDeviceNodeListLock);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Register the driver context debug table */
+	eError = PVRSRVRegisterDriverDbgTable();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Register the Server Thread Debug notifier */
+	eError = PVRSRVRegisterDriverDbgRequestNotify(&gpsPVRSRVData->hThreadsDbgReqNotify,
+		                                          _ThreadsDebugRequestNotify,
+		                                          DEBUG_REQUEST_SRV,
+		                                          NULL);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/*
+	 * Initialise the server bridges
+	 */
+	eError = ServerBridgeInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = DevmemIntInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = DebugCommonInitDriver();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = BridgeDispatcherInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Init any OS specific's */
+	eError = OSInitEnvData();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Early init. server cache maintenance */
+	eError = CacheOpInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RIInitKM();
+#endif
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG;
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnablePageFaultDebug,
+			&bAppHintDefault, &bEnablePageFaultDebug);
+	OSFreeAppHintState(pvAppHintState);
+
+	if (bEnablePageFaultDebug)
+	{
+		eError = DevicememHistoryInitKM();
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevicememHistoryInitKM", Error);
+	}
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	eError = CPUMappingHistoryInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	eError = PMRInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = DCInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+#endif
+
+	/* Initialise overall system state */
+	gpsPVRSRVData->eServicesState = PVRSRV_SERVICES_STATE_OK;
+
+	/* Create an event object */
+	eError = OSEventObjectCreate("PVRSRV_GLOBAL_EVENTOBJECT", &gpsPVRSRVData->hGlobalEventObject);
+	PVR_GOTO_IF_ERROR(eError, Error);
+	gpsPVRSRVData->ui32GEOConsecutiveTimeouts = 0;
+
+	eError = PVRSRVCmdCompleteInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = PVRSRVHandleInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = PVRSRV_APPHINT_CLEANUPTHREADPRIORITY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, CleanupThreadPriority,
+	                     &ui32AppHintDefault, &ui32AppHintCleanupThreadPriority);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_WATCHDOGTHREADPRIORITY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, WatchdogThreadPriority,
+	                     &ui32AppHintDefault, &ui32AppHintWatchdogThreadPriority);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFULLSYNCTRACKING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnableFullSyncTracking,
+			&bAppHintDefault, &bEnableFullSyncTracking);
+
+	_InitDefaultVzDriverMode(gpsPVRSRVData, pvAppHintState);
+
+
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	eError = HostMemDeviceCreate(&gpsPVRSRVData->psHostMemDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = _CleanupThreadPrepare(gpsPVRSRVData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CleanupThreadPrepare", Error);
+
+	/* Create a thread which is used to do the deferred cleanup */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hCleanupThread,
+	                                "pvr_defer_free",
+	                                CleanupThread,
+	                                CleanupThreadDumpInfo,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                ui32AppHintCleanupThreadPriority);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:1", Error);
+
+	/* Create the devices watchdog event object */
+	eError = OSEventObjectCreate("PVRSRV_DEVICESWATCHDOG_EVENTOBJECT", &gpsPVRSRVData->hDevicesWatchdogEvObj);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Error);
+
+	/* Create a thread which is used to detect fatal errors */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hDevicesWatchdogThread,
+	                                "pvr_device_wdg",
+	                                DevicesWatchdogThread,
+	                                NULL,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                ui32AppHintWatchdogThreadPriority);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:2", Error);
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Create the devices watchdog event object */
+	eError = OSEventObjectCreate("PVRSRV_AUTOVZ_WATCHDOG_EVENTOBJECT", &gpsPVRSRVData->hAutoVzWatchdogEvObj);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectCreate", Error);
+
+	/* Create a thread that maintains the FW-KM connection by regularly updating the virtualization watchdog */
+	eError = OSThreadCreatePriority(&gpsPVRSRVData->hAutoVzWatchdogThread,
+	                                "pvr_autovz_wdg",
+	                                AutoVzWatchdogThread,
+	                                NULL,
+	                                IMG_TRUE,
+	                                gpsPVRSRVData,
+	                                OS_THREAD_HIGHEST_PRIORITY);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSThreadCreatePriority:3", Error);
+#endif /* SUPPORT_AUTOVZ */
+
+#if defined(SUPPORT_RGX)
+	eError = OSLockCreate(&gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", Error);
+#endif
+
+	/* Initialise the Transport Layer */
+	eError = TLInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	/* Initialise pdump */
+	eError = PDUMPINIT();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	g_ui32InitFlags |= INIT_DATA_ENABLE_PDUMPINIT;
+
+	/* Initialise TL control stream */
+	eError = TLStreamCreate(&psPVRSRVData->hTLCtrlStream,
+	                        PVRSRV_TL_CTRL_STREAM,
+	                        PVRSRV_TL_CTRL_STREAM_SIZE,
+	                        TL_OPMODE_DROP_OLDEST,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+	                        NULL,
+                            NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "TLStreamCreate");
+		psPVRSRVData->hTLCtrlStream = NULL;
+	}
+
+	eError = InfoPageCreate(psPVRSRVData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InfoPageCreate", Error);
+
+
+	/* Initialise the Timeout Info */
+	eError = InitialiseInfoPageTimeouts(psPVRSRVData);
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	eError = PopulateInfoPageBridges(psPVRSRVData);
+
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+	if (bEnableFullSyncTracking)
+	{
+		psPVRSRVData->pui32InfoPage[DEBUG_FEATURE_FLAGS] |= DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED;
+	}
+	if (bEnablePageFaultDebug)
+	{
+		psPVRSRVData->pui32InfoPage[DEBUG_FEATURE_FLAGS] |= DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED;
+	}
+
+	{
+		IMG_UINT64 *pui64InfoPage;
+
+		pui64InfoPage = IMG_OFFSET_ADDR_DW(psPVRSRVData->pui32InfoPage,
+		                                   DEVMEM_INFO_PHYS_BUF_MAX_SIZE);
+		*pui64InfoPage = PMR_MAX_SUPPORTED_SIZE;
+	}
+
+	/* Initialise the Host Trace Buffer */
+	eError = HTBInit();
+	PVR_GOTO_IF_ERROR(eError, Error);
+
+#if defined(SUPPORT_RGX)
+	RGXHWPerfClientInitAppHintCallbacks();
+#endif
+
+	/* Late init. client cache maintenance via info. page */
+	eError = CacheOpInit2();
+	PVR_LOG_GOTO_IF_ERROR(eError, "CacheOpInit2", Error);
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	eError = SyncFbRegisterSyncFunctions();
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncFbRegisterSyncFunctions", Error);
+#endif
+
+#if defined(PDUMP)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	/* If initialising the device on first connection, we will
+	 * bind PDump capture to the first device we connect to later.
+	 */
+	psPVRSRVData->ui32PDumpBoundDevice = PVRSRV_MAX_DEVICES;
+#else
+	/* If not initialising the device on first connection, bind PDump
+	 * capture to device 0. This is because we need to capture PDump
+	 * during device initialisation but only want to capture PDump for
+	 * a single device (by default, device 0).
+	 */
+	psPVRSRVData->ui32PDumpBoundDevice = 0;
+#endif
+#endif
+
+	return 0;
+
+Error:
+	PVRSRVCommonDriverDeInit();
+	return eError;
+}
+
+void
+PVRSRVCommonDriverDeInit(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bEnablePageFaultDebug = IMG_FALSE;
+
+	if (gpsPVRSRVData == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: missing device-independent data",
+				 __func__));
+		return;
+	}
+
+	if (gpsPVRSRVData->pui32InfoPage != NULL)
+	{
+		bEnablePageFaultDebug = GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED;
+	}
+
+	gpsPVRSRVData->bUnload = IMG_TRUE;
+
+#if defined(SUPPORT_RGX)
+	PVRSRVDestroyHWPerfHostThread();
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock)
+	{
+		OSLockDestroy(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+		gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock = NULL;
+	}
+#endif
+
+	if (gpsPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectSignal(gpsPVRSRVData->hGlobalEventObject);
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Stop and cleanup the devices watchdog thread */
+	if (gpsPVRSRVData->hAutoVzWatchdogThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hAutoVzWatchdogEvObj)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hAutoVzWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hAutoVzWatchdogThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hAutoVzWatchdogThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hAutoVzWatchdogEvObj)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hAutoVzWatchdogEvObj);
+		gpsPVRSRVData->hAutoVzWatchdogEvObj = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+#endif /* SUPPORT_AUTOVZ */
+
+	/* Stop and cleanup the devices watchdog thread */
+	if (gpsPVRSRVData->hDevicesWatchdogThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hDevicesWatchdogEvObj)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hDevicesWatchdogEvObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hDevicesWatchdogThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hDevicesWatchdogThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hDevicesWatchdogEvObj)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hDevicesWatchdogEvObj);
+		gpsPVRSRVData->hDevicesWatchdogEvObj = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	/* Stop and cleanup the deferred clean up thread, event object and
+	 * deferred context list.
+	 */
+	if (gpsPVRSRVData->hCleanupThread)
+	{
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			if (gpsPVRSRVData->hCleanupEventObject)
+			{
+				eError = OSEventObjectSignal(gpsPVRSRVData->hCleanupEventObject);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+			}
+
+			eError = OSThreadDestroy(gpsPVRSRVData->hCleanupThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hCleanupThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+	}
+
+	if (gpsPVRSRVData->hCleanupEventObject)
+	{
+		eError = OSEventObjectDestroy(gpsPVRSRVData->hCleanupEventObject);
+		gpsPVRSRVData->hCleanupEventObject = NULL;
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+	}
+
+	/* Tear down the HTB before PVRSRVHandleDeInit() removes its TL handle */
+	/* HTB De-init happens in device de-registration currently */
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	/* Tear down CacheOp framework information page first */
+	CacheOpDeInit2();
+
+	/* Clean up information page */
+	InfoPageDestroy(gpsPVRSRVData);
+
+	/* Close the TL control plane stream. */
+	if (gpsPVRSRVData->hTLCtrlStream != NULL)
+	{
+		TLStreamClose(gpsPVRSRVData->hTLCtrlStream);
+	}
+
+	/* deinitialise pdump */
+	if ((g_ui32InitFlags & INIT_DATA_ENABLE_PDUMPINIT) > 0)
+	{
+		PDUMPDEINIT();
+	}
+
+	/* Clean up Transport Layer resources that remain */
+	TLDeInit();
+
+	HostMemDeviceDestroy(gpsPVRSRVData->psHostMemDeviceNode);
+	gpsPVRSRVData->psHostMemDeviceNode = NULL;
+
+	eError = PVRSRVHandleDeInit();
+	PVR_LOG_IF_ERROR(eError, "PVRSRVHandleDeInit");
+
+	/* destroy event object */
+	if (gpsPVRSRVData->hGlobalEventObject)
+	{
+		OSEventObjectDestroy(gpsPVRSRVData->hGlobalEventObject);
+		gpsPVRSRVData->hGlobalEventObject = NULL;
+	}
+
+	PVRSRVCmdCompleteDeinit();
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = DCDeInit();
+	PVR_LOG_IF_ERROR(eError, "DCDeInit");
+#endif
+
+	eError = PMRDeInit();
+	PVR_LOG_IF_ERROR(eError, "PMRDeInit");
+
+	BridgeDispatcherDeinit();
+
+#if defined(PVRSRV_PHYSMEM_CPUMAP_HISTORY)
+	CPUMappingHistoryDeInit();
+#endif
+
+	if (bEnablePageFaultDebug)
+	{
+		/* Clear all allocated history tracking data */
+		DevicememHistoryDeInitKM();
+	}
+
+	CacheOpDeInit();
+
+	OSDeInitEnvData();
+
+	(void) DevmemIntDeInit();
+
+	ServerBridgeDeInit();
+
+	HTB_DestroyDIEntry();
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDestroyDI();		/* Stage 1 freeing */
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+	DebugCommonDeInitDriver();
+
+	DIDeInit();
+
+	if (gpsPVRSRVData->hThreadsDbgReqNotify)
+	{
+		PVRSRVUnregisterDriverDbgRequestNotify(gpsPVRSRVData->hThreadsDbgReqNotify);
+	}
+
+	PVRSRVUnregisterDriverDbgTable();
+
+	OSWRLockDestroy(gpsPVRSRVData->hDeviceNodeListLock);
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsDestroy();		/* Stage 2 freeing */
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RIDeInitKM();
+#endif
+#endif /* PVRSRV_ENABLE_PROCESS_STATS */
+
+	OSFreeMemNoStats(gpsPVRSRVData);		/* Not trackable */
+
+	gpsPVRSRVData = NULL;
+}
+
+static void _SysDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	/* Only dump info once */
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hDebugRequestHandle;
+
+	PVR_UNREFERENCED_PARAMETER(ui32VerbLevel);
+
+	PVR_DUMPDEBUG_LOG("------[ System Summary Device ID:%d ]------", psDeviceNode->sDevId.ui32InternalID);
+
+	switch (psDeviceNode->eCurrentSysPowerState)
+	{
+		case PVRSRV_SYS_POWER_STATE_OFF:
+			PVR_DUMPDEBUG_LOG("Device System Power State: OFF");
+			break;
+		case PVRSRV_SYS_POWER_STATE_ON:
+			PVR_DUMPDEBUG_LOG("Device System Power State: ON");
+			break;
+		default:
+			PVR_DUMPDEBUG_LOG("Device System Power State: UNKNOWN (%d)",
+							   psDeviceNode->eCurrentSysPowerState);
+			break;
+	}
+
+	PVR_DUMPDEBUG_LOG("MaxHWTOut: %dus, WtTryCt: %d, WDGTOut(on,off): (%dms,%dms)",
+	                  MAX_HW_TIME_US, WAIT_TRY_COUNT, DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT, DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT);
+
+	SysDebugInfo(psDeviceNode->psDevConfig, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+PHYS_HEAP_CONFIG* PVRSRVFindPhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										   PHYS_HEAP_USAGE_FLAGS ui32Flags)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < psDevConfig->ui32PhysHeapCount; i++)
+	{
+		if (BITMASK_HAS(psDevConfig->pasPhysHeaps[i].ui32UsageFlags, ui32Flags))
+		{
+			return &psDevConfig->pasPhysHeaps[i];
+		}
+	}
+
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVAcquireInternalID(IMG_UINT32 *pui32InternalID)
+{
+	IMG_UINT32 ui32InternalID = 0;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	for (ui32InternalID = 0;
+		 ui32InternalID < PVRSRV_MAX_DEVICES;
+		 ui32InternalID++)
+	{
+		if (PVRSRVGetDeviceInstance(ui32InternalID) == NULL)
+		{
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (bFound)
+	{
+		*pui32InternalID = ui32InternalID;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		return PVRSRV_ERROR_NO_FREE_DEVICEIDS_AVAILABLE;
+	}
+}
+
+PVRSRV_ERROR PVRSRVCommonDeviceCreate(void *pvOSDevice,
+											 IMG_INT32 i32KernelDeviceID,
+											 PVRSRV_DEVICE_NODE **ppsDeviceNode)
+{
+	PVRSRV_DATA				*psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR			eError;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig;
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+	IMG_UINT32				ui32InternalID;
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	IMG_UINT32				ui32AppHintPhysMemTestPasses;
+#endif
+	void *pvAppHintState    = NULL;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_HANDLE				hProcessStats;
+#endif
+	IMG_BOOL				bAppHintDefault;
+	IMG_BOOL				bEnablePageFaultDebug = IMG_FALSE;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL				bAutoVzGPUPowerdown   = IMG_FALSE;
+#endif
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceCreate: DevId %d", i32KernelDeviceID);
+
+	/* Determine if we've got EnablePageFaultDebug set or not */
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEPAGEFAULTDEBUG;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, EnablePageFaultDebug,
+			&bAppHintDefault, &bEnablePageFaultDebug);
+
+#if defined(SUPPORT_AUTOVZ)
+	bAppHintDefault = PVRSRV_APPHINT_AUTOVZGPUPOWERDOWN;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState, AutoVzGPUPowerdown,
+			&bAppHintDefault, &bAutoVzGPUPowerdown);
+#endif
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	psDeviceNode = OSAllocZMemNoStats(sizeof(*psDeviceNode));
+	PVR_LOG_RETURN_IF_NOMEM(psDeviceNode, "psDeviceNode");
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Allocate process statistics */
+	eError = PVRSRVStatsRegisterProcess(&hProcessStats);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVStatsRegisterProcess", ErrorFreeDeviceNode);
+#endif
+
+	dllist_init(&psDeviceNode->sCleanupThreadWorkList);
+	OSAtomicWrite(&psDeviceNode->i32NumCleanupItems, 0);
+
+	/* Record setting of EnablePageFaultDebug in device-node */
+	psDeviceNode->bEnablePFDebug = bEnablePageFaultDebug;
+#if defined(SUPPORT_AUTOVZ)
+	psDeviceNode->bAutoVzAllowGPUPowerdown = bAutoVzGPUPowerdown;
+#endif
+	psDeviceNode->sDevId.i32KernelDeviceID = i32KernelDeviceID;
+	eError = PVRSRVAcquireInternalID(&ui32InternalID);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVAcquireInternalID", ErrorDeregisterStats);
+
+	eError = SysDevInit(pvOSDevice, &psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysDevInit", ErrorDeregisterStats);
+
+	/* Lock down the InternalID for this device by saving the value into
+	 * its device node. This is done after SysDevInit to allow the system
+	 * layer to identify the current device's the internal ID the same way. */
+	psDeviceNode->sDevId.ui32InternalID = ui32InternalID;
+
+	PVR_ASSERT(psDevConfig);
+	PVR_ASSERT(psDevConfig->pvOSDevice == pvOSDevice);
+	PVR_ASSERT(!psDevConfig->psDevNode);
+
+	if ((psDevConfig->eDefaultHeap != PVRSRV_PHYS_HEAP_GPU_LOCAL) &&
+	    (psDevConfig->eDefaultHeap != PVRSRV_PHYS_HEAP_CPU_LOCAL))
+	{
+		PVR_LOG_MSG(PVR_DBG_ERROR, "DEFAULT Heap is invalid, "
+		                           "it must be GPU_LOCAL or CPU_LOCAL");
+		PVR_LOG_GOTO_IF_ERROR(eError, "SysDevInit", ErrorDeregisterStats);
+	}
+	PVR_DPF((PVR_DBG_MESSAGE, "Device PhysHeap Default: %s",
+		(psDevConfig->eDefaultHeap == PVRSRV_PHYS_HEAP_CPU_LOCAL)
+		? "CPU_LOCAL" : "GPU_LOCAL"));
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_CREATING);
+
+	psDeviceNode->psDevConfig = psDevConfig;
+	psDevConfig->psDevNode = psDeviceNode;
+
+	if (psDevConfig->pfnGpuDomainPower)
+	{
+		psDeviceNode->eCurrentSysPowerState = psDevConfig->pfnGpuDomainPower(psDeviceNode);
+	}
+	else
+	{
+		/* If the System Layer doesn't provide a function to query the power state
+		 * of the system hardware, assume the system starting state below. */
+
+#if defined(SUPPORT_AUTOVZ)
+		psDeviceNode->eCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_ON;
+#else
+		psDeviceNode->eCurrentSysPowerState = PVRSRV_SYS_POWER_STATE_OFF;
+#endif
+	}
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	if (PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		IMG_UINT32 ui32AppHintDefault = 0;
+
+		/* Read AppHint - Configurable memory test pass count */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, PhysMemTestPasses,
+				&ui32AppHintDefault, &ui32AppHintPhysMemTestPasses);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		if (ui32AppHintPhysMemTestPasses > 0)
+		{
+			eError = PhysMemTest(psDevConfig, ui32AppHintPhysMemTestPasses);
+			PVR_LOG_GOTO_IF_ERROR(eError, "PhysMemTest", ErrorSysDevDeInit);
+		}
+	}
+#endif
+
+	/* Next update value will be 0xFFFFFFF7 since sync prim starts with 0xFFFFFFF6.
+	 * Has to be set before call to PMRInitDevice(). */
+	psDeviceNode->ui32NextMMUInvalidateUpdate = 0xFFFFFFF7U;
+#if defined(SUPPORT_PMR_DEFERRED_FREE) || defined(SUPPORT_MMU_DEFERRED_FREE)
+	psDeviceNode->uiPowerOffCounter = 0;
+	psDeviceNode->uiPowerOffCounterNext = 1;
+#endif
+
+	eError = PVRSRVRegisterDeviceDbgTable(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorPvzConnectionDeInit);
+
+	eError = PVRSRVPowerLockInit(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnregisterDbgTable);
+
+	eError = PhysHeapInitDeviceHeaps(psDeviceNode, psDevConfig);
+	PVR_GOTO_IF_ERROR(eError, ErrorPowerLockDeInit);
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	eError = PMRInitDevice(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorPhysHeapDeInitDeviceHeaps);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+#if defined(SUPPORT_RGX)
+	/* Requirements:
+	 *  registered GPU and FW local heaps */
+	/*  debug table */
+	eError = RGXRegisterDevice(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXRegisterDevice");
+		eError = PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+		goto ErrorPMRDeInitDevice;
+	}
+#endif
+
+	/* Inform the device layer PhysHeaps are now initialised so that device
+	 * specific heaps can be obtained along with carrying out any Vz setup. */
+	if (psDeviceNode->pfnPhysMemDeviceHeapsInit != NULL)
+	{
+		eError = psDeviceNode->pfnPhysMemDeviceHeapsInit(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorPMRDeInitDevice);
+	}
+
+	/* Carry out initialisation of a dedicated FW MMU data, if the FW CPU has
+	 * an MMU separate to the GPU MMU e.g. MIPS based FW. */
+	if (psDeviceNode->pfnFwMMUInit != NULL)
+	{
+		eError = psDeviceNode->pfnFwMMUInit(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorFwMMUDeinit);
+	}
+
+	eError = SyncServerInit(psDeviceNode);
+	PVR_GOTO_IF_ERROR(eError, ErrorDeInitRgx);
+
+	eError = SyncCheckpointInit(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SyncCheckpointInit", ErrorSyncCheckpointInit);
+
+	/*
+	 * This is registered before doing device specific initialisation to ensure
+	 * generic device information is dumped first during a debug request.
+	 */
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDeviceNode->hDbgReqNotify,
+		                                          psDeviceNode,
+		                                          _SysDebugRequestNotify,
+		                                          DEBUG_REQUEST_SYS,
+		                                          psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterDeviceDbgRequestNotify", ErrorRegDbgReqNotify);
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	eError = InitDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InitDVFS", ErrorDVFSInitFail);
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+	eError = InitPDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "InitPDVFS", ErrorPDVFSInitFail);
+#endif
+
+	OSAtomicWrite(&psDeviceNode->iNumClockSpeedChanges, 0);
+
+	OSWRLockCreate(&psDeviceNode->hPageFaultNotifyLock);
+	if (psDeviceNode->hPageFaultNotifyLock == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for PF notify list",
+		        __func__));
+		goto ErrorPageFaultLockFailCreate;
+	}
+
+	dllist_init(&psDeviceNode->sMemoryContextPageFaultNotifyListHead);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Registered device %p", psDeviceNode));
+	PVR_DPF((PVR_DBG_MESSAGE, "Register bank address = 0x%08lx",
+			 (unsigned long)psDevConfig->sRegsCpuPBase.uiAddr));
+	PVR_DPF((PVR_DBG_MESSAGE, "IRQ = %d", psDevConfig->ui32IRQ));
+
+/* SUPPORT_ALT_REGBASE is defined for rogue cores only */
+#if defined(SUPPORT_RGX) && defined(SUPPORT_ALT_REGBASE)
+	{
+		IMG_DEV_PHYADDR sRegsGpuPBase;
+
+		PhysHeapCpuPAddrToDevPAddr(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+		                           1,
+		                           &sRegsGpuPBase,
+		                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+
+		PVR_LOG(("%s: Using alternate Register bank GPU address: 0x%08lx (orig: 0x%08lx)", __func__,
+		         (unsigned long)psDevConfig->sAltRegsGpuPBase.uiAddr,
+		         (unsigned long)sRegsGpuPBase.uiAddr));
+	}
+#endif
+
+	eError = DebugCommonInitDevice(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DebugCommonInitDevice",
+	                      ErrorDestroyMemoryContextPageFaultNotifyListLock);
+
+	/* Create the devicemem_history hook for the device. We need to
+	 * have the debug-info instantiated before calling this.
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		eError = DevicememHistoryDeviceCreate(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevicememHistoryDeviceCreate", ErrorDebugCommonDeInitDevice);
+	}
+
+#if defined(__linux__)
+	/* Register the device specific AppHints so individual AppHints can be
+	 * configured before the FW is initialised. This must be called after
+	 * DebugCommonInitDevice() above as it depends on the created gpuXX/apphint
+	 * DI Group.
+	 */
+	{
+		int iError = pvr_apphint_device_register(psDeviceNode);
+		PVR_LOG_IF_FALSE(iError == 0, "pvr_apphint_device_register() failed");
+	}
+#endif /* defined(__linux__) */
+
+#if defined(SUPPORT_RGX)
+	RGXHWPerfInitAppHintCallbacks(psDeviceNode);
+#endif
+
+	/* Finally insert the device into the dev-list and set it as active */
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_InsertTail(&psPVRSRVData->psDeviceNodeList,
+									   psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices++;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+	/* Initialise the paravirtualised connection */
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		PvzConnectionInit(psDevConfig);
+		PVR_GOTO_IF_ERROR(eError, ErrorSysDevDeInit);
+	}
+
+	BIT_SET(psDevConfig->psDevNode->ui32VmState, RGXFW_HOST_DRIVER_ID);
+
+	*ppsDeviceNode = psDeviceNode;
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	/* Register the DVFS device now the device node is present in the dev-list */
+	eError = RegisterDVFSDevice(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RegisterDVFSDevice", ErrorRegisterDVFSDeviceFail);
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+#endif
+
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_CREATED);
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+ErrorRegisterDVFSDeviceFail:
+	/* Remove the device from the list */
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_Remove(psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices--;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+#if defined(__linux__)
+	pvr_apphint_device_unregister(psDeviceNode);
+#endif /* defined(__linux__) */
+
+	/* Remove the devicemem_history hook if we created it */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		DevicememHistoryDeviceDestroy(psDeviceNode);
+	}
+#endif
+
+ErrorDebugCommonDeInitDevice:
+	DebugCommonDeInitDevice(psDeviceNode);
+
+ErrorDestroyMemoryContextPageFaultNotifyListLock:
+	OSWRLockDestroy(psDeviceNode->hPageFaultNotifyLock);
+	psDeviceNode->hPageFaultNotifyLock = NULL;
+
+ErrorPageFaultLockFailCreate:
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+ErrorDVFSInitFail:
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+ErrorPDVFSInitFail:
+#endif
+
+	if (psDeviceNode->hDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceNode->hDbgReqNotify);
+	}
+
+ErrorRegDbgReqNotify:
+	SyncCheckpointDeinit(psDeviceNode);
+
+ErrorSyncCheckpointInit:
+	SyncServerDeinit(psDeviceNode);
+
+ErrorDeInitRgx:
+#if defined(SUPPORT_RGX)
+	DevDeInitRGX(psDeviceNode);
+#endif
+ErrorFwMMUDeinit:
+ErrorPMRDeInitDevice:
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	PMRDeInitDevice(psDeviceNode);
+ErrorPhysHeapDeInitDeviceHeaps:
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+ErrorPowerLockDeInit:
+	PVRSRVPowerLockDeInit(psDeviceNode);
+ErrorUnregisterDbgTable:
+	PVRSRVUnregisterDeviceDbgTable(psDeviceNode);
+ErrorPvzConnectionDeInit:
+	psDevConfig->psDevNode = NULL;
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVCFG, psDevConfig))
+	{
+		PvzConnectionDeInit(psDevConfig);
+	}
+ErrorSysDevDeInit:
+	SysDevDeInit(psDevConfig);
+ErrorDeregisterStats:
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+ErrorFreeDeviceNode:
+#endif
+	OSFreeMemNoStats(psDeviceNode);
+
+	return eError;
+}
+
+#if defined(SUPPORT_RGX)
+static PVRSRV_ERROR _SetDeviceFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL bValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXSetDeviceFlags((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                            ui32Flag, bValue);
+
+	return eResult;
+}
+
+static PVRSRV_ERROR _ReadDeviceFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL *pbValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+	IMG_UINT32 ui32State;
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXGetDeviceFlags((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                            &ui32State);
+
+	if (PVRSRV_OK == eResult)
+	{
+		*pbValue = (ui32State & ui32Flag)? IMG_TRUE: IMG_FALSE;
+	}
+
+	return eResult;
+}
+static PVRSRV_ERROR _SetStateFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                  const void *psPrivate, IMG_BOOL bValue)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	eResult = RGXStateFlagCtrl((PVRSRV_RGXDEV_INFO *)psDevice->pvDevice,
+	                           ui32Flag, NULL, bValue);
+
+	return eResult;
+}
+
+static PVRSRV_ERROR _ReadStateFlag(const PVRSRV_DEVICE_NODE *psDevice,
+                                   const void *psPrivate, IMG_BOOL *pbValue)
+{
+	IMG_UINT32 ui32Flag = (IMG_UINT32)((uintptr_t)psPrivate);
+	IMG_UINT32 ui32State;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(ui32Flag);
+	PVR_RETURN_IF_FALSE(psDevice != APPHINT_OF_DRIVER_NO_DEVICE,
+	                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDevice->pvDevice;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags,
+	                           INVALIDATE);
+	ui32State = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+
+	if (pbValue)
+	{
+		*pbValue = (ui32State & ui32Flag)? IMG_TRUE: IMG_FALSE;
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+PVRSRV_ERROR PVRSRVCommonDeviceInitialise(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_BOOL bInitSuccessful = IMG_FALSE;
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	IMG_HANDLE hProcessStats;
+#endif
+	PVRSRV_ERROR eError;
+
+	PDUMPCOMMENT(psDeviceNode, "Common Device Initialisation");
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceInitialise: DevId %d", psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_CREATED)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device already initialised", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Allocate an OSEventObject to use in the Freeze / Thaw transitioning to
+	 * allow LBIST requests to be serviced. This object is held for the life of
+	 * this device-node.
+	 */
+	eError = OSEventObjectCreate("PVRSRV_DEVICE_THREAD_EVENT_OBJECT",
+	                             &psDeviceNode->hDeviceThreadEvObj);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSEventObjectCreate");
+
+	eError = OSEventObjectOpen(psDeviceNode->hDeviceThreadEvObj,
+	                           &psDeviceNode->hDeviceFreezeThaw);
+
+	if (PVRSRV_OK != eError)
+	{
+		OSEventObjectDestroy(psDeviceNode->hDeviceThreadEvObj);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, "OSEventObjectOpen");
+	}
+
+	/* Initial zero-set for the number of Frozen threads */
+	OSAtomicWrite(&psDeviceNode->iFreezeCount, 0);
+	OSAtomicWrite(&psDeviceNode->iTotalFreezes, 0);
+
+	/* Initial zero-set for number of active threads on this device */
+	OSAtomicWrite(&psDeviceNode->iThreadsActive, 0);
+
+	/* Allocate devmem_history backing store for the device if we have
+	 * EnablePageFaultDebug set
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		eError = DevicememHistoryDeviceInit(psDeviceNode);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevicememHistoryDeviceInit");
+	}
+
+#if defined(PDUMP)
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_CONNECT)
+	{
+		PVRSRV_DATA *psSRVData = PVRSRVGetPVRSRVData();
+
+		/* If first connection, bind this and future PDump clients to use this device */
+		if (psSRVData->ui32PDumpBoundDevice == PVRSRV_MAX_DEVICES)
+		{
+			psSRVData->ui32PDumpBoundDevice = psDeviceNode->sDevId.ui32InternalID;
+		}
+	}
+#endif
+#endif
+
+	/* Initialise Connection_Data access mechanism */
+	dllist_init(&psDeviceNode->sConnections);
+	eError = OSLockCreate(&psDeviceNode->hConnectionsLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	/* Allocate process statistics */
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	eError = PVRSRVStatsRegisterProcess(&hProcessStats);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVStatsRegisterProcess");
+#endif
+
+	eError = MMU_InitDevice(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "MMU_InitDevice");
+
+#if defined(SUPPORT_RGX)
+	eError = RGXInit(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInit", Exit);
+#endif
+
+	bInitSuccessful = IMG_TRUE;
+
+#if defined(SUPPORT_RGX)
+Exit:
+#endif
+	eError = PVRSRVDeviceFinalise(psDeviceNode, bInitSuccessful);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceFinalise");
+
+#if defined(SUPPORT_RGX)
+	if (bInitSuccessful && (eError == PVRSRV_OK))
+	{
+		RGXInitMultiCoreInfo(psDeviceNode);
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableClockGating,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  APPHINT_OF_DRIVER_NO_DEVICE,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_DISABLE_CLKGATING_EN));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableDMOverlap,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  APPHINT_OF_DRIVER_NO_DEVICE,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_DISABLE_DM_OVERLAP));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_AssertOnHWRTrigger,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_AssertOutOfMemory,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY));
+			PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_CheckMList,
+			                                  _ReadStateFlag, _SetStateFlag,
+			                                  psDeviceNode,
+			                                  (void*)((uintptr_t)RGXFWIF_INICFG_CHECK_MLIST_EN));
+		}
+
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisableFEDLogging,
+		                                  _ReadDeviceFlag, _SetDeviceFlag,
+		                                  psDeviceNode,
+		                                  (void*)((uintptr_t)RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN));
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_ZeroFreelist,
+		                                  _ReadDeviceFlag, _SetDeviceFlag,
+		                                  psDeviceNode,
+		                                  (void*)((uintptr_t)RGXKM_DEVICE_STATE_ZERO_FREELIST));
+		PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_DisablePDumpPanic,
+		                                  RGXQueryPdumpPanicDisable, RGXSetPdumpPanicDisable,
+		                                  psDeviceNode,
+		                                  NULL);
+	}
+#endif
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Close the process statistics */
+	PVRSRVStatsDeregisterProcess(hProcessStats);
+#endif
+
+	return eError;
+}
+
+void PVRSRVCommonDeviceDestroy(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_DATA				*psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_ERROR			eError;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL				bForceUnload = IMG_FALSE;
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDeviceNode->eHealthStatus);
+
+	if ((PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK) ||
+		(eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_DEAD) ||
+		(eHealthStatus == PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+	{
+		bForceUnload = IMG_TRUE;
+	}
+#endif
+
+	/* Remove DI hook for the devicemem_history for this device (if any).
+	 * The associated devicemem_history buffers are freed by the final
+	 * call to DevicememHistoryDeInitKM() as they are used asynchronously
+	 * by other parts of the DDK.
+	 */
+	if (psDeviceNode->bEnablePFDebug)
+	{
+		DevicememHistoryDeviceDestroy(psDeviceNode);
+	}
+
+	MULTI_DEVICE_BRINGUP_DPF("PVRSRVCommonDeviceDestroy: DevId %d", psDeviceNode->sDevId.i32KernelDeviceID);
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_DEINIT);
+
+	PVRSRVCleanupThreadWaitForDevice(psDeviceNode);
+
+	OSWRLockAcquireWrite(psPVRSRVData->hDeviceNodeListLock);
+	List_PVRSRV_DEVICE_NODE_Remove(psDeviceNode);
+	psPVRSRVData->ui32RegisteredDevices--;
+	OSWRLockReleaseWrite(psPVRSRVData->hDeviceNodeListLock);
+
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_DESTRUCTING);
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	UnregisterDVFSDevice(psDeviceNode);
+#endif
+
+#if defined(__linux__)
+	pvr_apphint_device_unregister(psDeviceNode);
+#endif /* defined(__linux__) */
+
+	DebugCommonDeInitDevice(psDeviceNode);
+
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	SyncFbDeregisterDevice(psDeviceNode);
+#endif
+	/* Counter part to what gets done in PVRSRVDeviceFinalise */
+	if (psDeviceNode->hSyncCheckpointContext)
+	{
+		SyncCheckpointContextDestroy(psDeviceNode->hSyncCheckpointContext);
+		psDeviceNode->hSyncCheckpointContext = NULL;
+	}
+	if (psDeviceNode->hSyncPrimContext)
+	{
+		if (psDeviceNode->psMMUCacheSyncPrim)
+		{
+			PVRSRV_CLIENT_SYNC_PRIM *psSync = psDeviceNode->psMMUCacheSyncPrim;
+
+			/* Ensure there are no pending MMU Cache Ops in progress before freeing this sync. */
+			eError = PVRSRVPollForValueKM(psDeviceNode,
+			                              psSync->pui32LinAddr,
+			                              psDeviceNode->ui32NextMMUInvalidateUpdate-1,
+			                              0xFFFFFFFF,
+			                              POLL_FLAG_LOG_ERROR,
+			                              NULL);
+			PVR_LOG_IF_ERROR(eError, "PVRSRVPollForValueKM");
+
+			/* Important to set the device node pointer to NULL
+			 * before we free the sync-prim to make sure we don't
+			 * defer the freeing of the sync-prim's page tables itself.
+			 * The sync is used to defer the MMU page table
+			 * freeing. */
+			psDeviceNode->psMMUCacheSyncPrim = NULL;
+
+			/* Free general purpose sync primitive */
+			SyncPrimFree(psSync);
+		}
+
+		SyncPrimContextDestroy(psDeviceNode->hSyncPrimContext);
+		psDeviceNode->hSyncPrimContext = NULL;
+	}
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError == PVRSRV_OK)
+	{
+		IMG_BOOL bHasPowerLock = IMG_TRUE;
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+		/*
+		 * Firmware probably not responding if bForceUnload is set, but we still want to unload the
+		 * driver.
+		 */
+		if (!bForceUnload)
+#endif
+		{
+			/* Force idle device */
+			eError = PVRSRVDeviceIdleRequestKM(psDeviceNode, NULL, IMG_TRUE);
+			PVR_LOG_IF_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+			if (eError == PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				bHasPowerLock = IMG_FALSE;
+			}
+		}
+
+		if (bHasPowerLock)
+		{
+			/* Power down the device if necessary */
+			eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+												 PVRSRV_DEV_POWER_STATE_OFF,
+												 PVRSRV_POWER_FLAGS_FORCED);
+			PVRSRVPowerUnlock(psDeviceNode);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "PVRSRVSetDevicePowerStateKM");
+
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+		}
+	}
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+	DeinitDVFS(psDeviceNode);
+#elif defined(SUPPORT_PDVFS) && !defined(NO_HARDWARE)
+	DeinitPDVFS(psDeviceNode);
+#endif
+
+	if (psDeviceNode->hDbgReqNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDeviceNode->hDbgReqNotify);
+	}
+
+	SyncCheckpointDeinit(psDeviceNode);
+
+	SyncServerDeinit(psDeviceNode);
+
+	MMU_DeInitDevice(psDeviceNode);
+
+#if defined(SUPPORT_RGX)
+	DevDeInitRGX(psDeviceNode);
+#endif
+
+#if defined(SUPPORT_PMR_DEFERRED_FREE)
+	/* must be called before PhysHeapDeInitDeviceHeaps() */
+	PMRDeInitDevice(psDeviceNode);
+#endif /* defined(SUPPORT_PMR_DEFERRED_FREE) */
+
+	PhysHeapDeInitDeviceHeaps(psDeviceNode);
+	PVRSRVPowerLockDeInit(psDeviceNode);
+
+	PVRSRVUnregisterDeviceDbgTable(psDeviceNode);
+
+	if (psDeviceNode->hPageFaultNotifyLock != NULL)
+	{
+		OSWRLockDestroy(psDeviceNode->hPageFaultNotifyLock);
+	}
+
+	/* Release the Connection-Data lock as late as possible. */
+	if (psDeviceNode->hConnectionsLock)
+	{
+		OSLockDestroy(psDeviceNode->hConnectionsLock);
+	}
+
+	/* Release the hDeviceThreadEvObj as late as possible. */
+	if (psDeviceNode->hDeviceThreadEvObj)
+	{
+		OSEventObjectClose(psDeviceNode->hDeviceFreezeThaw);
+
+		OSEventObjectDestroy(psDeviceNode->hDeviceThreadEvObj);
+	}
+
+	PVR_ASSERT(OSAtomicRead(&psDeviceNode->iFreezeCount) == 0);
+
+	psDeviceNode->psDevConfig->psDevNode = NULL;
+
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		PvzConnectionDeInit(psDeviceNode->psDevConfig);
+	}
+	SysDevDeInit(psDeviceNode->psDevConfig);
+
+	OSFreeMemNoStats(psDeviceNode);
+}
+
+/**************************************************************************/ /*!
+@Function     PVRSRVDeviceFinalise
+@Description  Performs the final parts of device initialisation.
+@Input        psDeviceNode            Device node of the device to finish
+                                      initialising
+@Input        bInitSuccessful         Whether or not device specific
+                                      initialisation was successful
+@Return       PVRSRV_ERROR     PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceFinalise(PVRSRV_DEVICE_NODE *psDeviceNode,
+											   IMG_BOOL bInitSuccessful)
+{
+	PVRSRV_ERROR eError;
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	if (bInitSuccessful)
+	{
+		eError = SyncCheckpointContextCreate(psDeviceNode,
+											 &psDeviceNode->hSyncCheckpointContext);
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncCheckpointContextCreate", ErrorExit);
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+		eError = SyncFbRegisterDevice(psDeviceNode);
+		PVR_GOTO_IF_ERROR(eError, ErrorExit);
+#endif
+		eError = SyncPrimContextCreate(psDeviceNode,
+									   &psDeviceNode->hSyncPrimContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "SyncPrimContextCreate");
+			SyncCheckpointContextDestroy(psDeviceNode->hSyncCheckpointContext);
+			goto ErrorExit;
+		}
+
+		/* Allocate MMU cache invalidate sync */
+		eError = SyncPrimAlloc(psDeviceNode->hSyncPrimContext,
+							   &psDeviceNode->psMMUCacheSyncPrim,
+							   "pvrsrv dev MMU cache");
+		PVR_LOG_GOTO_IF_ERROR(eError, "SyncPrimAlloc", ErrorExit);
+
+		/* Set the sync prim value to a much higher value near the
+		 * wrapping range. This is so any wrapping bugs would be
+		 * seen early in the driver start-up.
+		 */
+		SyncPrimSet(psDeviceNode->psMMUCacheSyncPrim, 0xFFFFFFF6UL);
+
+		eError = PVRSRVPowerLock(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPowerLock", ErrorExit);
+
+		/*
+		 * Always ensure a single power on command appears in the pdump. This
+		 * should be the only power related call outside of PDUMPPOWCMDSTART
+		 * and PDUMPPOWCMDEND.
+		 */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_FORCED);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set device %p power state to 'on' (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			PVRSRVPowerUnlock(psDeviceNode);
+			goto ErrorExit;
+		}
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+		eError = ValidateFWOnLoad(psDeviceNode->pvDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "ValidateFWOnLoad");
+			PVRSRVPowerUnlock(psDeviceNode);
+			return eError;
+		}
+#endif
+
+		/*
+		 * Guest driver must do a runtime compatibility check against the
+		 * data provided by the Firmware.
+		 */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Failed compatibility check for device %p (%s)",
+						 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVPowerUnlock(psDeviceNode);
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				goto ErrorExit;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+
+		/* Force the device to idle if its default power state is off */
+		eError = PVRSRVDeviceIdleRequestKM(psDeviceNode,
+										   &PVRSRVDeviceIsDefaultStateOFF,
+										   IMG_TRUE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "PVRSRVDeviceIdleRequestKM");
+			if (eError != PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+			{
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+			goto ErrorExit;
+		}
+
+		/* Place device into its default power state. */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_DEFAULT,
+											 PVRSRV_POWER_FLAGS_FORCED);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set device %p into its default power state (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			goto ErrorExit;
+		}
+
+		PVRSRVPowerUnlock(psDeviceNode);
+
+		/*
+		 * If PDUMP is enabled and RGX device is supported, then initialise the
+		 * performance counters that can be further modified in PDUMP. Then,
+		 * before ending the init phase of the pdump, drain the commands put in
+		 * the kCCB during the init phase.
+		 */
+#if defined(SUPPORT_RGX)
+#if defined(PDUMP)
+		{
+			eError = RGXInitHWPerfCounters(psDeviceNode);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitHWPerfCounters", ErrorExit);
+
+			eError = RGXPdumpDrainKCCB(psDevInfo,
+									   psDevInfo->psKernelCCBCtlLocal->ui32WriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", ErrorExit);
+		}
+#endif
+#endif /* defined(SUPPORT_RGX) */
+		/* Now that the device(s) are fully initialised set them as active */
+		PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_ACTIVE);
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		/* Initialisation failed so set the device(s) into a bad state */
+		PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_BAD);
+		eError = PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	/* Give PDump control a chance to end the init phase, depends on OS */
+	PDUMPENDINITPHASE(psDeviceNode);
+	return eError;
+
+ErrorExit:
+	/* Initialisation failed so set the device(s) into a bad state */
+	PVRSRVDeviceSetState(psDeviceNode, PVRSRV_DEVICE_STATE_BAD);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* Only check devices which specify a compatibility check callback */
+	if (psDeviceNode->pfnInitDeviceCompatCheck)
+		return psDeviceNode->pfnInitDeviceCompatCheck(psDeviceNode);
+	else
+		return PVRSRV_OK;
+}
+
+/*
+	PollForValueKM
+*/
+static
+PVRSRV_ERROR PollForValueKM (volatile IMG_UINT32 __iomem           *pui32LinMemAddr,
+										  IMG_UINT32                ui32Value,
+										  IMG_UINT32                ui32Mask,
+										  IMG_UINT32                ui32Timeoutus,
+										  IMG_UINT32                ui32PollPeriodus,
+										  POLL_FLAGS                ePollFlags,
+										  PFN_INVALIDATE_CACHEFUNC  pfnFwInvalidate)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(pui32LinMemAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(ui32Timeoutus);
+	PVR_UNREFERENCED_PARAMETER(ui32PollPeriodus);
+	PVR_UNREFERENCED_PARAMETER(ePollFlags);
+	return PVRSRV_OK;
+#else
+	IMG_UINT32 ui32ActualValue = 0xFFFFFFFFU; /* Initialiser only required to prevent incorrect warning */
+
+	LOOP_UNTIL_TIMEOUT_US(ui32Timeoutus)
+	{
+		if (pfnFwInvalidate)
+		{
+			pfnFwInvalidate((const volatile void __force *)pui32LinMemAddr,
+			                sizeof(*pui32LinMemAddr),
+			                PVRSRV_CACHE_OP_INVALIDATE);
+		}
+
+		ui32ActualValue = OSReadHWReg32((void __iomem *)pui32LinMemAddr, 0) & ui32Mask;
+
+		if (ui32ActualValue == ui32Value)
+		{
+			return PVRSRV_OK;
+		}
+
+		if (gpsPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+#define ONE_MSEC_IN_USECS	(1000U)
+
+		if (ui32PollPeriodus <= ONE_MSEC_IN_USECS)
+		{
+			OSWaitus(ui32PollPeriodus);
+		}
+		else
+		{
+			OSSleepms(ui32PollPeriodus / ONE_MSEC_IN_USECS);
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (BITMASK_HAS(ePollFlags, POLL_FLAG_LOG_ERROR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PollForValueKM: Timeout. Expected 0x%x but found 0x%x (mask 0x%x).",
+		         ui32Value, ui32ActualValue, ui32Mask));
+	}
+
+	return PVRSRV_ERROR_TIMEOUT;
+#endif /* NO_HARDWARE */
+}
+
+
+/*
+	PVRSRVPollForValueKM
+*/
+PVRSRV_ERROR PVRSRVPollForValueKM (PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem   *pui32LinMemAddr,
+												IMG_UINT32                     ui32Value,
+												IMG_UINT32                     ui32Mask,
+												POLL_FLAGS                     ePollFlags,
+												PFN_INVALIDATE_CACHEFUNC       pfnFwInvalidate)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PollForValueKM(pui32LinMemAddr, ui32Value, ui32Mask,
+						  MAX_HW_TIME_US,
+						  MAX_HW_TIME_US/WAIT_TRY_COUNT,
+						  ePollFlags,
+						  pfnFwInvalidate);
+	if (eError != PVRSRV_OK && BITMASK_HAS(ePollFlags, POLL_FLAG_DEBUG_DUMP))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		                        __func__, PVRSRVGetErrorString(eError),
+								pui32LinMemAddr, ui32Value));
+		PVRSRVDebugRequest(psDevNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVWaitForValueKM(volatile IMG_UINT32 __iomem   *pui32LinMemAddr,
+                     IMG_UINT32                    ui32Value,
+                     IMG_UINT32                    ui32Mask,
+                     PFN_INVALIDATE_CACHEFUNC      pfnFwInvalidate)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(pui32LinMemAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	return PVRSRV_OK;
+#else
+
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	IMG_HANDLE hOSEvent;
+	PVRSRV_ERROR eError;
+	PVRSRV_ERROR eErrorWait;
+	IMG_UINT32 ui32ActualValue;
+
+	eError = OSEventObjectOpen(psPVRSRVData->hGlobalEventObject, &hOSEvent);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectOpen", EventObjectOpenError);
+
+	eError = PVRSRV_ERROR_TIMEOUT; /* Initialiser for following loop */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (pfnFwInvalidate)
+		{
+			pfnFwInvalidate((const volatile void __force *)pui32LinMemAddr,
+			                sizeof(*pui32LinMemAddr),
+			                PVRSRV_CACHE_OP_INVALIDATE);
+		}
+
+		ui32ActualValue = (OSReadDeviceMem32(pui32LinMemAddr) & ui32Mask);
+
+		if (ui32ActualValue == ui32Value)
+		{
+			/* Expected value has been found */
+			eError = PVRSRV_OK;
+			break;
+		}
+		else if (psPVRSRVData->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			/* Services in bad state, don't wait any more */
+			eError = PVRSRV_ERROR_NOT_READY;
+			break;
+		}
+		else
+		{
+			/* wait for event and retry */
+			eErrorWait = OSEventObjectWait(hOSEvent);
+			if (eErrorWait != PVRSRV_OK  &&  eErrorWait != PVRSRV_ERROR_TIMEOUT)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Failed with error %d. Found value 0x%x but was expected "
+				         "to be 0x%x (Mask 0x%08x). Retrying",
+						 __func__,
+						 eErrorWait,
+						 ui32ActualValue,
+						 ui32Value,
+						 ui32Mask));
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	OSEventObjectClose(hOSEvent);
+
+	/* One last check in case the object wait ended after the loop timeout... */
+	if (eError != PVRSRV_OK &&
+	    (OSReadDeviceMem32(pui32LinMemAddr) & ui32Mask) == ui32Value)
+	{
+		eError = PVRSRV_OK;
+	}
+
+	/* Provide event timeout information to aid the Device Watchdog Thread... */
+	if (eError == PVRSRV_OK)
+	{
+		psPVRSRVData->ui32GEOConsecutiveTimeouts = 0;
+	}
+	else if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		psPVRSRVData->ui32GEOConsecutiveTimeouts++;
+	}
+
+EventObjectOpenError:
+
+	return eError;
+
+#endif /* NO_HARDWARE */
+}
+
+int PVRSRVGetDriverStatus(void)
+{
+	return PVRSRVGetPVRSRVData()->eServicesState;
+}
+
+/*
+	PVRSRVSystemHasCacheSnooping
+*/
+IMG_BOOL PVRSRVSystemHasCacheSnooping(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if ((psDevConfig->eCacheSnoopingMode != PVRSRV_DEVICE_SNOOP_NONE) &&
+		(psDevConfig->eCacheSnoopingMode != PVRSRV_DEVICE_SNOOP_EMULATED))
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemSnoopingIsEmulated(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if (psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_EMULATED)
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemSnoopingOfCPUCache(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	if ((psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_CPU_ONLY) ||
+		(psDevConfig->eCacheSnoopingMode == PVRSRV_DEVICE_SNOOP_CROSS))
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+IMG_BOOL PVRSRVSystemHasNonMappableLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	return psDevConfig->bHasNonMappableLocalMemory;
+}
+
+/*
+	PVRSRVSystemWaitCycles
+*/
+void PVRSRVSystemWaitCycles(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT32 ui32Cycles)
+{
+	/* Delay in us */
+	IMG_UINT32 ui32Delayus = 1;
+
+	/* obtain the device freq */
+	if (psDevConfig->pfnClockFreqGet != NULL)
+	{
+		IMG_UINT32 ui32DeviceFreq;
+
+		ui32DeviceFreq = psDevConfig->pfnClockFreqGet(psDevConfig->hSysData);
+
+		ui32Delayus = (ui32Cycles*1000000)/ui32DeviceFreq;
+
+		if (ui32Delayus == 0)
+		{
+			ui32Delayus = 1;
+		}
+	}
+
+	OSWaitus(ui32Delayus);
+}
+
+static void *
+PVRSRVSystemInstallDeviceLISR_Match_AnyVaCb(PVRSRV_DEVICE_NODE *psDeviceNode,
+											va_list va)
+{
+	void *pvOSDevice = va_arg(va, void *);
+
+	if (psDeviceNode->psDevConfig->pvOSDevice == pvOSDevice)
+	{
+		return psDeviceNode;
+	}
+
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVSystemInstallDeviceLISR(void *pvOSDevice,
+										   IMG_UINT32 ui32IRQ,
+										   const IMG_CHAR *pszName,
+										   PFN_LISR pfnLISR,
+										   void *pvData,
+										   IMG_HANDLE *phLISRData)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode =
+		List_PVRSRV_DEVICE_NODE_Any_va(psPVRSRVData->psDeviceNodeList,
+									   &PVRSRVSystemInstallDeviceLISR_Match_AnyVaCb,
+									   pvOSDevice);
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (!psDeviceNode)
+	{
+		/* Device can't be found in the list so it isn't in the system */
+		PVR_DPF((PVR_DBG_ERROR, "%s: device %p with irq %d is not present",
+				 __func__, pvOSDevice, ui32IRQ));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	return SysInstallDeviceLISR(psDeviceNode->psDevConfig->hSysData, ui32IRQ,
+								pszName, pfnLISR, pvData, phLISRData);
+}
+
+PVRSRV_ERROR PVRSRVSystemUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return SysUninstallDeviceLISR(hLISRData);
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION) && defined(EMULATOR)
+/* functions only used on rogue, but header defining them is common */
+void SetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState)
+{
+	SysSetAxiProtOSid(hSysData, ui32OSid, bState);
+}
+
+void SetTrustedDeviceAceEnabled(IMG_HANDLE hSysData)
+{
+	SysSetTrustedDeviceAceEnabled(hSysData);
+}
+#endif
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR PVRSRVCreateHWPerfHostThread(IMG_UINT32 ui32Timeout)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!ui32Timeout)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	OSLockAcquire(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+
+	/* Create only once */
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread == NULL)
+	{
+		/* Create the HWPerf event object */
+		eError = OSEventObjectCreate("PVRSRV_HWPERFHOSTPERIODIC_EVENTOBJECT", &gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectCreate");
+
+		if (eError == PVRSRV_OK)
+		{
+			gpsPVRSRVData->bHWPerfHostThreadStop = IMG_FALSE;
+			gpsPVRSRVData->ui32HWPerfHostThreadTimeout = ui32Timeout;
+			/* Create a thread which is used to periodically emit host stream packets */
+			eError = OSThreadCreate(&gpsPVRSRVData->hHWPerfHostPeriodicThread,
+				"pvr_hwperf_host",
+				HWPerfPeriodicHostEventsThread,
+				NULL, IMG_TRUE, gpsPVRSRVData);
+			PVR_LOG_IF_ERROR(eError, "OSThreadCreate");
+		}
+	}
+	/* If the thread has already been created then just update the timeout and wake up thread */
+	else
+	{
+		gpsPVRSRVData->ui32HWPerfHostThreadTimeout = ui32Timeout;
+		eError = OSEventObjectSignal(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+	}
+
+	OSLockRelease(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVDestroyHWPerfHostThread(void)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	OSLockAcquire(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+
+	/* Stop and cleanup the HWPerf periodic thread */
+	if (gpsPVRSRVData->hHWPerfHostPeriodicThread)
+	{
+		if (gpsPVRSRVData->hHWPerfHostPeriodicEvObj)
+		{
+			gpsPVRSRVData->bHWPerfHostThreadStop = IMG_TRUE;
+			eError = OSEventObjectSignal(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectSignal");
+		}
+		LOOP_UNTIL_TIMEOUT_US(OS_THREAD_DESTROY_TIMEOUT_US)
+		{
+			eError = OSThreadDestroy(gpsPVRSRVData->hHWPerfHostPeriodicThread);
+			if (PVRSRV_OK == eError)
+			{
+				gpsPVRSRVData->hHWPerfHostPeriodicThread = NULL;
+				break;
+			}
+			OSWaitus(OS_THREAD_DESTROY_TIMEOUT_US/OS_THREAD_DESTROY_RETRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+		PVR_LOG_IF_ERROR(eError, "OSThreadDestroy");
+
+		if (gpsPVRSRVData->hHWPerfHostPeriodicEvObj)
+		{
+			eError = OSEventObjectDestroy(gpsPVRSRVData->hHWPerfHostPeriodicEvObj);
+			gpsPVRSRVData->hHWPerfHostPeriodicEvObj = NULL;
+			PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+		}
+	}
+
+	OSLockRelease(gpsPVRSRVData->hHWPerfHostPeriodicThread_Lock);
+	return eError;
+}
+#endif
+
+/*
+ * Scan the list of known devices until we find the specific instance or
+ * exhaust the list
+ */
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstance(IMG_UINT32 uiInstance)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	if (uiInstance >= gpsPVRSRVData->ui32RegisteredDevices)
+	{
+		return NULL;
+	}
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (psDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     psDevNode != NULL; psDevNode = psDevNode->psNext)
+	{
+		if (uiInstance == psDevNode->sDevId.ui32InternalID)
+		{
+			break;
+		}
+	}
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	return psDevNode;
+}
+
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstanceByKernelDevID(IMG_INT32 i32OSInstance)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (psDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     psDevNode != NULL; psDevNode = psDevNode->psNext)
+	{
+		if (i32OSInstance == psDevNode->sDevId.i32KernelDeviceID)
+		{
+			MULTI_DEVICE_BRINGUP_DPF("%s: Found DevId %d. Retrieving node.", __func__, i32OSInstance);
+			break;
+		}
+		else
+		{
+			MULTI_DEVICE_BRINGUP_DPF("%s: Searching for DevId %d: Id %d not matching", __func__, i32OSInstance, psDevNode->sDevId.i32KernelDeviceID);
+		}
+	}
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	if (psDevNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: DevId %d not found.", __func__, i32OSInstance));
+	}
+	return psDevNode;
+}
+
+#define _FROZEN 1		/* Device is already frozen */
+#define _NOT_FROZEN 0	/* Device is not frozen */
+
+/* Freeze the specified device if not already frozen */
+PVRSRV_ERROR PVRSRVDeviceFreeze(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL     bHasPowerLock = IMG_FALSE;
+
+	/* Verify that given argument *IS* a recognised device node */
+	PVRSRV_DEVICE_NODE *lpsDevNode;
+
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (lpsDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     lpsDevNode != NULL && lpsDevNode != psDevNode;
+	     lpsDevNode = lpsDevNode->psNext)
+		;
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	PVR_LOG_RETURN_IF_FALSE((lpsDevNode == psDevNode), "Device node not known",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Mark the device as 'frozen' if not already marked */
+	if (OSAtomicCompareExchange(&psDevNode->eFrozen, _NOT_FROZEN,
+	                            _FROZEN) == _FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device %p already frozen", __func__,
+		        psDevNode));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure there are no other threads active within this device. */
+	while (OSAtomicRead(&psDevNode->iThreadsActive) > 0)
+	{
+		OSReleaseThreadQuanta();	/* Let other threads execute */
+	}
+
+	/* Attempt to idle the GPU without power-down. If this fails, we try with
+	 * a potential power-down.
+	 */
+	eError = PVRSRVPowerLock(psDevNode);
+	if (eError != PVRSRV_OK)
+	{
+		/* Device is powered down, we can continue
+		 * as there are no in-flight requests in the GPU.
+		 */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unable to Idle Device (%p) [%u/%d]"
+		        " Device powered down",
+		        __func__, psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		bHasPowerLock = IMG_TRUE;
+
+		eError = PVRSRVDeviceIdleRequestKM(psDevNode,
+		                                   &PVRSRVDeviceIsDefaultStateOFF,
+		                                   IMG_TRUE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Failed to Idle. Need to remove the pending _FROZEN state. */
+		(void) OSAtomicExchange(&psDevNode->eFrozen, _NOT_FROZEN);
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unable to Idle Device (%p) [%u/%d]",
+		        __func__, psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+
+		if (bHasPowerLock)
+		{
+			PVRSRVPowerUnlock(psDevNode);
+		}
+		return eError;
+	}
+
+	/* Now change the device-state to not be ACTIVE until we unfreeze */
+	if (psDevNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Unexpected Device State %u for device %p",
+		        __func__, psDevNode->eDevState, psDevNode));
+
+		/* Reset device to UNFROZEN */
+		(void) OSAtomicExchange(&psDevNode->eFrozen, _NOT_FROZEN);
+
+		if (bHasPowerLock)
+		{
+			(void) PVRSRVDeviceIdleCancelRequestKM(psDevNode);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+		else
+		{
+			(void) PVRSRVSetDeviceSystemPowerState(psDevNode,
+	                                               PVRSRV_SYS_POWER_STATE_ON,
+	                                               PVRSRV_POWER_FLAGS_NONE);
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_FROZEN);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Device (%p) [%u/%d] FROZEN", __func__,
+	        psDevNode,
+	        psDevNode->sDevId.ui32InternalID,
+	        psDevNode->sDevId.i32KernelDeviceID));
+
+	OSAtomicIncrement(&psDevNode->iTotalFreezes);
+
+	if (bHasPowerLock)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+	}
+
+	return PVRSRV_OK;
+}
+
+/* Unfreeze / Thaw the specified device if frozen */
+PVRSRV_ERROR PVRSRVDeviceThaw(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *lpsDevNode;
+	IMG_BOOL           bHasPowerLock = IMG_FALSE;
+
+	/* Verify that the given argument *IS* a recognised device node */
+	OSWRLockAcquireRead(gpsPVRSRVData->hDeviceNodeListLock);
+	for (lpsDevNode = gpsPVRSRVData->psDeviceNodeList;
+	     lpsDevNode != NULL && lpsDevNode != psDevNode;
+	     lpsDevNode = lpsDevNode->psNext)
+		;
+	OSWRLockReleaseRead(gpsPVRSRVData->hDeviceNodeListLock);
+
+	PVR_LOG_RETURN_IF_FALSE((lpsDevNode == psDevNode), "Device node not known",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Unfreeze the device */
+	if (OSAtomicCompareExchange(&psDevNode->eFrozen, _FROZEN,
+	                            _NOT_FROZEN) == _NOT_FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Device (%p) [%u/%d] already unfrozen",
+		        __func__,
+		        psDevNode, psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (psDevNode->eDevState != PVRSRV_DEVICE_STATE_FROZEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Unexpected Device state %u for device (%p) [%u/%d]",
+		        __func__, psDevNode->eDevState, psDevNode,
+		        psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+	}
+	else
+	{
+		PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_ACTIVE);
+
+		PVR_DPF((PVR_DBG_VERBOSE, "%s: Device (%p) [%u/%d] UNFROZEN", __func__,
+		        psDevNode,
+		        psDevNode->sDevId.ui32InternalID,
+		        psDevNode->sDevId.i32KernelDeviceID));
+	}
+
+	/* Now unblock the device by clearing any ForcedIdle state and/or
+	 * powering-on the device if needed.
+	 */
+	eError = PVRSRVPowerLock(psDevNode);
+
+	bHasPowerLock = (eError == PVRSRV_OK) ? IMG_TRUE : IMG_FALSE;
+
+	if (bHasPowerLock)
+	{
+		eError = PVRSRVDeviceIdleCancelRequestKM(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Could not cancel Idle state: Device (%p) [%u/%d] '%s'",
+			        __func__,
+			        psDevNode,
+			        psDevNode->sDevId.ui32InternalID,
+			        psDevNode->sDevId.i32KernelDeviceID,
+			        PVRSRVGetErrorString(eError)));
+		}
+
+		PVRSRVPowerUnlock(psDevNode);
+	}
+	else
+	{
+		/* Force device back into POWER_STATE_ON as it must already be OFF. */
+		(void) PVRSRVSetDeviceSystemPowerState(psDevNode,
+	                                           PVRSRV_SYS_POWER_STATE_ON,
+	                                           PVRSRV_POWER_FLAGS_NONE);
+	}
+
+	/* Unblock any waiting threads */
+	while (OSAtomicRead(&psDevNode->iFreezeCount) > 0)
+	{
+		eError = OSEventObjectSignal(psDevNode->hDeviceThreadEvObj);
+		if (OSAtomicRead(&psDevNode->iFreezeCount) > 0)
+		{
+			/* Sleep for 1ms to allow waiter to receive signal */
+			OSSleepms(1U);
+		}
+	}
+
+	/* Ensure that any blocked queues get rescheduled if we've woken up a
+	 * waiter.
+	 */
+	PVRSRVCheckStatus(NULL);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_DRIVER_MODE PVRSRVGetVzModeByDevNum(IMG_UINT32 ui32DevNum)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DRIVER_MODE eRetMode = DRIVER_MODE_NATIVE;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+
+	/* Iterate over all devices. */
+	for (psDevNode = psPVRSRVData->psDeviceNodeList;
+		 psDevNode != NULL;
+		 psDevNode = psDevNode->psNext)
+	{
+		if (psDevNode->sDevId.ui32InternalID == ui32DevNum)
+		{
+			eRetMode = psDevNode->psDevConfig->eDriverMode;
+			break;
+		}
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+	return eRetMode;
+}
+
+/*****************************************************************************
+ End of file (pvrsrv.c)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv.h b/drivers/gpu/drm/img-rogue/pvrsrv.h
new file mode 100644
index 000000000000..86d607092c60
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv.h
@@ -0,0 +1,518 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR services server header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_H
+#define PVRSRV_H
+
+#include "connection_server.h"
+#include "pvrsrv_pool.h"
+#include "device.h"
+#include "power.h"
+#include "syscommon.h"
+#include "sysinfo.h"
+#include "physheap.h"
+#include "cache_ops.h"
+#include "pvr_notifier.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "dma_support.h"
+#include "vz_vmm_pvz.h"
+
+/*!
+ * For OSThreadDestroy(), which may require a retry
+ * Try for 100 ms to destroy an OS thread before failing
+ */
+#define OS_THREAD_DESTROY_TIMEOUT_US 100000ULL
+#define OS_THREAD_DESTROY_RETRY_COUNT 10
+
+typedef enum _POLL_FLAGS_
+{
+	POLL_FLAG_NONE = 0, /* No message or dump is printed on poll timeout */
+	POLL_FLAG_LOG_ERROR = 1, /* Log error on poll timeout */
+	POLL_FLAG_DEBUG_DUMP = 2 /* Print debug dump on poll timeout */
+} POLL_FLAGS;
+
+typedef struct _BUILD_INFO_
+{
+	IMG_UINT32	ui32BuildOptions;
+	IMG_UINT32	ui32BuildVersion;
+	IMG_UINT32	ui32BuildRevision;
+	IMG_UINT32	ui32BuildType;
+#define BUILD_TYPE_DEBUG	0
+#define BUILD_TYPE_RELEASE	1
+	/* The above fields are self explanatory */
+	/* B.V.N.C can be added later if required */
+} BUILD_INFO;
+
+typedef struct _DRIVER_INFO_
+{
+	BUILD_INFO	sUMBuildInfo;
+	BUILD_INFO	sKMBuildInfo;
+	IMG_UINT8	ui8UMSupportedArch;
+	IMG_UINT8	ui8KMBitArch;
+
+#define	BUILD_ARCH_64BIT			(1 << 0)
+#define	BUILD_ARCH_32BIT			(1 << 1)
+#define	BUILD_ARCH_BOTH		(BUILD_ARCH_32BIT | BUILD_ARCH_64BIT)
+	IMG_BOOL	bIsNoMatch;
+}DRIVER_INFO;
+
+
+typedef struct PVRSRV_DATA_TAG
+{
+	DRIVER_INFO           sDriverInfo;
+	IMG_UINT32            ui32DPFErrorCount;                 /*!< Number of Fatal/Error DPFs */
+
+	POSWR_LOCK            hDeviceNodeListLock;            /*!< Read-Write lock to protect the list of devices */
+	PVRSRV_DEVICE_NODE    *psDeviceNodeList;              /*!< List head of device nodes */
+	IMG_UINT32            ui32RegisteredDevices;
+	PVRSRV_DEVICE_NODE    *psHostMemDeviceNode;           /*!< DeviceNode to be used for device independent
+	                                                        host based memory allocations where the DevMem
+	                                                        framework is to be used e.g. TL */
+	PVRSRV_SERVICES_STATE eServicesState;                 /*!< global driver state */
+
+	IMG_HANDLE            hGlobalEventObject;             /*!< OS Global Event Object */
+	IMG_UINT32            ui32GEOConsecutiveTimeouts;     /*!< OS Global Event Object Timeouts */
+
+	IMG_HANDLE            hCleanupThread;                 /*!< Cleanup thread */
+	IMG_HANDLE            hCleanupEventObject;            /*!< Event object to drive cleanup thread */
+	POS_SPINLOCK          hCleanupThreadWorkListLock;     /*!< Lock protecting the cleanup thread work list */
+	IMG_PID               cleanupThreadPid;               /*!< Cleanup thread process id */
+	uintptr_t             cleanupThreadTid;               /*!< Cleanup thread id */
+	ATOMIC_T              i32NumCleanupItemsQueued;       /*!< Number of items in cleanup thread work list */
+	ATOMIC_T              i32NumCleanupItemsNotCompleted; /*!< Number of items dropped from cleanup thread work list
+	                                                           after retry limit reached */
+	ATOMIC_T              i32CleanupItemTypes[PVRSRV_CLEANUP_TYPE_LAST];         /*!< Array containing the counts for different cleanup item types. */
+
+	IMG_HANDLE            hDevicesWatchdogThread;         /*!< Devices watchdog thread */
+	IMG_HANDLE            hDevicesWatchdogEvObj;          /*! Event object to drive devices watchdog thread */
+	volatile IMG_UINT32   ui32DevicesWatchdogPwrTrans;    /*! Number of off -> on power state transitions */
+#if !defined(PVRSRV_SERVER_THREADS_INDEFINITE_SLEEP)
+	volatile IMG_UINT32   ui32DevicesWatchdogTimeout;     /*! Timeout for the Devices watchdog Thread */
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_HANDLE            hAutoVzWatchdogThread;          /*!< Devices watchdog thread */
+	IMG_HANDLE            hAutoVzWatchdogEvObj;           /*! Event object to drive devices watchdog thread */
+#endif
+
+	POS_LOCK              hHWPerfHostPeriodicThread_Lock; /*!< Lock for the HWPerf Host periodic thread */
+	IMG_HANDLE            hHWPerfHostPeriodicThread;      /*!< HWPerf Host periodic thread */
+	IMG_HANDLE            hHWPerfHostPeriodicEvObj;       /*! Event object to drive HWPerf thread */
+	volatile IMG_BOOL     bHWPerfHostThreadStop;
+	IMG_UINT32            ui32HWPerfHostThreadTimeout;
+
+	IMG_HANDLE            hPvzConnection;                 /*!< PVZ connection used for cross-VM hyper-calls */
+	POS_LOCK              hPvzConnectionLock;             /*!< Lock protecting PVZ connection */
+
+	IMG_BOOL              bUnload;                        /*!< Driver unload is in progress */
+
+	IMG_HANDLE            hTLCtrlStream;                  /*! Control plane for TL streams */
+
+	IMG_HANDLE            hDriverThreadEventObject;       /*! Event object relating to multi-threading in the Server */
+	IMG_BOOL              bDriverSuspended;               /*! if TRUE, the driver is suspended and new threads should not enter */
+	ATOMIC_T              iNumActiveDriverThreads;        /*! Number of threads active in the Server */
+
+	PMR                   *psInfoPagePMR;                 /*! Handle to exportable PMR of the information page. */
+	IMG_UINT32            *pui32InfoPage;                 /*! CPU memory mapping for information page. */
+	DEVMEM_MEMDESC        *psInfoPageMemDesc;             /*! Memory descriptor of the information page. */
+	POS_LOCK              hInfoPageLock;                  /*! Lock guarding access to information page. */
+
+	IMG_HANDLE            hThreadsDbgReqNotify;
+
+
+	IMG_UINT32            ui32PDumpBoundDevice;           /*!< PDump is bound to the device first connected to */
+	ATOMIC_T              iNumDriverTasksActive;          /*!< Number of device-agnostic tasks active in the server */
+	PVRSRV_DRIVER_MODE    aeModuleParamDriverMode[PVRSRV_MAX_DEVICES]; /*!< Driver Mode for each device requested at launch */
+} PVRSRV_DATA;
+
+/* Function pointer used to invalidate cache between loops in wait/poll for value functions */
+typedef PVRSRV_ERROR (*PFN_INVALIDATE_CACHEFUNC)(const volatile void*, IMG_UINT64, PVRSRV_CACHE_OP);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVGetPVRSRVData
+
+ @Description	Get a pointer to the global data
+
+ @Return   PVRSRV_DATA *
+******************************************************************************/
+PVRSRV_DATA *PVRSRVGetPVRSRVData(void);
+PVRSRV_DRIVER_MODE PVRSRVGetVzModeByDevNum(IMG_UINT32 ui32DevNum);
+
+#define PVRSRV_KM_ERRORS                     ( PVRSRVGetPVRSRVData() ? PVRSRVGetPVRSRVData()->ui32DPFErrorCount : IMG_UINT32_MAX)
+#define PVRSRV_ERROR_LIMIT_REACHED                (PVRSRV_KM_ERRORS == IMG_UINT32_MAX)
+#define PVRSRV_REPORT_ERROR()                do { if (!(PVRSRV_ERROR_LIMIT_REACHED)) { PVRSRVGetPVRSRVData()->ui32DPFErrorCount++; } } while (0)
+
+#define PVRSRV_VZ_MODE_FROM_DEVNODE(pnode)     (pnode->psDevConfig->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVINFO(pdevinfo)  (pdevinfo->psDeviceNode->psDevConfig->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVCFG(pdevcfg)    (pdevcfg->eDriverMode)
+#define PVRSRV_VZ_MODE_FROM_DEVID(devid)       (PVRSRVGetVzModeByDevNum(devid))
+
+#define PVRSRV_VZ_MODE_IS(_expr, _struct, dev) (DRIVER_MODE_##_expr == PVRSRV_VZ_MODE_FROM_##_struct(dev))
+
+#define PVRSRV_VZ_RETN_IF_MODE(_expr, _struct, dev)         do { if (  PVRSRV_VZ_MODE_IS(_expr, _struct, dev)) { return; } } while (0)
+#define PVRSRV_VZ_RET_IF_MODE(_expr, _struct, dev, _rc)     do { if (  PVRSRV_VZ_MODE_IS(_expr, _struct, dev)) { return (_rc); } } while (0)
+
+#define PVRSRV_VZ_TIME_SLICE_MAX	(100UL)
+
+typedef struct _PHYS_HEAP_ITERATOR_ PHYS_HEAP_ITERATOR;
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorCreate
+
+ @Description
+ Creates iterator for traversing physical heap requested by ePhysHeap. The
+ iterator will go through all of the segments (a segment is physically
+ contiguous) of the physical heap and return their CPU physical address and
+ size.
+
+ @Input psDevNode: Pointer to device node struct.
+ @Input ePhysHeap: Find the matching heap.
+ @Output ppsIter: Pointer to the iterator object.
+
+ @Return PVRSRV_OK upon success and PVRSRV_ERROR otherwise.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorCreate(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_PHYS_HEAP ePhysHeap,
+                                    PHYS_HEAP_ITERATOR **ppsIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorDestroy
+
+ @Description
+ Frees the iterator object created with LMA_HeapIteratorCreate.
+
+ @Input psIter: Pointer to the iterator object.
+******************************************************************************/
+void LMA_HeapIteratorDestroy(PHYS_HEAP_ITERATOR *psIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorReset
+
+ @Description
+ Resets the iterator the first segment of the physical heap.
+
+ @Input psIter: Pointer to the iterator object.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorReset(PHYS_HEAP_ITERATOR *psIter);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorNext
+
+ @Description
+ Retrieves current segment's physical device address and size and moves the
+ iterator to the next element (if exists). If the iterator reached an end of
+ the heap and no segment was retrieved, this function returns IMG_FALSE.
+
+ @Input psIter: Pointer to the iterator object.
+ @Output psDevPAddr: Device physical address of the current segment.
+ @Output puiSize: Size of the current segment.
+
+ @Return IMG TRUE if a segment was found and retrieved, IMG_FALSE otherwise.
+******************************************************************************/
+IMG_BOOL LMA_HeapIteratorNext(PHYS_HEAP_ITERATOR *psIter,
+                              IMG_DEV_PHYADDR *psDevPAddr,
+                              IMG_UINT64 *puiSize);
+
+/*!
+******************************************************************************
+ @Function LMA_HeapIteratorGetHeapStats
+
+ @Description
+ Retrieves phys heap's usage statistics.
+
+ @Input psPhysHeap: Pointer to the physical heap object.
+ @Output puiTotalSize: Total size of the physical heap.
+ @Output puiInUseSize: Used space in the physical heap.
+
+ @Return PVRSRV_OK upon success and PVRSRV_otherwise.
+******************************************************************************/
+PVRSRV_ERROR LMA_HeapIteratorGetHeapStats(PHYS_HEAP_ITERATOR *psIter,
+                                          IMG_UINT64 *puiTotalSize,
+                                          IMG_UINT64 *puiInUseSize);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVPollForValueKM
+
+ @Description
+ Polls for a value to match a masked read
+
+ @Input psDevNode : Pointer to device node struct
+ @Input pui32LinMemAddr : CPU linear address to poll
+ @Input ui32Value : required value
+ @Input ui32Mask : Mask
+ @Input bDebugDumpOnFailure : Whether poll failure should result into a debug
+        dump. CAUTION: When calling this function from code paths which are
+        also used by debug-dumping code, this argument MUST be IMG_FALSE
+        otherwise, we might end up requesting debug-dump in recursion and
+        eventually blow-up call stack.
+ @Input pfnFwInvalidate : Function pointer to invalidation function used
+        each loop / poll. This is only used for FWmemctx allocations.
+
+ @Return   PVRSRV_ERROR :
+******************************************************************************/
+PVRSRV_ERROR PVRSRVPollForValueKM(PVRSRV_DEVICE_NODE *psDevNode,
+		volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+		IMG_UINT32                   ui32Value,
+		IMG_UINT32                   ui32Mask,
+		POLL_FLAGS                   ePollFlags,
+		PFN_INVALIDATE_CACHEFUNC     pfnFwInvalidate);
+
+/*!
+******************************************************************************
+ @Function	PVRSRVWaitForValueKM
+
+ @Description
+ Waits (using EventObjects) for a value to match a masked read
+
+ @Input  pui32LinMemAddr       : CPU linear address to poll
+ @Input  ui32Value             : Required value
+ @Input  ui32Mask              : Mask to be applied before checking against
+                                 ui32Value
+ @Input  pfnFwInvalidate       : Function pointer to invalidation function used
+                                 each loop / wait. This is only used for
+                                 FWmemctx allocations.
+ @Return PVRSRV_ERROR          :
+******************************************************************************/
+PVRSRV_ERROR
+PVRSRVWaitForValueKM(volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+                     IMG_UINT32                  ui32Value,
+                     IMG_UINT32                  ui32Mask,
+                     PFN_INVALIDATE_CACHEFUNC    pfnFwInvalidate);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemHasCacheSnooping
+
+ @Description	: Returns whether the system has cache snooping
+
+ @Return : IMG_TRUE if the system has cache snooping
+******************************************************************************/
+IMG_BOOL PVRSRVSystemHasCacheSnooping(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemSnoopingIsEmulated
+
+ @Description : Returns whether system cache snooping support is emulated
+
+ @Return : IMG_TRUE if the system cache snooping is emulated in software
+******************************************************************************/
+IMG_BOOL PVRSRVSystemSnoopingIsEmulated(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemSnoopingOfCPUCache
+
+ @Description	: Returns whether the system supports snooping of the CPU cache
+
+ @Return : IMG_TRUE if the system has CPU cache snooping
+******************************************************************************/
+IMG_BOOL PVRSRVSystemSnoopingOfCPUCache(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemHasNonMappableLocalMemory
+
+ @Description	: Returns whether the device has non-mappable part of local memory
+
+ @Return : IMG_TRUE if the device has non-mappable part of local memory
+******************************************************************************/
+IMG_BOOL PVRSRVSystemHasNonMappableLocalMemory(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVSystemWaitCycles
+
+ @Description	: Waits for at least ui32Cycles of the Device clk.
+******************************************************************************/
+void PVRSRVSystemWaitCycles(PVRSRV_DEVICE_CONFIG *psDevConfig, IMG_UINT32 ui32Cycles);
+
+PVRSRV_ERROR PVRSRVSystemInstallDeviceLISR(void *pvOSDevice,
+										   IMG_UINT32 ui32IRQ,
+										   const IMG_CHAR *pszName,
+										   PFN_LISR pfnLISR,
+										   void *pvData,
+										   IMG_HANDLE *phLISRData);
+
+PVRSRV_ERROR PVRSRVSystemUninstallDeviceLISR(IMG_HANDLE hLISRData);
+
+int PVRSRVGetDriverStatus(void);
+
+/*!
+******************************************************************************
+ @Function	: PVRSRVIsBridgeEnabled
+
+ @Description	: Returns whether the given bridge group is enabled
+
+ @Return : IMG_TRUE if the given bridge group is enabled
+******************************************************************************/
+static inline IMG_BOOL PVRSRVIsBridgeEnabled(IMG_HANDLE hServices, IMG_UINT32 ui32BridgeGroup)
+{
+	IMG_UINT32 ui32Bridges;
+	IMG_UINT32 ui32Offset;
+
+	PVR_UNREFERENCED_PARAMETER(hServices);
+
+#if defined(SUPPORT_RGX)
+	if (ui32BridgeGroup >= PVRSRV_BRIDGE_RGX_FIRST)
+	{
+		ui32Bridges = gui32RGXBridges;
+		ui32Offset = PVRSRV_BRIDGE_RGX_FIRST;
+	}
+	else
+#endif /* SUPPORT_RGX */
+	{
+		ui32Bridges = gui32PVRBridges;
+		ui32Offset = PVRSRV_BRIDGE_FIRST;
+	}
+
+	return (IMG_BOOL)(((1U << (ui32BridgeGroup - ui32Offset)) & ui32Bridges) != 0);
+}
+
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	void SetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState);
+	void SetTrustedDeviceAceEnabled(IMG_HANDLE hSysData);
+#endif
+#endif
+
+/*!
+******************************************************************************
+ @Function			: PVRSRVCreateHWPerfHostThread
+
+ @Description		: Creates HWPerf event object and thread unless already created
+
+ @Input ui32Timeout	: Initial timeout (ms) between updates on the HWPerf thread
+
+ @Return			: PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+******************************************************************************/
+PVRSRV_ERROR PVRSRVCreateHWPerfHostThread(IMG_UINT32 ui32Timeout);
+
+/*!
+******************************************************************************
+ @Function			: PVRSRVDestroyHWPerfHostThread
+
+ @Description		: Destroys HWPerf event object and thread if created
+
+ @Return			: PVRSRV_ERROR	PVRSRV_OK on success. Otherwise, a PVRSRV_
+									error code
+******************************************************************************/
+PVRSRV_ERROR PVRSRVDestroyHWPerfHostThread(void);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVFindPhysHeapConfig
+@Description    Find Phys Heap Config from Device Config.
+@Input          psDevConfig  Pointer to device config.
+@Input          ui32Flags    Find heap that matches flags.
+@Return         PHYS_HEAP_CONFIG*  Return a config, or NULL if not found.
+*/ /**************************************************************************/
+PHYS_HEAP_CONFIG* PVRSRVFindPhysHeapConfig(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										   PHYS_HEAP_USAGE_FLAGS ui32Flags);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDeviceInstance
+@Description    Return the specified device instance from Device node list.
+@Input          ui32Instance         Device instance to find
+@Return         PVRSRV_DEVICE_NODE*  Return a device node, or NULL if not found.
+*/ /**************************************************************************/
+PVRSRV_DEVICE_NODE* PVRSRVGetDeviceInstance(IMG_UINT32 ui32Instance);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetDeviceInstanceByKernelDevID
+@Description    Return the specified device instance by OS Id.
+@Input          i32OSInstance        OS device Id to find
+@Return         PVRSRV_DEVICE_NODE*  Return a device node, or NULL if not found.
+*/ /**************************************************************************/
+PVRSRV_DEVICE_NODE *PVRSRVGetDeviceInstanceByKernelDevID(IMG_INT32 i32OSInstance);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVAcquireInternalID
+@Description    Returns the lowest free device ID.
+@Output         pui32InternalID  The device ID
+@Return         PVRSRV_ERROR     PVRSRV_OK or an error code
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVAcquireInternalID(IMG_UINT32 *pui32InternalID);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDeviceFreeze
+@Description    Stops further processing from occurring on the device after the
+                 current work ends and blocks user-mode tasks on the specified
+                 device. Device node is put into POWER-DOWN state.
+@Input          PVRSRV_DEVICE_NODE*     Device node
+@Return         PVRSRV_ERROR            PVRSRV_OK on success otherwise a
+                                        PVRSRV_ERR_ code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceFreeze(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDeviceThaw
+@Description    Unfreezes a previously frozen device. Restarts work on the
+                 device, if work queues are non-empty, and unblocks any
+                 user-mode tasks on the specified device.
+@Input          PVRSRV_DEVICE_NODE*     Device node
+@Return         PVRSRV_ERROR            PVRSRV_OK on success otherwise a
+                                        PVRSRV_ERR_ code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVDeviceThaw(PVRSRV_DEVICE_NODE *psDevNode);
+
+#endif /* PVRSRV_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_apphint.h b/drivers/gpu/drm/img-rogue/pvrsrv_apphint.h
new file mode 100644
index 000000000000..e35426680795
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_apphint.h
@@ -0,0 +1,71 @@
+/**************************************************************************/ /*!
+@File
+@Title          PowerVR AppHint generic interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PVRSRV_APPHINT_H)
+#define PVRSRV_APPHINT_H
+
+/* Supplied to PVRSRVAppHintRegisterHandlers*() functions when the apphint
+ * is a global driver apphint, i.e. apphints not present in
+ * APPHINT_DEBUGFS_DEVICE_ID, i.e. not per device.
+ */
+#define APPHINT_OF_DRIVER_NO_DEVICE ((void*)-1U)
+
+#if defined(__linux__)
+
+#include "km_apphint.h"
+#define PVRSRVAppHintDumpState(d) pvr_apphint_dump_state(d)
+#define PVRSRVAppHintRegisterHandlersUINT64(i,q,s,d,p) pvr_apphint_register_handlers_uint64(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersUINT32(i,q,s,d,p) pvr_apphint_register_handlers_uint32(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersBOOL(i,q,s,d,p) pvr_apphint_register_handlers_bool(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersSTRING(i,q,s,d,p) pvr_apphint_register_handlers_string(i,q,s,d,p)
+
+#else
+
+#define PVRSRVAppHintDumpState(d)
+#define PVRSRVAppHintRegisterHandlersUINT64(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersUINT32(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersBOOL(i,q,s,d,p)
+#define PVRSRVAppHintRegisterHandlersSTRING(i,q,s,d,p)
+
+#endif
+
+#endif /* PVRSRV_APPHINT_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_bridge_init.c b/drivers/gpu/drm/img-rogue/pvrsrv_bridge_init.c
new file mode 100644
index 000000000000..13beb3f5c454
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_bridge_init.c
@@ -0,0 +1,379 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements common PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_bridge_init.h"
+#include "srvcore.h"
+
+/* These will go when full bridge gen comes in */
+#if defined(PDUMP)
+PVRSRV_ERROR InitPDUMPCTRLBridge(void);
+void DeinitPDUMPCTRLBridge(void);
+PVRSRV_ERROR InitPDUMPBridge(void);
+void DeinitPDUMPBridge(void);
+PVRSRV_ERROR InitRGXPDUMPBridge(void);
+void DeinitRGXPDUMPBridge(void);
+#endif
+#if defined(SUPPORT_DISPLAY_CLASS)
+PVRSRV_ERROR InitDCBridge(void);
+void DeinitDCBridge(void);
+#endif
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+#endif
+PVRSRV_ERROR InitPDUMPMMBridge(void);
+void DeinitPDUMPMMBridge(void);
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+#if defined(SUPPORT_DMA_TRANSFER)
+PVRSRV_ERROR InitDMABridge(void);
+void DeinitDMABridge(void);
+#endif
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+PVRSRV_ERROR InitRGXTQBridge(void);
+void DeinitRGXTQBridge(void);
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void);
+void DeinitRGXBREAKPOINTBridge(void);
+#endif
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+#endif
+#endif /* SUPPORT_RGX */
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+#if defined(SUPPORT_SECURE_EXPORT)
+PVRSRV_ERROR InitSMMBridge(void);
+void DeinitSMMBridge(void);
+#endif
+#if defined(PVRSRV_ENABLE_HTB)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+#endif
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+#endif
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+PVRSRV_ERROR InitVALIDATIONBridge(void);
+void DeinitVALIDATIONBridge(void);
+#endif
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+#if defined(SUPPORT_WRAP_EXTMEM)
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+#endif
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+PVRSRV_ERROR InitSYNCFALLBACKBridge(void);
+void DeinitSYNCFALLBACKBridge(void);
+#endif
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+#if defined(SUPPORT_DI_BRG_IMPL)
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+#endif
+
+PVRSRV_ERROR
+ServerBridgeInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	BridgeDispatchTableStartOffsetsInit();
+
+	eError = InitSRVCOREBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSRVCOREBridge");
+
+	eError = InitSYNCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCBridge");
+
+#if defined(PDUMP)
+	eError = InitPDUMPCTRLBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPCTRLBridge");
+#endif
+
+	eError = InitMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitMMBridge");
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+	eError = InitCMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitCMMBridge");
+#endif
+
+#if defined(PDUMP)
+	eError = InitPDUMPMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPMMBridge");
+
+	eError = InitPDUMPBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPDUMPBridge");
+#endif
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	eError = InitDCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDCBridge");
+#endif
+
+	eError = InitCACHEBridge();
+	PVR_LOG_IF_ERROR(eError, "InitCACHEBridge");
+
+#if defined(SUPPORT_SECURE_EXPORT)
+	eError = InitSMMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSMMBridge");
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+	eError = InitHTBUFFERBridge();
+	PVR_LOG_IF_ERROR(eError, "InitHTBUFFERBridge");
+#endif
+
+	eError = InitPVRTLBridge();
+	PVR_LOG_IF_ERROR(eError, "InitPVRTLBridge");
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = InitRIBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRIBridge");
+#endif
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+	eError = InitVALIDATIONBridge();
+	PVR_LOG_IF_ERROR(eError, "InitVALIDATIONBridge");
+#endif
+
+
+	eError = InitDEVICEMEMHISTORYBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDEVICEMEMHISTORYBridge");
+
+	eError = InitSYNCTRACKINGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCTRACKINGBridge");
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	eError = InitDMABridge();
+	PVR_LOG_IF_ERROR(eError, "InitDMABridge");
+#endif
+
+#if defined(SUPPORT_RGX)
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+	eError = InitRGXTQBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTQBridge");
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+	eError = InitRGXTA3DBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTA3DBridge");
+
+	#if defined(SUPPORT_USC_BREAKPOINT)
+	eError = InitRGXBREAKPOINTBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXBREAKPOINTBridge");
+#endif
+
+	eError = InitRGXFWDBGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXFWDBGBridge");
+
+#if defined(PDUMP)
+	eError = InitRGXPDUMPBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXPDUMPBridge");
+#endif
+
+	eError = InitRGXHWPERFBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXHWPERFBridge");
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+	eError = InitRGXREGCONFIGBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXREGCONFIGBridge");
+#endif
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	eError = InitRGXKICKSYNCBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXKICKSYNCBridge");
+#endif
+
+	eError = InitRGXTIMERQUERYBridge();
+	PVR_LOG_IF_ERROR(eError, "InitRGXTIMERQUERYBridge");
+
+#endif /* SUPPORT_RGX */
+
+#if defined(SUPPORT_WRAP_EXTMEM)
+	eError = InitMMEXTMEMBridge();
+	PVR_LOG_IF_ERROR(eError, "InitMMEXTMEMBridge");
+#endif
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	eError = InitSYNCFALLBACKBridge();
+	PVR_LOG_IF_ERROR(eError, "InitSYNCFALLBACKBridge");
+#endif
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	eError = InitDIBridge();
+	PVR_LOG_IF_ERROR(eError, "InitDIBridge");
+#endif
+
+	eError = OSPlatformBridgeInit();
+	PVR_LOG_IF_ERROR(eError, "OSPlatformBridgeInit");
+
+	return eError;
+}
+
+void ServerBridgeDeInit(void)
+{
+	OSPlatformBridgeDeInit();
+
+#if defined(SUPPORT_DI_BRG_IMPL)
+	DeinitDIBridge();
+#endif
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	DeinitSYNCFALLBACKBridge();
+#endif
+
+#if defined(SUPPORT_WRAP_EXTMEM)
+	DeinitMMEXTMEMBridge();
+#endif
+
+	DeinitSRVCOREBridge();
+
+	DeinitSYNCBridge();
+
+#if defined(PDUMP)
+	DeinitPDUMPCTRLBridge();
+#endif
+
+	DeinitMMBridge();
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+	DeinitCMMBridge();
+#endif
+
+#if defined(PDUMP)
+	DeinitPDUMPMMBridge();
+
+	DeinitPDUMPBridge();
+#endif
+
+
+#if defined(SUPPORT_DISPLAY_CLASS)
+	DeinitDCBridge();
+#endif
+
+	DeinitCACHEBridge();
+
+#if defined(SUPPORT_SECURE_EXPORT)
+	DeinitSMMBridge();
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+	DeinitHTBUFFERBridge();
+#endif
+
+	DeinitPVRTLBridge();
+
+#if defined(SUPPORT_VALIDATION_BRIDGE)
+	DeinitVALIDATIONBridge();
+#endif
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	DeinitRIBridge();
+#endif
+
+	DeinitDEVICEMEMHISTORYBridge();
+
+	DeinitSYNCTRACKINGBridge();
+
+#if defined(SUPPORT_DMA_TRANSFER)
+	DeinitDMABridge();
+#endif
+
+#if defined(SUPPORT_RGX)
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+	DeinitRGXTQBridge();
+#endif /* defined(SUPPORT_RGXTQ_BRIDGE) */
+
+	DeinitRGXTA3DBridge();
+
+#if defined(SUPPORT_USC_BREAKPOINT)
+	DeinitRGXBREAKPOINTBridge();
+#endif
+
+	DeinitRGXFWDBGBridge();
+
+#if defined(PDUMP)
+	DeinitRGXPDUMPBridge();
+#endif
+
+	DeinitRGXHWPERFBridge();
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+	DeinitRGXREGCONFIGBridge();
+#endif
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DeinitRGXKICKSYNCBridge();
+#endif
+
+	DeinitRGXTIMERQUERYBridge();
+#endif /* SUPPORT_RGX */
+}
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_bridge_init.h b/drivers/gpu/drm/img-rogue/pvrsrv_bridge_init.h
new file mode 100644
index 000000000000..b225da6d368e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_bridge_init.h
@@ -0,0 +1,53 @@
+/**************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the common PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_BRIDGE_INIT_H
+#define PVRSRV_BRIDGE_INIT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+PVRSRV_ERROR ServerBridgeInit(void);
+void ServerBridgeDeInit(void);
+
+#endif /* PVRSRV_BRIDGE_INIT_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_cleanup.h b/drivers/gpu/drm/img-rogue/pvrsrv_cleanup.h
new file mode 100644
index 000000000000..a197ca9ec908
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_cleanup.h
@@ -0,0 +1,256 @@
+/**************************************************************************/ /*!
+@File
+@Title          PowerVR SrvKM cleanup thread deferred work interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_CLEANUP_H
+#define PVRSRV_CLEANUP_H
+
+#include "dllist.h"
+#include "device.h"
+
+/**************************************************************************/ /*!
+@Brief          CLEANUP_THREAD_FN
+
+@Description    This is the function prototype for the pfnFree member found in
+                the structure PVRSRV_CLEANUP_THREAD_WORK. The function is
+                responsible for carrying out the clean up work and if successful
+                freeing the memory originally supplied to the call
+                PVRSRVCleanupThreadAddWork().
+
+@Input          pvParam  This is private data originally supplied by the caller
+                         to PVRSRVCleanupThreadAddWork() when registering the
+                         clean up work item, psDAta->pvData. Itr can be cast
+                         to a relevant type within the using module.
+
+@Return         PVRSRV_OK if the cleanup operation was successful and the
+                callback has freed the PVRSRV_CLEANUP_THREAD_WORK* work item
+                memory original supplied to PVRSRVCleanupThreadAddWork()
+                Any other error code will lead to the work item
+                being re-queued and hence the original
+                PVRSRV_CLEANUP_THREAD_WORK* must not be freed.
+*/ /***************************************************************************/
+
+typedef PVRSRV_ERROR (*CLEANUP_THREAD_FN)(void *pvParam);
+
+
+/* Typical number of times a caller should want the work to be retried in case
+ * of the callback function (pfnFree) returning an error.
+ * Callers to PVRSRVCleanupThreadAddWork should provide this value as the retry
+ * count (ui32RetryCount) unless there are special requirements.
+ * A value of 200 corresponds to around ~20s (200 * 100ms). If it is not
+ * successful by then give up as an unrecoverable problem has occurred.
+ */
+#define CLEANUP_THREAD_RETRY_COUNT_DEFAULT 200u
+/* Like for CLEANUP_THREAD_RETRY_COUNT_DEFAULT but call will wait for
+ * a specified amount of time rather than number of retries.
+ */
+#define CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT 20000u /* 20s */
+
+/* Use to set retry count on a cleanup item.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * _count - retry count
+ */
+#define CLEANUP_THREAD_SET_RETRY_COUNT(_item,_count) \
+	do { \
+		(_item)->ui32RetryCount = (_count); \
+		(_item)->ui32TimeStart = 0; \
+		(_item)->ui32TimeEnd = 0; \
+	} while (0)
+
+/* Use to set timeout deadline on a cleanup item.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * _timeout - timeout in milliseconds, if 0
+ *            CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT is used
+ */
+#define CLEANUP_THREAD_SET_RETRY_TIMEOUT(_item,_timeout) \
+	do { \
+		(_item)->ui32RetryCount = 0; \
+		(_item)->ui32TimeStart = OSClockms(); \
+		(_item)->ui32TimeEnd = (_item)->ui32TimeStart + ((_timeout) > 0 ? \
+				(_timeout) : CLEANUP_THREAD_RETRY_TIMEOUT_MS_DEFAULT); \
+	} while (0)
+
+/* Indicates if the timeout on a given item has been reached.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ */
+#define CLEANUP_THREAD_RETRY_TIMEOUT_NOT_REACHED(_item) \
+	((_item)->ui32TimeEnd - (_item)->ui32TimeStart >= \
+			OSClockms() - (_item)->ui32TimeStart)
+
+/* Indicates if the current item is waiting on timeout or retry count.
+ * _item - pointer to the PVRSRV_CLEANUP_THREAD_WORK
+ * */
+#define CLEANUP_THREAD_IS_RETRY_TIMEOUT(_item) \
+	((_item)->ui32TimeStart != (_item->ui32TimeEnd))
+
+#define CLEANUP_TYPE_LIST \
+	X(UNDEF)      /**/ \
+	X(CONNECTION)     /**/ \
+	X(MMU)     /**/ \
+	X(OSMEM)   /**/ \
+	X(PMR)       /**/ \
+	X(LAST)      /**/ \
+
+#define CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE 11     /* CONNECTION\0 */
+#define CLEANUP_TYPE_ITEM_DPF " %1.11s : %1.5d"
+#define CLEANUP_TYPE_ITEM_DPF_MAX_SIZE CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE+sizeof(" : ")+5+1
+
+typedef enum _PVRSRV_CLEANUP_TYPE_
+{
+#define X(_name) PVRSRV_CLEANUP_TYPE_ ## _name,
+	CLEANUP_TYPE_LIST
+#undef X
+
+} PVRSRV_CLEANUP_TYPE;
+
+#if defined(CLEANUP_TYPE_STRINGS)
+
+static const char *const _pszCleanupStrings[] = {
+#define X(_name) #_name,
+	CLEANUP_TYPE_LIST
+#undef X
+};
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetCleanupName
+@Description    Returns the name of a Cleanup Type.
+
+@Input          eCleanupType   The enum value of the cleanup type.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetCleanupName(PVRSRV_CLEANUP_TYPE eCleanupType)
+{
+	if (eCleanupType < 0 || eCleanupType > PVRSRV_CLEANUP_TYPE_LAST)
+	{
+		return "Undefined";
+	}
+
+	PVR_ASSERT(sizeof(_pszCleanupStrings[eCleanupType]) < CLEANUP_TYPE_ITEM_LABEL_MAX_SIZE);
+
+	return _pszCleanupStrings[eCleanupType];
+}
+
+#endif /* CLEANUP_TYPE_STRINGS */
+
+/* Clean up work item specifics so that the task can be managed by the
+* pvr_defer_free cleanup thread in the Server.
+*/
+typedef struct _PVRSRV_CLEANUP_THREAD_WORK_
+{
+	DLLIST_NODE sNode;               /*!< List node used internally by the cleanup
+	                                    thread */
+	CLEANUP_THREAD_FN pfnFree;       /*!< Pointer to the function to be called to
+	                                    carry out the deferred cleanup */
+	void *pvData;                    /*!< private data for pfnFree, usually a way back
+	                                    to the original PVRSRV_CLEANUP_THREAD_WORK*
+	                                    pointer supplied in the call to
+	                                    PVRSRVCleanupThreadAddWork(). */
+	IMG_UINT32 ui32TimeStart;        /*!< Timestamp in ms of the moment when
+	                                    cleanup item has been created. */
+	IMG_UINT32 ui32TimeEnd;          /*!< Time in ms after which no further retry
+	                                    attempts will be made, item discard and
+	                                    error logged when this is reached. */
+	IMG_UINT32 ui32RetryCount;       /*!< Number of times the callback should be
+	                                    re-tried when it returns error. */
+	IMG_BOOL bDependsOnHW;           /*!< Don't drop the cleanup task if retry limit
+	                                      is reached, we could depend on event from
+	                                      device to continue. */
+	PVRSRV_CLEANUP_TYPE eCleanupType;/*!< Type of work item added to queue */
+#if defined(DEBUG)
+	const char *pszFun;
+	unsigned int ui32LineNum;
+#endif
+} PVRSRV_CLEANUP_THREAD_WORK;
+
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadAddWork
+
+@Description    Add a work item to be called from the cleanup thread
+
+@Input          psDevNode : Pointer to the device node
+@Input          psData :    The function pointer and private data for the
+                            callback
+
+@Return         None
+*/ /***************************************************************************/
+#if defined(DEBUG)
+#define PVRSRVCleanupThreadAddWork(DEV, DATA)		PVRSRVCleanupThreadAddWork_Debug(DEV, DATA, __FILE__, __LINE__)
+void PVRSRVCleanupThreadAddWork_Debug(PVRSRV_DEVICE_NODE *psDevNode,
+                                      PVRSRV_CLEANUP_THREAD_WORK *psData,
+                                      const char *pszFun, const unsigned int ui32LineNum);
+#else
+#define PVRSRVCleanupThreadAddWork					PVRSRVCleanupThreadAddWork_Int
+void PVRSRVCleanupThreadAddWork_Int(PVRSRV_DEVICE_NODE *psDevNode,
+                                    PVRSRV_CLEANUP_THREAD_WORK *psData);
+#endif
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadWaitForDevice
+
+@Description    Blocking wait for all of the device's items to be cleaned.
+
+@Input          psDevNode : Pointer to the device node
+
+@Return         None
+*/ /***************************************************************************/
+void PVRSRVCleanupThreadWaitForDevice(PVRSRV_DEVICE_NODE *psDevNode);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadGetPid
+
+@Description    Returns Cleanup Thread's PID.
+
+@Return         PID of the Cleanup Thread
+*/ /***************************************************************************/
+IMG_PID PVRSRVCleanupThreadGetPid(void);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVCleanupThreadGetTid
+
+@Description    Returns Cleanup Thread's TID.
+
+@Return         TID of the Cleanup Thread
+*/ /***************************************************************************/
+uintptr_t PVRSRVCleanupThreadGetTid(void);
+
+#endif /* PVRSRV_CLEANUP_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_device.h b/drivers/gpu/drm/img-rogue/pvrsrv_device.h
new file mode 100644
index 000000000000..1ad03bb91323
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_device.h
@@ -0,0 +1,422 @@
+/**************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_DEVICE_H
+#define PVRSRV_DEVICE_H
+
+#include "img_types.h"
+#include "physheap_config.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_memalloc_physheap.h"
+#include "rgx_firmware_boot.h"
+#include "rgx_fwif_km.h"
+#include "servicesext.h"
+#include "cache_ops.h"
+#include "opaque_types.h"
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+#include "pvr_dvfs.h"
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+typedef struct _PVRSRV_DEVICE_CONFIG_ PVRSRV_DEVICE_CONFIG;
+typedef enum _DRIVER_MODE_
+{
+/* Do not use these enumerations directly, to query the
+   current driver mode, use the PVRSRV_VZ_MODE_IS()
+   macro */
+	DRIVER_MODE_NATIVE	= 0,
+	DRIVER_MODE_HOST	= 1,
+	DRIVER_MODE_GUEST	= 2,
+	DRIVER_MODE_DEFAULT	= 3,
+} PVRSRV_DRIVER_MODE;
+
+typedef enum
+{
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_MAPPABLE = 0,
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_NON_MAPPABLE = 1,
+	PVRSRV_DEVICE_LOCAL_MEMORY_ARENA_LAST
+} PVRSRV_DEVICE_LOCAL_MEMORY_ARENA;
+
+typedef enum _PVRSRV_DEVICE_SNOOP_MODE_
+{
+	PVRSRV_DEVICE_SNOOP_NONE = 0,
+	PVRSRV_DEVICE_SNOOP_CPU_ONLY,
+	PVRSRV_DEVICE_SNOOP_DEVICE_ONLY,
+	PVRSRV_DEVICE_SNOOP_CROSS,
+	PVRSRV_DEVICE_SNOOP_EMULATED,
+} PVRSRV_DEVICE_SNOOP_MODE;
+
+#if defined(SUPPORT_SOC_TIMER)
+typedef IMG_UINT64
+(*PFN_SYS_DEV_SOC_TIMER_READ)(IMG_HANDLE hSysData);
+#endif
+
+typedef enum _PVRSRV_DEVICE_FABRIC_TYPE_
+{
+	PVRSRV_DEVICE_FABRIC_NONE = 0,
+	PVRSRV_DEVICE_FABRIC_ACELITE,
+	PVRSRV_DEVICE_FABRIC_FULLACE,
+} PVRSRV_DEVICE_FABRIC_TYPE;
+
+typedef IMG_UINT32
+(*PFN_SYS_DEV_CLK_FREQ_GET)(IMG_HANDLE hSysData);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_PRE_POWER)(IMG_HANDLE hSysData,
+						 PVRSRV_SYS_POWER_STATE eNewPowerState,
+						 PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+						 PVRSRV_POWER_FLAGS ePwrFlags);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_POST_POWER)(IMG_HANDLE hSysData,
+						  PVRSRV_SYS_POWER_STATE eNewPowerState,
+						  PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+						  PVRSRV_POWER_FLAGS ePwrFlags);
+
+/*************************************************************************/ /*!
+@Brief          Callback function type PFN_SYS_GET_POWER
+
+@Description    This function queries the SoC power registers to determine
+                if the power domain on which the GPU resides is powered on.
+
+   Implementation of this callback is optional - where it is not provided,
+   the driver will assume the domain power state depending on driver type:
+   regular drivers assume it is unpowered at startup, while drivers with
+   AutoVz support expect the GPU domain to be powered on initially. The power
+   state will be then tracked internally according to the pfnPrePowerState
+   and pfnPostPowerState calls using a fallback function.
+
+@Input          psDevNode                  Pointer to node struct of the
+                                           device being initialised
+
+@Return         PVRSRV_SYS_POWER_STATE_ON  if the respective device's hardware
+                                           domain is powered on
+                PVRSRV_SYS_POWER_STATE_OFF if the domain is powered off
+*/ /**************************************************************************/
+typedef PVRSRV_SYS_POWER_STATE
+(*PFN_SYS_GET_POWER)(PPVRSRV_DEVICE_NODE psDevNode);
+
+typedef void
+(*PFN_SYS_DEV_INTERRUPT_HANDLED)(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+typedef PVRSRV_ERROR
+(*PFN_SYS_DEV_CHECK_MEM_ALLOC_SIZE)(IMG_HANDLE hSysData,
+									IMG_UINT64 ui64MemSize);
+
+typedef void (*PFN_SYS_DEV_FEAT_DEP_INIT)(PVRSRV_DEVICE_CONFIG *, IMG_UINT64);
+
+typedef void
+(*PFN_SYS_DEV_HOST_CACHE_MAINTENANCE)(IMG_HANDLE hSysData,
+									PVRSRV_CACHE_OP eRequestType,
+									void *pvVirtStart,
+									void *pvVirtEnd,
+									IMG_CPU_PHYADDR sCPUPhysStart,
+									IMG_CPU_PHYADDR sCPUPhysEnd);
+
+typedef void*
+(*PFN_SLAVE_DMA_CHAN)(PVRSRV_DEVICE_CONFIG*, char*);
+
+typedef void
+(*PFN_SLAVE_DMA_FREE)(PVRSRV_DEVICE_CONFIG*,
+					  void*);
+
+typedef void
+(*PFN_DEV_PHY_ADDR_2_DMA_ADDR)(PVRSRV_DEVICE_CONFIG *,
+							   IMG_DMA_ADDR *,
+							   IMG_DEV_PHYADDR *,
+							   IMG_BOOL *,
+							   IMG_UINT32,
+							   IMG_BOOL);
+
+
+typedef struct _PVRSRV_FW_PARAMS_
+{
+	const void *pvFirmware;
+	IMG_UINT32 ui32FirmwareSize;
+	const void *pvSignature;
+	IMG_UINT32 ui32SignatureSize;
+	PVRSRV_FW_BOOT_PARAMS uFWP;
+} PVRSRV_FW_PARAMS;
+
+typedef PVRSRV_ERROR
+(*PFN_PREPARE_FW_IMAGE)(IMG_HANDLE hSysData,
+						PVRSRV_FW_PARAMS *psFWParams);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+
+typedef PVRSRV_ERROR
+(*PFN_TD_SEND_FW_IMAGE)(IMG_HANDLE hSysData,
+						PVRSRV_FW_PARAMS *psTDFWParams);
+
+typedef struct _PVRSRV_TD_POWER_PARAMS_
+{
+	IMG_DEV_PHYADDR sPCAddr;
+
+	/* MIPS-only fields */
+	IMG_DEV_PHYADDR sGPURegAddr;
+	IMG_DEV_PHYADDR sBootRemapAddr;
+	IMG_DEV_PHYADDR sCodeRemapAddr;
+	IMG_DEV_PHYADDR sDataRemapAddr;
+} PVRSRV_TD_POWER_PARAMS;
+
+typedef PVRSRV_ERROR
+(*PFN_TD_SET_POWER_PARAMS)(IMG_HANDLE hSysData,
+						   PVRSRV_TD_POWER_PARAMS *psTDPowerParams);
+
+typedef PVRSRV_ERROR
+(*PFN_TD_RGXSTART)(IMG_HANDLE hSysData);
+
+typedef PVRSRV_ERROR
+(*PFN_TD_RGXSTOP)(IMG_HANDLE hSysData);
+
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+typedef void (*PFN_SYS_INIT_FIREWALL)(IMG_HANDLE hSysData,
+                                      IMG_UINT64[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                                      IMG_UINT64[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG_
+{
+	IMG_UINT32 ui32Status;     /*!< FW status */
+	IMG_UINT32 ui32Reason;     /*!< Reason for FW status */
+} PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG;
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF_
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;     /*!< FW page fault address */
+} PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF;
+
+typedef struct _PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM_
+{
+	IMG_UINT32 ui32ExtJobRef;     /*!< External Job Reference of any affected GPU work */
+	RGXFWIF_DM eDM;               /*!< Data Master which was running any affected GPU work */
+} PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM;
+
+typedef struct _PVRSRV_ROBUSTNESS_NOTIFY_DATA_
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reason for error/reset */
+	IMG_PID                  pid;          /*!< Pid of process which created the errored context */
+	union
+	{
+		PVRSRV_ROBUSTNESS_ERR_DATA_CHECKSUM sChecksumErrData; /*!< Data returned for checksum errors */
+		PVRSRV_ROBUSTNESS_ERR_DATA_FW_PF    sFwPFErrData;     /*!< Data returned for FW page faults */
+		PVRSRV_ROBUSTNESS_ERR_DATA_HOST_WDG sHostWdgData;     /*!< Data returned for Host Wdg FW faults */
+	} uErrData;
+} PVRSRV_ROBUSTNESS_NOTIFY_DATA;
+
+typedef void
+(*PFN_SYS_DEV_ERROR_NOTIFY)(IMG_HANDLE hSysData,
+						    PVRSRV_ROBUSTNESS_NOTIFY_DATA *psRobustnessErrorData);
+
+struct _PVRSRV_DEVICE_CONFIG_
+{
+	/*! OS device passed to SysDevInit (linux: 'struct device') */
+	void *pvOSDevice;
+
+	/*!
+	 *! Service representation of pvOSDevice. Should be set to NULL when the
+	 *! config is created in SysDevInit. Set by Services once a device node has
+	 *! been created for this config and unset before SysDevDeInit is called.
+	 */
+	struct _PVRSRV_DEVICE_NODE_ *psDevNode;
+
+	/*! Name of the device */
+	IMG_CHAR *pszName;
+
+	/*! Version of the device (optional) */
+	IMG_CHAR *pszVersion;
+
+	/*! Register bank address */
+	IMG_CPU_PHYADDR sRegsCpuPBase;
+	/*! Register bank size */
+	IMG_UINT32 ui32RegsSize;
+	/*! Device interrupt number */
+	IMG_UINT32 ui32IRQ;
+
+	PVRSRV_DEVICE_SNOOP_MODE eCacheSnoopingMode;
+
+	/*! Device specific data handle */
+	IMG_HANDLE hDevData;
+
+	/*! System specific data that gets passed into system callback functions. */
+	IMG_HANDLE hSysData;
+
+	IMG_BOOL bHasNonMappableLocalMemory;
+
+	/*! Indicates if system supports FBCDC v3.1 */
+	IMG_BOOL bHasFBCDCVersion31;
+
+	/*! Physical Heap definitions for this device.
+	 * eDefaultHeap must be set to GPU_LOCAL or CPU_LOCAL. Specifying any other value
+	 *    (e.g. DEFAULT) will lead to an error at device discovery.
+	 * pasPhysHeap array must contain at least one PhysHeap, the declared default heap.
+	 */
+	PVRSRV_PHYS_HEAP  eDefaultHeap;
+	PHYS_HEAP_CONFIG *pasPhysHeaps;
+	IMG_UINT32 ui32PhysHeapCount;
+
+	/*!
+	 *! Callbacks to change system device power state at the beginning and end
+	 *! of a power state change (optional).
+	 */
+	PFN_SYS_PRE_POWER pfnPrePowerState;
+	PFN_SYS_POST_POWER pfnPostPowerState;
+	PFN_SYS_GET_POWER  pfnGpuDomainPower;
+
+	/*! Callback to obtain the clock frequency from the device (optional). */
+	PFN_SYS_DEV_CLK_FREQ_GET pfnClockFreqGet;
+
+#if defined(SUPPORT_SOC_TIMER)
+	/*! Callback to read SoC timer register value (mandatory). */
+	PFN_SYS_DEV_SOC_TIMER_READ	pfnSoCTimerRead;
+#endif
+
+	/*!
+	 *! Callback to perform host CPU cache maintenance. Might be needed for
+	 *! architectures which allow extensions such as RISC-V (optional).
+	 */
+	PFN_SYS_DEV_HOST_CACHE_MAINTENANCE pfnHostCacheMaintenance;
+	IMG_BOOL bHasPhysicalCacheMaintenance;
+
+	/*!
+	 *! Callback to prepare FW image after it has been loaded.  This may
+	 *! be used to separate a signature/header from the firmware proper,
+	 *! potentially modifying pvFirmware and ui32FirmwareSize to point to the
+	 *! actual firmware to be loaded.
+	 */
+	PFN_PREPARE_FW_IMAGE pfnPrepareFWImage;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/*!
+	 *! Callback to send FW image and FW boot time parameters to the trusted
+	 *! device.
+	 */
+	PFN_TD_SEND_FW_IMAGE pfnTDSendFWImage;
+
+	/*!
+	 *! Callback to send parameters needed in a power transition to the trusted
+	 *! device.
+	 */
+	PFN_TD_SET_POWER_PARAMS pfnTDSetPowerParams;
+
+	/*! Callbacks to ping the trusted device to securely run RGXStart/Stop() */
+	PFN_TD_RGXSTART pfnTDRGXStart;
+	PFN_TD_RGXSTOP pfnTDRGXStop;
+
+#if defined(PVR_ANDROID_HAS_DMA_HEAP_FIND)
+	/*! Name of DMA heap to allocate secure memory from. Used with dma_heap_find. */
+	IMG_CHAR *pszSecureDMAHeapName;
+#endif
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) */
+
+	/*! Function that does device feature specific system layer initialisation */
+	PFN_SYS_DEV_FEAT_DEP_INIT	pfnSysDevFeatureDepInit;
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	PVRSRV_DVFS sDVFS;
+#endif
+
+#if defined(SUPPORT_ALT_REGBASE)
+	IMG_DEV_PHYADDR sAltRegsGpuPBase;
+#endif
+
+	/*!
+	 *! Indicates if device physical address 0x0 might be used as GPU memory
+	 *! (e.g. LMA system or UMA system with CPU PA 0x0 reserved by the OS,
+	 *!  but CPU PA != device PA and device PA 0x0 available for the GPU)
+	 */
+	IMG_BOOL bDevicePA0IsValid;
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+	/*!
+	 *! System-specific function to initialise firewall mechanism needed to
+	 *! validate the correctness of memory accesses tagged with OSIDs.
+	 *! Used for hardware validation of virtualization features only.
+	 */
+	PFN_SYS_INIT_FIREWALL		pfnSysInitFirewall;
+#endif
+
+	/*!
+	 *! Callback to notify system layer of device errors.
+	 *! NB. implementers should ensure that the minimal amount of work is
+	 *! done in the callback function, as it will be executed in the main
+	 *! RGX MISR. (e.g. any blocking or lengthy work should be performed by
+	 *! a worker queue/thread instead.)
+	 */
+	PFN_SYS_DEV_ERROR_NOTIFY	pfnSysDevErrorNotify;
+
+	/*!
+	 *!  Slave DMA channel request callbacks
+	 */
+	PFN_SLAVE_DMA_CHAN pfnSlaveDMAGetChan;
+	PFN_SLAVE_DMA_FREE pfnSlaveDMAFreeChan;
+	/*!
+	 *!  Conversion of device memory to DMA addresses
+	 */
+	PFN_DEV_PHY_ADDR_2_DMA_ADDR pfnDevPhysAddr2DmaAddr;
+	/*!
+	 *!  DMA channel names
+	 */
+	IMG_CHAR *pszDmaTxChanName;
+	IMG_CHAR *pszDmaRxChanName;
+	/*!
+	 *!  DMA device transfer restrictions
+	 */
+	IMG_UINT32 ui32DmaAlignment;
+	IMG_UINT32 ui32DmaTransferUnit;
+	/*!
+	 *!  System-wide presence of DMA capabilities
+	 */
+	IMG_BOOL bHasDma;
+
+	/*!
+	 *!  DriverMode required
+	 */
+	PVRSRV_DRIVER_MODE eDriverMode;
+};
+
+#endif /* PVRSRV_DEVICE_H*/
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_device_types.h b/drivers/gpu/drm/img-rogue/pvrsrv_device_types.h
new file mode 100644
index 000000000000..20e0d5ce3ed5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_device_types.h
@@ -0,0 +1,60 @@
+/*************************************************************************/ /*!
+@File
+@Title          PowerVR device type definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVRSRV_DEVICE_TYPES_H)
+#define PVRSRV_DEVICE_TYPES_H
+
+#include "img_types.h"
+
+#if !defined(PVRSRV_MAX_DEVICES)
+#error "PVRSRV_MAX_DEVICES must be defined in the DDK build environment"
+#endif	/* !defined(PVRSRV_NUM_DEVICES) */
+#define PVRSRV_HOST_DEVICE_ID	255U /*!< Device ID used for host (non-GPU) device. */
+
+static_assert(PVRSRV_MAX_DEVICES < PVRSRV_HOST_DEVICE_ID, "Invalid host device ID.");
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+#endif /* PVRSRV_DEVICE_TYPES_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_devvar.h b/drivers/gpu/drm/img-rogue/pvrsrv_devvar.h
new file mode 100644
index 000000000000..a8c64e309fda
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_devvar.h
@@ -0,0 +1,291 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Device Variable interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for device variables
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_DEVVAR_H
+#define PVRSRV_DEVVAR_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include <powervr/pvrsrv_sync_ext.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define DEVVAR_MAX_NAME_LEN 32
+
+typedef struct SYNC_PRIM_CONTEXT_TAG *PDEVVARCTX;
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_TAG *PDEVVAR;
+
+typedef struct PVRSRV_DEV_VAR_UPDATE_TAG
+{
+	PDEVVAR					psDevVar;			/*!< Pointer to the dev var */
+	IMG_UINT32				ui32UpdateValue;	/*!< the update value */
+} PVRSRV_DEV_VAR_UPDATE;
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarContextCreate
+
+@Description    Create a new device variable context
+
+@Input          psDevConnection         Device to create the device
+                                        variable context on
+
+@Output         phDevVarContext         Handle to the created device
+                                        variable context
+
+@Return         PVRSRV_OK if the device variable context was successfully
+                created
+*/
+/*****************************************************************************/
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVDevVarContextCreate(const PVRSRV_DEV_CONNECTION *psDevConnection,
+                          PDEVVARCTX                  *phDevVarContext);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarContextDestroy
+
+@Description    Destroy a device variable context
+
+@Input          hDevVarContext          Handle to the device variable
+                                        context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarContextDestroy(PDEVVARCTX hDevVarContext);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarAlloc
+
+@Description    Allocate a new device variable on the specified device
+                variable context. The device variable's value is initialised
+                with the value passed in ui32InitialValue.
+
+@Input          hDevVarContext          Handle to the device variable
+                                        context
+@Input          ui32InitialValue        Value to initially assign to the
+                                        new variable
+@Input          pszDevVarName           Name assigned to the device variable
+                                        (for debug purposes)
+
+@Output         ppsDevVar               Created device variable
+
+@Return         PVRSRV_OK if the device variable was successfully created
+*/
+/*****************************************************************************/
+IMG_EXPORT PVRSRV_ERROR
+PVRSRVDevVarAllocI(PDEVVARCTX hDevVarContext,
+                   PDEVVAR *ppsDevVar,
+                   IMG_UINT32 ui32InitialValue,
+                   const IMG_CHAR *pszDevVarName
+                   PVR_DBG_FILELINE_PARAM);
+#define PVRSRVDevVarAlloc(hDevVarContext, ppsDevVar, ui32InitialValue, pszDevVarName) \
+	PVRSRVDevVarAllocI( (hDevVarContext), (ppsDevVar), (ui32InitialValue), (pszDevVarName) \
+	                    PVR_DBG_FILELINE )
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarFree
+
+@Description    Free a device variable
+
+@Input          psDevVar                The device variable to free
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarFree(PDEVVAR psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarSet
+
+@Description    Set the device variable to a value
+
+@Input          psDevVar                The device variable to set
+
+@Input          ui32Value               Value to set it to
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarSet(PDEVVAR		psDevVar,
+                IMG_UINT32	ui32Value);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarGet
+
+@Description    Get the current value of the device variable
+
+@Input          psDevVar                The device variable to get the
+                                        value of
+
+@Return         Value of the variable
+*/
+/*****************************************************************************/
+IMG_EXPORT IMG_UINT32
+PVRSRVDevVarGet(PDEVVAR		psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarGetFirmwareAddr
+
+@Description    Returns the address of the associated firmware value for a
+                specified device integer (not exposed to client)
+
+@Input          psDevVar               The device variable to resolve
+
+@Return         The firmware address of the device variable
+*/
+/*****************************************************************************/
+IMG_EXPORT IMG_UINT32
+PVRSRVDevVarGetFirmwareAddr(PDEVVAR	psDevVar);
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDump
+
+@Description    PDump the current value of the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDump(PDEVVAR psDevVar);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDumpPol
+
+@Description    Do a PDump poll of the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Input          ui32Value               Value to Poll for
+
+@Input          ui32Mask                PDump mask operator
+
+@Input          ui32PDumpFlags          PDump flags
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDumpPol(PDEVVAR psDevVar,
+                     IMG_UINT32 ui32Value,
+                     IMG_UINT32 ui32Mask,
+                     PDUMP_POLL_OPERATOR eOperator,
+                     IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVDevVarPDumpCBP
+
+@Description    Do a PDump CB poll using the device variable
+
+@Input          psDevVar                The device variable to PDump
+
+@Input          uiWriteOffset           Current write offset of buffer
+
+@Input          uiPacketSize            Size of the packet to write into CB
+
+@Input          uiBufferSize            Size of the CB
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_EXPORT void
+PVRSRVDevVarPDumpCBP(PDEVVAR psDevVar,
+                     IMG_UINT64 uiWriteOffset,
+                     IMG_UINT64 uiPacketSize,
+                     IMG_UINT64 uiBufferSize);
+#else /* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDump)
+#endif
+static INLINE void
+PVRSRVDevVarPDump(PDEVVAR psDevVar)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevVar);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDumpPol)
+#endif
+static INLINE void
+PVRSRVDevVarPDumpPol(PDEVVAR psDevVar,
+                     IMG_UINT32 ui32Value,
+                     IMG_UINT32 ui32Mask,
+                     PDUMP_POLL_OPERATOR eOperator,
+                     IMG_UINT32 ui32PDumpFlags)
+{
+	 PVR_UNREFERENCED_PARAMETER(psDevVar);
+	 PVR_UNREFERENCED_PARAMETER(ui32Value);
+	 PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	 PVR_UNREFERENCED_PARAMETER(eOperator);
+	 PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVDevVarPDumpCBP)
+#endif
+static INLINE void
+PVRSRVDevVarPDumpCBP(PDEVVAR psDevVar,
+                     IMG_UINT64 uiWriteOffset,
+                     IMG_UINT64 uiPacketSize,
+                     IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevVar);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+}
+#endif /* PDUMP */
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* PVRSRV_DEVVAR_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_error.c b/drivers/gpu/drm/img-rogue/pvrsrv_error.c
new file mode 100644
index 000000000000..5cd02a28d5a1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_error.c
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services error support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+
+IMG_EXPORT
+const IMG_CHAR *PVRSRVGetErrorString(PVRSRV_ERROR eError)
+{
+	switch (eError)
+	{
+		case PVRSRV_OK:
+			return "PVRSRV_OK";
+#define PVRE(x) \
+		case x: \
+			return #x;
+#include "pvrsrv_errors.h"
+#undef PVRE
+		default:
+			return "Unknown PVRSRV error number";
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_error.h b/drivers/gpu/drm/img-rogue/pvrsrv_error.h
new file mode 100644
index 000000000000..120c42d87155
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_error.h
@@ -0,0 +1,75 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_error.h
+@Title          services error enumerant
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines error codes used by any/all services modules
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(PVRSRV_ERROR_H)
+#define PVRSRV_ERROR_H
+
+/*!
+ *****************************************************************************
+ * Error values
+ *****************************************************************************/
+typedef enum PVRSRV_ERROR_TAG
+{
+	PVRSRV_OK,
+
+#define PVRE(x) x,
+#include "pvrsrv_errors.h"
+#undef PVRE
+
+	PVRSRV_ERROR_FORCE_I32 = 0x7fffffff
+
+} PVRSRV_ERROR;
+
+/*!
+ * @Function PVRSRVIsRetryError
+ * @Description Checks if error code is one of the errors that require retry
+ *              from the caller.
+ * @Input eError Error code.
+ * @Return IMG_TRUE if eError is one of the error codes that require the caller
+ *         to retry.
+ */
+#define PVRSRVIsRetryError(eError) \
+	(((eError == PVRSRV_ERROR_RETRY) || (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)) ? \
+	 IMG_TRUE : IMG_FALSE)
+
+#endif /* !defined(PVRSRV_ERROR_H) */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_errors.h b/drivers/gpu/drm/img-rogue/pvrsrv_errors.h
new file mode 100644
index 000000000000..9fcbab6cf02d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_errors.h
@@ -0,0 +1,422 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_errors.h
+@Title          services error codes
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines error codes used by any/all services modules
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Don't add include guards to this file! */
+
+PVRE(PVRSRV_ERROR_OUT_OF_MEMORY)
+PVRE(PVRSRV_ERROR_TOO_FEW_BUFFERS)
+PVRE(PVRSRV_ERROR_INVALID_PARAMS)
+PVRE(PVRSRV_ERROR_INIT_FAILURE)
+PVRE(PVRSRV_ERROR_CANT_REGISTER_CALLBACK)
+PVRE(PVRSRV_ERROR_INVALID_DEVICE)
+PVRE(PVRSRV_ERROR_NOT_OWNER)
+PVRE(PVRSRV_ERROR_BAD_MAPPING)
+PVRE(PVRSRV_ERROR_TIMEOUT)
+PVRE(PVRSRV_ERROR_NOT_IMPLEMENTED)
+PVRE(PVRSRV_ERROR_FLIP_CHAIN_EXISTS)
+PVRE(PVRSRV_ERROR_INVALID_SWAPINTERVAL)
+PVRE(PVRSRV_ERROR_SCENE_INVALID)
+PVRE(PVRSRV_ERROR_STREAM_ERROR)
+PVRE(PVRSRV_ERROR_FAILED_DEPENDENCIES)
+PVRE(PVRSRV_ERROR_CMD_NOT_PROCESSED)
+PVRE(PVRSRV_ERROR_CMD_TOO_BIG)
+PVRE(PVRSRV_ERROR_DEVICE_REGISTER_FAILED)
+PVRE(PVRSRV_ERROR_TOOMANYBUFFERS)
+PVRE(PVRSRV_ERROR_NOT_SUPPORTED)
+PVRE(PVRSRV_ERROR_PROCESSING_BLOCKED)
+PVRE(PVRSRV_ERROR_CANNOT_FLUSH_QUEUE)
+PVRE(PVRSRV_ERROR_CANNOT_GET_QUEUE_SPACE)
+PVRE(PVRSRV_ERROR_CANNOT_GET_RENDERDETAILS)
+PVRE(PVRSRV_ERROR_RETRY)
+PVRE(PVRSRV_ERROR_DDK_VERSION_MISMATCH)
+PVRE(PVRSRV_ERROR_DDK_BUILD_MISMATCH)
+PVRE(PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH)
+PVRE(PVRSRV_ERROR_BVNC_MISMATCH)
+PVRE(PVRSRV_ERROR_FWPROCESSOR_MISMATCH)
+PVRE(PVRSRV_ERROR_UPLOAD_TOO_BIG)
+PVRE(PVRSRV_ERROR_INVALID_FLAGS)
+PVRE(PVRSRV_ERROR_FAILED_TO_REGISTER_PROCESS)
+PVRE(PVRSRV_ERROR_UNABLE_TO_LOAD_LIBRARY)
+PVRE(PVRSRV_ERROR_UNABLE_GET_FUNC_ADDR)
+PVRE(PVRSRV_ERROR_UNLOAD_LIBRARY_FAILED)
+PVRE(PVRSRV_ERROR_BRIDGE_CALL_FAILED)
+PVRE(PVRSRV_ERROR_IOCTL_CALL_FAILED)
+PVRE(PVRSRV_ERROR_MMU_API_PROTOCOL_ERROR)
+PVRE(PVRSRV_ERROR_MMU_CONFIG_IS_WRONG)
+PVRE(PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_ALLOCATE_PAGETABLES)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_CREATE_HEAP)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_MMU_FAILED_TO_UNMAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE)
+PVRE(PVRSRV_ERROR_MMU_LIVE_ALLOCATIONS_IN_HEAP)
+PVRE(PVRSRV_ERROR_MMU_RESERVATION_NOT_INSIDE_HEAP)
+PVRE(PVRSRV_ERROR_PMR_NEW_MEMORY)
+PVRE(PVRSRV_ERROR_PMR_STILL_REFERENCED)
+PVRE(PVRSRV_ERROR_PMR_CLIENT_NOT_TRUSTED)
+PVRE(PVRSRV_ERROR_PMR_FAILED_TO_ALLOC_PAGES)
+PVRE(PVRSRV_ERROR_PMR_INCOMPATIBLE_CONTIGUITY)
+PVRE(PVRSRV_ERROR_PMR_MISMATCHED_ATTRIBUTES)
+PVRE(PVRSRV_ERROR_PMR_NOT_PAGE_MULTIPLE)
+PVRE(PVRSRV_ERROR_PMR_NOT_PERMITTED)
+PVRE(PVRSRV_ERROR_PMR_ALREADY_OCCUPIED)
+PVRE(PVRSRV_ERROR_PMR_UNRECOVERABLE_ERROR)
+PVRE(PVRSRV_ERROR_PMR_WRONG_PASSWORD_OR_STALE_PMR)
+PVRE(PVRSRV_ERROR_PMR_WRONG_PMR_TYPE)
+PVRE(PVRSRV_ERROR_PMR_MAPPING_ALREADY_EXISTS)
+PVRE(PVRSRV_ERROR_PMR_BAD_MAPPINGTABLE_SIZE)
+PVRE(PVRSRV_ERROR_PMR_BAD_CHUNK_SIZE)
+PVRE(PVRSRV_ERROR_PMR_MAPPINGTABLE_MISMATCH)
+PVRE(PVRSRV_ERROR_PMR_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_PMR_NO_KERNEL_MAPPING)
+PVRE(PVRSRV_ERROR_PMR_EMPTY)
+PVRE(PVRSRV_ERROR_PMR_NO_CPU_MAP_FOUND)
+PVRE(PVRSRV_ERROR_PMR_CPU_PAGE_UNMAP_FAILED)
+PVRE(PVRSRV_ERROR_PMR_CPU_PAGE_MAP_FAILED)
+PVRE(PVRSRV_ERROR_PMR_PAGE_POISONING_FAILED)
+PVRE(PVRSRV_ERROR_PMR_INVALID_MAP_INDEX_ARRAY)
+PVRE(PVRSRV_ERROR_PMR_FREE_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_PMR_TOO_LARGE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP)
+PVRE(PVRSRV_ERROR_DEVICEMEM_BAD_IMPORT_SIZE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_CANT_EXPORT_SUBALLOCATION)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_CONFIG_INDEX)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_HEAP_INDEX)
+PVRE(PVRSRV_ERROR_DEVICEMEM_MAP_FAILED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_NON_ZERO_USAGE_COUNT)
+PVRE(PVRSRV_ERROR_DEVICEMEM_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_VA_ALLOC_FAILED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_UNABLE_TO_CREATE_ARENA)
+PVRE(PVRSRV_ERROR_DEVICEMEM_OUT_OF_DEVICE_VM)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ALREADY_MAPPED)
+PVRE(PVRSRV_ERROR_DEVICEMEM_NO_MAPPING)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_PMR_FLAGS)
+PVRE(PVRSRV_ERROR_DEVICEMEM_INVALID_LMA_HEAP)
+PVRE(PVRSRV_ERROR_INVALID_MMU_TYPE)
+PVRE(PVRSRV_ERROR_BUFFER_DEVICE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_BUFFER_DEVICE_ALREADY_PRESENT)
+PVRE(PVRSRV_ERROR_PCI_DEVICE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_PCI_CALL_FAILED)
+PVRE(PVRSRV_ERROR_PCI_REGION_TOO_SMALL)
+PVRE(PVRSRV_ERROR_PCI_REGION_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_BAD_REGION_SIZE_MISMATCH)
+PVRE(PVRSRV_ERROR_REGISTER_BASE_NOT_SET)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_USER_MEM)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_VP_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_SHARED_PBDESC)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_KERNELVIRTUAL)
+PVRE(PVRSRV_ERROR_FAILED_TO_GET_PHYS_ADDR)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_VIRT_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_COPY_VIRT_MEMORY)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES)
+PVRE(PVRSRV_ERROR_FAILED_TO_FREE_PAGES)
+PVRE(PVRSRV_ERROR_FAILED_TO_COPY_PAGES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_LOCK_PAGES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNLOCK_PAGES)
+PVRE(PVRSRV_ERROR_STILL_MAPPED)
+PVRE(PVRSRV_ERROR_MAPPING_NOT_FOUND)
+PVRE(PVRSRV_ERROR_PHYS_ADDRESS_EXCEEDS_32BIT)
+PVRE(PVRSRV_ERROR_FAILED_TO_MAP_PAGE_TABLE)
+PVRE(PVRSRV_ERROR_INVALID_SEGMENT_BLOCK)
+PVRE(PVRSRV_ERROR_INVALID_GFXDEVDEVDATA)
+PVRE(PVRSRV_ERROR_INVALID_DEVINFO)
+PVRE(PVRSRV_ERROR_INVALID_MEMINFO)
+PVRE(PVRSRV_ERROR_INVALID_MISCINFO)
+PVRE(PVRSRV_ERROR_UNKNOWN_IOCTL)
+PVRE(PVRSRV_ERROR_INVALID_CONTEXT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT)
+PVRE(PVRSRV_ERROR_INVALID_HEAP)
+PVRE(PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE)
+PVRE(PVRSRV_ERROR_INVALID_KERNELINFO)
+PVRE(PVRSRV_ERROR_UNKNOWN_POWER_STATE)
+PVRE(PVRSRV_ERROR_INVALID_HANDLE_TYPE)
+PVRE(PVRSRV_ERROR_INVALID_WRAP_TYPE)
+PVRE(PVRSRV_ERROR_INVALID_PHYS_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_CPU_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_HEAPINFO)
+PVRE(PVRSRV_ERROR_INVALID_PERPROC)
+PVRE(PVRSRV_ERROR_FAILED_TO_RETRIEVE_HEAPINFO)
+PVRE(PVRSRV_ERROR_INVALID_MAP_REQUEST)
+PVRE(PVRSRV_ERROR_INVALID_UNMAP_REQUEST)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FIND_MAPPING_HEAP)
+PVRE(PVRSRV_ERROR_MAPPING_STILL_IN_USE)
+PVRE(PVRSRV_ERROR_EXCEEDED_HW_LIMITS)
+PVRE(PVRSRV_ERROR_NO_STAGING_BUFFER_ALLOCATED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_PERPROC_AREA)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_THREAD)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CLOSE_THREAD)
+PVRE(PVRSRV_ERROR_THREAD_READ_ERROR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_ISR_HANDLER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSTALL_ISR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNINSTALL_ISR)
+PVRE(PVRSRV_ERROR_ISR_ALREADY_INSTALLED)
+PVRE(PVRSRV_ERROR_ISR_NOT_INSTALLED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INITIALISE_INTERRUPT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RETRIEVE_INFO)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DO_BACKWARDS_BLIT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CLOSE_SERVICES)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_CONTEXT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_RESOURCE)
+PVRE(PVRSRV_ERROR_INVALID_CCB_COMMAND)
+PVRE(PVRSRV_ERROR_KERNEL_CCB_FULL)
+PVRE(PVRSRV_ERROR_FLIP_FAILED)
+PVRE(PVRSRV_ERROR_UNBLANK_DISPLAY_FAILED)
+PVRE(PVRSRV_ERROR_TIMEOUT_POLLING_FOR_VALUE)
+PVRE(PVRSRV_ERROR_TIMEOUT_WAITING_FOR_CLIENT_CCB)
+PVRE(PVRSRV_ERROR_CREATE_RENDER_CONTEXT_FAILED)
+PVRE(PVRSRV_ERROR_UNKNOWN_PRIMARY_FRAG)
+PVRE(PVRSRV_ERROR_UNEXPECTED_SECONDARY_FRAG)
+PVRE(PVRSRV_ERROR_UNEXPECTED_PRIMARY_FRAG)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSERT_FENCE_ID)
+PVRE(PVRSRV_ERROR_BLIT_SETUP_FAILED)
+PVRE(PVRSRV_ERROR_SUBMIT_NEEDED)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_AVAILABLE)
+PVRE(PVRSRV_ERROR_PDUMP_BUFFER_FULL)
+PVRE(PVRSRV_ERROR_PDUMP_BUF_OVERFLOW)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_ACTIVE)
+PVRE(PVRSRV_ERROR_INCOMPLETE_LINE_OVERLAPS_PAGES)
+PVRE(PVRSRV_ERROR_MUTEX_DESTROY_FAILED)
+PVRE(PVRSRV_ERROR_MUTEX_INTERRUPTIBLE_ERROR)
+PVRE(PVRSRV_ERROR_INSUFFICIENT_SPACE_FOR_COMMAND)
+PVRE(PVRSRV_ERROR_PROCESS_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_PROCESS_NOT_FOUND)
+PVRE(PVRSRV_ERROR_SRV_CONNECT_FAILED)
+PVRE(PVRSRV_ERROR_SRV_DISCONNECT_FAILED)
+PVRE(PVRSRV_ERROR_DEINT_PHASE_FAILED)
+PVRE(PVRSRV_ERROR_INIT2_PHASE_FAILED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FIND_RESOURCE)
+PVRE(PVRSRV_ERROR_NO_DC_DEVICES_FOUND)
+PVRE(PVRSRV_ERROR_DC_DEVICE_INACCESSIBLE)
+PVRE(PVRSRV_ERROR_DC_INVALID_MAXDEPTH)
+PVRE(PVRSRV_ERROR_UNABLE_TO_OPEN_DC_DEVICE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_UNREGISTER_DEVICE)
+PVRE(PVRSRV_ERROR_NO_DEVICEDATA_FOUND)
+PVRE(PVRSRV_ERROR_NO_DEVICENODE_FOUND)
+PVRE(PVRSRV_ERROR_NO_CLIENTNODE_FOUND)
+PVRE(PVRSRV_ERROR_FAILED_TO_PROCESS_QUEUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INIT_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SCHEDULE_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_KILL_TASK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_TIMER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DISABLE_TIMER)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REMOVE_TIMER)
+PVRE(PVRSRV_ERROR_UNKNOWN_PIXEL_FORMAT)
+PVRE(PVRSRV_ERROR_HANDLE_INDEX_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_ALLOCATED)
+PVRE(PVRSRV_ERROR_HANDLE_TYPE_MISMATCH)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_SHAREABLE)
+PVRE(PVRSRV_ERROR_HANDLE_NOT_FOUND)
+PVRE(PVRSRV_ERROR_INVALID_SUBHANDLE)
+PVRE(PVRSRV_ERROR_HANDLE_BATCH_IN_USE)
+PVRE(PVRSRV_ERROR_HANDLE_BATCH_COMMIT_FAILURE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_HASH_TABLE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REMOVE_HASH_VALUE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_INSERT_HASH_VALUE)
+PVRE(PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_BACKING_STORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_BM_HEAP)
+PVRE(PVRSRV_ERROR_UNKNOWN_INIT_SERVER_STATE)
+PVRE(PVRSRV_ERROR_NO_FREE_DEVICEIDS_AVAILABLE)
+PVRE(PVRSRV_ERROR_INVALID_DEVICEID)
+PVRE(PVRSRV_ERROR_DEVICEID_NOT_FOUND)
+PVRE(PVRSRV_ERROR_MEMORY_TEST_FAILED)
+PVRE(PVRSRV_ERROR_CPUPADDR_TEST_FAILED)
+PVRE(PVRSRV_ERROR_COPY_TEST_FAILED)
+PVRE(PVRSRV_ERROR_SEMAPHORE_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RELEASE_CLOCK)
+PVRE(PVRSRV_ERROR_CLOCK_REQUEST_FAILED)
+PVRE(PVRSRV_ERROR_DISABLE_CLOCK_FAILURE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SET_CLOCK_RATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ROUND_CLOCK_RATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ENABLE_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_PARENT_CLOCK)
+PVRE(PVRSRV_ERROR_UNABLE_TO_GET_SYSTEM_CLOCK)
+PVRE(PVRSRV_ERROR_UNKNOWN_SGL_ERROR)
+PVRE(PVRSRV_ERROR_SYSTEM_POWER_CHANGE_FAILURE)
+PVRE(PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE)
+PVRE(PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+PVRE(PVRSRV_ERROR_BAD_SYNC_STATE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SET_CACHE_MODE)
+PVRE(PVRSRV_ERROR_FAILED_TO_ALLOC_MMUCONTEXT_ID)
+PVRE(PVRSRV_ERROR_PARAMETER_BUFFER_INVALID_ALIGNMENT)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ACQUIRE_CONNECTION)
+PVRE(PVRSRV_ERROR_UNABLE_TO_RELEASE_CONNECTION)
+PVRE(PVRSRV_ERROR_PHYSHEAP_ID_IN_USE)
+PVRE(PVRSRV_ERROR_PHYSHEAP_ID_INVALID)
+PVRE(PVRSRV_ERROR_PHYSHEAP_CONFIG)
+PVRE(PVRSRV_ERROR_HP_REQUEST_TOO_LONG)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_PRIM)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_PRIM_OP)
+PVRE(PVRSRV_ERROR_INVALID_SYNC_CONTEXT)
+PVRE(PVRSRV_ERROR_BP_NOT_SET)
+PVRE(PVRSRV_ERROR_BP_ALREADY_SET)
+PVRE(PVRSRV_ERROR_FEATURE_DISABLED)
+PVRE(PVRSRV_ERROR_REG_CONFIG_ENABLED)
+PVRE(PVRSRV_ERROR_REG_CONFIG_FULL)
+PVRE(PVRSRV_ERROR_REG_CONFIG_INVALID_TYPE)
+PVRE(PVRSRV_ERROR_MEMORY_ACCESS)
+PVRE(PVRSRV_ERROR_NO_SYSTEM_BUFFER)
+PVRE(PVRSRV_ERROR_DC_INVALID_CONFIG)
+PVRE(PVRSRV_ERROR_DC_INVALID_CROP_RECT)
+PVRE(PVRSRV_ERROR_DC_INVALID_DISPLAY_RECT)
+PVRE(PVRSRV_ERROR_DC_INVALID_BUFFER_DIMS)
+PVRE(PVRSRV_ERROR_DC_INVALID_TRANSFORM)
+PVRE(PVRSRV_ERROR_DC_INVALID_SCALE)
+PVRE(PVRSRV_ERROR_DC_INVALID_CUSTOM)
+PVRE(PVRSRV_ERROR_DC_TOO_MANY_PIPES)
+PVRE(PVRSRV_ERROR_DC_INVALID_PLANE_ALPHA)
+PVRE(PVRSRV_ERROR_NOT_READY)
+PVRE(PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_PIXEL_FORMAT)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_MEMORY_LAYOUT)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_FB_COMPRESSION_MODE)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_DIMS)
+PVRE(PVRSRV_ERROR_UNSUPPORTED_CACHE_MODE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_ADD_TIMER)
+PVRE(PVRSRV_ERROR_NOT_FOUND)
+PVRE(PVRSRV_ERROR_ALREADY_OPEN)
+PVRE(PVRSRV_ERROR_STREAM_MISUSE)
+PVRE(PVRSRV_ERROR_STREAM_FULL)
+PVRE(PVRSRV_ERROR_STREAM_READLIMIT_REACHED)
+PVRE(PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE)
+PVRE(PVRSRV_ERROR_PHYSMEM_NOT_ALLOCATED)
+PVRE(PVRSRV_ERROR_PBSIZE_ALREADY_MAX)
+PVRE(PVRSRV_ERROR_PBSIZE_ALREADY_MIN)
+PVRE(PVRSRV_ERROR_INVALID_PB_CONFIG)
+PVRE(PVRSRV_ERROR_META_THREAD0_NOT_ENABLED)
+PVRE(PVRSRV_ERROR_NOT_AUTHENTICATED)
+PVRE(PVRSRV_ERROR_REQUEST_TDFWMEM_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_INIT_TDFWMEM_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_REQUEST_TDSECUREBUF_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_INIT_TDSECUREBUF_PAGES_FAIL)
+PVRE(PVRSRV_ERROR_MUTEX_ALREADY_CREATED)
+PVRE(PVRSRV_ERROR_DBGTABLE_ALREADY_REGISTERED)
+PVRE(PVRSRV_ERROR_ALREADY_EXISTS)
+PVRE(PVRSRV_ERROR_UNABLE_TO_SEND_PULSE)
+PVRE(PVRSRV_ERROR_TASK_FAILED)
+PVRE(PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED)
+PVRE(PVRSRV_ERROR_INVALID_GPU_ADDR)
+PVRE(PVRSRV_ERROR_INVALID_OFFSET)
+PVRE(PVRSRV_ERROR_CCCB_STALLED)
+PVRE(PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE)
+PVRE(PVRSRV_ERROR_NOT_ENABLED)
+PVRE(PVRSRV_ERROR_SYSTEM_LOCAL_MEMORY_INIT_FAIL)
+PVRE(PVRSRV_ERROR_FW_IMAGE_MISMATCH)
+PVRE(PVRSRV_ERROR_PDUMP_NOT_ALLOWED)
+PVRE(PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL)
+PVRE(PVRSRV_ERROR_RPM_PBSIZE_ALREADY_MAX)
+PVRE(PVRSRV_ERROR_NONZERO_REFCOUNT)
+PVRE(PVRSRV_ERROR_SETAFFINITY_FAILED)
+PVRE(PVRSRV_ERROR_UNABLE_TO_COMPILE_PDS)
+PVRE(PVRSRV_ERROR_INTERNAL_ERROR)
+PVRE(PVRSRV_ERROR_BRIDGE_EFAULT)
+PVRE(PVRSRV_ERROR_BRIDGE_EINVAL)
+PVRE(PVRSRV_ERROR_BRIDGE_ENOMEM)
+PVRE(PVRSRV_ERROR_BRIDGE_ERANGE)
+PVRE(PVRSRV_ERROR_BRIDGE_EPERM)
+PVRE(PVRSRV_ERROR_BRIDGE_ENOTTY)
+PVRE(PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)
+PVRE(PVRSRV_ERROR_PROBE_DEFER)
+PVRE(PVRSRV_ERROR_INVALID_ALIGNMENT)
+PVRE(PVRSRV_ERROR_CLOSE_FAILED)
+PVRE(PVRSRV_ERROR_NOT_INITIALISED)
+PVRE(PVRSRV_ERROR_CONVERSION_FAILED)
+PVRE(PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL)
+PVRE(PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL)
+PVRE(PVRSRV_ERROR_RA_OUT_OF_RESOURCE)
+PVRE(PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS)
+PVRE(PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED)
+PVRE(PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED)
+PVRE(PVRSRV_ERROR_RA_FREE_INVALID_CHUNK)
+PVRE(PVRSRV_ERROR_OBJECT_STILL_REFERENCED)
+PVRE(PVRSRV_ERROR_BVNC_UNSUPPORTED)
+PVRE(PVRSRV_ERROR_INVALID_BVNC_PARAMS)
+PVRE(PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE)
+PVRE(PVRSRV_ERROR_DEVICEMEM_ADDITIONAL_HEAPS_IN_CONTEXT)
+PVRE(PVRSRV_ERROR_PID_ALREADY_REGISTERED)
+PVRE(PVRSRV_ERROR_PID_NOT_REGISTERED)
+PVRE(PVRSRV_ERROR_SIGNAL_FAILED)
+PVRE(PVRSRV_ERROR_INVALID_NOTIF_STREAM)
+PVRE(PVRSRV_ERROR_INVALID_SPU_MASK)
+PVRE(PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED)
+PVRE(PVRSRV_ERROR_INVALID_PVZ_CONFIG)
+PVRE(PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED)
+PVRE(PVRSRV_ERROR_NOT_SW_TIMELINE)
+PVRE(PVRSRV_ERROR_SW_TIMELINE_AT_LATEST_POINT)
+PVRE(PVRSRV_ERROR_INVALID_PVZ_OSID)
+PVRE(PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE)
+PVRE(PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG)
+PVRE(PVRSRV_ERROR_INTERRUPTED)
+PVRE(PVRSRV_ERROR_PWLOCK_RELEASED_REACQ_FAILED)
+PVRE(PVRSRV_ERROR_PDUMP_INVALID_BLOCKLEN)
+PVRE(PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF)
+PVRE(PVRSRV_ERROR_MULTIPLE_SECURITY_PDUMPS)
+PVRE(PVRSRV_ERROR_BAD_PARAM_SIZE)
+PVRE(PVRSRV_ERROR_INVALID_REQUEST)
+PVRE(PVRSRV_ERROR_FAILED_TO_ACQUIRE_PAGES)
+PVRE(PVRSRV_ERROR_TEST_FAILED)
+PVRE(PVRSRV_ERROR_SYNC_PRIM_OP_NOT_SUPPORTED)
+PVRE(PVRSRV_ERROR_FAILED_TO_GET_VIRT_ADDR)
+PVRE(PVRSRV_ERROR_UNABLE_TO_FREE_RESOURCE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_CREATE_SEMAPHORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_REGISTER_SEMAPHORE)
+PVRE(PVRSRV_ERROR_UNABLE_TO_DESTROY_SEMAPHORE)
+PVRE(PVRSRV_ERROR_TOO_MANY_SYNCS)
+PVRE(PVRSRV_ERROR_ION_NO_CLIENT)
+PVRE(PVRSRV_ERROR_ION_FAILED_TO_ALLOC)
+PVRE(PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE)
+PVRE(PVRSRV_ERROR_OUT_OF_RANGE)
+PVRE(PVRSRV_ERROR_OUT_OF_APP_POOL_MEMORY)
+PVRE(PVRSRV_ERROR_REFCOUNT_OVERFLOW)
+PVRE(PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+PVRE(PVRSRV_ERROR_UNEXPECTED_TRUE_EXPR)
+PVRE(PVRSRV_ERROR_UNEXPECTED_FALSE_EXPR)
+PVRE(PVRSRV_ERROR_KERNEL_CCB_OFFSET)
+PVRE(PVRSRV_ERROR_PMB_NOT_PAGE_MULTIPLE)
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_firmware_boot.h b/drivers/gpu/drm/img-rogue/pvrsrv_firmware_boot.h
new file mode 100644
index 000000000000..14a196d2767c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_firmware_boot.h
@@ -0,0 +1,87 @@
+/**************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_FIRMWARE_BOOT_H
+#define PVRSRV_FIRMWARE_BOOT_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+
+#define TD_MAX_NUM_MIPS_PAGETABLE_PAGES (4U)
+
+typedef union _PVRSRV_FW_BOOT_PARAMS_
+{
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCodeDevVAddr;
+		IMG_DEV_VIRTADDR sFWDataDevVAddr;
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_UINT32 ui32NumThreads;
+	} sMeta;
+
+	struct
+	{
+		IMG_DEV_PHYADDR sGPURegAddr;
+		IMG_DEV_PHYADDR asFWPageTableAddr[TD_MAX_NUM_MIPS_PAGETABLE_PAGES];
+		IMG_DEV_PHYADDR sFWStackAddr;
+		IMG_UINT32 ui32FWPageTableLog2PageSize;
+		IMG_UINT32 ui32FWPageTableNumPages;
+	} sMips;
+
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememDataSize;
+	} sRISCV;
+
+} PVRSRV_FW_BOOT_PARAMS;
+
+
+#endif /* PVRSRV_FIRMWARE_BOOT_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_memalloc_physheap.h b/drivers/gpu/drm/img-rogue/pvrsrv_memalloc_physheap.h
new file mode 100644
index 000000000000..a02fe954cd68
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_memalloc_physheap.h
@@ -0,0 +1,201 @@
+/*************************************************************************/ /*!
+@File           pvrsrv_memalloc_physheap.h
+@Title          Services Phys Heap types
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Used in creating and allocating from Physical Heaps.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVRSRV_MEMALLOC_PHYSHEAP_H
+#define PVRSRV_MEMALLOC_PHYSHEAP_H
+
+#include "img_defs.h"
+
+/*
+ * These IDs are replicated in the Device Memory allocation flags to allow
+ * allocations to be made in terms of their locality/use to ensure the correct
+ * physical heap is accessed for the given system/platform configuration.
+ * A system Phys Heap Config is linked to one or more Phys Heaps. When a heap
+ * is not present in the system configuration the allocation will fallback to
+ * the default GPU_LOCAL physical heap which all systems must define.
+ * See PVRSRV_MEMALLOCFLAGS_*_MAPPABLE_MASK.
+ *
+ * NOTE: Enum order important, table in physheap.c must change if order changed.
+ */
+#define PHYS_HEAP_LIST                                                                                                                           \
+	X(DEFAULT)       /* Client: default phys heap for device memory allocations */                                                               \
+	X(CPU_LOCAL)     /* Client: used for buffers with more CPU access than GPU */                                                                \
+	X(GPU_LOCAL)     /* Client: used for buffers with more GPU access than CPU */                                                                \
+	X(GPU_PRIVATE)   /* Client: used for buffers that only required GPU read/write access, not visible to the CPU. */                            \
+	X(FW_MAIN)       /* Internal: runtime data, e.g. CCBs, sync objects */                                                                       \
+	X(EXTERNAL)      /* Internal: used by some PMR import/export factories where the physical memory heap is not managed by the pvrsrv driver */ \
+	X(GPU_COHERENT)  /* Internal: used for a cache coherent region */                                                                            \
+	X(GPU_SECURE)    /* Internal: used by security validation */                                                                                 \
+	X(FW_CONFIG)     /* Internal: subheap of FW_MAIN, configuration data for FW init */                                                          \
+	X(FW_CODE)       /* Internal: used by security validation or dedicated fw */                                                                 \
+	X(FW_PRIV_DATA)  /* Internal: internal FW data (like the stack, FW control data structures, etc.) */                                         \
+	X(FW_PREMAP_PT)  /* Internal: page tables for premapped firmware memory */                                                                   \
+	X(FW_PREMAP0)    /* Internal: Host OS premap fw heap */                                                                                      \
+	X(FW_PREMAP1)    /* Internal: Guest OS 1 premap fw heap */                                                                                   \
+	X(FW_PREMAP2)    /* Internal: Guest OS 2 premap fw heap */                                                                                   \
+	X(FW_PREMAP3)    /* Internal: Guest OS 3 premap fw heap */                                                                                   \
+	X(FW_PREMAP4)    /* Internal: Guest OS 4 premap fw heap */                                                                                   \
+	X(FW_PREMAP5)    /* Internal: Guest OS 5 premap fw heap */                                                                                   \
+	X(FW_PREMAP6)    /* Internal: Guest OS 6 premap fw heap */                                                                                   \
+	X(FW_PREMAP7)    /* Internal: Guest OS 7 premap fw heap */                                                                                   \
+	X(WRAP)          /* External: Wrap memory */                                                                                                 \
+	X(DISPLAY)       /* External: Display memory */                                                                                              \
+	X(LAST)
+
+typedef enum _PVRSRV_PHYS_HEAP_
+{
+#define X(_name) PVRSRV_PHYS_HEAP_ ## _name,
+	PHYS_HEAP_LIST
+#undef X
+
+	PVRSRV_PHYS_HEAP_INVALID = 0x7FFFFFFF
+} PVRSRV_PHYS_HEAP;
+
+/* Defines the number of user mode physheaps. These physheaps are: DEFAULT, GPU_LOCAL,
+ * CPU_LOCAL, GPU_PRIVATE, GPU_SECURE. */
+#define MAX_USER_MODE_ALLOC_PHYS_HEAPS 5
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= (0x1FU + 1U), "Ensure enum fits in memalloc flags bitfield.");
+
+/*! Type conveys the class of physical heap to instantiate within Services
+ * for the physical pool of memory. */
+#define PHYS_HEAP_TYPE_LIST                                                                                                                                                                                                                                           \
+	X(UNKNOWN)              /* Not a valid value for any config */                                                                                                                                                                                                    \
+	X(UMA)                  /* Heap represents OS managed physical memory heap i.e. system RAM. Unified Memory Architecture physmem_osmem PMR factory */                                                                                                              \
+	X(LMA)                  /* Heap represents physical memory pool managed by Services i.e. carve out from system RAM or local card memory. Local Memory Architecture physmem_lma PMR factory */                                                                     \
+	X(DLM)                  /* Heap represents local card memory. Used in a DLM heap (Dedicated Local Memory) system. */                                                                                                                                              \
+	X(IMA)                  /* Heap represents phys heap that imports PMBs from a DLM heap.*/                                                                                                                                                                         \
+	X(DMA)                  /* Heap represents a physical memory pool managed by Services, alias of LMA and is only used on VZ non-native system configurations for a heap used for allocations tagged with PVRSRV_PHYS_HEAP_FW_MAIN or PVRSRV_PHYS_HEAP_FW_CONFIG */ \
+	X(WRAP)                 /* Heap used to group UM buffers given to Services. Integrity OS port only. */                                                                                                                                                            \
+	X(LAST)                                                                                                                                                                                                                                                           \
+
+typedef enum _PHYS_HEAP_TYPE_
+{
+#define X(_name) PHYS_HEAP_TYPE_ ## _name,
+	PHYS_HEAP_TYPE_LIST
+#undef X
+
+} PHYS_HEAP_TYPE;
+
+/* Defines used when interpreting the ui32PhysHeapFlags in PHYS_HEAP_MEM_STATS
+     0x000000000000000d
+     d = is this the default heap? (1=yes, 0=no)
+*/
+#define PVRSRV_PHYS_HEAP_FLAGS_IS_DEFAULT (0x1U)
+
+/* Force PHYS_HEAP_MEM_STATS size to be a multiple of 8 bytes
+ * (as type is a parameter in bridge calls)
+ */
+typedef struct PHYS_HEAP_MEM_STATS_TAG
+{
+	IMG_UINT64	ui64TotalSize;
+	IMG_UINT64	ui64FreeSize;
+	IMG_UINT32	ui32PhysHeapFlags;
+	PHYS_HEAP_TYPE	ePhysHeapType;
+}PHYS_HEAP_MEM_STATS, *PHYS_HEAP_MEM_STATS_PTR;
+
+#if defined(PHYSHEAP_STRINGS)
+
+static const char *const _pszPhysHeapStrings[] = {
+#define X(_name) #_name,
+	PHYS_HEAP_LIST
+#undef X
+};
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetClientPhysHeapTypeName
+@Description    Returns the phys heap type as a string.
+
+@Input          ePhysHeapType   The physheap type.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetClientPhysHeapTypeName(PHYS_HEAP_TYPE ePhysHeapType)
+{
+	switch (ePhysHeapType)
+	{
+#define X(_name) case PHYS_HEAP_TYPE_ ## _name: return "PHYS_HEAP_TYPE_" # _name;
+		PHYS_HEAP_TYPE_LIST
+#undef X
+		default:
+			return "Unknown Heap Type";
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetPhysHeapName
+@Description    Returns the name of a PhysHeap.
+
+@Input          ePhysHeap   The enum value of the physheap.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetPhysHeapName(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+	if (ePhysHeap < 0 || ePhysHeap >= PVRSRV_PHYS_HEAP_LAST)
+	{
+		return "Undefined";
+	}
+
+	return _pszPhysHeapStrings[ePhysHeap];
+}
+
+/*************************************************************************/ /*!
+@Function       PVRSRVGetClientPhysHeapName
+@Description    Returns the name of a client PhysHeap.
+
+@Input          ePhysHeap   The enum value of the physheap.
+
+@Return         const IMG_CHAR pointer.
+*/ /**************************************************************************/
+static inline const IMG_CHAR *PVRSRVGetClientPhysHeapName(PVRSRV_PHYS_HEAP ePhysHeap)
+{
+    if (ePhysHeap > PVRSRV_PHYS_HEAP_GPU_PRIVATE)
+    {
+        return "Unknown Heap";
+    }
+
+    return PVRSRVGetPhysHeapName(ePhysHeap);
+}
+#endif /* PHYSHEAP_STRINGS */
+
+#endif /* PVRSRV_MEMALLOC_PHYSHEAP_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_memallocflags.h b/drivers/gpu/drm/img-rogue/pvrsrv_memallocflags.h
new file mode 100644
index 000000000000..e08777797f65
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_memallocflags.h
@@ -0,0 +1,1130 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines flags used on memory allocations and mappings
+                These flags are relevant throughout the memory management
+                software stack and are specified by users of services and
+                understood by all levels of the memory management in both
+                client and server.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_MEMALLOCFLAGS_H
+#define PVRSRV_MEMALLOCFLAGS_H
+
+#include "img_types.h"
+#include "pvrsrv_memalloc_physheap.h"
+
+/*!
+  Type for specifying memory allocation flags.
+ */
+
+typedef IMG_UINT64 PVRSRV_MEMALLOCFLAGS_T;
+#define PVRSRV_MEMALLOCFLAGS_FMTSPEC IMG_UINT64_FMTSPECx
+
+#if defined(__KERNEL__)
+#include "pvrsrv_memallocflags_internal.h"
+#endif /* __KERNEL__ */
+
+/*
+ * --- MAPPING FLAGS      0..14 (15-bits) ---
+ * | 0-3    | 4-7    | 8-10        | 11-13       | 14          |
+ * | GPU-RW | CPU-RW | GPU-Caching | CPU-Caching | KM-Mappable |
+ *
+ * --- MISC FLAGS         15..20 (9-bits) ---
+ * | 15    | 16        | 17  | 18         | 19              | 20      |
+ * | Defer | Alloc-Now | SVM | Scratch-Pg | CPU-Cache-Clean | Zero-Pg |
+ *
+ * --- RI FLAGS  21..23 (3-bits) ---
+ * | 21     | 22       | 23        |
+ * | Import | Suballoc | FW alloc  |
+ *
+ * --- DEV CONTROL FLAGS  26..27 (2-bits) ---
+ * | 24-25 | 26-27        |
+ * | ...   | Device-Flags |
+ *
+ * --- MISC FLAGS         28..31 (4-bits) ---
+ * | 28             | 29             | 30          | 31            |
+ * | No-Cache-Align | Poison-On-Free | P.-On-Alloc | Zero-On-Alloc |
+ *
+ * --- VALIDATION FLAGS ---
+ * | 35             |
+ * | Shared-buffer  |
+ *
+ * --- OS SPECIFIC FLAGS ---
+ * | 36             | 37            |
+ * | Linux Pref CMA | Linux Movable |
+ *
+ * --- IPA Policy ---
+ * | 53-55      |
+ * | IPA Policy |
+ *
+ * --- PHYS HEAP HINTS ---
+ * | 56            | 57-58 | 59-63          |
+ * | Mandate Heap  |       | PhysHeap Hints |
+ *
+ */
+
+/*
+ *  **********************************************************
+ *  *                                                        *
+ *  *                      MAPPING FLAGS                     *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*!
+ * This flag affects the device MMU protection flags, and specifies
+ * that the memory may be read by the GPU.
+ *
+ * Typically all device memory allocations would specify this flag.
+ *
+ * At the moment, memory allocations without this flag are not supported
+ *
+ * This flag will live with the PMR, thus subsequent mappings would
+ * honour this flag.
+ *
+ * This is a dual purpose flag.  It specifies that memory is permitted
+ * to be read by the GPU, and also requests that the allocation is
+ * mapped into the GPU as a readable mapping
+ *
+ * To be clear:
+ * - When used as an argument on PMR creation; it specifies
+ *       that GPU readable mappings will be _permitted_
+ * - When used as an argument to a "map" function: it specifies
+ *       that a GPU readable mapping is _desired_
+ * - When used as an argument to "AllocDeviceMem": it specifies
+ *       that the PMR will be created with permission to be mapped
+ *       with a GPU readable mapping, _and_ that this PMR will be
+ *       mapped with a GPU readable mapping.
+ * This distinction becomes important when (a) we export allocations;
+ * and (b) when we separate the creation of the PMR from the mapping.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_READABLE		(IMG_UINT64_C(1)<<0)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_READABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_READABLE(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_READABLE) != 0U)
+
+/*!
+ * This flag affects the device MMU protection flags, and specifies
+ * that the memory may be written by the GPU
+ *
+ * Using this flag on an allocation signifies that the allocation is
+ * intended to be written by the GPU.
+ *
+ * Omitting this flag causes a read-only mapping.
+ *
+ * This flag will live with the PMR, thus subsequent mappings would
+ * honour this flag.
+ *
+ * This is a dual purpose flag.  It specifies that memory is permitted
+ * to be written by the GPU, and also requests that the allocation is
+ * mapped into the GPU as a writable mapping (see note above about
+ * permission vs. mapping mode, and why this flag causes permissions
+ * to be inferred from mapping mode on first allocation)
+ *
+ * N.B.  This flag has no relevance to the CPU's MMU mapping, if any,
+ * and would therefore not enforce read-only mapping on CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE       (IMG_UINT64_C(1)<<1)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITEABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as GPU readable in another GPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED  (IMG_UINT64_C(1)<<2)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_READ_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as GPU writable in another GPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED (IMG_UINT64_C(1)<<3)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITE_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED) != 0U)
+
+/*!
+  The flag indicates that an allocation is mapped as readable to the CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_READABLE        (IMG_UINT64_C(1)<<4)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_READABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_READABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_READABLE) != 0U)
+
+/*!
+  The flag indicates that an allocation is mapped as writable to the CPU.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE       (IMG_UINT64_C(1)<<5)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITEABLE(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as CPU readable in another CPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED  (IMG_UINT64_C(1)<<6)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_READ_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED) != 0U)
+
+/*!
+  The flag indicates whether an allocation can be mapped as CPU writable in another CPU memory context.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED (IMG_UINT64_C(1)<<7)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITE_PERMITTED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED) != 0U)
+
+
+/*
+ *  **********************************************************
+ *  *                                                        *
+ *  *                   CACHE CONTROL FLAGS                  *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*
+	GPU domain
+	==========
+
+	The following defines are used to control the GPU cache bit field.
+	The defines are mutually exclusive.
+
+	A helper macro, PVRSRV_GPU_CACHE_MODE, is provided to obtain just the GPU
+	cache bit field from the flags. This should be used whenever the GPU cache
+	mode needs to be determined.
+*/
+
+/*!
+  GPU domain. Flag indicating uncached memory. This means that any writes to memory
+  allocated with this flag are written straight to memory and thus are
+  coherent for any device in the system.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_UNCACHED				(IMG_UINT64_C(1)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_UNCACHED(uiFlags)				(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_UNCACHED)
+
+/*!
+   GPU domain. Use write combiner (if supported) to combine sequential writes
+   together to reduce memory access by doing burst writes.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC			(IMG_UINT64_C(0)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_WRITE_COMBINE(uiFlags)			(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC)
+
+/*!
+    GPU domain. This flag affects the GPU MMU protection flags.
+    The allocation will be cached.
+    Services will try to set the coherent bit in the GPU MMU tables so the
+    GPU cache is snooping the CPU cache. If coherency is not supported the
+    caller is responsible to ensure the caches are up to date.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT			(IMG_UINT64_C(2)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHE_COHERENT(uiFlags)		(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT)
+
+/*!
+   GPU domain. Request cached memory, but not coherent (i.e. no cache
+   snooping). Services will flush the GPU internal caches after every GPU
+   task so no cache maintenance requests from the users are necessary.
+
+    Note: We reserve 3 bits in the CPU/GPU cache mode to allow for future
+    expansion.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT		(IMG_UINT64_C(3)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHE_INCOHERENT(uiFlags)		(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT)
+
+/*!
+    GPU domain. This flag is for internal use only and is used to indicate
+    that the underlying allocation should be cached on the GPU after all
+    the snooping and coherent checks have been done
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHED					(IMG_UINT64_C(7)<<8)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_GPU_CACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_GPU_CACHED(uiFlags)				(PVRSRV_GPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_GPU_CACHED)
+
+/*!
+    GPU domain. GPU cache mode mask.
+*/
+#define PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK			(IMG_UINT64_C(7)<<8)
+
+/*!
+  @Description    A helper macro to obtain just the GPU	cache bit field from the flags.
+                  This should be used whenever the GPU cache mode needs to be determined.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the GPU cache bit field.
+ */
+#define PVRSRV_GPU_CACHE_MODE(uiFlags)					((uiFlags) & PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK)
+
+
+/*
+	CPU domain
+	==========
+
+	The following defines are used to control the CPU cache bit field.
+	The defines are mutually exclusive.
+
+	A helper macro, PVRSRV_CPU_CACHE_MODE, is provided to obtain just the CPU
+	cache bit field from the flags. This should be used whenever the CPU cache
+	mode needs to be determined.
+*/
+
+/*!
+   CPU domain. Use write combiner (if supported) to combine sequential writes
+   together to reduce memory access by doing burst writes.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC			(IMG_UINT64_C(0)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_WRITE_COMBINE(uiFlags)			(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC)
+
+/*!
+    CPU domain. This flag affects the CPU MMU protection flags.
+    The allocation will be cached.
+    Services will try to set the coherent bit in the CPU MMU tables so the
+    CPU cache is snooping the GPU cache. If coherency is not supported the
+    caller is responsible to ensure the caches are up to date.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT			(IMG_UINT64_C(2)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_COHERENT(uiFlags)		(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+
+/*!
+    CPU domain. Request cached memory, but not coherent (i.e. no cache
+    snooping). This means that if the allocation needs to transition from
+    one device to another services has to be informed so it can
+    flush/invalidate the appropriate caches.
+
+    Note: We reserve 3 bits in the CPU/GPU cache mode to allow for future
+    expansion.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT		(IMG_UINT64_C(3)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_INCOHERENT(uiFlags)		(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT)
+
+/*!
+    CPU domain. This flag is for internal use only and is used to indicate
+    that the underlying allocation should be cached on the CPU
+    after all the snooping and coherent checks have been done
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHED					(IMG_UINT64_C(7)<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHED(uiFlags)				(PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_CACHED)
+
+/*!
+	CPU domain. CPU cache mode mask
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK			(IMG_UINT64_C(7)<<11)
+
+/*!
+  @Description    A helper macro to obtain just the CPU	cache bit field from the flags.
+                  This should be used whenever the CPU cache mode needs to be determined.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the CPU cache bit field.
+ */
+#define PVRSRV_CPU_CACHE_MODE(uiFlags)					((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)
+
+/* Helper flags for usual cases */
+
+/*!
+ * Memory will be write-combined on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_UNCACHED_WC		(PVRSRV_MEMALLOCFLAG_GPU_UNCACHED_WC | PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_UNCACHED_WC mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_WRITE_COMBINE(uiFlags)				(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_UNCACHED_WC)
+
+/*!
+ * Memory will be cached.
+ * Services will try to set the correct flags in the MMU tables.
+ * In case there is no coherency support the caller has to ensure caches are up to date */
+#define PVRSRV_MEMALLOCFLAG_CACHE_COHERENT				(PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT | PVRSRV_MEMALLOCFLAG_CPU_CACHE_COHERENT)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CACHE_COHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CACHE_COHERENT(uiFlags)			(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CACHE_COHERENT)
+
+/*!
+ * Memory will be cache-incoherent on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT			(PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CACHE_INCOHERENT(uiFlags)			(PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CACHE_INCOHERENT)
+
+/*!
+	Cache mode mask
+*/
+#define PVRSRV_CACHE_MODE(uiFlags)						(PVRSRV_GPU_CACHE_MODE(uiFlags) | PVRSRV_CPU_CACHE_MODE(uiFlags))
+
+
+/*!
+   CPU MMU Flags mask -- intended for use internal to services only
+ */
+#define PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK  (PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+												PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+												PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK)
+
+/*!
+   MMU Flags mask -- intended for use internal to services only - used for
+   partitioning the flags bits and determining which flags to pass down to
+   mmu_common.c
+ */
+#define PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK  (PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK)
+
+/*!
+    Indicates that the PMR created due to this allocation will support
+    in-kernel CPU mappings.  Only privileged processes may use this flag as
+    it may cause wastage of precious kernel virtual memory on some platforms.
+ */
+#define PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE			(IMG_UINT64_C(1)<<14)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_KERNEL_CPU_MAPPABLE(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE) != 0U)
+
+
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                   ALLOC MEMORY FLAGS                   *
+ *  *                                                        *
+ *  **********************************************************
+ */
+
+/*!	----- Bit 15
+
+    Indicates when the allocation of physical memory pages backing the PMR
+    is carried out. When set, pages are not allocated at PMR creation but are
+    instead deferred until they are first needed, i.e. "on demand".
+    When unset, the pages may be allocated at the same time the PMR is created
+    or deferred (at the KM/Server's discretion).
+    See also PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW (below). Note that at most one
+    of these two flags may be set.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC			(IMG_UINT64_C(1)<<15)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_ON_DEMAND(uiFlags)					(((uiFlags) & PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC) != 0U)
+
+/*! ----- Bit 16
+
+    Indicates when the allocation of physical memory pages backing the PMR
+    is carried out. When set, pages are allocated at PMR creation.
+    When unset, the pages may be allocated at the same time the PMR is created
+    or deferred (at the KM/Server's discretion).
+    See also PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC (above). Note that at most one
+    of these two flags may be set.
+ */
+#define PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW				(IMG_UINT64_C(1)<<16)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_PHYS_ALLOC_NOW(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW) != 0U)
+
+/*! ----- Bit 17
+
+    Indicates that the allocation will be accessed by the CPU and GPU using
+    the same virtual address, i.e. for all SVM allocs,
+    IMG_CPU_VIRTADDR == IMG_DEV_VIRTADDR
+ */
+#define PVRSRV_MEMALLOCFLAG_SVM_ALLOC					(IMG_UINT64_C(1)<<17)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_SVM_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_SVM_ALLOC(uiFlags)					(((uiFlags) & PVRSRV_MEMALLOCFLAG_SVM_ALLOC) != 0U)
+
+/*! ----- Bit 18
+
+    Indicates the particular memory that's being allocated is sparse and the
+    sparse regions should not be backed by scratch page
+ */
+#define PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING		(IMG_UINT64_C(1) << 18)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_IS_SPARSE_SCRATCH_BACKING_REQUIRED(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING) == 0U)
+
+/*! ----- Bit 19
+
+  Used to force Services to carry out at least one CPU cache flush and
+  invalidate on a CPU cached buffer during allocation of the memory. Applicable
+  to incoherent systems, it must be used for buffers which are CPU cached and
+  which will not be 100% written to by the CPU before the GPU accesses it. For
+  performance reasons, avoid usage if the whole buffer that is allocated is
+  written to by the CPU anyway before the next GPU kick, or if the system is
+  coherent.
+ */
+#define PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN				(IMG_UINT64_C(1)<<19)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_CACHE_CLEAN(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN) != 0U)
+
+/*! ----- Bit 20
+
+    Indicates the particular memory region should be backed by zero page.
+    This is different with zero on alloc flag such that only physically unbacked
+    pages are backed by zero page at the time of mapping.
+    The zero backed page is always with read only attribute irrespective of its
+    original attributes.
+ */
+#define PVRSRV_MEMALLOCFLAG_ZERO_BACKING			(IMG_UINT64_C(1) << 20)
+#define PVRSRV_IS_ZERO_BACKING_REQUIRED(uiFlags)		(((uiFlags) & \
+			PVRSRV_MEMALLOCFLAG_ZERO_BACKING) == PVRSRV_MEMALLOCFLAG_ZERO_BACKING)
+
+/*
+ ************************************************************
+ *                         RI flags                         *
+ ************************************************************
+ * The Flags are used exclusively by the RI (Reference Info)
+ * server for tracking KM allocations by various processes
+ *
+ * Import - Handle imported from other process
+ * Suballoc - Handle Suballocation of existing PMR
+ * FW Alloc - Used by FW during driver initialisation
+ *
+ * --- RI FLAGS  21..23 (3-bits) ---
+ * | 21     | 22       | 23        |
+ * | Import | Suballoc | FW alloc  |
+ *
+ */
+
+/*! ----- Bit 21
+  Used by RI server to register Allocation as an import from other process
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_IMPORT					(IMG_UINT64_C(1)<<21)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_IMPORT flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_IMPORT(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_IMPORT) != 0U)
+
+/*! ----- Bit 22
+  Used by RI server to register Allocation as a suballocation of existing PMR
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_SUBALLOC					(IMG_UINT64_C(1)<<22)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_SUBALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_SUBALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_SUBALLOC) != 0U)
+
+/*! ----- Bit 23
+  Used by RI server to register Allocation as FW/System process
+  'FW Alloc' also means the allocation is considered to belong to the SYS process
+  (ie that it will have a lifetime longer than the process which allocated it)
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC				(IMG_UINT64_C(1)<<23)
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC) != 0U)
+
+/*!
+  @Description    Macro passing the PVRSRV_MEMALLOCFLAG_RI_ Flags.
+  @Input  uiFlags Allocation flags.
+  @Return         Value of the RI server bit field
+ */
+#define PVRSRV_MEMALLOCFLAG_RI_MASK(uiFlags)			((uiFlags) & (IMG_UINT64_C(7)<<21))
+
+
+/*! ----- Bit 24
+  *
+    Not used.
+ */
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *           MEMORY ZEROING AND POISONING FLAGS           *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * Zero / Poison, on alloc/free
+ *
+ * We think the following usecases are required:
+ *
+ *  don't poison or zero on alloc or free
+ *     (normal operation, also most efficient)
+ *  poison on alloc
+ *     (for helping to highlight bugs)
+ *  poison on alloc and free
+ *     (for helping to highlight bugs)
+ *  zero on alloc
+ *     (avoid highlighting security issues in other uses of memory)
+ *  zero on alloc and poison on free
+ *     (avoid highlighting security issues in other uses of memory, while
+ *      helping to highlight a subset of bugs e.g. memory freed prematurely)
+ *
+ * Since there are more than 4, we can't encode this in just two bits,
+ * so we might as well have a separate flag for each of the three
+ * actions.
+ */
+
+/*!
+    Ensures that the memory allocated is initialised with zeroes.
+ */
+#define PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC				(IMG_UINT64_C(1)<<31)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_ZERO_ON_ALLOC(uiFlags)				(((uiFlags) & PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) != 0U)
+
+/*!
+    Scribbles over the allocated memory with a poison value
+
+    Not compatible with ZERO_ON_ALLOC
+
+    Poisoning is very deliberately _not_ reflected in PDump as we want
+    a simulation to cry loudly if the initialised data propagates to a
+    result.
+ */
+#define PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC				(IMG_UINT64_C(1)<<30)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_POISON_ON_ALLOC(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC) != 0U)
+
+#if defined(DEBUG)
+/*!
+    Causes memory to be trashed when freed, used when debugging only, not to be used
+    as a security measure.
+ */
+#define PVRSRV_MEMALLOCFLAG_POISON_ON_FREE				(IMG_UINT64_C(1)<<29)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_POISON_ON_FREE flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_POISON_ON_FREE(uiFlags)			(((uiFlags) & PVRSRV_MEMALLOCFLAG_POISON_ON_FREE) != 0U)
+#endif /* DEBUG */
+
+/*!
+    Avoid address alignment to a CPU or GPU cache line size.
+ */
+#define PVRSRV_MEMALLOCFLAG_NO_CACHE_LINE_ALIGN			(IMG_UINT64_C(1)<<28)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_CHECK_NO_CACHE_LINE_ALIGN flag is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the flag is set, false otherwise
+ */
+#define PVRSRV_CHECK_NO_CACHE_LINE_ALIGN(uiFlags)		(((uiFlags) & PVRSRV_MEMALLOCFLAG_NO_CACHE_LINE_ALIGN) != 0U)
+
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                Device specific MMU flags               *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 26 to 27)
+ *
+ * Some services controlled devices have device specific control bits in
+ * their page table entries, we need to allow these flags to be passed down
+ * the memory management layers so the user can control these bits.
+ * For example, RGX device has the file rgx_memallocflags.h
+ */
+
+/*!
+ * Offset of device specific MMU flags.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET		26
+
+/*!
+ * Mask for retrieving device specific MMU flags.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK		(IMG_UINT64_C(3) << PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET)
+
+/*!
+  @Description    Helper macro for setting device specific MMU flags.
+  @Input  uiFlags Flag index.
+  @Return         Flag vector with the specified bit set.
+ */
+#define PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(uiFlags)	\
+			(((PVRSRV_MEMALLOCFLAGS_T)(uiFlags) << PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_OFFSET) & \
+			PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                 Secure validation flags                *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bit 35)
+ *
+ */
+
+/*!
+    PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER
+ */
+
+#define PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER           (IMG_UINT64_C(1)<<35)
+#define PVRSRV_CHECK_SHARED_BUFFER(uiFlags)             (((uiFlags) & PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER) != 0U)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                OS Specific alloc flags                 *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 36 to 37)
+ *
+ */
+#define PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_OFFSET 36
+#define PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK          (IMG_UINT64_C(3) << PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_OFFSET)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA         (IMG_UINT64_C(1)<<36)
+#define PVRSRV_CHECK_OS_LINUX_PREFER_CMA(uiFlags)       (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA) != 0U)
+
+#define PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE           (IMG_UINT64_C(1)<<37)
+#define PVRSRV_CHECK_OS_LINUX_MOVABLE(uiFlags)         (((uiFlags) & PVRSRV_MEMALLOCFLAG_OS_LINUX_MOVABLE) != 0U)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                       IPA Policy                       *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 53 to 55)
+ *
+ */
+
+/*!
+ * Offset of Intermediate Physical Address (IPA) policy.
+ */
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET      53
+
+/*!
+ * Mask for retrieving IPA policy.
+ */
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK        (IMG_UINT64_C(7) << PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET)
+#define PVRSRV_MEMALLOCFLAG_IPA_POLICY(uiFlags)    (((uiFlags) & PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK) >> PVRSRV_MEMALLOCFLAG_IPA_POLICY_OFFSET)
+
+/*
+ *
+ *  **********************************************************
+ *  *                                                        *
+ *  *                 Phys Heap Hints                        *
+ *  *                                                        *
+ *  **********************************************************
+ *
+ * (Bits 56 to 63)
+ *
+ */
+
+/*!
+ * Ensures Physheap isn't reassigned when considered favourable by driver under a OOM condition.
+ */
+#define PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP            (IMG_UINT64_C(1)<<56)
+#define PVRSRV_CHECK_MANDATED_PHYSHEAP(uiFlags)         (((uiFlags) & PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP) != 0U)
+
+/*!
+ * Value of enum PVRSRV_PHYS_HEAP stored in memalloc flags. If not set
+ * i.e. PVRSRV_PHYS_HEAP_DEFAULT (value 0) used, the system layer defined default physical heap is used.
+ */
+#define PVRSRV_PHYS_HEAP_HINT_SHIFT        (59)
+#define PVRSRV_PHYS_HEAP_HINT_MASK         (IMG_UINT64_C(0x1F) << PVRSRV_PHYS_HEAP_HINT_SHIFT)
+
+
+/*!
+  @Description    Macro extracting the Phys Heap hint from memalloc flag value.
+  @Input uiFlags  Allocation flags
+  @Return         returns the value of the PHYS_HEAP_HINT bitfield
+ */
+#define PVRSRV_GET_PHYS_HEAP_HINT(uiFlags)      ((PVRSRV_PHYS_HEAP)(((uiFlags) & PVRSRV_PHYS_HEAP_HINT_MASK) \
+                                                 >> PVRSRV_PHYS_HEAP_HINT_SHIFT))
+
+/*!
+  @Description    Macro converting a Phys Heap value into a memalloc bitfield
+  @Input uiFlags  Device Phys Heap
+  @Return         returns a shifted bitfield with the Device Phys Heap value
+ */
+#define PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(PhysHeap)      ((((PVRSRV_MEMALLOCFLAGS_T)PVRSRV_PHYS_HEAP_ ## PhysHeap) << \
+                                                            PVRSRV_PHYS_HEAP_HINT_SHIFT) \
+                                                           & PVRSRV_PHYS_HEAP_HINT_MASK)
+/*!
+  @Description    Macro to replace an existing phys heap hint value in flags.
+  @Input PhysHeap Phys Heap Macro
+  @Input uiFlags  Allocation flags
+  @Return         N/A
+ */
+#define PVRSRV_SET_PHYS_HEAP_HINT(PhysHeap, uiFlags)	  (uiFlags) = ((uiFlags) & ~PVRSRV_PHYS_HEAP_HINT_MASK) | \
+                                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(PhysHeap)
+
+/*!
+  @Description    Macro to replace an existing phys heap hint value using Phys Heap value.
+  @Input PhysHeap Phys Heap Value
+  @Input uiFlags  Allocation flags
+  @Return         N/A
+ */
+#define PVRSRV_CHANGE_PHYS_HEAP_HINT(Physheap, uiFlags)	  (uiFlags) = ((uiFlags) & ~PVRSRV_PHYS_HEAP_HINT_MASK) | \
+                                                           (((PVRSRV_MEMALLOCFLAGS_T)(Physheap) << \
+                                                            PVRSRV_PHYS_HEAP_HINT_SHIFT) \
+                                                           & PVRSRV_PHYS_HEAP_HINT_MASK)
+
+/*!
+  @Description    Macros checking if a Phys Heap hint is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the hint is set, false otherwise
+ */
+#define PVRSRV_CHECK_PHYS_HEAP(PhysHeap, uiFlags) (PVRSRV_PHYS_HEAP_ ## PhysHeap == PVRSRV_GET_PHYS_HEAP_HINT(uiFlags))
+
+#define PVRSRV_CHECK_FW_MAIN(uiFlags)            (PVRSRV_CHECK_PHYS_HEAP(FW_MAIN, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_CONFIG, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_CODE, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PRIV_DATA, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP0, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP1, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP2, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP3, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP4, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP5, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP6, uiFlags) || \
+                                                  PVRSRV_CHECK_PHYS_HEAP(FW_PREMAP7, uiFlags))
+
+/*!
+ * Secure buffer mask -- Flags in the mask are allowed for secure buffers
+ * because they are not related to CPU mappings.
+ */
+#define PVRSRV_MEMALLOCFLAGS_SECBUFMASK  ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                           PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED)
+
+/*!
+ * Trusted device mask -- Flags in the mask are allowed for trusted device
+ * because the driver cannot access the memory
+ */
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_TDFWMASK    ~(PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                           PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                           PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+#else
+#define PVRSRV_MEMALLOCFLAGS_TDFWMASK    ~(PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                           PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+#endif
+
+/*!
+  PMR flags mask -- for internal services use only.  This is the set of flags
+  that will be passed down and stored with the PMR, this also includes the
+  MMU flags which the PMR has to pass down to mm_common.c at PMRMap time.
+*/
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK  (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                            PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                            PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW | \
+                                            PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER | \
+                                            PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                            PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                            PVRSRV_PHYS_HEAP_HINT_MASK)
+#else
+#define PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK  (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                            PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                            PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                            PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW | \
+                                            PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                            PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER | \
+                                            PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP  | \
+                                            PVRSRV_MEMALLOCFLAG_OS_ALLOCFLAG_MASK | \
+                                            PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                            PVRSRV_PHYS_HEAP_HINT_MASK)
+#endif
+
+/*!
+ * CPU mappable mask -- Any flag set in the mask requires memory to be CPU mappable
+ */
+#define PVRSRV_MEMALLOCFLAGS_CPU_MAPPABLE_MASK (PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                                PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                                PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)
+/*!
+  RA differentiation mask
+
+  for use internal to services
+
+  this is the set of flags bits that are able to determine whether a pair of
+  allocations are permitted to live in the same page table. Allocations
+  whose flags differ in any of these places would be allocated from separate
+  RA Imports and therefore would never coexist in the same page.
+  Special cases are zeroing and poisoning of memory. The caller is responsible
+  to set the sub-allocations to the value he wants it to be. To differentiate
+  between zeroed and poisoned RA Imports does not make sense because the
+  memory might be reused.
+
+*/
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK (PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK \
+                                                      & \
+                                                      ~(PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC   | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_FREE))
+#else
+#define PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK (PVRSRV_MEMALLOCFLAGS_PMRFLAGSMASK \
+                                                      & \
+                                                      ~(PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC   | \
+                                                        PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC))
+#endif
+/*!
+  Flags that affect _allocation_
+*/
+#define PVRSRV_MEMALLOCFLAGS_PERALLOCFLAGSMASK (0xFFFFFFFFU)
+
+/*!
+  Flags that affect _mapping_
+*/
+#define PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK   (PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK | \
+                                                    PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_DEFER_PHYS_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_SVM_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_BACKING | \
+                                                    PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING)
+
+#if ((~(PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK) & PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK) != 0U)
+#error PVRSRV_MEMALLOCFLAGS_PERMAPPINGFLAGSMASK is not a subset of PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK
+#endif
+
+
+/*!
+  Flags that affect _physical allocations_ in the DevMemX API
+ */
+#if defined(DEBUG)
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_PHYSICAL_MASK (PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_FREE | \
+                                                    PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW | \
+                                                    PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                                    PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                                    PVRSRV_PHYS_HEAP_HINT_MASK)
+#else
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_PHYSICAL_MASK (PVRSRV_MEMALLOCFLAGS_CPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_CACHE_MODE_MASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_WRITE_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_CPU_CACHE_CLEAN | \
+                                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC | \
+                                                    PVRSRV_MEMALLOCFLAG_PHYS_ALLOC_NOW | \
+                                                    PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP | \
+                                                    PVRSRV_MEMALLOCFLAG_IPA_POLICY_MASK | \
+                                                    PVRSRV_PHYS_HEAP_HINT_MASK)
+#endif
+
+/*!
+  Flags that affect _virtual allocations_ in the DevMemX API
+ */
+#define PVRSRV_MEMALLOCFLAGS_DEVMEMX_VIRTUAL_MASK  (PVRSRV_MEMALLOCFLAGS_GPU_MMUFLAGSMASK | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_READ_PERMITTED | \
+                                                    PVRSRV_MEMALLOCFLAG_GPU_WRITE_PERMITTED)
+
+#endif /* PVRSRV_MEMALLOCFLAGS_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_memallocflags_internal.h b/drivers/gpu/drm/img-rogue/pvrsrv_memallocflags_internal.h
new file mode 100644
index 000000000000..4fee3d49b927
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_memallocflags_internal.h
@@ -0,0 +1,78 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device Memory Management allocation flags for internal Services
+                use only
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This file defines flags used on memory allocations and mappings
+                These flags are relevant throughout the memory management
+                software stack and are specified by users of services and
+                understood by all levels of the memory management in the server
+                and in special cases in the client.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_MEMALLOCFLAGS_INTERNAL_H
+#define PVRSRV_MEMALLOCFLAGS_INTERNAL_H
+
+/*!
+   CPU domain. Request uncached memory. This means that any writes to memory
+   allocated with this flag are written straight to memory and thus are
+   coherent for any device in the system.
+*/
+#define PVRSRV_MEMALLOCFLAG_CPU_UNCACHED (1ULL<<11)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_CPU_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_CPU_UNCACHED(uiFlags) (PVRSRV_CPU_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_CPU_UNCACHED)
+
+/*!
+ * Memory will be uncached on CPU and GPU
+ */
+#define PVRSRV_MEMALLOCFLAG_UNCACHED (PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | PVRSRV_MEMALLOCFLAG_CPU_UNCACHED)
+
+/*!
+  @Description    Macro checking whether the PVRSRV_MEMALLOCFLAG_UNCACHED mode is set.
+  @Input  uiFlags Allocation flags.
+  @Return         True if the mode is set, false otherwise
+ */
+#define PVRSRV_CHECK_UNCACHED(uiFlags) (PVRSRV_CACHE_MODE(uiFlags) == PVRSRV_MEMALLOCFLAG_UNCACHED)
+
+#endif /* PVRSRV_MEMALLOCFLAGS_INTERNAL_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_pool.c b/drivers/gpu/drm/img-rogue/pvrsrv_pool.c
new file mode 100644
index 000000000000..9fa95ea98756
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_pool.c
@@ -0,0 +1,260 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services pool implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a generic pool implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "lock.h"
+#include "dllist.h"
+#include "allocmem.h"
+
+struct _PVRSRV_POOL_
+{
+	POS_LOCK hLock;
+	/* total max number of permitted entries in the pool */
+	IMG_UINT uiMaxEntries;
+	/* currently number of pool entries created. these may be in the pool
+	 * or in-use
+	 */
+	IMG_UINT uiNumBusy;
+	/* number of not-in-use entries currently free in the pool */
+	IMG_UINT uiNumFree;
+
+	DLLIST_NODE sFreeList;
+
+	const IMG_CHAR *pszName;
+
+	PVRSRV_POOL_ALLOC_FUNC *pfnAlloc;
+	PVRSRV_POOL_FREE_FUNC *pfnFree;
+	void *pvPrivData;
+};
+
+typedef struct _PVRSRV_POOL_ENTRY_
+{
+	DLLIST_NODE sNode;
+	void *pvData;
+} PVRSRV_POOL_ENTRY;
+
+PVRSRV_ERROR PVRSRVPoolCreate(PVRSRV_POOL_ALLOC_FUNC *pfnAlloc,
+					PVRSRV_POOL_FREE_FUNC *pfnFree,
+					IMG_UINT32 ui32MaxEntries,
+					const IMG_CHAR *pszName,
+					void *pvPrivData,
+					PVRSRV_POOL **ppsPool)
+{
+	PVRSRV_POOL *psPool;
+	PVRSRV_ERROR eError;
+
+	psPool = OSAllocMem(sizeof(PVRSRV_POOL));
+	PVR_GOTO_IF_NOMEM(psPool, eError, err_alloc);
+
+	eError = OSLockCreate(&psPool->hLock);
+
+	PVR_GOTO_IF_ERROR(eError, err_lock_create);
+
+	psPool->uiMaxEntries = ui32MaxEntries;
+	psPool->uiNumBusy = 0;
+	psPool->uiNumFree = 0;
+	psPool->pfnAlloc = pfnAlloc;
+	psPool->pfnFree = pfnFree;
+	psPool->pvPrivData = pvPrivData;
+	psPool->pszName = pszName;
+
+	dllist_init(&psPool->sFreeList);
+
+	*ppsPool = psPool;
+
+	return PVRSRV_OK;
+
+err_lock_create:
+	OSFreeMem(psPool);
+err_alloc:
+	return eError;
+}
+
+static PVRSRV_ERROR _DestroyPoolEntry(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_ENTRY *psEntry)
+{
+	psPool->pfnFree(psPool->pvPrivData, psEntry->pvData);
+	OSFreeMem(psEntry);
+
+	return PVRSRV_OK;
+}
+
+void PVRSRVPoolDestroy(PVRSRV_POOL *psPool)
+{
+	if (psPool->uiNumBusy != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Attempt to destroy pool %s "
+						"with %u entries still in use",
+						__func__,
+						psPool->pszName,
+						psPool->uiNumBusy));
+		return;
+	}
+
+	OSLockDestroy(psPool->hLock);
+
+	if (psPool->uiNumFree)
+	{
+		PVRSRV_POOL_ENTRY *psEntry;
+		DLLIST_NODE *psChosenNode;
+
+		psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+
+		while (psChosenNode)
+		{
+			dllist_remove_node(psChosenNode);
+
+			psEntry = IMG_CONTAINER_OF(psChosenNode, PVRSRV_POOL_ENTRY, sNode);
+			_DestroyPoolEntry(psPool, psEntry);
+
+			psPool->uiNumFree--;
+
+			psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+		}
+
+		PVR_ASSERT(psPool->uiNumFree == 0);
+	}
+
+	OSFreeMem(psPool);
+}
+
+static PVRSRV_ERROR _CreateNewPoolEntry(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_ENTRY **ppsEntry)
+{
+	PVRSRV_POOL_ENTRY *psNewEntry;
+	PVRSRV_ERROR eError;
+
+	psNewEntry = OSAllocMem(sizeof(PVRSRV_POOL_ENTRY));
+	PVR_GOTO_IF_NOMEM(psNewEntry, eError, err_allocmem);
+
+	dllist_init(&psNewEntry->sNode);
+
+	eError = psPool->pfnAlloc(psPool->pvPrivData, &psNewEntry->pvData);
+
+	PVR_GOTO_IF_ERROR(eError, err_pfn_alloc);
+
+	*ppsEntry = psNewEntry;
+
+	return PVRSRV_OK;
+
+err_pfn_alloc:
+	OSFreeMem(psNewEntry);
+err_allocmem:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVPoolGet(PVRSRV_POOL *psPool,
+					PVRSRV_POOL_TOKEN *hToken,
+					void **ppvDataOut)
+{
+	PVRSRV_POOL_ENTRY *psEntry;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psChosenNode;
+
+	OSLockAcquire(psPool->hLock);
+
+	psChosenNode = dllist_get_next_node(&psPool->sFreeList);
+	if (unlikely(psChosenNode == NULL))
+	{
+		/* no available elements in the pool. try to create one */
+
+		eError = _CreateNewPoolEntry(psPool, &psEntry);
+
+		PVR_GOTO_IF_ERROR(eError, out_unlock);
+	}
+	else
+	{
+		dllist_remove_node(psChosenNode);
+
+		psEntry = IMG_CONTAINER_OF(psChosenNode, PVRSRV_POOL_ENTRY, sNode);
+
+		psPool->uiNumFree--;
+	}
+
+#if defined(DEBUG)
+	/* Don't poison the IN buffer as that is copied from client and would be
+	 * waste of cycles.
+	 */
+	OSCachedMemSet(((IMG_PBYTE)psEntry->pvData)+PVRSRV_MAX_BRIDGE_IN_SIZE,
+			PVRSRV_POISON_ON_ALLOC_VALUE, PVRSRV_MAX_BRIDGE_OUT_SIZE);
+#endif
+
+	psPool->uiNumBusy++;
+	*hToken = psEntry;
+	*ppvDataOut = psEntry->pvData;
+
+out_unlock:
+	OSLockRelease(psPool->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVPoolPut(PVRSRV_POOL *psPool, PVRSRV_POOL_TOKEN hToken)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_POOL_ENTRY *psEntry = hToken;
+
+	PVR_ASSERT(psPool->uiNumBusy > 0);
+
+	OSLockAcquire(psPool->hLock);
+
+	/* put this entry in the pool if the pool has space,
+	 * otherwise free it
+	 */
+	if (psPool->uiNumFree < psPool->uiMaxEntries)
+	{
+		dllist_add_to_tail(&psPool->sFreeList, &psEntry->sNode);
+		psPool->uiNumFree++;
+	}
+	else
+	{
+		eError = _DestroyPoolEntry(psPool, psEntry);
+	}
+
+	psPool->uiNumBusy--;
+
+	OSLockRelease(psPool->hLock);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_pool.h b/drivers/gpu/drm/img-rogue/pvrsrv_pool.h
new file mode 100644
index 000000000000..2272fc50ce6e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_pool.h
@@ -0,0 +1,135 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services pool implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides a generic pool implementation.
+                The pool allows to dynamically retrieve and return entries from
+                it using functions pair PVRSRVPoolGet/PVRSRVPoolPut. The entries
+                are created in lazy manner which means not until first usage.
+                The pool API allows to pass and allocation/free functions
+                pair that will allocate entry's private data and return it
+                to the caller on every entry 'Get'.
+                The pool will keep up to ui32MaxEntries entries allocated.
+                Every entry that exceeds this number and is 'Put' back to the
+                pool will be freed on the spot instead being returned to the
+                pool.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(PVRSRVPOOL_H)
+#define PVRSRVPOOL_H
+
+/**************************************************************************/ /*!
+ @Description  Callback function called during creation of the new element. This
+               function allocates an object that will be stored in the pool.
+               The object can be retrieved from the pool by calling
+               PVRSRVPoolGet.
+ @Input        pvPrivData      Private data passed to the alloc function.
+ @Output       pvOut           Allocated object.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+typedef PVRSRV_ERROR (PVRSRV_POOL_ALLOC_FUNC)(void *pvPrivData, void **pvOut);
+
+/**************************************************************************/ /*!
+ @Description  Callback function called to free the object allocated by
+               the counterpart alloc function.
+ @Input        pvPrivData      Private data passed to the free function.
+ @Output       pvFreeData      Object allocated by PVRSRV_POOL_ALLOC_FUNC.
+*/ /***************************************************************************/
+typedef void (PVRSRV_POOL_FREE_FUNC)(void *pvPrivData, void *pvFreeData);
+
+typedef IMG_HANDLE PVRSRV_POOL_TOKEN;
+
+typedef struct _PVRSRV_POOL_ PVRSRV_POOL;
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolCreate
+ @Description  Creates new buffer pool.
+ @Input        pfnAlloc        Allocation function pointer. Function is used
+                               to allocate new pool entries' data.
+ @Input        pfnFree         Free function pointer. Function is used to
+                               free memory allocated by pfnAlloc function.
+ @Input        ui32MaxEntries  Total maximum number of entries in the pool.
+ @Input        pszName         Name of the pool. String has to be NULL
+                               terminated.
+ @Input        pvPrivData      Private data that will be passed to pfnAlloc and
+                               pfnFree functions.
+ @Output       ppsPool         New buffer pool object.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolCreate(PVRSRV_POOL_ALLOC_FUNC *pfnAlloc,
+					PVRSRV_POOL_FREE_FUNC *pfnFree,
+					IMG_UINT32 ui32MaxEntries,
+					const IMG_CHAR *pszName,
+					void *pvPrivData,
+					PVRSRV_POOL **ppsPool);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolDestroy
+ @Description  Destroys pool created by PVRSRVPoolCreate.
+ @Input        psPool          Buffer pool object meant to be destroyed.
+*/ /***************************************************************************/
+void PVRSRVPoolDestroy(PVRSRV_POOL *psPool);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolGet
+ @Description  Retrieves an entry from a pool. If no free elements are
+               available new entry will be allocated.
+ @Input        psPool          Pointer to the pool.
+ @Output       hToken          Pointer to the entry handle.
+ @Output       ppvDataOut      Pointer to data stored in the entry (the data
+                               allocated by the pfnAlloc function).
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolGet(PVRSRV_POOL *psPool,
+						PVRSRV_POOL_TOKEN *hToken,
+						void **ppvDataOut);
+
+/**************************************************************************/ /*!
+ @Function     PVRSRVPoolPut
+ @Description  Returns entry to the pool. If number of entries is greater
+               than ui32MaxEntries set during pool creation the entry will
+               be freed instead.
+ @Input        psPool          Pointer to the pool.
+ @Input        hToken          Entry handle.
+ @Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR PVRSRVPoolPut(PVRSRV_POOL *psPool,
+						PVRSRV_POOL_TOKEN hToken);
+
+#endif /* PVRSRVPOOL_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_sync_km.h b/drivers/gpu/drm/img-rogue/pvrsrv_sync_km.h
new file mode 100644
index 000000000000..04611f9f7cee
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_sync_km.h
@@ -0,0 +1,65 @@
+/*************************************************************************/ /*!
+@File
+@Title         PVR synchronisation interface
+@Copyright     Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description   Types for server side code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVRSRV_SYNC_KM_H
+#define PVRSRV_SYNC_KM_H
+
+#include <powervr/pvrsrv_sync_ext.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define SYNC_FB_FILE_STRING_MAX			256
+#define SYNC_FB_MODULE_STRING_LEN_MAX	(32)
+#define	SYNC_FB_DESC_STRING_LEN_MAX		(32)
+
+/* By default, fence-sync module emits into HWPerf (of course, if enabled) and
+ * considers a process (sleepable) context */
+#define PVRSRV_FENCE_FLAG_NONE             (0U)
+#define PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT (1U << 0)
+#define PVRSRV_FENCE_FLAG_CTX_ATOMIC       (1U << 1)
+
+#if defined(__cplusplus)
+}
+#endif
+#endif	/* PVRSRV_SYNC_KM_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_sync_server.h b/drivers/gpu/drm/img-rogue/pvrsrv_sync_server.h
new file mode 100644
index 000000000000..5d1a10cd0a9c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_sync_server.h
@@ -0,0 +1,278 @@
+/**************************************************************************/ /*!
+@File
+@Title          Fence sync server interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef PVRSRV_SYNC_SERVER_H
+#define PVRSRV_SYNC_SERVER_H
+
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+#include "sync_fallback_server.h"
+#include "pvr_notifier.h"
+#include "img_types.h"
+#include "pvrsrv_sync_km.h"
+#elif defined(SUPPORT_NATIVE_FENCE_SYNC)
+#include "pvr_sync.h"
+#endif
+
+#include "rgxhwperf.h"
+
+#define SYNC_SW_TIMELINE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+#define SYNC_SW_FENCE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+
+typedef struct _SYNC_TIMELINE_OBJ_
+{
+	void *pvTlObj; /* Implementation specific timeline object */
+
+	PVRSRV_TIMELINE hTimeline; /* Reference to implementation-independent timeline object */
+} SYNC_TIMELINE_OBJ;
+
+typedef struct _SYNC_FENCE_OBJ_
+{
+	void *pvFenceObj; /* Implementation specific fence object */
+
+	PVRSRV_FENCE hFence; /* Reference to implementation-independent fence object */
+} SYNC_FENCE_OBJ;
+
+static inline void SyncClearTimelineObj(SYNC_TIMELINE_OBJ *psSTO)
+{
+	psSTO->pvTlObj = NULL;
+	psSTO->hTimeline = PVRSRV_NO_TIMELINE;
+}
+
+static inline IMG_BOOL SyncIsTimelineObjValid(const SYNC_TIMELINE_OBJ *psSTO)
+{
+	return (IMG_BOOL)(psSTO->pvTlObj != NULL);
+}
+
+static inline void SyncClearFenceObj(SYNC_FENCE_OBJ *psSFO)
+{
+	psSFO->pvFenceObj = NULL;
+	psSFO->hFence = PVRSRV_NO_FENCE;
+}
+
+static inline IMG_BOOL SyncIsFenceObjValid(const SYNC_FENCE_OBJ *psSFO)
+{
+	return (IMG_BOOL)(psSFO->pvFenceObj != NULL);
+}
+
+
+/* Mapping of each required function to its appropriate sync-implementation function */
+#if defined(SUPPORT_FALLBACK_FENCE_SYNC)
+	#define SyncFenceWaitKM_                SyncFbFenceWait
+	#define SyncGetFenceObj_                SyncFbGetFenceObj
+	#define SyncFenceReleaseKM_             SyncFbFenceReleaseKM
+	#define SyncSWTimelineFenceCreateKM_    SyncFbSWTimelineFenceCreateKM
+	#define SyncSWTimelineAdvanceKM_        SyncFbSWTimelineAdvanceKM
+	#define SyncSWGetTimelineObj_           SyncFbSWGetTimelineObj
+	#define SyncSWTimelineReleaseKM_        SyncFbTimelineRelease
+	#define SyncDumpFence_                  SyncFbDumpFenceKM
+	#define SyncSWDumpTimeline_             SyncFbSWDumpTimelineKM
+#elif defined(SUPPORT_NATIVE_FENCE_SYNC)
+	#define SyncFenceWaitKM_                pvr_sync_fence_wait
+	#define SyncGetFenceObj_                pvr_sync_fence_get
+	#define SyncFenceReleaseKM_             pvr_sync_fence_release
+	#define SyncSWTimelineFenceCreateKM_    pvr_sync_sw_timeline_fence_create
+	#define SyncSWTimelineAdvanceKM_        pvr_sync_sw_timeline_advance
+	#define SyncSWGetTimelineObj_           pvr_sync_sw_timeline_get
+	#define SyncSWTimelineReleaseKM_        pvr_sync_sw_timeline_release
+	#define SyncDumpFence_                  sync_dump_fence
+	#define SyncSWDumpTimeline_             sync_sw_dump_timeline
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncFenceWaitKM
+
+@Description    Wait for all the sync points in the fence to be signalled.
+
+@Input          psFenceObj          Fence to wait on
+
+@Input          ui32TimeoutInMs     Maximum time to wait (in milliseconds)
+
+@Return         PVRSRV_OK               once the fence has been passed (all
+                                        containing check points have either
+                                        signalled or errored)
+                PVRSRV_ERROR_TIMEOUT    if the poll has exceeded the timeout
+                PVRSRV_ERROR_FAILED_DEPENDENCIES Other sync-impl specific error
+*/ /**************************************************************************/
+static inline PVRSRV_ERROR
+SyncFenceWaitKM(PVRSRV_DEVICE_NODE *psDevNode,
+                const SYNC_FENCE_OBJ *psFenceObj,
+                IMG_UINT32 ui32TimeoutInMs)
+{
+	PVRSRV_ERROR eError;
+
+	RGXSRV_HWPERF_SYNC_FENCE_WAIT(psDevNode->pvDevice,
+								  BEGIN,
+								  OSGetCurrentProcessID(),
+								  psFenceObj->hFence,
+								  ui32TimeoutInMs);
+
+	eError = SyncFenceWaitKM_(psFenceObj->pvFenceObj, ui32TimeoutInMs);
+
+	RGXSRV_HWPERF_SYNC_FENCE_WAIT(psDevNode->pvDevice,
+								  END,
+								  OSGetCurrentProcessID(),
+								  psFenceObj->hFence,
+								  ((eError == PVRSRV_OK) ?
+									  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_PASSED :
+									  ((eError == PVRSRV_ERROR_TIMEOUT) ?
+										  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_TIMEOUT :
+										  RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_ERROR)));
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       SyncGetFenceObj
+
+@Description    Get the implementation specific server fence object from
+                opaque implementation independent PVRSRV_FENCE type.
+                When successful, this function gets a reference on the base
+                fence, which needs to be dropped using SyncFenceReleaseKM,
+                when fence object is no longer in use.
+
+@Input          iFence        Input opaque fence object
+
+@Output         psFenceObj    Pointer to implementation specific fence object
+
+@Return         PVRSRV_ERROR  PVRSRV_OK, on success
+*/ /**************************************************************************/
+static inline PVRSRV_ERROR
+SyncGetFenceObj(PVRSRV_FENCE iFence,
+                SYNC_FENCE_OBJ *psFenceObj)
+{
+	psFenceObj->hFence = iFence;
+	return SyncGetFenceObj_(iFence, &psFenceObj->pvFenceObj);
+}
+
+/*************************************************************************/ /*!
+@Function       SyncFenceReleaseKM
+
+@Description    Release reference on this fence.
+
+@Input          psFenceObj     Fence to be released
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static inline
+PVRSRV_ERROR SyncFenceReleaseKM(const SYNC_FENCE_OBJ *psFenceObj)
+{
+	return SyncFenceReleaseKM_(psFenceObj->pvFenceObj);
+}
+
+/*****************************************************************************/
+/*                                                                           */
+/*                      SW TIMELINE SPECIFIC FUNCTIONS                       */
+/*                                                                           */
+/*****************************************************************************/
+
+static inline PVRSRV_ERROR
+SyncSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDevNode,
+                            PVRSRV_TIMELINE hSWTimeline,
+                            const IMG_CHAR *pszFenceName,
+                            PVRSRV_FENCE *phOutFence)
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+	eError = SyncSWTimelineFenceCreateKM_(psDevNode,
+	                                      hSWTimeline,
+	                                      pszFenceName,
+	                                      phOutFence,
+	                                      &ui64SyncPtIdx);
+	if (eError == PVRSRV_OK)
+	{
+		RGXSRV_HWPERF_ALLOC_SW_FENCE(psDevNode, OSGetCurrentProcessID(),
+		                             *phOutFence, hSWTimeline, ui64SyncPtIdx,
+		                             pszFenceName, OSStringLength(pszFenceName));
+	}
+	return eError;
+}
+
+static inline PVRSRV_ERROR
+SyncSWTimelineAdvanceKM(PVRSRV_DEVICE_NODE *psDevNode,
+                        const SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	IMG_UINT64 ui64SyncPtIdx;
+	PVRSRV_ERROR eError;
+	eError = SyncSWTimelineAdvanceKM_(psSWTimelineObj->pvTlObj,
+	                                  &ui64SyncPtIdx);
+
+	if (eError == PVRSRV_OK)
+	{
+		RGXSRV_HWPERF_SYNC_SW_TL_ADV(psDevNode->pvDevice,
+		                             OSGetCurrentProcessID(),
+		                             psSWTimelineObj->hTimeline,
+		                             ui64SyncPtIdx);
+	}
+	return eError;
+}
+
+static inline PVRSRV_ERROR
+SyncSWGetTimelineObj(PVRSRV_TIMELINE hSWTimeline,
+                     SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	psSWTimelineObj->hTimeline = hSWTimeline;
+	return SyncSWGetTimelineObj_(hSWTimeline, &psSWTimelineObj->pvTlObj);
+}
+
+static inline PVRSRV_ERROR
+SyncSWTimelineReleaseKM(const SYNC_TIMELINE_OBJ *psSWTimelineObj)
+{
+	return SyncSWTimelineReleaseKM_(psSWTimelineObj->pvTlObj);
+}
+
+static inline PVRSRV_ERROR
+SyncDumpFence(const SYNC_FENCE_OBJ *psFenceObj,
+              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+              void *pvDumpDebugFile)
+{
+	return SyncDumpFence_(psFenceObj->pvFenceObj, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+static inline PVRSRV_ERROR
+SyncSWDumpTimeline(const SYNC_TIMELINE_OBJ *psSWTimelineObj,
+                   DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                   void *pvDumpDebugFile)
+{
+	return SyncSWDumpTimeline_(psSWTimelineObj->pvTlObj, pfnDumpDebugPrintf, pvDumpDebugFile);
+}
+
+
+#endif /* PVRSRV_SYNC_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_tlcommon.h b/drivers/gpu/drm/img-rogue/pvrsrv_tlcommon.h
new file mode 100644
index 000000000000..9bc65fe05f98
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_tlcommon.h
@@ -0,0 +1,260 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Transport Layer common types and definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common types and definitions included into
+                both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef PVR_TLCOMMON_H
+#define PVR_TLCOMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+
+/*! Handle type for stream descriptor objects as created by this API */
+typedef IMG_HANDLE PVRSRVTL_SD;
+
+/*! Maximum stream name length including the null byte */
+#define PRVSRVTL_MAX_STREAM_NAME_SIZE	40U
+
+/*! Maximum number of streams expected to exist */
+#define PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER (32*PRVSRVTL_MAX_STREAM_NAME_SIZE)
+
+/*! Packet lengths are always rounded up to a multiple of 8 bytes */
+#define PVRSRVTL_PACKET_ALIGNMENT		8U
+#define PVRSRVTL_ALIGN(x)				PVR_ALIGN(x, PVRSRVTL_PACKET_ALIGNMENT)
+
+
+/*! A packet is made up of a header structure followed by the data bytes.
+ * There are 3 types of packet: normal (has data), data lost and padding,
+ * see packet flags. Header kept small to reduce data overhead.
+ *
+ * if the ORDER of the structure members is changed, please UPDATE the
+ * PVRSRVTL_PACKET_FLAG_OFFSET macro.
+ *
+ * Layout of uiTypeSize member is :
+ *
+ * |<---------------------------32-bits------------------------------>|
+ * |<----8---->|<-----1----->|<----7--->|<------------16------------->|
+ * |    Type   | Drop-Oldest |  UNUSED  |             Size            |
+ *
+ */
+typedef struct
+{
+	IMG_UINT32 uiTypeSize;	/*!< Type, Drop-Oldest flag & number of bytes following header */
+	IMG_UINT32 uiReserved;	/*!< Reserve, packets and data must be 8 byte aligned */
+
+	/* First bytes of TL packet data follow header ... */
+} PVRSRVTL_PACKETHDR, *PVRSRVTL_PPACKETHDR;
+
+/* Structure must always be a size multiple of 8 as stream buffer
+ * still an array of IMG_UINT32s.
+ */
+static_assert((sizeof(PVRSRVTL_PACKETHDR) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(PVRSRVTL_PACKETHDR) must be a multiple of 8");
+
+/*! Packet header reserved word fingerprint "TLP1" */
+#define PVRSRVTL_PACKETHDR_RESERVED 0x31504C54U
+
+/*! Packet header mask used to extract the size from the uiTypeSize member.
+ * Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_SIZE_MASK    0x0000FFFFU
+#define PVRSRVTL_MAX_PACKET_SIZE        (PVRSRVTL_PACKETHDR_SIZE_MASK & ~0xFU)
+
+
+/*! Packet header mask used to extract the type from the uiTypeSize member.
+ * Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_TYPE_MASK    0xFF000000U
+#define PVRSRVTL_PACKETHDR_TYPE_OFFSET  24U
+
+/*! Packet header mask used to check if packets before this one were dropped
+ * or not. Do not use directly, see GET macros.
+ */
+#define PVRSRVTL_PACKETHDR_OLDEST_DROPPED_MASK    0x00800000U
+#define PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET    23U
+
+/*! Packet type enumeration.
+ */
+typedef IMG_UINT32 PVRSRVTL_PACKETTYPE;
+
+/*! Undefined packet */
+#define PVRSRVTL_PACKETTYPE_UNDEF 0U
+
+/*! Normal packet type. Indicates data follows the header.
+ */
+#define PVRSRVTL_PACKETTYPE_DATA 1U
+
+/*! When seen this packet type indicates that at this moment in the stream
+ * packet(s) were not able to be accepted due to space constraints and
+ * that recent data may be lost - depends on how the producer handles the
+ * error. Such packets have no data, data length is 0.
+ */
+#define PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED 2U
+
+/*! Packets with this type set are padding packets that contain undefined
+ * data and must be ignored/skipped by the client. They are used when the
+ * circular stream buffer wraps around and there is not enough space for
+ * the data at the end of the buffer. Such packets have a length of 0 or
+ * more.
+ */
+#define PVRSRVTL_PACKETTYPE_PADDING 3U
+
+/*! This packet type conveys to the stream consumer that the stream
+ * producer has reached the end of data for that data sequence. The
+ * TLDaemon has several options for processing these packets that can
+ * be selected on a per stream basis.
+ */
+#define PVRSRVTL_PACKETTYPE_MARKER_EOS 4U
+
+/*! This is same as PVRSRVTL_PACKETTYPE_MARKER_EOS but additionally removes
+ * old data record output file before opening new/next one
+ */
+#define PVRSRVTL_PACKETTYPE_MARKER_EOS_REMOVEOLD 5U
+
+/*! Packet emitted on first stream opened by writer. Packet carries a name
+ * of the opened stream in a form of null-terminated string.
+ */
+#define PVRSRVTL_PACKETTYPE_STREAM_OPEN_FOR_WRITE 6U
+
+/*! Packet emitted on last stream closed by writer. Packet carries a name
+ * of the closed stream in a form of null-terminated string.
+ */
+#define PVRSRVTL_PACKETTYPE_STREAM_CLOSE_FOR_WRITE 7U
+
+#define PVRSRVTL_PACKETTYPE_LAST 8U
+
+/* The SET_PACKET_* macros rely on the order the PVRSRVTL_PACKETHDR members are declared:
+ * uiFlags is the upper half of a structure consisting of 2 uint16 quantities.
+ */
+#define PVRSRVTL_SET_PACKET_DATA(len)       (len) | (PVRSRVTL_PACKETTYPE_DATA                     << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_PADDING(len)    (len) | (PVRSRVTL_PACKETTYPE_PADDING                  << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_WRITE_FAILED    (0U)   | (PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+#define PVRSRVTL_SET_PACKET_HDR(len, type)  (len) | ((type)                                       << PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+
+/*! Returns the number of bytes of data in the packet.
+ * p may be any address type.
+ */
+#define GET_PACKET_DATA_LEN(p)	\
+	((IMG_UINT32) ((PVRSRVTL_PPACKETHDR) (void *) (p))->uiTypeSize & PVRSRVTL_PACKETHDR_SIZE_MASK)
+
+
+/*! Returns a IMG_BYTE* pointer to the first byte of data in the packet */
+#define GET_PACKET_DATA_PTR(p)	\
+	(((IMG_UINT8 *) (void *) (p)) + sizeof(PVRSRVTL_PACKETHDR))
+
+/*! Turns the packet address p into a PVRSRVTL_PPACKETHDR pointer type.
+ */
+#define GET_PACKET_HDR(p)		((PVRSRVTL_PPACKETHDR) ((void *) (p)))
+
+/*! Given a PVRSRVTL_PPACKETHDR address, return the address of the next pack
+ *  It is up to the caller to determine if the new address is within the
+ *  packet buffer.
+ */
+#define GET_NEXT_PACKET_ADDR(p) \
+	GET_PACKET_HDR( \
+		GET_PACKET_DATA_PTR(p) + \
+		( \
+			(GET_PACKET_DATA_LEN(p) + (PVRSRVTL_PACKET_ALIGNMENT-1U)) & \
+			(~(PVRSRVTL_PACKET_ALIGNMENT-1U)) \
+		) \
+	)
+
+/*! Get the type of the packet. p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define GET_PACKET_TYPE(p)		(((p)->uiTypeSize & PVRSRVTL_PACKETHDR_TYPE_MASK)>>PVRSRVTL_PACKETHDR_TYPE_OFFSET)
+
+/*! Set PACKETS_DROPPED flag in packet header as a part of uiTypeSize.
+ * p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define SET_PACKETS_DROPPED(p)		(((p)->uiTypeSize) | (1UL << PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET))
+
+/*! Check if packets were dropped before this packet.
+ * p is of type PVRSRVTL_PPACKETHDR.
+ */
+#define CHECK_PACKETS_DROPPED(p)	((((p)->uiTypeSize & PVRSRVTL_PACKETHDR_OLDEST_DROPPED_MASK)>>PVRSRVTL_PACKETHDR_OLDEST_DROPPED_OFFSET) != 0U)
+
+/*! Flags for use with PVRSRVTLOpenStream
+ * 0x01 - Do not block in PVRSRVTLAcquireData() when no bytes are available
+ * 0x02 - When the stream does not exist wait for a bit (2s) in
+ *        PVRSRVTLOpenStream() and then exit with a timeout error if it still
+ *        does not exist.
+ * 0x04 - Open stream for write only operations.
+ *        If flag is not used stream is opened as read-only. This flag is
+ *        required if one wants to call reserve/commit/write function on the
+ *        stream descriptor. Read from on the stream descriptor opened
+ *        with this flag will fail.
+ * 0x08 - Disable Producer Callback.
+ *        If this flag is set and the stream becomes empty, do not call any
+ *        associated producer callback to generate more data from the reader
+ *        context.
+ * 0x10 - Reset stream on open.
+ *        When this flag is used the stream will drop all of the stored data.
+ * 0x20 - Limit read position to the write position at time the stream
+ *        was opened. Hence this flag will freeze the content read to that
+ *        produced before the stream was opened for reading.
+ * 0x40 - Ignore Open Callback.
+ *        When this flag is set ignore any OnReaderOpenCallback setting for
+ *        the stream. This allows access to the stream to be made without
+ *        generating any extra packets into the stream.
+ */
+
+#define PVRSRV_STREAM_FLAG_NONE                        (0U)
+#define PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING         (1U<<0)
+#define PVRSRV_STREAM_FLAG_OPEN_WAIT                   (1U<<1)
+#define PVRSRV_STREAM_FLAG_OPEN_WO                     (1U<<2)
+#define PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK   (1U<<3)
+#define PVRSRV_STREAM_FLAG_RESET_ON_OPEN               (1U<<4)
+#define PVRSRV_STREAM_FLAG_READ_LIMIT                  (1U<<5)
+#define PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK        (1U<<6)
+
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* PVR_TLCOMMON_H */
+/******************************************************************************
+ End of file (pvrsrv_tlcommon.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pvrsrv_tlstreams.h b/drivers/gpu/drm/img-rogue/pvrsrv_tlstreams.h
new file mode 100644
index 000000000000..624b8eb1f47d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrv_tlstreams.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Transport Layer stream names
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common types and definitions included into
+                both user mode and kernel mode source.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRSRV_TLSTREAMS_H
+#define PVRSRV_TLSTREAMS_H
+
+#define PVRSRV_TL_CTRL_STREAM "tlctrl"
+
+#define PVRSRV_TL_HWPERF_RGX_FW_STREAM      "hwperf_fw_"
+#define PVRSRV_TL_HWPERF_HOST_SERVER_STREAM "hwperf_host_"
+
+#define PVRSRV_TL_FTRACE_RGX_FW_STREAM      "ftrace_fw_"
+
+/* Host HWPerf client stream names are of the form 'hwperf_client_<pid>' */
+#define PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM         "hwperf_client_"
+#define PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC "hwperf_client_%u_%u"
+
+#endif /* PVRSRV_TLSTREAMS_H */
+
+/******************************************************************************
+ End of file (pvrsrv_tlstreams.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/pvrsrvkm.mk b/drivers/gpu/drm/img-rogue/pvrsrvkm.mk
new file mode 100644
index 000000000000..a4aba4f11c31
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrsrvkm.mk
@@ -0,0 +1,158 @@
+pvrsrvkm-y += \
+ client_cache_direct_bridge.o \
+ server_cache_bridge.o \
+ server_cmm_bridge.o \
+ client_devicememhistory_direct_bridge.o \
+ server_devicememhistory_bridge.o \
+ server_di_bridge.o \
+ server_dmabuf_bridge.o \
+ client_mm_direct_bridge.o \
+ server_mm_bridge.o \
+ server_mmextmem_bridge.o \
+ client_pvrtl_direct_bridge.o \
+ server_pvrtl_bridge.o \
+ server_rgxbreakpoint_bridge.o \
+ server_rgxcmp_bridge.o \
+ server_rgxfwdbg_bridge.o \
+ server_rgxhwperf_bridge.o \
+ server_rgxregconfig_bridge.o \
+ server_rgxta3d_bridge.o \
+ server_rgxtimerquery_bridge.o \
+ server_rgxtq2_bridge.o \
+ server_rgxtq_bridge.o \
+ server_srvcore_bridge.o \
+ client_sync_direct_bridge.o \
+ server_sync_bridge.o \
+ client_synctracking_direct_bridge.o \
+ server_synctracking_bridge.o \
+ cache_km.o \
+ connection_server.o \
+ debug_common.o \
+ devicemem_heapcfg.o \
+ devicemem_history_server.o \
+ devicemem_server.o \
+ di_impl_brg.o \
+ di_server.o \
+ handle.o \
+ info_page_km.o \
+ lists.o \
+ mmu_common.o \
+ physheap.o \
+ physmem.o \
+ physmem_extmem.o \
+ physmem_hostmem.o \
+ physmem_lma.o \
+ physmem_osmem.o \
+ physmem_ramem.o \
+ pmr.o \
+ power.o \
+ process_stats.o \
+ pvr_notifier.o \
+ pvrsrv.o \
+ pvrsrv_bridge_init.o \
+ pvrsrv_pool.o \
+ srvcore.o \
+ sync_checkpoint.o \
+ sync_server.o \
+ tlintern.o \
+ tlserver.o \
+ tlstream.o \
+ vmm_pvz_client.o \
+ vmm_pvz_server.o \
+ vz_vmm_pvz.o \
+ vz_vmm_vm.o \
+ rgx_bridge_init.o \
+ rgxbreakpoint.o \
+ rgxbvnc.o \
+ rgxccb.o \
+ rgxcompute.o \
+ rgxdebug_common.o \
+ rgxfwcmnctx.o \
+ rgxfwdbg.o \
+ rgxfwimageutils.o \
+ rgxfwriscv.o \
+ rgxfwtrace_strings.o \
+ rgxhwperf_common.o \
+ rgxlayer_impl_common.o \
+ rgxmem.o \
+ rgxmmuinit.o \
+ rgxpower.o \
+ rgxregconfig.o \
+ rgxshader.o \
+ rgxsyncutils.o \
+ rgxtdmtransfer.o \
+ rgxtimecorr.o \
+ rgxtimerquery.o \
+ rgxutils.o \
+ rgxdebug.o \
+ rgxfwutils.o \
+ rgxhwperf.o \
+ rgxinit.o \
+ rgxlayer_impl.o \
+ rgxmipsmmuinit.o \
+ rgxmulticore.o \
+ rgxsrvinit.o \
+ rgxstartstop.o \
+ rgxta3d.o \
+ rgxtransfer.o \
+ allocmem.o \
+ dkf_server.o \
+ event.o \
+ fwload.o \
+ handle_idr.o \
+ km_apphint.o \
+ module_common.o \
+ osconnection_server.o \
+ osfunc.o \
+ osmmap_stub.o \
+ physmem_dmabuf.o \
+ physmem_extmem_linux.o \
+ physmem_osmem_linux.o \
+ physmem_test.o \
+ pmr_env.o \
+ pmr_os.o \
+ pvr_bridge_k.o \
+ pvr_buffer_sync.o \
+ pvr_counting_timeline.o \
+ pvr_debug.o \
+ pvr_debugfs.o \
+ pvr_drm.o \
+ pvr_export_fence.o \
+ pvr_fence.o \
+ pvr_gputrace.o \
+ pvr_platform_drv.o \
+ pvr_sw_fence.o \
+ pvr_sync_file.o \
+ pvr_sync_ioctl_common.o \
+ pvr_sync_ioctl_drm.o \
+ devicemem.o \
+ devicemem_utils.o \
+ hash.o \
+ mem_utils.o \
+ pvrsrv_error.o \
+ ra.o \
+ sync.o \
+ tlclient.o \
+ uniq_key_splay_tree.o \
+ rgx_hwperf_table.o \
+ interrupt_support.o \
+ sysconfig_cmn.o \
+ dma_support.o \
+ vmm_type_stub.o \
+ apollo/sysconfig.o
+pvrsrvkm-$(CONFIG_DRM_POWERVR_ROGUE_DEBUG) += \
+ client_htbuffer_direct_bridge.o \
+ server_htbuffer_bridge.o \
+ server_rgxkicksync_bridge.o \
+ client_ri_direct_bridge.o \
+ server_ri_bridge.o \
+ htb_debug.o \
+ htbserver.o \
+ ri_server.o \
+ rgxkicksync.o \
+ htbuffer.o
+pvrsrvkm-$(CONFIG_ARM)   += osfunc_arm.o
+pvrsrvkm-$(CONFIG_ARM64) += osfunc_arm64.o
+pvrsrvkm-$(CONFIG_EVENT_TRACING) += trace_events.o
+pvrsrvkm-$(CONFIG_RISCV) += osfunc_riscv.c
+pvrsrvkm-$(CONFIG_X86)   += osfunc_x86.o
diff --git a/drivers/gpu/drm/img-rogue/pvrversion.h b/drivers/gpu/drm/img-rogue/pvrversion.h
new file mode 100644
index 000000000000..1026b039627b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/pvrversion.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File           pvrversion.h
+@Title          PowerVR version numbers and strings.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Version numbers and strings for PowerVR components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef PVRVERSION_H
+#define PVRVERSION_H
+
+#define PVRVERSION_MAJ               24U
+#define PVRVERSION_MIN               2U
+
+#define PVRVERSION_FAMILY           "rogueddk"
+#define PVRVERSION_BRANCHNAME       "24.2"
+#define PVRVERSION_BUILD             6603887
+#define PVRVERSION_BSCONTROL        "Rogue_DDK_Linux_WS"
+
+#define PVRVERSION_STRING           "Rogue_DDK_Linux_WS rogueddk 24.2@6603887"
+#define PVRVERSION_STRING_SHORT     "24.2@6603887"
+
+#define COPYRIGHT_TXT               "Copyright (c) Imagination Technologies Ltd. All Rights Reserved."
+
+#define PVRVERSION_BUILD_HI          660
+#define PVRVERSION_BUILD_LO          3887
+#define PVRVERSION_STRING_NUMERIC   "24.2.660.3887"
+
+#define PVRVERSION_PACK(MAJOR,MINOR) (((IMG_UINT32)((IMG_UINT32)(MAJOR) & 0xFFFFU) << 16U) | (((MINOR) & 0xFFFFU) << 0U))
+#define PVRVERSION_UNPACK_MAJ(VERSION) (((VERSION) >> 16U) & 0xFFFFU)
+#define PVRVERSION_UNPACK_MIN(VERSION) (((VERSION) >> 0U) & 0xFFFFU)
+
+#endif /* PVRVERSION_H */
diff --git a/drivers/gpu/drm/img-rogue/ra.c b/drivers/gpu/drm/img-rogue/ra.c
new file mode 100644
index 000000000000..d1646b309f6d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/ra.c
@@ -0,0 +1,4040 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Allocator
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+@Description
+ Implements generic resource allocation. The resource allocator was originally
+ intended to manage address spaces. In practice the resource allocator is
+ generic and can manage arbitrary sets of integers.
+
+ Resources are allocated from arenas. Arenas can be created with an initial
+ span of resources. Further resources spans can be added to arenas. A
+ callback mechanism allows an arena to request further resource spans on
+ demand.
+
+ Each arena maintains an ordered list of resource segments each described by a
+ boundary tag. Each boundary tag describes a segment of resources which are
+ either 'free', available for allocation, or 'busy' currently allocated.
+ Adjacent 'free' segments are always coalesced to avoid fragmentation.
+
+ For allocation, all 'free' segments are kept on lists of 'free' segments in
+ a table index by pvr_log2(segment size) i.e., each table index n holds 'free'
+ segments in the size range 2^n -> 2^(n+1) - 1.
+
+ Allocation policy is based on an *almost* good fit strategy.
+
+ Allocated segments are inserted into a self-scaling hash table which maps
+ the base resource of the span to the relevant boundary tag. This allows the
+ code to get back to the boundary tag without exporting explicit boundary tag
+ references through the API.
+
+ Each arena has an associated quantum size, all allocations from the arena are
+ made in multiples of the basic quantum.
+
+ On resource exhaustion in an arena, a callback if provided will be used to
+ request further resources. Resource spans allocated by the callback mechanism
+ will be returned when freed (through one of the two callbacks).
+*/ /**************************************************************************/
+
+/* Issues:
+ * - flags, flags are passed into the resource allocator but are not currently used.
+ * - determination, of import size, is currently braindead.
+ * - debug code should be moved out to own module and #ifdef'd
+ */
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "dllist.h"
+#include "uniq_key_splay_tree.h"
+
+#include "hash.h"
+#include "ra.h"
+#include "pvrsrv_memallocflags.h"
+
+#include "osfunc.h"
+#include "allocmem.h"
+#include "lock.h"
+#include "pvr_intrinsics.h"
+
+/* The initial, and minimum size of the live address -> boundary tag structure
+ * hash table. The value 64 is a fairly arbitrary choice. The hash table
+ * resizes on demand so the value chosen is not critical.
+ */
+#define MINIMUM_HASH_SIZE (64)
+
+/* #define RA_VALIDATE */
+
+#if defined(__KLOCWORK__)
+	/* Make sure Klocwork analyses all the code (including the debug one) */
+	#if !defined(RA_VALIDATE)
+		#define RA_VALIDATE
+	#endif
+#endif
+
+#if !defined(PVRSRV_NEED_PVR_ASSERT) || !defined(RA_VALIDATE)
+/* Disable the asserts unless explicitly told otherwise.
+ * They slow the driver too much for other people
+ */
+
+#undef PVR_ASSERT
+/* Use a macro that really do not do anything when compiling in release
+ * mode!
+ */
+#define PVR_ASSERT(x)
+#endif
+
+/* boundary tags, used to describe a resource segment */
+struct _BT_
+{
+	enum bt_type
+	{
+		btt_free,		/* free resource segment */
+		btt_live		/* allocated resource segment */
+	} type;
+
+	unsigned int is_leftmost;
+	unsigned int is_rightmost;
+	unsigned int free_import;
+
+	/* The base resource and extent of this segment */
+	RA_BASE_T base;
+	RA_LENGTH_T uSize;
+
+	/* doubly linked ordered list of all segments within the arena */
+	struct _BT_ *pNextSegment;
+	struct _BT_ *pPrevSegment;
+
+	/* doubly linked un-ordered list of free segments with the same flags. */
+	struct _BT_ *next_free;
+	struct _BT_ *prev_free;
+
+	/* A user reference associated with this span, user references are
+	 * currently only provided in the callback mechanism
+	 */
+	IMG_HANDLE hPriv;
+
+	/* Flags to match on this span */
+	RA_FLAGS_T uFlags;
+
+};
+typedef struct _BT_ BT;
+
+
+/* resource allocation arena */
+struct _RA_ARENA_
+{
+	/* arena name for diagnostics output */
+	IMG_CHAR name[RA_MAX_NAME_LENGTH];
+
+	/* Spans / Imports within this arena are at least quantum sized
+	 * and are a multiple of the uQuantum. This also has the effect of
+	 * aligning these Spans to the uQuantum.
+	 */
+	RA_LENGTH_T uQuantum;
+
+	/* import interface, if provided */
+	union {
+		void* pImportAlloc;
+		PFN_RA_IMPORT_ALLOC_SINGLE pImportAllocSingle;
+		PFN_RA_IMPORT_ALLOC_MULTI pImportAllocMulti;
+	};
+	enum RA_IMPORT_ALLOC_METHOD {
+		RA_IMPORT_ALLOC_METHOD_SINGLE,
+		RA_IMPORT_ALLOC_METHOD_MULTI,
+	} eImportAllocMethod;
+
+	PFN_RA_IMPORT_FREE pImportFree;
+
+	/* Arbitrary handle provided by arena owner to be passed into the
+	 * import alloc and free hooks
+	 */
+	void *pImportHandle;
+
+	IMG_PSPLAY_TREE per_flags_buckets;
+
+	/* resource segment list */
+	BT *pHeadSegment;
+
+	/* segment address to boundary tag hash table */
+	HASH_TABLE *pSegmentHash;
+
+	/* Lock for this arena */
+	POS_LOCK hLock;
+
+	/* Policies that govern the resource area */
+	RA_POLICY_T ui32PolicyFlags;
+
+	/* LockClass of this arena. This is used within lockdep to decide if a
+	 * recursive call sequence with the same lock class is allowed or not.
+	 */
+	IMG_UINT32 ui32LockClass;
+
+	/* Total Size of the Arena */
+	IMG_UINT64	ui64TotalArenaSize;
+
+	/* Size available for allocation in the arena */
+	IMG_UINT64	ui64FreeArenaSize;
+
+};
+
+static_assert(sizeof(void*) == sizeof(PFN_RA_IMPORT_ALLOC_SINGLE), "pImportAlloc == pImportAllocSingle. Import callbacks must be pointers of the same size.");
+static_assert(sizeof(void*) == sizeof(PFN_RA_IMPORT_ALLOC_MULTI), "pImportAlloc == pImportAllocMulti. Import callbacks must be pointers of the same size.");
+
+struct _RA_ARENA_ITERATOR_
+{
+	RA_ARENA *pArena;
+	BT *pCurrent;
+	IMG_BOOL bIncludeFreeSegments;
+};
+
+/* Enum for selecting behaviour of _ConvertAndFree* functions*/
+typedef enum {
+	CONVERT_AND_FREE = 0,
+	CONVERT_DONT_FREE = 1
+} RA_CONVERT_AND_FREE_BEHAVIOUR;
+
+static PVRSRV_ERROR _RA_FreeMultiUnlocked(RA_ARENA *pArena,
+                                   RA_BASE_ARRAY_T aBaseArray,
+                                   RA_BASE_ARRAY_SIZE_T uiBaseArraySize);
+static PVRSRV_ERROR
+_RA_FreeMultiUnlockedSparse(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 *puiFreeIndices,
+                             IMG_UINT32 *puiFreeCount);
+
+/*************************************************************************/ /*!
+@Function       _RequestAllocFail
+@Description    Default callback allocator used if no callback is specified,
+                always fails to allocate further resources to the arena.
+@Input          _h - callback handle
+@Input          _uSize - requested allocation size
+@Input          _uflags - allocation flags
+@Input          _uBaseAlignment - Alignment for the returned allocated base
+
+
+@Output         _psImport - The base and size of the span to insert.
+@Return         PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL, this function always fails
+                to allocate.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_RequestAllocFail(RA_PERARENA_HANDLE _h,
+                  RA_LENGTH_T _uSize,
+                  RA_FLAGS_T _uFlags,
+                  RA_LENGTH_T _uBaseAlignment,
+                  const IMG_CHAR *_pszAnnotation,
+                  RA_IMPORT *_psImport)
+{
+	PVR_UNREFERENCED_PARAMETER(_h);
+	PVR_UNREFERENCED_PARAMETER(_uSize);
+	PVR_UNREFERENCED_PARAMETER(_uFlags);
+	PVR_UNREFERENCED_PARAMETER(_uBaseAlignment);
+	PVR_UNREFERENCED_PARAMETER(_pszAnnotation);
+	PVR_UNREFERENCED_PARAMETER(_psImport);
+
+	return PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL;
+}
+
+
+#if defined(PVR_CTZLL)
+	/* Make sure to trigger an error if someone change the buckets or the bHasEltsMapping size
+	   the bHasEltsMapping is used to quickly determine the smallest bucket containing elements.
+	   therefore it must have at least as many bits has the buckets array have buckets. The RA
+	   implementation actually uses one more bit. */
+	static_assert(ARRAY_SIZE(((IMG_PSPLAY_TREE)0)->buckets)
+				  < 8 * sizeof(((IMG_PSPLAY_TREE) 0)->bHasEltsMapping),
+				  "Too many buckets for bHasEltsMapping bitmap");
+#endif
+
+
+/*************************************************************************/ /*!
+@Function       pvr_log2
+@Description    Computes the floor of the log base 2 of a unsigned integer
+@Input          n       Unsigned integer
+@Return         Floor(Log2(n))
+*/ /**************************************************************************/
+#if defined(PVR_CLZLL)
+/* make sure to trigger a problem if someone changes the RA_LENGTH_T type
+   indeed the __builtin_clzll is for unsigned long long variables.
+
+   if someone changes RA_LENGTH to unsigned long, then use __builtin_clzl
+   if it changes to unsigned int, use __builtin_clz
+
+   if it changes for something bigger than unsigned long long,
+   then revert the pvr_log2 to the classic implementation */
+static_assert(sizeof(RA_LENGTH_T) == sizeof(unsigned long long),
+			  "RA log routines not tuned for sizeof(RA_LENGTH_T)");
+
+static inline IMG_UINT32 pvr_log2(RA_LENGTH_T n)
+{
+	PVR_ASSERT(n != 0); /* Log2 is not defined on 0 */
+
+	return (8 * sizeof(RA_LENGTH_T)) - 1 - PVR_CLZLL(n);
+}
+#else
+static IMG_UINT32
+pvr_log2(RA_LENGTH_T n)
+{
+	IMG_UINT32 l = 0;
+
+	PVR_ASSERT(n != 0); /* Log2 is not defined on 0 */
+
+	n >>= 1;
+	while (n > 0)
+	{
+		n >>= 1;
+		l++;
+	}
+	return l;
+}
+#endif
+
+static INLINE void _FreeTableLimitBoundsCheck(IMG_UINT32 *uiIndex)
+{
+	if (*uiIndex >= FREE_TABLE_LIMIT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Index exceeds FREE_TABLE_LIMIT (1TB), "
+		                        "Clamping Index to FREE_TABLE_LIMIT"));
+		*uiIndex = FREE_TABLE_LIMIT - 1;
+	}
+}
+
+
+#if defined(RA_VALIDATE)
+/*************************************************************************/ /*!
+@Function       _IsInSegmentList
+@Description    Tests if a BT is in the segment list.
+@Input          pArena           The arena.
+@Input          pBT              The boundary tag to look for.
+@Return         IMG_FALSE  BT was not in the arena's segment list.
+                IMG_TRUE   BT was in the arena's segment list.
+*/ /**************************************************************************/
+static IMG_BOOL
+_IsInSegmentList(RA_ARENA *pArena, BT *pBT)
+{
+	BT* pBTScan;
+
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(pBT != NULL);
+
+	/* Walk the segment list until we see the BT pointer... */
+	pBTScan = pArena->pHeadSegment;
+	while (pBTScan != NULL  &&  pBTScan != pBT)
+	{
+		pBTScan = pBTScan->pNextSegment;
+	}
+
+	/* Test if we found it and then return */
+	return (pBTScan == pBT);
+}
+
+/*************************************************************************/ /*!
+@Function       _IsInFreeList
+@Description    Tests if a BT is in the free list.
+@Input          pArena           The arena.
+@Input          pBT              The boundary tag to look for.
+@Return         IMG_FALSE  BT was not in the arena's free list.
+                IMG_TRUE   BT was in the arena's free list.
+*/ /**************************************************************************/
+static IMG_BOOL
+_IsInFreeList(RA_ARENA *pArena, BT *pBT)
+{
+	BT* pBTScan;
+	IMG_UINT32 uIndex;
+
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(pBT != NULL);
+
+	/* Look for the free list that holds BTs of this size... */
+	uIndex = pvr_log2(pBT->uSize);
+	PVR_ASSERT(uIndex < FREE_TABLE_LIMIT);
+
+	pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != pBT->uFlags))
+	{
+		return 0;
+	}
+	else
+	{
+		pBTScan = pArena->per_flags_buckets->buckets[uIndex];
+		while (pBTScan != NULL  &&  pBTScan != pBT)
+		{
+			pBTScan = pBTScan->next_free;
+		}
+
+		/* Test if we found it and then return */
+		return (pBTScan == pBT);
+	}
+}
+
+/* is_arena_valid should only be used in debug mode.
+ * It checks that some properties an arena must have are verified
+ */
+static int is_arena_valid(struct _RA_ARENA_ *arena)
+{
+	struct _BT_ *chunk;
+#if defined(PVR_CTZLL)
+	unsigned int i;
+#endif
+
+	for (chunk = arena->pHeadSegment; chunk != NULL; chunk = chunk->pNextSegment)
+	{
+		/* if next segment is NULL, then it must be a rightmost */
+		PVR_ASSERT((chunk->pNextSegment != NULL) || (chunk->is_rightmost));
+		/* if prev segment is NULL, then it must be a leftmost */
+		PVR_ASSERT((chunk->pPrevSegment != NULL) || (chunk->is_leftmost));
+
+		if (chunk->type == btt_free)
+		{
+			/* checks the correctness of the type field */
+			PVR_ASSERT(_IsInFreeList(arena, chunk));
+
+			/* check that there can't be two consecutive free chunks.
+			   Indeed, instead of having two consecutive free chunks,
+			   there should be only one that span the size of the two. */
+			PVR_ASSERT((chunk->is_leftmost) || (chunk->pPrevSegment->type != btt_free));
+			PVR_ASSERT((chunk->is_rightmost) || (chunk->pNextSegment->type != btt_free));
+		}
+		else
+		{
+			/* checks the correctness of the type field */
+			PVR_ASSERT(!_IsInFreeList(arena, chunk));
+		}
+
+		PVR_ASSERT((chunk->is_leftmost) || (chunk->pPrevSegment->base + chunk->pPrevSegment->uSize == chunk->base));
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->base + chunk->uSize == chunk->pNextSegment->base));
+
+		/* all segments of the same imports must have the same flags ... */
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->uFlags == chunk->pNextSegment->uFlags));
+		/* ... and the same import handle */
+		PVR_ASSERT((chunk->is_rightmost) || (chunk->hPriv == chunk->pNextSegment->hPriv));
+
+
+		/* if a free chunk spans a whole import, then it must be an 'not to free import'.
+		   Otherwise it should have been freed. */
+		PVR_ASSERT((!chunk->is_leftmost) || (!chunk->is_rightmost) || (chunk->type == btt_live) || (!chunk->free_import));
+	}
+
+#if defined(PVR_CTZLL)
+	if (arena->per_flags_buckets != NULL)
+	{
+		for (i = 0; i < FREE_TABLE_LIMIT; ++i)
+		{
+			/* verify that the bHasEltsMapping is correct for this flags bucket */
+			PVR_ASSERT(
+				((arena->per_flags_buckets->buckets[i] == NULL) &&
+				 (((arena->per_flags_buckets->bHasEltsMapping & ((IMG_ELTS_MAPPINGS) 1 << i)) == 0)))
+				||
+				((arena->per_flags_buckets->buckets[i] != NULL) &&
+				 (((arena->per_flags_buckets->bHasEltsMapping & ((IMG_ELTS_MAPPINGS) 1 << i)) != 0)))
+				);
+		}
+	}
+#endif
+
+	/* if arena was not valid, an earlier assert should have triggered */
+	return 1;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _SegmentListInsertAfter
+@Description    Insert a boundary tag into an arena segment list after a
+                specified boundary tag.
+@Input          pInsertionPoint  The insertion point.
+@Input          pBT              The boundary tag to insert.
+*/ /**************************************************************************/
+static INLINE void
+_SegmentListInsertAfter(BT *pInsertionPoint,
+                        BT *pBT)
+{
+	PVR_ASSERT(pBT != NULL);
+	PVR_ASSERT(pInsertionPoint != NULL);
+
+	pBT->pNextSegment = pInsertionPoint->pNextSegment;
+	pBT->pPrevSegment = pInsertionPoint;
+	if (pInsertionPoint->pNextSegment != NULL)
+	{
+		pInsertionPoint->pNextSegment->pPrevSegment = pBT;
+	}
+	pInsertionPoint->pNextSegment = pBT;
+}
+
+/*************************************************************************/ /*!
+@Function       _SegmentListInsert
+@Description    Insert a boundary tag into an arena segment list
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag to insert.
+*/ /**************************************************************************/
+static INLINE void
+_SegmentListInsert(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(!_IsInSegmentList(pArena, pBT));
+
+	/* insert into the segment chain */
+	pBT->pNextSegment = pArena->pHeadSegment;
+	pArena->pHeadSegment = pBT;
+	if (pBT->pNextSegment != NULL)
+	{
+		pBT->pNextSegment->pPrevSegment = pBT;
+	}
+
+	pBT->pPrevSegment = NULL;
+}
+
+/*************************************************************************/ /*!
+@Function       _SegmentListRemove
+@Description    Remove a boundary tag from an arena segment list.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag to remove.
+*/ /**************************************************************************/
+static void
+_SegmentListRemove(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(_IsInSegmentList(pArena, pBT));
+
+	if (pBT->pPrevSegment == NULL)
+		pArena->pHeadSegment = pBT->pNextSegment;
+	else
+		pBT->pPrevSegment->pNextSegment = pBT->pNextSegment;
+
+	if (pBT->pNextSegment != NULL)
+		pBT->pNextSegment->pPrevSegment = pBT->pPrevSegment;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _BuildBT
+@Description    Construct a boundary tag for a free segment.
+@Input          base     The base of the resource segment.
+@Input          uSize    The extent of the resource segment.
+@Input          uFlags   The flags to give to the boundary tag
+@Return         Boundary tag or NULL
+*/ /**************************************************************************/
+static BT *
+_BuildBT(RA_BASE_T base, RA_LENGTH_T uSize, RA_FLAGS_T uFlags)
+{
+	BT *pBT;
+
+	pBT = OSAllocZMem(sizeof(BT));
+	if (pBT == NULL)
+	{
+		return NULL;
+	}
+
+	pBT->is_leftmost = 1;
+	pBT->is_rightmost = 1;
+	/* pBT->free_import = 0; */
+	pBT->type = btt_live;
+	pBT->base = base;
+	pBT->uSize = uSize;
+	pBT->uFlags = uFlags;
+
+	return pBT;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _SegmentSplit
+@Description    Split a segment into two, maintain the arena segment list. The
+                boundary tag should not be in the free table. Neither the
+                original or the new neighbour boundary tag will be in the free
+                table.
+@Input          pBT       The boundary tag to split.
+@Input          uSize     The required segment size of boundary tag after
+                          splitting.
+@Return         New neighbour boundary tag or NULL.
+*/ /**************************************************************************/
+static BT *
+_SegmentSplit(BT *pBT, RA_LENGTH_T uSize)
+{
+	BT *pNeighbour;
+
+	pNeighbour = _BuildBT(pBT->base + uSize, pBT->uSize - uSize, pBT->uFlags);
+	if (pNeighbour == NULL)
+	{
+		return NULL;
+	}
+
+	_SegmentListInsertAfter(pBT, pNeighbour);
+
+	pNeighbour->is_leftmost = 0;
+	pNeighbour->is_rightmost = pBT->is_rightmost;
+	pNeighbour->free_import = pBT->free_import;
+	pBT->is_rightmost = 0;
+	pNeighbour->hPriv = pBT->hPriv;
+	pBT->uSize = uSize;
+	pNeighbour->uFlags = pBT->uFlags;
+
+	return pNeighbour;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeListInsert
+@Description    Insert a boundary tag into an arena free table.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag.
+*/ /**************************************************************************/
+static void
+_FreeListInsert(RA_ARENA *pArena, BT *pBT)
+{
+	IMG_UINT32 uIndex;
+	BT *pBTTemp = NULL;
+	uIndex = pvr_log2(pBT->uSize);
+
+	_FreeTableLimitBoundsCheck(&uIndex);
+
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+
+	pBT->type = btt_free;
+
+	pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+	/* the flags item in the splay tree must have been created before-hand by
+	   _InsertResource */
+	PVR_ASSERT(pArena->per_flags_buckets != NULL);
+
+	/* Handle NULL values for RELEASE builds and/or disabled ASSERT DEBUG builds */
+	if (unlikely(pArena->per_flags_buckets == NULL))
+	{
+		return;
+	}
+
+	/* Get the first node in the bucket */
+	pBTTemp = pArena->per_flags_buckets->buckets[uIndex];
+
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_ALLOC_NODE_SELECT_MASK) == RA_POLICY_ALLOC_OPTIMAL))
+	{
+		/* Add the node to the start if the bucket is empty */
+		if (NULL == pBTTemp)
+		{
+			pArena->per_flags_buckets->buckets[uIndex] = pBT;
+			pBT->next_free = NULL;
+			pBT->prev_free = NULL;
+
+		}
+		else
+		{
+			BT *pBTPrev = NULL;
+			/* Traverse the list and identify the appropriate
+			 * place based on the size of the Boundary being inserted */
+			while (pBTTemp && (pBTTemp->uSize < pBT->uSize))
+			{
+				pBTPrev = pBTTemp;
+				pBTTemp = pBTTemp->next_free;
+			}
+			/* point the new node to the first higher size element */
+			pBT->next_free = pBTTemp;
+			pBT->prev_free = pBTPrev;
+
+			if (pBTPrev)
+			{
+				/* Set the lower size element in the
+				 * chain to point new node */
+				pBTPrev->next_free = pBT;
+			}
+			else
+			{
+				/* Assign the new node to the start of the bucket
+				 * if the bucket is empty */
+				pArena->per_flags_buckets->buckets[uIndex] = pBT;
+			}
+			/* Make sure the higher size element in the chain points back
+			 * to the new node to be introduced */
+			if (pBTTemp)
+			{
+				pBTTemp->prev_free = pBT;
+			}
+		}
+	}
+	else
+	{
+		pBT->next_free =  pBTTemp;
+		if (pBT->next_free != NULL)
+		{
+			pBT->next_free->prev_free = pBT;
+		}
+		pBT->prev_free = NULL;
+		pArena->per_flags_buckets->buckets[uIndex] = pBT;
+	}
+
+#if defined(PVR_CTZLL)
+	/* tells that bucket[index] now contains elements */
+	pArena->per_flags_buckets->bHasEltsMapping |= ((IMG_ELTS_MAPPINGS) 1 << uIndex);
+#endif
+
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeListRemove
+@Description    Remove a boundary tag from an arena free table.
+@Input          pArena    The arena.
+@Input          pBT       The boundary tag.
+*/ /**************************************************************************/
+static void
+_FreeListRemove(RA_ARENA *pArena, BT *pBT)
+{
+	IMG_UINT32 uIndex;
+	uIndex = pvr_log2(pBT->uSize);
+
+	_FreeTableLimitBoundsCheck(&uIndex);
+
+	PVR_ASSERT(_IsInFreeList(pArena, pBT));
+
+	if (pBT->next_free != NULL)
+	{
+		pBT->next_free->prev_free = pBT->prev_free;
+	}
+
+	if (pBT->prev_free != NULL)
+	{
+		pBT->prev_free->next_free = pBT->next_free;
+	}
+	else
+	{
+		pArena->per_flags_buckets = PVRSRVSplay(pBT->uFlags, pArena->per_flags_buckets);
+		/* the flags item in the splay tree must have already been created
+		   (otherwise how could there be a segment with these flags */
+		PVR_ASSERT(pArena->per_flags_buckets != NULL);
+
+		/* Handle unlikely NULL values for RELEASE or ASSERT-disabled builds */
+		if (unlikely(pArena->per_flags_buckets == NULL))
+		{
+			pBT->type = btt_live;
+			return;
+		}
+
+		pArena->per_flags_buckets->buckets[uIndex] = pBT->next_free;
+#if defined(PVR_CTZLL)
+		if (pArena->per_flags_buckets->buckets[uIndex] == NULL)
+		{
+			/* there is no more elements in this bucket. Update the mapping. */
+			pArena->per_flags_buckets->bHasEltsMapping &= ~((IMG_ELTS_MAPPINGS) 1 << uIndex);
+		}
+#endif
+	}
+
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+	pBT->type = btt_live;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _InsertResource
+@Description    Add a free resource segment to an arena.
+@Input          pArena    The arena.
+@Input          base      The base of the resource segment.
+@Input          uSize     The extent of the resource segment.
+@Input          uFlags    The flags of the new resources.
+@Return         New bucket pointer
+                NULL on failure
+*/ /**************************************************************************/
+static BT *
+_InsertResource(RA_ARENA *pArena, RA_BASE_T base, RA_LENGTH_T uSize,
+                RA_FLAGS_T uFlags)
+{
+	BT *pBT;
+	PVR_ASSERT(pArena!=NULL);
+
+	pBT = _BuildBT(base, uSize, uFlags);
+
+	if (pBT != NULL)
+	{
+		IMG_PSPLAY_TREE tmp = PVRSRVInsert(pBT->uFlags, pArena->per_flags_buckets);
+		if (tmp == NULL)
+		{
+			OSFreeMem(pBT);
+			return NULL;
+		}
+
+		pArena->per_flags_buckets = tmp;
+		_SegmentListInsert(pArena, pBT);
+		_FreeListInsert(pArena, pBT);
+	}
+	return pBT;
+}
+
+/*************************************************************************/ /*!
+@Function       _InsertResourceSpan
+@Description    Add a free resource span to an arena, marked for free_import.
+@Input          pArena    The arena.
+@Input          base      The base of the resource segment.
+@Input          uSize     The extent of the resource segment.
+@Return         The boundary tag representing the free resource segment,
+                or NULL on failure.
+*/ /**************************************************************************/
+static INLINE BT *
+_InsertResourceSpan(RA_ARENA *pArena,
+                    RA_BASE_T base,
+                    RA_LENGTH_T uSize,
+                    RA_FLAGS_T uFlags)
+{
+	BT *pBT = _InsertResource(pArena, base, uSize, uFlags);
+	if (pBT != NULL)
+	{
+		pBT->free_import = 1;
+	}
+	return pBT;
+}
+
+
+/*************************************************************************/ /*!
+@Function       _RemoveResourceSpan
+@Description    Frees a resource span from an arena, returning the imported
+                span via the callback.
+@Input          pArena     The arena.
+@Input          pBT        The boundary tag to free.
+@Return         IMG_FALSE failure - span was still in use
+                IMG_TRUE  success - span was removed and returned
+*/ /**************************************************************************/
+static INLINE IMG_BOOL
+_RemoveResourceSpan(RA_ARENA *pArena, BT *pBT)
+{
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(pBT!=NULL);
+
+	if (pBT->free_import &&
+		pBT->is_leftmost &&
+		pBT->is_rightmost)
+	{
+		_SegmentListRemove(pArena, pBT);
+		pArena->ui64TotalArenaSize -= pBT->uSize;
+		pArena->ui64FreeArenaSize -= pBT->uSize;
+		pArena->pImportFree(pArena->pImportHandle, pBT->base, pBT->hPriv);
+		OSFreeMem(pBT);
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+/*************************************************************************/ /*!
+@Function       _FreeBT
+@Description    Free a boundary tag taking care of the segment list and the
+                boundary tag free table.
+@Input          pArena     The arena.
+@Input          pBT        The boundary tag to free.
+*/ /**************************************************************************/
+static void
+_FreeBT(RA_ARENA *pArena, BT *pBT)
+{
+	BT *pNeighbour;
+
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(pBT!=NULL);
+	PVR_ASSERT(!_IsInFreeList(pArena, pBT));
+
+	/* try and coalesce with left neighbour */
+	pNeighbour = pBT->pPrevSegment;
+	if ((!pBT->is_leftmost)	&& (pNeighbour->type == btt_free))
+	{
+		/* Verify list correctness */
+		PVR_ASSERT(pNeighbour->base + pNeighbour->uSize == pBT->base);
+
+		_FreeListRemove(pArena, pNeighbour);
+		_SegmentListRemove(pArena, pNeighbour);
+		pBT->base = pNeighbour->base;
+
+		pBT->uSize += pNeighbour->uSize;
+		pBT->is_leftmost = pNeighbour->is_leftmost;
+		OSFreeMem(pNeighbour);
+	}
+
+	/* try to coalesce with right neighbour */
+	pNeighbour = pBT->pNextSegment;
+	if ((!pBT->is_rightmost) && (pNeighbour->type == btt_free))
+	{
+		/* Verify list correctness */
+		PVR_ASSERT(pBT->base + pBT->uSize == pNeighbour->base);
+
+		_FreeListRemove(pArena, pNeighbour);
+		_SegmentListRemove(pArena, pNeighbour);
+		pBT->uSize += pNeighbour->uSize;
+		pBT->is_rightmost = pNeighbour->is_rightmost;
+		OSFreeMem(pNeighbour);
+	}
+
+	if (_RemoveResourceSpan(pArena, pBT) == IMG_FALSE)
+	{
+		_FreeListInsert(pArena, pBT);
+		PVR_ASSERT((!pBT->is_rightmost) || (!pBT->is_leftmost) || (!pBT->free_import));
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+}
+
+
+/*
+  This function returns the first element in a bucket that can be split
+  in a way that one of the sub-segments can meet the size and alignment
+  criteria.
+
+  The first_elt is the bucket to look into. Remember that a bucket is
+  implemented as a pointer to the first element of the linked list.
+
+  nb_max_try is used to limit the number of elements considered.
+  This is used to only consider the first nb_max_try elements in the
+  free-list. The special value ~0 is used to say unlimited i.e. consider
+  all elements in the free list
+ */
+static INLINE
+struct _BT_ *find_chunk_in_bucket(struct _BT_ * first_elt,
+                                  RA_LENGTH_T uSize,
+                                  RA_LENGTH_T uAlignment,
+                                  unsigned int nb_max_try)
+{
+	struct _BT_ *walker;
+
+	for (walker = first_elt; (walker != NULL) && (nb_max_try != 0); walker = walker->next_free)
+	{
+		const RA_BASE_T aligned_base = (uAlignment > 1) ?
+			PVR_ALIGN(walker->base, uAlignment)
+			: walker->base;
+
+		if (walker->base + walker->uSize >= aligned_base + uSize)
+		{
+			return walker;
+		}
+
+		/* 0xFFFF...FFFF is used has nb_max_try = infinity. */
+		if (nb_max_try != (unsigned int) ~0)
+		{
+			nb_max_try--;
+		}
+	}
+
+	return NULL;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FreeMultiBaseArray
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 free the region given as the array and size. This function
+ *                 should be used only when it is known that multiple Real
+ *                 bases will be freed from the array.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_FreeMultiBaseArray(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	IMG_UINT32 i;
+	for (i = 0; i < uiBaseArraySize; i++)
+	{
+		if (RA_BASE_IS_REAL(aBaseArray[i]))
+		{
+			BT *pBT;
+			pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &aBaseArray[i]);
+
+			if (pBT)
+			{
+				pArena->ui64FreeArenaSize += pBT->uSize;
+
+				PVR_ASSERT(pBT->base == aBaseArray[i]);
+				_FreeBT(pArena, pBT);
+				aBaseArray[i] = INVALID_BASE_ADDR;
+			}
+			else
+			{
+				/* Did we attempt to remove a ghost page?
+				 * Essentially the base was marked real but was actually a ghost.
+				 */
+				PVR_ASSERT(!"Attempt to free non-existing real base!");
+				return PVRSRV_ERROR_INVALID_REQUEST;
+			}
+		}
+#if defined(DEBUG)
+		else
+		{
+			aBaseArray[i] = INVALID_BASE_ADDR;
+		}
+#endif
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FreeSingleBaseArray
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 free the region given as the array and size. This function
+ *                 should be used only when it is known that a single Real
+ *                 base will be freed from the array. All Bases will be
+ *                 sanitised after the real has been freed.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from, entry 0 should be a
+ *                       Real base
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_FreeSingleBaseArray(RA_ARENA *pArena,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	BT *pBT;
+	PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[0]));
+
+	pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &aBaseArray[0]);
+
+	if (pBT)
+	{
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		PVR_ASSERT(pBT->base == aBaseArray[0]);
+		_FreeBT(pArena, pBT);
+	}
+	else
+	{
+		/* Did we attempt to remove a ghost page?
+		 * Essentially the base was marked real but was actually ghost.
+		 */
+		PVR_ASSERT(!"Attempt to free non-existing real base!");
+		return PVRSRV_ERROR_INVALID_REQUEST;
+	}
+
+	/* Set all entries to INVALID_BASE_ADDR */
+	OSCachedMemSet(aBaseArray, 0xFF, uiBaseArraySize * sizeof(RA_BASE_T));
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _GenerateGhostBases
+ *
+ *  @Description   Given an array (Could be complete or partial reference)
+ *                 generate Ghost bases for the allocation and size.
+ *
+ *  @Input  uiBase - The Real base to generate Ghost Bases from.
+ *  @Input  uiBaseSize - The size of the Real Base
+ *  @Input  uiChunkSize - The Base chunk size used to generate Ghost
+ *                           bases on specific boundaries.
+ *  @Input  aBaseArray - The array to add the Ghost bases to.
+ *
+ *  @Return array index of element past last Ghost base of given array.
+*/ /**************************************************************************/
+static IMG_UINT32
+_GenerateGhostBases(RA_BASE_T uiRealBase,
+                    RA_LENGTH_T uiBaseSize,
+                    RA_LENGTH_T uiChunkSize,
+                    RA_BASE_ARRAY_T aBaseArray)
+{
+	IMG_UINT32 ui32Index = 0;
+	RA_LENGTH_T uiRemaining = uiBaseSize - uiChunkSize;
+	RA_LENGTH_T uiCurrentBase = uiRealBase + uiChunkSize;
+	aBaseArray[ui32Index] = uiRealBase;
+
+	for (ui32Index = 1; uiRemaining != 0; ui32Index++)
+	{
+		aBaseArray[ui32Index] = RA_BASE_SET_GHOST_BIT(uiCurrentBase);
+		uiCurrentBase += uiChunkSize;
+		uiRemaining -= uiChunkSize;
+	}
+
+	return ui32Index;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _FindRealBaseFromGhost
+ *
+ *  @Description   Given an array and an index into that array for the Ghost Base
+ *                 find the Real Base hosting the Ghost base in the RA.
+ *  @Input aBaseArray - The array the Ghost and Real base reside on.
+ *  @Input ui32GhostBaseIndex - The index into the given array for the Ghost Base.
+ *  @Output pRealBase - The Real Base hosting the Ghost base.
+ *  @Output pui32RealBaseIndex -  The index of the Real Base found in the array.
+ *
+ *  @Return None.
+*/ /**************************************************************************/
+static void
+_FindRealBaseFromGhost(RA_BASE_ARRAY_T aBaseArray,
+                       IMG_UINT32 ui32GhostBaseIndex,
+                       RA_BASE_T *pRealBase,
+                       IMG_UINT32 *pui32RealBaseIndex)
+{
+	IMG_UINT32 ui32Index = ui32GhostBaseIndex;
+
+	PVR_ASSERT(RA_BASE_IS_GHOST(aBaseArray[ui32GhostBaseIndex]));
+
+	while (ui32Index != 0 &&
+	       RA_BASE_IS_GHOST(aBaseArray[ui32Index]))
+	{
+		ui32Index--;
+	}
+
+	*pRealBase = aBaseArray[ui32Index];
+	*pui32RealBaseIndex = ui32Index;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertGhostBaseToReal
+ *
+ *  @Description   Convert the given Ghost Base to a Real Base in the
+ *                 RA. This is mainly used in free paths so we can be
+ *                 agile with memory regions.
+ *  @Input pArena - The RA Arena to convert the base on.
+ *  @Input aBaseArray - The Base array to convert the base on.
+ *  @Input uiRealBase - The Base hosting the Ghost base to convert.
+ *  @Input ui32RealBaseArrayIndex - The index in the array of the Real Base.
+ *  @Input ui32GhostBaseArrayIndex - The index in the array of the Ghost Base.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost bases on.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertGhostBaseToReal(RA_ARENA *pArena,
+                        RA_BASE_ARRAY_T aBaseArray,
+                        RA_BASE_T uiRealBase,
+                        IMG_UINT32 ui32RealBaseArrayIndex,
+                        IMG_UINT32 ui32GhostBaseArrayIndex,
+                        RA_LENGTH_T uiChunkSize)
+{
+	BT *pOrigRealBT;
+	BT *pNewRealBT;
+
+	pOrigRealBT = (BT *) HASH_Retrieve_Extended(pArena->pSegmentHash, &uiRealBase);
+	pNewRealBT = _SegmentSplit(pOrigRealBT,
+	                           uiChunkSize *
+	                            (ui32GhostBaseArrayIndex - ui32RealBaseArrayIndex));
+	PVR_LOG_RETURN_IF_FALSE(pNewRealBT != NULL,
+	                        "Unable to split BT, no memory available to allocate new BT",
+	                        PVRSRV_ERROR_OUT_OF_MEMORY);
+
+	if (!HASH_Insert_Extended(pArena->pSegmentHash, &pNewRealBT->base, (uintptr_t) pNewRealBT))
+	{
+		PVR_LOG_RETURN_IF_ERROR(PVRSRV_ERROR_UNABLE_TO_INSERT_HASH_VALUE, "HASH_Insert_Extended");
+	}
+
+	aBaseArray[ui32GhostBaseArrayIndex] = pNewRealBT->base;
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeStartFromGhost
+ *
+ *  @Description   Given a ghost base and size, convert and free contiguous
+ *                 ghost bases. This has the effect of shrinking the size of
+ *                 the real base holding the range to free. If ghost pages
+ *                 remain after the free region, a new Real base will be
+ *                 created to host them.
+ *
+ *  @Input pArena - The RA Arena to free the Ghost Bases from.
+ *  @Input aBaseArray - The array to remove bases from
+ *  @Input uiBaseArraySize - The size of the Base array to free from.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost Bases.
+ *  @Input ui32GhostBaseIndex - The index into the array of the initial Ghost base to free
+ *  @Input ui32FreeCount - The number of Ghost bases to free from the Real base.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeStartFromGhost(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                              RA_LENGTH_T uiChunkSize,
+                              IMG_UINT32 ui32GhostBaseIndex,
+                              IMG_UINT32 ui32FreeCount,
+                              RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiRealBase;
+	IMG_UINT32 ui32RealBaseIndex;
+	IMG_UINT32 ui32FreeEndIndex;
+
+	_FindRealBaseFromGhost(aBaseArray,
+	                       ui32GhostBaseIndex,
+	                       &uiRealBase,
+	                       &ui32RealBaseIndex);
+
+	/* Make the first Ghost Base to free, real. */
+	eError = _ConvertGhostBaseToReal(pArena,
+	                                 aBaseArray,
+	                                 uiRealBase,
+	                                 ui32RealBaseIndex,
+	                                 ui32GhostBaseIndex,
+	                                 uiChunkSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+
+	/* Calculate the Base after the last to free. */
+	ui32FreeEndIndex = ui32GhostBaseIndex + ui32FreeCount;
+
+	/*
+	 * If the end of the free region is a Ghost base then we need to
+	 * make it a real base so that we can free the intended middle region.
+	 */
+	if (ui32FreeEndIndex != uiBaseArraySize &&
+	    RA_BASE_IS_GHOST(aBaseArray[ui32FreeEndIndex]))
+	{
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 aBaseArray[ui32GhostBaseIndex],
+		                                 ui32GhostBaseIndex,
+		                                 ui32FreeEndIndex,
+		                                 uiChunkSize);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	if (eBehaviour == CONVERT_AND_FREE)
+	{
+		/* Free the region calculated */
+		eError = _FreeSingleBaseArray(pArena,
+		                              &aBaseArray[ui32GhostBaseIndex],
+		                              ui32FreeCount);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeStartFromReal
+ *
+ *  @Description   Used in the case that we want to keep some indices that are ghost pages
+ *                 but the indices to free start with the real base. In this case we can
+ *                 convert the keep point to a real base, then free the original real base
+ *                 along with all ghost bases prior to the new real.
+ *
+ *  @Input pArena - The RA Arena to free the bases from.
+ *  @Input aBaseArray - The Base array to free from.
+ *  @Input uiChunkSize - The chunk size used to generate the Ghost bases.
+ *  @Input uiGhostBaseIndex - The index into the array of the Ghost base to convert.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeStartFromReal(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 uiRealBaseIndex,
+                             IMG_UINT32 uiGhostBaseIndex,
+                             RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiRealBase = aBaseArray[uiRealBaseIndex];
+
+	eError = _ConvertGhostBaseToReal(pArena,
+	                                 aBaseArray,
+	                                 uiRealBase,
+	                                 uiRealBaseIndex,
+	                                 uiGhostBaseIndex,
+	                                 uiChunkSize);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+
+	if (eBehaviour == CONVERT_AND_FREE)
+	{
+		eError = _FreeSingleBaseArray(pArena,
+		                              &aBaseArray[uiRealBaseIndex],
+		                              uiGhostBaseIndex - uiRealBaseIndex);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_FreeBaseArray");
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+ *  @Function   _ConvertAndFreeBaseArraySlice
+ *
+ *  @Description   Convert and maybe free Bases in an Array Slice.
+ *                 This function might convert some ghosts into real bases.
+ *                 This function assumes that the slice is within a single Real base alloc.
+ *                 i.e the uiFreeStartIndex and uiFreeCount remain fully within a single real
+ *                 base alloc and do not cross into another Real base region.
+ *
+ *  @Input pArena - The RA Arena to free bases from.
+ *  @Input aBaseArray - The Base array to free from.
+ *  @Input uiBaseArraySize - The size of the Base array to free from.
+ *  @Input uiChunkSize - The base chunk size used to generate the Ghost bases.
+ *  @Input uiFreeStartIndex - The index in the array to start freeing from
+ *  @Input uiFreeCount - The number of bases to free.
+ *  @Input eBehaviour - Specifies if the function should convert and free or only convert.
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code on Failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_ConvertAndFreeBaseArraySlice(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 uiFreeStartIndex,
+                             IMG_UINT32 uiFreeCount,
+                             RA_CONVERT_AND_FREE_BEHAVIOUR eBehaviour)
+{
+	/*3 cases:
+	 * Key: () = Region to Free
+	 *      [R] = Newly Real
+	 *      R = Real Base
+	 *      G = Ghost Base
+	 * 1. We free the whole Realbase (inc all Ghost bases)
+	 *    e.g. (RGGGGG)
+	 *    e.g. RGGG(R)RGG
+	 * 2 .We free the Real base but not all the Ghost bases meaning the first
+	 *    ghost base after the last freed will become a real base.
+	 *    e.g. RGGGG(RGGGG)[R]GGG
+	 *    e.g. (RGGGG)[R]GGGG
+	 * 3. We free some ghost bases from the real base
+	 *    e.g. RGGG(GGG)
+	 *    e.g. RGGG(GGG)[R]GGG
+	 *
+	 * Invalid Scenarios:
+	 * 1. RGG(GR)GGGRG
+	 * 2. RGG(GRG)GGRG
+	 * Higher levels should prevent these situations by ensuring that the free
+	 * index and count always focus on a single real base.
+	 * Scenario 1 & 2, correctly handled, would be a case 3. followed by a case 2.
+	 */
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(uiBaseArraySize >= uiFreeStartIndex &&
+	                        uiBaseArraySize >= uiFreeStartIndex + (uiFreeCount - 1),
+	                        "Free Index given out of array bounds",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Find which case we have */
+
+	/* Case 1 or 2 */
+	if (RA_BASE_IS_REAL(aBaseArray[uiFreeStartIndex]))
+	{
+		/* Case 1 */
+		if (uiFreeStartIndex + uiFreeCount == uiBaseArraySize ||
+		    RA_BASE_IS_REAL(aBaseArray[uiFreeStartIndex + uiFreeCount]) ||
+		    RA_BASE_IS_INVALID(aBaseArray[uiFreeStartIndex + uiFreeCount]))
+		{
+			if (eBehaviour == CONVERT_AND_FREE)
+			{
+				eError = _FreeSingleBaseArray(pArena,
+				                              &aBaseArray[uiFreeStartIndex],
+				                              uiFreeCount);
+				PVR_LOG_RETURN_IF_ERROR(eError, "_FreeBaseArray");
+			}
+		}
+		/* Case 2*/
+		else
+		{
+			eError = _ConvertAndFreeStartFromReal(pArena,
+			                                      aBaseArray,
+			                                      uiChunkSize,
+			                                      uiFreeStartIndex,
+			                                      uiFreeStartIndex + uiFreeCount,
+			                                      eBehaviour);
+			PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeStartFromReal");
+		}
+	}
+	/* Case 3 */
+	else if (RA_BASE_IS_GHOST(aBaseArray[uiFreeStartIndex]))
+	{
+		eError = _ConvertAndFreeStartFromGhost(pArena,
+		                                       aBaseArray,
+		                                       uiBaseArraySize,
+		                                       uiChunkSize,
+		                                       uiFreeStartIndex,
+		                                       uiFreeCount,
+		                                       eBehaviour);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeStartFromGhost");
+	}
+	/* Attempt to free an invalid base, this could be a duplicated
+	 * value in the free sparse index array */
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Attempt to free already free base Index %u", uiFreeStartIndex));
+		PVR_ASSERT(!"Attempted double free.");
+		return PVRSRV_ERROR_RA_FREE_INVALID_CHUNK;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       _AllocAlignSplit
+@Description    Given a valid BT, trim the start and end of the BT according
+                to alignment and size requirements. Also add the resulting
+                BT to the live hash table.
+@Input          pArena       The arena.
+@Input          pBT          The BT to trim and add to live hash table
+@Input          uSize        The requested allocation size.
+@Input          uAlignment   The alignment requirements of the allocation
+                             Required uAlignment, or 0.
+                             Must be a power of 2 if not 0
+@Output         pBase        Allocated, corrected, resource base
+                             (non-optional, must not be NULL)
+@Output         phPriv       The user references associated with
+                             the imported segment. (optional)
+@Return         IMG_FALSE failure
+                IMG_TRUE success
+*/ /**************************************************************************/
+static IMG_BOOL
+_AllocAlignSplit(RA_ARENA *pArena,
+                 BT *pBT,
+                 RA_LENGTH_T uSize,
+                 RA_LENGTH_T uAlignment,
+                 RA_BASE_T *pBase,
+                 RA_PERISPAN_HANDLE *phPriv)
+{
+	RA_BASE_T aligned_base;
+
+	aligned_base = (uAlignment > 1) ? PVR_ALIGN(pBT->base, uAlignment) : pBT->base;
+
+	_FreeListRemove(pArena, pBT);
+
+	if ((pArena->ui32PolicyFlags & RA_POLICY_NO_SPLIT_MASK) == RA_POLICY_NO_SPLIT)
+	{
+		goto nosplit;
+	}
+
+	/* with uAlignment we might need to discard the front of this segment */
+	if (aligned_base > pBT->base)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, (RA_LENGTH_T)(aligned_base - pBT->base));
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Front split failed", __func__));
+			/* Put pBT back in the list */
+			_FreeListInsert(pArena, pBT);
+			return IMG_FALSE;
+		}
+
+		_FreeListInsert(pArena, pBT);
+		pBT = pNeighbour;
+	}
+
+	/* the segment might be too big, if so, discard the back of the segment */
+	if (pBT->uSize > uSize)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, uSize);
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Back split failed", __func__));
+			/* Put pBT back in the list */
+			_FreeListInsert(pArena, pBT);
+			return IMG_FALSE;
+		}
+
+		_FreeListInsert(pArena, pNeighbour);
+	}
+nosplit:
+	pBT->type = btt_live;
+
+	if (!HASH_Insert_Extended(pArena->pSegmentHash, &aligned_base, (uintptr_t)pBT))
+	{
+		_FreeBT(pArena, pBT);
+		return IMG_FALSE;
+	}
+
+	if (phPriv != NULL)
+		*phPriv = pBT->hPriv;
+
+	*pBase = aligned_base;
+
+	return IMG_TRUE;
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptAllocAligned
+@Description    Attempt an allocation from an arena.
+@Input          pArena       The arena.
+@Input          uSize        The requested allocation size.
+@Input          uFlags       Allocation flags
+@Output         phPriv       The user references associated with
+                             the imported segment. (optional)
+@Input          uAlignment   Required uAlignment, or 0.
+                             Must be a power of 2 if not 0
+@Output         base         Allocated resource base (non-optional, must not
+                             be NULL)
+@Return         IMG_FALSE failure
+                IMG_TRUE success
+*/ /**************************************************************************/
+static IMG_BOOL
+_AttemptAllocAligned(RA_ARENA *pArena,
+                     RA_LENGTH_T uSize,
+                     RA_FLAGS_T uFlags,
+                     RA_LENGTH_T uAlignment,
+                     RA_BASE_T *base,
+                     RA_PERISPAN_HANDLE *phPriv) /* this is the "per-import" private data */
+{
+
+	IMG_UINT32 index_low;
+	IMG_UINT32 index_high;
+	IMG_UINT32 i;
+	struct _BT_ *pBT = NULL;
+
+	PVR_ASSERT(pArena!=NULL);
+	PVR_ASSERT(base != NULL);
+
+	pArena->per_flags_buckets = PVRSRVSplay(uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != uFlags))
+	{
+		/* no chunks with these flags. */
+		return IMG_FALSE;
+	}
+
+	index_low = pvr_log2(uSize);
+	if (uAlignment)
+	{
+		index_high = pvr_log2(uSize + uAlignment - 1);
+	}
+	else
+	{
+		index_high = index_low;
+	}
+
+	_FreeTableLimitBoundsCheck(&index_high);
+	_FreeTableLimitBoundsCheck(&index_low);
+
+	PVR_ASSERT(index_low <= index_high);
+
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_BUCKET_MASK) == RA_POLICY_BUCKET_BEST_FIT))
+	{
+		/* This policy ensures the selection of the first lowest size bucket that
+		 * satisfies the request size is selected */
+#if defined(PVR_CTZLL)
+		i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_low )) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+		i = index_low;
+#endif
+		for ( ; (i < FREE_TABLE_LIMIT) && (pBT == NULL); ++i)
+		{
+			if (pArena->per_flags_buckets->buckets[i])
+			{
+				pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, (unsigned int) ~0);
+			}
+		}
+	}
+	else
+	{
+#if defined(PVR_CTZLL)
+		i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_high + 1)) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+		for (i = index_high + 1; (i < FREE_TABLE_LIMIT) && (pArena->per_flags_buckets->buckets[i] == NULL); ++i)
+		{
+		}
+#endif
+		PVR_ASSERT(i <= FREE_TABLE_LIMIT);
+
+		if (i != FREE_TABLE_LIMIT)
+		{
+			/* since we start at index_high + 1, we are guaranteed to exit */
+			pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, 1);
+		}
+		else
+		{
+			for (i = index_high; (i != index_low - 1) && (pBT == NULL); --i)
+			{
+				pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, (unsigned int) ~0);
+			}
+		}
+	}
+
+	if (pBT == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	return _AllocAlignSplit(pArena, pBT, uSize, uAlignment, base, phPriv);
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptAllocAlignedAssured
+@Description    Attempt an allocation from an arena. If the arena allows
+                non-contiguous allocations, the allocation is guaranteed
+                given there is enough memory to satisfy the full allocation.
+@Input          pArena              The arena.
+@Input          uSize               The requested allocation size.
+@Input          uLog2MinContigSize  The Log2 minimum contiguity of the bases returned.
+@Input          uFlags              Allocation flags
+@Input          uAlignment          Required uAlignment, or 0.
+                                    Must be a power of 2 if not 0
+@Input          aBaseArray          Array to allocate bases to.
+@Input          bSparseAlloc        Is the allocation we are making sparse.
+@Output         bPhysContig         Is the allocation we made physically contiguous
+                                    or did we use the scoop logic
+@Return         Success: PVRSRV_OK
+                Fail:    PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptAllocAlignedAssured(RA_ARENA *pArena,
+                            RA_LENGTH_T uSize,
+                            IMG_UINT32 uLog2MinContigSize,
+                            RA_FLAGS_T uFlags,
+                            RA_LENGTH_T uAlignment,
+                            RA_BASE_ARRAY_T aBaseArray,
+                            IMG_BOOL bSparseAlloc,
+                            IMG_BOOL *bPhysContig)
+{
+	IMG_UINT32 index_low;  /* log2 Lowest contiguity required */
+	IMG_UINT32 index_high; /* log2 Size of full alloc */
+	IMG_UINT32 i;
+	struct _BT_ *pBT = NULL;
+	RA_PERISPAN_HANDLE phPriv;
+	RA_LENGTH_T uiRemaining = uSize;
+	RA_BASE_T uiBase;
+	IMG_UINT32 uiCurrentArrayIndex = 0;
+
+	PVR_ASSERT(pArena != NULL);
+
+	pArena->per_flags_buckets = PVRSRVSplay(uFlags, pArena->per_flags_buckets);
+	if ((pArena->per_flags_buckets == NULL) || (pArena->per_flags_buckets->uiFlags != uFlags))
+	{
+		/* no chunks with these flags. */
+		return PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS;
+	}
+
+	if (pArena->ui64FreeArenaSize < uSize)
+	{
+		/* Not enough memory to accommodate kick back for a chance to import more */
+		return PVRSRV_ERROR_RA_OUT_OF_RESOURCE;
+	}
+
+	if (uLog2MinContigSize && uAlignment)
+	{
+		index_low = uLog2MinContigSize;
+		index_high = pvr_log2(uSize);
+	}
+	else if (uLog2MinContigSize)
+	{
+		index_low = uLog2MinContigSize;
+		index_high = pvr_log2(uSize);
+	}
+	else if (uAlignment)
+	{
+		index_low = 0;
+		index_high = pvr_log2(uSize + uAlignment - 1);
+	}
+	else
+	{
+		index_low = 0;
+		index_high = pvr_log2(uSize);
+	}
+
+	PVR_ASSERT(index_low < FREE_TABLE_LIMIT);
+	PVR_ASSERT(index_high < FREE_TABLE_LIMIT);
+	PVR_ASSERT(index_low <= index_high);
+
+	/* Start at index_high + 1 as then we can check all buckets larger than the desired alloc
+	 * If we don't find one larger then we could still find one of requested size in index_high and
+	 * shortcut the non-contiguous allocation path. We check index_high + 1 first as it is
+	 * guaranteed to have a free region of the requested size if the bucket has entries. Whereas
+	 * index_high is not guaranteed to have an allocation that meets the size requested due to it
+	 * representing all free regions of size 2^bucket index to 2^bucket index +1. e.g we could have
+	 * a request for 19*4k Pages which would be represented by bucket 16, bucket 16 represents free
+	 * entries from 16*4k pages to 31*4k Pages in size, if this bucket only had free entries of
+	 * 17*4k pages the search would fail, hence not guaranteed at index_high.
+	 */
+#if defined(PVR_CTZLL)
+	i = PVR_CTZLL((~(((IMG_ELTS_MAPPINGS)1 << (index_high + 1)) - 1)) & pArena->per_flags_buckets->bHasEltsMapping);
+#else
+	for (i = index_high + 1; (i < FREE_TABLE_LIMIT) && (pArena->per_flags_buckets->buckets[i] == NULL); i++)
+	{
+	}
+#endif
+
+	PVR_ASSERT(i <= FREE_TABLE_LIMIT);
+
+	if (i != FREE_TABLE_LIMIT)
+	{
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], uSize, uAlignment, 1);
+	}
+	else
+	{
+		/* In this case we have searched all buckets index_high + 1 to FREE_TABLE_LIMIT and not found an
+		 * available bucket with the required allocation size.
+		 * Because we haven't found an allocation of the requested size in index_high + 1 there is still a chance
+		 * that we can find an allocation of correct size in index_high, when index_high references the bucket
+		 * containing the largest free chunks in the RA Arena. i.e All buckets > index_high == NULL.
+		 * We do a final search in that bucket here before we attempt to scoop memory or return NULL.
+		 */
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[index_high], uSize, uAlignment, 1);
+	}
+
+	/* We managed to find a contiguous allocation block of sufficient size */
+	if (pBT != NULL)
+	{
+		IMG_BOOL bResult;
+		bResult =  _AllocAlignSplit(pArena, pBT, uSize, uAlignment, &uiBase, &phPriv);
+		if (bResult)
+		{
+			if (!bSparseAlloc)
+			{
+				aBaseArray[0] = uiBase;
+			}
+			else
+			{
+				_GenerateGhostBases(uiBase, uSize, 1ULL << uLog2MinContigSize, aBaseArray);
+			}
+		}
+		else
+		{
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+		*bPhysContig = IMG_TRUE;
+
+		return PVRSRV_OK;
+	}
+
+	/*
+	 * If this arena doesn't have the non-contiguous allocation functionality enabled, then
+	 * don't attempt to scoop for non physically contiguous allocations. Sparse allocations
+	 * are still able to use the scoop functionality as they map in a chunk at a time in the
+	 * worst case.
+	 */
+	if (unlikely((pArena->ui32PolicyFlags & RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK) == 0) &&
+	    !bSparseAlloc)
+	{
+		return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+	}
+
+	/* Attempt to Scoop memory from non-contiguous blocks */
+	for (i = index_high; i >= index_low && uiRemaining != 0; i--)
+	{
+		/* While we have chunks of at least our contig size in the bucket to use */
+		for (
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], 1ULL << uLog2MinContigSize, uAlignment,(unsigned int) ~0);
+		pBT != NULL && uiRemaining != 0;
+		pBT = find_chunk_in_bucket(pArena->per_flags_buckets->buckets[i], 1ULL << uLog2MinContigSize, uAlignment,(unsigned int) ~0))//~0 Try all elements in bucket
+		{
+			/* Grab largest chunk possible that is a multiple of our min contiguity size
+			 * N.B: C always rounds towards 0 so this effectively floors for us */
+			IMG_BOOL bResult;
+			RA_BASE_T uiAlignedBase =
+			(uAlignment > 1) ? PVR_ALIGN(pBT->base, uAlignment) : pBT->base;
+			RA_LENGTH_T uiMaxSizeAvailable = (pBT->uSize - (uiAlignedBase - pBT->base));
+			RA_LENGTH_T uiMaxMultipleOfContig = (uiMaxSizeAvailable >> uLog2MinContigSize) << uLog2MinContigSize;
+
+			/*
+			 * If the size of the BT is larger than the remaining memory to allocate
+			 * then just allocate what we need. The rest will be trimmed and put back
+			 * into the pool in _AllocAlignSplit
+			 */
+			if (uiMaxMultipleOfContig > uiRemaining)
+			{
+				uiMaxMultipleOfContig = uiRemaining;
+			}
+
+			bResult = _AllocAlignSplit(pArena, pBT, uiMaxMultipleOfContig, uAlignment, &uiBase, &phPriv);
+			if (!bResult)
+			{
+				/* Something went wrong with splitting or adding to hash,
+				 * We can try find another chunk, although this should
+				 * never occur.
+				 */
+				PVR_ASSERT(!"_AllocAlignSplit issue.");
+				continue;
+			}
+
+			uiRemaining -= uiMaxMultipleOfContig;
+
+			uiCurrentArrayIndex += _GenerateGhostBases(uiBase,
+			                                           uiMaxMultipleOfContig,
+			                                           1ULL << uLog2MinContigSize,
+			                                           &aBaseArray[uiCurrentArrayIndex]);
+		}
+	}
+
+	/* If we didn't manage to scoop enough memory then we need to unwind the allocations we just made */
+	if (uiRemaining != 0)
+	{
+		goto error_unwind;
+	}
+	*bPhysContig = IMG_FALSE;
+
+	return PVRSRV_OK;
+
+error_unwind:
+	_RA_FreeMultiUnlocked(pArena,
+	                       aBaseArray,
+	                       uiCurrentArrayIndex);
+	return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+}
+
+/* Used as an inout parameter to reduce required memory allocations.
+ * The import callbacks return one or more RA_IMPORT which is then converted into a BT*.
+ * Once a RA_IMPORT is converted to a BT*, it is no longer needed. Therefore,
+ * the space taken up by the RA_IMPORT can be reused. */
+typedef union {
+	RA_IMPORT import;
+	BT *pBT;
+} RA_IMPORT_INTERNAL;
+
+
+static_assert(sizeof(RA_IMPORT_INTERNAL) == sizeof(RA_IMPORT),
+              "An RA_IMPORT[] array must be equivalent to an RA_IMPORT_INTERNAL[] array. "
+              "This allows RA_IMPORT[] returned by callbacks to be reused internally "
+              "as BT*[] without leaking scope to the user API.");
+
+static_assert(offsetof(RA_IMPORT_INTERNAL, pBT) == 0,
+              "The pBT member must be at offset 0 to match a BT*[].");
+
+/* Support importing 16 spans at once before allocating more memory.
+ * Almost all imports will only require a single span.
+ * However, if importing multiple spans, shouldn't always require memory allocations.
+ * Nominal use of multi imports shouldn't require many spans. */
+#define RA_DEFAULT_IMPORTS_BUFFER_SIZE (16 * (sizeof(RA_IMPORT) / sizeof(BT*)))
+
+static_assert((RA_DEFAULT_IMPORTS_BUFFER_SIZE * sizeof(BT*)) >= sizeof(RA_IMPORT),
+              "The default must be able to hold at least a single RA_IMPORT.");
+
+typedef struct {
+	BT **apsBTs;
+	IMG_UINT32 uiBTCount;
+} BT_PTR_ARRAY;
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAllocSingle
+@Description    Attempt to import more memory and create a new span.
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form a new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation.
+@Output         psImportInternal  The pBT member must be set to the new span.
+                                  Unless an error is returned.
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAllocSingle(RA_ARENA *pArena,
+                              RA_LENGTH_T uRequestSize,
+                              RA_FLAGS_T uImportFlags,
+                              RA_LENGTH_T uAlignment,
+                              const IMG_CHAR *pszAnnotation,
+                              RA_IMPORT_INTERNAL *psImportInternal)
+{
+	PVRSRV_ERROR eError;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+	RA_IMPORT *psImport = &psImportInternal->import;
+	BT *pBT;
+
+	eError = pArena->pImportAllocSingle(pArena->pImportHandle,
+	                                    uRequestSize, uImportFlags,
+	                                    uAlignment,
+	                                    pszAnnotation,
+	                                    psImport);
+	if (PVRSRV_OK != eError)
+	{
+		return eError;
+	}
+
+	/* If we successfully import more resource, create a span to
+	 * represent it else free the resource we imported.
+	 */
+	pBT = _InsertResourceSpan(pArena, psImport->base, psImport->uSize, uFlags);
+	if (pBT == NULL)
+	{
+		/* insufficient resources to insert the newly acquired span,
+		   so free it back again */
+		pArena->pImportFree(pArena->pImportHandle, psImport->base, psImport->hPriv);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+		        "size=0x%llx failed!", __func__, pArena->name,
+		        (unsigned long long)uRequestSize));
+		/* RA_Dump (arena); */
+
+		return PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	}
+
+	pBT->hPriv = psImport->hPriv;
+	/* Replace the RA_IMPORT with a BT*.
+	 * To free an import we only need the BT so we can reuse the elements slot in ppsImports. */
+	psImportInternal->pBT = pBT;
+
+	pArena->ui64TotalArenaSize += pBT->uSize;
+	pArena->ui64FreeArenaSize += pBT->uSize;
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAllocMulti
+@Description    Attempt to import more memory and create new span(s).
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form at least one new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation.
+@Inout          puiBTCount        The number of elements in *ppsImports. The
+                                  count will be modified to reflect how many
+                                  BT*s have been created.
+@Inout          ppsImports        Pointer to an array of RA_IMPORT_INTERNAL.
+                                  If the must be array resized, this will point
+                                  to the new array. It will be freed higher up
+                                  the callstack if necessary. If successful,
+                                  the pBT member must be set to a new span for
+                                  every element in the array of length puiBTCount.
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAllocMulti(RA_ARENA *pArena,
+                             RA_LENGTH_T uRequestSize,
+                             RA_FLAGS_T uImportFlags,
+                             RA_LENGTH_T uAlignment,
+                             const IMG_CHAR *pszAnnotation,
+                             IMG_UINT32 *puiImportCount,
+                             RA_IMPORT_INTERNAL **ppsImports)
+{
+	PVRSRV_ERROR eError;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+	IMG_UINT32 i;
+
+	eError = pArena->pImportAllocMulti(pArena->pImportHandle,
+	                                   uRequestSize, uImportFlags,
+	                                   uAlignment,
+	                                   pszAnnotation,
+	                                   puiImportCount,
+	                                   (RA_IMPORT**) ppsImports);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* An error should be returned by the callback if the count == 0. */
+	PVR_ASSERT(*puiImportCount >= 1);
+	/* An error should be returned by the callback if *ppsImports == NULL. */
+	PVR_ASSERT(*ppsImports != NULL);
+
+	/* If we successfully import more resource, create spans to represent it. */
+	for (i = 0; i < *puiImportCount; i++)
+	{
+		BT *pBT;
+		RA_IMPORT *psImport = &(*ppsImports)[i].import;
+
+		pBT = _InsertResourceSpan(pArena, psImport->base, psImport->uSize, uFlags);
+		PVR_GOTO_IF_FALSE(pBT != NULL, release_bts);
+
+		pBT->hPriv = psImport->hPriv;
+		pArena->ui64TotalArenaSize += pBT->uSize;
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		/* Replace the RA_IMPORT with a BT*.
+		 * To free an import we only need the BT so we can reuse the elements slot in ppsImports. */
+		(*ppsImports)[i].pBT = pBT;
+	}
+
+	return PVRSRV_OK;
+
+release_bts:
+	{
+		/* Free the just imported span that failed to allocate a BT. */
+		IMG_UINT32 j;
+		RA_IMPORT *psImport = &(*ppsImports)[i].import;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+		        "request=0x%llx size=0x%llx failed!", __func__, pArena->name,
+		        (unsigned long long)uRequestSize, (unsigned long long)psImport->uSize));
+
+		/* Insufficient resources to insert the newly acquired span so free it back again. */
+		pArena->pImportFree(pArena->pImportHandle, psImport->base, psImport->hPriv);
+
+		/* Remove the previously inserted BTs. */
+		for (j = 0; j < i; j++)
+		{
+			BT *pBT = (*ppsImports)[j].pBT;
+			pArena->ui64TotalArenaSize -= pBT->uSize;
+			pArena->ui64FreeArenaSize -= pBT->uSize;
+			_FreeBT(pArena, pBT);
+		}
+
+		return PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       _AttemptImportSpanAlloc
+@Description    Attempt to import more memory and create new span(s).
+                Function attempts to import more memory from the callback
+                provided at RA creation time, if successful the memory
+                will form a new span in the RA.
+@Input          pArena            The arena.
+@Input          uRequestSize      The requested allocation size.
+@Input          uImportMultiplier Import x-times more for future requests if
+                                  we have to import new memory.
+@Input          uImportFlags      Flags influencing allocation policy.
+@Input          uAlignment        The alignment requirements of the allocation
+                                  Required uAlignment, or 0.
+                                  Must be a power of 2 if not 0
+@Input          pszAnnotation     String to describe the allocation
+@Inout          psBTArray         A container to a BT*[] and it's length.
+                                  If the array is not large enough, a new array
+                                  will be returned and must be freed with OSFreeMem
+                                  regardless of the return error code.
+                                  On error, the array may be NULL!
+@Return         PVRSRV_OK - Success
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_AttemptImportSpanAlloc(RA_ARENA *pArena,
+                        RA_LENGTH_T uRequestSize,
+                        IMG_UINT8 uImportMultiplier,
+                        RA_FLAGS_T uImportFlags,
+                        RA_LENGTH_T uAlignment,
+                        const IMG_CHAR *pszAnnotation,
+                        BT_PTR_ARRAY *psBTArray)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_RA_INSERT_RESOURCE_SPAN_FAILED;
+	RA_IMPORT_INTERNAL *psImports;
+
+	/* Interpret the BT*[] as a RA_IMPORT_INTERNAL[].
+	 * Any extra space is ignored.
+	 * Example:
+	 * |---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|---BT*--|
+	 * |----RA_IMPORT_INTERNAL----|----RA_IMPORT_INTERNAL----|........|
+	 *
+	 * It must be able to hold at least 1 RA_IMPORT_INTERNAL.
+	 * This should never fail if RA_DEFAULT_IMPORTS_BUFFER_SIZE is used. */
+	PVR_ASSERT(psBTArray->uiBTCount * sizeof(BT*) >= sizeof(RA_IMPORT_INTERNAL));
+	PVR_ASSERT(psBTArray != NULL && psBTArray->apsBTs != NULL);
+	psImports = (RA_IMPORT_INTERNAL*) psBTArray->apsBTs;
+
+	/* apply over-allocation multiplier after all alignment adjustments */
+	uRequestSize *= uImportMultiplier;
+	/* ensure that we import according to the quanta of this arena */
+	uRequestSize = PVR_ALIGN(uRequestSize, pArena->uQuantum);
+
+	switch (pArena->eImportAllocMethod)
+	{
+	/* Exactly 1 BT will be created. */
+	case RA_IMPORT_ALLOC_METHOD_SINGLE:
+	{
+		eError = _AttemptImportSpanAllocSingle(pArena, uRequestSize, uImportFlags, uAlignment, pszAnnotation,  &psImports[0]);
+		PVR_GOTO_IF_ERROR(eError, err_);
+		psBTArray->uiBTCount = 1;
+
+		/* Reinterpret the RA_IMPORT_INTERNAL[] back into a BT*[].
+		 * We only care about psBTArray->apsBTs[0] being valid as we return a count of 1. */
+		psBTArray->apsBTs = (BT**) psImports;
+		break;
+	}
+	/* 1 or more BTs will be created. */
+	case RA_IMPORT_ALLOC_METHOD_MULTI:
+	{
+		/* The number of RA_IMPORT_INTERNAL that can be stored using the same space. */
+		IMG_UINT32 uiImportCount = (psBTArray->uiBTCount) / (sizeof(RA_IMPORT_INTERNAL) / sizeof(BT*));
+
+		eError = _AttemptImportSpanAllocMulti(pArena, uRequestSize, uImportFlags, uAlignment, pszAnnotation, &uiImportCount, &psImports);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_AttemptImportSpanAllocMulti");
+
+		/* Reinterpret the RA_IMPORT_INTERNAL[] back into a BT*[]. */
+		psBTArray->apsBTs = (BT**) psImports;
+
+		/* If a BT* is smaller than an RA_IMPORT_INTERNAL, the array of BT*s will not be tightly packed.
+		 * Therefore, it is not a valid BT*[] and the elements need to be compressed.
+		 * Example: Given the array below
+		 * |----RA_IMPORT_INTERNAL-0--|----RA_IMPORT_INTERNAL-1--|........|
+		 * |--BT*-0-|.................|--BT*-1-|..........................|
+		 * It must be packed into this:
+		 * |--BT*-0-|--BT*-1-|............................................| */
+		if (sizeof(BT*) < sizeof(RA_IMPORT_INTERNAL))
+		{
+			IMG_UINT32 i;
+			/* The first element will always be in the correct place. Skip redundant move. */
+			for (i = 1; i < uiImportCount; i++)
+			{
+				psBTArray->apsBTs[i] = psImports[i].pBT;
+			}
+		}
+		psBTArray->uiBTCount = uiImportCount;
+		break;
+	}
+	default:
+		PVR_ASSERT(!"unreachable");
+		break;
+	}
+
+	return eError;
+
+err_:
+	if ((void*) psImports != (void*) psBTArray->apsBTs && psImports != NULL)
+	{
+		OSFreeMem(psImports);
+	}
+	return eError;
+}
+
+static RA_ARENA*
+RA_CreateCommon(IMG_CHAR *name,
+                 RA_LOG2QUANTUM_T uLog2Quantum,
+                 IMG_UINT32 ui32LockClass,
+                 enum RA_IMPORT_ALLOC_METHOD eImportAllocMethod,
+                 void* imp_alloc,
+                 PFN_RA_IMPORT_FREE imp_free,
+                 RA_PERARENA_HANDLE arena_handle,
+                 RA_POLICY_T ui32PolicyFlags)
+{
+	RA_ARENA *pArena;
+	PVRSRV_ERROR eError;
+
+	if (name == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter 'name' (NULL not accepted)", __func__));
+		return NULL;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s'", __func__, name));
+
+	pArena = OSAllocMem(sizeof(*pArena));
+	if (pArena == NULL)
+	{
+		goto arena_fail;
+	}
+
+	eError = OSLockCreate(&pArena->hLock);
+	if (eError != PVRSRV_OK)
+	{
+		goto lock_fail;
+	}
+
+	pArena->pSegmentHash = HASH_Create_Extended(MINIMUM_HASH_SIZE, sizeof(RA_BASE_T), HASH_Func_Default, HASH_Key_Comp_Default);
+
+	if (pArena->pSegmentHash==NULL)
+	{
+		goto hash_fail;
+	}
+
+	if (imp_alloc == NULL) {
+		eImportAllocMethod = RA_IMPORT_ALLOC_METHOD_SINGLE;
+		imp_alloc = &_RequestAllocFail;
+	}
+
+	OSStringSafeCopy(pArena->name, name, RA_MAX_NAME_LENGTH);
+	pArena->eImportAllocMethod = eImportAllocMethod;
+	pArena->pImportAlloc = imp_alloc; /* import callback union */
+	pArena->pImportFree = imp_free;
+	pArena->pImportHandle = arena_handle;
+	pArena->pHeadSegment = NULL;
+	pArena->uQuantum = IMG_PAGE2BYTES64(uLog2Quantum);
+	pArena->per_flags_buckets = NULL;
+	pArena->ui32LockClass = ui32LockClass;
+	pArena->ui32PolicyFlags = ui32PolicyFlags;
+	pArena->ui64TotalArenaSize = 0;
+	pArena->ui64FreeArenaSize = 0;
+
+	PVR_ASSERT(is_arena_valid(pArena));
+	return pArena;
+
+hash_fail:
+	OSLockDestroy(pArena->hLock);
+lock_fail:
+	OSFreeMem(pArena);
+	/* not nulling pointer, out of scope */
+arena_fail:
+	return NULL;
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_Create(IMG_CHAR *name,
+          RA_LOG2QUANTUM_T uLog2Quantum,
+          IMG_UINT32 ui32LockClass,
+          PFN_RA_IMPORT_ALLOC_SINGLE imp_alloc,
+          PFN_RA_IMPORT_FREE imp_free,
+          RA_PERARENA_HANDLE arena_handle,
+          RA_POLICY_T ui32PolicyFlags)
+{
+	return RA_CreateCommon(name,
+	                       uLog2Quantum,
+	                       ui32LockClass,
+	                       RA_IMPORT_ALLOC_METHOD_SINGLE,
+	                       imp_alloc,
+	                       imp_free,
+	                       arena_handle,
+	                       ui32PolicyFlags
+	);
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_CreateMulti(IMG_CHAR *name,
+               RA_LOG2QUANTUM_T uLog2Quantum,
+               IMG_UINT32 ui32LockClass,
+               PFN_RA_IMPORT_ALLOC_MULTI imp_alloc,
+               PFN_RA_IMPORT_FREE imp_free,
+               RA_PERARENA_HANDLE arena_handle,
+               RA_POLICY_T ui32PolicyFlags)
+{
+	return RA_CreateCommon(name,
+	                       uLog2Quantum,
+	                       ui32LockClass,
+	                       RA_IMPORT_ALLOC_METHOD_MULTI,
+	                       imp_alloc,
+	                       imp_free,
+	                       arena_handle,
+	                       ui32PolicyFlags
+	);
+}
+
+static void _LogRegionCreation(const char *pszMemType,
+                               IMG_UINT64 ui64CpuPA,
+                               IMG_UINT64 ui64DevPA,
+                               IMG_UINT64 ui64Size)
+{
+#if !defined(DEBUG)
+	PVR_UNREFERENCED_PARAMETER(pszMemType);
+	PVR_UNREFERENCED_PARAMETER(ui64CpuPA);
+	PVR_UNREFERENCED_PARAMETER(ui64DevPA);
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+#else
+	if ((ui64CpuPA != 0) && (ui64DevPA != 0) && (ui64CpuPA != ui64DevPA))
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		        "Creating RA for \"%s\" memory"
+		        " - Cpu PA 0x%016" IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx
+		        " - Dev PA 0x%016" IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx,
+		        pszMemType,
+		        ui64CpuPA, ui64CpuPA + ui64Size,
+		        ui64DevPA, ui64DevPA + ui64Size));
+	}
+	else
+	{
+		__maybe_unused IMG_UINT64 ui64PA =
+			ui64CpuPA != 0 ? ui64CpuPA : ui64DevPA;
+		__maybe_unused const IMG_CHAR *pszAddrType =
+			ui64CpuPA == ui64DevPA ? "Cpu/Dev" : (ui64CpuPA != 0 ? "Cpu" : "Dev");
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+		        "Creating RA for \"%s\" memory - %s PA 0x%016"
+		        IMG_UINT64_FMTSPECx "-0x%016" IMG_UINT64_FMTSPECx,
+		        pszMemType, pszAddrType,
+		        ui64PA, ui64PA + ui64Size));
+	}
+#endif
+}
+
+IMG_INTERNAL RA_ARENA *
+RA_Create_With_Span(IMG_CHAR *name,
+                    RA_LOG2QUANTUM_T uLog2Quantum,
+                    IMG_UINT64 ui64CpuBase,
+                    IMG_UINT64 ui64SpanDevBase,
+                    IMG_UINT64 ui64SpanSize,
+                    RA_POLICY_T ui32PolicyFlags)
+{
+	RA_ARENA *psRA;
+	IMG_BOOL bSuccess;
+
+	psRA = RA_Create(name,
+	                 uLog2Quantum,       /* Use OS page size, keeps things simple */
+	                 RA_LOCKCLASS_0,     /* This arena doesn't use any other arenas. */
+	                 NULL,               /* No Import */
+	                 NULL,               /* No free import */
+	                 NULL,               /* No import handle */
+	                 ui32PolicyFlags);   /* No restriction on import splitting */
+	PVR_LOG_GOTO_IF_FALSE(psRA != NULL, "RA_Create() failed", return_);
+
+	bSuccess = RA_Add(psRA, (RA_BASE_T) ui64SpanDevBase, (RA_LENGTH_T) ui64SpanSize, 0, NULL);
+	PVR_LOG_GOTO_IF_FALSE(bSuccess, "RA_Add() failed", cleanup_);
+
+	_LogRegionCreation(name, ui64CpuBase, ui64SpanDevBase, ui64SpanSize);
+
+	return psRA;
+
+cleanup_:
+	RA_Delete(psRA);
+return_:
+	return NULL;
+}
+
+IMG_INTERNAL void
+RA_Delete(RA_ARENA *pArena)
+{
+	IMG_UINT32 uIndex;
+	IMG_BOOL bPrintWarn = IMG_TRUE;
+
+	PVR_ASSERT(pArena != NULL);
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return;
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			  "%s: name='%s'", __func__, pArena->name));
+
+	while (pArena->pHeadSegment != NULL)
+	{
+		BT *pBT = pArena->pHeadSegment;
+
+		if (pBT->type != btt_free)
+		{
+#if !defined(DEBUG)
+			if (bPrintWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Allocations still exist in the arena that is being destroyed. Arena Name: %s", __func__, pArena->name));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				PVR_DPF((PVR_DBG_ERROR, "%s: base = 0x%llx size=0x%llx", __func__,
+					  (unsigned long long)pBT->base, (unsigned long long)pBT->uSize));
+				PVR_DPF((PVR_DBG_ERROR, "%s: This warning will be issued only once for the first allocation found!", __func__));
+				bPrintWarn = IMG_FALSE;
+			}
+#else
+			if (bPrintWarn)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Allocations still exist in the arena that is being destroyed. Arena Name: %s", __func__, pArena->name));
+				PVR_DPF((PVR_DBG_ERROR, "%s: Likely Cause: client drivers not freeing allocations before destroying devmem context", __func__));
+				bPrintWarn = IMG_FALSE;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: base = 0x%llx size=0x%llx flags=0x%llx", __func__,
+				  (unsigned long long)pBT->base, (unsigned long long)pBT->uSize,
+				  (unsigned long long)pBT->uFlags));
+#endif
+		}
+		else
+		{
+			_FreeListRemove(pArena, pBT);
+		}
+
+		_SegmentListRemove(pArena, pBT);
+		OSFreeMem(pBT);
+		/* not nulling original pointer, it has changed */
+	}
+
+	while (pArena->per_flags_buckets != NULL)
+	{
+		for (uIndex=0; uIndex<FREE_TABLE_LIMIT; uIndex++)
+		{
+			PVR_ASSERT(pArena->per_flags_buckets->buckets[uIndex] == NULL);
+		}
+
+		pArena->per_flags_buckets = PVRSRVDelete(pArena->per_flags_buckets->uiFlags, pArena->per_flags_buckets);
+	}
+
+	HASH_Delete(pArena->pSegmentHash);
+	OSLockDestroy(pArena->hLock);
+	OSFreeMem(pArena);
+	/* not nulling pointer, copy on stack */
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_Add(RA_ARENA *pArena,
+       RA_BASE_T base,
+       RA_LENGTH_T uSize,
+       RA_FLAGS_T uFlags,
+       RA_PERISPAN_HANDLE hPriv)
+{
+	struct _BT_* bt;
+	PVR_ASSERT(pArena != NULL);
+	PVR_ASSERT(uSize != 0);
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return IMG_FALSE;
+	}
+
+	if (uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid size 0 added to arena %s", __func__, pArena->name));
+		return IMG_FALSE;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', "
+			 "base=0x%llx, size=0x%llx", __func__, pArena->name,
+			 (unsigned long long)base, (unsigned long long)uSize));
+
+	uSize = PVR_ALIGN(uSize, pArena->uQuantum);
+	bt = _InsertResource(pArena, base, uSize, uFlags);
+	if (bt != NULL)
+	{
+		bt->hPriv = hPriv;
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64TotalArenaSize += uSize;
+	pArena->ui64FreeArenaSize += uSize;
+	OSLockRelease(pArena->hLock);
+
+	return bt != NULL;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_Alloc(RA_ARENA *pArena,
+         RA_LENGTH_T uRequestSize,
+         IMG_UINT8 uImportMultiplier,
+         RA_FLAGS_T uImportFlags,
+         RA_LENGTH_T uAlignment,
+         const IMG_CHAR *pszAnnotation,
+         RA_BASE_T *base,
+         RA_LENGTH_T *pActualSize,
+         RA_PERISPAN_HANDLE *phPriv)
+{
+	PVRSRV_ERROR eError;
+	IMG_BOOL bResult;
+	RA_LENGTH_T uSize = uRequestSize;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+
+	if (pArena == NULL || uImportMultiplier == 0 || uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: One of the necessary parameters is 0", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	if (pActualSize != NULL)
+	{
+		*pActualSize = uSize;
+	}
+
+	/* Must be a power of 2 or 0 */
+	PVR_ASSERT((uAlignment == 0) || (uAlignment & (uAlignment - 1)) == 0);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: arena='%s', size=0x%llx(0x%llx), "
+	        "alignment=0x%llx", __func__, pArena->name,
+	        (unsigned long long)uSize, (unsigned long long)uRequestSize,
+	        (unsigned long long)uAlignment));
+
+	/* if allocation failed then we might have an import source which
+	   can provide more resource, else we will have to fail the
+	   allocation to the caller. */
+	bResult = _AttemptAllocAligned(pArena, uSize, uFlags, uAlignment, base, phPriv);
+	if (!bResult)
+	{
+		BT *apsBTsStatic[RA_DEFAULT_IMPORTS_BUFFER_SIZE];
+		BT_PTR_ARRAY sBTs = {
+			.uiBTCount = ARRAY_SIZE(apsBTsStatic),
+			.apsBTs = apsBTsStatic,
+		};
+
+		eError = _AttemptImportSpanAlloc(pArena,
+		                                 uSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 uAlignment,
+		                                 pszAnnotation,
+		                                 &sBTs);
+		if (eError != PVRSRV_OK)
+		{
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			OSLockRelease(pArena->hLock);
+			return eError;
+		}
+
+		bResult = _AttemptAllocAligned(pArena, uSize, uFlags, uAlignment, base, phPriv);
+		if (!bResult)
+		{
+			IMG_UINT32 i;
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: name='%s' second alloc failed!",
+			        __func__, pArena->name));
+
+			/*
+			  On failure of _AttemptAllocAligned() depending on the exact point
+			  of failure, the imported segment may have been used and freed, or
+			  left untouched. If the later, we need to return it.
+			*/
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				_FreeBT(pArena, sBTs.apsBTs[i]);
+			}
+
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			OSLockRelease(pArena->hLock);
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+		else
+		{
+			IMG_UINT32 i;
+			IMG_BOOL bBaseInNewSpans = IMG_FALSE;
+
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				BT *pBT = sBTs.apsBTs[i];
+				/* Check if the new allocation was in a span we just added... */
+				if (*base >= pBT->base && *base < (pBT->base + pBT->uSize))
+				{
+					bBaseInNewSpans = IMG_TRUE;
+					break;
+				}
+			}
+			if (!bBaseInNewSpans)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: name='%s' alloc did not occur in the imported span!",
+				         __func__, pArena->name));
+				/* Remove the imported spans which should not be in use (if it is then that is okay).
+				 * No spans should exist and not be used unless added through the import multiplier. */
+				for (i = 0; i < sBTs.uiBTCount; i++)
+				{
+					_FreeBT(pArena, sBTs.apsBTs[i]);
+				}
+			}
+		}
+		if (sBTs.apsBTs != apsBTsStatic)
+		{
+			OSFreeMem(sBTs.apsBTs);
+		}
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', size=0x%llx, "
+	        "*base=0x%llx = %d", __func__, pArena->name, (unsigned long long)uSize,
+	        (unsigned long long)*base, bResult));
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+	OSLockRelease(pArena->hLock);
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_RA_AllocMultiUnlocked(RA_ARENA *pArena,
+                       RA_LENGTH_T uRequestSize,
+                       IMG_UINT32 uiLog2ChunkSize,
+                       IMG_UINT8 uImportMultiplier,
+                       RA_FLAGS_T uImportFlags,
+                       const IMG_CHAR *pszAnnotation,
+                       RA_BASE_ARRAY_T aBaseArray,
+                       RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                       IMG_BOOL bSparseAlloc,
+                       IMG_BOOL *bPhysContig)
+{
+	PVRSRV_ERROR eError;
+	RA_LENGTH_T uSize = uRequestSize;
+	RA_FLAGS_T uFlags = (uImportFlags & PVRSRV_MEMALLOCFLAGS_RA_DIFFERENTIATION_MASK);
+
+	PVR_LOG_RETURN_IF_FALSE(pArena != NULL && uImportMultiplier != 0 && uSize != 0,
+	                        "One of the necessary parameters is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT((uRequestSize & (IMG_PAGE2BYTES64(uiLog2ChunkSize) - 1)) == 0);
+	PVR_LOG_RETURN_IF_FALSE((uRequestSize & (IMG_PAGE2BYTES64(uiLog2ChunkSize) - 1)) == 0,
+	                        "Require uiLog2ChunkSize pow 2 & multiple of uRequestSize",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Enforce these constraints so we can use those bits to handle Ghost bases. */
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure Base Array is large enough for intended allocation */
+	PVR_LOG_RETURN_IF_FALSE(((RA_LENGTH_T)uiBaseArraySize * (IMG_PAGE2BYTES64(uiLog2ChunkSize)) >= uRequestSize),
+		                    "Not enough array space to store alloc bases",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "%s: arena='%s', size=0x%llx(0x%llx), "
+	        "log2ChunkSize=0x%llx", __func__, pArena->name,
+	        (unsigned long long)uSize, (unsigned long long)uRequestSize,
+	        (unsigned long long)uiLog2ChunkSize));
+
+	/* if allocation failed then we might have an import source which
+	   can provide more resource, else we will have to fail the
+	   allocation to the caller. */
+	eError = _AttemptAllocAlignedAssured(pArena,
+	                                      uSize,
+	                                      uiLog2ChunkSize,
+	                                      uFlags,
+	                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                      aBaseArray,
+	                                      bSparseAlloc,
+	                                      bPhysContig);
+	if (eError)
+	{
+		BT *apsBTsStatic[RA_DEFAULT_IMPORTS_BUFFER_SIZE];
+		BT_PTR_ARRAY sBTs = {
+			.uiBTCount = ARRAY_SIZE(apsBTsStatic),
+			.apsBTs = apsBTsStatic,
+		};
+
+		if (eError == PVRSRV_ERROR_RA_OUT_OF_RESOURCE)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA out of resource, attempt to import more if possible:"
+			                         " uSize:0x%llx"
+			                         " uFlags:0x%llx",
+			                         (unsigned long long) uSize,
+			                         (unsigned long long) uFlags));
+		}
+		else if (eError == PVRSRV_ERROR_RA_NO_RESOURCE_WITH_FLAGS)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA no resource for flags, attempt to import some if possible:"
+			                         " uSize:0x%llx"
+			                         " uFlags:0x%llx",
+			                         (unsigned long long) uSize,
+			                         (unsigned long long) uFlags));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,"RA Failed to Allocate, could be fragmented, attempt to import"
+			                         " more resource if possible."));
+		}
+
+		eError = _AttemptImportSpanAlloc(pArena,
+		                                 uSize,
+		                                 uImportMultiplier,
+		                                 uFlags,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                 pszAnnotation,
+		                                 &sBTs);
+		if (eError != PVRSRV_OK)
+		{
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			return eError;
+		}
+
+		eError = _AttemptAllocAlignedAssured(pArena,
+		                                      uSize,
+		                                      uiLog2ChunkSize,
+		                                      uFlags,
+		                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                      aBaseArray,
+		                                      bSparseAlloc,
+		                                      bPhysContig);
+		if (eError)
+		{
+			IMG_UINT32 i;
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: name='%s' second alloc failed!",
+					 __func__, pArena->name));
+			/*
+			  On failure of _AttemptAllocAligned() depending on the exact point
+			  of failure, the imported segment may have been used and freed, or
+			  left untouched. If the later, we need to return it.
+			*/
+			for (i = 0; i < sBTs.uiBTCount; i++)
+			{
+				_FreeBT(pArena, sBTs.apsBTs[i]);
+			}
+
+			if (sBTs.apsBTs != apsBTsStatic)
+			{
+				OSFreeMem(sBTs.apsBTs);
+			}
+			return PVRSRV_ERROR_RA_ATTEMPT_ALLOC_ALIGNED_FAILED;
+		}
+#if defined(DEBUG)
+		/*
+		 * This block of code checks to see if the extra memory we just imported was
+		 * used for the second allocation. If we imported memory but did not use it,
+		 * it indicates there is a bug in the allocation logic. We can still recover by
+		 * freeing the imported span(s) but we emit an error to signal that there is an
+		 * issue.
+		 * */
+		else
+		{
+			IMG_UINT32 bt_index;
+			IMG_UINT32 base_index;
+			IMG_BOOL bBasesInNewSpan = IMG_FALSE;
+
+			for (bt_index = 0; bt_index < sBTs.uiBTCount && !bBasesInNewSpan; bt_index++)
+			{
+				BT *pBT = sBTs.apsBTs[bt_index];
+				for (base_index = 0; base_index < uiBaseArraySize; base_index++)
+				{
+					RA_BASE_T uiBase = RA_BASE_STRIP_GHOST_BIT(aBaseArray[base_index]);
+
+					/* If the base hasn't been allocated then skip it */
+					if (aBaseArray[base_index] == INVALID_BASE_ADDR)
+					{
+						continue;
+					}
+
+					if (uiBase >= pBT->base &&
+					    uiBase <= pBT->base + pBT->uSize)
+					{
+						bBasesInNewSpan = IMG_TRUE;
+						break;
+					}
+				}
+			}
+			if (!bBasesInNewSpan)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: name='%s' alloc did not occur in the imported span!",
+						__func__, pArena->name));
+				/* Remove the imported spans which should not be in use (if it is then that is okay).
+				 * No spans should exist and not be used unless added through the import multiplier. */
+				for (bt_index = 0; bt_index < sBTs.uiBTCount; bt_index++)
+				{
+					_FreeBT(pArena, sBTs.apsBTs[bt_index]);
+				}
+			}
+		}
+#endif
+		if (sBTs.apsBTs != apsBTsStatic)
+		{
+			OSFreeMem(sBTs.apsBTs);
+		}
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_AllocMulti(RA_ARENA *pArena,
+              RA_LENGTH_T uRequestSize,
+              IMG_UINT32 uiLog2ChunkSize,
+              IMG_UINT8 uImportMultiplier,
+              RA_FLAGS_T uImportFlags,
+              const IMG_CHAR *pszAnnotation,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+              IMG_BOOL *bPhysContig)
+{
+	PVRSRV_ERROR eError;
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_AllocMultiUnlocked(pArena,
+	                                 uRequestSize,
+	                                 uiLog2ChunkSize,
+	                                 uImportMultiplier,
+	                                 uImportFlags,
+	                                 pszAnnotation,
+	                                 aBaseArray,
+	                                 uiBaseArraySize,
+	                                 IMG_FALSE, /* Sparse alloc */
+	                                 bPhysContig);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_AllocMultiSparse(RA_ARENA *pArena,
+                     IMG_UINT32 uiLog2ChunkSize,
+                     IMG_UINT8 uImportMultiplier,
+                     RA_FLAGS_T uImportFlags,
+                     const IMG_CHAR *pszAnnotation,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                     IMG_UINT32 *puiAllocIndices,
+                     IMG_UINT32 uiAllocCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bPhysContig;
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+
+	/*
+	 * In this case the arguments given show the allocation is
+	 * sparse but has no specific indices, this indicates
+	 * we want to populate the full aBaseArray
+	 */
+	if (puiAllocIndices == NULL)
+	{
+		RA_LENGTH_T uRequestSize = (RA_LENGTH_T) uiAllocCount << uiLog2ChunkSize;
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 uRequestSize,
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 aBaseArray,
+		                                 uiBaseArraySize,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_IF_ERROR(eError, "RA_AllocMulti");
+		OSLockRelease(pArena->hLock);
+		return eError;
+	}
+
+	/*
+	 * This case is optimised for single allocations as we can skip
+	 * some of the iteration logic in the full allocation path.
+	 */
+	if (uiAllocCount == 1)
+	{
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 &aBaseArray[puiAllocIndices[0]],
+		                                 1,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_IF_ERROR(eError, "RA_AllocMulti");
+		OSLockRelease(pArena->hLock);
+		return eError;
+	}
+
+	/*
+	 * By consolidating / grouping the indices given we can perform sparse allocations
+	 * in blocks, this has the effect of reducing fragmentation and creating optimal free
+	 * scenarios. Free can be performed in blocks rather than a chunk at a time, this reduces
+	 * the amount of BT merging cycles we perform.
+	 */
+	for (i = 0; i < uiAllocCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		for (j = i;
+		     j + 1 != uiAllocCount &&
+		     puiAllocIndices[j + 1] == puiAllocIndices[j] + 1;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _RA_AllocMultiUnlocked(pArena,
+		                                 IMG_PAGES2BYTES64(uiConsolidate, uiLog2ChunkSize),
+		                                 uiLog2ChunkSize,
+		                                 uImportMultiplier,
+		                                 uImportFlags,
+		                                 pszAnnotation,
+		                                 &aBaseArray[puiAllocIndices[i]],
+		                                 uiConsolidate,
+		                                 IMG_TRUE, /* Sparse alloc */
+		                                 &bPhysContig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RA_AllocMulti", unwind_alloc);
+		i += uiConsolidate;
+	}
+
+	OSLockRelease(pArena->hLock);
+	return PVRSRV_OK;
+
+unwind_alloc:
+	if (i != 0)
+	{
+		PVRSRV_ERROR eFreeError;
+		eFreeError = _RA_FreeMultiUnlockedSparse(pArena,
+		                                          aBaseArray,
+		                                          uiBaseArraySize,
+		                                          IMG_PAGE2BYTES64(uiLog2ChunkSize),
+		                                          puiAllocIndices,
+		                                          &i);
+		PVR_LOG_IF_ERROR(eFreeError, "_RA_FreeMultiUnlockedSparse");
+	}
+
+	OSLockRelease(pArena->hLock);
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RA_Find_BT_VARange
+@Description    To find the boundary tag associated with the given device
+                                  virtual address.
+@Input          pArena            The arena
+@input          base              Allocated base resource
+@Input          uRequestSize      The size of resource segment requested.
+@Input          uImportFlags            Flags influencing allocation policy.
+@Return         Boundary Tag - success, NULL on failure
+*/ /**************************************************************************/
+static BT *RA_Find_BT_VARange(RA_ARENA *pArena,
+                              RA_BASE_T base,
+                              RA_LENGTH_T uRequestSize,
+                              RA_FLAGS_T uImportFlags)
+{
+	IMG_PSPLAY_TREE psSplaynode;
+	IMG_UINT32 uIndex;
+
+	/* Find the splay node associated with these import flags */
+	psSplaynode = PVRSRVFindNode(uImportFlags, pArena->per_flags_buckets);
+
+	if (psSplaynode == NULL)
+	{
+		return NULL;
+	}
+
+	uIndex = pvr_log2(uRequestSize);
+
+	/* Find the free Boundary Tag from the bucket that holds the requested range */
+	while (uIndex < FREE_TABLE_LIMIT)
+	{
+		BT *pBT = psSplaynode->buckets[uIndex];
+
+		while (pBT)
+		{
+			if ((pBT->base <= base) && ((pBT->base + pBT->uSize) >= (base + uRequestSize)))
+			{
+				if (pBT->type == btt_free)
+				{
+					return pBT;
+				}
+				else
+				{
+					PVR_ASSERT(pBT->type == btt_free);
+				}
+			}
+			else{
+				pBT = pBT->next_free;
+			}
+		}
+
+#if defined(PVR_CTZLL)
+		/* This could further be optimised to get the next valid bucket */
+		while (!(psSplaynode->bHasEltsMapping & (1ULL << ++uIndex)));
+#else
+		uIndex++;
+#endif
+	}
+
+	return NULL;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_Alloc_Range(RA_ARENA *pArena,
+               RA_LENGTH_T uRequestSize,
+               RA_FLAGS_T uImportFlags,
+               RA_LENGTH_T uAlignment,
+               RA_BASE_T base,
+               RA_LENGTH_T *pActualSize)
+{
+	RA_LENGTH_T uSize = uRequestSize;
+	BT *pBT = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (pArena == NULL || uSize == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: One of the necessary parameters is 0", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	/* Align the requested size to the Arena Quantum */
+	uSize = PVR_ALIGN(uSize, pArena->uQuantum);
+
+	/* Must be a power of 2 or 0 */
+	PVR_ASSERT((uAlignment == 0) || (uAlignment & (uAlignment - 1)) == 0);
+
+	if (uAlignment > 1)
+	{
+		if (base != PVR_ALIGN(base, uAlignment))
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, unlock_);
+		}
+	}
+
+	/* Find if the segment in the range exists and is free
+	 * Check if the segment can be split
+	 * Find the bucket that points to this segment
+	 * Find the free segment is in the free list
+	 * remove the free segment
+	 * split the segment into three segments one prior free, alloc range,
+	 *     free segment after the range.
+	 * remove the allocated range segment from the free list
+	 * hook up the prior and after segments back to free list
+	 * For each free, find the bucket the segment should go to
+	 */
+
+	pBT = RA_Find_BT_VARange(pArena, base, uSize, uImportFlags);
+
+	if (pBT == NULL)
+	{
+		PVR_GOTO_WITH_ERROR(eError,
+		                    PVRSRV_ERROR_RA_REQUEST_VIRT_ADDR_FAIL,
+		                    unlock_);
+	}
+
+	/* Remove the boundary tag from the free list */
+	_FreeListRemove (pArena, pBT);
+
+	/* if requested VA start in the middle of the BT, split the BT accordingly */
+	if (base > pBT->base)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit (pBT, (RA_LENGTH_T)(base - pBT->base));
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			/* Put pBT back in the list */
+			_FreeListInsert (pArena, pBT);
+			PVR_LOG_GOTO_WITH_ERROR("_SegmentSplit (1)", eError,
+			                        PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL,
+			                        unlock_);
+		}
+
+		/* Insert back the free BT to the free list */
+		_FreeListInsert(pArena, pBT);
+		pBT = pNeighbour;
+	}
+
+	/* the segment might be too big, if so, discard the back of the segment */
+	if (pBT->uSize > uSize)
+	{
+		BT *pNeighbour;
+		pNeighbour = _SegmentSplit(pBT, uSize);
+		/* partition the buffer, create a new boundary tag */
+		if (pNeighbour == NULL)
+		{
+			/* Put pBT back in the list */
+			_FreeListInsert (pArena, pBT);
+			PVR_LOG_GOTO_WITH_ERROR("_SegmentSplit (2)", eError,
+			                        PVRSRV_ERROR_RA_REQUEST_ALLOC_FAIL,
+			                        unlock_);
+		}
+
+		/* Insert back the free BT to the free list */
+		_FreeListInsert (pArena, pNeighbour);
+	}
+
+	pBT->type = btt_live;
+
+	if (!HASH_Insert_Extended (pArena->pSegmentHash, &base, (uintptr_t)pBT))
+	{
+		_FreeBT (pArena, pBT);
+		PVR_GOTO_WITH_ERROR(eError,
+		                    PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED,
+		                    unlock_);
+	}
+
+	if (pActualSize != NULL)
+	{
+		*pActualSize = uSize;
+	}
+
+	pArena->ui64FreeArenaSize -= uSize;
+
+unlock_:
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+IMG_INTERNAL void
+RA_Free(RA_ARENA *pArena, RA_BASE_T base)
+{
+	BT *pBT;
+
+	if (pArena == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: invalid parameter - pArena", __func__));
+		return;
+	}
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	PVR_ASSERT(is_arena_valid(pArena));
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: name='%s', base=0x%llx", __func__, pArena->name,
+	        (unsigned long long)base));
+
+	pBT = (BT *) HASH_Remove_Extended(pArena->pSegmentHash, &base);
+	PVR_ASSERT(pBT != NULL);
+
+	if (pBT)
+	{
+		pArena->ui64FreeArenaSize += pBT->uSize;
+
+		PVR_ASSERT(pBT->base == base);
+		_FreeBT(pArena, pBT);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: no resource span found for given base (0x%llX) in arena %s",
+		         __func__, (unsigned long long) base, pArena->name));
+	}
+
+	PVR_ASSERT(is_arena_valid(pArena));
+	OSLockRelease(pArena->hLock);
+}
+
+static PVRSRV_ERROR
+_RA_FreeMultiUnlocked(RA_ARENA *pArena,
+                       RA_BASE_ARRAY_T aBaseArray,
+                       RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	PVRSRV_ERROR eError;
+
+	/* Free the whole array */
+	if (uiBaseArraySize == 1)
+	{
+		eError = _FreeSingleBaseArray(pArena, aBaseArray, uiBaseArraySize);
+		PVR_LOG_IF_ERROR(eError, "_FreeSingleBaseArray");
+	}
+	else
+	{
+		eError = _FreeMultiBaseArray(pArena, aBaseArray, uiBaseArraySize);
+		PVR_LOG_IF_ERROR(eError, "_FreeMultiBaseArray");
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_FreeMulti(RA_ARENA *pArena,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize)
+{
+	PVRSRV_ERROR eError;
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_FreeMultiUnlocked(pArena,
+	                                aBaseArray,
+	                                uiBaseArraySize);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+static PVRSRV_ERROR
+_RA_FreeMultiUnlockedSparse(RA_ARENA *pArena,
+                             RA_BASE_ARRAY_T aBaseArray,
+                             RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                             RA_LENGTH_T uiChunkSize,
+                             IMG_UINT32 *puiFreeIndices,
+                             IMG_UINT32 *puiFreeCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiFreeCount = *puiFreeCount;
+	*puiFreeCount = 0;
+
+	/* Handle case where we only have 1 base to free. */
+	if (uiFreeCount == 1)
+	{
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiFreeIndices[0],
+		                                      1,
+		                                      CONVERT_AND_FREE);
+		PVR_LOG_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+		if (eError == PVRSRV_OK)
+		{
+			*puiFreeCount = uiFreeCount;
+		}
+		return eError;
+	}
+
+	for (i = 0; i < uiFreeCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[i]));
+
+		for (j = i;
+		     puiFreeIndices[j + 1] == puiFreeIndices[j] + 1 &&
+		     RA_BASE_IS_GHOST(aBaseArray[puiFreeIndices[j + 1]]) &&
+		     j + 1 < uiFreeCount;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiFreeIndices[i],
+		                                      uiConsolidate,
+		                                      CONVERT_AND_FREE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+
+		i += uiConsolidate;
+		*puiFreeCount += uiConsolidate;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_FreeMultiSparse(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                    IMG_UINT32 uiLog2ChunkSize,
+                    IMG_UINT32 *puiFreeIndices,
+                    IMG_UINT32 *puiFreeCount)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE(puiFreeCount != NULL,
+		                        "puiFreeCount Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure Base Array is large enough for intended free */
+	PVR_LOG_RETURN_IF_FALSE(uiBaseArraySize >= *puiFreeCount,
+		                        "Attempt to free more bases than array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(puiFreeIndices != NULL,
+		                        "puiFreeIndices Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	eError = _RA_FreeMultiUnlockedSparse(pArena,
+	                                      aBaseArray,
+	                                      uiBaseArraySize,
+	                                      IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                      puiFreeIndices,
+	                                      puiFreeCount);
+	OSLockRelease(pArena->hLock);
+
+	return eError;
+}
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/*************************************************************************/ /*!
+ *  @Function   _RA_RealiseMultiSparseIndicesUnlocked
+ *
+ *  @Description   Given an array of indices to extract, prepares the base
+ *                 array so that the indices in puiExtractIndices can be
+ *                 moved to another base array.
+ *                 Called when some pages of the base array need to be
+ *                 transferred to another base array. As a result of this call,
+ *                 some ghost addresses in aBaseArray might be converted to
+ *                 real addresses.
+ *
+ *  @Input  pArena - The RA Arena to free the bases on.
+ *  @Input  aBaseArray - The Base array to free from
+ *  @Input  uiBaseArraySize - The Size of the base array to free.
+ *  @Input  uiChunkSize -  The Base chunk size used to generate ghost entries
+ *  @Input  puiExtractIndices - Array of indices to extract
+ *  @Input  puiExtractCount - Number of indices to extract
+ *
+ *  @Return PVRSRV_OK on Success, PVRSRV_ERROR code otherwise.
+*/ /**************************************************************************/
+static PVRSRV_ERROR
+_RA_RealiseMultiSparseIndicesUnlocked(RA_ARENA *pArena,
+                                      RA_BASE_ARRAY_T aBaseArray,
+                                      RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                                      RA_LENGTH_T uiChunkSize,
+                                      IMG_UINT32 *puiExtractIndices,
+                                      IMG_UINT32 *puiExtractCount)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiExtractCount = *puiExtractCount;
+	*puiExtractCount = 0;
+
+	/* Handle case where we only have 1 base to extract. */
+	if (uiExtractCount == 1)
+	{
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiExtractIndices[0],
+		                                      1,
+		                                      CONVERT_DONT_FREE);
+		PVR_LOG_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+		if (eError == PVRSRV_OK)
+		{
+			*puiExtractCount = uiExtractCount;
+		}
+		return eError;
+	}
+
+	for (i = 0; i < uiExtractCount;)
+	{
+		IMG_UINT32 j;
+		IMG_UINT32 uiConsolidate = 1;
+
+		PVR_ASSERT(RA_BASE_IS_REAL(aBaseArray[i]));
+
+		for (j = i;
+		     puiExtractIndices[j + 1] == puiExtractIndices[j] + 1 &&
+		     RA_BASE_IS_GHOST(aBaseArray[puiExtractIndices[j + 1]]) &&
+		     j + 1 < uiExtractCount;
+		     j++)
+		{
+			uiConsolidate++;
+		}
+
+		eError = _ConvertAndFreeBaseArraySlice(pArena,
+		                                      aBaseArray,
+		                                      uiBaseArraySize,
+		                                      uiChunkSize,
+		                                      puiExtractIndices[i],
+		                                      uiConsolidate,
+		                                      CONVERT_DONT_FREE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertAndFreeBaseArraySlice");
+
+		i += uiConsolidate;
+		*puiExtractCount += uiConsolidate;
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_TransferMultiSparseIndices(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aSrcBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiSrcBaseArraySize,
+                              RA_BASE_ARRAY_T aDstBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiDstBaseArraySize,
+                              IMG_UINT32 uiLog2ChunkSize,
+                              IMG_UINT32 *puiTransferIndices,
+                              IMG_UINT32 *puiTransferCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PVR_LOG_RETURN_IF_FALSE(puiTransferCount != NULL,
+		                        "puiTransferCount Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure source Base Array is large enough for intended extract */
+	PVR_LOG_RETURN_IF_FALSE(uiSrcBaseArraySize >= *puiTransferCount,
+		                        "Attempt to transfer more bases than src array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Ensure dst Base Array is large enough for intended extract */
+	PVR_LOG_RETURN_IF_FALSE(uiDstBaseArraySize <= *puiTransferCount,
+		                        "Attempt to transfer more bases than dst array holds",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(puiTransferIndices != NULL,
+		                        "puiTransferIndices Required",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_LOG_RETURN_IF_FALSE(uiLog2ChunkSize >= RA_BASE_FLAGS_LOG2 &&
+	                        uiLog2ChunkSize <= RA_BASE_CHUNK_LOG2_MAX,
+		                    "Log2 chunk size must be 12-64",
+		                    PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+	/* First prepare the base array for subsequent transfer */
+	eError = _RA_RealiseMultiSparseIndicesUnlocked(pArena,
+	                                               aSrcBaseArray,
+	                                               uiSrcBaseArraySize,
+	                                               IMG_PAGE2BYTES64(uiLog2ChunkSize),
+	                                               puiTransferIndices,
+	                                               puiTransferCount);
+	OSLockRelease(pArena->hLock);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	/* Now do the transfer */
+	for (i=0; i<*puiTransferCount; i++)
+	{
+		IMG_UINT32 idxSrc = puiTransferIndices[i];
+		aDstBaseArray[i] = aSrcBaseArray[idxSrc];
+		aSrcBaseArray[idxSrc] = INVALID_BASE_ADDR;
+	}
+
+	return PVRSRV_OK;
+e0:
+	return eError;
+}
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+static PVRSRV_ERROR
+_TrimBlockMakeReal(RA_ARENA *pArena,
+                   RA_BASE_ARRAY_T aBaseArray,
+                   RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                   IMG_UINT32 uiLog2ChunkSize,
+                   IMG_UINT32 uiStartIndex,
+                   IMG_UINT32 uiEndIndex)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RA_BASE_T sRealBase;
+	IMG_UINT32 uiRealBaseIndex;
+
+	/* Note: Error return paths in this function do not require unwinding.
+	 * Free logic is performed based upon indices and detection of Real base regions,
+	 * performance wise it would be more costly to unwind the conversion here than to
+	 * just free a smaller Real base region.
+	 */
+
+	/* Check Start index is real, if not make it real */
+	if (RA_BASE_IS_GHOST(aBaseArray[uiStartIndex]))
+	{
+		_FindRealBaseFromGhost(aBaseArray,
+		                       uiStartIndex,
+		                       &sRealBase,
+		                       &uiRealBaseIndex);
+
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 sRealBase,
+		                                 uiRealBaseIndex,
+		                                 uiStartIndex,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize));
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	/* Check end +1 is real or end of array , if ghost make real */
+	if (uiEndIndex + 1 != uiBaseArraySize &&
+	    RA_BASE_IS_GHOST(aBaseArray[uiEndIndex + 1]))
+	{
+		_FindRealBaseFromGhost(aBaseArray,
+		                       uiEndIndex + 1,
+		                       &sRealBase,
+		                       &uiRealBaseIndex);
+
+		eError = _ConvertGhostBaseToReal(pArena,
+		                                 aBaseArray,
+		                                 sRealBase,
+		                                 uiRealBaseIndex,
+		                                 uiEndIndex + 1,
+		                                 IMG_PAGE2BYTES64(uiLog2ChunkSize));
+		PVR_LOG_RETURN_IF_ERROR(eError, "_ConvertGhostBaseToReal");
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_SwapSparseMem(RA_ARENA *pArena,
+                  RA_BASE_ARRAY_T aBaseArray,
+                  RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                  IMG_UINT32 uiLog2ChunkSize,
+                  IMG_UINT32 *puiXIndices,
+                  IMG_UINT32 *puiYIndices,
+                  IMG_UINT32 uiSwapCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiSwapped = 0;
+	IMG_UINT32 uiStartIndex;
+	/* Consolidation values counting the bases after the start index*/
+	IMG_UINT32 uiXConsol;
+	IMG_UINT32 uiYConsol;
+	/* Consolidation limit, the smallest consecutive indices between the
+	 * two inputs
+	 */
+	IMG_UINT32 uiConsolidateLimit;
+	IMG_UINT32 uiTotalSwapCount;
+	IMG_UINT32 i;
+
+	/*
+	 * The algorithm below aims to swap the desired indices whilst also
+	 * maintaining a maximum contiguity of allocation blocks where possible.
+	 * It does this by:
+	 * Consolidating the contiguous indices of X and Y.
+	 * Selecting the smallest of these consolidations as a range to swap in a block.
+	 * Trim both block ranges using the indices range to ensure that Real bases are
+	 * created to represent regions that have been split due to the indices.
+	 * Perform the swap and update the swapped count ready for the next iteration.
+	 * Note: Maintaining contiguity improves performance of free logic for sparse
+	 * allocations because we can free in regions rather than chunks.
+	 */
+	while (uiSwapped != uiSwapCount)
+	{
+		IMG_UINT32 x, y;
+		uiTotalSwapCount = 1;
+		uiStartIndex = uiSwapped;
+		uiXConsol = 0;
+		uiYConsol = 0;
+
+		/* Calculate contiguous indices at X */
+		for (x = uiStartIndex;
+		     x < uiSwapCount &&
+		     puiXIndices[x] + 1 == puiXIndices[x + 1];
+		     x++)
+		{
+			uiXConsol++;
+		}
+
+		/* Calculate contiguous indices at Y */
+		for (y = uiStartIndex;
+		     y < uiSwapCount &&
+		     puiYIndices[y] + 1 == puiYIndices[y + 1];
+		     y++)
+		{
+			uiYConsol++;
+		}
+
+		/* Find lowest consolidation value */
+		uiConsolidateLimit = (uiXConsol < uiYConsol) ? uiXConsol : uiYConsol;
+
+		/* Perform RealBase translation where required */
+		eError = _TrimBlockMakeReal(pArena,
+		                            aBaseArray,
+		                            uiBaseArraySize,
+		                            uiLog2ChunkSize,
+		                            puiXIndices[uiStartIndex],
+		                            puiXIndices[uiStartIndex + uiConsolidateLimit]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_TrimBlockMakeReal", unwind);
+
+		eError = _TrimBlockMakeReal(pArena,
+		                            aBaseArray,
+		                            uiBaseArraySize,
+		                            uiLog2ChunkSize,
+		                            puiYIndices[uiStartIndex],
+		                            puiYIndices[uiStartIndex + uiConsolidateLimit]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "_TrimBlockMakeReal", unwind);
+
+		uiTotalSwapCount += uiConsolidateLimit;
+		uiSwapped += uiTotalSwapCount;
+		i = uiStartIndex;
+
+		do
+		{
+			SWAP(aBaseArray[puiXIndices[i]], aBaseArray[puiYIndices[i]]);
+			uiTotalSwapCount--;
+			i++;
+		}
+		while (uiTotalSwapCount != 0);
+	}
+
+	return PVRSRV_OK;
+
+unwind:
+	/* If we hit an error when Trimming we should revert the swapping
+	 * that has already been performed.
+	 */
+	for (i = 0; i < uiSwapped; i++)
+	{
+		SWAP(aBaseArray[puiXIndices[i]], aBaseArray[puiYIndices[i]]);
+	}
+
+	return eError;
+}
+
+IMG_INTERNAL void
+RA_Get_Usage_Stats(RA_ARENA *pArena, PRA_USAGE_STATS psRAStats)
+{
+	psRAStats->ui64TotalArenaSize = pArena->ui64TotalArenaSize;
+	psRAStats->ui64FreeArenaSize = pArena->ui64FreeArenaSize;
+}
+
+IMG_INTERNAL IMG_CHAR *
+RA_GetArenaName(RA_ARENA *pArena)
+{
+	return pArena->name;
+}
+
+/* #define _DBG(...) PVR_LOG((__VA_ARGS__)) */
+#define _DBG(...)
+
+IMG_INTERNAL RA_ARENA_ITERATOR *
+RA_IteratorAcquire(RA_ARENA *pArena, IMG_BOOL bIncludeFreeSegments)
+{
+	RA_ARENA_ITERATOR *pIter = OSAllocMem(sizeof(*pIter));
+	PVR_LOG_RETURN_IF_FALSE(pIter != NULL, "OSAllocMem", NULL);
+
+	OSLockAcquireNested(pArena->hLock, pArena->ui32LockClass);
+
+	pIter->pArena = pArena;
+	pIter->bIncludeFreeSegments = bIncludeFreeSegments;
+
+	RA_IteratorReset(pIter);
+
+	return pIter;
+}
+
+IMG_INTERNAL void
+RA_IteratorRelease(RA_ARENA_ITERATOR *pIter)
+{
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		return;
+	}
+
+	OSLockRelease(pIter->pArena->hLock);
+
+	OSFreeMem(pIter);
+}
+
+IMG_INTERNAL void
+RA_IteratorReset(RA_ARENA_ITERATOR *pIter)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	pNext = pIter->pArena->pHeadSegment;
+
+	/* find next element if we're not including the free ones */
+	if (!pIter->bIncludeFreeSegments)
+	{
+		while (pNext != NULL && pNext->type != btt_live)
+		{
+			_DBG("(%s()) skipping segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+			     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+			     pNext->type);
+			pNext = pNext->pNextSegment;
+		}
+	}
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	/* if bIncludeFreeSegments then pNext here is either a valid pointer to
+	 * "live" segment or NULL and if !bIncludeFreeSegments then it's either
+	 * a valid pointer to any next segment or NULL */
+	pIter->pCurrent = pNext;
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNext(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	if (pIter->pCurrent == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pNext = pIter->pCurrent;
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+	     pNext->type);
+
+	pData->uiAddr = pIter->pCurrent->base;
+	pData->uiSize = pIter->pCurrent->uSize;
+	pData->bFree = pIter->pCurrent->type == btt_free;
+
+	/* combine contiguous segments */
+	while ((pNext = pNext->pNextSegment) != NULL &&
+	        pNext->type == pNext->pPrevSegment->type &&
+	        pNext->type == btt_live &&
+	        pNext->base == pData->uiAddr + pData->uiSize)
+	{
+		_DBG("(%s()) combining segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+		     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+		     pNext->type);
+		pData->uiSize += pNext->uSize;
+	}
+
+	/* advance to next */
+	if (!pIter->bIncludeFreeSegments)
+	{
+		while (pNext != NULL && pNext->type != btt_live)
+		{
+			_DBG("(%s()) skipping segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+			     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+			     pNext->type);
+			pNext = pNext->pNextSegment;
+		}
+	}
+
+	_DBG("(%s()) next segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	/* if bIncludeFreeSegments then pNext here is either a valid pointer to
+	 * "live" segment or NULL and if !bIncludeFreeSegments then it's either
+	 * a valid pointer to any next segment or NULL */
+	pIter->pCurrent = pNext;
+
+	return IMG_TRUE;
+}
+
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNextSpan(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData)
+{
+	BT *pNext;
+
+	PVR_ASSERT(pIter != NULL);
+
+	if (pIter == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "pIter in %s() is NULL", __func__));
+		return IMG_FALSE;
+	}
+
+	/* We have no more iterations to perform */
+	if (pIter->pCurrent == NULL)
+	{
+		return IMG_FALSE;
+	}
+
+	pNext = pIter->pCurrent;
+
+	_DBG("(%s()) current segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+	     pNext->type);
+
+	/* Extract details of current span */
+	pData->uiAddr = pIter->pCurrent->base;
+	pData->uiSize = pIter->pCurrent->uSize;
+	pData->bFree = pIter->pCurrent->free_import;
+
+	/* If this BT is a full span, can occur if we have no allocations
+	 * on a span.
+	 */
+	if (pNext->is_leftmost == 1 &&
+	    pNext->is_rightmost == 1)
+	{
+		pNext = pNext->pNextSegment;
+	}
+	else
+	{
+		/* combine contiguous segments of span, these are allocations
+		 * and free segments in a span.
+		 */
+		while ((pNext = pNext->pNextSegment) != NULL &&
+		        pNext->is_leftmost == 0)
+		{
+			_DBG("(%s()) combining segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+				 "type=%u", __func__, (void *) pNext->base, pNext->uSize,
+				 pNext->type);
+			pData->uiSize += pNext->uSize;
+		}
+	}
+
+	_DBG("(%s()) next segment=%px, size=0x%" IMG_UINT64_FMTSPECx ", "
+	     "type=%u", __func__,
+	     pNext != NULL ? (void *) pNext->base : NULL,
+	     pNext != NULL ? pNext->uSize : 0,
+	     pNext != NULL ? pNext->type : 0);
+
+	pIter->pCurrent = pNext;
+
+	return IMG_TRUE;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+RA_BlockDump(RA_ARENA *pArena, void (*pfnLogDump)(void*, IMG_CHAR*, ...), void *pPrivData)
+{
+	RA_ARENA_ITERATOR *pIter = NULL;
+	RA_ITERATOR_DATA sIterData;
+	const IMG_UINT32 uiLineWidth = 64;
+
+	IMG_UINT32 **papRegionArray = NULL;
+	IMG_UINT32 uiRegionCount = 0;
+
+	const IMG_UINT32 uiChunkSize = 32; /* 32-bit chunks */
+	const IMG_UINT32 uiChunkCount = (uiLineWidth / uiChunkSize) * 2; /* This should equal 2 or a multiple of 2 */
+	const IMG_UINT32 uiRegionSize = uiChunkSize * uiChunkCount;
+
+	IMG_UINT32 uiRecognisedQuantum = 0;
+
+	IMG_UINT64 uiLastBase = 0;
+	IMG_UINT64 uiLastSize = 0;
+
+	IMG_UINT32 i;
+	IMG_UINT32 uiRemainder;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	IMG_UINT64 uiLargestFreeSegmentSize = 0;
+	IMG_UINT32 uiFragPercentage = 0;
+
+	/* -- papRegionArray Structure --
+	 *  papRegionArray Indexes
+	 *  |         Chunk 0      Chunk 1      Chunk 2      Chunk 3
+	 *  v     |------------|------------|------------|------------|
+	 * [0] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 | -- |
+	 * [1] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [2] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [3] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    | Regions
+	 * [4] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [5] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 |    |
+	 * [6] -> | 0000000000 | 0000000000 | 0000000000 | 0000000000 | -- |
+	 * ...
+	 */
+
+	if (pArena == NULL || pfnLogDump == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	pIter = RA_IteratorAcquire(pArena, IMG_TRUE);
+	PVR_LOG_RETURN_IF_NOMEM(pIter, "RA_IteratorAcquire");
+
+	uiRecognisedQuantum = pArena->uQuantum > 0 ? pArena->uQuantum : 4096;
+
+	while (RA_IteratorNext(pIter, &sIterData))
+	{
+		if (!sIterData.bFree && sIterData.uiAddr >= uiLastBase)
+		{
+			uiLastBase = sIterData.uiAddr;
+			uiLastSize = sIterData.uiSize;
+		}
+	}
+
+	uiRegionCount = OSDivide64(uiLastBase + uiLastSize, uiRecognisedQuantum,
+	                           &uiRemainder);
+	uiRegionCount = OSDivide64(uiRegionCount, uiRegionSize, &uiRemainder);
+	if (uiRemainder != 0 || uiRegionCount == 0)
+	{
+		uiRegionCount += 1;
+	}
+
+	papRegionArray = OSAllocZMem(sizeof(IMG_UINT32*) * uiRegionCount);
+	PVR_LOG_GOTO_IF_NOMEM(papRegionArray, eError, cleanup_array);
+
+	RA_IteratorReset(pIter);
+
+	while (RA_IteratorNext(pIter, &sIterData))
+	{
+		IMG_UINT64 uiDataDivRecQuant;
+
+		IMG_UINT32 uiAddrRegionIdx = 0;
+		IMG_UINT32 uiAddrRegionOffset = 0;
+		IMG_UINT32 uiAddrChunkIdx = 0;
+		IMG_UINT32 uiAddrChunkOffset = 0;
+		IMG_UINT32 uiAddrChunkShift; /* The bit-shift needed to fill the chunk */
+
+		IMG_UINT32 uiQuantisedSize;
+		IMG_UINT32 uiQuantisedSizeMod;
+		IMG_UINT32 uiAllocLastRegionIdx = 0; /* The last region that this alloc appears in */
+		IMG_UINT32 uiAllocChunkSize = 0; /* The number of chunks this alloc spans */
+
+		IMG_INT32 iBitSetCount = 0;
+		IMG_INT32 iOverflowCheck = 0;
+		IMG_INT32 iOverflow = 0;
+		IMG_UINT32 uiRegionIdx = 0;
+		IMG_UINT32 uiChunkIdx = 0;
+
+		/* If the current data is for a free block, use it to track largest
+		 * contiguous free segment size.
+		 */
+		if (sIterData.bFree && sIterData.uiSize > uiLargestFreeSegmentSize)
+		{
+			uiLargestFreeSegmentSize = sIterData.uiSize;
+			continue;
+		}
+
+		uiDataDivRecQuant = OSDivide64(sIterData.uiAddr, uiRecognisedQuantum,
+		                               &uiRemainder);
+		uiAddrRegionIdx = OSDivide64(uiDataDivRecQuant, uiRegionSize,
+		                             &uiAddrRegionOffset);
+		uiQuantisedSize = OSDivide64(sIterData.uiSize, uiRecognisedQuantum,
+		                             &uiQuantisedSizeMod);
+
+		uiAddrChunkIdx = uiAddrRegionOffset / uiChunkSize;
+		uiAddrChunkOffset = uiAddrRegionOffset % uiChunkSize;
+		uiAddrChunkShift = uiChunkSize - uiAddrChunkOffset;
+		uiRegionIdx = uiAddrRegionIdx;
+		uiChunkIdx = uiAddrChunkIdx;
+
+		if ((uiQuantisedSize == 0) || (uiQuantisedSizeMod != 0))
+		{
+			uiQuantisedSize += 1;
+		}
+
+		uiAllocLastRegionIdx = OSDivide64(uiDataDivRecQuant + uiQuantisedSize - 1,
+		                                  uiRegionSize, &uiRemainder);
+		uiAllocChunkSize = (uiAddrChunkOffset + uiQuantisedSize) / uiChunkSize;
+
+		if ((uiAddrChunkOffset + uiQuantisedSize) % uiChunkSize > 0)
+		{
+			uiAllocChunkSize += 1;
+		}
+
+		iBitSetCount = uiQuantisedSize;
+		iOverflowCheck = uiQuantisedSize - uiAddrChunkShift;
+
+		if (iOverflowCheck > 0)
+		{
+			iOverflow = iOverflowCheck;
+			iBitSetCount = uiQuantisedSize - iOverflow;
+		}
+
+		/**
+		 * Allocate memory to represent the chunks for each region the allocation
+		 * spans. If one was already allocated before don't do it again.
+		 */
+		for (i = 0; uiAddrRegionIdx + i <= uiAllocLastRegionIdx; i++)
+		{
+			if (papRegionArray[uiAddrRegionIdx + i] == NULL)
+			{
+				papRegionArray[uiAddrRegionIdx + i] = OSAllocZMem(sizeof(IMG_UINT32) * uiChunkCount);
+				PVR_LOG_GOTO_IF_NOMEM(papRegionArray[uiAddrRegionIdx + i], eError, cleanup_regions);
+			}
+		}
+
+		for (i = 0; i < uiAllocChunkSize; i++)
+		{
+			if (uiChunkIdx >= uiChunkCount)
+			{
+				uiRegionIdx++;
+				uiChunkIdx = 0;
+			}
+
+			if ((IMG_UINT32)iBitSetCount != uiChunkSize)
+			{
+				IMG_UINT32 uiBitMask = 0;
+
+				uiBitMask = (1U << iBitSetCount) - 1;
+				uiBitMask <<= (uiAddrChunkShift - iBitSetCount);
+
+				papRegionArray[uiRegionIdx][uiChunkIdx] |= uiBitMask;
+			}
+			else
+			{
+				papRegionArray[uiRegionIdx][uiChunkIdx] |= 0xFFFFFFFF;
+			}
+
+			uiChunkIdx++;
+			iOverflow -= uiChunkSize;
+			iBitSetCount = iOverflow >= 0 ? uiChunkSize : uiChunkSize + iOverflow;
+			if (iOverflow < 0)
+			{
+				uiAddrChunkShift = 32;
+			}
+		}
+	}
+	if (pArena->ui64FreeArenaSize && uiLargestFreeSegmentSize)
+	{
+		/* N.B This can look strange in a dual RA when comparing to the dump visualisation
+		 * as spans that are freed are not included in the segment list, regardless it is
+		 * an accurate representation for the spans in the Arena.
+		 */
+		uiFragPercentage = OSDivide64(100 * pArena->ui64FreeArenaSize,
+		                              pArena->ui64FreeArenaSize + uiLargestFreeSegmentSize,
+		                              &uiRemainder);
+	}
+
+	pfnLogDump(pPrivData, "~~~ '%s' Resource Arena Block Dump", pArena->name);
+	pfnLogDump(pPrivData, "    Block Size: %uB", uiRecognisedQuantum);
+	pfnLogDump(pPrivData,
+	           "    Span Memory Usage: %"IMG_UINT64_FMTSPEC"B"
+	           "    Free Span Memory: %"IMG_UINT64_FMTSPEC"B"
+	           "    Largest Free Region Size: %"IMG_UINT64_FMTSPEC"B"
+	           "    Percent Fragmented %u%%",
+	           pArena->ui64TotalArenaSize,
+	           pArena->ui64FreeArenaSize,
+	           uiLargestFreeSegmentSize,
+	           uiFragPercentage);
+	pfnLogDump(pPrivData,
+	           "===============================================================================");
+
+	for (i = 0; i < uiRegionCount; i++)
+	{
+		static IMG_BOOL bEmptyRegion = IMG_FALSE;
+		if (papRegionArray[i] != NULL)
+		{
+			IMG_CHAR pszLine[65];
+			IMG_UINT32 j;
+
+			bEmptyRegion = IMG_FALSE;
+			pszLine[64] = '\0';
+
+			for (j = 0; j < uiChunkCount; j+=2)
+			{
+				IMG_UINT8 uiBit = 0;
+				IMG_UINT32 k;
+				IMG_UINT64 uiLineAddress =
+				    (i * uiRegionSize + (j >> 1) * uiLineWidth) * uiRecognisedQuantum;
+
+				/**
+				 * Move through each of the 32 bits in the chunk and check their
+				 * value. If it is 1 we set the corresponding character to '#',
+				 * otherwise it is set to '.' representing empty space
+				 */
+				for (k = 1 << 31; k != 0; k >>= 1)
+				{
+					pszLine[uiBit] = papRegionArray[i][j] & k ? '#' : '.';
+					pszLine[32 + uiBit] = papRegionArray[i][j+1] & k ? '#' : '.';
+					uiBit++;
+				}
+
+				pfnLogDump(pPrivData,
+				           "| 0x%08"IMG_UINT64_FMTSPECx" | %s",
+				           uiLineAddress,
+				           pszLine);
+			}
+			OSFreeMem(papRegionArray[i]);
+		}
+		else
+		{
+			/* We only print this once per gap of n regions */
+			if (!bEmptyRegion)
+			{
+				pfnLogDump(pPrivData, "     ....");
+				bEmptyRegion = IMG_TRUE;
+			}
+		}
+	}
+
+	RA_IteratorRelease(pIter);
+
+	OSFreeMem(papRegionArray);
+	return eError;
+
+cleanup_regions:
+	for (i = 0; i < uiRegionCount; i++)
+	{
+		if (papRegionArray[i] != NULL)
+		{
+			OSFreeMem(papRegionArray[i]);
+		}
+	}
+
+cleanup_array:
+	OSFreeMem(papRegionArray);
+	RA_IteratorRelease(pIter);
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/ra.h b/drivers/gpu/drm/img-rogue/ra.h
new file mode 100644
index 000000000000..ee4492afa1ed
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/ra.h
@@ -0,0 +1,763 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Allocator API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RA_H
+#define RA_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#define RA_MAX_NAME_LENGTH 20
+
+/** Resource arena.
+ *  struct _RA_ARENA_ deliberately opaque
+ */
+typedef struct _RA_ARENA_ RA_ARENA;			//PRQA S 3313
+
+/** Resource arena's iterator.
+ *  struct _RA_ARENA_ITERATOR_ deliberately opaque
+ */
+typedef struct _RA_ARENA_ITERATOR_ RA_ARENA_ITERATOR;
+
+typedef struct _RA_ITERATOR_DATA_ {
+	IMG_UINT64 uiAddr;
+	IMG_UINT64 uiSize;
+	/* Standard Iteration: is the segment free
+	 * Span Iteration: is the span a free import
+	 */
+	IMG_BOOL bFree;
+} RA_ITERATOR_DATA;
+
+/** Resource arena usage statistics.
+ *  struct _RA_USAGE_STATS
+ */
+typedef struct _RA_USAGE_STATS {
+	IMG_UINT64	ui64TotalArenaSize;
+	IMG_UINT64	ui64FreeArenaSize;
+} RA_USAGE_STATS, *PRA_USAGE_STATS;
+
+/*
+ * Per-Arena handle - this is private data for the caller of the RA.
+ * The RA knows nothing about this data. It is given it in RA_Create, and
+ * promises to pass it to calls to the ImportAlloc and ImportFree callbacks
+ */
+typedef IMG_HANDLE RA_PERARENA_HANDLE;
+/*
+ * Per-Import handle - this is private data for the caller of the RA.
+ * The RA knows nothing about this data. It is given it on a per-import basis,
+ * basis, either the "initial" import at RA_Create time, or further imports
+ * via the ImportAlloc callback. It sends it back via the ImportFree callback,
+ * and also provides it in answer to any RA_Alloc request to signify from
+ * which "import" the allocation came.
+ */
+typedef IMG_HANDLE RA_PERISPAN_HANDLE;
+
+typedef IMG_UINT64 RA_BASE_T;
+typedef IMG_UINT32 RA_LOG2QUANTUM_T;
+typedef IMG_UINT64 RA_LENGTH_T;
+typedef IMG_UINT32 RA_POLICY_T;
+
+typedef struct _RA_BASE_MULTI_ RA_BASE_MULTI_T;
+
+typedef IMG_UINT32 RA_BASE_ARRAY_SIZE_T;
+
+
+/*
+ * RA_BASE_ARRAY can represent a number of bases of which are packed,
+ * that is, they can be one of two types, a Real Base or a Ghost base.
+ * A Real Base is a base that has been created by the RA and is used to
+ * represent an allocated region, it has an entry in the RA Hash table and
+ * as such has a BT associated with it.
+ * A Ghost base is a fabricated base address generated at chunk boundaries
+ * given by the caller. These are used to divide a RealBase into
+ * arbitrary regions that the caller requires e.g. 4k pages. Ghost bases don't
+ * exist from the RA memory tracking point of view but they do exist and are treated
+ * as base addresses from the PoV of the caller. This allows the RA to allocate in
+ * largest possible lengths meaning fewer alloc calls whilst allowing the chunk
+ * flexibility for callers. Ghost refers to the concept that they
+ * don't exist in this RA internals context but do in the callers (LMA) context i.e.
+ * they appear Real from another perspective but we the RA know they are a ghost of the
+ * Real Base.
+ * */
+typedef RA_BASE_T RA_BASE_ARRAY_T[IMG_FLEX_ARRAY_MEMBER];
+
+/* Since 0x0 is a valid BaseAddr, we rely on max 64-bit value to be an invalid
+ * page address.
+ */
+#define INVALID_BASE_ADDR (IMG_UINT64_MAX)
+/* Used to check for duplicated alloc indices in sparse alloc path
+ * prior to attempting allocations */
+#define RA_BASE_SPARSE_PREP_ALLOC_ADDR (IMG_UINT64_MAX - 1)
+#define RA_BASE_FLAGS_MASK 0xFFF /* 12 Bits 4k alignment. */
+#define RA_BASE_FLAGS_LOG2 12
+#define RA_BASE_CHUNK_LOG2_MAX 64
+#define RA_BASE_GHOST_BIT (1ULL << 0)
+#define RA_BASE_STRIP_GHOST_BIT(uiBase) ((uiBase) & ~(RA_BASE_GHOST_BIT))
+#define RA_BASE_SET_GHOST_BIT(uiBase)   ((uiBase) |= RA_BASE_GHOST_BIT)
+#define RA_BASE_IS_GHOST(uiBase) (BITMASK_HAS((uiBase), RA_BASE_GHOST_BIT) && (uiBase) != INVALID_BASE_ADDR)
+#define RA_BASE_IS_REAL(uiBase) (!BITMASK_HAS((uiBase), RA_BASE_GHOST_BIT))
+#define RA_BASE_IS_SPARSE_PREP(uiBase) ((uiBase) == RA_BASE_SPARSE_PREP_ALLOC_ADDR)
+#define RA_BASE_IS_INVALID(uiBase) ((uiBase) == INVALID_BASE_ADDR)
+
+typedef struct _RA_MULTIBASE_ITERATOR_ RA_MULTIBASE_ITERATOR;
+
+/* Lock classes: describes the level of nesting between different arenas.
+ * A RA with lockclass x is permitted to obtain locks of a lockclass less than x.
+ * A RA with lockclass 0 cannot obtain the lock of any other RA. */
+#define RA_LOCKCLASS_0 0
+#define RA_LOCKCLASS_1 1
+#define RA_LOCKCLASS_2 2
+#define RA_LOCKCLASS_3 3
+
+#define RA_NO_IMPORT_MULTIPLIER 1
+
+/*
+ * Allocation Policies that govern the resource areas.
+ * */
+
+/* --- Resource allocation policy definitions ---
+* | 31.........5|.......4....|......3....|........2.............|1...................0|
+* | Reserved    | Non-Contig | No split  | Area bucket selection| Alloc node selection|
+*/
+
+/*
+ * Fast allocation policy allows to pick the first node
+ * that satisfies the request.
+ * It is the default policy for all arenas.
+ *  */
+#define RA_POLICY_ALLOC_FAST			(0U)
+/*
+ * Optimal allocation policy allows to pick the lowest size node
+ * that satisfies the request. This picking policy helps in reducing the fragmentation.
+ * This minimises the necessity to split the nodes more often as the optimal
+ * ones are picked.
+ * As a result any future higher size allocation requests are likely to succeed
+ */
+#define RA_POLICY_ALLOC_OPTIMAL		(1U)
+#define RA_POLICY_ALLOC_NODE_SELECT_MASK			(3U)
+
+/*
+ * Bucket selection policies
+ * */
+/* Assured bucket policy makes sure the selected bucket is guaranteed
+ * to satisfy the given request. Generally Nodes picked up from such a
+ * bucket need to be further split. However picking node that belongs to this
+ * bucket is likely to succeed and thus promises better response times */
+#define RA_POLICY_BUCKET_ASSURED_FIT		(0U)
+/*
+ * Best fit bucket policy selects a bucket with free nodes that are likely
+ * to satisfy the request and nodes that are close to the requested size.
+ * Nodes picked up from this bucket may likely to satisfy the request but not
+ * guaranteed. Failing to satisfy the request from this bucket mean further
+ * higher size buckets are selected in the later iterations till the request
+ * is satisfied.
+ *
+ * Hence response times may vary depending on availability of free nodes
+ * that satisfy the request.
+ * */
+#define RA_POLICY_BUCKET_BEST_FIT		(4U)
+#define RA_POLICY_BUCKET_MASK			(4U)
+
+/* This flag ensures the imports will not be split up and Allocations will always get
+ * their own import
+ */
+#define RA_POLICY_NO_SPLIT			(8U)
+#define RA_POLICY_NO_SPLIT_MASK		(8U)
+
+/* This flag is used in physmem_lma only. it is used to decide if we should
+ * activate the non-contiguous allocation feature of RA MultiAlloc.
+ * Requirements for activation are that the OS implements the
+ * OSMapPageArrayToKernelVA function in osfunc which allows for mapping
+ * physically sparse pages as a virtually contiguous range.
+ * */
+#define RA_POLICY_ALLOC_ALLOW_NONCONTIG      (16U)
+#define RA_POLICY_ALLOC_ALLOW_NONCONTIG_MASK (16U)
+
+/*
+ * Default Arena Policy
+ * */
+#define RA_POLICY_DEFAULT			(RA_POLICY_ALLOC_FAST | RA_POLICY_BUCKET_ASSURED_FIT)
+
+/*
+ * Flags in an "import" must match the flags for an allocation
+ */
+typedef IMG_UINT64 RA_FLAGS_T;
+
+typedef struct _RA_IMPORT_ {
+	RA_BASE_T base;           /* Allocation base */
+	RA_LENGTH_T uSize;        /* Allocation size */
+	RA_PERISPAN_HANDLE hPriv; /* Per import private data */
+} RA_IMPORT;
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_ALLOC_SINGLE
+@Description    RA import allocate function create a single span when
+                requesting extra resource for the arena.
+@Input          RA_PERARENA_HANDLE RA handle
+@Input          RA_LENGTH_T        Request size
+@Input          RA_FLAGS_T         RA flags
+@Input          RA_LENGTH_T        Base Alignment
+@Input          IMG_CHAR           Annotation
+@Output         RA_IMPORT          Attributes of an import.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_RA_IMPORT_ALLOC_SINGLE)(RA_PERARENA_HANDLE,
+                                                   RA_LENGTH_T,
+                                                   RA_FLAGS_T,
+                                                   RA_LENGTH_T,
+                                                   const IMG_CHAR*,
+                                                   RA_IMPORT*);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_ALLOC_MULTI
+@Description    RA import allocate function is able to create multiple spans
+                for a single import request.
+                The pointer to the array of inputs will be provided but
+                can be set to a larger array via OSAllocMem
+                if the count is not large enough.
+                The pointer will be freed by OSFreeMem after the callback.
+                It is permissible for the callback to set:
+                count == 0 or array pointer == NULL
+                but the callback MUST return an error in these cases.
+@Input          RA_PERARENA_HANDLE RA handle
+@Input          RA_LENGTH_T        Request size
+@Input          RA_FLAGS_T         RA flags
+@Input          RA_LENGTH_T        Base Alignment
+@Input          IMG_CHAR           Annotation
+@Inout          IMG_UINT32         Num of imports = Num of elements in array below.
+                                   Will always be >= 1. Must be set to the number of
+                                   imports returned.
+@Inout          RA_IMPORT          Pointer to array of imports. Will always point
+                                   to an array of equal length as Num of Imports.
+                                   Will be freed by OSFreeMem.
+@Return         PVRSRV_ERROR       PVRSRV_OK or error code
+*/ /**************************************************************************/
+typedef PVRSRV_ERROR (*PFN_RA_IMPORT_ALLOC_MULTI)(RA_PERARENA_HANDLE,
+                                                  RA_LENGTH_T,
+                                                  RA_FLAGS_T,
+                                                  RA_LENGTH_T,
+                                                  const IMG_CHAR*,
+                                                  IMG_UINT32*,
+                                                  RA_IMPORT**);
+
+/*************************************************************************/ /*!
+@Function       Callback function PFN_RA_IMPORT_FREE
+@Description    RA free imported allocation
+@Input          RA_PERARENA_HANDLE   RA handle
+@Input          RA_BASE_T            Allocation base
+@Input          RA_PERISPAN_HANDLE   Per import private data
+*/ /**************************************************************************/
+typedef void (*PFN_RA_IMPORT_FREE)(RA_PERARENA_HANDLE,
+                                   RA_BASE_T,
+                                   RA_PERISPAN_HANDLE);
+
+/**
+ *  @Function   RA_Create
+ *
+ *  @Description    To create a resource arena.
+ *
+ *  @Input name - the name of the arena for diagnostic purposes.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui32LockClass - the lock class level this arena uses.
+ *  @Input imp_alloc - a resource allocation callback or 0.
+ *  @Input imp_free - a resource de-allocation callback or 0.
+ *  @Input per_arena_handle - private handle passed to alloc and free or 0.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+ */
+RA_ARENA *
+RA_Create(IMG_CHAR *name,
+          /* subsequent imports: */
+          RA_LOG2QUANTUM_T uLog2Quantum,
+          IMG_UINT32 ui32LockClass,
+          PFN_RA_IMPORT_ALLOC_SINGLE imp_alloc,
+          PFN_RA_IMPORT_FREE imp_free,
+          RA_PERARENA_HANDLE per_arena_handle,
+          RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_CreateMulti
+ *
+ *  @Description    To create a resource arena using the
+ *                  ALLOC_MULTI callback.
+ *
+ *  @Input name - the name of the arena for diagnostic purposes.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui32LockClass - the lock class level this arena uses.
+ *  @Input imp_alloc - a resource allocation callback or 0.
+ *  @Input imp_free - a resource de-allocation callback or 0.
+ *  @Input arena_handle - private handle passed to alloc and free or 0.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+ */
+RA_ARENA *
+RA_CreateMulti(IMG_CHAR *name,
+               RA_LOG2QUANTUM_T uLog2Quantum,
+               IMG_UINT32 ui32LockClass,
+               PFN_RA_IMPORT_ALLOC_MULTI imp_alloc,
+               PFN_RA_IMPORT_FREE imp_free,
+               RA_PERARENA_HANDLE arena_handle,
+               RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_Create_With_Span
+ *
+ *  @Description
+ *
+ *  Create a resource arena and initialises it, with a given resource span.
+ *
+ *  @Input name - String briefly describing the RA's purpose.
+ *  @Input uLog2Quantum - the arena allocation quantum.
+ *  @Input ui64CpuBase - CPU Physical Base Address of the RA.
+ *  @Input ui64SpanDevBase - Device Physical Base Address of the RA.
+ *  @Input ui64SpanSize - Size of the span to add to the created RA.
+ *  @Input ui32PolicyFlags - Policies that govern the arena.
+ *  @Return pointer to arena, or NULL.
+*/
+RA_ARENA *
+RA_Create_With_Span(IMG_CHAR *name,
+                    RA_LOG2QUANTUM_T uLog2Quantum,
+                    IMG_UINT64 ui64CpuBase,
+                    IMG_UINT64 ui64SpanDevBase,
+                    IMG_UINT64 ui64SpanSize,
+                    RA_POLICY_T ui32PolicyFlags);
+
+/**
+ *  @Function   RA_Delete
+ *
+ *  @Description
+ *
+ *  To delete a resource arena. All resources allocated from the arena
+ *  must be freed before deleting the arena.
+ *
+ *  @Input  pArena - the arena to delete.
+ *  @Return None
+ */
+void
+RA_Delete(RA_ARENA *pArena);
+
+/**
+ *  @Function   RA_Add
+ *
+ *  @Description
+ *
+ *  To add a resource span to an arena. The span must not overlap with
+ *  any span previously added to the arena.
+ *
+ *  @Input pArena - the arena to add a span into.
+ *  @Input base - the base of the span.
+ *  @Input uSize - the extent of the span.
+ *  @Input hPriv - handle associated to the span (reserved for user uses)
+ *  @Return IMG_TRUE - success, IMG_FALSE - failure
+ */
+IMG_BOOL
+RA_Add(RA_ARENA *pArena,
+       RA_BASE_T base,
+       RA_LENGTH_T uSize,
+       RA_FLAGS_T uFlags,
+       RA_PERISPAN_HANDLE hPriv);
+
+/**
+ *  @Function   RA_Alloc
+ *
+ *  @Description    To allocate resource from an arena.
+ *
+ *  @Input  pArena - the arena
+ *  @Input  uRequestSize - the size of resource segment requested.
+ *  @Input  uImportMultiplier - Import x-times of the uRequestSize
+ *          for future RA_Alloc calls.
+ *          Use RA_NO_IMPORT_MULTIPLIER to import the exact size.
+ *  @Input  uImportFlags - flags influencing allocation policy.
+ *  @Input  uAlignment - the alignment constraint required for the
+ *          allocated segment, use 0 if alignment not required.
+ *  @Input  pszAnnotation - a string to describe the allocation
+ *  @Output base - allocated base resource
+ *  @Output pActualSize - the actual_size of resource segment allocated,
+ *          typically rounded up by quantum.
+ *  @Output phPriv - the user reference associated with allocated
+ *          resource span.
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_Alloc(RA_ARENA *pArena,
+         RA_LENGTH_T uRequestSize,
+         IMG_UINT8 uImportMultiplier,
+         RA_FLAGS_T uImportFlags,
+         RA_LENGTH_T uAlignment,
+         const IMG_CHAR *pszAnnotation,
+         RA_BASE_T *base,
+         RA_LENGTH_T *pActualSize,
+         RA_PERISPAN_HANDLE *phPriv);
+
+/*************************************************************************/ /*!
+@Function       RA_AllocMulti
+@Description    To allocate resource from an arena.
+                This method of allocation can be used to guarantee that if there
+                is enough space in the RA and the contiguity given is the
+                greatest common divisor of the contiguities used on this RA
+                the allocation can be made.
+                Allocations with contiguity less than the current GCD
+                (Greatest Common Divisor) abiding to pow2 are also guaranteed to
+                succeed. See scenario 4.
+                Allocations are not guaranteed but still reduce fragmentation
+                using this method when multiple contiguities are used e.g.
+                4k & 16k and the current allocation has a contiguity higher than
+                the greatest common divisor used.
+                Scenarios with Log 2 contiguity examples:
+                1. All allocations have contiguity of 4k. Allocations can be
+                guaranteed given enough RA space since the GCD is always used.
+                2. Allocations of 4k and 16k contiguity have been previously
+                made on this RA. A new allocation of 4k contiguity is guaranteed
+                to succeed given enough RA space since the contiguity is the GCD.
+                3. Allocations of 4k and 16k contiguity have been previously made
+                on this RA. A new allocation of 16k contiguity is not guaranteed
+                to succeed since it is not the GCD of all contiguities used.
+                4. Contiguity 16k and 64k already exist, a 4k contiguity
+                allocation would be guaranteed to succeed but would now be the
+                new GCD. So further allocations would be required to match this
+                GCD to guarantee success.
+                This method does not suffer the same fragmentation pitfalls
+                as RA_Alloc as it constructs the allocation size from many
+                smaller constituent allocations, these are represented and returned
+                in the given array. In addition, Ghost bases are generated in
+                array entries conforming to the chunk size, this allows for
+                representing chunks of any size that work as page addrs
+                in upper levels.
+                The aforementioned array must be at least of size
+                uRequestsize / uiChunkSize, this ensures there is at least one
+                array entry per chunk required.
+                This function must have a uiChunkSize value of
+                at least 4096, this is to ensure space for the base type encoding.
+@Input          pArena            The arena
+@Input          uRequestSize      The size of resource requested.
+@Input          uiLog2ChunkSize   The log2 contiguity multiple of the bases i.e all
+                                  Real bases must be a multiple in size of this
+                                  size, also used to generate Ghost bases.
+                                  Allocations will also be aligned to this value.
+@Input          uImportMultiplier Import x-times more for future requests if
+                                  we have to import new resource.
+@Input          uImportFlags      Flags influencing allocation policy.
+                                  required, otherwise must be a power of 2.
+@Input          pszAnnotation     String to describe the allocation
+@InOut          aBaseArray        Array of bases to populate.
+@Input          uiBaseArraySize   Size of the array to populate.
+@Output         bPhysContig       Are the allocations made in the RA physically
+                                  contiguous.
+@Return         PVRSRV_OK - success
+*/ /**************************************************************************/
+PVRSRV_ERROR
+RA_AllocMulti(RA_ARENA *pArena,
+               RA_LENGTH_T uRequestSize,
+               IMG_UINT32 uiLog2ChunkSize,
+               IMG_UINT8 uImportMultiplier,
+               RA_FLAGS_T uImportFlags,
+               const IMG_CHAR *pszAnnotation,
+               RA_BASE_ARRAY_T aBaseArray,
+               RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+               IMG_BOOL *bPhysContig);
+
+/**
+ * @Function   RA_AllocMultiSparse
+ *
+ * @Description    To Alloc resource from an RA arena at the specified indices.
+ *                 This function follows the same conditions and functionality as
+ *                 RA_AllocMulti although with the added aspect of specifying the
+ *                 indices to allocate in the Base Array. This means we can still
+ *                 attempt to maintain contiguity where possible with the aim of
+ *                 reducing fragmentation and increasing occurrence of optimal free
+ *                 scenarios.
+ * @Input          pArena            The Arena
+ * @Input          uiLog2ChunkSize   The log2 contiguity multiple of the bases i.e all
+ *                                   Real bases must be a multiple in size of this
+ *                                   size, also used to generate Ghost bases.
+ *                                   Allocations will also be aligned to this value.
+ * @Input          uImportMultiplier Import x-times more for future requests if
+ *                                   we have to import new resource.
+ * @Input          uImportFlags      Flags influencing allocation policy.
+ *                                   required, otherwise must be a power of 2.
+ * @Input          pszAnnotation     String to describe the allocation
+ * @InOut          aBaseArray        Array of bases to populate.
+ * @Input          uiBaseArraySize   Size of the array to populate.
+ * @Input          puiAllocIndices   The indices into the array to alloc, if indices are NULL
+ *                                   then we will allocate uiAllocCount chunks sequentially.
+ * @InOut          uiAllocCount      The number of bases to alloc from the array.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_AllocMultiSparse(RA_ARENA *pArena,
+                     IMG_UINT32 uiLog2ChunkSize,
+                     IMG_UINT8 uImportMultiplier,
+                     RA_FLAGS_T uImportFlags,
+                     const IMG_CHAR *pszAnnotation,
+                     RA_BASE_ARRAY_T aBaseArray,
+                     RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                     IMG_UINT32 *puiAllocIndices,
+                     IMG_UINT32 uiAllocCount);
+/**
+ *  @Function   RA_FreeMulti
+ *
+ *  @Description    To free a multi-base resource constructed using
+ *                  a call to RA_AllocMulti.
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aBaseArray - The array to free bases from.
+ *  @Input  uiBaseArraysize - Size of the array to free bases from.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_FreeMulti(RA_ARENA *pArena,
+              RA_BASE_ARRAY_T aBaseArray,
+              RA_BASE_ARRAY_SIZE_T uiBaseArraySize);
+
+/**
+ *  @Function   RA_FreeMultiSparse
+ *
+ *  @Description    To free part of a multi-base resource constructed using
+ *                  a call to RA_AllocMulti.
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aBaseArray - The array to free bases from.
+ *  @Input  uiBaseArraysize - Size of the array to free bases from.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases.
+ *  @Input  puiFreeIndices - The indices into the array to free.
+ *  @InOut  puiFreeCount - The number of bases to free from the array, becomes the number
+ *                         of bases actually free'd. The in value may differ from the out
+ *                         value in cases of error when freeing. The out value can then be
+ *                         used in upper levels to keep any mem tracking structures consistent
+ *                         with what was actually freed before the error occurred.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_FreeMultiSparse(RA_ARENA *pArena,
+                    RA_BASE_ARRAY_T aBaseArray,
+                    RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                    IMG_UINT32 uiLog2ChunkSize,
+                    IMG_UINT32 *puiFreeIndices,
+                    IMG_UINT32 *puiFreeCount);
+
+#if defined(SUPPORT_PMR_PAGES_DEFERRED_FREE)
+/**
+ *  @Function   RA_TransferMultiSparseIndices
+ *
+ *  @Description   Transfers a set of indices specified in puiTransferIndices from
+ *                 aSrcBaseArray to aDstBaseArray.
+ *                 Called when some pages of the base array need to be
+ *                 transfered to another base array. As a result of this call,
+ *                 some ghost addresses in aBaseArray might be converted to
+ *                 real addresses before being transferred..
+ *
+ *  @Input  pArena     - The arena the segment was originally allocated from.
+ *  @Input  aSrcBaseArray - The array to transfer bases from.
+ *  @Input  uiSrcBaseArraySize - Size of the array to transfer bases from.
+ *  @Input  aDstBaseArray - The array to transfer bases to.
+ *  @Input  uiDstBaseArraySize - Size of the array to transfer bases to.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases.
+ *  @Input  puiTransferIndices - The indices into the array to be extracted.
+ *  @InOut  puiTransferCount - The number of bases to prepare for extraction.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_TransferMultiSparseIndices(RA_ARENA *pArena,
+                              RA_BASE_ARRAY_T aSrcBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiSrcBaseArraySize,
+                              RA_BASE_ARRAY_T aDstBaseArray,
+                              RA_BASE_ARRAY_SIZE_T uiDstBaseArraySize,
+                              IMG_UINT32 uiLog2ChunkSize,
+                              IMG_UINT32 *puiTransferIndices,
+                              IMG_UINT32 *puiTransferCount);
+#endif /* defined(SUPPORT_PMR_PAGES_DEFERRED_FREE) */
+
+/**
+ *  @Function   RA_Alloc_Range
+ *
+ *  @Description
+ *
+ *  To allocate a resource at a specified base from an arena.
+ *
+ *  @Input  pArena - the arena
+ *  @Input  uRequestSize - the size of resource segment requested.
+ *  @Input  uImportFlags - flags influencing allocation policy.
+ *  @Input  uAlignment - the alignment constraint required for the
+ *          allocated segment, use 0 if alignment not required.
+ *  @Input  base - allocated base resource
+ *  @Output pActualSize - the actual_size of resource segment allocated,
+ *          typically rounded up by quantum.
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_Alloc_Range(RA_ARENA *pArena,
+		  RA_LENGTH_T uRequestSize,
+		  RA_FLAGS_T uImportFlags,
+		  RA_LENGTH_T uAlignment,
+		  RA_BASE_T base,
+		  RA_LENGTH_T *pActualSize);
+
+/**
+ *  @Function   RA_Free
+ *
+ *  @Description    To free a resource segment.
+ *
+ *  @Input  pArena - the arena the segment was originally allocated from.
+ *  @Input  base - the base of the resource span to free.
+ *
+ *  @Return None
+ */
+void
+RA_Free(RA_ARENA *pArena, RA_BASE_T base);
+
+/**
+ *  @Function   RA_SwapSparseMem
+ *
+ *  @Description    Swaps chunk sized allocations at X<->Y indices.
+ *                  The function is most optimal when Indices are provided
+ *                  in ascending order, this allows the internals to optimally
+ *                  swap based on contiguity and reduces the amount of ghost to
+ *                  real conversion performed. Note this function can also be used
+ *                  to move pages, in this case, we effectively swap real allocations
+ *                  with invalid marked bases.
+ *  @Input  pArena     - The arena.
+ *  @InOut  aBaseArray - The array to Swap bases in.
+ *  @Input  uiBaseArraysize - Size of the array to Swap bases in.
+ *  @Input  uiLog2ChunkSize - The log2 chunk size used to generate the Ghost bases
+ *                            and size the Real chunks.
+ *  @Input  puiXIndices - Set of X indices to swap with parallel indices in Y.
+ *  @Input  puiYIndices - Set of Y indices to swap with parallel indices in X.
+ *  @Input  uiSwapCount - Number of indices to swap.
+ *
+ *  @Return PVRSRV_OK - success
+ */
+PVRSRV_ERROR
+RA_SwapSparseMem(RA_ARENA *pArena,
+                  RA_BASE_ARRAY_T aBaseArray,
+                  RA_BASE_ARRAY_SIZE_T uiBaseArraySize,
+                  IMG_UINT32 uiLog2ChunkSize,
+                  IMG_UINT32 *puiXIndices,
+                  IMG_UINT32 *puiYIndices,
+                  IMG_UINT32 uiSwapCount);
+
+/**
+ *  @Function   RA_Get_Usage_Stats
+ *
+ *  @Description    To collect the arena usage statistics.
+ *
+ *  @Input  pArena - the arena to acquire usage statistics from.
+ *  @Input  psRAStats - the buffer to hold the usage statistics of the arena.
+ *
+ *  @Return None
+ */
+IMG_INTERNAL void
+RA_Get_Usage_Stats(RA_ARENA *pArena, PRA_USAGE_STATS psRAStats);
+
+/**
+ *  @Function   RA_GetArenaName
+ *
+ *  @Description    To obtain the arena name.
+ *
+ *  @Input  pArena - the arena to acquire the name from.
+ *
+ *  @Return IMG_CHAR* Arena name.
+ */
+IMG_INTERNAL IMG_CHAR *
+RA_GetArenaName(RA_ARENA *pArena);
+
+IMG_INTERNAL RA_ARENA_ITERATOR *
+RA_IteratorAcquire(RA_ARENA *pArena, IMG_BOOL bIncludeFreeSegments);
+
+IMG_INTERNAL void
+RA_IteratorReset(RA_ARENA_ITERATOR *pIter);
+
+IMG_INTERNAL void
+RA_IteratorRelease(RA_ARENA_ITERATOR *pIter);
+
+/*************************************************************************/ /*!
+@Function       RA_IteratorNext
+@Description    Function used to iterate over segments in the arena. Segments
+                can either be allocated or free. Iteration can be configured
+                to include free segments when creating the iterator via
+                RA_IteratorAcquire.
+
+@Input          pIter        Iteration handle used to keep state between calls
+@InOut          pData        Data about current block iteration, base and size
+                             etc...
+@Return         IMG_BOOL     Boolean value signalling if next element exists
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNext(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData);
+
+/*************************************************************************/ /*
+@Function       RA_IteratorNextSpan
+@Description    Function used to iterate over spans in the arena. Spans
+                can either be allocated or free. Iteration can be configured
+                to include free spans when creating the iterator via
+                RA_IteratorAcquire. Spans are not allocations on the arena but
+                instead the regions of resource added that allocations can
+                be made from.
+                See RA_Add().
+
+@Input          pIter        Iteration handle used to keep state between calls
+@InOut          pData        Data about current block iteration, base and size
+                             etc...
+@Return         IMG_BOOL     Boolean value signalling if next element exists
+*/ /**************************************************************************/
+IMG_INTERNAL IMG_BOOL
+RA_IteratorNextSpan(RA_ARENA_ITERATOR *pIter, RA_ITERATOR_DATA *pData);
+
+/*************************************************************************/ /*!
+@Function       RA_BlockDump
+@Description    Debug dump of all memory allocations within the RA and the space
+                between. A '#' represents a block of memory (the arena's quantum
+                in size) that has been allocated whereas a '.' represents a free
+                block.
+@Input          pArena        The arena to dump.
+@Input          pfnLogDump    The dumping method.
+@Input          pPrivData     Data to be passed into the pfnLogDump method.
+*/ /**************************************************************************/
+IMG_INTERNAL PVRSRV_ERROR
+RA_BlockDump(RA_ARENA *pArena,
+             __printf(2, 3) void (*pfnLogDump)(void*, IMG_CHAR*, ...),
+             void *pPrivData);
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/rgx_bridge.h b/drivers/gpu/drm/img-rogue/rgx_bridge.h
new file mode 100644
index 000000000000..5168f6f23ea5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_bridge.h
@@ -0,0 +1,250 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the Rogue Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_BRIDGE_H
+#define RGX_BRIDGE_H
+
+#include "pvr_bridge.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define RGXFWINITPARAMS_VERSION   1
+#define RGXFWINITPARAMS_EXTENSION 128
+
+#include "common_rgxta3d_bridge.h"
+#include "common_rgxcmp_bridge.h"
+#if defined(SUPPORT_FASTRENDER_DM)
+#include "common_rgxtq2_bridge.h"
+#endif
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+#include "common_rgxtq_bridge.h"
+#endif
+#if defined(SUPPORT_USC_BREAKPOINT)
+#include "common_rgxbreakpoint_bridge.h"
+#endif
+#include "common_rgxfwdbg_bridge.h"
+#if defined(PDUMP)
+#include "common_rgxpdump_bridge.h"
+#endif
+#include "common_rgxhwperf_bridge.h"
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+#include "common_rgxregconfig_bridge.h"
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "common_rgxkicksync_bridge.h"
+#endif
+#include "common_rgxtimerquery_bridge.h"
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+#include "common_rgxray_bridge.h"
+#endif
+/*
+ * Bridge Cmd Ids
+ */
+
+/* *REMEMBER* to update PVRSRV_BRIDGE_RGX_LAST if you add/remove a bridge
+ * group!
+ * Also you need to ensure all PVRSRV_BRIDGE_RGX_xxx_DISPATCH_FIRST offsets
+ * follow on from the previous bridge group's commands!
+ *
+ * If a bridge group is optional, ensure you *ALWAYS* define its index
+ * (e.g. PVRSRV_BRIDGE_RGXCMP is always 151, even is the feature is not
+ * defined). If an optional bridge group is not defined you must still
+ * define PVRSRV_BRIDGE_RGX_xxx_DISPATCH_FIRST for it with an assigned
+ * value of 0.
+ */
+
+/* The RGX bridge groups start at 128 (PVRSRV_BRIDGE_RGX_FIRST) rather than
+ * follow-on from the other non-device bridge groups (meaning that they then
+ * won't be displaced if other non-device bridge groups are added).
+ */
+
+#define PVRSRV_BRIDGE_RGX_FIRST                  128UL
+
+/* 128: RGX TQ interface functions */
+#define PVRSRV_BRIDGE_RGXTQ                      128UL
+/* The RGXTQ bridge is conditional since the definitions in this header file
+ * support both the rogue and volcanic servers, but the RGXTQ bridge is not
+ * required at all on the Volcanic architecture.
+ */
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST       (PVRSRV_BRIDGE_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST        (PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTQ_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST       0
+#define PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST        (PVRSRV_BRIDGE_DISPATCH_LAST)
+#endif
+
+/* 129: RGX Compute interface functions */
+#define PVRSRV_BRIDGE_RGXCMP                     129UL
+#define PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST      (PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXCMP_CMD_LAST)
+
+/* 130: RGX TA/3D interface functions */
+#define PVRSRV_BRIDGE_RGXTA3D                    130UL
+#define PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST     (PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST      (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTA3D_CMD_LAST)
+
+/* 131: RGX Breakpoint interface functions */
+#define PVRSRV_BRIDGE_RGXBREAKPOINT                 131UL
+#if defined(SUPPORT_USC_BREAKPOINT)
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXBREAKPOINT_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST  0
+#define PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST)
+#endif
+
+/* 132: RGX Debug/Misc interface functions */
+#define PVRSRV_BRIDGE_RGXFWDBG                   132UL
+#define PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST    (PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXFWDBG_CMD_LAST)
+
+/* 133: RGX PDump interface functions */
+#define PVRSRV_BRIDGE_RGXPDUMP                   133UL
+#if defined(PDUMP)
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST    (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXPDUMP_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST    0
+#define PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST     (PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST)
+#endif
+
+/* 134: RGX HWPerf interface functions */
+#define PVRSRV_BRIDGE_RGXHWPERF                  134UL
+#define PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST   (PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXHWPERF_CMD_LAST)
+
+/* 135: RGX Register Configuration interface functions */
+#define PVRSRV_BRIDGE_RGXREGCONFIG                  135UL
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST   (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXREGCONFIG_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST   0
+#define PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST)
+#endif
+
+/* 136: RGX kicksync interface */
+#define PVRSRV_BRIDGE_RGXKICKSYNC                136UL
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST  (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXKICKSYNC_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST   0
+#define PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST    (PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST)
+#endif
+/* 137: RGX TQ2 interface */
+#define PVRSRV_BRIDGE_RGXTQ2                     137UL
+#if defined(SUPPORT_FASTRENDER_DM)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST      (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTQ2_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST      (0)
+#define PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST       (PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST)
+#endif
+
+/* 138: RGX timer query interface */
+#define PVRSRV_BRIDGE_RGXTIMERQUERY                 138UL
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXTIMERQUERY_CMD_LAST)
+
+/* 139: RGX Ray tracing interface */
+#define PVRSRV_BRIDGE_RGXRAY                 139UL
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST  (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST + 1)
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST + PVRSRV_BRIDGE_RGXRAY_CMD_LAST)
+#else
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST  0
+#define PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST   (PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST)
+#endif
+
+#define PVRSRV_BRIDGE_RGX_LAST                   (PVRSRV_BRIDGE_RGXRAY)
+#define PVRSRV_BRIDGE_RGX_DISPATCH_LAST          (PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST)
+
+/* bit mask representing the enabled RGX bridges */
+
+static const IMG_UINT32 gui32RGXBridges =
+	  (1U << (PVRSRV_BRIDGE_RGXTQ - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(RGX_FEATURE_COMPUTE) || defined(__KERNEL__)
+	| (1U << (PVRSRV_BRIDGE_RGXCMP - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXTA3D - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_BREAKPOINT)
+	| (1U << (PVRSRV_BRIDGE_BREAKPOINT - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXFWDBG - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(PDUMP)
+	| (1U << (PVRSRV_BRIDGE_RGXPDUMP - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	| (1U << (PVRSRV_BRIDGE_RGXKICKSYNC - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXHWPERF - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_REGCONFIG)
+	| (1U << (PVRSRV_BRIDGE_RGXREGCONFIG - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXKICKSYNC - PVRSRV_BRIDGE_RGX_FIRST))
+#if defined(SUPPORT_FASTRENDER_DM) || defined(__KERNEL__)
+	| (1U << (PVRSRV_BRIDGE_RGXTQ2 - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+#if defined(SUPPORT_TIMERQUERY)
+	| (1U << (PVRSRV_BRIDGE_RGXTIMERQUERY - PVRSRV_BRIDGE_RGX_FIRST))
+#endif
+	| (1U << (PVRSRV_BRIDGE_RGXRAY - PVRSRV_BRIDGE_RGX_FIRST))
+	;
+/* bit field representing which RGX bridge groups may optionally not
+ * be present in the server
+ */
+
+#define RGX_BRIDGES_OPTIONAL \
+	( \
+		0 /* no RGX bridges are currently optional */ \
+	)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_BRIDGE_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_bridge_init.c b/drivers/gpu/drm/img-rogue/rgx_bridge_init.c
new file mode 100644
index 000000000000..1de4f08dd2b6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_bridge_init.c
@@ -0,0 +1,166 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR device dependent bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements device dependent PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_bridge_init.h"
+#include "rgxdevice.h"
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+PVRSRV_ERROR InitRGXRAYBridge(void);
+void DeinitRGXRAYBridge(void);
+#endif
+
+/* Reference counts for device-conditional
+ * bridges. This ensures that bridges remain
+ * valid while there are still devices using
+ * them.
+ */
+static ATOMIC_T i32RGXCMPBridgeRefCt;
+static ATOMIC_T i32RGXTQ2BridgeRefCt;
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+static ATOMIC_T i32RGXRayBridgeRefCt;
+#endif
+static IMG_BOOL bAtomicsInitialised = IMG_FALSE;
+
+void RGXBridgeDriverInit(void)
+{
+	if (!bAtomicsInitialised)
+	{
+		bAtomicsInitialised = IMG_TRUE;
+		OSAtomicWrite(&i32RGXCMPBridgeRefCt, 0);
+		OSAtomicWrite(&i32RGXTQ2BridgeRefCt, 0);
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+		OSAtomicWrite(&i32RGXRayBridgeRefCt, 0);
+#endif
+	}
+}
+
+PVRSRV_ERROR RGXRegisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	if (!bAtomicsInitialised)
+	{
+		eError = PVRSRV_ERROR_NOT_INITIALISED;
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXBridgeRefCts");
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		if (OSAtomicIncrement(&i32RGXCMPBridgeRefCt) == 1)
+		{
+			eError = InitRGXCMPBridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXCMPBridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXCMPBridge");
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		if (OSAtomicIncrement(&i32RGXTQ2BridgeRefCt) == 1)
+		{
+			eError = InitRGXTQ2Bridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXTQ2BridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXTQ2Bridge");
+		}
+	}
+
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 0)
+	{
+		if (OSAtomicIncrement(&i32RGXRayBridgeRefCt) == 1)
+		{
+			eError = InitRGXRAYBridge();
+			if (eError != PVRSRV_OK)
+			{
+				OSAtomicDecrement(&i32RGXRayBridgeRefCt);
+			}
+			PVR_LOG_RETURN_IF_ERROR(eError, "InitRGXRAYBridge");
+		}
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+void RGXUnregisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		if (OSAtomicDecrement(&i32RGXCMPBridgeRefCt) == 0)
+		{
+			DeinitRGXCMPBridge();
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		if (OSAtomicDecrement(&i32RGXTQ2BridgeRefCt) == 0)
+		{
+			DeinitRGXTQ2Bridge();
+		}
+	}
+
+#if defined(SUPPORT_RGXRAY_BRIDGE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 0)
+	{
+		if (OSAtomicDecrement(&i32RGXRayBridgeRefCt) == 0)
+		{
+			DeinitRGXRAYBridge();
+		}
+	}
+#endif
+}
diff --git a/drivers/gpu/drm/img-rogue/rgx_bridge_init.h b/drivers/gpu/drm/img-rogue/rgx_bridge_init.h
new file mode 100644
index 000000000000..8622848b51b1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_bridge_init.h
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR device dependent bridge Init/Deinit Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements device dependent PVR Bridge init/deinit code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_BRIDGE_INIT_H)
+#define RGX_BRIDGE_INIT_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "device.h"
+#include "rgxdevice.h"
+
+void RGXBridgeDriverInit(void);
+PVRSRV_ERROR RGXRegisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo);
+void RGXUnregisterBridges(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#endif /* RGX_BRIDGE_INIT_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_common.h b/drivers/gpu/drm/img-rogue/rgx_common.h
new file mode 100644
index 000000000000..55b0ed450e1c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_common.h
@@ -0,0 +1,262 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Common Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common types and definitions for RGX software
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_COMMON_H
+#define RGX_COMMON_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+
+/* Included to get the BVNC_KM_N defined and other feature defs */
+#include "km/rgxdefs_km.h"
+
+#include "rgx_common_asserts.h"
+
+typedef struct RGXFWIF_DEV_VIRTADDR_
+{
+	IMG_UINT32	ui32Addr;
+} RGXFWIF_DEV_VIRTADDR;
+
+/* Virtualisation validation builds are meant to test the VZ-related hardware without a fully virtualised platform.
+ * As such a driver can support either the vz-validation code or real virtualisation. */
+#if defined(SUPPORT_GPUVIRT_VALIDATION) && (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+#error "Invalid build configuration: Virtualisation support (RGX_NUM_DRIVERS_SUPPORTED > 1) and virtualisation validation code (SUPPORT_GPUVIRT_VALIDATION) are mutually exclusive."
+#endif
+
+/* The RGXFWIF_DM defines assume only one of RGX_FEATURE_TLA or
+ * RGX_FEATURE_FASTRENDER_DM is present. Ensure this with a compile-time check.
+ */
+#if defined(RGX_FEATURE_TLA) && defined(RGX_FEATURE_FASTRENDER_DM)
+#error "Both RGX_FEATURE_TLA and RGX_FEATURE_FASTRENDER_DM defined. Fix code to handle this!"
+#endif
+
+#define FW_OSID							(0U)
+#define MMU_CONTEXT_MAPPING_FWPRIV		(0U) /* FW code/private data */
+
+#if (RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID) || defined(RGX_FEATURE_MIPS)
+/* The Firmware accesses its private code & data and the interface
+ * memory it shares with the KM drivers using the same MMU context */
+#define MMU_CONTEXT_MAPPING_FWIF		MMU_CONTEXT_MAPPING_FWPRIV
+#else
+/* The Firmware accesses the interface memory it shares
+ * with the KM drivers using a reserved MMU context */
+#define MMU_CONTEXT_MAPPING_FWIF		(7U)
+#endif
+
+/*! The master definition for data masters known to the firmware of RGX.
+ * When a new DM is added to this list, relevant entry should be added to
+ * RGX_HWPERF_DM enum list.
+ * The DM in a V1 HWPerf packet uses this definition. */
+
+typedef IMG_UINT32 RGXFWIF_DM;
+
+#define	RGXFWIF_DM_GP			IMG_UINT32_C(0)
+/* Either TDM or 2D DM is present. The above build time error is present to verify this */
+#define	RGXFWIF_DM_2D			IMG_UINT32_C(1) /* when RGX_FEATURE_TLA defined */
+#define	RGXFWIF_DM_TDM			IMG_UINT32_C(1) /* when RGX_FEATURE_FASTRENDER_DM defined */
+
+#define	RGXFWIF_DM_GEOM			IMG_UINT32_C(2)
+#define	RGXFWIF_DM_3D			IMG_UINT32_C(3)
+#define	RGXFWIF_DM_CDM			IMG_UINT32_C(4)
+#define	RGXFWIF_DM_RAY			IMG_UINT32_C(5)
+#define	RGXFWIF_DM_GEOM2		IMG_UINT32_C(6)
+#define	RGXFWIF_DM_GEOM3		IMG_UINT32_C(7)
+#define	RGXFWIF_DM_GEOM4		IMG_UINT32_C(8)
+
+#define	RGXFWIF_DM_LAST			RGXFWIF_DM_GEOM4
+
+typedef IMG_UINT32 RGX_KICK_TYPE_DM;
+#define RGX_KICK_TYPE_DM_GP		IMG_UINT32_C(0x001)
+#define RGX_KICK_TYPE_DM_TDM_2D	IMG_UINT32_C(0x002)
+#define RGX_KICK_TYPE_DM_TA		IMG_UINT32_C(0x004)
+#define RGX_KICK_TYPE_DM_3D		IMG_UINT32_C(0x008)
+#define RGX_KICK_TYPE_DM_CDM	IMG_UINT32_C(0x010)
+#define RGX_KICK_TYPE_DM_RTU	IMG_UINT32_C(0x020)
+#define RGX_KICK_TYPE_DM_SHG	IMG_UINT32_C(0x040)
+#define RGX_KICK_TYPE_DM_TQ2D	IMG_UINT32_C(0x080)
+#define RGX_KICK_TYPE_DM_TQ3D	IMG_UINT32_C(0x100)
+#define RGX_KICK_TYPE_DM_RAY	IMG_UINT32_C(0x200)
+#define RGX_KICK_TYPE_DM_LAST	IMG_UINT32_C(0x400)
+
+/* Maximum number of DM in use: GP, 2D/TDM, GEOM, 3D, CDM, RDM, GEOM2, GEOM3, GEOM4 */
+#define RGXFWIF_DM_MAX			(RGXFWIF_DM_LAST + 1U)
+/* The set of DMs for gathering stats on GPU utilisation excludes GP */
+#define RGXFWIF_GPU_UTIL_DM_MAX (RGXFWIF_DM_MAX - 1U)
+
+/*
+ * Data Master Tags to be appended to resources created on behalf of each RGX
+ * Context.
+ */
+#define RGX_RI_DM_TAG_KS   'K'
+#define RGX_RI_DM_TAG_CDM  'C'
+#define RGX_RI_DM_TAG_RC   'R' /* To be removed once TA/3D Timelines are split */
+#define RGX_RI_DM_TAG_TA   'V'
+#define RGX_RI_DM_TAG_GEOM 'V'
+#define RGX_RI_DM_TAG_3D   'P'
+#define RGX_RI_DM_TAG_TDM  'T'
+#define RGX_RI_DM_TAG_TQ2D '2'
+#define RGX_RI_DM_TAG_TQ3D 'Q'
+#define RGX_RI_DM_TAG_RAY  'r'
+
+/*
+ * Client API Tags to be appended to resources created on behalf of each
+ * Client API.
+ */
+#define RGX_RI_CLIENT_API_GLES1    '1'
+#define RGX_RI_CLIENT_API_GLES3    '3'
+#define RGX_RI_CLIENT_API_VULKAN   'V'
+#define RGX_RI_CLIENT_API_EGL      'E'
+#define RGX_RI_CLIENT_API_OPENCL   'C'
+#define RGX_RI_CLIENT_API_OPENGL   'G'
+#define RGX_RI_CLIENT_API_SERVICES 'S'
+#define RGX_RI_CLIENT_API_WSEGL    'W'
+#define RGX_RI_CLIENT_API_ANDROID  'A'
+#define RGX_RI_CLIENT_API_LWS      'L'
+
+/*
+ * Format a RI annotation for a given RGX Data Master context
+ */
+#define RGX_RI_FORMAT_DM_ANNOTATION(annotation, dmTag, clientAPI) do         \
+	{                                                                        \
+		(annotation)[0] = (dmTag);                                           \
+		(annotation)[1] = (clientAPI);                                       \
+		(annotation)[2] = '\0';                                              \
+	} while (false)
+
+/*!
+ ******************************************************************************
+ * RGXFW Compiler alignment definitions
+ *****************************************************************************/
+#if defined(__GNUC__) || defined(HAS_GNUC_ATTRIBUTES) || defined(INTEGRITY_OS)
+#define RGXFW_ALIGN			__attribute__ ((aligned (8)))
+#define	RGXFW_ALIGN_DCACHEL		__attribute__((aligned (64)))
+#elif defined(_MSC_VER)
+#define RGXFW_ALIGN			__declspec(align(8))
+#define	RGXFW_ALIGN_DCACHEL		__declspec(align(64))
+#pragma warning (disable : 4324)
+#else
+#error "Align MACROS need to be defined for this compiler"
+#endif
+
+/*!
+ ******************************************************************************
+ * Force 8-byte alignment for structures allocated uncached.
+ *****************************************************************************/
+#define UNCACHED_ALIGN      RGXFW_ALIGN
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation states
+ *****************************************************************************/
+#define RGXFWIF_GPU_UTIL_STATE_IDLE      (0U)
+#define RGXFWIF_GPU_UTIL_STATE_ACTIVE    (1U)
+#define RGXFWIF_GPU_UTIL_STATE_BLOCKED   (2U)
+#define RGXFWIF_GPU_UTIL_STATE_NUM       (3U)
+/* the state below "combines" IDLE and BLOCKED
+ * and is used when we only care about GPU being in ACTIVE or not */
+#define RGXFWIF_GPU_UTIL_STATE_INACTIVE  (0U)
+/* when we combine IDLE and BLOCKED we end up with one state less */
+#define RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM       (RGXFWIF_GPU_UTIL_STATE_NUM-1U)
+#define RGXFWIF_GPU_UTIL_STATE_MASK      IMG_UINT64_C(0x0000000000000003)
+#define RGXFWIF_GPU_UTIL_STATE_MASK32    IMG_UINT32_C(0x00000003)
+
+
+/*
+ * Maximum amount of register writes that can be done by the register
+ * programmer (FW or META DMA). This is not a HW limitation, it is only
+ * a protection against malformed inputs to the register programmer.
+ */
+#define RGX_MAX_NUM_REGISTER_PROGRAMMER_WRITES  (256U)
+
+/* FW common context priority. */
+/*!
+ * @AddToGroup WorkloadContexts
+ * @{
+ */
+#define RGX_CTX_PRIORITY_REALTIME  (INT32_MAX)
+#define RGX_CTX_PRIORITY_HIGH      (2) /*!< HIGH priority */
+#define RGX_CTX_PRIORITY_MEDIUM    (1) /*!< MEDIUM priority */
+#define RGX_CTX_PRIORITY_LOW       (0) /*!< LOW priority */
+/*!
+ * @} End of AddToGroup WorkloadContexts
+ */
+
+
+/*
+ *   Use of the 32-bit context property flags mask
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *                                   0
+ *                                   |
+ *    -------------------------------x
+ */
+/*
+ * Context creation flags
+ * (specify a context's properties at creation time)
+ */
+#define RGX_CONTEXT_FLAG_DISABLESLR					(1UL << 0) /*!< Disable SLR */
+
+/* Bitmask of context flags allowed to be modified after context create. */
+#define RGX_CONTEXT_FLAGS_WRITEABLE_MASK            (RGX_CONTEXT_FLAG_DISABLESLR)
+
+/* List of attributes that may be set for a context */
+typedef IMG_UINT32 RGX_CONTEXT_PROPERTY;
+#define RGX_CONTEXT_PROPERTY_FLAGS 0U /*!< Context flags */
+
+/* MMU4 supported number of ranges */
+#define RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES (4U)
+#define RGX_MMU_RANGE_NON4KHEAP          (0U)
+#define RGX_MMU_RANGE_GLOBAL             (RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES - 1)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_COMMON_H */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_common_asserts.h b/drivers/gpu/drm/img-rogue/rgx_common_asserts.h
new file mode 100644
index 000000000000..c571cc6f008e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_common_asserts.h
@@ -0,0 +1,73 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Common Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common types and definitions for RGX software
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_COMMON_ASSERTS_H
+#define RGX_COMMON_ASSERTS_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*! This macro represents a mask of LSBs that must be zero on data structure
+ * sizes and offsets to ensure they are 8-byte granular on types shared between
+ * the FW and host driver */
+#define RGX_FW_ALIGNMENT_LSB (7U)
+
+/*! Macro to test structure size alignment */
+#define RGX_FW_STRUCT_SIZE_ASSERT(_a)	\
+	static_assert((sizeof(_a) & RGX_FW_ALIGNMENT_LSB) == 0U,	\
+				  "Size of " #_a " is not properly aligned")
+
+/*! Macro to test structure member alignment */
+#define RGX_FW_STRUCT_OFFSET_ASSERT(_a, _b)	\
+	static_assert((offsetof(_a, _b) & RGX_FW_ALIGNMENT_LSB) == 0U,	\
+				  "Offset of " #_a "." #_b " is not properly aligned")
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_COMMON_ASSERTS_H */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_compat_bvnc.h b/drivers/gpu/drm/img-rogue/rgx_compat_bvnc.h
new file mode 100644
index 000000000000..c3e1333cdb0f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_compat_bvnc.h
@@ -0,0 +1,140 @@
+/*************************************************************************/ /*!
+@File           rgx_compat_bvnc.h
+@Title          BVNC compatibility check utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used for packing BNC and V.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_COMPAT_BVNC_H)
+#define RGX_COMPAT_BVNC_H
+
+#include "img_types.h"
+
+#if defined(RGX_FIRMWARE)               /* Services firmware */
+# include "rgxfw_utils.h"
+# define PVR_COMPAT_ASSERT RGXFW_ASSERT
+#elif !defined(RGX_BUILD_BINARY)        /* Services host driver code */
+# include "pvr_debug.h"
+# define PVR_COMPAT_ASSERT PVR_ASSERT
+#else                                   /* FW user-mode tools */
+# include <assert.h>
+# define PVR_COMPAT_ASSERT assert
+#endif
+
+/* 64bit endian conversion macros */
+#if defined(__BIG_ENDIAN__)
+#define RGX_INT64_TO_BE(N) (N)
+#define RGX_INT64_FROM_BE(N) (N)
+#define RGX_INT32_TO_BE(N) (N)
+#define RGX_INT32_FROM_BE(N) (N)
+#else
+#define RGX_INT64_TO_BE(N)        \
+	((((N) >> 56)   & 0xff)       \
+	 | (((N) >> 40) & 0xff00)     \
+	 | (((N) >> 24) & 0xff0000)   \
+	 | (((N) >> 8)  & 0xff000000U) \
+	 | ((N)                << 56) \
+	 | (((N) & 0xff00)     << 40) \
+	 | (((N) & 0xff0000)   << 24) \
+	 | (((N) & 0xff000000U) << 8))
+#define RGX_INT64_FROM_BE(N) RGX_INT64_TO_BE(N)
+
+#define RGX_INT32_TO_BE(N)   \
+	((((N) >> 24)  & 0xff)   \
+	 | (((N) >> 8) & 0xff00) \
+	 | ((N)           << 24) \
+	 | ((((N) & 0xff00) << 8)))
+#define RGX_INT32_FROM_BE(N) RGX_INT32_TO_BE(N)
+#endif
+
+/******************************************************************************
+ * RGX Version packed into 64-bit (BVNC) to be used by Compatibility Check
+ *****************************************************************************/
+
+#define RGX_BVNC_PACK_SHIFT_B 48
+#define RGX_BVNC_PACK_SHIFT_V 32
+#define RGX_BVNC_PACK_SHIFT_N 16
+#define RGX_BVNC_PACK_SHIFT_C 0
+
+#define RGX_BVNC_PACK_MASK_B (IMG_UINT64_C(0xFFFF000000000000))
+#define RGX_BVNC_PACK_MASK_V (IMG_UINT64_C(0x0000FFFF00000000))
+#define RGX_BVNC_PACK_MASK_N (IMG_UINT64_C(0x00000000FFFF0000))
+#define RGX_BVNC_PACK_MASK_C (IMG_UINT64_C(0x000000000000FFFF))
+
+#define RGX_BVNC_PACKED_EXTR_B(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_B) >> RGX_BVNC_PACK_SHIFT_B))
+#define RGX_BVNC_PACKED_EXTR_V(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_V) >> RGX_BVNC_PACK_SHIFT_V))
+#define RGX_BVNC_PACKED_EXTR_N(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_N) >> RGX_BVNC_PACK_SHIFT_N))
+#define RGX_BVNC_PACKED_EXTR_C(BVNC) ((IMG_UINT32)(((BVNC) & RGX_BVNC_PACK_MASK_C) >> RGX_BVNC_PACK_SHIFT_C))
+
+#define RGX_BVNC_EQUAL(L,R,all,version,bvnc) do {															\
+										(bvnc) = IMG_FALSE;													\
+										(version) = ((L).ui32LayoutVersion == (R).ui32LayoutVersion);		\
+										if (version)														\
+										{																	\
+											(bvnc) = ((L).ui64BVNC == (R).ui64BVNC);						\
+										}																	\
+										(all) = (version) && (bvnc);										\
+									} while (false)
+
+
+/**************************************************************************//**
+ * Utility function for packing BVNC
+ *****************************************************************************/
+static inline IMG_UINT64 rgx_bvnc_pack(IMG_UINT32 ui32B, IMG_UINT32 ui32V, IMG_UINT32 ui32N, IMG_UINT32 ui32C)
+{
+	/*
+	 * Test for input B, V, N and C exceeding max bit width.
+	 */
+	PVR_COMPAT_ASSERT((ui32B & (~(RGX_BVNC_PACK_MASK_B >> RGX_BVNC_PACK_SHIFT_B))) == 0U);
+	PVR_COMPAT_ASSERT((ui32V & (~(RGX_BVNC_PACK_MASK_V >> RGX_BVNC_PACK_SHIFT_V))) == 0U);
+	PVR_COMPAT_ASSERT((ui32N & (~(RGX_BVNC_PACK_MASK_N >> RGX_BVNC_PACK_SHIFT_N))) == 0U);
+	PVR_COMPAT_ASSERT((ui32C & (~(RGX_BVNC_PACK_MASK_C >> RGX_BVNC_PACK_SHIFT_C))) == 0U);
+
+	return (((IMG_UINT64)ui32B << RGX_BVNC_PACK_SHIFT_B) |
+			((IMG_UINT64)ui32V << RGX_BVNC_PACK_SHIFT_V) |
+			((IMG_UINT64)ui32N << RGX_BVNC_PACK_SHIFT_N) |
+			((IMG_UINT64)ui32C << RGX_BVNC_PACK_SHIFT_C));
+}
+
+
+#endif /* RGX_COMPAT_BVNC_H */
+
+/******************************************************************************
+ End of file (rgx_compat_bvnc.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_firmware_boot.h b/drivers/gpu/drm/img-rogue/rgx_firmware_boot.h
new file mode 100644
index 000000000000..b0a14683e1a3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_firmware_boot.h
@@ -0,0 +1,91 @@
+/**************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef RGX_FIRMWARE_BOOT_H
+#define RGX_FIRMWARE_BOOT_H
+
+#include "img_types.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+
+#define TD_MAX_NUM_MIPS_PAGETABLE_PAGES (4U)
+
+typedef union _PVRSRV_FW_BOOT_PARAMS_
+{
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCodeDevVAddr;
+		IMG_DEV_VIRTADDR sFWDataDevVAddr;
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_UINT32 ui32NumThreads;
+	} sMeta;
+
+	struct
+	{
+		IMG_DEV_PHYADDR sGPURegAddr;
+		IMG_DEV_PHYADDR asFWPageTableAddr[TD_MAX_NUM_MIPS_PAGETABLE_PAGES];
+		IMG_DEV_PHYADDR sFWStackAddr;
+		IMG_UINT32 ui32FWPageTableLog2PageSize;
+		IMG_UINT32 ui32FWPageTableNumPages;
+	} sMips;
+
+	struct
+	{
+		IMG_DEV_VIRTADDR sFWCodeDevVAddr;
+		IMG_DEV_VIRTADDR sFWDataDevVAddr;
+
+		IMG_DEV_VIRTADDR sFWCorememCodeDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememCodeFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememCodeSize;
+
+		IMG_DEV_VIRTADDR sFWCorememDataDevVAddr;
+		RGXFWIF_DEV_VIRTADDR sFWCorememDataFWAddr;
+		IMG_DEVMEM_SIZE_T uiFWCorememDataSize;
+	} sRISCV;
+
+} PVRSRV_FW_BOOT_PARAMS;
+
+
+#endif /* RGX_FIRMWARE_BOOT_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_fw_info.h b/drivers/gpu/drm/img-rogue/rgx_fw_info.h
new file mode 100644
index 000000000000..fb31ac440024
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_fw_info.h
@@ -0,0 +1,144 @@
+/*************************************************************************/ /*!
+@File
+@Title          FW image information
+
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FW_INFO_H)
+#define RGX_FW_INFO_H
+
+#include "img_types.h"
+#include "rgx_common.h"
+
+/*
+ * Firmware binary block unit in bytes.
+ * Raw data stored in FW binary will be aligned to this size.
+ */
+#define FW_BLOCK_SIZE 4096L
+
+typedef enum
+{
+	META_CODE = 0,
+	META_PRIVATE_DATA,
+	META_COREMEM_CODE,
+	META_COREMEM_DATA,
+	MIPS_CODE,
+	MIPS_EXCEPTIONS_CODE,
+	MIPS_BOOT_CODE,
+	MIPS_PRIVATE_DATA,
+	MIPS_BOOT_DATA,
+	MIPS_STACK,
+	RISCV_UNCACHED_CODE,
+	RISCV_CACHED_CODE,
+	RISCV_PRIVATE_DATA,
+	RISCV_COREMEM_CODE,
+	RISCV_COREMEM_DATA,
+} RGX_FW_SECTION_ID;
+
+typedef enum
+{
+	NONE = 0,
+	FW_CODE,
+	FW_DATA,
+	FW_COREMEM_CODE,
+	FW_COREMEM_DATA
+} RGX_FW_SECTION_TYPE;
+
+
+/*
+ * FW binary format with FW info attached:
+ *
+ *          Contents        Offset
+ *     +-----------------+
+ *     |                 |    0
+ *     |                 |
+ *     | Original binary |
+ *     |      file       |
+ *     |   (.ldr/.elf)   |
+ *     |                 |
+ *     |                 |
+ *     +-----------------+
+ *     | FW info header  |  FILE_SIZE - 4K
+ *     +-----------------+
+ *     |                 |
+ *     | FW layout table |
+ *     |                 |
+ *     +-----------------+
+ *                          FILE_SIZE
+ */
+
+#define FW_INFO_VERSION  (2)
+
+/* Firmware is built for open source driver and uses open source version numbering */
+#define FW_INFO_FLAGS_OPEN_SOURCE (1U)
+
+typedef struct
+{
+	/* FW_INFO_VERSION 1 */
+	IMG_UINT32 ui32InfoVersion;      /* FW info version */
+	IMG_UINT32 ui32HeaderLen;        /* Header length */
+	IMG_UINT32 ui32LayoutEntryNum;   /* Number of entries in the layout table */
+	IMG_UINT32 ui32LayoutEntrySize;  /* Size of an entry in the layout table */
+	IMG_UINT64 RGXFW_ALIGN ui64BVNC; /* BVNC */
+	IMG_UINT32 ui32FwPageSize;       /* Page size of processor on which firmware executes */
+	IMG_UINT32 ui32Flags;            /* Compatibility flags */
+
+	/* FW_INFO_VERSION 2 */
+	IMG_UINT16 ui16PVRVersionMajor;  /* DDK major version number */
+	IMG_UINT16 ui16PVRVersionMinor;  /* DDK minor version number */
+	IMG_UINT32 ui32PVRVersionBuild;  /* DDK build number */
+} RGX_FW_INFO_HEADER;
+
+typedef struct
+{
+	RGX_FW_SECTION_ID eId;
+	RGX_FW_SECTION_TYPE eType;
+	IMG_UINT32 ui32BaseAddr;
+	IMG_UINT32 ui32MaxSize;
+	IMG_UINT32 ui32AllocSize;
+	IMG_UINT32 ui32AllocOffset;
+} RGX_FW_LAYOUT_ENTRY;
+
+#endif /* RGX_FW_INFO_H */
+
+/******************************************************************************
+ End of file (rgx_fw_info.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_fwif_alignchecks.h b/drivers/gpu/drm/img-rogue/rgx_fwif_alignchecks.h
new file mode 100644
index 000000000000..a33b6d541c65
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_fwif_alignchecks.h
@@ -0,0 +1,192 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX fw interface alignment checks
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Checks to avoid disalignment in RGX fw data structures
+                shared with the host
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_ALIGNCHECKS_H)
+#define RGX_FWIF_ALIGNCHECKS_H
+
+/* for the offsetof macro */
+#if defined(__KERNEL__) && defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+/*!
+ ******************************************************************************
+ * Alignment UM/FW checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_UM_MAX 128U
+
+#define RGXFW_ALIGN_CHECKS_INIT0								\
+		(IMG_UINT32)sizeof(RGXFWIF_TRACEBUF),								\
+		offsetof(RGXFWIF_TRACEBUF, ui32LogType),				\
+		offsetof(RGXFWIF_TRACEBUF, sTraceBuf),					\
+		offsetof(RGXFWIF_TRACEBUF, ui32TraceBufSizeInDWords),	\
+		offsetof(RGXFWIF_TRACEBUF, ui32TracebufFlags),			\
+																\
+		(IMG_UINT32)sizeof(RGXFWIF_SYSDATA),								\
+		offsetof(RGXFWIF_SYSDATA, ePowState),					\
+		offsetof(RGXFWIF_SYSDATA, ui32HWPerfDropCount),			\
+		offsetof(RGXFWIF_SYSDATA, ui32LastDropOrdinal),			\
+		offsetof(RGXFWIF_SYSDATA, ui32FWFaults),				\
+		offsetof(RGXFWIF_SYSDATA, ui32HWRStateFlags),			\
+																\
+		(IMG_UINT32)sizeof(RGXFWIF_OSDATA),									\
+		offsetof(RGXFWIF_OSDATA, ui32HostSyncCheckMark),		\
+		offsetof(RGXFWIF_OSDATA, ui32KCCBCmdsExecuted),			\
+																	\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRINFOBUF),									\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmLockedUpCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmOverranCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmRecoveredCount),		\
+		offsetof(RGXFWIF_HWRINFOBUF, aui32HwrDmFalseDetectCount),	\
+																	\
+		/* RGXFWIF_CMDTA checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTA),							\
+		offsetof(RGXFWIF_CMDTA, sGeomRegs),				\
+														\
+		/* RGXFWIF_CMD3D checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_CMD3D),							\
+		offsetof(RGXFWIF_CMD3D, s3DRegs),				\
+														\
+		/* RGXFWIF_CMDTRANSFER checks */				\
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTRANSFER),					\
+		offsetof(RGXFWIF_CMDTRANSFER, sTransRegs),		\
+														\
+														\
+		/* RGXFWIF_CMD_COMPUTE checks */				\
+		(IMG_UINT32)sizeof(RGXFWIF_CMD_COMPUTE),					\
+		offsetof(RGXFWIF_CMD_COMPUTE, sCDMRegs),		\
+														\
+		/* RGXFWIF_FREELIST checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_FREELIST),						\
+		offsetof(RGXFWIF_FREELIST, psFreeListDevVAddr),	\
+		offsetof(RGXFWIF_FREELIST, ui32MaxPages),		\
+		offsetof(RGXFWIF_FREELIST, ui32CurrentPages),	\
+														\
+		/* RGXFWIF_HWRTDATA checks */						\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRTDATA),							\
+		offsetof(RGXFWIF_HWRTDATA, psVHeapTableDevVAddr),	\
+		offsetof(RGXFWIF_HWRTDATA, psPMMListDevVAddr),		\
+		offsetof(RGXFWIF_HWRTDATA, apsFreeLists),			\
+		offsetof(RGXFWIF_HWRTDATA, ui64VCECatBase),			\
+		offsetof(RGXFWIF_HWRTDATA, eState),					\
+															\
+		/* RGXFWIF_HWRTDATA_COMMON checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_HWRTDATA_COMMON),						\
+		offsetof(RGXFWIF_HWRTDATA_COMMON, bTACachesNeedZeroing),\
+																\
+		/* RGXFWIF_HWPERF_CTL_BLK checks */					\
+		(IMG_UINT32)sizeof(RGXFWIF_HWPERF_CTL_BLK),						\
+		offsetof(RGXFWIF_HWPERF_CTL_BLK, aui64CounterCfg),	\
+															\
+		/* RGXFWIF_HWPERF_CTL checks */			\
+		(IMG_UINT32)sizeof(RGXFWIF_HWPERF_CTL),				\
+		offsetof(RGXFWIF_HWPERF_CTL, SelCntr)
+
+#if defined(RGX_FEATURE_TLA)
+#define RGXFW_ALIGN_CHECKS_INIT1                   \
+		RGXFW_ALIGN_CHECKS_INIT0,                  \
+		/* RGXFWIF_CMD2D checks */                 \
+		(IMG_UINT32)sizeof(RGXFWIF_CMD2D),                     \
+		offsetof(RGXFWIF_CMD2D, s2DRegs)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT1		RGXFW_ALIGN_CHECKS_INIT0
+#endif	/* RGX_FEATURE_TLA */
+
+
+#if defined(RGX_FEATURE_FASTRENDER_DM)
+#define RGXFW_ALIGN_CHECKS_INIT                    \
+		RGXFW_ALIGN_CHECKS_INIT1,                  \
+		/* RGXFWIF_CMDTDM checks */                \
+		(IMG_UINT32)sizeof(RGXFWIF_CMDTDM),                    \
+		offsetof(RGXFWIF_CMDTDM, sTDMRegs)
+#else
+#define RGXFW_ALIGN_CHECKS_INIT		RGXFW_ALIGN_CHECKS_INIT1
+#endif /* ! RGX_FEATURE_FASTRENDER_DM */
+
+
+
+/*!
+ ******************************************************************************
+ * Alignment KM checks array
+ *****************************************************************************/
+
+#define RGXFW_ALIGN_CHECKS_INIT_KM                                   \
+		(IMG_UINT32)sizeof(RGXFWIF_SYSINIT),                                     \
+		offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr),                   \
+		offsetof(RGXFWIF_SYSINIT, sPDSExecBase),                     \
+		offsetof(RGXFWIF_SYSINIT, sUSCExecBase),                     \
+		offsetof(RGXFWIF_SYSINIT, aui32TPUTrilinearFracMask),        \
+		offsetof(RGXFWIF_SYSINIT, sTraceBufCtl),                     \
+		offsetof(RGXFWIF_SYSINIT, sFwSysData),                       \
+		(IMG_UINT32)sizeof(RGXFWIF_OSINIT),                                      \
+		offsetof(RGXFWIF_OSINIT, psKernelCCBCtl),                    \
+		offsetof(RGXFWIF_OSINIT, psKernelCCB),                       \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCBCtl),                  \
+		offsetof(RGXFWIF_OSINIT, psFirmwareCCB),                     \
+		offsetof(RGXFWIF_OSINIT, sFwOsData),                         \
+		offsetof(RGXFWIF_OSINIT, sRGXCompChecks),                    \
+		                                                             \
+		/* RGXFWIF_FWRENDERCONTEXT checks */                         \
+		(IMG_UINT32)sizeof(RGXFWIF_FWRENDERCONTEXT),                             \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),               \
+		offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),               \
+		                                                             \
+		(IMG_UINT32)sizeof(RGXFWIF_FWCOMMONCONTEXT),                             \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psFWMemContext),           \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, sRunNode),                 \
+		offsetof(RGXFWIF_FWCOMMONCONTEXT, psCCB),                    \
+		                                                             \
+		(IMG_UINT32)sizeof(RGXFWIF_MMUCACHEDATA),                                \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32CacheFlags),              \
+		offsetof(RGXFWIF_MMUCACHEDATA, sMMUCacheSync),               \
+		offsetof(RGXFWIF_MMUCACHEDATA, ui32MMUCacheSyncUpdateValue)
+
+#endif /* RGX_FWIF_ALIGNCHECKS_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_alignchecks.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_fwif_hwperf.h b/drivers/gpu/drm/img-rogue/rgx_fwif_hwperf.h
new file mode 100644
index 000000000000..ec2d9bb071e7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_fwif_hwperf.h
@@ -0,0 +1,253 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_hwperf.h
+@Title          RGX HWPerf support
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared header between RGX firmware and Init process
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_HWPERF_H
+#define RGX_FWIF_HWPERF_H
+
+#include "rgx_fwif_shared.h"
+#include "rgx_hwperf.h"
+#include "rgxdefs_km.h"
+
+
+/*****************************************************************************/
+
+/* Structure to hold a block's parameters for passing between the BG context
+ * and the IRQ context when applying a configuration request. */
+typedef struct
+{
+	/* Few members could be booleans but padded to IMG_UINT32
+	 * to workaround pdump alignment requirements */
+	IMG_UINT32              ui32Valid;
+	IMG_UINT32              ui32Enabled;
+	IMG_UINT32              eBlockID;
+	IMG_UINT32              uiCounterMask;
+	IMG_UINT64  RGXFW_ALIGN aui64CounterCfg[RGX_CNTBLK_MUX_COUNTERS_MAX];
+}  RGXFWIF_HWPERF_CTL_BLK;
+
+/* Structure used to hold the configuration of the non-mux counters blocks */
+typedef struct
+{
+	IMG_UINT32            ui32NumSelectedCounters;
+	IMG_UINT32            aui32SelectedCountersIDs[RGX_HWPERF_MAX_CUSTOM_CNTRS];
+} RGXFW_HWPERF_SELECT;
+
+/* Structure used to hold a Direct-Addressable block's parameters for passing
+ * between the BG context and the IRQ context when applying a configuration
+ * request. HWPERF_UNIFIED use only.
+ */
+typedef struct
+{
+	IMG_UINT32               uiEnabled;
+	IMG_UINT32               uiNumCounters;
+	IMG_UINT32               eBlockID;
+	RGXFWIF_DEV_VIRTADDR     psModel;
+	IMG_UINT32               aui32Counters[RGX_CNTBLK_COUNTERS_MAX];
+} RGXFWIF_HWPERF_DA_BLK;
+
+
+/* Structure to hold the whole configuration request details for all blocks
+ * The block masks and counts are used to optimise reading of this data. */
+typedef struct
+{
+	IMG_UINT32                         ui32HWPerfCtlFlags;
+
+	IMG_UINT32                         ui32SelectedCountersBlockMask;
+	RGXFW_HWPERF_SELECT RGXFW_ALIGN    SelCntr[RGX_HWPERF_MAX_CUSTOM_BLKS];
+
+	IMG_UINT32                         ui32EnabledMUXBlksCount;
+	RGXFWIF_HWPERF_CTL_BLK RGXFW_ALIGN sBlkCfg[RGX_HWPERF_MAX_MUX_BLKS];
+} UNCACHED_ALIGN RGXFWIF_HWPERF_CTL;
+
+/* NOTE: The switch statement in this function must be kept in alignment with
+ * the enumeration RGX_HWPERF_CNTBLK_ID defined in rgx_hwperf.h. ASSERTs may
+ * result if not.
+ * The function provides a hash lookup to get a handle on the global store for
+ * a block's configuration store from it's block ID.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(rgxfw_hwperf_get_block_ctl)
+#endif
+static INLINE RGXFWIF_HWPERF_CTL_BLK *rgxfw_hwperf_get_block_ctl(
+		RGX_HWPERF_CNTBLK_ID eBlockID, RGXFWIF_HWPERF_CTL *psHWPerfInitData)
+{
+	IMG_UINT32 ui32Idx;
+
+	/* Hash the block ID into a control configuration array index */
+	switch (eBlockID)
+	{
+		case RGX_CNTBLK_ID_TA:
+		case RGX_CNTBLK_ID_RASTER:
+		case RGX_CNTBLK_ID_HUB:
+		case RGX_CNTBLK_ID_TORNADO:
+		case RGX_CNTBLK_ID_JONES:
+		{
+			ui32Idx = eBlockID;
+			break;
+		}
+		case RGX_CNTBLK_ID_TPU_MCU0:
+		case RGX_CNTBLK_ID_TPU_MCU1:
+		case RGX_CNTBLK_ID_TPU_MCU2:
+		case RGX_CNTBLK_ID_TPU_MCU3:
+		case RGX_CNTBLK_ID_TPU_MCU4:
+		case RGX_CNTBLK_ID_TPU_MCU5:
+		case RGX_CNTBLK_ID_TPU_MCU6:
+		case RGX_CNTBLK_ID_TPU_MCU7:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_USC0:
+		case RGX_CNTBLK_ID_USC1:
+		case RGX_CNTBLK_ID_USC2:
+		case RGX_CNTBLK_ID_USC3:
+		case RGX_CNTBLK_ID_USC4:
+		case RGX_CNTBLK_ID_USC5:
+		case RGX_CNTBLK_ID_USC6:
+		case RGX_CNTBLK_ID_USC7:
+		case RGX_CNTBLK_ID_USC8:
+		case RGX_CNTBLK_ID_USC9:
+		case RGX_CNTBLK_ID_USC10:
+		case RGX_CNTBLK_ID_USC11:
+		case RGX_CNTBLK_ID_USC12:
+		case RGX_CNTBLK_ID_USC13:
+		case RGX_CNTBLK_ID_USC14:
+		case RGX_CNTBLK_ID_USC15:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_TEXAS0:
+		case RGX_CNTBLK_ID_TEXAS1:
+		case RGX_CNTBLK_ID_TEXAS2:
+		case RGX_CNTBLK_ID_TEXAS3:
+		case RGX_CNTBLK_ID_TEXAS4:
+		case RGX_CNTBLK_ID_TEXAS5:
+		case RGX_CNTBLK_ID_TEXAS6:
+		case RGX_CNTBLK_ID_TEXAS7:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_RASTER0:
+		case RGX_CNTBLK_ID_RASTER1:
+		case RGX_CNTBLK_ID_RASTER2:
+		case RGX_CNTBLK_ID_RASTER3:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_BLACKPEARL0:
+		case RGX_CNTBLK_ID_BLACKPEARL1:
+		case RGX_CNTBLK_ID_BLACKPEARL2:
+		case RGX_CNTBLK_ID_BLACKPEARL3:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(RASTER, 3) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		case RGX_CNTBLK_ID_PBE0:
+		case RGX_CNTBLK_ID_PBE1:
+		case RGX_CNTBLK_ID_PBE2:
+		case RGX_CNTBLK_ID_PBE3:
+		case RGX_CNTBLK_ID_PBE4:
+		case RGX_CNTBLK_ID_PBE5:
+		case RGX_CNTBLK_ID_PBE6:
+		case RGX_CNTBLK_ID_PBE7:
+		case RGX_CNTBLK_ID_PBE8:
+		case RGX_CNTBLK_ID_PBE9:
+		case RGX_CNTBLK_ID_PBE10:
+		case RGX_CNTBLK_ID_PBE11:
+		case RGX_CNTBLK_ID_PBE12:
+		case RGX_CNTBLK_ID_PBE13:
+		case RGX_CNTBLK_ID_PBE14:
+		case RGX_CNTBLK_ID_PBE15:
+		{
+			ui32Idx = RGX_CNTBLK_ID_DIRECT_LAST +
+						RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(USC, 15) +
+						RGX_CNTBLK_INDIRECT_COUNT(TEXAS, 7) +
+						RGX_CNTBLK_INDIRECT_COUNT(RASTER, 3) +
+						RGX_CNTBLK_INDIRECT_COUNT(BLACKPEARL, 3) +
+						(eBlockID & RGX_CNTBLK_ID_UNIT_MASK);
+			break;
+		}
+		default:
+		{
+			ui32Idx = RGX_HWPERF_MAX_DEFINED_BLKS;
+			break;
+		}
+	}
+	if (ui32Idx >= RGX_HWPERF_MAX_DEFINED_BLKS)
+	{
+		return NULL;
+	}
+	return &psHWPerfInitData->sBlkCfg[ui32Idx];
+}
+
+/* Stub routine for rgxfw_hwperf_get_da_block_ctl(). Just return a NULL.
+ */
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(rgxfw_hwperf_get_da_block_ctl)
+#endif
+static INLINE RGXFWIF_HWPERF_DA_BLK* rgxfw_hwperf_get_da_block_ctl(
+		RGX_HWPERF_CNTBLK_ID eBlockID, RGXFWIF_HWPERF_CTL *psHWPerfInitData)
+{
+	PVR_UNREFERENCED_PARAMETER(eBlockID);
+	PVR_UNREFERENCED_PARAMETER(psHWPerfInitData);
+
+	return NULL;
+}
+#endif
diff --git a/drivers/gpu/drm/img-rogue/rgx_fwif_km.h b/drivers/gpu/drm/img-rogue/rgx_fwif_km.h
new file mode 100644
index 000000000000..22da7ec4cc9d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_fwif_km.h
@@ -0,0 +1,1888 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures used by pvrsrvkm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures used by pvrsrvkm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_KM_H)
+#define RGX_FWIF_KM_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+#include "rgxdefs_km.h"
+#include "dllist.h"
+#include "rgx_hwperf.h"
+#include "rgx_mips.h"
+#include "rgxheapconfig.h"
+
+/************************************************************************
+* RGX FW signature checks
+************************************************************************/
+#define RGXFW_SIG_BUFFER_SIZE_MIN       (8192)
+
+#define RGXFWIF_TIMEDIFF_ID			((0x1UL << 28) | RGX_CR_TIMER)
+
+#define RGXFW_POLL_TYPE_SET 0x80000000U
+
+/* Firmware per-DM HWR states */
+#define RGXFWIF_DM_STATE_WORKING					(0x00U)		/*!< DM is working if all flags are cleared */
+#define RGXFWIF_DM_STATE_READY_FOR_HWR				(IMG_UINT32_C(0x1) << 0)	/*!< DM is idle and ready for HWR */
+#define RGXFWIF_DM_STATE_NEEDS_SKIP					(IMG_UINT32_C(0x1) << 2)	/*!< DM need to skip to next cmd before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP			(IMG_UINT32_C(0x1) << 3)	/*!< DM need partial render cleanup before resuming processing */
+#define RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR			(IMG_UINT32_C(0x1) << 4)	/*!< DM need to increment Recovery Count once fully recovered */
+#define RGXFWIF_DM_STATE_GUILTY_LOCKUP				(IMG_UINT32_C(0x1) << 5)	/*!< DM was identified as locking up and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_LOCKUP			(IMG_UINT32_C(0x1) << 6)	/*!< DM was innocently affected by another lockup which caused HWR */
+#define RGXFWIF_DM_STATE_GUILTY_OVERRUNING			(IMG_UINT32_C(0x1) << 7)	/*!< DM was identified as over-running and causing HWR */
+#define RGXFWIF_DM_STATE_INNOCENT_OVERRUNING		(IMG_UINT32_C(0x1) << 8)	/*!< DM was innocently affected by another DM over-running which caused HWR */
+#define RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH		(IMG_UINT32_C(0x1) << 9)	/*!< DM was forced into HWR as it delayed more important workloads */
+#define RGXFWIF_DM_STATE_GPU_ECC_HWR				(IMG_UINT32_C(0x1) << 10)	/*!< DM was forced into HWR due to an uncorrected GPU ECC error */
+#define RGXFWIF_DM_STATE_GPU_PARITY_HWR				(IMG_UINT32_C(0x1) << 11)	/*!< DM was forced into HWR due to an uncorrected GPU PARITY error */
+#define RGXFWIF_DM_STATE_GPU_LATENT_HWR				(IMG_UINT32_C(0x1) << 12)	/*!< DM was forced into HWR due to an uncorrected GPU LATENT error */
+
+
+/* Firmware's connection state */
+typedef IMG_UINT32 RGXFWIF_CONNECTION_FW_STATE;
+#define RGXFW_CONNECTION_FW_OFFLINE				0U	/*!< Firmware is offline */
+#define RGXFW_CONNECTION_FW_READY				1U	/*!< Firmware is initialised */
+#define RGXFW_CONNECTION_FW_ACTIVE				2U	/*!< Firmware connection is fully established */
+#define RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD	3U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_FORCED_OFFLOAD		4U	/*!< Firmware is clearing up connection data */
+#define RGXFW_CONNECTION_FW_COOLDOWN			5U	/*!< Firmware connection is in cooldown period */
+#define RGXFW_CONNECTION_FW_STATE_COUNT			6U
+
+#define RGXFW_FORCED_OFFLOAD_HCS_DEADLINE_MS	2U		/*!< Workloads of Guest being forcefully offloaded must be stopped quickly */
+#define RGXFW_GRACEFUL_OFFLOAD_HCS_DEADLINE_MS	1000U	/*!< Workloads of Guest being gracefully offloaded are allowed more time to finish */
+
+/* OS' connection state */
+typedef enum
+{
+	RGXFW_CONNECTION_OS_OFFLINE = 0,	/*!< OS is offline */
+	RGXFW_CONNECTION_OS_READY,			/*!< OS's KM driver is setup and waiting */
+	RGXFW_CONNECTION_OS_ACTIVE,			/*!< OS connection is fully established */
+	RGXFW_CONNECTION_OS_STATE_COUNT
+} RGXFWIF_CONNECTION_OS_STATE;
+
+#define RGXFWIF_CTXSWITCH_PROFILE_FAST_EN		(IMG_UINT32_C(0x1))
+#define RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN		(IMG_UINT32_C(0x2))
+#define RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN		(IMG_UINT32_C(0x3))
+#define RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN	(IMG_UINT32_C(0x4))
+
+#define RGXFWIF_ISP_SCHEDMODE_VER1_IPP			(IMG_UINT32_C(0x1))
+#define RGXFWIF_ISP_SCHEDMODE_VER2_ISP			(IMG_UINT32_C(0x2))
+/*!
+ ******************************************************************************
+ * RGX firmware Init Config Data
+ *****************************************************************************/
+
+/* Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_CTXSWITCH_MODE_RAND				(IMG_UINT32_C(0x1) << 0)	/*!< Randomise context switch requests */
+#define RGXFWIF_INICFG_CTXSWITCH_SRESET_EN				(IMG_UINT32_C(0x1) << 1)
+#define RGXFWIF_INICFG_HWPERF_EN						(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN				(IMG_UINT32_C(0x1) << 3)	/*!< Randomise DM-killing requests */
+#define RGXFWIF_INICFG_POW_RASCALDUST					(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_SPU_CLOCK_GATE					(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_FBCDC_V3_1_EN					(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_CHECK_MLIST_EN					(IMG_UINT32_C(0x1) << 7)
+#define RGXFWIF_INICFG_DISABLE_CLKGATING_EN				(IMG_UINT32_C(0x1) << 8)
+#define RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN	(IMG_UINT32_C(0x1) << 9)
+#define RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN		(IMG_UINT32_C(0x1) << 10)
+/* 11 unused */
+#define RGXFWIF_INICFG_REGCONFIG_EN						(IMG_UINT32_C(0x1) << 12)
+#define RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY			(IMG_UINT32_C(0x1) << 13)
+#define RGXFWIF_INICFG_HWP_DISABLE_FILTER				(IMG_UINT32_C(0x1) << 14)
+/* 15 unused */
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT			(16)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST			(RGXFWIF_CTXSWITCH_PROFILE_FAST_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM			(RGXFWIF_CTXSWITCH_PROFILE_MEDIUM_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW			(RGXFWIF_CTXSWITCH_PROFILE_SLOW_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY		(RGXFWIF_CTXSWITCH_PROFILE_NODELAY_EN << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK			(IMG_UINT32_C(0x7) << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT)
+#define RGXFWIF_INICFG_DISABLE_DM_OVERLAP				(IMG_UINT32_C(0x1) << 19)
+#define RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER			(IMG_UINT32_C(0x1) << 20)
+#define RGXFWIF_INICFG_INJECT_ICS_FAULT					(IMG_UINT32_C(0x1) << 21)
+#define RGXFWIF_INICFG_VALIDATE_IRQ						(IMG_UINT32_C(0x1) << 22)
+#define RGXFWIF_INICFG_DISABLE_PDP_EN					(IMG_UINT32_C(0x1) << 23)
+#define RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN	(IMG_UINT32_C(0x1) << 24)
+#define RGXFWIF_INICFG_WORKEST							(IMG_UINT32_C(0x1) << 25)
+#define RGXFWIF_INICFG_PDVFS							(IMG_UINT32_C(0x1) << 26)
+/* 27 unused */
+/* 28 unused */
+#define RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT				(29)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_NONE				(0)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP			(RGXFWIF_ISP_SCHEDMODE_VER1_IPP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP			(RGXFWIF_ISP_SCHEDMODE_VER2_ISP << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT)
+#define RGXFWIF_INICFG_ISPSCHEDMODE_MASK				(RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP |\
+                                                         RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP)
+#define RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER			(IMG_UINT32_C(0x1) << 31)
+
+#define RGXFWIF_INICFG_ALL								(0xFFFFFFFFU)
+
+/* Extended Flag definitions affecting the firmware globally */
+#define RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT			(0)
+/* [8]   Lossy min channel override
+ * [7]   YUV10 override
+ * [6:4] Quality
+ * [3]   Quality enable
+ * [2:1] Compression scheme
+ * [0]   Lossy group */
+#define RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK			(IMG_UINT32_C(0x1FF)) /* RGX_CR_TFBC_COMPRESSION_CONTROL_MASKFULL */
+#define RGXFWIF_INICFG_EXT_ALL							(RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK)
+
+#define RGXFWIF_INICFG_SYS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_CTXSWITCH_MODE_RAND | \
+														  RGXFWIF_INICFG_CTXSWITCH_SRESET_EN)
+
+/* Flag definitions affecting only workloads submitted by a particular OS */
+
+/*!
+ * @AddToGroup ContextSwitching
+ * @{
+ * @Name Per-OS DM context switch configuration flags
+ * @{
+ */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN				(IMG_UINT32_C(0x1) << 0) /*!< Enables TDM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN				(IMG_UINT32_C(0x1) << 1) /*!< Enables GEOM DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN				(IMG_UINT32_C(0x1) << 2) /*!< Enables FRAG DM context switch */
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN				(IMG_UINT32_C(0x1) << 3) /*!< Enables CDM context switch */
+
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM				(IMG_UINT32_C(0x1) << 4)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM				(IMG_UINT32_C(0x1) << 5)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D				(IMG_UINT32_C(0x1) << 6)
+#define RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM				(IMG_UINT32_C(0x1) << 7)
+
+#define RGXFWIF_INICFG_OS_ALL							(0xFFU)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL				(RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN | \
+														 RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN)
+
+#define RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK				~(RGXFWIF_INICFG_OS_CTXSWITCH_DM_ALL)
+
+/*!
+ * @} End of Per-OS Context switch configuration flags
+ * @} End of AddToGroup ContextSwitching
+ */
+
+#define RGXFWIF_FILTCFG_TRUNCATE_HALF					(IMG_UINT32_C(0x1) << 3)
+#define RGXFWIF_FILTCFG_TRUNCATE_INT					(IMG_UINT32_C(0x1) << 2)
+#define RGXFWIF_FILTCFG_NEW_FILTER_MODE					(IMG_UINT32_C(0x1) << 1)
+
+typedef IMG_UINT32 RGX_ACTIVEPM_CONF;
+#define RGX_ACTIVEPM_FORCE_OFF	0U
+#define RGX_ACTIVEPM_FORCE_ON	1U
+#define RGX_ACTIVEPM_DEFAULT	2U
+
+typedef IMG_UINT32 RGX_RD_POWER_ISLAND_CONF;
+#define RGX_RD_POWER_ISLAND_FORCE_OFF	0U
+#define RGX_RD_POWER_ISLAND_FORCE_ON	1U
+#define RGX_RD_POWER_ISLAND_DEFAULT		2U
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+/* Unused registers re-purposed for storing counters of the Firmware's
+ * interrupts for each OS
+ */
+#if (RGXFW_MAX_NUM_OSIDS == 8)
+#define IRQ_COUNTER_STORAGE_REGS                        \
+		0x2028U, /* RGX_CR_PM_TA_MMU_FSTACK         */  \
+		0x2050U, /* RGX_CR_PM_3D_MMU_FSTACK         */  \
+		0x2030U, /* RGX_CR_PM_START_OF_MMU_TACONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/  \
+		0x2058U, /* RGX_CR_PM_START_OF_MMU_3DCONTEXT*/
+#elif (RGXFW_MAX_NUM_OSIDS == 2)
+#define IRQ_COUNTER_STORAGE_REGS                        \
+		0x2028U, /* RGX_CR_PM_TA_MMU_FSTACK         */  \
+		0x2050U, /* RGX_CR_PM_3D_MMU_FSTACK         */
+#else
+#error Unsupported number of IRQ_COUNTER_STORAGE_REGS registers!
+#endif
+#endif
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SIGBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TRACEBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_SYSDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_OSDATA;
+#if defined(SUPPORT_TBI_INTERFACE)
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TBIBUF;
+#endif
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERFBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRINFOBUF;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RUNTIME_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_GPU_UTIL_FW;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_REG_CFG;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWPERF_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_MUX_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_CONFIG_CNTBLK;
+typedef RGXFWIF_DEV_VIRTADDR  PRGX_HWPERF_SELECT_CUSTOM_CNTRS;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_RTN_SLOTS;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWCOMMONCONTEXT;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_ZSBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CORE_CLK_RATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COUNTERBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FIRMWAREGCOVBUFFER;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_HWRTDATA;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_TIMESTAMP_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCB;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+/*!
+ * @Brief Buffer to store KM active client contexts
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+} RGXFWIF_ACTIVE_CONTEXT_BUF_DATA;
+#endif
+
+/*!
+ * This number is used to represent an invalid page catalogue physical address
+ */
+#define RGXFWIF_INVALID_PC_PHYADDR 0xFFFFFFFFFFFFFFFFLLU
+
+/*!
+ * This number is used to represent an unallocated set of page catalog base registers
+ */
+#define RGXFW_BIF_INVALID_PCSET 0xFFFFFFFFU
+
+/*!
+ * This number is used to represent an invalid OS ID for the purpose of tracking PC set ownership
+ */
+#define RGXFW_BIF_INVALID_OSID 0xFFFFFFFFU
+
+/*!
+ * Firmware memory context.
+ */
+typedef struct
+{
+	IMG_DEV_PHYADDR			RGXFW_ALIGN sPCDevPAddr;	/*!< device physical address of context's page catalogue */
+	IMG_UINT32				uiPageCatBaseRegSet;	/*!< associated page catalog base register (RGXFW_BIF_INVALID_PCSET == unallocated) */
+	IMG_UINT32				uiBreakpointAddr; /*!< breakpoint address */
+	IMG_UINT32				uiBPHandlerAddr; /*!< breakpoint handler address */
+	IMG_UINT32				uiBreakpointCtl; /*!< DM and enable control for BP */
+	IMG_UINT32				ui32FwMemCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32              ui32OSid;
+	IMG_BOOL                bOSidAxiProt;
+#endif
+
+} UNCACHED_ALIGN RGXFWIF_FWMEMCONTEXT;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWMEMCONTEXT) == 32,
+				"RGXFWIF_FWMEMCONTEXT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * FW context state flags
+ */
+#define RGXFWIF_CONTEXT_FLAGS_NEED_RESUME				(0x00000001U)
+#define RGXFWIF_CONTEXT_FLAGS_MC_NEED_RESUME_MASKFULL	(0x000000FFU)
+#define RGXFWIF_CONTEXT_FLAGS_TDM_HEADER_STALE			(0x00000100U)
+#define RGXFWIF_CONTEXT_FLAGS_LAST_KICK_SECURE			(0x00000200U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend state (per GEOM core)
+ */
+typedef struct
+{
+	/* FW-accessible TA state which must be written out to memory on context store */
+	IMG_UINT64	RGXFW_ALIGN uTAReg_VDM_CALL_STACK_POINTER;		 /*!< VDM control stream stack pointer, to store in mid-TA */
+	IMG_UINT64	RGXFW_ALIGN uTAReg_VDM_CALL_STACK_POINTER_Init;	 /*!< Initial value of VDM control stream stack pointer (in case is 'lost' due to a lock-up) */
+	IMG_UINT16	ui16TACurrentIdx;
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE_PER_GEOM;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_GEOM_CORE_SIZE 4
+
+static_assert(RGX_NUM_GEOM_CORES <= MAX_GEOM_CORE_SIZE,
+				"RGX_NUM_GEOM_CORES is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware GEOM/TA context suspend states for all GEOM cores
+ */
+typedef struct
+{
+	/*! FW-accessible TA state which must be written out to memory on context store */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[MAX_GEOM_CORE_SIZE];
+#else
+	RGXFWIF_TACTX_STATE_PER_GEOM asGeomCore[RGX_NUM_GEOM_CORES];
+#endif
+} UNCACHED_ALIGN RGXFWIF_TACTX_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_TACTX_STATE) == 160,
+				"RGXFWIF_TACTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware FRAG/3D context suspend state
+ */
+typedef struct
+{
+	/* FW-accessible ISP state which must be written out to memory on context store */
+	IMG_UINT32	u3DReg_PM_DEALLOCATED_MASK_STATUS; /*!< PM deallocation status */
+	IMG_UINT32	u3DReg_PM_PDS_MTILEFREE_STATUS; /*!< Macro-tiles (MTs) finished status */
+	IMG_UINT32	ui32CtxStateFlags;	/*!< Compatibility and other flags */
+	/* au3DReg_ISP_STORE should be the last element of the structure
+	 * as this is an array whose size is determined at runtime
+	 * after detecting the RGX core */
+	IMG_UINT32	au3DReg_ISP_STORE[]; /*!< ISP state (per-pipe) */
+} UNCACHED_ALIGN RGXFWIF_3DCTX_STATE;
+
+static_assert(sizeof(RGXFWIF_3DCTX_STATE) <= 16U,
+              "Size of structure RGXFWIF_3DCTX_STATE exceeds maximum expected size.");
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_3DCTX_STATE) == 16,
+				"RGXFWIF_3DCTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_CTX_USING_BUFFER_A		(0)
+#define RGXFWIF_CTX_USING_BUFFER_B		(1U)
+
+typedef struct
+{
+	IMG_UINT32	ui32CtxStateFlags; /*!< Target buffer and other flags */
+} RGXFWIF_COMPUTECTX_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_COMPUTECTX_STATE) == 4,
+				"RGXFWIF_COMPUTECTX_STATE is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_STATS_PENDING   (1U << 0)
+#define RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT (1U << 1)
+
+typedef struct
+{
+	IMG_UINT32	ui32ExtJobRefToDisableZSStore;
+	IMG_BOOL	bDisableZStore;
+	IMG_BOOL	bDisableSStore;
+} RGXFWIF_DISABLE_ZSSTORE;
+
+#define MAX_ZSSTORE_DISABLE 8
+
+typedef struct
+{
+	bool       bSaved;
+	IMG_UINT64 ui64CheckSum[4];
+}RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY;
+
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_TQ[RGX_TRP_MAX_NUM_CORES][1];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_2D[RGX_TRP_MAX_NUM_CORES][2];
+typedef IMG_UINT64 RGXFWIF_TRP_CHECKSUM_3D[RGX_TRP_MAX_NUM_CORES][4];
+typedef RGXFWIF_TRP_CHECKSUM_GEOM_ENTRY RGXFWIF_TRP_CHECKSUM_GEOM[RGX_TRP_MAX_NUM_CORES];
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware render context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTAContext;				/*!< Firmware context for the TA */
+	RGXFWIF_FWCOMMONCONTEXT	s3DContext;				/*!< Firmware context for the 3D */
+
+	RGXFWIF_STATIC_RENDERCONTEXT_STATE sStaticRenderContextState;
+
+	RGXFWIF_DISABLE_ZSSTORE sDisableZSStoreQueue[MAX_ZSSTORE_DISABLE];
+
+	IMG_UINT32			ui32ZSStoreQueueCount;
+	IMG_UINT32			ui32WriteOffsetOfDisableZSStore;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32			ui32FwRenderCtxFlags; /*!< Compatibility and other flags */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_3D		aui64TRPChecksums3D;	/*!< Used by Firmware to store checksums during 3D WRR */
+	RGXFWIF_TRP_CHECKSUM_GEOM	aui64TRPChecksumsGeom;	/*!< Used by Firmware to store checksums during TA WRR */
+	RGXFWIF_DM			eTRPGeomCoreAffinity; /* !< Represent the DM affinity for pending 2nd TRP pass of GEOM otherwise points RGXFWIF_DM_MAX. */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWRENDERCONTEXT;
+
+/*!
+	Firmware compute context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT sCDMContext;				/*!< Firmware context for the CDM */
+
+	RGXFWIF_STATIC_COMPUTECONTEXT_STATE sStaticComputeContextState;
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+	IMG_UINT32 ui32ComputeCtxFlags; /*!< Compatibility and other flags */
+
+	IMG_UINT32		aui32WGPChecksum[RGX_WGP_MAX_NUM_CORES];
+} UNCACHED_ALIGN RGXFWIF_FWCOMPUTECONTEXT;
+
+/*!
+	Firmware TDM context.
+*/
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTDMContext;				/*!< Firmware context for the TDM */
+
+	IMG_UINT32			ui32WorkEstCCBSubmitted; /*!< Number of commands submitted to the WorkEst FW CCB */
+
+} UNCACHED_ALIGN RGXFWIF_FWTDMCONTEXT;
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware transfer context.
+ */
+typedef struct
+{
+	RGXFWIF_FWCOMMONCONTEXT	sTQContext;			 /*!< Firmware context for TQ3D */
+
+#if defined(SUPPORT_TRP)
+	RGXFWIF_TRP_CHECKSUM_TQ aui64TRPChecksumsTQ;/*!< Used by Firmware to store checksums during TQ WRR */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FWTRANSFERCONTEXT;
+
+/*!
+ ******************************************************************************
+ * Defines for CMD_TYPE corruption detection and forward compatibility check
+ *****************************************************************************/
+
+/* CMD_TYPE 32bit contains:
+ * 31:16	Reserved for magic value to detect corruption (16 bits)
+ * 15		Reserved for RGX_CCB_TYPE_TASK (1 bit)
+ * 14:0		Bits available for CMD_TYPEs (15 bits) */
+
+
+/* Magic value to detect corruption */
+#define RGX_CMD_MAGIC_DWORD			IMG_UINT32_C(0x2ABC)
+#define RGX_CMD_MAGIC_DWORD_MASK	(0xFFFF0000U)
+#define RGX_CMD_MAGIC_DWORD_SHIFT	(16U)
+#define RGX_CMD_MAGIC_DWORD_SHIFTED	(RGX_CMD_MAGIC_DWORD << RGX_CMD_MAGIC_DWORD_SHIFT)
+
+/*!
+ * @InGroup KCCBTypes ClientCCBTypes
+ * @Brief Generic CCB control structure
+ */
+typedef struct
+{
+	volatile IMG_UINT32		ui32WriteOffset;		/*!< write offset into array of commands (MUST be aligned to 16 bytes!) */
+	volatile IMG_UINT32		ui32ReadOffset;			/*!< read offset into array of commands */
+	IMG_UINT32				ui32WrapMask;			/*!< Offset wrapping mask (Total capacity of the CCB - 1) */
+} UNCACHED_ALIGN RGXFWIF_CCB_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCB_CTL) == 16,
+				"RGXFWIF_CCB_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @Defgroup KCCBTypes Kernel CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the KCCB functionality
+ * @{
+ */
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PT      (0x1U) /* MMU_CTRL_INVAL_PT_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PD      (0x2U) /* MMU_CTRL_INVAL_PD_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PC      (0x4U) /* MMU_CTRL_INVAL_PC_EN */
+
+#if !defined(__KERNEL__)
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB   (0x10U) /* can't use PM_TLB0 bit from BIFPM_CTRL reg because it collides with PT bit from BIF_CTRL reg */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB | 0x8U) /* BIF_CTRL_INVAL_TLB1_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x0U) /* not used */
+
+#else
+#define RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB   (0x10U) /* can't use PM_TLB0 bit from BIFPM_CTRL reg because it collides with PT bit from BIF_CTRL reg */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_TLB     (RGXFWIF_MMUCACHEDATA_FLAGS_PMTLB | 0x8U) /* BIF_CTRL_INVAL_TLB1_EN */
+#define RGXFWIF_MMUCACHEDATA_FLAGS_CTX_ALL (0x800U) /* MMU_CTRL_INVAL_ALL_CONTEXTS_EN */
+#endif
+
+#define RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT (0x4000000U) /* indicates FW should interrupt the host */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_MMUCACHE type command
+ */
+typedef struct
+{
+	IMG_UINT32            ui32CacheFlags;
+	RGXFWIF_DEV_VIRTADDR  sMMUCacheSync;
+	IMG_UINT32            ui32MMUCacheSyncUpdateValue;
+} RGXFWIF_MMUCACHEDATA;
+
+#define RGXFWIF_BPDATA_FLAGS_ENABLE (1U << 0)
+#define RGXFWIF_BPDATA_FLAGS_WRITE  (1U << 1)
+#define RGXFWIF_BPDATA_FLAGS_CTL    (1U << 2)
+#define RGXFWIF_BPDATA_FLAGS_REGS   (1U << 3)
+
+typedef struct
+{
+	PRGXFWIF_FWMEMCONTEXT		psFWMemContext;		/*!< Memory context */
+	IMG_UINT32					ui32BPAddr;			/*!< Breakpoint address */
+	IMG_UINT32					ui32HandlerAddr;	/*!< Breakpoint handler */
+	IMG_UINT32					ui32BPDM;			/*!< Breakpoint control */
+	IMG_UINT32					ui32BPDataFlags;
+	IMG_UINT32					ui32TempRegs;		/*!< Number of temporary registers to overallocate */
+	IMG_UINT32					ui32SharedRegs;		/*!< Number of shared registers to overallocate */
+	IMG_UINT64 RGXFW_ALIGN		ui64SpillAddr;
+	RGXFWIF_DM					eDM;				/*!< DM associated with the breakpoint */
+} RGXFWIF_BPDATA;
+
+#define RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS (RGXFWIF_PRBUFFER_MAXSUPPORTED + 1U) /* +1 is RTDATASET cleanup */
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_KICK type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CWoffUpdate;	/*!< Client CCB woff update */
+	IMG_UINT32					ui32CWrapMaskUpdate; /*!< Client CCB wrap mask update after CCCB growth */
+	IMG_UINT32					ui32NumCleanupCtl;		/*!< number of CleanupCtl pointers attached */
+	PRGXFWIF_CLEANUP_CTL		apsCleanupCtl[RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS]; /*!< CleanupCtl structures associated with command */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32					ui32WorkEstCmdHeaderOffset; /*!< offset to the CmdHeader which houses the workload estimation kick data. */
+#endif
+} RGXFWIF_KCCB_CMD_KICK_DATA;
+
+/*!
+ * @Brief Command data for @Ref RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK type command
+ */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_KICK_DATA	sTACmdKickData; /*!< GEOM DM kick command data */
+	RGXFWIF_KCCB_CMD_KICK_DATA	s3DCmdKickData; /*!< FRAG DM kick command data */
+} RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FORCE_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;			/*!< address of the firmware context */
+	IMG_UINT32					ui32CCBFenceOffset;	/*!< Client CCB fence offset */
+} RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT	psContext;
+	RGXFWIF_DISABLE_ZSSTORE		sDisableZSStore;
+} RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA;
+
+/*!
+ * @Brief Resource types supported by \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef enum
+{
+	RGXFWIF_CLEANUP_FWCOMMONCONTEXT,		/*!< FW common context cleanup */
+	RGXFWIF_CLEANUP_HWRTDATA,				/*!< FW HW RT data cleanup */
+	RGXFWIF_CLEANUP_FREELIST,				/*!< FW freelist cleanup */
+	RGXFWIF_CLEANUP_ZSBUFFER,				/*!< FW ZS Buffer cleanup */
+} RGXFWIF_CLEANUP_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CLEANUP type command
+ */
+typedef struct
+{
+	RGXFWIF_CLEANUP_TYPE			eCleanupType;			/*!< Cleanup type */
+	union {
+		PRGXFWIF_FWCOMMONCONTEXT	psContext;				/*!< FW common context to cleanup */
+		PRGXFWIF_HWRTDATA			psHWRTData;				/*!< HW RT to cleanup */
+		PRGXFWIF_FREELIST			psFreelist;				/*!< Freelist to cleanup */
+		PRGXFWIF_ZSBUFFER			psZSBuffer;				/*!< ZS Buffer to cleanup */
+	} uCleanupData;
+} RGXFWIF_CLEANUP_REQUEST;
+
+/*!
+ * @Brief Type of power requests supported in \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef enum
+{
+	RGXFWIF_POW_OFF_REQ = 1,           /*!< GPU power-off request */
+	RGXFWIF_POW_FORCED_IDLE_REQ,       /*!< Force-idle related request */
+	RGXFWIF_POW_NUM_UNITS_CHANGE,      /*!< Request to change default powered scalable units */
+	RGXFWIF_POW_APM_LATENCY_CHANGE     /*!< Request to change the APM latency period */
+} RGXFWIF_POWER_TYPE;
+
+/*!
+ * @Brief Supported force-idle related requests with \ref RGXFWIF_POW_FORCED_IDLE_REQ type request
+ */
+typedef enum
+{
+	RGXFWIF_POWER_FORCE_IDLE = 1,      /*!< Request to force-idle GPU */
+	RGXFWIF_POWER_CANCEL_FORCED_IDLE,  /*!< Request to cancel a previously successful force-idle transition */
+	RGXFWIF_POWER_HOST_TIMEOUT,        /*!< Notification that host timed-out waiting for force-idle state */
+} RGXFWIF_POWER_FORCE_IDLE_TYPE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_POW type command
+ */
+typedef struct
+{
+	RGXFWIF_POWER_TYPE					ePowType;					/*!< Type of power request */
+	union
+	{
+		IMG_UINT32						ui32PowUnits;			/*!< Number of active Dusts */
+		IMG_BOOL						bForced;				/*!< If the operation is mandatory */
+		RGXFWIF_POWER_FORCE_IDLE_TYPE	ePowRequestType;		/*!< Type of Request. Consolidating Force Idle, Cancel Forced Idle, Host Timeout */
+	} uPowerReqData;
+} RGXFWIF_POWER_REQUEST;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_SLCFLUSHINVAL type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext; /*!< Context to fence on (only useful when bDMContext == TRUE) */
+	IMG_BOOL    bInval;                 /*!< Invalidate the cache as well as flushing */
+	IMG_BOOL    bDMContext;             /*!< The data to flush/invalidate belongs to a specific DM context */
+	IMG_UINT64	RGXFW_ALIGN ui64Address;	/*!< Optional address of range (only useful when bDMContext == FALSE) */
+	IMG_UINT64	RGXFW_ALIGN ui64Size;		/*!< Optional size of range (only useful when bDMContext == FALSE) */
+} RGXFWIF_SLCFLUSHINVALDATA;
+
+typedef enum
+{
+	RGXFWIF_HWPERF_CTRL_TOGGLE = 0,
+	RGXFWIF_HWPERF_CTRL_SET    = 1,
+	RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV = 2
+} RGXFWIF_HWPERF_UPDATE_CONFIG;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG type command
+ */
+typedef struct
+{
+	RGXFWIF_HWPERF_UPDATE_CONFIG eOpCode; /*!< Control operation code */
+	IMG_UINT64	RGXFW_ALIGN	ui64Mask;   /*!< Mask of events to toggle */
+} RGXFWIF_HWPERF_CTRL;
+
+typedef struct
+{
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_MUX_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_MUX_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_MUX_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS type command
+ */
+typedef struct
+{
+	IMG_UINT32                ui32NumBlocks;    /*!< Number of RGX_HWPERF_CONFIG_CNTBLK in the array */
+	PRGX_HWPERF_CONFIG_CNTBLK sBlockConfigs;    /*!< Address of the RGX_HWPERF_CONFIG_CNTBLK array */
+} RGXFWIF_HWPERF_CONFIG_DA_BLKS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE type command
+ */
+typedef struct
+{
+	IMG_UINT32	ui32NewClockSpeed;			/*!< New clock speed */
+} RGXFWIF_CORECLKSPEEDCHANGE_DATA;
+
+#define RGXFWIF_HWPERF_CTRL_BLKS_MAX	16U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS type command
+ */
+typedef struct
+{
+	bool		bEnable;
+	IMG_UINT32	ui32NumBlocks;                              /*!< Number of block IDs in the array */
+	IMG_UINT16	aeBlockIDs[RGXFWIF_HWPERF_CTRL_BLKS_MAX];   /*!< Array of RGX_HWPERF_CNTBLK_ID values */
+} RGXFWIF_HWPERF_CTRL_BLKS;
+
+
+typedef struct
+{
+	IMG_UINT16                      ui16CustomBlock;
+	IMG_UINT16                      ui16NumCounters;
+	PRGX_HWPERF_SELECT_CUSTOM_CNTRS sCustomCounterIDs;
+} RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE & \ref RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE type commands
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;				/*!< ZS-Buffer FW address */
+	IMG_BOOL				bDone;								/*!< action backing/unbacking succeeded */
+} RGXFWIF_ZSBUFFER_BACKING_DATA;
+
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE type command
+ */
+typedef struct
+{
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;				/*!< Freelist FW address */
+	IMG_UINT32				ui32DeltaPages;						/*!< Amount of the Freelist change */
+	IMG_UINT32				ui32NewPages;						/*!< New amount of pages on the freelist (including ready pages) */
+	IMG_UINT32              ui32ReadyPages;                     /*!< Number of ready pages to be held in reserve until OOM */
+} RGXFWIF_FREELIST_GS_DATA;
+
+#define RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT         (MAX_HW_TA3DCONTEXTS * RGXFW_MAX_FREELISTS * 2U)
+#define RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG 0x80000000U
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE type command
+ */
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+} RGXFWIF_FREELISTS_RECONSTRUCTION_DATA;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE type command
+ */
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT  psContext; /*!< Context to that may need to be resumed following write offset update */
+} UNCACHED_ALIGN RGXFWIF_WRITE_OFFSET_UPDATE_DATA;
+
+/*!
+ ******************************************************************************
+ * Proactive DVFS Structures
+ *****************************************************************************/
+#define NUM_OPP_LEVELS 16
+
+typedef struct
+{
+	IMG_UINT32			ui32Volt; /* V  */
+	IMG_UINT32			ui32Freq; /* Hz */
+} UNCACHED_ALIGN OPP_LEVEL;
+
+typedef struct
+{
+	OPP_LEVEL		asOPPValues[NUM_OPP_LEVELS];
+#if defined(DEBUG)
+	IMG_UINT32		ui32MinOPPPoint;
+#endif
+	IMG_UINT32		ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_OPP_INFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32MaxOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MAX_FREQ_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32MinOPPPoint;
+} UNCACHED_ALIGN RGXFWIF_PDVFS_MIN_FREQ_DATA;
+
+/*!
+ ******************************************************************************
+ * Register configuration structures
+ *****************************************************************************/
+
+#define RGXFWIF_REG_CFG_MAX_SIZE 512
+
+typedef enum
+{
+	RGXFWIF_REGCFG_CMD_ADD				= 101,
+	RGXFWIF_REGCFG_CMD_CLEAR			= 102,
+	RGXFWIF_REGCFG_CMD_ENABLE			= 103,
+	RGXFWIF_REGCFG_CMD_DISABLE			= 104
+} RGXFWIF_REGDATA_CMD_TYPE;
+
+typedef struct
+{
+	IMG_UINT64		ui64Addr;
+	IMG_UINT64		ui64Mask;
+	IMG_UINT64		ui64Value;
+} RGXFWIF_REG_CFG_REC;
+
+typedef struct
+{
+	RGXFWIF_REGDATA_CMD_TYPE         eCmdType;
+	RGXFWIF_REG_CFG_TYPE             eRegConfigType;
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  sRegConfig;
+
+} RGXFWIF_REGCONFIG_DATA;
+
+typedef struct
+{
+	/**
+	 * PDump WRW command write granularity is 32 bits.
+	 * Add padding to ensure array size is 32 bit granular.
+	 */
+	IMG_UINT8           RGXFW_ALIGN  aui8NumRegsType[PVR_ALIGN((IMG_UINT32)RGXFWIF_REG_CFG_TYPE_ALL,sizeof(IMG_UINT32))];
+	RGXFWIF_REG_CFG_REC RGXFW_ALIGN  asRegConfigs[RGXFWIF_REG_CFG_MAX_SIZE];
+} UNCACHED_ALIGN RGXFWIF_REG_CFG;
+
+typedef enum
+{
+	RGXFWIF_OS_ONLINE = 1,
+	RGXFWIF_OS_OFFLINE
+} RGXFWIF_OS_STATE_CHANGE;
+
+/*!
+ * @Brief Command data for \ref RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE type command
+ */
+typedef struct
+{
+	IMG_UINT32 ui32DriverID;
+	RGXFWIF_OS_STATE_CHANGE eNewOSState;
+} UNCACHED_ALIGN RGXFWIF_OS_STATE_CHANGE_DATA;
+
+typedef struct
+{
+	PRGXFWIF_FWCOMMONCONTEXT psContext;
+	IMG_UINT32               ui32FirstIntJobRefToCancel;
+	IMG_UINT32               ui32LastIntJobRefToCancel;
+} UNCACHED_ALIGN RGXFWIF_CANCEL_WORK_DATA;
+
+typedef enum
+{
+	RGXFWIF_PWR_COUNTER_DUMP_START = 1,
+	RGXFWIF_PWR_COUNTER_DUMP_STOP,
+	RGXFWIF_PWR_COUNTER_DUMP_SAMPLE,
+} RGXFWIF_COUNTER_DUMP_REQUEST;
+
+typedef struct
+{
+	RGXFWIF_COUNTER_DUMP_REQUEST eCounterDumpRequest;
+}  RGXFW_ALIGN RGXFWIF_COUNTER_DUMP_DATA;
+
+/*!
+ * @Brief List of command types supported by the Kernel CCB
+ */
+typedef enum
+{
+	/* Common commands */
+	RGXFWIF_KCCB_CMD_KICK								= 101U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< DM workload kick command */
+	RGXFWIF_KCCB_CMD_MMUCACHE							= 102U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< MMU cache invalidation request */
+	RGXFWIF_KCCB_CMD_BP									= 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_SLCFLUSHINVAL						= 104U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< SLC flush and invalidation request */
+	RGXFWIF_KCCB_CMD_CLEANUP							= 105U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Requests cleanup of a FW resource (type specified in the command data) */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE			= 106U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Backing for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE			= 107U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Unbacking for on-demand ZS-Buffer done */
+	RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE				= 108U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelist Grow done */
+	RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE	= 109U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Freelists Reconstruction done */
+	RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE			= 110U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the firmware that the host has added more data to a CDM2 Circular Buffer */
+	RGXFWIF_KCCB_CMD_HEALTH_CHECK						= 111U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Health check request */
+	RGXFWIF_KCCB_CMD_FORCE_UPDATE						= 112U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Forcing signalling of all unmet UFOs for a given CCB offset */
+	RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK				= 113U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< There is a TA and a 3D command in this single kick */
+	RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE			= 114U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW that a Guest OS has come online / offline. */
+	RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE					= 115U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Informs the FW to disable zs store of a running 3D or add it to queue of render context. */
+	RGXFWIF_KCCB_CMD_CANCEL_WORK						= 116U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Cancel all work up to and including a given intjobref for a given context */
+	/* Commands only permitted to the native or host OS */
+	RGXFWIF_KCCB_CMD_POW								= 200U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Power request */
+	RGXFWIF_KCCB_CMD_REGCONFIG							= 201U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE					= 202U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Core clock speed change event */
+	RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE						= 203U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Ask the firmware to update its cached ui32LogType value from the (shared) tracebuf control structure */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ				= 204U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a maximum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE				= 205U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the priority/group for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL					= 206U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set or clear firmware state flags */
+	RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MIN_FREQ				= 207U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Set a minimum frequency/OPP point */
+	RGXFWIF_KCCB_CMD_PHR_CFG							= 208U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Periodic Hardware Reset behaviour */
+	RGXFWIF_KCCB_CMD_WDG_CFG							= 210U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure Safety Firmware Watchdog */
+	RGXFWIF_KCCB_CMD_COUNTER_DUMP						= 211U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Controls counter dumping in the FW */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE					= 213U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice for a particular driver. It can only be serviced for the Host DDK */
+	RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL			= 214U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Changes the GPU time slice interval for all drivers. It can only be serviced for the Host DDK */
+
+	/* HWPerf commands */
+	RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG				= 300U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure HWPerf events (to be generated) and HWPerf buffer address (if required) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS					= 301U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure directly addressable counters for HWPerf */
+	RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS					= 302U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Enable or disable multiple HWPerf blocks (reusing existing configuration) */
+	RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS			= 303U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure, clear and enable multiple HWPerf blocks */
+	RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS			= 304U | RGX_CMD_MAGIC_DWORD_SHIFTED, /*!< Configure the custom counters for HWPerf */
+
+} RGXFWIF_KCCB_CMD_TYPE;
+
+#define RGXFWIF_LAST_ALLOWED_GUEST_KCCB_CMD (RGXFWIF_KCCB_CMD_POW - 1)
+
+/*! @Brief Kernel CCB command packet */
+typedef struct
+{
+	RGXFWIF_KCCB_CMD_TYPE  eCmdType;      /*!< Command type */
+	IMG_UINT32             ui32KCCBFlags; /*!< Compatibility and other flags */
+
+	/* NOTE: Make sure that uCmdData is the last member of this struct
+	 * This is to calculate actual command size for device mem copy.
+	 * (Refer RGXGetCmdMemCopySize())
+	 * */
+	union
+	{
+		RGXFWIF_KCCB_CMD_KICK_DATA			sCmdKickData;			/*!< Data for Kick command */
+		RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA	sCombinedTA3DCmdKickData;	/*!< Data for combined TA/3D Kick command */
+		RGXFWIF_MMUCACHEDATA				sMMUCacheData;			/*!< Data for MMU cache command */
+		RGXFWIF_BPDATA						sBPData;				/*!< Data for Breakpoint Commands */
+		RGXFWIF_SLCFLUSHINVALDATA			sSLCFlushInvalData;		/*!< Data for SLC Flush/Inval commands */
+		RGXFWIF_CLEANUP_REQUEST				sCleanupData;			/*!< Data for cleanup commands */
+		RGXFWIF_POWER_REQUEST				sPowData;				/*!< Data for power request commands */
+		RGXFWIF_HWPERF_CTRL					sHWPerfCtrl;			/*!< Data for HWPerf control command */
+		RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS	sHWPerfCfgEnableBlks;	/*!< Data for HWPerf configure, clear and enable performance counter block command */
+		RGXFWIF_HWPERF_CTRL_BLKS			sHWPerfCtrlBlks;		/*!< Data for HWPerf enable or disable performance counter block commands */
+		RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS	sHWPerfSelectCstmCntrs;	/*!< Data for HWPerf configure the custom counters to read */
+		RGXFWIF_HWPERF_CONFIG_DA_BLKS		sHWPerfCfgDABlks;		/*!< Data for HWPerf configure Directly Addressable blocks */
+		RGXFWIF_CORECLKSPEEDCHANGE_DATA		sCoreClkSpeedChangeData;/*!< Data for core clock speed change */
+		RGXFWIF_ZSBUFFER_BACKING_DATA		sZSBufferBackingData;	/*!< Feedback for Z/S Buffer backing/unbacking */
+		RGXFWIF_FREELIST_GS_DATA			sFreeListGSData;		/*!< Feedback for Freelist grow/shrink */
+		RGXFWIF_FREELISTS_RECONSTRUCTION_DATA	sFreeListsReconstructionData;	/*!< Feedback for Freelists reconstruction */
+		RGXFWIF_REGCONFIG_DATA				sRegConfigData;			/*!< Data for custom register configuration */
+		RGXFWIF_WRITE_OFFSET_UPDATE_DATA    sWriteOffsetUpdateData; /*!< Data for informing the FW about the write offset update */
+		RGXFWIF_PDVFS_MAX_FREQ_DATA			sPDVFSMaxFreqData;		/*!< Data for setting the max frequency/OPP */
+		RGXFWIF_PDVFS_MIN_FREQ_DATA			sPDVFSMinFreqData;		/*!< Data for setting the min frequency/OPP */
+		RGXFWIF_OS_STATE_CHANGE_DATA        sCmdOSOnlineStateData;  /*!< Data for updating the Guest Online states */
+		RGXFWIF_DEV_VIRTADDR                sTBIBuffer;             /*!< Dev address for TBI buffer allocated on demand */
+		RGXFWIF_COUNTER_DUMP_DATA			sCounterDumpConfigData; /*!< Data for dumping of register ranges */
+		RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA	sDisableZSStoreData;	/*!< Data for disabling zs store of a 3D workload */
+		RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA  sForceUpdateData;       /*!< Data for signalling all unmet fences for a given CCB */
+		RGXFWIF_CANCEL_WORK_DATA			sCancelWorkData;		/*!< Data for cancelling work */
+	} UNCACHED_ALIGN uCmdData;
+} UNCACHED_ALIGN RGXFWIF_KCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_KCCB_CMD);
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_KCCB_CMD) == 64,
+				"RGXFWIF_KCCB_CMD is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of KCCBTypes */
+
+/*!
+ * @Defgroup FWCCBTypes Firmware CCB data interface
+ * @Brief Types grouping data structures and defines used in realising the Firmware CCB functionality
+ * @{
+ */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING and the
+ * \ref RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING Firmware CCB commands
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32ZSBufferID; /*!< ZS buffer ID */
+} RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELIST_GROW Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32				ui32FreelistID; /*!< Freelist ID */
+} RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32			ui32FreelistsCount;                                     /*!< Freelists count */
+	IMG_UINT32			ui32HwrCounter;                                         /*!< HWR counter */
+	IMG_UINT32			aui32FreelistIDs[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT]; /*!< Array of freelist IDs to reconstruct */
+} RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA;
+
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF			(1U<<0)	/*!< 1 if a page fault happened */
+#define RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS	(1U<<1)	/*!< 1 if applicable to all contexts */
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT32						ui32ServerCommonContextID;	/*!< Context affected by the reset */
+	RGX_CONTEXT_RESET_REASON		eResetReason;				/*!< Reason for reset */
+	RGXFWIF_DM						eDM;						/*!< Data Master affected by the reset */
+	IMG_UINT32						ui32ResetJobRef;			/*!< Job ref running at the time of reset */
+	IMG_UINT32						ui32Flags;					/*!< RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG bitfield */
+	IMG_UINT64 RGXFW_ALIGN			ui64PCAddress;				/*!< At what page catalog address */
+	IMG_DEV_VIRTADDR RGXFW_ALIGN	sFaultAddress;				/*!< Page fault address (only when applicable) */
+} RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION
+ * Firmware CCB command
+ *****************************************************************************/
+typedef struct
+{
+	IMG_DEV_VIRTADDR sFWFaultAddr;	/*!< Page fault address */
+} RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA;
+
+/*!
+ ******************************************************************************
+ * List of command types supported by the Firmware CCB
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING              = 101U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be backed with physical pages
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING            = 102U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests ZSBuffer to be unbacked
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA */
+	RGXFWIF_FWCCB_CMD_FREELIST_GROW                 = 103U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand freelist grow
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA */
+	RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION      = 104U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests freelists reconstruction
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA */
+	RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION    = 105U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a HWR event on a context
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA */
+	RGXFWIF_FWCCB_CMD_DEBUG_DUMP                    = 106U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand debug dump
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS                  = 107U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests an on-demand update on process stats
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA */
+
+	RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE          = 108U | RGX_CMD_MAGIC_DWORD_SHIFTED,
+	RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART           = 109U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Requests GPU restart
+	                                                                                          \n Command data: None */
+	RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION    = 112U | RGX_CMD_MAGIC_DWORD_SHIFTED,   /*!< Notifies host of a FW pagefault
+	                                                                                          \n Command data: RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA */
+} RGXFWIF_FWCCB_CMD_TYPE;
+
+/*!
+ ******************************************************************************
+ * List of the various stats of the process to update/increment
+ *****************************************************************************/
+typedef enum
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS=1,		/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumPartialRenders stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32TotalNumOutOfMemory stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTAStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES,				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32Num3DStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES,			/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumCDMStores stat */
+	RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES				/*!< PVRSRVStatsUpdateRenderContextStats should increase the value of the ui32NumTDMStores stat */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE;
+
+/*!
+ ******************************************************************************
+ * @Brief Command data of the \ref RGXFWIF_FWCCB_CMD_UPDATE_STATS Firmware CCB
+ * command
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_UPDATE_STATS_TYPE		eElementToUpdate;			/*!< Element to update */
+	IMG_PID									pidOwner;					/*!< The pid of the process whose stats are being updated */
+	IMG_INT32								i32AdjustmentValue;			/*!< Adjustment to be made to the statistic */
+} RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA;
+
+typedef struct
+{
+	IMG_UINT32 ui32CoreClkRate;
+} UNCACHED_ALIGN RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA;
+
+
+/*!
+ ******************************************************************************
+ * @Brief Firmware CCB command structure
+ *****************************************************************************/
+typedef struct
+{
+	RGXFWIF_FWCCB_CMD_TYPE  eCmdType;       /*!< Command type */
+	IMG_UINT32              ui32FWCCBFlags; /*!< Compatibility and other flags */
+
+	union
+	{
+		RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING_DATA				sCmdZSBufferBacking;			/*!< Data for Z/S-Buffer on-demand (un)backing*/
+		RGXFWIF_FWCCB_CMD_FREELIST_GS_DATA					sCmdFreeListGS;					/*!< Data for on-demand freelist grow/shrink */
+		RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION_DATA		sCmdFreeListsReconstruction;	/*!< Data for freelists reconstruction */
+		RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA				sCmdContextResetNotification;	/*!< Data for context reset notification */
+		RGXFWIF_FWCCB_CMD_UPDATE_STATS_DATA					sCmdUpdateStatsData;			/*!< Data for updating process stats */
+		RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE_DATA			sCmdCoreClkRateChange;
+		RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA					sCmdFWPagefault;				/*!< Data for context reset notification */
+	} RGXFW_ALIGN uCmdData;
+} RGXFW_ALIGN RGXFWIF_FWCCB_CMD;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGXFWIF_FWCCB_CMD);
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWCCB_CMD) == 48,
+				"RGXFWIF_FWCCB_CMD is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of FWCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Workload estimation Firmware CCB command structure for RGX
+ *****************************************************************************/
+typedef struct
+{
+	IMG_UINT16 ui16ReturnDataIndex; /*!< Index for return data array */
+	IMG_UINT16 ui16CyclesTakenHigh;	/*!< Not used on Rogue GPU cores. */
+	IMG_UINT32 ui32CyclesTaken;     /*!< The cycles the workload took on the hardware */
+} RGXFWIF_WORKEST_FWCCB_CMD;
+
+/*!
+ * @Defgroup ClientCCBTypes Client CCB data interface
+ * @Brief Types grouping data structures and defines used in realising Client CCB commands/functionality
+ * @{
+ */
+
+/* Required memory alignment for 64-bit variables accessible by Meta
+  (The gcc meta aligns 64-bit variables to 64-bit; therefore, memory shared
+   between the host and meta that contains 64-bit variables has to maintain
+   this alignment) */
+#define RGXFWIF_FWALLOC_ALIGN	sizeof(IMG_UINT64)
+
+#define RGX_CCB_TYPE_TASK			(IMG_UINT32_C(1) << 15)
+#define RGX_CCB_FWALLOC_ALIGN(size)	(PVR_ALIGN(size, RGXFWIF_FWALLOC_ALIGN))
+
+typedef IMG_UINT32 RGXFWIF_CCB_CMD_TYPE;
+
+/*!
+ * @Name Client CCB command types
+ * @{
+ */
+#define RGXFWIF_CCB_CMD_TYPE_GEOM			(201U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TA DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_3D			(202U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_3D				(203U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command */
+#define RGXFWIF_CCB_CMD_TYPE_3D_PR			(204U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 3D DM command for Partial render */
+#define RGXFWIF_CCB_CMD_TYPE_CDM			(205U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Compute DM command */
+#define RGXFWIF_CCB_CMD_TYPE_TQ_TDM			(206U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< TDM command */
+#define RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE (207U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_TQ_2D			(208U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< 2D DM command for TQ operation */
+#define RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP	(209U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_NULL			(210U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+#define RGXFWIF_CCB_CMD_TYPE_ABORT			(211U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK)
+
+/* Leave a gap between CCB specific commands and generic commands */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE          (212U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a command */
+#define RGXFWIF_CCB_CMD_TYPE_UPDATE         (213U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates of a command */
+#define RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE     (214U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence updates related to workload resources */
+#define RGXFWIF_CCB_CMD_TYPE_FENCE_PR       (215U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Fence dependencies of a PR command */
+#define RGXFWIF_CCB_CMD_TYPE_PRIORITY       (216U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Context priority update command */
+/* Pre and Post timestamp commands are supposed to sandwich the DM cmd. The
+   padding code with the CCB wrap upsets the FW if we don't have the task type
+   bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+*/
+#define RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP (217U | RGX_CMD_MAGIC_DWORD_SHIFTED)
+/* UNFENCED type is not supported but keeping defines as it is for now */
+
+
+#define RGXFWIF_CCB_CMD_TYPE_PADDING	(221U | RGX_CMD_MAGIC_DWORD_SHIFTED) /*!< Skip without action type command */
+#define RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP	(223U | RGX_CMD_MAGIC_DWORD_SHIFTED | RGX_CCB_TYPE_TASK) /*!< Process a vulkan timestamp */
+/*! @} End of Client CCB command types */
+
+
+#define RGXFWIF_TRP_STATUS_UNKNOWN				0x000U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_OK			0x001U
+#define RGXFWIF_TRP_STATUS_CHECKSUMS_ERROR		0x002U
+
+#define RGXFWIF_CR_TRP_SIGNATURE_STATUS			(RGX_CR_SCRATCH10)
+
+
+typedef struct
+{
+	/* Index for the KM Workload estimation return data array */
+	IMG_UINT16 RGXFW_ALIGN         ui16ReturnDataIndex;
+	/* Predicted time taken to do the work in cycles */
+	IMG_UINT32 RGXFW_ALIGN         ui32CyclesPrediction;
+	/* Deadline for the workload (in usecs) */
+	IMG_UINT64 RGXFW_ALIGN         ui64Deadline;
+} RGXFWIF_WORKEST_KICK_DATA;
+
+/*! @Brief Command header of a command in the client CCB buffer.
+ *
+ *  Followed by this header is the command-data specific to the
+ *  command-type as specified in the header.
+ */
+typedef struct
+{
+	RGXFWIF_CCB_CMD_TYPE					eCmdType;      /*!< Command data type following this command header */
+	IMG_UINT32								ui32CmdSize;   /*!< Size of the command following this header */
+	IMG_UINT32								ui32ExtJobRef; /*!< external job reference - provided by client and used in debug for tracking submitted work */
+	IMG_UINT32								ui32IntJobRef; /*!< internal job reference - generated by services and used in debug for tracking submitted work */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_WORKEST_KICK_DATA RGXFW_ALIGN	sWorkEstKickData; /*!< Workload Estimation - Workload Estimation Data */
+#endif
+} RGXFWIF_CCB_CMD_HEADER;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCB_CMD_HEADER) == 16,
+				"RGXFWIF_CCB_CMD_HEADER is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*
+ ******************************************************************************
+ * Client CCB commands which are only required by the kernel
+ *****************************************************************************/
+
+/*! @Brief Command data for \ref RGXFWIF_CCB_CMD_TYPE_PRIORITY type client CCB command */
+typedef struct
+{
+	IMG_INT32              i32Priority; /*!< Priority level */
+} RGXFWIF_CMD_PRIORITY;
+
+/*! @} End of ClientCCBTypes */
+
+/*!
+ ******************************************************************************
+ * Signature and Checksums Buffer
+ *****************************************************************************/
+typedef struct
+{
+	PRGXFWIF_SIGBUFFER		sBuffer;			/*!< Ptr to Signature Buffer memory */
+	IMG_UINT32				ui32LeftSizeInRegs;	/*!< Amount of space left for storing regs in the buffer */
+} UNCACHED_ALIGN RGXFWIF_SIGBUF_CTL;
+
+typedef struct
+{
+	PRGXFWIF_COUNTERBUFFER	sBuffer;			/*!< Ptr to counter dump buffer */
+	IMG_UINT32				ui32SizeInDwords;	/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_COUNTER_DUMP_CTL;
+
+typedef struct
+{
+	PRGXFWIF_FIRMWAREGCOVBUFFER	sBuffer;		/*!< Ptr to firmware gcov buffer */
+	IMG_UINT32					ui32Size;		/*!< Amount of space for storing in the buffer */
+} UNCACHED_ALIGN RGXFWIF_FIRMWARE_GCOV_CTL;
+
+/*!
+ *****************************************************************************
+ * RGX Compatibility checks
+ *****************************************************************************/
+
+/* WARNING: Whenever the layout of RGXFWIF_COMPCHECKS_BVNC changes, the
+	following define should be increased by 1 to indicate to the
+	compatibility logic that layout has changed. */
+#define RGXFWIF_COMPCHECKS_LAYOUT_VERSION 3
+
+typedef struct
+{
+	IMG_UINT32	ui32LayoutVersion; /* WARNING: This field must be defined as first one in this structure */
+	IMG_UINT64	RGXFW_ALIGN ui64BVNC;
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS_BVNC;
+
+typedef struct
+{
+	IMG_UINT8	ui8OsCountSupport;
+} UNCACHED_ALIGN RGXFWIF_INIT_OPTIONS;
+
+#define RGXFWIF_COMPCHECKS_BVNC_DECLARE_AND_INIT(name) \
+	RGXFWIF_COMPCHECKS_BVNC (name) = { \
+		RGXFWIF_COMPCHECKS_LAYOUT_VERSION, \
+		0, \
+	}
+#define RGXFWIF_COMPCHECKS_BVNC_INIT(name) \
+	do { \
+		(name).ui32LayoutVersion = RGXFWIF_COMPCHECKS_LAYOUT_VERSION; \
+		(name).ui64BVNC = 0; \
+	} while (false)
+
+typedef struct
+{
+	RGXFWIF_COMPCHECKS_BVNC		sHWBVNC;				/*!< hardware BVNC (from the RGX registers) */
+	RGXFWIF_COMPCHECKS_BVNC		sFWBVNC;				/*!< firmware BVNC */
+	IMG_UINT32					ui32DDKVersion;			/*!< software DDK version */
+	IMG_UINT32					ui32DDKBuild;			/*!< software DDK build no. */
+	IMG_UINT32					ui32BuildOptions;		/*!< build options bit-field */
+	RGXFWIF_INIT_OPTIONS		sInitOptions;			/*!< initialisation options bit-field */
+	IMG_BOOL					bUpdated;				/*!< Information is valid */
+} UNCACHED_ALIGN RGXFWIF_COMPCHECKS;
+
+typedef struct
+{
+	IMG_UINT32                 ui32NumCores;
+	IMG_UINT64 RGXFW_ALIGN     aui64MultiCoreCapabilities[RGX_MULTICORE_MAX_NOHW_CORES];
+} UNCACHED_ALIGN RGXFWIF_MULTICORE_INFO;
+
+/*! @Brief Firmware Runtime configuration data \ref RGXFWIF_RUNTIME_CFG
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	IMG_UINT32         ui32ActivePMLatencyms;                    /*!< APM latency in ms before signalling IDLE to the host */
+	IMG_UINT32         ui32RuntimeCfgFlags;                      /*!< Compatibility and other flags */
+	IMG_BOOL           bActivePMLatencyPersistant;               /*!< If set, APM latency does not reset to system default each GPU power transition */
+	IMG_UINT32         ui32CoreClockSpeed;                       /*!< Core clock speed, currently only used to calculate timer ticks */
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32         ui32SOCClockSpeed;                        /*!< SOC clock speed, used for deadline scheduling */
+#endif
+	IMG_UINT32         ui32PowUnitsState;                        /*!< Last number of dusts change requested by the host */
+	IMG_UINT32         ui32PHRMode;                              /*!< Periodic Hardware Reset configuration values */
+	IMG_UINT32         ui32HCSDeadlineMS;                        /*!< New number of milliseconds C/S is allowed to last */
+	IMG_UINT32         ui32WdgPeriodUs;                          /*!< The watchdog period in microseconds */
+	IMG_INT32          ai32DriverPriority[RGXFW_MAX_NUM_OSIDS];  /*!< Array of priorities per OS */
+	IMG_UINT32         aui32DriverIsolationGroup[RGXFW_MAX_NUM_OSIDS]; /*!< Array of isolation groups per OS */
+	IMG_UINT32         aui32TSPercentage[RGXFW_MAX_NUM_OSIDS];   /*!< Array of time slice per OS */
+	IMG_UINT32         ui32TSIntervalMs;                         /*!< Time slice interval */
+	IMG_UINT32         ui32VzConnectionCooldownPeriodInSec;      /*!< Vz Connection Cooldown period in secs */
+
+	PRGXFWIF_HWPERFBUF sHWPerfBuf;                               /*!< On-demand allocated HWPerf buffer address, to be passed to the FW */
+	RGXFWIF_DMA_ADDR   sHWPerfDMABuf;
+	RGXFWIF_DMA_ADDR   sHWPerfCtlDMABuf;
+} RGXFWIF_RUNTIME_CFG;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_RUNTIME_CFG) == 68,
+				"RGXFWIF_RUNTIME_CFG is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ *****************************************************************************
+ * Control data for RGX
+ *****************************************************************************/
+
+#define RGXFWIF_HWR_DEBUG_DUMP_ALL (99999U)
+
+#if defined(PDUMP)
+
+#define RGXFWIF_PID_FILTER_MAX_NUM_PIDS 32U
+
+typedef enum
+{
+	RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+	RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT
+} RGXFWIF_PID_FILTER_MODE;
+
+typedef struct
+{
+	IMG_PID uiPID;
+	IMG_UINT32 ui32DriverID;
+} RGXFW_ALIGN RGXFWIF_PID_FILTER_ITEM;
+
+typedef struct
+{
+	RGXFWIF_PID_FILTER_MODE eMode;
+	/* each process in the filter list is specified by a PID and OS ID pair.
+	 * each PID and OS pair is an item in the items array (asItems).
+	 * if the array contains less than RGXFWIF_PID_FILTER_MAX_NUM_PIDS entries
+	 * then it must be terminated by an item with pid of zero.
+	 */
+	RGXFWIF_PID_FILTER_ITEM asItems[RGXFWIF_PID_FILTER_MAX_NUM_PIDS];
+} RGXFW_ALIGN RGXFWIF_PID_FILTER;
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA  (0x1U << 0)
+#define RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE  (0x1U << 1)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE  (0x1U << 2)
+#define RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE     (0x1U << 3)
+#endif
+
+typedef enum
+{
+	RGXFWIF_TPU_DM_PDM = 0,
+	RGXFWIF_TPU_DM_VDM = 1,
+	RGXFWIF_TPU_DM_CDM = 2,
+	RGXFWIF_TPU_DM_TDM = 3,
+	RGXFWIF_TPU_DM_LAST
+} RGXFWIF_TPU_DM;
+
+typedef enum
+{
+	RGXFWIF_GPIO_VAL_OFF           = 0, /*!< No GPIO validation */
+	RGXFWIF_GPIO_VAL_GENERAL       = 1, /*!< Simple test case that
+	                                         initiates by sending data via the
+	                                         GPIO and then sends back any data
+	                                         received over the GPIO */
+	RGXFWIF_GPIO_VAL_AP            = 2, /*!< More complex test case that writes
+	                                         and reads data across the entire
+	                                         GPIO AP address range.*/
+#if defined(SUPPORT_STRIP_RENDERING)
+	RGXFWIF_GPIO_VAL_SR_BASIC      = 3, /*!< Strip Rendering AP based basic test.*/
+	RGXFWIF_GPIO_VAL_SR_COMPLEX    = 4, /*!< Strip Rendering AP based complex test.*/
+#endif
+	RGXFWIF_GPIO_VAL_TESTBENCH     = 5, /*!< Validates the GPIO Testbench. */
+	RGXFWIF_GPIO_VAL_LOOPBACK      = 6, /*!< Send and then receive each byte
+	                                         in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LOOPBACK_LITE = 7, /*!< Send and then receive each power-of-2
+	                                         byte in the range 0-255. */
+	RGXFWIF_GPIO_VAL_LAST
+} RGXFWIF_GPIO_VAL_MODE;
+
+typedef IMG_UINT32 FW_PERF_CONF;
+#define FW_PERF_CONF_NONE			0U
+#define FW_PERF_CONF_ICACHE			1U
+#define FW_PERF_CONF_DCACHE			2U
+#define FW_PERF_CONF_JTLB_INSTR		5U
+#define FW_PERF_CONF_INSTRUCTIONS	6U
+
+typedef enum
+{
+	FW_BOOT_STAGE_TLB_INIT_FAILURE = -2,
+	FW_BOOT_STAGE_NOT_AVAILABLE = -1,
+	FW_BOOT_NOT_STARTED = 0,
+	FW_BOOT_BLDR_STARTED = 1,
+	FW_BOOT_CACHE_DONE,
+	FW_BOOT_TLB_DONE,
+	FW_BOOT_MAIN_STARTED,
+	FW_BOOT_ALIGNCHECKS_DONE,
+	FW_BOOT_INIT_DONE,
+} FW_BOOT_STAGE;
+
+/*!
+ * @AddToGroup KCCBTypes
+ * @{
+ * @Name Kernel CCB return slot responses
+ * @{
+ * Usage of bit-fields instead of bare integers
+ * allows FW to possibly pack-in several responses for each single kCCB command.
+ */
+
+#define RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED   (1U << 0) /*!< Command executed (return status from FW) */
+#define RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY   (1U << 1) /*!< A cleanup was requested but resource busy */
+#define RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE   (1U << 2) /*!< Poll failed in FW for a HW operation to complete */
+
+#define RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE            0x0U      /*!< Reset value of a kCCB return slot (set by host) */
+/*!
+ * @} End of Name Kernel CCB return slot responses
+ * @} End of AddToGroup KCCBTypes
+ */
+
+/*! @Brief OS connection data \ref RGXFWIF_CONNECTION_CTL allocated
+ * by services and used to track OS state in Firmware and Services
+ **/
+typedef struct
+{
+	/* Fw-Os connection states */
+	volatile RGXFWIF_CONNECTION_FW_STATE eConnectionFwState;    /*!< Firmware-OS connection state */
+	volatile RGXFWIF_CONNECTION_OS_STATE eConnectionOsState;    /*!< Services-OS connection state */
+	volatile IMG_UINT32                  ui32AliveFwToken;      /*!< OS Alive token updated by Firmware */
+	volatile IMG_UINT32                  ui32AliveOsToken;      /*!< OS Alive token updated by Services */
+} UNCACHED_ALIGN RGXFWIF_CONNECTION_CTL;
+
+/*! @Brief Firmware OS Initialization data \ref RGXFWIF_OSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	/* Kernel CCB */
+	PRGXFWIF_CCB_CTL        psKernelCCBCtl; /*!< Kernel CCB Control */
+	PRGXFWIF_CCB            psKernelCCB; /*!<  Kernel CCB */
+	PRGXFWIF_CCB_RTN_SLOTS  psKernelCCBRtnSlots; /*!<  Kernel CCB return slots */
+
+	/* Firmware CCB */
+	PRGXFWIF_CCB_CTL        psFirmwareCCBCtl; /*!<  Firmware CCB control */
+	PRGXFWIF_CCB            psFirmwareCCB; /*!<  Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	PRGXFWIF_CCB_CTL        psWorkEstFirmwareCCBCtl; /*!<  Workload estimation control */
+	PRGXFWIF_CCB            psWorkEstFirmwareCCB; /*!<  Workload estimation buffer */
+
+	PRGXFWIF_HWRINFOBUF     sRGXFWIfHWRInfoBufCtl; /*!<  HWRecoveryInfo control */
+
+	IMG_UINT32              ui32HWRDebugDumpLimit; /*!< Firmware debug dump maximum limit */
+
+	PRGXFWIF_OSDATA         sFwOsData; /*!<  Firmware per-os shared data */
+
+	RGXFWIF_COMPCHECKS      sRGXCompChecks; /*!< Compatibility checks to be populated by the Firmware */
+	RGXFWIF_MULTICORE_INFO  sRGXMulticoreInfo; /*! < Multicore capability info */
+
+} UNCACHED_ALIGN RGXFWIF_OSINIT;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_OSINIT) == 104,
+				"RGXFWIF_OSINIT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @Brief Firmware System Initialization data \ref RGXFWIF_SYSINIT
+ * allocated by services and used by the Firmware on boot
+ **/
+typedef struct
+{
+	RGX_MIPS_STATE          sMIPSState; /*!< MIPS Debug Data; this must be the first member in the structure */
+
+	IMG_DEV_PHYADDR         RGXFW_ALIGN sFaultPhysAddr; /*!< Fault read address */
+
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sPDSExecBase; /*!< PDS execution base */
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN sUSCExecBase; /*!< USC execution base */
+
+	IMG_UINT64              RGXFW_ALIGN ui64HWPerfFilter; /*! Event filter for Firmware events */
+
+	IMG_UINT32              RGXFW_ALIGN aui32TPUTrilinearFracMask[RGXFWIF_TPU_DM_LAST];
+
+#if defined(PDUMP)
+	RGXFWIF_SIGBUF_CTL      asSigBufCtl[RGXFWIF_DM_MAX]; /*!< Signature and Checksum Buffers for DMs */
+#endif
+
+	RGXFWIF_DMA_ADDR        sCorememDataStore; /*!< Firmware coremem data */
+
+	RGXFWIF_COUNTER_DUMP_CTL sCounterDumpCtl;
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	RGXFWIF_FIRMWARE_GCOV_CTL sFirmwareGcovCtl; /*!< Firmware gcov buffer control */
+#endif
+
+	IMG_UINT32              ui32FilterFlags;
+
+	PRGXFWIF_RUNTIME_CFG    sRuntimeCfg; /*!<  Firmware Runtime configuration */
+
+	PRGXFWIF_TRACEBUF       sTraceBufCtl; /*!<  Firmware Trace buffer control */
+	PRGXFWIF_SYSDATA        sFwSysData; /*!< Firmware System shared data */
+#if defined(SUPPORT_TBI_INTERFACE)
+	PRGXFWIF_TBIBUF         sTBIBuf; /*!< Tbi log buffer */
+#endif
+
+	PRGXFWIF_GPU_UTIL_FW    sGpuUtilFWCtl; /*!< GPU utilization buffer */
+	PRGXFWIF_REG_CFG        sRegCfg; /*!< Firmware register user configuration */
+	PRGXFWIF_HWPERF_CTL     sHWPerfCtl; /*!< HWPerf counter block configuration.*/
+
+	RGXFWIF_DEV_VIRTADDR    sAlignChecks; /*!< Array holding Server structures alignment data */
+
+	IMG_UINT32              ui32InitialCoreClockSpeed; /*!< Core clock speed at FW boot time */
+
+#if defined(SUPPORT_SOC_TIMER)
+	IMG_UINT32              ui32InitialSOCClockSpeed; /*!< System/SOC clock speed at FW boot time */
+#endif
+
+	IMG_UINT32              ui32InitialActivePMLatencyms; /*!< APM latency in ms before signalling IDLE to the host */
+
+	IMG_BOOL                bFirmwareStarted; /*!< Flag to be set by the Firmware after successful start */
+
+	IMG_UINT32              ui32MarkerVal; /*!< Host/FW Trace synchronisation Partition Marker */
+
+	IMG_UINT32              ui32FirmwareStartedTimeStamp; /*!< Firmware initialization complete time */
+
+	FW_PERF_CONF            eFirmwarePerf; /*!< Firmware performance counter config */
+
+#if defined(SUPPORT_FW_OPP_TABLE)
+	RGXFWIF_OPP_INFO        RGXFW_ALIGN sOPPInfo;
+
+	/**
+	 * FW Pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory when running on non primary FW thread
+	 * to communicate to host driver.
+	 */
+	PRGXFWIF_CORE_CLK_RATE  RGXFW_ALIGN sCoreClockRate;
+#endif
+
+#if defined(PDUMP)
+	RGXFWIF_PID_FILTER      sPIDFilter;
+#endif
+
+	RGXFWIF_GPIO_VAL_MODE   eGPIOValidationMode;
+
+	RGX_HWPERF_BVNC         sBvncKmFeatureFlags; /*!< Used in HWPerf for decoding BVNC Features*/
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32              ui32SecurityTestFlags;
+	RGXFWIF_DEV_VIRTADDR    pbSecureBuffer;
+	RGXFWIF_DEV_VIRTADDR    pbNonSecureBuffer;
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	RGXFWIF_DEV_VIRTADDR    sActiveContextBufBase; /*!< Active context buffer base */
+#endif
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	/*
+	 * Used when validation is enabled to allow the host to check
+	 * that MTS sent the correct sideband in response to a kick
+	 * from a given OSes schedule register.
+	 * Testing is enabled if RGXFWIF_KICK_TEST_ENABLED_BIT is set
+	 *
+	 * Set by the host to:
+	 * (osid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT
+	 * reset to 0 by FW when kicked by the given OSid
+	 */
+	IMG_UINT32              ui32OSKickTest;
+#endif
+
+	/* Value to write into RGX_CR_TFBC_COMPRESSION_CONTROL */
+	IMG_UINT32              ui32TFBCCompressionControl;
+
+#if defined(SUPPORT_AUTOVZ)
+	IMG_UINT32              ui32VzWdgPeriod;
+#endif
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	/* notify firmware power-up on host-side recovery */
+	IMG_BOOL                bFwHostRecoveryMode;
+#endif
+} UNCACHED_ALIGN RGXFWIF_SYSINIT;
+
+static_assert(offsetof(RGXFWIF_SYSINIT, sMIPSState) == 0,
+				"sMIPSState is not the first member of the RGXFWIF_SYSINIT struct");
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+#define RGXFWIF_KICK_TEST_ENABLED_BIT  0x1
+#define RGXFWIF_KICK_TEST_OSID_SHIFT   0x1
+#endif
+
+/*!
+ *****************************************************************************
+ * Timer correlation shared data and defines
+ *****************************************************************************/
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64OSMonoTimeStamp;
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeStamp;
+
+	/* Utility variable used to convert CR timer deltas to OS timer deltas (nS),
+	 * where the deltas are relative to the timestamps above:
+	 * deltaOS = (deltaCR * K) >> decimal_shift, see full explanation below */
+	IMG_UINT64 RGXFW_ALIGN ui64CRDeltaToOSDeltaKNs;
+
+	IMG_UINT32             ui32CoreClockSpeed;
+	IMG_UINT32             ui32Reserved;
+} UNCACHED_ALIGN RGXFWIF_TIME_CORR;
+
+
+/* The following macros are used to help converting FW timestamps to the Host
+ * time domain. On the FW the RGX_CR_TIMER counter is used to keep track of
+ * time; it increments by 1 every 256 GPU clock ticks, so the general
+ * formula to perform the conversion is:
+ *
+ * [ GPU clock speed in Hz, if (scale == 10^9) then deltaOS is in nS,
+ *   otherwise if (scale == 10^6) then deltaOS is in uS ]
+ *
+ *             deltaCR * 256                                   256 * scale
+ *  deltaOS = --------------- * scale = deltaCR * K    [ K = --------------- ]
+ *             GPUclockspeed                                  GPUclockspeed
+ *
+ * The actual K is multiplied by 2^20 (and deltaCR * K is divided by 2^20)
+ * to get some better accuracy and to avoid returning 0 in the integer
+ * division 256000000/GPUfreq if GPUfreq is greater than 256MHz.
+ * This is the same as keeping K as a decimal number.
+ *
+ * The maximum deltaOS is slightly more than 5hrs for all GPU frequencies
+ * (deltaCR * K is more or less a constant), and it's relative to the base
+ * OS timestamp sampled as a part of the timer correlation data.
+ * This base is refreshed on GPU power-on, DVFS transition and periodic
+ * frequency calibration (executed every few seconds if the FW is doing
+ * some work), so as long as the GPU is doing something and one of these
+ * events is triggered then deltaCR * K will not overflow and deltaOS will be
+ * correct.
+ */
+
+#define RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT  (20)
+
+#define RGXFWIF_GET_DELTA_OSTIME_NS(deltaCR, K) \
+	(((deltaCR) * (K)) >> RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT)
+
+
+/*!
+ ******************************************************************************
+ * GPU Utilisation
+ *****************************************************************************/
+
+/* See rgx_common.h for a list of GPU states */
+#define RGXFWIF_GPU_UTIL_TIME_MASK       (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_TIME_MASK32     (IMG_UINT32_C(0xFFFFFFFF) & ~RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+#define RGXFWIF_GPU_UTIL_GET_TIME(word)    ((word) & RGXFWIF_GPU_UTIL_TIME_MASK)
+#define RGXFWIF_GPU_UTIL_GET_STATE(word)   ((word) & RGXFWIF_GPU_UTIL_STATE_MASK)
+#define RGXFWIF_GPU_UTIL_GET_TIME32(word)  ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_TIME_MASK32)
+#define RGXFWIF_GPU_UTIL_GET_STATE32(word) ((IMG_UINT32)(word) & RGXFWIF_GPU_UTIL_STATE_MASK32)
+
+/* The OS timestamps computed by the FW are approximations of the real time,
+ * which means they could be slightly behind or ahead the real timer on the Host.
+ * In some cases we can perform subtractions between FW approximated
+ * timestamps and real OS timestamps, so we need a form of protection against
+ * negative results if for instance the FW one is a bit ahead of time.
+ */
+#define RGXFWIF_GPU_UTIL_GET_PERIOD(newtime,oldtime) \
+	(((newtime) > (oldtime)) ? ((newtime) - (oldtime)) : 0U)
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME(time) | RGXFWIF_GPU_UTIL_GET_STATE(state))
+
+#define RGXFWIF_GPU_UTIL_MAKE_WORD32(time,state) \
+	(RGXFWIF_GPU_UTIL_GET_TIME32(time) | RGXFWIF_GPU_UTIL_GET_STATE32(state))
+
+
+/* The timer correlation array must be big enough to ensure old entries won't be
+ * overwritten before all the HWPerf events linked to those entries are processed
+ * by the MISR. The update frequency of this array depends on how fast the system
+ * can change state (basically how small the APM latency is) and perform DVFS transitions.
+ *
+ * The minimum size is 2 (not 1) to avoid race conditions between the FW reading
+ * an entry while the Host is updating it. With 2 entries in the worst case the FW
+ * will read old data, which is still quite ok if the Host is updating the timer
+ * correlation at that time.
+ */
+#define RGXFWIF_TIME_CORR_ARRAY_SIZE            256U
+#define RGXFWIF_TIME_CORR_CURR_INDEX(seqcount)  ((seqcount) % RGXFWIF_TIME_CORR_ARRAY_SIZE)
+
+/* Make sure the timer correlation array size is a power of 2 */
+static_assert((RGXFWIF_TIME_CORR_ARRAY_SIZE & (RGXFWIF_TIME_CORR_ARRAY_SIZE - 1U)) == 0U,
+			  "RGXFWIF_TIME_CORR_ARRAY_SIZE must be a power of two");
+
+/* The time is stored in DM state time-stamps, and as a result in DMs states counters, in "approximately microseconds",
+ * dividing the time originally obtained in nanoseconds by 2^10 for the sake of reducing coremem usage */
+#define RGXFWIF_DM_OS_TIMESTAMP_SHIFT    10U
+
+typedef struct
+{
+	/* Last GPU DM per-OS states + OS time of the last state update */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWord[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* DMs time-stamps are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aui32DMOSLastWordWrap[RGXFWIF_GPU_UTIL_DM_MAX];
+	/* Counters for the amount of time the GPU DMs were active or inactive(idle or blocked) */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSStatsCounters[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	/* DMs Counters are cached in coremem - to reduce coremem usage we allocate 32 bits for each of them
+	 * and save their values divided by 2^10, so they wrap around in ~73 mins, consequently
+	 * we keep the count of the wrapping around instances */
+	IMG_UINT32 RGXFW_ALIGN aaui32DMOSCountersWrap[RGXFWIF_GPU_UTIL_DM_MAX][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+} RGXFWIF_GPU_STATS;
+
+typedef struct
+{
+	RGXFWIF_TIME_CORR      sTimeCorr[RGXFWIF_TIME_CORR_ARRAY_SIZE];
+	IMG_UINT32             ui32TimeCorrSeqCount;
+
+	/* Compatibility and other flags */
+	IMG_UINT32             ui32GpuUtilFlags;
+
+	/* Last GPU state + OS time of the last state update */
+	IMG_UINT64 RGXFW_ALIGN ui64GpuLastWord;
+	/* Counters for the amount of time the GPU was active/idle/blocked */
+	IMG_UINT64 RGXFW_ALIGN aui64GpuStatsCounters[RGXFWIF_GPU_UTIL_STATE_NUM];
+	/* Device off period timestamp offset */
+	IMG_INT64 RGXFW_ALIGN i64DeviceTimestampOffset;
+	/* Stats per OSID/DriverID */
+	RGXFWIF_GPU_STATS sStats[RGXFW_MAX_NUM_OSIDS];
+} UNCACHED_ALIGN RGXFWIF_GPU_UTIL_FW;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_GPU_UTIL_FW) == 12584,
+				"RGXFWIF_GPU_UTIL_FW is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES      RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES      (RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES)
+#define RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES    2048U
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      PVR_ALIGN((RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_MAX_SIZE_BYTES + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_MAX_SIZE_BYTES), DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY)
+#else
+#define RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE                      0
+#endif
+
+#define RGXFWIF_TDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_TDM_SECURE_QUEUE_OFFSET_BYTES)
+#define RGXFWIF_CDM_SECURE_QUEUE_VADDR                             (RGX_GENERAL_HEAP_BASE + RGXFWIF_KM_GENERAL_HEAP_CDM_SECURE_QUEUE_OFFSET_BYTES)
+
+/*!
+ ******************************************************************************
+ * Virtualisation and Security
+ *****************************************************************************/
+#if defined(SECURE_FW_CODE_OSID)
+/* software workaround for SoCs without fw_code, fw_priv_data signals, MIPS only */
+#if defined(RGX_FEATURE_META)
+#error "SECURE_FW_CODE_OSID is not supported on META cores"
+#elif defined(RGX_FEATURE_RISCV_FW_PROCESSOR)
+#error "SECURE_FW_CODE_OSID is not supported on RISC-V cores"
+#elif (RGX_NUM_DRIVERS_SUPPORTED > 1)
+#error "SECURE_FW_CODE_OSID is not supported on virtualization drivers"
+#elif (SECURE_FW_CODE_OSID + 1 > 2)
+#define MIPS_FW_CODE_OSID				(SECURE_FW_CODE_OSID)
+#else
+#define MIPS_FW_CODE_OSID				(1U)
+#endif
+#endif /* defined(SECURE_FW_CODE_OSID) */
+
+static_assert((RGX_FW_HEAP_OSID_ASSIGNMENT == RGX_FW_HEAP_USES_FIRMWARE_OSID),
+			  " Invalid RGX_FW_HEAP_OSID_ASSIGNMENT value. Rogue cores support only the RGX_FW_HEAP_USES_FIRMWARE_OSID config");
+
+/* Firmware and Host driver share the same OSID */
+#define FW_HEAP_OSID					(FW_OSID)
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* virtualization without security support */
+#define DRIVER_ID(osid)					(osid)
+#define OSID(did)						(did)
+/* Time slice support */
+/* Bits 30 and 31 reserved by FW private driver priority */
+#define RGXFW_VZ_PRIORITY_MAX_SHIFT	(30U)
+#define RGXFW_VZ_PRIORITY_MASK		((1U << RGXFW_VZ_PRIORITY_MAX_SHIFT) - 1U)
+#define RGXFW_VZ_TIME_SLICE_MAX		(100U)
+#define RGXFW_VZ_TIME_SLICE_MIN		(5U)
+#else
+#define DRIVER_ID(osid)					(0U)
+#define OSID(did)						(did)
+#define RGXFW_VZ_PRIORITY_MASK			(0U)
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) < RGX_NUM_DRIVERS_SUPPORTED; (did)++)
+
+#if defined(__KERNEL__)
+/* Driver implementation */
+#define FOREACH_ACTIVE_DRIVER(devinfo, did)        RGXFwSharedMemCacheOpValue(psFwSysData->asOsRuntimeFlagsMirror[RGXFW_HOST_DRIVER_ID], \
+                                                                              INVALIDATE); \
+                                                   FOREACH_SUPPORTED_DRIVER(did)                                \
+                                                   {                                                            \
+                                                   if (devinfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[did].bfOsState != RGXFW_CONNECTION_FW_ACTIVE) continue;
+
+#define END_FOREACH_ACTIVE_DRIVER                  }
+
+#else
+/* Firmware implementation */
+#define FOREACH_ACTIVE_DRIVER(did)                 do {                                                                      \
+                                                   unsigned int idx;                                                         \
+                                                   for (idx = 0; idx < RGXFW_NUM_ACTIVE_DRIVERS; idx++)                      \
+                                                   {                                                                         \
+                                                       (did) = gsRGXFWCtl.aui32ActiveDrivers[idx];                           \
+                                                       {
+
+#define END_FOREACH_ACTIVE_DRIVER                  }}} while (false);
+#endif /* defined(__KERNEL__) */
+
+
+#else
+#define FOREACH_SUPPORTED_DRIVER(did)              for ((did)=RGXFW_HOST_DRIVER_ID; (did) <= RGXFW_HOST_DRIVER_ID; (did)++)
+
+#define FOREACH_ACTIVE_DRIVER(did)                 FOREACH_SUPPORTED_DRIVER(did)
+#define END_FOREACH_ACTIVE_DRIVER
+
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#define FOREACH_VALIDATION_OSID(osid)              for ((osid)=0; (osid) < GPUVIRT_VALIDATION_NUM_OS; (osid)++)
+#define FOREACH_HW_OSID(osid)                      for ((osid)=0; (osid) < RGXFW_MAX_NUM_OSIDS; (osid)++)
+#define FOREACH_DRIVER_RAW_HEAP(did, _struct, dev) for ((did)=RGX_FIRST_RAW_HEAP_DRIVER_ID; (did) < ((PVRSRV_VZ_MODE_IS(NATIVE, _struct, dev) ? 1 : RGX_NUM_DRIVERS_SUPPORTED)); (did)++)
+
+#endif /* RGX_FWIF_KM_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_fwif_resetframework.h b/drivers/gpu/drm/img-rogue/rgx_fwif_resetframework.h
new file mode 100644
index 000000000000..e60bafd84536
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_fwif_resetframework.h
@@ -0,0 +1,70 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_resetframework.h
+@Title          Post-reset work-around framework FW interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_RESETFRAMEWORK_H)
+#define RGX_FWIF_RESETFRAMEWORK_H
+
+#include "img_types.h"
+#include "rgx_fwif_shared.h"
+
+typedef struct
+{
+	union
+	{
+		IMG_UINT64	uCDMReg_CDM_CB_BASE;			//  defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+		IMG_UINT64	uCDMReg_CDM_CTRL_STREAM_BASE;	// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+	};
+	IMG_UINT64	uCDMReg_CDM_CB_QUEUE;				// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+	IMG_UINT64	uCDMReg_CDM_CB;						// !defined(RGX_FEATURE_CDM_USER_MODE_QUEUE)
+} RGXFWIF_RF_REGISTERS;
+
+typedef struct
+{
+	/* THIS MUST BE THE LAST MEMBER OF THE CONTAINING STRUCTURE */
+	RGXFWIF_RF_REGISTERS RGXFW_ALIGN sFWRegisters;
+
+} RGXFWIF_RF_CMD;
+
+/* to opaquely allocate and copy in the kernel */
+#define RGXFWIF_RF_CMD_SIZE  sizeof(RGXFWIF_RF_CMD)
+
+#endif /* RGX_FWIF_RESETFRAMEWORK_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_fwif_sf.h b/drivers/gpu/drm/img-rogue/rgx_fwif_sf.h
new file mode 100644
index 000000000000..a190875b5e6d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_fwif_sf.h
@@ -0,0 +1,1067 @@
+/*************************************************************************/ /*!
+@File           rgx_fwif_sf.h
+@Title          RGX firmware interface string format specifiers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the rgx firmware logging messages. The following
+                list are the messages the firmware prints. Changing anything
+                but the first column or spelling mistakes in the strings will
+                break compatibility with log files created with older/newer
+                firmware versions.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_FWIF_SF_H
+#define RGX_FWIF_SF_H
+
+/******************************************************************************
+ * *DO*NOT* rearrange or delete lines in SFIDLIST or SFGROUPLIST or you
+ *           WILL BREAK fw tracing message compatibility with previous
+ *           fw versions. Only add new ones, if so required.
+ *****************************************************************************/
+/* Available log groups */
+#define RGXFW_LOG_SFGROUPLIST       \
+	X(RGXFW_GROUP_NULL,NULL)        \
+	X(RGXFW_GROUP_MAIN,MAIN)        \
+	X(RGXFW_GROUP_CLEANUP,CLEANUP)  \
+	X(RGXFW_GROUP_CSW,CSW)          \
+	X(RGXFW_GROUP_PM, PM)           \
+	X(RGXFW_GROUP_RTD,RTD)          \
+	X(RGXFW_GROUP_SPM,SPM)          \
+	X(RGXFW_GROUP_MTS,MTS)          \
+	X(RGXFW_GROUP_BIF,BIF)          \
+	X(RGXFW_GROUP_MISC,MISC)        \
+	X(RGXFW_GROUP_POW,POW)          \
+	X(RGXFW_GROUP_HWR,HWR)          \
+	X(RGXFW_GROUP_HWP,HWP)          \
+	X(RGXFW_GROUP_RPM,RPM)          \
+	X(RGXFW_GROUP_DMA,DMA)          \
+	X(RGXFW_GROUP_DBG,DBG)          \
+	X(RGXFW_GROUP_VZ,VZ)            \
+	X(RGXFW_GROUP_SAFETY,SAFETY)    \
+	X(RGXFW_GROUP_VERBOSE,VERBOSE)  \
+	X(RGXFW_GROUP_CUSTOMER,CUSTOMER)
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief FW Trace log groups(GID) list
+ */
+enum RGXFW_LOG_SFGROUPS {
+#define X(A,B) A,
+	RGXFW_LOG_SFGROUPLIST
+#undef X
+};
+
+#define IMG_SF_STRING_MAX_SIZE 256U
+
+typedef struct {
+	IMG_UINT32 ui32Id;
+	IMG_CHAR sName[IMG_SF_STRING_MAX_SIZE];
+} RGXFW_STID_FMT; /*  pair of string format id and string formats */
+
+typedef struct {
+	IMG_UINT32 ui32Id;
+	const IMG_CHAR *psName;
+} RGXKM_STID_FMT; /*  pair of string format id and string formats */
+
+/* Table of String Format specifiers, the group they belong and the number of
+ * arguments each expects. Xmacro styled macros are used to generate what is
+ * needed without requiring hand editing.
+ *
+ * id		: id within a group
+ * gid		: group id
+ * Sym name	: name of enumerations used to identify message strings
+ * String	: Actual string
+ * #args	: number of arguments the string format requires
+ */
+#define RGXFW_LOG_SFIDLIST \
+/*id, gid,              id name,        string,                           # arguments */ \
+X(  0, RGXFW_GROUP_NULL, RGXFW_SF_FIRST, "You should not use this string", 0) \
+\
+X(  1, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D_DEPRECATED, "Kick 3D: FWCtx 0x%08.8x @ %u, RTD 0x%08x. Partial render:%u, CSW resume:%u, prio:%d", 6) \
+X(  2, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_FINISHED, "3D finished, HWRTData0State=%x, HWRTData1State=%x", 2) \
+X(  3, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK3D_TQ_DEPRECATED, "Kick 3D TQ: FWCtx 0x%08.8x @ %u, CSW resume:%u, prio: %d", 4) \
+X(  4, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_TQ_FINISHED, "3D Transfer finished", 0) \
+X(  5, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_DEPRECATED, "Kick Compute: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X(  6, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_FINISHED, "Compute finished", 0) \
+X(  7, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA_DEPRECATED, "Kick TA: FWCtx 0x%08.8x @ %u, RTD 0x%08x. First kick:%u, Last kick:%u, CSW resume:%u, prio:%d", 7) \
+X(  8, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_FINISHED, "TA finished", 0) \
+X(  9, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_RESTART_AFTER_PRENDER, "Restart TA after partial render", 0) \
+X( 10, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_RESUME_WOUT_PRENDER, "Resume TA without partial render", 0) \
+X( 11, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OOM, "Out of memory! Context 0x%08x, HWRTData 0x%x", 2) \
+X( 12, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TLA_DEPRECATED, "Kick TLA: FWCtx 0x%08.8x @ %u, prio:%d", 3) \
+X( 13, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TLA_FINISHED, "TLA finished", 0) \
+X( 14, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CCCB_WOFF_UPDATE, "cCCB Woff update = %u, DM = %u, FWCtx = 0x%08.8x", 3) \
+X( 16, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_START, "UFO Checks for FWCtx 0x%08.8x @ %u", 2) \
+X( 17, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK, "UFO Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 18, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_SUCCEEDED, "UFO Checks succeeded", 0) \
+X( 19, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_PR_CHECK, "UFO PR-Check: [0x%08.8x] is 0x%08.8x requires >= 0x%08.8x", 3) \
+X( 20, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK_START, "UFO SPM PR-Checks for FWCtx 0x%08.8x", 1) \
+X( 21, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK_DEPRECATED, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires >= ????????, [0x%08.8x] is ???????? requires 0x%08.8x", 4) \
+X( 22, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_UPDATE_START, "UFO Updates for FWCtx 0x%08.8x @ %u", 2) \
+X( 23, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_UPDATE, "UFO Update: [0x%08.8x] = 0x%08.8x", 2) \
+X( 24, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ASSERT_FAILED, "ASSERT Failed: line %u of:", 1) \
+X( 25, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_LOCKUP_DEPRECATED, "HWR: Lockup detected on DM%u, FWCtx: 0x%08.8x", 2) \
+X( 26, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_RESET_FW_DEPRECATED, "HWR: Reset fw state for DM%u, FWCtx: 0x%08.8x, MemCtx: 0x%08.8x", 3) \
+X( 27, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_RESET_HW_DEPRECATED, "HWR: Reset HW", 0) \
+X( 28, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_TERMINATED_DEPRECATED, "HWR: Lockup recovered.", 0) \
+X( 29, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_FALSE_LOCKUP_DEPRECATED, "HWR: False lockup detected for DM%u", 1) \
+X( 30, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ALIGN_FAILED, "Alignment check %u failed: host = 0x%x, fw = 0x%x", 3) \
+X( 31, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GP_USC_TRIGGERED, "GP USC triggered", 0) \
+X( 32, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_OVERALLOC_REGS, "Overallocating %u temporary registers and %u shared registers for breakpoint handler", 2) \
+X( 33, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET_DEPRECATED, "Setting breakpoint: Addr 0x%08.8x", 1) \
+X( 34, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_STORE, "Store breakpoint state", 0) \
+X( 35, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_UNSET, "Unsetting BP Registers", 0) \
+X( 36, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NONZERO_RT, "Active RTs expected to be zero, actually %u", 1) \
+X( 37, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTC_PRESENT, "RTC present, %u active render targets", 1) \
+X( 38, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_EST_POWER_DEPRECATED, "Estimated Power 0x%x", 1) \
+X( 39, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTA_TARGET, "RTA render target %u", 1) \
+X( 40, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTA_KICK_RENDER, "Kick RTA render %u of %u", 2) \
+X( 41, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_SIZES_CHECK_DEPRECATED, "HWR sizes check %u failed: addresses = %u, sizes = %u", 3) \
+X( 42, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_ENABLE_DEPRECATED, "Pow: DUSTS_ENABLE = 0x%x", 1) \
+X( 43, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_HWREQ_DEPRECATED, "Pow: On(1)/Off(0): %u, Units: 0x%08.8x", 2) \
+X( 44, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_CHANGE_DEPRECATED, "Pow: Changing number of dusts from %u to %u", 2) \
+X( 45, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_SIDEKICK_IDLE_DEPRECATED, "Pow: Sidekick ready to be powered down", 0) \
+X( 46, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_POW_DUSTS_CHANGE_REQ_DEPRECATED, "Pow: Request to change num of dusts to %u (bPowRascalDust=%u)", 2) \
+X( 47, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PARTIALRENDER_WITHOUT_ZSBUFFER_STORE, "No ZS Buffer used for partial render (store)", 0) \
+X( 48, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PARTIALRENDER_WITHOUT_ZSBUFFER_LOAD, "No Depth/Stencil Buffer used for partial render (load)", 0) \
+X( 49, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_SET_LOCKUP_DEPRECATED, "HWR: Lock-up DM%u FWCtx: 0x%08.8x", 2) \
+X( 50, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_REG_VALUE_DEPRECATED, "MLIST%u checker: CatBase TE=0x%08x (%u Pages), VCE=0x%08x (%u Pages), ALIST=0x%08x, IsTA=%u", 7) \
+X( 51, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_MLIST_VALUE, "MLIST%u checker: MList[%u] = 0x%08x", 3) \
+X( 52, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_OK, "MLIST%u OK", 1) \
+X( 53, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_EMPTY, "MLIST%u is empty", 1) \
+X( 54, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_MLIST_CHECKER_REG_VALUE, "MLIST%u checker: CatBase TE=0x%08x%08x, VCE=0x%08x%08x, ALIST=0x%08x%08x, IsTA=%u", 8) \
+X( 55, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_40480KICK, "3D OQ flush kick", 0) \
+X( 56, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWP_UNSUPPORTED_BLOCK, "HWPerf block ID (0x%x) unsupported by device", 1) \
+X( 57, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET_DEPRECATED2, "Setting breakpoint: Addr 0x%08.8x DM%u", 2) \
+X( 58, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED, "Kick RTU: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X( 59, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTU_FINISHED_DEPRECATED, "RDM finished on context %u", 1) \
+X( 60, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_SHG_DEPRECATED, "Kick SHG: FWCtx 0x%08.8x @ %u, prio: %d", 3) \
+X( 61, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SHG_FINISHED_DEPRECATED, "SHG finished", 0) \
+X( 62, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBA_FINISHED_DEPRECATED, "FBA finished on context %u", 1) \
+X( 63, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_FAILED, "UFO Checks failed", 0) \
+X( 64, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_START, "Kill DM%u start", 1) \
+X( 65, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_COMPLETE, "Kill DM%u complete", 1) \
+X( 66, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FC_CCB_UPDATE_DEPRECATED, "FC%u cCCB Woff update = %u", 2) \
+X( 67, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED2, "Kick RTU: FWCtx 0x%08.8x @ %u, prio: %d, Frame Context: %u", 4) \
+X( 68, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_INIT, "GPU init", 0) \
+X( 69, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNITS_INIT, "GPU Units init (# mask: 0x%x)", 1) \
+X( 70, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGTIMES, "Register access cycles: read: %u cycles, write: %u cycles, iterations: %u", 3) \
+X( 71, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGCONFIG_ADD, "Register configuration added. Address: 0x%x Value: 0x%x%x", 3) \
+X( 72, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REGCONFIG_SET, "Register configuration applied to type %u. (0:pow on, 1:Rascal/dust init, 2-5: TA,3D,CDM,TLA, 6:All)", 1) \
+X( 73, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TPC_FLUSH, "Perform TPC flush.", 0) \
+X( 74, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP_DEPRECATED, "GPU has locked up (see HWR logs for more info)", 0) \
+X( 75, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_OUTOFTIME, "HWR has been triggered - GPU has overrun its deadline (see HWR logs)", 0) \
+X( 76, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_POLLFAILURE_DEPRECATED, "HWR has been triggered - GPU has failed a poll (see HWR logs)", 0) \
+X( 77, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DOPPLER_OOM_DEPRECATED, "Doppler out of memory event for FC %u", 1) \
+X( 78, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK1, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires >= 0x%08.8x", 3) \
+X( 79, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_SPM_PR_CHECK2, "UFO SPM special PR-Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 80, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIMESTAMP, "TIMESTAMP -> [0x%08.8x]", 1) \
+X( 81, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_RMW_UPDATE_START, "UFO RMW Updates for FWCtx 0x%08.8x @ %u", 2) \
+X( 82, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_RMW_UPDATE, "UFO Update: [0x%08.8x] = 0x%08.8x", 2) \
+X( 83, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NULLCMD, "Kick Null cmd: FWCtx 0x%08.8x @ %u", 2) \
+X( 84, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RPM_OOM_DEPRECATED, "RPM Out of memory! Context 0x%08x, SH requestor %u", 2) \
+X( 85, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTU_ABORT_DISCARD_DEPRECATED, "Discard RTU due to RPM abort: FWCtx 0x%08.8x @ %u, prio: %d, Frame Context: %u", 4) \
+X( 86, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED, "Deferring DM%u from running context 0x%08x @ %u (deferred DMs = 0x%08x)", 4) \
+X( 87, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED_WAITING_TURN_DEPRECATED, "Deferring DM%u from running context 0x%08x @ %u to let other deferred DMs run (deferred DMs = 0x%08x)", 4) \
+X( 88, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEFERRED_NO_LONGER, "No longer deferring DM%u from running context = 0x%08x @ %u (deferred DMs = 0x%08x)", 4) \
+X( 89, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_FWCCB_DEPRECATED, "FWCCB for DM%u is full, we will have to wait for space! (Roff = %u, Woff = %u)", 3) \
+X( 90, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_FWCCB, "FWCCB for Driver ID %u is full, we will have to wait for space! (Roff = %u, Woff = %u)", 3) \
+X( 91, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SYNC_PART, "Host Sync Partition marker: %u", 1) \
+X( 92, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SYNC_PART_RPT, "Host Sync Partition repeat: %u", 1) \
+X( 93, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CLOCK_SPEED_CHANGE, "Core clock set to %u Hz", 1) \
+X( 94, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_OFFSETS, "Compute Queue: FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 7) \
+X( 95, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_DEPRECATED, "Signal check failed, Required Data: 0x%x, Address: 0x%08x%08x", 3) \
+X( 96, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE_DEPRECATED, "Signal update, Snoop Filter: %u, MMU Ctx: %u, Signal Id: %u, Signals Base: 0x%08x%08x", 5) \
+X( 97, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWCONTEXT_SIGNALED, "Signalled the previously waiting FWCtx: 0x%08.8x, OSId: %u, Signal Address: 0x%08x%08x", 4) \
+X( 98, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_STALLED_DEPRECATED, "Compute stalled", 0) \
+X( 99, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_STALLED, "Compute stalled (Roff = %u, Woff = %u, Size = %u)", 3) \
+X(100, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_RESUMED_FROM_STALL, "Compute resumed (Roff = %u, Woff = %u, Size = %u)", 3) \
+X(101, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOTIFY_SIGNAL_UPDATE, "Signal update notification from the host, PC Physical Address: 0x%08x%08x, Signal Virtual Address: 0x%08x%08x", 4) \
+X(102, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE_OSID_DM_DEPRECATED, "Signal update from DM: %u, OSId: %u, PC Physical Address: 0x%08x%08x", 4) \
+X(103, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_DM_DEPRECATED, "DM: %u signal check failed", 1) \
+X(104, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED, "Kick TDM: FWCtx 0x%08.8x @ %u, prio:%d", 3) \
+X(105, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_FINISHED, "TDM finished", 0) \
+X(106, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TE_PIPE_STATUS_DEPRECATED, "MMU_PM_CAT_BASE_TE[%u]_PIPE[%u]:  0x%08x 0x%08x)", 4) \
+X(107, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_HIT_DEPRECATED, "BRN 54141 HIT", 0) \
+X(108, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_APPLYING_DUMMY_TA_DEPRECATED, "BRN 54141 Dummy TA kicked", 0) \
+X(109, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_RESUME_TA_DEPRECATED, "BRN 54141 resume TA", 0) \
+X(110, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_DOUBLE_HIT_DEPRECATED, "BRN 54141 double hit after applying WA", 0) \
+X(111, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BRN_54141_DUMMY_TA_VDM_BASE_DEPRECATED, "BRN 54141 Dummy TA VDM base address: 0x%08x%08x", 2) \
+X(112, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_WAIT_FAILURE_WITH_CURRENT, "Signal check failed, Required Data: 0x%x, Current Data: 0x%x, Address: 0x%08x%08x", 4) \
+X(113, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BUFFER_STALL_DEPRECATED, "TDM stalled (Roff = %u, Woff = %u)", 2) \
+X(114, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOTIFY_WRITE_OFFSET_UPDATE, "Write Offset update notification for stalled FWCtx 0x%08.8x", 1) \
+X(115, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED, "Changing OSid %u's priority from %u to %u", 3) \
+X(116, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_RESUMED, "Compute resumed", 0) \
+X(117, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TLA, "Kick TLA: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(118, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED3, "Kick TDM: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(119, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA, "Kick TA: FWCtx 0x%08.8x @ %u, RTD 0x%08x, First kick:%u, Last kick:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(120, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D, "Kick 3D: FWCtx 0x%08.8x @ %u, RTD 0x%08x, Partial render:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 10) \
+X(121, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3DTQ, "Kick 3D TQ: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(122, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_DEPRECATED2, "Kick Compute: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 6) \
+X(123, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RTU_DEPRECATED3, "Kick RTU: FWCtx 0x%08.8x @ %u, Frame Context:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(124, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_SHG_DEPRECATED2, "Kick SHG: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7) \
+X(125, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CSRM_RECONFIG, "Reconfigure CSRM: special coeff support enable %u.", 1) \
+X(127, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_REQ_MAX_COEFFS, "TA requires max coeff mode, deferring: %u.", 1) \
+X(128, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_REQ_MAX_COEFFS, "3D requires max coeff mode, deferring: %u.", 1) \
+X(129, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_FAILED, "Kill DM%u failed", 1) \
+X(130, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_QUEUE, "Thread Queue is full, we will have to wait for space! (Roff = %u, Woff = %u)", 2) \
+X(131, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_WAITING_FOR_QUEUE_FENCE, "Thread Queue is fencing, we are waiting for Roff = %u (Roff = %u, Woff = %u)", 3) \
+X(132, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_HCS_TRIGGERED_DEPRECATED, "DM %u failed to Context Switch on time. Triggered HCS (see HWR logs).", 1) \
+X(133, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HCS_SET_DEPRECATED, "HCS changed to %u ms", 1) \
+X(134, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UPDATE_TILES_IN_FLIGHT_DEPRECATED, "Updating Tiles In Flight (Dusts=%u, PartitionMask=0x%08x, ISPCtl=0x%08x%08x)", 4) \
+X(135, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_TILES_IN_FLIGHT, "  Phantom %u: USCTiles=%u", 2) \
+X(136, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ISOLATION_CONF_OFF_DEPRECATED, "Isolation grouping is disabled", 0) \
+X(137, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ISOLATION_CONF_DEPRECATED, "Isolation group configured with a priority threshold of %u", 1) \
+X(138, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_ONLINE_DEPRECATED, "OS %u has come online", 1) \
+X(139, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_OFFLINE_DEPRECATED, "OS %u has gone offline", 1) \
+X(140, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWCONTEXT_SIGNAL_REKICK, "Signalled the previously stalled FWCtx: 0x%08.8x, OSId: %u, Signal Address: 0x%08x%08x", 4) \
+X(141, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSETS_DEPRECATED, "TDM Queue: FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 7) \
+X(142, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSET_READ_RESET, "Reset TDM Queue Read Offset: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u becomes 0, Woff = %u, Size = %u)", 6) \
+X(143, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UMQ_MISMATCHED_READ_OFFSET, "User Mode Queue mismatched stream start: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u, StreamStartOffset = %u)", 5) \
+X(144, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_DEINIT, "GPU deinit", 0) \
+X(145, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNITS_DEINIT, "GPU units deinit", 0) \
+X(146, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CONFIG_DEPRECATED, "Initialised OS %u with config flags 0x%08x", 2) \
+X(147, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_LIMIT, "UFO limit exceeded %u/%u", 2) \
+X(148, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_62850KICK, "3D Dummy stencil store", 0) \
+X(149, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CONFIG_DEPRECATED2, "Initialised OS %u with config flags 0x%08x and extended config flags 0x%08x", 3) \
+X(150, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_COMMAND_DEPRECATED, "Unknown Command (eCmdType=0x%08x)", 1) \
+X(151, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE, "UFO forced update: FWCtx 0x%08.8x @ %u [0x%08.8x] = 0x%08.8x", 4) \
+X(152, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE_NOP_DEPRECATED, "UFO forced update NOP: FWCtx 0x%08.8x @ %u [0x%08.8x] = 0x%08.8x, reason %u", 5) \
+X(153, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN66075_CHECK, "TDM context switch check: Roff %u points to 0x%08x, Match=%u", 3) \
+X(154, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CCBS_DEPRECATED, "Driver ID %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x", 6) \
+X(155, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWIRQ, "FW IRQ # %u @ %u", 2) \
+X(156, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_SET, "Setting breakpoint: Addr 0x%08.8x DM%u usc_breakpoint_ctrl_dm = %u", 3) \
+X(157, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KERNEL_CCB_DEPRECATED, "Invalid KCCB setup for OSid %u: KCCB 0x%08x, KCCB Ctrl 0x%08x", 3) \
+X(158, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KERNEL_CCB_CMD, "Invalid KCCB cmd (%u) for Driver ID %u @ KCCB 0x%08x", 3) \
+X(159, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_FAULT_DEPRECATED, "FW FAULT: At line %u in file 0x%08x%08x, additional data=0x%08x", 4) \
+X(160, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_INVALID, "Invalid breakpoint: MemCtx 0x%08x Addr 0x%08.8x DM%u usc_breakpoint_ctrl_dm = %u", 4) \
+X(161, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FLUSHINVAL_CMD_INVALID_DEPRECATED, "Discarding invalid SLC flushinval command for OSid %u: DM %u, FWCtx 0x%08x", 3) \
+X(162, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NOTIFY_WRITE_OFFSET_UPDATE_DEPRECATED, "Invalid Write Offset update notification from OSid %u to DM %u: FWCtx 0x%08x, MemCtx 0x%08x", 4) \
+X(163, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KCCB_KICK_CMD_DEPRECATED, "Null FWCtx in KCCB kick cmd for OSid %u: KCCB 0x%08x, ROff %u, WOff %u", 4) \
+X(164, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FULL_CHPTCCB, "Checkpoint CCB for Driver ID %u is full, signalling host for full check state (Roff = %u, Woff = %u)", 3) \
+X(165, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_INIT_CCBS_DEPRECATED2, "OSid %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x chptCCBCtl@0x%x chptCCB@0x%x", 8) \
+X(166, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_STATE_CHANGE_DEPRECATED, "Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-offloading 4-cooldown). Status %u (1-ok 0-fail)", 4) \
+X(167, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_STALE_KCCB_CMDS, "Driver ID %u has %u stale commands in its KCCB", 2) \
+X(168, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_VCE_PAUSE, "Applying VCE pause", 0) \
+X(169, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KCCB_UPDATE_RTN_SLOT_DEPRECATED, "OSid %u KCCB slot %u value updated to %u", 3) \
+X(170, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_KCCB_COMMAND, "Unknown KCCB Command: KCCBCtl=0x%08x, KCCB=0x%08x, Roff=%u, Woff=%u, Wrap=%u, Cmd=0x%08x, CmdType=0x%08x", 7) \
+X(171, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_CCB_COMMAND1, "Unknown Client CCB Command processing fences: FWCtx=0x%08x, CCBCtl=0x%08x, CCB=0x%08x, Roff=%u, Doff=%u, Woff=%u, Wrap=%u, CmdHdr=0x%08x, CmdType=0x%08x, CmdSize=%u", 10) \
+X(172, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UNKNOWN_CCB_COMMAND2, "Unknown Client CCB Command executing kick: FWCtx=0x%08x, CCBCtl=0x%08x, CCB=0x%08x, Roff=%u, Doff=%u, Woff=%u, Wrap=%u, CmdHdr=0x%08x, CmdType=0x%08x, CmdSize=%u", 10) \
+X(173, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_KCCB_KICK_CMD, "Null FWCtx in KCCB kick cmd for Driver ID %u with WOff %u", 2) \
+X(174, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FLUSHINVAL_CMD_INVALID, "Discarding invalid SLC flushinval command for Driver ID %u, FWCtx 0x%08x", 2) \
+X(175, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NOTIFY_WRITE_OFFSET_UPDATE, "Invalid Write Offset update notification from Driver ID %u: FWCtx 0x%08x, MemCtx 0x%08x", 3) \
+X(176, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_INIT_CONFIG, "Initialised Firmware with config flags 0x%08x and extended config flags 0x%08x", 2) \
+X(177, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_CONFIG, "Set Periodic Hardware Reset Mode: %u", 1) \
+X(179, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_TRIG, "PHR mode %u, FW state: 0x%08x, HWR flags: 0x%08x", 3) \
+X(180, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PHR_RESET_DEPRECATED, "PHR mode %u triggered a reset", 1) \
+X(181, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SIGNAL_UPDATE, "Signal update, Snoop Filter: %u, Signal Id: %u", 2) \
+X(182, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DEV_SERIES8_DEPRECATED, "WARNING: Skipping FW KCCB Cmd type %u which is not yet supported on Series8.", 1) \
+X(183, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INCONSISTENT_MMU_FLAGS, "MMU context cache data NULL, but cache flags=0x%x (sync counter=%u, update value=%u) OSId=%u", 4) \
+X(184, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SLC_FLUSH, "SLC range based flush: Context=%u VAddr=0x%02x%08x, Size=0x%08x, Invalidate=%u", 5) \
+X(185, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBSC_INVAL_DEPRECATED, "FBSC invalidate for Context Set [0x%08x]: Entry mask 0x%08x%08x.", 3) \
+X(186, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN66284_UPDATE, "TDM context switch check: Roff %u was not valid for kick starting at %u, moving back to %u", 3) \
+X(187, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SPFILTER_UPDATES, "Signal updates: FIFO: %u, Signals: 0x%08x", 2) \
+X(188, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_FBSC_CMD, "Invalid FBSC cmd: FWCtx 0x%08x, MemCtx 0x%08x", 2) \
+X(189, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BRN68497_BLIT, "Insert BRN68497 WA blit after TDM Context store.", 0) \
+X(190, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PENDING_UFO_UPDATE_START, "UFO Updates for previously finished FWCtx 0x%08.8x", 1) \
+X(191, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RTC_RTA_PRESENT, "RTC with RTA present, %u active render targets", 1) \
+X(192, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NULL_RTAS, "Invalid RTA Set-up. The ValidRenderTargets array in RTACtl is Null!", 0) \
+X(193, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_COUNTER, "Block 0x%x / Counter 0x%x INVALID and ignored", 2) \
+X(194, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ECC_FAULT_DEPRECATED, "ECC fault GPU=0x%08x FW=0x%08x", 2) \
+X(195, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_PROCESS_XPU_EVENT, "Processing XPU event on DM = %u", 1) \
+X(196, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_VZ_WDG_TRIGGER_DEPRECATED, "Driver ID %u failed to respond to the virtualisation watchdog in time. Timestamp of its last input = %u", 2) \
+X(197, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP, "GPU-%u has locked up (see HWR logs for more info)", 1) \
+X(198, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UPDATE_TILES_IN_FLIGHT, "Updating Tiles In Flight (Dusts=%u, PartitionMask=0x%08x, ISPCtl=0x%08x)", 3) \
+X(199, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_LOCKUP_DM, "GPU has locked up (see HWR logs for more info)", 0) \
+X(200, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_REPROCESS_XPU_EVENTS, "Reprocessing outstanding XPU events from cores 0x%02x", 1) \
+X(201, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SECONDARY_XPU_EVENT, "Secondary XPU event on DM=%u, CoreMask=0x%02x, Raised=0x%02x", 3) \
+X(202, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_OFFSETS, "TDM Queue: Core %u, FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 8) \
+X(203, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_BUFFER_STALL, "TDM stalled Core %u (Roff = %u, Woff = %u)", 3) \
+X(204, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_OFFSETS, "Compute Queue: Core %u, FWCtx 0x%08.8x, prio: %d, queue: 0x%08x%08x (Roff = %u, Woff = %u, Size = %u)", 8) \
+X(205, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_STALLED, "Compute stalled core %u (Roff = %u, Woff = %u, Size = %u)", 4) \
+X(206, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UMQ_MISMATCHED_CORE_READ_OFFSET, "User Mode Queue mismatched stream start: Core %u, FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u, StreamStartOffset = %u)", 6) \
+X(207, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_RESUMED_FROM_STALL, "TDM resumed core %u (Roff = %u, Woff = %u)", 3) \
+X(208, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_CORE_RESUMED_FROM_STALL, "Compute resumed core %u (Roff = %u, Woff = %u, Size = %u)", 4) \
+X(209, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_MTS_PERMISSION_CHANGED_DEPRECATED, " Updated permission for Driver ID %u to perform MTS kicks: %u (1 = allowed, 0 = not allowed)", 2) \
+X(210, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TEST1, "Mask = 0x%X, mask2 = 0x%X", 2) \
+X(211, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TEST2, "  core %u, reg = %u, mask = 0x%X)", 3) \
+X(212, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ECC_FAULT_SAFETY_BUS, "ECC fault received from safety bus: 0x%08x", 1) \
+X(213, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SAFETY_WDG_CONFIG, "Safety Watchdog threshold period set to 0x%x clock cycles", 1) \
+X(214, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SAFETY_WDG_TRIGGER, "MTS Safety Event triggered by the safety watchdog.", 0) \
+X(215, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_USC_TASKS_RANGE, "DM%u USC tasks range limit 0 - %u, stride %u", 3) \
+X(216, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_ECC_FAULT, "ECC fault GPU=0x%08x", 1) \
+X(217, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GPU_SAFETY_RESET, "GPU Hardware units reset to prevent transient faults.", 0) \
+X(218, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ABORTCMD, "Kick Abort cmd: FWCtx 0x%08.8x @ %u", 2) \
+X(219, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_DEPRECATED, "Kick Ray: FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 7)\
+X(220, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_FINISHED_DEPRECATED, "Ray finished", 0) \
+X(221, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FWDATA_INIT_STATUS, "State of firmware's private data at boot time: %u (0 = uninitialised, 1 = initialised); Fw State Flags = 0x%08X", 2) \
+X(222, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CFI_TIMEOUT, "CFI Timeout detected (%u increasing to %u)", 2) \
+X(223, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CFI_TIMEOUT_FBM, "CFI Timeout detected for FBM (%u increasing to %u)", 2) \
+X(224, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GEOM_OOM_DISALLOWED, "Geom OOM event not allowed", 0) \
+X(225, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED2, "Changing Driver ID %u's priority from %u to %u; Isolation = %u (0 = off; 1 = on)", 4) \
+X(226, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SKIP_ALREADY_RUN_GEOM, "Skipping already executed TA FWCtx 0x%08.8x @ %u", 2) \
+X(227, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ATTEMPT_TO_RUN_AHEAD_GEOM, "Attempt to execute TA FWCtx 0x%08.8x @ %u ahead of time on other GEOM", 2) \
+X(228, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_DEPRECATED2, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(229, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TA_PIPELINE, "Kick TA: Kick ID %u FWCtx 0x%08.8x @ %u, RTD 0x%08x, First kick:%u, Last kick:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 12) \
+X(230, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_3D_PIPELINE, "Kick 3D: Kick ID %u FWCtx 0x%08.8x @ %u, RTD 0x%08x, Partial render:%u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(231, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_PIPELINE_DEPRECATED, "Kick Compute: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 7) \
+X(232, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_FINISHED_PIPELINE, "TDM finished: Kick ID %u", 1) \
+X(233, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_FINISHED_PIPELINE, "TA finished: Kick ID %u", 1) \
+X(234, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_FINISHED_PIPELINE, "3D finished: Kick ID %u, HWRTData0State=%x, HWRTData1State=%x", 3) \
+X(235, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_COMPUTE_FINISHED_PIPELINE, "Compute finished: Kick ID %u", 1) \
+X(236, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_PIPELINE_DEPRECATED, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u, Base 0x%08x%08x. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 10) \
+X(237, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_PIPELINE_DEPRECATED, "Kick Ray: Kick ID %u FWCtx 0x%08.8x @ %u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8)\
+X(238, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_FINISHED_PIPELINE, "Ray finished: Kick ID %u", 1) \
+X(239, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RT_UNITS_INIT, "GPU RT Units init (# mask: 0x%08x%08x)", 2) \
+X(240, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_CHECK_PENDING_PASS, "UFO Check: [0x%08.8x] is pending update to 0x%08.8x and therefore passes", 2) \
+X(241, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_PR_CHECK_PENDING_PASS, "UFO PR-Check: [0x%08.8x] is pending update to 0x%08.8x and therefore passes", 2) \
+X(242, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DELAY_DM_TO_OVERLAP_PIPES, "Holding kick of DM %u pipe %u to encourage pipeline overlap", 2) \
+X(243, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RELEASE_DM_PIPE, "Releasing kick for DM %u pipe %u", 2) \
+X(244, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_PRIORITY_CHANGE_DEPRECATED3, "Changing Driver ID %u's priority from %u to %u", 3) \
+X(245, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_ISOLATION_GROUP_CHANGE_DEPRECATED, "Changing Driver ID %u's isolation group from %u to %u", 3) \
+X(246, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_VK_TIMESTAMP, "VK Timestamp: addr=0x%08x%08x, avail=0x%08x%08x stamp=0x%08x%08x", 6) \
+X(247, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SET_HCS_TRIGGERED, "DM %u failed to Context Switch on time (Current time: 0x%08x%08x, deadline: 0x%08x%08x). Triggered HCS (see HWR logs).", 5) \
+X(248, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_HWR_HIT_POLLFAILURE, "HWR has been triggered - GPU has failed a poll, RGX_CR_EVENT_STATUS=0x%08x (see HWR logs)", 1) \
+X(249, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBCDC_FAILURE_DETECTED_DEPRECATED, "FBCDC signature failure detected so block scheduling more work", 0) \
+X(250, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBCDC_FAILURE_CLEARED_DEPRECATED, "FBCDC signature cleared which unlocks scheduling more work", 0) \
+X(251, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FW_FAULT, "FW FAULT: At line %u in file 0x%08x%08x, additional data=0x%08x%08x", 5) \
+X(252, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_UFO_FORCED_UPDATE_NOP, "UFO forced update NOP: FWCtx 0x%08.8x @ %u, reason %u", 3) \
+X(253, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_CONTEXT_STORED, "TDM FWCtx:0x%08.8x stored", 1) \
+X(254, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CDM_CONTEXT_STORED, "CDM FWCtx:0x%08.8x stored, resumeMask:0x%08x", 2) \
+X(255, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_GEOM_CONTEXT_STORED, "GEOM FWCtx:0x%08.8x stored", 1) \
+X(256, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_CONTEXT_STORED, "3D FWCtx:0x%08.8x stored, resumeMask:0x%08x", 2) \
+X(257, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RDM_CONTEXT_STORED, "RAY FWCtx:0x%08.8x stored", 1) \
+X(258, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM, "Kick TDM: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 8) \
+X(259, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE, "Kick Compute: FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 7) \
+X(260, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_COMPUTE_PIPELINE, "Kick Compute: Kick ID %u FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, ext:0x%08x, int:0x%08x)", 8) \
+X(261, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_TDM_PIPELINE, "Kick TDM: Kick ID %u FWCtx 0x%08.8x @ %u, Base 0x%08x%08x, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 11) \
+X(262, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KICK_RAY_PIPELINE, "Kick Ray: Kick ID %u FWCtx 0x%08.8x @ %u, CSW resume:%u. (PID:%u, prio:%d, frame:%u, ext:0x%08x, int:0x%08x)", 9)\
+X(263, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DEPTH, "3D Disable Depth. ExtJobRef = 0x%08x", 1) \
+X(264, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_STENCIL, "3D Disable Stencil. ExtJobRef = 0x%08x", 1) \
+X(265, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DS_IN_3D_RUNNING, "3D Disable DS in 3D running. RenderContext 0x%08.8x ExtJobRef 0x%08x", 2) \
+X(266, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISABLE_DS_IN_KICK_3D, "3D Disable DS in kick 3D. RenderContext 0x%08.8x ExtJobRef 0x%08x", 2) \
+X(267, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ADD_DISABLE_DS_QUEUE, "Add disable DS in queue. RenderContext 0x%08.8x DisableJobRef 0x%08x Uncheck %u", 3) \
+X(268, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_NOT_FIND_USABLE_DS_IN_QUEUE, "Not find usable DS in queue. RenderContext 0x%08.8x ExtJobRef 0x%08x DisableJobRef 0x%08x Uncheck %u index %u", 5) \
+X(269, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_BREAKPOINT_LIST_FULL, "Unable to set breakpoint for MemCtx 0x%08x as the breakpoint list is full.", 1) \
+X(270, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_INVALID_NUM_LOG_PARAMS, "Invalid number of log parameters passed! (Group:%u ID:%u Params:%u Passed:%u)", 4) \
+X(271, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TDM_CANCEL_PIPELINE, "TDM cancelled: Kick ID %u", 1) \
+X(272, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TA_CANCEL_PIPELINE, "TA cancelled: Kick ID %u", 1) \
+X(273, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_3D_CANCEL_PIPELINE, "3D cancelled: Kick ID %u", 1) \
+X(274, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CDM_CANCEL_PIPELINE, "Compute cancelled: Kick ID %u", 1) \
+X(275, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_RAY_CANCEL_PIPELINE, "Ray cancelled: Kick ID %u", 1) \
+X(276, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_MAX_DEPRECATED, "Time Slice: Update Failed, Max total allocations exceeded 100pc", 0)\
+X(277, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_DYNAMIC_LOW_DEPRECATED, "Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 0)\
+X(278, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_UPDATE_SUCCESS_DEPRECATED, "Time Slice: Updated Successfully", 0)\
+X(279, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_DISCARD_FWCCB, "FWCCB for Driver ID %u is full, discarding command! (Roff = %u, Woff = %u)", 3) \
+X(280, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_RECEIVED_DEPRECATED, "Cancelling jobs with intjobref<0x%08x, DM = %u, FWCtx = 0x%08.8x", 3) \
+X(281, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_RECEIVED, "Cancelling jobs with intjobref from 0x%08x to 0x%08x, DM = %u, FWCtx = 0x%08.8x", 4) \
+X(282, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_MAX_DEPRECATED2, "Time Slice: Update Failed, Max total allocations exceeded 100pc", 0)\
+X(283, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_DYNAMIC_LOW_DEPRECATED2, "Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 0)\
+X(284, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_TIME_SLICE_UPDATE_SUCCESS_DEPRECATED2, "Time Slice: Updated Successfully", 0)\
+X(285, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_OS_STATE_CHANGE_DEPRECATED2, "Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-graceful_offloading 4-forced_offloading 5-cooldown). Status %u (1-ok 0-fail)", 4) \
+X(286, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FBSC_INVAL, "FBSC invalidate for Context Set [0x%08x]", 1) \
+X(287, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_CANCEL_WORK_CMD_DROPPED, "Dropped cancel request with intjobref from 0x%08x to 0x%08x, DM = %u, FWCtx = 0x%08.8x", 4) \
+X(288, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_ZERO_LM, "Zeroing local memory after context storing FWCtx = 0x%08.8x", 1) \
+X(289, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_FAILED_IRQ_CLEAR, "Host failed to clear IRQ DriverID: %u Reg: 0x%08x",2) \
+X(290, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_KILLDM_ABORTED, "Kill DM%u aborted", 1) \
+X(291, RGXFW_GROUP_MAIN, RGXFW_SF_MAIN_SOC_CLOCK_SPEED_CHANGE, "SOC clock set to %u Hz", 1) \
+\
+X(  1, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK_DEPRECATED, "Bg Task DM = %u, counted = %u", 2) \
+X(  2, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_COMPLETE_DEPRECATED, "Bg Task complete DM = %u", 1) \
+X(  3, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_KICK_DEPRECATED, "Irq Task DM = %u, Breq = %u, SBIrq = 0x%x", 3) \
+X(  4, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_COMPLETE_DEPRECATED, "Irq Task complete DM = %u", 1) \
+X(  5, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KICK_MTS_BG_ALL_DEPRECATED, "Kick MTS Bg task DM=All", 0) \
+X(  6, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KICK_MTS_IRQ, "Kick MTS Irq task DM=%u", 1) \
+X(  7, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYCELLTYPE_DEPRECATED, "Ready queue debug DM = %u, celltype = %u", 2) \
+X(  8, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYTORUN_DEPRECATED, "Ready-to-run debug DM = %u, item = 0x%x", 2) \
+X(  9, RGXFW_GROUP_MTS, RGXFW_SF_MTS_CMDHEADER, "Client command header DM = %u, client CCB = 0x%x, cmd = 0x%x", 3) \
+X( 10, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYTORUN, "Ready-to-run debug Driver ID = %u, DM = %u, item = 0x%x", 3) \
+X( 11, RGXFW_GROUP_MTS, RGXFW_SF_MTS_READYCELLTYPE_DEPRECATED2, "Ready queue debug DM = %u, celltype = %u, OSid = %u", 3) \
+X( 12, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK_DEPRECATED2, "Bg Task DM = %u, counted = %u, OSid = %u", 3) \
+X( 13, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_COMPLETE, "Bg Task complete DM Bitfield: %u", 1) \
+X( 14, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_COMPLETE, "Irq Task complete.", 0) \
+X( 15, RGXFW_GROUP_MTS, RGXFW_SF_MTS_CMD_DISCARD, "Discarded Command Type: %u OS ID = %u PID = %u context = 0x%08x cccb ROff = 0x%x, due to USC breakpoint hit by OS ID = %u PID = %u.", 7) \
+X( 16, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_EXEC_DEPRECATED, "KCCB Slot %u: DM=%u, Cmd=0x%08x, OSid=%u", 4) \
+X( 17, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_RTN_VALUE, "KCCB Slot %u: Return value %u", 2) \
+X( 18, RGXFW_GROUP_MTS, RGXFW_SF_MTS_BG_KICK, "Bg Task Driver ID = %u", 1) \
+X( 19, RGXFW_GROUP_MTS, RGXFW_SF_MTS_KCCBCMD_EXEC, "KCCB Slot %u: Cmd=0x%08x, Driver ID=%u", 3) \
+X( 20, RGXFW_GROUP_MTS, RGXFW_SF_MTS_IRQ_KICK, "Irq Task (EVENT_STATUS=0x%08x)", 1) \
+X( 21, RGXFW_GROUP_MTS, RGXFW_SF_MTS_VZ_SIDEBAND, "VZ sideband test, kicked with DriverID=%u from MTS, OSid for test=%u", 2) \
+\
+X(  1, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FWCTX_CLEANUP, "FwCommonContext [0x%08x] cleaned", 1) \
+X(  2, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FWCTX_BUSY, "FwCommonContext [0x%08x] is busy: ReadOffset = %u, WriteOffset = %u", 3) \
+X(  3, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANUP_DEPRECATED, "HWRTData [0x%08x] for DM=%u, received cleanup request", 2) \
+X(  4, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED_FOR_DM_DEPRECATED, "HWRTData [0x%08x] HW Context cleaned for DM%u, executed commands = %u", 3) \
+X(  5, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY_DEPRECATED, "HWRTData [0x%08x] HW Context for DM%u is busy", 2) \
+X(  6, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED_DEPRECATED, "HWRTData [0x%08x] HW Context %u cleaned", 2) \
+X(  7, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_FL_CLEANED, "Freelist [0x%08x] cleaned", 1) \
+X(  8, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_ZSBUFFER_CLEANED, "ZSBuffer [0x%08x] cleaned", 1) \
+X(  9, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_ZSBUFFER_BUSY, "ZSBuffer [0x%08x] is busy: submitted = %u, executed = %u", 3) \
+X( 10, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY_DEPRECATED2, "HWRTData [0x%08x] HW Context for DM%u is busy: submitted = %u, executed = %u", 4) \
+X( 11, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANUP_DEPRECATED, "HW Ray Frame data [0x%08x] for DM=%u, received cleanup request", 2) \
+X( 12, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANED_FOR_DM_DEPRECATED, "HW Ray Frame Data [0x%08x] cleaned for DM%u, executed commands = %u", 3) \
+X( 13, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_BUSY_DEPRECATED, "HW Ray Frame Data [0x%08x] for DM%u is busy: submitted = %u, executed = %u", 4) \
+X( 14, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRFD_CLEANED_DEPRECATED, "HW Ray Frame Data [0x%08x] HW Context %u cleaned", 2) \
+X( 15, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_INVALID_REQUEST, "Discarding invalid cleanup request of type 0x%x", 1) \
+X( 16, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANUP, "Received cleanup request for HWRTData [0x%08x]", 1) \
+X( 17, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_BUSY, "HWRTData [0x%08x] HW Context is busy: submitted = %u, executed = %u", 3) \
+X( 18, RGXFW_GROUP_CLEANUP, RGXFW_SF_CLEANUP_HWRTD_CLEANED, "HWRTData [0x%08x] HW Context %u cleaned, executed commands = %u", 3) \
+\
+X(  1, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_NEEDS_RESUME, "CDM FWCtx 0x%08.8x needs resume", 1) \
+X(  2, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_RESUME_DEPRECATED, "*** CDM FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X(  3, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_SHARED, "CDM FWCtx shared alloc size load 0x%x", 1) \
+X(  4, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_COMPLETE, "*** CDM FWCtx store complete", 0) \
+X(  5, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_START, "*** CDM FWCtx store start", 0) \
+X(  6, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_SOFT_RESET, "CDM Soft Reset", 0) \
+X(  7, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_NEEDS_RESUME, "3D FWCtx 0x%08.8x needs resume", 1) \
+X(  8, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME, "*** 3D FWCtx 0x%08.8x resume", 1) \
+X(  9, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_COMPLETE, "*** 3D context store complete", 0) \
+X( 10, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_DEPRECATED, "3D context store pipe state: 0x%08.8x 0x%08.8x 0x%08.8x", 3) \
+X( 11, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_START, "*** 3D context store start", 0) \
+X( 12, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_TQ_RESUME, "*** 3D TQ FWCtx 0x%08.8x resume", 1) \
+X( 13, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_NEEDS_RESUME, "TA FWCtx 0x%08.8x needs resume", 1) \
+X( 14, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_RESUME, "*** TA FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X( 15, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_SHARED, "TA context shared alloc size store 0x%x, load 0x%x", 2) \
+X( 16, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_COMPLETE, "*** TA context store complete", 0) \
+X( 17, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_START, "*** TA context store start", 0) \
+X( 18, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM_DEPRECATED, "Higher priority context requests DM %u, old prio:%d, new prio:%d", 3) \
+X( 19, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SET_CONTEXT_PRIORITY, "Set FWCtx 0x%x priority to %u", 2) \
+X( 20, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_DEPRECATED2, "3D context store pipe%u state: 0x%08.8x", 2) \
+X( 21, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE_DEPRECATED, "3D context resume pipe%u state: 0x%08.8x", 2) \
+X( 22, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_NEEDS_RESUME_DEPRECATED, "SHG FWCtx 0x%08.8x needs resume", 1) \
+X( 23, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_RESUME_DEPRECATED, "*** SHG FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x", 3) \
+X( 24, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_SHARED_DEPRECATED, "SHG context shared alloc size store 0x%x, load 0x%x", 2) \
+X( 25, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_STORE_COMPLETE_DEPRECATED, "*** SHG context store complete", 0) \
+X( 26, RGXFW_GROUP_CSW, RGXFW_SF_CSW_SHG_STORE_START_DEPRECATED, "*** SHG context store start", 0) \
+X( 27, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_PIPE_INDIRECT, "Performing TA indirection, last used pipe %u", 1) \
+X( 28, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_STORE_CTRL_STREAM_TERMINATE, "CDM context store hit ctrl stream terminate. Skip resume.", 0) \
+X( 29, RGXFW_GROUP_CSW, RGXFW_SF_CSW_CDM_RESUME_AB_BUFFER, "*** CDM FWCtx 0x%08.8x resume from snapshot buffer 0x%08x%08x, shader state %u", 4) \
+X( 30, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STATE_BUFFER_FLIP, "TA PDS/USC state buffer flip (%u->%u)", 2) \
+X( 31, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_52563_HIT_DEPRECATED, "TA context store hit BRN 52563: vertex store tasks outstanding", 0) \
+X( 32, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_USC_POLL_FAILED, "TA USC poll failed (USC vertex task count: %u)", 1) \
+X( 33, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TA_STORE_DEFERRED_DEPRECATED, "TA context store deferred due to BRN 54141.", 0) \
+X( 34, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM_DEPRECATED2, "Higher priority context requests DM %u. Prios (OSid, OSid Prio, Context Prio): Current: %u, %u, %u New: %u, %u, %u", 7) \
+X( 35, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_START, "*** TDM context store start", 0) \
+X( 36, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_COMPLETE, "*** TDM context store complete", 0) \
+X( 37, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_STORE_NEEDS_RESUME_DEPRECATED, "TDM context needs resume, header [0x%08.8x, 0x%08.8x]", 2) \
+X( 38, RGXFW_GROUP_CSW, RGXFW_SF_CSW_HIGHER_PRIORITY_REQUESTS_DM, "Higher priority context requests DM %u. Prios (Driver ID, Driver ID Prio, Context Prio): Current: %u, %u, %u New: %u, %u, %u. Hard Context Switching: %u", 8) \
+X( 39, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE, "3D context store pipe %2d (%2d) state: 0x%08.8x", 3) \
+X( 40, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE, "3D context resume pipe %2d (%2d) state: 0x%08.8x", 3) \
+X( 41, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_START_VOLCANIC, "*** 3D context store start version %u (1=IPP_TILE, 2=ISP_TILE)", 1) \
+X( 42, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_STORE_PIPE_STATE_VOLCANIC, "3D context store pipe%u state: 0x%08.8x%08x", 3) \
+X( 43, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_PIPE_STATE_VOLCANIC, "3D context resume pipe%u state: 0x%08.8x%08x", 3) \
+X( 44, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_RESUME_IPP_STATE,  "3D context resume IPP state: 0x%08.8x%08x", 2) \
+X( 45, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_PIPES_EMPTY, "All 3D pipes empty after ISP tile mode store! IPP_status: 0x%08x", 1) \
+X( 46, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_RESUME_PIPE_STATE_DEPRECATED, "TDM context resume pipe%u state: 0x%08.8x%08x", 3) \
+X( 47, RGXFW_GROUP_CSW, RGXFW_SF_CSW_3D_LEVEL4_STORE_START, "*** 3D context store start version 4", 0) \
+X( 48, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RESUME_MULTICORE, "Multicore context resume on DM%u active core mask 0x%04.4x", 2) \
+X( 49, RGXFW_GROUP_CSW, RGXFW_SF_CSW_STORE_MULTICORE, "Multicore context store on DM%u active core mask 0x%04.4x", 2) \
+X( 50, RGXFW_GROUP_CSW, RGXFW_SF_CSW_TDM_RESUME_PIPE_STATE, "TDM context resume Core %u, pipe%u state: 0x%08.8x%08x%08x", 5) \
+X( 51, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_STORE_COMPLETE, "*** RDM FWCtx store complete", 0) \
+X( 52, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_STORE_START, "*** RDM FWCtx store start", 0) \
+X( 53, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_NEEDS_RESUME, "RDM FWCtx 0x%08.8x needs resume", 1) \
+X( 54, RGXFW_GROUP_CSW, RGXFW_SF_CSW_RDM_RESUME, "RDM FWCtx 0x%08.8x resume", 1) \
+\
+X(  1, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_BIFREQ_DEPRECATED, "Activate MemCtx=0x%08x BIFreq=%u secure=%u", 3) \
+X(  2, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE, "Deactivate MemCtx=0x%08x", 1) \
+X(  3, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCREG_ALLOC_DEPRECATED, "Alloc PC reg %u", 1) \
+X(  4, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_GRAB, "Grab reg set %u refcount now %u", 2) \
+X(  5, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_UNGRAB_DEPRECATED, "Ungrab reg set %u refcount now %u", 2) \
+X(  6, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_BIFREQ_DEPRECATED, "Setup reg=%u BIFreq=%u, expect=0x%08x%08x, actual=0x%08x%08x", 6) \
+X(  7, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TRUST_DEPRECATED, "Trust enabled:%u, for BIFreq=%u", 2) \
+X(  8, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TILECFG_DEPRECATED, "BIF Tiling Cfg %u base 0x%08x%08x len 0x%08x%08x enable %u stride %u --> 0x%08x%08x", 9) \
+X(  9, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSID0, "Wrote the Value %u to OSID0, Cat Base %u, Register's contents are now 0x%08x 0x%08x", 4) \
+X( 10, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSID1, "Wrote the Value %u to OSID1, Context  %u, Register's contents are now 0x%04x", 3) \
+X( 11, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSIDx_DEPRECATED, "ui32OSid = %u, Catbase = %u, Reg Address = 0x%x, Reg index = %u, Bitshift index = %u, Val = 0x%08x%08x", 7) \
+X( 12, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY_BIFREQ_DEPRECATED, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u, BIFREQ %u", 5) \
+X( 13, RGXFW_GROUP_BIF, RGXFW_SF_BIF_UNMAP_GPU_MEMORY, "Unmap GPU memory (event status 0x%x)", 1) \
+X( 14, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_DM, "Activate MemCtx=0x%08x DM=%u secure=%u", 3) \
+X( 15, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM_DEPRECATED, "Setup reg=%u DM=%u, expect=0x%08x%08x, actual=0x%08x%08x", 6) \
+X( 16, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u", 4) \
+X( 17, RGXFW_GROUP_BIF, RGXFW_SF_BIF_TRUST_DM, "Trust enabled:%u, for DM=%u", 2) \
+X( 18, RGXFW_GROUP_BIF, RGXFW_SF_BIF_MAP_GPU_MEMORY_DM, "Map GPU memory DevVAddr 0x%x%08x, Size %u, Context ID %u, DM %u", 5) \
+X( 19, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM, "Setup register set=%u DM=%u, PC address=0x%08x%08x, OSid=%u, NewPCRegRequired=%u", 6) \
+X( 20, RGXFW_GROUP_BIF, RGXFW_SF_BIF_PCSET_ALLOC, "Alloc PC set %u as register range [%u - %u]", 3) \
+X( 21, RGXFW_GROUP_BIF, RGXFW_SF_BIF_OSIDx, "ui32CoreID = %u, ui32OSid = %u, Catbase = %u, Reg Address = 0x%x, Reg index = %u, Bitshift index = %u, Val = 0x%08x%08x", 8) \
+X( 22, RGXFW_GROUP_BIF, RGXFW_SF_BIF_SETUP_REG_DM_SECURE, "Setup secure register=%u DM=%u, PC address=0x%08x%08x, OSid=%u, NewContext=%u", 6) \
+X( 23, RGXFW_GROUP_BIF, RGXFW_SF_BIF_ACTIVATE_DM_DEPRECATED, "Activate MemCtx=0x%08x DM=%u secure=%u CtxFlags=0x%08x", 4) \
+X( 24, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE_DEPRECATED, "Deactivate MemCtx=0x%08x CtxFlags=0x%08x", 2) \
+X( 25, RGXFW_GROUP_BIF, RGXFW_SF_BIF_DEACTIVATE_AND_UNGRAB_PCSET, "Deactivate MemCtx=0x%08x, ungrab reg set %u refcount now %u", 3) \
+\
+X(  1, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_WRITE, "GPIO write 0x%02x", 1) \
+X(  2, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_READ, "GPIO read 0x%02x", 1) \
+X(  3, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_ENABLED, "GPIO enabled", 0) \
+X(  4, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_DISABLED, "GPIO disabled", 0) \
+X(  5, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_STATUS, "GPIO status=%u (0=OK, 1=Disabled)", 1) \
+X(  6, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_READ, "GPIO_AP: Read address=0x%02x (%u byte(s))", 2) \
+X(  7, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_WRITE, "GPIO_AP: Write address=0x%02x (%u byte(s))", 2) \
+X(  8, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_TIMEOUT, "GPIO_AP timeout!", 0) \
+X(  9, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_AP_ERROR, "GPIO_AP error. GPIO status=%u (0=OK, 1=Disabled)", 1) \
+X( 10, RGXFW_GROUP_MISC, RGXFW_SF_MISC_GPIO_ALREADY_READ, "GPIO already read 0x%02x", 1) \
+X( 11, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_CHECK_BUFFER_AVAILABLE, "SR: Check buffer %u available returned %u", 2) \
+X( 12, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_WAITING_BUFFER_AVAILABLE, "SR: Waiting for buffer %u", 1) \
+X( 13, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_WAIT_BUFFER_TIMEOUT, "SR: Timeout waiting for buffer %u (after %u ticks)", 2) \
+X( 14, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_FRAME_CHECK, "SR: Skip frame check for strip %u returned %u (0=No skip, 1=Skip frame)", 2) \
+X( 15, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_REMAINING_STRIPS, "SR: Skip remaining strip %u in frame", 1) \
+X( 16, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_FRAME_SKIP_NEW_FRAME, "SR: Inform HW that strip %u is a new frame", 1) \
+X( 17, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_SKIP_FRAME_TIMEOUT, "SR: Timeout waiting for INTERRUPT_FRAME_SKIP (after %u ticks)", 1) \
+X( 18, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_STRIP_MODE, "SR: Strip mode is %u", 1) \
+X( 19, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_STRIP_INDEX, "SR: Strip Render start (strip %u)", 1) \
+X( 20, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_BUFFER_RENDERED, "SR: Strip Render complete (buffer %u)", 1) \
+X( 21, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SR_BUFFER_FAULT, "SR: Strip Render fault (buffer %u)", 1) \
+X( 22, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_DEPRECATED2, "TRP state: %u", 1) \
+X( 23, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_FAILURE_DEPRECATED, "TRP failure: %u", 1) \
+X( 24, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SW_TRP_STATE_DEPRECATED, "SW TRP State: %u", 1) \
+X( 25, RGXFW_GROUP_MISC, RGXFW_SF_MISC_SW_TRP_FAILURE_DEPRECATED2, "SW TRP failure: %u", 1) \
+X( 26, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HW_KICK_DEPRECATED, "HW kick event (%u)", 1) \
+X( 27, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WGP_CHECKSUMS_DEPRECATED, "GPU core (%u/%u): checksum 0x%08x vs. 0x%08x", 4) \
+X( 28, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WGP_UNIT_CHECKSUMS_DEPRECATED, "GPU core (%u/%u), unit (%u,%u): checksum 0x%08x vs. 0x%08x", 6) \
+X( 29, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_CHECK_REG_DEPRECATED, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 30, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_USC_SLOTS_CHECK_DEPRECATED, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X( 31, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HWR_USC_REG_CHECK_DEPRECATED, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 32, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_HWRTDATA_DEPRECATED, "TRP HWRTData: 0x%08x, state: %u", 2) \
+X( 33, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_STATE_CNTX_DEPRECATED, "TRP Context: 0x%08x, state: %u", 2) \
+X( 34, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_FAILURE_CNTX_DEPRECATED, "TRP Context: 0x%08x, failure: %u", 2) \
+X( 35, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DEPRECATED, "Memory dump: Addr=0x%02x%08x, Size=%d, ContextId=%u, DM=%u", 5) \
+X( 36, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS4_DEPRECATED, "  0x%02x%08x  %08x %08x %08x %08x", 6) \
+X( 37, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS3_DEPRECATED, "  0x%02x%08x  %08x %08x %08x", 5) \
+X( 38, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS2_DEPRECATED, "  0x%02x%08x  %08x %08x", 4) \
+X( 39, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_GPU_DWORDS1_DEPRECATED, "  0x%02x%08x  %08x", 3) \
+X( 40, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DEPRECATED, "Memory dump: Addr=0x%08x, Size=%d", 2) \
+X( 41, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS4_DEPRECATED, "  0x%08x  %08x %08x %08x %08x", 5) \
+X( 42, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS3_DEPRECATED, "  0x%08x  %08x %08x %08x", 4) \
+X( 43, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS2_DEPRECATED, "  0x%08x  %08x %08x", 3) \
+X( 44, RGXFW_GROUP_MISC, RGXFW_SF_MISC_HEXDUMP_FW_DWORDS1_DEPRECATED, "  0x%08x  %08x", 2) \
+X( 45, RGXFW_GROUP_MISC, RGXFW_SF_MISC_FBCDC_FAILURE_STATUS_DEPRECATED, "FBCDC: Core=0x%08x, Status=0x%08x, Signature status=0x%08x", 3) \
+X( 46, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WORK_CYCLES_PIPEDM_EN, "FWCtx 0x%08.8x, PipeDM state %04x, (start) %08x, (end) %08x, elapsed %08x", 5) \
+X( 47, RGXFW_GROUP_MISC, RGXFW_SF_MISC_WORK_CYCLES, "FWCtx 0x%08.8x, elapsed %08x", 2) \
+X( 48, RGXFW_GROUP_MISC, RGXFW_SF_MISC_TRP_UNEXPECTED_EVENT_DEPRECATED, "Unexpected TRP pass completion on DM: %u",1) \
+X( 49, RGXFW_GROUP_MISC, RGXFW_SF_MISC_FAILED_IRQ_CLEAR_DEPRECATED, "Host failed to clear IRQ, Reg: 0x%08x",1) \
+\
+X(  1, RGXFW_GROUP_PM, RGXFW_SF_PM_AMLIST, "ALIST%u SP = %u, MLIST%u SP = %u (VCE 0x%08x%08x, TE 0x%08x%08x, ALIST 0x%08x%08x)", 10) \
+X(  2, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED_DEPRECATED, "Is TA: %u, finished: %u on HW %u (HWRTData = 0x%08x, MemCtx = 0x%08x). FL different between TA/3D: global:%u, local:%u, mmu:%u", 8) \
+X(  3, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE_DEPRECATED, "UFL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), MFL-3D-Base: 0x%08x%08x (SP = %u, 4PT = %u)", 14) \
+X(  4, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE_DEPRECATED, "UFL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), MFL-TA-Base: 0x%08x%08x (SP = %u, 4PT = %u)", 14) \
+X(  5, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_GROW_COMPLETE_DEPRECATED, "Freelist grow completed [0x%08x]: added pages 0x%08x, total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 5) \
+X(  6, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_GROW_DENIED_DEPRECATED, "Grow for freelist ID=0x%08x denied by host", 1) \
+X(  7, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_COMPLETE, "Freelist update completed [0x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 5) \
+X(  8, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_RECONSTRUCTION_FAILED_DEPRECATED, "Reconstruction of freelist ID=0x%08x failed", 1) \
+X(  9, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_WARNING, "Ignored attempt to pause or unpause the DM while there is no relevant operation in progress (0-TA,1-3D): %u, operation(0-unpause, 1-pause): %u", 2) \
+X( 10, RGXFW_GROUP_PM, RGXFW_SF_PM_3D_TIMEOUT_STATUS, "Force free 3D Context memory, FWCtx: 0x%08x, status(1:success, 0:fail): %u", 2)\
+X( 11, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_ALLOC, "PM pause TA ALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 12, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_UNPAUSE_ALLOC, "PM unpause TA ALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 13, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_DALLOC, "PM pause 3D DALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 14, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_UNPAUSE_DALLOC, "PM unpause 3D DALLOC: PM_PAGE_MANAGEOP set to 0x%x", 1) \
+X( 15, RGXFW_GROUP_PM, RGXFW_SF_PM_DM_PAUSE_FAILED, "PM ALLOC/DALLOC change was not actioned: PM_PAGE_MANAGEOP_STATUS=0x%x", 1) \
+X( 16, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED, "Is TA: %u, finished: %u on HW %u (HWRTData = 0x%08x, MemCtx = 0x%08x). FL different between TA/3D: global:%u, local:%u", 7) \
+X( 17, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE, "UFL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 18, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE, "UFL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 19, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_COMPLETE_VOLCANIC, "Freelist update completed [0x%08x / FL State 0x%08x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 7) \
+X( 20, RGXFW_GROUP_PM, RGXFW_SF_PM_FL_UPDATE_FAILED, "Freelist update failed [0x%08x / FL State 0x%08x%08x]: old total pages 0x%08x, new total pages 0x%08x, new DevVirtAddr 0x%08x%08x", 7) \
+X( 21, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_3DBASE_VOLCANIC, "UFL-3D-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-3D-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 22, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_TABASE_VOLCANIC, "UFL-TA-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u), FL-TA-State-Base: 0x%08x%08x (SP = %u, 4PB = %u, 4PT = %u)", 10) \
+X( 23, RGXFW_GROUP_PM, RGXFW_SF_PM_CHECK_FL_BASEADDR, "Freelist 0x%08x base address from HW: 0x%02x%08x (expected value: 0x%02x%08x)", 5) \
+X( 24, RGXFW_GROUP_PM, RGXFW_SF_PM_ANALYSE_FL_GROW, "Analysis of FL grow: Pause=(%u,%u) Paused+Valid(%u,%u) PMStateBuffer=0x%x", 5) \
+X( 25, RGXFW_GROUP_PM, RGXFW_SF_PM_ATTEMPT_FL_GROW, "Attempt FL grow for FL: 0x%08x, new dev address: 0x%02x%08x, new page count: %u, new ready count: %u", 5) \
+X( 26, RGXFW_GROUP_PM, RGXFW_SF_PM_DEFER_FL_GROW, "Deferring FL grow for non-loaded FL: 0x%08x, new dev address: 0x%02x%08x, new page count: %u, new ready count: %u", 5) \
+X( 27, RGXFW_GROUP_PM, RGXFW_SF_PM_UFL_SHARED_ALBIORIX, "Is GEOM: %u, finished: %u (HWRTData = 0x%08x, MemCtx = 0x%08x)", 4) \
+X( 28, RGXFW_GROUP_PM, RGXFW_SF_PM_3D_TIMEOUT, "3D Timeout Now for FWCtx 0x%08.8x", 1) \
+X( 29, RGXFW_GROUP_PM, RGXFW_SF_PM_RECYCLE, "GEOM PM Recycle for FWCtx 0x%08.8x", 1) \
+X( 30, RGXFW_GROUP_PM, RGXFW_SF_PM_PRIMARY_CONFIG, "PM running primary config (Core %u)", 1) \
+X( 31, RGXFW_GROUP_PM, RGXFW_SF_PM_SECONDARY_CONFIG, "PM running secondary config (Core %u)", 1) \
+X( 32, RGXFW_GROUP_PM, RGXFW_SF_PM_TERTIARY_CONFIG, "PM running tertiary config (Core %u)", 1) \
+X( 33, RGXFW_GROUP_PM, RGXFW_SF_PM_QUATERNARY_CONFIG, "PM running quaternary config (Core %u)", 1) \
+X( 34, RGXFW_GROUP_PM, RGXFW_SF_PM_REVERT_CONFIG, "PM reverting to previous config (Core %u)", 1) \
+\
+X(  1, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GLL_DYNAMIC_STATUS_DEPRECATED, "Global link list dynamic page count: vertex 0x%x, varying 0x%x, node 0x%x", 3) \
+X(  2, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GLL_STATIC_STATUS_DEPRECATED, "Global link list static page count: vertex 0x%x, varying 0x%x, node 0x%x", 3) \
+X(  3, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_WAIT_FOR_GROW_DEPRECATED, "RPM request failed. Waiting for freelist grow.", 0) \
+X(  4, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_ABORT_DEPRECATED, "RPM request failed. Aborting the current frame.", 0) \
+X(  5, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_WAIT_FOR_PENDING_GROW_DEPRECATED, "RPM waiting for pending grow on freelist 0x%08x", 1) \
+X(  6, RGXFW_GROUP_RPM, RGXFW_SF_RPM_REQUEST_HOST_GROW_DEPRECATED, "Request freelist grow [0x%08x] current pages %u, grow size %u", 3) \
+X(  7, RGXFW_GROUP_RPM, RGXFW_SF_RPM_FREELIST_LOAD_DEPRECATED, "Freelist load: SHF = 0x%08x, SHG = 0x%08x", 2) \
+X(  8, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_DEPRECATED, "SHF FPL register: 0x%08x.0x%08x", 2) \
+X(  9, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_DEPRECATED, "SHG FPL register: 0x%08x.0x%08x", 2) \
+X( 10, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_FREELIST_DEPRECATED, "Kernel requested RPM grow on freelist (type %u) at 0x%08x from current size %u to new size %u, RPM restart: %u (1=Yes)", 5) \
+X( 11, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_RESTART_DEPRECATED, "Restarting SHG", 0) \
+X( 12, RGXFW_GROUP_RPM, RGXFW_SF_RPM_GROW_ABORTED_DEPRECATED, "Grow failed, aborting the current frame.", 0) \
+X( 13, RGXFW_GROUP_RPM, RGXFW_SF_RPM_ABORT_COMPLETE_DEPRECATED, "RPM abort complete on HWFrameData [0x%08x].", 1) \
+X( 14, RGXFW_GROUP_RPM, RGXFW_SF_RPM_CLEANUP_NEEDS_ABORT_DEPRECATED, "RPM freelist cleanup [0x%08x] requires abort to proceed.", 1) \
+X( 15, RGXFW_GROUP_RPM, RGXFW_SF_RPM_RPM_PT_DEPRECATED, "RPM page table base register: 0x%08x.0x%08x", 2) \
+X( 16, RGXFW_GROUP_RPM, RGXFW_SF_RPM_OOM_ABORT_DEPRECATED, "Issuing RPM abort.", 0) \
+X( 17, RGXFW_GROUP_RPM, RGXFW_SF_RPM_OOM_TOGGLE_CHECK_FULL_DEPRECATED, "RPM OOM received but toggle bits indicate free pages available", 0) \
+X( 18, RGXFW_GROUP_RPM, RGXFW_SF_RPM_STATE_HW_TIMEOUT_DEPRECATED, "RPM hardware timeout. Unable to process OOM event.", 0) \
+X( 19, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_LOAD_DEPRECATED_DEPRECATED, "SHF FL (0x%08x) load, FPL: 0x%08x.0x%08x, roff: 0x%08x, woff: 0x%08x", 5) \
+X( 20, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_LOAD_DEPRECATED, "SHG FL (0x%08x) load, FPL: 0x%08x.0x%08x, roff: 0x%08x, woff: 0x%08x", 5) \
+X( 21, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHF_FPL_STORE_DEPRECATED, "SHF FL (0x%08x) store, roff: 0x%08x, woff: 0x%08x", 3) \
+X( 22, RGXFW_GROUP_RPM, RGXFW_SF_RPM_SHG_FPL_STORE_DEPRECATED, "SHG FL (0x%08x) store, roff: 0x%08x, woff: 0x%08x", 3) \
+\
+X(  1, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_FINISHED, "3D RTData 0x%08x finished on HW context %u", 2) \
+X(  2, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_READY, "3D RTData 0x%08x ready on HW context %u", 2) \
+X(  3, RGXFW_GROUP_RTD, RGXFW_SF_RTD_PB_SET_TO_DEPRECATED, "CONTEXT_PB_BASE set to 0x%x, FL different between TA/3D: local: %u, global: %u, mmu: %u", 4) \
+X(  4, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOADVFP_3D_DEPRECATED, "Loading VFP table 0x%08x%08x for 3D", 2) \
+X(  5, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOADVFP_TA_DEPRECATED, "Loading VFP table 0x%08x%08x for TA", 2) \
+X(  6, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_DEPRECATED, "Load Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: TotalPMPages = %u, FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 10) \
+X(  7, RGXFW_GROUP_RTD, RGXFW_SF_RTD_VHEAP_STORE, "Perform VHEAP table store", 0) \
+X(  8, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_MATCH_FOUND, "RTData 0x%08x: found match in Context=%u: Load=No, Store=No", 2) \
+X(  9, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_NULL_FOUND, "RTData 0x%08x: found NULL in Context=%u: Load=Yes, Store=No", 2) \
+X( 10, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_3D_FINISHED, "RTData 0x%08x: found state 3D finished (0x%08x) in Context=%u: Load=Yes, Store=Yes", 3) \
+X( 11, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RTDATA_TA_FINISHED, "RTData 0x%08x: found state TA finished (0x%08x) in Context=%u: Load=Yes, Store=Yes", 3) \
+X( 12, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_STACK_POINTERS, "Loading stack-pointers for %u (0:MidTA,1:3D) on context %u, MLIST = 0x%08x, ALIST = 0x%08x%08x", 5) \
+X( 13, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_PB_DEPRECATED, "Store Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: TotalPMPages = %u, FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 10) \
+X( 14, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_FINISHED, "TA RTData 0x%08x finished on HW context %u", 2) \
+X( 15, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_LOADED, "TA RTData 0x%08x loaded on HW context %u", 2) \
+X( 16, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_PB_DEPRECATED2, "Store Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 17, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_DEPRECATED2, "Load Freelist 0x%x type: %u (0:local,1:global,2:mmu) for DM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 18, RGXFW_GROUP_RTD, RGXFW_SF_RTD_DEBUG_DEPRECATED, "Freelist 0x%x RESET!!!!!!!!", 1) \
+X( 19, RGXFW_GROUP_RTD, RGXFW_SF_RTD_DEBUG2_DEPRECATED, "Freelist 0x%x stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 5) \
+X( 20, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_DEPRECATED, "Request reconstruction of Freelist 0x%x type: %u (0:local,1:global,2:mmu) on HW context %u", 3) \
+X( 21, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_ACK_DEPRECATED, "Freelist reconstruction ACK from host (HWR state :%u)", 1) \
+X( 22, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FL_RECON_ACK_DEPRECATED2, "Freelist reconstruction completed", 0) \
+X( 23, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_LOADED_DEPRECATED, "TA RTData 0x%08x loaded on HW context %u HWRTDataNeedsLoading=%u", 3) \
+X( 24, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TE_RGNHDR_INFO, "TE Region headers base 0x%08x%08x (RGNHDR Init: %u)", 3) \
+X( 25, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_BUFFER_ADDRS_DEPRECATED, "TA Buffers: FWCtx 0x%08x, RT 0x%08x, RTData 0x%08x, VHeap 0x%08x%08x, TPC 0x%08x%08x (MemCtx 0x%08x)", 8) \
+X( 26, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_LOADED_DEPRECATED, "3D RTData 0x%08x loaded on HW context %u", 2) \
+X( 27, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS_DEPRECATED, "3D Buffers: FWCtx 0x%08x, RT 0x%08x, RTData 0x%08x (MemCtx 0x%08x)", 4) \
+X( 28, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RESTART_AFTER_PR_EXECUTED, "Restarting TA after partial render, HWRTData0State=0x%x, HWRTData1State=0x%x", 2) \
+X( 29, RGXFW_GROUP_RTD, RGXFW_SF_RTD_PB_SET_TO, "CONTEXT_PB_BASE set to 0x%x, FL different between TA/3D: local: %u, global: %u", 3) \
+X( 30, RGXFW_GROUP_RTD, RGXFW_SF_RTD_STORE_FL, "Store Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 31, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL, "Load Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u), FL-addr = 0x%08x%08x, stacktop = 0x%08x%08x, Alloc Page Count = %u, Alloc MMU Page Count = %u", 12) \
+X( 32, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS_DEPRECATED2, "3D Buffers: FWCtx 0x%08x, parent RT 0x%08x, RTData 0x%08x on ctx %u, (MemCtx 0x%08x)", 5) \
+X( 33, RGXFW_GROUP_RTD, RGXFW_SF_RTD_TA_RTDATA_BUFFER_ADDRS, "TA Buffers: FWCtx 0x%08x, RTData 0x%08x, VHeap 0x%08x%08x, TPC 0x%08x%08x (MemCtx 0x%08x)", 7) \
+X( 34, RGXFW_GROUP_RTD, RGXFW_SF_RTD_3D_RTDATA_BUFFER_ADDRS, "3D Buffers: FWCtx 0x%08x, RTData 0x%08x on ctx %u, (MemCtx 0x%08x)", 4) \
+X( 35, RGXFW_GROUP_RTD, RGXFW_SF_RTD_LOAD_FL_V2, "Load Freelist 0x%x type: %u (0:local,1:global) for PMDM%u: FL Total Pages %u (max=%u,grow size=%u)", 6) \
+X( 36, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILLED_TA, "TA RTData 0x%08x marked as killed.", 1) \
+X( 37, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILLED_3D, "3D RTData 0x%08x marked as killed.", 1) \
+X( 38, RGXFW_GROUP_RTD, RGXFW_SF_RTD_KILL_TA_AFTER_RESTART, "RTData 0x%08x will be killed after TA restart.", 1) \
+X( 39, RGXFW_GROUP_RTD, RGXFW_SF_RTD_RENDERSTATE_RESET, "RTData 0x%08x Render State Buffer 0x%02x%08x will be reset.", 3) \
+X( 40, RGXFW_GROUP_RTD, RGXFW_SF_RTD_GEOM_RENDERSTATE, "GEOM RTData 0x%08x using Render State Buffer 0x%02x%08x.", 3) \
+X( 41, RGXFW_GROUP_RTD, RGXFW_SF_RTD_FRAG_RENDERSTATE, "FRAG RTData 0x%08x using Render State Buffer 0x%02x%08x.", 3) \
+\
+X(  1, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZLOAD_DEPRECATED, "Force Z-Load for partial render", 0) \
+X(  2, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSTORE_DEPRECATED, "Force Z-Store for partial render", 0) \
+X(  3, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_LOCAL_DEPRECATED, "3D MemFree: Local FL 0x%08x", 1) \
+X(  4, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_MMU_DEPRECATED, "3D MemFree: MMU FL 0x%08x", 1) \
+X(  5, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_GLOBAL_DEPRECATED, "3D MemFree: Global FL 0x%08x", 1) \
+X(  6, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_DEPRECATED, "OOM TA/3D PR Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x, HardwareSync Fence [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 6) \
+X(  7, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_UN_FL, "OOM TA_cmd=0x%08x, U-FL 0x%08x, N-FL 0x%08x", 3) \
+X(  8, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD_UN_MMU_FL_DEPRECATED, "OOM TA_cmd=0x%08x, OOM MMU:%u, U-FL 0x%08x, N-FL 0x%08x, MMU-FL 0x%08x", 5) \
+X(  9, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_AVOIDED_DEPRECATED, "Partial render avoided", 0) \
+X( 10, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_DISCARDED_DEPRECATED, "Partial render discarded", 0) \
+X( 11, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PRENDER_FINISHED, "Partial Render finished", 0) \
+X( 12, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_3DBG_DEPRECATED, "SPM Owner = 3D-BG", 0) \
+X( 13, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_3DIRQ_DEPRECATED, "SPM Owner = 3D-IRQ", 0) \
+X( 14, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_NONE_DEPRECATED, "SPM Owner = NONE", 0) \
+X( 15, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_TABG_DEPRECATED, "SPM Owner = TA-BG", 0) \
+X( 16, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OWNER_TAIRQ_DEPRECATED, "SPM Owner = TA-IRQ", 0) \
+X( 17, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSTORE_ADDRESS, "ZStore address 0x%08x%08x", 2) \
+X( 18, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SSTORE_ADDRESS, "SStore address 0x%08x%08x", 2) \
+X( 19, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZLOAD_ADDRESS, "ZLoad address 0x%08x%08x", 2) \
+X( 20, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SLOAD_ADDRESS, "SLoad address 0x%08x%08x", 2) \
+X( 21, RGXFW_GROUP_SPM, RGXFW_SF_SPM_NO_DEFERRED_ZSBUFFER_DEPRECATED, "No deferred ZS Buffer provided", 0) \
+X( 22, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_POPULATED, "ZS Buffer successfully populated (ID=0x%08x)", 1) \
+X( 23, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_POP_UNNEEDED_DEPRECATED, "No need to populate ZS Buffer (ID=0x%08x)", 1) \
+X( 24, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNPOPULATED, "ZS Buffer successfully unpopulated (ID=0x%08x)", 1) \
+X( 25, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNPOP_UNNEEDED_DEPRECATED, "No need to unpopulate ZS Buffer (ID=0x%08x)", 1) \
+X( 26, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_DEPRECATED, "Send ZS-Buffer backing request to host (ID=0x%08x)", 1) \
+X( 27, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNBACKING_REQUEST_DEPRECATED, "Send ZS-Buffer unbacking request to host (ID=0x%08x)", 1) \
+X( 28, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_PENDING_DEPRECATED, "Don't send ZS-Buffer backing request. Previous request still pending (ID=0x%08x)", 1) \
+X( 29, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_UNBACKING_REQUEST_PENDING_DEPRECATED, "Don't send ZS-Buffer unbacking request. Previous request still pending (ID=0x%08x)", 1) \
+X( 30, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZBUFFER_NOT_READY_DEPRECATED, "Partial Render waiting for ZBuffer to be backed (ID=0x%08x)", 1) \
+X( 31, RGXFW_GROUP_SPM, RGXFW_SF_SPM_SBUFFER_NOT_READY_DEPRECATED, "Partial Render waiting for SBuffer to be backed (ID=0x%08x)", 1) \
+X( 32, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_NONE, "SPM State = none", 0) \
+X( 33, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_BLOCKED, "SPM State = PR blocked", 0) \
+X( 34, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_GROW, "SPM State = wait for grow", 0) \
+X( 35, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_HW, "SPM State = wait for HW", 0) \
+X( 36, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_RUNNING, "SPM State = PR running", 0) \
+X( 37, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_AVOIDED, "SPM State = PR avoided", 0) \
+X( 38, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_EXECUTED, "SPM State = PR executed", 0) \
+X( 39, RGXFW_GROUP_SPM, RGXFW_SF_SPM_FREELIST_MATCH, "3DMemFree matches freelist 0x%08x (FL type = %u)", 2) \
+X( 40, RGXFW_GROUP_SPM, RGXFW_SF_SPM_3DMEMFREE_FLAG_SET, "Raise the 3DMemFreeDetected flag", 0) \
+X( 41, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_WAIT_FOR_PENDING_GROW, "Wait for pending grow on Freelist 0x%08x", 1) \
+X( 42, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ZSBUFFER_BACKING_REQUEST_FAILED, "ZS Buffer failed to be populated (ID=0x%08x)", 1) \
+X( 43, RGXFW_GROUP_SPM, RGXFW_SF_SPM_FL_GROW_DEBUG, "Grow update inconsistency: FL addr: 0x%02x%08x, curr pages: %u, ready: %u, new: %u", 5) \
+X( 44, RGXFW_GROUP_SPM, RGXFW_SF_SPM_RESUMED_TA_WITH_SP, "OOM: Resumed TA with ready pages, FL addr: 0x%02x%08x, current pages: %u, SP : %u", 4) \
+X( 45, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ACK_GROW_UPDATE_DEPRECATED, "Received grow update, FL addr: 0x%02x%08x, current pages: %u, ready pages: %u, threshold: %u", 5) \
+X( 46, RGXFW_GROUP_SPM, RGXFW_SF_SPM_NO_DEFERRED_PRBUFFER, "No deferred partial render FW (Type=%u) Buffer provided", 1) \
+X( 47, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_POP_UNNEEDED, "No need to populate PR Buffer (ID=0x%08x)", 1) \
+X( 48, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNPOP_UNNEEDED, "No need to unpopulate PR Buffer (ID=0x%08x)", 1) \
+X( 49, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_REQUEST, "Send PR Buffer backing request to host (ID=0x%08x)", 1) \
+X( 50, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNBACKING_REQUEST, "Send PR Buffer unbacking request to host (ID=0x%08x)", 1) \
+X( 51, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_REQUEST_PENDING, "Don't send PR Buffer backing request. Previous request still pending (ID=0x%08x)", 1) \
+X( 52, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_UNBACKING_REQUEST_PENDING, "Don't send PR Buffer unbacking request. Previous request still pending (ID=0x%08x)", 1) \
+X( 53, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_NOT_READY, "Partial Render waiting for Buffer %u type to be backed (ID=0x%08x)", 2) \
+X( 54, RGXFW_GROUP_SPM, RGXFW_SF_SPM_ACK_GROW_UPDATE, "Received grow update, FL addr: 0x%02x%08x, new pages: %u, ready pages: %u", 4) \
+X( 66, RGXFW_GROUP_SPM, RGXFW_SF_SPM_OOM_TACMD, "OOM TA/3D PR Check: [0x%08.8x] is 0x%08.8x requires 0x%08.8x", 3) \
+X( 67, RGXFW_GROUP_SPM, RGXFW_SF_SPM_RESUMED_TA, "OOM: Resumed TA with ready pages, FL addr: 0x%02x%08x, current pages: %u", 3) \
+X( 68, RGXFW_GROUP_SPM, RGXFW_SF_SPM_PR_DEADLOCK_UNBLOCKED, "OOM TA/3D PR deadlock unblocked reordering DM%u runlist head from Context 0x%08x to 0x%08x", 3) \
+X( 69, RGXFW_GROUP_SPM, RGXFW_SF_SPM_STATE_PR_FORCEFREE, "SPM State = PR force free", 0) \
+X( 70, RGXFW_GROUP_SPM, RGXFW_SF_SPM_BUFFER_BACKING_FAILED, "Failure to back PR Buffer", 0) \
+\
+X(  1, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK_DEPRECATED, "Check Pow state DM%u int: 0x%x, ext: 0x%x, pow flags: 0x%x", 4) \
+X(  2, RGXFW_GROUP_POW, RGXFW_SF_POW_GPU_IDLE, "GPU idle (might be powered down). Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X(  3, RGXFW_GROUP_POW, RGXFW_SF_POW_OSREQ_DEPRECATED, "OS requested pow off (forced = %u), DM%u, pow flags: 0x%x", 3) \
+X(  4, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_DEPRECATED, "Initiate powoff query. Inactive DMs: %u %u %u %u", 4) \
+X(  5, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECKOFF_DEPRECATED, "Any RD-DM pending? %u, Any RD-DM Active? %u", 2) \
+X(  6, RGXFW_GROUP_POW, RGXFW_SF_POW_GPU_OFF, "GPU ready to be powered down. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X(  7, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ, "HW Request On(1)/Off(0): %u, Units: 0x%08.8x", 2) \
+X(  8, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE_REQ, "Request to change num of dusts to %u (Power flags=%u)", 2) \
+X(  9, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE, "Changing number of dusts from %u to %u", 2) \
+X( 11, RGXFW_GROUP_POW, RGXFW_SF_POW_SIDEKICK_INIT_DEPRECATED, "Sidekick init", 0) \
+X( 12, RGXFW_GROUP_POW, RGXFW_SF_POW_RD_INIT_DEPRECATED, "Rascal+Dusts init (# dusts mask: 0x%x)", 1) \
+X( 13, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_RD, "Initiate powoff query for RD-DMs.", 0) \
+X( 14, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_TLA, "Initiate powoff query for TLA-DM.", 0) \
+X( 15, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_RD, "Any RD-DM pending? %u, Any RD-DM Active? %u", 2) \
+X( 16, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_TLA, "TLA-DM pending? %u, TLA-DM Active? %u", 2) \
+X( 17, RGXFW_GROUP_POW, RGXFW_SF_POW_BRN37270_DEPRECATED, "Request power up due to BRN37270. Pow stat int: 0x%x", 1) \
+X( 18, RGXFW_GROUP_POW, RGXFW_SF_POW_REQ_CANCEL, "Cancel power off request int: 0x%x, ext: 0x%x, pow flags: 0x%x", 3) \
+X( 19, RGXFW_GROUP_POW, RGXFW_SF_POW_FORCED_IDLE, "OS requested forced IDLE, pow flags: 0x%x", 1) \
+X( 20, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_FORCED_IDLE, "OS cancelled forced IDLE, pow flags: 0x%x", 1) \
+X( 21, RGXFW_GROUP_POW, RGXFW_SF_POW_IDLE_TIMER, "Idle timer start. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 22, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_IDLE_TIMER, "Cancel idle timer. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 23, RGXFW_GROUP_POW, RGXFW_SF_POW_APM_LATENCY_CHANGE, "Active PM latency set to %ums. Core clock: %u Hz", 2) \
+X( 24, RGXFW_GROUP_POW, RGXFW_SF_POW_CDM_CLUSTERS, "Compute cluster mask change to 0x%x, %u dusts powered.", 2) \
+X( 25, RGXFW_GROUP_POW, RGXFW_SF_POW_NULL_CMD_INIOFF_RD, "Null command executed, repeating initiate powoff query for RD-DMs.", 0) \
+X( 26, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_ENERGY, "Power monitor: Estimate of dynamic energy %u", 1) \
+X( 27, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK_DEPRECATED2, "Check Pow state: Int: 0x%x, Ext: 0x%x, Pow flags: 0x%x", 3) \
+X( 28, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_NEW_DEADLINE, "Proactive DVFS: New deadline, time = 0x%08x%08x", 2) \
+X( 29, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_NEW_WORKLOAD, "Proactive DVFS: New workload, cycles = 0x%08x%08x", 2) \
+X( 30, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_CALCULATE, "Proactive DVFS: Proactive frequency calculated = %u", 1) \
+X( 31, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UTILISATION, "Proactive DVFS: Reactive utilisation = %u percent", 1) \
+X( 32, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_REACT, "Proactive DVFS: Reactive frequency calculated = %u.%u", 2) \
+X( 33, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_SEND_DEPRECATED, "Proactive DVFS: OPP Point Sent = 0x%x", 1) \
+X( 34, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DEADLINE_REMOVED, "Proactive DVFS: Deadline removed = 0x%08x%08x", 2) \
+X( 35, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_WORKLOAD_REMOVED, "Proactive DVFS: Workload removed = 0x%08x%08x", 2) \
+X( 36, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_THROTTLE, "Proactive DVFS: Throttle to a maximum = 0x%x", 1) \
+X( 37, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_FAILURE, "Proactive DVFS: Failed to pass OPP point via GPIO.", 0) \
+X( 38, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_INVALID_NODE_DEPRECATED, "Proactive DVFS: Invalid node passed to function.", 0) \
+X( 39, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GUEST_BAD_ACCESS_DEPRECATED, "Proactive DVFS: Guest OS attempted to do a privileged action. OSid = %u", 1) \
+X( 40, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UNPROFILED_STARTED, "Proactive DVFS: Unprofiled work started. Total unprofiled work present: %u", 1) \
+X( 41, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_UNPROFILED_FINISHED, "Proactive DVFS: Unprofiled work finished. Total unprofiled work present: %u", 1) \
+X( 42, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DISABLED, "Proactive DVFS: Disabled: Not enabled by host.", 0) \
+X( 43, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ_RESULT, "HW Request Completed(1)/Aborted(0): %u, Ticks: %u", 2) \
+X( 44, RGXFW_GROUP_POW, RGXFW_SF_POW_DUSTS_CHANGE_FIX_59042_DEPRECATED, "Allowed number of dusts is %u due to BRN59042.", 1) \
+X( 45, RGXFW_GROUP_POW, RGXFW_SF_POW_HOST_TIMEOUT_NOTIFICATION, "Host timed out while waiting for a forced idle state. Pow state int: 0x%x, ext: 0x%x, flags: 0x%x", 3) \
+X( 46, RGXFW_GROUP_POW, RGXFW_SF_POW_CHECK, "Check Pow state: Int: 0x%x, Ext: 0x%x, Pow flags: 0x%x, Fence Counters: Check: %u - Update: %u", 5) \
+X( 47, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_GPIO_SEND, "Custom/Proactive DVFS: OPP Point Sent = 0x%x, Success = 0x%x", 2) \
+X( 48, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_TO_IDLE, "Proactive DVFS: GPU transitioned to idle", 0) \
+X( 49, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_TO_ACTIVE, "Proactive DVFS: GPU transitioned to active", 0) \
+X( 50, RGXFW_GROUP_POW, RGXFW_SF_POW_POWDUMP_BUFFER_SIZE, "Power counter dumping: Data truncated writing register %u. Buffer too small.", 1) \
+X( 51, RGXFW_GROUP_POW, RGXFW_SF_POW_POWCTRL_ABORT, "Power controller returned ABORT for last request so retrying.", 0) \
+X( 52, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_POWER_REQUEST_DEPRECATED, "Discarding invalid power request: type 0x%x, DM %u", 2) \
+X( 53, RGXFW_GROUP_POW, RGXFW_SF_POW_CANCEL_FORCED_IDLE_NOT_IDLE, "Detected attempt to cancel forced idle while not forced idle (pow state 0x%x, pow flags 0x%x)", 2) \
+X( 54, RGXFW_GROUP_POW, RGXFW_SF_POW_FORCED_POW_OFF_NOT_IDLE, "Detected attempt to force power off while not forced idle (pow state 0x%x, pow flags 0x%x)", 2) \
+X( 55, RGXFW_GROUP_POW, RGXFW_SF_POW_NUMDUST_CHANGE_NOT_IDLE, "Detected attempt to change dust count while not forced idle (pow state 0x%x)", 1) \
+X( 56, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_RESULT, "Power monitor: Type = %u (0 = power, 1 = energy), Estimate result = 0x%08x%08x", 3) \
+X( 57, RGXFW_GROUP_POW, RGXFW_SF_POW_MINMAX_CONFLICT, "Conflicting clock frequency range: OPP min = %u, max = %u", 2) \
+X( 58, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_FLOOR, "Proactive DVFS: Set floor to a minimum = 0x%x", 1) \
+X( 59, RGXFW_GROUP_POW, RGXFW_SF_POW_OSREQ, "OS requested pow off (forced = %u), pow flags: 0x%x", 2) \
+X( 60, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_POWER_REQUEST, "Discarding invalid power request: type 0x%x", 1) \
+X( 61, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_STATE_CHANGE_REQ, "Request to change SPU power state mask from 0x%x to 0x%x. Pow flags: 0x%x", 3) \
+X( 62, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_STATE_CHANGE, "Changing SPU power state mask from 0x%x to 0x%x", 2) \
+X( 63, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_POW_CHANGE_NOT_IDLE, "Detected attempt to change SPU power state mask while not forced idle (pow state 0x%x)", 1) \
+X( 64, RGXFW_GROUP_POW, RGXFW_SF_POW_INVALID_SPU_POWER_MASK, "Invalid SPU power mask 0x%x! Changing to 1", 1) \
+X( 65, RGXFW_GROUP_POW, RGXFW_SF_POW_CLKDIV_UPDATE, "Proactive DVFS: Send OPP %u with clock divider value %u", 2) \
+X( 66, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_PERF_MODE, "Power counters in raw/validation mode.", 0) \
+X( 67, RGXFW_GROUP_POW, RGXFW_SF_POW_POWMON_RESET, "Reset PPA block state %u (1=reset, 0=recalculate).", 1) \
+X( 68, RGXFW_GROUP_POW, RGXFW_SF_POW_POWCTRL_ABORT_WITH_CORE, "Power controller returned ABORT for Core-%u last request so retrying.", 1) \
+X( 69, RGXFW_GROUP_POW, RGXFW_SF_POW_HWREQ64BIT, "HW Request On(1)/Off(0): %u, Units: 0x%08x%08x", 3) \
+X( 70, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_RAC_POW_STATE_CHANGE_REQ, "Request to change SPU power state mask from 0x%x to 0x%x and RAC from 0x%x to 0x%x. Pow flags: 0x%x", 5) \
+X( 71, RGXFW_GROUP_POW, RGXFW_SF_POW_SPU_RAC_POW_STATE_CHANGE, "Changing SPU power state mask from 0x%x to 0x%x and RAC from 0x%x to 0x%x", 4) \
+X( 72, RGXFW_GROUP_POW, RGXFW_SF_POW_REQUESTEDOFF_RAC, "RAC pending? %u, RAC Active? %u", 2) \
+X( 73, RGXFW_GROUP_POW, RGXFW_SF_POW_INIOFF_RAC, "Initiate powoff query for RAC.", 0) \
+X( 74, RGXFW_GROUP_POW, RGXFW_SF_POW_PDVFS_DEFER_REACTIVE_UPDATE, "Proactive DVFS: Defer reactive update to meet next deadline 0x%08x%08x", 2) \
+\
+X(  1, RGXFW_GROUP_HWR, RGXFW_SF_HWR_LOCKUP_DEPRECATED, "Lockup detected on DM%u, FWCtx: 0x%08.8x", 2) \
+X(  2, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_FW_DEPRECATED, "Reset fw state for DM%u, FWCtx: 0x%08.8x, MemCtx: 0x%08.8x", 3) \
+X(  3, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED, "Reset HW", 0) \
+X(  4, RGXFW_GROUP_HWR, RGXFW_SF_HWR_TERMINATED_DEPRECATED, "Lockup recovered.", 0) \
+X(  5, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DEPRECATED, "Lock-up DM%u FWCtx: 0x%08.8x", 2) \
+X(  6, RGXFW_GROUP_HWR, RGXFW_SF_HWR_LOCKUP_DETECTED_DEPRECATED, "Lockup detected: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X(  7, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EARLY_FAULT_DETECTION_DEPRECATED, "Early fault detection: GLB(%u->%u), PER-DM(0x%08x)", 3) \
+X(  8, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_LOCKUP_DEPRECATED, "Hold scheduling due lockup: GLB(%u), PER-DM(0x%08x->0x%08x)", 3) \
+X(  9, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FALSE_LOCKUP_DEPRECATED, "False lockup detected: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X( 10, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BRN37729_DEPRECATED, "BRN37729: GLB(%u->%u), PER-DM(0x%08x->0x%08x)", 4) \
+X( 11, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FREELISTS_RECONSTRUCTED_DEPRECATED, "Freelists reconstructed: GLB(%u->%u), PER-DM(0x%08x)", 3) \
+X( 12, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RECONSTRUCTING_FREELISTS_DEPRECATED, "Reconstructing freelists: %u (0-No, 1-Yes): GLB(%u->%u), PER-DM(0x%08x)", 4) \
+X( 13, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FAILED_HW_POLL, "HW poll %u (0-Unset 1-Set) failed (reg:0x%08x val:0x%08x)", 3) \
+X( 14, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_DISCARDED_DEPRECATED, "Discarded cmd on DM%u FWCtx=0x%08x", 2) \
+X( 15, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_DISCARDED, "Discarded cmd on DM%u (reason=%u) HWRTData=0x%08x (st: %u), FWCtx 0x%08x @ %u", 6) \
+X( 16, RGXFW_GROUP_HWR, RGXFW_SF_HWR_PM_FENCE_DEPRECATED, "PM fence WA could not be applied, Valid TA Setup: %u, RD powered off: %u", 2) \
+X( 17, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_SNAPSHOT, "FL snapshot RTD 0x%08.8x - local (0x%08.8x): %u, global (0x%08.8x): %u", 5) \
+X( 18, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_CHECK, "FL check RTD 0x%08.8x, discard: %u - local (0x%08.8x): s%u?=c%u, global (0x%08.8x): s%u?=c%u", 8) \
+X( 19, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_DEPRECATED, "FL reconstruction 0x%08.8x c%u", 2) \
+X( 20, RGXFW_GROUP_HWR, RGXFW_SF_HWR_3D_CHECK, "3D check: missing TA FWCtx 0x%08.8x @ %u, RTD 0x%08x.", 3) \
+X( 21, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED2, "Reset HW (mmu:%u, extmem: %u)", 2) \
+X( 22, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ZERO_TA_CACHES, "Zero TA caches for FWCtx: 0x%08.8x (TPC addr: 0x%08x%08x, size: %u bytes)", 4) \
+X( 23, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FREELISTS_RECONSTRUCTED_DEPRECATED2, "Recovery DM%u: Freelists reconstructed. New R-Flags=0x%08x", 2) \
+X( 24, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SKIPPED_CMD, "Recovery DM%u: FWCtx 0x%08x skipped to command @ %u. PR=%u. New R-Flags=0x%08x", 5) \
+X( 25, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_RECOVERED, "Recovery DM%u: DM fully recovered", 1) \
+X( 26, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_LOCKUP, "DM%u: Hold scheduling due to R-Flag = 0x%08x", 2) \
+X( 27, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_RECONSTRUCTION, "Analysis: Need freelist reconstruction", 0) \
+X( 28, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_SKIP, "Analysis DM%u: Lockup FWCtx: 0x%08.8x. Need to skip to next command", 2) \
+X( 29, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_SKIP_OOM_TA, "Analysis DM%u: Lockup while TA is OOM FWCtx: 0x%08.8x. Need to skip to next command", 2) \
+X( 30, RGXFW_GROUP_HWR, RGXFW_SF_HWR_NEEDS_PR_CLEANUP, "Analysis DM%u: Lockup while partial render FWCtx: 0x%08.8x. Need PR cleanup", 2) \
+X( 31, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DEPRECATED2, "GPU has locked up", 0) \
+X( 32, RGXFW_GROUP_HWR, RGXFW_SF_HWR_READY, "DM%u ready for HWR", 1) \
+X( 33, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_UPDATE_RECOVERY, "Recovery DM%u: Updated Recovery counter. New R-Flags=0x%08x", 2) \
+X( 34, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BRN37729_DEPRECATED2, "Analysis: BRN37729 detected, reset TA and re-kicked 0x%08x)", 1) \
+X( 35, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_TIMED_OUT_DEPRECATED, "DM%u timed out", 1) \
+X( 36, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EVENT_STATUS_REG, "RGX_CR_EVENT_STATUS=0x%08x", 1) \
+X( 37, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_FALSE_LOCKUP, "DM%u lockup falsely detected, R-Flags=0x%08x", 2) \
+X( 38, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_OUTOFTIME, "GPU has overrun its deadline", 0) \
+X( 39, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_POLLFAILURE, "GPU has failed a poll", 0) \
+X( 40, RGXFW_GROUP_HWR, RGXFW_SF_HWR_PERF_PHASE_REG, "RGX DM%u phase count=0x%08x", 2) \
+X( 41, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW_DEPRECATED3, "Reset HW (loop:%u, poll failures: 0x%08x)", 2) \
+X( 42, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_FAULT_EVENT, "MMU fault event: 0x%08x", 1) \
+X( 43, RGXFW_GROUP_HWR, RGXFW_SF_HWR_BIF1_FAULT, "BIF1 page fault detected (Bank1 MMU Status: 0x%08x)", 1) \
+X( 44, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK_TRUE_DEPRECATED, "Fast CRC Failed. Proceeding to full register checking (DM: %u).", 1) \
+X( 45, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_META_FAULT, "Meta MMU page fault detected (Meta MMU Status: 0x%08x%08x)", 2) \
+X( 46, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK_DEPRECATED, "Fast CRC Check result for DM%u is HWRNeeded=%u", 2) \
+X( 47, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FULL_CHECK_DEPRECATED, "Full Signature Check result for DM%u is HWRNeeded=%u", 2) \
+X( 48, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FINAL_RESULT, "Final result for DM%u is HWRNeeded=%u with HWRChecksToGo=%u", 3) \
+X( 49, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_CHECK_DEPRECATED, "USC Slots result for DM%u is HWRNeeded=%u USCSlotsUsedByDM=%u", 3) \
+X( 50, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DEADLINE_CHECK_DEPRECATED, "Deadline counter for DM%u is HWRDeadline=%u", 2) \
+X( 51, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_FREELIST_DEPRECATED, "Holding Scheduling on OSid %u due to pending freelist reconstruction", 1) \
+X( 52, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_REQUEST, "Requesting reconstruction for freelist 0x%x (ID=%u)", 2) \
+X( 53, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_PASSED, "Reconstruction of freelist ID=%u complete", 1) \
+X( 54, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global,2:mmu) on HW context %u", 4) \
+X( 55, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_FAILED, "Reconstruction of freelist ID=%u failed", 1) \
+X( 56, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESTRICTING_PDS_TASKS, "Restricting PDS Tasks to help other stalling DMs (RunningMask=0x%02x, StallingMask=0x%02x, PDS_CTRL=0x%08x%08x)", 4) \
+X( 57, RGXFW_GROUP_HWR, RGXFW_SF_HWR_UNRESTRICTING_PDS_TASKS, "Unrestricting PDS Tasks again (RunningMask=0x%02x, StallingMask=0x%02x, PDS_CTRL=0x%08x%08x)", 4) \
+X( 58, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_USED, "USC slots: %u used by DM%u", 2) \
+X( 59, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_EMPTY, "USC slots: %u empty", 1) \
+X( 60, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HCS_FIRE, "HCS DM%u's Context Switch failed to meet deadline. Current time: 0x%08x%08x, deadline: 0x%08x%08x", 5) \
+X( 61, RGXFW_GROUP_HWR, RGXFW_SF_HWR_START_HW_RESET, "Begin hardware reset (HWR Counter=%u)", 1) \
+X( 62, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FINISH_HW_RESET, "Finished hardware reset (HWR Counter=%u)", 1) \
+X( 63, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HOLD_SCHEDULING_DUE_TO_FREELIST, "Holding Scheduling on DM %u for Driver ID %u due to pending freelist reconstruction", 2) \
+X( 64, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_UMQ_READ_OFFSET, "User Mode Queue ROff reset: FWCtx 0x%08.8x, queue: 0x%08x%08x (Roff = %u becomes StreamStartOffset = %u)", 5) \
+X( 65, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED2, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global) on HW context %u", 4) \
+X( 66, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MIPS_FAULT, "Mips page fault detected (BadVAddr: 0x%08x, EntryLo0: 0x%08x, EntryLo1: 0x%08x)", 3) \
+X( 67, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ANOTHER_CHANCE, "At least one other DM is running okay so DM%u will get another chance", 1) \
+X( 68, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_FW, "Reconstructing in FW, FL: 0x%x (ID=%u)", 2) \
+X( 69, RGXFW_GROUP_HWR, RGXFW_SF_HWR_ZERO_RTC, "Zero RTC for FWCtx: 0x%08.8x (RTC addr: 0x%08x%08x, size: %u bytes)", 4) \
+X( 70, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED_DEPRECATED3, "Reconstruction needed for freelist 0x%x (ID=%u) type: %u (0:local,1:global) phase: %u (0:TA, 1:3D) on HW context %u", 5) \
+X( 71, RGXFW_GROUP_HWR, RGXFW_SF_HWR_START_LONG_HW_POLL, "Start long HW poll %u (0-Unset 1-Set) for (reg:0x%08x val:0x%08x)", 3) \
+X( 72, RGXFW_GROUP_HWR, RGXFW_SF_HWR_END_LONG_HW_POLL, "End long HW poll (result=%u)", 1) \
+X( 73, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DEADLINE_CHECK, "DM%u has taken %u ticks and deadline is %u ticks", 3) \
+X( 74, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WATCHDOG_CHECK_DEPRECATED, "USC Watchdog result for DM%u is HWRNeeded=%u Status=%u USCs={0x%x} with HWRChecksToGo=%u", 5) \
+X( 75, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FL_RECON_NEEDED, "Reconstruction needed for freelist 0x%x (ID=%u) Driver ID: %u type: %u (0:local,1:global) phase: %u (0:TA, 1:3D) on HW context %u", 6) \
+X( 76, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP, "GPU-%u has locked up", 1) \
+X( 77, RGXFW_GROUP_HWR, RGXFW_SF_HWR_SET_LOCKUP_DM, "DM%u has locked up", 1) \
+X( 78, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_EVENT_STATUS_REG, "Core %u RGX_CR_EVENT_STATUS=0x%08x", 2) \
+X( 79, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MULTICORE_EVENT_STATUS_REG, "RGX_CR_MULTICORE_EVENT_STATUS%u=0x%08x", 2) \
+X( 80, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_BIF0_FAULT, "BIF0 page fault detected (Core %u MMU Status: 0x%08x%08x Req Status: 0x%08x%08x)", 5) \
+X( 81, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_MMU_FAULT_S7, "MMU page fault detected (Core %u MMU Status: 0x%08x%08x)", 3) \
+X( 82, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CORE_MMU_FAULT, "MMU page fault detected (Core %u MMU Status: 0x%08x%08x 0x%08x)", 4) \
+X( 83, RGXFW_GROUP_HWR, RGXFW_SF_HWR_RESET_HW, "Reset HW (core:%u of %u, loop:%u, poll failures: 0x%08x)", 4) \
+X( 84, RGXFW_GROUP_HWR, RGXFW_SF_HWR_CRC_CHECK, "Fast CRC Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 85, RGXFW_GROUP_HWR, RGXFW_SF_HWR_FULL_CHECK, "Full Signature Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 86, RGXFW_GROUP_HWR, RGXFW_SF_HWR_USC_SLOTS_CHECK, "USC Slots result for Core%u, DM%u is HWRNeeded=%u USCSlotsUsedByDM=%u", 4) \
+X( 87, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WATCHDOG_CHECK, "USC Watchdog result for Core%u DM%u is HWRNeeded=%u Status=%u USCs={0x%x} with HWRChecksToGo=%u", 6) \
+X( 88, RGXFW_GROUP_HWR, RGXFW_SF_HWR_MMU_RISCV_FAULT, "RISC-V MMU page fault detected (FWCORE MMU Status 0x%08x Req Status 0x%08x%08x)", 3) \
+X( 89, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_TEXAS1_PFS_DEPRECATED, "TEXAS1_PFS poll failed on core %u with value 0x%08x", 2) \
+X( 90, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_PFS, "BIF_PFS poll failed on core %u with value 0x%08x", 2) \
+X( 91, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SET_ABORT_PM_STATUS, "MMU_ABORT_PM_STATUS set poll failed on core %u with value 0x%08x", 2) \
+X( 92, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_UNSET_ABORT_PM_STATUS, "MMU_ABORT_PM_STATUS unset poll failed on core %u with value 0x%08x", 2) \
+X( 93, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SLC_INVAL, "MMU_CTRL_INVAL poll (all but fw) failed on core %u with value 0x%08x", 2) \
+X( 94, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_SLCMMU_INVAL, "MMU_CTRL_INVAL poll (all) failed on core %u with value 0x%08x", 2) \
+X( 95, RGXFW_GROUP_HWR, RGXFW_SF_HWR_HWR_FAULT_POLL_BIF_TEXAS_PFS, "TEXAS%u_PFS poll failed on core %u with value 0x%08x", 3) \
+X( 96, RGXFW_GROUP_HWR, RGXFW_SF_HWR_EXTRA_CHECK, "Extra Registers Check result for Core%u, DM%u is HWRNeeded=%u", 3) \
+X( 97, RGXFW_GROUP_HWR, RGXFW_SF_HWR_WRITE_TO_GPU_READONLY_ADDR, "FW attempted to write to read-only GPU address 0x%08x", 1) \
+X( 98, RGXFW_GROUP_HWR, RGXFW_SF_HWR_DM_TIMED_OUT, "DM%u timed out (phase count=0x%08x)", 2) \
+\
+X(  1, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CFGBLK, "Block 0x%x mapped to Config Idx %u", 2) \
+X(  2, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_OMTBLK, "Block 0x%x omitted from event - not enabled in HW", 1) \
+X(  3, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INCBLK, "Block 0x%x included in event - enabled in HW", 1) \
+X(  4, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_SELREG, "Select register state hi_0x%x lo_0x%x", 2) \
+X(  5, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CSBHDR, "Counter stream block header word 0x%x", 1) \
+X(  6, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTROFF, "Counter register offset 0x%x", 1) \
+X(  7, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CFGSKP, "Block 0x%x config unset, skipping", 1) \
+X(  8, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INDBLK, "Accessing Indirect block 0x%x", 1) \
+X(  9, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DIRBLK, "Accessing Direct block 0x%x", 1) \
+X( 10, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CNTPRG, "Programmed counter select register at offset 0x%x", 1) \
+X( 11, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKPRG, "Block register offset 0x%x and value 0x%x", 2) \
+X( 12, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKCG, "Reading config block from driver 0x%x", 1) \
+X( 13, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKRG, "Reading block range 0x%x to 0x%x", 2) \
+X( 14, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKREC, "Recording block 0x%x config from driver", 1) \
+X( 15, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_UBLKED, "Finished reading config block from driver", 0) \
+X( 16, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CUSTOM_COUNTER, "Custom Counter offset: 0x%x  value: 0x%x", 2) \
+X( 17, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_SELECT_CNTR, "Select counter n:%u  ID:0x%x", 2) \
+X( 18, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_SELECT_PACK, "The counter ID 0x%x is not allowed. The package [b:%u, n:%u] will be discarded", 3) \
+X( 19, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHANGE_FILTER_STATUS_CUSTOM, "Custom Counters filter status %u", 1) \
+X( 20, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_WRONG_BLOCK, "The Custom block %u is not allowed. Use only blocks lower than %u. The package will be discarded", 2) \
+X( 21, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_DROP_TOO_MANY_ID, "The package will be discarded because it contains %u counters IDs while the upper limit is %u", 2) \
+X( 22, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHECK_FILTER, "Check Filter 0x%x is 0x%x ?", 2) \
+X( 23, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_RESET_CUSTOM_BLOCK, "The custom block %u is reset", 1) \
+X( 24, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INVALID_CMD_DEPRECATED, "Encountered an invalid command (%u)", 1) \
+X( 25, RGXFW_GROUP_HWP, RGXFW_SF_HWP_WAITING_FOR_QUEUE_DEPRECATED, "HWPerf Queue is full, we will have to wait for space! (Roff = %u, Woff = %u)", 2) \
+X( 26, RGXFW_GROUP_HWP, RGXFW_SF_HWP_WAITING_FOR_QUEUE_FENCE_DEPRECATED, "HWPerf Queue is fencing, we are waiting for Roff = %u (Roff = %u, Woff = %u)", 3) \
+X( 27, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CUSTOM_BLOCK, "Custom Counter block: %u", 1) \
+X( 28, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKENA, "Block 0x%x ENABLED", 1) \
+X( 29, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_BLKDIS, "Block 0x%x DISABLED", 1) \
+X( 30, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_INDBLK_INSTANCE, "Accessing Indirect block 0x%x, instance %u", 2) \
+X( 31, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTRVAL, "Counter register 0x%x, Value 0x%x", 2) \
+X( 32, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CHANGE_FILTER_STATUS, "Counters filter status %u", 1) \
+X( 33, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CTLBLK, "Block 0x%x mapped to Ctl Idx %u", 2) \
+X( 34, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_WORKEST_EN, "Block(s) in use for workload estimation.", 0) \
+X( 35, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CYCCTR, "GPU %u Cycle counter 0x%x, Value 0x%x", 3) \
+X( 36, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_CYCMAX, "GPU Mask 0x%x Cycle counter 0x%x, Value 0x%x", 3) \
+X( 37, RGXFW_GROUP_HWP, RGXFW_SF_HWP_I_IGNORE_BLOCKS, "Blocks IGNORED for GPU %u", 1) \
+\
+X(  1, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_REQUEST_DEPRECATED, "Transfer 0x%02x request: 0x%02x%08x -> 0x%08x, size %u", 5) \
+X(  2, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_COMPLETE, "Transfer of type 0x%02x expected on channel %u, 0x%02x found, status %u", 4) \
+X(  3, RGXFW_GROUP_DMA, RGXFW_SF_DMA_INT_REG, "DMA Interrupt register 0x%08x", 1) \
+X(  4, RGXFW_GROUP_DMA, RGXFW_SF_DMA_WAIT, "Waiting for transfer of type 0x%02x completion...", 1) \
+X(  5, RGXFW_GROUP_DMA, RGXFW_SF_DMA_CCB_LOADING_FAILED, "Loading of cCCB data from FW common context 0x%08x (offset: %u, size: %u) failed", 3) \
+X(  6, RGXFW_GROUP_DMA, RGXFW_SF_DMA_CCB_LOAD_INVALID, "Invalid load of cCCB data from FW common context 0x%08x (offset: %u, size: %u)", 3) \
+X(  7, RGXFW_GROUP_DMA, RGXFW_SF_DMA_POLL_FAILED, "Transfer 0x%02x request poll failure", 1) \
+X(  8, RGXFW_GROUP_DMA, RGXFW_SF_DMA_BOOT_TRANSFER_FAILED_DEPRECATED, "Boot transfer(s) failed (code? %u, data? %u), used slower memcpy instead", 2) \
+X(  9, RGXFW_GROUP_DMA, RGXFW_SF_DMA_TRANSFER_REQUEST, "Transfer 0x%02x request on ch. %u: system 0x%02x%08x, coremem 0x%08x, flags 0x%x, size %u", 7) \
+\
+X(  1, RGXFW_GROUP_DBG, RGXFW_SF_DBG_INTPAIR, "0x%08x 0x%08x", 2) \
+X(  2, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1HEX, "0x%08x", 1) \
+X(  3, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2HEX, "0x%08x 0x%08x", 2) \
+X(  4, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3HEX, "0x%08x 0x%08x 0x%08x", 3) \
+X(  5, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4HEX, "0x%08x 0x%08x 0x%08x 0x%08x", 4) \
+X(  6, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 5) \
+X(  7, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 6) \
+X(  8, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 7) \
+X(  9, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8HEX, "0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x", 8) \
+X( 10, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1SIGNED, "%d", 1) \
+X( 11, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2SIGNED, "%d %d", 2) \
+X( 12, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3SIGNED, "%d %d %d", 3) \
+X( 13, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4SIGNED, "%d %d %d %d", 4) \
+X( 14, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5SIGNED, "%d %d %d %d %d", 5) \
+X( 15, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6SIGNED, "%d %d %d %d %d %d", 6) \
+X( 16, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7SIGNED, "%d %d %d %d %d %d %d", 7) \
+X( 17, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8SIGNED, "%d %d %d %d %d %d %d %d", 8) \
+X( 18, RGXFW_GROUP_DBG, RGXFW_SF_DBG_1UNSIGNED, "%u", 1) \
+X( 19, RGXFW_GROUP_DBG, RGXFW_SF_DBG_2UNSIGNED, "%u %u", 2) \
+X( 20, RGXFW_GROUP_DBG, RGXFW_SF_DBG_3UNSIGNED, "%u %u %u", 3) \
+X( 21, RGXFW_GROUP_DBG, RGXFW_SF_DBG_4UNSIGNED, "%u %u %u %u", 4) \
+X( 22, RGXFW_GROUP_DBG, RGXFW_SF_DBG_5UNSIGNED, "%u %u %u %u %u", 5) \
+X( 23, RGXFW_GROUP_DBG, RGXFW_SF_DBG_6UNSIGNED, "%u %u %u %u %u %u", 6) \
+X( 24, RGXFW_GROUP_DBG, RGXFW_SF_DBG_7UNSIGNED, "%u %u %u %u %u %u %u", 7) \
+X( 25, RGXFW_GROUP_DBG, RGXFW_SF_DBG_8UNSIGNED, "%u %u %u %u %u %u %u %u", 8) \
+\
+X(  1, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_STATE_CHANGE, "[Host time %09u%09u%09u] Driver ID %u fw state transition request: from %u to %u (0-offline 1-ready 2-active 3-graceful_offloading 4-forced_offloading 5-cooldown). Status %u (1-ok 0-fail)", 7) \
+X(  2, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_MTS_PERMISSION_CHANGED, "[Host time %09u%09u%09u] Updated permission for Driver ID %u to perform MTS kicks: %u (1 = allowed, 0 = not allowed)", 5) \
+X(  3, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_INIT_CCBS, "[Host time %09u%09u%09u] Driver ID %u CCB init status: %u (1-ok 0-fail): kCCBCtl@0x%x kCCB@0x%x fwCCBCtl@0x%x fwCCB@0x%x", 9) \
+X(  4, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_INIT_CONFIG, "[Host time %09u%09u%09u] Initialised OS %u with config flags 0x%08x", 5) \
+X(  5, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_MAX, "[Host time %09u%09u%09u] Time Slice: Update Failed, Max total allocations exceeded 100pc", 3)\
+X(  6, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_DYNAMIC_LOW, "[Host time %09u%09u%09u] Time Slice: Dynamic time slice low, allocating zero time slice to dynamic drivers", 3)\
+X(  7, RGXFW_GROUP_VZ, RGXFW_SF_VZ_TIME_SLICE_UPDATE_SUCCESS, "[Host time %09u%09u%09u] Time Slice: Updated Successfully", 3)\
+X(  8, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_ISOLATION_GROUP_CHANGE, "[Host time %09u%09u%09u] Changing Driver ID %u's isolation group from %u to %u", 6) \
+X(  9, RGXFW_GROUP_VZ, RGXFW_SF_VZ_OS_PRIORITY_CHANGE, "[Host time %09u%09u%09u] Changing Driver ID %u's priority from %u to %u", 6) \
+X( 10, RGXFW_GROUP_VZ, RGXFW_SF_VZ_WDG_TRIGGER, "[Host time %09u%09u%09u] Driver ID %u failed to respond to the virtualisation watchdog in time. Timestamp of its last input = %u", 5) \
+\
+\
+X(  1, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_STATUS, "FBCDC: Core=0x%08x, Status=0x%08x, Signature status=0x%08x", 3) \
+X(  2, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_DETECTED, "FBCDC signature failure detected so block scheduling more work", 0) \
+X(  3, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_FBCDC_FAILURE_CLEARED, "FBCDC signature cleared which unlocks scheduling more work", 0) \
+X(  4, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_FAILURE, "TRP failure: %u", 1) \
+X(  5, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SW_TRP_STATE, "SW TRP State: %u", 1) \
+X(  6, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_HW_KICK, "HW kick event (%u)", 1) \
+X(  7, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_CHECKSUMS, "GPU core (%u/%u): checksum 0x%08x vs. 0x%08x", 4) \
+X(  8, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_WGP_UNIT_CHECKSUMS, "GPU core (%u/%u), unit (%u,%u): checksum 0x%08x vs. 0x%08x", 6) \
+X(  9, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_CHECK_REG, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 10, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_USC_SLOTS_CHECK, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X( 11, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_HWR_USC_REG_CHECK, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X( 12, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_STATE_HWRTDATA_DEPRECATED, "TRP HWRTData: 0x%08x, state: %u", 2) \
+X( 13, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_STATE_CNTX_DEPRECATED, "TRP Context: 0x%08x, state: %u", 2) \
+X( 14, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_FAILURE_CNTX, "TRP Context: 0x%08x, failure: %u", 2) \
+X( 15, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_TRP_UNEXPECTED_EVENT_DEPRECATED, "Unexpected TRP pass completion on DM: %u",1) \
+X( 16, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_ENABLED, "ICS enabled for DM: %u",1) \
+X( 17, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_INTERVAL,"ICS: interval: %u cycles, threshold: %u cycles",2) \
+X( 18, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_FAULT,"ICS fault detected, GPU cores mask 0x%08x",1) \
+X( 19, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_MODE,"Set ICS mode to: %u ",1) \
+X( 20, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_TESTS_COMPLETION_FAILURE,"ICS tests not completed in mode %u",1) \
+X( 21, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_ICS_TESTS_START_FAILURE,"ICS tests failed to start",0) \
+X( 22, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SELF_TEST_START,"Start Safety self-test",0) \
+X( 23, RGXFW_GROUP_SAFETY, RGXFW_SF_SAFETY_SELF_TEST_END,"Safety self-test completed",0) \
+\
+X(  1, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_CHECK_REG, "HWR: Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X(  2, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_USC_SLOTS_CHECK, "HWR: USC Core%u, ui32TotalSlotsUsedByDM=0x%08x, psDMHWCtl->ui32USCSlotsUsedByDM=0x%08x, bHWRNeeded=%u", 4) \
+X(  3, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HWR_USC_REG_CHECK, "HWR: USC Core%u, Register=0x%08x, OldValue=0x%08x%08x, CurrValue=0x%08x%08x", 6) \
+X(  4, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU, "Memory dump: Addr=0x%02x%08x, Size=%d, ContextId=%u, DM=%u", 5) \
+X(  5, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS4, "  0x%02x%08x  %08x %08x %08x %08x", 6) \
+X(  6, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS3, "  0x%02x%08x  %08x %08x %08x", 5) \
+X(  7, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS2, "  0x%02x%08x  %08x %08x", 4) \
+X(  8, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_GPU_DWORDS1, "  0x%02x%08x  %08x", 3) \
+X(  9, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW, "Memory dump: Addr=0x%08x, Size=%d", 2) \
+X(  0, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS4, "  0x%08x  %08x %08x %08x %08x", 5) \
+X( 11, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS3, "  0x%08x  %08x %08x %08x", 4) \
+X( 12, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS2, "  0x%08x  %08x %08x", 3) \
+X( 13, RGXFW_GROUP_VERBOSE, RGXFW_SF_VERBOSE_HEXDUMP_FW_DWORDS1, "  0x%08x  %08x", 2) \
+\
+X(65535, RGXFW_GROUP_NULL, RGXFW_SF_LAST, "You should not use this string", 15)
+
+
+/*  The symbolic names found in the table above are assigned an ui32 value of
+ *  the following format:
+ *  31 30 28 26      20   19  16    15  12      11  10           0   bits
+ *  -   ---   --- ----     ----      ----        -   --- ---- ----
+ *        0-10: id number
+ *    12-15,11: group id number (bits 0-3 and bit 4)
+ *       16-19: number of parameters
+ *       20-26: unused
+ *       28-30: active: identify SF packet, otherwise regular int32
+ *          31: reserved for signed/unsigned compatibility
+ *
+ *   The following macro assigns those values to the enum generated SF ids list.
+ */
+#define RGXFW_LOG_IDMARKER			(0x70000000U)
+#define RGXFW_LOG_CREATESFID(a,b,e) ((IMG_UINT32)(a) | (((IMG_UINT32)(b)&0x10U)<<7U) | (((IMG_UINT32)(b)&0xFU)<<12U) | ((IMG_UINT32)(e)<<16U)) | RGXFW_LOG_IDMARKER
+
+#define RGXFW_LOG_IDMASK			(0xFFF00000U)
+#define RGXFW_LOG_VALIDID(I)		(((I) & RGXFW_LOG_IDMASK) == RGXFW_LOG_IDMARKER)
+
+typedef enum {
+#define X(a, b, c, d, e) c = RGXFW_LOG_CREATESFID(a,b,e),
+	RGXFW_LOG_SFIDLIST
+#undef X
+} RGXFW_LOG_SFids;
+
+/* Return the group id number that the given (enum generated) id belongs to */
+#define RGXFW_SF_GID(x) ((((IMG_UINT32)(x)>>12) & 0xfU)|(((IMG_UINT32)(x)>>7) & 0x10U))
+/* Return the id number that the given (enum generated) id belongs to */
+#define RGXFW_SF_ID(x) ((IMG_UINT32)(x) & 0x7ffU)
+/* Returns how many arguments the SF(string format) for the given (enum generated) id requires */
+#define RGXFW_SF_PARAMNUM(x) (((IMG_UINT32)(x)>>16) & 0xfU)
+
+#endif /* RGX_FWIF_SF_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_fwif_shared.h b/drivers/gpu/drm/img-rogue/rgx_fwif_shared.h
new file mode 100644
index 000000000000..ccab45ae8fc0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_fwif_shared.h
@@ -0,0 +1,1140 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware interface structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware interface structures shared by both host client
+                and host server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_FWIF_SHARED_H)
+#define RGX_FWIF_SHARED_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_common.h"
+#include "powervr/mem_types.h"
+#include "devicemem_typedefs.h"
+
+/* Indicates the number of RTDATAs per RTDATASET */
+#if defined(SUPPORT_AGP)
+#define RGXMKIF_NUM_RTDATAS           4U
+#define RGXMKIF_NUM_GEOMDATAS         4U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  12U /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (2U)
+#else
+#define RGXMKIF_NUM_RTDATAS           2U
+#define RGXMKIF_NUM_GEOMDATAS         1U
+#define RGXMKIF_NUM_RTDATA_FREELISTS  2U  /* RGXMKIF_NUM_RTDATAS * RGXFW_MAX_FREELISTS */
+#define RGX_NUM_GEOM_CORES           (1U)
+#endif
+
+/* Maximum number of UFOs in a CCB command.
+ * The number is based on having 32 sync prims (as originally), plus 32 sync
+ * checkpoints.
+ * Once the use of sync prims is no longer supported, we will retain
+ * the same total (64) as the number of sync checkpoints which may be
+ * supporting a fence is not visible to the client driver and has to
+ * allow for the number of different timelines involved in fence merges.
+ */
+#define RGXFWIF_CCB_CMD_MAX_UFOS			(32U+32U)
+
+/*
+ * This is a generic limit imposed on any DM (TA,3D,CDM,TDM,2D,TRANSFER)
+ * command passed through the bridge.
+ * Just across the bridge in the server, any incoming kick command size is
+ * checked against this maximum limit.
+ * In case the incoming command size is larger than the specified limit,
+ * the bridge call is retired with error.
+ */
+#define RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE	(1024U)
+
+typedef struct
+{
+	IMG_DEV_VIRTADDR        RGXFW_ALIGN psDevVirtAddr;
+	RGXFWIF_DEV_VIRTADDR    pbyFWAddr;
+} UNCACHED_ALIGN RGXFWIF_DMA_ADDR;
+
+typedef IMG_UINT8	RGXFWIF_CCCB;
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_UFO_ADDR;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CLEANUP_CTL;
+
+
+/*!
+ * @InGroup ClientCCBTypes
+ * @Brief Command data for fence & update types Client CCB commands.
+ */
+typedef struct
+{
+	PRGXFWIF_UFO_ADDR	puiAddrUFO; /*!< Address to be checked/updated */
+	IMG_UINT32			ui32Value;  /*!< Value to check-against/update-to */
+} RGXFWIF_UFO;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Track pending and executed workloads of HWRTDATA and ZSBUFFER
+ */
+typedef struct
+{
+	IMG_UINT32			ui32SubmittedCommands;	/*!< Number of commands received by the FW */
+	IMG_UINT32			ui32ExecutedCommands;	/*!< Number of commands executed by the FW */
+} UNCACHED_ALIGN RGXFWIF_CLEANUP_CTL;
+
+#define	RGXFWIF_PRBUFFER_START        IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_ZSBUFFER     IMG_UINT32_C(0)
+#define	RGXFWIF_PRBUFFER_MSAABUFFER   IMG_UINT32_C(1)
+#define	RGXFWIF_PRBUFFER_MAXSUPPORTED IMG_UINT32_C(2)
+
+typedef IMG_UINT32 RGXFWIF_PRBUFFER_TYPE;
+
+typedef enum
+{
+	RGXFWIF_PRBUFFER_UNBACKED = 0,
+	RGXFWIF_PRBUFFER_BACKED,
+	RGXFWIF_PRBUFFER_BACKING_PENDING,
+	RGXFWIF_PRBUFFER_UNBACKING_PENDING,
+}RGXFWIF_PRBUFFER_STATE;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief OnDemand Z/S/MSAA Buffers
+ */
+typedef struct
+{
+	IMG_UINT32		ui32BufferID;		/*!< Buffer ID*/
+	IMG_BOOL		bOnDemand;		/*!< Needs On-demand Z/S/MSAA Buffer allocation */
+	RGXFWIF_PRBUFFER_STATE	eState;			/*!< Z/S/MSAA -Buffer state */
+	RGXFWIF_CLEANUP_CTL	sCleanupState;		/*!< Cleanup state */
+	IMG_UINT32		ui32PRBufferFlags;	/*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_PRBUFFER;
+
+/*
+ * Used to share frame numbers across UM-KM-FW,
+ * frame number is set in UM,
+ * frame number is required in both KM for HTB and FW for FW trace.
+ *
+ * May be used to house Kick flags in the future.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32FrameNum; /*!< associated frame number */
+} CMD_COMMON;
+
+/*
+ * TA and 3D commands require set of firmware addresses that are stored in the
+ * Kernel. Client has handle(s) to Kernel containers storing these addresses,
+ * instead of raw addresses. We have to patch/write these addresses in KM to
+ * prevent UM from controlling FW addresses directly.
+ * Typedefs for TA and 3D commands are shared between Client and Firmware (both
+ * single-BVNC). Kernel is implemented in a multi-BVNC manner, so it can't use
+ * TA|3D CMD type definitions directly. Therefore we have a SHARED block that
+ * is shared between UM-KM-FW across all BVNC configurations.
+ */
+typedef struct
+{
+	CMD_COMMON           sCmn;      /*!< Common command attributes */
+	RGXFWIF_DEV_VIRTADDR sHWRTData; /* RTData associated with this command,
+									   this is used for context selection and for storing out HW-context,
+									   when TA is switched out for continuing later */
+
+	RGXFWIF_DEV_VIRTADDR asPRBuffer[RGXFWIF_PRBUFFER_MAXSUPPORTED];	/* Supported PR Buffers like Z/S/MSAA Scratch */
+
+} CMDTA3D_SHARED;
+
+/*!
+ * Client Circular Command Buffer (CCCB) control structure.
+ * This is shared between the Server and the Firmware and holds byte offsets
+ * into the CCCB as well as the wrapping mask to aid wrap around. A given
+ * snapshot of this queue with Cmd 1 running on the GPU might be:
+ *
+ *          Roff                           Doff                 Woff
+ * [..........|-1----------|=2===|=3===|=4===|~5~~~~|~6~~~~|~7~~~~|..........]
+ *            <      runnable commands       ><   !ready to run   >
+ *
+ * Cmd 1    : Currently executing on the GPU data master.
+ * Cmd 2,3,4: Fence dependencies met, commands runnable.
+ * Cmd 5... : Fence dependency not met yet.
+ */
+typedef struct
+{
+	IMG_UINT32  ui32WriteOffset;    /*!< Host write offset into CCB. This
+	                                 *    must be aligned to 16 bytes. */
+	IMG_UINT32  ui32ReadOffset;     /*!< Firmware read offset into CCB.
+	                                      Points to the command that is
+	                                 *    runnable on GPU, if R!=W */
+	IMG_UINT32  ui32DepOffset;      /*!< Firmware fence dependency offset.
+	                                 *    Points to commands not ready, i.e.
+	                                 *    fence dependencies are not met. */
+	IMG_UINT32  ui32WrapMask;       /*!< Offset wrapping mask, total capacity
+	                                      in bytes of the CCB-1 */
+
+	IMG_UINT32  ui32ReadOffset2;     /*!< Firmware 2nd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R2!=W */
+	IMG_UINT32  ui32ReadOffset3;     /*!< Firmware 3rd read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R3!=W */
+	IMG_UINT32  ui32ReadOffset4;     /*!< Firmware 4th read offset into CCB for AGP.
+	                                      Points to the command that is
+	                                      runnable on GPU, if R4!=W */
+
+} UNCACHED_ALIGN RGXFWIF_CCCB_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_CCCB_CTL) == 32,
+				"RGXFWIF_CCCB_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef IMG_UINT32 RGXFW_FREELIST_TYPE;
+
+#define RGXFW_LOCAL_FREELIST     IMG_UINT32_C(0)
+#define RGXFW_GLOBAL_FREELIST    IMG_UINT32_C(1)
+#if defined(SUPPORT_AGP)
+#define RGXFW_GLOBAL2_FREELIST   IMG_UINT32_C(2)
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL2_FREELIST + 1U)
+#else
+#define RGXFW_MAX_FREELISTS      (RGXFW_GLOBAL_FREELIST + 1U)
+#endif
+#define RGXFW_MAX_HWFREELISTS    (2U)
+
+/*!
+ * @Defgroup ContextSwitching Context switching data interface
+ * @Brief Types grouping data structures and defines used in realising the Context Switching (CSW) functionality
+ * @{
+ */
+
+/*!
+ * @Brief GEOM DM or TA register controls for context switch
+ */
+typedef struct
+{
+	IMG_UINT64	uTAReg_VDM_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the VDM's context state buffer */
+	IMG_UINT64	uTAReg_VDM_CONTEXT_STATE_RESUME_ADDR;
+	IMG_UINT64	uTAReg_TA_CONTEXT_STATE_BASE_ADDR; /*!< The base address of the TA's context state buffer */
+
+	struct
+	{
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK0; /*!< VDM context store task 0 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK1; /*!< VDM context store task 1 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_STORE_TASK2; /*!< VDM context store task 2 */
+
+		/* VDM resume state update controls */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK0; /*!< VDM context resume task 0 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK1; /*!< VDM context resume task 1 */
+		IMG_UINT64	uTAReg_VDM_CONTEXT_RESUME_TASK2; /*!< VDM context resume task 2 */
+	} asTAState[2];
+
+} RGXFWIF_TAREGISTERS_CSWITCH;
+/*! @} End of Defgroup ContextSwitching */
+
+#define RGXFWIF_TAREGISTERS_CSWITCH_SIZE sizeof(RGXFWIF_TAREGISTERS_CSWITCH)
+
+typedef struct
+{
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS1;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS;
+	IMG_UINT64	uCDMReg_CDM_TERMINATE_PDS1;
+
+	/* CDM resume controls */
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0;
+	IMG_UINT64	uCDMReg_CDM_CONTEXT_PDS0_B;
+	IMG_UINT64	uCDMReg_CDM_RESUME_PDS0_B;
+
+} RGXFWIF_CDM_REGISTERS_CSWITCH;
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Render context static register controls for context switch
+ */
+typedef struct
+{
+	RGXFWIF_TAREGISTERS_CSWITCH	RGXFW_ALIGN asCtxSwitch_GeomRegs[RGX_NUM_GEOM_CORES];	/*!< Geom registers for ctx switch */
+} RGXFWIF_STATIC_RENDERCONTEXT_STATE;
+
+#define RGXFWIF_STATIC_RENDERCONTEXT_SIZE sizeof(RGXFWIF_STATIC_RENDERCONTEXT_STATE)
+
+typedef struct
+{
+	RGXFWIF_CDM_REGISTERS_CSWITCH	RGXFW_ALIGN sCtxSwitch_Regs;	/*!< CDM registers for ctx switch */
+} RGXFWIF_STATIC_COMPUTECONTEXT_STATE;
+
+#define RGXFWIF_STATIC_COMPUTECONTEXT_SIZE sizeof(RGXFWIF_STATIC_COMPUTECONTEXT_STATE)
+
+/*!
+	@Brief Context reset reason. Last reset reason for a reset context.
+*/
+typedef enum
+{
+	RGX_CONTEXT_RESET_REASON_NONE                = 0,	/*!< No reset reason recorded */
+	RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP       = 1,	/*!< Caused a reset due to locking up */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP     = 2,	/*!< Affected by another context locking up */
+	RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING   = 3,	/*!< Overran the global deadline */
+	RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING = 4,	/*!< Affected by another context overrunning */
+	RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH = 5,	/*!< Forced reset to ensure scheduling requirements */
+	RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM        = 6,	/*!< CDM Mission/safety checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM        = 7,	/*!< TRP checksum mismatch */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_OK          = 8,	/*!< GPU ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR         = 9,	/*!< GPU ECC error (uncorrected, HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_OK           = 10,	/*!< FW ECC error (corrected, OK) */
+	RGX_CONTEXT_RESET_REASON_FW_ECC_ERR          = 11,	/*!< FW ECC error (uncorrected, ERR) */
+	RGX_CONTEXT_RESET_REASON_FW_WATCHDOG         = 12,	/*!< FW Safety watchdog triggered */
+	RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT        = 13,	/*!< FW page fault (no HWR) */
+	RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR         = 14,	/*!< FW execution error (GPU reset requested) */
+	RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR     = 15,	/*!< Host watchdog detected FW error */
+	RGX_CONTEXT_GEOM_OOM_DISABLED                = 16,	/*!< Geometry DM OOM event is not allowed */
+	RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH         = 17,	/*!< PVRIC Signature mismatch */
+	RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR   = 18,	/*!< Parity error in MMU Page Table Entry */
+	RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR       = 19,	/*!< Parity error in MH, system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR      = 20,	/*!< Parity error in system bus or Control Status registers */
+	RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR      = 21,	/*!< Latent/ICS signature mismatch error */
+	RGX_CONTEXT_RESET_REASON_DCLS_ERR            = 22,	/*!< Dual Core Lock Step FW error detected */
+} RGX_CONTEXT_RESET_REASON;
+
+/*!
+	@Brief Context reset data shared with the host
+*/
+typedef struct
+{
+	RGX_CONTEXT_RESET_REASON eResetReason; /*!< Reset reason */
+	IMG_UINT32 ui32ResetExtJobRef;  /*!< External Job ID */
+} RGX_CONTEXT_RESET_REASON_DATA;
+
+#define RGX_HEAP_UM_PDS_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_PDS_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_PDS_RESERVED_TOTAL_SIZE            RGX_HEAP_UM_PDS_RESERVED_SIZE
+
+#define RGX_HEAP_UM_USC_RESERVED_SIZE               DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_USC_RESERVED_REGION_OFFSET      0
+#define RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET      RGX_HEAP_UM_USC_RESERVED_SIZE
+
+#define RGX_HEAP_UM_GENERAL_RESERVED_SIZE           DEVMEM_HEAP_RESERVED_SIZE_GRANULARITY
+#define RGX_HEAP_UM_GENERAL_RESERVED_REGION_OFFSET  0
+#define RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET  RGX_HEAP_UM_GENERAL_RESERVED_SIZE
+
+/*************************************************************************/ /*!
+ Logging type
+*/ /**************************************************************************/
+#define RGXFWIF_LOG_TYPE_NONE			0x00000000U
+#define RGXFWIF_LOG_TYPE_TRACE			0x00000001U
+#define RGXFWIF_LOG_TYPE_GROUP_MAIN		0x00000002U
+#define RGXFWIF_LOG_TYPE_GROUP_MTS		0x00000004U
+#define RGXFWIF_LOG_TYPE_GROUP_CLEANUP	0x00000008U
+#define RGXFWIF_LOG_TYPE_GROUP_CSW		0x00000010U
+#define RGXFWIF_LOG_TYPE_GROUP_BIF		0x00000020U
+#define RGXFWIF_LOG_TYPE_GROUP_PM		0x00000040U
+#define RGXFWIF_LOG_TYPE_GROUP_RTD		0x00000080U
+#define RGXFWIF_LOG_TYPE_GROUP_SPM		0x00000100U
+#define RGXFWIF_LOG_TYPE_GROUP_POW		0x00000200U
+#define RGXFWIF_LOG_TYPE_GROUP_HWR		0x00000400U
+#define RGXFWIF_LOG_TYPE_GROUP_HWP		0x00000800U
+#define RGXFWIF_LOG_TYPE_GROUP_RPM		0x00001000U
+#define RGXFWIF_LOG_TYPE_GROUP_DMA		0x00002000U
+#define RGXFWIF_LOG_TYPE_GROUP_MISC		0x00004000U
+#define RGXFWIF_LOG_TYPE_GROUP_VZ		0x00008000U
+#define RGXFWIF_LOG_TYPE_GROUP_SAFETY	0x00010000U
+#define RGXFWIF_LOG_TYPE_GROUP_VERBOSE	0x00020000U
+#define RGXFWIF_LOG_TYPE_GROUP_CUSTOMER	0x00040000U
+#define RGXFWIF_LOG_TYPE_GROUP_DEBUG	0x80000000U
+#define RGXFWIF_LOG_TYPE_GROUP_MASK		0x8007FFFEU
+#define RGXFWIF_LOG_TYPE_MASK			0x8007FFFFU
+
+/* String used in pvrdebug -h output */
+#define RGXFWIF_LOG_GROUPS_STRING_LIST   "main,mts,cleanup,csw,bif,pm,rtd,spm,pow,hwr,hwp,rpm,dma,misc,vz,safety,verbose,customer,debug"
+
+/* Table entry to map log group strings to log type value */
+typedef struct {
+	const IMG_CHAR* pszLogGroupName;
+	IMG_UINT32      ui32LogGroupType;
+} RGXFWIF_LOG_GROUP_MAP_ENTRY;
+
+/*
+  Macro for use with the RGXFWIF_LOG_GROUP_MAP_ENTRY type to create a lookup
+  table where needed. Keep log group names short, no more than 20 chars.
+*/
+#define RGXFWIF_LOG_GROUP_NAME_VALUE_MAP { "none",    RGXFWIF_LOG_TYPE_NONE }, \
+                                         { "main",    RGXFWIF_LOG_TYPE_GROUP_MAIN }, \
+                                         { "mts",     RGXFWIF_LOG_TYPE_GROUP_MTS }, \
+                                         { "cleanup", RGXFWIF_LOG_TYPE_GROUP_CLEANUP }, \
+                                         { "csw",     RGXFWIF_LOG_TYPE_GROUP_CSW }, \
+                                         { "bif",     RGXFWIF_LOG_TYPE_GROUP_BIF }, \
+                                         { "pm",      RGXFWIF_LOG_TYPE_GROUP_PM }, \
+                                         { "rtd",     RGXFWIF_LOG_TYPE_GROUP_RTD }, \
+                                         { "spm",     RGXFWIF_LOG_TYPE_GROUP_SPM }, \
+                                         { "pow",     RGXFWIF_LOG_TYPE_GROUP_POW }, \
+                                         { "hwr",     RGXFWIF_LOG_TYPE_GROUP_HWR }, \
+                                         { "hwp",     RGXFWIF_LOG_TYPE_GROUP_HWP }, \
+                                         { "rpm",     RGXFWIF_LOG_TYPE_GROUP_RPM }, \
+                                         { "dma",     RGXFWIF_LOG_TYPE_GROUP_DMA }, \
+                                         { "misc",    RGXFWIF_LOG_TYPE_GROUP_MISC }, \
+                                         { "vz",      RGXFWIF_LOG_TYPE_GROUP_VZ }, \
+                                         { "safety",  RGXFWIF_LOG_TYPE_GROUP_SAFETY }, \
+                                         { "verbose", RGXFWIF_LOG_TYPE_GROUP_VERBOSE }, \
+                                         { "customer",RGXFWIF_LOG_TYPE_GROUP_CUSTOMER }, \
+                                         { "debug",   RGXFWIF_LOG_TYPE_GROUP_DEBUG }
+
+/* Used in print statements to display log group state, one %s per group defined */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC  "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s"
+
+/* Used in a print statement to display log group state, one per group */
+#define RGXFWIF_LOG_ENABLED_GROUPS_LIST(types)  ((((types) & RGXFWIF_LOG_TYPE_GROUP_MAIN) != 0U)	?("main ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MTS) != 0U)		?("mts ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CLEANUP) != 0U)	?("cleanup ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CSW) != 0U)		?("csw ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_BIF) != 0U)		?("bif ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_PM) != 0U)		?("pm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RTD) != 0U)		?("rtd ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SPM) != 0U)		?("spm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_POW) != 0U)		?("pow ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWR) != 0U)		?("hwr ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_HWP) != 0U)		?("hwp ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_RPM) != 0U)		?("rpm ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DMA) != 0U)		?("dma ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_MISC) != 0U)	?("misc ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VZ) != 0U)		?("vz ")		:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_SAFETY) != 0U)	?("safety ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_VERBOSE) != 0U)	?("verbose ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_CUSTOMER) != 0U)?("customer ")	:("")),		\
+                                                ((((types) & RGXFWIF_LOG_TYPE_GROUP_DEBUG) != 0U)	?("debug ")		:(""))
+
+/*!
+ ******************************************************************************
+ * Trace Buffer
+ *****************************************************************************/
+
+/*! Min, Max, and Default size of RGXFWIF_TRACEBUF_SPACE in DWords */
+#define RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS      8192U  /*  32KB */
+#define RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS 12000U  /* ~48KB */
+#define RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS     32768U  /* 128KB */
+
+#define RGXFW_TRACE_BUFFER_ASSERT_SIZE 200U
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+#define RGXFW_THREAD_NUM 2U
+#else
+#define RGXFW_THREAD_NUM 1U
+#endif
+
+typedef struct
+{
+	IMG_CHAR	szPath[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_CHAR	szInfo[RGXFW_TRACE_BUFFER_ASSERT_SIZE];
+	IMG_UINT32	ui32LineNum;
+} UNCACHED_ALIGN RGXFWIF_FILE_INFO_BUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FILE_INFO_BUF) == 408,
+				"RGXFWIF_FILE_INFO_BUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @Defgroup SRVAndFWTracing Services and Firmware Tracing data interface
+ * @Brief The document groups/lists the data structures and the interfaces related to Services and Firmware Tracing
+ * @{
+ */
+
+/*!
+ * @Brief Firmware trace buffer details
+ */
+typedef struct
+{
+	IMG_UINT32                         ui32TracePointer;          /*!< Trace pointer (write index into Trace Buffer) */
+	IMG_UINT32                         ui32WrapCount;             /*!< Number of times the Trace Buffer has wrapped */
+
+	RGXFWIF_DEV_VIRTADDR               pui32RGXFWIfTraceBuffer;   /*!< Trace buffer address (FW address), to be used by firmware for writing into trace buffer */
+
+	RGXFWIF_FILE_INFO_BUF RGXFW_ALIGN  sAssertBuf;
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF_SPACE;
+
+/*! @} End of Defgroup SRVAndFWTracing */
+
+/*!
+ * @InGroup SRVAndFWTracing
+ * @Brief Firmware trace control data
+ */
+typedef struct
+{
+	IMG_UINT32              ui32LogType;                  /*!< FW trace log group configuration */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[MAX_THREAD_NUM];  /*!< FW Trace buffer */
+#else
+	RGXFWIF_TRACEBUF_SPACE  sTraceBuf[RGXFW_THREAD_NUM];  /*!< FW Trace buffer */
+#endif
+	IMG_UINT32              ui32TraceBufSizeInDWords;     /*!< FW Trace buffer size in dwords, Member initialised only when sTraceBuf is actually allocated
+															(in RGXTraceBufferInitOnDemandResources) */
+	IMG_UINT32              ui32TracebufFlags;            /*!< Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_TRACEBUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_TRACEBUF) == 880,
+				"RGXFWIF_TRACEBUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Debug-info sub-fields */
+/* Bit 0: RGX_CR_EVENT_STATUS_MMU_PAGE_FAULT bit from RGX_CR_EVENT_STATUS register */
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT        (0U)
+#define RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET          (1U << RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SHIFT)
+
+/* Bit 1: RGX_CR_BIF_MMU_ENTRY_PENDING bit from RGX_CR_BIF_MMU_ENTRY register */
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT     (1U)
+#define RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET       (1U << RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SHIFT)
+
+/* Bit 2: RGX_CR_SLAVE_EVENT register is non-zero */
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT          (2U)
+#define RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET            (1U << RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SHIFT)
+
+/* Bit 3-15: Unused bits */
+
+#define RGXFWT_DEBUG_INFO_STR_MAXLEN                  64
+#define RGXFWT_DEBUG_INFO_STR_PREPEND                 " (debug info: "
+#define RGXFWT_DEBUG_INFO_STR_APPEND                  ")"
+
+/* Table of debug info sub-field's masks and corresponding message strings
+ * to be appended to firmware trace
+ *
+ * Mask     : 16 bit mask to be applied to debug-info field
+ * String   : debug info message string
+ */
+
+#define RGXFWT_DEBUG_INFO_MSKSTRLIST \
+/*Mask,                                           String*/ \
+X(RGXFWT_DEBUG_INFO_MMU_PAGE_FAULT_SET,      "mmu pf") \
+X(RGXFWT_DEBUG_INFO_MMU_ENTRY_PENDING_SET,   "mmu pending") \
+X(RGXFWT_DEBUG_INFO_SLAVE_EVENTS_SET,        "slave events")
+
+/* Firmware trace time-stamp field breakup */
+
+/* RGX_CR_TIMER register read (48 bits) value*/
+#define RGXFWT_TIMESTAMP_TIME_SHIFT                   (0U)
+#define RGXFWT_TIMESTAMP_TIME_CLRMSK                  (IMG_UINT64_C(0xFFFF000000000000))
+
+/* Extra debug-info (16 bits) */
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT             (48U)
+#define RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK            ~RGXFWT_TIMESTAMP_TIME_CLRMSK
+
+typedef struct
+{
+	IMG_UINT			bfOsState		: 3;
+	IMG_UINT			bfFLOk			: 1;
+	IMG_UINT			bfFLGrowPending	: 1;
+	IMG_UINT			bfReserved		: 27;
+} RGXFWIF_OS_RUNTIME_FLAGS;
+
+#define RGXFWIF_FWFAULTINFO_MAX		(8U)			/* Total number of FW fault logs stored */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64CRTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64OSTimer;
+	IMG_UINT64 RGXFW_ALIGN	ui64Data;
+	RGXFWIF_FILE_INFO_BUF	sFaultBuf;
+} UNCACHED_ALIGN RGX_FWFAULTINFO;
+
+#define RGXFWIF_POW_STATES \
+  X(RGXFWIF_POW_OFF)			/* idle and handshaked with the host (ready to full power down) */ \
+  X(RGXFWIF_POW_ON)				/* running HW commands */ \
+  X(RGXFWIF_POW_FORCED_IDLE)	/* forced idle */ \
+  X(RGXFWIF_POW_IDLE)			/* idle waiting for host handshake */
+
+typedef enum
+{
+#define X(NAME) NAME,
+	RGXFWIF_POW_STATES
+#undef X
+} RGXFWIF_POW_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_THREAD_NUM 2
+
+static_assert(RGXFW_THREAD_NUM <= MAX_THREAD_NUM,
+				"RGXFW_THREAD_NUM is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Firmware HWR states */
+#define RGXFWIF_HWR_HARDWARE_OK			(IMG_UINT32_C(0x1) << 0U)	/*!< The HW state is ok or locked up */
+#define RGXFWIF_HWR_RESET_IN_PROGRESS	(IMG_UINT32_C(0x1) << 1U)	/*!< Tells if a HWR reset is in progress */
+#define RGXFWIF_HWR_GENERAL_LOCKUP		(IMG_UINT32_C(0x1) << 3U)	/*!< A DM unrelated lockup has been detected */
+#define RGXFWIF_HWR_DM_RUNNING_OK		(IMG_UINT32_C(0x1) << 4U)	/*!< At least one DM is running without being close to a lockup */
+#define RGXFWIF_HWR_DM_STALLING			(IMG_UINT32_C(0x1) << 5U)	/*!< At least one DM is close to lockup */
+#define RGXFWIF_HWR_FW_FAULT			(IMG_UINT32_C(0x1) << 6U)	/*!< The FW has faulted and needs to restart */
+#define RGXFWIF_HWR_RESTART_REQUESTED	(IMG_UINT32_C(0x1) << 7U)	/*!< The FW has requested the host to restart it */
+
+#define RGXFWIF_PHR_MODE_OFF			(0UL)
+#define RGXFWIF_PHR_MODE_RD_RESET		(1UL)
+
+typedef IMG_UINT32 RGXFWIF_HWR_STATEFLAGS;
+
+typedef IMG_UINT32 RGXFWIF_HWR_RECOVERYFLAGS;
+
+/*! @Brief Firmware system data shared with the Host driver */
+typedef struct
+{
+	IMG_UINT32                 ui32ConfigFlags;                       /*!< Configuration flags from host */
+	IMG_UINT32                 ui32ConfigFlagsExt;                    /*!< Extended configuration flags from host */
+	volatile RGXFWIF_POW_STATE ePowState;
+	struct {
+		volatile IMG_UINT32        ui32HWPerfRIdx;
+		volatile IMG_UINT32        ui32HWPerfWIdx;
+		volatile IMG_UINT32        ui32HWPerfWrapCount;
+	} sHWPerfCtrl; /* Struct used to inval/flush HWPerfCtrl members */
+	IMG_UINT32                 ui32HWPerfSize;                        /*!< Constant after setup, needed in FW */
+	IMG_UINT32                 ui32HWPerfDropCount;                   /*!< The number of times the FW drops a packet due to buffer full */
+
+	/* ui32HWPerfUt, ui32FirstDropOrdinal, ui32LastDropOrdinal only valid when FW is built with
+	 * RGX_HWPERF_UTILIZATION & RGX_HWPERF_DROP_TRACKING defined in rgxfw_hwperf.c */
+	IMG_UINT32                 ui32HWPerfUt;                          /*!< Buffer utilisation, high watermark of bytes in use */
+	IMG_UINT32                 ui32FirstDropOrdinal;                  /*!< The ordinal of the first packet the FW dropped */
+	IMG_UINT32                 ui32LastDropOrdinal;                   /*!< The ordinal of the last packet the FW dropped */
+	RGXFWIF_OS_RUNTIME_FLAGS   asOsRuntimeFlagsMirror[RGXFW_MAX_NUM_OSIDS];/*!< State flags for each Operating System mirrored from Fw coremem */
+	RGX_FWFAULTINFO            sFaultInfo[RGXFWIF_FWFAULTINFO_MAX];   /*!< Firmware fault info */
+	IMG_UINT32                 ui32FWFaults;                          /*!< Firmware faults count */
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32                 aui32CrPollAddr[MAX_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[MAX_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[MAX_THREAD_NUM];    /*!< Failed poll count */
+#else
+	IMG_UINT32                 aui32CrPollAddr[RGXFW_THREAD_NUM];     /*!< Failed poll address */
+	IMG_UINT32                 aui32CrPollMask[RGXFW_THREAD_NUM];     /*!< Failed poll mask */
+	IMG_UINT32                 aui32CrPollCount[RGXFW_THREAD_NUM];    /*!< Failed poll count */
+#endif
+	IMG_UINT64 RGXFW_ALIGN     ui64StartIdleTime;
+	IMG_UINT32                 aui32TSMirror[RGXFW_MAX_NUM_OSIDS];    /*!< Array of time slice per OS Mirrored from the FW */
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define RGXFWIF_STATS_FRAMEWORK_LINESIZE    (8)
+#define RGXFWIF_STATS_FRAMEWORK_MAX         (2048*RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+	IMG_UINT32 RGXFW_ALIGN     aui32FWStatsBuf[RGXFWIF_STATS_FRAMEWORK_MAX];
+#endif
+	RGXFWIF_HWR_STATEFLAGS     ui32HWRStateFlags; /*!< Firmware's Current HWR state */
+	RGXFWIF_HWR_RECOVERYFLAGS  aui32HWRRecoveryFlags[RGXFWIF_DM_MAX]; /*!< Each DM's HWR state */
+	IMG_UINT32                 ui32FwSysDataFlags;                      /*!< Compatibility and other flags */
+	IMG_UINT32                 ui32McConfig;                            /*!< Identify whether MC config is P-P or P-S */
+	IMG_UINT32                 ui32MemFaultCheck;                       /*!< Device mem fault check on PCI systems */
+} UNCACHED_ALIGN RGXFWIF_SYSDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_SYSDATA) == 3624,
+				"RGXFWIF_SYSDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER == 3624");
+#endif
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+#define PVR_SLR_LOG_ENTRIES 10U
+#define PVR_SLR_LOG_STRLEN  30 /*!< MAX_CLIENT_CCB_NAME not visible to this header */
+
+typedef struct
+{
+	IMG_UINT64 RGXFW_ALIGN	ui64Timestamp;
+	IMG_UINT32				ui32FWCtxAddr;
+	IMG_UINT32				ui32NumUFOs;
+	IMG_CHAR				aszCCBName[PVR_SLR_LOG_STRLEN];
+} UNCACHED_ALIGN RGXFWIF_SLR_ENTRY;
+#endif
+
+#define RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS (128U)
+
+/*!
+ * @InGroup ContextSwitching
+ * @Brief Firmware per-os data and configuration
+ */
+typedef struct
+{
+	IMG_UINT32                 ui32FwOsConfigFlags;                   /*!< Configuration flags from an OS */
+	IMG_UINT32                 ui32FWSyncCheckMark;                   /*!< Markers to signal that the host should perform a full sync check */
+	IMG_UINT32                 ui32HostSyncCheckMark;                  /*!< Markers to signal that the Firmware should perform a full sync check */
+#if defined(PVRSRV_STALLED_CCB_ACTION) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32                 ui32ForcedUpdatesRequested;
+	IMG_UINT8                  ui8SLRLogWp;
+	RGXFWIF_SLR_ENTRY          sSLRLogFirst;
+	RGXFWIF_SLR_ENTRY          sSLRLog[PVR_SLR_LOG_ENTRIES];
+	IMG_UINT64 RGXFW_ALIGN     ui64LastForcedUpdateTime;
+#endif
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	volatile IMG_UINT32        aui32InterruptCount[MAX_THREAD_NUM]; /*!< Interrupt count from Threads > */
+#else
+	volatile IMG_UINT32        aui32InterruptCount[RGXFW_THREAD_NUM]; /*!< Interrupt count from Threads > */
+#endif
+	IMG_UINT32                 ui32KCCBCmdsExecuted;                  /*!< Executed Kernel CCB command count */
+	RGXFWIF_DEV_VIRTADDR       sPowerSync;                            /*!< Sync prim used to signal the host the power off state */
+	IMG_UINT32                 ui32FwOsDataFlags;                       /*!< Compatibility and other flags */
+	IMG_UINT32                 aui32OfflineBuffer[RGXFWIF_OFFLINE_DATA_BUFFER_SIZE_IN_WORDS];
+} UNCACHED_ALIGN RGXFWIF_OSDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_OSDATA) == 584,
+				"RGXFWIF_OSDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+/*!
+ ******************************************************************************
+ * HWR Data
+ *****************************************************************************/
+/*!
+ * @Defgroup HWRInfo FW HWR shared data interface
+ * @Brief Types grouping data structures and defines used in realising the HWR record.
+ * @{
+ */
+
+#define RGXFW_PROCESS_NAME_LEN	(16)
+
+/*! @Brief HWR Lockup types */
+typedef enum
+{
+	RGX_HWRTYPE_UNKNOWNFAILURE = 0, /*!< Unknown failure */
+	RGX_HWRTYPE_OVERRUN        = 1, /*!< DM overrun */
+	RGX_HWRTYPE_POLLFAILURE    = 2, /*!< Poll failure */
+	RGX_HWRTYPE_BIF0FAULT      = 3, /*!< BIF0 fault */
+	RGX_HWRTYPE_BIF1FAULT      = 4, /*!< BIF1 fault */
+	RGX_HWRTYPE_TEXASBIF0FAULT = 5, /*!< TEXASBIF0 fault */
+	RGX_HWRTYPE_MMUFAULT       = 6, /*!< MMU fault */
+	RGX_HWRTYPE_MMUMETAFAULT   = 7, /*!< MMU META fault */
+	RGX_HWRTYPE_MIPSTLBFAULT   = 8, /*!< MIPS TLB fault */
+	RGX_HWRTYPE_ECCFAULT       = 9, /*!< ECC fault */
+	RGX_HWRTYPE_MMURISCVFAULT  = 10, /*!< MMU RISCV fault */
+} RGX_HWRTYPE;
+
+#define RGXFWIF_HWRTYPE_BIF_BANK_GET(eHWRType) (((eHWRType) == RGX_HWRTYPE_BIF0FAULT) ? 0 : 1)
+
+#define RGXFWIF_HWRTYPE_PAGE_FAULT_GET(eHWRType) ((((eHWRType) == RGX_HWRTYPE_BIF0FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_BIF1FAULT)      ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_TEXASBIF0FAULT) ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUFAULT)       ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMUMETAFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MIPSTLBFAULT)   ||       \
+                                                   ((eHWRType) == RGX_HWRTYPE_MMURISCVFAULT)) ? true : false)
+
+/************************
+ *  GPU HW error codes  *
+ ************************/
+typedef enum
+{
+	RGX_HW_ERR_NA = 0x0,
+	RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL = 0x101,
+} RGX_HW_ERR;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFReqStatus; /*!< BIF request status */
+	IMG_UINT64	RGXFW_ALIGN		ui64BIFMMUStatus; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_BIFINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32FaultGPU; /*!< ECC fault in GPU */
+} RGX_ECCINFO;
+
+typedef struct
+{
+	IMG_UINT64	RGXFW_ALIGN		aui64MMUStatus[2]; /*!< MMU status */
+	IMG_UINT64	RGXFW_ALIGN		ui64PCAddress; /*!< phys address of the page catalogue */
+	IMG_UINT64	RGXFW_ALIGN		ui64Reserved;
+} RGX_MMUINFO;
+
+typedef struct
+{
+	IMG_UINT32	ui32ThreadNum; /*!< Thread ID performing poll operation */
+	IMG_UINT32	ui32CrPollAddr; /*!< CR Poll Address */
+	IMG_UINT32	ui32CrPollMask; /*!< CR Poll mask */
+	IMG_UINT32	ui32CrPollLastValue; /*!< CR Poll last value */
+	IMG_UINT64	RGXFW_ALIGN ui64Reserved;
+} UNCACHED_ALIGN RGX_POLLINFO;
+
+typedef struct
+{
+	IMG_UINT32 ui32BadVAddr; /*!< VA address */
+	IMG_UINT32 ui32EntryLo;
+} RGX_TLBINFO;
+
+/*! @Brief Structure to keep information specific to a lockup e.g. DM, timer, lockup type etc. */
+typedef struct
+{
+	union
+	{
+		RGX_BIFINFO  sBIFInfo; /*!< BIF failure details */
+		RGX_MMUINFO  sMMUInfo; /*!< MMU failure details */
+		RGX_POLLINFO sPollInfo; /*!< Poll failure details */
+		RGX_TLBINFO  sTLBInfo; /*!< TLB failure details */
+		RGX_ECCINFO  sECCInfo; /*!< ECC failure details */
+	} uHWRData;
+
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimer; /*!< Timer value at the time of lockup */
+	IMG_UINT64 RGXFW_ALIGN ui64OSTimer; /*!< OS timer value at the time of lockup */
+	IMG_UINT32             ui32FrameNum; /*!< Frame number of the workload */
+	IMG_UINT32             ui32PID; /*!< PID belonging to the workload */
+	IMG_UINT32             ui32ActiveHWRTData; /*!< HWRT data of the workload */
+	IMG_UINT32             ui32HWRNumber; /*!< HWR number */
+	IMG_UINT32             ui32EventStatus; /*!< Core specific event status register at the time of lockup */
+	IMG_UINT32             ui32HWRRecoveryFlags; /*!< DM state flags */
+	RGX_HWRTYPE            eHWRType; /*!< Type of lockup */
+	RGXFWIF_DM             eDM; /*!< Recovery triggered for the DM */
+	IMG_UINT32             ui32CoreID; /*!< Core ID of the GPU */
+	RGX_HW_ERR             eHWErrorCode; /*!< Error code used to determine HW fault */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeOfKick; /*!< Workload kick time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetStart; /*!< HW reset start time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeHWResetFinish; /*!< HW reset stop time */
+	IMG_UINT64 RGXFW_ALIGN ui64CRTimeFreelistReady; /*!< freelist ready time on the last HWR */
+	IMG_CHAR   RGXFW_ALIGN szProcName[RGXFW_PROCESS_NAME_LEN]; /*!< User process name */
+} UNCACHED_ALIGN RGX_HWRINFO;
+
+#define RGXFWIF_HWINFO_MAX_FIRST 8U							/* Number of first HWR logs recorded (never overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX_LAST 8U							/* Number of latest HWR logs (older logs are overwritten by newer logs) */
+#define RGXFWIF_HWINFO_MAX (RGXFWIF_HWINFO_MAX_FIRST + RGXFWIF_HWINFO_MAX_LAST)	/* Total number of HWR logs stored in a buffer */
+#define RGXFWIF_HWINFO_LAST_INDEX (RGXFWIF_HWINFO_MAX - 1U)	/* Index of the last log in the HWR log buffer */
+
+/*! @Brief Firmware HWR information structure allocated by the Services and used by the Firmware to update recovery information. */
+typedef struct
+{
+	RGX_HWRINFO sHWRInfo[RGXFWIF_HWINFO_MAX]; /*!< Max number of recovery record */
+	IMG_UINT32  ui32HwrCounter; /*!< HWR counter used in FL reconstruction */
+	IMG_UINT32  ui32WriteIndex; /*!< Index for updating recovery information in sHWRInfo */
+	IMG_UINT32  ui32DDReqCount; /*!< Count of DebugDump requested to the host after recovery */
+	IMG_UINT32  ui32HWRInfoBufFlags; /* Compatibility and other flags */
+	IMG_UINT32  aui32HwrDmLockedUpCount[RGXFWIF_DM_MAX]; /*!< Lockup count for each DM */
+	IMG_UINT32  aui32HwrDmOverranCount[RGXFWIF_DM_MAX]; /*!< Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmRecoveredCount[RGXFWIF_DM_MAX]; /*!< Lockup + Overrun count for each DM */
+	IMG_UINT32  aui32HwrDmFalseDetectCount[RGXFWIF_DM_MAX]; /*!< False lockup detection count for each DM */
+} UNCACHED_ALIGN RGXFWIF_HWRINFOBUF;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRINFOBUF) == 2336,
+				"RGXFWIF_HWRINFOBUF is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*! @} End of HWRInfo */
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_CCCB_CTL;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_COMMONCTX_STATE;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_RF_CMD;
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FWMEMCONTEXT;
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Freelist holding usage state of the Parameter Buffers
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR	RGXFW_ALIGN psFreeListDevVAddr;	/*!< Freelist page table base */
+	IMG_UINT64		RGXFW_ALIGN ui64CurrentDevVAddr;/*!< Freelist page table entry for current free page */
+	IMG_UINT32		ui32CurrentStackTop;		/*!< Freelist current free page */
+	IMG_UINT32		ui32MaxPages;			/*!< Max no. of pages can be added to the freelist */
+	IMG_UINT32		ui32GrowPages;			/*!< No pages to add in each freelist grow */
+	IMG_UINT32		ui32CurrentPages;		/*!< Total no. of pages made available to the PM HW */
+	IMG_UINT32		ui32AllocatedPageCount;		/*!< No. of pages allocated by PM HW */
+	IMG_UINT32		ui32AllocatedMMUPageCount;	/*!< No. of pages allocated for GPU MMU for PM*/
+#if defined(SUPPORT_SHADOW_FREELISTS) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32		ui32HWRCounter;
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;
+#endif
+	IMG_UINT32		ui32FreeListID;			/*!< Unique Freelist ID */
+	IMG_BOOL		bGrowPending;			/*!< Freelist grow is pending */
+	IMG_UINT32		ui32ReadyPages;			/*!< Reserved pages to be used only on PM OOM event */
+	IMG_UINT32		ui32FreelistFlags;		/*!< Compatibility and other flags */
+#if defined(SUPPORT_AGP) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32		ui32PmGlobalPb;			/*!< PM Global PB on which Freelist is loaded */
+#endif
+} UNCACHED_ALIGN RGXFWIF_FREELIST;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FREELIST) == 64,
+				"RGXFWIF_FREELIST is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct {RGXFWIF_DEV_VIRTADDR sNext;
+                RGXFWIF_DEV_VIRTADDR sPrev;}	RGXFW_DLLIST_NODE;
+
+#define RGXFWIF_MAX_NUM_CANCEL_REQUESTS		(8U)		/* Maximum number of workload cancellation requests */
+
+/*!
+ * @InGroup WorkloadContexts
+ * @Brief Firmware Common Context (or FWCC)
+ */
+typedef struct RGXFWIF_FWCOMMONCONTEXT_
+{
+	/* CCB details for this firmware context */
+	PRGXFWIF_CCCB_CTL		psCCBCtl;				/*!< CCB control */
+	PRGXFWIF_CCCB			psCCB;					/*!< CCB base */
+
+	/* Context suspend state */
+	PRGXFWIF_COMMONCTX_STATE	RGXFW_ALIGN psContextState;		/*!< TA/3D context suspend state, read/written by FW */
+
+	/* Flags e.g. for context switching */
+	IMG_UINT32				ui32FWComCtxFlags;
+	IMG_INT32				i32Priority;  /*!< Priority level */
+	IMG_UINT32				ui32PrioritySeqNum;
+
+	/* Framework state */
+	PRGXFWIF_RF_CMD			RGXFW_ALIGN psRFCmd;		/*!< Register updates for Framework */
+
+	/* Misc and compatibility flags */
+	IMG_UINT32				ui32CompatFlags;
+
+	/* Statistic updates waiting to be passed back to the host... */
+	IMG_INT32				i32StatsNumStores;		/*!< Number of stores on this context since last update */
+	IMG_INT32				i32StatsNumOutOfMemory;		/*!< Number of OOMs on this context since last update */
+	IMG_INT32				i32StatsNumPartialRenders;	/*!< Number of PRs on this context since last update */
+	RGXFWIF_DM				eDM;				/*!< Data Master type */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN  sBufStalledNode;			/*!< List entry for the buffer stalled list */
+	IMG_UINT64				RGXFW_ALIGN  ui64CBufQueueCtrlAddr;	/*!< Address of the circular buffer queue pointers */
+
+	IMG_UINT64				RGXFW_ALIGN  ui64RobustnessAddress;
+	IMG_UINT32				ui32MaxDeadlineMS;			/*!< Max HWR deadline limit in ms */
+	bool					bReadOffsetNeedsReset;			/*!< Following HWR circular buffer read-offset needs resetting */
+
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sWaitingNode;		/*!< List entry for the waiting list */
+	RGXFW_DLLIST_NODE		RGXFW_ALIGN sRunNode;			/*!< List entry for the run list */
+	RGXFWIF_UFO				sLastFailedUFO;			/*!< UFO that last failed (or NULL) */
+
+	PRGXFWIF_FWMEMCONTEXT	psFWMemContext;					/*!< Memory context */
+
+	/* References to the host side originators */
+	IMG_UINT32				ui32ServerCommonContextID;	/*!< the Server Common Context */
+	IMG_UINT32				ui32PID;			/*!< associated process ID */
+
+	IMG_BOOL				bGeomOOMDisabled;		/*!< True when Geom DM OOM is not allowed */
+	IMG_CHAR				szProcName[RGXFW_PROCESS_NAME_LEN];	/*!< User process name */
+	IMG_UINT32				ui32DeferCount;		/*!< Number of context defers before forced scheduling of context */
+	IMG_UINT32				aui32FirstIntJobRefToCancel[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the beginning of range of IntJobRefs at and above which workloads will be discarded */
+	IMG_UINT32				aui32FirstValidIntJobRef[RGXFWIF_MAX_NUM_CANCEL_REQUESTS];	/*!< Saved values of the end of range of IntJobRef below which workloads will be discarded */
+	IMG_BOOL				bCancelRangesActive;	/*!< True if any active ranges in aui32FirstIntJobRefToCancel and aui32FirstValidIntJobRef arrays */
+	IMG_BOOL				bLastKickedCmdWasSafetyOnly;
+	IMG_UINT32				ui32UID;			/*!< associated process UID used in FW managed gpu work period hwperf events */
+} UNCACHED_ALIGN RGXFWIF_FWCOMMONCONTEXT;
+
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) <= 256U,
+              "Size of structure RGXFWIF_FWCOMMONCONTEXT exceeds maximum expected size.");
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_FWCOMMONCONTEXT) == 168,
+				"RGXFWIF_FWCOMMONCONTEXT is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ ******************************************************************************
+ * HWRTData
+ *****************************************************************************/
+
+typedef RGXFWIF_DEV_VIRTADDR  PRGXFWIF_FREELIST;
+
+/* HWRTData flags */
+/* Deprecated flags 1:0 */
+#define HWRTDATA_HAS_LAST_TA              (IMG_UINT32_C(1) << 2)
+#define HWRTDATA_PARTIAL_RENDERED         (IMG_UINT32_C(1) << 3)
+#define HWRTDATA_DISABLE_TILE_REORDERING  (IMG_UINT32_C(1) << 4)
+#define HWRTDATA_NEED_BRN65101_BLIT       (IMG_UINT32_C(1) << 5)
+#define HWRTDATA_FIRST_BRN65101_STRIP     (IMG_UINT32_C(1) << 6)
+#define HWRTDATA_NEED_BRN67182_2ND_RENDER (IMG_UINT32_C(1) << 7)
+#if defined(SUPPORT_AGP)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT0    (IMG_UINT32_C(1) << 8)
+#if defined(SUPPORT_AGP4)
+#define HWRTDATA_GLOBAL_PB_NUMBER_BIT1    (IMG_UINT32_C(1) << 9)
+#endif
+#define HWRTDATA_GEOM_NEEDS_RESUME        (IMG_UINT32_C(1) << 10)
+#endif
+#if defined(SUPPORT_TRP)
+#define HWRTDATA_GEOM_TRP_IN_PROGRESS          (IMG_UINT32_C(1) << 9)
+#endif
+
+typedef enum
+{
+	RGXFWIF_RTDATA_STATE_NONE = 0,
+	RGXFWIF_RTDATA_STATE_KICKTA,
+	RGXFWIF_RTDATA_STATE_KICKTAFIRST,
+	RGXFWIF_RTDATA_STATE_TAFINISHED,
+	RGXFWIF_RTDATA_STATE_KICK3D,
+	RGXFWIF_RTDATA_STATE_3DFINISHED,
+	RGXFWIF_RTDATA_STATE_3DCONTEXTSTORED,
+	RGXFWIF_RTDATA_STATE_TAOUTOFMEM,
+	RGXFWIF_RTDATA_STATE_PARTIALRENDERFINISHED,
+	/* In case of HWR, we can't set the RTDATA state to NONE,
+	 * as this will cause any TA to become a first TA.
+	 * To ensure all related TA's are skipped, we use the HWR state */
+	RGXFWIF_RTDATA_STATE_HWR,
+	RGXFWIF_RTDATA_STATE_UNKNOWN = 0x7FFFFFFFU
+} RGXFWIF_RTDATA_STATE;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+#define MAX_FREELISTS_SIZE 3
+
+static_assert(RGXFW_MAX_FREELISTS <= MAX_FREELISTS_SIZE,
+				"RGXFW_MAX_FREELISTS is outside of allowable range for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct
+{
+	IMG_BOOL							bTACachesNeedZeroing;
+
+	IMG_UINT32							ui32ScreenPixelMax;
+	IMG_UINT64							RGXFW_ALIGN ui64MultiSampleCtl;
+	IMG_UINT64							ui64FlippedMultiSampleCtl;
+	IMG_UINT32							ui32TPCStride;
+	IMG_UINT32							ui32TPCSize;
+	IMG_UINT32							ui32TEScreen;
+	IMG_UINT32							ui32MTileStride;
+	IMG_UINT32							ui32TEAA;
+	IMG_UINT32							ui32TEMTILE1;
+	IMG_UINT32							ui32TEMTILE2;
+	IMG_UINT32							ui32ISPMergeLowerX;
+	IMG_UINT32							ui32ISPMergeLowerY;
+	IMG_UINT32							ui32ISPMergeUpperX;
+	IMG_UINT32							ui32ISPMergeUpperY;
+	IMG_UINT32							ui32ISPMergeScaleX;
+	IMG_UINT32							ui32ISPMergeScaleY;
+	IMG_UINT32							uiRgnHeaderSize;
+	IMG_UINT32							ui32ISPMtileSize;
+} UNCACHED_ALIGN RGXFWIF_HWRTDATA_COMMON;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRTDATA_COMMON) == 88,
+				"RGXFWIF_HWRTDATA_COMMON is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+typedef struct
+{
+	IMG_UINT32           ui32RenderTargetIndex;		//Render number
+	IMG_UINT32           ui32CurrentRenderTarget;	//index in RTA
+	IMG_UINT32           ui32ActiveRenderTargets;	//total active RTs
+	IMG_UINT32           ui32CumulActiveRenderTargets;   //total active RTs from the first TA kick, for OOM
+	RGXFWIF_DEV_VIRTADDR sValidRenderTargets;  //Array of valid RT indices
+	RGXFWIF_DEV_VIRTADDR sRTANumPartialRenders;  //Array of number of occurred partial renders per render target
+	IMG_UINT32           ui32MaxRTs;   //Number of render targets in the array
+	IMG_UINT32           ui32RTACtlFlags; /* Compatibility and other flags */
+} UNCACHED_ALIGN RGXFWIF_RTA_CTL;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_RTA_CTL) == 32,
+				"RGXFWIF_RTA_CTL is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/*!
+ * @InGroup RenderTarget
+ * @Brief Firmware Render Target data i.e. HWRTDATA used to hold the PM context
+ */
+typedef struct
+{
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psPMMListDevVAddr;			/*!< MList Data Store */
+
+	IMG_UINT64			RGXFW_ALIGN ui64VCECatBase[1];			/*!< VCE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64VCELastCatBase[1];
+	IMG_UINT64			RGXFW_ALIGN ui64TECatBase[1];			/*!< TE Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64TELastCatBase[1];
+	IMG_UINT64			RGXFW_ALIGN ui64AlistCatBase;			/*!< Alist Page Catalogue base */
+	IMG_UINT64			RGXFW_ALIGN ui64AlistLastCatBase;
+
+	IMG_UINT64			RGXFW_ALIGN ui64PMAListStackPointer;		/*!< Freelist page table entry for current Mlist page */
+	IMG_UINT32			ui32PMMListStackPointer;			/*!< Current Mlist page */
+
+	RGXFWIF_DEV_VIRTADDR		sHWRTDataCommonFwAddr;				/*!< Render target dimension dependent data */
+
+	IMG_UINT32			ui32HWRTDataFlags;
+	RGXFWIF_RTDATA_STATE		eState;						/*!< Current workload processing state of HWRTDATA */
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[MAX_FREELISTS_SIZE];	/*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[MAX_FREELISTS_SIZE];
+#else
+	PRGXFWIF_FREELIST		RGXFW_ALIGN apsFreeLists[RGXFW_MAX_FREELISTS];	/*!< Freelist to use */
+	IMG_UINT32			aui32FreeListHWRSnapshot[RGXFW_MAX_FREELISTS];
+#endif
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN psVHeapTableDevVAddr;		/*!< VHeap table base */
+
+	RGXFWIF_RTA_CTL			sRTACtl;					/*!< Render target array data */
+
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sTailPtrsDevVAddr;			/*!< Tail pointers base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sMacrotileArrayDevVAddr;		/*!< Macrotiling array base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sRgnHeaderDevVAddr;			/*!< Region headers base */
+	IMG_DEV_VIRTADDR		RGXFW_ALIGN sRTCDevVAddr;			/*!< Render target cache base */
+#if defined(RGX_FIRMWARE)
+	struct RGXFWIF_FWCOMMONCONTEXT_* RGXFW_ALIGN psOwnerGeom;
+#else
+	RGXFWIF_DEV_VIRTADDR		RGXFW_ALIGN pui32OwnerGeomNotUsedByHost;
+#endif
+#if defined(SUPPORT_TRP) && !defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_UINT32			ui32KickFlagsCopy;
+	IMG_UINT32			ui32TEPageCopy;
+	IMG_UINT32			ui32VCEPageCopy;
+#endif
+#if defined(SUPPORT_AGP) || defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	IMG_BOOL			bTACachesNeedZeroing;
+#endif
+
+	RGXFWIF_CLEANUP_CTL		RGXFW_ALIGN_DCACHEL sCleanupState;					/*!< Render target clean up state */
+} RGXFW_ALIGN_DCACHEL RGXFWIF_HWRTDATA;
+
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+static_assert(sizeof(RGXFWIF_HWRTDATA) == 256,
+				"RGXFWIF_HWRTDATA is incorrect size for SUPPORT_OPEN_SOURCE_DRIVER");
+#endif
+
+/* Sync_checkpoint firmware object.
+ * This is the FW-addressable structure used to hold the sync checkpoint's
+ * state and other information which needs to be accessed by the firmware.
+ */
+typedef struct
+{
+	IMG_UINT32	ui32State;          /*!< Holds the current state of the sync checkpoint */
+	IMG_UINT32	ui32FwRefCount;     /*!< Holds the FW reference count (num of fences/updates processed) */
+} SYNC_CHECKPOINT_FW_OBJ;
+
+/* Bit mask Firmware can use to test if a checkpoint has signalled or errored */
+#define SYNC_CHECKPOINT_SIGNALLED_MASK (0x1 << 0)
+
+typedef enum
+{
+	RGXFWIF_REG_CFG_TYPE_PWR_ON=0,      /* Sidekick power event */
+	RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,   /* Rascal / dust power event */
+	RGXFWIF_REG_CFG_TYPE_TA,            /* TA kick */
+	RGXFWIF_REG_CFG_TYPE_3D,            /* 3D kick */
+	RGXFWIF_REG_CFG_TYPE_CDM,           /* Compute kick */
+	RGXFWIF_REG_CFG_TYPE_TLA,           /* TLA kick */
+	RGXFWIF_REG_CFG_TYPE_TDM,           /* TDM kick */
+	RGXFWIF_REG_CFG_TYPE_ALL            /* Applies to all types. Keep as last element */
+} RGXFWIF_REG_CFG_TYPE;
+
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_OFFSET_BYTES                 RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET
+#define RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES               (1U << 19)
+
+#define RGX_HEAP_KM_USC_RESERVED_SIZE                              RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES
+#define RGX_HEAP_USC_RESERVED_TOTAL_SIZE                           RGX_HEAP_UM_USC_RESERVED_SIZE + RGX_HEAP_KM_USC_RESERVED_SIZE
+
+#endif /*  RGX_FWIF_SHARED_H */
+
+/******************************************************************************
+ End of file (rgx_fwif_shared.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_heap_firmware.h b/drivers/gpu/drm/img-rogue/rgx_heap_firmware.h
new file mode 100644
index 000000000000..2628cd82e38e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_heap_firmware.h
@@ -0,0 +1,135 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX FW heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAP_FIRMWARE_H)
+#define RGX_HEAP_FIRMWARE_H
+
+/* Start at 903GiB. Size of 32MB per OSID (see rgxheapconfig.h)
+ * NOTE:
+ *      The firmware heaps bases and sizes are defined here to
+ *      simplify #include dependencies, see rgxheapconfig.h
+ *      for the full RGX virtual address space layout.
+ */
+
+/*
+ * The Config heap holds initialisation data shared between the
+ * the driver and firmware (e.g. pointers to the KCCB and FWCCB).
+ * The Main Firmware heap size is adjusted accordingly but most
+ * of the map / unmap functions must take into consideration
+ * the entire range (i.e. main and config heap).
+ */
+#define RGX_FIRMWARE_NUMBER_OF_FW_HEAPS              (IMG_UINT32_C(2))
+#define RGX_FIRMWARE_HEAP_SHIFT                      RGX_FW_HEAP_SHIFT
+#define RGX_FIRMWARE_RAW_HEAP_BASE                   (IMG_UINT64_C(0xE1C0000000))
+#define RGX_FIRMWARE_RAW_HEAP_SIZE                   (IMG_UINT32_C(1) << RGX_FIRMWARE_HEAP_SHIFT)
+
+/* To enable the firmware to compute the exact address of structures allocated by the KM
+ * in the Fw Config subheap, regardless of the KM's page size (and PMR granularity),
+ * objects allocated consecutively but from different PMRs (due to differing memalloc flags)
+ * are allocated with a 64kb offset. This way, all structures will be located at the same base
+ * addresses when the KM is running with a page size of 4k, 16k or 64k.  */
+#define RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY    (IMG_UINT32_C(0x10000))
+
+/* Ensure the heap can hold 3 PMRs of maximum supported granularity (192KB):
+ * 1st PMR: RGXFWIF_CONNECTION_CTL
+ * 2nd PMR: RGXFWIF_OSINIT
+ * 3rd PMR: RGXFWIF_SYSINIT */
+#define RGX_FIRMWARE_CONFIG_HEAP_SIZE                (IMG_UINT32_C(3)*RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY)
+
+#define RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE          (RGX_FIRMWARE_RAW_HEAP_SIZE - RGX_FIRMWARE_CONFIG_HEAP_SIZE)
+/*
+ * MIPS FW needs space in the Main heap to map GPU memory.
+ * This space is taken from the MAIN heap, to avoid creating a new heap.
+ */
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL       (IMG_UINT32_C(0x100000)) /* 1MB */
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101     (IMG_UINT32_C(0x400000)) /* 4MB */
+
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL      (RGX_FIRMWARE_RAW_HEAP_SIZE -  RGX_FIRMWARE_CONFIG_HEAP_SIZE - \
+                                                           RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL)
+
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101    (RGX_FIRMWARE_RAW_HEAP_SIZE -  RGX_FIRMWARE_CONFIG_HEAP_SIZE - \
+                                                           RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101)
+
+#if !defined(__KERNEL__)
+#if defined(FIX_HW_BRN_65101)
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE      RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE        RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101
+
+#include "img_defs.h"
+static_assert((RGX_FIRMWARE_RAW_HEAP_SIZE) >= IMG_UINT32_C(0x800000), "MIPS GPU map size cannot be increased due to BRN65101 with a small FW heap");
+
+#else
+#define RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE      RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL
+#define RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE        RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL
+#endif
+#endif /* !defined(__KERNEL__) */
+
+#define RGX_FIRMWARE_MAIN_HEAP_BASE             RGX_FIRMWARE_RAW_HEAP_BASE
+#define RGX_FIRMWARE_CONFIG_HEAP_BASE           (RGX_FIRMWARE_MAIN_HEAP_BASE + \
+                                                 RGX_FIRMWARE_RAW_HEAP_SIZE - \
+                                                 RGX_FIRMWARE_CONFIG_HEAP_SIZE)
+
+
+/* 1 Mb can hold the maximum amount of page tables for the memory shared between the firmware and all KM drivers:
+ *  MAX(RAW_HEAP_SIZE) = 32 Mb; MAX(NUMBER_OS) = 8; Total shared memory = 256 Mb;
+ *  MMU objects required: 65536 PTEs; 16 PDEs; 1 PCE; */
+#define RGX_FIRMWARE_MAX_PAGETABLE_SIZE (1 * 1024 * 1024)
+
+#define RGX_FW_CONFIG_HEAP_SIZE (1 << RGX_FW_HEAP_SHIFT)
+#define RGX_FW_MAX_HEAP_SIZE (1 << 28)
+/*
+ * The maximum configurable size via RGX_FW_HEAP_SHIFT is 256MiB (1<<28) and
+ * the minimum is 4MiB (1<<22); the firmware heap size is dependent
+ * on the number of drivers supported.
+ */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	#if defined(RGX_FW_HEAP_SHIFT) && (RGX_FW_HEAP_SHIFT < 22 || (RGX_FW_CONFIG_HEAP_SIZE > (RGX_FW_MAX_HEAP_SIZE / RGX_NUM_DRIVERS_SUPPORTED)))
+	#error "RGX_FW_HEAP_SHIFT is outside valid range"
+	#endif
+#else
+	#if defined(RGX_FW_HEAP_SHIFT) && (RGX_FW_HEAP_SHIFT < 22 || RGX_FW_HEAP_SHIFT > 28)
+	#error "RGX_FW_HEAP_SHIFT is outside valid range [22,28]"
+	#endif
+#endif
+
+
+#endif /* RGX_HEAP_FIRMWARE_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_heaps.h b/drivers/gpu/drm/img-rogue/rgx_heaps.h
new file mode 100644
index 000000000000..f8fcfe1fe27e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_heaps.h
@@ -0,0 +1,67 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_HEAPS_H)
+#define RGX_HEAPS_H
+
+/*
+  Identify heaps by their names
+*/
+#define RGX_GENERAL_SVM_HEAP_IDENT          "General SVM"               /*!< RGX General SVM (shared virtual memory) Heap Identifier */
+#define RGX_GENERAL_HEAP_IDENT              "General"                   /*!< RGX General Heap Identifier */
+#define RGX_GENERAL_NON4K_HEAP_IDENT        "General NON-4K"            /*!< RGX General non-4K Heap Identifier */
+#define RGX_PDSCODEDATA_HEAP_IDENT          "PDS Code and Data"         /*!< RGX PDS Code/Data Heap Identifier */
+#define RGX_USCCODE_HEAP_IDENT              "USC Code"                  /*!< RGX USC Code Heap Identifier */
+#define RGX_VK_CAPT_REPLAY_HEAP_IDENT       "Vulkan Capture Replay"     /*!< RGX Vulkan capture replay buffer Heap Identifier */
+#define RGX_FBCDC_HEAP_IDENT                "FBCDC"                     /*!< RGX FBCDC State Table Heap Identifier */
+#define RGX_FBCDC_LARGE_HEAP_IDENT          "Large FBCDC"               /*!< RGX Large FBCDC State Table Heap Identifier */
+#define RGX_CMP_MISSION_RMW_HEAP_IDENT      "Compute Mission RMW"       /*!< Compute Mission RMW Heap Identifier */
+#define RGX_CMP_SAFETY_RMW_HEAP_IDENT       "Compute Safety RMW"        /*!< Compute Safety RMW Heap Identifier */
+#define RGX_TEXTURE_STATE_HEAP_IDENT        "Texture State"             /*!< Texture State Heap Identifier */
+#define RGX_VISIBILITY_TEST_HEAP_IDENT      "Visibility Test"           /*!< Visibility Test Heap Identifier */
+
+/* Services client internal heap identification */
+#define RGX_RGNHDR_BRN_63142_HEAP_IDENT     "RgnHdr BRN63142"           /*!< RGX RgnHdr BRN63142 Heap Identifier */
+#define RGX_TQ3DPARAMETERS_HEAP_IDENT       "TQ3DParameters"            /*!< RGX TQ 3D Parameters Heap Identifier */
+#define RGX_MMU_INIA_BRN_65273_HEAP_IDENT   "MMU INIA BRN65273"         /*!< MMU BRN65273 Heap A Identifier */
+#define RGX_MMU_INIB_BRN_65273_HEAP_IDENT   "MMU INIB BRN65273"         /*!< MMU BRN65273 Heap B Identifier */
+#endif /* RGX_HEAPS_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_heaps_server.h b/drivers/gpu/drm/img-rogue/rgx_heaps_server.h
new file mode 100644
index 000000000000..a1dd31113fa3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_heaps_server.h
@@ -0,0 +1,86 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX heap (server) definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#if !defined RGX_HEAPS_SERVER_H
+#define RGX_HEAPS_SERVER_H
+
+#include "img_types.h"
+#include "rgx_heaps.h"
+
+/*
+ *  Supported log2 page size values for RGX_GENERAL_NON_4K_HEAP_ID
+ */
+#define RGX_HEAP_PAGE_SHIFTS_DEF \
+	X(4KB, 12U) \
+	X(16KB, 14U) \
+	X(64KB, 16U) \
+	X(256KB, 18U) \
+	X(1MB, 20U) \
+	X(2MB, 21U)
+
+typedef enum RGX_HEAP_PAGE_SHIFTS_TAG
+{
+#define X(_name, _shift) RGX_HEAP_ ## _name ## _PAGE_SHIFT = _shift,
+	RGX_HEAP_PAGE_SHIFTS_DEF
+#undef X
+} RGX_HEAP_PAGE_SHIFTS;
+
+/* Base and size alignment 2MB */
+#define RGX_HEAP_BASE_SIZE_ALIGN 0x200000UL
+#define RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT 0x8000UL
+
+/*************************************************************************/ /*!
+@Function       RGXGetValidHeapPageSizeMask
+@Description    Returns a bitmask indicating all supported virtual heap page sizes.
+
+@Return         IMG_UINT32  A 32-bit mask with enabled bits indicating valid
+                            page sizes.
+*/ /**************************************************************************/
+static inline IMG_UINT32 RGXGetValidHeapPageSizeMask(void)
+{
+	/* Generates a bit mask with the values of RGX_HEAP_PAGE_SHIFTS_DEF.
+	 * 0 is required for the first shift to properly bitwise OR. */
+#define X(_name, _shift) | (1 << _shift)
+	return 0 RGX_HEAP_PAGE_SHIFTS_DEF;
+#undef X
+}
+
+#endif /* RGX_HEAPS_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_hwperf.h b/drivers/gpu/drm/img-rogue/rgx_hwperf.h
new file mode 100644
index 000000000000..d164c20a2417
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_hwperf.h
@@ -0,0 +1,483 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_H_
+#define RGX_HWPERF_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_hwperf_common.h"
+
+#if !defined(__KERNEL__)
+/* User-mode and Firmware definitions only */
+
+#if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER)
+
+/* HWPerf interface assumption checks */
+static_assert(RGX_FEATURE_NUM_CLUSTERS <= 16U, "Cluster count too large for HWPerf protocol definition");
+
+/*! The number of indirectly addressable TPU_MSC blocks in the GPU */
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST MAX(((IMG_UINT32)RGX_FEATURE_NUM_CLUSTERS >> 1), 1U)
+
+/*! The number of indirectly addressable USC blocks in the GPU */
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER (RGX_FEATURE_NUM_CLUSTERS)
+
+# if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+
+ /*! Defines the number of performance counter blocks that are directly
+  * addressable in the RGX register map for S. */
+#  define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      1 /* JONES */
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       (RGX_NUM_PHANTOMS)
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      1 /* BLACKPEARL */
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         2 /* TPU, TEXAS */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 2 /* USC, PBE */
+
+# elif defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+
+  /*! Defines the number of performance counter blocks that are directly
+   * addressable in the RGX register map. */
+#   define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS    2 /* TORNADO, TA */
+
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       (RGX_NUM_PHANTOMS)
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      2 /* RASTER, TEXAS */
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         1 /* TPU */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 1 /* USC */
+
+# else /* !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && !defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) i.e. S6 */
+
+ /*! Defines the number of performance counter blocks that are
+  * addressable in the RGX register map for Series 6. */
+#  define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      3 /* TA, RASTER, HUB */
+#  define RGX_HWPERF_INDIRECT_BY_PHANTOM       0 /* PHANTOM is not there in Rogue1. Just using it to keep naming same as later series (RogueXT n Rogue XT+) */
+#  define RGX_HWPERF_PHANTOM_NONDUST_BLKS      0
+#  define RGX_HWPERF_PHANTOM_DUST_BLKS         1 /* TPU */
+#  define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS 1 /* USC */
+
+# endif
+
+/*! The number of performance counters in each layout block defined for UM/FW code */
+#if defined(RGX_FEATURE_CLUSTER_GROUPING)
+  #define RGX_HWPERF_CNTRS_IN_BLK 6
+ #else
+  #define RGX_HWPERF_CNTRS_IN_BLK 4
+#endif
+
+#endif /* #if defined(RGX_BVNC_CORE_KM_HEADER) && defined(RGX_BNC_CONFIG_KM_HEADER) */
+#else /* defined(__KERNEL__) */
+/* Kernel/server definitions - not used, hence invalid definitions */
+
+# define RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC 0xFF
+
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST    RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+
+# define RGX_HWPERF_MAX_DIRECT_ADDR_BLKS      RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_INDIRECT_BY_PHANTOM       RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_NONDUST_BLKS      RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_DUST_BLKS         RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+# define RGX_HWPERF_PHANTOM_DUST_CLUSTER_BLKS RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+
+#endif
+
+/*! The number of custom non-mux counter blocks supported */
+#define RGX_HWPERF_MAX_CUSTOM_BLKS 5U
+
+/*! The number of counters supported in each non-mux counter block */
+#define RGX_HWPERF_MAX_CUSTOM_CNTRS 8U
+
+
+/******************************************************************************
+ * Data Stream Common Types
+ *****************************************************************************/
+
+/*! All the Data Masters HWPerf is aware of. When a new DM is added to this
+ * list, it should be appended at the end to maintain backward compatibility
+ * of HWPerf data.
+ */
+#define	RGX_HWPERF_DM_GP                  0x00000000U
+#define	RGX_HWPERF_DM_2D                  0x00000001U
+#define	RGX_HWPERF_DM_TA                  0x00000002U
+#define	RGX_HWPERF_DM_3D                  0x00000003U
+#define	RGX_HWPERF_DM_CDM                 0x00000004U
+#define	RGX_HWPERF_DM_RTU                 0x00000005U
+#define	RGX_HWPERF_DM_SHG                 0x00000006U
+#define	RGX_HWPERF_DM_TDM                 0x00000007U
+
+#define	RGX_HWPERF_DM_LAST                0x00000008U
+
+#define	RGX_HWPERF_DM_INVALID             0x1FFFFFFFU
+
+/******************************************************************************
+ * API Types
+ *****************************************************************************/
+
+/*! Counter block IDs for all the hardware blocks with counters.
+ * Directly addressable blocks must have a value between 0..15 [0..0xF].
+ * Indirect groups have following encoding:
+ * First hex digit (LSB) represents a unit number within the group
+ * and the second hex digit represents the group number.
+ * Group 0 is the direct group, all others are indirect groups.
+ */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_ID;
+
+/*! Directly addressable counter blocks */
+#if defined(DOXYGEN)
+/*! _RGX_HWPERF_CNTBLK_ID */
+#endif
+#define	RGX_CNTBLK_ID_TA			 0x0000U
+#define	RGX_CNTBLK_ID_RASTER		 0x0001U /*!< Non-cluster grouping cores */
+#define	RGX_CNTBLK_ID_HUB			 0x0002U /*!< Non-cluster grouping cores */
+#define	RGX_CNTBLK_ID_TORNADO		 0x0003U /*!< XT cores */
+#define	RGX_CNTBLK_ID_JONES			 0x0004U /*!< S7 cores */
+#define	RGX_CNTBLK_ID_DIRECT_LAST	 0x0005U /*!< Indirect blocks start from here */
+#define	RGX_CNTBLK_ID_BF_DEPRECATED	 0x0005U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_BT_DEPRECATED	 0x0006U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_RT_DEPRECATED	 0x0007U /*!< Doppler unit (DEPRECATED) */
+#define	RGX_CNTBLK_ID_SH_DEPRECATED	 0x0008U /*!< Ray tracing unit (DEPRECATED) */
+
+
+/*! Indirectly addressable counter blocks. DA blocks indicate counter blocks
+ *  where the counter registers are directly accessible
+ */
+#define	RGX_CNTBLK_ID_TPU_MCU0		 0x0010U /*!< Addressable by Dust */
+#define RGX_CNTBLK_ID_TPU_MCU0_DA	 0x8010U
+#define	RGX_CNTBLK_ID_TPU_MCU1		 0x0011U
+#define RGX_CNTBLK_ID_TPU_MCU1_DA	 0x8011U
+#define	RGX_CNTBLK_ID_TPU_MCU2		 0x0012U
+#define RGX_CNTBLK_ID_TPU_MCU2_DA	 0x8012U
+#define	RGX_CNTBLK_ID_TPU_MCU3		 0x0013U
+#define RGX_CNTBLK_ID_TPU_MCU3_DA	 0x8013U
+#define	RGX_CNTBLK_ID_TPU_MCU4		 0x0014U
+#define RGX_CNTBLK_ID_TPU_MCU4_DA	 0x8014U
+#define	RGX_CNTBLK_ID_TPU_MCU5		 0x0015U
+#define RGX_CNTBLK_ID_TPU_MCU5_DA	 0x8015U
+#define	RGX_CNTBLK_ID_TPU_MCU6		 0x0016U
+#define RGX_CNTBLK_ID_TPU_MCU6_DA	 0x8016U
+#define	RGX_CNTBLK_ID_TPU_MCU7		 0x0017U
+#define RGX_CNTBLK_ID_TPU_MCU7_DA	 0x8017U
+#define	RGX_CNTBLK_ID_TPU_MCU_ALL	 0x4010U
+#define	RGX_CNTBLK_ID_TPU_MCU_ALL_DA	 0xC010U
+
+#define	RGX_CNTBLK_ID_USC0			 0x0020U /*!< Addressable by Cluster */
+#define RGX_CNTBLK_ID_USC0_DA		 0x8020U
+#define	RGX_CNTBLK_ID_USC1			 0x0021U
+#define RGX_CNTBLK_ID_USC1_DA		 0x8021U
+#define	RGX_CNTBLK_ID_USC2			 0x0022U
+#define RGX_CNTBLK_ID_USC2_DA		 0x8022U
+#define	RGX_CNTBLK_ID_USC3			 0x0023U
+#define RGX_CNTBLK_ID_USC3_DA		 0x8023U
+#define	RGX_CNTBLK_ID_USC4			 0x0024U
+#define RGX_CNTBLK_ID_USC4_DA		 0x8024U
+#define	RGX_CNTBLK_ID_USC5			 0x0025U
+#define RGX_CNTBLK_ID_USC5_DA		 0x8025U
+#define	RGX_CNTBLK_ID_USC6			 0x0026U
+#define RGX_CNTBLK_ID_USC6_DA		 0x8026U
+#define	RGX_CNTBLK_ID_USC7			 0x0027U
+#define RGX_CNTBLK_ID_USC7_DA		 0x8027U
+#define	RGX_CNTBLK_ID_USC8			 0x0028U
+#define RGX_CNTBLK_ID_USC8_DA		 0x8028U
+#define	RGX_CNTBLK_ID_USC9			 0x0029U
+#define RGX_CNTBLK_ID_USC9_DA		 0x8029U
+#define	RGX_CNTBLK_ID_USC10			 0x002AU
+#define RGX_CNTBLK_ID_USC10_DA		 0x802AU
+#define	RGX_CNTBLK_ID_USC11			 0x002BU
+#define RGX_CNTBLK_ID_USC11_DA		 0x802BU
+#define	RGX_CNTBLK_ID_USC12			 0x002CU
+#define RGX_CNTBLK_ID_USC12_DA		 0x802CU
+#define	RGX_CNTBLK_ID_USC13			 0x002DU
+#define RGX_CNTBLK_ID_USC13_DA		 0x802DU
+#define	RGX_CNTBLK_ID_USC14			 0x002EU
+#define RGX_CNTBLK_ID_USC14_DA		 0x802EU
+#define	RGX_CNTBLK_ID_USC15			 0x002FU
+#define RGX_CNTBLK_ID_USC15_DA		 0x802FU
+#define	RGX_CNTBLK_ID_USC_ALL		 0x4020U
+#define	RGX_CNTBLK_ID_USC_ALL_DA	 0xC020U
+
+#define	RGX_CNTBLK_ID_TEXAS0		 0x0030U /*!< Addressable by Phantom in XT, Dust in S7 */
+#define	RGX_CNTBLK_ID_TEXAS1		 0x0031U
+#define	RGX_CNTBLK_ID_TEXAS2		 0x0032U
+#define	RGX_CNTBLK_ID_TEXAS3		 0x0033U
+#define	RGX_CNTBLK_ID_TEXAS4		 0x0034U
+#define	RGX_CNTBLK_ID_TEXAS5		 0x0035U
+#define	RGX_CNTBLK_ID_TEXAS6		 0x0036U
+#define	RGX_CNTBLK_ID_TEXAS7		 0x0037U
+#define	RGX_CNTBLK_ID_TEXAS_ALL		 0x4030U
+
+#define	RGX_CNTBLK_ID_RASTER0		 0x0040U /*!< Addressable by Phantom, XT only */
+#define	RGX_CNTBLK_ID_RASTER1		 0x0041U
+#define	RGX_CNTBLK_ID_RASTER2		 0x0042U
+#define	RGX_CNTBLK_ID_RASTER3		 0x0043U
+#define	RGX_CNTBLK_ID_RASTER_ALL	 0x4040U
+
+#define	RGX_CNTBLK_ID_BLACKPEARL0	 0x0050U /*!< Addressable by Phantom, S7, only */
+#define	RGX_CNTBLK_ID_BLACKPEARL1	 0x0051U
+#define	RGX_CNTBLK_ID_BLACKPEARL2	 0x0052U
+#define	RGX_CNTBLK_ID_BLACKPEARL3	 0x0053U
+#define	RGX_CNTBLK_ID_BLACKPEARL_ALL 0x4050U
+
+#define	RGX_CNTBLK_ID_PBE0			 0x0060U /*!< Addressable by Cluster in S7 and PBE2_IN_XE */
+#define	RGX_CNTBLK_ID_PBE1			 0x0061U
+#define	RGX_CNTBLK_ID_PBE2			 0x0062U
+#define	RGX_CNTBLK_ID_PBE3			 0x0063U
+#define	RGX_CNTBLK_ID_PBE4			 0x0064U
+#define	RGX_CNTBLK_ID_PBE5			 0x0065U
+#define	RGX_CNTBLK_ID_PBE6			 0x0066U
+#define	RGX_CNTBLK_ID_PBE7			 0x0067U
+#define	RGX_CNTBLK_ID_PBE8			 0x0068U
+#define	RGX_CNTBLK_ID_PBE9			 0x0069U
+#define	RGX_CNTBLK_ID_PBE10			 0x006AU
+#define	RGX_CNTBLK_ID_PBE11			 0x006BU
+#define	RGX_CNTBLK_ID_PBE12			 0x006CU
+#define	RGX_CNTBLK_ID_PBE13			 0x006DU
+#define	RGX_CNTBLK_ID_PBE14			 0x006EU
+#define	RGX_CNTBLK_ID_PBE15			 0x006FU
+#define	RGX_CNTBLK_ID_PBE_ALL		 0x4060U
+
+#define	RGX_CNTBLK_ID_LAST			 0x0070U /*!< End of PBE block */
+
+#define	RGX_CNTBLK_ID_BX_TU0_DEPRECATED		 0x0070U /*!< Doppler unit, DEPRECATED */
+#define	RGX_CNTBLK_ID_BX_TU1_DEPRECATED		 0x0071U
+#define	RGX_CNTBLK_ID_BX_TU2_DEPRECATED		 0x0072U
+#define	RGX_CNTBLK_ID_BX_TU3_DEPRECATED		 0x0073U
+#define	RGX_CNTBLK_ID_BX_TU_ALL_DEPRECATED	 0x4070U
+
+#define	RGX_CNTBLK_ID_CUSTOM0		 0x70F0U
+#define	RGX_CNTBLK_ID_CUSTOM1		 0x70F1U
+#define	RGX_CNTBLK_ID_CUSTOM2		 0x70F2U
+#define	RGX_CNTBLK_ID_CUSTOM3		 0x70F3U
+#define	RGX_CNTBLK_ID_CUSTOM4_FW	 0x70F4U /*!< Custom block used for getting statistics held in the FW */
+#define	RGX_CNTBLK_ID_CUSTOM_MASK	 0x70FFU
+
+
+/* Masks for the counter block ID*/
+#define	RGX_CNTBLK_ID_UNIT_MASK      (0x000FU)
+#define	RGX_CNTBLK_ID_GROUP_MASK     (0x00F0U)
+#define	RGX_CNTBLK_ID_GROUP_SHIFT    (4U)
+#define	RGX_CNTBLK_ID_MC_GPU_MASK    (0x0F00U)
+#define	RGX_CNTBLK_ID_MC_GPU_SHIFT   (8U)
+#define	RGX_CNTBLK_ID_UNIT_ALL_MASK  (0x4000U)
+#define	RGX_CNTBLK_ID_DA_MASK        (0x8000U) /*!< Block with directly accessible counter registers */
+
+#define RGX_CNTBLK_INDIRECT_COUNT(_class, _n) ((IMG_UINT32)(RGX_CNTBLK_ID_ ## _class ## _n) - (IMG_UINT32)(RGX_CNTBLK_ID_ ## _class ## 0) + 1u)
+
+/*! The number of layout blocks defined with configurable multiplexed
+ * performance counters, hence excludes custom counter blocks.
+ */
+#define RGX_HWPERF_MAX_DEFINED_BLKS  (\
+	(IMG_UINT32)RGX_CNTBLK_ID_DIRECT_LAST    +\
+	RGX_CNTBLK_INDIRECT_COUNT(TPU_MCU,     7)+\
+	RGX_CNTBLK_INDIRECT_COUNT(USC,        15)+\
+	RGX_CNTBLK_INDIRECT_COUNT(TEXAS,       7)+\
+	RGX_CNTBLK_INDIRECT_COUNT(RASTER,      3)+\
+	RGX_CNTBLK_INDIRECT_COUNT(BLACKPEARL,  3)+\
+	RGX_CNTBLK_INDIRECT_COUNT(PBE,        15) )
+#define RGX_HWPERF_MAX_MUX_BLKS      (\
+    RGX_HWPERF_MAX_DEFINED_BLKS      )
+
+static_assert(
+	((RGX_CNTBLK_ID_DIRECT_LAST + ((RGX_CNTBLK_ID_LAST & RGX_CNTBLK_ID_GROUP_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT)) <= RGX_HWPERF_MAX_BVNC_BLOCK_LEN),
+	"RGX_HWPERF_MAX_BVNC_BLOCK_LEN insufficient");
+
+#define RGX_HWPERF_EVENT_MASK_VALUE(e)      (IMG_UINT64_C(1) << (IMG_UINT32)(e))
+
+#define RGX_CUSTOM_FW_CNTRS	\
+                X(TA_LOCAL_FL_SIZE,    0x0, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(TA_GLOBAL_FL_SIZE,   0x1, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TAPAUSE) |  \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PMOOM_TARESUME) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_TAFINISHED))      \
+                                                                                                        \
+                X(3D_LOCAL_FL_SIZE,    0x2, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(3D_GLOBAL_FL_SIZE,   0x3, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DFINISHED))      \
+                                                                                                        \
+                X(ISP_TILES_IN_FLIGHT, 0x4, RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DKICK) |         \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_3DSPMKICK))
+
+/*! Counter IDs for the firmware held statistics */
+typedef enum
+{
+#define X(ctr, id, allow_mask)	RGX_CUSTOM_FW_CNTR_##ctr = id,
+	RGX_CUSTOM_FW_CNTRS
+#undef X
+
+	/* always the last entry in the list */
+	RGX_CUSTOM_FW_CNTR_LAST
+} RGX_HWPERF_CUSTOM_FW_CNTR_ID;
+
+/*! Identifier for each counter in a performance counting module */
+typedef IMG_UINT32 RGX_HWPERF_CNTBLK_COUNTER_ID;
+
+#define	RGX_CNTBLK_COUNTER0_ID 0U
+#define	RGX_CNTBLK_COUNTER1_ID 1U
+#define	RGX_CNTBLK_COUNTER2_ID 2U
+#define	RGX_CNTBLK_COUNTER3_ID 3U
+#define	RGX_CNTBLK_COUNTER4_ID 4U
+#define	RGX_CNTBLK_COUNTER5_ID 5U
+	/* MAX value used in server handling of counter config arrays */
+#define	RGX_CNTBLK_MUX_COUNTERS_MAX 6U
+
+
+/* sets all the bits from bit _b1 to _b2, in a IMG_UINT64 type */
+#define MASK_RANGE_IMPL(b1, b2)	((IMG_UINT64)((IMG_UINT64_C(1) << ((IMG_UINT32)(b2)-(IMG_UINT32)(b1) + 1U)) - 1U) << (IMG_UINT32)(b1))
+#define MASK_RANGE(R)			MASK_RANGE_IMPL(R##_FIRST_TYPE, R##_LAST_TYPE)
+#define RGX_HWPERF_HOST_EVENT_MASK_VALUE(e) (IMG_UINT32_C(1) << (e))
+
+/*! Mask macros for use with RGXCtrlHWPerf() API.
+ */
+#define RGX_HWPERF_EVENT_MASK_NONE          (IMG_UINT64_C(0x0000000000000000))
+#define RGX_HWPERF_EVENT_MASK_DEFAULT       RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_FWACT) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG) | \
+                                            RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG)
+#define RGX_HWPERF_EVENT_MASK_ALL           (IMG_UINT64_C(0xFFFFFFFFFFFFFFFF))
+
+/*! HWPerf Firmware event masks
+ * @par
+ * All FW Start/End/Debug (SED) events. */
+#define RGX_HWPERF_EVENT_MASK_FW_SED    (MASK_RANGE(RGX_HWPERF_FW_EVENT_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_FW_UFO    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_UFO))
+#define RGX_HWPERF_EVENT_MASK_FW_CSW    (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_START) |\
+                                          RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CSW_FINISHED))
+/*! All FW events. */
+#define RGX_HWPERF_EVENT_MASK_ALL_FW    (RGX_HWPERF_EVENT_MASK_FW_SED |\
+                                          RGX_HWPERF_EVENT_MASK_FW_UFO |\
+                                          RGX_HWPERF_EVENT_MASK_FW_CSW)
+
+/*! HW Periodic events (1ms interval). */
+#define RGX_HWPERF_EVENT_MASK_HW_PERIODIC   (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HW_PERIODIC))
+/*! All HW Kick/Finish events. */
+#define RGX_HWPERF_EVENT_MASK_HW_KICKFINISH ((MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE0) |\
+                                               MASK_RANGE(RGX_HWPERF_HW_EVENT_RANGE1)) &\
+                                              ~(RGX_HWPERF_EVENT_MASK_HW_PERIODIC))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_HW        (RGX_HWPERF_EVENT_MASK_HW_KICKFINISH |\
+                                              RGX_HWPERF_EVENT_MASK_HW_PERIODIC)
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR_EST   (MASK_RANGE(RGX_HWPERF_PWR_EST_RANGE))
+
+#define RGX_HWPERF_EVENT_MASK_ALL_PWR       (RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_CLKS_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_GPU_STATE_CHG) |\
+                                              RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_PWR_CHG))
+
+/*! HWPerf Host event masks
+ */
+#define RGX_HWPERF_EVENT_MASK_HOST_WORK_ENQ  (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_ENQ))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_UFO   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_UFO))
+#define RGX_HWPERF_EVENT_MASK_HOST_ALL_PWR   (RGX_HWPERF_HOST_EVENT_MASK_VALUE(RGX_HWPERF_HOST_CLK_SYNC))
+
+
+/*! Type used in the RGX API RGXConfigMuxHWPerfCounters() */
+typedef struct
+{
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! 4 or 6 LSBs used to select counters to configure in this block. */
+	IMG_UINT8  ui8CounterSelect;
+
+	/*! 4 or 6 LSBs used as MODE bits for the counters in the group. */
+	IMG_UINT8  ui8Mode;
+
+	/*! 5 or 6 LSBs used as the GROUP_SELECT value for the counter. */
+	IMG_UINT8  aui8GroupSelect[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 16 LSBs used as the BIT_SELECT value for the counter. */
+	IMG_UINT16 aui16BitSelect[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 14 LSBs used as the BATCH_MAX value for the counter. */
+	IMG_UINT32 aui32BatchMax[RGX_CNTBLK_MUX_COUNTERS_MAX];
+
+	/*! 14 LSBs used as the BATCH_MIN value for the counter. */
+	IMG_UINT32 aui32BatchMin[RGX_CNTBLK_MUX_COUNTERS_MAX];
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_MUX_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+
+/*! Type used in the RGX API RGXConfigHWPerfCounters() */
+typedef struct
+{
+	/*! Reserved for future use */
+	IMG_UINT32 ui32Reserved;
+
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of configured counters within this block */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Counter register values */
+	IMG_UINT16 ui16Counters[RGX_CNTBLK_COUNTERS_MAX];
+} UNCACHED_ALIGN RGX_HWPERF_CONFIG_CNTBLK;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CONFIG_CNTBLK);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_hwperf_common.h b/drivers/gpu/drm/img-rogue/rgx_hwperf_common.h
new file mode 100644
index 000000000000..23576c7832ac
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_hwperf_common.h
@@ -0,0 +1,1641 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HWPerf and Debug Types and Defines Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common data types definitions for hardware performance API
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGX_HWPERF_COMMON_H_
+#define RGX_HWPERF_COMMON_H_
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* These structures are used on both GPU and CPU and must be a size that is a
+ * multiple of 64 bits, 8 bytes to allow the FW to write 8 byte quantities at
+ * 8 byte aligned addresses. RGX_FW_STRUCT_*_ASSERT() is used to check this.
+ */
+
+/******************************************************************************
+ * Includes and Defines
+ *****************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+
+#include "rgx_common.h"
+#include "rgx_common_asserts.h"
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_sync_km.h"
+
+
+/******************************************************************************
+ * Packet Event Type Enumerations
+ *****************************************************************************/
+
+/*! Type used to encode the event that generated the packet.
+ * NOTE: When this type is updated the corresponding hwperfbin2json tool
+ * source needs to be updated as well. The RGX_HWPERF_EVENT_MASK_* macros will
+ * also need updating when adding new types.
+ *
+ * @par
+ * The event type values are incrementing integers for use as a shift ordinal
+ * in the event filtering process at the point events are generated.
+ * This scheme thus implies a limit of 63 event types.
+ */
+
+typedef IMG_UINT32 RGX_HWPERF_EVENT_TYPE;
+
+#define RGX_HWPERF_INVALID				0x00U /*!< Invalid. Reserved value. */
+
+/*! FW types 0x01..0x06 */
+#define RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE	0x01U
+
+#define RGX_HWPERF_FW_BGSTART			0x01U /*!< Background task processing start */
+#define RGX_HWPERF_FW_BGEND				0x02U /*!< Background task end */
+#define RGX_HWPERF_FW_IRQSTART			0x03U /*!< IRQ task processing start */
+
+#define RGX_HWPERF_FW_IRQEND			0x04U /*!< IRQ task end */
+#define RGX_HWPERF_FW_DBGSTART			0x05U /*!< Debug event start */
+#define RGX_HWPERF_FW_DBGEND			0x06U /*!< Debug event end */
+
+#define RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE		0x06U
+
+/*! HW types 0x07..0x19 */
+#define RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE	0x07U
+
+#define RGX_HWPERF_HW_PMOOM_TAPAUSE		0x07U /*!< TA Pause at PM Out of Memory */
+
+#define RGX_HWPERF_HW_TAKICK			0x08U /*!< TA task started */
+#define RGX_HWPERF_HW_TAFINISHED		0x09U /*!< TA task finished */
+#define RGX_HWPERF_HW_3DTQKICK			0x0AU /*!< 3D TQ started */
+#define RGX_HWPERF_HW_3DKICK			0x0BU /*!< 3D task started */
+#define RGX_HWPERF_HW_3DFINISHED		0x0CU /*!< 3D task finished */
+#define RGX_HWPERF_HW_CDMKICK			0x0DU /*!< CDM task started */
+#define RGX_HWPERF_HW_CDMFINISHED		0x0EU /*!< CDM task finished */
+#define RGX_HWPERF_HW_TLAKICK			0x0FU /*!< TLA task started */
+#define RGX_HWPERF_HW_TLAFINISHED		0x10U /*!< TLS task finished */
+#define RGX_HWPERF_HW_3DSPMKICK			0x11U /*!< 3D SPM task started */
+#define RGX_HWPERF_HW_PERIODIC			0x12U /*!< Periodic event with updated HW counters */
+#define RGX_HWPERF_HW_RTUKICK			0x13U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_RTUFINISHED		0x14U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_SHGKICK			0x15U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_SHGFINISHED		0x16U /*!< Reserved, future use */
+#define RGX_HWPERF_HW_3DTQFINISHED		0x17U /*!< 3D TQ finished */
+#define RGX_HWPERF_HW_3DSPMFINISHED		0x18U /*!< 3D SPM task finished */
+
+#define RGX_HWPERF_HW_PMOOM_TARESUME	0x19U /*!< TA Resume after PM Out of Memory */
+
+/*! HW_EVENT_RANGE0 used up. Use next empty range below to add new hardware events */
+#define RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE	0x19U
+
+/*! other types 0x1A..0x1F */
+#define RGX_HWPERF_CLKS_CHG				0x1AU /*!< Clock speed change in GPU */
+#define RGX_HWPERF_GPU_STATE_CHG		0x1BU /*!< GPU work state change */
+
+/*! power types 0x20..0x27 */
+#define RGX_HWPERF_PWR_EST_RANGE_FIRST_TYPE	0x20U
+#define RGX_HWPERF_PWR_EST_REQUEST		0x20U /*!< Power estimate requested (via GPIO) */
+#define RGX_HWPERF_PWR_EST_READY		0x21U /*!< Power estimate inputs ready */
+#define RGX_HWPERF_PWR_EST_RESULT		0x22U /*!< Power estimate result calculated */
+#define RGX_HWPERF_PWR_EST_RANGE_LAST_TYPE	0x22U
+
+#define RGX_HWPERF_PWR_CHG				0x23U /*!< Power state change */
+
+/*! HW_EVENT_RANGE1 0x28..0x2F, for accommodating new hardware events */
+#define RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE	0x28U
+
+#define RGX_HWPERF_HW_TDMKICK			0x28U /*!< TDM task started */
+#define RGX_HWPERF_HW_TDMFINISHED		0x29U /*!< TDM task finished */
+#define RGX_HWPERF_HW_NULLKICK			0x2AU /*!< NULL event */
+
+#define RGX_HWPERF_HW_EVENT_RANGE1_LAST_TYPE 0x2AU
+#define RGX_HWPERF_TRACE_EVENT_GPU_WORK_PERIOD		0x2BU
+
+/*! context switch types 0x30..0x31 */
+#define RGX_HWPERF_CSW_START			0x30U /*!< HW context store started */
+#define RGX_HWPERF_CSW_FINISHED			0x31U /*!< HW context store finished */
+
+/*! DVFS events */
+#define RGX_HWPERF_DVFS					0x32U /*!< Dynamic voltage/frequency scaling events */
+
+/*! firmware misc 0x38..0x39 */
+#define RGX_HWPERF_UFO					0x38U /*!< FW UFO Check / Update */
+#define RGX_HWPERF_FWACT				0x39U /*!< FW Activity notification */
+
+/*! last */
+#define RGX_HWPERF_LAST_TYPE			0x3BU
+
+/*! This enumeration must have a value that is a power of two as it is
+ * used in masks and a filter bit field (currently 64 bits long).
+ */
+#define RGX_HWPERF_MAX_TYPE				0x40U
+
+static_assert(RGX_HWPERF_LAST_TYPE < RGX_HWPERF_MAX_TYPE, "Too many HWPerf event types");
+
+/*! Macro used to check if an event type ID is present in the known set of hardware type events */
+#define HWPERF_PACKET_IS_HW_TYPE(_etype)	(((_etype) >= RGX_HWPERF_HW_EVENT_RANGE0_FIRST_TYPE && (_etype) <= RGX_HWPERF_HW_EVENT_RANGE0_LAST_TYPE) || \
+											 ((_etype) >= RGX_HWPERF_HW_EVENT_RANGE1_FIRST_TYPE && (_etype) <= RGX_HWPERF_HW_EVENT_RANGE1_LAST_TYPE))
+
+/*! Macro used to check if an event type ID is present in the known set of firmware type events */
+#define HWPERF_PACKET_IS_FW_TYPE(_etype)					\
+	((_etype) >= RGX_HWPERF_FW_EVENT_RANGE_FIRST_TYPE &&	\
+	 (_etype) <= RGX_HWPERF_FW_EVENT_RANGE_LAST_TYPE)
+
+
+typedef enum {
+	RGX_HWPERF_HOST_INVALID   = 0x00,           /*!< Invalid, do not use. */
+	RGX_HWPERF_HOST_ENQ       = 0x01,           /*!< ``0x01`` Kernel driver has queued GPU work.
+	                                             See RGX_HWPERF_HOST_ENQ_DATA */
+	RGX_HWPERF_HOST_UFO       = 0x02,           /*!< ``0x02`` UFO updated by the driver.
+	                                             See RGX_HWPERF_HOST_UFO_DATA */
+	RGX_HWPERF_HOST_ALLOC     = 0x03,           /*!< ``0x03`` Resource allocated.
+	                                             See RGX_HWPERF_HOST_ALLOC_DATA */
+	RGX_HWPERF_HOST_CLK_SYNC  = 0x04,           /*!< ``0x04`` GPU / Host clocks correlation data.
+	                                             See RGX_HWPERF_HOST_CLK_SYNC_DATA */
+	RGX_HWPERF_HOST_FREE      = 0x05,           /*!< ``0x05`` Resource freed,
+	                                             See RGX_HWPERF_HOST_FREE_DATA */
+	RGX_HWPERF_HOST_MODIFY    = 0x06,           /*!< ``0x06`` Resource modified / updated.
+	                                             See RGX_HWPERF_HOST_MODIFY_DATA */
+	RGX_HWPERF_HOST_DEV_INFO  = 0x07,           /*!< ``0x07`` Device Health status.
+	                                             See RGX_HWPERF_HOST_DEV_INFO_DATA */
+	RGX_HWPERF_HOST_INFO      = 0x08,           /*!< ``0x08`` Device memory usage information.
+	                                             See RGX_HWPERF_HOST_INFO_DATA */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT = 0x09,     /*!< ``0x09`` Wait for sync event.
+	                                             See RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA */
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE  = 0x0A, /*!< ``0x0A`` Software timeline advanced.
+	                                             See RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA */
+	RGX_HWPERF_HOST_CLIENT_INFO = 0x0B,			/*!< ``0x0B`` Additional client info.
+	                                             See RGX_HWPERF_HOST_CLIENT_INFO_DATA */
+
+	/*! last */
+	RGX_HWPERF_HOST_LAST_TYPE,
+
+	/*! This enumeration must have a value that is a power of two as it is
+	 * used in masks and a filter bit field (currently 32 bits long).
+	 */
+	RGX_HWPERF_HOST_MAX_TYPE  = 0x20
+} RGX_HWPERF_HOST_EVENT_TYPE;
+
+/*!< The event type values are incrementing integers for use as a shift ordinal
+ * in the event filtering process at the point events are generated.
+ * This scheme thus implies a limit of 31 event types.
+ */
+static_assert(RGX_HWPERF_HOST_LAST_TYPE < RGX_HWPERF_HOST_MAX_TYPE, "Too many HWPerf host event types");
+
+/*! Define containing bit position for 32bit feature flags used in hwperf and api */
+typedef IMG_UINT32 RGX_HWPERF_FEATURE_FLAGS;
+#define RGX_HWPERF_FEATURE_PERFBUS_FLAG                    0x00000001U
+#define RGX_HWPERF_FEATURE_S7_TOP_INFRASTRUCTURE_FLAG      0x00000002U
+#define RGX_HWPERF_FEATURE_XT_TOP_INFRASTRUCTURE_FLAG      0x00000004U
+#define RGX_HWPERF_FEATURE_PERF_COUNTER_BATCH_FLAG         0x00000008U
+#define RGX_HWPERF_FEATURE_ROGUEXE_FLAG                    0x00000010U
+#define RGX_HWPERF_FEATURE_DUST_POWER_ISLAND_S7_FLAG       0x00000020U
+#define RGX_HWPERF_FEATURE_PBE2_IN_XE_FLAG                 0x00000040U
+#define RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION             0x00000080U
+#define RGX_HWPERF_FEATURE_MULTICORE_FLAG                  0x00000100U
+#define RGX_HWPERF_FEATURE_RAYTRACING_FLAG                 0x00000200U
+#define RGX_HWPERF_FEATURE_CXT_TOP_INFRASTRUCTURE_FLAG     0x00000400U
+#define RGX_HWPERF_FEATURE_VOLCANIC_FLAG                   0x00000800U
+#define RGX_HWPERF_FEATURE_ROGUE_FLAG                      0x00001000U
+#define RGX_HWPERF_FEATURE_RESERVED1_FLAG                  0x00002000U
+#define RGX_HWPERF_FEATURE_CXT_XTP_TOP_INFRASTRUCTURE_FLAG 0x00004000U
+#define RGX_HWPERF_FEATURE_AX_TOP_INFRASTRUCTURE_FLAG      0x00008000U
+#define RGX_HWPERF_FEATURE_BX_TOP_INFRASTRUCTURE_FLAG      0x00010000U
+#define RGX_HWPERF_FEATURE_DX_TOP_INFRASTRUCTURE_FLAG      0x00020000U
+
+/* ! Define for RGX_HWPERF_DM type. The values are architecture specific */
+typedef IMG_UINT32 RGX_HWPERF_DM;
+
+/******************************************************************************
+ * Packet Header Format Version 2 Types
+ *****************************************************************************/
+
+/*! Major version number of the protocol in operation
+ */
+#define RGX_HWPERF_V2_FORMAT 2
+
+/*! Signature ASCII pattern 'HWP2' found in the first word of a HWPerfV2 packet
+ */
+#define HWPERF_PACKET_V2_SIG		0x48575032
+
+/*! Signature ASCII pattern 'HWPA' found in the first word of a HWPerfV2a packet
+ */
+#define HWPERF_PACKET_V2A_SIG		0x48575041
+
+/*! Signature ASCII pattern 'HWPB' found in the first word of a HWPerfV2b packet
+ */
+#define HWPERF_PACKET_V2B_SIG		0x48575042
+
+/*! Signature ASCII pattern 'HWPC' found in the first word of a HWPerfV2c packet
+ */
+#define HWPERF_PACKET_V2C_SIG		0x48575043
+
+#define HWPERF_PACKET_ISVALID(_val) (((_val) == HWPERF_PACKET_V2_SIG) || ((_val) == HWPERF_PACKET_V2A_SIG) || ((_val) == HWPERF_PACKET_V2B_SIG) || ((_val) == HWPERF_PACKET_V2C_SIG))
+/*!< Checks that the packet signature is one of the supported versions */
+
+/*! Type defines the HWPerf packet header common to all events. */
+typedef struct
+{
+	IMG_UINT32  ui32Sig;        /*!< Always the value HWPERF_PACKET_SIG */
+	IMG_UINT32  ui32Size;       /*!< Overall packet size in bytes */
+	IMG_UINT32  eTypeId;        /*!< Event type information field */
+	IMG_UINT32  ui32Ordinal;    /*!< Sequential number of the packet */
+	IMG_UINT64  ui64Timestamp;  /*!< Event timestamp */
+} RGX_HWPERF_V2_PACKET_HDR, *RGX_PHWPERF_V2_PACKET_HDR;
+
+RGX_FW_STRUCT_OFFSET_ASSERT(RGX_HWPERF_V2_PACKET_HDR, ui64Timestamp);
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_V2_PACKET_HDR);
+
+
+/*! Mask for use with the IMG_UINT32 ui32Size header field */
+#define RGX_HWPERF_SIZE_MASK         0xFFFFU
+
+/*! This macro defines an upper limit to which the size of the largest variable
+ * length HWPerf packet must fall within, currently 3KB. This constant may be
+ * used to allocate a buffer to hold one packet.
+ * This upper limit is policed by packet producing code.
+ */
+#define RGX_HWPERF_MAX_PACKET_SIZE   0xC00U
+
+/*! Defines an upper limit to the size of a variable length packet payload.
+ */
+#define RGX_HWPERF_MAX_PAYLOAD_SIZE	 ((IMG_UINT32)(RGX_HWPERF_MAX_PACKET_SIZE-\
+	sizeof(RGX_HWPERF_V2_PACKET_HDR)))
+
+/*! Macro which takes a structure name and provides the packet size for
+ * a fixed size payload packet, rounded up to 8 bytes to align packets
+ * for 64 bit architectures. */
+#define RGX_HWPERF_MAKE_SIZE_FIXED(_struct)       ((IMG_UINT32)(RGX_HWPERF_SIZE_MASK&(sizeof(RGX_HWPERF_V2_PACKET_HDR)+PVR_ALIGN(sizeof(_struct), PVRSRVTL_PACKET_ALIGNMENT))))
+
+/*! Macro which takes the number of bytes written in the data payload of a
+ * packet for a variable size payload packet, rounded up to 8 bytes to
+ * align packets for 64 bit architectures. */
+#define RGX_HWPERF_MAKE_SIZE_VARIABLE(_size)      ((IMG_UINT32)(RGX_HWPERF_SIZE_MASK&((IMG_UINT32)sizeof(RGX_HWPERF_V2_PACKET_HDR)+PVR_ALIGN((_size), PVRSRVTL_PACKET_ALIGNMENT))))
+
+/*! Macro to obtain the size of the packet */
+#define RGX_HWPERF_GET_SIZE(_packet_addr)         ((IMG_UINT16)(((_packet_addr)->ui32Size) & RGX_HWPERF_SIZE_MASK))
+
+/*! Macro to obtain the size of the packet data */
+#define RGX_HWPERF_GET_DATA_SIZE(_packet_addr)    (RGX_HWPERF_GET_SIZE(_packet_addr) - sizeof(RGX_HWPERF_V2_PACKET_HDR))
+
+/*! Masks for use with the IMG_UINT32 eTypeId header field */
+#define RGX_HWPERF_TYPEID_MASK			0x0007FFFFU
+#define RGX_HWPERF_TYPEID_EVENT_MASK	0x00007FFFU
+#define RGX_HWPERF_TYPEID_THREAD_MASK	0x00008000U
+#define RGX_HWPERF_TYPEID_STREAM_MASK	0x00070000U
+#define RGX_HWPERF_TYPEID_META_DMA_MASK	0x00080000U
+#define RGX_HWPERF_TYPEID_M_CORE_MASK	0x00100000U
+#define RGX_HWPERF_TYPEID_PIPEDM_MASK	0x00200000U
+#define RGX_HWPERF_TYPEID_OSID_MASK		0x1f000000U
+
+/*! Meta thread macros for encoding the ID into the type field of a packet */
+#define RGX_HWPERF_META_THREAD_SHIFT	15U
+#define RGX_HWPERF_META_THREAD_ID0		0x0U  /*!< Meta Thread 0 ID */
+#define RGX_HWPERF_META_THREAD_ID1		0x1U  /*!< Meta Thread 1 ID */
+/*! Obsolete, kept for source compatibility */
+#define RGX_HWPERF_META_THREAD_MASK		0x1U
+/*! Stream ID macros for encoding the ID into the type field of a packet */
+#define RGX_HWPERF_STREAM_SHIFT			16U
+/*! Meta DMA macro for encoding how the packet was generated into the type field of a packet */
+#define RGX_HWPERF_META_DMA_SHIFT		19U
+/*! Bit-shift macro used for encoding multi-core data into the type field of a packet */
+#define RGX_HWPERF_M_CORE_SHIFT			20U
+/*! Bit-shift macro used for encoding Pipeline DM data into the type field of a packet */
+#define RGX_HWPERF_PIPEDM_SHIFT			21U
+/*! OSID bit-shift macro used for encoding OSID into type field of a packet */
+#define RGX_HWPERF_OSID_SHIFT			24U
+
+/*! HWPerf Stream ID type definition. Maximum of 32bits. */
+typedef IMG_UINT32 RGX_HWPERF_STREAM_ID;
+/*! Events from the Firmware/GPU */
+#define RGX_HWPERF_STREAM_ID0_FW		0U
+/*! Events from the Server host driver component */
+#define RGX_HWPERF_STREAM_ID1_HOST		1U
+/*! Events from the Client host driver component */
+#define RGX_HWPERF_STREAM_ID2_CLIENT	2U
+#define RGX_HWPERF_STREAM_ID_LAST		3U
+
+/* Checks if all stream IDs can fit under RGX_HWPERF_TYPEID_STREAM_MASK. */
+static_assert(((IMG_UINT32)RGX_HWPERF_STREAM_ID_LAST - 1U) < (RGX_HWPERF_TYPEID_STREAM_MASK >> RGX_HWPERF_STREAM_SHIFT),
+		"Too many HWPerf stream IDs.");
+
+/*! Compile-time value used to seed the Multi-Core (MC) bit in the typeID field.
+ *  Only set by RGX_FIRMWARE builds.
+ */
+#if defined(RGX_FIRMWARE)
+# if defined(RGX_FEATURE_GPU_MULTICORE_SUPPORT)
+#define RGX_HWPERF_M_CORE_VALUE 1U  /*!< 1 => Multi-core supported */
+# else
+#define RGX_HWPERF_M_CORE_VALUE 0U  /*!< 0 => Multi-core not supported */
+# endif
+# if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION) && (RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION > 0)
+#define RGX_HWPERF_PIPEDM_VALUE 1U  /*!< 1 => Pipeline DM supported */
+# else
+#define RGX_HWPERF_PIPEDM_VALUE 0U  /*!< 0 => Pipeline DM not supported */
+# endif
+#else
+#define RGX_HWPERF_M_CORE_VALUE 0U  /*!< 0 => Multi-core not supported */
+#define RGX_HWPERF_PIPEDM_VALUE 0U  /*!< 0 => Pipeline DM not supported */
+#endif
+
+/*! Macros used to set the packet type and encode meta thread ID (0|1),
+ * HWPerf stream ID, multi-core capability and OSID within the typeID */
+#define RGX_HWPERF_MAKE_TYPEID(_stream, _type, _thread, _metadma, _osid)\
+		((IMG_UINT32) ((RGX_HWPERF_TYPEID_STREAM_MASK&((IMG_UINT32)(_stream) << RGX_HWPERF_STREAM_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_THREAD_MASK & ((IMG_UINT32)(_thread) << RGX_HWPERF_META_THREAD_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_EVENT_MASK & (IMG_UINT32)(_type)) | \
+		(RGX_HWPERF_TYPEID_META_DMA_MASK & ((IMG_UINT32)(_metadma) << RGX_HWPERF_META_DMA_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_OSID_MASK & ((IMG_UINT32)(_osid) << RGX_HWPERF_OSID_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_M_CORE_MASK & ((IMG_UINT32)(RGX_HWPERF_M_CORE_VALUE) << RGX_HWPERF_M_CORE_SHIFT)) | \
+		(RGX_HWPERF_TYPEID_PIPEDM_MASK & ((IMG_UINT32)(RGX_HWPERF_PIPEDM_VALUE) << RGX_HWPERF_PIPEDM_SHIFT))))
+
+/*! Obtains the event type that generated the packet */
+#define RGX_HWPERF_GET_TYPE(_packet_addr)            (((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_EVENT_MASK)
+
+/*! Obtains the META Thread number that generated the packet */
+#define RGX_HWPERF_GET_THREAD_ID(_packet_addr)       (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_THREAD_MASK) >> RGX_HWPERF_META_THREAD_SHIFT))
+
+/*! Determines if the packet generated contains multi-core data */
+#define RGX_HWPERF_GET_M_CORE(_packet_addr)          (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_M_CORE_MASK) >> RGX_HWPERF_M_CORE_SHIFT)
+
+/*! Determines if the packet generated contains multi-core data */
+#define RGX_HWPERF_GET_PIPEDM(_packet_addr)          (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_PIPEDM_MASK) >> RGX_HWPERF_PIPEDM_SHIFT)
+
+/*! Obtains the guest OSID which resulted in packet generation */
+#define RGX_HWPERF_GET_OSID(_packet_addr)            (((_packet_addr)->eTypeId & RGX_HWPERF_TYPEID_OSID_MASK) >> RGX_HWPERF_OSID_SHIFT)
+
+/*! Obtain stream id */
+#define RGX_HWPERF_GET_STREAM_ID(_packet_addr)       (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_STREAM_MASK) >> RGX_HWPERF_STREAM_SHIFT))
+
+/*! Obtain information about how the packet was generated, which might affect payload total size */
+#define RGX_HWPERF_GET_META_DMA_INFO(_packet_addr)   (((((_packet_addr)->eTypeId) & RGX_HWPERF_TYPEID_META_DMA_MASK) >> RGX_HWPERF_META_DMA_SHIFT))
+
+/*! Obtains a typed pointer to a packet given a buffer address */
+#define RGX_HWPERF_GET_PACKET(_buffer_addr)            ((RGX_HWPERF_V2_PACKET_HDR *)(void *)  (_buffer_addr))
+/*! Obtains a typed pointer to a data structure given a packet address */
+#define RGX_HWPERF_GET_PACKET_DATA_BYTES(_packet_addr) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR)))
+/*! Obtains a typed pointer to the next packet given a packet address */
+#define RGX_HWPERF_GET_NEXT_PACKET(_packet_addr)       ((RGX_HWPERF_V2_PACKET_HDR *)  (IMG_OFFSET_ADDR((_packet_addr), RGX_HWPERF_SIZE_MASK&((_packet_addr)->ui32Size))))
+
+/*! Obtains a typed pointer to a packet header given the packet data address */
+#define RGX_HWPERF_GET_PACKET_HEADER(_packet_addr)     ((RGX_HWPERF_V2_PACKET_HDR *)  (IMG_OFFSET_ADDR((_packet_addr), -(IMG_INT32)sizeof(RGX_HWPERF_V2_PACKET_HDR))))
+
+
+/******************************************************************************
+ * Other Common Defines
+ *****************************************************************************/
+
+/*! Used to align structs using a flexible array member as the definition of the member
+ * size can change between C standards */
+#if (IMG_FLEX_ARRAY_MEMBER + 0)
+#define RGX_HWPERF_FLEX_ARRAY_ALIGN(_align_var) _align_var
+#else
+#define RGX_HWPERF_FLEX_ARRAY_ALIGN(...)
+#endif
+
+/*! Masks for use with the IMG_UINT32 ui32BlkInfo field */
+#define RGX_HWPERF_BLKINFO_BLKCOUNT_MASK	0xFFFF0000U
+#define RGX_HWPERF_BLKINFO_BLKOFFSET_MASK	0x0000FFFFU
+
+/*! Shift for the NumBlocks and counter block offset field in ui32BlkInfo */
+#define RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT	16U
+#define RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT 0U
+
+/*! Macro used to set the block info word as a combination of two 16-bit integers */
+#define RGX_HWPERF_MAKE_BLKINFO(_numblks, _blkoffset) ((IMG_UINT32) ((RGX_HWPERF_BLKINFO_BLKCOUNT_MASK&((_numblks) << RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT)) | (RGX_HWPERF_BLKINFO_BLKOFFSET_MASK&((_blkoffset) << RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT))))
+
+/*! Macro used to obtain the number of counter blocks present in the packet */
+#define RGX_HWPERF_GET_BLKCOUNT(_blkinfo)            (((_blkinfo) & RGX_HWPERF_BLKINFO_BLKCOUNT_MASK) >> RGX_HWPERF_BLKINFO_BLKCOUNT_SHIFT)
+
+/*! Obtains the offset of the counter block stream in the packet */
+#define RGX_HWPERF_GET_BLKOFFSET(_blkinfo)           (((_blkinfo) & RGX_HWPERF_BLKINFO_BLKOFFSET_MASK) >> RGX_HWPERF_BLKINFO_BLKOFFSET_SHIFT)
+
+/*! This macro gets the number of blocks depending on the packet version */
+#define RGX_HWPERF_GET_NUMBLKS(_sig, _packet_data, _numblocks) \
+	do { \
+		if (HWPERF_PACKET_V2B_SIG == (_sig) || HWPERF_PACKET_V2C_SIG == (_sig)) \
+		{ \
+			(_numblocks) = RGX_HWPERF_GET_BLKCOUNT((_packet_data)->ui32BlkInfo);\
+		} \
+		else \
+		{ \
+			IMG_UINT32 ui32VersionOffset = (((_sig) == HWPERF_PACKET_V2_SIG) ? 1 : 3);\
+			(_numblocks) = *(IMG_UINT16 *)(IMG_OFFSET_ADDR(&(_packet_data)->ui32WorkTarget, ui32VersionOffset)); \
+		} \
+	} while (0)
+
+/*! This macro gets the counter stream pointer depending on the packet version */
+#define RGX_HWPERF_GET_CNTSTRM(_sig, _hw_packet_data, _cntstream_ptr) \
+{ \
+	if (HWPERF_PACKET_V2B_SIG == (_sig) || HWPERF_PACKET_V2C_SIG == (_sig)) \
+	{ \
+		(_cntstream_ptr) = (IMG_UINT32 *)(IMG_OFFSET_ADDR((_hw_packet_data), RGX_HWPERF_GET_BLKOFFSET((_hw_packet_data)->ui32BlkInfo))); \
+	} \
+	else \
+	{ \
+		IMG_UINT32 ui32BlkStreamOffsetInWords = (((_sig) == HWPERF_PACKET_V2_SIG) ? 6 : 8); \
+		(_cntstream_ptr) = (IMG_UINT32 *)(IMG_OFFSET_ADDR_DW((_hw_packet_data), ui32BlkStreamOffsetInWords)); \
+	} \
+}
+
+/*! Masks for use with the IMG_UINT32 ui32KickInfo field */
+#define RGX_HWPERF_KICKINFO_STARTBE_MASK	0xFFFFFF00U
+#define RGX_HWPERF_KICKINFO_KICKID_MASK		0x000000FFU
+
+/*! Shift for the Kick ID field in ui32KickInfo */
+#define RGX_HWPERF_KICKINFO_STARTBE_SHIFT 8U
+#define RGX_HWPERF_KICKINFO_KICKID_SHIFT  0U
+
+/*! Macro used to set the kick info field. */
+#define RGX_HWPERF_MAKE_KICKINFO(_startbe, _kickid) \
+        ((IMG_UINT32) (RGX_HWPERF_KICKINFO_STARTBE_MASK&((_startbe) << RGX_HWPERF_KICKINFO_STARTBE_SHIFT))| \
+                      (RGX_HWPERF_KICKINFO_KICKID_MASK&((_kickid) << RGX_HWPERF_KICKINFO_KICKID_SHIFT)))
+
+/*! Macro used to obtain the lowest 24 bits of START_BE if present in the packet */
+#define RGX_HWPERF_GET_STARTBE(_kickinfo)           (((_kickinfo) & RGX_HWPERF_KICKINFO_STARTBE_MASK) >> RGX_HWPERF_KICKINFO_STARTBE_SHIFT)
+
+/*! Macro used to obtain the Kick ID if present in the packet */
+#define RGX_HWPERF_GET_KICKID(_kickinfo)            (((_kickinfo) & RGX_HWPERF_KICKINFO_KICKID_MASK) >> RGX_HWPERF_KICKINFO_KICKID_SHIFT)
+
+/*! Masks for use with the RGX_HWPERF_UFO_EV eEvType field */
+#define RGX_HWPERF_UFO_STREAMSIZE_MASK 0xFFFF0000U
+#define RGX_HWPERF_UFO_STREAMOFFSET_MASK 0x0000FFFFU
+
+/*! Shift for the UFO count and data stream fields */
+#define RGX_HWPERF_UFO_STREAMSIZE_SHIFT 16U
+#define RGX_HWPERF_UFO_STREAMOFFSET_SHIFT 0U
+
+/*! Macro used to set UFO stream info word as a combination of two 16-bit integers */
+#define RGX_HWPERF_MAKE_UFOPKTINFO(_ssize, _soff) \
+        ((IMG_UINT32) ((RGX_HWPERF_UFO_STREAMSIZE_MASK&((_ssize) << RGX_HWPERF_UFO_STREAMSIZE_SHIFT)) | \
+        (RGX_HWPERF_UFO_STREAMOFFSET_MASK&((_soff) << RGX_HWPERF_UFO_STREAMOFFSET_SHIFT))))
+
+/*! Macro used to obtain UFO count*/
+#define RGX_HWPERF_GET_UFO_STREAMSIZE(_streaminfo) \
+        (((_streaminfo) & RGX_HWPERF_UFO_STREAMSIZE_MASK) >> RGX_HWPERF_UFO_STREAMSIZE_SHIFT)
+
+/*! Obtains the offset of the UFO stream in the packet */
+#define RGX_HWPERF_GET_UFO_STREAMOFFSET(_streaminfo) \
+        (((_streaminfo) & RGX_HWPERF_UFO_STREAMOFFSET_MASK) >> RGX_HWPERF_UFO_STREAMOFFSET_SHIFT)
+
+
+/*! This structure holds the data of a firmware packet. */
+typedef struct
+{
+	RGX_HWPERF_DM eDM;             /*!< DataMaster identifier, see RGX_HWPERF_DM */
+	IMG_UINT32 ui32TxtActCyc;      /*!< Meta TXTACTCYC register value */
+	IMG_UINT32 ui32FWPerfCount0;   /*!< Meta/MIPS PERF_COUNT0 register */
+	IMG_UINT32 ui32FWPerfCount1;   /*!< Meta/MIPS PERF_COUNT1 register */
+	IMG_UINT32 ui32TimeCorrIndex;  /*!< Internal field */
+	IMG_UINT32 ui32Padding;        /*!< Reserved */
+} RGX_HWPERF_FW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FW_DATA);
+
+
+/*! This structure holds the data of a gpu work period packet. */
+typedef struct
+{
+	IMG_UINT64 ui64GPUWorkPeriodStartTime;
+	IMG_UINT32 ui32UID;               /*!< Process identifier */
+	IMG_UINT32 ui32TimeCorrIndex;     /*!< Index to the time correlation at the beginning of gpu work period */
+	IMG_UINT32 ui32StartTimeCorrIndex;     /*!< Index to the time correlation at the time the packet was generated */
+	IMG_UINT32 ui32Padding2;          /*!< Reserved. To ensure correct alignment (not written in the packet) */
+} RGX_HWPERF_GPU_WORK_PERIOD_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_GPU_WORK_PERIOD_DATA);
+
+/*! This structure holds the data of a hardware packet, including counters. */
+typedef struct
+{
+	union
+	{
+		IMG_UINT32 ui32DMCyc;         /*!< DataMaster cycle count register, 0 if none */
+		IMG_UINT32 ui32KickStartTime; /*!< Front End start time for Pipeline DMs */
+	};
+	IMG_UINT32 ui32FrameNum;          /*!< Frame number, undefined on some DataMasters */
+	IMG_UINT32 ui32PID;               /*!< Process identifier */
+	IMG_UINT32 ui32DMContext;         /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32WorkTarget;        /*!< RenderTarget for a TA,3D; Frame context for RTU, 0x0 otherwise */
+	IMG_UINT32 ui32ExtJobRef;         /*!< Client driver context job reference used for tracking/debugging */
+	IMG_UINT32 ui32IntJobRef;         /*!< RGX Data master context job reference used for tracking/debugging */
+	IMG_UINT32 ui32TimeCorrIndex;     /*!< Index to the time correlation at the time the packet was generated */
+	IMG_UINT32 ui32BlkInfo;           /*!< <31..16> NumBlocks <15..0> Counter block stream offset */
+	IMG_UINT32 ui32WorkCtx;           /*!< Work context: Render Context for TA/3D; RayTracing Context for RTU/SHG; 0x0 otherwise */
+	IMG_UINT32 ui32CtxPriority;       /*!< Context priority */
+	IMG_UINT32 ui32GPUIdMask;         /*!< GPU IDs active within this event */
+	IMG_UINT32 ui32KickInfo;          /*!< <31..8> Back End start time lowest 24 bits <7..0> GPU Pipeline DM kick ID, 0 if not using Pipeline DMs */
+	IMG_UINT32 ui32KickEndTime;       /*!< Back End finish time for Pipeline DMs */
+	IMG_UINT32 aui32CountBlksStream[IMG_FLEX_ARRAY_MEMBER]; /*!< Optional variable length Counter data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);          /*!< Reserved. To ensure correct alignment (not written in the packet) */
+} RGX_HWPERF_HW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_HW_DATA);
+RGX_FW_STRUCT_OFFSET_ASSERT(RGX_HWPERF_HW_DATA, aui32CountBlksStream);
+
+/*! Mask for use with the aui32CountBlksStream field when decoding the
+ * counter block ID and mask word. */
+#define RGX_HWPERF_CNTBLK_ID_MASK	0xFFFF0000U
+#define RGX_HWPERF_CNTBLK_ID_SHIFT	16U
+
+/*! MAX value used in server handling of counter config arrays */
+#define RGX_CNTBLK_COUNTERS_MAX PVRSRV_HWPERF_COUNTERS_PERBLK
+
+/*! Obtains the counter block ID word from an aui32CountBlksStream field.
+ * The word combines Control bits (15-12), GPU-Id (11-8), Group (7-4), Unit
+ * within group (3-0) */
+#define RGX_HWPERF_GET_CNTBLK_IDW(_word)           ((IMG_UINT16)(((_word)&RGX_HWPERF_CNTBLK_ID_MASK)>>RGX_HWPERF_CNTBLK_ID_SHIFT))
+
+/*! Obtains the counter block ID from the supplied RGX_HWPERF_HW_DATA address
+ * and stream index. May be used in decoding the counter block stream words of
+ * a RGX_HWPERF_HW_DATA structure. */
+#define RGX_HWPERF_GET_CNTBLK_ID(_data_addr, _idx) RGX_HWPERF_GET_CNTBLK_IDW((_data_addr)->aui32CountBlksStream[(_idx)])
+
+/*! Obtains the GPU ID from the supplied RGX_HWPERF_HW_DATA CNTBLK_IDW */
+#define RGX_HWPERF_GET_CNTBLK_GPUW(_word)          ((IMG_UINT16)(((_word)&RGX_CNTBLK_ID_MC_GPU_MASK)>>RGX_CNTBLK_ID_MC_GPU_SHIFT))
+
+#define RGX_HWPERF_GET_CNT_MASKW(_word)           ((IMG_UINT16)((_word)&(~RGX_HWPERF_CNTBLK_ID_MASK)))
+
+/*! Obtains the counter mask from the supplied RGX_HWPERF_HW_DATA address
+ * and stream index. May be used in decoding the counter block stream words
+ * of a RGX_HWPERF_HW_DATA structure. */
+#define RGX_HWPERF_GET_CNT_MASK(_data_addr, _idx) RGX_HWPERF_GET_CNT_MASKW((_data_addr)->aui32CountBlksStream[(_idx)])
+
+
+/*! Context switch packet event */
+typedef struct
+{
+	RGX_HWPERF_DM	eDM;					/*!< DataMaster identifier, see RGX_HWPERF_DM */
+	IMG_UINT32		ui32DMContext;			/*!< GPU Data Master (FW) Context */
+	IMG_UINT32		ui32FrameNum;			/*!< Client Frame number (TA, 3D only) */
+	IMG_UINT32		ui32TxtActCyc;			/*!< Meta TXTACTCYC register value */
+	IMG_UINT32		ui32PerfCycle;			/*!< Cycle count. Used to measure HW context store latency */
+	IMG_UINT32		ui32PerfPhase;			/*!< Phase. Used to determine geometry content */
+	IMG_UINT32		ui32GPUIdMask;			/*!< GPU IDs active within this event */
+	IMG_UINT32		ui32Padding;			/*!< Padding to 8 DWords */
+} RGX_HWPERF_CSW_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CSW_DATA);
+
+/*! Enumeration of clocks supporting this event */
+typedef enum
+{
+	RGX_HWPERF_CLKS_CHG_INVALID = 0,
+
+	RGX_HWPERF_CLKS_CHG_NAME_CORE = 1,
+
+	RGX_HWPERF_CLKS_CHG_LAST,
+} RGX_HWPERF_CLKS_CHG_NAME;
+
+/*! This structure holds the data of a clocks change packet. */
+typedef struct
+{
+	IMG_UINT64                ui64NewClockSpeed;         /*!< New Clock Speed (in Hz) */
+	RGX_HWPERF_CLKS_CHG_NAME  eClockName;                /*!< Clock name */
+	IMG_UINT32                ui32CalibratedClockSpeed;  /*!< Calibrated new GPU clock speed (in Hz) */
+	IMG_UINT64                ui64OSTimeStamp;           /*!< OSTimeStamp sampled by the host */
+	IMG_UINT64                ui64CRTimeStamp;           /*!< CRTimeStamp sampled by the host and
+	                                                          correlated to OSTimeStamp */
+} RGX_HWPERF_CLKS_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_CLKS_CHG_DATA);
+
+/*! Enumeration of GPU utilisation states supported by this event */
+typedef IMG_UINT32 RGX_HWPERF_GPU_STATE;
+
+/*! This structure holds the data of a GPU utilisation state change packet. */
+typedef struct
+{
+	RGX_HWPERF_GPU_STATE	eState;		/*!< New GPU utilisation state */
+	IMG_UINT32				uiUnused1;	/*!< Padding */
+	IMG_UINT32				uiUnused2;	/*!< Padding */
+	IMG_UINT32				uiUnused3;	/*!< Padding */
+} RGX_HWPERF_GPU_STATE_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_GPU_STATE_CHG_DATA);
+
+
+/*! Signature pattern 'HPE1' found in the first word of a PWR_EST packet data */
+#define HWPERF_PWR_EST_V1_SIG	0x48504531
+
+/*! Macros to obtain a component field from a counter ID word */
+#define RGX_HWPERF_GET_PWR_EST_HIGH_FLAG(_word) (((_word)&0x80000000)>>31)
+#define RGX_HWPERF_GET_PWR_EST_GPUID(_word)     (((_word)&0x70000000)>>28)
+/*!< Obtains the GPU ID from a counter ID word */
+#define RGX_HWPERF_GET_PWR_EST_UNIT(_word)      (((_word)&0x0F000000)>>24)
+#define RGX_HWPERF_GET_PWR_EST_NUMBER(_word)    ((_word)&0x0000FFFF)
+
+#define RGX_HWPERF_PWR_EST_HIGH_OFFSET		(31)
+#define RGX_HWPERF_PWR_EST_GPUID_OFFSET		(28)
+#define RGX_HWPERF_PWR_EST_GPUID_MASK		(0x7U)
+#define RGX_HWPERF_PWR_EST_UNIT_OFFSET		(24)
+#define RGX_HWPERF_PWR_EST_UNIT_MASK		(0xFU)
+#define RGX_HWPERF_PWR_EST_VALUE_MASK		(0xFFFFU)
+
+/*! This macro constructs a counter ID for a power estimate data stream from
+ * the component parts of: high word flag, unit id, GPU id, counter number */
+#define RGX_HWPERF_MAKE_PWR_EST_COUNTERID(_high, _unit, _core, _number) \
+			((IMG_UINT32)(((IMG_UINT32)((IMG_UINT32)(_high)&0x1U)<<RGX_HWPERF_PWR_EST_HIGH_OFFSET) | \
+			((IMG_UINT32) ((IMG_UINT32)(_unit)&RGX_HWPERF_PWR_EST_UNIT_MASK)<<RGX_HWPERF_PWR_EST_UNIT_OFFSET) | \
+			((IMG_UINT32) ((IMG_UINT32)(_core)&RGX_HWPERF_PWR_EST_GPUID_MASK)<<RGX_HWPERF_PWR_EST_GPUID_OFFSET) | \
+			                           ((_number)&RGX_HWPERF_PWR_EST_VALUE_MASK)))
+
+/*! This structure holds the data for a power estimate packet. */
+typedef struct
+{
+	IMG_UINT32  ui32StreamVersion;  /*!< Version word, HWPERF_PWR_EST_V1_SIG */
+	IMG_UINT32  ui32StreamSize;     /*!< Size of array in bytes of stream data
+	                                     held in the aui32StreamData member */
+	IMG_UINT32  aui32StreamData[IMG_FLEX_ARRAY_MEMBER]; /*!< Counter data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved); /*!< Reserved. To ensure correct alignment */
+} RGX_HWPERF_PWR_EST_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_PWR_EST_DATA);
+
+/*! Enumeration of the kinds of power change events that can occur */
+typedef enum
+{
+	RGX_HWPERF_PWR_UNDEFINED    = 0,
+	RGX_HWPERF_PWR_ON           = 1, /*!< Whole device powered on */
+	RGX_HWPERF_PWR_OFF          = 2, /*!< Whole device powered off */
+	RGX_HWPERF_PWR_UP           = 3, /*!< Power turned on to a HW domain */
+	RGX_HWPERF_PWR_DOWN         = 4, /*!< Power turned off to a HW domain */
+	RGX_HWPERF_PWR_SAFETY_RESET = 5, /*!< Resetting the GPU HW units for safety reasons */
+	RGX_HWPERF_PWR_PHR_FULL     = 6, /*!< Periodic HW full GPU Reset */
+	RGX_HWPERF_PWR_RD_UP        = 7, /*!< Power turned on to a Rascal+Dust */
+	RGX_HWPERF_PWR_RD_DOWN      = 8, /*!< Power turned off to a Rascal+Dust */
+	RGX_HWPERF_PWR_SPU_UP       = 9, /*!< Power turned on to a SPU */
+	RGX_HWPERF_PWR_SPU_DOWN     = 10, /*!< Power turned off to a SPU */
+	RGX_HWPERF_PWR_CLUSTER_UP   = 11, /*!< Power turned on to a cluster */
+	RGX_HWPERF_PWR_CLUSTER_DOWN = 12, /*!< Power turned off to a cluster */
+	RGX_HWPERF_PWR_RAC_UP       = 13, /*!< Power turned on to a RAC */
+	RGX_HWPERF_PWR_RAC_DOWN     = 14, /*!< Power turned off to a RAC */
+
+	RGX_HWPERF_PWR_LAST,
+} RGX_HWPERF_PWR;
+
+/*! This structure holds the data of a power packet. */
+typedef struct
+{
+	RGX_HWPERF_PWR eChange;                  /*!< Defines the type of power change */
+	IMG_UINT32     ui32Domains;              /*!< HW Domains affected */
+	IMG_UINT64     ui64OSTimeStamp;          /*!< OSTimeStamp sampled by the host */
+	IMG_UINT64     ui64CRTimeStamp;          /*!< CRTimeStamp sampled by the host and
+	                                              correlated to OSTimeStamp */
+	IMG_UINT32     ui32CalibratedClockSpeed; /*!< GPU clock speed (in Hz) at the time
+	                                              the two timers were correlated */
+	IMG_UINT32     ui32Unused1;              /*!< Padding */
+} RGX_HWPERF_PWR_CHG_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_PWR_CHG_DATA);
+
+/*
+ * PDVFS, GPU clock frequency changes and workload estimation profiling
+ * data.
+ */
+/*! DVFS and work estimation events. */
+typedef enum
+{
+	RGX_HWPERF_DVFS_EV_INVALID,                 /*!< Invalid value. */
+	RGX_HWPERF_DVFS_EV_PROACTIVE_EST_START,     /*!< Proactive DVFS estimate start */
+	RGX_HWPERF_DVFS_EV_PROACTIVE_EST_FINISHED,  /*!< Proactive DVFS estimate finished */
+	RGX_HWPERF_DVFS_EV_REACTIVE_EST_START,      /*!< Reactive DVFS estimate start */
+	RGX_HWPERF_DVFS_EV_REACTIVE_EST_FINISHED,   /*!< Reactive DVFS estimate finished */
+	/* workload estimation */
+	RGX_HWPERF_DVFS_EV_WORK_EST_START,          /*!< Workload estimation start */
+	RGX_HWPERF_DVFS_EV_WORK_EST_FINISHED,       /*!< Workload estimation finished */
+	RGX_HWPERF_DVFS_EV_FREQ_CHG,                /*!< DVFS OPP/clock frequency change */
+
+	RGX_HWPERF_DVFS_EV_LAST               /*!< Number of element. */
+} RGX_HWPERF_DVFS_EV;
+
+/*! Enumeration of DVFS transitions that can occur */
+typedef enum
+{
+	RGX_HWPERF_DVFS_OPP_NONE        = 0x0,  /*!< No OPP change, already operating at required freq */
+	RGX_HWPERF_DVFS_OPP_IDLE        = 0x1,  /*!< GPU is idle, defer the OPP change */
+	/* 0x2 to 0xF reserved */
+	RGX_HWPERF_DVFS_OPP_UPDATE      = 0x10, /*!< OPP change, new point is encoded in bits [3:0] */
+	RGX_HWPERF_DVFS_OPP_LAST        = 0x20,
+} RGX_HWPERF_DVFS_OPP;
+
+typedef union
+{
+	/*! This structure holds the data of a proactive DVFS calculation packet. */
+	struct
+	{
+		IMG_UINT64     ui64DeadlineInus;         /*!< Next deadline in microseconds */
+		IMG_UINT32     ui32Frequency;            /*!< Required freq to meet deadline at 90% utilisation */
+		IMG_UINT32     ui32WorkloadCycles;       /*!< Current workload estimate in cycles */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sProDVFSCalc;
+
+	/*! This structure holds the data of a reactive DVFS calculation packet. */
+	struct
+	{
+		IMG_UINT32     ui32Frequency;            /*!< Required freq to achieve average 90% utilisation */
+		IMG_UINT32     ui32Utilisation;          /*!< GPU utilisation since last update in percentage points (0..100%) */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sDVFSCalc;
+
+	/*! This structure holds the data of a work estimation packet. */
+	struct
+	{
+		IMG_UINT32     ui32CyclesPrediction;     /*!< Predicted cycle count for this workload */
+		IMG_UINT32     ui32CyclesTaken;          /*!< Actual cycle count for this workload */
+		RGX_HWPERF_DM  eDM;                      /*!< Target DM */
+		IMG_UINT32     ui32ReturnDataIndex;      /*!< Index into workload estimation table */
+		IMG_UINT32     ui32TxtActCyc;            /*!< Meta TXTACTCYC register value */
+	} sWorkEst;
+
+	/*! This structure holds the data of an Operating Performance Points (OPP) clock frequency transition packet. */
+	struct
+	{
+		IMG_UINT32     ui32OPPData;              /*!< New Operating Performance Point (OPP) */
+	} sOPP;
+
+} RGX_HWPERF_DVFS_DETAIL;
+
+/*! DVFS sub-event data structure */
+typedef struct {
+	RGX_HWPERF_DVFS_EV      eEventType;          /*!< DVFS sub-event type */
+	RGX_HWPERF_DVFS_DETAIL  uData;               /*!< DVFS sub-event data */
+} RGX_HWPERF_DVFS_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_DVFS_DATA);
+
+typedef enum {
+	RGX_HWPERF_UFO_EV_UPDATE,          /*!< Update on the UFO objects. */
+	RGX_HWPERF_UFO_EV_CHECK_SUCCESS,   /*!< Successful check on UFO objects. */
+	RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS, /*!< Successful partial render check on UFO objects. */
+	RGX_HWPERF_UFO_EV_CHECK_FAIL,      /*!< Unsuccessful check on UFO objects. */
+	RGX_HWPERF_UFO_EV_PRCHECK_FAIL,    /*!< Unsuccessful partial render check on UFO objects. */
+	RGX_HWPERF_UFO_EV_FORCE_UPDATE,    /*!< Forced erroring of the UFO objects. */
+
+	RGX_HWPERF_UFO_EV_LAST             /*!< Reserved. Do not use. */
+} RGX_HWPERF_UFO_EV;
+
+/*! Data stream tuple. */
+typedef union
+{
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32Value;         /*!< Value of the UFO object */
+	} sCheckSuccess;
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32Value;         /*!< Value of the UFO object */
+		IMG_UINT32 ui32Required;      /*!< Value of the UFO object required by the fence */
+	} sCheckFail;
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< UFO's unique address */
+		IMG_UINT32 ui32OldValue;      /*!< Value of UFO object before update */
+		IMG_UINT32 ui32NewValue;      /*!< Value of UFO object after update */
+	} sUpdate;
+} RGX_HWPERF_UFO_DATA_ELEMENT;
+
+/*! This structure holds the packet payload data for UFO event. */
+typedef struct
+{
+	RGX_HWPERF_UFO_EV eEvType;     /*!< Subtype of the event. See RGX_HWPERF_UFO_EV */
+	IMG_UINT32 ui32TimeCorrIndex;  /*!< Index to the timer correlation data
+	                                 at the time the packet was generated.
+	                                 Used to approximate Host timestamps for
+	                                 these events. */
+	IMG_UINT32 ui32PID;            /*!< Client process identifier */
+	IMG_UINT32 ui32ExtJobRef;      /*!< Reference used by callers of the RGX
+	                                 API to track submitted work (for
+	                                 debugging/trace purposes) */
+	IMG_UINT32 ui32IntJobRef;      /*!< Internal reference used to track
+	                                 submitted work (for debugging / trace
+	                                 purposes) */
+	IMG_UINT32 ui32DMContext;      /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32StreamInfo;     /*!< Encoded number of elements in the
+	                                 stream and stream data offset in the
+	                                 payload */
+	RGX_HWPERF_DM eDM;             /*!< Data Master number, see RGX_HWPERF_DM */
+	IMG_UINT32 aui32StreamData[IMG_FLEX_ARRAY_MEMBER];  /*!< Series of tuples holding UFO objects data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);        /*!< Unused, reserved */
+} RGX_HWPERF_UFO_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_UFO_DATA);
+
+
+/*!
+ * RGX_HWPERF_KICK_TYPE describes the type of kick for events received / sent
+ * between KICK_START / KICK_END inclusively for all event types.
+ */
+typedef enum
+{
+	RGX_HWPERF_KICK_TYPE_RESERVED_0,  /*!< Replaced by separate TA and 3D types (Deprecated) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_1,  /*!< Compute Data Master Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_CDM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_2,  /*!< Ray Store Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_RS) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_3,  /*!< Scene Hierarchy Generator Kick (Deprecated) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_4,  /*!< TQ 2D Data Master Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_TQTDM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_5,  /*!< Sync Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_SYNC) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_6,  /*!< TA Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_GEOM) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_7,  /*!< 3D Kick (Deprecated - Use RGX_HWPERF_KICK_TYPE2_3D) */
+	RGX_HWPERF_KICK_TYPE_RESERVED_8,
+	RGX_HWPERF_KICK_TYPE_RESERVED_9,
+
+	RGX_HWPERF_KICK_TYPE_UNIFIED = 0x10,
+
+	RGX_HWPERF_KICK_TYPE2_TQ2D,  /*!< 2D TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_TQ3D,  /*!< 3D TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_TQTDM, /*!< 2D Data Master TQ Kick */
+	RGX_HWPERF_KICK_TYPE2_CDM,   /*!< Compute Kick */
+	RGX_HWPERF_KICK_TYPE2_GEOM,  /*!< GEOM Kick */
+	RGX_HWPERF_KICK_TYPE2_3D,    /*!< 3D Kick */
+	RGX_HWPERF_KICK_TYPE2_SYNC,  /*!< Sync Kick */
+	RGX_HWPERF_KICK_TYPE2_RS,    /*!< Ray Store Kick */
+	RGX_HWPERF_KICK_TYPE2_LAST,
+
+	RGX_HWPERF_KICK_TYPE_FORCE_32BIT = 0x7fffffff
+} RGX_HWPERF_KICK_TYPE;
+
+typedef enum
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_NONE,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_DEFAULT_FRAMEBUFFER,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_OFFSCREEN_FB_ATTACHMENTS,
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE_TILE_LIFETIME_DATA,
+
+	RGX_HWPERF_RESOURCE_TYPE_COUNT
+} RGX_HWPERF_RESOURCE_CAPTURE_TYPE;
+
+typedef struct
+{
+	IMG_UINT32 ui32Height;
+	IMG_UINT32 ui32Width;
+	IMG_UINT32 ui32BPP;
+	IMG_UINT32 ui32PixFormat;
+} RGX_RESOURCE_PER_SURFACE_INFO, *PRGX_RESOURCE_PER_SURFACE_INFO;
+
+typedef struct
+{
+	IMG_INT32  i32XOffset;        /*!< render surface X shift */
+	IMG_INT32  i32YOffset;        /*!< render surface Y shift */
+	IMG_UINT32 ui32WidthInTiles;  /*!< number of TLT data points in X */
+	IMG_UINT32 ui32HeightInTiles; /*!< number of TLT data points in Y */
+} RGX_RESOURCE_PER_TLT_BUFFER_INFO, *PRGX_RESOURCE_PER_TLT_BUFFER_INFO;
+
+typedef union
+{
+	struct RGX_RESOURCE_CAPTURE_RENDER_SURFACES
+	{
+		IMG_UINT32 ui32RenderSurfaceCount;
+		RGX_RESOURCE_PER_SURFACE_INFO sSurface[IMG_FLEX_ARRAY_MEMBER];
+	} sRenderSurfaces;
+
+	struct RGX_RESOURCE_CAPTURE_TILE_LIFETIME_BUFFERS
+	{
+		IMG_UINT32 uiReserved;
+		RGX_RESOURCE_PER_TLT_BUFFER_INFO sTLTBufInfo[IMG_FLEX_ARRAY_MEMBER];
+	} sTLTBuffers;
+} RGX_RESOURCE_CAPTURE_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_TYPE eType;
+	IMG_PID uPID;
+	IMG_UINT32 ui32ContextID;
+	IMG_UINT32 ui32FrameNum;
+	IMG_UINT32 ui32CapturedTaskJobRef;	/* The job ref of the HW task that emitted the data */
+	IMG_INT32 eClientModule;			/* RGX_HWPERF_CLIENT_API - ID that the capture is originating from. */
+	RGX_RESOURCE_CAPTURE_DETAIL uDetail; /* eType determines the value of the union */
+} RGX_RESOURCE_CAPTURE_INFO, *PRGX_RESOURCE_CAPTURE_INFO;
+
+#define RGX_RESOURCE_CAPTURE_INFO_BASE_SIZE() offsetof(RGX_RESOURCE_CAPTURE_INFO, uDetail)
+
+/*! Tile Lifetime Tracking header size. Only available if
+ * RGX_FEATURE_ISP_TILE_LIFETIME_TRACKING is present and enabled via
+ * SUPPORT_TLT_PERF
+ */
+#define RGX_TLT_HARDWARE_HDR_SIZE   (16U)
+
+/* PVRSRVGetHWPerfResourceCaptureResult */
+typedef enum
+{
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_NONE = 0,
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK,					/* We got data ok, expect more packets for this request. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_NOT_READY,			/* Signals a timeout on the connection - no data available yet. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_COMPLETE_SUCCESS,	/* The request completed successfully, signals the end of packets for the request. */
+	RGX_HWPERF_RESOURCE_CAPTURE_RESULT_COMPLETE_FAILURE		/* The request failed, signals the end of packets for the request. */
+} RGX_HWPERF_RESOURCE_CAPTURE_RESULT_STATUS;
+
+typedef struct
+{
+	IMG_PID uPID;						/* In case of a failed request pass the caller the PID and context ID. */
+	IMG_UINT32 ui32CtxID;
+	RGX_RESOURCE_CAPTURE_INFO *psInfo;	/* Various meta-data regarding the captured resource which aid the requester when,
+											unpacking the resource data, valid if RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK is returned. */
+	IMG_BYTE *pbData;					/* Buffer containing the captured resource data, valid if RGX_HWPERF_RESOURCE_CAPTURE_RESULT_OK is returned. */
+} RGX_RESOURCE_CAPTURE_RESULT;
+
+/*! Firmware Activity event. */
+typedef enum
+{
+	RGX_HWPERF_FWACT_EV_INVALID,            /*!< Invalid value. */
+	RGX_HWPERF_FWACT_EV_REGS_SET,           /*!< Registers set. */
+	RGX_HWPERF_FWACT_EV_HWR_DETECTED,       /*!< HWR detected. */
+	RGX_HWPERF_FWACT_EV_HWR_RESET_REQUIRED, /*!< Reset required. */
+	RGX_HWPERF_FWACT_EV_HWR_RECOVERED,      /*!< HWR recovered. */
+	RGX_HWPERF_FWACT_EV_HWR_FREELIST_READY, /*!< Freelist ready. */
+	RGX_HWPERF_FWACT_EV_FEATURES,           /*!< Features present */
+	RGX_HWPERF_FWACT_EV_FILTER_SET,         /*!< Event filter set. */
+	RGX_HWPERF_FWACT_EV_KICK_CANCEL,        /*!< A previous pipelined kick cancel. */
+
+	RGX_HWPERF_FWACT_EV_LAST                /*!< Number of element. */
+} RGX_HWPERF_FWACT_EV;
+
+/*! Cause of the HWR event. */
+typedef enum
+{
+	RGX_HWPERF_HWR_REASON_INVALID,              /*!< Invalid value. */
+	RGX_HWPERF_HWR_REASON_LOCKUP,               /*!< Lockup. */
+	RGX_HWPERF_HWR_REASON_PAGEFAULT,            /*!< Page fault. */
+	RGX_HWPERF_HWR_REASON_POLLFAIL,             /*!< Poll fail. */
+	RGX_HWPERF_HWR_REASON_DEADLINE_OVERRUN,     /*!< Deadline overrun. */
+	RGX_HWPERF_HWR_REASON_CSW_DEADLINE_OVERRUN, /*!< Hard Context Switch deadline overrun. */
+
+	RGX_HWPERF_HWR_REASON_LAST                  /*!< Number of elements. */
+} RGX_HWPERF_HWR_REASON;
+
+
+/* Fixed size for BVNC string so it does not alter packet data format
+ * Check it is large enough against official BVNC string length maximum
+ */
+#define RGX_HWPERF_MAX_BVNC_LEN (24)
+static_assert((RGX_HWPERF_MAX_BVNC_LEN >= RGX_BVNC_STR_SIZE_MAX),
+			  "Space inside HWPerf packet data for BVNC string insufficient");
+
+#define RGX_HWPERF_MAX_BVNC_BLOCK_LEN (20U)
+
+/*! BVNC Features */
+typedef struct
+{
+	/*! Counter block ID, see RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT16 ui16BlockID;
+
+	/*! Number of counters in this block type */
+	IMG_UINT16 ui16NumCounters;
+
+	/*! Number of blocks of this type */
+	IMG_UINT16 ui16NumBlocks;
+
+	/*! Reserved for future use */
+	IMG_UINT16 ui16Reserved;
+} RGX_HWPERF_BVNC_BLOCK;
+
+/*! BVNC Features */
+typedef struct
+{
+	IMG_CHAR aszBvncString[RGX_HWPERF_MAX_BVNC_LEN]; /*!< BVNC string */
+	IMG_UINT32 ui32BvncKmFeatureFlags;               /*!< See RGX_HWPERF_FEATURE_FLAGS */
+	IMG_UINT16 ui16BvncBlocks;                       /*!< Number of blocks described in aBvncBlocks */
+	IMG_UINT16 ui16BvncGPUCores;                     /*!< Number of GPU cores present */
+	RGX_HWPERF_BVNC_BLOCK aBvncBlocks[RGX_HWPERF_MAX_BVNC_BLOCK_LEN];  /*!< Supported Performance Blocks for BVNC. See RGX_HWPERF_BVNC_BLOCK */
+} RGX_HWPERF_BVNC;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_BVNC);
+
+/*! Performance Counter Configuration data element. */
+typedef struct
+{
+	IMG_UINT32 ui32BlockID;               /*!< Counter Block ID. See RGX_HWPERF_CNTBLK_ID */
+	IMG_UINT32 ui32NumCounters;           /*!< Number of counters configured */
+	IMG_UINT32 ui32CounterVals[RGX_CNTBLK_COUNTERS_MAX];  /*!< Counters configured (ui32NumCounters worth of entries) */
+} RGX_HWPERF_COUNTER_CFG_DATA_EL;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_COUNTER_CFG_DATA_EL);
+
+/*! Performance Counter Configuration data. */
+typedef struct
+{
+	IMG_UINT32 ui32EnabledBlocks;          /*!< Number of Enabled Blocks. */
+	RGX_HWPERF_COUNTER_CFG_DATA_EL uData;  /*!< Start of variable length data. See RGX_HWPERF_COUNTER_CFG_DATA_EL */
+	IMG_UINT32 ui32Padding;                /*!< reserved */
+} RGX_HWPERF_COUNTER_CFG;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_COUNTER_CFG);
+
+/*! Sub-event's data. */
+typedef union
+{
+	struct
+	{
+		RGX_HWPERF_DM eDM;				/*!< Data Master ID. */
+		RGX_HWPERF_HWR_REASON eReason;	/*!< Reason of the HWR. */
+		IMG_UINT32 ui32DMContext;		/*!< FW render context */
+	} sHWR;								/*!< HWR sub-event data. */
+
+	RGX_HWPERF_BVNC sBVNC;              /*!< BVNC Features. See RGX_HWPERF_BVNC */
+	struct
+	{
+		IMG_UINT32 ui32EvMaskLo;		/*!< Low order 32 bits of Filter Mask */
+		IMG_UINT32 ui32EvMaskHi;		/*!< High order 32 bits of Filter Mask */
+	} sEvMsk;							/*!< HW Filter Mask */
+	RGX_HWPERF_COUNTER_CFG sPCC;        /*!< Performance Counter Config. See RGX_HWPERF_COUNTER_CFG */
+
+	struct
+	{
+		RGX_HWPERF_DM eDM;				/*!< Data Master ID. */
+		IMG_UINT32 ui32DMContext;		/*!< FW context */
+		IMG_UINT32 ui32GPUIdMask;		/*!< Multicore mask. */
+		IMG_UINT32 ui32KickID;			/*!< Kick Id cancelled. */
+	} sKickCancel;						/*!< Kick cancel sub-event data. */
+} RGX_HWPERF_FWACT_DETAIL;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FWACT_DETAIL);
+
+/*! This structure holds the data of a FW activity event packet */
+typedef struct
+{
+	RGX_HWPERF_FWACT_EV eEvType;           /*!< Event type. */
+	RGX_HWPERF_FWACT_DETAIL uFwActDetail;  /*!< Data of the sub-event. */
+	IMG_UINT32 ui32Padding;                /*!< Reserved. */
+} RGX_HWPERF_FWACT_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_FWACT_DATA);
+
+typedef struct
+{
+	RGX_HWPERF_KICK_TYPE ui32EnqType; /*!< Workload type sent to FW for
+	                                   scheduling on GPU hardware.
+	                                   See RGX_HWPERF_KICK_TYPE */
+	IMG_UINT32 ui32PID;               /*!< Client process identifier */
+	IMG_UINT32 ui32ExtJobRef;         /*!< Reference used by callers of the RGX API
+	                                   to track submitted work (for debugging /
+	                                   trace purposes) */
+	IMG_UINT32 ui32IntJobRef;         /*!< internal reference used to track submitted
+	                                   work (for debugging / trace purposes) */
+	IMG_UINT32 ui32DMContext;         /*!< GPU Data Master (FW) Context */
+	IMG_UINT32 ui32Padding;           /*!< Unused, reserved */
+	IMG_UINT64 ui64CheckFence_UID;    /*!< ID of fence gating work execution on GPU */
+	IMG_UINT64 ui64UpdateFence_UID;   /*!< ID of fence triggered after work completes on GPU */
+	IMG_UINT64 ui64DeadlineInus;      /*!< Workload deadline in system monotonic time */
+	IMG_UINT32 ui32CycleEstimate;     /*!< Estimated cycle time for the workload */
+	PVRSRV_FENCE hCheckFence;         /*!< Fence this enqueue task waits for, before starting */
+	PVRSRV_FENCE hUpdateFence;        /*!< Fence this enqueue task signals, on completion */
+	PVRSRV_TIMELINE hUpdateTimeline;  /*!< Timeline on which the above hUpdateFence is created */
+
+	/* Align structure size to 8 bytes */
+} RGX_HWPERF_HOST_ENQ_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_ENQ_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_ENQ_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef struct
+{
+	RGX_HWPERF_UFO_EV eEvType;   /*!< Subtype of the event */
+	IMG_UINT32 ui32StreamInfo;   /*!< Encoded number of elements in the stream and
+	                              stream data offset in the payload */
+#ifdef __CHECKER__
+	/* Since we're not conforming to the C99 standard by not using a flexible
+	 * array member need to add a special case for Smatch static code analyser. */
+	IMG_UINT32 aui32StreamData[];
+#else
+	IMG_UINT32 aui32StreamData[IMG_FLEX_ARRAY_MEMBER];       /*!< Series of tuples holding UFO objects data */
+	RGX_HWPERF_FLEX_ARRAY_ALIGN(IMG_UINT32 uiReserved);      /*!< Reserved, align structure size to 8 bytes */
+#endif
+} RGX_HWPERF_HOST_UFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_UFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_UFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*!
+ * RGX_HWPERF_HOST_RESOURCE_TYPE describes the type of resource which has been
+ * Allocated, Freed or Modified. The values are used to determine which event
+ * data structure to use to decode the data from the event stream
+ */
+typedef enum
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE_INVALID,   /*!< Invalid */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC,      /*!< SyncPrim */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_TIMELINE_DEPRECATED,
+	                                         /*!< Timeline resource packets are
+	                                           now emitted in client hwperf buffer */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR, /*!< Fence for use on GPU (SYNC_CP backed) */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP,   /*!< Sync Checkpoint */
+	RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW,  /*!< Fence created on SW timeline */
+
+	RGX_HWPERF_HOST_RESOURCE_TYPE_LAST       /*!< End of enumeration */
+} RGX_HWPERF_HOST_RESOURCE_TYPE;
+
+typedef union
+{
+	/*! Data for TYPE_TIMELINE (*Deprecated*). This sub-event is no longer
+     *  generated in the HOST stream. Timeline data is now provided in the
+     *  CLIENT stream instead.
+     */
+	struct
+	{
+		IMG_UINT32 uiPid;             /*!< Identifier of owning process */
+		IMG_UINT64 ui64Timeline_UID1; /*!< Unique identifier for timeline resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sTimelineAlloc;
+
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_PID uiPID;                /*!< Identifier of owning process */
+		PVRSRV_FENCE hFence;          /*!< Unique identifier for the fence resource */
+		IMG_UINT32 ui32CheckPt_FWAddr; /*!< Unique identifier of the check point
+		                                backing this fence on the GPU */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+	} sFenceAlloc;
+
+	/*! Data for TYPE_SYNC_CP */
+	struct
+	{
+		IMG_UINT32 ui32CheckPt_FWAddr; /*!< Unique identifier for the check point resource */
+		PVRSRV_TIMELINE hTimeline;     /*!< Unique identifier for the timeline resource */
+		IMG_PID uiPID;                 /*!< Identifier of owning process */
+		PVRSRV_FENCE hFence;           /*!< Unique identifier for the fence resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSyncCheckPointAlloc;
+
+	/*! Data for TYPE_FENCE_SW */
+	struct
+	{
+		IMG_PID uiPID;                 /*!< Identifier of owning process */
+		PVRSRV_FENCE hSWFence;         /*!< Unique identifier for the SWFence resource */
+		PVRSRV_TIMELINE hSWTimeline;   /*!< Unique identifier for the timeline resource */
+		IMG_UINT64 ui64SyncPtIndex;    /*!< Sync-pt index where this SW timeline has reached */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSWFenceAlloc;
+
+	/*! Data for TYPE_SYNC */
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;         /*!< Identifier of sync resource */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                               /*!< Label or name given to the sync resource */
+	} sSyncAlloc;
+} RGX_HWPERF_HOST_ALLOC_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32AllocType;
+	                                 /*!< This describes the type of the resource
+	                                  allocated in the driver. See
+	                                  RGX_HWPERF_HOST_RESOURCE_TYPE */
+	RGX_HWPERF_HOST_ALLOC_DETAIL RGXFW_ALIGN uAllocDetail;
+	                                 /*!< Union of structures providing further
+	                                  data regarding the resource allocated.
+	                                  Size of data varies with union member that
+	                                  is present, check ``ui32AllocType`` value
+	                                  to decode */
+} RGX_HWPERF_HOST_ALLOC_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_ALLOC_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_ALLOC_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef union
+{
+	/*! Data for TYPE_TIMELINE (*Deprecated*) */
+	struct
+	{
+		IMG_UINT32 uiPid;             /*!< Identifier of owning process */
+		IMG_UINT64 ui64Timeline_UID1; /*!< Unique identifier for the timeline resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sTimelineDestroy;
+
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_UINT64 ui64Fence_UID;     /*!< Unique identifier for the fence resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. */
+	} sFenceDestroy;
+
+	/*! Data for TYPE_SYNC_CP */
+	struct
+	{
+		IMG_UINT32 ui32CheckPt_FWAddr;  /*!< Unique identifier for the check point resource */
+	} sSyncCheckPointFree;
+
+	/*! Data for TYPE_SYNC */
+	struct
+	{
+		IMG_UINT32 ui32FWAddr;        /*!< Unique identifier for the sync resource */
+	} sSyncFree;
+} RGX_HWPERF_HOST_FREE_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32FreeType;
+	                              /*!< This describes the type of the resource
+	                               freed or released by the driver. See
+	                               RGX_HWPERF_HOST_RESOURCE_TYPE */
+	RGX_HWPERF_HOST_FREE_DETAIL uFreeDetail;
+	                              /*!< Union of structures providing further data
+	                               regarding the resource freed. Size of data
+	                               varies with union member that is present,
+	                               check ``ui32FreeType`` value to decode */
+	IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+} RGX_HWPERF_HOST_FREE_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_FREE_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_FREE_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef struct
+{
+	IMG_UINT64 ui64CRTimestamp;   /*!< CR timer value from the latest entry of
+	                               the time domains correlation table */
+	IMG_UINT64 ui64OSTimestamp;   /*!< OS timestamp from the latest entry of the
+	                               time domains correlation table */
+	IMG_UINT32 ui32ClockSpeed;    /*!< GPU clock speed from the latest entry of
+	                               the time domains correlation table */
+	IMG_UINT32 ui32Padding;       /*!< Reserved, align structure size to 8 bytes */
+} RGX_HWPERF_HOST_CLK_SYNC_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_CLK_SYNC_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_CLK_SYNC_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef union
+{
+	/*! Data for TYPE_FENCE_PVR */
+	struct
+	{
+		IMG_UINT64 ui64NewFence_UID;  /*!< Unique identifier for the new merged fence
+		                               resource that has been created */
+		IMG_UINT64 ui64InFence1_UID;  /*!< Unique identifier for the fence resource */
+		IMG_UINT64 ui64InFence2_UID;  /*!< Unique identifier of the check point backing
+		                               the fence on the GPU */
+		IMG_CHAR acName[PVRSRV_SYNC_NAME_LENGTH];
+		                              /*!< Label or name given to the sync resource */
+		IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	} sFenceMerge;
+} RGX_HWPERF_HOST_MODIFY_DETAIL;
+
+typedef struct
+{
+	RGX_HWPERF_HOST_RESOURCE_TYPE ui32ModifyType;
+	                                /*!< Describes the type of the resource
+	                                 modified by the driver. See
+	                                 RGX_HWPERF_HOST_RESOURCE_TYPE */
+
+	RGX_HWPERF_HOST_MODIFY_DETAIL uModifyDetail;
+	                                /*!< Union of structures providing further
+	                                 data regarding the resource modified.
+	                                 Size of data varies with union member that
+	                                 is present.
+	                                 Check ``uiModifyType`` value to decode */
+} RGX_HWPERF_HOST_MODIFY_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_MODIFY_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_MODIFY_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef enum
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED = 0, /*!< Invalid */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_OK,            /*!< Device OK */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_NOT_RESPONDING,/*!< Device not responding to requests */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_DEAD,          /*!< Device not responding */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_FAULT,         /*!< Device has faulted */
+
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_LAST
+} RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS;
+
+typedef enum
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_UNDEFINED = 0,     /*!< Invalid */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_NONE,              /*!< No underlying health reason. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_ASSERTED,          /*!< Device has asserted. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_POLL_FAILING,      /*!< Device poll has failed. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_TIMEOUTS,          /*!< Device timeout has fired. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_CORRUPT,     /*!< Queue has become corrupt. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_STALLED,     /*!< Queue has stalled. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_IDLING,            /*!< Device is idling. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_RESTARTING,        /*!< Device restarting. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS,/*!< Interrupts have been discarded. */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_PCI_ERROR,         /*!< PCI error detected. */
+
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_LAST
+} RGX_HWPERF_HOST_DEVICE_HEALTH_REASON;
+
+/*! Data for device status event */
+typedef struct
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS eDeviceHealthStatus;
+	                                 /*!< Device's health status */
+	RGX_HWPERF_HOST_DEVICE_HEALTH_REASON eDeviceHealthReason;
+	                                 /*!< Reason for device's health status */
+} RGX_HWPERF_HOST_DEVICE_HEALTH;
+
+/*! RGX_HWPERF_DEV_INFO_EV values */
+typedef enum
+{
+	RGX_HWPERF_DEV_INFO_EV_HEALTH,      /*!< Health sub-event */
+	RGX_HWPERF_DEV_INFO_EV_FEATURES,    /*!< Features sub-event */
+
+	RGX_HWPERF_DEV_INFO_EV_LAST         /*!< Last enumeration value */
+} RGX_HWPERF_DEV_INFO_EV;
+
+/*! RGX_HWPERF_HOST_DEV_INFO_DETAIL is a union of structures providing
+ *  further data regarding the device's status
+ */
+typedef union
+{
+	RGX_HWPERF_HOST_DEVICE_HEALTH sDeviceStatus;    /*!< Device health status */
+	RGX_HWPERF_BVNC               sBVNC;            /*!< Device features */
+} RGX_HWPERF_HOST_DEV_INFO_DETAIL;
+
+/*! RGX_HWPERF_HOST_DEV_INFO_DATA contains device health status information */
+typedef struct
+{
+	IMG_UINT32                      ui32Padding;
+	                            /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_DEV_INFO_EV          eEvType;
+	                            /*!< Type of the sub-event. See
+	                              RGX_HWPERF_DEV_INFO_EV */
+	RGX_HWPERF_HOST_DEV_INFO_DETAIL uDevInfoDetail;
+	                            /*!< Union of structures providing further data
+	                              regarding the device's status. Size of data
+	                              varies with union member that is present,
+	                              check ``eEvType`` value to decode */
+} RGX_HWPERF_HOST_DEV_INFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_DEV_INFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			 "sizeof(RGX_HWPERF_HOST_DEV_INFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! RGX_HWPERF_INFO_EV event subtype for RGX_HWPERF_HOST_INFO_DATA events */
+typedef enum
+{
+	RGX_HWPERF_INFO_EV_RESERVED_0,
+	RGX_HWPERF_INFO_EV_MEM64_USAGE,     /*!< 64-bit Memory usage event */
+	RGX_HWPERF_INFO_EV_LAST             /*!< End of enumeration */
+} RGX_HWPERF_INFO_EV;
+
+/*! RGX_HWPERF_HOST_INFO_DETAIL contains the data payload for the
+ * RGX_HWPERF_HOST_INFO_DATA event.
+ */
+typedef union
+{
+	/*! Host Memory usage statistics */
+	struct
+	{
+		IMG_UINT64 ui64TotalMemoryUsage;   /*!< Total memory usage (bytes) */
+		/*! Detailed memory usage */
+		struct _RGX_HWPERF_HOST_INFO_PER_PROC_USAGE_
+		{
+			IMG_UINT32 ui32Pid;              /*!< Process ID */
+			IMG_UINT32 ui32Padding;          /*!< Padding */
+			IMG_UINT64 ui64KernelMemUsage;   /*!< Kernel memory usage (bytes) */
+			IMG_UINT64 ui64GraphicsMemUsage; /*!< GPU memory usage (bytes) */
+		} sPerProcessUsage[IMG_FLEX_ARRAY_MEMBER];
+	} sMemUsageStats;
+} RGX_HWPERF_HOST_INFO_DETAIL;
+
+/*! RGX_HWPERF_HOST_INFO_DATA. Host Info data event payload contains device
+ * memory usage information.
+ */
+typedef struct
+{
+	IMG_UINT32 ui32Padding;       /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_INFO_EV eEvType;   /*!< Type of subevent. See RGX_HWPERF_INFO_EV */
+	RGX_HWPERF_HOST_INFO_DETAIL uInfoDetail;
+	                              /*!< Union of structures providing further data
+	                               regarding memory usage. Size varies with union
+	                               member that is present, check ``eEvType``
+	                               value to decode */
+} RGX_HWPERF_HOST_INFO_DATA;
+
+/* Payload size must be multiple of 8 bytes to align start of next packet. */
+static_assert((sizeof(RGX_HWPERF_HOST_INFO_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_INFO_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! FENCE_WAIT_TYPE definitions */
+typedef enum
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN = 0,    /*!< Begin */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END,          /*!< End */
+
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_LAST,         /*!< Do not use */
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE;
+
+/*! FENCE_WAIT_RESULT definitions */
+typedef enum
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_INVALID = 0, /*!< Invalid */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_TIMEOUT,     /*!< Timed Out */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_PASSED,      /*!< Passed */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_ERROR,       /*!< Errored */
+
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT_LAST,        /*!< Do not use */
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT;
+
+/*! FENCE_WAIT_DETAIL Event Payload */
+typedef union
+{
+/*! Data for SYNC_FENCE_WAIT_TYPE_BEGIN */
+	struct
+	{
+		IMG_UINT32 ui32TimeoutInMs;                     /*!< Wait timeout (ms) */
+	} sBegin;
+
+	/*! Data for SYNC_FENCE_WAIT_TYPE_END */
+	struct
+	{
+		RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT eResult; /*!< Wait result */
+	} sEnd;
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DETAIL;
+
+/*! RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA Event Payload. This data structure
+ * is received whenever the host driver handles a wait for sync event request.
+ */
+typedef struct
+{
+	IMG_PID uiPID;          /*!< Identifier of the owning process */
+	PVRSRV_FENCE hFence;    /*!< Unique identifier for the fence resource */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType;
+	                        /*!< Type of the subevent, see
+	                         RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DETAIL uDetail;
+	                        /*!< Union of structures providing further data
+	                         regarding device's status. Size of data varies with
+	                         union member that is present, check ``eType`` value
+	                         to decode */
+
+} RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA;
+
+static_assert((sizeof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+/*! RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA.
+ * Software Timeline Advanced Event Payload. This data structure is received
+ * whenever the host driver processes a Software Timeline Advanced event.
+ */
+typedef struct
+{
+	IMG_PID uiPID;                /*!< Identifier of the owning process */
+	PVRSRV_TIMELINE hTimeline;    /*!< Unique identifier for the timeline resource */
+	IMG_UINT64 ui64SyncPtIndex;   /*!< Index of the sync point to which the
+	                               timeline has advanced */
+
+} RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA;
+
+static_assert((sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA) & (PVRSRVTL_PACKET_ALIGNMENT-1U)) == 0U,
+			  "sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA) must be a multiple PVRSRVTL_PACKET_ALIGNMENT");
+
+typedef enum
+{
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_INVALID = 0,  /*!< Invalid */
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME, /*!< Process Name */
+
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE_LAST,         /*!< Do not use */
+} RGX_HWPERF_HOST_CLIENT_INFO_TYPE;
+
+typedef struct
+{
+	IMG_PID uiClientPID; /*!< Client process identifier */
+	IMG_UINT32 ui32Length;  /*!< Number of bytes present in ``acName`` */
+	IMG_CHAR acName[IMG_FLEX_ARRAY_MEMBER]; /*!< Process name string, null terminated */
+} RGX_HWPERF_HOST_CLIENT_PROC_NAME;
+
+#define RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen) \
+	((IMG_UINT32)(offsetof(RGX_HWPERF_HOST_CLIENT_PROC_NAME, acName) + (ui32NameLen)))
+
+typedef union
+{
+	struct
+	{
+		IMG_UINT32 ui32Count; /*!< Number of elements in ``asProcNames`` */
+		RGX_HWPERF_HOST_CLIENT_PROC_NAME asProcNames[IMG_FLEX_ARRAY_MEMBER];
+	} sProcName;
+} RGX_HWPERF_HOST_CLIENT_INFO_DETAIL;
+
+typedef struct
+{
+	IMG_UINT32 uiReserved; /*!< Reserved. Align structure size to 8 bytes */
+	RGX_HWPERF_HOST_CLIENT_INFO_TYPE eType;
+	                        /*!< Type of the subevent, see
+	                         RGX_HWPERF_HOST_CLIENT_INFO_TYPE */
+	RGX_HWPERF_HOST_CLIENT_INFO_DETAIL uDetail;
+	                        /*!< Union of structures. Size of data
+	                         varies with union member that is present,
+	                         check ``eType`` value to decode */
+} RGX_HWPERF_HOST_CLIENT_INFO_DATA;
+
+/*! This type is a union of packet payload data structures associated with
+ * various FW and Host events */
+typedef union
+{
+	RGX_HWPERF_FW_DATA             sFW;           /*!< Firmware event packet data,
+	                                               events ``0x01-0x06`` */
+	RGX_HWPERF_HW_DATA             sHW;           /*!< Hardware event packet data,
+	                                               events ``0x07-0x19``, ``0x28-0x29``
+	                                               See RGX_HWPERF_HW_DATA */
+	RGX_HWPERF_GPU_WORK_PERIOD_DATA sGPUWorkPeriod;
+
+	RGX_HWPERF_CLKS_CHG_DATA       sCLKSCHG;      /*!< Clock change event packet
+	                                               data, events ``0x1A`` */
+	RGX_HWPERF_GPU_STATE_CHG_DATA  sGPUSTATECHG;  /*!< GPU utilisation state
+	                                               change event packet data,
+	                                               events ``0x1B`` */
+	RGX_HWPERF_PWR_EST_DATA        sPWREST;       /*!< Power estimate event
+	                                               packet data,
+	                                               events ``0x20-0x22`` */
+	RGX_HWPERF_PWR_CHG_DATA        sPWR;          /*!< Power event packet data,
+	                                               events ``0x23`` */
+	RGX_HWPERF_CSW_DATA            sCSW;          /*!< Context switch packet data,
+	                                               events ``0x30-0x31`` */
+	RGX_HWPERF_DVFS_DATA           sDVFS;         /*!< DVFS activity data,
+	                                               events ``0x32`` */
+	RGX_HWPERF_UFO_DATA            sUFO;          /*!< UFO data, events ``0x38`` */
+	RGX_HWPERF_FWACT_DATA          sFWACT;        /*!< Firmware activity event
+	                                               packet data,
+	                                               events ``0x39`` */
+	/* */
+	RGX_HWPERF_HOST_ENQ_DATA       sENQ;          /*!< Host ENQ data,
+	                                               events ``0x01`` (Host) */
+	RGX_HWPERF_HOST_UFO_DATA       sHUFO;         /*!< Host UFO data,
+	                                               events ``0x02`` (Host) */
+	RGX_HWPERF_HOST_ALLOC_DATA     sHALLOC;       /*!< Host Alloc data,
+	                                               events ``0x03`` (Host) */
+	RGX_HWPERF_HOST_CLK_SYNC_DATA  sHCLKSYNC;     /*!< Host CLK_SYNC data,
+	                                               events ``0x04`` (Host) */
+	RGX_HWPERF_HOST_FREE_DATA      sHFREE;        /*!< Host Free data,
+	                                               events ``0x05`` (Host) */
+	RGX_HWPERF_HOST_MODIFY_DATA    sHMOD;         /*!< Host Modify data,
+	                                               events ``0x06`` (Host) */
+	RGX_HWPERF_HOST_DEV_INFO_DATA  sHDEVINFO;     /*!< Host device info data,
+	                                               events ``0x07`` (Host) */
+	RGX_HWPERF_HOST_INFO_DATA      sHINFO;        /*!< Host info data,
+	                                               events ``0x08`` (Host) */
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA sWAIT;   /*!< Host fence-wait data,
+	                                               events ``0x09`` (Host) */
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA sSWTLADV; /*!< Host SW-timeline advance
+	                                               data, events ``0x0A`` (Host) */
+	RGX_HWPERF_HOST_CLIENT_INFO_DATA sHClientInfo; /*!< Host client info,
+	                                               events ``0x0B`` (Host) */
+} RGX_HWPERF_V2_PACKET_DATA, *RGX_PHWPERF_V2_PACKET_DATA;
+
+RGX_FW_STRUCT_SIZE_ASSERT(RGX_HWPERF_V2_PACKET_DATA);
+
+#define RGX_HWPERF_GET_PACKET_DATA(_packet_addr) ((RGX_PHWPERF_V2_PACKET_DATA) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR))))
+
+#define RGX_HWPERF_GET_DVFS_EVENT_TYPE_PTR(_packet_addr)	\
+	((RGX_HWPERF_DVFS_EV*) (IMG_OFFSET_ADDR((_packet_addr), sizeof(RGX_HWPERF_V2_PACKET_HDR) + offsetof(RGX_HWPERF_DVFS_DATA,eEventType))))
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGX_HWPERF_COMMON_H_ */
+
+/******************************************************************************
+ End of file
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_hwperf_table.c b/drivers/gpu/drm/img-rogue/rgx_hwperf_table.c
new file mode 100644
index 000000000000..78d731c98eb6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_hwperf_table.c
@@ -0,0 +1,629 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance counter table
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance counters table
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(__KERNEL__)
+#include "rgxdefs_km.h"
+#else
+#include "rgxdefs.h"
+#endif
+#include "rgx_hwperf_table.h"
+
+/* Includes needed for PVRSRVKM (Server) context */
+#	include "rgx_bvnc_defs_km.h"
+#	if defined(__KERNEL__)
+#		include "rgxdevice.h"
+#	endif
+
+/* Shared compile-time context ASSERT macro */
+#if defined(RGX_FIRMWARE)
+#	include "rgxfw_utils.h"
+/* firmware context */
+#	define DBG_ASSERT(_c) RGXFW_ASSERT((_c))
+#else
+#	include "pvr_debug.h"
+/* host client/server context */
+#	define DBG_ASSERT(_c) PVR_ASSERT((_c))
+#endif
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered()
+
+ Referenced in gasCntBlkTypeModel[] table below and only called from
+ RGX_FIRMWARE run-time context. Therefore compile time configuration is used.
+ *****************************************************************************/
+
+#if defined(RGX_FIRMWARE) && defined(RGX_FEATURE_PERFBUS)
+#	include "rgxfw_pow.h"
+#	include "rgxfw_utils.h"
+
+static bool rgxfw_hwperf_pow_st_direct(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	PVR_UNREFERENCED_PARAMETER(eBlkType);
+	PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+	/* S6XT: TA, TORNADO */
+	return true;
+#else
+	/* S6  : TA, HUB, RASTER (RASCAL) */
+	return rgxfw_pow_is_rd_on();
+#endif
+}
+
+/* Only use conditional compilation when counter blocks appear in different
+ * islands for different Rogue families.
+ */
+static bool rgxfw_hwperf_pow_st_indirect(RGX_HWPERF_CNTBLK_ID eBlkType, IMG_UINT8 ui8UnitId)
+{
+	IMG_UINT32 ui32NumDustsEnabled = rgxfw_pow_get_enabled_units();
+
+	if (rgxfw_pow_is_rd_on() && (ui32NumDustsEnabled > 0U))
+	{
+#if defined(RGX_FEATURE_DYNAMIC_DUST_POWER)
+		IMG_UINT32 ui32NumUscEnabled = ui32NumDustsEnabled*2U;
+
+		switch (eBlkType)
+		{
+		case RGX_CNTBLK_ID_TPU_MCU0:                   /* S6 and S6XT */
+			if (ui8UnitId >= ui32NumDustsEnabled)
+			{
+				return false;
+			}
+			break;
+		case RGX_CNTBLK_ID_USC0:                       /* S6, S6XT, S7 */
+		case RGX_CNTBLK_ID_PBE0:                       /* S7, PBE2_IN_XE */
+			/* Handle single cluster cores */
+			if (ui8UnitId >= ((ui32NumUscEnabled > RGX_FEATURE_NUM_CLUSTERS) ? RGX_FEATURE_NUM_CLUSTERS : ui32NumUscEnabled))
+			{
+				return false;
+			}
+			break;
+		case RGX_CNTBLK_ID_BLACKPEARL0:                /* S7 */
+		case RGX_CNTBLK_ID_RASTER0:                    /* S6XT */
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+		case RGX_CNTBLK_ID_TEXAS0:                     /* S6XT */
+#endif
+			if (ui8UnitId >= (RGX_REQ_NUM_PHANTOMS(ui32NumUscEnabled)))
+			{
+				return false;
+			}
+			break;
+		default:
+			RGXFW_ASSERT(false); /* should never get here, table error */
+			break;
+		}
+#else
+		/* Always true, no fused DUSTs, all powered so do not check unit */
+		PVR_UNREFERENCED_PARAMETER(eBlkType);
+		PVR_UNREFERENCED_PARAMETER(ui8UnitId);
+#endif
+	}
+	else
+	{
+		return false;
+	}
+	return true;
+}
+
+#else /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+# define rgxfw_hwperf_pow_st_direct   ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+# define rgxfw_hwperf_pow_st_indirect ((PFN_RGXFW_HWPERF_CNTBLK_POWERED)NULL)
+
+#endif /* !defined(RGX_FIRMWARE) || !defined(RGX_FEATURE_PERFBUS) */
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPowered() end
+ *****************************************************************************/
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() start
+
+ Referenced in gasCntBlkTypeModel[] table below and called from all build
+ contexts:
+ RGX_FIRMWARE, PVRSRVCTL (UM) and PVRSRVKM (Server).
+
+ Therefore each function has two implementations, one for compile time and one
+ run time configuration depending on the context. The functions will inform the
+ caller whether this block is valid for this particular RGX device. Other
+ run-time dependent data is returned in psRtInfo for the caller to use.
+ *****************************************************************************/
+
+/* Used for block types: USC */
+static IMG_BOOL rgx_hwperf_blk_present_perfbus(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_USC0);
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+		{
+			psRtInfo->ui32NumUnits = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) ? RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) : 0;
+			psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+			return IMG_TRUE;
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+# if defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: Direct RASTERISATION, HUB */
+static IMG_BOOL rgx_hwperf_blk_present_not_clustergrouping(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_HUB));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if ((!RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING)) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS)))
+		{
+			psRtInfo->ui32NumUnits = 1;
+			psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+			return IMG_TRUE;
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+#if defined(__KERNEL__) /* Server context */
+static IMG_UINT32 rgx_units_indirect_by_phantom(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_INDIRECT_BY_PHANTOM */
+	return ((psFeatCfg->ui64Features & RGX_FEATURE_CLUSTER_GROUPING_BIT_MASK) == 0) ? 1
+			: (psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX]+3)/4;
+}
+
+static IMG_UINT32 rgx_units_phantom_indirect_by_dust(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST */
+	return MAX((psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX]>>1),1);
+}
+
+static IMG_UINT32 rgx_units_phantom_indirect_by_cluster(const PVRSRV_DEVICE_FEATURE_CONFIG *psFeatCfg)
+{
+	/* Run-time math for RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER */
+	return psFeatCfg->ui32FeaturesValues[RGX_FEATURE_NUM_CLUSTERS_IDX];
+}
+#endif /* defined(__KERNEL__) */
+
+/* Used for block types: TORNADO, TEXAS, Indirect RASTERISATION */
+static IMG_BOOL rgx_hwperf_blk_present_xttop(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TORNADO) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER0));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TORNADO)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TEXAS_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_RASTER0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS)
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: JONES, TPU_MCU, TEXAS, BLACKPEARL, PBE */
+static IMG_BOOL rgx_hwperf_blk_present_s7top(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_JONES) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_BLACKPEARL0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0));
+
+#if defined(__KERNEL__) /* Server context */
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE_IDX)
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, S7_TOP_INFRASTRUCTURE))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TPU_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TEXAS0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TEXAS3_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_BLACKPEARL0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_indirect_by_phantom(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_cluster(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_JONES)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+		}
+	}
+#endif
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return IMG_TRUE;
+# else
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+/* Used for block types: TA, TPU_MCU. Also PBE when PBE2_IN_XE is present */
+static IMG_BOOL rgx_hwperf_blk_present_not_s7top(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	DBG_ASSERT(psBlkTypeDesc != NULL);
+	DBG_ASSERT((psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TA) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0) ||
+			(psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0));
+
+#if defined(__KERNEL__) /* Server context */
+	PVR_ASSERT(pvDev_km != NULL);
+	PVR_ASSERT(pvRtInfo != NULL);
+	{
+		RGX_HWPERF_CNTBLK_RT_INFO *psRtInfo = (RGX_HWPERF_CNTBLK_RT_INFO *) pvRtInfo;
+		const PVRSRV_RGXDEV_INFO *psDevInfo = (const PVRSRV_RGXDEV_INFO *)pvDev_km;
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+		{
+			if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TA)
+			{
+				psRtInfo->ui32NumUnits = 1;
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+			{
+				if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+				{
+					/* PBE counters are not present on this config */
+					return IMG_FALSE;
+				}
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_cluster(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = psBlkTypeDesc->ui32IndirectReg;
+				return IMG_TRUE;
+			}
+			else if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_TPU_MCU0)
+			{
+				psRtInfo->ui32NumUnits = rgx_units_phantom_indirect_by_dust(&psDevInfo->sDevFeatureCfg);
+				psRtInfo->ui32IndirectReg = RGX_CR_TPU_MCU_L0_PERF_INDIRECT;
+				return IMG_TRUE;
+			}
+		}
+	}
+#else /* FW context */
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+# if !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS)
+#  if !defined(RGX_FEATURE_PBE2_IN_XE)
+	if (psBlkTypeDesc->ui32CntBlkIdBase == RGX_CNTBLK_ID_PBE0)
+	{
+		/* No support for PBE counters without PBE2_IN_XE */
+		return IMG_FALSE;
+	}
+#  endif
+	return IMG_TRUE;
+# endif
+#endif
+	return IMG_FALSE;
+}
+
+static IMG_BOOL rgx_hwperf_blk_present_check_s7top_or_not(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+#if defined(__KERNEL__)
+	return (rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo)
+	     || rgx_hwperf_blk_present_not_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo));
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+
+#elif defined(RGX_FEATURE_PBE2_IN_XE) || defined(RGX_FEATURE_PERFBUS)
+	return rgx_hwperf_blk_present_not_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	return IMG_FALSE;
+#endif
+}
+
+static IMG_BOOL rgx_hwperf_blk_present_check_s7top_or_xttop(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+#if defined(__KERNEL__)
+	return (rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo)
+	     || rgx_hwperf_blk_present_xttop(psBlkTypeDesc, pvDev_km, pvRtInfo));
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_s7top(psBlkTypeDesc, pvDev_km, pvRtInfo);
+
+#elif defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+	return rgx_hwperf_blk_present_xttop(psBlkTypeDesc, pvDev_km, pvRtInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+	return IMG_FALSE;
+#endif
+}
+
+#if !defined(__KERNEL__) /* Firmware or User-mode context */
+static IMG_BOOL rgx_hwperf_blk_present_false(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc, const void *pvDev_km, void *pvRtInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(psBlkTypeDesc);
+	PVR_UNREFERENCED_PARAMETER(pvDev_km);
+	PVR_UNREFERENCED_PARAMETER(pvRtInfo);
+
+	/* Some functions not used on some BVNCs, silence compiler warnings */
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_perfbus);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_not_clustergrouping);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_xttop);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_s7top);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_not_s7top);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_check_s7top_or_not);
+	PVR_UNREFERENCED_PARAMETER(rgx_hwperf_blk_present_check_s7top_or_xttop);
+
+	return IMG_FALSE;
+}
+
+/* Used to instantiate a null row in the block type model table below where the
+ * block is not supported for a given build BVNC in firmware/user mode context.
+ * This is needed as the blockid to block type lookup uses the table as well
+ * and clients may try to access blocks not in the hardware. */
+#define RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(_blkid) {_blkid, 0, 0, 0, 0, 0, 0, 0, 0, #_blkid, NULL, rgx_hwperf_blk_present_false}
+
+#endif
+
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL struct PFNs pfnIsBlkPresent() end
+ *****************************************************************************/
+
+#if defined(__KERNEL__) /* Values will be calculated at run-time */
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_NUM_BLOCK_UNITS_RUNTIME_CALC
+#define RGX_INDIRECT_REG_TEXAS 0xFFFFFFFF
+#define RGX_INDIRECT_REG_TPU 0xFFFFFFFF
+
+#elif defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST
+#define RGX_INDIRECT_REG_TEXAS RGX_CR_TEXAS3_PERF_INDIRECT
+#define RGX_INDIRECT_REG_TPU RGX_CR_TPU_PERF_INDIRECT
+
+#else
+
+#if defined(RGX_FEATURE_PERFBUS)
+#define RGX_INDIRECT_REG_TPU RGX_CR_TPU_MCU_L0_PERF_INDIRECT
+#endif
+
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+#define RGX_HWPERF_NUM_BLOCK_UNITS RGX_HWPERF_INDIRECT_BY_PHANTOM
+#define RGX_INDIRECT_REG_TEXAS RGX_CR_TEXAS_PERF_INDIRECT
+#endif
+
+#endif
+
+
+/*****************************************************************************
+ RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] table
+
+ This table holds the entries for the performance counter block type model.
+ Where the block is not present on an RGX device in question the
+ pfnIsBlkPresent() returns false, if valid and present it returns true.
+ Columns in the table with a ** indicate the value is a default and the
+ value returned in RGX_HWPERF_CNTBLK_RT_INFO when calling pfnIsBlkPresent()
+ should be used at runtime by the caller. These columns are only valid for
+ compile time BVNC configured contexts.
+
+ Order of table rows must match order of counter block IDs in the enumeration
+ RGX_HWPERF_CNTBLK_ID.
+ *****************************************************************************/
+
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL gasCntBlkTypeModel[] =
+{
+		/*   ui32CntBlkIdBase,         ui32IndirectReg,                  ui32PerfReg,                  ui32Select0BaseReg,                    ui32Counter0BaseReg                   ui8NumCounters,  ui32NumUnits**,                  ui8SelectRegModeShift, ui8SelectRegOffsetShift,            pfnIsBlkPowered               pfnIsBlkPresent
+		 *                                                                                                                                                                                                                                                   pszBlockNameComment,  */
+		/*RGX_CNTBLK_ID_TA*/
+#if !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_TA,       0, /* direct */                RGX_CR_TA_PERF,             RGX_CR_TA_PERF_SELECT0,              RGX_CR_TA_PERF_COUNTER_0,             4,              1,                              21,                  3,  "RGX_CR_TA_PERF",              rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TA),
+#endif
+
+		/*RGX_CNTBLK_ID_RASTER*/
+#if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_RASTER,   0, /* direct */                RGX_CR_RASTERISATION_PERF,  RGX_CR_RASTERISATION_PERF_SELECT0,   RGX_CR_RASTERISATION_PERF_COUNTER_0,  4,              1,                              21,                  3,  "RGX_CR_RASTERISATION_PERF",   rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_clustergrouping },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_RASTER),
+#endif
+
+		/*RGX_CNTBLK_ID_HUB*/
+#if !defined(RGX_FEATURE_CLUSTER_GROUPING) && defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_HUB,      0, /* direct */                RGX_CR_HUB_BIFPMCACHE_PERF, RGX_CR_HUB_BIFPMCACHE_PERF_SELECT0,  RGX_CR_HUB_BIFPMCACHE_PERF_COUNTER_0, 4,              1,                              21,                  3,  "RGX_CR_HUB_BIFPMCACHE_PERF",  rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_not_clustergrouping },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_HUB),
+#endif
+
+		/*RGX_CNTBLK_ID_TORNADO*/
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_TORNADO,  0, /* direct */                RGX_CR_TORNADO_PERF,        RGX_CR_TORNADO_PERF_SELECT0,         RGX_CR_TORNADO_PERF_COUNTER_0,        4,              1,                              21,                  4,  "RGX_CR_TORNADO_PERF",         rgxfw_hwperf_pow_st_direct,   rgx_hwperf_blk_present_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TORNADO),
+#endif
+
+		/*RGX_CNTBLK_ID_JONES*/
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_JONES,   0, /* direct */                 RGX_CR_JONES_PERF,          RGX_CR_JONES_PERF_SELECT0,           RGX_CR_JONES_PERF_COUNTER_0,          4,              1,                              21,                  3,  "RGX_CR_JONES_PERF",           rgxfw_hwperf_pow_st_direct,    rgx_hwperf_blk_present_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_JONES),
+#endif
+
+		/*RGX_CNTBLK_ID_TPU_MCU0*/
+#if defined(__KERNEL__) || (defined(RGX_FEATURE_PERFBUS) && !defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
+		{RGX_CNTBLK_ID_TPU_MCU0, RGX_INDIRECT_REG_TPU,      RGX_CR_TPU_MCU_L0_PERF,   RGX_CR_TPU_MCU_L0_PERF_SELECT0,     RGX_CR_TPU_MCU_L0_PERF_COUNTER_0,     4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_DUST,    21,          3,  "RGX_CR_TPU_MCU_L0_PERF",      rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_not },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TPU_MCU0),
+#endif
+
+		/*RGX_CNTBLK_ID_USC0*/
+#if defined(RGX_FEATURE_PERFBUS) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_USC0,    RGX_CR_USC_PERF_INDIRECT,       RGX_CR_USC_PERF,            RGX_CR_USC_PERF_SELECT0,            RGX_CR_USC_PERF_COUNTER_0,            4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER, 21,          3,  "RGX_CR_USC_PERF",             rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_perfbus },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_USC0),
+#endif
+
+		/*RGX_CNTBLK_ID_TEXAS0*/
+#if defined(__KERNEL__) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE)
+		{RGX_CNTBLK_ID_TEXAS0,  RGX_INDIRECT_REG_TEXAS,      RGX_CR_TEXAS_PERF,          RGX_CR_TEXAS_PERF_SELECT0,          RGX_CR_TEXAS_PERF_COUNTER_0,          6,              RGX_HWPERF_NUM_BLOCK_UNITS,             31,          3,  "RGX_CR_TEXAS_PERF",           rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_TEXAS0),
+#endif
+
+		/*RGX_CNTBLK_ID_RASTER0*/
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_RASTER0, RGX_CR_RASTERISATION_PERF_INDIRECT, RGX_CR_RASTERISATION_PERF, RGX_CR_RASTERISATION_PERF_SELECT0, RGX_CR_RASTERISATION_PERF_COUNTER_0,  4,            RGX_HWPERF_INDIRECT_BY_PHANTOM,         21,          3,  "RGX_CR_RASTERISATION_PERF",   rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_xttop },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_RASTER0),
+#endif
+
+		/*RGX_CNTBLK_ID_BLACKPEARL0*/
+#if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(__KERNEL__)
+		{RGX_CNTBLK_ID_BLACKPEARL0, RGX_CR_BLACKPEARL_PERF_INDIRECT, RGX_CR_BLACKPEARL_PERF, RGX_CR_BLACKPEARL_PERF_SELECT0,    RGX_CR_BLACKPEARL_PERF_COUNTER_0,     6,              RGX_HWPERF_INDIRECT_BY_PHANTOM,         21,          3,  "RGX_CR_BLACKPEARL_PERF",      rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_s7top },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_BLACKPEARL0),
+#endif
+
+		/*RGX_CNTBLK_ID_PBE0*/
+#if defined(__KERNEL__) || defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE) || defined(RGX_FEATURE_PBE2_IN_XE)
+		{RGX_CNTBLK_ID_PBE0,    RGX_CR_PBE_PERF_INDIRECT,        RGX_CR_PBE_PERF,            RGX_CR_PBE_PERF_SELECT0,            RGX_CR_PBE_PERF_COUNTER_0,            4,              RGX_HWPERF_PHANTOM_INDIRECT_BY_CLUSTER, 21,          3,  "RGX_CR_PBE_PERF",             rgxfw_hwperf_pow_st_indirect, rgx_hwperf_blk_present_check_s7top_or_not },
+#else
+		RGXFW_HWPERF_CNTBLK_TYPE_UNSUPPORTED(RGX_CNTBLK_ID_PBE0),
+#endif
+};
+
+
+IMG_INTERNAL IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel)
+{
+	*ppsModel = gasCntBlkTypeModel;
+	return ARRAY_SIZE(gasCntBlkTypeModel);
+}
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_hwperf_table.h b/drivers/gpu/drm/img-rogue/rgx_hwperf_table.h
new file mode 100644
index 000000000000..449885cdcb64
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_hwperf_table.h
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File
+@Title          HWPerf counter table header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Utility functions used internally for HWPerf data retrieval
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_HWPERF_TABLE_H
+#define RGX_HWPERF_TABLE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(__KERNEL__)
+#include "rgxdevice.h"
+#endif
+/*****************************************************************************/
+
+/* Forward declaration */
+typedef struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_ RGXFW_HWPERF_CNTBLK_TYPE_MODEL;
+
+/* Function pointer type for functions to check dynamic power state of
+ * counter block instance. Used only in firmware. */
+typedef bool (*PFN_RGXFW_HWPERF_CNTBLK_POWERED)(
+		RGX_HWPERF_CNTBLK_ID eBlkType,
+		IMG_UINT8 ui8UnitId);
+
+#if defined(__KERNEL__)
+/* Counter block run-time info */
+typedef struct
+{
+	IMG_UINT32 ui32IndirectReg;          /* 0 if direct type otherwise the indirect control register to select indirect unit */
+	IMG_UINT32 ui32NumUnits;             /* Number of instances of this block type in the core */
+} RGX_HWPERF_CNTBLK_RT_INFO;
+#endif
+
+/* Function pointer type for functions to check block is valid and present
+ * on that RGX Device at runtime. It may have compile logic or run-time
+ * logic depending on where the code executes: server, srvinit or firmware.
+ * Values in the psRtInfo output parameter are only valid if true returned.
+ */
+typedef IMG_BOOL (*PFN_RGXFW_HWPERF_CNTBLK_PRESENT)(
+		const struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_* psBlkTypeDesc,
+		const void *pvDev_km,
+		void *pvRtInfo);
+
+/* This structure encodes properties of a type of performance counter block.
+ * The structure is sometimes referred to as a block type descriptor. These
+ * properties contained in this structure represent the columns in the block
+ * type model table variable below. These values vary depending on the build
+ * BVNC and core type.
+ * Each direct block has a unique type descriptor and each indirect group has
+ * a type descriptor.
+ */
+struct RGXFW_HWPERF_CNTBLK_TYPE_MODEL_
+{
+	/* Could use RGXFW_ALIGN_DCACHEL here but then we would waste 40% of the cache line? */
+	IMG_UINT32 ui32CntBlkIdBase;         /* The starting block id for this block type */
+	IMG_UINT32 ui32IndirectReg;          /* 0 if direct type otherwise the indirect control register to select indirect unit */
+	IMG_UINT32 ui32PerfReg;              /* RGX_CR_*_PERF register for this block type */
+	IMG_UINT32 ui32Select0BaseReg;       /* RGX_CR_*_PERF_SELECT0 register for this block type */
+	IMG_UINT32 ui32Counter0BaseReg;      /* RGX_CR_*_PERF_COUNTER_0 register for this block type */
+	IMG_UINT8  ui8NumCounters;          /* Number of counters in this block type */
+	IMG_UINT8  ui8NumUnits;             /* Number of instances of this block type in the core */
+	IMG_UINT8  ui8SelectRegModeShift;   /* Mode field shift value of select registers */
+	IMG_UINT8  ui8SelectRegOffsetShift; /* Interval between select registers, either 8 bytes or 16, hence << 3 or << 4 */
+	const IMG_CHAR *pszBlockNameComment;             /* Name of the PERF register. Used while dumping the perf counters to pdumps */
+	PFN_RGXFW_HWPERF_CNTBLK_POWERED pfnIsBlkPowered; /* A function to determine dynamic power state for the block type */
+	PFN_RGXFW_HWPERF_CNTBLK_PRESENT pfnIsBlkPresent; /* A function to determine presence on RGX Device at run-time */
+};
+
+/*****************************************************************************/
+
+IMG_INTERNAL IMG_UINT32 RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel);
+
+#endif /* RGX_HWPERF_TABLE_H */
+
+/******************************************************************************
+ End of file (rgx_hwperf_table.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_memallocflags.h b/drivers/gpu/drm/img-rogue/rgx_memallocflags.h
new file mode 100644
index 000000000000..e26f42c4f935
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_memallocflags.h
@@ -0,0 +1,58 @@
+/**************************************************************************/ /*!
+@File
+@Title          RGX device specific memory allocation flags
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_MEMALLOCFLAGS_H
+#define RGX_MEMALLOCFLAGS_H
+
+
+/* Include pvrsrv layer header as the flags below are used in the device
+ * field defined in this header inside Services code.
+ * See PVRSRV_MEMALLOCFLAG_DEVICE_FLAGS_MASK */
+#include "pvrsrv_memallocflags.h"
+
+
+/* Device specific MMU flags */
+#define PMMETA_PROTECT          (1U << 0)      /*!< Memory that only the PM and Meta can access */
+#define FIRMWARE_CACHED         (1U << 1)      /*!< Memory that is cached in META/MIPS */
+
+
+#endif /* RGX_MEMALLOCFLAGS_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_meta.h b/drivers/gpu/drm/img-rogue/rgx_meta.h
new file mode 100644
index 000000000000..1e6214c9ba1c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_meta.h
@@ -0,0 +1,389 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX META definitions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX META helper definitions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_META_H)
+#define RGX_META_H
+
+
+/***** The META HW register definitions in the file are updated manually *****/
+
+
+#include "img_defs.h"
+#include "km/rgxdefs_km.h"
+
+
+/******************************************************************************
+* META registers and MACROS
+******************************************************************************/
+#define META_CR_CTRLREG_BASE(T)					(0x04800000U + (0x1000U*(T)))
+
+#define META_CR_TXPRIVEXT						(0x048000E8)
+#define META_CR_TXPRIVEXT_MINIM_EN				(IMG_UINT32_C(0x1) << 7)
+
+#define META_CR_SYSC_JTAG_THREAD				(0x04830030)
+#define META_CR_SYSC_JTAG_THREAD_PRIV_EN		(0x00000004)
+
+#define META_CR_PERF_COUNT0						(0x0480FFE0)
+#define META_CR_PERF_COUNT1						(0x0480FFE8)
+#define META_CR_PERF_COUNT_CTRL_SHIFT			(28)
+#define META_CR_PERF_COUNT_CTRL_MASK			(0xF0000000)
+#define META_CR_PERF_COUNT_CTRL_DCACHEHITS		(IMG_UINT32_C(0x8) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICACHEHITS		(IMG_UINT32_C(0x9) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICACHEMISS		(IMG_UINT32_C(0xA) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_CTRL_ICORE			(IMG_UINT32_C(0xD) << META_CR_PERF_COUNT_CTRL_SHIFT)
+#define META_CR_PERF_COUNT_THR_SHIFT			(24)
+#define META_CR_PERF_COUNT_THR_MASK				(0x0F000000)
+#define META_CR_PERF_COUNT_THR_0				(IMG_UINT32_C(0x1) << META_CR_PERF_COUNT_THR_SHIFT)
+#define META_CR_PERF_COUNT_THR_1				(IMG_UINT32_C(0x2) << META_CR_PERF_COUNT_THR_SHIFT)
+
+#define META_CR_TxVECINT_BHALT					(0x04820500)
+#define META_CR_PERF_ICORE0						(0x0480FFD0)
+#define META_CR_PERF_ICORE1						(0x0480FFD8)
+#define META_CR_PERF_ICORE_DCACHEMISS			(0x8)
+
+#define META_CR_PERF_COUNT(CTRL, THR)			((META_CR_PERF_COUNT_CTRL_##CTRL << META_CR_PERF_COUNT_CTRL_SHIFT) | \
+												 (THR << META_CR_PERF_COUNT_THR_SHIFT))
+
+#define META_CR_TXUXXRXDT_OFFSET				(META_CR_CTRLREG_BASE(0U) + 0x0000FFF0U)
+#define META_CR_TXUXXRXRQ_OFFSET				(META_CR_CTRLREG_BASE(0U) + 0x0000FFF8U)
+
+#define META_CR_TXUXXRXRQ_DREADY_BIT			(0x80000000U)	/* Poll for done */
+#define META_CR_TXUXXRXRQ_RDnWR_BIT				(0x00010000U)	/* Set for read  */
+#define META_CR_TXUXXRXRQ_TX_S					(12)
+#define META_CR_TXUXXRXRQ_RX_S					(4)
+#define META_CR_TXUXXRXRQ_UXX_S					(0)
+
+#define META_CR_TXUIN_ID						(0x0)			/* Internal ctrl regs */
+#define META_CR_TXUD0_ID						(0x1)			/* Data unit regs */
+#define META_CR_TXUD1_ID						(0x2)			/* Data unit regs */
+#define META_CR_TXUA0_ID						(0x3)			/* Address unit regs */
+#define META_CR_TXUA1_ID						(0x4)			/* Address unit regs */
+#define META_CR_TXUPC_ID						(0x5)			/* PC registers */
+
+/* Macros to calculate register access values */
+#define META_CR_CORE_REG(Thr, RegNum, Unit)	(((IMG_UINT32)(Thr)		<< META_CR_TXUXXRXRQ_TX_S) | \
+											 ((IMG_UINT32)(RegNum)	<< META_CR_TXUXXRXRQ_RX_S) | \
+											 ((IMG_UINT32)(Unit)	<< META_CR_TXUXXRXRQ_UXX_S))
+
+#define META_CR_THR0_PC		META_CR_CORE_REG(0, 0, META_CR_TXUPC_ID)
+#define META_CR_THR0_PCX	META_CR_CORE_REG(0, 1, META_CR_TXUPC_ID)
+#define META_CR_THR0_SP		META_CR_CORE_REG(0, 0, META_CR_TXUA0_ID)
+
+#define META_CR_THR1_PC		META_CR_CORE_REG(1, 0, META_CR_TXUPC_ID)
+#define META_CR_THR1_PCX	META_CR_CORE_REG(1, 1, META_CR_TXUPC_ID)
+#define META_CR_THR1_SP		META_CR_CORE_REG(1, 0, META_CR_TXUA0_ID)
+
+#define SP_ACCESS(Thread)	META_CR_CORE_REG(Thread, 0, META_CR_TXUA0_ID)
+#define PC_ACCESS(Thread)	META_CR_CORE_REG(Thread, 0, META_CR_TXUPC_ID)
+
+#define META_CR_COREREG_ENABLE			(0x0000000U)
+#define META_CR_COREREG_STATUS			(0x0000010U)
+#define META_CR_COREREG_DEFR			(0x00000A0U)
+#define META_CR_COREREG_PRIVEXT			(0x00000E8U)
+
+#define META_CR_T0ENABLE_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_ENABLE)
+#define META_CR_T0STATUS_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_STATUS)
+#define META_CR_T0DEFR_OFFSET			(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_DEFR)
+#define META_CR_T0PRIVEXT_OFFSET		(META_CR_CTRLREG_BASE(0U) + META_CR_COREREG_PRIVEXT)
+
+#define META_CR_T1ENABLE_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_ENABLE)
+#define META_CR_T1STATUS_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_STATUS)
+#define META_CR_T1DEFR_OFFSET			(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_DEFR)
+#define META_CR_T1PRIVEXT_OFFSET		(META_CR_CTRLREG_BASE(1U) + META_CR_COREREG_PRIVEXT)
+
+#define META_CR_TXENABLE_ENABLE_BIT		(0x00000001U)   /* Set if running */
+#define META_CR_TXSTATUS_PRIV			(0x00020000U)
+#define META_CR_TXPRIVEXT_MINIM			(0x00000080U)
+
+#define META_MEM_GLOBAL_RANGE_BIT		(0x80000000U)
+
+#define META_CR_TXCLKCTRL          (0x048000B0)
+#define META_CR_TXCLKCTRL_ALL_ON   (0x55111111)
+#define META_CR_TXCLKCTRL_ALL_AUTO (0xAA222222)
+
+
+/******************************************************************************
+* META LDR Format
+******************************************************************************/
+/* Block header structure */
+typedef struct
+{
+	IMG_UINT32	ui32DevID;
+	IMG_UINT32	ui32SLCode;
+	IMG_UINT32	ui32SLData;
+	IMG_UINT16	ui16PLCtrl;
+	IMG_UINT16	ui16CRC;
+
+} RGX_META_LDR_BLOCK_HDR;
+
+/* High level data stream block structure */
+typedef struct
+{
+	IMG_UINT16	ui16Cmd;
+	IMG_UINT16	ui16Length;
+	IMG_UINT32	ui32Next;
+	IMG_UINT32	aui32CmdData[4];
+
+} RGX_META_LDR_L1_DATA_BLK;
+
+/* High level data stream block structure */
+typedef struct
+{
+	IMG_UINT16	ui16Tag;
+	IMG_UINT16	ui16Length;
+	IMG_UINT32	aui32BlockData[4];
+
+} RGX_META_LDR_L2_DATA_BLK;
+
+/* Config command structure */
+typedef struct
+{
+	IMG_UINT32	ui32Type;
+	IMG_UINT32	aui32BlockData[4];
+
+} RGX_META_LDR_CFG_BLK;
+
+/* Block type definitions */
+#define RGX_META_LDR_COMMENT_TYPE_MASK			(0x0010U)
+#define RGX_META_LDR_BLK_IS_COMMENT(X)			((X & RGX_META_LDR_COMMENT_TYPE_MASK) != 0U)
+
+/* Command definitions
+ *  Value   Name            Description
+ *  0       LoadMem         Load memory with binary data.
+ *  1       LoadCore        Load a set of core registers.
+ *  2       LoadMMReg       Load a set of memory mapped registers.
+ *  3       StartThreads    Set each thread PC and SP, then enable threads.
+ *  4       ZeroMem         Zeros a memory region.
+ *  5       Config          Perform a configuration command.
+ */
+#define RGX_META_LDR_CMD_MASK				(0x000FU)
+
+#define RGX_META_LDR_CMD_LOADMEM			(0x0000U)
+#define RGX_META_LDR_CMD_LOADCORE			(0x0001U)
+#define RGX_META_LDR_CMD_LOADMMREG			(0x0002U)
+#define RGX_META_LDR_CMD_START_THREADS		(0x0003U)
+#define RGX_META_LDR_CMD_ZEROMEM			(0x0004U)
+#define RGX_META_LDR_CMD_CONFIG			(0x0005U)
+
+/* Config Command definitions
+ *  Value   Name        Description
+ *  0       Pause       Pause for x times 100 instructions
+ *  1       Read        Read a value from register - No value return needed.
+ *                      Utilises effects of issuing reads to certain registers
+ *  2       Write       Write to mem location
+ *  3       MemSet      Set mem to value
+ *  4       MemCheck    check mem for specific value.
+ */
+#define RGX_META_LDR_CFG_PAUSE			(0x0000)
+#define RGX_META_LDR_CFG_READ			(0x0001)
+#define RGX_META_LDR_CFG_WRITE			(0x0002)
+#define RGX_META_LDR_CFG_MEMSET			(0x0003)
+#define RGX_META_LDR_CFG_MEMCHECK		(0x0004)
+
+
+/******************************************************************************
+* RGX FW segmented MMU definitions
+******************************************************************************/
+/* All threads can access the segment */
+#define RGXFW_SEGMMU_ALLTHRS	(IMG_UINT32_C(0xf) << 8U)
+/* Writable */
+#define RGXFW_SEGMMU_WRITEABLE	(0x1U << 1U)
+/* All threads can access and writable */
+#define RGXFW_SEGMMU_ALLTHRS_WRITEABLE	(RGXFW_SEGMMU_ALLTHRS | RGXFW_SEGMMU_WRITEABLE)
+
+/* Direct map region 8 optionally used for custom mappings - max 8MB */
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_ID			(8U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_ADDR_START	(0x06000000U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM0_MAX_SIZE		(0x00800000U)
+
+/* Direct map region 9 optionally used for custom mappings - max 8MB */
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_ID			(9U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_ADDR_START	(0x06800000U)
+#define RGXFW_SEGMMU_DMAP_CUSTOM1_MAX_SIZE		(0x00800000U)
+
+/* Direct map region 10 used for mapping GPU memory - max 8MB */
+#define RGXFW_SEGMMU_DMAP_GPU_ID			(10U)
+#define RGXFW_SEGMMU_DMAP_GPU_ADDR_START	(0x07000000U)
+#define RGXFW_SEGMMU_DMAP_GPU_MAX_SIZE		(0x00800000U)
+
+/* Segment IDs */
+#define RGXFW_SEGMMU_DATA_ID			(1U)
+#define RGXFW_SEGMMU_BOOTLDR_ID			(2U)
+#define RGXFW_SEGMMU_TEXT_ID			(RGXFW_SEGMMU_BOOTLDR_ID)
+
+/*
+ * SLC caching strategy in S7 and volcanic is emitted through the segment MMU.
+ * All the segments configured through the macro RGXFW_SEGMMU_OUTADDR_TOP are
+ * CACHED in the SLC.
+ * The interface has been kept the same to simplify the code changes.
+ * The bifdm argument is ignored (no longer relevant) in S7 and volcanic.
+ */
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(pers, slc_policy, mmu_ctx)      ((((IMG_UINT64) ((pers)    & 0x3U))  << 52) | \
+                                                                           (((IMG_UINT64) ((mmu_ctx) & 0xFFU)) << 44) | \
+                                                                           (((IMG_UINT64) ((slc_policy) & 0x1U))  << 40))
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(mmu_ctx)      RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(0x3U, 0x0U, mmu_ctx)
+#define RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED(mmu_ctx)    RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC(0x0U, 0x1U, mmu_ctx)
+
+/* To configure the Page Catalog and BIF-DM fed into the BIF for Garten
+ * accesses through this segment
+ */
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC(pc, bifdm)              (((IMG_UINT64)((IMG_UINT64)(pc)    & 0xFU) << 44U) | \
+                                                              ((IMG_UINT64)((IMG_UINT64)(bifdm) & 0xFU) << 40U))
+
+#define RGXFW_SEGMMU_META_BIFDM_ID   (0x7U)
+#if !(defined(__KERNEL__) || defined(TEE_DDK)) && defined(RGX_FEATURE_META)
+#if defined(RGX_FEATURE_SLC_VIVT)
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED    RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_UNCACHED  RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_UNCACHED
+#define RGXFW_SEGMMU_OUTADDR_TOP_META          RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED
+#else
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_CACHED    RGXFW_SEGMMU_OUTADDR_TOP_SLC
+#define RGXFW_SEGMMU_OUTADDR_TOP_SLC_UNCACHED  RGXFW_SEGMMU_OUTADDR_TOP_SLC
+#define RGXFW_SEGMMU_OUTADDR_TOP_META(pc)      RGXFW_SEGMMU_OUTADDR_TOP_SLC(pc, RGXFW_SEGMMU_META_BIFDM_ID)
+#endif
+#endif
+
+/* META segments have 4kB minimum size */
+#define RGXFW_SEGMMU_ALIGN			(0x1000U)
+
+/* Segmented MMU registers (n = segment id) */
+#define META_CR_MMCU_SEGMENTn_BASE(n)			(0x04850000U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_LIMIT(n)			(0x04850004U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_OUTA0(n)			(0x04850008U + ((n)*0x10U))
+#define META_CR_MMCU_SEGMENTn_OUTA1(n)			(0x0485000CU + ((n)*0x10U))
+
+/* The following defines must be recalculated if the Meta MMU segments used
+ * to access Host-FW data are changed
+ * Current combinations are:
+ * - SLC uncached, META cached,   FW base address 0x70000000
+ * - SLC uncached, META uncached, FW base address 0xF0000000
+ * - SLC cached,   META cached,   FW base address 0x10000000
+ * - SLC cached,   META uncached, FW base address 0x90000000
+ */
+#define RGXFW_SEGMMU_DATA_BASE_ADDRESS        (0x10000000U)
+#define RGXFW_SEGMMU_DATA_META_CACHED         (0x0U)
+#define RGXFW_SEGMMU_DATA_META_UNCACHED       (META_MEM_GLOBAL_RANGE_BIT) // 0x80000000
+#define RGXFW_SEGMMU_DATA_META_CACHE_MASK     (META_MEM_GLOBAL_RANGE_BIT)
+/* For non-VIVT SLCs the cacheability of the FW data in the SLC is selected in
+ * the PTEs for the FW data, not in the Meta Segment MMU, which means these
+ * defines have no real effect in those cases
+ */
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED     (0x0U)
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED   (0x60000000U)
+#define RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK (0x60000000U)
+
+/******************************************************************************
+* RGX FW Bootloader defaults
+******************************************************************************/
+#define RGXFW_BOOTLDR_META_ADDR		(0x40000000U)
+#define RGXFW_BOOTLDR_DEVV_ADDR_0	(0xC0000000U)
+#define RGXFW_BOOTLDR_DEVV_ADDR_1	(0x000000E1)
+#define RGXFW_BOOTLDR_DEVV_ADDR		((((IMG_UINT64) RGXFW_BOOTLDR_DEVV_ADDR_1) << 32) | RGXFW_BOOTLDR_DEVV_ADDR_0)
+#define RGXFW_BOOTLDR_LIMIT		(0x1FFFF000)
+#define RGXFW_MAX_BOOTLDR_OFFSET	(0x1000)
+
+/* Bootloader configuration offset is in dwords (512 bytes) */
+#define RGXFW_BOOTLDR_CONF_OFFSET	(0x80)
+
+
+/******************************************************************************
+* RGX META Stack
+******************************************************************************/
+#define RGX_META_STACK_SIZE  (0x1000U)
+
+/******************************************************************************
+ RGX META Core memory
+******************************************************************************/
+/* code and data both map to the same physical memory */
+#define RGX_META_COREMEM_CODE_ADDR   (0x80000000U)
+#define RGX_META_COREMEM_DATA_ADDR   (0x82000000U)
+#define RGX_META_COREMEM_OFFSET_MASK (0x01ffffffU)
+
+#if defined(__KERNEL__) || defined(TEE_DDK)
+#define RGX_META_IS_COREMEM_CODE(A, B)  (((A) >= RGX_META_COREMEM_CODE_ADDR) && ((A) < (RGX_META_COREMEM_CODE_ADDR + (B))))
+#define RGX_META_IS_COREMEM_DATA(A, B)  (((A) >= RGX_META_COREMEM_DATA_ADDR) && ((A) < (RGX_META_COREMEM_DATA_ADDR + (B))))
+#endif
+
+/******************************************************************************
+* 2nd thread
+******************************************************************************/
+#define RGXFW_THR1_PC		(0x18930000)
+#define RGXFW_THR1_SP		(0x78890000)
+
+/******************************************************************************
+* META compatibility
+******************************************************************************/
+
+#define META_CR_CORE_ID			(0x04831000)
+#define META_CR_CORE_ID_VER_SHIFT	(16U)
+#define META_CR_CORE_ID_VER_CLRMSK	(0XFF00FFFFU)
+
+#if !(defined(__KERNEL__) || defined(TEE_DDK)) && defined(RGX_FEATURE_META)
+
+	#if (RGX_FEATURE_META == MTP218)
+	#define RGX_CR_META_CORE_ID_VALUE 0x19
+	#elif (RGX_FEATURE_META == MTP219)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1E
+	#elif (RGX_FEATURE_META == LTP218)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1C
+	#elif (RGX_FEATURE_META == LTP217)
+	#define RGX_CR_META_CORE_ID_VALUE 0x1F
+	#else
+	#error "Unknown META ID"
+	#endif
+#else
+
+	#define RGX_CR_META_MTP218_CORE_ID_VALUE 0x19
+	#define RGX_CR_META_MTP219_CORE_ID_VALUE 0x1E
+	#define RGX_CR_META_LTP218_CORE_ID_VALUE 0x1C
+	#define RGX_CR_META_LTP217_CORE_ID_VALUE 0x1F
+
+#endif
+#define RGXFW_PROCESSOR_META        "META"
+
+
+#endif /* RGX_META_H */
+
+/******************************************************************************
+ End of file (rgx_meta.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgx_mips.h b/drivers/gpu/drm/img-rogue/rgx_mips.h
new file mode 100644
index 000000000000..a80d2f66f0b7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_mips.h
@@ -0,0 +1,406 @@
+/*************************************************************************/ /*!
+@File           rgx_mips.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    RGX MIPS definitions, kernel/user space
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_MIPS_H)
+#define RGX_MIPS_H
+
+/*
+ * Utility defines for memory management
+ */
+#define RGXMIPSFW_LOG2_PAGE_SIZE_4K              (12)
+#define RGXMIPSFW_PAGE_SIZE_4K                   (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+#define RGXMIPSFW_PAGE_MASK_4K                   (RGXMIPSFW_PAGE_SIZE_4K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_64K             (16)
+#define RGXMIPSFW_PAGE_SIZE_64K                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_64K)
+#define RGXMIPSFW_PAGE_MASK_64K                  (RGXMIPSFW_PAGE_SIZE_64K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_256K            (18)
+#define RGXMIPSFW_PAGE_SIZE_256K                 (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_256K)
+#define RGXMIPSFW_PAGE_MASK_256K                 (RGXMIPSFW_PAGE_SIZE_256K - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_1MB             (20)
+#define RGXMIPSFW_PAGE_SIZE_1MB                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_1MB)
+#define RGXMIPSFW_PAGE_MASK_1MB                  (RGXMIPSFW_PAGE_SIZE_1MB - 1)
+#define RGXMIPSFW_LOG2_PAGE_SIZE_4MB             (22)
+#define RGXMIPSFW_PAGE_SIZE_4MB                  (0x1 << RGXMIPSFW_LOG2_PAGE_SIZE_4MB)
+#define RGXMIPSFW_PAGE_MASK_4MB                  (RGXMIPSFW_PAGE_SIZE_4MB - 1)
+#define RGXMIPSFW_LOG2_PTE_ENTRY_SIZE            (2)
+/* log2 page table sizes dependent on FW heap size and page size (for each OS) */
+#define RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K         (RGX_FIRMWARE_HEAP_SHIFT - RGXMIPSFW_LOG2_PAGE_SIZE_4K + RGXMIPSFW_LOG2_PTE_ENTRY_SIZE)
+#define RGXMIPSFW_LOG2_PAGETABLE_SIZE_64K        (RGX_FIRMWARE_HEAP_SHIFT - RGXMIPSFW_LOG2_PAGE_SIZE_64K + RGXMIPSFW_LOG2_PTE_ENTRY_SIZE)
+/* Maximum number of page table pages (both Host and MIPS pages) */
+#define RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES        (4)
+/* Total number of TLB entries */
+#define RGXMIPSFW_NUMBER_OF_TLB_ENTRIES          (16)
+/* "Uncached" caching policy */
+#define RGXMIPSFW_UNCACHED_CACHE_POLICY          (0X00000002U)
+/* "Write-back write-allocate" caching policy */
+#define RGXMIPSFW_WRITEBACK_CACHE_POLICY         (0X00000003)
+/* "Write-through no write-allocate" caching policy */
+#define RGXMIPSFW_WRITETHROUGH_CACHE_POLICY      (0X00000001)
+/* Cached policy used by MIPS in case of physical bus on 32 bit */
+#define RGXMIPSFW_CACHED_POLICY                  (RGXMIPSFW_WRITEBACK_CACHE_POLICY)
+/* Cached policy used by MIPS in case of physical bus on more than 32 bit */
+#define RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT      (RGXMIPSFW_WRITETHROUGH_CACHE_POLICY)
+/* Total number of Remap entries */
+#define RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES        (2 * RGXMIPSFW_NUMBER_OF_TLB_ENTRIES)
+
+
+/*
+ * MIPS EntryLo/PTE format
+ */
+
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_SHIFT     (31U)
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_CLRMSK    (0X7FFFFFFF)
+#define RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN        (0X80000000U)
+
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_SHIFT     (30U)
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_CLRMSK    (0XBFFFFFFF)
+#define RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN        (0X40000000U)
+
+/* Page Frame Number */
+#define RGXMIPSFW_ENTRYLO_PFN_SHIFT              (6)
+#define RGXMIPSFW_ENTRYLO_PFN_ALIGNSHIFT         (12)
+/* Mask used for the MIPS Page Table in case of physical bus on 32 bit */
+#define RGXMIPSFW_ENTRYLO_PFN_MASK               (0x03FFFFC0)
+#define RGXMIPSFW_ENTRYLO_PFN_SIZE               (20)
+/* Mask used for the MIPS Page Table in case of physical bus on more than 32 bit */
+#define RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT   (0x3FFFFFC0U)
+#define RGXMIPSFW_ENTRYLO_PFN_SIZE_ABOVE_32BIT   (24)
+#define RGXMIPSFW_ADDR_TO_ENTRYLO_PFN_RSHIFT     (RGXMIPSFW_ENTRYLO_PFN_ALIGNSHIFT - \
+                                                  RGXMIPSFW_ENTRYLO_PFN_SHIFT)
+
+#define RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT     (3U)
+#define RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK    (0XFFFFFFC7U)
+
+#define RGXMIPSFW_ENTRYLO_DIRTY_SHIFT            (2U)
+#define RGXMIPSFW_ENTRYLO_DIRTY_CLRMSK           (0XFFFFFFFB)
+#define RGXMIPSFW_ENTRYLO_DIRTY_EN               (0X00000004U)
+
+#define RGXMIPSFW_ENTRYLO_VALID_SHIFT            (1U)
+#define RGXMIPSFW_ENTRYLO_VALID_CLRMSK           (0XFFFFFFFD)
+#define RGXMIPSFW_ENTRYLO_VALID_EN               (0X00000002U)
+
+#define RGXMIPSFW_ENTRYLO_GLOBAL_SHIFT           (0U)
+#define RGXMIPSFW_ENTRYLO_GLOBAL_CLRMSK          (0XFFFFFFFE)
+#define RGXMIPSFW_ENTRYLO_GLOBAL_EN              (0X00000001U)
+
+#define RGXMIPSFW_ENTRYLO_DVG                    (RGXMIPSFW_ENTRYLO_DIRTY_EN | \
+                                                  RGXMIPSFW_ENTRYLO_VALID_EN | \
+                                                  RGXMIPSFW_ENTRYLO_GLOBAL_EN)
+#define RGXMIPSFW_ENTRYLO_UNCACHED               (RGXMIPSFW_UNCACHED_CACHE_POLICY << \
+                                                  RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT)
+#define RGXMIPSFW_ENTRYLO_DVG_UNCACHED           (RGXMIPSFW_ENTRYLO_DVG | RGXMIPSFW_ENTRYLO_UNCACHED)
+
+
+/* Remap Range Config Addr Out */
+/* These defines refer to the upper half of the Remap Range Config register */
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_MASK      (0x0FFFFFF0)
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_SHIFT     (4)  /* wrt upper half of the register */
+#define RGXMIPSFW_REMAP_RANGE_ADDR_OUT_ALIGNSHIFT (12)
+#define RGXMIPSFW_ADDR_TO_RR_ADDR_OUT_RSHIFT     (RGXMIPSFW_REMAP_RANGE_ADDR_OUT_ALIGNSHIFT - \
+                                                  RGXMIPSFW_REMAP_RANGE_ADDR_OUT_SHIFT)
+/*
+ * Pages to trampoline problematic physical addresses:
+ *   - RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN : 0x1FC0_0000
+ *   - RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN : 0x1FC0_1000
+ *   - RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN : 0x1FC0_2000
+ *   - (benign trampoline)               : 0x1FC0_3000
+ * that would otherwise be erroneously remapped by the MIPS wrapper
+ * (see "Firmware virtual layout and remap configuration" section below)
+ */
+
+#define RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES       (2)
+#define RGXMIPSFW_TRAMPOLINE_NUMPAGES            (1U << RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES)
+#define RGXMIPSFW_TRAMPOLINE_SIZE                (RGXMIPSFW_TRAMPOLINE_NUMPAGES << RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+#define RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE   (RGXMIPSFW_TRAMPOLINE_LOG2_NUMPAGES + RGXMIPSFW_LOG2_PAGE_SIZE_4K)
+
+#define RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR    (RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN)
+#define RGXMIPSFW_TRAMPOLINE_OFFSET(a)           (a - RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN)
+
+#define RGXMIPSFW_SENSITIVE_ADDR(a)              (RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN == (~((1UL << RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE)-1U) & a))
+
+#define RGXMIPSFW_C0_PAGEMASK_4K                 (0x00001800)
+#define RGXMIPSFW_C0_PAGEMASK_16K                (0x00007800)
+#define RGXMIPSFW_C0_PAGEMASK_64K                (0x0001F800)
+#define RGXMIPSFW_C0_PAGEMASK_256K               (0x0007F800)
+#define RGXMIPSFW_C0_PAGEMASK_1MB                (0x001FF800)
+#define RGXMIPSFW_C0_PAGEMASK_4MB                (0x007FF800)
+
+#if defined(RGX_FEATURE_GPU_MULTICORE_SUPPORT)
+/* GPU_COUNT: number of physical cores in the system
+ * NUM_OF_REGBANKS = GPU_COUNT + 1 //XPU BROADCAST BANK
+ * RGXMIPSFW_REGISTERS_PAGE_SIZE = NUM_OF_REGBANKS * REGBANK_SIZE(64KB) * NUM_OF_OSID(8)
+ * For RGXMIPSFW_REGISTERS_PAGE_SIZE = 4MB, NUM_OF_REGBANKS = 8 so supports upto GPU_COUNT = 7 cores
+ */
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_4MB)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_4MB)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_4MB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED == 1)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_64K)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_64K)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_64KB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED <= 4)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_256K)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_256K)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_256KB)
+#elif (RGX_NUM_DRIVERS_SUPPORTED <= 8)
+#define RGXMIPSFW_C0_PAGEMASK_REGISTERS                    (RGXMIPSFW_C0_PAGEMASK_1MB)
+#define RGXMIPSFW_REGISTERS_PAGE_SIZE                      (RGXMIPSFW_PAGE_SIZE_1MB)
+#define RGXMIPSFW_REGISTERS_REMAP_RANGE_CONFIG_REGION_SIZE (RGX_CR_MIPS_ADDR_REMAP_RANGE_CONFIG_REGION_SIZE_1MB)
+#else
+#error "MIPS TLB invalid params"
+#endif
+
+#define RGXMIPSFW_DECODE_REMAP_CONFIG_REGION_SIZE(r)       ((1U << (((r >> 7) + 1U) << 1U))*0x400)
+
+/*
+ * Firmware virtual layout and remap configuration
+ */
+/*
+ * For each remap region we define:
+ * - the virtual base used by the Firmware to access code/data through that region
+ * - the microAptivAP physical address correspondent to the virtual base address,
+ *   used as input address and remapped to the actual physical address
+ * - log2 of size of the region remapped by the MIPS wrapper, i.e. number of bits from
+ *   the bottom of the base input address that survive onto the output address
+ *   (this defines both the alignment and the maximum size of the remapped region)
+ * - one or more code/data segments within the remapped region
+ */
+
+/* Boot remap setup */
+#define RGXMIPSFW_BOOT_REMAP_VIRTUAL_BASE        (0xBFC00000)
+#define RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN        (0x1FC00000U)
+#define RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_BOOT_NMI_CODE_VIRTUAL_BASE     (RGXMIPSFW_BOOT_REMAP_VIRTUAL_BASE)
+
+/* Data remap setup */
+#define RGXMIPSFW_DATA_REMAP_VIRTUAL_BASE        (0xBFC01000)
+#define RGXMIPSFW_DATA_CACHED_REMAP_VIRTUAL_BASE (0x9FC01000)
+#define RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN        (0x1FC01000U)
+#define RGXMIPSFW_DATA_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_BOOT_NMI_DATA_VIRTUAL_BASE     (RGXMIPSFW_DATA_REMAP_VIRTUAL_BASE)
+
+/* Code remap setup */
+#define RGXMIPSFW_CODE_REMAP_VIRTUAL_BASE        (0x9FC02000)
+#define RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN        (0x1FC02000U)
+#define RGXMIPSFW_CODE_REMAP_LOG2_SEGMENT_SIZE   (12)
+#define RGXMIPSFW_EXCEPTIONS_VIRTUAL_BASE        (RGXMIPSFW_CODE_REMAP_VIRTUAL_BASE)
+
+/* Permanent mappings setup */
+#define RGXMIPSFW_PT_VIRTUAL_BASE                (0xCF000000)
+#define RGXMIPSFW_REGISTERS_VIRTUAL_BASE         (0xCF800000)
+#define RGXMIPSFW_STACK_VIRTUAL_BASE             (0xCF600000)
+#define RGXMIPSFW_MIPS_STATE_VIRTUAL_BASE        (RGXMIPSFW_REGISTERS_VIRTUAL_BASE + RGXMIPSFW_REGISTERS_PAGE_SIZE)
+
+/* Offset inside the bootloader data page where the general_exception handler saves the error state.
+ * The error value is then copied by the NMI handler to the MipsState struct in shared memory.
+ * This is done because it's difficult to obtain the address of MipsState inside the general
+ * exception handler. */
+#define RGXMIPSFW_ERROR_STATE_BASE                        (0x100)
+
+/*
+ * Bootloader configuration data
+ */
+/* Bootloader configuration offset (where RGXMIPSFW_BOOT_DATA lives)
+ * within the bootloader/NMI data page */
+#define RGXMIPSFW_BOOTLDR_CONF_OFFSET                         (0x0U)
+
+/*
+ * MIPS boot stage
+ */
+#define RGXMIPSFW_BOOT_STAGE_OFFSET                           (0x400)
+
+/*
+ * MIPS private data in the bootloader data page.
+ * Memory below this offset is used by the FW only, no interface data allowed.
+ */
+#define RGXMIPSFW_PRIVATE_DATA_OFFSET                         (0x800)
+
+
+/* The things that follow are excluded when compiling assembly sources */
+#if !defined(RGXMIPSFW_ASSEMBLY_CODE)
+#include "img_types.h"
+#include "km/rgxdefs_km.h"
+
+typedef struct
+{
+	IMG_UINT64 ui64StackPhyAddr;
+	IMG_UINT64 ui64RegBase;
+	IMG_UINT64 aui64PTPhyAddr[RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES];
+	IMG_UINT32 ui32PTLog2PageSize;
+	IMG_UINT32 ui32PTNumPages;
+	IMG_UINT32 ui32Reserved1;
+	IMG_UINT32 ui32Reserved2;
+} RGXMIPSFW_BOOT_DATA;
+
+#define RGXMIPSFW_GET_OFFSET_IN_DWORDS(offset)                (offset / sizeof(IMG_UINT32))
+#define RGXMIPSFW_GET_OFFSET_IN_QWORDS(offset)                (offset / sizeof(IMG_UINT64))
+
+/* Used for compatibility checks */
+#define RGXMIPSFW_ARCHTYPE_VER_CLRMSK                         (0xFFFFE3FFU)
+#define RGXMIPSFW_ARCHTYPE_VER_SHIFT                          (10U)
+#define RGXMIPSFW_CORE_ID_VALUE                               (0x001U)
+#define RGXFW_PROCESSOR_MIPS                                  "MIPS"
+
+/* microAptivAP cache line size */
+#define RGXMIPSFW_MICROAPTIVEAP_CACHELINE_SIZE                (16U)
+
+/* The SOCIF transactions are identified with the top 16 bits of the physical address emitted by the MIPS */
+#define RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN           (16U)
+
+/* Values to put in the MIPS selectors for performance counters */
+#define RGXMIPSFW_PERF_COUNT_CTRL_ICACHE_ACCESSES_C0          (9U)   /* Icache accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_ICACHE_MISSES_C1            (9U)   /* Icache misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_DCACHE_ACCESSES_C0          (10U)  /* Dcache accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_DCACHE_MISSES_C1            (11U) /* Dcache misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_ITLB_INSTR_ACCESSES_C0      (5U)  /* ITLB instruction accesses in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_JTLB_INSTR_MISSES_C1        (7U)  /* JTLB instruction accesses misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_INSTR_COMPLETED_C0          (1U)  /* Instructions completed in COUNTER0 */
+#define RGXMIPSFW_PERF_COUNT_CTRL_JTLB_DATA_MISSES_C1         (8U)  /* JTLB data misses in COUNTER1 */
+
+#define RGXMIPSFW_PERF_COUNT_CTRL_EVENT_SHIFT                 (5U)  /* Shift for the Event field in the MIPS perf ctrl registers */
+/* Additional flags for performance counters. See MIPS manual for further reference */
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_USER_MODE             (8U)
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_KERNEL_MODE           (2U)
+#define RGXMIPSFW_PERF_COUNT_CTRL_COUNT_EXL                   (1U)
+
+
+#define RGXMIPSFW_C0_NBHWIRQ	8
+
+/* Macros to decode C0_Cause register */
+#define RGXMIPSFW_C0_CAUSE_EXCCODE(CAUSE)       (((CAUSE) & 0x7cU) >> 2U)
+#define RGXMIPSFW_C0_CAUSE_EXCCODE_FWERROR      9
+/* Use only when Coprocessor Unusable exception */
+#define RGXMIPSFW_C0_CAUSE_UNUSABLE_UNIT(CAUSE) (((CAUSE) >> 28U) & 0x3U)
+#define RGXMIPSFW_C0_CAUSE_PENDING_HWIRQ(CAUSE) (((CAUSE) & 0x3fc00) >> 10)
+#define RGXMIPSFW_C0_CAUSE_FDCIPENDING          (1UL << 21)
+#define RGXMIPSFW_C0_CAUSE_IV                   (1UL << 23)
+#define RGXMIPSFW_C0_CAUSE_IC                   (1UL << 25)
+#define RGXMIPSFW_C0_CAUSE_PCIPENDING           (1UL << 26)
+#define RGXMIPSFW_C0_CAUSE_TIPENDING            (1UL << 30)
+#define RGXMIPSFW_C0_CAUSE_BRANCH_DELAY         (1UL << 31)
+
+/* Macros to decode C0_Debug register */
+#define RGXMIPSFW_C0_DEBUG_EXCCODE(DEBUG) (((DEBUG) >> 10U) & 0x1fU)
+#define RGXMIPSFW_C0_DEBUG_DSS            (1UL << 0)
+#define RGXMIPSFW_C0_DEBUG_DBP            (1UL << 1)
+#define RGXMIPSFW_C0_DEBUG_DDBL           (1UL << 2)
+#define RGXMIPSFW_C0_DEBUG_DDBS           (1UL << 3)
+#define RGXMIPSFW_C0_DEBUG_DIB            (1UL << 4)
+#define RGXMIPSFW_C0_DEBUG_DINT           (1UL << 5)
+#define RGXMIPSFW_C0_DEBUG_DIBIMPR        (1UL << 6)
+#define RGXMIPSFW_C0_DEBUG_DDBLIMPR       (1UL << 18)
+#define RGXMIPSFW_C0_DEBUG_DDBSIMPR       (1UL << 19)
+#define RGXMIPSFW_C0_DEBUG_IEXI           (1UL << 20)
+#define RGXMIPSFW_C0_DEBUG_DBUSEP         (1UL << 21)
+#define RGXMIPSFW_C0_DEBUG_CACHEEP        (1UL << 22)
+#define RGXMIPSFW_C0_DEBUG_MCHECKP        (1UL << 23)
+#define RGXMIPSFW_C0_DEBUG_IBUSEP         (1UL << 24)
+#define RGXMIPSFW_C0_DEBUG_DM             (1UL << 30)
+#define RGXMIPSFW_C0_DEBUG_DBD            (1UL << 31)
+
+/* Macros to decode TLB entries */
+#define RGXMIPSFW_TLB_GET_MASK(PAGE_MASK)       (((PAGE_MASK) >> 13) & 0XFFFFU)
+#define RGXMIPSFW_TLB_GET_PAGE_SIZE(PAGE_MASK)  ((((PAGE_MASK) | 0x1FFFU) + 1U) >> 11U) /* page size in KB */
+#define RGXMIPSFW_TLB_GET_PAGE_MASK(PAGE_SIZE)  ((((PAGE_SIZE) << 11) - 1) & ~0x7FF) /* page size in KB */
+#define RGXMIPSFW_TLB_GET_VPN2(ENTRY_HI)        ((ENTRY_HI) >> 13)
+#define RGXMIPSFW_TLB_GET_COHERENCY(ENTRY_LO)   (((ENTRY_LO) >> 3) & 0x7U)
+#define RGXMIPSFW_TLB_GET_PFN(ENTRY_LO)         (((ENTRY_LO) >> 6) & 0XFFFFFU)
+/* GET_PA uses a non-standard PFN mask for 36 bit addresses */
+#define RGXMIPSFW_TLB_GET_PA(ENTRY_LO)          (((IMG_UINT64)(ENTRY_LO) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) << 6)
+#define RGXMIPSFW_TLB_GET_INHIBIT(ENTRY_LO)     (((ENTRY_LO) >> 30) & 0x3U)
+#define RGXMIPSFW_TLB_GET_DGV(ENTRY_LO)         ((ENTRY_LO) & 0x7U)
+#define RGXMIPSFW_TLB_GLOBAL                    (1U)
+#define RGXMIPSFW_TLB_VALID                     (1U << 1)
+#define RGXMIPSFW_TLB_DIRTY                     (1U << 2)
+#define RGXMIPSFW_TLB_XI                        (1U << 30)
+#define RGXMIPSFW_TLB_RI                        (1U << 31)
+
+typedef struct {
+	IMG_UINT32 ui32TLBPageMask;
+	IMG_UINT32 ui32TLBHi;
+	IMG_UINT32 ui32TLBLo0;
+	IMG_UINT32 ui32TLBLo1;
+} RGX_MIPS_TLB_ENTRY;
+
+typedef struct {
+	IMG_UINT32 ui32RemapAddrIn;     /* always 4k aligned */
+	IMG_UINT32 ui32RemapAddrOut;    /* always 4k aligned */
+	IMG_UINT32 ui32RemapRegionSize;
+} RGX_MIPS_REMAP_ENTRY;
+
+typedef struct {
+	IMG_UINT32 ui32ErrorState; /* This must come first in the structure */
+	IMG_UINT32 ui32Sync;
+	IMG_UINT32 ui32ErrorEPC;
+	IMG_UINT32 ui32StatusRegister;
+	IMG_UINT32 ui32CauseRegister;
+	IMG_UINT32 ui32BadRegister;
+	IMG_UINT32 ui32EPC;
+	IMG_UINT32 ui32SP;
+	IMG_UINT32 ui32Debug;
+	IMG_UINT32 ui32DEPC;
+	IMG_UINT32 ui32BadInstr;
+	IMG_UINT32 ui32UnmappedAddress;
+	RGX_MIPS_TLB_ENTRY asTLB[RGXMIPSFW_NUMBER_OF_TLB_ENTRIES];
+	IMG_UINT64 aui64Remap[RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES];
+} RGX_MIPS_STATE;
+
+static_assert(offsetof(RGX_MIPS_STATE, ui32ErrorState) == 0,
+				"ui32ErrorState is not the first member of the RGX_MIPS_STATE struct");
+
+#if defined(SUPPORT_MIPS_64K_PAGE_SIZE)
+static_assert(RGXMIPSFW_REGISTERS_PAGE_SIZE >= RGXMIPSFW_PAGE_SIZE_64K,
+				"Register page size must be greater or equal to MIPS page size");
+#else
+static_assert(RGXMIPSFW_REGISTERS_PAGE_SIZE >= RGXMIPSFW_PAGE_SIZE_4K,
+				"Register page size must be greater or equal to MIPS page size");
+#endif
+
+
+#endif /* RGXMIPSFW_ASSEMBLY_CODE */
+
+#endif /* RGX_MIPS_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_options.h b/drivers/gpu/drm/img-rogue/rgx_options.h
new file mode 100644
index 000000000000..df9c949f4a1f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_options.h
@@ -0,0 +1,350 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX build options
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* Each build option listed here is packed into a dword which provides up to
+ *  log2(RGX_BUILD_OPTIONS_MASK_KM + 1) flags for KM and
+ *  (32 - log2(RGX_BUILD_OPTIONS_MASK_KM + 1)) flags for UM.
+ * The corresponding bit is set if the build option was enabled at compile
+ * time.
+ *
+ * IMPORTANT: add new options to unused bits or define a new dword
+ * (e.g. RGX_BUILD_OPTIONS_KM2 or RGX_BUILD_OPTIONS2) so that the bitfield
+ * remains backwards compatible.
+ */
+
+#ifndef RGX_OPTIONS_H
+#define RGX_OPTIONS_H
+
+#define OPTIONS_OPEN_SOURCE_EN					(0x1UL << 0)
+#define OPTIONS_PDUMP_EN						(0x1UL << 1)
+#define OPTIONS_UNUSED1_EN						(0x1UL << 2)
+#define OPTIONS_SECURE_ALLOC_KM_EN				(0x1UL << 3)
+#define OPTIONS_RGX_EN							(0x1UL << 4)
+#define OPTIONS_SECURE_EXPORT_EN				(0x1UL << 5)
+#define OPTIONS_INSECURE_EXPORT_EN				(0x1UL << 6)
+#define OPTIONS_VFP_EN							(0x1UL << 7)
+#define OPTIONS_WORKLOAD_ESTIMATION_EN			(0x1UL << 8)
+#define OPTIONS_PDVFS_EN						(0x1UL << 9)
+#define OPTIONS_DEBUG_EN						(0x1UL << 10)
+#define OPTIONS_BUFFER_SYNC_EN					(0x1UL << 11)
+#define OPTIONS_AUTOVZ_EN						(0x1UL << 12)
+#define OPTIONS_AUTOVZ_HW_REGS_EN				(0x1UL << 13)
+#define OPTIONS_FW_IRQ_REG_COUNTERS_EN			(0x1UL << 14)
+#define OPTIONS_VALIDATION_EN					(0x1UL << 15)
+#define OPTIONS_NO_HARDWARE_EN					(0x1UL << 16)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN                  (0x1UL << 17)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_MASK                      (0x1fUL << 18)
+#define OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT                     (18)
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_PERCONTEXT_FREELIST_EN			(0x1UL << 31)
+
+#define RGX_BUILD_OPTIONS_MASK_KM				  \
+			(OPTIONS_OPEN_SOURCE_EN				| \
+			 OPTIONS_PDUMP_EN					| \
+			 OPTIONS_SECURE_ALLOC_KM_EN			| \
+			 OPTIONS_RGX_EN						| \
+			 OPTIONS_SECURE_EXPORT_EN			| \
+			 OPTIONS_INSECURE_EXPORT_EN			| \
+			 OPTIONS_VFP_EN						| \
+			 OPTIONS_WORKLOAD_ESTIMATION_EN		| \
+			 OPTIONS_PDVFS_EN					| \
+			 OPTIONS_DEBUG_EN					| \
+			 OPTIONS_BUFFER_SYNC_EN				| \
+			 OPTIONS_AUTOVZ_EN					| \
+			 OPTIONS_AUTOVZ_HW_REGS_EN			| \
+			 OPTIONS_FW_IRQ_REG_COUNTERS_EN		| \
+			 OPTIONS_VALIDATION_EN				| \
+			 OPTIONS_NO_HARDWARE_EN				| \
+			 OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN)
+
+#define RGX_BUILD_OPTIONS_MASK_FW \
+			(RGX_BUILD_OPTIONS_MASK_KM & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN & \
+			 ~OPTIONS_NUM_DRIVERS_SUPPORTED_MASK & \
+			 ~OPTIONS_BUFFER_SYNC_EN)
+
+/* Build options that the FW must have if the present on the KM */
+#define FW_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the UM must have if the present on the KM */
+#define UM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_PDVFS_EN))
+
+/* Build options that the KM must have if the present on the UM */
+#define KM_OPTIONS_STRICT ((RGX_BUILD_OPTIONS_MASK_KM			| \
+							OPTIONS_PERCONTEXT_FREELIST_EN)		& \
+							~(OPTIONS_DEBUG_EN					| \
+							  OPTIONS_WORKLOAD_ESTIMATION_EN	| \
+							  OPTIONS_PDVFS_EN					| \
+							  OPTIONS_OPEN_SOURCE_EN			| \
+							  OPTIONS_BUFFER_SYNC_EN))
+
+#define OPEN_SOURCE_OPTION	"OPEN_SOURCE_DRIVER  "
+#if defined(SUPPORT_OPEN_SOURCE_DRIVER)
+	#define OPTIONS_BIT0		OPTIONS_OPEN_SOURCE_EN
+	#if OPTIONS_BIT0 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT0		0x0UL
+#endif /* SUPPORT_OPEN_SOURCE_DRIVER */
+
+#define PDUMP_OPTION	"PDUMP  "
+#if defined(PDUMP)
+	#define OPTIONS_BIT1		OPTIONS_PDUMP_EN
+	#if OPTIONS_BIT1 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT1		0x0UL
+#endif /* PDUMP */
+
+/* No longer used */
+#define INTERNAL_UNUSED1_OPTION	"INTERNAL_UNUSED1  "
+#if defined(INTERNAL_UNUSED1)
+	#define OPTIONS_BIT2		OPTIONS_UNUSED1_EN
+	#if OPTIONS_BIT2 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT2		0x0UL
+#endif
+
+#define SECURE_ALLOC_KM_OPTION "SECURE_ALLOC_KM  "
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	#define OPTIONS_BIT3		OPTIONS_SECURE_ALLOC_KM_EN
+	#if OPTIONS_BIT3 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT3 0x0UL
+#endif /* SUPPORT_SECURE_ALLOC_KM */
+
+#define RGX_OPTION	" "
+#if defined(SUPPORT_RGX)
+	#define OPTIONS_BIT4		OPTIONS_RGX_EN
+	#if OPTIONS_BIT4 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT4		0x0UL
+#endif /* SUPPORT_RGX */
+
+#define SECURE_EXPORT_OPTION	"SECURE_EXPORTS  "
+#if defined(SUPPORT_SECURE_EXPORT)
+	#define OPTIONS_BIT5		OPTIONS_SECURE_EXPORT_EN
+	#if OPTIONS_BIT5 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT5		0x0UL
+#endif /* SUPPORT_SECURE_EXPORT */
+
+#define INSECURE_EXPORT_OPTION	"INSECURE_EXPORTS  "
+#if defined(SUPPORT_INSECURE_EXPORT)
+	#define OPTIONS_BIT6		OPTIONS_INSECURE_EXPORT_EN
+	#if OPTIONS_BIT6 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT6		0x0UL
+#endif /* SUPPORT_INSECURE_EXPORT */
+
+#define VFP_OPTION	"VFP  "
+#if defined(SUPPORT_VFP)
+	#define OPTIONS_BIT7		OPTIONS_VFP_EN
+	#if OPTIONS_BIT7 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT7		0x0UL
+#endif /* SUPPORT_VFP */
+
+#define WORKLOAD_ESTIMATION_OPTION	"WORKLOAD_ESTIMATION  "
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	#define OPTIONS_BIT8		OPTIONS_WORKLOAD_ESTIMATION_EN
+	#if OPTIONS_BIT8 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT8		0x0UL
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+
+#define PDVFS_OPTION	"PDVFS  "
+#if defined(SUPPORT_PDVFS)
+	#define OPTIONS_BIT9		OPTIONS_PDVFS_EN
+	#if OPTIONS_BIT9 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT9		0x0UL
+#endif /* SUPPORT_PDVFS */
+
+#define DEBUG_OPTION	"DEBUG  "
+#if defined(DEBUG)
+	#define OPTIONS_BIT10		OPTIONS_DEBUG_EN
+	#if OPTIONS_BIT10 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT10		0x0UL
+#endif /* DEBUG */
+
+#define BUFFER_SYNC_OPTION	"BUFFER_SYNC  "
+#if defined(SUPPORT_BUFFER_SYNC)
+	#define OPTIONS_BIT11		OPTIONS_BUFFER_SYNC_EN
+	#if OPTIONS_BIT11 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT11		0x0UL
+#endif /* SUPPORT_BUFFER_SYNC */
+
+#define AUTOVZ_OPTION	"AUTOVZ  "
+#if defined(SUPPORT_AUTOVZ)
+	#define OPTIONS_BIT12		OPTIONS_AUTOVZ_EN
+	#if OPTIONS_BIT12 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT12		0x0UL
+#endif /* SUPPORT_AUTOVZ */
+
+#define AUTOVZ_HW_REGS_OPTION	"AUTOVZ_HW_REGS  "
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+	#define OPTIONS_BIT13		OPTIONS_AUTOVZ_HW_REGS_EN
+	#if OPTIONS_BIT13 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT13		0x0UL
+#endif /* SUPPORT_AUTOVZ_HW_REGS */
+
+#define RGX_FW_IRQ_OS_COUNTERS_OPTION	"FW_IRQ_OS_COUNTERS  "
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	#define OPTIONS_BIT14		OPTIONS_FW_IRQ_REG_COUNTERS_EN
+	#if OPTIONS_BIT14 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT14		0x0UL
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+#define VALIDATION_OPTION	"VALIDATION  "
+	#define OPTIONS_BIT15		0x0UL
+
+#define NO_HARDWARE_OPTION	"NO_HARDWARE  "
+#if defined(NO_HARDWARE)
+	#define OPTIONS_BIT16		OPTIONS_NO_HARDWARE_EN
+	#if OPTIONS_BIT16 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif
+#else
+	#define OPTIONS_BIT16		0x0UL
+#endif /* NO_HARDWARE */
+
+#define NUM_DRIVERS_SUPPORTED_CHECK_OPTION	"NUM_DRIVERS_SUPPORTED_CHECK  "
+	#define OPTIONS_BIT17		OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN
+	#if OPTIONS_BIT17 > RGX_BUILD_OPTIONS_MASK_KM
+	#error "Bit exceeds reserved range"
+	#endif /* NUM_DRIVERS_SUPPORTED_CHECK */
+/* Option bits[22:18] are used for max number of drivers supported by the FW. */
+
+#define OPTIONS_BIT31		OPTIONS_PERCONTEXT_FREELIST_EN
+#if OPTIONS_BIT31 <= RGX_BUILD_OPTIONS_MASK_KM
+#error "Bit exceeds reserved range"
+#endif
+
+#define RGX_BUILD_OPTIONS_KM	\
+	(OPTIONS_BIT0				|\
+	 OPTIONS_BIT1				|\
+	 OPTIONS_BIT2				|\
+	 OPTIONS_BIT3				|\
+	 OPTIONS_BIT4				|\
+	 OPTIONS_BIT5				|\
+	 OPTIONS_BIT6				|\
+	 OPTIONS_BIT7				|\
+	 OPTIONS_BIT8				|\
+	 OPTIONS_BIT9				|\
+	 OPTIONS_BIT10				|\
+	 OPTIONS_BIT11				|\
+	 OPTIONS_BIT12				|\
+	 OPTIONS_BIT13				|\
+	 OPTIONS_BIT14				|\
+	 OPTIONS_BIT15				|\
+	 OPTIONS_BIT16				|\
+	 OPTIONS_BIT17)
+
+#define RGX_BUILD_OPTIONS (RGX_BUILD_OPTIONS_KM | OPTIONS_BIT31)
+
+#define RGX_BUILD_OPTIONS_LIST			\
+	{									\
+		OPEN_SOURCE_OPTION,				\
+		PDUMP_OPTION,					\
+		INTERNAL_UNUSED1_OPTION,		\
+		SECURE_ALLOC_KM_OPTION,			\
+		RGX_OPTION,						\
+		SECURE_EXPORT_OPTION,			\
+		INSECURE_EXPORT_OPTION,			\
+		VFP_OPTION,						\
+		WORKLOAD_ESTIMATION_OPTION,		\
+		PDVFS_OPTION,					\
+		DEBUG_OPTION,					\
+		BUFFER_SYNC_OPTION,				\
+		AUTOVZ_OPTION,					\
+		AUTOVZ_HW_REGS_OPTION,			\
+		RGX_FW_IRQ_OS_COUNTERS_OPTION,	\
+		VALIDATION_OPTION,				\
+		NO_HARDWARE_OPTION,				\
+		NUM_DRIVERS_SUPPORTED_CHECK_OPTION				\
+	}
+
+#endif /* RGX_OPTIONS_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_pdump_panics.h b/drivers/gpu/drm/img-rogue/rgx_pdump_panics.h
new file mode 100644
index 000000000000..fce2b3efab69
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_pdump_panics.h
@@ -0,0 +1,64 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX PDump panic definitions header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX PDump panic definitions header
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_PDUMP_PANICS_H_)
+#define RGX_PDUMP_PANICS_H_
+
+/*! Unique device specific IMG_UINT16 panic IDs to identify the cause of an
+ * RGX PDump panic in a PDump script. */
+typedef enum
+{
+	RGX_PDUMP_PANIC_UNDEFINED = 0,
+
+	/* These panics occur when test parameters and driver configuration
+	 * enable features that require the firmware and host driver to
+	 * communicate. Such features are not supported with off-line playback.
+	 */
+	RGX_PDUMP_PANIC_ZSBUFFER_BACKING         = 101, /*!< Requests ZSBuffer to be backed with physical pages */
+	RGX_PDUMP_PANIC_ZSBUFFER_UNBACKING       = 102, /*!< Requests ZSBuffer to be unbacked */
+	RGX_PDUMP_PANIC_FREELIST_GROW            = 103, /*!< Requests an on-demand freelist grow/shrink */
+	RGX_PDUMP_PANIC_FREELISTS_RECONSTRUCTION = 104, /*!< Requests freelists reconstruction */
+	RGX_PDUMP_PANIC_SPARSEMEM_SWAP           = 105, /*!< Requests sparse remap memory swap feature */
+} RGX_PDUMP_PANIC;
+
+#endif /* RGX_PDUMP_PANICS_H_ */
diff --git a/drivers/gpu/drm/img-rogue/rgx_riscv.h b/drivers/gpu/drm/img-rogue/rgx_riscv.h
new file mode 100644
index 000000000000..f98b77658d8f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_riscv.h
@@ -0,0 +1,258 @@
+/*************************************************************************/ /*!
+@File           rgx_riscv.h
+@Title
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       RGX
+@Description    RGX RISCV definitions, kernel/user space
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_RISCV_H)
+#define RGX_RISCV_H
+
+#include "km/rgxdefs_km.h"
+
+
+/* Utility defines to convert regions to virtual addresses and remaps */
+#define RGXRISCVFW_NUM_REGIONS_LOG2             IMG_UINT32_C(4)
+#define RGXRISCVFW_REGION_SIZE_LOG2             IMG_UINT32_C(28)
+#define RGXRISCVFW_NUM_REGIONS                  (IMG_UINT32_C(1) << RGXRISCVFW_NUM_REGIONS_LOG2)
+#define RGXRISCVFW_GET_REGION_BASE(r)           ((r) << RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_GET_REGION(a)                ((a) >> RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_MAX_REGION_SIZE              (IMG_UINT32_C(1) << RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_GET_REMAP(r)                 (RGX_CR_FWCORE_ADDR_REMAP_CONFIG0 + ((r) * 8U))
+
+#define RGXRISCVFW_REGION_MASK                  (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_NUM_REGIONS - IMG_UINT32_C(1)))
+
+#if !defined(RGXRISCVFW_ASSEMBLY_CODE)
+static_assert(RGXRISCVFW_NUM_REGIONS_LOG2 + RGXRISCVFW_REGION_SIZE_LOG2 == 32,
+              "Mismatch between number of remap regions and remap size");
+#endif
+
+/* RISCV remap output is aligned to 4K */
+#define RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN  (0x1000U)
+
+/*
+ * FW bootloader defines
+ */
+#define RGXRISCVFW_BOOTLDR_CODE_REGION          (IMG_UINT32_C(0xC0000000) >> RGXRISCVFW_REGION_SIZE_LOG2)
+#define RGXRISCVFW_BOOTLDR_DATA_REGION          IMG_UINT32_C(0x5)
+#define RGXRISCVFW_BOOTLDR_CODE_BASE            (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_BASE            (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_BOOTLDR_DATA_REGION))
+#define RGXRISCVFW_BOOTLDR_CODE_REMAP           (RGXRISCVFW_GET_REMAP(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_REMAP           (RGXRISCVFW_GET_REMAP(RGXRISCVFW_BOOTLDR_DATA_REGION))
+
+/* Bootloader data offset in dwords from the beginning of the FW data allocation */
+#define RGXRISCVFW_BOOTLDR_CONF_OFFSET          (0x0)
+
+/*
+ * FW coremem region defines
+ */
+#define RGXRISCVFW_COREMEM_REGION               IMG_UINT32_C(0x8)
+#define RGXRISCVFW_COREMEM_MAX_SIZE             IMG_UINT32_C(0x10000000) /* 256 MB */
+#define RGXRISCVFW_COREMEM_BASE                 (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_COREMEM_REGION))
+#define RGXRISCVFW_COREMEM_END                  (RGXRISCVFW_COREMEM_BASE + RGXRISCVFW_COREMEM_MAX_SIZE - 1U)
+
+
+/*
+ * Host-FW shared data defines
+ */
+#define RGXRISCVFW_SHARED_CACHED_DATA_REGION    (0x0UL)
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_REGION  (0x1UL)
+#define RGXRISCVFW_SHARED_CACHED_DATA_BASE      (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SHARED_CACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_BASE    (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SHARED_UNCACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_CACHED_DATA_REMAP     (RGXRISCVFW_GET_REMAP(RGXRISCVFW_SHARED_CACHED_DATA_REGION))
+#define RGXRISCVFW_SHARED_UNCACHED_DATA_REMAP   (RGXRISCVFW_GET_REMAP(RGXRISCVFW_SHARED_UNCACHED_DATA_REGION))
+
+
+/*
+ * GPU SOCIF access defines
+ */
+#define RGXRISCVFW_SOCIF_REGION                 (0x2U)
+#define RGXRISCVFW_SOCIF_BASE                   (RGXRISCVFW_GET_REGION_BASE(RGXRISCVFW_SOCIF_REGION))
+
+
+/* The things that follow are excluded when compiling assembly sources */
+#if !defined(RGXRISCVFW_ASSEMBLY_CODE)
+#include "img_types.h"
+
+#define RGXFW_PROCESSOR_RISCV       "RISCV"
+#define RGXRISCVFW_CORE_ID_VALUE    (0x00450B02U)
+#define RGXRISCVFW_MISA_ADDR        (0x301U)
+#define RGXRISCVFW_MISA_VALUE       (0x40001104U)
+#define RGXRISCVFW_MSCRATCH_ADDR    (0x340U)
+
+typedef struct
+{
+	IMG_UINT64 ui64CorememCodeDevVAddr;
+	IMG_UINT64 ui64CorememDataDevVAddr;
+	IMG_UINT32 ui32CorememCodeFWAddr;
+	IMG_UINT32 ui32CorememDataFWAddr;
+	IMG_UINT32 ui32CorememCodeSize;
+	IMG_UINT32 ui32CorememDataSize;
+	IMG_UINT32 ui32Flags;
+	IMG_UINT32 ui32Reserved;
+} RGXRISCVFW_BOOT_DATA;
+
+/*
+ * List of registers to be printed in debug dump.
+ * First column:  register names (general purpose or control/status registers)
+ * Second column: register number to be used in abstract access register command
+ * (see RISC-V debug spec v0.13)
+ */
+#define RGXRISCVFW_DEBUG_DUMP_REGISTERS \
+	X(pc,        0x7b1) /* dpc */ \
+	X(ra,       0x1001) \
+	X(sp,       0x1002) \
+	X(mepc,      0x341) \
+	X(mcause,    0x342) \
+	X(mdseac,    0xfc0) \
+	X(mstatus,   0x300) \
+	X(mie,       0x304) \
+	X(mip,       0x344) \
+	X(mscratch,  0x340) \
+	X(mbvnc0,    0xffe) \
+	X(mbvnc1,    0xfff) \
+	X(micect,    0x7f0) \
+	X(mdcect,    0x7f3) \
+	X(mdcrfct,   0x7f4) \
+
+typedef struct
+{
+#define X(name, address) \
+	IMG_UINT32 name;
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+} RGXRISCVFW_STATE;
+
+
+#define RGXRISCVFW_MCAUSE_INTERRUPT  (1U << 31)
+
+#define RGXRISCVFW_MCAUSE_TABLE \
+	X(0x00000000U, IMG_FALSE, "NMI pin assertion") /* Also reset value */ \
+	X(0x00000001U, IMG_TRUE,  "Instruction access fault") \
+	X(0x00000002U, IMG_TRUE,  "Illegal instruction") \
+	X(0x00000003U, IMG_TRUE,  "Breakpoint") \
+	X(0x00000004U, IMG_TRUE,  "Load address misaligned") \
+	X(0x00000005U, IMG_TRUE,  "Load access fault") \
+	X(0x00000006U, IMG_TRUE,  "Store/AMO address misaligned") \
+	X(0x00000007U, IMG_TRUE,  "Store/AMO access fault") \
+	X(0x0000000BU, IMG_TRUE,  "Environment call from M-mode (FW assert)") \
+	X(0x80000007U, IMG_FALSE, "Machine timer interrupt") \
+	X(0x8000000BU, IMG_FALSE, "Machine external interrupt") \
+	X(0x8000001EU, IMG_FALSE, "Machine correctable error local interrupt") \
+	X(0xF0000000U, IMG_TRUE,  "Machine D-bus store error NMI") \
+	X(0xF0000001U, IMG_TRUE,  "Machine D-bus non-blocking load error NMI") \
+	X(0xF0000002U, IMG_TRUE,  "dCache unrecoverable NMI")
+
+
+/* Debug module HW defines */
+#define RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER (0U)
+#define RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY   (2U)
+#define RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT   (2UL << 20)
+#define RGXRISCVFW_DMI_COMMAND_WRITE           (1UL << 16)
+#define RGXRISCVFW_DMI_COMMAND_READ            (0UL << 16)
+#define RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT     (2U)
+
+/* Abstract command error codes (descriptions from RISC-V debug spec v0.13) */
+typedef IMG_UINT32 RGXRISCVFW_ABSTRACT_CMD_ERR;
+
+/* No error. */
+#define RISCV_ABSTRACT_CMD_NO_ERROR 0U
+
+/*
+ * An abstract command was executing while command, abstractcs, or abstractauto
+ * was written, or when one of the data or progbuf registers was read or
+ * written. This status is only written if cmderr contains 0.
+ */
+#define RISCV_ABSTRACT_CMD_BUSY 1U
+
+/*
+ * The requested command is not supported, regardless of whether
+ * the hart is running or not.
+ */
+#define RISCV_ABSTRACT_CMD_NOT_SUPPORTED 2U
+
+/*
+ * An exception occurred while executing the command
+ * (e.g. while executing the Program Buffer).
+ */
+#define RISCV_ABSTRACT_CMD_EXCEPTION 3U
+
+/*
+ * The abstract command couldn't execute because the hart wasn't in the required
+ * state (running/halted), or unavailable.
+ */
+#define RISCV_ABSTRACT_CMD_HALT_RESUME 4U
+
+/*
+ * The abstract command failed due to a bus error
+ * (e.g. alignment, access size, or timeout).
+ */
+#define RISCV_ABSTRACT_CMD_BUS_ERROR 5U
+
+/* The command failed for another reason. */
+#define RISCV_ABSTRACT_CMD_OTHER_ERROR 7U
+
+
+/* System Bus error codes (descriptions from RISC-V debug spec v0.13) */
+typedef IMG_UINT32 RGXRISCVFW_SYSBUS_ERR;
+
+/* There was no bus error. */
+#define RISCV_SYSBUS_NO_ERROR 0U
+
+/* There was a timeout. */
+#define RISCV_SYSBUS_TIMEOUT 1U
+
+/* A bad address was accessed. */
+#define RISCV_SYSBUS_BAD_ADDRESS 2U
+
+/* There was an alignment error. */
+#define RISCV_SYSBUS_BAD_ALIGNMENT 3U
+
+/* An access of unsupported size was requested. */
+#define RISCV_SYSBUS_UNSUPPORTED_SIZE 4U
+
+/* Other. */
+#define RISCV_SYSBUS_OTHER_ERROR 7U
+
+
+#endif /* RGXRISCVFW_ASSEMBLY_CODE */
+
+#endif /* RGX_RISCV_H */
diff --git a/drivers/gpu/drm/img-rogue/rgx_tq_shared.h b/drivers/gpu/drm/img-rogue/rgx_tq_shared.h
new file mode 100644
index 000000000000..2db19224c441
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgx_tq_shared.h
@@ -0,0 +1,61 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX transfer queue shared
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shared definitions between client and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGX_TQ_SHARED_H
+#define RGX_TQ_SHARED_H
+
+#define TQ_MAX_PREPARES_PER_SUBMIT		16U
+
+#define TQ_PREP_FLAGS_COMMAND_3D		0x0U
+#define TQ_PREP_FLAGS_COMMAND_2D		0x1U
+#define TQ_PREP_FLAGS_COMMAND_MASK		(0xfU)
+#define TQ_PREP_FLAGS_COMMAND_SHIFT		0
+#define TQ_PREP_FLAGS_PDUMPCONTINUOUS	(1U << 4)
+
+#define TQ_PREP_FLAGS_COMMAND_SET(m) \
+	((TQ_PREP_FLAGS_COMMAND_##m << TQ_PREP_FLAGS_COMMAND_SHIFT) & TQ_PREP_FLAGS_COMMAND_MASK)
+
+#define TQ_PREP_FLAGS_COMMAND_IS(m,n) \
+	(((m & TQ_PREP_FLAGS_COMMAND_MASK) >> TQ_PREP_FLAGS_COMMAND_SHIFT)  == TQ_PREP_FLAGS_COMMAND_##n)
+
+#endif /* RGX_TQ_SHARED_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxapi_km.h b/drivers/gpu/drm/img-rogue/rgxapi_km.h
new file mode 100644
index 000000000000..7d7efd0afb5e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxapi_km.h
@@ -0,0 +1,322 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX API Header kernel mode
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported RGX API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXAPI_KM_H
+#define RGXAPI_KM_H
+
+#include "rgx_hwperf.h"
+
+/******************************************************************************
+ * RGX HW Performance Profiling Control API(s)
+ *****************************************************************************/
+
+/*! HWPerf device identification structure */
+typedef struct _RGX_HWPERF_DEVICE_
+{
+	IMG_CHAR pszName[20];	/*!< Helps identify this device uniquely */
+	IMG_HANDLE hDevData;	/*!< Handle for the server */
+
+	struct _RGX_HWPERF_DEVICE_ *psNext;	/*!< Next device if any */
+} RGX_HWPERF_DEVICE;
+
+/*! HWPerf connection structure */
+typedef struct
+{
+	RGX_HWPERF_DEVICE *psHWPerfDevList;	/*!< Pointer to list of devices */
+} RGX_HWPERF_CONNECTION;
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfLazyConnect
+@Description    Obtain a HWPerf connection object to the RGX device(s). The
+                 connections to devices are not actually opened until
+                 HWPerfOpen() is called.
+
+@Output         ppsHWPerfConnection Address of a HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfOpen
+@Description    Opens connection(s) to the RGX device(s). Valid handle to the
+                 connection object has to be provided which means the this
+                 function needs to be preceded by the call to
+                 RGXHWPerfLazyConnect() function.
+
+@Input          psHWPerfConnection HWPerf connection object
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION* psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConnect
+@Description    Obtain a connection object to the RGX HWPerf module. Allocated
+                 connection object(s) reference opened connection(s). Calling
+                 this function is an equivalent of calling RGXHWPerfLazyConnect
+                 and RGXHWPerfOpen. This connect should be used when the caller
+                 will be retrieving event data.
+
+@Output         ppsHWPerfConnection Address of HWPerf connection object
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfFreeConnection
+@Description    Frees the HWPerf connection object
+
+@Input          psHWPerfConnection Pointer to connection object as returned
+                                    from RGXHWPerfLazyConnect()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfClose
+@Description    Closes all the opened connection(s) to RGX device(s)
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object as
+                                    returned from RGXHWPerfConnect() or
+                                    RGXHWPerfOpen()
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisconnect
+@Description    Disconnect from the RGX device
+
+@Input          ppsHWPerfConnection Pointer to HWPerf connection object as
+                                     returned from RGXHWPerfConnect() or
+                                     RGXHWPerfOpen(). Calling this function is
+                                     an equivalent of calling RGXHWPerfClose()
+                                     and RGXHWPerfFreeConnection().
+@Return         PVRSRV_ERROR        System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfControl
+@Description    Enable or disable the generation of RGX HWPerf event packets.
+                 See RGXCtrlHWPerf().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          eStreamId          ID of the HWPerf stream
+@Input          bToggle            Switch to toggle or apply mask.
+@Input          ui64Mask           Mask of events to control.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfGetFilter
+@Description    Reads HWPerf stream filter where stream is identified by the
+                 given stream ID.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ui64Filter   HWPerf filter value
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter
+);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigMuxCounters
+@Description    Enable and configure the performance counter block for one or
+                 more device layout modules.
+                 See RGXHWPerfConfigureAndEnableCustomCounters().
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          asBlockConfigs     Address of the array of configuration blocks
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigMuxCounters(
+		RGX_HWPERF_CONNECTION         *psHWPerfConnection,
+		IMG_UINT32                     ui32NumBlocks,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK  *asBlockConfigs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConfigureAndEnableCustomCounters
+@Description    Enable and configure custom performance counters
+
+@Input          psHWPerfConnection    Pointer to HWPerf connection object
+@Input          ui16CustomBlockID     ID of the custom block to configure
+@Input          ui16NumCustomCounters Number of custom counters
+@Input          pui32CustomCounterIDs Pointer to array containing custom
+                                       counter IDs
+@Return         PVRSRV_ERROR          System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfConfigureAndEnableCustomCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT16             ui16CustomBlockID,
+		IMG_UINT16             ui16NumCustomCounters,
+		IMG_UINT32            *pui32CustomCounterIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfDisableCounters
+@Description    Disable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from
+                                    the <tt>RGX_HWPERF_CNTBLK_ID</tt>
+                                    enumeration.
+@Return         PVRSRV_ERROR       System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfEnableCounters
+@Description    Enable the performance counter block for one or more device
+                 layout modules.
+
+@Input          psHWPerfConnection Pointer to HWPerf connection object
+@Input          ui32NumBlocks      Number of elements in the array
+@Input          aeBlockIDs         An array of words with values taken from the
+                                    <tt>RGX_HWPERF_CNTBLK_ID</tt> enumeration.
+@Return         PVRSRV_ERROR  System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling Retrieval API(s)
+ *
+ * The client must ensure their use of this acquire/release API for a single
+ * connection/stream must not be shared with multiple execution contexts e.g.
+ * between a kernel thread and an ISR handler. It is the client's
+ * responsibility to ensure this API is not interrupted by a high priority
+ * thread/ISR
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfAcquireEvents
+@Description    When there is data available to read this call returns with OK
+                 and the address and length of the data buffer the client can
+                 safely read. This buffer may contain one or more event packets.
+                 When there is no data to read, this call returns with OK and
+                 sets *puiBufLen to 0 on exit.
+                 Clients must pair this call with a RGXHWPerfReleaseEvents()
+                 call.
+                 Data returned in ppBuf will be in the form of a sequence of
+                 HWPerf packets which should be traversed using the pointers,
+                 structures and macros provided by rgx_hwperf.h.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Output         ppBuf        Address of a pointer to a byte buffer. On exit it
+                              contains the address of buffer to read from
+@Output         pui32BufLen  Pointer to an integer. On exit it is the size of
+                              the data to read from the buffer
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfReleaseEvents
+@Description    Called after client has read the event data out of the buffer
+                 retrieved from the Acquire Events call to release resources.
+
+@Input          hDevData     Handle to connection/device object
+@Input          eStreamId    ID of the HWPerf stream
+@Return         PVRSRV_ERROR System error code
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId);
+
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfConvertCRTimeStamp
+@Description    Converts the timestamp given by FW events to the common OS
+                 timestamp. The first three inputs are obtained via a CLK_SYNC
+                 event, ui64CRTimeStamp is the CR timestamp from the FW event
+                 to be converted.
+
+@Input          ui32ClkSpeed        Clock speed given by sync event
+@Input          ui64CorrCRTimeStamp CR Timestamp given by sync event
+@Input          ui64CorrOSTimeStamp Correlating OS Timestamp given by sync
+                                     event
+@Input          ui64CRTimeStamp     CR Timestamp to convert
+@Return         IMG_UINT64          Calculated OS Timestamp
+*/ /**************************************************************************/
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp);
+
+#endif /* RGXAPI_KM_H */
+
+/******************************************************************************
+ End of file (rgxapi_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxbreakpoint.c b/drivers/gpu/drm/img-rogue/rgxbreakpoint.c
new file mode 100644
index 000000000000..902cbe0318d9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxbreakpoint.c
@@ -0,0 +1,292 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Breakpoint routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Breakpoint routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxbreakpoint.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxmem.h"
+#include "device.h"
+#include "sync_internal.h"
+#include "pdump_km.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR PVRSRVRGXSetBreakpointKM(CONNECTION_DATA    * psConnection,
+                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                      IMG_HANDLE           hMemCtxPrivData,
+                                      RGXFWIF_DM           eFWDataMaster,
+                                      IMG_UINT64           ui64TempSpillingAddr,
+                                      IMG_UINT32           ui32BPAddr,
+                                      IMG_UINT32           ui32HandlerAddr,
+                                      IMG_UINT32           ui32DataMaster)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet)
+	{
+		eError = PVRSRV_ERROR_BP_ALREADY_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPAddr = ui32BPAddr;
+	sBPCmd.uCmdData.sBPData.ui32HandlerAddr = ui32HandlerAddr;
+	sBPCmd.uCmdData.sBPData.ui32BPDM = ui32DataMaster;
+	sBPCmd.uCmdData.sBPData.ui64SpillAddr = ui64TempSpillingAddr;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_WRITE | RGXFWIF_BPDATA_FLAGS_ENABLE;
+	sBPCmd.uCmdData.sBPData.eDM = eFWDataMaster;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          eFWDataMaster,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+	psDevInfo->eBPDM = eFWDataMaster;
+	psDevInfo->bBPSet = IMG_TRUE;
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXClearBreakpointKM(CONNECTION_DATA    * psConnection,
+                                        PVRSRV_DEVICE_NODE * psDeviceNode,
+                                        IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPAddr = 0;
+	sBPCmd.uCmdData.sBPData.ui32HandlerAddr = 0;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_WRITE | RGXFWIF_BPDATA_FLAGS_CTL;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+	psDevInfo->bBPSet = IMG_FALSE;
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXEnableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                         PVRSRV_DEVICE_NODE * psDeviceNode,
+                                         IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet == IMG_FALSE)
+	{
+		eError = PVRSRV_ERROR_BP_NOT_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_CTL | RGXFWIF_BPDATA_FLAGS_ENABLE;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDisableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_HANDLE           hMemCtxPrivData)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC		*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	if (psDevInfo->bBPSet == IMG_FALSE)
+	{
+		eError = PVRSRV_ERROR_BP_NOT_SET;
+		goto unlock;
+	}
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_CTL;
+	sBPCmd.uCmdData.sBPData.eDM = psDevInfo->eBPDM;
+
+	eError = RGXSetFirmwareAddress(&sBPCmd.uCmdData.sBPData.psFWMemContext,
+				psFWMemContextMemDesc,
+				0 ,
+				RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", unlock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+	                                          psDevInfo->eBPDM,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXOverallocateBPRegistersKM(CONNECTION_DATA    * psConnection,
+                                                PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                IMG_UINT32           ui32TempRegs,
+                                                IMG_UINT32           ui32SharedRegs)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sBPCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	sBPCmd.eCmdType = RGXFWIF_KCCB_CMD_BP;
+	sBPCmd.uCmdData.sBPData.ui32BPDataFlags = RGXFWIF_BPDATA_FLAGS_REGS;
+	sBPCmd.uCmdData.sBPData.ui32TempRegs = ui32TempRegs;
+	sBPCmd.uCmdData.sBPData.ui32SharedRegs = ui32SharedRegs;
+	sBPCmd.uCmdData.sBPData.psFWMemContext.ui32Addr = 0U;
+	sBPCmd.uCmdData.sBPData.eDM = RGXFWIF_DM_GP;
+
+	OSLockAcquire(psDevInfo->hBPLock);
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sBPCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", unlock);
+
+	/* Wait for FW to complete command execution */
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", unlock);
+
+unlock:
+	OSLockRelease(psDevInfo->hBPLock);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxbreakpoint.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxbreakpoint.h b/drivers/gpu/drm/img-rogue/rgxbreakpoint.h
new file mode 100644
index 000000000000..2470de250914
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxbreakpoint.h
@@ -0,0 +1,142 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX breakpoint functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX breakpoint functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXBREAKPOINT_H)
+#define RGXBREAKPOINT_H
+
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetBreakpointKM
+
+ @Description
+	Server-side implementation of RGXSetBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input eDataMaster - Data Master to schedule command for
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32BPAddr - Address of breakpoint
+ @Input ui32HandlerAddr - Address of breakpoint handler
+ @Input ui32BPCtl - Breakpoint controls
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetBreakpointKM(CONNECTION_DATA    * psConnection,
+                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                      IMG_HANDLE           hMemCtxPrivData,
+                                      RGXFWIF_DM           eFWDataMaster,
+                                      IMG_UINT64           ui64TempSpillingAddr,
+                                      IMG_UINT32           ui32BPAddr,
+                                      IMG_UINT32           ui32HandlerAddr,
+                                      IMG_UINT32           ui32DataMaster);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXClearBreakpointKM
+
+ @Description
+	Server-side implementation of RGXClearBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXClearBreakpointKM(CONNECTION_DATA    * psConnection,
+                                        PVRSRV_DEVICE_NODE * psDeviceNode,
+                                        IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXEnableBreakpointKM
+
+ @Description
+	Server-side implementation of RGXEnableBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXEnableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                         PVRSRV_DEVICE_NODE * psDeviceNode,
+                                         IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDisableBreakpointKM
+
+ @Description
+	Server-side implementation of RGXDisableBreakpoint
+
+ @Input psDeviceNode - RGX Device node
+ @Input hMemCtxPrivData - memory context private data
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDisableBreakpointKM(CONNECTION_DATA    * psConnection,
+                                          PVRSRV_DEVICE_NODE * psDeviceNode,
+                                          IMG_HANDLE           hMemCtxPrivData);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXOverallocateBPRegistersKM
+
+ @Description
+	Server-side implementation of RGXOverallocateBPRegisters
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui32TempRegs - Number of temporary registers to overallocate
+ @Input ui32SharedRegs - Number of shared registers to overallocate
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXOverallocateBPRegistersKM(CONNECTION_DATA    * psConnection,
+                                                PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                IMG_UINT32           ui32TempRegs,
+                                                IMG_UINT32           ui32SharedRegs);
+#endif /* RGXBREAKPOINT_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxbvnc.c b/drivers/gpu/drm/img-rogue/rgxbvnc.c
new file mode 100644
index 000000000000..9f82ff1260bb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxbvnc.c
@@ -0,0 +1,960 @@
+/*************************************************************************/ /*!
+@File
+@Title          BVNC handling specific routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Functions used for BVNC related work
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxbvnc.h"
+#define RGXBVNC_C
+#include "rgx_bvnc_table_km.h"
+#undef RGXBVNC_C
+#include "os_apphint.h"
+#include "pvrsrv.h"
+#include "pdump_km.h"
+#include "rgx_compat_bvnc.h"
+#include "allocmem.h"
+
+#define RGX_FEATURE_TRUE_VALUE_TYPE_UINT16 (RGX_FEATURE_VALUE_TYPE_UINT16 >> RGX_FEATURE_TYPE_BIT_SHIFT)
+#define RGX_FEATURE_TRUE_VALUE_TYPE_UINT32 (RGX_FEATURE_VALUE_TYPE_UINT32 >> RGX_FEATURE_TYPE_BIT_SHIFT)
+#define RGXBVNC_BUFFER_SIZE (((PVRSRV_MAX_DEVICES)*(RGX_BVNC_STR_SIZE_MAX))+1)
+
+/* This function searches the given array for a given search value */
+static IMG_UINT64* _RGXSearchBVNCTable( IMG_UINT64 *pui64Array,
+								IMG_UINT uiEnd,
+								IMG_UINT64 ui64SearchValue,
+								IMG_UINT uiColCount)
+{
+	IMG_UINT uiStart = 0, index;
+	IMG_UINT64 value, *pui64Ptr = NULL;
+
+	while (uiStart < uiEnd)
+	{
+		index = (uiStart + uiEnd)/2;
+		pui64Ptr = pui64Array + (index * uiColCount);
+		value = *(pui64Ptr);
+
+		if (value == ui64SearchValue)
+		{
+			return pui64Ptr;
+		}
+
+		if (value > ui64SearchValue)
+		{
+			uiEnd = index;
+		}else
+		{
+			uiStart = index + 1;
+		}
+	}
+	return NULL;
+}
+#define RGX_SEARCH_BVNC_TABLE(t, b) (_RGXSearchBVNCTable((IMG_UINT64*)(t), \
+                                ARRAY_SIZE(t), (b), \
+                                sizeof((t)[0])/sizeof(IMG_UINT64)) )
+
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@brief		This function reads the (P)BVNC core_ID register and extracts
+			the BVNC configuration. Supports the old scheme and the newer
+			PBVNC scheme.
+@param		psDeviceNode - Device Node pointer
+@param		ui32CoreNum  - Core/bank number (0 for single core)
+@param		pB           - Address of branch value (output)
+@param		pV           - Address of version value (output)
+@param		pN           - Address of number of clusters/scalable shading units value (output)
+@param		pC           - Address of configuration value (output)
+@return		BVNC encoded in 64-bit value, 16-bits per field
+*/ /**************************************************************************/
+static
+IMG_UINT64 _RGXReadBVNCFromReg(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32CoreNum,
+							   IMG_UINT32 *pB, IMG_UINT32 *pV, IMG_UINT32 *pN, IMG_UINT32 *pC)
+{
+	IMG_UINT64 ui64BVNC;
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_CORE_ID__PBVNC)
+	/* Core ID reading code for Rogue */
+
+	/* Read the BVNC, in to new way first, if B not set, use old scheme */
+	ui64BVNC = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID__PBVNC + (ui32CoreNum << 16));
+
+	if (GET_B(ui64BVNC))
+	{
+		B = GET_PBVNC_B(ui64BVNC);
+		V = GET_PBVNC_V(ui64BVNC);
+		N = GET_PBVNC_N(ui64BVNC);
+		C = GET_PBVNC_C(ui64BVNC);
+	}
+	else
+	{
+		IMG_UINT64 ui32CoreID, ui32CoreRev;
+		ui32CoreRev = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_REVISION + (ui32CoreNum << 16));
+		ui32CoreID = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID + (ui32CoreNum << 16));
+		B = (ui32CoreRev & ~RGX_CR_CORE_REVISION_MAJOR_CLRMSK) >>
+												RGX_CR_CORE_REVISION_MAJOR_SHIFT;
+		V = (ui32CoreRev & ~RGX_CR_CORE_REVISION_MINOR_CLRMSK) >>
+												RGX_CR_CORE_REVISION_MINOR_SHIFT;
+		N = (ui32CoreID & ~RGX_CR_CORE_ID_CONFIG_N_CLRMSK) >>
+												RGX_CR_CORE_ID_CONFIG_N_SHIFT;
+		C = (ui32CoreID & ~RGX_CR_CORE_ID_CONFIG_C_CLRMSK) >>
+												RGX_CR_CORE_ID_CONFIG_C_SHIFT;
+		ui64BVNC = rgx_bvnc_pack(B, V, N, C);
+	}
+#else
+	/* Core ID reading code for Volcanic */
+
+	ui64BVNC = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CORE_ID + (ui32CoreNum << 16));
+
+	B = (ui64BVNC & ~RGX_CR_CORE_ID_BRANCH_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_BRANCH_ID_SHIFT;
+	V = (ui64BVNC & ~RGX_CR_CORE_ID_VERSION_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_VERSION_ID_SHIFT;
+	N = (ui64BVNC & ~RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_CLRMSK) >>
+											RGX_CR_CORE_ID_NUMBER_OF_SCALABLE_UNITS_SHIFT;
+	C = (ui64BVNC & ~RGX_CR_CORE_ID_CONFIG_ID_CLRMSK) >>
+											RGX_CR_CORE_ID_CONFIG_ID_SHIFT;
+#endif
+
+	*pB = B; *pV = V; *pN = N; *pC = C;
+	return ui64BVNC;
+}
+#endif
+
+#if defined(DEBUG) || defined(SUPPORT_PERFORMANCE_RUN)
+
+#define PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, szShortName, Feature)															\
+	if ( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] != RGX_FEATURE_VALUE_DISABLED )			\
+		{ PVR_LOG(("%s %d", szShortName, psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX])); }		\
+	else																\
+		{ PVR_LOG(("%s N/A", szShortName)); }
+
+static void _RGXBvncDumpParsedConfig(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT64 ui64Mask = 0, ui32IdOrNameIdx = 1;
+
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NC:       ", NUM_CLUSTERS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "CSF:      ", CDM_CONTROL_STREAM_FORMAT);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "FBCDCA:   ", FBCDC_ARCHITECTURE);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "META:     ", META);
+#if defined(RGX_FEATURE_META_COREMEM_BANKS_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MCMB:     ", META_COREMEM_BANKS);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MCMS:     ", META_COREMEM_SIZE);
+#if defined(RGX_FEATURE_META_DMA_CHANNEL_COUNT_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "MDMACnt:  ", META_DMA_CHANNEL_COUNT);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NIIP:     ", NUM_ISP_IPP_PIPES);
+#if defined(RGX_FEATURE_NUM_ISP_PER_SPU_MAX_VALUE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NIPS:     ", NUM_ISP_PER_SPU);
+#endif
+#if defined(RGX_FEATURE_PBE_PER_SPU_MAX_VALUE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "PPS:      ", PBE_PER_SPU);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NSPU:     ", NUM_SPU);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "PBW:      ", PHYS_BUS_WIDTH);
+#if defined(RGX_FEATURE_SCALABLE_TE_ARCH_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "STEArch:  ", SCALABLE_TE_ARCH);
+#endif
+#if defined(RGX_FEATURE_SCALABLE_VCE_IDX)
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SVCEA:    ", SCALABLE_VCE);
+#endif
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SLCBanks: ", SLC_BANKS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "SLCCLS:   ", SLC_CACHE_LINE_SIZE_BITS);
+	PVR_LOG(("SLCSize:   %d",  psDevInfo->sDevFeatureCfg.ui32SLCSizeInBytes));
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "VASB:     ", VIRTUAL_ADDRESS_SPACE_BITS);
+	PVR_LOG_DUMP_FEATURE_VALUE(psDevInfo, "NOSIDS:   ", NUM_OSIDS);
+
+#if defined(FEATURE_NO_VALUES_NAMES_MAX_IDX)
+	/* Dump the features with no values */
+	ui64Mask = psDevInfo->sDevFeatureCfg.ui64Features;
+	while (ui64Mask)
+	{
+		if (ui64Mask & 0x01)
+		{
+			if (ui32IdOrNameIdx <= FEATURE_NO_VALUES_NAMES_MAX_IDX)
+			{
+				PVR_LOG(("%s", gaszFeaturesNoValuesNames[ui32IdOrNameIdx - 1]));
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				         "Feature with Mask doesn't exist: 0x%016" IMG_UINT64_FMTSPECx,
+				         ((IMG_UINT64)1 << (ui32IdOrNameIdx - 1))));
+			}
+		}
+		ui64Mask >>= 1;
+		ui32IdOrNameIdx++;
+	}
+#endif
+
+#if defined(ERNSBRNS_IDS_MAX_IDX)
+	/* Dump the ERN and BRN flags for this core */
+	ui64Mask = psDevInfo->sDevFeatureCfg.ui64ErnsBrns;
+	ui32IdOrNameIdx = 1;
+
+	while (ui64Mask)
+	{
+		if (ui64Mask & 0x1)
+		{
+			if (ui32IdOrNameIdx <= ERNSBRNS_IDS_MAX_IDX)
+			{
+				PVR_LOG(("ERN/BRN : %d", gaui64ErnsBrnsIDs[ui32IdOrNameIdx - 1]));
+			}
+			else
+			{
+				PVR_LOG(("Unknown ErnBrn bit: 0x%0" IMG_UINT64_FMTSPECx, ((IMG_UINT64)1 << (ui32IdOrNameIdx - 1))));
+			}
+		}
+		ui64Mask >>= 1;
+		ui32IdOrNameIdx++;
+	}
+#endif
+
+#if !defined(ERNSBRNS_IDS_MAX_IDX) && !defined(FEATURE_NO_VALUES_NAMES_MAX_IDX)
+	PVR_UNREFERENCED_PARAMETER(ui64Mask);
+	PVR_UNREFERENCED_PARAMETER(ui32IdOrNameIdx);
+#endif
+
+}
+#endif
+
+static PVRSRV_ERROR _RGXBvncParseFeatureValues(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                               IMG_UINT64 *pui64Cfg)
+{
+	IMG_UINT32 ui32Index;
+
+	/* Read the feature values for the runtime BVNC */
+	for (ui32Index = 0; ui32Index < RGX_FEATURE_WITH_VALUES_MAX_IDX; ui32Index++)
+	{
+		IMG_UINT16 ui16BitPosition = aui16FeaturesWithValuesBitPositions[ui32Index];
+		IMG_UINT64 ui64PackedValues = pui64Cfg[2 + ui16BitPosition / 64];
+		IMG_UINT16 ui16ValueIndex = (ui64PackedValues & aui64FeaturesWithValuesBitMasks[ui32Index]) >> (ui16BitPosition % 64);
+
+		if (ui16ValueIndex >= gaFeaturesValuesMaxIndexes[ui32Index])
+		{
+			/* This case should never be reached */
+			psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] = RGX_FEATURE_VALUE_INVALID;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Feature with index (%d) decoded wrong value index (%d)", __func__, ui32Index, ui16ValueIndex));
+			PVR_ASSERT(ui16ValueIndex < gaFeaturesValuesMaxIndexes[ui32Index]);
+			return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+
+		switch (ui16BitPosition >> RGX_FEATURE_TYPE_BIT_SHIFT)
+		{
+			case RGX_FEATURE_TRUE_VALUE_TYPE_UINT16:
+			{
+				IMG_UINT16 *pui16FeatureValues = (IMG_UINT16*)gaFeaturesValues[ui32Index];
+				if (pui16FeatureValues[ui16ValueIndex] == (IMG_UINT16)RGX_FEATURE_VALUE_DISABLED)
+				{
+					psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+						RGX_FEATURE_VALUE_DISABLED;
+				}
+				else
+				{
+					psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+						pui16FeatureValues[ui16ValueIndex];
+				}
+				break;
+			}
+			case RGX_FEATURE_TRUE_VALUE_TYPE_UINT32:
+				psDevInfo->sDevFeatureCfg.ui32FeaturesValues[ui32Index] =
+					((IMG_UINT32*)gaFeaturesValues[ui32Index])[ui16ValueIndex];
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Feature with index %d has invalid feature type",
+				         __func__,
+				         ui32Index));
+				return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+	}
+
+
+#if defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX)
+	/* Code path for Volcanic */
+
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = RGXFWIF_DM_CDM+1;
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_RAY+1);
+	}
+#if defined(SUPPORT_AGP)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM2+1);
+#if defined(SUPPORT_AGP4)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM4+1);
+#endif
+#endif
+
+	/* Get the max number of dusts in the core */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS))
+	{
+		RGX_LAYER_PARAMS sParams = {.psDevInfo = psDevInfo};
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) == 1)
+		{
+			/* per SPU power island */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = MAX(1, (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) / 2));
+		}
+		else if (RGX_DEVICE_GET_FEATURE_VALUE(&sParams, POWER_ISLAND_VERSION) >= 2)
+		{
+			/* per Cluster power island */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS);
+		}
+		else
+		{
+			/* All volcanic cores support power islanding */
+			psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Power island feature version not found!", __func__));
+			PVR_ASSERT(0);
+			return PVRSRV_ERROR_FEATURE_DISABLED;
+		}
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+			RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+		{
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RT_RAC_PER_SPU))
+			{
+				psDevInfo->sDevFeatureCfg.ui32MAXRACCount = RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU);
+			}
+			else
+			{
+				psDevInfo->sDevFeatureCfg.ui32MAXRACCount = 1;
+			}
+		}
+	}
+	else
+	{
+		/* This case should never be reached as all cores have clusters */
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of clusters feature value missing!", __func__));
+		PVR_ASSERT(0);
+		return PVRSRV_ERROR_FEATURE_DISABLED;
+	}
+#else /* defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX) */
+	/* Code path for Rogue */
+
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = RGXFWIF_DM_CDM+1;
+#if defined(SUPPORT_AGP)
+	psDevInfo->sDevFeatureCfg.ui32MAXDMCount = MAX(psDevInfo->sDevFeatureCfg.ui32MAXDMCount, RGXFWIF_DM_GEOM2+1);
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_META_IDX] = RGX_FEATURE_VALUE_DISABLED;
+	}
+
+	/* Get the max number of dusts in the core */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS))
+	{
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = MAX(1, (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS) / 2));
+	}
+	else
+	{
+		/* This case should never be reached as all cores have clusters */
+		psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount = RGX_FEATURE_VALUE_INVALID;
+		PVR_DPF((PVR_DBG_ERROR, "%s: Number of clusters feature value missing!", __func__));
+		PVR_ASSERT(0);
+		return PVRSRV_ERROR_FEATURE_DISABLED;
+	}
+#endif /* defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX) */
+
+	/* Transform the META coremem size info in bytes */
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_META_COREMEM_SIZE_IDX] *= 1024;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RGXBvncAcquireAppHint(IMG_CHAR *pszBVNC, const IMG_UINT32 ui32RGXDevCount)
+{
+	const IMG_CHAR *pszAppHintDefault = PVRSRV_APPHINT_RGXBVNC;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32BVNCCount = 0;
+	IMG_BOOL bRet;
+	IMG_CHAR *pszBVNCAppHint;
+	IMG_CHAR *pszCurrentBVNC;
+	pszBVNCAppHint = (IMG_CHAR *)OSAllocMem(RGXBVNC_BUFFER_SIZE);
+	if (pszBVNCAppHint == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	pszBVNCAppHint[0] = '\0';
+
+	pszCurrentBVNC = pszBVNCAppHint;
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bRet = (IMG_BOOL)OSGetAppHintSTRING(APPHINT_NO_DEVICE,
+						pvAppHintState,
+						RGXBVNC,
+						pszAppHintDefault,
+						pszBVNCAppHint,
+						RGXBVNC_BUFFER_SIZE
+	);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	if (!bRet || (pszBVNCAppHint[0] == '\0'))
+	{
+		OSFreeMem(pszBVNCAppHint);
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC module param list: %s",__func__, pszBVNCAppHint));
+
+	while (*pszCurrentBVNC != '\0')
+	{
+		IMG_CHAR *pszNext = pszCurrentBVNC;
+
+		if (ui32BVNCCount >= PVRSRV_MAX_DEVICES)
+		{
+			break;
+		}
+
+		while (1)
+		{
+			if (*pszNext == ',')
+			{
+				pszNext[0] = '\0';
+				pszNext++;
+				break;
+			} else if (*pszNext == '\0')
+			{
+				break;
+			}
+			pszNext++;
+		}
+
+		if (ui32BVNCCount == ui32RGXDevCount)
+		{
+			OSStringSafeCopy(pszBVNC, pszCurrentBVNC, RGX_BVNC_STR_SIZE_MAX);
+			OSFreeMem(pszBVNCAppHint);
+			return PVRSRV_OK;
+		}
+
+		ui32BVNCCount++;
+		pszCurrentBVNC = pszNext;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Given module parameters list is shorter than "
+	"number of actual devices", __func__));
+
+	/* If only one BVNC parameter is specified, the same is applied for all RGX
+	 * devices detected */
+	if (1 == ui32BVNCCount)
+	{
+		OSStringSafeCopy(pszBVNC, pszBVNCAppHint, RGX_BVNC_STR_SIZE_MAX);
+	}
+
+	OSFreeMem(pszBVNCAppHint);
+
+	return PVRSRV_OK;
+}
+
+/* Function that parses the BVNC List passed as module parameter */
+static PVRSRV_ERROR _RGXBvncParseList(IMG_UINT32 *pB,
+									  IMG_UINT32 *pV,
+									  IMG_UINT32 *pN,
+									  IMG_UINT32 *pC,
+									  const IMG_UINT32 ui32RGXDevCount)
+{
+	IMG_CHAR aszBVNCString[RGX_BVNC_STR_SIZE_MAX];
+	IMG_CHAR *pcTemp, *pcNext;
+	PVRSRV_ERROR eError;
+
+	aszBVNCString[0] = '\0';
+
+	/* 4 components of a BVNC string is B, V, N & C */
+#define RGX_BVNC_INFO_PARAMS (4)
+
+	eError = _RGXBvncAcquireAppHint(aszBVNCString, ui32RGXDevCount);
+
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	if ('\0' == aszBVNCString[0])
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+
+	/* Parse the given RGX_BVNC string */
+	pcTemp = &aszBVNCString[0];
+	pcNext = strchr(pcTemp, '.');
+	if (pcNext == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+
+	*pcNext = '\0';
+	if (OSStringToUINT32(pcTemp, 0, pB) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	/* remove any 'p' from the V string, as this will
+	 * cause the call to OSStringToUINT32 to fail
+	 */
+	pcNext = strchr(pcTemp, 'p');
+	if (pcNext)
+	{
+		/* found one- - changing to '\0' */
+		*pcNext = '\0';
+		/* Move to next '.' */
+		pcNext++;
+	}
+	else
+	{
+		/* none found, so find next '.' and change to '\0' */
+		pcNext = strchr(pcTemp, '.');
+		if (pcNext == NULL)
+		{
+			return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+		}
+		*pcNext = '\0';
+	}
+	if (OSStringToUINT32(pcTemp, 0, pV) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	pcNext = strchr(pcTemp, '.');
+	if (pcNext == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	*pcNext = '\0';
+	if (OSStringToUINT32(pcTemp, 0, pN) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	pcTemp = pcNext+1;
+	if (OSStringToUINT32(pcTemp, 0, pC) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_INVALID_BVNC_PARAMS;
+	}
+	PVR_LOG(("BVNC module parameter honoured: %d.%d.%d.%d", *pB, *pV, *pN, *pC));
+
+	return PVRSRV_OK;
+}
+
+#if !defined(NO_HARDWARE)
+/*
+ * This function obtains the SLCSize from the physical device for GPUs which provide
+ * this information. If the GPU does not provide support we return a value of 0 which will
+ * result in the BVNC supplied definition being used to provide the SLCSize.
+ * Must only be called from driver-live with hardware powered-on.
+ */
+static IMG_UINT32 _RGXBvncReadSLCSize(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT64 ui64SLCSize = 0ULL;
+
+#if defined(RGX_CR_SLC_SIZE_IN_KB)
+	/* Rogue hardware */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_SIZE_CONFIGURABLE))
+	{
+		ui64SLCSize = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SLC_SIZE_IN_KB);
+		if (ui64SLCSize == 0ULL)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Unexpected 0 SLC size. Using default", __func__));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: RGX_CR_SIZE_IN_KB = %u", __func__,
+			        (IMG_UINT32) ui64SLCSize));
+		}
+	}
+#else
+	/* Volcanic hardware */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_SIZE_ADJUSTMENT))
+	{
+		ui64SLCSize = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SLC_STATUS2);
+		ui64SLCSize &= ~RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_CLRMSK;
+		ui64SLCSize >>= RGX_CR_SLC_STATUS2_SLC_SIZE_IN_KB_SHIFT;
+
+		if (ui64SLCSize == 0ULL)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Unexpected 0 SLC size. Using default", __func__));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: SLC_SIZE_IN_KILOBYTES = %u", __func__,
+			        (IMG_UINT32) ui64SLCSize));
+		}
+	}
+#endif
+
+	return (IMG_UINT32)ui64SLCSize * 1024U;
+}
+#endif /* !defined(NO_HARDWARE) */
+
+/* This function detects the Rogue variant and configures the essential
+ * config info associated with such a device.
+ * The config info includes features, errata, etc
+ */
+PVRSRV_ERROR RGXBvncInitialiseConfiguration(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	static IMG_UINT32 ui32RGXDevCnt = 0;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT64 ui64BVNC=0;
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+	IMG_UINT64 *pui64Cfg = NULL;
+	IMG_UINT32 ui32Cores = 1U;
+	IMG_UINT32 ui32SLCSize = 0;
+
+	/* Check for load time RGX BVNC parameter */
+	eError = _RGXBvncParseList(&B,&V,&N,&C, ui32RGXDevCnt);
+	if (PVRSRV_OK == eError)
+	{
+		PVR_LOG(("Read BVNC " RGX_BVNC_STR_FMTSPEC
+				" from driver load parameter", B, V, N, C));
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+		PVR_LOG_IF_FALSE((pui64Cfg != NULL), "Driver parameter BVNC configuration not found!");
+	}
+
+	{
+		void *pvAppHintState = NULL;
+		const IMG_BOOL bAppHintDefault = PVRSRV_APPHINT_IGNOREHWREPORTEDBVNC;
+
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							IgnoreHWReportedBVNC,
+							&bAppHintDefault,
+							&psDevInfo->bIgnoreHWReportedBVNC);
+		OSFreeAppHintState(pvAppHintState);
+	}
+
+#if !defined(NO_HARDWARE)
+
+	/* Try to detect the RGX BVNC from the HW device */
+	if ((NULL == pui64Cfg) && !psDevInfo->bIgnoreHWReportedBVNC)
+	{
+		IMG_BOOL bPowerDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+		}
+
+		/* Read the BVNC from HW */
+		_RGXReadBVNCFromReg(psDeviceNode, 0 /*core0*/, &B, &V, &N, &C);
+
+		PVR_LOG(("Read BVNC " RGX_BVNC_STR_FMTSPEC
+				" from HW device registers", B, V, N, C));
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Read the number of cores in the system for newer BVNC (Branch ID > 20) */
+			if (B > 20)
+			{
+				ui32Cores = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM);
+			}
+		}
+
+		/* Obtain the SLC size from the device */
+		ui32SLCSize = _RGXBvncReadSLCSize(psDeviceNode);
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: SLC Size reported as %u", __func__, ui32SLCSize));
+
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		if (ui64BVNC != 0)
+		{
+			pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+			PVR_LOG_IF_FALSE((pui64Cfg != NULL), "HW device BVNC configuration not found!");
+		}
+		else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/*
+			 * On host OS we should not get here as CORE_ID should not be zero, so flag an error.
+			 * On older cores, guest OS only has CORE_ID if defined(RGX_FEATURE_COREID_PER_OS)
+			 */
+			PVR_LOG_ERROR(PVRSRV_ERROR_DEVICE_REGISTER_FAILED, "CORE_ID register returns zero. Unknown BVNC");
+		}
+	}
+#endif
+
+#if defined(RGX_BVNC_KM_B) && defined(RGX_BVNC_KM_N) && defined(RGX_BVNC_KM_C)
+	if (NULL == pui64Cfg)
+	{
+		IMG_CHAR acVStr[5] = RGX_BVNC_KM_V_ST;
+
+		/* We reach here if the HW is not present,
+		 * or we are running in a guest OS with no COREID_PER_OS feature,
+		 * or HW is unstable during register read giving invalid values,
+		 * or runtime detection has been disabled - fall back to compile time BVNC
+		 */
+		B = RGX_BVNC_KM_B;
+		N = RGX_BVNC_KM_N;
+		C = RGX_BVNC_KM_C;
+
+		/* Clear any 'p' that may have been in RGX_BVNC_KM_V_ST,
+		 * as OSStringToUINT32() will otherwise return an error.
+		 */
+		if (acVStr[strlen(acVStr)-1] == 'p')
+		{
+			acVStr[strlen(acVStr)-1] = '\0';
+		}
+
+		if (OSStringToUINT32(&acVStr[0], 0, &V) != PVRSRV_OK)
+		{
+			V = 0;
+		}
+		PVR_LOG(("Reverting to compile time BVNC %s", RGX_BVNC_KM));
+
+		/* Extract the BVNC config from the Features table */
+		ui64BVNC = BVNC_PACK(B,0,N,C);
+		pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaFeatures, ui64BVNC);
+		PVR_LOG_IF_FALSE((pui64Cfg != NULL), "Compile time BVNC configuration not found!");
+	}
+#endif /* defined(RGX_BVNC) */
+
+	/* Have we failed to identify the BVNC to use? */
+	if (NULL == pui64Cfg)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: BVNC Detection and feature lookup failed. "
+		    "Unsupported BVNC: 0x%016" IMG_UINT64_FMTSPECx, __func__, ui64BVNC));
+		return PVRSRV_ERROR_BVNC_UNSUPPORTED;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC Feature found config: 0x%016"
+	    IMG_UINT64_FMTSPECx " 0x%016" IMG_UINT64_FMTSPECx " 0x%016"
+	    IMG_UINT64_FMTSPECx " 0x%016" IMG_UINT64_FMTSPECx "\n", __func__,
+	    pui64Cfg[0], pui64Cfg[1], pui64Cfg[2], pui64Cfg[3]));
+
+	/* Parsing feature config depends on available features on the core
+	 * hence this parsing should always follow the above feature assignment */
+	psDevInfo->sDevFeatureCfg.ui64Features = pui64Cfg[1];
+	eError = _RGXBvncParseFeatureValues(psDevInfo, pui64Cfg);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Add 'V' to the packed BVNC value to get the BVNC ERN and BRN config. */
+	ui64BVNC = BVNC_PACK(B,V,N,C);
+	pui64Cfg = RGX_SEARCH_BVNC_TABLE(gaErnsBrns, ui64BVNC);
+	if (NULL == pui64Cfg)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: BVNC ERN/BRN lookup failed. "
+		    "Unsupported BVNC: 0x%016" IMG_UINT64_FMTSPECx, __func__, ui64BVNC));
+		psDevInfo->sDevFeatureCfg.ui64ErnsBrns = 0;
+		return PVRSRV_ERROR_BVNC_UNSUPPORTED;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: BVNC ERN/BRN Cfg: 0x%016" IMG_UINT64_FMTSPECx
+	    " 0x%016" IMG_UINT64_FMTSPECx, __func__, *pui64Cfg, pui64Cfg[1]));
+	psDevInfo->sDevFeatureCfg.ui64ErnsBrns = pui64Cfg[1];
+
+	psDevInfo->sDevFeatureCfg.ui32B = B;
+	psDevInfo->sDevFeatureCfg.ui32V = V;
+	psDevInfo->sDevFeatureCfg.ui32N = N;
+	psDevInfo->sDevFeatureCfg.ui32C = C;
+
+
+	/*
+	 * Store the SLCSize in the device info field. If 0 it means the device uses the BVNC
+	 * values so grab them here as we've already populated the internal structures.
+	 */
+	if (ui32SLCSize == 0U)
+	{
+		ui32SLCSize = RGX_GET_FEATURE_VALUE(psDevInfo, SLC_SIZE_IN_KILOBYTES) * 1024U;
+
+		/* Verify that we have a valid value returned from the BVNC */
+		PVR_ASSERT(ui32SLCSize != 0U);
+	}
+	psDevInfo->sDevFeatureCfg.ui32SLCSizeInBytes = ui32SLCSize;
+
+	/* Message to confirm configuration look up was a success */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT) &&
+		!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(NO_HARDWARE)
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32Cores);
+			PVR_LOG(("RGX Device registered with BVNC " RGX_BVNC_STR_FMTSPEC,
+					B, V, N, C));
+		}
+#else
+		{
+			PVR_LOG(("RGX Device registered BVNC " RGX_BVNC_STR_FMTSPEC
+					" with %u %s in the system", B ,V ,N ,C, ui32Cores ,
+					((ui32Cores == 1U)?"core":"cores")));
+		}
+#endif
+	}
+	else
+	{
+		PVR_LOG(("RGX Device registered with BVNC " RGX_BVNC_STR_FMTSPEC,
+					B, V, N, C));
+	}
+
+	ui32RGXDevCnt++;
+
+#if defined(DEBUG) || defined(SUPPORT_PERFORMANCE_RUN)
+	_RGXBvncDumpParsedConfig(psDeviceNode);
+#endif
+	return PVRSRV_OK;
+}
+
+/*
+ * This function checks if a particular feature is available on the given rgx device */
+IMG_BOOL RGXBvncCheckFeatureSupported(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64FeatureMask)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->sDevFeatureCfg.ui64Features & ui64FeatureMask)
+	{
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+/*
+ * This function returns the value of a feature on the given rgx device */
+IMG_INT32 RGXBvncGetSupportedFeatureValue(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_FEATURE_WITH_VALUE_INDEX eFeatureIndex)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+
+	if (eFeatureIndex >= RGX_FEATURE_WITH_VALUES_MAX_IDX)
+	{
+		return -1;
+	}
+
+	if (psDevInfo->sDevFeatureCfg.ui32FeaturesValues[eFeatureIndex] == RGX_FEATURE_VALUE_DISABLED)
+	{
+		return -1;
+	}
+
+	return psDevInfo->sDevFeatureCfg.ui32FeaturesValues[eFeatureIndex];
+}
+
+/**************************************************************************/ /*!
+@Function       RGXVerifyBVNC
+@Description    Checks that the device's BVNC registers have the correct values.
+@Input          psDeviceNode		Device node
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+#define NUM_RGX_CORE_IDS    8
+PVRSRV_ERROR RGXVerifyBVNC(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT64 ui64MatchBVNC;
+#if !defined(NO_HARDWARE)
+	IMG_UINT32 B=0, V=0, N=0, C=0;
+#endif
+	IMG_UINT32 i;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	/* The device info */
+	psDevInfo = psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "PDUMP VERIFY CORE_ID registers for all OSIDs\n");
+
+	/* construct the value to match against */
+	if ((ui64GivenBVNC | ui64CoreIdMask) == 0) /* both zero means use configured DDK value */
+	{
+		ui64MatchBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+									psDevInfo->sDevFeatureCfg.ui32V,
+									psDevInfo->sDevFeatureCfg.ui32N,
+									psDevInfo->sDevFeatureCfg.ui32C);
+	}
+	else
+	{
+		/* use the value in CORE_ID for any zero elements in the BVNC */
+#if !defined(NO_HARDWARE)
+		IMG_UINT64 ui64BVNC = _RGXReadBVNCFromReg(psDeviceNode, 0, &B, &V, &N, &C);
+		ui64MatchBVNC = (ui64GivenBVNC & ~ui64CoreIdMask) | (ui64BVNC & ui64CoreIdMask);
+#else
+		ui64MatchBVNC = 0;
+#endif
+	}
+	PVR_LOG(("matchBVNC %d.%d.%d.%d",
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_B) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_V) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_N) & 0xffff),
+	        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_C) & 0xffff)));
+
+	/* read in all the CORE_ID registers */
+	for (i = 0; i < NUM_RGX_CORE_IDS; ++i)
+	{
+#if !defined(NO_HARDWARE)
+		IMG_UINT64 ui64BVNC = _RGXReadBVNCFromReg(psDeviceNode, i, &B, &V, &N, &C);
+		PVR_LOG(("CORE_ID%d returned %d.%d.%d.%d", i, B, V, N, C));
+
+		if (ui64BVNC != ui64MatchBVNC)
+		{
+			eError = PVRSRV_ERROR_BVNC_MISMATCH;
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid CORE_ID%d %d.%d.%d.%d, Expected %d.%d.%d.%d", __func__, i,
+					B, V, N, C,
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_B) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_V) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_N) & 0xffff),
+			        (int) ((ui64MatchBVNC >> RGX_BVNC_PACK_SHIFT_C) & 0xffff)));
+			break;
+		}
+#endif
+
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxbvnc.h b/drivers/gpu/drm/img-rogue/rgxbvnc.h
new file mode 100644
index 000000000000..64c418bc4edb
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxbvnc.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          BVNC handling specific header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the BVNC related work
+                (see hwdefs/km/rgx_bvnc_table_km.h and
+                hwdefs/km/rgx_bvnc_defs_km.h
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXBVNC_H)
+#define RGXBVNC_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "rgxdevice.h"
+
+/*************************************************************************/ /*!
+@brief		This function detects the Rogue variant and configures the
+			essential config info associated with such a device.
+			The config info includes features, errata, etc
+@param		psDeviceNode - Device Node pointer
+@return		PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXBvncInitialiseConfiguration(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@brief		This function checks if a particular feature is available on
+			the given rgx device
+@param		psDeviceNode - Device Node pointer
+@param		ui64FeatureMask - feature to be checked
+@return		true if feature is supported, false otherwise
+*/ /**************************************************************************/
+IMG_BOOL RGXBvncCheckFeatureSupported(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64FeatureMask);
+
+/*************************************************************************/ /*!
+@brief		This function returns the value of a feature on the given
+			rgx device
+@param		psDeviceNode - Device Node pointer
+@param		ui64FeatureMask - feature for which to return the value
+@return		the value for the specified feature
+*/ /**************************************************************************/
+IMG_INT32 RGXBvncGetSupportedFeatureValue(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_FEATURE_WITH_VALUE_INDEX eFeatureIndex);
+
+/*************************************************************************/ /*!
+@brief		This function validates that the BVNC values in CORE_ID regs are
+			consistent and correct.
+@param		psDeviceNode - Device Node pointer
+@param		GivenBVNC - BVNC to be verified against as supplied by caller
+@param		CoreIdMask - mask of components to pull from CORE_ID register
+@return		success or fail
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXVerifyBVNC(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64GivenBVNC, IMG_UINT64 ui64CoreIdMask);
+
+#endif /* RGXBVNC_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxccb.c b/drivers/gpu/drm/img-rogue/rgxccb.c
new file mode 100644
index 000000000000..b30a36d41157
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxccb.c
@@ -0,0 +1,2864 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX CCB routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX CCB routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgx_memallocflags.h"
+#include "devicemem_pdump.h"
+#include "dllist.h"
+#if defined(__linux__)
+#include "trace_events.h"
+#endif
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "rgxutils.h"
+#include "info_page.h"
+#include "rgxtimerquery.h"
+
+#if defined(PVRSRV_FORCE_FLUSH_CCCB_ON_KICK)
+#include "cache_km.h"
+#endif
+
+/*
+ *  Uncomment PVRSRV_ENABLE_CCCB_UTILISATION_INFO define for verbose
+ *  info and statistics regarding CCB usage.
+ */
+//#define PVRSRV_ENABLE_CCCB_UTILISATION_INFO
+
+/* Default threshold (as a percentage) for the PVRSRV_ENABLE_CCCB_UTILISATION_INFO feature. */
+#define PVRSRV_ENABLE_CCCB_UTILISATION_INFO_THRESHOLD  (90)
+
+/*
+ * Defines the number of fence updates to record so that future fences in the
+ * CCB. Can be checked to see if they are already known to be satisfied.
+ */
+#define RGX_CCCB_FENCE_UPDATE_LIST_SIZE  (32)
+
+#define RGX_UFO_PTR_ADDR(ufoptr) \
+	(((ufoptr)->puiAddrUFO.ui32Addr) & 0xFFFFFFFC)
+
+#define GET_CCB_SPACE(WOff, ROff, CCBSize) \
+	((((ROff) - (WOff)) + ((CCBSize) - 1)) & ((CCBSize) - 1))
+
+#define UPDATE_CCB_OFFSET(Off, PacketSize, CCBSize) \
+	(Off) = (((Off) + (PacketSize)) & ((CCBSize) - 1))
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_THRESHOLD 0x1
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED 0x2
+#define PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_FULL_CCB 0x4
+
+typedef struct _RGX_CLIENT_CCB_UTILISATION_
+{
+	/* the threshold in bytes.
+	 * when the CCB utilisation hits the threshold then we will print
+	 * a warning message.
+	 */
+	IMG_UINT32 ui32ThresholdBytes;
+	/* Maximum cCCB usage at some point in time */
+	IMG_UINT32 ui32HighWaterMark;
+	/* keep track of the warnings already printed.
+	 * bit mask of PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_xyz
+	 */
+	IMG_UINT32 ui32Warnings;
+	/* Keep track how many times CCB was full.
+	 * Counters are reset after every grow.
+	 */
+	IMG_UINT32 ui32CCBFull;
+	IMG_UINT32 ui32CCBAcquired;
+} RGX_CLIENT_CCB_UTILISATION;
+
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+struct _RGX_CLIENT_CCB_ {
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;				/*!< CPU mapping of the CCB control structure used by the fw */
+	void						*pvClientCCB;					/*!< CPU mapping of the CCB */
+	DEVMEM_MEMDESC				*psClientCCBMemDesc;			/*!< MemDesc for the CCB */
+	DEVMEM_MEMDESC				*psClientCCBCtrlMemDesc;		/*!< MemDesc for the CCB control */
+	IMG_UINT32					ui32HostWriteOffset;			/*!< CCB write offset from the driver side */
+	IMG_UINT32					ui32LastPDumpWriteOffset;		/*!< CCB write offset from the last time we submitted a command in capture range */
+	IMG_UINT32					ui32LastROff;					/*!< Last CCB Read offset to help detect any CCB wedge */
+	IMG_UINT32					ui32LastWOff;					/*!< Last CCB Write offset to help detect any CCB wedge */
+	IMG_UINT32					ui32ByteCount;					/*!< Count of the number of bytes written to CCCB */
+	IMG_UINT32					ui32LastByteCount;				/*!< Last value of ui32ByteCount to help detect any CCB wedge */
+	IMG_UINT32					ui32Size;						/*!< Size of the CCB */
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	POS_LOCK					hCCBGrowLock;					/*!< Prevents CCB Grow while DumpCCB() is called and vice versa */
+	IMG_UINT32					ui32VirtualAllocSize;			/*!< Virtual size of the CCB */
+	IMG_UINT32					ui32ChunkSize;					/*!< CCB Sparse allocation chunk size */
+	IMG_PUINT32					pui32MappingTable;				/*!< Mapping table for sparse allocation of the CCB */
+#endif
+	DLLIST_NODE					sNode;							/*!< Node used to store this CCB on the per connection list */
+	PDUMP_CONNECTION_DATA		*psPDumpConnectionData;			/*!< Pointer to the per connection data in which we reside */
+	void						*hTransition;					/*!< Handle for Transition callback */
+	IMG_CHAR					szName[MAX_CLIENT_CCB_NAME];	/*!< Name of this client CCB */
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;			/*!< Parent server common context that this CCB belongs to */
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	RGX_CCB_REQUESTOR_TYPE		eRGXCCBRequestor;
+	RGX_CLIENT_CCB_UTILISATION	sUtilisation;					/*!< CCB utilisation data */
+#endif
+#if defined(DEBUG)
+	IMG_UINT32					ui32UpdateEntries;				/*!< Number of Fence Updates in asFenceUpdateList */
+	RGXFWIF_UFO					asFenceUpdateList[RGX_CCCB_FENCE_UPDATE_LIST_SIZE]; /*!< List of recent updates written in this CCB */
+#endif
+	IMG_UINT32					ui32CCBFlags;                   /*!< Bitmask for various flags relating to CCB. Bit defines in rgxccb.h */
+};
+
+/* Forms a table, with array of strings for each requestor type (listed in RGX_CCB_REQUESTORS X macro), to be used for
+   DevMemAllocation comments and PDump comments. Each tuple in the table consists of 3 strings:
+	{ "FwClientCCB:" <requestor_name>, "FwClientCCBControl:" <requestor_name>, <requestor_name> },
+   The first string being used as comment when allocating ClientCCB for the given requestor, the second for CCBControl
+   structure, and the 3rd one for use in PDUMP comments. The number of tuples in the table must adhere to the following
+   build assert. */
+const IMG_CHAR *const aszCCBRequestors[][3] =
+{
+#define REQUESTOR_STRING(prefix,req) #prefix ":" #req
+#define FORM_REQUESTOR_TUPLE(req) { REQUESTOR_STRING(FwClientCCB,req), REQUESTOR_STRING(FwClientCCBControl,req), #req },
+	RGX_CCB_REQUESTORS(FORM_REQUESTOR_TUPLE)
+#undef FORM_REQUESTOR_TUPLE
+};
+
+PVRSRV_ERROR RGXCCBPDumpDrainCCB(RGX_CLIENT_CCB *psClientCCB,
+						IMG_UINT32 ui32PDumpFlags)
+{
+
+	IMG_UINT32 ui32PollOffset;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+#endif
+
+	/* Command to a finished CCB stream and FW is drained to empty
+	 * out remaining commands until R==W.
+	 */
+	ui32PollOffset = psClientCCB->ui32LastPDumpWriteOffset;
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+						  ui32PDumpFlags,
+						  "cCCB(%s@%p): Draining CCB rgxfw_roff == woff (%d)",
+						  psClientCCB->szName,
+						  psClientCCB,
+						  ui32PollOffset);
+
+	return DevmemPDumpDevmemPol32(psClientCCB->psClientCCBCtrlMemDesc,
+									offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+									ui32PollOffset,
+									0xffffffff,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXCCBPDumpSyncCCB
+
+ PURPOSE	: Synchronise Client CCBs from both live and playback contexts.
+               Waits for live-FW to empty live-CCB.
+               Waits for sim-FW to empty sim-CCB by adding POL
+
+ PARAMETERS	: psClientCCB		- The client CCB
+              ui32PDumpFlags    - PDump flags
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXCCBPDumpSyncCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Wait for the live FW to catch up/empty CCB. This is done by returning
+	 * retry which will get pushed back out to Services client where it
+	 * waits on the event object and then resubmits the command.
+	 */
+	if (psClientCCB->psClientCCBCtrl->ui32ReadOffset != psClientCCB->ui32HostWriteOffset)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Wait for the sim FW to catch up/empty sim CCB.
+	 * We drain whenever capture range is entered, even if no commands
+	 * have been issued on this CCB when out of capture range. We have to
+	 * wait for commands that might have been issued in the last capture
+	 * range to finish so the connection's sync block snapshot dumped after
+	 * all the PDumpTransition callbacks have been execute doesn't clobber
+	 * syncs which the sim FW is currently working on.
+	 *
+	 * Although this is sub-optimal for play-back - while out of capture
+	 * range for every continuous operation we synchronise the sim
+	 * play-back processing the script and the sim FW, there is no easy
+	 * solution. Not all modules that work with syncs register a
+	 * PDumpTransition callback and thus we have no way of knowing if we
+	 * can skip this sim CCB drain and sync block dump or not.
+	 */
+
+	eError = RGXCCBPDumpDrainCCB(psClientCCB, ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "RGXCCBPDumpDrainCCB");
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* Live CCB and simulation CCB now empty, FW idle on CCB in both
+	 * contexts.
+	 */
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXCCBPDumpFastForwardCCB
+
+ PURPOSE	: Fast-forward sim-CCB and live-CCB offsets to live app-thread
+              values.
+               This helps to skip any commands submitted when out of capture
+               range and start with first command in capture range in both
+               live and playback contexts. In case of Block mode, this helps
+               to playback any intermediate PDump block directly after first
+               block.
+
+
+ PARAMETERS	: psClientCCB		- The client CCB
+			  ui32PDumpFlags    - PDump flags
+
+ RETURNS	: void
+******************************************************************************/
+static void RGXCCBPDumpFastForwardCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32PDumpFlags)
+{
+	volatile RGXFWIF_CCCB_CTL *psCCBCtl = psClientCCB->psClientCCBCtrl;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+#endif
+
+	/* Make sure that we have synced live-FW and live-App threads */
+	PVR_ASSERT(psCCBCtl->ui32ReadOffset == psClientCCB->ui32HostWriteOffset);
+
+	psCCBCtl->ui32ReadOffset = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32DepOffset = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32WriteOffset = psClientCCB->ui32HostWriteOffset;
+
+	psCCBCtl->ui32ReadOffset2 = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32ReadOffset3 = psClientCCB->ui32HostWriteOffset;
+	psCCBCtl->ui32ReadOffset4 = psClientCCB->ui32HostWriteOffset;
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+			ui32PDumpFlags,
+			"cCCB(%s@%p): Fast-forward from %d to %d",
+			psClientCCB->szName,
+			psClientCCB,
+			psClientCCB->ui32LastPDumpWriteOffset,
+			psClientCCB->ui32HostWriteOffset);
+
+	DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+			0,
+			sizeof(RGXFWIF_CCCB_CTL),
+			ui32PDumpFlags);
+
+	/* Although we've entered capture range for this process connection
+	 * we might not do any work on this CCB so update the
+	 * ui32LastPDumpWriteOffset to reflect where we got to for next
+	 * time so we start the drain from where we got to last time.
+	 */
+	psClientCCB->ui32LastPDumpWriteOffset = psClientCCB->ui32HostWriteOffset;
+
+}
+
+static PVRSRV_ERROR _RGXCCBPDumpTransition(void *pvData, void *pvDevice, PDUMP_TRANSITION_EVENT eEvent, IMG_UINT32 ui32PDumpFlags)
+{
+	RGX_CLIENT_CCB *psClientCCB = (RGX_CLIENT_CCB *) pvData;
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) pvDevice;
+#else
+	PVR_UNREFERENCED_PARAMETER(pvDevice);
+#endif
+
+	/* Block mode:
+	 * Here is block structure at transition (ui32BlockLength=N frames):
+	 *
+	 * ...
+	 * ...
+	 * PDUMP_BLOCK_START_0x0000000x{
+	 *   <Fast-forward sim-CCCB>
+	 *   <Re-dump SyncBlocks>
+	 *   ...
+	 *   ...
+	 *   ... (N frames data)
+	 *   ...
+	 *   ...
+	 *   <(1) Drain sim-KCCB>                     ''|
+	 *   <(2) Sync live and sim CCCB>               |
+	 * }PDUMP_BLOCK_END_0x0000000x                  | <- BlockTransition Steps
+	 *   <(3) Split MAIN and BLOCK stream script>   |
+	 * PDUMP_BLOCK_START_0x0000000y{                |
+	 *   <(4) Fast-forward sim-CCCB>                |
+	 *   <(5) Re-dump SyncBlocks>                 ,,|
+	 *   ...
+	 *   ...
+	 *   ... (N frames data)
+	 *   ...
+	 *   ...
+	 *   <Drain sim-KCCB>
+	 *   <Sync live and sim CCCB>
+	 * }PDUMP_BLOCK_END_0x0000000y
+	 * ...
+	 * ...
+	 *
+	 * Steps (3) and (5) are done in pdump_server.c
+	 * */
+	switch (eEvent)
+	{
+		case PDUMP_TRANSITION_EVENT_RANGE_ENTERED:
+			{
+				/* We're about to transition into capture range and we've submitted
+				 * new commands since the last time we entered capture range so drain
+				 * the live CCB and simulation (sim) CCB as required, i.e. leave CCB
+				 * idle in both live and sim contexts.
+				 * This requires the host driver to ensure the live FW & the sim FW
+				 * have both emptied out the remaining commands until R==W (CCB empty).
+				 */
+
+				eError = RGXCCBPDumpSyncCCB(psClientCCB, ui32PDumpFlags);
+				PVR_RETURN_IF_ERROR(eError);
+
+				if (psClientCCB->ui32LastPDumpWriteOffset != psClientCCB->ui32HostWriteOffset)
+				{
+					/* If new commands have been written when out of capture range in
+					 * the live CCB then we need to fast forward the sim CCBCtl
+					 * offsets past uncaptured commands. This is done by PDUMPing
+					 * the CCBCtl memory to align sim values with the live CCBCtl
+					 * values. Both live and sim FWs can start with the 1st command
+					 * which is in the new capture range.
+					 */
+					RGXCCBPDumpFastForwardCCB(psClientCCB, ui32PDumpFlags);
+				}
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_RANGE_APPEND:
+			{
+				/* Capture range already entered. Skip dump sync and fast forward.
+				 */
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_RANGE_EXITED:
+			{
+				/* Nothing to do */
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_BLOCK_FINISHED:
+			{
+				/* (1) Drain KCCB from current block before starting new:
+				 *
+				 * At playback, this will ensure that sim-FW drains all commands in KCCB
+				 * belongs to current block before 'jumping' to any future commands (from
+				 * next block). This will synchronise script-thread and sim-FW thread KCCBs
+				 * at end of each pdump block.
+				 *
+				 * This will additionally force redump of KCCBCtl structure at start of next/new block.
+				 * */
+
+#if defined(PDUMP)
+				eError = RGXPdumpDrainKCCB(psDevInfo, psDevInfo->psKernelCCBCtl->ui32WriteOffset);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXPdumpDrainKCCB");
+#endif
+
+				/* (2) Synchronise Client CCBs from live and playback contexts before starting new block:
+				 *
+				 * This operation will,
+				 * a. Force synchronisation between app-thread and live-FW thread (i.e. Wait
+				 *    for live-FW to empty live Client CCB).
+				 *
+				 * b. Next, it will dump poll command to drain Client CCB at end of every
+				 *    pdump block. At playback time this will synchronise sim-FW and
+				 *    script-thread Client CCBs at end of each block.
+				 *
+				 * This is to ensure that all commands in CCB from current block are processed
+				 * before moving on to future commands.
+				 * */
+
+				eError = RGXCCBPDumpSyncCCB(psClientCCB, ui32PDumpFlags);
+				PVR_RETURN_IF_ERROR(eError);
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_BLOCK_STARTED:
+			{
+				/* (4) Fast-forward CCB write offsets to current live values:
+				 *
+				 * We have already synchronised live-FW and app-thread above at end of each
+				 * block (in Step 2a above), now fast-forward Client CCBCtl write offsets to that of
+				 * current app-thread values at start of every block. This will allow us to
+				 * skip any intermediate pdump blocks and start with last (or any next) block
+				 * immediately after first pdump block.
+				 * */
+
+				RGXCCBPDumpFastForwardCCB(psClientCCB, ui32PDumpFlags);
+				break;
+			}
+		case PDUMP_TRANSITION_EVENT_NONE:
+			/* Invalid event for transition */
+		default:
+			{
+				/* Unknown Transition event */
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+	}
+	return PVRSRV_OK;
+}
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+
+static INLINE void _RGXInitCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	psClientCCB->sUtilisation.ui32HighWaterMark = 0; /* initialize ui32HighWaterMark level to zero */
+	psClientCCB->sUtilisation.ui32ThresholdBytes = (psClientCCB->ui32Size *
+							PVRSRV_ENABLE_CCCB_UTILISATION_INFO_THRESHOLD)	/ 100;
+	psClientCCB->sUtilisation.ui32Warnings = 0;
+	psClientCCB->sUtilisation.ui32CCBAcquired = 0;
+	psClientCCB->sUtilisation.ui32CCBFull = 0;
+}
+
+static INLINE void _RGXCCBUtilisationEvent(RGX_CLIENT_CCB *psClientCCB,
+						IMG_UINT32 ui32WarningType,
+						IMG_UINT32 ui32CmdSize)
+{
+	/* in VERBOSE mode we will print a message for each different
+	 * event type as they happen.
+	 */
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	if (!(psClientCCB->sUtilisation.ui32Warnings & ui32WarningType))
+	{
+		if (ui32WarningType == PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED)
+		{
+			PVR_LOG(("Failed to acquire CCB space for %u byte command:", ui32CmdSize));
+		}
+
+		PVR_LOG(("%s: Client CCB (%s) watermark (%u) hit %d%% of its allocation size (%u)",
+									__func__,
+									psClientCCB->szName,
+									psClientCCB->sUtilisation.ui32HighWaterMark,
+									psClientCCB->sUtilisation.ui32HighWaterMark * 100 / psClientCCB->ui32Size,
+									psClientCCB->ui32Size));
+
+		/* record that we have issued a warning of this type */
+		psClientCCB->sUtilisation.ui32Warnings |= ui32WarningType;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psClientCCB);
+	PVR_UNREFERENCED_PARAMETER(ui32WarningType);
+	PVR_UNREFERENCED_PARAMETER(ui32CmdSize);
+#endif
+}
+
+/* Check the current CCB utilisation. Print a one-time warning message if it is above the
+ * specified threshold
+ */
+static INLINE void _RGXCheckCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	/* Print a warning message if the cCCB watermark is above the threshold value */
+	if (psClientCCB->sUtilisation.ui32HighWaterMark >= psClientCCB->sUtilisation.ui32ThresholdBytes)
+	{
+		_RGXCCBUtilisationEvent(psClientCCB,
+					PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_THRESHOLD,
+					0);
+	}
+}
+
+/* Update the cCCB high watermark level if necessary */
+static void _RGXUpdateCCBUtilisation(RGX_CLIENT_CCB *psClientCCB)
+{
+	IMG_UINT32 ui32FreeSpace, ui32MemCurrentUsage;
+
+	RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+	                           INVALIDATE);
+
+	ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+									  psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+									  psClientCCB->ui32Size);
+	ui32MemCurrentUsage = psClientCCB->ui32Size - ui32FreeSpace;
+
+	if (ui32MemCurrentUsage > psClientCCB->sUtilisation.ui32HighWaterMark)
+	{
+		psClientCCB->sUtilisation.ui32HighWaterMark = ui32MemCurrentUsage;
+
+		/* The high water mark has increased. Check if it is above the
+		 * threshold so we can print a warning if necessary.
+		 */
+		_RGXCheckCCBUtilisation(psClientCCB);
+	}
+}
+
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+PVRSRV_ERROR RGXCreateCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+						  IMG_UINT32			ui32CCBSizeLog2,
+						  IMG_UINT32			ui32CCBMaxSizeLog2,
+						  IMG_UINT32			ui32ContextFlags,
+						  CONNECTION_DATA		*psConnectionData,
+						  RGX_CCB_REQUESTOR_TYPE		eRGXCCBRequestor,
+						  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+						  RGX_CLIENT_CCB		**ppsClientCCB,
+						  DEVMEM_MEMDESC		**ppsClientCCBMemDesc,
+						  DEVMEM_MEMDESC		**ppsClientCCBCtrlMemDesc)
+{
+	PVRSRV_ERROR	eError = PVRSRV_OK;
+	PVRSRV_MEMALLOCFLAGS_T	uiClientCCBMemAllocFlags, uiClientCCBCtlMemAllocFlags;
+	IMG_UINT32		ui32FWMainLog2PageSize = DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap);
+	IMG_UINT32		ui32ChunkSize = (1U << ui32FWMainLog2PageSize);
+	IMG_UINT32		ui32AllocSize = MAX((1U << ui32CCBSizeLog2), ui32ChunkSize);
+	IMG_UINT32		ui32MinAllocSize = MAX((1U << MIN_SAFE_CCB_SIZE_LOG2), ui32ChunkSize);
+	RGX_CLIENT_CCB	*psClientCCB;
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	IMG_UINT32		ui32NumChunks = ui32AllocSize / ui32ChunkSize;
+	IMG_UINT32		ui32VirtualAllocSize = (1U << ui32CCBMaxSizeLog2);
+	IMG_UINT32		ui32NumVirtChunks = ui32VirtualAllocSize / ui32ChunkSize;
+	IMG_UINT32		i;
+
+	/* For the allocation request to be valid, at least one page is required.
+	 * This is relevant on systems where the page size is greater than the client CCB size. */
+	ui32NumVirtChunks = MAX(1, ui32NumVirtChunks);
+	PVR_ASSERT((ui32ChunkSize >= (1U << PAGE_SHIFT)));
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32CCBMaxSizeLog2);
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+	/* All client CCBs should be at-least of the "minimum" size and not to exceed "maximum" */
+	if ((ui32CCBSizeLog2 < MIN_SAFE_CCB_SIZE_LOG2) ||
+		(ui32CCBSizeLog2 > MAX_SAFE_CCB_SIZE_LOG2))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: %s CCB size is invalid (%d). Should be from %d to %d",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         ui32CCBSizeLog2, MIN_SAFE_CCB_SIZE_LOG2, MAX_SAFE_CCB_SIZE_LOG2));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if ((ui32CCBMaxSizeLog2 < ui32CCBSizeLog2) ||
+		(ui32CCBMaxSizeLog2 > MAX_SAFE_CCB_SIZE_LOG2))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: %s CCB maximum size is invalid (%d). Should be from %d to %d",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         ui32CCBMaxSizeLog2, ui32CCBSizeLog2, MAX_SAFE_CCB_SIZE_LOG2));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+
+	psClientCCB = OSAllocMem(sizeof(*psClientCCB));
+	if (psClientCCB == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+	psClientCCB->psServerCommonContext = psServerCommonContext;
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	psClientCCB->ui32VirtualAllocSize = 0;
+	psClientCCB->pui32MappingTable = NULL;
+	psClientCCB->ui32ChunkSize = ui32ChunkSize;
+#endif
+
+	uiClientCCBMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+								PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+								PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+								PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+#if defined(DEBUG)
+								PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC |
+#endif
+								PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+								PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	uiClientCCBCtlMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+								PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+								PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+								PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+								PVRSRV_MEMALLOCFLAG_CPU_UNCACHED |
+								PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+								PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+								PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	/* If connection data indicates Sync Lockup Recovery (SLR) should be disabled,
+	 * or if the caller has set ui32ContextFlags to disable SLR for this context,
+	 * indicate this in psClientCCB->ui32CCBFlags.
+	 */
+	if ((psConnectionData->ui32ClientFlags & SRV_FLAGS_CLIENT_SLR_DISABLED) ||
+	    (ui32ContextFlags & RGX_CONTEXT_FLAG_DISABLESLR))
+	{
+		BIT_SET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGXFW cCCB");
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if (BITMASK_HAS(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN))
+	{
+		PHYS_HEAP *psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+		PHYS_HEAP_POLICY uiHeapPolicy = PhysHeapGetPolicy(psPhysHeap);
+
+		psClientCCB->ui32VirtualAllocSize = ui32VirtualAllocSize;
+
+		if (uiHeapPolicy != PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+		{
+			psClientCCB->pui32MappingTable = NULL;
+			/*
+			 * On LMA sparse memory can't be mapped to kernel without support for non physically
+			 * sparse allocations.
+			 * To work around this whole ccb memory is allocated at once as contiguous.
+			 */
+			eError = DevmemFwAllocate(psDevInfo,
+									ui32VirtualAllocSize,
+									uiClientCCBMemAllocFlags,
+									aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+									&psClientCCB->psClientCCBMemDesc);
+		}
+		else
+		{
+			/*
+			 * Growing CCB is doubling the size. Last grow would require only ui32NumVirtChunks/2 new chunks
+			 * because another ui32NumVirtChunks/2 is already allocated.
+			 * Sometimes initial chunk count would be higher (when CCB size is equal to CCB maximum size) so MAX is needed.
+			 */
+			psClientCCB->pui32MappingTable = OSAllocMem(MAX(ui32NumChunks, ui32NumVirtChunks/2) * sizeof(IMG_UINT32));
+			if (psClientCCB->pui32MappingTable == NULL)
+			{
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_mtable;
+			}
+
+			for (i = 0; i < ui32NumChunks; i++)
+			{
+				psClientCCB->pui32MappingTable[i] = i;
+			}
+
+			eError = DevmemFwAllocateSparse(psDevInfo,
+											ui32VirtualAllocSize,
+											ui32NumChunks,
+											ui32NumVirtChunks,
+											psClientCCB->pui32MappingTable,
+											uiClientCCBMemAllocFlags,
+											aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+											&psClientCCB->psClientCCBMemDesc);
+		}
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+		psClientCCB->pui32MappingTable = NULL;
+		psClientCCB->ui32VirtualAllocSize = 0;
+	}
+
+	if (!BITMASK_HAS(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN) ||
+	    (eError != PVRSRV_OK))
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+	{
+		/* Allocate ui32AllocSize, or the next best POT allocation */
+		do
+		{
+			eError = DevmemFwAllocate(psDevInfo,
+									ui32AllocSize,
+									uiClientCCBMemAllocFlags,
+									aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_STRING],
+									&psClientCCB->psClientCCBMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				/* Failed to allocate - ensure CCB grow is disabled from
+				 * now on for this device.
+				 */
+				BITMASK_UNSET(psDevInfo->ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+
+				/* Failed to allocate, try next POT down */
+				ui32AllocSize >>= 1;
+			}
+		} while ((eError != PVRSRV_OK) && (ui32AllocSize > ui32MinAllocSize));
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate RGX client CCB (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_alloc_ccb;
+	}
+
+	OSSNPrintf(psClientCCB->szName, MAX_CLIENT_CCB_NAME, "%s-P%lu-T%lu-%s",
+									aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+									(unsigned long) OSGetCurrentClientProcessIDKM(),
+									(unsigned long) OSGetCurrentClientThreadIDKM(),
+									OSGetCurrentClientProcessNameKM());
+
+	if (ui32AllocSize < (1U << ui32CCBSizeLog2))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Unable to allocate %d bytes for RGX client CCB (%s) but allocated %d bytes",
+		         __func__,
+		         (1U << ui32CCBSizeLog2),
+		         psClientCCB->szName,
+		         ui32AllocSize));
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									  &psClientCCB->pvClientCCB);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map RGX client CCB (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_map_ccb;
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGXFW cCCB control");
+	eError = DevmemFwAllocate(psDevInfo,
+										sizeof(RGXFWIF_CCCB_CTL),
+										uiClientCCBCtlMemAllocFlags,
+										aszCCBRequestors[eRGXCCBRequestor][REQ_RGX_FW_CLIENT_CCB_CONTROL_STRING],
+										&psClientCCB->psClientCCBCtrlMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate RGX client CCB control (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_alloc_ccbctrl;
+	}
+
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc,
+									  (void **) &psClientCCB->psClientCCBCtrl);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map RGX client CCB control (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_map_ccbctrl;
+	}
+
+	/* psClientCCBCtrlMemDesc was zero alloc'd so no need to initialise offsets. */
+	psClientCCB->psClientCCBCtrl->ui32WrapMask = ui32AllocSize - 1;
+
+	/* Flush the whole struct since other parts are implicitly init (zero'd) */
+	RGXFwSharedMemCacheOpPtr(psClientCCB->psClientCCBCtrl,
+	                         FLUSH);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "cCCB control");
+	DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCCB_CTL),
+					   PDUMP_FLAGS_CONTINUOUS);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psClientCCB->ui32HostWriteOffset = 0;
+	psClientCCB->ui32LastPDumpWriteOffset = 0;
+	psClientCCB->ui32Size = ui32AllocSize;
+	psClientCCB->ui32LastROff = ui32AllocSize - 1;
+	psClientCCB->ui32ByteCount = 0;
+	psClientCCB->ui32LastByteCount = 0;
+	BIT_UNSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	eError = OSLockCreate(&psClientCCB->hCCBGrowLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create hCCBGrowLock (%s)",
+		         __func__,
+		         PVRSRVGetErrorString(eError)));
+		goto fail_create_ccbgrow_lock;
+	}
+#endif
+#if defined(DEBUG)
+	psClientCCB->ui32UpdateEntries = 0;
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXInitCCBUtilisation(psClientCCB);
+	psClientCCB->eRGXCCBRequestor = eRGXCCBRequestor;
+#endif
+	eError = PDumpRegisterTransitionCallback(psConnectionData->psPDumpConnectionData,
+											  _RGXCCBPDumpTransition,
+											  psClientCCB,
+											  psDevInfo,
+											  &psClientCCB->hTransition);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_pdumpreg;
+	}
+
+	/*
+	 * Note:
+	 * Save the PDump specific structure, which is ref counted unlike
+	 * the connection data, to ensure it's not freed too early
+	 */
+	psClientCCB->psPDumpConnectionData = psConnectionData->psPDumpConnectionData;
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "New RGXFW cCCB(%s@%p) created",
+				 psClientCCB->szName,
+				 psClientCCB);
+
+	*ppsClientCCB = psClientCCB;
+	*ppsClientCCBMemDesc = psClientCCB->psClientCCBMemDesc;
+	*ppsClientCCBCtrlMemDesc = psClientCCB->psClientCCBCtrlMemDesc;
+	return PVRSRV_OK;
+
+fail_pdumpreg:
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockDestroy(psClientCCB->hCCBGrowLock);
+fail_create_ccbgrow_lock:
+#endif
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc);
+fail_map_ccbctrl:
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBCtrlMemDesc);
+fail_alloc_ccbctrl:
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+fail_map_ccb:
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBMemDesc);
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+fail_alloc_ccb:
+	if (psClientCCB->pui32MappingTable)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+	}
+fail_alloc_mtable:
+#else
+fail_alloc_ccb:
+#endif
+	OSFreeMem(psClientCCB);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+void RGXDestroyCCB(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_CLIENT_CCB *psClientCCB)
+{
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	if (psClientCCB->sUtilisation.ui32CCBFull)
+	{
+		PVR_LOG(("CCBUtilisationInfo: GPU %s command buffer was full %d times out of %d. "
+				"This is not an error but the application may not run optimally.",
+				aszCCBRequestors[psClientCCB->eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+				psClientCCB->sUtilisation.ui32CCBFull,
+				psClientCCB->sUtilisation.ui32CCBAcquired));
+	}
+#endif
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockDestroy(psClientCCB->hCCBGrowLock);
+#endif
+	PDumpUnregisterTransitionCallback(psClientCCB->hTransition);
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBCtrlMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBCtrlMemDesc);
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psClientCCB->psClientCCBMemDesc);
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	if (psClientCCB->pui32MappingTable)
+	{
+		OSFreeMem(psClientCCB->pui32MappingTable);
+	}
+#endif
+	OSFreeMem(psClientCCB);
+}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+static PVRSRV_ERROR _RGXCCBMemChangeSparse(RGX_CLIENT_CCB *psClientCCB,
+										  IMG_UINT32 ui32AllocPageCount)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32	 i;
+
+	DevmemReleaseCpuVirtAddr(psClientCCB->psClientCCBMemDesc);
+
+	for (i = 0; i < ui32AllocPageCount; i++)
+	{
+		psClientCCB->pui32MappingTable[i] = ui32AllocPageCount + i;
+	}
+
+	/* Double the CCB size (CCB must be POT) by adding ui32AllocPageCount new pages */
+	eError = DeviceMemChangeSparse(psClientCCB->psClientCCBMemDesc,
+									ui32AllocPageCount,
+									psClientCCB->pui32MappingTable,
+									0,
+									NULL,
+									SPARSE_RESIZE_ALLOC);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to grow RGX client CCB (%s)",
+				PVRSRVGetErrorString(eError)));
+
+		if (DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									&psClientCCB->pvClientCCB) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to reacquire CCB mapping"));
+			psClientCCB->pvClientCCB = NULL;
+		}
+
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psClientCCB->psClientCCBMemDesc,
+									&psClientCCB->pvClientCCB);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXAcquireCCB: Failed to map RGX client CCB (%s)",
+				PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+PVRSRV_ERROR RGXCheckSpaceCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32CmdSize)
+{
+	IMG_UINT32 ui32FreeSpace;
+
+	/* Check that the CCB can hold this command + padding */
+	if ((ui32CmdSize + PADDING_COMMAND_SIZE + 1) > psClientCCB->ui32Size)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Command size (%d bytes) too big for CCB"
+		        " (%d bytes)", ui32CmdSize, psClientCCB->ui32Size));
+		return PVRSRV_ERROR_CMD_TOO_BIG;
+	}
+
+	/*
+		Check we don't overflow the end of the buffer and make sure we have
+		enough space for the padding command. If we don't have enough space
+		(including the minimum amount for the padding command) we need to make
+		sure we insert a padding command now and wrap before adding the main
+		command.
+	*/
+
+	/* Invalidate read offset */
+	RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+	                           INVALIDATE);
+	if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) <= psClientCCB->ui32Size)
+	{
+		ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+		                              psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+		                              psClientCCB->ui32Size);
+
+		/* Don't allow all the space to be used */
+		if (ui32FreeSpace > ui32CmdSize)
+		{
+			return PVRSRV_OK;
+		}
+
+		goto e_retry;
+	}
+	else
+	{
+		IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+		ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+		                              psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+		                              psClientCCB->ui32Size);
+
+		/* Check there is space for both the command and the padding command */
+		if (ui32FreeSpace > ui32Remain + ui32CmdSize)
+		{
+			return PVRSRV_OK;
+		}
+
+		goto e_retry;
+	}
+
+e_retry:
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXCCBUtilisationEvent(psClientCCB,
+	            PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_FULL_CCB,
+	            ui32CmdSize);
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+
+	return PVRSRV_ERROR_RETRY;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireCCB
+
+ PURPOSE	: Obtains access to write some commands to a CCB
+
+ PARAMETERS	: psClientCCB		- The client CCB
+			  ui32CmdSize		- How much space is required
+			  ppvBufferSpace	- Pointer to space in the buffer
+			  ui32PDumpFlags - Should this be PDump continuous?
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXAcquireCCB(RGX_CLIENT_CCB *psClientCCB,
+										IMG_UINT32		ui32CmdSize,
+										void			**ppvBufferSpace,
+										IMG_UINT32		ui32PDumpFlags)
+{
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	IMG_UINT32	ui32RetryCount = 2;
+#endif
+
+#if defined(PDUMP)
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_BOOL	bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, ui32PDumpFlags);
+	IMG_BOOL	bPDumpFlagsContinuous = PDUMP_IS_CONTINUOUS(ui32PDumpFlags);
+
+	/*
+		PDumpSetFrame will detect as we Transition into capture range for
+		frame based data but if we are PDumping continuous data then we
+		need to inform the PDump layer ourselves
+
+		First check is to confirm we are in continuous mode
+		Second check is to confirm the pdump client is connected and ready.
+		Third check is to confirm we are not in capture range.
+	*/
+	if (bPDumpFlagsContinuous &&
+		bPDumpEnabled &&
+		!PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_NONE))
+	{
+		PDUMP_TRANSITION_EVENT eTransitionEvent = BIT_ISSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN) ?
+			PDUMP_TRANSITION_EVENT_RANGE_APPEND : PDUMP_TRANSITION_EVENT_RANGE_ENTERED;
+
+		eError = PDumpTransition(psDeviceNode,
+		                         psClientCCB->psPDumpConnectionData,
+		                         eTransitionEvent,
+		                         ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	/* Check that the CCB can hold this command + padding */
+	if ((ui32CmdSize + PADDING_COMMAND_SIZE + 1) > psClientCCB->ui32Size)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Command size (%d bytes) too big for CCB (%d bytes)",
+								ui32CmdSize, psClientCCB->ui32Size));
+		return PVRSRV_ERROR_CMD_TOO_BIG;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	while (ui32RetryCount--)
+#endif
+	{
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+		psClientCCB->sUtilisation.ui32CCBAcquired++;
+#endif
+
+		/*
+			Check we don't overflow the end of the buffer and make sure we have
+			enough space for the padding command. We don't have enough space (including the
+			minimum amount for the padding command) we will need to make sure we insert a
+			padding command now and wrap before adding the main command.
+		*/
+
+		if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) <= psClientCCB->ui32Size)
+		{
+			/* The command can fit without wrapping... */
+			IMG_UINT32 ui32FreeSpace;
+
+#if defined(PDUMP)
+			/* Wait for sufficient CCB space to become available */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+								  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+								  ui32CmdSize, psClientCCB->ui32HostWriteOffset,
+								  psClientCCB->szName);
+			DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+						offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+						psClientCCB->ui32HostWriteOffset,
+						ui32CmdSize,
+						psClientCCB->ui32Size);
+#endif
+
+			RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+			                           INVALIDATE);
+			ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+										psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+										psClientCCB->ui32Size);
+
+			/* Can command fit? */
+			if (ui32FreeSpace > ui32CmdSize)
+			{
+				*ppvBufferSpace = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+				return PVRSRV_OK;
+			}
+			/* There is not enough free space in CCB. */
+			goto e_retry;
+		}
+		else
+		{
+			/*
+				We're at the end of the buffer without enough contiguous space.
+				The command cannot fit without wrapping, we need to insert a
+				padding command and wrap. We need to do this in one go otherwise
+				we would be leaving unflushed commands and forcing the client to
+				deal with flushing the padding command but not the command they
+				wanted to write. Therefore we either do all or nothing.
+			*/
+			RGXFWIF_CCB_CMD_HEADER *psHeader;
+			IMG_UINT32 ui32FreeSpace;
+			IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+			RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+			                           INVALIDATE);
+			/* Check this is a growable CCB */
+			if (psClientCCB->ui32VirtualAllocSize > 0)
+			{
+				PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+
+				ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+											psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+											psClientCCB->ui32Size);
+				/*
+				 * Check if CCB should grow or be wrapped.
+				 * Wrap CCB if there is no need for grow (CCB is half empty) or CCB can't grow,
+				 * and when is free space for command and padding.
+				 */
+				if (((ui32FreeSpace > psClientCCB->ui32Size/2) || (psClientCCB->ui32Size == psClientCCB->ui32VirtualAllocSize)) &&
+					(ui32FreeSpace > ui32Remain + ui32CmdSize))
+				{
+					/* Wrap CCB */
+					psHeader = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+					psHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PADDING;
+					psHeader->ui32CmdSize = ui32Remain - sizeof(RGXFWIF_CCB_CMD_HEADER);
+
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags,
+										  "cCCB(%p): Padding cmd %d", psClientCCB, psHeader->ui32CmdSize);
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+										psClientCCB->ui32HostWriteOffset,
+										ui32Remain,
+										ui32PDumpFlags);
+					}
+#endif
+
+					*ppvBufferSpace = psClientCCB->pvClientCCB;
+					return PVRSRV_OK;
+				}
+				else if ((psClientCCB->ui32Size < psClientCCB->ui32VirtualAllocSize) &&
+				         (psClientCCB->ui32HostWriteOffset >= psClientCCB->psClientCCBCtrl->ui32ReadOffset))
+				{
+					/* Grow CCB */
+					PHYS_HEAP *psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+					PHYS_HEAP_POLICY uiHeapPolicy = PhysHeapGetPolicy(psPhysHeap);
+					PVRSRV_ERROR eErr = PVRSRV_OK;
+
+					/* Something went wrong if we are here a second time */
+					PVR_ASSERT(ui32RetryCount != 0);
+					OSLockAcquire(psClientCCB->hCCBGrowLock);
+
+					/*
+					 * On LMA sparse memory can't be mapped to kernel without support for non physically
+					 * sparse allocations.
+					 * To work around this whole ccb memory was allocated at once as contiguous.
+					 * In such case below sparse change is not needed because memory is already allocated.
+					 */
+					if (uiHeapPolicy == PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG)
+					{
+						IMG_UINT32 ui32AllocChunkCount = psClientCCB->ui32Size / psClientCCB->ui32ChunkSize;
+
+						eErr = _RGXCCBMemChangeSparse(psClientCCB, ui32AllocChunkCount);
+					}
+
+					/* Setup new CCB size */
+					if (eErr == PVRSRV_OK)
+					{
+						psClientCCB->ui32Size += psClientCCB->ui32Size;
+					}
+					else
+					{
+						PVR_LOG(("%s: Client CCB (%s) grow failed (%s)", __func__, psClientCCB->szName, PVRSRVGetErrorString(eErr)));
+						OSLockRelease(psClientCCB->hCCBGrowLock);
+						goto e_retry;
+					}
+
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "cCCB update for grow");
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBCtrlMemDesc,
+											offsetof(RGXFWIF_CCCB_CTL, ui32WrapMask),
+											sizeof(psClientCCB->psClientCCBCtrl->ui32WrapMask),
+											ui32PDumpFlags);
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+											offsetof(RGX_CLIENT_CCB, ui32Size),
+											sizeof(psClientCCB->ui32Size),
+											ui32PDumpFlags);
+					}
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+					PVR_LOG(("%s: Client CCB (%s) grew to %u", __func__, psClientCCB->szName, psClientCCB->ui32Size));
+					/* Reset counters */
+					_RGXInitCCBUtilisation(psClientCCB);
+#endif
+
+					/* CCB doubled the size so retry now. */
+					OSLockRelease(psClientCCB->hCCBGrowLock);
+				}
+				else
+				{
+					/* CCB can't grow anymore and can't be wrapped */
+#if defined(PDUMP)
+					/* Wait for sufficient CCB space to become available */
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+										  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+										  ui32Remain, psClientCCB->ui32HostWriteOffset,
+										  psClientCCB->szName);
+					DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+								offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+								psClientCCB->ui32HostWriteOffset,
+								ui32Remain,
+								psClientCCB->ui32Size);
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+										  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+										  ui32CmdSize, 0 /*ui32HostWriteOffset after wrap */,
+										  psClientCCB->szName);
+					DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+								offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+								0 /*ui32HostWriteOffset after wrap */,
+								ui32CmdSize,
+								psClientCCB->ui32Size);
+					/* CCB has now space for our command so try wrapping again. Retry now. */
+#else /* defined(PDUMP) */
+					goto e_retry;
+#endif /* defined(PDUMP) */
+				}
+			}
+			else
+#endif /* defined(PVRSRV_ENABLE_CCCB_GROW) */
+			{
+#if defined(PDUMP)
+				/* Wait for sufficient CCB space to become available */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+									  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+									  ui32Remain, psClientCCB->ui32HostWriteOffset,
+									  psClientCCB->szName);
+				DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+							offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+							psClientCCB->ui32HostWriteOffset,
+							ui32Remain,
+							psClientCCB->ui32Size);
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, 0,
+									  "Wait for %u bytes to become available according cCCB Ctl (woff=%x) for %s",
+									  ui32CmdSize, 0 /*ui32HostWriteOffset after wrap */,
+									  psClientCCB->szName);
+				DevmemPDumpCBP(psClientCCB->psClientCCBCtrlMemDesc,
+							offsetof(RGXFWIF_CCCB_CTL, ui32ReadOffset),
+							0 /*ui32HostWriteOffset after wrap */,
+							ui32CmdSize,
+							psClientCCB->ui32Size);
+#endif
+				RGXFwSharedMemCacheOpValue(psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+				                           INVALIDATE);
+				ui32FreeSpace = GET_CCB_SPACE(psClientCCB->ui32HostWriteOffset,
+											psClientCCB->psClientCCBCtrl->ui32ReadOffset,
+											psClientCCB->ui32Size);
+
+				if (ui32FreeSpace > ui32Remain + ui32CmdSize)
+				{
+					psHeader = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+					psHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PADDING;
+					psHeader->ui32CmdSize = ui32Remain - sizeof(RGXFWIF_CCB_CMD_HEADER);
+#if defined(PDUMP)
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags, "cCCB(%p): Padding cmd %d", psClientCCB, psHeader->ui32CmdSize);
+					if (bPDumpEnabled)
+					{
+						DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+										psClientCCB->ui32HostWriteOffset,
+										ui32Remain,
+										ui32PDumpFlags);
+					}
+#endif
+
+					*ppvBufferSpace = psClientCCB->pvClientCCB;
+					return PVRSRV_OK;
+				}
+
+				goto e_retry;
+			}
+		}
+	}
+e_retry:
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	psClientCCB->sUtilisation.ui32CCBFull++;
+	_RGXCCBUtilisationEvent(psClientCCB,
+				PVRSRV_CLIENT_CCCB_UTILISATION_WARNING_ACQUIRE_FAILED,
+				ui32CmdSize);
+#endif /* PVRSRV_ENABLE_CCCB_UTILISATION_INFO */
+	return PVRSRV_ERROR_RETRY;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXReleaseCCB
+
+ PURPOSE	: Release a CCB that we have been writing to.
+
+ PARAMETERS	: psDevData			- device data
+			  psCCB				- the CCB
+
+ RETURNS	: None
+******************************************************************************/
+void RGXReleaseCCB(RGX_CLIENT_CCB *psClientCCB,
+								IMG_UINT32		ui32CmdSize,
+								IMG_UINT32		ui32PDumpFlags)
+{
+	IMG_UINT32 ui32NewWriteOffset;
+
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psClientCCB->psServerCommonContext);
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_BOOL	bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, ui32PDumpFlags);
+	IMG_BOOL	bPDumpFlagsContinuous = PDUMP_IS_CONTINUOUS(ui32PDumpFlags);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockAcquire(psClientCCB->hCCBGrowLock);
+#endif
+	/*
+	 * If a padding command was needed then we should now move ui32HostWriteOffset
+	 * forward. The command has already be dumped (if bPDumpEnabled).
+	 */
+	if ((psClientCCB->ui32HostWriteOffset + ui32CmdSize + PADDING_COMMAND_SIZE) > psClientCCB->ui32Size)
+	{
+		IMG_UINT32 ui32Remain = psClientCCB->ui32Size - psClientCCB->ui32HostWriteOffset;
+
+		UPDATE_CCB_OFFSET(psClientCCB->ui32HostWriteOffset,
+						  ui32Remain,
+						  psClientCCB->ui32Size);
+		psClientCCB->ui32ByteCount += ui32Remain;
+	}
+
+#if defined(PDUMP)
+	/* Dump the CCB data */
+	if (bPDumpEnabled)
+	{
+		DevmemPDumpLoadMem(psClientCCB->psClientCCBMemDesc,
+						   psClientCCB->ui32HostWriteOffset,
+						   ui32CmdSize,
+						   ui32PDumpFlags);
+	}
+#endif
+
+	/*
+	 * Check if there any fences being written that will already be
+	 * satisfied by the last written update command in this CCB. At the
+	 * same time we can ASSERT that all sync addresses are not NULL.
+	 */
+#if defined(DEBUG)
+	{
+		void *pvBufferStart = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset);
+		void *pvBufferEnd   = IMG_OFFSET_ADDR(psClientCCB->pvClientCCB, psClientCCB->ui32HostWriteOffset + ui32CmdSize);
+		IMG_BOOL  bMessagePrinted  = IMG_FALSE;
+
+		/* Walk through the commands in this section of CCB being released... */
+		while (pvBufferStart < pvBufferEnd)
+		{
+			RGXFWIF_CCB_CMD_HEADER *psCmdHeader = pvBufferStart;
+
+			if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_UPDATE)
+			{
+				/* If an UPDATE then record the values in case an adjacent fence uses it. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				psClientCCB->ui32UpdateEntries = 0;
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+					if (psClientCCB->ui32UpdateEntries < RGX_CCCB_FENCE_UPDATE_LIST_SIZE)
+					{
+						psClientCCB->asFenceUpdateList[psClientCCB->ui32UpdateEntries++] = *psUFOPtr++;
+					}
+				}
+			}
+			else if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE)
+			{
+				/* If a FENCE then check the values against the last UPDATE issued. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+
+					if (bMessagePrinted == IMG_FALSE)
+					{
+						RGXFWIF_UFO *psUpdatePtr = psClientCCB->asFenceUpdateList;
+						IMG_UINT32  ui32UpdateIndex;
+
+						for (ui32UpdateIndex = 0; ui32UpdateIndex < psClientCCB->ui32UpdateEntries; ui32UpdateIndex++)
+						{
+							if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+							{
+								if (RGX_UFO_PTR_ADDR(psUFOPtr) == RGX_UFO_PTR_ADDR(psUpdatePtr))
+								{
+									PVR_DPF((PVR_DBG_MESSAGE, "Redundant sync checkpoint check found in cCCB(%p) - 0x%x -> 0x%x",
+											psClientCCB, RGX_UFO_PTR_ADDR(psUFOPtr), psUFOPtr->ui32Value));
+									bMessagePrinted = IMG_TRUE;
+									break;
+								}
+							}
+							else
+							{
+								if (psUFOPtr->puiAddrUFO.ui32Addr == psUpdatePtr->puiAddrUFO.ui32Addr  &&
+									psUFOPtr->ui32Value == psUpdatePtr->ui32Value)
+								{
+									PVR_DPF((PVR_DBG_MESSAGE, "Redundant fence check found in cCCB(%p) - 0x%x -> 0x%x",
+											psClientCCB, psUFOPtr->puiAddrUFO.ui32Addr, psUFOPtr->ui32Value));
+									bMessagePrinted = IMG_TRUE;
+									break;
+								}
+							}
+							psUpdatePtr++;
+						}
+					}
+
+					psUFOPtr++;
+				}
+			}
+			else if (psCmdHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR)
+			{
+				/* For all other UFO ops check the UFO address is not NULL. */
+				IMG_UINT32  ui32NumUFOs = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+				RGXFWIF_UFO *psUFOPtr   = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+				PVR_ASSERT(ui32NumUFOs != 0);
+
+				while (ui32NumUFOs-- > 0)
+				{
+					PVR_ASSERT(psUFOPtr->puiAddrUFO.ui32Addr != 0);
+					psUFOPtr++;
+				}
+			}
+
+			/* Move to the next command in this section of CCB being released... */
+			pvBufferStart = IMG_OFFSET_ADDR(pvBufferStart, sizeof(RGXFWIF_CCB_CMD_HEADER) + psCmdHeader->ui32CmdSize);
+		}
+	}
+#endif /* REDUNDANT_SYNCS_DEBUG */
+
+
+#if defined(PVRSRV_FORCE_FLUSH_CCCB_ON_KICK)
+	{
+		DEVMEM_MEMDESC* psClientCCBMemDesc = psClientCCB->psClientCCBMemDesc;
+		void *pvClientCCBAddr = psClientCCB->pvClientCCB;
+		PMR *psClientCCBMemDescPMR = NULL;
+		IMG_DEVMEM_OFFSET_T uiPMROffset;
+
+		DevmemGetPMRData(psClientCCBMemDesc,
+			             (IMG_HANDLE*)&psClientCCBMemDescPMR,
+			             &uiPMROffset);
+
+		CacheOpValExec(psClientCCBMemDescPMR,
+					   (IMG_UINT64)(uintptr_t) pvClientCCBAddr,
+					   uiPMROffset,
+					   psClientCCBMemDesc->uiAllocSize,
+					   PVRSRV_CACHE_OP_FLUSH);
+
+	}
+#endif
+
+	ui32NewWriteOffset = psClientCCB->ui32HostWriteOffset;
+	/*
+	 * Update the CCB write offset.
+	 */
+	UPDATE_CCB_OFFSET(ui32NewWriteOffset, ui32CmdSize,
+	                  psClientCCB->ui32Size);
+	psClientCCB->ui32ByteCount += ui32CmdSize;
+
+	/* Flush the CCB data */
+	RGXFwSharedMemFlushCCB(psClientCCB->pvClientCCB,
+	                       psClientCCB->ui32HostWriteOffset,
+	                       ui32NewWriteOffset,
+	                       psClientCCB->psClientCCBCtrl->ui32WrapMask + 1);
+
+	psClientCCB->ui32HostWriteOffset = ui32NewWriteOffset;
+
+#if defined(PVRSRV_ENABLE_CCCB_UTILISATION_INFO)
+	_RGXUpdateCCBUtilisation(psClientCCB);
+#endif
+	/*
+		PDumpSetFrame will detect as we Transition out of capture range for
+		frame based data but if we are PDumping continuous data then we
+		need to inform the PDump layer ourselves
+
+		First check is to confirm we are in continuous mode
+		Second check is to confirm the pdump client is connected and ready.
+		Third check is to confirm we are not in capture range.
+	*/
+#if defined(PDUMP)
+	if (bPDumpFlagsContinuous &&
+		bPDumpEnabled &&
+		!PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_NONE))
+	{
+		PVRSRV_ERROR eError;
+
+		/* Only Transitioning into capture range can cause an error */
+		eError = PDumpTransition(psDeviceNode,
+		                         psClientCCB->psPDumpConnectionData,
+		                         PDUMP_TRANSITION_EVENT_RANGE_EXITED,
+		                         ui32PDumpFlags);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	if (bPDumpEnabled)
+	{
+		/* Update the PDump write offset to show we PDumped this command */
+		psClientCCB->ui32LastPDumpWriteOffset = psClientCCB->ui32HostWriteOffset;
+	}
+#endif
+
+#if defined(NO_HARDWARE)
+	/*
+		The firmware is not running, it cannot update these; we do here instead.
+	*/
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32DepOffset = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset2 = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset3 = psClientCCB->ui32HostWriteOffset;
+	psClientCCB->psClientCCBCtrl->ui32ReadOffset4 = psClientCCB->ui32HostWriteOffset;
+#endif
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psClientCCB->hCCBGrowLock);
+#endif
+}
+
+IMG_UINT32 RGXGetHostWriteOffsetCCB(RGX_CLIENT_CCB *psClientCCB)
+{
+	return psClientCCB->ui32HostWriteOffset;
+}
+
+IMG_UINT32 RGXGetWrapMaskCCB(RGX_CLIENT_CCB *psClientCCB)
+{
+	return psClientCCB->ui32Size-1;
+}
+
+void RGXSetCCBFlags(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32		ui32Flags)
+{
+	if ((ui32Flags & RGX_CONTEXT_FLAG_DISABLESLR))
+	{
+		BIT_SET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+	else
+	{
+		BIT_UNSET(psClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED);
+	}
+}
+
+void RGXCmdHelperInitCmdCCB_CommandSize(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                        IMG_UINT64 ui64FBSCEntryMask,
+                                        IMG_UINT32 ui32ClientFenceCount,
+                                        IMG_UINT32 ui32ClientUpdateCount,
+                                        IMG_UINT32 ui32CmdSize,
+                                        PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                                        PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                                        PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                                        RGX_CCB_CMD_HELPER_DATA *psCmdHelperData)
+{
+	IMG_UINT32 ui32FenceCmdSize = 0;
+	IMG_UINT32 ui32UpdateCmdSize = 0;
+
+	/* Init the generated data members */
+	psCmdHelperData->ui32FBSCInvalCmdSize = 0;
+	psCmdHelperData->ui64FBSCEntryMask = 0;
+	psCmdHelperData->ui32PreTimeStampCmdSize = 0;
+	psCmdHelperData->ui32PostTimeStampCmdSize = 0;
+	psCmdHelperData->ui32RMWUFOCmdSize = 0;
+
+	/* Total FBSC invalidate command size (header plus command data) */
+	if (ui64FBSCEntryMask != 0)
+	{
+		psCmdHelperData->ui32FBSCInvalCmdSize =
+			        RGX_CCB_FWALLOC_ALIGN(sizeof(psCmdHelperData->ui64FBSCEntryMask) +
+			                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+		psCmdHelperData->ui64FBSCEntryMask = ui64FBSCEntryMask;
+	}
+
+	/* total DM command size (header plus command data) */
+
+	psCmdHelperData->ui32DMCmdSize =
+	        RGX_CCB_FWALLOC_ALIGN(ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+	if (ui32ClientFenceCount != 0)
+	{
+		ui32FenceCmdSize =
+		        RGX_CCB_FWALLOC_ALIGN(ui32ClientFenceCount * sizeof(RGXFWIF_UFO) +
+		                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+	}
+
+	if (ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateCmdSize =
+		        RGX_CCB_FWALLOC_ALIGN(ui32ClientUpdateCount * sizeof(RGXFWIF_UFO) +
+		                              sizeof(RGXFWIF_CCB_CMD_HEADER));
+	}
+
+	if (ppPreAddr && (ppPreAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32PreTimeStampCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER)
+			+ PVR_ALIGN(sizeof(RGXFWIF_DEV_VIRTADDR), RGXFWIF_FWALLOC_ALIGN);
+	}
+
+	if (ppPostAddr && (ppPostAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32PostTimeStampCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER)
+			+ PVR_ALIGN(sizeof(RGXFWIF_DEV_VIRTADDR), RGXFWIF_FWALLOC_ALIGN);
+	}
+
+	if (ppRMWUFOAddr && (ppRMWUFOAddr->ui32Addr != 0))
+	{
+		psCmdHelperData->ui32RMWUFOCmdSize = sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_UFO);
+	}
+
+	psCmdHelperData->ui32TotalSize =
+		ui32FenceCmdSize +
+		psCmdHelperData->ui32FBSCInvalCmdSize +
+		psCmdHelperData->ui32DMCmdSize +
+		ui32UpdateCmdSize +
+		psCmdHelperData->ui32PreTimeStampCmdSize +
+		psCmdHelperData->ui32PostTimeStampCmdSize +
+		psCmdHelperData->ui32RMWUFOCmdSize;
+
+	psCmdHelperData->ui32DMCmdOffset =
+		ui32FenceCmdSize +
+		psCmdHelperData->ui32PreTimeStampCmdSize +
+		psCmdHelperData->ui32FBSCInvalCmdSize;
+}
+
+/*
+	Work out how much space this command will require
+*/
+void RGXCmdHelperInitCmdCCB_OtherData(PVRSRV_RGXDEV_INFO        *psDevInfo,
+                                      RGX_CLIENT_CCB            *psClientCCB,
+                                      IMG_UINT32                ui32ClientFenceCount,
+                                      PRGXFWIF_UFO_ADDR         *pauiFenceUFOAddress,
+                                      IMG_UINT32                *paui32FenceValue,
+                                      IMG_UINT32                ui32ClientUpdateCount,
+                                      PRGXFWIF_UFO_ADDR         *pauiUpdateUFOAddress,
+                                      IMG_UINT32                *paui32UpdateValue,
+                                      IMG_UINT32                ui32CmdSize,
+                                      IMG_PBYTE                 pui8DMCmd,
+                                      PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                                      PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                                      PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                                      RGXFWIF_CCB_CMD_TYPE      eType,
+                                      IMG_UINT32                ui32ExtJobRef,
+                                      IMG_UINT32                ui32IntJobRef,
+                                      IMG_UINT32                ui32PDumpFlags,
+                                      RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                                      IMG_CHAR                  *pszCommandName,
+                                      IMG_BOOL                  bCCBStateOpen,
+                                      RGX_CCB_CMD_HELPER_DATA   *psCmdHelperData)
+{
+	/* Job reference values */
+	psCmdHelperData->ui32ExtJobRef = ui32ExtJobRef;
+	psCmdHelperData->ui32IntJobRef = ui32IntJobRef;
+
+	/* Save the data we require in the submit call */
+	psCmdHelperData->psClientCCB = psClientCCB;
+#if defined(PDUMP)
+	psCmdHelperData->ui32PDumpFlags = ui32PDumpFlags;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+	psCmdHelperData->pszCommandName = pszCommandName;
+	if (bCCBStateOpen)
+	{
+		BIT_SET(psCmdHelperData->psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+	}
+	else
+	{
+		BIT_UNSET(psCmdHelperData->psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+	}
+
+	/* Client sync data */
+	psCmdHelperData->ui32ClientFenceCount = ui32ClientFenceCount;
+	psCmdHelperData->pauiFenceUFOAddress = pauiFenceUFOAddress;
+	psCmdHelperData->paui32FenceValue = paui32FenceValue;
+	psCmdHelperData->ui32ClientUpdateCount = ui32ClientUpdateCount;
+	psCmdHelperData->pauiUpdateUFOAddress = pauiUpdateUFOAddress;
+	psCmdHelperData->paui32UpdateValue = paui32UpdateValue;
+
+	/* Command data */
+	psCmdHelperData->ui32CmdSize = ui32CmdSize;
+	psCmdHelperData->pui8DMCmd = pui8DMCmd;
+	psCmdHelperData->eType = eType;
+
+	if (ppPreAddr)
+	{
+		psCmdHelperData->pPreTimestampAddr = *ppPreAddr;
+	}
+
+	if (ppPostAddr)
+	{
+		psCmdHelperData->pPostTimestampAddr = *ppPostAddr;
+	}
+
+	if (ppRMWUFOAddr)
+	{
+		psCmdHelperData->pRMWUFOAddr = *ppRMWUFOAddr;
+	}
+
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+			"%s Command Server Init on FWCtx %08x", pszCommandName,
+			FWCommonContextGetFWAddress(psClientCCB->psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Workload Data added */
+		psCmdHelperData->psWorkEstKickData = psWorkEstKickData;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(psWorkEstKickData);
+#endif
+}
+
+/*
+	Work out how much space this command will require
+*/
+void RGXCmdHelperInitCmdCCB(PVRSRV_RGXDEV_INFO		*psDevInfo,
+                            RGX_CLIENT_CCB            *psClientCCB,
+                            IMG_UINT64                ui64FBSCEntryMask,
+                            IMG_UINT32                ui32ClientFenceCount,
+                            PRGXFWIF_UFO_ADDR         *pauiFenceUFOAddress,
+                            IMG_UINT32                *paui32FenceValue,
+                            IMG_UINT32                ui32ClientUpdateCount,
+                            PRGXFWIF_UFO_ADDR         *pauiUpdateUFOAddress,
+                            IMG_UINT32                *paui32UpdateValue,
+                            IMG_UINT32                ui32CmdSize,
+                            IMG_PBYTE                 pui8DMCmd,
+                            PRGXFWIF_TIMESTAMP_ADDR   *ppPreAddr,
+                            PRGXFWIF_TIMESTAMP_ADDR   *ppPostAddr,
+                            PRGXFWIF_UFO_ADDR         *ppRMWUFOAddr,
+                            RGXFWIF_CCB_CMD_TYPE      eType,
+                            IMG_UINT32                ui32ExtJobRef,
+                            IMG_UINT32                ui32IntJobRef,
+                            IMG_UINT32                ui32PDumpFlags,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                            IMG_CHAR                  *pszCommandName,
+                            IMG_BOOL                  bCCBStateOpen,
+                            RGX_CCB_CMD_HELPER_DATA   *psCmdHelperData)
+{
+	RGXCmdHelperInitCmdCCB_CommandSize(psDevInfo,
+	                                 ui64FBSCEntryMask,
+	                                 ui32ClientFenceCount,
+	                                 ui32ClientUpdateCount,
+	                                 ui32CmdSize,
+	                                 ppPreAddr,
+	                                 ppPostAddr,
+	                                 ppRMWUFOAddr,
+	                                 psCmdHelperData);
+
+	RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+	                                 psClientCCB,
+	                                 ui32ClientFenceCount,
+	                                 pauiFenceUFOAddress,
+	                                 paui32FenceValue,
+	                                 ui32ClientUpdateCount,
+	                                 pauiUpdateUFOAddress,
+	                                 paui32UpdateValue,
+	                                 ui32CmdSize,
+	                                 pui8DMCmd,
+	                                 ppPreAddr,
+	                                 ppPostAddr,
+	                                 ppRMWUFOAddr,
+	                                 eType,
+	                                 ui32ExtJobRef,
+	                                 ui32IntJobRef,
+	                                 ui32PDumpFlags,
+	                                 psWorkEstKickData,
+	                                 pszCommandName,
+	                                 bCCBStateOpen,
+	                                 psCmdHelperData);
+}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*************************************************************************/ /*!
+@Function       RGXIsValidWorkloadEstCCBCommand
+
+@Description    Checks if command type can be used for workload estimation
+
+@Input          eType       Command type to check
+
+
+@Return        IMG_BOOL
+******************************************************************************/
+FORCE_INLINE IMG_BOOL RGXIsValidWorkloadEstCCBCommand(RGXFWIF_CCB_CMD_TYPE eType)
+{
+	switch (eType)
+	{
+			case RGXFWIF_CCB_CMD_TYPE_GEOM:
+			case RGXFWIF_CCB_CMD_TYPE_3D:
+			case RGXFWIF_CCB_CMD_TYPE_CDM:
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+			case RGXFWIF_CCB_CMD_TYPE_RAY:
+#endif
+			case RGXFWIF_CCB_CMD_TYPE_TQ_TDM:
+				return IMG_TRUE;
+			default:
+				PVR_ASSERT(IMG_FALSE);
+				return IMG_FALSE;
+	}
+}
+#endif
+
+static inline void RGXWriteCmdHeader(void *pvCCB, IMG_UINT32 eCmdType, IMG_UINT32 ui32TotalSize,
+									 IMG_UINT32 ui32ExtJobRef, IMG_UINT32 ui32IntJobRef,
+									 RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData)
+{
+	RGXFWIF_CCB_CMD_HEADER sCmdHeader;
+
+	sCmdHeader.eCmdType = eCmdType;
+	sCmdHeader.ui32CmdSize = ui32TotalSize - sizeof(RGXFWIF_CCB_CMD_HEADER);
+	sCmdHeader.ui32ExtJobRef = ui32ExtJobRef;
+	sCmdHeader.ui32IntJobRef = ui32IntJobRef;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (psWorkEstKickData != NULL &&
+		RGXIsValidWorkloadEstCCBCommand(eCmdType))
+	{
+		sCmdHeader.sWorkEstKickData = *psWorkEstKickData;
+	}
+	else
+	{
+		sCmdHeader.sWorkEstKickData.ui16ReturnDataIndex = 0;
+		sCmdHeader.sWorkEstKickData.ui64Deadline = 0;
+		sCmdHeader.sWorkEstKickData.ui32CyclesPrediction = 0;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psWorkEstKickData);
+#endif
+
+	OSCachedMemCopy(pvCCB, &sCmdHeader, sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+}
+
+/*
+	Reserve space in the CCB and fill in the command and client sync data
+*/
+PVRSRV_ERROR RGXCmdHelperAcquireCmdCCB(IMG_UINT32 ui32CmdCount,
+									   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+	void *pvStartPtr;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(asCmdHelperData->psClientCCB->psServerCommonContext);
+	RGXFWIF_UFO asUFOs[RGXFWIF_CCB_CMD_MAX_UFOS];
+
+	/*
+		Check the number of fences & updates are valid.
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = &asCmdHelperData[i];
+
+		if (psCmdHelperData->ui32ClientFenceCount > RGXFWIF_CCB_CMD_MAX_UFOS ||
+		    psCmdHelperData->ui32ClientUpdateCount > RGXFWIF_CCB_CMD_MAX_UFOS)
+		{
+			return PVRSRV_ERROR_TOO_MANY_SYNCS;
+		}
+	}
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	ui32AllocSize = RGXCmdHelperGetCommandSize(ui32CmdCount, asCmdHelperData);
+
+#if defined(PDUMP)
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		if ((asCmdHelperData[0].ui32PDumpFlags ^ asCmdHelperData[i].ui32PDumpFlags) & PDUMP_FLAGS_CONTINUOUS)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PDump continuous is not consistent (%s != %s) for command %d",
+					 __func__,
+					 PDUMP_IS_CONTINUOUS(asCmdHelperData[0].ui32PDumpFlags)?"IMG_TRUE":"IMG_FALSE",
+					 PDUMP_IS_CONTINUOUS(asCmdHelperData[i].ui32PDumpFlags)?"IMG_TRUE":"IMG_FALSE",
+					 ui32CmdCount));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	/*
+		Acquire space in the CCB for all the command(s).
+	*/
+	eError = RGXAcquireCCB(asCmdHelperData[0].psClientCCB,
+						   ui32AllocSize,
+						   &pvStartPtr,
+						   asCmdHelperData[0].ui32PDumpFlags);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		return eError;
+	}
+
+	/*
+		For each command fill in the fence, DM, and update command
+
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = & asCmdHelperData[i];
+		void *pvCmdPtr;
+#if defined(PDUMP)
+		IMG_UINT32 ui32CtxAddr = FWCommonContextGetFWAddress(asCmdHelperData->psClientCCB->psServerCommonContext).ui32Addr;
+		IMG_UINT32 ui32CcbWoff = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(asCmdHelperData->psClientCCB->psServerCommonContext));
+#endif
+
+		if (psCmdHelperData->ui32ClientFenceCount+psCmdHelperData->ui32ClientUpdateCount != 0)
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "Start of %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+
+		pvCmdPtr = pvStartPtr;
+
+		/*
+			Create the fence command.
+		*/
+		if (psCmdHelperData->ui32ClientFenceCount > 0)
+		{
+			IMG_UINT k, uiNextValueIndex;
+			IMG_UINT32 ui32FenceCmdSize =
+				RGX_CCB_FWALLOC_ALIGN(psCmdHelperData->ui32ClientFenceCount * sizeof(RGXFWIF_UFO) +
+									  sizeof(RGXFWIF_CCB_CMD_HEADER));
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_FENCE,
+							  ui32FenceCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			/* Fill in the client fences */
+			uiNextValueIndex = 0;
+			for (k = 0; k < psCmdHelperData->ui32ClientFenceCount; k++)
+			{
+				PVR_ASSERT(k < RGXFWIF_CCB_CMD_MAX_UFOS);
+
+				asUFOs[k].puiAddrUFO = psCmdHelperData->pauiFenceUFOAddress[k];
+
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(psCmdHelperData->pauiFenceUFOAddress[k].ui32Addr))
+				{
+					asUFOs[k].ui32Value = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				}
+				else
+				{
+					/* Only increment uiNextValueIndex for non sync checkpoints
+					 * (as paui32FenceValue only contains values for sync prims)
+					 */
+					asUFOs[k].ui32Value = psCmdHelperData->paui32FenceValue[uiNextValueIndex++];
+				}
+
+#if defined(SYNC_COMMAND_DEBUG)
+				PVR_DPF((PVR_DBG_ERROR, "%s client sync fence - 0x%x -> 0x%x",
+						psCmdHelperData->psClientCCB->szName, asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value));
+#endif
+				PDUMPCOMMENT(psDevInfo->psDeviceNode,
+							 ".. %s client sync fence - 0x%x -> 0x%x",
+							 psCmdHelperData->psClientCCB->szName,
+							 asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value);
+
+
+			}
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&asUFOs, psCmdHelperData->ui32ClientFenceCount * sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, ui32FenceCmdSize);
+		}
+
+		/*
+			Create the FBSC invalidate command.
+		*/
+		if (psCmdHelperData->ui32FBSCInvalCmdSize)
+		{
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE,
+							  psCmdHelperData->ui32FBSCInvalCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->ui64FBSCEntryMask,
+							sizeof(psCmdHelperData->ui64FBSCEntryMask));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32FBSCInvalCmdSize);
+		}
+
+		/*
+		  Create the pre DM timestamp commands. Pre and Post timestamp commands are supposed to
+		  sandwich the DM cmd. The padding code with the CCB wrap upsets the FW if we don't have
+		  the task type bit cleared for POST_TIMESTAMPs. That's why we have 2 different cmd types.
+		*/
+		if (psCmdHelperData->ui32PreTimeStampCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP,
+							  psCmdHelperData->ui32PreTimeStampCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->pPreTimestampAddr.ui32Addr,
+							sizeof(psCmdHelperData->pPreTimestampAddr.ui32Addr));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32PreTimeStampCmdSize);
+
+		}
+
+		/*
+			Create the DM command
+		*/
+		if (psCmdHelperData->ui32DMCmdSize)
+		{
+			PVR_ASSERT(psCmdHelperData->ui32DMCmdSize == sizeof(RGXFWIF_CCB_CMD_HEADER) + psCmdHelperData->ui32CmdSize);
+
+			RGXWriteCmdHeader(pvCmdPtr,
+							  psCmdHelperData->eType,
+							  psCmdHelperData->ui32DMCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)) ? NULL : psCmdHelperData->psWorkEstKickData);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							psCmdHelperData->pui8DMCmd,
+							psCmdHelperData->ui32CmdSize);
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32DMCmdSize);
+		}
+
+
+		if (psCmdHelperData->ui32PostTimeStampCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP,
+							  psCmdHelperData->ui32PostTimeStampCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&psCmdHelperData->pPostTimestampAddr.ui32Addr,
+							sizeof(psCmdHelperData->pPostTimestampAddr.ui32Addr));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32PostTimeStampCmdSize);
+
+		}
+
+
+		if (psCmdHelperData->ui32RMWUFOCmdSize != 0)
+		{
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE,
+							  psCmdHelperData->ui32RMWUFOCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			asUFOs[0].puiAddrUFO = psCmdHelperData->pRMWUFOAddr;
+			asUFOs[0].ui32Value = 0;
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							asUFOs, sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, psCmdHelperData->ui32RMWUFOCmdSize);
+		}
+
+		/*
+			Create the update command.
+		*/
+		if (psCmdHelperData->ui32ClientUpdateCount > 0)
+		{
+			IMG_UINT k, uiNextValueIndex;
+			IMG_UINT32 ui32UpdateCmdSize =
+				RGX_CCB_FWALLOC_ALIGN(psCmdHelperData->ui32ClientUpdateCount * sizeof(RGXFWIF_UFO) +
+									  sizeof(RGXFWIF_CCB_CMD_HEADER));
+			RGXWriteCmdHeader(pvCmdPtr,
+							  RGXFWIF_CCB_CMD_TYPE_UPDATE,
+							  ui32UpdateCmdSize,
+							  psCmdHelperData->ui32ExtJobRef,
+							  psCmdHelperData->ui32IntJobRef,
+							  NULL);
+
+			/* Fill in the client updates */
+			uiNextValueIndex = 0;
+			for (k = 0; k < psCmdHelperData->ui32ClientUpdateCount; k++)
+			{
+				PVR_ASSERT(k < RGXFWIF_CCB_CMD_MAX_UFOS);
+
+				asUFOs[k].puiAddrUFO = psCmdHelperData->pauiUpdateUFOAddress[k];
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(psCmdHelperData->pauiUpdateUFOAddress[k].ui32Addr))
+				{
+					asUFOs[k].ui32Value = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				}
+				else
+				{
+					/* Only increment uiNextValueIndex for non sync checkpoints
+					 * (as paui32UpdateValue only contains values for sync prims)
+					 */
+					asUFOs[k].ui32Value = psCmdHelperData->paui32UpdateValue[uiNextValueIndex++];
+				}
+
+#if defined(SYNC_COMMAND_DEBUG)
+				PVR_DPF((PVR_DBG_ERROR, "%s client sync update - 0x%x -> 0x%x",
+						psCmdHelperData->psClientCCB->szName, asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value));
+#endif
+				PDUMPCOMMENT(psDevInfo->psDeviceNode,
+							 ".. %s client sync update - 0x%x -> 0x%x",
+							 psCmdHelperData->psClientCCB->szName,
+							 asUFOs[k].puiAddrUFO.ui32Addr, asUFOs[k].ui32Value);
+
+			}
+
+			OSCachedMemCopy(IMG_OFFSET_ADDR(pvCmdPtr, sizeof(RGXFWIF_CCB_CMD_HEADER)),
+							&asUFOs, psCmdHelperData->ui32ClientUpdateCount * sizeof(RGXFWIF_UFO));
+			pvCmdPtr = IMG_OFFSET_ADDR(pvCmdPtr, ui32UpdateCmdSize);
+		}
+
+		/* Set the start pointer for the next iteration around the loop */
+		pvStartPtr = IMG_OFFSET_ADDR(pvStartPtr, psCmdHelperData->ui32TotalSize);
+
+		if (psCmdHelperData->ui32ClientFenceCount+psCmdHelperData->ui32ClientUpdateCount != 0)
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "End of %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+		else
+		{
+			PDUMPCOMMENT(psDevInfo->psDeviceNode,
+						 "No %s client syncs for cmd[%d] on FWCtx %08x Woff 0x%x bytes",
+						 psCmdHelperData->psClientCCB->szName, i, ui32CtxAddr, ui32CcbWoff);
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+/*
+	Fill in the server syncs data and release the CCB space
+*/
+void RGXCmdHelperReleaseCmdCCB(IMG_UINT32 ui32CmdCount,
+							   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+							   const IMG_CHAR *pcszDMName,
+							   IMG_UINT32 ui32CtxAddr)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	IMG_BOOL bTraceChecks = trace_rogue_are_fence_checks_traced();
+	IMG_BOOL bTraceUpdates = trace_rogue_are_fence_updates_traced();
+#endif
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	ui32AllocSize = RGXCmdHelperGetCommandSize(ui32CmdCount, asCmdHelperData);
+	/*
+		For each command write PDump comments and emit FTrace events for fence
+		checks and updates if they exist.
+	*/
+	for (i=0;i<ui32CmdCount;i++)
+	{
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelperData = &asCmdHelperData[i];
+#if defined(PDUMP) || (defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS))
+		PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psCmdHelperData->psClientCCB->psServerCommonContext);
+#endif
+
+#if (!defined(__linux__) || !defined(PDUMP))
+		PVR_UNREFERENCED_PARAMETER(psCmdHelperData);
+#endif
+
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+		if (bTraceChecks)
+		{
+			trace_rogue_fence_checks(psCmdHelperData->pszCommandName,
+									 pcszDMName,
+									 psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+									 ui32CtxAddr,
+									 psCmdHelperData->psClientCCB->ui32HostWriteOffset + ui32AllocSize,
+									 psCmdHelperData->ui32ClientFenceCount,
+									 psCmdHelperData->pauiFenceUFOAddress,
+									 psCmdHelperData->paui32FenceValue);
+		}
+		if (bTraceUpdates)
+		{
+			trace_rogue_fence_updates(psCmdHelperData->pszCommandName,
+									  pcszDMName,
+									  psDevInfo->psDeviceNode->sDevId.ui32InternalID,
+									  ui32CtxAddr,
+									  psCmdHelperData->psClientCCB->ui32HostWriteOffset + ui32AllocSize,
+									  psCmdHelperData->ui32ClientUpdateCount,
+									  psCmdHelperData->pauiUpdateUFOAddress,
+									  psCmdHelperData->paui32UpdateValue);
+		}
+#else
+	PVR_UNREFERENCED_PARAMETER(pcszDMName);
+	PVR_UNREFERENCED_PARAMETER(ui32CtxAddr);
+#endif
+
+		/*
+			All the commands have been filled in so release the CCB space.
+			The FW still won't run this command until we kick it
+		*/
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+				psCmdHelperData->ui32PDumpFlags,
+				"%s Command Server Release on FWCtx %08x",
+				psCmdHelperData->pszCommandName, ui32CtxAddr);
+	}
+
+	RGXReleaseCCB(asCmdHelperData[0].psClientCCB,
+				  ui32AllocSize,
+				  asCmdHelperData[0].ui32PDumpFlags);
+
+	BIT_UNSET(asCmdHelperData[0].psClientCCB->ui32CCBFlags, CCB_FLAGS_CCB_STATE_OPEN);
+}
+
+IMG_UINT32 RGXCmdHelperGetCommandSize(IMG_UINT32              ui32CmdCount,
+                                      RGX_CCB_CMD_HELPER_DATA *asCmdHelperData)
+{
+	IMG_UINT32 ui32AllocSize = 0;
+	IMG_UINT32 i;
+
+	/*
+		Work out how much space we need for all the command(s)
+	*/
+	for (i = 0; i < ui32CmdCount; i++)
+	{
+		ui32AllocSize += asCmdHelperData[i].ui32TotalSize;
+	}
+
+	return ui32AllocSize;
+}
+
+/* Work out how much of an offset there is to a specific command. */
+IMG_UINT32 RGXCmdHelperGetCommandOffset(RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+                                        IMG_UINT32              ui32Cmdindex)
+{
+	return RGXCmdHelperGetCommandSize(ui32Cmdindex, asCmdHelperData);
+}
+
+/* Returns the offset of the data master command from a write offset */
+IMG_UINT32 RGXCmdHelperGetDMCommandHeaderOffset(RGX_CCB_CMD_HELPER_DATA *psCmdHelperData)
+{
+	return psCmdHelperData->ui32DMCmdOffset;
+}
+
+static const char *_CCBCmdTypename(RGXFWIF_CCB_CMD_TYPE cmdType)
+{
+	switch (cmdType)
+	{
+		case RGXFWIF_CCB_CMD_TYPE_GEOM: return "TA";
+		case RGXFWIF_CCB_CMD_TYPE_3D: return "3D";
+		case RGXFWIF_CCB_CMD_TYPE_3D_PR: return "3D_PR";
+		case RGXFWIF_CCB_CMD_TYPE_CDM: return "CDM";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_3D: return "TQ_3D";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_2D: return "TQ_2D";
+		case RGXFWIF_CCB_CMD_TYPE_TQ_TDM: return "TQ_TDM";
+		case RGXFWIF_CCB_CMD_TYPE_FBSC_INVALIDATE: return "FBSC_INVALIDATE";
+		case RGXFWIF_CCB_CMD_TYPE_NULL: return "NULL";
+		case RGXFWIF_CCB_CMD_TYPE_FENCE: return "FENCE";
+		case RGXFWIF_CCB_CMD_TYPE_UPDATE: return "UPDATE";
+		case RGXFWIF_CCB_CMD_TYPE_FENCE_PR: return "FENCE_PR";
+		case RGXFWIF_CCB_CMD_TYPE_PRIORITY: return "PRIORITY";
+		case RGXFWIF_CCB_CMD_TYPE_PRE_TIMESTAMP: return "PRE_TIMESTAMP";
+		case RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE: return "RMW_UPDATE";
+		case RGXFWIF_CCB_CMD_TYPE_POST_TIMESTAMP: return "POST_TIMESTAMP";
+		case RGXFWIF_CCB_CMD_TYPE_PADDING: return "PADDING";
+
+		default:
+			PVR_ASSERT(IMG_FALSE);
+		break;
+	}
+
+	return "INVALID";
+}
+
+PVRSRV_ERROR CheckForStalledCCB(PVRSRV_DEVICE_NODE *psDevNode, RGX_CLIENT_CCB *psCurrentClientCCB, RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;
+	IMG_UINT32					ui32SampledRdOff, ui32SampledDpOff, ui32SampledWrOff, ui32WrapMask;
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+
+	if (psCurrentClientCCB == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "CheckForStalledCCB: CCCB is NULL"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	/* If CCB grow is enabled, take the lock while sampling offsets
+	 * (to guard against a grow happening mid-sample)
+	 */
+	OSLockAcquire(psCurrentClientCCB->hCCBGrowLock);
+#endif
+	/* NB. use psCurrentClientCCB->ui32Size as basis for wrap mask (rather than psClientCCBCtrl->ui32WrapMask)
+	 * as if CCB grow happens, psCurrentClientCCB->ui32Size will have been updated but
+	 * psClientCCBCtrl->ui32WrapMask is only updated once the firmware sees the CCB has grown.
+	 * If we use the wrong value, we might incorrectly determine that the offsets are invalid.
+	 */
+	ui32WrapMask = RGXGetWrapMaskCCB(psCurrentClientCCB);
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	ui32SampledRdOff = psClientCCBCtrl->ui32ReadOffset;
+	ui32SampledDpOff = psClientCCBCtrl->ui32DepOffset;
+	ui32SampledWrOff = psCurrentClientCCB->ui32HostWriteOffset;
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+
+	if (ui32SampledRdOff > ui32WrapMask ||
+		ui32SampledDpOff > ui32WrapMask ||
+		ui32SampledWrOff > ui32WrapMask)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "CheckForStalledCCB: CCCB has invalid offset (ROFF=%d DOFF=%d WOFF=%d)",
+				ui32SampledRdOff, ui32SampledDpOff, ui32SampledWrOff));
+		return PVRSRV_ERROR_INVALID_OFFSET;
+	}
+
+	if (ui32SampledRdOff != ui32SampledWrOff &&
+				psCurrentClientCCB->ui32LastROff != psCurrentClientCCB->ui32LastWOff &&
+				ui32SampledRdOff == psCurrentClientCCB->ui32LastROff &&
+				(psCurrentClientCCB->ui32ByteCount - psCurrentClientCCB->ui32LastByteCount) < psCurrentClientCCB->ui32Size)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+
+		/* Only log a stalled CCB if GPU is idle (any state other than POW_ON is considered idle).
+		 * Guest drivers do not initialize psRGXFWIfFwSysData, so they assume FW internal state is ON. */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+		                           INVALIDATE);
+		if (((psDevInfo->psRGXFWIfFwSysData == NULL) || (psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_ON)) &&
+			(psDevInfo->ui32SLRHoldoffCounter == 0))
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Don't log this by default unless debugging since a higher up
+			 * function will log the stalled condition. Helps avoid double
+			 * messages in the log.
+			 */
+			PVR_DPF((PVR_DBG_ERROR, "%s (%s): CCCB has not progressed (ROFF=%d DOFF=%d WOFF=%d) for \"%s\"",
+					__func__, pszStalledAction, ui32SampledRdOff,
+					ui32SampledDpOff, ui32SampledWrOff,
+					psCurrentClientCCB->szName));
+			eError = PVRSRV_ERROR_CCCB_STALLED;
+
+			{
+				void				*pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+				RGXFWIF_CCB_CMD_HEADER	*psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledRdOff);
+				PVRSRV_RGXDEV_INFO		*psDevInfo = FWCommonContextGetRGXDevInfo(psCurrentClientCCB->psServerCommonContext);
+
+				/* Special case - if readOffset is on a PADDING packet, CCB has wrapped.
+				 * In this case, skip over the PADDING packet.
+				 */
+				if (psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_PADDING)
+				{
+					psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff,
+					                                             ((ui32SampledRdOff +
+					                                               psCommandHeader->ui32CmdSize +
+					                                               sizeof(RGXFWIF_CCB_CMD_HEADER))
+					                                              & psCurrentClientCCB->psClientCCBCtrl->ui32WrapMask));
+				}
+
+				/* Only try to recover a 'stalled' context (ie one waiting on a fence), as some work (eg compute) could
+				 * take a long time to complete, during which time the CCB ptrs would not advance.
+				 */
+				if (((psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE) ||
+				     (psCommandHeader->eCmdType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR)) &&
+				    (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff)))
+				{
+					/* Acquire the cCCB recovery lock */
+					OSLockAcquire(psDevInfo->hCCBRecoveryLock);
+
+					if (!psDevInfo->pvEarliestStalledClientCCB)
+					{
+						psDevInfo->pvEarliestStalledClientCCB = (void*)psCurrentClientCCB;
+						psDevInfo->ui32OldestSubmissionOrdinal = psCommandHeader->ui32IntJobRef;
+					}
+					else
+					{
+						/* Check if this fence cmd header has an older submission stamp than the one we are currently considering unblocking
+						 * (account for submission stamp wrap by checking diff is less than 0x80000000) - if it is older, then this becomes
+						 * our preferred fence to be unblocked/
+						 */
+						if ((psCommandHeader->ui32IntJobRef < psDevInfo->ui32OldestSubmissionOrdinal) &&
+						    ((psDevInfo->ui32OldestSubmissionOrdinal - psCommandHeader->ui32IntJobRef) < 0x8000000))
+						{
+							psDevInfo->pvEarliestStalledClientCCB = (void*)psCurrentClientCCB;
+							psDevInfo->ui32OldestSubmissionOrdinal = psCommandHeader->ui32IntJobRef;
+						}
+					}
+
+					/* Release the cCCB recovery lock */
+					OSLockRelease(psDevInfo->hCCBRecoveryLock);
+				}
+			}
+		}
+	}
+
+	psCurrentClientCCB->ui32LastROff = ui32SampledRdOff;
+	psCurrentClientCCB->ui32LastWOff = ui32SampledWrOff;
+	psCurrentClientCCB->ui32LastByteCount = psCurrentClientCCB->ui32ByteCount;
+
+	return eError;
+}
+
+void DumpCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+			PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+			RGX_CLIENT_CCB *psCurrentClientCCB,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	volatile RGXFWIF_CCCB_CTL *psClientCCBCtrl;
+	void *pvClientCCBBuff;
+	IMG_UINT32 ui32Offset;
+	IMG_UINT32 ui32DepOffset;
+	IMG_UINT32 ui32EndOffset;
+	IMG_UINT32 ui32WrapMask;
+	IMG_CHAR * pszState = "Ready";
+
+	/* Ensure hCCBGrowLock is acquired before reading
+	 * psCurrentClientCCB->pvClientCCB as a CCB grow
+	 * could remap the virtual addresses.
+	 */
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockAcquire(psCurrentClientCCB->hCCBGrowLock);
+#endif
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+	ui32EndOffset = psCurrentClientCCB->ui32HostWriteOffset;
+	OSMemoryBarrier(NULL);
+	ui32Offset = psClientCCBCtrl->ui32ReadOffset;
+	ui32DepOffset = psClientCCBCtrl->ui32DepOffset;
+	/* NB. Use psCurrentClientCCB->ui32Size as basis for wrap mask (rather
+	 * than psClientCCBCtrl->ui32WrapMask) as if CCB grow happened,
+	 * psCurrentClientCCB->ui32Size will have been updated but
+	 * psClientCCBCtrl->ui32WrapMask is only updated once the firmware
+	 * sees the CCB has grown. If we use the wrong value, ui32NextOffset
+	 * can end up being wrapped prematurely and pointing to garbage.
+	 */
+	ui32WrapMask = RGXGetWrapMaskCCB(psCurrentClientCCB);
+
+	PVR_DUMPDEBUG_LOG("FWCtx 0x%08X (%s)", sFWCommonContext.ui32Addr, psCurrentClientCCB->szName);
+	if (ui32Offset == ui32EndOffset)
+	{
+		PVR_DUMPDEBUG_LOG("  `--<Empty>");
+	}
+
+	if (ui32Offset > ui32WrapMask)
+	{
+		PVR_DUMPDEBUG_LOG("  `--<Invalid offset>");
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+		OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+		return;
+	}
+
+	while (ui32Offset != ui32EndOffset)
+	{
+		RGXFWIF_CCB_CMD_HEADER *psCmdHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32Offset);
+		IMG_UINT32 ui32NextOffset = (ui32Offset + psCmdHeader->ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER)) & ui32WrapMask;
+		IMG_BOOL bLastCommand = (ui32NextOffset == ui32EndOffset)? IMG_TRUE: IMG_FALSE;
+		IMG_BOOL bLastUFO;
+		#define CCB_SYNC_INFO_LEN 80
+		IMG_CHAR pszSyncInfo[CCB_SYNC_INFO_LEN];
+		IMG_UINT32 ui32NoOfUpdates, i;
+		RGXFWIF_UFO *psUFOPtr;
+
+		ui32NoOfUpdates = psCmdHeader->ui32CmdSize / sizeof(RGXFWIF_UFO);
+		psUFOPtr = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32Offset + sizeof(RGXFWIF_CCB_CMD_HEADER));
+		pszSyncInfo[0] = '\0';
+
+		if (ui32Offset == ui32DepOffset)
+		{
+			pszState = "Waiting";
+		}
+
+		PVR_DUMPDEBUG_LOG("  %s--%s %s @ %u Int=%u Ext=%u",
+			bLastCommand? "`": "|",
+			pszState, _CCBCmdTypename(psCmdHeader->eCmdType),
+			ui32Offset, psCmdHeader->ui32IntJobRef, psCmdHeader->ui32ExtJobRef
+			);
+
+		/* switch on type and write checks and updates */
+		switch (psCmdHeader->eCmdType)
+		{
+			case RGXFWIF_CCB_CMD_TYPE_UPDATE:
+			case RGXFWIF_CCB_CMD_TYPE_FENCE:
+			case RGXFWIF_CCB_CMD_TYPE_FENCE_PR:
+			{
+				for (i = 0; i < ui32NoOfUpdates; i++, psUFOPtr++)
+				{
+					bLastUFO = (ui32NoOfUpdates-1 == i)? IMG_TRUE: IMG_FALSE;
+
+					if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+					{
+						if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+						{
+							SyncCheckpointRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+						else
+						{
+							SyncRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+					}
+
+					PVR_DUMPDEBUG_LOG("  %s  %s--Addr:0x%08x Val=0x%08x %s",
+						bLastCommand? " ": "|",
+						bLastUFO? "`": "|",
+						psUFOPtr->puiAddrUFO.ui32Addr, psUFOPtr->ui32Value,
+						pszSyncInfo
+						);
+				}
+				break;
+			}
+			case RGXFWIF_CCB_CMD_TYPE_RMW_UPDATE:
+			{
+				for (i = 0; i < ui32NoOfUpdates; i++, psUFOPtr++)
+				{
+					bLastUFO = (ui32NoOfUpdates-1 == i)? IMG_TRUE: IMG_FALSE;
+
+					if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+					{
+						if (PVRSRV_UFO_IS_SYNC_CHECKPOINT(psUFOPtr))
+						{
+							SyncCheckpointRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+						else
+						{
+							SyncRecordLookup(psDeviceNode, psUFOPtr->puiAddrUFO.ui32Addr,
+										pszSyncInfo, CCB_SYNC_INFO_LEN);
+						}
+					}
+
+					PVR_DUMPDEBUG_LOG("  %s  %s--Addr:0x%08x Val++ %s",
+						bLastCommand? " ": "|",
+						bLastUFO? "`": "|",
+						psUFOPtr->puiAddrUFO.ui32Addr,
+						pszSyncInfo
+						);
+				}
+				break;
+			}
+			default:
+				break;
+		}
+
+		/* Check the command size is valid, otherwise if corruption was present this loop might hang... */
+		if ((ui32Offset == ui32NextOffset) ||
+		    ((ui32Offset + psCmdHeader->ui32CmdSize + sizeof(RGXFWIF_CCB_CMD_HEADER)) > ui32WrapMask+1))
+		{
+			PVR_DUMPDEBUG_LOG("  `--Invalid CCB offset!");
+			break;
+		}
+
+		ui32Offset = ui32NextOffset;
+	}
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	OSLockRelease(psCurrentClientCCB->hCCBGrowLock);
+#endif
+}
+
+void DumpFirstCCBCmd(PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+				RGX_CLIENT_CCB *psCurrentClientCCB,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile)
+{
+	volatile RGXFWIF_CCCB_CTL	*psClientCCBCtrl;
+	void					*pvClientCCBBuff = psCurrentClientCCB->pvClientCCB;
+	IMG_UINT32					ui32SampledWrOff = psCurrentClientCCB->ui32HostWriteOffset;
+	IMG_UINT32					ui32SampledRdOff;
+	IMG_UINT32					ui32SampledDepOff;
+
+	RGXFwSharedMemCacheOpPtr(psCurrentClientCCB->psClientCCBCtrl,
+	                         INVALIDATE);
+	psClientCCBCtrl = psCurrentClientCCB->psClientCCBCtrl;
+	ui32SampledRdOff = psClientCCBCtrl->ui32ReadOffset;
+	ui32SampledDepOff = psClientCCBCtrl->ui32DepOffset;
+
+	if ((ui32SampledRdOff == ui32SampledDepOff) &&
+		(ui32SampledRdOff != ui32SampledWrOff))
+	{
+		volatile RGXFWIF_CCB_CMD_HEADER *psCommandHeader = IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledRdOff);
+		RGXFWIF_CCB_CMD_TYPE	eCommandType = psCommandHeader->eCmdType;
+		volatile void				*pvPtr = psCommandHeader;
+
+		/* CCB is stalled on a fence... */
+		if ((eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE) || (eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR))
+		{
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+			PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psCurrentClientCCB->psServerCommonContext);
+			IMG_UINT32 ui32Val;
+#endif
+			RGXFWIF_UFO *psUFOPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader));
+			IMG_UINT32 jj;
+
+			/* Display details of the fence object on which the context is pending */
+			PVR_DUMPDEBUG_LOG("FWCtx 0x%08X @ %d (%s) pending on %s:",
+							   sFWCommonContext.ui32Addr,
+							   ui32SampledRdOff,
+							   psCurrentClientCCB->szName,
+							   _CCBCmdTypename(eCommandType));
+			for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+			{
+#if !defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+				PVR_DUMPDEBUG_LOG("  Addr:0x%08x  Value=0x%08x",psUFOPtr[jj].puiAddrUFO.ui32Addr, psUFOPtr[jj].ui32Value);
+#else
+				ui32Val = 0;
+				RGXReadFWModuleAddr(psDevInfo, psUFOPtr[jj].puiAddrUFO.ui32Addr, &ui32Val);
+				PVR_DUMPDEBUG_LOG("  Addr:0x%08x Value(Host)=0x%08x Value(FW)=0x%08x",
+				                   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+				                   psUFOPtr[jj].ui32Value, ui32Val);
+#endif
+			}
+
+			/* Advance psCommandHeader past the FENCE to the next command header (this will be the TA/3D command that is fenced) */
+			pvPtr = IMG_OFFSET_ADDR(psUFOPtr, psCommandHeader->ui32CmdSize);
+			psCommandHeader = pvPtr;
+			if (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff))
+			{
+				PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X fenced command is of type %s",sFWCommonContext.ui32Addr, _CCBCmdTypename(psCommandHeader->eCmdType));
+				/* Advance psCommandHeader past the TA/3D to the next command header (this will possibly be an UPDATE) */
+				pvPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader) + psCommandHeader->ui32CmdSize);
+				psCommandHeader = pvPtr;
+				/* If the next command is an update, display details of that so we can see what would then become unblocked */
+				if (psCommandHeader != IMG_OFFSET_ADDR(pvClientCCBBuff, ui32SampledWrOff))
+				{
+					eCommandType = psCommandHeader->eCmdType;
+
+					if (eCommandType == RGXFWIF_CCB_CMD_TYPE_UPDATE)
+					{
+						psUFOPtr = IMG_OFFSET_ADDR(psCommandHeader, sizeof(*psCommandHeader));
+						PVR_DUMPDEBUG_LOG(" preventing %s:",_CCBCmdTypename(eCommandType));
+						for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+						{
+#if !defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+							PVR_DUMPDEBUG_LOG("  Addr:0x%08x  Value=0x%08x",psUFOPtr[jj].puiAddrUFO.ui32Addr, psUFOPtr[jj].ui32Value);
+#else
+							ui32Val = 0;
+							RGXReadFWModuleAddr(psDevInfo, psUFOPtr[jj].puiAddrUFO.ui32Addr, &ui32Val);
+							PVR_DUMPDEBUG_LOG("  Addr:0x%08x Value(Host)=0x%08x Value(FW)=0x%08x",
+							                   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							                   psUFOPtr[jj].ui32Value,
+							                   ui32Val);
+#endif
+						}
+					}
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X has no further commands",sFWCommonContext.ui32Addr);
+				}
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG(" FWCtx 0x%08X has no further commands",sFWCommonContext.ui32Addr);
+			}
+		}
+	}
+}
+
+void DumpStalledContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_CLIENT_CCB *psStalledClientCCB;
+
+	PVR_ASSERT(psDevInfo);
+
+	psStalledClientCCB = (RGX_CLIENT_CCB *)psDevInfo->pvEarliestStalledClientCCB;
+
+	if (psStalledClientCCB)
+	{
+		volatile RGXFWIF_CCCB_CTL *psClientCCBCtrl;
+		IMG_UINT32 ui32SampledDepOffset;
+		void                 *pvPtr;
+		RGXFWIF_CCB_CMD_HEADER    *psCommandHeader;
+		RGXFWIF_CCB_CMD_TYPE      eCommandType;
+
+		RGXFwSharedMemCacheOpValue(psStalledClientCCB->psClientCCBCtrl->ui32DepOffset,
+		                           INVALIDATE);
+		psClientCCBCtrl = psStalledClientCCB->psClientCCBCtrl;
+		ui32SampledDepOffset = psClientCCBCtrl->ui32DepOffset;
+		/* No need to invalidate CCCB as FW doesn't write to it, only read. */
+		pvPtr = IMG_OFFSET_ADDR(psStalledClientCCB->pvClientCCB, ui32SampledDepOffset);
+		psCommandHeader = pvPtr;
+		eCommandType = psCommandHeader->eCmdType;
+
+		if ((eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE) || (eCommandType == RGXFWIF_CCB_CMD_TYPE_FENCE_PR))
+		{
+			RGXFWIF_UFO *psUFOPtr = IMG_OFFSET_ADDR(pvPtr, sizeof(*psCommandHeader));
+			IMG_UINT32 jj;
+			IMG_UINT32 ui32NumUnsignalledUFOs = 0;
+			IMG_UINT32 ui32UnsignalledUFOVaddrs[PVRSRV_MAX_SYNCS];
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+			                         INVALIDATE);
+			if (!psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.aszCCBName[0])
+			{
+				OSClockMonotonicns64(&psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui64Timestamp);
+				psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui32NumUFOs = (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO));
+				psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.ui32FWCtxAddr = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr;
+				OSStringSafeCopy(psDevInfo->psRGXFWIfFwOsData->sSLRLogFirst.aszCCBName,
+				              psStalledClientCCB->szName,
+				              MAX_CLIENT_CCB_NAME);
+			}
+			else
+			{
+				OSClockMonotonicns64(&psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui64Timestamp);
+				psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui32NumUFOs = (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO));
+				psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].ui32FWCtxAddr = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr;
+				OSStringSafeCopy(psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp].aszCCBName,
+				              psStalledClientCCB->szName,
+				              MAX_CLIENT_CCB_NAME);
+				psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp = (psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp + 1) % PVR_SLR_LOG_ENTRIES;
+			}
+			psDevInfo->psRGXFWIfFwOsData->ui32ForcedUpdatesRequested++;
+			/* flush write buffers for psRGXFWIfFwOsData */
+			OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwOsData->sSLRLog[psDevInfo->psRGXFWIfFwOsData->ui8SLRLogWp]);
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+			                         FLUSH);
+#endif
+			PVR_LOG(("Fence found on context 0x%x '%s' @ %d has %d UFOs",
+			         FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext).ui32Addr,
+			         psStalledClientCCB->szName, ui32SampledDepOffset,
+			         (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO))));
+
+			for (jj=0; jj<psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO); jj++)
+			{
+				if (PVRSRV_UFO_IS_SYNC_CHECKPOINT((RGXFWIF_UFO *)&psUFOPtr[jj]))
+				{
+					IMG_UINT32 ui32ReadValue = SyncCheckpointStateFromUFO(psDevInfo->psDeviceNode,
+					                                           psUFOPtr[jj].puiAddrUFO.ui32Addr);
+					PVR_LOG(("  %d/%d FWAddr 0x%x requires 0x%x (currently 0x%x)", jj+1,
+							   (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO)),
+							   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							   psUFOPtr[jj].ui32Value,
+							   ui32ReadValue));
+					/* If fence is unmet, dump debug info on it */
+					if (ui32ReadValue != psUFOPtr[jj].ui32Value)
+					{
+						/* Add to our list to pass to pvr_sync */
+						ui32UnsignalledUFOVaddrs[ui32NumUnsignalledUFOs] = psUFOPtr[jj].puiAddrUFO.ui32Addr;
+						ui32NumUnsignalledUFOs++;
+					}
+				}
+				else
+				{
+					PVR_LOG(("  %d/%d FWAddr 0x%x requires 0x%x", jj+1,
+							   (IMG_UINT32)(psCommandHeader->ui32CmdSize/sizeof(RGXFWIF_UFO)),
+							   psUFOPtr[jj].puiAddrUFO.ui32Addr,
+							   psUFOPtr[jj].ui32Value));
+				}
+			}
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_FALLBACK_FENCE_SYNC)
+			if (ui32NumUnsignalledUFOs > 0)
+			{
+				IMG_UINT32 ui32NumSyncsOwned;
+				PVRSRV_ERROR eErr = SyncCheckpointDumpInfoOnStalledUFOs(ui32NumUnsignalledUFOs, &ui32UnsignalledUFOVaddrs[0], &ui32NumSyncsOwned);
+
+				PVR_LOG_IF_ERROR(eErr, "SyncCheckpointDumpInfoOnStalledUFOs() call failed.");
+			}
+#endif
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+			if (BIT_ISSET(psStalledClientCCB->ui32CCBFlags, CCB_FLAGS_SLR_DISABLED))
+			{
+				PRGXFWIF_FWCOMMONCONTEXT psContext = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext);
+
+				PVR_LOG(("SLR disabled for FWCtx 0x%08X", psContext.ui32Addr));
+			}
+			else
+			{
+				if (ui32NumUnsignalledUFOs > 0)
+				{
+					RGXFWIF_KCCB_CMD sSignalFencesCmd;
+
+					sSignalFencesCmd.eCmdType = RGXFWIF_KCCB_CMD_FORCE_UPDATE;
+					sSignalFencesCmd.ui32KCCBFlags = 0;
+					sSignalFencesCmd.uCmdData.sForceUpdateData.psContext = FWCommonContextGetFWAddress(psStalledClientCCB->psServerCommonContext);
+					sSignalFencesCmd.uCmdData.sForceUpdateData.ui32CCBFenceOffset = ui32SampledDepOffset;
+
+					PVR_LOG(("Forced update command issued for FWCtx 0x%08X", sSignalFencesCmd.uCmdData.sForceUpdateData.psContext.ui32Addr));
+
+					RGXScheduleCommand(FWCommonContextGetRGXDevInfo(psStalledClientCCB->psServerCommonContext),
+					                   RGXFWIF_DM_GP,
+					                   &sSignalFencesCmd,
+					                   PDUMP_FLAGS_CONTINUOUS);
+				}
+			}
+#endif
+		}
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+}
+
+/******************************************************************************
+ End of file (rgxccb.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxccb.h b/drivers/gpu/drm/img-rogue/rgxccb.h
new file mode 100644
index 000000000000..e9edd9a37bc3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxccb.h
@@ -0,0 +1,357 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Circular Command Buffer functionality.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Circular Command Buffer functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXCCB_H)
+#define RGXCCB_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "rgxdefs_km.h"
+#include "pvr_notifier.h"
+
+#define MAX_CLIENT_CCB_NAME	30
+#define SYNC_FLAG_MASK_ALL  IMG_UINT32_MAX
+
+/*
+ * This size is to be used when a client CCB is found to consume very
+ * negligible space (e.g. a few hundred bytes to few KBs - less than a page).
+ * In such a case, instead of allocating CCB of size of only a few KBs, we
+ * allocate at-least this much to be future risk-free.
+ */
+#define MIN_SAFE_CCB_SIZE_LOG2         13  /* 8K (2 Pages) */
+#define MAX_SAFE_CCB_SIZE_LOG2         18  /* 256K (64 Pages) */
+
+#define RGX_TQ3D_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D
+static_assert(RGX_TQ3D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TQ3D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ3D CCB size is invalid");
+#define RGX_TQ3D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ3D
+static_assert(RGX_TQ3D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ3D
+	&& RGX_TQ3D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ3D max CCB size is invalid");
+
+#define RGX_TQ2D_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D
+static_assert(RGX_TQ2D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TQ2D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ2D CCB size is invalid");
+#define RGX_TQ2D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TQ2D
+static_assert(RGX_TQ2D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TQ2D &&
+	RGX_TQ2D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TQ2D max CCB size is invalid");
+
+#define RGX_CDM_CCB_SIZE_LOG2          PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM
+static_assert(RGX_CDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_CDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "CDM CCB size is invalid");
+#define RGX_CDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_CDM
+static_assert(RGX_CDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_CDM &&
+	RGX_CDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "CDM max CCB size is invalid");
+
+#define RGX_TA_CCB_SIZE_LOG2           PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA
+static_assert(RGX_TA_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TA_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TA CCB size is invalid");
+#define RGX_TA_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TA
+static_assert(RGX_TA_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TA &&
+	RGX_TA_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TA max CCB size is invalid");
+
+#define RGX_3D_CCB_SIZE_LOG2           PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D
+static_assert(RGX_3D_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_3D_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "3D CCB size is invalid");
+#define RGX_3D_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_3D
+static_assert(RGX_3D_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_3D &&
+	RGX_3D_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "3D max CCB size is invalid");
+
+#define RGX_KICKSYNC_CCB_SIZE_LOG2     PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC
+static_assert(RGX_KICKSYNC_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_KICKSYNC_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "KickSync CCB size is invalid");
+#define RGX_KICKSYNC_CCB_MAX_SIZE_LOG2	PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_KICKSYNC
+static_assert(RGX_KICKSYNC_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_KICKSYNC &&
+	RGX_KICKSYNC_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "KickSync max CCB size is invalid");
+
+#define RGX_TDM_CCB_SIZE_LOG2         PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM
+static_assert(RGX_TDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_TDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TDM CCB size is invalid");
+#define RGX_TDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_TDM
+static_assert(RGX_TDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_TDM &&
+	RGX_TDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "TDM max CCB size is invalid");
+
+#define RGX_RDM_CCB_SIZE_LOG2          PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM
+static_assert(RGX_RDM_CCB_SIZE_LOG2 >= MIN_SAFE_CCB_SIZE_LOG2 &&
+	RGX_RDM_CCB_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "RDM CCB size is invalid");
+#define RGX_RDM_CCB_MAX_SIZE_LOG2		PVRSRV_RGX_LOG2_CLIENT_CCB_MAX_SIZE_RDM
+static_assert(RGX_RDM_CCB_MAX_SIZE_LOG2 >= PVRSRV_RGX_LOG2_CLIENT_CCB_SIZE_RDM &&
+	RGX_RDM_CCB_MAX_SIZE_LOG2 <= MAX_SAFE_CCB_SIZE_LOG2, "RDM max CCB size is invalid");
+
+typedef struct _RGX_CLIENT_CCB_ RGX_CLIENT_CCB;
+
+/*
+	This structure is declared here as it's allocated on the heap by
+	the callers
+*/
+
+typedef struct _RGX_CCB_CMD_HELPER_DATA_ {
+	/* Data setup at command init time */
+	RGX_CLIENT_CCB			*psClientCCB;
+	IMG_CHAR				*pszCommandName;
+	IMG_UINT32				ui32PDumpFlags;
+
+	IMG_UINT32				ui32ClientFenceCount;
+	PRGXFWIF_UFO_ADDR		*pauiFenceUFOAddress;
+	IMG_UINT32				*paui32FenceValue;
+	IMG_UINT32				ui32ClientUpdateCount;
+	PRGXFWIF_UFO_ADDR		*pauiUpdateUFOAddress;
+	IMG_UINT32				*paui32UpdateValue;
+	RGXFWIF_CCB_CMD_TYPE	eType;
+	IMG_UINT32				ui32CmdSize;
+	IMG_UINT8				*pui8DMCmd;
+	IMG_UINT32				ui32FBSCInvalCmdSize;
+	IMG_UINT32				ui32DMCmdSize;
+	IMG_UINT32				ui32TotalSize;
+	IMG_UINT32				ui32DMCmdOffset;
+
+	/* data for FBSC invalidate command */
+	IMG_UINT64				ui64FBSCEntryMask;
+
+	/* timestamp commands */
+	PRGXFWIF_TIMESTAMP_ADDR pPreTimestampAddr;
+	IMG_UINT32              ui32PreTimeStampCmdSize;
+	PRGXFWIF_TIMESTAMP_ADDR pPostTimestampAddr;
+	IMG_UINT32              ui32PostTimeStampCmdSize;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+	IMG_UINT32              ui32RMWUFOCmdSize;
+
+	/* Job reference fields */
+	IMG_UINT32				ui32ExtJobRef;
+	IMG_UINT32				ui32IntJobRef;
+
+	/* FW Memdesc for Workload information */
+	RGXFWIF_WORKEST_KICK_DATA	*psWorkEstKickData;
+
+} RGX_CCB_CMD_HELPER_DATA;
+
+#define PADDING_COMMAND_SIZE	(sizeof(RGXFWIF_CCB_CMD_HEADER))
+
+
+#define RGX_CCB_REQUESTORS(TYPE) \
+	/* for debugging purposes */ TYPE(UNDEF)	\
+	TYPE(TA)	\
+	TYPE(3D)	\
+	TYPE(CDM)	\
+	TYPE(SH)	\
+	TYPE(RS)	\
+	TYPE(TQ_3D)	\
+	TYPE(TQ_2D)	\
+	TYPE(TQ_TDM)	\
+	TYPE(KICKSYNC)	\
+	TYPE(RAY)	\
+
+/* Forms an enum constant for each type present in RGX_CCB_REQUESTORS list. The enum is mainly used as
+   an index to the aszCCBRequestors table defined in rgxccb.c. The total number of enums must adhere
+   to the following build assert.
+*/
+typedef enum _RGX_CCB_REQUESTOR_TYPE_
+{
+#define CONSTRUCT_ENUM(req) REQ_TYPE_##req,
+	RGX_CCB_REQUESTORS (CONSTRUCT_ENUM)
+#undef CONSTRUCT_ENUM
+
+	/* should always be at the end */
+	REQ_TYPE_TOTAL_COUNT,
+} RGX_CCB_REQUESTOR_TYPE;
+
+/* Tuple describing the columns of the following table */
+typedef enum _RGX_CCB_REQUESTOR_TUPLE_
+{
+	REQ_RGX_FW_CLIENT_CCB_STRING,          /* Index to comment to be dumped in DevMemAllocs when allocating FirmwareClientCCB for this requestor */
+	REQ_RGX_FW_CLIENT_CCB_CONTROL_STRING,  /* Index to comment to be dumped in DevMemAllocs when allocating FirmwareClientCCBControl for this requestor */
+	REQ_PDUMP_COMMENT,                     /* Index to comment to be dumped in PDUMPs */
+
+	/* should always be at the end */
+	REQ_TUPLE_CARDINALITY,
+} RGX_CCB_REQUESTOR_TUPLE;
+
+/* Unpack U8 values from U32. */
+#define U32toU8_Unpack1(U32Packed) (U32Packed & 0xFF)
+#define U32toU8_Unpack2(U32Packed) ((U32Packed>>8) & 0xFF)
+#define U32toU8_Unpack3(U32Packed) ((U32Packed>>16) & 0xFF)
+#define U32toU8_Unpack4(U32Packed) ((U32Packed>>24) & 0xFF)
+
+/* Defines for bit meanings within the ui32CCBFlags member of struct _RGX_CLIENT_CCB_
+ *
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *   31                             10
+ *    |                             ||
+ *    ------------------------------XX
+ *  Bit   Meaning
+ *    0 = If set, CCB is still open and commands will be appended to it
+ *    1 = If set, do not perform Sync Lockup Recovery (SLR) for this CCB
+ */
+#define CCB_FLAGS_CCB_STATE_OPEN (0)  /*!< This bit is set to indicate CCB is in the 'Open' state. */
+#define CCB_FLAGS_SLR_DISABLED   (1)  /*!< This bit is set to disable Sync Lockup Recovery (SLR) for this CCB. */
+
+
+/*	Table containing an array of strings for each requestor type in the list of RGX_CCB_REQUESTORS. In addition to its use in
+	this module (rgxccb.c), this table is also used to access string to be dumped in PDUMP comments, hence, marking it extern for
+	use in other modules.
+*/
+extern const IMG_CHAR *const aszCCBRequestors[][REQ_TUPLE_CARDINALITY];
+
+PVRSRV_ERROR RGXCCBPDumpDrainCCB(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32 ui32PDumpFlags);
+
+PVRSRV_ERROR RGXCreateCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+						  IMG_UINT32			ui32CCBSizeLog2,
+						  IMG_UINT32			ui32CCBMaxSizeLog2,
+						  IMG_UINT32			ui32ContextFlags,
+						  CONNECTION_DATA		*psConnectionData,
+						  RGX_CCB_REQUESTOR_TYPE	eCCBRequestor,
+						  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+						  RGX_CLIENT_CCB		**ppsClientCCB,
+						  DEVMEM_MEMDESC		**ppsClientCCBMemDesc,
+						  DEVMEM_MEMDESC		**ppsClientCCBCtlMemDesc);
+
+void RGXDestroyCCB(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_CLIENT_CCB *psClientCCB);
+
+PVRSRV_ERROR RGXCheckSpaceCCB(RGX_CLIENT_CCB *psClientCCB, IMG_UINT32 ui32CmdSize);
+
+PVRSRV_ERROR RGXAcquireCCB(RGX_CLIENT_CCB *psClientCCB,
+						   IMG_UINT32		ui32CmdSize,
+						   void				**ppvBufferSpace,
+						   IMG_UINT32		ui32PDumpFlags);
+
+void RGXReleaseCCB(RGX_CLIENT_CCB *psClientCCB,
+								IMG_UINT32		ui32CmdSize,
+								IMG_UINT32		ui32PDumpFlags);
+
+IMG_UINT32 RGXGetHostWriteOffsetCCB(RGX_CLIENT_CCB *psClientCCB);
+IMG_UINT32 RGXGetWrapMaskCCB(RGX_CLIENT_CCB *psClientCCB);
+
+void RGXSetCCBFlags(RGX_CLIENT_CCB *psClientCCB,
+					IMG_UINT32		ui32Flags);
+
+void RGXCmdHelperInitCmdCCB_CommandSize(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT64 ui64FBSCEntryMask,
+                                        IMG_UINT32 ui32ClientFenceCount,
+                                        IMG_UINT32 ui32ClientUpdateCount,
+                                        IMG_UINT32 ui32CmdSize,
+                                        PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                                        PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                                        PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                                        RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void RGXCmdHelperInitCmdCCB_OtherData(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      RGX_CLIENT_CCB *psClientCCB,
+                                      IMG_UINT32 ui32ClientFenceCount,
+                                      PRGXFWIF_UFO_ADDR *pauiFenceUFOAddress,
+                                      IMG_UINT32 *paui32FenceValue,
+                                      IMG_UINT32 ui32ClientUpdateCount,
+                                      PRGXFWIF_UFO_ADDR *pauiUpdateUFOAddress,
+                                      IMG_UINT32 *paui32UpdateValue,
+                                      IMG_UINT32 ui32CmdSize,
+                                      IMG_PBYTE pui8DMCmd,
+                                      PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                                      PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                                      PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                                      RGXFWIF_CCB_CMD_TYPE eType,
+                                      IMG_UINT32 ui32ExtJobRef,
+                                      IMG_UINT32 ui32IntJobRef,
+                                      IMG_UINT32 ui32PDumpFlags,
+                                      RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                                      IMG_CHAR *pszCommandName,
+                                      IMG_BOOL bCCBStateOpen,
+                                      RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void RGXCmdHelperInitCmdCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+							RGX_CLIENT_CCB          *psClientCCB,
+                            IMG_UINT64              ui64FBSCEntryMask,
+                            IMG_UINT32              ui32ClientFenceCount,
+                            PRGXFWIF_UFO_ADDR       *pauiFenceUFOAddress,
+                            IMG_UINT32              *paui32FenceValue,
+                            IMG_UINT32              ui32ClientUpdateCount,
+                            PRGXFWIF_UFO_ADDR       *pauiUpdateUFOAddress,
+                            IMG_UINT32              *paui32UpdateValue,
+                            IMG_UINT32              ui32CmdSize,
+                            IMG_UINT8               *pui8DMCmd,
+                            PRGXFWIF_TIMESTAMP_ADDR *ppPreAddr,
+                            PRGXFWIF_TIMESTAMP_ADDR *ppPostAddr,
+                            PRGXFWIF_UFO_ADDR       *ppRMWUFOAddr,
+                            RGXFWIF_CCB_CMD_TYPE    eType,
+                            IMG_UINT32              ui32ExtJobRef,
+                            IMG_UINT32              ui32IntJobRef,
+                            IMG_UINT32              ui32PDumpFlags,
+                            RGXFWIF_WORKEST_KICK_DATA *psWorkEstKickData,
+                            IMG_CHAR                *pszCommandName,
+                            IMG_BOOL                bCCBStateOpen,
+                            RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+PVRSRV_ERROR RGXCmdHelperAcquireCmdCCB(IMG_UINT32 ui32CmdCount,
+									   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData);
+
+void RGXCmdHelperReleaseCmdCCB(IMG_UINT32 ui32CmdCount,
+							   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+							   const IMG_CHAR *pcszDMName,
+							   IMG_UINT32 ui32CtxAddr);
+
+IMG_UINT32 RGXCmdHelperGetCommandSize(IMG_UINT32 ui32CmdCount,
+								   RGX_CCB_CMD_HELPER_DATA *asCmdHelperData);
+
+IMG_UINT32 RGXCmdHelperGetCommandOffset(RGX_CCB_CMD_HELPER_DATA *asCmdHelperData,
+                                        IMG_UINT32              ui32Cmdindex);
+
+IMG_UINT32 RGXCmdHelperGetDMCommandHeaderOffset(RGX_CCB_CMD_HELPER_DATA *psCmdHelperData);
+
+void DumpFirstCCBCmd(PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+				RGX_CLIENT_CCB  *psCurrentClientCCB,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile);
+
+void DumpCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+			PRGXFWIF_FWCOMMONCONTEXT sFWCommonContext,
+			RGX_CLIENT_CCB *psCurrentClientCCB,
+			DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+			void *pvDumpDebugFile);
+
+PVRSRV_ERROR CheckForStalledCCB(PVRSRV_DEVICE_NODE *psDevNode, RGX_CLIENT_CCB  *psCurrentClientCCB, RGX_KICK_TYPE_DM eKickTypeDM);
+
+void DumpStalledContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif /* RGXCCB_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxcompute.c b/drivers/gpu/drm/img-rogue/rgxcompute.c
new file mode 100644
index 000000000000..119dc69a57c4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxcompute.c
@@ -0,0 +1,1872 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Compute routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Compute routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvkm.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxcompute.h"
+#include "rgx_bvnc_defs_km.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "htbserver.h"
+#include "rgxshader.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_memallocflags.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_CMP_UFO_DUMP	0
+
+//#define CMP_CHECKPOINT_DEBUG 1
+
+#if defined(CMP_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+struct _RGX_SERVER_COMPUTE_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	DEVMEM_MEMDESC				*psFWComputeContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC				*psFWComputeContextStateMemDesc;
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListFence;
+	SYNC_ADDR_LIST				sSyncAddrListUpdate;
+	POS_LOCK					hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+PVRSRV_ERROR PVRSRVRGXCreateComputeContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE			*psDeviceNode,
+											 IMG_INT32					i32Priority,
+											 IMG_UINT32					ui32FrameworkCommandSize,
+											 IMG_PBYTE					pbyFrameworkCommand,
+											 IMG_HANDLE					hMemCtxPrivData,
+											 IMG_UINT32					ui32StaticComputeContextStateSize,
+											 IMG_PBYTE					pStaticComputeContextState,
+											 IMG_UINT32					ui32PackedCCBSizeU88,
+											 IMG_UINT32					ui32ContextFlags,
+											 IMG_UINT64					ui64RobustnessAddress,
+											 IMG_UINT32					ui32MaxDeadlineMS,
+											 RGX_SERVER_COMPUTE_CONTEXT	**ppsComputeContext)
+{
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext;
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+	RGXFWIF_FWCOMPUTECONTEXT	*psFWComputeContext;
+	IMG_UINT32					ui32CCBAllocSizeLog2, ui32CCBMaxAllocSizeLog2;
+
+	/* Prepare cleanup struct */
+	*ppsComputeContext = NULL;
+
+	psComputeContext = OSAllocZMem(sizeof(*psComputeContext));
+	if (psComputeContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW compute context, this has the CDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWCOMPUTECONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwComputeContext",
+			&psComputeContext->psFWComputeContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwcomputecontext;
+	}
+
+	eError = OSLockCreate(&psComputeContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	psComputeContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware compute context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_COMPUTECTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwComputeContextState",
+							  &psComputeContext->psFWComputeContextStateMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate firmware GPU context suspend state (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_contextsuspendalloc;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitCompute(psDevInfo, &psComputeContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psComputeContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psComputeContext->psFWFrameworkMemDesc,
+				pbyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psComputeContext->psFWFrameworkMemDesc;
+	}
+
+	ui32CCBAllocSizeLog2 = U32toU8_Unpack1(ui32PackedCCBSizeU88);
+	ui32CCBMaxAllocSizeLog2 = U32toU8_Unpack2(ui32PackedCCBSizeU88);
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_CDM,
+									 RGXFWIF_DM_CDM,
+									 hMemCtxPrivData,
+									 psComputeContext->psFWComputeContextMemDesc,
+									 offsetof(RGXFWIF_FWCOMPUTECONTEXT, sCDMContext),
+									 psFWMemContextMemDesc,
+									 psComputeContext->psFWComputeContextStateMemDesc,
+									 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_CDM_CCB_SIZE_LOG2,
+									 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_CDM_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 i32Priority,
+									 ui32MaxDeadlineMS,
+									 ui64RobustnessAddress,
+									 &sInfo,
+									 &psComputeContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc,
+			(void **)&psFWComputeContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	OSDeviceMemCopy(&psFWComputeContext->sStaticComputeContextState, pStaticComputeContextState, ui32StaticComputeContextStateSize);
+	DevmemPDumpLoadMem(psComputeContext->psFWComputeContextMemDesc, 0, sizeof(RGXFWIF_FWCOMPUTECONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpValue(psFWComputeContext->sStaticComputeContextState, FLUSH);
+	DevmemReleaseCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psComputeContext->psBufferSyncContext =
+			pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+					"rogue-cdm");
+	if (IS_ERR(psComputeContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to create buffer_sync context (err=%ld)",
+				__func__, PTR_ERR(psComputeContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psComputeContext->sSyncAddrListFence);
+	SyncAddrListInit(&psComputeContext->sSyncAddrListUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hComputeCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sComputeCtxtListHead), &(psComputeContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hComputeCtxListLock);
+	}
+
+	*ppsComputeContext = psComputeContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	FWCommonContextFree(psComputeContext->psServerCommonContext);
+fail_contextalloc:
+fail_frameworkcopy:
+	if (psComputeContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextStateMemDesc);
+fail_contextsuspendalloc:
+	OSLockDestroy(psComputeContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextMemDesc);
+fail_fwcomputecontext:
+	OSFreeMem(psComputeContext);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyComputeContextKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext)
+{
+	PVRSRV_ERROR				eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psComputeContext->psDeviceNode,
+											  psComputeContext->psServerCommonContext,
+											  RGXFWIF_DM_CDM,
+											  PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psComputeContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* remove after RGXFWRequestCommonContextCleanUp() because we might return
+	 * RETRY and don't want to be calling this twice */
+	if (psComputeContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psComputeContext->psBufferSyncContext);
+		psComputeContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_FWCOMPUTECONTEXT	*psFWComputeContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc,
+				(void **)&psFWComputeContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware compute context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+		RGXFwSharedMemCacheOpValue(psFWComputeContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+		ui32WorkEstCCBSubmitted = psFWComputeContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psComputeContext->psFWComputeContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psComputeContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psComputeContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+	/* ... it has so we can free its resources */
+
+	OSWRLockAcquireWrite(psDevInfo->hComputeCtxListLock);
+	dllist_remove_node(&(psComputeContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hComputeCtxListLock);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		WorkEstDeInitCompute(psDevInfo, &psComputeContext->sWorkEstData);
+	}
+#endif
+
+	SyncAddrListDeinit(&psComputeContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psComputeContext->sSyncAddrListUpdate);
+
+	FWCommonContextFree(psComputeContext->psServerCommonContext);
+	psComputeContext->psServerCommonContext = NULL;
+
+	if (psComputeContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWFrameworkMemDesc);
+	}
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextStateMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psComputeContext->psFWComputeContextMemDesc);
+
+	OSLockDestroy(psComputeContext->hLock);
+	OSFreeMem(psComputeContext);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXKickCDMKM(RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext,
+								IMG_UINT32					ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK		**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								IMG_UINT32					*paui32ClientUpdateValue,
+								PVRSRV_FENCE				iCheckFence,
+								PVRSRV_TIMELINE				iUpdateTimeline,
+								PVRSRV_FENCE				*piUpdateFence,
+								IMG_CHAR					pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								PVRSRV_FENCE				iExportFenceToSignal,
+								IMG_UINT32					ui32CmdSize,
+								IMG_PBYTE					pui8DMCmd,
+								IMG_UINT32					ui32PDumpFlags,
+								IMG_UINT32					ui32ExtJobRef,
+								IMG_UINT32					ui32SyncPMRCount,
+								IMG_UINT32					*paui32SyncPMRFlags,
+								PMR							**ppsSyncPMRs,
+								IMG_UINT32					ui32NumWorkgroups,
+								IMG_UINT32					ui32NumWorkitems,
+								IMG_UINT64					ui64DeadlineInus,
+								IMG_PUINT32					pui32IntJobRef)
+{
+	RGXFWIF_KCCB_CMD		sCmpKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	PVRSRV_ERROR			eError;
+	IMG_UINT32				ui32CDMCmdOffset = 0;
+	PVRSRV_RGXDEV_INFO      *psDevInfo = FWCommonContextGetRGXDevInfo(psComputeContext->psServerCommonContext);
+	RGX_CLIENT_CCB          *psClientCCB = FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext);
+	IMG_UINT32              ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32				ui32FWCtx;
+	IMG_BOOL				bCCBStateOpen = IMG_FALSE;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataCompute = {0};
+	IMG_UINT32 ui32CDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32CDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32CDMCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT64 ui64FBSCEntryMask = 0;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64 uiCheckFenceUID = 0;
+	IMG_UINT64 uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT psExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	CMD_COMMON *psComputeCmdCmn = IMG_OFFSET_ADDR(pui8DMCmd, 0);
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				"iUpdateTimeline=%d but piUpdateFence is NULL - PVRSRV_ERROR_INVALID_PARAMS",
+				__func__, iUpdateTimeline));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (pui32IntJobRef)
+	{
+		*pui32IntJobRef = ui32IntJobRef;
+	}
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	ui32IntClientUpdateCount = ui32ClientUpdateCount;
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListUpdate,
+									ui32ClientUpdateCount,
+									pauiClientUpdateUFODevVarBlock,
+									paui32ClientUpdateSyncOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount)
+	{
+		pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+	if (ui32SyncPMRCount != 0)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling "
+		          "pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psComputeContext->psBufferSyncContext,
+		    psComputeContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%s)",
+				        __func__, PVRSRVGetErrorString(eError)));
+			}
+
+			goto fail_resolve_input_fence;
+		}
+
+		/* Append buffer sync fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints "
+			          "to CDM Fence (&psTransferContext->sSyncAddrListFence=<%p>, "
+			          "pauiIntFenceUFOAddress=<%p>)...", __func__,
+			          ui32BufferFenceSyncCheckpointCount,
+			          (void *) &psComputeContext->sSyncAddrListFence ,
+			          (void *) pauiIntFenceUFOAddress));
+
+			SyncAddrListAppendAndDeRefCheckpoints(&psComputeContext->sSyncAddrListFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (pauiIntFenceUFOAddress == NULL)
+			{
+				pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		/* Append the update (from output fence) */
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+			                              1, &psBufferUpdateSyncCheckpoint);
+			if (pauiIntUpdateUFOAddress == NULL)
+			{
+				pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+		}
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+		PVR_UNREFERENCED_PARAMETER(ppsSyncPMRs);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers",
+		        __func__, ui32SyncPMRCount));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_populate_sync_addr_list;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										iCheckFence,
+										&ui32FenceSyncCheckpointCount,
+										&apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (%s)", __func__, PVRSRVGetErrorString(eError)));
+		goto fail_free_buffer_sync_data;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, iCheckFence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, ii, (void*)psNextCheckpoint, SyncCheckpointGetFirmwareAddr(psNextCheckpoint)));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psComputeContext->psDeviceNode,
+		                                   pszUpdateFenceName,
+										   iUpdateTimeline,
+										   psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s)", __func__, PVRSRVGetErrorString(eError)));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: iExportFenceToSignal=%d", __func__, iExportFenceToSignal));
+		/* Resolve the iExportFenceToSignal (if required) */
+		if (iExportFenceToSignal != PVRSRV_NO_FENCE)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+			eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+			                                          psComputeContext->psDeviceNode->hSyncCheckpointContext,
+			                                          &psExportFenceSyncCheckpoint,
+			                                          ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), psExportFenceSyncCheckpoint));
+				goto fail_resolve_export_fence;
+			}
+
+			/* Check that the export fence was not also included as part of the
+			 * check fence (which is an error and would lead to a stalled kick).
+			 */
+			if (ui32FenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Checking export fence is not part of check fence...", __func__));
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   ui32FenceSyncCheckpointCount=%d",
+						   __func__, ui32FenceSyncCheckpointCount));
+				if (ui32FenceSyncCheckpointCount > 0)
+				{
+					IMG_UINT32 iii;
+
+					for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+						if (apsFenceSyncCheckpoints[iii] == psExportFenceSyncCheckpoint)
+						{
+							CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, psExportFenceSyncCheckpoint));
+							eError = PVRSRV_ERROR_INVALID_PARAMS;
+							PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+							goto fail_check_fence_includes_export_fence;
+						}
+					}
+				}
+			}
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			if (paui32IntUpdateValue)
+			{
+				OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+			}
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the compute context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the compute context update list */
+			SyncAddrListAppendSyncPrim(&psComputeContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u:", __func__, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Compute CDM Fence (&psComputeContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psComputeContext->sSyncAddrListFence));
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			IMG_UINT32 iii;
+
+			for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+			}
+		}
+#endif
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32IntClientUpdateCount > 0)
+	{
+		IMG_UINT32 iii;
+		IMG_UINT32 *pui32Tmp = (IMG_UINT32*)paui32IntUpdateValue;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Dumping %d update values (paui32IntUpdateValue=<%p>)...", __func__, ui32IntClientUpdateCount, (void*)paui32IntUpdateValue));
+		for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: paui32IntUpdateValue[%d] = <%p>", __func__, iii, (void*)pui32Tmp));
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: *paui32IntUpdateValue[%d] = 0x%x", __func__, iii, *pui32Tmp));
+			pui32Tmp++;
+		}
+	}
+#endif
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+
+	if (psExportFenceSyncCheckpoint)
+	{
+		/* Append the update (from export fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psExportFenceSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psExportFenceSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+		if (ui32IntClientUpdateCount > 0)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+			for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_CMP_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping Compute (CDM) fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Compute (CDM) fence syncs (&psComputeContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psComputeContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d Compute (CDM) update syncs (&psComputeContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psComputeContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+	/*
+	 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+	 * in other words, take the value and set it to zero afterwards.
+	 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+	 * as it must be ready at the time of context activation.
+	 */
+	{
+		eError = RGXExtractFBSCEntryMaskFromMMUContext(psComputeContext->psDeviceNode,
+													   FWCommonContextGetServerMMUCtx(psComputeContext->psServerCommonContext),
+													   &ui64FBSCEntryMask);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%s)", PVRSRVGetErrorString(eError)));
+			goto fail_cmdinvalfbsc;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psComputeContext->psDeviceNode))
+	{
+		sWorkloadCharacteristics.sCompute.ui32NumberOfWorkgroups = ui32NumWorkgroups;
+		sWorkloadCharacteristics.sCompute.ui32NumberOfWorkitems  = ui32NumWorkitems;
+
+		/* Prepare workload estimation */
+		WorkEstPrepare(psComputeContext->psDeviceNode->pvDevice,
+				&psComputeContext->sWorkEstData,
+				&psComputeContext->sWorkEstData.uWorkloadMatchingData.sCompute.sDataCDM,
+				RGXFWIF_CCB_CMD_TYPE_CDM,
+				&sWorkloadCharacteristics,
+				ui64DeadlineInus,
+				&sWorkloadKickDataCompute);
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32NumWorkgroups);
+	PVR_UNREFERENCED_PARAMETER(ui32NumWorkitems);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psComputeContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       ui64FBSCEntryMask,
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+	                       &pPreAddr,
+	                       &pPostAddr,
+	                       &pRMWUFOAddr,
+	                       RGXFWIF_CCB_CMD_TYPE_CDM,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	                       &sWorkloadKickDataCompute,
+#else
+	                       NULL,
+#endif
+	                       "Compute",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+
+	/*
+		We should reserve space in the kernel CCB here and fill in the command
+		directly.
+		This is so if there isn't space in the kernel CCB we can return with
+		retry back to services client before we take any operations
+	*/
+
+	/*
+		We might only be kicking for flush out a padding packet so only submit
+		the command if the create was successful
+	*/
+
+	/*
+	  All the required resources are ready at this point, we can't fail so
+	  take the required server sync operations and commit all the resources
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	ui32CDMCmdOffset = RGXGetHostWriteOffsetCCB(psClientCCB);
+	RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "CDM", FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+	{
+		/* The following is used to determine the offset of the command header containing
+		   the workload estimation data so that can be accessed when the KCCB is read */
+		ui32CDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(asCmdHelperData);
+
+		ui32CDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32CDMCmdOffset < ui32CDMCmdOffsetWrapCheck)
+		{
+			ui32CDMWorkloadDataRO = ui32CDMCmdOffset;
+		}
+		else
+		{
+			ui32CDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/* Construct the kernel compute CCB command. */
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sCmpKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+	{
+		/* Store the offset to the CCCB command header so that it can be referenced
+		 * when the KCCB command reaches the FW */
+		sCmpKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32CDMWorkloadDataRO + ui32CDMCmdHeaderOffset;
+	}
+#endif
+
+	ui32FWCtx = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr;
+
+	if (psComputeCmdCmn)
+	{
+		HTBLOGK(HTB_SF_MAIN_KICK_CDM,
+				sCmpKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui32CDMCmdOffset,
+				psComputeCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef);
+	}
+
+	RGXSRV_HWPERF_ENQ(psComputeContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_CDM,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	/*
+	 * Submit the compute command to the firmware.
+	 */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandWithoutPowerLock(psComputeContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_CDM,
+									&sCmpKCCBCmd,
+									ui32PDumpFlags);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_schedulecmd;
+	}
+	else
+	{
+		PVRGpuTraceEnqueueEvent(psComputeContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_CDM);
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__,
+		           (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+	if (psExportFenceSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)psExportFenceSyncCheckpoint, SyncCheckpointGetId(psExportFenceSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(psExportFenceSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psExportFenceSyncCheckpoint);
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal=%d)", __func__, iExportFenceToSignal));
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psComputeContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pszUpdateFenceName);
+	}
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_schedulecmd:
+fail_acquirepowerlock:
+fail_cmdacquire:
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+fail_cmdinvalfbsc:
+#endif
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+fail_check_fence_includes_export_fence:
+fail_resolve_export_fence:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+
+fail_free_buffer_sync_data:
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+
+fail_resolve_input_fence:
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendCancelCmdKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext, IMG_UINT32 ui32FirstIntJobRefToCancel, IMG_UINT32 ui32LastIntJobRefToCancel)
+{
+	RGXFWIF_KCCB_CMD sCancelWorkKCCBCmd;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	sCancelWorkKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_CANCEL_WORK;
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.ui32FirstIntJobRefToCancel = ui32FirstIntJobRefToCancel;
+	sCancelWorkKCCBCmd.uCmdData.sCancelWorkData.ui32LastIntJobRefToCancel = ui32LastIntJobRefToCancel;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psComputeContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_CDM,
+		                            &sCancelWorkKCCBCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+
+		/* Iterate if we hit a PVRSRV_ERROR_KERNEL_CCB_FULL error */
+		if ((eError != PVRSRV_ERROR_RETRY) &&
+		    (eError != PVRSRV_ERROR_KERNEL_CCB_FULL))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Sending compute cancel command for context <%p>. Work with IntJobRef below ui32FirstValidIntJobRef=0x%x might be discarded.",
+	    psComputeContext->psServerCommonContext, ui32LastIntJobRefToCancel + 1));
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXFlushComputeDataKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext)
+{
+	RGXFWIF_KCCB_CMD sFlushCmd;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psComputeContext->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Submit Compute flush");
+#endif
+	sFlushCmd.eCmdType = RGXFWIF_KCCB_CMD_SLCFLUSHINVAL;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.bInval = IMG_FALSE;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.bDMContext = IMG_TRUE;
+	sFlushCmd.uCmdData.sSLCFlushInvalData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+									RGXFWIF_DM_CDM,
+									&sFlushCmd,
+									PDUMP_FLAGS_CONTINUOUS,
+									&ui32kCCBCommandSlot);
+		/* Iterate if we hit a PVRSRV_ERROR_KERNEL_CCB_FULL error */
+		if ((eError != PVRSRV_ERROR_RETRY) &&
+		    (eError != PVRSRV_ERROR_KERNEL_CCB_FULL))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If we hit a temporary KCCB exhaustion, return a RETRY to caller */
+		if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: Returning RETRY to caller", __func__));
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to schedule SLC flush command (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+		}
+	}
+	else
+	{
+		/* Wait for the SLC flush to complete */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Compute flush aborted (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+			goto error_exit;
+		}
+
+		if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] &
+		                  RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+	}
+
+error_exit:
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(RGX_SERVER_COMPUTE_CONTEXT  *psComputeContext)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psComputeContext->psDeviceNode->pvDevice;
+	IMG_UINT32 ui32ControlStreamFormat =
+		RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, CDM_CONTROL_STREAM_FORMAT) ?
+		RGX_GET_FEATURE_VALUE(psDevInfo, CDM_CONTROL_STREAM_FORMAT) : 0;
+
+	if (ui32ControlStreamFormat >= 2U && ui32ControlStreamFormat < 5U)
+	{
+
+		RGXFWIF_KCCB_CMD  sKCCBCmd;
+		PVRSRV_ERROR      eError;
+
+		OSLockAcquire(psComputeContext->hLock);
+
+		/* Schedule the firmware command */
+		sKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE;
+		sKCCBCmd.uCmdData.sWriteOffsetUpdateData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = RGXScheduleCommand(psComputeContext->psDeviceNode->pvDevice,
+										RGXFWIF_DM_CDM,
+										&sKCCBCmd,
+										PDUMP_FLAGS_NONE);
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to schedule the FW command %d (%s)",
+					__func__,
+					eError,
+					PVRSRVGetErrorString(eError)));
+		}
+
+		OSLockRelease(psComputeContext->hLock);
+		return eError;
+	}
+	else
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                  PVRSRV_DEVICE_NODE * psDeviceNode,
+												  RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = ContextSetPriority(psComputeContext->psServerCommonContext,
+								psConnection,
+								psComputeContext->psDeviceNode->pvDevice,
+								i32Priority,
+								RGXFWIF_DM_CDM);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the compute context (%s)", __func__, PVRSRVGetErrorString(eError)));
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+/*
+ * PVRSRVRGXSetComputeContextPropertyKM
+ */
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPropertyKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                                  RGX_CONTEXT_PROPERTY eContextProperty,
+                                                  IMG_UINT64 ui64Input,
+                                                  IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psComputeContext->hLock);
+			eError = FWCommonContextSetFlags(psComputeContext->psServerCommonContext,
+			                                 ui32ContextFlags);
+			OSLockRelease(psComputeContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpComputeCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hComputeCtxListLock);
+	dllist_foreach_node(&psDevInfo->sComputeCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMPUTE_CONTEXT *psCurrentServerComputeCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMPUTE_CONTEXT, sListNode);
+		DumpFWCommonContextInfo(psCurrentServerComputeCtx->psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hComputeCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientComputeCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 ui32ContextBitMask = 0;
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hComputeCtxListLock);
+	dllist_foreach_node(&psDevInfo->sComputeCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMPUTE_CONTEXT *psCurrentServerComputeCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMPUTE_CONTEXT, sListNode);
+
+		if (CheckStalledClientCommonContext(psCurrentServerComputeCtx->psServerCommonContext, RGX_KICK_TYPE_DM_CDM)
+			== PVRSRV_ERROR_CCCB_STALLED)
+		{
+			ui32ContextBitMask |= RGX_KICK_TYPE_DM_CDM;
+		}
+	}
+	OSWRLockReleaseRead(psDevInfo->hComputeCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * PVRSRVRGXGetLastDeviceErrorKM
+ */
+PVRSRV_ERROR PVRSRVRGXGetLastDeviceErrorKM(CONNECTION_DATA    *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32         *ui32Error)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	*ui32Error = psDevInfo->eLastDeviceError;
+	psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_NONE;
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXKickTimestampQueryKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTimestampQueryKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                           PVRSRV_FENCE iCheckFence,
+										   PVRSRV_TIMELINE iUpdateTimeline,
+										   PVRSRV_FENCE *piUpdateFence,
+                                           IMG_CHAR pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                           IMG_UINT32 ui32CmdSize,
+                                           IMG_PBYTE pui8DMCmd,
+                                           IMG_UINT32 ui32ExtJobRef)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psComputeContext->psServerCommonContext);
+	RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psComputeContext->psServerCommonContext);
+	RGX_CCB_CMD_HELPER_DATA	asCmdHelperData[1];
+	IMG_UINT32 ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32 ui32PDumpFlags = 0;
+	IMG_UINT32 ui32IntClientFenceCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+	IMG_UINT32 ui32IntClientUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+	IMG_UINT32 *paui32IntUpdateValue = NULL;
+	PVRSRV_FENCE  iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT64 uiCheckFenceUID = 0;
+	IMG_UINT64 uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *paui32ClientUpdateValue = NULL;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+	PVRSRV_ERROR eError;
+    PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+    IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+    void *pvUpdateFenceFinaliseData = NULL;
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psComputeContext->hLock);
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListFence,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+
+	eError = SyncAddrListPopulate(&psComputeContext->sSyncAddrListUpdate,
+									0,
+									NULL,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		goto err_populate_sync_addr_list;
+	}
+	if (ui32IntClientUpdateCount)
+	{
+		pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	paui32IntUpdateValue = paui32ClientUpdateValue;
+
+
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										iCheckFence,
+										&ui32FenceSyncCheckpointCount,
+										&apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID, ui32PDumpFlags);
+	if (eError != PVRSRV_OK)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (%s)", __func__, PVRSRVGetErrorString(eError)));
+		goto err_populate_sync_addr_list;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(CMP_CHECKPOINT_DEBUG)
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		IMG_UINT32 ii;
+		for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, ii, (void*)psNextCheckpoint, SyncCheckpointGetFirmwareAddr(psNextCheckpoint)));
+		}
+	}
+#endif
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (iUpdateFence=%d, iUpdateTimeline=%d,  psComputeContext->psDeviceNode->hSyncCheckpointContext=<%p>)...", __func__, iUpdateFence, iUpdateTimeline, (void*)psComputeContext->psDeviceNode->hSyncCheckpointContext));
+		eError = SyncCheckpointCreateFence(psComputeContext->psDeviceNode,
+		                                   pszUpdateFenceName,
+										   iUpdateTimeline,
+										   psComputeContext->psDeviceNode->hSyncCheckpointContext,
+										   &iUpdateFence,
+										   &uiUpdateFenceUID,
+										   &pvUpdateFenceFinaliseData,
+										   &psUpdateSyncCheckpoint,
+										   (void*)&psFenceTimelineUpdateSync,
+										   &ui32FenceTimelineUpdateValue,
+										   ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s)", __func__, PVRSRVGetErrorString(eError)));
+			goto fail_create_output_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned from SyncCheckpointCreateFence (iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, ui32FenceTimelineUpdateValue=%u)", __func__, iUpdateFence, psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ui32IntClientUpdateCount=%u, psFenceTimelineUpdateSync=<%p>", __func__, ui32IntClientUpdateCount, (void*)psFenceTimelineUpdateSync));
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			if (paui32IntUpdateValue)
+			{
+				OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+			}
+
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32IntClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: append the timeline sync prim addr <%p> to the compute context update list", __func__,  (void*)psFenceTimelineUpdateSync));
+			/* Now append the timeline sync prim addr to the compute context update list */
+			SyncAddrListAppendSyncPrim(&psComputeContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+
+			/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+			paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+		}
+	}
+
+	/* Append the checks (from input fence) */
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to Compute CDM Fence (&psComputeContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psComputeContext->sSyncAddrListFence));
+
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+		if (!pauiIntFenceUFOAddress)
+		{
+			pauiIntFenceUFOAddress = psComputeContext->sSyncAddrListFence.pasFWAddrs;
+		}
+		ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+	}
+
+	if (psUpdateSyncCheckpoint)
+	{
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Compute CDM Update (&psComputeContext->sSyncAddrListUpdate=<%p>, psUpdateSyncCheckpoint=<%p>)...", __func__, (void*)&psComputeContext->sSyncAddrListUpdate , (void*)psUpdateSyncCheckpoint));
+		SyncAddrListAppendCheckpoints(&psComputeContext->sSyncAddrListUpdate,
+									  1,
+									  &psUpdateSyncCheckpoint);
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psComputeContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32IntClientUpdateCount++;
+	}
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       0, /* empty ui64FBSCEntryMask */
+	                       ui32IntClientFenceCount,
+	                       pauiIntFenceUFOAddress,
+	                       NULL,
+	                       ui32IntClientUpdateCount,
+	                       pauiIntUpdateUFOAddress,
+	                       paui32IntUpdateValue,
+	                       ui32CmdSize,
+	                       pui8DMCmd,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       PDUMP_FLAGS_NONE,
+	                       NULL,
+	                       "VkTimestamp",
+	                       IMG_FALSE, /* bCCBStateOpen */
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXCmdHelperAcquireCmdCCB", fail_cmdacquire);
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	RGXCmdHelperReleaseCmdCCB(1, asCmdHelperData, "CDM",
+	                          FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext).ui32Addr);
+
+	/* Construct the kernel compute CCB command. */
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sCmpKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psComputeContext->psServerCommonContext);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sCmpKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+	/*
+	 * Submit the RGXFWIF_CCB_CMD_TYPE_VK_TIMESTAMP
+	 * command to the firmware.
+	 */
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommandWithoutPowerLock(psComputeContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_CDM,
+		                            &sCmpKCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s failed to schedule kernel CCB command (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_schedulecmd;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)psUpdateSyncCheckpoint, SyncCheckpointGetId(psUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating NOHW sync prim<%p> to %d", __func__, (void*)psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif /* defined(NO_HARDWARE) */
+
+
+	*piUpdateFence = iUpdateFence;
+
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psComputeContext->psDeviceNode, iUpdateFence,
+		                            pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, pszUpdateFenceName);
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned
+	 * by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psComputeContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_schedulecmd:
+fail_acquirepowerlock:
+fail_cmdacquire:
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psComputeContext->psDeviceNode, &psComputeContext->sSyncAddrListUpdate);
+fail_alloc_update_values_mem:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+err_populate_sync_addr_list:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+	OSLockRelease(psComputeContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXCDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXCDMReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxcompute.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxcompute.h b/drivers/gpu/drm/img-rogue/rgxcompute.h
new file mode 100644
index 000000000000..b93c42efc044
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxcompute.h
@@ -0,0 +1,226 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX compute functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX compute functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXCOMPUTE_H)
+#define RGXCOMPUTE_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+
+
+typedef struct _RGX_SERVER_COMPUTE_CONTEXT_ RGX_SERVER_COMPUTE_CONTEXT;
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXCreateComputeContextKM
+@Description    Creates an RGX device context for submitting commands to CDM.
+@Input          psConnection          Device connection
+@Input          psDeviceNode          Services-managed device
+@Input          i32Priority           Scheduling priority for commands
+                                      on this context
+@Input          ui32FrameworkCommandSize
+                                      Framework command size
+@Input          pabyFrameworkCommand  Pointer to framework command
+@Input          hMemCtxPrivData       Private data
+@Input          ui32StaticComputeContextStateSize
+                                      Size of fixed compute context state
+@Input          pStaticComputeContextState
+                                      Compute context state
+@Input          ui32PackedCCBSizeU88  Packed CCB size. The first byte contains
+                                      the log2 CCB size and the second byte
+                                      the log2 maximum CCB size.
+@Input          ui32ContextFlags      Flags with context properties
+@Input          ui64RobustnessAddress Address for FW to signal a context reset
+@Input          ui32MaxDeadlineMS     Max deadline limit in MS that the
+                                      workload can run
+@Output         ppsComputeContext     Cleanup data
+@Return         PVRSRV_ERROR          Returns PVRSRV_OK or an error.
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateComputeContextKM(CONNECTION_DATA			*psConnection,
+											 PVRSRV_DEVICE_NODE			*psDeviceNode,
+											 IMG_INT32					i32Priority,
+											 IMG_UINT32					ui32FrameworkCommandSize,
+											 IMG_PBYTE					pabyFrameworkCommand,
+											 IMG_HANDLE					hMemCtxPrivData,
+											 IMG_UINT32					ui32StaticComputeContextStateSize,
+											 IMG_PBYTE					pStaticComputeContextState,
+											 IMG_UINT32					ui32PackedCCBSizeU88,
+											 IMG_UINT32					ui32ContextFlags,
+											 IMG_UINT64					ui64RobustnessAddress,
+											 IMG_UINT32					ui32MaxDeadlineMS,
+											 RGX_SERVER_COMPUTE_CONTEXT	**ppsComputeContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDestroyComputeContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyComputeContext
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyComputeContextKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXKickCDMKM
+
+ @Description
+	Server-side implementation of RGXKickCDM
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickCDMKM(RGX_SERVER_COMPUTE_CONTEXT	*psComputeContext,
+								IMG_UINT32					ui32ClientUpdateCount,
+								SYNC_PRIMITIVE_BLOCK		**pauiClientUpdateUFODevVarBlock,
+								IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								IMG_UINT32					*paui32ClientUpdateValue,
+								PVRSRV_FENCE				iCheckFence,
+								PVRSRV_TIMELINE				iUpdateTimeline,
+								PVRSRV_FENCE				*piUpdateFence,
+								IMG_CHAR					pcszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								PVRSRV_FENCE				iExportFenceToSignal,
+								IMG_UINT32					ui32CmdSize,
+								IMG_PBYTE					pui8DMCmd,
+								IMG_UINT32					ui32PDumpFlags,
+								IMG_UINT32					ui32ExtJobRef,
+								IMG_UINT32					ui32SyncPMRCount,
+								IMG_UINT32					*paui32SyncPMRFlags,
+								PMR							**ppsSyncPMRs,
+								IMG_UINT32					ui32NumWorkgroups,
+								IMG_UINT32					ui32NumWorkitems,
+								IMG_UINT64					ui64DeadlineInus,
+								IMG_PUINT32					pui32IntJobRef);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXFlushComputeDataKM
+
+ @Description
+	Server-side implementation of RGXFlushComputeData
+
+ @Input psComputeContext - Compute context to flush
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFlushComputeDataKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSendCancelCmd
+
+ @Description
+	Server-side implementation of RGXSendCancelCmd
+
+ @Input psComputeContext - Compute context to cancel work on
+ @Input ui32LastIntJobRefToCancel - Last IntJobRef to cancel
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSendCancelCmdKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                      IMG_UINT32 ui32FirstIntJobRefToCancel,
+                                      IMG_UINT32 ui32LastIntJobRefToCancel);
+
+
+/*!
+*******************************************************************************
+
+ @Function	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM
+ @Description   Server-side implementation of RGXNotifyComputeWriteOffsetUpdate
+
+ @Input         psComputeContext - Compute context to flush
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext);
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPriorityKM(CONNECTION_DATA *psConnection,
+												  PVRSRV_DEVICE_NODE *psDeviceNode,
+												  RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetComputeContextPropertyKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+												  RGX_CONTEXT_PROPERTY eContextProperty,
+												  IMG_UINT64 ui64Input,
+												  IMG_UINT64 *pui64Output);
+
+PVRSRV_ERROR PVRSRVRGXGetLastDeviceErrorKM(CONNECTION_DATA    *psConnection,
+                                           PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           IMG_UINT32         *ui32Error);
+
+PVRSRV_ERROR PVRSRVRGXKickTimestampQueryKM(RGX_SERVER_COMPUTE_CONTEXT *psComputeContext,
+                                           PVRSRV_FENCE iCheckFence,
+										   PVRSRV_TIMELINE iUpdateTimeline,
+										   PVRSRV_FENCE *piUpdateFence,
+                                           IMG_CHAR pszUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                           IMG_UINT32 ui32CmdSize,
+                                           IMG_PBYTE pui8DMCmd,
+                                           IMG_UINT32 ui32ExtJobRef);
+
+/* Debug - Dump debug info of compute contexts on this device */
+void DumpComputeCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client compute contexts are stalled */
+IMG_UINT32 CheckForStalledClientComputeCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR PVRSRVRGXCDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+PVRSRV_ERROR PVRSRVRGXCDMReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+#endif /* RGXCOMPUTE_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxdebug.c b/drivers/gpu/drm/img-rogue/rgxdebug.c
new file mode 100644
index 000000000000..93738fdf6f9a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxdebug.c
@@ -0,0 +1,2580 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "cache_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvr_debug.h"
+#include "srvkm.h"
+#include "rgxutils.h"
+#include "tlstream.h"
+#include "rgxfwriscv.h"
+#include "pvrsrv.h"
+#include "services_km.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_utils.h"
+#include "rgx_fwif_km.h"
+#include "rgx_fwif_sf.h"
+#include "debug_common.h"
+
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxcompute.h"
+#include "rgxtransfer.h"
+#include "rgxtdmtransfer.h"
+#include "rgxtimecorr.h"
+#include "rgx_options.h"
+#include "rgxinit.h"
+#include "rgxlayer_impl.h"
+#include "devicemem_history_server.h"
+
+#define DD_SUMMARY_INDENT  ""
+
+#define RGX_DEBUG_STR_SIZE			(150U)
+
+#define RGX_CR_BIF_CAT_BASE0                              (0x1200U)
+#define RGX_CR_BIF_CAT_BASE1                              (0x1208U)
+
+#define RGX_CR_BIF_CAT_BASEN(n) \
+	RGX_CR_BIF_CAT_BASE0 + \
+	((RGX_CR_BIF_CAT_BASE1 - RGX_CR_BIF_CAT_BASE0) * n)
+
+
+#define RGXDBG_BIF_IDS \
+	X(BIF0)\
+	X(BIF1)\
+	X(TEXAS_BIF)\
+	X(DPX_BIF) \
+	X(FWCORE)
+
+#define RGXDBG_SIDEBAND_TYPES \
+	X(META)\
+	X(TLA)\
+	X(DMA)\
+	X(VDMM)\
+	X(CDM)\
+	X(IPP)\
+	X(PM)\
+	X(TILING)\
+	X(MCU)\
+	X(PDS)\
+	X(PBE)\
+	X(VDMS)\
+	X(IPF)\
+	X(ISP)\
+	X(TPF)\
+	X(USCS)\
+	X(PPP)\
+	X(VCE)\
+	X(TPF_CPF)\
+	X(IPF_CPF)\
+	X(FBCDC)
+
+typedef enum
+{
+#define X(NAME) RGXDBG_##NAME,
+	RGXDBG_BIF_IDS
+#undef X
+} RGXDBG_BIF_ID;
+
+typedef enum
+{
+#define X(NAME) RGXDBG_##NAME,
+	RGXDBG_SIDEBAND_TYPES
+#undef X
+} RGXDBG_SIDEBAND_TYPE;
+
+static const IMG_CHAR *const pszPowStateName[] =
+{
+#define X(NAME)	#NAME,
+	RGXFWIF_POW_STATES
+#undef X
+};
+
+static const IMG_CHAR *const pszBIFNames[] =
+{
+#define X(NAME)	#NAME,
+	RGXDBG_BIF_IDS
+#undef X
+};
+
+static const IMG_FLAGS2DESC asHwrState2Description[] =
+{
+	{RGXFWIF_HWR_HARDWARE_OK, " HWR OK;"},
+	{RGXFWIF_HWR_GENERAL_LOCKUP, " General lockup;"},
+	{RGXFWIF_HWR_DM_RUNNING_OK, " DM running ok;"},
+	{RGXFWIF_HWR_DM_STALLING, " DM stalling;"},
+	{RGXFWIF_HWR_FW_FAULT, " FW fault;"},
+	{RGXFWIF_HWR_RESTART_REQUESTED, " Restart requested;"},
+};
+
+static const IMG_FLAGS2DESC asDmState2Description[] =
+{
+	{RGXFWIF_DM_STATE_READY_FOR_HWR, " ready for hwr;"},
+	{RGXFWIF_DM_STATE_NEEDS_SKIP, " needs skip;"},
+	{RGXFWIF_DM_STATE_NEEDS_PR_CLEANUP, " needs PR cleanup;"},
+	{RGXFWIF_DM_STATE_NEEDS_TRACE_CLEAR, " needs trace clear;"},
+	{RGXFWIF_DM_STATE_GUILTY_LOCKUP, " guilty lockup;"},
+	{RGXFWIF_DM_STATE_INNOCENT_LOCKUP, " innocent lockup;"},
+	{RGXFWIF_DM_STATE_GUILTY_OVERRUNING, " guilty overrunning;"},
+	{RGXFWIF_DM_STATE_INNOCENT_OVERRUNING, " innocent overrunning;"},
+	{RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH, " hard context switching;"},
+	{RGXFWIF_DM_STATE_GPU_ECC_HWR, " GPU ECC hwr;"},
+};
+
+static const IMG_CHAR * const apszFwOsStateName[RGXFW_CONNECTION_FW_STATE_COUNT] =
+{
+	"offline",
+	"ready",
+	"active",
+	"offloading",
+	"cooldown"
+};
+
+#if defined(PVR_ENABLE_PHR)
+static const IMG_FLAGS2DESC asPHRConfig2Description[] =
+{
+	{BIT_ULL(RGXFWIF_PHR_MODE_OFF), "off"},
+	{BIT_ULL(RGXFWIF_PHR_MODE_RD_RESET), "reset RD hardware"},
+};
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodePMPC
+
+ @Description
+
+ Return the name for the PM managed Page Catalogues
+
+ @Input ui32PC	 - Page Catalogue number
+
+ @Return   void
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodePMPC(IMG_UINT32 ui32PC)
+{
+	const IMG_CHAR* pszPMPC = " (-)";
+
+	switch (ui32PC)
+	{
+		case 0x8: pszPMPC = " (PM-VCE0)"; break;
+		case 0x9: pszPMPC = " (PM-TE0)"; break;
+		case 0xA: pszPMPC = " (PM-ZLS0)"; break;
+		case 0xB: pszPMPC = " (PM-ALIST0)"; break;
+		case 0xC: pszPMPC = " (PM-VCE1)"; break;
+		case 0xD: pszPMPC = " (PM-TE1)"; break;
+		case 0xE: pszPMPC = " (PM-ZLS1)"; break;
+		case 0xF: pszPMPC = " (PM-ALIST1)"; break;
+	}
+
+	return pszPMPC;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeBIFReqTags
+
+ @Description
+
+ Decode the BIF Tag ID and sideband data fields from BIF_FAULT_BANK_REQ_STATUS regs
+
+ @Input eBankID             - BIF identifier
+ @Input ui32TagID           - Tag ID value
+ @Input ui32TagSB           - Tag Sideband data
+ @Output ppszTagID          - Decoded string from the Tag ID
+ @Output ppszTagSB          - Decoded string from the Tag SB
+ @Output pszScratchBuf      - Buffer provided to the function to generate the debug strings
+ @Input ui32ScratchBufSize  - Size of the provided buffer
+
+ @Return   void
+
+******************************************************************************/
+#include "rgxmhdefs_km.h"
+
+static void _RGXDecodeBIFReqTagsXE(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								   IMG_UINT32	ui32TagID,
+								   IMG_UINT32	ui32TagSB,
+								   IMG_CHAR		**ppszTagID,
+								   IMG_CHAR		**ppszTagSB,
+								   IMG_CHAR		*pszScratchBuf,
+								   IMG_UINT32	ui32ScratchBufSize)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+	IMG_BOOL bNewTagEncoding = IMG_FALSE;
+
+	PVR_ASSERT(ppszTagID != NULL);
+	PVR_ASSERT(ppszTagSB != NULL);
+
+	/* tags updated for all cores (auto & consumer) with branch > 36 or only auto cores with branch = 36 */
+	if ((psDevInfo->sDevFeatureCfg.ui32B > 36) ||
+	    (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TILE_REGION_PROTECTION) && (psDevInfo->sDevFeatureCfg.ui32B == 36)))
+	{
+		bNewTagEncoding = IMG_TRUE;
+	}
+
+	switch (ui32TagID)
+	{
+		/* MMU tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MMU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_MMU:	    pszTagID = "MMU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:	pszTagID = "CPU MMU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:	pszTagID = "CPU IFU"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:	pszTagID = "CPU LSU"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PT_REQUEST:		pszTagSB = "PT"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PD_REQUEST:		pszTagSB = "PD"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PC_REQUEST:		pszTagSB = "PC"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PT_REQUEST:	pszTagSB = "PM PT"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_REQUEST:	pszTagSB = "PM PD"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_REQUEST:	pszTagSB = "PM PC"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PD_WREQUEST:	pszTagSB = "PM PD W"; break;
+				case RGX_MH_TAG_SB_MMU_ENCODING_MMU_TAG_PM_PC_WREQUEST:	pszTagSB = "PM PC W"; break;
+			}
+			break;
+		}
+
+		/* MIPS */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MIPS:
+		{
+			pszTagID = "MIPS";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_OPCODE_FETCH:	pszTagSB = "Opcode"; break;
+				case RGX_MH_TAG_SB_MIPS_ENCODING_MIPS_TAG_DATA_ACCESS:	pszTagSB = "Data"; break;
+			}
+			break;
+		}
+
+		/* CDM tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2:
+		case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG0:	pszTagID = "CDM Stage 0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG1:	pszTagID = "CDM Stage 1"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG2:	pszTagID = "CDM Stage 2"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_CDM_STG3:	pszTagID = "CDM Stage 3"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTROL_STREAM:	pszTagSB = "Control"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_INDIRECT_DATA:	pszTagSB = "Indirect"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_EVENT_DATA:		pszTagSB = "Event"; break;
+				case RGX_MH_TAG_SB_CDM_ENCODING_CDM_TAG_CONTEXT_STATE:	pszTagSB = "Context"; break;
+			}
+			break;
+		}
+
+		/* VDM tags */
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3:
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG0:	pszTagID = "VDM Stage 0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG1:	pszTagID = "VDM Stage 1"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG2:	pszTagID = "VDM Stage 2"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG3:	pszTagID = "VDM Stage 3"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG4:	pszTagID = "VDM Stage 4"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTROL:	pszTagSB = "Control"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STATE:		pszTagSB = "State"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_INDEX:		pszTagSB = "Index"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_STACK:		pszTagSB = "Stack"; break;
+				case RGX_MH_TAG_SB_VDM_ENCODING_VDM_TAG_CONTEXT:	pszTagSB = "Context"; break;
+			}
+			break;
+		}
+
+		/* PDS */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PDS_0:
+			pszTagID = "PDS req 0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PDS_1:
+			pszTagID = "PDS req 1"; break;
+
+		/* MCU */
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCA:
+			pszTagID = "MCU USCA"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCB:
+			pszTagID = "MCU USCB"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCC:
+			pszTagID = "MCU USCC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_USCD:
+			pszTagID = "MCU USCD"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCA:
+			pszTagID = "MCU PDS USCA"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCB:
+			pszTagID = "MCU PDS USCB"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCC:
+			pszTagID = "MCU PDS USCC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDS_USCD:
+			pszTagID = "MCU PDSUSCD"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_MCU_PDSRW:
+			pszTagID = "MCU PDS PDSRW"; break;
+
+		/* TCU */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TCU_0:
+			pszTagID = "TCU req 0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TCU_1:
+			pszTagID = "TCU req 1"; break;
+
+		/* FBCDC */
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_0:
+			pszTagID = bNewTagEncoding ? "TFBDC_TCU0" : "FBCDC0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_1:
+			pszTagID = bNewTagEncoding ? "TFBDC_ZLS0" : "FBCDC1"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_2:
+			pszTagID = bNewTagEncoding ? "TFBDC_TCU1" : "FBCDC2"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_FBCDC_3:
+			pszTagID = bNewTagEncoding ? "TFBDC_ZLS1" : "FBCDC3"; break;
+
+		/* USC Shared */
+		case RGX_MH_TAG_ENCODING_MH_TAG_USC:
+			pszTagID = "USCS"; break;
+
+		/* ISP */
+		case RGX_MH_TAG_ENCODING_MH_TAG_ISP_ZLS:
+			pszTagID = "ISP0 ZLS"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_ISP_DS:
+			pszTagID = "ISP0 DS"; break;
+
+		/* TPF */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF:
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS:
+		case RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF:           pszTagID = "TPF0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF_PBCDBIAS:  pszTagID = "TPF0 DBIAS"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_TPF_SPF:       pszTagID = "TPF0 SPF"; break;
+			}
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_PDS_STATE:	pszTagSB = "PDS state"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DEPTH_BIAS:	pszTagSB = "Depth bias"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_FLOOR_OFFSET_DATA:	pszTagSB = "Floor offset"; break;
+				case RGX_MH_TAG_SB_TPF_ENCODING_TPF_TAG_DELTA_DATA:	pszTagSB = "Delta"; break;
+			}
+			break;
+		}
+
+		/* IPF */
+		case RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ:
+		case RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS:
+		{
+			switch (ui32TagID)
+			{
+				case RGX_MH_TAG_ENCODING_MH_TAG_IPF_CREQ:      pszTagID = "IPF0"; break;
+				case RGX_MH_TAG_ENCODING_MH_TAG_IPF_OTHERS:    pszTagID = "IPF0"; break;
+			}
+
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_ISP_IPP_PIPES))
+			{
+				if (ui32TagID < RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+				{
+					OSSNPrintf(pszScratchBuf, ui32ScratchBufSize, "CReq%d", ui32TagID);
+					pszTagSB = pszScratchBuf;
+				}
+				else if (ui32TagID < 2 * RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+				{
+					ui32TagID -= RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES);
+					OSSNPrintf(pszScratchBuf, ui32ScratchBufSize, "PReq%d", ui32TagID);
+					pszTagSB = pszScratchBuf;
+				}
+				else
+				{
+					switch (ui32TagSB - 2 * RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_IPP_PIPES))
+					{
+						case 0:	pszTagSB = "RReq"; break;
+						case 1:	pszTagSB = "DBSC"; break;
+						case 2:	pszTagSB = "CPF"; break;
+						case 3:	pszTagSB = "Delta"; break;
+					}
+				}
+			}
+			break;
+		}
+
+		/* VDM Stage 5 (temporary) */
+		case RGX_MH_TAG_ENCODING_MH_TAG_VDM_STG5:
+			pszTagID = "VDM Stage 5"; break;
+
+		/* TA */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PPP:
+			pszTagID = "PPP"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPWRTC:
+			pszTagID = "TPW RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TEACRTC:
+			pszTagID = "TEAC RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGRTC:
+			pszTagID = "PSG RTC"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGREGION:
+			pszTagID = "PSG Region"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_PSGSTREAM:
+			pszTagID = "PSG Stream"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPW:
+			pszTagID = "TPW"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_TA_TPC:
+			pszTagID = "TPC"; break;
+
+		/* PM */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PM_ALLOC:
+		{
+			pszTagID = "PMA";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAFSTACK:	pszTagSB = "TA Fstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMLIST:		pszTagSB = "TA MList"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DFSTACK:	pszTagSB = "3D Fstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMLIST:		pszTagSB = "3D MList"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX0:		pszTagSB = "Context0"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_PMCTX1:		pszTagSB = "Context1"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_MAVP:		pszTagSB = "MAVP"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_UFSTACK:		pszTagSB = "UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAMMUSTACK:	pszTagSB = "TA MMUstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DMMUSTACK:	pszTagSB = "3D MMUstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAUFSTACK:	pszTagSB = "TA UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_3DUFSTACK:	pszTagSB = "3D UFstack"; break;
+				case RGX_MH_TAG_SB_PMA_ENCODING_PM_TAG_PMA_TAVFP:		pszTagSB = "TA VFP"; break;
+			}
+			break;
+		}
+		case RGX_MH_TAG_ENCODING_MH_TAG_PM_DEALLOC:
+		{
+			pszTagID = "PMD";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAFSTACK:	pszTagSB = "TA Fstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMLIST:		pszTagSB = "TA MList"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DFSTACK:	pszTagSB = "3D Fstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMLIST:		pszTagSB = "3D MList"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX0:		pszTagSB = "Context0"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_PMCTX1:		pszTagSB = "Context1"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_UFSTACK:		pszTagSB = "UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAMMUSTACK:	pszTagSB = "TA MMUstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DMMUSTACK:	pszTagSB = "3D MMUstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAUFSTACK:	pszTagSB = "TA UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DUFSTACK:	pszTagSB = "3D UFstack"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_TAVFP:		pszTagSB = "TA VFP"; break;
+				case RGX_MH_TAG_SB_PMD_ENCODING_PM_TAG_PMD_3DVFP:		pszTagSB = "3D VFP"; break;
+			}
+			break;
+		}
+
+		/* TDM */
+		case RGX_MH_TAG_ENCODING_MH_TAG_TDM_DMA:
+		{
+			pszTagID = "TDM DMA";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTL_STREAM: pszTagSB = "Ctl stream"; break;
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_CTX_BUFFER: pszTagSB = "Ctx buffer"; break;
+				case RGX_MH_TAG_SB_TDM_DMA_ENCODING_TDM_DMA_TAG_QUEUE_CTL:  pszTagSB = "Queue ctl"; break;
+			}
+			break;
+		}
+		case RGX_MH_TAG_ENCODING_MH_TAG_TDM_CTL:
+		{
+			pszTagID = "TDM CTL";
+			switch (ui32TagSB)
+			{
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_FENCE:   pszTagSB = "Fence"; break;
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_CONTEXT: pszTagSB = "Context"; break;
+				case RGX_MH_TAG_SB_TDM_CTL_ENCODING_TDM_CTL_TAG_QUEUE:   pszTagSB = "Queue"; break;
+			}
+			break;
+		}
+
+		/* PBE */
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE0:
+			pszTagID = "PBE0"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE1:
+			pszTagID = "PBE1"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE2:
+			pszTagID = "PBE2"; break;
+		case RGX_MH_TAG_ENCODING_MH_TAG_PBE3:
+			pszTagID = "PBE3"; break;
+	}
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+/* RISC-V pf tags */
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU  (0x00000001U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU  (0x00000002U)
+#define RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU  (0x00000003U)
+
+static void _RGXDecodeBIFReqTagsFwcore(PVRSRV_RGXDEV_INFO *psDevInfo,
+									   IMG_UINT32 ui32TagID,
+									   IMG_UINT32 ui32TagSB,
+									   IMG_CHAR **ppszTagID,
+									   IMG_CHAR **ppszTagSB)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		pszTagSB = "RISC-V";
+
+		switch (ui32TagID)
+		{
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_MMU:	pszTagID = "RISC-V MMU"; break;
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_IFU:	pszTagID = "RISC-V Instruction Fetch Unit"; break;
+			case RGX_MH_TAG_ENCODING_MH_TAG_CPU_LSU:	pszTagID = "RISC-V Load/Store Unit"; break; /* Or Debug Module System Bus */
+		}
+	}
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+static void _RGXDecodeBIFReqTags(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								 RGXDBG_BIF_ID	eBankID,
+								 IMG_UINT32		ui32TagID,
+								 IMG_UINT32		ui32TagSB,
+								 IMG_CHAR		**ppszTagID,
+								 IMG_CHAR		**ppszTagSB,
+								 IMG_CHAR		*pszScratchBuf,
+								 IMG_UINT32		ui32ScratchBufSize)
+{
+	/* default to unknown */
+	IMG_CHAR *pszTagID = "-";
+	IMG_CHAR *pszTagSB = "-";
+
+	PVR_ASSERT(ppszTagID != NULL);
+	PVR_ASSERT(ppszTagSB != NULL);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		if (eBankID == RGXDBG_FWCORE)
+		{
+			_RGXDecodeBIFReqTagsFwcore(psDevInfo, ui32TagID, ui32TagSB, ppszTagID, ppszTagSB);
+		}
+		else
+		{
+			_RGXDecodeBIFReqTagsXE(psDevInfo, ui32TagID, ui32TagSB, ppszTagID, ppszTagSB, pszScratchBuf, ui32ScratchBufSize);
+		}
+		return;
+	}
+
+	switch (ui32TagID)
+	{
+		case 0x0:
+		{
+			pszTagID = "MMU";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Table"; break;
+				case 0x1: pszTagSB = "Directory"; break;
+				case 0x2: pszTagSB = "Catalogue"; break;
+			}
+			break;
+		}
+		case 0x1:
+		{
+			pszTagID = "TLA";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Pixel data"; break;
+				case 0x1: pszTagSB = "Command stream data"; break;
+				case 0x2: pszTagSB = "Fence or flush"; break;
+			}
+			break;
+		}
+		case 0x2:
+		{
+			pszTagID = "HOST";
+			break;
+		}
+		case 0x3:
+		{
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+			{
+					pszTagID = "META";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "DCache - Thread 0"; break;
+						case 0x1: pszTagSB = "ICache - Thread 0"; break;
+						case 0x2: pszTagSB = "JTag - Thread 0"; break;
+						case 0x3: pszTagSB = "Slave bus - Thread 0"; break;
+						case 0x4: pszTagSB = "DCache - Thread "; break;
+						case 0x5: pszTagSB = "ICache - Thread 1"; break;
+						case 0x6: pszTagSB = "JTag - Thread 1"; break;
+						case 0x7: pszTagSB = "Slave bus - Thread 1"; break;
+					}
+			}
+			else if (RGX_IS_ERN_SUPPORTED(psDevInfo, 57596))
+			{
+				pszTagID="TCU";
+			}
+			else
+			{
+				/* Unreachable code */
+				PVR_ASSERT(IMG_FALSE);
+			}
+			break;
+		}
+		case 0x4:
+		{
+			pszTagID = "USC";
+			OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+			           "Cache line %d", (ui32TagSB & 0x3f));
+			pszTagSB = pszScratchBuf;
+			break;
+		}
+		case 0x5:
+		{
+			pszTagID = "PBE";
+			break;
+		}
+		case 0x6:
+		{
+			pszTagID = "ISP";
+			switch (ui32TagSB)
+			{
+				case 0x00: pszTagSB = "ZLS"; break;
+				case 0x20: pszTagSB = "Occlusion Query"; break;
+			}
+			break;
+		}
+		case 0x7:
+		{
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+			{
+				if (eBankID == RGXDBG_TEXAS_BIF)
+				{
+					pszTagID = "IPF";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "CPF"; break;
+						case 0x1: pszTagSB = "DBSC"; break;
+						case 0x2:
+						case 0x4:
+						case 0x6:
+						case 0x8: pszTagSB = "Control Stream"; break;
+						case 0x3:
+						case 0x5:
+						case 0x7:
+						case 0x9: pszTagSB = "Primitive Block"; break;
+					}
+				}
+				else
+				{
+					pszTagID = "IPP";
+					switch (ui32TagSB)
+					{
+						case 0x0: pszTagSB = "Macrotile Header"; break;
+						case 0x1: pszTagSB = "Region Header"; break;
+					}
+				}
+			}
+			else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, SIMPLE_INTERNAL_PARAMETER_FORMAT))
+			{
+				pszTagID = "IPF";
+				switch (ui32TagSB)
+				{
+					case 0x0: pszTagSB = "Region Header"; break;
+					case 0x1: pszTagSB = "DBSC"; break;
+					case 0x2: pszTagSB = "CPF"; break;
+					case 0x3: pszTagSB = "Control Stream"; break;
+					case 0x4: pszTagSB = "Primitive Block"; break;
+				}
+			}
+			else
+			{
+				pszTagID = "IPF";
+				switch (ui32TagSB)
+				{
+					case 0x0: pszTagSB = "Macrotile Header"; break;
+					case 0x1: pszTagSB = "Region Header"; break;
+					case 0x2: pszTagSB = "DBSC"; break;
+					case 0x3: pszTagSB = "CPF"; break;
+					case 0x4:
+					case 0x6:
+					case 0x8: pszTagSB = "Control Stream"; break;
+					case 0x5:
+					case 0x7:
+					case 0x9: pszTagSB = "Primitive Block"; break;
+				}
+			}
+			break;
+		}
+		case 0x8:
+		{
+			pszTagID = "CDM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Control Stream"; break;
+				case 0x1: pszTagSB = "Indirect Data"; break;
+				case 0x2: pszTagSB = "Event Write"; break;
+				case 0x3: pszTagSB = "Context State"; break;
+			}
+			break;
+		}
+		case 0x9:
+		{
+			pszTagID = "VDM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "Control Stream"; break;
+				case 0x1: pszTagSB = "PPP State"; break;
+				case 0x2: pszTagSB = "Index Data"; break;
+				case 0x4: pszTagSB = "Call Stack"; break;
+				case 0x8: pszTagSB = "Context State"; break;
+			}
+			break;
+		}
+		case 0xA:
+		{
+			pszTagID = "PM";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "PMA_TAFSTACK"; break;
+				case 0x1: pszTagSB = "PMA_TAMLIST"; break;
+				case 0x2: pszTagSB = "PMA_3DFSTACK"; break;
+				case 0x3: pszTagSB = "PMA_3DMLIST"; break;
+				case 0x4: pszTagSB = "PMA_PMCTX0"; break;
+				case 0x5: pszTagSB = "PMA_PMCTX1"; break;
+				case 0x6: pszTagSB = "PMA_MAVP"; break;
+				case 0x7: pszTagSB = "PMA_UFSTACK"; break;
+				case 0x8: pszTagSB = "PMD_TAFSTACK"; break;
+				case 0x9: pszTagSB = "PMD_TAMLIST"; break;
+				case 0xA: pszTagSB = "PMD_3DFSTACK"; break;
+				case 0xB: pszTagSB = "PMD_3DMLIST"; break;
+				case 0xC: pszTagSB = "PMD_PMCTX0"; break;
+				case 0xD: pszTagSB = "PMD_PMCTX1"; break;
+				case 0xF: pszTagSB = "PMD_UFSTACK"; break;
+				case 0x10: pszTagSB = "PMA_TAMMUSTACK"; break;
+				case 0x11: pszTagSB = "PMA_3DMMUSTACK"; break;
+				case 0x12: pszTagSB = "PMD_TAMMUSTACK"; break;
+				case 0x13: pszTagSB = "PMD_3DMMUSTACK"; break;
+				case 0x14: pszTagSB = "PMA_TAUFSTACK"; break;
+				case 0x15: pszTagSB = "PMA_3DUFSTACK"; break;
+				case 0x16: pszTagSB = "PMD_TAUFSTACK"; break;
+				case 0x17: pszTagSB = "PMD_3DUFSTACK"; break;
+				case 0x18: pszTagSB = "PMA_TAVFP"; break;
+				case 0x19: pszTagSB = "PMD_3DVFP"; break;
+				case 0x1A: pszTagSB = "PMD_TAVFP"; break;
+			}
+			break;
+		}
+		case 0xB:
+		{
+			pszTagID = "TA";
+			switch (ui32TagSB)
+			{
+				case 0x1: pszTagSB = "VCE"; break;
+				case 0x2: pszTagSB = "TPC"; break;
+				case 0x3: pszTagSB = "TE Control Stream"; break;
+				case 0x4: pszTagSB = "TE Region Header"; break;
+				case 0x5: pszTagSB = "TE Render Target Cache"; break;
+				case 0x6: pszTagSB = "TEAC Render Target Cache"; break;
+				case 0x7: pszTagSB = "VCE Render Target Cache"; break;
+				case 0x8: pszTagSB = "PPP Context State"; break;
+			}
+			break;
+		}
+		case 0xC:
+		{
+			pszTagID = "TPF";
+			switch (ui32TagSB)
+			{
+				case 0x0: pszTagSB = "TPF0: Primitive Block"; break;
+				case 0x1: pszTagSB = "TPF0: Depth Bias"; break;
+				case 0x2: pszTagSB = "TPF0: Per Primitive IDs"; break;
+				case 0x3: pszTagSB = "CPF - Tables"; break;
+				case 0x4: pszTagSB = "TPF1: Primitive Block"; break;
+				case 0x5: pszTagSB = "TPF1: Depth Bias"; break;
+				case 0x6: pszTagSB = "TPF1: Per Primitive IDs"; break;
+				case 0x7: pszTagSB = "CPF - Data: Pipe 0"; break;
+				case 0x8: pszTagSB = "TPF2: Primitive Block"; break;
+				case 0x9: pszTagSB = "TPF2: Depth Bias"; break;
+				case 0xA: pszTagSB = "TPF2: Per Primitive IDs"; break;
+				case 0xB: pszTagSB = "CPF - Data: Pipe 1"; break;
+				case 0xC: pszTagSB = "TPF3: Primitive Block"; break;
+				case 0xD: pszTagSB = "TPF3: Depth Bias"; break;
+				case 0xE: pszTagSB = "TPF3: Per Primitive IDs"; break;
+				case 0xF: pszTagSB = "CPF - Data: Pipe 2"; break;
+			}
+			break;
+		}
+		case 0xD:
+		{
+			pszTagID = "PDS";
+			break;
+		}
+		case 0xE:
+		{
+			pszTagID = "MCU";
+			{
+				IMG_UINT32 ui32Burst = (ui32TagSB >> 5) & 0x7;
+				IMG_UINT32 ui32GroupEnc = (ui32TagSB >> 2) & 0x7;
+				IMG_UINT32 ui32Group = ui32TagSB & 0x3;
+
+				IMG_CHAR* pszBurst = "";
+				IMG_CHAR* pszGroupEnc = "";
+				IMG_CHAR* pszGroup = "";
+
+				switch (ui32Burst)
+				{
+					case 0x0:
+					case 0x1: pszBurst = "128bit word within the Lower 256bits"; break;
+					case 0x2:
+					case 0x3: pszBurst = "128bit word within the Upper 256bits"; break;
+					case 0x4: pszBurst = "Lower 256bits"; break;
+					case 0x5: pszBurst = "Upper 256bits"; break;
+					case 0x6: pszBurst = "512 bits"; break;
+				}
+				switch (ui32GroupEnc)
+				{
+					case 0x0: pszGroupEnc = "TPUA_USC"; break;
+					case 0x1: pszGroupEnc = "TPUB_USC"; break;
+					case 0x2: pszGroupEnc = "USCA_USC"; break;
+					case 0x3: pszGroupEnc = "USCB_USC"; break;
+					case 0x4: pszGroupEnc = "PDS_USC"; break;
+					case 0x5:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 > RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "PDSRW";
+						} else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "UPUC_USC";
+						}
+						break;
+					case 0x6:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "TPUC_USC";
+						}
+						break;
+					case 0x7:
+						if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) &&
+							6 == RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS))
+						{
+							pszGroupEnc = "PDSRW";
+						}
+						break;
+				}
+				switch (ui32Group)
+				{
+					case 0x0: pszGroup = "Banks 0-3"; break;
+					case 0x1: pszGroup = "Banks 4-7"; break;
+					case 0x2: pszGroup = "Banks 8-11"; break;
+					case 0x3: pszGroup = "Banks 12-15"; break;
+				}
+
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+								"%s, %s, %s", pszBurst, pszGroupEnc, pszGroup);
+				pszTagSB = pszScratchBuf;
+			}
+			break;
+		}
+		case 0xF:
+		{
+			pszTagID = "FB_CDC";
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+			{
+				IMG_UINT32 ui32Req   = (ui32TagSB >> 0) & 0xf;
+				IMG_UINT32 ui32MCUSB = (ui32TagSB >> 4) & 0x3;
+				IMG_CHAR* pszReqOrig = "";
+
+				switch (ui32Req)
+				{
+					case 0x0: pszReqOrig = "FBC Request, originator ZLS"; break;
+					case 0x1: pszReqOrig = "FBC Request, originator PBE"; break;
+					case 0x2: pszReqOrig = "FBC Request, originator Host"; break;
+					case 0x3: pszReqOrig = "FBC Request, originator TLA"; break;
+					case 0x4: pszReqOrig = "FBDC Request, originator ZLS"; break;
+					case 0x5: pszReqOrig = "FBDC Request, originator MCU"; break;
+					case 0x6: pszReqOrig = "FBDC Request, originator Host"; break;
+					case 0x7: pszReqOrig = "FBDC Request, originator TLA"; break;
+					case 0x8: pszReqOrig = "FBC Request, originator ZLS Requester Fence"; break;
+					case 0x9: pszReqOrig = "FBC Request, originator PBE Requester Fence"; break;
+					case 0xa: pszReqOrig = "FBC Request, originator Host Requester Fence"; break;
+					case 0xb: pszReqOrig = "FBC Request, originator TLA Requester Fence"; break;
+					case 0xc: pszReqOrig = "Reserved"; break;
+					case 0xd: pszReqOrig = "Reserved"; break;
+					case 0xe: pszReqOrig = "FBDC Request, originator FBCDC(Host) Memory Fence"; break;
+					case 0xf: pszReqOrig = "FBDC Request, originator FBCDC(TLA) Memory Fence"; break;
+				}
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+				           "%s, MCU sideband 0x%X", pszReqOrig, ui32MCUSB);
+				pszTagSB = pszScratchBuf;
+			}
+			else
+			{
+				IMG_UINT32 ui32Req   = (ui32TagSB >> 2) & 0x7;
+				IMG_UINT32 ui32MCUSB = (ui32TagSB >> 0) & 0x3;
+				IMG_CHAR* pszReqOrig = "";
+
+				switch (ui32Req)
+				{
+					case 0x0: pszReqOrig = "FBC Request, originator ZLS";   break;
+					case 0x1: pszReqOrig = "FBC Request, originator PBE";   break;
+					case 0x2: pszReqOrig = "FBC Request, originator Host";  break;
+					case 0x3: pszReqOrig = "FBC Request, originator TLA";   break;
+					case 0x4: pszReqOrig = "FBDC Request, originator ZLS";  break;
+					case 0x5: pszReqOrig = "FBDC Request, originator MCU";  break;
+					case 0x6: pszReqOrig = "FBDC Request, originator Host"; break;
+					case 0x7: pszReqOrig = "FBDC Request, originator TLA";  break;
+				}
+				OSSNPrintf(pszScratchBuf, ui32ScratchBufSize,
+				           "%s, MCU sideband 0x%X", pszReqOrig, ui32MCUSB);
+				pszTagSB = pszScratchBuf;
+			}
+			break;
+		}
+	} /* switch (TagID) */
+
+	*ppszTagID = pszTagID;
+	*ppszTagSB = pszTagSB;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDecodeMMULevel
+
+ @Description
+
+ Return the name for the MMU level that faulted.
+
+ @Input ui32MMULevel	 - MMU level
+
+ @Return   IMG_CHAR* to the string describing the MMU level that faulted.
+
+******************************************************************************/
+static const IMG_CHAR* _RGXDecodeMMULevel(IMG_UINT32 ui32MMULevel)
+{
+	const IMG_CHAR* pszMMULevel = "";
+
+	switch (ui32MMULevel)
+	{
+		case 0x0: pszMMULevel = " (Page Table)"; break;
+		case 0x1: pszMMULevel = " (Page Directory)"; break;
+		case 0x2: pszMMULevel = " (Page Catalog)"; break;
+		case 0x3: pszMMULevel = " (Cat Base Reg)"; break;
+	}
+
+	return pszMMULevel;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpRGXBIFBank
+
+ @Description
+
+ Dump BIF Bank state in human readable form.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo            - RGX device info
+ @Input eBankID              - BIF identifier
+ @Input ui64MMUStatus        - MMU Status register value
+ @Input ui64ReqStatus        - BIF request Status register value
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpRGXBIFBank(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					RGXDBG_BIF_ID eBankID,
+					IMG_UINT64 ui64MMUStatus,
+					IMG_UINT64 ui64ReqStatus,
+					const IMG_CHAR *pszIndent)
+{
+	if (ui64MMUStatus == 0x0)
+	{
+		PVR_DUMPDEBUG_LOG("%s - OK", pszBIFNames[eBankID]);
+	}
+	else
+	{
+		IMG_UINT32 ui32PageSize;
+		IMG_UINT32 ui32PC =
+			(ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+				RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+
+		/* Bank 0 & 1 share the same fields */
+		PVR_DUMPDEBUG_LOG("%s%s - FAULT:",
+						  pszIndent,
+						  pszBIFNames[eBankID]);
+
+		/* MMU Status */
+		{
+			IMG_UINT32 ui32MMUDataType =
+				(ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_DATA_TYPE_SHIFT;
+
+			IMG_BOOL bROFault = (ui64MMUStatus & RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_RO_EN) != 0;
+			IMG_BOOL bProtFault = (ui64MMUStatus & RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_FAULT_PM_META_RO_EN) != 0;
+
+			ui32PageSize = (ui64MMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+						RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+
+			PVR_DUMPDEBUG_LOG("%s  * MMU status (0x%016" IMG_UINT64_FMTSPECx "): PC = %d%s, Page Size = %d%s%s%s.",
+						pszIndent,
+						ui64MMUStatus,
+						ui32PC,
+						(ui32PC < 0x8)?"":_RGXDecodePMPC(ui32PC),
+						ui32PageSize,
+						(bROFault)?", Read Only fault":"",
+						(bProtFault)?", PM/META protection fault":"",
+						_RGXDecodeMMULevel(ui32MMUDataType));
+		}
+
+		/* Req Status */
+		{
+			IMG_CHAR *pszTagID;
+			IMG_CHAR *pszTagSB;
+			IMG_CHAR aszScratch[RGX_DEBUG_STR_SIZE];
+			IMG_BOOL bRead;
+			IMG_UINT32 ui32TagSB, ui32TagID;
+			IMG_UINT64 ui64Addr;
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+			{
+				bRead = (ui64ReqStatus & RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN) != 0;
+				ui32TagSB = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT;
+				ui32TagID = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT;
+			}
+			else
+			{
+				bRead = (ui64ReqStatus & RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_RNW_EN) != 0;
+				ui32TagSB = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_SB_SHIFT;
+				ui32TagID = (ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_CLRMSK) >>
+					RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_TAG_ID_SHIFT;
+			}
+			ui64Addr = ((ui64ReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK) >>
+				RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT) <<
+				RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT;
+
+			_RGXDecodeBIFReqTags(psDevInfo, eBankID, ui32TagID, ui32TagSB, &pszTagID, &pszTagSB, &aszScratch[0], RGX_DEBUG_STR_SIZE);
+
+			PVR_DUMPDEBUG_LOG("%s  * Request (0x%016" IMG_UINT64_FMTSPECx
+						"): %s (%s), %s " IMG_DEV_VIRTADDR_FMTSPEC ".",
+							  pszIndent,
+							  ui64ReqStatus,
+			                  pszTagID,
+			                  pszTagSB,
+			                  (bRead)?"Reading from":"Writing to",
+			                  ui64Addr);
+		}
+	}
+}
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__RNW_EN == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_RNW_EN mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_SB_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_SB_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS__XE_MEM__TAG_ID_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_TAG_ID_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_SHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_SHIFT mismatch!");
+static_assert((RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_ALIGNSHIFT == RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT),
+			  "RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_ALIGNSHIFT mismatch!");
+
+static_assert((RGX_CR_MMU_FAULT_STATUS_CONTEXT_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_CONTEXT_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_CONTEXT_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_ADDRESS_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_ADDRESS_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_ADDRESS_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TAG_SB_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TAG_SB_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TAG_SB_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_REQ_ID_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_REQ_ID_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_REQ_ID_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_LEVEL_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_LEVEL_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_LEVEL_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_RNW_EN == RGX_CR_MMU_FAULT_STATUS_META_RNW_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_FAULT_EN == RGX_CR_MMU_FAULT_STATUS_META_FAULT_EN),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_CLRMSK == RGX_CR_MMU_FAULT_STATUS_META_TYPE_CLRMSK),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+static_assert((RGX_CR_MMU_FAULT_STATUS_TYPE_SHIFT == RGX_CR_MMU_FAULT_STATUS_META_TYPE_SHIFT),
+			  "RGX_CR_MMU_FAULT_STATUS_META mismatch!");
+
+
+#if !defined(NO_HARDWARE)
+static inline IMG_CHAR const *_GetRISCVException(IMG_UINT32 ui32Mcause)
+{
+	switch (ui32Mcause)
+	{
+#define X(value, fatal, description) \
+		case value: \
+			if (fatal) \
+				return description; \
+			return NULL;
+
+		RGXRISCVFW_MCAUSE_TABLE
+#undef X
+
+		default:
+			PVR_DPF((PVR_DBG_WARNING, "Invalid RISC-V FW mcause value 0x%08x", ui32Mcause));
+			return NULL;
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+static const IMG_FLAGS2DESC asHWErrorState[] =
+{
+	{RGX_HW_ERR_NA, "N/A"},
+	{RGX_HW_ERR_PRIMID_FAILURE_DURING_DMKILL, "Primitive ID failure during DM kill."},
+};
+
+/*
+ *  Translate ID code to descriptive string.
+ *  Returns on the first match.
+ */
+static void _ID2Description(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, const IMG_FLAGS2DESC *psConvTable, IMG_UINT32 ui32TableSize, IMG_UINT32 ui32ID)
+{
+	IMG_UINT32 ui32Idx;
+
+	for (ui32Idx = 0; ui32Idx < ui32TableSize; ui32Idx++)
+	{
+		if (ui32ID == psConvTable[ui32Idx].uiFlag)
+		{
+			OSStringSafeCopy(psDesc, psConvTable[ui32Idx].pszLabel, ui32DescSize);
+			return;
+		}
+	}
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWAssert
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psRGXFWIfTraceBufCtl - RGX FW trace buffer
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWAssert(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl)
+{
+	const IMG_CHAR *pszTraceAssertPath;
+	const IMG_CHAR *pszTraceAssertInfo;
+	IMG_INT32 ui32TraceAssertLine;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf, INVALIDATE);
+		pszTraceAssertPath = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szPath;
+		pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.szInfo;
+		ui32TraceAssertLine = psRGXFWIfTraceBufCtl->sTraceBuf[i].sAssertBuf.ui32LineNum;
+
+		/* print non-null assert strings */
+		if (*pszTraceAssertInfo)
+		{
+			PVR_DUMPDEBUG_LOG("FW-T%d Assert: %.*s (%.*s:%d)",
+			                  i, RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, pszTraceAssertPath, ui32TraceAssertLine);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_RGXDumpFWFaults
+
+ @Description
+
+ Dump FW assert strings when a thread asserts.
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psFwSysData       - RGX FW shared system data
+
+ @Return   void
+
+******************************************************************************/
+static void _RGXDumpFWFaults(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             const RGXFWIF_SYSDATA *psFwSysData)
+{
+	if (psFwSysData->ui32FWFaults > 0)
+	{
+		IMG_UINT32	ui32StartFault = psFwSysData->ui32FWFaults - RGXFWIF_FWFAULTINFO_MAX;
+		IMG_UINT32	ui32EndFault   = psFwSysData->ui32FWFaults - 1;
+		IMG_UINT32  ui32Index;
+
+		if (psFwSysData->ui32FWFaults < RGXFWIF_FWFAULTINFO_MAX)
+		{
+			ui32StartFault = 0;
+		}
+
+		for (ui32Index = ui32StartFault; ui32Index <= ui32EndFault; ui32Index++)
+		{
+			const RGX_FWFAULTINFO *psFaultInfo = &psFwSysData->sFaultInfo[ui32Index % RGXFWIF_FWFAULTINFO_MAX];
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			/* Split OS timestamp in seconds and nanoseconds */
+			RGXConvertOSTimestampToSAndNS(psFaultInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+			PVR_DUMPDEBUG_LOG("FW Fault %d: %.*s (%.*s:%d)",
+			                  ui32Index+1, RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szInfo,
+			                  RGXFW_TRACE_BUFFER_ASSERT_SIZE, psFaultInfo->sFaultBuf.szPath,
+			                  psFaultInfo->sFaultBuf.ui32LineNum);
+			PVR_DUMPDEBUG_LOG("            Data = 0x%016"IMG_UINT64_FMTSPECx", CRTimer = 0x%012"IMG_UINT64_FMTSPECx", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+			                  psFaultInfo->ui64Data,
+			                  psFaultInfo->ui64CRTimer,
+			                  ui64Seconds, ui64Nanoseconds);
+		}
+	}
+}
+
+static void _RGXDumpFWPoll(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGXFWIF_SYSDATA *psFwSysData)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psFwSysData->aui32CrPollAddr[i])
+		{
+			PVR_DUMPDEBUG_LOG("T%u polling %s (reg:0x%08X mask:0x%08X)",
+			                  i,
+			                  ((psFwSysData->aui32CrPollAddr[i] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+			                  psFwSysData->aui32CrPollAddr[i] & ~RGXFW_POLL_TYPE_SET,
+			                  psFwSysData->aui32CrPollMask[i]);
+		}
+	}
+
+}
+
+static void _RGXDumpFWHWRInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							  void *pvDumpDebugFile,
+							  const RGXFWIF_SYSDATA *psFwSysData,
+							  const RGXFWIF_HWRINFOBUF *psHWRInfoBuf,
+							  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL          bAnyLocked = IMG_FALSE;
+	IMG_UINT32        dm, i;
+	IMG_UINT32        ui32LineSize;
+	IMG_CHAR          *pszLine, *pszTemp;
+	const IMG_CHAR    *apszDmNames[RGXFWIF_DM_MAX] = {"GP", "TDM", "GEOM", "3D", "CDM", "RAY", "GEOM2", "GEOM3", "GEOM4"};
+	const IMG_CHAR    szMsgHeader[] = "Number of HWR: ";
+	const IMG_CHAR    szMsgFalse[] = "FALSE(";
+	IMG_CHAR          *pszLockupType = "";
+	const IMG_UINT32  ui32MsgHeaderCharCount = ARRAY_SIZE(szMsgHeader) - 1; /* size includes the null */
+	const IMG_UINT32  ui32MsgFalseCharCount = ARRAY_SIZE(szMsgFalse) - 1;
+	IMG_UINT32        ui32HWRRecoveryFlags;
+	IMG_UINT32        ui32ReadIndex;
+
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM)))
+	{
+		apszDmNames[RGXFWIF_DM_TDM] = "2D";
+	}
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] ||
+		    psHWRInfoBuf->aui32HwrDmOverranCount[dm])
+		{
+			bAnyLocked = IMG_TRUE;
+			break;
+		}
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) && !bAnyLocked && (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_HARDWARE_OK))
+	{
+		/* No HWR situation, print nothing */
+		return;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_BOOL bAnyHWROccurred = IMG_FALSE;
+
+		for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+		{
+			if (psHWRInfoBuf->aui32HwrDmRecoveredCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm] != 0 ||
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm] !=0)
+				{
+					bAnyHWROccurred = IMG_TRUE;
+					break;
+				}
+		}
+
+		if (!bAnyHWROccurred)
+		{
+			return;
+		}
+	}
+
+/* <DM name + left parenthesis> + <UINT32 max num of digits> + <slash> + <UINT32 max num of digits> +
+   <plus> + <UINT32 max num of digits> + <right parenthesis + comma + space> */
+#define FWHWRINFO_DM_STR_SIZE (5U + 10U + 1U + 10U + 1U + 10U + 3U)
+
+	ui32LineSize = sizeof(IMG_CHAR) * (
+			ui32MsgHeaderCharCount +
+			(psDevInfo->sDevFeatureCfg.ui32MAXDMCount * FWHWRINFO_DM_STR_SIZE) +
+			ui32MsgFalseCharCount + 1 + (psDevInfo->sDevFeatureCfg.ui32MAXDMCount*6) + 1
+				/* 'FALSE(' + ')' + (UINT16 max num + comma) per DM + \0 */
+			);
+
+	pszLine = OSAllocMem(ui32LineSize);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			"%s: Out of mem allocating line string (size: %d)",
+			__func__,
+			ui32LineSize));
+		return;
+	}
+
+	OSStringSafeCopy(pszLine, szMsgHeader, ui32LineSize);
+	pszTemp = pszLine + ui32MsgHeaderCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				FWHWRINFO_DM_STR_SIZE,
+				"%s(%u/%u+%u), ",
+				apszDmNames[dm],
+				psHWRInfoBuf->aui32HwrDmRecoveredCount[dm],
+				psHWRInfoBuf->aui32HwrDmLockedUpCount[dm],
+				psHWRInfoBuf->aui32HwrDmOverranCount[dm]);
+	}
+
+	OSStringLCat(pszLine, szMsgFalse, ui32LineSize);
+	pszTemp += ui32MsgFalseCharCount;
+
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		pszTemp += OSSNPrintf(pszTemp,
+				10 + 1 + 1 /* UINT32 max num + comma + \0 */,
+				(dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1 ? "%u," : "%u)"),
+				psHWRInfoBuf->aui32HwrDmFalseDetectCount[dm]);
+	}
+
+	PVR_DUMPDEBUG_LOG("%s", pszLine);
+
+	OSFreeMem(pszLine);
+
+	/* Print out per HWR info */
+	for (dm = 0; dm < psDevInfo->sDevFeatureCfg.ui32MAXDMCount; dm++)
+	{
+		if (dm == RGXFWIF_DM_GP)
+		{
+			PVR_DUMPDEBUG_LOG("DM %d (GP)", dm);
+		}
+		else
+		{
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+			{
+				IMG_UINT32 ui32HWRRecoveryFlags = psFwSysData->aui32HWRRecoveryFlags[dm];
+				IMG_CHAR sPerDmHwrDescription[RGX_DEBUG_STR_SIZE];
+				sPerDmHwrDescription[0] = '\0';
+
+				if (ui32HWRRecoveryFlags == RGXFWIF_DM_STATE_WORKING)
+				{
+					OSStringSafeCopy(sPerDmHwrDescription, " working;", RGX_DEBUG_STR_SIZE);
+				}
+				else
+				{
+					DebugCommonFlagStrings(sPerDmHwrDescription, RGX_DEBUG_STR_SIZE,
+						asDmState2Description, ARRAY_SIZE(asDmState2Description),
+						ui32HWRRecoveryFlags);
+				}
+				PVR_DUMPDEBUG_LOG("DM %d (HWRflags 0x%08x:%.*s)", dm, ui32HWRRecoveryFlags,
+								  RGX_DEBUG_STR_SIZE, sPerDmHwrDescription);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("DM %d", dm);
+			}
+		}
+
+		ui32ReadIndex = 0;
+		for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+		{
+			IMG_BOOL bPMFault = IMG_FALSE;
+			IMG_UINT32 ui32PC;
+			IMG_UINT32 ui32PageSize = 0;
+			IMG_DEV_PHYADDR sPCDevPAddr = { 0 };
+			const RGX_HWRINFO *psHWRInfo = &psHWRInfoBuf->sHWRInfo[ui32ReadIndex];
+
+			if (ui32ReadIndex >= RGXFWIF_HWINFO_MAX)
+			{
+				PVR_DUMPDEBUG_LOG("HWINFO index error: %u", ui32ReadIndex);
+				break;
+			}
+
+			if ((psHWRInfo->eDM == dm) && (psHWRInfo->ui32HWRNumber != 0))
+			{
+				IMG_CHAR aui8RecoveryNum[10+10+1];
+				IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+				IMG_BOOL bPageFault = IMG_FALSE;
+				IMG_DEV_VIRTADDR sFaultDevVAddr;
+
+				/* Split OS timestamp in seconds and nanoseconds */
+				RGXConvertOSTimestampToSAndNS(psHWRInfo->ui64OSTimer, &ui64Seconds, &ui64Nanoseconds);
+
+				ui32HWRRecoveryFlags = psHWRInfo->ui32HWRRecoveryFlags;
+				if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_LOCKUP) { pszLockupType = ", Guilty Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_LOCKUP) { pszLockupType = ", Innocent Lockup"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GUILTY_OVERRUNING) { pszLockupType = ", Guilty Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_INNOCENT_OVERRUNING) { pszLockupType = ", Innocent Overrun"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_HARD_CONTEXT_SWITCH) { pszLockupType = ", Hard Context Switch"; }
+				else if (ui32HWRRecoveryFlags & RGXFWIF_DM_STATE_GPU_ECC_HWR) { pszLockupType = ", GPU ECC HWR"; }
+
+				OSSNPrintf(aui8RecoveryNum, sizeof(aui8RecoveryNum), "Recovery %d:", psHWRInfo->ui32HWRNumber);
+				if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+				{
+					PVR_DUMPDEBUG_LOG("  %s Core = %u, PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32CoreID,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s PID = %u / %.*s, frame = %d, HWRTData = 0x%08X, EventStatus = 0x%08X%s",
+				                   aui8RecoveryNum,
+				                   psHWRInfo->ui32PID,
+				                   RGXFW_PROCESS_NAME_LEN, psHWRInfo->szProcName,
+				                   psHWRInfo->ui32FrameNum,
+				                   psHWRInfo->ui32ActiveHWRTData,
+				                   psHWRInfo->ui32EventStatus,
+				                   pszLockupType);
+				}
+
+				if (psHWRInfo->eHWErrorCode != RGX_HW_ERR_NA)
+				{
+					IMG_CHAR sHWDebugInfo[RGX_DEBUG_STR_SIZE] = "";
+
+					_ID2Description(sHWDebugInfo, RGX_DEBUG_STR_SIZE, asHWErrorState, ARRAY_SIZE(asHWErrorState),
+						psHWRInfo->eHWErrorCode);
+					PVR_DUMPDEBUG_LOG("  HW error code = 0x%X: %s",
+									  psHWRInfo->eHWErrorCode, sHWDebugInfo);
+				}
+
+				pszTemp = &aui8RecoveryNum[0];
+				while (*pszTemp != '\0')
+				{
+					*pszTemp++ = ' ';
+				}
+
+				/* There's currently no time correlation for the Guest OSes on the Firmware so there's no point printing OS Timestamps on Guests */
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", OSTimer = %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   ui64Seconds,
+									   ui64Nanoseconds,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("  %s CRTimer = 0x%012"IMG_UINT64_FMTSPECX", CyclesElapsed = %" IMG_INT64_FMTSPECd,
+									   aui8RecoveryNum,
+									   psHWRInfo->ui64CRTimer,
+									   (psHWRInfo->ui64CRTimer-psHWRInfo->ui64CRTimeOfKick)*256);
+				}
+
+				if (psHWRInfo->ui64CRTimeHWResetFinish != 0)
+				{
+					if (psHWRInfo->ui64CRTimeFreelistReady != 0)
+					{
+						/* If ui64CRTimeFreelistReady is less than ui64CRTimeHWResetFinish it means APM kicked in and the time is not valid. */
+						if (psHWRInfo->ui64CRTimeHWResetFinish < psHWRInfo->ui64CRTimeFreelistReady)
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalRecoveryTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimeHWResetFinish)*256,
+											   (psHWRInfo->ui64CRTimeFreelistReady-psHWRInfo->ui64CRTimer)*256);
+						}
+						else
+						{
+							PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", FreelistReconTimeInCycles = <not_timed>, TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+											   aui8RecoveryNum,
+											   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+											   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+						}
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("  %s PreResetTimeInCycles = %" IMG_INT64_FMTSPECd ", HWResetTimeInCycles = %" IMG_INT64_FMTSPECd ", TotalResetTimeInCycles = %" IMG_INT64_FMTSPECd,
+										   aui8RecoveryNum,
+										   (psHWRInfo->ui64CRTimeHWResetStart-psHWRInfo->ui64CRTimer)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimeHWResetStart)*256,
+										   (psHWRInfo->ui64CRTimeHWResetFinish-psHWRInfo->ui64CRTimer)*256);
+					}
+				}
+
+#if defined(HW_ERN_65104_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 65104))
+				{
+					PVR_DUMPDEBUG_LOG("    Active PDS DM USCs = 0x%08x", psHWRInfo->ui32PDSActiveDMUSCs);
+				}
+#endif
+
+#if defined(HW_ERN_69700_BIT_MASK)
+				if (RGX_IS_ERN_SUPPORTED(psDevInfo, 69700))
+				{
+					PVR_DUMPDEBUG_LOG("    DMs stalled waiting on PDS Store space = 0x%08x", psHWRInfo->ui32PDSStalledDMs);
+				}
+#endif
+
+				switch (psHWRInfo->eHWRType)
+				{
+					case RGX_HWRTYPE_BIF0FAULT:
+					case RGX_HWRTYPE_BIF1FAULT:
+					{
+						_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXFWIF_HWRTYPE_BIF_BANK_GET(psHWRInfo->eHWRType),
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+										DD_NORMAL_INDENT);
+
+						bPageFault = IMG_TRUE;
+						sFaultDevVAddr.uiAddr = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK);
+						ui32PC = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+								RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+						bPMFault = (ui32PC >= 8);
+						ui32PageSize = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+									RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+						sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+					}
+					break;
+					case RGX_HWRTYPE_TEXASBIF0FAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+						{
+							_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+										psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+										DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							sFaultDevVAddr.uiAddr = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus & ~RGX_CR_BIF_FAULT_BANK0_REQ_STATUS_ADDRESS_CLRMSK);
+							ui32PC = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_CLRMSK) >>
+									RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_CAT_BASE_SHIFT;
+							bPMFault = (ui32PC >= 8);
+							ui32PageSize = (psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus & ~RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+										RGX_CR_BIF_FAULT_BANK0_MMU_STATUS_PAGE_SIZE_SHIFT;
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_ECCFAULT:
+					{
+						PVR_DUMPDEBUG_LOG("    ECC fault GPU=0x%08x", psHWRInfo->uHWRData.sECCInfo.ui32FaultGPU);
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUFAULT:
+					{
+					}
+					break;
+
+					case RGX_HWRTYPE_MMUMETAFAULT:
+					{
+					}
+					break;
+
+					case RGX_HWRTYPE_POLLFAILURE:
+					{
+						PVR_DUMPDEBUG_LOG("    T%u polling %s (reg:0x%08X mask:0x%08X last:0x%08X)",
+										  psHWRInfo->uHWRData.sPollInfo.ui32ThreadNum,
+										  ((psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollAddr & ~RGXFW_POLL_TYPE_SET,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollMask,
+										  psHWRInfo->uHWRData.sPollInfo.ui32CrPollLastValue);
+					}
+					break;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+					case RGX_HWRTYPE_MIPSTLBFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+						{
+							IMG_UINT32 ui32EntryLo = psHWRInfo->uHWRData.sTLBInfo.ui32EntryLo;
+
+							/* This is not exactly what the MMU code does, but the result should be the same */
+							const IMG_UINT32 ui32UnmappedEntry =
+								((IMG_UINT32)(MMU_BAD_PHYS_ADDR & 0xffffffff) & RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT) | RGXMIPSFW_ENTRYLO_UNCACHED;
+
+							PVR_DUMPDEBUG_LOG("    MIPS TLB fault: BadVA = 0x%08X, EntryLo = 0x%08X"
+											  " (page PA 0x%" IMG_UINT64_FMTSPECx", V %u)",
+											  psHWRInfo->uHWRData.sTLBInfo.ui32BadVAddr,
+											  ui32EntryLo,
+											  RGXMIPSFW_TLB_GET_PA(ui32EntryLo),
+											  ui32EntryLo & RGXMIPSFW_TLB_VALID ? 1 : 0);
+
+							if (ui32EntryLo == ui32UnmappedEntry)
+							{
+								PVR_DUMPDEBUG_LOG("    Potential use-after-free detected");
+							}
+						}
+					}
+					break;
+#endif
+
+					case RGX_HWRTYPE_MMURISCVFAULT:
+					{
+						if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+						{
+							_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_FWCORE,
+											psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus,
+											psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus,
+											DD_NORMAL_INDENT);
+
+							bPageFault = IMG_TRUE;
+							bPMFault = IMG_FALSE;
+							sFaultDevVAddr.uiAddr =
+								(psHWRInfo->uHWRData.sBIFInfo.ui64BIFReqStatus &
+								 ~RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS_ADDRESS_CLRMSK);
+							ui32PageSize =
+								(psHWRInfo->uHWRData.sBIFInfo.ui64BIFMMUStatus &
+								 ~RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_CLRMSK) >>
+								RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS_PAGE_SIZE_SHIFT;
+							sPCDevPAddr.uiAddr = psHWRInfo->uHWRData.sBIFInfo.ui64PCAddress;
+						}
+					}
+					break;
+
+					case RGX_HWRTYPE_OVERRUN:
+					case RGX_HWRTYPE_UNKNOWNFAILURE:
+					{
+						/* Nothing to dump */
+					}
+					break;
+
+					default:
+					{
+						PVR_DUMPDEBUG_LOG("    Unknown HWR Info type: 0x%x", psHWRInfo->eHWRType);
+					}
+					break;
+				}
+
+				if (bPageFault)
+				{
+					RGXDumpFaultInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psHWRInfo,
+					                 ui32ReadIndex, &sFaultDevVAddr, &sPCDevPAddr, bPMFault, ui32PageSize);
+				}
+
+			}
+
+			if (ui32ReadIndex == RGXFWIF_HWINFO_MAX_FIRST - 1)
+				ui32ReadIndex = psHWRInfoBuf->ui32WriteIndex;
+			else
+				ui32ReadIndex = (ui32ReadIndex + 1) - (ui32ReadIndex / RGXFWIF_HWINFO_LAST_INDEX) * RGXFWIF_HWINFO_MAX_LAST;
+		}
+	}
+}
+
+
+
+
+#if !defined(NO_HARDWARE)
+
+/*!
+*******************************************************************************
+
+ @Function	_CheckForPendingPage
+
+ @Description
+
+ Check if the MMU indicates it is blocked on a pending page
+ MMU4 does not support pending pages, so return false.
+
+ @Input psDevInfo	 - RGX device info
+
+ @Return   IMG_BOOL      - IMG_TRUE if there is a pending page
+
+******************************************************************************/
+static INLINE IMG_BOOL _CheckForPendingPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* MMU4 doesn't support pending pages */
+	return (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) < 4) &&
+		   (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MMU_ENTRY) & RGX_CR_MMU_ENTRY_PENDING_EN);
+#else
+	IMG_UINT32 ui32BIFMMUEntry;
+
+	ui32BIFMMUEntry = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY);
+
+	if (ui32BIFMMUEntry & RGX_CR_BIF_MMU_ENTRY_PENDING_EN)
+	{
+		return IMG_TRUE;
+	}
+	else
+	{
+		return IMG_FALSE;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetPendingPageInfo
+
+ @Description
+
+ Get information about the pending page from the MMU status registers
+
+ @Input psDevInfo	 - RGX device info
+ @Output psDevVAddr      - The device virtual address of the pending MMU address translation
+ @Output pui32CatBase    - The page catalog base
+
+ @Return   void
+
+******************************************************************************/
+static void _GetPendingPageInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_VIRTADDR *psDevVAddr,
+								IMG_UINT32 *pui32CatBase)
+{
+	IMG_UINT64 ui64BIFMMUEntryStatus;
+
+	ui64BIFMMUEntryStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_MMU_ENTRY_STATUS);
+
+	psDevVAddr->uiAddr = (ui64BIFMMUEntryStatus & ~RGX_CR_BIF_MMU_ENTRY_STATUS_ADDRESS_CLRMSK);
+
+	*pui32CatBase = (ui64BIFMMUEntryStatus & ~RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_CLRMSK) >>
+								RGX_CR_BIF_MMU_ENTRY_STATUS_CAT_BASE_SHIFT;
+}
+
+#endif
+
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON)
+{
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	const RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	IMG_UINT32 ui32DriverID;
+	const RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	/* space for the current clock speed and 3 previous */
+	RGXFWIF_TIME_CORR asTimeCorrs[4];
+	IMG_UINT32 ui32NumClockSpeedChanges;
+
+	/* Should invalidate all reads below including when passed to functions. */
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, INVALIDATE);
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(bRGXPoweredON);
+#else
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		{
+			IMG_UINT64	ui64RegValMMUStatus, ui64RegValREQStatus;
+
+			ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK0_MMU_STATUS);
+			ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK0_REQ_STATUS);
+
+			_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_BIF0, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+
+			if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, SINGLE_BIF)))
+			{
+				ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK1_MMU_STATUS);
+				ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_FAULT_BANK1_REQ_STATUS);
+				_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_BIF1, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+			}
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+			{
+				ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_FWCORE_MEM_FAULT_MMU_STATUS);
+				ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_FWCORE_MEM_FAULT_REQ_STATUS);
+				_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_FWCORE, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+			}
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, CLUSTER_GROUPING))
+			{
+				IMG_UINT32  ui32PhantomCnt = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, NUM_CLUSTERS) ?  RGX_REQ_NUM_PHANTOMS(RGX_GET_FEATURE_VALUE(psDevInfo, NUM_CLUSTERS)) : 0;
+
+				if (ui32PhantomCnt > 1)
+				{
+					IMG_UINT32  ui32Phantom;
+					for (ui32Phantom = 0;  ui32Phantom < ui32PhantomCnt;  ui32Phantom++)
+					{
+						/* This can't be done as it may interfere with the FW... */
+						/*OSWriteHWReg64(RGX_CR_TEXAS_INDIRECT, ui32Phantom);*/
+
+						ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS);
+						ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS);
+
+						_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+					}
+				}else
+				{
+					ui64RegValMMUStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_MMU_STATUS);
+					ui64RegValREQStatus = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TEXAS_BIF_FAULT_BANK0_REQ_STATUS);
+
+					_RGXDumpRGXBIFBank(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, RGXDBG_TEXAS_BIF, ui64RegValMMUStatus, ui64RegValREQStatus, DD_SUMMARY_INDENT);
+				}
+			}
+		}
+
+		if (_CheckForPendingPage(psDevInfo))
+		{
+			IMG_UINT32 ui32CatBase;
+			IMG_DEV_VIRTADDR sDevVAddr;
+
+			PVR_DUMPDEBUG_LOG("MMU Pending page: Yes");
+
+			_GetPendingPageInfo(psDevInfo, &sDevVAddr, &ui32CatBase);
+
+			if (ui32CatBase >= 8)
+			{
+				PVR_DUMPDEBUG_LOG("Cannot check address on PM cat base %u", ui32CatBase);
+			}
+			else
+			{
+				IMG_DEV_PHYADDR sPCDevPAddr;
+				MMU_FAULT_DATA sFaultData;
+				IMG_BOOL bIsValid = IMG_TRUE;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+				IMG_UINT64 ui64CBaseMapping;
+				IMG_UINT32 ui32CBaseMapCtxReg;
+
+				if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__INVALID_EN);
+				}
+				else
+				{
+					ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+
+					OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32CBaseMapCtxReg, ui32CatBase);
+
+					ui64CBaseMapping = OSReadUncheckedHWReg64(psDevInfo->pvSecureRegsBaseKM, RGX_CR_MMU_CBASE_MAPPING);
+					sPCDevPAddr.uiAddr = (((ui64CBaseMapping & ~RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT);
+					bIsValid = !(ui64CBaseMapping & RGX_CR_MMU_CBASE_MAPPING_INVALID_EN);
+				}
+#else
+				sPCDevPAddr.uiAddr = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_CAT_BASEN(ui32CatBase));
+#endif
+
+				PVR_DUMPDEBUG_LOG("Checking device virtual address " IMG_DEV_VIRTADDR_FMTSPEC
+							" on cat base %u. PC Addr = 0x%" IMG_UINT64_FMTSPECx " is %s",
+								sDevVAddr.uiAddr,
+								ui32CatBase,
+								sPCDevPAddr.uiAddr,
+								bIsValid ? "valid":"invalid");
+				RGXCheckFaultAddress(psDevInfo, &sDevVAddr, &sPCDevPAddr, &sFaultData);
+				RGXDumpFaultAddressHostView(&sFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_SUMMARY_INDENT);
+			}
+		}
+	}
+#endif /* NO_HARDWARE */
+
+#if !defined(NO_HARDWARE)
+	/* Determine the type virtualisation support used */
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+#if defined(SUPPORT_AUTOVZ)
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with HW register support");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: AutoVz with shared memory");
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with static Fw heap allocation");
+#endif /* defined(SUPPORT_AUTOVZ) */
+#else
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation type: Hypervisor-assisted with dynamic Fw heap allocation");
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+	}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		RGXFWIF_CONNECTION_FW_STATE eFwState;
+		RGXFWIF_CONNECTION_OS_STATE eOsState;
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+		eFwState = KM_GET_FW_CONNECTION(psDevInfo);
+		eOsState = KM_GET_OS_CONNECTION(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation firmware connection state: %s (Fw=%s; OS=%s)",
+						  ((eFwState == RGXFW_CONNECTION_FW_ACTIVE) && (eOsState == RGXFW_CONNECTION_OS_ACTIVE)) ? ("UP") : ("DOWN"),
+						  (eFwState < RGXFW_CONNECTION_FW_STATE_COUNT) ? (apszFwOsStateName[eFwState]) : ("invalid"),
+						  (eOsState < RGXFW_CONNECTION_OS_STATE_COUNT) ? (apszFwOsStateName[eOsState]) : ("invalid"));
+
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32FwAliveTS;
+		IMG_UINT32 ui32OsAliveTS;
+
+		KM_ALIVE_TOKEN_CACHEOP(Fw, INVALIDATE);
+		KM_ALIVE_TOKEN_CACHEOP(Os, INVALIDATE);
+
+		ui32FwAliveTS = KM_GET_FW_ALIVE_TOKEN(psDevInfo);
+		ui32OsAliveTS = KM_GET_OS_ALIVE_TOKEN(psDevInfo);
+
+		PVR_DUMPDEBUG_LOG("RGX Virtualisation watchdog timestamps (in GPU timer ticks): Fw=%u; OS=%u; diff(FW, OS) = %u",
+						  ui32FwAliveTS, ui32OsAliveTS, ui32FwAliveTS - ui32OsAliveTS);
+	}
+#endif
+#endif /* !defined(NO_HARDWARE) */
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_CHAR sHwrStateDescription[RGX_DEBUG_STR_SIZE];
+		IMG_BOOL bDriverIsolationEnabled = IMG_FALSE;
+		IMG_UINT32 ui32HostIsolationGroup;
+
+		sHwrStateDescription[0] = '\0';
+
+		DebugCommonFlagStrings(sHwrStateDescription, RGX_DEBUG_STR_SIZE,
+			asHwrState2Description, ARRAY_SIZE(asHwrState2Description),
+			psFwSysData->ui32HWRStateFlags);
+		PVR_DUMPDEBUG_LOG("RGX HWR State 0x%08x:%s", psFwSysData->ui32HWRStateFlags, sHwrStateDescription);
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: %s (APM %s: %d ok, %d denied, %d non-idle, %d retry, %d other, %d total. Latency: %u ms)",
+		                  (psFwSysData->ePowState < ARRAY_SIZE(pszPowStateName) ? pszPowStateName[psFwSysData->ePowState] : "???"),
+		                  (psDevInfo->pvAPMISRData)?"enabled":"disabled",
+		                  psDevInfo->ui32ActivePMReqOk - psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqDenied,
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqRetry,
+		                  psDevInfo->ui32ActivePMReqTotal -
+		                  psDevInfo->ui32ActivePMReqOk -
+		                  psDevInfo->ui32ActivePMReqDenied -
+		                  psDevInfo->ui32ActivePMReqRetry -
+		                  psDevInfo->ui32ActivePMReqNonIdle,
+		                  psDevInfo->ui32ActivePMReqTotal,
+		                  psRuntimeCfg->ui32ActivePMLatencyms);
+
+		ui32NumClockSpeedChanges = (IMG_UINT32) OSAtomicRead(&psDevInfo->psDeviceNode->iNumClockSpeedChanges);
+		RGXGetTimeCorrData(psDevInfo->psDeviceNode, asTimeCorrs, ARRAY_SIZE(asTimeCorrs));
+
+		PVR_DUMPDEBUG_LOG("RGX DVFS: %u frequency changes. "
+		                  "Current frequency: %u.%03u MHz (sampled at %" IMG_UINT64_FMTSPEC " ns). "
+		                  "FW frequency: %u.%03u MHz.",
+		                  ui32NumClockSpeedChanges,
+		                  asTimeCorrs[0].ui32CoreClockSpeed / 1000000,
+		                  (asTimeCorrs[0].ui32CoreClockSpeed / 1000) % 1000,
+		                  asTimeCorrs[0].ui64OSTimeStamp,
+		                  psRuntimeCfg->ui32CoreClockSpeed / 1000000,
+		                  (psRuntimeCfg->ui32CoreClockSpeed / 1000) % 1000);
+		if (ui32NumClockSpeedChanges > 0)
+		{
+			PVR_DUMPDEBUG_LOG("          Previous frequencies: %u.%03u, %u.%03u, %u.%03u MHz (Sampled at "
+							"%" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ", %" IMG_UINT64_FMTSPEC ")",
+												asTimeCorrs[1].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[1].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[2].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[2].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[3].ui32CoreClockSpeed / 1000000,
+												(asTimeCorrs[3].ui32CoreClockSpeed / 1000) % 1000,
+												asTimeCorrs[1].ui64OSTimeStamp,
+												asTimeCorrs[2].ui64OSTimeStamp,
+												asTimeCorrs[3].ui64OSTimeStamp);
+		}
+
+		ui32HostIsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID];
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			RGXFWIF_OS_RUNTIME_FLAGS sFwRunFlags = psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+			IMG_UINT32 ui32IsolationGroup = psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID];
+			IMG_BOOL bMTSEnabled = IMG_FALSE;
+
+#if !defined(NO_HARDWARE)
+			if (bRGXPoweredON)
+			{
+				bMTSEnabled = (
+#if defined(FIX_HW_BRN_64502_BIT_MASK)
+								RGX_IS_BRN_SUPPORTED(psDevInfo, 64502) ||
+#endif
+								!RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_VIRTUALISATION)) ?
+								IMG_TRUE : ((OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE_ENABLE) & BIT(ui32DriverID)) != 0);
+			}
+#endif
+
+			PVR_DUMPDEBUG_LOG("RGX FW OS %u - State: %s; Freelists: %s%s; Priority: %u; Isolation group: %u;  Time Slice%s: %u%% (%ums); %s", ui32DriverID,
+							  apszFwOsStateName[sFwRunFlags.bfOsState],
+							  (sFwRunFlags.bfFLOk) ? "Ok" : "Not Ok",
+							  (sFwRunFlags.bfFLGrowPending) ? "; Grow Request Pending" : "",
+							  psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID],
+							  ui32IsolationGroup,
+							  (psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] != 0) ? "" : "*",
+							  psFwSysData->aui32TSMirror[ui32DriverID],
+							  (psFwSysData->aui32TSMirror[ui32DriverID] *
+							   psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs / 100),
+							  (bMTSEnabled) ? "MTS on;" : "MTS off;"
+							 );
+
+			if (ui32IsolationGroup != ui32HostIsolationGroup)
+			{
+				bDriverIsolationEnabled = IMG_TRUE;
+			}
+
+			if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+			{
+				/* don't print guest information on native mode drivers */
+				break;
+			}
+		}
+
+#if defined(PVR_ENABLE_PHR)
+		{
+			IMG_CHAR sPHRConfigDescription[RGX_DEBUG_STR_SIZE];
+
+			sPHRConfigDescription[0] = '\0';
+			DebugCommonFlagStrings(sPHRConfigDescription, RGX_DEBUG_STR_SIZE,
+			                   asPHRConfig2Description, ARRAY_SIZE(asPHRConfig2Description),
+			                   BIT_ULL(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode));
+
+			PVR_DUMPDEBUG_LOG("RGX PHR configuration: (%d) %.*s", psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, RGX_DEBUG_STR_SIZE, sPHRConfigDescription);
+		}
+#endif
+
+		if (bRGXPoweredON && RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			if (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM) > 1U)
+			{
+				PVR_DUMPDEBUG_LOG("RGX MC Configuration: 0x%X (1:primary, 0:secondary)", psFwSysData->ui32McConfig);
+			}
+		}
+
+		if (bDriverIsolationEnabled)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Hard Context Switch deadline: %u ms", psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+		}
+
+		_RGXDumpFWAssert(pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl);
+		_RGXDumpFWFaults(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+		_RGXDumpFWPoll(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("RGX FW State: Unavailable under Guest Mode of operation");
+		PVR_DUMPDEBUG_LOG("RGX FW Power State: Unavailable under Guest Mode of operation");
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, INVALIDATE);
+	_RGXDumpFWHWRInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psFwSysData, psDevInfo->psRGXFWIfHWRInfoBufCtl, psDevInfo);
+
+#if defined(SUPPORT_RGXFW_STATS_FRAMEWORK)
+	/* Dump all non-zero values in lines of 8... */
+	{
+		IMG_CHAR    pszLine[(9*RGXFWIF_STATS_FRAMEWORK_LINESIZE)+1];
+		const IMG_UINT32 *pui32FWStatsBuf = psFwSysData->aui32FWStatsBuf;
+		IMG_UINT32  ui32Index1, ui32Index2;
+
+		PVR_DUMPDEBUG_LOG("STATS[START]: RGXFWIF_STATS_FRAMEWORK_MAX=%d", RGXFWIF_STATS_FRAMEWORK_MAX);
+		for (ui32Index1 = 0;  ui32Index1 < RGXFWIF_STATS_FRAMEWORK_MAX;  ui32Index1 += RGXFWIF_STATS_FRAMEWORK_LINESIZE)
+		{
+			IMG_UINT32  ui32OrOfValues = 0;
+			IMG_CHAR    *pszBuf = pszLine;
+
+			/* Print all values in this line and skip if all zero... */
+			for (ui32Index2 = 0;  ui32Index2 < RGXFWIF_STATS_FRAMEWORK_LINESIZE;  ui32Index2++)
+			{
+				ui32OrOfValues |= pui32FWStatsBuf[ui32Index1+ui32Index2];
+				OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32FWStatsBuf[ui32Index1+ui32Index2]);
+				pszBuf += 9; /* write over the '\0' */
+			}
+
+			if (ui32OrOfValues != 0)
+			{
+				PVR_DUMPDEBUG_LOG("STATS[%08x]:%s", ui32Index1, pszLine);
+			}
+		}
+		PVR_DUMPDEBUG_LOG("STATS[END]");
+	}
+#endif
+}
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	RGXRISCVFW_STATE sRiscvState;
+	const IMG_CHAR *pszException;
+	PVRSRV_ERROR eError;
+
+	DDLOG64(FWCORE_MEM_CAT_BASE0);
+	DDLOG64(FWCORE_MEM_CAT_BASE1);
+	DDLOG64(FWCORE_MEM_CAT_BASE2);
+	DDLOG64(FWCORE_MEM_CAT_BASE3);
+	DDLOG64(FWCORE_MEM_CAT_BASE4);
+	DDLOG64(FWCORE_MEM_CAT_BASE5);
+	DDLOG64(FWCORE_MEM_CAT_BASE6);
+	DDLOG64(FWCORE_MEM_CAT_BASE7);
+
+	/* Limit dump to what is currently being used */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG4__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG5__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG6__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG12__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG13__HOST_SECURITY_GEQ4);
+		DDLOGUNCHECKED64(FWCORE_ADDR_REMAP_CONFIG14__HOST_SECURITY_GEQ4);
+	}
+	else
+#endif
+	{
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG4);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG5);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG6);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG12);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG13);
+		DDLOG64(FWCORE_ADDR_REMAP_CONFIG14);
+	}
+
+	DDLOG32(FWCORE_MEM_FAULT_MMU_STATUS);
+	DDLOG64(FWCORE_MEM_FAULT_REQ_STATUS);
+	DDLOG32(FWCORE_MEM_MMU_STATUS);
+	DDLOG32(FWCORE_MEM_READS_EXT_STATUS);
+	DDLOG32(FWCORE_MEM_READS_INT_STATUS);
+
+	PVR_DUMPDEBUG_LOG("---- [ RISC-V internal state ] ----");
+
+#if  defined(SUPPORT_RISCV_GDB)
+	if (RGXRiscvIsHalted(psDevInfo))
+	{
+		/* Avoid resuming the RISC-V FW as most operations
+		 * on the debug module require a halted core */
+		PVR_DUMPDEBUG_LOG("(skipping as RISC-V found halted)");
+		return PVRSRV_OK;
+	}
+#endif
+
+	eError = RGXRiscvHalt(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+#define X(name, address)												\
+	eError = RGXRiscvReadReg(psDevInfo, address, &sRiscvState.name);	\
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXRiscvReadReg", _RISCVDMError);	\
+	DDLOGVAL32(#name, sRiscvState.name);
+
+	RGXRISCVFW_DEBUG_DUMP_REGISTERS
+#undef X
+
+	eError = RGXRiscvResume(psDevInfo);
+	PVR_GOTO_IF_ERROR(eError, _RISCVDMError);
+
+	pszException = _GetRISCVException(sRiscvState.mcause);
+	if (pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG("RISC-V FW hit an exception: %s", pszException);
+
+		eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+		}
+	}
+
+	return PVRSRV_OK;
+
+_RISCVDMError:
+	PVR_DPF((PVR_DBG_ERROR, "Failed to communicate with the Debug Module"));
+
+	return eError;
+}
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBVNC_COREID_REG))
+	{
+		DDLOG64(CORE_ID__PBVNC);
+	}
+	else
+	{
+		DDLOG32(CORE_ID);
+		DDLOG32(CORE_REVISION);
+	}
+	DDLOG32(DESIGNER_REV_FIELD1);
+	DDLOG32(DESIGNER_REV_FIELD2);
+	DDLOG64(CHANGESET_NUMBER);
+}
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(MULTICORE_SYSTEM);
+	DDLOG32(MULTICORE_GPU);
+}
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG64(CLK_CTRL);
+	DDLOG64(CLK_STATUS);
+	DDLOG64(CLK_CTRL2);
+	DDLOG64(CLK_STATUS2);
+}
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	{
+		DDLOG32(BIF_FAULT_BANK0_MMU_STATUS);
+		DDLOG64(BIF_FAULT_BANK0_REQ_STATUS);
+		DDLOG32(BIF_FAULT_BANK1_MMU_STATUS);
+		DDLOG64(BIF_FAULT_BANK1_REQ_STATUS);
+	}
+	DDLOG32(BIF_MMU_STATUS);
+	DDLOG32(BIF_MMU_ENTRY);
+	DDLOG64(BIF_MMU_ENTRY_STATUS);
+
+	{
+		if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE)))
+		{
+			DDLOG32(BIF_STATUS_MMU);
+			DDLOG32(BIF_READS_EXT_STATUS);
+			DDLOG32(BIF_READS_INT_STATUS);
+		}
+		DDLOG32(BIFPM_STATUS_MMU);
+		DDLOG32(BIFPM_READS_EXT_STATUS);
+		DDLOG32(BIFPM_READS_INT_STATUS);
+	}
+
+	{
+		DDLOG64(BIF_CAT_BASE_INDEX);
+		DDLOG64(BIF_CAT_BASE0);
+		DDLOG64(BIF_CAT_BASE1);
+		DDLOG64(BIF_CAT_BASE2);
+		DDLOG64(BIF_CAT_BASE3);
+		DDLOG64(BIF_CAT_BASE4);
+		DDLOG64(BIF_CAT_BASE5);
+		DDLOG64(BIF_CAT_BASE6);
+		DDLOG64(BIF_CAT_BASE7);
+	}
+
+	DDLOG32(BIF_CTRL_INVAL);
+	DDLOG32(BIF_CTRL);
+}
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	IMG_UINT32   ui32TACycles, ui323DCycles, ui32TAOr3DCycles, ui32TAAnd3DCycles;
+
+	DDLOG64(BIF_PM_CAT_BASE_VCE0);
+	DDLOG64(BIF_PM_CAT_BASE_TE0);
+	DDLOG64(BIF_PM_CAT_BASE_ALIST0);
+	DDLOG64(BIF_PM_CAT_BASE_VCE1);
+	DDLOG64(BIF_PM_CAT_BASE_TE1);
+	DDLOG64(BIF_PM_CAT_BASE_ALIST1);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		DDLOG32(MULTICORE_GEOMETRY_CTRL_COMMON);
+		DDLOG32(MULTICORE_FRAGMENT_CTRL_COMMON);
+		DDLOG32(MULTICORE_COMPUTE_CTRL_COMMON);
+	}
+
+	DDLOG32(PERF_TA_PHASE);
+	DDLOG32(PERF_TA_CYCLE);
+	DDLOG32(PERF_3D_PHASE);
+	DDLOG32(PERF_3D_CYCLE);
+
+	ui32TACycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_TA_CYCLE);
+	ui323DCycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_3D_CYCLE);
+	ui32TAOr3DCycles = OSReadHWReg32(pvRegsBaseKM, RGX_CR_PERF_TA_OR_3D_CYCLE);
+	ui32TAAnd3DCycles = ((ui32TACycles + ui323DCycles) > ui32TAOr3DCycles) ? (ui32TACycles + ui323DCycles - ui32TAOr3DCycles) : 0;
+	DDLOGVAL32("PERF_TA_OR_3D_CYCLE", ui32TAOr3DCycles);
+	DDLOGVAL32("PERF_TA_AND_3D_CYCLE", ui32TAAnd3DCycles);
+
+	DDLOG32(PERF_COMPUTE_PHASE);
+	DDLOG32(PERF_COMPUTE_CYCLE);
+
+	DDLOG32(PM_PARTIAL_RENDER_ENABLE);
+
+	DDLOG32(ISP_RENDER);
+	DDLOG64(TLA_STATUS);
+	DDLOG64(MCU_FENCE);
+
+	DDLOG32(VDM_CONTEXT_STORE_STATUS);
+	DDLOG64(VDM_CONTEXT_STORE_TASK0);
+	DDLOG64(VDM_CONTEXT_STORE_TASK1);
+	DDLOG64(VDM_CONTEXT_STORE_TASK2);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK0);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK1);
+	DDLOG64(VDM_CONTEXT_RESUME_TASK2);
+
+	DDLOG32(ISP_CTL);
+	DDLOG32(ISP_STATUS);
+
+	DDLOG32(CDM_CONTEXT_STORE_STATUS);
+	DDLOG64(CDM_CONTEXT_PDS0);
+	DDLOG64(CDM_CONTEXT_PDS1);
+	DDLOG64(CDM_TERMINATE_PDS);
+	DDLOG64(CDM_TERMINATE_PDS1);
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 47025))
+	{
+		DDLOG64(CDM_CONTEXT_LOAD_PDS0);
+		DDLOG64(CDM_CONTEXT_LOAD_PDS1);
+	}
+}
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	{
+		DDLOG32(SLC_IDLE);
+		DDLOG32(SLC_STATUS0);
+		DDLOG64(SLC_STATUS1);
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_BANKS) && RGX_GET_FEATURE_VALUE(psDevInfo, SLC_BANKS))
+		{
+			DDLOG64(SLC_STATUS2);
+		}
+
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+		{
+			DDLOG64(SLC_CTRL_BYPASS);
+		}
+		else
+		{
+			DDLOG32(SLC_CTRL_BYPASS);
+		}
+		DDLOG64(SLC_CTRL_MISC);
+	}
+}
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+
+	DDLOG32(SIDEKICK_IDLE);
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ROGUEXE) &&
+		RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER))
+	{
+		DDLOG32(SAFETY_EVENT_STATUS__ROGUEXE);
+		DDLOG32(MTS_SAFETY_EVENT_ENABLE__ROGUEXE);
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER))
+	{
+		DDLOG32(FWCORE_WDT_CTRL);
+	}
+
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDevInfo->psDeviceNode, LAYOUT_MARS) > 0)
+	{
+		DDLOG32(SCRATCH0);
+		DDLOG32(SCRATCH1);
+		DDLOG32(SCRATCH2);
+		DDLOG32(SCRATCH3);
+		DDLOG32(SCRATCH4);
+		DDLOG32(SCRATCH5);
+		DDLOG32(SCRATCH6);
+		DDLOG32(SCRATCH7);
+		DDLOG32(SCRATCH8);
+		DDLOG32(SCRATCH9);
+		DDLOG32(SCRATCH10);
+		DDLOG32(SCRATCH11);
+		DDLOG32(SCRATCH12);
+		DDLOG32(SCRATCH13);
+		DDLOG32(SCRATCH14);
+		DDLOG32(SCRATCH15);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#undef REG32_FMTSPEC
+#undef REG64_FMTSPEC
+#undef DDLOG32
+#undef DDLOG64
+#undef DDLOG32_DPX
+#undef DDLOG64_DPX
+#undef DDLOGVAL32
+
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32VerbLevel)
+{
+	DumpTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	DumpRenderCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	DumpKickSyncCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE))
+	{
+		DumpComputeCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, FASTRENDER_DM))
+	{
+		DumpTDMTransferCtxtsInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+}
+
+/******************************************************************************
+ End of file (rgxdebug.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxdebug_common.c b/drivers/gpu/drm/img-rogue/rgxdebug_common.c
new file mode 100644
index 000000000000..6486d035db9d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxdebug_common.c
@@ -0,0 +1,3233 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rgx debug information
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "rgxdebug_common.h"
+#include "pvrversion.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_sf.h"
+#include "rgxfw_log_helper.h"
+#include "fwtrace_string.h"
+#include "rgxmmudefs_km.h"
+#include "rgxinit.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "rgxfwimageutils.h"
+#include "fwload.h"
+#include "rgx_options.h"
+#include "devicemem_history_server.h"
+#include "debug_common.h"
+#include "info_page.h"
+#include "osfunc.h"
+#if defined(SUPPORT_SOC_TIMER)
+#include "rgxtimecorr.h"
+#endif
+
+#define MAX_FW_DESCRIPTION_LENGTH	(600U)
+
+#define PVR_DUMP_FIRMWARE_INFO(x)														\
+	PVR_DUMPDEBUG_LOG("FW info: %d.%d @ %8d (%s) build options: 0x%08x",				\
+						PVRVERSION_UNPACK_MAJ((x).ui32DDKVersion),						\
+						PVRVERSION_UNPACK_MIN((x).ui32DDKVersion),						\
+						(x).ui32DDKBuild,												\
+						((x).ui32BuildOptions & OPTIONS_DEBUG_EN) ? "debug":"release",	\
+						(x).ui32BuildOptions);
+
+#define PVR_DUMP_FIRMWARE_INFO_HDR(x)												\
+	PVR_DUMPDEBUG_LOG("FW info: %d.%d @ %8d (%s) build options: 0x%08x",		\
+						(x).ui16PVRVersionMajor,								\
+						(x).ui16PVRVersionMinor,								\
+						(x).ui32PVRVersionBuild,								\
+						((x).ui32Flags & OPTIONS_DEBUG_EN) ? "debug":"release",	\
+						(x).ui32Flags);
+
+typedef struct {
+	IMG_UINT16     ui16Mask;
+	const IMG_CHAR *pszStr;
+} RGXFWT_DEBUG_INFO_MSKSTR; /* pair of bit mask and debug info message string */
+
+/*
+ *  Array of all the Firmware Trace log IDs used to convert the trace data.
+ */
+typedef struct _TRACEBUF_LOG_ {
+	RGXFW_LOG_SFids	eSFId;
+	const IMG_CHAR	*pszName;
+	const IMG_CHAR	*pszFmt;
+	IMG_UINT32		ui32ArgNum;
+} TRACEBUF_LOG;
+
+static const TRACEBUF_LOG aLogDefinitions[] =
+{
+#define X(a, b, c, d, e) {RGXFW_LOG_CREATESFID(a,b,e), #c, d, e},
+	RGXFW_LOG_SFIDLIST
+#undef X
+};
+
+static const IMG_FLAGS2DESC asCswOpts2Description[] =
+{
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_FAST, " Fast CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_MEDIUM, " Medium CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_SLOW, " Slow CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_PROFILE_NODELAY, " No Delay CSW profile;"},
+	{RGXFWIF_INICFG_CTXSWITCH_MODE_RAND, " Random Csw enabled;"},
+	{RGXFWIF_INICFG_CTXSWITCH_SRESET_EN, " SoftReset;"},
+};
+
+static const IMG_FLAGS2DESC asMisc2Description[] =
+{
+	{RGXFWIF_INICFG_POW_RASCALDUST, " Power Rascal/Dust;"},
+	{RGXFWIF_INICFG_SPU_CLOCK_GATE, " SPU Clock Gating (requires Power Rascal/Dust);"},
+	{RGXFWIF_INICFG_HWPERF_EN, " HwPerf EN;"},
+	{RGXFWIF_INICFG_FBCDC_V3_1_EN, " FBCDCv3.1;"},
+	{RGXFWIF_INICFG_CHECK_MLIST_EN, " Check MList;"},
+	{RGXFWIF_INICFG_DISABLE_CLKGATING_EN, " ClockGating Off;"},
+	{RGXFWIF_INICFG_REGCONFIG_EN, " Register Config;"},
+	{RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY, " Assert on OOM;"},
+	{RGXFWIF_INICFG_HWP_DISABLE_FILTER, " HWP Filter Off;"},
+	{RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN, " CDM Random kill;"},
+	{RGXFWIF_INICFG_DISABLE_DM_OVERLAP, " DM Overlap Off;"},
+	{RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER, " Assert on HWR;"},
+	{RGXFWIF_INICFG_INJECT_ICS_FAULT, " Inject ICS Fault;"},
+	{RGXFWIF_INICFG_VALIDATE_IRQ, " Validate IRQ;"},
+	{RGXFWIF_INICFG_DISABLE_PDP_EN, " PDUMP Panic off;"},
+	{RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN, " SPU Pow mask change on;"},
+	{RGXFWIF_INICFG_WORKEST, " Workload Estim;"},
+	{RGXFWIF_INICFG_PDVFS, " PDVFS;"},
+	{RGXFWIF_INICFG_ISPSCHEDMODE_VER1_IPP, " ISP v1 scheduling;"},
+	{RGXFWIF_INICFG_ISPSCHEDMODE_VER2_ISP, " ISP v2 scheduling;"},
+	{RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER, " Validate SOC&USC timers;"},
+};
+
+static const IMG_FLAGS2DESC asFwOsCfg2Description[] =
+{
+	{RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN, " TDM;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN, " GEOM;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN, " 3D;"},
+	{RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN, " CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN, " RDM;"},
+#endif
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM, " LowPrio TDM;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM, " LowPrio GEOM;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D, " LowPrio 3D;"},
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM, " LowPrio CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM, " LowPrio RDM;"},
+#endif
+#if defined(SUPPORT_ICS)
+	{RGXFWIF_INICFG_OS_ICS_TDM_EN, " TDM;"},
+	{RGXFWIF_INICFG_OS_ICS_GEOM_EN, " GEOM;"},
+	{RGXFWIF_INICFG_OS_ICS_3D_EN, " 3D;"},
+	{RGXFWIF_INICFG_OS_ICS_CDM_EN, " CDM;"},
+#if defined(SUPPORT_RAY_TRACING)
+	{RGXFWIF_INICFG_OS_ICS_RDM_EN, " RDM;"},
+#endif
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_LONG, " Long FDTI profile;"},
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_MEDIUM, " Medium FDTI profile;"},
+	{RGXFWIF_INICFG_OS_FDTI_PROFILE_SHORT, " Short FDTI profile;"},
+#endif
+};
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+const IMG_CHAR * const gapszMipsPermissionPTFlags[4] =
+{
+	"    ",
+	"XI  ",
+	"RI  ",
+	"RIXI"
+};
+
+const IMG_CHAR * const gapszMipsCoherencyPTFlags[8] =
+{
+	"C",
+	"C",
+	" ",
+	"C",
+	"C",
+	"C",
+	"C",
+	" "
+};
+
+const IMG_CHAR * const gapszMipsDirtyGlobalValidPTFlags[8] =
+{
+	"   ",
+	"  G",
+	" V ",
+	" VG",
+	"D  ",
+	"D G",
+	"DV ",
+	"DVG"
+};
+
+#if !defined(NO_HARDWARE)
+/* Translation of MIPS exception encoding */
+typedef struct _MIPS_EXCEPTION_ENCODING_
+{
+	const IMG_CHAR *const pszStr;	/* Error type */
+	const IMG_BOOL bIsFatal;	/* Error is fatal or non-fatal */
+} MIPS_EXCEPTION_ENCODING;
+
+static const MIPS_EXCEPTION_ENCODING apsMIPSExcCodes[] =
+{
+	{"Interrupt", IMG_FALSE},
+	{"TLB modified exception", IMG_FALSE},
+	{"TLB exception (load/instruction fetch)", IMG_FALSE},
+	{"TLB exception (store)", IMG_FALSE},
+	{"Address error exception (load/instruction fetch)", IMG_TRUE},
+	{"Address error exception (store)", IMG_TRUE},
+	{"Bus error exception (instruction fetch)", IMG_TRUE},
+	{"Bus error exception (load/store)", IMG_TRUE},
+	{"Syscall exception", IMG_FALSE},
+	{"Breakpoint exception (FW assert)", IMG_FALSE},
+	{"Reserved instruction exception", IMG_TRUE},
+	{"Coprocessor Unusable exception", IMG_FALSE},
+	{"Arithmetic Overflow exception", IMG_FALSE},
+	{"Trap exception", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"Implementation-Specific Exception 1 (COP2)", IMG_FALSE},
+	{"CorExtend Unusable", IMG_FALSE},
+	{"Coprocessor 2 exceptions", IMG_FALSE},
+	{"TLB Read-Inhibit", IMG_TRUE},
+	{"TLB Execute-Inhibit", IMG_TRUE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"Reference to WatchHi/WatchLo address", IMG_FALSE},
+	{"Machine check", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{"DSP Module State Disabled exception", IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	{NULL, IMG_FALSE},
+	/* Can only happen in MIPS debug mode */
+	{"Parity error", IMG_FALSE},
+	{NULL, IMG_FALSE}
+};
+
+static IMG_CHAR const *_GetMIPSExcString(IMG_UINT32 ui32ExcCode)
+{
+	if (ui32ExcCode >= sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Only %lu exceptions available in MIPS, %u is not a valid exception code",
+		         (unsigned long)sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING), ui32ExcCode));
+		return NULL;
+	}
+
+	return apsMIPSExcCodes[ui32ExcCode].pszStr;
+}
+#endif
+
+typedef struct _RGXMIPSFW_C0_DEBUG_TBL_ENTRY_
+{
+    IMG_UINT32 ui32Mask;
+    const IMG_CHAR * pszExplanation;
+} RGXMIPSFW_C0_DEBUG_TBL_ENTRY;
+
+#if !defined(NO_HARDWARE)
+static const RGXMIPSFW_C0_DEBUG_TBL_ENTRY sMIPS_C0_DebugTable[] =
+{
+    { RGXMIPSFW_C0_DEBUG_DSS,      "Debug single-step exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DBP,      "Debug software breakpoint exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBL,     "Debug data break exception occurred on a load" },
+    { RGXMIPSFW_C0_DEBUG_DDBS,     "Debug data break exception occurred on a store" },
+    { RGXMIPSFW_C0_DEBUG_DIB,      "Debug instruction break exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DINT,     "Debug interrupt exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DIBIMPR,  "Imprecise debug instruction break exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBLIMPR, "Imprecise debug data break load exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DDBSIMPR, "Imprecise debug data break store exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_IEXI,     "Imprecise error exception inhibit controls exception occurred" },
+    { RGXMIPSFW_C0_DEBUG_DBUSEP,   "Data access Bus Error exception pending" },
+    { RGXMIPSFW_C0_DEBUG_CACHEEP,  "Imprecise Cache Error pending" },
+    { RGXMIPSFW_C0_DEBUG_MCHECKP,  "Imprecise Machine Check exception pending" },
+    { RGXMIPSFW_C0_DEBUG_IBUSEP,   "Instruction fetch Bus Error exception pending" },
+    { (IMG_UINT32)RGXMIPSFW_C0_DEBUG_DBD,      "Debug exception occurred in branch delay slot" }
+};
+#endif
+#endif
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)*/
+
+#define NARGS_MASK ~(0xF<<16)
+static IMG_BOOL _FirmwareTraceIntegrityCheck(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile)
+{
+	const TRACEBUF_LOG *psLogDef = &aLogDefinitions[0];
+	IMG_BOOL bIntegrityOk = IMG_TRUE;
+
+	/*
+	 * For every log ID, check the format string and number of arguments is valid.
+	 */
+	while (psLogDef->eSFId != RGXFW_SF_LAST)
+	{
+		const TRACEBUF_LOG *psLogDef2;
+		const IMG_CHAR *pszString;
+		IMG_UINT32 ui32Count;
+
+		/*
+		 * Check the number of arguments matches the number of '%' in the string and
+		 * check that no string uses %s which is not supported as it requires a
+		 * pointer to memory that is not going to be valid.
+		 */
+		pszString = psLogDef->pszFmt;
+		ui32Count = 0;
+
+		while (*pszString != '\0')
+		{
+			if (*pszString++ == '%')
+			{
+				ui32Count++;
+				if (*pszString == 's')
+				{
+					bIntegrityOk = IMG_FALSE;
+					PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has an unsupported type not recognized (fmt: %%%c). Please fix.",
+									  psLogDef->pszName, *pszString);
+				}
+				else if (*pszString == '%')
+				{
+					/* Double % is a printable % sign and not a format string... */
+					ui32Count--;
+					pszString++;
+				}
+			}
+		}
+
+		if (ui32Count != psLogDef->ui32ArgNum)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has %d arguments but only %d are specified. Please fix.",
+			                  psLogDef->pszName, ui32Count, psLogDef->ui32ArgNum);
+		}
+
+		/* RGXDumpFirmwareTrace() has a hardcoded limit of supporting up to 20 arguments... */
+		if (ui32Count > 20)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s has %d arguments but a maximum of 20 are supported. Please fix.",
+			                  psLogDef->pszName, ui32Count);
+		}
+
+		/* Check the id number is unique (don't take into account the number of arguments) */
+		ui32Count = 0;
+		psLogDef2 = &aLogDefinitions[0];
+
+		while (psLogDef2->eSFId != RGXFW_SF_LAST)
+		{
+			if ((psLogDef->eSFId & NARGS_MASK) == (psLogDef2->eSFId & NARGS_MASK))
+			{
+				ui32Count++;
+			}
+			psLogDef2++;
+		}
+
+		if (ui32Count != 1)
+		{
+			bIntegrityOk = IMG_FALSE;
+			PVR_DUMPDEBUG_LOG("Integrity Check FAIL: %s id %x is not unique, there are %d more. Please fix.",
+			                  psLogDef->pszName, psLogDef->eSFId, ui32Count - 1);
+		}
+
+		/* Move to the next log ID... */
+		psLogDef++;
+	}
+
+	return bIntegrityOk;
+}
+
+void RGXDumpFirmwareTrace(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO  *psDevInfo)
+{
+	RGXFWIF_TRACEBUF  *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	static IMG_BOOL   bIntegrityCheckPassed = IMG_FALSE;
+
+	/* Check that the firmware trace is correctly defined... */
+	if (!bIntegrityCheckPassed)
+	{
+		bIntegrityCheckPassed = _FirmwareTraceIntegrityCheck(pfnDumpDebugPrintf, pvDumpDebugFile);
+		if (!bIntegrityCheckPassed)
+		{
+			return;
+		}
+	}
+
+	/* Dump FW trace information... */
+	if (psRGXFWIfTraceBufCtl != NULL)
+	{
+		IMG_UINT32  tid;
+
+		PVR_DUMPDEBUG_LOG("Device ID: %u", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+		RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+
+		/* Print the log type settings... */
+		if (psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+		{
+			PVR_DUMPDEBUG_LOG("Debug log type: %s ( " RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC ")",
+							  ((psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)?("trace"):("tbi")),
+							  RGXFWIF_LOG_ENABLED_GROUPS_LIST(psRGXFWIfTraceBufCtl->ui32LogType)
+							  );
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("Debug log type: none");
+		}
+
+		/* Print the decoded log for each thread... */
+		for (tid = 0;  tid < RGXFW_THREAD_NUM;  tid++)
+		{
+			RGXDumpFirmwareTraceDecoded(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXPrepareExtraDebugInfo
+
+ @Description
+
+ Prepares debug info string by decoding ui16DebugInfo value passed
+
+ @Input pszBuffer	 - pointer to debug info string buffer
+
+ @Return   void
+
+******************************************************************************/
+static void RGXPrepareExtraDebugInfo(IMG_CHAR *pszBuffer, IMG_UINT32 ui32BufferSize, IMG_UINT16 ui16DebugInfo)
+{
+	const RGXFWT_DEBUG_INFO_MSKSTR aDebugInfoMskStr[] =
+	{
+#define X(a, b) {a, b},
+		RGXFWT_DEBUG_INFO_MSKSTRLIST
+#undef X
+	};
+
+	IMG_UINT32 ui32NumFields = sizeof(aDebugInfoMskStr)/sizeof(RGXFWT_DEBUG_INFO_MSKSTR);
+	IMG_UINT32 i;
+	IMG_BOOL   bHasExtraDebugInfo = IMG_FALSE;
+
+	/* Add prepend string */
+	OSStringSafeCopy(pszBuffer, RGXFWT_DEBUG_INFO_STR_PREPEND, ui32BufferSize);
+
+	/* Add debug info strings */
+	for (i = 0; i < ui32NumFields; i++)
+	{
+		if (ui16DebugInfo & aDebugInfoMskStr[i].ui16Mask)
+		{
+			if (bHasExtraDebugInfo)
+			{
+				OSStringLCat(pszBuffer, ", ", ui32BufferSize); /* Add comma separator */
+			}
+			OSStringLCat(pszBuffer, aDebugInfoMskStr[i].pszStr, ui32BufferSize);
+			bHasExtraDebugInfo = IMG_TRUE;
+		}
+	}
+
+	/* Add append string */
+	OSStringLCat(pszBuffer, RGXFWT_DEBUG_INFO_STR_APPEND, ui32BufferSize);
+}
+
+#define PVR_MAX_DEBUG_PARTIAL_LINES (40U)
+#define PVR_DUMPDEBUG_LOG_LINES(fmt, ...) \
+	if (!bPrintAllLines) { \
+		OSSNPrintf(&pszLineBuffer[ui32LastLineIdx * PVR_MAX_DEBUG_MESSAGE_LEN], PVR_MAX_DEBUG_MESSAGE_LEN, (fmt), ##__VA_ARGS__); \
+		ui32LineCount++; \
+		ui32LastLineIdx = ui32LineCount % PVR_MAX_DEBUG_PARTIAL_LINES; \
+	} else { \
+		PVR_UNREFERENCED_PARAMETER(pszLineBuffer); \
+		PVR_UNREFERENCED_PARAMETER(ui32LineCount); \
+		PVR_UNREFERENCED_PARAMETER(ui32LastLineIdx); \
+		PVR_DUMPDEBUG_LOG((fmt), ##__VA_ARGS__); \
+	}
+
+static void RGXDumpFirmwareTraceLines(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID,
+				bool bPrintAllLines)
+{
+	volatile IMG_UINT32  *pui32FWWrapCount;
+	volatile IMG_UINT32  *pui32FWTracePtr;
+	IMG_UINT32           *pui32TraceBuf;
+	IMG_UINT32           *pui32LocalTraceBuf = NULL;
+	IMG_UINT32           ui32HostWrapCount;
+	IMG_UINT32           ui32HostTracePtr;
+	IMG_UINT32           ui32Count           = 0;
+	IMG_UINT32           ui32LineCount       = 0;
+	IMG_UINT32           ui32LastLineIdx     = 0;
+	IMG_CHAR             *pszLineBuffer      = NULL;
+	IMG_UINT32           ui32TraceBufSizeInDWords;
+
+	RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID], INVALIDATE);
+	RGXFwSharedMemCacheOpExec(psDevInfo->apui32TraceBuffer[ui32TID],
+	                          psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	pui32FWWrapCount = &(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].ui32WrapCount);
+	pui32FWTracePtr  = &(psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].ui32TracePointer);
+	pui32TraceBuf    = psDevInfo->apui32TraceBuffer[ui32TID];
+	ui32HostWrapCount = *pui32FWWrapCount;
+	ui32HostTracePtr  = *pui32FWTracePtr;
+
+	if (pui32TraceBuf == NULL)
+	{
+		/* trace buffer not yet allocated */
+		return;
+	}
+
+	if (!bPrintAllLines)
+	{
+		pszLineBuffer = OSAllocMem(PVR_MAX_DEBUG_MESSAGE_LEN * PVR_MAX_DEBUG_PARTIAL_LINES);
+		PVR_LOG_RETURN_VOID_IF_FALSE(pszLineBuffer != NULL, "pszLineBuffer alloc failed");
+	}
+
+	ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+	{
+		PVR_DUMPDEBUG_LOG_LINES("WARNING: Trace pointer (%d) greater than buffer size (%d).",
+								ui32HostTracePtr, ui32TraceBufSizeInDWords);
+		ui32HostTracePtr %= ui32TraceBufSizeInDWords;
+	}
+
+	/*
+	 *  Allocate a local copy of the trace buffer which will contain a static non-changing
+	 *  snapshot view of the buffer. This removes the issue of a fast GPU wrapping and
+	 *  overwriting the tail data of the buffer.
+	 */
+	pui32LocalTraceBuf = OSAllocMem(ui32TraceBufSizeInDWords * sizeof(IMG_UINT32));
+	if (pui32LocalTraceBuf != NULL)
+	{
+		memcpy(pui32LocalTraceBuf, pui32TraceBuf, ui32TraceBufSizeInDWords * sizeof(IMG_UINT32));
+		ui32HostTracePtr = *pui32FWTracePtr;
+		pui32TraceBuf = pui32LocalTraceBuf;
+	}
+
+	while (ui32Count < ui32TraceBufSizeInDWords)
+	{
+		IMG_UINT32  ui32Data, ui32DataToId;
+
+		/* Find the first valid log ID, skipping whitespace... */
+		do
+		{
+			IMG_UINT32 ui32ValidatedHostTracePtr;
+			ui32ValidatedHostTracePtr = OSConfineArrayIndexNoSpeculation(ui32HostTracePtr,
+			                                                             ui32TraceBufSizeInDWords);
+			ui32Data     = pui32TraceBuf[ui32ValidatedHostTracePtr];
+			ui32DataToId = idToStringID(ui32Data, SFs);
+
+			/* If an unrecognized id is found it may be inconsistent data or a firmware trace error. */
+			if (ui32DataToId == RGXFW_SF_LAST  &&  RGXFW_LOG_VALIDID(ui32Data))
+			{
+				PVR_DUMPDEBUG_LOG_LINES("WARNING: Unrecognized id (%x). From here on the trace might be wrong!", ui32Data);
+			}
+
+			/* Update the trace pointer... */
+			ui32HostTracePtr++;
+			if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+			{
+				ui32HostTracePtr = 0;
+				ui32HostWrapCount++;
+			}
+			ui32Count++;
+		} while ((RGXFW_SF_LAST == ui32DataToId)  &&
+				 ui32Count < ui32TraceBufSizeInDWords);
+
+		if (ui32Count < ui32TraceBufSizeInDWords)
+		{
+			IMG_CHAR   szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN] = "%" IMG_UINT64_FMTSPEC ":T%u-%s> ";
+			IMG_CHAR   szDebugInfoBuffer[RGXFWT_DEBUG_INFO_STR_MAXLEN] = "";
+			IMG_UINT64 ui64Timestamp;
+			IMG_UINT16 ui16DebugInfo;
+
+			/* If we hit the ASSERT message then this is the end of the log... */
+			if (ui32Data == RGXFW_SF_MAIN_ASSERT_FAILED)
+			{
+				PVR_DUMPDEBUG_LOG_LINES("ASSERTION %.*s failed at %.*s:%u",
+				                        RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.szInfo,
+				                        RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.szPath,
+				                        psRGXFWIfTraceBufCtl->sTraceBuf[ui32TID].sAssertBuf.ui32LineNum);
+				break;
+			}
+
+			ui64Timestamp = (IMG_UINT64)(pui32TraceBuf[(ui32HostTracePtr + 0) % ui32TraceBufSizeInDWords]) << 32 |
+							(IMG_UINT64)(pui32TraceBuf[(ui32HostTracePtr + 1) % ui32TraceBufSizeInDWords]);
+
+			ui16DebugInfo = (IMG_UINT16) ((ui64Timestamp & ~RGXFWT_TIMESTAMP_DEBUG_INFO_CLRMSK) >> RGXFWT_TIMESTAMP_DEBUG_INFO_SHIFT);
+			ui64Timestamp = (ui64Timestamp & ~RGXFWT_TIMESTAMP_TIME_CLRMSK) >> RGXFWT_TIMESTAMP_TIME_SHIFT;
+
+			/*
+			 * Print the trace string and provide up to 20 arguments which
+			 * printf function will be able to use. We have already checked
+			 * that no string uses more than this.
+			 */
+			OSStringLCat(szBuffer, SFs[ui32DataToId].psName, PVR_MAX_DEBUG_MESSAGE_LEN);
+
+			/* Check and append any extra debug info available */
+			if (ui16DebugInfo)
+			{
+				/* Prepare debug info string */
+				RGXPrepareExtraDebugInfo(szDebugInfoBuffer, RGXFWT_DEBUG_INFO_STR_MAXLEN, ui16DebugInfo);
+
+				/* Append debug info string */
+				OSStringLCat(szBuffer, szDebugInfoBuffer, PVR_MAX_DEBUG_MESSAGE_LEN);
+			}
+
+			PVR_DUMPDEBUG_LOG_LINES(szBuffer, ui64Timestamp, ui32TID, groups[RGXFW_SF_GID(ui32Data)],
+			                        pui32TraceBuf[(ui32HostTracePtr +  2) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  3) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  4) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  5) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  6) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  7) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  8) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr +  9) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 10) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 11) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 12) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 13) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 14) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 15) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 16) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 17) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 18) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 19) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 20) % ui32TraceBufSizeInDWords],
+			                        pui32TraceBuf[(ui32HostTracePtr + 21) % ui32TraceBufSizeInDWords]);
+
+			/* Update the trace pointer... */
+			ui32HostTracePtr = ui32HostTracePtr + 2 + RGXFW_SF_PARAMNUM(ui32Data);
+			if (ui32HostTracePtr >= ui32TraceBufSizeInDWords)
+			{
+				ui32HostTracePtr = ui32HostTracePtr % ui32TraceBufSizeInDWords;
+				ui32HostWrapCount++;
+			}
+			ui32Count = (ui32Count + 2 + RGXFW_SF_PARAMNUM(ui32Data));
+
+			/* Has the FW trace buffer overtaken the host pointer during the last line printed??? */
+			if ((pui32LocalTraceBuf == NULL)  &&
+			    ((*pui32FWWrapCount > ui32HostWrapCount) ||
+			     ((*pui32FWWrapCount == ui32HostWrapCount) && (*pui32FWTracePtr > ui32HostTracePtr))))
+			{
+				/* Move forward to the oldest entry again... */
+				PVR_DUMPDEBUG_LOG_LINES(". . .");
+				ui32HostWrapCount = *pui32FWWrapCount;
+				ui32HostTracePtr  = *pui32FWTracePtr;
+			}
+		}
+	}
+
+	/* Free the local copy of the trace buffer if it was allocated... */
+	if (pui32LocalTraceBuf != NULL)
+	{
+		OSFreeMem(pui32LocalTraceBuf);
+	}
+
+	if (!bPrintAllLines)
+	{
+		IMG_UINT32 ui32FirstLineIdx;
+
+		if (ui32LineCount > PVR_MAX_DEBUG_PARTIAL_LINES)
+		{
+			ui32FirstLineIdx = ui32LastLineIdx;
+			ui32LineCount = PVR_MAX_DEBUG_PARTIAL_LINES;
+		}
+		else
+		{
+			ui32FirstLineIdx = 0;
+		}
+
+		for (ui32Count = 0; ui32Count < ui32LineCount; ui32Count++)
+		{
+			PVR_DUMPDEBUG_LOG("%s", &pszLineBuffer[((ui32FirstLineIdx + ui32Count) % PVR_MAX_DEBUG_PARTIAL_LINES) * PVR_MAX_DEBUG_MESSAGE_LEN]);
+		}
+
+		OSFreeMem(pszLineBuffer);
+	}
+}
+
+void RGXDumpFirmwareTraceDecoded(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	RGXDumpFirmwareTraceLines(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+	                          psRGXFWIfTraceBufCtl, ui32TID, true);
+}
+
+void RGXDumpFirmwareTracePartial(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	RGXDumpFirmwareTraceLines(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+	                          psRGXFWIfTraceBufCtl, ui32TID, false);
+}
+
+void RGXDumpFirmwareTraceBinary(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID)
+{
+	IMG_UINT32	i;
+	IMG_BOOL	bPrevLineWasZero = IMG_FALSE;
+	IMG_BOOL	bLineIsAllZeros = IMG_FALSE;
+	IMG_UINT32	ui32CountLines = 0;
+	IMG_UINT32	*pui32TraceBuffer;
+	IMG_CHAR *pszLine;
+
+	RGXFwSharedMemCacheOpExec(psDevInfo->apui32TraceBuffer[ui32TID],
+	                          psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+	pui32TraceBuffer = psDevInfo->apui32TraceBuffer[ui32TID];
+
+/* Max number of DWords to be printed per line, in debug dump binary output */
+#define PVR_DD_FW_TRACEBUF_LINESIZE 30U
+	/* each element in the line is 8 characters plus a space.  The '+ 1' is because of the final trailing '\0'. */
+	pszLine = OSAllocMem(9 * PVR_DD_FW_TRACEBUF_LINESIZE + 1);
+	if (pszLine == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Out of mem allocating line string (size: %d)",
+				__func__,
+				9 * PVR_DD_FW_TRACEBUF_LINESIZE + 1));
+		return;
+	}
+
+	for (i = 0; i < psDevInfo->ui32TraceBufSizeInDWords; i += PVR_DD_FW_TRACEBUF_LINESIZE)
+	{
+		IMG_UINT32 k = 0;
+		IMG_UINT32 ui32Line = 0x0;
+		IMG_UINT32 ui32LineOffset = i*sizeof(IMG_UINT32);
+		IMG_CHAR   *pszBuf = pszLine;
+
+		for (k = 0; k < PVR_DD_FW_TRACEBUF_LINESIZE; k++)
+		{
+			if ((i + k) >= psDevInfo->ui32TraceBufSizeInDWords)
+			{
+				/* Stop reading when the index goes beyond trace buffer size. This condition is
+				 * hit during printing the last line in DD when ui32TraceBufSizeInDWords is not
+				 * a multiple of PVR_DD_FW_TRACEBUF_LINESIZE */
+				break;
+			}
+
+			ui32Line |= pui32TraceBuffer[i + k];
+
+			/* prepare the line to print it. The '+1' is because of the trailing '\0' added */
+			OSSNPrintf(pszBuf, 9 + 1, " %08x", pui32TraceBuffer[i + k]);
+			pszBuf += 9; /* write over the '\0' */
+		}
+
+		bLineIsAllZeros = (ui32Line == 0x0);
+
+		if (bLineIsAllZeros)
+		{
+			if (bPrevLineWasZero)
+			{
+				ui32CountLines++;
+			}
+			else
+			{
+				bPrevLineWasZero = IMG_TRUE;
+				ui32CountLines = 1;
+				PVR_DUMPDEBUG_LOG("FWT[%08x]: 00000000 ... 00000000", ui32LineOffset);
+			}
+		}
+		else
+		{
+			if (bPrevLineWasZero  &&  ui32CountLines > 1)
+			{
+				PVR_DUMPDEBUG_LOG("FWT[...]: %d lines were all zero", ui32CountLines);
+			}
+			bPrevLineWasZero = IMG_FALSE;
+
+			PVR_DUMPDEBUG_LOG("FWT[%08x]:%s", ui32LineOffset, pszLine);
+		}
+	}
+
+	if (bPrevLineWasZero)
+	{
+		PVR_DUMPDEBUG_LOG("FWT[END]: %d lines were all zero", ui32CountLines);
+	}
+
+	OSFreeMem(pszLine);
+}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+void RGXDocumentFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				const IMG_UINT32 ui32FwVA,
+				const IMG_CPU_PHYADDR sCpuPA,
+				const IMG_DEV_PHYADDR sDevPA,
+				const IMG_UINT64 ui64PTE)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		PVR_DUMPDEBUG_LOG("|    0x%08X   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "%s%s%s   |",
+						  ui32FwVA,
+						  (IMG_UINT64) sCpuPA.uiAddr,
+						  sDevPA.uiAddr,
+						  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(ui64PTE)],
+						  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(ui64PTE)],
+						  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(ui64PTE)]);
+	}
+	else
+#endif
+	{
+		const char *pszSLCBypass =
+#if defined(RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN)
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN) ? "B" : " ";
+#else
+						  " ";
+#endif
+
+		/* META and RISCV use a subset of the GPU's virtual address space */
+		PVR_DUMPDEBUG_LOG("|    0x%08X   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "0x%016" IMG_UINT64_FMTSPECX "   |   "
+						  "%s%s%s%s%s%s   |",
+						  ui32FwVA,
+						  (IMG_UINT64) sCpuPA.uiAddr,
+						  sDevPA.uiAddr,
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN)   ? "P" : " ",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_PM_SRC_EN)          ? "PM" : "  ",
+						  pszSLCBypass,
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_CC_EN)              ? "C" : " ",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_READ_ONLY_EN)       ? "RO" : "RW",
+						  BITMASK_HAS(ui64PTE, RGX_MMUCTRL_PT_DATA_VALID_EN)           ? "V" : " ");
+	}
+}
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+#if !defined(NO_HARDWARE)
+static PVRSRV_ERROR
+RGXPollMetaRegThroughSP(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32RegOffset,
+                        IMG_UINT32 ui32PollValue, IMG_UINT32 ui32Mask)
+{
+	IMG_UINT32 ui32RegValue, ui32NumPolls = 0;
+	PVRSRV_ERROR eError;
+
+	do
+	{
+		eError = RGXReadFWModuleAddr(psDevInfo, ui32RegOffset, &ui32RegValue);
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+	} while (((ui32RegValue & ui32Mask) != ui32PollValue) && (ui32NumPolls++ < 1000));
+
+	return ((ui32RegValue & ui32Mask) == ui32PollValue) ? PVRSRV_OK : PVRSRV_ERROR_RETRY;
+}
+
+PVRSRV_ERROR
+RGXReadMetaCoreReg(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32RegAddr, IMG_UINT32 *pui32RegVal)
+{
+	PVRSRV_ERROR eError;
+
+	/* Core Read Ready? */
+	eError = RGXPollMetaRegThroughSP(psDevInfo,
+	                                 META_CR_TXUXXRXRQ_OFFSET,
+	                                 META_CR_TXUXXRXRQ_DREADY_BIT,
+									 META_CR_TXUXXRXRQ_DREADY_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXPollMetaRegThroughSP");
+
+	/* Set the reg we are interested in reading */
+	eError = RGXWriteFWModuleAddr(psDevInfo, META_CR_TXUXXRXRQ_OFFSET,
+	                        ui32RegAddr | META_CR_TXUXXRXRQ_RDnWR_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXWriteFWModuleAddr");
+
+	/* Core Read Done? */
+	eError = RGXPollMetaRegThroughSP(psDevInfo,
+	                                 META_CR_TXUXXRXRQ_OFFSET,
+	                                 META_CR_TXUXXRXRQ_DREADY_BIT,
+									 META_CR_TXUXXRXRQ_DREADY_BIT);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXPollMetaRegThroughSP");
+
+	/* Read the value */
+	return RGXReadFWModuleAddr(psDevInfo, META_CR_TXUXXRXDT_OFFSET, pui32RegVal);
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR _ValidateWithFWModule(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo,
+						RGXFWIF_DEV_VIRTADDR *psFWAddr,
+						void *pvHostCodeAddr,
+						IMG_UINT32 ui32MaxLen,
+						const IMG_CHAR *pszDesc,
+						IMG_UINT32 ui32StartOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32Value = 0;
+	IMG_UINT32 ui32FWCodeDevVAAddr = psFWAddr->ui32Addr + ui32StartOffset;
+	IMG_UINT32 *pui32FWCode = (IMG_UINT32*) IMG_OFFSET_ADDR(pvHostCodeAddr,ui32StartOffset);
+	IMG_UINT32 i;
+
+#if defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	ui32MaxLen -= ui32StartOffset;
+	ui32MaxLen /= sizeof(IMG_UINT32); /* Byte -> 32 bit words */
+
+	for (i = 0; i < ui32MaxLen; i++)
+	{
+		eError = RGXReadFWModuleAddr(psDevInfo, ui32FWCodeDevVAAddr, &ui32Value);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+#if defined(EMULATOR)
+		if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+#endif
+		{
+			PVR_DPF((PVR_DBG_VERBOSE, "0x%x: CPU 0x%08x, FW 0x%08x", i * 4, pui32FWCode[i], ui32Value));
+
+			if (pui32FWCode[i] != ui32Value)
+			{
+				PVR_DUMPDEBUG_LOG("%s: Mismatch while validating %s at offset 0x%x: CPU 0x%08x (%p), FW 0x%08x (%x)",
+					 __func__, pszDesc,
+					 (i * 4) + ui32StartOffset, pui32FWCode[i], pui32FWCode, ui32Value, ui32FWCodeDevVAAddr);
+				return PVRSRV_ERROR_FW_IMAGE_MISMATCH;
+			}
+		}
+
+		ui32FWCodeDevVAAddr += 4;
+	}
+
+	PVR_DUMPDEBUG_LOG("Match between Host and Firmware view of the %s", pszDesc);
+	return PVRSRV_OK;
+}
+#endif
+
+#if !defined(NO_HARDWARE)
+PVRSRV_ERROR RGXValidateFWImage(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if !defined(SUPPORT_TRUSTED_DEVICE)
+	PVRSRV_ERROR eError;
+	IMG_UINT32 *pui32HostFWCode = NULL, *pui32HostFWCoremem = NULL;
+	OS_FW_IMAGE *psRGXFW = NULL;
+	const IMG_BYTE *pbRGXFirmware = NULL;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_UINT32 *pui32CodeMemoryPointer;
+#endif
+	RGXFWIF_DEV_VIRTADDR sFWAddr;
+	IMG_UINT32 ui32StartOffset = 0;
+
+#if defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		PVR_DUMPDEBUG_LOG("Validation of RISC-V FW code is disabled on emulator");
+		return PVRSRV_OK;
+	}
+#endif
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: RGX registers not mapped yet!", __func__));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	/* Load FW from system for code verification */
+	pui32HostFWCode = OSAllocZMem(psDevInfo->ui32FWCodeSizeInBytes);
+	if (pui32HostFWCode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed in allocating memory for FW code. "
+				"So skipping FW code verification",
+				__func__));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Coremem is not present on all GPU cores, so size can be zero */
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		pui32HostFWCoremem = OSAllocZMem(psDevInfo->ui32FWCorememCodeSizeInBytes);
+		if (pui32HostFWCoremem == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed in allocating memory for FW core code. "
+					"So skipping FW code verification",
+					__func__));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto freeHostFWCode;
+		}
+	}
+
+	/* Load FW image */
+	eError = RGXLoadAndGetFWData(psDevInfo->psDeviceNode, &psRGXFW);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to load FW image file (%s).",
+		         __func__, PVRSRVGetErrorString(eError)));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto cleanup_initfw;
+	}
+
+	pbRGXFirmware = (const IMG_BYTE *)OSFirmwareData(psRGXFW);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		eError = ProcessLDRCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+						(void*) pui32HostFWCode, NULL,
+						(void*) pui32HostFWCoremem, NULL, NULL);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = ProcessELFCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+		                                 pui32HostFWCode, NULL,
+		                                 NULL, NULL);
+	}
+#endif
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = ProcessELFCommandStream(&psDevInfo->sLayerParams, pbRGXFirmware,
+		                                 pui32HostFWCode, NULL,
+		                                 pui32HostFWCoremem, NULL);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed in parsing FW image file.", __func__));
+		goto cleanup_initfw;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, (void **)&pui32CodeMemoryPointer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Error in acquiring MIPS FW code memory area (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+
+		RGXFwSharedMemCacheOpExec(pui32CodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, PVRSRV_CACHE_OP_INVALIDATE);
+
+		if (OSMemCmp(pui32HostFWCode, pui32CodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes) == 0)
+		{
+			PVR_DUMPDEBUG_LOG("Match between Host and MIPS views of the FW code" );
+		}
+		else
+		{
+			IMG_UINT32 ui32Count = 10; /* Show only the first 10 mismatches */
+			IMG_UINT32 ui32Offset;
+
+			PVR_DUMPDEBUG_LOG("Mismatch between Host and MIPS views of the FW code");
+			for (ui32Offset = 0; (ui32Offset*4 < psDevInfo->ui32FWCodeSizeInBytes) || (ui32Count == 0); ui32Offset++)
+			{
+				if (pui32HostFWCode[ui32Offset] != pui32CodeMemoryPointer[ui32Offset])
+				{
+					PVR_DUMPDEBUG_LOG("At %d bytes, code should be 0x%x but it is instead 0x%x",
+					   ui32Offset*4, pui32HostFWCode[ui32Offset], pui32CodeMemoryPointer[ui32Offset]);
+					ui32Count--;
+				}
+			}
+		}
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+	else
+#endif
+	{
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			/* starting checking after BOOT LOADER config */
+			sFWAddr.ui32Addr = RGXFW_BOOTLDR_META_ADDR;
+
+			ui32StartOffset = RGXFW_MAX_BOOTLDR_OFFSET;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			/* Use bootloader code remap which is always configured before the FW is started */
+			if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+			{
+				sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE;
+			}
+			else
+#endif
+			{
+				sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+			}
+		}
+
+		eError = _ValidateWithFWModule(pfnDumpDebugPrintf, pvDumpDebugFile,
+						psDevInfo, &sFWAddr,
+						pui32HostFWCode, psDevInfo->ui32FWCodeSizeInBytes,
+						"FW code", ui32StartOffset);
+		if (eError != PVRSRV_OK)
+		{
+			goto cleanup_initfw;
+		}
+
+		/* Coremem is not present on all GPU cores, so may not be alloc'd */
+		if (pui32HostFWCoremem != NULL) // && psDevInfo->ui32FWCorememCodeSizeInBytes
+		{
+			if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+			{
+				sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, META_COREMEM_CODE);
+			}
+			else
+			{
+				sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, RISCV_COREMEM_CODE);
+
+				/* Core must be halted while issuing abstract commands */
+				eError = RGXRiscvHalt(psDevInfo);
+				PVR_GOTO_IF_ERROR(eError, cleanup_initfw);
+			}
+
+			eError = _ValidateWithFWModule(pfnDumpDebugPrintf, pvDumpDebugFile,
+							psDevInfo, &sFWAddr,
+							pui32HostFWCoremem, psDevInfo->ui32FWCorememCodeSizeInBytes,
+							"FW coremem code", 0);
+
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+			{
+				eError = RGXRiscvResume(psDevInfo);
+				PVR_GOTO_IF_ERROR(eError, cleanup_initfw);
+			}
+		}
+	}
+
+cleanup_initfw:
+	if (psRGXFW)
+	{
+		OSUnloadFirmware(psRGXFW);
+	}
+
+	if (pui32HostFWCoremem)
+	{
+		OSFreeMem(pui32HostFWCoremem);
+	}
+freeHostFWCode:
+	if (pui32HostFWCode)
+	{
+		OSFreeMem(pui32HostFWCode);
+	}
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+PVRSRV_ERROR ValidateFWOnLoad(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_TRUSTED_DEVICE)
+	IMG_PBYTE pbCodeMemoryPointer;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR sFWAddr;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, (void **)&pbCodeMemoryPointer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	RGXFwSharedMemCacheOpExec(pbCodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, PVRSRV_CACHE_OP_INVALIDATE);
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		sFWAddr.ui32Addr = RGXFW_BOOTLDR_META_ADDR;
+	}
+	else
+	{
+		PVR_ASSERT(RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR));
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+		{
+			sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE;
+		}
+		else
+#endif
+		{
+			sFWAddr.ui32Addr = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+		}
+	};
+
+	eError = _ValidateWithFWModule(NULL, NULL, psDevInfo, &sFWAddr, pbCodeMemoryPointer, psDevInfo->ui32FWCodeSizeInBytes, "FW code", 0);
+	if (eError != PVRSRV_OK)
+	{
+		goto releaseFWCodeMapping;
+	}
+
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, (void **)&pbCodeMemoryPointer);
+		if (eError != PVRSRV_OK)
+		{
+			goto releaseFWCoreCodeMapping;
+		}
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, META_COREMEM_CODE);
+		}
+		else
+		{
+			PVR_ASSERT(RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR));
+			sFWAddr.ui32Addr = RGXGetFWImageSectionAddress(NULL, RISCV_COREMEM_CODE);
+		}
+
+		eError = _ValidateWithFWModule(NULL, NULL, psDevInfo, &sFWAddr, pbCodeMemoryPointer,
+						psDevInfo->ui32FWCorememCodeSizeInBytes, "FW coremem code", 0);
+	}
+
+releaseFWCoreCodeMapping:
+	if (psDevInfo->ui32FWCorememCodeSizeInBytes)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+releaseFWCodeMapping:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	return PVRSRV_OK;
+#endif
+}
+#endif
+
+static const IMG_CHAR* _RGXGetDebugDevPowerStateString(PVRSRV_DEV_POWER_STATE ePowerState)
+{
+	switch (ePowerState)
+	{
+		case PVRSRV_DEV_POWER_STATE_DEFAULT: return "DEFAULT";
+		case PVRSRV_DEV_POWER_STATE_OFF: return "OFF";
+		case PVRSRV_DEV_POWER_STATE_ON: return "ON";
+		default: return "UNKNOWN";
+	}
+}
+
+/*
+	Writes flags strings to an uninitialised buffer.
+*/
+static void _GetFwSysFlagsDescription(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, IMG_UINT32 ui32RawFlags)
+{
+	const IMG_CHAR szCswLabel[] = "Ctx switch options:";
+	size_t uLabelLen = sizeof(szCswLabel) - 1;
+	const size_t uiBytesPerDesc = (ui32DescSize - uLabelLen) / 2U - 1U;
+
+	OSStringSafeCopy(psDesc, szCswLabel, ui32DescSize);
+
+	DebugCommonFlagStrings(psDesc, uiBytesPerDesc + uLabelLen, asCswOpts2Description, ARRAY_SIZE(asCswOpts2Description), ui32RawFlags);
+	DebugCommonFlagStrings(psDesc, ui32DescSize, asMisc2Description, ARRAY_SIZE(asMisc2Description), ui32RawFlags);
+}
+
+static void _GetFwOsFlagsDescription(IMG_CHAR *psDesc, IMG_UINT32 ui32DescSize, IMG_UINT32 ui32RawFlags)
+{
+	const IMG_CHAR szCswLabel[] = "Ctx switch:";
+	size_t uLabelLen = sizeof(szCswLabel) - 1;
+	const size_t uiBytesPerDesc = (ui32DescSize - uLabelLen) / 2U - 1U;
+
+	OSStringSafeCopy(psDesc, szCswLabel, ui32DescSize);
+
+	DebugCommonFlagStrings(psDesc, uiBytesPerDesc + uLabelLen, asFwOsCfg2Description, ARRAY_SIZE(asFwOsCfg2Description), ui32RawFlags);
+}
+
+
+typedef enum _DEVICEMEM_HISTORY_QUERY_INDEX_
+{
+	DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING,
+	DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED,
+	DEVICEMEM_HISTORY_QUERY_INDEX_NEXT,
+	DEVICEMEM_HISTORY_QUERY_INDEX_COUNT,
+} DEVICEMEM_HISTORY_QUERY_INDEX;
+
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintDevicememHistoryQueryResult
+
+ @Description
+
+ Print details of a single result from a DevicememHistory query
+
+ @Input pfnDumpDebugPrintf       - Debug printf function
+ @Input pvDumpDebugFile          - Optional file identifier to be passed to the
+                                   'printf' function if required
+ @Input psFaultProcessInfo       - The process info derived from the page fault
+ @Input psResult                 - The DevicememHistory result to be printed
+ @Input ui32Index                - The index of the result
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintDevicememHistoryQueryResult(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						RGXMEM_PROCESS_INFO *psFaultProcessInfo,
+						DEVICEMEM_HISTORY_QUERY_OUT_RESULT *psResult,
+						IMG_UINT32 ui32Index,
+						const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+	RGXConvertOSTimestampToSAndNS(psResult->ui64When,
+							&ui64Seconds,
+							&ui64Nanoseconds);
+
+	if (psFaultProcessInfo->uiPID != RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		PVR_DUMPDEBUG_LOG("%s    [%u] Name: %s Base address: " IMG_DEV_VIRTADDR_FMTSPEC
+					" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+					" Operation: %s Modified: %" IMG_UINT64_FMTSPEC
+					" us ago (OS time %" IMG_UINT64_FMTSPEC
+					".%09" IMG_UINT64_FMTSPEC " s)",
+						pszIndent,
+						ui32Index,
+						psResult->szString,
+						psResult->sBaseDevVAddr.uiAddr,
+						psResult->uiSize,
+						psResult->bMap ? "Map": "Unmap",
+						OSDivide64r64(psResult->ui64Age, 1000, &ui32Remainder),
+						ui64Seconds,
+						ui64Nanoseconds);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s    [%u] Name: %s Base address: " IMG_DEV_VIRTADDR_FMTSPEC
+					" Size: " IMG_DEVMEM_SIZE_FMTSPEC
+					" Operation: %s Modified: %" IMG_UINT64_FMTSPEC
+					" us ago (OS time %" IMG_UINT64_FMTSPEC
+					".%09" IMG_UINT64_FMTSPEC
+					") PID: %u (%s)",
+						pszIndent,
+						ui32Index,
+						psResult->szString,
+						psResult->sBaseDevVAddr.uiAddr,
+						psResult->uiSize,
+						psResult->bMap ? "Map": "Unmap",
+						OSDivide64r64(psResult->ui64Age, 1000, &ui32Remainder),
+						ui64Seconds,
+						ui64Nanoseconds,
+						psResult->sProcessInfo.uiPID,
+						psResult->sProcessInfo.szProcessName);
+	}
+
+	if (!psResult->bRange)
+	{
+		PVR_DUMPDEBUG_LOG("%s        Whole allocation was %s", pszIndent, psResult->bMap ? "mapped": "unmapped");
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s        Pages %u to %u (" IMG_DEV_VIRTADDR_FMTSPEC "-" IMG_DEV_VIRTADDR_FMTSPEC ") %s%s",
+										pszIndent,
+										psResult->ui32StartPage,
+										psResult->ui32StartPage + psResult->ui32PageCount - 1,
+										psResult->sMapStartAddr.uiAddr,
+										psResult->sMapEndAddr.uiAddr,
+										psResult->bAll ? "(whole allocation) " : "",
+										psResult->bMap ? "mapped": "unmapped");
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintDevicememHistoryQueryOut
+
+ @Description
+
+ Print details of all the results from a DevicememHistory query
+
+ @Input pfnDumpDebugPrintf       - Debug printf function
+ @Input pvDumpDebugFile          - Optional file identifier to be passed to the
+                                   'printf' function if required
+ @Input psFaultProcessInfo       - The process info derived from the page fault
+ @Input psQueryOut               - Storage for the query results
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintDevicememHistoryQueryOut(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						RGXMEM_PROCESS_INFO *psFaultProcessInfo,
+						DEVICEMEM_HISTORY_QUERY_OUT *psQueryOut,
+						const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 i;
+
+	if (psQueryOut->ui32NumResults == 0)
+	{
+		PVR_DUMPDEBUG_LOG("%s    No results", pszIndent);
+	}
+	else
+	{
+		for (i = 0; i < psQueryOut->ui32NumResults; i++)
+		{
+			_PrintDevicememHistoryQueryResult(pfnDumpDebugPrintf, pvDumpDebugFile,
+									psFaultProcessInfo,
+									&psQueryOut->sResults[i],
+									i,
+									pszIndent);
+		}
+	}
+}
+
+/* table of HW page size values and the equivalent */
+static const unsigned int aui32HWPageSizeTable[][2] =
+{
+	{ 0, PVRSRV_4K_PAGE_SIZE },
+	{ 1, PVRSRV_16K_PAGE_SIZE },
+	{ 2, PVRSRV_64K_PAGE_SIZE },
+	{ 3, PVRSRV_256K_PAGE_SIZE },
+	{ 4, PVRSRV_1M_PAGE_SIZE },
+	{ 5, PVRSRV_2M_PAGE_SIZE }
+};
+
+/*!
+*******************************************************************************
+
+ @Function	_PageSizeHWToBytes
+
+ @Description
+
+ Convert a HW page size value to its size in bytes
+
+ @Input ui32PageSizeHW     - The HW page size value
+
+ @Return   IMG_UINT32      The page size in bytes
+
+******************************************************************************/
+static IMG_UINT32 _PageSizeHWToBytes(IMG_UINT32 ui32PageSizeHW)
+{
+	if (ui32PageSizeHW > 5)
+	{
+		/* This is invalid, so return a default value as we cannot ASSERT in this code! */
+		return PVRSRV_4K_PAGE_SIZE;
+	}
+
+	return aui32HWPageSizeTable[ui32PageSizeHW][1];
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_GetDevicememHistoryData
+
+ @Description
+
+ Get the DevicememHistory results for the given PID and faulting device virtual address.
+ The function will query DevicememHistory for information about the faulting page, as well
+ as the page before and after.
+
+ @Input psDeviceNode       - The device which this allocation search should be made on
+ @Input uiPID              - The process ID to search for allocations belonging to
+ @Input sFaultDevVAddr     - The device address to search for allocations at/before/after
+ @Input asQueryOut         - Storage for the query results
+ @Input ui32PageSizeBytes  - Faulted page size in bytes
+
+ @Return IMG_BOOL          - IMG_TRUE if any results were found for this page fault
+
+******************************************************************************/
+static IMG_BOOL _GetDevicememHistoryData(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PID uiPID,
+							IMG_DEV_VIRTADDR sFaultDevVAddr,
+							DEVICEMEM_HISTORY_QUERY_OUT asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_COUNT],
+							IMG_UINT32 ui32PageSizeBytes)
+{
+	DEVICEMEM_HISTORY_QUERY_IN sQueryIn;
+	IMG_BOOL bAnyHits = IMG_FALSE;
+
+	/* if the page fault originated in the firmware then the allocation may
+	 * appear to belong to any PID, because FW allocations are attributed
+	 * to the client process creating the allocation, so instruct the
+	 * devicemem_history query to search all available PIDs
+	 */
+	if (uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		sQueryIn.uiPID = DEVICEMEM_HISTORY_PID_ANY;
+	}
+	else
+	{
+		sQueryIn.uiPID = uiPID;
+	}
+
+	sQueryIn.psDevNode = psDeviceNode;
+	/* Query the DevicememHistory for all allocations in the previous page... */
+	sQueryIn.sDevVAddr.uiAddr = (sFaultDevVAddr.uiAddr & ~(IMG_UINT64)(ui32PageSizeBytes - 1)) - ui32PageSizeBytes;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING],
+	                          ui32PageSizeBytes, IMG_TRUE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+
+	/* Query the DevicememHistory for any record at the exact address... */
+	sQueryIn.sDevVAddr = sFaultDevVAddr;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED],
+	                          ui32PageSizeBytes, IMG_FALSE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+	else
+	{
+		/* If not matched then try matching any record in the faulting page... */
+		if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED],
+		                          ui32PageSizeBytes, IMG_TRUE))
+		{
+			bAnyHits = IMG_TRUE;
+		}
+	}
+
+	/* Query the DevicememHistory for all allocations in the next page... */
+	sQueryIn.sDevVAddr.uiAddr = (sFaultDevVAddr.uiAddr & ~(IMG_UINT64)(ui32PageSizeBytes - 1)) + ui32PageSizeBytes;
+	if (DevicememHistoryQuery(&sQueryIn, &asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_NEXT],
+	                          ui32PageSizeBytes, IMG_TRUE))
+	{
+		bAnyHits = IMG_TRUE;
+	}
+
+	return bAnyHits;
+}
+
+/* stored data about one page fault */
+typedef struct _FAULT_INFO_
+{
+	/* the process info of the memory context that page faulted */
+	RGXMEM_PROCESS_INFO sProcessInfo;
+	IMG_DEV_VIRTADDR sFaultDevVAddr;
+	MMU_FAULT_DATA   sMMUFaultData;
+	DEVICEMEM_HISTORY_QUERY_OUT asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_COUNT];
+	/* the CR timer value at the time of the fault, recorded by the FW.
+	 * used to differentiate different page faults
+	 */
+	IMG_UINT64 ui64CRTimer;
+	/* time when this FAULT_INFO entry was added. used for timing
+	 * reference against the map/unmap information
+	 */
+	IMG_UINT64 ui64When;
+	IMG_UINT32 ui32FaultInfoFlags;
+} FAULT_INFO;
+
+/* history list of page faults.
+ * Keeps the first `n` page faults and the last `n` page faults, like the FW
+ * HWR log
+ */
+typedef struct _FAULT_INFO_LOG_
+{
+	IMG_UINT32 ui32Head;
+	/* the number of faults in this log need not correspond exactly to
+	 * the HWINFO number of the FW, as the FW HWINFO log may contain
+	 * non-page fault HWRs
+	 */
+	FAULT_INFO asFaults[RGXFWIF_HWINFO_MAX];
+} FAULT_INFO_LOG;
+
+#define FAULT_INFO_PROC_INFO   (0x1U)
+#define FAULT_INFO_DEVMEM_HIST (0x2U)
+
+static FAULT_INFO_LOG gsFaultInfoLog = { 0 };
+
+static void _FillAppForFWFaults(PVRSRV_RGXDEV_INFO *psDevInfo,
+							FAULT_INFO *psInfo)
+{
+	IMG_UINT32 i, j;
+
+	for (i = 0; i < DEVICEMEM_HISTORY_QUERY_INDEX_COUNT; i++)
+	{
+		for (j = 0; j < DEVICEMEM_HISTORY_QUERY_OUT_MAX_RESULTS; j++)
+		{
+			IMG_BOOL bFound;
+
+			RGXMEM_PROCESS_INFO *psProcInfo = &psInfo->asQueryOut[i].sResults[j].sProcessInfo;
+			bFound = RGXPCPIDToProcessInfo(psDevInfo,
+								psProcInfo->uiPID,
+								psProcInfo);
+			if (!bFound) {
+				OSStringSafeCopy(psProcInfo->szProcessName,
+						 "(unknown)",
+						 sizeof(psProcInfo->szProcessName));
+			}
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_PrintFaultInfo
+
+ @Description
+
+ Print all the details of a page fault from a FAULT_INFO structure
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psInfo               - The page fault occurrence to print
+
+ @Return   void
+
+******************************************************************************/
+static void _PrintFaultInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					PVRSRV_DEVICE_NODE *psDevNode,
+					void *pvDumpDebugFile,
+					FAULT_INFO *psInfo,
+					const IMG_CHAR* pszIndent)
+{
+	IMG_UINT32 i;
+	IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+	RGXConvertOSTimestampToSAndNS(psInfo->ui64When, &ui64Seconds, &ui64Nanoseconds);
+
+	if (BITMASK_HAS(psInfo->ui32FaultInfoFlags, FAULT_INFO_PROC_INFO))
+	{
+		IMG_PID uiPID = (psInfo->sProcessInfo.uiPID == RGXMEM_SERVER_PID_FIRMWARE || psInfo->sProcessInfo.uiPID == RGXMEM_SERVER_PID_PM) ?
+							0 : psInfo->sProcessInfo.uiPID;
+
+		PVR_DUMPDEBUG_LOG("%sDevice memory history for page fault address " IMG_DEV_VIRTADDR_FMTSPEC
+							", CRTimer: 0x%016" IMG_UINT64_FMTSPECX
+							", PID: %u (%s, unregistered: %u) OS time: "
+							"%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC,
+					pszIndent,
+					psInfo->sFaultDevVAddr.uiAddr,
+					psInfo->ui64CRTimer,
+					uiPID,
+					psInfo->sProcessInfo.szProcessName,
+					psInfo->sProcessInfo.bUnregistered,
+					ui64Seconds,
+					ui64Nanoseconds);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%sCould not find PID for device memory history on PC of the fault", pszIndent);
+	}
+
+	if (BITMASK_HAS(psInfo->ui32FaultInfoFlags, FAULT_INFO_DEVMEM_HIST))
+	{
+		for (i = DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING; i < DEVICEMEM_HISTORY_QUERY_INDEX_COUNT; i++)
+		{
+			const IMG_CHAR *pszWhich = NULL;
+
+			switch (i)
+			{
+				case DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING:
+					pszWhich = "Preceding page";
+					break;
+				case DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED:
+					pszWhich = "Faulted page";
+					break;
+				case DEVICEMEM_HISTORY_QUERY_INDEX_NEXT:
+					pszWhich = "Next page";
+					break;
+			}
+
+			PVR_DUMPDEBUG_LOG("%s  %s:", pszIndent, pszWhich);
+			_PrintDevicememHistoryQueryOut(pfnDumpDebugPrintf, pvDumpDebugFile,
+								&psInfo->sProcessInfo,
+								&psInfo->asQueryOut[i],
+								pszIndent);
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("%s  No matching Devmem History for fault address", pszIndent);
+		DevicememHistoryDumpRecordStats(psDevNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+		PVR_DUMPDEBUG_LOG("%s  Records Searched -"
+		                  " PP:%"IMG_UINT64_FMTSPEC
+		                  " FP:%"IMG_UINT64_FMTSPEC
+		                  " NP:%"IMG_UINT64_FMTSPEC,
+		                  pszIndent,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_PRECEDING].ui64SearchCount,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_FAULTED].ui64SearchCount,
+		                  psInfo->asQueryOut[DEVICEMEM_HISTORY_QUERY_INDEX_NEXT].ui64SearchCount);
+	}
+}
+
+static void _RecordFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					FAULT_INFO *psInfo,
+					IMG_DEV_VIRTADDR sFaultDevVAddr,
+					IMG_DEV_PHYADDR sPCDevPAddr,
+					IMG_UINT64 ui64CRTimer,
+					IMG_UINT32 ui32PageSizeBytes)
+{
+	IMG_BOOL bFound = IMG_FALSE, bIsPMFault = IMG_FALSE;
+	RGXMEM_PROCESS_INFO sProcessInfo;
+
+	psInfo->ui32FaultInfoFlags = 0;
+	psInfo->sFaultDevVAddr = sFaultDevVAddr;
+	psInfo->ui64CRTimer = ui64CRTimer;
+	psInfo->ui64When = OSClockns64();
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		/* Check if this is PM fault */
+		if (psInfo->sMMUFaultData.eType == MMU_FAULT_TYPE_PM)
+		{
+			bIsPMFault = IMG_TRUE;
+			bFound = IMG_TRUE;
+			sProcessInfo.uiPID = RGXMEM_SERVER_PID_PM;
+			OSStringSafeCopy(sProcessInfo.szProcessName, "PM", sizeof(sProcessInfo.szProcessName));
+			sProcessInfo.szProcessName[sizeof(sProcessInfo.szProcessName) - 1] = '\0';
+			sProcessInfo.bUnregistered = IMG_FALSE;
+		}
+		else
+		{
+			/* look up the process details for the faulting page catalogue */
+			bFound = RGXPCAddrToProcessInfo(psDevInfo, sPCDevPAddr, &sProcessInfo);
+		}
+
+		if (bFound)
+		{
+			IMG_BOOL bHits;
+
+			psInfo->ui32FaultInfoFlags = FAULT_INFO_PROC_INFO;
+			psInfo->sProcessInfo = sProcessInfo;
+
+			if (bIsPMFault)
+			{
+				bHits = IMG_TRUE;
+			}
+			else
+			{
+				/* get any DevicememHistory data for the faulting address */
+				bHits = _GetDevicememHistoryData(psDevInfo->psDeviceNode,
+								 sProcessInfo.uiPID,
+								 sFaultDevVAddr,
+								 psInfo->asQueryOut,
+								 ui32PageSizeBytes);
+
+				if (bHits)
+				{
+					psInfo->ui32FaultInfoFlags |= FAULT_INFO_DEVMEM_HIST;
+
+					/* if the page fault was caused by the firmware then get information about
+					 * which client application created the related allocations.
+					 *
+					 * Fill in the process info data for each query result.
+					 */
+
+					if (sProcessInfo.uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+					{
+						_FillAppForFWFaults(psDevInfo, psInfo);
+					}
+				}
+			}
+		}
+	}
+}
+
+void RGXDumpFaultAddressHostView(MMU_FAULT_DATA *psFaultData,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const IMG_CHAR* pszIndent)
+{
+	MMU_LEVEL eTopLevel;
+	const IMG_CHAR szPageLevel[][4] = {"", "PTE", "PDE", "PCE" };
+	const IMG_CHAR szPageError[][3] = {"", "PT",  "PD",  "PC"  };
+
+	eTopLevel = psFaultData->eTopLevel;
+
+	if (psFaultData->eType == MMU_FAULT_TYPE_UNKNOWN)
+	{
+		PVR_DUMPDEBUG_LOG("%sNo live host MMU data available", pszIndent);
+		return;
+	}
+	else if (psFaultData->eType == MMU_FAULT_TYPE_PM)
+	{
+		PVR_DUMPDEBUG_LOG("%sPM faulted at PC address = 0x%016" IMG_UINT64_FMTSPECx, pszIndent, psFaultData->sLevelData[MMU_LEVEL_0].ui64Address);
+	}
+	else
+	{
+		MMU_LEVEL eCurrLevel;
+		PVR_ASSERT(eTopLevel < MMU_LEVEL_LAST);
+
+		for (eCurrLevel = eTopLevel; eCurrLevel > MMU_LEVEL_0; eCurrLevel--)
+		{
+			MMU_LEVEL_DATA *psMMULevelData = &psFaultData->sLevelData[eCurrLevel];
+			if (psMMULevelData->ui64Address)
+			{
+				if (psMMULevelData->uiBytesPerEntry == 4)
+				{
+					PVR_DUMPDEBUG_LOG("%s%s for index %d = 0x%08x and is %s",
+								pszIndent,
+								szPageLevel[eCurrLevel],
+								psMMULevelData->ui32Index,
+								(IMG_UINT) psMMULevelData->ui64Address,
+								psMMULevelData->psDebugStr);
+				}
+				else
+				{
+					PVR_DUMPDEBUG_LOG("%s%s for index %d = 0x%016" IMG_UINT64_FMTSPECx " and is %s",
+								pszIndent,
+								szPageLevel[eCurrLevel],
+								psMMULevelData->ui32Index,
+								psMMULevelData->ui64Address,
+								psMMULevelData->psDebugStr);
+				}
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("%s%s index (%d) out of bounds (%d)",
+							pszIndent,
+							szPageError[eCurrLevel],
+							psMMULevelData->ui32Index,
+							psMMULevelData->ui32NumOfEntries);
+				break;
+			}
+		}
+	}
+
+}
+
+void RGXDumpFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile,
+					  const RGX_HWRINFO *psHWRInfo,
+                      IMG_UINT32 ui32ReadIndex,
+                      IMG_DEV_VIRTADDR *psFaultDevVAddr,
+                      IMG_DEV_PHYADDR *psPCDevPAddr,
+                      bool bPMFault,
+                      IMG_UINT32 ui32PageSize)
+{
+	FAULT_INFO *psInfo;
+
+	OSLockAcquire(psDevInfo->hDebugFaultInfoLock);
+
+	/* Find the matching Fault Info for this HWRInfo */
+	psInfo = &gsFaultInfoLog.asFaults[ui32ReadIndex];
+
+	/* if they do not match, we need to update the psInfo */
+	if ((psInfo->ui64CRTimer != psHWRInfo->ui64CRTimer) ||
+		(psInfo->sFaultDevVAddr.uiAddr != psFaultDevVAddr->uiAddr))
+	{
+		MMU_FAULT_DATA *psFaultData = &psInfo->sMMUFaultData;
+
+		psFaultData->eType = MMU_FAULT_TYPE_UNKNOWN;
+
+		if (bPMFault)
+		{
+			/* PM fault and we dump PC details only */
+			psFaultData->eTopLevel = MMU_LEVEL_0;
+			psFaultData->eType     = MMU_FAULT_TYPE_PM;
+			psFaultData->sLevelData[MMU_LEVEL_0].ui64Address = psPCDevPAddr->uiAddr;
+		}
+		else
+		{
+			RGXCheckFaultAddress(psDevInfo, psFaultDevVAddr, psPCDevPAddr, psFaultData);
+		}
+
+		_RecordFaultInfo(psDevInfo, psInfo,
+					*psFaultDevVAddr, *psPCDevPAddr, psHWRInfo->ui64CRTimer,
+					_PageSizeHWToBytes(ui32PageSize));
+
+	}
+
+	RGXDumpFaultAddressHostView(&psInfo->sMMUFaultData, pfnDumpDebugPrintf, pvDumpDebugFile, DD_NORMAL_INDENT);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		_PrintFaultInfo(pfnDumpDebugPrintf, psDevInfo->psDeviceNode, pvDumpDebugFile, psInfo, DD_NORMAL_INDENT);
+	}
+
+	OSLockRelease(psDevInfo->hDebugFaultInfoLock);
+}
+
+void RGXConvertOSTimestampToSAndNS(IMG_UINT64 ui64OSTimer,
+							IMG_UINT64 *pui64Seconds,
+							IMG_UINT64 *pui64Nanoseconds)
+{
+	IMG_UINT32 ui32Remainder;
+
+	*pui64Seconds = OSDivide64r64(ui64OSTimer, 1000000000, &ui32Remainder);
+	*pui64Nanoseconds = ui64OSTimer - (*pui64Seconds * 1000000000ULL);
+}
+
+#if !defined(NO_HARDWARE)
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static PVRSRV_ERROR _RGXMipsExtraDebug(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_MIPS_STATE *psMIPSState)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	IMG_UINT32 ui32RegRead;
+	IMG_UINT32 eError = PVRSRV_OK;
+	IMG_UINT32 volatile *pui32SyncFlag;
+
+	/* Acquire the NMI operations lock */
+	OSLockAcquire(psDevInfo->hNMILock);
+
+	/* Make sure the synchronisation flag is set to 0 */
+	pui32SyncFlag = &psDevInfo->psRGXFWIfSysInit->sMIPSState.ui32Sync;
+	*pui32SyncFlag = 0;
+
+	/* Readback performed as a part of memory barrier */
+	OSWriteMemoryBarrier(pui32SyncFlag);
+	RGXFwSharedMemCacheOpPtr(pui32SyncFlag,
+	                         FLUSH);
+
+
+	/* Enable NMI issuing in the MIPS wrapper */
+	OSWriteHWReg64(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE_EVENT_EN);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_ENABLE);
+
+	/* Check the MIPS is not in error state already (e.g. it is booting or an NMI has already been requested) */
+	ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_EXCEPTION_STATUS);
+	if ((ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN) || (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN))
+	{
+
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Issue NMI */
+	OSWriteHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_EVENT,
+				   RGX_CR_MIPS_WRAPPER_NMI_EVENT_TRIGGER_EN);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_EVENT);
+
+
+	/* Wait for NMI Taken to be asserted */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+									RGX_CR_MIPS_EXCEPTION_STATUS);
+		if (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if ((ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_NMI_TAKEN_EN) == 0)
+	{
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Allow the firmware to proceed */
+	*pui32SyncFlag = 1;
+
+	/* Readback performed as a part of memory barrier */
+	OSWriteMemoryBarrier(pui32SyncFlag);
+	RGXFwSharedMemCacheOpPtr(pui32SyncFlag,
+	                         FLUSH);
+
+
+	/* Wait for the FW to have finished the NMI routine */
+	ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+								RGX_CR_MIPS_EXCEPTION_STATUS);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		ui32RegRead = OSReadHWReg32(pvRegsBaseKM,
+									RGX_CR_MIPS_EXCEPTION_STATUS);
+		if (!(ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	if (ui32RegRead & RGX_CR_MIPS_EXCEPTION_STATUS_SI_ERL_EN)
+	{
+		eError = PVRSRV_ERROR_MIPS_STATUS_UNAVAILABLE;
+		goto fail;
+	}
+	ui32RegRead = 0;
+
+	/* Copy state */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->sMIPSState,
+	                           INVALIDATE);
+	OSDeviceMemCopy(psMIPSState, &psDevInfo->psRGXFWIfSysInit->sMIPSState, sizeof(*psMIPSState));
+
+	--(psMIPSState->ui32ErrorEPC);
+	--(psMIPSState->ui32EPC);
+
+	/* Disable NMI issuing in the MIPS wrapper */
+	OSWriteHWReg32(pvRegsBaseKM,
+				   RGX_CR_MIPS_WRAPPER_NMI_ENABLE,
+				   0);
+	(void) OSReadHWReg64(pvRegsBaseKM, RGX_CR_MIPS_WRAPPER_NMI_ENABLE);
+
+fail:
+	/* Release the NMI operations lock */
+	OSLockRelease(psDevInfo->hNMILock);
+	return eError;
+}
+
+/* Print decoded information from cause register */
+static void _RGXMipsDumpCauseDecode(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile,
+                                    IMG_UINT32 ui32Cause,
+                                    IMG_UINT32 ui32ErrorState)
+{
+#define INDENT "    "
+	const IMG_UINT32 ui32ExcCode = RGXMIPSFW_C0_CAUSE_EXCCODE(ui32Cause);
+	const IMG_CHAR * const pszException = _GetMIPSExcString(ui32ExcCode);
+
+	if (ui32ErrorState != 0 &&
+	    pszException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Cause exception: %s", pszException);
+	}
+
+	if (ui32Cause & RGXMIPSFW_C0_CAUSE_FDCIPENDING)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "FDC interrupt pending");
+	}
+
+	if (!(ui32Cause & RGXMIPSFW_C0_CAUSE_IV))
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Interrupt uses general interrupt vector");
+	}
+
+	if (ui32Cause & RGXMIPSFW_C0_CAUSE_PCIPENDING)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Performance Counter Interrupt pending");
+	}
+
+	/* Unusable Coproc exception */
+	if (ui32ExcCode == 11)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Unusable Coprocessor: %d", RGXMIPSFW_C0_CAUSE_UNUSABLE_UNIT(ui32Cause));
+	}
+
+#undef INDENT
+}
+
+static IMG_BOOL _IsFWCodeException(IMG_UINT32 ui32ExcCode)
+{
+	if (ui32ExcCode >= sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Only %lu exceptions available in MIPS, %u is not a valid exception code",
+		         (unsigned long)sizeof(apsMIPSExcCodes)/sizeof(MIPS_EXCEPTION_ENCODING), ui32ExcCode));
+		return IMG_FALSE;
+	}
+
+	return apsMIPSExcCodes[ui32ExcCode].bIsFatal;
+}
+
+static void _RGXMipsDumpDebugDecode(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					IMG_UINT32 ui32Debug,
+					IMG_UINT32 ui32DEPC)
+{
+	const IMG_CHAR *pszDException = NULL;
+	IMG_UINT32 i;
+#define INDENT "    "
+
+	if (!(ui32Debug & RGXMIPSFW_C0_DEBUG_DM))
+	{
+		return;
+	}
+
+	PVR_DUMPDEBUG_LOG("DEBUG                        :");
+
+	pszDException = _GetMIPSExcString(RGXMIPSFW_C0_DEBUG_EXCCODE(ui32Debug));
+
+	if (pszDException != NULL)
+	{
+		PVR_DUMPDEBUG_LOG(INDENT "Debug exception: %s", pszDException);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sMIPS_C0_DebugTable); ++i)
+	{
+		const RGXMIPSFW_C0_DEBUG_TBL_ENTRY * const psDebugEntry = &sMIPS_C0_DebugTable[i];
+
+		if (ui32Debug & psDebugEntry->ui32Mask)
+		{
+			PVR_DUMPDEBUG_LOG(INDENT "%s", psDebugEntry->pszExplanation);
+		}
+	}
+#undef INDENT
+	PVR_DUMPDEBUG_LOG("DEPC                    :0x%08X", ui32DEPC);
+}
+
+static inline void _GetMipsTLBPARanges(const RGX_MIPS_TLB_ENTRY *psTLBEntry,
+                                       const RGX_MIPS_REMAP_ENTRY *psRemapEntry0,
+                                       const RGX_MIPS_REMAP_ENTRY *psRemapEntry1,
+                                       IMG_UINT64 *pui64PA0Start,
+                                       IMG_UINT64 *pui64PA0End,
+                                       IMG_UINT64 *pui64PA1Start,
+                                       IMG_UINT64 *pui64PA1End)
+{
+	IMG_BOOL bUseRemapOutput = (psRemapEntry0 != NULL && psRemapEntry1 != NULL) ? IMG_TRUE : IMG_FALSE;
+	IMG_UINT64 ui64PageSize = RGXMIPSFW_TLB_GET_PAGE_SIZE(psTLBEntry->ui32TLBPageMask);
+
+	if ((psTLBEntry->ui32TLBLo0 & RGXMIPSFW_TLB_VALID) == 0)
+	{
+		/* Dummy values to fail the range checks later */
+		*pui64PA0Start = -1ULL;
+		*pui64PA0End   = -1ULL;
+	}
+	else if (bUseRemapOutput)
+	{
+		*pui64PA0Start = (IMG_UINT64)psRemapEntry0->ui32RemapAddrOut << 12;
+		*pui64PA0End   = *pui64PA0Start + ui64PageSize - 1;
+	}
+	else
+	{
+		*pui64PA0Start = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo0);
+		*pui64PA0End   = *pui64PA0Start + ui64PageSize - 1;
+	}
+
+	if ((psTLBEntry->ui32TLBLo1 & RGXMIPSFW_TLB_VALID) == 0)
+	{
+		/* Dummy values to fail the range checks later */
+		*pui64PA1Start = -1ULL;
+		*pui64PA1End   = -1ULL;
+	}
+	else if (bUseRemapOutput)
+	{
+		*pui64PA1Start = (IMG_UINT64)psRemapEntry1->ui32RemapAddrOut << 12;
+		*pui64PA1End   = *pui64PA1Start + ui64PageSize - 1;
+	}
+	else
+	{
+		*pui64PA1Start = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo1);
+		*pui64PA1End   = *pui64PA1Start + ui64PageSize - 1;
+	}
+}
+
+static void _CheckMipsTLBDuplicatePAs(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                      void *pvDumpDebugFile,
+                                      const RGX_MIPS_TLB_ENTRY *psTLB,
+                                      const RGX_MIPS_REMAP_ENTRY *psRemap)
+{
+	IMG_UINT64 ui64PA0StartI, ui64PA1StartI, ui64PA0StartJ, ui64PA1StartJ;
+	IMG_UINT64 ui64PA0EndI,   ui64PA1EndI,   ui64PA0EndJ,   ui64PA1EndJ;
+	IMG_UINT32 i, j;
+
+#define RANGES_OVERLAP(start0,end0,start1,end1)  ((start0) < (end1) && (start1) < (end0))
+
+	for (i = 0; i < RGXMIPSFW_NUMBER_OF_TLB_ENTRIES; i++)
+	{
+		_GetMipsTLBPARanges(&psTLB[i],
+		                    psRemap ? &psRemap[i] : NULL,
+		                    psRemap ? &psRemap[i + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+		                    &ui64PA0StartI, &ui64PA0EndI,
+		                    &ui64PA1StartI, &ui64PA1EndI);
+
+		for (j = i + 1; j < RGXMIPSFW_NUMBER_OF_TLB_ENTRIES; j++)
+		{
+			_GetMipsTLBPARanges(&psTLB[j],
+			                    psRemap ? &psRemap[j] : NULL,
+			                    psRemap ? &psRemap[j + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+			                    &ui64PA0StartJ, &ui64PA0EndJ,
+			                    &ui64PA1StartJ, &ui64PA1EndJ);
+
+			if (RANGES_OVERLAP(ui64PA0StartI, ui64PA0EndI, ui64PA0StartJ, ui64PA0EndJ) ||
+			    RANGES_OVERLAP(ui64PA0StartI, ui64PA0EndI, ui64PA1StartJ, ui64PA1EndJ) ||
+			    RANGES_OVERLAP(ui64PA1StartI, ui64PA1EndI, ui64PA0StartJ, ui64PA0EndJ) ||
+			    RANGES_OVERLAP(ui64PA1StartI, ui64PA1EndI, ui64PA1StartJ, ui64PA1EndJ)  )
+			{
+				PVR_DUMPDEBUG_LOG("Overlap between TLB entry %u and %u", i , j);
+			}
+		}
+	}
+}
+
+static inline IMG_UINT32 _GetMIPSRemapRegionSize(IMG_UINT32 ui32RegionSizeEncoding)
+{
+    return 1U << ((ui32RegionSizeEncoding + 1U) << 1U);
+}
+
+static inline void _RGXMipsDumpTLBEntry(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                        void *pvDumpDebugFile,
+                                        const RGX_MIPS_TLB_ENTRY *psTLBEntry,
+                                        const RGX_MIPS_REMAP_ENTRY *psRemapEntry0,
+                                        const RGX_MIPS_REMAP_ENTRY *psRemapEntry1,
+                                        IMG_UINT32 ui32Index)
+{
+	IMG_BOOL bDumpRemapEntries = (psRemapEntry0 != NULL && psRemapEntry1 != NULL) ? IMG_TRUE : IMG_FALSE;
+	IMG_UINT64 ui64PA0 = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo0);
+	IMG_UINT64 ui64PA1 = RGXMIPSFW_TLB_GET_PA(psTLBEntry->ui32TLBLo1);
+	IMG_UINT64 ui64Remap0AddrOut = 0, ui64Remap1AddrOut = 0;
+	IMG_UINT32 ui32Remap0AddrIn = 0, ui32Remap1AddrIn = 0;
+
+	if (bDumpRemapEntries)
+	{
+		/* RemapAddrIn is always 4k aligned and on 32 bit */
+		ui32Remap0AddrIn = psRemapEntry0->ui32RemapAddrIn << 12;
+		ui32Remap1AddrIn = psRemapEntry1->ui32RemapAddrIn << 12;
+
+		/* RemapAddrOut is always 4k aligned and on 32 or 36 bit */
+		ui64Remap0AddrOut = (IMG_UINT64)psRemapEntry0->ui32RemapAddrOut << 12;
+		ui64Remap1AddrOut = (IMG_UINT64)psRemapEntry1->ui32RemapAddrOut << 12;
+
+		/* If TLB and remap entries match, then merge them else, print them separately */
+		if ((IMG_UINT32)ui64PA0 == ui32Remap0AddrIn &&
+		    (IMG_UINT32)ui64PA1 == ui32Remap1AddrIn)
+		{
+			ui64PA0 = ui64Remap0AddrOut;
+			ui64PA1 = ui64Remap1AddrOut;
+			bDumpRemapEntries = IMG_FALSE;
+		}
+	}
+
+	PVR_DUMPDEBUG_LOG("%2u) VA 0x%08X (%3uk) -> PA0 0x%08" IMG_UINT64_FMTSPECx " %s%s%s, "
+	                                           "PA1 0x%08" IMG_UINT64_FMTSPECx " %s%s%s",
+	                  ui32Index,
+	                  psTLBEntry->ui32TLBHi,
+	                  RGXMIPSFW_TLB_GET_PAGE_SIZE(psTLBEntry->ui32TLBPageMask),
+	                  ui64PA0,
+	                  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(psTLBEntry->ui32TLBLo0)],
+	                  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(psTLBEntry->ui32TLBLo0)],
+	                  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(psTLBEntry->ui32TLBLo0)],
+	                  ui64PA1,
+	                  gapszMipsPermissionPTFlags[RGXMIPSFW_TLB_GET_INHIBIT(psTLBEntry->ui32TLBLo1)],
+	                  gapszMipsDirtyGlobalValidPTFlags[RGXMIPSFW_TLB_GET_DGV(psTLBEntry->ui32TLBLo1)],
+	                  gapszMipsCoherencyPTFlags[RGXMIPSFW_TLB_GET_COHERENCY(psTLBEntry->ui32TLBLo1)]);
+
+	if (bDumpRemapEntries)
+	{
+		PVR_DUMPDEBUG_LOG("    Remap %2u : IN 0x%08X (%3uk) => OUT 0x%08" IMG_UINT64_FMTSPECx,
+		                  ui32Index,
+		                  ui32Remap0AddrIn,
+		                  _GetMIPSRemapRegionSize(psRemapEntry0->ui32RemapRegionSize),
+		                  ui64Remap0AddrOut);
+
+		PVR_DUMPDEBUG_LOG("    Remap %2u : IN 0x%08X (%3uk) => OUT 0x%08" IMG_UINT64_FMTSPECx,
+		                  ui32Index + RGXMIPSFW_NUMBER_OF_TLB_ENTRIES,
+		                  ui32Remap1AddrIn,
+		                  _GetMIPSRemapRegionSize(psRemapEntry1->ui32RemapRegionSize),
+		                  ui64Remap1AddrOut);
+	}
+}
+
+static RGX_MIPS_REMAP_ENTRY RGXDecodeMIPSRemap(IMG_UINT64 ui64RemapReg)
+{
+	RGX_MIPS_REMAP_ENTRY sRemapInfo;
+
+	sRemapInfo.ui32RemapAddrIn =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_BASE_ADDR_IN_SHIFT;
+
+	sRemapInfo.ui32RemapAddrOut =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_ADDR_OUT_SHIFT;
+
+	sRemapInfo.ui32RemapRegionSize =
+			(ui64RemapReg & ~RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_CLRMSK)
+				>> RGX_CR_MIPS_ADDR_REMAP_RANGE_DATA_REGION_SIZE_SHIFT;
+
+	return sRemapInfo;
+}
+
+static void RGXDumpMIPSState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+							 void *pvDumpDebugFile,
+							 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	RGX_MIPS_STATE sMIPSState = {0};
+	PVRSRV_ERROR eError;
+
+	eError = _RGXMipsExtraDebug(psDevInfo, &sMIPSState);
+	PVR_DUMPDEBUG_LOG("---- [ MIPS internal state ] ----");
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DUMPDEBUG_LOG("MIPS extra debug not available");
+	}
+	else
+	{
+		DDLOGVAL32("PC", sMIPSState.ui32ErrorEPC);
+		DDLOGVAL32("STATUS_REGISTER", sMIPSState.ui32StatusRegister);
+		DDLOGVAL32("CAUSE_REGISTER", sMIPSState.ui32CauseRegister);
+		_RGXMipsDumpCauseDecode(pfnDumpDebugPrintf, pvDumpDebugFile,
+								sMIPSState.ui32CauseRegister, sMIPSState.ui32ErrorState);
+		DDLOGVAL32("BAD_REGISTER", sMIPSState.ui32BadRegister);
+		DDLOGVAL32("EPC", sMIPSState.ui32EPC);
+		DDLOGVAL32("SP", sMIPSState.ui32SP);
+		DDLOGVAL32("BAD_INSTRUCTION", sMIPSState.ui32BadInstr);
+		_RGXMipsDumpDebugDecode(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile,
+								sMIPSState.ui32Debug, sMIPSState.ui32DEPC);
+
+		{
+			IMG_UINT32 ui32Idx;
+			RGX_MIPS_REMAP_ENTRY *psMipsRemaps = NULL;
+
+			IMG_BOOL bCheckBRN63553WA =
+				RGX_IS_BRN_SUPPORTED(psDevInfo, 63553) &&
+				(OSReadHWReg32(pvRegsBaseKM, RGX_CR_MIPS_ADDR_REMAP5_CONFIG1) == (0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN));
+
+			IMG_BOOL bUseRemapRanges = RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32;
+
+			if (bUseRemapRanges)
+			{
+				psMipsRemaps = OSAllocMem(sizeof(RGX_MIPS_REMAP_ENTRY) * RGXMIPSFW_NUMBER_OF_REMAP_ENTRIES);
+				PVR_LOG_RETURN_VOID_IF_FALSE(psMipsRemaps != NULL, "psMipsRemaps alloc failed.");
+			}
+
+			PVR_DUMPDEBUG_LOG("TLB                           :");
+
+			for (ui32Idx = 0; ui32Idx < ARRAY_SIZE(sMIPSState.asTLB); ui32Idx++)
+			{
+				if (bUseRemapRanges)
+				{
+					psMipsRemaps[ui32Idx] =
+							RGXDecodeMIPSRemap(sMIPSState.aui64Remap[ui32Idx]);
+
+					psMipsRemaps[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] =
+							RGXDecodeMIPSRemap(sMIPSState.aui64Remap[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES]);
+				}
+
+				_RGXMipsDumpTLBEntry(pfnDumpDebugPrintf,
+								     pvDumpDebugFile,
+								     &sMIPSState.asTLB[ui32Idx],
+								     (bUseRemapRanges) ? &psMipsRemaps[ui32Idx] : NULL,
+								     (bUseRemapRanges) ? &psMipsRemaps[ui32Idx+RGXMIPSFW_NUMBER_OF_TLB_ENTRIES] : NULL,
+								     ui32Idx);
+
+				if (bCheckBRN63553WA)
+				{
+					const RGX_MIPS_TLB_ENTRY *psTLBEntry = &sMIPSState.asTLB[ui32Idx];
+
+					#define BRN63553_TLB_IS_NUL(X)  (((X) & RGXMIPSFW_TLB_VALID) && (RGXMIPSFW_TLB_GET_PA(X) == 0x0))
+
+					if (BRN63553_TLB_IS_NUL(psTLBEntry->ui32TLBLo0) || BRN63553_TLB_IS_NUL(psTLBEntry->ui32TLBLo1))
+					{
+						PVR_DUMPDEBUG_LOG("BRN63553 WA present with a valid TLB entry mapping address 0x0.");
+					}
+				}
+			}
+
+			/* This implicitly also checks for overlaps between memory and regbank addresses */
+			_CheckMipsTLBDuplicatePAs(pfnDumpDebugPrintf,
+									  pvDumpDebugFile,
+									  sMIPSState.asTLB,
+									  bUseRemapRanges ? psMipsRemaps : NULL);
+
+			if (bUseRemapRanges)
+			{
+				/* Dump unmapped address if it was dumped in FW, otherwise it will be 0 */
+				if (sMIPSState.ui32UnmappedAddress)
+				{
+					PVR_DUMPDEBUG_LOG("Remap unmapped address => 0x%08X",
+									  sMIPSState.ui32UnmappedAddress);
+				}
+			}
+
+			if (psMipsRemaps != NULL)
+			{
+				OSFreeMem(psMipsRemaps);
+			}
+		}
+
+		/* Check FW code corruption in case of known errors */
+		if (_IsFWCodeException(RGXMIPSFW_C0_CAUSE_EXCCODE(sMIPSState.ui32CauseRegister)))
+		{
+			eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+			}
+		}
+	}
+	PVR_DUMPDEBUG_LOG("--------------------------------");
+}
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+static void _RGXDumpMetaSPExtraDebugInfo(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+/* List of extra META Slave Port debug registers */
+#define RGX_META_SP_EXTRA_DEBUG \
+			X(RGX_CR_META_SP_MSLVCTRL0) \
+			X(RGX_CR_META_SP_MSLVCTRL1) \
+			X(RGX_CR_META_SP_MSLVDATAX) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL)
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+/* Order in these two initialisers and the one above must match */
+#define RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_EQ1_AND_MRUA_ACCESSES \
+			X(RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_EQ1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_EQ1_AND_MRUA)
+
+#define RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_GT1_AND_MRUA_ACCESSES \
+			X(RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQENABLE__HOST_SECURITY_GT1_AND_MRUA) \
+			X(RGX_CR_META_SP_MSLVIRQLEVEL__HOST_SECURITY_GT1_AND_MRUA)
+#endif
+
+	IMG_UINT32 ui32Idx;
+	IMG_UINT32 ui32RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+	const IMG_UINT32* pui32DebugRegAddr;
+	const IMG_UINT32 aui32DebugRegAddr[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG
+#undef X
+		};
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	const IMG_UINT32 aui32DebugRegAddrUAHSV1[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_EQ1_AND_MRUA_ACCESSES
+#undef X
+		};
+
+	const IMG_UINT32 aui32DebugRegAddrUAHSGT1[] = {
+#define X(A) A,
+		RGX_META_SP_EXTRA_DEBUG__HOST_SECURITY_GT1_AND_MRUA_ACCESSES
+#undef X
+		};
+#endif
+
+	const IMG_CHAR* apszDebugRegName[] = {
+#define X(A) #A,
+	RGX_META_SP_EXTRA_DEBUG
+#undef X
+	};
+
+	PVR_DUMPDEBUG_LOG("META Slave Port extra debug:");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* array of register offset values depends on feature. But don't augment names in apszDebugRegName */
+	PVR_ASSERT(sizeof(aui32DebugRegAddrUAHSGT1) == sizeof(aui32DebugRegAddr));
+	PVR_ASSERT(sizeof(aui32DebugRegAddrUAHSV1) == sizeof(aui32DebugRegAddr));
+	pui32DebugRegAddr = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+						((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ? (aui32DebugRegAddrUAHSGT1) : (aui32DebugRegAddrUAHSV1)) : aui32DebugRegAddr;
+#else
+	pui32DebugRegAddr = aui32DebugRegAddr;
+#endif
+
+	/* dump set of Slave Port debug registers */
+	for (ui32Idx = 0; ui32Idx < sizeof(aui32DebugRegAddr)/sizeof(IMG_UINT32); ui32Idx++)
+	{
+		const IMG_CHAR* pszRegName = apszDebugRegName[ui32Idx];
+
+		ui32RegAddr = pui32DebugRegAddr[ui32Idx];
+		ui32RegVal = OSReadUncheckedHWReg32(RGX_GET_RISCV_REGS_BASE(psDevInfo), ui32RegAddr);
+		PVR_DUMPDEBUG_LOG("  * %s: 0x%8.8X", pszRegName, ui32RegVal);
+	}
+}
+#endif
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+PVRSRV_ERROR RGXDumpRGXRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(NO_HARDWARE)
+	PVR_DUMPDEBUG_LOG("------[ RGX registers ]------");
+	PVR_DUMPDEBUG_LOG("(Not supported for NO_HARDWARE builds)");
+
+	return PVRSRV_OK;
+#else /* !defined(NO_HARDWARE) */
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_UINT32   ui32Meta = RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ? RGX_GET_FEATURE_VALUE(psDevInfo, META) : 0;
+#endif
+	IMG_BOOL     bFirmwarePerf;
+	void __iomem *pvRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_DUMPDEBUG_LOG("------[ RGX registers ]------");
+	PVR_DUMPDEBUG_LOG("RGX Register Base Address (Linear):   0x%p", psDevInfo->pvRegsBaseKM);
+	PVR_DUMPDEBUG_LOG("RGX Register Base Address (Physical): 0x%08lX", (unsigned long)psDevInfo->sRegsPhysBase.uiAddr);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Linear): N/A in REE ");
+#else
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Linear):   0x%p",
+							psDevInfo->pvSecureRegsBaseKM);
+#endif
+		PVR_DUMPDEBUG_LOG("RGX Host Secure Register Base Address (Physical): 0x%08lX",
+							(unsigned long)psDevInfo->sRegsPhysBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+#endif
+
+	/* Check if firmware perf was set at Init time */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->eFirmwarePerf,
+	                           INVALIDATE);
+	bFirmwarePerf = (psDevInfo->psRGXFWIfSysInit->eFirmwarePerf != FW_PERF_CONF_NONE);
+
+	RGXDumpCoreRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (ui32Meta)
+	{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		IMG_UINT32 ui32MSlvCtrl1Reg = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+				((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ?
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA :
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA) :
+				RGX_CR_META_SP_MSLVCTRL1;
+
+		/* Forcing bit 6 of MslvCtrl1 to 0 to avoid internal reg read going through the core */
+		OSWriteUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32MSlvCtrl1Reg, 0x0);
+#else
+		/* Forcing bit 6 of MslvCtrl1 to 0 to avoid internal reg read going through the core */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL1, 0x0);
+#endif
+
+		DDLOG32(META_SP_MSLVIRQSTATUS);
+	}
+#endif /* !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION) */
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		RGXDumpMulticoreRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+	}
+
+	RGXDumpClkRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	DDLOG32(EVENT_STATUS);
+	DDLOG64(TIMER);
+
+	RGXDumpMMURegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	RGXDumpDMRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	DDLOG32(MTS_INTCTX);
+	DDLOG32(MTS_BGCTX);
+	DDLOG32(MTS_BGCTX_COUNTED_SCHEDULE);
+	DDLOG32(MTS_SCHEDULE);
+	DDLOG32(MTS_GPU_INT_STATUS);
+
+	RGXDumpSLCRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	RGXDumpMiscRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	PVR_DUMPDEBUG_LOG("FW Core Registers not available to REE");
+#else
+	if (ui32Meta)
+	{
+		IMG_BOOL bIsT0Enabled = IMG_FALSE, bIsFWFaulted = IMG_FALSE;
+		IMG_UINT32 ui32RegVal;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		IMG_UINT32 ui32MSlvIrqStatusReg = RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES) ?
+				((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1) ?
+					RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA :
+					RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA) :
+				RGX_CR_META_SP_MSLVIRQSTATUS;
+
+		PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, "META_SP_MSLVIRQSTATUS", OSReadUncheckedHWReg32(psDevInfo->pvSecureRegsBaseKM, ui32MSlvIrqStatusReg));
+#endif
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0ENABLE_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXENABLE", ui32RegVal);
+		if (ui32RegVal & META_CR_TXENABLE_ENABLE_BIT)
+		{
+			bIsT0Enabled = IMG_TRUE;
+		}
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0STATUS_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXSTATUS", ui32RegVal);
+
+		/* check for FW fault */
+		if (((ui32RegVal >> 20) & 0x3) == 0x2)
+		{
+			bIsFWFaulted = IMG_TRUE;
+		}
+
+		eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T0DEFR_OFFSET, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+		DDLOGVAL32("T0 TXDEFR", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_PC, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 PC", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_PCX, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 PCX", ui32RegVal);
+
+		eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR0_SP, &ui32RegVal);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+		DDLOGVAL32("T0 SP", ui32RegVal);
+
+		if ((ui32Meta == MTP218) || (ui32Meta == MTP219))
+		{
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1ENABLE_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXENABLE", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1STATUS_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXSTATUS", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_T1DEFR_OFFSET, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("T1 TXDEFR", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_PC, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 PC", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_PCX, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 PCX", ui32RegVal);
+
+			eError = RGXReadMetaCoreReg(psDevInfo, META_CR_THR1_SP, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadMetaCoreReg", _METASPError);
+			DDLOGVAL32("T1 SP", ui32RegVal);
+		}
+
+		if (bFirmwarePerf)
+		{
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_PERF_COUNT0, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("META_CR_PERF_COUNT0", ui32RegVal);
+
+			eError = RGXReadFWModuleAddr(psDevInfo, META_CR_PERF_COUNT1, &ui32RegVal);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXReadFWModuleAddr", _METASPError);
+			DDLOGVAL32("META_CR_PERF_COUNT1", ui32RegVal);
+		}
+
+		if (bIsT0Enabled & bIsFWFaulted)
+		{
+			eError = RGXValidateFWImage(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DUMPDEBUG_LOG("Failed to validate any FW code corruption");
+			}
+		}
+		else if (bIsFWFaulted)
+		{
+			PVR_DUMPDEBUG_LOG("Skipping FW code memory corruption checking as META is disabled");
+		}
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		DDLOG32(MIPS_ADDR_REMAP1_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP1_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP2_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP2_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP3_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP3_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP4_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP4_CONFIG2);
+		DDLOG32(MIPS_ADDR_REMAP5_CONFIG1);
+		DDLOG64(MIPS_ADDR_REMAP5_CONFIG2);
+		DDLOG64(MIPS_WRAPPER_CONFIG);
+		DDLOG32(MIPS_EXCEPTION_STATUS);
+
+		RGXDumpMIPSState(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+	}
+#endif
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = RGXDumpRISCVState(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		PVR_RETURN_IF_ERROR(eError);
+	}
+#endif /* defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION) */
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		DDLOGVAL32("TFBC_VERSION", RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION));
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		DDLOGVAL32("TFBC_COMPRESSION_CONTROL", psDevInfo->psRGXFWIfSysInit->ui32TFBCCompressionControl);
+	}
+#endif
+
+	return eError;
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(SUPPORT_SECURITY_VALIDATION)
+_METASPError:
+	PVR_DUMPDEBUG_LOG("Dump Slave Port debug information");
+	_RGXDumpMetaSPExtraDebugInfo(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+
+	return eError;
+#endif
+#endif /* defined(NO_HARDWARE) */
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDebugRequestProcess
+
+ @Description
+
+ This function will print out the debug for the specified level of verbosity
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+ @Input ui32VerbLevel       - Verbosity level
+
+ @Return   void
+
+******************************************************************************/
+static
+void RGXDebugRequestProcess(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32VerbLevel)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_DEV_POWER_STATE  ePowerState;
+#if defined(SUPPORT_SOC_TIMER)
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#endif
+	IMG_BOOL                bRGXPoweredON;
+	RGXFWIF_TRACEBUF        *psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	const RGXFWIF_OSDATA    *psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+	IMG_BOOL                bPwrLockAlreadyHeld;
+	IMG_BOOL                bPwrLockAcquired = IMG_TRUE;
+	IMG_UINT8               ui8FwOsCount;
+
+	bPwrLockAlreadyHeld = PVRSRVPwrLockIsLockedByMe(psDeviceNode);
+	if (!bPwrLockAlreadyHeld)
+	{
+		/* Only acquire the power-lock if not already held by the calling context */
+		eError = PVRSRVPowerTryLockWaitForTimeout(psDeviceNode);
+		if (eError == PVRSRV_ERROR_TIMEOUT)
+		{
+			bPwrLockAcquired = IMG_FALSE;
+#if defined(DEBUG)
+			PVR_DPF((PVR_DBG_WARNING, "Power lock owner: PID = %u at timestamp %" IMG_UINT64_FMTSPEC " (%s:%u)",
+		             psDeviceNode->uiPwrLockOwnerPID, psDeviceNode->sPowerLockOwner.ui64Timestamp,
+		             psDeviceNode->sPowerLockOwner.pszFile, psDeviceNode->sPowerLockOwner.ui32LineNum));
+#endif
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire lock (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+#if !defined(DEBUG_OVERRIDE_PWRLOCK)
+			/* Bail out if another thread has the power lock and may be
+			 * powering down the device. */
+			return;
+#endif
+		}
+		else if (eError != PVRSRV_OK)
+		{
+			return;
+		}
+	}
+	/* This should satisfy all accesses below */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                           INVALIDATE);
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error retrieving RGX power state. No debug info dumped.",
+				__func__));
+		goto Exit;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	{
+		PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						   1, RGX_NUM_DRIVERS_SUPPORTED);
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Device ID:%d Start ]------", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+	bRGXPoweredON = (ePowerState == PVRSRV_DEV_POWER_STATE_ON);
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Info ]------");
+	PVR_DUMPDEBUG_LOG("Device Node (Info): %p (%p)", psDevInfo->psDeviceNode, psDevInfo);
+	DevicememHistoryDumpRecordStats(psDevInfo->psDeviceNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+	PVR_DUMPDEBUG_LOG("RGX BVNC: %d.%d.%d.%d (%s)", psDevInfo->sDevFeatureCfg.ui32B,
+											   psDevInfo->sDevFeatureCfg.ui32V,
+											   psDevInfo->sDevFeatureCfg.ui32N,
+											   psDevInfo->sDevFeatureCfg.ui32C,
+											   PVR_ARCH_NAME);
+	PVR_DUMPDEBUG_LOG("RGX Device State: %s", PVRSRVGetDebugDevStateString(psDeviceNode->eDevState));
+	PVR_DUMPDEBUG_LOG("RGX Power State: %s", _RGXGetDebugDevPowerStateString(ePowerState));
+
+	PVR_DUMPDEBUG_LOG("RGX Health Status: %s, Reason: %s",PVRSRVGetDebugHealthStatusString(OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus)),
+						PVRSRVGetDebugHealthReasonString(OSAtomicRead(&psDevInfo->psDeviceNode->eHealthReason)));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		if (psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated)
+		{
+			PVR_DUMP_FIRMWARE_INFO(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks);
+			ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount);
+			}
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("FW info: UNINITIALIZED");
+		}
+	}
+	else
+	{
+		PVR_DUMP_FIRMWARE_INFO_HDR(psDevInfo->sFWInfoHeader);
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode) && BITMASK_HAS(psDevInfo->sFWInfoHeader.ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psDevInfo->sFWInfoHeader.ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DUMPDEBUG_LOG("Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+						RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount);
+			}
+		}
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TILE_REGION_PROTECTION))
+	{
+#if defined(SUPPORT_TRP)
+		PVR_DUMPDEBUG_LOG("TRP: HW support - Yes; SW enabled");
+#else
+		PVR_DUMPDEBUG_LOG("TRP: HW support - Yes; SW disabled");
+#endif
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("TRP: HW support - No");
+	}
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, WORKGROUP_PROTECTION))
+	{
+#if defined(SUPPORT_WGP)
+		PVR_DUMPDEBUG_LOG("WGP: HW support - Yes; SW enabled");
+#else
+		PVR_DUMPDEBUG_LOG("WGP: HW support - Yes; SW disabled");
+#endif
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("WGP: HW support - No");
+	}
+
+#if defined(SUPPORT_SOC_TIMER)
+	if (psDevConfig && psDevConfig->pfnSoCTimerRead)
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*)psDevConfig->hDevData;
+		RGX_TIMING_INFORMATION *psRGXTimingInfo = psRGXData->psRGXTimingInfo;
+		IMG_UINT32 ui32Remainder;
+		IMG_UINT64 ui64CurrentSoCTime = psDevConfig->pfnSoCTimerRead(psDevConfig->hSysData);
+		IMG_UINT64 ui64CurrentSoCTimeInNS =
+			OSDivide64r64(ui64CurrentSoCTime * SECONDS_TO_MICROSECONDS, RGXFWIF_CONVERT_TO_KHZ(psRGXTimingInfo->ui32SOCClockSpeed), &ui32Remainder);
+		IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+		RGXConvertOSTimestampToSAndNS(ui64CurrentSoCTimeInNS, &ui64Seconds, &ui64Nanoseconds);
+		PVR_DUMPDEBUG_LOG("SoC timer counter: 0x%" IMG_UINT64_FMTSPECx, ui64CurrentSoCTime);
+		PVR_DUMPDEBUG_LOG("SoC timer: %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC " seconds",
+				ui64Seconds, ui64Nanoseconds);
+	}
+#endif
+
+	RGXDumpRGXDebugSummary(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, bRGXPoweredON);
+
+	/* Dump out the kernel CCB. */
+	{
+		const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		const RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+
+		if (psKCCBCtl != NULL)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Kernel CCB WO:0x%X RO:0x%X",
+							  psKCCBCtlLocal->ui32WriteOffset,
+							  psKCCBCtl->ui32ReadOffset);
+		}
+	}
+
+	/* Dump out the firmware CCB. */
+	{
+		const RGXFWIF_CCB_CTL *psFCCBCtl = psDevInfo->psFirmwareCCBCtl;
+		const RGXFWIF_CCB_CTL *psFCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psFirmwareCCBCtl, INVALIDATE);
+
+		if (psFCCBCtl != NULL)
+		{
+			PVR_DUMPDEBUG_LOG("RGX Firmware CCB WO:0x%X RO:0x%X",
+							   psFCCBCtl->ui32WriteOffset,
+							   psFCCBCtlLocal->ui32ReadOffset);
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Dump out the Workload estimation CCB. */
+		const RGXFWIF_CCB_CTL *psWorkEstCCBCtl = psDevInfo->psWorkEstFirmwareCCBCtl;
+		const RGXFWIF_CCB_CTL *psWorkEstCCBCtlLocal = psDevInfo->psWorkEstFirmwareCCBCtlLocal;
+
+		if (psWorkEstCCBCtl != NULL)
+		{
+			RGXFwSharedMemCacheOpPtr(psWorkEstCCBCtl, INVALIDATE);
+			PVR_DUMPDEBUG_LOG("RGX WorkEst CCB WO:0x%X RO:0x%X",
+							  psWorkEstCCBCtl->ui32WriteOffset,
+							  psWorkEstCCBCtlLocal->ui32ReadOffset);
+		}
+	}
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psFwOsData,
+	                         INVALIDATE);
+
+	if (psFwOsData != NULL)
+	{
+		/* Dump the KCCB commands executed */
+		PVR_DUMPDEBUG_LOG("RGX Kernel CCB commands executed = %d",
+						  psFwOsData->ui32KCCBCmdsExecuted);
+
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+		/* Dump the number of times we have performed a forced UFO update,
+		 * and (if non-zero) the timestamp of the most recent occurrence/
+		 */
+		PVR_DUMPDEBUG_LOG("RGX SLR: Forced UFO updates requested = %d",
+						  psFwOsData->ui32ForcedUpdatesRequested);
+		if (psFwOsData->ui32ForcedUpdatesRequested > 0)
+		{
+			IMG_UINT8 ui8Idx;
+			IMG_UINT64 ui64Seconds, ui64Nanoseconds;
+
+			if (psFwOsData->ui64LastForcedUpdateTime > 0ULL)
+			{
+				RGXConvertOSTimestampToSAndNS(psFwOsData->ui64LastForcedUpdateTime, &ui64Seconds, &ui64Nanoseconds);
+				PVR_DUMPDEBUG_LOG("RGX SLR: (most recent forced update was around %" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC ")",
+								  ui64Seconds, ui64Nanoseconds);
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("RGX SLR: (unable to force update as fence contained no sync checkpoints)");
+			}
+			/* Dump SLR log */
+			if (psFwOsData->sSLRLogFirst.aszCCBName[0])
+			{
+				RGXConvertOSTimestampToSAndNS(psFwOsData->sSLRLogFirst.ui64Timestamp, &ui64Seconds, &ui64Nanoseconds);
+				PVR_DUMPDEBUG_LOG("RGX SLR:{%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC
+								  "} Fence found on context 0x%x '%.*s' has %d UFOs",
+								  ui64Seconds, ui64Nanoseconds,
+								  psFwOsData->sSLRLogFirst.ui32FWCtxAddr,
+								  PVR_SLR_LOG_STRLEN, psFwOsData->sSLRLogFirst.aszCCBName,
+								  psFwOsData->sSLRLogFirst.ui32NumUFOs);
+			}
+			for (ui8Idx=0; ui8Idx<PVR_SLR_LOG_ENTRIES;ui8Idx++)
+			{
+				if (psFwOsData->sSLRLog[ui8Idx].aszCCBName[0])
+				{
+					RGXConvertOSTimestampToSAndNS(psFwOsData->sSLRLog[ui8Idx].ui64Timestamp, &ui64Seconds, &ui64Nanoseconds);
+					PVR_DUMPDEBUG_LOG("RGX SLR:[%" IMG_UINT64_FMTSPEC ".%09" IMG_UINT64_FMTSPEC
+									  "] Fence found on context 0x%x '%.*s' has %d UFOs",
+									  ui64Seconds, ui64Nanoseconds,
+									  psFwOsData->sSLRLog[ui8Idx].ui32FWCtxAddr,
+									  PVR_SLR_LOG_STRLEN, psFwOsData->sSLRLog[ui8Idx].aszCCBName,
+									  psFwOsData->sSLRLog[ui8Idx].ui32NumUFOs);
+				}
+			}
+		}
+#else
+		PVR_DUMPDEBUG_LOG("RGX SLR: Disabled");
+#endif
+
+		/* Dump the error counts */
+		PVR_DUMPDEBUG_LOG("RGX Errors: WGP:%d, TRP:%d",
+						  psDevInfo->sErrorCounts.ui32WGPErrorCount,
+						  psDevInfo->sErrorCounts.ui32TRPErrorCount);
+
+		/* Dump the IRQ info for threads or OS IDs */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* only Host has access to registers containing IRQ counters */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+#endif
+		{
+			IMG_UINT32 ui32idx;
+
+			for_each_irq_cnt(ui32idx)
+			{
+				IMG_UINT32 ui32IrqCnt;
+
+				get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+				if (ui32IrqCnt)
+				{
+					PVR_DUMPDEBUG_LOG(MSG_IRQ_CNT_TYPE "%u: FW IRQ count = %u", ui32idx, ui32IrqCnt);
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+					if (ui32idx == RGXFW_HOST_DRIVER_ID)
+#endif
+					{
+						PVR_DUMPDEBUG_LOG("Last sampled IRQ count in LISR = %u", psDevInfo->aui32SampleIRQCount[ui32idx]);
+					}
+				}
+			}
+		}
+	}
+
+	/* Dump the FW Sys config flags on the Host */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+		IMG_CHAR sFwSysFlagsDescription[MAX_FW_DESCRIPTION_LENGTH];
+
+		if (!psFwSysData)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Fw Sys Data is not mapped into CPU space", __func__));
+			goto Exit;
+		}
+
+		RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags,
+		                           INVALIDATE);
+
+		_GetFwSysFlagsDescription(sFwSysFlagsDescription, MAX_FW_DESCRIPTION_LENGTH, psFwSysData->ui32ConfigFlags);
+		PVR_DUMPDEBUG_LOG("FW System config flags = 0x%08X (%s)", psFwSysData->ui32ConfigFlags, sFwSysFlagsDescription);
+	}
+
+	/* Dump the FW OS config flags */
+	{
+		IMG_CHAR sFwOsFlagsDescription[MAX_FW_DESCRIPTION_LENGTH];
+
+		if (!psFwOsData)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Fw Os Data is not mapped into CPU space", __func__));
+			goto Exit;
+		}
+
+		_GetFwOsFlagsDescription(sFwOsFlagsDescription, MAX_FW_DESCRIPTION_LENGTH, psFwOsData->ui32FwOsConfigFlags);
+		PVR_DUMPDEBUG_LOG("FW OS config flags = 0x%08X (%s)", psFwOsData->ui32FwOsConfigFlags, sFwOsFlagsDescription);
+	}
+
+	if ((bRGXPoweredON) && !PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDumpRGXRegisters(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: RGXDumpRGXRegisters failed (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+		}
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG(" (!) %s. No registers dumped", PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? "Guest Mode of operation" : "RGX power is down");
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX FW Trace Info ]------");
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_INT tid;
+		/* Dump FW trace information */
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			RGX_FWT_LOGTYPE eFWTLogType = psDevInfo->eDebugDumpFWTLogType;
+
+			if (eFWTLogType == RGX_FWT_LOGTYPE_NONE)
+			{
+				PVR_DUMPDEBUG_LOG("Firmware trace printing disabled.");
+			}
+			else
+			{
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+
+				for (tid = 0; tid < RGXFW_THREAD_NUM; tid++)
+				{
+					IMG_UINT32 *pui32TraceBuffer;
+
+					if (psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+					{
+						PVR_DUMPDEBUG_LOG("Debug log type: %s ( " RGXFWIF_LOG_ENABLED_GROUPS_LIST_PFSPEC ")",
+										  ((psRGXFWIfTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)?("trace"):("tbi")),
+										  RGXFWIF_LOG_ENABLED_GROUPS_LIST(psRGXFWIfTraceBufCtl->ui32LogType)
+										  );
+					}
+					else
+					{
+						PVR_DUMPDEBUG_LOG("Debug log type: none");
+					}
+
+					pui32TraceBuffer = psDevInfo->apui32TraceBuffer[tid];
+
+					/* Skip if trace buffer is not allocated */
+					if (pui32TraceBuffer == NULL)
+					{
+						PVR_DUMPDEBUG_LOG("RGX FW thread %d: Trace buffer not yet allocated",tid);
+						continue;
+					}
+
+					RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[tid].ui32TracePointer, INVALIDATE);
+					PVR_DUMPDEBUG_LOG("------[ RGX FW thread %d trace START ]------", tid);
+					PVR_DUMPDEBUG_LOG("FWT[traceptr]: %X", psRGXFWIfTraceBufCtl->sTraceBuf[tid].ui32TracePointer);
+					PVR_DUMPDEBUG_LOG("FWT[tracebufsize]: %X", psDevInfo->ui32TraceBufSizeInDWords);
+
+					if (eFWTLogType == RGX_FWT_LOGTYPE_BINARY)
+					{
+						RGXDumpFirmwareTraceBinary(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+					else if (eFWTLogType == RGX_FWT_LOGTYPE_DECODED)
+					{
+						RGXDumpFirmwareTraceDecoded(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+					else if (eFWTLogType == RGX_FWT_LOGTYPE_PARTIAL)
+					{
+						RGXDumpFirmwareTracePartial(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, psRGXFWIfTraceBufCtl, tid);
+					}
+
+					PVR_DUMPDEBUG_LOG("------[ RGX FW thread %d trace END ]------", tid);
+				}
+			}
+		}
+
+		{
+			if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+			{
+				PVR_DUMPDEBUG_LOG("------[ Full CCB Status ]------");
+			}
+			else
+			{
+				PVR_DUMPDEBUG_LOG("------[ FWCtxs Next CMD ]------");
+			}
+
+			RGXDumpAllContextInfo(psDevInfo, pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+
+	PVR_DUMPDEBUG_LOG("------[ RGX Device ID:%d End ]------", psDevInfo->psDeviceNode->sDevId.ui32InternalID);
+
+Exit:
+	if (!bPwrLockAlreadyHeld && bPwrLockAcquired)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+}
+
+/*!
+ ******************************************************************************
+
+ @Function	RGXDebugRequestNotify
+
+ @Description Dump the debug data for RGX
+
+ ******************************************************************************/
+static void RGXDebugRequestNotify(PVRSRV_DBGREQ_HANDLE hDbgRequestHandle,
+		IMG_UINT32 ui32VerbLevel,
+		DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+		void *pvDumpDebugFile)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = hDbgRequestHandle;
+
+	/* Only action the request if we've fully init'ed */
+	if (psDevInfo->bDevInit2Done)
+	{
+		RGXDebugRequestProcess(pfnDumpDebugPrintf, pvDumpDebugFile, psDevInfo, ui32VerbLevel);
+	}
+}
+
+PVRSRV_ERROR RGXDebugInit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return PVRSRVRegisterDeviceDbgRequestNotify(&psDevInfo->hDbgReqNotify,
+		                                        psDevInfo->psDeviceNode,
+		                                        RGXDebugRequestNotify,
+		                                        DEBUG_REQUEST_RGX,
+		                                        psDevInfo);
+}
+
+PVRSRV_ERROR RGXDebugDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (psDevInfo->hDbgReqNotify)
+	{
+		return PVRSRVUnregisterDeviceDbgRequestNotify(psDevInfo->hDbgReqNotify);
+	}
+
+	/* No notifier registered */
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxdebug_common.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxdebug_common.h b/drivers/gpu/drm/img-rogue/rgxdebug_common.h
new file mode 100644
index 000000000000..f7707b87fa9f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxdebug_common.h
@@ -0,0 +1,433 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX debug header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX debugging functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEBUG_COMMON_H)
+#define RGXDEBUG_COMMON_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "rgxdevice.h"
+#include "rgxfwmemctx.h"
+
+#define DD_NORMAL_INDENT   "    "
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+extern const IMG_CHAR * const gapszMipsPermissionPTFlags[4];
+extern const IMG_CHAR * const gapszMipsCoherencyPTFlags[8];
+extern const IMG_CHAR * const gapszMipsDirtyGlobalValidPTFlags[8];
+#endif
+
+/**
+ * Debug utility macro for printing FW IRQ count and Last sampled IRQ count in
+ * LISR for each RGX FW thread.
+ * Macro takes pointer to PVRSRV_RGXDEV_INFO as input.
+ */
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+#define for_each_irq_cnt(ui32idx)   FOREACH_SUPPORTED_DRIVER(ui32idx)
+
+#define get_irq_cnt_val(ui32Dest, ui32idx, psRgxDevInfo) \
+	do { \
+		extern const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS]; \
+		ui32Dest = PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psRgxDevInfo) ? 0 : OSReadHWReg32((psRgxDevInfo)->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[ui32idx]); \
+	} while (false)
+
+#define MSG_IRQ_CNT_TYPE "OS"
+
+#else
+
+#define for_each_irq_cnt(ui32idx) \
+	for (ui32idx = 0; ui32idx < RGXFW_THREAD_NUM; ui32idx++)
+
+#define get_irq_cnt_val(ui32Dest, ui32idx, psRgxDevInfo) \
+	do { \
+		RGXFwSharedMemCacheOpValue(psRgxDevInfo->psRGXFWIfFwOsData->aui32InterruptCount[ui32idx], \
+		                           INVALIDATE); \
+		ui32Dest = (psRgxDevInfo)->psRGXFWIfFwOsData->aui32InterruptCount[ui32idx]; \
+	} while (false)
+#define MSG_IRQ_CNT_TYPE "Thread"
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+static inline void RGXDEBUG_PRINT_IRQ_COUNT(PVRSRV_RGXDEV_INFO* psRgxDevInfo)
+{
+#if defined(PVRSRV_NEED_PVR_DPF) && defined(DEBUG)
+	IMG_UINT32 ui32idx;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		IMG_UINT32 ui32IrqCnt;
+
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psRgxDevInfo);
+
+		PVR_DPF((DBGPRIV_VERBOSE, MSG_IRQ_CNT_TYPE
+		         " %u FW IRQ count = %u", ui32idx, ui32IrqCnt));
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		if (ui32idx == RGXFW_HOST_DRIVER_ID)
+#endif
+		{
+			PVR_DPF((DBGPRIV_VERBOSE, "Last sampled IRQ count in LISR = %u",
+			        (psRgxDevInfo)->aui32SampleIRQCount[ui32idx]));
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psRgxDevInfo);
+#endif /* PVRSRV_NEED_PVR_DPF */
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpFirmwareTrace
+
+ @Description Dumps the decoded version of the firmware trace buffer.
+
+ Dump useful debugging info
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+
+ @Return   void
+
+******************************************************************************/
+void RGXDumpFirmwareTrace(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				PVRSRV_RGXDEV_INFO  *psDevInfo);
+
+void RGXDumpFirmwareTraceBinary(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+void RGXDumpFirmwareTracePartial(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+void RGXDumpFirmwareTraceDecoded(PVRSRV_RGXDEV_INFO *psDevInfo,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile,
+				RGXFWIF_TRACEBUF *psRGXFWIfTraceBufCtl,
+				IMG_UINT32 ui32TID);
+
+
+/* Helper macros to emit data */
+#define REG32_FMTSPEC   "%-30s: 0x%08X"
+#define REG64_FMTSPEC   "%-30s: 0x%016" IMG_UINT64_FMTSPECX
+#define DDLOG32(R)      PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, #R, OSReadHWReg32(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOG64(R)      PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadHWReg64(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOGUNCHECKED64(R)      PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadUncheckedHWReg64(pvRegsBaseKM, RGX_CR_##R));
+#define DDLOG32_DPX(R)  PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, #R, OSReadHWReg32(pvRegsBaseKM, DPX_CR_##R));
+#define DDLOG64_DPX(R)  PVR_DUMPDEBUG_LOG(REG64_FMTSPEC, #R, OSReadHWReg64(pvRegsBaseKM, DPX_CR_##R));
+#define DDLOGVAL32(S,V) PVR_DUMPDEBUG_LOG(REG32_FMTSPEC, S, V);
+
+PVRSRV_ERROR RGXDumpRISCVState(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+									  void *pvDumpDebugFile,
+									  PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpCoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMulticoreRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpClkRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMMURegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpDMRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpSLCRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+void RGXDumpMiscRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpRGXRegisters
+
+ @Description
+
+ Dumps an extensive list of RGX registers required for debugging
+
+ @Input pfnDumpDebugPrintf  - Optional replacement print function
+ @Input pvDumpDebugFile     - Optional file identifier to be passed to the
+                              'printf' function if required
+ @Input psDevInfo           - RGX device info
+
+ @Return PVRSRV_ERROR         PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXDumpRGXRegisters(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+								 void *pvDumpDebugFile,
+								 PVRSRV_RGXDEV_INFO *psDevInfo);
+
+#if !defined(NO_HARDWARE)
+/*!
+*******************************************************************************
+
+ @Function	RGXReadMetaCoreReg
+
+ @Description  Read a META core register's value
+
+ @Input psDevInfo      RGX device info
+ @Input ui32RegAddr    Register address to read from
+ @Output pui32RegVal   Pointer to the resulting register value
+
+ @Return PVRSRV_ERROR  PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXReadMetaCoreReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT32 ui32RegAddr,
+                                IMG_UINT32 *pui32RegVal);
+
+/*!
+*******************************************************************************
+
+ @Function  RGXValidateFWImage
+
+ @Description  Validate the currently running firmware
+               against the firmware image
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo	         - RGX device info
+
+ @Return PVRSRV_ERROR  PVRSRV_OK on success, error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXValidateFWImage(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						void *pvDumpDebugFile,
+						PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#if defined(SUPPORT_FW_VIEW_EXTRA_DEBUG)
+/*!
+*******************************************************************************
+
+ @Function     ValidateFWOnLoad
+
+ @Description  Compare the Firmware image as seen from the CPU point of view
+               against the same memory area as seen from the firmware point
+               of view after first power up.
+
+ @Input        psDevInfo - Device Info
+
+ @Return       PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR ValidateFWOnLoad(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpRGXDebugSummary
+
+ @Description
+
+ Dump a summary in human readable form with the RGX state
+
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Input psDevInfo	         - RGX device info
+ @Input bRGXPoweredON        - IMG_TRUE if RGX device is on
+
+ @Return   void
+
+******************************************************************************/
+void RGXDumpRGXDebugSummary(DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					PVRSRV_RGXDEV_INFO *psDevInfo,
+					IMG_BOOL bRGXPoweredON);
+
+/*!
+*******************************************************************************
+
+ @Function RGXDebugInit
+
+ @Description
+
+ Setup debug requests, calls into PVRSRVRegisterDeviceDbgRequestNotify
+
+ @Input          psDevInfo            RGX device info
+ @Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+
+******************************************************************************/
+PVRSRV_ERROR RGXDebugInit(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function RGXDebugDeinit
+
+ @Description
+
+ Remove debug requests, calls into PVRSRVUnregisterDeviceDbgRequestNotify
+
+ @Output         phNotify             Points to debug notifier handle
+ @Return         PVRSRV_ERROR         PVRSRV_OK on success otherwise an error
+
+******************************************************************************/
+PVRSRV_ERROR RGXDebugDeinit(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetFwMapping
+
+ @Description    Retrieve any of the CPU Physical Address, Device Physical
+                Address or the raw value of the page table entry associated
+                with the firmware virtual address given.
+
+ @Input          psDevInfo           Pointer to device info
+ @Input          pfnDumpDebugPrintf  The debug printf function
+ @Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+ @Input          ui32FwVA            The Fw VA that needs decoding
+ @Output         psCpuPA             Pointer to the resulting CPU PA
+ @Output         psDevPA             Pointer to the resulting Dev PA
+ @Output         pui64PTE            Pointer to the raw Page Table Entry value
+
+ @Return         void
+
+******************************************************************************/
+void RGXDocumentFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                          DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                          void *pvDumpDebugFile,
+                          const IMG_UINT32 ui32FwVA,
+                          const IMG_CPU_PHYADDR sCpuPA,
+                          const IMG_DEV_PHYADDR sDevPA,
+                          const IMG_UINT64 ui64PTE);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXConvertOSTimestampToSAndNS
+
+ @Description    Convert the OS time to seconds and nanoseconds
+
+ @Input          ui64OSTimer       OS time to convert
+ @Output         pui64Seconds      Pointer to the resulting seconds
+ @Output         pui64Nanoseconds  Pointer to the resulting nanoseconds
+
+ @Return         void
+
+******************************************************************************/
+void RGXConvertOSTimestampToSAndNS(IMG_UINT64 ui64OSTimer,
+							IMG_UINT64 *pui64Seconds,
+							IMG_UINT64 *pui64Nanoseconds);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDumpAllContextInfo
+
+ @Description    Dump debug info of all contexts on a device
+
+ @Input          psDevInfo           Pointer to device info
+ @Input          pfnDumpDebugPrintf  The debug printf function
+ @Input          pvDumpDebugFile     Optional file identifier to be passed to
+                                    the 'printf' function if required
+ @Input          ui32VerbLevel       Verbosity level
+
+ @Return         void
+
+******************************************************************************/
+void RGXDumpAllContextInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDumpFaultAddressHostView
+
+ @Description
+
+ Dump FW HWR fault status in human readable form.
+
+ @Input ui32Index            - Index of global Fault info
+ @Input pfnDumpDebugPrintf   - The debug printf function
+ @Input pvDumpDebugFile      - Optional file identifier to be passed to the
+                               'printf' function if required
+ @Return   void
+
+******************************************************************************/
+void RGXDumpFaultAddressHostView(MMU_FAULT_DATA *psFaultData,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const IMG_CHAR* pszIndent);
+
+void RGXDumpFaultInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile,
+					const RGX_HWRINFO *psHWRInfo,
+					IMG_UINT32 ui32ReadIndex,
+					IMG_DEV_VIRTADDR *psFaultDevVAddr,
+					IMG_DEV_PHYADDR *psPCDevPAddr,
+					bool bPMFault,
+					IMG_UINT32 ui32PageSize);
+
+#endif /* RGXDEBUG_COMMON_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxdevice.h b/drivers/gpu/drm/img-rogue/rgxdevice.h
new file mode 100644
index 000000000000..95e0a0b53cca
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxdevice.h
@@ -0,0 +1,938 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX device node header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX device node
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXDEVICE_H)
+#define RGXDEVICE_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_device_types.h"
+#include "mmu_common.h"
+#include "rgx_fwif_km.h"
+#include "cache_ops.h"
+#include "device.h"
+#include "osfunc.h"
+#include "rgxlayer_impl.h"
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "hash.h"
+#endif
+typedef struct _RGX_SERVER_COMMON_CONTEXT_ RGX_SERVER_COMMON_CONTEXT;
+
+typedef struct {
+	DEVMEM_MEMDESC		*psFWFrameworkMemDesc;
+} RGX_COMMON_CONTEXT_INFO;
+
+
+/*!
+ ******************************************************************************
+ * Device state flags
+ *****************************************************************************/
+#define RGXKM_DEVICE_STATE_ZERO_FREELIST                          (0x1)  /*!< Zeroing the physical pages of reconstructed free lists */
+#define RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN                  (0x2)  /*!< Used to disable the Devices Watchdog logging */
+#define RGXKM_DEVICE_STATE_GPU_UNITS_POWER_CHANGE_EN              (0x4)  /*!< Used for validation to inject power units state change every DM kick */
+#define RGXKM_DEVICE_STATE_CCB_GROW_EN                            (0x8)  /*!< Used to indicate CCB grow is permitted */
+#define RGXKM_DEVICE_STATE_ENABLE_SPU_UNITS_POWER_MASK_CHANGE_EN  (0x10) /*!< Used for validation to enable SPU power state mask change */
+#define RGXKM_DEVICE_STATE_MASK                                   (0x1F)
+
+/*!
+ ******************************************************************************
+ * ECC RAM Fault Validation
+ *****************************************************************************/
+#define RGXKM_ECC_ERR_INJ_DISABLE 0
+#define RGXKM_ECC_ERR_INJ_SLC     1
+#define RGXKM_ECC_ERR_INJ_USC     2
+#define RGXKM_ECC_ERR_INJ_TPU     3
+#define RGXKM_ECC_ERR_INJ_RASCAL  4
+#define RGXKM_ECC_ERR_INJ_MARS    5
+
+#define RGXKM_ECC_ERR_INJ_INTERVAL 10U
+
+/*!
+ ******************************************************************************
+ * GPU DVFS Table
+ *****************************************************************************/
+
+#define RGX_GPU_DVFS_TABLE_SIZE                      32
+#define RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US       25000     /* Time required to calibrate a clock frequency the first time */
+#define RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US  150000    /* Time required for a recalibration after a DVFS transition */
+#define RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US    10000000  /* Time before the next periodic calibration and correlation */
+
+
+/*!
+ ******************************************************************************
+ * Global flags for driver validation
+ *****************************************************************************/
+#define RGX_VAL_FBDC_SIG_CHECK_NOERR_EN           (0x2U)  /*!< Enable FBDC signature check. Signatures must match */
+#define RGX_VAL_FBDC_SIG_CHECK_ERR_EN             (0x4U)  /*!< Enable FBDC signature check. Signatures must not match */
+#define RGX_VAL_GPUSTATEPIN_EN                    (0x8U)  /*!< Enable GPU state pin check */
+#define RGX_VAL_WGP_SIG_CHECK_NOERR_EN           (0x10U)  /*!< Enable WGP signature check. Signatures must match */
+#define RGX_VAL_WGP_SIG_CHECK_ERR_EN             (0x20U)  /*!< Enable WGP signature check. Signatures must not match */
+#define RGX_VAL_TRP_SIG_CHECK_NOERR_EN           (0x40U)  /*!< Enable TRP signature check. Signatures must match */
+#define RGX_VAL_TRP_SIG_CHECK_ERR_EN             (0x80U)  /*!< Enable TRP signature check. Signatures must not match */
+
+/*!
+ ******************************************************************************
+ * HWPerf L2 Stream ID type definition.
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_HWPERF_L2_STREAM_ID;
+/* HWPerf stream for Client HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_HWPERF 0U
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+/* HWPerf stream for FTrace HWPerf access. */
+#define RGX_HWPERF_L2_STREAM_FTRACE 1U
+#define RGX_HWPERF_L2_STREAM_LAST   2U
+#else
+#define RGX_HWPERF_L2_STREAM_LAST   1U
+#endif
+
+typedef struct _GPU_FREQ_TRACKING_DATA_
+{
+	/* Core clock speed estimated by the driver */
+	IMG_UINT32 ui32EstCoreClockSpeed;
+
+	/* Amount of successful calculations of the estimated core clock speed */
+	IMG_UINT32 ui32CalibrationCount;
+} GPU_FREQ_TRACKING_DATA;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+#define RGX_GPU_FREQ_TRACKING_SIZE 16
+
+typedef struct
+{
+	IMG_UINT64 ui64BeginCRTimestamp;
+	IMG_UINT64 ui64BeginOSTimestamp;
+
+	IMG_UINT64 ui64EndCRTimestamp;
+	IMG_UINT64 ui64EndOSTimestamp;
+
+	IMG_UINT32 ui32EstCoreClockSpeed;
+	IMG_UINT32 ui32CoreClockSpeed;
+} GPU_FREQ_TRACKING_HISTORY;
+#endif
+
+typedef struct _RGX_GPU_DVFS_TABLE_
+{
+	/* Beginning of current calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimestamp;
+	IMG_UINT64 ui64CalibrationOSTimestamp;
+
+	/* Calculated calibration period (in us) */
+	IMG_UINT64 ui64CalibrationCRTimediff;
+	IMG_UINT64 ui64CalibrationOSTimediff;
+
+	/* Current calibration period (in us) */
+	IMG_UINT32 ui32CalibrationPeriod;
+
+	/* System layer frequency table and frequency tracking data */
+	IMG_UINT32 ui32FreqIndex;
+	IMG_UINT32 aui32GPUFrequency[RGX_GPU_DVFS_TABLE_SIZE];
+	GPU_FREQ_TRACKING_DATA asTrackingData[RGX_GPU_DVFS_TABLE_SIZE];
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	IMG_UINT32 ui32HistoryIndex;
+	GPU_FREQ_TRACKING_HISTORY asTrackingHistory[RGX_GPU_FREQ_TRACKING_SIZE];
+#endif
+} RGX_GPU_DVFS_TABLE;
+
+
+/*!
+ ******************************************************************************
+ * GPU utilisation statistics
+ *****************************************************************************/
+
+typedef struct _RGXFWIF_TEMP_GPU_UTIL_STATS_
+{
+	IMG_UINT64 aaaui64DMOSTmpCounters[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 aaui64DMOSTmpLastWord[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastState[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastPeriod[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 aaui64DMOSTmpLastTime[RGXFWIF_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];
+} RGXFWIF_TEMP_GPU_UTIL_STATS;
+
+typedef struct _RGXFWIF_GPU_UTIL_STATS_
+{
+	IMG_BOOL   bValid;                /* If TRUE, statistics are valid.
+	                                     FALSE if the driver couldn't get reliable stats. */
+	IMG_UINT64 ui64GpuStatActive;     /* GPU active statistic */
+	IMG_UINT64 ui64GpuStatBlocked;    /* GPU blocked statistic */
+	IMG_UINT64 ui64GpuStatIdle;       /* GPU idle statistic */
+	IMG_UINT64 ui64GpuStatCumulative; /* Sum of active/blocked/idle stats */
+
+	IMG_UINT64 aaui64DMOSStatInactive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];   /* Per-DM per-OS sum of idle and blocked stats */
+	IMG_UINT64 aaui64DMOSStatActive[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED];     /* Per-DM per-OS active statistic */
+	IMG_UINT64 aaui64DMOSStatCumulative[RGXFWIF_GPU_UTIL_DM_MAX][RGX_NUM_DRIVERS_SUPPORTED]; /* Per-DM per-OS sum of active/blocked/idle stats */
+
+	IMG_UINT64 ui64TimeStamp;         /* Timestamp of the most recent sample of the GPU stats */
+
+	RGXFWIF_TEMP_GPU_UTIL_STATS sTempGpuStats; /* Temporary data used to calculate the per-DM per-OS statistics */
+} RGXFWIF_GPU_UTIL_STATS;
+
+
+typedef struct _RGX_REG_CONFIG_
+{
+	IMG_BOOL               bEnabled;
+	RGXFWIF_REG_CFG_TYPE   eRegCfgTypeToPush;
+	IMG_UINT32             ui32NumRegRecords;
+	POS_LOCK               hLock;
+} RGX_REG_CONFIG;
+
+typedef struct _PVRSRV_STUB_PBDESC_ PVRSRV_STUB_PBDESC;
+
+
+typedef struct _PVRSRV_DEVICE_FEATURE_CONFIG_
+{
+	IMG_UINT64 ui64ErnsBrns;
+	IMG_UINT64 ui64Features;
+	IMG_UINT32 ui32B;
+	IMG_UINT32 ui32V;
+	IMG_UINT32 ui32N;
+	IMG_UINT32 ui32C;
+	IMG_UINT32 ui32FeaturesValues[RGX_FEATURE_WITH_VALUES_MAX_IDX];
+	IMG_UINT32 ui32MAXDMCount;
+	IMG_UINT32 ui32MAXPowUnitCount;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	IMG_UINT32 ui32MAXRACCount;
+#endif
+	IMG_UINT32 ui32SLCSizeInBytes;
+	IMG_PCHAR  pszBVNCString;
+}PVRSRV_DEVICE_FEATURE_CONFIG;
+
+/* This is used to get the value of a specific feature.
+ * Note that it will assert if the feature is disabled or value is invalid. */
+#define RGX_GET_FEATURE_VALUE(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] )
+
+/* This is used to check if the feature value (e.g. with an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, Feature) \
+			( psDevInfo->sDevFeatureCfg.ui32FeaturesValues[RGX_FEATURE_##Feature##_IDX] < RGX_FEATURE_VALUE_DISABLED )
+
+/* This is used to check if the Boolean feature (e.g. WITHOUT an integer value) is available for the currently running BVNC or not */
+#define RGX_IS_FEATURE_SUPPORTED(psDevInfo, Feature) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64Features, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if the ERN is available for the currently running BVNC or not */
+#define RGX_IS_ERN_SUPPORTED(psDevInfo, ERN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if the BRN is available for the currently running BVNC or not */
+#define RGX_IS_BRN_SUPPORTED(psDevInfo, BRN) \
+			BITMASK_HAS(psDevInfo->sDevFeatureCfg.ui64ErnsBrns, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+/* there is a corresponding define in rgxapi.h */
+#define RGX_MAX_TIMER_QUERIES 16U
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+/*!
+ * The host maintains a 512-deep cache of submitted workloads per device,
+ * i.e. a global look-up table for TA, 3D and compute (depending on the RGX
+ * hardware support present)
+ */
+
+/*
+ * For the workload estimation return data array, the max amount of commands the
+ * MTS can have is 255, therefore 512 (LOG2 = 9) is large enough to account for
+ * all corner cases
+ */
+#define RETURN_DATA_ARRAY_SIZE_LOG2 (9)
+#define RETURN_DATA_ARRAY_SIZE      ((1U) << RETURN_DATA_ARRAY_SIZE_LOG2)
+#define RETURN_DATA_ARRAY_WRAP_MASK (RETURN_DATA_ARRAY_SIZE - 1)
+
+#define WORKLOAD_HASH_SIZE_LOG2		6
+#define WORKLOAD_HASH_SIZE			((1U) << WORKLOAD_HASH_SIZE_LOG2)
+#define WORKLOAD_HASH_WRAP_MASK		(WORKLOAD_HASH_SIZE - 1)
+
+/*!
+ * Workload characteristics for supported data masters.
+ * All characteristics must match for the workload estimate to be used/updated.
+ */
+typedef union _RGX_WORKLOAD_
+{
+	struct
+	{
+		IMG_UINT32				ui32RenderTargetSize;
+		IMG_UINT32				ui32NumberOfDrawCalls;
+		IMG_UINT32				ui32NumberOfIndices;
+		IMG_UINT32				ui32NumberOfMRTs;
+	} sTA3D;
+
+	struct
+	{
+		IMG_UINT32				ui32NumberOfWorkgroups;
+		IMG_UINT32				ui32NumberOfWorkitems;
+	} sCompute;
+
+	struct
+	{
+		IMG_UINT32				ui32Characteristic1;
+		IMG_UINT32				ui32Characteristic2;
+	} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	struct
+	{
+		IMG_UINT32				ui32DispatchSize;
+		IMG_UINT32				ui32AccStructSize;
+	} sRay;
+#endif
+
+} RGX_WORKLOAD;
+
+/*!
+ * Host data used to match the return data (actual cycles count) to the
+ * submitted command packet.
+ * The hash table is a per-DM circular buffer containing a key based on the
+ * workload characteristics. On job completion, the oldest workload data
+ * is evicted if the CB is full and the driver matches the characteristics
+ * to the matching data.
+ *
+ * o If the driver finds a match the existing cycle estimate is averaged with
+ *   the actual cycles used.
+ * o Otherwise a new hash entry is created with the actual cycles for this
+ *   workload.
+ *
+ * Subsequently if a match is found during command submission, the estimate
+ * is passed to the scheduler, e.g. adjust the GPU frequency if PDVFS is enabled.
+ */
+typedef struct _WORKLOAD_MATCHING_DATA_
+{
+	POS_LOCK				psHashLock;
+	HASH_TABLE				*psHashTable;		/*! existing workload cycle estimates for this DM */
+	RGX_WORKLOAD			asHashKeys[WORKLOAD_HASH_SIZE];
+	IMG_UINT64				aui64HashData[WORKLOAD_HASH_SIZE];
+	IMG_UINT32				ui32HashArrayWO;	/*! track the most recent workload estimates */
+} WORKLOAD_MATCHING_DATA;
+
+/*!
+ * A generic container for the workload matching data for GPU contexts:
+ * rendering (TA, 3D), compute, etc.
+ */
+typedef struct _WORKEST_HOST_DATA_
+{
+	union
+	{
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTA;	/*!< matching data for TA commands */
+			WORKLOAD_MATCHING_DATA	sData3D;	/*!< matching data for 3D commands */
+		} sTA3D;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataCDM;	/*!< matching data for CDM commands */
+		} sCompute;
+
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataTDM;	/*!< matching data for TDM-TQ commands */
+		} sTransfer;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		struct
+		{
+			WORKLOAD_MATCHING_DATA	sDataRDM;	/*!< matching data for RDM commands */
+		} sRay;
+#endif
+	} uWorkloadMatchingData;
+
+	/*
+	 * This is a per-context property, hence the TA and 3D share the same
+	 * per render context counter.
+	 */
+	IMG_UINT32				ui32WorkEstCCBReceived;	/*!< Used to ensure all submitted work
+														 estimation commands are received
+														 by the host before clean up. */
+} WORKEST_HOST_DATA;
+
+/*!
+ * Entries in the list of submitted workloads, used when the completed command
+ * returns data to the host.
+ *
+ * - the matching data is needed as it holds the hash data
+ * - the host data is needed for completion updates, ensuring memory is not
+ *   freed while workload estimates are in-flight.
+ * - the workload characteristic is used in the hash table look-up.
+ */
+typedef struct _WORKEST_RETURN_DATA_
+{
+	WORKEST_HOST_DATA		*psWorkEstHostData;
+	WORKLOAD_MATCHING_DATA	*psWorkloadMatchingData;
+	RGX_WORKLOAD			sWorkloadCharacteristics;
+} WORKEST_RETURN_DATA;
+#endif
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+#define RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES    4
+#endif
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+typedef struct
+{
+#if defined(PDUMP)
+	IMG_HANDLE      hPdumpPages;
+#endif
+	PG_HANDLE       sPages;
+	IMG_DEV_PHYADDR sPhysAddr;
+} RGX_MIPS_ADDRESS_TRAMPOLINE;
+#endif
+
+
+/*!
+ ******************************************************************************
+ * RGX Device error counts
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_ERROR_COUNTS_
+{
+	IMG_UINT32 ui32WGPErrorCount;		/*!< count of the number of WGP checksum errors */
+	IMG_UINT32 ui32TRPErrorCount;		/*!< count of the number of TRP checksum errors */
+} PVRSRV_RGXDEV_ERROR_COUNTS;
+
+/*!
+ ******************************************************************************
+ * RGX Debug dump firmware trace log type
+ *****************************************************************************/
+typedef IMG_UINT32 RGX_FWT_LOGTYPE;
+#define RGX_FWT_LOGTYPE_NONE    0U
+#define RGX_FWT_LOGTYPE_BINARY  1U
+#define RGX_FWT_LOGTYPE_DECODED 2U
+#define RGX_FWT_LOGTYPE_PARTIAL 3U
+
+/*!
+ ******************************************************************************
+ * RGX Device info
+ *****************************************************************************/
+typedef struct _PVRSRV_RGXDEV_INFO_
+{
+	PVRSRV_DEVICE_NODE		*psDeviceNode;
+
+	PVRSRV_DEVICE_FEATURE_CONFIG	sDevFeatureCfg;
+
+	IMG_BOOL				bDevInit2Done;
+
+	IMG_BOOL				bFirmwareInitialised;
+	IMG_BOOL				bPDPEnabled;
+
+	IMG_HANDLE				hDbgReqNotify;
+
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvRegsBaseKM;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/* Kernel mode linear address of device registers */
+	void __iomem			*pvSecureRegsBaseKM;
+#endif
+
+	IMG_HANDLE				hRegMapping;
+
+	/* System physical address of device registers */
+	IMG_CPU_PHYADDR			sRegsPhysBase;
+	/* Register region size in bytes */
+	IMG_UINT32				ui32RegSize;
+
+	PVRSRV_STUB_PBDESC		*psStubPBDescListKM;
+
+	/* Firmware memory context info */
+	DEVMEM_CONTEXT			*psKernelDevmemCtx;
+	DEVMEM_HEAP				*psFirmwareMainHeap;
+	DEVMEM_HEAP				*psFirmwareConfigHeap;
+	MMU_CONTEXT				*psKernelMMUCtx;
+
+	void					*pvDeviceMemoryHeap;
+
+	/* Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBCtlMemDesc;      /*!< memdesc for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtl;             /*!< kernel mapping for Kernel CCB control */
+	RGXFWIF_CCB_CTL			*psKernelCCBCtlLocal;        /*!< cpu local copy of Kernel CCB control */
+	DEVMEM_MEMDESC			*psKernelCCBMemDesc;         /*!< memdesc for Kernel CCB */
+	IMG_UINT8				*psKernelCCB;                /*!< kernel mapping for Kernel CCB */
+	DEVMEM_MEMDESC			*psKernelCCBRtnSlotsMemDesc; /*!< Return slot array for Kernel CCB commands */
+	IMG_UINT32				*pui32KernelCCBRtnSlots;     /*!< kernel mapping for return slot array */
+
+	/* Firmware CCB */
+	DEVMEM_MEMDESC			*psFirmwareCCBCtlMemDesc;   /*!< memdesc for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtl;          /*!< kernel mapping for Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psFirmwareCCBCtlLocal;     /*!< cpu local copy of Firmware CCB control */
+	DEVMEM_MEMDESC			*psFirmwareCCBMemDesc;      /*!< memdesc for Firmware CCB */
+	IMG_UINT8				*psFirmwareCCB;             /*!< kernel mapping for Firmware CCB */
+
+	/* Workload Estimation Firmware CCB */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBCtlMemDesc;   /*!< memdesc for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtl;          /*!< kernel mapping for Workload Estimation Firmware CCB control */
+	RGXFWIF_CCB_CTL			*psWorkEstFirmwareCCBCtlLocal;     /*!< cpu local copy of Workload Estimation Firmware CCB control */
+	DEVMEM_MEMDESC			*psWorkEstFirmwareCCBMemDesc;      /*!< memdesc for Workload Estimation Firmware CCB */
+	IMG_UINT8				*psWorkEstFirmwareCCB;             /*!< kernel mapping for Workload Estimation Firmware CCB */
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	/* Counter dumping */
+	DEVMEM_MEMDESC			*psCounterBufferMemDesc;      /*!< mem desc for counter dumping buffer */
+	POS_LOCK				hCounterDumpingLock;          /*!< Lock for guarding access to counter dumping buffer */
+#endif
+
+	PVRSRV_MEMALLOCFLAGS_T  uiFWPoisonOnFreeFlag;           /*!< Flag for poisoning FW allocations when freed */
+
+	IMG_BOOL				bIgnoreHWReportedBVNC;			/*!< Ignore BVNC reported by HW */
+
+    /* multicore configuration information */
+    IMG_UINT32              ui32MultiCoreNumCores;      /* total cores primary + secondaries. 0 for non-multi core */
+    IMG_UINT32              ui32MultiCorePrimaryId;     /* primary core id for this device */
+    IMG_UINT64             *pui64MultiCoreCapabilities; /* capabilities for each core */
+
+	/*
+		if we don't preallocate the pagetables we must
+		insert newly allocated page tables dynamically
+	*/
+	void					*pvMMUContextList;
+
+	IMG_UINT32				ui32ClkGateStatusReg;
+	IMG_UINT32				ui32ClkGateStatusMask;
+
+	DEVMEM_MEMDESC			*psRGXFWCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCodeDevVAddrBase;
+	IMG_UINT32			ui32FWCodeSizeInBytes;
+	DEVMEM_MEMDESC			*psRGXFWDataMemDesc;
+	IMG_DEV_VIRTADDR		sFWDataDevVAddrBase;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	RGX_MIPS_ADDRESS_TRAMPOLINE	*psTrampoline;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWCorememCodeMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememCodeDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememCodeFWAddr;
+	IMG_UINT32			ui32FWCorememCodeSizeInBytes;
+
+	DEVMEM_MEMDESC			*psRGXFWIfCorememDataStoreMemDesc;
+	IMG_DEV_VIRTADDR		sFWCorememDataStoreDevVAddrBase;
+	RGXFWIF_DEV_VIRTADDR		sFWCorememDataStoreFWAddr;
+
+	DEVMEM_MEMDESC			*psRGXFWAlignChecksMemDesc;
+
+#if defined(PDUMP)
+	DEVMEM_MEMDESC			*psRGXFWSigTAChecksMemDesc;
+	IMG_UINT32				ui32SigTAChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSig3DChecksMemDesc;
+	IMG_UINT32				ui32Sig3DChecksSize;
+
+	DEVMEM_MEMDESC			*psRGXFWSigTDMChecksMemDesc;
+	IMG_UINT32				ui32SigTDMChecksSize;
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	DEVMEM_MEMDESC			*psRGXFWSigRDMChecksMemDesc;
+	IMG_UINT32				ui32SigRDMChecksSize;
+#endif
+
+	IMG_BOOL				bDumpedKCCBCtlAlready;
+
+	POS_SPINLOCK			hSyncCheckpointSignalSpinLock;						/*!< Guards data shared between an atomic & sleepable-context */
+#endif
+
+	POS_LOCK				hRGXFWIfBufInitLock;								/*!< trace buffer lock for initialisation phase */
+
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufCtlMemDesc;						/*!< memdesc of trace buffer control structure */
+	DEVMEM_MEMDESC			*psRGXFWIfTraceBufferMemDesc[RGXFW_THREAD_NUM];		/*!< memdesc of actual FW trace (log) buffer(s) */
+	IMG_PUINT32				apui32TraceBuffer[RGXFW_THREAD_NUM];				/*!< Trace buffer address (Host address), to be used by host when reading from trace buffer */
+	IMG_UINT32				ui32TraceBufSizeInDWords;							/*!< CPU local copy of FW Trace buffer size in dwords */
+	RGXFWIF_TRACEBUF		*psRGXFWIfTraceBufCtl;								/*!< structure containing trace control data and actual trace buffer */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwSysDataMemDesc;							/*!< memdesc of the firmware-shared system data structure */
+	RGXFWIF_SYSDATA			*psRGXFWIfFwSysData;								/*!< structure containing km-firmware shared system data */
+
+	DEVMEM_MEMDESC			*psRGXFWIfFwOsDataMemDesc;							/*!< memdesc of the firmware-shared os structure */
+	RGXFWIF_OSDATA			*psRGXFWIfFwOsData;									/*!< structure containing km-firmware shared os data */
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	DEVMEM_MEMDESC			*psRGXFWIfTBIBufferMemDesc;							/*!< memdesc of actual FW TBI buffer */
+	RGXFWIF_DEV_VIRTADDR	sRGXFWIfTBIBuffer;									/*!< TBI buffer data */
+	IMG_UINT32				ui32FWIfTBIBufferSize;
+#endif
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWRInfoBufCtlMemDesc;
+	RGXFWIF_HWRINFOBUF		*psRGXFWIfHWRInfoBufCtl;
+	IMG_UINT32				ui32ClockSource;
+	IMG_UINT32				ui32LastClockSource;
+
+	DEVMEM_MEMDESC			*psRGXFWIfGpuUtilFWCtlMemDesc;
+	RGXFWIF_GPU_UTIL_FW     *psRGXFWIfGpuUtilFW;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfBufMemDesc;
+	IMG_BYTE				*psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32RGXFWIfHWPerfBufSize; /* in bytes */
+	IMG_UINT32				ui32RGXL2HWPerfBufSize;   /* in bytes */
+
+	DEVMEM_MEMDESC			*psRGXFWIfRegCfgMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfHWPerfCountersMemDesc;
+
+	DEVMEM_MEMDESC			*psRGXFWIfConnectionCtlMemDesc;
+	RGXFWIF_CONNECTION_CTL	*psRGXFWIfConnectionCtl;
+
+	DEVMEM_MEMDESC			*psRGXFWHeapGuardPageReserveMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfSysInitMemDesc;
+	RGXFWIF_SYSINIT			*psRGXFWIfSysInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfOsInitMemDesc;
+	RGXFWIF_OSINIT			*psRGXFWIfOsInit;
+
+	DEVMEM_MEMDESC			*psRGXFWIfRuntimeCfgMemDesc;
+	RGXFWIF_RUNTIME_CFG		*psRGXFWIfRuntimeCfg;
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	DEVMEM_MEMDESC			*psRGXFWIfActiveContextBufDesc;
+	RGXFWIF_ACTIVE_CONTEXT_BUF_DATA		*psRGXFWIfActiveContextBuf;
+#endif
+
+	/* Premapped firmware memory context info */
+	DEVMEM_HEAP				*psPremappedFwRawHeap[RGX_NUM_DRIVERS_SUPPORTED];
+	DEVMEM_MEMDESC			*psPremappedFwRawMemDesc[RGX_NUM_DRIVERS_SUPPORTED];
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Array to store data needed for workload estimation when a workload
+	   has finished and its cycle time is returned to the host.	 */
+	WORKEST_RETURN_DATA     asReturnData[RETURN_DATA_ARRAY_SIZE];
+	IMG_UINT32              ui32ReturnDataWO;
+	POS_LOCK                hWorkEstLock;
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/**
+	 * Host memdesc and pointer to memory containing core clock rate in Hz.
+	 * Firmware (PDVFS) updates the memory on changing the core clock rate over
+	 * GPIO.
+	 * Note: Shared memory needs atomic access from Host driver and firmware,
+	 * hence size should not be greater than memory transaction granularity.
+	 * Currently it is chosen to be 32 bits.
+	 */
+	DEVMEM_MEMDESC			*psRGXFWIFCoreClkRateMemDesc;
+	volatile IMG_UINT32		*pui32RGXFWIFCoreClkRate;
+	/**
+	 * Last sampled core clk rate.
+	 */
+	volatile IMG_UINT32		ui32CoreClkRateSnapshot;
+#endif
+
+	/*
+	   HWPerf data for the RGX device
+	 */
+
+	POS_LOCK    hHWPerfLock;  /*! Critical section lock that protects HWPerf code
+	                           *  from multiple thread duplicate init/deinit
+	                           *  and loss/freeing of FW & Host resources while in
+	                           *  use in another thread e.g. MSIR. */
+
+	IMG_UINT64  ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_LAST];         /*! Event filter for FW events (settable by AppHint) */
+	IMG_HANDLE  hHWPerfStream[RGX_HWPERF_L2_STREAM_LAST];            /*! TL Stream buffer (L2) for firmware event stream */
+	IMG_UINT32  ui32L2BufMaxPacketSize[RGX_HWPERF_L2_STREAM_LAST];   /*! Max allowed packet size in FW HWPerf TL (L2) buffer */
+	IMG_BOOL    bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_LAST]; /*! Flag to indicate if copying HWPerf data is suspended */
+	IMG_UINT64  ui64HWPerfFwFilter;                                  /*! Event filter for FW events created from OR-ing ui64HWPerfFilter values. */
+	IMG_UINT32  uiHWPerfStreamCount;                                 /*! Value indicating if any of the HWPerf streams has been created */
+
+	IMG_UINT32  ui32HWPerfHostFilter;      /*! Event filter for HWPerfHost stream (settable by AppHint) */
+	POS_LOCK    hLockHWPerfHostStream;     /*! Lock guarding access to HWPerfHost stream from multiple threads */
+	IMG_HANDLE  hHWPerfHostStream;         /*! TL Stream buffer for host only event stream */
+	IMG_UINT32  ui32HWPerfHostBufSize;     /*! Host side buffer size in bytes */
+	IMG_UINT32  ui32HWPerfHostLastOrdinal; /*! Ordinal of the last packet emitted in HWPerfHost TL stream.
+	                                        *  Guarded by hLockHWPerfHostStream */
+	IMG_UINT32  ui32HWPerfHostNextOrdinal; /*! Ordinal number for HWPerfHost events. Guarded by hHWPerfHostSpinLock */
+	IMG_UINT8   *pui8DeferredEvents;       /*! List of HWPerfHost events yet to be emitted in the TL stream.
+	                                        *  Events generated from atomic context are deferred "emitted"
+											*  as the "emission" code can sleep */
+	IMG_UINT16  ui16DEReadIdx;             /*! Read index in the above deferred events buffer */
+	IMG_UINT16  ui16DEWriteIdx;            /*! Write index in the above deferred events buffer */
+	void        *pvHostHWPerfMISR;         /*! MISR to emit pending/deferred events in HWPerfHost TL stream */
+	POS_SPINLOCK hHWPerfHostSpinLock;      /*! Guards data shared between an atomic & sleepable-context */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32  ui32DEHighWatermark;       /*! High watermark of deferred events buffer usage. Protected by
+	                                        *! hHWPerfHostSpinLock */
+	/* Max number of times DeferredEmission waited for an atomic-context to "finish" packet write */
+	IMG_UINT32  ui32WaitForAtomicCtxPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an atomic-context packet loss (due to too long wait for "write" finish) */
+	IMG_BOOL    bWarnedAtomicCtxPktLost;
+	/* Max number of times DeferredEmission scheduled-out to give a chance to the right-ordinal packet to be emitted */
+	IMG_UINT32  ui32WaitForRightOrdPktHighWatermark; /*! Protected by hLockHWPerfHostStream */
+	/* Whether warning has been logged about an packet loss (due to too long wait for right ordinal to emit) */
+	IMG_BOOL    bWarnedPktOrdinalBroke;
+#endif
+
+	void        *pvGpuFtraceData;
+
+	/* Poll data for detecting firmware fatal errors */
+	IMG_UINT32				aui32CrLastPollCount[RGXFW_THREAD_NUM];
+	IMG_UINT32				ui32KCCBCmdsExecutedLastTime;
+	IMG_BOOL				bKCCBCmdsWaitingLastTime;
+	IMG_UINT32				ui32GEOTimeoutsLastTime;
+	IMG_UINT32				ui32InterruptCountLastTime;
+	IMG_UINT32				ui32MissingInterruptsLastTime;
+
+	/* Client stall detection */
+	IMG_UINT32				ui32StalledClientMask;
+
+	IMG_BOOL				bWorkEstEnabled;
+	IMG_BOOL				bPDVFSEnabled;
+
+	void					*pvLISRData;
+	void					*pvMISRData;
+	void					*pvAPMISRData;
+	RGX_ACTIVEPM_CONF		eActivePMConf;
+
+	volatile IMG_UINT32		aui32SampleIRQCount[RGXFW_THREAD_NUM];
+
+	DEVMEM_MEMDESC			*psRGXFaultAddressMemDesc;
+
+	DEVMEM_MEMDESC			*psSLC3FenceMemDesc;
+
+	/* If we do 10 deferred memory allocations per second, then the ID would wrap around after 13 years */
+	IMG_UINT32				ui32ZSBufferCurrID;	/*!< ID assigned to the next deferred devmem allocation */
+	IMG_UINT32				ui32FreelistCurrID;	/*!< ID assigned to the next freelist */
+
+	POS_LOCK				hLockZSBuffer;		/*!< Lock to protect simultaneous access to ZSBuffers */
+	DLLIST_NODE				sZSBufferHead;		/*!< List of on-demand ZSBuffers */
+	POS_LOCK				hLockFreeList;		/*!< Lock to protect simultaneous access to Freelists */
+	DLLIST_NODE				sFreeListHead;		/*!< List of growable Freelists */
+	PSYNC_PRIM_CONTEXT		hSyncPrimContext;
+	PVRSRV_CLIENT_SYNC_PRIM	*psPowSyncPrim;
+
+	IMG_UINT32				ui32ActivePMReqOk;
+	IMG_UINT32				ui32ActivePMReqDenied;
+	IMG_UINT32				ui32ActivePMReqNonIdle;
+	IMG_UINT32				ui32ActivePMReqRetry;
+	IMG_UINT32				ui32ActivePMReqTotal;
+
+	IMG_HANDLE				hProcessQueuesMISR;
+
+	IMG_UINT32				ui32DeviceFlags;		/*!< Flags to track general device state */
+
+	/* GPU DVFS Table */
+	RGX_GPU_DVFS_TABLE		*psGpuDVFSTable;
+
+	/* Pointer to function returning the GPU utilisation statistics since the last
+	 * time the function was called. Supports different users at the same time.
+	 *
+	 * psReturnStats [out]: GPU utilisation statistics (active high/active low/idle/blocked)
+	 *                      in microseconds since the last time the function was called
+	 *                      by a specific user (identified by hGpuUtilUser)
+	 *
+	 * Returns PVRSRV_OK in case the call completed without errors,
+	 * some other value otherwise.
+	 */
+	PVRSRV_ERROR (*pfnGetGpuUtilStats) (PVRSRV_DEVICE_NODE *psDeviceNode,
+	                                    IMG_HANDLE hGpuUtilUser,
+	                                    RGXFWIF_GPU_UTIL_STATS *psReturnStats);
+
+	/* Pointer to function that checks if the physical GPU IRQ
+	 * line has been asserted and clears it if so */
+	IMG_BOOL (*pfnRGXAckIrq) (struct _PVRSRV_RGXDEV_INFO_ *psDevInfo);
+
+	POS_LOCK				hGPUUtilLock;
+
+	/* Register configuration */
+	RGX_REG_CONFIG			sRegConfig;
+
+	IMG_BOOL				bRGXPowered;
+	DLLIST_NODE				sMemoryContextList;
+
+	POSWR_LOCK				hRenderCtxListLock;
+	POSWR_LOCK				hComputeCtxListLock;
+	POSWR_LOCK				hTransferCtxListLock;
+	POSWR_LOCK				hTDMCtxListLock;
+	POSWR_LOCK				hMemoryCtxListLock;
+	POSWR_LOCK				hKickSyncCtxListLock;
+
+	/* Linked list of deferred KCCB commands due to a full KCCB.
+	 * Access to members sKCCBDeferredCommandsListHead and ui32KCCBDeferredCommandsCount
+	 * are protected by the hLockKCCBDeferredCommandsList spin lock. */
+	POS_SPINLOCK			hLockKCCBDeferredCommandsList; /*!< Protects deferred KCCB commands list */
+	DLLIST_NODE				sKCCBDeferredCommandsListHead;
+	IMG_UINT32				ui32KCCBDeferredCommandsCount; /*!< No of commands in the deferred list */
+
+	/* Linked lists of contexts on this device */
+	DLLIST_NODE				sRenderCtxtListHead;
+	DLLIST_NODE				sComputeCtxtListHead;
+	DLLIST_NODE				sTransferCtxtListHead;
+	DLLIST_NODE				sTDMCtxtListHead;
+	DLLIST_NODE				sKickSyncCtxtListHead;
+
+	DLLIST_NODE				sCommonCtxtListHead;
+	POSWR_LOCK				hCommonCtxtListLock;
+	IMG_UINT32				ui32CommonCtxtCurrentID;	/*!< ID assigned to the next common context */
+
+	POS_LOCK				hDebugFaultInfoLock;	/*!< Lock to protect the debug fault info list */
+	POS_LOCK				hMMUCtxUnregLock;		/*!< Lock to protect list of unregistered MMU contexts */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	POS_LOCK				hNMILock; /*!< Lock to protect NMI operations */
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32				ui32AvailablePowUnitsMask;
+	IMG_UINT32				ui32AvailableRACMask;
+#endif
+
+	RGX_LAYER_PARAMS		sLayerParams;
+
+	RGXFWIF_DM				eBPDM;					/*!< Current breakpoint data master */
+	IMG_BOOL				bBPSet;					/*!< A Breakpoint has been set */
+	POS_LOCK				hBPLock;				/*!< Lock for break point operations */
+
+	IMG_UINT32				ui32CoherencyTestsDone;
+
+	ATOMIC_T				iCCBSubmissionOrdinal; /* Rolling count used to indicate CCB submission order (all CCBs) */
+	POS_LOCK				hCCBRecoveryLock;      /* Lock to protect pvEarliestStalledClientCCB and ui32OldestSubmissionOrdinal variables */
+	void					*pvEarliestStalledClientCCB; /* Will point to cCCB command to unblock in the event of a stall */
+	IMG_UINT32				ui32OldestSubmissionOrdinal; /* Earliest submission ordinal of CCB entry found so far */
+	IMG_UINT32				ui32SLRHoldoffCounter;   /* Decremented each time health check is called until zero. SLR only happen when zero. */
+
+	POS_LOCK				hCCBStallCheckLock; /* Lock used to guard against multiple threads simultaneously checking for stalled CCBs */
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	/* Firmware gcov buffer */
+	DEVMEM_MEMDESC			*psFirmwareGcovBufferMemDesc;      /*!< mem desc for Firmware gcov dumping buffer */
+	IMG_UINT32				ui32FirmwareGcovSize;
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	/* Value to store for each page size range config register in MMU4 */
+	IMG_UINT64				aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES];
+#endif
+
+
+
+	IMG_HANDLE				hTQCLISharedMem;		/*!< TQ Client Shared Mem PMR */
+	IMG_HANDLE				hTQUSCSharedMem;		/*!< TQ USC Shared Mem PMR */
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	DEVMEM_MEMDESC			*psRGXFWIfSecureBufMemDesc;
+	DEVMEM_MEMDESC			*psRGXFWIfNonSecureBufMemDesc;
+#endif
+
+	/* Timer Queries */
+	IMG_UINT32				ui32ActiveQueryId;		/*!< id of the active line */
+	IMG_BOOL				bSaveStart;				/*!< save the start time of the next kick on the device*/
+	IMG_BOOL				bSaveEnd;				/*!< save the end time of the next kick on the device*/
+
+	DEVMEM_MEMDESC			*psStartTimeMemDesc;    /*!< memdesc for Start Times */
+	IMG_UINT64				*pui64StartTimeById;    /*!< CPU mapping of the above */
+
+	DEVMEM_MEMDESC			*psEndTimeMemDesc;      /*!< memdesc for End Timer */
+	IMG_UINT64				*pui64EndTimeById;      /*!< CPU mapping of the above */
+
+	IMG_UINT32				aui32ScheduledOnId[RGX_MAX_TIMER_QUERIES];	/*!< kicks Scheduled on QueryId */
+	DEVMEM_MEMDESC			*psCompletedMemDesc;	/*!< kicks Completed on QueryId */
+	IMG_UINT32				*pui32CompletedById;	/*!< CPU mapping of the above */
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	POS_LOCK				hTimerQueryLock;		/*!< lock to protect simultaneous access to timer query members */
+#endif
+
+	PVRSRV_RGXDEV_ERROR_COUNTS sErrorCounts;		/*!< struct containing device error counts */
+
+	IMG_UINT32				ui32HostSafetyEventMask;/*!< mask of the safety events handled by the driver */
+
+	RGX_CONTEXT_RESET_REASON	eLastDeviceError;	/*!< device error reported to client */
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	PMR						*psGenHeapSecMem;		/*!< An allocation of secure memory mapped to
+													  the general devmem heap. The allocation is
+													  created and mapped at driver init. It's used for
+													  various purposes. See rgx_fwif_km.h for all use cases. */
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	DEVMEM_MEMDESC			*psRGXFWScratchBufMemDesc;
+#endif
+
+	RGX_FWT_LOGTYPE			eDebugDumpFWTLogType;
+
+	RGX_FW_INFO_HEADER      sFWInfoHeader;
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	IMG_UINT32              ui32TFBCLossyGroup;     /*!< TFBCCompressionControlGroup
+													  setting for those cores which support
+													  this feature. */
+#endif
+	RGXFWIF_GPU_UTIL_STATS	sGpuUtilStats;          /*!< GPU usage statistics */
+	POS_LOCK				hGpuUtilStatsLock;
+
+} PVRSRV_RGXDEV_INFO;
+
+
+
+typedef struct _RGX_TIMING_INFORMATION_
+{
+	/*! GPU default core clock speed in Hz */
+	IMG_UINT32			ui32CoreClockSpeed;
+
+	/*! Default SOC clock speed in Hz */
+	IMG_UINT32			ui32SOCClockSpeed;
+
+	/*! Active Power Management: GPU actively requests the host driver to be powered off */
+	IMG_BOOL			bEnableActivePM;
+
+	/*! Enable the GPU to power off internal Power Islands independently from the host driver */
+	IMG_BOOL			bEnableRDPowIsland;
+
+	/*! Active Power Management: Delay between the GPU idle and the request to the host */
+	IMG_UINT32			ui32ActivePMLatencyms;
+
+} RGX_TIMING_INFORMATION;
+
+typedef struct _RGX_DATA_
+{
+	/*! Timing information */
+	RGX_TIMING_INFORMATION	*psRGXTimingInfo;
+} RGX_DATA;
+
+typedef enum _RGX_QUERY_TIMESTAMP_TYPE_
+{
+	RGX_QUERY_HOST_TIMESTAMP,
+	RGX_QUERY_DEVICE_TIMESTAMP,
+} RGX_QUERY_TIMESTAMP_TYPE;
+
+
+/*
+	RGX PDUMP register bank name (prefix)
+*/
+#define RGX_PDUMPREG_NAME		"RGXREG"
+
+#endif /* RGXDEVICE_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxfw_log_helper.h b/drivers/gpu/drm/img-rogue/rgxfw_log_helper.h
new file mode 100644
index 000000000000..275b63aca46b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfw_log_helper.h
@@ -0,0 +1,79 @@
+/*************************************************************************/ /*!
+@File           rgxfw_log_helper.h
+@Title          Firmware TBI logging helper function
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Platform       Generic
+@Description    This file contains some helper code to make TBI logging possible
+                Specifically, it uses the SFIDLIST xmacro to trace ids back to
+                the original strings.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef RGXFW_LOG_HELPER_H
+#define RGXFW_LOG_HELPER_H
+
+#include "rgx_fwif_sf.h"
+
+static const IMG_CHAR *const groups[]= {
+#define X(A,B) #B,
+	RGXFW_LOG_SFGROUPLIST
+#undef X
+};
+
+/*  idToStringID : Search SFs tuples {id,string} for a matching id.
+ *   return index to array if found or RGXFW_SF_LAST if none found.
+ *   bsearch could be used as ids are in increasing order. */
+#if defined(RGX_FIRMWARE)
+static IMG_UINT32 idToStringID(IMG_UINT32 ui32CheckData, const RGXFW_STID_FMT *const psSFs)
+#else
+static IMG_UINT32 idToStringID(IMG_UINT32 ui32CheckData, const RGXKM_STID_FMT *const psSFs)
+#endif
+{
+	IMG_UINT32 i = 0, ui32Id = (IMG_UINT32)RGXFW_SF_LAST;
+
+	for ( i = 0 ; psSFs[i].ui32Id != (IMG_UINT32)RGXFW_SF_LAST ; i++)
+	{
+		if ( ui32CheckData == psSFs[i].ui32Id )
+		{
+			ui32Id = i;
+			break;
+		}
+	}
+	return ui32Id;
+}
+
+#endif /* RGXFW_LOG_HELPER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxfwcmnctx.c b/drivers/gpu/drm/img-rogue/rgxfwcmnctx.c
new file mode 100644
index 000000000000..d340307d542d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwcmnctx.c
@@ -0,0 +1,803 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware common context utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware common context utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxfwcmnctx.h"
+#include "rgxfwutils.h"
+#include "devicemem_pdump.h"
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#include "rogue_trace_events.h"
+#endif
+
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+#include "linux/cred.h"
+#include "linux/pid.h"
+#endif
+/*
+ * Maximum length of time a DM can run for before the DM will be marked
+ * as out-of-time. CDM has an increased value due to longer running kernels.
+ *
+ * These deadlines are increased on FPGA, EMU and VP due to the slower
+ * execution time of these platforms. PDUMPS are also included since they
+ * are often run on EMU, FPGA or in CSim.
+ */
+#if defined(FPGA) || defined(EMULATOR) || defined(VIRTUAL_PLATFORM) || defined(PDUMP)
+#define RGXFWIF_MAX_WORKLOAD_DEADLINE_MS     (480000)
+#define RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS (10800000)
+#else
+#define RGXFWIF_MAX_WORKLOAD_DEADLINE_MS     (40000)
+#define RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS (600000)
+#endif
+#define RGXFWIF_MAX_RDM_WORKLOAD_DEADLINE_MS (36000000)
+
+struct _RGX_SERVER_COMMON_CONTEXT_ {
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	DEVMEM_MEMDESC *psFWCommonContextMemDesc;
+	PRGXFWIF_FWCOMMONCONTEXT sFWCommonContextFWAddr;
+	SERVER_MMU_CONTEXT *psServerMMUContext;
+	DEVMEM_MEMDESC *psFWMemContextMemDesc;
+	DEVMEM_MEMDESC *psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC *psContextStateMemDesc;
+	RGX_CLIENT_CCB *psClientCCB;
+	DEVMEM_MEMDESC *psClientCCBMemDesc;
+	DEVMEM_MEMDESC *psClientCCBCtrlMemDesc;
+	IMG_BOOL bCommonContextMemProvided;
+	IMG_UINT32 ui32ContextID;
+	DLLIST_NODE sListNode;
+	RGX_CONTEXT_RESET_REASON eLastResetReason;
+	IMG_UINT32 ui32LastResetJobRef;
+	IMG_INT32 i32Priority;
+	RGX_CCB_REQUESTOR_TYPE eRequestor;
+};
+
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+static IMG_UINT32 _GetUID(IMG_PID pid)
+{
+	struct task_struct *psTask;
+	struct pid *psPid;
+
+	psPid = find_get_pid((pid_t)pid);
+	if (!psPid)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to lookup PID %u.",
+		                        __func__, pid));
+		return 0;
+	}
+
+	psTask = get_pid_task(psPid, PIDTYPE_PID);
+	if (!psTask)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to get pid task for PID %u.",
+		                        __func__, pid));
+	}
+	put_pid(psPid);
+
+	return psTask ? from_kuid(&init_user_ns, psTask->cred->uid) : 0;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       _CheckPriority
+@Description    Check if priority is allowed for requestor type
+@Input          psDevInfo    pointer to DevInfo struct
+@Input          i32Priority Requested priority
+@Input          eRequestor   Requestor type specifying data master
+@Return         PVRSRV_ERROR PVRSRV_OK on success
+*/ /**************************************************************************/
+static PVRSRV_ERROR _CheckPriority(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_INT32 i32Priority,
+								   RGX_CCB_REQUESTOR_TYPE eRequestor)
+{
+	/* Only contexts from a single PID allowed with real time priority (highest priority) */
+	if (i32Priority == RGX_CTX_PRIORITY_REALTIME)
+	{
+		DLLIST_NODE *psNode, *psNext;
+
+		dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+		{
+			RGX_SERVER_COMMON_CONTEXT *psThisContext =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+			if (psThisContext->i32Priority == RGX_CTX_PRIORITY_REALTIME &&
+				psThisContext->eRequestor == eRequestor &&
+				RGXGetPIDFromServerMMUContext(psThisContext->psServerMMUContext) != OSGetCurrentClientProcessIDKM())
+			{
+				PVR_LOG(("Only one process can have contexts with real time priority"));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
+									 PVRSRV_DEVICE_NODE *psDeviceNode,
+									 RGX_CCB_REQUESTOR_TYPE eRGXCCBRequestor,
+									 RGXFWIF_DM eDM,
+									 SERVER_MMU_CONTEXT *psServerMMUContext,
+									 DEVMEM_MEMDESC *psAllocatedMemDesc,
+									 IMG_UINT32 ui32AllocatedOffset,
+									 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+									 DEVMEM_MEMDESC *psContextStateMemDesc,
+									 IMG_UINT32 ui32CCBAllocSizeLog2,
+									 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+									 IMG_UINT32 ui32ContextFlags,
+									 IMG_INT32 i32Priority,
+									 IMG_UINT32 ui32MaxDeadlineMS,
+									 IMG_UINT64 ui64RobustnessAddress,
+									 RGX_COMMON_CONTEXT_INFO *psInfo,
+									 RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_COMMON_CONTEXT *psServerCommonContext;
+	RGXFWIF_FWCOMMONCONTEXT sFWCommonContext = {{0}};
+	IMG_UINT32 ui32FWCommonContextOffset;
+	IMG_UINT8 *pui8Ptr;
+	PVRSRV_ERROR eError;
+
+	/*
+	 * Allocate all the resources that are required
+	 */
+	psServerCommonContext = OSAllocMem(sizeof(*psServerCommonContext));
+	if (psServerCommonContext == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+
+	psServerCommonContext->psDevInfo = psDevInfo;
+	psServerCommonContext->psServerMMUContext = psServerMMUContext;
+
+	if (psAllocatedMemDesc)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+					 "Using existing MemDesc for Rogue firmware %s context (offset = %d)",
+					 aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+					 ui32AllocatedOffset);
+		ui32FWCommonContextOffset = ui32AllocatedOffset;
+		psServerCommonContext->psFWCommonContextMemDesc = psAllocatedMemDesc;
+		psServerCommonContext->bCommonContextMemProvided = IMG_TRUE;
+	}
+	else
+	{
+		/* Allocate device memory for the firmware context */
+		PDUMPCOMMENT(psDeviceNode,
+					 "Allocate Rogue firmware %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
+		eError = DevmemFwAllocate(psDevInfo,
+								sizeof(sFWCommonContext),
+								RGX_FWCOMCTX_ALLOCFLAGS,
+								"FwContext",
+								&psServerCommonContext->psFWCommonContextMemDesc);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to allocate firmware %s context (%s)",
+			         __func__,
+			         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+			         PVRSRVGetErrorString(eError)));
+			goto fail_contextalloc;
+		}
+		ui32FWCommonContextOffset = 0;
+		psServerCommonContext->bCommonContextMemProvided = IMG_FALSE;
+	}
+
+	/* Record this context so we can refer to it if the FW needs to tell us it was reset. */
+	psServerCommonContext->eLastResetReason    = RGX_CONTEXT_RESET_REASON_NONE;
+	psServerCommonContext->ui32LastResetJobRef = 0;
+	psServerCommonContext->ui32ContextID       = psDevInfo->ui32CommonCtxtCurrentID++;
+
+	/*
+	 * Temporarily map the firmware context to the kernel and initialise it
+	 */
+	eError = DevmemAcquireCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc,
+	                                  (void **)&pui8Ptr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to map firmware %s context to CPU (%s)",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         PVRSRVGetErrorString(eError)));
+		goto fail_cpuvirtacquire;
+	}
+
+	/* Allocate the client CCB */
+	eError = RGXCreateCCB(psDevInfo,
+						  ui32CCBAllocSizeLog2,
+						  ui32CCBMaxAllocSizeLog2,
+						  ui32ContextFlags,
+						  psConnection,
+						  eRGXCCBRequestor,
+						  psServerCommonContext,
+						  &psServerCommonContext->psClientCCB,
+						  &psServerCommonContext->psClientCCBMemDesc,
+						  &psServerCommonContext->psClientCCBCtrlMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: failed to create CCB for %s context (%s)",
+		         __func__,
+		         aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+		         PVRSRVGetErrorString(eError)));
+		goto fail_allocateccb;
+	}
+
+	sFWCommonContext.eDM = eDM;
+	BITMASK_SET(sFWCommonContext.ui32CompatFlags, RGXFWIF_CONTEXT_COMPAT_FLAGS_HAS_DEFER_COUNT);
+
+	/* Set the firmware CCB device addresses in the firmware common context */
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psCCB,
+						  psServerCommonContext->psClientCCBMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail_cccbfwaddr);
+
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psCCBCtl,
+						  psServerCommonContext->psClientCCBCtrlMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail_cccbctrlfwaddr);
+
+#if defined(RGX_FEATURE_META_DMA_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_DMA))
+	{
+		RGXSetMetaDMAAddress(&sFWCommonContext.sCCBMetaDMAAddr,
+							 psServerCommonContext->psClientCCBMemDesc,
+							 &sFWCommonContext.psCCB,
+							 0);
+	}
+#endif
+
+	/* Set the memory context device address */
+	psServerCommonContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
+	eError = RGXSetFirmwareAddress(&sFWCommonContext.psFWMemContext,
+						  psFWMemContextMemDesc,
+						  0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", fail_fwmemctxfwaddr);
+
+	/* Set the framework register updates address */
+	psServerCommonContext->psFWFrameworkMemDesc = psInfo->psFWFrameworkMemDesc;
+	if (psInfo->psFWFrameworkMemDesc != NULL)
+	{
+		eError = RGXSetFirmwareAddress(&sFWCommonContext.psRFCmd,
+				psInfo->psFWFrameworkMemDesc,
+				0, RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:4", fail_fwframeworkfwaddr);
+	}
+	else
+	{
+		/* This should never be touched in this contexts without a framework
+		 * memdesc, but ensure it is zero so we see crashes if it is.
+		 */
+		sFWCommonContext.psRFCmd.ui32Addr = 0;
+	}
+
+	eError = _CheckPriority(psDevInfo, i32Priority, eRGXCCBRequestor);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CheckPriority", fail_checkpriority);
+
+	psServerCommonContext->i32Priority = i32Priority;
+	psServerCommonContext->eRequestor = eRGXCCBRequestor;
+
+	sFWCommonContext.i32Priority = i32Priority;
+	sFWCommonContext.ui32PrioritySeqNum = 0;
+
+	if (eDM == RGXFWIF_DM_CDM)
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_CDM_WORKLOAD_DEADLINE_MS);
+	}
+	else if (eDM == RGXFWIF_DM_RAY)
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_RDM_WORKLOAD_DEADLINE_MS);
+	}
+	else
+	{
+		sFWCommonContext.ui32MaxDeadlineMS = MIN(ui32MaxDeadlineMS, RGXFWIF_MAX_WORKLOAD_DEADLINE_MS);
+	}
+
+	sFWCommonContext.ui64RobustnessAddress = ui64RobustnessAddress;
+
+	/* Store a references to Server Common Context and PID for notifications back from the FW. */
+	sFWCommonContext.ui32ServerCommonContextID = psServerCommonContext->ui32ContextID;
+	sFWCommonContext.ui32PID                   = OSGetCurrentClientProcessIDKM();
+#if defined(__linux__) && defined(PVRSRV_ANDROID_TRACE_GPU_WORK_PERIOD)
+	sFWCommonContext.ui32UID                   = _GetUID(sFWCommonContext.ui32PID);
+#endif
+	OSStringSafeCopy(sFWCommonContext.szProcName, psConnection->pszProcName, RGXFW_PROCESS_NAME_LEN);
+
+	/* Set the firmware GPU context state buffer */
+	psServerCommonContext->psContextStateMemDesc = psContextStateMemDesc;
+	if (psContextStateMemDesc)
+	{
+		eError = RGXSetFirmwareAddress(&sFWCommonContext.psContextState,
+							  psContextStateMemDesc,
+							  0,
+							  RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:5", fail_ctxstatefwaddr);
+	}
+
+	OSCachedMemCopy(IMG_OFFSET_ADDR(pui8Ptr, ui32FWCommonContextOffset), &sFWCommonContext, sizeof(sFWCommonContext));
+	RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pui8Ptr, ui32FWCommonContextOffset),
+	                          sizeof(sFWCommonContext),
+	                          PVRSRV_CACHE_OP_FLUSH);
+
+	/*
+	 * Dump the created context
+	 */
+	PDUMPCOMMENT(psDeviceNode,
+				 "Dump %s context", aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT]);
+	DevmemPDumpLoadMem(psServerCommonContext->psFWCommonContextMemDesc,
+					   ui32FWCommonContextOffset,
+					   sizeof(sFWCommonContext),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* We've finished the setup so release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
+
+	/* Map this allocation into the FW */
+	eError = RGXSetFirmwareAddress(&psServerCommonContext->sFWCommonContextFWAddr,
+						  psServerCommonContext->psFWCommonContextMemDesc,
+						  ui32FWCommonContextOffset,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:6", fail_fwcommonctxfwaddr);
+
+#if defined(__linux__) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+	{
+		IMG_UINT32 ui32FWAddr;
+		switch (eDM) {
+			case RGXFWIF_DM_GEOM:
+				ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
+						psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, sTAContext));
+				break;
+			case RGXFWIF_DM_3D:
+				ui32FWAddr = (IMG_UINT32) ((uintptr_t) IMG_CONTAINER_OF((void *) ((uintptr_t)
+						psServerCommonContext->sFWCommonContextFWAddr.ui32Addr), RGXFWIF_FWRENDERCONTEXT, s3DContext));
+				break;
+			default:
+				ui32FWAddr = psServerCommonContext->sFWCommonContextFWAddr.ui32Addr;
+				break;
+		}
+
+		trace_rogue_create_fw_context(OSGetCurrentClientProcessNameKM(),
+									  aszCCBRequestors[eRGXCCBRequestor][REQ_PDUMP_COMMENT],
+									  psDeviceNode->sDevId.ui32InternalID,
+									  ui32FWAddr);
+	}
+#endif
+
+	/*Add the node to the list when finalised */
+	OSWRLockAcquireWrite(psDevInfo->hCommonCtxtListLock);
+	dllist_add_to_tail(&(psDevInfo->sCommonCtxtListHead), &(psServerCommonContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hCommonCtxtListLock);
+
+	*ppsServerCommonContext = psServerCommonContext;
+	return PVRSRV_OK;
+
+fail_fwcommonctxfwaddr:
+	if (psContextStateMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psContextStateMemDesc);
+	}
+fail_ctxstatefwaddr:
+fail_checkpriority:
+	if (psInfo->psFWFrameworkMemDesc != NULL)
+	{
+		RGXUnsetFirmwareAddress(psInfo->psFWFrameworkMemDesc);
+	}
+fail_fwframeworkfwaddr:
+	RGXUnsetFirmwareAddress(psFWMemContextMemDesc);
+fail_fwmemctxfwaddr:
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
+fail_cccbctrlfwaddr:
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
+fail_cccbfwaddr:
+	RGXDestroyCCB(psDevInfo, psServerCommonContext->psClientCCB);
+fail_allocateccb:
+	DevmemReleaseCpuVirtAddr(psServerCommonContext->psFWCommonContextMemDesc);
+fail_cpuvirtacquire:
+	if (!psServerCommonContext->bCommonContextMemProvided)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psServerCommonContext->psFWCommonContextMemDesc);
+		psServerCommonContext->psFWCommonContextMemDesc = NULL;
+	}
+fail_contextalloc:
+	OSFreeMem(psServerCommonContext);
+fail_alloc:
+	return eError;
+}
+
+void FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+
+	OSWRLockAcquireWrite(psServerCommonContext->psDevInfo->hCommonCtxtListLock);
+	/* Remove the context from the list of all contexts. */
+	dllist_remove_node(&psServerCommonContext->sListNode);
+	OSWRLockReleaseWrite(psServerCommonContext->psDevInfo->hCommonCtxtListLock);
+
+	/*
+		Unmap the context itself and then all its resources
+	*/
+
+	/* Unmap the FW common context */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psFWCommonContextMemDesc);
+	/* Umap context state buffer (if there was one) */
+	if (psServerCommonContext->psContextStateMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psServerCommonContext->psContextStateMemDesc);
+	}
+	/* Unmap the framework buffer */
+	if (psServerCommonContext->psFWFrameworkMemDesc != NULL)
+	{
+		RGXUnsetFirmwareAddress(psServerCommonContext->psFWFrameworkMemDesc);
+	}
+	/* Unmap client CCB and CCB control */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBCtrlMemDesc);
+	RGXUnsetFirmwareAddress(psServerCommonContext->psClientCCBMemDesc);
+	/* Unmap the memory context */
+	RGXUnsetFirmwareAddress(psServerCommonContext->psFWMemContextMemDesc);
+
+	/* Destroy the client CCB */
+	RGXDestroyCCB(psServerCommonContext->psDevInfo, psServerCommonContext->psClientCCB);
+
+
+	/* Free the FW common context (if there was one) */
+	if (!psServerCommonContext->bCommonContextMemProvided)
+	{
+		DevmemFwUnmapAndFree(psServerCommonContext->psDevInfo,
+						psServerCommonContext->psFWCommonContextMemDesc);
+		psServerCommonContext->psFWCommonContextMemDesc = NULL;
+	}
+	/* Free the hosts representation of the common context */
+	OSFreeMem(psServerCommonContext);
+}
+
+PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->sFWCommonContextFWAddr;
+}
+
+RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psClientCCB;
+}
+
+SERVER_MMU_CONTEXT *FWCommonContextGetServerMMUCtx(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psServerMMUContext;
+}
+
+RGX_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                                               IMG_UINT32 *pui32LastResetJobRef)
+{
+	RGX_CONTEXT_RESET_REASON eLastResetReason;
+
+	PVR_ASSERT(psServerCommonContext != NULL);
+	PVR_ASSERT(pui32LastResetJobRef != NULL);
+
+	/* Take the most recent reason & job ref and reset for next time... */
+	eLastResetReason      = psServerCommonContext->eLastResetReason;
+	*pui32LastResetJobRef = psServerCommonContext->ui32LastResetJobRef;
+	psServerCommonContext->eLastResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+	psServerCommonContext->ui32LastResetJobRef = 0;
+
+	if (eLastResetReason == RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "A Hard Context Switch was triggered on the GPU to ensure Quality of Service."));
+	}
+
+	return eLastResetReason;
+}
+
+PVRSRV_RGXDEV_INFO* FWCommonContextGetRGXDevInfo(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext)
+{
+	return psServerCommonContext->psDevInfo;
+}
+
+PVRSRV_ERROR RGXGetFWCommonContextAddrFromServerMMUCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													   SERVER_MMU_CONTEXT *psServerMMUContext,
+													   PRGXFWIF_FWCOMMONCONTEXT *psFWCommonContextFWAddr)
+{
+	DLLIST_NODE *psNode, *psNext;
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMMON_CONTEXT *psThisContext =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+		if (psThisContext->psServerMMUContext == psServerMMUContext)
+		{
+			psFWCommonContextFWAddr->ui32Addr = psThisContext->sFWCommonContextFWAddr.ui32Addr;
+			return PVRSRV_OK;
+		}
+	}
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+PRGXFWIF_FWCOMMONCONTEXT RGXGetFWCommonContextAddrFromServerCommonCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													                  DLLIST_NODE *psNode)
+{
+	RGX_SERVER_COMMON_CONTEXT *psThisContext =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+	return FWCommonContextGetFWAddress(psThisContext);
+}
+
+PVRSRV_ERROR FWCommonContextSetFlags(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                     IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (BITMASK_ANY(ui32ContextFlags, ~RGX_CONTEXT_FLAGS_WRITEABLE_MASK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Context flag(s) invalid or not writeable (%d)",
+		         __func__, ui32ContextFlags));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else
+	{
+		RGXSetCCBFlags(psServerCommonContext->psClientCCB,
+		               ui32ContextFlags);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
+								CONNECTION_DATA *psConnection,
+								PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_INT32 i32Priority,
+								RGXFWIF_DM eDM)
+{
+	IMG_UINT32				ui32CmdSize;
+	IMG_UINT8				*pui8CmdPtr;
+	RGXFWIF_KCCB_CMD		sPriorityCmd = { 0 };
+	RGXFWIF_CCB_CMD_HEADER	*psCmdHeader;
+	RGXFWIF_CMD_PRIORITY	*psCmd;
+	PVRSRV_ERROR			eError;
+	RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psContext);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = _CheckPriority(psDevInfo, i32Priority, psContext->eRequestor);
+	PVR_LOG_GOTO_IF_ERROR(eError, "_CheckPriority", fail_checkpriority);
+
+	/*
+		Get space for command
+	*/
+	ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CCB_CMD_HEADER) + sizeof(RGXFWIF_CMD_PRIORITY));
+
+	eError = RGXAcquireCCB(psClientCCB,
+						   ui32CmdSize,
+						   (void **) &pui8CmdPtr,
+						   PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire space for client CCB", __func__));
+		}
+		goto fail_ccbacquire;
+	}
+
+	/*
+		Write the command header and command
+	*/
+	psCmdHeader = IMG_OFFSET_ADDR(pui8CmdPtr, 0);
+	psCmdHeader->eCmdType = RGXFWIF_CCB_CMD_TYPE_PRIORITY;
+	psCmdHeader->ui32CmdSize = RGX_CCB_FWALLOC_ALIGN(sizeof(RGXFWIF_CMD_PRIORITY));
+	pui8CmdPtr += sizeof(*psCmdHeader);
+
+	psCmd = IMG_OFFSET_ADDR(pui8CmdPtr, 0);
+	psCmd->i32Priority = i32Priority;
+	pui8CmdPtr += sizeof(*psCmd);
+
+	/*
+		We should reserve space in the kernel CCB here and fill in the command
+		directly.
+		This is so if there isn't space in the kernel CCB we can return with
+		retry back to services client before we take any operations
+	*/
+
+	/*
+		Submit the command
+	*/
+	RGXReleaseCCB(psClientCCB,
+				  ui32CmdSize,
+				  PDUMP_FLAGS_CONTINUOUS);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to release space in client CCB", __func__));
+		return eError;
+	}
+
+	/* Construct the priority command. */
+	sPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sPriorityCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psContext);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+	sPriorityCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	sPriorityCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									eDM,
+									&sPriorityCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit set priority command with error (%u)",
+				__func__,
+				eError));
+		goto fail_cmdacquire;
+	}
+
+	psContext->i32Priority = i32Priority;
+
+	return PVRSRV_OK;
+
+fail_ccbacquire:
+fail_checkpriority:
+fail_cmdacquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext, RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	if (psCurrentServerCommonContext == NULL)
+	{
+		/* the context has already been freed so there is nothing to do here */
+		return PVRSRV_OK;
+	}
+
+	return CheckForStalledCCB(psCurrentServerCommonContext->psDevInfo->psDeviceNode,
+	                          psCurrentServerCommonContext->psClientCCB,
+	                          eKickTypeDM);
+}
+
+void DumpFWCommonContextInfo(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
+                             DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             IMG_UINT32 ui32VerbLevel)
+{
+	if (psCurrentServerCommonContext == NULL)
+	{
+		/* the context has already been freed so there is nothing to do here */
+		return;
+	}
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_HIGH))
+	{
+		/* If high verbosity requested, dump whole CCB */
+		DumpCCB(psCurrentServerCommonContext->psDevInfo,
+		        psCurrentServerCommonContext->sFWCommonContextFWAddr,
+		        psCurrentServerCommonContext->psClientCCB,
+		        pfnDumpDebugPrintf,
+		        pvDumpDebugFile);
+	}
+	else
+	{
+		/* Otherwise, only dump first command in the CCB */
+		DumpFirstCCBCmd(psCurrentServerCommonContext->sFWCommonContextFWAddr,
+		                      psCurrentServerCommonContext->psClientCCB,
+		                      pfnDumpDebugPrintf,
+		                      pvDumpDebugFile);
+	}
+}
+
+void FWCommonContextListSetLastResetReason(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                           IMG_UINT32 *pui32ErrorPid,
+                                           const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_SERVER_COMMON_CONTEXT *psServerCommonContext = NULL;
+	IMG_UINT32 ui32ErrorPid = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hCommonCtxtListLock);
+
+	dllist_foreach_node(&psDevInfo->sCommonCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_COMMON_CONTEXT *psThisContext =
+		    IMG_CONTAINER_OF(psNode, RGX_SERVER_COMMON_CONTEXT, sListNode);
+
+		/* If the notification applies to all contexts update reset info
+		* for all contexts, otherwise only do so for the appropriate ID.
+		*/
+		if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS)
+		{
+			/* Notification applies to all contexts */
+			psThisContext->eLastResetReason    = psCmdContextResetNotification->eResetReason;
+			psThisContext->ui32LastResetJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+		}
+		else
+		{
+			/* Notification applies to one context only */
+			if (psThisContext->ui32ContextID == psCmdContextResetNotification->ui32ServerCommonContextID)
+			{
+				psServerCommonContext = psThisContext;
+				psServerCommonContext->eLastResetReason    = psCmdContextResetNotification->eResetReason;
+				psServerCommonContext->ui32LastResetJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+				ui32ErrorPid = RGXGetPIDFromServerMMUContext(psServerCommonContext->psServerMMUContext);
+				break;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hCommonCtxtListLock);
+
+	if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_ALL_CTXS)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: All contexts reset (Reason=%d, JobRef=0x%08x)",
+				__func__,
+				(IMG_UINT32)(psCmdContextResetNotification->eResetReason),
+				psCmdContextResetNotification->ui32ResetJobRef));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Context 0x%p reset (ID=0x%08x, Reason=%d, JobRef=0x%08x)",
+				__func__,
+				psServerCommonContext,
+				psCmdContextResetNotification->ui32ServerCommonContextID,
+				(IMG_UINT32)(psCmdContextResetNotification->eResetReason),
+				psCmdContextResetNotification->ui32ResetJobRef));
+	}
+
+	if (pui32ErrorPid)
+	{
+		*pui32ErrorPid = ui32ErrorPid;
+	}
+}
+
+/******************************************************************************
+ End of file (rgxfwcmnctx.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxfwcmnctx.h b/drivers/gpu/drm/img-rogue/rgxfwcmnctx.h
new file mode 100644
index 000000000000..eba2fac705da
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwcmnctx.h
@@ -0,0 +1,150 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware common context utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware common context utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWCMNCTX_H
+#define RGXFWCMNCTX_H
+
+#include "connection_server.h"
+#include "device.h"
+#include "rgxccb.h"
+#include "rgx_common.h"
+#include "devicemem_typedefs.h"
+#include "rgxdevice.h"
+#include "rgxmem.h"
+
+/*************************************************************************/ /*!
+@Function       FWCommonContextAllocate
+
+@Description    Allocate a FW common context. This allocates the HW memory
+                for the context, the CCB and wires it all together.
+
+@Input          psConnection            Connection this context is being created on
+@Input          psDeviceNode            Device node to create the FW context on
+                                        (must be RGX device node)
+@Input          eRGXCCBRequestor        RGX_CCB_REQUESTOR_TYPE enum constant which
+                                        represents the requestor of this FWCC
+@Input          eDM                     Data Master type
+@Input          psServerMMUContext      Server MMU memory context.
+@Input          psAllocatedMemDesc      Pointer to pre-allocated MemDesc to use
+                                        as the FW context or NULL if this function
+                                        should allocate it
+@Input          ui32AllocatedOffset     Offset into pre-allocate MemDesc to use
+                                        as the FW context. If psAllocatedMemDesc
+                                        is NULL then this parameter is ignored
+@Input          psFWMemContextMemDesc   MemDesc of the FW memory context this
+                                        common context resides on
+@Input          psContextStateMemDesc   FW context state (context switch) MemDesc
+@Input          ui32CCBAllocSizeLog2    Size of the CCB for this context
+@Input          ui32CCBMaxAllocSizeLog2 Maximum size to which CCB can grow for this context
+@Input          ui32ContextFlags        Flags which specify properties of the context
+@Input          i32Priority             Priority of the context
+@Input          ui32MaxDeadlineMS       Max deadline limit in MS that the workload can run
+@Input          ui64RobustnessAddress   Address for FW to signal a context reset
+@Input          psInfo                  Structure that contains extra info
+                                        required for the creation of the context
+                                        (elements might change from core to core)
+@Return         PVRSRV_OK if the context was successfully created
+*/ /**************************************************************************/
+PVRSRV_ERROR FWCommonContextAllocate(CONNECTION_DATA *psConnection,
+									 PVRSRV_DEVICE_NODE *psDeviceNode,
+									 RGX_CCB_REQUESTOR_TYPE eRGXCCBRequestor,
+									 RGXFWIF_DM eDM,
+									 SERVER_MMU_CONTEXT *psServerMMUContext,
+									 DEVMEM_MEMDESC *psAllocatedMemDesc,
+									 IMG_UINT32 ui32AllocatedOffset,
+									 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+									 DEVMEM_MEMDESC *psContextStateMemDesc,
+									 IMG_UINT32 ui32CCBAllocSizeLog2,
+									 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+									 IMG_UINT32 ui32ContextFlags,
+									 IMG_INT32 i32Priority,
+									 IMG_UINT32 ui32MaxDeadlineMS,
+									 IMG_UINT64 ui64RobustnessAddress,
+									 RGX_COMMON_CONTEXT_INFO *psInfo,
+									 RGX_SERVER_COMMON_CONTEXT **ppsServerCommonContext);
+
+
+void FWCommonContextFree(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+PRGXFWIF_FWCOMMONCONTEXT FWCommonContextGetFWAddress(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+RGX_CLIENT_CCB *FWCommonContextGetClientCCB(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+SERVER_MMU_CONTEXT *FWCommonContextGetServerMMUCtx(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+RGX_CONTEXT_RESET_REASON FWCommonContextGetLastResetReason(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                                           IMG_UINT32 *pui32LastResetJobRef);
+
+PVRSRV_RGXDEV_INFO* FWCommonContextGetRGXDevInfo(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext);
+
+PVRSRV_ERROR RGXGetFWCommonContextAddrFromServerMMUCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													   SERVER_MMU_CONTEXT *psServerMMUContext,
+													   PRGXFWIF_FWCOMMONCONTEXT *psFWCommonContextFWAddr);
+
+PRGXFWIF_FWCOMMONCONTEXT RGXGetFWCommonContextAddrFromServerCommonCtx(PVRSRV_RGXDEV_INFO *psDevInfo,
+													                  DLLIST_NODE *psNode);
+
+PVRSRV_ERROR FWCommonContextSetFlags(RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+                                     IMG_UINT32 ui32ContextFlags);
+
+PVRSRV_ERROR ContextSetPriority(RGX_SERVER_COMMON_CONTEXT *psContext,
+								CONNECTION_DATA *psConnection,
+								PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_INT32 i32Priority,
+								RGXFWIF_DM eDM);
+
+PVRSRV_ERROR CheckStalledClientCommonContext(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext, RGX_KICK_TYPE_DM eKickTypeDM);
+
+void DumpFWCommonContextInfo(RGX_SERVER_COMMON_CONTEXT *psCurrentServerCommonContext,
+                             DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                             void *pvDumpDebugFile,
+                             IMG_UINT32 ui32VerbLevel);
+
+void FWCommonContextListSetLastResetReason(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                           IMG_UINT32 *pui32ErrorPid,
+                                           const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification);
+
+#endif /* RGXFWCMNCTX_H */
+/******************************************************************************
+ End of file (rgxfwcmnctx.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxfwdbg.c b/drivers/gpu/drm/img-rogue/rgxfwdbg.c
new file mode 100644
index 000000000000..2271adf3ea9c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwdbg.c
@@ -0,0 +1,666 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debugging and miscellaneous functions server implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel services functions for debugging and other
+                miscellaneous functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "pvr_debug.h"
+#include "rgxfwdbg.h"
+#include "rgxfwutils.h"
+#include "rgxta3d.h"
+#include "pdump_km.h"
+#include "mmu_common.h"
+#include "devicemem_server.h"
+#include "osfunc.h"
+#include "vmm_pvz_server.h"
+#include "vz_vm.h"
+#if defined(PDUMP)
+#include "devicemem_pdump.h"
+#endif
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugQueryFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 *pui32RGXFWLogType)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!psDeviceNode || !pui32RGXFWLogType)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+	psDevInfo = psDeviceNode->pvDevice;
+
+	if (!psDevInfo || !psDevInfo->psRGXFWIfTraceBufCtl)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+	*pui32RGXFWLogType = psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType;
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetFWLogKM(
+	const CONNECTION_DATA * psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32RGXFWLogType)
+{
+	RGXFWIF_KCCB_CMD sLogTypeUpdateCmd;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevInfo;
+	IMG_UINT32 ui32OldRGXFWLogTpe;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (!psDeviceNode)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo = psDeviceNode->pvDevice;
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, INVALIDATE);
+	ui32OldRGXFWLogTpe = psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType;
+
+	/* check log type is valid */
+	if (ui32RGXFWLogType & ~RGXFWIF_LOG_TYPE_MASK)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	OSLockAcquire(psDevInfo->hRGXFWIfBufInitLock);
+
+	/* set the new log type and ensure the new log type is written to memory
+	 * before requesting the FW to read it
+	 */
+	psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = ui32RGXFWLogType;
+	OSMemoryBarrier(&psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+
+	/* Allocate firmware trace buffer resource(s) if not already done */
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo, RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	/* Check if LogType is TBI then allocate resource on demand and copy
+	 * SFs to it
+	 */
+	else if (RGXTBIBufferIsInitRequired(psDevInfo))
+	{
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+	}
+
+	/* TBI buffer address will be 0 if not initialised */
+	sLogTypeUpdateCmd.uCmdData.sTBIBuffer = psDevInfo->sRGXFWIfTBIBuffer;
+#else
+	sLogTypeUpdateCmd.uCmdData.sTBIBuffer.ui32Addr = 0;
+#endif
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate resource on-demand. Reverting to old value",
+		         __func__));
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = ui32OldRGXFWLogTpe;
+		OSMemoryBarrier(&psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType);
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+
+		OSLockRelease(psDevInfo->hRGXFWIfBufInitLock);
+
+		return eError;
+	}
+
+	OSLockRelease(psDevInfo->hRGXFWIfBufInitLock);
+
+	eError = PVRSRVPowerLock((PPVRSRV_DEVICE_NODE) psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire power lock (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState((PPVRSRV_DEVICE_NODE) psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of logType value */
+		sLogTypeUpdateCmd.eCmdType = RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sLogTypeUpdateCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock( (PPVRSRV_DEVICE_NODE) psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the LogType value to be updated in FW */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetHCSDeadlineKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32HCSDeadlineMS)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWSetHCSDeadline(psDevInfo, ui32HCSDeadlineMS);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugMapGuestHeapKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32DriverID,
+	IMG_UINT64 ui64GuestHeapBase)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32DeviceID = psDeviceNode->sDevId.ui32InternalID;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		if (ui64GuestHeapBase == IMG_UINT64_MAX)
+		{
+			/* unmap heap and set DriverID to offline */
+			eError = PvzServerUnmapDevPhysHeap(ui32DriverID, ui32DeviceID);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PvzServerUnmapDevPhysHeap()");
+			eError = PvzServerOnVmOffline(ui32DriverID, ui32DeviceID);
+		}
+		else
+		{
+			/* set DriverID online if necessary and map firmware heap */
+			if (!IsVmOnline(ui32DriverID, ui32DeviceID))
+			{
+				eError = PvzServerOnVmOnline(ui32DriverID, ui32DeviceID);
+				PVR_LOG_RETURN_IF_ERROR(eError, "PvzServerOnVmOnline()");
+			}
+
+			eError = PvzServerMapDevPhysHeap(ui32DriverID, ui32DeviceID, RGX_FIRMWARE_RAW_HEAP_SIZE, ui64GuestHeapBase);
+		}
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		PVR_DPF((PVR_DBG_ERROR, " %s: Driver must run in Host mode to support Guest Mapping operations\n", __func__));
+	}
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32DriverID);
+	PVR_UNREFERENCED_PARAMETER(ui64GuestHeapBase);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32TSIntervalMs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzTSIntervalCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	sVzTSIntervalCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs = ui32TSIntervalMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32TSIntervalMs);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the timeslice interval inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32TSIntervalMs),
+							  ui32TSIntervalMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzTSIntervalCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32TSPercentage)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sVzTimeSliceCmd = { 0 };
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg;
+	IMG_INT32 ui32TSPercentageMax = 0;
+	IMG_UINT32 ui32DriverIDLoop;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_NOT_INITIALISED);
+
+	psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVR_RETURN_IF_FALSE(psRuntimeCfg != NULL, PVRSRV_ERROR_NOT_INITIALISED);
+
+	/*
+	 * Each time slice is a number between 0 -> 100.
+	 * Use '0' to disable time slice based CSW for the driver.
+	 */
+	 /* Check if the sum exceeds PVRSRV_VZ_TIME_SLICE_MAX */
+	if (ui32TSPercentage)
+	{
+		PVR_RETURN_IF_FALSE(ui32TSPercentage <= PVRSRV_VZ_TIME_SLICE_MAX, PVRSRV_ERROR_INVALID_PARAMS);
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverIDLoop)
+		{
+			if (ui32DriverID != ui32DriverIDLoop)
+			{
+				ui32TSPercentageMax += psRuntimeCfg->aui32TSPercentage[ui32DriverIDLoop];
+			}
+			else
+			{
+				ui32TSPercentageMax += ui32TSPercentage;
+			}
+
+			PVR_RETURN_IF_FALSE(ui32TSPercentageMax <= PVRSRV_VZ_TIME_SLICE_MAX, PVRSRV_ERROR_INVALID_PARAMS);
+		}
+	}
+
+	sVzTimeSliceCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE;
+	psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID] = ui32TSPercentage;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->aui32TSPercentage[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the timeslice of DriverID %u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, aui32TSPercentage) + (ui32DriverID * sizeof(ui32TSPercentage)),
+							  ui32TSPercentage,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzTimeSliceCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverPriorityKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_INT32   i32DriverPriority)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzPriorityCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((i32DriverPriority & ~RGXFW_VZ_PRIORITY_MASK) != 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sVzPriorityCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE;
+	psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID] = i32DriverPriority;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ai32DriverPriority[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the priority of DriverID %u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ai32DriverPriority) + (ui32DriverID * sizeof(i32DriverPriority)),
+							  i32DriverPriority,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sVzPriorityCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverIsolationGroupKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32DriverIsolationGroup)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sVzIsolationGroupCmd = { 0 };
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	if (psDevInfo->psRGXFWIfRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sVzIsolationGroupCmd.eCmdType = RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE;
+	psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = ui32DriverIsolationGroup;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID], FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the isolation group of DriverID%u inside RGXFWIfRuntimeCfg", ui32DriverID);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, aui32DriverIsolationGroup) + (ui32DriverID * sizeof(ui32DriverIsolationGroup)),
+							  ui32DriverIsolationGroup,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+				RGXFWIF_DM_GP,
+				&sVzIsolationGroupCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetOSNewOnlineStateKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32OSNewState)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_OS_STATE_CHANGE eState;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eState = (ui32OSNewState) ? (RGXFWIF_OS_ONLINE) : (RGXFWIF_OS_OFFLINE);
+	return RGXFWSetFwOsState(psDevInfo, ui32DriverID, eState);
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32DriverID);
+	PVR_UNREFERENCED_PARAMETER(ui32OSNewState);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPHRConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWConfigPHR(psDevInfo,
+	                      ui32PHRMode);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugWdgConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWConfigWdg(psDevInfo,
+	                      ui32WdgPeriodUs);
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugDumpFreelistPageListKM(
+	CONNECTION_DATA * psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (dllist_is_empty(&psDevInfo->sFreeListHead))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_LOG(("---------------[ Begin Freelist Page List Dump ]------------------"));
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		RGXDumpFreeListPageList(psFreeList);
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	PVR_LOG(("----------------[ End Freelist Page List Dump ]-------------------"));
+
+	return PVRSRV_OK;
+
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInjectFaultKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWInjectFault(psDevInfo);
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOffKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+#if defined(SUPPORT_AUTOVZ)
+	psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+#endif
+
+	return PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+					       PVRSRV_SYS_POWER_STATE_OFF,
+					       PVRSRV_POWER_FLAGS_NONE);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOnKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(ENABLE_PVRDEBUG_PRIVILEGED_CMDS)
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return PVRSRVSetDeviceSystemPowerState(psDeviceNode,
+					       PVRSRV_SYS_POWER_STATE_ON,
+					       PVRSRV_POWER_FLAGS_NONE);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	return RGXFWSetVzConnectionCooldownPeriod(psDevInfo, ui32VzConnectionCooldownPeriodInSec);
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxfwdbg.h b/drivers/gpu/drm/img-rogue/rgxfwdbg.h
new file mode 100644
index 000000000000..59c48ed5f4f8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwdbg.h
@@ -0,0 +1,160 @@
+/*************************************************************************/ /*!
+@File
+@Title          Debugging and miscellaneous functions server interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel services functions for debugging and other
+                miscellaneous functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXFWDBG_H)
+#define RGXFWDBG_H
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "device.h"
+#include "pmr.h"
+
+#include "connection_server.h"
+
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInitFWImageKM(
+	PMR *psFWImgDestPMR,
+	PMR *psFWImgSrcPMR,
+	IMG_UINT64 ui64FWImgLen,
+	PMR *psFWImgSigPMR,
+	IMG_UINT64 ui64FWSigLen);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugQueryFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 *pui32RGXFWLogType);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetFWLogKM(
+	const CONNECTION_DATA *psConnection,
+	const PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32RGXFWLogType);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetHCSDeadlineKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32HCSDeadlineMS);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32TSPercentage);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32TSIntervalMs);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverPriorityKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_INT32   i32DriverPriority);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetDriverIsolationGroupKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32DriverIsolationGroup);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetOSNewOnlineStateKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32DriverID,
+	IMG_UINT32  ui32OSNewState);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugMapGuestHeapKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32DriverID,
+	IMG_UINT64 ui64GuestHeapBase);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPHRConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32PHRMode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugWdgConfigureKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32 ui32WdgPeriodUs);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugDumpFreelistPageListKM(
+	CONNECTION_DATA * psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugInjectFaultKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOffKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugPowerOnKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR
+PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(
+	CONNECTION_DATA *psConnection,
+	PVRSRV_DEVICE_NODE *psDeviceNode,
+	IMG_UINT32  ui32VzConnectionCooldownPeriodInSec);
+#endif
diff --git a/drivers/gpu/drm/img-rogue/rgxfwimageutils.c b/drivers/gpu/drm/img-rogue/rgxfwimageutils.c
new file mode 100644
index 000000000000..afc69f7228f8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwimageutils.c
@@ -0,0 +1,1165 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services Firmware image utilities used at init time
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Services Firmware image utilities used at init time
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxfwimageutils.h"
+#include "pvrversion.h"
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+#include "rgxfwutils.h"
+#endif
+
+/************************************************************************
+* FW layout information
+************************************************************************/
+#define MAX_NUM_ENTRIES (8)
+static RGX_FW_LAYOUT_ENTRY asRGXFWLayoutTable[MAX_NUM_ENTRIES];
+static IMG_UINT32 ui32LayoutEntryNum;
+
+
+static RGX_FW_LAYOUT_ENTRY* GetTableEntry(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		if (asRGXFWLayoutTable[i].eId == eId)
+		{
+			return &asRGXFWLayoutTable[i];
+		}
+	}
+
+	RGXErrorLog(hPrivate, "%s: id %u not found, returning entry 0\n",
+	            __func__, eId);
+
+	return &asRGXFWLayoutTable[0];
+}
+
+/*!
+*******************************************************************************
+
+ @Function      FindMMUSegment
+
+ @Description   Given a 32 bit FW address attempt to find the corresponding
+                pointer to FW allocation
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         ui32OffsetIn             : 32 bit FW address
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem code
+ @Input         uiHostAddrOut            : CPU pointer equivalent to ui32OffsetIn
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR FindMMUSegment(const void *hPrivate,
+                                   IMG_UINT32 ui32OffsetIn,
+                                   void *pvHostFWCodeAddr,
+                                   void *pvHostFWDataAddr,
+                                   void *pvHostFWCorememCodeAddr,
+                                   void *pvHostFWCorememDataAddr,
+                                   void **uiHostAddrOut)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		if ((ui32OffsetIn >= asRGXFWLayoutTable[i].ui32BaseAddr) &&
+		    (ui32OffsetIn < (asRGXFWLayoutTable[i].ui32BaseAddr + asRGXFWLayoutTable[i].ui32AllocSize)))
+		{
+			switch (asRGXFWLayoutTable[i].eType)
+			{
+				case FW_CODE:
+					*uiHostAddrOut = pvHostFWCodeAddr;
+					break;
+
+				case FW_DATA:
+					if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+					{
+						*uiHostAddrOut = RGXCalculateHostFWDataAddress(hPrivate, pvHostFWDataAddr);
+					}
+					else
+					{
+						*uiHostAddrOut = pvHostFWDataAddr;
+					}
+					break;
+
+				case FW_COREMEM_CODE:
+					*uiHostAddrOut = pvHostFWCorememCodeAddr;
+					break;
+
+				case FW_COREMEM_DATA:
+					*uiHostAddrOut = pvHostFWCorememDataAddr;
+					break;
+
+				default:
+					return PVRSRV_ERROR_INIT_FAILURE;
+			}
+
+			goto found;
+		}
+	}
+
+	return PVRSRV_ERROR_INIT_FAILURE;
+
+found:
+	if (*uiHostAddrOut == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Add offset to pointer to FW allocation now that allocation is found */
+
+	/* Direct Mem write to mapped memory */
+	ui32OffsetIn -= asRGXFWLayoutTable[i].ui32BaseAddr;
+	ui32OffsetIn += asRGXFWLayoutTable[i].ui32AllocOffset;
+
+	*(IMG_UINT8 **)uiHostAddrOut += ui32OffsetIn;
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureSegID
+
+ @Description   Configures a single segment of the Segment MMU
+                (base, limit and out_addr)
+
+ @Input         hPrivate        : Implementation specific data
+ @Input         ui64SegOutAddr  : Segment output base address (40 bit devVaddr)
+ @Input         ui32SegBase     : Segment input base address (32 bit FW address)
+ @Input         ui32SegLimit    : Segment size
+ @Input         ui32SegID       : Segment ID
+ @Input         pszName         : Segment name
+ @Input         ppui32BootConf  : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureSegID(const void *hPrivate,
+                                IMG_UINT64 ui64SegOutAddr,
+                                IMG_UINT32 ui32SegBase,
+                                IMG_UINT32 ui32SegLimit,
+                                IMG_UINT32 ui32SegID,
+                                IMG_UINT32 **ppui32BootConf)
+{
+	IMG_UINT32 *pui32BootConf = *ppui32BootConf;
+	IMG_UINT32 ui32SegOutAddr0 = ui64SegOutAddr & 0x00000000FFFFFFFFUL;
+	IMG_UINT32 ui32SegOutAddr1 = (ui64SegOutAddr >> 32) & 0x00000000FFFFFFFFUL;
+
+	/* META segments have a minimum size */
+	IMG_UINT32 ui32LimitOff = (ui32SegLimit < RGXFW_SEGMMU_ALIGN) ?
+	                          RGXFW_SEGMMU_ALIGN : ui32SegLimit;
+	/* the limit is an offset, therefore off = size - 1 */
+	ui32LimitOff -= 1;
+
+	RGXCommentLog(hPrivate,
+	              "* Seg%d: meta_addr = 0x%08x, devv_addr = 0x%" IMG_UINT64_FMTSPECx ", limit = 0x%x",
+	              ui32SegID,
+	              ui32SegBase,
+	              ui64SegOutAddr,
+	              ui32LimitOff);
+
+	ui32SegBase |= RGXFW_SEGMMU_ALLTHRS_WRITEABLE;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_BASE(ui32SegID);
+	*pui32BootConf++ = ui32SegBase;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_LIMIT(ui32SegID);
+	*pui32BootConf++ = ui32LimitOff;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_OUTA0(ui32SegID);
+	*pui32BootConf++ = ui32SegOutAddr0;
+
+	*pui32BootConf++ = META_CR_MMCU_SEGMENTn_OUTA1(ui32SegID);
+	*pui32BootConf++ = ui32SegOutAddr1;
+
+	*ppui32BootConf = pui32BootConf;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureSegMMU
+
+ @Description   Configures META's Segment MMU
+
+ @Input         hPrivate             : Implementation specific data
+ @Input         psFWCodeDevVAddrBase : FW code base device virtual address
+ @Input         psFWDataDevVAddrBase : FW data base device virtual address
+ @Input         ppui32BootConf       : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureSegMMU(const void       *hPrivate,
+                                 IMG_DEV_VIRTADDR *psFWCodeDevVAddrBase,
+                                 IMG_DEV_VIRTADDR *psFWDataDevVAddrBase,
+                                 IMG_UINT32       **ppui32BootConf)
+{
+	IMG_UINT64 ui64SegOutAddrTop;
+	IMG_UINT32 i;
+
+	PVR_UNREFERENCED_PARAMETER(psFWCodeDevVAddrBase);
+
+	/* Configure Segment MMU */
+	RGXCommentLog(hPrivate, "********** FW configure Segment MMU **********");
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SLC_VIVT))
+	{
+		ui64SegOutAddrTop = RGXFW_SEGMMU_OUTADDR_TOP_VIVT_SLC_CACHED(MMU_CONTEXT_MAPPING_FWPRIV);
+	}
+	else
+#endif
+	{
+		ui64SegOutAddrTop = RGXFW_SEGMMU_OUTADDR_TOP_SLC(MMU_CONTEXT_MAPPING_FWPRIV, RGXFW_SEGMMU_META_BIFDM_ID);
+	}
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		/*
+		 * FW code is using the bootloader segment which is already configured on boot.
+		 * FW coremem code and data don't use the segment MMU.
+		 * Only the FW data segment needs to be configured.
+		 */
+
+		if (asRGXFWLayoutTable[i].eType == FW_DATA)
+		{
+			IMG_UINT64 ui64SegOutAddr;
+			IMG_UINT32 ui32SegId = RGXFW_SEGMMU_DATA_ID;
+
+			ui64SegOutAddr = (psFWDataDevVAddrBase->uiAddr | ui64SegOutAddrTop) +
+			                  asRGXFWLayoutTable[i].ui32AllocOffset;
+
+			RGXFWConfigureSegID(hPrivate,
+			                    ui64SegOutAddr,
+			                    asRGXFWLayoutTable[i].ui32BaseAddr,
+			                    asRGXFWLayoutTable[i].ui32AllocSize,
+			                    ui32SegId,
+			                    ppui32BootConf); /*write the sequence to the bootldr */
+
+			break;
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXFWConfigureMetaCaches
+
+ @Description   Configure and enable the Meta instruction and data caches
+
+ @Input         hPrivate          : Implementation specific data
+ @Input         ui32NumThreads    : Number of FW threads in use
+ @Input         ppui32BootConf    : Pointer to bootloader data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXFWConfigureMetaCaches(const void *hPrivate,
+                                     IMG_UINT32 ui32NumThreads,
+                                     IMG_UINT32 **ppui32BootConf)
+{
+	IMG_UINT32 *pui32BootConf = *ppui32BootConf;
+	IMG_UINT32 ui32DCacheT0, ui32ICacheT0;
+	IMG_UINT32 ui32DCacheT1, ui32ICacheT1;
+	IMG_UINT32 ui32DCacheT2, ui32ICacheT2;
+	IMG_UINT32 ui32DCacheT3, ui32ICacheT3;
+
+#define META_CR_MMCU_LOCAL_EBCTRL                        (0x04830600)
+#define META_CR_MMCU_LOCAL_EBCTRL_ICWIN                  (0x3 << 14)
+#define META_CR_MMCU_LOCAL_EBCTRL_DCWIN                  (0x3 << 6)
+#define META_CR_SYSC_DCPART(n)                           (0x04830200 + (n)*0x8)
+#define META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE         (0x1 << 31)
+#define META_CR_SYSC_ICPART(n)                           (0x04830220 + (n)*0x8)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF  (0x8 << 16)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE       (0xF)
+#define META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE       (0x7)
+#define META_CR_MMCU_DCACHE_CTRL                         (0x04830018)
+#define META_CR_MMCU_ICACHE_CTRL                         (0x04830020)
+#define META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN           (0x1)
+
+	RGXCommentLog(hPrivate, "********** Meta caches configuration *********");
+
+	/* Initialise I/Dcache settings */
+	ui32DCacheT0 = ui32DCacheT1 = (IMG_UINT32)META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE;
+	ui32DCacheT2 = ui32DCacheT3 = (IMG_UINT32)META_CR_SYSC_DCPARTX_CACHED_WRITE_ENABLE;
+	ui32ICacheT0 = ui32ICacheT1 = ui32ICacheT2 = ui32ICacheT3 = 0;
+
+	if (ui32NumThreads == 1)
+	{
+		ui32DCacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE;
+		ui32ICacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_FULL_CACHE;
+	}
+	else
+	{
+		ui32DCacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE;
+		ui32ICacheT0 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE;
+
+		ui32DCacheT1 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE |
+		                META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF;
+		ui32ICacheT1 |= META_CR_SYSC_XCPARTX_LOCAL_ADDR_HALF_CACHE |
+		                META_CR_SYSC_XCPARTX_LOCAL_ADDR_OFFSET_TOP_HALF;
+	}
+
+	/* Local region MMU enhanced bypass: WIN-3 mode for code and data caches */
+	*pui32BootConf++ = META_CR_MMCU_LOCAL_EBCTRL;
+	*pui32BootConf++ = META_CR_MMCU_LOCAL_EBCTRL_ICWIN |
+	                   META_CR_MMCU_LOCAL_EBCTRL_DCWIN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_LOCAL_EBCTRL,
+	              META_CR_MMCU_LOCAL_EBCTRL_ICWIN | META_CR_MMCU_LOCAL_EBCTRL_DCWIN);
+
+	/* Data cache partitioning thread 0 to 3 */
+	*pui32BootConf++ = META_CR_SYSC_DCPART(0);
+	*pui32BootConf++ = ui32DCacheT0;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(1);
+	*pui32BootConf++ = ui32DCacheT1;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(2);
+	*pui32BootConf++ = ui32DCacheT2;
+	*pui32BootConf++ = META_CR_SYSC_DCPART(3);
+	*pui32BootConf++ = ui32DCacheT3;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(0), ui32DCacheT0);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(1), ui32DCacheT1);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(2), ui32DCacheT2);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_DCPART(3), ui32DCacheT3);
+
+	/* Enable data cache hits */
+	*pui32BootConf++ = META_CR_MMCU_DCACHE_CTRL;
+	*pui32BootConf++ = META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_DCACHE_CTRL,
+	              META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN);
+
+	/* Instruction cache partitioning thread 0 to 3 */
+	*pui32BootConf++ = META_CR_SYSC_ICPART(0);
+	*pui32BootConf++ = ui32ICacheT0;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(1);
+	*pui32BootConf++ = ui32ICacheT1;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(2);
+	*pui32BootConf++ = ui32ICacheT2;
+	*pui32BootConf++ = META_CR_SYSC_ICPART(3);
+	*pui32BootConf++ = ui32ICacheT3;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(0), ui32ICacheT0);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(1), ui32ICacheT1);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(2), ui32ICacheT2);
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_SYSC_ICPART(3), ui32ICacheT3);
+
+	/* Enable instruction cache hits */
+	*pui32BootConf++ = META_CR_MMCU_ICACHE_CTRL;
+	*pui32BootConf++ = META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+	              META_CR_MMCU_ICACHE_CTRL,
+	              META_CR_MMCU_XCACHE_CTRL_CACHE_HITS_EN);
+
+	*pui32BootConf++ = 0x040000C0;
+	*pui32BootConf++ = 0;
+
+	RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x", 0x040000C0, 0);
+
+	*ppui32BootConf = pui32BootConf;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessLDRCommandStream
+
+ @Description   Process the output of the Meta toolchain in the .LDR format
+                copying code and data sections into their final location and
+                passing some information to the Meta bootloader
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbLDR                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+ @Input         ppui32BootConf           : Pointer to bootloader data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessLDRCommandStream(const void *hPrivate,
+                                     const IMG_BYTE* pbLDR,
+                                     void* pvHostFWCodeAddr,
+                                     void* pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr,
+                                     IMG_UINT32 **ppui32BootConf)
+{
+	RGX_META_LDR_BLOCK_HDR *psHeader = (RGX_META_LDR_BLOCK_HDR *) pbLDR;
+	RGX_META_LDR_L1_DATA_BLK *psL1Data =
+	    (RGX_META_LDR_L1_DATA_BLK*) ((IMG_UINT8 *) pbLDR + psHeader->ui32SLData);
+
+	IMG_UINT32 *pui32BootConf  = ppui32BootConf ? *ppui32BootConf : NULL;
+	IMG_UINT32 ui32CorememSize = RGXGetFWCorememSize(hPrivate);
+
+	RGXCommentLog(hPrivate, "**********************************************");
+	RGXCommentLog(hPrivate, "************** Begin LDR Parsing *************");
+	RGXCommentLog(hPrivate, "**********************************************");
+
+	while (psL1Data != NULL)
+	{
+		if (RGX_META_LDR_BLK_IS_COMMENT(psL1Data->ui16Cmd))
+		{
+			/* Don't process comment blocks */
+			goto NextBlock;
+		}
+
+		switch (psL1Data->ui16Cmd & RGX_META_LDR_CMD_MASK)
+		{
+			case RGX_META_LDR_CMD_LOADMEM:
+			{
+				RGX_META_LDR_L2_DATA_BLK *psL2Block =
+				    (RGX_META_LDR_L2_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->aui32CmdData[1]);
+				IMG_UINT32 ui32Offset = psL1Data->aui32CmdData[0];
+				IMG_UINT32 ui32DataSize = psL2Block->ui16Length - 6 /* L2 Tag length and checksum */;
+				void *pvWriteAddr;
+				PVRSRV_ERROR eError;
+
+				if (!RGX_META_IS_COREMEM_CODE(ui32Offset, ui32CorememSize) &&
+				    !RGX_META_IS_COREMEM_DATA(ui32Offset, ui32CorememSize))
+				{
+					/* Global range is aliased to local range */
+					ui32Offset &= ~META_MEM_GLOBAL_RANGE_BIT;
+				}
+
+				eError = FindMMUSegment(hPrivate,
+				                        ui32Offset,
+				                        pvHostFWCodeAddr,
+				                        pvHostFWDataAddr,
+				                        pvHostFWCorememCodeAddr,
+				                        pvHostFWCorememDataAddr,
+				                        &pvWriteAddr);
+
+				if (eError != PVRSRV_OK)
+				{
+					RGXErrorLog(hPrivate,
+					            "ProcessLDRCommandStream: Addr 0x%x (size: %d) not found in any segment",
+					            ui32Offset, ui32DataSize);
+					return eError;
+				}
+
+				/* Write to FW allocation only if available */
+				if (pvWriteAddr)
+				{
+					RGXMemCopy(hPrivate,
+					           pvWriteAddr,
+					           psL2Block->aui32BlockData,
+					           ui32DataSize);
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+					RGXFwSharedMemCacheOpExec(pvWriteAddr, ui32DataSize, PVRSRV_CACHE_OP_FLUSH);
+#endif
+				}
+
+				break;
+			}
+			case RGX_META_LDR_CMD_LOADCORE:
+			case RGX_META_LDR_CMD_LOADMMREG:
+			{
+				return PVRSRV_ERROR_INIT_FAILURE;
+			}
+			case RGX_META_LDR_CMD_START_THREADS:
+			{
+				/* Don't process this block */
+				break;
+			}
+			case RGX_META_LDR_CMD_ZEROMEM:
+			{
+				IMG_UINT32 ui32Offset = psL1Data->aui32CmdData[0];
+				IMG_UINT32 ui32ByteCount = psL1Data->aui32CmdData[1];
+				void *pvWriteAddr;
+				PVRSRV_ERROR  eError;
+
+				if (RGX_META_IS_COREMEM_DATA(ui32Offset, ui32CorememSize))
+				{
+					/* cannot zero coremem directly */
+					break;
+				}
+
+				/* Global range is aliased to local range */
+				ui32Offset &= ~META_MEM_GLOBAL_RANGE_BIT;
+
+				eError = FindMMUSegment(hPrivate,
+				                        ui32Offset,
+				                        pvHostFWCodeAddr,
+				                        pvHostFWDataAddr,
+				                        pvHostFWCorememCodeAddr,
+				                        pvHostFWCorememDataAddr,
+				                        &pvWriteAddr);
+
+				if (eError != PVRSRV_OK)
+				{
+					RGXErrorLog(hPrivate,
+					            "ProcessLDRCommandStream: Addr 0x%x (size: %d) not found in any segment",
+					            ui32Offset, ui32ByteCount);
+					return eError;
+				}
+
+				/* Write to FW allocation only if available */
+				if (pvWriteAddr)
+				{
+					RGXMemSet(hPrivate, pvWriteAddr, 0, ui32ByteCount);
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+					RGXFwSharedMemCacheOpExec(pvWriteAddr, ui32ByteCount, PVRSRV_CACHE_OP_FLUSH);
+#endif
+				}
+
+				break;
+			}
+			case RGX_META_LDR_CMD_CONFIG:
+			{
+				RGX_META_LDR_L2_DATA_BLK *psL2Block =
+				    (RGX_META_LDR_L2_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->aui32CmdData[0]);
+				RGX_META_LDR_CFG_BLK *psConfigCommand = (RGX_META_LDR_CFG_BLK*) psL2Block->aui32BlockData;
+				IMG_UINT32 ui32L2BlockSize = psL2Block->ui16Length - 6 /* L2 Tag length and checksum */;
+				IMG_UINT32 ui32CurrBlockSize = 0;
+
+				while (ui32L2BlockSize)
+				{
+					switch (psConfigCommand->ui32Type)
+					{
+						case RGX_META_LDR_CFG_PAUSE:
+						case RGX_META_LDR_CFG_READ:
+						{
+							ui32CurrBlockSize = 8;
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+						case RGX_META_LDR_CFG_WRITE:
+						{
+							IMG_UINT32 ui32RegisterOffset = psConfigCommand->aui32BlockData[0];
+							IMG_UINT32 ui32RegisterValue  = psConfigCommand->aui32BlockData[1];
+
+							/* Only write to bootloader if we got a valid
+							 * pointer to the FW code allocation
+							 */
+							if (pui32BootConf)
+							{
+								/* Do register write */
+								*pui32BootConf++ = ui32RegisterOffset;
+								*pui32BootConf++ = ui32RegisterValue;
+							}
+
+							RGXCommentLog(hPrivate, "Meta SP: [0x%08x] = 0x%08x",
+							              ui32RegisterOffset, ui32RegisterValue);
+
+							ui32CurrBlockSize = 12;
+							break;
+						}
+						case RGX_META_LDR_CFG_MEMSET:
+						case RGX_META_LDR_CFG_MEMCHECK:
+						{
+							ui32CurrBlockSize = 20;
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+						default:
+						{
+							return PVRSRV_ERROR_INIT_FAILURE;
+						}
+					}
+					ui32L2BlockSize -= ui32CurrBlockSize;
+					psConfigCommand = (RGX_META_LDR_CFG_BLK*) (((IMG_UINT8*) psConfigCommand) + ui32CurrBlockSize);
+				}
+
+				break;
+			}
+			default:
+			{
+				return PVRSRV_ERROR_INIT_FAILURE;
+			}
+		}
+
+NextBlock:
+
+		if (psL1Data->ui32Next == 0xFFFFFFFF)
+		{
+			psL1Data = NULL;
+		}
+		else
+		{
+			psL1Data = (RGX_META_LDR_L1_DATA_BLK*) (((IMG_UINT8 *) pbLDR) + psL1Data->ui32Next);
+		}
+	}
+
+	if (pui32BootConf)
+	{
+		*ppui32BootConf = pui32BootConf;
+	}
+
+	RGXCommentLog(hPrivate, "**********************************************");
+	RGXCommentLog(hPrivate, "************** End Loader Parsing ************");
+	RGXCommentLog(hPrivate, "**********************************************");
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessELFCommandStream
+
+ @Description   Process a file in .ELF format copying code and data sections
+                into their final location
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbELF                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessELFCommandStream(const void *hPrivate,
+                                     const IMG_BYTE *pbELF,
+                                     void *pvHostFWCodeAddr,
+                                     void *pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr)
+{
+	IMG_UINT32 ui32Entry;
+	IMG_ELF_HDR *psHeader = (IMG_ELF_HDR *)pbELF;
+	IMG_ELF_PROGRAM_HDR *psProgramHeader =
+	    (IMG_ELF_PROGRAM_HDR *)(pbELF + psHeader->ui32Ephoff);
+	PVRSRV_ERROR eError;
+
+	for (ui32Entry = 0; ui32Entry < psHeader->ui32Ephnum; ui32Entry++, psProgramHeader++)
+	{
+		void *pvWriteAddr;
+
+		/* Only consider loadable entries in the ELF segment table */
+		if (psProgramHeader->ui32Ptype != ELF_PT_LOAD) continue;
+
+		eError = FindMMUSegment(hPrivate,
+		                        psProgramHeader->ui32Pvaddr,
+		                        pvHostFWCodeAddr,
+		                        pvHostFWDataAddr,
+		                        pvHostFWCorememCodeAddr,
+		                        pvHostFWCorememDataAddr,
+		                        &pvWriteAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate,
+			            "%s: Addr 0x%x (size: %d) not found in any segment",__func__,
+			            psProgramHeader->ui32Pvaddr,
+			            psProgramHeader->ui32Pfilesz);
+			return eError;
+		}
+
+		/* Write to FW allocation only if available */
+		if (pvWriteAddr)
+		{
+			RGXMemCopy(hPrivate,
+			           pvWriteAddr,
+			           (IMG_PBYTE)(pbELF + psProgramHeader->ui32Poffset),
+			           psProgramHeader->ui32Pfilesz);
+
+			RGXMemSet(hPrivate,
+			          (IMG_PBYTE)pvWriteAddr + psProgramHeader->ui32Pfilesz,
+			          0,
+			          psProgramHeader->ui32Pmemsz - psProgramHeader->ui32Pfilesz);
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+			RGXFwSharedMemCacheOpExec(pvWriteAddr, psProgramHeader->ui32Pmemsz, PVRSRV_CACHE_OP_FLUSH);
+#endif
+		}
+	}
+
+	return PVRSRV_OK;
+}
+
+IMG_UINT32 RGXGetFWImageSectionOffset(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32AllocOffset;
+}
+
+IMG_UINT32 RGXGetFWImageSectionMaxSize(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32MaxSize;
+}
+
+IMG_UINT32 RGXGetFWImageSectionAllocSize(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32AllocSize;
+}
+
+IMG_UINT32 RGXGetFWImageSectionAddress(const void *hPrivate, RGX_FW_SECTION_ID eId)
+{
+	RGX_FW_LAYOUT_ENTRY *psEntry = GetTableEntry(hPrivate, eId);
+
+	return psEntry->ui32BaseAddr;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion1(const void *hPrivate,
+                                         const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	/* Applicable to any FW_INFO_VERSION */
+	if (psInfoHeader->ui32LayoutEntrySize != sizeof(RGX_FW_LAYOUT_ENTRY))
+	{
+		RGXErrorLog(hPrivate, "%s: FW layout entry sizes mismatch (expected: %u, found: %u)",
+		            __func__,
+		            (IMG_UINT32) sizeof(RGX_FW_LAYOUT_ENTRY),
+		            psInfoHeader->ui32LayoutEntrySize);
+	}
+
+	/* Applicable to any FW_INFO_VERSION */
+	if (psInfoHeader->ui32LayoutEntryNum > MAX_NUM_ENTRIES)
+	{
+		RGXErrorLog(hPrivate, "%s: Not enough storage for the FW layout table (max: %u entries, found: %u)",
+		            __func__,
+		            MAX_NUM_ENTRIES,
+		            psInfoHeader->ui32LayoutEntryNum);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Applicable to any FW_INFO_VERSION */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		if (psInfoHeader->ui32FwPageSize != RGXGetOSPageSize(hPrivate))
+		{
+			RGXErrorLog(hPrivate, "%s: FW page size mismatch (expected: %u, found: %u)",
+			            __func__,
+			            (IMG_UINT32) RGXGetOSPageSize(hPrivate),
+			            psInfoHeader->ui32FwPageSize);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	if (psInfoHeader->ui32InfoVersion != FW_INFO_VERSION)
+	{
+		/* Not an error because RGX_FW_INFO_HEADER is now versioned. It can grow
+		 * incrementally and it must be backwards compatible.
+		 */
+		RGXCommentLog(hPrivate, "%s: FW info version mismatch (expected: %u, found: %u)",
+		              __func__,
+		              (IMG_UINT32) FW_INFO_VERSION,
+		              psInfoHeader->ui32InfoVersion);
+		goto exit_version1_validation;
+	}
+
+	if (psInfoHeader->ui32HeaderLen != sizeof(RGX_FW_INFO_HEADER))
+	{
+		RGXErrorLog(hPrivate, "%s: FW info header sizes mismatch (expected: %u, found: %u)",
+		            __func__,
+		            (IMG_UINT32) sizeof(RGX_FW_INFO_HEADER),
+		            psInfoHeader->ui32HeaderLen);
+	}
+
+exit_version1_validation:
+	return PVRSRV_OK;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion2(const void *hPrivate,
+                                         const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	if (psInfoHeader->ui16PVRVersionMajor != PVRVERSION_MAJ ||
+	    psInfoHeader->ui16PVRVersionMinor != PVRVERSION_MIN)
+	{
+		RGXErrorLog(hPrivate, "%s: KM and FW version mismatch (expected: %u.%u, found: %u.%u)",
+		            __func__,
+		            PVRVERSION_MAJ,
+		            PVRVERSION_MIN,
+		            psInfoHeader->ui16PVRVersionMajor,
+		            psInfoHeader->ui16PVRVersionMinor);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+static inline
+PVRSRV_ERROR RGXValidateFWHeaderVersion(const void *hPrivate,
+                                        const RGX_FW_INFO_HEADER *psInfoHeader)
+{
+	PVRSRV_ERROR eError;
+
+	switch (psInfoHeader->ui32InfoVersion)
+	{
+		default:
+			__fallthrough;
+		case 2:
+			eError = RGXValidateFWHeaderVersion2(hPrivate, psInfoHeader);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			__fallthrough;
+		case 1:
+			eError = RGXValidateFWHeaderVersion1(hPrivate, psInfoHeader);
+			if (eError != PVRSRV_OK)
+			{
+				return eError;
+			}
+
+			break;
+		case 0:
+			RGXErrorLog(hPrivate, "%s: invalid FW_INFO_VERSION", __func__);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetFWImageAllocSize(const void *hPrivate,
+                                    const IMG_BYTE    *pbRGXFirmware,
+                                    const IMG_UINT32  ui32RGXFirmwareSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWDataAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememDataAllocSize,
+                                    RGX_FW_INFO_HEADER *psFWInfoHeader)
+{
+	RGX_FW_INFO_HEADER *psInfoHeader;
+	const IMG_BYTE *pbRGXFirmwareInfo;
+	const IMG_BYTE *pbRGXFirmwareLayout;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	if (pbRGXFirmware == NULL || ui32RGXFirmwareSize == 0 || ui32RGXFirmwareSize <= FW_BLOCK_SIZE)
+	{
+		RGXErrorLog(hPrivate, "%s: Invalid FW binary at %p, size %u",
+		            __func__, pbRGXFirmware, ui32RGXFirmwareSize);
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/*
+	 * Acquire pointer to the FW info header within the FW image.
+	 * The format of the header in the FW image might not be the one expected
+	 * by the driver, but the driver should still be able to correctly read
+	 * the information below, as long as new/incompatible elements are added
+	 * at the end of the header (they will be ignored by the driver).
+	 */
+
+	pbRGXFirmwareInfo = pbRGXFirmware + ui32RGXFirmwareSize - FW_BLOCK_SIZE;
+	psInfoHeader = (RGX_FW_INFO_HEADER*)pbRGXFirmwareInfo;
+
+	eError = RGXValidateFWHeaderVersion(hPrivate, psInfoHeader);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	ui32LayoutEntryNum = psInfoHeader->ui32LayoutEntryNum;
+
+
+	/*
+	 * Copy FW layout table from FW image to local array.
+	 * One entry is copied at a time and the copy is limited to what the driver
+	 * expects to find in it. Assuming that new/incompatible elements
+	 * are added at the end of each entry, the loop below adapts the table
+	 * in the FW image into the format expected by the driver.
+	 */
+
+	pbRGXFirmwareLayout = pbRGXFirmwareInfo + psInfoHeader->ui32HeaderLen;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		RGX_FW_LAYOUT_ENTRY *psOutEntry = &asRGXFWLayoutTable[i];
+
+		RGX_FW_LAYOUT_ENTRY *psInEntry = (RGX_FW_LAYOUT_ENTRY*)
+			(pbRGXFirmwareLayout + i * psInfoHeader->ui32LayoutEntrySize);
+
+		RGXMemCopy(hPrivate,
+		           (void*)psOutEntry,
+		           (void*)psInEntry,
+		           sizeof(RGX_FW_LAYOUT_ENTRY));
+	}
+
+
+	/* Calculate how much memory the FW needs for its code and data segments */
+
+	*puiFWCodeAllocSize = 0;
+	*puiFWDataAllocSize = 0;
+	*puiFWCorememCodeAllocSize = 0;
+	*puiFWCorememDataAllocSize = 0;
+
+	for (i = 0; i < ui32LayoutEntryNum; i++)
+	{
+		switch (asRGXFWLayoutTable[i].eType)
+		{
+			case FW_CODE:
+				*puiFWCodeAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_DATA:
+				*puiFWDataAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_COREMEM_CODE:
+				*puiFWCorememCodeAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			case FW_COREMEM_DATA:
+				*puiFWCorememDataAllocSize += asRGXFWLayoutTable[i].ui32AllocSize;
+				break;
+
+			default:
+				RGXErrorLog(hPrivate, "%s: Unknown FW section type %u\n",
+				            __func__, asRGXFWLayoutTable[i].eType);
+				break;
+		}
+	}
+
+	*psFWInfoHeader = *psInfoHeader;
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXProcessFWImage(const void *hPrivate,
+                               const IMG_BYTE *pbRGXFirmware,
+                               void *pvFWCode,
+                               void *pvFWData,
+                               void *pvFWCorememCode,
+                               void *pvFWCorememData,
+                               PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMIPS = IMG_FALSE;
+	IMG_BOOL bRISCV = RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR);
+	IMG_BOOL bMETA;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	bMIPS = (IMG_BOOL)RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS);
+#endif
+	bMETA = (IMG_BOOL)(!bMIPS && !bRISCV);
+
+	if (bMETA)
+	{
+		IMG_UINT32 *pui32BootConf = NULL;
+		/* Skip bootloader configuration if a pointer to the FW code
+		 * allocation is not available
+		 */
+		if (pvFWCode)
+		{
+			/* This variable points to the bootloader code which is mostly
+			 * a sequence of <register address,register value> pairs
+			 */
+			pui32BootConf = ((IMG_UINT32*) pvFWCode) + RGXFW_BOOTLDR_CONF_OFFSET;
+
+			/* Slave port and JTAG accesses are privileged */
+			*pui32BootConf++ = META_CR_SYSC_JTAG_THREAD;
+			*pui32BootConf++ = META_CR_SYSC_JTAG_THREAD_PRIV_EN;
+
+			RGXFWConfigureSegMMU(hPrivate,
+			                     &puFWParams->sMeta.sFWCodeDevVAddr,
+			                     &puFWParams->sMeta.sFWDataDevVAddr,
+			                     &pui32BootConf);
+		}
+
+		/* Process FW image data stream */
+		eError = ProcessLDRCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 pvFWCorememCode,
+		                                 pvFWCorememData,
+		                                 &pui32BootConf);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		/* Skip bootloader configuration if a pointer to the FW code
+		 * allocation is not available
+		 */
+		if (pvFWCode)
+		{
+			IMG_UINT32 ui32NumThreads   = puFWParams->sMeta.ui32NumThreads;
+
+			if ((ui32NumThreads == 0) || (ui32NumThreads > 2))
+			{
+				RGXErrorLog(hPrivate,
+				            "ProcessFWImage: Wrong Meta threads configuration, using one thread only");
+
+				ui32NumThreads = 1;
+			}
+
+			RGXFWConfigureMetaCaches(hPrivate,
+			                         ui32NumThreads,
+			                         &pui32BootConf);
+
+			/* Signal the end of the conf sequence */
+			*pui32BootConf++ = 0x0;
+			*pui32BootConf++ = 0x0;
+
+			if (puFWParams->sMeta.uiFWCorememCodeSize && (puFWParams->sMeta.sFWCorememCodeFWAddr.ui32Addr != 0))
+			{
+				*pui32BootConf++ = puFWParams->sMeta.sFWCorememCodeFWAddr.ui32Addr;
+				*pui32BootConf++ = puFWParams->sMeta.uiFWCorememCodeSize;
+			}
+			else
+			{
+				*pui32BootConf++ = 0;
+				*pui32BootConf++ = 0;
+			}
+
+#if defined(RGX_FEATURE_META_DMA_BIT_MASK)
+			if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_DMA))
+			{
+				*pui32BootConf++ = (IMG_UINT32) (puFWParams->sMeta.sFWCorememCodeDevVAddr.uiAddr >> 32);
+				*pui32BootConf++ = (IMG_UINT32) puFWParams->sMeta.sFWCorememCodeDevVAddr.uiAddr;
+			}
+			else
+#endif
+			{
+				*pui32BootConf++ = 0;
+				*pui32BootConf++ = 0;
+			}
+		}
+	}
+#if defined(RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES)
+	else if (bMIPS)
+	{
+		/* Process FW image data stream */
+		eError = ProcessELFCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 NULL,
+		                                 NULL);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		if (pvFWData)
+		{
+			RGXMIPSFW_BOOT_DATA *psBootData = (RGXMIPSFW_BOOT_DATA*)
+				/* To get a pointer to the bootloader configuration data start from a pointer to the FW image... */
+				IMG_OFFSET_ADDR(pvFWData,
+				/* ... jump to the boot/NMI data page... */
+				(RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA)
+				/* ... and then jump to the bootloader data offset within the page */
+				+ RGXMIPSFW_BOOTLDR_CONF_OFFSET));
+
+			/* Rogue Registers physical address */
+			psBootData->ui64RegBase = puFWParams->sMips.sGPURegAddr.uiAddr;
+
+			/* MIPS Page Table physical address */
+			psBootData->ui32PTLog2PageSize = puFWParams->sMips.ui32FWPageTableLog2PageSize;
+			psBootData->ui32PTNumPages     = puFWParams->sMips.ui32FWPageTableNumPages;
+			psBootData->aui64PTPhyAddr[0U] = puFWParams->sMips.asFWPageTableAddr[0U].uiAddr;
+			psBootData->aui64PTPhyAddr[1U] = puFWParams->sMips.asFWPageTableAddr[1U].uiAddr;
+			psBootData->aui64PTPhyAddr[2U] = puFWParams->sMips.asFWPageTableAddr[2U].uiAddr;
+			psBootData->aui64PTPhyAddr[3U] = puFWParams->sMips.asFWPageTableAddr[3U].uiAddr;
+
+			/* MIPS Stack Pointer Physical Address */
+			psBootData->ui64StackPhyAddr = puFWParams->sMips.sFWStackAddr.uiAddr;
+
+			/* Reserved for future use */
+			psBootData->ui32Reserved1 = 0;
+			psBootData->ui32Reserved2 = 0;
+		}
+	}
+#endif /* #if defined(RGXMIPSFW_MAX_NUM_PAGETABLE_PAGES) */
+	else
+	{
+		/* Process FW image data stream */
+		eError = ProcessELFCommandStream(hPrivate,
+		                                 pbRGXFirmware,
+		                                 pvFWCode,
+		                                 pvFWData,
+		                                 pvFWCorememCode,
+		                                 pvFWCorememData);
+		if (eError != PVRSRV_OK)
+		{
+			RGXErrorLog(hPrivate, "RGXProcessFWImage: Processing FW image failed (%d)", eError);
+			return eError;
+		}
+
+		if (pvFWData)
+		{
+			RGXRISCVFW_BOOT_DATA *psBootData = (RGXRISCVFW_BOOT_DATA*)
+				IMG_OFFSET_ADDR(pvFWData, RGXRISCVFW_BOOTLDR_CONF_OFFSET);
+
+			psBootData->ui64CorememCodeDevVAddr = puFWParams->sRISCV.sFWCorememCodeDevVAddr.uiAddr;
+			psBootData->ui32CorememCodeFWAddr   = puFWParams->sRISCV.sFWCorememCodeFWAddr.ui32Addr;
+			psBootData->ui32CorememCodeSize     = puFWParams->sRISCV.uiFWCorememCodeSize;
+
+			psBootData->ui64CorememDataDevVAddr = puFWParams->sRISCV.sFWCorememDataDevVAddr.uiAddr;
+			psBootData->ui32CorememDataFWAddr   = puFWParams->sRISCV.sFWCorememDataFWAddr.ui32Addr;
+			psBootData->ui32CorememDataSize     = puFWParams->sRISCV.uiFWCorememDataSize;
+		}
+	}
+
+	return eError;
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxfwimageutils.h b/drivers/gpu/drm/img-rogue/rgxfwimageutils.h
new file mode 100644
index 000000000000..b4cedb8d3271
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwimageutils.h
@@ -0,0 +1,224 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for Services Firmware image utilities used at init time
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for Services Firmware image utilities used at init time
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWIMAGEUTILS_H
+#define RGXFWIMAGEUTILS_H
+
+/* The routines declared here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when DRM security is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary.
+ */
+#include "rgxlayer.h"
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionOffset
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return offset of a Firmware section, relative to the beginning
+               of the code or data allocation (depending on the section id)
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionOffset(const void *hPrivate,
+                                      RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionMaxSize
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return maximum size (not allocation size) of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionMaxSize(const void *hPrivate,
+                                       RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionAllocSize
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return allocation size of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionAllocSize(const void *hPrivate,
+                                         RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageSectionAddress
+
+ @Input        hPrivate : Implementation specific data
+ @Input        eId      : Section id
+
+ @Description  Return base address of a Firmware section
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWImageSectionAddress(const void *hPrivate,
+                                       RGX_FW_SECTION_ID eId);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXGetFWImageAllocSize
+
+ @Description  Return size of Firmware code/data/coremem code allocations
+
+ @Input        hPrivate            : Implementation specific data
+ @Input        pbRGXFirmware       : Pointer to FW binary
+ @Input        ui32RGXFirmwareSize : FW binary size
+ @Output       puiFWCodeAllocSize  : Code size
+ @Output       puiFWDataAllocSize  : Data size
+ @Output       puiFWCorememCodeAllocSize : Coremem code size (0 if N/A)
+ @Output       puiFWCorememDataAllocSize : Coremem data size (0 if N/A)
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXGetFWImageAllocSize(const void *hPrivate,
+                                    const IMG_BYTE    *pbRGXFirmware,
+                                    const IMG_UINT32  ui32RGXFirmwareSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWDataAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememCodeAllocSize,
+                                    IMG_DEVMEM_SIZE_T *puiFWCorememDataAllocSize,
+                                    RGX_FW_INFO_HEADER *psFWInfoHeader);
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessLDRCommandStream
+
+ @Description   Process the output of the Meta toolchain in the .LDR format
+                copying code and data sections into their final location and
+                passing some information to the Meta bootloader
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbLDR                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+ @Input         ppui32BootConf           : Pointer to bootloader data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessLDRCommandStream(const void *hPrivate,
+                                     const IMG_BYTE* pbLDR,
+                                     void* pvHostFWCodeAddr,
+                                     void* pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr,
+                                     IMG_UINT32 **ppui32BootConf);
+
+/*!
+*******************************************************************************
+
+ @Function      ProcessELFCommandStream
+
+ @Description   Process a file in .ELF format copying code and data sections
+                into their final location
+
+ @Input         hPrivate                 : Implementation specific data
+ @Input         pbELF                    : Pointer to FW blob
+ @Input         pvHostFWCodeAddr         : Pointer to FW code
+ @Input         pvHostFWDataAddr         : Pointer to FW data
+ @Input         pvHostFWCorememCodeAddr  : Pointer to FW coremem code
+ @Input         pvHostFWCorememDataAddr  : Pointer to FW coremem data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR ProcessELFCommandStream(const void *hPrivate,
+                                     const IMG_BYTE *pbELF,
+                                     void *pvHostFWCodeAddr,
+                                     void *pvHostFWDataAddr,
+                                     void* pvHostFWCorememCodeAddr,
+                                     void* pvHostFWCorememDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function     RGXProcessFWImage
+
+ @Description  Process the Firmware binary blob copying code and data
+               sections into their final location and passing some
+               information to the Firmware bootloader.
+               If a pointer to the final memory location for FW code or data
+               is not valid (NULL) then the relative section will not be
+               processed.
+
+ @Input        hPrivate        : Implementation specific data
+ @Input        pbRGXFirmware   : Pointer to FW blob
+ @Input        pvFWCode        : Pointer to FW code
+ @Input        pvFWData        : Pointer to FW data
+ @Input        pvFWCorememCode : Pointer to FW coremem code
+ @Input        pvFWCorememData : Pointer to FW coremem data
+ @Input        puFWParams      : Parameters used by the FW at boot time
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessFWImage(const void *hPrivate,
+                               const IMG_BYTE *pbRGXFirmware,
+                               void *pvFWCode,
+                               void *pvFWData,
+                               void *pvFWCorememCode,
+                               void *pvFWCorememData,
+                               PVRSRV_FW_BOOT_PARAMS *puFWParams);
+
+#endif /* RGXFWIMAGEUTILS_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxfwmemctx.h b/drivers/gpu/drm/img-rogue/rgxfwmemctx.h
new file mode 100644
index 000000000000..49f37b1c27aa
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwmemctx.h
@@ -0,0 +1,156 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware Memctx routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for operations on FWKM Shared memory context.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWMEMCTX_H
+#define RGXFWMEMCTX_H
+
+#include "device.h"
+#include "rgx_memallocflags.h"
+#include "pvrsrv.h"
+#include "cache_ops.h"
+#include "cache_km.h"
+#include "pvr_debug.h"
+
+#if defined(CONFIG_ARM64) && defined(__linux__) && defined(SUPPORT_CPUCACHED_FWMEMCTX)
+/*
+ * RGXFwSharedMemCPUCacheMode()
+ * We upgrade allocations on ARM64 Linux when CPU Cache snooping is enabled.
+ * This is because of the Linux direct mapping causing interference due to PIPT
+ * cache. All allocations are normally UCWC but snooping can return a bad value from the
+ * direct mapping as it is cached. Upgrade our allocations to cached to prevent bad cached
+ * values but in turn we require flushing.
+ */
+static INLINE void RGXFwSharedMemCPUCacheMode(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_MEMALLOCFLAGS_T *puiFlags)
+{
+	if ((*puiFlags & (PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                  PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                  PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)) == 0)
+	{
+		/* We don't need to upgrade if we don't map into the CPU */
+		return;
+	}
+
+	/* Clear the existing CPU cache flags */
+	*puiFlags &= ~(PVRSRV_MEMALLOCFLAG_CPU_CACHE_MODE_MASK);
+
+	if (PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig))
+	{
+		*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT;
+	}
+	else
+	{
+		*puiFlags |= PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC;
+	}
+}
+
+#define RGXFwSharedMemCheckSnoopMode(psDeviceConfig) PVR_ASSERT(PVRSRVSystemSnoopingOfCPUCache(psDeviceConfig))
+
+/*
+ * FWSharedMemCacheOpExec()
+ * This is the CPU data-cache maintenance interface for FW shared allocations.
+ * We have to be very careful that the VAs supplied to this function are
+ * sensible as to not cause a kernel oops. Given that this should only be
+ * used for allocations used for the FW this should be guaranteed.
+ */
+static INLINE PVRSRV_ERROR RGXFwSharedMemCacheOpExec(const volatile void *pvVirtStart,
+                                                     IMG_UINT64 uiSize,
+                                                     PVRSRV_CACHE_OP uiCacheOp)
+{
+	IMG_UINT64 uiEndAddr = (IMG_UINT64) pvVirtStart + uiSize;
+	IMG_CPU_PHYADDR uiUnusedPhysAddr = {.uiAddr = 0};
+
+	if (!pvVirtStart || uiSize == 0)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return CacheOpExec(NULL,
+					   (void*) pvVirtStart,
+					   (void*) uiEndAddr,
+					   uiUnusedPhysAddr,
+					   uiUnusedPhysAddr,
+					   uiCacheOp);
+}
+
+#define RGXFwSharedMemCacheOpValue(value, cacheop) (RGXFwSharedMemCacheOpExec(&value, sizeof(value), PVRSRV_CACHE_OP_##cacheop))
+#define RGXFwSharedMemCacheOpPtr(ptr, cacheop) (RGXFwSharedMemCacheOpExec(ptr, sizeof(*ptr), PVRSRV_CACHE_OP_##cacheop))
+#define RGXFwSharedMemCacheOpExecPfn RGXFwSharedMemCacheOpExec
+
+static INLINE void RGXFwSharedMemFlushCCB(void *pvCCBVirtAddr,
+                                          IMG_UINT64 uiStart,
+                                          IMG_UINT64 uiFinish,
+                                          IMG_UINT64 uiLimit)
+{
+	if (uiFinish >= uiStart)
+	{
+		/* Flush the CCB data */
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, uiStart),
+		                          uiFinish - uiStart,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+	else
+	{
+		/* CCCB wrapped around - flush the pre and post wrap boundary separately */
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, uiStart),
+		                          uiLimit - uiStart,
+		                          PVRSRV_CACHE_OP_FLUSH);
+
+		RGXFwSharedMemCacheOpExec(IMG_OFFSET_ADDR(pvCCBVirtAddr, 0),
+		                          uiFinish,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+}
+#else
+#define RGXFwSharedMemCPUCacheMode(...)
+#define RGXFwSharedMemCheckSnoopMode(...)
+/* NULL value required for function callbacks */
+#define RGXFwSharedMemCacheOpExec(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpValue(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpPtr(...) ((void)NULL)
+#define RGXFwSharedMemCacheOpExecPfn NULL
+#define RGXFwSharedMemFlushCCB(...)
+#endif
+
+
+#endif /* RGXFWMEMCTX_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxfwriscv.c b/drivers/gpu/drm/img-rogue/rgxfwriscv.c
new file mode 100644
index 000000000000..555691010517
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwriscv.c
@@ -0,0 +1,1074 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware RISC-V utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware RISC-V utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGX_GET_DMI_REG(psDevInfo, value) \
+	((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? \
+	 RGX_CR_FWCORE_DMI_##value##__HOST_SECURITY_GEQ4 : RGX_CR_FWCORE_DMI_##value)
+#else
+#define RGX_GET_DMI_REG(psDevInfo, value) RGX_CR_FWCORE_DMI_##value
+#endif
+
+/*
+ * RGXRiscvHalt
+ */
+PVRSRV_ERROR RGXRiscvHalt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_DMCONTROL_Reg = RGX_GET_DMI_REG(psDevInfo, DMCONTROL);
+	__maybe_unused IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Halt RISC-V FW");
+
+	/* Send halt request (no need to select one or more harts on this RISC-V core) */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN |
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until hart is halted */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DMSTATUS_Reg,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	/* Clear halt request */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send halt request (no need to select one or more harts on this RISC-V core) */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_HALTREQ_EN |
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+
+	/* Wait until hart is halted */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_DMSTATUS_Reg/sizeof(IMG_UINT32),
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Hart not halted (0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Clear halt request */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXRiscvIsHalted
+ */
+IMG_BOOL RGXRiscvIsHalted(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	/* Assume the core is always halted in nohw */
+	return IMG_TRUE;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+	IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+	return (OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg) &
+	        RGX_CR_FWCORE_DMI_DMSTATUS_ALLHALTED_EN) != 0U;
+#endif
+}
+
+/*
+ * RGXRiscvResume
+ */
+PVRSRV_ERROR RGXRiscvResume(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_DMCONTROL_Reg = RGX_GET_DMI_REG(psDevInfo, DMCONTROL);
+	__maybe_unused IMG_UINT32 ui32_DMI_DMSTATUS_Reg = RGX_GET_DMI_REG(psDevInfo, DMSTATUS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS, "Resume RISC-V FW");
+
+	/* Send resume request (no need to select one or more harts on this RISC-V core) */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN |
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until hart is resumed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DMSTATUS_Reg,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	            RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	/* Clear resume request */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_DMCONTROL_Reg,
+	           RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send resume request (no need to select one or more harts on this RISC-V core) */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_RESUMEREQ_EN |
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+
+	/* Wait until hart is resumed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_DMSTATUS_Reg/sizeof(IMG_UINT32),
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	                         RGX_CR_FWCORE_DMI_DMSTATUS_ALLRESUMEACK_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Hart not resumed (0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMSTATUS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Clear resume request */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DMCONTROL_Reg,
+	               RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvCheckAbstractCmdError
+
+@Description    Check for RISC-V abstract command errors and clear them
+
+@Input          psDevInfo    Pointer to GPU device info
+
+@Return         RGXRISCVFW_ABSTRACT_CMD_ERR
+******************************************************************************/
+static RGXRISCVFW_ABSTRACT_CMD_ERR RGXRiscvCheckAbstractCmdError(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXRISCVFW_ABSTRACT_CMD_ERR eCmdErr;
+
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	eCmdErr = RISCV_ABSTRACT_CMD_NO_ERROR;
+
+	/* Check error status */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            RISCV_ABSTRACT_CMD_NO_ERROR << RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT,
+	            ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	void __iomem *pvRegsBaseKM = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Check error status */
+	eCmdErr = (OSReadUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_ABSTRACTCS_Reg)
+	          & ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK)
+	          >> RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_SHIFT;
+
+	if (eCmdErr != RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RISC-V FW abstract command error %u", eCmdErr));
+
+		/* Clear the error (note CMDERR field is write-1-to-clear) */
+		OSWriteUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_ABSTRACTCS_Reg,
+		               ~RGX_CR_FWCORE_DMI_ABSTRACTCS_CMDERR_CLRMSK);
+	}
+#endif
+
+	return eCmdErr;
+}
+
+/*
+ * RGXRiscReadReg
+ */
+PVRSRV_ERROR RGXRiscvReadReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading HW registers is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Send abstract register read command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_READ |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	               ui32RegAddr);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckAbstractCmdError(psDevInfo) == RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		/* Read register value */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+
+/*
+ * RGXRiscvPollReg
+ */
+PVRSRV_ERROR RGXRiscvPollReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V register 0x%x (expected 0x%08x)",
+	                      ui32RegAddr, ui32Value);
+
+	/* Send abstract register read command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_READ |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	           ui32RegAddr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckAbstractCmdError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling HW registers is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+/*
+ * RGXRiscvWriteReg
+ */
+PVRSRV_ERROR RGXRiscvWriteReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32RegAddr,
+                              IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V register 0x%x (value 0x%08x)",
+	                      ui32RegAddr, ui32Value);
+
+	/* Prepare data to be written to register */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract register write command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_WRITE |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	           ui32RegAddr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare data to be written to register */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg, ui32Value);
+
+	/* Send abstract register write command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_REGISTER << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_WRITE |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT |
+	               ui32RegAddr);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvCheckSysBusError
+
+@Description    Check for RISC-V system bus errors and clear them
+
+@Input          psDevInfo    Pointer to GPU device info
+
+@Return         RGXRISCVFW_SYSBUS_ERR
+******************************************************************************/
+static __maybe_unused RGXRISCVFW_SYSBUS_ERR RGXRiscvCheckSysBusError(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXRISCVFW_SYSBUS_ERR eSBError;
+
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	eSBError = RISCV_SYSBUS_NO_ERROR;
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            RISCV_SYSBUS_NO_ERROR << RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT,
+	            ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	void __iomem *pvRegsBaseKM = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	eSBError = (OSReadUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_SBCS_Reg)
+	         & ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK)
+	         >> RGX_CR_FWCORE_DMI_SBCS_SBERROR_SHIFT;
+
+	if (eSBError != RISCV_SYSBUS_NO_ERROR)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RISC-V FW system bus error %u", eSBError));
+
+		/* Clear the error (note SBERROR field is write-1-to-clear) */
+		OSWriteUncheckedHWReg32(pvRegsBaseKM, ui32_DMI_SBCS_Reg,
+		               ~RGX_CR_FWCORE_DMI_SBCS_SBERROR_CLRMSK);
+	}
+#endif
+
+	return eSBError;
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadAbstractMem
+
+@Description    Read a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvReadAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading memory is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare read address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA1_Reg, ui32Addr);
+
+	/* Send abstract memory read command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_READ |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckAbstractCmdError(psDevInfo) == RISCV_ABSTRACT_CMD_NO_ERROR)
+	{
+		/* Read memory value */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollAbstractMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvPollAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+	                      PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V address 0x%x (expected 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Prepare read address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA1_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract memory read command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_READ |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckAbstractCmdError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_DATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling memory is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadSysBusMem
+
+@Description    Read a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvReadSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 *pui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(pui32Value);
+
+	/* Reading memory is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_SBCS_Reg,
+	               (RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT) |
+	               RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN);
+
+	/* Perform read */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBADDRESS0_Reg, ui32Addr);
+
+	/* Wait until system bus is idle */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_SBCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: System Bus did not go idle in time (sbcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	if (RGXRiscvCheckSysBusError(psDevInfo) == RISCV_SYSBUS_NO_ERROR)
+	{
+		/* Read value from debug system bus */
+		*pui32Value = OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg);
+	}
+	else
+	{
+		*pui32Value = 0U;
+	}
+
+	return PVRSRV_OK;
+#endif
+}
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollSysBusMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvPollSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBDATA0_Reg = RGX_GET_DMI_REG(psDevInfo, SBDATA0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Poll RISC-V address 0x%x (expected 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBCS_Reg,
+	           (RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT) |
+	           RGX_CR_FWCORE_DMI_SBCS_SBREADONADDR_EN,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Perform read */
+	PDUMPREG32(psDevInfo->psDeviceNode,
+	           RGX_PDUMPREG_NAME, ui32_DMI_SBADDRESS0_Reg,
+	           ui32Addr,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until system bus is idle */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	RGXRiscvCheckSysBusError(psDevInfo);
+
+	/* Check read value */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBDATA0_Reg,
+	            ui32Value,
+	            0xFFFFFFFF,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(ui32Addr);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+
+	/* Polling memory is currently not required driverlive */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+}
+
+#if !defined(EMULATOR)
+/*
+ * RGXRiscvReadMem
+ */
+PVRSRV_ERROR RGXRiscvReadMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 *pui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvReadAbstractMem(psDevInfo, ui32Addr, pui32Value);
+	}
+
+	return RGXRiscvReadSysBusMem(psDevInfo, ui32Addr, pui32Value);
+}
+#endif /* !defined(EMULATOR) */
+
+/*
+ * RGXRiscvPollMem
+ */
+PVRSRV_ERROR RGXRiscvPollMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 ui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvPollAbstractMem(psDevInfo, ui32Addr, ui32Value);
+	}
+
+	return RGXRiscvPollSysBusMem(psDevInfo, ui32Addr, ui32Value);
+}
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteAbstractMem
+
+@Description    Write a value at the given address in RISC-V memory space
+                using RISC-V abstract memory commands
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvWriteAbstractMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V address 0x%x (value 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Prepare write address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA1_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write data */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_DATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Send abstract register write command */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_COMMAND_Reg,
+	           (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	           RGXRISCVFW_DMI_COMMAND_WRITE |
+	           RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until abstract command is completed */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_ABSTRACTCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Prepare write address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA1_Reg, ui32Addr);
+
+	/* Prepare write data */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_DATA0_Reg, ui32Value);
+
+	/* Send abstract memory write command */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_COMMAND_Reg,
+	               (RGXRISCVFW_DMI_COMMAND_ACCESS_MEMORY << RGX_CR_FWCORE_DMI_COMMAND_CMDTYPE_SHIFT) |
+	               RGXRISCVFW_DMI_COMMAND_WRITE |
+	               RGXRISCVFW_DMI_COMMAND_AAxSIZE_32BIT);
+
+	/* Wait until abstract command is completed */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_ABSTRACTCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_ABSTRACTCS_BUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Abstract command did not complete in time (abstractcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_ABSTRACTCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteSysBusMem
+
+@Description    Write a value at the given address in RISC-V memory space
+                using the RISC-V system bus
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR
+RGXRiscvWriteSysBusMem(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Addr, IMG_UINT32 ui32Value)
+{
+	__maybe_unused IMG_UINT32 ui32_DMI_ABSTRACTCS_Reg = RGX_GET_DMI_REG(psDevInfo, ABSTRACTCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_COMMAND_Reg = RGX_GET_DMI_REG(psDevInfo, COMMAND);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA0_Reg = RGX_GET_DMI_REG(psDevInfo, DATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBDATA0_Reg = RGX_GET_DMI_REG(psDevInfo, SBDATA0);
+	__maybe_unused IMG_UINT32 ui32_DMI_DATA1_Reg = RGX_GET_DMI_REG(psDevInfo, DATA1);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBCS_Reg = RGX_GET_DMI_REG(psDevInfo, SBCS);
+	__maybe_unused IMG_UINT32 ui32_DMI_SBADDRESS0_Reg = RGX_GET_DMI_REG(psDevInfo, SBADDRESS0);
+
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Write RISC-V address 0x%x (value 0x%08x)",
+	                      ui32Addr, ui32Value);
+
+	/* Configure system bus to read 32 bit every time a new address is provided */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBCS_Reg,
+	           RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT,
+	           PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write address */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBADDRESS0_Reg,
+	           ui32Addr, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Prepare write data and initiate write */
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, ui32_DMI_SBDATA0_Reg,
+	           ui32Value, PDUMP_FLAGS_CONTINUOUS);
+
+	/* Wait until system bus is idle */
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32_DMI_SBCS_Reg,
+	            0U,
+	            RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	            PDUMP_FLAGS_CONTINUOUS,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+#else
+	IMG_UINT32 __iomem *pui32RegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+
+	/* Configure system bus for 32 bit accesses */
+	OSWriteUncheckedHWReg32(pui32RegsBase,
+	               ui32_DMI_SBCS_Reg,
+	               RGXRISCVFW_DMI_SBCS_SBACCESS_32BIT << RGX_CR_FWCORE_DMI_SBCS_SBACCESS_SHIFT);
+
+	/* Prepare write address */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBADDRESS0_Reg, ui32Addr);
+
+	/* Prepare write data and initiate write */
+	OSWriteUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBDATA0_Reg, ui32Value);
+
+	/* Wait until system bus is idle */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+	                         pui32RegsBase + ui32_DMI_SBCS_Reg/sizeof(IMG_UINT32),
+	                         0U,
+	                         RGX_CR_FWCORE_DMI_SBCS_SBBUSY_EN,
+	                         POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: System Bus did not go idle in time (sbcs = 0x%x)",
+		         __func__, OSReadUncheckedHWReg32(pui32RegsBase, ui32_DMI_SBCS_Reg)));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXRiscvWriteMem
+ */
+PVRSRV_ERROR RGXRiscvWriteMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32Addr,
+                              IMG_UINT32 ui32Value)
+{
+	if (ui32Addr >= RGXRISCVFW_COREMEM_BASE && ui32Addr <= RGXRISCVFW_COREMEM_END)
+	{
+		return RGXRiscvWriteAbstractMem(psDevInfo, ui32Addr, ui32Value);
+	}
+
+	return RGXRiscvWriteSysBusMem(psDevInfo, ui32Addr, ui32Value);
+}
+#endif /* !defined(EMULATOR) */
+
+/*
+ * RGXRiscvDmiOp
+ */
+PVRSRV_ERROR RGXRiscvDmiOp(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT64 *pui64DMI)
+{
+#if defined(NO_HARDWARE) && defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+	PVR_UNREFERENCED_PARAMETER(pui64DMI);
+
+	/* Accessing DM registers is not supported in nohw/pdump */
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#else
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define DMI_BASE     ((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? RGX_CR_FWCORE_DMI_RESERVED00__HOST_SECURITY_GEQ4 : RGX_CR_FWCORE_DMI_RESERVED00)
+#else
+#define DMI_BASE     RGX_CR_FWCORE_DMI_RESERVED00
+#endif
+#define DMI_STRIDE  (RGX_CR_FWCORE_DMI_RESERVED01 - RGX_CR_FWCORE_DMI_RESERVED00)
+#define DMI_REG(r)  ((DMI_BASE) + (DMI_STRIDE) * (r))
+
+#define DMI_OP_SHIFT            0U
+#define DMI_OP_MASK             0x3ULL
+#define DMI_DATA_SHIFT          2U
+#define DMI_DATA_MASK           0x3FFFFFFFCULL
+#define DMI_ADDRESS_SHIFT       34U
+#define DMI_ADDRESS_MASK        0xFC00000000ULL
+
+#define DMI_OP_NOP	            0U
+#define DMI_OP_READ	            1U
+#define DMI_OP_WRITE	        2U
+#define DMI_OP_RESERVED	        3U
+
+#define DMI_OP_STATUS_SUCCESS	0U
+#define DMI_OP_STATUS_RESERVED	1U
+#define DMI_OP_STATUS_FAILED	2U
+#define DMI_OP_STATUS_BUSY	    3U
+
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+	IMG_UINT64 ui64Op, ui64Address, ui64Data;
+
+	ui64Op      = (*pui64DMI & DMI_OP_MASK) >> DMI_OP_SHIFT;
+	ui64Address = (*pui64DMI & DMI_ADDRESS_MASK) >> DMI_ADDRESS_SHIFT;
+	ui64Data    = (*pui64DMI & DMI_DATA_MASK) >> DMI_DATA_SHIFT;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		ui64Op = DMI_OP_STATUS_FAILED;
+		goto dmiop_update;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to retrieve RGX power state (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		ui64Op = DMI_OP_STATUS_FAILED;
+		goto dmiop_release_lock;
+	}
+
+	if (ePowerState == PVRSRV_DEV_POWER_STATE_ON)
+	{
+		void __iomem *pvRegsBase = RGX_GET_RISCV_REGS_BASE(psDevInfo);
+		switch (ui64Op)
+		{
+			case DMI_OP_NOP:
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			case DMI_OP_WRITE:
+				OSWriteUncheckedHWReg32(pvRegsBase,
+						DMI_REG(ui64Address),
+						(IMG_UINT32)ui64Data);
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			case DMI_OP_READ:
+				ui64Data = (IMG_UINT64)OSReadUncheckedHWReg32(pvRegsBase,
+						DMI_REG(ui64Address));
+				ui64Op = DMI_OP_STATUS_SUCCESS;
+				break;
+			default:
+				PVR_DPF((PVR_DBG_ERROR, "%s: unknown op %u", __func__, (IMG_UINT32)ui64Op));
+				ui64Op = DMI_OP_STATUS_FAILED;
+				break;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: Accessing RISC-V Debug Module is not "
+					"possible while the GPU is powered off", __func__));
+
+		ui64Op = DMI_OP_STATUS_FAILED;
+	}
+
+dmiop_release_lock:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+dmiop_update:
+	*pui64DMI = (ui64Op << DMI_OP_SHIFT) |
+		(ui64Address << DMI_ADDRESS_SHIFT) |
+		(ui64Data << DMI_DATA_SHIFT);
+
+	return eError;
+#endif
+}
+
+/******************************************************************************
+ End of file (rgxfwriscv.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxfwriscv.h b/drivers/gpu/drm/img-rogue/rgxfwriscv.h
new file mode 100644
index 000000000000..7821c167fa2a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwriscv.h
@@ -0,0 +1,217 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware RISC-V utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware RISC-V utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWRISCV_H
+#define RGXFWRISCV_H
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGXRISCVFW_GET_REMAP_SECURE(r)          (RGX_CR_FWCORE_ADDR_REMAP_CONFIG0__HOST_SECURITY_GEQ4 + ((r) * 8U))
+#define RGXRISCVFW_BOOTLDR_CODE_REMAP_SECURE    (RGXRISCVFW_GET_REMAP_SECURE(RGXRISCVFW_BOOTLDR_CODE_REGION))
+#define RGXRISCVFW_BOOTLDR_DATA_REMAP_SECURE    (RGXRISCVFW_GET_REMAP_SECURE(RGXRISCVFW_BOOTLDR_DATA_REGION))
+#define RGX_GET_RISCV_REGS_BASE(psDevInfo) \
+	((RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4) ? \
+	 (psDevInfo)->pvSecureRegsBaseKM : (psDevInfo)->pvRegsBaseKM)
+#else
+#define RGX_GET_RISCV_REGS_BASE(psDevInfo) ((psDevInfo)->pvRegsBaseKM)
+#endif
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvHalt
+
+@Description    Halt the RISC-V FW core (required for certain operations
+                done through Debug Module)
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvHalt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvIsHalted
+
+@Description    Check if the RISC-V FW is halted
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         IMG_BOOL
+******************************************************************************/
+IMG_BOOL RGXRiscvIsHalted(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvResume
+
+@Description    Resume the RISC-V FW core
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvResume(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadReg
+
+@Description    Read a value from the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvReadReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollReg
+
+@Description    Poll for a value from the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvPollReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32RegAddr,
+                             IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteReg
+
+@Description    Write a value to the given RISC-V register (GPR or CSR)
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32RegAddr     RISC-V register address
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvWriteReg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32RegAddr,
+                              IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvPollMem
+
+@Description    Poll for a value at the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Expected value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvPollMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 ui32Value);
+
+#if !defined(EMULATOR)
+/*!
+*******************************************************************************
+@Function       RGXRiscvReadMem
+
+@Description    Read a value at the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvReadMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32Addr,
+                             IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvWriteMem
+
+@Description    Write a value to the given address in RISC-V memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvWriteMem(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              IMG_UINT32 ui32Addr,
+                              IMG_UINT32 ui32Value);
+#endif /* !defined(EMULATOR) */
+
+/*!
+*******************************************************************************
+@Function       RGXRiscvDmiOp
+
+@Description    Acquire the powerlock and perform an operation on the RISC-V
+                Debug Module Interface, but only if the GPU is powered on.
+
+@Input          psDevInfo       Pointer to device info
+@InOut          pui64DMI        Encoding of a request for the RISC-V Debug
+                                Module with same format as the 'dmi' register
+                                from the RISC-V debug specification (v0.13+).
+                                On return, this is updated with the result of
+                                the request, encoded the same way.
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXRiscvDmiOp(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT64 *pui64DMI);
+
+#endif /* RGXFWRISCV_H */
+/******************************************************************************
+ End of file (rgxfwriscv.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxfwtrace_strings.c b/drivers/gpu/drm/img-rogue/rgxfwtrace_strings.c
new file mode 100644
index 000000000000..d95050871994
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwtrace_strings.c
@@ -0,0 +1,56 @@
+/*************************************************************************/ /*!
+@File           rgxfwtrace_strings.c
+@Title          RGX Firmware trace strings
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgx_fwif_sf.h"
+#include "fwtrace_string.h"
+
+/* The tuple pairs that will be generated using XMacros will be stored here.
+ * This macro definition must match the definition of SFids in rgx_fwif_sf.h
+ */
+const RGXKM_STID_FMT SFs[]= {
+#define X(a, b, c, d, e) { RGXFW_LOG_CREATESFID(a,b,e), d },
+	RGXFW_LOG_SFIDLIST
+#undef X
+};
+
+const IMG_UINT32 g_ui32SFsCount = ARRAY_SIZE(SFs);
diff --git a/drivers/gpu/drm/img-rogue/rgxfwutils.c b/drivers/gpu/drm/img-rogue/rgxfwutils.c
new file mode 100644
index 000000000000..56d4940d2a6c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwutils.c
@@ -0,0 +1,6561 @@
+/*************************************************************************/ /*!
+@File
+@Title          Rogue firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Rogue firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "os_apphint.h"
+#include "cache_km.h"
+#include "allocmem.h"
+#include "physheap.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "devicemem_server.h"
+
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxfwriscv.h"
+#include "rgx_options.h"
+#include "rgx_fwif_alignchecks.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_pdump_panics.h"
+#include "fwtrace_string.h"
+#include "rgxheapconfig.h"
+#include "pvrsrv.h"
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "rgxccb.h"
+#include "rgxcompute.h"
+#include "rgxtransfer.h"
+#include "rgxtdmtransfer.h"
+#include "rgxpower.h"
+#if defined(SUPPORT_DISPLAY_CLASS)
+#include "dc_server.h"
+#endif
+#include "rgxmem.h"
+#include "rgxmmudefs_km.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#endif
+#include "rgxta3d.h"
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+#include "rgxkicksync.h"
+#endif
+#include "rgxutils.h"
+#include "rgxtimecorr.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_external.h"
+#include "tlstream.h"
+#include "devicemem_server_utils.h"
+#include "htbserver.h"
+#include "rgx_bvnc_defs_km.h"
+#include "info_page.h"
+
+#include "physmem_lma.h"
+#include "physmem_osmem.h"
+
+#ifdef __linux__
+#include <linux/kernel.h>	/* sprintf */
+#else
+#include <stdio.h>
+#include <string.h>
+#endif
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_FW_OPP_TABLE) && defined(CONFIG_OF)
+#include "pvr_dvfs_common.h"
+#endif
+
+
+#include "vz_vmm_pvz.h"
+#include "rgx_heaps.h"
+
+/*!
+ ******************************************************************************
+ * HWPERF
+ *****************************************************************************/
+/* Size of the Firmware L1 HWPERF buffer in bytes (2MB). Accessed by the
+ * Firmware and host driver. */
+#define RGXFW_HWPERF_L1_SIZE_MIN        (16U)
+#define RGXFW_HWPERF_L1_SIZE_DEFAULT    PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB
+#define RGXFW_HWPERF_L1_SIZE_MAX        (12288U)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#if defined(DEBUG)
+/* Catch the use of auto-increment when meta_registers_unpacked_accesses feature is
+ * present in case we ever use it. No WA exists so it must not be used */
+#define CHECK_HWBRN_68777(v) \
+	do { \
+		PVR_ASSERT(((v) & RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN) == 0); \
+	} while (0)
+#else
+#define CHECK_HWBRN_68777(v)
+#endif
+#endif
+
+/* Firmware CCB length */
+#if defined(NO_HARDWARE) && defined(PDUMP)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (10)
+#elif defined(SUPPORT_PDVFS) || defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (8)
+#else
+#define RGXFWIF_FWCCB_NUMCMDS_LOG2   (5)
+#endif
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+const IMG_UINT32 gaui32FwOsIrqCntRegAddr[RGXFW_MAX_NUM_OSIDS] = {IRQ_COUNTER_STORAGE_REGS};
+#endif
+
+/* Workload Estimation Firmware CCB length */
+#define RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2   (7)
+
+/* Size of memory buffer for firmware gcov data
+ * The actual data size is several hundred kilobytes. The buffer is an order of magnitude larger. */
+#define RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE (4*1024*1024)
+
+typedef struct
+{
+	RGXFWIF_KCCB_CMD        sKCCBcmd;
+	DLLIST_NODE             sListNode;
+	PDUMP_FLAGS_T           uiPDumpFlags;
+	PVRSRV_RGXDEV_INFO      *psDevInfo;
+} RGX_DEFERRED_KCCB_CMD;
+
+typedef struct
+{
+	IMG_INT32  i32Priority;
+	IMG_UINT32 ui32IsolationGroups;
+	IMG_UINT32 ui32TSPercentage;
+}RGX_QOS_DEFAULTS;
+
+#define RGX_QOS_DEFAULTS_INIT(osid) \
+	{RGX_DRIVERID_##osid##_DEFAULT_PRIORITY,\
+	RGX_DRIVERID_##osid##_DEFAULT_ISOLATION_GROUP,\
+	RGX_DRIVERID_##osid##_DEFAULT_TIME_SLICE}
+
+#if defined(PDUMP)
+/* ensure PIDs are 32-bit because a 32-bit PDump load is generated for the
+ * PID filter example entries
+ */
+static_assert(sizeof(IMG_PID) == sizeof(IMG_UINT32),
+		"FW PID filtering assumes the IMG_PID type is 32-bits wide as it "
+		"generates WRW commands for loading the PID values");
+#endif
+
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+static_assert(RGX_DRIVER_DEFAULT_TIME_SLICES_SUM <= PVRSRV_VZ_TIME_SLICE_MAX, "Invalid driverid time slice aggregate");
+#endif
+
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo);
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+
+static void __MTSScheduleWrite(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32Value)
+{
+	/* This should *NOT* happen. Try to trace what caused this and avoid a NPE
+	 * with the Write/Read at the foot of the function.
+	 */
+	PVR_ASSERT((psDevInfo != NULL));
+	if (psDevInfo == NULL)
+	{
+		return;
+	}
+
+	/* Kick MTS to wake firmware. */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE, ui32Value);
+
+	/* Uncached device/IO mapping will ensure MTS kick leaves CPU, read back
+	 * will ensure it reaches the regbank via inter-connects (AXI, PCIe etc)
+	 * before continuing.
+	 */
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_SCHEDULE);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          ppsMemDesc      pointer to the allocation's memory descriptor
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO*  psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags)
+{
+	PVRSRV_ERROR eError;
+#if defined(SUPPORT_AUTOVZ)
+	IMG_BOOL bClearByMemset;
+	if (PVRSRV_CHECK_ZERO_ON_ALLOC(uiAllocFlags))
+	{
+		/* Under AutoVz the ZERO_ON_ALLOC flag is avoided as it causes the memory to
+		 * be allocated from a different PMR than an allocation without the flag.
+		 * When the content of an allocation needs to be recovered from physical memory
+		 * on a later driver reboot, the memory then cannot be zeroed but the allocation
+		 * addresses must still match.
+		 * If the memory requires clearing, perform a memset after the allocation. */
+		uiAllocFlags &= ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+		bClearByMemset = IMG_TRUE;
+	}
+	else
+	{
+		bClearByMemset = IMG_FALSE;
+	}
+#endif
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate %s", pszName);
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui32Size,
+							  uiAllocFlags,
+							  pszName,
+							  ppsMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate %u bytes for %s (%u)",
+				 __func__,
+				 ui32Size,
+				 pszName,
+				 eError));
+		goto fail_alloc;
+	}
+
+	if (psFwPtr)
+	{
+		eError = RGXSetFirmwareAddress(psFwPtr, *ppsMemDesc, 0, ui32DevVAFlags);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire firmware virtual address for %s (%u)",
+					 __func__,
+					 pszName,
+					 eError));
+			goto fail_fwaddr;
+		}
+	}
+
+#if defined(SUPPORT_AUTOVZ)
+	if ((bClearByMemset) || (ppvCpuPtr))
+#else
+	if (ppvCpuPtr)
+#endif
+	{
+		void *pvTempCpuPtr;
+
+		eError = DevmemAcquireCpuVirtAddr(*ppsMemDesc, &pvTempCpuPtr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to acquire CPU virtual address for %s (%u)",
+					__func__,
+					 pszName,
+					eError));
+			goto fail_cpuva;
+		}
+
+#if defined(SUPPORT_AUTOVZ)
+		if (bClearByMemset)
+		{
+			if (PVRSRV_CHECK_CPU_WRITE_COMBINE(uiAllocFlags))
+			{
+				OSCachedMemSetWMB(pvTempCpuPtr, 0, ui32Size);
+			}
+			else
+			{
+				OSDeviceMemSet(pvTempCpuPtr, 0, ui32Size);
+			}
+		}
+		if (ppvCpuPtr)
+#endif
+		{
+			*ppvCpuPtr = pvTempCpuPtr;
+		}
+#if defined(SUPPORT_AUTOVZ)
+		else
+		{
+			DevmemReleaseCpuVirtAddr(*ppsMemDesc);
+			pvTempCpuPtr = NULL;
+		}
+#endif
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s set up at Fw VA 0x%x and CPU VA 0x%p with alloc flags 0x%" IMG_UINT64_FMTSPECX,
+			 __func__, pszName,
+			 (psFwPtr)   ? (psFwPtr->ui32Addr) : (0),
+			 (ppvCpuPtr) ? (*ppvCpuPtr)        : (NULL),
+			 uiAllocFlags));
+
+	return eError;
+
+fail_cpuva:
+	if (psFwPtr)
+	{
+		RGXUnsetFirmwareAddress(*ppsMemDesc);
+	}
+fail_fwaddr:
+	DevmemFree(*ppsMemDesc);
+fail_alloc:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHwPerfBufferSize
+
+@Description    Computes the effective size of the HW Perf Buffer
+@Input          ui32HWPerfFWBufSizeKB       Device Info struct
+@Return         HwPerfBufferSize
+*/ /**************************************************************************/
+static IMG_UINT32 GetHwPerfBufferSize(IMG_UINT32 ui32HWPerfFWBufSizeKB)
+{
+	IMG_UINT32 HwPerfBufferSize;
+
+	/* HWPerf: Determine the size of the FW buffer */
+	if (ui32HWPerfFWBufSizeKB == 0 ||
+			ui32HWPerfFWBufSizeKB == RGXFW_HWPERF_L1_SIZE_DEFAULT)
+	{
+		/* Under pvrsrvctl 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. Set it to the default
+		 * size, no logging.
+		 */
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_DEFAULT<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MAX))
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too big, using maximum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MAX));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MAX<<10;
+	}
+	else if (ui32HWPerfFWBufSizeKB > (RGXFW_HWPERF_L1_SIZE_MIN))
+	{
+		/* Size specified as in AppHint HWPerfFWBufSizeInKB */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Using HWPerf FW buffer size of %u KB",
+				__func__,
+				ui32HWPerfFWBufSizeKB));
+		HwPerfBufferSize = ui32HWPerfFWBufSizeKB<<10;
+	}
+	else
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: HWPerfFWBufSizeInKB value (%u) too small, using minimum (%u)",
+				__func__,
+				ui32HWPerfFWBufSizeKB, RGXFW_HWPERF_L1_SIZE_MIN));
+		HwPerfBufferSize = RGXFW_HWPERF_L1_SIZE_MIN<<10;
+	}
+
+	return HwPerfBufferSize;
+}
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupSignatureChecks
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupSignatureChecks(PVRSRV_RGXDEV_INFO* psDevInfo,
+                                              DEVMEM_MEMDESC**    ppsSigChecksMemDesc,
+                                              IMG_UINT32          ui32SigChecksBufSize,
+                                              RGXFWIF_SIGBUF_CTL* psSigBufCtl)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+								  ui32SigChecksBufSize,
+								  "FwSignatureChecks",
+								  ppsSigChecksMemDesc,
+								  &psSigBufCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	DevmemPDumpLoadMem(	*ppsSigChecksMemDesc,
+			0,
+			ui32SigChecksBufSize,
+			PDUMP_FLAGS_CONTINUOUS);
+
+	psSigBufCtl->ui32LeftSizeInRegs = ui32SigChecksBufSize / sizeof(IMG_UINT32);
+fail:
+	return eError;
+}
+#endif
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+/*!
+*******************************************************************************
+ @Function		RGXFWSetupFirmwareGcovBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupFirmwareGcovBuffer(PVRSRV_RGXDEV_INFO*			psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32FirmwareGcovBufferSize,
+		RGXFWIF_FIRMWARE_GCOV_CTL*	psFirmwareGcovCtl,
+		const IMG_CHAR*				pszBufferName)
+{
+	PVRSRV_ERROR	eError;
+
+	/* Allocate memory for gcov */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32FirmwareGcovBufferSize,
+								  pszBufferName,
+								  ppsBufferMemDesc,
+								  &psFirmwareGcovCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psFirmwareGcovCtl->ui32Size = ui32FirmwareGcovBufferSize;
+
+	return PVRSRV_OK;
+}
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+/*!
+ ******************************************************************************
+ @Function		RGXFWSetupCounterBuffer
+ @Description
+ @Input			psDevInfo
+
+ @Return		PVRSRV_ERROR
+ *****************************************************************************/
+static PVRSRV_ERROR RGXFWSetupCounterBuffer(PVRSRV_RGXDEV_INFO* psDevInfo,
+		DEVMEM_MEMDESC**			ppsBufferMemDesc,
+		IMG_UINT32					ui32CounterDataBufferSize,
+		RGXFWIF_COUNTER_DUMP_CTL*	psCounterDumpCtl)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+								  ui32CounterDataBufferSize,
+								  "FwCounterBuffer",
+								  ppsBufferMemDesc,
+								  &psCounterDumpCtl->sBuffer,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXSetupFwAllocation");
+
+	psCounterDumpCtl->ui32SizeInDwords = ui32CounterDataBufferSize >> 2;
+	RGXFwSharedMemCacheOpValue(psCounterDumpCtl->ui32SizeInDwords, FLUSH);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+ @Function      RGXFWSetupAlignChecks
+ @Description   This functions allocates and fills memory needed for the
+                aligns checks of the UM and KM structures shared with the
+                firmware. The format of the data in the memory is as follows:
+                    <number of elements in the KM array>
+                    <array of KM structures' sizes and members' offsets>
+                    <number of elements in the UM array>
+                    <array of UM structures' sizes and members' offsets>
+                The UM array is passed from the user side. Now the firmware is
+                responsible for filling this part of the memory. If that
+                happens the check of the UM structures will be performed
+                by the host driver on client's connect.
+                If the macro is not defined the client driver fills the memory
+                and the firmware checks for the alignment of all structures.
+ @Input			psDeviceNode
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXFWSetupAlignChecks(PVRSRV_DEVICE_NODE *psDeviceNode,
+								RGXFWIF_DEV_VIRTADDR	*psAlignChecksDevFW)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32			aui32RGXFWAlignChecksKM[] = { RGXFW_ALIGN_CHECKS_INIT_KM };
+	IMG_UINT32			ui32RGXFWAlignChecksTotal;
+	IMG_UINT32*			paui32AlignChecks;
+	PVRSRV_ERROR		eError;
+
+	/* In this case we don't know the number of elements in UM array.
+	 * We have to assume something so we assume RGXFW_ALIGN_CHECKS_UM_MAX.
+	 */
+	ui32RGXFWAlignChecksTotal = sizeof(aui32RGXFWAlignChecksKM)
+	                            + RGXFW_ALIGN_CHECKS_UM_MAX * sizeof(IMG_UINT32)
+	                            + 2 * sizeof(IMG_UINT32);
+
+	/* Allocate memory for the checks */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32RGXFWAlignChecksTotal,
+								  "FwAlignmentChecks",
+								  &psDevInfo->psRGXFWAlignChecksMemDesc,
+								  psAlignChecksDevFW,
+								  (void**) &paui32AlignChecks,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Copy the values */
+		*paui32AlignChecks++ = ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+		OSCachedMemCopy(paui32AlignChecks, &aui32RGXFWAlignChecksKM[0],
+		                sizeof(aui32RGXFWAlignChecksKM));
+		paui32AlignChecks += ARRAY_SIZE(aui32RGXFWAlignChecksKM);
+
+		*paui32AlignChecks = 0;
+
+		OSWriteMemoryBarrier(paui32AlignChecks);
+		RGXFwSharedMemCacheOpExec(paui32AlignChecks - (ARRAY_SIZE(aui32RGXFWAlignChecksKM) + 1),
+		                          ui32RGXFWAlignChecksTotal,
+		                          PVRSRV_CACHE_OP_FLUSH);
+	}
+
+
+	DevmemPDumpLoadMem(	psDevInfo->psRGXFWAlignChecksMemDesc,
+						0,
+						ui32RGXFWAlignChecksTotal,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+
+fail:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXFWFreeAlignChecks(PVRSRV_RGXDEV_INFO* psDevInfo)
+{
+	if (psDevInfo->psRGXFWAlignChecksMemDesc != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWAlignChecksMemDesc);
+		psDevInfo->psRGXFWAlignChecksMemDesc = NULL;
+	}
+}
+
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+						   DEVMEM_MEMDESC		*psSrc,
+						   IMG_UINT32			uiExtraOffset,
+						   IMG_UINT32			ui32Flags)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	psDevVirtAddr;
+	PVRSRV_DEVICE_NODE	*psDeviceNode;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+
+	psDeviceNode = (PVRSRV_DEVICE_NODE *) DevmemGetConnection(psSrc);
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		IMG_UINT32          ui32Offset;
+		IMG_BOOL            bCachedInMETA;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+		IMG_UINT32          uiGPUCacheMode;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in META memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the META cache flags */
+		bCachedInMETA = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		/* Honour the SLC cache flags */
+		eError = DevmemDeviceCacheMode(uiDevFlags, &uiGPUCacheMode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemDeviceCacheMode", failDevCacheMode);
+
+		/*
+		 * Choose Meta virtual address based on Meta and SLC cacheability.
+		 */
+		ui32Offset += RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+		if (bCachedInMETA)
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+		}
+
+		if (PVRSRV_CHECK_GPU_CACHED(uiGPUCacheMode))
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_CACHED;
+		}
+		else
+		{
+			ui32Offset |= RGXFW_SEGMMU_DATA_VIVT_SLC_UNCACHED;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_GOTO_IF_ERROR(eError, failDevVAAcquire);
+
+		ppDest->ui32Addr = (IMG_UINT32)((psDevVirtAddr.uiAddr + uiExtraOffset) & 0xFFFFFFFF);
+	}
+#endif
+	else
+	{
+		IMG_UINT32      ui32Offset;
+		IMG_BOOL        bCachedInRISCV;
+		PVRSRV_MEMALLOCFLAGS_T uiDevFlags;
+
+		eError = DevmemAcquireDevVirtAddr(psSrc, &psDevVirtAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireDevVirtAddr", failDevVAAcquire);
+
+		/* Convert to an address in RISCV memmap */
+		ui32Offset = psDevVirtAddr.uiAddr + uiExtraOffset - RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Check in the devmem flags whether this memory is cached/uncached */
+		DevmemGetFlags(psSrc, &uiDevFlags);
+
+		/* Honour the RISCV cache flags */
+		bCachedInRISCV = (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) & uiDevFlags) != 0;
+
+		if (bCachedInRISCV)
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_CACHED_DATA_BASE;
+		}
+		else
+		{
+			ui32Offset |= RGXRISCVFW_SHARED_UNCACHED_DATA_BASE;
+		}
+
+		ppDest->ui32Addr = ui32Offset;
+	}
+
+	if ((ppDest->ui32Addr & 0x3U) != 0)
+	{
+		IMG_CHAR *pszAnnotation;
+		/* It is expected that the annotation returned by DevmemGetAnnotation() is always valid */
+		DevmemGetAnnotation(psSrc, &pszAnnotation);
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: %s @ 0x%x is not aligned to 32 bit",
+				 __func__, pszAnnotation, ppDest->ui32Addr));
+
+		return PVRSRV_ERROR_INVALID_ALIGNMENT;
+	}
+
+	if (ui32Flags & RFW_FWADDR_NOREF_FLAG)
+	{
+		DevmemReleaseDevVirtAddr(psSrc);
+	}
+
+	return PVRSRV_OK;
+
+failDevCacheMode:
+	DevmemReleaseDevVirtAddr(psSrc);
+failDevVAAcquire:
+	return eError;
+}
+
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset)
+{
+	PVRSRV_ERROR		eError;
+	IMG_DEV_VIRTADDR	sDevVirtAddr;
+
+	eError = DevmemAcquireDevVirtAddr(psSrcMemDesc, &sDevVirtAddr);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psDest->psDevVirtAddr.uiAddr = sDevVirtAddr.uiAddr;
+	psDest->psDevVirtAddr.uiAddr += uiOffset;
+	psDest->pbyFWAddr.ui32Addr = psSrcFWDevVAddr->ui32Addr;
+
+	DevmemReleaseDevVirtAddr(psSrcMemDesc);
+}
+
+
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc)
+{
+	DevmemReleaseDevVirtAddr(psSrc);
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCB
+ @Description	Free the kernel or firmware CCB
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppvCCBCtlLocal
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+******************************************************************************/
+static void RGXFreeCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtl,
+					   RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+					   DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+					   IMG_UINT8			**ppui8CCB,
+					   DEVMEM_MEMDESC		**ppsCCBMemDesc)
+{
+	if (*ppsCCBMemDesc != NULL)
+	{
+		if (*ppui8CCB != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBMemDesc);
+			*ppui8CCB = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBMemDesc);
+		*ppsCCBMemDesc = NULL;
+	}
+	if (*ppsCCBCtlMemDesc != NULL)
+	{
+		if (*ppsCCBCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBCtlMemDesc);
+			*ppsCCBCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBCtlMemDesc);
+		*ppsCCBCtlMemDesc = NULL;
+	}
+	if (*ppsCCBCtlLocal != NULL)
+	{
+		OSFreeMem(*ppsCCBCtlLocal);
+		*ppsCCBCtlLocal = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXFreeCCBReturnSlots
+ @Description	Free the kernel CCB's return slot array and associated mappings
+ @Input			psDevInfo              Device Info struct
+ @Input			ppui32CCBRtnSlots      CPU mapping of slot array
+ @Input			ppsCCBRtnSlotsMemDesc  Slot array's device memdesc
+******************************************************************************/
+static void RGXFreeCCBReturnSlots(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32         **ppui32CCBRtnSlots,
+								  DEVMEM_MEMDESC     **ppsCCBRtnSlotsMemDesc)
+{
+	/* Free the return slot array if allocated */
+	if (*ppsCCBRtnSlotsMemDesc != NULL)
+	{
+		/* Before freeing, ensure the CPU mapping as well is released */
+		if (*ppui32CCBRtnSlots != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(*ppsCCBRtnSlotsMemDesc);
+			*ppui32CCBRtnSlots = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, *ppsCCBRtnSlotsMemDesc);
+		*ppsCCBRtnSlotsMemDesc = NULL;
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function		RGXSetupCCB
+ @Description	Allocate and initialise a circular command buffer
+ @Input			psDevInfo
+ @Input			ppsCCBCtl
+ @Input			ppsCCBCtlMemDesc
+ @Input			ppui8CCB
+ @Input			ppsCCBMemDesc
+ @Input			psCCBCtlFWAddr
+ @Input			ui32NumCmdsLog2
+ @Input			ui32CmdSize
+ @Input			uiCCBMemAllocFlags
+ @Input			pszName
+
+ @Return		PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupCCB(PVRSRV_RGXDEV_INFO	*psDevInfo,
+								RGXFWIF_CCB_CTL		**ppsCCBCtl,
+								RGXFWIF_CCB_CTL		**ppsCCBCtlLocal,
+								DEVMEM_MEMDESC		**ppsCCBCtlMemDesc,
+								IMG_UINT8			**ppui8CCB,
+								DEVMEM_MEMDESC		**ppsCCBMemDesc,
+								PRGXFWIF_CCB_CTL	*psCCBCtlFWAddr,
+								PRGXFWIF_CCB		*psCCBFWAddr,
+								IMG_UINT32			ui32NumCmdsLog2,
+								IMG_UINT32			ui32CmdSize,
+								PVRSRV_MEMALLOCFLAGS_T uiCCBMemAllocFlags,
+								const IMG_CHAR		*pszName)
+{
+	PVRSRV_ERROR		eError;
+	RGXFWIF_CCB_CTL		*psCCBCtl;
+	IMG_UINT32		ui32CCBSize = (1U << ui32NumCmdsLog2);
+	IMG_CHAR		szCCBCtlName[DEVMEM_ANNOTATION_MAX_LEN];
+	IMG_INT32		iStrLen;
+
+	/* Append "Control" to the name for the control struct. */
+	iStrLen = OSSNPrintf(szCCBCtlName, sizeof(szCCBCtlName), "%sControl", pszName);
+	PVR_ASSERT(iStrLen < sizeof(szCCBCtlName));
+
+	if (unlikely(iStrLen < 0))
+	{
+		OSStringSafeCopy(szCCBCtlName, "FwCCBControl", DEVMEM_ANNOTATION_MAX_LEN);
+	}
+
+	/* Allocate memory for the CCB control.*/
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_CCB_CTL),
+								  szCCBCtlName,
+								  ppsCCBCtlMemDesc,
+								  psCCBCtlFWAddr,
+								  (void**) ppsCCBCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Allocate memory for the CCB.
+	 * (this will reference further command data in non-shared CCBs)
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  uiCCBMemAllocFlags,
+								  ui32CCBSize * ui32CmdSize,
+								  pszName,
+								  ppsCCBMemDesc,
+								  psCCBFWAddr,
+								  (void**) ppui8CCB,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/*
+	 * Initialise the CCB control.
+	 */
+	psCCBCtl = OSAllocZMem(sizeof(*psCCBCtl));
+	PVR_LOG_GOTO_IF_NOMEM(psCCBCtl, eError, fail);
+
+	psCCBCtl->ui32WrapMask = ui32CCBSize - 1;
+
+	OSDeviceMemCopy(*ppsCCBCtl, psCCBCtl, sizeof(*psCCBCtl));
+	RGXFwSharedMemCacheOpPtr(psCCBCtl, FLUSH);
+
+	*ppsCCBCtlLocal = psCCBCtl;
+
+	/* Pdump the CCB control */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Initialise %s", szCCBCtlName);
+	DevmemPDumpLoadMem(*ppsCCBCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_CCB_CTL),
+					   0);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeCCB(psDevInfo,
+			   ppsCCBCtl,
+			   ppsCCBCtlLocal,
+			   ppsCCBCtlMemDesc,
+			   ppui8CCB,
+			   ppsCCBMemDesc);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+static void RGXSetupFaultReadRegisterRollback(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+#endif
+	{
+		if (psDevInfo->psRGXFaultAddressMemDesc)
+		{
+			if (DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR) == PVRSRV_OK)
+			{
+				PMRUnlockSysPhysAddresses(psPMR);
+			}
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+			psDevInfo->psRGXFaultAddressMemDesc = NULL;
+		}
+	}
+}
+
+static PVRSRV_ERROR RGXSetupFaultReadRegister(PVRSRV_DEVICE_NODE *psDeviceNode, RGXFWIF_SYSINIT *psFwSysInit)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			*pui32MemoryVirtAddr;
+	IMG_UINT32			i;
+	size_t				ui32PageSize = OSGetPageSize();
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PMR					*psPMR;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	/* Allocate page of memory to use for page faults on non-blocking memory transactions.
+	 * Doesn't need to be cleared as it is initialised with the 0xDEADBEEF pattern below. */
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+			ui32PageSize,
+			ui32PageSize,
+			RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS & ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+			"FwExFaultAddress",
+			&psDevInfo->psRGXFaultAddressMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to allocate mem for fault address (%u)",
+		         __func__, eError));
+		goto failFaultAddressDescAlloc;
+	}
+
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc,
+										  (void **)&pui32MemoryVirtAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to acquire mem for fault address (%u)",
+					 __func__, eError));
+			goto failFaultAddressDescAqCpuVirt;
+		}
+
+		/* fill the page with a known pattern when booting the firmware */
+		for (i = 0; i < ui32PageSize/sizeof(IMG_UINT32); i++)
+		{
+			*(pui32MemoryVirtAddr + i) = 0xDEADBEEF;
+		}
+
+		OSWriteMemoryBarrier(pui32MemoryVirtAddr);
+		RGXFwSharedMemCacheOpExec(pui32MemoryVirtAddr, ui32PageSize, PVRSRV_CACHE_OP_FLUSH);
+
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFaultAddressMemDesc);
+	}
+
+	eError = DevmemServerGetImportHandle(psDevInfo->psRGXFaultAddressMemDesc, (void **)&psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Error getting PMR for fault address (%u)",
+		         __func__, eError));
+
+		goto failFaultAddressDescGetPMR;
+	}
+	else
+	{
+		IMG_BOOL bValid;
+		IMG_UINT32 ui32Log2PageSize = OSGetPageShift();
+
+		eError = PMRLockSysPhysAddresses(psPMR);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error locking physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescLockPhys;
+		}
+
+		eError = PMR_DevPhysAddr(psPMR,ui32Log2PageSize, 1, 0, &(psFwSysInit->sFaultPhysAddr), &bValid, DEVICE_USE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error getting physical address for fault address MemDesc (%u)",
+			         __func__, eError));
+
+			goto failFaultAddressDescGetPhys;
+		}
+
+		if (!bValid)
+		{
+			psFwSysInit->sFaultPhysAddr.uiAddr = 0;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed getting physical address for fault address MemDesc - invalid page (0x%" IMG_UINT64_FMTSPECX ")",
+			         __func__, psFwSysInit->sFaultPhysAddr.uiAddr));
+
+			goto failFaultAddressDescGetPhys;
+		}
+	}
+
+	return PVRSRV_OK;
+
+failFaultAddressDescGetPhys:
+	PMRUnlockSysPhysAddresses(psPMR);
+
+failFaultAddressDescLockPhys:
+failFaultAddressDescGetPMR:
+failFaultAddressDescAqCpuVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFaultAddressMemDesc);
+	psDevInfo->psRGXFaultAddressMemDesc = NULL;
+
+failFaultAddressDescAlloc:
+
+	return eError;
+}
+
+#if defined(PDUMP)
+/* Replace the DevPhy address with the one Pdump allocates at pdump_player run time */
+static PVRSRV_ERROR RGXPDumpFaultReadRegister(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+	PMR *psFWInitPMR, *psFaultAddrPMR;
+	IMG_UINT32 ui32Dstoffset;
+
+#if defined(RGX_FEATURE_SLC_FAULT_ACCESS_ADDR_PHYS_BIT_MASK)
+	/* Run-time check feature support */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDevInfo->psDeviceNode, SLC_FAULT_ACCESS_ADDR_PHYS))
+	{
+		return PVRSRV_OK;
+	}
+#endif
+
+	psFWInitPMR = (PMR *)(psDevInfo->psRGXFWIfSysInitMemDesc->psImport->hPMR);
+	ui32Dstoffset = psDevInfo->psRGXFWIfSysInitMemDesc->uiOffset + offsetof(RGXFWIF_SYSINIT, sFaultPhysAddr.uiAddr);
+
+	psFaultAddrPMR = (PMR *)(psDevInfo->psRGXFaultAddressMemDesc->psImport->hPMR);
+
+	eError = PDumpMemLabelToMem64(psFaultAddrPMR,
+				psFWInitPMR,
+				0,
+				ui32Dstoffset,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dump of Fault Page Phys address failed(%u)", __func__, eError));
+	}
+	return eError;
+}
+#endif
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW tbi buffer
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTBIBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTBIBufferMemDesc);
+	psDevInfo->psRGXFWIfTBIBufferMemDesc = NULL;
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = 0;
+	psDevInfo->ui32RGXL2HWPerfBufSize = 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferInitOnDemandResources
+
+@Description    Allocates the firmware TBI buffer required for reading SFs
+		strings and initialize it with SFs.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         i, ui32Len;
+	const IMG_UINT32   ui32FWTBIBufsize = g_ui32SFsCount * sizeof(RGXFW_STID_FMT);
+	RGXFW_STID_FMT     *psFW_SFs = NULL;
+
+	/* Firmware address should not be already set */
+	if (psDevInfo->sRGXFWIfTBIBuffer.ui32Addr)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW address for FWTBI is already set. Resetting it with newly allocated one",
+		         __func__));
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS,
+								  ui32FWTBIBufsize,
+								  "FwTBIBuffer",
+								  &psDevInfo->psRGXFWIfTBIBufferMemDesc,
+								  &psDevInfo->sRGXFWIfTBIBuffer,
+								  (void**)&psFW_SFs,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* Copy SFs entries to FW buffer */
+	for (i = 0; i < g_ui32SFsCount; i++)
+	{
+		OSCachedMemCopy(&psFW_SFs[i].ui32Id, &SFs[i].ui32Id, sizeof(SFs[i].ui32Id));
+		ui32Len = OSStringLength(SFs[i].psName);
+		OSCachedMemCopy(psFW_SFs[i].sName, SFs[i].psName, MIN(ui32Len, IMG_SF_STRING_MAX_SIZE - 1));
+	}
+
+	/* flush write buffers for psFW_SFs */
+	OSWriteMemoryBarrier(psFW_SFs);
+
+	/* Set size of TBI buffer */
+	psDevInfo->ui32FWIfTBIBufferSize = ui32FWTBIBufsize;
+
+	/* release CPU mapping */
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTBIBufferMemDesc);
+
+	return PVRSRV_OK;
+fail:
+	RGXTBIBufferDeinit(psDevInfo);
+	return eError;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferDeinit
+
+@Description    Deinitialises all the allocations and references that are made
+		for the FW trace buffer(s)
+
+@Input          ppsDevInfo	 RGX device info
+@Return		void
+*/ /**************************************************************************/
+static void RGXTraceBufferDeinit(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGXFW_THREAD_NUM; i++)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufferMemDesc[i])
+		{
+			if (psDevInfo->apui32TraceBuffer[i] != NULL)
+			{
+				DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+				psDevInfo->apui32TraceBuffer[i] = NULL;
+			}
+
+			DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufferMemDesc[i]);
+			psDevInfo->psRGXFWIfTraceBufferMemDesc[i] = NULL;
+		}
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferInitOnDemandResources
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+		info from the firmware.
+
+@Input          psDevInfo	 RGX device info
+
+@Return		PVRSRV_OK	If all went good, PVRSRV_ERROR otherwise.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo,
+												 PVRSRV_MEMALLOCFLAGS_T uiAllocFlags)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	IMG_UINT32         ui32FwThreadNum;
+	IMG_UINT32         ui32DefaultTraceBufSize;
+	IMG_DEVMEM_SIZE_T  uiTraceBufSizeInBytes;
+	void               *pvAppHintState = NULL;
+	IMG_CHAR           pszBufferName[] = "FwTraceBuffer_Thread0";
+
+	/* Check AppHint value for module-param FWTraceBufSizeInDWords */
+	OSCreateAppHintState(&pvAppHintState);
+	ui32DefaultTraceBufSize = RGXFW_TRACE_BUF_DEFAULT_SIZE_IN_DWORDS;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,
+						 pvAppHintState,
+						 FWTraceBufSizeInDWords,
+						 &ui32DefaultTraceBufSize,
+						 &psDevInfo->ui32TraceBufSizeInDWords);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/* Write tracebuf size once to devmem */
+	psTraceBufCtl->ui32TraceBufSizeInDWords = psDevInfo->ui32TraceBufSizeInDWords;
+
+	if (psDevInfo->ui32TraceBufSizeInDWords < RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS ||
+		psDevInfo->ui32TraceBufSizeInDWords > RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Requested trace buffer size (%u) out of its minimum (%u) & maximum (%u) range. Exiting error.",
+				  __func__,
+				  psDevInfo->ui32TraceBufSizeInDWords,
+				  RGXFW_TRACE_BUF_MIN_SIZE_IN_DWORDS,
+				  RGXFW_TRACE_BUF_MAX_SIZE_IN_DWORDS));
+		eError = PVRSRV_ERROR_OUT_OF_RANGE;
+		goto exit_error;
+	}
+
+	uiTraceBufSizeInBytes = psDevInfo->ui32TraceBufSizeInDWords * sizeof(IMG_UINT32);
+
+	for (ui32FwThreadNum = 0; ui32FwThreadNum < RGXFW_THREAD_NUM; ui32FwThreadNum++)
+	{
+#if !defined(SUPPORT_AUTOVZ)
+		/* Ensure allocation API is only called when not already allocated */
+		PVR_ASSERT(psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum] == NULL);
+		/* Firmware address should not be already set */
+		PVR_ASSERT(psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer.ui32Addr == 0x0);
+#endif
+
+		/* update the firmware thread number in the Trace Buffer's name */
+		pszBufferName[sizeof(pszBufferName) - 2] += ui32FwThreadNum;
+
+		eError = RGXSetupFwAllocation(psDevInfo,
+									  uiAllocFlags,
+									  uiTraceBufSizeInBytes,
+									  pszBufferName,
+									  &psDevInfo->psRGXFWIfTraceBufferMemDesc[ui32FwThreadNum],
+									  &psTraceBufCtl->sTraceBuf[ui32FwThreadNum].pui32RGXFWIfTraceBuffer,
+									  (void**)&psDevInfo->apui32TraceBuffer[ui32FwThreadNum],
+									  RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+	}
+
+	return PVRSRV_OK;
+
+fail:
+	RGXTraceBufferDeinit(psDevInfo);
+exit_error:
+	return eError;
+}
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       RGXPDumpLoadFWInitData
+
+@Description    Allocates the firmware trace buffer required for dumping trace
+                info from the firmware.
+
+@Input          psDevInfo RGX device info
+ */ /*************************************************************************/
+static void RGXPDumpLoadFWInitData(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   RGX_INIT_APPHINTS  *psApphints,
+								   IMG_UINT32         ui32HWPerfCountersDataSize)
+{
+	IMG_UINT32 ui32ConfigFlags  = psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags;
+	IMG_UINT32 ui32FwOsCfgFlags = psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Dump RGXFW Init data");
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "(to enable rgxfw signatures place the following line after the RTCONF line)");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, asSigBufCtl),
+						   sizeof(RGXFWIF_SIGBUF_CTL)*(RGXFWIF_DM_MAX),
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump initial state of FW runtime configuration");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_RUNTIME_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw hwperfctl structure");
+	DevmemPDumpLoadZeroMem (psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+							0,
+							ui32HWPerfCountersDataSize,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw trace control structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TRACEBUF),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware system data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump firmware OS data structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSDATA),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgx TBI buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfTBIBufferMemDesc,
+					   0,
+					   psDevInfo->ui32FWIfTBIBufferSize,
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw register configuration buffer");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfRegCfgMemDesc,
+					   0,
+					   sizeof(RGXFWIF_REG_CFG),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_USER_REGISTER_CONFIGURATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw system init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_SYSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Dump rgxfw os init structure");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWIfOsInitMemDesc,
+					   0,
+					   sizeof(RGXFWIF_OSINIT),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	/* RGXFW Init structure needs to be loaded before we overwrite FaultPhysAddr, else this address patching won't have any effect */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Overwrite FaultPhysAddr of FwSysInit in pdump with actual physical address");
+	RGXPDumpFaultReadRegister(psDevInfo);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "RTCONF: run-time configuration");
+
+	/* Dump the config options so they can be edited. */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW system config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Rand mode:                      0x%08x)", RGXFWIF_INICFG_CTXSWITCH_MODE_RAND);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch Soft Reset Enable:              0x%08x)", RGXFWIF_INICFG_CTXSWITCH_SRESET_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable HWPerf:                             0x%08x)", RGXFWIF_INICFG_HWPERF_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Rascal+Dust Power Island:                  0x%08x)", RGXFWIF_INICFG_POW_RASCALDUST);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( FBCDC Version 3.1 Enable:                  0x%08x)", RGXFWIF_INICFG_FBCDC_V3_1_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Check MList:                               0x%08x)", RGXFWIF_INICFG_CHECK_MLIST_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable Auto Clock Gating:                 0x%08x)", RGXFWIF_INICFG_DISABLE_CLKGATING_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+				 "( Try overlapping DM pipelines:              0x%08x)", RGXFWIF_INICFG_TRY_OVERLAPPING_DM_PIPELINES_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable DM pipeline roadblocks:             0x%08x)", RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+#endif
+				 "( Enable register configuration:             0x%08x)", RGXFWIF_INICFG_REGCONFIG_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on TA Out-of-Memory:                0x%08x)", RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable HWPerf custom counter filter:      0x%08x)", RGXFWIF_INICFG_HWP_DISABLE_FILTER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable Ctx Switch profile mode: 0x%08x (none=b'000, fast=b'001, medium=b'010, slow=b'011, nodelay=b'100))", RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Disable DM overlap (except TA during SPM): 0x%08x)", RGXFWIF_INICFG_DISABLE_DM_OVERLAP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Assert on HWR trigger (page fault, lockup, overrun or poll failure): 0x%08x)", RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Enable IRQ validation:                     0x%08x)", RGXFWIF_INICFG_VALIDATE_IRQ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPU power state mask change Enable:        0x%08x)", RGXFWIF_INICFG_SPU_POWER_STATE_MASK_CHANGE_EN);
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Workload Estimation:                0x%08x)", RGXFWIF_INICFG_WORKEST);
+#if defined(SUPPORT_PDVFS)
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+					 "( Enable Proactive DVFS:                     0x%08x)", RGXFWIF_INICFG_PDVFS);
+#endif /* defined(SUPPORT_PDVFS) */
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( ISP Scheduling Mode (v1=b'01, v2=b'10):    0x%08x)", RGXFWIF_INICFG_ISPSCHEDMODE_MASK);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Validate SOC & USC timers:                 0x%08x)", RGXFWIF_INICFG_VALIDATE_SOCUSC_TIMER);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+							offsetof(RGXFWIF_SYSDATA, ui32ConfigFlags),
+							ui32ConfigFlags,
+							PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Extended FW system config options not used.)");
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the FW OS config options here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch TDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_TDM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch GEOM Enable:                    0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_GEOM_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch 3D Enable:                      0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_3D_EN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch CDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_CDM_EN);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Ctx Switch RDM Enable:                     0x%08x)", RGXFWIF_INICFG_OS_CTXSWITCH_RDM_EN);
+#endif
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  2D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_TDM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  TA Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_GEOM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch  3D Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_3D);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch CDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_CDM);
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Lower Priority Ctx Switch RDM Enable:      0x%08x)", RGXFWIF_INICFG_OS_LOW_PRIO_CS_RDM);
+#endif
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfFwOsDataMemDesc,
+							  offsetof(RGXFWIF_OSDATA, ui32FwOsConfigFlags),
+							  ui32FwOsCfgFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	{
+		PDUMP_FLAGS_T      ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+		IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+		IMG_BOOL           bRunTimeUpdate = IMG_FALSE;
+		IMG_UINT32         ui32DstOffset = psDevInfo->psRGXFWIfRuntimeCfgMemDesc->uiOffset + offsetof(RGXFWIF_RUNTIME_CFG, ui32PowUnitsState);
+		IMG_CHAR           aszPowUnitsMaskRegVar[] = ":SYSMEM:$1";
+		IMG_CHAR           aszPowUnitsEnable[] = "RUNTIME_POW_UNITS_MASK";
+		PMR                *psPMR = (PMR *)(psDevInfo->psRGXFWIfRuntimeCfgMemDesc->psImport->hPMR);
+
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPIF(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+		                      "Load initial value power units mask in FW runtime configuration");
+		DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  ui32DstOffset,
+								  psDevInfo->psRGXFWIfRuntimeCfg->ui32PowUnitsState,
+								  ui32PDumpFlags);
+
+		if (bRunTimeUpdate)
+		{
+			PDUMPELSE(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+			PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags, "Read initial SPU mask value from HW registers");
+			PDumpRegRead32ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_SPU_ENABLE, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, aszPowUnitsMaskRegVar, ui32AllPowUnitsMask, ui32PDumpFlags);
+			PDumpInternalVarToMemLabel(psPMR, ui32DstOffset, aszPowUnitsMaskRegVar, ui32PDumpFlags);
+			PDUMPFI(psDevInfo->psDeviceNode, aszPowUnitsEnable, ui32PDumpFlags);
+		}
+	}
+#endif
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Select one or more security tests here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW private data from non-FW contexts: 0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_DATA);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Read/write FW code from non-FW contexts:         0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_READ_WRITE_FW_CODE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from non-secure memory:          0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_NONSECURE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Execute FW code from secure (non-FW) memory:     0x%08x)", RGXFWIF_SECURE_ACCESS_TEST_RUN_FROM_SECURE);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui32SecurityTestFlags),
+							  psDevInfo->psRGXFWIfSysInit->ui32SecurityTestFlags,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* defined(SUPPORT_SECURITY_VALIDATION) */
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter type: %X=INCLUDE_ALL_EXCEPT, %X=EXCLUDE_ALL_EXCEPT)",
+				 RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT,
+				 RGXFW_PID_FILTER_EXCLUDE_ALL_EXCEPT);
+
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, sPIDFilter.eMode),
+							  psDevInfo->psRGXFWIfSysInit->sPIDFilter.eMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PID filter PID/DriverID list (Up to %u entries. Terminate with a zero PID))",
+				 RGXFWIF_PID_FILTER_MAX_NUM_PIDS);
+	{
+		IMG_UINT32 i;
+
+		/* generate a few WRWs in the pdump stream as an example */
+		for (i = 0; i < MIN(RGXFWIF_PID_FILTER_MAX_NUM_PIDS, 8); i++)
+		{
+			/*
+			 * Some compilers cannot cope with the uses of offsetof() below - the specific problem being the use of
+			 * a non-const variable in the expression, which it needs to be const. Typical compiler output is
+			 * "expression must have a constant value".
+			 */
+			const IMG_DEVMEM_OFFSET_T uiPIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].uiPID);
+
+			const IMG_DEVMEM_OFFSET_T uiDriverIDOff
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_SYSINIT *)0)->sPIDFilter.asItems[i].ui32DriverID);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID and DriverID pair %u)", i);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(PID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiPIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+
+			PDUMPCOMMENT(psDevInfo->psDeviceNode, "(DriverID)");
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfSysInitMemDesc,
+									  uiDriverIDOff,
+									  0,
+									  PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	/*
+	 * Dump the log config so it can be edited.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Set the log config here)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( Log Type: TRACE mode using shared memory buffer: 0x00000001)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           TBI mode via external interface or sim support: 0x00000000)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(           Note: TBI mode will hang on most hardware devices!)");
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MAIN Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MAIN);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MTS Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MTS);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CLEANUP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CLEANUP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CSW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CSW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( BIF Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_BIF);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( PM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_PM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( RTD Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_RTD);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SPM Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SPM);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( POW Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_POW);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWR Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWR);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( HWP Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_HWP);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( MISC Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_MISC);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VZ Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VZ);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( SAFETY Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_SAFETY);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( VERBOSE Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_VERBOSE);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( CUSTOMER Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_CUSTOMER);
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "( DEBUG Group Enable: 0x%08x)", RGXFWIF_LOG_TYPE_GROUP_DEBUG);
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+							  offsetof(RGXFWIF_TRACEBUF, ui32LogType),
+							  psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Set the HWPerf Filter config here, see \"hwperfbin2jsont -h\"");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfSysInitMemDesc,
+							  offsetof(RGXFWIF_SYSINIT, ui64HWPerfFilter),
+							  psDevInfo->psRGXFWIfSysInit->ui64HWPerfFilter,
+							  PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "(Number of registers configurations for types(byte index): pow on(%d), dust change(%d), ta(%d), 3d(%d), cdm(%d), "
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 "tla(%d), "
+#endif
+				 "tdm(%d))",
+				 RGXFWIF_REG_CFG_TYPE_PWR_ON,
+				 RGXFWIF_REG_CFG_TYPE_DUST_CHANGE,
+				 RGXFWIF_REG_CFG_TYPE_TA,
+				 RGXFWIF_REG_CFG_TYPE_3D,
+				 RGXFWIF_REG_CFG_TYPE_CDM,
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+				 RGXFWIF_REG_CFG_TYPE_TLA,
+#endif
+				 RGXFWIF_REG_CFG_TYPE_TDM);
+
+	{
+		IMG_UINT32 i;
+
+		/* Write 32 bits in each iteration as required by PDUMP WRW command */
+		for (i = 0; i < RGXFWIF_REG_CFG_TYPE_ALL; i += sizeof(IMG_UINT32))
+		{
+			DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRegCfgMemDesc,
+									offsetof(RGXFWIF_REG_CFG, aui8NumRegsType[i]),
+									0,
+									PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "(Set registers here: address, mask, value)");
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Addr),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Mask),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+	DevmemPDumpLoadMemValue64(psDevInfo->psRGXFWIfRegCfgMemDesc,
+							  offsetof(RGXFWIF_REG_CFG, asRegConfigs[0].ui64Value),
+							  0,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif /* SUPPORT_USER_REGISTER_CONFIGURATION */
+}
+#endif /* defined(PDUMP) */
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwGuardPage
+
+ @Description Allocate a Guard Page at the start of a Guest's Main Heap
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwGuardPage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+								   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)),
+								  OSGetPageSize(),
+								  "FwGuardPage",
+								  &psDevInfo->psRGXFWHeapGuardPageReserveMemDesc,
+								  NULL,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwSysData
+
+ @Description Sets up all system-wide firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwSysData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									  RGX_INIT_APPHINTS        *psApphints,
+									  IMG_UINT32               ui32ConfigFlags,
+									  IMG_UINT32               ui32ConfigFlagsExt,
+									  IMG_UINT32               ui32HWPerfCountersDataSize)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32         ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32         ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+	RGXFWIF_SYSINIT    *psFwSysInitScratch = NULL;
+#if defined(PDUMP)
+	IMG_UINT32         ui32SignatureChecksBufSize = psApphints->ui32SignatureChecksBufSize;
+#endif
+
+	psFwSysInitScratch = OSAllocZMem(sizeof(*psFwSysInitScratch));
+	PVR_LOG_GOTO_IF_NOMEM(psFwSysInitScratch, eError, fail);
+
+	/* Sys Fw init data */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              (RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+	                              PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSINIT),
+	                              "FwSysInitStructure",
+	                              &psDevInfo->psRGXFWIfSysInitMemDesc,
+	                              NULL,
+	                              (void**) &psDevInfo->psRGXFWIfSysInit,
+	                              RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Sys Init structure allocation", fail);
+
+	/* Setup Fault read register */
+	eError = RGXSetupFaultReadRegister(psDeviceNode, psFwSysInitScratch);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Fault read register setup", fail);
+
+#if defined(SUPPORT_AUTOVZ)
+	psFwSysInitScratch->ui32VzWdgPeriod = PVR_AUTOVZ_WDG_PERIOD_MS;
+#endif
+
+	/* RD Power Island */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableRDPowIsland = psRGXData->psRGXTimingInfo->bEnableRDPowIsland;
+		IMG_BOOL bEnableRDPowIsland = ((psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_DEFAULT) && bSysEnableRDPowIsland) ||
+		                               (psApphints->eRGXRDPowerIslandConf == RGX_RD_POWER_ISLAND_FORCE_ON);
+
+		ui32ConfigFlags |= bEnableRDPowIsland? RGXFWIF_INICFG_POW_RASCALDUST : 0;
+	}
+
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		ui32ConfigFlags |= RGXFWIF_INICFG_WORKEST;
+#endif
+#if defined(SUPPORT_FW_OPP_TABLE)
+		{
+			RGXFWIF_OPP_INFO    *psOPPInfo;
+			IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg;
+
+			/* Pro-active DVFS depends on Workload Estimation */
+			psOPPInfo = &psFwSysInitScratch->sOPPInfo;
+			psDVFSDeviceCfg = &psDeviceNode->psDevConfig->sDVFS.sDVFSDeviceCfg;
+#if defined(CONFIG_OF) && defined(CONFIG_PM_OPP) && !defined(NO_HARDWARE)
+			if (psDVFSDeviceCfg->bDTConfig)
+			{
+				/* OPP table configured from Device tree */
+				eError = DVFSCopyOPPTable(psDeviceNode,
+										  psOPPInfo,
+										  ARRAY_SIZE(psOPPInfo->asOPPValues));
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Unable to copy OPP table to FW init buffer (%u)", eError));
+					goto fail;
+				}
+			}
+#endif
+			if (!psDVFSDeviceCfg->bDTConfig)
+			{
+				PVR_LOG_IF_FALSE(psDVFSDeviceCfg->pasOPPTable, "RGXSetupFwSysData: Missing OPP Table");
+
+				if (psDVFSDeviceCfg->pasOPPTable != NULL)
+				{
+					if (psDVFSDeviceCfg->ui32OPPTableSize > ARRAY_SIZE(psOPPInfo->asOPPValues))
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								 "%s: OPP Table too large: Size = %u, Maximum size = %lu",
+								 __func__,
+								 psDVFSDeviceCfg->ui32OPPTableSize,
+								 (unsigned long)(ARRAY_SIZE(psOPPInfo->asOPPValues))));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						goto fail;
+					}
+
+					OSDeviceMemCopy(psOPPInfo->asOPPValues,
+									psDVFSDeviceCfg->pasOPPTable,
+									sizeof(psOPPInfo->asOPPValues));
+
+					psOPPInfo->ui32MaxOPPPoint = psDVFSDeviceCfg->ui32OPPTableSize - 1;
+
+				}
+			}
+
+#if defined(SUPPORT_PDVFS)
+			ui32ConfigFlags |= RGXFWIF_INICFG_PDVFS;
+#endif
+		}
+#endif /* defined(SUPPORT_FW_OPP_TABLE) */
+	}
+
+	/* FW trace control structure */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_TRACEBUF),
+								  "FwTraceCtlStruct",
+								  &psDevInfo->psRGXFWIfTraceBufCtlMemDesc,
+								  &psFwSysInitScratch->sTraceBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfTraceBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		/* Set initial firmware log type/group(s) */
+		if (psApphints->ui32LogType & ~RGXFWIF_LOG_TYPE_MASK)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid initial log type (0x%X)",
+			         __func__, psApphints->ui32LogType));
+			goto fail;
+		}
+		psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType = psApphints->ui32LogType;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfTraceBufCtl->ui32LogType, FLUSH);
+	}
+
+	/* When PDUMP is enabled, ALWAYS allocate on-demand trace buffer resource
+	 * (irrespective of loggroup(s) enabled), given that logtype/loggroups can
+	 * be set during PDump playback in logconfig, at any point of time,
+	 * Otherwise, allocate only if required. */
+#if !defined(PDUMP)
+#if defined(SUPPORT_AUTOVZ)
+	/* always allocate trace buffer for AutoVz Host drivers to allow
+	 * deterministic addresses of all SysData structures */
+	if ((PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo)) || (RGXTraceBufferIsInitRequired(psDevInfo)))
+#else
+	if (RGXTraceBufferIsInitRequired(psDevInfo))
+#endif
+#endif
+	{
+		eError = RGXTraceBufferInitOnDemandResources(psDevInfo,
+													 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+													 RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp));
+	}
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXTraceBufferInitOnDemandResources", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(RGXFWIF_SYSDATA),
+	                              "FwSysData",
+	                              &psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                              &psFwSysInitScratch->sFwSysData,
+	                              (void**) &psDevInfo->psRGXFWIfFwSysData,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	/* GPIO validation setup */
+	psFwSysInitScratch->eGPIOValidationMode = RGXFWIF_GPIO_VAL_OFF;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = RGXFWSetupCounterBuffer(psDevInfo,
+									 &psDevInfo->psCounterBufferMemDesc,
+									 OSGetPageSize(),
+									 &psFwSysInitScratch->sCounterDumpCtl);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Counter Buffer allocation", fail);
+
+	PVR_DPF((PVR_DBG_WARNING, "Counter buffer allocated at %p, size %zu Bytes.", psDevInfo->psCounterBufferMemDesc, OSGetPageSize()));
+#endif /* defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS) */
+
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	eError = RGXFWSetupFirmwareGcovBuffer(psDevInfo,
+	                                      &psDevInfo->psFirmwareGcovBufferMemDesc,
+	                                      RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE,
+	                                      &psFwSysInitScratch->sFirmwareGcovCtl,
+	                                      "FirmwareGcovBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware GCOV buffer allocation", fail);
+	psDevInfo->ui32FirmwareGcovSize = RGXFWIF_FIRMWARE_GCOV_BUFFER_SIZE;
+#endif /* defined(SUPPORT_FIRMWARE_GCOV) */
+
+#if defined(PDUMP)
+	/* Require a minimum amount of memory for the signature buffers */
+	if (ui32SignatureChecksBufSize < RGXFW_SIG_BUFFER_SIZE_MIN)
+	{
+		ui32SignatureChecksBufSize = RGXFW_SIG_BUFFER_SIZE_MIN;
+	}
+
+	/* Setup Signature and Checksum Buffers */
+	psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	psDevInfo->ui32SigTDMChecksSize = 0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM))
+	{
+		/* Buffer allocated only when feature present because all known TDM
+		 * signature registers are dependent on this feature being present */
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigTDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TDM Signature check setup", fail);
+		psDevInfo->ui32SigTDMChecksSize = ui32SignatureChecksBufSize;
+	}
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSigTAChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GEOM Signature check setup", fail);
+	psDevInfo->ui32SigTAChecksSize = ui32SignatureChecksBufSize;
+
+	eError = RGXFWSetupSignatureChecks(psDevInfo,
+	                                   &psDevInfo->psRGXFWSig3DChecksMemDesc,
+	                                   ui32SignatureChecksBufSize,
+	                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "3D Signature check setup", fail);
+	psDevInfo->ui32Sig3DChecksSize = ui32SignatureChecksBufSize;
+
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1)
+	{
+		eError = RGXFWSetupSignatureChecks(psDevInfo,
+		                                   &psDevInfo->psRGXFWSigRDMChecksMemDesc,
+		                                   ui32SignatureChecksBufSize,
+		                                   &psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RDM Signature check setup", fail);
+		psDevInfo->ui32SigRDMChecksSize = ui32SignatureChecksBufSize;
+	}
+#endif
+
+	if (!psApphints->bEnableSignatureChecks)
+	{
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_TDM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_GEOM].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_3D].sBuffer.ui32Addr = 0x0;
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_CDM].sBuffer.ui32Addr = 0x0;
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+		psFwSysInitScratch->asSigBufCtl[RGXFWIF_DM_RAY].sBuffer.ui32Addr = 0x0;
+#endif
+	}
+#endif /* defined(PDUMP) */
+
+	eError = RGXFWSetupAlignChecks(psDeviceNode,
+	                               &psFwSysInitScratch->sAlignChecks);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Alignment checks setup", fail);
+
+	psFwSysInitScratch->ui32FilterFlags = psApphints->ui32FilterFlags;
+
+	/* Fill the remaining bits of fw the init data */
+	psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_HEAP_BASE;
+	psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_HEAP_BASE;
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/* Fill the remaining bits of fw the init data */
+		psFwSysInitScratch->sPDSExecBase.uiAddr = RGX_PDSCODEDATA_BRN_65273_HEAP_BASE;
+		psFwSysInitScratch->sUSCExecBase.uiAddr = RGX_USCCODE_BRN_65273_HEAP_BASE;
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/* Core clock rate */
+	eError = RGXSetupFwAllocation(psDevInfo,
+	                              RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+	                              RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+	                              sizeof(IMG_UINT32),
+	                              "FwPDVFSCoreClkRate",
+	                              &psDevInfo->psRGXFWIFCoreClkRateMemDesc,
+	                              &psFwSysInitScratch->sCoreClockRate,
+	                              (void**) &psDevInfo->pui32RGXFWIFCoreClkRate,
+	                              RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PDVFS core clock rate memory setup", fail);
+#endif
+	{
+	/* Timestamps */
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags =
+		PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+		PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		PVRSRV_MEMALLOCFLAG_GPU_READABLE | /* XXX ?? */
+		PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+	/*
+	  the timer query arrays
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate timer query arrays (FW)");
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwStartTimesArray",
+	                          & psDevInfo->psStartTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psStartTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64StartTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map start times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT64) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE,
+	                          "FwEndTimesArray",
+	                          & psDevInfo->psEndTimeMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psEndTimeMemDesc,
+	                                  (void **)& psDevInfo->pui64EndTimeById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map end times array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(IMG_UINT32) * RGX_MAX_TIMER_QUERIES,
+	                          uiMemAllocFlags,
+	                          "FwCompletedOpsArray",
+	                          & psDevInfo->psCompletedMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to completed ops array",
+				__func__));
+		goto fail;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psCompletedMemDesc,
+	                                  (void **)& psDevInfo->pui32CompletedById);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map completed ops array",
+				__func__));
+		goto fail;
+	}
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	eError = OSLockCreate(&psDevInfo->hTimerQueryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate log for timer query",
+				__func__));
+		goto fail;
+	}
+#endif
+#if defined(SUPPORT_TBI_INTERFACE)
+#if !defined(PDUMP)
+	/* allocate only if required */
+	if (RGXTBIBufferIsInitRequired(psDevInfo))
+#endif /* !defined(PDUMP) */
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand TBI buffer resource
+		 * (irrespective of loggroup(s) enabled), given that logtype/loggroups
+		 * can be set during PDump playback in logconfig, at any point of time
+		 */
+		eError = RGXTBIBufferInitOnDemandResources(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXTBIBufferInitOnDemandResources", fail);
+	}
+
+	psFwSysInitScratch->sTBIBuf = psDevInfo->sRGXFWIfTBIBuffer;
+#endif /* defined(SUPPORT_TBI_INTERFACE) */
+
+	/* Allocate shared buffer for GPU utilisation.
+	 * Enable FIRMWARE_CACHED to reduce read latency in the FW.
+	 * The FW flushes the cache after any writes.
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_GPU_UTIL_FW),
+								  "FwGPUUtilisationBuffer",
+								  &psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc,
+								  &psFwSysInitScratch->sGpuUtilFWCtl,
+								  (void**) &psDevInfo->psRGXFWIfGpuUtilFW,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "GPU Utilisation Buffer ctl allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_RUNTIME_CFG),
+								  "FwRuntimeCfg",
+								  &psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+								  &psFwSysInitScratch->sRuntimeCfg,
+								  (void**) &psDevInfo->psRGXFWIfRuntimeCfg,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware runtime configuration memory allocation", fail);
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  sizeof(RGXFWIF_REG_CFG),
+								  "FwRegisterConfigStructure",
+								  &psDevInfo->psRGXFWIfRegCfgMemDesc,
+								  &psFwSysInitScratch->sRegCfg,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware register user configuration structure allocation", fail);
+#endif
+
+#if defined(SUPPORT_SECURE_CONTEXT_SWITCH)
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS | PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN)) &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  RGXFW_SCRATCH_BUF_SIZE,
+								  "FwScratchBuf",
+								  &psDevInfo->psRGXFWScratchBufMemDesc,
+								  &psFwSysInitScratch->pbFwScratchBuf,
+								  NULL,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware scratch buffer allocation", fail);
+#endif
+
+	psDevInfo->ui32RGXFWIfHWPerfBufSize = GetHwPerfBufferSize(psApphints->ui32HWPerfFWBufSize);
+
+	/* Host L2 HWPERF buffer size in bytes must be bigger than the L1 buffer
+	 * accessed by the FW. The MISR may try to write one packet the size of the L1
+	 * buffer in some scenarios. When logging is enabled in the MISR, it can be seen
+	 * if the L2 buffer hits a full condition. The closer in size the L2 and L1 buffers
+	 * are the more chance of this happening.
+	 * Size chosen to allow MISR to write an L1 sized packet and for the client
+	 * application/daemon to drain a L1 sized packet e.g. ~ 1.5*L1.
+	 */
+	psDevInfo->ui32RGXL2HWPerfBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize +
+	        (psDevInfo->ui32RGXFWIfHWPerfBufSize>>1);
+
+	/* Second stage initialisation or HWPerf, hHWPerfLock created in first
+	 * stage. See RGXRegisterDevice() call to RGXHWPerfInit(). */
+	if (psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF] == 0)
+	{
+		psFwSysInitScratch->ui64HWPerfFilter =
+		    RGXHWPerfFwSetEventFilter(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF,
+		                              (IMG_UINT64) psApphints->ui32HWPerfFilter0 |
+		                              ((IMG_UINT64) psApphints->ui32HWPerfFilter1 << 32));
+	}
+	else
+	{
+		/* The filter has already been modified. This can happen if
+		 * pvr/apphint/EnableFTraceGPU was enabled. */
+		psFwSysInitScratch->ui64HWPerfFilter = psDevInfo->ui64HWPerfFwFilter;
+	}
+
+#if !defined(PDUMP)
+	/* Allocate if HWPerf filter has already been set. This is possible either
+	 * by setting a proper AppHint or enabling GPU ftrace events. */
+	if (psFwSysInitScratch->ui64HWPerfFilter != 0)
+#endif
+	{
+		/* When PDUMP is enabled, ALWAYS allocate on-demand HWPerf resources
+		 * (irrespective of HWPerf enabled or not), given that HWPerf can be
+		 * enabled during PDump playback via RTCONF at any point of time. */
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL1Buffer", fail);
+
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInitOnDemandL2Stream", fail);
+	}
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWCOMCTX_ALLOCFLAGS &
+								  RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp),
+								  ui32HWPerfCountersDataSize,
+								  "FwHWPerfControlStructure",
+								  &psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+								  &psFwSysInitScratch->sHWPerfCtl,
+								  NULL,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware HW Perf control struct allocation", fail);
+
+	psDevInfo->bPDPEnabled = (ui32ConfigFlags & RGXFWIF_INICFG_DISABLE_PDP_EN)
+							  ? IMG_FALSE : IMG_TRUE;
+
+	psFwSysInitScratch->eFirmwarePerf = psApphints->eFirmwarePerf;
+
+#if defined(PDUMP)
+	/* default: no filter */
+	psFwSysInitScratch->sPIDFilter.eMode = RGXFW_PID_FILTER_INCLUDE_ALL_EXCEPT;
+	psFwSysInitScratch->sPIDFilter.asItems[0].uiPID = 0;
+#endif
+
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	{
+		PVRSRV_MEMALLOCFLAGS_T uiFlags = RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS;
+		PVRSRV_SET_PHYS_HEAP_HINT(FW_PRIV_DATA, uiFlags);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate non-secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+											"FwExNonSecureBuffer",
+											&psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Non-secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbNonSecureBuffer,
+									   psDevInfo->psRGXFWIfNonSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", fail);
+
+		PDUMPCOMMENT(psDeviceNode, "Allocate secure buffer for security validation test");
+		eError = DevmemFwAllocateExportable(psDeviceNode,
+											OSGetPageSize(),
+											OSGetPageSize(),
+											uiFlags,
+											"FwExSecureBuffer",
+											&psDevInfo->psRGXFWIfSecureBufMemDesc);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Secure buffer allocation", fail);
+
+		eError = RGXSetFirmwareAddress(&psFwSysInitScratch->pbSecureBuffer,
+									   psDevInfo->psRGXFWIfSecureBufMemDesc,
+									   0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", fail);
+	}
+#endif /* SUPPORT_SECURITY_VALIDATION */
+
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT) || RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_DELTA_CORRELATION))
+	{
+		psFwSysInitScratch->ui32TFBCCompressionControl =
+			(ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK) >> RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT;
+	}
+#endif /* RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK */
+
+	/* Initialize FW started flag */
+	psFwSysInitScratch->bFirmwareStarted = IMG_FALSE;
+	psFwSysInitScratch->ui32MarkerVal = 1;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = RGX_VZ_CONNECTION_COOLDOWN_PERIOD;
+
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		IMG_UINT32 ui32DriverID;
+
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+
+		/* Required info by FW to calculate the ActivePM idle timer latency */
+		psFwSysInitScratch->ui32InitialCoreClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psFwSysInitScratch->ui32InitialSOCClockSpeed = psRGXData->psRGXTimingInfo->ui32SOCClockSpeed;
+#endif
+		psFwSysInitScratch->ui32InitialActivePMLatencyms = psRGXData->psRGXTimingInfo->ui32ActivePMLatencyms;
+
+		/* Initialise variable runtime configuration to the system defaults */
+		psRuntimeCfg->ui32CoreClockSpeed = psFwSysInitScratch->ui32InitialCoreClockSpeed;
+#if defined(SUPPORT_SOC_TIMER)
+		psRuntimeCfg->ui32SOCClockSpeed = psFwSysInitScratch->ui32InitialSOCClockSpeed;
+#endif
+		psRuntimeCfg->ui32ActivePMLatencyms = psFwSysInitScratch->ui32InitialActivePMLatencyms;
+		psRuntimeCfg->bActivePMLatencyPersistant = IMG_TRUE;
+		psRuntimeCfg->ui32HCSDeadlineMS = RGX_HCS_DEFAULT_DEADLINE_MS;
+
+		if ((RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US > 0U) && (RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US < 1000U))
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			PVR_LOG_GOTO_IF_ERROR(eError,
+								  "RGXSetupFwSysData: RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US must be either 0 (disabled) or greater than 1000",
+								  fail);
+		}
+		else
+		{
+			psRuntimeCfg->ui32WdgPeriodUs = RGXFW_SAFETY_WATCHDOG_PERIOD_IN_US;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo))
+		{
+			psRuntimeCfg->ai32DriverPriority[RGXFW_HOST_DRIVER_ID] = 0;
+			psRuntimeCfg->aui32DriverIsolationGroup[RGXFW_HOST_DRIVER_ID] = RGX_DRIVERID_0_DEFAULT_ISOLATION_GROUP;
+			psRuntimeCfg->aui32TSPercentage[RGXFW_HOST_DRIVER_ID] = (IMG_UINT8)RGX_DRIVERID_0_DEFAULT_TIME_SLICE;
+		}
+		else
+		{
+			const RGX_QOS_DEFAULTS asQosDefaults[RGXFW_MAX_NUM_OSIDS] = {
+					RGX_QOS_DEFAULTS_INIT(0),
+#if (RGXFW_MAX_NUM_OSIDS > 1)
+					RGX_QOS_DEFAULTS_INIT(1),
+#if (RGXFW_MAX_NUM_OSIDS > 2)
+					RGX_QOS_DEFAULTS_INIT(2),
+					RGX_QOS_DEFAULTS_INIT(3),
+					RGX_QOS_DEFAULTS_INIT(4),
+					RGX_QOS_DEFAULTS_INIT(5),
+					RGX_QOS_DEFAULTS_INIT(6),
+					RGX_QOS_DEFAULTS_INIT(7),
+#if (RGXFW_MAX_NUM_OSIDS > 8)
+					RGX_QOS_DEFAULTS_INIT(8),
+					RGX_QOS_DEFAULTS_INIT(9),
+					RGX_QOS_DEFAULTS_INIT(10),
+					RGX_QOS_DEFAULTS_INIT(11),
+					RGX_QOS_DEFAULTS_INIT(12),
+					RGX_QOS_DEFAULTS_INIT(13),
+					RGX_QOS_DEFAULTS_INIT(14),
+					RGX_QOS_DEFAULTS_INIT(15),
+					RGX_QOS_DEFAULTS_INIT(16),
+					RGX_QOS_DEFAULTS_INIT(17),
+					RGX_QOS_DEFAULTS_INIT(18),
+					RGX_QOS_DEFAULTS_INIT(19),
+					RGX_QOS_DEFAULTS_INIT(20),
+					RGX_QOS_DEFAULTS_INIT(21),
+					RGX_QOS_DEFAULTS_INIT(22),
+					RGX_QOS_DEFAULTS_INIT(23),
+					RGX_QOS_DEFAULTS_INIT(24),
+					RGX_QOS_DEFAULTS_INIT(25),
+					RGX_QOS_DEFAULTS_INIT(26),
+					RGX_QOS_DEFAULTS_INIT(27),
+					RGX_QOS_DEFAULTS_INIT(28),
+					RGX_QOS_DEFAULTS_INIT(29),
+					RGX_QOS_DEFAULTS_INIT(30),
+					RGX_QOS_DEFAULTS_INIT(31),
+#if (RGXFW_MAX_NUM_OSIDS > 32)
+#error "Support for more than 32 OSIDs not implemented."
+#endif
+#endif /* RGXFW_MAX_NUM_OSIDS > 8 */
+#endif /* RGXFW_MAX_NUM_OSIDS > 2 */
+
+#endif /* RGXFW_MAX_NUM_OSIDS > 1 */
+			};
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				/* Set up initial priorities between different OSes */
+				psRuntimeCfg->ai32DriverPriority[ui32DriverID] = asQosDefaults[ui32DriverID].i32Priority;
+				psRuntimeCfg->aui32DriverIsolationGroup[ui32DriverID] = asQosDefaults[ui32DriverID].ui32IsolationGroups;
+				psRuntimeCfg->aui32TSPercentage[ui32DriverID] = (asQosDefaults[ui32DriverID].ui32TSPercentage <=
+																				PVRSRV_VZ_TIME_SLICE_MAX) ?
+																				asQosDefaults[ui32DriverID].ui32TSPercentage:(0);
+			}
+		}
+		psRuntimeCfg->ui32TSIntervalMs = RGX_DRIVER_DEFAULT_TIME_SLICE_INTERVAL;
+
+#if defined(PVR_ENABLE_PHR) && defined(PDUMP)
+		psRuntimeCfg->ui32PHRMode = RGXFWIF_PHR_MODE_RD_RESET;
+#else
+		psRuntimeCfg->ui32PHRMode = 0;
+#endif
+
+		/* Initialize the PowUnitsState Field to Max Dusts */
+		psRuntimeCfg->ui32PowUnitsState = psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount;
+
+		/* flush write buffers for psDevInfo->psRGXFWIfRuntimeCfg */
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfRuntimeCfg);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfRuntimeCfg, FLUSH);
+
+		/* Setup FW coremem data */
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psFwSysInitScratch->sCorememDataStore.pbyFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+		}
+
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlags    = ui32ConfigFlags    & RGXFWIF_INICFG_ALL;
+		psDevInfo->psRGXFWIfFwSysData->ui32ConfigFlagsExt = ui32ConfigFlagsExt & RGXFWIF_INICFG_EXT_ALL;
+
+		/* Initialise GPU utilisation buffer */
+		{
+			IMG_UINT64 ui64LastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64(), RGXFWIF_GPU_UTIL_STATE_IDLE);
+			RGXFWIF_DM eDM;
+			IMG_UINT64 ui64LastWordTimeShifted =
+			        RGXFWIF_GPU_UTIL_MAKE_WORD(OSClockns64() >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT, RGXFWIF_GPU_UTIL_STATE_IDLE);
+			IMG_UINT32 ui32DriverID;
+
+			psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord = ui64LastWord;
+
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				RGXFWIF_GPU_STATS *psStats = &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID];
+
+				for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+				{
+					psStats->aui32DMOSLastWord[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted & IMG_UINT32_MAX);
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64LastWordTimeShifted >> 32);
+				}
+			}
+			RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+		}
+
+		/* init HWPERF data */
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWIdx = 0;
+		psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfUt = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32HWPerfDropCount = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32FirstDropOrdinal = 0;
+		psDevInfo->psRGXFWIfFwSysData->ui32LastDropOrdinal = 0;
+
+		psDevInfo->psRGXFWIfFwSysData->ui32MemFaultCheck = 0;
+
+		// flush write buffers for psRGXFWIfFwSysData
+		OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwSysData);
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, FLUSH);
+
+		/*Send through the BVNC Feature Flags*/
+		eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, &psFwSysInitScratch->sBvncKmFeatureFlags);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags", fail);
+
+		/* populate the real FwOsInit structure with the values stored in the scratch copy */
+		OSCachedMemCopyWMB(psDevInfo->psRGXFWIfSysInit, psFwSysInitScratch, sizeof(RGXFWIF_SYSINIT));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfSysInit,
+		                         FLUSH);
+	}
+
+	OSFreeMem(psFwSysInitScratch);
+
+
+	return PVRSRV_OK;
+
+fail:
+	if (psFwSysInitScratch)
+	{
+		OSFreeMem(psFwSysInitScratch);
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFwOsData
+
+ @Description Sets up all os-specific firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXSetupFwOsData(PVRSRV_DEVICE_NODE       *psDeviceNode,
+									 IMG_UINT32               ui32KCCBSizeLog2,
+									 IMG_UINT32               ui32HWRDebugDumpLimit,
+									 IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_OSINIT     sFwOsInitScratch;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSCachedMemSet(&sFwOsInitScratch, 0, sizeof(RGXFWIF_OSINIT));
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXSetupFwGuardPage(psDevInfo);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware heap guard pages", fail);
+	}
+
+	/* Memory tracking the connection state should be non-volatile and
+	 * is not cleared on allocation to prevent loss of pre-reset information */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS &
+								  ~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC,
+								  sizeof(RGXFWIF_CONNECTION_CTL),
+								  "FwConnectionCtl",
+								  &psDevInfo->psRGXFWIfConnectionCtlMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfConnectionCtl,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Connection Control structure allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS |
+								  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED),
+								  sizeof(RGXFWIF_OSINIT),
+								  "FwOsInitStructure",
+								  &psDevInfo->psRGXFWIfOsInitMemDesc,
+								  NULL,
+								  (void**) &psDevInfo->psRGXFWIfOsInit,
+								  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware Os Init structure allocation", fail);
+
+	/* init HWR frame info */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_HWRINFOBUF),
+								  "FwHWRInfoBuffer",
+								  &psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc,
+								  &sFwOsInitScratch.sRGXFWIfHWRInfoBufCtl,
+								  (void**) &psDevInfo->psRGXFWIfHWRInfoBufCtl,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "HWR Info Buffer allocation", fail);
+
+	/* Might be uncached. Be conservative and use a DeviceMemSet */
+	OSDeviceMemSet(psDevInfo->psRGXFWIfHWRInfoBufCtl, 0, sizeof(RGXFWIF_HWRINFOBUF));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfHWRInfoBufCtl, FLUSH);
+
+	/* Allocate a sync for power management */
+	eError = SyncPrimContextCreate(psDevInfo->psDeviceNode,
+	                               &psDevInfo->hSyncPrimContext);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive context allocation", fail);
+
+	eError = SyncPrimAlloc(psDevInfo->hSyncPrimContext, &psDevInfo->psPowSyncPrim, "fw power ack");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Sync primitive allocation", fail);
+
+	/* Set up kernel CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psKernelCCBCtl,
+						 &psDevInfo->psKernelCCBCtlLocal,
+						 &psDevInfo->psKernelCCBCtlMemDesc,
+						 &psDevInfo->psKernelCCB,
+						 &psDevInfo->psKernelCCBMemDesc,
+						 &sFwOsInitScratch.psKernelCCBCtl,
+						 &sFwOsInitScratch.psKernelCCB,
+						 ui32KCCBSizeLog2,
+						 sizeof(RGXFWIF_KCCB_CMD),
+						 (RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS |
+						 PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)),
+						 "FwKernelCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB allocation", fail);
+
+	/* KCCB additionally uses a return slot array for FW to be able to send back
+	 * return codes for each required command
+	 */
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  (1U << ui32KCCBSizeLog2) * sizeof(IMG_UINT32),
+								  "FwKernelCCBRtnSlots",
+								  &psDevInfo->psKernelCCBRtnSlotsMemDesc,
+								  &sFwOsInitScratch.psKernelCCBRtnSlots,
+								  (void**) &psDevInfo->pui32KernelCCBRtnSlots,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Kernel CCB return slot array allocation", fail);
+
+	/* Set up firmware CCB */
+	eError = RGXSetupCCB(psDevInfo,
+						 &psDevInfo->psFirmwareCCBCtl,
+						 &psDevInfo->psFirmwareCCBCtlLocal,
+						 &psDevInfo->psFirmwareCCBCtlMemDesc,
+						 &psDevInfo->psFirmwareCCB,
+						 &psDevInfo->psFirmwareCCBMemDesc,
+						 &sFwOsInitScratch.psFirmwareCCBCtl,
+						 &sFwOsInitScratch.psFirmwareCCB,
+						 RGXFWIF_FWCCB_NUMCMDS_LOG2,
+						 sizeof(RGXFWIF_FWCCB_CMD),
+						 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+						 "FwCCB");
+	PVR_LOG_GOTO_IF_ERROR(eError, "Firmware CCB allocation", fail);
+
+	eError = RGXSetupFwAllocation(psDevInfo,
+								  RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS,
+								  sizeof(RGXFWIF_OSDATA),
+								  "FwOsData",
+								  &psDevInfo->psRGXFWIfFwOsDataMemDesc,
+								  &sFwOsInitScratch.sFwOsData,
+								  (void**) &psDevInfo->psRGXFWIfFwOsData,
+								  RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetupFwAllocation", fail);
+
+	psDevInfo->psRGXFWIfFwOsData->ui32FwOsConfigFlags = ui32FwOsCfgFlags & RGXFWIF_INICFG_OS_ALL;
+
+	eError = SyncPrimGetFirmwareAddr(psDevInfo->psPowSyncPrim, &psDevInfo->psRGXFWIfFwOsData->sPowerSync.ui32Addr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Get Sync Prim FW address", fail);
+
+	/* flush write buffers for psRGXFWIfFwOsData */
+	OSWriteMemoryBarrier(psDevInfo->psRGXFWIfFwOsData);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         FLUSH);
+
+	sFwOsInitScratch.ui32HWRDebugDumpLimit = ui32HWRDebugDumpLimit;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Set up Workload Estimation firmware CCB */
+		eError = RGXSetupCCB(psDevInfo,
+							 &psDevInfo->psWorkEstFirmwareCCBCtl,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+							 &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+							 &psDevInfo->psWorkEstFirmwareCCB,
+							 &psDevInfo->psWorkEstFirmwareCCBMemDesc,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCBCtl,
+							 &sFwOsInitScratch.psWorkEstFirmwareCCB,
+							 RGXFWIF_WORKEST_FWCCB_NUMCMDS_LOG2,
+							 sizeof(RGXFWIF_WORKEST_FWCCB_CMD),
+							 RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS,
+							 "FwWEstCCB");
+		PVR_LOG_GOTO_IF_ERROR(eError, "Workload Estimation Firmware CCB allocation", fail);
+	}
+#endif /* defined(SUPPORT_WORKLOAD_ESTIMATION) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise the compatibility check data */
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sFWBVNC);
+		RGXFWIF_COMPCHECKS_BVNC_INIT(sFwOsInitScratch.sRGXCompChecks.sHWBVNC);
+	}
+
+	/* populate the real FwOsInit structure with the values stored in the scratch copy */
+	OSCachedMemCopyWMB(psDevInfo->psRGXFWIfOsInit, &sFwOsInitScratch, sizeof(RGXFWIF_OSINIT));
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfOsInit,
+	                         FLUSH);
+
+	return PVRSRV_OK;
+
+fail:
+	RGXFreeFwOsData(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXSetupFirmware
+
+ @Description Sets up all firmware related data
+
+ @Input       psDevInfo
+
+ @Return      PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+							  RGX_INIT_APPHINTS        *psApphints,
+							  IMG_UINT32               ui32ConfigFlags,
+							  IMG_UINT32               ui32ConfigFlagsExt,
+							  IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_UINT32 ui32HWPerfCountersDataSize;
+
+	eError = RGXSetupFwOsData(psDeviceNode,
+							  psApphints->ui32KCCBSizeLog2,
+							  psApphints->ui32HWRDebugDumpLimit,
+							  ui32FwOsCfgFlags);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware os data", fail);
+
+	ui32HWPerfCountersDataSize = sizeof(RGXFWIF_HWPERF_CTL);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest drivers do not configure system-wide firmware data */
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+	else
+	{
+		/* Native and Host drivers must initialise the firmware's system data */
+		eError = RGXSetupFwSysData(psDeviceNode,
+								   psApphints,
+								   ui32ConfigFlags,
+								   ui32ConfigFlagsExt,
+								   ui32HWPerfCountersDataSize);
+		PVR_LOG_GOTO_IF_ERROR(eError, "Setting up firmware system data", fail);
+	}
+
+	psDevInfo->bFirmwareInitialised = IMG_TRUE;
+
+#if defined(PDUMP)
+	RGXPDumpLoadFWInitData(psDevInfo,
+	                       psApphints,
+	                       ui32HWPerfCountersDataSize);
+#endif /* PDUMP */
+
+fail:
+	return eError;
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwSysData
+
+ @Description Frees all system-wide firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwSysData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	psDevInfo->bFirmwareInitialised = IMG_FALSE;
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc)
+	{
+		RGXFWFreeAlignChecks(psDevInfo);
+	}
+
+#if defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TDM_PDS_CHECKSUM) &&
+	    psDevInfo->psRGXFWSigTDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTDMChecksMemDesc);
+		psDevInfo->psRGXFWSigTDMChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSigTAChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigTAChecksMemDesc);
+		psDevInfo->psRGXFWSigTAChecksMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWSig3DChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSig3DChecksMemDesc);
+		psDevInfo->psRGXFWSig3DChecksMemDesc = NULL;
+	}
+
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, RAY_TRACING_ARCH) &&
+		RGX_GET_FEATURE_VALUE(psDevInfo, RAY_TRACING_ARCH) > 1 &&
+	   psDevInfo->psRGXFWSigRDMChecksMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWSigRDMChecksMemDesc);
+		psDevInfo->psRGXFWSigRDMChecksMemDesc = NULL;
+	}
+#endif
+#endif
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	if (psDevInfo->psCounterBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCounterBufferMemDesc);
+		psDevInfo->psCounterBufferMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FIRMWARE_GCOV)
+	if (psDevInfo->psFirmwareGcovBufferMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psFirmwareGcovBufferMemDesc);
+		psDevInfo->psFirmwareGcovBufferMemDesc = NULL;
+	}
+#endif
+
+	RGXSetupFaultReadRegisterRollback(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfGpuUtilFW != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+			psDevInfo->psRGXFWIfGpuUtilFW = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc);
+		psDevInfo->psRGXFWIfGpuUtilFWCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfRuntimeCfgMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfRuntimeCfg != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+			psDevInfo->psRGXFWIfRuntimeCfg = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRuntimeCfgMemDesc);
+		psDevInfo->psRGXFWIfRuntimeCfgMemDesc = NULL;
+	}
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+		{
+			psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+		}
+	}
+
+	if (psDevInfo->psRGXFWIfTraceBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfTraceBufCtl != NULL)
+		{
+			/* deinit/free the tracebuffer allocation */
+			RGXTraceBufferDeinit(psDevInfo);
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+			psDevInfo->psRGXFWIfTraceBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfTraceBufCtlMemDesc);
+		psDevInfo->psRGXFWIfTraceBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwSysDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwSysData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwSysDataMemDesc);
+			psDevInfo->psRGXFWIfFwSysData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwSysDataMemDesc);
+		psDevInfo->psRGXFWIfFwSysDataMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_TBI_INTERFACE)
+	if (psDevInfo->psRGXFWIfTBIBufferMemDesc)
+	{
+		RGXTBIBufferDeinit(psDevInfo);
+	}
+#endif
+
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	if (psDevInfo->psRGXFWIfRegCfgMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfRegCfgMemDesc);
+		psDevInfo->psRGXFWIfRegCfgMemDesc = NULL;
+	}
+#endif
+	if (psDevInfo->psRGXFWIfHWPerfCountersMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+		psDevInfo->psRGXFWIfHWPerfCountersMemDesc = NULL;
+	}
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDevInfo->psRGXFWIfNonSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfNonSecureBufMemDesc);
+		psDevInfo->psRGXFWIfNonSecureBufMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSecureBufMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSecureBufMemDesc);
+		psDevInfo->psRGXFWIfSecureBufMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	if (psDevInfo->psRGXFWIFCoreClkRateMemDesc)
+	{
+		if (psDevInfo->pui32RGXFWIFCoreClkRate != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+			psDevInfo->pui32RGXFWIFCoreClkRate = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIFCoreClkRateMemDesc);
+		psDevInfo->psRGXFWIFCoreClkRateMemDesc = NULL;
+	}
+#endif
+
+#if defined(SUPPORT_FW_HOST_SIDE_RECOVERY)
+	if (psDevInfo->psRGXFWIfActiveContextBufDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfActiveContextBufDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfActiveContextBufDesc);
+		psDevInfo->psRGXFWIfActiveContextBufDesc = NULL;
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFwOsData
+
+ @Description Frees all os-specific firmware related data
+
+ @Input       psDevInfo
+******************************************************************************/
+static void RGXFreeFwOsData(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFreeCCBReturnSlots(psDevInfo,
+	                      &psDevInfo->pui32KernelCCBRtnSlots,
+	                      &psDevInfo->psKernelCCBRtnSlotsMemDesc);
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psKernelCCBCtl,
+	           &psDevInfo->psKernelCCBCtlLocal,
+	           &psDevInfo->psKernelCCBCtlMemDesc,
+	           &psDevInfo->psKernelCCB,
+	           &psDevInfo->psKernelCCBMemDesc);
+
+	RGXFreeCCB(psDevInfo,
+	           &psDevInfo->psFirmwareCCBCtl,
+	           &psDevInfo->psFirmwareCCBCtlLocal,
+	           &psDevInfo->psFirmwareCCBCtlMemDesc,
+	           &psDevInfo->psFirmwareCCB,
+	           &psDevInfo->psFirmwareCCBMemDesc);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		RGXFreeCCB(psDevInfo,
+		           &psDevInfo->psWorkEstFirmwareCCBCtl,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlLocal,
+		           &psDevInfo->psWorkEstFirmwareCCBCtlMemDesc,
+		           &psDevInfo->psWorkEstFirmwareCCB,
+		           &psDevInfo->psWorkEstFirmwareCCBMemDesc);
+	}
+#endif
+
+	if (psDevInfo->psPowSyncPrim != NULL)
+	{
+		SyncPrimFree(psDevInfo->psPowSyncPrim);
+		psDevInfo->psPowSyncPrim = NULL;
+	}
+
+	if (psDevInfo->hSyncPrimContext != (IMG_HANDLE) NULL)
+	{
+		SyncPrimContextDestroy(psDevInfo->hSyncPrimContext);
+		psDevInfo->hSyncPrimContext = (IMG_HANDLE) NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfHWRInfoBufCtl != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+			psDevInfo->psRGXFWIfHWRInfoBufCtl = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc);
+		psDevInfo->psRGXFWIfHWRInfoBufCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfFwOsDataMemDesc)
+	{
+		if (psDevInfo->psRGXFWIfFwOsData != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfFwOsDataMemDesc);
+			psDevInfo->psRGXFWIfFwOsData = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfFwOsDataMemDesc);
+		psDevInfo->psRGXFWIfFwOsDataMemDesc = NULL;
+	}
+
+	if (psDevInfo->psCompletedMemDesc)
+	{
+		if (psDevInfo->pui32CompletedById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psCompletedMemDesc);
+			psDevInfo->pui32CompletedById = NULL;
+		}
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psCompletedMemDesc);
+		psDevInfo->psCompletedMemDesc = NULL;
+	}
+	if (psDevInfo->psEndTimeMemDesc)
+	{
+		if (psDevInfo->pui64EndTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psEndTimeMemDesc);
+			psDevInfo->pui64EndTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psEndTimeMemDesc);
+		psDevInfo->psEndTimeMemDesc = NULL;
+	}
+	if (psDevInfo->psStartTimeMemDesc)
+	{
+		if (psDevInfo->pui64StartTimeById)
+		{
+			DevmemReleaseCpuVirtAddr(psDevInfo->psStartTimeMemDesc);
+			psDevInfo->pui64StartTimeById = NULL;
+		}
+
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psStartTimeMemDesc);
+		psDevInfo->psStartTimeMemDesc = NULL;
+	}
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	if (psDevInfo->hTimerQueryLock)
+	{
+		OSLockDestroy(psDevInfo->hTimerQueryLock);
+		psDevInfo->hTimerQueryLock = NULL;
+	}
+#endif
+
+	if (psDevInfo->psRGXFWHeapGuardPageReserveMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWHeapGuardPageReserveMemDesc);
+	}
+}
+
+/*!
+*******************************************************************************
+ @Function    RGXFreeFirmware
+
+ @Description Frees all the firmware-related allocations
+
+ @Input       psDevInfo
+******************************************************************************/
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFreeFwOsData(psDevInfo);
+
+	if (psDevInfo->psRGXFWIfConnectionCtl)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtl = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfConnectionCtlMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfConnectionCtlMemDesc);
+		psDevInfo->psRGXFWIfConnectionCtlMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfOsInitMemDesc);
+		psDevInfo->psRGXFWIfOsInitMemDesc = NULL;
+	}
+
+	RGXFreeFwSysData(psDevInfo);
+	if (psDevInfo->psRGXFWIfSysInit)
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInit = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfSysInitMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfSysInitMemDesc);
+		psDevInfo->psRGXFWIfSysInitMemDesc = NULL;
+	}
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalKCCBRoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT32 ui32ReadOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update KCCB read offset */
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+	ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+	if (ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psKCCBCtlLocal->ui32ReadOffset = ui32ReadOffset;
+
+	return PVRSRV_OK;
+}
+
+static INLINE PVRSRV_ERROR RGXUpdateLocalFWCCBWoff(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	const RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT32 ui32WriteOffset;
+
+	barrier(); /* Don't optimise order. Reads from device memory follow. */
+
+	/* update FWCCB write offset */
+	RGXFwSharedMemCacheOpValue(psFWCCBCtl->ui32WriteOffset, INVALIDATE);
+	ui32WriteOffset = psFWCCBCtl->ui32WriteOffset;
+
+	if (ui32WriteOffset > psFWCCBCtlLocal->ui32WrapMask)
+	{
+		return PVRSRV_ERROR_KERNEL_CCB_OFFSET;
+	}
+
+	psFWCCBCtlLocal->ui32WriteOffset = ui32WriteOffset;
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXAcquireKernelCCBSlot
+
+ PURPOSE	: Attempts to obtain a slot in the Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32		*pui32Offset)
+{
+	IMG_UINT32 ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+#if defined(PDUMP)
+	/* Wait for sufficient CCB space to become available */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, 0,
+	                      "Wait for space to write kCCB woff=%u", psKCCBCtlLocal->ui32WriteOffset);
+	DevmemPDumpCBP(psDevInfo->psKernelCCBCtlMemDesc,
+	               offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+	               psKCCBCtlLocal->ui32WriteOffset,
+	               1,
+	               (psKCCBCtlLocal->ui32WrapMask + 1));
+#endif
+
+	if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset == psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+	}
+	*pui32Offset = ui32NextWriteOffset;
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXPollKernelCCBSlot
+
+ PURPOSE	: Poll for space in Kernel CCB
+
+ PARAMETERS	: psCCB - the CCB
+			: Address of space if available, NULL otherwise
+
+ RETURNS	: PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXPollKernelCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32	ui32NextWriteOffset;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	ui32NextWriteOffset = (psKCCBCtlLocal->ui32WriteOffset + 1) & psKCCBCtlLocal->ui32WrapMask;
+
+	if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+	{
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+		if (ui32NextWriteOffset != psKCCBCtlLocal->ui32ReadOffset)
+		{
+			return PVRSRV_OK;
+		}
+
+		/*
+		 * The following check doesn't impact performance, since the
+		 * CPU has to wait for the GPU anyway (full kernel CCB).
+		 */
+		if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+		{
+			return PVRSRV_ERROR_KERNEL_CCB_FULL;
+		}
+
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return PVRSRV_ERROR_KERNEL_CCB_FULL;
+}
+
+/******************************************************************************
+ FUNCTION	: RGXGetCmdMemCopySize
+
+ PURPOSE	: Calculates actual size of KCCB command getting used
+
+ PARAMETERS	: eCmdType     Type of KCCB command
+
+ RETURNS	: Returns actual size of KCCB command on success else zero
+******************************************************************************/
+static IMG_UINT32 RGXGetCmdMemCopySize(RGXFWIF_KCCB_CMD_TYPE eCmdType)
+{
+	/* First get offset of uCmdData inside the struct RGXFWIF_KCCB_CMD
+	 * This will account alignment requirement of uCmdData union
+	 *
+	 * Then add command-data size depending on command type to calculate actual
+	 * command size required to do mem copy
+	 *
+	 * NOTE: Make sure that uCmdData is the last member of RGXFWIF_KCCB_CMD struct.
+	 */
+	switch (eCmdType)
+	{
+		case RGXFWIF_KCCB_CMD_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_MMUCACHE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_MMUCACHEDATA);
+		}
+#if defined(SUPPORT_USC_BREAKPOINT)
+		case RGXFWIF_KCCB_CMD_BP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_BPDATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_SLCFLUSHINVAL:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_SLCFLUSHINVALDATA);
+		}
+		case RGXFWIF_KCCB_CMD_CLEANUP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CLEANUP_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_POW:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_POWER_REQUEST);
+		}
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE:
+		case RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_ZSBUFFER_BACKING_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELIST_GS_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_FREELISTS_RECONSTRUCTION_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_WRITE_OFFSET_UPDATE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_FORCE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_KCCB_CMD_FORCE_UPDATE_DATA);
+		}
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+		case RGXFWIF_KCCB_CMD_REGCONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_REGCONFIG_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_SELECT_CUSTOM_CNTRS);
+		}
+#if defined(SUPPORT_PDVFS)
+		case RGXFWIF_KCCB_CMD_PDVFS_LIMIT_MAX_FREQ:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_PDVFS_MAX_FREQ_DATA);
+		}
+#endif
+		case RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_OS_STATE_CHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_COUNTER_DUMP:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_COUNTER_DUMP_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_ENABLE_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CONFIG_DA_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_HWPERF_CTRL_BLKS);
+		}
+		case RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CORECLKSPEEDCHANGE_DATA);
+		}
+		case RGXFWIF_KCCB_CMD_VZ_DRV_ARRAY_CHANGE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE:
+		case RGXFWIF_KCCB_CMD_VZ_DRV_TIME_SLICE_INTERVAL:
+		case RGXFWIF_KCCB_CMD_WDG_CFG:
+		case RGXFWIF_KCCB_CMD_PHR_CFG:
+		case RGXFWIF_KCCB_CMD_HEALTH_CHECK:
+		case RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL:
+		{
+			/* No command specific data */
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData);
+		}
+		case RGXFWIF_KCCB_CMD_LOGTYPE_UPDATE:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_DEV_VIRTADDR);
+		}
+		case RGXFWIF_KCCB_CMD_CANCEL_WORK:
+		{
+			return offsetof(RGXFWIF_KCCB_CMD, uCmdData) + sizeof(RGXFWIF_CANCEL_WORK_DATA);
+		}
+		default:
+		{
+			/* Invalid (OR) Unused (OR) Newly added command type */
+			return 0; /* Error */
+		}
+	}
+}
+
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+	                           INVALIDATE);
+	eError = PVRSRVWaitForValueKM(
+	              (IMG_UINT32 __iomem *)&psDevInfo->pui32KernelCCBRtnSlots[ui32SlotNum],
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+				  RGXFwSharedMemCacheOpExecPfn);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVWaitForValueKM");
+
+#if defined(PDUMP)
+	/* PDumping conditions same as RGXSendCommandRaw for the actual command and poll command to go in harmony */
+	if (PDumpCheckFlagsWrite(psDevInfo->psDeviceNode, ui32PDumpFlags))
+	{
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Poll on KCCB slot %u for value %u (mask: 0x%x)", ui32SlotNum,
+					 RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED, RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED);
+
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+										ui32SlotNum * sizeof(IMG_UINT32),
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										RGXFWIF_KCCB_RTN_SLOT_CMD_EXECUTED,
+										PDUMP_POLL_OPERATOR_EQUAL,
+										ui32PDumpFlags);
+		PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXSendCommandRaw(PVRSRV_RGXDEV_INFO  *psDevInfo,
+									  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+									  IMG_UINT32          uiPDumpFlags,
+									  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE	*psDeviceNode = psDevInfo->psDeviceNode;
+	RGXFWIF_CCB_CTL		*psKCCBCtl;
+	RGXFWIF_CCB_CTL		*psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+	IMG_UINT8			*pui8KCCB = psDevInfo->psKernelCCB;
+	IMG_UINT32			ui32NewWriteOffset;
+	IMG_UINT32			ui32OldWriteOffset;
+	IMG_UINT32			ui32CmdMemCopySize;
+
+#if !defined(PDUMP)
+	PVR_UNREFERENCED_PARAMETER(uiPDumpFlags);
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+#else
+	IMG_BOOL bContCaptureOn = PDumpCheckFlagsWrite(psDeviceNode, PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER); /* client connected or in pdump init phase */
+	IMG_BOOL bPDumpEnabled = PDumpCheckFlagsWrite(psDeviceNode, uiPDumpFlags); /* Are we in capture range or continuous and not in a power transition */
+
+	psKCCBCtl = psDevInfo->psKernelCCBCtl;
+	ui32OldWriteOffset = psKCCBCtlLocal->ui32WriteOffset;
+
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			if (!psDevInfo->bDumpedKCCBCtlAlready)
+			{
+				/* entering capture range */
+				psDevInfo->bDumpedKCCBCtlAlready = IMG_TRUE;
+
+				/* Wait for the live FW to catch up */
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: waiting on fw to catch-up, roff: %d, woff: %d",
+						__func__,
+						psKCCBCtl->ui32ReadOffset, ui32OldWriteOffset));
+				PVRSRVPollForValueKM(psDeviceNode,
+				                     (IMG_UINT32 __iomem *)&psKCCBCtl->ui32ReadOffset,
+				                     ui32OldWriteOffset, 0xFFFFFFFF,
+				                     POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+				                     NULL);
+
+				/* Dump Init state of Kernel CCB control (read and write offset) */
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+						"Initial state of kernel CCB Control, roff: %d, woff: %d",
+						psKCCBCtl->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset);
+
+				DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+						0,
+						sizeof(RGXFWIF_CCB_CTL),
+						uiPDumpFlags);
+			}
+		}
+	}
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode))
+	{
+		if ((likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+			(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo)))) ||
+			(KM_FW_CONNECTION_IS(READY, psDevInfo) && KM_OS_CONNECTION_IS(READY, psDevInfo)))
+		{
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: The firmware-driver connection is invalid:"
+									"driver state = %u / firmware state = %u;"
+									"expected READY (%u/%u) or ACTIVE (%u/%u) or in transition (%u/%u);",
+									__func__, KM_GET_OS_CONNECTION(psDevInfo), KM_GET_FW_CONNECTION(psDevInfo),
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_READY,
+									RGXFW_CONNECTION_OS_ACTIVE, RGXFW_CONNECTION_FW_ACTIVE,
+									RGXFW_CONNECTION_OS_READY, RGXFW_CONNECTION_FW_ACTIVE));
+			eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+			goto _RGXSendCommandRaw_Exit;
+		}
+	}
+#endif
+
+	if (!OSLockIsLocked(psDeviceNode->hPowerLock))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called without power lock held!",
+				__func__));
+		PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+	}
+
+	/* Acquire a slot in the CCB */
+	eError = RGXAcquireKernelCCBSlot(psDevInfo, &ui32NewWriteOffset);
+	if (eError != PVRSRV_OK)
+	{
+		goto _RGXSendCommandRaw_Exit;
+	}
+
+	/* Calculate actual size of command to optimize device mem copy */
+	ui32CmdMemCopySize = RGXGetCmdMemCopySize(psKCCBCmd->eCmdType);
+	PVR_LOG_RETURN_IF_FALSE(ui32CmdMemCopySize !=0, "RGXGetCmdMemCopySize failed", PVRSRV_ERROR_INVALID_CCB_COMMAND);
+
+	/* Copy the command into the CCB */
+	OSCachedMemCopy(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)],
+	                psKCCBCmd, ui32CmdMemCopySize);
+	RGXFwSharedMemCacheOpExec(&pui8KCCB[ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD)], ui32CmdMemCopySize, PVRSRV_CACHE_OP_FLUSH);
+
+	/* If non-NULL pui32CmdKCCBSlot passed-in, return the kCCB slot in which the command was enqueued */
+	if (pui32CmdKCCBSlot)
+	{
+		*pui32CmdKCCBSlot = ui32OldWriteOffset;
+
+		/* Each such command enqueue needs to reset the slot value first. This is so that a caller
+		 * doesn't get to see stale/false value in allotted slot */
+		OSWriteDeviceMem32WithWMB(&psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                          RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32OldWriteOffset],
+		                           FLUSH);
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags,
+							  "Reset kCCB slot number %u", ui32OldWriteOffset);
+		DevmemPDumpLoadMem(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+		                   ui32OldWriteOffset * sizeof(IMG_UINT32),
+						   sizeof(IMG_UINT32),
+						   uiPDumpFlags);
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Device (%p) KCCB slot %u reset with value %u for command type %x",
+		         __func__, psDevInfo, ui32OldWriteOffset, RGXFWIF_KCCB_RTN_SLOT_NO_RESPONSE, psKCCBCmd->eCmdType));
+	}
+
+	/* Memory barrier before KCCB write offset update. */
+	OSWriteMemoryBarrier(NULL);
+
+
+	/* Move past the current command */
+	psKCCBCtlLocal->ui32WriteOffset = ui32NewWriteOffset;
+	psKCCBCtl->ui32WriteOffset = ui32NewWriteOffset;
+	/* Read-back of memory before Kick MTS */
+	OSWriteMemoryBarrier(&psKCCBCtl->ui32WriteOffset);
+	RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32WriteOffset, FLUSH);
+
+#if defined(PDUMP)
+	if (bContCaptureOn)
+	{
+		/* in capture range */
+		if (bPDumpEnabled)
+		{
+			/* Dump new Kernel CCB content */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCB cmd woff = %d",
+					ui32OldWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBMemDesc,
+					ui32OldWriteOffset * sizeof(RGXFWIF_KCCB_CMD),
+					ui32CmdMemCopySize,
+					uiPDumpFlags);
+
+			/* Dump new kernel CCB write offset */
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode,
+					uiPDumpFlags, "Dump kCCBCtl woff: %d",
+					ui32NewWriteOffset);
+			DevmemPDumpLoadMem(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					sizeof(IMG_UINT32),
+					uiPDumpFlags);
+
+			/* mimic the read-back of the write from above */
+			DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+					offsetof(RGXFWIF_CCB_CTL, ui32WriteOffset),
+					ui32NewWriteOffset,
+					0xFFFFFFFF,
+					PDUMP_POLL_OPERATOR_EQUAL,
+					uiPDumpFlags);
+		}
+		/* out of capture range */
+		else
+		{
+			eError = RGXPdumpDrainKCCB(psDevInfo, ui32OldWriteOffset);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXPdumpDrainKCCB", _RGXSendCommandRaw_Exit);
+		}
+	}
+#endif
+
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, uiPDumpFlags, "MTS kick for kernel CCB");
+	/*
+	 * Kick the MTS to schedule the firmware.
+	 */
+	__MTSScheduleWrite(psDevInfo, RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK);
+
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME, RGX_CR_MTS_SCHEDULE,
+	           RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK, uiPDumpFlags);
+
+#if defined(SUPPORT_AUTOVZ)
+	RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+
+#if defined(NO_HARDWARE)
+	/* keep the roff updated because fw isn't there to update it */
+	psKCCBCtl->ui32ReadOffset = psKCCBCtlLocal->ui32WriteOffset;
+#endif
+
+_RGXSendCommandRaw_Exit:
+	return eError;
+}
+
+/******************************************************************************
+ FUNCTION	: _AllocDeferredCommand
+
+ PURPOSE	: Allocate a KCCB command and add it to KCCB deferred list
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: eKCCBType		Firmware Command type
+			: psKCCBCmd		Firmware Command
+			: uiPDumpFlags	Pdump flags
+
+ RETURNS	: PVRSRV_OK	If all went good, PVRSRV_ERROR_RETRY otherwise.
+******************************************************************************/
+static PVRSRV_ERROR _AllocDeferredCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                          RGXFWIF_KCCB_CMD   *psKCCBCmd,
+                                          IMG_UINT32         uiPDumpFlags)
+{
+	RGX_DEFERRED_KCCB_CMD *psDeferredCommand;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psDeferredCommand = OSAllocMem(sizeof(*psDeferredCommand));
+
+	if (!psDeferredCommand)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Deferring a KCCB command failed: allocation failure: requesting retry"));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psDeferredCommand->sKCCBcmd = *psKCCBCmd;
+	psDeferredCommand->uiPDumpFlags = uiPDumpFlags;
+	psDeferredCommand->psDevInfo = psDevInfo;
+
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_add_to_tail(&(psDevInfo->sKCCBDeferredCommandsListHead), &(psDeferredCommand->sListNode));
+	psDevInfo->ui32KCCBDeferredCommandsCount++;
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ FUNCTION	: _FreeDeferredCommand
+
+ PURPOSE	: Remove from the deferred list the sent deferred KCCB command
+
+ PARAMETERS	: psNode			Node in deferred list
+			: psDeferredKCCBCmd	KCCB Command to free
+
+ RETURNS	: None
+******************************************************************************/
+static void _FreeDeferredCommand(DLLIST_NODE *psNode, RGX_DEFERRED_KCCB_CMD *psDeferredKCCBCmd)
+{
+	dllist_remove_node(psNode);
+	psDeferredKCCBCmd->psDevInfo->ui32KCCBDeferredCommandsCount--;
+	OSFreeMem(psDeferredKCCBCmd);
+}
+
+/******************************************************************************
+ FUNCTION	: RGXSendCommandsFromDeferredList
+
+ PURPOSE	: Try send KCCB commands in deferred list to KCCB
+		  Should be called by holding PowerLock
+
+ PARAMETERS	: psDevInfo	RGX device info
+			: bPoll		Poll for space in KCCB
+
+ RETURNS	: PVRSRV_OK	If all commands in deferred list are sent to KCCB,
+			  PVRSRV_ERROR_KERNEL_CCB_FULL otherwise.
+******************************************************************************/
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	DLLIST_NODE *psNode, *psNext;
+	RGX_DEFERRED_KCCB_CMD *psTempDeferredKCCBCmd;
+	DLLIST_NODE sCommandList;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	PVR_ASSERT(PVRSRVPwrLockIsLockedByMe(psDevInfo->psDeviceNode));
+
+	/* !!! Important !!!
+	 *
+	 * The idea of moving the whole list hLockKCCBDeferredCommandsList below
+	 * to the temporary list is only valid under the principle that all of the
+	 * operations are also protected by the power lock. It must be held
+	 * so that the order of the commands doesn't get messed up while we're
+	 * performing the operations on the local list.
+	 *
+	 * The necessity of releasing the hLockKCCBDeferredCommandsList comes from
+	 * the fact that _FreeDeferredCommand() is allocating memory and it can't
+	 * be done in atomic context (inside section protected by a spin lock).
+	 *
+	 * We're using spin lock here instead of mutex to quickly perform a check
+	 * if the list is empty in MISR without a risk that the MISR is going
+	 * to sleep due to a lock.
+	 */
+
+	/* move the whole list to a local list so it can be processed without lock */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_replace_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		if (dllist_is_empty(&sCommandList))
+		{
+			return PVRSRV_OK;
+		}
+
+		/* For every deferred KCCB command, try to send it*/
+		dllist_foreach_node(&sCommandList, psNode, psNext)
+		{
+			psTempDeferredKCCBCmd = IMG_CONTAINER_OF(psNode, RGX_DEFERRED_KCCB_CMD, sListNode);
+			eError = RGXSendCommandRaw(psTempDeferredKCCBCmd->psDevInfo,
+			                           &psTempDeferredKCCBCmd->sKCCBcmd,
+			                           psTempDeferredKCCBCmd->uiPDumpFlags,
+			                           NULL /* We surely aren't interested in kCCB slot number of deferred command */);
+			if (eError != PVRSRV_OK)
+			{
+				if (!bPoll)
+				{
+					eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+					goto cleanup_;
+				}
+				break;
+			}
+
+			_FreeDeferredCommand(psNode, psTempDeferredKCCBCmd);
+		}
+
+		if (bPoll)
+		{
+			PVRSRV_ERROR eErrPollForKCCBSlot;
+
+			/* Don't overwrite eError because if RGXPollKernelCCBSlot returns OK and the
+			 * outer loop times-out, we'll still want to return KCCB_FULL to caller
+			 */
+			eErrPollForKCCBSlot = RGXPollKernelCCBSlot(psDevInfo);
+			if (eErrPollForKCCBSlot == PVRSRV_ERROR_KERNEL_CCB_FULL)
+			{
+				eError = PVRSRV_ERROR_KERNEL_CCB_FULL;
+				goto cleanup_;
+			}
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+cleanup_:
+	/* if the local list is not empty put it back to the deferred list head
+	 * so that the old order of commands is retained */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	dllist_insert_list_at_head(&psDevInfo->sKCCBDeferredCommandsListHead, &sCommandList);
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+										  RGXFWIF_KCCB_CMD    *psKCCBCmd,
+										  IMG_UINT32          uiPDumpFlags,
+										  IMG_UINT32          *pui32CmdKCCBSlot)
+{
+	IMG_BOOL     bPoll = (pui32CmdKCCBSlot != NULL);
+	PVRSRV_ERROR eError;
+
+	/*
+	 * First try to Flush all the cmds in deferred list.
+	 *
+	 * We cannot defer an incoming command if the caller is interested in
+	 * knowing the command's kCCB slot: it plans to poll/wait for a
+	 * response from the FW just after the command is enqueued, so we must
+	 * poll for space to be available.
+	 */
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, bPoll);
+	if (eError == PVRSRV_OK)
+	{
+		eError = RGXSendCommandRaw(psDevInfo,
+								   psKCCBCmd,
+								   uiPDumpFlags,
+								   pui32CmdKCCBSlot);
+	}
+
+	/*
+	 * If we don't manage to enqueue one of the deferred commands or the command
+	 * passed as argument because the KCCB is full, insert the latter into the deferred commands list.
+	 * The deferred commands will also be flushed eventually by:
+	 *  - one more KCCB command sent for any DM
+	 *  - RGX_MISRHandler_CheckFWActivePowerState
+	 */
+	if (eError == PVRSRV_ERROR_KERNEL_CCB_FULL)
+	{
+		if (pui32CmdKCCBSlot == NULL)
+		{
+			eError = _AllocDeferredCommand(psDevInfo, psKCCBCmd, uiPDumpFlags);
+		}
+		else
+		{
+			/* Let the caller retry. Otherwise if we deferred the command and returned OK,
+			 * the caller can end up looking in a stale CCB slot.
+			 */
+			PVR_DPF((PVR_DBG_WARNING, "%s: Couldn't flush the deferred queue for a command (Type:%d) "
+			                        "- will be retried", __func__, psKCCBCmd->eCmdType));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32         *pui32CmdKCCBSlot)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+
+	/* Ensure Rogue is powered up before kicking MTS */
+	eError = PVRSRVPowerLock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: failed to acquire powerlock (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVPowerLock_Exit;
+	}
+
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition Rogue to ON (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+										  psKCCBCmd,
+										  ui32PDumpFlags,
+	                                      pui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to schedule command (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+#if defined(DEBUG)
+		/* PVRSRVDebugRequest must be called without powerlock */
+		PVRSRVPowerUnlock(psDeviceNode);
+		PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+		goto _PVRSRVPowerLock_Exit;
+#endif
+	}
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_PVRSRVPowerLock_Exit:
+	return eError;
+}
+
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE*) hCmdCompHandle;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSScheduleMISR(psDevInfo->hProcessQueuesMISR);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGX_MISRHandler_ScheduleProcessQueues
+
+ @Description - Sends uncounted kick to all the DMs (the FW will process all
+				the queue for all the DMs)
+******************************************************************************/
+static void RGX_MISRHandler_ScheduleProcessQueues(void *pvData)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO     *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR           eError;
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_OFF;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Check whether it's worth waking up the GPU */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guests are not permitted to change the device power state */
+		if ((eError != PVRSRV_OK) || (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+	else
+	{
+		if ((eError == PVRSRV_OK) && (ePowerState == PVRSRV_DEV_POWER_STATE_OFF))
+		{
+			RGXFWIF_GPU_UTIL_FW    *psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+			IMG_BOOL               bGPUHasWorkWaiting;
+
+			/* Check whether it's worth waking up the GPU */
+			RGXFwSharedMemCacheOpValue(psUtilFW->ui64GpuLastWord, INVALIDATE);
+			bGPUHasWorkWaiting =
+			    (RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord) == RGXFWIF_GPU_UTIL_STATE_BLOCKED);
+
+			if (!bGPUHasWorkWaiting)
+			{
+				/* all queues are empty, don't wake up the GPU */
+				PVRSRVPowerUnlock(psDeviceNode);
+				return;
+			}
+		}
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		/* wake up the GPU */
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+											 PVRSRV_DEV_POWER_STATE_ON,
+											 PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition device to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+
+			PVRSRVPowerUnlock(psDeviceNode);
+			return;
+		}
+	}
+
+	/* Memory barrier before Kick MTS */
+	OSWriteMemoryBarrier(NULL);
+
+	/* uncounted kick to the FW */
+	HTBLOGK(HTB_SF_MAIN_KICK_UNCOUNTED);
+	__MTSScheduleWrite(psDevInfo, (RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK) | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return OSInstallMISR(phMISR,
+			RGX_MISRHandler_ScheduleProcessQueues,
+			psDeviceNode,
+			"RGX_ScheduleProcessQueues");
+}
+
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO  *psDevInfo,
+                                              RGXFWIF_DM          eKCCBType,
+                                              RGXFWIF_KCCB_CMD    *psKCCBCmd,
+                                              IMG_UINT32          ui32PDumpFlags,
+                                              IMG_UINT32          *pui32CmdKCCBSlot,
+                                              IMG_BOOL            bCallerHasPwrLock)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 uiMMUSyncUpdate;
+
+	/* Don't send the command/power up request if device not available. */
+	if (unlikely((psDevInfo == NULL) ||
+	             (psDevInfo->psDeviceNode == NULL) ||
+				 (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)))
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	/* Don't send the command/power up request if device in deinit phase.
+	 * The de-init thread could destroy the device whilst the power up
+	 * sequence below is accessing the HW registers.
+	 * Not yet safe to free resources. Caller should retry later.
+	 */
+	if (psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	    psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_ERROR_RETRY;
+	}
+
+
+
+	if (!bCallerHasPwrLock)
+	{
+		/* PVRSRVPowerLock guarantees atomicity between commands. This is helpful
+		   in a scenario with several applications allocating resources. */
+		eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+
+			/* If system is found powered OFF, Retry scheduling the command */
+			if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+			{
+				eError = PVRSRV_ERROR_RETRY;
+			}
+
+			goto RGXScheduleCommand_exit;
+		}
+	}
+
+	if (unlikely(psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+	             psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING))
+	{
+		/* If we have the power lock the device is valid but the deinit
+		 * thread could be waiting for the lock. */
+		eError = PVRSRV_ERROR_RETRY;
+		goto _PVRSRVInvalidDeviceError_Exit;
+	}
+
+	/* Ensure device is powered up before sending any commands */
+	PDUMPPOWCMDSTART(psDevInfo->psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDevInfo->psDeviceNode,
+	                                     PVRSRV_DEV_POWER_STATE_ON,
+	                                     PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		goto _PVRSRVSetDevicePowerStateKM_Exit;
+	}
+
+	eError = RGXPreKickCacheCommand(psDevInfo, eKCCBType, &uiMMUSyncUpdate);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot);
+	if (unlikely(eError != PVRSRV_OK)) goto _PVRSRVSetDevicePowerStateKM_Exit;
+
+_PVRSRVSetDevicePowerStateKM_Exit:
+_PVRSRVInvalidDeviceError_Exit:
+	if (!bCallerHasPwrLock)
+	{
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+	}
+RGXScheduleCommand_exit:
+	return eError;
+}
+
+/*
+ * RGXCheckFirmwareCCB
+ */
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_CCB_CTL *psFWCCBCtl = psDevInfo->psFirmwareCCBCtl;
+	RGXFWIF_CCB_CTL *psFWCCBCtlLocal = psDevInfo->psFirmwareCCBCtlLocal;
+	IMG_UINT8 *psFWCCB = psDevInfo->psFirmwareCCB;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	PVR_LOG_RETURN_VOID_IF_FALSE(PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo) ||
+								 (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+								  (KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))),
+								 "FW-KM connection is down");
+#endif
+
+	eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+		return;
+	}
+
+	while (psFWCCBCtlLocal->ui32ReadOffset != psFWCCBCtlLocal->ui32WriteOffset)
+	{
+		/* Point to the next command */
+		const RGXFWIF_FWCCB_CMD *psFwCCBCmd = ((RGXFWIF_FWCCB_CMD *)psFWCCB) + psFWCCBCtlLocal->ui32ReadOffset;
+		RGXFwSharedMemCacheOpPtr(psFwCCBCmd, INVALIDATE);
+
+
+		HTBLOGK(HTB_SF_MAIN_FWCCB_CMD, psFwCCBCmd->eCmdType);
+		switch (psFwCCBCmd->eCmdType)
+		{
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_BACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_BACKING, "Request to add backing to ZSBuffer");
+				}
+				RGXProcessRequestZSBufferBacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_ZSBUFFER_UNBACKING:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, ZSBUFFER_UNBACKING, "Request to remove backing from ZSBuffer");
+				}
+				RGXProcessRequestZSBufferUnbacking(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdZSBufferBacking.ui32ZSBufferID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELIST_GROW:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELIST_GROW, "Request to grow the free list");
+				}
+				RGXProcessRequestGrow(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListGS.ui32FreelistID);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
+			{
+				if (psDevInfo->bPDPEnabled)
+				{
+					PDUMP_PANIC(psDevInfo->psDeviceNode, FREELISTS_RECONSTRUCTION, "Request to reconstruct free lists");
+				}
+
+				if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "%s: Freelist reconstruction request (%d/%d) for %d freelists",
+							__func__,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32HwrCounter+1,
+							psDevInfo->psRGXFWIfHWRInfoBufCtl->ui32HwrCounter+1,
+							psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount));
+				}
+
+				RGXProcessRequestFreelistsReconstruction(psDevInfo,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.ui32FreelistsCount,
+				        psFwCCBCmd->uCmdData.sCmdFreeListsReconstruction.aui32FreelistIDs);
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_FW_PF_NOTIFICATION:
+			{
+				/* Notify client drivers */
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+					const RGXFWIF_FWCCB_CMD_FW_PAGEFAULT_DATA *psCmdFwPagefault =
+							&psFwCCBCmd->uCmdData.sCmdFWPagefault;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT;
+						sErrorData.uErrData.sFwPFErrData.sFWFaultAddr.uiAddr = psCmdFwPagefault->sFWFaultAddr.uiAddr;
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig,
+														  &sErrorData);
+					}
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
+			{
+				const RGXFWIF_FWCCB_CMD_CONTEXT_RESET_DATA *psCmdContextResetNotification =
+						&psFwCCBCmd->uCmdData.sCmdContextResetNotification;
+				IMG_UINT32 ui32ErrorPid = 0;
+
+				FWCommonContextListSetLastResetReason(psDevInfo,
+				                                      &ui32ErrorPid,
+				                                      psCmdContextResetNotification);
+
+				/* Increment error counter (if appropriate) */
+				if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32WGPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32WGPErrorCount++;
+					}
+				}
+				else if (psCmdContextResetNotification->eResetReason == RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM)
+				{
+					/* Avoid wrapping the error count (which would then
+					 * make it appear we had far fewer errors), by limiting
+					 * it to IMG_UINT32_MAX.
+					 */
+					if (psDevInfo->sErrorCounts.ui32TRPErrorCount < IMG_UINT32_MAX)
+					{
+						psDevInfo->sErrorCounts.ui32TRPErrorCount++;
+					}
+				}
+
+				/* Notify system layer */
+				{
+					PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+					PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+					if (psDevConfig->pfnSysDevErrorNotify)
+					{
+						PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+						sErrorData.eResetReason = psCmdContextResetNotification->eResetReason;
+						sErrorData.pid = ui32ErrorPid;
+
+						/* Populate error data according to reset reason */
+						switch (psCmdContextResetNotification->eResetReason)
+						{
+							case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+							case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+							{
+								sErrorData.uErrData.sChecksumErrData.ui32ExtJobRef = psCmdContextResetNotification->ui32ResetJobRef;
+								sErrorData.uErrData.sChecksumErrData.eDM = psCmdContextResetNotification->eDM;
+								break;
+							}
+							default:
+							{
+								break;
+							}
+						}
+
+						psDevConfig->pfnSysDevErrorNotify(psDevConfig,
+						                                  &sErrorData);
+					}
+				}
+
+				/* Notify if a page fault */
+				if (psCmdContextResetNotification->ui32Flags & RGXFWIF_FWCCB_CMD_CONTEXT_RESET_FLAG_PF)
+				{
+					DevmemIntPFNotify(psDevInfo->psDeviceNode,
+							psCmdContextResetNotification->ui64PCAddress,
+							psCmdContextResetNotification->sFaultAddress);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_DEBUG_DUMP:
+			{
+				PVRSRV_ERROR eError;
+				PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+				OSAtomicWrite(&psDevInfo->psDeviceNode->eDebugDumpRequested, PVRSRV_DEVICE_DEBUG_DUMP_CAPTURE);
+				eError = OSEventObjectSignal(psPVRSRVData->hDevicesWatchdogEvObj);
+				if (eError != PVRSRV_OK)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Failed to signal FW Cmd debug dump event, dumping now instead", __func__));
+					PVRSRVDebugRequest(psDevInfo->psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+				}
+				break;
+			}
+
+			case RGXFWIF_FWCCB_CMD_UPDATE_STATS:
+			{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+				IMG_PID pidTmp = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.pidOwner;
+				IMG_INT32 i32AdjustmentValue = psFwCCBCmd->uCmdData.sCmdUpdateStatsData.i32AdjustmentValue;
+
+				switch (psFwCCBCmd->uCmdData.sCmdUpdateStatsData.eElementToUpdate)
+				{
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_PARTIAL_RENDERS:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,i32AdjustmentValue,0,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_OUT_OF_MEMORY:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,i32AdjustmentValue,0,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TA_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,i32AdjustmentValue,0,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_3D_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,i32AdjustmentValue,0,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_CDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,i32AdjustmentValue,0,0,pidTmp);
+						break;
+					}
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_TDM_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,i32AdjustmentValue,0,pidTmp);
+						break;
+					}
+#if defined(RGX_FEATURE_RAY_TRACING_ARCH_MAX_VALUE_IDX)
+					case RGXFWIF_FWCCB_CMD_UPDATE_NUM_RAY_STORES:
+					{
+						PVRSRVStatsUpdateRenderContextStats(psDevInfo->psDeviceNode,0,0,0,0,0,0,i32AdjustmentValue,pidTmp);
+						break;
+					}
+#endif
+				}
+#endif
+				break;
+			}
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+			case RGXFWIF_FWCCB_CMD_CORE_CLK_RATE_CHANGE:
+			{
+				RGX_PROCESS_CORE_CLK_RATE_CHANGE(psDevInfo,
+											  psFwCCBCmd->uCmdData.sCmdCoreClkRateChange.ui32CoreClkRate);
+				break;
+			}
+#endif
+			case RGXFWIF_FWCCB_CMD_REQUEST_GPU_RESTART:
+			{
+				RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+				if (psDevInfo->psRGXFWIfFwSysData != NULL  &&
+				    psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)
+				{
+					PVRSRV_ERROR eError;
+
+					/* Power down... */
+					eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+															 PVRSRV_SYS_POWER_STATE_OFF,
+															 PVRSRV_POWER_FLAGS_NONE);
+					if (eError == PVRSRV_OK)
+					{
+						/* Clear the FW faulted flags... */
+						psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags &= ~(RGXFWIF_HWR_FW_FAULT|RGXFWIF_HWR_RESTART_REQUESTED);
+						OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags);
+						RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ui32HWRStateFlags,
+						                           FLUSH);
+
+						/* Power back up again... */
+						eError = PVRSRVSetDeviceSystemPowerState(psDevInfo->psDeviceNode,
+																 PVRSRV_SYS_POWER_STATE_ON,
+																 PVRSRV_POWER_FLAGS_NONE);
+
+						/* Send a dummy KCCB command to ensure the FW wakes up and checks the queues... */
+						if (eError == PVRSRV_OK)
+						{
+							LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+							{
+								eError = RGXFWHealthCheckCmd(psDevInfo);
+								if (eError != PVRSRV_ERROR_RETRY)
+								{
+									break;
+								}
+								OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+							} END_LOOP_UNTIL_TIMEOUT_US();
+						}
+					}
+
+					/* Notify client drivers and system layer of FW fault */
+					{
+						PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+						PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+						/* Client notification of device error will be achieved by
+						 * clients calling UM function RGXGetLastDeviceError() */
+						psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+
+						/* Notify system layer */
+						if (psDevConfig->pfnSysDevErrorNotify)
+						{
+							PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+							sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR;
+							psDevConfig->pfnSysDevErrorNotify(psDevConfig,
+							                                  &sErrorData);
+						}
+					}
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Failed firmware restart (%s)",
+								 __func__, PVRSRVGetErrorString(eError)));
+					}
+				}
+				break;
+			}
+			default:
+			{
+				/* unknown command */
+				PVR_DPF((PVR_DBG_WARNING, "%s: Unknown Command (eCmdType=0x%08x)",
+				         __func__, psFwCCBCmd->eCmdType));
+				/* Assert on magic value corruption */
+				PVR_ASSERT((((IMG_UINT32)psFwCCBCmd->eCmdType & RGX_CMD_MAGIC_DWORD_MASK) >> RGX_CMD_MAGIC_DWORD_SHIFT) == RGX_CMD_MAGIC_DWORD);
+			}
+		}
+
+		/* Update read offset */
+		psFWCCBCtlLocal->ui32ReadOffset = (psFWCCBCtlLocal->ui32ReadOffset + 1) & psFWCCBCtlLocal->ui32WrapMask;
+		OSMemoryBarrier(NULL);
+		psFWCCBCtl->ui32ReadOffset = psFWCCBCtlLocal->ui32ReadOffset;
+		OSWriteMemoryBarrier(NULL);
+
+		if (psFWCCBCtlLocal->ui32ReadOffset == psFWCCBCtlLocal->ui32WriteOffset)
+		{
+			eError = RGXUpdateLocalFWCCBWoff(psDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "RGXUpdateLocalFWCCBWoff");
+				return;
+			}
+		}
+	}
+}
+
+/*
+ * PVRSRVRGXFrameworkCopyCommand
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+		DEVMEM_MEMDESC	*psFWFrameworkMemDesc,
+		IMG_PBYTE		pbyGPUFRegisterList,
+		IMG_UINT32		ui32FrameworkRegisterSize)
+{
+	PVRSRV_ERROR	eError;
+	RGXFWIF_RF_REGISTERS	*psRFReg;
+
+	eError = DevmemAcquireCpuVirtAddr(psFWFrameworkMemDesc,
+			(void **)&psRFReg);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	OSDeviceMemCopy(psRFReg, pbyGPUFRegisterList, ui32FrameworkRegisterSize);
+	RGXFwSharedMemCacheOpPtr(psRFReg, FLUSH);
+
+	/* Release the CPU mapping */
+	DevmemReleaseCpuVirtAddr(psFWFrameworkMemDesc);
+
+	/*
+	 * Dump the FW framework buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FWFramework buffer");
+	DevmemPDumpLoadMem(psFWFrameworkMemDesc, 0, ui32FrameworkRegisterSize, PDUMP_FLAGS_CONTINUOUS);
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ * PVRSRVRGXFrameworkCreateKM
+*/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE	*psDeviceNode,
+		DEVMEM_MEMDESC		**ppsFWFrameworkMemDesc,
+		IMG_UINT32			ui32FrameworkCommandSize)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+		Allocate device memory for the firmware GPU framework state.
+		Sufficient info to kick one or more DMs should be contained in this buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware framework state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+			ui32FrameworkCommandSize,
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwGPUFrameworkState",
+			ppsFWFrameworkMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware framework state (%u)",
+				__func__, eError));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE  *psDevNode,
+												volatile IMG_UINT32	__iomem *pui32LinMemAddr,
+												IMG_UINT32			ui32Value,
+												IMG_UINT32			ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32CurrentQueueLength, ui32MaxRetries;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDevNode->pvDevice;
+	RGXFWIF_CCB_CTL *psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+	eError = RGXUpdateLocalKCCBRoff(psDevInfo);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXUpdateLocalKCCBRoff");
+
+	ui32CurrentQueueLength = (psKCCBCtlLocal->ui32WrapMask+1 +
+					psKCCBCtlLocal->ui32WriteOffset -
+					psKCCBCtlLocal->ui32ReadOffset) & psKCCBCtlLocal->ui32WrapMask;
+	ui32CurrentQueueLength += psDevInfo->ui32KCCBDeferredCommandsCount;
+
+	for (ui32MaxRetries = ui32CurrentQueueLength + 1;
+				ui32MaxRetries > 0;
+				ui32MaxRetries--)
+	{
+
+		/*
+		 * PVRSRVPollForValueKM flags are set to POLL_FLAG_NONE in this case so that the function
+		 * does not generate an error message. In this case, the PollForValueKM is expected to
+		 * timeout as there is work ongoing on the GPU which may take longer than the timeout period.
+		 */
+		eError = PVRSRVPollForValueKM(psDevNode, pui32LinMemAddr, ui32Value, ui32Mask, POLL_FLAG_NONE, NULL);
+		if (eError != PVRSRV_ERROR_TIMEOUT)
+		{
+			break;
+		}
+
+		RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+	}
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed! Error(%s) CPU linear address(%p) Expected value(%u)",
+		         __func__, PVRSRVGetErrorString(eError),
+		         pui32LinMemAddr, ui32Value));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32ConfigState,
+				IMG_BOOL bSetNotClear)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	RGXFWIF_KCCB_CMD sStateFlagCmd = { 0 };
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_SYSDATA *psFwSysData;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_BOOL bWaitForFwUpdate = IMG_FALSE;
+
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (NULL == psFwSysData)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Fw Sys Config is not mapped into CPU space", __func__));
+		return PVRSRV_ERROR_INVALID_CPU_ADDR;
+	}
+
+	/* apply change and ensure the new data is written to memory
+	 * before requesting the FW to read it
+	 */
+	ui32Config = ui32Config & RGXFWIF_INICFG_ALL;
+	if (bSetNotClear)
+	{
+		psFwSysData->ui32ConfigFlags |= ui32Config;
+	}
+	else
+	{
+		psFwSysData->ui32ConfigFlags &= ~ui32Config;
+	}
+	OSWriteMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+	RGXFwSharedMemCacheOpValue(psFwSysData->ui32ConfigFlags, FLUSH);
+
+	/* return current/new value to caller */
+	if (pui32ConfigState)
+	{
+		*pui32ConfigState = psFwSysData->ui32ConfigFlags;
+	}
+
+	OSMemoryBarrier(&psFwSysData->ui32ConfigFlags);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerLock");
+
+	/* notify FW to update setting */
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		/* Ask the FW to update its cached version of the value */
+		sStateFlagCmd.eCmdType = RGXFWIF_KCCB_CMD_STATEFLAGS_CTRL;
+
+		eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+											  &sStateFlagCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSendCommandAndGetKCCBSlot", unlock);
+		bWaitForFwUpdate = IMG_TRUE;
+	}
+
+unlock:
+	PVRSRVPowerUnlock(psDeviceNode);
+	if (bWaitForFwUpdate)
+	{
+		/* Wait for the value to be updated as the FW validates
+		 * the parameters and modifies the ui32ConfigFlags
+		 * accordingly
+		 * (for completeness as registered callbacks should also
+		 *  not permit invalid transitions)
+		 */
+		eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+		PVR_LOG_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+	}
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXScheduleCleanupCommand(PVRSRV_RGXDEV_INFO	*psDevInfo,
+									   RGXFWIF_DM			eDM,
+									   RGXFWIF_KCCB_CMD		*psKCCBCmd,
+									   RGXFWIF_CLEANUP_TYPE	eCleanupType,
+									   IMG_UINT32			ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32kCCBCommandSlot;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	PVRSRV_DEVICE_HEALTH_STATUS eHealthStatus = OSAtomicRead(&psDevInfo->psDeviceNode->eHealthStatus);
+
+	PVR_LOG_RETURN_IF_FALSE((eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+							(eHealthStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT),
+							"Cleanup aborted: Device in bad state", PVRSRV_OK);
+#endif
+
+
+	/* Clean-up commands sent during frame capture intervals must be dumped even when not in capture range... */
+	ui32PDumpFlags |= PDUMP_FLAGS_INTERVAL;
+
+	psKCCBCmd->eCmdType = RGXFWIF_KCCB_CMD_CLEANUP;
+	psKCCBCmd->uCmdData.sCleanupData.eCleanupType = eCleanupType;
+
+	/*
+		Send the cleanup request to the firmware. If the resource is still busy
+		the firmware will tell us and we'll drop out with a retry.
+	*/
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+											  eDM,
+											  psKCCBCmd,
+											  ui32PDumpFlags,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR ,"RGXScheduleCommandAndGetKCCBSlot() failed (%s) in %s()",
+			         PVRSRVGETERRORSTRING(eError), __func__));
+		}
+		goto fail_command;
+	}
+
+	/* Wait for command kCCB slot to be updated by FW */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+						  "Wait for the firmware to reply to the cleanup command");
+	eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot,
+									  ui32PDumpFlags);
+	/*
+		If the firmware hasn't got back to us in a timely manner
+		then bail and let the caller retry the command.
+	 */
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: RGXWaitForKCCBSlotUpdate timed out. Dump debug information.",
+		         __func__));
+
+		eError = PVRSRV_ERROR_RETRY;
+#if defined(DEBUG)
+		PVRSRVDebugRequest(psDevInfo->psDeviceNode,
+				DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+#endif
+		goto fail_poll;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		goto fail_poll;
+	}
+
+#if defined(PDUMP)
+	/*
+	 * The cleanup request to the firmware will tell us if a given resource is busy or not.
+	 * If the RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY flag is set, this means that the resource is
+	 * still in use. In this case we return a PVRSRV_ERROR_RETRY error to the client drivers
+	 * and they will re-issue the cleanup request until it succeed.
+	 *
+	 * Since this retry mechanism doesn't work for pdumps, client drivers should ensure
+	 * that cleanup requests are only submitted if the resource is unused.
+	 * If this is not the case, the following poll will block infinitely, making sure
+	 * the issue doesn't go unnoticed.
+	 */
+	PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, ui32PDumpFlags,
+					"Cleanup: If this poll fails, the following resource is still in use (DM=%u, type=%u, address=0x%08x), which is incorrect in pdumps",
+					eDM,
+					psKCCBCmd->uCmdData.sCleanupData.eCleanupType,
+					psKCCBCmd->uCmdData.sCleanupData.uCleanupData.psContext.ui32Addr);
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBRtnSlotsMemDesc,
+									ui32kCCBCommandSlot * sizeof(IMG_UINT32),
+									0,
+									RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY,
+									PDUMP_POLL_OPERATOR_EQUAL,
+									ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+#endif
+
+	/*
+		If the command has was run but a resource was busy, then the request
+		will need to be retried.
+	*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot],
+	                           INVALIDATE);
+
+	if (unlikely(psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_CLEANUP_BUSY))
+	{
+		if (psDevInfo->pui32KernelCCBRtnSlots[ui32kCCBCommandSlot] & RGXFWIF_KCCB_RTN_SLOT_POLL_FAILURE)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: FW poll on a HW operation failed", __func__));
+		}
+		eError = PVRSRV_ERROR_RETRY;
+		goto fail_requestbusy;
+	}
+
+	return PVRSRV_OK;
+
+fail_requestbusy:
+fail_poll:
+fail_command:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+	RGXRequestCommonContextCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD			sRCCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+	PRGXFWIF_FWCOMMONCONTEXT	psFWCommonContextFWAddr;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	/* Force retry if this context's CCB is currently being dumped
+	 * as part of the stalled CCB debug */
+	if (psDevInfo->pvEarliestStalledClientCCB == (void*)FWCommonContextGetClientCCB(psServerCommonContext))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: Forcing retry as psDevInfo->pvEarliestStalledClientCCB = psCtxClientCCB <%p>",
+		         __func__,
+		         psDevInfo->pvEarliestStalledClientCCB));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	psFWCommonContextFWAddr = FWCommonContextGetFWAddress(psServerCommonContext);
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Common ctx cleanup Request DM%d [context = 0x%08x]",
+			eDM, psFWCommonContextFWAddr.ui32Addr);
+	PDUMPCOMMENT(psDeviceNode, "Wait for CCB to be empty before common ctx cleanup");
+
+	RGXCCBPDumpDrainCCB(FWCommonContextGetClientCCB(psServerCommonContext), ui32PDumpFlags);
+#endif
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sRCCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psContext = psFWCommonContextFWAddr;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+									   eDM,
+									   &sRCCleanUpCmd,
+									   RGXFWIF_CLEANUP_FWCOMMONCONTEXT,
+									   ui32PDumpFlags);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+/*
+ * RGXFWRequestHWRTDataCleanUp
+ */
+
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                         PRGXFWIF_HWRTDATA psHWRTData)
+{
+	RGXFWIF_KCCB_CMD			sHWRTDataCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDeviceNode, "HW RTData cleanup Request [HWRTData = 0x%08x]", psHWRTData.ui32Addr);
+
+	sHWRTDataCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psHWRTData = psHWRTData;
+
+	eError = RGXScheduleCleanupCommand(psDeviceNode->pvDevice,
+	                                   RGXFWIF_DM_GP,
+	                                   &sHWRTDataCleanUpCmd,
+	                                   RGXFWIF_CLEANUP_HWRTDATA,
+	                                   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a HWRTData cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestFreeListCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_FREELIST psFWFreeList)
+{
+	RGXFWIF_KCCB_CMD			sFLCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free list cleanup Request [FreeList = 0x%08x]", psFWFreeList.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sFLCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psFreelist = psFWFreeList;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_GP,
+									   &sFLCleanUpCmd,
+									   RGXFWIF_CLEANUP_FREELIST,
+									   PDUMP_FLAGS_NONE);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* If caller may retry, fail with no error message */
+		if (!PVRSRVIsRetryError(eError))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to schedule a memory context cleanup with error (%u)",
+			         __func__, eError));
+		}
+	}
+
+	return eError;
+}
+
+/*
+	RGXFWRequestZSBufferCleanUp
+*/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer)
+{
+	RGXFWIF_KCCB_CMD			sZSBufferCleanUpCmd = {0};
+	PVRSRV_ERROR				eError;
+
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "ZS Buffer cleanup Request [ZS Buffer = 0x%08x]", psFWZSBuffer.ui32Addr);
+
+	/* Setup our command data, the cleanup call will fill in the rest */
+	sZSBufferCleanUpCmd.uCmdData.sCleanupData.uCleanupData.psZSBuffer = psFWZSBuffer;
+
+	/* Request cleanup of the firmware resource */
+	eError = RGXScheduleCleanupCommand(psDevInfo,
+									   RGXFWIF_DM_3D,
+									   &sZSBufferCleanUpCmd,
+									   RGXFWIF_CLEANUP_ZSBUFFER,
+									   PDUMP_FLAGS_NONE);
+
+	if ((eError != PVRSRV_OK) && !PVRSRVIsRetryError(eError))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule a memory context cleanup with error (%u)",
+		         __func__, eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS = ui32HCSDeadlineMs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32HCSDeadlineMS, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Hard Context Switching deadline inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32HCSDeadlineMS),
+							  ui32HCSDeadlineMs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock)
+{
+	RGXFWIF_KCCB_CMD	sCmpKCCBCmd = { 0 };
+
+	sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+	if (bCallerHasPwrLock)
+	{
+		return	RGXScheduleCommandWithoutPowerLock(psDevInfo,
+												   RGXFWIF_DM_GP,
+												   &sCmpKCCBCmd,
+												   PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		return	RGXScheduleCommand(psDevInfo,
+								   RGXFWIF_DM_GP,
+								   &sCmpKCCBCmd,
+								   PDUMP_FLAGS_CONTINUOUS);
+	}
+}
+
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32CBaseMapCtxReg;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT__HOST_SECURITY_GT1_AND_MHPW_LT6_AND_MMU_VER_GEQ4;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING__HOST_SECURITY_GT1__BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+	else
+	{
+		ui32CBaseMapCtxReg = RGX_CR_MMU_CBASE_MAPPING_CONTEXT;
+		/* Set the mapping context */
+		RGXWriteReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg, MMU_CONTEXT_MAPPING_FWPRIV);
+		(void)RGXReadReg32(&psDevInfo->sLayerParams, ui32CBaseMapCtxReg); /* Fence write */
+
+		/*
+		 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+		 * FW crash from its memory context
+		 */
+		RGXWriteKernelMMUPC32(&psDevInfo->sLayerParams,
+		                      RGX_CR_MMU_CBASE_MAPPING,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT,
+		                      RGX_CR_MMU_CBASE_MAPPING_BASE_ADDR_SHIFT,
+		                      0xDEADBEEF);
+	}
+#else
+	/*
+	 * Catbase-0 (FW MMU context) pointing to unmapped mem to make
+	 * FW crash from its memory context
+	 */
+	RGXWriteKernelMMUPC64(&psDevInfo->sLayerParams,
+				                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+				                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+				                      ((0xDEADBEEF
+				                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+				                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+				                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               IMG_UINT32 ui32DriverID,
+                               RGXFWIF_OS_STATE_CHANGE eOSOnlineState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD sOSOnlineStateCmd = { 0 };
+	const RGXFWIF_SYSDATA *psFwSysData;
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData, INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	sOSOnlineStateCmd.eCmdType = RGXFWIF_KCCB_CMD_OS_ONLINE_STATE_CONFIGURE;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.ui32DriverID = ui32DriverID;
+	sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = eOSOnlineState;
+
+#if defined(SUPPORT_AUTOVZ)
+	{
+		IMG_BOOL bConnectionDown = IMG_FALSE;
+
+		PVR_UNREFERENCED_PARAMETER(psFwSysData);
+		sOSOnlineStateCmd.uCmdData.sCmdOSOnlineStateData.eNewOSState = RGXFWIF_OS_OFFLINE;
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			/* Send the offline command regardless if power lock is held or not.
+			 * Under AutoVz this is done during regular driver deinit, store-to-ram suspend
+			 * or (optionally) from a kernel panic callback. Deinit and suspend operations
+			 * take the lock in the rgx pre/post power functions as expected.
+			 * The kernel panic callback is a last resort way of letting the firmware know that
+			 * the VM is unrecoverable and the vz connection must be disabled. It cannot wait
+			 * on other kernel threads to finish and release the lock. */
+			eError = RGXSendCommand(psDevInfo,
+									&sOSOnlineStateCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Guests and Host going offline should wait for confirmation
+		 * from the Firmware of the state change. If this fails, break
+		 * the connection on the OS Driver's end as backup. */
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (ui32DriverID == RGXFW_HOST_DRIVER_ID))
+		{
+			LOOP_UNTIL_TIMEOUT_US(SECONDS_TO_MICROSECONDS/2)
+			{
+				KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+				if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+				{
+					bConnectionDown = IMG_TRUE;
+					break;
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (!bConnectionDown)
+			{
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+		}
+	}
+#else
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* no reason for Guests to update their state or any other VM's.
+		 * This is the Hypervisor and Host driver's responsibility. */
+		return PVRSRV_OK;
+	}
+	else
+	{
+		const volatile RGXFWIF_OS_RUNTIME_FLAGS *psFwRunFlags;
+
+		PVR_ASSERT(psFwSysData != NULL);
+
+		psFwRunFlags = (const volatile RGXFWIF_OS_RUNTIME_FLAGS*) &psFwSysData->asOsRuntimeFlagsMirror[ui32DriverID];
+		/* Attempt several times until the FW manages to offload the OS */
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			IMG_UINT32 ui32kCCBCommandSlot;
+
+			/* Send request */
+			eError = RGXScheduleCommandAndGetKCCBSlot(psDevInfo,
+													  RGXFWIF_DM_GP,
+													  &sOSOnlineStateCmd,
+													  PDUMP_FLAGS_CONTINUOUS,
+													  &ui32kCCBCommandSlot);
+			if (unlikely(eError == PVRSRV_ERROR_RETRY)) continue;
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommand", return_);
+
+			/* Wait for FW to process the cmd */
+			eError = RGXWaitForKCCBSlotUpdate(psDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", return_);
+
+			/* read the OS state */
+			OSMemoryBarrier(NULL);
+			/* check if FW finished offloading the driver and is stopped */
+			if ((eOSOnlineState == RGXFWIF_OS_ONLINE && (psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_READY || psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_ACTIVE)) ||
+			    (eOSOnlineState == RGXFWIF_OS_OFFLINE && psFwRunFlags->bfOsState == RGXFW_CONNECTION_FW_OFFLINE))
+			{
+				eError = PVRSRV_OK;
+				break;
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_TIMEOUT;
+			}
+
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+
+return_ :
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgPHRCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE((ui32PHRMode == RGXFWIF_PHR_MODE_OFF) ||
+							(ui32PHRMode == RGXFWIF_PHR_MODE_RD_RESET),
+							"Invalid PHR Mode.", PVRSRV_ERROR_INVALID_PARAMS);
+
+	sCfgPHRCmd.eCmdType = RGXFWIF_KCCB_CMD_PHR_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode = ui32PHRMode;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32PHRMode, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Periodic Hardware Reset Mode inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32PHRMode),
+							  ui32PHRMode,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GP,
+		                            &sCfgPHRCmd,
+		                            PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+							IMG_UINT32 ui32WdgPeriodUs)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sCfgWdgCmd = { 0 };
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	sCfgWdgCmd.eCmdType = RGXFWIF_KCCB_CMD_WDG_CFG;
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs = ui32WdgPeriodUs;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32WdgPeriodUs, FLUSH);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the firmware watchdog period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32WdgPeriodUs),
+							  ui32WdgPeriodUs,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+									RGXFWIF_DM_GP,
+									&sCfgWdgCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	return eError;
+}
+
+
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious)
+{
+	/* Attempt to detect and deal with any stalled client contexts.
+	 * bIgnorePrevious may be set by the caller if they know a context to be
+	 * stalled, as otherwise this function will only identify stalled
+	 * contexts which have not been previously reported.
+	 */
+
+	IMG_UINT32 ui32StalledClientMask = 0;
+
+	if (!(OSTryLockAcquire(psDevInfo->hCCBStallCheckLock)))
+	{
+		PVR_LOG(("RGXCheckForStalledClientContexts: Failed to acquire hCCBStallCheckLock, returning..."));
+		return;
+	}
+
+	ui32StalledClientMask |= CheckForStalledClientTransferCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientRenderCtxt(psDevInfo);
+
+	ui32StalledClientMask |= CheckForStalledClientComputeCtxt(psDevInfo);
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+	ui32StalledClientMask |= CheckForStalledClientKickSyncCtxt(psDevInfo);
+#endif
+	/* If at least one DM stalled bit is different than before */
+	if (bIgnorePrevious || (psDevInfo->ui32StalledClientMask != ui32StalledClientMask))
+	{
+		if (ui32StalledClientMask > 0)
+		{
+			static __maybe_unused const char *pszStalledAction =
+#if defined(PVRSRV_STALLED_CCB_ACTION)
+					"force";
+#else
+					"warn";
+#endif
+			/* Print all the stalled DMs */
+			PVR_LOG(("Possible stalled client RGX contexts detected: %s%s%s%s%s%s%s",
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_GP),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TDM_2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TA),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_3D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_CDM),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ2D),
+					 RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(ui32StalledClientMask, RGX_KICK_TYPE_DM_TQ3D)));
+
+			PVR_LOG(("Trying to identify stalled context...(%s) [%d]",
+			         pszStalledAction, bIgnorePrevious));
+
+			DumpStalledContextInfo(psDevInfo);
+		}
+		else
+		{
+			if (psDevInfo->ui32StalledClientMask> 0)
+			{
+				/* Indicate there are no stalled DMs */
+				PVR_LOG(("No further stalled client contexts exist"));
+			}
+		}
+		psDevInfo->ui32StalledClientMask = ui32StalledClientMask;
+		psDevInfo->pvEarliestStalledClientCCB = NULL;
+	}
+	OSLockRelease(psDevInfo->hCCBStallCheckLock);
+}
+
+/*
+	RGXUpdateHealthStatus
+*/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed)
+{
+	const PVRSRV_DATA*           psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_HEALTH_STATUS  eNewStatus   = PVRSRV_DEVICE_HEALTH_STATUS_OK;
+	PVRSRV_DEVICE_HEALTH_REASON  eNewReason   = PVRSRV_DEVICE_HEALTH_REASON_NONE;
+	PVRSRV_RGXDEV_INFO*          psDevInfo;
+	const RGXFWIF_TRACEBUF*      psRGXFWIfTraceBufCtl;
+	const RGXFWIF_SYSDATA*       psFwSysData;
+	const RGXFWIF_OSDATA*        psFwOsData;
+	const RGXFWIF_CCB_CTL*       psKCCBCtl;
+	RGXFWIF_CCB_CTL*             psKCCBCtlLocal;
+	IMG_UINT32                   ui32ThreadCount;
+	IMG_BOOL                     bKCCBCmdsWaiting;
+	PVRSRV_ERROR                 eError;
+
+	PVR_ASSERT(psDevNode != NULL);
+	psDevInfo = psDevNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, PVRSRV_OK);
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		return PVRSRV_OK;
+	}
+
+	psRGXFWIfTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwSysData,
+								 INVALIDATE);
+		psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	}
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfFwOsData,
+	                         INVALIDATE);
+	psFwOsData = psDevInfo->psRGXFWIfFwOsData;
+
+	/* If this is a quick update, then include the last current value... */
+	if (!bCheckAfterTimePassed)
+	{
+		eNewStatus = OSAtomicRead(&psDevNode->eHealthStatus);
+		eNewReason = OSAtomicRead(&psDevNode->eHealthReason);
+	}
+
+	/* Decrement the SLR holdoff counter (if non-zero) */
+	if (psDevInfo->ui32SLRHoldoffCounter > 0)
+	{
+		psDevInfo->ui32SLRHoldoffCounter--;
+	}
+
+	/* Take power lock, retry if it's in use in another task. */
+	eError = PVRSRVPowerTryLockWaitForTimeout(psDevNode);
+	if (eError == PVRSRV_ERROR_TIMEOUT)
+	{
+		/* Skip health status update if timeout */
+		PVR_DPF((PVR_DBG_WARNING, "%s: Power lock timeout, increase OS_POWERLOCK_TIMEOUT_US.", __func__));
+		goto _RGXUpdateHealthStatus_Exit;
+	}
+	PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVPowerTryLockWaitForTimeout");
+
+	/* If the firmware is not yet initialised or has already deinitialised, stop here */
+	if (psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || psDevInfo->pvRegsBaseKM == NULL ||
+		psDevInfo->psDeviceNode == NULL || psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+		psDevInfo->psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)
+	{
+		PVRSRVPowerUnlock(psDevNode);
+		return PVRSRV_OK;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+	{
+		/* On a PCI error all reads from the PCI bar may return 0xFFFFFFFF.
+		   This value is not valid for a core ID. */
+		if (psFwSysData->ui32MemFaultCheck == RGX_PCI_ERROR_VALUE_DWORD)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: PCI error", __func__));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR;
+			PVRSRVDeviceSetState(psDevNode, PVRSRV_DEVICE_STATE_PCI_ERROR);
+			PVRSRVPowerUnlock(psDevNode);
+			goto _RGXUpdateHealthStatus_Exit;
+		}
+	}
+
+	/* If Rogue is not powered on, just skip ahead and check for stalled client CCBs */
+	if (PVRSRVIsDevicePowered(psDevNode))
+	{
+		if (psRGXFWIfTraceBufCtl != NULL)
+		{
+			/*
+			   Firmware thread checks...
+			 */
+			for (ui32ThreadCount = 0; ui32ThreadCount < RGXFW_THREAD_NUM; ui32ThreadCount++)
+			{
+				const IMG_CHAR* pszTraceAssertInfo;
+
+				RGXFwSharedMemCacheOpValue(psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf, INVALIDATE);
+				pszTraceAssertInfo = psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szInfo;
+
+				/*
+				Check if the FW has hit an assert...
+				*/
+				if (*pszTraceAssertInfo != '\0')
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: Firmware thread %d has asserted: %.*s (%.*s:%d)",
+							__func__, ui32ThreadCount, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							pszTraceAssertInfo, RGXFW_TRACE_BUFFER_ASSERT_SIZE,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.szPath,
+							psRGXFWIfTraceBufCtl->sTraceBuf[ui32ThreadCount].sAssertBuf.ui32LineNum));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_ASSERTED;
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+
+				if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+				{
+					/*
+					   Check the threads to see if they are in the same poll locations as last time...
+					*/
+					if (bCheckAfterTimePassed)
+					{
+						if (psFwSysData->aui32CrPollAddr[ui32ThreadCount] != 0  &&
+							psFwSysData->aui32CrPollCount[ui32ThreadCount] == psDevInfo->aui32CrLastPollCount[ui32ThreadCount])
+						{
+							PVR_DPF((PVR_DBG_WARNING, "%s: Firmware stuck on CR poll: T%u polling %s (reg:0x%08X mask:0x%08X)",
+									__func__, ui32ThreadCount,
+									((psFwSysData->aui32CrPollAddr[ui32ThreadCount] & RGXFW_POLL_TYPE_SET)?("set"):("unset")),
+									psFwSysData->aui32CrPollAddr[ui32ThreadCount] & ~RGXFW_POLL_TYPE_SET,
+									psFwSysData->aui32CrPollMask[ui32ThreadCount]));
+							eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+							eNewReason = PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING;
+							PVRSRVPowerUnlock(psDevNode);
+							goto _RGXUpdateHealthStatus_Exit;
+						}
+						psDevInfo->aui32CrLastPollCount[ui32ThreadCount] = psFwSysData->aui32CrPollCount[ui32ThreadCount];
+					}
+				}
+			}
+
+			if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevNode))
+			{
+				/*
+				Check if the FW has faulted...
+				*/
+				if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_FW_FAULT)
+				{
+					PVR_DPF((PVR_DBG_WARNING,
+							"%s: Firmware has faulted and needs to restart",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_FAULT;
+					if (psFwSysData->ui32HWRStateFlags & RGXFWIF_HWR_RESTART_REQUESTED)
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_RESTARTING;
+					}
+					else
+					{
+						eNewReason = PVRSRV_DEVICE_HEALTH_REASON_IDLING;
+					}
+					PVRSRVPowerUnlock(psDevNode);
+					goto _RGXUpdateHealthStatus_Exit;
+				}
+			}
+		}
+
+		/*
+		   Event Object Timeouts check...
+		*/
+		if (!bCheckAfterTimePassed)
+		{
+			if (psDevInfo->ui32GEOTimeoutsLastTime > 1 && psPVRSRVData->ui32GEOConsecutiveTimeouts > psDevInfo->ui32GEOTimeoutsLastTime)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Global Event Object Timeouts have risen (from %d to %d)",
+				         __func__,
+				         psDevInfo->ui32GEOTimeoutsLastTime, psPVRSRVData->ui32GEOConsecutiveTimeouts));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS;
+			}
+			psDevInfo->ui32GEOTimeoutsLastTime = psPVRSRVData->ui32GEOConsecutiveTimeouts;
+		}
+
+		/*
+		   Check the Kernel CCB pointer is valid. If any commands were waiting last time, then check
+		   that some have executed since then.
+		*/
+		bKCCBCmdsWaiting = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psKernelCCBCtl, INVALIDATE);
+		psKCCBCtl = psDevInfo->psKernelCCBCtl;
+		psKCCBCtlLocal = psDevInfo->psKernelCCBCtlLocal;
+
+		if (psKCCBCtl != NULL && psKCCBCtlLocal != NULL)
+		{
+			/* update KCCB read offset */
+			RGXFwSharedMemCacheOpValue(psKCCBCtl->ui32ReadOffset, INVALIDATE);
+			psKCCBCtlLocal->ui32ReadOffset = psKCCBCtl->ui32ReadOffset;
+
+			if (psKCCBCtlLocal->ui32ReadOffset > psKCCBCtlLocal->ui32WrapMask  ||
+				psKCCBCtlLocal->ui32WriteOffset > psKCCBCtlLocal->ui32WrapMask)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: KCCB has invalid offset (ROFF=%d WOFF=%d)",
+						__func__, psKCCBCtlLocal->ui32ReadOffset, psKCCBCtlLocal->ui32WriteOffset));
+				eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_DEAD;
+				eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+			}
+
+			if (psKCCBCtlLocal->ui32ReadOffset != psKCCBCtlLocal->ui32WriteOffset)
+			{
+				bKCCBCmdsWaiting = IMG_TRUE;
+			}
+		}
+
+		if (bCheckAfterTimePassed && psFwOsData != NULL)
+		{
+			IMG_UINT32 ui32KCCBCmdsExecuted = psFwOsData->ui32KCCBCmdsExecuted;
+
+			if (psDevInfo->ui32KCCBCmdsExecutedLastTime == ui32KCCBCmdsExecuted)
+			{
+				/*
+				   If something was waiting last time then the Firmware has stopped processing commands.
+				*/
+				if (psDevInfo->bKCCBCmdsWaitingLastTime)
+				{
+					PVR_DPF((PVR_DBG_WARNING, "%s: No KCCB commands executed since check!",
+							__func__));
+					eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+					eNewReason = PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+				}
+
+				/*
+				   If no commands are currently pending and nothing happened since the last poll, then
+				   schedule a dummy command to ping the firmware so we know it is alive and processing.
+				*/
+				if ((!bKCCBCmdsWaiting) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_DEAD) &&
+					(eNewStatus != PVRSRV_DEVICE_HEALTH_STATUS_FAULT))
+				{
+					/* Protect the PDumpLoadMem. RGXScheduleCommand() cannot take the
+					 * PMR lock itself, because some bridge functions will take the PMR lock
+					 * before calling RGXScheduleCommand
+					 */
+					PVRSRV_ERROR eError = RGXFWHealthCheckCmdWithoutPowerLock(psDevNode->pvDevice);
+
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Cannot schedule Health Check command! (0x%x)",
+								__func__, eError));
+					}
+					else
+					{
+						bKCCBCmdsWaiting = IMG_TRUE;
+					}
+				}
+			}
+
+			psDevInfo->bKCCBCmdsWaitingLastTime     = bKCCBCmdsWaiting;
+			psDevInfo->ui32KCCBCmdsExecutedLastTime = ui32KCCBCmdsExecuted;
+		}
+	}
+
+	/*
+	   Interrupt counts check...
+	*/
+	if (bCheckAfterTimePassed  &&  psFwOsData != NULL)
+	{
+		IMG_UINT32  ui32LISRCount   = 0;
+		IMG_UINT32  ui32FWCount     = 0;
+		IMG_UINT32  ui32MissingInts = 0;
+
+		/* Add up the total number of interrupts issued, sampled/received and missed... */
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+		/* Only the Host OS has a sample count, so only one counter to check. */
+		ui32LISRCount += psDevInfo->aui32SampleIRQCount[RGXFW_HOST_DRIVER_ID];
+		ui32FWCount   += OSReadHWReg32(psDevInfo->pvRegsBaseKM, gaui32FwOsIrqCntRegAddr[RGXFW_HOST_DRIVER_ID]);
+#else
+		IMG_UINT32  ui32Index;
+
+		for (ui32Index = 0;  ui32Index < RGXFW_THREAD_NUM;  ui32Index++)
+		{
+			ui32LISRCount += psDevInfo->aui32SampleIRQCount[ui32Index];
+			ui32FWCount   += psFwOsData->aui32InterruptCount[ui32Index];
+		}
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+
+		if (ui32LISRCount < ui32FWCount)
+		{
+			ui32MissingInts = (ui32FWCount-ui32LISRCount);
+		}
+
+		if (ui32LISRCount == psDevInfo->ui32InterruptCountLastTime  &&
+		    ui32MissingInts >= psDevInfo->ui32MissingInterruptsLastTime  &&
+		    psDevInfo->ui32MissingInterruptsLastTime > 1)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: LISR has not received the last %d interrupts",
+					__func__, ui32MissingInts));
+			eNewStatus = PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+			eNewReason = PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+
+			/* Schedule the MISRs to help mitigate the problems of missing interrupts. */
+			OSScheduleMISR(psDevInfo->pvMISRData);
+			if (psDevInfo->pvAPMISRData != NULL)
+			{
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+		psDevInfo->ui32InterruptCountLastTime    = ui32LISRCount;
+		psDevInfo->ui32MissingInterruptsLastTime = ui32MissingInts;
+	}
+
+	/* Release power lock before RGXCheckForStalledClientContexts */
+	PVRSRVPowerUnlock(psDevNode);
+
+	/*
+	   Stalled CCB check...
+	*/
+	if (bCheckAfterTimePassed && (PVRSRV_DEVICE_HEALTH_STATUS_OK==eNewStatus))
+	{
+		RGXCheckForStalledClientContexts(psDevInfo, IMG_FALSE);
+	}
+
+	/* Notify client driver and system layer of any eNewStatus errors */
+	if (eNewStatus > PVRSRV_DEVICE_HEALTH_STATUS_OK)
+	{
+		/* Client notification of device error will be achieved by
+		 * clients calling UM function RGXGetLastDeviceError() */
+		psDevInfo->eLastDeviceError = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+
+		/* Notify system layer */
+		{
+			PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+			PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+			if (psDevConfig->pfnSysDevErrorNotify)
+			{
+				PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+				sErrorData.eResetReason = RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR;
+				sErrorData.uErrData.sHostWdgData.ui32Status = (IMG_UINT32)eNewStatus;
+				sErrorData.uErrData.sHostWdgData.ui32Reason = (IMG_UINT32)eNewReason;
+
+				psDevConfig->pfnSysDevErrorNotify(psDevConfig,
+				                                  &sErrorData);
+			}
+		}
+	}
+
+	/*
+	   Finished, save the new status...
+	*/
+_RGXUpdateHealthStatus_Exit:
+	OSAtomicWrite(&psDevNode->eHealthStatus, eNewStatus);
+	OSAtomicWrite(&psDevNode->eHealthReason, eNewReason);
+	RGXSRV_HWPERF_DEVICE_INFO_HEALTH(psDevInfo, eNewStatus, eNewReason);
+
+	/*
+	 * Attempt to service the HWPerf buffer to regularly transport idle/periodic
+	 * packets to host buffer.
+	 */
+	if (psDevNode->pfnServiceHWPerf != NULL)
+	{
+		PVRSRV_ERROR eError = psDevNode->pfnServiceHWPerf(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: "
+			         "Error occurred when servicing HWPerf buffer (%d)",
+			         __func__, eError));
+		}
+	}
+
+	/* Attempt to refresh timer correlation data */
+	RGXTimeCorrRestartPeriodic(psDevNode);
+
+	return PVRSRV_OK;
+} /* RGXUpdateHealthStatus */
+
+#if defined(SUPPORT_AUTOVZ)
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+	if (likely(KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+		(KM_OS_CONNECTION_IS(ACTIVE, psDevInfo) || KM_OS_CONNECTION_IS(READY, psDevInfo))))
+	{
+		/* read and write back the alive token value to confirm to the
+		 * virtualisation watchdog that this connection is healthy */
+		KM_SET_OS_ALIVE_TOKEN(KM_GET_FW_ALIVE_TOKEN(psDevInfo), psDevInfo);
+		KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH);
+	}
+}
+
+/*
+	RGXUpdateAutoVzWatchdog
+*/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode)
+{
+	if (likely(psDevNode != NULL))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+
+		if (unlikely((psDevInfo  == NULL || !psDevInfo->bFirmwareInitialised || !psDevInfo->bRGXPowered ||
+					  psDevInfo->pvRegsBaseKM == NULL || psDevNode->eDevState == PVRSRV_DEVICE_STATE_DEINIT ||
+					  psDevNode->eDevState == PVRSRV_DEVICE_STATE_DESTRUCTING)))
+		{
+			/* If the firmware is not initialised, stop here */
+			return;
+		}
+		else
+		{
+			PVRSRV_ERROR eError = PVRSRVPowerLock(psDevNode);
+			PVR_LOG_RETURN_VOID_IF_ERROR(eError, "PVRSRVPowerLock");
+
+			RGXUpdateAutoVzWdgToken(psDevInfo);
+			PVRSRVPowerUnlock(psDevNode);
+		}
+	}
+}
+
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 ui32DriverID;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+	                           INVALIDATE);
+
+	for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+	     ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED;
+	     ui32DriverID++)
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+			psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+		if (eGuestState == RGXFW_CONNECTION_FW_ACTIVE ||
+		    eGuestState == RGXFW_CONNECTION_FW_READY)
+		{
+			PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+		}
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* SUPPORT_AUTOVZ */
+
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer)
+{
+	PVRSRV_ERROR eError;
+	PRGXFWIF_CLEANUP_CTL *psCleanupCtlWrite = apsCleanupCtl;
+
+	PVR_ASSERT((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+	PVR_RETURN_IF_INVALID_PARAM((eDM == RGXFWIF_DM_GEOM) || (eDM == RGXFWIF_DM_3D));
+
+	if (bKick)
+	{
+		if (psKMHWRTDataSet)
+		{
+			PRGXFWIF_CLEANUP_CTL psCleanupCtl;
+
+			eError = RGXSetFirmwareAddress(&psCleanupCtl, psKMHWRTDataSet->psHWRTDataFwMemDesc,
+					offsetof(RGXFWIF_HWRTDATA, sCleanupState),
+					RFW_FWADDR_NOREF_FLAG);
+			PVR_RETURN_IF_ERROR(eError);
+
+			*(psCleanupCtlWrite++) = psCleanupCtl;
+		}
+
+		if (eDM == RGXFWIF_DM_3D)
+		{
+			RGXFWIF_PRBUFFER_TYPE eBufferType;
+			RGX_ZSBUFFER_DATA *psBuffer = NULL;
+
+			for (eBufferType = RGXFWIF_PRBUFFER_START; eBufferType < RGXFWIF_PRBUFFER_MAXSUPPORTED; eBufferType++)
+			{
+				switch (eBufferType)
+				{
+				case RGXFWIF_PRBUFFER_ZSBUFFER:
+					psBuffer = psZSBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MSAABUFFER:
+					psBuffer = psMSAAScratchBuffer;
+					break;
+				case RGXFWIF_PRBUFFER_MAXSUPPORTED:
+					psBuffer = NULL;
+					break;
+				}
+				if (psBuffer)
+				{
+					(psCleanupCtlWrite++)->ui32Addr = psBuffer->sZSBufferFWDevVAddr.ui32Addr +
+									offsetof(RGXFWIF_PRBUFFER, sCleanupState);
+					psBuffer = NULL;
+				}
+			}
+		}
+	}
+
+	*pui32NumCleanupCtl = psCleanupCtlWrite - apsCleanupCtl;
+	PVR_ASSERT(*pui32NumCleanupCtl <= RGXFWIF_KCCB_CMD_KICK_DATA_MAX_NUM_CLEANUP_CTLS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO       *psDevInfo;
+	RGXFWIF_HWRINFOBUF       *psHWRInfoBuf;
+	IMG_UINT32               i;
+
+	if (psDevNode->pvDevice == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_DEVINFO;
+	}
+	psDevInfo = psDevNode->pvDevice;
+
+	psHWRInfoBuf = psDevInfo->psRGXFWIfHWRInfoBufCtl;
+
+	for (i = 0 ; i < RGXFWIF_DM_MAX ; i++)
+	{
+		/* Reset the HWR numbers */
+		psHWRInfoBuf->aui32HwrDmLockedUpCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmFalseDetectCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmRecoveredCount[i] = 0;
+		psHWRInfoBuf->aui32HwrDmOverranCount[i] = 0;
+	}
+
+	for (i = 0 ; i < RGXFWIF_HWINFO_MAX ; i++)
+	{
+		psHWRInfoBuf->sHWRInfo[i].ui32HWRNumber = 0;
+	}
+
+	psHWRInfoBuf->ui32WriteIndex = 0;
+	psHWRInfoBuf->ui32DDReqCount = 0;
+
+	OSWriteMemoryBarrier(&psHWRInfoBuf->ui32DDReqCount);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid)
+{
+
+	PVRSRV_ERROR eError;
+
+	eError = PMRLockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	eError = PMR_DevPhysAddr(psPMR,
+								 ui32Log2PageSize,
+								 ui32NumOfPages,
+								 ui32LogicalOffset,
+								 psPhyAddr,
+								 bValid,
+								 DEVICE_USE);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMR_DevPhysAddr failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+
+	eError = PMRUnlockSysPhysAddresses(psPMR);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: PMRUnLockSysPhysAddresses failed (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	return eError;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32WriteOffset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_OK);
+
+	if (psDevInfo->bDumpedKCCBCtlAlready)
+	{
+		/* exiting capture range or pdump block */
+		psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+		/* make sure previous cmd is drained in pdump in case we will 'jump' over some future cmds */
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode,
+                              PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER,
+                              "kCCB(%p): Draining rgxfw_roff (0x%x) == woff (0x%x)",
+                              psDevInfo->psKernelCCBCtl,
+                              ui32WriteOffset,
+                              ui32WriteOffset);
+		eError = DevmemPDumpDevmemPol32(psDevInfo->psKernelCCBCtlMemDesc,
+				offsetof(RGXFWIF_CCB_CTL, ui32ReadOffset),
+				ui32WriteOffset,
+				0xffffffff,
+				PDUMP_POLL_OPERATOR_EQUAL,
+				PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_POWER);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: problem pdumping POL for kCCBCtl (%d)", __func__, eError));
+		}
+	}
+
+	return eError;
+
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXClientConnectCompatCheck_ClientAgainstFW
+
+ @Description
+
+ Check compatibility of client and firmware (build options)
+ at the connection time.
+
+ @Input psDeviceNode - device node
+ @Input ui32ClientBuildOptions - build options for the client
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+******************************************************************************/
+PVRSRV_ERROR RGXClientConnectCompatCheck_ClientAgainstFW(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT32 ui32ClientBuildOptions)
+{
+	IMG_UINT32		ui32BuildOptionsMismatch;
+	IMG_UINT32		ui32BuildOptionsFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_FW_INFO_HEADER *psFWInfoHeader;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psDevInfo == NULL || psDevInfo->psRGXFWIfOsInitMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Cannot acquire kernel fw compatibility check info, RGXFWIF_OSINIT structure not allocated.",
+		         __func__));
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *) &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+	}
+#endif
+
+	ui32BuildOptionsFW = psFWInfoHeader->ui32Flags;
+	ui32BuildOptionsMismatch = ui32ClientBuildOptions ^ ui32BuildOptionsFW;
+
+	if (ui32BuildOptionsMismatch != 0)
+	{
+		if ((ui32ClientBuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in client: (0x%x). Please check rgx_options.h",
+					 ui32ClientBuildOptions & ui32BuildOptionsMismatch ));
+		}
+
+		if ((ui32BuildOptionsFW & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and client build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFW & ui32BuildOptionsMismatch ));
+		}
+
+		return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware and client build options match. [ OK ]", __func__));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapAllocMap
+
+ @Description Register firmware heap for the specified driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID - Guest driver
+ @Input sDevPAddr    - Heap address
+ @Input ui64DevPSize - Heap size
+
+ @Return   PVRSRV_ERROR - PVRSRV_OK if heap setup was successful.
+
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize)
+{
+	PVRSRV_ERROR eError;
+	IMG_CHAR szRegionRAName[RA_MAX_NAME_LENGTH];
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_MEMALLOCFLAGS_T uiRawFwHeapAllocFlags = (RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS |
+													PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PREMAP0 + ui32DriverID));
+	PHYS_HEAP_CONFIG *psFwHeapConfig = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+	PHYS_HEAP_CONFIG sFwHeapConfig;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psFwHeapConfig == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW_MAIN heap config not found."));
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	OSSNPrintf(szRegionRAName, sizeof(szRegionRAName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	if (!ui64DevPSize ||
+		!sDevPAddr.uiAddr ||
+		ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED ||
+		ui64DevPSize != RGX_FIRMWARE_RAW_HEAP_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid parameters for %s", szRegionRAName));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	sFwHeapConfig = *psFwHeapConfig;
+	sFwHeapConfig.eType = PHYS_HEAP_TYPE_LMA;
+	sFwHeapConfig.ui32UsageFlags = PHYS_HEAP_USAGE_FW_PREMAP;
+	sFwHeapConfig.uConfig.sLMA.sStartAddr.uiAddr = 0;
+	sFwHeapConfig.uConfig.sLMA.sCardBase.uiAddr = sDevPAddr.uiAddr;
+	sFwHeapConfig.uConfig.sLMA.uiSize = RGX_FIRMWARE_RAW_HEAP_SIZE;
+
+	eError = PhysmemCreateHeapLMA(psDeviceNode,
+	                              RGXPhysHeapGetLMAPolicy(sFwHeapConfig.ui32UsageFlags, psDeviceNode),
+	                              &sFwHeapConfig,
+	                              szRegionRAName,
+	                              &psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysmemCreateHeapLMA:PREMAP [%d]", ui32DriverID);
+
+	eError = PhysHeapAcquire(psDeviceNode->apsFWPremapPhysHeap[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR_VA(eError, "PhysHeapAcquire:PREMAP [%d]", ui32DriverID);
+
+	psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID] = psDeviceNode->apsFWPremapPhysHeap[ui32DriverID];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate and map raw firmware heap for DriverID: [%d]", ui32DriverID);
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* don't clear the heap of other guests on allocation */
+	uiRawFwHeapAllocFlags &= (ui32DriverID > RGXFW_HOST_DRIVER_ID) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL);
+#endif
+
+	/* if the firmware is already powered up, consider the firmware heaps are pre-mapped. */
+	if (psDeviceNode->bAutoVzFwIsUp)
+	{
+		uiRawFwHeapAllocFlags &= RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Allocation and mapping for Firmware heaps done by TEE.", __func__));
+#else
+	eError = DevmemFwAllocate(psDevInfo,
+							  RGX_FIRMWARE_RAW_HEAP_SIZE,
+							  uiRawFwHeapAllocFlags,
+							  psDevInfo->psPremappedFwRawHeap[ui32DriverID]->pszName,
+							  &psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+#endif
+
+	/* Mark this devmem heap as premapped so allocations will not require device mapping. */
+	DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_TRUE);
+
+	if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+	{
+		/* if the Host's raw fw heap is premapped, mark its main & config sub-heaps accordingly
+		 * No memory allocated from these sub-heaps will be individually mapped into the device's
+		 * address space so they can remain marked permanently as premapped. */
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXFwRawHeapUnmapFree
+
+ @Description Unregister firmware heap for the specified guest driver
+
+ @Input psDeviceNode - device node
+ @Input ui32DriverID
+
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* remove the premap status, so the heap can be unmapped and freed */
+	if (psDevInfo->psPremappedFwRawHeap[ui32DriverID])
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psPremappedFwRawHeap[ui32DriverID], IMG_FALSE);
+	}
+
+	if (psDevInfo->psPremappedFwRawMemDesc[ui32DriverID])
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psPremappedFwRawMemDesc[ui32DriverID]);
+		psDevInfo->psPremappedFwRawMemDesc[ui32DriverID] = NULL;
+	}
+}
+
+/*
+	RGXReadMETAAddr
+*/
+static PVRSRV_ERROR RGXReadMETAAddr(PVRSRV_RGXDEV_INFO	*psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 *pui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem  *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem  *pui8RegBase = pvRegBase;
+	IMG_UINT32 ui32PollValue;
+	IMG_UINT32 ui32PollMask;
+	IMG_UINT32 ui32PollRegOffset;
+	IMG_UINT32 ui32ReadOffset;
+	IMG_UINT32 ui32WriteOffset;
+	IMG_UINT32 ui32WriteValue;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32PollValue = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollMask = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+							| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN;
+			CHECK_HWBRN_68777(ui32WriteValue);
+			ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+	{
+		ui32PollValue = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollMask = RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32PollRegOffset = RGX_CR_META_SP_MSLVCTRL1;
+		ui32WriteOffset = RGX_CR_META_SP_MSLVCTRL0;
+		ui32WriteValue = ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN;
+		ui32ReadOffset = RGX_CR_META_SP_MSLVDATAX;
+	}
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteUncheckedHWReg32(pvRegBase, ui32WriteOffset, ui32WriteValue);
+	(void)OSReadUncheckedHWReg32(pvRegBase, ui32WriteOffset);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + ui32PollRegOffset),
+	        ui32PollValue,
+	        ui32PollMask,
+	        POLL_FLAG_LOG_ERROR,
+	        NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadUncheckedHWReg32(pvRegBase, ui32ReadOffset);
+#else
+	IMG_UINT8 __iomem  *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Read */
+	OSWriteHWReg32(
+			psDevInfo->pvRegsBaseKM,
+			RGX_CR_META_SP_MSLVCTRL0,
+			ui32METAAddr | RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	(void) OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0);
+
+	/* Wait for Slave Port to be Ready: read complete */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *) (pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Read the value */
+	*pui32Value = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAX);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGXWriteMETAAddr
+*/
+static PVRSRV_ERROR RGXWriteMETAAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32METAAddr, IMG_UINT32 ui32Value)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	void __iomem *pvRegBase = psDevInfo->pvSecureRegsBaseKM;
+	IMG_UINT8 __iomem *pui8RegBase = pvRegBase;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1)
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA, ui32Value);
+		}
+		else
+		{
+			/* Wait for Slave Port to be Ready */
+			if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+					(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA),
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+					| RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+					POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+			{
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			/* Issue the Write */
+			CHECK_HWBRN_68777(ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA, ui32METAAddr);
+			OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_EQ1_AND_MRUA, ui32Value);
+		}
+	}
+	else
+	{
+		/* Wait for Slave Port to be Ready */
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+				(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+				POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+		{
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Issue the Write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVCTRL0); /* Fence write */
+		OSWriteUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+		(void) OSReadUncheckedHWReg32(pvRegBase, RGX_CR_META_SP_MSLVDATAT); /* Fence write */
+	}
+#else
+	IMG_UINT8 __iomem *pui8RegBase = psDevInfo->pvRegsBaseKM;
+
+	/* Wait for Slave Port to be Ready */
+	if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+			(IMG_UINT32 __iomem *)(pui8RegBase + RGX_CR_META_SP_MSLVCTRL1),
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			RGX_CR_META_SP_MSLVCTRL1_READY_EN|RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+			POLL_FLAG_LOG_ERROR, NULL) != PVRSRV_OK)
+	{
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	/* Issue the Write */
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVCTRL0, ui32METAAddr);
+	OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_SP_MSLVDATAT, ui32Value);
+#endif
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 *pui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXReadMETAAddr(psDevInfo, ui32FWAddr, pui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvReadMem(psDevInfo, ui32FWAddr, pui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FWAddr, IMG_UINT32 ui32Value)
+{
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		return RGXWriteMETAAddr(psDevInfo, ui32FWAddr, ui32Value);
+	}
+
+#if !defined(EMULATOR)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		return RGXRiscvWriteMem(psDevInfo, ui32FWAddr, ui32Value);
+	}
+#endif
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+}
+
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                             IMG_UINT32 ui32FwVA,
+                             IMG_CPU_PHYADDR *psCpuPA,
+                             IMG_DEV_PHYADDR *psDevPA,
+                             IMG_UINT64 *pui64RawPTE)
+{
+	PVRSRV_ERROR eError       = PVRSRV_OK;
+	IMG_CPU_PHYADDR sCpuPA    = {0U};
+	IMG_DEV_PHYADDR sDevPA    = {0U};
+	IMG_UINT64 ui64RawPTE     = 0U;
+	MMU_FAULT_DATA sFaultData = {0U};
+	MMU_CONTEXT *psFwMMUCtx   = psDevInfo->psKernelMMUCtx;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32FwHeapEnd  = ui32FwHeapBase + (RGX_NUM_DRIVERS_SUPPORTED * RGX_FIRMWARE_RAW_HEAP_SIZE);
+	IMG_UINT32 ui32DriverID   = (ui32FwVA - ui32FwHeapBase) / RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT32 ui32HeapId;
+	PHYS_HEAP *psPhysHeap;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* MIPS uses the same page size as the OS, while others default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) ?
+	                             OSGetPageSize() : BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#else
+	/* default to 4K pages */
+	IMG_UINT32 ui32FwPageSize = BIT(RGX_MMUCTRL_PAGE_4KB_RANGE_SHIFT);
+#endif
+
+	IMG_UINT32 ui32PageOffset = (ui32FwVA & (ui32FwPageSize - 1));
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM((ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((psCpuPA != NULL) ||
+	                               (psDevPA != NULL) ||
+	                               (pui64RawPTE != NULL)),
+	                              eError, ErrorExit);
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(((ui32FwVA >= ui32FwHeapBase) &&
+	                              (ui32FwVA < ui32FwHeapEnd)),
+	                              eError, ErrorExit);
+
+	ui32HeapId = (ui32DriverID == RGXFW_HOST_DRIVER_ID) ?
+	              PVRSRV_PHYS_HEAP_FW_MAIN : (PVRSRV_PHYS_HEAP_FW_PREMAP0 + ui32DriverID);
+	psPhysHeap = psDevInfo->psDeviceNode->apsPhysHeap[ui32HeapId];
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS is equipped with a dedicated MMU */
+		RGXMipsCheckFaultAddress(psFwMMUCtx, ui32FwVA, &sFaultData);
+	}
+	else
+#endif
+	{
+		IMG_UINT64 ui64FwDataBaseMask;
+		IMG_DEV_VIRTADDR sDevVAddr;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			ui64FwDataBaseMask = ~(RGXFW_SEGMMU_DATA_META_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK |
+			                         RGXFW_SEGMMU_DATA_BASE_ADDRESS);
+		}
+#if !defined(EMULATOR)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			ui64FwDataBaseMask = ~RGXRISCVFW_REGION_MASK;
+		}
+#endif
+		else
+		{
+			PVR_LOG_GOTO_WITH_ERROR("RGXGetFwMapping", eError, PVRSRV_ERROR_NOT_IMPLEMENTED, ErrorExit);
+		}
+
+		sDevVAddr.uiAddr = (ui32FwVA & ui64FwDataBaseMask) | RGX_FIRMWARE_RAW_HEAP_BASE;
+
+		/* Fw CPU shares a subset of the GPU's VA space */
+		MMU_CheckFaultAddress(psFwMMUCtx, &sDevVAddr, &sFaultData);
+	}
+
+	ui64RawPTE = sFaultData.sLevelData[MMU_LEVEL_1].ui64Address;
+
+	if (eError == PVRSRV_OK)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		IMG_BOOL bValidPage = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+		                       BITMASK_HAS(ui64RawPTE, RGXMIPSFW_TLB_VALID) :
+		                       BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#else
+		IMG_BOOL bValidPage = BITMASK_HAS(ui64RawPTE, RGX_MMUCTRL_PT_DATA_VALID_EN);
+#endif
+
+		if (!bValidPage)
+		{
+			/* don't report invalid pages */
+			eError = PVRSRV_ERROR_DEVICEMEM_NO_MAPPING;
+		}
+		else
+		{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+			sDevPA.uiAddr = ui32PageOffset + ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			                RGXMIPSFW_TLB_GET_PA(ui64RawPTE) :
+			                (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK));
+#else
+			sDevPA.uiAddr = ui32PageOffset + (ui64RawPTE & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK);
+#endif
+
+			/* Only the Host's Firmware heap is present in the Host's CPU IPA space */
+			if (ui32DriverID == RGXFW_HOST_DRIVER_ID)
+			{
+				PhysHeapDevPAddrToCpuPAddr(psPhysHeap, 1, &sCpuPA, &sDevPA);
+			}
+			else
+			{
+				sCpuPA.uiAddr = 0U;
+			}
+		}
+	}
+
+	if (psCpuPA != NULL)
+	{
+		*psCpuPA = sCpuPA;
+	}
+
+	if (psDevPA != NULL)
+	{
+		*psDevPA = sDevPA;
+	}
+
+	if (pui64RawPTE != NULL)
+	{
+		*pui64RawPTE = ui64RawPTE;
+	}
+
+ErrorExit:
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec)
+{
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psDevInfo, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec = ui32VzConnectionCooldownPeriodInSec;
+	OSWriteMemoryBarrier(&psDevInfo->psRGXFWIfRuntimeCfg->ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDevInfo->psDeviceNode,
+				 "Updating the Vz reconnect request cooldown period inside RGXFWIfRuntimeCfg");
+	DevmemPDumpLoadMemValue32(psDevInfo->psRGXFWIfRuntimeCfgMemDesc,
+							  offsetof(RGXFWIF_RUNTIME_CFG, ui32VzConnectionCooldownPeriodInSec),
+							  ui32VzConnectionCooldownPeriodInSec,
+							  PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxfwutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxfwutils.h b/drivers/gpu/drm/img-rogue/rgxfwutils.h
new file mode 100644
index 000000000000..6752bc95fa18
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxfwutils.h
@@ -0,0 +1,1298 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX firmware utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX firmware utility routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXFWUTILS_H
+#define RGXFWUTILS_H
+
+#include "rgx_memallocflags.h"
+#include "log2.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "devicemem.h"
+#include "device.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "connection_server.h"
+#include "rgxta3d.h"
+#include "devicemem_utils.h"
+#include "rgxmem.h"
+#include "rgxfwmemctx.h"
+#include "rgxinit_apphints.h"
+
+#define RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT   "FwRawDriverID%d" /*!< RGX Raw Firmware Heap identifier */
+
+static INLINE PVRSRV_ERROR _SelectDevMemHeap(PVRSRV_RGXDEV_INFO *psDevInfo,
+											 PVRSRV_MEMALLOCFLAGS_T *puiFlags,
+											 DEVMEM_HEAP **ppsFwHeap)
+{
+	PVRSRV_PHYS_HEAP ePhysHeap = (PVRSRV_PHYS_HEAP)PVRSRV_GET_PHYS_HEAP_HINT(*puiFlags);
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (ePhysHeap)
+	{
+		case PVRSRV_PHYS_HEAP_FW_CODE:
+		case PVRSRV_PHYS_HEAP_FW_PRIV_DATA:
+		case PVRSRV_PHYS_HEAP_FW_MAIN:
+		{
+			*ppsFwHeap = psDevInfo->psFirmwareMainHeap;
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_FW_CONFIG:
+		{
+			*ppsFwHeap = psDevInfo->psFirmwareConfigHeap;
+			break;
+		}
+		case PVRSRV_PHYS_HEAP_FW_PREMAP0:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP1:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP2:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP3:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP4:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP5:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP6:
+		case PVRSRV_PHYS_HEAP_FW_PREMAP7:
+		{
+			IMG_UINT32 ui32DriverID = ePhysHeap - PVRSRV_PHYS_HEAP_FW_PREMAP0;
+
+			PVR_LOG_RETURN_IF_INVALID_PARAM(ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED, "ui32DriverID");
+			*ppsFwHeap = psDevInfo->psPremappedFwRawHeap[ui32DriverID];
+			break;
+		}
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: invalid phys heap", __func__));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			break;
+		}
+	}
+
+	return eError;
+}
+
+/*
+ * Firmware-only allocation (which are initialised by the host) must be aligned to the SLC cache line size.
+ * This is because firmware-only allocations are GPU_CACHE_INCOHERENT and this causes problems
+ * if two allocations share the same cache line; e.g. the initialisation of the second allocation won't
+ * make it into the SLC cache because it has been already loaded when accessing the content of the first allocation.
+ */
+static INLINE PVRSRV_ERROR DevmemFwAllocate(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_DEVMEM_SIZE_T uiSize,
+											PVRSRV_MEMALLOCFLAGS_T uiFlags,
+											const IMG_CHAR *pszText,
+											DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_DEVMEM_ALIGN_T uiAlign;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) && (pszText[0] == 'F') && (pszText[1] == 'w'));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	if (psFwHeap == psDevInfo->psFirmwareConfigHeap)
+	{
+		/*
+		 * All structures allocated from the Firmware Config sub-heap must start at the same pre-determined
+		 * offsets, regardless of the system's page size (e.g. 4k,16k,64k). The alignment requirement is
+		 * satisfied for virtual addresses during the mapping stage. Physical allocations do not take
+		 * alignment into consideration.
+		 * VZ drivers usually preallocate and pre-map the entire Firmware heap range. Any allocations from
+		 * this heap are physical allocations only, having their device VAs derived from their PAs. This makes
+		 * it impossible to fulfil alignment requirements.
+		 * To work around this limitation, allocation sizes are rounded to the nearest multiple of 64kb,
+		 * regardless of the actual size of object.
+		 */
+		uiAlign = RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY;
+
+		uiSize = PVR_ALIGN(uiSize, RGX_FIRMWARE_CONFIG_HEAP_ALLOC_GRANULARITY);
+	}
+	else
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			/* Aligning FW based allocations for MIPS based rogue cores at cache line boundary(16 bytes) instead
+			 * of SLC(64 bytes) to have more compact memory with less waste and hopefully save some TLB misses.
+			 * MIPS CPU cores alignment.
+			 */
+			uiAlign = RGXMIPSFW_MICROAPTIVEAP_CACHELINE_SIZE;
+		}
+		else
+#endif
+		{
+			/* Non-MIPS CPU cores alignment */
+			uiAlign = (GET_ROGUE_CACHE_LINE_SIZE(RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS)));
+		}
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+	eError = DevmemAllocateAndMap(psFwHeap,
+				uiSize,
+				uiAlign,
+				uiFlags,
+				pszText,
+				ppsMemDescPtr,
+				&sTmpDevVAddr);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+static INLINE PVRSRV_ERROR DevmemFwAllocateExportable(PVRSRV_DEVICE_NODE *psDeviceNode,
+													  IMG_DEVMEM_SIZE_T uiSize,
+													  IMG_DEVMEM_ALIGN_T uiAlign,
+													  PVRSRV_MEMALLOCFLAGS_T uiFlags,
+													  const IMG_CHAR *pszText,
+													  DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_UINT32 ui32HeapLog2PageSize;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) &&
+			(pszText[0] == 'F') && (pszText[1] == 'w') &&
+			(pszText[2] == 'E') && (pszText[3] == 'x'));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* MIPS cores */
+		ui32HeapLog2PageSize = ExactLog2(uiAlign);
+	}
+	else
+#endif
+	{
+		/* Meta and RiscV cores */
+		ui32HeapLog2PageSize = DevmemGetHeapLog2PageSize(psFwHeap);
+	}
+
+	eError = DevmemAllocateExportable(psDeviceNode,
+									  uiSize,
+									  uiAlign,
+									  ui32HeapLog2PageSize,
+									  uiFlags,
+									  pszText,
+									  ppsMemDescPtr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "FW DevmemAllocateExportable failed (%u)", eError));
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	/*
+		We need to map it so the heap for this allocation
+		is set
+	*/
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+							   psFwHeap,
+							   &sTmpDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsMemDescPtr);
+		PVR_DPF((PVR_DBG_ERROR, "FW DevmemMapToDevice failed (%u)", eError));
+	}
+
+	PVR_DPF_RETURN_RC1(eError, *ppsMemDescPtr);
+}
+
+static INLINE PVRSRV_ERROR DevmemFwAllocateSparse(PVRSRV_RGXDEV_INFO *psDevInfo,
+												IMG_DEVMEM_SIZE_T uiSize,
+												IMG_UINT32 ui32NumPhysChunks,
+												IMG_UINT32 ui32NumVirtChunks,
+												IMG_UINT32 *pui32MappingTable,
+												PVRSRV_MEMALLOCFLAGS_T uiFlags,
+												const IMG_CHAR *pszText,
+												DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	IMG_DEV_VIRTADDR sTmpDevVAddr;
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP *psFwHeap;
+	IMG_UINT32 ui32Align;
+
+	PVR_DPF_ENTERED;
+
+	/* Enforce the standard pre-fix naming scheme callers must follow */
+	PVR_ASSERT((pszText != NULL) && (pszText[0] == 'F') && (pszText[1] == 'w'));
+	ui32Align = GET_ROGUE_CACHE_LINE_SIZE(RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS));
+
+	/* Imported from AppHint , flag to poison allocations when freed */
+	uiFlags |= psDevInfo->uiFWPoisonOnFreeFlag;
+
+	eError = _SelectDevMemHeap(psDevInfo, &uiFlags, &psFwHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	RGXFwSharedMemCPUCacheMode(psDevInfo->psDeviceNode,
+	                           &uiFlags);
+
+	eError = DevmemAllocateSparse(psDevInfo->psDeviceNode,
+								uiSize,
+								ui32NumPhysChunks,
+								ui32NumVirtChunks,
+								pui32MappingTable,
+								ui32Align,
+								DevmemGetHeapLog2PageSize(psFwHeap),
+								uiFlags | PVRSRV_MEMALLOCFLAG_SPARSE_NO_SCRATCH_BACKING,
+								pszText,
+								ppsMemDescPtr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	/*
+		We need to map it so the heap for this allocation
+		is set
+	*/
+	eError = DevmemMapToDevice(*ppsMemDescPtr,
+				   psFwHeap,
+				   &sTmpDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		DevmemFree(*ppsMemDescPtr);
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+static INLINE void DevmemFwUnmapAndFree(PVRSRV_RGXDEV_INFO *psDevInfo,
+								DEVMEM_MEMDESC *psMemDesc)
+{
+	PVR_DPF_ENTERED1(psMemDesc);
+
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+
+	DevmemReleaseDevVirtAddr(psMemDesc);
+	DevmemFree(psMemDesc);
+
+	PVR_DPF_RETURN;
+}
+
+/*
+ * This function returns the value of the hardware register RGX_CR_TIMER
+ * which is a timer counting in ticks.
+ */
+
+static INLINE IMG_UINT64 RGXReadHWTimerReg(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 ui64Time = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TIMER);
+
+	/*
+	*  In order to avoid having to issue three 32-bit reads to detect the
+	*  lower 32-bits wrapping, the MSB of the low 32-bit word is duplicated
+	*  in the MSB of the high 32-bit word. If the wrap happens, we just read
+	*  the register again (it will not wrap again so soon).
+	*/
+	if ((ui64Time ^ (ui64Time << 32)) & ~RGX_CR_TIMER_BIT31_CLRMSK)
+	{
+		ui64Time = OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_TIMER);
+	}
+
+	return (ui64Time & ~RGX_CR_TIMER_VALUE_CLRMSK) >> RGX_CR_TIMER_VALUE_SHIFT;
+}
+
+/*
+ * This FW Common Context is only mapped into kernel for initialisation and cleanup purposes.
+ * Otherwise this allocation is only used by the FW.
+ * Therefore the GPU cache doesn't need coherency, and write-combine will
+ * suffice on the CPU side (WC buffer will be flushed at the first kick)
+ */
+#define RGX_FWCOMCTX_ALLOCFLAGS      (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                      PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED)| \
+                                      PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                      PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                      PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+#define RGX_FWCODEDATA_ALLOCFLAGS    (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                      PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                      PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                      PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT | \
+                                      PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                      PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE)
+
+#define RGX_FWSHAREDMEM_MAIN_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                         PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                         PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                         PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                         PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                         PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                         PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+#define RGX_FWSHAREDMEM_CONFIG_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CONFIG))
+
+#define RGX_FWSHAREDMEM_GPU_RO_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN))
+
+/* Firmware memory that is not accessible by the CPU. */
+#define RGX_FWSHAREDMEM_GPU_ONLY_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                             PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                             PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                             PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC)
+
+/* Firmware shared memory that is supposed to be read-only to the CPU.
+ * In reality it isn't due to ZERO_ON_ALLOC which enforces CPU_WRITEABLE
+ * flag on the allocations. */
+#define RGX_FWSHAREDMEM_CPU_RO_ALLOCFLAGS (PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) | \
+                                           PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_READABLE | \
+                                           PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE | \
+                                           PVRSRV_MEMALLOCFLAG_GPU_UNCACHED | \
+                                           PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC | \
+                                           PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC | \
+                                           PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC)
+
+/* data content being kept from previous boot cycles from physical memory must not be cleared during allocation */
+#define RGX_AUTOVZ_KEEP_FW_DATA_MASK(bKeepMem) ((bKeepMem) ? (~PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) : (~0ULL))
+
+/******************************************************************************
+ * RGXSetFirmwareAddress Flags
+ *****************************************************************************/
+#define RFW_FWADDR_FLAG_NONE		(0)			/*!< Void flag */
+#define RFW_FWADDR_NOREF_FLAG		(1U << 0)	/*!< It is safe to immediately release the reference to the pointer,
+												  otherwise RGXUnsetFirmwareAddress() must be call when finished. */
+
+/*************************************************************************/ /*!
+@Function       RGXTraceBufferIsInitRequired
+
+@Description    Returns true if the firmware trace buffer is not allocated and
+                might be required by the firmware soon. Trace buffer allocated
+                on-demand to reduce RAM footprint on systems not needing
+                firmware trace.
+
+@Input          psDevInfo RGX device info
+
+@Return         IMG_BOOL  Whether on-demand allocation(s) is/are needed or not
+*/ /**************************************************************************/
+FORCE_INLINE IMG_BOOL RGXTraceBufferIsInitRequired(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	RGXFwSharedMemCacheOpValue(psTraceBufCtl->ui32LogType, INVALIDATE);
+
+	/* The firmware expects a trace buffer only when:
+	 *	- Logtype is "trace" AND
+	 *	- at least one LogGroup is configured
+	 *	- the Driver Mode is not Guest
+	 */
+	if ((psDevInfo->psRGXFWIfTraceBufferMemDesc[0] == NULL)
+		&& (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_TRACE)
+		&& (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK)
+		&& !PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXTraceBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO* psDevInfo, PVRSRV_MEMALLOCFLAGS_T uiAllocFlags);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+/*************************************************************************/ /*!
+@Function       RGXTBIBufferIsInitRequired
+
+@Description    Returns true if the firmware tbi buffer is not allocated and
+                might be required by the firmware soon. TBI buffer allocated
+                on-demand to reduce RAM footprint on systems not needing
+                tbi.
+
+@Input          psDevInfo RGX device info
+
+@Return         IMG_BOOL  Whether on-demand allocation(s) is/are needed or not
+*/ /**************************************************************************/
+FORCE_INLINE IMG_BOOL RGXTBIBufferIsInitRequired(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_TRACEBUF*  psTraceBufCtl = psDevInfo->psRGXFWIfTraceBufCtl;
+
+	RGXFwSharedMemCacheOpValue(psTraceBufCtl->ui32LogType, INVALIDATE);
+
+	/* The firmware expects a tbi buffer only when:
+	 *	- Logtype is "tbi"
+	 */
+	if ((psDevInfo->psRGXFWIfTBIBufferMemDesc == NULL)
+		 && (psTraceBufCtl->ui32LogType & ~RGXFWIF_LOG_TYPE_TRACE)
+		 && (psTraceBufCtl->ui32LogType & RGXFWIF_LOG_TYPE_GROUP_MASK))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXTBIBufferInitOnDemandResources(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+PVRSRV_ERROR RGXSetupFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                              RGX_INIT_APPHINTS        *psApphints,
+                              IMG_UINT32               ui32ConfigFlags,
+                              IMG_UINT32               ui32ConfigFlagsExt,
+                              IMG_UINT32               ui32FwOsCfgFlags);
+
+
+void RGXFreeFirmware(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*************************************************************************/ /*!
+@Function       RGXSetupFwAllocation
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          psDevInfo       Device Info struct
+@Input          uiAllocFlags    Flags determining type of memory allocation
+@Input          ui32Size        Size of memory allocation
+@Input          pszName         Allocation label
+@Input          psFwPtr         Address of the firmware pointer to set
+@Input          ppvCpuPtr       Address of the cpu pointer to set
+@Input          ui32DevVAFlags  Any combination of  RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetupFwAllocation(PVRSRV_RGXDEV_INFO   *psDevInfo,
+								  PVRSRV_MEMALLOCFLAGS_T uiAllocFlags,
+								  IMG_UINT32           ui32Size,
+								  const IMG_CHAR       *pszName,
+								  DEVMEM_MEMDESC       **ppsMemDesc,
+								  RGXFWIF_DEV_VIRTADDR *psFwPtr,
+								  void                 **ppvCpuPtr,
+								  IMG_UINT32           ui32DevVAFlags);
+
+/*************************************************************************/ /*!
+@Function       RGXSetFirmwareAddress
+
+@Description    Sets a pointer in a firmware data structure.
+
+@Input          ppDest          Address of the pointer to set
+@Input          psSrc           MemDesc describing the pointer
+@Input          ui32Flags       Any combination of RFW_FWADDR_*_FLAG
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSetFirmwareAddress(RGXFWIF_DEV_VIRTADDR	*ppDest,
+								   DEVMEM_MEMDESC		*psSrc,
+								   IMG_UINT32			uiOffset,
+								   IMG_UINT32			ui32Flags);
+
+
+/*************************************************************************/ /*!
+@Function       RGXSetMetaDMAAddress
+
+@Description    Fills a Firmware structure used to setup the Meta DMA with two
+                pointers to the same data, one on 40 bit and one on 32 bit
+                (pointer in the FW memory space).
+
+@Input          ppDest          Address of the structure to set
+@Input          psSrcMemDesc    MemDesc describing the pointer
+@Input          psSrcFWDevVAddr Firmware memory space pointer
+
+@Return         void
+*/ /**************************************************************************/
+void RGXSetMetaDMAAddress(RGXFWIF_DMA_ADDR		*psDest,
+						  DEVMEM_MEMDESC		*psSrcMemDesc,
+						  RGXFWIF_DEV_VIRTADDR	*psSrcFWDevVAddr,
+						  IMG_UINT32			uiOffset);
+
+
+/*************************************************************************/ /*!
+@Function       RGXUnsetFirmwareAddress
+
+@Description    Unsets a pointer in a firmware data structure
+
+@Input          psSrc           MemDesc describing the pointer
+
+@Return         void
+*/ /**************************************************************************/
+void RGXUnsetFirmwareAddress(DEVMEM_MEMDESC *psSrc);
+
+/*!
+*******************************************************************************
+@Function       RGXScheduleProcessQueuesKM
+
+@Description    Software command complete handler
+                (sends uncounted kicks for all the DMs through the MISR)
+
+@Input          hCmdCompHandle  RGX device node
+
+@Return         None
+******************************************************************************/
+void RGXScheduleProcessQueuesKM(PVRSRV_CMDCOMP_HANDLE hCmdCompHandle);
+
+
+/*!
+*******************************************************************************
+
+@Function       RGXInstallProcessQueuesMISR
+
+@Description    Installs the MISR to handle Process Queues operations
+
+@Input          phMISR          Pointer to the MISR handler
+@Input          psDeviceNode    RGX Device node
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXInstallProcessQueuesMISR(IMG_HANDLE *phMISR, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR RGXSendCommandsFromDeferredList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bPoll);
+
+/*************************************************************************/ /*!
+@Function       RGXSendCommandWithPowLockAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM without honouring
+                pending cache operations but taking the power lock.
+
+@Input          psDevInfo       Device Info
+@Input          psKCCBCmd       The cmd to send.
+@Input          ui32PDumpFlags  Pdump flags
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSendCommandWithPowLockAndGetKCCBSlot(PVRSRV_RGXDEV_INFO	*psDevInfo,
+													 RGXFWIF_KCCB_CMD	*psKCCBCmd,
+													 IMG_UINT32			ui32PDumpFlags,
+													 IMG_UINT32			*pui32CmdKCCBSlot);
+
+#define RGXSendCommandWithPowLock(psDevInfo, psKCCBCmd, ui32PDumpFlags) \
+  RGXSendCommandWithPowLockAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, NULL)
+
+/*************************************************************************/ /*!
+@Function       RGXSendCommandAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM without honouring
+                pending cache operations or the power lock.
+                The function flushes any deferred KCCB commands first.
+
+@Input          psDevInfo       Device Info
+@Input          psKCCBCmd       The cmd to send.
+@Input          uiPdumpFlags    PDump flags.
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXSendCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+										  RGXFWIF_KCCB_CMD   *psKCCBCmd,
+										  PDUMP_FLAGS_T      uiPdumpFlags,
+										  IMG_UINT32         *pui32CmdKCCBSlot);
+
+#define RGXSendCommand(psDevInfo, psKCCBCmd, ui32PDumpFlags) \
+  RGXSendCommandAndGetKCCBSlot(psDevInfo, psKCCBCmd, ui32PDumpFlags, NULL)
+
+/*************************************************************************/ /*!
+@Function       _RGXScheduleCommandAndGetKCCBSlot
+
+@Description    Sends a command to a particular DM and kicks the firmware but
+                first schedules any commands which have to happen before
+                handle
+
+@Input          psDevInfo          Device Info
+@Input          eDM                To which DM the cmd is sent.
+@Input          psKCCBCmd          The cmd to send.
+@Input          ui32PDumpFlags     PDump flags
+@Input          bCallerHasPwrLock  Caller already has power lock
+@Output         pui32CmdKCCBSlot   When non-NULL:
+                                   - Pointer on return contains the kCCB slot
+                                     number in which the command was enqueued.
+                                   - Resets the value of the allotted slot to
+                                     RGXFWIF_KCCB_RTN_SLOT_RST
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR _RGXScheduleCommandAndGetKCCBSlot(PVRSRV_RGXDEV_INFO *psDevInfo,
+											   RGXFWIF_DM         eKCCBType,
+											   RGXFWIF_KCCB_CMD   *psKCCBCmd,
+											   IMG_UINT32         ui32PDumpFlags,
+											   IMG_UINT32         *pui32CmdKCCBSlot,
+											   IMG_BOOL           bCallerHasPwrLock);
+
+#define RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot) \
+	_RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, pui32CmdKCCBSlot, IMG_FALSE)
+
+#define RGXScheduleCommand(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags) \
+  RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, NULL)
+
+#define RGXScheduleCommandWithoutPowerLock(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags) \
+	_RGXScheduleCommandAndGetKCCBSlot(psDevInfo, eKCCBType, psKCCBCmd, ui32PDumpFlags, NULL, IMG_TRUE)
+
+/*************************************************************************/ /*!
+@Function       RGXWaitForKCCBSlotUpdate
+
+@Description    Waits until the required kCCB slot value is updated by the FW
+                (signifies command completion). Additionally, dumps a relevant
+                PDump poll command.
+
+@Input          psDevInfo       Device Info
+@Input          ui32SlotNum     The kCCB slot number to wait for an update on
+@Input          ui32PDumpFlags
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXWaitForKCCBSlotUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                      IMG_UINT32 ui32SlotNum,
+									  IMG_UINT32 ui32PDumpFlags);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXFrameworkCopyCommand
+
+@Description    Copy framework command into FW addressable buffer
+
+@param          psDeviceNode
+@param          psFWFrameworkMemDesc
+@param          pbyGPUFRegisterList
+@param          ui32FrameworkRegisterSize
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFrameworkCopyCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+										   DEVMEM_MEMDESC   *psFWFrameworkMemDesc,
+										   IMG_PBYTE        pbyGPUFRegisterList,
+										   IMG_UINT32       ui32FrameworkRegisterSize);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXFrameworkCreateKM
+
+@Description    Create FW addressable buffer for framework
+
+@param          psDeviceNode
+@param          ppsFWFrameworkMemDesc
+@param          ui32FrameworkRegisterSize
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXFrameworkCreateKM(PVRSRV_DEVICE_NODE * psDeviceNode,
+										DEVMEM_MEMDESC     ** ppsFWFrameworkMemDesc,
+										IMG_UINT32         ui32FrameworkRegisterSize);
+
+/*************************************************************************/ /*!
+@Function       RGXPollForGPCommandCompletion
+
+@Description    Polls for completion of a submitted GP command. Poll is done
+                on a value matching a masked read from the address.
+
+@Input          psDevNode       Pointer to device node struct
+@Input          pui32LinMemAddr CPU linear address to poll
+@Input          ui32Value       Required value
+@Input          ui32Mask        Mask
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPollForGPCommandCompletion(PVRSRV_DEVICE_NODE *psDevNode,
+									volatile IMG_UINT32 __iomem *pui32LinMemAddr,
+									IMG_UINT32                   ui32Value,
+									IMG_UINT32                   ui32Mask);
+
+/*************************************************************************/ /*!
+@Function       RGXStateFlagCtrl
+
+@Description    Set and return FW internal state flags.
+
+@Input          psDevInfo       Device Info
+@Input          ui32Config      AppHint config flags
+@Output         pui32State      Current AppHint state flag configuration
+@Input          bSetNotClear    Set or clear the provided config flags
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXStateFlagCtrl(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_UINT32 *pui32State,
+				IMG_BOOL bSetNotClear);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestCommonContextCleanUp
+
+@Description    Schedules a FW common context cleanup. The firmware doesn't
+                block waiting for the resource to become idle but rather
+                notifies the host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psServerCommonContext context to be cleaned up
+@Input          eDM             Data master, to which the cleanup command should
+                                be sent
+@Input          ui32PDumpFlags  PDump continuous flag
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestCommonContextCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+											  RGX_SERVER_COMMON_CONTEXT *psServerCommonContext,
+											  RGXFWIF_DM eDM,
+											  IMG_UINT32 ui32PDumpFlags);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestHWRTDataCleanUp
+
+@Description    Schedules a FW HWRTData memory cleanup. The firmware doesn't
+                block waiting for the resource to become idle but rather
+                notifies the host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psHWRTData      firmware address of the HWRTData for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestHWRTDataCleanUp(PVRSRV_DEVICE_NODE *psDeviceNode,
+										 PRGXFWIF_HWRTDATA psHWRTData);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestFreeListCleanUp
+
+@Description    Schedules a FW FreeList cleanup. The firmware doesn't block
+                waiting for the resource to become idle but rather notifies the
+                host that the resource is busy.
+
+@Input          psDeviceNode    pointer to device node
+@Input          psFWFreeList    firmware address of the FreeList for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestFreeListCleanUp(PVRSRV_RGXDEV_INFO *psDeviceNode,
+										 PRGXFWIF_FREELIST psFWFreeList);
+
+/*!
+*******************************************************************************
+@Function       RGXFWRequestZSBufferCleanUp
+
+@Description    Schedules a FW ZS Buffer cleanup. The firmware doesn't block
+                waiting for the resource to become idle but rather notifies the
+                host that the resource is busy.
+
+@Input          psDevInfo       pointer to device node
+@Input          psFWZSBuffer    firmware address of the ZS Buffer for clean-up
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWRequestZSBufferCleanUp(PVRSRV_RGXDEV_INFO *psDevInfo,
+										 PRGXFWIF_ZSBUFFER psFWZSBuffer);
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetHCSDeadline
+
+@Description    Requests the Firmware to set a new Hard Context Switch timeout
+                deadline. Context switches that surpass that deadline cause the
+                system to kill the currently running workloads.
+
+@Input          psDeviceNode    pointer to device node
+@Input          ui32HCSDeadlineMs  The deadline in milliseconds.
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWSetHCSDeadline(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32HCSDeadlineMs);
+
+/*!
+*******************************************************************************
+@Function       RGXFWHealthCheckCmdInt
+
+@Description    Ping the firmware to check if it is responsive.
+
+@Input          psDevInfo          pointer to device info
+@Input          bCallerHasPwrLock  Caller already has power lock
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWHealthCheckCmdInt(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bCallerHasPwrLock);
+
+#define RGXFWHealthCheckCmd(psDevInfo)	\
+	RGXFWHealthCheckCmdInt(psDevInfo, IMG_FALSE)
+
+#define RGXFWHealthCheckCmdWithoutPowerLock(psDevInfo)	\
+	RGXFWHealthCheckCmdInt(psDevInfo, IMG_TRUE)
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetFwOsState
+
+@Description    Requests the Firmware to change the guest OS Online states.
+                This should be initiated by the VMM when a guest VM comes
+                online or goes offline. If offline, the FW offloads any current
+                resource from that DriverID. The request is repeated until the
+                FW has had time to free all the resources or has waited for
+                workloads to finish.
+
+@Input          psDevInfo       pointer to device info
+@Input          ui32DriverID    The driver whose state is being altered
+@Input          eOSOnlineState  The new state (Online or Offline)
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWSetFwOsState(PVRSRV_RGXDEV_INFO *psDevInfo,
+								IMG_UINT32 ui32DriverID,
+								RGXFWIF_OS_STATE_CHANGE eOSOnlineState);
+
+#if defined(SUPPORT_AUTOVZ)
+/*!
+*******************************************************************************
+@Function       RGXUpdateAutoVzWdgToken
+
+@Description    If the driver-firmware connection is active, read the
+                firmware's watchdog token and copy its value back into the OS
+                token. This indicates to the firmware that this driver is alive
+                and responsive.
+
+@Input          psDevInfo       pointer to device info
+******************************************************************************/
+void RGXUpdateAutoVzWdgToken(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXDisconnectAllGuests
+
+@Description    Send requests to FW to disconnect all guest connections.
+
+@Input          psDeviceNode    pointer to device node
+******************************************************************************/
+PVRSRV_ERROR RGXDisconnectAllGuests(PVRSRV_DEVICE_NODE *psDeviceNode);
+#endif
+
+/*!
+*******************************************************************************
+@Function       RGXFWConfigPHR
+
+@Description    Configure the Periodic Hardware Reset functionality
+
+@Input          psDevInfo       pointer to device info
+@Input          ui32PHRMode     desired PHR mode
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWConfigPHR(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32PHRMode);
+
+/*!
+*******************************************************************************
+@Function       RGXFWConfigWdg
+
+@Description    Configure the Safety watchdog trigger period
+
+@Input          psDevInfo        pointer to device info
+@Input          ui32WdgPeriodUs  requested period in microseconds
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWConfigWdg(PVRSRV_RGXDEV_INFO *psDevInfo,
+                            IMG_UINT32 ui32WdgPeriod);
+
+/*!
+*******************************************************************************
+@Function       RGXCheckFirmwareCCB
+
+@Description    Processes all commands that are found in the Firmware CCB.
+
+@Input          psDevInfo       pointer to device
+
+@Return         None
+******************************************************************************/
+void RGXCheckFirmwareCCB(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXCheckForStalledClientContexts
+
+@Description    Checks all client contexts, for the device with device info
+                provided, to see if any are waiting for a fence to signal and
+                optionally force signalling of the fence for the context which
+                has been waiting the longest.
+                This function is called by RGXUpdateHealthStatus() and also
+                may be invoked from other trigger points.
+
+@Input          psDevInfo       pointer to device info
+@Input          bIgnorePrevious If IMG_TRUE, any stalled contexts will be
+                                indicated immediately, rather than only
+                                checking against any previous stalled contexts
+
+@Return         None
+******************************************************************************/
+void RGXCheckForStalledClientContexts(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_BOOL bIgnorePrevious);
+
+/*!
+*******************************************************************************
+@Function       RGXUpdateHealthStatus
+
+@Description    Tests a number of conditions which might indicate a fatal error
+                has occurred in the firmware. The result is stored in the
+                device node eHealthStatus.
+
+@Input         psDevNode        Pointer to device node structure.
+@Input         bCheckAfterTimePassed  When TRUE, the function will also test
+                                for firmware queues and polls not changing
+                                since the previous test.
+
+                                Note: if not enough time has passed since the
+                                last call, false positives may occur.
+
+@Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXUpdateHealthStatus(PVRSRV_DEVICE_NODE* psDevNode,
+                                   IMG_BOOL bCheckAfterTimePassed);
+
+#if defined(SUPPORT_AUTOVZ)
+/*!
+*******************************************************************************
+@Function       RGXUpdateAutoVzWatchdog
+
+@Description    Updates AutoVz watchdog that maintains the fw-driver connection
+
+@Input         psDevNode        Pointer to device node structure.
+******************************************************************************/
+void RGXUpdateAutoVzWatchdog(PVRSRV_DEVICE_NODE* psDevNode);
+#endif /* SUPPORT_AUTOVZ */
+
+/*!
+*******************************************************************************
+@Function       AttachKickResourcesCleanupCtls
+
+@Description    Attaches the cleanup structures to a kick command so that
+                submission reference counting can be performed when the
+                firmware processes the command
+
+@Output         apsCleanupCtl   Array of CleanupCtl structure pointers to populate.
+@Output         pui32NumCleanupCtl  Number of CleanupCtl structure pointers written out.
+@Input          eDM             Which data master is the subject of the command.
+@Input          bKick           TRUE if the client originally wanted to kick this DM.
+@Input          psRTDataCleanup Optional RTData cleanup associated with the command.
+@Input          psZBuffer       Optional ZSBuffer associated with the command.
+
+@Return        PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR AttachKickResourcesCleanupCtls(PRGXFWIF_CLEANUP_CTL *apsCleanupCtl,
+									IMG_UINT32 *pui32NumCleanupCtl,
+									RGXFWIF_DM eDM,
+									IMG_BOOL bKick,
+									RGX_KM_HW_RT_DATASET           *psKMHWRTDataSet,
+									RGX_ZSBUFFER_DATA              *psZSBuffer,
+									RGX_ZSBUFFER_DATA              *psMSAAScratchBuffer);
+
+/*!
+*******************************************************************************
+@Function       RGXResetHWRLogs
+
+@Description    Resets the HWR Logs buffer
+                (the hardware recovery count is not reset)
+
+@Input          psDevNode       Pointer to the device
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXResetHWRLogs(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*!
+*******************************************************************************
+@Function       RGXGetPhyAddr
+
+@Description    Get the physical address of a PMR at an offset within it
+
+@Input          psPMR           PMR of the allocation
+@Input          ui32LogicalOffset  Logical offset
+
+@Output         psPhyAddr       Physical address of the allocation
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXGetPhyAddr(PMR *psPMR,
+						   IMG_DEV_PHYADDR *psPhyAddr,
+						   IMG_UINT32 ui32LogicalOffset,
+						   IMG_UINT32 ui32Log2PageSize,
+						   IMG_UINT32 ui32NumOfPages,
+						   IMG_BOOL *bValid);
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+@Function       RGXPdumpDrainKCCB
+
+@Description    Wait for the firmware to execute all the commands in the kCCB
+
+@Input          psDevInfo       Pointer to the device
+@Input          ui32WriteOffset Woff we have to POL for the Roff to be equal to
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXPdumpDrainKCCB(PVRSRV_RGXDEV_INFO *psDevInfo,
+							   IMG_UINT32 ui32WriteOffset);
+#endif /* PDUMP */
+
+/*!
+*******************************************************************************
+@Function       RGXFwRawHeapAllocMap
+
+@Description    Register and maps to device, a raw firmware physheap
+
+@Return         PVRSRV_ERROR    PVRSRV_OK on success.
+                                Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR RGXFwRawHeapAllocMap(PVRSRV_DEVICE_NODE *psDeviceNode,
+								  IMG_UINT32 ui32DriverID,
+								  IMG_DEV_PHYADDR sDevPAddr,
+								  IMG_UINT64 ui64DevPSize);
+
+/*!
+*******************************************************************************
+@Function       RGXFwRawHeapUnmapFree
+
+@Description    Unregister and unmap from device, a raw firmware physheap
+******************************************************************************/
+void RGXFwRawHeapUnmapFree(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   IMG_UINT32 ui32DriverID);
+
+/*!
+*******************************************************************************
+@Function       RGXReadFWModuleAddr
+
+@Description    Read a value at the given address in META or RISCV memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in META or RISCV memory space
+
+@Output         pui32Value      Read value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXReadFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                 IMG_UINT32 ui32Addr,
+                                 IMG_UINT32 *pui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXWriteFWModuleAddr
+
+@Description    Write a value to the given address in META or RISC memory space
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32Addr        Address in RISC-V memory space
+@Input          ui32Value       Write value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXWriteFWModuleAddr(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                  IMG_UINT32 ui32MemAddr,
+                                  IMG_UINT32 ui32Value);
+
+/*!
+*******************************************************************************
+@Function       RGXGetFwMapping
+
+@Description    Retrieve any of the CPU Physical Address, Device Physical
+                Address or the raw value of the page table entry associated
+                with the firmware virtual address given.
+
+@Input          psDevInfo       Pointer to device info
+@Input          ui32FwVA        The Fw VA that needs decoding
+@Output         psCpuPA         Pointer to the resulting CPU PA
+@Output         psDevPA         Pointer to the resulting Dev PA
+@Output         pui64RawPTE     Pointer to the raw Page Table Entry value
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXGetFwMapping(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                    IMG_UINT32 ui32FwVA,
+                                    IMG_CPU_PHYADDR *psCpuPA,
+                                    IMG_DEV_PHYADDR *psDevPA,
+                                    IMG_UINT64 *pui64RawPTE);
+
+/*!
+*******************************************************************************
+@Function       RGXFWInjectFault
+
+@Description    Injecting firmware fault to validate recovery through Host
+
+@Input          psDevInfo       Pointer to device info
+
+@Return         PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR RGXFWInjectFault(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+@Function       RGXFWSetVzConnectionCooldownPeriod
+
+@Description    Set Vz connection cooldown period
+
+@Input          psDevInfo                              pointer to device info
+@Input          ui32VzConnectionCooldownPeriodInSec    Cooldown period in secs
+******************************************************************************/
+PVRSRV_ERROR
+RGXFWSetVzConnectionCooldownPeriod(PVRSRV_RGXDEV_INFO *psDevInfo,
+				   IMG_UINT32 ui32VzConnectionCooldownPeriodInSec);
+
+#if defined(SUPPORT_AUTOVZ_HW_REGS) && !defined(SUPPORT_AUTOVZ)
+#error "VZ build configuration error: use of OS scratch registers supported only in AutoVz drivers."
+#endif
+
+#if defined(SUPPORT_AUTOVZ_HW_REGS)
+/* AutoVz with hw support */
+#define KM_GET_FW_CONNECTION(psDevInfo)				OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH3)
+#define KM_GET_OS_CONNECTION(psDevInfo)				OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH2)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH2, RGXFW_CONNECTION_OS_##val)
+
+#define KM_GET_FW_ALIVE_TOKEN(psDevInfo)			OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH1)
+#define KM_GET_OS_ALIVE_TOKEN(psDevInfo)			OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH0)
+#define KM_SET_OS_ALIVE_TOKEN(val, psDevInfo)		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_OS0_SCRATCH0, val)
+
+#define KM_ALIVE_TOKEN_CACHEOP(Target, CacheOp)
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)
+
+#else
+
+#if defined(SUPPORT_AUTOVZ)
+#define KM_GET_FW_ALIVE_TOKEN(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->ui32AliveFwToken)
+#define KM_GET_OS_ALIVE_TOKEN(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->ui32AliveOsToken)
+#define KM_SET_OS_ALIVE_TOKEN(val, psDevInfo)		do { \
+                                                    OSWriteDeviceMem32WithWMB((volatile IMG_UINT32 *) &psDevInfo->psRGXFWIfConnectionCtl->ui32AliveOsToken, val); \
+                                                    KM_ALIVE_TOKEN_CACHEOP(Os, FLUSH); \
+                                                    } while (0)
+
+#define KM_ALIVE_TOKEN_CACHEOP(Target, CacheOp)     RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfConnectionCtl->ui32Alive##Target##Token, \
+													                           CacheOp);
+#endif /* defined(SUPPORT_AUTOVZ) */
+
+#if !defined(NO_HARDWARE) && defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/* static, dynamic and AutoVz DDKs using shared memory */
+#define KM_GET_FW_CONNECTION(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->eConnectionFwState)
+#define KM_GET_OS_CONNECTION(psDevInfo)			(psDevInfo->psRGXFWIfConnectionCtl->eConnectionOsState)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)	do { \
+                                                OSWriteDeviceMem32WithWMB((void*)&psDevInfo->psRGXFWIfConnectionCtl->eConnectionOsState, RGXFW_CONNECTION_OS_##val); \
+                                                KM_CONNECTION_CACHEOP(Os, FLUSH); \
+                                                } while (0)
+
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)  RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfConnectionCtl->eConnection##Target##State, \
+												                           CacheOp);
+#else
+/* nohw & native */
+#define KM_GET_FW_CONNECTION(psDevInfo)			(RGXFW_CONNECTION_FW_ACTIVE)
+#define KM_GET_OS_CONNECTION(psDevInfo)			(RGXFW_CONNECTION_OS_ACTIVE)
+#define KM_SET_OS_CONNECTION(val, psDevInfo)
+#define KM_CONNECTION_CACHEOP(Target, CacheOp)
+#endif /* defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED == 1) */
+#endif /* defined(SUPPORT_AUTOVZ_HW_REGS) */
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+#define RGX_FIRST_RAW_HEAP_DRIVER_ID		RGXFW_HOST_DRIVER_ID
+#else
+#define RGX_FIRST_RAW_HEAP_DRIVER_ID		RGXFW_GUEST_DRIVER_ID_START
+#endif
+
+#define KM_OS_CONNECTION_IS(val, psDevInfo)		(KM_GET_OS_CONNECTION(psDevInfo) == RGXFW_CONNECTION_OS_##val)
+#define KM_FW_CONNECTION_IS(val, psDevInfo)		(KM_GET_FW_CONNECTION(psDevInfo) == RGXFW_CONNECTION_FW_##val)
+
+#endif /* RGXFWUTILS_H */
+/******************************************************************************
+ End of file (rgxfwutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxheapconfig.h b/drivers/gpu/drm/img-rogue/rgxheapconfig.h
new file mode 100644
index 000000000000..6825da5caa3b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxheapconfig.h
@@ -0,0 +1,288 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_H
+#define RGXHEAPCONFIG_H
+
+#include "rgxdefs_km.h"
+
+#define RGX_HEAP_SIZE_32KiB      IMG_UINT64_C(0x0000008000)
+#define RGX_HEAP_SIZE_2MiB       IMG_UINT64_C(0x0000200000)
+#define RGX_HEAP_SIZE_4MiB       IMG_UINT64_C(0x0000400000)
+#define RGX_HEAP_SIZE_16MiB      IMG_UINT64_C(0x0001000000)
+#define RGX_HEAP_SIZE_32MiB      IMG_UINT64_C(0x0002000000)
+#define RGX_HEAP_SIZE_256MiB     IMG_UINT64_C(0x0010000000)
+
+#define RGX_HEAP_SIZE_1GiB       IMG_UINT64_C(0x0040000000)
+#define RGX_HEAP_SIZE_2GiB       IMG_UINT64_C(0x0080000000)
+#define RGX_HEAP_SIZE_4GiB       IMG_UINT64_C(0x0100000000)
+#define RGX_HEAP_SIZE_16GiB      IMG_UINT64_C(0x0400000000)
+#define RGX_HEAP_SIZE_32GiB      IMG_UINT64_C(0x0800000000)
+#define RGX_HEAP_SIZE_64GiB      IMG_UINT64_C(0x1000000000)
+#define RGX_HEAP_SIZE_128GiB     IMG_UINT64_C(0x2000000000)
+#define RGX_HEAP_SIZE_256GiB     IMG_UINT64_C(0x4000000000)
+#define RGX_HEAP_SIZE_512GiB     IMG_UINT64_C(0x8000000000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address heaps that are used in
+	application memory contexts. It also shows where the Firmware memory heap
+	fits into this, but the firmware heap is only ever created in the
+	Services KM/server component.
+
+	RGX_PDSCODEDATA_HEAP_BASE and RGX_USCCODE_HEAP_BASE will be programmed,
+	on a global basis, into RGX_CR_PDS_EXEC_BASE and RGX_CR_USC_CODE_BASE_*
+	respectively. Therefore if clients use multiple configs they must still
+	be consistent with their definitions for these heaps.
+
+	Shared virtual memory (GENERAL_SVM) support requires half of the address
+	space (512 GiB) be reserved for SVM allocations to mirror application CPU
+	addresses. However, if BRN_65273 WA is active in which case the SVM heap
+	is disabled. This is reflected in the device connection capability bits
+	returned to user space.
+
+	The GENERAL non-SVM region is 512 GiB to 768 GiB and is shared between the
+	general (4KiB) heap and the general non-4K heap. The first 128 GiB is used
+	for the GENERAL_HEAP (4KiB) and the last 32 GiB is used for the
+	GENERAL_NON4K_HEAP. This heap has a default page-size of 16K.
+	AppHint PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE can be used to forced it
+	to these values: 4K,64K,256K,1M,2M.
+
+	The heaps defined for BRN_65273 _replace_ the non-BRN equivalents below
+	when this BRN WA is active on affected cores. This is different to most
+	other BRNs and hence has been given its own header file for clarity,
+	see below. This	is a special case, other BRNs that need 1 or 2 additional
+	heaps should be added to this file, like BRN_63142 below.
+	NOTE: All regular heaps below greater than 1GB require a BRN_65273 WA heap.
+
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range. Often this is the reserved
+	size of the heap within the range.
+*/
+
+/* This BRN requires a different virtual memory map from the standard one
+ * defined in this file below. Hence the alternative heap definitions for this
+ * BRN are provided in a separate file for clarity. */
+#include "rgxheapconfig_65273.h"
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0000_0000 - 0x00_0020_0000 **/
+	/* 0 MiB to 2 MiB, size of 2 MiB : RESERVED (only when General SVM
+	 *                                           doesn't exist) **/
+
+	/* BRN_65273 TQ3DPARAMETERS base 0x0000010000 */
+	/* BRN_65273 GENERAL base        0x65C0000000 */
+	/* BRN_65273 GENERAL_NON4K base  0x73C0000000 */
+
+/* 0x00_0000_8000 - 0x7F_FFFF_8000 **/
+	/* MAX(32 KiB, PAGE_SIZE) to 512 GiB, size of 512 GiB less MAX(32 KiB, PAGE_SIZE) : GENERAL_SVM_HEAP **/
+
+	/* The MAX is determined at runtime (PAGE_SIZE isn't available on all platforms)
+	 * so the #define's must NOT be used directly. Use the heap config after initialisation. */
+	#define RGX_GENERAL_SVM_HEAP_BASE           IMG_UINT64_C(0x0000008000)
+	#define RGX_GENERAL_SVM_HEAP_SIZE           (RGX_HEAP_SIZE_512GiB - RGX_HEAP_SIZE_32KiB)
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0x80_0000_0000 - 0x9F_FFFF_FFFF **/
+	/* 512 GiB to 640 GiB, size of 128 GiB : GENERAL_HEAP **/
+	#define RGX_GENERAL_HEAP_BASE               IMG_UINT64_C(0x8000000000)
+	#define RGX_GENERAL_HEAP_SIZE               RGX_HEAP_SIZE_128GiB
+
+	/* BRN_65273 PDSCODEDATA base    0xA800000000 */
+
+/* 0xA0_0000_0000 - 0xAF_FFFF_FFFF **/
+	/* 640 GiB to 704 GiB, size of 64 GiB : FREE **/
+
+/* B0_0000_0000 - 0xB7_FFFF_FFFF **/
+	/* 704 GiB to 736 GiB, size of 32 GiB : FREE **/
+
+	/* BRN_65273 USCCODE base        0xBA00000000 */
+
+/* 0xB8_0000_0000 - 0xBF_FFFF_FFFF **/
+	/* 736 GiB to 768 GiB, size of 32 GiB : GENERAL_NON4K_HEAP **/
+	#define RGX_GENERAL_NON4K_HEAP_BASE         IMG_UINT64_C(0xB800000000)
+	#define RGX_GENERAL_NON4K_HEAP_SIZE         RGX_HEAP_SIZE_32GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xC0_0000_0000 - 0xD9_FFFF_FFFF **/
+	/* 768 GiB to 872 GiB, size of 104 GiB : FREE **/
+
+/* 0xDA_0000_0000 - 0xDA_FFFF_FFFF **/
+	/* 872 GiB to 876 GiB, size of 4 GiB : PDSCODEDATA_HEAP **/
+	#define RGX_PDSCODEDATA_HEAP_BASE           IMG_UINT64_C(0xDA00000000)
+	#define RGX_PDSCODEDATA_HEAP_SIZE           RGX_HEAP_SIZE_4GiB
+
+/* 0xDB_0000_0000 - 0xDB_FFFF_FFFF **/
+	/* 876 GiB to 880 GiB, size of 256 MiB (reserved 4GiB) : BRN **/
+	/* HWBRN63142 workaround requires Region Header memory to be at the top
+	   of a 16GiB aligned range. This is so when masked with 0x03FFFFFFFF the
+	   address will avoid aliasing PB addresses. Start at 879.75GiB. Size of 256MiB. */
+	#define RGX_RGNHDR_BRN_63142_HEAP_BASE      IMG_UINT64_C(0xDBF0000000)
+	#define RGX_RGNHDR_BRN_63142_HEAP_SIZE      RGX_HEAP_SIZE_256MiB
+
+/* 0xDC_0000_0000 - 0xDF_FFFF_FFFF **/
+	/* 880 GiB to 896 GiB, size of 16 GiB : FREE **/
+
+/* 0xE0_0000_0000 - 0xE0_FDFF_FFFF **/
+	/* 896 GiB to 900 GiB, size of 4 GiB less 32 MiB : USCCODE_HEAP **/
+	#define RGX_USCCODE_HEAP_BASE               IMG_UINT64_C(0xE000000000)
+	#define RGX_USCCODE_HEAP_SIZE               (RGX_HEAP_SIZE_4GiB - RGX_HEAP_SIZE_32MiB)
+
+/* 0xE0_FE00_0000 - 0xE0_FFFF_FFFF **/
+	/* 900 GiB less 32 MiB to 900 GiB, size of 32 MiB : RESERVED VOLCANIC **/
+
+/* 0xE1_0000_0000 - 0xE1_BFFF_FFFF **/
+	/* 900 GiB to 903 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xE1_C000_000 - 0xE1_FFFF_FFFF **/
+	/* 903 GiB to 904 GiB, reserved 1 GiB, : FIRMWARE_HEAP **/
+
+	/* Firmware heaps defined in rgx_heap_firmware.h as they are not present in
+	   application memory contexts, see:
+	    RGX_FIRMWARE_RAW_HEAP_BASE
+	    RGX_FIRMWARE_RAW_HEAP_SIZE
+	   See header for other sub-heaps details
+	*/
+
+/* 0xE2_0000_0000 - 0xE3_FFFF_FFFF **/
+	/* 904 GiB to 912 GiB, size of 8 GiB : FREE **/
+
+	/* BRN_65273 VISIBILITY_TEST base 0xE400000000 */
+
+/* 0xE4_0000_0000 - 0xE7_FFFF_FFFF **/
+	/* 912 GiB to 928 GiB, size 16 GiB : TQ3DPARAMETERS_HEAP **/
+	/* Aligned to match RGX_CR_ISP_PIXEL_BASE at 16 GiB */
+	#define RGX_TQ3DPARAMETERS_HEAP_BASE        IMG_UINT64_C(0xE400000000)
+	#define RGX_TQ3DPARAMETERS_HEAP_SIZE        RGX_HEAP_SIZE_16GiB
+
+/* 0xE8_0000_0000 - 0xE8_FFFF_FFFF **/
+	/* 928 GiB to 932 GiB, size of 4 GiB : FREE **/
+
+/* 0xE9_0000_0000 - 0xE9_3FFF_FFFF **/
+	/* 932 GiB to 933 GiB, size of 1 GiB : VK_CAPT_REPLAY_HEAP **/
+	#define RGX_VK_CAPT_REPLAY_HEAP_BASE        IMG_UINT64_C(0xE900000000)
+	#define RGX_VK_CAPT_REPLAY_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xE9_4000_0000 - 0xE9_FFFF_FFFF **/
+	/* 933 GiB to 936 GiB, size of 3 GiB : FREE **/
+
+/* 0xEA_0000_0000 - 0xEA_001F_FFFF **/
+	/* 936 GiB to 937 GiB, size of 1 GiB : FREE **/
+
+/* 0xEA_4000_0000 - 0xEA_FFFF_FFFF **/
+	/* 937 GiB to 940 GiB, size of 3 GiB : FREE **/
+
+/* 0xEB_0000_0000 - 0xEB_FFFF_FFFF **/
+	/* 940 GiB to 944 GiB, size of 4 GiB : RESERVED VOLCANIC  **/
+
+/* 0xEC_0000_0000 - 0xEC_001F_FFFF **/
+	/* 944 GiB to 945 GiB, size 2 MiB : FBCDC_HEAP **/
+	#define RGX_FBCDC_HEAP_BASE                 IMG_UINT64_C(0xEC00000000)
+	#define RGX_FBCDC_HEAP_SIZE                 RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_4000_0000 - 0xEC_401F_FFFF **/
+	/* 945 GiB to 946 GiB, size 2 MiB : FBCDC_LARGE_HEAP **/
+	#define RGX_FBCDC_LARGE_HEAP_BASE           IMG_UINT64_C(0xEC40000000)
+	#define RGX_FBCDC_LARGE_HEAP_SIZE           RGX_HEAP_SIZE_2MiB
+
+/* 0xEC_8000_0000 - 0xED_FFFF_FFFF **/
+	/* 946 GiB to 952 GiB, size of 6 GiB : RESERVED VOLCANIC  **/
+
+/* 0xEE_0000_0000 - 0xEE_3FFF_FFFF **/
+	/* 952 GiB to 953 GiB, size of 1 GiB : CMP_MISSION_RMW_HEAP **/
+	#define RGX_CMP_MISSION_RMW_HEAP_BASE       IMG_UINT64_C(0xEE00000000)
+	#define RGX_CMP_MISSION_RMW_HEAP_SIZE       RGX_HEAP_SIZE_1GiB
+
+/* 0xEE_4000_0000 - 0xEE_FFFF_FFFF **/
+	/* 953 GiB to 956 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xEF_0000_0000 - 0xEF_3FFF_FFFF **/
+	/* 956 GiB to 957 GiB, size of 1 GiB : CMP_SAFETY_RMW_HEAP **/
+	#define RGX_CMP_SAFETY_RMW_HEAP_BASE        IMG_UINT64_C(0xEF00000000)
+	#define RGX_CMP_SAFETY_RMW_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+/* 0xEF_4000_0000 - 0xEF_FFFF_FFFF **/
+	/* 957 GiB to 960 GiB, size of 3 GiB : RESERVED **/
+
+/* 0xF0_0000_0000 - 0xF0_FFFF_FFFF **/
+	/* 960 GiB to 964 GiB, size of 4 GiB : TEXTURE_STATE_HEAP (36-bit aligned) */
+	#define RGX_TEXTURE_STATE_HEAP_BASE         IMG_UINT64_C(0xF000000000)
+	#define RGX_TEXTURE_STATE_HEAP_SIZE         RGX_HEAP_SIZE_4GiB
+
+/* 0xF1_0000_0000 - 0xF1_FFFF_FFFF **/
+	/* 964 GiB to 968 GiB, size of 4 GiB : FREE **/
+
+/* 0xF2_0000_0000 - 0xF2_001F_FFFF **/
+	/* 968 GiB to 969 GiB, size of 2 MiB : VISIBILITY_TEST_HEAP **/
+	#define RGX_VISIBILITY_TEST_HEAP_BASE       IMG_UINT64_C(0xF200000000)
+	#define RGX_VISIBILITY_TEST_HEAP_SIZE       RGX_HEAP_SIZE_2MiB
+
+/* 0xF2_4000_0000 - 0xF2_FFFF_FFFF **/
+	/* 969 GiB to 972 GiB, size of 3 GiB : FREE **/
+
+	/* BRN_65273 MMU_INIA base 0xF800000000 */
+	/* BRN_65273 MMU_INIB base 0xF900000000 */
+
+/* 0xF3_0000_0000 - 0xFF_FFFF_FFFF **/
+	/* 972 GiB to 1024 GiB, size of 52 GiB : FREE **/
+
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxheapconfig_65273.h b/drivers/gpu/drm/img-rogue/rgxheapconfig_65273.h
new file mode 100644
index 000000000000..66f44dc8c18b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxheapconfig_65273.h
@@ -0,0 +1,129 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Device virtual memory map for BRN_65273.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Memory heaps device specific configuration
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHEAPCONFIG_65273_H
+#define RGXHEAPCONFIG_65273_H
+
+#define RGX_HEAP_SIZE_64KiB      IMG_UINT64_C(0x0000010000)
+
+/*
+	RGX Device Virtual Address Space Definitions
+
+	This file defines the RGX virtual address replacement heaps that are used
+	in application memory contexts when BRN_65273 is present. The addresses
+	used are specifically crafted to avoid the BRN - under no circumstances
+	can they be moved, altered in size or additional heaps added.
+
+	The heaps defined for BRN_65273 _replace_ the non-BRN equivalents when this
+	BRN WA is active on affected cores. This is different to most other BRNs
+	and hence has been given its own header file for clarity. The SVM_HEAP is
+	also disabled and unavailable when the WA is active. This is reflected
+	in the device connection capability bits returned to user space.
+
+	NOTE: All regular heaps in rgxheapconfig.h greater than 1GB require
+	      a BRN_65273 WA heap.
+
+	Base addresses must have to be a multiple of 4MiB
+	Heaps must not start at 0x0000000000, as this is reserved for internal
+	use within device memory layer.
+	Range comments, those starting in column 0 below are a section heading of
+	sorts and are above the heaps in that range.
+*/
+
+
+/* 0x00_0000_0000 ************************************************************/
+
+/* 0x00_0001_0000 - 0x00_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires TQ memory to start at 64 KiB and use a
+	 * unique single 0.99GiB PCE entry. */
+	#define RGX_TQ3DPARAMETERS_BRN_65273_HEAP_BASE  IMG_UINT64_C(0x0000010000)
+	#define RGX_TQ3DPARAMETERS_BRN_65273_HEAP_SIZE  (RGX_HEAP_SIZE_1GiB - RGX_HEAP_SIZE_64KiB)
+
+/* 0x65_C000_0000 - 0x66_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires General Heap to use a unique PCE entry for each GiB in range */
+	#define RGX_GENERAL_BRN_65273_HEAP_BASE         IMG_UINT64_C(0x65C0000000)
+	#define RGX_GENERAL_BRN_65273_HEAP_SIZE         RGX_HEAP_SIZE_2GiB
+
+/* 0x73_C000_0000 - 0x74_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires Non4K memory to use a unique PCE entry for each GiB in range */
+	#define RGX_GENERAL_NON4K_BRN_65273_HEAP_BASE   IMG_UINT64_C(0x73C0000000)
+	#define RGX_GENERAL_NON4K_BRN_65273_HEAP_SIZE   RGX_HEAP_SIZE_2GiB
+
+
+/* 0x80_0000_0000 ************************************************************/
+
+/* 0xA8_0000_0000 - 0xA8_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires PDS memory to use a unique single 1GiB PCE entry. */
+	#define RGX_PDSCODEDATA_BRN_65273_HEAP_BASE     IMG_UINT64_C(0xA800000000)
+	#define RGX_PDSCODEDATA_BRN_65273_HEAP_SIZE     RGX_HEAP_SIZE_1GiB
+
+/* 0xBA_0000_0000 - 0xBA_3FFF_FFFF **/
+	/* HWBRN65273 workaround requires USC memory to use a unique single 1GiB PCE entry. */
+	#define RGX_USCCODE_BRN_65273_HEAP_BASE         IMG_UINT64_C(0xBA00000000)
+	#define RGX_USCCODE_BRN_65273_HEAP_SIZE         RGX_HEAP_SIZE_1GiB
+
+
+/* 0xC0_0000_0000 ************************************************************/
+
+/* 0xE4_0000_0000 - 0xE4_001F_FFFF **/
+	/* HWBRN65273 workaround requires USC memory to use a unique single 1GiB PCE entry. */
+	#define RGX_VISIBILITY_TEST_BRN_65273_HEAP_BASE IMG_UINT64_C(0xE400000000)
+	#define RGX_VISIBILITY_TEST_BRN_65273_HEAP_SIZE RGX_HEAP_SIZE_2MiB
+
+/* 0xF8_0000_0000 - 0xF9_FFFF_FFFF **/
+	/* HWBRN65273 workaround requires two Region Header buffers 4GiB apart. */
+	#define RGX_MMU_INIA_BRN_65273_HEAP_BASE        IMG_UINT64_C(0xF800000000)
+	#define RGX_MMU_INIA_BRN_65273_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+	#define RGX_MMU_INIB_BRN_65273_HEAP_BASE        IMG_UINT64_C(0xF900000000)
+	#define RGX_MMU_INIB_BRN_65273_HEAP_SIZE        RGX_HEAP_SIZE_1GiB
+
+
+/* 0xFF_FFFF_FFFF ************************************************************/
+
+/*	End of RGX Device Virtual Address Space definitions */
+
+#endif /* RGXHEAPCONFIG_65273_H */
+
+/******************************************************************************
+ End of file (rgxheapconfig_65273.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxhwperf.c b/drivers/gpu/drm/img-rogue/rgxhwperf.c
new file mode 100644
index 000000000000..e43fc499d9db
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxhwperf.c
@@ -0,0 +1,1030 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+
+#include "info_page_defs.h"
+
+/* This is defined by default to enable producer callbacks.
+ * Clients of the TL interface can disable the use of the callback
+ * with PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK. */
+#define SUPPORT_TL_PRODUCER_CALLBACK 1
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+/* Defines size of buffers returned from acquire/release calls */
+#define FW_STREAM_BUFFER_SIZE (0x80000)
+#define HOST_STREAM_BUFFER_SIZE (0x20000)
+
+/* Must be at least as large as two tl packets of maximum size */
+static_assert(HOST_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "HOST_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+static_assert(FW_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "FW_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+
+/******************************************************************************
+ * RGX HW Performance Profiling Server API(s)
+ *****************************************************************************/
+
+static IMG_BOOL RGXServerFeatureFlagsToHWPerfFlagsAddBlock(
+	RGX_HWPERF_BVNC_BLOCK	* const psBlocks,
+	IMG_UINT16				* const pui16Count,
+	const IMG_UINT16		ui16BlockID, /* see RGX_HWPERF_CNTBLK_ID */
+	const IMG_UINT16		ui16NumCounters,
+	const IMG_UINT16		ui16NumBlocks)
+{
+	const IMG_UINT16 ui16Count = *pui16Count;
+
+	if (ui16Count < RGX_HWPERF_MAX_BVNC_BLOCK_LEN)
+	{
+		RGX_HWPERF_BVNC_BLOCK * const psBlock = &psBlocks[ui16Count];
+
+		/* If the GROUP is non-zero, convert from e.g. RGX_CNTBLK_ID_USC0 to RGX_CNTBLK_ID_USC_ALL. The table stores the former (plus the
+		number of blocks and counters) but PVRScopeServices expects the latter (plus the number of blocks and counters). The conversion
+		could always be moved to PVRScopeServices, but it's less code this way. */
+		psBlock->ui16BlockID		= (ui16BlockID & RGX_CNTBLK_ID_GROUP_MASK) ? (ui16BlockID | RGX_CNTBLK_ID_UNIT_ALL_MASK) : ui16BlockID;
+		if ((ui16BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+		{
+			psBlock->ui16NumCounters	= RGX_CNTBLK_COUNTERS_MAX;
+		}
+		else
+		{
+			psBlock->ui16NumCounters	= ui16NumCounters;
+		}
+		psBlock->ui16NumBlocks		= ui16NumBlocks;
+
+		*pui16Count = ui16Count + 1;
+		return IMG_TRUE;
+	}
+	return IMG_FALSE;
+}
+
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_HWPERF_BVNC *psBVNC)
+{
+	IMG_PCHAR pszBVNC;
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDevInfo), "psDevInfo invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((pszBVNC = RGXDevBVNCString(psDevInfo)))
+	{
+		size_t uiStringLength = OSStringNLength(pszBVNC, RGX_HWPERF_MAX_BVNC_LEN - 1);
+		OSStringSafeCopy(psBVNC->aszBvncString, pszBVNC, uiStringLength + 1);
+		memset(&psBVNC->aszBvncString[uiStringLength], 0, RGX_HWPERF_MAX_BVNC_LEN - uiStringLength);
+	}
+	else
+	{
+		*psBVNC->aszBvncString = 0;
+	}
+
+	psBVNC->ui32BvncKmFeatureFlags = 0x0;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERFBUS))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERFBUS_FLAG;
+	}
+#if defined(RGX_FEATURE_XT_TOP_INFRASTRUCTURE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XT_TOP_INFRASTRUCTURE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_XT_TOP_INFRASTRUCTURE_FLAG;
+	}
+#endif
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PERF_COUNTER_BATCH))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PERF_COUNTER_BATCH_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, ROGUEXE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_ROGUEXE_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_PBE2_IN_XE_FLAG;
+	}
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_MULTICORE_FLAG;
+	}
+
+#ifdef SUPPORT_WORKLOAD_ESTIMATION
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		/* Not a part of BVNC feature line and so doesn't need the feature supported check */
+		psBVNC->ui32BvncKmFeatureFlags |= RGX_HWPERF_FEATURE_WORKLOAD_ESTIMATION;
+	}
+#endif
+
+	/* Define the HW counter block counts. */
+	{
+		RGX_HWPERF_BVNC_BLOCK					* const psBlocks	= psBVNC->aBvncBlocks;
+		IMG_UINT16								* const pui16Count	= &psBVNC->ui16BvncBlocks;
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	*asCntBlkTypeModel;
+		const IMG_UINT32						ui32CntBlkModelLen	= RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+		IMG_UINT32								ui32BlkCfgIdx;
+		size_t									uiCount;
+		IMG_BOOL								bOk					= IMG_TRUE;
+
+		// Initialise to zero blocks
+		*pui16Count = 0;
+
+		// Add all the blocks
+		for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+		{
+			const RGXFW_HWPERF_CNTBLK_TYPE_MODEL	* const psCntBlkInfo = &asCntBlkTypeModel[ui32BlkCfgIdx];
+			RGX_HWPERF_CNTBLK_RT_INFO				sCntBlkRtInfo;
+			/* psCntBlkInfo->ui8NumUnits gives compile-time info. For BVNC agnosticism, we use this: */
+			if (psCntBlkInfo->pfnIsBlkPresent(psCntBlkInfo, psDevInfo, &sCntBlkRtInfo))
+			{
+				bOk &= RGXServerFeatureFlagsToHWPerfFlagsAddBlock(psBlocks, pui16Count, psCntBlkInfo->ui32CntBlkIdBase, psCntBlkInfo->ui8NumCounters, sCntBlkRtInfo.ui32NumUnits);
+			}
+		}
+
+		/* If this fails, consider why the static_assert didn't fail, and consider increasing RGX_HWPERF_MAX_BVNC_BLOCK_LEN */
+		PVR_ASSERT(bOk);
+
+		// Zero the remaining entries
+		uiCount = *pui16Count;
+		OSDeviceMemSet(&psBlocks[uiCount], 0, (RGX_HWPERF_MAX_BVNC_BLOCK_LEN - uiCount) * sizeof(*psBlocks));
+	}
+
+	/* The GPU core count is overwritten by the FW */
+	psBVNC->ui16BvncGPUCores = 0;
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVRGXConfigMuxHWPerfCountersKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigMuxHWPerfCountersKM(
+		CONNECTION_DATA               *psConnection,
+		PVRSRV_DEVICE_NODE            *psDeviceNode,
+		IMG_UINT32                     ui32ArrayLen,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK  *psBlockConfigs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	DEVMEM_MEMDESC*		psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_MUX_CNTBLK* psFwArray;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed
+	 */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_ENABLE_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgEnableBlks.sBlockConfigs,
+	                      psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM parameters set, calling FW"));*/
+
+	/* Ask the FW to carry out the HWPerf configuration command
+	 */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM command scheduled for FW"));*/
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration
+	 */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	/*PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXConfigMuxHWPerfCountersKM firmware completed"));*/
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED", ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+/*
+	PVRSRVRGXConfigCustomCountersReadingHWPerfKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigCustomCountersKM(
+		CONNECTION_DATA             * psConnection,
+		PVRSRV_DEVICE_NODE          * psDeviceNode,
+		IMG_UINT16                    ui16CustomBlockID,
+		IMG_UINT16                    ui16NumCustomCounters,
+		IMG_UINT32                  * pui32CustomCounterIDs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	DEVMEM_MEMDESC*		psFwSelectCntrsMemDesc = NULL;
+	IMG_UINT32*			psFwArray;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVRGXSelectCustomCountersKM: configure block %u to read %u counters", ui16CustomBlockID, ui16NumCustomCounters));
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_SELECT_CUSTOM_CNTRS;
+	sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.ui16NumCounters = ui16NumCustomCounters;
+	sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.ui16CustomBlock = ui16CustomBlockID;
+
+	if (ui16NumCustomCounters > 0)
+	{
+		PVR_ASSERT(pui32CustomCounterIDs);
+
+		/* used for passing counters config to the Firmware, write-only for the CPU */
+		eError = DevmemFwAllocate(psDevice,
+		                          sizeof(IMG_UINT32) * ui16NumCustomCounters,
+		                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+		                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+		                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+		                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+		                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+		                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+		                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+		                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+		                          "FwHWPerfConfigCustomCounters",
+		                          &psFwSelectCntrsMemDesc);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+		eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfSelectCstmCntrs.sCustomCounterIDs,
+		                      psFwSelectCntrsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+		eError = DevmemAcquireCpuVirtAddr(psFwSelectCntrsMemDesc, (void **)&psFwArray);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+		OSCachedMemCopyWMB(psFwArray, pui32CustomCounterIDs, sizeof(IMG_UINT32) * ui16NumCustomCounters);
+		DevmemPDumpLoadMem(psFwSelectCntrsMemDesc,
+		                   0,
+		                   sizeof(IMG_UINT32) * ui16NumCustomCounters,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	/* Push in the KCCB the command to configure the custom counters block */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail3);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXSelectCustomCountersKM: Command scheduled"));
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "PVRSRVRGXSelectCustomCountersKM: FW operation completed"));
+
+	if (ui16NumCustomCounters > 0)
+	{
+		/* Release temporary memory used for block configuration */
+		RGXUnsetFirmwareAddress(psFwSelectCntrsMemDesc);
+		DevmemReleaseCpuVirtAddr(psFwSelectCntrsMemDesc);
+		DevmemFwUnmapAndFree(psDevice, psFwSelectCntrsMemDesc);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf custom counters %u reading will be sent with the next HW events", ui16NumCustomCounters));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	if (psFwSelectCntrsMemDesc)
+	{
+		DevmemReleaseCpuVirtAddr(psFwSelectCntrsMemDesc);
+	}
+fail2:
+	if (psFwSelectCntrsMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psFwSelectCntrsMemDesc);
+	}
+fail1:
+	if (psFwSelectCntrsMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevice, psFwSelectCntrsMemDesc);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*
+	PVRSRVRGXConfigureHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+		CONNECTION_DATA          * psConnection,
+		PVRSRV_DEVICE_NODE       * psDeviceNode,
+		IMG_UINT32                 ui32CtrlWord,
+		IMG_UINT32                 ui32ArrayLen,
+		RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs)
+{
+	PVRSRV_ERROR             eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD         sKccbCmd;
+	DEVMEM_MEMDESC           *psFwBlkConfigsMemDesc;
+	RGX_HWPERF_CONFIG_CNTBLK *psFwArray;
+	IMG_UINT32               ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO       *psDevice;
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevice = psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(ui32CtrlWord);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_LOG_RETURN_IF_FALSE(ui32ArrayLen > 0, "ui32ArrayLen is 0",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psBlockConfigs != NULL, "psBlockConfigs is NULL",
+	                        PVRSRV_ERROR_INVALID_PARAMS);
+
+	PVR_DPF_ENTERED;
+
+	/* Fill in the command structure with the parameters needed */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CONFIG_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCfgDABlks.ui32NumBlocks = ui32ArrayLen;
+
+	/* used for passing counters config to the Firmware, write-only for the CPU */
+	eError = DevmemFwAllocate(psDevice,
+	                          sizeof(RGX_HWPERF_CONFIG_CNTBLK) * ui32ArrayLen,
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwHWPerfCountersDAConfigBlock",
+	                          &psFwBlkConfigsMemDesc);
+	PVR_LOG_RETURN_IF_ERROR(eError, "DevmemFwAllocate");
+
+	eError = RGXSetFirmwareAddress(&sKccbCmd.uCmdData.sHWPerfCfgDABlks.sBlockConfigs,
+	                               psFwBlkConfigsMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", fail1);
+
+	eError = DevmemAcquireCpuVirtAddr(psFwBlkConfigsMemDesc, (void **)&psFwArray);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail2);
+
+	OSCachedMemCopyWMB(psFwArray, psBlockConfigs, sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen);
+	DevmemPDumpLoadMem(psFwBlkConfigsMemDesc,
+	                   0,
+	                   sizeof(RGX_HWPERF_CONFIG_CNTBLK)*ui32ArrayLen,
+	                   PDUMP_FLAGS_CONTINUOUS);
+
+	/* Ask the FW to carry out the HWPerf configuration command. */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot", fail2);
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", fail3);
+
+	/* Release temporary memory used for block configuration. */
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks configured and ENABLED",
+	         ui32ArrayLen));
+
+	PVR_DPF_RETURN_OK;
+
+fail3:
+	DevmemReleaseCpuVirtAddr(psFwBlkConfigsMemDesc);
+fail2:
+	RGXUnsetFirmwareAddress(psFwBlkConfigsMemDesc);
+fail1:
+	DevmemFwUnmapAndFree(psDevice, psFwBlkConfigsMemDesc);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfConfigMuxCounters(
+		RGX_HWPERF_CONNECTION           *psHWPerfConnection,
+		IMG_UINT32					     ui32NumBlocks,
+		RGX_HWPERF_CONFIG_MUX_CNTBLK	*asBlockConfigs)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !asBlockConfigs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXConfigMuxHWPerfCountersKM(NULL,
+		                                            psDevData->psRgxDevNode,
+		                                            ui32NumBlocks,
+		                                            asBlockConfigs);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXConfigMuxHWPerfCountersKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfConfigureAndEnableCustomCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT16              ui16CustomBlockID,
+		IMG_UINT16          ui16NumCustomCounters,
+		IMG_UINT32         *pui32CustomCounterIDs)
+{
+	PVRSRV_ERROR            eError;
+	RGX_HWPERF_DEVICE       *psHWPerfDev;
+
+	/* Validate input arguments supplied by the caller */
+	PVR_LOG_RETURN_IF_FALSE((NULL != psHWPerfConnection), "psHWPerfConnection invalid",
+	                   PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((0 != ui16NumCustomCounters), "uiNumBlocks invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((NULL != pui32CustomCounterIDs),"asBlockConfigs invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Check # of blocks */
+	PVR_LOG_RETURN_IF_FALSE((!(ui16CustomBlockID > RGX_HWPERF_MAX_CUSTOM_BLKS)),"ui16CustomBlockID invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	/* Check # of counters */
+	PVR_LOG_RETURN_IF_FALSE((!(ui16NumCustomCounters > RGX_HWPERF_MAX_CUSTOM_CNTRS)),"ui16NumCustomCounters invalid",
+			           PVRSRV_ERROR_INVALID_PARAMS);
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		RGX_KM_HWPERF_DEVDATA *psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		eError = PVRSRVRGXConfigCustomCountersKM(NULL,
+				                                 psDevData->psRgxDevNode,
+												 ui16CustomBlockID, ui16NumCustomCounters, pui32CustomCounterIDs);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXCtrlCustHWPerfKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       GetHWPerfBlockTypeByID
+@Description    Lookup function to obtain a block type descriptor for a given
+                counter block identifier.
+@Input          psDevInfo                       A pointer to current device info.
+@Input          ui32BlockID                     The block ID for which a type
+                                                descriptor should be retrieved.
+@Return         RGXFW_HWPERF_CNTBLK_TYPE_MODEL  Block type descriptor.
+*/ /**************************************************************************/
+static const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *
+GetHWPerfBlockTypeByID(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32BlockID)
+{
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	IMG_UINT32 ui32TableIdx = 0xFFFF;
+	RGX_HWPERF_CNTBLK_RT_INFO sRtInfo; /* Only used to satisfy pfnIsBlkPresent requirements. */
+
+#if defined(HWPERF_UNIFIED)
+	IMG_UINT32 uiBlockID = (IMG_UINT32)(ui32BlockID & ~(RGX_CNTBLK_ID_UNIT_ALL_MASK|RGX_CNTBLK_ID_DA_MASK));
+#else
+	IMG_UINT32 uiBlockID = (IMG_UINT32)(ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK);
+#endif
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	/* Is it a direct block? */
+	if (uiBlockID < RGX_CNTBLK_ID_DIRECT_LAST)
+	{
+		ui32TableIdx = uiBlockID;
+	}
+	/* Is it an indirect block */
+	else if ((uiBlockID > RGX_CNTBLK_ID_DIRECT_LAST) && (uiBlockID < RGX_CNTBLK_ID_LAST))
+	{
+		ui32TableIdx = RGX_CNTBLK_ID_DIRECT_LAST + (((uiBlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) >> RGX_CNTBLK_ID_GROUP_SHIFT) - 1U);
+	}
+	/* Unknown mapping from CNTBLK_ID to Table index */
+	else
+	{
+		return NULL;
+	}
+
+	PVR_ASSERT(ui32TableIdx < ui32CntBlkModelLen);
+
+	if (psDevInfo == NULL)
+	{
+		PVR_LOG(("psDevInfo invalid"));
+		return NULL;
+	}
+
+	if ((ui32TableIdx < ui32CntBlkModelLen) &&
+		(asCntBlkTypeModel[ui32TableIdx].pfnIsBlkPresent(&asCntBlkTypeModel[ui32TableIdx], psDevInfo, &sRtInfo) != IMG_FALSE))
+	{
+		return &asCntBlkTypeModel[ui32TableIdx];
+	}
+
+	/* Fall through, block not valid from run-time validation */
+	return NULL;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(CONNECTION_DATA *psConnection,
+                                                       PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                       const IMG_UINT32 ui32BlockID,
+                                                       RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXAcquireHWPerfCtlCPUAddr");
+
+	eError = PVRSRVRGXGetConfiguredHWPerfMuxCounters(psDeviceNode,
+	                                                 psHWPerfCtl,
+	                                                 ui32BlockID,
+	                                                 psConfiguredMuxCounters);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRGXGetConfiguredHWPerfMuxCounters");
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                     RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                     IMG_UINT32 ui32BlockID,
+                                                     RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredMuxCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & ~RGX_CNTBLK_ID_UNIT_ALL_MASK) < RGX_CNTBLK_ID_LAST)
+	{
+		RGXFWIF_HWPERF_CTL_BLK *psBlock = rgxfw_hwperf_get_block_ctl(ui32BlockID, psHWPerfCtl);
+		const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *psBlkTypeDesc;
+		IMG_UINT32 i, ui32LastCountIdx = 0, ui8CurCountIdx = 0;
+		RGX_HWPERF_CONFIG_MUX_CNTBLK sBlockConfig;
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		if (!psBlock->ui32Enabled || !psBlock->ui32Valid)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not %s",
+			         ui32BlockID,
+			         !psBlock->ui32Enabled ? "enabled." : "configured."));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		psBlkTypeDesc = GetHWPerfBlockTypeByID(psDevInfo, psBlock->eBlockID);
+		if (psBlkTypeDesc == NULL)
+		{
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->eBlockID;
+		sBlockConfig.ui8Mode = 0;
+
+		for (i = 0; ((psBlock->uiCounterMask >> i) != 0) &&
+		     (ui8CurCountIdx < psBlkTypeDesc->ui8NumCounters); i++)
+		{
+			if (psBlock->uiCounterMask & (1 << i))
+			{
+				IMG_UINT8 ui8Mode = 0;
+
+				ui8Mode = (psBlock->aui64CounterCfg[i] >> psBlkTypeDesc->ui8SelectRegModeShift) & 1U;
+				sBlockConfig.ui8Mode |= ui8Mode << ui32LastCountIdx;
+
+				sBlockConfig.aui8GroupSelect[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_GROUP_SELECT_SHIFT) & 0x1F;
+
+				sBlockConfig.aui16BitSelect[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BIT_SELECT_SHIFT) & 0x7FFF;
+
+#if defined(RGX_FEATURE_PERF_COUNTER_BATCH)
+				sBlockConfig.aui32BatchMax[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BATCH_MAX_SHIFT) & 0x1FFF;
+
+				sBlockConfig.aui32BatchMin[ui32LastCountIdx] =
+					(psBlock->aui64CounterCfg[i] >> RGX_CR_TA_PERF_SELECT0_BATCH_MIN_SHIFT) & 0x1FFF;
+#endif
+				ui32LastCountIdx++;
+				ui8CurCountIdx++;
+			}
+		}
+
+		sBlockConfig.ui8CounterSelect = (1 << ui32LastCountIdx) - 1;
+		*psConfiguredMuxCounters = sBlockConfig;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	RGX_HWPERF_CONFIG_CNTBLK sBlockConfig;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psConfiguredCounters != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	if ((ui32BlockID & RGX_CNTBLK_ID_CUSTOM_MASK) >= RGX_CNTBLK_ID_LAST)
+	{
+		/* Validate block ID */
+		switch (ui32BlockID)
+		{
+			case RGX_CNTBLK_ID_CUSTOM0:
+			case RGX_CNTBLK_ID_CUSTOM1:
+			case RGX_CNTBLK_ID_CUSTOM2:
+			case RGX_CNTBLK_ID_CUSTOM3:
+			case RGX_CNTBLK_ID_CUSTOM4_FW:
+			{
+				PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+				/* Check to see if this block is enabled */
+				if (psHWPerfCtl->ui32SelectedCountersBlockMask & (1 << (ui32BlockID & 0x0F)))
+				{
+					RGXFW_HWPERF_SELECT *psBlock = &psHWPerfCtl->SelCntr[ui32BlockID & 0x0F];
+
+					sBlockConfig.ui16BlockID = ui32BlockID;
+					sBlockConfig.ui16NumCounters = psBlock->ui32NumSelectedCounters;
+
+					for (i = 0; i < psBlock->ui32NumSelectedCounters; i++)
+					{
+						sBlockConfig.ui16Counters[i] = psBlock->aui32SelectedCountersIDs[i];
+					}
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+					PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+				}
+				break;
+			}
+			default:
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+			}
+		}
+	}
+#if defined(HWPERF_UNIFIED)
+	else if ((ui32BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+	{
+		RGXFWIF_HWPERF_DA_BLK *psBlock = rgxfw_hwperf_get_da_block_ctl(ui32BlockID, psHWPerfCtl);
+
+		PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+		if (psBlock == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		if (!psBlock->uiEnabled)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Block (0x%04x) is not enabled.", ui32BlockID));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, Error);
+		}
+
+		sBlockConfig.ui16BlockID = psBlock->eBlockID;
+		sBlockConfig.ui16NumCounters = psBlock->uiNumCounters;
+
+		for (i = 0; i < psBlock->uiNumCounters; i++)
+		{
+			sBlockConfig.ui16Counters[i] = psBlock->aui32Counters[i];
+		}
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Block ID (0x%04x) was invalid.", ui32BlockID));
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, InvalidIDError);
+	}
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+
+InvalidIDError:
+	if (eError == PVRSRV_OK)
+	{
+		*psConfiguredCounters = sBlockConfig;
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLen,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	IMG_UINT32 ui32LastIdx = 0;
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(psHWPerfCtl != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_RETURN_IF_FALSE(pui32BlockCount != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	*pui32BlockCount = 0;
+
+	if (ui32ArrayLen > 0 && pui32EnabledBlockIDs == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen is greater than 0 but pui32EnabledBlockIDs is NULL"));
+	}
+
+	PVR_RETURN_IF_ERROR(PVRSRVPowerLock(psDevNode));
+
+	for (i = 0; i < RGX_HWPERF_MAX_MUX_BLKS; i++)
+	{
+		if (psHWPerfCtl->sBlkCfg[i].ui32Enabled && psHWPerfCtl->sBlkCfg[i].ui32Valid)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx + 1 > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sBlkCfg[i].eBlockID;
+			ui32LastIdx++;
+		}
+	}
+
+	for (i = 0; i < RGX_HWPERF_MAX_CUSTOM_BLKS; i++)
+	{
+		if (psHWPerfCtl->ui32SelectedCountersBlockMask == 0)
+		{
+			break;
+		}
+
+		if (psHWPerfCtl->ui32SelectedCountersBlockMask & (1 << i))
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx + 1 > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = RGX_CNTBLK_ID_CUSTOM0 + i;
+			ui32LastIdx++;
+		}
+	}
+
+#if defined(HWPERF_UNIFIED)
+	for (i = 0; i < RGX_HWPERF_MAX_DA_BLKS; i++)
+	{
+		if (psHWPerfCtl->sDABlkCfg[i].uiEnabled)
+		{
+			*pui32BlockCount += 1;
+
+			if (pui32EnabledBlockIDs == NULL)
+			{
+				continue;
+			}
+
+			if (ui32LastIdx > ui32ArrayLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+				PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+			}
+
+			pui32EnabledBlockIDs[ui32LastIdx] = psHWPerfCtl->sDABlkCfg[i].eBlockID;
+			ui32LastIdx++;
+		}
+	}
+#endif
+
+Error:
+	PVRSRVPowerUnlock(psDevNode);
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxhwperf.h b/drivers/gpu/drm/img-rogue/rgxhwperf.h
new file mode 100644
index 000000000000..5191f18f1307
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxhwperf.h
@@ -0,0 +1,99 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX HWPerf functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHWPERF_H_
+#define RGXHWPERF_H_
+
+#include "rgxhwperf_common.h"
+
+/******************************************************************************
+ * RGX HW Performance Profiling API(s)
+ *****************************************************************************/
+
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+PVRSRV_ERROR PVRSRVRGXConfigMuxHWPerfCountersKM(
+	CONNECTION_DATA               *psConnection,
+	PVRSRV_DEVICE_NODE            *psDeviceNode,
+	IMG_UINT32                     ui32ArrayLen,
+	RGX_HWPERF_CONFIG_MUX_CNTBLK  *psBlockConfigs);
+
+
+PVRSRV_ERROR PVRSRVRGXConfigCustomCountersKM(
+	CONNECTION_DATA    * psConnection,
+	PVRSRV_DEVICE_NODE * psDeviceNode,
+	IMG_UINT16           ui16CustomBlockID,
+	IMG_UINT16           ui16NumCustomCounters,
+	IMG_UINT32         * pui32CustomCounterIDs);
+#endif
+
+PVRSRV_ERROR PVRSRVRGXConfigureHWPerfBlocksKM(
+	CONNECTION_DATA       * psConnection,
+	PVRSRV_DEVICE_NODE    * psDeviceNode,
+	IMG_UINT32            ui32CtrlWord,
+	IMG_UINT32            ui32ArrayLen,
+	RGX_HWPERF_CONFIG_CNTBLK * psBlockConfigs);
+
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(CONNECTION_DATA *psConnection,
+                                                       PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                       const IMG_UINT32 ui32BlockID,
+                                                       RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfMuxCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                     RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                     IMG_UINT32 ui32BlockID,
+                                                     RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCounters);
+#endif
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCounters(PVRSRV_DEVICE_NODE *psDevNode,
+                                                  RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                                  IMG_UINT32 ui32BlockID,
+                                                  RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocks(PVRSRV_DEVICE_NODE *psDevNode,
+                                             RGXFWIF_HWPERF_CTL *psHWPerfCtl,
+                                             IMG_UINT32 ui32ArrayLength,
+                                             IMG_UINT32 *pui32BlockCount,
+                                             IMG_UINT32 *pui32EnabledBlockIDs);
+
+#endif /* RGXHWPERF_H_ */
diff --git a/drivers/gpu/drm/img-rogue/rgxhwperf_common.c b/drivers/gpu/drm/img-rogue/rgxhwperf_common.c
new file mode 100644
index 000000000000..4a2452651077
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxhwperf_common.c
@@ -0,0 +1,4142 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX HW Performance implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "rgxdevice.h"
+#include "pvrsrv_error.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "allocmem.h"
+
+#include "pvrsrv.h"
+#include "pvrsrv_tlstreams.h"
+#include "pvrsrv_tlcommon.h"
+#include "tlclient.h"
+#include "tlstream.h"
+
+#include "rgxhwperf.h"
+#include "rgxapi_km.h"
+#include "rgxfwutils.h"
+#include "rgxtimecorr.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pdump_km.h"
+#include "pvrsrv_apphint.h"
+#include "process_stats.h"
+#include "rgx_hwperf_table.h"
+#include "rgxinit.h"
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+#include "ospvr_gputrace.h"
+#endif
+
+#include "info_page_defs.h"
+
+/* This is defined by default to enable producer callbacks.
+ * Clients of the TL interface can disable the use of the callback
+ * with PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK. */
+#define SUPPORT_TL_PRODUCER_CALLBACK 1
+
+/* Maximum enum value to prevent access to RGX_HWPERF_STREAM_ID2_CLIENT stream */
+#define RGX_HWPERF_MAX_STREAM_ID (RGX_HWPERF_STREAM_ID2_CLIENT)
+
+/* Defines size of buffers returned from acquire/release calls */
+#define FW_STREAM_BUFFER_SIZE (0x80000)
+#define HOST_STREAM_BUFFER_SIZE (0x20000)
+
+/* Must be at least as large as two tl packets of maximum size */
+static_assert(HOST_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "HOST_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+static_assert(FW_STREAM_BUFFER_SIZE >= (PVRSRVTL_MAX_PACKET_SIZE<<1),
+              "FW_STREAM_BUFFER_SIZE is less than (PVRSRVTL_MAX_PACKET_SIZE<<1)");
+
+IMG_INTERNAL /*static inline*/ IMG_UINT32 RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **);
+static IMG_UINT64 RGXHWPerfFwSetEventFilterNoLock(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                                  IMG_UINT64 uiFilter);
+
+static inline IMG_UINT32
+RGXHWPerfGetPackets(IMG_UINT32  ui32BytesExp,
+                    IMG_UINT32  ui32AllowedSize,
+                    RGX_PHWPERF_V2_PACKET_HDR psCurPkt )
+{
+	IMG_UINT32 sizeSum = 0;
+	RGXFwSharedMemCacheOpValue(psCurPkt->ui32Size, INVALIDATE);
+
+	/* Traverse the array to find how many packets will fit in the available space. */
+	while ( sizeSum < ui32BytesExp  &&
+			sizeSum + RGX_HWPERF_GET_SIZE(psCurPkt) < ui32AllowedSize )
+	{
+		sizeSum += RGX_HWPERF_GET_SIZE(psCurPkt);
+		psCurPkt = RGX_HWPERF_GET_NEXT_PACKET(psCurPkt);
+		RGXFwSharedMemCacheOpValue(psCurPkt->ui32Size, INVALIDATE);
+	}
+
+	return sizeSum;
+}
+
+static inline void
+RGXSuspendHWPerfL2DataCopy(PVRSRV_RGXDEV_INFO* psDeviceInfo,
+                           RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                           IMG_BOOL bIsReaderConnected)
+{
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	if (!bIsReaderConnected)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s : HWPerf FW events enabled but L2 host buffer "
+		        "for stream %u is full and no reader is currently connected, suspending "
+		        "event collection. Connect a reader or restart driver to avoid event loss.",
+		        __func__, eL2StreamId));
+		psDeviceInfo->bSuspendHWPerfL2DataCopy[eL2StreamId] = IMG_TRUE;
+	}
+}
+
+static IMG_UINT32 RGXHWPerfCopyData(PVRSRV_RGXDEV_INFO *psDeviceInfo,
+                                    IMG_BYTE *pbSrcBuffer,
+                                    RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                    IMG_UINT32 uiBytesToCopy)
+{
+	IMG_BYTE *pbDestBuffer;
+	IMG_UINT32 uiBytesCopied = 0;
+	IMG_UINT32 uiFreeSpace;
+	IMG_UINT32 uiBytesToCopyMin = RGX_HWPERF_GET_SIZE(RGX_HWPERF_GET_PACKET(pbSrcBuffer));
+	IMG_BOOL bIsReaderConnected;
+	PVRSRV_ERROR eError;
+	IMG_HANDLE hHWPerfDestStream;
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	hHWPerfDestStream = psDeviceInfo->hHWPerfStream[eL2StreamId];
+
+	PVR_DPF_ENTERED;
+
+	/* Try submitting all data in one TL packet. */
+	eError = TLStreamReserve2(hHWPerfDestStream, &pbDestBuffer, uiBytesToCopy, uiBytesToCopyMin,
+	                          &uiFreeSpace, &bIsReaderConnected);
+	if (eError == PVRSRV_OK)
+	{
+		RGXFwSharedMemCacheOpExec(pbSrcBuffer, uiBytesToCopy, PVRSRV_CACHE_OP_INVALIDATE);
+		OSDeviceMemCopy(pbDestBuffer, pbSrcBuffer, (size_t) uiBytesToCopy);
+
+		eError = TLStreamCommit(hHWPerfDestStream, uiBytesToCopy);
+		PVR_LOG_GOTO_IF_ERROR_VA(eError, ErrReturn, "TLStreamCommit() failed with error %d, "
+		                         "unable to copy packet from L1 to L2 buffer", eError);
+
+		/* Data were successfully written */
+		uiBytesCopied = (size_t) uiBytesToCopy;
+	}
+	else if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		/* There was not enough space for all data, copy as much as possible */
+		IMG_UINT32 uiSizeSum = RGXHWPerfGetPackets(uiBytesToCopy, uiFreeSpace,
+		                                           RGX_HWPERF_GET_PACKET(pbSrcBuffer));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Unable to reserve space (%d) in host buffer on first attempt, "
+		         "remaining free space: %d", uiBytesToCopy, uiFreeSpace));
+
+		if (uiSizeSum != 0)
+		{
+			eError = TLStreamReserve(hHWPerfDestStream, &pbDestBuffer, uiSizeSum);
+
+			if (eError == PVRSRV_OK)
+			{
+				RGXFwSharedMemCacheOpExec(pbSrcBuffer, uiSizeSum, PVRSRV_CACHE_OP_INVALIDATE);
+				OSDeviceMemCopy(pbDestBuffer, pbSrcBuffer, (size_t) uiSizeSum);
+
+				eError = TLStreamCommit(hHWPerfDestStream, uiSizeSum);
+				PVR_LOG_GOTO_IF_ERROR_VA(eError, ErrReturn, "TLStreamCommit() failed with error "
+				                         "%d, unable to copy packet from L1 to L2 buffer", eError);
+
+				/* uiSizeSum bytes of hwperf packets have been successfully written */
+				uiBytesCopied = uiSizeSum;
+			}
+			else if (eError == PVRSRV_ERROR_STREAM_FULL)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "Cannot write HWPerf packet into host buffer, check data "
+				         "in case of packet loss, remaining free space: %d", uiFreeSpace));
+				RGXSuspendHWPerfL2DataCopy(psDeviceInfo, eL2StreamId, bIsReaderConnected);
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "Cannot find space in host buffer, check data in case of "
+			         "packet loss, remaining free space: %d", uiFreeSpace));
+			RGXSuspendHWPerfL2DataCopy(psDeviceInfo, eL2StreamId, bIsReaderConnected);
+		}
+	}
+
+	/* Some other error occurred. Full error handled by caller, we returning the copied bytes count
+	 * to caller */
+	if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_STREAM_FULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "HWPerf enabled: Unexpected Error (%d) while copying FW buffer "
+		         "to destination buffer.", eError));
+	}
+
+ErrReturn:
+	/* Return the remaining packets left to be transported. */
+	PVR_DPF_RETURN_VAL(uiBytesCopied);
+}
+
+/*
+  RGXHWPerfGetMaxTransfer
+ */
+static IMG_UINT32 RGXHWPerfGetMaxTransfer(PVRSRV_RGXDEV_INFO *psDeviceInfo,
+                                          IMG_UINT32 ui32BytesExp,
+                                          IMG_UINT32 uiL2StreamCopyMask)
+{
+	IMG_UINT32 uiMaxXfer = ui32BytesExp;
+	IMG_UINT32 eL2StreamId;
+
+	for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+	{
+		if (BIT_ISSET(uiL2StreamCopyMask, eL2StreamId))
+		{
+			IMG_UINT32 uiMaxXferSize = TLStreamGetMaxTransfer(
+			                             ui32BytesExp,
+			                             psDeviceInfo->hHWPerfStream[eL2StreamId]);
+
+			if (uiMaxXferSize < uiMaxXfer)
+			{
+				/* New minimum size found, save it for later */
+#if defined(DEBUG)
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s(dev %u, len %u, mask %u) New/Old = [0x%x/0x%x]",
+				         __func__, eL2StreamId, ui32BytesExp,
+				         uiL2StreamCopyMask, uiMaxXferSize, uiMaxXfer));
+#endif
+
+				uiMaxXfer = uiMaxXferSize;
+			}
+#if defined(DEBUG)
+			else
+			{
+				PVR_DPF((PVR_DBG_VERBOSE,
+				         "%s(dev %u, len %u, mask %u) New/Old = [0x%x/0x%x]",
+				         __func__, eL2StreamId, ui32BytesExp,
+				         uiL2StreamCopyMask, uiMaxXferSize, uiMaxXfer));
+			}
+#endif
+		}
+	}
+
+	return uiMaxXfer;
+}
+
+/*
+	RGXHWPerfCopyDataL1toL2
+ */
+static IMG_UINT32 RGXHWPerfCopyDataL1toL2(PVRSRV_RGXDEV_INFO* psDeviceInfo,
+                                          IMG_BYTE   *pbFwBuffer,
+                                          IMG_UINT32 ui32BytesExp,
+                                          IMG_UINT32 uiL2StreamCopyMask)
+{
+	IMG_UINT32 eL2StreamId, uiHWPerfBytesCopied = 0;
+
+	/* HWPERF_MISR_FUNC_DEBUG enables debug code for investigating HWPerf issues */
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	static IMG_UINT32 gui32Ordinal = IMG_UINT32_MAX;
+#endif
+
+	/* Invalidate initial packet header, type/size cast via RGX_HWPEF_GET_PACKET */
+	RGXFwSharedMemCacheOpPtr(RGX_HWPERF_GET_PACKET(pbFwBuffer), INVALIDATE);
+
+	PVR_DPF_ENTERED;
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	PVR_DPF((PVR_DBG_VERBOSE, "EVENTS to copy from 0x%p length:%05d mask:0x%2x",
+			pbFwBuffer, ui32BytesExp, uiL2StreamCopyMask));
+#endif
+
+	/* Determine the maximum space available in all consumer (L2) streams.
+	 * This limits the amount of data that will be copied if we have multiple
+	 * L2 consumers registered. In this case we only transfer the amount of
+	 * data that can fit into all available L2 consumer streams and so we will
+	 * exert back-pressure onto the L1 buffer whenever one of the L2 consumer
+	 * streams gets filled.
+	 * This effectively lock-steps the L2 consumer streams together.
+	 */
+	ui32BytesExp = RGXHWPerfGetMaxTransfer(psDeviceInfo, ui32BytesExp, uiL2StreamCopyMask);
+
+	for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+	{
+		if (BIT_ISSET(uiL2StreamCopyMask, eL2StreamId))
+		{
+			IMG_UINT32 uiBytesCopied, uiPacketDataSize = ui32BytesExp;
+			IMG_UINT32 uiMaxPacketSize = psDeviceInfo->ui32L2BufMaxPacketSize[eL2StreamId];
+
+			if (ui32BytesExp > uiMaxPacketSize)
+			{
+				uiPacketDataSize = RGXHWPerfGetPackets(ui32BytesExp, uiMaxPacketSize,
+				                                       RGX_HWPERF_GET_PACKET(pbFwBuffer));
+
+				if (uiPacketDataSize == 0)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "Failed to write data into host buffer "
+					        "(%u) as packet is too big and hence it breaches TL "
+					        "packet size limit (TLBufferSize / 2.5)", eL2StreamId));
+
+					continue;
+				}
+			}
+
+			uiBytesCopied = RGXHWPerfCopyData(psDeviceInfo, pbFwBuffer, eL2StreamId,
+			                                  uiPacketDataSize);
+
+			uiHWPerfBytesCopied = MAX(uiBytesCopied, uiHWPerfBytesCopied);
+		}
+	}
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	{
+		/* Check the incoming buffer of data has not lost any packets */
+		IMG_BYTE *pbFwBufferIter = pbFwBuffer;
+		IMG_BYTE *pbFwBufferEnd = pbFwBuffer+ui32BytesExp;
+		do
+		{
+			RGX_HWPERF_V2_PACKET_HDR *asCurPos = RGX_HWPERF_GET_PACKET(pbFwBufferIter);
+			IMG_UINT32 ui32CurOrdinal;
+			/* Invalidate HDR pointed to by asCurPos as we use both ordinal for detecting
+			 * lost packets and size for iteration.
+			 */
+			RGXFwSharedMemCacheOpPtr(asCurPos, INVALIDATE);
+			ui32CurOrdinal = asCurPos->ui32Ordinal;
+			if (gui32Ordinal != IMG_UINT32_MAX)
+			{
+				if ((gui32Ordinal+1) != ui32CurOrdinal)
+				{
+					if (gui32Ordinal < ui32CurOrdinal)
+					{
+						PVR_DPF((PVR_DBG_WARNING,
+								"HWPerf [%p] packets lost (%u packets) between ordinal %u...%u",
+								pbFwBufferIter,
+								ui32CurOrdinal - gui32Ordinal - 1,
+								gui32Ordinal,
+								ui32CurOrdinal));
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_WARNING,
+								"HWPerf [%p] packet ordinal out of sequence last: %u, current: %u",
+								pbFwBufferIter,
+								gui32Ordinal,
+								ui32CurOrdinal));
+					}
+				}
+			}
+			gui32Ordinal = asCurPos->ui32Ordinal;
+			pbFwBufferIter += RGX_HWPERF_GET_SIZE(asCurPos);
+		} while (pbFwBufferIter < pbFwBufferEnd && pbFwBufferIter < (pbFwBuffer + uiHWPerfBytesCopied));
+	}
+#endif
+
+	/* Return the remaining packets left to be transported. */
+	PVR_DPF_RETURN_VAL(uiHWPerfBytesCopied);
+}
+
+
+static INLINE IMG_UINT32 RGXHWPerfAdvanceRIdx(
+		const IMG_UINT32 ui32BufSize,
+		const IMG_UINT32 ui32Pos,
+		const IMG_UINT32 ui32Size)
+{
+	return ( ui32Pos + ui32Size < ui32BufSize ? ui32Pos + ui32Size : 0 );
+}
+
+
+/*
+	RGXHWPerfDataStore
+
+	This function copies HWPerf data from L1 buffer to all L2 streams.
+	The number of copied data is always the maximum read number of packets.
+	In case where one of the stream is not able to accept the same amount of
+	data as other streams it will suffer from gaps in the data.
+
+	To avoid these gaps, we will exert back-pressure on the L1 buffer by
+	pre-calculating the maximum amount of data that can be copied to all
+	L2 streams (currently a maximum of 2).
+ */
+static IMG_UINT32 RGXHWPerfDataStore(PVRSRV_RGXDEV_INFO	*psDevInfo)
+{
+	RGXFWIF_SYSDATA	*psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	IMG_BYTE*				psHwPerfInfo = psDevInfo->psRGXFWIfHWPerfBuf;
+	IMG_UINT32				ui32SrcRIdx, ui32SrcWIdx, ui32SrcWrapCount;
+	IMG_UINT32				ui32BytesExp = 0, ui32BytesCopied = 0, ui32BytesCopiedSum = 0;
+	IMG_UINT32				uiStreamCopyMask = 0;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+	IMG_UINT32				ui32BytesExpSum = 0;
+#endif
+
+	/* It's unlikely that we're ever going to have more than 32 consumers
+	 * for the Firmware L1 buffer but check just to be safe. */
+	static_assert(RGX_HWPERF_L2_STREAM_LAST <= sizeof(uiStreamCopyMask) * 8,
+	              "RGX_HWPERF_L2_STREAM_LAST cannot be greater than 32.")
+
+	PVR_DPF_ENTERED;
+
+	/* Caller should check this member is valid before calling */
+	{
+		RGX_HWPERF_L2_STREAM_ID eL2StreamId;
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+		IMG_UINT32 uiNotNullCount = 0;
+#endif
+		for (eL2StreamId = 0; eL2StreamId < RGX_HWPERF_L2_STREAM_LAST; eL2StreamId++)
+		{
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+			if (psDevInfo->hHWPerfStream[eL2StreamId] != NULL)
+			{
+				uiNotNullCount++;
+			}
+#endif
+			if (!psDevInfo->bSuspendHWPerfL2DataCopy[eL2StreamId])
+			{
+				BIT_SET(uiStreamCopyMask, eL2StreamId);
+			}
+		}
+
+#if defined(PVRSRV_NEED_PVR_ASSERT)
+		/* At least one stream must exist. */
+		PVR_ASSERT(uiNotNullCount > 0);
+#endif
+
+		/* Only proceed if any of the streams are not suspended.
+		 * Build bit field representing each L2 stream's suspend status. This
+		 * will be passed down the stack for copy function to determine if the
+		 * data should or should not be copied. */
+		if (uiStreamCopyMask == 0)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s : Copying data to all L2 host buffers for FW events is "
+			        "suspended. Start at least one of the HWPerf consumers or restart the driver "
+			        "if HWPerf FW events are needed", __func__));
+
+			PVR_DPF_RETURN_VAL(0);
+		}
+	}
+
+	/* Invalidate partial region of struct */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl,
+	                           INVALIDATE);
+
+	/* Get a copy of the current
+	 *   read (first packet to read)
+	 *   write (empty location for the next write to be inserted)
+	 *   WrapCount (size in bytes of the buffer at or past end)
+	 * indexes of the FW buffer */
+	ui32SrcRIdx = psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx;
+	ui32SrcWIdx = psFwSysData->sHWPerfCtrl.ui32HWPerfWIdx;
+	OSMemoryBarrier(NULL);
+	ui32SrcWrapCount = psFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount;
+
+#if defined(HWPERF_MISR_FUNC_DEBUG) || defined(EMULATOR)
+	{
+		IMG_UINT32  ui32SrcBufSize = psDevInfo->ui32RGXFWIfHWPerfBufSize;
+
+		if (ui32SrcRIdx >= ui32SrcBufSize || ui32SrcWIdx >= ui32SrcBufSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s : Invalid read/write offsets found! srcRIdx:%u srcWIdx:%u srcBufSize:%u",
+			        __func__, ui32SrcRIdx, ui32SrcWIdx, ui32SrcBufSize));
+
+			PVR_DPF_RETURN_VAL(ui32BytesCopiedSum);
+		}
+	}
+#endif
+
+	/* Is there any data in the buffer not yet retrieved? */
+	if ( ui32SrcRIdx != ui32SrcWIdx )
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfDataStore EVENTS found srcRIdx:%d srcWIdx: %d", ui32SrcRIdx, ui32SrcWIdx));
+
+		/* Is the write position higher than the read position? */
+		if ( ui32SrcWIdx > ui32SrcRIdx )
+		{
+			/* Yes, buffer has not wrapped */
+			ui32BytesExp = ui32SrcWIdx - ui32SrcRIdx;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+			ui32BytesExpSum += ui32BytesExp;
+#endif
+			ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+			                                          psHwPerfInfo + ui32SrcRIdx,
+			                                          ui32BytesExp,
+			                                          uiStreamCopyMask);
+
+			/* Advance the read index and the free bytes counter by the number
+			 * of bytes transported. Items will be left in buffer if not all data
+			 * could be transported. Exit to allow buffer to drain. */
+			OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                          RGXHWPerfAdvanceRIdx(psDevInfo->ui32RGXFWIfHWPerfBufSize,
+			                                               ui32SrcRIdx,
+			                                               ui32BytesCopied));
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                           FLUSH);
+
+			ui32BytesCopiedSum += ui32BytesCopied;
+		}
+		/* No, buffer has wrapped and write position is behind read position */
+		else
+		{
+			/* Byte count equal to
+			 *     number of bytes from read position to the end of the buffer,
+			 *   + data in the extra space in the end of the buffer. */
+			ui32BytesExp = ui32SrcWrapCount - ui32SrcRIdx;
+
+#ifdef HWPERF_MISR_FUNC_DEBUG
+			ui32BytesExpSum += ui32BytesExp;
+#endif
+			ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+			                                          psHwPerfInfo + ui32SrcRIdx,
+			                                          ui32BytesExp,
+			                                          uiStreamCopyMask);
+
+			/* Advance read index as before and Update the local copy of the
+			 * read index as it might be used in the last if branch*/
+			ui32SrcRIdx = RGXHWPerfAdvanceRIdx(
+					psDevInfo->ui32RGXFWIfHWPerfBufSize, ui32SrcRIdx,
+					ui32BytesCopied);
+
+			/* Update Wrap Count */
+			if ( ui32SrcRIdx == 0)
+			{
+				OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount,
+				                          psDevInfo->ui32RGXFWIfHWPerfBufSize);
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfWrapCount,
+				                           FLUSH);
+			}
+			OSWriteDeviceMem32WithWMB(&psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx, ui32SrcRIdx);
+
+			ui32BytesCopiedSum += ui32BytesCopied;
+
+			/* If all the data in the end of the array was copied, try copying
+			 * wrapped data in the beginning of the array, assuming there is
+			 * any and the RIdx was wrapped. */
+			if (   (ui32BytesCopied == ui32BytesExp)
+					&& (ui32SrcWIdx > 0)
+					&& (ui32SrcRIdx == 0) )
+			{
+				ui32BytesExp = ui32SrcWIdx;
+#ifdef HWPERF_MISR_FUNC_DEBUG
+				ui32BytesExpSum += ui32BytesExp;
+#endif
+				ui32BytesCopied = RGXHWPerfCopyDataL1toL2(psDevInfo,
+				                                          psHwPerfInfo,
+				                                          ui32BytesExp,
+				                                          uiStreamCopyMask);
+				/* Advance the FW buffer read position. */
+				psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx = RGXHWPerfAdvanceRIdx(
+						psDevInfo->ui32RGXFWIfHWPerfBufSize, ui32SrcRIdx,
+						ui32BytesCopied);
+
+				ui32BytesCopiedSum += ui32BytesCopied;
+			}
+			/* This flush covers both writes above */
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->sHWPerfCtrl.ui32HWPerfRIdx,
+			                           FLUSH);
+		}
+#ifdef HWPERF_MISR_FUNC_DEBUG
+		if (ui32BytesCopiedSum != ui32BytesExpSum)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfDataStore: FW L1 RIdx:%u. Not all bytes copied to L2: %u bytes out of %u expected", psFwSysData->sHWPerfCtrl.ui32HWPerfRIdx, ui32BytesCopiedSum, ui32BytesExpSum));
+		}
+#endif
+
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_VERBOSE, "RGXHWPerfDataStore NO EVENTS to transport"));
+	}
+
+	PVR_DPF_RETURN_VAL(ui32BytesCopiedSum);
+}
+
+/* Function called from MISR to copy data from L1 buffer to L2 streams. */
+PVRSRV_ERROR RGXHWPerfDataStoreCB(PVRSRV_DEVICE_NODE *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+	IMG_UINT32          ui32BytesCopied;
+
+	PVR_ASSERT(psDevInfo);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevInfo, PVRSRV_OK);
+
+	PVR_DPF_ENTERED;
+
+	psRgxDevInfo = psDevInfo->pvDevice;
+
+	/* Store FW event data if the destination buffer exists.*/
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->uiHWPerfStreamCount > 0)
+	{
+		ui32BytesCopied = RGXHWPerfDataStore(psRgxDevInfo);
+		if ( ui32BytesCopied )
+		{
+			/* It's possible that the HWPerf stream doesn't exist yet. It's
+			 * possible that only FTrace L2 stream has been created so far. */
+			if (psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] != NULL)
+			{
+				/* Signal consumers that packets may be available to read when
+				 * running from a HW kick, not when called by client APP thread
+				 * via the transport layer CB as this can lead to stream
+				 * corruption. */
+				eError = TLStreamSync(psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF]);
+				PVR_ASSERT(eError == PVRSRV_OK);
+			}
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfDataStoreCB: Zero bytes copied"));
+			RGXDEBUG_PRINT_IRQ_COUNT(psRgxDevInfo);
+		}
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+	PVR_DPF_RETURN_OK;
+}
+
+
+/* Currently supported by default */
+#if !defined(NO_HARDWARE) && defined(SUPPORT_TL_PRODUCER_CALLBACK)
+static PVRSRV_ERROR RGXHWPerfTLCB(IMG_HANDLE hStream,
+                                  IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*)pvUser;
+
+	PVR_UNREFERENCED_PARAMETER(hStream);
+	PVR_UNREFERENCED_PARAMETER(ui32Resp);
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	switch (ui32ReqOp)
+	{
+		case TL_SOURCECB_OP_CLIENT_EOS:
+			/* Keep HWPerf resource init check and use of
+			 * resources atomic, they may not be freed during use
+			 */
+
+			/* This solution is for avoiding a deadlock situation where -
+			 * in DoTLStreamReserve(), writer has acquired HWPerfLock and
+			 * ReadLock and is waiting on ReadPending (which will be reset
+			 * by reader), And
+			 * the reader after setting ReadPending in TLStreamAcquireReadPos(),
+			 * is waiting for HWPerfLock in RGXHWPerfTLCB().
+			 * So here in RGXHWPerfTLCB(), if HWPerfLock is already acquired we
+			 * will return to the reader without waiting to acquire HWPerfLock.
+			 */
+			if (!OSTryLockAcquire(psRgxDevInfo->hHWPerfLock))
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "hHWPerfLock is already acquired, a write "
+						"operation might already be in process"));
+				return PVRSRV_OK;
+			}
+
+			if (psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] != NULL)
+			{
+				(void) RGXHWPerfDataStore(psRgxDevInfo);
+			}
+			OSLockRelease(psRgxDevInfo->hHWPerfLock);
+			break;
+
+		default:
+			break;
+	}
+
+	return eError;
+}
+#endif
+
+
+static void RGXHWPerfL1BufferDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	if (psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc)
+	{
+		if (psRgxDevInfo->psRGXFWIfHWPerfBuf != NULL)
+		{
+			DevmemReleaseCpuVirtAddr(psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+			psRgxDevInfo->psRGXFWIfHWPerfBuf = NULL;
+		}
+		DevmemFwUnmapAndFree(psRgxDevInfo, psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+		psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc = NULL;
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInit
+
+@Description    Called during driver init for initialization of HWPerf module
+				in the Rogue device driver. This function keeps allocated
+				only the minimal necessary resources, which are required for
+				functioning of HWPerf server module.
+
+@Input          psRgxDevInfo	RGX Device Info
+
+@Return			PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 i;
+
+	PVR_DPF_ENTERED;
+
+	/* expecting a valid device info */
+	PVR_RETURN_IF_INVALID_PARAM(psRgxDevInfo != NULL);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_OK);
+
+	/* Create a lock for HWPerf server module used for serializing, L1 to L2
+	 * copy calls (e.g. in case of TL producer callback) and L1, L2 resource
+	 * allocation */
+	eError = OSLockCreate(&psRgxDevInfo->hHWPerfLock);
+	PVR_LOG_RETURN_IF_ERROR(eError, "OSLockCreate");
+
+	/* Initialise only non-zero fields since psRgxDevInfo is zeroed
+	 * on allocation. */
+	for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+	{
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[i] = IMG_TRUE;
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfIsInitRequired
+
+@Description    Returns true if the HWperf firmware buffer (L1 buffer) and host
+                driver TL buffer (L2 buffer) are not already allocated. Caller
+                must possess hHWPerfLock lock before calling this
+                function so the state tested is not inconsistent.
+
+@Input          psRgxDevInfo RGX Device Info, on which init requirement is
+                checked.
+
+@Return         IMG_BOOL	Whether initialization (allocation) is required
+ */ /**************************************************************************/
+static INLINE IMG_BOOL RGXHWPerfIsInitRequired(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                               RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hHWPerfLock));
+
+#if !defined(NO_HARDWARE)
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Both L1 and L2 buffers are required (for HWPerf functioning) on driver
+	 * built for actual hardware (TC, EMU, etc.)
+	 */
+	return psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc == NULL ||
+	       psRgxDevInfo->hHWPerfStream[eL2StreamId] == NULL;
+#else
+	/* On a NO-HW driver L2 is not allocated. So, no point in checking its
+	 * allocation */
+	return psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc == NULL;
+#endif
+}
+#if !defined(NO_HARDWARE)
+static void _HWPerfFWOnReaderOpenCB(void *pvArg)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+	PVRSRV_DEVICE_NODE* psDevNode = (PVRSRV_DEVICE_NODE*) psRgxDevInfo->psDeviceNode;
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	IMG_UINT32 ui32kCCBCommandSlot;
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDevNode);
+
+	/* Clear any previously suspended state for bSuspendHWPerfL2DataCopy as we
+	 * now have a reader attached so the data will be delivered upstream. */
+	if (psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF])
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Resuming HWPerf FW event collection.",
+		        __func__));
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF] = IMG_FALSE;
+	}
+
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG;
+	sKccbCmd.uCmdData.sHWPerfCtrl.eOpCode = RGXFWIF_HWPERF_CTRL_EMIT_FEATURES_EV;
+	sKccbCmd.uCmdData.sHWPerfCtrl.ui64Mask = 0;
+
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevNode->pvDevice,
+											  RGXFWIF_DM_GP,
+											  &sKccbCmd,
+											  PDUMP_FLAGS_CONTINUOUS,
+											  &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to generate feature packet in "
+				"firmware (error = %d)", __func__, eError));
+		return;
+	}
+
+	eError = RGXWaitForKCCBSlotUpdate(psRgxDevInfo, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_RETURN_VOID_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+}
+
+static void _HWPerfFWOnReaderCloseCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo = (PVRSRV_RGXDEV_INFO*) pvArg;
+
+	psRgxDevInfo->bSuspendHWPerfL2DataCopy[RGX_HWPERF_L2_STREAM_HWPERF] = IMG_TRUE;
+}
+#endif
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInitOnDemandL1Buffer
+
+@Description    This function allocates the HWperf firmware buffer (L1 buffer)
+                if HWPerf is enabled at driver load time. Otherwise, this
+                buffer is allocated on-demand as and when required. Caller must
+                possess hHWPerfLock lock before calling this function so the
+                state tested is not inconsistent if called outside of
+                initialisation.
+
+@Input          psRgxDevInfo RGX Device Info, on which init is done
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInitOnDemandL1Buffer(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_ERROR eError;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_DPF_ENTERED;
+
+	/* This function might be called more than once due to initialisation of
+	 * multiple consumers. Make sure that L1 is only ever initialised once. */
+	if (psRgxDevInfo->psRGXFWIfHWPerfBuf != NULL)
+	{
+		PVR_DPF_RETURN_OK;
+	}
+
+	/* Create the L1 HWPerf buffer on demand, read-only for the CPU
+	 * (except for the zero/poison operations) */
+	uiMemAllocFlags = PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)
+						| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+						| PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE
+						| PVRSRV_MEMALLOCFLAG_GPU_UNCACHED
+						| PVRSRV_MEMALLOCFLAG_CPU_READABLE
+						| PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC
+						| PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+#if defined(PDUMP) /* Helps show where the packet data ends */
+						| PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC
+#else /* Helps show corruption issues in driver-live */
+						| PVRSRV_MEMALLOCFLAG_POISON_ON_ALLOC
+#endif
+						| PVRSRV_MEMALLOCFLAG_RI_FWKMD_ALLOC
+						| PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN);
+
+	/* Allocate HWPerf FW L1 buffer */
+	eError = DevmemFwAllocate(psRgxDevInfo,
+	                          /* Pad it enough to hold the biggest variable sized packet. */
+	                          psRgxDevInfo->ui32RGXFWIfHWPerfBufSize+RGX_HWPERF_MAX_PACKET_SIZE,
+	                          uiMemAllocFlags,
+	                          "FwHWPerfBuffer",
+	                          &psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate kernel fw hwperf buffer (%u)",
+				__func__, eError));
+		goto ErrReturn;
+	}
+
+	/* Expecting the RuntimeCfg structure is mapped into CPU virtual memory.
+	 * Also, make sure the FW address is not already set */
+	PVR_ASSERT(psRgxDevInfo->psRGXFWIfRuntimeCfg && psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr == 0x0);
+
+	/* Meta cached flag removed from this allocation as it was found
+	 * FW performance was better without it. */
+	eError = RGXSetFirmwareAddress(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf,
+	                      psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                      0, RFW_FWADDR_NOREF_FLAG);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrDeInitL1Buffer);
+
+#if defined(RGX_FEATURE_HWPERF_VOLCANIC)
+	RGXSetMetaDMAAddress(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfDMABuf,
+	                     psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                     &psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf,
+	                     0);
+#endif
+
+	/* flush write buffers for psRgxDevInfo->psRGXFWIfRuntimeCfg */
+	OSWriteMemoryBarrier(&psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr);
+	RGXFwSharedMemCacheOpValue(psRgxDevInfo->psRGXFWIfRuntimeCfg->sHWPerfBuf.ui32Addr, FLUSH);
+
+	eError = DevmemAcquireCpuVirtAddr(psRgxDevInfo->psRGXFWIfHWPerfBufMemDesc,
+	                                  (void**)&psRgxDevInfo->psRGXFWIfHWPerfBuf);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire kernel hwperf buffer (%u)",
+				__func__, eError));
+		goto ErrDeInitL1Buffer;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerf buffer size in bytes: L1: %d",
+	         psRgxDevInfo->ui32RGXFWIfHWPerfBufSize));
+
+	PVR_DPF_RETURN_OK;
+
+ErrDeInitL1Buffer:
+	/* L1 buffer initialisation failures */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+ErrReturn:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfInitOnDemandL2Stream
+
+@Description    This function allocates the HWperf firmware buffer (L1 buffer)
+                and host driver TL buffer (L2 buffer) if HWPerf is enabled at
+                driver load time. Otherwise, these buffers are allocated
+                on-demand as and when required. Caller must possess hHWPerfLock
+                lock before calling this function so the state tested is not
+                inconsistent if called outside of driver initialisation.
+
+@Input          psRgxDevInfo RGX Device Info, on which init is done
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfInitOnDemandL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                           RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if !defined(NO_HARDWARE)
+	IMG_HANDLE hStream;
+	TL_STREAM_INFO sTLStreamInfo;
+#endif
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	PVR_DPF_ENTERED;
+
+#if !defined(NO_HARDWARE)
+	if (eL2StreamId == RGX_HWPERF_L2_STREAM_HWPERF)
+	{
+		/* On NO-HW driver, there is no MISR installed to copy data from L1 to L2. Hence,
+		 * L2 buffer is not allocated */
+		IMG_CHAR pszHWPerfStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 4];
+			/* + 4 is used to allow names up to "hwperf_fw_999", which is enough */
+
+		/* form the HWPerf stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfStreamName, sizeof(pszHWPerfStreamName), "%s%d",
+					   PVRSRV_TL_HWPERF_RGX_FW_STREAM,
+					   psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		eError = TLStreamCreate(&hStream,
+								pszHWPerfStreamName,
+								psRgxDevInfo->ui32RGXL2HWPerfBufSize,
+								TL_OPMODE_DROP_NEWER | TL_FLAG_NO_SIGNAL_ON_COMMIT,
+								_HWPerfFWOnReaderOpenCB, psRgxDevInfo,
+								_HWPerfFWOnReaderCloseCB, psRgxDevInfo,
+#if !defined(SUPPORT_TL_PRODUCER_CALLBACK)
+								NULL, NULL
+#else
+								/* Not enabled by default */
+								RGXHWPerfTLCB, psRgxDevInfo
+#endif
+								);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TLStreamCreate", ErrClearStream);
+
+		eError = TLStreamSetNotifStream(hStream,
+										PVRSRVGetPVRSRVData()->hTLCtrlStream);
+		/* we can still discover host stream so leave it as is and just log error */
+		PVR_LOG_IF_ERROR(eError, "TLStreamSetNotifStream");
+
+		/* send the event here because host stream is implicitly opened for write
+		 * in TLStreamCreate and TLStreamOpen is never called (so the event is
+		 * never emitted) */
+		TLStreamMarkStreamOpen(hStream);
+
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf buffer size in bytes: L1: %d  L2: %d",
+			psRgxDevInfo->ui32RGXFWIfHWPerfBufSize,
+			psRgxDevInfo->ui32RGXL2HWPerfBufSize));
+
+		psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] = hStream;
+		psRgxDevInfo->uiHWPerfStreamCount++;
+		PVR_ASSERT(psRgxDevInfo->uiHWPerfStreamCount <= RGX_HWPERF_L2_STREAM_LAST);
+	}
+#if (defined(__linux__) && !defined(__QNXNTO__) && !defined(INTEGRITY_OS))
+	else if (eL2StreamId == RGX_HWPERF_L2_STREAM_FTRACE)
+	{
+		eError = PVRGpuTraceInitStream(psRgxDevInfo);
+		PVR_LOG_IF_ERROR(eError, "PVRGpuTraceInitStream");
+	}
+#endif
+
+	TLStreamInfo(psRgxDevInfo->hHWPerfStream[eL2StreamId], &sTLStreamInfo);
+	psRgxDevInfo->ui32L2BufMaxPacketSize[eL2StreamId] = sTLStreamInfo.maxTLpacketSize;
+#else
+	psRgxDevInfo->hHWPerfStream[eL2StreamId] = NULL;
+#endif /* !defined(NO_HARDWARE) */
+
+	PVR_DPF_RETURN_OK;
+
+#if !defined(NO_HARDWARE)
+ErrClearStream: /* L2 buffer initialisation failures */
+	psRgxDevInfo->hHWPerfStream[RGX_HWPERF_L2_STREAM_HWPERF] = NULL;
+#endif
+	/* L1 buffer initialisation failures */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+
+void RGXHWPerfDeinitL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                             RGX_HWPERF_L2_STREAM_ID eL2StreamId)
+{
+	IMG_HANDLE hStream;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRgxDevInfo);
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+
+	hStream = psRgxDevInfo->hHWPerfStream[eL2StreamId];
+
+	/* Clean up the L2 buffer stream object if allocated */
+	if (hStream)
+	{
+		psRgxDevInfo->hHWPerfStream[eL2StreamId] = NULL;
+		psRgxDevInfo->bSuspendHWPerfL2DataCopy[eL2StreamId] = IMG_TRUE;
+		psRgxDevInfo->uiHWPerfStreamCount--;
+		PVR_ASSERT(psRgxDevInfo->uiHWPerfStreamCount < RGX_HWPERF_L2_STREAM_LAST);
+
+		/* send the event here because host stream is implicitly opened for
+		 * write in TLStreamCreate and TLStreamClose is never called (so the
+		 * event is never emitted) */
+		TLStreamMarkStreamClose(hStream);
+		TLStreamClose(hStream);
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void RGXHWPerfDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	/* Cleanup L1 buffer resources */
+	RGXHWPerfL1BufferDeinit(psRgxDevInfo);
+
+	/* Cleanup the HWPerf server module lock resource */
+	if (psRgxDevInfo->hHWPerfLock)
+	{
+		OSLockDestroy(psRgxDevInfo->hHWPerfLock);
+		psRgxDevInfo->hHWPerfLock = NULL;
+	}
+}
+
+
+/******************************************************************************
+ * RGX HW Performance Profiling Server API(s)
+ *****************************************************************************/
+
+static PVRSRV_ERROR RGXHWPerfCtrlFwBuffer(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                          RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                          IMG_BOOL bToggle,
+                                          IMG_UINT64 ui64Mask)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psDevice = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD sKccbCmd;
+	IMG_UINT32 ui32kCCBCommandSlot;
+	IMG_UINT64 ui64MaskValue = ui64Mask;
+	IMG_UINT64 ui64OldMaskValue;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Modify mask to include the default bit settings if it is non-zero */
+	if (!bToggle && (ui64Mask != 0ULL))
+	{
+		ui64MaskValue = ui64Mask | RGX_HWPERF_EVENT_MASK_DEFAULT;
+	}
+
+	/* If this method is being used whether to enable or disable
+	 * then the hwperf buffers (host and FW) are likely to be needed
+	 * eventually so create them, also helps unit testing. Buffers
+	 * allocated on demand to reduce RAM foot print on systems not
+	 * needing HWPerf resources.
+	 * Obtain lock first, test and init if required. */
+	OSLockAcquire(psDevice->hHWPerfLock);
+
+	if (!psDevice->bFirmwareInitialised)
+	{
+		/* No other initialisation can be done at this point until the FW is
+		 * initialised so unlock, log and return Ok so the caller knows
+		 * the filter was set. */
+		(void) RGXHWPerfFwSetEventFilterNoLock(psDevice, eL2StreamId, ui64MaskValue);
+		OSLockRelease(psDevice->hHWPerfLock);
+		goto done_;
+	}
+
+	if (RGXHWPerfIsInitRequired(psDevice, eL2StreamId))
+	{
+		eError = RGXHWPerfInitOnDemandL1Buffer(psDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Initialisation of on-demand HWPerfFW "
+					"resources failed", __func__));
+			goto unlock_and_return;
+		}
+
+		/* if this fails it also cleans up L1 buffer */
+		eError = RGXHWPerfInitOnDemandL2Stream(psDevice, eL2StreamId);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Initialisation of on-demand HWPerfFW "
+					"resources failed", __func__));
+			goto unlock_and_return;
+		}
+	}
+
+	/* Unlock here as no further HWPerf resources are used below that would be
+	 * affected if freed by another thread */
+	OSLockRelease(psDevice->hHWPerfLock);
+
+	/* Return if the filter is the same */
+	if (!bToggle && psDevice->ui64HWPerfFilter[eL2StreamId] == ui64MaskValue)
+	{
+		goto done_;
+	}
+
+	ui64OldMaskValue = psDevice->ui64HWPerfFilter[eL2StreamId];
+	ui64MaskValue = RGXHWPerfFwSetEventFilter(psDevice, eL2StreamId, bToggle
+	                                          ? psDevice->ui64HWPerfFilter[eL2StreamId] ^ ui64MaskValue
+	                                          : ui64MaskValue);
+
+	/* Prepare command parameters ... */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_UPDATE_CONFIG;
+	sKccbCmd.uCmdData.sHWPerfCtrl.eOpCode = bToggle ? RGXFWIF_HWPERF_CTRL_TOGGLE : RGXFWIF_HWPERF_CTRL_SET;
+	sKccbCmd.uCmdData.sHWPerfCtrl.ui64Mask = ui64MaskValue;
+
+	/* Ask the FW to carry out the HWPerf configuration command */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set new HWPerfFW filter in "
+				"firmware (error = %d)", __func__, eError));
+		goto restore_mask_;
+	}
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate", restore_mask_);
+
+done_:
+	return PVRSRV_OK;
+
+unlock_and_return:
+	OSLockRelease(psDevice->hHWPerfLock);
+
+	return eError;
+
+restore_mask_:
+	(void) RGXHWPerfFwSetEventFilter(psDevice, eL2StreamId, ui64OldMaskValue);
+
+	return eError;
+}
+
+#define HWPERF_HOST_MAX_DEFERRED_PACKETS 800
+
+static PVRSRV_ERROR RGXHWPerfCtrlHostBuffer(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            IMG_BOOL bToggle,
+                                            IMG_UINT32 ui32Mask)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO* psDevice = psDeviceNode->pvDevice;
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	IMG_UINT32 ui32OldFilter = psDevice->ui32HWPerfHostFilter;
+#endif
+
+	if (psDevice->hHWPerfHostStream == NULL)
+	{
+		eError = RGXHWPerfHostInitOnDemandResources(psDevice);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Initialisation of on-demand HWPerfHost resources failed",
+					 __func__));
+			return eError;
+		}
+	}
+
+	OSLockAcquire(psDevice->hLockHWPerfHostStream);
+	psDevice->ui32HWPerfHostFilter = bToggle ?
+			psDevice->ui32HWPerfHostFilter ^ ui32Mask : ui32Mask;
+
+	// Deferred creation of host periodic events thread
+	if (psDevice->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HOST_INFO))
+	{
+		eError = PVRSRVCreateHWPerfHostThread(PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVCreateHWPerfHostThread");
+	}
+	else
+	{
+		eError = PVRSRVDestroyHWPerfHostThread();
+		PVR_LOG_IF_ERROR(eError, "PVRSRVDestroyHWPerfHostThread");
+	}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	// Log deferred events stats if filter changed from non-zero to zero
+	if ((ui32OldFilter != 0) && (psDevice->ui32HWPerfHostFilter == 0))
+	{
+		PVR_LOG(("HWPerfHost deferred events buffer high-watermark / size: (%u / %u)",
+				psDevice->ui32DEHighWatermark, HWPERF_HOST_MAX_DEFERRED_PACKETS));
+
+		PVR_LOG(("HWPerfHost deferred event retries: WaitForAtomicCtxPktHighWatermark(%u) "
+				"WaitForRightOrdPktHighWatermark(%u)",
+				psDevice->ui32WaitForAtomicCtxPktHighWatermark,
+				psDevice->ui32WaitForRightOrdPktHighWatermark));
+	}
+#endif
+
+	OSLockRelease(psDevice->hLockHWPerfHostStream);
+
+	if (bToggle)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfHost events (%x) have been TOGGLED",
+				ui32Mask));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfHost mask has been SET to (%x)",
+				ui32Mask));
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXHWPerfCtrlClientBuffer(IMG_BOOL bToggle,
+                                              IMG_UINT32 ui32InfoPageIdx,
+                                              IMG_UINT32 ui32Mask)
+{
+	PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+
+	PVR_LOG_RETURN_IF_FALSE(ui32InfoPageIdx >= INFO_PAGE_HWPERF_BLOCK_START &&
+	                        ui32InfoPageIdx < INFO_PAGE_HWPERF_BLOCK_END,
+	                        "invalid info page index", PVRSRV_ERROR_INVALID_PARAMS);
+
+	OSLockAcquire(psData->hInfoPageLock);
+	psData->pui32InfoPage[ui32InfoPageIdx] = bToggle ?
+			psData->pui32InfoPage[ui32InfoPageIdx] ^ ui32Mask : ui32Mask;
+	OSLockRelease(psData->hInfoPageLock);
+
+	if (bToggle)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfClient (%u) events (%x) have been TOGGLED",
+				ui32InfoPageIdx, ui32Mask));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerfClient (%u) mask has been SET to (%x)",
+				ui32InfoPageIdx, ui32Mask));
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(CONNECTION_DATA    *psConnection,
+                                                  PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                  RGX_HWPERF_BVNC    *psBVNC)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR        eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE((NULL != psDeviceNode), "psDeviceNode invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = psDeviceNode->pvDevice;
+	eError = RGXServerFeatureFlagsToHWPerfFlags(psDevInfo, psBVNC);
+
+	return eError;
+}
+
+/*
+	AppHint interfaces
+ */
+static
+PVRSRV_ERROR RGXHWPerfSetFwFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT64 ui64Value)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	eError = RGXHWPerfCtrlFwBuffer(psDeviceNode, RGX_HWPERF_L2_STREAM_HWPERF,
+	                               IMG_FALSE, ui64Value);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Failed to set HWPerf firmware filter for device (%u)",
+		        psDeviceNode->sDevId.ui32InternalID));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfReadFwFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT64 *pui64Value)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pui64Value = psDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF];
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfSetHostFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    const void *psPrivate,
+                                    IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	eError = RGXHWPerfCtrlHostBuffer(psDeviceNode, IMG_FALSE, ui32Value);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "Failed to set HWPerf firmware filter for device (%u)",
+		        psDeviceNode->sDevId.ui32InternalID));
+		return eError;
+	}
+
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXHWPerfReadHostFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     const void *psPrivate,
+                                     IMG_UINT32 *pui32Value)
+{
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL);
+	PVR_RETURN_IF_INVALID_PARAM(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	*pui32Value =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32HWPerfHostFilter;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _ReadClientFilter(const PVRSRV_DEVICE_NODE *psDevice,
+                                      const void *psPrivData,
+                                      IMG_UINT32 *pui32Value)
+{
+	PVRSRV_DATA *psData = PVRSRVGetPVRSRVData();
+	IMG_UINT32 ui32Idx = (IMG_UINT32) (uintptr_t) psPrivData;
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	OSLockAcquire(psData->hInfoPageLock);
+	*pui32Value = psData->pui32InfoPage[ui32Idx];
+	OSLockRelease(psData->hInfoPageLock);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _WriteClientFilter(const PVRSRV_DEVICE_NODE *psDevice,
+                                       const void *psPrivData,
+                                       IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 ui32Idx = (IMG_UINT32) (uintptr_t) psPrivData;
+	PVR_UNREFERENCED_PARAMETER(psDevice);
+
+	return RGXHWPerfCtrlClientBuffer(IMG_FALSE, ui32Idx, ui32Value);
+}
+
+void RGXHWPerfInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersUINT64(APPHINT_ID_HWPerfFWFilter,
+	                                    RGXHWPerfReadFwFilter,
+	                                    RGXHWPerfSetFwFilter,
+	                                    psDeviceNode,
+	                                    NULL);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfHostFilter,
+	                                    RGXHWPerfReadHostFilter,
+	                                    RGXHWPerfSetHostFilter,
+	                                    psDeviceNode,
+	                                    NULL);
+}
+
+void RGXHWPerfClientInitAppHintCallbacks(void)
+{
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_Services,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_SERVICES_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_EGL,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_EGL_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenGLES,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_OPENGLES_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenCL,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_OPENCL_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_Vulkan,
+	                                    _ReadClientFilter,
+	                                    _WriteClientFilter,
+	                                    APPHINT_OF_DRIVER_NO_DEVICE,
+	                                    (void *) HWPERF_FILTER_VULKAN_IDX);
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_HWPerfClientFilter_OpenGL,
+		                                _ReadClientFilter,
+		                                _WriteClientFilter,
+		                                APPHINT_OF_DRIVER_NO_DEVICE,
+		                                (void *) HWPERF_FILTER_OPENGL_IDX);
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCountersKM(CONNECTION_DATA *psConnection,
+                                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                    const IMG_UINT32 ui32BlockID,
+                                                    RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(psConfiguredCounters != NULL, "psConfiguredCounters is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXGetHWPerfCtl");
+
+	eError = PVRSRVRGXGetConfiguredHWPerfCounters(psDeviceNode,
+	                                              psHWPerfCtl,
+	                                              ui32BlockID,
+	                                              psConfiguredCounters);
+	PVR_LOG_IF_ERROR(eError, "PVRSRVRGXGetConfiguredHWPerfCounters");
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocksKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               const IMG_UINT32 ui32ArrayLen,
+                                               IMG_UINT32 *pui32BlockCount,
+                                               IMG_UINT32 *pui32EnabledBlockIDs)
+{
+	RGXFWIF_HWPERF_CTL *psHWPerfCtl;
+	IMG_UINT32 *pui32BlockIDs = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_LOG_RETURN_IF_FALSE(psDeviceNode != NULL, "psDeviceNode is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE(pui32BlockCount != NULL, "pui32BlockCount is invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = RGXAcquireHWPerfCtlCPUAddr(psDeviceNode, &psHWPerfCtl);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXGetHWPerfCtl");
+
+	if (pui32EnabledBlockIDs != NULL)
+	{
+		pui32BlockIDs = OSAllocMem(sizeof(IMG_UINT32) * ui32ArrayLen);
+		if (pui32BlockIDs == NULL)
+		{
+			PVR_LOG_GOTO_WITH_ERROR("OSAllocMem", eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+		}
+	}
+
+	eError = PVRSRVRGXGetEnabledHWPerfBlocks(psDeviceNode,
+	                                         psHWPerfCtl,
+	                                         ui32ArrayLen,
+	                                         pui32BlockCount,
+	                                         pui32BlockIDs);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRGXGetEnabledHWPerfBlocks", Error);
+
+	if (pui32EnabledBlockIDs != NULL)
+	{
+		IMG_UINT32 i;
+		if (*pui32BlockCount > ui32ArrayLen)
+		{
+			*pui32BlockCount = 0;
+			PVR_DPF((PVR_DBG_ERROR, "ui32ArrayLen less than the number of enabled blocks."));
+			PVR_LOG_GOTO_WITH_ERROR(__func__, eError, PVRSRV_ERROR_OUT_OF_MEMORY, Error);
+		}
+		else if (*pui32BlockCount < ui32ArrayLen)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "ui32ArrayLen greater than the number of enabled blocks."));
+		}
+
+		for (i = 0; i < *pui32BlockCount; i++)
+		{
+			pui32EnabledBlockIDs[i] = pui32BlockIDs[i];
+		}
+	}
+
+Error:
+	if (pui32BlockIDs != NULL)
+	{
+		OSFreeMem(pui32BlockIDs);
+	}
+
+	RGXReleaseHWPerfCtlCPUAddr(psDeviceNode);
+
+	return eError;
+}
+
+static INLINE IMG_UINT32 _RGXHWPerfFixBufferSize(IMG_UINT32 ui32BufSizeKB)
+{
+	if (ui32BufSizeKB > HWPERF_HOST_TL_STREAM_SIZE_MAX)
+	{
+		/* Size specified as a AppHint but it is too big */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "RGXHWPerfHostInit: HWPerf Host buffer size "
+		         "value (%u) too big, using maximum (%u)",
+		         ui32BufSizeKB, HWPERF_HOST_TL_STREAM_SIZE_MAX));
+		return HWPERF_HOST_TL_STREAM_SIZE_MAX<<10;
+	}
+	else if (ui32BufSizeKB >= HWPERF_HOST_TL_STREAM_SIZE_MIN)
+	{
+		return ui32BufSizeKB<<10;
+	}
+	else if (ui32BufSizeKB > 0)
+	{
+		/* Size specified as a AppHint but it is too small */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "RGXHWPerfHostInit: HWPerf Host buffer size "
+		         "value (%u) too small, using minimum (%u)",
+		         ui32BufSizeKB, HWPERF_HOST_TL_STREAM_SIZE_MIN));
+		return HWPERF_HOST_TL_STREAM_SIZE_MIN<<10;
+	}
+	else
+	{
+		/* 0 size implies AppHint not set or is set to zero,
+		 * use default size from driver constant. */
+		return HWPERF_HOST_TL_STREAM_SIZE_DEFAULT<<10;
+	}
+}
+
+/******************************************************************************
+ * RGX HW Performance Host Stream API
+ *****************************************************************************/
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfHostInit
+
+@Description    Called during driver init for initialisation of HWPerfHost
+                stream in the Rogue device driver. This function keeps allocated
+                only the minimal necessary resources, which are required for
+                functioning of HWPerf server module.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfHostInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32BufSizeKB)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_INVALID_PARAM(psRgxDevInfo != NULL);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_OK);
+
+	eError = OSLockCreate(&psRgxDevInfo->hLockHWPerfHostStream);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", error);
+
+	psRgxDevInfo->hHWPerfHostStream = NULL;
+	psRgxDevInfo->ui32HWPerfHostFilter = 0; /* disable all events */
+	psRgxDevInfo->ui32HWPerfHostNextOrdinal = 1;
+	psRgxDevInfo->ui32HWPerfHostBufSize = _RGXHWPerfFixBufferSize(ui32BufSizeKB);
+	psRgxDevInfo->pvHostHWPerfMISR = NULL;
+	psRgxDevInfo->pui8DeferredEvents = NULL;
+	/* First packet has ordinal=1, so LastOrdinal=0 will ensure ordering logic
+	 * is maintained */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = 0;
+	psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+
+error:
+	return eError;
+}
+
+#define RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE \
+	((IMG_UINT32)(offsetof(RGX_HWPERF_HOST_CLIENT_INFO_DATA, uDetail) + \
+		sizeof(((RGX_HWPERF_HOST_CLIENT_INFO_DETAIL*)0)->sProcName.ui32Count)))
+
+static void _HWPerfHostOnConnectCB(void *pvArg)
+{
+	PVRSRV_RGXDEV_INFO* psDevice;
+	PVRSRV_ERROR eError;
+
+	RGXSRV_HWPERF_CLK_SYNC(pvArg);
+
+	psDevice = (PVRSRV_RGXDEV_INFO*) pvArg;
+
+	/* Handle the case where we may be being called as part of a multi-device
+	 * initialisation sequence. If the bDevInit2Done flag is not yet set we can
+	 * perform no action for this device. Simply return.
+	 */
+	if (!psDevice->bDevInit2Done)
+	{
+		return;
+	}
+
+	/* Handle the case where the RGX_HWPERF_HOST_INFO bit is set in the event filter
+	 * before the host stream is opened for reading by a HWPerf client.
+	 * Which can result in the host periodic thread sleeping for a long duration as TLStreamIsOpenForReading may return false. */
+	if (psDevice->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(RGX_HWPERF_HOST_INFO))
+	{
+		eError = PVRSRVCreateHWPerfHostThread(PVRSRV_APPHINT_HWPERFHOSTTHREADTIMEOUTINMS);
+		PVR_LOG_IF_ERROR(eError, "PVRSRVCreateHWPerfHostThread");
+	}
+
+	RGXSRV_HWPERF_DEVICE_INFO_FEATURES(psDevice);
+
+	if (RGXHWPerfHostIsEventEnabled(psDevice, RGX_HWPERF_HOST_CLIENT_INFO))
+	{
+		// GCC throws -Werror=frame-larger-than error if the frame size is > 1024 bytes,
+		// so use a heap allocation - is there an alternate solution?
+		IMG_BYTE *pbPktPayload = (IMG_BYTE*)OSAllocMem(RGX_HWPERF_MAX_PAYLOAD_SIZE);
+
+		if (pbPktPayload)
+		{
+			RGX_HWPERF_HOST_CLIENT_INFO_DATA *psHostClientInfo;
+			RGX_HWPERF_HOST_CLIENT_PROC_NAME *psProcName;
+			IMG_UINT32 ui32TotalPayloadSize, ui32NameLen, ui32ProcNamePktSize;
+			DLLIST_NODE *pNode, *pNext;
+
+			psHostClientInfo = IMG_OFFSET_ADDR(pbPktPayload,0);
+			psHostClientInfo->eType = RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME;
+			psHostClientInfo->uDetail.sProcName.ui32Count = 0U;
+			psProcName = psHostClientInfo->uDetail.sProcName.asProcNames;
+			ui32TotalPayloadSize = RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE;
+
+			OSLockAcquire(psDevice->psDeviceNode->hConnectionsLock);
+
+			// Announce current client connections to the reader
+			dllist_foreach_node(&psDevice->psDeviceNode->sConnections, pNode, pNext)
+			{
+				CONNECTION_DATA *psData = IMG_CONTAINER_OF(pNode, CONNECTION_DATA, sConnectionListNode);
+
+				ui32NameLen = OSStringLength(psData->pszProcName) + 1U;
+				ui32ProcNamePktSize = RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen);
+
+				// Unlikely case where we have too much data to fit into a single hwperf packet
+				if (ui32ProcNamePktSize + ui32TotalPayloadSize > RGX_HWPERF_MAX_PAYLOAD_SIZE)
+				{
+					RGXHWPerfHostPostRaw(psDevice, RGX_HWPERF_HOST_CLIENT_INFO, pbPktPayload, ui32TotalPayloadSize);
+
+					psHostClientInfo->uDetail.sProcName.ui32Count = 0U;
+					psProcName = psHostClientInfo->uDetail.sProcName.asProcNames;
+					ui32TotalPayloadSize = RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE;
+				}
+
+				// Setup packet data
+				psHostClientInfo->uDetail.sProcName.ui32Count++;
+				psProcName->uiClientPID = psData->pid;
+				psProcName->ui32Length = ui32NameLen;
+				(void)OSCachedMemCopy(psProcName->acName, psData->pszProcName, ui32NameLen);
+
+				psProcName = (RGX_HWPERF_HOST_CLIENT_PROC_NAME*)IMG_OFFSET_ADDR(psProcName, ui32ProcNamePktSize);
+				ui32TotalPayloadSize += ui32ProcNamePktSize;
+			}
+
+			OSLockRelease(psDevice->psDeviceNode->hConnectionsLock);
+			RGXHWPerfHostPostRaw(psDevice, RGX_HWPERF_HOST_CLIENT_INFO, pbPktPayload, ui32TotalPayloadSize);
+			OSFreeMem(pbPktPayload);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: OUT OF MEMORY. Could not allocate memory for RGX_HWPERF_HOST_CLIENT_INFO_DATA packet.", __func__));
+		}
+	}
+}
+
+/* Avoiding a holder struct using fields below, as a struct gets along padding,
+ * packing, and other compiler dependencies, and we want a continuous stream of
+ * bytes for (header+data) for use in TLStreamWrite. See
+ * _HWPerfHostDeferredEventsEmitter().
+ *
+ * A deferred (UFO) packet is represented in memory as:
+ *     - IMG_BOOL                 --> Indicates whether a packet write is
+ *                                    "complete" by atomic context or not.
+ *     - RGX_HWPERF_V2_PACKET_HDR --.
+ *                                  |--> Fed together to TLStreamWrite for
+ *                                  |    deferred packet to be written to
+ *                                  |    HWPerfHost buffer
+ *     - RGX_HWPERF_HOST_UFO_DATA---`
+ *
+ * PS: Currently only UFO events are supported in deferred list */
+#define HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE (sizeof(IMG_BOOL) +\
+		sizeof(RGX_HWPERF_V2_PACKET_HDR) +\
+		sizeof(RGX_HWPERF_HOST_UFO_DATA))
+
+static void RGX_MISRHandler_HWPerfPostDeferredHostEvents(void *pvData);
+static void _HWPerfHostDeferredEventsEmitter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                             IMG_UINT32 ui32MaxOrdinal);
+
+/*************************************************************************/ /*!
+@Function       RGXHWPerfHostInitOnDemandResources
+
+@Description    This function allocates the HWPerfHost buffer if HWPerf is
+                enabled at driver load time. Otherwise, these buffers are
+                allocated on-demand as and when required.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+PVRSRV_ERROR RGXHWPerfHostInitOnDemandResources(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVRSRV_ERROR eError;
+	/* 4 makes space up to "hwperf_host_999" streams */
+	IMG_CHAR pszHWPerfHostStreamName[sizeof(PVRSRV_TL_HWPERF_HOST_SERVER_STREAM) + 4];
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psRgxDevInfo->hHWPerfHostStream != NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf host stream already initialised"));
+		return PVRSRV_OK;
+	}
+
+	/* form the HWPerf host stream name, corresponding to this DevNode; which can make sense in the UM */
+	if (OSSNPrintf(pszHWPerfHostStreamName, sizeof(pszHWPerfHostStreamName), "%s%d",
+	               PVRSRV_TL_HWPERF_HOST_SERVER_STREAM,
+	               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to form HWPerf host stream name for device %d",
+				__func__,
+				psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = TLStreamCreate(&psRgxDevInfo->hHWPerfHostStream,
+	                        pszHWPerfHostStreamName, psRgxDevInfo->ui32HWPerfHostBufSize,
+	                        TL_OPMODE_DROP_NEWER,
+	                        _HWPerfHostOnConnectCB, psRgxDevInfo,
+	                        NULL, NULL, NULL, NULL);
+	PVR_LOG_RETURN_IF_ERROR(eError, "TLStreamCreate");
+
+	eError = TLStreamSetNotifStream(psRgxDevInfo->hHWPerfHostStream,
+	                                PVRSRVGetPVRSRVData()->hTLCtrlStream);
+	/* we can still discover host stream so leave it as is and just log error */
+	PVR_LOG_IF_ERROR(eError, "TLStreamSetNotifStream");
+
+	/* send the event here because host stream is implicitly opened for write
+	 * in TLStreamCreate and TLStreamOpen is never called (so the event is
+	 * never emitted) */
+	eError = TLStreamMarkStreamOpen(psRgxDevInfo->hHWPerfHostStream);
+	PVR_LOG_IF_ERROR(eError, "TLStreamMarkStreamOpen");
+
+	/* HWPerfHost deferred events specific initialization */
+	eError = OSInstallMISR(&psRgxDevInfo->pvHostHWPerfMISR,
+	                       RGX_MISRHandler_HWPerfPostDeferredHostEvents,
+	                       psRgxDevInfo,
+	                       "RGX_HWPerfDeferredEventPoster");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR", err_install_misr);
+
+	eError = OSSpinLockCreate(&psRgxDevInfo->hHWPerfHostSpinLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate", err_spinlock_create);
+
+	psRgxDevInfo->pui8DeferredEvents = OSAllocMem(HWPERF_HOST_MAX_DEFERRED_PACKETS
+	                                              * HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE);
+	if (NULL == psRgxDevInfo->pui8DeferredEvents)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: OUT OF MEMORY. Could not allocate memory for "
+				"HWPerfHost deferred events array", __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_alloc_deferred_events;
+	}
+	psRgxDevInfo->ui16DEReadIdx = 0;
+	psRgxDevInfo->ui16DEWriteIdx = 0;
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	psRgxDevInfo->ui32DEHighWatermark = 0;
+	psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark = 0;
+	psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark = 0;
+#endif
+
+	PVR_DPF((DBGPRIV_MESSAGE, "HWPerf Host buffer size is %uKB",
+			psRgxDevInfo->ui32HWPerfHostBufSize));
+
+	return PVRSRV_OK;
+
+err_alloc_deferred_events:
+	OSSpinLockDestroy(psRgxDevInfo->hHWPerfHostSpinLock);
+	psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+
+err_spinlock_create:
+	(void) OSUninstallMISR(psRgxDevInfo->pvHostHWPerfMISR);
+	psRgxDevInfo->pvHostHWPerfMISR = NULL;
+
+err_install_misr:
+	TLStreamMarkStreamClose(psRgxDevInfo->hHWPerfHostStream);
+	TLStreamClose(psRgxDevInfo->hHWPerfHostStream);
+	psRgxDevInfo->hHWPerfHostStream = NULL;
+
+	return eError;
+}
+
+void RGXHWPerfHostDeInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	PVR_ASSERT (psRgxDevInfo);
+
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+
+	if (psRgxDevInfo->pui8DeferredEvents)
+	{
+		OSFreeMem(psRgxDevInfo->pui8DeferredEvents);
+		psRgxDevInfo->pui8DeferredEvents = NULL;
+	}
+
+	if (psRgxDevInfo->hHWPerfHostSpinLock)
+	{
+		OSSpinLockDestroy(psRgxDevInfo->hHWPerfHostSpinLock);
+		psRgxDevInfo->hHWPerfHostSpinLock = NULL;
+	}
+
+	if (psRgxDevInfo->pvHostHWPerfMISR)
+	{
+		(void) OSUninstallMISR(psRgxDevInfo->pvHostHWPerfMISR);
+		psRgxDevInfo->pvHostHWPerfMISR = NULL;
+	}
+
+	if (psRgxDevInfo->hHWPerfHostStream)
+	{
+		/* send the event here because host stream is implicitly opened for
+		 * write in TLStreamCreate and TLStreamClose is never called (so the
+		 * event is never emitted) */
+		TLStreamMarkStreamClose(psRgxDevInfo->hHWPerfHostStream);
+		TLStreamClose(psRgxDevInfo->hHWPerfHostStream);
+		psRgxDevInfo->hHWPerfHostStream = NULL;
+	}
+
+	if (psRgxDevInfo->hLockHWPerfHostStream)
+	{
+		OSLockDestroy(psRgxDevInfo->hLockHWPerfHostStream);
+		psRgxDevInfo->hLockHWPerfHostStream = NULL;
+	}
+}
+
+static IMG_UINT64 RGXHWPerfFwSetEventFilterNoLock(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                                  IMG_UINT64 uiFilter)
+{
+	IMG_UINT64 uiTmpFilter = 0;
+	IMG_UINT32 i;
+
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	/* Set filter for the given L2 stream. */
+	psRgxDevInfo->ui64HWPerfFilter[eL2StreamId] = uiFilter;
+
+	/* Compute compound filter from all existing L2 streams' filters. */
+	for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+	{
+		uiTmpFilter |= psRgxDevInfo->ui64HWPerfFilter[i];
+	}
+
+	psRgxDevInfo->ui64HWPerfFwFilter = uiTmpFilter;
+
+#if !defined(NO_HARDWARE)
+	PVR_DPF((PVR_DBG_MESSAGE, "HWPerfFW mask has been SET to 0x%" IMG_UINT64_FMTSPECx
+	         " (stream %u value SET to 0x%" IMG_UINT64_FMTSPECx ")",
+	         psRgxDevInfo->ui64HWPerfFwFilter, eL2StreamId,
+	         psRgxDevInfo->ui64HWPerfFilter[eL2StreamId]));
+#endif
+
+	return uiTmpFilter;
+}
+
+IMG_UINT64 RGXHWPerfFwSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                     RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                     IMG_UINT64 uiFilter)
+{
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	uiFilter = RGXHWPerfFwSetEventFilterNoLock(psRgxDevInfo, eL2StreamId, uiFilter);
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+	return uiFilter;
+}
+
+inline void RGXHWPerfHostSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Filter)
+{
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVINFO, psRgxDevInfo);
+	psRgxDevInfo->ui32HWPerfHostFilter = ui32Filter;
+}
+
+inline IMG_BOOL RGXHWPerfHostIsEventEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo, RGX_HWPERF_HOST_EVENT_TYPE eEvent)
+{
+	PVR_ASSERT(psRgxDevInfo);
+	return (psRgxDevInfo->ui32HWPerfHostFilter & RGX_HWPERF_EVENT_MASK_VALUE(eEvent)) ? IMG_TRUE : IMG_FALSE;
+}
+
+#define MAX_RETRY_COUNT 80
+static inline void _PostFunctionPrologue(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_UINT32 ui32CurrentOrdinal)
+{
+	IMG_UINT32 ui32Retry = MAX_RETRY_COUNT;
+
+	PVR_ASSERT(psRgxDevInfo->hLockHWPerfHostStream != NULL);
+	PVR_ASSERT(psRgxDevInfo->hHWPerfHostStream != NULL);
+
+	OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+
+	/* First, flush pending events (if any) */
+	_HWPerfHostDeferredEventsEmitter(psRgxDevInfo, ui32CurrentOrdinal);
+
+	while ((ui32CurrentOrdinal != psRgxDevInfo->ui32HWPerfHostLastOrdinal + 1)
+		   && (--ui32Retry != 0))
+	{
+		/* Release lock and give a chance to a waiting context to emit the
+		 * expected packet */
+		OSLockRelease (psRgxDevInfo->hLockHWPerfHostStream);
+		OSSleepms(100);
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+	}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+	if ((ui32Retry == 0) && !(psRgxDevInfo->bWarnedPktOrdinalBroke))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Will warn only once! Potential packet(s) lost after ordinal"
+				 " %u (Current ordinal = %u)",
+				 __func__,
+				 psRgxDevInfo->ui32HWPerfHostLastOrdinal, ui32CurrentOrdinal));
+		psRgxDevInfo->bWarnedPktOrdinalBroke = IMG_TRUE;
+	}
+
+	if (psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark < (MAX_RETRY_COUNT - ui32Retry))
+	{
+		psRgxDevInfo->ui32WaitForRightOrdPktHighWatermark = MAX_RETRY_COUNT - ui32Retry;
+	}
+#endif
+}
+
+static inline void _PostFunctionEpilogue(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_UINT32 ui32CurrentOrdinal)
+{
+	/* update last ordinal emitted */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = ui32CurrentOrdinal;
+
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hLockHWPerfHostStream));
+	OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+}
+
+static inline IMG_UINT8 *_ReserveHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Size)
+{
+	IMG_UINT8 *pui8Dest;
+
+	PVRSRV_ERROR eError = TLStreamReserve(psRgxDevInfo->hHWPerfHostStream,
+	                                      &pui8Dest, ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not reserve space in %s buffer"
+				" (%d). Dropping packet.",
+				__func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+		return NULL;
+	}
+	PVR_ASSERT(pui8Dest != NULL);
+
+	return pui8Dest;
+}
+
+static inline void _CommitHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError = TLStreamCommit(psRgxDevInfo->hHWPerfHostStream,
+	                                     ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not commit data to %s"
+				" (%d)", __func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+	}
+}
+
+/* Returns IMG_TRUE if packet write passes, IMG_FALSE otherwise */
+static inline IMG_BOOL _WriteHWPerfStream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                          RGX_HWPERF_V2_PACKET_HDR *psHeader)
+{
+	PVRSRV_ERROR eError = TLStreamWrite(psRgxDevInfo->hHWPerfHostStream,
+	                                    IMG_OFFSET_ADDR(psHeader, 0), psHeader->ui32Size);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Could not write packet in %s buffer"
+				" (%d). Dropping packet.",
+				__func__, PVRSRV_TL_HWPERF_HOST_SERVER_STREAM, eError));
+	}
+
+	/* Regardless of whether write passed/failed, we consider it "written" */
+	psRgxDevInfo->ui32HWPerfHostLastOrdinal = psHeader->ui32Ordinal;
+
+	return (eError == PVRSRV_OK);
+}
+
+/* Helper macros for deferred events operations */
+#define GET_DE_NEXT_IDX(_curridx) ((_curridx + 1) % HWPERF_HOST_MAX_DEFERRED_PACKETS)
+#define GET_DE_EVENT_BASE(_idx)   (IMG_OFFSET_ADDR(psRgxDevInfo->pui8DeferredEvents, \
+		(_idx) * HWPERF_HOST_DEFERRED_UFO_PACKET_SIZE))
+
+#define GET_DE_EVENT_WRITE_STATUS(_base) ((IMG_BOOL*)((void *)(_base)))
+#define GET_DE_EVENT_DATA(_base)         (IMG_OFFSET_ADDR((_base), sizeof(IMG_BOOL)))
+
+/* Emits HWPerfHost event packets present in the deferred list stopping when one
+ * of the following cases is hit:
+ * case 1: Packet ordering breaks i.e. a packet found doesn't meet ordering
+ *         criteria (ordinal == last_ordinal + 1)
+ *
+ * case 2: A packet with ordinal > ui32MaxOrdinal is found
+ *
+ * case 3: Deferred list's (read == write) i.e. no more deferred packets.
+ *
+ * NOTE: Caller must possess the hLockHWPerfHostStream lock before calling
+ *       this function.*/
+static void _HWPerfHostDeferredEventsEmitter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                             IMG_UINT32 ui32MaxOrdinal)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader;
+	IMG_UINT32 ui32Retry;
+	IMG_UINT8  *pui8DeferredEvent;
+	IMG_BOOL   *pbPacketWritten;
+	IMG_BOOL   bWritePassed;
+
+	PVR_ASSERT(OSLockIsLocked(psRgxDevInfo->hLockHWPerfHostStream));
+
+	while (psRgxDevInfo->ui16DEReadIdx != psRgxDevInfo->ui16DEWriteIdx)
+	{
+		pui8DeferredEvent = GET_DE_EVENT_BASE(psRgxDevInfo->ui16DEReadIdx);
+		pbPacketWritten   = GET_DE_EVENT_WRITE_STATUS(pui8DeferredEvent);
+		psHeader          = (RGX_HWPERF_V2_PACKET_HDR*) GET_DE_EVENT_DATA(pui8DeferredEvent);
+
+		for (ui32Retry = MAX_RETRY_COUNT; !(*pbPacketWritten) && (ui32Retry != 0); ui32Retry--)
+		{
+			/* Packet not yet written, re-check after a while. Wait for a short period as
+			 * atomic contexts are generally expected to finish fast */
+			OSWaitus(10);
+		}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+		if ((ui32Retry == 0) && !(psRgxDevInfo->bWarnedAtomicCtxPktLost))
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: Will warn only once. Dropping a deferred packet as atomic context"
+					 " took too long to write it",
+					 __func__));
+			psRgxDevInfo->bWarnedAtomicCtxPktLost = IMG_TRUE;
+		}
+
+		if (psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark < (MAX_RETRY_COUNT - ui32Retry))
+		{
+			psRgxDevInfo->ui32WaitForAtomicCtxPktHighWatermark = MAX_RETRY_COUNT - ui32Retry;
+		}
+#endif
+
+		if (*pbPacketWritten)
+		{
+			if ((psHeader->ui32Ordinal > ui32MaxOrdinal) ||
+					(psHeader->ui32Ordinal != (psRgxDevInfo->ui32HWPerfHostLastOrdinal + 1)))
+			{
+				/* Leave remaining events to be emitted by next call to this function */
+				break;
+			}
+			bWritePassed = _WriteHWPerfStream(psRgxDevInfo, psHeader);
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%s: Atomic context packet lost!", __func__));
+			bWritePassed = IMG_FALSE;
+		}
+
+		/* Move on to next packet */
+		psRgxDevInfo->ui16DEReadIdx = GET_DE_NEXT_IDX(psRgxDevInfo->ui16DEReadIdx);
+
+		if (!bWritePassed // if write failed
+				&& ui32MaxOrdinal == IMG_UINT32_MAX // and we are from MISR
+				&& psRgxDevInfo->ui16DEReadIdx != psRgxDevInfo->ui16DEWriteIdx) // and there are more events
+		{
+			/* Stop emitting here and re-schedule MISR */
+			OSScheduleMISR(psRgxDevInfo->pvHostHWPerfMISR);
+			break;
+		}
+	}
+}
+
+static void RGX_MISRHandler_HWPerfPostDeferredHostEvents(void *pvData)
+{
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo = pvData;
+
+	OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+
+	/* Since we're called from MISR, there is no upper cap of ordinal to be emitted.
+	 * Send IMG_UINT32_MAX to signify all possible packets. */
+	_HWPerfHostDeferredEventsEmitter(psRgxDevInfo, IMG_UINT32_MAX);
+
+	OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+}
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+static inline void _UpdateDEBufferHighWatermark(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	IMG_UINT32 ui32DEWatermark;
+	IMG_UINT16 ui16LRead = psRgxDevInfo->ui16DEReadIdx;
+	IMG_UINT16 ui16LWrite = psRgxDevInfo->ui16DEWriteIdx;
+
+	if (ui16LWrite >= ui16LRead)
+	{
+		ui32DEWatermark = ui16LWrite - ui16LRead;
+	}
+	else
+	{
+		ui32DEWatermark = (HWPERF_HOST_MAX_DEFERRED_PACKETS - ui16LRead) + (ui16LWrite);
+	}
+
+	if (ui32DEWatermark > psRgxDevInfo->ui32DEHighWatermark)
+	{
+		psRgxDevInfo->ui32DEHighWatermark = ui32DEWatermark;
+	}
+}
+#endif
+
+/* @Description Gets the data/members that concerns the accuracy of a packet in HWPerfHost
+                buffer. Since the data returned by this function is required in both, an
+                atomic as well as a process/sleepable context, it is protected under spinlock
+
+   @Output      pui32Ordinal Pointer to ordinal number assigned to this packet
+   @Output      pui64Timestamp Timestamp value for this packet
+   @Output      ppui8Dest If the current context cannot sleep, pointer to a place in
+                          deferred events buffer where the packet data should be written.
+                          Don't care, otherwise.
+ */
+static void _GetHWPerfHostPacketSpecifics(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                          IMG_UINT32 *pui32Ordinal,
+                                          IMG_UINT64 *pui64Timestamp,
+                                          IMG_UINT8 **ppui8Dest,
+                                          IMG_BOOL    bSleepAllowed)
+{
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* Spin lock is required to avoid getting scheduled out by a higher priority
+	 * context while we're getting header specific details and packet place in
+	 * HWPerf buffer (when in atomic context) for ourselves */
+	OSSpinLockAcquire(psRgxDevInfo->hHWPerfHostSpinLock, uiFlags);
+
+	*pui32Ordinal = psRgxDevInfo->ui32HWPerfHostNextOrdinal++;
+	*pui64Timestamp = RGXTimeCorrGetClockus64(psRgxDevInfo->psDeviceNode);
+
+	if (!bSleepAllowed)
+	{
+		/* We're in an atomic context. So return the next position available in
+		 * deferred events buffer */
+		IMG_UINT16 ui16NewWriteIdx;
+		IMG_BOOL *pbPacketWritten;
+
+		PVR_ASSERT(ppui8Dest != NULL);
+
+		ui16NewWriteIdx = GET_DE_NEXT_IDX(psRgxDevInfo->ui16DEWriteIdx);
+		if (ui16NewWriteIdx == psRgxDevInfo->ui16DEReadIdx)
+		{
+			/* This shouldn't happen. HWPERF_HOST_MAX_DEFERRED_PACKETS should be
+			 * big enough to avoid any such scenario */
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+			/* PVR_LOG/printk isn't recommended in atomic context. Perhaps we'll do
+			 * this debug output here when trace_printk support is added to DDK */
+//			PVR_LOG(("%s: No more space in deferred events buffer (%u/%u) W=%u,R=%u",
+//                     __func__, psRgxDevInfo->ui32DEHighWatermark,
+//					 HWPERF_HOST_MAX_DEFERRED_PACKETS, psRgxDevInfo->ui16DEWriteIdx,
+//					 psRgxDevInfo->ui16DEReadIdx));
+#endif
+			*ppui8Dest = NULL;
+		}
+		else
+		{
+			/* Return the position where deferred event would be written */
+			*ppui8Dest = GET_DE_EVENT_BASE(psRgxDevInfo->ui16DEWriteIdx);
+
+			/* Make sure packet write "state" is "write-pending" _before_ moving write
+			 * pointer forward */
+			pbPacketWritten = GET_DE_EVENT_WRITE_STATUS(*ppui8Dest);
+			*pbPacketWritten = IMG_FALSE;
+
+			psRgxDevInfo->ui16DEWriteIdx = ui16NewWriteIdx;
+
+#if defined(PVRSRV_HWPERF_HOST_DEBUG_DEFERRED_EVENTS)
+			_UpdateDEBufferHighWatermark(psRgxDevInfo);
+#endif
+		}
+	}
+
+	OSSpinLockRelease(psRgxDevInfo->hHWPerfHostSpinLock, uiFlags);
+}
+
+static inline void _SetupHostPacketHeader(IMG_UINT8 *pui8Dest,
+                                          RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+                                          IMG_UINT32 ui32Size,
+                                          IMG_UINT32 ui32Ordinal,
+                                          IMG_UINT64 ui64Timestamp)
+{
+	RGX_HWPERF_V2_PACKET_HDR *psHeader = (RGX_HWPERF_V2_PACKET_HDR *) ((void *)pui8Dest);
+
+	PVR_ASSERT(ui32Size<=RGX_HWPERF_MAX_PACKET_SIZE);
+
+	psHeader->ui32Ordinal = ui32Ordinal;
+	psHeader->ui64Timestamp = ui64Timestamp;
+	psHeader->ui32Sig = HWPERF_PACKET_V2B_SIG;
+	psHeader->eTypeId = RGX_HWPERF_MAKE_TYPEID(RGX_HWPERF_STREAM_ID1_HOST,
+	                                           eEvType, 0, 0, 0);
+	psHeader->ui32Size = ui32Size;
+}
+
+static inline void _SetupHostEnqPacketData(IMG_UINT8 *pui8Dest,
+                                           RGX_HWPERF_KICK_TYPE eEnqType,
+                                           IMG_UINT32 ui32Pid,
+                                           IMG_UINT32 ui32FWDMContext,
+                                           IMG_UINT32 ui32ExtJobRef,
+                                           IMG_UINT32 ui32IntJobRef,
+                                           PVRSRV_FENCE hCheckFence,
+                                           PVRSRV_FENCE hUpdateFence,
+                                           PVRSRV_TIMELINE hUpdateTimeline,
+                                           IMG_UINT64 ui64CheckFenceUID,
+                                           IMG_UINT64 ui64UpdateFenceUID,
+                                           IMG_UINT64 ui64DeadlineInus,
+                                           IMG_UINT32 ui32CycleEstimate)
+{
+	RGX_HWPERF_HOST_ENQ_DATA *psData = (RGX_HWPERF_HOST_ENQ_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->ui32EnqType = eEnqType;
+	psData->ui32PID = ui32Pid;
+	psData->ui32ExtJobRef = ui32ExtJobRef;
+	psData->ui32IntJobRef = ui32IntJobRef;
+	psData->ui32DMContext = ui32FWDMContext;
+	psData->hCheckFence = hCheckFence;
+	psData->hUpdateFence = hUpdateFence;
+	psData->hUpdateTimeline = hUpdateTimeline;
+	psData->ui64CheckFence_UID = ui64CheckFenceUID;
+	psData->ui64UpdateFence_UID = ui64UpdateFenceUID;
+	psData->ui64DeadlineInus = ui64DeadlineInus;
+	psData->ui32CycleEstimate = ui32CycleEstimate;
+}
+
+void RGXHWPerfHostPostRaw(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+						  RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+						  IMG_BYTE *pbPayload,
+						  IMG_UINT32 ui32PayloadSize)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32PktSize;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	PVR_ASSERT(ui32PayloadSize <= RGX_HWPERF_MAX_PAYLOAD_SIZE);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32PktSize = RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32PayloadSize);
+	pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32PktSize);
+
+	if (pui8Dest == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, eEvType, ui32PktSize, ui32Ordinal, ui64Timestamp);
+	OSDeviceMemCopy((IMG_UINT8*)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR)), pbPayload, ui32PayloadSize);
+	_CommitHWPerfStream(psRgxDevInfo, ui32PktSize);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+void RGXHWPerfHostPostEnqEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_KICK_TYPE eEnqType,
+                               IMG_UINT32 ui32Pid,
+                               IMG_UINT32 ui32FWDMContext,
+                               IMG_UINT32 ui32ExtJobRef,
+                               IMG_UINT32 ui32IntJobRef,
+                               PVRSRV_FENCE hCheckFence,
+                               PVRSRV_FENCE hUpdateFence,
+                               PVRSRV_TIMELINE hUpdateTimeline,
+                               IMG_UINT64 ui64CheckFenceUID,
+                               IMG_UINT64 ui64UpdateFenceUID,
+                               IMG_UINT64 ui64DeadlineInus,
+                               IMG_UINT32 ui32CycleEstimate )
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_ENQ_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_ENQ, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostEnqPacketData(pui8Dest,
+	                        eEnqType,
+	                        ui32Pid,
+	                        ui32FWDMContext,
+	                        ui32ExtJobRef,
+	                        ui32IntJobRef,
+	                        hCheckFence,
+	                        hUpdateFence,
+	                        hUpdateTimeline,
+	                        ui64CheckFenceUID,
+	                        ui64UpdateFenceUID,
+	                        ui64DeadlineInus,
+	                        ui32CycleEstimate);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _CalculateHostUfoPacketSize(RGX_HWPERF_UFO_EV eUfoType)
+{
+	IMG_UINT32 ui32Size =
+			(IMG_UINT32) offsetof(RGX_HWPERF_HOST_UFO_DATA, aui32StreamData);
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData;
+
+	switch (eUfoType)
+	{
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			ui32Size += sizeof(puData->sCheckSuccess);
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			ui32Size += sizeof(puData->sCheckFail);
+			break;
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			ui32Size += sizeof(puData->sUpdate);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostUfoEvent: Invalid UFO"
+					" event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostUfoPacketData(IMG_UINT8 *pui8Dest,
+                                           RGX_HWPERF_UFO_EV eUfoType,
+                                           RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData)
+{
+	RGX_HWPERF_HOST_UFO_DATA *psData = (RGX_HWPERF_HOST_UFO_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	RGX_HWPERF_UFO_DATA_ELEMENT *puData = (RGX_HWPERF_UFO_DATA_ELEMENT *)
+					 psData->aui32StreamData;
+
+	psData->eEvType = eUfoType;
+	/* HWPerfHost always emits 1 UFO at a time, since each UFO has 1-to-1 mapping
+	 * with an underlying DevNode, and each DevNode has a dedicated HWPerf buffer */
+	psData->ui32StreamInfo = RGX_HWPERF_MAKE_UFOPKTINFO(1,
+	                                                    offsetof(RGX_HWPERF_HOST_UFO_DATA, aui32StreamData));
+
+	switch (eUfoType)
+	{
+		case RGX_HWPERF_UFO_EV_CHECK_SUCCESS:
+		case RGX_HWPERF_UFO_EV_PRCHECK_SUCCESS:
+			puData->sCheckSuccess.ui32FWAddr =
+					psUFOData->sCheckSuccess.ui32FWAddr;
+			puData->sCheckSuccess.ui32Value =
+					psUFOData->sCheckSuccess.ui32Value;
+			break;
+		case RGX_HWPERF_UFO_EV_CHECK_FAIL:
+		case RGX_HWPERF_UFO_EV_PRCHECK_FAIL:
+			puData->sCheckFail.ui32FWAddr =
+					psUFOData->sCheckFail.ui32FWAddr;
+			puData->sCheckFail.ui32Value =
+					psUFOData->sCheckFail.ui32Value;
+			puData->sCheckFail.ui32Required =
+					psUFOData->sCheckFail.ui32Required;
+			break;
+		case RGX_HWPERF_UFO_EV_UPDATE:
+			puData->sUpdate.ui32FWAddr =
+					psUFOData->sUpdate.ui32FWAddr;
+			puData->sUpdate.ui32OldValue =
+					psUFOData->sUpdate.ui32OldValue;
+			puData->sUpdate.ui32NewValue =
+					psUFOData->sUpdate.ui32NewValue;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostUfoEvent: Invalid UFO"
+					" event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+void RGXHWPerfHostPostUfoEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_UFO_EV eUfoType,
+                               RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData,
+                               const IMG_BOOL bSleepAllowed)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = _CalculateHostUfoPacketSize(eUfoType);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_BOOL   *pbPacketWritten = NULL;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              &pui8Dest, bSleepAllowed);
+
+	if (bSleepAllowed)
+	{
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+		{
+			goto cleanup;
+		}
+	}
+	else
+	{
+		if (pui8Dest == NULL)
+		{
+			// Give-up if we couldn't get a place in deferred events buffer
+			goto cleanup;
+		}
+		pbPacketWritten = GET_DE_EVENT_WRITE_STATUS(pui8Dest);
+		pui8Dest = GET_DE_EVENT_DATA(pui8Dest);
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_UFO, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostUfoPacketData(pui8Dest, eUfoType, psUFOData);
+
+	if (bSleepAllowed)
+	{
+		_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+	}
+	else
+	{
+		*pbPacketWritten = IMG_TRUE;
+		OSScheduleMISR(psRgxDevInfo->pvHostHWPerfMISR);
+	}
+
+cleanup:
+	if (bSleepAllowed)
+	{
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+	}
+}
+
+#define UNKNOWN_SYNC_NAME "UnknownSync"
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH==PVRSRV_SYNC_NAME_LENGTH, "Sync class name max does not match Fence Sync name max");
+
+static inline IMG_UINT32 _FixNameAndCalculateHostAllocPacketSize(
+		RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+		const IMG_CHAR **ppsName,
+		IMG_UINT32 *ui32NameSize)
+{
+	RGX_HWPERF_HOST_ALLOC_DATA *psData;
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_ALLOC_DATA, uAllocDetail);
+
+	if (*ppsName != NULL && *ui32NameSize > 0)
+	{
+		/* if string longer than maximum cut it (leave space for '\0') */
+		if ((*ui32NameSize +1U) >= PVRSRV_SYNC_NAME_LENGTH)
+			*ui32NameSize = PVRSRV_SYNC_NAME_LENGTH;
+		else
+			*ui32NameSize += 1U;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfHostPostAllocEvent: Invalid"
+				" resource name given."));
+		*ppsName = UNKNOWN_SYNC_NAME;
+		*ui32NameSize = sizeof(UNKNOWN_SYNC_NAME);
+	}
+
+	switch (eAllocType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			ui32Size += sizeof(psData->uAllocDetail.sSyncAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			ui32Size += sizeof(psData->uAllocDetail.sFenceAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW:
+			ui32Size += sizeof(psData->uAllocDetail.sSWFenceAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			ui32Size += sizeof(psData->uAllocDetail.sSyncCheckPointAlloc) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostAllocEvent: Invalid alloc event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostAllocPacketData(IMG_UINT8 *pui8Dest,
+                                             RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                             RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail,
+                                             const IMG_CHAR *psName,
+                                             IMG_UINT32 ui32NameSize)
+{
+	RGX_HWPERF_HOST_ALLOC_DATA *psData = (RGX_HWPERF_HOST_ALLOC_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	IMG_CHAR *acName = NULL;
+
+	psData->ui32AllocType = eAllocType;
+
+	switch (eAllocType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			psData->uAllocDetail.sSyncAlloc.ui32FWAddr = puAllocDetail->sSyncAlloc.ui32FWAddr;
+			acName = psData->uAllocDetail.sSyncAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uAllocDetail.sFenceAlloc.uiPID = puAllocDetail->sFenceAlloc.uiPID;
+			psData->uAllocDetail.sFenceAlloc.hFence = puAllocDetail->sFenceAlloc.hFence;
+			psData->uAllocDetail.sFenceAlloc.ui32CheckPt_FWAddr = puAllocDetail->sFenceAlloc.ui32CheckPt_FWAddr;
+			acName = psData->uAllocDetail.sFenceAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW:
+			psData->uAllocDetail.sSWFenceAlloc.uiPID = puAllocDetail->sSWFenceAlloc.uiPID;
+			psData->uAllocDetail.sSWFenceAlloc.hSWFence = puAllocDetail->sSWFenceAlloc.hSWFence;
+			psData->uAllocDetail.sSWFenceAlloc.hSWTimeline = puAllocDetail->sSWFenceAlloc.hSWTimeline;
+			psData->uAllocDetail.sSWFenceAlloc.ui64SyncPtIndex = puAllocDetail->sSWFenceAlloc.ui64SyncPtIndex;
+			acName = psData->uAllocDetail.sSWFenceAlloc.acName;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			psData->uAllocDetail.sSyncCheckPointAlloc.ui32CheckPt_FWAddr = puAllocDetail->sSyncCheckPointAlloc.ui32CheckPt_FWAddr;
+			psData->uAllocDetail.sSyncCheckPointAlloc.hTimeline = puAllocDetail->sSyncCheckPointAlloc.hTimeline;
+			psData->uAllocDetail.sSyncCheckPointAlloc.uiPID = puAllocDetail->sSyncCheckPointAlloc.uiPID;
+			psData->uAllocDetail.sSyncCheckPointAlloc.hFence = puAllocDetail->sSyncCheckPointAlloc.hFence;
+			acName = psData->uAllocDetail.sSyncCheckPointAlloc.acName;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostAllocEvent: Invalid alloc event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+
+
+	if (acName != NULL)
+	{
+		if (ui32NameSize)
+		{
+			OSStringSafeCopy(acName, psName, ui32NameSize);
+		}
+		else
+		{
+			/* In case no name was given make sure we don't access random
+			 * memory */
+			acName[0] = '\0';
+		}
+	}
+}
+
+void RGXHWPerfHostPostAllocEvent(PVRSRV_RGXDEV_INFO* psRgxDevInfo,
+                                 RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                 const IMG_CHAR *psName,
+                                 IMG_UINT32 ui32NameSize,
+                                 RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT32 ui32Size = _FixNameAndCalculateHostAllocPacketSize(eAllocType,
+	                                                              &psName,
+	                                                              &ui32NameSize);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_ALLOC, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+
+	_SetupHostAllocPacketData(pui8Dest,
+	                          eAllocType,
+	                          puAllocDetail,
+	                          psName,
+	                          ui32NameSize);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostFreePacketData(IMG_UINT8 *pui8Dest,
+                                            RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                            IMG_UINT64 ui64UID,
+                                            IMG_UINT32 ui32FWAddr)
+{
+	RGX_HWPERF_HOST_FREE_DATA *psData = (RGX_HWPERF_HOST_FREE_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->ui32FreeType = eFreeType;
+
+	switch (eFreeType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC:
+			psData->uFreeDetail.sSyncFree.ui32FWAddr = ui32FWAddr;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uFreeDetail.sFenceDestroy.ui64Fence_UID = ui64UID;
+			break;
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP:
+			psData->uFreeDetail.sSyncCheckPointFree.ui32CheckPt_FWAddr = ui32FWAddr;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostFreeEvent: Invalid free event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+}
+
+void RGXHWPerfHostPostFreeEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                IMG_UINT64 ui64UID,
+                                IMG_UINT32 ui32PID,
+                                IMG_UINT32 ui32FWAddr)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size = RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_FREE_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	PVR_UNREFERENCED_PARAMETER(ui32PID);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_FREE, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostFreePacketData(pui8Dest,
+	                         eFreeType,
+	                         ui64UID,
+	                         ui32FWAddr);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _FixNameAndCalculateHostModifyPacketSize(
+		RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+		const IMG_CHAR **ppsName,
+		IMG_UINT32 *ui32NameSize)
+{
+	RGX_HWPERF_HOST_MODIFY_DATA *psData;
+	RGX_HWPERF_HOST_MODIFY_DETAIL *puData;
+	IMG_UINT32 ui32Size = sizeof(psData->ui32ModifyType);
+
+	if (*ppsName != NULL && *ui32NameSize > 0)
+	{
+		/* first strip the terminator */
+		if ((*ppsName)[*ui32NameSize - 1] == '\0')
+			*ui32NameSize -= 1;
+		/* if string longer than maximum cut it (leave space for '\0') */
+		if (*ui32NameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			*ui32NameSize = PVRSRV_SYNC_NAME_LENGTH - 1;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfHostPostModifyEvent: Invalid"
+				" resource name given."));
+		*ppsName = UNKNOWN_SYNC_NAME;
+		*ui32NameSize = sizeof(UNKNOWN_SYNC_NAME) - 1;
+	}
+
+	switch (eModifyType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			ui32Size += sizeof(puData->sFenceMerge) - PVRSRV_SYNC_NAME_LENGTH +
+			*ui32NameSize + 1; /* +1 for '\0' */
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostModifyEvent: Invalid modify event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void _SetupHostModifyPacketData(IMG_UINT8 *pui8Dest,
+                                              RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                              IMG_UINT64 ui64NewUID,
+                                              IMG_UINT64 ui64UID1,
+                                              IMG_UINT64 ui64UID2,
+                                              const IMG_CHAR *psName,
+                                              IMG_UINT32 ui32NameSize)
+{
+	RGX_HWPERF_HOST_MODIFY_DATA *psData = (RGX_HWPERF_HOST_MODIFY_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	IMG_CHAR *acName = NULL;
+
+	psData->ui32ModifyType = eModifyType;
+
+	switch (eModifyType)
+	{
+		case RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR:
+			psData->uModifyDetail.sFenceMerge.ui64NewFence_UID = ui64NewUID;
+			psData->uModifyDetail.sFenceMerge.ui64InFence1_UID = ui64UID1;
+			psData->uModifyDetail.sFenceMerge.ui64InFence2_UID = ui64UID2;
+			acName = psData->uModifyDetail.sFenceMerge.acName;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"RGXHWPerfHostPostModifyEvent: Invalid modify event type"));
+			PVR_ASSERT(IMG_FALSE);
+	}
+
+	if (acName != NULL)
+	{
+		if (ui32NameSize)
+		{
+			OSStringSafeCopy(acName, psName, ui32NameSize);
+		}
+		else
+		{
+			/* In case no name was given make sure we don't access random
+			 * memory */
+			acName[0] = '\0';
+		}
+	}
+}
+
+void RGXHWPerfHostPostModifyEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                  RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                  IMG_UINT64 ui64NewUID,
+                                  IMG_UINT64 ui64UID1,
+                                  IMG_UINT64 ui64UID2,
+                                  const IMG_CHAR *psName,
+                                  IMG_UINT32 ui32NameSize)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT32 ui32Size = _FixNameAndCalculateHostModifyPacketSize(eModifyType,
+	                                                               &psName,
+	                                                               &ui32NameSize);
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_MODIFY, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostModifyPacketData(pui8Dest,
+	                           eModifyType,
+	                           ui64NewUID,
+	                           ui64UID1,
+	                           ui64UID2,
+	                           psName,
+	                           ui32NameSize);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostClkSyncPacketData(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT8 *pui8Dest)
+{
+	RGX_HWPERF_HOST_CLK_SYNC_DATA *psData = (RGX_HWPERF_HOST_CLK_SYNC_DATA *)
+					IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psRgxDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32CurrIdx;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32CurrIdx = RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount);
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32CurrIdx], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32CurrIdx];
+
+	psData->ui64CRTimestamp = psTimeCorr->ui64CRTimeStamp;
+	psData->ui64OSTimestamp = psTimeCorr->ui64OSTimeStamp;
+	psData->ui32ClockSpeed = psTimeCorr->ui32CoreClockSpeed;
+}
+
+void RGXHWPerfHostPostClkSyncEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size =
+			RGX_HWPERF_MAKE_SIZE_FIXED(RGX_HWPERF_HOST_CLK_SYNC_DATA);
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	/* if the buffer for time correlation data is not yet available (possibly
+	 * device not initialised yet) skip this event */
+	if (psRgxDevInfo->psRGXFWIfGpuUtilFW == NULL)
+	{
+		return;
+	}
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_CLK_SYNC, ui32Size,
+	                       ui32Ordinal, ui64Timestamp);
+	_SetupHostClkSyncPacketData(psRgxDevInfo, pui8Dest);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline void _SetupHostDeviceInfoPacketData(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                                  RGX_HWPERF_DEV_INFO_EV eEvType,
+                                                  RGX_HWPERF_HOST_DEV_INFO_DETAIL *puPacketData,
+                                                  IMG_UINT8 *pui8Dest)
+{
+	RGX_HWPERF_HOST_DEV_INFO_DATA *psData = (RGX_HWPERF_HOST_DEV_INFO_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->eEvType = eEvType;
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_DEV_INFO_EV_HEALTH:
+			if (puPacketData != NULL)
+			{
+				psData->uDevInfoDetail.sDeviceStatus.eDeviceHealthStatus =
+					puPacketData->sDeviceStatus.eDeviceHealthStatus;
+				psData->uDevInfoDetail.sDeviceStatus.eDeviceHealthReason =
+					puPacketData->sDeviceStatus.eDeviceHealthReason;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: puPacketData is invalid."));
+			}
+			break;
+		case RGX_HWPERF_DEV_INFO_EV_FEATURES:
+			{
+				PVRSRV_ERROR eError;
+				eError = RGXServerFeatureFlagsToHWPerfFlags(psRgxDevInfo,
+				                                            &psData->uDevInfoDetail.sBVNC);
+				PVR_LOG_IF_ERROR(eError, "RGXServerFeatureFlagsToHWPerfFlags");
+				psData->uDevInfoDetail.sBVNC.ui32BvncKmFeatureFlags |=
+#if defined(RGX_FEATURE_HWPERF_ROGUE)
+					RGX_HWPERF_FEATURE_ROGUE_FLAG;
+#elif defined(RGX_FEATURE_HWPERF_VOLCANIC)
+					RGX_HWPERF_FEATURE_VOLCANIC_FLAG;
+#else
+					0x0;
+#endif
+			}
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+static inline IMG_UINT32 _CalculateHostDeviceInfoPacketSize(RGX_HWPERF_DEV_INFO_EV eEvType)
+{
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_DEV_INFO_DATA, uDevInfoDetail);
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_DEV_INFO_EV_HEALTH:
+			ui32Size += sizeof(((RGX_HWPERF_HOST_DEV_INFO_DATA*)0)->uDevInfoDetail.sDeviceStatus);
+			break;
+		case RGX_HWPERF_DEV_INFO_EV_FEATURES:
+			ui32Size += sizeof(((RGX_HWPERF_HOST_DEV_INFO_DATA*)0)->uDevInfoDetail.sBVNC);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostDeviceInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+void RGXHWPerfHostPostDeviceInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 RGX_HWPERF_DEV_INFO_EV eEvType,
+                                 RGX_HWPERF_HOST_DEV_INFO_DETAIL *puData)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT32 ui32Size;
+
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->hHWPerfHostStream != (IMG_HANDLE) NULL)
+	{
+		_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+		ui32Size = _CalculateHostDeviceInfoPacketSize(eEvType);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) != NULL)
+		{
+			_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_DEV_INFO, ui32Size, ui32Ordinal, ui64Timestamp);
+			_SetupHostDeviceInfoPacketData(psRgxDevInfo, eEvType, puData, pui8Dest);
+			_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+		}
+
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+}
+
+static inline void _SetupHostInfoPacketData(RGX_HWPERF_INFO_EV eEvType,
+												  IMG_UINT64 ui64TotalMemoryUsage,
+												  IMG_UINT32 ui32LivePids,
+												  PVRSRV_PER_PROCESS_MEM_USAGE *psPerProcessMemUsage,
+												  IMG_UINT8 *pui8Dest)
+{
+	IMG_INT i;
+	RGX_HWPERF_HOST_INFO_DATA *psData = (RGX_HWPERF_HOST_INFO_DATA *)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psData->eEvType = eEvType;
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_INFO_EV_MEM64_USAGE:
+			psData->uInfoDetail.sMemUsageStats.ui64TotalMemoryUsage = ui64TotalMemoryUsage;
+
+			if (psPerProcessMemUsage)
+			{
+				for (i = 0; i < ui32LivePids; ++i)
+				{
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui32Pid = psPerProcessMemUsage[i].ui32Pid;
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui64KernelMemUsage = psPerProcessMemUsage[i].ui64KernelMemUsage;
+					psData->uInfoDetail.sMemUsageStats.sPerProcessUsage[i].ui64GraphicsMemUsage = psPerProcessMemUsage[i].ui64GraphicsMemUsage;
+				}
+			}
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+}
+
+static inline IMG_UINT32 _CalculateHostInfoPacketSize(RGX_HWPERF_INFO_EV eEvType,
+															IMG_UINT64 *pui64TotalMemoryUsage,
+															IMG_UINT32 *pui32LivePids,
+															PVRSRV_PER_PROCESS_MEM_USAGE **ppsPerProcessMemUsage)
+{
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_INFO_DATA, uInfoDetail);
+
+	switch (eEvType)
+	{
+		case RGX_HWPERF_INFO_EV_MEM64_USAGE:
+#if !defined(__QNXNTO__)
+			if (PVRSRVGetProcessMemUsage(pui64TotalMemoryUsage, pui32LivePids, ppsPerProcessMemUsage) == PVRSRV_OK)
+			{
+				ui32Size += offsetof(RGX_HWPERF_HOST_INFO_DETAIL, sMemUsageStats.sPerProcessUsage)
+					+ ((*pui32LivePids) * sizeof(struct _RGX_HWPERF_HOST_INFO_PER_PROC_USAGE_));
+			}
+#else
+			PVR_DPF((PVR_DBG_ERROR, "This functionality is not yet implemented for this platform"));
+#endif
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfHostPostInfo: Invalid event type"));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+void RGXHWPerfHostPostInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_INFO_EV eEvType)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+	IMG_UINT64 ui64TotalMemoryUsage = 0;
+	PVRSRV_PER_PROCESS_MEM_USAGE *psPerProcessMemUsage = NULL;
+	IMG_UINT32 ui32LivePids = 0;
+
+	OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+
+	if (psRgxDevInfo->hHWPerfHostStream != (IMG_HANDLE) NULL)
+	{
+		_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+		_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+		ui32Size = _CalculateHostInfoPacketSize(eEvType, &ui64TotalMemoryUsage, &ui32LivePids, &psPerProcessMemUsage);
+
+		if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) != NULL)
+		{
+			_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_INFO, ui32Size, ui32Ordinal, ui64Timestamp);
+			_SetupHostInfoPacketData(eEvType, ui64TotalMemoryUsage, ui32LivePids, psPerProcessMemUsage, pui8Dest);
+			_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+		}
+
+		_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+
+		if (psPerProcessMemUsage)
+			OSFreeMemNoStats(psPerProcessMemUsage); // psPerProcessMemUsage was allocated with OSAllocZMemNoStats
+	}
+
+	OSLockRelease(psRgxDevInfo->hHWPerfLock);
+}
+
+static inline IMG_UINT32
+_CalculateHostFenceWaitPacketSize(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eWaitType)
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *psSizeCalculator;
+	IMG_UINT32 ui32Size = offsetof(RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA, uDetail);
+
+	switch (eWaitType)
+	{
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN:
+			ui32Size += sizeof(psSizeCalculator->uDetail.sBegin);
+			break;
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END:
+			ui32Size += sizeof(psSizeCalculator->uDetail.sEnd);
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid wait event type (%u)", __func__,
+			         eWaitType));
+			PVR_ASSERT(IMG_FALSE);
+			break;
+	}
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void
+_SetupHostFenceWaitPacketData(IMG_UINT8 *pui8Dest,
+                              RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eWaitType,
+                              IMG_PID uiPID,
+                              PVRSRV_FENCE hFence,
+                              IMG_UINT32 ui32Data)
+{
+	RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *psData = (RGX_HWPERF_HOST_SYNC_FENCE_WAIT_DATA *)
+	                IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->eType = eWaitType;
+	psData->uiPID = uiPID;
+	psData->hFence = hFence;
+
+	switch (eWaitType)
+	{
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_BEGIN:
+			psData->uDetail.sBegin.ui32TimeoutInMs = ui32Data;
+			break;
+		case RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_END:
+			psData->uDetail.sEnd.eResult =
+			    (RGX_HWPERF_HOST_SYNC_FENCE_WAIT_RESULT) ui32Data;
+			break;
+		default:
+			// unknown type - this should never happen
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid fence-wait event type", __func__));
+			PVR_ASSERT(IMG_FALSE);
+	}
+}
+
+void RGXHWPerfHostPostFenceWait(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType,
+								IMG_PID uiPID,
+								PVRSRV_FENCE hFence,
+								IMG_UINT32 ui32Data)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32Size = _CalculateHostFenceWaitPacketSize(eType);
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_SYNC_FENCE_WAIT,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+	_SetupHostFenceWaitPacketData(pui8Dest, eType, uiPID, hFence, ui32Data);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+static inline IMG_UINT32 _CalculateHostSWTimelineAdvPacketSize(void)
+{
+	IMG_UINT32 ui32Size = sizeof(RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA);
+	return RGX_HWPERF_MAKE_SIZE_VARIABLE(ui32Size);
+}
+
+static inline void
+_SetupHostSWTimelineAdvPacketData(IMG_UINT8 *pui8Dest,
+                                  IMG_PID uiPID,
+								  PVRSRV_TIMELINE hSWTimeline,
+								  IMG_UINT64 ui64SyncPtIndex)
+
+{
+	RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA *psData = (RGX_HWPERF_HOST_SYNC_SW_TL_ADV_DATA *)
+	                IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+
+	psData->uiPID = uiPID;
+	psData->hTimeline = hSWTimeline;
+	psData->ui64SyncPtIndex = ui64SyncPtIndex;
+}
+
+void RGXHWPerfHostPostSWTimelineAdv(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                    IMG_PID uiPID,
+                                    PVRSRV_TIMELINE hSWTimeline,
+                                    IMG_UINT64 ui64SyncPtIndex)
+{
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp,
+	                              NULL, IMG_TRUE);
+
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32Size = _CalculateHostSWTimelineAdvPacketSize();
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+	_SetupHostSWTimelineAdvPacketData(pui8Dest, uiPID, hSWTimeline, ui64SyncPtIndex);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+
+}
+
+void RGXHWPerfHostPostClientInfoProcName(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_PID uiPID,
+									     const IMG_CHAR *psName)
+{
+	RGX_HWPERF_HOST_CLIENT_INFO_DATA* psPkt;
+	IMG_UINT8 *pui8Dest;
+	IMG_UINT32 ui32Size;
+	IMG_UINT32 ui32NameLen;
+	IMG_UINT32 ui32Ordinal;
+	IMG_UINT64 ui64Timestamp;
+
+	_GetHWPerfHostPacketSpecifics(psRgxDevInfo, &ui32Ordinal, &ui64Timestamp, NULL, IMG_TRUE);
+	_PostFunctionPrologue(psRgxDevInfo, ui32Ordinal);
+
+	ui32NameLen = OSStringLength(psName) + 1U;
+	ui32Size = RGX_HWPERF_MAKE_SIZE_VARIABLE(RGX_HWPERF_HOST_CLIENT_INFO_PROC_NAME_BASE_SIZE
+		+ RGX_HWPERF_HOST_CLIENT_PROC_NAME_SIZE(ui32NameLen));
+
+	if ((pui8Dest = _ReserveHWPerfStream(psRgxDevInfo, ui32Size)) == NULL)
+	{
+		goto cleanup;
+	}
+
+	_SetupHostPacketHeader(pui8Dest, RGX_HWPERF_HOST_CLIENT_INFO,
+	                       ui32Size, ui32Ordinal, ui64Timestamp);
+
+	psPkt = (RGX_HWPERF_HOST_CLIENT_INFO_DATA*)IMG_OFFSET_ADDR(pui8Dest, sizeof(RGX_HWPERF_V2_PACKET_HDR));
+	psPkt->eType = RGX_HWPERF_HOST_CLIENT_INFO_TYPE_PROCESS_NAME;
+	psPkt->uDetail.sProcName.ui32Count = 1U;
+	psPkt->uDetail.sProcName.asProcNames[0].uiClientPID = uiPID;
+	psPkt->uDetail.sProcName.asProcNames[0].ui32Length = ui32NameLen;
+	(void)OSCachedMemCopy(psPkt->uDetail.sProcName.asProcNames[0].acName, psName, ui32NameLen);
+
+	_CommitHWPerfStream(psRgxDevInfo, ui32Size);
+
+cleanup:
+	_PostFunctionEpilogue(psRgxDevInfo, ui32Ordinal);
+}
+
+/******************************************************************************
+ * Currently only implemented on Linux. Feature can be enabled to provide
+ * an interface to 3rd-party kernel modules that wish to access the
+ * HWPerf data. The API is documented in the rgxapi_km.h header and
+ * the rgx_hwperf* headers.
+ *****************************************************************************/
+
+/* Internal HWPerf kernel connection/device data object to track the state
+ * of a client session.
+ */
+typedef struct
+{
+	PVRSRV_DEVICE_NODE* psRgxDevNode;
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo;
+
+	/* TL Open/close state */
+	IMG_HANDLE          hSD[RGX_HWPERF_MAX_STREAM_ID];
+
+	/* TL Acquire/release state */
+	IMG_PBYTE			pHwpBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer returned to user in acquire call */
+	IMG_PBYTE			pHwpBufEnd[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to end of HwpBuf */
+	IMG_PBYTE			pTlBuf[RGX_HWPERF_MAX_STREAM_ID];			/*!< buffer obtained via TlAcquireData */
+	IMG_PBYTE			pTlBufPos[RGX_HWPERF_MAX_STREAM_ID];		/*!< initial position in TlBuf to acquire packets */
+	IMG_PBYTE			pTlBufRead[RGX_HWPERF_MAX_STREAM_ID];		/*!< pointer to the last packet read */
+	IMG_UINT32			ui32AcqDataLen[RGX_HWPERF_MAX_STREAM_ID];	/*!< length of acquired TlBuf */
+	IMG_BOOL			bRelease[RGX_HWPERF_MAX_STREAM_ID];		/*!< used to determine whether or not to release currently held TlBuf */
+
+
+} RGX_KM_HWPERF_DEVDATA;
+
+PVRSRV_ERROR RGXHWPerfLazyConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGX_KM_HWPERF_DEVDATA *psDevData;
+	RGX_HWPERF_DEVICE *psNewHWPerfDevice;
+	RGX_HWPERF_CONNECTION* psHWPerfConnection;
+	IMG_BOOL bFWActive = IMG_FALSE;
+
+	/* avoid uninitialised data */
+	PVR_ASSERT(*ppsHWPerfConnection == NULL);
+	PVR_ASSERT(psPVRSRVData);
+
+	/* Allocate connection object */
+	psHWPerfConnection = OSAllocZMem(sizeof(*psHWPerfConnection));
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	/* early save the return pointer to aid clean-up if failure occurs */
+	*ppsHWPerfConnection = psHWPerfConnection;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	psDeviceNode = psPVRSRVData->psDeviceNodeList;
+
+	while (psDeviceNode)
+	{
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_NOT_IMPLEMENTED;
+		}
+
+		if (psDeviceNode->eDevState != PVRSRV_DEVICE_STATE_ACTIVE)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: HWPerf: Device not currently active. ID:%u",
+					 __func__,
+					 psDeviceNode->sDevId.i32KernelDeviceID));
+			psDeviceNode = psDeviceNode->psNext;
+			continue;
+		}
+		/* Create a list node to be attached to connection object's list */
+		psNewHWPerfDevice = OSAllocMem(sizeof(*psNewHWPerfDevice));
+		if (!psNewHWPerfDevice)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		/* Insert node at head of the list */
+		psNewHWPerfDevice->psNext = psHWPerfConnection->psHWPerfDevList;
+		psHWPerfConnection->psHWPerfDevList = psNewHWPerfDevice;
+
+		/* create a device data object for kernel server */
+		psDevData = OSAllocZMem(sizeof(*psDevData));
+		psNewHWPerfDevice->hDevData = (IMG_HANDLE)psDevData;
+		if (!psDevData)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		if (OSSNPrintf(psNewHWPerfDevice->pszName, sizeof(psNewHWPerfDevice->pszName),
+			           "hwperf_device_%d", psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf device name for device %d",
+					__func__,
+					psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		psDevData->psRgxDevNode = psDeviceNode;
+		psDevData->psRgxDevInfo = psDeviceNode->pvDevice;
+
+		psDeviceNode = psDeviceNode->psNext;
+
+		/* At least one device is active */
+		bFWActive = IMG_TRUE;
+	}
+
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (!bFWActive)
+	{
+		return PVRSRV_ERROR_NOT_READY;
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfOpen(RGX_HWPERF_CONNECTION *psHWPerfConnection)
+{
+	RGX_KM_HWPERF_DEVDATA *psDevData;
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+	PVRSRV_RGXDEV_INFO *psRgxDevInfo;
+	PVRSRV_ERROR eError;
+	IMG_CHAR pszHWPerfFwStreamName[sizeof(PVRSRV_TL_HWPERF_RGX_FW_STREAM) + 5];
+	IMG_CHAR pszHWPerfHostStreamName[sizeof(PVRSRV_TL_HWPERF_HOST_SERVER_STREAM) + 5];
+	IMG_UINT32 ui32BufSize;
+
+	/* Disable producer callback by default for the Kernel API. */
+	IMG_UINT32 ui32StreamFlags = PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING |
+			PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK;
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+		psRgxDevInfo = psDevData->psRgxDevInfo;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* In the case where the AppHint has not been set we need to
+		 * initialise the HWPerf resources here. Allocated on-demand
+		 * to reduce RAM foot print on systems not needing HWPerf.
+		 */
+		OSLockAcquire(psRgxDevInfo->hHWPerfLock);
+		if (RGXHWPerfIsInitRequired(psRgxDevInfo, RGX_HWPERF_L2_STREAM_HWPERF))
+		{
+			eError = RGXHWPerfInitOnDemandL1Buffer(psRgxDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfFW resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hHWPerfLock);
+				return eError;
+			}
+
+			/* if this fails it also cleans up L1 buffer */
+			eError = RGXHWPerfInitOnDemandL2Stream(psRgxDevInfo, RGX_HWPERF_L2_STREAM_HWPERF);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfFW resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hHWPerfLock);
+				return eError;
+			}
+		}
+		OSLockRelease(psRgxDevInfo->hHWPerfLock);
+
+		OSLockAcquire(psRgxDevInfo->hLockHWPerfHostStream);
+		if (psRgxDevInfo->hHWPerfHostStream == NULL)
+		{
+			eError = RGXHWPerfHostInitOnDemandResources(psRgxDevInfo);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						 "%s: Initialisation of on-demand HWPerfHost resources failed",
+						 __func__));
+				OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+				return eError;
+			}
+		}
+		OSLockRelease(psRgxDevInfo->hLockHWPerfHostStream);
+
+		/* form the HWPerf stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfFwStreamName, sizeof(pszHWPerfFwStreamName), "%s%d",
+		               PVRSRV_TL_HWPERF_RGX_FW_STREAM,
+		               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+		/* Open the RGX TL stream for reading in this session */
+		eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+		                            pszHWPerfFwStreamName,
+		                            ui32StreamFlags,
+		                            &psDevData->hSD[RGX_HWPERF_STREAM_ID0_FW]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientOpenStream(RGX_HWPerf)");
+
+		/* form the HWPerf host stream name, corresponding to this DevNode; which can make sense in the UM */
+		if (OSSNPrintf(pszHWPerfHostStreamName, sizeof(pszHWPerfHostStreamName), "%s%d",
+		               PVRSRV_TL_HWPERF_HOST_SERVER_STREAM,
+		               psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID) < 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to form HWPerf host stream name for device %d",
+					__func__,
+					psRgxDevInfo->psDeviceNode->sDevId.i32KernelDeviceID));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Open the host TL stream for reading in this session */
+		eError = TLClientOpenStream(DIRECT_BRIDGE_HANDLE,
+		                            pszHWPerfHostStreamName,
+		                            PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING,
+		                            &psDevData->hSD[RGX_HWPERF_STREAM_ID1_HOST]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientOpenStream(Host_HWPerf)");
+
+		/* Allocate a large enough buffer for use during the entire session to
+		 * avoid the need to resize in the Acquire call as this might be in an ISR
+		 * Choose size that can contain at least one packet.
+		 */
+		/* Allocate buffer for FW Stream */
+		ui32BufSize = FW_STREAM_BUFFER_SIZE;
+		psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW] = OSAllocMem(ui32BufSize);
+		if (psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW] == NULL)
+		{
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		psDevData->pHwpBufEnd[RGX_HWPERF_STREAM_ID0_FW] = psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW]+ui32BufSize;
+
+		/* Allocate buffer for Host Stream */
+		ui32BufSize = HOST_STREAM_BUFFER_SIZE;
+		psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST] = OSAllocMem(ui32BufSize);
+		if (psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST] == NULL)
+		{
+			OSFreeMem(psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID0_FW]);
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		psDevData->pHwpBufEnd[RGX_HWPERF_STREAM_ID1_HOST] = psDevData->pHwpBuf[RGX_HWPERF_STREAM_ID1_HOST]+ui32BufSize;
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfConnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_ERROR eError;
+
+	eError = RGXHWPerfLazyConnect(ppsHWPerfConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfLazyConnect", e0);
+
+	eError = RGXHWPerfOpen(*ppsHWPerfConnection);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfOpen", e1);
+
+	return PVRSRV_OK;
+
+e1: /* HWPerfOpen might have opened some, and then failed */
+	RGXHWPerfClose(*ppsHWPerfConnection);
+e0: /* LazyConnect might have allocated some resources and then failed,
+	 * make sure they are cleaned up */
+	RGXHWPerfFreeConnection(ppsHWPerfConnection);
+	return eError;
+}
+
+/*
+    PVRSRVRGXControlHWPerfBlocksKM
+ */
+PVRSRV_ERROR PVRSRVRGXControlHWPerfBlocksKM(
+		CONNECTION_DATA             * psConnection,
+		PVRSRV_DEVICE_NODE          * psDeviceNode,
+		IMG_BOOL                      bEnable,
+		IMG_UINT32                    ui32ArrayLen,
+		IMG_UINT16                  * psBlockIDs)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sKccbCmd;
+	IMG_UINT32			ui32kCCBCommandSlot;
+	PVRSRV_RGXDEV_INFO	*psDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_ASSERT(psDeviceNode);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	PVR_DPF_ENTERED;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psBlockIDs != NULL, "psBlockIDs");
+	PVR_LOG_RETURN_IF_INVALID_PARAM((ui32ArrayLen>0) && (ui32ArrayLen <= RGXFWIF_HWPERF_CTRL_BLKS_MAX), "ui32ArrayLen");
+
+	psDevice = psDeviceNode->pvDevice;
+
+	/* Fill in the command structure with the parameters needed
+	 */
+	sKccbCmd.eCmdType = RGXFWIF_KCCB_CMD_HWPERF_CTRL_BLKS;
+	sKccbCmd.uCmdData.sHWPerfCtrlBlks.bEnable = bEnable;
+	sKccbCmd.uCmdData.sHWPerfCtrlBlks.ui32NumBlocks = ui32ArrayLen;
+
+	OSDeviceMemCopy(sKccbCmd.uCmdData.sHWPerfCtrlBlks.aeBlockIDs, psBlockIDs, sizeof(IMG_UINT16) * ui32ArrayLen);
+
+
+	/* Ask the FW to carry out the HWPerf configuration command
+	 */
+	eError = RGXScheduleCommandAndGetKCCBSlot(psDevice,
+	                                          RGXFWIF_DM_GP,
+	                                          &sKccbCmd,
+	                                          PDUMP_FLAGS_CONTINUOUS,
+	                                          &ui32kCCBCommandSlot);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXScheduleCommandAndGetKCCBSlot");
+
+	/* Wait for FW to complete */
+	eError = RGXWaitForKCCBSlotUpdate(psDevice, ui32kCCBCommandSlot, PDUMP_FLAGS_CONTINUOUS);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXWaitForKCCBSlotUpdate");
+
+	if (bEnable)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks have been ENABLED", ui32ArrayLen));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "HWPerf %d counter blocks have been DISABLED", ui32ArrayLen));
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+/*
+	PVRSRVRGXCtrlHWPerfKM
+ */
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		RGX_HWPERF_STREAM_ID     eStreamId,
+		IMG_BOOL                 bToggle,
+		IMG_UINT64               ui64Mask)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVR_ASSERT(psDeviceNode);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	PVR_DPF_ENTERED;
+
+	if (eStreamId == RGX_HWPERF_STREAM_ID0_FW)
+	{
+		return RGXHWPerfCtrlFwBuffer(psDeviceNode, RGX_HWPERF_L2_STREAM_HWPERF,
+		                             bToggle, ui64Mask);
+	}
+	else if (eStreamId == RGX_HWPERF_STREAM_ID1_HOST)
+	{
+		return RGXHWPerfCtrlHostBuffer(psDeviceNode, bToggle, (IMG_UINT32) ui64Mask);
+	}
+	else if (eStreamId == RGX_HWPERF_STREAM_ID2_CLIENT)
+	{
+		IMG_UINT32 ui32Index = (IMG_UINT32) (ui64Mask >> 32);
+		IMG_UINT32 ui32Mask = (IMG_UINT32) ui64Mask;
+
+		return RGXHWPerfCtrlClientBuffer(bToggle, ui32Index, ui32Mask);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXCtrlHWPerfKM: Unknown stream id."));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfFW(
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		RGX_HWPERF_L2_STREAM_ID  eL2StreamId,
+		IMG_UINT64               ui64Mask,
+		HWPERF_FILTER_OPERATION  eMaskOp)
+{
+	IMG_UINT64 uiTmpFilter;
+
+	PVR_DPF_ENTERED;
+	PVR_ASSERT(psDeviceNode);
+	PVR_ASSERT(eL2StreamId < RGX_HWPERF_L2_STREAM_LAST);
+
+	uiTmpFilter =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui64HWPerfFilter[eL2StreamId];
+
+	switch (eMaskOp)
+	{
+		case HWPERF_FILTER_OPERATION_SET:
+			uiTmpFilter = ui64Mask;
+			break;
+		case HWPERF_FILTER_OPERATION_BIT_CLR:
+			uiTmpFilter &= ~ui64Mask;
+			break;
+		case HWPERF_FILTER_OPERATION_BIT_OR:
+			uiTmpFilter |= ui64Mask;
+			break;
+	}
+
+	PVR_DPF_RETURN_RC(RGXHWPerfCtrlFwBuffer(psDeviceNode, eL2StreamId, IMG_FALSE,
+	                  uiTmpFilter));
+}
+
+#if defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+/*
+	PVRSRVRGXGetHWPerfTimeStampKM
+ */
+PVRSRV_ERROR PVRSRVRGXGetHWPerfTimeStampKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		IMG_UINT64              *pui64TimeStamp)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	*pui64TimeStamp = RGXTimeCorrGetClockus64(psDeviceNode);
+	return PVRSRV_OK;
+}
+#endif
+
+PVRSRV_ERROR RGXHWPerfControl(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_BOOL             bToggle,
+		IMG_UINT64           ui64Mask)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE* psHWPerfDev;
+
+	/* Validate input argument values supplied by the caller */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXCtrlHWPerfKM(NULL, psDevData->psRgxDevNode, eStreamId, bToggle, ui64Mask);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVRGXCtrlHWPerfKM");
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+IMG_INTERNAL PVRSRV_ERROR RGXHWPerfToggleCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs,
+		IMG_BOOL     bToggle,
+		const char* szFunctionString);
+
+IMG_INTERNAL PVRSRV_ERROR RGXHWPerfToggleCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32   ui32NumBlocks,
+		IMG_UINT16*  aeBlockIDs,
+		IMG_BOOL     bToggle,
+		const char* szFunctionString)
+{
+	PVRSRV_ERROR           eError;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	RGX_HWPERF_DEVICE*     psHWPerfDev;
+
+	if (!psHWPerfConnection || ui32NumBlocks==0 || !aeBlockIDs)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32NumBlocks > RGXFWIF_HWPERF_CTRL_BLKS_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		/* Call the internal server API */
+		eError = PVRSRVRGXControlHWPerfBlocksKM(NULL,
+		                                        psDevData->psRgxDevNode,
+		                                        bToggle,
+		                                        ui32NumBlocks,
+		                                        aeBlockIDs);
+
+		PVR_LOG_RETURN_IF_ERROR(eError, szFunctionString);
+
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXHWPerfDisableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32            ui32NumBlocks,
+		IMG_UINT16*           aeBlockIDs)
+{
+	return RGXHWPerfToggleCounters(psHWPerfConnection,
+			                        ui32NumBlocks,
+			                        aeBlockIDs,
+			                        IMG_FALSE,
+			                        __func__);
+}
+
+
+PVRSRV_ERROR RGXHWPerfEnableCounters(
+		RGX_HWPERF_CONNECTION *psHWPerfConnection,
+		IMG_UINT32            ui32NumBlocks,
+		IMG_UINT16*           aeBlockIDs)
+{
+	return RGXHWPerfToggleCounters(psHWPerfConnection,
+			                        ui32NumBlocks,
+			                        aeBlockIDs,
+			                        IMG_TRUE,
+			                        __func__);
+}
+
+
+PVRSRV_ERROR RGXHWPerfAcquireEvents(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_PBYTE*  ppBuf,
+		IMG_UINT32* pui32BufLen)
+{
+	PVRSRV_ERROR			eError;
+	RGX_KM_HWPERF_DEVDATA*	psDevData = (RGX_KM_HWPERF_DEVDATA*)hDevData;
+	IMG_PBYTE				pDataDest;
+	IMG_UINT32			ui32TlPackets = 0;
+	IMG_PBYTE			pBufferEnd;
+	PVRSRVTL_PPACKETHDR psHDRptr;
+	PVRSRVTL_PACKETTYPE ui16TlType;
+
+	/* Reset the output arguments in case we discover an error */
+	*ppBuf = NULL;
+	*pui32BufLen = 0;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psDevData || eStreamId >= RGX_HWPERF_MAX_STREAM_ID)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psDevData->pTlBuf[eStreamId] == NULL)
+	{
+		/* Acquire some data to read from the HWPerf TL stream */
+		eError = TLClientAcquireData(DIRECT_BRIDGE_HANDLE,
+		                             psDevData->hSD[eStreamId],
+		                             &psDevData->pTlBuf[eStreamId],
+		                             &psDevData->ui32AcqDataLen[eStreamId]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "TLClientAcquireData");
+
+		psDevData->pTlBufPos[eStreamId] = psDevData->pTlBuf[eStreamId];
+	}
+
+	/* TL indicates no data exists so return OK and zero. */
+	if ((psDevData->pTlBufPos[eStreamId] == NULL) || (psDevData->ui32AcqDataLen[eStreamId] == 0))
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Process each TL packet in the data buffer we have acquired */
+	pBufferEnd = psDevData->pTlBuf[eStreamId]+psDevData->ui32AcqDataLen[eStreamId];
+	pDataDest = psDevData->pHwpBuf[eStreamId];
+	psHDRptr = GET_PACKET_HDR(psDevData->pTlBufPos[eStreamId]);
+	psDevData->pTlBufRead[eStreamId] = psDevData->pTlBufPos[eStreamId];
+	while (psHDRptr < (PVRSRVTL_PPACKETHDR)((void *)pBufferEnd))
+	{
+		ui16TlType = GET_PACKET_TYPE(psHDRptr);
+		if (ui16TlType == PVRSRVTL_PACKETTYPE_DATA)
+		{
+			IMG_UINT16 ui16DataLen = GET_PACKET_DATA_LEN(psHDRptr);
+			if (0 == ui16DataLen)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "RGXHWPerfAcquireEvents: ZERO Data in TL data packet: %p", psHDRptr));
+			}
+			else
+			{
+				/* Check next packet does not fill buffer */
+				if (pDataDest + ui16DataLen > psDevData->pHwpBufEnd[eStreamId])
+				{
+					break;
+				}
+
+				/* For valid data copy it into the client buffer and move
+				 * the write position on */
+				OSDeviceMemCopy(pDataDest, GET_PACKET_DATA_PTR(psHDRptr), ui16DataLen);
+				pDataDest += ui16DataLen;
+			}
+		}
+		else if (ui16TlType == PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfAcquireEvents: Indication that the transport buffer was full"));
+		}
+		else
+		{
+			/* else Ignore padding packet type and others */
+			PVR_DPF((PVR_DBG_MESSAGE, "RGXHWPerfAcquireEvents: Ignoring TL packet, type %d", ui16TlType ));
+		}
+
+		/* Update loop variable to the next packet and increment counts */
+		psHDRptr = GET_NEXT_PACKET_ADDR(psHDRptr);
+		/* Updated to keep track of the next packet to be read. */
+		psDevData->pTlBufRead[eStreamId] = (IMG_PBYTE) ((void *)psHDRptr);
+		ui32TlPackets++;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "RGXHWPerfAcquireEvents: TL Packets processed %03d", ui32TlPackets));
+
+	psDevData->bRelease[eStreamId] = IMG_FALSE;
+	if (psHDRptr >= (PVRSRVTL_PPACKETHDR)((void *)pBufferEnd))
+	{
+		psDevData->bRelease[eStreamId] = IMG_TRUE;
+	}
+
+	/* Update output arguments with client buffer details and true length */
+	*ppBuf = psDevData->pHwpBuf[eStreamId];
+	*pui32BufLen = pDataDest - psDevData->pHwpBuf[eStreamId];
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfReleaseEvents(
+		IMG_HANDLE hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId)
+{
+	PVRSRV_ERROR			eError = PVRSRV_OK;
+	RGX_KM_HWPERF_DEVDATA*	psDevData = (RGX_KM_HWPERF_DEVDATA*)hDevData;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psDevData || eStreamId >= RGX_HWPERF_MAX_STREAM_ID)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	if (psDevData->bRelease[eStreamId])
+	{
+		/* Inform the TL that we are done with reading the data. */
+		eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, psDevData->hSD[eStreamId]);
+		psDevData->ui32AcqDataLen[eStreamId] = 0;
+		psDevData->pTlBuf[eStreamId] = NULL;
+	}
+	else
+	{
+		psDevData->pTlBufPos[eStreamId] = psDevData->pTlBufRead[eStreamId];
+	}
+	return eError;
+}
+
+
+PVRSRV_ERROR RGXHWPerfGetFilter(
+		IMG_HANDLE  hDevData,
+		RGX_HWPERF_STREAM_ID eStreamId,
+		IMG_UINT64 *ui64Filter)
+{
+	PVRSRV_RGXDEV_INFO* psRgxDevInfo =
+			hDevData ? ((RGX_KM_HWPERF_DEVDATA*) hDevData)->psRgxDevInfo : NULL;
+
+	/* Valid input argument values supplied by the caller */
+	if (!psRgxDevInfo)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid pointer to the RGX device",
+				__func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVINFO, psRgxDevInfo, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+	/* No need to take hHWPerfLock here since we are only reading data
+	 * from always existing integers to return to debugfs which is an
+	 * atomic operation.
+	 */
+	switch (eStreamId) {
+		case RGX_HWPERF_STREAM_ID0_FW:
+			*ui64Filter = psRgxDevInfo->ui64HWPerfFilter[RGX_HWPERF_L2_STREAM_HWPERF];
+			break;
+		case RGX_HWPERF_STREAM_ID1_HOST:
+			*ui64Filter = psRgxDevInfo->ui32HWPerfHostFilter;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR, "%s: Invalid stream ID",
+					__func__));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfFreeConnection(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	RGX_HWPERF_DEVICE *psHWPerfDev, *psHWPerfNextDev;
+	RGX_HWPERF_CONNECTION *psHWPerfConnection = *ppsHWPerfConnection;
+
+	/* if connection object itself is NULL, nothing to free */
+	if (psHWPerfConnection == NULL)
+	{
+		return PVRSRV_OK;
+	}
+
+	psHWPerfNextDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfNextDev)
+	{
+		psHWPerfDev = psHWPerfNextDev;
+		psHWPerfNextDev = psHWPerfNextDev->psNext;
+
+		/* Free the session memory */
+		if (psHWPerfDev->hDevData)
+			OSFreeMem(psHWPerfDev->hDevData);
+		OSFreeMem(psHWPerfDev);
+	}
+	OSFreeMem(psHWPerfConnection);
+	*ppsHWPerfConnection = NULL;
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfClose(RGX_HWPERF_CONNECTION *psHWPerfConnection)
+{
+	RGX_HWPERF_DEVICE *psHWPerfDev;
+	RGX_KM_HWPERF_DEVDATA* psDevData;
+	IMG_UINT uiStreamId;
+	PVRSRV_ERROR eError;
+
+	/* Check session connection is not zero */
+	if (!psHWPerfConnection)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psHWPerfDev = psHWPerfConnection->psHWPerfDevList;
+	while (psHWPerfDev)
+	{
+		psDevData = (RGX_KM_HWPERF_DEVDATA *) psHWPerfDev->hDevData;
+
+		PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevData->psRgxDevNode, PVRSRV_ERROR_NOT_IMPLEMENTED);
+
+		for (uiStreamId = 0; uiStreamId < RGX_HWPERF_MAX_STREAM_ID; uiStreamId++)
+		{
+			/* If the TL buffer exists they have not called ReleaseData
+			 * before disconnecting so clean it up */
+			if (psDevData->pTlBuf[uiStreamId])
+			{
+				/* TLClientReleaseData call and null out the buffer fields
+				 * and length */
+				eError = TLClientReleaseData(DIRECT_BRIDGE_HANDLE, psDevData->hSD[uiStreamId]);
+				psDevData->ui32AcqDataLen[uiStreamId] = 0;
+				psDevData->pTlBuf[uiStreamId] = NULL;
+				PVR_LOG_IF_ERROR(eError, "TLClientReleaseData");
+				/* Packets may be lost if release was not required */
+				if (!psDevData->bRelease[uiStreamId])
+				{
+					PVR_DPF((PVR_DBG_WARNING, "RGXHWPerfClose: Events in buffer waiting to be read, remaining events may be lost."));
+				}
+			}
+
+			/* Close the TL stream, ignore the error if it occurs as we
+			 * are disconnecting */
+			if (psDevData->hSD[uiStreamId])
+			{
+				eError = TLClientCloseStream(DIRECT_BRIDGE_HANDLE,
+				                             psDevData->hSD[uiStreamId]);
+				PVR_LOG_IF_ERROR(eError, "TLClientCloseStream");
+				psDevData->hSD[uiStreamId] = NULL;
+			}
+
+			/* Free the client buffer used in session */
+			if (psDevData->pHwpBuf[uiStreamId])
+			{
+				OSFreeMem(psDevData->pHwpBuf[uiStreamId]);
+				psDevData->pHwpBuf[uiStreamId] = NULL;
+			}
+		}
+		psHWPerfDev = psHWPerfDev->psNext;
+	}
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR RGXHWPerfDisconnect(RGX_HWPERF_CONNECTION** ppsHWPerfConnection)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXHWPerfClose(*ppsHWPerfConnection);
+	PVR_LOG_IF_ERROR(eError, "RGXHWPerfClose");
+
+	eError = RGXHWPerfFreeConnection(ppsHWPerfConnection);
+	PVR_LOG_IF_ERROR(eError, "RGXHWPerfFreeConnection");
+
+	return eError;
+}
+
+IMG_UINT64 RGXHWPerfConvertCRTimeStamp(
+		IMG_UINT32 ui32ClkSpeed,
+		IMG_UINT64 ui64CorrCRTimeStamp,
+		IMG_UINT64 ui64CorrOSTimeStamp,
+		IMG_UINT64 ui64CRTimeStamp)
+{
+	IMG_UINT64 ui64CRDeltaToOSDeltaKNs;
+	IMG_UINT64 ui64EventOSTimestamp, deltaRgxTimer, delta_ns;
+
+	if (!(ui64CRTimeStamp) || !(ui32ClkSpeed) || !(ui64CorrCRTimeStamp) || !(ui64CorrOSTimeStamp))
+	{
+		return 0;
+	}
+
+	ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(ui32ClkSpeed);
+
+	/* RGX CR timer ticks delta */
+	deltaRgxTimer = ui64CRTimeStamp - ui64CorrCRTimeStamp;
+	/* RGX time delta in nanoseconds */
+	delta_ns = RGXFWIF_GET_DELTA_OSTIME_NS(deltaRgxTimer, ui64CRDeltaToOSDeltaKNs);
+	/* Calculate OS time of HWPerf event */
+	ui64EventOSTimestamp = ui64CorrOSTimeStamp + delta_ns;
+
+	return ui64EventOSTimestamp;
+}
+
+/******************************************************************************
+ End of file (rgxhwperf_common.c)
+ ******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxhwperf_common.h b/drivers/gpu/drm/img-rogue/rgxhwperf_common.h
new file mode 100644
index 000000000000..9b14d15cb64f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxhwperf_common.h
@@ -0,0 +1,657 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX HW Performance header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX HWPerf functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXHWPERF_COMMON_H_
+#define RGXHWPERF_COMMON_H_
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#include "device.h"
+#include "connection_server.h"
+#include "rgxdevice.h"
+#include "rgx_hwperf.h"
+#include "rgx_fwif_hwperf.h"
+#if defined(PVR_ARCH_VOLCANIC)
+#include "rgx_hwperf_table.h"
+#endif
+#include "cache_ops.h"
+#include "rgxfwmemctx.h"
+
+/* HWPerf host buffer size constraints in KBs */
+#define HWPERF_HOST_TL_STREAM_SIZE_DEFAULT PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB
+#define HWPERF_HOST_TL_STREAM_SIZE_MIN     (32U)
+#define HWPERF_HOST_TL_STREAM_SIZE_MAX     (3072U)
+
+/* Operations on HWPerf filter. */
+typedef enum HWPERF_FILTER_OPERATION_TAG
+{
+	HWPERF_FILTER_OPERATION_SET,
+	HWPERF_FILTER_OPERATION_BIT_CLR,
+	HWPERF_FILTER_OPERATION_BIT_OR,
+} HWPERF_FILTER_OPERATION;
+
+/******************************************************************************
+ * RGX HW Performance decode Bvnc Features for HWPerf
+ *****************************************************************************/
+PVRSRV_ERROR RGXServerFeatureFlagsToHWPerfFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+		                                        RGX_HWPERF_BVNC    *psBVNC);
+
+PVRSRV_ERROR PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(CONNECTION_DATA    *psConnection,
+                                                  PVRSRV_DEVICE_NODE *psDeviceNode,
+											      RGX_HWPERF_BVNC    *psBVNC);
+
+/******************************************************************************
+ * RGX HW Performance Data Transport Routines
+ *****************************************************************************/
+
+PVRSRV_ERROR RGXHWPerfDataStoreCB(PVRSRV_DEVICE_NODE* psDevInfo);
+
+PVRSRV_ERROR RGXHWPerfInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+void RGXHWPerfDeinit(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+
+PVRSRV_ERROR RGXHWPerfInitOnDemandL1Buffer(PVRSRV_RGXDEV_INFO* psRgxDevInfo);
+PVRSRV_ERROR RGXHWPerfInitOnDemandL2Stream(PVRSRV_RGXDEV_INFO* psRgxDevInfo,
+                                           RGX_HWPERF_L2_STREAM_ID eL2StreamId);
+void RGXHWPerfDeinitL2Stream(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                             RGX_HWPERF_L2_STREAM_ID eL2StreamId);
+void RGXHWPerfInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+void RGXHWPerfClientInitAppHintCallbacks(void);
+
+static INLINE PVRSRV_ERROR RGXAcquireHWPerfCtlCPUAddr(PVRSRV_DEVICE_NODE *psDevNode,
+                                                      RGXFWIF_HWPERF_CTL **ppsHWPerfCtl)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+
+	PVR_RETURN_IF_FALSE(psDevNode != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_RETURN_IF_FALSE(psDevInfo != NULL, PVRSRV_ERROR_INVALID_PARAMS);
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc,
+	                                  (void**)ppsHWPerfCtl);
+	RGXFwSharedMemCacheOpPtr(ppsHWPerfCtl, INVALIDATE);
+
+	return eError;
+}
+
+static INLINE void RGXReleaseHWPerfCtlCPUAddr(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevNode != NULL, "psDevNode is invalid");
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO*)psDevNode->pvDevice;
+	PVR_LOG_RETURN_VOID_IF_FALSE(psDevInfo != NULL, "psDevInfo invalid");
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+}
+
+PVRSRV_ERROR PVRSRVRGXGetConfiguredHWPerfCountersKM(CONNECTION_DATA *psConnection,
+                                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                    const IMG_UINT32 ui32BlockID,
+                                                    RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCounters);
+
+PVRSRV_ERROR PVRSRVRGXGetEnabledHWPerfBlocksKM(CONNECTION_DATA *psConnection,
+                                               PVRSRV_DEVICE_NODE *psDeviceNode,
+                                               const IMG_UINT32 ui32ArrayLen,
+                                               IMG_UINT32 *pui32BlockCount,
+                                               IMG_UINT32 *pui32EnabledBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Profiling API(s)
+ *****************************************************************************/
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfKM(
+	CONNECTION_DATA      * psConnection,
+	PVRSRV_DEVICE_NODE   * psDeviceNode,
+	 RGX_HWPERF_STREAM_ID  eStreamId,
+	IMG_BOOL               bToggle,
+	IMG_UINT64             ui64Mask);
+
+PVRSRV_ERROR PVRSRVRGXCtrlHWPerfFW(
+	PVRSRV_DEVICE_NODE      *psDeviceNode,
+	RGX_HWPERF_L2_STREAM_ID  eL2StreamId,
+	IMG_UINT64               ui64Mask,
+	HWPERF_FILTER_OPERATION  eMaskOp);
+
+#if defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+PVRSRV_ERROR PVRSRVRGXGetHWPerfTimeStampKM(
+		CONNECTION_DATA         *psConnection,
+		PVRSRV_DEVICE_NODE      *psDeviceNode,
+		IMG_UINT64              *pui64TimeStamp);
+#endif
+
+PVRSRV_ERROR PVRSRVRGXControlHWPerfBlocksKM(
+	CONNECTION_DATA       * psConnection,
+	PVRSRV_DEVICE_NODE    * psDeviceNode,
+	IMG_BOOL              bEnable,
+	IMG_UINT32            ui32ArrayLen,
+	IMG_UINT16            * psBlockIDs);
+
+/******************************************************************************
+ * RGX HW Performance Host Stream API
+ *****************************************************************************/
+
+static inline RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS
+RGXHWPerfConvDeviceHealthStatus(PVRSRV_DEVICE_HEALTH_STATUS eDeviceHealthStatus)
+{
+	switch (eDeviceHealthStatus)
+	{
+		case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:         return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED;
+		case PVRSRV_DEVICE_HEALTH_STATUS_OK:                return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:    return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_NOT_RESPONDING;
+		case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:              return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_DEAD;
+		case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:             return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_FAULT;
+		default:                                            return RGX_HWPERF_HOST_DEVICE_HEALTH_STATUS_UNDEFINED;
+	}
+}
+
+static inline RGX_HWPERF_HOST_DEVICE_HEALTH_REASON
+RGXHWPerfConvDeviceHealthReason(PVRSRV_DEVICE_HEALTH_REASON eDeviceHealthReason)
+{
+	switch (eDeviceHealthReason)
+	{
+		case PVRSRV_DEVICE_HEALTH_REASON_NONE:                  return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_NONE;
+		case PVRSRV_DEVICE_HEALTH_REASON_ASSERTED:              return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_ASSERTED;
+		case PVRSRV_DEVICE_HEALTH_REASON_POLL_FAILING:          return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_POLL_FAILING;
+		case PVRSRV_DEVICE_HEALTH_REASON_TIMEOUTS:              return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_TIMEOUTS;
+		case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_CORRUPT:         return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_CORRUPT;
+		case PVRSRV_DEVICE_HEALTH_REASON_QUEUE_STALLED:         return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_QUEUE_STALLED;
+		case PVRSRV_DEVICE_HEALTH_REASON_IDLING:                return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_IDLING;
+		case PVRSRV_DEVICE_HEALTH_REASON_RESTARTING:            return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_RESTARTING;
+		case PVRSRV_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS:    return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_MISSING_INTERRUPTS;
+		case PVRSRV_DEVICE_HEALTH_REASON_PCI_ERROR:             return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_PCI_ERROR;
+		default:                                                return RGX_HWPERF_HOST_DEVICE_HEALTH_REASON_UNDEFINED;
+	}
+}
+
+PVRSRV_ERROR RGXHWPerfHostInit(PVRSRV_RGXDEV_INFO *psRgxDevInfo, IMG_UINT32 ui32BufSizeKB);
+PVRSRV_ERROR RGXHWPerfHostInitOnDemandResources(PVRSRV_RGXDEV_INFO* psRgxDevInfo);
+void RGXHWPerfHostDeInit(PVRSRV_RGXDEV_INFO	*psRgxDevInfo);
+
+IMG_UINT64 RGXHWPerfFwSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                     RGX_HWPERF_L2_STREAM_ID eL2StreamId,
+                                     IMG_UINT64 uiFilter);
+
+void RGXHWPerfHostSetEventFilter(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 IMG_UINT32 ui32Filter);
+
+void RGXHWPerfHostPostRaw(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+						  RGX_HWPERF_HOST_EVENT_TYPE eEvType,
+						  IMG_BYTE *pbPayload,
+						  IMG_UINT32 ui32PayloadSize);
+
+void RGXHWPerfHostPostEnqEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_KICK_TYPE eEnqType,
+                               IMG_UINT32 ui32Pid,
+                               IMG_UINT32 ui32FWDMContext,
+                               IMG_UINT32 ui32ExtJobRef,
+                               IMG_UINT32 ui32IntJobRef,
+                               PVRSRV_FENCE hCheckFence,
+                               PVRSRV_FENCE hUpdateFence,
+                               PVRSRV_TIMELINE hUpdateTimeline,
+                               IMG_UINT64 ui64CheckFenceUID,
+                               IMG_UINT64 ui64UpdateFenceUID,
+                               IMG_UINT64 ui64DeadlineInus,
+                               IMG_UINT32 ui32CycleEstimate);
+
+void RGXHWPerfHostPostAllocEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                 RGX_HWPERF_HOST_RESOURCE_TYPE eAllocType,
+                                 const IMG_CHAR *psName,
+                                 IMG_UINT32 ui32NameSize,
+                                 RGX_HWPERF_HOST_ALLOC_DETAIL *puAllocDetail);
+
+void RGXHWPerfHostPostFreeEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                RGX_HWPERF_HOST_RESOURCE_TYPE eFreeType,
+                                IMG_UINT64 ui64UID,
+                                IMG_UINT32 ui32PID,
+                                IMG_UINT32 ui32FWAddr);
+
+void RGXHWPerfHostPostModifyEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                  RGX_HWPERF_HOST_RESOURCE_TYPE eModifyType,
+                                  IMG_UINT64 ui64NewUID,
+                                  IMG_UINT64 ui64UID1,
+                                  IMG_UINT64 ui64UID2,
+                                  const IMG_CHAR *psName,
+                                  IMG_UINT32 ui32NameSize);
+
+void RGXHWPerfHostPostUfoEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                               RGX_HWPERF_UFO_EV eUfoType,
+                               RGX_HWPERF_UFO_DATA_ELEMENT *psUFOData,
+							   const IMG_BOOL bSleepAllowed);
+
+void RGXHWPerfHostPostClkSyncEvent(PVRSRV_RGXDEV_INFO *psRgxDevInfo);
+
+void RGXHWPerfHostPostDeviceInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_DEV_INFO_EV eEvType,
+								 RGX_HWPERF_HOST_DEV_INFO_DETAIL *psData);
+
+void RGXHWPerfHostPostInfo(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								 RGX_HWPERF_INFO_EV eEvType);
+
+void RGXHWPerfHostPostFenceWait(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+								RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE eType,
+								IMG_PID uiPID,
+								PVRSRV_FENCE hFence,
+								IMG_UINT32 ui32Data);
+
+void RGXHWPerfHostPostSWTimelineAdv(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                    IMG_PID uiPID,
+									PVRSRV_TIMELINE hSWTimeline,
+									IMG_UINT64 ui64SyncPtIndex);
+
+void RGXHWPerfHostPostClientInfoProcName(PVRSRV_RGXDEV_INFO *psRgxDevInfo,
+                                         IMG_PID uiPID,
+									     const IMG_CHAR *psName);
+
+IMG_BOOL RGXHWPerfHostIsEventEnabled(PVRSRV_RGXDEV_INFO *psRgxDevInfo, RGX_HWPERF_HOST_EVENT_TYPE eEvent);
+
+#if defined(PVR_ARCH_VOLCANIC)
+IMG_INTERNAL /*static inline*/ IMG_UINT32
+RGXGetHWPerfBlockConfig(const RGXFW_HWPERF_CNTBLK_TYPE_MODEL **ppsModel);
+
+/*!
+*******************************************************************************
+ @Function    RGXHWPerfMaxDefinedBlks
+
+ @Description Return the number of valid block-IDs for the given device node
+
+ @Input       (PVRSRV_RGXDEV_INFO *)   pvDevice    device-node to query
+
+ @Returns     (IMG_UINT32)             Number of block-IDs (RGX_CNTBLK_ID)
+                                       valid for this device.
+******************************************************************************/
+IMG_INTERNAL IMG_UINT32
+RGXHWPerfMaxDefinedBlks(PVRSRV_RGXDEV_INFO *psDevInfo);
+#endif
+
+#define _RGX_HWPERF_HOST_FILTER(CTX, EV) \
+		(((PVRSRV_RGXDEV_INFO *)CTX->psDeviceNode->pvDevice)->ui32HWPerfHostFilter \
+		& RGX_HWPERF_EVENT_MASK_VALUE(EV))
+
+#define _RGX_DEVICE_INFO_FROM_CTX(CTX) \
+		((PVRSRV_RGXDEV_INFO *)CTX->psDeviceNode->pvDevice)
+
+#define _RGX_DEVICE_INFO_FROM_NODE(DEVNODE) \
+		((PVRSRV_RGXDEV_INFO *)DEVNODE->pvDevice)
+
+/* Deadline and cycle estimate is not supported for all ENQ events */
+#define NO_DEADLINE 0
+#define NO_CYCEST   0
+
+
+#if defined(SUPPORT_RGX)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param C      Kick context
+ * @param P      Pid of kicking process
+ * @param X      Related FW context
+ * @param E      External job reference
+ * @param I      Job ID
+ * @param K      Kick type
+ * @param CF     Check fence handle
+ * @param UF     Update fence handle
+ * @param UT     Update timeline (on which above UF was created) handle
+ * @param CHKUID Check fence UID
+ * @param UPDUID Update fence UID
+ * @param D      Deadline
+ * @param CE     Cycle estimate
+ */
+#define RGXSRV_HWPERF_ENQ(C, P, X, E, I, K, CF, UF, UT, CHKUID, UPDUID, D, CE) \
+		do { \
+			if (_RGX_HWPERF_HOST_FILTER(C, RGX_HWPERF_HOST_ENQ)) \
+			{ \
+				RGXHWPerfHostPostEnqEvent(_RGX_DEVICE_INFO_FROM_CTX(C), \
+				                          (K), (P), (X), (E), (I), \
+				                          (CF), (UF), (UT), \
+				                          (CHKUID), (UPDUID), (D), (CE)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I Device Info pointer
+ * @param T Host UFO event type
+ * @param D Pointer to UFO data
+ * @param S Is sleeping allowed?
+ */
+#define RGXSRV_HWPERF_UFO(I, T, D, S) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_UFO)) \
+			{ \
+				RGXHWPerfHostPostUfoEvent((I), (T), (D), (S)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device node pointer
+ * @param T Host ALLOC event type
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC(D, T, FWADDR, N, Z) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSyncAlloc.ui32FWAddr = (FWADDR); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+											(N), (Z), &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param PID ID of allocating process
+ * @param FENCE PVRSRV_FENCE object
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_FENCE(D, PID, FENCE, FWADDR, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sFenceAlloc.uiPID = (PID); \
+				uAllocDetail.sFenceAlloc.hFence = (FENCE); \
+				uAllocDetail.sFenceAlloc.ui32CheckPt_FWAddr = (FWADDR); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_PVR, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * @param D Device Node pointer
+ * @param TL PVRSRV_TIMELINE on which CP is allocated
+ * @param PID Allocating process ID of this TL/FENCE
+ * @param FENCE PVRSRV_FENCE as passed to SyncCheckpointResolveFence OR PVRSRV_NO_FENCE
+ * @param FWADDR sync firmware address
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_SYNC_CP(D, TL, PID, FENCE, FWADDR, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSyncCheckPointAlloc.ui32CheckPt_FWAddr = (FWADDR); \
+				uAllocDetail.sSyncCheckPointAlloc.hTimeline = (TL); \
+				uAllocDetail.sSyncCheckPointAlloc.uiPID = (PID); \
+				uAllocDetail.sSyncCheckPointAlloc.hFence = (FENCE); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_SYNC_CP, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * @param D Device Node pointer
+ * @param PID ID of allocating process
+ * @param SW_FENCE PVRSRV_FENCE object
+ * @param SW_TL PVRSRV_TIMELINE on which SW_FENCE is allocated
+ * @param SPI Sync point index on the SW_TL on which this SW_FENCE is allocated
+ * @param N string containing sync name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_ALLOC_SW_FENCE(D, PID, SW_FENCE, SW_TL, SPI, N, Z)  \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_ALLOC)) \
+			{ \
+				RGX_HWPERF_HOST_ALLOC_DETAIL uAllocDetail; \
+				uAllocDetail.sSWFenceAlloc.uiPID = (PID); \
+				uAllocDetail.sSWFenceAlloc.hSWFence = (SW_FENCE); \
+				uAllocDetail.sSWFenceAlloc.hSWTimeline = (SW_TL); \
+				uAllocDetail.sSWFenceAlloc.ui64SyncPtIndex = (SPI); \
+				RGXHWPerfHostPostAllocEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                            RGX_HWPERF_HOST_RESOURCE_TYPE_FENCE_SW, \
+				                            N, Z, &uAllocDetail); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param FWADDR sync firmware address
+ */
+#define RGXSRV_HWPERF_FREE(D, T, FWADDR) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_FREE)) \
+			{ \
+				RGXHWPerfHostPostFreeEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                           RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                           (0), (0), (FWADDR)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param UID ID of input object
+ * @param PID ID of allocating process
+ * @param FWADDR sync firmware address
+ */
+#define RGXSRV_HWPERF_FREE_FENCE_SYNC(D, T, UID, PID, FWADDR) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_FREE)) \
+			{ \
+				RGXHWPerfHostPostFreeEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                           RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                           (UID), (PID), (FWADDR)); \
+			} \
+		} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param D Device Node pointer
+ * @param T Host ALLOC event type
+ * @param NEWUID ID of output object
+ * @param UID1 ID of first input object
+ * @param UID2 ID of second input object
+ * @param N string containing new object's name
+ * @param Z string size including null terminating character
+ */
+#define RGXSRV_HWPERF_MODIFY_FENCE_SYNC(D, T, NEWUID, UID1, UID2, N, Z) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_MODIFY)) \
+			{ \
+				RGXHWPerfHostPostModifyEvent(_RGX_DEVICE_INFO_FROM_NODE(D), \
+				                             RGX_HWPERF_HOST_RESOURCE_TYPE_##T, \
+				                             (NEWUID), (UID1), (UID2), N, Z); \
+			} \
+		} while (0)
+
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I Device info pointer
+ */
+#define RGXSRV_HWPERF_CLK_SYNC(I) \
+		do { \
+			if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_CLK_SYNC)) \
+			{ \
+				RGXHWPerfHostPostClkSyncEvent((I)); \
+			} \
+		} while (0)
+
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts a device info health event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ * @param H      Health status enum
+ * @param R      Health reason enum
+ */
+#define RGXSRV_HWPERF_DEVICE_INFO_HEALTH(I, H, R) \
+	do { \
+		if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_DEV_INFO)) \
+		{ \
+			RGX_HWPERF_HOST_DEV_INFO_DETAIL uDevDetail; \
+			uDevDetail.sDeviceStatus.eDeviceHealthStatus = RGXHWPerfConvDeviceHealthStatus(H); \
+			uDevDetail.sDeviceStatus.eDeviceHealthReason = RGXHWPerfConvDeviceHealthReason(R); \
+			RGXHWPerfHostPostDeviceInfo((I), RGX_HWPERF_DEV_INFO_EV_HEALTH, &uDevDetail); \
+		} \
+	} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts a device info features event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ */
+#define RGXSRV_HWPERF_DEVICE_INFO_FEATURES(I) \
+	do { \
+		if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_DEV_INFO)) \
+		{ \
+			RGXHWPerfHostPostDeviceInfo((I), RGX_HWPERF_DEV_INFO_EV_FEATURES, NULL); \
+		} \
+	} while (0)
+
+/**
+ * This macro checks if HWPerfHost and the event are enabled and if they are
+ * it posts event to the HWPerfHost stream.
+ *
+ * @param I      Device info pointer
+ * @param T      Event type
+ */
+#define RGXSRV_HWPERF_HOST_INFO(I, T) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_INFO)) \
+	{ \
+		RGXHWPerfHostPostInfo((I), (T)); \
+	} \
+} while (0)
+
+/**
+ * @param I      Device info pointer
+ * @param T      Wait Event type
+ * @param PID    Process ID that the following fence belongs to
+ * @param F      Fence handle
+ * @param D      Data for this wait event type
+ */
+#define RGXSRV_HWPERF_SYNC_FENCE_WAIT(I, T, PID, F, D) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_SYNC_FENCE_WAIT)) \
+	{ \
+		RGXHWPerfHostPostFenceWait(I, RGX_HWPERF_HOST_SYNC_FENCE_WAIT_TYPE_##T, \
+		                           (PID), (F), (D)); \
+	} \
+} while (0)
+
+/**
+ * @param I      Device info pointer
+ * @param PID    Process ID that the following timeline belongs to
+ * @param F      SW-timeline handle
+ * @param SPI    Sync-pt index where this SW-timeline has reached
+ */
+#define RGXSRV_HWPERF_SYNC_SW_TL_ADV(I, PID, SW_TL, SPI)\
+do { \
+	if (RGXHWPerfHostIsEventEnabled((I), RGX_HWPERF_HOST_SYNC_SW_TL_ADVANCE)) \
+	{ \
+		RGXHWPerfHostPostSWTimelineAdv((I), (PID), (SW_TL), (SPI)); \
+	} \
+} while (0)
+
+/**
+ * @param D      Device Node pointer
+ * @param PID    Process ID that the following timeline belongs to
+ * @param N      Null terminated string containing the process name
+ */
+#define RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(D, PID, N) \
+do { \
+	if (RGXHWPerfHostIsEventEnabled(_RGX_DEVICE_INFO_FROM_NODE(D), RGX_HWPERF_HOST_CLIENT_INFO)) \
+	{ \
+		RGXHWPerfHostPostClientInfoProcName(_RGX_DEVICE_INFO_FROM_NODE(D), (PID), (N)); \
+	} \
+} while (0)
+
+#else
+
+#define RGXSRV_HWPERF_ENQ(C, P, X, E, I, K, CF, UF, UT, CHKUID, UPDUID, D, CE)
+#define RGXSRV_HWPERF_UFO(I, T, D, S)
+#define RGXSRV_HWPERF_ALLOC(D, T, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_FENCE(D, PID, FENCE, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_SYNC_CP(D, TL, PID, FENCE, FWADDR, N, Z)
+#define RGXSRV_HWPERF_ALLOC_SW_FENCE(D, PID, SW_FENCE, SW_TL, SPI, N, Z)
+#define RGXSRV_HWPERF_FREE(D, T, FWADDR)
+#define RGXSRV_HWPERF_FREE_FENCE_SYNC(D, T, UID, PID, FWADDR)
+#define RGXSRV_HWPERF_MODIFY_FENCE_SYNC(D, T, NEWUID, UID1, UID2, N, Z)
+#define RGXSRV_HWPERF_CLK_SYNC(I)
+#define RGXSRV_HWPERF_DEVICE_INFO_HEALTH(I, H, R)
+#define RGXSRV_HWPERF_DEVICE_INFO_FEATURES(I)
+#define RGXSRV_HWPERF_HOST_INFO(I, T)
+#define RGXSRV_HWPERF_SYNC_FENCE_WAIT(I, T, PID, F, D)
+#define RGXSRV_HWPERF_SYNC_SW_TL_ADV(I, PID, SW_TL, SPI)
+#define RGXSRV_HWPERF_HOST_CLIENT_INFO_PROCESS_NAME(D, PID, N)
+
+#endif
+
+#endif /* RGXHWPERF_COMMON_H_ */
diff --git a/drivers/gpu/drm/img-rogue/rgxinit.c b/drivers/gpu/drm/img-rogue/rgxinit.c
new file mode 100644
index 000000000000..88f243d899f7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxinit.c
@@ -0,0 +1,5610 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "img_defs.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "pvrsrv_bridge_init.h"
+#include "rgx_bridge_init.h"
+#include "syscommon.h"
+#include "rgx_heaps_server.h"
+#include "rgxheapconfig.h"
+#include "rgxpower.h"
+#include "tlstream.h"
+#include "pvrsrv_tlstreams.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "pvr_ricommon.h"
+#endif
+
+#include "rgxinit.h"
+#include "rgxbvnc.h"
+#include "rgxmulticore.h"
+
+#include "pdump_km.h"
+#include "handle.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxmem.h"
+#include "sync_internal.h"
+#include "pvrsrv_apphint.h"
+#include "os_apphint.h"
+#include "rgxfwdbg.h"
+#include "info_page.h"
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxfwimageutils.h"
+#endif
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+#include "rgxmmuinit.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgxmipsmmuinit.h"
+#include "physmem.h"
+#endif
+#include "devicemem_utils.h"
+#include "devicemem_server.h"
+#include "physmem_osmem.h"
+#include "physmem_lma.h"
+
+#include "rgxdebug_common.h"
+#include "rgxhwperf.h"
+#include "htbserver.h"
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "rgxta3d.h"
+#include "rgxtimecorr.h"
+#include "rgxshader.h"
+
+#if defined(PDUMP)
+#include "rgxstartstop.h"
+#endif
+
+#include "rgx_fwif_alignchecks.h"
+#include "vmm_pvz_client.h"
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+
+#if defined(PDUMP) && defined(SUPPORT_SECURITY_VALIDATION)
+#include "pdump_physmem.h"
+#endif
+
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode);
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_CHAR **ppszVersionString);
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_PUINT32 pui32RGXClockSpeed);
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64ResetValue1, IMG_UINT64 ui64ResetValue2);
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID);
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/* Services internal heap identification used in this file only */
+#define RGX_FIRMWARE_MAIN_HEAP_IDENT   "FwMain"   /*!< RGX Main Firmware Heap identifier */
+#define RGX_FIRMWARE_CONFIG_HEAP_IDENT "FwConfig" /*!< RGX Config firmware Heap identifier */
+
+#define RGX_MMU_PAGE_SIZE_4KB   (   4 * 1024)
+#define RGX_MMU_PAGE_SIZE_16KB  (  16 * 1024)
+#define RGX_MMU_PAGE_SIZE_64KB  (  64 * 1024)
+#define RGX_MMU_PAGE_SIZE_256KB ( 256 * 1024)
+#define RGX_MMU_PAGE_SIZE_1MB   (1024 * 1024)
+#define RGX_MMU_PAGE_SIZE_2MB   (2048 * 1024)
+#define RGX_MMU_PAGE_SIZE_MIN RGX_MMU_PAGE_SIZE_4KB
+#define RGX_MMU_PAGE_SIZE_MAX RGX_MMU_PAGE_SIZE_2MB
+
+#define VAR(x) #x
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#if !defined(NO_HARDWARE)
+/*************************************************************************/ /*!
+@Function       SampleIRQCount
+@Description    Utility function taking snapshots of RGX FW interrupt count.
+@Input          psDevInfo    Device Info structure
+
+@Return         IMG_BOOL     Returns IMG_TRUE if RGX FW IRQ is not equal to
+                             sampled RGX FW IRQ count for any RGX FW thread.
+ */ /**************************************************************************/
+static INLINE IMG_BOOL SampleIRQCount(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bReturnVal = IMG_FALSE;
+	volatile IMG_UINT32 *pui32SampleIrqCount = psDevInfo->aui32SampleIRQCount;
+	IMG_UINT32 ui32IrqCnt;
+
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bReturnVal = IMG_TRUE;
+	}
+	else
+	{
+		get_irq_cnt_val(ui32IrqCnt, RGXFW_HOST_DRIVER_ID, psDevInfo);
+
+		if (ui32IrqCnt != pui32SampleIrqCount[RGXFW_THREAD_0])
+		{
+			pui32SampleIrqCount[RGXFW_THREAD_0] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#else
+	IMG_UINT32 ui32TID;
+
+	for_each_irq_cnt(ui32TID)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32TID, psDevInfo);
+
+		/* treat unhandled interrupts here to align host count with fw count */
+		if (pui32SampleIrqCount[ui32TID] != ui32IrqCnt)
+		{
+			pui32SampleIrqCount[ui32TID] = ui32IrqCnt;
+			bReturnVal = IMG_TRUE;
+		}
+	}
+#endif
+
+	return bReturnVal;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXHostSafetyEvents
+@Description    Returns the event status masked to keep only the safety
+                events handled by the Host
+@Input          psDevInfo    Device Info structure
+@Return         IMG_UINT32   Status of Host-handled safety events
+ */ /**************************************************************************/
+static INLINE IMG_UINT32 RGXHostSafetyEvents(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo) || (psDevInfo->ui32HostSafetyEventMask == 0))
+	{
+		return 0;
+	}
+	else
+	{
+		IMG_UINT32 ui32SafetyEventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE);
+		return (ui32SafetyEventStatus & psDevInfo->ui32HostSafetyEventMask);
+	}
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventCheck
+@Description    Clears the Event Status register and checks if any of the
+                safety events need Host handling
+@Input          psDevInfo    Device Info structure
+@Return         IMG_BOOL     Are there any safety events for Host to handle ?
+ */ /**************************************************************************/
+static INLINE IMG_BOOL RGXSafetyEventCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bSafetyEvent = IMG_FALSE;
+
+	if (psDevInfo->ui32HostSafetyEventMask != 0)
+	{
+		IMG_UINT32 ui32EventStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_STATUS);
+
+		if (BIT_ISSET(ui32EventStatus, RGX_CR_EVENT_STATUS__ROGUEXE__SAFETY_SHIFT))
+		{
+			/* clear the safety event */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_EVENT_CLEAR, RGX_CR_EVENT_CLEAR__ROGUEXE__SAFETY_EN);
+
+			/* report if there is anything for the Host to handle */
+			bSafetyEvent = (RGXHostSafetyEvents(psDevInfo) != 0);
+		}
+	}
+
+	return bSafetyEvent;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSafetyEventHandler
+@Description    Handles the Safety Events that the Host is responsible for
+@Input          psDevInfo    Device Info structure
+ */ /**************************************************************************/
+static void RGXSafetyEventHandler(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	if (psDevInfo->bRGXPowered)
+	{
+		RGX_CONTEXT_RESET_REASON eResetReason = RGX_CONTEXT_RESET_REASON_NONE;
+		IMG_UINT32 ui32HostSafetyStatus = RGXHostSafetyEvents(psDevInfo);
+
+		if (ui32HostSafetyStatus != 0)
+		{
+			/* clear the safety bus events handled by the Host */
+			OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SAFETY_EVENT_CLEAR__ROGUEXE, ui32HostSafetyStatus);
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_SHIFT))
+			{
+				IMG_UINT32 ui32FaultFlag;
+				IMG_UINT32 ui32FaultFW = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_STATUS);
+				IMG_UINT32 ui32CorrectedBitOffset = RGX_CR_FAULT_FW_STATUS_CPU_CORRECT_SHIFT -
+													RGX_CR_FAULT_FW_STATUS_CPU_DETECT_SHIFT;
+
+				PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety fault status: 0x%X", __func__, ui32FaultFW));
+
+				for (ui32FaultFlag = 0; ui32FaultFlag < ui32CorrectedBitOffset; ui32FaultFlag++)
+				{
+					if (BIT_ISSET(ui32FaultFW, ui32FaultFlag))
+					{
+						PVR_DPF((PVR_DBG_ERROR, "%s: Firmware safety hardware fault detected (0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_ERR;
+					}
+					else if BIT_ISSET(ui32FaultFW, ui32FaultFlag + ui32CorrectedBitOffset)
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: Firmware safety hardware fault corrected.(0x%lX).",
+							 __func__, BIT(ui32FaultFlag)));
+
+						/* Only report this if we haven't detected a more serious error */
+						if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR)
+						{
+							eResetReason = RGX_CONTEXT_RESET_REASON_FW_ECC_OK;
+						}
+					}
+				}
+
+				OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_FAULT_FW_CLEAR, ui32FaultFW);
+			}
+
+			if (BIT_ISSET(ui32HostSafetyStatus, RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_SHIFT))
+			{
+				volatile RGXFWIF_POW_STATE ePowState;
+
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+										   INVALIDATE);
+				ePowState = psDevInfo->psRGXFWIfFwSysData->ePowState;
+
+				if (ePowState != RGXFWIF_POW_OFF)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: Safety Watchdog Trigger !", __func__));
+
+					/* Only report this if we haven't detected a more serious error */
+					if (eResetReason != RGX_CONTEXT_RESET_REASON_FW_ECC_ERR)
+					{
+						eResetReason = RGX_CONTEXT_RESET_REASON_FW_WATCHDOG;
+					}
+				}
+			}
+
+			/* Notify client and system layer of any error */
+			if (eResetReason != RGX_CONTEXT_RESET_REASON_NONE)
+			{
+				PVRSRV_DEVICE_NODE *psDevNode = psDevInfo->psDeviceNode;
+				PVRSRV_DEVICE_CONFIG *psDevConfig = psDevNode->psDevConfig;
+
+				/* Client notification of device error will be achieved by
+				 * clients calling UM function RGXGetLastDeviceError() */
+				psDevInfo->eLastDeviceError = eResetReason;
+
+				/* Notify system layer of any error */
+				if (psDevConfig->pfnSysDevErrorNotify)
+				{
+					PVRSRV_ROBUSTNESS_NOTIFY_DATA sErrorData = {0};
+
+					sErrorData.eResetReason = eResetReason;
+
+					psDevConfig->pfnSysDevErrorNotify(psDevConfig,
+													  &sErrorData);
+				}
+			}
+		}
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+static IMG_BOOL _WaitForInterruptsTimeoutCheck(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32idx;
+#endif
+
+	RGXDEBUG_PRINT_IRQ_COUNT(psDevInfo);
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	PVR_DPF((PVR_DBG_ERROR,
+	        "Last RGX_LISRHandler State (DevID %u): 0x%08X Clock: %" IMG_UINT64_FMTSPEC,
+			 psDeviceNode->sDevId.ui32InternalID,
+			 psDeviceNode->sLISRExecutionInfo.ui32Status,
+			 psDeviceNode->sLISRExecutionInfo.ui64Clockns));
+
+	for_each_irq_cnt(ui32idx)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				MSG_IRQ_CNT_TYPE " %u: InterruptCountSnapshot: 0x%X",
+				ui32idx, psDeviceNode->sLISRExecutionInfo.aui32InterruptCountSnapshot[ui32idx]));
+	}
+#else
+	PVR_DPF((PVR_DBG_ERROR, "No further information available. Please enable PVRSRV_DEBUG_LISR_EXECUTION"));
+#endif
+
+	return SampleIRQCount(psDevInfo);
+}
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_BOOL bScheduleMISR;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		bScheduleMISR = IMG_TRUE;
+	}
+	else
+	{
+		bScheduleMISR = _WaitForInterruptsTimeoutCheck(psDevInfo);
+	}
+
+	if (bScheduleMISR)
+	{
+		OSScheduleMISR(psDevInfo->pvMISRData);
+
+		if (psDevInfo->pvAPMISRData != NULL)
+		{
+			OSScheduleMISR(psDevInfo->pvAPMISRData);
+		}
+	}
+}
+
+static inline IMG_BOOL RGXAckHwIrq(PVRSRV_RGXDEV_INFO *psDevInfo,
+								   IMG_UINT32 ui32IRQStatusReg,
+								   IMG_UINT32 ui32IRQStatusEventMsk,
+								   IMG_UINT32 ui32IRQClearReg,
+								   IMG_UINT32 ui32IRQClearMask)
+{
+	IMG_UINT32 ui32IRQStatus = OSReadHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQStatusReg);
+
+	/* clear only the pending bit of the thread that triggered this interrupt */
+	ui32IRQClearMask &= ui32IRQStatus;
+
+	if (ui32IRQStatus & ui32IRQStatusEventMsk)
+	{
+		/* acknowledge and clear the interrupt */
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32IRQClearReg, ui32IRQClearMask);
+		return IMG_TRUE;
+	}
+	else
+	{
+		/* spurious interrupt */
+		return IMG_FALSE;
+	}
+}
+
+static __maybe_unused IMG_BOOL RGXAckIrqMETA(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_EN,
+					   RGX_CR_META_SP_MSLVIRQSTATUS,
+					   RGX_CR_META_SP_MSLVIRQSTATUS_TRIGVECT2_CLRMSK);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static IMG_BOOL RGXAckIrqMIPS(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS,
+					   RGX_CR_MIPS_WRAPPER_IRQ_STATUS_EVENT_EN,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR,
+					   RGX_CR_MIPS_WRAPPER_IRQ_CLEAR_EVENT_EN);
+}
+#endif
+
+static IMG_BOOL RGXAckIrqDedicated(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+		/* status & clearing registers are available on both Host and Guests
+		 * and are agnostic of the Fw CPU type. Due to the remappings done by
+		 * the 2nd stage device MMU, all drivers assume they are accessing
+		 * register bank 0 */
+	return RGXAckHwIrq(psDevInfo,
+					   RGX_CR_IRQ_OS0_EVENT_STATUS,
+					   ~RGX_CR_IRQ_OS0_EVENT_STATUS_SOURCE_CLRMSK,
+					   RGX_CR_IRQ_OS0_EVENT_CLEAR,
+					   ~RGX_CR_IRQ_OS0_EVENT_CLEAR_SOURCE_CLRMSK);
+}
+
+static PVRSRV_ERROR RGXSetAckIrq(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	if ((RGX_IS_FEATURE_SUPPORTED(psDevInfo, IRQ_PER_OS)) && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVINFO, psDevInfo)))
+	{
+		psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psDevInfo->pfnRGXAckIrq = NULL;
+	}
+	else
+	{
+		/* native and host drivers must clear the unique GPU physical interrupt */
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqMIPS;
+		}
+		else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqMETA;
+		}
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			psDevInfo->pfnRGXAckIrq = RGXAckIrqDedicated;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU IRQ clearing mechanism not implemented "
+									"for this architecture.", __func__));
+			return PVRSRV_ERROR_NOT_IMPLEMENTED;
+		}
+	}
+
+#if defined(RGX_IRQ_HYPERV_HANDLER)
+	/* The hypervisor receives and acknowledges the GPU irq, then it injects an
+	 * irq only in the recipient OS. The KM driver doesn't handle the GPU irq line */
+	psDevInfo->pfnRGXAckIrq = NULL;
+#endif
+
+	return PVRSRV_OK;
+}
+
+static IMG_BOOL RGX_LISRHandler(void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bIrqAcknowledged = IMG_FALSE;
+
+#if defined(PVRSRV_DEBUG_LISR_EXECUTION)
+	IMG_UINT32 ui32idx, ui32IrqCnt;
+
+	for_each_irq_cnt(ui32idx)
+	{
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+		UPDATE_LISR_DBG_SNAPSHOT(ui32idx, ui32IrqCnt);
+	}
+
+	UPDATE_LISR_DBG_STATUS(RGX_LISR_INIT);
+	UPDATE_LISR_DBG_TIMESTAMP();
+#endif
+
+	UPDATE_LISR_DBG_COUNTER();
+
+	if (psDevInfo->bRGXPowered)
+	{
+		IMG_BOOL bSafetyEvent = RGXSafetyEventCheck(psDevInfo);
+
+		if ((psDevInfo->pfnRGXAckIrq == NULL) || psDevInfo->pfnRGXAckIrq(psDevInfo) || bSafetyEvent)
+		{
+			bIrqAcknowledged = IMG_TRUE;
+
+			if (bSafetyEvent || SampleIRQCount(psDevInfo))
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_PROCESSED);
+				UPDATE_MISR_DBG_COUNTER();
+
+				OSScheduleMISR(psDevInfo->pvMISRData);
+
+#if defined(SUPPORT_AUTOVZ)
+				RGXUpdateAutoVzWdgToken(psDevInfo);
+#endif
+				if (psDevInfo->pvAPMISRData != NULL)
+				{
+					OSScheduleMISR(psDevInfo->pvAPMISRData);
+				}
+			}
+			else
+			{
+				UPDATE_LISR_DBG_STATUS(RGX_LISR_FW_IRQ_COUNTER_NOT_UPDATED);
+			}
+		}
+		else
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_NOT_TRIGGERED_BY_HW);
+		}
+	}
+	else
+	{
+#if defined(SUPPORT_AUTOVZ)
+		/* AutoVz drivers rebooting while the firmware is active must acknowledge
+		 * and clear the hw IRQ line before the RGXInit() has finished. */
+		if ((psDevInfo->pfnRGXAckIrq != NULL) &&
+			 psDevInfo->pfnRGXAckIrq(psDevInfo))
+		{
+			bIrqAcknowledged = IMG_TRUE;
+		}
+		else
+#endif
+		{
+			UPDATE_LISR_DBG_STATUS(RGX_LISR_DEVICE_NOT_POWERED);
+		}
+	}
+
+	return bIrqAcknowledged;
+}
+
+static void RGX_MISR_ProcessKCCBDeferredList(PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	/* First check whether there are pending commands in Deferred KCCB List */
+	OSSpinLockAcquire(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+	if (dllist_is_empty(&psDevInfo->sKCCBDeferredCommandsListHead))
+	{
+		OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+		return;
+	}
+	OSSpinLockRelease(psDevInfo->hLockKCCBDeferredCommandsList, uiFlags);
+
+	/* Powerlock to avoid further Power transition requests
+	   while KCCB deferred list is being processed */
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to acquire PowerLock (device: %p, error: %s)",
+				 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+		return;
+	}
+
+	/* Try to send deferred KCCB commands Do not Poll from here*/
+	eError = RGXSendCommandsFromDeferredList(psDevInfo, IMG_FALSE);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s could not flush Deferred KCCB list, KCCB is full.",
+				 __func__));
+	}
+}
+
+static void RGX_MISRHandler_CheckFWActivePowerState(void *psDevice)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevice;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_ON || psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+		/* The FW is IDLE and therefore could be shut down */
+		eError = RGXActivePowerRequest(psDeviceNode);
+
+		if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED))
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+					"%s: Failed RGXActivePowerRequest call (device: %p) with %s",
+					__func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+				PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+			}
+			else
+			{
+				/* Re-schedule the power down request as it was deferred. */
+				OSScheduleMISR(psDevInfo->pvAPMISRData);
+			}
+		}
+	}
+
+}
+
+/* Shorter defines to keep the code a bit shorter */
+#define GPU_IDLE       RGXFWIF_GPU_UTIL_STATE_IDLE
+#define GPU_ACTIVE     RGXFWIF_GPU_UTIL_STATE_ACTIVE
+#define GPU_BLOCKED    RGXFWIF_GPU_UTIL_STATE_BLOCKED
+#define GPU_INACTIVE   RGXFWIF_GPU_UTIL_STATE_INACTIVE
+#define MAX_ITERATIONS 64
+#define MAX_DIFF_TIME_NS (300000ULL)
+#define MAX_DIFF_DM_TIME_NS (MAX_DIFF_TIME_NS >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT)
+
+static PVRSRV_ERROR RGXGetGpuUtilStats(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_HANDLE hGpuUtilUser,
+                                       RGXFWIF_GPU_UTIL_STATS *psReturnStats)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_STATS sStats;
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+	IMG_UINT64 (*paaui64DMOSTmpCounters)[RGX_NUM_DRIVERS_SUPPORTED][RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM];
+	IMG_UINT64 (*paui64DMOSTmpLastWord)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastState)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastPeriod)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 (*paui64DMOSTmpLastTime)[RGX_NUM_DRIVERS_SUPPORTED];
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT64 ui64TimeNowShifted;
+	IMG_UINT32 ui32Attempts;
+	IMG_UINT32 ui32Remainder;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT32 ui32MaxDMCount;
+	RGXFWIF_DM eDM;
+
+	/***** (1) Initialise return stats *****/
+
+	psReturnStats->bValid = IMG_FALSE;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	psAggregateStats = hGpuUtilUser;
+
+	/* decrease by 1 to account for excluding GP DM from the statics */;
+	ui32MaxDMCount =  psDevInfo->sDevFeatureCfg.ui32MAXDMCount-1;
+
+	/* Reset temporary counters used in the attempts loop */
+	paaui64DMOSTmpCounters  = &psAggregateStats->sTempGpuStats.aaaui64DMOSTmpCounters[0];
+	paui64DMOSTmpLastWord   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastWord[0];
+	paui64DMOSTmpLastState  = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastState[0];
+	paui64DMOSTmpLastPeriod = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastPeriod[0];
+	paui64DMOSTmpLastTime   = &psAggregateStats->sTempGpuStats.aaui64DMOSTmpLastTime[0];
+
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	/* Try to acquire GPU utilisation counters and repeat if the FW is in the middle of an update */
+	for (ui32Attempts = 0; ui32Attempts < 4; ui32Attempts++)
+	{
+		IMG_UINT64 aui64GpuTmpCounters[RGXFWIF_GPU_UTIL_STATE_NUM] = {0};
+		IMG_UINT64 ui64GpuLastPeriod = 0, ui64GpuLastWord = 0, ui64GpuLastState = 0, ui64GpuLastTime = 0;
+
+		/***** (2) Get latest data from shared area *****/
+
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			IMG_UINT64 aui64StatsCountersNew[RGXFWIF_GPU_UTIL_STATE_NUM];
+			IMG_UINT64 ui64GpuLastWordNew;
+			RGXFWIF_GPU_STATS sStatsNew;
+			IMG_UINT32 i = 0;
+
+			ui64GpuLastWord = 0;
+			ui64GpuLastState = 0;
+
+			OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+			/* Copy data from device memory */
+			memcpy(&sStatsNew, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+			memcpy(&ui64GpuLastWordNew, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+			memcpy(aui64StatsCountersNew, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64StatsCountersNew));
+
+			/*
+			 * First attempt at detecting if the FW is in the middle of an update.
+			 * This should also help if the FW is in the middle of a 64 bit variable update.
+			 * This loop must be fast. Faster than FW updates the stats.
+			 */
+			for (i = 0; i < MAX_ITERATIONS; i++)
+			{
+				IMG_UINT32 j,k;
+				IMG_BOOL bRetry = IMG_FALSE;
+
+				if (i > 0)
+				{
+					/* On retry keep previous data */
+					ui64GpuLastWordNew = ui64GpuLastWord;
+					memcpy(aui64StatsCountersNew, aui64GpuTmpCounters, sizeof(aui64StatsCountersNew));
+					memcpy(&sStatsNew, &sStats, sizeof(sStatsNew));
+				}
+
+				/* Copy data from device memory */
+				memcpy(&sStats, &psDevInfo->psRGXFWIfGpuUtilFW->sStats[ui32DriverID], sizeof(sStats));
+				memcpy(&ui64GpuLastWord, &psDevInfo->psRGXFWIfGpuUtilFW->ui64GpuLastWord, sizeof(ui64GpuLastWord));
+				memcpy(aui64GpuTmpCounters, psDevInfo->psRGXFWIfGpuUtilFW->aui64GpuStatsCounters, sizeof(aui64GpuTmpCounters));
+
+				/* Check for abnormal time difference between reads */
+				if (RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord) - RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWordNew) > MAX_DIFF_TIME_NS)
+				{
+					bRetry = IMG_TRUE;
+					continue;
+				}
+
+				for (j = 0; j < RGXFWIF_GPU_UTIL_STATE_NUM; j++)
+				{
+					/* Check for abnormal time difference between reads */
+					if (aui64GpuTmpCounters[j] - aui64StatsCountersNew[j] > MAX_DIFF_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+				}
+
+				if (bRetry)
+				{
+					continue;
+				}
+
+				/* Check for DM counters wrapped or
+				   abnormal time difference between reads.
+				   The DM time is shifted by RGXFWIF_DM_OS_TIMESTAMP_SHIFT */
+				for (j = 0; j < RGXFWIF_GPU_UTIL_DM_MAX; j++)
+				{
+					if (sStats.aui32DMOSLastWordWrap[j] != sStatsNew.aui32DMOSLastWordWrap[j] ||
+						RGXFWIF_GPU_UTIL_GET_TIME32(sStats.aui32DMOSLastWord[j]) - RGXFWIF_GPU_UTIL_GET_TIME32(sStatsNew.aui32DMOSLastWord[j]) > MAX_DIFF_DM_TIME_NS)
+					{
+						bRetry = IMG_TRUE;
+						break;
+					}
+
+					for (k = 0; k < RGXFWIF_GPU_UTIL_REDUCED_STATES_NUM; k++)
+					{
+						if (sStats.aaui32DMOSCountersWrap[j][k] != sStatsNew.aaui32DMOSCountersWrap[j][k] ||
+							sStats.aaui32DMOSStatsCounters[j][k] - sStatsNew.aaui32DMOSStatsCounters[j][k] > MAX_DIFF_DM_TIME_NS)
+						{
+							bRetry = IMG_TRUE;
+							break;
+						}
+
+					}
+
+					if (bRetry)
+					{
+						break;
+					}
+				}
+
+				if (!bRetry)
+				{
+					/* Stats are good*/
+					break;
+				}
+			}
+
+			OSLockRelease(psDevInfo->hGPUUtilLock);
+
+			ui64GpuLastState = RGXFWIF_GPU_UTIL_GET_STATE(ui64GpuLastWord);
+
+			if (i == MAX_ITERATIONS)
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+						 "RGXGetGpuUtilStats could not get reliable data after trying %u times", i));
+
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+
+			for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+			{
+				paui64DMOSTmpLastWord[eDM][ui32DriverID]  =
+					((IMG_UINT64)sStats.aui32DMOSLastWordWrap[eDM] << 32) + sStats.aui32DMOSLastWord[eDM];
+				paui64DMOSTmpLastState[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_STATE(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				if (paui64DMOSTmpLastState[eDM][ui32DriverID] != GPU_ACTIVE)
+				{
+					paui64DMOSTmpLastState[eDM][ui32DriverID] = GPU_INACTIVE;
+				}
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE] = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_INACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_INACTIVE] << 32);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE]  = (IMG_UINT64)sStats.aaui32DMOSStatsCounters[eDM][GPU_ACTIVE] +
+					((IMG_UINT64)sStats.aaui32DMOSCountersWrap[eDM][GPU_ACTIVE] << 32);
+			}
+
+		} /* FOREACH_SUPPORTED_DRIVER(ui32DriverID) */
+
+
+		/***** (3) Compute return stats *****/
+
+		/* Update temp counters to account for the time since the last update to the shared ones */
+		OSMemoryBarrier(NULL); /* Ensure the current time is read after the loop above */
+		ui64TimeNow    = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDeviceNode));
+
+		ui64GpuLastTime   = RGXFWIF_GPU_UTIL_GET_TIME(ui64GpuLastWord);
+		ui64GpuLastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64GpuLastTime);
+		aui64GpuTmpCounters[ui64GpuLastState] += ui64GpuLastPeriod;
+
+		/* Get statistics for a user since its last request */
+		psReturnStats->ui64GpuStatIdle = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_IDLE],
+		                                                             psAggregateStats->ui64GpuStatIdle);
+		psReturnStats->ui64GpuStatActive = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_ACTIVE],
+		                                                               psAggregateStats->ui64GpuStatActive);
+		psReturnStats->ui64GpuStatBlocked = RGXFWIF_GPU_UTIL_GET_PERIOD(aui64GpuTmpCounters[GPU_BLOCKED],
+		                                                                psAggregateStats->ui64GpuStatBlocked);
+		psReturnStats->ui64GpuStatCumulative = psReturnStats->ui64GpuStatIdle +
+		                                       psReturnStats->ui64GpuStatActive +
+		                                       psReturnStats->ui64GpuStatBlocked;
+
+		/* convert time into the same units as used by fw */
+		ui64TimeNowShifted  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+		for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+		{
+			FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+			{
+				paui64DMOSTmpLastTime[eDM][ui32DriverID]   = RGXFWIF_GPU_UTIL_GET_TIME(paui64DMOSTmpLastWord[eDM][ui32DriverID]);
+				paui64DMOSTmpLastPeriod[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNowShifted , paui64DMOSTmpLastTime[eDM][ui32DriverID]);
+				paaui64DMOSTmpCounters[eDM][ui32DriverID][paui64DMOSTmpLastState[eDM][ui32DriverID]] += paui64DMOSTmpLastPeriod[eDM][ui32DriverID];
+				/* Get statistics for a user since its last request */
+				psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_INACTIVE],
+				                                                             psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID] = RGXFWIF_GPU_UTIL_GET_PERIOD(paaui64DMOSTmpCounters[eDM][ui32DriverID][GPU_ACTIVE],
+				                                                               psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]);
+				psReturnStats->aaui64DMOSStatCumulative[eDM][ui32DriverID] = psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID] +
+				                                       psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+			}
+		}
+
+		if (psAggregateStats->ui64TimeStamp != 0)
+		{
+			IMG_UINT64 ui64TimeSinceLastCall = ui64TimeNow - psAggregateStats->ui64TimeStamp;
+			/* We expect to return at least 75% of the time since the last call in GPU stats */
+			IMG_UINT64 ui64MinReturnedStats = ui64TimeSinceLastCall - (ui64TimeSinceLastCall / 4);
+
+			/*
+			 * If the returned stats are substantially lower than the time since
+			 * the last call, then the Host might have read a partial update from the FW.
+			 * If this happens, try sampling the shared counters again.
+			 */
+			if (psReturnStats->ui64GpuStatCumulative < ui64MinReturnedStats)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE,
+				         "%s: Return stats (%" IMG_UINT64_FMTSPEC ") too low "
+				         "(call period %" IMG_UINT64_FMTSPEC ")",
+				         __func__, psReturnStats->ui64GpuStatCumulative, ui64TimeSinceLastCall));
+				PVR_DPF((PVR_DBG_MESSAGE, "%s: Attempt #%u has failed, trying again",
+				         __func__, ui32Attempts));
+				continue;
+			}
+		}
+
+		break;
+	}
+
+	/***** (4) Update aggregate stats for the current user *****/
+
+	psAggregateStats->ui64GpuStatIdle    += psReturnStats->ui64GpuStatIdle;
+	psAggregateStats->ui64GpuStatActive  += psReturnStats->ui64GpuStatActive;
+	psAggregateStats->ui64GpuStatBlocked += psReturnStats->ui64GpuStatBlocked;
+	psAggregateStats->ui64TimeStamp       = ui64TimeNow;
+
+	for (eDM = 0; eDM < ui32MaxDMCount; eDM++)
+	{
+		FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+		{
+			psAggregateStats->aaui64DMOSStatInactive[eDM][ui32DriverID]    += psReturnStats->aaui64DMOSStatInactive[eDM][ui32DriverID];
+			psAggregateStats->aaui64DMOSStatActive[eDM][ui32DriverID]  += psReturnStats->aaui64DMOSStatActive[eDM][ui32DriverID];
+		}
+	}
+
+	/***** (5) Convert return stats to microseconds *****/
+
+	psReturnStats->ui64GpuStatIdle       = OSDivide64(psReturnStats->ui64GpuStatIdle, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatActive     = OSDivide64(psReturnStats->ui64GpuStatActive, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatBlocked    = OSDivide64(psReturnStats->ui64GpuStatBlocked, 1000, &ui32Remainder);
+	psReturnStats->ui64GpuStatCumulative = OSDivide64(psReturnStats->ui64GpuStatCumulative, 1000, &ui32Remainder);
+
+	/* Check that the return stats make sense */
+	if (psReturnStats->ui64GpuStatCumulative == 0)
+	{
+		/* We can enter here only if allocating the temporary stats
+		 * buffers failed, or all the RGXFWIF_GPU_UTIL_GET_PERIOD
+		 * returned 0. The latter could happen if the GPU frequency value
+		 * is not well calibrated and the FW is updating the GPU state
+		 * while the Host is reading it.
+		 * When such an event happens frequently, timers or the aggregate
+		 * stats might not be accurate...
+		 */
+#if defined(VIRTUAL_PLATFORM)
+		/* To avoid spamming the console logging system on emulated devices,
+		 * we special-case so that we will only produce a single message per
+		 * driver invocation. This should reduce the time spent logging
+		 * information which is not relevant for very slow timers found in
+		 * VP device configurations
+		 */
+		static IMG_BOOL bFirstTime = IMG_TRUE;
+
+		if (bFirstTime)
+		{
+			bFirstTime = IMG_FALSE;
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "RGXGetGpuUtilStats could not get reliable data."));
+#if defined(VIRTUAL_PLATFORM)
+		}
+#endif	/* defined(VIRTUAL_PLATFORM) */
+		return PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	psReturnStats->bValid = IMG_TRUE;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	/* NoStats used since this may be called outside of the register/de-register
+	 * process calls which track memory use. */
+	psAggregateStats = OSAllocZMemNoStats(sizeof(RGXFWIF_GPU_UTIL_STATS));
+	if (psAggregateStats == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	*phGpuUtilUser = psAggregateStats;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser)
+{
+	RGXFWIF_GPU_UTIL_STATS *psAggregateStats;
+
+	if (hGpuUtilUser == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAggregateStats = hGpuUtilUser;
+	OSFreeMemNoStats(psAggregateStats);
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGX MISR Handler
+*/
+static void RGX_MISRHandler_Main (void *pvData)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = pvData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	/* Give the HWPerf service a chance to transfer some data from the FW
+	 * buffer to the host driver transport layer buffer.
+	 */
+	RGXHWPerfDataStoreCB(psDeviceNode);
+
+	/* Inform other services devices that we have finished an operation */
+	PVRSRVNotifyCommandCompletion(psDeviceNode);
+
+#if defined(SUPPORT_PDVFS) && defined(RGXFW_META_SUPPORT_2ND_THREAD)
+	/* Normally, firmware CCB only exists for the primary FW thread unless PDVFS
+	   is running on the second[ary] FW thread, here we process said CCB */
+	RGXPDVFSCheckCoreClkRateChange(psDeviceNode->pvDevice);
+#endif
+
+	/* Only execute SafetyEventHandler if RGX_FEATURE_SAFETY_EVENT is on */
+	if (PVRSRV_GET_DEVICE_FEATURE_VALUE(psDeviceNode, ECC_RAMS) > 0 ||
+		PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, WATCHDOG_TIMER))
+	{
+		/* Handle Safety events if necessary */
+		RGXSafetyEventHandler(psDeviceNode->pvDevice);
+	}
+
+	/* Signal the global event object */
+	PVRSRVSignalDriverWideEO();
+
+	/* Process the Firmware CCB for pending commands */
+	RGXCheckFirmwareCCB(psDeviceNode->pvDevice);
+
+	/* Calibrate the GPU frequency and recorrelate Host and GPU timers (done every few seconds) */
+	RGXTimeCorrRestartPeriodic(psDeviceNode);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Process Workload Estimation Specific commands from the FW */
+		WorkEstCheckFirmwareCCB(psDeviceNode->pvDevice);
+	}
+#endif
+
+	if (psDevInfo->pvAPMISRData == NULL)
+	{
+		RGX_MISR_ProcessKCCBDeferredList(psDeviceNode);
+	}
+}
+#endif /* !defined(NO_HARDWARE) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+static PVRSRV_ERROR RGXPDumpBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+		PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PMR *psFWDataPMR;
+	RGXMIPSFW_BOOT_DATA *psBootData;
+	IMG_DEV_PHYADDR sTmpAddr;
+	IMG_UINT32 ui32BootConfOffset, ui32ParamOffset, i;
+	PVRSRV_ERROR eError;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+	ui32BootConfOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+	ui32BootConfOffset += RGXMIPSFW_BOOTLDR_CONF_OFFSET;
+
+	/* The physical addresses used by a pdump player will be different
+	 * than the ones we have put in the MIPS bootloader configuration data.
+	 * We have to tell the pdump player to replace the original values with the real ones.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Pass new boot parameters to the FW");
+
+	/* Rogue Registers physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64RegBase);
+
+	eError = PDumpRegLabelToMem64(RGX_PDUMPREG_NAME,
+			0x0,
+			psFWDataPMR,
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of Rogue registers phy address failed (%u)", eError));
+		return eError;
+	}
+
+	/* Page Table physical Address */
+	eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sTmpAddr);
+	if (eError !=  PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXBootldrDataInit: MMU_AcquireBaseAddr failed (%u)",
+		         eError));
+		return eError;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+									 (void **)&psBootData);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire pointer to FW data (%s)",
+				__func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	psBootData = IMG_OFFSET_ADDR(psBootData, ui32BootConfOffset);
+
+	for (i = 0; i < psBootData->ui32PTNumPages; i++)
+	{
+		ui32ParamOffset = ui32BootConfOffset +
+			offsetof(RGXMIPSFW_BOOT_DATA, aui64PTPhyAddr[0])
+			+ i * sizeof(psBootData->aui64PTPhyAddr[0]);
+
+		eError = PDumpPTBaseObjectToMem64(psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				psFWDataPMR,
+				0,
+				ui32ParamOffset,
+				PDUMP_FLAGS_CONTINUOUS,
+				MMU_LEVEL_1,
+				sTmpAddr.uiAddr,
+				i << psBootData->ui32PTLog2PageSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of page tables phy address failed (%u)", eError));
+			return eError;
+		}
+	}
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+
+	/* Stack physical address */
+	ui32ParamOffset = ui32BootConfOffset + offsetof(RGXMIPSFW_BOOT_DATA, ui64StackPhyAddr);
+
+	eError = PDumpMemLabelToMem64(psFWDataPMR,
+			psFWDataPMR,
+			RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+			ui32ParamOffset,
+			PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPDumpBootldrData: Dump of stack phy address failed (%u)", eError));
+		return eError;
+	}
+
+	return eError;
+}
+#endif /* PDUMP */
+
+static PVRSRV_ERROR RGXSetPowerParams(PVRSRV_RGXDEV_INFO   *psDevInfo,
+                                      PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEV_PHYADDR sKernelMMUCtxPCAddr;
+	IMG_BOOL bPremappedFw;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVCFG, psDevConfig, PVRSRV_OK);
+
+	/* Save information used on power transitions for later
+	 * (when RGXStart and RGXStop are executed)
+	 */
+#if defined(PDUMP)
+	psDevInfo->sLayerParams.ui32PdumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_PREMAP_FW_HEAPS)
+	/* Rogue drivers with security support and premapped fw heaps
+	 * always have their fw heap premapped by the TEE */
+	bPremappedFw = IMG_TRUE;
+#else
+	/* If AutoVz firmware is up at this stage, the driver initialised it
+	 * during a previous life-cycle. The firmware's memory is already pre-mapped
+	 * and the MMU page tables reside in the predetermined memory carveout.
+	 * The Kernel MMU Context created in this life-cycle is a dummy structure
+	 * that is not used for mapping.
+	 * To program the Device's BIF with the correct PC address, use the base
+	 * address of the carveout reserved for MMU mappings as Kernel MMU PC Address */
+	bPremappedFw = psDevInfo->psDeviceNode->bAutoVzFwIsUp;
+#endif
+
+	if (bPremappedFw)
+	{
+		IMG_DEV_PHYADDR sDevPAddr;
+		PHYS_HEAP *psFwPageTableHeap = psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_RETURN_IF_FALSE((NULL != psFwPageTableHeap),
+								"Firmware Page Table heap not defined.",
+								PVRSRV_ERROR_INVALID_HEAP);
+
+		PhysHeapGetDevPAddr(psFwPageTableHeap, &sDevPAddr);
+		sKernelMMUCtxPCAddr.uiAddr = sDevPAddr.uiAddr;
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+	else if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+	    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+	{
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx,
+		                             &sKernelMMUCtxPCAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire Kernel MMU Ctx page catalog"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sPCAddr = sKernelMMUCtxPCAddr;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		PMR *psFWCodePMR = (PMR *)(psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR);
+		PMR *psFWDataPMR = (PMR *)(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR);
+		IMG_DEV_PHYADDR sPhyAddr;
+		IMG_BOOL bValid;
+
+#if defined(SUPPORT_ALT_REGBASE)
+		psDevInfo->sLayerParams.sGPURegAddr = psDevConfig->sAltRegsGpuPBase;
+#else
+		/* The physical address of the GPU registers needs to be translated
+		 * in case we are in a LMA scenario
+		 */
+		PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+				1,
+				&sPhyAddr,
+				&(psDevConfig->sRegsCpuPBase));
+
+		psDevInfo->sLayerParams.sGPURegAddr = sPhyAddr;
+#endif
+
+		/* Register bank must be aligned to 512KB (as per the core integration) to
+		 * prevent the FW accessing incorrect registers */
+		if ((psDevInfo->sLayerParams.sGPURegAddr.uiAddr & 0x7FFFFU) != 0U)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Register bank must be aligned to 512KB, but current address (0x%016"IMG_UINT64_FMTSPECX") is not",
+						psDevInfo->sLayerParams.sGPURegAddr.uiAddr));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE),
+				OSGetPageShift(), /* FW will be using the same page size as the OS */
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI code address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sBootRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWDataPMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW boot/NMI data address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sDataRemapAddr = sPhyAddr;
+
+		eError = RGXGetPhyAddr(psFWCodePMR,
+				&sPhyAddr,
+				RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE),
+				OSGetPageShift(),
+				1,
+				&bValid);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: Failed to acquire FW exceptions address"));
+			return eError;
+		}
+
+		psDevInfo->sLayerParams.sCodeRemapAddr = sPhyAddr;
+
+		psDevInfo->sLayerParams.sTrampolineRemapAddr.uiAddr = psDevInfo->psTrampoline->sPhysAddr.uiAddr;
+
+		psDevInfo->sLayerParams.bDevicePA0IsValid = psDevConfig->bDevicePA0IsValid;
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	/* Send information used on power transitions to the trusted device as
+	 * in this setup the driver cannot start/stop the GPU and perform resets
+	 */
+	if (psDevConfig->pfnTDSetPowerParams)
+	{
+		PVRSRV_TD_POWER_PARAMS sTDPowerParams;
+
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+		    RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR))
+		{
+			sTDPowerParams.sPCAddr = psDevInfo->sLayerParams.sPCAddr;
+		}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+		{
+			sTDPowerParams.sGPURegAddr    = psDevInfo->sLayerParams.sGPURegAddr;
+			sTDPowerParams.sBootRemapAddr = psDevInfo->sLayerParams.sBootRemapAddr;
+			sTDPowerParams.sCodeRemapAddr = psDevInfo->sLayerParams.sCodeRemapAddr;
+			sTDPowerParams.sDataRemapAddr = psDevInfo->sLayerParams.sDataRemapAddr;
+		}
+#endif
+
+		eError = psDevConfig->pfnTDSetPowerParams(psDevConfig->hSysData,
+												  &sTDPowerParams);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXSetPowerParams: TDSetPowerParams not implemented!"));
+		eError = PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+#endif
+
+	return eError;
+}
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*
+	RGXSystemGetFabricCoherency
+*/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDevConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode)
+{
+	IMG_CHAR *aszLabels[] = {"none", "acelite", "fullace", "unknown"};
+	PVRSRV_DEVICE_SNOOP_MODE eAppHintCacheSnoopingMode;
+	PVRSRV_DEVICE_SNOOP_MODE eDeviceCacheSnoopingMode;
+	IMG_UINT32 ui32AppHintFabricCoherency;
+	IMG_UINT32 ui32DeviceFabricCoherency;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+#if !defined(NO_HARDWARE)
+	void *pvRegsBaseKM;
+	IMG_BOOL bPowerDown = IMG_TRUE;
+	PVRSRV_ERROR eError;
+#endif
+
+	if (!sRegsCpuPBase.uiAddr || !ui32RegsSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Invalid RGX register base/size parameters"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(NO_HARDWARE)
+	pvRegsBaseKM = OSMapPhysToLin(sRegsCpuPBase, ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+	if (!pvRegsBaseKM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXSystemGetFabricCoherency: Failed to create RGX register mapping"));
+		return PVRSRV_ERROR_BAD_MAPPING;
+	}
+
+	bPowerDown = ! PVRSRVIsSystemPowered(psDevConfig->psDevNode);
+
+	/* Power-up the device as required to read the registers */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState ON");
+	}
+
+	/* AXI support within the SoC, bitfield COHERENCY_SUPPORT [1 .. 0]
+		value NO_COHERENCY        0x0 {SoC does not support any form of Coherency}
+		value ACE_LITE_COHERENCY  0x1 {SoC supports ACE-Lite or I/O Coherency}
+		value FULL_ACE_COHERENCY  0x2 {SoC supports full ACE or 2-Way Coherency} */
+	ui32DeviceFabricCoherency = OSReadHWReg32((void __iomem *)pvRegsBaseKM, RGX_CR_SOC_AXI);
+	PVR_LOG(("AXI fabric coherency (RGX_CR_SOC_AXI): 0x%x", ui32DeviceFabricCoherency));
+#if defined(DEBUG)
+	if (ui32DeviceFabricCoherency & ~((IMG_UINT32)RGX_CR_SOC_AXI_MASKFULL))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid RGX_CR_SOC_AXI value.", __func__));
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+#endif
+	ui32DeviceFabricCoherency &= ~((IMG_UINT32)RGX_CR_SOC_AXI_COHERENCY_SUPPORT_CLRMSK);
+	ui32DeviceFabricCoherency >>= RGX_CR_SOC_AXI_COHERENCY_SUPPORT_SHIFT;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVCFG, psDevConfig) && bPowerDown)
+	{
+		eError = PVRSRVSetSystemPowerState(psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+	}
+
+	/* UnMap Regs */
+	OSUnMapPhysToLin(pvRegsBaseKM, ui32RegsSize);
+
+	switch (ui32DeviceFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_FULLACE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+	default:
+		eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		*peDevFabricType = PVRSRV_DEVICE_FABRIC_NONE;
+		break;
+	}
+#else /* !defined(NO_HARDWARE) */
+    *peDevFabricType = PVRSRV_DEVICE_FABRIC_ACELITE;
+    eDeviceCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+    ui32DeviceFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY;
+#endif /* !defined(NO_HARDWARE) */
+
+	OSCreateAppHintState(&pvAppHintState);
+	ui32AppHintDefault = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, FabricCoherencyOverride,
+						 &ui32AppHintDefault, &ui32AppHintFabricCoherency);
+	OSFreeAppHintState(pvAppHintState);
+
+#if defined(SUPPORT_SECURITY_VALIDATION)
+	/* Temporarily disable coherency */
+	ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY;
+#endif
+
+	/* Suppress invalid AppHint value */
+	switch (ui32AppHintFabricCoherency)
+	{
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_NO_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_NONE;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_ACE_LITE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+		break;
+
+	case RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY:
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"Invalid FabricCoherencyOverride AppHint %d, ignoring",
+				ui32AppHintFabricCoherency));
+		eAppHintCacheSnoopingMode = PVRSRV_DEVICE_SNOOP_CROSS;
+		ui32AppHintFabricCoherency = RGX_CR_SOC_AXI_COHERENCY_SUPPORT_FULL_ACE_COHERENCY;
+		break;
+	}
+
+	if (ui32AppHintFabricCoherency < ui32DeviceFabricCoherency)
+	{
+		PVR_LOG(("Downgrading device fabric coherency from %s to %s",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+		eDeviceCacheSnoopingMode = eAppHintCacheSnoopingMode;
+	}
+	else if (ui32AppHintFabricCoherency > ui32DeviceFabricCoherency)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Cannot upgrade device fabric coherency from %s to %s, not supported by device!",
+				aszLabels[ui32DeviceFabricCoherency],
+				aszLabels[ui32AppHintFabricCoherency]));
+
+		/* Override requested-for app-hint with actual app-hint value being used */
+		ui32AppHintFabricCoherency = ui32DeviceFabricCoherency;
+	}
+
+	*peCacheSnoopingMode = eDeviceCacheSnoopingMode;
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXSystemHasFBCDCVersion31
+*/
+static IMG_BOOL RGXSystemHasFBCDCVersion31(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(HW_ERN_66622_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_ERN_SUPPORTED(psDevInfo, 66622))
+#endif
+	{
+		{
+			if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+			{
+				return IMG_TRUE;
+			}
+		}
+	}
+#if defined(HW_ERN_66622_BIT_MASK)
+	else
+	{
+
+#if !defined(NO_HARDWARE)
+		if (psDeviceNode->psDevConfig->bHasFBCDCVersion31)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: System uses FBCDC3.1 but GPU doesn't support it!",
+			         __func__));
+		}
+#endif
+	}
+#endif /* defined(HW_ERN_66622_BIT_MASK) */
+
+	return IMG_FALSE;
+}
+
+/*
+	RGXGetTFBCLossyGroup
+*/
+static IMG_UINT32 RGXGetTFBCLossyGroup(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	return psDevInfo->ui32TFBCLossyGroup;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	return 0;
+#endif
+}
+
+/*
+	RGXDevMMUAttributes
+*/
+static MMU_DEVICEATTRIBS *RGXDevMMUAttributes(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              IMG_BOOL bKernelFWMemoryCtx)
+{
+	MMU_DEVICEATTRIBS *psMMUDevAttrs = NULL;
+
+	if (psDeviceNode->pfnCheckDeviceFeature)
+	{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		if (PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, MIPS))
+		{
+			psMMUDevAttrs = bKernelFWMemoryCtx ?
+			                psDeviceNode->psFirmwareMMUDevAttrs :
+			                psDeviceNode->psMMUDevAttrs;
+		}
+		else
+#endif
+		{
+			PVR_UNREFERENCED_PARAMETER(bKernelFWMemoryCtx);
+			psMMUDevAttrs = psDeviceNode->psMMUDevAttrs;
+		}
+	}
+
+	return psMMUDevAttrs;
+}
+
+/*
+	RGXDevSnoopMode
+*/
+static PVRSRV_DEVICE_SNOOP_MODE RGXDevSnoopMode(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+	{
+		return PVRSRV_DEVICE_SNOOP_CPU_ONLY;
+	}
+
+	return PVRSRV_DEVICE_SNOOP_NONE;
+}
+
+/*
+ * RGXInitDevPart2
+ */
+PVRSRV_ERROR RGXInitDevPart2(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                             RGX_INIT_APPHINTS	*psApphints)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_DEV_POWER_STATE	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	PVRSRV_DEVICE_CONFIG	*psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32			ui32AllPowUnitsMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount) - 1;
+	IMG_UINT32			ui32AllRACMask = (1 << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 2");
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(PDUMP)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		RGXPDumpBootldrData(psDeviceNode, psDevInfo);
+	}
+#endif
+#if defined(TIMING) || defined(DEBUG)
+	OSUserModeAccessToPerfCountersEn();
+#endif
+
+	/* Initialise Device Flags */
+	psDevInfo->ui32DeviceFlags = 0;
+	RGXSetDeviceFlags(psDevInfo, psApphints->ui32DeviceFlags, IMG_TRUE);
+
+	/* Allocate DVFS Table (needs to be allocated before GPU trace events
+	 *  component is initialised because there is a dependency between them) */
+	psDevInfo->psGpuDVFSTable = OSAllocZMem(sizeof(*(psDevInfo->psGpuDVFSTable)));
+	PVR_LOG_GOTO_IF_NOMEM(psDevInfo->psGpuDVFSTable, eError, ErrorExit);
+
+	if (psDevInfo->ui32HWPerfHostFilter == 0)
+	{
+		RGXHWPerfHostSetEventFilter(psDevInfo, psApphints->ui32HWPerfHostFilter);
+	}
+
+	/* If HWPerf enabled allocate all resources for the host side buffer. */
+	if (psDevInfo->ui32HWPerfHostFilter != 0)
+	{
+		if (RGXHWPerfHostInitOnDemandResources(psDevInfo) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "HWPerfHost buffer on demand"
+			        " initialisation failed."));
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Initialise work estimation lock */
+		eError = OSLockCreate(&psDevInfo->hWorkEstLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(WorkEstLock)", ErrorExit);
+	}
+#endif
+
+	/* Initialise lists of ZSBuffers */
+	eError = OSLockCreate(&psDevInfo->hLockZSBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockZSBuffer)", ErrorExit);
+	dllist_init(&psDevInfo->sZSBufferHead);
+	psDevInfo->ui32ZSBufferCurrID = 1;
+
+	/* Initialise lists of growable Freelists */
+	eError = OSLockCreate(&psDevInfo->hLockFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(LockFreeList)", ErrorExit);
+	dllist_init(&psDevInfo->sFreeListHead);
+	psDevInfo->ui32FreelistCurrID = 1;
+
+	eError = OSLockCreate(&psDevInfo->hDebugFaultInfoLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(DebugFaultInfoLock)", ErrorExit);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		eError = OSLockCreate(&psDevInfo->hMMUCtxUnregLock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(MMUCtxUnregLock)", ErrorExit);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = OSLockCreate(&psDevInfo->hNMILock);
+		PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(NMILock)", ErrorExit);
+	}
+#endif
+
+	/* Setup GPU utilisation stats update callback */
+	eError = OSLockCreate(&psDevInfo->hGPUUtilLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate(GPUUtilLock)", ErrorExit);
+#if !defined(NO_HARDWARE)
+	psDevInfo->pfnGetGpuUtilStats = RGXGetGpuUtilStats;
+#endif
+
+	eDefaultPowerState = PVRSRV_DEV_POWER_STATE_ON;
+	psDevInfo->eActivePMConf = psApphints->eRGXActivePMConf;
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	/* Validate the SPU mask and initialize to number of SPUs to power up */
+	if ((psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s:Invalid SPU mask (All=0x%X, Non Fused=0x%X). At-least one SPU must to be powered up.",
+		         __func__,
+		         ui32AllPowUnitsMask,
+		         psApphints->ui32AvailablePowUnitsMask));
+		PVR_LOG_GOTO_WITH_ERROR("ui32AvailablePowUnitsMask", eError, PVRSRV_ERROR_INVALID_SPU_MASK, ErrorExit);
+	}
+
+	psDevInfo->ui32AvailablePowUnitsMask = psApphints->ui32AvailablePowUnitsMask & ui32AllPowUnitsMask;
+
+	psDevInfo->ui32AvailableRACMask = psApphints->ui32AvailableRACMask & ui32AllRACMask;
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* set-up the Active Power Mgmt callback */
+	{
+		RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+		IMG_BOOL bSysEnableAPM = psRGXData->psRGXTimingInfo->bEnableActivePM;
+		IMG_BOOL bEnableAPM = ((psApphints->eRGXActivePMConf == RGX_ACTIVEPM_DEFAULT) && bSysEnableAPM) ||
+							   (psApphints->eRGXActivePMConf == RGX_ACTIVEPM_FORCE_ON);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1) && defined(SUPPORT_AUTOVZ)
+		/* The AutoVz driver enables a virtualisation watchdog not compatible with APM */
+		if (bEnableAPM && (!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)))
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Active Power Management disabled in AutoVz mode", __func__));
+			bEnableAPM = IMG_FALSE;
+		}
+
+		PVR_ASSERT(bEnableAPM == IMG_FALSE);
+#endif
+
+		if (bEnableAPM)
+		{
+			eError = OSInstallMISR(&psDevInfo->pvAPMISRData,
+					RGX_MISRHandler_CheckFWActivePowerState,
+					psDeviceNode,
+					"RGX_CheckFWActivePower");
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(APMISR)", ErrorExit);
+
+			/* Prevent the device being woken up before there is something to do. */
+			eDefaultPowerState = PVRSRV_DEV_POWER_STATE_OFF;
+		}
+	}
+#endif
+
+	psDevInfo->eDebugDumpFWTLogType = psApphints->eDebugDumpFWTLogType;
+
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableAPM,
+	                                    RGXQueryAPMState,
+	                                    RGXSetAPMState,
+	                                    psDeviceNode,
+	                                    NULL);
+
+	RGXTimeCorrInitAppHintCallbacks(psDeviceNode);
+
+	/* Register the device with the power manager */
+	eError = PVRSRVRegisterPowerDevice(psDeviceNode,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPrePowerState : &RGXVzPrePowerState,
+	                                   (PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode)) ? &RGXPostPowerState : &RGXVzPostPowerState,
+	                                   psDevConfig->pfnPrePowerState, psDevConfig->pfnPostPowerState,
+	                                   &RGXPreClockSpeedChange, &RGXPostClockSpeedChange,
+	                                   &RGXForcedIdleRequest, &RGXCancelForcedIdleRequest,
+	                                   &RGXPowUnitsChange,
+	                                   (IMG_HANDLE)psDeviceNode,
+	                                   PVRSRV_DEV_POWER_STATE_OFF,
+	                                   eDefaultPowerState);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVRegisterPowerDevice", ErrorExit);
+
+	eError = RGXSetPowerParams(psDevInfo, psDevConfig);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetPowerParams", ErrorExit);
+
+
+#if defined(PDUMP)
+#if defined(NO_HARDWARE)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT, "Wait for the FW to signal idle");
+
+	/* Kick the FW once, in case it still needs to detect and set the idle state */
+	PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+			   RGX_CR_MTS_SCHEDULE,
+			   RGXFWIF_DM_GP & ~RGX_CR_MTS_SCHEDULE_DM_CLRMSK,
+			   PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfFwSysDataMemDesc,
+	                                offsetof(RGXFWIF_SYSDATA, ePowState),
+	                                RGXFWIF_POW_IDLE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS | PDUMP_FLAGS_DEINIT);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", ErrorExit);
+#endif
+
+	/* Run RGXStop with the correct PDump flags to feed the last-frame deinit buffer */
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_DEINIT,
+	                      "RGX deinitialisation commands");
+
+	psDevInfo->sLayerParams.ui32PdumpFlags |= PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW;
+
+	if (! PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXStop(&psDevInfo->sLayerParams);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXStop", ErrorExit);
+	}
+
+	psDevInfo->sLayerParams.ui32PdumpFlags &= ~(PDUMP_FLAGS_DEINIT | PDUMP_FLAGS_NOHW);
+#endif
+
+#if !defined(NO_HARDWARE)
+	eError = RGXInstallProcessQueuesMISR(&psDevInfo->hProcessQueuesMISR, psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInstallProcessQueuesMISR", ErrorExit);
+
+	/* Register RGX to receive notifies when other devices complete some work */
+	PVRSRVRegisterCmdCompleteNotify(&psDeviceNode->hCmdCompNotify, &RGXScheduleProcessQueuesKM, psDeviceNode);
+
+	/* Register the interrupt handlers */
+	eError = OSInstallMISR(&psDevInfo->pvMISRData,
+			RGX_MISRHandler_Main,
+			psDeviceNode,
+			"RGX_Main");
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSInstallMISR(MISR)", ErrorExit);
+
+	/* Register appropriate mechanism for clearing hw interrupts */
+	eError = RGXSetAckIrq(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetAckIrq", ErrorExit);
+
+	eError = SysInstallDeviceLISR(psDevConfig->hSysData,
+								  psDevConfig->ui32IRQ,
+								  PVRSRV_MODNAME,
+								  RGX_LISRHandler,
+								  psDeviceNode,
+								  &psDevInfo->pvLISRData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "SysInstallDeviceLISR", ErrorExit);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(PDUMP)
+    if (PVRSRVSystemSnoopingOfCPUCache(psDevConfig))
+    {
+        PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+                              "System has CPU cache snooping");
+    }
+    else
+    {
+        PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+                              "System has NO cache snooping");
+    }
+#endif
+
+#if defined(RGX_FEATURE_COMPUTE_ONLY_BIT_MASK)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE_ONLY))
+#endif
+	{
+		eError = PVRSRVTQLoadShaders(psDeviceNode);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVTQLoadShaders", ErrorExit);
+	}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	eError = OSAllocateSecBuf(psDeviceNode, RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE, "SharedSecMem", &psDevInfo->psGenHeapSecMem);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSAllocateSecBuf", ErrorExit);
+#endif
+
+	psDevInfo->bDevInit2Done = IMG_TRUE;
+
+	return PVRSRV_OK;
+
+ErrorExit:
+	DevPart2DeInitRGX(psDeviceNode);
+
+	return eError;
+}
+
+#define VZ_RGX_FW_FILENAME_SUFFIX ".vz"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#define RGX_64K_FW_FILENAME_SUFFIX ".64k"
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX) + sizeof(RGX_64K_FW_FILENAME_SUFFIX)))
+#else
+#define RGX_FW_FILENAME_MAX_SIZE   ((sizeof(RGX_FW_FILENAME)+ \
+			RGX_BVNC_STR_SIZE_MAX+sizeof(VZ_RGX_FW_FILENAME_SUFFIX)))
+#endif
+
+static void _GetFWFileName(PVRSRV_DEVICE_NODE *psDeviceNode,
+		IMG_CHAR *pszFWFilenameStr,
+		IMG_CHAR *pszFWpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const IMG_CHAR * const pszFWFilenameSuffix =
+			PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) ? "" : VZ_RGX_FW_FILENAME_SUFFIX;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	const IMG_CHAR * const pszFWFilenameSuffix2 =
+			((OSGetPageSize() == RGX_MMU_PAGE_SIZE_64KB) &&
+			 RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+			? RGX_64K_FW_FILENAME_SUFFIX : "";
+#else
+	const IMG_CHAR * const pszFWFilenameSuffix2 = "";
+#endif
+
+	OSSNPrintf(pszFWFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STR_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+
+	OSSNPrintf(pszFWpFilenameStr, RGX_FW_FILENAME_MAX_SIZE,
+			RGX_FW_FILENAME "." RGX_BVNC_STRP_FMTSPEC "%s%s",
+			psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C,
+			pszFWFilenameSuffix, pszFWFilenameSuffix2);
+}
+
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW)
+{
+	IMG_CHAR aszFWFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR aszFWpFilenameStr[RGX_FW_FILENAME_MAX_SIZE];
+	IMG_CHAR *pszLoadedFwStr;
+	PVRSRV_ERROR eErr;
+
+	/* Prepare the image filenames to use in the following code */
+	_GetFWFileName(psDeviceNode, aszFWFilenameStr, aszFWpFilenameStr);
+
+	/* Get pointer to Firmware image */
+	pszLoadedFwStr = aszFWFilenameStr;
+	eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+	if (eErr == PVRSRV_ERROR_NOT_FOUND)
+	{
+		pszLoadedFwStr = aszFWpFilenameStr;
+		eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+		if (eErr == PVRSRV_ERROR_NOT_FOUND)
+		{
+			pszLoadedFwStr = RGX_FW_FILENAME;
+			eErr = OSLoadFirmware(psDeviceNode, pszLoadedFwStr, OS_FW_VERIFY_FUNCTION, ppsRGXFW);
+			if (eErr == PVRSRV_ERROR_NOT_FOUND)
+			{
+				PVR_DPF((PVR_DBG_FATAL, "All RGX Firmware image loads failed for '%s' (%s)",
+						aszFWFilenameStr, PVRSRVGetErrorString(eErr)));
+			}
+		}
+	}
+
+	if (eErr == PVRSRV_OK)
+	{
+		PVR_LOG(("RGX Firmware image '%s' loaded", pszLoadedFwStr));
+	}
+
+	return eErr;
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return PVRSRV_OK;
+}
+#endif
+
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	/* set up fw memory contexts */
+	PVRSRV_RGXDEV_INFO   *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+	PVRSRV_ERROR eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	IMG_BOOL  bNativeFwUMAHeap = PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDeviceNode) &&
+	                             (PhysHeapGetType(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM]) == PHYS_HEAP_TYPE_UMA);
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)) && (!psDeviceNode->bAutoVzFwIsUp) && (!bNativeFwUMAHeap))
+	{
+		PHYS_HEAP *psFwPageTableHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		PVR_LOG_GOTO_IF_INVALID_PARAM((psFwPageTableHeap != NULL),
+		                              eError, failed_to_create_ctx);
+
+		/* Temporarily swap the MMU and default GPU physheap to allow the page
+		 * tables of all memory mapped by the FwKernel context to be placed
+		 * in a dedicated memory carveout. This should allow the firmware mappings to
+		 * persist after a Host kernel crash or driver reset. */
+		psDeviceNode->psMMUPhysHeap = psFwPageTableHeap;
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+	/* Set the device fabric coherency before FW context creation */
+	eError = RGXSystemGetFabricCoherency(psDevConfig,
+										 psDevConfig->sRegsCpuPBase,
+										 psDevConfig->ui32RegsSize,
+										 &psDeviceNode->eDevFabricType,
+										 &psDevConfig->eCacheSnoopingMode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed RGXSystemGetFabricCoherency (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+#endif
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	RGXFwSharedMemCheckSnoopMode(psDevConfig);
+
+	/* Create the memory context for the firmware. */
+	eError = DevmemCreateContext(psDeviceNode, DEVMEM_HEAPCFG_FORFW,
+	                             &psDevInfo->psKernelDevmemCtx);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemCreateContext (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_create_ctx;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_MAIN_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareMainHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+	eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, RGX_FIRMWARE_CONFIG_HEAP_IDENT,
+	                              &psDevInfo->psFirmwareConfigHeap);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed DevmemFindHeapByName (%u)",
+		         __func__,
+		         eError));
+		goto failed_to_find_heap;
+	}
+
+#if (defined(RGX_PREMAP_FW_HEAPS)) || (defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1))
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_CHAR szHeapName[RA_MAX_NAME_LENGTH];
+
+			OSSNPrintf(szHeapName, sizeof(szHeapName), RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+			eError = DevmemFindHeapByName(psDevInfo->psKernelDevmemCtx, szHeapName,
+										  &psDevInfo->psPremappedFwRawHeap[ui32DriverID]);
+			PVR_LOG_GOTO_IF_ERROR(eError, "DevmemFindHeapByName", failed_to_find_heap);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && !bNativeFwUMAHeap)
+	{
+		IMG_DEV_PHYADDR sPhysHeapBase;
+		IMG_UINT32 ui32DriverID;
+		void *pvAppHintState = NULL;
+		IMG_UINT64 ui64DefaultHeapStride;
+		IMG_UINT64 ui64GuestHeapDevBaseStride;
+
+		OSCreateAppHintState(&pvAppHintState);
+		ui64DefaultHeapStride = PVRSRV_APPHINT_GUESTFWHEAPSTRIDE;
+		OSGetAppHintUINT64(APPHINT_NO_DEVICE,
+							pvAppHintState,
+							GuestFWHeapStride,
+							&ui64DefaultHeapStride,
+							&ui64GuestHeapDevBaseStride);
+		OSFreeAppHintState(pvAppHintState);
+		pvAppHintState = NULL;
+
+		eError = PhysHeapGetDevPAddr(psDeviceNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM], &sPhysHeapBase);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapGetDevPAddr", failed_to_find_heap);
+
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			IMG_DEV_PHYADDR sRawFwHeapBase = {sPhysHeapBase.uiAddr + (ui32DriverID * ui64GuestHeapDevBaseStride)};
+
+			eError = RGXFwRawHeapAllocMap(psDeviceNode,
+										  ui32DriverID,
+										  sRawFwHeapBase,
+										  RGX_FIRMWARE_RAW_HEAP_SIZE);
+			if (eError != PVRSRV_OK)
+			{
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+				}
+				PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", failed_to_find_heap);
+			}
+		}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+		/* restore default Px setup */
+		psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* On setups with dynamically mapped Guest heaps, the Guest makes
+	 * a PVZ call to the Host to request the mapping during init. */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PvzClientMapDevPhysHeap(psDevConfig);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PvzClientMapDevPhysHeap", failed_to_find_heap);
+	}
+#endif /* !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_TRUE);
+		DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_TRUE);
+	}
+
+	return eError;
+
+failed_to_find_heap:
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+	DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+	psDevInfo->psKernelDevmemCtx = NULL;
+failed_to_create_ctx:
+	return eError;
+}
+
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR       eError;
+#if defined(RGX_PREMAP_FW_HEAPS)
+	PHYS_HEAP *psDefaultPhysHeap = psDeviceNode->psMMUPhysHeap;
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+#if defined(RGX_PREMAP_FW_HEAPS)
+		psDeviceNode->psMMUPhysHeap =
+				psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT];
+
+		if (!psDeviceNode->bAutoVzFwIsUp)
+#endif
+		{
+			IMG_UINT32 ui32DriverID;
+
+			FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+			{
+				RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+			}
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) */
+
+#if !defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		(void) PvzClientUnmapDevPhysHeap(psDeviceNode->psDevConfig);
+
+		if (psDevInfo->psFirmwareMainHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareMainHeap, IMG_FALSE);
+		}
+		if (psDevInfo->psFirmwareConfigHeap)
+		{
+			DevmemHeapSetPremapStatus(psDevInfo->psFirmwareConfigHeap, IMG_FALSE);
+		}
+	}
+#endif
+
+	/*
+	 * Clear the mem context create callbacks before destroying the RGX firmware
+	 * context to avoid a spurious callback.
+	 */
+	psDeviceNode->pfnRegisterMemoryContext = NULL;
+	psDeviceNode->pfnUnregisterMemoryContext = NULL;
+
+	if (psDevInfo->psKernelDevmemCtx)
+	{
+		eError = DevmemDestroyContext(psDevInfo->psKernelDevmemCtx);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	psDeviceNode->psMMUPhysHeap = psDefaultPhysHeap;
+#endif
+}
+
+static PVRSRV_ERROR RGXAlignmentCheck(PVRSRV_DEVICE_NODE *psDevNode,
+                                      IMG_UINT32 ui32AlignChecksSizeUM,
+                                      IMG_UINT32 aui32AlignChecksUM[])
+{
+	static const IMG_UINT32 aui32AlignChecksKM[] = {RGXFW_ALIGN_CHECKS_INIT_KM};
+	IMG_UINT32 ui32UMChecksOffset = ARRAY_SIZE(aui32AlignChecksKM) + 1;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32 i, *paui32FWAlignChecks;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Skip the alignment check if the driver is guest
+	   since there is no firmware to check against */
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDevNode, eError);
+
+	if (psDevInfo->psRGXFWAlignChecksMemDesc == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: FW Alignment Check Mem Descriptor is NULL",
+		         __func__));
+		return PVRSRV_ERROR_ALIGNMENT_ARRAY_NOT_AVAILABLE;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc,
+	                                  (void **) &paui32FWAlignChecks);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to acquire kernel address for alignment checks (%u)",
+		         __func__,
+		         eError));
+		return eError;
+	}
+
+	paui32FWAlignChecks += ui32UMChecksOffset;
+	/* Invalidate the size value, check the next region size (UM) and invalidate */
+	RGXFwSharedMemCacheOpPtr(paui32FWAlignChecks, INVALIDATE);
+	if (*paui32FWAlignChecks++ != ui32AlignChecksSizeUM)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Mismatching sizes of RGXFW_ALIGN_CHECKS_INIT"
+		         " array between UM(%d) and FW(%d)",
+		         __func__,
+		         ui32AlignChecksSizeUM,
+		         *paui32FWAlignChecks));
+		eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		goto return_;
+	}
+
+	RGXFwSharedMemCacheOpExec(paui32FWAlignChecks,
+	                          ui32AlignChecksSizeUM * sizeof(IMG_UINT32),
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+
+	for (i = 0; i < ui32AlignChecksSizeUM; i++)
+	{
+		if (aui32AlignChecksUM[i] != paui32FWAlignChecks[i])
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: size/offset mismatch in RGXFW_ALIGN_CHECKS_INIT[%d]"
+					" between UM(%d) and FW(%d)",
+					__func__, i, aui32AlignChecksUM[i], paui32FWAlignChecks[i]));
+			eError = PVRSRV_ERROR_INVALID_ALIGNMENT;
+		}
+	}
+
+	if (eError == PVRSRV_ERROR_INVALID_ALIGNMENT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Check for FW/KM structure"
+				" alignment failed.", __func__));
+	}
+
+return_:
+
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWAlignChecksMemDesc);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR RGXAllocateFWMemoryRegion(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_DEVMEM_SIZE_T ui32Size,
+                                       PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags,
+                                       const IMG_PCHAR pszText,
+                                       DEVMEM_MEMDESC **ppsMemDescPtr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_DEVMEM_LOG2ALIGN_T uiLog2Align = OSGetPageShift();
+#if defined(RGX_FEATURE_MIPS_BIT_MASK) && defined(SUPPORT_MIPS_CONTIGUOUS_FW_MEMORY)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		uiLog2Align = RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(uiLog2Align);
+#endif
+
+	uiMemAllocFlags = (uiMemAllocFlags |
+					   PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+					   PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC) &
+	                   RGX_AUTOVZ_KEEP_FW_DATA_MASK(psDeviceNode->bAutoVzFwIsUp);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	uiMemAllocFlags &= PVRSRV_MEMALLOCFLAGS_TDFWMASK;
+#endif
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate FW %s memory", pszText);
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										ui32Size,
+										1ULL << uiLog2Align,
+										uiMemAllocFlags,
+										pszText,
+										ppsMemDescPtr);
+
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver
+
+ @Description
+
+ Validate the FW build options against KM driver build options (KM build options only)
+
+ Following check is redundant, because next check checks the same bits.
+ Redundancy occurs because if client-server are build-compatible and client-firmware are
+ build-compatible then server-firmware are build-compatible as well.
+
+ This check is left for clarity in error messages if any incompatibility occurs.
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+
+	IMG_UINT32			ui32BuildOptions, ui32BuildOptionsFWKMPart, ui32BuildOptionsMismatch;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+	IMG_UINT8               ui8FwOsCount;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32BuildOptions = (RGX_BUILD_OPTIONS_KM & RGX_BUILD_OPTIONS_MASK_FW);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32BuildOptionsFWKMPart = psFwOsInit->sRGXCompChecks.ui32BuildOptions & RGX_BUILD_OPTIONS_MASK_FW;
+
+		ui8FwOsCount = psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.sInitOptions.ui8OsCountSupport;
+		if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32BuildOptionsFWKMPart = psFWInfoHeader->ui32Flags & RGX_BUILD_OPTIONS_MASK_FW;
+
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVINFO, psDevInfo) && BITMASK_HAS(psFWInfoHeader->ui32Flags, OPTIONS_NUM_DRIVERS_SUPPORTED_CHECK_EN))
+		{
+			ui8FwOsCount = (psFWInfoHeader->ui32Flags & OPTIONS_NUM_DRIVERS_SUPPORTED_MASK) >> OPTIONS_NUM_DRIVERS_SUPPORTED_SHIFT;
+			ui8FwOsCount++;
+			if (ui8FwOsCount != RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch between the number of Operating Systems supported by KM driver (%d) and FW (%d)",
+					__func__, RGX_NUM_DRIVERS_SUPPORTED, ui8FwOsCount));
+				return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+			}
+		}
+	}
+
+	/* Check if the FW is missing support for any features required by the driver */
+	if (~ui32BuildOptionsFWKMPart & ui32BuildOptions)
+	{
+		ui32BuildOptionsMismatch = ui32BuildOptions ^ ui32BuildOptionsFWKMPart;
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask non-critical options out as we do support combining them in UM & KM */
+		ui32BuildOptionsMismatch &= FW_OPTIONS_STRICT;
+#endif
+		if ((ui32BuildOptions & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware and KM driver build options; "
+					 "extra options present in the KM driver: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptions & ui32BuildOptionsMismatch));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+
+		if ((ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in Firmware-side and KM driver build options; "
+					 "extra options present in Firmware: (0x%x). Please check rgx_options.h",
+					 ui32BuildOptionsFWKMPart & ui32BuildOptionsMismatch ));
+			return PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "RGXDevInitCompatCheck: Firmware and KM driver build options differ."));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: Firmware and KM driver build options match. [ OK ]"));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK version against driver DDK version
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT32			ui32KMDDKVersion;
+	IMG_UINT32			ui32FWDDKVersion;
+	PVRSRV_ERROR		eError;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	ui32KMDDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKVersion = psFwOsInit->sRGXCompChecks.ui32DDKVersion;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+		ui32FWDDKVersion = PVRVERSION_PACK(psFWInfoHeader->ui16PVRVersionMajor, psFWInfoHeader->ui16PVRVersionMinor);
+	}
+
+	if (ui32FWDDKVersion != ui32KMDDKVersion)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Incompatible driver DDK version (%u.%u) / Firmware DDK version (%u.%u).",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_UNPACK_MAJ(ui32FWDDKVersion),
+				 PVRVERSION_UNPACK_MIN(ui32FWDDKVersion)));
+		eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK version (%u.%u) and Firmware DDK version (%u.%u) match. [ OK ]",
+				 PVRVERSION_MAJ, PVRVERSION_MIN,
+				 PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver
+
+ @Description
+
+ Validate FW DDK build against driver DDK build
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32KMDDKBuild;
+	IMG_UINT32			ui32FWDDKBuild;
+	RGX_FW_INFO_HEADER	*psFWInfoHeader = NULL;
+	RGXFWIF_OSINIT		*psFwOsInit = NULL;
+
+	ui32KMDDKBuild = PVRVERSION_BUILD;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		psFwOsInit = psDevInfo->psRGXFWIfOsInit;
+		if (psFwOsInit == NULL)
+			return PVRSRV_ERROR_INVALID_PARAMS;
+
+		ui32FWDDKBuild = psFwOsInit->sRGXCompChecks.ui32DDKBuild;
+	}
+	else
+	{
+		psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+		ui32FWDDKBuild = psFWInfoHeader->ui32PVRVersionBuild;
+	}
+
+	if (ui32FWDDKBuild != ui32KMDDKBuild)
+	{
+		PVR_LOG(("(WARN) RGXDevInitCompatCheck: Different driver DDK build version (%d) / Firmware DDK build version (%d).",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		eError = PVRSRV_ERROR_DDK_BUILD_MISMATCH;
+		PVR_DBG_BREAK;
+		return eError;
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: driver DDK build version (%d) and Firmware DDK build version (%d) match. [ OK ]",
+				 ui32KMDDKBuild, ui32FWDDKBuild));
+	}
+	return eError;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function	RGXDevInitCompatCheck_BVNC_FWAgainstDriver
+
+ @Description
+
+ Validate FW BVNC against driver BVNC
+
+ @Input psDevInfo - device info
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck_BVNC_FWAgainstDriver(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+
+	RGX_FW_INFO_HEADER	*psFWInfoHeader;
+	IMG_UINT64			ui64KMBVNC;
+
+	if (psDevInfo == NULL)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psFWInfoHeader = &psDevInfo->sFWInfoHeader;
+
+	ui64KMBVNC = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+							   psDevInfo->sDevFeatureCfg.ui32V,
+							   psDevInfo->sDevFeatureCfg.ui32N,
+							   psDevInfo->sDevFeatureCfg.ui32C);
+
+	if (ui64KMBVNC != psFWInfoHeader->ui64BVNC)
+	{
+		PVR_LOG(("(FAIL) RGXDevInitCompatCheck: Mismatch in KM driver BVNC (%u.%u.%u.%u) and Firmware BVNC (%u.%u.%u.%u)",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_ERROR_BVNC_MISMATCH;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXDevInitCompatCheck: KM driver BVNC (%d.%d.%d.%d) and FW BVNC (%d.%d.%d.%d) match. [ OK ]",
+				 RGX_BVNC_PACKED_EXTR_B(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_V(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_N(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_C(ui64KMBVNC),
+				 RGX_BVNC_PACKED_EXTR_B(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_V(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_N(psFWInfoHeader->ui64BVNC),
+				 RGX_BVNC_PACKED_EXTR_C(psFWInfoHeader->ui64BVNC)));
+
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevInitCompatCheck
+
+ @Description
+
+ Check compatibility of host driver and firmware (DDK and build options)
+ for RGX devices at services/device initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR - depending on mismatch found
+
+ ******************************************************************************/
+static PVRSRV_ERROR RGXDevInitCompatCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR		eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#if !defined(NO_HARDWARE)
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32			ui32FwTimeout = MAX_HW_TIME_US;
+
+		LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+		{
+			RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                               INVALIDATE);
+			if (*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+			{
+				/* No need to wait if the FW has already updated the values */
+				break;
+			}
+			OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		/* Flush covers this instance and the reads in the functions below */
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks,
+	                               INVALIDATE);
+		if (!*((volatile IMG_BOOL *)&psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated))
+		{
+			eError = PVRSRV_ERROR_TIMEOUT;
+			PVR_DPF((PVR_DBG_ERROR, "%s: GPU Firmware not responding: failed to supply compatibility info (%u)",
+					__func__, eError));
+
+			PVR_DPF((PVR_DBG_ERROR, "%s: Potential causes: firmware not initialised or the current Guest driver's "
+									"OsConfig initialisation data was not accepted by the firmware", __func__));
+			goto chk_exit;
+		}
+	}
+#endif /* defined(NO_HARDWARE) */
+
+	eError = RGXDevInitCompatCheck_KMBuildOptions_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKVersion_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	eError = RGXDevInitCompatCheck_DDKBuild_FWAgainstDriver(psDevInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto chk_exit;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXDevInitCompatCheck_BVNC_FWAgainstDriver(psDevInfo);
+		if (eError != PVRSRV_OK)
+		{
+			goto chk_exit;
+		}
+	}
+
+	eError = PVRSRV_OK;
+chk_exit:
+
+	return eError;
+}
+
+static void _RGXSoftResetToggle(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                IMG_UINT64  ui64ResetValue1,
+                                IMG_UINT64  ui64ResetValue2)
+{
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, ui64ResetValue1);
+
+	/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+	(void) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
+}
+
+/**************************************************************************/ /*!
+@Function       RGXSoftReset
+@Description    Resets some modules of the RGX device
+@Input          psDeviceNode		Device node
+@Input          ui64ResetValue1 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET
+                                register).
+@Input          ui64ResetValue2 A mask for which each bit set corresponds
+                                to a module to reset (via the SOFT_RESET2
+                                register).
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXSoftReset(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT64  ui64ResetValue1,
+                                 IMG_UINT64  ui64ResetValue2)
+{
+	PVRSRV_RGXDEV_INFO        *psDevInfo;
+	IMG_BOOL	bSoftReset = IMG_FALSE;
+	IMG_UINT64	ui64SoftResetMask = 0;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+	if (((ui64ResetValue1 & ui64SoftResetMask) != ui64ResetValue1) || bSoftReset)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Set in soft-reset */
+	_RGXSoftResetToggle(psDevInfo, ui64ResetValue1, ui64ResetValue2);
+
+	/* Take the modules out of reset... */
+	_RGXSoftResetToggle(psDevInfo, 0, 0);
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+static const RGX_MIPS_ADDRESS_TRAMPOLINE sNullTrampoline;
+
+static void RGXFreeTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+			psDevInfo->psTrampoline->hPdumpPages,
+#endif
+			&psDevInfo->psTrampoline->sPages);
+
+	if (psDevInfo->psTrampoline != &sNullTrampoline)
+	{
+		OSFreeMem(psDevInfo->psTrampoline);
+	}
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+}
+
+#define RANGES_OVERLAP(x,y,size) (x < (y+size) && y < (x+size))
+#define TRAMPOLINE_ALLOC_MAX_RETRIES (3)
+
+static PVRSRV_ERROR RGXAllocTrampoline(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	IMG_INT32 i, j;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_MIPS_ADDRESS_TRAMPOLINE *pasTrampoline[TRAMPOLINE_ALLOC_MAX_RETRIES];
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate pages for trampoline");
+
+	/* Retry the allocation of the trampoline block (16KB), retaining any
+	 * previous allocations overlapping  with the target range until we get an
+	 * allocation that doesn't overlap with the target range.
+	 * Any allocation like this will require a maximum of 3 tries as we are
+	 * allocating a physical contiguous block of memory, not individual pages.
+	 * Free the unused allocations at the end only after the desired range
+	 * is obtained to prevent the alloc function from returning the same bad
+	 * range repeatedly.
+	 */
+	for (i = 0; i < TRAMPOLINE_ALLOC_MAX_RETRIES; i++)
+	{
+		pasTrampoline[i] = OSAllocMem(sizeof(RGX_MIPS_ADDRESS_TRAMPOLINE));
+		eError = DevPhysMemAlloc(psDeviceNode,
+				RGXMIPSFW_TRAMPOLINE_SIZE,
+				RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE,
+				0,         // (init) u8Value
+				IMG_FALSE, // bInitPage,
+#if defined(PDUMP)
+				psDeviceNode->psFirmwareMMUDevAttrs->pszMMUPxPDumpMemSpaceName,
+				"TrampolineRegion",
+				&pasTrampoline[i]->hPdumpPages,
+#endif
+				PVR_SYS_ALLOC_PID,
+				&pasTrampoline[i]->sPages,
+				&pasTrampoline[i]->sPhysAddr);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s failed (%u)",
+			         __func__,
+			         eError));
+			goto fail;
+		}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+		/* Set the persistent uiOSid value so that we free from the correct
+		 * base arena when unloading the driver and freeing the trampoline.
+		 */
+		pasTrampoline[i]->sPages.uiOSid = 0;	/* Firmware global arena */
+#endif
+
+		if (!RANGES_OVERLAP(pasTrampoline[i]->sPhysAddr.uiAddr,
+				RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+				RGXMIPSFW_TRAMPOLINE_SIZE))
+		{
+			break;
+		}
+	}
+	if (TRAMPOLINE_ALLOC_MAX_RETRIES == i)
+	{
+		/* Failed to find a physical allocation after 3 attempts */
+		eError = PVRSRV_ERROR_FAILED_TO_ALLOC_PAGES;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s failed to allocate non-overlapping pages (%u)",
+				__func__, eError));
+		/* Fall through, clean up and return error. */
+	}
+	else
+	{
+		/* Remember the last physical block allocated, it will not be freed */
+		psDevInfo->psTrampoline = pasTrampoline[i];
+	}
+
+fail:
+	/* free all unused allocations */
+	for (j = 0; j < i; j++)
+	{
+		DevPhysMemFree(psDeviceNode,
+#if defined(PDUMP)
+				pasTrampoline[j]->hPdumpPages,
+#endif
+				&pasTrampoline[j]->sPages);
+		OSFreeMem(pasTrampoline[j]);
+	}
+
+	return eError;
+}
+
+#undef RANGES_OVERLAP
+#endif
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    uiFWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen)
+{
+	PVRSRV_MEMALLOCFLAGS_T uiMemAllocFlags;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR        eError;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEVMEM_SIZE_T	uiDummyLen;
+	DEVMEM_MEMDESC		*psDummyMemDesc = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		eError = RGXAllocTrampoline(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Failed to allocate trampoline region (%u)",
+					eError));
+			goto failTrampolineMemDescAlloc;
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW code section
+	 */
+	uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+	                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE);
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWCodeLen,
+	                                   uiMemAllocFlags,
+	                                   "FwExCodeRegion",
+	                                   &psDevInfo->psRGXFWCodeMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc,
+	                                  &psDevInfo->sFWCodeDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw code mem (%u)",
+		         eError));
+		goto failFWCodeMemDescAqDevVirt;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) || (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))))
+#endif
+	{
+		/*
+		 * The FW code must be the first allocation in the firmware heap, otherwise
+		 * the bootloader will not work (the FW will not be able to find the bootloader).
+		 */
+		PVR_ASSERT(psDevInfo->sFWCodeDevVAddrBase.uiAddr == RGX_FIRMWARE_RAW_HEAP_BASE);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/*
+		 * Allocate Dummy Pages so that Data segment allocation gets the same
+		 * device virtual address as specified in MIPS firmware linker script
+		 */
+		uiDummyLen = RGXGetFWImageSectionMaxSize(NULL, MIPS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_EXCEPTIONS_CODE) +
+				RGXGetFWImageSectionMaxSize(NULL, MIPS_BOOT_CODE) -
+				uiFWCodeLen; /* code actual size */
+
+		if (uiDummyLen > 0)
+		{
+			eError = DevmemFwAllocateExportable(psDeviceNode,
+					uiDummyLen,
+					OSGetPageSize(),
+					uiMemAllocFlags,
+					"FwExDummyPages",
+					&psDummyMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "Failed to allocate fw dummy mem (%u)",
+				         eError));
+				goto failDummyMemDescAlloc;
+			}
+		}
+	}
+#endif
+
+	/*
+	 * Set up Allocation for FW data section
+	 */
+	eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+	                                   uiFWDataLen,
+	                                   RGX_FWCODEDATA_ALLOCFLAGS |
+	                                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA),
+	                                   "FwExDataRegion",
+	                                   &psDevInfo->psRGXFWDataMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to allocate fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAlloc;
+	}
+
+	eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+	                                  &psDevInfo->sFWDataDevVAddrBase);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Failed to acquire devVAddr for fw data mem (%u)",
+		         eError));
+		goto failFWDataMemDescAqDevVirt;
+	}
+
+	if (uiFWCorememCodeLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem code section
+		 */
+		uiMemAllocFlags = (RGX_FWCODEDATA_ALLOCFLAGS |
+		                   PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_CODE)) &
+		                   ~PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE;
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+		                                   uiFWCorememCodeLen,
+		                                   uiMemAllocFlags,
+		                                   "FwExCorememCodeRegion",
+		                                   &psDevInfo->psRGXFWCorememCodeMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem code mem, size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememCodeLen, uiMemAllocFlags, eError));
+			goto failFWCorememCodeMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc,
+		                                  &psDevInfo->sFWCorememCodeDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem code (%u)",
+			         eError));
+			goto failFWCorememCodeMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememCodeFWAddr,
+		                      psDevInfo->psRGXFWCorememCodeMemDesc,
+		                      0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", failFWCorememCodeMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememCodeDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememCodeFWAddr.ui32Addr = 0;
+	}
+
+	if (uiFWCorememDataLen != 0)
+	{
+		/*
+		 * Set up Allocation for FW coremem data section
+		 */
+		uiMemAllocFlags = RGX_FWCODEDATA_ALLOCFLAGS |
+		                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_PRIV_DATA);
+		eError = RGXAllocateFWMemoryRegion(psDeviceNode,
+				uiFWCorememDataLen,
+				uiMemAllocFlags,
+				"FwExCorememDataRegion",
+				&psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to allocate fw coremem data mem, "
+			         "size: %"  IMG_INT64_FMTSPECd ", flags: %" PVRSRV_MEMALLOCFLAGS_FMTSPEC " (%u)",
+			         uiFWCorememDataLen,
+			         uiMemAllocFlags,
+			         eError));
+			goto failFWCorememDataMemDescAlloc;
+		}
+
+		eError = DevmemAcquireDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				&psDevInfo->sFWCorememDataStoreDevVAddrBase);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Failed to acquire devVAddr for fw coremem mem data (%u)",
+			         eError));
+			goto failFWCorememDataMemDescAqDevVirt;
+		}
+
+		eError = RGXSetFirmwareAddress(&psDevInfo->sFWCorememDataStoreFWAddr,
+				psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+				0, RFW_FWADDR_NOREF_FLAG);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", failFWCorememDataMemDescFwAddr);
+	}
+	else
+	{
+		psDevInfo->sFWCorememDataStoreDevVAddrBase.uiAddr = 0;
+		psDevInfo->sFWCorememDataStoreFWAddr.ui32Addr = 0;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Free Dummy Pages */
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+#endif
+
+	return PVRSRV_OK;
+
+failFWCorememDataMemDescFwAddr:
+failFWCorememDataMemDescAqDevVirt:
+	if (uiFWCorememDataLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+failFWCorememDataMemDescAlloc:
+failFWCorememCodeMemDescFwAddr:
+failFWCorememCodeMemDescAqDevVirt:
+	if (uiFWCorememCodeLen != 0)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+failFWCorememCodeMemDescAlloc:
+failFWDataMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+	psDevInfo->psRGXFWDataMemDesc = NULL;
+failFWDataMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (psDummyMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psDummyMemDesc);
+	}
+failDummyMemDescAlloc:
+#endif
+failFWCodeMemDescAqDevVirt:
+	DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+	psDevInfo->psRGXFWCodeMemDesc = NULL;
+failFWCodeMemDescAlloc:
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		RGXFreeTrampoline(psDeviceNode);
+	}
+failTrampolineMemDescAlloc:
+#endif
+	return eError;
+}
+
+/*
+	AppHint parameter interface
+ */
+static
+PVRSRV_ERROR RGXFWTraceQueryFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	*pui32Value &= RGXFWIF_LOG_TYPE_GROUP_MASK;
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceQueryLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   const void *psPrivate,
+                                   IMG_UINT32 *pui32Value)
+{
+	PVRSRV_ERROR eResult;
+
+	eResult = PVRSRVRGXFWDebugQueryFWLogKM(NULL, psDeviceNode, pui32Value);
+	if (PVRSRV_OK == eResult)
+	{
+		if (*pui32Value & RGXFWIF_LOG_TYPE_TRACE)
+		{
+			*pui32Value = 0; /* Trace */
+		}
+		else
+		{
+			*pui32Value = 1; /* TBI */
+		}
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetFilter(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 const void *psPrivate,
+                                 IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType;
+
+	eResult = RGXFWTraceQueryLogType(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK == eResult)
+	{
+		if (0 == ui32RGXFWLogType)
+		{
+			BITMASK_SET(ui32Value, RGXFWIF_LOG_TYPE_TRACE);
+		}
+		eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32Value);
+	}
+	return eResult;
+}
+
+static
+PVRSRV_ERROR RGXFWTraceSetLogType(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  const void *psPrivate,
+                                  IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eResult;
+	IMG_UINT32 ui32RGXFWLogType = ui32Value;
+
+	eResult = RGXFWTraceQueryFilter(psDeviceNode, NULL, &ui32RGXFWLogType);
+	if (PVRSRV_OK != eResult)
+	{
+		return eResult;
+	}
+
+	/* 0 - trace, 1 - tbi */
+	if (0 == ui32Value)
+	{
+		BITMASK_SET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#if defined(SUPPORT_TBI_INTERFACE)
+	else if (1 == ui32Value)
+	{
+		BITMASK_UNSET(ui32RGXFWLogType, RGXFWIF_LOG_TYPE_TRACE);
+	}
+#endif
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid parameter %u specified to set FW log type AppHint.",
+		         __func__, ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eResult = PVRSRVRGXFWDebugSetFWLogKM(NULL, psDeviceNode, ui32RGXFWLogType);
+	return eResult;
+}
+
+#if defined(DEBUG)
+static
+PVRSRV_ERROR RGXQueryFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL *pbValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+	*pbValue = (PVRSRV_MEMALLOCFLAG_POISON_ON_FREE == psDevInfo->uiFWPoisonOnFreeFlag)
+		? IMG_TRUE
+		: IMG_FALSE;
+	return PVRSRV_OK;
+}
+
+static
+PVRSRV_ERROR RGXSetFWPoisonOnFree(const PVRSRV_DEVICE_NODE *psDeviceNode,
+									const void *psPrivate,
+									IMG_BOOL bValue)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+	psDevInfo->uiFWPoisonOnFreeFlag = bValue
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+ * RGXInitFirmware
+ */
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+#if defined(DEBUG)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bEnableFWPoisonOnFree = IMG_FALSE;
+#endif
+
+	eError = RGXSetupFirmware(psDeviceNode,
+	                          psApphints,
+	                          ui32ConfigFlags,
+	                          ui32ConfigFlagsExt,
+	                          ui32FwOsCfgFlags);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXInitFirmwareKM: RGXSetupFirmware failed (%u)",
+		         eError));
+		goto failed_init_firmware;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_EnableLogGroup,
+		                                    RGXFWTraceQueryFilter,
+		                                    RGXFWTraceSetFilter,
+		                                    psDeviceNode,
+		                                    NULL);
+		PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_FirmwareLogType,
+		                                    RGXFWTraceQueryLogType,
+		                                    RGXFWTraceSetLogType,
+		                                    psDeviceNode,
+		                                    NULL);
+	}
+
+#if defined(DEBUG)
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLEFWPOISONONFREE;
+	OSGetAppHintBOOL(psDeviceNode,
+			pvAppHintState,
+			EnableFWPoisonOnFree,
+			&bAppHintDefault,
+			&bEnableFWPoisonOnFree);
+
+	OSFreeAppHintState(pvAppHintState);
+
+	PVRSRVAppHintRegisterHandlersBOOL(APPHINT_ID_EnableFWPoisonOnFree,
+	                                   RGXQueryFWPoisonOnFree,
+	                                   RGXSetFWPoisonOnFree,
+	                                   psDeviceNode,
+	                                   NULL);
+
+	psDevInfo->uiFWPoisonOnFreeFlag = bEnableFWPoisonOnFree
+			? PVRSRV_MEMALLOCFLAG_POISON_ON_FREE
+			: 0ULL;
+#else
+	psDevInfo->uiFWPoisonOnFreeFlag = 0ULL;
+#endif
+
+	psDevInfo->ui32ClockSource = PVRSRV_APPHINT_TIMECORRCLOCK;
+	psDevInfo->ui32LastClockSource = PVRSRV_APPHINT_TIMECORRCLOCK;
+
+	return PVRSRV_OK;
+
+failed_init_firmware:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* See device.h for function declaration */
+static PVRSRV_ERROR RGXAllocUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+									 IMG_UINT32 ui32RequestedSize,
+									 DEVMEM_MEMDESC **psMemDesc,
+									 IMG_UINT32 *puiSyncPrimVAddr,
+									 IMG_UINT32 *puiSyncPrimBlockSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+	RGXFWIF_DEV_VIRTADDR pFirmwareAddr;
+	IMG_DEVMEM_ALIGN_T uiUFOBlockAlign = MAX(sizeof(IMG_UINT32), sizeof(SYNC_CHECKPOINT_FW_OBJ));
+	IMG_DEVMEM_SIZE_T uiUFOBlockSize = PVR_ALIGN(ui32RequestedSize, uiUFOBlockAlign);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Size and align are 'expanded' because we request an Exportalign allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiUFOBlockSize,
+	                                             &uiUFOBlockAlign);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = DevmemFwAllocateExportable(psDeviceNode,
+										uiUFOBlockSize,
+										uiUFOBlockAlign,
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_UNCACHED,
+										"FwExUFOBlock",
+										psMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	eError = RGXSetFirmwareAddress(&pFirmwareAddr, *psMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = pFirmwareAddr.ui32Addr;
+	*puiSyncPrimBlockSize = TRUNCATE_64BITS_TO_32BITS(uiUFOBlockSize);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, *psMemDesc);
+e0:
+	return eError;
+}
+
+/* See device.h for function declaration */
+static void RGXFreeUFOBlock(PVRSRV_DEVICE_NODE *psDeviceNode,
+							DEVMEM_MEMDESC *psMemDesc)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	RGXUnsetFirmwareAddress(psMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psMemDesc);
+}
+
+static void DevPart2DeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+
+	psDevInfo->bDevInit2Done = IMG_FALSE;
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+	if (psDevInfo->psGenHeapSecMem != NULL)
+	{
+		OSFreeSecBuf(psDevInfo->psGenHeapSecMem);
+	}
+#endif
+
+#if defined(RGX_FEATURE_COMPUTE_ONLY_BIT_MASK)
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, COMPUTE_ONLY))
+#endif
+	{
+		if ((psDevInfo->hTQUSCSharedMem != NULL) &&
+		    (psDevInfo->hTQCLISharedMem != NULL))
+		{
+			PVRSRVTQUnloadShaders(psDeviceNode);
+		}
+	}
+
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvLISRData != NULL)
+	{
+		(void) SysUninstallDeviceLISR(psDevInfo->pvLISRData);
+	}
+	if (psDevInfo->pvMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvMISRData);
+	}
+	if (psDevInfo->hProcessQueuesMISR != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->hProcessQueuesMISR);
+	}
+	if (psDevInfo->pvAPMISRData != NULL)
+	{
+		(void) OSUninstallMISR(psDevInfo->pvAPMISRData);
+	}
+	if (psDeviceNode->hCmdCompNotify != NULL)
+	{
+		/* Cancel notifications to this device */
+		PVRSRVUnregisterCmdCompleteNotify(psDeviceNode->hCmdCompNotify);
+		psDeviceNode->hCmdCompNotify = NULL;
+	}
+#endif /* !NO_HARDWARE */
+
+	/* Remove the device from the power manager */
+	PVRSRVRemovePowerDevice(psDeviceNode);
+
+	psDevInfo->pfnGetGpuUtilStats = NULL;
+	if (psDevInfo->hGPUUtilLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hGPUUtilLock);
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(psDevInfo->hNMILock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hNMILock);
+	}
+#endif
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(psDevInfo->hMMUCtxUnregLock != NULL))
+	{
+		OSLockDestroy(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	if (psDevInfo->hDebugFaultInfoLock != NULL)
+	{
+		OSLockDestroy(psDevInfo->hDebugFaultInfoLock);
+	}
+
+	/* De-init Freelists/ZBuffers... */
+	if (psDevInfo->hLockFreeList != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockFreeList);
+	}
+
+	if (psDevInfo->hLockZSBuffer != NULL)
+	{
+		OSLockDestroy(psDevInfo->hLockZSBuffer);
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* De-init work estimation lock */
+		if (psDevInfo->hWorkEstLock != NULL)
+		{
+			OSLockDestroy(psDevInfo->hWorkEstLock);
+		}
+	}
+#endif
+
+	/* Free DVFS Table */
+	if (psDevInfo->psGpuDVFSTable != NULL)
+	{
+		OSFreeMem(psDevInfo->psGpuDVFSTable);
+		psDevInfo->psGpuDVFSTable = NULL;
+	}
+}
+
+/*
+	DevDeInitRGX
+ */
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO		*psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	PVRSRV_ERROR			eError;
+	DEVICE_MEMORY_INFO		*psDevMemoryInfo;
+
+	if (!psDevInfo)
+	{
+		/* Can happen if DevInitRGX failed */
+		PVR_DPF((PVR_DBG_ERROR, "DevDeInitRGX: Null DevInfo"));
+		return PVRSRV_OK;
+	}
+
+	if (psDevInfo->psRGXFWIfOsInit)
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	RGXUnregisterBridges(psDevInfo);
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	OSLockDestroy(psDevInfo->hCounterDumpingLock);
+#endif
+
+	/* Unregister debug request notifiers first as they could depend on anything. */
+
+	RGXDebugDeinit(psDevInfo);
+
+	/* De-initialise in reverse order, so stage 2 init is undone first. */
+	if (psDevInfo->bDevInit2Done)
+	{
+		DevPart2DeInitRGX(psDeviceNode);
+	}
+
+	/* Unregister MMU related stuff */
+	eError = RGXMMUInit_Unregister(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevDeInitRGX: Failed RGXMMUInit_Unregister (0x%x)",
+		         eError));
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		/* Unregister MMU related stuff */
+		eError = RGXMipsMMUInit_Unregister(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed RGXMipsMMUInit_Unregister (0x%x)",
+			         eError));
+		}
+	}
+#endif
+
+	/* UnMap Regs */
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+						 psDevInfo->ui32RegSize);
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvRegsBaseKM = NULL;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+#if !defined(NO_HARDWARE)
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+			(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* undo the VA offset performed in RGXRegisterDevice() to allow the allocation to be unmapped */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM, RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+#endif /* !NO_HARDWARE */
+		psDevInfo->pvSecureRegsBaseKM = NULL;
+	}
+#endif
+
+#if 0 /* not required at this time */
+	if (psDevInfo->hTimer)
+	{
+		eError = OSRemoveTimer(psDevInfo->hTimer);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "DevDeInitRGX: Failed to remove timer"));
+			return eError;
+		}
+		psDevInfo->hTimer = NULL;
+	}
+#endif
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+
+	RGXDeInitHeaps(psDevMemoryInfo, psDeviceNode);
+
+	if (psDevInfo->psRGXFWCodeMemDesc)
+	{
+		/* Free fw code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCodeMemDesc);
+		psDevInfo->psRGXFWCodeMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware code memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) == 32))
+	{
+		if (psDevInfo->psTrampoline->sPages.u.pvHandle)
+		{
+			/* Free trampoline region */
+			PDUMPCOMMENT(psDeviceNode, "Freeing trampoline memory");
+			RGXFreeTrampoline(psDeviceNode);
+		}
+	}
+#endif
+
+	if (psDevInfo->psRGXFWDataMemDesc)
+	{
+		/* Free fw data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW data memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWDataMemDesc);
+		psDevInfo->psRGXFWDataMemDesc = NULL;
+	}
+#if !defined(NO_HARDWARE)
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "No firmware data memory to free"));
+	}
+#endif	/* !defined(NO_HARDWARE) */
+
+	if (psDevInfo->psRGXFWCorememCodeMemDesc)
+	{
+		/* Free fw core mem code */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem code memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWCorememCodeMemDesc);
+		psDevInfo->psRGXFWCorememCodeMemDesc = NULL;
+	}
+
+	if (psDevInfo->psRGXFWIfCorememDataStoreMemDesc)
+	{
+		/* Free fw core mem data */
+		PDUMPCOMMENT(psDeviceNode, "Freeing FW coremem data store memory");
+		DevmemReleaseDevVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+		psDevInfo->psRGXFWIfCorememDataStoreMemDesc = NULL;
+	}
+
+	/*
+	   Free the firmware allocations.
+	 */
+	RGXFreeFirmware(psDevInfo);
+
+
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+
+	/* De-initialise non-device specific (TL) users of RGX device memory */
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < RGX_HWPERF_L2_STREAM_LAST; i++)
+		{
+			RGXHWPerfDeinitL2Stream(psDevInfo, i);
+		}
+
+		RGXHWPerfDeinit(psDevInfo);
+	}
+
+	RGXDeInitDestroyFWKernelMemoryContext(psDeviceNode);
+
+	RGXHWPerfHostDeInit(psDevInfo);
+	eError = HTBDeInit();
+	PVR_LOG_IF_ERROR(eError, "HTBDeInit");
+
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+
+	/* destroy the stalled CCB locks */
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+
+	/* destroy the context list locks */
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+	OSLockDestroy(psDevInfo->hBPLock);
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+
+	/* Free device BVNC string */
+	if (NULL != psDevInfo->sDevFeatureCfg.pszBVNCString)
+	{
+		OSFreeMem(psDevInfo->sDevFeatureCfg.pszBVNCString);
+	}
+
+
+	/* DeAllocate devinfo */
+	OSFreeMem(psDevInfo);
+
+	psDeviceNode->pvDevice = NULL;
+
+	return PVRSRV_OK;
+}
+
+#if defined(PDUMP)
+static
+PVRSRV_ERROR RGXResetPDump(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)(psDeviceNode->pvDevice);
+
+	psDevInfo->bDumpedKCCBCtlAlready = IMG_FALSE;
+
+	return PVRSRV_OK;
+}
+#endif /* PDUMP */
+
+/* Takes a log2 page size parameter and calculates a suitable page size
+ * for the RGX heaps. Returns 0 if parameter is wrong.*/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize)
+{
+	IMG_BOOL bFound = IMG_FALSE;
+	IMG_UINT32 ui32PageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	/* OS page shift must be at least RGX_HEAP_4KB_PAGE_SHIFT,
+	 * max RGX_HEAP_2MB_PAGE_SHIFT, non-zero and a power of two*/
+	if (uiLog2PageSize == 0U ||
+	    (uiLog2PageSize < RGX_HEAP_4KB_PAGE_SHIFT) ||
+	    (uiLog2PageSize > RGX_HEAP_2MB_PAGE_SHIFT))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Provided incompatible log2 page size %u",
+				__func__,
+				uiLog2PageSize));
+		PVR_ASSERT(0);
+		return 0;
+	}
+
+	do
+	{
+		if ((IMG_PAGE2BYTES32(uiLog2PageSize) & ui32PageSizeMask) == 0)
+		{
+			/* We have to fall back to a smaller device
+			 * page size than given page size because there
+			 * is no exact match for any supported size. */
+			uiLog2PageSize -= 1U;
+		}
+		else
+		{
+			/* All good, RGX page size equals given page size
+			 * => use it as default for heaps */
+			bFound = IMG_TRUE;
+		}
+	} while (!bFound);
+
+	return uiLog2PageSize;
+}
+
+/* First 16-bits define possible types */
+#define HEAP_INST_VALUE_MASK     (0xFFFF)
+#define HEAP_INST_DEFAULT_VALUE  (1U)  /* Used to show either the heap is always instantiated by default (pfn = NULL)
+	                                      OR
+	                                      that this is the default configuration of the heap with an Alternative BRN */
+#define HEAP_INST_BRN_DEP_VALUE  (2U)  /* The inclusion of this heap is dependent on the brn being present */
+#define HEAP_INST_FEAT_DEP_VALUE (3U)  /* The inclusion of this heap is dependent on the feature being present */
+#define HEAP_INST_BRN_ALT_VALUE  (4U)  /* This entry is a possible alternative to the default determined by a BRN */
+#define HEAP_INST_FEAT_ALT_VALUE (5U)  /* The entry is a possible alternative to the default determined by a Feature define */
+
+/* Latter 16-bits define other flags we may need */
+#define HEAP_INST_NON4K_FLAG     (1 << 16U) /* This is a possible NON4K Entry and we should use the device
+                                               NON4K size when instantiating */
+
+typedef struct RGX_HEAP_INFO_TAG RGX_HEAP_INFO; // Forward declaration
+typedef IMG_BOOL (*PFN_IS_PRESENT)(PVRSRV_RGXDEV_INFO*, const RGX_HEAP_INFO*);
+typedef void (*PFN_HEAP_DYNAMIC)(PVRSRV_DEVICE_NODE*, RGX_HEAP_INFO*);
+
+struct RGX_HEAP_INFO_TAG
+{
+	IMG_CHAR           *pszName;
+	IMG_UINT64         ui64HeapBase;
+	IMG_DEVMEM_SIZE_T  uiHeapLength;
+	IMG_DEVMEM_SIZE_T  uiHeapReservedRegionLength;
+	IMG_UINT32         ui32Log2ImportAlignment;
+	PFN_IS_PRESENT     pfnIsHeapPresent;
+	PFN_HEAP_DYNAMIC   pfnDynamicBaseSize; /* May modify the psHeapInfo's base & length. May be NULL.
+	                                          Only called once if the heap is present otherwise never. */
+	PFN_HEAP_INIT      pfnInit;
+	PFN_HEAP_DEINIT    pfnDeInit;
+	IMG_UINT32         ui32HeapInstanceFlags;
+};
+/* RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES is the total amount of reserved space, to be specified in gasRGXHeapLayoutApp[] */
+#define RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES	(RGX_HEAP_UM_GENERAL_RESERVED_SIZE + RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE)
+
+/* Private heap data struct. */
+typedef struct RGX_HEAP_DATA_TAG
+{
+	DEVMEMINT_RESERVATION  *psMemReservation;
+} RGX_HEAP_DATA;
+
+static PVRSRV_ERROR HeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+							 DEVMEMINT_HEAP *psDevmemHeap,
+							 PMR *psPMR,
+							 IMG_DEVMEM_SIZE_T uiSize,
+							 IMG_UINT64 ui64Offset,
+							 IMG_BOOL bWriteAble,
+							 IMG_HANDLE *phPrivData)
+{
+	RGX_HEAP_DATA *psHeapData;
+	IMG_DEV_VIRTADDR sCarveOutAddr;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDevmemHeap, "psDevmemHeap");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psPMR, "psPMR");
+	PVR_LOG_RETURN_IF_INVALID_PARAM(phPrivData, "phPrivData");
+
+	psHeapData = OSAllocMem(sizeof(*psHeapData));
+	PVR_LOG_RETURN_IF_NOMEM(psHeapData, "psHeapData");
+
+	/* Map the per device secure mem PMR allocation to the general devmem heap carveout. */
+	sCarveOutAddr = DevmemIntHeapGetBaseAddr(psDevmemHeap);
+	sCarveOutAddr.uiAddr += ui64Offset;
+
+	eError = DevmemIntReserveRange(psDevmemHeap,
+									sCarveOutAddr,
+									uiSize,
+									PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+									(bWriteAble ? PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE : 0),
+									&psHeapData->psMemReservation);
+	PVR_GOTO_IF_ERROR(eError, ErrorFreeHeapData);
+
+	eError = DevmemIntMapPMR(psHeapData->psMemReservation, psPMR);
+	PVR_GOTO_IF_ERROR(eError, ErrorUnreserve);
+
+	*phPrivData = (IMG_HANDLE)psHeapData;
+
+	return PVRSRV_OK;
+
+ErrorUnreserve:
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+ErrorFreeHeapData:
+	OSFreeMem(psHeapData);
+
+	return eError;
+}
+
+/* Deinit callback function for general heap. */
+static void HeapDeInit(IMG_HANDLE hPrivData)
+{
+	RGX_HEAP_DATA *psHeapData = (RGX_HEAP_DATA*)hPrivData;
+
+	PVR_ASSERT(hPrivData);
+
+	DevmemIntUnmapPMR(psHeapData->psMemReservation);
+	DevmemIntUnreserveRange(psHeapData->psMemReservation);
+
+	OSFreeMem(psHeapData);
+}
+
+/* Init callback function for general heap. */
+static PVRSRV_ERROR USCHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_FALSE;
+	IMG_DEVMEM_SIZE_T uiSize;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	uiSize = PMR_PhysicalSize(psDevInfo->hTQUSCSharedMem);
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->hTQUSCSharedMem,
+					  uiSize,
+					  RGX_HEAP_KM_USC_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#if defined(SUPPORT_SECURE_ALLOC_KM)
+/* Init callback function for general heap. */
+static PVRSRV_ERROR GeneralHeapInit(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    DEVMEMINT_HEAP *psDevmemHeap,
+                                    IMG_HANDLE *phPrivData)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_BOOL bWriteAble = IMG_TRUE;
+	PVRSRV_ERROR eError;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	eError = HeapInit(psDeviceNode,
+					  psDevmemHeap,
+					  psDevInfo->psGenHeapSecMem,
+					  RGXFWIF_KM_GENERAL_HEAP_RESERVED_SIZE,
+					  RGX_HEAP_KM_GENERAL_RESERVED_REGION_OFFSET,
+					  bWriteAble,
+					  phPrivData);
+
+	return eError;
+}
+
+#define GeneralHeapDeInit HeapDeInit
+#else
+/* Callbacks not used */
+#define GeneralHeapInit NULL
+#define GeneralHeapDeInit NULL
+#endif
+
+static void SVMHeapDynamic(PVRSRV_DEVICE_NODE *psDeviceNode,
+                           RGX_HEAP_INFO *psHeapInfo)
+{
+	IMG_UINT64 ui64OSPageSize = OSGetPageSize();
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	PVRSRV_RGXDEV_INFO* psDevInfo = psDeviceNode->pvDevice;
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+
+	/* Ensures the SVM heap has the correct alignment & size for any OS page size.
+	 *
+	 * The SVM heap's base must be the smallest possible address mappable by UM.
+	 * This is 32KB unless the page size is larger than 32KB. [1]
+	 * If the page size > 32KB, raise the SVM heap base to the next page boundary.
+	 * Also reduce the length to ensure it's still page aligned and doesn't go
+	 * into another heap.
+	 *
+	 * [1]: https://chromium.googlesource.com/chromium/src/+/fe24932ee14aa93e1fe4d3e7003b9362591a54d4/docs/security/faq.md#why-aren_t-null-pointer-dereferences-considered-security-bugs
+	 */
+	IMG_UINT64 ui64Base = PVR_ALIGN(psHeapInfo->ui64HeapBase, ui64OSPageSize);
+	IMG_UINT64 ui64BaseDiff = ui64Base - psHeapInfo->ui64HeapBase;
+	psHeapInfo->ui64HeapBase = ui64Base;
+	if (psHeapInfo->uiHeapLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapLength -= ui64BaseDiff;
+	if (psHeapInfo->uiHeapReservedRegionLength >= ui64BaseDiff)
+	    psHeapInfo->uiHeapReservedRegionLength -= ui64BaseDiff;
+
+	/* The device shared-virtual-memory heap address-space size is stored on the device for
+	   faster look-up without having to walk the device heap configuration structures during
+	   client device connection  (i.e. this size is relative to a zero-based offset) */
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = 0;
+	}
+	else
+#endif /* defined(FIX_HW_BRN_65273_BIT_MASK) */
+	{
+		psDeviceNode->ui64GeneralSVMHeapTopVA = psHeapInfo->ui64HeapBase + psHeapInfo->uiHeapLength;
+	}
+}
+
+/* Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65273_BIT_MASK)
+static IMG_BOOL BRN65273IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		return (((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_ALT_VALUE) ||
+		        ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_BRN_DEP_VALUE)) ?
+		        IMG_TRUE : IMG_FALSE;
+	}
+	else
+	{
+		return ((pksHeapInfo->ui32HeapInstanceFlags & HEAP_INST_VALUE_MASK) == HEAP_INST_DEFAULT_VALUE) ? IMG_TRUE : IMG_FALSE;
+	}
+}
+#endif
+
+#if defined(FIX_HW_BRN_63142_BIT_MASK)
+static IMG_BOOL BRN63142IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 63142))
+	{
+		PVR_ASSERT((pksHeapInfo->ui64HeapBase & IMG_UINT64_C(0x3FFFFFFFF)) +
+		            pksHeapInfo->uiHeapLength == IMG_UINT64_C(0x400000000));
+
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FBCDescriptorIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, FBC_MAX_DEFAULT_DESCRIPTORS))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_BOOL FBCLargeDescriptorIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, FBC_MAX_LARGE_DESCRIPTORS))
+	{
+		return IMG_TRUE;
+	}
+
+	return IMG_FALSE;
+}
+
+static IMG_BOOL TextureStateIsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	PVR_UNREFERENCED_PARAMETER(pksHeapInfo);
+#if defined(RGX_FEATURE_BINDLESS_IMAGE_AND_TEXTURE_STATE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, BINDLESS_IMAGE_AND_TEXTURE_STATE))
+	{
+		return IMG_TRUE;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevInfo);
+#endif
+	return IMG_FALSE;
+}
+
+/* FW Feature Present function prototypes */
+
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+static IMG_BOOL FWBRN65101IsPresent(PVRSRV_RGXDEV_INFO *psDevInfo, const RGX_HEAP_INFO *pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap size and base at run time VS the current table instance
+	 */
+	IMG_UINT64 ui64MainSubHeapSize;
+
+	/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo)))
+	{
+		if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101;
+		}
+		else
+		{
+			ui64MainSubHeapSize = RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL;
+		}
+	}
+	else
+	{
+		ui64MainSubHeapSize = RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE;
+	}
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->uiHeapLength == ui64MainSubHeapSize &&
+	        pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_MAIN_HEAP_BASE) ?
+	        IMG_TRUE : IMG_FALSE;
+}
+#endif
+
+static IMG_BOOL FWVZConfigPresent(PVRSRV_RGXDEV_INFO* psDevInfo, const RGX_HEAP_INFO* pksHeapInfo)
+{
+	/* Used to determine the correct table row to instantiate as a heap by checking
+	 * the Heap base at run time VS the current table instance
+	 */
+
+	/* Determine if we should include this entry based upon previous checks */
+	return (pksHeapInfo->ui64HeapBase == RGX_FIRMWARE_CONFIG_HEAP_BASE) ? IMG_TRUE : IMG_FALSE;
+}
+
+/* Blueprint array. note: not all heaps are available to clients*/
+
+static RGX_HEAP_INFO gasRGXHeapLayoutApp[] =
+{
+	/* Name                             HeapBase                                 HeapLength                               HeapReservedRegionLength                     Log2ImportAlignment pfnIsHeapPresent             pfnDynamicBaseSize pfnInit           pfnDeInit          HeapInstanceFlags   */
+	{RGX_GENERAL_SVM_HEAP_IDENT,        RGX_GENERAL_SVM_HEAP_BASE,               RGX_GENERAL_SVM_HEAP_SIZE,               0,                                           0,                  NULL,                        SVMHeapDynamic,    NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_HEAP_BASE,                   RGX_GENERAL_HEAP_SIZE,                   RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  BRN65273IsPresent,           NULL,              GeneralHeapInit,  GeneralHeapDeInit, HEAP_INST_DEFAULT_VALUE },
+	{RGX_GENERAL_HEAP_IDENT,            RGX_GENERAL_BRN_65273_HEAP_BASE,         RGX_GENERAL_BRN_65273_HEAP_SIZE,         RGX_GENERAL_HEAP_RESERVED_TOTAL_BYTES,       0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_HEAP_BASE,             RGX_GENERAL_NON4K_HEAP_SIZE,             0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_GENERAL_NON4K_HEAP_IDENT,      RGX_GENERAL_NON4K_BRN_65273_HEAP_BASE,   RGX_GENERAL_NON4K_BRN_65273_HEAP_SIZE,   0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE | HEAP_INST_NON4K_FLAG },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_HEAP_BASE,               RGX_PDSCODEDATA_HEAP_SIZE,               RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_PDSCODEDATA_HEAP_IDENT,        RGX_PDSCODEDATA_BRN_65273_HEAP_BASE,     RGX_PDSCODEDATA_BRN_65273_HEAP_SIZE,     RGX_HEAP_PDS_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_RGNHDR_BRN_63142_HEAP_IDENT,   RGX_RGNHDR_BRN_63142_HEAP_BASE,          RGX_RGNHDR_BRN_63142_HEAP_SIZE,          0,                                           0,                  BRN63142IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_HEAP_BASE,                   RGX_USCCODE_HEAP_SIZE,                   RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_DEFAULT_VALUE },
+	{RGX_USCCODE_HEAP_IDENT,            RGX_USCCODE_BRN_65273_HEAP_BASE,         RGX_USCCODE_BRN_65273_HEAP_SIZE,         RGX_HEAP_USC_RESERVED_TOTAL_SIZE,            0,                  BRN65273IsPresent,           NULL,              USCHeapInit,      HeapDeInit,        HEAP_INST_BRN_ALT_VALUE },
+	{RGX_TQ3DPARAMETERS_HEAP_IDENT,     RGX_TQ3DPARAMETERS_HEAP_BASE,            RGX_TQ3DPARAMETERS_HEAP_SIZE,            0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TQ3DPARAMETERS_HEAP_IDENT,     RGX_TQ3DPARAMETERS_BRN_65273_HEAP_BASE,  RGX_TQ3DPARAMETERS_BRN_65273_HEAP_SIZE,  0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_VK_CAPT_REPLAY_HEAP_IDENT,     RGX_VK_CAPT_REPLAY_HEAP_BASE,            RGX_VK_CAPT_REPLAY_HEAP_SIZE,            0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_FBCDC_HEAP_IDENT,              RGX_FBCDC_HEAP_BASE,                     RGX_FBCDC_HEAP_SIZE,                     0,                                           0,                  FBCDescriptorIsPresent,      NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_FBCDC_LARGE_HEAP_IDENT,        RGX_FBCDC_LARGE_HEAP_BASE,               RGX_FBCDC_LARGE_HEAP_SIZE,               0,                                           0,                  FBCLargeDescriptorIsPresent, NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_CMP_MISSION_RMW_HEAP_IDENT,    RGX_CMP_MISSION_RMW_HEAP_BASE,           RGX_CMP_MISSION_RMW_HEAP_SIZE,           0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_CMP_SAFETY_RMW_HEAP_IDENT,     RGX_CMP_SAFETY_RMW_HEAP_BASE,            RGX_CMP_SAFETY_RMW_HEAP_SIZE,            0,                                           0,                  NULL,                        NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_TEXTURE_STATE_HEAP_IDENT,      RGX_TEXTURE_STATE_HEAP_BASE,             RGX_TEXTURE_STATE_HEAP_SIZE,             0,                                           0,                  TextureStateIsPresent,       NULL,              NULL,             NULL,              HEAP_INST_FEAT_DEP_VALUE},
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_HEAP_BASE,           RGX_VISIBILITY_TEST_HEAP_SIZE,           0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_DEFAULT_VALUE },
+	{RGX_VISIBILITY_TEST_HEAP_IDENT,    RGX_VISIBILITY_TEST_BRN_65273_HEAP_BASE, RGX_VISIBILITY_TEST_BRN_65273_HEAP_SIZE, 0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_ALT_VALUE },
+	{RGX_MMU_INIA_BRN_65273_HEAP_IDENT, RGX_MMU_INIA_BRN_65273_HEAP_BASE,        RGX_MMU_INIA_BRN_65273_HEAP_SIZE,        0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+	{RGX_MMU_INIB_BRN_65273_HEAP_IDENT, RGX_MMU_INIB_BRN_65273_HEAP_BASE,        RGX_MMU_INIB_BRN_65273_HEAP_SIZE,        0,                                           0,                  BRN65273IsPresent,           NULL,              NULL,             NULL,              HEAP_INST_BRN_DEP_VALUE },
+};
+
+static RGX_HEAP_INFO gasRGXHeapLayoutFW[] =
+{
+	/* Name                          HeapBase                        HeapLength                                      HeapReservedRegionLength Log2ImportAlignment pfnIsHeapPresent     pfnDynamicBaseSize pfnInit pfnDeInit HeapInstanceFlags*/
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_DEFAULT_MAIN_HEAP_SIZE,            0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_NORMAL,   0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+	{RGX_FIRMWARE_MAIN_HEAP_IDENT,   RGX_FIRMWARE_MAIN_HEAP_BASE,    RGX_FIRMWARE_HOST_MIPS_MAIN_HEAP_SIZE_BRN65101, 0,                       0,                  FWBRN65101IsPresent, NULL,              NULL,   NULL,      HEAP_INST_BRN_ALT_VALUE},
+	{RGX_FIRMWARE_CONFIG_HEAP_IDENT, RGX_FIRMWARE_CONFIG_HEAP_BASE,  RGX_FIRMWARE_CONFIG_HEAP_SIZE,                  0,                       0,                  FWVZConfigPresent,   NULL,              NULL,   NULL,      HEAP_INST_DEFAULT_VALUE},
+};
+
+static INLINE IMG_BOOL IsFwHeapLayout(const RGX_HEAP_INFO *psHeapInfo)
+{
+	return psHeapInfo->pszName[0] == 'F' &&
+	       psHeapInfo->pszName[1] == 'w' ? IMG_TRUE : IMG_FALSE;
+}
+
+static INLINE void CheckHeapAlignment(const RGX_HEAP_INFO *psHeapInfo,
+                                      const PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_UINT64 uiAlignment = RGX_HEAP_BASE_SIZE_ALIGN - 1;
+
+	if (IsFwHeapLayout(psHeapInfo) ||
+	    RGX_IS_BRN_SUPPORTED(psDevInfo, 65273))
+	{
+		/*
+		 * 1) Main heap starts at 2MB offset 0xEC10000000UL
+		 * 2) Config Sub heap is created at the end of the main heap making the entire unit start and end at 2MB offset aligned,
+		 *    these 2 heaps will always have the same page size
+		 *    There are no other heaps in between these two heaps, there is no risk for another devmem heap be created between them.
+		 */
+		return;
+	}
+
+	/* General SVM Heap must be placed below 2MiB boundary so we need to adjust
+	 * the validity condition. This is because an OS might return virtual
+	 * addresses below 2MiB threshold. By default (based on testing on Linux)
+	 * this is around 32KiB. */
+	if (OSStringNCompare(psHeapInfo->pszName, RGX_GENERAL_SVM_HEAP_IDENT,
+	                     sizeof(RGX_GENERAL_SVM_HEAP_IDENT)) == 0)
+	{
+		uiAlignment = RGX_GENERAL_SVM_BASE_SIZE_ALIGNMENT - 1;
+	}
+
+	/* All UM accessible heap bases should be aligned to 2MB */
+	if (psHeapInfo->ui64HeapBase & uiAlignment)
+	{
+		PVR_ASSERT(!"Heap Base not aligned to RGX_HEAP_BASE_SIZE_ALIGN");
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid Heap \"%s\" Base: "
+		         "%"IMG_UINT64_FMTSPEC")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->ui64HeapBase));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Heap Base (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         psHeapInfo->ui64HeapBase,
+		         uiAlignment + 1));
+	}
+
+	/* All UM accessible heaps should also be size aligned to 2MB */
+	if (psHeapInfo->uiHeapLength & uiAlignment)
+	{
+		PVR_ASSERT(!"Heap Size not aligned to RGX_HEAP_BASE_SIZE_ALIGN");
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Invalid Heap \"%s\" Size: "
+		         "%"IMG_UINT64_FMTSPEC")",
+		         __func__,
+		         psHeapInfo->pszName,
+		         psHeapInfo->uiHeapLength));
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Heap Size (0x%"IMG_UINT64_FMTSPECX") should always be aligned to "
+		         "RGX_HEAP_BASE_SIZE_ALIGN (0x%" IMG_UINT64_FMTSPECX ")",
+		         psHeapInfo->uiHeapLength,
+		         uiAlignment + 1));
+	}
+}
+
+/* Generic counting method. */
+static void _CountRequiredHeaps(PVRSRV_RGXDEV_INFO  *psDevInfo,
+	                            const RGX_HEAP_INFO  pksHeapInfo[],
+	                            IMG_UINT32           ui32HeapListSize,
+	                            IMG_UINT32*          ui32HeapCount)
+{
+	IMG_UINT32 i;
+
+	/* Loop over rows in the heap data array using callback to decide if we
+	 * should include the heap
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo = &pksHeapInfo[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		(*ui32HeapCount)++;
+	}
+}
+/* Generic heap instantiator */
+static void _InstantiateRequiredHeaps(PVRSRV_RGXDEV_INFO     *psDevInfo,
+                                      RGX_HEAP_INFO           psHeapInfos[],
+                                      IMG_UINT32              ui32HeapListSize,
+                                      const IMG_UINT32        ui32Log2RgxDefaultPageShift,
+                                      DEVMEM_HEAP_BLUEPRINT **psDeviceMemoryHeapCursor)
+{
+	IMG_UINT32 i;
+
+#if defined DEBUG
+	IMG_UINT32 ui32heapListCnt;
+	bool bHeapPageSizeMisMatch = false;
+
+	/*
+	 * To ensure all heaps within a 2MB region have the same page sizes
+	 */
+	for (ui32heapListCnt = 0; ui32heapListCnt < (ui32HeapListSize-1); ui32heapListCnt++)
+	{
+		const RGX_HEAP_INFO *psHeapInfo1 = &psHeapInfos[ui32heapListCnt];
+		const RGX_HEAP_INFO *psHeapInfo2 = &psHeapInfos[ui32heapListCnt+1];
+
+		if (((psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) &&
+			((psHeapInfo1->ui64HeapBase + psHeapInfo1->uiHeapLength) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)) ==
+			((psHeapInfo2->ui64HeapBase) & (RGX_HEAP_BASE_SIZE_ALIGN - 1)))
+		{
+			if (psHeapInfo1->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				if (!(psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG))
+				{
+					bHeapPageSizeMisMatch = true;
+				}
+			}
+			else if (psHeapInfo2->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+			{
+				bHeapPageSizeMisMatch = true;
+			}
+		}
+		if (bHeapPageSizeMisMatch)
+		{
+			PVR_ASSERT(!"Two Heap with Different Page Size allocated in the same PD space(2MB)");
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid Heaps 1) \"%s\" and 2) \"%s\"",
+					__func__,
+					psHeapInfo1->pszName,
+					psHeapInfo2->pszName));
+		}
+	}
+#endif
+
+	/* We now have a list of the heaps to include and so we should loop over this
+	 * list and instantiate.
+	 */
+	for (i = 0; i < ui32HeapListSize; i++)
+	{
+		IMG_UINT32 ui32Log2DataPageSize = 0;
+		RGX_HEAP_INFO *psHeapInfo = &psHeapInfos[i];
+
+		if (psHeapInfo->pfnIsHeapPresent)
+		{
+			if (!psHeapInfo->pfnIsHeapPresent(psDevInfo, psHeapInfo))
+			{
+				/* We don't need to create this heap */
+				continue;
+			}
+		}
+
+		if (psHeapInfo->pfnDynamicBaseSize != NULL)
+		{
+			psHeapInfo->pfnDynamicBaseSize(psDevInfo->psDeviceNode, psHeapInfo);
+		}
+
+		if (psHeapInfo->ui32HeapInstanceFlags & HEAP_INST_NON4K_FLAG)
+		{
+			ui32Log2DataPageSize = psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2;
+		}
+		else
+		{
+			ui32Log2DataPageSize = ui32Log2RgxDefaultPageShift;
+		}
+
+		CheckHeapAlignment(psHeapInfo, psDevInfo);
+
+		HeapCfgBlueprintInit(psHeapInfo->pszName,
+		                     psHeapInfo->ui64HeapBase,
+		                     psHeapInfo->uiHeapLength,
+		                     psHeapInfo->uiHeapReservedRegionLength,
+		                     ui32Log2DataPageSize,
+		                     psHeapInfo->ui32Log2ImportAlignment,
+		                     psHeapInfo->pfnInit,
+		                     psHeapInfo->pfnDeInit,
+		                     *psDeviceMemoryHeapCursor);
+
+		(*psDeviceMemoryHeapCursor)++;
+	}
+}
+
+static PVRSRV_ERROR RGXInitHeaps(PVRSRV_RGXDEV_INFO *psDevInfo,
+	                             DEVICE_MEMORY_INFO *psNewMemoryInfo)
+{
+	PVRSRV_ERROR eError;
+	DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor;
+
+	IMG_UINT32 ui32AppHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutApp);
+	IMG_UINT32 ui32FWHeapListSize = ARRAY_SIZE(gasRGXHeapLayoutFW);
+	IMG_UINT32 ui32CountedHeapSize;
+
+	IMG_UINT32 ui32AppHeapCount = 0;
+	IMG_UINT32 ui32FWHeapCount = 0;
+
+	IMG_UINT32 ui32Log2DefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	if (ui32Log2DefaultPageShift == 0)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2DefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2DefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e0;
+		}
+	}
+#endif
+
+	/* Count heaps required for the app heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutApp,
+		                ui32AppHeapListSize,
+		                &ui32AppHeapCount);
+
+	/* Count heaps required for the FW heaps */
+	_CountRequiredHeaps(psDevInfo,
+		                gasRGXHeapLayoutFW,
+		                ui32FWHeapListSize,
+		                &ui32FWHeapCount);
+
+	ui32CountedHeapSize = (ui32AppHeapCount + ui32FWHeapCount + RGX_NUM_DRIVERS_SUPPORTED);
+
+	psNewMemoryInfo->psDeviceMemoryHeap = OSAllocMem(sizeof(DEVMEM_HEAP_BLUEPRINT) * ui32CountedHeapSize);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeap, eError, e0);
+
+	/* Initialise the heaps */
+	psDeviceMemoryHeapCursor = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	/* Instantiate App Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutApp,
+	                          ui32AppHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* Instantiate FW Heaps */
+	_InstantiateRequiredHeaps(psDevInfo,
+	                          gasRGXHeapLayoutFW,
+	                          ui32FWHeapListSize,
+	                          ui32Log2DefaultPageShift,
+	                          &psDeviceMemoryHeapCursor);
+
+	/* set the heap count */
+	psNewMemoryInfo->ui32HeapCount = (IMG_UINT32)(psDeviceMemoryHeapCursor - psNewMemoryInfo->psDeviceMemoryHeap);
+
+	/* Check we have allocated the correct # of heaps, minus any VZ heaps as these
+	 * have not been created at this point
+	 */
+	PVR_ASSERT(psNewMemoryInfo->ui32HeapCount == (ui32CountedHeapSize - RGX_NUM_DRIVERS_SUPPORTED));
+
+	/*
+	   In the new heap setup, we initialise 2 configurations:
+		1 - One will be for the firmware only (index 1 in array)
+			a. This primarily has the firmware heap in it.
+			b. It also has additional guest OSID firmware heap(s)
+				- Only if the number of support firmware OSID > 1
+		2 - Others shall be for clients only (index 0 in array)
+			a. This has all the other client heaps in it.
+	 */
+	psNewMemoryInfo->uiNumHeapConfigs = 2;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray = OSAllocMem(sizeof(DEVMEM_HEAP_CONFIG) * psNewMemoryInfo->uiNumHeapConfigs);
+	PVR_LOG_GOTO_IF_NOMEM(psNewMemoryInfo->psDeviceMemoryHeapConfigArray, eError, e1);
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].pszName = "Default Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].uiNumHeaps = psNewMemoryInfo->ui32HeapCount - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[0].psHeapBlueprintArray = psNewMemoryInfo->psDeviceMemoryHeap;
+
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].pszName = "Firmware Heap Configuration";
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps = RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+	psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].psHeapBlueprintArray = psDeviceMemoryHeapCursor - RGX_FIRMWARE_NUMBER_OF_FW_HEAPS;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4)
+	{
+		IMG_UINT32 i;
+		const IMG_UINT32 ui32GeneralNon4KHeapPageSize = (1 << psDevInfo->psDeviceNode->ui32Non4KPageSizeLog2);
+		const IMG_UINT32 ui32RgxDefaultPageSize = (1 << RGXHeapDerivePageSize(OSGetPageShift()));
+
+		/*
+		 * Initialise all MMU Page Size Range Config register to the default page size
+		 * used by the OS, leaving the address range 0;
+		 */
+		for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[i] =
+					RGXMMUInit_GetConfigRangeValue(ui32RgxDefaultPageSize,
+												   0,
+												   (1 << RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT));
+		}
+
+
+		/* set the last MMU config range covering the entire virtual memory to the OS's page size */
+		psDevInfo->aui64MMUPageSizeRangeValue[RGX_MAX_NUM_MMU_PAGE_SIZE_RANGES - 1] =
+				RGXMMUInit_GetConfigRangeValue(ui32RgxDefaultPageSize, 0, (1ULL << 40));
+
+		/*
+		 * If the Non4K heap has a different page size than the OS's page size
+		 * (used as default for all other heaps), configure one MMU config range
+		 * for the Non4K heap
+		 */
+		if (ui32GeneralNon4KHeapPageSize != ui32RgxDefaultPageSize)
+		{
+			psDevInfo->aui64MMUPageSizeRangeValue[0] =
+					RGXMMUInit_GetConfigRangeValue(ui32GeneralNon4KHeapPageSize,
+												   RGX_GENERAL_NON4K_HEAP_BASE,
+												   RGX_GENERAL_NON4K_HEAP_SIZE);
+		}
+	}
+#endif
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_UINT32 ui32DriverID;
+
+		/* Create additional raw firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVINFO, psDevInfo)
+		{
+			eError = RGXInitFwRawHeap(psDevInfo, psDeviceMemoryHeapCursor, ui32DriverID);
+			if (eError != PVRSRV_OK)
+			{
+				/* if any allocation fails, free previously allocated heaps and abandon initialisation */
+				for (; ui32DriverID > RGX_FIRST_RAW_HEAP_DRIVER_ID; ui32DriverID--)
+				{
+					RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+					psDeviceMemoryHeapCursor--;
+				}
+				goto e1;
+			}
+
+			/* Append additional firmware heaps to host driver firmware context heap configuration */
+			psNewMemoryInfo->psDeviceMemoryHeapConfigArray[1].uiNumHeaps += 1;
+
+			/* advance to the next heap */
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	return PVRSRV_OK;
+e1:
+	OSFreeMem(psNewMemoryInfo->psDeviceMemoryHeap);
+e0:
+	return eError;
+}
+
+static void RGXDeInitHeaps(DEVICE_MEMORY_INFO *psDevMemoryInfo, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		IMG_UINT32 ui32DriverID;
+		DEVMEM_HEAP_BLUEPRINT *psDeviceMemoryHeapCursor = psDevMemoryInfo->psDeviceMemoryHeap;
+
+		/* Delete all guest firmware heaps */
+		FOREACH_DRIVER_RAW_HEAP(ui32DriverID, DEVNODE, psDeviceNode)
+		{
+			RGXDeInitFwRawHeap(psDeviceMemoryHeapCursor);
+			psDeviceMemoryHeapCursor++;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeapConfigArray);
+	OSFreeMem(psDevMemoryInfo->psDeviceMemoryHeap);
+}
+
+static PVRSRV_ERROR RGXInitSharedFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psSysHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																PHYS_HEAP_USAGE_FW_SHARED);
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+	/* VZ heap validation */
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg != NULL,
+								"FW Main heap is required for VZ Guest.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+#endif
+
+	if (psSysHeapCfg != NULL)
+	{
+		/* Check FW_SHARED for multiple usage flags. Because FW_SHARED is divided
+		   into subheaps, shared usage with other heaps is not allowed.  */
+		PVR_LOG_RETURN_IF_FALSE(psSysHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_SHARED,
+								"FW_SHARED phys heap config not specified with more than one usage."
+								"FW_SHARED heap must be exclusively used as FW_SHARED.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+	}
+
+	if (psSysHeapCfg == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap not set", __func__));
+		/* Nothing to do. Default to the physheap fallback option */
+	}
+	else if (psSysHeapCfg->eType == PHYS_HEAP_TYPE_UMA)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses OS System memory (UMA)", __func__));
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psSysHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG] = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	}
+	else /* PHYS_HEAP_TYPE_LMA or PHYS_HEAP_TYPE_DMA */
+	{
+		PHYS_HEAP_CONFIG sFwMainHeapCfg, sFwCfgHeapCfg;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Firmware physical heap uses local memory managed by the driver (LMA)", __func__));
+
+
+		/* Subheap layout: Main + (optional MIPS reserved range) + Config */
+		sFwMainHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwMainHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		/* Reserve space for the Config heap */
+		sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+			/* MIPS Firmware must reserve some space in its Host/Native heap for GPU memory mappings */
+			if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) && (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)))
+			{
+#if defined(FIX_HW_BRN_65101_BIT_MASK)
+				if (RGX_IS_BRN_SUPPORTED(psDevInfo, 65101))
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_BRN65101;
+				}
+				else
+#endif
+				{
+					sFwMainHeapCfg.uConfig.sLMA.uiSize -= RGX_FIRMWARE_MIPS_GPU_MAP_RESERVED_SIZE_NORMAL;
+				}
+			}
+		}
+#endif
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwMainHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwMainHeapCfg,
+		                              "Fw Main subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:MAIN", ErrorDeinit);
+
+		sFwCfgHeapCfg = *psSysHeapCfg;
+		PVR_ASSERT(sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_LMA ||
+		           sFwCfgHeapCfg.eType == PHYS_HEAP_TYPE_DMA);
+
+		sFwCfgHeapCfg.uConfig.sLMA.sStartAddr.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+		sFwCfgHeapCfg.uConfig.sLMA.sCardBase.uiAddr += psSysHeapCfg->uConfig.sLMA.uiSize - RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		sFwCfgHeapCfg.uConfig.sLMA.uiSize = RGX_FIRMWARE_CONFIG_HEAP_SIZE;
+
+		eError = PhysmemCreateHeapLMA(psDeviceNode,
+		                              RGXPhysHeapGetLMAPolicy(sFwCfgHeapCfg.ui32UsageFlags, psDeviceNode),
+		                              &sFwCfgHeapCfg,
+		                              "Fw Cfg subheap",
+		                              &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+		PVR_LOG_GOTO_IF_ERROR(eError, "PhysmemCreateHeapLMA:CFG", ErrorDeinit);
+	}
+
+	/* Acquire FW heaps */
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_MAIN, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_MAIN", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CONFIG, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CONFIG]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CONFIG", ErrorDeinit);
+
+	return eError;
+
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitPrivateFwPhysHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PHYS_HEAP_CONFIG *psFwCodeHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_CODE);
+	PHYS_HEAP_CONFIG *psFwDataHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIV_DATA);
+	PHYS_HEAP_CONFIG *psFwPrivateHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																 PHYS_HEAP_USAGE_FW_PRIVATE);
+	PHYS_HEAP_CONFIG sFwPrivateTempCfg;
+
+	if (psFwPrivateHeapCfg != NULL)
+	{
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg == NULL) && (psFwDataHeapCfg == NULL),
+								"FW_PRIVATE and the FW_CODE & FW_PRIV_DATA usage flags "
+								"achieve the same goal and are mutually exclusive.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Fw code and data are both allocated from this unified heap */
+		sFwPrivateTempCfg = *psFwPrivateHeapCfg;
+		sFwPrivateTempCfg.ui32UsageFlags = PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA;
+
+		psFwCodeHeapCfg = &sFwPrivateTempCfg;
+		psFwDataHeapCfg = &sFwPrivateTempCfg;
+	}
+
+	if ((psFwCodeHeapCfg == NULL) || (psFwDataHeapCfg == NULL))
+	{
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware code and data heaps must be either both defined
+			 * or both undefined. There is no point in isolating one but not
+			 * the other.*/
+			eError = PVRSRV_ERROR_PHYSHEAP_CONFIG;
+			PVR_LOG_GOTO_IF_ERROR(eError, "PrivateFwPhysHeap check", ErrorDeinit);
+		}
+		else
+		{
+			/* No dedicated heaps, default to the physheap fallback option */
+		}
+	}
+	else if (psFwCodeHeapCfg == psFwDataHeapCfg)
+	{
+		if (psFwCodeHeapCfg->ui32UsageFlags ==
+			(PHYS_HEAP_USAGE_FW_CODE | PHYS_HEAP_USAGE_FW_PRIV_DATA))
+		{
+			/* Fw code and private data allocations come from the same system heap
+			 * Instantiate one physheap and share it between them. */
+
+			eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+												  psFwCodeHeapCfg,
+												  NULL);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig");
+		}
+		else
+		{
+			/* Not an exclusive heap, can be used for other purposes (e.g. secure buffers).
+			 * Expect the PVR layer to have already created a heap for the other uses. */
+		}
+	}
+	else
+	{
+		/*
+		 * Separating private Firmware code and data is allowed for backwards compatibility
+		 * purposes. New platforms should use the unified FW_PRIVATE heap instead.
+		 *
+		 * Early security implementations on Rogue cores required separate FW_PRIV_DATA
+		 * and FW_CODE heaps, as access permissions to Firmware were granted differently
+		 * based on the transaction types (code or data).
+		 */
+		PVR_LOG_RETURN_IF_FALSE((psFwCodeHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_CODE) &&
+								(psFwDataHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PRIV_DATA),
+								"Dedicated private heaps for Fw code and "
+								"data must have one usage flag exclusively.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		/* Dedicated Fw code heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwCodeHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+
+		/* Dedicated Fw private data heap */
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwDataHeapCfg,
+											  NULL);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysmemCreateHeap");
+	}
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+	/* When premapping distinct private and shared Firmware phys heaps
+	 * inside the same virtual devmem heap, their sizes must add up to
+	 * the fixed RGX_FIRMWARE_RAW_HEAP_SIZE for the premapping to work */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwSharedHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		PHYS_HEAP_USAGE_FW_SHARED);
+		IMG_UINT64 ui64FwCodeHeapSize = PhysHeapConfigGetSize(psFwCodeHeapCfg);
+		IMG_UINT64 ui64FwDataHeapSize = PhysHeapConfigGetSize(psFwDataHeapCfg);
+		IMG_UINT64 ui64FwSharedHeapSize = PhysHeapConfigGetSize(psFwSharedHeapCfg);
+		IMG_UINT64 ui64FwPrivateHeapSize;
+
+		PVR_LOG_GOTO_IF_FALSE((psFwCodeHeapCfg != NULL) && (psFwDataHeapCfg != NULL),
+							  "Security support requires Fw code and data memory be"
+							  " separate from the heap shared with the kernel driver.", FailDeinit);
+
+		if (psFwCodeHeapCfg != psFwDataHeapCfg)
+		{
+			/* Private Firmware allocations come from 2 different heaps */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize + ui64FwDataHeapSize;
+		}
+		else
+		{
+			/* Private Firmware allocations come from a single heap */
+			ui64FwPrivateHeapSize = ui64FwCodeHeapSize;
+		}
+
+				PVR_LOG_GOTO_IF_FALSE((ui64FwSharedHeapSize +
+									   ui64FwPrivateHeapSize) ==
+									   RGX_FIRMWARE_RAW_HEAP_SIZE,
+									   "Invalid firmware physical heap size.", FailDeinit);
+	}
+#endif
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_CODE, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_CODE]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_CODE", ErrorDeinit);
+
+	eError = PhysHeapAcquireByID(PVRSRV_PHYS_HEAP_FW_PRIV_DATA, psDeviceNode,
+								 &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PRIV_DATA]);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PhysHeapAcquire:FW_DATA", ErrorDeinit);
+
+	return eError;
+
+#if defined(RGX_PREMAP_FW_HEAPS) && defined(SUPPORT_TRUSTED_DEVICE)
+FailDeinit:
+    eError = PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+ErrorDeinit:
+	PVR_ASSERT(IMG_FALSE);
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXInitFwPageTableHeap(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if defined(RGX_PREMAP_FW_HEAPS)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		PHYS_HEAP_CONFIG *psFwPageTableHeapCfg = PVRSRVFindPhysHeapConfig(psDeviceNode->psDevConfig,
+																		  PHYS_HEAP_USAGE_FW_PREMAP_PT);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg != NULL),
+								"The Firmware Page Table phys heap config not found.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->ui32UsageFlags == PHYS_HEAP_USAGE_FW_PREMAP_PT),
+								"The Firmware Page Table heap must be used exclusively for this purpose",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_LMA) ||
+								(psFwPageTableHeapCfg->eType == PHYS_HEAP_TYPE_DMA),
+								"The Firmware Page Table heap must be LMA or DMA memory.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		PVR_LOG_RETURN_IF_FALSE((psFwPageTableHeapCfg->uConfig.sLMA.uiSize >= RGX_FIRMWARE_MAX_PAGETABLE_SIZE),
+								"The Firmware Page Table heap must be able to hold the maximum "
+								"number of pagetables needed to cover the Firmware's VA space.",
+								PVRSRV_ERROR_PHYSHEAP_CONFIG);
+
+		eError = PhysHeapCreateHeapFromConfig(psDeviceNode,
+											  psFwPageTableHeapCfg,
+											  &psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapCreateHeapFromConfig:FwPageTableHeap");
+
+		eError = PhysHeapAcquire(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_PREMAP_PT]);
+		PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapAcquire:FwPageTableHeap");
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+#endif /* defined(RGX_PREMAP_FW_HEAPS) */
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXPhysMemDeviceHeapsInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	eError = RGXInitFwPageTableHeap(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitFwPageTableHeap", ErrorDeinit);
+	eError = RGXInitSharedFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitSharedFwPhysHeaps", ErrorDeinit);
+	eError = RGXInitPrivateFwPhysHeaps(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXInitPrivateFwPhysHeaps", ErrorDeinit);
+
+ErrorDeinit:
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDeviceFWMainHeapMemCheck
+@Description    Checks the free memory in FW Main PhysHeap of a device to ensure
+                there is enough for a connection to be made.
+
+@Input          psDeviceNode    The device of the FW Main PhysHeap to be checked.
+
+@Return         On success PVRSRV_OK, else a PVRSRV_ERROR code.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDeviceFWMainHeapMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP *psFWMainPhysHeap;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode, "psDeviceNode");
+
+	psFWMainPhysHeap = psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN];
+	if (psFWMainPhysHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's FW Main PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psFWMainPhysHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		const IMG_UINT32 ui32MinMemInKBs = RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION;
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psFWMainPhysHeap,
+		                              KB2B(ui32MinMemInKBs),
+		                              &ui64FreePhysHeapMem);
+
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "FW_MAIN PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"KB "
+				"Minimum required: %uKB",
+				B2KB(ui64FreePhysHeapMem),
+				ui32MinMemInKBs));
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate, IMG_UINT32 *pui32Log2Non4KPgShift)
+{
+	IMG_UINT32 ui32GeneralNon4KHeapPageSize;
+	IMG_UINT32 uiLog2OSPageShift = OSGetPageShift();
+
+	/* We support Non4K pages only on platforms with 4KB pages. On all platforms
+	 * where OS pages are larger than 4KB we must ensure the non4K device memory
+	 * heap matches the page size used in all other device memory heaps, which
+	 * is the OS page size, see RGXHeapDerivePageSize. */
+	if (uiLog2OSPageShift > RGX_HEAP_4KB_PAGE_SHIFT)
+	{
+		*pui32Log2Non4KPgShift = RGXHeapDerivePageSize(uiLog2OSPageShift);
+	}
+	else
+	{
+		void *pvAppHintState = NULL;
+		IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_GENERALNON4KHEAPPAGESIZE;
+		IMG_UINT32 ui32Log2Non4KPgShift;
+
+		/* Get the page size for the dummy page from the NON4K heap apphint */
+		OSCreateAppHintState(&pvAppHintState);
+		OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState,
+		                   GeneralNon4KHeapPageSize, &ui32AppHintDefault,
+		                   &ui32GeneralNon4KHeapPageSize);
+		OSFreeAppHintState(pvAppHintState);
+
+		/* Validate the specified parameter to be one of the supported values */
+		ui32Log2Non4KPgShift = RGXHeapDerivePageSize(ExactLog2(ui32GeneralNon4KHeapPageSize));
+		if (ui32AppHintDefault != ui32GeneralNon4KHeapPageSize)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Invalid Non4K Page-size, default=%u, requested=%u,"
+			         " actual=%u, pageshift 0x%x",
+			         __func__, ui32AppHintDefault, ui32GeneralNon4KHeapPageSize,
+			         (1U << ui32Log2Non4KPgShift), ui32Log2Non4KPgShift));
+		}
+
+		if (ui32Log2Non4KPgShift == 0U)
+		{
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+
+		*pui32Log2Non4KPgShift = ui32Log2Non4KPgShift;
+	}
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 71317))
+	{
+		if (*pui32Log2Non4KPgShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || *pui32Log2Non4KPgShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "Page sizes of 2MB or 1MB cause page faults."));
+			return PVRSRV_ERROR_INVALID_NON4K_HEAP_PAGESIZE;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+#endif
+
+	/* Check the Non4k page size is at least the size of the OS page size
+	 * or larger. The Non4k page size also has to be a multiple of the OS page
+	 * size but since we have the log2 value from the apphint we know powers of 2
+	 * will always be multiples. If the Non4k page size is less than OS page size
+	 * we notify and upgrade the size.
+	 */
+	if (*pui32Log2Non4KPgShift < uiLog2OSPageShift)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "Non4K page size smaller than OS page size, upgrading to "
+		                          "match OS page size."));
+		*pui32Log2Non4KPgShift = uiLog2OSPageShift;
+	}
+
+	return PVRSRV_OK;
+}
+
+/* RGXRegisterDevice
+ *
+ * WARNING!
+ *
+ * No PDUMP statements are allowed until device initialisation starts.
+ */
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	DEVICE_MEMORY_INFO *psDevMemoryInfo;
+	PVRSRV_RGXDEV_INFO	*psDevInfo;
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault = HWPERF_HOST_TL_STREAM_SIZE_DEFAULT, ui32HWPerfHostBufSizeKB;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault, &ui32HWPerfHostBufSizeKB);
+	OSFreeAppHintState(pvAppHintState);
+	pvAppHintState = NULL;
+
+	/*********************
+	 * Device node setup *
+	 *********************/
+	/* Setup static data and callbacks on the device agnostic device node */
+#if defined(PDUMP)
+	psDeviceNode->sDevId.pszPDumpRegName	= RGX_PDUMPREG_NAME;
+	psDeviceNode->sDevId.pszPDumpDevName	= PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+	psDeviceNode->pfnPDumpInitDevice = &RGXResetPDump;
+#endif /* PDUMP */
+
+	OSAtomicWrite(&psDeviceNode->eHealthStatus, PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	OSAtomicWrite(&psDeviceNode->eHealthReason, PVRSRV_DEVICE_HEALTH_REASON_NONE);
+
+	/* Configure MMU specific stuff */
+	RGXMMUInit_Register(psDeviceNode);
+
+	psDeviceNode->pfnInvalFBSCTable = NULL;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = NULL;
+
+	/* Callback for getting the MMU device attributes */
+	psDeviceNode->pfnGetMMUDeviceAttributes = RGXDevMMUAttributes;
+	psDeviceNode->pfnGetDeviceSnoopMode = RGXDevSnoopMode;
+	psDeviceNode->pfnMMUCacheInvalidate = RGXMMUCacheInvalidate;
+	psDeviceNode->pfnMMUCacheInvalidateKick = RGXMMUCacheInvalidateKick;
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = RGXMapBRN71422TargetPhysicalAddress;
+#else
+	psDeviceNode->pfnMMUTopLevelPxWorkarounds = NULL;
+#endif
+	/* pfnMMUTweakProtFlags is set later on once BVNC features are setup */
+	psDeviceNode->pfnMMUTweakProtFlags = NULL;
+
+	psDeviceNode->pfnInitDeviceCompatCheck	= &RGXDevInitCompatCheck;
+
+	/* Register callbacks for creation of device memory contexts */
+	psDeviceNode->pfnRegisterMemoryContext = RGXRegisterMemoryContext;
+	psDeviceNode->pfnUnregisterMemoryContext = RGXUnregisterMemoryContext;
+
+	/* Register callbacks for Unified Fence Objects */
+	psDeviceNode->pfnAllocUFOBlock = RGXAllocUFOBlock;
+	psDeviceNode->pfnFreeUFOBlock = RGXFreeUFOBlock;
+
+	/* Register callback for checking the device's health */
+	psDeviceNode->pfnUpdateHealthStatus = PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) ? NULL : RGXUpdateHealthStatus;
+
+#if defined(SUPPORT_AUTOVZ)
+	/* Register callback for updating the virtualization watchdog */
+	psDeviceNode->pfnUpdateAutoVzWatchdog = RGXUpdateAutoVzWatchdog;
+#endif
+
+	/* Register method to service the FW HWPerf buffer */
+	psDeviceNode->pfnServiceHWPerf = RGXHWPerfDataStoreCB;
+
+	/* Register callback for getting the device version information string */
+	psDeviceNode->pfnDeviceVersionString = RGXDevVersionString;
+
+	/* Register callback for getting the device clock speed */
+	psDeviceNode->pfnDeviceClockSpeed = RGXDevClockSpeed;
+
+	/* Register callback for soft resetting some device modules */
+	psDeviceNode->pfnSoftReset = RGXSoftReset;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnResetHWRLogs = RGXResetHWRLogs;
+
+	/* Register callback for resetting the HWR logs */
+	psDeviceNode->pfnVerifyBVNC = RGXVerifyBVNC;
+
+	/* Register callback for checking alignment of UM structures */
+	psDeviceNode->pfnAlignmentCheck = RGXAlignmentCheck;
+
+	/*Register callback for checking the supported features and getting the
+	 * corresponding values */
+	psDeviceNode->pfnCheckDeviceFeature = RGXBvncCheckFeatureSupported;
+	psDeviceNode->pfnGetDeviceFeatureValue = RGXBvncGetSupportedFeatureValue;
+
+	/* Callback for checking if system layer supports FBC 3.1 */
+	psDeviceNode->pfnHasFBCDCVersion31 = RGXSystemHasFBCDCVersion31;
+
+	/* Callback for getting TFBC configuration */
+	psDeviceNode->pfnGetTFBCLossyGroup = RGXGetTFBCLossyGroup;
+
+	/* Register callback for initialising device-specific physical memory heaps */
+	psDeviceNode->pfnPhysMemDeviceHeapsInit = RGXPhysMemDeviceHeapsInit;
+
+	/* Register callback for checking a device's FW Main physical heap for sufficient free memory */
+	psDeviceNode->pfnCheckForSufficientFWPhysMem = RGXDeviceFWMainHeapMemCheck;
+
+	/* Register callback for determining the appropriate LMA allocation policy for a phys heap */
+	psDeviceNode->pfnPhysHeapGetLMAPolicy = RGXPhysHeapGetLMAPolicy;
+
+	/*********************
+	 * Device info setup *
+	 *********************/
+	/* Allocate device control block */
+	psDevInfo = OSAllocZMem(sizeof(*psDevInfo));
+	if (psDevInfo == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "DevInitRGXPart1 : Failed to alloc memory for DevInfo"));
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* initialise the layer parameters needed for early hw feature checks */
+	psDevInfo->sLayerParams.psDevInfo = psDevInfo;
+	psDevInfo->sLayerParams.psDevConfig = psDeviceNode->psDevConfig;
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Default psTrampoline to point to null struct */
+	psDevInfo->psTrampoline = (RGX_MIPS_ADDRESS_TRAMPOLINE *)&sNullTrampoline;
+#endif
+
+	/* create locks for the context lists stored in the DevInfo structure.
+	 * these lists are modified on context create/destroy and read by the
+	 * watchdog thread
+	 */
+
+	eError = OSWRLockCreate(&(psDevInfo->hRenderCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create render context list lock", __func__));
+		goto e0;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hComputeCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create compute context list lock", __func__));
+		goto e1;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTransferCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create transfer context list lock", __func__));
+		goto e2;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hTDMCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create TDM context list lock", __func__));
+		goto e3;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hKickSyncCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create kick sync context list lock", __func__));
+		goto e4;
+	}
+
+	eError = OSWRLockCreate(&(psDevInfo->hMemoryCtxListLock));
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create memory context list lock", __func__));
+		goto e5;
+	}
+
+	eError = OSSpinLockCreate(&psDevInfo->hLockKCCBDeferredCommandsList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to KCCB deferred commands list lock", __func__));
+		goto e6;
+	}
+	dllist_init(&(psDevInfo->sKCCBDeferredCommandsListHead));
+
+	dllist_init(&(psDevInfo->sRenderCtxtListHead));
+	dllist_init(&(psDevInfo->sComputeCtxtListHead));
+	dllist_init(&(psDevInfo->sTransferCtxtListHead));
+	dllist_init(&(psDevInfo->sTDMCtxtListHead));
+	dllist_init(&(psDevInfo->sKickSyncCtxtListHead));
+
+	dllist_init(&(psDevInfo->sCommonCtxtListHead));
+	psDevInfo->ui32CommonCtxtCurrentID = 1;
+
+
+	eError = OSWRLockCreate(&psDevInfo->hCommonCtxtListLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create common context list lock", __func__));
+		goto e7;
+	}
+
+	eError = OSLockCreate(&psDevInfo->sRegConfig.hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create register configuration lock", __func__));
+		goto e8;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hBPLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for break points", __func__));
+		goto e9;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hRGXFWIfBufInitLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for trace buffers", __func__));
+		goto e10;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hCCBStallCheckLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB checking lock", __func__));
+		goto e11;
+	}
+	eError = OSLockCreate(&psDevInfo->hCCBRecoveryLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create stalled CCB recovery lock", __func__));
+		goto e12;
+	}
+
+	eError = OSLockCreate(&psDevInfo->hGpuUtilStatsLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create GPU stats lock", __func__));
+		goto e13;
+	}
+
+	dllist_init(&psDevInfo->sMemoryContextList);
+
+	/* initialise ui32SLRHoldoffCounter */
+	if (RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS > DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT)
+	{
+		psDevInfo->ui32SLRHoldoffCounter = RGX_INITIAL_SLR_HOLDOFF_PERIOD_MS / DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT;
+	}
+	else
+	{
+		psDevInfo->ui32SLRHoldoffCounter = 0;
+	}
+
+	/* Setup static data and callbacks on the device specific device info */
+	psDevInfo->psDeviceNode		= psDeviceNode;
+
+	psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
+	psDevInfo->pvDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
+
+	/*
+	 * Map RGX Registers
+	 */
+	psDevInfo->ui32RegSize = psDeviceNode->psDevConfig->ui32RegsSize;
+	psDevInfo->sRegsPhysBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+#if !defined(NO_HARDWARE)
+	psDevInfo->pvRegsBaseKM = (void __iomem *) OSMapPhysToLin(psDeviceNode->psDevConfig->sRegsCpuPBase,
+			psDeviceNode->psDevConfig->ui32RegsSize,
+			PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+	if (psDevInfo->pvRegsBaseKM == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to create RGX register mapping",
+		         __func__));
+		eError = PVRSRV_ERROR_BAD_MAPPING;
+		goto e14;
+	}
+#endif /* !NO_HARDWARE */
+
+	psDeviceNode->pvDevice = psDevInfo;
+
+	eError = RGXBvncInitialiseConfiguration(psDeviceNode);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Unsupported HW device detected by driver",
+		         __func__));
+		goto e15;
+	}
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	/*
+	 * We must now setup the SECURITY mappings if supported. We cannot
+	 * check on the features until we have reached here as the BVNC is
+	 * not setup before now.
+	 */
+#if !defined(NO_HARDWARE)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+	{
+		IMG_CPU_PHYADDR sHostSecureRegBankBase = {psDeviceNode->psDevConfig->sRegsCpuPBase.uiAddr + RGX_HOST_SECURE_REGBANK_OFFSET};
+
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *) OSMapPhysToLin(sHostSecureRegBankBase,
+																		RGX_HOST_SECURE_REGBANK_SIZE,
+																		PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+
+		if (psDevInfo->pvSecureRegsBaseKM == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "PVRSRVRGXInitDevPart2KM: Failed to create RGX secure register mapping"));
+			eError = PVRSRV_ERROR_BAD_MAPPING;
+			goto e15;
+		}
+
+		/*
+		 * The secure register bank is mapped into the CPU VA space starting from
+		 * the base of the normal register bank + an offset of RGX_HOST_SECURE_REGBAK_OFFSET.
+		 * The hardware register addresses are all indexed from the base of the regular register bank.
+		 * For the RegBankBase+RegOffset computation to still be accurate for host-secure registers,
+		 * we need to compensate for offsets of registers in the secure bank
+		 */
+		psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM - RGX_HOST_SECURE_REGBANK_OFFSET);
+	}
+	else
+	{
+		psDevInfo->pvSecureRegsBaseKM = psDevInfo->pvRegsBaseKM;
+	}
+#endif /* !NO_HARDWARE */
+#endif /* defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	psDeviceNode->pfnMMUTweakProtFlags = (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS)) ?
+			RGXMMUTweakProtFlags : NULL;
+#endif
+
+	eError = RGXGetNon4KHeapPageShift(&psDevInfo->sLayerParams,
+	                                &psDeviceNode->ui32Non4KPageSizeLog2);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXGetNon4KHeapPageSize", e16);
+
+	eError = RGXInitHeaps(psDevInfo, psDevMemoryInfo);
+	if (eError != PVRSRV_OK)
+	{
+		goto e16;
+	}
+
+	eError = RGXHWPerfInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfInit", e16);
+
+	eError = RGXHWPerfHostInit(psDeviceNode->pvDevice, ui32HWPerfHostBufSizeKB);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXHWPerfHostInit", ErrorDeInitHWPerfFw);
+
+
+	/* Register callback for dumping debug info */
+	eError = RGXDebugInit(psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDebugInit", e17);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	/* Register callback for fw mmu init */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		psDeviceNode->pfnFwMMUInit = RGXMipsMMUInit_Register;
+	}
+#endif
+
+	if (NULL != psDeviceNode->psDevConfig->pfnSysDevFeatureDepInit)
+	{
+		psDeviceNode->psDevConfig->pfnSysDevFeatureDepInit(psDeviceNode->psDevConfig,
+				psDevInfo->sDevFeatureCfg.ui64Features);
+	}
+
+	/* Initialise the device dependent bridges */
+	eError = RGXRegisterBridges(psDevInfo);
+	PVR_LOG_IF_ERROR(eError, "RGXRegisterBridges");
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+	eError = OSLockCreate(&psDevInfo->hCounterDumpingLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock for counter sampling.", __func__));
+		goto ErrorDeInitDeviceDepBridge;
+	}
+#endif
+
+	/* Initialise error counters */
+	memset(&psDevInfo->sErrorCounts, 0, sizeof(PVRSRV_RGXDEV_ERROR_COUNTS));
+
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_POWER_SAMPLING_VIA_DEBUGFS)
+ErrorDeInitDeviceDepBridge:
+	RGXUnregisterBridges(psDevInfo);
+#endif
+
+e17:
+	RGXHWPerfHostDeInit(psDevInfo);
+ErrorDeInitHWPerfFw:
+	RGXHWPerfDeinit(psDevInfo);
+e16:
+#if !defined(NO_HARDWARE)
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (psDevInfo->pvSecureRegsBaseKM != NULL)
+	{
+		/* Adjust pvSecureRegsBaseKM if device has SECURITY_VERSION > 1 */
+		if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, HOST_SECURITY_VERSION) &&
+		    (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) > 1))
+		{
+			/* Undo the VA offset adjustment to unmap correct VAddr */
+			psDevInfo->pvSecureRegsBaseKM = (void __iomem *)((uintptr_t)psDevInfo->pvSecureRegsBaseKM + RGX_HOST_SECURE_REGBANK_OFFSET);
+			OSUnMapPhysToLin((void __force *) psDevInfo->pvSecureRegsBaseKM,
+			                 RGX_HOST_SECURE_REGBANK_SIZE);
+		}
+	}
+#endif
+#endif /* !NO_HARDWARE */
+e15:
+#if !defined(NO_HARDWARE)
+	if (psDevInfo->pvRegsBaseKM != NULL)
+	{
+		OSUnMapPhysToLin((void __force *) psDevInfo->pvRegsBaseKM,
+		                 psDevInfo->ui32RegSize);
+	}
+e14:
+#endif /* !NO_HARDWARE */
+	OSLockDestroy(psDevInfo->hGpuUtilStatsLock);
+e13:
+	OSLockDestroy(psDevInfo->hCCBRecoveryLock);
+e12:
+	OSLockDestroy(psDevInfo->hCCBStallCheckLock);
+e11:
+	OSLockDestroy(psDevInfo->hRGXFWIfBufInitLock);
+e10:
+	OSLockDestroy(psDevInfo->hBPLock);
+e9:
+	OSLockDestroy(psDevInfo->sRegConfig.hLock);
+e8:
+	OSWRLockDestroy(psDevInfo->hCommonCtxtListLock);
+e7:
+	OSSpinLockDestroy(psDevInfo->hLockKCCBDeferredCommandsList);
+e6:
+	OSWRLockDestroy(psDevInfo->hMemoryCtxListLock);
+e5:
+	OSWRLockDestroy(psDevInfo->hKickSyncCtxListLock);
+e4:
+	OSWRLockDestroy(psDevInfo->hTDMCtxListLock);
+e3:
+	OSWRLockDestroy(psDevInfo->hTransferCtxListLock);
+e2:
+	OSWRLockDestroy(psDevInfo->hComputeCtxListLock);
+e1:
+	OSWRLockDestroy(psDevInfo->hRenderCtxListLock);
+e0:
+	OSFreeMem(psDevInfo);
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	IMG_PCHAR psz = psDevInfo->sDevFeatureCfg.pszBVNCString;
+	if (NULL == psz)
+	{
+		IMG_CHAR pszBVNCInfo[RGX_HWPERF_MAX_BVNC_LEN];
+		size_t uiBVNCStringSize;
+		size_t uiStringLength;
+
+		uiStringLength = OSSNPrintf(pszBVNCInfo, RGX_HWPERF_MAX_BVNC_LEN, "%d.%d.%d.%d",
+				psDevInfo->sDevFeatureCfg.ui32B,
+				psDevInfo->sDevFeatureCfg.ui32V,
+				psDevInfo->sDevFeatureCfg.ui32N,
+				psDevInfo->sDevFeatureCfg.ui32C);
+		PVR_ASSERT(uiStringLength < RGX_HWPERF_MAX_BVNC_LEN);
+
+		uiBVNCStringSize = (uiStringLength + 1) * sizeof(IMG_CHAR);
+		psz = OSAllocMem(uiBVNCStringSize);
+		if (NULL != psz)
+		{
+			OSCachedMemCopy(psz, pszBVNCInfo, uiBVNCStringSize);
+			psDevInfo->sDevFeatureCfg.pszBVNCString = psz;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_MESSAGE,
+					"%s: Allocating memory for BVNC Info string failed",
+					__func__));
+		}
+	}
+
+	return psz;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDevVersionString
+@Description    Gets the version string for the given device node and returns
+                a pointer to it in ppszVersionString. It is then the
+                responsibility of the caller to free this memory.
+@Input          psDeviceNode            Device node from which to obtain the
+                                        version string
+@Output	        ppszVersionString	Contains the version string upon return
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXDevVersionString(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_CHAR **ppszVersionString)
+{
+#if defined(NO_HARDWARE) || defined(EMULATOR)
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (SW)";
+#else
+	const IMG_CHAR szFormatString[] = "GPU variant BVNC: %s (HW)";
+#endif
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_PCHAR pszBVNC;
+	size_t uiStringLength;
+
+	if (psDeviceNode == NULL || ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	pszBVNC = RGXDevBVNCString(psDevInfo);
+
+	if (NULL == pszBVNC)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	uiStringLength = OSStringLength(pszBVNC);
+	uiStringLength += (sizeof(szFormatString) - 2); /* sizeof includes the null, -2 for "%s" */
+	*ppszVersionString = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (*ppszVersionString == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	OSSNPrintf(*ppszVersionString, uiStringLength, szFormatString,
+		pszBVNC);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       RGXDevClockSpeed
+@Description    Gets the clock speed for the given device node and returns
+                it in pui32RGXClockSpeed.
+@Input          psDeviceNode		Device node
+@Output         pui32RGXClockSpeed  Variable for storing the clock speed
+@Return         PVRSRV_ERROR
+*/ /***************************************************************************/
+static PVRSRV_ERROR RGXDevClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_PUINT32  pui32RGXClockSpeed)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*) psDeviceNode->psDevConfig->hDevData;
+
+	/* get clock speed */
+	*pui32RGXClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1)
+/*!
+ *******************************************************************************
+
+ @Function		RGXInitFwRawHeap
+
+ @Description	Called to perform additional initialisation
+ ******************************************************************************/
+static PVRSRV_ERROR RGXInitFwRawHeap(PVRSRV_RGXDEV_INFO *psDevInfo, DEVMEM_HEAP_BLUEPRINT *psDevMemHeap, IMG_UINT32 ui32DriverID)
+{
+	IMG_UINT32 uiStringLength;
+	IMG_UINT32 uiStringLengthMax = 32;
+
+	IMG_UINT32 ui32Log2RgxDefaultPageShift = RGXHeapDerivePageSize(OSGetPageShift());
+
+	PVR_RETURN_IF_FALSE(ui32Log2RgxDefaultPageShift != 0, PVRSRV_ERROR_INVALID_PARAMS);
+
+#if defined(FIX_HW_BRN_71317_BIT_MASK)
+	if (RGX_IS_BRN_SUPPORTED(psDevInfo, 71317))
+	{
+		if (ui32Log2RgxDefaultPageShift == RGX_HEAP_2MB_PAGE_SHIFT
+		   || ui32Log2RgxDefaultPageShift == RGX_HEAP_1MB_PAGE_SHIFT)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "OS page size too large for device virtual heaps. "
+			         "Maximum page size supported is 256KB when BRN71317 is present."));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+#endif
+
+	uiStringLength = MIN(sizeof(RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT), uiStringLengthMax + 1);
+
+	/* Start by allocating memory for this DriverID heap identification string */
+	psDevMemHeap->pszName = OSAllocMem(uiStringLength * sizeof(IMG_CHAR));
+	if (psDevMemHeap->pszName == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/* Append the DriverID number to the RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT string */
+	OSSNPrintf((IMG_CHAR *)psDevMemHeap->pszName, uiStringLength, RGX_FIRMWARE_GUEST_RAW_HEAP_IDENT, ui32DriverID);
+
+	/* Use the common blueprint template support function to initialise the heap */
+	HeapCfgBlueprintInit(psDevMemHeap->pszName,
+		                 RGX_FIRMWARE_RAW_HEAP_BASE + (ui32DriverID * RGX_FIRMWARE_RAW_HEAP_SIZE),
+		                 RGX_FIRMWARE_RAW_HEAP_SIZE,
+		                 0,
+		                 ui32Log2RgxDefaultPageShift,
+		                 0,
+		                 NULL,
+		                 NULL,
+		                 psDevMemHeap);
+
+	return PVRSRV_OK;
+}
+
+/*!
+ *******************************************************************************
+
+ @Function		RGXDeInitFwRawHeap
+
+ @Description	Called to perform additional deinitialisation
+ ******************************************************************************/
+static void RGXDeInitFwRawHeap(DEVMEM_HEAP_BLUEPRINT *psDevMemHeap)
+{
+	IMG_UINT64 uiBase = RGX_FIRMWARE_RAW_HEAP_BASE + RGX_FIRMWARE_RAW_HEAP_SIZE;
+	IMG_UINT64 uiSpan = uiBase + ((RGX_NUM_DRIVERS_SUPPORTED - 1) * RGX_FIRMWARE_RAW_HEAP_SIZE);
+
+	/* Safe to do as the guest firmware heaps are last in the list */
+	if (psDevMemHeap->sHeapBaseAddr.uiAddr >= uiBase &&
+	    psDevMemHeap->sHeapBaseAddr.uiAddr < uiSpan)
+	{
+		void *pszName = (void*)psDevMemHeap->pszName;
+		OSFreeMem(pszName);
+	}
+}
+#endif /* defined(RGX_PREMAP_FW_HEAPS) || (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+/******************************************************************************
+ End of file (rgxinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxinit.h b/drivers/gpu/drm/img-rogue/rgxinit.h
new file mode 100644
index 000000000000..807f6a0b6fce
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxinit.h
@@ -0,0 +1,313 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX initialisation header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXINIT_H)
+#define RGXINIT_H
+
+#include "connection_server.h"
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_bridge.h"
+#include "fwload.h"
+#include "rgxinit_apphints.h"
+
+#if defined(__linux__)
+#define OS_FW_VERIFY_FUNCTION OSVerifyFirmware
+#else
+#define OS_FW_VERIFY_FUNCTION NULL
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXInitDevPart2
+
+ @Description
+
+ Second part of server-side RGX initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInitDevPart2 (PVRSRV_DEVICE_NODE	*psDeviceNode,
+							  RGX_INIT_APPHINTS		*psApphints);
+
+PVRSRV_ERROR RGXInitAllocFWImgMem(PVRSRV_DEVICE_NODE   *psDeviceNode,
+                                  IMG_DEVMEM_SIZE_T    ui32FWCodeLen,
+                                  IMG_DEVMEM_SIZE_T    ui32FWDataLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememCodeLen,
+                                  IMG_DEVMEM_SIZE_T    uiFWCorememDataLen);
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXInitFirmware
+
+ @Description
+
+ Server-side RGX firmware initialisation
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+RGXInitFirmware(PVRSRV_DEVICE_NODE       *psDeviceNode,
+                RGX_INIT_APPHINTS        *psApphints,
+                IMG_UINT32               ui32ConfigFlags,
+                IMG_UINT32               ui32ConfigFlagsExt,
+                IMG_UINT32               ui32FwOsCfgFlags);
+
+
+/*!
+*******************************************************************************
+
+ @Function	RGXLoadAndGetFWData
+
+ @Description
+
+ Load FW and return pointer to FW data.
+
+ @Input psDeviceNode - device node
+
+ @Input ppsRGXFW - fw pointer
+
+ @Return PVRSRV_ERROR - PVRSRV_OK on success
+                        PVRSRV_ERROR_NOT_READY if filesystem is not ready
+                        PVRSRV_ERROR_NOT_FOUND if no suitable FW image found
+                        PVRSRV_ERROR_OUT_OF_MEMORY if unable to alloc memory for FW image
+                        PVRSRV_ERROR_NOT_AUTHENTICATED if FW image failed verification
+
+******************************************************************************/
+PVRSRV_ERROR RGXLoadAndGetFWData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 OS_FW_IMAGE **ppsRGXFW);
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function	RGXInitHWPerfCounters
+
+ @Description
+
+ Initialisation of the performance counters
+
+ @Input psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInitHWPerfCounters(PVRSRV_DEVICE_NODE	*psDeviceNode);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function	RGXRegisterDevice
+
+ @Description
+
+ Registers the device with the system
+
+ @Input:   psDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+*******************************************************************************
+
+ @Function	RGXDevBVNCString
+
+ @Description
+
+ Returns the Device BVNC string. It will allocate and fill it first, if necessary.
+
+ @Input:   psDevInfo - device info (must not be null)
+
+ @Return   IMG_PCHAR - pointer to BVNC string
+
+******************************************************************************/
+IMG_PCHAR RGXDevBVNCString(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function	DevDeInitRGX
+
+ @Description
+
+ Reset and deinitialise Chip
+
+ @Input psDeviceNode - device info. structure
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR DevDeInitRGX(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+
+#if !defined(NO_HARDWARE)
+
+void RGX_WaitForInterruptsTimeout(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsRegister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Initialise data used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument). This function must be called only once for each
+               different user/handle.
+
+ @Input        phGpuUtilUser - Pointer to handle used to identify a user of
+                               RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser);
+
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsUnregister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Free data previously used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument).
+
+ @Input        hGpuUtilUser - Handle used to identify a user of
+                              RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser);
+#endif /* !defined(NO_HARDWARE) */
+
+#if defined(RGX_FEATURE_AXI_ACE_BIT_MASK)
+/*!
+************************************************************************************
+ @Function		RGXSystemGetFabricCoherency
+
+ @Description	Get the system fabric coherency for the device by reading default
+				configuration from device register, subject to AppHint overrides.
+
+ @Input			sRegsCpuPBase		: Device register CPU physical address base
+				ui32RegsSize		: Device register size
+				peDevFabricType		: Device memory bus fabric type
+				peCacheSnoopingMode : Fabric coherency override
+
+ @Return		PVRSRV_ERROR
+************************************************************************************/
+PVRSRV_ERROR RGXSystemGetFabricCoherency(PVRSRV_DEVICE_CONFIG *psDeviceConfig,
+										 IMG_CPU_PHYADDR sRegsCpuPBase,
+										 IMG_UINT32 ui32RegsSize,
+										 PVRSRV_DEVICE_FABRIC_TYPE *peDevFabricType,
+										 PVRSRV_DEVICE_SNOOP_MODE *peCacheSnoopingMode);
+#endif
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXInitCreateFWKernelMemoryContext
+
+ @Description   Called to perform initialisation during firmware kernel context
+                creation.
+
+ @Input         psDeviceNode  device node
+ ******************************************************************************/
+PVRSRV_ERROR RGXInitCreateFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXDeInitDestroyFWKernelMemoryContext
+
+ @Description   Called to perform deinitialisation during firmware kernel
+                context destruction.
+
+ @Input         psDeviceNode  device node
+ ******************************************************************************/
+void RGXDeInitDestroyFWKernelMemoryContext(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXHeapDerivePageSize
+
+ @Description   Ensure the desire page size is suitable for the RGX hardware
+
+ @Input         uiLog2PageSize target page log2 size
+
+ @Return        IMG_UINT32 valid page log2 size
+ ******************************************************************************/
+IMG_UINT32 RGXHeapDerivePageSize(IMG_UINT32 uiLog2PageSize);
+
+/*!
+ *******************************************************************************
+
+ @Function      RGXGetNon4KHeapPageShift
+
+ @Description   Retrieves the log2 page size of the General non 4k heap
+
+ @Output        pui32Log2Non4KPgShift page shift
+ ******************************************************************************/
+PVRSRV_ERROR RGXGetNon4KHeapPageShift(const void *hPrivate,
+                                     IMG_UINT32 *pui32Log2Non4KPgShift);
+
+
+#endif /* RGXINIT_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxinit_apphints.h b/drivers/gpu/drm/img-rogue/rgxinit_apphints.h
new file mode 100644
index 000000000000..99cecc630b5c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxinit_apphints.h
@@ -0,0 +1,120 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services device initialisation settings
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device initialisation settings
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXINIT_APPHINTS_H)
+#define RGXINIT_APPHINTS_H
+
+#include "img_defs.h"
+
+#include "rgx_fwif_km.h"
+#include "rgxdefs_km.h"
+#include "rgxdevice.h"
+
+/*
+ * Container for all the apphints used by this module
+ */
+typedef struct _RGX_INIT_APPHINTS_
+{
+	IMG_BOOL   bEnableSignatureChecks;
+	IMG_UINT32 ui32SignatureChecksBufSize;
+
+	IMG_BOOL   bAssertOnOutOfMem;
+	IMG_BOOL   bAssertOnHWRTrigger;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32TFBCVersion;
+	IMG_UINT32 ui32TFBCCompressionControlGroup;
+	IMG_UINT32 ui32TFBCCompressionControlScheme;
+	IMG_BOOL   bTFBCCompressionControlYUVFormat;
+	IMG_BOOL   bTFBCCompressionControlLossyMinChannel;
+#endif
+	IMG_BOOL   bCheckMlist;
+	IMG_BOOL   bDisableClockGating;
+	IMG_BOOL   bDisableDMOverlap;
+	IMG_BOOL   bDisablePDP;
+	IMG_BOOL   bEnableDMKillRand;
+	IMG_BOOL   bEnableRandomCsw;
+	IMG_BOOL   bEnableSoftResetCsw;
+	IMG_BOOL   bHWPerfDisableCounterFilter;
+	IMG_UINT32 ui32DeviceFlags;
+	IMG_UINT32 ui32FilterFlags;
+	IMG_UINT32 ui32EnableFWContextSwitch;
+	IMG_UINT32 ui32FWContextSwitchProfile;
+	IMG_UINT32 ui32HWPerfFWBufSize;
+	IMG_UINT32 ui32HWPerfHostBufSize;
+	IMG_UINT32 ui32HWPerfFilter0;
+	IMG_UINT32 ui32HWPerfFilter1;
+	IMG_UINT32 ui32HWPerfHostFilter;
+	IMG_UINT32 ui32TimeCorrClock;
+	IMG_UINT32 ui32HWRDebugDumpLimit;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32KCCBSizeLog2;
+#if defined(PVR_ARCH_VOLCANIC)
+	IMG_UINT32 ui32ISPSchedulingLatencyMode;
+#endif
+	FW_PERF_CONF eFirmwarePerf;
+	RGX_ACTIVEPM_CONF eRGXActivePMConf;
+	RGX_RD_POWER_ISLAND_CONF eRGXRDPowerIslandConf;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	IMG_UINT32 ui32AvailablePowUnitsMask;
+	IMG_UINT32 ui32AvailableRACMask;
+	IMG_BOOL bSPUClockGating;
+#endif
+	IMG_BOOL   bEnableTrustedDeviceAceConfig;
+	IMG_UINT32 ui32FWContextSwitchCrossDM;
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	IMG_UINT32 ui32PhysMemTestPasses;
+#endif
+	RGX_FWT_LOGTYPE eDebugDumpFWTLogType;
+#if defined(SUPPORT_ICS)
+	IMG_UINT32 ui32EnableIdleCycleStealing;
+	IMG_UINT32 ui32FDTI;
+	IMG_UINT32 ui32ICSThreshold;
+	IMG_BOOL   bTestModeOn;
+#endif
+} RGX_INIT_APPHINTS;
+
+#endif /* RGXINIT_APPHINTS_H */
+
+/******************************************************************************
+ End of file (rgxinit_apphints.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxkicksync.c b/drivers/gpu/drm/img-rogue/rgxkicksync.c
new file mode 100644
index 000000000000..6ab8942d84a8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxkicksync.c
@@ -0,0 +1,797 @@
+/*************************************************************************/ /*!
+@File           rgxkicksync.c
+@Title          Server side of the sync only kick API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxkicksync.h"
+
+#include "rgxdevice.h"
+#include "rgxmem.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "allocmem.h"
+#include "sync.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_KICKSYNC_UFO_DUMP	0
+
+//#define KICKSYNC_CHECKPOINT_DEBUG 1
+
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+struct _RGX_SERVER_KICKSYNC_CONTEXT_
+{
+	PVRSRV_DEVICE_NODE        * psDeviceNode;
+	RGX_SERVER_COMMON_CONTEXT * psServerCommonContext;
+	DLLIST_NODE                 sListNode;
+	SYNC_ADDR_LIST              sSyncAddrListFence;
+	SYNC_ADDR_LIST              sSyncAddrListUpdate;
+	POS_LOCK                    hLock;
+};
+
+
+PVRSRV_ERROR PVRSRVRGXCreateKickSyncContextKM(CONNECTION_DATA              *psConnection,
+                                              PVRSRV_DEVICE_NODE           *psDeviceNode,
+                                              IMG_HANDLE                    hMemCtxPrivData,
+                                              IMG_UINT32                    ui32PackedCCBSizeU88,
+                                              IMG_UINT32                    ui32ContextFlags,
+                                              RGX_SERVER_KICKSYNC_CONTEXT **ppsKickSyncContext)
+{
+	PVRSRV_RGXDEV_INFO          * psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC              * psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext;
+	RGX_COMMON_CONTEXT_INFO      sInfo;
+	PVRSRV_ERROR                 eError;
+	IMG_UINT32                   ui32CCBAllocSizeLog2, ui32CCBMaxAllocSizeLog2;
+
+	memset(&sInfo, 0, sizeof(sInfo));
+
+	/* Prepare cleanup struct */
+	* ppsKickSyncContext = NULL;
+	psKickSyncContext = OSAllocZMem(sizeof(*psKickSyncContext));
+	if (psKickSyncContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psKickSyncContext->hLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+									__func__,
+									PVRSRVGetErrorString(eError)));
+		goto err_lockcreate;
+	}
+
+	psKickSyncContext->psDeviceNode = psDeviceNode;
+
+	ui32CCBAllocSizeLog2 = U32toU8_Unpack1(ui32PackedCCBSizeU88);
+	ui32CCBMaxAllocSizeLog2 = U32toU8_Unpack2(ui32PackedCCBSizeU88);
+	eError = FWCommonContextAllocate(psConnection,
+									 psDeviceNode,
+									 REQ_TYPE_KICKSYNC,
+									 RGXFWIF_DM_GP,
+									 hMemCtxPrivData,
+									 NULL,
+									 0,
+									 psFWMemContextMemDesc,
+									 NULL,
+									 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_KICKSYNC_CCB_SIZE_LOG2,
+									 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_KICKSYNC_CCB_MAX_SIZE_LOG2,
+									 ui32ContextFlags,
+									 0, /* priority */
+									 0, /* max deadline MS */
+									 0, /* robustness address */
+									 & sInfo,
+									 & psKickSyncContext->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	OSWRLockAcquireWrite(psDevInfo->hKickSyncCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sKickSyncCtxtListHead), &(psKickSyncContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hKickSyncCtxListLock);
+
+	SyncAddrListInit(&psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListInit(&psKickSyncContext->sSyncAddrListUpdate);
+
+	* ppsKickSyncContext = psKickSyncContext;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+	OSLockDestroy(psKickSyncContext->hLock);
+err_lockcreate:
+	OSFreeMem(psKickSyncContext);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXDestroyKickSyncContextKM(RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = psKickSyncContext->psDeviceNode->pvDevice;
+	PVRSRV_ERROR         eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psKickSyncContext->psDeviceNode,
+	                                          psKickSyncContext->psServerCommonContext,
+	                                          RGXFWIF_DM_GP,
+	                                          PDUMP_FLAGS_NONE);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psKickSyncContext->psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+
+	OSWRLockAcquireWrite(psDevInfo->hKickSyncCtxListLock);
+	dllist_remove_node(&(psKickSyncContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hKickSyncCtxListLock);
+
+	FWCommonContextFree(psKickSyncContext->psServerCommonContext);
+	psKickSyncContext->psServerCommonContext = NULL;
+
+	SyncAddrListDeinit(&psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psKickSyncContext->sSyncAddrListUpdate);
+
+	OSLockDestroy(psKickSyncContext->hLock);
+
+	OSFreeMem(psKickSyncContext);
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetKickSyncContextPropertyKM(RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContext,
+                                                   RGX_CONTEXT_PROPERTY eContextProperty,
+                                                   IMG_UINT64 ui64Input,
+                                                   IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psKickSyncContext->hLock);
+			eError = FWCommonContextSetFlags(psKickSyncContext->psServerCommonContext,
+			                                 ui32ContextFlags);
+
+			OSLockRelease(psKickSyncContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpKickSyncCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hKickSyncCtxListLock);
+	dllist_foreach_node(&psDevInfo->sKickSyncCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_KICKSYNC_CONTEXT *psCurrentServerKickSyncCtx =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_KICKSYNC_CONTEXT, sListNode);
+
+		if (NULL != psCurrentServerKickSyncCtx->psServerCommonContext)
+		{
+			DumpFWCommonContextInfo(psCurrentServerKickSyncCtx->psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+	OSWRLockReleaseRead(psDevInfo->hKickSyncCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientKickSyncCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hKickSyncCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sKickSyncCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_KICKSYNC_CONTEXT *psCurrentServerKickSyncCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_KICKSYNC_CONTEXT, sListNode);
+
+		if (NULL != psCurrentServerKickSyncCtx->psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerKickSyncCtx->psServerCommonContext, RGX_KICK_TYPE_DM_GP) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_GP;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hKickSyncCtxListLock);
+	return ui32ContextBitMask;
+}
+
+PVRSRV_ERROR PVRSRVRGXKickSyncKM(RGX_SERVER_KICKSYNC_CONTEXT * psKickSyncContext,
+                                 IMG_UINT32                    ui32ClientUpdateCount,
+                                 SYNC_PRIMITIVE_BLOCK       ** pauiClientUpdateUFODevVarBlock,
+                                 IMG_UINT32                  * paui32ClientUpdateOffset,
+                                 IMG_UINT32                  * paui32ClientUpdateValue,
+                                 PVRSRV_FENCE                  iCheckFence,
+                                 PVRSRV_TIMELINE               iUpdateTimeline,
+                                 PVRSRV_FENCE                * piUpdateFence,
+                                 IMG_CHAR                      szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+                                 IMG_UINT32                    ui32ExtJobRef)
+{
+	RGXFWIF_KCCB_CMD         sKickSyncKCCBCmd;
+	RGX_CCB_CMD_HELPER_DATA  asCmdHelperData[1];
+	PVRSRV_ERROR             eError;
+	PVRSRV_ERROR             eError2;
+	IMG_BOOL                 bCCBStateOpen = IMG_FALSE;
+	PRGXFWIF_UFO_ADDR        *pauiClientFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR        *pauiClientUpdateUFOAddress = NULL;
+	IMG_UINT32               ui32ClientFenceCount = 0;
+	IMG_UINT32               *paui32ClientFenceValue = NULL;
+	PVRSRV_FENCE             iUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT32               ui32FWCtx = FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext).ui32Addr;
+	PVRSRV_RGXDEV_INFO       *psDevInfo = FWCommonContextGetRGXDevInfo(psKickSyncContext->psServerCommonContext);
+	RGX_CLIENT_CCB           *psClientCCB = FWCommonContextGetClientCCB(psKickSyncContext->psServerCommonContext);
+	IMG_UINT32               ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have dev var updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	OSLockAcquire(psKickSyncContext->hLock);
+	eError = SyncAddrListPopulate(&psKickSyncContext->sSyncAddrListUpdate,
+							ui32ClientUpdateCount,
+							pauiClientUpdateUFODevVarBlock,
+							paui32ClientUpdateOffset);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_syncaddrlist;
+	}
+
+	if (ui32ClientUpdateCount > 0)
+	{
+		pauiClientUpdateUFOAddress = psKickSyncContext->sSyncAddrListUpdate.pasFWAddrs;
+	}
+	/* Ensure the string is null-terminated (Required for safety) */
+	szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	/* This will never be true if called from the bridge since piUpdateFence will always be valid */
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto out_unlock;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), "
+			   "psKickSyncContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+			   __func__, iCheckFence,
+			   (void*)psKickSyncContext->psDeviceNode->hSyncCheckpointContext));
+	/* Resolve the sync checkpoints that make up the input fence */
+	eError = SyncCheckpointResolveFence(psKickSyncContext->psDeviceNode->hSyncCheckpointContext,
+	                                    iCheckFence,
+	                                    &ui32FenceSyncCheckpointCount,
+	                                    &apsFenceSyncCheckpoints,
+	                                    &uiCheckFenceUID,
+	                                    PDUMP_FLAGS_NONE);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_resolve_fence;
+	}
+
+	/* Create the output fence (if required) */
+	if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: calling SyncCheckpointCreateFence (iUpdateTimeline=%d)...",
+				   __func__, iUpdateTimeline));
+		eError = SyncCheckpointCreateFence(psKickSyncContext->psDeviceNode,
+		                                   szUpdateFenceName,
+		                                   iUpdateTimeline,
+		                                   psKickSyncContext->psDeviceNode->hSyncCheckpointContext,
+		                                   &iUpdateFence,
+		                                   &uiUpdateFenceUID,
+		                                   &pvUpdateFenceFinaliseData,
+		                                   &psUpdateSyncCheckpoint,
+		                                   (void*)&psFenceTimelineUpdateSync,
+		                                   &ui32FenceTimelineUpdateValue,
+		                                   PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%d)",
+					   __func__, eError));
+			goto fail_create_output_fence;
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: ...returned from SyncCheckpointCreateFence "
+				   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+				   "ui32FenceTimelineUpdateValue=%u)",
+				   __func__, iUpdateFence, psFenceTimelineUpdateSync,
+				   ui32FenceTimelineUpdateValue));
+
+		/* Append the sync prim update for the timeline (if required) */
+		if (psFenceTimelineUpdateSync)
+		{
+			IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+			/* Allocate memory to hold the list of update values (including our timeline update) */
+			pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*paui32ClientUpdateValue) * (ui32ClientUpdateCount+1));
+			if (!pui32IntAllocatedUpdateValues)
+			{
+				/* Failed to allocate memory */
+				eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto fail_alloc_update_values_mem;
+			}
+			OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32ClientUpdateCount+1));
+			/* Copy the update values into the new memory, then append our timeline update value */
+			OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32ClientUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32ClientUpdateCount);
+			/* Now set the additional update value */
+			pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32ClientUpdateCount;
+			*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+			ui32ClientUpdateCount++;
+			/* Now make sure paui32ClientUpdateValue points to pui32IntAllocatedUpdateValues */
+			paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32ClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x",
+							   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			/* Now append the timeline sync prim addr to the kicksync context update list */
+			SyncAddrListAppendSyncPrim(&psKickSyncContext->sSyncAddrListUpdate,
+			                           psFenceTimelineUpdateSync);
+		}
+	}
+
+	/* Reset number of fence syncs in kicksync context fence list to 0 */
+	SyncAddrListPopulate(&psKickSyncContext->sSyncAddrListFence,
+	                     0, NULL, NULL);
+
+	if (ui32FenceSyncCheckpointCount > 0)
+	{
+		/* Append the checks (from input fence) */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Append %d sync checkpoints to KickSync Fence "
+				   "(&psKickSyncContext->sSyncAddrListFence=<%p>)...",
+				   __func__, ui32FenceSyncCheckpointCount,
+				   (void*)&psKickSyncContext->sSyncAddrListFence));
+		SyncAddrListAppendCheckpoints(&psKickSyncContext->sSyncAddrListFence,
+									  ui32FenceSyncCheckpointCount,
+									  apsFenceSyncCheckpoints);
+
+		pauiClientFenceUFOAddress = psKickSyncContext->sSyncAddrListFence.pasFWAddrs;
+
+		ui32ClientFenceCount += ui32FenceSyncCheckpointCount;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiClientFenceUFOAddress;
+
+				for (iii=0; iii<ui32ClientFenceCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientFenceUFOAddress[%d](<%p>) = 0x%x",
+							   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+	}
+
+	if (psUpdateSyncCheckpoint)
+	{
+		PVRSRV_ERROR eErr;
+
+		/* Append the update (from output fence) */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Append 1 sync checkpoint to KickSync Update "
+				   "(&psKickSyncContext->sSyncAddrListUpdate=<%p>)...",
+				   __func__, (void*)&psKickSyncContext->sSyncAddrListUpdate));
+		eErr = SyncAddrListAppendCheckpoints(&psKickSyncContext->sSyncAddrListUpdate,
+											 1,
+											 &psUpdateSyncCheckpoint);
+		if (eErr != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:  ...done. SyncAddrListAppendCheckpoints() returned error (%d)",
+					   __func__, eErr));
+		}
+		else
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:  ...done.", __func__));
+		}
+		if (!pauiClientUpdateUFOAddress)
+		{
+			pauiClientUpdateUFOAddress = psKickSyncContext->sSyncAddrListUpdate.pasFWAddrs;
+		}
+		ui32ClientUpdateCount++;
+#if defined(KICKSYNC_CHECKPOINT_DEBUG)
+		{
+			IMG_UINT32 iii;
+			IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiClientUpdateUFOAddress;
+
+			for (iii=0; iii<ui32ClientUpdateCount; iii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: pauiClientUpdateUFOAddress[%d](<%p>) = 0x%x",
+						   __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+				pui32Tmp++;
+			}
+		}
+#endif
+	}
+
+#if (ENABLE_KICKSYNC_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping KICKSYNC fence/updates syncs...",
+				 __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiClientFenceUFOAddress;
+			IMG_UINT32 *pui32TmpIntFenceValue = paui32ClientFenceValue;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiClientUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32ClientUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Prepared %d KickSync fence syncs "
+					 "(&psKickSyncContext->sSyncAddrListFence=<%p>, "
+					 "pauiClientFenceUFOAddress=<%p>):",
+					 __func__, ui32ClientFenceCount,
+					 (void*)&psKickSyncContext->sSyncAddrListFence,
+					 (void*)pauiClientFenceUFOAddress));
+			for (ii=0; ii<ui32ClientFenceCount; ii++)
+			{
+				if (psTmpIntFenceUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, "
+							 "CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+							 __func__, ii + 1, ui32ClientFenceCount,
+							 (void*)psTmpIntFenceUFOAddress,
+							 psTmpIntFenceUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+							 __func__, ii + 1, ui32ClientFenceCount,
+							 (void*)psTmpIntFenceUFOAddress,
+							 psTmpIntFenceUFOAddress->ui32Addr,
+							 *pui32TmpIntFenceValue,
+							 *pui32TmpIntFenceValue));
+					pui32TmpIntFenceValue++;
+				}
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Prepared %d KickSync update syncs "
+					 "(&psKickSyncContext->sSyncAddrListUpdate=<%p>, "
+					 "pauiClientUpdateUFOAddress=<%p>):",
+					 __func__, ui32ClientUpdateCount,
+					 (void*)&psKickSyncContext->sSyncAddrListUpdate,
+					 (void*)pauiClientUpdateUFOAddress));
+			for (ii=0; ii<ui32ClientUpdateCount; ii++)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:  Line %d, psTmpIntUpdateUFOAddress=<%p>",
+						   __func__, __LINE__,
+						   (void*)psTmpIntUpdateUFOAddress));
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:  Line %d, pui32TmpIntUpdateValue=<%p>",
+						   __func__, __LINE__,
+						   (void*)pui32TmpIntUpdateValue));
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, "
+							 "UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+							 __func__, ii + 1, ui32ClientUpdateCount,
+							 (void*)psTmpIntUpdateUFOAddress,
+							 psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							 "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d",
+							 __func__, ii + 1, ui32ClientUpdateCount,
+							 (void*)psTmpIntUpdateUFOAddress,
+							 psTmpIntUpdateUFOAddress->ui32Addr,
+							 *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+	RGXCmdHelperInitCmdCCB(psDevInfo,
+	                       psClientCCB,
+	                       0, /* empty ui64FBSCEntryMask */
+	                       ui32ClientFenceCount,
+	                       pauiClientFenceUFOAddress,
+	                       paui32ClientFenceValue,
+	                       ui32ClientUpdateCount,
+	                       pauiClientUpdateUFOAddress,
+	                       paui32ClientUpdateValue,
+	                       0,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       NULL,
+	                       RGXFWIF_CCB_CMD_TYPE_NULL,
+	                       ui32ExtJobRef,
+	                       ui32IntJobRef,
+	                       PDUMP_FLAGS_NONE,
+	                       NULL,
+	                       "KickSync",
+	                       bCCBStateOpen,
+	                       asCmdHelperData);
+
+	eError = RGXCmdHelperAcquireCmdCCB(ARRAY_SIZE(asCmdHelperData), asCmdHelperData);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_cmdacquire;
+	}
+
+	/*
+	 *  We should reserve space in the kernel CCB here and fill in the command
+	 *  directly.
+	 *  This is so if there isn't space in the kernel CCB we can return with
+	 *  retry back to services client before we take any operations
+	 */
+
+	/*
+	 * All the required resources are ready at this point, we can't fail so
+	 * take the required server sync operations and commit all the resources
+	 */
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	RGXCmdHelperReleaseCmdCCB(1,
+							  asCmdHelperData,
+							  "KickSync",
+							  FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext).ui32Addr);
+
+	/* Construct the kernel kicksync CCB command. */
+	sKickSyncKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psKickSyncContext->psServerCommonContext);
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	sKickSyncKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+	/*
+	 * Submit the kicksync command to the firmware.
+	 */
+	RGXSRV_HWPERF_ENQ(psKickSyncContext,
+	                  OSGetCurrentClientProcessIDKM(),
+	                  ui32FWCtx,
+	                  ui32ExtJobRef,
+	                  ui32IntJobRef,
+	                  RGX_HWPERF_KICK_TYPE2_SYNC,
+	                  iCheckFence,
+	                  iUpdateFence,
+	                  iUpdateTimeline,
+	                  uiCheckFenceUID,
+	                  uiUpdateFenceUID,
+	                  NO_DEADLINE,
+	                  NO_CYCEST);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError2 = RGXScheduleCommandWithoutPowerLock(psKickSyncContext->psDeviceNode->pvDevice,
+		                             RGXFWIF_DM_GP,
+		                             & sKickSyncKCCBCmd,
+		                             PDUMP_FLAGS_NONE);
+		if (eError2 != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	PVRGpuTraceEnqueueEvent(psKickSyncContext->psDeviceNode,
+	                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+	                        RGX_HWPERF_KICK_TYPE2_SYNC);
+
+	if (eError2 != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVRGXKickSync failed to schedule kernel CCB command. (0x%x)",
+		         eError));
+		eError = eError2;
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier call
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdacquire;
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim<%p> to %d",
+				   __func__, (void*)psFenceTimelineUpdateSync,
+				   ui32FenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+#endif
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	*piUpdateFence = iUpdateFence;
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psKickSyncContext->psDeviceNode, iUpdateFence,
+									pvUpdateFenceFinaliseData,
+									psUpdateSyncCheckpoint, szUpdateFenceName);
+	}
+
+	OSLockRelease(psKickSyncContext->hLock);
+	return PVRSRV_OK;
+
+fail_acquirepowerlock:
+fail_cmdacquire:
+	SyncAddrListRollbackCheckpoints(psKickSyncContext->psDeviceNode, &psKickSyncContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psKickSyncContext->psDeviceNode, &psKickSyncContext->sSyncAddrListUpdate);
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+
+	/* Free memory allocated to hold update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+	}
+fail_alloc_update_values_mem:
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free memory allocated to hold the resolved fence's checkpoints */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+fail_resolve_fence:
+fail_syncaddrlist:
+out_unlock:
+	OSLockRelease(psKickSyncContext->hLock);
+	return eError;
+}
+
+/**************************************************************************//**
+ End of file (rgxkicksync.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxkicksync.h b/drivers/gpu/drm/img-rogue/rgxkicksync.h
new file mode 100644
index 000000000000..57b49a03da5c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxkicksync.h
@@ -0,0 +1,128 @@
+/*************************************************************************/ /*!
+@File           rgxkicksync.h
+@Title          Server side of the sync only kick API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXKICKSYNC_H)
+#define RGXKICKSYNC_H
+
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+#include "sync_server.h"
+#include "rgxdevice.h"
+
+
+typedef struct _RGX_SERVER_KICKSYNC_CONTEXT_ RGX_SERVER_KICKSYNC_CONTEXT;
+
+/**************************************************************************/ /*!
+@Function       DumpKickSyncCtxtsInfo
+@Description    Function that dumps debug info of kick sync ctxs on this device
+@Return         none
+*/ /**************************************************************************/
+void
+DumpKickSyncCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                      DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                      void *pvDumpDebugFile,
+                      IMG_UINT32 ui32VerbLevel);
+
+/**************************************************************************/ /*!
+@Function       CheckForStalledClientKickSyncCtxt
+@Description    Function that checks if a kick sync client is stalled
+@Return         RGX_KICK_TYPE_DM_GP on stalled context. Otherwise, 0
+*/ /**************************************************************************/
+IMG_UINT32 CheckForStalledClientKickSyncCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXCreateKickSyncContextKM
+@Description    Server-side implementation of RGXCreateKicksyncContext
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXCreateKickSyncContextKM(CONNECTION_DATA              * psConnection,
+                                 PVRSRV_DEVICE_NODE           * psDeviceNode,
+                                 IMG_HANDLE                     hMemCtxPrivData,
+                                 IMG_UINT32                     ui32PackedCCBSizeU88,
+                                 IMG_UINT32                     ui32ContextFlags,
+                                 RGX_SERVER_KICKSYNC_CONTEXT ** ppsKicksyncContext);
+
+
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXDestroyKickSyncContextKM
+@Description    Server-side implementation of RGXDestroyKicksyncContext
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXDestroyKickSyncContextKM(RGX_SERVER_KICKSYNC_CONTEXT * psKicksyncContext);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXSetKickSyncContextPropertyKM
+@Description    Server-side implementation of RGXSetKickSyncContextProperty
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+ */ /**************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetKickSyncContextPropertyKM(RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContext,
+                                                   RGX_CONTEXT_PROPERTY eContextProperty,
+                                                   IMG_UINT64 ui64Input,
+                                                   IMG_UINT64 *pui64Output);
+
+/**************************************************************************/ /*!
+@Function       PVRSRVRGXKickSyncKM
+@Description    Kicks a sync only command
+@Return         PVRSRV_OK on success. Otherwise, a PVRSRV_ error code
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXKickSyncKM(RGX_SERVER_KICKSYNC_CONTEXT * psKicksyncContext,
+                    IMG_UINT32                    ui32ClientUpdateCount,
+                    SYNC_PRIMITIVE_BLOCK       ** pauiClientUpdateUFODevVarBlock,
+                    IMG_UINT32                  * paui32ClientUpdateDevVarOffset,
+                    IMG_UINT32                  * paui32ClientUpdateValue,
+                    PVRSRV_FENCE                  iCheckFence,
+                    PVRSRV_TIMELINE               iUpdateTimeline,
+                    PVRSRV_FENCE                * piUpdateFence,
+                    IMG_CHAR                      szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+
+                    IMG_UINT32                    ui32ExtJobRef);
+
+#endif /* RGXKICKSYNC_H */
+
+/**************************************************************************//**
+ End of file (rgxkicksync.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxlayer.h b/drivers/gpu/drm/img-rogue/rgxlayer.h
new file mode 100644
index 000000000000..3141db5c4488
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxlayer.h
@@ -0,0 +1,920 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Declaration of an interface layer used to abstract code that
+                can be compiled outside of the DDK, potentially in a
+                completely different OS.
+                All the headers included by this file must also be copied to
+                the alternative source tree.
+                All the functions declared here must have a DDK implementation
+                inside the DDK source tree (e.g. rgxlayer_impl.h/.c) and
+                another different implementation in case they are used outside
+                of the DDK.
+                All of the functions accept as a first parameter a
+                "const void *hPrivate" argument. It should be used to pass
+                around any implementation specific data required.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXLAYER_H)
+#define RGXLAYER_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "img_elf.h"
+#include "pvrsrv_error.h" /* includes pvrsrv_errors.h */
+#include "rgx_firmware_boot.h"
+#include "rgx_bvnc_defs_km.h"
+#include "rgx_fw_info.h"
+#include "rgx_common.h"
+#include "rgx_meta.h"
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+#include "rgx_mips.h"
+#endif
+#include "rgx_riscv.h"
+
+#include "rgxdefs_km.h"
+/* includes:
+ * rgx_cr_defs_km.h,
+ * RGX_BVNC_CORE_KM_HEADER (rgxcore_km_B.V.N.C.h),
+ * RGX_BNC_CONFIG_KM_HEADER (rgxconfig_km_B.V.N.C.h)
+ */
+
+/*
+ * Specific fields for RGX_CR_IDLE must not be polled in pdumps
+ * (technical reasons)
+ */
+#define CR_IDLE_UNSELECTED_MASK ((~RGX_CR_SLC_IDLE_ACE_CONVERTERS_CLRMSK) |	\
+								 (~RGX_CR_SLC_IDLE_OWDB_CLRMSK) |			\
+								 (RGX_CR_SLC_IDLE_FBCDC_ARB_EN))
+
+
+/*!
+*******************************************************************************
+
+ @Function       RGXMemCopy
+
+ @Description    MemCopy implementation
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pvDst      : Pointer to the destination
+ @Input          pvSrc      : Pointer to the source location
+ @Input          uiSize     : The amount of memory to copy in bytes
+
+ @Return         void
+
+******************************************************************************/
+void RGXMemCopy(const void *hPrivate,
+                void *pvDst,
+                void *pvSrc,
+                size_t uiSize);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXMemSet
+
+ @Description    MemSet implementation
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pvDst      : Pointer to the start of the memory region
+ @Input          ui8Value   : The value to be written
+ @Input          uiSize     : The number of bytes to be set to ui8Value
+
+ @Return         void
+
+******************************************************************************/
+void RGXMemSet(const void *hPrivate,
+               void *pvDst,
+               IMG_UINT8 ui8Value,
+               size_t uiSize);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXCommentLog
+
+ @Description    Generic log function used for debugging or other purposes
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pszString  : Message to be printed
+ @Input          ...        : Variadic arguments
+
+ @Return         void
+
+******************************************************************************/
+__printf(2, 3)
+void RGXCommentLog(const void *hPrivate,
+                   const IMG_CHAR *pszString,
+                   ...);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXErrorLog
+
+ @Description    Generic error log function used for debugging or other purposes
+
+ @Input          hPrivate   : Implementation specific data
+ @Input          pszString  : Message to be printed
+ @Input          ...        : Variadic arguments
+
+ @Return         void
+
+******************************************************************************/
+__printf(2, 3)
+void RGXErrorLog(const void *hPrivate,
+                 const IMG_CHAR *pszString,
+                 ...);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function       RGXGetOSPageSize
+
+ @Description    Return Page size used on OS
+
+ @Input          hPrivate   : Implementation specific data
+
+ @Return         IMG_UINT32
+
+******************************************************************************/
+
+IMG_UINT32 RGXGetOSPageSize(const void *hPrivate);
+#endif
+
+/* This is used to check if a specific feature is enabled.
+ * Should be used instead of calling RGXDeviceHasFeature.  */
+#define RGX_DEVICE_HAS_FEATURE(hPrivate, Feature) \
+			RGXDeviceHasFeature(hPrivate, RGX_FEATURE_##Feature##_BIT_MASK)
+
+/* This is used to check if a specific feature with value is enabled.
+ * Should be used instead of calling RGXDeviceGetFeatureValue.  */
+#define RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, Feature) \
+			(RGXDeviceGetFeatureValue(hPrivate, RGX_FEATURE_##Feature##_IDX) >= 0)
+
+/* This is used to get the value of a specific feature from hPrivate.
+ * Should be used instead of calling RGXDeviceGetFeatureValue.  */
+#define RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, Feature) \
+			RGXDeviceGetFeatureValue(hPrivate, RGX_FEATURE_##Feature##_IDX)
+
+
+/* This is used to check if a specific ERN is enabled from hPrivate.
+ * Should be used instead of calling RGXDeviceHasErnBrn.  */
+#define RGX_DEVICE_HAS_ERN(hPrivate, ERN) \
+			RGXDeviceHasErnBrn(hPrivate, HW_ERN_##ERN##_BIT_MASK)
+
+/* This is used to check if a specific BRN is enabled from hPrivate.
+ * Should be used instead of calling RGXDeviceHasErnBrn.  */
+#define RGX_DEVICE_HAS_BRN(hPrivate, BRN) \
+			RGXDeviceHasErnBrn(hPrivate, FIX_HW_BRN_##BRN##_BIT_MASK)
+
+#define CLK_CTRL_FORCE_ON(X, Module) \
+			X = (((X) & RGX_CR_##Module##_CLRMSK) | RGX_CR_##Module##_ON)
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceGetFeatureValue
+
+ @Description    Checks if a device has a particular feature with values
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64Feature  : Feature with values to check
+
+ @Return         Value >= 0 if the given feature is available, -1 otherwise
+
+******************************************************************************/
+IMG_INT32 RGXDeviceGetFeatureValue(const void *hPrivate, IMG_UINT64 ui64Feature);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceHasFeature
+
+ @Description    Checks if a device has a particular feature
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64Feature  : Feature to check
+
+ @Return         IMG_TRUE if the given feature is available, IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDeviceHasFeature(const void *hPrivate, IMG_UINT64 ui64Feature);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDeviceHasErnBrn
+
+ @Description    Checks if a device has a particular errata
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui64ErnsBrns : Flags to check
+
+ @Return         IMG_TRUE if the given errata is available, IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDeviceHasErnBrn(const void *hPrivate, IMG_UINT64 ui64ErnsBrns);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetFWCorememSize
+
+ @Description    Get the FW coremem size
+
+ @Input          hPrivate   : Implementation specific data
+
+ @Return         FW coremem size
+
+******************************************************************************/
+IMG_UINT32 RGXGetFWCorememSize(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXWriteReg32/64
+
+ @Description   Write a value to a 32/64 bit RGX register
+
+ @Input         hPrivate         : Implementation specific data
+ @Input         ui32RegAddr      : Register offset inside the register bank
+ @Input         ui32/64RegValue  : New register value
+
+ @Return        void
+
+******************************************************************************/
+void RGXWriteReg32(const void *hPrivate,
+                   IMG_UINT32 ui32RegAddr,
+                   IMG_UINT32 ui32RegValue);
+
+void RGXWriteReg64(const void *hPrivate,
+                   IMG_UINT32 ui32RegAddr,
+                   IMG_UINT64 ui64RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadReg32/64
+
+ @Description    Read a 32/64 bit RGX register
+
+ @Input          hPrivate     : Implementation specific data
+ @Input          ui32RegAddr  : Register offset inside the register bank
+
+ @Return         Register value
+
+******************************************************************************/
+IMG_UINT32 RGXReadReg32(const void *hPrivate,
+                        IMG_UINT32 ui32RegAddr);
+
+IMG_UINT64 RGXReadReg64(const void *hPrivate,
+                        IMG_UINT32 ui32RegAddr);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadModifyWriteReg32
+
+ @Description    Read-modify-write a 32 bit RGX register
+
+ @Input          hPrivate     : Implementation specific data.
+ @Input          ui32RegAddr  : Register offset inside the register bank.
+ @Input          ui32RegValue : New register value.
+ @Input          ui32RegMask  : Keep the bits set in the mask.
+
+ @Return         Always returns PVRSRV_OK
+
+******************************************************************************/
+IMG_UINT32 RGXReadModifyWriteReg64(const void *hPrivate,
+                                   IMG_UINT32 ui32RegAddr,
+                                   IMG_UINT64 ui64RegValue,
+                                   IMG_UINT64 ui64RegKeepMask);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXPollReg32/64
+
+ @Description    Poll on a 32/64 bit RGX register until some bits are set/unset
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          ui32/64RegValue  : Value expected from the register
+ @Input          ui32/64RegMask   : Only the bits set in this mask will be
+                                    checked against uiRegValue
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV_ERROR_TIMEOUT if the poll takes too long
+
+******************************************************************************/
+PVRSRV_ERROR RGXPollReg32(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT32 ui32RegValue,
+                          IMG_UINT32 ui32RegMask);
+
+PVRSRV_ERROR RGXPollReg64(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64RegValue,
+                          IMG_UINT64 ui64RegMask);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXWriteMetaRegThroughSP
+
+ @Description    Write a register value using the META slave port
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          ui32RegValue     : Value written to the register
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXReadMetaRegThroughSP
+
+ @Description    Read a register value using the META slave port
+
+ @Input          hPrivate         : Implementation specific data
+ @Input          ui32RegAddr      : Register offset inside the register bank
+ @Input          *ui32RegValue    : Value read from the register
+
+ @Return         PVRSRV_OK if the poll succeeds,
+                 PVRSRV error code otherwise
+
+******************************************************************************/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXSetPoweredState
+
+ @Description    Declare if the device is powered or not
+
+ @Input          hPrivate    : Implementation specific data
+ @Input          bPowered    : true is powered, false otherwise
+
+ @Return         void
+
+******************************************************************************/
+void RGXSetPoweredState(const void *hPrivate, IMG_BOOL bPowered);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXWaitCycles
+
+ @Description    Wait for a number of GPU cycles and/or microseconds
+
+ @Input          hPrivate    : Implementation specific data
+ @Input          ui32Cycles  : Number of GPU cycles to wait for in pdumps,
+                               it can also be used when running driver-live
+                               if desired (ignoring the next parameter)
+ @Input          ui32WaitUs  : Number of microseconds to wait for when running
+                               driver-live
+
+ @Return         void
+
+******************************************************************************/
+void RGXWaitCycles(const void *hPrivate,
+                   IMG_UINT32 ui32Cycles,
+                   IMG_UINT32 ui32WaitUs);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireKernelMMUPC
+
+ @Description     Acquire the Kernel MMU Page Catalogue device physical address
+
+ @Input           hPrivate  : Implementation specific data
+ @Input           psPCAddr  : Returned page catalog address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireKernelMMUPC(const void *hPrivate, IMG_DEV_PHYADDR *psPCAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXWriteKernelMMUPC32/64
+
+ @Description     Write the Kernel MMU Page Catalogue to the 32/64 bit
+                  RGX register passed as argument.
+                  In a driver-live scenario without PDump these functions
+                  are the same as RGXWriteReg32/64 and they don't need
+                  to be reimplemented.
+
+ @Input           hPrivate        : Implementation specific data
+ @Input           ui32PCReg       : Register offset inside the register bank
+ @Input           ui32AlignShift  : PC register alignshift
+ @Input           ui32Shift       : PC register shift
+ @Input           ui32/64PCVal    : Page catalog value (aligned and shifted)
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+void RGXWriteKernelMMUPC64(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT64 ui64PCVal);
+#endif
+
+void RGXWriteKernelMMUPC32(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT32 ui32PCVal);
+#else /* defined(PDUMP) */
+
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGXWriteKernelMMUPC64(priv, pcreg, alignshift, shift, pcval) \
+	RGXWriteReg64(priv, pcreg, pcval)
+#endif
+
+#define RGXWriteKernelMMUPC32(priv, pcreg, alignshift, shift, pcval) \
+	RGXWriteReg32(priv, pcreg, pcval)
+
+#endif /* defined(PDUMP) */
+
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireGPURegsAddr
+
+ @Description     Acquire the GPU registers base device physical address
+
+ @Input           hPrivate       : Implementation specific data
+ @Input           psGPURegsAddr  : Returned GPU registers base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireGPURegsAddr(const void *hPrivate, IMG_DEV_PHYADDR *psGPURegsAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXMIPSWrapperConfig
+
+ @Description     Write GPU register bank transaction ID and MIPS boot mode
+                  to the MIPS wrapper config register (passed as argument).
+                  In a driver-live scenario without PDump this is the same as
+                  RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate          : Implementation specific data
+ @Input           ui32RegAddr       : Register offset inside the register bank
+ @Input           ui64GPURegsAddr   : GPU registers base address
+ @Input           ui32GPURegsAlign  : Register bank transactions alignment
+ @Input           ui32BootMode      : Mips BOOT ISA mode
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXMIPSWrapperConfig(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64GPURegsAddr,
+                          IMG_UINT32 ui32GPURegsAlign,
+                          IMG_UINT32 ui32BootMode);
+#else
+#define RGXMIPSWrapperConfig(priv, regaddr, gpuregsaddr, gpuregsalign, bootmode) \
+	RGXWriteReg64(priv, regaddr, ((gpuregsaddr) >> (gpuregsalign)) | (bootmode))
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS bootloader
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootRemapAddr  : Base address of the remapped bootloader
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psBootRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXBootRemapConfig
+
+ @Description     Configure the bootloader remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXBootRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXBootRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireCodeRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS code
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psCodeRemapAddr  : Base address of the remapped code
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireCodeRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psCodeRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXCodeRemapConfig
+
+ @Description     Configure the code remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXCodeRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXCodeRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireDataRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS data
+                  accessed through remap region
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psDataRemapAddr  : Base address of the remapped data
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireDataRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psDataRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXDataRemapConfig
+
+ @Description     Configure the data remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXDataRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXDataRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireTrampolineRemapAddr
+
+ @Description     Acquire the device physical address of the MIPS data
+                  accessed through remap region
+
+ @Input           hPrivate             : Implementation specific data
+ @Output          psTrampolineRemapAddr: Base address of the remapped data
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireTrampolineRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psTrampolineRemapAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXTrampolineRemapConfig
+
+ @Description     Configure the trampoline remap registers passed as arguments.
+                  In a driver-live scenario without PDump this is the same as
+                  two RGXWriteReg64 and it doesn't need to be reimplemented.
+
+ @Input           hPrivate             : Implementation specific data
+ @Input           ui32Config1RegAddr   : Remap config1 register offset
+ @Input           ui64Config1RegValue  : Remap config1 register value
+ @Input           ui32Config2RegAddr   : Remap config2 register offset
+ @Input           ui64Config2PhyAddr   : Output remapped aligned physical address
+ @Input           ui64Config2PhyMask   : Mask for the output physical address
+ @Input           ui64Config2Settings  : Extra settings for this remap region
+
+ @Return          void
+
+******************************************************************************/
+#if defined(PDUMP)
+void RGXTrampolineRemapConfig(const void *hPrivate,
+                        IMG_UINT32 ui32Config1RegAddr,
+                        IMG_UINT64 ui64Config1RegValue,
+                        IMG_UINT32 ui32Config2RegAddr,
+                        IMG_UINT64 ui64Config2PhyAddr,
+                        IMG_UINT64 ui64Config2PhyMask,
+                        IMG_UINT64 ui64Config2Settings);
+#else
+#define RGXTrampolineRemapConfig(priv, c1reg, c1val, c2reg, c2phyaddr, c2phymask, c2settings) do { \
+		RGXWriteReg64(priv, c1reg, (c1val)); \
+		RGXWriteReg64(priv, c2reg, ((c2phyaddr) & (c2phymask)) | (c2settings)); \
+	} while (0)
+#endif
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+/*!
+*******************************************************************************
+
+ @Function        RGXDoFWSlaveBoot
+
+ @Description     Returns whether or not a FW Slave Boot is required
+                  while powering on
+
+ @Input           hPrivate       : Implementation specific data
+
+ @Return          IMG_BOOL
+
+******************************************************************************/
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXFabricCoherencyTest
+
+ @Description    Performs fabric coherency test
+
+ @Input          hPrivate         : Implementation specific data
+
+ @Return         PVRSRV_OK if the test succeeds,
+                 PVRSRV_ERROR_INIT_FAILURE if the test fails at some point
+
+******************************************************************************/
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDeviceSLCBanks
+
+ @Description    Returns the number of SLC banks used by the device
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Number of SLC banks
+
+******************************************************************************/
+IMG_UINT32 RGXGetDeviceSLCBanks(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDeviceCacheLineSize
+
+ @Description    Returns the device cache line size
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Cache line size
+
+******************************************************************************/
+IMG_UINT32 RGXGetDeviceCacheLineSize(const void *hPrivate);
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function       RGXGetDevicePhysBusWidth
+
+ @Description    Returns the device physical bus width
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         Physical bus width
+
+******************************************************************************/
+IMG_UINT32 RGXGetDevicePhysBusWidth(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function       RGXDevicePA0IsValid
+
+ @Description    Returns true if the device physical address 0x0 is a valid
+                 address and can be accessed by the GPU.
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return         IMG_TRUE if device physical address 0x0 is a valid address,
+                 IMG_FALSE otherwise
+
+******************************************************************************/
+IMG_BOOL RGXDevicePA0IsValid(const void *hPrivate);
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootCodeAddr
+
+ @Description     Acquire the device virtual address of the RISCV boot code
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootCodeAddr   : Boot code base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootCodeAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootCodeAddr);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXCalculateHostFWDataAddress
+
+ @Description   Calculates the base host address of the RISCV firmware data
+
+ @Input         hPrivate         : Implementation specific data
+ @Input         pvHostFWDataAddr : Initial host address of the firmware data
+
+ @Return:       Base host address of the RISCV firmware data
+
+******************************************************************************/
+void *RGXCalculateHostFWDataAddress(const void *hPrivate, void *pvHostFWDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function        RGXAcquireBootDataAddr
+
+ @Description     Acquire the device virtual address of the RISCV boot data
+
+ @Input           hPrivate         : Implementation specific data
+ @Output          psBootDataAddr   : Boot data base address
+
+ @Return          void
+
+******************************************************************************/
+void RGXAcquireBootDataAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootDataAddr);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXDeviceAckIrq
+
+ @Description   Checks the implementation specific IRQ status register,
+                clearing it if necessary and returning the IRQ status.
+
+ @Input          hPrivate    : Implementation specific data
+
+ @Return:       IRQ status
+
+******************************************************************************/
+IMG_BOOL RGXDeviceAckIrq(const void *hPrivate);
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*!
+*******************************************************************************
+
+@Function       RGXMMUInitRangeValue
+
+@Description    Returns the appropriate config value for each MMU range
+
+@Input          ui32MMURange   : Hardware MMU range to be initialised
+
+@Return         64-bit register value
+
+******************************************************************************/
+IMG_UINT64 RGXMMUInitRangeValue(IMG_UINT32 ui32MMURange);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* RGXLAYER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxlayer_impl.c b/drivers/gpu/drm/img-rogue/rgxlayer_impl.c
new file mode 100644
index 000000000000..2141554bfdae
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxlayer_impl.c
@@ -0,0 +1,478 @@
+/*************************************************************************/ /*!
+@File
+@Title          DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "osfunc.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwimageutils.h"
+#include "cache_km.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+
+#define MAX_NUM_COHERENCY_TESTS  (10)
+IMG_BOOL RGXDoFWSlaveBoot(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	if (psDevInfo->ui32CoherencyTestsDone >= MAX_NUM_COHERENCY_TESTS)
+	{
+		return IMG_FALSE;
+	}
+
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	return PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig);
+}
+
+/*
+ * The fabric coherency test is performed when platform supports fabric coherency
+ * either in the form of ACE-lite or Full-ACE. This test is done quite early
+ * with the firmware processor quiescent and makes exclusive use of the slave
+ * port interface for reading/writing through the device memory hierarchy. The
+ * rationale for the test is to ensure that what the CPU writes to its dcache
+ * is visible to the GPU via coherency snoop miss/hit and vice-versa without
+ * any intervening cache maintenance by the writing agent.
+ */
+PVRSRV_ERROR RGXFabricCoherencyTest(const void *hPrivate)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 *pui32FabricCohTestBufferCpuVA;
+	DEVMEM_MEMDESC *psFabricCohTestBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR sFabricCohTestBufferDevVA;
+	IMG_DEVMEM_SIZE_T uiFabricCohTestBlockSize = sizeof(IMG_UINT64);
+	IMG_DEVMEM_ALIGN_T uiFabricCohTestBlockAlign = sizeof(IMG_UINT64);
+	IMG_UINT32 ui32SLCCTRL = 0;
+	IMG_UINT32 ui32OddEven;
+	IMG_UINT32 ui32OddEvenSeed = 1;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bFullTestPassed = IMG_TRUE;
+	IMG_BOOL bExit = IMG_FALSE;
+#if defined(DEBUG)
+	IMG_BOOL bSubTestPassed = IMG_FALSE;
+#endif
+	enum TEST_TYPE {
+		CPU_WRITE_GPU_READ_SM=0, GPU_WRITE_CPU_READ_SM,
+		CPU_WRITE_GPU_READ_SH,   GPU_WRITE_CPU_READ_SH
+	} eTestType;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	PVR_LOG(("Starting fabric coherency test ....."));
+
+	{
+		/* Bypass the SLC when IO coherency is enabled */
+		ui32SLCCTRL = RGXReadReg32(hPrivate, RGX_CR_SLC_CTRL_BYPASS);
+		RGXWriteReg32(hPrivate,
+				RGX_CR_SLC_CTRL_BYPASS,
+				ui32SLCCTRL | RGX_CR_SLC_CTRL_BYPASS_BYP_CC_EN);
+	}
+
+	/* Size and align are 'expanded' because we request an export align allocation */
+	eError = DevmemExportalignAdjustSizeAndAlign(DevmemGetHeapLog2PageSize(psDevInfo->psFirmwareMainHeap),
+	                                             &uiFabricCohTestBlockSize,
+	                                             &uiFabricCohTestBlockAlign);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemExportalignAdjustSizeAndAlign() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	/* Allocate, acquire cpu address and set firmware address */
+	eError = DevmemFwAllocateExportable(psDevInfo->psDeviceNode,
+										uiFabricCohTestBlockSize,
+										uiFabricCohTestBlockAlign,
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_GPU_CACHE_COHERENT |
+										PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwExFabricCoherencyTestBuffer",
+										&psFabricCohTestBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemFwAllocateExportable() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e0;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psFabricCohTestBufferMemDesc, (void **) &pui32FabricCohTestBufferCpuVA);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"DevmemAcquireCpuVirtAddr() error: %s, exiting",
+				PVRSRVGetErrorString(eError)));
+		goto e1;
+	}
+
+	/* Create a FW address which is uncached in the Meta DCache and in the SLC using the Meta bootloader segment.
+	   This segment is the only one configured correctly out of reset (when this test is meant to be executed) */
+
+	eError = RGXSetFirmwareAddress(&sFabricCohTestBufferDevVA,
+						  psFabricCohTestBufferMemDesc,
+						  0,
+						  RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", e2);
+
+	/* Undo most of the FW mappings done by RGXSetFirmwareAddress */
+	sFabricCohTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_META_CACHE_MASK;
+	sFabricCohTestBufferDevVA.ui32Addr &= ~RGXFW_SEGMMU_DATA_VIVT_SLC_CACHE_MASK;
+	sFabricCohTestBufferDevVA.ui32Addr -= RGXFW_SEGMMU_DATA_BASE_ADDRESS;
+
+	/* Map the buffer in the bootloader segment as uncached */
+	sFabricCohTestBufferDevVA.ui32Addr |= RGXFW_BOOTLDR_META_ADDR;
+	sFabricCohTestBufferDevVA.ui32Addr |= RGXFW_SEGMMU_DATA_META_UNCACHED;
+
+	for (eTestType = CPU_WRITE_GPU_READ_SH; eTestType <= GPU_WRITE_CPU_READ_SH && bExit == IMG_FALSE; eTestType++)
+	{
+		IMG_CPU_PHYADDR sCpuPhyAddr;
+		IMG_BOOL bValid;
+		PMR *psPMR;
+
+		/* Acquire underlying PMR CpuPA in preparation for cache maintenance */
+		(void) DevmemLocalGetImportHandle(psFabricCohTestBufferMemDesc, (void**)&psPMR);
+		eError = PMR_CpuPhysAddr(psPMR, OSGetPageShift(), 1, 0, &sCpuPhyAddr, &bValid);
+		if (eError != PVRSRV_OK || bValid == IMG_FALSE)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"PMR_CpuPhysAddr error: %s, exiting",
+					PVRSRVGetErrorString(eError)));
+			bExit = IMG_TRUE;
+			continue;
+		}
+
+		/* Here we do two passes mostly to account for the effects of using a different
+		   seed (i.e. ui32OddEvenSeed) value to read and write */
+		for (ui32OddEven = 1; ui32OddEven < 3 && bExit == IMG_FALSE; ui32OddEven++)
+		{
+			IMG_UINT32 i;
+
+#if defined(DEBUG)
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: starting [run #%u]", ui32OddEven));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit  Test: starting [run #%u]", ui32OddEven));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+
+			/* Do multiple sub-dword cache line tests */
+			for (i = 0; i < 2 && bExit == IMG_FALSE; i++)
+			{
+				IMG_UINT32 ui32FWAddr;
+				IMG_UINT32 ui32FWValue;
+				IMG_UINT32 ui32FWValue2;
+				IMG_CPU_PHYADDR sCpuPhyAddrStart;
+				IMG_CPU_PHYADDR sCpuPhyAddrEnd;
+				IMG_UINT32 ui32LastFWValue = ~0;
+				IMG_UINT32 ui32Offset = i * sizeof(IMG_UINT32);
+
+				/* Calculate next address and seed value to write/read from slave-port */
+				ui32FWAddr = sFabricCohTestBufferDevVA.ui32Addr + ui32Offset;
+				sCpuPhyAddrStart.uiAddr = sCpuPhyAddr.uiAddr + ui32Offset;
+				sCpuPhyAddrEnd.uiAddr = sCpuPhyAddrStart.uiAddr;
+				ui32OddEvenSeed += 1;
+
+				if (eTestType == GPU_WRITE_CPU_READ_SM || eTestType == GPU_WRITE_CPU_READ_SH)
+				{
+					/* Clean dcache to ensure there is no stale data in dcache that might over-write
+					   what we are about to write via slave-port here because if it drains from the CPU
+					   dcache before we read it, it would corrupt what we are going to read back via
+					   the CPU */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_FLUSH);
+
+					/* Calculate a new value to write */
+					ui32FWValue = i + ui32OddEvenSeed;
+
+					/* Write the value using the RGX slave-port interface */
+					eError = RGXWriteFWModuleAddr(psDevInfo, ui32FWAddr, ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXWriteFWModuleAddr error: %s, exiting",
+						          PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Read back value using RGX slave-port interface, this is used
+					   as a sort of memory barrier for the above write */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue2);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+						         "RGXReadFWModuleAddr error: %s, exiting",
+						         PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+					else if (ui32FWValue != ui32FWValue2)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW via SlavePort failed: expected: %x, got: %x",
+								i,
+								ui32FWValue,
+								ui32FWValue2));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Invalidate dcache to ensure that any prefetched data by the CPU from this memory
+					   region is discarded before we read (i.e. next read must trigger a cache miss).
+					   Previously there was snooping of device cache, where prefetching done by the CPU
+					   would reflect the most up to date datum writing by GPU into said location,
+					   that is to say prefetching was coherent so CPU d-flush was not needed */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_INVALIDATE);
+				}
+				else
+				{
+					IMG_UINT32 ui32RAWCpuValue;
+
+					/* Ensures line is in dcache */
+					ui32FWValue = IMG_UINT32_MAX;
+
+					/* Dirty allocation in dcache */
+					ui32RAWCpuValue = i + ui32OddEvenSeed;
+					pui32FabricCohTestBufferCpuVA[i] = i + ui32OddEvenSeed;
+
+					/* Flush possible cpu store-buffer(ing) on LMA */
+					OSWriteMemoryBarrier(&pui32FabricCohTestBufferCpuVA[i]);
+
+					if (eTestType == CPU_WRITE_GPU_READ_SM)
+					{
+						/* Flush dcache to force subsequent incoming CPU-bound snoop to miss so
+						   memory is coherent before the SlavePort reads */
+						sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+						CacheOpExec(psDevInfo->psDeviceNode,
+								(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+								(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+								sCpuPhyAddrStart,
+								sCpuPhyAddrEnd,
+								PVRSRV_CACHE_OP_FLUSH);
+					}
+
+					/* Read back value using RGX slave-port interface */
+					eError = RGXReadFWModuleAddr(psDevInfo, ui32FWAddr, &ui32FWValue);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"RGXReadFWModuleAddr error: %s, exiting",
+								PVRSRVGetErrorString(eError)));
+						bExit = IMG_TRUE;
+						continue;
+					}
+
+					/* Being mostly paranoid here, verify that CPU RAW operation is valid
+					   after the above slave port read */
+					sCpuPhyAddrEnd.uiAddr += sizeof(IMG_UINT32);
+					CacheOpExec(psDevInfo->psDeviceNode,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset,
+							(IMG_CHAR *)pui32FabricCohTestBufferCpuVA + ui32Offset + sizeof(IMG_UINT32),
+							sCpuPhyAddrStart,
+							sCpuPhyAddrEnd,
+							PVRSRV_CACHE_OP_FLUSH);
+					if (pui32FabricCohTestBufferCpuVA[i] != ui32RAWCpuValue)
+					{
+						/* Fatal error, we should abort */
+						PVR_DPF((PVR_DBG_ERROR,
+								"At Offset: %d, RAW by CPU failed: expected: %x, got: %x",
+								i,
+								ui32RAWCpuValue,
+								pui32FabricCohTestBufferCpuVA[i]));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				/* Compare to see if sub-test passed */
+				if (pui32FabricCohTestBufferCpuVA[i] == ui32FWValue)
+				{
+#if defined(DEBUG)
+					bSubTestPassed = IMG_TRUE;
+#endif
+				}
+				else
+				{
+					bFullTestPassed = IMG_FALSE;
+					eError = PVRSRV_ERROR_INIT_FAILURE;
+#if defined(DEBUG)
+					bSubTestPassed = IMG_FALSE;
+#endif
+					if (ui32LastFWValue != ui32FWValue)
+					{
+#if defined(DEBUG)
+						PVR_LOG(("At Offset: %d, Expected: %x, Got: %x",
+								 i,
+								 (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i],
+								 (eTestType & 0x1) ? pui32FabricCohTestBufferCpuVA[i] : ui32FWValue));
+#endif
+					}
+					else
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+								"test encountered unexpected error, exiting"));
+						eError = PVRSRV_ERROR_INIT_FAILURE;
+						bExit = IMG_TRUE;
+						continue;
+					}
+				}
+
+				ui32LastFWValue = (eTestType & 0x1) ? ui32FWValue : pui32FabricCohTestBufferCpuVA[i];
+			}
+
+#if defined(DEBUG)
+			if (bExit)
+			{
+				continue;
+			}
+
+			switch (eTestType)
+			{
+			case CPU_WRITE_GPU_READ_SM:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SM:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Miss Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case CPU_WRITE_GPU_READ_SH:
+				PVR_LOG(("CPU:Write/GPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			case GPU_WRITE_CPU_READ_SH:
+				PVR_LOG(("GPU:Write/CPU:Read Snoop Hit Test: completed [run #%u]: %s",
+						 ui32OddEven, bSubTestPassed ? "PASSED" : "FAILED"));
+				break;
+			default:
+				PVR_LOG(("Internal error, exiting test"));
+				eError = PVRSRV_ERROR_INIT_FAILURE;
+				bExit = IMG_TRUE;
+				continue;
+			}
+#endif
+		}
+	}
+
+	RGXUnsetFirmwareAddress(psFabricCohTestBufferMemDesc);
+e2:
+	DevmemReleaseCpuVirtAddr(psFabricCohTestBufferMemDesc);
+e1:
+	DevmemFwUnmapAndFree(psDevInfo, psFabricCohTestBufferMemDesc);
+
+e0:
+	{
+		/* Restore SLC bypass settings */
+		RGXWriteReg32(hPrivate, RGX_CR_SLC_CTRL_BYPASS, ui32SLCCTRL);
+	}
+
+	bFullTestPassed = bExit ? IMG_FALSE: bFullTestPassed;
+	if (bFullTestPassed)
+	{
+		PVR_LOG(("fabric coherency test: PASSED"));
+		psDevInfo->ui32CoherencyTestsDone = MAX_NUM_COHERENCY_TESTS + 1;
+	}
+	else
+	{
+		PVR_LOG(("fabric coherency test: FAILED"));
+		psDevInfo->ui32CoherencyTestsDone++;
+	}
+
+	return eError;
+}
+
diff --git a/drivers/gpu/drm/img-rogue/rgxlayer_impl.h b/drivers/gpu/drm/img-rogue/rgxlayer_impl.h
new file mode 100644
index 000000000000..b46efee8c71c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxlayer_impl.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File
+@Title          Header for DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXLAYER_IMPL_H)
+#define RGXLAYER_IMPL_H
+
+#include "rgxlayer.h"
+#include "device_connection.h"
+
+typedef struct _RGX_LAYER_PARAMS_
+{
+	void *psDevInfo;
+	void *psDevConfig;
+#if defined(PDUMP)
+	IMG_UINT32 ui32PdumpFlags;
+#endif
+
+	IMG_DEV_PHYADDR sPCAddr;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	IMG_DEV_PHYADDR sGPURegAddr;
+	IMG_DEV_PHYADDR sBootRemapAddr;
+	IMG_DEV_PHYADDR sCodeRemapAddr;
+	IMG_DEV_PHYADDR sDataRemapAddr;
+	IMG_DEV_PHYADDR sTrampolineRemapAddr;
+	IMG_BOOL bDevicePA0IsValid;
+#endif
+} RGX_LAYER_PARAMS;
+
+#endif /* RGXLAYER_IMPL_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxlayer_impl_common.c b/drivers/gpu/drm/img-rogue/rgxlayer_impl_common.c
new file mode 100644
index 000000000000..7eab6fde4dcd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxlayer_impl_common.c
@@ -0,0 +1,914 @@
+/*************************************************************************/ /*!
+@File
+@Title          Common DDK implementation of the Services abstraction layer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Common DDK implementation of the Services abstraction layer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxlayer_impl.h"
+#include "pdump_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwimageutils.h"
+
+#if defined(PDUMP)
+#if defined(__linux__)
+ #include <linux/version.h>
+
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#endif
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+#define RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr) \
+	(((ui32RegAddr) < RGX_HOST_SECURE_REGBANK_OFFSET) ? \
+	 ((psDevInfo)->pvRegsBaseKM) : ((psDevInfo)->pvSecureRegsBaseKM))
+#else
+#define RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr) ((psDevInfo)->pvRegsBaseKM)
+#endif
+
+void RGXMemCopy(const void *hPrivate,
+                void *pvDst,
+                void *pvSrc,
+                size_t uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	OSDeviceMemCopy(pvDst, pvSrc, uiSize);
+}
+
+void RGXMemSet(const void *hPrivate,
+               void *pvDst,
+               IMG_UINT8 ui8Value,
+               size_t uiSize)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	OSDeviceMemSet(pvDst, ui8Value, uiSize);
+}
+
+void RGXCommentLog(const void *hPrivate,
+                   const IMG_CHAR *pszString,
+                   ...)
+{
+#if defined(PDUMP)
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	va_list argList;
+	va_start(argList, pszString);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	PDumpCommentWithFlagsVA(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS, pszString, argList);
+	va_end(argList);
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	PVR_UNREFERENCED_PARAMETER(pszString);
+#endif
+}
+
+void RGXErrorLog(const void *hPrivate,
+                 const IMG_CHAR *pszString,
+                 ...)
+{
+	IMG_CHAR szBuffer[PVR_MAX_DEBUG_MESSAGE_LEN];
+	va_list argList;
+
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+
+	va_start(argList, pszString);
+	vsnprintf(szBuffer, sizeof(szBuffer), pszString, argList);
+	va_end(argList);
+
+	PVR_DPF((PVR_DBG_ERROR, "%s", szBuffer));
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+IMG_UINT32 RGXGetOSPageSize(const void *hPrivate)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+	return OSGetPageSize();
+}
+#endif
+
+IMG_INT32 RGXDeviceGetFeatureValue(const void *hPrivate, IMG_UINT64 ui64Feature)
+{
+	IMG_INT32 i32Ret = -1;
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	psDeviceNode = psDevInfo->psDeviceNode;
+
+	if ((psDeviceNode->pfnGetDeviceFeatureValue))
+	{
+		i32Ret = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode, ui64Feature);
+	}
+
+	return i32Ret;
+}
+
+IMG_BOOL RGXDeviceHasFeature(const void *hPrivate, IMG_UINT64 ui64Feature)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->sDevFeatureCfg.ui64Features & ui64Feature) != 0;
+}
+
+IMG_UINT32 RGXGetFWCorememSize(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32CorememSize = 0;
+
+	PVR_ASSERT(hPrivate != NULL);
+
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META_COREMEM_SIZE))
+	{
+		ui32CorememSize = RGX_GET_FEATURE_VALUE(psDevInfo, META_COREMEM_SIZE);
+	}
+
+	return ui32CorememSize;
+}
+
+void RGXWriteReg32(const void *hPrivate, IMG_UINT32 ui32RegAddr, IMG_UINT32 ui32RegValue)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		OSWriteUncheckedHWReg32(pvRegsBase, ui32RegAddr, ui32RegValue);
+	}
+
+	PDUMPREG32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	           ui32RegAddr, ui32RegValue, psParams->ui32PdumpFlags);
+}
+
+void RGXWriteReg64(const void *hPrivate, IMG_UINT32 ui32RegAddr, IMG_UINT64 ui64RegValue)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		OSWriteUncheckedHWReg64(pvRegsBase, ui32RegAddr, ui64RegValue);
+	}
+
+	PDUMPREG64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	           ui32RegAddr, ui64RegValue, psParams->ui32PdumpFlags);
+}
+
+IMG_UINT32 RGXReadReg32(const void *hPrivate, IMG_UINT32 ui32RegAddr)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+	IMG_UINT32 ui32RegValue;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW)
+	{
+		ui32RegValue = IMG_UINT32_MAX;
+	}
+	else
+#endif
+	{
+		ui32RegValue = OSReadUncheckedHWReg32(pvRegsBase, ui32RegAddr);
+	}
+
+	PDUMPREGREAD32(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	               ui32RegAddr, psParams->ui32PdumpFlags);
+
+	return ui32RegValue;
+}
+
+IMG_UINT64 RGXReadReg64(const void *hPrivate, IMG_UINT32 ui32RegAddr)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+	IMG_UINT64 ui64RegValue;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW)
+	{
+		ui64RegValue = IMG_UINT64_MAX;
+	}
+	else
+#endif
+	{
+		ui64RegValue = OSReadUncheckedHWReg64(pvRegsBase, ui32RegAddr);
+	}
+
+	PDUMPREGREAD64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	               ui32RegAddr, PDUMP_FLAGS_CONTINUOUS);
+
+	return ui64RegValue;
+}
+
+IMG_UINT32 RGXReadModifyWriteReg64(const void *hPrivate,
+                                   IMG_UINT32 ui32RegAddr,
+                                   IMG_UINT64 uiRegValueNew,
+                                   IMG_UINT64 uiRegKeepMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+#if defined(PDUMP)
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+#endif
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+	/* only use the new values for bits we update according to the keep mask */
+	uiRegValueNew &= ~uiRegKeepMask;
+
+#if defined(PDUMP)
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store register offset to temp PDump variable */
+	PDumpRegRead64ToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                            ":SYSMEM:$1", ui32RegAddr, ui32PDumpFlags);
+
+	/* Keep the bits set in the mask */
+	PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                        uiRegKeepMask, ui32PDumpFlags);
+
+	/* OR the new values */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                       uiRegValueNew, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                        ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+
+	{
+		IMG_UINT64 uiRegValue = OSReadUncheckedHWReg64(pvRegsBase, ui32RegAddr);
+		uiRegValue &= uiRegKeepMask;
+		OSWriteUncheckedHWReg64(pvRegsBase, ui32RegAddr, uiRegValue | uiRegValueNew);
+	}
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollReg32(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT32 ui32RegValue,
+                          IMG_UINT32 ui32RegMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr),
+		                         ui32RegValue,
+		                         ui32RegMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg32: Poll for Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+	}
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr,
+	            ui32RegValue,
+	            ui32RegMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXPollReg64(const void *hPrivate,
+                          IMG_UINT32 ui32RegAddr,
+                          IMG_UINT64 ui64RegValue,
+                          IMG_UINT64 ui64RegMask)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	/* Split lower and upper words */
+	IMG_UINT32 ui32UpperValue = (IMG_UINT32) (ui64RegValue >> 32);
+	IMG_UINT32 ui32LowerValue = (IMG_UINT32) (ui64RegValue);
+	IMG_UINT32 ui32UpperMask = (IMG_UINT32) (ui64RegMask >> 32);
+	IMG_UINT32 ui32LowerMask = (IMG_UINT32) (ui64RegMask);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32RegAddr);
+
+#if defined(PDUMP)
+	if (!(psParams->ui32PdumpFlags & PDUMP_FLAGS_NOHW))
+#endif
+	{
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr + 4),
+		                         ui32UpperValue,
+		                         ui32UpperMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg64: Poll for upper part of Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		if (PVRSRVPollForValueKM(psDevInfo->psDeviceNode,
+		                         (IMG_UINT32 __iomem *)((IMG_UINT8 __iomem *)pvRegsBase + ui32RegAddr),
+		                         ui32LowerValue,
+		                         ui32LowerMask,
+		                         POLL_FLAG_LOG_ERROR,
+		                         NULL) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "RGXPollReg64: Poll for lower part of Reg (0x%x) failed", ui32RegAddr));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+	}
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr + 4,
+	            ui32UpperValue,
+	            ui32UpperMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+
+	PDUMPREGPOL(psDevInfo->psDeviceNode,
+	            RGX_PDUMPREG_NAME,
+	            ui32RegAddr,
+	            ui32LowerValue,
+	            ui32LowerMask,
+	            psParams->ui32PdumpFlags,
+	            PDUMP_POLL_OPERATOR_EQUAL);
+
+	return PVRSRV_OK;
+}
+
+void RGXSetPoweredState(const void *hPrivate, IMG_BOOL bPowered)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	psDevInfo->bRGXPowered = bPowered;
+}
+
+void RGXWaitCycles(const void *hPrivate, IMG_UINT32 ui32Cycles, IMG_UINT32 ui32TimeUs)
+{
+	__maybe_unused PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+	OSWaitus(ui32TimeUs);
+	PDUMPIDLWITHFLAGS(psDevInfo->psDeviceNode, ui32Cycles, PDUMP_FLAGS_CONTINUOUS);
+}
+
+void RGXAcquireKernelMMUPC(const void *hPrivate, IMG_DEV_PHYADDR *psPCAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psPCAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sPCAddr;
+}
+
+#if defined(PDUMP)
+#if !defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+void RGXWriteKernelMMUPC64(const void *hPrivate,
+		IMG_UINT32 ui32PCReg,
+		IMG_UINT32 ui32PCRegAlignShift,
+		IMG_UINT32 ui32PCRegShift,
+		IMG_UINT64 ui64PCVal)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write the cat-base address */
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM, ui32PCReg, ui64PCVal);
+
+	/* Pdump catbase address */
+	MMU_PDumpWritePageCatBase(psDevInfo->psKernelMMUCtx,
+			RGX_PDUMPREG_NAME,
+			ui32PCReg,
+			8,
+			ui32PCRegAlignShift,
+			ui32PCRegShift,
+			PDUMP_FLAGS_CONTINUOUS);
+}
+#endif
+
+void RGXWriteKernelMMUPC32(const void *hPrivate,
+                           IMG_UINT32 ui32PCReg,
+                           IMG_UINT32 ui32PCRegAlignShift,
+                           IMG_UINT32 ui32PCRegShift,
+                           IMG_UINT32 ui32PCVal)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	void __iomem *pvRegsBase;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+	pvRegsBase = RGX_GET_REGS_BASE(psDevInfo, ui32PCReg);
+
+	/* Write the cat-base address */
+	OSWriteUncheckedHWReg32(pvRegsBase, ui32PCReg, ui32PCVal);
+
+	/* Pdump catbase address */
+	MMU_PDumpWritePageCatBase(psDevInfo->psKernelMMUCtx,
+	                          RGX_PDUMPREG_NAME,
+	                          ui32PCReg,
+	                          4,
+	                          ui32PCRegAlignShift,
+	                          ui32PCRegShift,
+	                          PDUMP_FLAGS_CONTINUOUS);
+}
+#endif /* defined(PDUMP) */
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXAcquireGPURegsAddr(const void *hPrivate, IMG_DEV_PHYADDR *psGPURegsAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psGPURegsAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sGPURegAddr;
+}
+
+#if defined(PDUMP)
+void RGXMIPSWrapperConfig(const void *hPrivate,
+		IMG_UINT32 ui32RegAddr,
+		IMG_UINT64 ui64GPURegsAddr,
+		IMG_UINT32 ui32GPURegsAlign,
+		IMG_UINT32 ui32BootMode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
+			ui32RegAddr,
+			(ui64GPURegsAddr >> ui32GPURegsAlign) | ui32BootMode);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store register offset to temp PDump variable */
+	PDumpRegLabelToInternalVar(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                           ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	/* Align register transactions identifier */
+	PDumpWriteVarSHRValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                        ui32GPURegsAlign, ui32PDumpFlags);
+
+	/* Enable micromips instruction encoding */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+	                       ui32BootMode, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode, RGX_PDUMPREG_NAME,
+	                        ui32RegAddr, ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+}
+#endif
+
+void RGXAcquireBootRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psBootRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psBootRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sBootRemapAddr;
+}
+
+void RGXAcquireCodeRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psCodeRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psCodeRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sCodeRemapAddr;
+}
+
+void RGXAcquireDataRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psDataRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psDataRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sDataRemapAddr;
+}
+
+void RGXAcquireTrampolineRemapAddr(const void *hPrivate, IMG_DEV_PHYADDR *psTrampolineRemapAddr)
+{
+	PVR_ASSERT(hPrivate != NULL);
+	*psTrampolineRemapAddr = ((RGX_LAYER_PARAMS*)hPrivate)->sTrampolineRemapAddr;
+}
+
+#if defined(PDUMP)
+static inline
+void RGXWriteRemapConfig2Reg(void __iomem *pvRegs,
+		PMR *psPMR,
+		IMG_DEVMEM_OFFSET_T uiLogicalOffset,
+		IMG_UINT32 ui32RegAddr,
+		IMG_UINT64 ui64PhyAddr,
+		IMG_UINT64 ui64PhyMask,
+		IMG_UINT64 ui64Settings)
+{
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_ASSERT(psPMR != NULL);
+	psDevNode = PMR_DeviceNode(psPMR);
+
+	OSWriteHWReg64(pvRegs, ui32RegAddr, (ui64PhyAddr & ui64PhyMask) | ui64Settings);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store memory offset to temp PDump variable */
+	PDumpMemLabelToInternalVar64(":SYSMEM:$1", psPMR,
+	                             uiLogicalOffset, ui32PDumpFlags);
+
+	/* Keep only the relevant bits of the output physical address */
+	PDumpWriteVarANDValueOp(psDevNode, ":SYSMEM:$1", ui64PhyMask, ui32PDumpFlags);
+
+	/* Extra settings for this remapped region */
+	PDumpWriteVarORValueOp(psDevNode, ":SYSMEM:$1", ui64Settings, ui32PDumpFlags);
+
+	/* Do the actual register write */
+	PDumpInternalVarToReg64(psDevNode, RGX_PDUMPREG_NAME, ui32RegAddr,
+	                        ":SYSMEM:$1", ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+}
+
+void RGXBootRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32BootRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_CODE);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWCodeMemDesc->uiOffset + ui32BootRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXCodeRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32CodeRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_EXCEPTIONS_CODE);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWCodeMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWCodeMemDesc->uiOffset + ui32CodeRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXDataRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32DataRemapMemOffset = RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* Write remap config1 register */
+	RGXWriteReg64(hPrivate,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	/* Write remap config2 register */
+	RGXWriteRemapConfig2Reg(psDevInfo->pvRegsBaseKM,
+			psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+			psDevInfo->psRGXFWDataMemDesc->uiOffset + ui32DataRemapMemOffset,
+			ui32Config2RegAddr,
+			ui64Config2PhyAddr,
+			ui64Config2PhyMask,
+			ui64Config2Settings);
+}
+
+void RGXTrampolineRemapConfig(const void *hPrivate,
+		IMG_UINT32 ui32Config1RegAddr,
+		IMG_UINT64 ui64Config1RegValue,
+		IMG_UINT32 ui32Config2RegAddr,
+		IMG_UINT64 ui64Config2PhyAddr,
+		IMG_UINT64 ui64Config2PhyMask,
+		IMG_UINT64 ui64Config2Settings)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PDUMP_FLAGS_T ui32PDumpFlags = PDUMP_FLAGS_CONTINUOUS;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	/* write the register for real, without PDump */
+	OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
+			ui32Config1RegAddr,
+			ui64Config1RegValue);
+
+	PDUMP_BLKSTART(ui32PDumpFlags);
+
+	/* Store the memory address in a PDump variable */
+	PDumpPhysHandleToInternalVar64(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			psDevInfo->psTrampoline->hPdumpPages,
+			ui32PDumpFlags);
+
+	/* Keep only the relevant bits of the input physical address */
+	PDumpWriteVarANDValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			~RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_BASE_ADDR_IN_CLRMSK,
+			ui32PDumpFlags);
+
+	/* Enable bit */
+	PDumpWriteVarORValueOp(psDevInfo->psDeviceNode, ":SYSMEM:$1",
+			RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+			ui32PDumpFlags);
+
+	/* Do the PDump register write */
+	PDumpInternalVarToReg64(psDevInfo->psDeviceNode,
+			RGX_PDUMPREG_NAME,
+			ui32Config1RegAddr,
+			":SYSMEM:$1",
+			ui32PDumpFlags);
+
+	PDUMP_BLKEND(ui32PDumpFlags);
+
+	/* this can be written directly */
+	RGXWriteReg64(hPrivate,
+			ui32Config2RegAddr,
+			(ui64Config2PhyAddr & ui64Config2PhyMask) | ui64Config2Settings);
+}
+#endif /* defined(PDUMP) */
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+
+IMG_BOOL RGXDeviceHasErnBrn(const void *hPrivate, IMG_UINT64 ui64ErnsBrns)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->sDevFeatureCfg.ui64ErnsBrns & ui64ErnsBrns) != 0;
+}
+
+IMG_UINT32 RGXGetDeviceSLCBanks(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_BANKS))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, SLC_BANKS);
+}
+
+IMG_UINT32 RGXGetDeviceCacheLineSize(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, SLC_CACHE_LINE_SIZE_BITS))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, SLC_CACHE_LINE_SIZE_BITS);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+IMG_UINT32 RGXGetDevicePhysBusWidth(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (!RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PHYS_BUS_WIDTH))
+	{
+		return 0;
+	}
+	return RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH);
+}
+
+IMG_BOOL RGXDevicePA0IsValid(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return psDevInfo->sLayerParams.bDevicePA0IsValid;
+}
+#endif
+
+void RGXAcquireBootCodeAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootCodeAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	*psBootCodeAddr = psDevInfo->sFWCodeDevVAddrBase;
+}
+
+void RGXAcquireBootDataAddr(const void *hPrivate, IMG_DEV_VIRTADDR *psBootDataAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psDevInfo = ((RGX_LAYER_PARAMS*)hPrivate)->psDevInfo;
+
+	*psBootDataAddr = psDevInfo->sFWDataDevVAddrBase;
+}
+
+void *RGXCalculateHostFWDataAddress(const void *hPrivate, void *pvHostFWDataAddr)
+{
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT8 *ui8HostFWDataAddr = (IMG_UINT8*)pvHostFWDataAddr;
+	IMG_UINT32 ui32Offset = 0U;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, HOST_SECURITY_VERSION) >= 4)
+	{
+		ui32Offset =
+			PVR_ALIGN(RGXGetFWImageSectionAllocSize(hPrivate, RISCV_UNCACHED_CODE),
+			          RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN) +
+			PVR_ALIGN(RGXGetFWImageSectionAllocSize(hPrivate, RISCV_CACHED_CODE),
+			          RGXRISCVFW_REMAP_CONFIG_DEVVADDR_ALIGN);
+	}
+
+	ui8HostFWDataAddr -= ui32Offset;
+	return (void*)ui8HostFWDataAddr;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPrivate);
+
+	return pvHostFWDataAddr;
+#endif
+}
+
+IMG_BOOL RGXDeviceAckIrq(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_ASSERT(hPrivate != NULL);
+	psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	psDevInfo = psParams->psDevInfo;
+
+	return (psDevInfo->pfnRGXAckIrq != NULL) ?
+			psDevInfo->pfnRGXAckIrq(psDevInfo) : IMG_TRUE;
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxmem.c b/drivers/gpu/drm/img-rogue/rgxmem.c
new file mode 100644
index 000000000000..4df36ef5d1f0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmem.c
@@ -0,0 +1,973 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX memory context management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX memory context management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvr_debug.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_server_utils.h"
+#include "devicemem_pdump.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_km.h"
+#include "rgxfwutils.h"
+#include "pdump_km.h"
+#include "pdump_physmem.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+#include "sync_internal.h"
+#include "rgx_memallocflags.h"
+#include "rgx_bvnc_defs_km.h"
+#include "info_page.h"
+
+# include "rgxmmudefs_km.h"
+
+#if defined(PDUMP)
+#include "sync.h"
+#endif
+
+struct SERVER_MMU_CONTEXT_TAG
+{
+	DEVMEM_MEMDESC *psFWMemContextMemDesc;
+	PRGXFWIF_FWMEMCONTEXT sFWMemContextDevVirtAddr;
+	MMU_CONTEXT *psMMUContext;
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_UINT64 ui64FBSCEntryMask;
+	DLLIST_NODE sNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+}; /* SERVER_MMU_CONTEXT is typedef-ed in rgxmem.h */
+
+PVRSRV_ERROR RGXInvalidateFBSCTable(PVRSRV_DEVICE_NODE *psDeviceNode,
+									MMU_CONTEXT *psMMUContext,
+									IMG_UINT64 ui64FBSCEntryMask)
+{
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSWRLockAcquireRead(psDevInfo->hMemoryCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psIter = IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+		if (psIter->psMMUContext == psMMUContext)
+		{
+			psServerMMUContext = psIter;
+			break;
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hMemoryCtxListLock);
+
+	if (! psServerMMUContext)
+	{
+		return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+	}
+
+	/* Accumulate the FBSC invalidate request */
+	psServerMMUContext->ui64FBSCEntryMask |= ui64FBSCEntryMask;
+
+	return PVRSRV_OK;
+}
+
+/*
+ * RGXExtractFBSCEntryMaskFromMMUContext
+ *
+ */
+PVRSRV_ERROR RGXExtractFBSCEntryMaskFromMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+												   SERVER_MMU_CONTEXT *psServerMMUContext,
+												   IMG_UINT64 *pui64FBSCEntryMask)
+{
+	if (!psServerMMUContext)
+	{
+		return PVRSRV_ERROR_MMU_CONTEXT_NOT_FOUND;
+	}
+
+	*pui64FBSCEntryMask = psServerMMUContext->ui64FBSCEntryMask;
+	psServerMMUContext->ui64FBSCEntryMask = 0;
+
+	return PVRSRV_OK;
+}
+
+void RGXMMUCacheInvalidate(PVRSRV_DEVICE_NODE *psDeviceNode,
+						   MMU_CONTEXT *psMMUContext,
+						   MMU_LEVEL eMMULevel,
+						   IMG_BOOL bUnmap)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT32 ui32NewCacheFlags;
+
+	PVR_UNREFERENCED_PARAMETER(bUnmap);
+
+	switch (eMMULevel)
+	{
+		case MMU_LEVEL_3:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PC;
+
+			break;
+		case MMU_LEVEL_2:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PD;
+
+			break;
+		case MMU_LEVEL_1:
+			ui32NewCacheFlags = RGXFWIF_MMUCACHEDATA_FLAGS_PT;
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+			if (!(RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT)))
+#endif
+			{
+				ui32NewCacheFlags |= RGXFWIF_MMUCACHEDATA_FLAGS_TLB;
+			}
+
+			break;
+		default:
+			ui32NewCacheFlags = 0;
+			PVR_ASSERT(0);
+
+			break;
+	}
+
+#if defined(RGX_FEATURE_SLC_VIVT_BIT_MASK)
+	/* For VIVT devices only accumulate the flags on the Firmware MMU context
+	 * since the Firmware/HW invalidates caches on every kick automatically. */
+	if (!RGX_IS_FEATURE_SUPPORTED(psDevInfo, SLC_VIVT) ||
+	    psMMUContext == psDevInfo->psKernelMMUCtx)
+#endif
+	{
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32NewCacheFlags);
+	}
+}
+
+static
+PVRSRV_ERROR _PrepareAndSubmitCacheCommand(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           RGXFWIF_DM eDM,
+                                           IMG_UINT32 ui32CacheFlags,
+                                           IMG_BOOL bInterrupt,
+                                           IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD sFlushCmd;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	*pui32MMUInvalidateUpdate = psDeviceNode->ui32NextMMUInvalidateUpdate++;
+
+	/* Setup cmd and add the device nodes sync object */
+	sFlushCmd.eCmdType = RGXFWIF_KCCB_CMD_MMUCACHE;
+	sFlushCmd.uCmdData.sMMUCacheData.ui32MMUCacheSyncUpdateValue = *pui32MMUInvalidateUpdate;
+	SyncPrimGetFirmwareAddr(psDeviceNode->psMMUCacheSyncPrim,
+	                        &sFlushCmd.uCmdData.sMMUCacheData.sMMUCacheSync.ui32Addr);
+
+	/*
+	 *  Indicate if the firmware should signal command completion to the host.
+	 *  The sync update will always happen. This flag requests that the FW pass
+	 *  back the result of the KCCB command and interrupt the host. The KCCB
+	 *  response is not used but the interrupt signal to the host is as a way
+	 *  to know the sync update may have happened.
+	 */
+	if (bInterrupt)
+	{
+		ui32CacheFlags |= RGXFWIF_MMUCACHEDATA_FLAGS_INTERRUPT;
+	}
+
+	sFlushCmd.uCmdData.sMMUCacheData.ui32CacheFlags = ui32CacheFlags;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Submit MMU flush and invalidate (flags = 0x%08x)",
+	                      ui32CacheFlags);
+#endif
+
+	/* Schedule MMU cache command */
+	eError = RGXSendCommand(psDevInfo,
+							&sFlushCmd,
+							PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Failed to schedule MMU cache command to "
+		         "DM=%d with error (%u)",
+		         __func__, eDM, eError));
+		psDeviceNode->ui32NextMMUInvalidateUpdate--;
+
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32CacheFlags);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXMMUCacheInvalidateKick(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32FWCacheFlags;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+		goto RGXMMUCacheInvalidateKick_exit;
+	}
+
+	/*
+	 * Atomically clear flags to ensure we never accidentally read state
+	 * inconsistently or overwrite valid cache flags with 0.
+	 */
+	ui32FWCacheFlags = MMU_GetAndResetCacheFlags(psDevInfo->psKernelMMUCtx);
+	if (ui32FWCacheFlags == 0)
+	{
+		/* Nothing to do if no cache ops pending */
+		eError = PVRSRV_OK;
+		goto _PowerUnlockAndReturnErr;
+	}
+
+	/* Ensure device is powered up before sending cache command */
+	PDUMPPOWCMDSTART(psDeviceNode);
+	eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+										 PVRSRV_DEV_POWER_STATE_ON,
+										 PVRSRV_POWER_FLAGS_NONE);
+	PDUMPPOWCMDEND(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to transition RGX to ON (%s)",
+					__func__, PVRSRVGetErrorString(eError)));
+		MMU_AppendCacheFlags(psDevInfo->psKernelMMUCtx, ui32FWCacheFlags);
+		goto _PowerUnlockAndReturnErr;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = _PrepareAndSubmitCacheCommand(psDeviceNode, RGXFWIF_DM_GP, ui32FWCacheFlags,
+											   IMG_TRUE, pui32MMUInvalidateUpdate);
+		if (!PVRSRVIsRetryError(eError))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_LOG_IF_ERROR(eError, "_PrepareAndSubmitCacheCommand");
+
+_PowerUnlockAndReturnErr:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+RGXMMUCacheInvalidateKick_exit:
+	return eError;
+}
+
+PVRSRV_ERROR RGXPreKickCacheCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+									RGXFWIF_DM eDM,
+									IMG_UINT32 *pui32MMUInvalidateUpdate)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psDevInfo->psDeviceNode;
+	IMG_UINT32 ui32FWCacheFlags;
+
+	/* Caller should ensure that power lock is held before calling this function */
+	PVR_ASSERT(OSLockIsLocked(psDeviceNode->hPowerLock));
+
+	/*
+	 * Atomically clear flags to ensure we never accidentally read state
+	 * inconsistently or overwrite valid cache flags with 0.
+	 */
+	ui32FWCacheFlags = MMU_GetAndResetCacheFlags(psDevInfo->psKernelMMUCtx);
+	if (ui32FWCacheFlags == 0)
+	{
+		/* Nothing to do if no cache ops pending */
+		return PVRSRV_OK;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = _PrepareAndSubmitCacheCommand(psDeviceNode, eDM, ui32FWCacheFlags,
+		                                       IMG_FALSE, pui32MMUInvalidateUpdate);
+		if (!PVRSRVIsRetryError(eError))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "_PrepareAndSubmitCacheCommand");
+
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+/*
+ * RGXMapBRN71422TargetPhysicalAddress
+ *
+ * Only used on early Volcanic cores.
+ * Set-up a special MMU tree mapping with a single page that eventually points
+ * to RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR. This is supplied by the
+ * customer (in rgxdefs_km.h) and hence this WA is off by default.
+ *
+ * PC entries are 32b, with the last 4 bits being 0 except for the LSB bit that should be 1 (Valid). Addr is 4KB aligned.
+ * PD entries are 64b, with addr in bits 39:5 and everything else 0 except for LSB bit that is Valid. Addr is byte aligned?
+ * PT entries are 64b, with phy addr in bits 39:12 and everything else 0 except for LSB bit that is Valid. Addr is 4KB aligned.
+ * So, we can construct the page tables in a single page like this:
+ *   0x00 : PCE (PCE index 0)
+ *   0x04 : 0x0
+ *   0x08 : PDEa (PDE index 1)
+ *   0x0C : PDEb
+ *   0x10 : PTEa (PTE index 2)
+ *   0x14 : PTEb
+ *
+ * With the PCE and the PDE pointing to this same page.
+ * The VA address that we are mapping is therefore:
+ *  VA = PCE_idx*PCE_size + PDE_idx*PDE_size + PTE_idx*PTE_size =
+ *     =      0 * 1GB     +      1 * 2MB     +      2 * 4KB     =
+ *     =        0         +    0x20_0000     +      0x2000      =
+ *     = 0x00_0020_2000
+ */
+void RGXMapBRN71422TargetPhysicalAddress(CONNECTION_DATA *psConnection,
+                                         PVRSRV_DEVICE_NODE *psDevNode,
+                                         IMG_DEV_PHYADDR sPhysAddrL1Px,
+                                         void *pxL1PxCpuVAddr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	IMG_UINT32       *pui32Px    = pxL1PxCpuVAddr;
+	IMG_UINT64       *pui64Px    = pxL1PxCpuVAddr;
+	IMG_UINT64       ui64Entry;
+
+
+	/* Only setup the BRN71422 workaround if this is the FW memory
+	 * context and BRN present.
+	 */
+	if ((psConnection != NULL) || !RGX_IS_BRN_SUPPORTED(psDevInfo, 71422))
+	{
+		return;
+	}
+
+	/* Setup dummy mapping to fast constant time physical address. */
+	/* PCE points to PC */
+	ui64Entry = sPhysAddrL1Px.uiAddr;
+	ui64Entry = ui64Entry >> RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT;
+	ui64Entry = ui64Entry << RGX_MMUCTRL_PC_DATA_PD_BASE_SHIFT;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PC_DATA_PD_BASE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PC_DATA_VALID_EN;
+	pui32Px[0] = (IMG_UINT32) ui64Entry;
+
+	/* PDE points to PC */
+	ui64Entry = sPhysAddrL1Px.uiAddr;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PD_DATA_PT_BASE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PD_DATA_VALID_EN;
+	pui64Px[1] = ui64Entry;
+
+	/* PTE points to PAddr */
+	ui64Entry = RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR;
+	ui64Entry = ui64Entry & ~RGX_MMUCTRL_PT_DATA_PAGE_CLRMSK;
+	ui64Entry = ui64Entry | RGX_MMUCTRL_PT_DATA_VALID_EN;
+	pui64Px[2] = ui64Entry;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Mapping the BRN71422 workaround to target physical address 0x%" IMG_UINT64_FMTSPECx ".",
+	         __func__, RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR));
+
+}
+#endif
+
+/* Common header, only needed for architectures with MIPS FW CPU */
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXMMUTweakProtFlags(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                  MMU_DEVICEATTRIBS *psDevAttrs,
+		                  PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+		                  MMU_PROTFLAGS_T *puiMMUProtFlags)
+{
+	/* If we are allocating on the MMU of the firmware processor, the
+	 * cached/uncached attributes must depend on the FIRMWARE_CACHED
+	 * allocation flag.
+	 */
+	if (psDevAttrs == psDevNode->psFirmwareMMUDevAttrs)
+	{
+		if (uiMappingFlags & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED))
+		{
+			*puiMMUProtFlags |= MMU_PROTFLAGS_CACHED;
+		}
+		else
+		{
+			*puiMMUProtFlags &= ~MMU_PROTFLAGS_CACHED;
+
+		}
+		*puiMMUProtFlags &= ~MMU_PROTFLAGS_CACHE_COHERENT;
+	}
+}
+#endif
+
+
+/* page fault debug is the only current use case for needing to find process info
+ * after that process device memory context has been destroyed
+ */
+
+typedef struct _UNREGISTERED_MEMORY_CONTEXT_
+{
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_DEV_PHYADDR sPCDevPAddr;
+} UNREGISTERED_MEMORY_CONTEXT;
+
+/* must be a power of two */
+#define UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE (1 << 3)
+
+static UNREGISTERED_MEMORY_CONTEXT gasUnregisteredMemCtxs[UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE];
+static IMG_UINT32 gui32UnregisteredMemCtxsHead;
+
+/* record a device memory context being unregistered.
+ * the list of unregistered contexts can be used to find the PID and process name
+ * belonging to a memory context which has been destroyed
+ */
+static void _RecordUnregisteredMemoryContext(PVRSRV_RGXDEV_INFO *psDevInfo, SERVER_MMU_CONTEXT *psServerMMUContext)
+{
+	UNREGISTERED_MEMORY_CONTEXT *psRecord;
+
+	OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+	psRecord = &gasUnregisteredMemCtxs[gui32UnregisteredMemCtxsHead];
+
+	gui32UnregisteredMemCtxsHead = (gui32UnregisteredMemCtxsHead + 1)
+					& (UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1);
+
+	OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+
+	psRecord->uiPID = psServerMMUContext->uiPID;
+	if (MMU_AcquireBaseAddr(psServerMMUContext->psMMUContext, &psRecord->sPCDevPAddr) != PVRSRV_OK)
+	{
+		PVR_LOG(("_RecordUnregisteredMemoryContext: Failed to get PC address for memory context"));
+	}
+	OSStringSafeCopy(psRecord->szProcessName, psServerMMUContext->szProcessName, sizeof(psRecord->szProcessName));
+}
+
+
+void RGXUnregisterMemoryContext(IMG_HANDLE hPrivData)
+{
+	SERVER_MMU_CONTEXT *psServerMMUContext = hPrivData;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psServerMMUContext->psDevInfo;
+
+#if defined(PDUMP)
+	{
+		RGXFWIF_DEV_VIRTADDR sFWAddr;
+
+		RGXSetFirmwareAddress(&sFWAddr,
+		                      psServerMMUContext->psFWMemContextMemDesc,
+		                      0,
+		                      RFW_FWADDR_NOREF_FLAG);
+
+		/*
+		 * MMU cache commands (always dumped) might have a pointer to this FW
+		 * memory context, wait until the FW has caught-up to the latest command.
+		 */
+		PDUMPCOMMENT(psDevInfo->psDeviceNode,
+		             "Ensure FW has executed all MMU invalidations on FW memory "
+		             "context 0x%x before freeing it", sFWAddr.ui32Addr);
+		SyncPrimPDumpPol(psDevInfo->psDeviceNode->psMMUCacheSyncPrim,
+		                 psDevInfo->psDeviceNode->ui32NextMMUInvalidateUpdate - 1,
+		                 0xFFFFFFFF,
+		                 PDUMP_POLL_OPERATOR_GREATEREQUAL,
+		                 PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+	OSWRLockAcquireWrite(psDevInfo->hMemoryCtxListLock);
+	dllist_remove_node(&psServerMMUContext->sNode);
+	OSWRLockReleaseWrite(psDevInfo->hMemoryCtxListLock);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED)
+	{
+		_RecordUnregisteredMemoryContext(psDevInfo, psServerMMUContext);
+	}
+
+	/*
+	 * Release the page catalogue address acquired in RGXRegisterMemoryContext().
+	 */
+	MMU_ReleaseBaseAddr(NULL);
+
+	/*
+	 * Free the firmware memory context.
+	 */
+	PDUMPCOMMENT(psDevInfo->psDeviceNode, "Free FW memory context");
+	DevmemFwUnmapAndFree(psDevInfo, psServerMMUContext->psFWMemContextMemDesc);
+
+	OSFreeMem(psServerMMUContext);
+}
+
+/*
+ * RGXRegisterMemoryContext
+ */
+PVRSRV_ERROR RGXRegisterMemoryContext(PVRSRV_DEVICE_NODE	*psDeviceNode,
+									  MMU_CONTEXT			*psMMUContext,
+									  IMG_HANDLE			*hPrivData)
+{
+	PVRSRV_ERROR			eError;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_FWMEMCONTEXT	*psFWMemContext;
+	DEVMEM_MEMDESC			*psFWMemContextMemDesc;
+	SERVER_MMU_CONTEXT *psServerMMUContext;
+
+	if (psDevInfo->psKernelMMUCtx == NULL)
+	{
+		/*
+		 * This must be the creation of the Kernel memory context. Take a copy
+		 * of the MMU context for use when programming the BIF.
+		 */
+		psDevInfo->psKernelMMUCtx = psMMUContext;
+	}
+	else
+	{
+		psServerMMUContext = OSAllocMem(sizeof(*psServerMMUContext));
+		if (psServerMMUContext == NULL)
+		{
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto fail_alloc_server_ctx;
+		}
+
+		psServerMMUContext->psDevInfo = psDevInfo;
+		psServerMMUContext->ui64FBSCEntryMask = 0;
+		psServerMMUContext->sFWMemContextDevVirtAddr.ui32Addr = 0;
+
+		/*
+			Allocate device memory for the firmware memory context for the new
+			application.
+		*/
+		PDUMPCOMMENT(psDevInfo->psDeviceNode, "Allocate RGX firmware memory context");
+		eError = DevmemFwAllocate(psDevInfo,
+								sizeof(*psFWMemContext),
+								RGX_FWCOMCTX_ALLOCFLAGS,
+								"FwMemoryContext",
+								&psFWMemContextMemDesc);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to allocate firmware memory context (%u)",
+			         __func__,
+			         eError));
+			goto fail_alloc_fw_ctx;
+		}
+
+		/*
+			Temporarily map the firmware memory context to the kernel.
+		*/
+		eError = DevmemAcquireCpuVirtAddr(psFWMemContextMemDesc,
+										  (void **)&psFWMemContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map firmware memory context (%u)",
+			         __func__,
+			         eError));
+			goto fail_acquire_cpu_addr;
+		}
+
+		/*
+		 * Write the new memory context's page catalogue into the firmware memory
+		 * context for the client.
+		 */
+		eError = MMU_AcquireBaseAddr(psMMUContext, &psFWMemContext->sPCDevPAddr);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to acquire Page Catalogue address (%u)",
+			         __func__,
+			         eError));
+			DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+			goto fail_acquire_base_addr;
+		}
+
+		/*
+		 * Set default values for the rest of the structure.
+		 */
+		psFWMemContext->uiPageCatBaseRegSet = RGXFW_BIF_INVALID_PCSET;
+		psFWMemContext->uiBreakpointAddr = 0;
+		psFWMemContext->uiBPHandlerAddr = 0;
+		psFWMemContext->uiBreakpointCtl = 0;
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+{
+		IMG_UINT32 ui32OSid = 0, ui32OSidReg = 0;
+		IMG_BOOL   bOSidAxiProt;
+
+		MMU_GetOSids(psMMUContext, &ui32OSid, &ui32OSidReg, &bOSidAxiProt);
+
+		psFWMemContext->ui32OSid     = ui32OSidReg;
+		psFWMemContext->bOSidAxiProt = bOSidAxiProt;
+}
+#endif
+
+#if defined(PDUMP)
+		{
+			IMG_CHAR			aszName[PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH];
+			IMG_DEVMEM_OFFSET_T uiOffset = 0;
+
+			/*
+			 * Dump the Mem context allocation
+			 */
+			DevmemPDumpLoadMem(psFWMemContextMemDesc, 0, sizeof(*psFWMemContext), PDUMP_FLAGS_CONTINUOUS);
+
+
+			/*
+			 * Obtain a symbolic addr of the mem context structure
+			 */
+			eError = DevmemPDumpPageCatBaseToSAddr(psFWMemContextMemDesc,
+												   &uiOffset,
+												   aszName,
+												   PHYSMEM_PDUMP_MEMSPNAME_SYMB_ADDR_MAX_LENGTH);
+
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Failed to generate a Dump Page Catalogue address (%u)",
+				         __func__,
+				         eError));
+				DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+				goto fail_pdump_cat_base_addr;
+			}
+
+			/*
+			 * Dump the Page Cat tag in the mem context (symbolic address)
+			 */
+			eError = MMU_PDumpWritePageCatBase(psMMUContext,
+												aszName,
+												uiOffset,
+												8, /* 64-bit register write */
+												0,
+												0,
+												0);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+				         "%s: Failed to acquire Page Catalogue address (%u)",
+				         __func__,
+				         eError));
+				DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+				goto fail_pdump_cat_base;
+			}
+		}
+#endif
+
+		/*
+		 * Release kernel address acquired above.
+		 */
+		RGXFwSharedMemCacheOpPtr(psFWMemContext, FLUSH);
+		DevmemReleaseCpuVirtAddr(psFWMemContextMemDesc);
+
+		/*
+		 * Store the process information for this device memory context
+		 * for use with the host page-fault analysis.
+		 */
+		psServerMMUContext->uiPID = OSGetCurrentClientProcessIDKM();
+		psServerMMUContext->psMMUContext = psMMUContext;
+		psServerMMUContext->psFWMemContextMemDesc = psFWMemContextMemDesc;
+		OSStringSafeCopy(psServerMMUContext->szProcessName,
+		              OSGetCurrentClientProcessNameKM(),
+		              sizeof(psServerMMUContext->szProcessName));
+
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "New memory context: Process Name: %s PID: %u (0x%08X)",
+		                      psServerMMUContext->szProcessName,
+		                      psServerMMUContext->uiPID,
+		                      psServerMMUContext->uiPID);
+
+		OSWRLockAcquireWrite(psDevInfo->hMemoryCtxListLock);
+		dllist_add_to_tail(&psDevInfo->sMemoryContextList, &psServerMMUContext->sNode);
+		OSWRLockReleaseWrite(psDevInfo->hMemoryCtxListLock);
+
+		*hPrivData = psServerMMUContext;
+	}
+
+	return PVRSRV_OK;
+
+#if defined(PDUMP)
+fail_pdump_cat_base:
+fail_pdump_cat_base_addr:
+	MMU_ReleaseBaseAddr(NULL);
+#endif
+fail_acquire_base_addr:
+	/* Done before jumping to the fail point as the release is done before exit */
+fail_acquire_cpu_addr:
+	DevmemFwUnmapAndFree(psDevInfo, psServerMMUContext->psFWMemContextMemDesc);
+fail_alloc_fw_ctx:
+	OSFreeMem(psServerMMUContext);
+fail_alloc_server_ctx:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+DEVMEM_MEMDESC *RGXGetFWMemDescFromMemoryContextHandle(IMG_HANDLE hPriv)
+{
+	SERVER_MMU_CONTEXT *psMMUContext = (SERVER_MMU_CONTEXT *) hPriv;
+
+	return psMMUContext->psFWMemContextMemDesc;
+}
+
+void RGXSetFWMemContextDevVirtAddr(SERVER_MMU_CONTEXT *psServerMMUContext,
+						RGXFWIF_DEV_VIRTADDR	sFWMemContextAddr)
+{
+	psServerMMUContext->sFWMemContextDevVirtAddr.ui32Addr = sFWMemContextAddr.ui32Addr;
+}
+
+void RGXCheckFaultAddress(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_DEV_VIRTADDR *psDevVAddr,
+				IMG_DEV_PHYADDR *psDevPAddr,
+				MMU_FAULT_DATA *psOutFaultData)
+{
+	IMG_DEV_PHYADDR sPCDevPAddr;
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hMemoryCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psServerMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+
+		if (MMU_AcquireBaseAddr(psServerMMUContext->psMMUContext, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for memory context"));
+			continue;
+		}
+
+		if (psDevPAddr->uiAddr == sPCDevPAddr.uiAddr)
+		{
+			MMU_CheckFaultAddress(psServerMMUContext->psMMUContext, psDevVAddr, psOutFaultData);
+			goto out_unlock;
+		}
+	}
+
+	/* Lastly check for fault in the kernel allocated memory */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		if (MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for kernel memory context"));
+		}
+
+		if (psDevPAddr->uiAddr == sPCDevPAddr.uiAddr)
+		{
+			MMU_CheckFaultAddress(psDevInfo->psKernelMMUCtx, psDevVAddr, psOutFaultData);
+		}
+	}
+
+out_unlock:
+	OSWRLockReleaseRead(psDevInfo->hMemoryCtxListLock);
+}
+
+/* given the physical address of a page catalogue, searches for a corresponding
+ * MMU context and if found, provides the caller details of the process.
+ * Returns IMG_TRUE if a process is found.
+ */
+IMG_BOOL RGXPCAddrToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_PHYADDR sPCAddress,
+								RGXMEM_PROCESS_INFO *psInfo)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+
+	/* check if the input PC addr corresponds to an active memory context */
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psThisMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+		IMG_DEV_PHYADDR sPCDevPAddr;
+
+		if (MMU_AcquireBaseAddr(psThisMMUContext->psMMUContext, &sPCDevPAddr) != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for memory context"));
+			continue;
+		}
+
+		if (sPCAddress.uiAddr == sPCDevPAddr.uiAddr)
+		{
+			psServerMMUContext = psThisMMUContext;
+			break;
+		}
+	}
+
+	if (psServerMMUContext != NULL)
+	{
+		psInfo->uiPID = psServerMMUContext->uiPID;
+		OSStringSafeCopy(psInfo->szProcessName, psServerMMUContext->szProcessName, sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+	/* else check if the input PC addr corresponds to the firmware */
+	else if (!PVRSRV_VZ_MODE_IS(GUEST, DEVINFO, psDevInfo))
+	{
+		IMG_DEV_PHYADDR sKernelPCDevPAddr;
+		PVRSRV_ERROR eError;
+
+		eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sKernelPCDevPAddr);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG(("Failed to get PC address for kernel memory context"));
+		}
+		else
+		{
+			if (sPCAddress.uiAddr == sKernelPCDevPAddr.uiAddr)
+			{
+				psInfo->uiPID = RGXMEM_SERVER_PID_FIRMWARE;
+				OSStringSafeCopy(psInfo->szProcessName, "Firmware", sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_FALSE;
+				bRet = IMG_TRUE;
+			}
+		}
+	}
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(bRet == IMG_FALSE))
+	{
+		/* no active memory context found with the given PC address.
+		 * Check the list of most recently freed memory contexts.
+		 */
+		const IMG_UINT32 ui32Mask = UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1;
+		IMG_UINT32 i, j;
+
+		OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+		/* iterate through the list of unregistered memory contexts
+		 * from newest (one before the head) to the oldest (the current head)
+		 */
+		for (i = (gui32UnregisteredMemCtxsHead - 1) & ui32Mask, j = 0;
+		     j < UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE;
+		     i = (i - 1) & ui32Mask, j++)
+		{
+			UNREGISTERED_MEMORY_CONTEXT *psRecord = &gasUnregisteredMemCtxs[i];
+
+			if (psRecord->sPCDevPAddr.uiAddr == sPCAddress.uiAddr)
+			{
+				psInfo->uiPID = psRecord->uiPID;
+				OSStringSafeCopy(psInfo->szProcessName, psRecord->szProcessName, sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_TRUE;
+				bRet = IMG_TRUE;
+				break;
+			}
+		}
+
+		OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	return bRet;
+}
+
+IMG_BOOL RGXPCPIDToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_PID uiPID,
+								RGXMEM_PROCESS_INFO *psInfo)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	DLLIST_NODE *psNode, *psNext;
+	SERVER_MMU_CONTEXT *psServerMMUContext = NULL;
+
+	/* check if the input PID corresponds to an active memory context */
+	dllist_foreach_node(&psDevInfo->sMemoryContextList, psNode, psNext)
+	{
+		SERVER_MMU_CONTEXT *psThisMMUContext =
+			IMG_CONTAINER_OF(psNode, SERVER_MMU_CONTEXT, sNode);
+
+		if (psThisMMUContext->uiPID == uiPID)
+		{
+			psServerMMUContext = psThisMMUContext;
+			break;
+		}
+	}
+
+	if (psServerMMUContext != NULL)
+	{
+		psInfo->uiPID = psServerMMUContext->uiPID;
+		OSStringSafeCopy(psInfo->szProcessName, psServerMMUContext->szProcessName, sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+	/* else check if the input PID corresponds to the firmware */
+	else if (uiPID == RGXMEM_SERVER_PID_FIRMWARE)
+	{
+		psInfo->uiPID = RGXMEM_SERVER_PID_FIRMWARE;
+		OSStringSafeCopy(psInfo->szProcessName, "Firmware", sizeof(psInfo->szProcessName));
+		psInfo->bUnregistered = IMG_FALSE;
+		bRet = IMG_TRUE;
+	}
+
+	if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_PAGE_FAULT_DEBUG_ENABLED) &&
+		(bRet == IMG_FALSE))
+	{
+		/* if the PID didn't correspond to an active context or the
+		 * FW address then see if it matches a recently unregistered context
+		 */
+		const IMG_UINT32 ui32Mask = UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE - 1;
+		IMG_UINT32 i, j;
+
+		OSLockAcquire(psDevInfo->hMMUCtxUnregLock);
+
+		for (i = (gui32UnregisteredMemCtxsHead - 1) & ui32Mask, j = 0;
+		     j < UNREGISTERED_MEMORY_CONTEXTS_HISTORY_SIZE;
+		     i = (i - 1) & ui32Mask, j++)
+		{
+			UNREGISTERED_MEMORY_CONTEXT *psRecord = &gasUnregisteredMemCtxs[i];
+
+			if (psRecord->uiPID == uiPID)
+			{
+				psInfo->uiPID = psRecord->uiPID;
+				OSStringSafeCopy(psInfo->szProcessName, psRecord->szProcessName, sizeof(psInfo->szProcessName));
+				psInfo->bUnregistered = IMG_TRUE;
+				bRet = IMG_TRUE;
+				break;
+			}
+		}
+
+		OSLockRelease(psDevInfo->hMMUCtxUnregLock);
+	}
+
+	return bRet;
+}
+
+IMG_PID RGXGetPIDFromServerMMUContext(SERVER_MMU_CONTEXT *psServerMMUContext)
+{
+	if (psServerMMUContext)
+	{
+		return psServerMMUContext->uiPID;
+	}
+	return 0;
+}
+
+/******************************************************************************
+ End of file (rgxmem.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxmem.h b/drivers/gpu/drm/img-rogue/rgxmem.h
new file mode 100644
index 000000000000..2e959fc79eda
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmem.h
@@ -0,0 +1,157 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX memory context management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for RGX memory context management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXMEM_H)
+#define RGXMEM_H
+
+#include "pvrsrv_error.h"
+#include "device.h"
+#include "mmu_common.h"
+#include "rgxdevice.h"
+
+#define RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME 16
+
+/* this PID denotes the firmware */
+#define RGXMEM_SERVER_PID_FIRMWARE 0xFFFFFFFF
+
+/* this PID denotes the PM */
+#define RGXMEM_SERVER_PID_PM 0xEFFFFFFF
+
+typedef struct _RGXMEM_PROCESS_INFO_
+{
+	IMG_PID uiPID;
+	IMG_CHAR szProcessName[RGXMEM_SERVER_MMU_CONTEXT_MAX_NAME];
+	IMG_BOOL bUnregistered;
+} RGXMEM_PROCESS_INFO;
+
+typedef struct SERVER_MMU_CONTEXT_TAG SERVER_MMU_CONTEXT;
+
+IMG_DEV_PHYADDR GetPC(MMU_CONTEXT * psContext);
+
+void RGXSetFWMemContextDevVirtAddr(SERVER_MMU_CONTEXT *psServerMMUContext,
+			RGXFWIF_DEV_VIRTADDR	sFWMemContextAddr);
+
+void RGXMMUSyncPrimAlloc(PVRSRV_DEVICE_NODE *psDevNode);
+void RGXMMUSyncPrimFree(void);
+
+PVRSRV_ERROR RGXInvalidateFBSCTable(PVRSRV_DEVICE_NODE *psDeviceNode,
+									MMU_CONTEXT *psMMUContext,
+									IMG_UINT64 ui64FBSCEntryMask);
+
+PVRSRV_ERROR RGXExtractFBSCEntryMaskFromMMUContext(PVRSRV_DEVICE_NODE *psDeviceNode,
+												   SERVER_MMU_CONTEXT *psServerMMUContext,
+												   IMG_UINT64 *pui64FBSCEntryMask);
+
+void RGXMMUCacheInvalidate(PVRSRV_DEVICE_NODE *psDevNode,
+						   MMU_CONTEXT *psMMUContext,
+						   MMU_LEVEL eMMULevel,
+						   IMG_BOOL bUnmap);
+
+/*************************************************************************/ /*!
+@Function       RGXMMUCacheInvalidateKick
+
+@Description    Sends a flush command to a particular DM but first takes
+                the power lock.
+
+@Input          psDevNode   Device Node pointer
+@Input          pui32NextMMUInvalidateUpdate
+
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXMMUCacheInvalidateKick(PVRSRV_DEVICE_NODE *psDevNode,
+                                       IMG_UINT32 *pui32NextMMUInvalidateUpdate);
+
+/*************************************************************************/ /*!
+@Function       RGXPreKickCacheCommand
+
+@Description    Sends a cache flush command to a particular DM without
+                honouring the power lock. It's the caller's responsibility
+                to ensure power lock is held before calling this function.
+
+@Input          psDevInfo   Device Info
+@Input          eDM			To which DM the cmd is sent.
+@Input          pui32MMUInvalidateUpdate
+
+@Return			PVRSRV_ERROR
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPreKickCacheCommand(PVRSRV_RGXDEV_INFO *psDevInfo,
+									RGXFWIF_DM eDM,
+									IMG_UINT32 *pui32MMUInvalidateUpdate);
+
+/* Needed for Volcanic architectures with BRN71422 */
+#if defined(RGX_BRN71422_TARGET_HARDWARE_PHYSICAL_ADDR)
+void RGXMapBRN71422TargetPhysicalAddress(struct _CONNECTION_DATA_ *psConnection,
+		                                 struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                                 IMG_DEV_PHYADDR sPhysAddrL1Px,
+		                                 void *pxL1PxCpuVAddr);
+#endif
+
+/* Needed for Rogue architecture where a MIPS FW CPU is used */
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+void RGXMMUTweakProtFlags(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+		                  MMU_DEVICEATTRIBS *psDevAttrs,
+		                  PVRSRV_MEMALLOCFLAGS_T uiMappingFlags,
+		                  MMU_PROTFLAGS_T *uiMMUProtFlags);
+#endif
+
+void RGXUnregisterMemoryContext(IMG_HANDLE hPrivData);
+PVRSRV_ERROR RGXRegisterMemoryContext(PVRSRV_DEVICE_NODE	*psDevNode,
+									  MMU_CONTEXT			*psMMUContext,
+									  IMG_HANDLE			*hPrivData);
+
+DEVMEM_MEMDESC *RGXGetFWMemDescFromMemoryContextHandle(IMG_HANDLE hPriv);
+
+void RGXCheckFaultAddress(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_DEV_VIRTADDR *psDevVAddr,
+				IMG_DEV_PHYADDR *psDevPAddr,
+				MMU_FAULT_DATA *psOutFaultData);
+
+IMG_BOOL RGXPCAddrToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_DEV_PHYADDR sPCAddress,
+								RGXMEM_PROCESS_INFO *psInfo);
+
+IMG_BOOL RGXPCPIDToProcessInfo(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_PID uiPID,
+                               RGXMEM_PROCESS_INFO *psInfo);
+
+IMG_PID RGXGetPIDFromServerMMUContext(SERVER_MMU_CONTEXT *psServerMMUContext);
+
+#endif /* RGXMEM_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxmipsmmuinit.c b/drivers/gpu/drm/img-rogue/rgxmipsmmuinit.c
new file mode 100644
index 000000000000..e0b17bf3dedd
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmipsmmuinit.c
@@ -0,0 +1,1068 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxmipsmmuinit.h"
+
+#include "device.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "mmu_common.h"
+#include "pdump_mmu.h"
+#include "rgxheapconfig.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "rgx_memallocflags.h"
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "log2.h"
+
+/*
+ * Bits of PT, PD and PC not involving addresses
+ */
+
+/* Currently there is no page directory for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PDE_PROTMASK        0
+/* Currently there is no page catalog for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PCE_PROTMASK	     0
+
+
+static MMU_PxE_CONFIG sRGXMMUPCEConfig;
+static MMU_DEVVADDR_CONFIG sRGXMMUTopLevelDevVAddrConfig;
+
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_4KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_4KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_4KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig4KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_16KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_16KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_16KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig16KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_64KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_64KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_64KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig64KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_256KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_256KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig256KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 1MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_1MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_1MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_1MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig1MB;
+
+
+/*
+ *
+ *  Configuration for heaps with 2MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_2MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_2MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_2MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig2MB;
+
+
+/* Forward declaration of protection bits derivation functions, for
+   the following structure */
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags);
+
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+										   const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+										   const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+										   const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+										   IMG_HANDLE *phPriv);
+
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv);
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+
+static MMU_DEVICEATTRIBS sRGXMMUDeviceAttributes;
+
+/* Cached policy */
+static IMG_UINT32 gui32CachedPolicy;
+
+static PVRSRV_ERROR RGXCheckTrampolineAddrs(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                            MMU_DEVICEATTRIBS *psDevAttrs,
+                                            IMG_UINT64 *pui64Addr);
+
+PVRSRV_ERROR RGXMipsMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_BOOL bPhysBusAbove32Bit = 0;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PHYS_BUS_WIDTH))
+	{
+		bPhysBusAbove32Bit = RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32;
+	}
+
+	sRGXMMUDeviceAttributes.pszMMUPxPDumpMemSpaceName =
+		PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_FW_MAIN]);
+
+	/*
+	 * Setup sRGXMMUPCEConfig, no PC in MIPS MMU currently
+	 */
+	sRGXMMUPCEConfig.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPCEConfig.pszPxLevelStr = "UnD";
+	sRGXMMUPCEConfig.uiBytesPerEntry = 0; /* 32 bit entries */
+
+	sRGXMMUPCEConfig.uiAddrMask = 0; /* Mask to get significant address bits of PC entry */
+	sRGXMMUPCEConfig.uiAddrShift = 0; /* Shift this many bits to get PD address in PC entry */
+	sRGXMMUPCEConfig.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K; /* Alignment of PD AND PC */
+
+	sRGXMMUPCEConfig.uiProtMask = RGX_MIPS_MMUCTRL_PCE_PROTMASK; /* Mask to get the status bits of the PC */
+	sRGXMMUPCEConfig.uiProtShift = 0; /* Shift this many bits to have status bits starting with bit 0 */
+
+	sRGXMMUPCEConfig.uiValidEnMask = RGX_MIPS_MMUCTRL_PC_DATA_VALID_EN; /* Mask to get entry valid bit of the PC */
+	sRGXMMUPCEConfig.uiValidEnShift = RGX_MIPS_MMUCTRL_PC_DATA_VALID_SHIFT; /* Shift this many bits to have entry valid bit starting with bit 0 */
+
+	/*
+	 *  Setup sRGXMMUTopLevelDevVAddrConfig
+	 */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask = 0; /* Get the PC address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift = 0;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPC = 0;
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask = 0; /* Get the PD address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift = 0;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPD = 0;
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPTIndexMask = IMG_UINT64_C(0xfffffff000); /* Get the PT address bits from a 40 bit virt. address (in a 64bit UINT) */
+	sRGXMMUTopLevelDevVAddrConfig.uiPTIndexShift = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_4KBDP. No PD in MIPS MMU currently
+	 */
+	sRGXMMUPDEConfig_4KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_4KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_4KBDP.uiBytesPerEntry = 0;
+
+	/* No PD used for MIPS */
+	sRGXMMUPDEConfig_4KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_4KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_4KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlMask = IMG_UINT64_C(0x0);
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiProtMask = RGX_MIPS_MMUCTRL_PDE_PROTMASK;
+	sRGXMMUPDEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiValidEnMask = RGX_MIPS_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_4KBDP.uiValidEnShift = RGX_MIPS_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_4KBDP.
+	 */
+	sRGXMMUPTEConfig_4KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_4KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+
+	if (bPhysBusAbove32Bit)
+	{
+		sRGXMMUPTEConfig_4KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT;
+	}
+	else
+	{
+		sRGXMMUPTEConfig_4KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY;
+	}
+
+	sRGXMMUPTEConfig_4KBDP.uiAddrShift = RGXMIPSFW_ENTRYLO_PFN_SHIFT;
+	sRGXMMUPTEConfig_4KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPTEConfig_4KBDP.uiProtMask = RGXMIPSFW_ENTRYLO_DVG | ~RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK |
+	                                    RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN | RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN;
+	sRGXMMUPTEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_4KBDP.uiValidEnMask = RGXMIPSFW_ENTRYLO_VALID_EN;
+	sRGXMMUPTEConfig_4KBDP.uiValidEnShift = RGXMIPSFW_ENTRYLO_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_4KBDP
+	 */
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPC = 0;
+
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask = ~RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift = RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000000fff);
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiOffsetInBytes = RGX_FIRMWARE_RAW_HEAP_BASE & IMG_UINT64_C(0x00ffffffff);
+
+	/*
+	 * Setup gsPageSizeConfig4KB
+	 */
+	gsPageSizeConfig4KB.psPDEConfig = &sRGXMMUPDEConfig_4KBDP;
+	gsPageSizeConfig4KB.psPTEConfig = &sRGXMMUPTEConfig_4KBDP;
+	gsPageSizeConfig4KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_4KBDP;
+	gsPageSizeConfig4KB.uiRefCount = 0;
+	gsPageSizeConfig4KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_16KBDP
+	 */
+	sRGXMMUPDEConfig_16KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_16KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_16KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiAddrShift = 0; /* These are for a page directory ENTRY, meaning the address of a PT cropped to suit the PD */
+	sRGXMMUPDEConfig_16KBDP.uiAddrLog2Align = 0; /* Alignment of the page tables NOT directories */
+
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_16KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_16KBDP. Not supported yet
+	 */
+	sRGXMMUPTEConfig_16KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_16KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiAddrShift = 0; /* These are for a page table ENTRY, meaning the address of a PAGE cropped to suit the PD */
+	sRGXMMUPTEConfig_16KBDP.uiAddrLog2Align = 0; /* Alignment of the pages NOT tables */
+
+	sRGXMMUPTEConfig_16KBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_16KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_16KBDP
+	 */
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig16KB
+	 */
+	gsPageSizeConfig16KB.psPDEConfig = &sRGXMMUPDEConfig_16KBDP;
+	gsPageSizeConfig16KB.psPTEConfig = &sRGXMMUPTEConfig_16KBDP;
+	gsPageSizeConfig16KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_16KBDP;
+	gsPageSizeConfig16KB.uiRefCount = 0;
+	gsPageSizeConfig16KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size. Not supported yet
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_64KBDP
+	 */
+	sRGXMMUPDEConfig_64KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_64KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_64KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_64KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_64KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_64KBDP.
+	 *
+	 */
+	sRGXMMUPTEConfig_64KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_64KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+	if (bPhysBusAbove32Bit)
+	{
+		sRGXMMUPTEConfig_64KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK_ABOVE_32BIT;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY_ABOVE_32BIT;
+	}
+	else
+	{
+		sRGXMMUPTEConfig_64KBDP.uiAddrMask = RGXMIPSFW_ENTRYLO_PFN_MASK;
+		gui32CachedPolicy = RGXMIPSFW_CACHED_POLICY;
+	}
+
+	/* Even while using 64K pages, MIPS still aligns addresses to 4K */
+	sRGXMMUPTEConfig_64KBDP.uiAddrShift = RGXMIPSFW_ENTRYLO_PFN_SHIFT;
+	sRGXMMUPTEConfig_64KBDP.uiAddrLog2Align = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_4K;
+
+	sRGXMMUPTEConfig_64KBDP.uiProtMask = RGXMIPSFW_ENTRYLO_DVG | ~RGXMIPSFW_ENTRYLO_CACHE_POLICY_CLRMSK |
+	                                     RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN | RGXMIPSFW_ENTRYLO_EXEC_INHIBIT_EN;
+	sRGXMMUPTEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_64KBDP.uiValidEnMask = RGXMIPSFW_ENTRYLO_VALID_EN;
+	sRGXMMUPTEConfig_64KBDP.uiValidEnShift = RGXMIPSFW_ENTRYLO_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_64KBDP.
+	 */
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask = IMG_UINT64_C(0x00ffff0000);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift = (IMG_UINT32)RGXMIPSFW_LOG2_PAGE_SIZE_64K;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT = (RGX_NUM_DRIVERS_SUPPORTED << RGXMIPSFW_LOG2_PAGETABLE_SIZE_64K) >> RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000000ffff);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiOffsetInBytes = RGX_FIRMWARE_RAW_HEAP_BASE & IMG_UINT64_C(0x00ffffffff);
+
+	/*
+	 * Setup gsPageSizeConfig64KB.
+	 */
+	gsPageSizeConfig64KB.psPDEConfig = &sRGXMMUPDEConfig_64KBDP;
+	gsPageSizeConfig64KB.psPTEConfig = &sRGXMMUPTEConfig_64KBDP;
+	gsPageSizeConfig64KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_64KBDP;
+	gsPageSizeConfig64KB.uiRefCount = 0;
+	gsPageSizeConfig64KB.uiMaxRefCount = 0;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size. Not supported yet
+ *
+ */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_256KBDP
+	 */
+	sRGXMMUPDEConfig_256KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_256KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_256KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_256KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_256KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP
+	 */
+	sRGXMMUPTEConfig_256KBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_256KBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_256KBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_256KBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_256KBDP
+	 */
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig256KB
+	 */
+	gsPageSizeConfig256KB.psPDEConfig = &sRGXMMUPDEConfig_256KBDP;
+	gsPageSizeConfig256KB.psPTEConfig = &sRGXMMUPTEConfig_256KBDP;
+	gsPageSizeConfig256KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_256KBDP;
+	gsPageSizeConfig256KB.uiRefCount = 0;
+	gsPageSizeConfig256KB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUPDEConfig_1MBDP.  Not supported yet
+	 */
+	sRGXMMUPDEConfig_1MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_1MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_1MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_1MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_1MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_1MBDP
+	 */
+	sRGXMMUPTEConfig_1MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_1MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_1MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_1MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_1MBDP
+	 */
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig1MB
+	 */
+	gsPageSizeConfig1MB.psPDEConfig = &sRGXMMUPDEConfig_1MBDP;
+	gsPageSizeConfig1MB.psPTEConfig = &sRGXMMUPTEConfig_1MBDP;
+	gsPageSizeConfig1MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_1MBDP;
+	gsPageSizeConfig1MB.uiRefCount = 0;
+	gsPageSizeConfig1MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUPDEConfig_2MBDP. Not supported yet
+	 */
+	sRGXMMUPDEConfig_2MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPDEConfig_2MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPDEConfig_2MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiAddrMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiAddrShift = 0;
+	sRGXMMUPDEConfig_2MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiProtMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiValidEnMask = 0;
+	sRGXMMUPDEConfig_2MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_2MBDP
+	 */
+	sRGXMMUPTEConfig_2MBDP.ePxLevel = MMU_LEVEL_LAST;
+	sRGXMMUPTEConfig_2MBDP.pszPxLevelStr = "UnD";
+	sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiAddrMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiAddrShift = 0;
+	sRGXMMUPTEConfig_2MBDP.uiAddrLog2Align = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiProtMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiValidEnMask = 0;
+	sRGXMMUPTEConfig_2MBDP.uiValidEnShift = 0;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_2MBDP
+	 */
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPC = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPD = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT = 0;
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetMask = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig2MB
+	 */
+	gsPageSizeConfig2MB.psPDEConfig = &sRGXMMUPDEConfig_2MBDP;
+	gsPageSizeConfig2MB.psPTEConfig = &sRGXMMUPTEConfig_2MBDP;
+	gsPageSizeConfig2MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_2MBDP;
+	gsPageSizeConfig2MB.uiRefCount = 0;
+	gsPageSizeConfig2MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUDeviceAttributes
+	 */
+	sRGXMMUDeviceAttributes.eMMUType = PDUMP_MMU_TYPE_MIPS_MICROAPTIV;
+
+	/*
+	 * The page table fits in one or more big physically adjacent pages,
+	 * at most as big as the page table itself.
+	 * To calculate its alignment/page size, calculate the log2 size of the page
+	 * table taking into account all OSes, then round that down to a valid MIPS
+	 * log2 page size (12, 14, 16 for a 4K, 16K, 64K page size).
+	 */
+	sRGXMMUDeviceAttributes.ui32BaseAlign =
+		(CeilLog2(RGX_NUM_DRIVERS_SUPPORTED) + RGXMIPSFW_LOG2_PAGETABLE_SIZE_4K) & ~1U;
+
+	/* 256K alignment might be too hard to achieve, fall back to 64K */
+	sRGXMMUDeviceAttributes.ui32BaseAlign =
+		MIN(sRGXMMUDeviceAttributes.ui32BaseAlign, RGXMIPSFW_LOG2_PAGE_SIZE_64K);
+
+
+
+	/* The base configuration is set to 4kB pages*/
+	sRGXMMUDeviceAttributes.psBaseConfig = &sRGXMMUPTEConfig_4KBDP;
+	sRGXMMUDeviceAttributes.psTopLevelDevVAddrConfig = &sRGXMMUTopLevelDevVAddrConfig;
+
+	/* Functions for deriving page table/dir/cat protection bits */
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt8 = RGXDerivePCEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt4 = RGXDerivePCEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXDerivePDEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt4 = RGXDerivePDEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt8 = RGXDerivePTEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt4 = RGXDerivePTEProt4;
+
+	/* Functions for establishing configurations for PDE/PTE/DEVVADDR
+	   on per-heap basis */
+	sRGXMMUDeviceAttributes.pfnGetPageSizeConfiguration = RGXGetPageSizeConfigCB;
+	sRGXMMUDeviceAttributes.pfnPutPageSizeConfiguration = RGXPutPageSizeConfigCB;
+
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE4 = RGXGetPageSizeFromPDE4;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXGetPageSizeFromPDE8;
+
+	psDeviceNode->psFirmwareMMUDevAttrs = &sRGXMMUDeviceAttributes;
+
+	psDeviceNode->pfnValidateOrTweakPhysAddrs = RGXCheckTrampolineAddrs;
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXCheckTrampolineAddrs(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+                                            MMU_DEVICEATTRIBS *psDevAttrs,
+                                            IMG_UINT64 *pui64Addr)
+{
+	if (PVRSRV_IS_FEATURE_SUPPORTED(psDevNode, MIPS))
+	{
+		/*
+		 * If mapping for the MIPS FW context, check for sensitive PAs
+		 */
+		if (psDevAttrs == psDevNode->psFirmwareMMUDevAttrs)
+		{
+			PVRSRV_RGXDEV_INFO *psDevice = (PVRSRV_RGXDEV_INFO *)psDevNode->pvDevice;
+
+			if ((RGX_GET_FEATURE_VALUE(psDevice, PHYS_BUS_WIDTH) == 32) &&
+				 RGXMIPSFW_SENSITIVE_ADDR(*pui64Addr))
+			{
+				*pui64Addr = psDevice->psTrampoline->sPhysAddr.uiAddr + RGXMIPSFW_TRAMPOLINE_OFFSET(*pui64Addr);
+			}
+			/* FIX_HW_BRN_63553 is mainlined for all MIPS cores */
+			else if (*pui64Addr == 0x0 && !psDevice->sLayerParams.bDevicePA0IsValid)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s attempt to map addr 0x0 in the FW but 0x0 is not considered valid.", __func__));
+				return PVRSRV_ERROR_MMU_FAILED_TO_MAP_PAGE_TABLE;
+			}
+		}
+	}
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXMipsMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	psDeviceNode->pfnMMUGetContextID = NULL;
+#endif
+
+	psDeviceNode->psFirmwareMMUDevAttrs = NULL;
+
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_MESSAGE, "Variable Page Size Heap Stats:"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 4K page heaps: %d",
+			 gsPageSizeConfig4KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 4K page heaps (should be 0): %d",
+			 gsPageSizeConfig4KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 16K page heaps: %d",
+			 gsPageSizeConfig16KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 16K page heaps (should be 0): %d",
+			 gsPageSizeConfig16KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 64K page heaps: %d",
+			 gsPageSizeConfig64KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 64K page heaps (should be 0): %d",
+			 gsPageSizeConfig64KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 256K page heaps: %d",
+			 gsPageSizeConfig256KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 256K page heaps (should be 0): %d",
+			 gsPageSizeConfig256KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 1M page heaps: %d",
+			 gsPageSizeConfig1MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 1M page heaps (should be 0): %d",
+			 gsPageSizeConfig1MB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 2M page heaps: %d",
+			 gsPageSizeConfig2MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 2M page heaps (should be 0): %d",
+			 gsPageSizeConfig2MB.uiRefCount));
+#endif
+	if (gsPageSizeConfig4KB.uiRefCount > 0 ||
+		gsPageSizeConfig16KB.uiRefCount > 0 ||
+		gsPageSizeConfig64KB.uiRefCount > 0 ||
+		gsPageSizeConfig256KB.uiRefCount > 0 ||
+		gsPageSizeConfig1MB.uiRefCount > 0 ||
+		gsPageSizeConfig2MB.uiRefCount > 0
+		)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXMMUInit_Unregister: Unbalanced MMU API Usage (Internal error)"));
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt4
+@Description    calculate the PCE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_DPF((PVR_DBG_ERROR, "Page Catalog not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt8
+@Description    calculate the PCE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "Page Catalog not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt4
+@Description    derive the PDE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "Page Directory not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize The log2 of the required page size.
+				E.g, for 4KiB pages, this parameter must be 12.
+				For 2MiB pages, it must be set to 21.
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "Page Directory not supported on MIPS MMU"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt4
+@Description    calculate the PTE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags)
+{
+	IMG_UINT32 ui32MMUFlags = 0;
+
+	if (((MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE) & uiProtFlags) == (MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE))
+	{
+		/* read/write */
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_DIRTY_EN;
+	}
+	else if (MMU_PROTFLAGS_READABLE & uiProtFlags)
+	{
+		/* read only */
+	}
+	else if (MMU_PROTFLAGS_WRITEABLE & uiProtFlags)
+	{
+		/* write only */
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_READ_INHIBIT_EN;
+	}
+	else if ((MMU_PROTFLAGS_INVALID & uiProtFlags) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt4: neither read nor write specified..."));
+	}
+
+	/* cache coherency */
+	if (MMU_PROTFLAGS_CACHE_COHERENT & uiProtFlags)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt4: cache coherency not supported for MIPS caches"));
+	}
+
+	/* cache setup */
+	if ((MMU_PROTFLAGS_CACHED & uiProtFlags) == 0)
+	{
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_UNCACHED;
+	}
+	else
+	{
+		ui32MMUFlags |= gui32CachedPolicy <<
+		                RGXMIPSFW_ENTRYLO_CACHE_POLICY_SHIFT;
+	}
+
+	if ((uiProtFlags & MMU_PROTFLAGS_INVALID) == 0)
+	{
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_VALID_EN;
+		ui32MMUFlags |= RGXMIPSFW_ENTRYLO_GLOBAL_EN;
+	}
+
+	if (MMU_PROTFLAGS_DEVICE(PMMETA_PROTECT) & uiProtFlags)
+	{
+		/* PVR_DPF((PVR_DBG_WARNING, "RGXDerivePTEProt4: PMMETA Protect not existent for MIPS, option discarded")); */
+	}
+
+	return ui32MMUFlags;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt8
+@Description    calculate the PTE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "8-byte PTE not supported on this device"));
+
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXGetPageSizeConfig
+@Description    Set up configuration for variable sized data pages.
+				RGXPutPageSizeConfigCB has to be called to ensure correct
+				refcounting.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+										   const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+										   const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+										   const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+										   IMG_HANDLE *phPriv)
+{
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGXMIPSFW_LOG2_PAGE_SIZE_64K:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGXMIPSFW_LOG2_PAGE_SIZE_4K:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				 "RGXGetPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				 uiLog2DataPageSize));
+		*phPriv = NULL;
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Refer caller's pointers to the data */
+	*ppsMMUPDEConfig = psPageSizeConfig->psPDEConfig;
+	*ppsMMUPTEConfig = psPageSizeConfig->psPTEConfig;
+	*ppsMMUDevVAddrConfig = psPageSizeConfig->psDevVAddrConfig;
+
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	/* Increment ref-count - not that we're allocating anything here
+	   (I'm using static structs), but one day we might, so we want
+	   the Get/Put code to be balanced properly */
+	psPageSizeConfig->uiRefCount++;
+
+	/* This is purely for debug statistics */
+	psPageSizeConfig->uiMaxRefCount = MAX(psPageSizeConfig->uiMaxRefCount,
+										  psPageSizeConfig->uiRefCount);
+#endif
+
+	*phPriv = (IMG_HANDLE)(uintptr_t)uiLog2DataPageSize;
+	PVR_ASSERT (uiLog2DataPageSize == (IMG_UINT32)(uintptr_t)*phPriv);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPutPageSizeConfig
+@Description    Tells this code that the mmu module is done with the
+				configurations set in RGXGetPageSizeConfig.  This can
+				be a no-op.
+				Called after RGXGetPageSizeConfigCB.
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv)
+{
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+	IMG_UINT32 uiLog2DataPageSize;
+
+	uiLog2DataPageSize = (IMG_UINT32)(uintptr_t) hPriv;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGXMIPSFW_LOG2_PAGE_SIZE_64K:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGXMIPSFW_LOG2_PAGE_SIZE_4K:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				 "RGXPutPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				 uiLog2DataPageSize));
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Ref-count here is not especially useful, but it's an extra
+	   check that the API is being used correctly */
+	psPageSizeConfig->uiRefCount--;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "PDE not supported on MIPS"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "PDE not supported on MIPS"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+void RGXMipsCheckFaultAddress(MMU_CONTEXT *psFwMMUCtx,
+                              IMG_UINT32 ui32FwVA,
+                              MMU_FAULT_DATA *psOutFaultData)
+{
+	IMG_UINT32 *pui32PageTable = NULL;
+	PVRSRV_ERROR eError = MMU_AcquireCPUBaseAddr(psFwMMUCtx, (void**) &pui32PageTable);
+	MMU_LEVEL_DATA *psMMULevelData;
+	IMG_UINT32 ui32FwHeapBase = (IMG_UINT32) (RGX_FIRMWARE_RAW_HEAP_BASE & UINT_MAX);
+	IMG_UINT32 ui32PageSize = OSGetPageSize();
+
+	/* MIPS Firmware CPU must use the same page size as the Host */
+	IMG_UINT32 ui32PTEIndex = ((ui32FwVA & ~(ui32PageSize - 1)) - ui32FwHeapBase) / ui32PageSize;
+
+	psOutFaultData->eTopLevel = MMU_LEVEL_1;
+	psOutFaultData->eType = MMU_FAULT_TYPE_NON_PM;
+
+	psMMULevelData = &psOutFaultData->sLevelData[MMU_LEVEL_1];
+	psMMULevelData->uiBytesPerEntry = 1 << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	psMMULevelData->ui32Index = ui32PTEIndex;
+	psMMULevelData->ui32NumOfEntries = RGX_FIRMWARE_RAW_HEAP_SIZE / ui32PageSize;
+
+	if ((eError == PVRSRV_OK) && (pui32PageTable != NULL))
+	{
+		psMMULevelData->ui64Address = pui32PageTable[ui32PTEIndex];
+	}
+	else
+	{
+		psMMULevelData->ui64Address = 0U;
+	}
+
+	psMMULevelData->psDebugStr = BITMASK_HAS(psMMULevelData->ui64Address,
+	                                         RGXMIPSFW_TLB_VALID) ?
+	                             ("valid") : ("not valid");
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxmipsmmuinit.h b/drivers/gpu/drm/img-rogue/rgxmipsmmuinit.h
new file mode 100644
index 000000000000..b2b39402cafe
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmipsmmuinit.h
@@ -0,0 +1,97 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation for the MIPS firmware
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* NB: this file is not to be included arbitrarily.  It exists solely
+   for the linkage between rgxinit.c and rgxmmuinit.c, the former
+   being otherwise cluttered by the contents of the latter */
+
+#ifndef SRVKM_RGXMIPSMMUINIT_H
+#define SRVKM_RGXMIPSMMUINIT_H
+
+#include "device.h"
+#include "img_types.h"
+#include "mmu_common.h"
+#include "img_defs.h"
+#include "rgx_mips.h"
+
+/*
+
+		Labelling of fields within virtual address. No PD and PC are used currently for
+		the MIPS MMU
+*/
+/*
+Page Table entry #
+*/
+#define RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_SHIFT        (12U)
+#define RGX_MIPS_MMUCTRL_VADDR_PT_INDEX_CLRMSK       (IMG_UINT64_C(0XFFFFFFFF00000FFF))
+
+
+/* PC entries related definitions */
+/* No PC is currently used for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_EN            (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_SHIFT         (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_VALID_CLRMSK        (0U)
+
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_SHIFT     (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_CLRMSK    (0U)
+#define RGX_MIPS_MMUCTRL_PC_DATA_READ_ONLY_EN        (0U)
+
+/* PD entries related definitions */
+/* No PD is currently used for MIPS MMU */
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_EN            (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_SHIFT         (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_VALID_CLRMSK        (0U)
+
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_SHIFT     (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_CLRMSK    (0U)
+#define RGX_MIPS_MMUCTRL_PD_DATA_READ_ONLY_EN        (0U)
+
+
+PVRSRV_ERROR RGXMipsMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR RGXMipsMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+void RGXMipsCheckFaultAddress(MMU_CONTEXT *psFwMMUCtx,
+                              IMG_UINT32 ui32FwVA,
+                              MMU_FAULT_DATA *psOutFaultData);
+
+#endif /* #ifndef SRVKM_RGXMIPSMMUINIT_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxmmuinit.c b/drivers/gpu/drm/img-rogue/rgxmmuinit.c
new file mode 100644
index 000000000000..fdedebe7b033
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmmuinit.c
@@ -0,0 +1,1421 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+#include "rgxmmuinit.h"
+#include "rgxmmudefs_km.h"
+
+#include "rgxdevice.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "mmu_common.h"
+#include "pdump_mmu.h"
+
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "rgx_memallocflags.h"
+#include "rgx_heaps_server.h"
+#include "pdump_km.h"
+#include "allocmem.h"
+
+/* useful macros */
+/* units represented in a bitfield */
+#define UNITS_IN_BITFIELD(Mask, Shift)	((Mask >> Shift) + 1)
+
+/*
+ * Bits of PT, PD and PC not involving addresses
+ */
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/* protection bits for MMU_VERSION <= 3 */
+#define RGX_MMUCTRL_PTE_PROTMASK	(RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN | \
+		~RGX_MMUCTRL_PT_DATA_AXCACHE_CLRMSK | \
+		RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PT_DATA_PM_SRC_EN | \
+		RGX_MMUCTRL_PT_DATA_CC_EN | \
+		RGX_MMUCTRL_PT_DATA_READ_ONLY_EN | \
+		RGX_MMUCTRL_PT_DATA_VALID_EN)
+#else
+#define RGX_MMUCTRL_PTE_PROTMASK	(RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN | \
+		RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PT_DATA_PM_SRC_EN | \
+		RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN | \
+		RGX_MMUCTRL_PT_DATA_CC_EN | \
+		RGX_MMUCTRL_PT_DATA_READ_ONLY_EN | \
+		RGX_MMUCTRL_PT_DATA_VALID_EN)
+#endif
+
+#define RGX_MMUCTRL_PDE_PROTMASK	(RGX_MMUCTRL_PD_DATA_ENTRY_PENDING_EN | \
+		~RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK | \
+		RGX_MMUCTRL_PD_DATA_VALID_EN)
+
+#define RGX_MMUCTRL_PCE_PROTMASK	(RGX_MMUCTRL_PC_DATA_ENTRY_PENDING_EN | \
+		RGX_MMUCTRL_PC_DATA_VALID_EN)
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*
+ * protection bits for MMU_VERSION >= 4
+ * MMU4 has no PENDING or PAGE_SIZE fields in PxE
+ */
+#define RGX_MMU4CTRL_PTE_PROTMASK	(RGX_MMUCTRL_PTE_PROTMASK & ~RGX_MMUCTRL_PT_DATA_ENTRY_PENDING_EN)
+
+#define RGX_MMU4CTRL_PDE_PROTMASK	(RGX_MMUCTRL_PD_DATA_VALID_EN)
+
+#define RGX_MMU4CTRL_PCE_PROTMASK	(RGX_MMUCTRL_PC_DATA_VALID_EN)
+#endif
+
+
+static MMU_PxE_CONFIG sRGXMMUPCEConfig;
+static MMU_DEVVADDR_CONFIG sRGXMMUTopLevelDevVAddrConfig;
+
+
+/*
+ *
+ *  Configuration for heaps with 4kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_4KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_4KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_4KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig4KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 16kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_16KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_16KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_16KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig16KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 64kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_64KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_64KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_64KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig64KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 256kB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_256KBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_256KBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig256KB;
+
+
+/*
+ *
+ *  Configuration for heaps with 1MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_1MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_1MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_1MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig1MB;
+
+
+/*
+ *
+ *  Configuration for heaps with 2MB Data-Page size
+ *
+ */
+
+static MMU_PxE_CONFIG sRGXMMUPDEConfig_2MBDP;
+static MMU_PxE_CONFIG sRGXMMUPTEConfig_2MBDP;
+static MMU_DEVVADDR_CONFIG sRGXMMUDevVAddrConfig_2MBDP;
+static MMU_PAGESIZECONFIG gsPageSizeConfig2MB;
+
+
+/* Forward declaration of protection bits derivation functions, for
+   the following structure */
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags);
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags);
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/* protection bits derivation functions for MMUv4 */
+static IMG_UINT64 RGXMMU4DerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize);
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromVirtAddr(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												   IMG_DEV_VIRTADDR sDevVAddr,
+												   IMG_UINT32 *pui32Log2PageSize);
+#endif
+
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+		const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+		const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+		const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+		IMG_HANDLE *phPriv);
+
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv);
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize);
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize);
+
+static MMU_DEVICEATTRIBS sRGXMMUDeviceAttributes;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+/* This is a basis for a pattern of parity bit values for consecutive VAs.
+   For each PT with 512 entries we'd get either this pattern or its reverse. */
+static const IMG_UINT8 ui8ParityPTPattern[512] = {
+    0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,
+    1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0,
+    0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+    1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
+    0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1,
+    0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1,
+    1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1,
+    1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1,
+    0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1,
+    1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0,
+    0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0,
+    1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0,
+    0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
+    1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
+    0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
+    1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1,
+    1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1};
+#endif
+
+PVRSRV_ERROR RGXMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK) || defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	IMG_BOOL bHaveParity = RGX_IS_FEATURE_SUPPORTED(psDevInfo, MH_PARITY);
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	IMG_BOOL bHaveMMU4 = (RGX_GET_FEATURE_VALUE(psDevInfo, MMU_VERSION) >= 4);
+#define RGX_GET_MMUCTRL_PROTMASK(entry) \
+	(bHaveMMU4 ? RGX_MMU4CTRL_##entry##_PROTMASK : RGX_MMUCTRL_##entry##_PROTMASK)
+#else
+#define RGX_GET_MMUCTRL_PROTMASK(entry) RGX_MMUCTRL_##entry##_PROTMASK
+#endif
+
+	/* Setup of Px Entries:
+	 *
+	 *
+	 * PAGE TABLE (8 Byte):
+	 *
+	 * | 62              | 61...40         | 39...12 (varies) | 11...6          | 5             | 4      | 3               | 2               | 1         | 0     |
+	 * | PM/Meta protect | VP Page (39:18) | Physical Page    | VP Page (17:12) | Entry Pending | PM src | SLC Bypass Ctrl | Cache Coherency | Read Only | Valid |
+	 *
+	 *
+	 * PAGE DIRECTORY (8 Byte):
+	 *
+	 *  | 40            | 39...5  (varies)        | 4          | 3...1     | 0     |
+	 *  | Entry Pending | Page Table base address | (reserved) | Page Size | Valid |
+	 *
+	 *
+	 * PAGE CATALOGUE (4 Byte):
+	 *
+	 *  | 31...4                      | 3...2      | 1             | 0     |
+	 *  | Page Directory base address | (reserved) | Entry Pending | Valid |
+	 *
+	 */
+
+
+	/* Example how to get the PD address from a PC entry.
+	 * The procedure is the same for PD and PT entries to retrieve PT and Page addresses:
+	 *
+	 * 1) sRGXMMUPCEConfig.uiAddrMask applied to PC entry with '&':
+	 *  | 31...4   | 3...2      | 1             | 0     |
+	 *  | PD Addr  | 0          | 0             | 0     |
+	 *
+	 * 2) sRGXMMUPCEConfig.uiAddrShift applied with '>>':
+	 *  | 27...0   |
+	 *  | PD Addr  |
+	 *
+	 * 3) sRGXMMUPCEConfig.uiAddrLog2Align applied with '<<':
+	 *  | 39...0   |
+	 *  | PD Addr  |
+	 *
+	 */
+
+
+	sRGXMMUDeviceAttributes.pszMMUPxPDumpMemSpaceName =
+			PhysHeapPDumpMemspaceName(psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL]);
+
+	/*
+	 * Setup sRGXMMUPCEConfig
+	 */
+	sRGXMMUPCEConfig.ePxLevel = MMU_LEVEL_3;
+	sRGXMMUPCEConfig.pszPxLevelStr = "PC";
+	sRGXMMUPCEConfig.uiBytesPerEntry = 4;     /* 32 bit entries */
+
+	sRGXMMUPCEConfig.uiAddrMask = 0xfffffff0; /* Mask to get significant address bits of PC entry i.e. the address of the PD */
+	sRGXMMUPCEConfig.uiAddrShift = 4;         /* Shift this many bits to get PD address */
+	sRGXMMUPCEConfig.uiAddrLog2Align = 12;    /* Alignment of PD physical addresses. */
+
+	sRGXMMUPCEConfig.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PCE); /* Mask to get the status bits */
+
+	sRGXMMUPCEConfig.uiProtShift = 0;                       /* Shift this many bits to get the status bits */
+
+	sRGXMMUPCEConfig.uiValidEnMask = RGX_MMUCTRL_PC_DATA_VALID_EN;     /* Mask to get entry valid bit of the PC */
+	sRGXMMUPCEConfig.uiValidEnShift = RGX_MMUCTRL_PC_DATA_VALID_SHIFT; /* Shift this many bits to get entry valid bit */
+
+	/*
+	 *  Setup sRGXMMUTopLevelDevVAddrConfig
+	 */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK; /* Mask to get PC index applied to a 40 bit virt. device address */
+	sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;  /* Shift a 40 bit virt. device address by this amount to get the PC index */
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUTopLevelDevVAddrConfig.uiPCIndexMask,
+			sRGXMMUTopLevelDevVAddrConfig.uiPCIndexShift));
+
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK; /* Mask to get PD index applied to a 40 bit virt. device address */
+	sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;  /* Shift a 40 bit virt. device address by this amount to get the PD index */
+	sRGXMMUTopLevelDevVAddrConfig.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUTopLevelDevVAddrConfig.uiPDIndexMask,
+			sRGXMMUTopLevelDevVAddrConfig.uiPDIndexShift));
+
+	/*
+	 *
+	 *  Configuration for heaps with 4kB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20   12   11        0
+	 *      |         \ /       \ /       \ /           \
+	 *      |....PCE...|...PDE...|...PTE...|....VAddr...|
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_4KBDP
+	 */
+	sRGXMMUPDEConfig_4KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_4KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_4KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_4KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_4KBDP.uiAddrShift = 12;
+	sRGXMMUPDEConfig_4KBDP.uiAddrLog2Align = 12;
+
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_4KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_4KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_4KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_4KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_4KBDP
+	 */
+	sRGXMMUPTEConfig_4KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_4KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_4KBDP.uiAddrMask = IMG_UINT64_C(0xfffffff000);
+	sRGXMMUPTEConfig_4KBDP.uiAddrShift = 12;
+	sRGXMMUPTEConfig_4KBDP.uiAddrLog2Align = 12; /* Alignment of the physical addresses of the pages NOT PTs */
+
+	sRGXMMUPTEConfig_4KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_4KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_4KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_4KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_4KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_4KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_4KBDP
+	 */
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPCIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPDIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask = ~RGX_MMUCTRL_VADDR_PT_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift = RGX_MMUCTRL_VADDR_PT_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_4KBDP.uiPTIndexShift));
+
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000000fff);
+	sRGXMMUDevVAddrConfig_4KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_4KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig4KB
+	 */
+	gsPageSizeConfig4KB.psPDEConfig = &sRGXMMUPDEConfig_4KBDP;
+	gsPageSizeConfig4KB.psPTEConfig = &sRGXMMUPTEConfig_4KBDP;
+	gsPageSizeConfig4KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_4KBDP;
+	gsPageSizeConfig4KB.uiRefCount = 0;
+	gsPageSizeConfig4KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 16kB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20 14   13          0
+	 *      |         \ /       \ /     \ /             \
+	 *      |....PCE...|...PDE...|..PTE..|.....VAddr....|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_16KBDP
+	 */
+	sRGXMMUPDEConfig_16KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_16KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_16KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_16KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_16KBDP.uiAddrShift = 10;
+	sRGXMMUPDEConfig_16KBDP.uiAddrLog2Align = 10;
+
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_16KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_16KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_16KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_16KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_16KBDP
+	 */
+	sRGXMMUPTEConfig_16KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_16KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_16KBDP.uiAddrMask = IMG_UINT64_C(0xffffffc000);
+	sRGXMMUPTEConfig_16KBDP.uiAddrShift = 14;
+	sRGXMMUPTEConfig_16KBDP.uiAddrLog2Align = 14;
+
+	sRGXMMUPTEConfig_16KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_16KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_16KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_16KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_16KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_16KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_16KBDP
+	 */
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001fc000);
+	sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift = 14;
+	sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_16KBDP.uiPTIndexShift));
+
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetMask = IMG_UINT64_C(0x0000003fff);
+	sRGXMMUDevVAddrConfig_16KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_16KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig16KB
+	 */
+	gsPageSizeConfig16KB.psPDEConfig = &sRGXMMUPDEConfig_16KBDP;
+	gsPageSizeConfig16KB.psPTEConfig = &sRGXMMUPTEConfig_16KBDP;
+	gsPageSizeConfig16KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_16KBDP;
+	gsPageSizeConfig16KB.uiRefCount = 0;
+	gsPageSizeConfig16KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 64kB Data-Page size
+	 *
+	 * Bit  39      30   29   21  20 16 15             0
+	 *      |         \ /       \ /   \ /               \
+	 *      |....PCE...|...PDE...|.PTE.|.....VAddr......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_64KBDP
+	 */
+	sRGXMMUPDEConfig_64KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_64KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_64KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_64KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_64KBDP.uiAddrShift = 8;
+	sRGXMMUPDEConfig_64KBDP.uiAddrLog2Align = 8;
+
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_64KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_64KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_64KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_64KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_64KBDP
+	 */
+	sRGXMMUPTEConfig_64KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_64KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_64KBDP.uiAddrMask = IMG_UINT64_C(0xffffff0000);
+	sRGXMMUPTEConfig_64KBDP.uiAddrShift = 16;
+	sRGXMMUPTEConfig_64KBDP.uiAddrLog2Align = 16;
+
+	sRGXMMUPTEConfig_64KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_64KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_64KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_64KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_64KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_64KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_64KBDP
+	 */
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001f0000);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift = 16;
+	sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_64KBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000000ffff);
+	sRGXMMUDevVAddrConfig_64KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_64KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig64KB
+	 */
+	gsPageSizeConfig64KB.psPDEConfig = &sRGXMMUPDEConfig_64KBDP;
+	gsPageSizeConfig64KB.psPTEConfig = &sRGXMMUPTEConfig_64KBDP;
+	gsPageSizeConfig64KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_64KBDP;
+	gsPageSizeConfig64KB.uiRefCount = 0;
+	gsPageSizeConfig64KB.uiMaxRefCount = 0;
+
+
+	/*
+	 *
+	 *  Configuration for heaps with 256kB Data-Page size
+	 *
+	 * Bit  39      30   29   21 20 18 17              0
+	 *      |         \ /       \|   |/                 \
+	 *      |....PCE...|...PDE...|PTE|.......VAddr......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_256KBDP
+	 */
+	sRGXMMUPDEConfig_256KBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_256KBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_256KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_256KBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	sRGXMMUPDEConfig_256KBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_256KBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_256KBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_256KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_256KBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_256KBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup MMU_PxE_CONFIG sRGXMMUPTEConfig_256KBDP
+	 */
+	sRGXMMUPTEConfig_256KBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_256KBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_256KBDP.uiAddrMask = IMG_UINT64_C(0xfffffc0000);
+	sRGXMMUPTEConfig_256KBDP.uiAddrShift = 18;
+	sRGXMMUPTEConfig_256KBDP.uiAddrLog2Align = 18;
+
+	sRGXMMUPTEConfig_256KBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_256KBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_256KBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_256KBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_256KBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_256KBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_256KBDP
+	 */
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask = IMG_UINT64_C(0x00001c0000);
+	sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift = 18;
+	sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_256KBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetMask = IMG_UINT64_C(0x000003ffff);
+	sRGXMMUDevVAddrConfig_256KBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_256KBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig256KB
+	 */
+	gsPageSizeConfig256KB.psPDEConfig = &sRGXMMUPDEConfig_256KBDP;
+	gsPageSizeConfig256KB.psPTEConfig = &sRGXMMUPTEConfig_256KBDP;
+	gsPageSizeConfig256KB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_256KBDP;
+	gsPageSizeConfig256KB.uiRefCount = 0;
+	gsPageSizeConfig256KB.uiMaxRefCount = 0;
+
+	/*
+	 *
+	 *  Configuration for heaps with 1MB Data-Page size
+	 *
+	 * Bit  39      30   29   21 20  19               0
+	 *      |         \ /       \ | /                  \
+	 *      |....PCE...|...PDE...|.|........VAddr.......|
+	 *                           PTE
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_1MBDP
+	 */
+	sRGXMMUPDEConfig_1MBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_1MBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_1MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_1MBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	/*
+	 * The hardware requires that PT tables need be 1<<6 = 64 byte aligned even
+	 * if they contain fewer entries.
+	 */
+	sRGXMMUPDEConfig_1MBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_1MBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_1MBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_1MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_1MBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_1MBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUPTEConfig_1MBDP
+	 */
+	sRGXMMUPTEConfig_1MBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_1MBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_1MBDP.uiAddrMask = IMG_UINT64_C(0xfffff00000);
+	sRGXMMUPTEConfig_1MBDP.uiAddrShift = 20;
+	sRGXMMUPTEConfig_1MBDP.uiAddrLog2Align = 20;
+
+	sRGXMMUPTEConfig_1MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_1MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_1MBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_1MBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_1MBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_1MBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_1MBDP
+	 */
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask = IMG_UINT64_C(0x0000100000);
+	sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift = 20;
+	sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_1MBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetMask = IMG_UINT64_C(0x00000fffff);
+	sRGXMMUDevVAddrConfig_1MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_1MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig1MB
+	 */
+	gsPageSizeConfig1MB.psPDEConfig = &sRGXMMUPDEConfig_1MBDP;
+	gsPageSizeConfig1MB.psPTEConfig = &sRGXMMUPTEConfig_1MBDP;
+	gsPageSizeConfig1MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_1MBDP;
+	gsPageSizeConfig1MB.uiRefCount = 0;
+	gsPageSizeConfig1MB.uiMaxRefCount = 0;
+
+	/*
+	 *
+	 *  Configuration for heaps with 2MB Data-Page size
+	 *
+	 * Bit  39      30   29   21   20                0
+	 *      |         \ /       \ /                   \
+	 *      |....PCE...|...PDE...|.........VAddr.......|
+	 *
+	 */
+
+	/*
+	 * Setup sRGXMMUPDEConfig_2MBDP
+	 */
+	sRGXMMUPDEConfig_2MBDP.ePxLevel = MMU_LEVEL_2;
+	sRGXMMUPDEConfig_2MBDP.pszPxLevelStr = "PD";
+	sRGXMMUPDEConfig_2MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPDEConfig_2MBDP.uiAddrMask = IMG_UINT64_C(0xfffffffff0);
+	/*
+	 * The hardware requires that PT tables need be 1<<6 = 64 byte aligned even
+	 * if they contain fewer entries.
+	 */
+	sRGXMMUPDEConfig_2MBDP.uiAddrShift = 6;
+	sRGXMMUPDEConfig_2MBDP.uiAddrLog2Align = 6;
+
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlMask = IMG_UINT64_C(0x000000000e);
+	sRGXMMUPDEConfig_2MBDP.uiVarCtrlShift = 1;
+
+	sRGXMMUPDEConfig_2MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PDE);
+	sRGXMMUPDEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPDEConfig_2MBDP.uiValidEnMask = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	sRGXMMUPDEConfig_2MBDP.uiValidEnShift = RGX_MMUCTRL_PD_DATA_VALID_SHIFT;
+
+#if defined(RGX_FEATURE_MH_PARITY_BIT_MASK)
+	if (bHaveParity)
+	{
+		sRGXMMUPTEConfig_2MBDP.uiParityBitMask = RGX_MMUCTRL_PT_DATA_PT_PARITY_EN;
+		sRGXMMUPTEConfig_2MBDP.uiParityBitShift = RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+#endif
+
+	/*
+	 * Setup sRGXMMUPTEConfig_2MBDP
+	 */
+	sRGXMMUPTEConfig_2MBDP.ePxLevel = MMU_LEVEL_1;
+	sRGXMMUPTEConfig_2MBDP.pszPxLevelStr = "PT";
+	sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry = 8;
+
+	sRGXMMUPTEConfig_2MBDP.uiAddrMask = IMG_UINT64_C(0xffffe00000);
+	sRGXMMUPTEConfig_2MBDP.uiAddrShift = 21;
+	sRGXMMUPTEConfig_2MBDP.uiAddrLog2Align = 21;
+
+	sRGXMMUPTEConfig_2MBDP.uiProtMask = RGX_GET_MMUCTRL_PROTMASK(PTE);
+	sRGXMMUPTEConfig_2MBDP.uiProtShift = 0;
+
+	sRGXMMUPTEConfig_2MBDP.uiValidEnMask = RGX_MMUCTRL_PT_DATA_VALID_EN;
+	sRGXMMUPTEConfig_2MBDP.uiValidEnShift = RGX_MMUCTRL_PT_DATA_VALID_SHIFT;
+
+	/*
+	 * Setup sRGXMMUDevVAddrConfig_2MBDP
+	 */
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask = ~RGX_MMUCTRL_VADDR_PC_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift = RGX_MMUCTRL_VADDR_PC_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPC = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPCIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask = ~RGX_MMUCTRL_VADDR_PD_INDEX_CLRMSK;
+	sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift = RGX_MMUCTRL_VADDR_PD_INDEX_SHIFT;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPD = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPDIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask = IMG_UINT64_C(0x0000000000);
+	sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift = 21;
+	sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT = TRUNCATE_64BITS_TO_32BITS(UNITS_IN_BITFIELD(sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexMask,
+			sRGXMMUDevVAddrConfig_2MBDP.uiPTIndexShift));
+
+
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetMask = IMG_UINT64_C(0x00001fffff);
+	sRGXMMUDevVAddrConfig_2MBDP.uiPageOffsetShift = 0;
+	sRGXMMUDevVAddrConfig_2MBDP.uiOffsetInBytes = 0;
+
+	/*
+	 * Setup gsPageSizeConfig2MB
+	 */
+	gsPageSizeConfig2MB.psPDEConfig = &sRGXMMUPDEConfig_2MBDP;
+	gsPageSizeConfig2MB.psPTEConfig = &sRGXMMUPTEConfig_2MBDP;
+	gsPageSizeConfig2MB.psDevVAddrConfig = &sRGXMMUDevVAddrConfig_2MBDP;
+	gsPageSizeConfig2MB.uiRefCount = 0;
+	gsPageSizeConfig2MB.uiMaxRefCount = 0;
+
+	/*
+	 * Setup sRGXMMUDeviceAttributes
+	 */
+	sRGXMMUDeviceAttributes.eMMUType = PDUMP_MMU_TYPE_VARPAGE_40BIT;
+	sRGXMMUDeviceAttributes.ui32BaseAlign = RGX_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT;
+	sRGXMMUDeviceAttributes.psBaseConfig = &sRGXMMUPCEConfig;
+	sRGXMMUDeviceAttributes.psTopLevelDevVAddrConfig = &sRGXMMUTopLevelDevVAddrConfig;
+
+#if defined(PVRSRV_MMU_PARITY_ON_PTALLOC_AND_PTEUNMAP)
+{
+	IMG_UINT32 i;
+
+	PVR_ASSERT(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT   <= 512);
+	PVR_ASSERT(sRGXMMUDevVAddrConfig_16KBDP.uiNumEntriesPT  <= 512);
+	PVR_ASSERT(sRGXMMUDevVAddrConfig_64KBDP.uiNumEntriesPT  <= 512);
+	PVR_ASSERT(sRGXMMUDevVAddrConfig_256KBDP.uiNumEntriesPT <= 512);
+	PVR_ASSERT(sRGXMMUDevVAddrConfig_1MBDP.uiNumEntriesPT   <= 512);
+	PVR_ASSERT(sRGXMMUDevVAddrConfig_2MBDP.uiNumEntriesPT   <= 512);
+
+	PVR_ASSERT(sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry   == 8);
+	PVR_ASSERT(sRGXMMUPTEConfig_16KBDP.uiBytesPerEntry  == 8);
+	PVR_ASSERT(sRGXMMUPTEConfig_64KBDP.uiBytesPerEntry  == 8);
+	PVR_ASSERT(sRGXMMUPTEConfig_256KBDP.uiBytesPerEntry == 8);
+	PVR_ASSERT(sRGXMMUPTEConfig_1MBDP.uiBytesPerEntry   == 8);
+	PVR_ASSERT(sRGXMMUPTEConfig_2MBDP.uiBytesPerEntry   == 8);
+
+	sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[0] =
+	    OSAllocZMem(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT * sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry);
+	sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[1] =
+	    OSAllocZMem(sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT * sRGXMMUPTEConfig_4KBDP.uiBytesPerEntry);
+
+	/* Generate two precomputed pages in host memory for zero-initialised PTs with parity bits set. */
+	for (i=0; i<sRGXMMUDevVAddrConfig_4KBDP.uiNumEntriesPT; i++)
+	{
+		sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[0][i] =
+		    ((IMG_UINT64)(ui8ParityPTPattern[i]))         << RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+		sRGXMMUDeviceAttributes.pui64PrecomputedAllocParity[1][i] =
+		    ((IMG_UINT64)(ui8ParityPTPattern[i] ? 0 : 1)) << RGX_MMUCTRL_PT_DATA_PT_PARITY_SHIFT;
+	}
+}
+#endif
+
+	/* Functions for deriving page table/dir/cat protection bits */
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt8 = RGXDerivePCEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePCEProt4 = RGXDerivePCEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXDerivePDEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePDEProt4 = RGXDerivePDEProt4;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt8 = RGXDerivePTEProt8;
+	sRGXMMUDeviceAttributes.pfnDerivePTEProt4 = RGXDerivePTEProt4;
+
+	/* Functions for establishing configurations for PDE/PTE/DEVVADDR
+	   on per-heap basis */
+	sRGXMMUDeviceAttributes.pfnGetPageSizeConfiguration = RGXGetPageSizeConfigCB;
+	sRGXMMUDeviceAttributes.pfnPutPageSizeConfiguration = RGXPutPageSizeConfigCB;
+
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE4 = RGXGetPageSizeFromPDE4;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXGetPageSizeFromPDE8;
+	sRGXMMUDeviceAttributes.pfnGetPageSizeFromVirtAddr = NULL;
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (bHaveMMU4)
+	{
+		/* override some of these functions for MMU4 as page size is not stored in PD entries */
+		sRGXMMUDeviceAttributes.pfnDerivePDEProt8 = RGXMMU4DerivePDEProt8;
+		sRGXMMUDeviceAttributes.pfnGetPageSizeFromPDE8 = RGXMMU4GetPageSizeFromPDE8;
+		sRGXMMUDeviceAttributes.pfnGetPageSizeFromVirtAddr = RGXMMU4GetPageSizeFromVirtAddr;
+	}
+#endif
+
+	sRGXMMUDeviceAttributes.ui32ValidPageSizeMask = RGXGetValidHeapPageSizeMask();
+
+	psDeviceNode->psMMUDevAttrs = &sRGXMMUDeviceAttributes;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRV_OK;
+
+#if defined(PDUMP)
+	psDeviceNode->pfnMMUGetContextID = NULL;
+#endif
+
+	psDeviceNode->psMMUDevAttrs = NULL;
+
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_MESSAGE, "Variable Page Size Heap Stats:"));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 4K page heaps: %d",
+			gsPageSizeConfig4KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 4K page heaps (should be 0): %d",
+			gsPageSizeConfig4KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 16K page heaps: %d",
+			gsPageSizeConfig16KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 16K page heaps (should be 0): %d",
+			gsPageSizeConfig16KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 64K page heaps: %d",
+			gsPageSizeConfig64KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 64K page heaps (should be 0): %d",
+			gsPageSizeConfig64KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 256K page heaps: %d",
+			gsPageSizeConfig256KB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 256K page heaps (should be 0): %d",
+			gsPageSizeConfig256KB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 1M page heaps: %d",
+			gsPageSizeConfig1MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 1M page heaps (should be 0): %d",
+			gsPageSizeConfig1MB.uiRefCount));
+	PVR_DPF((PVR_DBG_MESSAGE, "Max 2M page heaps: %d",
+			gsPageSizeConfig2MB.uiMaxRefCount));
+	PVR_DPF((PVR_DBG_VERBOSE, "Current 2M page heaps (should be 0): %d",
+			gsPageSizeConfig2MB.uiRefCount));
+#endif
+	if (gsPageSizeConfig4KB.uiRefCount > 0 ||
+			gsPageSizeConfig16KB.uiRefCount > 0 ||
+			gsPageSizeConfig64KB.uiRefCount > 0 ||
+			gsPageSizeConfig256KB.uiRefCount > 0 ||
+			gsPageSizeConfig1MB.uiRefCount > 0 ||
+			gsPageSizeConfig2MB.uiRefCount > 0
+	)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXMMUInit_Unregister: Unbalanced MMU API Usage (Internal error)"));
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt4
+@Description    calculate the PCE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePCEProt4(IMG_UINT32 uiProtFlags)
+{
+	return (uiProtFlags & MMU_PROTFLAGS_INVALID)?0:RGX_MMUCTRL_PC_DATA_VALID_EN;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePCEProt8
+@Description    calculate the PCE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePCEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	PVR_DPF((PVR_DBG_ERROR, "8-byte PCE not supported on this device"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt4
+@Description    derive the PDE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePDEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PDE not supported on this device"));
+	return 0;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize The log2 of the required page size.
+                E.g, for 4KiB pages, this parameter must be 12.
+                For 2MiB pages, it must be set to 21.
+
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ret_value = 0; /* 0 means invalid */
+
+	if (!(uiProtFlags & MMU_PROTFLAGS_INVALID)) /* if not invalid */
+	{
+		switch (uiLog2DataPageSize)
+		{
+		case RGX_HEAP_4KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB;
+			break;
+		case RGX_HEAP_16KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB;
+			break;
+		case RGX_HEAP_64KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB;
+			break;
+		case RGX_HEAP_256KB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB;
+			break;
+		case RGX_HEAP_1MB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB;
+			break;
+		case RGX_HEAP_2MB_PAGE_SHIFT:
+			ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN | RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB;
+			break;
+		default:
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Invalid parameter log2_page_size. Expected {12, 14, 16, 18, 20, 21}. Got [%u]",
+					__func__, uiLog2DataPageSize));
+		}
+	}
+	return ret_value;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt4
+@Description    calculate the PTE protection flags based on a 4 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT32 RGXDerivePTEProt4(IMG_UINT32 uiProtFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(uiProtFlags);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PTE not supported on this device"));
+
+	return 0;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXDerivePTEProt8
+@Description    calculate the PTE protection flags based on an 8 byte entry
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static IMG_UINT64 RGXDerivePTEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ui64MMUFlags=0;
+
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	if (((MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE) & uiProtFlags) == (MMU_PROTFLAGS_READABLE|MMU_PROTFLAGS_WRITEABLE))
+	{
+		/* read/write */
+	}
+	else if (MMU_PROTFLAGS_READABLE & uiProtFlags)
+	{
+		/* read only */
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_READ_ONLY_EN;
+	}
+	else if (MMU_PROTFLAGS_WRITEABLE & uiProtFlags)
+	{
+		/* write only */
+		PVR_DPF((PVR_DBG_WARNING, "RGXDerivePTEProt8: write-only is not possible on this device"));
+	}
+	else if ((MMU_PROTFLAGS_INVALID & uiProtFlags) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXDerivePTEProt8: neither read nor write specified..."));
+	}
+
+	/* cache coherency */
+	if (MMU_PROTFLAGS_CACHE_COHERENT & uiProtFlags)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_CC_EN;
+	}
+
+#if defined(RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN)
+	/* cache setup */
+	if ((MMU_PROTFLAGS_CACHED & uiProtFlags) == 0)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_SLC_BYPASS_CTRL_EN;
+	}
+#endif
+
+	if ((uiProtFlags & MMU_PROTFLAGS_INVALID) == 0)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_VALID_EN;
+	}
+
+	if (MMU_PROTFLAGS_DEVICE(PMMETA_PROTECT) & uiProtFlags)
+	{
+		ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_PM_META_PROTECT_EN;
+	}
+
+#if defined(RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC)
+	/**
+	 * Always enable caching on the fabric level cache irrespective of type of
+	 * cache coherent interconnect and memory cache attributes.
+	 * This needs to be updated, if selective caching policy needs to be
+	 * implemented based on cache attributes requested by caller and based on
+	 * cache coherent interconnect.
+	 */
+	ui64MMUFlags |= RGX_MMUCTRL_PT_DATA_AXCACHE_WBRWALLOC;
+#endif
+
+	return ui64MMUFlags;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXGetPageSizeConfig
+@Description    Set up configuration for variable sized data pages.
+                RGXPutPageSizeConfigCB has to be called to ensure correct
+                refcounting.
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static PVRSRV_ERROR RGXGetPageSizeConfigCB(IMG_UINT32 uiLog2DataPageSize,
+		const MMU_PxE_CONFIG **ppsMMUPDEConfig,
+		const MMU_PxE_CONFIG **ppsMMUPTEConfig,
+		const MMU_DEVVADDR_CONFIG **ppsMMUDevVAddrConfig,
+		IMG_HANDLE *phPriv)
+{
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGX_HEAP_4KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	case RGX_HEAP_16KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig16KB;
+		break;
+	case RGX_HEAP_64KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGX_HEAP_256KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig256KB;
+		break;
+	case RGX_HEAP_1MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig1MB;
+		break;
+	case RGX_HEAP_2MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig2MB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXGetPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				uiLog2DataPageSize));
+		*phPriv = NULL;
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Refer caller's pointers to the data */
+	*ppsMMUPDEConfig = psPageSizeConfig->psPDEConfig;
+	*ppsMMUPTEConfig = psPageSizeConfig->psPTEConfig;
+	*ppsMMUDevVAddrConfig = psPageSizeConfig->psDevVAddrConfig;
+
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	/* Increment ref-count - not that we're allocating anything here
+       (I'm using static structs), but one day we might, so we want
+       the Get/Put code to be balanced properly */
+	psPageSizeConfig->uiRefCount++;
+
+	/* This is purely for debug statistics */
+	psPageSizeConfig->uiMaxRefCount = MAX(psPageSizeConfig->uiMaxRefCount,
+			psPageSizeConfig->uiRefCount);
+#endif
+
+	*phPriv = (IMG_HANDLE)(uintptr_t)uiLog2DataPageSize;
+	PVR_ASSERT (uiLog2DataPageSize == (IMG_UINT32)(uintptr_t)*phPriv);
+
+	return PVRSRV_OK;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPutPageSizeConfig
+@Description    Tells this code that the mmu module is done with the
+                configurations set in RGXGetPageSizeConfig.  This can
+                be a no-op.
+                Called after RGXGetPageSizeConfigCB.
+@Return         PVRSRV_ERROR
+ */ /**************************************************************************/
+static PVRSRV_ERROR RGXPutPageSizeConfigCB(IMG_HANDLE hPriv)
+{
+#if defined(SUPPORT_MMU_PAGESIZECONFIG_REFCOUNT)
+	MMU_PAGESIZECONFIG *psPageSizeConfig;
+	IMG_UINT32 uiLog2DataPageSize;
+
+	uiLog2DataPageSize = (IMG_UINT32)(uintptr_t) hPriv;
+
+	switch (uiLog2DataPageSize)
+	{
+	case RGX_HEAP_4KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig4KB;
+		break;
+	case RGX_HEAP_16KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig16KB;
+		break;
+	case RGX_HEAP_64KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig64KB;
+		break;
+	case RGX_HEAP_256KB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig256KB;
+		break;
+	case RGX_HEAP_1MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig1MB;
+		break;
+	case RGX_HEAP_2MB_PAGE_SHIFT:
+		psPageSizeConfig = &gsPageSizeConfig2MB;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXPutPageSizeConfigCB: Invalid Data Page Size 1<<0x%x",
+				uiLog2DataPageSize));
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+
+	/* Ref-count here is not especially useful, but it's an extra
+	   check that the API is being used correctly */
+	psPageSizeConfig->uiRefCount--;
+#else
+	PVR_UNREFERENCED_PARAMETER(hPriv);
+#endif
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE4(IMG_UINT32 ui32PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui32PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+	PVR_DPF((PVR_DBG_ERROR, "4-byte PDE not supported on this device"));
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+static PVRSRV_ERROR RGXGetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	IMG_UINT64 ui64PageSizeBits = ui64PDE & (~RGX_MMUCTRL_PD_DATA_PAGE_SIZE_CLRMSK);
+
+	switch (ui64PageSizeBits)
+	{
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_4KB:
+		*pui32Log2PageSize = RGX_HEAP_4KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_16KB:
+		*pui32Log2PageSize = RGX_HEAP_16KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_64KB:
+		*pui32Log2PageSize = RGX_HEAP_64KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_256KB:
+		*pui32Log2PageSize = RGX_HEAP_256KB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_1MB:
+		*pui32Log2PageSize = RGX_HEAP_1MB_PAGE_SHIFT;
+		break;
+	case RGX_MMUCTRL_PD_DATA_PAGE_SIZE_2MB:
+		*pui32Log2PageSize = RGX_HEAP_2MB_PAGE_SHIFT;
+		break;
+	default:
+		PVR_DPF((PVR_DBG_ERROR,
+				"RGXGetPageSizeFromPDE8: Invalid page size bitfield %" IMG_UINT64_FMTSPECx " in PDE",
+				ui64PageSizeBits));
+
+		return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+	}
+	return PVRSRV_OK;
+}
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/*************************************************************************/ /*!
+@Function       RGXMMU4DerivePDEProt8
+@Description    derive the PDE protection flags based on an 8 byte entry
+
+@Input          uiLog2DataPageSize: ignored as MMU4 doesn't put page size in PD entries.
+
+@Return         PVRSRV_ERROR
+*/ /**************************************************************************/
+static IMG_UINT64 RGXMMU4DerivePDEProt8(IMG_UINT32 uiProtFlags, IMG_UINT32 uiLog2DataPageSize)
+{
+	IMG_UINT64 ret_value = 0; /* 0 means invalid */
+	PVR_UNREFERENCED_PARAMETER(uiLog2DataPageSize);
+
+	if (!(uiProtFlags & MMU_PROTFLAGS_INVALID)) /* if not invalid */
+	{
+		/*  page size in range config registers. Bits in PD entries are reserved */
+		ret_value = RGX_MMUCTRL_PD_DATA_VALID_EN;
+	}
+	return ret_value;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXMMU4GetPageSizeFromPDE8
+@Description    The upper layers should be such that this function is never called
+                as pages size are not stored in PD entries for MMU4.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromPDE8(IMG_UINT64 ui64PDE, IMG_UINT32 *pui32Log2PageSize)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64PDE);
+	PVR_UNREFERENCED_PARAMETER(pui32Log2PageSize);
+
+	PVR_ASSERT(0 && "RGXMMU4GetPageSizeFromPDE8 called in error. MMU4 does not store page sizes in PDT.");
+	return PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+}
+
+
+/*************************************************************************/ /*!
+@Function       RGXMMU4GetPageSizeFromVirtAddr
+@Description    Get page size by walking through range config registers
+                looking for a match against the virtual address.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXMMU4GetPageSizeFromVirtAddr(struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+												   IMG_DEV_VIRTADDR sDevVAddr,
+												   IMG_UINT32 *pui32Log2PageSize)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+
+	/* assume default of 4KB page size */
+	*pui32Log2PageSize = 12;
+
+	/* Loop through the range registers looking for the given target address */
+	for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i)
+	{
+		IMG_UINT64 ui64RangeVal = psDevInfo->aui64MMUPageSizeRangeValue[i];
+
+		if (ui64RangeVal != 0)
+		{
+			/* end addr in register is inclusive in the range so add 1 to move it over the end */
+			IMG_UINT64 ui64Base = ((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_CLRMSK)
+												>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_SHIFT)
+												<< RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_BASE_ADDR_ALIGNSHIFT;
+			IMG_UINT64 ui64End  = (((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_CLRMSK)
+												>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_SHIFT) + 1)
+												<< RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_END_ADDR_ALIGNSHIFT;
+
+			if ((sDevVAddr.uiAddr >= ui64Base) && (sDevVAddr.uiAddr < ui64End))
+			{
+				IMG_UINT32 ui32PageSizeField = (IMG_UINT32)((ui64RangeVal & ~RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_CLRMSK)
+														>> RGX_CR_MMU_PAGE_SIZE_RANGE_ONE_PAGE_SIZE_SHIFT);
+				if (ui32PageSizeField < 5)
+				{
+					*pui32Log2PageSize = (ui32PageSizeField << 1) + 12;   /* 12 (4K), 14 (16K), 16 (64K), 18 (256K), 20 (1MB) */
+				}
+				else if (ui32PageSizeField == 5)
+				{
+					*pui32Log2PageSize = 21;    /* 2MB */
+				}
+				else
+				{
+					eError = PVRSRV_ERROR_MMU_INVALID_PAGE_SIZE_FOR_DEVICE;
+				}
+				break;
+			}
+		}
+	}
+
+	return eError;
+}
+#endif
diff --git a/drivers/gpu/drm/img-rogue/rgxmmuinit.h b/drivers/gpu/drm/img-rogue/rgxmmuinit.h
new file mode 100644
index 000000000000..b70ddad0cb64
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmmuinit.h
@@ -0,0 +1,59 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific MMU initialisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* NB: this file is not to be included arbitrarily.  It exists solely
+   for the linkage between rgxinit.c and rgxmmuinit.c, the former
+   being otherwise cluttered by the contents of the latter */
+
+#ifndef SRVKM_RGXMMUINIT_H
+#define SRVKM_RGXMMUINIT_H
+
+#include "device.h"
+#include "img_types.h"
+#include "mmu_common.h"
+#include "img_defs.h"
+
+PVRSRV_ERROR RGXMMUInit_Register(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR RGXMMUInit_Unregister(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* #ifndef SRVKM_RGXMMUINIT_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxmulticore.c b/drivers/gpu/drm/img-rogue/rgxmulticore.c
new file mode 100644
index 000000000000..447c50a78e00
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmulticore.c
@@ -0,0 +1,328 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.c
+@Title          Functions related to multicore devices
+@Codingstyle    IMG
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Kernel mode workload estimation functionality.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxdevice.h"
+#include "rgxdefs_km.h"
+#include "pdump_km.h"
+#include "rgxmulticore.h"
+#include "multicore_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxfwmemctx.h"
+
+/*
+ * check that register defines match our hardcoded definitions.
+ * Rogue has these, volcanic does not.
+ */
+#if ((RGX_MULTICORE_CAPABILITY_FRAGMENT_EN != RGX_CR_MULTICORE_GPU_CAPABILITY_FRAGMENT_EN) || \
+     (RGX_MULTICORE_CAPABILITY_GEOMETRY_EN != RGX_CR_MULTICORE_GPU_CAPABILITY_GEOMETRY_EN) || \
+     (RGX_MULTICORE_CAPABILITY_COMPUTE_EN  != RGX_CR_MULTICORE_GPU_CAPABILITY_COMPUTE_EN) || \
+     (RGX_MULTICORE_CAPABILITY_PRIMARY_EN  != RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN) || \
+     (RGX_MULTICORE_ID_CLRMSK              != RGX_CR_MULTICORE_GPU_ID_CLRMSK))
+#error "Rogue definitions for RGX_CR_MULTICORE_GPU register have changed"
+#endif
+
+
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_UINT32 ui32CapsSize,
+                                        IMG_UINT32 *pui32NumCores,
+                                        IMG_UINT64 *pui64Caps);
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Return multicore information to clients.
+ * Return not supported on cores without multicore.
+ */
+static PVRSRV_ERROR RGXGetMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 IMG_UINT32 ui32CapsSize,
+                                 IMG_UINT32 *pui32NumCores,
+                                 IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDevInfo->ui32MultiCoreNumCores == 0)
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+	else
+	{
+		*pui32NumCores = psDevInfo->ui32MultiCoreNumCores;
+		if (ui32CapsSize > 0)
+		{
+			if (ui32CapsSize < psDevInfo->ui32MultiCoreNumCores)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore caps buffer too small"));
+				eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			}
+			else
+			{
+				IMG_UINT32 i;
+
+				for (i = 0; i < psDevInfo->ui32MultiCoreNumCores; ++i)
+				{
+					pui64Caps[i] = psDevInfo->pui64MultiCoreCapabilities[i];
+				}
+			}
+		}
+	}
+
+	return eError;
+}
+
+
+
+/*
+ * RGXInitMultiCoreInfo:
+ * Read multicore HW registers and fill in data structure for clients.
+ * Return not supported on cores without multicore.
+ */
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		/* we only set this up once */
+		return PVRSRV_OK;
+	}
+
+	/* defaults for non-multicore devices */
+	psDevInfo->ui32MultiCoreNumCores = 0;
+	psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	psDevInfo->pui64MultiCoreCapabilities = NULL;
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+	{
+		IMG_BOOL bPowerWasDown;
+		IMG_UINT32 ui32MulticoreRegBankOffset = (1 << RGX_GET_FEATURE_VALUE(psDevInfo, XPU_MAX_REGBANKS_ADDR_WIDTH));
+		IMG_UINT32 ui32MulticoreGPUReg = RGX_CR_MULTICORE_GPU;
+		IMG_UINT32 ui32NumCores;
+		IMG_UINT32 i;
+
+#if defined(RGX_HOST_SECURE_REGBANK_OFFSET) && defined(XPU_MAX_REGBANKS_ADDR_WIDTH)
+		/* Ensure the HOST_SECURITY reg bank definitions are correct */
+		if ((RGX_HOST_SECURE_REGBANK_OFFSET + RGX_HOST_SECURE_REGBANK_SIZE) != ui32MulticoreRegBankOffset)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Register bank definitions for HOST_SECURITY don't match core's configuration.", __func__));
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+#endif
+		bPowerWasDown = ! PVRSRVIsSystemPowered(psDeviceNode);
+
+		/* Power-up the device as required to read the registers */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			PVRSRVPowerLock(psDeviceNode);
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_ON);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRVSetSystemPowerState ON failed (%u)", __func__, eError));
+				PVRSRVPowerUnlock(psDeviceNode);
+				return eError;
+			}
+		}
+
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			IMG_UINT32 ui32FwTimeout = MAX_HW_TIME_US;
+
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores,
+		                               INVALIDATE);
+				if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores))
+				{
+					/* No need to wait if the FW has already updated the values */
+					break;
+				}
+				OSWaitus(ui32FwTimeout/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (*((volatile IMG_UINT32*)&psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores) == 0)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Multicore info not available for guest"));
+				return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+			}
+
+			ui32NumCores = psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.ui32NumCores;
+
+			PVR_LOG(("RGX Guest Device initialised with %u %s",
+					 ui32NumCores, (ui32NumCores == 1U) ? "core" : "cores"));
+		}
+		else
+#endif
+		{
+			ui32NumCores = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MULTICORE_SYSTEM);
+		}
+#if !defined(NO_HARDWARE)
+		/* check that the number of cores reported is in-bounds */
+		if (ui32NumCores > (RGX_CR_MULTICORE_SYSTEM_MASKFULL >> RGX_CR_MULTICORE_SYSTEM_GPU_COUNT_SHIFT))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "invalid return (%u) read from MULTICORE_SYSTEM", ui32NumCores));
+			return PVRSRV_ERROR_DEVICE_REGISTER_FAILED;
+		}
+#else
+		/* for nohw set to max so clients can allocate enough memory for all pdump runs on any config */
+		ui32NumCores = RGX_MULTICORE_MAX_NOHW_CORES;
+#endif
+		PVR_DPF((PVR_DBG_MESSAGE, "Multicore system has %u cores", ui32NumCores));
+		PDUMPCOMMENT(psDeviceNode, "RGX Multicore has %d cores\n", ui32NumCores);
+
+		/* allocate storage for capabilities */
+		psDevInfo->pui64MultiCoreCapabilities = OSAllocMem(ui32NumCores * sizeof(psDevInfo->pui64MultiCoreCapabilities[0]));
+		PVR_LOG_GOTO_IF_NOMEM(psDevInfo->pui64MultiCoreCapabilities, eError, err);
+
+		psDevInfo->ui32MultiCoreNumCores = ui32NumCores;
+
+		for (i = 0; i < ui32NumCores; ++i)
+		{
+	#if !defined(NO_HARDWARE)
+#if defined(RGX_NUM_DRIVERS_SUPPORTED) && (RGX_NUM_DRIVERS_SUPPORTED > 1)
+			if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				psDevInfo->pui64MultiCoreCapabilities[i] = psDevInfo->psRGXFWIfOsInit->sRGXMulticoreInfo.aui64MultiCoreCapabilities[i];
+			}
+			else
+#endif
+			{
+				IMG_BOOL bMulticoreRegsMapped = (psDeviceNode->psDevConfig->ui32RegsSize > ui32MulticoreGPUReg);
+				void __iomem *pvCoreRegBase;
+				IMG_INT32 ui32MulticoreRegOffset;
+
+				if (bMulticoreRegsMapped)
+				{
+					pvCoreRegBase = psDevInfo->pvRegsBaseKM;
+					ui32MulticoreRegOffset = ui32MulticoreGPUReg;
+				}
+				else
+				{
+					/* the register bank of this core is not mapped */
+					IMG_CPU_PHYADDR sMultiCoreRegsBase = psDeviceNode->psDevConfig->sRegsCpuPBase;
+
+					sMultiCoreRegsBase.uiAddr += i*ui32MulticoreRegBankOffset;
+					pvCoreRegBase = (void __iomem *) OSMapPhysToLin(sMultiCoreRegsBase, psDeviceNode->psDevConfig->ui32RegsSize, PVRSRV_MEMALLOCFLAG_CPU_UNCACHED);
+					PVR_LOG_GOTO_IF_NOMEM(pvCoreRegBase, eError, err);
+
+					/* adjust the register offset to point inside the newly mapped range */
+					ui32MulticoreRegOffset = RGX_CR_MULTICORE_GPU;
+				}
+
+				psDevInfo->pui64MultiCoreCapabilities[i] =
+							OSReadHWReg64(pvCoreRegBase, ui32MulticoreRegOffset) & RGX_CR_MULTICORE_GPU_MASKFULL;
+
+				if (!bMulticoreRegsMapped)
+				{
+					OSUnMapPhysToLin((void __force *) pvCoreRegBase, psDeviceNode->psDevConfig->ui32RegsSize);
+				}
+			}
+	#else
+			/* emulation for what we think caps are */
+			psDevInfo->pui64MultiCoreCapabilities[i] =
+							   i | ((i == 0) ? (RGX_MULTICORE_CAPABILITY_PRIMARY_EN
+											  | RGX_MULTICORE_CAPABILITY_GEOMETRY_EN) : 0)
+							   | RGX_MULTICORE_CAPABILITY_COMPUTE_EN
+							   | RGX_MULTICORE_CAPABILITY_FRAGMENT_EN;
+	#endif
+			PVR_DPF((PVR_DBG_MESSAGE, "Core %d has capabilities value 0x%x", i, (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[i] ));
+			PDUMPCOMMENT(psDeviceNode, "\tCore %d has caps 0x%08x\n", i,
+			             (IMG_UINT32)psDevInfo->pui64MultiCoreCapabilities[i]);
+
+			if (psDevInfo->pui64MultiCoreCapabilities[i] & RGX_CR_MULTICORE_GPU_CAPABILITY_PRIMARY_EN)
+			{
+				psDevInfo->ui32MultiCorePrimaryId = (psDevInfo->pui64MultiCoreCapabilities[i]
+														& ~RGX_CR_MULTICORE_GPU_ID_CLRMSK)
+														>> RGX_CR_MULTICORE_GPU_ID_SHIFT;
+			}
+
+			ui32MulticoreGPUReg += ui32MulticoreRegBankOffset;
+		}
+
+        /* revert power state to what it was on entry to this function */
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) && bPowerWasDown)
+		{
+			eError = PVRSRVSetSystemPowerState(psDeviceNode->psDevConfig, PVRSRV_SYS_POWER_STATE_OFF);
+			PVRSRVPowerUnlock(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "PVRSRVSetSystemPowerState OFF");
+		}
+
+		/* Register callback to return info about multicore setup to client bridge */
+		psDeviceNode->pfnGetMultiCoreInfo = RGXGetMultiCoreInfo;
+	}
+	else
+	{
+		/* MULTICORE not supported on this device */
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+	return eError;
+
+err:
+	RGXDeInitMultiCoreInfo(psDeviceNode);
+	return eError;
+}
+
+
+/*
+ * RGXDeinitMultiCoreInfo:
+ * Release resources and clear the MultiCore values in the DeviceNode.
+ */
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pui64MultiCoreCapabilities != NULL)
+	{
+		OSFreeMem(psDevInfo->pui64MultiCoreCapabilities);
+		psDevInfo->pui64MultiCoreCapabilities = NULL;
+		psDevInfo->ui32MultiCoreNumCores = 0;
+		psDevInfo->ui32MultiCorePrimaryId = (IMG_UINT32)(-1);
+	}
+	psDeviceNode->pfnGetMultiCoreInfo = NULL;
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxmulticore.h b/drivers/gpu/drm/img-rogue/rgxmulticore.h
new file mode 100644
index 000000000000..b45a20ab5e14
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxmulticore.h
@@ -0,0 +1,54 @@
+/*************************************************************************/ /*!
+@File           rgxmulticore.h
+@Title          Functions related to multicore devices
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    General purpose memory shared between kernel driver and user
+                mode.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXMULTICORE_H
+#define RGXMULTICORE_H
+
+#include "pvrsrv_error.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR RGXInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode);
+void RGXDeInitMultiCoreInfo(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* RGXMULTICORE_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxpower.c b/drivers/gpu/drm/img-rogue/rgxpower.c
new file mode 100644
index 000000000000..2d608b0f653d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxpower.c
@@ -0,0 +1,1829 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific power routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "rgxpower.h"
+#include "rgxinit.h"
+#include "rgx_fwif_km.h"
+#include "rgxfwutils.h"
+#include "rgxfwriscv.h"
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "rgxdebug_common.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "rgxtimecorr.h"
+#include "devicemem_utils.h"
+#include "htbserver.h"
+#include "rgxstartstop.h"
+#include "rgxfwimageutils.h"
+#include "sync.h"
+#include "rgxdefs_km.h"
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+#include "process_stats.h"
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+#include "pvr_dvfs_device.h"
+#endif
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_FREQ)
+#include "pvr_gpufreq.h"
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_FREQ) */
+
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+#include "rgxpdvfs.h"
+#endif
+
+static PVRSRV_ERROR RGXFWNotifyHostTimeout(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_KCCB_CMD sCmd;
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32CmdKCCBSlot;
+
+	/* Send the Timeout notification to the FW */
+	sCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_HOST_TIMEOUT;
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	return eError;
+}
+
+static void _RGXUpdateGPUUtilStats(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGXFWIF_GPU_UTIL_FW *psUtilFW;
+	IMG_UINT64 ui64LastPeriod;
+	IMG_UINT64 ui64LastState;
+	IMG_UINT64 ui64LastReducedState;
+	IMG_UINT64 ui64LastTime;
+	IMG_UINT64 ui64TimeNow;
+	IMG_UINT32 ui32DriverID;
+	IMG_UINT64 ui64DMOSStatsCounter;
+
+	psUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, INVALIDATE);
+
+	OSLockAcquire(psDevInfo->hGPUUtilLock);
+
+	ui64TimeNow = RGXFWIF_GPU_UTIL_GET_TIME(RGXTimeCorrGetClockns64(psDevInfo->psDeviceNode));
+
+	/* Update counters to account for the time since the last update */
+	ui64LastState  = RGXFWIF_GPU_UTIL_GET_STATE(psUtilFW->ui64GpuLastWord);
+	ui64LastTime   = RGXFWIF_GPU_UTIL_GET_TIME(psUtilFW->ui64GpuLastWord);
+	ui64LastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64LastTime);
+	psUtilFW->aui64GpuStatsCounters[ui64LastState] += ui64LastPeriod;
+
+	/* Update state and time of the latest update */
+	psUtilFW->ui64GpuLastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(ui64TimeNow, ui64LastState);
+
+	/* convert last period into the same units as used by fw */
+	ui64TimeNow  = ui64TimeNow >> RGXFWIF_DM_OS_TIMESTAMP_SHIFT;
+
+	FOREACH_SUPPORTED_DRIVER(ui32DriverID)
+	{
+		RGXFWIF_GPU_STATS *psStats = &psUtilFW->sStats[ui32DriverID];
+		RGXFWIF_DM eDM;
+
+		for (eDM = 0; eDM < RGXFWIF_GPU_UTIL_DM_MAX; eDM++)
+		{
+			ui64LastState  = (IMG_UINT64)RGXFWIF_GPU_UTIL_GET_STATE32(psStats->aui32DMOSLastWord[eDM]);
+			ui64LastTime   = (IMG_UINT64)RGXFWIF_GPU_UTIL_GET_TIME32(psStats->aui32DMOSLastWord[eDM]) +
+			                 ((IMG_UINT64)psStats->aui32DMOSLastWordWrap[eDM] << 32);
+			ui64LastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64LastTime);
+			/* for states statistics per DM per driver we only care about the time in Active state,
+			so we "combine" other states (Idle and Blocked) together */
+			ui64LastReducedState = (ui64LastState == RGXFWIF_GPU_UTIL_STATE_ACTIVE) ?
+			                       RGXFWIF_GPU_UTIL_STATE_ACTIVE : RGXFWIF_GPU_UTIL_STATE_INACTIVE;
+			ui64DMOSStatsCounter = (IMG_UINT64)psStats->aaui32DMOSStatsCounters[eDM][ui64LastReducedState] + ui64LastPeriod;
+			psStats->aaui32DMOSStatsCounters[eDM][ui64LastReducedState] = (IMG_UINT32)(ui64DMOSStatsCounter & IMG_UINT32_MAX);
+			if (ui64DMOSStatsCounter > IMG_UINT32_MAX)
+			{
+				psStats->aaui32DMOSCountersWrap[eDM][ui64LastReducedState] += (IMG_UINT32)(ui64DMOSStatsCounter >> 32);
+			}
+
+			/* Update state and time of the latest update */
+			psStats->aui32DMOSLastWord[eDM] = RGXFWIF_GPU_UTIL_MAKE_WORD32((ui64TimeNow & (IMG_UINT64)IMG_UINT32_MAX), ui64LastState);
+			if (ui64TimeNow > IMG_UINT32_MAX)
+			{
+				if (psStats->aui32DMOSLastWordWrap[eDM] != (IMG_UINT32)(ui64TimeNow >> 32))
+				{
+					psStats->aui32DMOSLastWordWrap[eDM] = (IMG_UINT32)(ui64TimeNow >> 32);
+				}
+			}
+		}
+	}
+	RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfGpuUtilFW, FLUSH);
+
+	OSLockRelease(psDevInfo->hGPUUtilLock);
+}
+
+static INLINE PVRSRV_ERROR RGXDoStop(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (psDeviceNode->psDevConfig->pfnTDRGXStop == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPrePowerState: TDRGXStop not implemented!"));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	psDevInfo->bRGXPowered = IMG_FALSE;
+	eError = psDeviceNode->psDevConfig->pfnTDRGXStop(psDeviceNode->psDevConfig->hSysData);
+#else
+	eError = RGXStop(&psDevInfo->sLayerParams);
+#endif
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXSendPowerOffKick
+@Description    Send a KCCB kick to power off the GPU FW. This function will wait
+                for completion of the command before exiting.
+
+@Input          psDeviceNode    The device node struct associated with the GPU.
+@Input          bForce          A boolean indicating if the power off command
+                                should be forced.
+
+@Return         Failure code if the virtual address is invalid.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXSendPowerOffKick(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        IMG_BOOL bForce)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_KCCB_CMD sPowCmd;
+	IMG_UINT32 ui32CmdKCCBSlot;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/* Send the Power off request to the FW */
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_OFF_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.bForced = bForce;
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+		         __func__));
+		return eError;
+	}
+
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send Power off request",
+		         __func__));
+		return eError;
+	}
+
+	/* Wait for the firmware to complete processing. It cannot use PVRSRVWaitForValueKM as it relies
+	 * on the EventObject which is signalled in this MISR */
+	return RGXPollForGPCommandCompletion(psDeviceNode,
+	                                     psDevInfo->psPowSyncPrim->pui32LinAddr,
+	                                     0x1, 0xFFFFFFFF);
+}
+
+/*************************************************************************/ /*!
+@Function       RGXFinalisePowerOff
+@Description    Finalises the GPU power transition to off.
+
+@Input          psDeviceNode    The device node struct associated with the GPU.
+
+@Return         Result code indicating the success or reason for failure.
+*/ /**************************************************************************/
+static PVRSRV_ERROR RGXFinalisePowerOff(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+#if !defined(NO_HARDWARE) && !defined(SUPPORT_SYNC_IRQ)
+	IMG_UINT32 ui32idx;
+#endif
+
+	PVR_ASSERT(psDeviceNode != NULL);
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+#if !defined(NO_HARDWARE)
+	/* Driver takes the VZ Fw-KM connection down, preventing the
+		* firmware from submitting further interrupts */
+	KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+	KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+#if defined(SUPPORT_SYNC_IRQ)
+	/* Wait for the pending IRQ handlers to complete. */
+	OSSyncIRQ(psDeviceNode->psDevConfig->ui32IRQ);
+#else
+#if defined(RGX_FW_IRQ_OS_COUNTERS)
+	ui32idx = RGXFW_HOST_DRIVER_ID;
+#else
+	for_each_irq_cnt(ui32idx)
+#endif /* RGX_FW_IRQ_OS_COUNTERS */
+	{
+		IMG_UINT32 ui32IrqCnt;
+
+		get_irq_cnt_val(ui32IrqCnt, ui32idx, psDevInfo);
+
+		/* Wait for the pending FW processor to host interrupts to come back. */
+		eError = PVRSRVPollForValueKM(psDeviceNode,
+		                              (IMG_UINT32 __iomem *)&psDevInfo->aui32SampleIRQCount[ui32idx],
+		                              ui32IrqCnt,
+		                              0xffffffff,
+		                              POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+		                              NULL);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Wait for pending interrupts failed (DevID %u)." MSG_IRQ_CNT_TYPE " %u Host: %u, FW: %u",
+			         __func__,
+			         psDeviceNode->sDevId.ui32InternalID,
+			         ui32idx,
+			         psDevInfo->aui32SampleIRQCount[ui32idx],
+			         ui32IrqCnt));
+
+			RGX_WaitForInterruptsTimeout(psDevInfo);
+#if !defined(RGX_FW_IRQ_OS_COUNTERS)
+			break;
+#endif
+		}
+	}
+#endif /* SUPPORT_SYNC_IRQ */
+#endif /* NO_HARDWARE */
+
+	/* Update GPU frequency and timer correlation related data */
+	RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_POWER);
+
+	/* Update GPU state counters */
+	_RGXUpdateGPUUtilStats(psDevInfo);
+
+#if defined(SUPPORT_LINUX_DVFS)
+	eError = SuspendDVFS(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to suspend DVFS", __func__));
+		return eError;
+	}
+#endif
+
+	eError = RGXDoStop(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		/* Power down failures are treated as successful since the power was removed but logged. */
+		PVR_DPF((PVR_DBG_WARNING, "%s: RGXDoStop failed (%s)",
+		         __func__, PVRSRVGetErrorString(eError)));
+		psDevInfo->ui32ActivePMReqNonIdle++;
+		eError = PVRSRV_OK;
+	}
+
+	return eError;
+}
+
+/*************************************************************************/ /*!
+@Function       RGXPrePowerState
+@Description    Initial step for setting power state, to be followed by
+                RGXPostPowerState.
+
+@Input          psDeviceNode        The device node struct associated with the GPU.
+@Input          eNewPowerState      The power state the GPU is to transition to.
+@Input          eCurrentPowerState  The current power state of the GPU.
+@Input          ePwrFlags           Flags indicating the behaviour of the transition.
+
+@Return         Result code indicating the success or reason for failure.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXPrePowerState(PVRSRV_DEVICE_NODE *psDeviceNode,
+                              PVRSRV_DEV_POWER_STATE eNewPowerState,
+                              PVRSRV_DEV_POWER_STATE eCurrentPowerState,
+                              PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*)psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData;
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	IMG_BOOL              bDeviceOk;
+#endif
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_ASSERT(psDevInfo != NULL);
+
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+
+	if ((eNewPowerState == eCurrentPowerState) ||
+	    (eNewPowerState == PVRSRV_DEV_POWER_STATE_ON))
+	{
+		return PVRSRV_OK;
+	}
+
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	bDeviceOk = (OSAtomicRead(&psDeviceNode->eHealthStatus) == PVRSRV_DEVICE_HEALTH_STATUS_OK);
+	if (bDeviceOk)
+#endif
+	{
+		IMG_BOOL bForce = IMG_FALSE;
+
+		if (BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED))
+		{
+			bForce = IMG_TRUE;
+		}
+
+		eError = RGXSendPowerOffKick(psDeviceNode, bForce);
+		if (eError == PVRSRV_ERROR_TIMEOUT)
+		{
+			/* timeout waiting for the FW to ack the request: return timeout */
+			PVR_DPF((PVR_DBG_WARNING,
+			         "%s: Timeout waiting for powoff ack from the FW",
+			         __func__));
+			return eError;
+		}
+		else if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Error waiting for powoff ack from the FW (%s)",
+			         __func__, PVRSRVGetErrorString(eError)));
+			return PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE;
+		}
+	}
+
+	/* Check the Power state after the answer */
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState, INVALIDATE);
+	if ((psFwSysData->ePowState != RGXFWIF_POW_OFF)
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+	    && (bDeviceOk)
+#endif
+		)
+	{
+		if (BITMASK_HAS(ePwrFlags, PVRSRV_POWER_FLAGS_FORCED))
+		{	/* It is an error for a forced request to be denied */
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failure to power off during a forced power off. FW: %d",
+			         __func__, psFwSysData->ePowState));
+		}
+
+		/* the sync was updated but the pow state isn't off -> the FW denied the transition */
+		return PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
+	}
+
+	return RGXFinalisePowerOff(psDeviceNode);
+}
+
+#if defined(SUPPORT_AUTOVZ)
+static PVRSRV_ERROR _RGXWaitForGuestsToDisconnect(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_ERROR_TIMEOUT;
+	IMG_UINT32 ui32FwTimeout = (20 * SECONDS_TO_MICROSECONDS);
+
+	LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+	{
+		IMG_UINT32 ui32DriverID;
+		IMG_BOOL bGuestOnline = IMG_FALSE;
+
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror,
+		                           INVALIDATE);
+
+		for (ui32DriverID = RGXFW_GUEST_DRIVER_ID_START;
+			 ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED; ui32DriverID++)
+		{
+			RGXFWIF_CONNECTION_FW_STATE eGuestState = (RGXFWIF_CONNECTION_FW_STATE)
+					psDevInfo->psRGXFWIfFwSysData->asOsRuntimeFlagsMirror[ui32DriverID].bfOsState;
+
+			if ((eGuestState == RGXFW_CONNECTION_FW_ACTIVE) ||
+				(eGuestState == RGXFW_CONNECTION_FW_GRACEFUL_OFFLOAD) ||
+				(eGuestState == RGXFW_CONNECTION_FW_FORCED_OFFLOAD))
+			{
+				bGuestOnline = IMG_TRUE;
+				PVR_DPF((PVR_DBG_WARNING, "%s: Guest OS %u still online.", __func__, ui32DriverID));
+			}
+		}
+
+		if (!bGuestOnline)
+		{
+			/* Allow Guests to finish reading Connection state registers before disconnecting. */
+			OSSleepms(100);
+
+			PVR_DPF((PVR_DBG_WARNING, "%s: All Guest connections are down. "
+									  "Host can power down the GPU.", __func__));
+			eError = PVRSRV_OK;
+			break;
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: Waiting for Guests to disconnect "
+									  "before powering down GPU.", __func__));
+
+			if (PVRSRVPwrLockIsLockedByMe(psDeviceNode))
+			{
+				/* Don't wait with the power lock held as this prevents the vz
+				 * watchdog thread from keeping the fw-km connection alive. */
+				PVRSRVPowerUnlock(psDeviceNode);
+			}
+		}
+
+		OSSleepms(10);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (!PVRSRVPwrLockIsLockedByMe(psDeviceNode))
+	{
+		/* Take back power lock after waiting for Guests */
+		eError = PVRSRVPowerLock(psDeviceNode);
+	}
+
+	return eError;
+}
+#endif /* defined(SUPPORT_AUTOVZ) */
+
+/*
+	RGXVzPrePowerState
+*/
+PVRSRV_ERROR RGXVzPrePowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                                PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                                PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", eError);
+
+	if (eNewPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering down */
+#if defined(SUPPORT_AUTOVZ)
+		if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode) && (!psDeviceNode->bAutoVzFwIsUp || psDeviceNode->bAutoVzAllowGPUPowerdown))
+		{
+			if (psDeviceNode->bAutoVzFwIsUp)
+			{
+				/* bAutoVzAllowGPUPowerdown must be TRUE here and
+				 * bAutoVzFwIsUp=TRUE indicates that this is a powerdown event
+				 * so send requests to the FW to disconnect all guest connections
+				 * before GPU is powered down. */
+				eError = RGXDisconnectAllGuests(psDeviceNode);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXDisconnectAllGuests");
+			}
+
+			/* The Host must ensure all Guest drivers have disconnected from the GPU before powering it down.
+			 * Guest drivers regularly access hardware registers during runtime. If an attempt is made to
+			 * access a GPU register while the GPU is down, the SoC might lock up. */
+			eError = _RGXWaitForGuestsToDisconnect(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "_RGXWaitForGuestsToDisconnect");
+
+			if (psDeviceNode->bAutoVzAllowGPUPowerdown)
+			{
+				psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+			}
+
+			/* Temporarily restore all power callbacks used by the driver to fully power down the GPU.
+			 * Under AutoVz, power transitions requests (e.g. on driver deinitialisation and unloading)
+			 * are generally ignored and the GPU power state is unaffected. Special power requests like
+			 * those triggered by Suspend/Resume calls must reinstate the callbacks when needed. */
+			PVRSRVSetPowerCallbacks(psDeviceNode, psDeviceNode->psPowerDev,
+									&RGXVzPrePowerState, &RGXVzPostPowerState,
+									psDeviceNode->psDevConfig->pfnPrePowerState,
+									psDeviceNode->psDevConfig->pfnPostPowerState,
+									&RGXForcedIdleRequest, &RGXCancelForcedIdleRequest);
+		}
+		else
+		{
+			PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+			KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+			KM_CONNECTION_CACHEOP(Os, INVALIDATE);
+
+			if (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo) &&
+				KM_OS_CONNECTION_IS(ACTIVE, psDevInfo))
+			{
+				PVRSRV_ERROR eError = RGXFWSetFwOsState(psDevInfo, 0, RGXFWIF_OS_OFFLINE);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXFWSetFwOsState");
+			}
+		}
+#endif
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering down: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+	}
+	else if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering up */
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering up: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+
+	}
+
+	if (!(PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp)))
+	{
+		/* call regular device power function */
+		eError = RGXPrePowerState(psDeviceNode, eNewPowerState, eCurrentPowerState, ePwrFlags);
+	}
+
+	return eError;
+}
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+static PVRSRV_ERROR RGXVzWaitFirmwareReady(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	if (!KM_FW_CONNECTION_IS(READY, psDevInfo))
+	{
+		PVR_LOG(("%s: Firmware Connection is not in Ready state. Waiting for Firmware ...", __func__));
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(RGX_VZ_CONNECTION_TIMEOUT_US)
+	{
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		if (KM_FW_CONNECTION_IS(READY, psDevInfo))
+		{
+			PVR_LOG(("%s: Firmware Connection is Ready. Initialisation proceeding.", __func__));
+			break;
+		}
+		else
+		{
+			OSSleepms(10);
+		}
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+	if (!KM_FW_CONNECTION_IS(READY, psDevInfo))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Timed out waiting for the Firmware to enter Ready state.", __func__));
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*
+	RGXVzPostPowerState
+*/
+PVRSRV_ERROR RGXVzPostPowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                                 PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                                 PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_LOG_RETURN_IF_FALSE((eNewPowerState != eCurrentPowerState), "no power change", eError);
+
+	if (!(PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp)))
+	{
+		if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+		{
+			KM_SET_OS_CONNECTION(READY, psDevInfo);
+			KM_CONNECTION_CACHEOP(Os, FLUSH);
+		}
+		/* call regular device power function */
+		eError = RGXPostPowerState(psDeviceNode, eNewPowerState, eCurrentPowerState, ePwrFlags);
+	}
+	else
+	{
+		KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	if (eNewPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering down */
+		if (psDeviceNode->bAutoVzFwIsUp)
+		{
+			PVR_LOG(("%s: AutoVz Fw active, power not changed", __func__));
+			return eError;
+		}
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering down: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+
+#if !defined(SUPPORT_AUTOVZ_HW_REGS)
+		/* The connection states must be reset on a GPU power cycle. If the states are kept
+		 * in hardware scratch registers, they will be cleared on power down. When using shared
+		 * memory the connection data must be explicitly cleared by the driver. */
+		OSCachedMemSetWMB(psDevInfo->psRGXFWIfConnectionCtl, 0, sizeof(RGXFWIF_CONNECTION_CTL));
+		RGXFwSharedMemCacheOpPtr(psDevInfo->psRGXFWIfConnectionCtl, FLUSH);
+#endif /* defined(SUPPORT_AUTOVZ) && !defined(SUPPORT_AUTOVZ_HW_REGS) */
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+		{
+#if defined(SUPPORT_AUTOVZ)
+			/* AutoVz Guests attempting to suspend have updated their connections earlier in RGXVzPrePowerState.
+			 * Skip this redundant register write, as the Host could have powered down the GPU by now. */
+			if (psDeviceNode->bAutoVzFwIsUp)
+#endif
+			{
+				/* Take the VZ connection down to prevent firmware from submitting further interrupts */
+				KM_SET_OS_CONNECTION(OFFLINE, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+			}
+			/* Power transition callbacks were not executed, update RGXPowered flag here */
+			psDevInfo->bRGXPowered = IMG_FALSE;
+		}
+	}
+	else if (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		/* powering up */
+		IMG_UINT32 ui32FwTimeout = (3 * SECONDS_TO_MICROSECONDS);
+		volatile IMG_BOOL *pbUpdatedFlag;
+		RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated,
+		                           INVALIDATE);
+		pbUpdatedFlag = &psDevInfo->psRGXFWIfOsInit->sRGXCompChecks.bUpdated;
+
+		PVR_DPF((PVR_DBG_WARNING, "%s: %s driver powering up: bAutoVzFwIsUp = %s",
+								__func__, PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode)? "GUEST" : "HOST",
+								psDeviceNode->bAutoVzFwIsUp ? "TRUE" : "FALSE"));
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Guests don't execute the power transition callbacks, so update their RGXPowered flag here */
+			psDevInfo->bRGXPowered = IMG_TRUE;
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+			/* Guest drivers expect the firmware to have set its end of the
+			 * connection to Ready state by now. */
+			eError = RGXVzWaitFirmwareReady(psDevInfo);
+			PVR_LOG_RETURN_IF_ERROR(eError, "RGXVzWaitFirmwareReady()");
+#endif /* RGX_VZ_STATIC_CARVEOUT_FW_HEAPS */
+
+			OSWriteDeviceMem32WithWMB(pbUpdatedFlag, IMG_FALSE);
+
+			/* Kick an initial dummy command to make the firmware initialise all
+			 * its internal guest OS data structures and compatibility information.
+			 * Use the lower-level RGXSendCommand() for the job, to make
+			 * sure only 1 KCCB command is issued to the firmware.
+			 * The default RGXFWHealthCheckCmd() prefaces each HealthCheck command with
+			 * a pre-kick cache command which can interfere with the FW-KM init handshake. */
+			{
+				RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+				sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+				KM_SET_OS_CONNECTION(READY, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+				eError = RGXSendCommand(psDevInfo, &sCmpKCCBCmd, PDUMP_FLAGS_CONTINUOUS);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXSendCommand()");
+			}
+		}
+		else
+		{
+#if defined(SUPPORT_AUTOVZ)
+			/* Disable power callbacks that should not be run on virtualised drivers after the GPU
+			 * is fully initialised: system layer pre/post functions and driver idle requests.
+			 * The original device RGX Pre/Post functions are called from this Vz wrapper. */
+			PVRSRVSetPowerCallbacks(psDeviceNode, psDeviceNode->psPowerDev,
+									&RGXVzPrePowerState, &RGXVzPostPowerState,
+									NULL, NULL, NULL, NULL);
+
+			/* AutoVz Host driver reconnecting to running Firmware */
+			if (psDeviceNode->bAutoVzFwIsUp)
+			{
+				/* Firmware already running, send a KCCB command to establish the new connection */
+				RGXFWIF_KCCB_CMD sCmpKCCBCmd;
+				sCmpKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_HEALTH_CHECK;
+
+				eError = RGXVzWaitFirmwareReady(psDevInfo);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXVzWaitFirmwareReady()");
+
+				KM_SET_OS_CONNECTION(READY, psDevInfo);
+				KM_CONNECTION_CACHEOP(Os, FLUSH);
+
+				eError = RGXSendCommand(psDevInfo, &sCmpKCCBCmd, PDUMP_FLAGS_CONTINUOUS);
+				PVR_LOG_RETURN_IF_ERROR(eError, "RGXSendCommand()");
+			}
+			else
+			{
+				/* During first-time boot the flag is set here, while subsequent reboots will already
+				 * have set it earlier in RGXInit. Set to true from this point on. */
+				psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+			}
+#endif
+		}
+
+		/* Wait for the firmware to accept and enable the connection with this OS by setting its state to Active */
+		LOOP_UNTIL_TIMEOUT_US(RGX_VZ_CONNECTION_TIMEOUT_US)
+		{
+			KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+			if (KM_FW_CONNECTION_IS(ACTIVE, psDevInfo))
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Connection is Active. Initialisation proceeding.", __func__));
+				break;
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_WARNING, "%s: Firmware Connection is not in Active state. Waiting for Firmware ...", __func__));
+				OSSleepms(10);
+			}
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		KM_CONNECTION_CACHEOP(Fw, INVALIDATE);
+		if (!KM_FW_CONNECTION_IS(ACTIVE, psDevInfo))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Timed out waiting for the Firmware to enter Active state.", __func__));
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* poll on the Firmware supplying the compatibility data */
+			LOOP_UNTIL_TIMEOUT_US(ui32FwTimeout)
+			{
+				if (*pbUpdatedFlag)
+				{
+					break;
+				}
+				OSSleepms(10);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			PVR_LOG_RETURN_IF_FALSE(*pbUpdatedFlag, "Firmware does not respond with compatibility data.", PVRSRV_ERROR_TIMEOUT);
+		}
+
+		KM_SET_OS_CONNECTION(ACTIVE, psDevInfo);
+		KM_CONNECTION_CACHEOP(Os, FLUSH);
+	}
+
+	return PVRSRV_OK;
+}
+
+#if defined(TRACK_FW_BOOT)
+static INLINE void RGXCheckFWBootStage(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	FW_BOOT_STAGE eStage;
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		/* Boot stage temporarily stored to the register below */
+		eStage = OSReadHWReg32(psDevInfo->pvRegsBaseKM,
+		                       RGX_FW_BOOT_STAGE_REGISTER);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		IMG_BYTE *pbBootData;
+
+		if (PVRSRV_OK != DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc,
+		                                          (void**)&pbBootData))
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Could not acquire pointer to FW boot stage", __func__));
+			eStage = FW_BOOT_STAGE_NOT_AVAILABLE;
+		}
+		else
+		{
+			pbBootData += RGXGetFWImageSectionOffset(NULL, MIPS_BOOT_DATA);
+
+			eStage = *(FW_BOOT_STAGE*)&pbBootData[RGXMIPSFW_BOOT_STAGE_OFFSET];
+
+			if (eStage == FW_BOOT_STAGE_TLB_INIT_FAILURE)
+			{
+				RGXMIPSFW_BOOT_DATA *psBootData =
+					(RGXMIPSFW_BOOT_DATA*) (pbBootData + RGXMIPSFW_BOOTLDR_CONF_OFFSET);
+
+				PVR_LOG(("MIPS TLB could not be initialised. Boot data info:"
+						 " num PT pages %u, log2 PT page size %u, PT page addresses"
+						 " %"IMG_UINT64_FMTSPECx " %"IMG_UINT64_FMTSPECx
+						 " %"IMG_UINT64_FMTSPECx " %"IMG_UINT64_FMTSPECx,
+						 psBootData->ui32PTNumPages,
+						 psBootData->ui32PTLog2PageSize,
+						 psBootData->aui64PTPhyAddr[0U],
+						 psBootData->aui64PTPhyAddr[1U],
+						 psBootData->aui64PTPhyAddr[2U],
+						 psBootData->aui64PTPhyAddr[3U]));
+			}
+
+			DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+		}
+	}
+#endif
+	else
+	{
+		eStage = OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SCRATCH14);
+	}
+
+	PVR_LOG(("%s: FW reached boot stage %i/%i.",
+	         __func__, eStage, FW_BOOT_INIT_DONE));
+}
+#endif
+
+static INLINE PVRSRV_ERROR RGXDoStart(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+
+	if (psDevConfig->pfnTDRGXStart == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPostPowerState: TDRGXStart not implemented!"));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+	eError = psDevConfig->pfnTDRGXStart(psDevConfig->hSysData);
+
+	if (eError == PVRSRV_OK)
+	{
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+#else
+	eError = RGXStart(&psDevInfo->sLayerParams);
+#endif
+
+	return eError;
+}
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+/*
+ * To validate the MTS unit we do the following:
+ *  - Immediately after firmware loading for each OSID
+ *    - Write the OSid to a memory location shared with FW
+ *    - Kick the register of that OSid
+ *         (Uncounted, DM 0)
+ *    - FW clears the memory location if OSid matches
+ *    - Host checks that memory location is cleared
+ *
+ *  See firmware/rgxfw_bg.c
+ */
+static PVRSRV_ERROR RGXVirtualisationPowerupSidebandTest(PVRSRV_DEVICE_NODE	 *psDeviceNode,
+														 RGXFWIF_SYSINIT *psFwSysInit,
+														 PVRSRV_RGXDEV_INFO	 *psDevInfo)
+{
+	IMG_UINT32 ui32ScheduleRegister;
+	IMG_UINT32 ui32OSid;
+	IMG_UINT32 ui32KickType;
+	IMG_UINT32 ui32OsRegBanksMapped = (psDeviceNode->psDevConfig->ui32RegsSize / RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+
+	/* Nothing to do if the device does not support GPU_VIRTUALISATION */
+	if (!PVRSRV_IS_FEATURE_SUPPORTED(psDeviceNode, GPU_VIRTUALISATION))
+	{
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Testing per-os kick registers:"));
+
+	ui32OsRegBanksMapped = MIN(ui32OsRegBanksMapped, GPUVIRT_VALIDATION_NUM_OS);
+
+	if (ui32OsRegBanksMapped != RGXFW_MAX_NUM_OSIDS)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "The register bank mapped into kernel VA does not cover all OS' registers:"));
+		PVR_DPF((PVR_DBG_WARNING, "Maximum OS count = %d / Per-os register banks mapped = %d", RGXFW_MAX_NUM_OSIDS, ui32OsRegBanksMapped));
+		PVR_DPF((PVR_DBG_WARNING, "Only first %d MTS registers will be tested", ui32OsRegBanksMapped));
+	}
+
+	ui32KickType = RGX_CR_MTS_SCHEDULE_DM_DM0 | RGX_CR_MTS_SCHEDULE_TASK_NON_COUNTED;
+
+	for (ui32OSid = 0; ui32OSid < ui32OsRegBanksMapped; ui32OSid++)
+	{
+		/* set Test field */
+		psFwSysInit->ui32OSKickTest = (ui32OSid << RGXFWIF_KICK_TEST_OSID_SHIFT) | RGXFWIF_KICK_TEST_ENABLED_BIT;
+
+#if defined(PDUMP)
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfSysInitMemDesc,
+						   offsetof(RGXFWIF_SYSINIT, ui32OSKickTest),
+						   sizeof(psFwSysInit->ui32OSKickTest),
+						   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+		/* Force a read-back to memory to avoid posted writes on certain buses */
+		OSWriteMemoryBarrier(&psFwSysInit->ui32OSKickTest);
+		RGXFwSharedMemCacheOpValue(psFwSysInit->ui32OSKickTest, FLUSH);
+
+		/* kick register */
+		ui32ScheduleRegister = RGX_CR_MTS_SCHEDULE + (ui32OSid * RGX_VIRTUALISATION_REG_SIZE_PER_OS);
+		PVR_DPF((PVR_DBG_MESSAGE, "  Testing OS: %u, Kick Reg: %X",
+				 ui32OSid,
+				 ui32ScheduleRegister));
+		OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32ScheduleRegister, ui32KickType);
+
+#if defined(PDUMP)
+		PDUMPCOMMENTWITHFLAGS(psDevInfo->psDeviceNode, PDUMP_FLAGS_CONTINUOUS, "VZ sideband test, kicking MTS register %u", ui32OSid);
+
+		PDUMPREG32(psDeviceNode, RGX_PDUMPREG_NAME,
+				ui32ScheduleRegister, ui32KickType, PDUMP_FLAGS_CONTINUOUS);
+
+		DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfSysInitMemDesc,
+							   offsetof(RGXFWIF_SYSINIT, ui32OSKickTest),
+							   0,
+							   0xFFFFFFFF,
+							   PDUMP_POLL_OPERATOR_EQUAL,
+							   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if !defined(NO_HARDWARE)
+		OSMemoryBarrier((IMG_BYTE*) psDevInfo->pvRegsBaseKM + ui32ScheduleRegister);
+
+		/* Wait test enable bit to be unset */
+		if (PVRSRVPollForValueKM(psDeviceNode,
+								 (volatile IMG_UINT32 __iomem *)&psFwSysInit->ui32OSKickTest,
+								 0,
+								 RGXFWIF_KICK_TEST_ENABLED_BIT,
+								 POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+								 RGXFwSharedMemCacheOpExecPfn) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Testing OS %u kick register failed: firmware did not clear test location (contents: 0x%X)",
+					 ui32OSid,
+					 psFwSysInit->ui32OSKickTest));
+
+			return PVRSRV_ERROR_TIMEOUT;
+		}
+
+		/* Check that the value is what we expect */
+		if (psFwSysInit->ui32OSKickTest != 0)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Testing OS %u kick register failed: firmware wrote 0x%X to test location",
+					 ui32OSid,
+					 psFwSysInit->ui32OSKickTest));
+			return PVRSRV_ERROR_INIT_FAILURE;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "    PASS"));
+#endif
+	}
+
+	PVR_LOG(("MTS passed sideband tests"));
+	return PVRSRV_OK;
+}
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION_MTS) */
+
+
+/*
+	RGXPostPowerState
+*/
+PVRSRV_ERROR RGXPostPowerState(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                               PVRSRV_DEV_POWER_STATE	eNewPowerState,
+                               PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+                               PVRSRV_POWER_FLAGS		ePwrFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(ePwrFlags);
+
+	if ((eNewPowerState == eCurrentPowerState) ||
+	    (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGXPostPowerState: Current state: %d, New state: %d",
+		             eCurrentPowerState, eNewPowerState);
+
+		return PVRSRV_OK;
+	}
+
+	/* Update timer correlation related data */
+	RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_POWER);
+
+	/* Update GPU state counters */
+	_RGXUpdateGPUUtilStats(psDevInfo);
+
+	eError = RGXDoStart(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXDoStart", fail);
+
+	OSMemoryBarrier(NULL);
+
+
+	/*
+		* Check whether the FW has started by polling on bFirmwareStarted flag
+		*/
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->bFirmwareStarted,
+	                           INVALIDATE);
+	if (PVRSRVPollForValueKM(psDeviceNode,
+	                         (IMG_UINT32 __iomem *)&psDevInfo->psRGXFWIfSysInit->bFirmwareStarted,
+	                         IMG_TRUE,
+	                         0xFFFFFFFF,
+	                         POLL_FLAG_LOG_ERROR | POLL_FLAG_DEBUG_DUMP,
+	                         RGXFwSharedMemCacheOpExecPfn) != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "RGXPostPowerState: Polling for 'FW started' flag failed."));
+		eError = PVRSRV_ERROR_TIMEOUT;
+
+#if defined(TRACK_FW_BOOT)
+		RGXCheckFWBootStage(psDevInfo);
+#endif
+
+		/*
+		* When bFirmwareStarted fails some info may be gained by doing the following
+		* debug dump but unfortunately it could be potentially dangerous if the reason
+		* for not booting is the GPU power is not ON. However, if we have reached this
+		* point the System Layer has returned without errors, we assume the GPU power
+		* is indeed ON.
+		*/
+		RGXDumpRGXDebugSummary(NULL, NULL, psDeviceNode->pvDevice, IMG_TRUE);
+		RGXDumpRGXRegisters(NULL, NULL, psDeviceNode->pvDevice);
+
+		PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPollForValueKM(bFirmwareStarted)", fail);
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS, "Wait for the Firmware to start.");
+	eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfSysInitMemDesc,
+	                                offsetof(RGXFWIF_SYSINIT, bFirmwareStarted),
+	                                IMG_TRUE,
+	                                0xFFFFFFFFU,
+	                                PDUMP_POLL_OPERATOR_EQUAL,
+	                                PDUMP_FLAGS_CONTINUOUS);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "RGXPostPowerState: problem pdumping POL for psRGXFWIfSysInitMemDesc (%d)",
+		         eError));
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemPDumpDevmemPol32", fail);
+	}
+
+#endif /* defined(PDUMP) */
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION_MTS)
+	eError = RGXVirtualisationPowerupSidebandTest(psDeviceNode, psDevInfo->psRGXFWIfSysInit, psDevInfo);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXVirtualisationPowerupSidebandTest", fail);
+#endif
+
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp,
+	                           INVALIDATE);
+	PVRSRVSetFirmwareStartTime(psDeviceNode->psPowerDev,
+	                           psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32FirmwareStartedTimeStamp,
+	                           FLUSH);
+#endif
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfSysInit->ui32MarkerVal,
+	                           INVALIDATE);
+	HTBSyncPartitionMarker(psDevInfo->psRGXFWIfSysInit->ui32MarkerVal);
+
+#if defined(SUPPORT_LINUX_DVFS)
+	eError = ResumeDVFS(psDeviceNode);
+	PVR_LOG_GOTO_IF_ERROR(eError, "ResumeDVFS", fail);
+#endif
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXPostPowerState: Current state: %d, New state: %d",
+	             eCurrentPowerState, eNewPowerState);
+
+	return eError;
+
+fail:
+	psDevInfo->bRGXPowered = IMG_FALSE;
+
+	return eError;
+}
+
+/*
+	RGXPreClockSpeedChange
+*/
+PVRSRV_ERROR RGXPreClockSpeedChange(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                    PVRSRV_DEV_POWER_STATE	eCurrentPowerState)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	const PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	__maybe_unused const RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "RGXPreClockSpeedChange: RGX clock speed was %uHz",
+			psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF) &&
+	    (psFwSysData->ePowState != RGXFWIF_POW_OFF))
+	{
+		/* Update GPU frequency and timer correlation related data */
+		RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+	}
+
+	return eError;
+}
+
+/*
+	RGXPostClockSpeedChange
+*/
+PVRSRV_ERROR RGXPostClockSpeedChange(PVRSRV_DEVICE_NODE		*psDeviceNode,
+                                     PVRSRV_DEV_POWER_STATE	eCurrentPowerState)
+{
+	const PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32NewClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	/* Update runtime configuration with the new value */
+	OSWriteDeviceMem32WithWMB(&psDevInfo->psRGXFWIfRuntimeCfg->ui32CoreClockSpeed,
+	                          ui32NewClockSpeed);
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfRuntimeCfg->ui32CoreClockSpeed, FLUSH);
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+	if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF) &&
+	    (psFwSysData->ePowState != RGXFWIF_POW_OFF))
+	{
+		RGXFWIF_KCCB_CMD sCOREClkSpeedChangeCmd;
+		IMG_UINT32 ui32CmdKCCBSlot;
+
+		RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+
+		sCOREClkSpeedChangeCmd.eCmdType = RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE;
+		sCOREClkSpeedChangeCmd.uCmdData.sCoreClkSpeedChangeData.ui32NewClockSpeed = ui32NewClockSpeed;
+
+		PDUMPCOMMENT(psDeviceNode, "Scheduling CORE clock speed change command");
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+		                                      &sCOREClkSpeedChangeCmd,
+		                                      PDUMP_FLAGS_NONE,
+		                                      &ui32CmdKCCBSlot);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Scheduling CORE clock speed change command failed");
+			PVR_DPF((PVR_DBG_ERROR, "RGXPostClockSpeedChange: Scheduling KCCB command failed. Error:%u", eError));
+			return eError;
+		}
+
+#if defined(PVRSRV_ANDROID_TRACE_GPU_FREQ)
+		GpuTraceFrequency(psDeviceNode->sDevId.ui32InternalID,
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed);
+#endif /* defined(PVRSRV_ANDROID_TRACE_GPU_FREQ) */
+
+		PVR_DPF((PVR_DBG_MESSAGE, "RGXPostClockSpeedChange: RGX clock speed changed to %uHz",
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
+	}
+
+	return eError;
+}
+
+/*!
+******************************************************************************
+
+ @Function	RGXPowUnitsChange
+
+ @Description Change power units state
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32PowUnits : On Rogue: Number of DUSTs to make transition to.
+                          On Volcanic: Mask containing power state of SPUs.
+                          Each bit corresponds to an SPU and value must be non-zero.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPowUnitsChange(PVRSRV_DEVICE_NODE *psDeviceNode,
+                               IMG_UINT32 ui32PowUnits)
+
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError;
+	RGXFWIF_KCCB_CMD	sPowUnitsChange;
+	IMG_UINT32			ui32AvailablePowUnits;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	RGXFWIF_RUNTIME_CFG *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AvailablePowUnits = psDevInfo->ui32AvailablePowUnitsMask;
+
+	/**
+	 * Validate the input. At-least one PU must be powered on and all requested
+	 * PU's must be a subset of full PU mask.
+	 */
+	if ((ui32PowUnits == 0) || (ui32PowUnits & ~ui32AvailablePowUnits))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid Power Units mask requested (0x%X). Value should be non-zero and sub-set of 0x%X mask",
+				__func__,
+				ui32PowUnits,
+				ui32AvailablePowUnits));
+		return PVRSRV_ERROR_INVALID_SPU_MASK;
+	}
+#else
+	ui32AvailablePowUnits = psDevInfo->sDevFeatureCfg.ui32MAXPowUnitCount;
+
+	if (ui32PowUnits > ui32AvailablePowUnits)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Invalid number of DUSTs (%u) while expecting value within <0,%u>",
+				__func__,
+				ui32PowUnits,
+				ui32AvailablePowUnits));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+
+	psRuntimeCfg->ui32PowUnitsState = ui32PowUnits;
+	OSWriteMemoryBarrier(&psRuntimeCfg->ui32PowUnitsState);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ui32PowUnitsState, FLUSH);
+
+#if !defined(NO_HARDWARE)
+	{
+		const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+		RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+		                           INVALIDATE);
+
+		if (psFwSysData->ePowState == RGXFWIF_POW_OFF)
+		{
+			return PVRSRV_OK;
+		}
+
+		if (psFwSysData->ePowState != RGXFWIF_POW_FORCED_IDLE)
+		{
+			eError = PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Powered units state can not be changed when not IDLE",
+					 __func__));
+			return eError;
+		}
+	}
+#endif
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+				__func__));
+		return eError;
+	}
+
+	sPowUnitsChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowUnitsChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_NUM_UNITS_CHANGE;
+	sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32PowUnits = ui32PowUnits;
+#if defined(RGX_FEATURE_POWER_ISLAND_VERSION_MAX_VALUE_IDX)
+	sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32RACUnits = 0;
+
+	if (RGX_GET_FEATURE_VALUE(psDevInfo, POWER_ISLAND_VERSION) >= 3)
+	{
+		sPowUnitsChange.uCmdData.sPowData.uPowerReqData.ui32RACUnits =
+			(1U << psDevInfo->sDevFeatureCfg.ui32MAXRACCount) - 1;
+	}
+#endif
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "Scheduling command to change power units state to 0x%X",
+	             ui32PowUnits);
+	eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+	                                      &sPowUnitsChange,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "Scheduling command to change power units state. Error:%u",
+		             eError);
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Scheduling KCCB to change power units state. Error:%u",
+				 __func__, eError));
+		return eError;
+	}
+
+	/* Wait for the firmware to answer. */
+	eError = RGXPollForGPCommandCompletion(psDeviceNode,
+	                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+								  0x1, 0xFFFFFFFF);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Timeout waiting for idle request", __func__));
+		return eError;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "%s: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             __func__, psDevInfo->psPowSyncPrim->pui32LinAddr,
+	             RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+ @Function	RGXAPMLatencyChange
+*/
+PVRSRV_ERROR RGXAPMLatencyChange(PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 IMG_UINT32			ui32ActivePMLatencyms,
+                                 IMG_BOOL			bActivePMLatencyPersistant)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR		eError;
+	RGXFWIF_RUNTIME_CFG	*psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	PVRSRV_DEV_POWER_STATE	ePowerState;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	if (psRuntimeCfg == NULL)
+	{
+		return PVRSRV_ERROR_NOT_INITIALISED;
+	}
+
+	/* Update runtime configuration with the new values and ensure the
+	 * new APM latency is written to memory before requesting the FW to
+	 * read it
+	 */
+	psRuntimeCfg->ui32ActivePMLatencyms = ui32ActivePMLatencyms;
+	psRuntimeCfg->bActivePMLatencyPersistant = bActivePMLatencyPersistant;
+	OSWriteMemoryBarrier(&psRuntimeCfg->bActivePMLatencyPersistant);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->ui32ActivePMLatencyms, FLUSH);
+	RGXFwSharedMemCacheOpValue(psRuntimeCfg->bActivePMLatencyPersistant, FLUSH);
+
+	eError = PVRSRVPowerLock(psDeviceNode);
+	if (eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF)
+	{
+		/* Power is off, APM latency will be read on next firmware boot */
+		return PVRSRV_OK;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to acquire power lock (%u)",
+		         __func__, eError));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+
+	if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
+	{
+		RGXFWIF_KCCB_CMD	sActivePMLatencyChange;
+		sActivePMLatencyChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
+		sActivePMLatencyChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_APM_LATENCY_CHANGE;
+
+		PDUMPCOMMENT(psDeviceNode,
+		             "Scheduling command to change APM latency to %u",
+		             ui32ActivePMLatencyms);
+		eError = RGXSendCommandAndGetKCCBSlot(psDeviceNode->pvDevice,
+		                                      &sActivePMLatencyChange,
+		                                      PDUMP_FLAGS_NONE,
+		                                      &ui32CmdKCCBSlot);
+
+		if (eError != PVRSRV_OK)
+		{
+			PDUMPCOMMENT(psDeviceNode,
+			             "Scheduling command to change APM latency failed. Error:%u",
+			             eError);
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Scheduling KCCB to change APM latency failed. Error:%u",
+			         __func__, eError));
+			goto ErrorExit;
+		}
+	}
+
+ErrorExit:
+	PVRSRVPowerUnlock(psDeviceNode);
+
+	return eError;
+}
+
+/*
+	RGXActivePowerRequest
+*/
+PVRSRV_ERROR RGXActivePowerRequest(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	const RGXFWIF_SYSDATA *psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+
+	psDevInfo->ui32ActivePMReqTotal++;
+
+	/* Powerlock to avoid further requests from racing with the FW hand-shake
+	 * from now on (previous kicks to this point are detected by the FW)
+	 * PVRSRVPowerLock is replaced with PVRSRVPowerTryLock to avoid
+	 * potential dead lock between PDumpWriteLock and PowerLock
+	 * during 'DriverLive + PDUMP=1 + EnableAPM=1'.
+	 */
+	eError = PVRSRVPowerTryLock(psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			PVR_LOG_ERROR(eError, "PVRSRVPowerTryLock");
+		}
+		else
+		{
+			psDevInfo->ui32ActivePMReqRetry++;
+		}
+		goto _RGXActivePowerRequest_PowerLock_failed;
+	}
+
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	/* Check again for IDLE once we have the power lock */
+	if (psFwSysData->ePowState == RGXFWIF_POW_IDLE)
+	{
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		PVRSRVSetFirmwareHandshakeIdleTime(psDeviceNode->psPowerDev,
+		                                   RGXReadHWTimerReg(psDevInfo)-psFwSysData->ui64StartIdleTime);
+#endif
+
+		PDUMPPOWCMDSTART(psDeviceNode);
+		eError = PVRSRVSetDevicePowerStateKM(psDeviceNode,
+		                                     PVRSRV_DEV_POWER_STATE_OFF,
+		                                     PVRSRV_POWER_FLAGS_NONE);
+		PDUMPPOWCMDEND(psDeviceNode);
+
+		if (eError == PVRSRV_OK)
+		{
+			psDevInfo->ui32ActivePMReqOk++;
+		}
+		else if (eError == PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
+		{
+			psDevInfo->ui32ActivePMReqDenied++;
+		}
+	}
+	else
+	{
+		psDevInfo->ui32ActivePMReqNonIdle++;
+	}
+
+	PVRSRVPowerUnlock(psDeviceNode);
+
+_RGXActivePowerRequest_PowerLock_failed:
+
+	return eError;
+}
+/*
+	RGXForcedIdleRequest
+*/
+
+#define RGX_FORCED_IDLE_RETRY_COUNT 10
+
+PVRSRV_ERROR RGXForcedIdleRequest(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_BOOL bDeviceOffPermitted)
+{
+	PVRSRV_RGXDEV_INFO    *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD      sPowCmd;
+	PVRSRV_ERROR          eError;
+	IMG_UINT32            ui32RetryCount = 0;
+	IMG_UINT32            ui32CmdKCCBSlot;
+#if !defined(NO_HARDWARE)
+	const RGXFWIF_SYSDATA *psFwSysData;
+#endif
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+#if !defined(NO_HARDWARE)
+	psFwSysData = psDevInfo->psRGXFWIfFwSysData;
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+
+	/* Firmware already forced idle */
+	if (psFwSysData->ePowState == RGXFWIF_POW_FORCED_IDLE)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Firmware is not powered. Sometimes this is permitted, for instance we were forcing idle to power down. */
+	if (psFwSysData->ePowState == RGXFWIF_POW_OFF)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Firmware is powered OFF (bDeviceOffPermitted = %s)",
+				 bDeviceOffPermitted ? "Yes" : "No"));
+		return (bDeviceOffPermitted) ? PVRSRV_OK : PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
+	}
+#endif
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+				__func__));
+		return eError;
+	}
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_FORCE_IDLE;
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Sending forced idle command");
+
+	/* Send one forced IDLE command to GP */
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to send idle request", __func__));
+		return eError;
+	}
+
+	/* Wait for GPU to finish current workload */
+	do {
+		eError = RGXPollForGPCommandCompletion(psDeviceNode,
+		                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+									  0x1, 0xFFFFFFFF);
+		if ((eError == PVRSRV_OK) || (ui32RetryCount == RGX_FORCED_IDLE_RETRY_COUNT))
+		{
+			break;
+		}
+		ui32RetryCount++;
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Request timeout. Retry %d of %d",
+				 __func__, ui32RetryCount, RGX_FORCED_IDLE_RETRY_COUNT));
+	} while (IMG_TRUE);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXFWNotifyHostTimeout(psDevInfo);
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Idle request failed. Firmware potentially left in forced idle state",
+				 __func__));
+		return eError;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+#if !defined(NO_HARDWARE)
+	/* Check the firmware state for idleness */
+	RGXFwSharedMemCacheOpValue(psFwSysData->ePowState,
+	                           INVALIDATE);
+	if (psFwSysData->ePowState != RGXFWIF_POW_FORCED_IDLE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "FW power state (%u) is not RGXFWIF_POW_FORCED_IDLE (%u)",
+				 psFwSysData->ePowState, RGXFWIF_POW_FORCED_IDLE));
+		return PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
+	}
+#endif
+
+	return PVRSRV_OK;
+}
+
+/*
+	RGXCancelForcedIdleRequest
+*/
+PVRSRV_ERROR RGXCancelForcedIdleRequest(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_KCCB_CMD	sPowCmd;
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32CmdKCCBSlot;
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_OK);
+
+	eError = SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set Power sync prim",
+				__func__));
+		goto ErrorExit;
+	}
+
+	/* Send the IDLE request to the FW */
+	sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
+	sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
+	sPowCmd.uCmdData.sPowData.uPowerReqData.ePowRequestType = RGXFWIF_POWER_CANCEL_FORCED_IDLE;
+
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXForcedIdleRequest: Sending cancel forced idle command");
+
+	/* Send cancel forced IDLE command to GP */
+	eError = RGXSendCommandAndGetKCCBSlot(psDevInfo,
+	                                      &sPowCmd,
+	                                      PDUMP_FLAGS_NONE,
+	                                      &ui32CmdKCCBSlot);
+
+	if (eError != PVRSRV_OK)
+	{
+		PDUMPCOMMENT(psDeviceNode,
+		             "RGXCancelForcedIdleRequest: Failed to send cancel IDLE request for DM%d",
+		             RGXFWIF_DM_GP);
+		goto ErrorExit;
+	}
+
+	/* Wait for the firmware to answer. */
+	eError = RGXPollForGPCommandCompletion(psDeviceNode,
+	                              psDevInfo->psPowSyncPrim->pui32LinAddr,
+								  1, 0xFFFFFFFF);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Timeout waiting for cancel idle request", __func__));
+		goto ErrorExit;
+	}
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGXCancelForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d",
+	             psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
+
+	SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
+	                 1,
+	                 0xffffffff,
+	                 PDUMP_POLL_OPERATOR_EQUAL,
+	                 0);
+#endif
+
+	return eError;
+
+ErrorExit:
+	PVR_DPF((PVR_DBG_ERROR, "%s: Firmware potentially left in forced idle state", __func__));
+	return eError;
+}
+
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+/*************************************************************************/ /*!
+@Function       PDVFSProcessCoreClkChangeRequest
+@Description    Processes a core clock rate change request.
+@Input          psDevInfo            A pointer to PVRSRV_RGXDEV_INFO.
+@Input          ui32CoreClockRate    New core clock rate.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeRequest(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	IMG_DVFS_DEVICE_CFG *psDVFSDeviceCfg = &psDevConfig->sDVFS.sDVFSDeviceCfg;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = ((RGX_DATA*)(psDevConfig->hDevData))->psRGXTimingInfo;
+	IMG_UINT32 ui32CoreClockRateCurrent = psRGXTimingInfo->ui32CoreClockSpeed;
+	const IMG_OPP *psOpp = NULL;
+	IMG_UINT32 ui32Index;
+	PVRSRV_ERROR eError;
+
+	if (!_PDVFSEnabled())
+	{
+		/* No error message to avoid excessive messages */
+		return PVRSRV_OK;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Core clock rate = %u", ui32CoreClockRate));
+
+	/* Find the matching OPP (Exact). */
+	for (ui32Index = 0; ui32Index < psDVFSDeviceCfg->ui32OPPTableSize; ui32Index++)
+	{
+		if (ui32CoreClockRate == psDVFSDeviceCfg->pasOPPTable[ui32Index].ui32Freq)
+		{
+			psOpp = &psDVFSDeviceCfg->pasOPPTable[ui32Index];
+			break;
+		}
+	}
+
+	if (! psOpp)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Frequency not present in OPP table - %u", ui32CoreClockRate));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	eError = PVRSRVDevicePreClockSpeedChange(psDevInfo->psDeviceNode, psDVFSDeviceCfg->bIdleReq, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PVRSRVDevicePreClockSpeedChange failed"));
+		return eError;
+	}
+
+	psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+
+	/* Increasing frequency, change voltage first */
+	if (ui32CoreClockRate > ui32CoreClockRateCurrent)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDevConfig->hSysData, psOpp->ui32Volt);
+	}
+
+	psDVFSDeviceCfg->pfnSetFrequency(psDevConfig->hSysData, ui32CoreClockRate);
+
+	/* Decreasing frequency, change frequency first */
+	if (ui32CoreClockRate < ui32CoreClockRateCurrent)
+	{
+		psDVFSDeviceCfg->pfnSetVoltage(psDevConfig->hSysData, psOpp->ui32Volt);
+	}
+
+	PVRSRVDevicePostClockSpeedChange(psDevInfo->psDeviceNode, psDVFSDeviceCfg->bIdleReq, NULL);
+
+	return PVRSRV_OK;
+}
+#else
+/*************************************************************************/ /*!
+@Function       PDVFSProcessCoreClkChangeNotification
+@Description    Processes a core clock rate change notification.
+@Input          psDevInfo            A pointer to PVRSRV_RGXDEV_INFO.
+@Input          ui32CoreClockRate    New core clock rate.
+@Return         PVRSRV_ERROR.
+*/ /**************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeNotification(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDevInfo->psDeviceNode->psDevConfig;
+	RGX_TIMING_INFORMATION *psRGXTimingInfo = ((RGX_DATA*)(psDevConfig->hDevData))->psRGXTimingInfo;
+	PVRSRV_DEV_POWER_STATE ePowerState;
+	PVRSRV_ERROR eError;
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire lock (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		return eError;
+	}
+
+	eError = PVRSRVGetDevicePowerState(psDevInfo->psDeviceNode, &ePowerState);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: failed to acquire power state (%s)",
+				 __func__, PVRSRVGetErrorString(eError)));
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+		return eError;
+	}
+
+	/* Guest drivers do not initialize psRGXFWIfFwSysData */
+	RGXFwSharedMemCacheOpValue(psDevInfo->psRGXFWIfFwSysData->ePowState,
+	                           INVALIDATE);
+	if ((ePowerState != PVRSRV_DEV_POWER_STATE_OFF)
+	    && ((psDevInfo->psRGXFWIfFwSysData == NULL) || (psDevInfo->psRGXFWIfFwSysData->ePowState != RGXFWIF_POW_OFF)))
+	{
+		/* Update GPU frequency and timer correlation related data */
+		RGXTimeCorrEnd(psDevInfo->psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+		psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+		RGXTimeCorrBegin(psDevInfo->psDeviceNode, RGXTIMECORR_EVENT_DVFS);
+	}
+	else
+	{
+		psRGXTimingInfo->ui32CoreClockSpeed = ui32CoreClockRate;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+	return PVRSRV_OK;
+}
+#endif
+#endif /* SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY */
+
+
+
+/******************************************************************************
+ End of file (rgxpower.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxpower.h b/drivers/gpu/drm/img-rogue/rgxpower.h
new file mode 100644
index 000000000000..dbe9130d7a7b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxpower.h
@@ -0,0 +1,296 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX power header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX power
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXPOWER_H)
+#define RGXPOWER_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "servicesext.h"
+#include "rgxdevice.h"
+
+
+/*!
+******************************************************************************
+
+ @Function	RGXPrePowerState
+
+ @Description
+
+ does necessary preparation before power state transition
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPrePowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+							  PVRSRV_DEV_POWER_STATE	eNewPowerState,
+							  PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+							  PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPostPowerState
+
+ @Description
+
+ does necessary preparation after power state transition
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPostPowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+							   PVRSRV_DEV_POWER_STATE	eNewPowerState,
+							   PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+							   PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXVzPrePowerState
+
+ @Description
+
+ does necessary preparation before power state transition on a vz driver
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXVzPrePowerState(PPVRSRV_DEVICE_NODE		psDeviceNode,
+								PVRSRV_DEV_POWER_STATE	eNewPowerState,
+								PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+								PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXVzPostPowerState
+
+ @Description
+
+ does necessary preparation after power state transition on a vz driver
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eNewPowerState : New power state
+ @Input	   eCurrentPowerState : Current power state
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXVzPostPowerState(PPVRSRV_DEVICE_NODE	psDeviceNode,
+								 PVRSRV_DEV_POWER_STATE	eNewPowerState,
+								 PVRSRV_DEV_POWER_STATE	eCurrentPowerState,
+								 PVRSRV_POWER_FLAGS		ePwrFlags);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPreClockSpeedChange
+
+ @Description
+
+	Does processing required before an RGX clock speed change.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eCurrentPowerState : Power state of the device
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPreClockSpeedChange(PPVRSRV_DEVICE_NODE		psDeviceNode,
+									PVRSRV_DEV_POWER_STATE	eCurrentPowerState);
+
+/*!
+******************************************************************************
+
+ @Function	RGXPostClockSpeedChange
+
+ @Description
+
+	Does processing required after an RGX clock speed change.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   eCurrentPowerState : Power state of the device
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPostClockSpeedChange(PPVRSRV_DEVICE_NODE	psDeviceNode,
+									 PVRSRV_DEV_POWER_STATE	eCurrentPowerState);
+
+
+#if defined(SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY)
+#if defined(SUPPORT_PDVFS) && (PDVFS_COM == PDVFS_COM_HOST)
+/*!
+******************************************************************************
+
+ @Function	RGXProcessCoreClkChangeRequest
+
+ @Input	   psDevInfo : RGX Device Info
+ @Input	   ui32CoreClockRate : New clock frequency to send to system layer.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeRequest(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate);
+#define RGX_PROCESS_CORE_CLK_RATE_CHANGE(devinfo, clk)  RGXProcessCoreClkChangeRequest(devinfo, clk)
+
+#else
+/*!
+******************************************************************************
+
+ @Function	RGXProcessCoreClkChangeNotification
+
+ @Input	   psDevInfo : RGX Device Info
+ @Input	   ui32CoreClockRate : New clock frequency.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXProcessCoreClkChangeNotification(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32CoreClockRate);
+#define RGX_PROCESS_CORE_CLK_RATE_CHANGE(devinfo, clk)  RGXProcessCoreClkChangeNotification(devinfo, clk)
+#endif
+#endif /* SUPPORT_FW_CORE_CLK_RATE_CHANGE_NOTIFY */
+
+/*!
+******************************************************************************
+
+ @Function	RGXPowUnitsChange
+
+ @Description Change power units state
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32PowUnits : On Rogue: Number of DUSTs to make transition to.
+                          On Volcanic: Mask containing power state of SPUs.
+                          Each bit corresponds to an SPU and value must be non-zero.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXPowUnitsChange(PPVRSRV_DEVICE_NODE psDeviceNode,
+                               IMG_UINT32 ui32PowUnits);
+
+/*!
+******************************************************************************
+
+ @Function	RGXAPMLatencyChange
+
+ @Description
+
+	Changes the wait duration used before firmware indicates IDLE.
+	Reducing this value will cause the firmware to shut off faster and
+	more often but may increase bubbles in GPU scheduling due to the added
+	power management activity. If bPersistent is NOT set, APM latency will
+	return back to system default on power up.
+
+ @Input	   psDeviceNode : RGX Device Node
+ @Input	   ui32ActivePMLatencyms : Number of milliseconds to wait
+ @Input	   bActivePMLatencyPersistant : Set to ensure new value is not reset
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXAPMLatencyChange(PPVRSRV_DEVICE_NODE	psDeviceNode,
+				IMG_UINT32				ui32ActivePMLatencyms,
+				IMG_BOOL				bActivePMLatencyPersistant);
+
+/*!
+******************************************************************************
+
+ @Function	RGXActivePowerRequest
+
+ @Description Initiate a handshake with the FW to power off the GPU
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXActivePowerRequest(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function	RGXForcedIdleRequest
+
+ @Description Initiate a handshake with the FW to idle the GPU
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Input    bDeviceOffPermitted : Set to indicate device state being off is not
+                                 erroneous.
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXForcedIdleRequest(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  IMG_BOOL bDeviceOffPermitted);
+
+/*!
+******************************************************************************
+
+ @Function	RGXCancelForcedIdleRequest
+
+ @Description Send a request to cancel idle to the firmware.
+
+ @Input	   psDeviceNode : RGX Device Node
+
+ @Return   PVRSRV_ERROR :
+
+******************************************************************************/
+PVRSRV_ERROR RGXCancelForcedIdleRequest(PPVRSRV_DEVICE_NODE psDeviceNode);
+
+
+#endif /* RGXPOWER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxregconfig.c b/drivers/gpu/drm/img-rogue/rgxregconfig.c
new file mode 100644
index 000000000000..c2f681c70f12
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxregconfig.c
@@ -0,0 +1,331 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Register configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Regconfig routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxregconfig.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "device.h"
+#include "sync_internal.h"
+#include "pdump_km.h"
+#include "pvrsrv.h"
+
+PVRSRV_ERROR PVRSRVRGXSetRegConfigTypeKM(CONNECTION_DATA * psDevConnection,
+                                         PVRSRV_DEVICE_NODE	 *psDeviceNode,
+                                         IMG_UINT8           ui8RegCfgType)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR          eError      = PVRSRV_OK;
+	PVRSRV_RGXDEV_INFO    *psDevInfo  = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG        *psRegCfg   = &psDevInfo->sRegConfig;
+	RGXFWIF_REG_CFG_TYPE  eRegCfgType = (RGXFWIF_REG_CFG_TYPE) ui8RegCfgType;
+
+	PVR_UNREFERENCED_PARAMETER(psDevConnection);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (eRegCfgType < psRegCfg->eRegCfgTypeToPush)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Register configuration requested (%d) is not valid since it has to be at least %d."
+			 " Configurations of different types need to go in order",
+			 __func__,
+			 eRegCfgType,
+			 psRegCfg->eRegCfgTypeToPush));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_INVALID_TYPE;
+	}
+
+	psRegCfg->eRegCfgTypeToPush = eRegCfgType;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psDevConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui8RegCfgType);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXAddRegConfigKM(CONNECTION_DATA * psConnection,
+                                     PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                     IMG_UINT32		ui32RegAddr,
+                                     IMG_UINT64		ui64RegValue,
+                                     IMG_UINT64		ui64RegMask)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (psRegCfg->bEnabled)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Cannot add record whilst register configuration active.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_ENABLED;
+	}
+	if (psRegCfg->ui32NumRegRecords == RGXFWIF_REG_CFG_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Register configuration full.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_FULL;
+	}
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Addr = (IMG_UINT64) ui32RegAddr;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Value = ui64RegValue;
+	sRegCfgCmd.uCmdData.sRegConfigData.sRegConfig.ui64Mask = ui64RegMask;
+	sRegCfgCmd.uCmdData.sRegConfigData.eRegConfigType = psRegCfg->eRegCfgTypeToPush;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_ADD;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->ui32NumRegRecords++;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32RegAddr);
+	PVR_UNREFERENCED_PARAMETER(ui64RegValue);
+	PVR_UNREFERENCED_PARAMETER(ui64RegMask);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXClearRegConfigKM(CONNECTION_DATA * psConnection,
+                                       PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	if (psRegCfg->bEnabled)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: Attempt to clear register configuration whilst active.",
+			 __func__));
+		OSLockRelease(psRegCfg->hLock);
+		return PVRSRV_ERROR_REG_CONFIG_ENABLED;
+	}
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_CLEAR;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->ui32NumRegRecords = 0;
+	psRegCfg->eRegCfgTypeToPush = RGXFWIF_REG_CFG_TYPE_PWR_ON;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXEnableRegConfigKM(CONNECTION_DATA * psConnection,
+                                        PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_ENABLE;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->bEnabled = IMG_TRUE;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+PVRSRV_ERROR PVRSRVRGXDisableRegConfigKM(CONNECTION_DATA * psConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode)
+{
+#if defined(SUPPORT_USER_REGISTER_CONFIGURATION)
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	RGXFWIF_KCCB_CMD	sRegCfgCmd;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+	RGX_REG_CONFIG		*psRegCfg = &psDevInfo->sRegConfig;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRV_VZ_RET_IF_MODE(GUEST, DEVNODE, psDeviceNode, PVRSRV_ERROR_NOT_SUPPORTED);
+
+	OSLockAcquire(psRegCfg->hLock);
+
+	sRegCfgCmd.eCmdType = RGXFWIF_KCCB_CMD_REGCONFIG;
+	sRegCfgCmd.uCmdData.sRegConfigData.eCmdType = RGXFWIF_REGCFG_CMD_DISABLE;
+
+	eError = RGXScheduleCommand(psDeviceNode->pvDevice,
+				RGXFWIF_DM_GP,
+				&sRegCfgCmd,
+				PDUMP_FLAGS_CONTINUOUS);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: RGXScheduleCommand failed. Error:%u",
+			 __func__,
+			 eError));
+		OSLockRelease(psRegCfg->hLock);
+		return eError;
+	}
+
+	psRegCfg->bEnabled = IMG_FALSE;
+
+	OSLockRelease(psRegCfg->hLock);
+
+	return eError;
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	PVR_DPF((PVR_DBG_ERROR,
+		 "%s: Feature disabled. Compile with SUPPORT_USER_REGISTER_CONFIGURATION",
+		 __func__));
+
+	return PVRSRV_ERROR_FEATURE_DISABLED;
+#endif
+}
+
+/******************************************************************************
+ End of file (rgxregconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxregconfig.h b/drivers/gpu/drm/img-rogue/rgxregconfig.h
new file mode 100644
index 000000000000..b0921d98cb14
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxregconfig.h
@@ -0,0 +1,130 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX register configuration functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX register configuration functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXREGCONFIG_H)
+#define RGXREGCONFIG_H
+
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_km.h"
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetRegConfigTypeKM
+
+ @Description
+	Server-side implementation of RGXSetRegConfig
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui8RegPowerIsland - Reg configuration
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSetRegConfigTypeKM(CONNECTION_DATA * psDevConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                         IMG_UINT8 ui8RegPowerIsland);
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXSetRegConfigKM
+
+ @Description
+	Server-side implementation of RGXSetRegConfig
+
+ @Input psDeviceNode - RGX Device node
+ @Input ui64RegAddr - Register address
+ @Input ui64RegValue - Reg value
+ @Input ui64RegMask - Reg mask
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+
+PVRSRV_ERROR PVRSRVRGXAddRegConfigKM(CONNECTION_DATA * psConnection,
+                                     PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                     IMG_UINT32	ui64RegAddr,
+                                     IMG_UINT64	ui64RegValue,
+                                     IMG_UINT64	ui64RegMask);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXClearRegConfigKM
+
+ @Description
+	Server-side implementation of RGXClearRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXClearRegConfigKM(CONNECTION_DATA * psConnection,
+                                       PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXEnableRegConfigKM
+
+ @Description
+	Server-side implementation of RGXEnableRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXEnableRegConfigKM(CONNECTION_DATA * psConnection,
+                                        PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+/*!
+*******************************************************************************
+ @Function	PVRSRVRGXDisableRegConfigKM
+
+ @Description
+	Server-side implementation of RGXDisableRegConfig
+
+ @Input psDeviceNode - RGX Device node
+
+ @Return   PVRSRV_ERROR
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDisableRegConfigKM(CONNECTION_DATA * psConnection,
+                                         PVRSRV_DEVICE_NODE	*psDeviceNode);
+
+#endif /* RGXREGCONFIG_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxshader.c b/drivers/gpu/drm/img-rogue/rgxshader.c
new file mode 100644
index 000000000000..85cf498f037c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxshader.c
@@ -0,0 +1,321 @@
+/*************************************************************************/ /*!
+@File           rgxshader.c
+@Title          TQ Shader Load
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shader code and info are shared for all context on the device.
+                If allocation doesn't already exist, read shader data from file
+                and allocate PMR memory. PMR memory is not deallocated until
+                device deinit.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxshader.h"
+#include "osfunc_common.h"
+#include "rgxdevice.h"
+#include "pdump_km.h"
+#include "physmem.h"
+#include "ri_server.h"
+#include "pvr_ricommon.h"
+#include "pvrsrv.h"
+
+static void
+RGXShaderReadHeader(OS_FW_IMAGE *psShaderFW, RGX_SHADER_HEADER *psHeader)
+{
+	const void * pvData;
+
+	pvData = OSFirmwareData(psShaderFW);
+
+	OSDeviceMemCopy(psHeader, pvData, sizeof(RGX_SHADER_HEADER));
+}
+
+static size_t
+RGXShaderCLIMemSize(OS_FW_IMAGE *psShaderFW)
+{
+	RGX_SHADER_HEADER sHeader;
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	return sHeader.ui32SizeClientMem;
+}
+
+static size_t
+RGXShaderUSCMemSize(OS_FW_IMAGE *psShaderFW)
+{
+	RGX_SHADER_HEADER sHeader;
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	return sHeader.ui32SizeFragment;
+}
+
+static void *
+RGXShaderCLIMem(OS_FW_IMAGE *psShaderFW)
+{
+	return (void*)OSFirmwareData(psShaderFW);
+}
+
+static void *
+RGXShaderUSCMem(OS_FW_IMAGE *psShaderFW)
+{
+	IMG_PBYTE pui8Data;
+
+	pui8Data = (IMG_PBYTE)OSFirmwareData(psShaderFW);
+
+	pui8Data += RGXShaderCLIMemSize(psShaderFW);
+
+	return (void*) pui8Data;
+}
+
+#define RGX_SHADER_FILENAME_MAX_SIZE   ((sizeof(RGX_SH_FILENAME)+	\
+										 RGX_BVNC_STR_SIZE_MAX))
+
+static void
+_GetShaderFileName(PVRSRV_DEVICE_NODE * psDeviceNode,
+				   IMG_CHAR           * pszShaderFilenameStr,
+				   IMG_CHAR           * pszShaderpFilenameStr)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	OSSNPrintf(pszShaderFilenameStr, RGX_SHADER_FILENAME_MAX_SIZE,
+			   RGX_SH_FILENAME "." RGX_BVNC_STR_FMTSPEC,
+			   psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			   psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C);
+
+	OSSNPrintf(pszShaderpFilenameStr, RGX_SHADER_FILENAME_MAX_SIZE,
+			   RGX_SH_FILENAME "." RGX_BVNC_STRP_FMTSPEC,
+			   psDevInfo->sDevFeatureCfg.ui32B, psDevInfo->sDevFeatureCfg.ui32V,
+			   psDevInfo->sDevFeatureCfg.ui32N, psDevInfo->sDevFeatureCfg.ui32C);
+}
+
+PVRSRV_ERROR
+PVRSRVTQLoadShaders(PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	OS_FW_IMAGE        *psShaderFW;
+	RGX_SHADER_HEADER   sHeader;
+	IMG_UINT32          ui32MappingTable = 0;
+	IMG_UINT32          ui32NumPages;
+	IMG_CHAR            aszShaderFilenameStr[RGX_SHADER_FILENAME_MAX_SIZE];
+	IMG_CHAR            aszShaderpFilenameStr[RGX_SHADER_FILENAME_MAX_SIZE];
+	const IMG_CHAR      *pszShaderFilenameStr = aszShaderFilenameStr;
+	size_t              uiNumBytes;
+	PVRSRV_DEVICE_NODE *psHostDevNode = PVRSRVGetPVRSRVData()->psHostMemDeviceNode;
+	size_t              uiLog2PageSize = OSGetPageShift();
+	IMG_DEVMEM_SIZE_T   uiTQUSCMemSize;
+	PVRSRV_ERROR        eError;
+
+	_GetShaderFileName(psDeviceNode, aszShaderFilenameStr, aszShaderpFilenameStr);
+
+	eError = OSLoadFirmware(psDeviceNode, aszShaderFilenameStr, NULL, &psShaderFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		eError = OSLoadFirmware(psDeviceNode, aszShaderpFilenameStr,
+		                        NULL, &psShaderFW);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to load shader binary file %s (%s)",
+			         __func__,
+			         aszShaderpFilenameStr,
+			         PVRSRVGetErrorString(eError)));
+			eError = PVRSRV_ERROR_UNABLE_TO_FIND_RESOURCE;
+			goto failed_init;
+		}
+
+		pszShaderFilenameStr = aszShaderpFilenameStr;
+	}
+
+	PVR_LOG(("Shader binary image '%s' loaded", pszShaderFilenameStr));
+
+	RGXShaderReadHeader(psShaderFW, &sHeader);
+
+	if (sHeader.ui32Version != RGX_TQ_SHADERS_VERSION_PACK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: unsupported TQ shaders version: %d != %d",
+				 __func__, sHeader.ui32Version, RGX_TQ_SHADERS_VERSION_PACK));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto failed_firmware;
+	}
+
+	ui32NumPages = (sHeader.ui32SizeFragment / IMG_PAGE2BYTES32(uiLog2PageSize)) + 1;
+
+	uiTQUSCMemSize = ui32NumPages * IMG_PAGE2BYTES32(uiLog2PageSize);
+
+	if (uiTQUSCMemSize > RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TQ shaders out of space ("IMG_DEVMEM_SIZE_FMTSPEC" > 0x%X)",
+				 __func__, uiTQUSCMemSize, RGXFWIF_KM_USC_TQ_SHADER_CODE_MAX_SIZE_BYTES));
+		eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		goto failed_firmware;
+	}
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate TDM USC PMR Block (Pages %08X)", ui32NumPages);
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psDeviceNode,
+									uiTQUSCMemSize,
+									1,
+									1,
+									&ui32MappingTable,
+									uiLog2PageSize,
+									PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT
+									| PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER,
+									sizeof("tquscpmr"),
+									"tquscpmr",
+									PVR_SYS_ALLOC_PID,
+									(PMR**)&psDevInfo->hTQUSCSharedMem,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PhysmemNewRamBackedPMR (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_firmware;
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(psDevInfo->hTQUSCSharedMem, PVR_SYS_ALLOC_PID);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RIWritePMREntryWithOwnerKM (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+#endif
+
+	eError = PMR_WriteBytes(psDevInfo->hTQUSCSharedMem, 0, RGXShaderUSCMem(psShaderFW), RGXShaderUSCMemSize(psShaderFW), &uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PMR_WriteBytes (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+
+	PDUMPCOMMENT(psDeviceNode, "Load TDM USC PMR Block (Size "IMG_SIZE_FMTSPECX")", RGXShaderUSCMemSize(psShaderFW));
+	PMRPDumpLoadMem(psDevInfo->hTQUSCSharedMem, 0, RGXShaderUSCMemSize(psShaderFW), PDUMP_FLAGS_CONTINUOUS, false);
+
+	ui32NumPages = (sHeader.ui32SizeClientMem / IMG_PAGE2BYTES32(uiLog2PageSize)) + 1;
+
+	PDUMPCOMMENT(psDeviceNode, "Allocate TDM Client PMR Block (Pages %08X)", ui32NumPages);
+
+	eError = PhysmemNewRamBackedPMR(NULL,
+									psHostDevNode,
+									(IMG_DEVMEM_SIZE_T)ui32NumPages * IMG_PAGE2BYTES32(uiLog2PageSize),
+									1,
+									1,
+									&ui32MappingTable,
+									uiLog2PageSize,
+									PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE
+									| PVRSRV_MEMALLOCFLAG_CPU_READABLE
+									| PVRSRV_MEMALLOCFLAG_CPU_CACHE_INCOHERENT
+									| PVRSRV_MEMALLOCFLAG_VAL_SHARED_BUFFER
+									| PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL),
+									sizeof("tqclipmr"),
+									"tqclipmr",
+									PVR_SYS_ALLOC_PID,
+									(PMR**)&psDevInfo->hTQCLISharedMem,
+									PDUMP_NONE,
+									NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PhysmemNewRamBackedPMR (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_uscpmr;
+	}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	eError = RIWritePMREntryWithOwnerKM(psDevInfo->hTQCLISharedMem, PVR_SYS_ALLOC_PID);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RIWritePMREntryWithOwnerKM (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_clipmr;
+	}
+#endif
+
+	eError = PMR_WriteBytes(psDevInfo->hTQCLISharedMem, 0, RGXShaderCLIMem(psShaderFW), RGXShaderCLIMemSize(psShaderFW), &uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from PMR_WriteBytes (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto failed_clipmr;
+	}
+
+	OSUnloadFirmware(psShaderFW);
+
+	PVR_ASSERT(psDevInfo->hTQUSCSharedMem != NULL);
+	PVR_ASSERT(psDevInfo->hTQCLISharedMem != NULL);
+
+	return PVRSRV_OK;
+
+failed_clipmr:
+	PMRUnrefPMR(psDevInfo->hTQCLISharedMem);
+failed_uscpmr:
+	PMRUnrefPMR(psDevInfo->hTQUSCSharedMem);
+failed_firmware:
+	OSUnloadFirmware(psShaderFW);
+failed_init:
+	return eError;
+}
+
+void
+PVRSRVTQAcquireShaders(PVRSRV_DEVICE_NODE  * psDeviceNode,
+					   PMR                ** ppsCLIPMRMem)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_ASSERT(psDevInfo->hTQCLISharedMem != NULL);
+
+	*ppsCLIPMRMem = psDevInfo->hTQCLISharedMem;
+}
+
+void PVRSRVTQUnloadShaders(PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	(void) PMRUnrefPMR(psDevInfo->hTQUSCSharedMem);
+	(void) PMRUnrefPMR(psDevInfo->hTQCLISharedMem);
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxshader.h b/drivers/gpu/drm/img-rogue/rgxshader.h
new file mode 100644
index 000000000000..4ba50fc3fc86
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxshader.h
@@ -0,0 +1,81 @@
+/*************************************************************************/ /*!
+@File           rgxshader.h
+@Title          TQ Shader Load
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Shader code and info are shared for all context on the device.
+                If allocation doesn't already exist, read shader data from file
+                and allocate PMR memory. PMR memory is not deallocated until
+                device deinit.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSHADER_H)
+#define RGXSHADER_H
+
+#include "fwload.h"
+#include "rgxtransfer_shader.h"
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQLoadShaders
+@Description    If PMR is not allocated, reads shader binary data from file
+                and allocates new PMR memory.
+@Input          psDeviceNode Device node
+@Return         PVRSRV_ERROR Returns PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVTQLoadShaders(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQAcquireShaders
+@Description    Get handle to ready allocated shader PMR memory
+@Input          psDeviceNode Device node
+@Output         ppsCLIPMRMem Shader data used by CPU client side.
+*/ /**************************************************************************/
+void
+PVRSRVTQAcquireShaders(PVRSRV_DEVICE_NODE  *psDeviceNode,
+                       PMR                **ppsCLIPMRMem);
+
+/*************************************************************************/ /*!
+@Function       PVRSRVTQUnLoadShaders
+@Description    Unref PMR memory.
+@Input          psDeviceNode Device node
+*/ /**************************************************************************/
+void PVRSRVTQUnloadShaders(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* RGXSHADER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxsrvinit.c b/drivers/gpu/drm/img-rogue/rgxsrvinit.c
new file mode 100644
index 000000000000..9b1077273542
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxsrvinit.c
@@ -0,0 +1,1711 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services initialisation routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "srvinit.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+#include "km_apphint_defs.h"
+#include "htbuffer_types.h"
+
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+
+#include "rgxinit_apphints.h"
+#include "rgxinit.h"
+#include "rgxmulticore.h"
+
+#include "rgx_compat_bvnc.h"
+
+#include "osfunc.h"
+
+#include "rgxdefs_km.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+#include "rgx_fwif_hwperf.h"
+#include "rgx_hwperf_table.h"
+
+#include "fwload.h"
+#include "rgxlayer_impl.h"
+#include "rgxfwimageutils.h"
+#include "rgxfwutils.h"
+
+#include "rgx_hwperf.h"
+#include "rgx_bvnc_defs_km.h"
+
+#include "rgxdevice.h"
+
+#include "pvrsrv.h"
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+#include "rgxdevice.h"
+#include "pvrsrv_device.h"
+#endif
+
+#define	HW_PERF_FILTER_DEFAULT         0x00000000 /* Default to no HWPerf */
+#define HW_PERF_FILTER_DEFAULT_ALL_ON  0xFFFFFFFF /* All events */
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+#define AVAIL_POW_UNITS_MASK_DEFAULT   (PVRSRV_APPHINT_HWVALAVAILABLESPUMASK)
+#define AVAIL_RAC_MASK_DEFAULT         (PVRSRV_APPHINT_HWVALAVAILABLERACMASK)
+#endif
+
+/* Kernel CCB size */
+
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE 4
+#endif
+#if !defined(PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+#define PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE 16
+#endif
+
+#if PVRSRV_APPHINT_KCCB_SIZE_LOG2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too low.
+#elif PVRSRV_APPHINT_KCCB_SIZE_LOG2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE
+#error PVRSRV_APPHINT_KCCB_SIZE_LOG2 is too high.
+#endif
+
+
+#include "os_apphint.h"
+
+/*
+ * _ParseHTBAppHints:
+ *
+ * Generate necessary references to the globally visible AppHints which are
+ * declared in the above #include "km_apphint_defs.h"
+ * Without these local references some compiler tool-chains will treat
+ * unreferenced declarations as fatal errors. This function duplicates the
+ * HTB_specific apphint references which are made in htbserver.c:HTBInit()
+ * However, it makes absolutely *NO* use of these hints.
+ */
+static void
+_ParseHTBAppHints(PVRSRV_DEVICE_NODE *psDeviceNode, void *pvAppHintState)
+{
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_UINT32 ui32LogType;
+	IMG_UINT32 ui32OpMode;
+	IMG_UINT32 ui32BufferSize;
+
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEHTBLOGGROUP;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, EnableHTBLogGroup,
+	                     &ui32AppHintDefault, &ui32LogType);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBOPERATIONMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBOperationMode,
+	                     &ui32AppHintDefault, &ui32OpMode);
+	ui32AppHintDefault = PVRSRV_APPHINT_HTBUFFERSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, HTBufferSizeInKB,
+	                     &ui32AppHintDefault, &ui32BufferSize);
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetFilterFlags
+
+ @Description   Initialise and return filter flags
+
+ @Input         bFilteringMode : Enable new TPU filtering mode
+ @Input         ui32TruncateMode : TPU Truncate mode
+
+ @Return        IMG_UINT32 : Filter flags
+
+******************************************************************************/
+static INLINE IMG_UINT32 GetFilterFlags(IMG_BOOL bFilteringMode, IMG_UINT32 ui32TruncateMode)
+{
+	IMG_UINT32 ui32FilterFlags = 0;
+
+	ui32FilterFlags |= bFilteringMode ? RGXFWIF_FILTCFG_NEW_FILTER_MODE : 0;
+	if (ui32TruncateMode == 2)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_INT;
+	}
+	else if (ui32TruncateMode == 3)
+	{
+		ui32FilterFlags |= RGXFWIF_FILTCFG_TRUNCATE_HALF;
+	}
+
+	return ui32FilterFlags;
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      InitDeviceFlags
+
+ @Description   Initialise and return device flags
+
+ @Input         psDeviceNode     : Pointer to device node
+ @Input         pvAppHintState   : Pointer to apphint state
+ @Input         psHints          : Apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void InitDeviceFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   void *pvAppHintState,
+                                   RGX_INIT_APPHINTS *psHints)
+{
+	IMG_UINT32 ui32DeviceFlags = 0;
+	IMG_BOOL bAppHintDefault;
+	IMG_BOOL bZeroFreelist;
+	IMG_BOOL bDisableFEDLogging;
+
+
+	bAppHintDefault = PVRSRV_APPHINT_ZEROFREELIST;
+	OSGetAppHintBOOL(psDeviceNode,  pvAppHintState,  ZeroFreelist,
+	                   &bAppHintDefault,         &bZeroFreelist);
+	ui32DeviceFlags |= bZeroFreelist ? RGXKM_DEVICE_STATE_ZERO_FREELIST : 0;
+
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEFEDLOGGING;
+	OSGetAppHintBOOL(psDeviceNode,    pvAppHintState,    DisableFEDLogging,
+	                   &bAppHintDefault,        &bDisableFEDLogging);
+	ui32DeviceFlags |= bDisableFEDLogging ? RGXKM_DEVICE_STATE_DISABLE_DW_LOGGING_EN : 0;
+
+
+#if defined(PVRSRV_ENABLE_CCCB_GROW)
+	BITMASK_SET(ui32DeviceFlags, RGXKM_DEVICE_STATE_CCB_GROW_EN);
+#endif
+
+	psHints->ui32DeviceFlags = ui32DeviceFlags;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      GetApphints
+
+ @Description   Read init time apphints and initialise apphints structure
+
+ @Input         psHints : Pointer to apphints container
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetApphints(PVRSRV_RGXDEV_INFO *psDevInfo, RGX_INIT_APPHINTS *psHints)
+{
+	void *pvAppHintState = NULL;
+	IMG_UINT32 ui32AppHintDefault;
+	IMG_BOOL bAppHintDefault;
+	IMG_UINT32 ui32ParamTemp;
+#if defined(__linux__)
+	IMG_UINT64 ui64AppHintDefault;
+#endif
+
+	OSCreateAppHintState(&pvAppHintState);
+
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESIGNATURECHECKS;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSignatureChecks,
+	                     &bAppHintDefault,    &psHints->bEnableSignatureChecks);
+	ui32AppHintDefault = PVRSRV_APPHINT_SIGNATURECHECKSBUFSIZE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    SignatureChecksBufSize,
+	                     &ui32AppHintDefault,    &psHints->ui32SignatureChecksBufSize);
+
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTOUTOFMEMORY;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOutOfMemory,
+	                   &bAppHintDefault,        &psHints->bAssertOnOutOfMem);
+	bAppHintDefault = PVRSRV_APPHINT_ASSERTONHWRTRIGGER;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    AssertOnHWRTrigger,
+	                   &bAppHintDefault,        &psHints->bAssertOnHWRTrigger);
+	bAppHintDefault = PVRSRV_APPHINT_CHECKMLIST;
+	OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,    CheckMList,
+	                   &bAppHintDefault,        &psHints->bCheckMlist);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLECLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableClockGating,
+	                   &bAppHintDefault,        &psHints->bDisableClockGating);
+	bAppHintDefault = PVRSRV_APPHINT_DISABLEDMOVERLAP;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    DisableDMOverlap,
+	                   &bAppHintDefault,        &psHints->bDisableDMOverlap);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEAPM;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    EnableAPM,
+	                     &ui32AppHintDefault,      &ui32ParamTemp);
+	psHints->eRGXActivePMConf = ui32ParamTemp;
+	bAppHintDefault = PVRSRV_APPHINT_ENABLECDMKILLINGRANDMODE;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableCDMKillingRandMode,
+	                   &bAppHintDefault,      &psHints->bEnableDMKillRand);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLERANDOMCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableRandomContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableRandomCsw);
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESOFTRESETCONTEXTSWITCH;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    EnableSoftResetContextSwitch,
+	                   &bAppHintDefault,      &psHints->bEnableSoftResetCsw);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLEFWCONTEXTSWITCH;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableFWContextSwitch,
+	                     &ui32AppHintDefault,    &psHints->ui32EnableFWContextSwitch);
+	ui32AppHintDefault = PVRSRV_APPHINT_ENABLERDPOWERISLAND;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    EnableRDPowerIsland,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eRGXRDPowerIslandConf = ui32ParamTemp;
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	bAppHintDefault = PVRSRV_APPHINT_ENABLESPUCLOCKGATING;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,          pvAppHintState,    EnableSPUClockGating,
+	                   &bAppHintDefault,           &psHints->bSPUClockGating);
+#endif
+	ui32AppHintDefault = PVRSRV_APPHINT_FIRMWAREPERF;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FirmwarePerf,
+	                     &ui32AppHintDefault,    &ui32ParamTemp);
+	psHints->eFirmwarePerf = ui32ParamTemp;
+	ui32AppHintDefault = PVRSRV_APPHINT_FWCONTEXTSWITCHPROFILE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    FWContextSwitchProfile,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchProfile);
+
+	/*
+	 * HWPerf apphints *
+	 */
+	bAppHintDefault = PVRSRV_APPHINT_HWPERFDISABLECUSTOMCOUNTERFILTER;
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE,        pvAppHintState,    HWPerfDisableCustomCounterFilter,
+	                     &bAppHintDefault,    &psHints->bHWPerfDisableCounterFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfHostBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfHostBufSize);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFFWBUFSIZEINKB;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,    HWPerfFWBufSizeInKB,
+	                     &ui32AppHintDefault,    &psHints->ui32HWPerfFWBufSize);
+
+	ui32AppHintDefault = PVRSRV_APPHINT_KCCB_SIZE_LOG2;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,  KernelCCBSizeLog2,
+	                     &ui32AppHintDefault,    &psHints->ui32KCCBSizeLog2);
+
+	if (psHints->ui32KCCBSizeLog2 < PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too low, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MIN_SIZE;
+	}
+	else if (psHints->ui32KCCBSizeLog2 > PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "KCCB size %u is too high, setting to %u",
+		         psHints->ui32KCCBSizeLog2, PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE));
+		psHints->ui32KCCBSizeLog2 = PVRSRV_RGX_LOG2_KERNEL_CCB_MAX_SIZE;
+	}
+
+
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32AppHintDefault = PVRSRV_APPHINT_ISPSCHEDULINGLATENCYMODE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  ISPSchedulingLatencyMode,
+	                     &ui32AppHintDefault,  &psHints->ui32ISPSchedulingLatencyMode);
+#endif
+
+#if defined(__linux__)
+	/* name changes */
+	{
+		IMG_UINT64 ui64Tmp;
+		bAppHintDefault = PVRSRV_APPHINT_DISABLEPDUMPPANIC;
+		OSGetAppHintBOOL(psDevInfo->psDeviceNode,    pvAppHintState,  DisablePDumpPanic,
+		                   &bAppHintDefault,           &psHints->bDisablePDP);
+		ui64AppHintDefault = PVRSRV_APPHINT_HWPERFFWFILTER;
+		OSGetAppHintUINT64(psDevInfo->psDeviceNode,  pvAppHintState,  HWPerfFWFilter,
+		                   &ui64AppHintDefault,        &ui64Tmp);
+		psHints->ui32HWPerfFilter0 = (IMG_UINT32)(ui64Tmp & 0xffffffffllu);
+		psHints->ui32HWPerfFilter1 = (IMG_UINT32)((ui64Tmp >> 32) & 0xffffffffllu);
+	}
+#endif
+
+	ui32AppHintDefault = PVRSRV_APPHINT_HWPERFHOSTFILTER;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWPerfHostFilter,
+	                     &ui32AppHintDefault,          &psHints->ui32HWPerfHostFilter);
+	ui32AppHintDefault = PVRSRV_APPHINT_TIMECORRCLOCK;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,  pvAppHintState,    TimeCorrClock,
+	                         &ui32AppHintDefault,      &psHints->ui32TimeCorrClock);
+	ui32AppHintDefault = PVRSRV_APPHINT_HWRDEBUGDUMPLIMIT;
+	OSGetAppHintUINT32(psDevInfo->psDeviceNode,      pvAppHintState,    HWRDebugDumpLimit,
+	                     &ui32AppHintDefault,          &ui32ParamTemp);
+	psHints->ui32HWRDebugDumpLimit = MIN(ui32ParamTemp, RGXFWIF_HWR_DEBUG_DUMP_ALL);
+
+	{
+		IMG_BOOL bFilteringMode = IMG_FALSE;
+		IMG_UINT32 ui32TruncateMode = 0U;
+
+#if defined(HW_ERN_42290_BIT_MASK) && defined(RGX_FEATURE_TPU_FILTERING_MODE_CONTROL_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42290) && RGX_IS_FEATURE_SUPPORTED(psDevInfo, TPU_FILTERING_MODE_CONTROL))
+#endif
+		{
+			bAppHintDefault = PVRSRV_APPHINT_NEWFILTERINGMODE;
+			OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  NewFilteringMode,
+			                   &bAppHintDefault,  &bFilteringMode);
+		}
+
+#if defined(HW_ERN_42606_BIT_MASK)
+		if (RGX_IS_ERN_SUPPORTED(psDevInfo, 42606))
+#endif
+		{
+			ui32AppHintDefault = PVRSRV_APPHINT_TRUNCATEMODE;
+			OSGetAppHintUINT32(APPHINT_NO_DEVICE,  pvAppHintState,  TruncateMode,
+			                     &ui32AppHintDefault,  &ui32TruncateMode);
+		}
+
+		psHints->ui32FilterFlags = GetFilterFlags(bFilteringMode, ui32TruncateMode);
+	}
+
+#if defined(EMULATOR)
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#endif
+	{
+		bAppHintDefault = PVRSRV_APPHINT_ENABLETRUSTEDDEVICEACECONFIG;
+		OSGetAppHintBOOL(APPHINT_NO_DEVICE,    pvAppHintState,  EnableTrustedDeviceAceConfig,
+		                   &bAppHintDefault,     &psHints->bEnableTrustedDeviceAceConfig);
+	}
+#endif
+
+#if defined(__linux__)
+	ui32AppHintDefault = 0;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  FWContextSwitchCrossDM,
+	                     &ui32AppHintDefault,    &psHints->ui32FWContextSwitchCrossDM);
+#endif
+
+#if defined(SUPPORT_PHYSMEM_TEST) && !defined(INTEGRITY_OS) && !defined(__QNXNTO__)
+	ui32AppHintDefault = PVRSRV_APPHINT_PHYSMEMTESTPASSES;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,      pvAppHintState,  PhysMemTestPasses,
+	                     &ui32AppHintDefault,    &psHints->ui32PhysMemTestPasses);
+#endif
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	psHints->ui32AvailablePowUnitsMask = AVAIL_POW_UNITS_MASK_DEFAULT;
+	psHints->ui32AvailableRACMask = AVAIL_RAC_MASK_DEFAULT;
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	ui32AppHintDefault = PVRSRV_APPHINT_TFBCVERSION;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, TFBCVersionDowngrade,
+	 &ui32AppHintDefault, &psHints->ui32TFBCVersion);
+
+	if (ui32AppHintDefault != psHints->ui32TFBCVersion)
+	{
+		PVR_LOG(("TFBCVersionDowngrade set to %u", psHints->ui32TFBCVersion));
+	}
+
+	psHints->bTFBCCompressionControlLossyMinChannel = false;
+	psHints->bTFBCCompressionControlYUVFormat = false;
+	psHints->ui32TFBCCompressionControlScheme =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLSCHEME;
+	psHints->ui32TFBCCompressionControlGroup =
+	    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	ui32AppHintDefault = PVRSRV_APPHINT_DEBUGDUMPFWTLOGTYPE;
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE,    pvAppHintState,  DebugDumpFWTLogType,
+	                     &ui32AppHintDefault,  &psHints->eDebugDumpFWTLogType);
+	if ((IMG_UINT32)psHints->eDebugDumpFWTLogType > RGX_FWT_LOGTYPE_PARTIAL)
+	{
+		psHints->eDebugDumpFWTLogType = RGX_FWT_LOGTYPE_NONE;
+		PVR_DPF((PVR_DBG_WARNING, "Invalid value for DebugDumpFWTLogType. Setting to 0 (disabled)."));
+	}
+
+	/*
+	 * FW logs apphints
+	 */
+	{
+		IMG_UINT32 ui32LogGroup, ui32TraceOrTBI;
+
+		ui32AppHintDefault = PVRSRV_APPHINT_ENABLELOGGROUP;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, EnableLogGroup, &ui32AppHintDefault, &ui32LogGroup);
+		ui32AppHintDefault = PVRSRV_APPHINT_FIRMWARELOGTYPE;
+		OSGetAppHintUINT32(psDevInfo->psDeviceNode, pvAppHintState, FirmwareLogType, &ui32AppHintDefault, &ui32TraceOrTBI);
+
+		/* Defaulting to TRACE */
+		BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+
+#if defined(SUPPORT_TBI_INTERFACE)
+		if (ui32TraceOrTBI == 1 /* TBI */)
+		{
+			if ((ui32LogGroup & RGXFWIF_LOG_TYPE_GROUP_MASK) == 0)
+			{
+				/* No groups configured - defaulting to MAIN group */
+				BITMASK_SET(ui32LogGroup, RGXFWIF_LOG_TYPE_GROUP_MAIN);
+			}
+			BITMASK_UNSET(ui32LogGroup, RGXFWIF_LOG_TYPE_TRACE);
+		}
+#endif
+		psHints->ui32LogType = ui32LogGroup;
+	}
+
+	_ParseHTBAppHints(psDevInfo->psDeviceNode, pvAppHintState);
+
+	InitDeviceFlags(psDevInfo->psDeviceNode, pvAppHintState, psHints);
+
+	OSFreeAppHintState(pvAppHintState);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      GetFWConfigFlags
+
+ @Description   Initialise and return FW config flags
+
+ @Input         psHints            : Apphints container
+ @Input         pui32FWConfigFlags : Pointer to config flags
+
+ @Return        void
+
+******************************************************************************/
+static INLINE void GetFWConfigFlags(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    RGX_INIT_APPHINTS *psHints,
+                                    IMG_UINT32 *pui32FWConfigFlags,
+                                    IMG_UINT32 *pui32FWConfigFlagsExt,
+                                    IMG_UINT32 *pui32FwOsCfgFlags)
+{
+	IMG_UINT32 ui32FWConfigFlags = 0;
+	IMG_UINT32 ui32FWConfigFlagsExt = 0;
+	IMG_UINT32 ui32FwOsCfgFlags = psHints->ui32FWContextSwitchCrossDM |
+	                              (psHints->ui32EnableFWContextSwitch & ~RGXFWIF_INICFG_OS_CTXSWITCH_CLRMSK);
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+	IMG_UINT32 ui32FWConfigFlagsSupValExt = 0;
+	IMG_UINT32 ui32TFBCVersion = 0U;
+#endif
+
+	ui32FWConfigFlags |= psHints->bAssertOnOutOfMem ? RGXFWIF_INICFG_ASSERT_ON_OUTOFMEMORY : 0;
+	ui32FWConfigFlags |= psHints->bAssertOnHWRTrigger ? RGXFWIF_INICFG_ASSERT_ON_HWR_TRIGGER : 0;
+	ui32FWConfigFlags |= psHints->bCheckMlist ? RGXFWIF_INICFG_CHECK_MLIST_EN : 0;
+	ui32FWConfigFlags |= psHints->bDisableClockGating ? RGXFWIF_INICFG_DISABLE_CLKGATING_EN : 0;
+
+#if defined(RGX_FEATURE_PIPELINED_DATAMASTERS_VERSION_MAX_VALUE_IDX)
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, PIPELINED_DATAMASTERS_VERSION)  &&
+		(RGX_GET_FEATURE_VALUE(psDevInfo, PIPELINED_DATAMASTERS_VERSION) > 0) &&
+		!RGX_IS_FEATURE_SUPPORTED(psDevInfo, ERYX_TOP_INFRASTRUCTURE))
+	{
+		/* Pipeline DM roadblocks are currently enabled pre-Eryx. */
+		ui32FWConfigFlags |= RGXFWIF_INICFG_DM_PIPELINE_ROADBLOCKS_EN;
+	}
+#endif
+
+	ui32FWConfigFlags |= psHints->bDisableDMOverlap ? RGXFWIF_INICFG_DISABLE_DM_OVERLAP : 0;
+
+	if ((RGX_GET_FEATURE_VALUE(psDevInfo, SLC_SIZE_IN_KILOBYTES) <= 2))
+	{
+		ui32FWConfigFlags |= RGXFWIF_INICFG_DISABLE_DM_OVERLAP;
+	}
+
+	ui32FWConfigFlags |= psHints->bDisablePDP ? RGXFWIF_INICFG_DISABLE_PDP_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableDMKillRand ? RGXFWIF_INICFG_DM_KILL_MODE_RAND_EN : 0;
+	ui32FWConfigFlags |= psHints->bEnableRandomCsw ? RGXFWIF_INICFG_CTXSWITCH_MODE_RAND : 0;
+	ui32FWConfigFlags |= psHints->bEnableSoftResetCsw ? RGXFWIF_INICFG_CTXSWITCH_SRESET_EN : 0;
+	ui32FWConfigFlags |= (psHints->ui32HWPerfFilter0 != 0 || psHints->ui32HWPerfFilter1 != 0) ? RGXFWIF_INICFG_HWPERF_EN : 0;
+	ui32FWConfigFlags |= psHints->bHWPerfDisableCounterFilter ? RGXFWIF_INICFG_HWP_DISABLE_FILTER : 0;
+	ui32FWConfigFlags |= (psHints->ui32FWContextSwitchProfile << RGXFWIF_INICFG_CTXSWITCH_PROFILE_SHIFT) & RGXFWIF_INICFG_CTXSWITCH_PROFILE_MASK;
+#if defined(PVR_ARCH_VOLCANIC)
+	ui32FWConfigFlags |= (psHints->ui32ISPSchedulingLatencyMode << RGXFWIF_INICFG_ISPSCHEDMODE_SHIFT) & RGXFWIF_INICFG_ISPSCHEDMODE_MASK;
+#endif
+
+	{
+		ui32FWConfigFlags |= psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode) ? RGXFWIF_INICFG_FBCDC_V3_1_EN : 0;
+	}
+
+
+#if defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX)
+
+	/* Determine if we need to present a TFBC v1.0, v1.1 or native
+	 * behaviour. For V1.0 we need to set the following features:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x1
+	 *  TFBCCompressionControlScheme = 0x2
+	 *  TFBCCompressionControlGroup = 0x0
+	 * For V1.1 we need to set the following:
+	 *  TFBCCompressionControlLossyMinChannel = 0x1
+	 *  TFBCCompressionControlYUVFormat = 0x0
+	 *  TFBCCompressionControlScheme = 0x1
+	 *  TFBCCompressionControlGroup = 0 / 1 (depends on LOSSY_37_PERCENT)
+	 * The gating for these values depends on whether the GPU supports
+	 * RGX_FEATURE_TFBC_VERSION = 20U
+	 */
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, TFBC_VERSION))
+	{
+		ui32TFBCVersion = RGX_GET_FEATURE_VALUE(psDevInfo, TFBC_VERSION);
+
+		if (ui32TFBCVersion >= 20U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 11:	/* TFBC Version 1.1 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = false;
+					psHints->ui32TFBCCompressionControlScheme = 1U;
+					psHints->ui32TFBCCompressionControlGroup =
+					    PVRSRV_APPHINT_TFBCCOMPRESSIONCONTROLGROUP;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.1, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+					         " Ignoring. Using value %u instead",
+					         __func__, psHints->ui32TFBCVersion,
+					         ui32TFBCVersion));
+					break;
+			}
+
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else if (ui32TFBCVersion == 11U)
+		{
+			switch (psHints->ui32TFBCVersion) {
+				case 10:	/* TFBC Version 1.0 */
+					psHints->bTFBCCompressionControlLossyMinChannel = true;
+					psHints->bTFBCCompressionControlYUVFormat = true;
+					psHints->ui32TFBCCompressionControlScheme = 2U;
+					psHints->ui32TFBCCompressionControlGroup = 0U;
+
+#if defined(DEBUG)
+					PVR_LOG(("%s: Setting TFBC Version 1.0, Native v%u",
+					         __func__, ui32TFBCVersion));
+#endif
+					break;
+
+				case 0:		/* Leave with whatever the ui32TFBCVersion is */
+					break;
+
+				default:	/* Unexpected / unsupported value */
+					PVR_DPF((PVR_DBG_WARNING,
+					         "%s: Unexpected TFBC Version %u"
+				             " Ignoring. Using value %u instead",
+				             __func__, psHints->ui32TFBCVersion,
+				             ui32TFBCVersion));
+					break;
+			}
+			ui32FWConfigFlagsExt |=
+				((((psHints->ui32TFBCCompressionControlGroup  << RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_GROUP_CONTROL_CLRMSK) |
+				  ((psHints->ui32TFBCCompressionControlScheme << RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_SHIFT) &
+															    ~RGX_CR_TFBC_COMPRESSION_CONTROL_SCHEME_CLRMSK) |
+				  ((psHints->bTFBCCompressionControlYUVFormat) ? RGX_CR_TFBC_COMPRESSION_CONTROL_YUV10_OVERRIDE_EN : 0) |
+				  ((psHints->bTFBCCompressionControlLossyMinChannel) ? RGX_CR_TFBC_COMPRESSION_CONTROL_LOSSY_MIN_CHANNEL_OVERRIDE_EN : 0))
+				<< RGXFWIF_INICFG_EXT_TFBC_CONTROL_SHIFT) & RGXFWIF_INICFG_EXT_TFBC_CONTROL_MASK;
+			/* Save the CompressionControlGroup for later use by
+			 * ->pfnGetTFBCLossyGroup() */
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = psHints->ui32TFBCCompressionControlGroup;
+#endif
+		}
+		else	/* TFBC v1.0 */
+		{
+			PVR_UNREFERENCED_PARAMETER(ui32FWConfigFlagsSupValExt);
+#if defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+			psDevInfo->ui32TFBCLossyGroup = 0;
+#endif
+			if ((psHints->ui32TFBCVersion != 0U) &&
+			    (psHints->ui32TFBCVersion != ui32TFBCVersion))
+			{
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s: Cannot specify TFBC version %u"
+				         " on a version %u GPU core", __func__,
+				         psHints->ui32TFBCVersion, ui32TFBCVersion));
+			}
+		}
+	}
+#endif /* defined(RGX_FEATURE_TFBC_VERSION_MAX_VALUE_IDX) */
+
+	*pui32FWConfigFlags    = ui32FWConfigFlags;
+	*pui32FWConfigFlagsExt = ui32FWConfigFlagsExt;
+	*pui32FwOsCfgFlags     = ui32FwOsCfgFlags;
+}
+
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+/*!
+*******************************************************************************
+
+ @Function      RGXTDProcessFWImage
+
+ @Description   Fetch and send data used by the trusted device to complete
+                the FW image setup
+
+ @Input         psDeviceNode : Device node
+ @Input         psFWParams : Firmware and parameters used by the FW
+
+ @Return        PVRSRV_ERROR
+******************************************************************************/
+static PVRSRV_ERROR RGXTDProcessFWImage(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                        PVRSRV_FW_PARAMS *psFWParams)
+{
+	PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+#endif
+	PVRSRV_ERROR eError;
+
+	if (psDevConfig->pfnTDSendFWImage == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: TDSendFWImage not implemented!", __func__));
+		return PVRSRV_ERROR_NOT_IMPLEMENTED;
+	}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		if (psFWParams->uFWP.sMips.ui32FWPageTableNumPages > TD_MAX_NUM_MIPS_PAGETABLE_PAGES)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Number of page table pages %u greater "
+					 "than what is allowed by the TD interface (%u), FW might "
+					 "not work properly!", __func__,
+					 psFWParams->uFWP.sMips.ui32FWPageTableNumPages,
+					 TD_MAX_NUM_MIPS_PAGETABLE_PAGES));
+		}
+	}
+#endif
+
+	eError = psDevConfig->pfnTDSendFWImage(psDevConfig->hSysData, psFWParams);
+
+	return eError;
+}
+#endif
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXAcquireMipsBootldrData
+
+ @Description   Acquire MIPS bootloader data parameters
+
+ @Input         psDeviceNode : Device node
+ @Input         puFWParams   : FW boot parameters
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR RGXAcquireMipsBootldrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                              PVRSRV_FW_BOOT_PARAMS *puFWParams)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice;
+	MMU_DEVICEATTRIBS *psFWMMUDevAttrs = psDevInfo->psDeviceNode->psFirmwareMMUDevAttrs;
+	IMG_DEV_PHYADDR sAddr;
+	IMG_UINT32 ui32PTSize, i;
+	PVRSRV_ERROR eError;
+	IMG_BOOL bValid;
+
+	/* Rogue Registers physical address */
+#if defined(SUPPORT_ALT_REGBASE)
+	puFWParams->sMips.sGPURegAddr = psDeviceNode->psDevConfig->sAltRegsGpuPBase;
+#else
+	PhysHeapCpuPAddrToDevPAddr(psDevInfo->psDeviceNode->apsPhysHeap[PVRSRV_PHYS_HEAP_GPU_LOCAL],
+	                           1,
+	                           &puFWParams->sMips.sGPURegAddr,
+	                           &(psDeviceNode->psDevConfig->sRegsCpuPBase));
+#endif
+
+	/* MIPS Page Table physical address */
+	MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sAddr);
+
+	/* MIPS Page Table allocation is contiguous. Pass one or more addresses
+	 * to the FW depending on the Page Table size and alignment. */
+
+	ui32PTSize = (psFWMMUDevAttrs->psTopLevelDevVAddrConfig->uiNumEntriesPT)
+	             << RGXMIPSFW_LOG2_PTE_ENTRY_SIZE;
+	ui32PTSize = PVR_ALIGN(ui32PTSize, 1U << psFWMMUDevAttrs->ui32BaseAlign);
+
+	puFWParams->sMips.ui32FWPageTableLog2PageSize = psFWMMUDevAttrs->ui32BaseAlign;
+	puFWParams->sMips.ui32FWPageTableNumPages = ui32PTSize >> psFWMMUDevAttrs->ui32BaseAlign;
+
+	if (puFWParams->sMips.ui32FWPageTableNumPages > 4U)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Page table cannot be mapped by the FW "
+		         "(size 0x%x, log2 page size %u, %u pages)",
+		         __func__, ui32PTSize, puFWParams->sMips.ui32FWPageTableLog2PageSize,
+		         puFWParams->sMips.ui32FWPageTableNumPages));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	/* Confirm page alignment fits in 64-bits */
+	if (psFWMMUDevAttrs->ui32BaseAlign > 63)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Invalid page alignment "
+		         "(psFWMMUDevAttrs->ui32BaseAlign = %u)",
+		         __func__, psFWMMUDevAttrs->ui32BaseAlign));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	for (i = 0; i < puFWParams->sMips.ui32FWPageTableNumPages; i++)
+	{
+		puFWParams->sMips.asFWPageTableAddr[i].uiAddr =
+		    sAddr.uiAddr + i * (1ULL << psFWMMUDevAttrs->ui32BaseAlign);
+	}
+
+	/* MIPS Stack Pointer Physical Address */
+	eError = RGXGetPhyAddr(psDevInfo->psRGXFWDataMemDesc->psImport->hPMR,
+	                       &puFWParams->sMips.sFWStackAddr,
+	                       RGXGetFWImageSectionOffset(NULL, MIPS_STACK),
+	                       OSGetPageShift(),
+	                       1,
+	                       &bValid);
+
+	return eError;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      InitFirmware
+
+ @Description   Allocate, initialise and pdump Firmware code and data memory
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitFirmware(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	OS_FW_IMAGE       *psRGXFW = NULL;
+	PVRSRV_FW_PARAMS  sFWParams = {0};
+
+	/* FW code memory */
+	IMG_DEVMEM_SIZE_T uiFWCodeAllocSize;
+	void              *pvFWCodeHostAddr;
+
+	/* FW data memory */
+	IMG_DEVMEM_SIZE_T uiFWDataAllocSize;
+	void              *pvFWDataHostAddr;
+
+	/* FW coremem code memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememCodeAllocSize;
+	void              *pvFWCorememCodeHostAddr = NULL;
+
+	/* FW coremem data memory */
+	IMG_DEVMEM_SIZE_T uiFWCorememDataAllocSize;
+	void              *pvFWCorememDataHostAddr = NULL;
+
+	PVRSRV_FW_BOOT_PARAMS uFWParams;
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	IMG_BOOL bUseSecureFWData =
+								RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META) ||
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	                            (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS) &&
+	                             RGX_GET_FEATURE_VALUE(psDevInfo, PHYS_BUS_WIDTH) > 32) ||
+#endif
+	                            RGX_IS_FEATURE_SUPPORTED(psDevInfo, RISCV_FW_PROCESSOR);
+#endif
+
+	/*
+	 * Get pointer to Firmware image
+	 */
+	eError = RGXLoadAndGetFWData(psDeviceNode, &psRGXFW);
+
+	if (eError != PVRSRV_OK)
+	{
+		/* Error or confirmation message generated in RGXLoadAndGetFWData */
+		goto fw_load_fail;
+	}
+
+	/*
+	 * Get pointer and size
+	 */
+	sFWParams.pvFirmware       = OSFirmwareData(psRGXFW);
+	sFWParams.ui32FirmwareSize = OSFirmwareSize(psRGXFW);
+
+	/*
+	 * Allow it to be pre-processed by the platform hook
+	 */
+	if (psDeviceNode->psDevConfig->pfnPrepareFWImage != NULL)
+	{
+		eError = psDeviceNode->psDevConfig->pfnPrepareFWImage(
+				psDeviceNode->psDevConfig->hSysData, &sFWParams);
+		if (eError != PVRSRV_OK)
+			goto cleanup_initfw;
+	}
+
+	/*
+	 * Allocate Firmware memory
+	 */
+
+	eError = RGXGetFWImageAllocSize(&psDevInfo->sLayerParams,
+	                                sFWParams.pvFirmware,
+	                                sFWParams.ui32FirmwareSize,
+	                                &uiFWCodeAllocSize,
+	                                &uiFWDataAllocSize,
+	                                &uiFWCorememCodeAllocSize,
+	                                &uiFWCorememDataAllocSize,
+	                                &psDevInfo->sFWInfoHeader);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXGetFWImageAllocSize failed",
+		        __func__));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Initiate FW compatibility check for Native and Host.
+	 * Guest compatibility check must be done after FW boot.
+	 */
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = PVRSRVDevInitCompatCheck(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed compatibility check for device %p (%s)",
+					 __func__, psDeviceNode, PVRSRVGetErrorString(eError)));
+			goto cleanup_initfw;
+		}
+	}
+
+	psDevInfo->ui32FWCodeSizeInBytes = uiFWCodeAllocSize;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+    PVR_DPF((PVR_DBG_WARNING,
+            "%s: META DMA not available, disabling core memory code/data",
+            __func__));
+    uiFWCorememCodeAllocSize = 0;
+    uiFWCorememDataAllocSize = 0;
+#endif
+
+	psDevInfo->ui32FWCorememCodeSizeInBytes = uiFWCorememCodeAllocSize;
+
+	eError = RGXInitAllocFWImgMem(psDeviceNode,
+	                              uiFWCodeAllocSize,
+	                              uiFWDataAllocSize,
+	                              uiFWCorememCodeAllocSize,
+	                              uiFWCorememDataAllocSize);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: RGXInitAllocFWImgMem failed (%d)",
+		        __func__,
+		        eError));
+		goto cleanup_initfw;
+	}
+
+	/*
+	 * Acquire pointers to Firmware allocations
+	 */
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCodeHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc, &pvFWCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", cleanup_initfw);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWDataHostAddr = NULL;
+	}
+	else
+#endif
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc, &pvFWDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_code);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		/* We can't get a pointer to a secure FW allocation from within the DDK */
+		pvFWCorememCodeHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc, &pvFWCorememCodeHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_data);
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (bUseSecureFWData)
+	{
+		pvFWCorememDataHostAddr = NULL;
+	}
+	else
+#endif
+	if (uiFWCorememDataAllocSize != 0)
+	{
+		eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc, &pvFWCorememDataHostAddr);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", release_corememcode);
+	}
+
+	/*
+	 * Prepare FW boot parameters
+	 */
+	OSCachedMemSet(&uFWParams, 0, sizeof(PVRSRV_FW_BOOT_PARAMS));
+
+	if (RGX_IS_FEATURE_VALUE_SUPPORTED(psDevInfo, META))
+	{
+		uFWParams.sMeta.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sMeta.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sMeta.sFWCorememCodeFWAddr = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sMeta.uiFWCorememCodeSize = uiFWCorememCodeAllocSize;
+		uFWParams.sMeta.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sMeta.sFWCorememDataFWAddr = psDevInfo->sFWCorememDataStoreFWAddr;
+#if defined(RGXFW_META_SUPPORT_2ND_THREAD)
+		uFWParams.sMeta.ui32NumThreads = 2;
+#else
+		uFWParams.sMeta.ui32NumThreads = 1;
+#endif
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, MIPS))
+	{
+		eError = RGXAcquireMipsBootldrData(psDeviceNode, &uFWParams);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXAcquireMipsBootldrData failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+	}
+#endif
+	else
+	{
+		uFWParams.sRISCV.sFWCodeDevVAddr = psDevInfo->sFWCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWDataDevVAddr = psDevInfo->sFWDataDevVAddrBase;
+
+		uFWParams.sRISCV.sFWCorememCodeDevVAddr = psDevInfo->sFWCorememCodeDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememCodeFWAddr   = psDevInfo->sFWCorememCodeFWAddr;
+		uFWParams.sRISCV.uiFWCorememCodeSize    = uiFWCorememCodeAllocSize;
+
+		uFWParams.sRISCV.sFWCorememDataDevVAddr = psDevInfo->sFWCorememDataStoreDevVAddrBase;
+		uFWParams.sRISCV.sFWCorememDataFWAddr   = psDevInfo->sFWCorememDataStoreFWAddr;
+		uFWParams.sRISCV.uiFWCorememDataSize    = uiFWCorememDataAllocSize;
+	}
+
+
+	/*
+	 * Process the Firmware image and setup code and data segments.
+	 *
+	 * When the trusted device is enabled and the FW code lives
+	 * in secure memory we will only setup the data segments here,
+	 * while the code segments will be loaded to secure memory
+	 * by the trusted device.
+	 */
+	if (!psDeviceNode->bAutoVzFwIsUp)
+	{
+		eError = RGXProcessFWImage(&psDevInfo->sLayerParams,
+								   sFWParams.pvFirmware,
+								   pvFWCodeHostAddr,
+								   pvFWDataHostAddr,
+								   pvFWCorememCodeHostAddr,
+								   pvFWCorememDataHostAddr,
+								   &uFWParams);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: RGXProcessFWImage failed (%d)",
+					 __func__, eError));
+			goto release_fw_allocations;
+		}
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+		RGXFwSharedMemCacheOpExec(pvFWCodeHostAddr,
+		                          sizeof(psDevInfo->psRGXFWCodeMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememCodeAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememCodeHostAddr,
+			                          sizeof(psDevInfo->psRGXFWCorememCodeMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+
+		RGXFwSharedMemCacheOpExec(pvFWDataHostAddr,
+		                          sizeof(psDevInfo->psRGXFWDataMemDesc->uiAllocSize),
+		                          PVRSRV_CACHE_OP_FLUSH);
+		if (uiFWCorememDataAllocSize)
+		{
+			RGXFwSharedMemCacheOpExec(pvFWCorememDataHostAddr,
+			                          sizeof(psDevInfo->psRGXFWIfCorememDataStoreMemDesc->uiAllocSize),
+			                          PVRSRV_CACHE_OP_FLUSH);
+		}
+#endif
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	sFWParams.uFWP = uFWParams;
+	RGXTDProcessFWImage(psDeviceNode, &sFWParams);
+#endif
+
+
+	/*
+	 * PDump Firmware allocations
+	 */
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Dump firmware code image");
+	DevmemPDumpLoadMem(psDevInfo->psRGXFWCodeMemDesc,
+	                   0,
+	                   uiFWCodeAllocSize,
+	                   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware data image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWDataMemDesc,
+		                   0,
+		                   uiFWDataAllocSize,
+		                   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+#if !defined(SUPPORT_TRUSTED_DEVICE) || defined(NO_HARDWARE) || defined(SUPPORT_SECURITY_VALIDATION)
+	if (uiFWCorememCodeAllocSize != 0)
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem code image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWCorememCodeMemDesc,
+						   0,
+						   uiFWCorememCodeAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+#endif
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "Dump firmware coremem data store image");
+		DevmemPDumpLoadMem(psDevInfo->psRGXFWIfCorememDataStoreMemDesc,
+						   0,
+						   uiFWCorememDataAllocSize,
+						   PDUMP_FLAGS_CONTINUOUS);
+	}
+
+	/*
+	 * Release Firmware allocations and clean up
+	 */
+release_fw_allocations:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && uiFWCorememDataAllocSize)
+#else
+	if (uiFWCorememDataAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfCorememDataStoreMemDesc);
+	}
+
+release_corememcode:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData && (uiFWCorememCodeAllocSize != 0))
+#else
+	if (uiFWCorememCodeAllocSize != 0)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCorememCodeMemDesc);
+	}
+
+release_data:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWDataMemDesc);
+	}
+
+release_code:
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(NO_HARDWARE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	if (!bUseSecureFWData)
+#endif
+	{
+		DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWCodeMemDesc);
+	}
+cleanup_initfw:
+	OSUnloadFirmware(psRGXFW);
+fw_load_fail:
+
+	return eError;
+}
+
+
+#if defined(PDUMP)
+/*!
+*******************************************************************************
+
+ @Function      InitialiseHWPerfCounters
+
+ @Description   Initialisation of hardware performance counters and dumping
+                them out to pdump, so that they can be modified at a later
+                point.
+
+ @Input         pvDevice
+ @Input         psHWPerfDataMemDesc
+ @Input         psHWPerfInitDataInt
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseHWPerfCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     void *pvDevice,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc,
+                                     RGXFWIF_HWPERF_CTL *psHWPerfInitDataInt)
+{
+	RGXFWIF_HWPERF_CTL_BLK *psHWPerfInitBlkData;
+#if defined(HWPERF_UNIFIED)
+	RGXFWIF_HWPERF_DA_BLK *psHWPerfInitDABlkData;
+#endif
+	IMG_UINT32 ui32CntBlkModelLen;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL *asCntBlkTypeModel;
+	const RGXFW_HWPERF_CNTBLK_TYPE_MODEL* psBlkTypeDesc;
+	IMG_UINT32 ui32BlockID, ui32BlkCfgIdx, ui32CounterIdx;
+	RGX_HWPERF_CNTBLK_RT_INFO sCntBlkRtInfo;
+
+	ui32CntBlkModelLen = RGXGetHWPerfBlockConfig(&asCntBlkTypeModel);
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "HWPerf Counter Config starts here.");
+
+	for (ui32BlkCfgIdx = 0; ui32BlkCfgIdx < ui32CntBlkModelLen; ui32BlkCfgIdx++)
+	{
+		__maybe_unused IMG_UINT32 uiUnit;
+		__maybe_unused IMG_BOOL bDirect;
+
+		/* Exit early if this core does not have any of these counter blocks
+		 * due to core type/BVNC features.... */
+		psBlkTypeDesc = &asCntBlkTypeModel[ui32BlkCfgIdx];
+		if (psBlkTypeDesc->pfnIsBlkPresent(psBlkTypeDesc, pvDevice, &sCntBlkRtInfo) == IMG_FALSE)
+		{
+			continue;
+		}
+
+		/* Program all counters in one block so those already on may
+		 * be configured off and vice-versa. */
+		for (ui32BlockID = psBlkTypeDesc->ui32CntBlkIdBase;
+			 ui32BlockID < psBlkTypeDesc->ui32CntBlkIdBase+sCntBlkRtInfo.ui32NumUnits;
+			 ui32BlockID++)
+		{
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "Unit %d Block : %s",
+			                      ui32BlockID-psBlkTypeDesc->ui32CntBlkIdBase,
+			                      psBlkTypeDesc->pszBlockNameComment);
+
+			/* Get the block configure store to update from the global store of
+			 * block configuration. This is used to remember the configuration
+			 * between configurations and core power on in APM.
+			 * For HWPERF_UNIFIED layout we will have a different
+			 * structure type to decode the HWPerf block. This is indicated by
+			 * the RGX_CNTBLK_ID_DA_MASK bit being set in the block-ID value. */
+
+			bDirect = (psBlkTypeDesc->ui32IndirectReg == 0U);
+			uiUnit = ui32BlockID - psBlkTypeDesc->ui32CntBlkIdBase;
+
+#if defined(HWPERF_UNIFIED)
+			if ((ui32BlockID & RGX_CNTBLK_ID_DA_MASK) == RGX_CNTBLK_ID_DA_MASK)
+			{
+				psHWPerfInitDABlkData = rgxfw_hwperf_get_da_block_ctl(ui32BlockID, psHWPerfInitDataInt);
+
+				PVR_ASSERT(psHWPerfInitDABlkData);
+
+				psHWPerfInitDABlkData->eBlockID = ui32BlockID;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "eBlockID: The Block ID for the layout block. See RGX_HWPERF_CNTBLK_ID for further information.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->eBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->eBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitDABlkData->uiEnabled = 0U;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->uiEnabled) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->uiEnabled,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitDABlkData->uiNumCounters = 0U;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiNumCounters (X): Specifies the number of valid counters"
+			                      " [0..%d] which follow.", RGX_CNTBLK_COUNTERS_MAX);
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->uiNumCounters) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitDABlkData->uiNumCounters,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				for (ui32CounterIdx = 0; ui32CounterIdx < RGX_CNTBLK_COUNTERS_MAX; ui32CounterIdx++)
+				{
+					psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx] = IMG_UINT32_C(0x00000000);
+
+					if (bDirect)
+					{
+						PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+					                      "%s_COUNTER_%d",
+					                      psBlkTypeDesc->pszBlockNameComment,
+					                      ui32CounterIdx);
+					}
+					else
+					{
+						PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+					                      "%s%d_COUNTER_%d",
+					                      psBlkTypeDesc->pszBlockNameComment,
+					                      uiUnit, ui32CounterIdx);
+					}
+
+					DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+					        psHWPerfInitDABlkData->aui32Counters[ui32CounterIdx],
+							PDUMP_FLAGS_CONTINUOUS);
+				}
+			}
+			else
+#endif	/* defined(HWPERF_UNIFIED) */
+			{
+				psHWPerfInitBlkData = rgxfw_hwperf_get_block_ctl(ui32BlockID, psHWPerfInitDataInt);
+				/* Assert to check for HWPerf block mis-configuration */
+				PVR_ASSERT(psHWPerfInitBlkData);
+
+				psHWPerfInitBlkData->ui32Valid = IMG_TRUE;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "bValid: This specifies if the layout block is valid for the given BVNC.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->ui32Valid) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->ui32Valid,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->ui32Enabled = IMG_FALSE;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "bEnabled: Set to 0x1 if the block needs to be enabled during playback.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->ui32Enabled) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->ui32Enabled,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->eBlockID = ui32BlockID;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "eBlockID: The Block ID for the layout block. See RGX_HWPERF_CNTBLK_ID for further information.");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->eBlockID) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->eBlockID,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				psHWPerfInitBlkData->uiCounterMask = 0x00;
+				PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "uiCounterMask: Bitmask for selecting the counters that need to be configured. (Bit 0 - counter0, bit 1 - counter1 and so on.)");
+				DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->uiCounterMask) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->uiCounterMask,
+							PDUMP_FLAGS_CONTINUOUS);
+
+				for (ui32CounterIdx = RGX_CNTBLK_COUNTER0_ID; ui32CounterIdx < psBlkTypeDesc->ui8NumCounters; ui32CounterIdx++)
+				{
+					psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx] = IMG_UINT64_C(0x0000000000000000);
+
+					PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+				                      "%s_COUNTER_%d", psBlkTypeDesc->pszBlockNameComment, ui32CounterIdx);
+					DevmemPDumpLoadMemValue64(psHWPerfDataMemDesc,
+							(size_t)&(psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx]) - (size_t)(psHWPerfInitDataInt),
+							psHWPerfInitBlkData->aui64CounterCfg[ui32CounterIdx],
+							PDUMP_FLAGS_CONTINUOUS);
+
+				}
+			}
+		}
+	}
+}
+/*!
+*******************************************************************************
+
+ @Function      InitialiseCustomCounters
+
+ @Description   Initialisation of custom counters and dumping them out to
+                pdump, so that they can be modified at a later point.
+
+ @Input         psHWPerfDataMemDesc
+
+ @Return        void
+
+******************************************************************************/
+
+static void InitialiseCustomCounters(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     DEVMEM_MEMDESC *psHWPerfDataMemDesc)
+{
+	IMG_UINT32 ui32CustomBlock, ui32CounterID;
+
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "ui32SelectedCountersBlockMask - The Bitmask of the custom counters that are to be selected");
+	DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+						offsetof(RGXFWIF_HWPERF_CTL, ui32SelectedCountersBlockMask),
+						0,
+						PDUMP_FLAGS_CONTINUOUS);
+
+	for (ui32CustomBlock = 0; ui32CustomBlock < RGX_HWPERF_MAX_CUSTOM_BLKS; ui32CustomBlock++)
+	{
+		/*
+		 * Some compilers cannot cope with the use of offsetof() below - the specific problem being the use of
+		 * a non-const variable in the expression, which it needs to be const. Typical compiler error produced is
+		 * "expression must have a constant value".
+		 */
+		const IMG_DEVMEM_OFFSET_T uiOffsetOfCustomBlockSelectedCounters
+		= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_HWPERF_CTL *)0)->SelCntr[ui32CustomBlock].ui32NumSelectedCounters);
+
+		PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+		                      "ui32NumSelectedCounters - The Number of counters selected for this Custom Block: %d",ui32CustomBlock );
+		DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+					uiOffsetOfCustomBlockSelectedCounters,
+					0,
+					PDUMP_FLAGS_CONTINUOUS);
+
+		for (ui32CounterID = 0; ui32CounterID < RGX_HWPERF_MAX_CUSTOM_CNTRS; ui32CounterID++ )
+		{
+			const IMG_DEVMEM_OFFSET_T uiOffsetOfCustomBlockSelectedCounterIDs
+			= (IMG_DEVMEM_OFFSET_T)(uintptr_t)&(((RGXFWIF_HWPERF_CTL *)0)->SelCntr[ui32CustomBlock].aui32SelectedCountersIDs[ui32CounterID]);
+
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+			                      "CUSTOMBLK_%d_COUNTERID_%d",ui32CustomBlock, ui32CounterID);
+			DevmemPDumpLoadMemValue32(psHWPerfDataMemDesc,
+					uiOffsetOfCustomBlockSelectedCounterIDs,
+					0,
+					PDUMP_FLAGS_CONTINUOUS);
+		}
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      InitialiseAllCounters
+
+ @Description   Initialise HWPerf and custom counters
+
+ @Input         psDeviceNode : Device Node
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR InitialiseAllCounters(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	RGXFWIF_HWPERF_CTL *psHWPerfInitData;
+	PVRSRV_ERROR eError;
+
+	eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc, (void **)&psHWPerfInitData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", failHWPerfCountersMemDescAqCpuVirt);
+
+	InitialiseHWPerfCounters(psDeviceNode, psDevInfo, psDevInfo->psRGXFWIfHWPerfCountersMemDesc, psHWPerfInitData);
+	InitialiseCustomCounters(psDeviceNode, psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+	RGXFwSharedMemCacheOpPtr(psHWPerfInitData, FLUSH);
+
+failHWPerfCountersMemDescAqCpuVirt:
+	DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfHWPerfCountersMemDesc);
+
+	return eError;
+}
+#endif /* PDUMP */
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+static PVRSRV_ERROR RGXValidateTDHeap(PVRSRV_DEVICE_NODE *psDeviceNode,
+									  PVRSRV_PHYS_HEAP ePhysHeap,
+									  PHYS_HEAP_USAGE_FLAGS ui32RequiredFlags)
+{
+	PHYS_HEAP *psHeap = psDeviceNode->apsPhysHeap[ePhysHeap];
+	PHYS_HEAP_USAGE_FLAGS ui32HeapFlags = PhysHeapGetFlags(psHeap);
+	PHYS_HEAP_USAGE_FLAGS ui32InvalidFlags = ~(PHYS_HEAP_USAGE_FW_PRIV_DATA | PHYS_HEAP_USAGE_FW_CODE
+											   | PHYS_HEAP_USAGE_GPU_SECURE | PHYS_HEAP_USAGE_FW_PRIVATE);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32RequiredFlags) != 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap is missing required flags. flags: 0x%x / required:0x%x",
+							   ui32HeapFlags,
+							   ui32RequiredFlags);
+
+	PVR_LOG_RETURN_IF_FALSE_VA((ui32HeapFlags & ui32InvalidFlags) == 0,
+							   PVRSRV_ERROR_NOT_SUPPORTED,
+							   "TD heap uses invalid flags. flags: 0x%x / invalid:0x%x",
+							   ui32HeapFlags,
+							   ui32InvalidFlags);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXValidateTDHeaps(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_PRIV_DATA, PHYS_HEAP_USAGE_FW_PRIV_DATA);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_PRIV_DATA");
+
+		eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_FW_CODE, PHYS_HEAP_USAGE_FW_CODE);
+		PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:FW_CODE");
+	}
+
+	eError = RGXValidateTDHeap(psDeviceNode, PVRSRV_PHYS_HEAP_GPU_SECURE, PHYS_HEAP_USAGE_GPU_SECURE);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeap:GPU_SECURE");
+
+	return PVRSRV_OK;
+}
+#endif
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInit
+
+ @Description   RGX Initialisation
+
+ @Input         psDeviceNode
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Services initialisation parameters */
+	RGX_INIT_APPHINTS sApphints = {0};
+	IMG_UINT32 ui32FWConfigFlags, ui32FWConfigFlagsExt, ui32FwOsCfgFlags;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PDUMPCOMMENT(psDeviceNode, "RGX Initialisation Part 1");
+
+	PDUMPCOMMENT(psDeviceNode, "Device Name: %s",
+	             psDeviceNode->psDevConfig->pszName);
+	PDUMPCOMMENT(psDeviceNode, "Device ID: %u (%d)",
+	             psDeviceNode->sDevId.ui32InternalID,
+	             psDeviceNode->sDevId.i32KernelDeviceID);
+
+	if (psDeviceNode->psDevConfig->pszVersion)
+	{
+		PDUMPCOMMENT(psDeviceNode, "Device Version: %s",
+		             psDeviceNode->psDevConfig->pszVersion);
+	}
+
+	/* pdump info about the core */
+	PDUMPCOMMENT(psDeviceNode,
+	             "RGX Version Information (KM): %d.%d.%d.%d",
+	             psDevInfo->sDevFeatureCfg.ui32B,
+	             psDevInfo->sDevFeatureCfg.ui32V,
+	             psDevInfo->sDevFeatureCfg.ui32N,
+	             psDevInfo->sDevFeatureCfg.ui32C);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	eError = RGXValidateTDHeaps(psDeviceNode);
+	PVR_LOG_RETURN_IF_ERROR(eError, "RGXValidateTDHeaps");
+#endif
+
+#if defined(SUPPORT_AUTOVZ)
+	if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+	{
+		/* The RGX_CR_MTS_DM4_INTERRUPT_ENABLE register is always set by the firmware during initialisation
+		 * and it provides a good method of determining if the firmware has been booted previously */
+		psDeviceNode->bAutoVzFwIsUp = (OSReadHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_DM4_INTERRUPT_ENABLE) != 0);
+
+		PVR_LOG(("AutoVz startup check: firmware is %s;",
+				(psDeviceNode->bAutoVzFwIsUp) ? "already running" : "powered down"));
+		PVR_LOG(("AutoVz allow GPU powerdown is %s:",
+				(psDeviceNode->bAutoVzAllowGPUPowerdown) ? "enabled" : "disabled"));
+	}
+	else if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* Guest assumes the firmware is always available */
+		psDeviceNode->bAutoVzFwIsUp = IMG_TRUE;
+		psDeviceNode->bAutoVzAllowGPUPowerdown = IMG_FALSE;
+	}
+	else
+#endif
+	{
+		/* Firmware does not follow the AutoVz life-cycle */
+		psDeviceNode->bAutoVzFwIsUp = IMG_FALSE;
+	}
+
+	if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode) || (psDeviceNode->bAutoVzFwIsUp))
+	{
+		/* set the device power state here as the regular power
+		 * callbacks will not be executed on this driver */
+		psDevInfo->bRGXPowered = IMG_TRUE;
+	}
+
+	/* Set which HW Safety Events will be handled by the driver */
+	psDevInfo->ui32HostSafetyEventMask |= RGX_IS_FEATURE_SUPPORTED(psDevInfo, WATCHDOG_TIMER) ?
+										  RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__WATCHDOG_TIMEOUT_EN : 0;
+	psDevInfo->ui32HostSafetyEventMask |= (RGX_DEVICE_HAS_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS)
+										   && (RGX_DEVICE_GET_FEATURE_VALUE(&psDevInfo->sLayerParams, ECC_RAMS) > 0)) ?
+										  RGX_CR_SAFETY_EVENT_STATUS__ROGUEXE__FAULT_FW_EN : 0;
+
+#if defined(PDUMP)
+	PDUMPCOMMENTWITHFLAGS(psDeviceNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Register defs revision: %d", RGX_CR_DEFS_KM_REVISION);
+#endif
+
+	/* Services initialisation parameters */
+	GetApphints(psDevInfo, &sApphints);
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#if defined(EMULATOR)
+	if ((sApphints.bEnableTrustedDeviceAceConfig) &&
+		(RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE)))
+	{
+		SetTrustedDeviceAceEnabled(psDeviceNode->psDevConfig->hSysData);
+	}
+#endif
+#endif
+
+	eError = RGXInitCreateFWKernelMemoryContext(psDeviceNode);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create FW kernel memory context (%u)",
+		         __func__, eError));
+		goto cleanup;
+	}
+
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitFirmware(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitFirmware failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+
+	/*
+	 * Setup Firmware initialisation data
+	 */
+
+	GetFWConfigFlags(psDeviceNode, &sApphints, &ui32FWConfigFlags, &ui32FWConfigFlagsExt, &ui32FwOsCfgFlags);
+
+	eError = RGXInitFirmware(psDeviceNode,
+	                         &sApphints,
+	                         ui32FWConfigFlags,
+	                         ui32FWConfigFlagsExt,
+	                         ui32FwOsCfgFlags);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitFirmware failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+#if defined(PDUMP)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		eError = InitialiseAllCounters(psDeviceNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: InitialiseAllCounters failed (%d)",
+					 __func__, eError));
+			goto cleanup;
+		}
+	}
+#endif
+
+	/*
+	 * Perform second stage of RGX initialisation
+	 */
+	eError = RGXInitDevPart2(psDeviceNode,
+	                         &sApphints);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: RGXInitDevPart2 failed (%d)",
+				 __func__, eError));
+		goto cleanup;
+	}
+
+
+	eError = PVRSRV_OK;
+
+cleanup:
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxsrvinit.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxstartstop.c b/drivers/gpu/drm/img-rogue/rgxstartstop.c
new file mode 100644
index 000000000000..2b93b46ff3ef
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxstartstop.c
@@ -0,0 +1,1245 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific start/stop routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific start/stop routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* The routines implemented here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when trusted device is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary. */
+#include "rgxstartstop.h"
+
+#define SOC_FEATURE_STRICT_SAME_ADDRESS_WRITE_ORDERING
+
+/*
+	RGXWriteMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXWriteMetaRegThroughSP(const void *hPrivate,
+                                      IMG_UINT32 ui32RegAddr,
+                                      IMG_UINT32 ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAT__HOST_SECURITY_GT1_AND_MRUA;
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAT;
+	}
+
+	eError = RGXPollReg32(hPrivate, ui32StateReg, ui32StateReadyFlag, ui32StateReadyFlag);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Write */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+		RGXWriteReg32(hPrivate, ui32DataReg, ui32RegValue);
+		(void) RGXReadReg32(hPrivate, ui32DataReg); /* Fence write */
+	}
+
+	return eError;
+}
+
+/*
+	RGXReadMetaRegThroughSP
+*/
+PVRSRV_ERROR RGXReadMetaRegThroughSP(const void *hPrivate,
+                                     IMG_UINT32 ui32RegAddr,
+                                     IMG_UINT32* ui32RegValue)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32StateReg, ui32StateReadyFlag;
+	IMG_UINT32 ui32CtrlReg, ui32DataReg;
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+	{
+		/* ensure the meta_registers_unpacked_accesses auto-increment feature is not used */
+		BITMASK_UNSET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_AUTOINCR_EN);
+
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_GT1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_GT1_AND_MRUA__RD_EN);
+		}
+		else
+		{
+			ui32StateReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN |
+								 RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN;
+			ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA;
+			ui32DataReg = RGX_CR_META_SP_MSLVDATAX__HOST_SECURITY_EQ1_AND_MRUA;
+			BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0__HOST_SECURITY_EQ1_AND_MRUA__RD_EN);
+		}
+	}
+	else
+#endif
+	{
+		ui32StateReg = RGX_CR_META_SP_MSLVCTRL1;
+		ui32StateReadyFlag = RGX_CR_META_SP_MSLVCTRL1_READY_EN |
+							 RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN;
+		ui32CtrlReg = RGX_CR_META_SP_MSLVCTRL0;
+		ui32DataReg = RGX_CR_META_SP_MSLVDATAX;
+		BITMASK_SET(ui32RegAddr, RGX_CR_META_SP_MSLVCTRL0_RD_EN);
+	}
+
+	/* Wait for Slave Port to be Ready */
+	eError = RGXPollReg32(hPrivate,
+						  ui32StateReg,
+						  ui32StateReadyFlag,
+						  ui32StateReadyFlag);
+	if (eError == PVRSRV_OK)
+	{
+		/* Issue a Read */
+		RGXWriteReg32(hPrivate, ui32CtrlReg, ui32RegAddr);
+		(void) RGXReadReg32(hPrivate, ui32CtrlReg); /* Fence write */
+
+		/* Wait for Slave Port to be Ready */
+		eError = RGXPollReg32(hPrivate,
+				  ui32StateReg,
+				  ui32StateReadyFlag,
+				  ui32StateReadyFlag);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+#if !defined(NO_HARDWARE)
+	*ui32RegValue = RGXReadReg32(hPrivate, ui32DataReg);
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32DataReg);
+	*ui32RegValue = 0xFFFFFFFF;
+#endif
+
+	return eError;
+}
+
+static PVRSRV_ERROR RGXWriteMetaCoreRegThoughSP(const void *hPrivate,
+                                                IMG_UINT32 ui32CoreReg,
+                                                IMG_UINT32 ui32Value)
+{
+	IMG_UINT32 i = 0;
+
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXDT_OFFSET, ui32Value);
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, ui32CoreReg & ~META_CR_TXUXXRXRQ_RDnWR_BIT);
+
+	do
+	{
+		RGXReadMetaRegThroughSP(hPrivate, META_CR_TXUXXRXRQ_OFFSET, &ui32Value);
+	} while (((ui32Value & META_CR_TXUXXRXRQ_DREADY_BIT) != META_CR_TXUXXRXRQ_DREADY_BIT) && (i++ < 1000));
+
+	if (i == 1000)
+	{
+		RGXCommentLog(hPrivate, "RGXWriteMetaCoreRegThoughSP: Timeout");
+		return PVRSRV_ERROR_TIMEOUT;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RGXStartFirmware(const void *hPrivate)
+{
+	PVRSRV_ERROR eError;
+
+	/* Give privilege to debug and slave port */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_SYSC_JTAG_THREAD, META_CR_SYSC_JTAG_THREAD_PRIV_EN);
+
+	/* Point Meta to the bootloader address, global (uncached) range */
+	eError = RGXWriteMetaCoreRegThoughSP(hPrivate,
+	                                     PC_ACCESS(0),
+	                                     RGXFW_BOOTLDR_META_ADDR | META_MEM_GLOBAL_RANGE_BIT);
+
+	if (eError != PVRSRV_OK)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start failed!");
+		return eError;
+	}
+
+	/* Enable minim encoding */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_TXPRIVEXT, META_CR_TXPRIVEXT_MINIM_EN);
+
+	/* Enable Meta thread */
+	RGXWriteMetaRegThroughSP(hPrivate, META_CR_T0ENABLE_OFFSET, META_CR_TXENABLE_ENABLE_BIT);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMetaProcWrapper
+
+ @Description   Configures the hardware wrapper of the META processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMetaProcWrapper(const void *hPrivate)
+{
+	IMG_UINT64 ui64GartenConfig;
+
+	/* Garten IDLE bit controlled by META */
+	ui64GartenConfig = RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META;
+
+#if defined(RGX_CR_MTS_GARTEN_WRAPPER_CONFIG__S7_INFRA__FENCE_PC_BASE_SHIFT)
+	/* Set the Garten Wrapper BIF Fence address */
+	{
+		/* Set PC = 0 for fences */
+		ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK;
+		ui64GartenConfig |= (IMG_UINT64)MMU_CONTEXT_MAPPING_FWPRIV
+		                    << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_SHIFT;
+
+		/* Set SLC DM=META */
+		ui64GartenConfig |= ((IMG_UINT64) RGXFW_SEGMMU_META_BIFDM_ID) << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT;
+	}
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure META wrapper");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, ui64GartenConfig);
+}
+
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+/*!
+*******************************************************************************
+
+ @Function      RGXInitMipsProcWrapper
+
+ @Description   Configures the hardware wrapper of the MIPS processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitMipsProcWrapper(const void *hPrivate)
+{
+	IMG_DEV_PHYADDR sPhyAddr;
+	IMG_UINT64 ui64RemapSettings = RGXMIPSFW_BOOT_REMAP_LOG2_SEGMENT_SIZE; /* Same for all remap registers */
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure MIPS wrapper");
+
+	/*
+	 * MIPS wrapper (registers transaction ID and ISA mode) setup
+	 */
+
+	RGXCommentLog(hPrivate, "RGXStart: Write wrapper config register");
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		RGXWriteReg32(hPrivate,
+		              RGX_CR_MIPS_WRAPPER_CONFIG,
+		              (RGXMIPSFW_REGISTERS_VIRTUAL_BASE >>
+		              RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN) |
+		              RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+	else
+	{
+		RGXAcquireGPURegsAddr(hPrivate, &sPhyAddr);
+
+		RGXMIPSWrapperConfig(hPrivate,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG,
+		                     sPhyAddr.uiAddr,
+		                     RGXMIPSFW_WRAPPER_CONFIG_REGBANK_ADDR_ALIGN,
+		                     RGX_CR_MIPS_WRAPPER_CONFIG_BOOT_ISA_MODE_MICROMIPS);
+	}
+
+	/*
+	 * Boot remap setup
+	 */
+
+	RGXAcquireBootRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write boot remap registers");
+	RGXBootRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG1,
+	                   RGXMIPSFW_BOOT_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP1_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP1_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+#if defined(FIX_HW_BRN_63553_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 63553))
+	{
+		IMG_BOOL bPhysBusAbove32Bit = RGXGetDevicePhysBusWidth(hPrivate) > 32;
+		IMG_BOOL bDevicePA0IsValid  = RGXDevicePA0IsValid(hPrivate);
+
+		/* WA always required on 36 bit cores, to avoid continuous unmapped memory accesses to address 0x0 */
+		if (bPhysBusAbove32Bit || !bDevicePA0IsValid)
+		{
+			RGXCodeRemapConfig(hPrivate,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG1,
+					0x0 | RGX_CR_MIPS_ADDR_REMAP5_CONFIG1_MODE_ENABLE_EN,
+					RGX_CR_MIPS_ADDR_REMAP5_CONFIG2,
+					sPhyAddr.uiAddr,
+					~RGX_CR_MIPS_ADDR_REMAP5_CONFIG2_ADDR_OUT_CLRMSK,
+					ui64RemapSettings);
+		}
+	}
+#endif
+
+	/*
+	 * Data remap setup
+	 */
+
+	RGXAcquireDataRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	if (RGXGetDevicePhysBusWidth(hPrivate) > 32)
+	{
+		/* Remapped private data in secure memory */
+		ui64RemapSettings |= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_EN;
+	}
+	else
+	{
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+	}
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+	RGXCommentLog(hPrivate, "RGXStart: Write data remap registers");
+	RGXDataRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG1,
+	                   RGXMIPSFW_DATA_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP2_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP2_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP2_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	/*
+	 * Code remap setup
+	 */
+
+	RGXAcquireCodeRemapAddr(hPrivate, &sPhyAddr);
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+	/* Do not mark accesses to a FW code remap region as DRM accesses */
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+	ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+#if defined(MIPS_FW_CODE_OSID)
+	ui64RemapSettings |= ((IMG_UINT64) MIPS_FW_CODE_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#else
+	ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+#endif
+
+	RGXCommentLog(hPrivate, "RGXStart: Write exceptions remap registers");
+	RGXCodeRemapConfig(hPrivate,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG1,
+	                   RGXMIPSFW_CODE_REMAP_PHYS_ADDR_IN | RGX_CR_MIPS_ADDR_REMAP3_CONFIG1_MODE_ENABLE_EN,
+	                   RGX_CR_MIPS_ADDR_REMAP3_CONFIG2,
+	                   sPhyAddr.uiAddr,
+	                   ~RGX_CR_MIPS_ADDR_REMAP3_CONFIG2_ADDR_OUT_CLRMSK,
+	                   ui64RemapSettings);
+
+	if (RGXGetDevicePhysBusWidth(hPrivate) == 32)
+	{
+		/*
+		 * Trampoline remap setup
+		 */
+
+		RGXAcquireTrampolineRemapAddr(hPrivate, &sPhyAddr);
+		ui64RemapSettings = RGXMIPSFW_TRAMPOLINE_LOG2_SEGMENT_SIZE;
+
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		/* Remapped data in non-secure memory */
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_TRUSTED_CLRMSK;
+#endif
+
+		ui64RemapSettings &= RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_CLRMSK;
+		ui64RemapSettings |= ((IMG_UINT64) FW_OSID) << RGX_CR_MIPS_ADDR_REMAP1_CONFIG2_OS_ID_SHIFT;
+
+		RGXCommentLog(hPrivate, "RGXStart: Write trampoline remap registers");
+		RGXTrampolineRemapConfig(hPrivate,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG1,
+		                         sPhyAddr.uiAddr | RGX_CR_MIPS_ADDR_REMAP4_CONFIG1_MODE_ENABLE_EN,
+		                         RGX_CR_MIPS_ADDR_REMAP4_CONFIG2,
+		                         RGXMIPSFW_TRAMPOLINE_TARGET_PHYS_ADDR,
+		                         ~RGX_CR_MIPS_ADDR_REMAP4_CONFIG2_ADDR_OUT_CLRMSK,
+		                         ui64RemapSettings);
+	}
+
+	/* Garten IDLE bit controlled by MIPS */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to MIPS");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+
+	/* Turn on the EJTAG probe (only useful driver live) */
+	RGXWriteReg32(hPrivate, RGX_CR_MIPS_DEBUG_CONFIG, 0);
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitRiscvProcWrapper
+
+ @Description   Configures the hardware wrapper of the RISCV processor
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitRiscvProcWrapper(const void *hPrivate)
+{
+	IMG_UINT32 ui32BootCodeRemap = RGXRISCVFW_BOOTLDR_CODE_REMAP;
+	IMG_UINT32 ui32BootDataRemap = RGXRISCVFW_BOOTLDR_DATA_REMAP;
+	IMG_DEV_VIRTADDR sTmp;
+
+	RGXCommentLog(hPrivate, "RGXStart: Configure RISCV wrapper");
+
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) < 4)
+#endif
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Write boot code remap");
+		RGXAcquireBootCodeAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootCodeRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN);
+
+		RGXCommentLog(hPrivate, "RGXStart: Write boot data remap");
+		RGXAcquireBootDataAddr(hPrivate, &sTmp);
+		RGXWriteReg64(hPrivate,
+		              ui32BootDataRemap,
+		              sTmp.uiAddr |
+		              (IMG_UINT64) (RGX_FIRMWARE_RAW_HEAP_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
+		                << RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT |
+		              (IMG_UINT64) MMU_CONTEXT_MAPPING_FWPRIV << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT |
+#if defined(SUPPORT_TRUSTED_DEVICE) && defined(RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN)
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN |
+#endif
+		              RGX_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN);
+	}
+
+	/* Garten IDLE bit controlled by RISCV */
+	RGXCommentLog(hPrivate, "RGXStart: Set GARTEN_IDLE type to RISCV");
+	RGXWriteReg64(hPrivate, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META);
+}
+
+
+/*!
+*******************************************************************************
+
+ @Function      __RGXInitSLC
+
+ @Description   Initialise RGX SLC
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void __RGXInitSLC(const void *hPrivate)
+{
+	{
+		IMG_UINT32 ui32Reg;
+		IMG_UINT32 ui32RegVal;
+		IMG_UINT64 ui64RegVal;
+
+		/*
+		 * SLC Bypass control
+		 */
+		ui32Reg = RGX_CR_SLC_CTRL_BYPASS;
+		ui64RegVal = 0;
+
+#if defined(RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN)
+		if ((RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, SLC_SIZE_IN_KILOBYTES) == 8)  ||
+		    RGX_DEVICE_HAS_BRN(hPrivate, 61450))
+		{
+			RGXCommentLog(hPrivate, "Bypass SLC for IPF_OBJ and IPF_CPF");
+			ui64RegVal |= (IMG_UINT64) RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_OBJ_EN |
+						(IMG_UINT64) RGX_CR_SLC_CTRL_BYPASS_REQ_IPF_CPF_EN;
+		}
+#endif
+
+		if (ui64RegVal != 0)
+		{
+			RGXReadModifyWriteReg64(hPrivate, ui32Reg, ui64RegVal, ~ui64RegVal);
+		}
+
+		/*
+		 * SLC Misc control.
+		 *
+		 * Note: This is a 64bit register and we set only the lower 32bits leaving the top
+		 *       32bits (RGX_CR_SLC_CTRL_MISC_SCRAMBLE_BITS) unchanged from the HW default.
+		 */
+		ui32Reg = RGX_CR_SLC_CTRL_MISC;
+		ui32RegVal = RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH1;
+		ui32RegVal |= RGXReadReg32(hPrivate, ui32Reg) & RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+
+#if defined(FIX_HW_BRN_60084_BIT_MASK)
+		if (RGX_DEVICE_HAS_BRN(hPrivate, 60084))
+		{
+#if !defined(SOC_FEATURE_STRICT_SAME_ADDRESS_WRITE_ORDERING)
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+#else
+			if (RGX_DEVICE_HAS_ERN(hPrivate, 61389))
+			{
+				ui32RegVal |= RGX_CR_SLC_CTRL_MISC_ENABLE_PSG_HAZARD_CHECK_EN;
+			}
+#endif
+		}
+#endif
+
+		/* Bypass burst combiner if SLC line size is smaller than 1024 bits */
+		if (RGXGetDeviceCacheLineSize(hPrivate) < 1024)
+		{
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_EN;
+		}
+
+		if (RGX_DEVICE_HAS_BRN(hPrivate, 71242) && !RGX_DEVICE_HAS_FEATURE(hPrivate, GPU_MULTICORE_SUPPORT))
+		{
+			ui32RegVal |= RGX_CR_SLC_CTRL_MISC_LAZYWB_OVERRIDE_EN;
+		}
+
+		RGXWriteReg32(hPrivate, ui32Reg, ui32RegVal);
+	}
+}
+
+
+static void RGXWriteKernelCatBase(const void *hPrivate, IMG_DEV_PHYADDR sPCAddr)
+{
+	{
+		/* Write the cat-base address */
+		RGXWriteKernelMMUPC64(hPrivate,
+		                      BIF_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT,
+		                      RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT,
+		                      ((sPCAddr.uiAddr
+		                      >> RGX_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT)
+		                      << RGX_CR_BIF_CAT_BASE0_ADDR_SHIFT)
+		                      & ~RGX_CR_BIF_CAT_BASE0_ADDR_CLRMSK);
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Keep catbase registers in sync */
+			RGXWriteKernelMMUPC64(hPrivate,
+			                      FWCORE_MEM_CAT_BASEx(MMU_CONTEXT_MAPPING_FWPRIV),
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT,
+			                      RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT,
+			                      ((sPCAddr.uiAddr
+			                      >> RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
+			                      << RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT)
+			                      & ~RGX_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK);
+		}
+
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXWriteKernelCatBase: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+}
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitBIF
+
+ @Description   Initialise RGX BIF
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitBIF(const void *hPrivate)
+{
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		/*
+		 * Trusted Firmware boot
+		 */
+#if defined(SUPPORT_TRUSTED_DEVICE)
+		RGXCommentLog(hPrivate, "RGXInitBIF: Trusted Device enabled");
+		RGXWriteReg32(hPrivate, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
+#endif
+	}
+	else
+#endif /* defined(RGX_FEATURE_MIPS_BIT_MASK) */
+	{
+		IMG_DEV_PHYADDR sPCAddr;
+
+		/*
+		 * Acquire the address of the Kernel Page Catalogue.
+		 */
+		RGXAcquireKernelMMUPC(hPrivate, &sPCAddr);
+
+		/*
+		 * Write the kernel catalogue base.
+		 */
+		RGXCommentLog(hPrivate, "RGX firmware MMU Page Catalogue");
+
+		RGXWriteKernelCatBase(hPrivate, sPCAddr);
+	}
+}
+
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+/**************************************************************************/ /*!
+@Function       RGXInitMMURangeRegisters
+@Description    Initialises MMU range registers for Non4K pages.
+@Input          hPrivate           Implementation specific data
+@Return         void
+ */ /**************************************************************************/
+static void RGXInitMMURangeRegisters(const void *hPrivate)
+{
+	RGX_LAYER_PARAMS *psParams = (RGX_LAYER_PARAMS*)hPrivate;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psParams->psDevInfo;
+	IMG_UINT32 ui32RegAddr = RGX_CR_MMU_PAGE_SIZE_RANGE_ONE;
+	IMG_UINT32 i;
+
+	for (i = 0; i < ARRAY_SIZE(psDevInfo->aui64MMUPageSizeRangeValue); ++i, ui32RegAddr += sizeof(IMG_UINT64))
+	{
+		RGXWriteReg64(hPrivate, ui32RegAddr, psDevInfo->aui64MMUPageSizeRangeValue[i]);
+	}
+}
+#endif
+
+
+/*!
+*******************************************************************************
+
+ @Function      RGXInitAXIACE
+
+ @Description    Initialise AXI-ACE interface
+
+ @Input         hPrivate : Implementation specific data
+
+ @Return        void
+
+******************************************************************************/
+static void RGXInitAXIACE(const void *hPrivate)
+{
+	IMG_UINT64 ui64RegVal;
+	IMG_UINT32 ui32RegAddr;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	ui32RegAddr = RGX_CR_AXI_ACE_LITE_CONFIGURATION;
+
+	/* Setup AXI-ACE config. Set everything to outer cache */
+	ui64RegVal = (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT) |
+	             (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT)  |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT) |
+	             (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT);
+
+#if defined(FIX_HW_BRN_42321_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 42321))
+	{
+		ui64RegVal |= (((IMG_UINT64) 1) << RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT);
+	}
+#endif
+
+#if defined(FIX_HW_BRN_68186_BIT_MASK)
+	if (RGX_DEVICE_HAS_BRN(hPrivate, 68186))
+	{
+		/* default value for reg_enable_fence_out is zero. Force to 1 to allow core_clk < mem_clk */
+		ui64RegVal |= (IMG_UINT64)1 << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ENABLE_FENCE_OUT_SHIFT;
+	}
+#endif
+#else /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+	ui32RegAddr = RGX_CR_ACE_CTRL;
+
+	/**
+	 * The below configuration is only applicable for RGX cores supporting
+	 * ACE/ACE-lite protocol and connected to ACE coherent interconnect.
+	 */
+
+	/**
+	 * Configure AxDomain and AxCache for MMU transactions.
+	 * AxDomain set to non sharable (0x0).
+	 */
+	ui64RegVal = RGX_CR_ACE_CTRL_MMU_AWCACHE_WRITE_BACK_WRITE_ALLOCATE |
+				 RGX_CR_ACE_CTRL_MMU_ARCACHE_WRITE_BACK_READ_ALLOCATE;
+
+	/**
+	 * Configure AxCache for PM/MMU transactions.
+	 * Set to same value (i.e WBRWALLOC caching, rgxmmunit.c:RGXDerivePTEProt8)
+	 * as non-coherent PTEs
+	 */
+	ui64RegVal |= (IMG_UINT64_C(0xF)) << RGX_CR_ACE_CTRL_PM_MMU_AXCACHE_SHIFT;
+
+	/**
+	 * Configure AxDomain for non MMU transactions.
+	 */
+	ui64RegVal |= (IMG_UINT64)(RGX_CR_ACE_CTRL_COH_DOMAIN_OUTER_SHAREABLE |
+							   RGX_CR_ACE_CTRL_NON_COH_DOMAIN_NON_SHAREABLE);
+#endif /* defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK) */
+
+	RGXCommentLog(hPrivate, "Init AXI-ACE interface");
+	RGXWriteReg64(hPrivate, ui32RegAddr, ui64RegVal);
+}
+
+static void RGXResetSequence(const void *hPrivate, const IMG_CHAR *pcRGXFW_PROCESSOR)
+{
+	IMG_UINT64 ui64SoftResetMask;
+
+#if defined(RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, PBE2_IN_XE))
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET__PBE2_XE__MASKFULL;
+	}
+	else
+#endif
+	{
+		ui64SoftResetMask = RGX_CR_SOFT_RESET_MASKFULL;
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXCommentLog(hPrivate, "RGXStart: soft reset cpu core");
+		RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 0);
+	}
+
+	{
+		/* Set RGX in soft-reset */
+		RGXCommentLog(hPrivate, "RGXStart: soft reset everything");
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, ui64SoftResetMask);
+
+		/* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+		/* Take Rascal and Dust out of reset */
+		RGXCommentLog(hPrivate, "RGXStart: Rascal and Dust out of reset");
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, ui64SoftResetMask ^ RGX_CR_SOFT_RESET_RASCALDUSTS_EN);
+
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+		/* Take everything out of reset but the FW processor */
+		RGXCommentLog(hPrivate, "RGXStart: Take everything out of reset but %s", pcRGXFW_PROCESSOR);
+
+		RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
+
+		(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+	}
+}
+
+static void DeassertMetaReset(const void *hPrivate)
+{
+	/* Need to wait for at least 32 cycles before taking the FW processor out of reset ... */
+	RGXWaitCycles(hPrivate, 32, 3);
+
+	RGXWriteReg64(hPrivate, RGX_CR_SOFT_RESET, 0x0);
+	(void) RGXReadReg64(hPrivate, RGX_CR_SOFT_RESET);
+
+	/* ... and afterwards */
+	RGXWaitCycles(hPrivate, 32, 3);
+}
+
+PVRSRV_ERROR RGXStart(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR *pcRGXFW_PROCESSOR;
+	IMG_BOOL bDoFWSlaveBoot = IMG_FALSE;
+	IMG_BOOL bMetaFW = IMG_FALSE;
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_RISCV;
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else if (RGX_DEVICE_HAS_FEATURE(hPrivate, MIPS))
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_MIPS;
+	}
+#endif
+	else
+	{
+		pcRGXFW_PROCESSOR = RGXFW_PROCESSOR_META;
+		bMetaFW = IMG_TRUE;
+		bDoFWSlaveBoot = RGXDoFWSlaveBoot(hPrivate);
+	}
+
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, SYS_BUS_SECURE_RESET))
+	{
+		/* Disable the default sys_bus_secure protection to perform minimal setup */
+		RGXCommentLog(hPrivate, "RGXStart: Disable sys_bus_secure");
+		RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, 0);
+		(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+	}
+
+#if defined(RGX_FEATURE_HYPERVISOR_MMU_BIT_MASK)
+	/* Only bypass HMMU if the module is present */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, HYPERVISOR_MMU))
+	{
+		/* Always set HMMU in bypass mode */
+		RGXWriteReg32(hPrivate, RGX_CR_HMMU_BYPASS, RGX_CR_HMMU_BYPASS_MASKFULL);
+		(void) RGXReadReg32(hPrivate, RGX_CR_HMMU_BYPASS);
+	}
+#endif
+
+	/*!
+	 * Start FW init sequence
+	 */
+	RGXResetSequence(hPrivate, pcRGXFW_PROCESSOR);
+
+	/*
+	 * Initialise SLC.
+	 */
+	__RGXInitSLC(hPrivate);
+
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, ECC_RAMS) > 0)
+	{
+		RGXCommentLog(hPrivate, "RGXStart: Enable safety events");
+		RGXWriteReg32(hPrivate, RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE,
+		                        RGX_CR_SAFETY_EVENT_ENABLE__ROGUEXE__MASKFULL);
+	}
+
+	if (bMetaFW)
+	{
+		if (bDoFWSlaveBoot)
+		{
+			/* Configure META to Slave boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Slave boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, 0);
+		}
+		else
+		{
+			/* Configure META to Master boot */
+			RGXCommentLog(hPrivate, "RGXStart: META Master boot");
+			RGXWriteReg32(hPrivate, RGX_CR_META_BOOT, RGX_CR_META_BOOT_MODE_EN);
+		}
+	}
+
+	/*
+	 * Initialise Firmware wrapper
+	 */
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+	{
+		RGXInitRiscvProcWrapper(hPrivate);
+	}
+	else if (bMetaFW)
+	{
+		RGXInitMetaProcWrapper(hPrivate);
+	}
+#if defined(RGX_FEATURE_MIPS_BIT_MASK)
+	else
+	{
+		RGXInitMipsProcWrapper(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_MMU_VERSION_MAX_VALUE_IDX)
+	if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, MMU_VERSION) >= 4)
+	{
+		/* initialise the MMU range based config registers for Non4K pages */
+		RGXInitMMURangeRegisters(hPrivate);
+	}
+#endif
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_DEVICE_HAS_FEATURE(hPrivate, AXI_ACELITE))
+#endif
+	{
+		/* We must init the AXI-ACE interface before 1st BIF transaction */
+		RGXInitAXIACE(hPrivate);
+	}
+
+	/*
+	 * Initialise BIF.
+	 */
+	RGXInitBIF(hPrivate);
+
+	RGXSetPoweredState(hPrivate, IMG_TRUE);
+
+	RGXCommentLog(hPrivate, "RGXStart: Take %s out of reset", pcRGXFW_PROCESSOR);
+	DeassertMetaReset(hPrivate);
+
+	if (bMetaFW && bDoFWSlaveBoot)
+	{
+		eError = RGXFabricCoherencyTest(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Slave boot Start");
+		eError = RGXStartFirmware(hPrivate);
+		if (eError != PVRSRV_OK) return eError;
+	}
+	else
+	{
+		RGXCommentLog(hPrivate, "RGXStart: RGX Firmware Master boot Start");
+
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, RISCV_FW_PROCESSOR))
+		{
+			/* Bring Debug Module out of reset */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) >= 4)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4, RGX_CR_FWCORE_DMI_DMCONTROL__HOST_SECURITY_GEQ4__DMACTIVE_EN);
+			}
+			else
+#endif
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_FWCORE_DMI_DMCONTROL, RGX_CR_FWCORE_DMI_DMCONTROL_DMACTIVE_EN);
+			}
+
+			/* Boot the FW */
+			RGXWriteReg32(hPrivate, RGX_CR_FWCORE_BOOT, 1);
+			RGXWaitCycles(hPrivate, 32, 3);
+		}
+	}
+
+#if defined(SUPPORT_TRUSTED_DEVICE) && !defined(SUPPORT_SECURITY_VALIDATION)
+	RGXCommentLog(hPrivate, "RGXStart: Enable sys_bus_secure");
+	RGXWriteReg32(hPrivate, RGX_CR_SYS_BUS_SECURE, RGX_CR_SYS_BUS_SECURE_ENABLE_EN);
+	(void) RGXReadReg32(hPrivate, RGX_CR_SYS_BUS_SECURE); /* Fence write */
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXStop(const void *hPrivate)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_BOOL bMetaFW = RGX_DEVICE_HAS_FEATURE_VALUE(hPrivate, META);
+
+	RGXDeviceAckIrq(hPrivate);
+
+	/* Set FW power state OFF to disable LISR handler */
+	RGXSetPoweredState(hPrivate, IMG_FALSE);
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper
+	 * For LAYOUT_MARS = 1, SIDEKICK would have been powered down by FW
+	 */
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN),
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN));
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		/*
+		 * Wait for SLC to signal IDLE
+		 * For LAYOUT_MARS = 1, SLC would have been powered down by FW
+		 */
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SLC_IDLE,
+					RGX_CR_SLC_IDLE_MASKFULL,
+					RGX_CR_SLC_IDLE_MASKFULL);
+		}
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	/* Unset MTS DM association with threads */
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_INTCTX_THREAD0_DM_ASSOC_MASKFULL);
+	RGXWriteReg32(hPrivate,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC,
+	              RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_DM_ASSOC_CLRMSK
+	              & RGX_CR_MTS_BGCTX_THREAD0_DM_ASSOC_MASKFULL);
+
+	if (bMetaFW)
+	{
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_INTCTX_THREAD1_DM_ASSOC_MASKFULL);
+		RGXWriteReg32(hPrivate,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC,
+					  RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_DM_ASSOC_CLRMSK
+					  & RGX_CR_MTS_BGCTX_THREAD1_DM_ASSOC_MASKFULL);
+	}
+
+#if defined(PDUMP)
+	if (bMetaFW)
+	{
+		/* Disabling threads is only required for pdumps to stop the fw gracefully */
+
+		/* Disable thread 0 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T0ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Disable thread 1 */
+		eError = RGXWriteMetaRegThroughSP(hPrivate,
+		                                  META_CR_T1ENABLE_OFFSET,
+		                                  ~META_CR_TXENABLE_ENABLE_BIT);
+		if (eError != PVRSRV_OK) return eError;
+
+		/* Clear down any irq raised by META (done after disabling the FW
+		 * threads to avoid a race condition).
+		 * This is only really needed for PDumps but we do it anyway driver-live.
+		 */
+#if defined(RGX_FEATURE_HOST_SECURITY_VERSION_MAX_VALUE_IDX)
+		/* Wait for the Slave Port to finish all the transactions */
+		if (RGX_DEVICE_HAS_FEATURE(hPrivate, META_REGISTER_UNPACKED_ACCESSES))
+		{
+			if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, HOST_SECURITY_VERSION) > 1)
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_GT1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_GT1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+			else
+			{
+				RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA, 0x0);
+				(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS__HOST_SECURITY_EQ1_AND_MRUA); /* Fence write */
+
+				eError = RGXPollReg32(hPrivate,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN,
+									  RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__READY_EN
+									  | RGX_CR_META_SP_MSLVCTRL1__HOST_SECURITY_EQ1_AND_MRUA__GBLPORT_IDLE_EN);
+			}
+		}
+		else
+#endif
+		{
+			RGXWriteReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS, 0x0);
+			(void)RGXReadReg32(hPrivate, RGX_CR_META_SP_MSLVIRQSTATUS); /* Fence write */
+
+			eError = RGXPollReg32(hPrivate,
+								  RGX_CR_META_SP_MSLVCTRL1,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN,
+								  RGX_CR_META_SP_MSLVCTRL1_READY_EN | RGX_CR_META_SP_MSLVCTRL1_GBLPORT_IDLE_EN);
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+#endif
+
+	/* Extra Idle checks */
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIF_STATUS_MMU,
+	                      0,
+	                      RGX_CR_BIF_STATUS_MMU_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIFPM_STATUS_MMU,
+	                      0,
+	                      RGX_CR_BIFPM_STATUS_MMU_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	if (!RGX_DEVICE_HAS_FEATURE(hPrivate, XT_TOP_INFRASTRUCTURE))
+	{
+		eError = RGXPollReg32(hPrivate,
+		                      RGX_CR_BIF_READS_EXT_STATUS,
+		                      0,
+		                      RGX_CR_BIF_READS_EXT_STATUS_MASKFULL);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	eError = RGXPollReg32(hPrivate,
+	                      RGX_CR_BIFPM_READS_EXT_STATUS,
+	                      0,
+	                      RGX_CR_BIFPM_READS_EXT_STATUS_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	eError = RGXPollReg64(hPrivate,
+	                      RGX_CR_SLC_STATUS1,
+	                      0,
+		                  RGX_CR_SLC_STATUS1_MASKFULL);
+	if (eError != PVRSRV_OK) return eError;
+
+	if (4 == RGXGetDeviceSLCBanks(hPrivate))
+	{
+		eError = RGXPollReg64(hPrivate,
+		                      RGX_CR_SLC_STATUS2,
+		                      0,
+		                      RGX_CR_SLC_STATUS2_MASKFULL);
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		/*
+		 * Wait for SLC to signal IDLE
+		 * For LAYOUT_MARS = 1, SLC would have been powered down by FW
+		 */
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SLC_IDLE,
+					RGX_CR_SLC_IDLE_MASKFULL,
+					RGX_CR_SLC_IDLE_MASKFULL);
+		}
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	/* Wait for Sidekick/Jones to signal IDLE except for the Garten Wrapper
+	 * For LAYOUT_MARS = 1, SIDEKICK would have been powered down by FW
+	 */
+	if (!(RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0))
+	{
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN),
+					RGX_CR_SIDEKICK_IDLE_MASKFULL^(RGX_CR_SIDEKICK_IDLE_GARTEN_EN|RGX_CR_SIDEKICK_IDLE_SOCIF_EN|RGX_CR_SIDEKICK_IDLE_HOSTIF_EN));
+		}
+
+		if (eError != PVRSRV_OK) return eError;
+	}
+
+	if (bMetaFW)
+	{
+		IMG_UINT32 ui32RegValue;
+
+		eError = RGXReadMetaRegThroughSP(hPrivate,
+		                                 META_CR_TxVECINT_BHALT,
+		                                 &ui32RegValue);
+		if (eError != PVRSRV_OK) return eError;
+
+		if ((ui32RegValue & 0xFFFFFFFFU) == 0x0)
+		{
+			/* Wait for Sidekick/Jones to signal IDLE including
+			 * the Garten Wrapper if there is no debugger attached
+			 * (TxVECINT_BHALT = 0x0) */
+			{
+				eError = RGXPollReg32(hPrivate,
+				                      RGX_CR_SIDEKICK_IDLE,
+				                      RGX_CR_SIDEKICK_IDLE_GARTEN_EN,
+				                      RGX_CR_SIDEKICK_IDLE_GARTEN_EN);
+				if (eError != PVRSRV_OK) return eError;
+			}
+		}
+	}
+	else
+	{
+		if (RGX_DEVICE_GET_FEATURE_VALUE(hPrivate, LAYOUT_MARS) > 0)
+		{
+			/* As FW core has been moved from SIDEKICK to the new MARS domain, checking
+			 * idle bits for CPU & System Arbiter excluding SOCIF which will never be Idle
+			 * if Host polling on this register
+			 */
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_MARS_IDLE,
+					RGX_CR_MARS_IDLE_CPU_EN | RGX_CR_MARS_IDLE_MH_SYSARB0_EN,
+					RGX_CR_MARS_IDLE_CPU_EN | RGX_CR_MARS_IDLE_MH_SYSARB0_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+		else
+		{
+			eError = RGXPollReg32(hPrivate,
+					RGX_CR_SIDEKICK_IDLE,
+					RGX_CR_SIDEKICK_IDLE_GARTEN_EN,
+					RGX_CR_SIDEKICK_IDLE_GARTEN_EN);
+			if (eError != PVRSRV_OK) return eError;
+		}
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxstartstop.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxstartstop.h b/drivers/gpu/drm/img-rogue/rgxstartstop.h
new file mode 100644
index 000000000000..178afe2849a0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxstartstop.h
@@ -0,0 +1,84 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX start/stop header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX start/stop functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSTARTSTOP_H)
+#define RGXSTARTSTOP_H
+
+/* The routines declared here are built on top of an abstraction layer to
+ * hide DDK/OS-specific details in case they are used outside of the DDK
+ * (e.g. when DRM security is enabled).
+ * Any new dependency should be added to rgxlayer.h.
+ * Any new code should be built on top of the existing abstraction layer,
+ * which should be extended when necessary.
+ */
+#include "rgxlayer.h"
+
+/*!
+*******************************************************************************
+
+ @Function      RGXStart
+
+ @Description   Perform GPU reset and initialisation
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXStart(const void *hPrivate);
+
+/*!
+*******************************************************************************
+
+ @Function      RGXStop
+
+ @Description   Stop Rogue in preparation for power down
+
+ @Input         hPrivate  : Implementation specific data
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXStop(const void *hPrivate);
+
+#endif /* RGXSTARTSTOP_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxsyncutils.c b/drivers/gpu/drm/img-rogue/rgxsyncutils.c
new file mode 100644
index 000000000000..cec0597d0301
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxsyncutils.c
@@ -0,0 +1,184 @@
+/*************************************************************************/ /*!
+@File           rgxsyncutils.c
+@Title          RGX Sync Utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Sync helper functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "rgxsyncutils.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "allocmem.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static
+void _DebugSyncValues(IMG_UINT32 *pui32UpdateValues,
+					  IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 iii;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (iii = 0; iii < ui32Count; iii++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+#else
+#define CHKPT_DBG(X)
+#endif
+
+
+PVRSRV_ERROR RGXSyncAppendTimelineUpdate(IMG_UINT32 ui32FenceTimelineUpdateValue,
+										 SYNC_ADDR_LIST	*psSyncList,
+										 SYNC_ADDR_LIST	*psPRSyncList,
+										 PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync,
+										 RGX_SYNC_DATA *psSyncData,
+										 IMG_BOOL bKick3D)
+{
+	IMG_UINT32 *pui32TimelineUpdateWOff = NULL;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+
+	IMG_UINT32 ui32ClientUpdateValueCount = psSyncData->ui32ClientUpdateValueCount;
+
+	/* Space for original client updates, and the one new update */
+	size_t uiUpdateSize = sizeof(*pui32IntAllocatedUpdateValues) * (ui32ClientUpdateValueCount + 1);
+
+	if (!bKick3D)
+	{
+		/* Additional space for one PR update, only the newest one */
+		uiUpdateSize += sizeof(*pui32IntAllocatedUpdateValues) * 1;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+		   "%s: About to allocate memory to hold updates in pui32IntAllocatedUpdateValues(<%p>)",
+		   __func__,
+		   (void*)pui32IntAllocatedUpdateValues));
+
+	/* Allocate memory to hold the list of update values (including our timeline update) */
+	pui32IntAllocatedUpdateValues = OSAllocMem(uiUpdateSize);
+	if (!pui32IntAllocatedUpdateValues)
+	{
+		/* Failed to allocate memory */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xcc, uiUpdateSize);
+	pui32TimelineUpdateWOff = pui32IntAllocatedUpdateValues;
+
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: Copying %d %s update values into pui32IntAllocatedUpdateValues(<%p>)",
+			   __func__,
+			   ui32ClientUpdateValueCount,
+			   bKick3D ? "TA/3D" : "TA/PR",
+			   (void*)pui32IntAllocatedUpdateValues));
+		/* Copy the update values into the new memory, then append our timeline update value */
+		OSCachedMemCopy(pui32TimelineUpdateWOff, psSyncData->paui32ClientUpdateValue, ui32ClientUpdateValueCount * sizeof(*psSyncData->paui32ClientUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(pui32TimelineUpdateWOff, ui32ClientUpdateValueCount);
+#endif
+
+		pui32TimelineUpdateWOff += ui32ClientUpdateValueCount;
+	}
+
+	/* Now set the additional update value and append the timeline sync prim addr to either the
+	 * render context 3D (or TA) update list
+	 */
+	CHKPT_DBG((PVR_DBG_ERROR,
+		   "%s: Appending the additional update value (0x%x) to psRenderContext->sSyncAddrList%sUpdate...",
+		   __func__,
+		   ui32FenceTimelineUpdateValue,
+		   bKick3D ? "TA/3D" : "TA/PR"));
+
+	/* Append the TA/3D update */
+	{
+		*pui32TimelineUpdateWOff++ = ui32FenceTimelineUpdateValue;
+		psSyncData->ui32ClientUpdateValueCount++;
+		psSyncData->ui32ClientUpdateCount++;
+		SyncAddrListAppendSyncPrim(psSyncList, psFenceTimelineUpdateSync);
+
+		if (!psSyncData->pauiClientUpdateUFOAddress)
+		{
+			psSyncData->pauiClientUpdateUFOAddress = psSyncList->pasFWAddrs;
+		}
+		/* Update paui32ClientUpdateValue to point to our new list of update values */
+		psSyncData->paui32ClientUpdateValue = pui32IntAllocatedUpdateValues;
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(pui32IntAllocatedUpdateValues, psSyncData->ui32ClientUpdateValueCount);
+#endif
+	}
+
+	if (!bKick3D)
+	{
+		/* Use the sSyncAddrList3DUpdate for PR (as it doesn't have one of its own) */
+		*pui32TimelineUpdateWOff++ = ui32FenceTimelineUpdateValue;
+		psSyncData->ui32ClientPRUpdateValueCount = 1;
+		psSyncData->ui32ClientPRUpdateCount = 1;
+		SyncAddrListAppendSyncPrim(psPRSyncList, psFenceTimelineUpdateSync);
+
+		if (!psSyncData->pauiClientPRUpdateUFOAddress)
+		{
+			psSyncData->pauiClientPRUpdateUFOAddress = psPRSyncList->pasFWAddrs;
+		}
+		/* Update paui32ClientPRUpdateValue to point to our new list of update values */
+		psSyncData->paui32ClientPRUpdateValue = &pui32IntAllocatedUpdateValues[psSyncData->ui32ClientUpdateValueCount];
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		_DebugSyncValues(psSyncData->paui32ClientPRUpdateValue, psSyncData->ui32ClientPRUpdateValueCount);
+#endif
+	}
+
+	/* Do not free the old psSyncData->ui32ClientUpdateValueCount,
+	 * as it was constant data passed through the bridge down to PVRSRVRGXKickTA3DKM() */
+
+	return PVRSRV_OK;
+}
diff --git a/drivers/gpu/drm/img-rogue/rgxsyncutils.h b/drivers/gpu/drm/img-rogue/rgxsyncutils.h
new file mode 100644
index 000000000000..bfc46937f185
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxsyncutils.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           rgxsyncutils.h
+@Title          RGX Sync Utilities
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Sync helper functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXSYNCUTILS_H
+#define RGXSYNCUTILS_H
+
+#include "rgxdevice.h"
+#include "sync_server.h"
+#include "rgxdebug_common.h"
+#include "rgx_fwif_km.h"
+
+typedef struct _RGX_SYNC_DATA_
+{
+	PRGXFWIF_UFO_ADDR *pauiClientUpdateUFOAddress;
+	IMG_UINT32 *paui32ClientUpdateValue;
+	IMG_UINT32 ui32ClientUpdateValueCount;
+	IMG_UINT32 ui32ClientUpdateCount;
+
+	PRGXFWIF_UFO_ADDR *pauiClientPRUpdateUFOAddress;
+	IMG_UINT32 *paui32ClientPRUpdateValue;
+	IMG_UINT32 ui32ClientPRUpdateValueCount;
+	IMG_UINT32 ui32ClientPRUpdateCount;
+} RGX_SYNC_DATA;
+
+PVRSRV_ERROR RGXSyncAppendTimelineUpdate(IMG_UINT32 ui32FenceTimelineUpdateValue,
+										 SYNC_ADDR_LIST	*psSyncList,
+										 SYNC_ADDR_LIST	*psPRSyncList,
+										 PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync,
+										 RGX_SYNC_DATA *psSyncData,
+										 IMG_BOOL bKick3D);
+
+#endif /* RGXSYNCUTILS_H */
+
+/******************************************************************************
+ End of file (rgxsyncutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxta3d.c b/drivers/gpu/drm/img-rogue/rgxta3d.c
new file mode 100644
index 000000000000..595c6e6f357e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxta3d.c
@@ -0,0 +1,5664 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA/3D routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX TA/3D routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+/* for the offsetof macro */
+#if defined(__linux__)
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
+
+#include "pdump_km.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "rgxfwutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxta3d.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "ri_server.h"
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "rgx_memallocflags.h"
+#include "rgxccb.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxsyncutils.h"
+#include "htbserver.h"
+
+#include "rgxdefs_km.h"
+#include "rgx_fwif_km.h"
+#include "physmem.h"
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "process_stats.h"
+
+#if !defined(PM_INTERACTIVE_MODE)
+#include "rgxpmdefs.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+
+#define HASH_CLEAN_LIMIT 6
+#endif
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TA3D_UFO_DUMP	0
+
+//#define TA3D_CHECKPOINT_DEBUG
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+static INLINE
+void _DebugSyncValues(const IMG_CHAR *pszFunction,
+					  const IMG_UINT32 *pui32UpdateValues,
+					  const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+	IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32UpdateValues;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", pszFunction, i, (void*)pui32Tmp, *pui32Tmp));
+		pui32Tmp++;
+	}
+}
+
+static INLINE
+void _DebugSyncCheckpoints(const IMG_CHAR *pszFunction,
+						   const IMG_CHAR *pszDMName,
+						   const PSYNC_CHECKPOINT *apsSyncCheckpoints,
+						   const IMG_UINT32 ui32Count)
+{
+	IMG_UINT32 i;
+
+	for (i = 0; i < ui32Count; i++)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFence%sSyncCheckpoints[%d]=<%p>", pszFunction, pszDMName, i, *(apsSyncCheckpoints + i)));
+	}
+}
+
+#else
+#define CHKPT_DBG(X)
+#endif
+
+/* define the number of commands required to be set up by the CCB helper */
+/* 1 command for the TA */
+#define CCB_CMD_HELPER_NUM_TA_COMMANDS 1
+/* Up to 3 commands for the 3D (partial render fence, partial render, and render) */
+#define CCB_CMD_HELPER_NUM_3D_COMMANDS 3
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#define WORKEST_CYCLES_PREDICTION_GET(x) ((x).ui32CyclesPrediction)
+#else
+#define WORKEST_CYCLES_PREDICTION_GET(x) (NO_CYCEST)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_TA_DATA;
+
+typedef struct {
+	DEVMEM_MEMDESC				*psContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+} RGX_SERVER_RC_3D_DATA;
+
+struct _RGX_SERVER_RENDER_CONTEXT_ {
+	/* this lock protects usage of the render context.
+	 * it ensures only one kick is being prepared and/or submitted on
+	 * this render context at any time
+	 */
+	POS_LOCK				hLock;
+	RGX_CCB_CMD_HELPER_DATA asTACmdHelperData[CCB_CMD_HELPER_NUM_TA_COMMANDS];
+	RGX_CCB_CMD_HELPER_DATA as3DCmdHelperData[CCB_CMD_HELPER_NUM_3D_COMMANDS];
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWRenderContextMemDesc;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	RGX_SERVER_RC_TA_DATA		sTAData;
+	RGX_SERVER_RC_3D_DATA		s3DData;
+	IMG_UINT32					ui32CleanupStatus;
+#define RC_CLEANUP_TA_COMPLETE		(1 << 0)
+#define RC_CLEANUP_3D_COMPLETE		(1 << 1)
+	DLLIST_NODE					sListNode;
+	SYNC_ADDR_LIST				sSyncAddrListTAFence;
+	SYNC_ADDR_LIST				sSyncAddrListTAUpdate;
+	SYNC_ADDR_LIST				sSyncAddrList3DFence;
+	SYNC_ADDR_LIST				sSyncAddrList3DUpdate;
+	ATOMIC_T					hIntJobRef;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA			sWorkEstData;
+#endif
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+};
+
+
+/*
+	Static functions used by render context code
+*/
+
+static
+PVRSRV_ERROR _DestroyTAContext(RGX_SERVER_RC_TA_DATA *psTAData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  psTAData->psServerCommonContext,
+											  RGXFWIF_DM_GEOM,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(psTAData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, psTAData->psContextStateMemDesc);
+	psTAData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Destroy3DContext(RGX_SERVER_RC_3D_DATA *ps3DData,
+							   PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psContextStateMemDesc);
+	ps3DData->psServerCommonContext = NULL;
+
+	return eError;
+}
+
+static void _RGXDumpPMRPageList(DLLIST_NODE *psNode)
+{
+	RGX_PMR_NODE *psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	PVRSRV_ERROR			eError;
+
+	eError = PMRDumpPageList(psPMRNode->psPMR,
+							RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Error (%s) printing pmr %p",
+				PVRSRVGetErrorString(eError),
+				psPMRNode->psPMR));
+	}
+}
+
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	PVR_LOG(("Freelist FWAddr 0x%08x, ID = %d, CheckSum 0x%016" IMG_UINT64_FMTSPECx,
+				psFreeList->sFreeListFWDevVAddr.ui32Addr,
+				psFreeList->ui32FreelistID,
+				psFreeList->ui64FreelistChecksum));
+
+	/* Dump Init FreeList page list */
+	PVR_LOG(("  Initial Memory block"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	/* Dump Grow FreeList page list */
+	PVR_LOG(("  Grow Memory blocks"));
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXDumpPMRPageList(psNode);
+	}
+
+	return IMG_TRUE;
+}
+
+static void _CheckFreelist(RGX_FREELIST *psFreeList,
+						   IMG_UINT32 ui32NumOfPagesToCheck,
+						   IMG_UINT64 ui64ExpectedCheckSum,
+						   IMG_UINT64 *pui64CalculatedCheckSum)
+{
+#if defined(NO_HARDWARE)
+	/* No checksum needed as we have all information in the pdumps */
+	PVR_UNREFERENCED_PARAMETER(psFreeList);
+	PVR_UNREFERENCED_PARAMETER(ui32NumOfPagesToCheck);
+	PVR_UNREFERENCED_PARAMETER(ui64ExpectedCheckSum);
+	*pui64CalculatedCheckSum = 0;
+#else
+	PVRSRV_ERROR eError;
+	size_t uiNumBytes;
+	IMG_UINT8* pui8Buffer;
+	IMG_UINT32* pui32Buffer;
+	IMG_UINT32 ui32CheckSumAdd = 0;
+	IMG_UINT32 ui32CheckSumXor = 0;
+	IMG_UINT32 ui32Entry;
+	IMG_UINT32 ui32Entry2;
+	IMG_BOOL bFreelistBad = IMG_FALSE;
+
+	*pui64CalculatedCheckSum = 0;
+
+	PVR_ASSERT(ui32NumOfPagesToCheck <= (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+	/* Allocate Buffer of the size of the freelist */
+	pui8Buffer = OSAllocMem(ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+	if (pui8Buffer == NULL)
+	{
+		PVR_LOG(("%s: Failed to allocate buffer to check freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	/* Copy freelist content into Buffer */
+	eError = PMR_ReadBytes(psFreeList->psFreeListPMR,
+					psFreeList->uiFreeListPMROffset +
+					(((psFreeList->ui32MaxFLPages -
+					   psFreeList->ui32CurrentFLPages - psFreeList->ui32ReadyFLPages) * sizeof(IMG_UINT32)) &
+					 ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1)),
+					 pui8Buffer,
+					 ui32NumOfPagesToCheck * sizeof(IMG_UINT32),
+					&uiNumBytes);
+	if (eError != PVRSRV_OK)
+	{
+		OSFreeMem(pui8Buffer);
+		PVR_LOG(("%s: Failed to get freelist data for freelist %p!",
+				__func__, psFreeList));
+		PVR_ASSERT(0);
+		return;
+	}
+
+	PVR_ASSERT(uiNumBytes == ui32NumOfPagesToCheck * sizeof(IMG_UINT32));
+
+	/* Generate checksum (skipping the first page if not allocated) */
+	pui32Buffer = (IMG_UINT32 *)pui8Buffer;
+	ui32Entry = ((psFreeList->ui32GrowFLPages == 0  &&  psFreeList->ui32CurrentFLPages > 1) ? 1 : 0);
+	for (/*ui32Entry*/ ; ui32Entry < ui32NumOfPagesToCheck; ui32Entry++)
+	{
+		ui32CheckSumAdd += pui32Buffer[ui32Entry];
+		ui32CheckSumXor ^= pui32Buffer[ui32Entry];
+
+		/* Check for double entries */
+		for (ui32Entry2 = ui32Entry+1; ui32Entry2 < ui32NumOfPagesToCheck; ui32Entry2++)
+		{
+			if (pui32Buffer[ui32Entry] == pui32Buffer[ui32Entry2])
+			{
+				PVR_LOG(("%s: Freelist consistency failure: FW addr: 0x%08X, Double entry found 0x%08x on idx: %d and %d of %d",
+						__func__,
+						psFreeList->sFreeListFWDevVAddr.ui32Addr,
+						pui32Buffer[ui32Entry2],
+						ui32Entry,
+						ui32Entry2,
+						psFreeList->ui32CurrentFLPages));
+				bFreelistBad = IMG_TRUE;
+				break;
+			}
+		}
+	}
+
+	OSFreeMem(pui8Buffer);
+
+	/* Check the calculated checksum against the expected checksum... */
+	*pui64CalculatedCheckSum = ((IMG_UINT64)ui32CheckSumXor << 32) | ui32CheckSumAdd;
+
+	if (ui64ExpectedCheckSum != 0  &&  ui64ExpectedCheckSum != *pui64CalculatedCheckSum)
+	{
+		PVR_LOG(("%s: Checksum mismatch for freelist %p! Expected 0x%016" IMG_UINT64_FMTSPECx " calculated 0x%016" IMG_UINT64_FMTSPECx,
+				__func__, psFreeList,
+				ui64ExpectedCheckSum, *pui64CalculatedCheckSum));
+		bFreelistBad = IMG_TRUE;
+	}
+
+	if (bFreelistBad)
+	{
+		PVR_LOG(("%s: Sleeping for ever!", __func__));
+		PVR_ASSERT(!bFreelistBad);
+	}
+#endif
+}
+
+
+/*
+ *  Function to work out the number of freelist pages to reserve for growing
+ *  within the FW without having to wait for the host to progress a grow
+ *  request.
+ *
+ *  The number of pages must be a multiple of 4 to align the PM addresses
+ *  for the initial freelist allocation and also be less than the grow size.
+ *
+ *  If the threshold or grow size means less than 4 pages, then the feature
+ *  is not used.
+ */
+static IMG_UINT32 _CalculateFreelistReadyPages(RGX_FREELIST *psFreeList,
+                                               IMG_UINT32  ui32FLPages)
+{
+	IMG_UINT32  ui32ReadyFLPages = ((ui32FLPages * psFreeList->ui32GrowThreshold) / 100) &
+	                               ~((RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE/sizeof(IMG_UINT32))-1);
+
+	if (ui32ReadyFLPages > psFreeList->ui32GrowFLPages)
+	{
+		ui32ReadyFLPages = psFreeList->ui32GrowFLPages;
+	}
+
+	return ui32ReadyFLPages;
+}
+
+
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader)
+{
+	RGX_PMR_NODE	*psPMRNode;
+	IMG_DEVMEM_SIZE_T uiSize;
+	IMG_UINT32  ui32MappingTable = 0;
+	IMG_DEVMEM_OFFSET_T uiOffset;
+	IMG_DEVMEM_SIZE_T uiLength;
+	IMG_DEVMEM_SIZE_T uistartPage;
+	PVRSRV_ERROR eError;
+	static const IMG_CHAR szAllocName[] = "Free List";
+
+	/* Are we allowed to grow ? */
+	if (psFreeList->ui32MaxFLPages - (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) < ui32NumPages)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: grow by %u pages denied. "
+				"Max PB size reached (current pages %u+%u/%u)",
+				psFreeList,
+				ui32NumPages,
+				psFreeList->ui32CurrentFLPages,
+				psFreeList->ui32ReadyFLPages,
+				psFreeList->ui32MaxFLPages));
+		return PVRSRV_ERROR_PBSIZE_ALREADY_MAX;
+	}
+
+	/* Allocate kernel memory block structure */
+	psPMRNode = OSAllocMem(sizeof(*psPMRNode));
+	if (psPMRNode == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to allocate host data structure",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocHost;
+	}
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages
+	 */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+
+	/*
+	 *  The PM never takes the last page in a freelist, so if this block
+	 *  of pages is the first one and there is no ability to grow, then
+	 *  we can skip allocating one 4K page for the lowest entry.
+	 */
+	if (OSGetPageSize() > RGX_BIF_PM_PHYSICAL_PAGE_SIZE)
+	{
+		/*
+		 * Allocation size will be rounded up to the OS page size,
+		 * any attempt to change it a bit now will be invalidated later.
+		 */
+		psPMRNode->bFirstPageMissing = IMG_FALSE;
+	}
+	else
+	{
+		psPMRNode->bFirstPageMissing = (psFreeList->ui32GrowFLPages == 0  &&  ui32NumPages > 1);
+	}
+
+	psPMRNode->ui32NumPages = ui32NumPages;
+	psPMRNode->psFreeList = psFreeList;
+
+	/* Allocate Memory Block */
+	PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Allocate PB Block (Pages %08X)", ui32NumPages);
+	uiSize = (IMG_DEVMEM_SIZE_T)ui32NumPages * RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	if (psPMRNode->bFirstPageMissing)
+	{
+		uiSize -= RGX_BIF_PM_PHYSICAL_PAGE_SIZE;
+	}
+
+	eError = PhysmemNewRamBackedPMR(psFreeList->psConnection,
+			psFreeList->psDevInfo->psDeviceNode,
+			uiSize,
+			1,
+			1,
+			&ui32MappingTable,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+			PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(GPU_PRIVATE) |
+			PVRSRV_MEMALLOCFLAG_MANDATE_PHYSHEAP,
+			sizeof(szAllocName),
+			szAllocName,
+			psFreeList->ownerPid,
+			&psPMRNode->psPMR,
+			PDUMP_NONE,
+			NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to allocate PB block of size: 0x%016" IMG_UINT64_FMTSPECX,
+				 __func__,
+				 (IMG_UINT64)uiSize));
+		goto ErrorBlockAlloc;
+	}
+
+	/* Zeroing physical pages pointed by the PMR */
+	if (psFreeList->psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	uistartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((uistartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+	eError = RIWritePMREntryWithOwnerKM(psPMRNode->psPMR,
+	                                    psFreeList->ownerPid);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: call to RIWritePMREntryWithOwnerKM failed (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+	}
+
+	 /* Attach RI information */
+	eError = RIWriteMEMDESCEntryKM(psFreeList->psConnection,
+	                               psFreeList->psDevInfo->psDeviceNode,
+	                               psPMRNode->psPMR,
+	                               OSStringNLength(szAllocName, DEVMEM_ANNOTATION_MAX_LEN),
+	                               szAllocName,
+	                               0,
+	                               uiSize,
+	                               0,
+	                               &psPMRNode->hRIHandle);
+	PVR_LOG_IF_ERROR(eError, "RIWriteMEMDESCEntryKM");
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+	/* write Freelist with Memory Block physical addresses */
+	eError = PMRWritePMPageList(
+						/* Target PMR, offset, and length */
+						psFreeList->psFreeListPMR,
+						(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+						(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+						/* Referenced PMR, and "page" granularity */
+						psPMRNode->psPMR,
+						RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+						&psPMRNode->psPageList);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Failed to write pages of Node %p",
+				 __func__,
+				 psPMRNode));
+		goto ErrorPopulateFreelist;
+	}
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+	/* Copy freelist memory to shadow freelist */
+	{
+		const IMG_UINT32 ui32FLMaxSize = psFreeList->ui32MaxFLPages * sizeof(IMG_UINT32);
+		const IMG_UINT32 ui32MapSize = ui32FLMaxSize * 2;
+		const IMG_UINT32 ui32CopyOffset = uiOffset - psFreeList->uiFreeListPMROffset;
+		IMG_BYTE *pFLMapAddr;
+		size_t uiNumBytes;
+		PVRSRV_ERROR res;
+		IMG_HANDLE hMapHandle;
+
+		/* Check for overflow. Validate size and offset. */
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize > psFreeList->uiFreeListPMROffset,
+		                          eError, ErrorPopulateFreelist);
+		PVR_GOTO_IF_INVALID_PARAM(psFreeList->uiFreeListPMROffset + ui32MapSize <= PMR_LogicalSize(psFreeList->psFreeListPMR),
+		                          eError, ErrorPopulateFreelist);
+
+		/* Map both the FL and the shadow FL */
+		res = PMRAcquireKernelMappingData(psFreeList->psFreeListPMR, psFreeList->uiFreeListPMROffset, ui32MapSize,
+		                                  (void**) &pFLMapAddr, &uiNumBytes, &hMapHandle);
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to map freelist (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			goto ErrorPopulateFreelist;
+		}
+
+		/* Copy only the newly added memory */
+		OSCachedMemCopy(pFLMapAddr + ui32FLMaxSize + ui32CopyOffset, pFLMapAddr + ui32CopyOffset , uiLength);
+		OSWriteMemoryBarrier(pFLMapAddr);
+
+#if defined(PDUMP)
+		PDUMPCOMMENT(psFreeList->psDevInfo->psDeviceNode, "Initialize shadow freelist");
+
+		/* Translate memcpy to pdump */
+		{
+			IMG_DEVMEM_OFFSET_T uiCurrOffset;
+
+			for (uiCurrOffset = uiOffset; (uiCurrOffset - uiOffset) < uiLength; uiCurrOffset += sizeof(IMG_UINT32))
+			{
+				PMRPDumpCopyMem32(psFreeList->psFreeListPMR,
+				                  uiCurrOffset + ui32FLMaxSize,
+				                  psFreeList->psFreeListPMR,
+				                  uiCurrOffset,
+				                  ":SYSMEM:$1",
+				                  PDUMP_FLAGS_CONTINUOUS);
+			}
+		}
+#endif
+
+
+		res = PMRReleaseKernelMappingData(psFreeList->psFreeListPMR, hMapHandle);
+
+		if (res != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			         "%s: Failed to release freelist mapping (ID=%d)",
+			         __func__,
+			         psFreeList->ui32FreelistID));
+			goto ErrorPopulateFreelist;
+		}
+	}
+#endif
+
+	/* We add It must be added to the tail, otherwise the freelist population won't work */
+	dllist_add_to_head(pListHeader, &psPMRNode->sMemoryBlock);
+
+	/* Update number of available pages */
+	psFreeList->ui32CurrentFLPages += ui32NumPages;
+
+	/* Update statistics (needs to happen before the ReadyFL calculation to also count those pages) */
+	if (psFreeList->ui32NumHighPages < psFreeList->ui32CurrentFLPages)
+	{
+		psFreeList->ui32NumHighPages = psFreeList->ui32CurrentFLPages;
+	}
+
+	/* Reserve a number ready pages to allow the FW to process OOM quickly and asynchronously request a grow. */
+	psFreeList->ui32ReadyFLPages    = _CalculateFreelistReadyPages(psFreeList, psFreeList->ui32CurrentFLPages);
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+
+	if (psFreeList->bCheckFreelist)
+	{
+		/*
+		 *  We can only calculate the freelist checksum when the list is full
+		 *  (e.g. at initial creation time). At other times the checksum cannot
+		 *  be calculated and has to be disabled for this freelist.
+		 */
+		if ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages)
+		{
+			_CheckFreelist(psFreeList, ui32NumPages, 0, &psFreeList->ui64FreelistChecksum);
+		}
+		else
+		{
+			psFreeList->ui64FreelistChecksum = 0;
+		}
+	}
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: %s %u pages (pages=%u+%u/%u checksum=0x%016" IMG_UINT64_FMTSPECx "%s)",
+			psFreeList,
+	        ((psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages) == ui32NumPages ? "Create initial" : "Grow by"),
+			ui32NumPages,
+			psFreeList->ui32CurrentFLPages,
+			psFreeList->ui32ReadyFLPages,
+			psFreeList->ui32MaxFLPages,
+			psFreeList->ui64FreelistChecksum,
+			(psPMRNode->bFirstPageMissing ? " - lowest page not allocated" : "")));
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+ErrorPopulateFreelist:
+	PMRUnrefPMR(psPMRNode->psPMR);
+
+ErrorBlockAlloc:
+	OSFreeMem(psPMRNode);
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+ErrorAllocHost:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+
+}
+
+static PVRSRV_ERROR RGXShrinkFreeList(PDLLIST_NODE pListHeader,
+										RGX_FREELIST *psFreeList)
+{
+	DLLIST_NODE *psNode;
+	RGX_PMR_NODE *psPMRNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32OldValue;
+
+	/*
+	 * Lock protects simultaneous manipulation of:
+	 * - the memory block list
+	 * - the freelist's ui32CurrentFLPages value
+	 */
+	PVR_ASSERT(pListHeader);
+	PVR_ASSERT(psFreeList);
+	PVR_ASSERT(psFreeList->psDevInfo);
+	PVR_ASSERT(psFreeList->psDevInfo->hLockFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+
+	/* Get node from head of list and remove it */
+	psNode = dllist_get_next_node(pListHeader);
+	if (psNode)
+	{
+		dllist_remove_node(psNode);
+
+		psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+		PVR_ASSERT(psPMRNode);
+		PVR_ASSERT(psPMRNode->psPMR);
+		PVR_ASSERT(psPMRNode->psFreeList);
+
+		/* remove block from freelist list */
+
+		/* Unwrite Freelist with Memory Block physical addresses */
+		eError = PMRUnwritePMPageList(psPMRNode->psPageList);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to unwrite pages of Node %p",
+					 __func__,
+					 psPMRNode));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+		if (psPMRNode->hRIHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			eError = RIDeleteMEMDESCEntryKM(psPMRNode->hRIHandle);
+			PVR_LOG_IF_ERROR(eError, "RIDeleteMEMDESCEntryKM");
+		}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
+
+		/* Free PMR (We should be the only one that holds a ref on the PMR) */
+		eError = PMRUnrefPMR(psPMRNode->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to free PB block %p (%s)",
+					 __func__,
+					 psPMRNode->psPMR,
+					 PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(IMG_FALSE);
+		}
+
+		/* update available pages in freelist */
+		ui32OldValue = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+
+		/*
+		 * Deallocated pages should first be deducted from ReadyPages bank, once
+		 * there are no more left, start deducting them from CurrentPage bank.
+		 */
+		if (psPMRNode->ui32NumPages > psFreeList->ui32ReadyFLPages)
+		{
+			psFreeList->ui32CurrentFLPages -= psPMRNode->ui32NumPages - psFreeList->ui32ReadyFLPages;
+			psFreeList->ui32ReadyFLPages = 0;
+		}
+		else
+		{
+			psFreeList->ui32ReadyFLPages -= psPMRNode->ui32NumPages;
+		}
+
+		/* check underflow */
+		PVR_ASSERT(ui32OldValue > (psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages));
+
+		PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: shrink by %u pages (current pages %u/%u)",
+								psFreeList,
+								psPMRNode->ui32NumPages,
+								psFreeList->ui32CurrentFLPages,
+								psFreeList->ui32MaxFLPages));
+
+		OSFreeMem(psPMRNode);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"Freelist [0x%p]: shrink denied. PB already at initial PB size (%u pages)",
+								psFreeList,
+								psFreeList->ui32InitFLPages));
+		eError = PVRSRV_ERROR_PBSIZE_ALREADY_MIN;
+	}
+
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	return eError;
+}
+
+static RGX_FREELIST *FindFreeList(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32FreelistID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_FREELIST *psFreeList = NULL;
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST *psThisFreeList = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		if (psThisFreeList->ui32FreelistID == ui32FreelistID)
+		{
+			psFreeList = psThisFreeList;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevInfo->hLockFreeList);
+	return psFreeList;
+}
+
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           IMG_UINT32 ui32FreelistID)
+{
+	RGX_FREELIST *psFreeList = NULL;
+	RGXFWIF_KCCB_CMD s3DCCBCmd;
+	IMG_UINT32 ui32GrowValue;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	psFreeList = FindFreeList(psDevInfo, ui32FreelistID);
+	if (psFreeList == NULL)
+	{
+		/* Should never happen */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "FreeList Lookup for FreeList ID 0x%08x failed (Populate)",
+		         ui32FreelistID));
+		PVR_ASSERT(IMG_FALSE);
+		return;
+	}
+
+	/* Since the FW made the request, it has already consumed the ready pages, update the host struct */
+	psFreeList->ui32CurrentFLPages += psFreeList->ui32ReadyFLPages;
+	psFreeList->ui32ReadyFLPages = 0;
+
+	/* Try to grow the freelist */
+	eError = RGXGrowFreeList(psFreeList,
+	                         psFreeList->ui32GrowFLPages,
+	                         &psFreeList->sMemoryBlockHead);
+
+	if (eError == PVRSRV_OK)
+	{
+		/* Grow successful, return size of grow size */
+		ui32GrowValue = psFreeList->ui32GrowFLPages;
+
+		psFreeList->ui32NumGrowReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+		/* Update Stats */
+		PVRSRVStatsUpdateFreelistStats(psDevInfo->psDeviceNode,
+		                               0,
+		                               1, /* Add 1 to the appropriate counter (Requests by FW) */
+		                               psFreeList->ui32InitFLPages,
+		                               psFreeList->ui32NumHighPages,
+		                               psFreeList->ownerPid);
+
+#endif
+
+	}
+	else
+	{
+		/* Grow failed */
+		ui32GrowValue = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+		         "Grow for FreeList %p failed (%s)",
+		         psFreeList,
+		         PVRSRVGetErrorString(eError)));
+	}
+
+	/* send feedback */
+	s3DCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELIST_GROW_UPDATE;
+	s3DCCBCmd.uCmdData.sFreeListGSData.sFreeListFWDevVAddr.ui32Addr = psFreeList->sFreeListFWDevVAddr.ui32Addr;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32DeltaPages = ui32GrowValue;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages = psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages;
+	s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages = psFreeList->ui32ReadyFLPages;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "Freelist [%p]: Grow pages=%u, new pages=%u, ready pages=%u, counter=%d",
+	         psFreeList,
+	         ui32GrowValue,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32NewPages,
+	         s3DCCBCmd.uCmdData.sFreeListGSData.ui32ReadyPages,
+	         psFreeList->ui32NumGrowReqByFW));
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_3D,
+		                            &s3DCCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+static void _RGXFreeListReconstruction(PDLLIST_NODE psNode)
+{
+
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	RGX_FREELIST			*psFreeList;
+	RGX_PMR_NODE			*psPMRNode;
+	PVRSRV_ERROR			eError;
+	IMG_DEVMEM_OFFSET_T		uiOffset;
+	IMG_DEVMEM_SIZE_T		uiLength;
+	IMG_UINT32				ui32StartPage;
+
+	psPMRNode = IMG_CONTAINER_OF(psNode, RGX_PMR_NODE, sMemoryBlock);
+	psFreeList = psPMRNode->psFreeList;
+	PVR_ASSERT(psFreeList);
+	psDevInfo = psFreeList->psDevInfo;
+	PVR_ASSERT(psDevInfo);
+
+	uiLength = psPMRNode->ui32NumPages * sizeof(IMG_UINT32);
+	ui32StartPage = (psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages - psPMRNode->ui32NumPages);
+	uiOffset = psFreeList->uiFreeListPMROffset + ((ui32StartPage * sizeof(IMG_UINT32)) & ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1));
+
+	PMRUnwritePMPageList(psPMRNode->psPageList);
+	psPMRNode->psPageList = NULL;
+	eError = PMRWritePMPageList(
+			/* Target PMR, offset, and length */
+			psFreeList->psFreeListPMR,
+			(psPMRNode->bFirstPageMissing ? uiOffset + sizeof(IMG_UINT32) : uiOffset),
+			(psPMRNode->bFirstPageMissing ? uiLength - sizeof(IMG_UINT32) : uiLength),
+			/* Referenced PMR, and "page" granularity */
+			psPMRNode->psPMR,
+			RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,
+			&psPMRNode->psPageList);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Error (%s) writing FL 0x%08x",
+				__func__,
+				PVRSRVGetErrorString(eError),
+				(IMG_UINT32)psFreeList->ui32FreelistID));
+	}
+
+	/* Zeroing physical pages pointed by the reconstructed freelist */
+	if (psDevInfo->ui32DeviceFlags & RGXKM_DEVICE_STATE_ZERO_FREELIST)
+	{
+		eError = PMRZeroingPMR(psPMRNode->psPMR, RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to zero PMR %p of freelist %p (%s)",
+					__func__,
+					psPMRNode->psPMR,
+					psFreeList,
+					PVRSRVGetErrorString(eError)));
+			PVR_ASSERT(0);
+		}
+	}
+
+	psFreeList->ui32CurrentFLPages += psPMRNode->ui32NumPages;
+}
+
+
+static PVRSRV_ERROR RGXReconstructFreeList(RGX_FREELIST *psFreeList)
+{
+	IMG_UINT32        ui32OriginalFLPages;
+	DLLIST_NODE       *psNode, *psNext;
+	RGXFWIF_FREELIST  *psFWFreeList;
+	PVRSRV_ERROR      eError;
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: Reconstructing freelist %p (ID=%u)", psFreeList, psFreeList->ui32FreelistID));
+
+	/* Do the FreeList Reconstruction */
+	ui32OriginalFLPages            = psFreeList->ui32CurrentFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+
+	/* Reconstructing Init FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockInitHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Reconstructing Grow FreeList pages */
+	dllist_foreach_node(&psFreeList->sMemoryBlockHead, psNode, psNext)
+	{
+		_RGXFreeListReconstruction(psNode);
+	}
+
+	/* Ready pages are allocated but kept hidden until OOM occurs. */
+	psFreeList->ui32CurrentFLPages -= psFreeList->ui32ReadyFLPages;
+	if (psFreeList->ui32CurrentFLPages != ui32OriginalFLPages)
+	{
+		PVR_ASSERT(psFreeList->ui32CurrentFLPages == ui32OriginalFLPages);
+		return PVRSRV_ERROR_FREELIST_RECONSTRUCTION_FAILED;
+	}
+
+	/* Update firmware freelist structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	psFWFreeList->ui32MaxPages     = psFreeList->ui32MaxFLPages;
+	psFWFreeList->ui32GrowPages    = psFreeList->ui32GrowFLPages;
+	psFWFreeList->ui32CurrentPages = psFreeList->ui32CurrentFLPages;
+	psFWFreeList->ui32FreeListID   = psFreeList->ui32FreelistID;
+	psFWFreeList->bGrowPending     = IMG_FALSE;
+	psFWFreeList->ui32ReadyPages   = psFreeList->ui32ReadyFLPages;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psFWFreeList->ui32CurrentStackTop       = psFWFreeList->ui32CurrentPages - 1;
+	psFWFreeList->ui32AllocatedPageCount    = 0;
+	psFWFreeList->ui32AllocatedMMUPageCount = 0;
+#else
+	psFWFreeList->bUpdatePending                   = IMG_FALSE;
+	psFWFreeList->ui32UpdateNewPages               = 0;
+	psFWFreeList->ui32UpdateNewReadyPages          = 0;
+	psFWFreeList->sFreeListLastGrowDevVAddr.uiAddr = 0;
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+#if !defined(PM_INTERACTIVE_MODE)
+	/* Reset freelist state buffer */
+	{
+		RGX_PM_FREELISTSTATE_BUFFER		sFLState;
+		size_t uiNbBytes;
+		IMG_DEV_VIRTADDR sFLBaseAddr;
+
+		eError = PMR_ReadBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+
+		sFLBaseAddr.uiAddr = (psFreeList->sFreeListBaseDevVAddr.uiAddr +
+		                      ((psFreeList->ui32MaxFLPages - psFreeList->ui32CurrentFLPages) * sizeof(IMG_UINT32))) &
+		                      ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+		/* Note: Freelist base address is packed shifted down. */
+		RGX_PM_FREELISTSTATE_BUFFER_SET_BASE_ADDR(sFLState, sFLBaseAddr.uiAddr >> RGX_PM_FREELISTSTATE_BASE_ADDR_ALIGNSHIFT);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_STACK_PTR(sFLState, psFreeList->ui32CurrentFLPages - 1);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_PAGE_STATUS(sFLState, 0);
+		RGX_PM_FREELISTSTATE_BUFFER_SET_MMUPAGE_STATUS(sFLState, 0);
+
+		eError = PMR_WriteBytes(psFreeList->psFreeListStatePMR, psFreeList->uiFreeListStatePMROffset, (IMG_UINT8*)&sFLState, sizeof(sFLState), &uiNbBytes);
+
+		if (eError != PVRSRV_OK)
+		{
+			return eError;
+		}
+
+		PVR_ASSERT(uiNbBytes == sizeof(sFLState));
+	}
+#endif
+
+	/* Check the Freelist checksum if required (as the list is fully populated) */
+	if (psFreeList->bCheckFreelist)
+	{
+		IMG_UINT64  ui64CheckSum;
+
+		_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+	}
+
+	return eError;
+}
+
+
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+                                              IMG_UINT32 ui32FreelistsCount,
+                                              const IMG_UINT32 *paui32Freelists)
+{
+	PVRSRV_ERROR      eError = PVRSRV_OK;
+	DLLIST_NODE       *psNode, *psNext;
+	IMG_UINT32        ui32Loop;
+	RGXFWIF_KCCB_CMD  sTACCBCmd;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE       *psNodeHWRTData, *psNextHWRTData;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+	RGXFWIF_HWRTDATA     *psHWRTData;
+#endif
+	IMG_UINT32        ui32PIDCount = 0;
+	IMG_UINT32        aui32PIDList[RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT];
+
+	PVR_ASSERT(psDevInfo != NULL);
+	PVR_ASSERT(ui32FreelistsCount <= RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT);
+	if (ui32FreelistsCount > RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT)
+	{
+		ui32FreelistsCount = RGXFWIF_MAX_FREELISTS_TO_RECONSTRUCT;
+	}
+
+	//PVR_DPF((PVR_DBG_ERROR, "FreeList RECONSTRUCTION: %u freelist(s) requested for reconstruction", ui32FreelistsCount));
+
+	/*
+	 *  Initialise the response command (in case we don't find a freelist ID).
+	 *  Also copy the list to the 'final' freelist array.
+	 */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE;
+	sTACCBCmd.uCmdData.sFreeListsReconstructionData.ui32FreelistsCount = ui32FreelistsCount;
+
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] = paui32Freelists[ui32Loop] |
+				RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+	}
+
+	/*
+	 *  All freelists belonging to the same PID will be reconstructed.
+	 *  This simplifies tracking for AGP, since there is no longer a
+	 *  single global freelist per local freelist. The list of unique
+	 *  PIDs is calculated from the list of freelists in this first pass.
+	 */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList   = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+
+		for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+		{
+			IMG_UINT32 ui32PIDLoop;
+
+			if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+			{
+				for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+				{
+					if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+					{
+						break;
+					}
+				}
+
+				if (ui32PIDLoop == ui32PIDCount)
+				{
+					aui32PIDList[ui32PIDCount++] = psFreeList->ownerPid;
+				}
+			}
+		}
+	}
+	dllist_foreach_node(&psDevInfo->sFreeListHead, psNode, psNext)
+	{
+		RGX_FREELIST  *psFreeList  = IMG_CONTAINER_OF(psNode, RGX_FREELIST, sNode);
+		IMG_BOOL      bReconstruct = IMG_FALSE;
+		IMG_UINT32    ui32PIDLoop;
+
+		/*
+		 *  Check if this freelist needs to be reconstructed (is it in the list
+		 *  of PIDs which will have every single one of their freelists reconstructed)
+		 */
+		for (ui32PIDLoop = 0; ui32PIDLoop < ui32PIDCount; ui32PIDLoop++)
+		{
+			if (aui32PIDList[ui32PIDLoop] == psFreeList->ownerPid)
+			{
+				bReconstruct = IMG_TRUE;
+				break;
+			}
+		}
+
+		if (bReconstruct)
+		{
+			eError = RGXReconstructFreeList(psFreeList);
+			if (eError == PVRSRV_OK)
+			{
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+				/* Mark all HWRTData's of reconstructing local freelists as HWR (applies to TA/3D's not finished yet) */
+				dllist_foreach_node(&psFreeList->sNodeHWRTDataHead, psNodeHWRTData, psNextHWRTData)
+				{
+					psKMHWRTDataSet = IMG_CONTAINER_OF(psNodeHWRTData, RGX_KM_HW_RT_DATASET, sNodeHWRTData);
+					eError = DevmemAcquireCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc, (void **)&psHWRTData);
+					if (eError != PVRSRV_OK)
+					{
+						PVR_DPF((PVR_DBG_ERROR,
+									"Devmem AcquireCpuVirtAddr Failed during Reconstructing of FreeList, FwMemDesc(%p),psHWRTData(%p)",
+									psKMHWRTDataSet->psHWRTDataFwMemDesc,
+									psHWRTData));
+						continue;
+					}
+
+					psHWRTData->eState = RGXFWIF_RTDATA_STATE_HWR;
+					psHWRTData->ui32HWRTDataFlags &= ~HWRTDATA_HAS_LAST_TA;
+					RGXFwSharedMemCacheOpValue(psHWRTData->eState, FLUSH);
+					RGXFwSharedMemCacheOpValue(psHWRTData->ui32HWRTDataFlags, FLUSH);
+
+					DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+				}
+#endif
+
+				/* Update the response for this freelist if it was specifically requested for reconstruction. */
+				for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+				{
+					if (paui32Freelists[ui32Loop] == psFreeList->ui32FreelistID)
+					{
+						/* Reconstruction of this requested freelist was successful... */
+						sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &= ~RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG;
+						break;
+					}
+				}
+			}
+			else
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"Reconstructing of FreeList %p failed (%s)",
+						psFreeList,
+						PVRSRVGetErrorString(eError)));
+			}
+		}
+	}
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Check that all freelists were found and reconstructed... */
+	for (ui32Loop = 0; ui32Loop < ui32FreelistsCount; ui32Loop++)
+	{
+		PVR_ASSERT((sTACCBCmd.uCmdData.sFreeListsReconstructionData.aui32FreelistIDs[ui32Loop] &
+		            RGXFWIF_FREELISTS_RECONSTRUCTION_FAILED_FLAG) == 0);
+	}
+
+	/* send feedback */
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+/* Create a single HWRTData instance */
+static PVRSRV_ERROR RGXCreateHWRTData_aux(CONNECTION_DATA		*psConnection,
+                                          PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                          IMG_DEV_VIRTADDR		psVHeapTableDevVAddr,
+                                          IMG_DEV_VIRTADDR		sPMDataDevVAddr, /* per-HWRTData */
+                                          RGX_FREELIST			*apsFreeLists[RGXFW_MAX_FREELISTS],
+                                          IMG_DEV_VIRTADDR		sTailPtrsDevVAddr,
+                                          IMG_DEV_VIRTADDR		sMacrotileArrayDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR		sRgnHeaderDevVAddr, /* per-HWRTData */
+                                          IMG_DEV_VIRTADDR		sRTCDevVAddr,
+                                          IMG_UINT16			ui16MaxRTs,
+                                          RGX_HWRTDATA_COMMON_COOKIE	*psHWRTDataCommonCookie,
+                                          RGX_KM_HW_RT_DATASET	**ppsKMHWRTDataSet) /* per-HWRTData */
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	/* KM cookie storing all the FW/HW data */
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSet;
+
+	/* local pointers for memory descriptors of FW allocations */
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc = NULL;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc = NULL;
+
+	/* local pointer for CPU-mapped [FW]HWRTData */
+	RGXFWIF_HWRTDATA *psHWRTData = NULL;
+
+	/*
+	 * Fill in a local copy of RGXFWIF_HWRTDATA first to reduces reads and writes to
+	 * device memory. Using a byte array buffer avoids the limitation that some OSs
+	 * cannot align local variables to 64 bytes, usually needed by RGXFWIF_HWRTDATA
+	 */
+	IMG_BYTE aui8HWRTDataLocal[sizeof(RGXFWIF_HWRTDATA)] = {0};
+	RGXFWIF_HWRTDATA *psHWRTDataLocal = (RGXFWIF_HWRTDATA *)aui8HWRTDataLocal;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Prepare the HW RT DataSet struct */
+	psKMHWRTDataSet = OSAllocZMem(sizeof(*psKMHWRTDataSet));
+	if (psKMHWRTDataSet == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto AllocError;
+	}
+
+	*ppsKMHWRTDataSet = psKMHWRTDataSet;
+	psKMHWRTDataSet->psDeviceNode = psDeviceNode;
+
+	psKMHWRTDataSet->psHWRTDataCommonCookie = psHWRTDataCommonCookie;
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	/*
+	 * This FW RT-Data is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency and write-combine will
+	 * suffice on the CPU side. (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(RGXFWIF_HWRTDATA),
+	                          RGX_FWCOMCTX_ALLOCFLAGS,
+	                          "FwHwRTData",
+	                          &psHWRTDataFwMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: DevmemAllocate for RGX_FWIF_HWRTDATA failed",
+				__func__));
+		goto FWRTDataAllocateError;
+	}
+
+	psKMHWRTDataSet->psHWRTDataFwMemDesc = psHWRTDataFwMemDesc;
+	eError = RGXSetFirmwareAddress( &psKMHWRTDataSet->sHWRTDataFwAddr,
+							psHWRTDataFwMemDesc,
+							0,
+							RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:1", FWRTDataFwAddrError);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataFwMemDesc,
+									  (void **)&psHWRTData);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWRTDataCpuMapError);
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psVHeapTableDevVAddr = psVHeapTableDevVAddr;
+#endif
+
+	psHWRTDataLocal->sHWRTDataCommonFwAddr = psHWRTDataCommonCookie->sHWRTDataCommonFwAddr;
+
+#if defined(PM_INTERACTIVE_MODE)
+	psHWRTDataLocal->psPMMListDevVAddr = sPMDataDevVAddr;
+#else
+	psHWRTDataLocal->sPMRenderStateDevVAddr = sPMDataDevVAddr;
+#endif
+
+	psHWRTDataLocal->sTailPtrsDevVAddr     = sTailPtrsDevVAddr;
+	psHWRTDataLocal->sMacrotileArrayDevVAddr = sMacrotileArrayDevVAddr;
+	psHWRTDataLocal->sRgnHeaderDevVAddr		= sRgnHeaderDevVAddr;
+	psHWRTDataLocal->sRTCDevVAddr			= sRTCDevVAddr;
+
+	{
+		RGXFWIF_RTA_CTL *psRTACtl = &psHWRTDataLocal->sRTACtl;
+
+		psRTACtl->ui32RenderTargetIndex = 0;
+		psRTACtl->ui32ActiveRenderTargets = 0;
+		psRTACtl->sValidRenderTargets.ui32Addr = 0;
+		psRTACtl->sRTANumPartialRenders.ui32Addr = 0;
+		psRTACtl->ui32MaxRTs = (IMG_UINT32) ui16MaxRTs;
+
+		if (ui16MaxRTs > 1)
+		{
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for shadow render target cache");
+			eError = DevmemFwAllocate(	psDevInfo,
+										ui16MaxRTs * sizeof(IMG_UINT32),
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+										PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+										PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+										PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+										PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+										PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+										"FwShadowRTCache",
+										&psRTArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTArryError;
+			}
+
+			psKMHWRTDataSet->psRTArrayFwMemDesc = psRTArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sValidRenderTargets,
+									psRTArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:2", FWAllocateRTArryFwAddrError);
+
+			PDUMPCOMMENT(psDeviceNode, "Allocate memory for tracking renders accumulation");
+			eError = DevmemFwAllocate(psDevInfo,
+			                          ui16MaxRTs * sizeof(IMG_UINT32),
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+					                  PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+					                  PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+					                  PVRSRV_MEMALLOCFLAG_GPU_UNCACHED |
+					                  PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC |
+					                  PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+					                  "FwRendersAccumulation",
+					                  &psRendersAccArrayFwMemDesc);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate %u bytes for render target array (%s) (renders accumulation)",
+						__func__,
+						ui16MaxRTs, PVRSRVGetErrorString(eError)));
+				goto FWAllocateRTAccArryError;
+			}
+
+			psKMHWRTDataSet->psRendersAccArrayFwMemDesc = psRendersAccArrayFwMemDesc;
+
+			eError = RGXSetFirmwareAddress(	&psRTACtl->sRTANumPartialRenders,
+									psRendersAccArrayFwMemDesc,
+									0,
+									RFW_FWADDR_FLAG_NONE	);
+			PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress:3", FWAllocRTAccArryFwAddrError);
+		}
+	}
+
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		psKMHWRTDataSet->apsFreeLists[ui32Loop] = apsFreeLists[ui32Loop];
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount++;
+		psHWRTDataLocal->apsFreeLists[ui32Loop].ui32Addr = psKMHWRTDataSet->apsFreeLists[ui32Loop]->sFreeListFWDevVAddr.ui32Addr;
+		/* Invalid initial snapshot value. The snapshot is always taken during the first
+		 * kick and hence this value gets replaced, so it's safe to set it to zero.
+		 */
+		psHWRTDataLocal->aui32FreeListHWRSnapshot[ui32Loop] = 0;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_add_to_tail(&apsFreeLists[RGXFW_LOCAL_FREELIST]->sNodeHWRTDataHead, &(psKMHWRTDataSet->sNodeHWRTData));
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	OSCachedMemCopy(psHWRTData, psHWRTDataLocal, sizeof(*psHWRTDataLocal));
+
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTData 0x%08X", psKMHWRTDataSet->sHWRTDataFwAddr.ui32Addr);
+	DevmemPDumpLoadMem(psKMHWRTDataSet->psHWRTDataFwMemDesc, 0, sizeof(*psHWRTData), PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	RGXFwSharedMemCacheOpPtr(psHWRTData, FLUSH);
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	return PVRSRV_OK;
+
+FWAllocRTAccArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRendersAccArrayFwMemDesc);
+FWAllocateRTAccArryError:
+	RGXUnsetFirmwareAddress(psRTArrayFwMemDesc);
+FWAllocateRTArryFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psRTArrayFwMemDesc);
+FWAllocateRTArryError:
+	DevmemReleaseCpuVirtAddr(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataCpuMapError:
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataFwAddrError:
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+FWRTDataAllocateError:
+	*ppsKMHWRTDataSet = NULL;
+	OSFreeMem(psKMHWRTDataSet);
+
+AllocError:
+	return eError;
+}
+
+static void RGXDestroyHWRTData_aux(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	IMG_UINT32 ui32Loop;
+
+	if (psKMHWRTDataSet == NULL)
+	{
+		return;
+	}
+
+	psDevInfo = psKMHWRTDataSet->psDeviceNode->pvDevice;
+
+	if (psKMHWRTDataSet->psRTArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRTArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRTArrayFwMemDesc);
+	}
+
+	if (psKMHWRTDataSet->psRendersAccArrayFwMemDesc)
+	{
+		RGXUnsetFirmwareAddress(psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+		DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psRendersAccArrayFwMemDesc);
+	}
+
+	/* Decrease freelist refcount */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	for (ui32Loop = 0; ui32Loop < RGXFW_MAX_FREELISTS; ui32Loop++)
+	{
+		PVR_ASSERT(psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount > 0);
+		psKMHWRTDataSet->apsFreeLists[ui32Loop]->ui32RefCount--;
+	}
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_remove_node(&psKMHWRTDataSet->sNodeHWRTData);
+#endif
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+	/* Freeing the memory has to happen _after_ removing the HWRTData from the freelist
+	 * otherwise we risk traversing the freelist to find a pointer from a freed data structure */
+	RGXUnsetFirmwareAddress(psKMHWRTDataSet->psHWRTDataFwMemDesc);
+	DevmemFwUnmapAndFree(psDevInfo, psKMHWRTDataSet->psHWRTDataFwMemDesc);
+
+	OSFreeMem(psKMHWRTDataSet);
+}
+
+/* Create set of HWRTData(s) and bind it with a shared FW HWRTDataCommon */
+PVRSRV_ERROR RGXCreateHWRTDataSet(
+		CONNECTION_DATA			*psConnection,
+		PVRSRV_DEVICE_NODE		*psDeviceNode,
+		IMG_DEV_VIRTADDR	asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_DEV_VIRTADDR		asPMMListDevVAddr[RGXMKIF_NUM_RTDATAS],
+		RGX_FREELIST			*apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+		IMG_UINT32           ui32ScreenPixelMax,
+		IMG_UINT64           ui64MultiSampleCtl,
+		IMG_UINT64           ui64FlippedMultiSampleCtl,
+		IMG_UINT32           ui32TPCStride,
+		IMG_DEV_VIRTADDR		asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT32           ui32TPCSize,
+		IMG_UINT32           ui32TEScreen,
+		IMG_UINT32           ui32TEAA,
+		IMG_UINT32           ui32TEMTILE1,
+		IMG_UINT32           ui32TEMTILE2,
+		IMG_UINT32           ui32MTileStride,
+		IMG_UINT32                 ui32ISPMergeLowerX,
+		IMG_UINT32                 ui32ISPMergeLowerY,
+		IMG_UINT32                 ui32ISPMergeUpperX,
+		IMG_UINT32                 ui32ISPMergeUpperY,
+		IMG_UINT32                 ui32ISPMergeScaleX,
+		IMG_UINT32                 ui32ISPMergeScaleY,
+		IMG_DEV_VIRTADDR	asMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR	asRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+		IMG_DEV_VIRTADDR	asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+		IMG_UINT32			uiRgnHeaderSize,
+		IMG_UINT32			ui32ISPMtileSize,
+		IMG_UINT16			ui16MaxRTs,
+		RGX_KM_HW_RT_DATASET	*pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS])
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RTDataID;
+	PVRSRV_RGXDEV_INFO		*psDevInfo = psDeviceNode->pvDevice;
+
+	RGX_HWRTDATA_COMMON_COOKIE	*psHWRTDataCommonCookie;
+	RGXFWIF_HWRTDATA_COMMON		*psHWRTDataCommon;
+	DEVMEM_MEMDESC				*psHWRTDataCommonFwMemDesc;
+	RGXFWIF_DEV_VIRTADDR		sHWRTDataCommonFwAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(0 < ui16MaxRTs && ui16MaxRTs <= RGX_MAX_TA_RENDER_TARGETS, "Number of TA Render targets outside the range (0, RGX_MAX_TA_RENDER_TARGETS) is unsupported");
+
+	/* Prepare KM cleanup object for HWRTDataCommon FW object */
+	psHWRTDataCommonCookie = OSAllocZMem(sizeof(*psHWRTDataCommonCookie));
+	if (psHWRTDataCommonCookie == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto err_HWRTDataCommonCookieAlloc;
+	}
+
+	/*
+	 * This FW common context is only mapped into kernel for initialisation.
+	 * Otherwise this allocation is only used by the FW.
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_HWRTDATA_COMMON),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwHWRTDataCommon",
+			&psHWRTDataCommonFwMemDesc);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: DevmemAllocate for FwHWRTDataCommon failed", __func__));
+		goto err_HWRTDataCommonAlloc;
+	}
+	eError = RGXSetFirmwareAddress(&sHWRTDataCommonFwAddr, psHWRTDataCommonFwMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", err_HWRTDataCommonFwAddr);
+
+	eError = DevmemAcquireCpuVirtAddr(psHWRTDataCommonFwMemDesc, (void **)&psHWRTDataCommon);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", err_HWRTDataCommonVA);
+
+	psHWRTDataCommon->bTACachesNeedZeroing = IMG_FALSE;
+	psHWRTDataCommon->ui32ScreenPixelMax    = ui32ScreenPixelMax;
+	psHWRTDataCommon->ui64MultiSampleCtl = ui64MultiSampleCtl;
+	psHWRTDataCommon->ui64FlippedMultiSampleCtl = ui64FlippedMultiSampleCtl;
+	psHWRTDataCommon->ui32TPCStride         = ui32TPCStride;
+	psHWRTDataCommon->ui32TPCSize           = ui32TPCSize;
+	psHWRTDataCommon->ui32TEScreen          = ui32TEScreen;
+	psHWRTDataCommon->ui32TEAA              = ui32TEAA;
+	psHWRTDataCommon->ui32TEMTILE1          = ui32TEMTILE1;
+	psHWRTDataCommon->ui32TEMTILE2          = ui32TEMTILE2;
+	psHWRTDataCommon->ui32MTileStride       = ui32MTileStride;
+	psHWRTDataCommon->ui32ISPMergeLowerX    = ui32ISPMergeLowerX;
+	psHWRTDataCommon->ui32ISPMergeLowerY    = ui32ISPMergeLowerY;
+	psHWRTDataCommon->ui32ISPMergeUpperX    = ui32ISPMergeUpperX;
+	psHWRTDataCommon->ui32ISPMergeUpperY    = ui32ISPMergeUpperY;
+	psHWRTDataCommon->ui32ISPMergeScaleX    = ui32ISPMergeScaleX;
+	psHWRTDataCommon->ui32ISPMergeScaleY    = ui32ISPMergeScaleY;
+	psHWRTDataCommon->uiRgnHeaderSize       = uiRgnHeaderSize;
+	psHWRTDataCommon->ui32ISPMtileSize      = ui32ISPMtileSize;
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump HWRTDataCommon");
+	DevmemPDumpLoadMem(psHWRTDataCommonFwMemDesc, 0, sizeof(*psHWRTDataCommon), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psHWRTDataCommon, FLUSH);
+	DevmemReleaseCpuVirtAddr(psHWRTDataCommonFwMemDesc);
+
+	psHWRTDataCommonCookie->ui32RefCount = 0;
+	psHWRTDataCommonCookie->psHWRTDataCommonFwMemDesc = psHWRTDataCommonFwMemDesc;
+	psHWRTDataCommonCookie->sHWRTDataCommonFwAddr = sHWRTDataCommonFwAddr;
+
+	/* Here we are creating a set of HWRTData(s)
+	   the number of elements in the set equals RGXMKIF_NUM_RTDATAS.
+	*/
+
+	for (ui32RTDataID = 0; ui32RTDataID < RGXMKIF_NUM_RTDATAS; ui32RTDataID++)
+	{
+		eError = RGXCreateHWRTData_aux(
+			psConnection,
+			psDeviceNode,
+			asVHeapTableDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+			asPMMListDevVAddr[ui32RTDataID],
+			&apsFreeLists[(ui32RTDataID % RGXMKIF_NUM_GEOMDATAS) * RGXFW_MAX_FREELISTS],
+			asTailPtrsDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+			asMacrotileArrayDevVAddr[ui32RTDataID],
+			asRgnHeaderDevVAddr[ui32RTDataID],
+			asRTCDevVAddr[ui32RTDataID % RGXMKIF_NUM_GEOMDATAS],
+			ui16MaxRTs,
+			psHWRTDataCommonCookie,
+			&pasKMHWRTDataSet[ui32RTDataID]);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to create HWRTData [slot %u] (%s)",
+					__func__,
+					ui32RTDataID,
+					PVRSRVGetErrorString(eError)));
+			goto err_HWRTDataAlloc;
+		}
+		psHWRTDataCommonCookie->ui32RefCount += 1;
+	}
+
+	return PVRSRV_OK;
+
+err_HWRTDataAlloc:
+	PVR_DPF((PVR_DBG_WARNING, "%s: err_HWRTDataAlloc %u",
+			 __func__, psHWRTDataCommonCookie->ui32RefCount));
+	if (pasKMHWRTDataSet)
+	{
+		for (ui32RTDataID = psHWRTDataCommonCookie->ui32RefCount; ui32RTDataID > 0; ui32RTDataID--)
+		{
+			if (pasKMHWRTDataSet[ui32RTDataID-1] != NULL)
+			{
+				RGXDestroyHWRTData_aux(pasKMHWRTDataSet[ui32RTDataID-1]);
+				pasKMHWRTDataSet[ui32RTDataID-1] = NULL;
+			}
+		}
+	}
+err_HWRTDataCommonVA:
+	RGXUnsetFirmwareAddress(psHWRTDataCommonFwMemDesc);
+err_HWRTDataCommonFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psHWRTDataCommonFwMemDesc);
+err_HWRTDataCommonAlloc:
+	OSFreeMem(psHWRTDataCommonCookie);
+err_HWRTDataCommonCookieAlloc:
+
+	return eError;
+}
+
+/* Destroy a single instance of HWRTData.
+   Additionally, destroy the HWRTDataCommon{Cookie} objects
+   when it is the last HWRTData within a corresponding set of HWRTDatas.
+*/
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_ERROR eError;
+	PRGXFWIF_HWRTDATA psHWRTData;
+	RGX_HWRTDATA_COMMON_COOKIE *psCommonCookie;
+
+	PVR_ASSERT(psKMHWRTDataSet);
+
+	psDevNode = psKMHWRTDataSet->psDeviceNode;
+
+	eError = RGXSetFirmwareAddress(&psHWRTData,
+	                               psKMHWRTDataSet->psHWRTDataFwMemDesc, 0,
+	                               RFW_FWADDR_NOREF_FLAG);
+	PVR_RETURN_IF_ERROR(eError);
+
+	/* Cleanup HWRTData */
+	eError = RGXFWRequestHWRTDataCleanUp(psDevNode, psHWRTData);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDevNode,
+						   eError,
+						   RGXFWRequestHWRTDataCleanUp);
+
+	psCommonCookie = psKMHWRTDataSet->psHWRTDataCommonCookie;
+
+	RGXDestroyHWRTData_aux(psKMHWRTDataSet);
+
+	/* We've got past potential PVRSRV_ERROR_RETRY events, so we are sure
+	   that the HWRTDATA instance will be destroyed during this call.
+	   Consequently, we decrease the ref count for HWRTDataCommonCookie.
+
+	   NOTE: This ref count does not require locks or atomics.
+	   -------------------------------------------------------
+	     HWRTDatas bound into one pair are always destroyed sequentially,
+	     within a single loop on the Client side.
+	     The Common/Cookie objects always belong to only one pair of
+	     HWRTDatas, and ref count is used to ensure that the Common/Cookie
+	     objects will be destroyed after destruction of all HWRTDatas
+	     within a single pair.
+	*/
+	psCommonCookie->ui32RefCount--;
+
+	/* When ref count for HWRTDataCommonCookie hits ZERO
+	 * we have to destroy the HWRTDataCommon [FW object] and the cookie
+	 * [KM object] afterwards. */
+	if (psCommonCookie->ui32RefCount == 0)
+	{
+		RGXUnsetFirmwareAddress(psCommonCookie->psHWRTDataCommonFwMemDesc);
+
+		/* We don't need to flush the SLC before freeing.
+		 * FW RequestCleanUp has already done that for HWRTData, so we're fine
+		 * now. */
+
+		DevmemFwUnmapAndFree(psDevNode->pvDevice,
+		                     psCommonCookie->psHWRTDataCommonFwMemDesc);
+		OSFreeMem(psCommonCookie);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA			*psConnection,
+                               PVRSRV_DEVICE_NODE			*psDeviceNode,
+                               IMG_HANDLE				hMemCtxPrivData,
+                               IMG_UINT32				ui32MaxFLPages,
+                               IMG_UINT32				ui32InitFLPages,
+                               IMG_UINT32				ui32GrowFLPages,
+                               IMG_UINT32				ui32GrowParamThreshold,
+                               RGX_FREELIST				*psGlobalFreeList,
+                               IMG_BOOL					bCheckFreelist,
+                               DEVMEMINT_RESERVATION		*psFreeListReservation,
+                               RGX_FREELIST				**ppsFreeList)
+{
+	PVRSRV_ERROR       eError;
+	RGXFWIF_FREELIST   *psFWFreeList;
+	RGXFWIF_FREELIST   sFWFreeList = {{0}};
+	DEVMEM_MEMDESC     *psFWFreelistMemDesc;
+	RGX_FREELIST       *psFreeList;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	IMG_DEV_VIRTADDR   sFreeListDevVAddr;
+	PMR*               psFreeListPMR = NULL;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psFreeListReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Failed to acquire reservation for freelist buffer",
+		        __func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	eError = DevmemIntGetReservationData(psFreeListReservation, &psFreeListPMR, &sFreeListDevVAddr);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Error from DevmemIntGetReservationData: %s",
+		        __func__, PVRSRVGetErrorString(eError)));
+
+		goto ErrorAllocHost;
+	}
+
+	/* Check if client properly allocated PMMETA_PROTECT */
+	if ((PMR_Flags(psFreeListPMR) & PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT)) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Freelist PMR must have PMMETA_PROTECT set",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_FLAGS;
+		goto ErrorAllocHost;
+	}
+
+	if (PMR_IsSparse(psFreeListPMR))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: Free list PMR cannot be sparse!",
+		        __func__));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto ErrorAllocHost;
+	}
+
+	/* Ref the PMR to prevent resource being destroyed before use */
+	PMRRefPMR(psFreeListPMR);
+
+	if (OSGetPageShift() > RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT)
+	{
+		IMG_UINT32 ui32Size, ui32NewInitFLPages, ui32NewMaxFLPages, ui32NewGrowFLPages;
+
+		/* Round up number of FL pages to the next multiple of the OS page size */
+
+		ui32Size = ui32InitFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewInitFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32GrowFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewGrowFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		ui32Size = ui32MaxFLPages << RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+		ui32Size = PVR_ALIGN(ui32Size, (IMG_DEVMEM_SIZE_T)OSGetPageSize());
+		ui32NewMaxFLPages = ui32Size >> RGX_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Increased number of PB pages: Init %u -> %u, Grow %u -> %u, Max %u -> %u",
+		        __func__, ui32InitFLPages, ui32NewInitFLPages, ui32GrowFLPages, ui32NewGrowFLPages, ui32MaxFLPages, ui32NewMaxFLPages));
+
+		ui32InitFLPages = ui32NewInitFLPages;
+		ui32GrowFLPages = ui32NewGrowFLPages;
+		ui32MaxFLPages = ui32NewMaxFLPages;
+	}
+
+	/* Allocate kernel freelist struct */
+	psFreeList = OSAllocZMem(sizeof(*psFreeList));
+	if (psFreeList == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: failed to allocate host data structure",
+		        __func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocHost;
+	}
+
+	/*
+	 * This FW FreeList context is only mapped into kernel for initialisation
+	 * and reconstruction (at other times it is not mapped and only used by the
+	 * FW).
+	 * Therefore the GPU cache doesn't need coherency, and write-combine will
+	 * suffice on the CPU side (WC buffer will be flushed at the first TA-kick)
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+	                          sizeof(*psFWFreeList),
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+	                          PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+	                          PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+	                          PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                          PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                          PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                          PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+	                          "FwFreeList",
+	                          &psFWFreelistMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: DevmemAllocate for RGXFWIF_FREELIST failed",
+		        __func__));
+		goto FWFreeListAlloc;
+	}
+
+	/* Initialise host data structures */
+	psFreeList->psDevInfo = psDevInfo;
+	psFreeList->psConnection = psConnection;
+
+	psFreeList->psFreeListPMR = psFreeListPMR;
+	psFreeList->psFreeListReservation = psFreeListReservation;
+
+	psFreeList->uiFreeListPMROffset = 0U;
+
+	psFreeList->psFWFreelistMemDesc = psFWFreelistMemDesc;
+	eError = RGXSetFirmwareAddress(&psFreeList->sFreeListFWDevVAddr, psFWFreelistMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	/* psFreeList->ui32FreelistID set below with lock... */
+	psFreeList->ui32FreelistGlobalID = (psGlobalFreeList ? psGlobalFreeList->ui32FreelistID : 0);
+	psFreeList->ui32MaxFLPages = ui32MaxFLPages;
+	psFreeList->ui32InitFLPages = ui32InitFLPages;
+	psFreeList->ui32GrowFLPages = ui32GrowFLPages;
+	psFreeList->ui32CurrentFLPages = 0;
+	psFreeList->ui32ReadyFLPages = 0;
+	psFreeList->ui32GrowThreshold = ui32GrowParamThreshold;
+	psFreeList->ui64FreelistChecksum = 0;
+	psFreeList->ui32RefCount = 0;
+	psFreeList->bCheckFreelist = bCheckFreelist;
+	dllist_init(&psFreeList->sMemoryBlockHead);
+	dllist_init(&psFreeList->sMemoryBlockInitHead);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	dllist_init(&psFreeList->sNodeHWRTDataHead);
+#endif
+	psFreeList->ownerPid = OSGetCurrentClientProcessIDKM();
+
+
+	/* Add to list of freelists */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	psFreeList->ui32FreelistID = psDevInfo->ui32FreelistCurrID++;
+	dllist_add_to_tail(&psDevInfo->sFreeListHead, &psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+
+
+	/* Initialise FW data structure */
+	eError = DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+	PVR_LOG_GOTO_IF_ERROR(eError, "Devmem AcquireCpuVirtAddr", FWFreeListCpuMap);
+
+	{
+		const IMG_UINT32 ui32ReadyPages = _CalculateFreelistReadyPages(psFreeList, ui32InitFLPages);
+
+		sFWFreeList.ui32MaxPages = ui32MaxFLPages;
+		sFWFreeList.ui32CurrentPages = ui32InitFLPages - ui32ReadyPages;
+		sFWFreeList.ui32GrowPages = ui32GrowFLPages;
+		sFWFreeList.psFreeListDevVAddr = sFreeListDevVAddr;
+#if defined(PM_INTERACTIVE_MODE)
+		sFWFreeList.ui32CurrentStackTop = sFWFreeList.ui32CurrentPages - 1;
+		sFWFreeList.ui64CurrentDevVAddr = (sFreeListDevVAddr.uiAddr +
+		        ((ui32MaxFLPages - sFWFreeList.ui32CurrentPages) * sizeof(IMG_UINT32))) &
+		                ~((IMG_UINT64)RGX_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE-1);
+#endif
+		sFWFreeList.ui32FreeListID = psFreeList->ui32FreelistID;
+		sFWFreeList.bGrowPending = IMG_FALSE;
+		sFWFreeList.ui32ReadyPages = ui32ReadyPages;
+
+#if defined(SUPPORT_SHADOW_FREELISTS)
+		/* Get the FW Memory Context address... */
+		eError = RGXSetFirmwareAddress(&sFWFreeList.psFWMemContext,
+		                               RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData),
+		                               0, RFW_FWADDR_NOREF_FLAG);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: RGXSetFirmwareAddress for RGXFWIF_FWMEMCONTEXT failed",
+			        __func__));
+			DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+			goto FWFreeListCpuMap;
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(hMemCtxPrivData);
+#endif
+	}
+
+	OSCachedMemCopy(psFWFreeList, &sFWFreeList, sizeof(sFWFreeList));
+	RGXFwSharedMemCacheOpPtr(psFWFreeList, FLUSH);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	        "Freelist %p created: Max pages 0x%08x, Init pages 0x%08x, "
+	        "Max FL base address 0x%016" IMG_UINT64_FMTSPECx ", "
+	        "Init FL base address 0x%016" IMG_UINT64_FMTSPECx,
+	        psFreeList,
+	        ui32MaxFLPages,
+	        ui32InitFLPages,
+	        sFreeListDevVAddr.uiAddr,
+	        sFWFreeList.ui64CurrentDevVAddr));
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump FW FreeList");
+	DevmemPDumpLoadMem(psFreeList->psFWFreelistMemDesc, 0, sizeof(*psFWFreeList), PDUMP_FLAGS_CONTINUOUS);
+
+#if defined(PM_INTERACTIVE_MODE)
+	/*
+	 * Separate dump of the Freelist's number of Pages and stack pointer.
+	 * This allows to easily modify the PB size in the out2.txt files.
+	 */
+	PDUMPCOMMENT(psDeviceNode, "FreeList TotalPages");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentPages),
+	                          sFWFreeList.ui32CurrentPages,
+	                          PDUMP_FLAGS_CONTINUOUS);
+	PDUMPCOMMENT(psDeviceNode, "FreeList StackPointer");
+	DevmemPDumpLoadMemValue32(psFreeList->psFWFreelistMemDesc,
+	                          offsetof(RGXFWIF_FREELIST, ui32CurrentStackTop),
+	                          sFWFreeList.ui32CurrentStackTop,
+	                          PDUMP_FLAGS_CONTINUOUS);
+#endif
+#endif
+	DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+	/* Add initial PB block */
+	eError = RGXGrowFreeList(psFreeList,
+	                         ui32InitFLPages,
+	                         &psFreeList->sMemoryBlockInitHead);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: failed to allocate initial memory block for free list 0x%016" IMG_UINT64_FMTSPECx " (%s)",
+		         __func__,
+		         sFreeListDevVAddr.uiAddr,
+		         PVRSRVGetErrorString(eError)));
+		goto FWFreeListCpuMap;
+	}
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	/* Update Stats */
+	PVRSRVStatsUpdateFreelistStats(psDeviceNode,
+	                               1, /* Add 1 to the appropriate counter (Requests by App)*/
+	                               0,
+	                               psFreeList->ui32InitFLPages,
+	                               psFreeList->ui32NumHighPages,
+	                               psFreeList->ownerPid);
+
+#endif
+
+	/* return values */
+	*ppsFreeList = psFreeList;
+
+	return PVRSRV_OK;
+
+	/* Error handling */
+
+FWFreeListCpuMap:
+	/* Remove freelists from list */
+	OSLockAcquire(psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+	OSLockRelease(psDevInfo->hLockFreeList);
+	RGXUnsetFirmwareAddress(psFWFreelistMemDesc);
+
+ErrorSetFwAddr:
+	DevmemFwUnmapAndFree(psDevInfo, psFWFreelistMemDesc);
+	PMRUnrefPMR(psFreeList->psFreeListPMR);
+
+FWFreeListAlloc:
+	OSFreeMem(psFreeList);
+
+ErrorAllocHost:
+	DevmemIntReservationRelease(psFreeListReservation);
+
+ErrorReservationAcquire:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyFreeList
+*/
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 ui32RefCount;
+
+	PVR_ASSERT(psFreeList);
+
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	ui32RefCount = psFreeList->ui32RefCount;
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+	if (ui32RefCount != 0)
+	{
+		/* Freelist still busy */
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Freelist is not in use => start firmware cleanup */
+	eError = RGXFWRequestFreeListCleanUp(psFreeList->psDevInfo,
+										 psFreeList->sFreeListFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psFreeList->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestFreeListCleanUp);
+
+	/* Device becomes unrecoverable if the firmware took too long to
+	 * handle the cleanup request, or if SLC-flushes didn't go through
+	 * (due to some GPU lockup) */
+
+	/* Remove FreeList from linked list before we destroy it... */
+	OSLockAcquire(psFreeList->psDevInfo->hLockFreeList);
+	dllist_remove_node(&psFreeList->sNode);
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* Confirm all HWRTData nodes are freed before releasing freelist */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sNodeHWRTDataHead));
+#endif
+	OSLockRelease(psFreeList->psDevInfo->hLockFreeList);
+
+#if defined(PM_INTERACTIVE_MODE)
+	if (psFreeList->bCheckFreelist)
+	{
+		RGXFWIF_FREELIST  *psFWFreeList;
+		IMG_UINT32        ui32CurrentStackTop;
+		IMG_UINT64        ui64CheckSum;
+
+		/* Get the current stack pointer for this free list */
+		DevmemAcquireCpuVirtAddr(psFreeList->psFWFreelistMemDesc, (void **)&psFWFreeList);
+		RGXFwSharedMemCacheOpValue(psFWFreeList->ui32CurrentStackTop, INVALIDATE);
+		ui32CurrentStackTop = psFWFreeList->ui32CurrentStackTop;
+		DevmemReleaseCpuVirtAddr(psFreeList->psFWFreelistMemDesc);
+
+		if (ui32CurrentStackTop > psFreeList->ui32MaxFLPages)
+		{
+			PVR_LOG(("%s: FW freelist corrupted (%d)",
+					 __func__,
+					 ui32CurrentStackTop));
+		}
+		else if (ui32CurrentStackTop == psFreeList->ui32CurrentFLPages-1)
+		{
+			/* Do consistency tests (as the list is fully populated) */
+			_CheckFreelist(psFreeList, psFreeList->ui32CurrentFLPages + psFreeList->ui32ReadyFLPages, psFreeList->ui64FreelistChecksum, &ui64CheckSum);
+		}
+		else
+		{
+			/* Check for duplicate pages, but don't check the checksum as the list is not fully populated */
+			_CheckFreelist(psFreeList, ui32CurrentStackTop+1, 0, &ui64CheckSum);
+		}
+	}
+#endif
+
+	/* Destroy FW structures */
+	RGXUnsetFirmwareAddress(psFreeList->psFWFreelistMemDesc);
+	DevmemFwUnmapAndFree(psFreeList->psDevInfo, psFreeList->psFWFreelistMemDesc);
+
+	/* Remove grow shrink blocks */
+	while (!dllist_is_empty(&psFreeList->sMemoryBlockHead))
+	{
+		eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockHead, psFreeList);
+		PVR_ASSERT(eError == PVRSRV_OK);
+	}
+
+	/* Remove initial PB block */
+	eError = RGXShrinkFreeList(&psFreeList->sMemoryBlockInitHead, psFreeList);
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	/* consistency checks */
+	PVR_ASSERT(dllist_is_empty(&psFreeList->sMemoryBlockInitHead));
+	PVR_ASSERT(psFreeList->ui32CurrentFLPages == 0);
+
+	/* Remove reference from the PMR and reservation resources */
+	PMRUnrefPMR(psFreeList->psFreeListPMR);
+	DevmemIntReservationRelease(psFreeList->psFreeListReservation);
+
+	/* free Freelist */
+	OSFreeMem(psFreeList);
+
+	return eError;
+}
+
+
+/*
+	RGXCreateZSBuffer
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_PRBUFFER			*psFWZSBuffer;
+	RGXFWIF_PRBUFFER			sFWZSBuffer = {0};
+	RGX_ZSBUFFER_DATA			*psZSBuffer;
+	DEVMEM_MEMDESC				*psFWZSBufferMemDesc;
+	IMG_BOOL					bOnDemand = PVRSRV_CHECK_ON_DEMAND(uiMapFlags) ? IMG_TRUE : IMG_FALSE;
+
+	/* Allocate host data structure */
+	psZSBuffer = OSAllocZMem(sizeof(*psZSBuffer));
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate cleanup data structure for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto ErrorAllocCleanup;
+	}
+
+	/* Populate Host data */
+	psZSBuffer->psDevInfo = psDevInfo;
+	psZSBuffer->psReservation = psReservation;
+
+	/* Obtain reference to reservation object */
+	if (!DevmemIntReservationAcquire(psZSBuffer->psReservation))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to acquire reservation for ZS-Buffer",
+				__func__));
+		eError = PVRSRV_ERROR_REFCOUNT_OVERFLOW;
+		goto ErrorReservationAcquire;
+	}
+
+	psZSBuffer->psPMR = psPMR;
+	/* Obtain reference to PMR */
+	PMRRefPMR(psZSBuffer->psPMR);
+
+	psZSBuffer->ui32RefCount = 0;
+	psZSBuffer->bIsBacked = IMG_FALSE;
+	psZSBuffer->bOnDemand = bOnDemand;
+
+	/* Allocate firmware memory for ZS-Buffer. */
+	PDUMPCOMMENT(psDeviceNode, "Allocate firmware ZS-Buffer data structure");
+	eError = DevmemFwAllocate(psDevInfo,
+							sizeof(*psFWZSBuffer),
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(PMMETA_PROTECT) |
+							PVRSRV_MEMALLOCFLAG_DEVICE_FLAG(FIRMWARE_CACHED) |
+							PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_GPU_CACHE_INCOHERENT |
+							PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+							PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+							PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+							PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(FW_MAIN),
+							"FwZSBuffer",
+							&psFWZSBufferMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAllocFWZSBuffer;
+	}
+	psZSBuffer->psFWZSBufferMemDesc = psFWZSBufferMemDesc;
+
+	/* Temporarily map the firmware render context to the kernel. */
+	eError = DevmemAcquireCpuVirtAddr(psFWZSBufferMemDesc,
+	                                  (void **)&psFWZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware ZS-Buffer (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto ErrorAcquireFWZSBuffer;
+	}
+
+	/* Get firmware address of ZS-Buffer. */
+	eError = RGXSetFirmwareAddress(&psZSBuffer->sZSBufferFWDevVAddr, psFWZSBufferMemDesc, 0, RFW_FWADDR_FLAG_NONE);
+	PVR_LOG_GOTO_IF_ERROR(eError, "RGXSetFirmwareAddress", ErrorSetFwAddr);
+
+	if (bOnDemand)
+	{
+		/* psZSBuffer->ui32ZSBufferID set below with lock... */
+		OSLockAcquire(psDevInfo->hLockZSBuffer);
+		psZSBuffer->ui32ZSBufferID = psDevInfo->ui32ZSBufferCurrID++;
+		dllist_add_to_tail(&psDevInfo->sZSBufferHead, &psZSBuffer->sNode);
+		OSLockRelease(psDevInfo->hLockZSBuffer);
+	}
+
+	/* Populate FW ZS-Buffer data structure */
+	sFWZSBuffer.bOnDemand = bOnDemand;
+	sFWZSBuffer.eState = (bOnDemand) ? RGXFWIF_PRBUFFER_UNBACKED : RGXFWIF_PRBUFFER_BACKED;
+	sFWZSBuffer.ui32BufferID = psZSBuffer->ui32ZSBufferID;
+
+	OSCachedMemCopy(psFWZSBuffer, &sFWZSBuffer, sizeof(sFWZSBuffer));
+	/* Dump the ZS-Buffer and the memory content */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump firmware ZS-Buffer");
+	DevmemPDumpLoadMem(psFWZSBufferMemDesc, 0, sizeof(*psFWZSBuffer), PDUMP_FLAGS_CONTINUOUS);
+#endif
+	RGXFwSharedMemCacheOpPtr(psFWZSBuffer, FLUSH);
+
+	/* Release address acquired above. */
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+
+
+	/* define return value */
+	*ppsZSBuffer = psZSBuffer;
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] created (%s)",
+							psZSBuffer,
+							(bOnDemand) ? "On-Demand": "Up-front"));
+
+	psZSBuffer->owner=OSGetCurrentClientProcessIDKM();
+
+	return PVRSRV_OK;
+
+	/* error handling */
+
+ErrorSetFwAddr:
+	DevmemReleaseCpuVirtAddr(psFWZSBufferMemDesc);
+ErrorAcquireFWZSBuffer:
+	DevmemFwUnmapAndFree(psDevInfo, psFWZSBufferMemDesc);
+
+ErrorAllocFWZSBuffer:
+	PMRUnrefPMR(psZSBuffer->psPMR);
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+ErrorReservationAcquire:
+	OSFreeMem(psZSBuffer);
+
+ErrorAllocCleanup:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+	RGXDestroyZSBuffer
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psZSBuffer);
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	/* Request ZS Buffer cleanup */
+	eError = RGXFWRequestZSBufferCleanUp(psZSBuffer->psDevInfo,
+										psZSBuffer->sZSBufferFWDevVAddr);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psZSBuffer->psDevInfo->psDeviceNode,
+						   eError,
+						   RGXFWRequestZSBufferCleanUp);
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->ui32RefCount != 0)
+	{
+		/* ZS-Buffer is still referenced */
+		OSLockRelease(hLockZSBuffer);
+		PVR_DPF((PVR_DBG_WARNING, "Trying to destroy a ZS-Buffer [%p] that's still in use.", psZSBuffer));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Free the firmware render context. */
+	RGXUnsetFirmwareAddress(psZSBuffer->psFWZSBufferMemDesc);
+	DevmemFwUnmapAndFree(psZSBuffer->psDevInfo, psZSBuffer->psFWZSBufferMemDesc);
+
+	/* Remove Deferred Allocation from list */
+	if (psZSBuffer->bOnDemand)
+	{
+		PVR_ASSERT(dllist_node_is_in_list(&psZSBuffer->sNode));
+		dllist_remove_node(&psZSBuffer->sNode);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "ZS-Buffer [%p] destroyed", psZSBuffer));
+
+	/* Release reference to reservation object and the PMR */
+	PMRUnrefPMR(psZSBuffer->psPMR);
+	DevmemIntReservationRelease(psZSBuffer->psReservation);
+
+	/* Free ZS-Buffer host data structure */
+	OSFreeMem(psZSBuffer);
+
+	OSLockRelease(hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (!psZSBuffer->bOnDemand)
+	{
+		/* Only deferred allocations can be populated */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bIsBacked == IMG_FALSE)
+	{
+		IMG_HANDLE hDevmemHeap;
+
+		/* Get Heap */
+		eError = DevmemServerGetHeapHandle(psZSBuffer->psReservation, &hDevmemHeap);
+		if (unlikely(hDevmemHeap == (IMG_HANDLE)NULL))
+		{
+			OSLockRelease(hLockZSBuffer);
+			return PVRSRV_ERROR_INVALID_HEAP;
+		}
+
+		eError = DevmemIntMapPMR(psZSBuffer->psReservation, psZSBuffer->psPMR);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable populate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+
+		}
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing acquired",
+		                          psZSBuffer,
+		                          psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_TRUE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+					RGX_POPULATION **ppsPopulation)
+{
+	RGX_POPULATION *psPopulation;
+	PVRSRV_ERROR eError;
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount++;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	psZSBuffer->ui32NumReqByApp++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psZSBuffer->psDevInfo->psDeviceNode,
+	                               1, 0, psZSBuffer->owner);
+#endif
+
+	/* Do the backing */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		goto OnErrorBacking;
+	}
+
+	/* Create the handle to the backing */
+	psPopulation = OSAllocMem(sizeof(*psPopulation));
+	if (psPopulation == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto OnErrorAlloc;
+	}
+
+	psPopulation->psZSBuffer = psZSBuffer;
+
+	/* return value */
+	*ppsPopulation = psPopulation;
+
+	return PVRSRV_OK;
+
+OnErrorAlloc:
+	RGXUnbackingZSBuffer(psZSBuffer);
+
+OnErrorBacking:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	OSLockAcquire(psZSBuffer->psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psZSBuffer->psDevInfo->hLockZSBuffer);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer)
+{
+	POS_LOCK hLockZSBuffer;
+	PVRSRV_ERROR eError;
+
+	if (!psZSBuffer)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_ASSERT(psZSBuffer->ui32RefCount);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+			"ZS Buffer [%p, ID=0x%08x]: Physical backing removal requested",
+								psZSBuffer,
+								psZSBuffer->ui32ZSBufferID));
+
+	hLockZSBuffer = psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	OSLockAcquire(hLockZSBuffer);
+
+	if (psZSBuffer->bOnDemand && psZSBuffer->bIsBacked == IMG_TRUE)
+	{
+		eError = DevmemIntUnmapPMR(psZSBuffer->psReservation);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"Unable to unpopulate ZS Buffer [%p, ID=0x%08x] (%s)",
+					psZSBuffer,
+					psZSBuffer->ui32ZSBufferID,
+					PVRSRVGetErrorString(eError)));
+			OSLockRelease(hLockZSBuffer);
+			return eError;
+		}
+
+		PVR_DPF((PVR_DBG_MESSAGE, "ZS Buffer [%p, ID=0x%08x]: Physical backing removed",
+									psZSBuffer,
+									psZSBuffer->ui32ZSBufferID));
+
+		psZSBuffer->bIsBacked = IMG_FALSE;
+	}
+
+	OSLockRelease(hLockZSBuffer);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation)
+{
+	PVRSRV_ERROR eError;
+	POS_LOCK hLockZSBuffer;
+	if (!psPopulation)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	hLockZSBuffer = psPopulation->psZSBuffer->psDevInfo->hLockZSBuffer;
+
+	eError = RGXUnbackingZSBuffer(psPopulation->psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		return eError;
+	}
+
+	OSLockAcquire(hLockZSBuffer);
+	psPopulation->psZSBuffer->ui32RefCount--;
+	OSLockRelease(hLockZSBuffer);
+
+	OSFreeMem(psPopulation);
+
+	return PVRSRV_OK;
+}
+
+static RGX_ZSBUFFER_DATA *FindAndRefZSBuffer(PVRSRV_RGXDEV_INFO *psDevInfo, IMG_UINT32 ui32ZSBufferID)
+{
+	DLLIST_NODE *psNode, *psNext;
+	RGX_ZSBUFFER_DATA *psZSBuffer = NULL;
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+
+	dllist_foreach_node(&psDevInfo->sZSBufferHead, psNode, psNext)
+	{
+		RGX_ZSBUFFER_DATA *psThisZSBuffer = IMG_CONTAINER_OF(psNode, RGX_ZSBUFFER_DATA, sNode);
+
+		if (psThisZSBuffer->ui32ZSBufferID == ui32ZSBufferID)
+		{
+			psZSBuffer = psThisZSBuffer;
+
+			psZSBuffer->ui32RefCount++;
+			break;
+		}
+	}
+
+
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+	return psZSBuffer;
+}
+
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID)
+{
+	IMG_BOOL bBackingDone = IMG_TRUE;
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (Populate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Populate ZLS */
+	eError = RGXBackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"Populating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		bBackingDone = IMG_FALSE;
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_BACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = bBackingDone;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	psZSBuffer->ui32NumReqByFW++;
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+	PVRSRVStatsUpdateZSBufferStats(psDevInfo->psDeviceNode,
+	                               0, 1, psZSBuffer->owner);
+#endif
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+
+}
+
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+											IMG_UINT32 ui32ZSBufferID)
+{
+	RGX_ZSBUFFER_DATA *psZSBuffer;
+	RGXFWIF_KCCB_CMD sTACCBCmd;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psDevInfo);
+
+	/* scan all deferred allocations */
+	psZSBuffer = FindAndRefZSBuffer(psDevInfo, ui32ZSBufferID);
+
+	if (psZSBuffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "ZS Buffer Lookup for ZS Buffer ID 0x%08x failed (UnPopulate)",
+		         ui32ZSBufferID));
+
+		return;
+	}
+
+	/* Unpopulate ZLS */
+	eError = RGXUnbackingZSBuffer(psZSBuffer);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"UnPopulating ZS-Buffer (ID = 0x%08x) failed (%s)",
+				ui32ZSBufferID,
+				PVRSRVGetErrorString(eError)));
+		PVR_ASSERT(IMG_FALSE);
+	}
+
+	/* send confirmation */
+	sTACCBCmd.eCmdType = RGXFWIF_KCCB_CMD_ZSBUFFER_UNBACKING_UPDATE;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.sZSBufferFWDevVAddr.ui32Addr = psZSBuffer->sZSBufferFWDevVAddr.ui32Addr;
+	sTACCBCmd.uCmdData.sZSBufferBackingData.bDone = IMG_TRUE;
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psDevInfo,
+		                            RGXFWIF_DM_GEOM,
+		                            &sTACCBCmd,
+		                            PDUMP_FLAGS_NONE);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	/* Kernel CCB should never fill up, as the FW is processing them right away */
+	PVR_ASSERT(eError == PVRSRV_OK);
+
+	OSLockAcquire(psDevInfo->hLockZSBuffer);
+	psZSBuffer->ui32RefCount--;
+	OSLockRelease(psDevInfo->hLockZSBuffer);
+}
+
+static
+PVRSRV_ERROR _CreateTAContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_DEV_VIRTADDR sVDMCallStackAddr,
+                              IMG_UINT32 ui32CallStackDepth,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_TA_DATA *psTAData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_TACTX_STATE *psContextState;
+	IMG_UINT32 uiCoreIdx;
+	PVRSRV_ERROR eError;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TA context suspend state");
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_TACTX_STATE),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwTAContextState",
+							  &psTAData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacontextsuspendalloc;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psTAData->psContextStateMemDesc,
+			(void **)&psContextState);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to map firmware render context state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_suspendcpuvirtacquire;
+	}
+
+	for (uiCoreIdx = 0; uiCoreIdx < RGX_NUM_GEOM_CORES; uiCoreIdx++)
+	{
+		psContextState->asGeomCore[uiCoreIdx].uTAReg_VDM_CALL_STACK_POINTER_Init =
+			sVDMCallStackAddr.uiAddr + (uiCoreIdx * ui32CallStackDepth * sizeof(IMG_UINT64));
+	}
+	RGXFwSharedMemCacheOpPtr(psContextState->asGeomCore, FLUSH);
+
+	DevmemReleaseCpuVirtAddr(psTAData->psContextStateMemDesc);
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TA,
+	                                 RGXFWIF_DM_GEOM,
+	                                 NULL,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 psTAData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TA_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TA_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &psTAData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init TA fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_tacommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+#if defined(PDUMP)
+	PDUMPCOMMENT(psDeviceNode, "Dump the TA context suspend state buffer");
+	DevmemPDumpLoadMem(psTAData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_TACTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+#endif
+
+	psTAData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_tacommoncontext:
+fail_suspendcpuvirtacquire:
+	DevmemFwUnmapAndFree(psDevInfo, psTAData->psContextStateMemDesc);
+fail_tacontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+static
+PVRSRV_ERROR _Create3DContext(CONNECTION_DATA *psConnection,
+                              PVRSRV_DEVICE_NODE *psDeviceNode,
+                              DEVMEM_MEMDESC *psAllocatedMemDesc,
+                              IMG_UINT32 ui32AllocatedOffset,
+                              DEVMEM_MEMDESC *psFWMemContextMemDesc,
+                              IMG_INT32 i32Priority,
+                              IMG_UINT32 ui32MaxDeadlineMS,
+                              IMG_UINT64 ui64RobustnessAddress,
+                              RGX_COMMON_CONTEXT_INFO *psInfo,
+                              RGX_SERVER_RC_3D_DATA *ps3DData,
+                              IMG_UINT32 ui32CCBAllocSizeLog2,
+                              IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+                              IMG_UINT32 ui32ContextFlags)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	uiNumISPStoreRegs;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware 3D context suspend state");
+
+#if defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX)
+	uiNumISPStoreRegs = RGXFWIF_IPP_RESUME_REG_COUNT; /* default 1 register for IPP_resume */
+
+	uiNumISPStoreRegs += (RGX_GET_FEATURE_VALUE(psDevInfo, NUM_SPU) *
+								RGX_GET_FEATURE_VALUE(psDevInfo, NUM_ISP_PER_SPU) *
+								RGXFWIF_PIPE_COUNT_PER_ISP);
+
+
+	if (uiNumISPStoreRegs > (RGXFWIF_ISP_PIPE_COUNT_MAX + RGXFWIF_IPP_RESUME_REG_COUNT))
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+#else /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+#endif /* defined(RGX_FEATURE_NUM_SPU_MAX_VALUE_IDX) */
+
+	/* Size of the CS buffer */
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			(uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]));
+
+	eError = DevmemFwAllocate(psDevInfo,
+							  ui3DRegISPStateStoreSize,
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "Fw3DContextState",
+							  &ps3DData->psContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to allocate firmware GPU context suspend state (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcontextsuspendalloc;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_3D,
+	                                 RGXFWIF_DM_3D,
+	                                 NULL,
+	                                 psAllocatedMemDesc,
+	                                 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 ui32MaxDeadlineMS,
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to init 3D fw common context (%s)",
+				__func__,
+				PVRSRVGetErrorString(eError)));
+		goto fail_3dcommoncontext;
+	}
+
+	/*
+	 * Dump the FW 3D context suspend state buffer
+	 */
+	PDUMPCOMMENT(psDeviceNode, "Dump the 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psContextStateMemDesc,
+					   0,
+					   sizeof(RGXFWIF_3DCTX_STATE),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_3dcommoncontext:
+	DevmemFwUnmapAndFree(psDevInfo, ps3DData->psContextStateMemDesc);
+fail_3dcontextsuspendalloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVRGXCreateRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+                                            PVRSRV_DEVICE_NODE			*psDeviceNode,
+                                            IMG_INT32					i32Priority,
+                                            IMG_DEV_VIRTADDR			sVDMCallStackAddr,
+                                            IMG_UINT32					ui32CallStackDepth,
+                                            IMG_UINT32					ui32FrameworkRegisterSize,
+                                            IMG_PBYTE					pabyFrameworkRegisters,
+                                            IMG_HANDLE					hMemCtxPrivData,
+                                            IMG_UINT32					ui32StaticRenderContextStateSize,
+                                            IMG_PBYTE					pStaticRenderContextState,
+                                            IMG_UINT32					ui32PackedCCBSizeU8888,
+                                            IMG_UINT32					ui32ContextFlags,
+                                            IMG_UINT64					ui64RobustnessAddress,
+                                            IMG_UINT32					ui32MaxTADeadlineMS,
+                                            IMG_UINT32					ui32Max3DDeadlineMS,
+                                            RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+	RGX_SERVER_RENDER_CONTEXT	*psRenderContext;
+	DEVMEM_MEMDESC				*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO		sInfo = {NULL};
+	RGXFWIF_FWRENDERCONTEXT		*psFWRenderContext;
+
+	*ppsRenderContext = NULL;
+
+	if (ui32StaticRenderContextStateSize > RGXFWIF_STATIC_RENDERCONTEXT_SIZE)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRenderContext = OSAllocZMem(sizeof(*psRenderContext));
+	if (psRenderContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	eError = OSLockCreate(&psRenderContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_lock;
+	}
+
+	psRenderContext->psDeviceNode = psDeviceNode;
+
+	/*
+		Create the FW render context, this has the TA and 3D FW common
+		contexts embedded within it
+	*/
+	eError = DevmemFwAllocate(psDevInfo,
+							  sizeof(RGXFWIF_FWRENDERCONTEXT),
+							  RGX_FWCOMCTX_ALLOCFLAGS,
+							  "FwRenderContext",
+							  &psRenderContext->psFWRenderContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwrendercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+	}
+#endif
+
+	if (ui32FrameworkRegisterSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psRenderContext->psFWFrameworkMemDesc,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psRenderContext->psFWFrameworkMemDesc,
+				pabyFrameworkRegisters,
+				ui32FrameworkRegisterSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+		sInfo.psFWFrameworkMemDesc = psRenderContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DContext(psConnection,
+	                          psDeviceNode,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, s3DContext),
+	                          psFWMemContextMemDesc,
+	                          i32Priority,
+	                          ui32Max3DDeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->s3DData,
+	                          U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcontext;
+	}
+
+	eError = _CreateTAContext(psConnection,
+	                          psDeviceNode,
+	                          psRenderContext->psFWRenderContextMemDesc,
+	                          offsetof(RGXFWIF_FWRENDERCONTEXT, sTAContext),
+	                          psFWMemContextMemDesc,
+	                          sVDMCallStackAddr,
+	                          ui32CallStackDepth,
+	                          i32Priority,
+	                          ui32MaxTADeadlineMS,
+	                          ui64RobustnessAddress,
+	                          &sInfo,
+	                          &psRenderContext->sTAData,
+	                          U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+	                          U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+	                          ui32ContextFlags);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tacontext;
+	}
+
+	eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+			(void **)&psFWRenderContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_acquire_cpu_mapping;
+	}
+
+	/* Copy the static render context data */
+	OSDeviceMemCopy(&psFWRenderContext->sStaticRenderContextState, pStaticRenderContextState, ui32StaticRenderContextStateSize);
+#if defined(SUPPORT_TRP)
+	psFWRenderContext->eTRPGeomCoreAffinity = RGXFWIF_DM_MAX;
+#endif
+	DevmemPDumpLoadMem(psRenderContext->psFWRenderContextMemDesc, 0, sizeof(RGXFWIF_FWRENDERCONTEXT), PDUMP_FLAGS_CONTINUOUS);
+	RGXFwSharedMemCacheOpPtr(psFWRenderContext, FLUSH);
+	DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psRenderContext->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-ta3d");
+	if (IS_ERR(psRenderContext->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(psRenderContext->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListInit(&psRenderContext->sSyncAddrList3DUpdate);
+
+	{
+		PVRSRV_RGXDEV_INFO			*psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	}
+
+	*ppsRenderContext = psRenderContext;
+	return PVRSRV_OK;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+fail_buffer_sync_context_create:
+#endif
+fail_acquire_cpu_mapping:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _DestroyTAContext(&psRenderContext->sTAData,
+		                                         psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_tacontext:
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		PVRSRV_ERROR eError2 = _Destroy3DContext(&psRenderContext->s3DData,
+		                                         psRenderContext->psDeviceNode);
+		if (!PVRSRVIsRetryError(eError2))
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	}
+	END_LOOP_UNTIL_TIMEOUT_US();
+fail_3dcontext:
+fail_frameworkcopy:
+	if (psRenderContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+fail_fwrendercontext:
+	OSLockDestroy(psRenderContext->hLock);
+fail_lock:
+	OSFreeMem(psRenderContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	return eError;
+}
+
+/*
+ * PVRSRVRGXDestroyRenderContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	PVRSRV_ERROR				eError;
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psRenderContext->psDeviceNode->pvDevice;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_remove_node(&(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	/* Check psBufferSyncContext has not been destroyed already (by a previous
+	 * call to this function which then later returned PVRSRV_ERROR_RETRY)
+	 */
+	if (psRenderContext->psBufferSyncContext != NULL)
+	{
+		pvr_buffer_sync_context_destroy(psRenderContext->psBufferSyncContext);
+		psRenderContext->psBufferSyncContext = NULL;
+	}
+#endif
+
+	/* Cleanup the TA if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_TA_COMPLETE) == 0)
+	{
+		eError = _DestroyTAContext(&psRenderContext->sTAData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_TA_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+	/* Cleanup the 3D if we haven't already */
+	if ((psRenderContext->ui32CleanupStatus & RC_CLEANUP_3D_COMPLETE) == 0)
+	{
+		eError = _Destroy3DContext(&psRenderContext->s3DData,
+								   psRenderContext->psDeviceNode);
+		if (eError == PVRSRV_OK)
+		{
+			psRenderContext->ui32CleanupStatus |= RC_CLEANUP_3D_COMPLETE;
+		}
+		else
+		{
+			goto e0;
+		}
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+	{
+		RGXFWIF_FWRENDERCONTEXT	*psFWRenderContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc,
+				(void **)&psFWRenderContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to map firmware render context (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+		RGXFwSharedMemCacheOpValue(psFWRenderContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+
+		ui32WorkEstCCBSubmitted = psFWRenderContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psRenderContext->psFWRenderContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psRenderContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					__func__, ui32WorkEstCCBSubmitted,
+					psRenderContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			eError = PVRSRV_ERROR_RETRY;
+			goto e0;
+		}
+	}
+#endif
+
+	/*
+		Only if both TA and 3D contexts have been cleaned up can we
+		free the shared resources
+	*/
+	if (psRenderContext->ui32CleanupStatus == (RC_CLEANUP_3D_COMPLETE | RC_CLEANUP_TA_COMPLETE))
+	{
+		if (psRenderContext->psFWFrameworkMemDesc)
+		{
+			/* Free the framework buffer */
+			DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWFrameworkMemDesc);
+		}
+
+		/* Free the firmware render context */
+		DevmemFwUnmapAndFree(psDevInfo, psRenderContext->psFWRenderContextMemDesc);
+
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrListTAUpdate);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DFence);
+		SyncAddrListDeinit(&psRenderContext->sSyncAddrList3DUpdate);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			WorkEstDeInitTA3D(psDevInfo, &psRenderContext->sWorkEstData);
+		}
+#endif
+
+		OSLockDestroy(psRenderContext->hLock);
+
+		OSFreeMem(psRenderContext);
+	}
+
+	return PVRSRV_OK;
+
+e0:
+	OSWRLockAcquireWrite(psDevInfo->hRenderCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sRenderCtxtListHead), &(psRenderContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hRenderCtxListLock);
+	return eError;
+}
+
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+static void DumpUfoList(IMG_UINT32 ui32ClientTAFenceCount,
+                        IMG_UINT32 ui32ClientTAUpdateCount,
+                        IMG_UINT32 ui32Client3DFenceCount,
+                        IMG_UINT32 ui32Client3DUpdateCount,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAFenceUFOAddress,
+                        IMG_UINT32 *paui32ClientTAFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClientTAUpdateUFOAddress,
+                        IMG_UINT32 *paui32ClientTAUpdateValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DFenceUFOAddress,
+                        IMG_UINT32 *paui32Client3DFenceValue,
+                        PRGXFWIF_UFO_ADDR *pauiClient3DUpdateUFOAddress,
+                        IMG_UINT32 *paui32Client3DUpdateValue)
+{
+	IMG_UINT32 i;
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: ~~~ After populating sync prims ~~~",
+			 __func__));
+
+	/* Dump Fence syncs, Update syncs and PR Update syncs */
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA fence syncs:",
+	        __func__, ui32ClientTAFenceCount));
+	for (i = 0; i < ui32ClientTAFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAFenceCount,
+			        (void *) pauiClientTAFenceUFOAddress,
+			        pauiClientTAFenceUFOAddress->ui32Addr,
+			        *paui32ClientTAFenceValue,
+			        *paui32ClientTAFenceValue));
+			paui32ClientTAFenceValue++;
+		}
+		pauiClientTAFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TA update syncs:",
+			 __func__, ui32ClientTAUpdateCount));
+	for (i = 0; i < ui32ClientTAUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClientTAUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32ClientTAUpdateCount,
+			        (void *) pauiClientTAUpdateUFOAddress,
+			        pauiClientTAUpdateUFOAddress->ui32Addr,
+			        *paui32ClientTAUpdateValue,
+			        *paui32ClientTAUpdateValue));
+			paui32ClientTAUpdateValue++;
+		}
+		pauiClientTAUpdateUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D fence syncs:",
+			 __func__, ui32Client3DFenceCount));
+	for (i = 0; i < ui32Client3DFenceCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DFenceUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DFenceCount,
+			        (void *) pauiClient3DFenceUFOAddress,
+			        pauiClient3DFenceUFOAddress->ui32Addr,
+			        *paui32Client3DFenceValue,
+			        *paui32Client3DFenceValue));
+			paui32Client3DFenceValue++;
+		}
+		pauiClient3DFenceUFOAddress++;
+	}
+
+	PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d 3D update syncs:",
+			 __func__, ui32Client3DUpdateCount));
+	for (i = 0; i < ui32Client3DUpdateCount; i++)
+	{
+		if (BITMASK_HAS(pauiClient3DUpdateUFOAddress->ui32Addr, 1))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x,"
+			        " UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr));
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d(0x%x)",
+			        __func__, i + 1, ui32Client3DUpdateCount,
+			        (void *) pauiClient3DUpdateUFOAddress,
+			        pauiClient3DUpdateUFOAddress->ui32Addr,
+			        *paui32Client3DUpdateValue,
+			        *paui32Client3DUpdateValue));
+			paui32Client3DUpdateValue++;
+		}
+		pauiClient3DUpdateUFOAddress++;
+	}
+}
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+/*
+ * PVRSRVRGXKickTA3DKM
+ */
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRFenceSyncPrimBlock,
+								 IMG_UINT32					ui32PRFenceSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckTAFence,
+								 PVRSRV_TIMELINE			iUpdateTATimeline,
+								 PVRSRV_FENCE				*piUpdateTAFence,
+								 IMG_CHAR					szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheck3DFence,
+								 PVRSRV_TIMELINE			iUpdate3DTimeline,
+								 PVRSRV_FENCE				*piUpdate3DFence,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus)
+{
+	/* per-context helper structures */
+	RGX_CCB_CMD_HELPER_DATA *pasTACmdHelperData = psRenderContext->asTACmdHelperData;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelperData = psRenderContext->as3DCmdHelperData;
+
+	IMG_UINT32                ui32TACmdCount=0;
+	IMG_UINT32                ui323DCmdCount=0;
+	__maybe_unused IMG_UINT32 ui32TACmdOffset=0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset=0;
+
+	RGXFWIF_UFO        sPRUFO;
+	IMG_UINT32         i;
+	PVRSRV_ERROR       eError = PVRSRV_OK;
+	PVRSRV_ERROR       eError2 = PVRSRV_OK;
+
+	PVRSRV_RGXDEV_INFO *psDevInfo = FWCommonContextGetRGXDevInfo(psRenderContext->s3DData.psServerCommonContext);
+	IMG_UINT32         ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_BOOL           bCCBStateOpen = IMG_FALSE;
+
+	IMG_UINT32         ui32ClientPRUpdateCount = 0;
+	PRGXFWIF_UFO_ADDR  *pauiClientPRUpdateUFOAddress = NULL;
+	IMG_UINT32         *paui32ClientPRUpdateValue = NULL;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	PRGXFWIF_UFO_ADDR  *pauiClientTAFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClientTAUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DFenceUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  *pauiClient3DUpdateUFOAddress = NULL;
+	PRGXFWIF_UFO_ADDR  uiPRFenceUFOAddress;
+
+	IMG_UINT64         uiCheckTAFenceUID = 0;
+	IMG_UINT64         uiCheck3DFenceUID = 0;
+	IMG_UINT64         uiUpdateTAFenceUID = 0;
+	IMG_UINT64         uiUpdate3DFenceUID = 0;
+
+	IMG_BOOL bUseCombined3DAnd3DPR = bKickPR && bKick3D && !pui83DPRDMCmd;
+
+	RGXFWIF_KCCB_CMD_KICK_DATA sTACmdKickData;
+	RGXFWIF_KCCB_CMD_KICK_DATA s3DCmdKickData;
+	IMG_BOOL bUseSingleFWCommand = bKickTA && (bKickPR || bKick3D);
+
+	IMG_UINT32 ui32TACmdSizeTmp = 0, ui323DCmdSizeTmp = 0;
+
+	IMG_BOOL bTAFenceOnSyncCheckpointsOnly = IMG_FALSE;
+
+	PVRSRV_FENCE iUpdateTAFence = PVRSRV_NO_FENCE;
+	PVRSRV_FENCE iUpdate3DFence = PVRSRV_NO_FENCE;
+
+	IMG_BOOL b3DFenceOnSyncCheckpointsOnly = IMG_FALSE;
+	IMG_UINT32 ui32TAFenceTimelineUpdateValue = 0;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+
+	/*
+	 * Count of the number of TA and 3D update values (may differ from number of
+	 * TA and 3D updates later, as sync checkpoints do not need to specify a value)
+	 */
+	IMG_UINT32 ui32ClientPRUpdateValueCount = 0;
+	IMG_UINT32 ui32ClientTAUpdateValueCount = ui32ClientTAUpdateCount;
+	IMG_UINT32 ui32Client3DUpdateValueCount = ui32Client3DUpdateCount;
+	PSYNC_CHECKPOINT *apsFenceTASyncCheckpoints = NULL;				/*!< TA fence checkpoints */
+	PSYNC_CHECKPOINT *apsFence3DSyncCheckpoints = NULL;				/*!< 3D fence checkpoints */
+	IMG_UINT32 ui32FenceTASyncCheckpointCount = 0;
+	IMG_UINT32 ui32Fence3DSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psUpdateTASyncCheckpoint = NULL;				/*!< TA update checkpoint (output) */
+	PSYNC_CHECKPOINT psUpdate3DSyncCheckpoint = NULL;				/*!< 3D update checkpoint (output) */
+	PVRSRV_CLIENT_SYNC_PRIM *psTAFenceTimelineUpdateSync = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	void *pvTAUpdateFenceFinaliseData = NULL;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+
+	RGX_SYNC_DATA sTASyncData = {NULL};		/*!< Contains internal update syncs for TA */
+	RGX_SYNC_DATA s3DSyncData = {NULL};		/*!< Contains internal update syncs for 3D */
+
+	IMG_BOOL bTestSLRAdd3DCheck = IMG_FALSE;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTA = {0};
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickData3D = {0};
+	IMG_UINT32 ui32TACommandOffset = 0;
+	IMG_UINT32 ui323DCommandOffset = 0;
+	IMG_UINT32 ui32TACmdHeaderOffset = 0;
+	IMG_UINT32 ui323DCmdHeaderOffset = 0;
+	IMG_UINT32 ui323DFullRenderCommandOffset = 0;
+	IMG_UINT32 ui32TACmdOffsetWrapCheck = 0;
+	IMG_UINT32 ui323DCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+	IMG_UINT32 ui32TAFenceCount, ui323DFenceCount;
+	IMG_UINT32 ui32TAUpdateCount, ui323DUpdateCount;
+	IMG_UINT32 ui32PRUpdateCount;
+
+	IMG_PID uiCurrentProcess = OSGetCurrentClientProcessIDKM();
+
+	IMG_UINT32 ui32Client3DFenceCount = 0;
+
+	/* Ensure we haven't been given a null ptr to
+	 * TA fence values if we have been told we
+	 * have TA sync prim fences
+	 */
+	if (ui32ClientTAFenceCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAFenceValue != NULL,
+		                        "paui32ClientTAFenceValue NULL but "
+		                        "ui32ClientTAFenceCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * TA update values if we have been told we
+	 * have TA updates
+	 */
+	if (ui32ClientTAUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientTAUpdateValue != NULL,
+		                        "paui32ClientTAUpdateValue NULL but "
+		                        "ui32ClientTAUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	/* Ensure we haven't been given a null ptr to
+	 * 3D update values if we have been told we
+	 * have 3D updates
+	 */
+	if (ui32Client3DUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32Client3DUpdateValue != NULL,
+		                        "paui32Client3DUpdateValue NULL but "
+		                        "ui32Client3DUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Write FW addresses into CMD SHARED BLOCKs */
+	{
+		CMDTA3D_SHARED *psGeomCmdShared = (CMDTA3D_SHARED *)pui8TADMCmd;
+		CMDTA3D_SHARED *ps3DCmdShared = (CMDTA3D_SHARED *)pui83DDMCmd;
+		CMDTA3D_SHARED *psPR3DCmdShared = (CMDTA3D_SHARED *)pui83DPRDMCmd;
+		RGXFWIF_DEV_VIRTADDR sNullFWAddr = {0};
+
+		if (psKMHWRTDataSet == NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "KMHWRTDataSet is a null-pointer"));
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+		/* Write FW address for TA CMD
+		*/
+		if (psGeomCmdShared != NULL)
+		{
+			if (ui32TACmdSize < sizeof(*psGeomCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid TACmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psGeomCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psGeomCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for 3D CMD
+		*/
+		if (ps3DCmdShared != NULL)
+		{
+			if (ui323DCmdSize < sizeof(*ps3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			ps3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			ps3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+
+		/* Write FW address for PR3D CMD
+		*/
+		if (psPR3DCmdShared != NULL)
+		{
+			if (ui323DPRCmdSize < sizeof(*psPR3DCmdShared))
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Invalid 3DPRCmd size", __func__));
+				return PVRSRV_ERROR_INVALID_PARAMS;
+			}
+
+			psPR3DCmdShared->sHWRTData = psKMHWRTDataSet->sHWRTDataFwAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_ZSBUFFER] =
+			    psZSBuffer ? psZSBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+			psPR3DCmdShared->asPRBuffer[RGXFWIF_PRBUFFER_MSAABUFFER] =
+			    psMSAAScratchBuffer ? psMSAAScratchBuffer->sZSBufferFWDevVAddr : sNullFWAddr;
+		}
+	}
+
+	if (unlikely(iUpdateTATimeline >= 0 && !piUpdateTAFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	if (unlikely(iUpdate3DTimeline >= 0 && !piUpdate3DFence))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d, "
+			   "ui32Client3DFenceCount=%d, ui32Client3DUpdateCount=%d",
+			   __func__,
+			   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+			   ui32Client3DFenceCount, ui32Client3DUpdateCount));
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psRenderContext->psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+
+	/* Double-check we have a PR kick if there are client fences */
+	if (unlikely(!bKickPR && ui32Client3DFenceCount != 0))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: 3D fence passed without a PR kick",
+		        __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceNameTA[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+	szFenceName3D[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	ui32TAFenceCount = ui32ClientTAFenceCount;
+	ui323DFenceCount = ui32Client3DFenceCount;
+	ui32TAUpdateCount = ui32ClientTAUpdateCount;
+	ui323DUpdateCount = ui32Client3DUpdateCount;
+	ui32PRUpdateCount = ui32ClientPRUpdateCount;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32SyncPMRCount)
+	{
+		int err;
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling"
+		          " pvr_buffer_sync_resolve_and_create_fences", __func__));
+
+		err = pvr_buffer_sync_resolve_and_create_fences(
+		    psRenderContext->psBufferSyncContext,
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    ui32SyncPMRCount,
+		    ppsSyncPMRs,
+		    paui32SyncPMRFlags,
+		    &ui32BufferFenceSyncCheckpointCount,
+		    &apsBufferFenceSyncCheckpoints,
+		    &psBufferUpdateSyncCheckpoint,
+		    &psBufferSyncData
+		);
+
+		if (unlikely(err))
+		{
+			switch (err)
+			{
+				case -EINTR:
+					eError = PVRSRV_ERROR_RETRY;
+					break;
+				case -ENOMEM:
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					break;
+				default:
+					eError = PVRSRV_ERROR_INVALID_PARAMS;
+					break;
+			}
+
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   "
+				        "pvr_buffer_sync_resolve_and_create_fences failed (%d)",
+				        __func__, eError));
+			}
+			OSLockRelease(psRenderContext->hLock);
+
+			return eError;
+		}
+
+#if !defined(SUPPORT_STRIP_RENDERING)
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+		{
+			ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+#else /* !defined(SUPPORT_STRIP_RENDERING) */
+		ui323DFenceCount += ui32BufferFenceSyncCheckpointCount;
+
+		PVR_UNREFERENCED_PARAMETER(bTAFenceOnSyncCheckpointsOnly);
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+
+		if (psBufferUpdateSyncCheckpoint != NULL)
+		{
+			if (bKick3D)
+			{
+				ui323DUpdateCount++;
+			}
+			else
+			{
+				ui32PRUpdateCount++;
+			}
+		}
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2
+#error "Invalid value for UPDATE_FENCE_CHECKPOINT_COUNT. Must be either 1 or 2."
+#endif /* !defined(UPDATE_FENCE_CHECKPOINT_COUNT) || UPDATE_FENCE_CHECKPOINT_COUNT != 1 && UPDATE_FENCE_CHECKPOINT_COUNT != 2 */
+
+	if (iCheckTAFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[TA]"
+		          " (iCheckFence=%d),"
+		          " psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheckTAFence,
+		          (void *) psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheckTAFence,
+		    &ui32FenceTASyncCheckpointCount,
+		    &apsFenceTASyncCheckpoints,
+		    &uiCheckTAFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_ta_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheckTAFence, ui32FenceTASyncCheckpointCount,
+		          (void *) apsFenceTASyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFenceTASyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "TA", apsFenceTASyncCheckpoints,
+			                      ui32FenceTASyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheck3DFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence[3D]"
+		          " (iCheckFence=%d), "
+		          "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>...",
+		          __func__, iCheck3DFence,
+		          (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(
+		    psRenderContext->psDeviceNode->hSyncCheckpointContext,
+		    iCheck3DFence,
+		    &ui32Fence3DSyncCheckpointCount,
+		    &apsFence3DSyncCheckpoints,
+		    &uiCheck3DFenceUID,
+		    ui32PDumpFlags);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)",
+			          __func__, eError));
+			goto fail_resolve_input_3d_fence;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d "
+		          "checkpoints (apsFenceSyncCheckpoints=<%p>)",
+		          __func__, iCheck3DFence, ui32Fence3DSyncCheckpointCount,
+		          (void*)apsFence3DSyncCheckpoints));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+		if (apsFence3DSyncCheckpoints)
+		{
+			_DebugSyncCheckpoints(__func__, "3D", apsFence3DSyncCheckpoints,
+			                      ui32Fence3DSyncCheckpointCount);
+		}
+#endif /* defined(TA3D_CHECKPOINT_DEBUG) */
+	}
+
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		IMG_UINT32 i;
+
+		if (bKickTA)
+		{
+			ui32TAFenceCount += ui32FenceTASyncCheckpointCount;
+
+			for (i = 0; i < ui32Fence3DSyncCheckpointCount; i++)
+			{
+				if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) !=
+				    uiCurrentProcess)
+				{
+					ui32TAFenceCount++;
+				}
+			}
+		}
+
+		if (bKick3D)
+		{
+			ui323DFenceCount += ui32Fence3DSyncCheckpointCount;
+		}
+
+		ui32TAUpdateCount += iUpdateTATimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui323DUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+		ui32PRUpdateCount += iUpdate3DTimeline != PVRSRV_NO_TIMELINE && !bKick3D ?
+				UPDATE_FENCE_CHECKPOINT_COUNT : 0;
+	}
+
+
+	if (bKickTA)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+		          __func__, ui32TAFenceCount, ui32TAUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+		    ui32TAFenceCount,
+		    ui32TAUpdateCount,
+		    ui32TACmdSize,
+		    &pPreAddr,
+		    (bKick3D ? NULL : &pPostAddr),
+		    (bKick3D ? NULL : &pRMWUFOAddr),
+		    pasTACmdHelperData
+		);
+	}
+
+	if (bKickPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+		    ui323DFenceCount,
+		    0,
+		    sizeof(sPRUFO),
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickPR && !bUseCombined3DAnd3DPR)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32PRUpdateCount=%d", __func__,
+		          ui32PRUpdateCount));
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+		    0,
+		    0,
+		    ui32PRUpdateCount,
+		    /* if the client has not provided a 3DPR command, the regular 3D
+		     * command should be used instead */
+		    pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize,
+			NULL,
+			NULL,
+			NULL,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKick3D || bAbort)
+	{
+		if (!bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   calling RGXCmdHelperInitCmdCCB(),"
+		          " ui32Client3DFenceCount=%d", __func__,
+		          ui323DFenceCount));
+		}
+
+		RGXCmdHelperInitCmdCCB_CommandSize(
+			psDevInfo,
+			0,
+			bKickTA ? 0 : ui323DFenceCount,
+		    ui323DUpdateCount,
+		    ui323DCmdSize,
+			(bKickTA ? NULL : & pPreAddr),
+			&pPostAddr,
+			&pRMWUFOAddr,
+		    &pas3DCmdHelperData[ui323DCmdCount++]
+		);
+	}
+
+	if (bKickTA)
+	{
+		ui32TACmdSizeTmp = RGXCmdHelperGetCommandSize(1, pasTACmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext),
+		    ui32TACmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	if (ui323DCmdCount > 0)
+	{
+		ui323DCmdSizeTmp = RGXCmdHelperGetCommandSize(ui323DCmdCount, pas3DCmdHelperData);
+
+		eError = RGXCheckSpaceCCB(
+		    FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext),
+		    ui323DCmdSizeTmp
+		);
+		if (eError != PVRSRV_OK)
+		{
+			goto err_not_enough_space;
+		}
+	}
+
+	/* need to reset the counter here */
+
+	ui323DCmdCount = 0;
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAFence, %d fences)...",
+			   __func__, ui32ClientTAFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAFence,
+										ui32ClientTAFenceCount,
+										apsClientTAFenceSyncPrimBlock,
+										paui32ClientTAFenceSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_fence;
+	}
+
+	if (ui32ClientTAFenceCount)
+	{
+		pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrListTAUpdate, %d updates)...",
+			   __func__, ui32ClientTAUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrListTAUpdate,
+										ui32ClientTAUpdateCount,
+										apsClientTAUpdateSyncPrimBlock,
+										paui32ClientTAUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_ta_update;
+	}
+
+	if (ui32ClientTAUpdateCount)
+	{
+		pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: pauiClientTAUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClientTAUpdateUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DFence, %d fences)...",
+			   __func__, ui32Client3DFenceCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DFence,
+										ui32Client3DFenceCount,
+										NULL,
+										NULL);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_fence;
+	}
+
+	if (ui32Client3DFenceCount)
+	{
+		pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DFenceUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DFenceUFOAddress));
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: SyncAddrListPopulate(psRenderContext->sSyncAddrList3DUpdate, %d updates)...",
+			   __func__, ui32Client3DUpdateCount));
+	eError = SyncAddrListPopulate(&psRenderContext->sSyncAddrList3DUpdate,
+										ui32Client3DUpdateCount,
+										apsClient3DUpdateSyncPrimBlock,
+										paui32Client3DUpdateSyncOffset);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_populate_sync_addr_list_3d_update;
+	}
+
+	if (ui32Client3DUpdateCount || (iUpdate3DTimeline != PVRSRV_NO_TIMELINE && piUpdate3DFence && bKick3D))
+	{
+		pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+	}
+	CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiClient3DUpdateUFOAddress=<%p> ",
+			   __func__, (void*)pauiClient3DUpdateUFOAddress));
+
+	eError = SyncPrimitiveBlockToFWAddr(psPRFenceSyncPrimBlock, ui32PRFenceSyncOffset, &uiPRFenceUFOAddress);
+
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		goto err_pr_fence_address;
+	}
+
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	if (ui32SyncPMRCount)
+	{
+#if defined(SUPPORT_BUFFER_SYNC)
+#if !defined(SUPPORT_STRIP_RENDERING)
+		/* Append buffer sync fences to TA fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0 && bKickTA)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to TA Fence "
+					   "(&psRenderContext->sSyncAddrListTAFence=<%p>, "
+					   "pauiClientTAFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrListTAFence ,
+					   (void*)pauiClientTAFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+			                                      ui32BufferFenceSyncCheckpointCount,
+			                                      apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClientTAFenceUFOAddress)
+			{
+				pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+			}
+			if (ui32ClientTAFenceCount == 0)
+			{
+				bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32ClientTAFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+		else
+#endif /* !defined(SUPPORT_STRIP_RENDERING) */
+		/* Append buffer sync fences to 3D fences */
+		if (ui32BufferFenceSyncCheckpointCount > 0)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   Append %d buffer sync checkpoints to 3D Fence "
+					   "(&psRenderContext->sSyncAddrList3DFence=<%p>, "
+					   "pauiClient3DFenceUFOAddress=<%p>)...",
+					   __func__,
+					   ui32BufferFenceSyncCheckpointCount,
+					   (void*)&psRenderContext->sSyncAddrList3DFence,
+					   (void*)pauiClient3DFenceUFOAddress));
+			SyncAddrListAppendAndDeRefCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+					ui32BufferFenceSyncCheckpointCount,
+					apsBufferFenceSyncCheckpoints);
+			if (apsBufferFenceSyncCheckpoints)
+			{
+				kfree(apsBufferFenceSyncCheckpoints);
+				apsBufferFenceSyncCheckpoints = NULL;
+			}
+			if (!pauiClient3DFenceUFOAddress)
+			{
+				pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+			}
+			if (ui32Client3DFenceCount == 0)
+			{
+				b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+			}
+			ui32Client3DFenceCount += ui32BufferFenceSyncCheckpointCount;
+		}
+
+		if (psBufferUpdateSyncCheckpoint)
+		{
+			/* If we have a 3D kick append update to the 3D updates else append to the PR update */
+			if (bKick3D)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   Append 1 buffer sync checkpoint<%p> to 3D Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClient3DUpdateUFOAddress=<%p>)...",
+						   __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClient3DUpdateUFOAddress));
+				/* Append buffer sync update to 3D updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClient3DUpdateUFOAddress)
+				{
+					pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32Client3DUpdateCount++;
+			}
+			else
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+				           "%s:   Append 1 buffer sync checkpoint<%p> to PR Update"
+						   " (&psRenderContext->sSyncAddrList3DUpdate=<%p>,"
+						   " pauiClientPRUpdateUFOAddress=<%p>)...",
+				           __func__,
+						   (void*)psBufferUpdateSyncCheckpoint,
+						   (void*)&psRenderContext->sSyncAddrList3DUpdate,
+						   (void*)pauiClientPRUpdateUFOAddress));
+				/* Attach update to the 3D (used for PR) Updates */
+				SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiClientPRUpdateUFOAddress)
+				{
+					pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+				}
+				ui32ClientPRUpdateCount++;
+			}
+		}
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   (after buffer_sync) ui32ClientTAFenceCount=%d, "
+				   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+				   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+				   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+				   ui32ClientPRUpdateCount));
+
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Buffer sync not supported but got %u buffers",
+				 __func__, ui32SyncPMRCount));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto err_no_buffer_sync_invalid_params;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	}
+
+	/*
+	 * The hardware requires a PR to be submitted if there is a TA (otherwise
+	 * it can wedge if we run out of PB space with no PR to run)
+	 *
+	 * If we only have a TA, attach native checks to the TA and updates to the PR
+	 * If we have a TA and 3D, attach checks to TA, updates to 3D
+	 * If we only have a 3D, attach checks and updates to the 3D
+	 *
+	 * Note that 'updates' includes the cleanup syncs for 'check' fence FDs, in
+	 * addition to the update fence FD (if supplied)
+	 *
+	 * Currently, the client driver never kicks only the 3D, so we only support
+	 * that for the time being.
+	 */
+	if (iCheckTAFence >= 0 || iUpdateTATimeline >= 0 ||
+	    iCheck3DFence >= 0 || iUpdate3DTimeline >= 0)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [TA] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheckTAFence, iUpdateTATimeline));
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s: [3D] iCheckFence = %d, iUpdateTimeline = %d",
+				   __func__, iCheck3DFence, iUpdate3DTimeline));
+
+		{
+			/* Create the output fence for TA (if required) */
+			if (iUpdateTATimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdateTAFence, iUpdateTATimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceNameTA,
+						iUpdateTATimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdateTAFence,
+						&uiUpdateTAFenceUID,
+						&pvTAUpdateFenceFinaliseData,
+						&psUpdateTASyncCheckpoint,
+						(void*)&psTAFenceTimelineUpdateSync,
+						&ui32TAFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[TA] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_ta_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[TA] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdateTAFence,
+						   (void*)psTAFenceTimelineUpdateSync,
+						   ui32TAFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClientTAIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClientTAIntUpdateUFOAddress */
+					pauiClientTAIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdateTASyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[TA]->ui32Addr=0x%x",
+							   __func__, pauiClientTAIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the TA timeline (if required) */
+			if (psTAFenceTimelineUpdateSync)
+			{
+				sTASyncData.ui32ClientUpdateCount		 = ui32ClientTAUpdateCount;
+				sTASyncData.ui32ClientUpdateValueCount	 = ui32ClientTAUpdateValueCount;
+				sTASyncData.ui32ClientPRUpdateValueCount = (bKick3D) ? 0 : ui32ClientPRUpdateValueCount;
+				sTASyncData.paui32ClientUpdateValue		 = paui32ClientTAUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui32TAFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrListTAUpdate,
+											(bKick3D) ? NULL : &psRenderContext->sSyncAddrList3DUpdate,
+											psTAFenceTimelineUpdateSync,
+											&sTASyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_TA;
+				}
+
+				paui32ClientTAUpdateValue = sTASyncData.paui32ClientUpdateValue;
+				ui32ClientTAUpdateValueCount = sTASyncData.ui32ClientUpdateValueCount;
+				pauiClientTAUpdateUFOAddress = sTASyncData.pauiClientUpdateUFOAddress;
+				ui32ClientTAUpdateCount = sTASyncData.ui32ClientUpdateCount;
+			}
+
+			/* Create the output fence for 3D (if required) */
+			if (iUpdate3DTimeline != PVRSRV_NO_TIMELINE)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: calling SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, iUpdateTimeline=%d, "
+						   "psRenderContext->psDeviceNode->hSyncCheckpointContext=<%p>)",
+						   __func__, iUpdate3DFence, iUpdate3DTimeline,
+						   (void*)psRenderContext->psDeviceNode->hSyncCheckpointContext));
+				eError = SyncCheckpointCreateFence(psRenderContext->psDeviceNode,
+						szFenceName3D,
+						iUpdate3DTimeline,
+						psRenderContext->psDeviceNode->hSyncCheckpointContext,
+						&iUpdate3DFence,
+						&uiUpdate3DFenceUID,
+						&pv3DUpdateFenceFinaliseData,
+						&psUpdate3DSyncCheckpoint,
+						(void*)&ps3DFenceTimelineUpdateSync,
+						&ui323DFenceTimelineUpdateValue,
+						ui32PDumpFlags);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					PVR_DPF((PVR_DBG_ERROR,
+							"%s:   SyncCheckpointCreateFence[3D] failed (%s)",
+							__func__,
+							PVRSRVGetErrorString(eError)));
+					goto fail_create_3d_fence;
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s: returned from SyncCheckpointCreateFence[3D] "
+						   "(iUpdateFence=%d, psFenceTimelineUpdateSync=<%p>, "
+						   "ui32FenceTimelineUpdateValue=0x%x)",
+						   __func__, iUpdate3DFence,
+						   (void*)ps3DFenceTimelineUpdateSync,
+						   ui323DFenceTimelineUpdateValue));
+
+#if defined(TA3D_CHECKPOINT_DEBUG)
+				{
+					PRGXFWIF_UFO_ADDR *pauiClient3DIntUpdateUFOAddress = NULL;
+
+					/* Store the FW address of the update sync checkpoint in pauiClient3DIntUpdateUFOAddress */
+					pauiClient3DIntUpdateUFOAddress = SyncCheckpointGetRGXFWIFUFOAddr(psUpdate3DSyncCheckpoint);
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: pauiClientIntUpdateUFOAddress[3D]->ui32Addr=0x%x",
+							   __func__, pauiClient3DIntUpdateUFOAddress->ui32Addr));
+				}
+#endif
+			}
+
+			/* Append the sync prim update for the 3D timeline (if required) */
+			if (ps3DFenceTimelineUpdateSync)
+			{
+				s3DSyncData.ui32ClientUpdateCount = ui32Client3DUpdateCount;
+				s3DSyncData.ui32ClientUpdateValueCount = ui32Client3DUpdateValueCount;
+				s3DSyncData.ui32ClientPRUpdateValueCount = ui32ClientPRUpdateValueCount;
+				s3DSyncData.paui32ClientUpdateValue = paui32Client3DUpdateValue;
+
+				eError = RGXSyncAppendTimelineUpdate(ui323DFenceTimelineUpdateValue,
+											&psRenderContext->sSyncAddrList3DUpdate,
+											&psRenderContext->sSyncAddrList3DUpdate,	/*!< PR update: is this required? */
+											ps3DFenceTimelineUpdateSync,
+											&s3DSyncData,
+											bKick3D);
+				if (unlikely(eError != PVRSRV_OK))
+				{
+					goto fail_alloc_update_values_mem_3D;
+				}
+
+				paui32Client3DUpdateValue = s3DSyncData.paui32ClientUpdateValue;
+				ui32Client3DUpdateValueCount = s3DSyncData.ui32ClientUpdateValueCount;
+				pauiClient3DUpdateUFOAddress = s3DSyncData.pauiClientUpdateUFOAddress;
+				ui32Client3DUpdateCount = s3DSyncData.ui32ClientUpdateCount;
+
+				if (!bKick3D)
+				{
+					paui32ClientPRUpdateValue = s3DSyncData.paui32ClientPRUpdateValue;
+					ui32ClientPRUpdateValueCount = s3DSyncData.ui32ClientPRUpdateValueCount;
+					pauiClientPRUpdateUFOAddress = s3DSyncData.pauiClientPRUpdateUFOAddress;
+					ui32ClientPRUpdateCount = s3DSyncData.ui32ClientPRUpdateCount;
+				}
+			}
+
+			/*
+			 * The hardware requires a PR to be submitted if there is a TA OOM.
+			 * If we only have a TA, attach native checks and updates to the TA
+			 * and 3D updates to the PR.
+			 * If we have a TA and 3D, attach the native TA checks and updates
+			 * to the TA and similarly for the 3D.
+			 * Note that 'updates' includes the cleanup syncs for 'check' fence
+			 * FDs, in addition to the update fence FD (if supplied).
+			 * Currently, the client driver never kicks only the 3D, so we don't
+			 * support that for the time being.
+			 */
+
+			{
+				if (bKickTA)
+				{
+					/* Attach checks and updates to TA */
+
+					/* Checks (from input fence) */
+					if (ui32FenceTASyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to TA Fence (apsFenceSyncCheckpoints=<%p>)...",
+								   __func__,
+								   ui32FenceTASyncCheckpointCount,
+								   (void*)apsFenceTASyncCheckpoints));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+													  ui32FenceTASyncCheckpointCount,
+													  apsFenceTASyncCheckpoints);
+						if (!pauiClientTAFenceUFOAddress)
+						{
+							pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+								   __func__, ui32ClientTAFenceCount,
+								   ui32ClientTAFenceCount + ui32FenceTASyncCheckpointCount));
+						if (ui32ClientTAFenceCount == 0)
+						{
+							bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32ClientTAFenceCount += ui32FenceTASyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32ClientTAFenceCount now %d}",
+							   __func__, ui32ClientTAFenceCount));
+
+					if (psUpdateTASyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Update...",
+								   __func__, (void*)psUpdateTASyncCheckpoint,
+								   SyncCheckpointGetId(psUpdateTASyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAUpdate,
+													  1,
+													  &psUpdateTASyncCheckpoint);
+						if (!pauiClientTAUpdateUFOAddress)
+						{
+							pauiClientTAUpdateUFOAddress = psRenderContext->sSyncAddrListTAUpdate.pasFWAddrs;
+						}
+						ui32ClientTAUpdateCount++;
+					}
+
+					if (!bKick3D && psUpdate3DSyncCheckpoint)
+					{
+						/* Attach update to the 3D (used for PR) Updates */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D(PR) Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClientPRUpdateUFOAddress)
+						{
+							pauiClientPRUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32ClientPRUpdateCount++;
+					}
+				}
+
+				if (bKick3D)
+				{
+					/* Attach checks and updates to the 3D */
+
+					/* Checks (from input fence) */
+					if (ui32Fence3DSyncCheckpointCount > 0)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append %d sync checkpoints to 3D Fence...",
+								   __func__, ui32Fence3DSyncCheckpointCount));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DFence,
+													  ui32Fence3DSyncCheckpointCount,
+													  apsFence3DSyncCheckpoints);
+						if (!pauiClient3DFenceUFOAddress)
+						{
+							pauiClient3DFenceUFOAddress = psRenderContext->sSyncAddrList3DFence.pasFWAddrs;
+						}
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   {ui32Client3DFenceCount was %d, now %d}",
+								   __func__, ui32Client3DFenceCount,
+								   ui32Client3DFenceCount + ui32Fence3DSyncCheckpointCount));
+						if (ui32Client3DFenceCount == 0)
+						{
+							b3DFenceOnSyncCheckpointsOnly = IMG_TRUE;
+						}
+						ui32Client3DFenceCount += ui32Fence3DSyncCheckpointCount;
+					}
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s:   {ui32Client3DFenceCount was %d}",
+							   __func__, ui32Client3DFenceCount));
+
+					if (psUpdate3DSyncCheckpoint)
+					{
+						/* Update (from output fence) */
+						CHKPT_DBG((PVR_DBG_ERROR,
+								   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to 3D Update...",
+								   __func__, (void*)psUpdate3DSyncCheckpoint,
+								   SyncCheckpointGetId(psUpdate3DSyncCheckpoint)));
+						SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrList3DUpdate,
+													  1,
+													  &psUpdate3DSyncCheckpoint);
+						if (!pauiClient3DUpdateUFOAddress)
+						{
+							pauiClient3DUpdateUFOAddress = psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs;
+						}
+						ui32Client3DUpdateCount++;
+					}
+				}
+
+				/*
+				 * Relocate sync check points from the 3D fence that are
+				 * external to the current process, to the TA fence.
+				 * This avoids a sync lockup when dependent renders are
+				 * submitted out-of-order and a PR must be scheduled.
+				 */
+				if (bKickTA)
+				{
+					/* Search for external timeline dependencies */
+					CHKPT_DBG((PVR_DBG_ERROR,
+							   "%s: Checking 3D fence for external sync points (%d)...",
+							   __func__, ui32Fence3DSyncCheckpointCount));
+
+					for (i=0; i<ui32Fence3DSyncCheckpointCount; i++)
+					{
+						/* Check to see if the checkpoint is on a TL owned by
+						 * another process.
+						 */
+						if (SyncCheckpointGetCreator(apsFence3DSyncCheckpoints[i]) != uiCurrentProcess)
+						{
+							/* 3D Sync point represents cross process
+							 * dependency, copy sync point to TA command fence. */
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   Append 1 sync checkpoint<%p> (ID=%d) to TA Fence...",
+									   __func__, (void*)apsFence3DSyncCheckpoints[i],
+									   SyncCheckpointGetId(apsFence3DSyncCheckpoints[i])));
+
+							SyncAddrListAppendCheckpoints(&psRenderContext->sSyncAddrListTAFence,
+														  1,
+														  &apsFence3DSyncCheckpoints[i]);
+
+							if (!pauiClientTAFenceUFOAddress)
+							{
+								pauiClientTAFenceUFOAddress = psRenderContext->sSyncAddrListTAFence.pasFWAddrs;
+							}
+
+							CHKPT_DBG((PVR_DBG_ERROR,
+									   "%s:   {ui32ClientTAFenceCount was %d, now %d}",
+									   __func__,
+									   ui32ClientTAFenceCount,
+									   ui32ClientTAFenceCount + 1));
+
+							if (ui32ClientTAFenceCount == 0)
+							{
+								bTAFenceOnSyncCheckpointsOnly = IMG_TRUE;
+							}
+
+							ui32ClientTAFenceCount++;
+						}
+					}
+				}
+
+				CHKPT_DBG((PVR_DBG_ERROR,
+						   "%s:   (after pvr_sync) ui32ClientTAFenceCount=%d, "
+						   "ui32ClientTAUpdateCount=%d, ui32Client3DFenceCount=%d, "
+						   "ui32Client3DUpdateCount=%d, ui32ClientPRUpdateCount=%d,",
+						   __func__,
+						   ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+						   ui32Client3DFenceCount, ui32Client3DUpdateCount,
+						   ui32ClientPRUpdateCount));
+			}
+		}
+
+		if (ui32ClientTAFenceCount)
+		{
+			PVR_ASSERT(pauiClientTAFenceUFOAddress);
+			if (!bTAFenceOnSyncCheckpointsOnly)
+			{
+				PVR_ASSERT(paui32ClientTAFenceValue);
+			}
+		}
+		if (ui32ClientTAUpdateCount)
+		{
+			PVR_ASSERT(pauiClientTAUpdateUFOAddress);
+			if (ui32ClientTAUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientTAUpdateValue);
+			}
+		}
+		if (ui32Client3DFenceCount)
+		{
+			PVR_ASSERT(pauiClient3DFenceUFOAddress);
+			PVR_ASSERT(b3DFenceOnSyncCheckpointsOnly);
+		}
+		if (ui32Client3DUpdateCount)
+		{
+			PVR_ASSERT(pauiClient3DUpdateUFOAddress);
+			if (ui32Client3DUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32Client3DUpdateValue);
+			}
+		}
+		if (ui32ClientPRUpdateCount)
+		{
+			PVR_ASSERT(pauiClientPRUpdateUFOAddress);
+			if (ui32ClientPRUpdateValueCount>0)
+			{
+				PVR_ASSERT(paui32ClientPRUpdateValue);
+			}
+		}
+
+	}
+
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAFenceCount=%d, pauiClientTAFenceUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAFenceCount,
+			   (void*)paui32ClientTAFenceValue));
+	CHKPT_DBG((PVR_DBG_ERROR,
+			   "%s: ui32ClientTAUpdateCount=%d, pauiClientTAUpdateUFOAddress=<%p> Line ",
+			   __func__,
+			   ui32ClientTAUpdateCount,
+			   (void*)pauiClientTAUpdateUFOAddress));
+#if (ENABLE_TA3D_UFO_DUMP == 1)
+	DumpUfoList(ui32ClientTAFenceCount, ui32ClientTAUpdateCount,
+	            ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+	            ui32Client3DUpdateCount,
+	            pauiClientTAFenceUFOAddress, paui32ClientTAFenceValue,
+	            pauiClientTAUpdateUFOAddress, paui32ClientTAUpdateValue,
+	            pauiClient3DFenceUFOAddress, NULL,
+	            pauiClient3DUpdateUFOAddress, paui32Client3DUpdateValue);
+#endif /* (ENABLE_TA3D_UFO_DUMP == 1) */
+
+	/* Command size check */
+	if (ui32TAFenceCount != ui32ClientTAFenceCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAFenceCount, ui32ClientTAFenceCount));
+	}
+	if (ui32TAUpdateCount != ui32ClientTAUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TA pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32TAUpdateCount, ui32ClientTAUpdateCount));
+	}
+	if (!bTestSLRAdd3DCheck && (ui323DFenceCount != ui32Client3DFenceCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of fences"
+		        " is different than the actual number (%u != %u)",
+		        ui323DFenceCount, ui32Client3DFenceCount));
+	}
+	if (ui323DUpdateCount != ui32Client3DUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "3D pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui323DUpdateCount, ui32Client3DUpdateCount));
+	}
+	if (ui32PRUpdateCount != ui32ClientPRUpdateCount)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "PR pre-calculated number of updates"
+		        " is different than the actual number (%u != %u)",
+		        ui32PRUpdateCount, ui32ClientPRUpdateCount));
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if ((!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode)) && (bKickTA || bKick3D || bAbort))
+	{
+		sWorkloadCharacteristics.sTA3D.ui32RenderTargetSize  = ui32RenderTargetSize;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfDrawCalls = ui32NumberOfDrawCalls;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfIndices   = ui32NumberOfIndices;
+		sWorkloadCharacteristics.sTA3D.ui32NumberOfMRTs      = ui32NumberOfMRTs;
+	}
+#endif
+
+	/* Init and acquire to TA command if required */
+	if (bKickTA)
+	{
+		RGX_SERVER_RC_TA_DATA *psTAData = &psRenderContext->sTAData;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sDataTA,
+					RGXFWIF_CCB_CMD_TYPE_GEOM,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTA);
+		}
+#endif
+
+		/* Init the TA command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientTAFenceCount=%d, ui32ClientTAUpdateCount=%d",
+				   __func__, ui32ClientTAFenceCount, ui32ClientTAUpdateCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(psTAData->psServerCommonContext),
+		                                 ui32ClientTAFenceCount,
+		                                 pauiClientTAFenceUFOAddress,
+		                                 paui32ClientTAFenceValue,
+		                                 ui32ClientTAUpdateCount,
+		                                 pauiClientTAUpdateUFOAddress,
+		                                 paui32ClientTAUpdateValue,
+		                                 ui32TACmdSize,
+		                                 pui8TADMCmd,
+								         &pPreAddr,
+								         (bKick3D ? NULL : & pPostAddr),
+								         (bKick3D ? NULL : & pRMWUFOAddr),
+		                                 RGXFWIF_CCB_CMD_TYPE_GEOM,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickDataTA,
+#else
+		                                 NULL,
+#endif
+		                                 "TA",
+		                                 bCCBStateOpen,
+		                                 pasTACmdHelperData);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following is used to determine the offset of the command header containing
+			   the workload estimation data so that can be accessed when the KCCB is read */
+			ui32TACmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(pasTACmdHelperData);
+		}
+#endif
+
+		eError = RGXCmdHelperAcquireCmdCCB(CCB_CMD_HELPER_NUM_TA_COMMANDS, pasTACmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taacquirecmd;
+		}
+		else
+		{
+			ui32TACmdCount++;
+		}
+	}
+
+	/* Only kick the 3D if required */
+	if (bKickPR)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+
+		/*
+			The command helper doesn't know about the PR fence so create
+			the command with all the fences against it and later create
+			the PR command itself which _must_ come after the PR fence.
+		*/
+		sPRUFO.puiAddrUFO = uiPRFenceUFOAddress;
+		sPRUFO.ui32Value = ui32PRFenceValue;
+
+		/* Init the PR fence command helper */
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32Client3DFenceCount=%d",
+				   __func__, ui32Client3DFenceCount));
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 ui32Client3DFenceCount + (bTestSLRAdd3DCheck ? 1 : 0),
+		                                 pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 0,
+		                                 NULL,
+		                                 NULL,
+		                                 sizeof(sPRUFO),
+		                                 (IMG_UINT8*) &sPRUFO,
+		                                 NULL,
+		                                 NULL,
+		                                 NULL,
+		                                 RGXFWIF_CCB_CMD_TYPE_FENCE_PR,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+		                                 NULL,
+		                                 "3D-PR-Fence",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+		/* Init the 3D PR command helper */
+		/*
+			Updates for Android (fence sync and Timeline sync prim) are provided in the PR-update
+			if no 3D is present. This is so the timeline update cannot happen out of order with any
+			other 3D already in flight for the same timeline (PR-updates are done in the 3D cCCB).
+			This out of order timeline sync prim update could happen if we attach it to the TA update.
+		*/
+		if (ui32ClientPRUpdateCount)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s: Line %d, ui32ClientPRUpdateCount=%d, "
+					   "pauiClientPRUpdateUFOAddress=0x%x, "
+					   "ui32ClientPRUpdateValueCount=%d, "
+					   "paui32ClientPRUpdateValue=0x%x",
+					   __func__, __LINE__, ui32ClientPRUpdateCount,
+					   pauiClientPRUpdateUFOAddress->ui32Addr,
+					   ui32ClientPRUpdateValueCount,
+					   (ui32ClientPRUpdateValueCount == 0) ? PVRSRV_SYNC_CHECKPOINT_SIGNALLED : *paui32ClientPRUpdateValue));
+		}
+
+		if (!bUseCombined3DAnd3DPR)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR,
+					   "%s:   calling RGXCmdHelperInitCmdCCB(), ui32ClientPRUpdateCount=%d",
+					   __func__, ui32ClientPRUpdateCount));
+			RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+			                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+			                                 0,
+			                                 NULL,
+			                                 NULL,
+			                                 ui32ClientPRUpdateCount,
+			                                 pauiClientPRUpdateUFOAddress,
+			                                 paui32ClientPRUpdateValue,
+			                                 pui83DPRDMCmd ? ui323DPRCmdSize : ui323DCmdSize, // If the client has not provided a 3DPR command, the regular 3D command should be used instead
+			                                 pui83DPRDMCmd ? pui83DPRDMCmd : pui83DDMCmd,
+			                                 NULL,
+			                                 NULL,
+			                                 NULL,
+			                                 RGXFWIF_CCB_CMD_TYPE_3D_PR,
+			                                 ui32ExtJobRef,
+			                                 ui32IntJobRef,
+			                                 ui32PDumpFlags,
+			                                 NULL,
+			                                 "3D-PR",
+			                                 bCCBStateOpen,
+			                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+		}
+	}
+
+	if (bKick3D || bAbort)
+	{
+		RGX_SERVER_RC_3D_DATA *ps3DData = &psRenderContext->s3DData;
+		const RGXFWIF_CCB_CMD_TYPE e3DCmdType = bAbort ? RGXFWIF_CCB_CMD_TYPE_ABORT : RGXFWIF_CCB_CMD_TYPE_3D;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* Prepare workload estimation */
+			WorkEstPrepare(psRenderContext->psDeviceNode->pvDevice,
+					&psRenderContext->sWorkEstData,
+					&psRenderContext->sWorkEstData.uWorkloadMatchingData.sTA3D.sData3D,
+					e3DCmdType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickData3D);
+		}
+#endif
+
+		/* Init the 3D command helper */
+		RGXCmdHelperInitCmdCCB_OtherData(psDevInfo,
+		                                 FWCommonContextGetClientCCB(ps3DData->psServerCommonContext),
+		                                 bKickTA ? 0 : ui32Client3DFenceCount,  /* For a kick with a TA, the 3D fences are added before the PR command instead */
+		                                 bKickTA ? NULL : pauiClient3DFenceUFOAddress,
+		                                 NULL,
+		                                 ui32Client3DUpdateCount,
+		                                 pauiClient3DUpdateUFOAddress,
+		                                 paui32Client3DUpdateValue,
+		                                 ui323DCmdSize,
+		                                 pui83DDMCmd,
+		                                 (bKickTA ? NULL : & pPreAddr),
+		                                 &pPostAddr,
+		                                 &pRMWUFOAddr,
+		                                 e3DCmdType,
+		                                 ui32ExtJobRef,
+		                                 ui32IntJobRef,
+		                                 ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                                 &sWorkloadKickData3D,
+#else
+		                                 NULL,
+#endif
+		                                 "3D",
+		                                 bCCBStateOpen,
+		                                 &pas3DCmdHelperData[ui323DCmdCount++]);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psRenderContext->psDeviceNode))
+		{
+			/* The following are used to determine the offset of the command header containing the workload estimation
+			   data so that can be accessed when the KCCB is read */
+			ui323DCmdHeaderOffset =	RGXCmdHelperGetDMCommandHeaderOffset(&pas3DCmdHelperData[ui323DCmdCount - 1]);
+			ui323DFullRenderCommandOffset =	RGXCmdHelperGetCommandOffset(pas3DCmdHelperData, ui323DCmdCount - 1);
+		}
+#endif
+	}
+
+	/* Protect against array overflow in RGXCmdHelperAcquireCmdCCB() */
+	if (unlikely(ui323DCmdCount > CCB_CMD_HELPER_NUM_3D_COMMANDS))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+		goto fail_3dcmdinit;
+	}
+
+	if (ui323DCmdCount)
+	{
+		PVR_ASSERT(bKickPR || bKick3D);
+
+		/* Acquire space for all the 3D command(s) */
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount, pas3DCmdHelperData);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			/* If RGXCmdHelperAcquireCmdCCB fails we skip the scheduling
+			 * of a new TA command with the same Write offset in Kernel CCB.
+			 */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line", __func__, eError));
+			goto fail_3dacquirecmd;
+		}
+	}
+
+	if (ui32TACmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &sTACmdKickData.apsCleanupCtl,
+				&sTACmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_GEOM,
+				bKickTA,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_taattachcleanupctls;
+		}
+	}
+
+	if (ui323DCmdCount)
+	{
+		eError = AttachKickResourcesCleanupCtls((PRGXFWIF_CLEANUP_CTL *) &s3DCmdKickData.apsCleanupCtl,
+				&s3DCmdKickData.ui32NumCleanupCtl,
+				RGXFWIF_DM_3D,
+				bKick3D,
+				psKMHWRTDataSet,
+				psZSBuffer,
+				psMSAAScratchBuffer);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+					   __func__, eError));
+			goto fail_3dattachcleanupctls;
+		}
+	}
+
+	/*
+		We should acquire the space in the kernel CCB here as after this point
+		we release the commands which will take operations on server syncs
+		which can't be undone
+	*/
+
+	/*
+		Everything is ready to go now, release the commands
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui32TACmdCount)
+	{
+		ui32TACmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui32TACmdCount,
+								  pasTACmdHelperData,
+								  "TA",
+								  FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui32TACmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext));
+
+			/* This checks if the command would wrap around at the end of the CCB and therefore  would start at an
+			   offset of 0 rather than the current command offset */
+			if (ui32TACmdOffset < ui32TACmdOffsetWrapCheck)
+			{
+				ui32TACommandOffset = ui32TACmdOffset;
+			}
+			else
+			{
+				ui32TACommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  pas3DCmdHelperData,
+								  "3D",
+								  FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			ui323DCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext));
+
+			if (ui323DCmdOffset < ui323DCmdOffsetWrapCheck)
+			{
+				ui323DCommandOffset = ui323DCmdOffset;
+			}
+			else
+			{
+				ui323DCommandOffset = 0;
+			}
+		}
+#endif
+	}
+
+	if (ui32TACmdCount)
+	{
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->sTAData.psServerCommonContext);
+		CMDTA3D_SHARED *psGeomCmdShared = IMG_OFFSET_ADDR(pui8TADMCmd, 0);
+
+		sTACmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->sTAData.psServerCommonContext);
+		sTACmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTACmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Add the Workload data into the KCCB kick */
+			sTACmdKickData.ui32WorkEstCmdHeaderOffset = ui32TACommandOffset + ui32TACmdHeaderOffset;
+		}
+#endif
+
+		if (psGeomCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_TA,
+					sTACmdKickData.psContext,
+					ui32TACmdOffset,
+					psGeomCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_GEOM,
+		                  iCheckTAFence,
+		                  iUpdateTAFence,
+		                  iUpdateTATimeline,
+		                  uiCheckTAFenceUID,
+		                  uiUpdateTAFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickDataTA));
+
+		if (!bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA CCB command. */
+			RGXFWIF_KCCB_CMD sTAKCCBCmd;
+			sTAKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			sTAKCCBCmd.uCmdData.sCmdKickData = sTACmdKickData;
+
+			LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+			{
+				eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+						RGXFWIF_DM_GEOM,
+						&sTAKCCBCmd,
+						ui32PDumpFlags);
+				if (eError2 != PVRSRV_ERROR_RETRY)
+				{
+					break;
+				}
+				OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+			} END_LOOP_UNTIL_TIMEOUT_US();
+		}
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			/* Mark the error and bail out */
+			eError = eError2;
+			goto fail_tasubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_GEOM);
+	}
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd = { 0 };
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *psClientCCB = FWCommonContextGetClientCCB(psRenderContext->s3DData.psServerCommonContext);
+		CMDTA3D_SHARED *ps3DCmdShared = IMG_OFFSET_ADDR(pui83DDMCmd, 0);
+
+		s3DCmdKickData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+		s3DCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		s3DCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDevInfo->psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced when the KCCB command reaches the FW */
+			s3DCmdKickData.ui32WorkEstCmdHeaderOffset = ui323DCommandOffset + ui323DCmdHeaderOffset + ui323DFullRenderCommandOffset;
+		}
+#endif
+
+		if (ps3DCmdShared)
+		{
+			HTBLOGK(HTB_SF_MAIN_KICK_3D,
+					s3DCmdKickData.psContext,
+					ui323DCmdOffset,
+					ps3DCmdShared->sCmn.ui32FrameNum,
+					ui32ExtJobRef,
+					ui32IntJobRef);
+		}
+
+		RGXSRV_HWPERF_ENQ(psRenderContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_3D,
+		                  iCheck3DFence,
+		                  iUpdate3DFence,
+		                  iUpdate3DTimeline,
+		                  uiCheck3DFenceUID,
+		                  uiUpdate3DFenceUID,
+		                  ui64DeadlineInus,
+		                  WORKEST_CYCLES_PREDICTION_GET(sWorkloadKickData3D));
+
+		if (bUseSingleFWCommand)
+		{
+			/* Construct the kernel TA/3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_COMBINED_TA_3D_KICK;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.sTACmdKickData = sTACmdKickData;
+			s3DKCCBCmd.uCmdData.sCombinedTA3DCmdKickData.s3DCmdKickData = s3DCmdKickData;
+		}
+		else
+		{
+			/* Construct the kernel 3D CCB command. */
+			s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+			s3DKCCBCmd.uCmdData.sCmdKickData = s3DCmdKickData;
+		}
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psRenderContext->psDeviceNode->pvDevice,
+										 RGXFWIF_DM_3D,
+										 &s3DKCCBCmd,
+										 ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXKicKTA3DKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			if (eError == PVRSRV_OK)
+			{
+				eError = eError2;
+			}
+			goto fail_3dsubmitcmd;
+		}
+
+		PVRGpuTraceEnqueueEvent(psRenderContext->psDeviceNode,
+		                        ui32FWCtx, ui32ExtJobRef, ui32IntJobRef,
+		                        RGX_HWPERF_KICK_TYPE2_3D);
+	}
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (unlikely(eError != PVRSRV_OK ))
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: Failed, eError=%d, Line",
+				   __func__, eError));
+		goto fail_3dsubmitcmd;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateTASyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [TA] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdateTASyncCheckpoint,
+				   SyncCheckpointGetId(psUpdateTASyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdateTASyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdateTASyncCheckpoint);
+	}
+	if (psTAFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [TA] <%p> to %d",
+				   __func__, (void*)psTAFenceTimelineUpdateSync,
+				   ui32TAFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(psTAFenceTimelineUpdateSync, ui32TAFenceTimelineUpdateValue);
+	}
+
+	if (psUpdate3DSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Signalling NOHW sync checkpoint [3D] <%p>, ID:%d, FwAddr=0x%x",
+				   __func__, (void*)psUpdate3DSyncCheckpoint,
+				   SyncCheckpointGetId(psUpdate3DSyncCheckpoint),
+				   SyncCheckpointGetFirmwareAddr(psUpdate3DSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(psUpdate3DSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   Updating NOHW sync prim [3D] <%p> to %d",
+				   __func__, (void*)ps3DFenceTimelineUpdateSync,
+				   ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR,
+				   "%s:   calling pvr_buffer_sync_kick_succeeded(psBufferSyncData=<%p>)...",
+				   __func__, (void*)psBufferSyncData));
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	if (piUpdateTAFence)
+	{
+		*piUpdateTAFence = iUpdateTAFence;
+	}
+	if (piUpdate3DFence)
+	{
+		*piUpdate3DFence = iUpdate3DFence;
+	}
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+
+	if (pvTAUpdateFenceFinaliseData && (iUpdateTAFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdateTAFence,
+									pvTAUpdateFenceFinaliseData,
+									psUpdateTASyncCheckpoint, szFenceNameTA);
+	}
+	if (pv3DUpdateFenceFinaliseData && (iUpdate3DFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psRenderContext->psDeviceNode, iUpdate3DFence,
+									pv3DUpdateFenceFinaliseData,
+									psUpdate3DSyncCheckpoint, szFenceName3D);
+	}
+
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+
+	return PVRSRV_OK;
+
+fail_3dsubmitcmd:
+fail_tasubmitcmd:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_3dattachcleanupctls:
+fail_taattachcleanupctls:
+fail_3dacquirecmd:
+fail_3dcmdinit:
+fail_taacquirecmd:
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrListTAUpdate);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DFence);
+	SyncAddrListRollbackCheckpoints(psRenderContext->psDeviceNode, &psRenderContext->sSyncAddrList3DUpdate);
+	/* Where a TA-only kick (ie no 3D) is submitted, the PR update will make use of the unused 3DUpdate list.
+	 * If this has happened, performing a rollback on pauiClientPRUpdateUFOAddress will simply repeat what
+	 * has already been done for the sSyncAddrList3DUpdate above and result in a double decrement of the
+	 * sync checkpoint's hEnqueuedCCBCount, so we need to check before rolling back the PRUpdate.
+	 */
+	if (pauiClientPRUpdateUFOAddress && (pauiClientPRUpdateUFOAddress != psRenderContext->sSyncAddrList3DUpdate.pasFWAddrs))
+	{
+		SyncCheckpointRollbackFromUFO(psRenderContext->psDeviceNode, pauiClientPRUpdateUFOAddress->ui32Addr);
+	}
+
+fail_alloc_update_values_mem_3D:
+	if (iUpdate3DFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdate3DFence, pv3DUpdateFenceFinaliseData);
+	}
+fail_create_3d_fence:
+fail_alloc_update_values_mem_TA:
+	if (iUpdateTAFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateTAFence, pvTAUpdateFenceFinaliseData);
+	}
+fail_create_ta_fence:
+#if !defined(SUPPORT_BUFFER_SYNC)
+err_no_buffer_sync_invalid_params:
+#endif /* !defined(SUPPORT_BUFFER_SYNC) */
+err_pr_fence_address:
+err_populate_sync_addr_list_3d_update:
+err_populate_sync_addr_list_3d_fence:
+err_populate_sync_addr_list_ta_update:
+err_populate_sync_addr_list_ta_fence:
+err_not_enough_space:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence.
+	 * NOTE: 3D fence is always submitted, either via 3D or TA(PR).
+	 */
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		SyncAddrListDeRefCheckpoints(ui32BufferFenceSyncCheckpointCount,
+					     apsBufferFenceSyncCheckpoints);
+	}
+#endif
+	SyncAddrListDeRefCheckpoints(ui32Fence3DSyncCheckpointCount, apsFence3DSyncCheckpoints);
+fail_resolve_input_3d_fence:
+	if (bKickTA)
+	{
+		SyncAddrListDeRefCheckpoints(ui32FenceTASyncCheckpointCount, apsFenceTASyncCheckpoints);
+	}
+fail_resolve_input_ta_fence:
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceTASyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceTASyncCheckpoints);
+	}
+	if (apsFence3DSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFence3DSyncCheckpoints);
+	}
+	if (sTASyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(sTASyncData.paui32ClientUpdateValue);
+	}
+	if (s3DSyncData.paui32ClientUpdateValue)
+	{
+		OSFreeMem(s3DSyncData.paui32ClientUpdateValue);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+	PVR_ASSERT(eError != PVRSRV_OK);
+	OSLockRelease(psRenderContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore)
+{
+	PVRSRV_ERROR eError;
+	RGXFWIF_KCCB_CMD		sDisableZSStoreCmd = { 0 };
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	sDisableZSStoreCmd.eCmdType = RGXFWIF_KCCB_CMD_DISABLE_ZSSTORE;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.psContext = FWCommonContextGetFWAddress(psRenderContext->s3DData.psServerCommonContext);
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableZStore = bDisableDepthStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.bDisableSStore = bDisableStencilStore;
+	sDisableZSStoreCmd.uCmdData.sDisableZSStoreData.sDisableZSStore.ui32ExtJobRefToDisableZSStore = ui32ExtJobRefToDisableZSStore;
+
+	if (psRenderContext->ui32CleanupStatus & (RC_CLEANUP_TA_COMPLETE | RC_CLEANUP_3D_COMPLETE))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: submit disable depth and stencil store command to render context that has been cleaned up (%u)",
+				__func__, PVRSRV_ERROR_INVALID_CONTEXT));
+		return PVRSRV_ERROR_INVALID_CONTEXT;
+	}
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psRenderContext->psDeviceNode->pvDevice,
+									RGXFWIF_DM_3D,
+									&sDisableZSStoreCmd,
+									PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to submit disable depth and stencil store command (%u)",
+				__func__,
+				eError));
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psRenderContext->hLock);
+
+	if (psRenderContext->sTAData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->sTAData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_GEOM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the TA part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_tacontext;
+		}
+		psRenderContext->sTAData.i32Priority = i32Priority;
+	}
+
+	if (psRenderContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psRenderContext->s3DData.psServerCommonContext,
+									psConnection,
+									psRenderContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to set the priority of the 3D part of the rendercontext (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto fail_3dcontext;
+		}
+		psRenderContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psRenderContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+fail_tacontext:
+	OSLockRelease(psRenderContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 RGX_CONTEXT_PROPERTY eContextProperty,
+                                                 IMG_UINT64 ui64Input,
+                                                 IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psRenderContext->hLock);
+			eError = FWCommonContextSetFlags(psRenderContext->sTAData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psRenderContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psRenderContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->sTAData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		DumpFWCommonContextInfo(psCurrentServerRenderCtx->s3DData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hRenderCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sRenderCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_RENDER_CONTEXT *psCurrentServerRenderCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_RENDER_CONTEXT, sListNode);
+		if (NULL != psCurrentServerRenderCtx->sTAData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->sTAData.psServerCommonContext, RGX_KICK_TYPE_DM_TA) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TA;
+			}
+		}
+
+		if (NULL != psCurrentServerRenderCtx->s3DData.psServerCommonContext)
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerRenderCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_3D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hRenderCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/*
+ * RGXRenderContextStalledKM
+ */
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext)
+{
+	RGXCheckForStalledClientContexts((PVRSRV_RGXDEV_INFO *) psRenderContext->psDeviceNode->pvDevice, IMG_TRUE);
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (rgxta3d.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxta3d.h b/drivers/gpu/drm/img-rogue/rgxta3d.h
new file mode 100644
index 000000000000..45c71796793e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxta3d.h
@@ -0,0 +1,509 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX TA and 3D Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX TA and 3D Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RGXTA3D_H
+#define RGXTA3D_H
+
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgx_fwif_shared.h"
+#include "rgx_fwif_resetframework.h"
+#include "sync_server.h"
+#include "connection_server.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+typedef struct _RGX_SERVER_RENDER_CONTEXT_ RGX_SERVER_RENDER_CONTEXT;
+typedef struct _RGX_FREELIST_ RGX_FREELIST;
+typedef struct _RGX_PMR_NODE_ RGX_PMR_NODE;
+
+/*****************************************************************************
+ * The Design of Data Storage System for Render Targets                      *
+ * ====================================================                      *
+ *   Relevant for                                                            *
+ *     understanding RGXCreateHWRTDataSet & RGXDestroyHWRTDataSet            *
+ *                                                                           *
+ *                                                                           *
+ *        +=========================================+                        *
+ *        |           RenderTargetDataSet           |                        *
+ *        +---------------|---------|---------------+                        *
+ *                        |         |                                        *
+ *                        V         V                                        *
+ *  +- - - - - - - - - - - - +   +- - - - - - - - - - - - +                  *
+ *  | KM_HW_RT_DATA_HANDLE_0 |   | KM_HW_RT_DATA_HANDLE_1 |                  *
+ *  +- - -|- - - - - - - - - +   +- - - - - - - - - | - - +                  *
+ *        |                                         |                        *
+ *        |                                         |           [UM]Client   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |               Bridge   *
+ *  ------|-----------------------------------------|----------------------- *
+ *        |                                         |           [KM]Server   *
+ *        |                                         |                        *
+ *        | KM-ptr                                  | KM-ptr                 *
+ *        V                                         V                        *
+ *  +====================+           +====================+                  *
+ *  |  KM_HW_RT_DATA_0   |           |   KM_HW_RT_DATA_1  |                  *
+ *  +-----|------------|-+           +-|------------|-----+                  *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            |               |            |                        *
+ *        |            | KM-ptr        | KM-ptr     |                        *
+ *        |            V               V            |                        *
+ *        |      +==========================+       |                        *
+ *        |      | HW_RT_DATA_COMMON_COOKIE |       |                        *
+ *        |      +--------------------------+       |                        *
+ *        |                   |                     |                        *
+ *        |                   |                     |                        *
+ *  ------|-------------------|---------------------|----------------------- *
+ *        |                   |                     |         [FW]Firmware   *
+ *        |                   |                     |                        *
+ *        | FW-addr           |                     | FW-addr                *
+ *        V                   |                     V                        *
+ *  +===============+         |           +===============+                  *
+ *  | HW_RT_DATA_0  |         |           | HW_RT_DATA_1  |                  *
+ *  +------------|--+         |           +--|------------+                  *
+ *               |            |              |                               *
+ *               | FW-addr    | FW-addr      | FW-addr                       *
+ *               V            V              V                               *
+ *        +=========================================+                        *
+ *        |           HW_RT_DATA_COMMON             |                        *
+ *        +-----------------------------------------+                        *
+ *                                                                           *
+ *****************************************************************************/
+
+typedef struct _RGX_HWRTDATA_COMMON_COOKIE_
+{
+	DEVMEM_MEMDESC			*psHWRTDataCommonFwMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sHWRTDataCommonFwAddr;
+	IMG_UINT32				ui32RefCount;
+
+} RGX_HWRTDATA_COMMON_COOKIE;
+
+typedef struct _RGX_KM_HW_RT_DATASET_
+{
+	RGX_HWRTDATA_COMMON_COOKIE *psHWRTDataCommonCookie;
+
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	RGXFWIF_DEV_VIRTADDR sHWRTDataFwAddr;
+
+	DEVMEM_MEMDESC *psHWRTDataFwMemDesc;
+	DEVMEM_MEMDESC *psRTArrayFwMemDesc;
+	DEVMEM_MEMDESC *psRendersAccArrayFwMemDesc;
+
+	RGX_FREELIST *apsFreeLists[RGXFW_MAX_FREELISTS];
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	DLLIST_NODE			sNodeHWRTData;
+#endif
+
+} RGX_KM_HW_RT_DATASET;
+
+struct _RGX_FREELIST_ {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	CONNECTION_DATA			*psConnection;
+
+	/* Free list PMR */
+	PMR						*psFreeListPMR;
+	IMG_DEVMEM_OFFSET_T		uiFreeListPMROffset;
+
+	DEVMEMINT_RESERVATION* psFreeListReservation;
+
+	/* Freelist config */
+	IMG_UINT32				ui32MaxFLPages;
+	IMG_UINT32				ui32InitFLPages;
+	IMG_UINT32				ui32CurrentFLPages;
+	IMG_UINT32				ui32GrowFLPages;
+	IMG_UINT32				ui32ReadyFLPages;
+	IMG_UINT32				ui32GrowThreshold;		/* Percentage of FL memory used that should trigger a new grow request */
+	IMG_UINT32				ui32FreelistID;
+	IMG_UINT32				ui32FreelistGlobalID;	/* related global freelist for this freelist */
+	IMG_UINT64				ui64FreelistChecksum;	/* checksum over freelist content */
+	IMG_BOOL				bCheckFreelist;			/* freelist check enabled */
+	IMG_UINT32				ui32RefCount;			/* freelist reference counting */
+
+	IMG_UINT32				ui32NumGrowReqByApp;	/* Total number of grow requests by Application */
+	IMG_UINT32				ui32NumGrowReqByFW;		/* Total Number of grow requests by Firmware */
+	IMG_UINT32				ui32NumHighPages;		/* High Mark of pages in the freelist */
+
+	IMG_PID					ownerPid;				/* Pid of the owner of the list */
+
+	/* Memory Blocks */
+	DLLIST_NODE				sMemoryBlockHead;
+	DLLIST_NODE				sMemoryBlockInitHead;
+	DLLIST_NODE				sNode;
+#if !defined(SUPPORT_SHADOW_FREELISTS)
+	/* HWRTData nodes linked to local freelist */
+	DLLIST_NODE				sNodeHWRTDataHead;
+#endif
+
+	/* FW data structures */
+	DEVMEM_MEMDESC			*psFWFreelistMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sFreeListFWDevVAddr;
+};
+
+struct _RGX_PMR_NODE_ {
+	RGX_FREELIST			*psFreeList;
+	PMR						*psPMR;
+	PMR_PAGELIST			*psPageList;
+	DLLIST_NODE				sMemoryBlock;
+	IMG_UINT32				ui32NumPages;
+	IMG_BOOL				bFirstPageMissing;
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+	RI_HANDLE				hRIHandle;
+#endif
+};
+
+typedef struct {
+	PVRSRV_RGXDEV_INFO		*psDevInfo;
+	DEVMEM_MEMDESC			*psFWZSBufferMemDesc;
+	RGXFWIF_DEV_VIRTADDR	sZSBufferFWDevVAddr;
+
+	DEVMEMINT_RESERVATION	*psReservation;
+	PMR						*psPMR;
+	IMG_UINT32				ui32ZSBufferID;
+	IMG_UINT32				ui32RefCount;
+	IMG_BOOL				bOnDemand;
+	IMG_BOOL				bIsBacked;
+
+	IMG_UINT32				ui32NumReqByApp;		/* Number of Backing Requests from Application */
+	IMG_UINT32				ui32NumReqByFW;			/* Number of Backing Requests from Firmware */
+
+	IMG_PID					owner;
+
+	DLLIST_NODE	sNode;
+}RGX_ZSBUFFER_DATA;
+
+typedef struct {
+	RGX_ZSBUFFER_DATA		*psZSBuffer;
+} RGX_POPULATION;
+
+/* Dump the physical pages of a freelist */
+IMG_BOOL RGXDumpFreeListPageList(RGX_FREELIST *psFreeList);
+
+
+/* Create set of HWRTData(s) */
+PVRSRV_ERROR RGXCreateHWRTDataSet(CONNECTION_DATA          *psConnection,
+                                  PVRSRV_DEVICE_NODE       *psDeviceNode,
+                                  IMG_DEV_VIRTADDR          asVHeapTableDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_DEV_VIRTADDR          psPMMListDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  RGX_FREELIST	           *apsFreeLists[RGXMKIF_NUM_RTDATA_FREELISTS],
+                                  IMG_UINT32                ui32ScreenPixelMax,
+                                  IMG_UINT64                ui64MultiSampleCtl,
+                                  IMG_UINT64                ui64FlippedMultiSampleCtl,
+                                  IMG_UINT32                ui32TPCStride,
+                                  IMG_DEV_VIRTADDR          asTailPtrsDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                ui32TPCSize,
+                                  IMG_UINT32                ui32TEScreen,
+                                  IMG_UINT32                ui32TEAA,
+                                  IMG_UINT32                ui32TEMTILE1,
+                                  IMG_UINT32                ui32TEMTILE2,
+                                  IMG_UINT32                ui32MTileStride,
+                                  IMG_UINT32                ui32ISPMergeLowerX,
+                                  IMG_UINT32                ui32ISPMergeLowerY,
+                                  IMG_UINT32                ui32ISPMergeUpperX,
+                                  IMG_UINT32                ui32ISPMergeUpperY,
+                                  IMG_UINT32                ui32ISPMergeScaleX,
+                                  IMG_UINT32                ui32ISPMergeScaleY,
+                                  IMG_DEV_VIRTADDR          sMacrotileArrayDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          sRgnHeaderDevVAddr[RGXMKIF_NUM_RTDATAS],
+                                  IMG_DEV_VIRTADDR          asRTCDevVAddr[RGXMKIF_NUM_GEOMDATAS],
+                                  IMG_UINT32                uiRgnHeaderSize,
+                                  IMG_UINT32                ui32ISPMtileSize,
+                                  IMG_UINT16                ui16MaxRTs,
+                                  RGX_KM_HW_RT_DATASET     *pasKMHWRTDataSet[RGXMKIF_NUM_RTDATAS]);
+
+/* Destroy HWRTDataSet */
+PVRSRV_ERROR RGXDestroyHWRTDataSet(RGX_KM_HW_RT_DATASET *psKMHWRTDataSet);
+
+/*
+	RGXCreateZSBufferKM
+*/
+PVRSRV_ERROR RGXCreateZSBufferKM(CONNECTION_DATA * psConnection,
+                                 PVRSRV_DEVICE_NODE	*psDeviceNode,
+                                 DEVMEMINT_RESERVATION	*psReservation,
+                                 PMR					*psPMR,
+                                 PVRSRV_MEMALLOCFLAGS_T		uiMapFlags,
+                                 RGX_ZSBUFFER_DATA **ppsZSBuffer);
+
+/*
+	RGXDestroyZSBufferKM
+*/
+PVRSRV_ERROR RGXDestroyZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+
+/*
+ * RGXBackingZSBuffer()
+ *
+ * Backs ZS-Buffer with physical pages
+ */
+PVRSRV_ERROR
+RGXBackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXPopulateZSBufferKM()
+ *
+ * Backs ZS-Buffer with physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXPopulateZSBufferKM(RGX_ZSBUFFER_DATA *psZSBuffer,
+								   RGX_POPULATION **ppsPopulation);
+
+/*
+ * RGXUnbackingZSBuffer()
+ *
+ * Frees ZS-Buffer's physical pages
+ */
+PVRSRV_ERROR RGXUnbackingZSBuffer(RGX_ZSBUFFER_DATA *psZSBuffer);
+
+/*
+ * RGXUnpopulateZSBufferKM()
+ *
+ * Frees ZS-Buffer's physical pages (called by Bridge calls)
+ */
+PVRSRV_ERROR RGXUnpopulateZSBufferKM(RGX_POPULATION *psPopulation);
+
+/*
+	RGXProcessRequestZSBufferBacking
+*/
+void RGXProcessRequestZSBufferBacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+									  IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXProcessRequestZSBufferUnbacking
+*/
+void RGXProcessRequestZSBufferUnbacking(PVRSRV_RGXDEV_INFO *psDevInfo,
+										IMG_UINT32 ui32ZSBufferID);
+
+/*
+	RGXGrowFreeList
+*/
+PVRSRV_ERROR RGXGrowFreeList(RGX_FREELIST *psFreeList,
+                             IMG_UINT32 ui32NumPages,
+                             PDLLIST_NODE pListHeader);
+
+/* Create free list */
+PVRSRV_ERROR RGXCreateFreeList(CONNECTION_DATA      *psConnection,
+                               PVRSRV_DEVICE_NODE	*psDeviceNode,
+                               IMG_HANDLE			hMemCtxPrivData,
+                               IMG_UINT32			ui32MaxFLPages,
+                               IMG_UINT32			ui32InitFLPages,
+                               IMG_UINT32			ui32GrowFLPages,
+                               IMG_UINT32           ui32GrowParamThreshold,
+                               RGX_FREELIST			*psGlobalFreeList,
+                               IMG_BOOL				bCheckFreelist,
+                               DEVMEMINT_RESERVATION* psFreeListReservation,
+                               RGX_FREELIST			**ppsFreeList);
+
+/* Destroy free list */
+PVRSRV_ERROR RGXDestroyFreeList(RGX_FREELIST *psFreeList);
+
+/*
+	RGXProcessRequestGrow
+*/
+void RGXProcessRequestGrow(PVRSRV_RGXDEV_INFO *psDevInfo,
+						   IMG_UINT32 ui32FreelistID);
+
+
+/* Reconstruct free list after Hardware Recovery */
+void RGXProcessRequestFreelistsReconstruction(PVRSRV_RGXDEV_INFO *psDevInfo,
+											  IMG_UINT32 ui32FreelistsCount,
+											  const IMG_UINT32 *paui32Freelists);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateRenderContextKM
+
+ @Description
+	Server-side implementation of RGXCreateRenderContext
+
+ @Input psConnection -
+ @Input psDeviceNode - device node
+ @Input i32Priority - context priority
+ @Input sVDMCallStackAddr - VDM call stack device virtual address
+ @Input ui32CallStackDepth - VDM call stack depth
+ @Input ui32FrameworkCommandSize - framework command size
+ @Input pabyFrameworkCommand - ptr to framework command
+ @Input hMemCtxPrivData - memory context private data
+ @Input ui32StaticRenderContextStateSize - size of fixed render state
+ @Input pStaticRenderContextState - ptr to fixed render state buffer
+ @Input ui32PackedCCBSizeU8888 :
+		ui8TACCBAllocSizeLog2 - TA CCB size
+		ui8TACCBMaxAllocSizeLog2 - maximum size to which TA CCB can grow
+		ui83DCCBAllocSizeLog2 - 3D CCB size
+		ui83DCCBMaxAllocSizeLog2 - maximum size to which 3D CCB can grow
+ @Input ui32ContextFlags - flags which specify properties of the context
+ @Output ppsRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateRenderContextKM(CONNECTION_DATA				*psConnection,
+											PVRSRV_DEVICE_NODE			*psDeviceNode,
+											IMG_INT32					i32Priority,
+											IMG_DEV_VIRTADDR			sVDMCallStackAddr,
+											IMG_UINT32					ui32CallStackDepth,
+											IMG_UINT32					ui32FrameworkCommandSize,
+											IMG_PBYTE					pabyFrameworkCommand,
+											IMG_HANDLE					hMemCtxPrivData,
+											IMG_UINT32					ui32StaticRenderContextStateSize,
+											IMG_PBYTE					pStaticRenderContextState,
+											IMG_UINT32					ui32PackedCCBSizeU8888,
+											IMG_UINT32					ui32ContextFlags,
+											IMG_UINT64					ui64RobustnessAddress,
+											IMG_UINT32					ui32MaxTADeadlineMS,
+											IMG_UINT32					ui32Max3DDeadlineMS,
+											RGX_SERVER_RENDER_CONTEXT	**ppsRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyRenderContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyRenderContext
+
+ @Input psRenderContext -
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyRenderContextKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXKickTA3DKM
+
+ @Description
+	Server-side implementation of RGXKickTA3D
+
+ @Input psRTDataCleanup - RT data associated with the kick (or NULL)
+ @Input psZBuffer - Z-buffer associated with the kick (or NULL)
+ @Input psSBuffer - S-buffer associated with the kick (or NULL)
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXKickTA3DKM(RGX_SERVER_RENDER_CONTEXT	*psRenderContext,
+								 IMG_UINT32					ui32ClientTAFenceCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientTAFenceSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientTAFenceSyncOffset,
+								 IMG_UINT32					*paui32ClientTAFenceValue,
+								 IMG_UINT32					ui32ClientTAUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClientUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32ClientUpdateSyncOffset,
+								 IMG_UINT32					*paui32ClientTAUpdateValue,
+								 IMG_UINT32					ui32Client3DUpdateCount,
+								 SYNC_PRIMITIVE_BLOCK		**apsClient3DUpdateSyncPrimBlock,
+								 IMG_UINT32					*paui32Client3DUpdateSyncOffset,
+								 IMG_UINT32					*paui32Client3DUpdateValue,
+								 SYNC_PRIMITIVE_BLOCK		*psPRSyncPrimBlock,
+								 IMG_UINT32					ui32PRSyncOffset,
+								 IMG_UINT32					ui32PRFenceValue,
+								 PVRSRV_FENCE				iCheckFence,
+								 PVRSRV_TIMELINE			iUpdateTimeline,
+								 PVRSRV_FENCE				*piUpdateFence,
+								 IMG_CHAR					szFenceName[PVRSRV_SYNC_NAME_LENGTH],
+								 PVRSRV_FENCE				iCheckFence3D,
+								 PVRSRV_TIMELINE			iUpdateTimeline3D,
+								 PVRSRV_FENCE				*piUpdateFence3D,
+								 IMG_CHAR					szFenceName3D[PVRSRV_SYNC_NAME_LENGTH],
+								 IMG_UINT32					ui32TACmdSize,
+								 IMG_PBYTE					pui8TADMCmd,
+								 IMG_UINT32					ui323DPRCmdSize,
+								 IMG_PBYTE					pui83DPRDMCmd,
+								 IMG_UINT32					ui323DCmdSize,
+								 IMG_PBYTE					pui83DDMCmd,
+								 IMG_UINT32					ui32ExtJobRef,
+								 IMG_BOOL					bKickTA,
+								 IMG_BOOL					bKickPR,
+								 IMG_BOOL					bKick3D,
+								 IMG_BOOL					bAbort,
+								 IMG_UINT32					ui32PDumpFlags,
+								 RGX_KM_HW_RT_DATASET		*psKMHWRTDataSet,
+								 RGX_ZSBUFFER_DATA			*psZSBuffer,
+								 RGX_ZSBUFFER_DATA			*psMSAAScratchBuffer,
+								 IMG_UINT32					ui32SyncPMRCount,
+								 IMG_UINT32					*paui32SyncPMRFlags,
+								 PMR						**ppsSyncPMRs,
+								 IMG_UINT32					ui32RenderTargetSize,
+								 IMG_UINT32					ui32NumberOfDrawCalls,
+								 IMG_UINT32					ui32NumberOfIndices,
+								 IMG_UINT32					ui32NumberOfMRTs,
+								 IMG_UINT64					ui64DeadlineInus);
+
+
+PVRSRV_ERROR PVRSRVRGXSendZSStoreDisableKM(CONNECTION_DATA *psConnection,
+									PVRSRV_DEVICE_NODE * psDeviceNode,
+									RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+									IMG_BOOL    bDisableDepthStore,
+									IMG_BOOL    bDisableStencilStore,
+									IMG_UINT32  ui32ExtJobRefToDisableZSStore);
+
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                 PVRSRV_DEVICE_NODE * psDevNode,
+                                                 RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+                                                 IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetRenderContextPropertyKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext,
+												 RGX_CONTEXT_PROPERTY eContextProperty,
+												 IMG_UINT64 ui64Input,
+												 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of render contexts on this device */
+void DumpRenderCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                         void *pvDumpDebugFile,
+                         IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client contexts are stalled */
+IMG_UINT32 CheckForStalledClientRenderCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR RGXRenderContextStalledKM(RGX_SERVER_RENDER_CONTEXT *psRenderContext);
+
+#endif /* RGXTA3D_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxtdmtransfer.c b/drivers/gpu/drm/img-rogue/rgxtdmtransfer.c
new file mode 100644
index 000000000000..ff33e864d971
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtdmtransfer.c
@@ -0,0 +1,1486 @@
+/*************************************************************************/ /*!
+@File           rgxtdmtransfer.c
+@Title          Device specific TDM transfer queue routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxtdmtransfer.h"
+#include "rgx_tq_shared.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_memallocflags.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "rgxshader.h"
+
+#include "pdump_km.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+#include "rgxworkest.h"
+#endif
+
+#include "rgxtimerquery.h"
+
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TDM_UFO_DUMP	0
+
+//#define TDM_CHECKPOINT_DEBUG 1
+
+#if defined(TDM_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+typedef struct {
+	RGX_SERVER_COMMON_CONTEXT * psServerCommonContext;
+	IMG_INT32                   i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_TDM_DATA;
+
+
+struct _RGX_SERVER_TQ_TDM_CONTEXT_ {
+	PVRSRV_DEVICE_NODE      *psDeviceNode;
+	DEVMEM_MEMDESC          *psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC          *psFWTransferContextMemDesc;
+	IMG_UINT32              ui32Flags;
+	RGX_SERVER_TQ_TDM_DATA  sTDMData;
+	DLLIST_NODE             sListNode;
+	SYNC_ADDR_LIST          sSyncAddrListFence;
+	SYNC_ADDR_LIST          sSyncAddrListUpdate;
+	POS_LOCK		hLock;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	WORKEST_HOST_DATA       sWorkEstData;
+#endif
+};
+
+static PVRSRV_ERROR _CreateTDMTransferContext(
+	CONNECTION_DATA         * psConnection,
+	PVRSRV_DEVICE_NODE      * psDeviceNode,
+	DEVMEM_MEMDESC          * psAllocatedMemDesc,
+	IMG_UINT32                ui32AllocatedOffset,
+	SERVER_MMU_CONTEXT      * psServerMMUContext,
+	DEVMEM_MEMDESC          * psFWMemContextMemDesc,
+	IMG_INT32                 i32Priority,
+	IMG_INT32                 ui32MaxDeadlineMS,
+	RGX_COMMON_CONTEXT_INFO * psInfo,
+	RGX_SERVER_TQ_TDM_DATA  * psTDMData,
+	IMG_UINT32                ui32CCBAllocSizeLog2,
+	IMG_UINT32                ui32CCBMaxAllocSizeLog2,
+	IMG_UINT32                ui32ContextFlags,
+	IMG_UINT64                ui64RobustnessAddress)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	psTDMData->psBufferSyncContext =
+			pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+			                               "rogue-tdm");
+	if (IS_ERR(psTDMData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: failed to create buffer_sync context (err=%ld)",
+				__func__, PTR_ERR(psTDMData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = FWCommonContextAllocate(
+			psConnection,
+			psDeviceNode,
+			REQ_TYPE_TQ_TDM,
+			RGXFWIF_DM_TDM,
+			psServerMMUContext,
+			psAllocatedMemDesc,
+			ui32AllocatedOffset,
+			psFWMemContextMemDesc,
+			NULL,
+			ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TDM_CCB_SIZE_LOG2,
+			ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TDM_CCB_MAX_SIZE_LOG2,
+			ui32ContextFlags,
+			i32Priority,
+			ui32MaxDeadlineMS,
+			ui64RobustnessAddress,
+			psInfo,
+			&psTDMData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	psTDMData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(psTDMData->psBufferSyncContext);
+	psTDMData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+static PVRSRV_ERROR _DestroyTDMTransferContext(
+	RGX_SERVER_TQ_TDM_DATA  * psTDMData,
+	PVRSRV_DEVICE_NODE      * psDeviceNode)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(
+		psDeviceNode,
+		psTDMData->psServerCommonContext,
+		RGXFWIF_DM_TDM,
+		PDUMP_FLAGS_CONTINUOUS);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free it's resources */
+	FWCommonContextFree(psTDMData->psServerCommonContext);
+	psTDMData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(psTDMData->psBufferSyncContext);
+	psTDMData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+
+/*
+ * PVRSRVCreateTransferContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXTDMCreateTransferContextKM(
+	CONNECTION_DATA            * psConnection,
+	PVRSRV_DEVICE_NODE         * psDeviceNode,
+	IMG_INT32                    i32Priority,
+	IMG_UINT32                   ui32FrameworkCommandSize,
+	IMG_PBYTE                    pabyFrameworkCommand,
+	IMG_HANDLE                   hMemCtxPrivData,
+	IMG_UINT32                   ui32PackedCCBSizeU88,
+	IMG_UINT32                   ui32ContextFlags,
+	IMG_UINT64                   ui64RobustnessAddress,
+	IMG_UINT32                   ui32MaxDeadlineMS,
+	RGX_SERVER_TQ_TDM_CONTEXT ** ppsTransferContext)
+{
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext;
+
+	DEVMEM_MEMDESC          * psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	PVRSRV_RGXDEV_INFO      * psDevInfo = psDeviceNode->pvDevice;
+	RGX_COMMON_CONTEXT_INFO   sInfo = {NULL};
+	PVRSRV_ERROR              eError = PVRSRV_OK;
+
+	/* Allocate the server side structure */
+	*ppsTransferContext = NULL;
+	psTransferContext = OSAllocZMem(sizeof(*psTransferContext));
+	if (psTransferContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW transfer context, this has the TDM common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWTDMCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwTransferContext",
+			&psTransferContext->psFWTransferContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwtransfercontext;
+	}
+
+	eError = OSLockCreate(&psTransferContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+		goto fail_lockcreate;
+	}
+
+	psTransferContext->psDeviceNode = psDeviceNode;
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psTransferContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to allocate firmware GPU framework state (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psTransferContext->psFWFrameworkMemDesc,
+				pabyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s: Failed to populate the framework buffer (%s)",
+					__func__,
+					PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psTransferContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _CreateTDMTransferContext(psConnection,
+	                                   psDeviceNode,
+	                                   psTransferContext->psFWTransferContextMemDesc,
+	                                   offsetof(RGXFWIF_FWTDMCONTEXT, sTDMContext),
+	                                   hMemCtxPrivData,
+	                                   psFWMemContextMemDesc,
+	                                   i32Priority,
+	                                   ui32MaxDeadlineMS,
+	                                   &sInfo,
+	                                   &psTransferContext->sTDMData,
+									   U32toU8_Unpack1(ui32PackedCCBSizeU88),
+									   U32toU8_Unpack2(ui32PackedCCBSizeU88),
+	                                   ui32ContextFlags,
+	                                   ui64RobustnessAddress);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_tdmtransfercontext;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstInitTDM(psDevInfo, &psTransferContext->sWorkEstData);
+	}
+#endif
+
+	SyncAddrListInit(&psTransferContext->sSyncAddrListFence);
+	SyncAddrListInit(&psTransferContext->sSyncAddrListUpdate);
+
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTDMCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+	*ppsTransferContext = psTransferContext;
+
+	return PVRSRV_OK;
+
+fail_tdmtransfercontext:
+fail_frameworkcopy:
+	if (psTransferContext->psFWFrameworkMemDesc != NULL)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	OSLockDestroy(psTransferContext->hLock);
+fail_lockcreate:
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+fail_fwtransfercontext:
+	OSFreeMem(psTransferContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*ppsTransferContext = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMDestroyTransferContextKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		RGXFWIF_FWTDMCONTEXT	*psFWTransferContext;
+		IMG_UINT32 ui32WorkEstCCBSubmitted;
+
+		eError = DevmemAcquireCpuVirtAddr(psTransferContext->psFWTransferContextMemDesc,
+										  (void **)&psFWTransferContext);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: Failed to map firmware transfer context (%s)",
+					 __func__,
+					 PVRSRVGetErrorString(eError)));
+			return eError;
+		}
+
+		RGXFwSharedMemCacheOpValue(psFWTransferContext->ui32WorkEstCCBSubmitted, INVALIDATE);
+		ui32WorkEstCCBSubmitted = psFWTransferContext->ui32WorkEstCCBSubmitted;
+
+		DevmemReleaseCpuVirtAddr(psTransferContext->psFWTransferContextMemDesc);
+
+		/* Check if all of the workload estimation CCB commands for this workload are read */
+		if (ui32WorkEstCCBSubmitted != psTransferContext->sWorkEstData.ui32WorkEstCCBReceived)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: WorkEst # cmds submitted (%u) and received (%u) mismatch",
+					 __func__, ui32WorkEstCCBSubmitted,
+					 psTransferContext->sWorkEstData.ui32WorkEstCCBReceived));
+
+			return PVRSRV_ERROR_RETRY;
+		}
+	}
+#endif
+
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_remove_node(&(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+
+
+	eError = _DestroyTDMTransferContext(&psTransferContext->sTDMData,
+	                                    psTransferContext->psDeviceNode);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_destroyTDM;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		WorkEstDeInitTDM(psDevInfo, &psTransferContext->sWorkEstData);
+	}
+#endif
+
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+
+	SyncAddrListDeinit(&psTransferContext->sSyncAddrListFence);
+	SyncAddrListDeinit(&psTransferContext->sSyncAddrListUpdate);
+
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+
+	OSLockDestroy(psTransferContext->hLock);
+
+	OSFreeMem(psTransferContext);
+
+	return PVRSRV_OK;
+
+fail_destroyTDM:
+
+	OSWRLockAcquireWrite(psDevInfo->hTDMCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTDMCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTDMCtxListLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+/*
+ * PVRSRVSubmitTQ3DKickKM
+ */
+/* Old bridge call for backwards compatibility. */
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransferKM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * paui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32                  ui32TDMCharacteristic1,
+	IMG_UINT32                  ui32TDMCharacteristic2,
+	IMG_UINT64                  ui64DeadlineInus)
+{
+	return PVRSRVRGXTDMSubmitTransfer3KM(psTransferContext,
+										 ui32PDumpFlags,
+										 ui32ClientUpdateCount,
+										 pauiClientUpdateUFODevVarBlock,
+										 paui32ClientUpdateSyncOffset,
+										 paui32ClientUpdateValue,
+										 iCheckFence,
+										 iUpdateTimeline,
+										 piUpdateFence,
+										 szUpdateFenceName,
+										 PVRSRV_NO_FENCE,
+										 ui32FWCommandSize,
+										 pui8FWCommand,
+										 ui32ExtJobRef,
+										 ui32SyncPMRCount,
+										 paui32SyncPMRFlags,
+										 ppsSyncPMRs,
+										 ui32TDMCharacteristic1,
+										 ui32TDMCharacteristic2,
+										 ui64DeadlineInus);
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransfer3KM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	PVRSRV_FENCE                iExportFenceToSignal,
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * paui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32                  ui32TDMCharacteristic1,
+	IMG_UINT32                  ui32TDMCharacteristic2,
+	IMG_UINT64                  ui64DeadlineInus)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	RGX_CCB_CMD_HELPER_DATA *psCmdHelper;
+	PRGXFWIF_UFO_ADDR * pauiIntFenceUFOAddress   = NULL;
+	PRGXFWIF_UFO_ADDR * pauiIntUpdateUFOAddress  = NULL;
+	IMG_UINT32          ui32IntClientFenceCount  = 0;
+	IMG_UINT32        * paui32IntUpdateValue     = paui32ClientUpdateValue;
+	IMG_UINT32          ui32IntClientUpdateCount = ui32ClientUpdateCount;
+	PVRSRV_ERROR eError;
+	PVRSRV_FENCE iUpdateFence = PVRSRV_NO_FENCE;
+	PVRSRV_RGXDEV_INFO  *psDevInfo = FWCommonContextGetRGXDevInfo(psTransferContext->sTDMData.psServerCommonContext);
+	RGX_CLIENT_CCB      *psClientCCB = FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext);
+	IMG_UINT32          ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+
+	IMG_UINT64          ui64FBSCEntryMask = 0;
+
+	IMG_BOOL bCCBStateOpen;
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+	IMG_UINT64               uiCheckFenceUID = 0;
+	IMG_UINT64               uiUpdateFenceUID = 0;
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	IMG_UINT32 ui32CmdOffset = 0;
+	RGXFWIF_WORKEST_KICK_DATA sWorkloadKickDataTransfer = {0};
+	IMG_UINT32 ui32TDMWorkloadDataRO = 0;
+	IMG_UINT32 ui32TDMCmdHeaderOffset = 0;
+	IMG_UINT32 ui32TDMCmdOffsetWrapCheck = 0;
+	RGX_WORKLOAD sWorkloadCharacteristics = {0};
+#endif
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+#endif
+
+	PSYNC_CHECKPOINT psUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT psExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui32IntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *psFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui32FenceTimelineUpdateValue = 0;
+	void *pvUpdateFenceFinaliseData = NULL;
+
+	if (iUpdateTimeline >= 0 && !piUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(SUPPORT_WORKLOAD_ESTIMATION)
+	PVR_UNREFERENCED_PARAMETER(ui32TDMCharacteristic1);
+	PVR_UNREFERENCED_PARAMETER(ui32TDMCharacteristic2);
+	PVR_UNREFERENCED_PARAMETER(ui64DeadlineInus);
+#endif
+
+	/* Ensure we haven't been given a null ptr to
+	 * update values if we have been told we
+	 * have updates
+	 */
+	if (ui32ClientUpdateCount > 0)
+	{
+		PVR_LOG_RETURN_IF_FALSE(paui32ClientUpdateValue != NULL,
+		                        "paui32ClientUpdateValue NULL but "
+		                        "ui32ClientUpdateCount > 0",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	if (ui32SyncPMRCount != 0)
+	{
+		if (!ppsSyncPMRs)
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* We can't allocate the required amount of stack space on all consumer architectures */
+	psCmdHelper = OSAllocMem(sizeof(RGX_CCB_CMD_HELPER_DATA));
+	if (psCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_allochelper;
+	}
+
+
+	/*
+		Init the command helper commands for all the prepares
+	*/
+	{
+		IMG_CHAR *pszCommandName;
+		RGXFWIF_CCB_CMD_TYPE eType;
+#if defined(SUPPORT_BUFFER_SYNC)
+		struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+
+		pszCommandName = "TQ-TDM";
+
+		if (ui32FWCommandSize == 0)
+		{
+			/* A NULL CMD for TDM is used to append updates to a non finished
+			 * FW command. bCCBStateOpen is used in case capture range is
+			 * entered on this command, to not drain CCB up to the Roff for this
+			 * command, but the finished command prior to this.
+			 */
+			bCCBStateOpen = IMG_TRUE;
+			eType = RGXFWIF_CCB_CMD_TYPE_NULL;
+		}
+		else
+		{
+			bCCBStateOpen = IMG_FALSE;
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_TDM;
+		}
+
+#if defined(SUPPORT_BUFFER_SYNC)
+		psBufferSyncContext = psTransferContext->sTDMData.psBufferSyncContext;
+#endif
+
+		eError = SyncAddrListPopulate(&psTransferContext->sSyncAddrListFence,
+		                              0,
+		                              NULL,
+		                              NULL);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_populate_sync_addr_list;
+		}
+
+		eError = SyncAddrListPopulate(&psTransferContext->sSyncAddrListUpdate,
+		                              ui32ClientUpdateCount,
+		                              pauiClientUpdateUFODevVarBlock,
+		                              paui32ClientUpdateSyncOffset);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_populate_sync_addr_list;
+		}
+		paui32IntUpdateValue    = paui32ClientUpdateValue;
+		pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+
+
+		if (ui32SyncPMRCount)
+		{
+#if defined(SUPPORT_BUFFER_SYNC)
+			int err;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling pvr_buffer_sync_resolve_and_create_fences", __func__));
+			err = pvr_buffer_sync_resolve_and_create_fences(psBufferSyncContext,
+			                                                psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                                ui32SyncPMRCount,
+			                                                ppsSyncPMRs,
+			                                                paui32SyncPMRFlags,
+			                                                &ui32BufferFenceSyncCheckpointCount,
+			                                                &apsBufferFenceSyncCheckpoints,
+			                                                &psBufferUpdateSyncCheckpoint,
+			                                                &psBufferSyncData);
+			if (err)
+			{
+				switch (err)
+				{
+					case -EINTR:
+						eError = PVRSRV_ERROR_RETRY;
+						break;
+					case -ENOMEM:
+						eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+						break;
+					default:
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						break;
+				}
+
+				if (eError != PVRSRV_ERROR_RETRY)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   pvr_buffer_sync_resolve_and_create_fences failed (%s)", __func__, PVRSRVGetErrorString(eError)));
+				}
+				goto fail_resolve_input_fence;
+			}
+
+			/* Append buffer sync fences */
+			if (ui32BufferFenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints to TQ Fence (&psTransferContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>)...", __func__, ui32BufferFenceSyncCheckpointCount, (void*)&psTransferContext->sSyncAddrListFence , (void*)pauiIntFenceUFOAddress));
+				SyncAddrListAppendAndDeRefCheckpoints(&psTransferContext->sSyncAddrListFence,
+				                                      ui32BufferFenceSyncCheckpointCount,
+				                                      apsBufferFenceSyncCheckpoints);
+				if (!pauiIntFenceUFOAddress)
+				{
+					pauiIntFenceUFOAddress = psTransferContext->sSyncAddrListFence.pasFWAddrs;
+				}
+				ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+			}
+
+			if (psBufferUpdateSyncCheckpoint)
+			{
+				/* Append the update (from output fence) */
+				SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+				                              1,
+				                              &psBufferUpdateSyncCheckpoint);
+				if (!pauiIntUpdateUFOAddress)
+				{
+					pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+				}
+				ui32IntClientUpdateCount++;
+			}
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+			PVR_UNREFERENCED_PARAMETER(paui32SyncPMRFlags);
+			PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers", __func__, ui32SyncPMRCount));
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto fail_populate_sync_addr_list;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+		}
+
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(psTransferContext->psDeviceNode->hSyncCheckpointContext,
+		                                    iCheckFence,
+		                                    &ui32FenceSyncCheckpointCount,
+		                                    &apsFenceSyncCheckpoints,
+		                                    &uiCheckFenceUID,
+		                                    ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_resolve_input_fence;
+		}
+#if defined(TDM_CHECKPOINT_DEBUG)
+		{
+			IMG_UINT32 ii;
+			for (ii=0; ii<32; ii++)
+			{
+				PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints +  ii);
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint)); //psFenceSyncCheckpoints[ii]));
+			}
+		}
+#endif
+		/* Create the output fence (if required) */
+		if (iUpdateTimeline != PVRSRV_NO_TIMELINE)
+		{
+			eError = SyncCheckpointCreateFence(psTransferContext->psDeviceNode,
+			                                   szUpdateFenceName,
+			                                   iUpdateTimeline,
+			                                   psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                   &iUpdateFence,
+			                                   &uiUpdateFenceUID,
+			                                   &pvUpdateFenceFinaliseData,
+			                                   &psUpdateSyncCheckpoint,
+			                                   (void*)&psFenceTimelineUpdateSync,
+			                                   &ui32FenceTimelineUpdateValue,
+			                                   ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				goto fail_create_output_fence;
+			}
+
+			/* Resolve the iExportFenceToSignal (if required) */
+			if (iExportFenceToSignal != PVRSRV_NO_FENCE)
+			{
+				IMG_UINT32 iii;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+				eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+														  psTransferContext->psDeviceNode->hSyncCheckpointContext,
+														  &psExportFenceSyncCheckpoint,
+														  ui32PDumpFlags);
+				if (eError != PVRSRV_OK)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), psExportFenceSyncCheckpoint));
+					goto fail_resolve_export_fence;
+				}
+
+				/* Check that the export fence was not also included as part of the
+				 * check fence (which is an error and would lead to a stalled kick).
+				 */
+				for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+				{
+					if (apsFenceSyncCheckpoints[iii] == psExportFenceSyncCheckpoint)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, psExportFenceSyncCheckpoint));
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+						goto fail_check_fence_includes_export_fence;
+					}
+				}
+			}
+
+			/* Append the sync prim update for the timeline (if required) */
+			if (psFenceTimelineUpdateSync)
+			{
+				IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+				/* Allocate memory to hold the list of update values (including our timeline update) */
+				pui32IntAllocatedUpdateValues = OSAllocMem(sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				if (!pui32IntAllocatedUpdateValues)
+				{
+					/* Failed to allocate memory */
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto fail_alloc_update_values_mem;
+				}
+				OSCachedMemSet(pui32IntAllocatedUpdateValues, 0xbb, sizeof(*pui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				/* Copy the update values into the new memory, then append our timeline update value */
+				if (paui32IntUpdateValue)
+				{
+					OSCachedMemCopy(pui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(*pui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+				}
+				/* Now set the additional update value */
+				pui32TimelineUpdateWp = pui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+				*pui32TimelineUpdateWp = ui32FenceTimelineUpdateValue;
+				ui32IntClientUpdateCount++;
+#if defined(TDM_CHECKPOINT_DEBUG)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Now append the timeline sync prim addr to the transfer context update list */
+				SyncAddrListAppendSyncPrim(&psTransferContext->sSyncAddrListUpdate,
+				                           psFenceTimelineUpdateSync);
+#if defined(TDM_CHECKPOINT_DEBUG)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+				paui32IntUpdateValue = pui32IntAllocatedUpdateValues;
+			}
+		}
+
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			/* Append the checks (from input fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to TQ Fence (&psTransferContext->sSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)&psTransferContext->sSyncAddrListFence));
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntFenceUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListFence,
+										  ui32FenceSyncCheckpointCount,
+										  apsFenceSyncCheckpoints);
+			if (!pauiIntFenceUFOAddress)
+			{
+				pauiIntFenceUFOAddress = psTransferContext->sSyncAddrListFence.pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+		if (psUpdateSyncCheckpoint)
+		{
+			/* Append the update (from output fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to TQ Update (&psTransferContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)&psTransferContext->sSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+			                              1,
+			                              &psUpdateSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(TDM_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pui32IntAllocatedUpdateValues;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		if (psExportFenceSyncCheckpoint)
+		{
+			/* Append the update (from export fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Transfer CDM Update (&psTransferContext->sSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)&psTransferContext->sSyncAddrListUpdate , (void*)psExportFenceSyncCheckpoint));
+			SyncAddrListAppendCheckpoints(&psTransferContext->sSyncAddrListUpdate,
+										  1,
+										  &psExportFenceSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psTransferContext->sSyncAddrListUpdate.pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+#if (ENABLE_TDM_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping TDM fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TDM fence syncs (&psTransferContext->sSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psTransferContext->sSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TDM update syncs (&psTransferContext->sSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psTransferContext->sSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+		RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psTransferContext->psDeviceNode->pvDevice,
+		                          &pPreAddr,
+		                          &pPostAddr,
+		                          &pRMWUFOAddr);
+
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+		/*
+		 * Extract the FBSC entries from MMU Context for the deferred FBSC invalidate command,
+		 * in other words, take the value and set it to zero afterwards.
+		 * FBSC Entry Mask must be extracted from MMU ctx and updated just before the kick starts
+		 * as it must be ready at the time of context activation.
+		 */
+		{
+			eError = RGXExtractFBSCEntryMaskFromMMUContext(psTransferContext->psDeviceNode,
+														   FWCommonContextGetServerMMUCtx(psTransferContext->sTDMData.psServerCommonContext),
+														   &ui64FBSCEntryMask);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "Failed to extract FBSC Entry Mask (%d)", eError));
+				goto fail_invalfbsc;
+			}
+		}
+#endif
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			sWorkloadCharacteristics.sTransfer.ui32Characteristic1 = ui32TDMCharacteristic1;
+			sWorkloadCharacteristics.sTransfer.ui32Characteristic2 = ui32TDMCharacteristic2;
+
+			/* Prepare workload estimation */
+			WorkEstPrepare(psDeviceNode->pvDevice,
+					&psTransferContext->sWorkEstData,
+					&psTransferContext->sWorkEstData.uWorkloadMatchingData.sTransfer.sDataTDM,
+					eType,
+					&sWorkloadCharacteristics,
+					ui64DeadlineInus,
+					&sWorkloadKickDataTransfer);
+		}
+#endif
+
+		/*
+			Create the command helper data for this command
+		*/
+		RGXCmdHelperInitCmdCCB(psDevInfo,
+		                       psClientCCB,
+		                       ui64FBSCEntryMask,
+		                       ui32IntClientFenceCount,
+		                       pauiIntFenceUFOAddress,
+		                       NULL,
+		                       ui32IntClientUpdateCount,
+		                       pauiIntUpdateUFOAddress,
+		                       paui32IntUpdateValue,
+		                       ui32FWCommandSize,
+		                       pui8FWCommand,
+		                       &pPreAddr,
+		                       &pPostAddr,
+		                       &pRMWUFOAddr,
+		                       eType,
+		                       ui32ExtJobRef,
+		                       ui32IntJobRef,
+		                       ui32PDumpFlags,
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		                       &sWorkloadKickDataTransfer,
+#else /* SUPPORT_WORKLOAD_ESTIMATION */
+		                       NULL,
+#endif /* SUPPORT_WORKLOAD_ESTIMATION */
+		                       pszCommandName,
+		                       bCCBStateOpen,
+		                       psCmdHelper);
+
+	}
+
+	/*
+		Acquire space for all the commands in one go
+	*/
+
+	eError = RGXCmdHelperAcquireCmdCCB(1, psCmdHelper);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dcmdacquire;
+	}
+
+
+	/*
+		We should acquire the kernel CCB(s) space here as the schedule could fail
+		and we would have to roll back all the syncs
+	*/
+
+	/*
+		Only do the command helper release (which takes the server sync
+		operations if the acquire succeeded
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		ui32CmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext));
+	}
+#endif
+	RGXCmdHelperReleaseCmdCCB(1,
+	                          psCmdHelper,
+	                          "TQ_TDM",
+	                          FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext).ui32Addr);
+
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		/* The following is used to determine the offset of the command header containing
+		   the workload estimation data so that can be accessed when the KCCB is read */
+		ui32TDMCmdHeaderOffset = RGXCmdHelperGetDMCommandHeaderOffset(psCmdHelper);
+
+		ui32TDMCmdOffsetWrapCheck = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->sTDMData.psServerCommonContext));
+
+		/* This checks if the command would wrap around at the end of the CCB and
+		 * therefore would start at an offset of 0 rather than the current command
+		 * offset */
+		if (ui32CmdOffset < ui32TDMCmdOffsetWrapCheck)
+		{
+			ui32TDMWorkloadDataRO = ui32CmdOffset;
+		}
+		else
+		{
+			ui32TDMWorkloadDataRO = 0;
+		}
+	}
+#endif
+
+	/*
+		Even if we failed to acquire the client CCB space we might still need
+		to kick the HW to process a padding packet to release space for us next
+		time round
+	*/
+	{
+		RGXFWIF_KCCB_CMD sTDMKCCBCmd;
+		IMG_UINT32 ui32FWAddr = FWCommonContextGetFWAddress(
+		        psTransferContext->sTDMData.psServerCommonContext).ui32Addr;
+
+		/* Construct the kernel 3D CCB command. */
+		sTDMKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		sTDMKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(psClientCCB);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(psClientCCB);
+		sTDMKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+		/* Add the Workload data into the KCCB kick */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+		{
+			/* Store the offset to the CCCB command header so that it can be referenced
+			 * when the KCCB command reaches the FW */
+			sTDMKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = ui32TDMWorkloadDataRO + ui32TDMCmdHeaderOffset;
+		}
+#endif
+
+		/* HTBLOGK(HTB_SF_MAIN_KICK_TDM, */
+		/*		s3DKCCBCmd.uCmdData.sCmdKickData.psContext, */
+		/*		ui323DCmdOffset); */
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext).ui32Addr,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQTDM,
+		                  iCheckFence,
+		                  iUpdateFence,
+		                  iUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  uiUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError = RGXScheduleCommandWithoutPowerLock(psDeviceNode->pvDevice,
+			                             RGXFWIF_DM_TDM,
+			                             & sTDMKCCBCmd,
+			                             ui32PDumpFlags);
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXTDMSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError));
+			goto fail_2dcmdacquire;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWAddr, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQTDM);
+	}
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+	if (psUpdateSyncCheckpoint)
+	{
+		SyncCheckpointSignalNoHW(psUpdateSyncCheckpoint);
+	}
+	if (psFenceTimelineUpdateSync)
+	{
+		SyncPrimNoHwUpdate(psFenceTimelineUpdateSync, ui32FenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointSignalNoHW(psExportFenceSyncCheckpoint);
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	* piUpdateFence = iUpdateFence;
+	if (pvUpdateFenceFinaliseData && (iUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, iUpdateFence, pvUpdateFenceFinaliseData,
+		                            psUpdateSyncCheckpoint, szUpdateFenceName);
+	}
+
+	OSFreeMem(psCmdHelper);
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+/*
+	No resources are created in this function so there is nothing to free
+	unless we had to merge syncs.
+	If we fail after the client CCB acquire there is still nothing to do
+	as only the client CCB release will modify the client CCB
+*/
+fail_2dcmdacquire:
+fail_acquirepowerlock:
+fail_3dcmdacquire:
+#if defined(RGX_FBSC_INVALIDATE_COMMAND_SUPPORTED)
+fail_invalfbsc:
+#endif
+	SyncAddrListRollbackCheckpoints(psTransferContext->psDeviceNode, &psTransferContext->sSyncAddrListFence);
+	SyncAddrListRollbackCheckpoints(psTransferContext->psDeviceNode, &psTransferContext->sSyncAddrListUpdate);
+
+	/* Free memory allocated to hold the internal list of update values */
+	if (pui32IntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui32IntAllocatedUpdateValues);
+		pui32IntAllocatedUpdateValues = NULL;
+	}
+fail_alloc_update_values_mem:
+	if (psExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+fail_check_fence_includes_export_fence:
+fail_resolve_export_fence:
+	if (iUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(iUpdateFence, pvUpdateFenceFinaliseData);
+	}
+fail_create_output_fence:
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+	                             apsFenceSyncCheckpoints);
+
+fail_resolve_input_fence:
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+fail_populate_sync_addr_list:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	OSFreeMem(psCmdHelper);
+fail_allochelper:
+
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	OSLockRelease(psTransferContext->hLock);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+	IMG_UINT32                 ui32PDumpFlags)
+{
+	RGXFWIF_KCCB_CMD  sKCCBCmd;
+	PVRSRV_ERROR      eError;
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* Schedule the firmware command */
+	sKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_NOTIFY_WRITE_OFFSET_UPDATE;
+	sKCCBCmd.uCmdData.sWriteOffsetUpdateData.psContext = FWCommonContextGetFWAddress(psTransferContext->sTDMData.psServerCommonContext);
+
+	LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+	{
+		eError = RGXScheduleCommand(psTransferContext->psDeviceNode->pvDevice,
+		                            RGXFWIF_DM_TDM,
+		                            &sKCCBCmd,
+		                            ui32PDumpFlags);
+		if (eError != PVRSRV_ERROR_RETRY)
+		{
+			break;
+		}
+		OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+	} END_LOOP_UNTIL_TIMEOUT_US();
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Failed to schedule the FW command %d (%s)",
+				__func__, eError, PVRSRVGETERRORSTRING(eError)));
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                                                      RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+                                                      IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	if (psTransferContext->sTDMData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psTransferContext->sTDMData.psServerCommonContext,
+		                            psConnection,
+		                            psTransferContext->psDeviceNode->pvDevice,
+		                            i32Priority,
+		                            RGXFWIF_DM_TDM);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority (%s)", __func__, PVRSRVGetErrorString(eError)));
+
+			OSLockRelease(psTransferContext->hLock);
+			return eError;
+		}
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPropertyKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+													  RGX_CONTEXT_PROPERTY eContextProperty,
+													  IMG_UINT64 ui64Input,
+													  IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psTransferContext->hLock);
+			eError = FWCommonContextSetFlags(psTransferContext->sTDMData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			OSLockRelease(psTransferContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpTDMTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                              void *pvDumpDebugFile,
+                              IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hTDMCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTDMCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_TDM_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_TDM_CONTEXT, sListNode);
+
+		DumpFWCommonContextInfo(psCurrentServerTransferCtx->sTDMData.psServerCommonContext,
+		                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTDMCtxListLock);
+}
+
+
+IMG_UINT32 CheckForStalledClientTDMTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hTDMCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTDMCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_TDM_CONTEXT *psCurrentServerTransferCtx =
+				IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_TDM_CONTEXT, sListNode);
+
+		if (CheckStalledClientCommonContext(
+				psCurrentServerTransferCtx->sTDMData.psServerCommonContext, RGX_KICK_TYPE_DM_TDM_2D)
+				== PVRSRV_ERROR_CCCB_STALLED) {
+			ui32ContextBitMask = RGX_KICK_TYPE_DM_TDM_2D;
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTDMCtxListLock);
+	return ui32ContextBitMask;
+}
+
+/**************************************************************************//**
+ End of file (rgxtdmtransfer.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxtdmtransfer.h b/drivers/gpu/drm/img-rogue/rgxtdmtransfer.h
new file mode 100644
index 000000000000..b0f003cf3ad3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtdmtransfer.h
@@ -0,0 +1,154 @@
+/*************************************************************************/ /*!
+@File           rgxtdmtransfer.h
+@Title          RGX Transfer queue 2 Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Transfer queue Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTDMTRANSFER_H)
+#define RGXTDMTRANSFER_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "connection_server.h"
+
+typedef struct _RGX_SERVER_TQ_TDM_CONTEXT_ RGX_SERVER_TQ_TDM_CONTEXT;
+
+
+PVRSRV_ERROR PVRSRVRGXTDMCreateTransferContextKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	IMG_INT32                   i32Priority,
+	IMG_UINT32                  ui32FrameworkCommandSize,
+	IMG_PBYTE                   pabyFrameworkCommand,
+	IMG_HANDLE                  hMemCtxPrivData,
+	IMG_UINT32					ui32PackedCCBSizeU88,
+	IMG_UINT32                  ui32ContextFlags,
+	IMG_UINT64					ui64RobustnessAddress,
+	IMG_UINT32                  ui32MaxDeadlineMS,
+	RGX_SERVER_TQ_TDM_CONTEXT **ppsTransferContext);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMDestroyTransferContextKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext);
+
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransferKM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * pui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32					ui32TDMCharacteristic1,
+	IMG_UINT32					ui32TDMCharacteristic2,
+	IMG_UINT64					ui64DeadlineInus);
+
+PVRSRV_ERROR PVRSRVRGXTDMSubmitTransfer3KM(
+	RGX_SERVER_TQ_TDM_CONTEXT * psTransferContext,
+	IMG_UINT32                  ui32PDumpFlags,
+	IMG_UINT32                  ui32ClientUpdateCount,
+	SYNC_PRIMITIVE_BLOCK     ** pauiClientUpdateUFODevVarBlock,
+	IMG_UINT32                * paui32ClientUpdateSyncOffset,
+	IMG_UINT32                * paui32ClientUpdateValue,
+	PVRSRV_FENCE                iCheckFence,
+	PVRSRV_TIMELINE             iUpdateTimeline,
+	PVRSRV_FENCE              * piUpdateFence,
+	IMG_CHAR                    szUpdateFenceName[PVRSRV_SYNC_NAME_LENGTH],
+	PVRSRV_FENCE                iExportFenceToSignal,
+	IMG_UINT32                  ui32FWCommandSize,
+	IMG_UINT8                 * pui8FWCommand,
+	IMG_UINT32                  ui32ExtJobRef,
+	IMG_UINT32                  ui32SyncPMRCount,
+	IMG_UINT32                * pui32SyncPMRFlags,
+	PMR                      ** ppsSyncPMRs,
+	IMG_UINT32					ui32TDMCharacteristic1,
+	IMG_UINT32					ui32TDMCharacteristic2,
+	IMG_UINT64					ui64DeadlineInus);
+
+PVRSRV_ERROR PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+	IMG_UINT32                 ui32PDumpFlags);
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDeviceNode,
+												   RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+												   IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXTDMSetTransferContextPropertyKM(RGX_SERVER_TQ_TDM_CONTEXT *psTransferContext,
+													 RGX_CONTEXT_PROPERTY eContextProperty,
+													 IMG_UINT64 ui64Input,
+													 IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of TDM transfer contexts on this device */
+void DumpTDMTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                              DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                              void *pvDumpDebugFile,
+                              IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client transfer contexts are stalled */
+IMG_UINT32 CheckForStalledClientTDMTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+
+#endif /* RGXTDMTRANSFER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxtimecorr.c b/drivers/gpu/drm/img-rogue/rgxtimecorr.c
new file mode 100644
index 000000000000..0b00a82afe2b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtimecorr.c
@@ -0,0 +1,789 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific time correlation and calibration routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific time correlation and calibration routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "rgxtimecorr.h"
+#include "rgxfwutils.h"
+#include "htbserver.h"
+#include "pvrsrv_apphint.h"
+#include "rgxpower.h"
+
+/******************************************************************************
+ *
+ * - A calibration period is started on power-on and after a DVFS transition,
+ *   and it's closed before a power-off and before a DVFS transition
+ *   (so power-on -> dfvs -> dvfs -> power-off , power on -> dvfs -> dvfs...,
+ *   where each arrow is a calibration period).
+ *
+ * - The timers on the Host and on the FW are correlated at the beginning of
+ *   each period together with the current GPU frequency.
+ *
+ * - Correlation and calibration are also done at regular intervals using
+ *   a best effort approach.
+ *
+ *****************************************************************************/
+
+/*
+	AppHint interfaces
+*/
+
+static PVRSRV_ERROR _SetClock(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 ui32Value)
+{
+	static __maybe_unused const char* const apszClocks[] = {
+		"mono", "mono_raw", "sched"
+	};
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	if (ui32Value >= RGXTIMECORR_CLOCK_LAST)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Invalid clock source type (%u)", ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_DPF((PVR_DBG_WARNING,
+	        "Setting time correlation clock from \"%s\" to \"%s\"",
+	        apszClocks[psDevInfo->ui32ClockSource],
+	        apszClocks[ui32Value]));
+
+	/* PVRSRVPowerLock() fails only when power is off. */
+	if (PVRSRVPowerLock((PVRSRV_DEVICE_NODE *) psDeviceNode) == PVRSRV_OK)
+	{
+		RGXTimeCorrEnd((PVRSRV_DEVICE_NODE *) psDeviceNode,
+		               RGXTIMECORR_EVENT_CLOCK_CHANGE);
+
+		psDevInfo->ui32ClockSource = ui32Value;
+
+		RGXTimeCorrBegin((PVRSRV_DEVICE_NODE *) psDeviceNode,
+		                 RGXTIMECORR_EVENT_CLOCK_CHANGE);
+
+		PVRSRVPowerUnlock((PVRSRV_DEVICE_NODE *)psDeviceNode);
+	}
+	else
+	{
+		/* Set the new clock source without updating the time correlation
+		 * data. This is going to be accounted for during the next power up. */
+		psDevInfo->ui32ClockSource = ui32Value;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _GetClock(const PVRSRV_DEVICE_NODE *psDeviceNode,
+                              const void *psPrivate,
+                              IMG_UINT32 *pui32Value)
+{
+	PVR_ASSERT(psDeviceNode->pvDevice != NULL);
+
+	*pui32Value =
+	    ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource;
+
+	PVR_UNREFERENCED_PARAMETER(psPrivate);
+
+	return PVRSRV_OK;
+}
+
+void RGXTimeCorrInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVRSRVAppHintRegisterHandlersUINT32(APPHINT_ID_TimeCorrClock, _GetClock,
+	                                    _SetClock, psDeviceNode, NULL);
+}
+
+/*
+	End of AppHint interface
+*/
+
+IMG_UINT64 RGXTimeCorrGetClockns64(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT64 ui64Clock;
+
+	switch (((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource) {
+		case RGXTIMECORR_CLOCK_MONO:
+			return ((void) OSClockMonotonicns64(&ui64Clock), ui64Clock);
+		case RGXTIMECORR_CLOCK_MONO_RAW:
+			return OSClockMonotonicRawns64();
+		case RGXTIMECORR_CLOCK_SCHED:
+			return OSClockns64();
+		default:
+			PVR_ASSERT(IMG_FALSE);
+			return 0;
+	}
+}
+
+IMG_UINT64 RGXTimeCorrGetClockus64(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT32 rem;
+	return OSDivide64r64(RGXTimeCorrGetClockns64(psDeviceNode), 1000, &rem);
+}
+
+static IMG_UINT64 RGXTimeGetDeviceTimestampInTicks(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	IMG_UINT64 ui64DeviceTimestamp;
+
+	/* In case of powerlock acquire fails or device is powered off, return
+	 * last device power off timestamp plus ticks corresponding to device off time. */
+	ui64DeviceTimestamp = psDeviceNode->ui64LastDeviceOffTimestamp +
+			       RGXTimeCorrDeltaOSNsToDeltaCR(psDeviceNode,
+						OSClockns64() - psDeviceNode->ui64LastDeviceOffHostTimestampNs);
+
+	if (PVRSRV_OK == PVRSRVPowerLock(psDeviceNode))
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+		RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+
+		if (PVRSRVIsDevicePowered(psDeviceNode))
+		{
+			ui64DeviceTimestamp = RGXReadHWTimerReg(psDevInfo) + psGpuUtilFW->i64DeviceTimestampOffset;
+		}
+		PVRSRVPowerUnlock(psDeviceNode);
+	}
+
+	return ui64DeviceTimestamp;
+}
+
+void RGXGetTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+							RGXFWIF_TIME_CORR *psTimeCorrs,
+							IMG_UINT32 ui32NumOut)
+{
+	PVRSRV_RGXDEV_INFO    *psDevInfo     = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW   *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32CurrentIndex;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32CurrentIndex = psGpuUtilFW->ui32TimeCorrSeqCount;
+
+	RGXFwSharedMemCacheOpExec(&psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32CurrentIndex - ui32NumOut)],
+	                          sizeof(psGpuUtilFW->sTimeCorr[0]) * ui32NumOut,
+	                          PVRSRV_CACHE_OP_INVALIDATE);
+	while (ui32NumOut--)
+	{
+		*(psTimeCorrs++) = psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32CurrentIndex)];
+		ui32CurrentIndex--;
+	}
+}
+
+static __maybe_unused const IMG_CHAR* _EventToString(RGXTIMECORR_EVENT eEvent)
+{
+	switch (eEvent)
+	{
+		case RGXTIMECORR_EVENT_POWER:
+			return "power";
+		case RGXTIMECORR_EVENT_DVFS:
+			return "dvfs";
+		case RGXTIMECORR_EVENT_PERIODIC:
+			return "periodic";
+		case RGXTIMECORR_EVENT_CLOCK_CHANGE:
+			return "clock source";
+		default:
+			return "n/a";
+	}
+}
+
+static inline IMG_UINT32 _RGXGetSystemLayerGPUClockSpeed(PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+
+	return psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
+}
+
+static inline IMG_UINT32 _RGXGetEstimatedGPUClockSpeed(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_GPU_DVFS_TABLE *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+
+	psTrackingData = &psGpuDVFSTable->asTrackingData[psGpuDVFSTable->ui32FreqIndex];
+
+	return psTrackingData->ui32EstCoreClockSpeed;
+}
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+static inline void _DumpTimerCorrelationHistory(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	RGX_GPU_DVFS_TABLE *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	IMG_UINT32 i = psGpuDVFSTable->ui32HistoryIndex;
+
+	PVR_DPF((PVR_DBG_ERROR, "Dumping history of timer correlation data (latest first):"));
+
+	do
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "  Begin times: OS %" IMG_UINT64_FMTSPEC ", CR %" IMG_UINT64_FMTSPEC ", "
+				 "End times: OS %" IMG_UINT64_FMTSPEC ", CR %" IMG_UINT64_FMTSPEC ", "
+				 "Core clk %u, Estimated clk %u",
+				 psGpuDVFSTable->asTrackingHistory[i].ui64BeginOSTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64BeginCRTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64EndOSTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui64EndCRTimestamp,
+				 psGpuDVFSTable->asTrackingHistory[i].ui32CoreClockSpeed,
+				 psGpuDVFSTable->asTrackingHistory[i].ui32EstCoreClockSpeed));
+
+		i = (i - 1) % RGX_GPU_FREQ_TRACKING_SIZE;
+
+	} while (i != psGpuDVFSTable->ui32HistoryIndex);
+}
+#endif
+
+static void _RGXMakeTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32NewSeqCount;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	RGXFWIF_TIME_CORR sTimeCorr = {0};
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32NewSeqCount = psGpuUtilFW->ui32TimeCorrSeqCount + 1;
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32NewSeqCount)], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(ui32NewSeqCount)];
+
+	/*
+	 * The following reads must be done as close together as possible, because
+	 * they represent the same current time sampled from different clock sources.
+	 */
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	if (!PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+	{
+		if (OSClockMonotonicns64(&sTimeCorr.ui64OSMonoTimeStamp) != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "_RGXMakeTimeCorrData: System Monotonic Clock not available."));
+			PVR_ASSERT(0);
+		}
+	}
+#endif
+
+	sTimeCorr.ui64CRTimeStamp = RGXReadHWTimerReg(psDevInfo);
+	sTimeCorr.ui64OSTimeStamp = RGXTimeCorrGetClockns64(psDeviceNode);
+	sTimeCorr.ui32CoreClockSpeed = _RGXGetEstimatedGPUClockSpeed(psDevInfo);
+	sTimeCorr.ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(sTimeCorr.ui32CoreClockSpeed);
+
+	if (sTimeCorr.ui64CRDeltaToOSDeltaKNs == 0)
+	{
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+		_DumpTimerCorrelationHistory(psDevInfo);
+#endif
+
+		/* Revert to original clock speed (error already printed) */
+		sTimeCorr.ui32CoreClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+		sTimeCorr.ui64CRDeltaToOSDeltaKNs = RGXTimeCorrGetConversionFactor(sTimeCorr.ui32CoreClockSpeed);
+	}
+
+	OSCachedMemCopy(psTimeCorr, &sTimeCorr, sizeof(sTimeCorr));
+	/* Make sure the values are written to memory before updating the index of the current entry */
+	OSWriteMemoryBarrier(psTimeCorr);
+	RGXFwSharedMemCacheOpPtr(psTimeCorr, FLUSH);
+
+
+	/* Update the index of the current entry in the timer correlation array */
+	psGpuUtilFW->ui32TimeCorrSeqCount = ui32NewSeqCount;
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, FLUSH);
+
+
+	if (RGXTIMECORR_EVENT_POWER == eEvent && psDeviceNode->ui64LastDeviceOffTimestamp)
+	{
+		/* Capture effective timestamp offset on device power on */
+		psGpuUtilFW->i64DeviceTimestampOffset = psDeviceNode->ui64LastDeviceOffTimestamp - RGXReadHWTimerReg(psDevInfo) +
+								RGXTimeCorrDeltaOSNsToDeltaCR(psDeviceNode,
+									OSClockns64() - psDeviceNode->ui64LastDeviceOffHostTimestampNs);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "Timer correlation data (post %s event): OS %" IMG_UINT64_FMTSPEC " ns, "
+	         "CR %" IMG_UINT64_FMTSPEC ", GPU freq. %u Hz (given as %u Hz)",
+	         _EventToString(eEvent),
+	         sTimeCorr.ui64OSTimeStamp,
+	         sTimeCorr.ui64CRTimeStamp,
+	         RGXFWIF_ROUND_TO_KHZ(sTimeCorr.ui32CoreClockSpeed),
+	         _RGXGetSystemLayerGPUClockSpeed(psDeviceNode)));
+
+	/*
+	 * Don't log timing data to the HTB log after a power(-on) event.
+	 * Otherwise this will be logged before the HTB partition marker, breaking
+	 * the log sync grammar. This data will be automatically repeated when the
+	 * partition marker is written.
+	 */
+	HTBSyncScale(eEvent != RGXTIMECORR_EVENT_POWER,
+	             sTimeCorr.ui64OSTimeStamp,
+	             sTimeCorr.ui64CRTimeStamp,
+	             sTimeCorr.ui32CoreClockSpeed);
+}
+
+static void _RGXCheckTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                  RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+#if !defined(NO_HARDWARE) && !defined(VIRTUAL_PLATFORM) && defined(DEBUG)
+#define SCALING_FACTOR (10)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	IMG_UINT32 ui32Index;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	IMG_UINT64 ui64EstimatedTime, ui64CRTimeStamp, ui64OSTimeStamp;
+	IMG_UINT64 ui64CRTimeDiff, ui64OSTimeDiff;
+	IMG_INT64 i64Diff;
+	IMG_UINT32 ui32Ratio, ui32Remainder;
+
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->ui32TimeCorrSeqCount, INVALIDATE);
+	ui32Index = RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount);
+	RGXFwSharedMemCacheOpValue(psGpuUtilFW->sTimeCorr[ui32Index], INVALIDATE);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[ui32Index];
+
+	/*
+	 * The following reads must be done as close together as possible, because
+	 * they represent the same current time sampled from different clock sources.
+	 */
+	ui64CRTimeStamp = RGXReadHWTimerReg(psDevInfo);
+	ui64OSTimeStamp = RGXTimeCorrGetClockns64(psDeviceNode);
+
+	if ((ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp) < (1 << SCALING_FACTOR))
+	{
+		/*
+		 * Less than ~1us has passed since the timer correlation data was generated.
+		 * A time frame this short is probably not enough to get an estimate
+		 * of how good the timer correlation data was.
+		 * Skip calculations for the above reason and to avoid a division by 0 below.
+		 */
+		return;
+	}
+
+
+	/* Calculate an estimated timestamp based on the latest timer correlation data */
+	ui64CRTimeDiff = ui64CRTimeStamp - psTimeCorr->ui64CRTimeStamp;
+	ui64OSTimeDiff = RGXFWIF_GET_DELTA_OSTIME_NS(ui64CRTimeDiff,
+	                                             psTimeCorr->ui64CRDeltaToOSDeltaKNs);
+	ui64EstimatedTime = psTimeCorr->ui64OSTimeStamp + ui64OSTimeDiff;
+
+	/* Get difference between estimated timestamp and current timestamp, in ns */
+	i64Diff = ui64EstimatedTime - ui64OSTimeStamp;
+
+	/*
+	 * Calculate ratio between estimated time diff and real time diff:
+	 * ratio% : 100% = (OSestimate - OStimecorr) : (OSreal - OStimecorr)
+	 *
+	 * The operands are scaled down (approximately from ns to us) so at least
+	 * the divisor fits on 32 bit.
+	 */
+	ui32Ratio = OSDivide64(((ui64EstimatedTime - psTimeCorr->ui64OSTimeStamp) * 100ULL) >> SCALING_FACTOR,
+	                       (ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp) >> SCALING_FACTOR,
+	                       &ui32Remainder);
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "Estimated timestamp check: diff %" IMG_INT64_FMTSPECd " ns over "
+	         "period %" IMG_UINT64_FMTSPEC " ns, estimated timer speed %u%%",
+	         i64Diff,
+	         ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp,
+	         ui32Ratio));
+
+	/* Warn if the estimated timestamp is not within +/- 1% of the current time */
+	if (ui32Ratio < 99 || ui32Ratio > 101)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Estimated timestamps generated in the last %" IMG_UINT64_FMTSPEC " ns "
+		         "were %s the real time (increasing at %u%% speed)",
+		         ui64OSTimeStamp - psTimeCorr->ui64OSTimeStamp,
+		         i64Diff > 0 ? "ahead of" : "behind",
+		         ui32Ratio));
+
+		/* Higher ratio == higher delta OS == higher delta CR == frequency higher than expected (and viceversa) */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "Current GPU frequency %u Hz (given as %u Hz) is probably %s than expected",
+		         RGXFWIF_ROUND_TO_KHZ(psTimeCorr->ui32CoreClockSpeed),
+		         _RGXGetSystemLayerGPUClockSpeed(psDeviceNode),
+		         i64Diff > 0 ? "lower" : "higher"));
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psGpuDVFSTable);
+#endif
+}
+
+static inline IMG_UINT32 _RGXGPUFreqGetIndex(RGX_GPU_DVFS_TABLE *psGpuDVFSTable, IMG_UINT32 ui32CoreClockSpeed)
+{
+	IMG_UINT32 *paui32GPUFrequencies = psGpuDVFSTable->aui32GPUFrequency;
+	IMG_UINT32 i;
+
+	for (i = 0; i < RGX_GPU_DVFS_TABLE_SIZE; i++)
+	{
+		if (paui32GPUFrequencies[i] == ui32CoreClockSpeed)
+		{
+			return i;
+		}
+
+		if (paui32GPUFrequencies[i] == 0)
+		{
+			paui32GPUFrequencies[i] = ui32CoreClockSpeed;
+			return i;
+		}
+	}
+
+	i--;
+
+	PVR_DPF((PVR_DBG_ERROR, "GPU frequency table in the driver is full! "
+	         "Table size should be increased! Overriding last entry (%u) with %u",
+	         paui32GPUFrequencies[i], ui32CoreClockSpeed));
+
+	paui32GPUFrequencies[i] = ui32CoreClockSpeed;
+
+	return i;
+}
+
+static void _RGXGPUFreqCalibrationPeriodStart(PVRSRV_DEVICE_NODE *psDeviceNode, RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+	IMG_UINT32 ui32CoreClockSpeed, ui32Index;
+
+	IMG_UINT64 ui64CRTimestamp = RGXReadHWTimerReg(psDevInfo);
+	IMG_UINT64 ui64OSTimestamp = RGXTimeCorrGetClockus64(psDeviceNode);
+
+	psGpuDVFSTable->ui64CalibrationCRTimestamp = ui64CRTimestamp;
+	psGpuDVFSTable->ui64CalibrationOSTimestamp = ui64OSTimestamp;
+
+	ui32CoreClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+	ui32Index          = _RGXGPUFreqGetIndex(psGpuDVFSTable, ui32CoreClockSpeed);
+	psTrackingData     = &psGpuDVFSTable->asTrackingData[ui32Index];
+
+	/* Set the time needed to (re)calibrate the GPU frequency */
+	if (psTrackingData->ui32CalibrationCount == 0) /* We never met this frequency */
+	{
+		psTrackingData->ui32EstCoreClockSpeed = ui32CoreClockSpeed;
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_FIRST_CALIBRATION_TIME_US;
+	}
+	else if (psTrackingData->ui32CalibrationCount == 1) /* We calibrated this frequency only once */
+	{
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_TRANSITION_CALIBRATION_TIME_US;
+	}
+	else
+	{
+		psGpuDVFSTable->ui32CalibrationPeriod = RGX_GPU_DVFS_PERIODIC_CALIBRATION_TIME_US;
+	}
+
+	/* Update the index to the DVFS table */
+	psGpuDVFSTable->ui32FreqIndex = ui32Index;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui32CoreClockSpeed    = ui32CoreClockSpeed;
+		psTrackingHistory->ui32EstCoreClockSpeed = psTrackingData->ui32EstCoreClockSpeed;
+		psTrackingHistory->ui64BeginCRTimestamp  = ui64CRTimestamp;
+		psTrackingHistory->ui64BeginOSTimestamp  = ui64OSTimestamp;
+		psTrackingHistory->ui64EndCRTimestamp    = 0ULL;
+		psTrackingHistory->ui64EndOSTimestamp    = 0ULL;
+	}
+#endif
+}
+
+static void _RGXGPUFreqCalibrationPeriodStop(PVRSRV_DEVICE_NODE *psDeviceNode,
+											 RGX_GPU_DVFS_TABLE *psGpuDVFSTable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+	IMG_UINT64 ui64CRTimestamp = RGXReadHWTimerReg(psDevInfo);
+	IMG_UINT64 ui64OSTimestamp = RGXTimeCorrGetClockus64(psDeviceNode);
+
+	psGpuDVFSTable->ui64CalibrationCRTimediff =
+	    ui64CRTimestamp - psGpuDVFSTable->ui64CalibrationCRTimestamp;
+	psGpuDVFSTable->ui64CalibrationOSTimediff =
+	    ui64OSTimestamp - psGpuDVFSTable->ui64CalibrationOSTimestamp;
+
+	/* Check if the current timer correlation data is good enough */
+	_RGXCheckTimeCorrData(psDeviceNode, psGpuDVFSTable);
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui64EndCRTimestamp = ui64CRTimestamp;
+		psTrackingHistory->ui64EndOSTimestamp = ui64OSTimestamp;
+	}
+#endif
+}
+
+static void _RGXGPUFreqCalibrationCalculate(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                            RGX_GPU_DVFS_TABLE *psGpuDVFSTable,
+                                            RGXTIMECORR_EVENT   eEvent)
+{
+#if !defined(DISABLE_GPU_FREQUENCY_CALIBRATION)
+	GPU_FREQ_TRACKING_DATA *psTrackingData;
+	IMG_UINT32 ui32EstCoreClockSpeed, ui32PrevCoreClockSpeed, ui32SysGPUClockSpeed;
+	IMG_INT32  i32Diff;
+	IMG_UINT32 ui32Remainder;
+
+	ui32SysGPUClockSpeed = _RGXGetSystemLayerGPUClockSpeed(psDeviceNode);
+	/*
+	 * Find out what the GPU frequency was in the last period.
+	 * This should return a value very close to the frequency passed by the system layer.
+	 */
+	ui32EstCoreClockSpeed =
+	    RGXFWIF_GET_GPU_CLOCK_FREQUENCY_HZ(psGpuDVFSTable->ui64CalibrationCRTimediff,
+	                                       psGpuDVFSTable->ui64CalibrationOSTimediff,
+	                                       ui32Remainder);
+
+	/* Update GPU frequency used by the driver for a given system layer frequency */
+	psTrackingData = &psGpuDVFSTable->asTrackingData[psGpuDVFSTable->ui32FreqIndex];
+
+	ui32PrevCoreClockSpeed = psTrackingData->ui32EstCoreClockSpeed;
+	psTrackingData->ui32EstCoreClockSpeed = ui32EstCoreClockSpeed;
+	psTrackingData->ui32CalibrationCount++;
+
+	i32Diff = (IMG_INT32) (ui32EstCoreClockSpeed - ui32PrevCoreClockSpeed);
+
+	if ((i32Diff < -1000000) || (i32Diff > 1000000))
+	{
+		/* Warn if the frequency changed by more than 1 MHz between recalculations */
+		PVR_DPF((PVR_DBG_WARNING,
+		         "GPU frequency calibration of system layer frequency %u Hz (pre %s event): "
+		         "more than 1 MHz difference between old and new value "
+		         "(%u Hz -> %u Hz over %"  IMG_UINT64_FMTSPEC " us)",
+		         ui32SysGPUClockSpeed,
+		         _EventToString(eEvent),
+		         RGXFWIF_ROUND_TO_KHZ(ui32PrevCoreClockSpeed),
+		         RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed),
+		         psGpuDVFSTable->ui64CalibrationOSTimediff));
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE,
+		         "GPU frequency calibration of system layer frequency %u Hz (pre %s event): "
+		         "%u Hz -> %u Hz done over %" IMG_UINT64_FMTSPEC " us",
+		         ui32SysGPUClockSpeed,
+		         _EventToString(eEvent),
+		         RGXFWIF_ROUND_TO_KHZ(ui32PrevCoreClockSpeed),
+		         RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed),
+		         psGpuDVFSTable->ui64CalibrationOSTimediff));
+	}
+
+	if (eEvent == RGXTIMECORR_EVENT_PERIODIC)
+	{
+		PVRSRV_DEV_POWER_STATE ePowerState;
+
+		i32Diff = (IMG_INT32) (ui32EstCoreClockSpeed - ui32SysGPUClockSpeed);
+
+		/*
+		 * Notify the Firmware about unexpected frequency differences observed during
+		 * periodic frequency calibration events only.
+		 * Other events like PDVFS and power transitions are already likely to call
+		 * the pre/post clock callback directly to set frequencies as needed.
+		 * Platforms without PDVFS or APM need a method to correct the Firmware's
+		 * internal timing measurements.
+		 */
+		if (((i32Diff < -1000000) || (i32Diff > 1000000)) &&
+			(PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState) == PVRSRV_OK))
+		{
+			PVRSRV_ERROR eError = RGXPreClockSpeedChange((IMG_HANDLE)psDeviceNode, ePowerState);
+
+			PVR_LOG_IF_ERROR(eError, "RGXPreClockSpeedChange");
+			if (eError == PVRSRV_OK)
+			{
+				RGX_DATA *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
+
+				/* Update the internal core frequency variable and notify the Firmware of the change */
+				psRGXData->psRGXTimingInfo->ui32CoreClockSpeed = RGXFWIF_ROUND_TO_KHZ(ui32EstCoreClockSpeed);
+
+				eError = RGXPostClockSpeedChange((IMG_HANDLE)psDeviceNode, ePowerState);
+				PVR_LOG_IF_ERROR(eError, "RGXPostClockSpeedChange");
+			}
+		}
+	}
+
+	/* Reset time deltas to avoid recalibrating the same frequency over and over again */
+	psGpuDVFSTable->ui64CalibrationCRTimediff = 0;
+	psGpuDVFSTable->ui64CalibrationOSTimediff = 0;
+
+#if defined(PVRSRV_TIMER_CORRELATION_HISTORY)
+	/* Update tracking history */
+	{
+		GPU_FREQ_TRACKING_HISTORY *psTrackingHistory;
+
+		psTrackingHistory = &psGpuDVFSTable->asTrackingHistory[psGpuDVFSTable->ui32HistoryIndex];
+		psTrackingHistory->ui32EstCoreClockSpeed = ui32EstCoreClockSpeed;
+		psGpuDVFSTable->ui32HistoryIndex =
+			(psGpuDVFSTable->ui32HistoryIndex + 1) % RGX_GPU_FREQ_TRACKING_SIZE;
+	}
+#endif
+
+#else
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(psGpuDVFSTable);
+	PVR_UNREFERENCED_PARAMETER(eEvent);
+#endif
+}
+
+void RGXTimeCorrBegin(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_DEVICE_NODE  *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO  *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE  *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	_RGXGPUFreqCalibrationPeriodStart(psDeviceNode, psGpuDVFSTable);
+	_RGXMakeTimeCorrData(psDeviceNode, eEvent);
+}
+
+void RGXTimeCorrEnd(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent)
+{
+	PVRSRV_DEVICE_NODE  *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO  *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE  *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	_RGXGPUFreqCalibrationPeriodStop(psDeviceNode, psGpuDVFSTable);
+
+	if (psGpuDVFSTable->ui64CalibrationOSTimediff >= psGpuDVFSTable->ui32CalibrationPeriod)
+	{
+		_RGXGPUFreqCalibrationCalculate(psDeviceNode, psGpuDVFSTable, eEvent);
+	}
+
+	if (RGXTIMECORR_EVENT_POWER == eEvent)
+	{
+		RGXFWIF_GPU_UTIL_FW *psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+
+		/* Capture effective device power off timestamp */
+		psDeviceNode->ui64LastDeviceOffTimestamp = RGXReadHWTimerReg(psDevInfo) + psGpuUtilFW->i64DeviceTimestampOffset;
+		psDeviceNode->ui64LastDeviceOffHostTimestampNs = OSClockns64();
+	}
+}
+
+void RGXTimeCorrRestartPeriodic(IMG_HANDLE hDevHandle)
+{
+	PVRSRV_DEVICE_NODE     *psDeviceNode   = hDevHandle;
+	PVRSRV_RGXDEV_INFO     *psDevInfo      = psDeviceNode->pvDevice;
+	RGX_GPU_DVFS_TABLE     *psGpuDVFSTable = psDevInfo->psGpuDVFSTable;
+	IMG_UINT64             ui64TimeNow     = RGXTimeCorrGetClockus64(psDeviceNode);
+	PVRSRV_DEV_POWER_STATE ePowerState = PVRSRV_DEV_POWER_STATE_DEFAULT;
+	PVRSRV_VZ_RETN_IF_MODE(GUEST, DEVNODE, psDeviceNode);
+
+	if (psGpuDVFSTable == NULL)
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: Required data not initialised yet", __func__));
+		return;
+	}
+
+	/* Check if it's the right time to recalibrate the GPU clock frequency */
+	if ((ui64TimeNow - psGpuDVFSTable->ui64CalibrationOSTimestamp) < psGpuDVFSTable->ui32CalibrationPeriod) return;
+
+	/* Try to acquire the powerlock, if not possible then don't wait */
+	if (PVRSRVPowerTryLock(psDeviceNode) != PVRSRV_OK) return;
+
+	/* If the GPU is off then we can't do anything */
+	PVRSRVGetDevicePowerState(psDeviceNode, &ePowerState);
+	if (ePowerState != PVRSRV_DEV_POWER_STATE_ON)
+	{
+		PVRSRVPowerUnlock(psDeviceNode);
+		return;
+	}
+
+	/* All checks passed, we can calibrate and correlate */
+	RGXTimeCorrEnd(psDeviceNode, RGXTIMECORR_EVENT_PERIODIC);
+	RGXTimeCorrBegin(psDeviceNode, RGXTIMECORR_EVENT_PERIODIC);
+
+	PVRSRVPowerUnlock(psDeviceNode);
+}
+
+/*
+	RGXTimeCorrGetClockSource
+*/
+RGXTIMECORR_CLOCK_TYPE RGXTimeCorrGetClockSource(const PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	return ((PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice)->ui32ClockSource;
+}
+
+/*
+	RGXTimeCorrSetClockSource
+*/
+PVRSRV_ERROR RGXTimeCorrSetClockSource(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       RGXTIMECORR_CLOCK_TYPE eClockType)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO* psRGXDevNode = psDeviceNode->pvDevice;
+	RGXTIMECORR_CLOCK_TYPE eLastClock = psRGXDevNode->ui32ClockSource;
+
+	eError = _SetClock(psDeviceNode, NULL, eClockType);
+	PVR_LOG_RETURN_IF_ERROR(eError, "_SetClock");
+
+	psRGXDevNode->ui32LastClockSource = eLastClock;
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVRGXCurrentTime(CONNECTION_DATA    *psConnection,
+                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT8           ui8TimestampType,
+                     IMG_UINT64         *pui64Time)
+{
+	RGX_QUERY_TIMESTAMP_TYPE eTimestampType = (RGX_QUERY_TIMESTAMP_TYPE)ui8TimestampType;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	switch (eTimestampType)
+	{
+		case RGX_QUERY_HOST_TIMESTAMP:
+		*pui64Time = RGXTimeCorrGetClockns64(psDeviceNode);
+		break;
+
+		case RGX_QUERY_DEVICE_TIMESTAMP:
+		*pui64Time = RGXTimeGetDeviceTimestampInTicks(psDeviceNode);
+		break;
+
+		default:
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		break;
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (rgxtimecorr.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxtimecorr.h b/drivers/gpu/drm/img-rogue/rgxtimecorr.h
new file mode 100644
index 000000000000..9758e61909c8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtimecorr.h
@@ -0,0 +1,316 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX time correlation and calibration header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX time correlation and calibration routines
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTIMECORR_H)
+#define RGXTIMECORR_H
+
+#include "img_types.h"
+#include "device.h"
+#include "osfunc.h"
+#include "connection_server.h"
+#include "rgxdevice.h"
+
+typedef enum
+{
+	RGXTIMECORR_CLOCK_MONO,
+	RGXTIMECORR_CLOCK_MONO_RAW,
+	RGXTIMECORR_CLOCK_SCHED,
+
+	RGXTIMECORR_CLOCK_LAST
+} RGXTIMECORR_CLOCK_TYPE;
+
+typedef enum
+{
+	RGXTIMECORR_EVENT_POWER,
+	RGXTIMECORR_EVENT_DVFS,
+	RGXTIMECORR_EVENT_PERIODIC,
+	RGXTIMECORR_EVENT_CLOCK_CHANGE
+} RGXTIMECORR_EVENT;
+
+/*
+ * Calibrated GPU frequencies are rounded to the nearest multiple of 1 KHz
+ * before use, to reduce the noise introduced by calculations done with
+ * imperfect operands (correlated timers not sampled at exactly the same
+ * time, GPU CR timer incrementing only once every 256 GPU cycles).
+ * This also helps reducing the variation between consecutive calculations.
+ */
+#define RGXFWIF_CONVERT_TO_KHZ(freq)   (((freq) + 500) / 1000)
+#define RGXFWIF_ROUND_TO_KHZ(freq)    ((((freq) + 500) / 1000) * 1000)
+
+/* Constants used in different calculations */
+#define SECONDS_TO_MICROSECONDS          (1000000ULL)
+#define CRTIME_TO_CYCLES_WITH_US_SCALE   (RGX_CRTIME_TICK_IN_CYCLES * SECONDS_TO_MICROSECONDS)
+
+/*
+ * Use this macro to get a more realistic GPU core clock speed than the one
+ * given by the upper layers (used when doing GPU frequency calibration)
+ */
+#define RGXFWIF_GET_GPU_CLOCK_FREQUENCY_HZ(deltacr_us, deltaos_us, remainder) \
+    OSDivide64((deltacr_us) * CRTIME_TO_CYCLES_WITH_US_SCALE, (deltaos_us), &(remainder))
+
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetConversionFactor
+
+ @Description Generate constant used to convert a GPU time difference into
+              an OS time difference (for more info see rgx_fwif_km.h).
+
+ @Input       ui32ClockSpeed : GPU clock speed
+
+ @Return      0 on failure, conversion factor otherwise
+
+******************************************************************************/
+static inline IMG_UINT64 RGXTimeCorrGetConversionFactor(IMG_UINT32 ui32ClockSpeed)
+{
+	IMG_UINT32 ui32Remainder;
+
+	if (RGXFWIF_CONVERT_TO_KHZ(ui32ClockSpeed) == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: GPU clock frequency %u is too low",
+				 __func__, ui32ClockSpeed));
+
+		return 0;
+	}
+
+	return OSDivide64r64(CRTIME_TO_CYCLES_WITH_US_SCALE << RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT,
+	                     RGXFWIF_CONVERT_TO_KHZ(ui32ClockSpeed), &ui32Remainder);
+}
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrDeltaOSNsToDeltaCR
+
+ @Description Convert OS timestamp difference in ns to device tick difference
+
+ @Input       psDeviceNode, ui64DeltaOSNs
+
+ @Return      0 on failure, deltaCR otherwise
+
+******************************************************************************/
+static inline IMG_UINT64 RGXTimeCorrDeltaOSNsToDeltaCR(PVRSRV_DEVICE_NODE *psDeviceNode, IMG_UINT64 ui64DeltaOSNs)
+{
+	RGXFWIF_GPU_UTIL_FW *psGpuUtilFW;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	RGXFWIF_TIME_CORR *psTimeCorr;
+	IMG_UINT32 ui32Remainder;
+
+	PVR_ASSERT(psDeviceNode && psDeviceNode->pvDevice);
+	psDevInfo = psDeviceNode->pvDevice;
+	psGpuUtilFW = psDevInfo->psRGXFWIfGpuUtilFW;
+	PVR_ASSERT(psGpuUtilFW);
+	psTimeCorr = &psGpuUtilFW->sTimeCorr[RGXFWIF_TIME_CORR_CURR_INDEX(psGpuUtilFW->ui32TimeCorrSeqCount)];
+
+	if (psTimeCorr->ui64CRDeltaToOSDeltaKNs == 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui64CRDeltaToOSDeltaKNs is 0", __func__));
+		return 0;
+	}
+
+	return OSDivide64r64(ui64DeltaOSNs << RGXFWIF_CRDELTA_TO_OSDELTA_ACCURACY_SHIFT,
+	                     psTimeCorr->ui64CRDeltaToOSDeltaKNs, &ui32Remainder);
+}
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrBegin
+
+ @Description Generate new timer correlation data, and start tracking
+              the current GPU frequency.
+
+ @Input       hDevHandle : RGX Device Node
+ @Input       eEvent     : Event associated with the beginning of a timer
+                           correlation period
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrBegin(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrEnd
+
+ @Description Stop tracking the CPU and GPU timers, and if possible
+              recalculate the GPU frequency to a value which makes the timer
+              correlation data more accurate.
+
+ @Input       hDevHandle : RGX Device Node
+ @Input       eEvent     : Event associated with the end of a timer
+                           correlation period
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrEnd(IMG_HANDLE hDevHandle, RGXTIMECORR_EVENT eEvent);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrRestartPeriodic
+
+ @Description Perform actions from RGXTimeCorrEnd and RGXTimeCorrBegin,
+              but only if enough time has passed since the last timer
+              correlation data was generated.
+
+ @Input       hDevHandle : RGX Device Node
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrRestartPeriodic(IMG_HANDLE hDevHandle);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockns64
+
+ @Description Returns value of currently selected clock (in ns).
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock value from currently selected clock source
+
+******************************************************************************/
+IMG_UINT64 RGXTimeCorrGetClockns64(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockus64
+
+ @Description Returns value of currently selected clock (in us).
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock value from currently selected clock source
+
+******************************************************************************/
+IMG_UINT64 RGXTimeCorrGetClockus64(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrGetClockSource
+
+ @Description Returns currently selected clock source
+
+ @Input       psDeviceNode : RGX Device Node
+ @Return      clock source type
+
+******************************************************************************/
+RGXTIMECORR_CLOCK_TYPE RGXTimeCorrGetClockSource(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrSetClockSource
+
+ @Description Sets clock source for correlation data.
+
+ @Input       psDeviceNode : RGX Device Node
+ @Input       eClockType : clock source type
+
+ @Return      error code
+
+******************************************************************************/
+PVRSRV_ERROR RGXTimeCorrSetClockSource(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                       RGXTIMECORR_CLOCK_TYPE eClockType);
+
+/*!
+******************************************************************************
+
+ @Function    RGXTimeCorrInitAppHintCallbacks
+
+ @Description Initialise apphint callbacks for timer correlation
+              related apphints.
+
+ @Input       psDeviceNode : RGX Device Node
+
+ @Return      void
+
+******************************************************************************/
+void RGXTimeCorrInitAppHintCallbacks(const PVRSRV_DEVICE_NODE *psDeviceNode);
+
+/*!
+******************************************************************************
+
+ @Function    RGXGetTimeCorrData
+
+ @Description Get a number of the most recent time correlation data points
+
+ @Input       psDeviceNode : RGX Device Node
+ @Output      psTimeCorrs  : Output array of RGXFWIF_TIME_CORR elements
+                             for data to be written to
+ @Input       ui32NumOut   : Number of elements to be written out
+
+ @Return      void
+
+******************************************************************************/
+void RGXGetTimeCorrData(PVRSRV_DEVICE_NODE *psDeviceNode,
+							RGXFWIF_TIME_CORR *psTimeCorrs,
+							IMG_UINT32 ui32NumOut);
+
+/*!
+******************************************************************************
+
+ @Function    PVRSRVRGXCurrentTime
+
+ @Description Server-side implementation of RGXCurrentTime
+
+ @Input       psConnection     : Connection handle
+ @Input       psDeviceNode     : RGX Device Node
+ @Input       ui8TimestampType : Timestamp type
+ @Output      pui64Time        : Timestamp
+
+ @Return      PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXCurrentTime(CONNECTION_DATA    *psConnection,
+                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                     IMG_UINT8           ui8TimestampType,
+                     IMG_UINT64         *pui64Time);
+#endif /* RGXTIMECORR_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxtimerquery.c b/drivers/gpu/drm/img-rogue/rgxtimerquery.c
new file mode 100644
index 000000000000..82900863ed26
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtimerquery.c
@@ -0,0 +1,225 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Timer queries
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    RGX Timer queries
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgxtimerquery.h"
+#include "rgxdevice.h"
+#include "rgxtimecorr.h"
+
+#include "rgxfwutils.h"
+#include "pdump_km.h"
+
+PVRSRV_ERROR
+PVRSRVRGXBeginTimerQueryKM(CONNECTION_DATA    * psConnection,
+                           PVRSRV_DEVICE_NODE * psDeviceNode,
+                           IMG_UINT32         ui32QueryId)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32QueryId >= RGX_MAX_TIMER_QUERIES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	psDevInfo->bSaveStart = IMG_TRUE;
+	psDevInfo->bSaveEnd   = IMG_TRUE;
+
+	/* clear the stamps, in case there is no Kick */
+	psDevInfo->pui64StartTimeById[ui32QueryId] = 0UL;
+	psDevInfo->pui64EndTimeById[ui32QueryId]   = 0UL;
+	OSWriteMemoryBarrier(&psDevInfo->pui64EndTimeById[ui32QueryId]);
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui64StartTimeById[ui32QueryId], FLUSH);
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui64EndTimeById[ui32QueryId], FLUSH);
+
+	/* save of the active query index */
+	psDevInfo->ui32ActiveQueryId = ui32QueryId;
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXEndTimerQueryKM(CONNECTION_DATA    * psConnection,
+                         PVRSRV_DEVICE_NODE * psDeviceNode)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	/* clear off the flags set by Begin(). Note that _START_TIME is
+	 * probably already cleared by Kick()
+	 */
+	psDevInfo->bSaveStart = IMG_FALSE;
+	psDevInfo->bSaveEnd   = IMG_FALSE;
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+
+	return PVRSRV_OK;
+}
+
+
+PVRSRV_ERROR
+PVRSRVRGXQueryTimerKM(CONNECTION_DATA    * psConnection,
+                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                      IMG_UINT32         ui32QueryId,
+                      IMG_UINT64         * pui64StartTime,
+                      IMG_UINT64         * pui64EndTime)
+{
+	PVRSRV_RGXDEV_INFO * psDevInfo = (PVRSRV_RGXDEV_INFO *)psDeviceNode->pvDevice;
+	IMG_UINT32         ui32Scheduled;
+	IMG_UINT32         ui32Completed;
+	PVRSRV_ERROR       eError;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32QueryId >= RGX_MAX_TIMER_QUERIES)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockAcquire(psDevInfo->hTimerQueryLock);
+#endif
+
+	ui32Scheduled = psDevInfo->aui32ScheduledOnId[ui32QueryId];
+
+	RGXFwSharedMemCacheOpValue(psDevInfo->pui32CompletedById[ui32QueryId], INVALIDATE);
+	ui32Completed = psDevInfo->pui32CompletedById[ui32QueryId];
+
+	/* if there was no kick since the Begin() on this id we return 0-s as Begin cleared
+	 * the stamps. If there was no begin the returned data is undefined - but still
+	 * safe from services pov
+	 */
+	if (ui32Completed >= ui32Scheduled)
+	{
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui64StartTimeById[ui32QueryId], INVALIDATE);
+		RGXFwSharedMemCacheOpValue(psDevInfo->pui64EndTimeById[ui32QueryId], INVALIDATE);
+		* pui64StartTime = psDevInfo->pui64StartTimeById[ui32QueryId];
+		* pui64EndTime   = psDevInfo->pui64EndTimeById[ui32QueryId];
+
+		eError = PVRSRV_OK;
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+#if !defined(PVRSRV_USE_BRIDGE_LOCK)
+	OSLockRelease(psDevInfo->hTimerQueryLock);
+#endif
+	return eError;
+}
+
+
+
+/******************************************************************************
+ NOT BRIDGED/EXPORTED FUNCS
+******************************************************************************/
+
+void
+RGX_GetTimestampCmdHelper(PVRSRV_RGXDEV_INFO      * psDevInfo,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPreAddr,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPostAddr,
+                          PRGXFWIF_UFO_ADDR       * ppUpdate)
+{
+	if (ppPreAddr != NULL)
+	{
+		if (psDevInfo->bSaveStart)
+		{
+			/* drop the SaveStart on the first Kick */
+			psDevInfo->bSaveStart = IMG_FALSE;
+
+			RGXSetFirmwareAddress(ppPreAddr,
+			                      psDevInfo->psStartTimeMemDesc,
+			                      sizeof(IMG_UINT64) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+		}
+		else
+		{
+			ppPreAddr->ui32Addr = 0;
+		}
+	}
+
+	if (ppPostAddr != NULL && ppUpdate != NULL)
+	{
+		if (psDevInfo->bSaveEnd)
+		{
+			RGXSetFirmwareAddress(ppPostAddr,
+			                      psDevInfo->psEndTimeMemDesc,
+			                      sizeof(IMG_UINT64) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+
+			psDevInfo->aui32ScheduledOnId[psDevInfo->ui32ActiveQueryId]++;
+
+			RGXSetFirmwareAddress(ppUpdate,
+			                      psDevInfo->psCompletedMemDesc,
+			                      sizeof(IMG_UINT32) * psDevInfo->ui32ActiveQueryId,
+			                      RFW_FWADDR_NOREF_FLAG);
+		}
+		else
+		{
+			ppUpdate->ui32Addr   = 0;
+			ppPostAddr->ui32Addr = 0;
+		}
+	}
+}
+
+
+/******************************************************************************
+ End of file (rgxtimerquery.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxtimerquery.h b/drivers/gpu/drm/img-rogue/rgxtimerquery.h
new file mode 100644
index 000000000000..7a43e7966f88
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtimerquery.h
@@ -0,0 +1,117 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Timer queries
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Timer queries functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGX_TIMERQUERIES_H)
+#define RGX_TIMERQUERIES_H
+
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "device.h"
+#include "rgxdevice.h"
+
+#include "connection_server.h"
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXBeginTimerQueryKM
+@Description    Opens a new timer query.
+
+@Input          ui32QueryId an identifier between [ 0 and RGX_MAX_TIMER_QUERIES - 1 ]
+@Return         PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXBeginTimerQueryKM(CONNECTION_DATA    * psConnection,
+                           PVRSRV_DEVICE_NODE * psDeviceNode,
+                           IMG_UINT32         ui32QueryId);
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXEndTimerQueryKM
+@Description    Closes a timer query
+
+                The lack of ui32QueryId argument expresses the fact that there
+                can't be overlapping queries open.
+@Return         PVRSRV_OK on success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXEndTimerQueryKM(CONNECTION_DATA    * psConnection,
+                         PVRSRV_DEVICE_NODE * psDeviceNode);
+
+
+
+/*************************************************************************/ /*!
+@Function       PVRSRVRGXQueryTimerKM
+@Description    Queries the state of the specified timer
+
+@Input          ui32QueryId an identifier between [ 0 and RGX_MAX_TIMER_QUERIES - 1 ]
+@Out            pui64StartTime
+@Out            pui64EndTime
+@Return         PVRSRV_OK                         on success.
+                PVRSRV_ERROR_RESOURCE_UNAVAILABLE if the device is still busy with
+                                                  operations from the queried period
+                other error code                  otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVRGXQueryTimerKM(CONNECTION_DATA    * psConnection,
+                      PVRSRV_DEVICE_NODE * psDeviceNode,
+                      IMG_UINT32         ui32QueryId,
+                      IMG_UINT64         * pui64StartTime,
+                      IMG_UINT64         * pui64EndTime);
+
+
+
+/******************************************************************************
+ NON BRIDGED/EXPORTED interface
+******************************************************************************/
+
+/* get the relevant data from the Kick to the helper*/
+void
+RGX_GetTimestampCmdHelper(PVRSRV_RGXDEV_INFO      * psDevInfo,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPreAddr,
+                          PRGXFWIF_TIMESTAMP_ADDR * ppPostAddr,
+                          PRGXFWIF_UFO_ADDR       * ppUpdate);
+
+#endif /* RGX_TIMERQUERIES_H */
+
+/******************************************************************************
+ End of file (rgxtimerquery.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxtransfer.c b/drivers/gpu/drm/img-rogue/rgxtransfer.c
new file mode 100644
index 000000000000..360d6282e674
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtransfer.c
@@ -0,0 +1,1990 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific transfer queue routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pdump_km.h"
+#include "rgxdevice.h"
+#include "rgxccb.h"
+#include "rgxutils.h"
+#include "rgxfwcmnctx.h"
+#include "rgxtransfer.h"
+#include "rgx_tq_shared.h"
+#include "rgxmem.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pvr_debug.h"
+#include "pvrsrv.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgx_memallocflags.h"
+#include "rgxhwperf.h"
+#include "ospvr_gputrace.h"
+#include "htbserver.h"
+#include "rgxshader.h"
+
+#include "pdump_km.h"
+
+#include "sync_server.h"
+#include "sync_internal.h"
+#include "sync.h"
+#include "rgx_bvnc_defs_km.h"
+
+#if defined(SUPPORT_BUFFER_SYNC)
+#include "pvr_buffer_sync.h"
+#endif
+
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+
+#include "rgxtimerquery.h"
+
+
+/* Enable this to dump the compiled list of UFOs prior to kick call */
+#define ENABLE_TQ_UFO_DUMP	0
+
+//#define TRANSFER_CHECKPOINT_DEBUG 1
+
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+#define CHKPT_DBG(X) PVR_DPF(X)
+#else
+#define CHKPT_DBG(X)
+#endif
+
+typedef struct {
+	DEVMEM_MEMDESC				*psFWContextStateMemDesc;
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_3D_DATA;
+
+typedef struct {
+	RGX_SERVER_COMMON_CONTEXT	*psServerCommonContext;
+	IMG_INT32					i32Priority;
+#if defined(SUPPORT_BUFFER_SYNC)
+	struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+} RGX_SERVER_TQ_2D_DATA;
+
+struct _RGX_SERVER_TQ_CONTEXT_ {
+	PVRSRV_DEVICE_NODE			*psDeviceNode;
+	DEVMEM_MEMDESC				*psFWFrameworkMemDesc;
+	DEVMEM_MEMDESC              *psFWTransferContextMemDesc;
+	IMG_UINT32					ui32Flags;
+#define RGX_SERVER_TQ_CONTEXT_FLAGS_2D		(1<<0)
+#define RGX_SERVER_TQ_CONTEXT_FLAGS_3D		(1<<1)
+	RGX_SERVER_TQ_3D_DATA		s3DData;
+	RGX_SERVER_TQ_2D_DATA		s2DData;
+	DLLIST_NODE					sListNode;
+	ATOMIC_T			hIntJobRef;
+	IMG_UINT32			ui32PDumpFlags;
+	/* per-prepare sync address lists */
+	SYNC_ADDR_LIST			asSyncAddrListFence[TQ_MAX_PREPARES_PER_SUBMIT];
+	SYNC_ADDR_LIST			asSyncAddrListUpdate[TQ_MAX_PREPARES_PER_SUBMIT];
+	POS_LOCK				hLock;
+};
+
+/*
+	Static functions used by transfer context code
+*/
+static PVRSRV_ERROR _Create3DTransferContext(CONNECTION_DATA *psConnection,
+											 PVRSRV_DEVICE_NODE *psDeviceNode,
+											 DEVMEM_MEMDESC *psAllocatedMemDesc,
+											 IMG_UINT32 ui32AllocatedOffset,
+											 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+											 IMG_INT32 i32Priority,
+											 RGX_COMMON_CONTEXT_INFO *psInfo,
+											 RGX_SERVER_TQ_3D_DATA *ps3DData,
+											 IMG_UINT32 ui32CCBAllocSizeLog2,
+											 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+											 IMG_UINT32 ui32ContextFlags,
+											 IMG_UINT64 ui64RobustnessAddress)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	PVRSRV_ERROR eError;
+	IMG_UINT	ui3DRegISPStateStoreSize = 0;
+	IMG_UINT	uiNumISPStoreRegs;
+	/*
+		Allocate device memory for the firmware GPU context suspend state.
+		Note: the FW reads/writes the state to memory by accessing the GPU register interface.
+	*/
+	PDUMPCOMMENT(psDeviceNode, "Allocate RGX firmware TQ/3D context suspend state");
+
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, XE_MEMORY_HIERARCHY))
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+				RGX_FEATURE_NUM_RASTER_PIPES_IDX);
+		if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, GPU_MULTICORE_SUPPORT))
+		{
+			uiNumISPStoreRegs *= (1U + psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+					RGX_FEATURE_XPU_MAX_SLAVES_IDX));
+		}
+	}
+	else
+	{
+		uiNumISPStoreRegs = psDeviceNode->pfnGetDeviceFeatureValue(psDeviceNode,
+													RGX_FEATURE_NUM_ISP_IPP_PIPES_IDX);
+	}
+
+	/* Calculate the size of the 3DCTX ISP state */
+	ui3DRegISPStateStoreSize = sizeof(RGXFWIF_3DCTX_STATE) +
+			uiNumISPStoreRegs * sizeof(((RGXFWIF_3DCTX_STATE *)0)->au3DReg_ISP_STORE[0]);
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	ps3DData->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-tq3d");
+	if (IS_ERR(ps3DData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(ps3DData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = DevmemFwAllocate(psDevInfo,
+							ui3DRegISPStateStoreSize,
+							RGX_FWCOMCTX_ALLOCFLAGS,
+							"FwTQ3DContext",
+							&ps3DData->psFWContextStateMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextswitchstate;
+	}
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TQ_3D,
+	                                 RGXFWIF_DM_3D,
+									 NULL,
+									 psAllocatedMemDesc,
+									 ui32AllocatedOffset,
+	                                 psFWMemContextMemDesc,
+	                                 ps3DData->psFWContextStateMemDesc,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TQ3D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TQ3D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 UINT_MAX, /* max deadline MS */
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps3DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+
+	PDUMPCOMMENT(psDeviceNode, "Dump 3D context suspend state buffer");
+	DevmemPDumpLoadMem(ps3DData->psFWContextStateMemDesc, 0, sizeof(RGXFWIF_3DCTX_STATE), PDUMP_FLAGS_CONTINUOUS);
+
+	ps3DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+	DevmemFwUnmapAndFree(psDevInfo, ps3DData->psFWContextStateMemDesc);
+fail_contextswitchstate:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps3DData->psBufferSyncContext);
+	ps3DData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+static PVRSRV_ERROR _Create2DTransferContext(CONNECTION_DATA *psConnection,
+											 PVRSRV_DEVICE_NODE *psDeviceNode,
+											 DEVMEM_MEMDESC *psFWMemContextMemDesc,
+											 IMG_INT32 i32Priority,
+											 RGX_COMMON_CONTEXT_INFO *psInfo,
+											 RGX_SERVER_TQ_2D_DATA *ps2DData,
+											 IMG_UINT32 ui32CCBAllocSizeLog2,
+											 IMG_UINT32 ui32CCBMaxAllocSizeLog2,
+											 IMG_UINT32 ui32ContextFlags,
+											 IMG_UINT64 ui64RobustnessAddress)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	ps2DData->psBufferSyncContext =
+		pvr_buffer_sync_context_create(psDeviceNode->psDevConfig->pvOSDevice,
+									   "rogue-tqtla");
+	if (IS_ERR(ps2DData->psBufferSyncContext))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: failed to create buffer_sync context (err=%ld)",
+				 __func__, PTR_ERR(ps2DData->psBufferSyncContext)));
+
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto fail_buffer_sync_context_create;
+	}
+#endif
+
+	eError = FWCommonContextAllocate(psConnection,
+	                                 psDeviceNode,
+	                                 REQ_TYPE_TQ_2D,
+	                                 RGXFWIF_DM_2D,
+									 NULL,
+	                                 NULL,
+	                                 0,
+	                                 psFWMemContextMemDesc,
+	                                 NULL,
+	                                 ui32CCBAllocSizeLog2 ? ui32CCBAllocSizeLog2 : RGX_TQ2D_CCB_SIZE_LOG2,
+	                                 ui32CCBMaxAllocSizeLog2 ? ui32CCBMaxAllocSizeLog2 : RGX_TQ2D_CCB_MAX_SIZE_LOG2,
+	                                 ui32ContextFlags,
+	                                 i32Priority,
+	                                 UINT_MAX, /* max deadline MS */
+	                                 ui64RobustnessAddress,
+	                                 psInfo,
+	                                 &ps2DData->psServerCommonContext);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_contextalloc;
+	}
+
+	ps2DData->i32Priority = i32Priority;
+	return PVRSRV_OK;
+
+fail_contextalloc:
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps2DData->psBufferSyncContext);
+	ps2DData->psBufferSyncContext = NULL;
+fail_buffer_sync_context_create:
+#endif
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+
+static PVRSRV_ERROR _Destroy2DTransferContext(RGX_SERVER_TQ_2D_DATA *ps2DData,
+											  PVRSRV_DEVICE_NODE *psDeviceNode,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps2DData->psServerCommonContext,
+											  RGXFWIF_DM_2D,
+											  ui32PDumpFlags);
+
+	RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode,
+						   eError,
+						   RGXFWRequestCommonContextCleanUp);
+
+	/* ... it has so we can free its resources */
+	FWCommonContextFree(ps2DData->psServerCommonContext);
+	ps2DData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps2DData->psBufferSyncContext);
+	ps2DData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+#endif /* #if defined(RGX_FEATURE_TLA_BIT_MASK) */
+
+static PVRSRV_ERROR _Destroy3DTransferContext(RGX_SERVER_TQ_3D_DATA *ps3DData,
+											  PVRSRV_DEVICE_NODE *psDeviceNode,
+											  IMG_UINT32 ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError;
+
+	/* Check if the FW has finished with this resource ... */
+	eError = RGXFWRequestCommonContextCleanUp(psDeviceNode,
+											  ps3DData->psServerCommonContext,
+											  RGXFWIF_DM_3D,
+											  ui32PDumpFlags);
+	if (PVRSRVIsRetryError(eError))
+	{
+		return eError;
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_LOG(("%s: Unexpected error from RGXFWRequestCommonContextCleanUp (%s)",
+				 __func__,
+				 PVRSRVGetErrorString(eError)));
+	}
+
+	/* ... it has so we can free its resources */
+	DevmemFwUnmapAndFree(psDeviceNode->pvDevice, ps3DData->psFWContextStateMemDesc);
+	FWCommonContextFree(ps3DData->psServerCommonContext);
+	ps3DData->psServerCommonContext = NULL;
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	pvr_buffer_sync_context_destroy(ps3DData->psBufferSyncContext);
+	ps3DData->psBufferSyncContext = NULL;
+#endif
+
+	return eError;
+}
+
+
+/*
+ * PVRSRVCreateTransferContextKM
+ */
+PVRSRV_ERROR PVRSRVRGXCreateTransferContextKM(CONNECTION_DATA		*psConnection,
+										   PVRSRV_DEVICE_NODE		*psDeviceNode,
+										   IMG_INT32				i32Priority,
+										   IMG_UINT32				ui32FrameworkCommandSize,
+										   IMG_PBYTE				pabyFrameworkCommand,
+										   IMG_HANDLE				hMemCtxPrivData,
+										   IMG_UINT32				ui32PackedCCBSizeU8888,
+										   IMG_UINT32				ui32ContextFlags,
+										   IMG_UINT64				ui64RobustnessAddress,
+										   RGX_SERVER_TQ_CONTEXT	**ppsTransferContext)
+{
+	RGX_SERVER_TQ_CONTEXT	*psTransferContext;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	DEVMEM_MEMDESC			*psFWMemContextMemDesc = RGXGetFWMemDescFromMemoryContextHandle(hMemCtxPrivData);
+	RGX_COMMON_CONTEXT_INFO	sInfo = {NULL};
+	PVRSRV_ERROR			eError = PVRSRV_OK;
+
+	/* Allocate the server side structure */
+	*ppsTransferContext = NULL;
+	psTransferContext = OSAllocZMem(sizeof(*psTransferContext));
+	if (psTransferContext == NULL)
+	{
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	/*
+		Create the FW transfer context, this has the TQ common
+		context embedded within it
+	 */
+	eError = DevmemFwAllocate(psDevInfo,
+			sizeof(RGXFWIF_FWTRANSFERCONTEXT),
+			RGX_FWCOMCTX_ALLOCFLAGS,
+			"FwTransferContext",
+			&psTransferContext->psFWTransferContextMemDesc);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_fwtransfercontext;
+	}
+
+	eError = OSLockCreate(&psTransferContext->hLock);
+
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create lock (%s)",
+									__func__,
+									PVRSRVGetErrorString(eError)));
+		goto fail_createlock;
+	}
+
+	psTransferContext->psDeviceNode = psDeviceNode;
+
+	if (ui32FrameworkCommandSize)
+	{
+		/*
+		 * Create the FW framework buffer
+		 */
+		eError = PVRSRVRGXFrameworkCreateKM(psDeviceNode,
+				&psTransferContext->psFWFrameworkMemDesc,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to allocate firmware GPU framework state (%s)",
+						__func__,
+						PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcreate;
+		}
+
+		/* Copy the Framework client data into the framework buffer */
+		eError = PVRSRVRGXFrameworkCopyCommand(psDeviceNode,
+				psTransferContext->psFWFrameworkMemDesc,
+				pabyFrameworkCommand,
+				ui32FrameworkCommandSize);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+						"%s: Failed to populate the framework buffer (%s)",
+						__func__,
+				PVRSRVGetErrorString(eError)));
+			goto fail_frameworkcopy;
+		}
+
+		sInfo.psFWFrameworkMemDesc = psTransferContext->psFWFrameworkMemDesc;
+	}
+
+	eError = _Create3DTransferContext(psConnection,
+									  psDeviceNode,
+									  psTransferContext->psFWTransferContextMemDesc,
+									  offsetof(RGXFWIF_FWTRANSFERCONTEXT, sTQContext),
+									  psFWMemContextMemDesc,
+									  i32Priority,
+									  &sInfo,
+									  &psTransferContext->s3DData,
+									  U32toU8_Unpack3(ui32PackedCCBSizeU8888),
+									  U32toU8_Unpack4(ui32PackedCCBSizeU8888),
+									  ui32ContextFlags,
+									  ui64RobustnessAddress);
+	if (eError != PVRSRV_OK)
+	{
+		goto fail_3dtransfercontext;
+	}
+	psTransferContext->ui32Flags |= RGX_SERVER_TQ_CONTEXT_FLAGS_3D;
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA))
+	{
+		eError = _Create2DTransferContext(psConnection,
+										  psDeviceNode,
+										  psFWMemContextMemDesc,
+										  i32Priority,
+										  &sInfo,
+										  &psTransferContext->s2DData,
+										  U32toU8_Unpack1(ui32PackedCCBSizeU8888),
+										  U32toU8_Unpack2(ui32PackedCCBSizeU8888),
+										  ui32ContextFlags,
+										  ui64RobustnessAddress);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_2dtransfercontext;
+		}
+		psTransferContext->ui32Flags |= RGX_SERVER_TQ_CONTEXT_FLAGS_2D;
+	}
+#endif
+
+	{
+		PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+
+		OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+		dllist_add_to_tail(&(psDevInfo->sTransferCtxtListHead), &(psTransferContext->sListNode));
+		OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+		*ppsTransferContext = psTransferContext;
+	}
+
+	return PVRSRV_OK;
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+fail_2dtransfercontext:
+	_Destroy3DTransferContext(&psTransferContext->s3DData,
+							  psTransferContext->psDeviceNode,
+							  psTransferContext->ui32PDumpFlags);
+#endif
+fail_3dtransfercontext:
+fail_frameworkcopy:
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+fail_frameworkcreate:
+	OSLockDestroy(psTransferContext->hLock);
+fail_createlock:
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+fail_fwtransfercontext:
+	OSFreeMem(psTransferContext);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	*ppsTransferContext = NULL;
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXDestroyTransferContextKM(RGX_SERVER_TQ_CONTEXT *psTransferContext)
+{
+	PVRSRV_ERROR eError;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psTransferContext->psDeviceNode->pvDevice;
+	IMG_UINT32 i;
+
+	/* remove node from list before calling destroy - as destroy, if successful
+	 * will invalidate the node
+	 * must be re-added if destroy fails
+	 */
+	OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+	dllist_remove_node(&(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((psTransferContext->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+			(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = _Destroy2DTransferContext(&psTransferContext->s2DData,
+										   psTransferContext->psDeviceNode,
+										   PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_destroy2d;
+		}
+		/* We've freed the 2D context, don't try to free it again */
+		psTransferContext->ui32Flags &= ~RGX_SERVER_TQ_CONTEXT_FLAGS_2D;
+	}
+#endif
+
+	if (psTransferContext->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D)
+	{
+		eError = _Destroy3DTransferContext(&psTransferContext->s3DData,
+										   psTransferContext->psDeviceNode,
+										   PDUMP_FLAGS_CONTINUOUS);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_destroy3d;
+		}
+		/* We've freed the 3D context, don't try to free it again */
+		psTransferContext->ui32Flags &= ~RGX_SERVER_TQ_CONTEXT_FLAGS_3D;
+	}
+
+	/* free any resources within the per-prepare UFO address stores */
+	for (i = 0; i < TQ_MAX_PREPARES_PER_SUBMIT; i++)
+	{
+		SyncAddrListDeinit(&psTransferContext->asSyncAddrListFence[i]);
+		SyncAddrListDeinit(&psTransferContext->asSyncAddrListUpdate[i]);
+	}
+
+	if (psTransferContext->psFWFrameworkMemDesc)
+	{
+		DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWFrameworkMemDesc);
+	}
+
+	DevmemFwUnmapAndFree(psDevInfo, psTransferContext->psFWTransferContextMemDesc);
+
+	OSLockDestroy(psTransferContext->hLock);
+
+	OSFreeMem(psTransferContext);
+
+	return PVRSRV_OK;
+
+fail_destroy3d:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+
+fail_destroy2d:
+#endif
+	OSWRLockAcquireWrite(psDevInfo->hTransferCtxListLock);
+	dllist_add_to_tail(&(psDevInfo->sTransferCtxtListHead), &(psTransferContext->sListNode));
+	OSWRLockReleaseWrite(psDevInfo->hTransferCtxListLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/*
+ * PVRSRVSubmitTQ3DKickKM
+ */
+/* Old bridge call for backwards compatibility. */
+PVRSRV_ERROR PVRSRVRGXSubmitTransferKM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									   IMG_UINT32				ui32PrepareCount,
+									   IMG_UINT32				*paui32ClientUpdateCount,
+									   SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									   IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									   IMG_UINT32				**papaui32ClientUpdateValue,
+									   PVRSRV_FENCE				iCheckFence,
+									   PVRSRV_TIMELINE			i2DUpdateTimeline,
+									   PVRSRV_FENCE				*pi2DUpdateFence,
+									   PVRSRV_TIMELINE			i3DUpdateTimeline,
+									   PVRSRV_FENCE				*pi3DUpdateFence,
+									   IMG_CHAR					szFenceName[32],
+									   IMG_UINT32				*paui32FWCommandSize,
+									   IMG_UINT8				**papaui8FWCommand,
+									   IMG_UINT32				*pui32TQPrepareFlags,
+									   IMG_UINT32				ui32ExtJobRef,
+									   IMG_UINT32				ui32SyncPMRCount,
+									   IMG_UINT32				*paui32SyncPMRFlags,
+									   PMR						**ppsSyncPMRs)
+{
+	return PVRSRVRGXSubmitTransfer3KM(psTransferContext,
+									  ui32PrepareCount,
+									  paui32ClientUpdateCount,
+									  papauiClientUpdateUFODevVarBlock,
+									  papaui32ClientUpdateSyncOffset,
+									  papaui32ClientUpdateValue,
+									  iCheckFence,
+									  i2DUpdateTimeline,
+									  pi2DUpdateFence,
+									  i3DUpdateTimeline,
+									  pi3DUpdateFence,
+									  szFenceName,
+									  PVRSRV_NO_FENCE,
+									  paui32FWCommandSize,
+									  papaui8FWCommand,
+									  pui32TQPrepareFlags,
+									  ui32ExtJobRef,
+									  ui32SyncPMRCount,
+									  paui32SyncPMRFlags,
+									  ppsSyncPMRs);
+}
+
+PVRSRV_ERROR PVRSRVRGXSubmitTransfer3KM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									   IMG_UINT32				ui32PrepareCount,
+									   IMG_UINT32				*paui32ClientUpdateCount,
+									   SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									   IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									   IMG_UINT32				**papaui32ClientUpdateValue,
+									   PVRSRV_FENCE				iCheckFence,
+									   PVRSRV_TIMELINE			i2DUpdateTimeline,
+									   PVRSRV_FENCE				*pi2DUpdateFence,
+									   PVRSRV_TIMELINE			i3DUpdateTimeline,
+									   PVRSRV_FENCE				*pi3DUpdateFence,
+									   IMG_CHAR					szFenceName[32],
+									   PVRSRV_FENCE				iExportFenceToSignal,
+									   IMG_UINT32				*paui32FWCommandSize,
+									   IMG_UINT8				**papaui8FWCommand,
+									   IMG_UINT32				*pui32TQPrepareFlags,
+									   IMG_UINT32				ui32ExtJobRef,
+									   IMG_UINT32				ui32SyncPMRCount,
+									   IMG_UINT32				*paui32SyncPMRFlags,
+									   PMR						**ppsSyncPMRs)
+{
+	PVRSRV_DEVICE_NODE *psDeviceNode = psTransferContext->psDeviceNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
+	RGX_CCB_CMD_HELPER_DATA *pas3DCmdHelper;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	RGX_CCB_CMD_HELPER_DATA *pas2DCmdHelper;
+#endif
+	IMG_UINT32                ui323DCmdCount = 0;
+	IMG_UINT32                ui323DCmdLast = 0;
+	__maybe_unused IMG_UINT32 ui323DCmdOffset = 0;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	IMG_UINT32                ui322DCmdCount = 0;
+	IMG_UINT32                ui322DCmdLast = 0;
+	__maybe_unused IMG_UINT32 ui322DCmdOffset = 0;
+#endif
+	IMG_UINT32 ui32PDumpFlags = PDUMP_FLAGS_NONE;
+	IMG_UINT32 i;
+	IMG_UINT64 uiCheckFenceUID = 0;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	IMG_UINT64 ui2DUpdateFenceUID = 0;
+#endif
+	IMG_UINT64 ui3DUpdateFenceUID = 0;
+
+	PSYNC_CHECKPOINT ps3DUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT ps3DExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *ppsExportFenceSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT *apsFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32FenceSyncCheckpointCount = 0;
+	IMG_UINT32 *pui323DIntAllocatedUpdateValues = NULL;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PSYNC_CHECKPOINT ps2DUpdateSyncCheckpoint = NULL;
+	PSYNC_CHECKPOINT ps2DExportFenceSyncCheckpoint = NULL;
+	IMG_UINT32 *pui322DIntAllocatedUpdateValues = NULL;
+	PVRSRV_CLIENT_SYNC_PRIM *ps2DFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui322DFenceTimelineUpdateValue = 0;
+	void *pv2DUpdateFenceFinaliseData = NULL;
+#endif
+	PVRSRV_CLIENT_SYNC_PRIM *ps3DFenceTimelineUpdateSync = NULL;
+	IMG_UINT32 ui323DFenceTimelineUpdateValue = 0;
+	void *pv3DUpdateFenceFinaliseData = NULL;
+#if defined(SUPPORT_BUFFER_SYNC)
+	PSYNC_CHECKPOINT psBufferUpdateSyncCheckpoint = NULL;
+	struct pvr_buffer_sync_append_data *psBufferSyncData = NULL;
+	PSYNC_CHECKPOINT *apsBufferFenceSyncCheckpoints = NULL;
+	IMG_UINT32 ui32BufferFenceSyncCheckpointCount = 0;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVRSRV_ERROR eError2;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVRSRV_FENCE i2DUpdateFence = PVRSRV_NO_FENCE;
+#endif
+	PVRSRV_FENCE i3DUpdateFence = PVRSRV_NO_FENCE;
+	IMG_UINT32   ui32IntJobRef = OSAtomicIncrement(&psDevInfo->iCCBSubmissionOrdinal);
+	IMG_UINT32   ui32PreparesDone = 0;
+
+
+	PRGXFWIF_TIMESTAMP_ADDR pPreAddr;
+	PRGXFWIF_TIMESTAMP_ADDR pPostAddr;
+	PRGXFWIF_UFO_ADDR       pRMWUFOAddr;
+
+#if !defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVR_UNREFERENCED_PARAMETER(i2DUpdateTimeline);
+	PVR_UNREFERENCED_PARAMETER(pi2DUpdateFence);
+#endif
+
+	RGX_GetTimestampCmdHelper((PVRSRV_RGXDEV_INFO*) psDeviceNode->pvDevice,
+	                          &pPreAddr,
+	                          &pPostAddr,
+	                          &pRMWUFOAddr);
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (i2DUpdateTimeline != PVRSRV_NO_TIMELINE && !pi2DUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+#endif
+	if (i3DUpdateTimeline != PVRSRV_NO_TIMELINE && !pi3DUpdateFence)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Validate sync prim fence/update value ptrs
+	 * for each prepare.
+	 */
+	{
+		IMG_UINT32 ui32Prepare;
+		IMG_UINT32 *pui32UpdateCount = paui32ClientUpdateCount;
+		IMG_UINT32 **papui32UpdateValue = papaui32ClientUpdateValue;
+
+		/* Check that we have not been given a null ptr for
+		 * update count parameters.
+		 */
+		PVR_LOG_RETURN_IF_FALSE((paui32ClientUpdateCount != NULL),
+		                        "paui32ClientUpdateCount NULL",
+		                        PVRSRV_ERROR_INVALID_PARAMS);
+
+		for (ui32Prepare=0; ui32Prepare<ui32PrepareCount; ui32Prepare++)
+		{
+			/* Ensure we haven't been given a null ptr to
+			 * update values if we have been told we
+			 * have updates for this prepare
+			 */
+			if (*pui32UpdateCount > 0)
+			{
+				PVR_LOG_RETURN_IF_FALSE(*papui32UpdateValue != NULL,
+				                        "paui32ClientUpdateValue NULL but "
+				                        "ui32ClientUpdateCount > 0",
+				                        PVRSRV_ERROR_INVALID_PARAMS);
+			}
+			/* Advance local ptr to update values ptr for next prepare. */
+			papui32UpdateValue++;
+			/* Advance local ptr to update count for next prepare. */
+			pui32UpdateCount++;
+		}
+	}
+
+	/* Ensure the string is null-terminated (Required for safety) */
+	szFenceName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+
+	if ((ui32PrepareCount == 0) || (ui32PrepareCount > TQ_MAX_PREPARES_PER_SUBMIT))
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (ui32SyncPMRCount != 0)
+	{
+		if (!ppsSyncPMRs)
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+
+#if defined(SUPPORT_BUFFER_SYNC)
+		/* PMR sync is valid only when there is no batching */
+		if ((ui32PrepareCount != 1))
+#endif
+		{
+			return PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+	OSLockAcquire(psTransferContext->hLock);
+
+	/* We can't allocate the required amount of stack space on all consumer architectures */
+	pas3DCmdHelper = OSAllocMem(sizeof(*pas3DCmdHelper) * ui32PrepareCount);
+	if (pas3DCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc3dhelper;
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	pas2DCmdHelper = OSAllocMem(sizeof(*pas2DCmdHelper) * ui32PrepareCount);
+	if (pas2DCmdHelper == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc2dhelper;
+	}
+#endif
+
+	if (iCheckFence != PVRSRV_NO_FENCE)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointResolveFence (iCheckFence=%d), psDeviceNode->hSyncCheckpointContext=<%p>...", __func__, iCheckFence, (void*)psDeviceNode->hSyncCheckpointContext));
+		/* Resolve the sync checkpoints that make up the input fence */
+		eError = SyncCheckpointResolveFence(psDeviceNode->hSyncCheckpointContext,
+											iCheckFence,
+											&ui32FenceSyncCheckpointCount,
+											&apsFenceSyncCheckpoints,
+											&uiCheckFenceUID,
+											ui32PDumpFlags);
+		if (eError != PVRSRV_OK)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, returned ERROR (eError=%d)", __func__, eError));
+			goto fail_resolve_fencesync_input_fence;
+		}
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: ...done, fence %d contained %d checkpoints (apsFenceSyncCheckpoints=<%p>)", __func__, iCheckFence, ui32FenceSyncCheckpointCount, (void*)apsFenceSyncCheckpoints));
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+		if (ui32FenceSyncCheckpointCount > 0)
+		{
+			IMG_UINT32 ii;
+			for (ii=0; ii<ui32FenceSyncCheckpointCount; ii++)
+			{
+				PSYNC_CHECKPOINT psNextCheckpoint = *(apsFenceSyncCheckpoints + ii);
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:    apsFenceSyncCheckpoints[%d]=<%p>", __func__, ii, (void*)psNextCheckpoint));
+			}
+		}
+#endif
+	}
+	/*
+		Ensure we do the right thing for server syncs which cross call boundaries
+	*/
+	for (i=0;i<ui32PrepareCount;i++)
+	{
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 3D))
+		{
+			ui323DCmdLast++;
+		}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		else if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			ui322DCmdLast++;
+		}
+#endif
+	}
+
+	/*
+		Init the command helper commands for all the prepares
+	*/
+	for (i=0;i<ui32PrepareCount;i++)
+	{
+		RGX_CLIENT_CCB *psClientCCB;
+		RGX_SERVER_COMMON_CONTEXT *psServerCommonCtx;
+		IMG_CHAR *pszCommandName;
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper;
+		RGXFWIF_CCB_CMD_TYPE eType;
+		PRGXFWIF_UFO_ADDR *pauiIntFenceUFOAddress = NULL;
+		PRGXFWIF_UFO_ADDR *pauiIntUpdateUFOAddress = NULL;
+		SYNC_ADDR_LIST *psSyncAddrListFence = &psTransferContext->asSyncAddrListFence[i];
+		SYNC_ADDR_LIST *psSyncAddrListUpdate = &psTransferContext->asSyncAddrListUpdate[i];
+		IMG_UINT32 ui32IntClientFenceCount = 0U;
+		IMG_UINT32 ui32IntClientUpdateCount = paui32ClientUpdateCount[i];
+		IMG_UINT32 *paui32IntUpdateValue = papaui32ClientUpdateValue[i];
+#if defined(SUPPORT_BUFFER_SYNC)
+		struct pvr_buffer_sync_context *psBufferSyncContext;
+#endif
+
+		PVRSRV_FENCE *piUpdateFence = NULL;
+		PVRSRV_TIMELINE	iUpdateTimeline = PVRSRV_NO_TIMELINE;
+		void **ppvUpdateFenceFinaliseData = NULL;
+		PSYNC_CHECKPOINT * ppsUpdateSyncCheckpoint = NULL;
+		PVRSRV_CLIENT_SYNC_PRIM **ppsFenceTimelineUpdateSync = NULL;
+		IMG_UINT32 *pui32FenceTimelineUpdateValue = NULL;
+		IMG_UINT32 **ppui32IntAllocatedUpdateValues = NULL;
+		IMG_BOOL bCheckFence = IMG_FALSE;
+		IMG_BOOL bUpdateFence = IMG_FALSE;
+		IMG_BOOL bExportFence = IMG_FALSE;
+		IMG_UINT64 *puiUpdateFenceUID = NULL;
+
+		IMG_BOOL bCCBStateOpen = IMG_FALSE;
+
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 3D))
+		{
+			psServerCommonCtx = psTransferContext->s3DData.psServerCommonContext;
+			psClientCCB = FWCommonContextGetClientCCB(psServerCommonCtx);
+			pszCommandName = "TQ-3D";
+			psCmdHelper = &pas3DCmdHelper[ui323DCmdCount++];
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_3D;
+#if defined(SUPPORT_BUFFER_SYNC)
+			psBufferSyncContext = psTransferContext->s3DData.psBufferSyncContext;
+#endif
+			bCheckFence = ui323DCmdCount == 1;
+			bUpdateFence = ui323DCmdCount == ui323DCmdLast
+				&& i3DUpdateTimeline != PVRSRV_NO_TIMELINE;
+			bExportFence = ui323DCmdCount == ui323DCmdLast
+				&& iExportFenceToSignal != PVRSRV_NO_FENCE;
+
+			if (bUpdateFence)
+			{
+				piUpdateFence = &i3DUpdateFence;
+				iUpdateTimeline = i3DUpdateTimeline;
+				ppvUpdateFenceFinaliseData = &pv3DUpdateFenceFinaliseData;
+				ppsUpdateSyncCheckpoint = &ps3DUpdateSyncCheckpoint;
+				ppsFenceTimelineUpdateSync = &ps3DFenceTimelineUpdateSync;
+				pui32FenceTimelineUpdateValue = &ui323DFenceTimelineUpdateValue;
+				ppui32IntAllocatedUpdateValues = &pui323DIntAllocatedUpdateValues;
+				puiUpdateFenceUID = &ui3DUpdateFenceUID;
+			}
+
+			if (bExportFence)
+			{
+				ppsExportFenceSyncCheckpoint = &ps3DExportFenceSyncCheckpoint;
+			}
+		}
+		else
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if (TQ_PREP_FLAGS_COMMAND_IS(pui32TQPrepareFlags[i], 2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			psServerCommonCtx = psTransferContext->s2DData.psServerCommonContext;
+			psClientCCB = FWCommonContextGetClientCCB(psServerCommonCtx);
+			pszCommandName = "TQ-2D";
+			psCmdHelper = &pas2DCmdHelper[ui322DCmdCount++];
+			eType = RGXFWIF_CCB_CMD_TYPE_TQ_2D;
+#if defined(SUPPORT_BUFFER_SYNC)
+			psBufferSyncContext = psTransferContext->s2DData.psBufferSyncContext;
+#endif
+			bCheckFence = ui322DCmdCount == 1;
+			bUpdateFence = ui322DCmdCount == ui322DCmdLast
+				&& i2DUpdateTimeline != PVRSRV_NO_TIMELINE;
+			bExportFence = ui322DCmdCount == ui322DCmdLast
+				&& iExportFenceToSignal != PVRSRV_NO_FENCE;
+
+			if (bUpdateFence)
+			{
+				piUpdateFence = &i2DUpdateFence;
+				iUpdateTimeline = i2DUpdateTimeline;
+				ppvUpdateFenceFinaliseData = &pv2DUpdateFenceFinaliseData;
+				ppsUpdateSyncCheckpoint = &ps2DUpdateSyncCheckpoint;
+				ppsFenceTimelineUpdateSync = &ps2DFenceTimelineUpdateSync;
+				pui32FenceTimelineUpdateValue = &ui322DFenceTimelineUpdateValue;
+				ppui32IntAllocatedUpdateValues = &pui322DIntAllocatedUpdateValues;
+				puiUpdateFenceUID = &ui2DUpdateFenceUID;
+			}
+
+			if (bExportFence)
+			{
+				ppsExportFenceSyncCheckpoint = &ps2DExportFenceSyncCheckpoint;
+			}
+		}
+		else
+#endif
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto fail_prepare_loop;
+		}
+
+		if (i == 0)
+		{
+			ui32PDumpFlags = ((pui32TQPrepareFlags[i] & TQ_PREP_FLAGS_PDUMPCONTINUOUS) != 0) ? PDUMP_FLAGS_CONTINUOUS : PDUMP_FLAGS_NONE;
+			PDUMPCOMMENTWITHFLAGS(psDeviceNode, ui32PDumpFlags,
+					"%s Command Server Submit on FWCtx %08x", pszCommandName, FWCommonContextGetFWAddress(psServerCommonCtx).ui32Addr);
+			psTransferContext->ui32PDumpFlags |= ui32PDumpFlags;
+		}
+		else
+		{
+			IMG_UINT32 ui32NewPDumpFlags = ((pui32TQPrepareFlags[i] & TQ_PREP_FLAGS_PDUMPCONTINUOUS) != 0) ? PDUMP_FLAGS_CONTINUOUS : PDUMP_FLAGS_NONE;
+			if (ui32NewPDumpFlags != ui32PDumpFlags)
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+				PVR_DPF((PVR_DBG_ERROR, "%s: Mixing of continuous and non-continuous command in a batch is not permitted", __func__));
+				goto fail_prepare_loop;
+			}
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncAddrListPopulate(psTransferContext->sSyncAddrListFence, %d fences)", __func__, ui32IntClientFenceCount));
+		eError = SyncAddrListPopulate(psSyncAddrListFence,
+										0,
+										NULL,
+										NULL);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_prepare_loop;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncAddrListPopulate(psTransferContext->asSyncAddrListUpdate[], %d updates)", __func__, ui32IntClientUpdateCount));
+		eError = SyncAddrListPopulate(psSyncAddrListUpdate,
+										ui32IntClientUpdateCount,
+										papauiClientUpdateUFODevVarBlock[i],
+										papaui32ClientUpdateSyncOffset[i]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_prepare_loop;
+		}
+		if (!pauiIntUpdateUFOAddress)
+		{
+			pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after sync prims) ui32IntClientUpdateCount=%d", __func__, ui32IntClientUpdateCount));
+		if (ui32SyncPMRCount)
+		{
+#if defined(SUPPORT_BUFFER_SYNC)
+			int err;
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Calling pvr_buffer_sync_resolve_and_create_fences", __func__));
+			err = pvr_buffer_sync_resolve_and_create_fences(psBufferSyncContext,
+			                                                psTransferContext->psDeviceNode->hSyncCheckpointContext,
+			                                                ui32SyncPMRCount,
+			                                                ppsSyncPMRs,
+			                                                paui32SyncPMRFlags,
+			                                                &ui32BufferFenceSyncCheckpointCount,
+			                                                &apsBufferFenceSyncCheckpoints,
+			                                                &psBufferUpdateSyncCheckpoint,
+			                                                &psBufferSyncData);
+			if (err)
+			{
+				switch (err)
+				{
+					case -EINTR:
+						eError = PVRSRV_ERROR_RETRY;
+						break;
+					case -ENOMEM:
+						eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+						break;
+					default:
+						eError = PVRSRV_ERROR_INVALID_PARAMS;
+						break;
+				}
+
+				if (eError != PVRSRV_ERROR_RETRY)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   pvr_buffer_sync_resolve_and_create_fences failed (%s)", __func__, PVRSRVGetErrorString(eError)));
+				}
+				goto fail_resolve_buffersync_input_fence;
+			}
+
+			/* Append buffer sync fences */
+			if (ui32BufferFenceSyncCheckpointCount > 0)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d buffer sync checkpoints to TQ Fence (psSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>)...", __func__, ui32BufferFenceSyncCheckpointCount, (void*)psSyncAddrListFence , (void*)pauiIntFenceUFOAddress));
+				SyncAddrListAppendAndDeRefCheckpoints(psSyncAddrListFence,
+													  ui32BufferFenceSyncCheckpointCount,
+													  apsBufferFenceSyncCheckpoints);
+				if (!pauiIntFenceUFOAddress)
+				{
+					pauiIntFenceUFOAddress = psSyncAddrListFence->pasFWAddrs;
+				}
+				ui32IntClientFenceCount += ui32BufferFenceSyncCheckpointCount;
+			}
+
+			if (psBufferUpdateSyncCheckpoint)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 buffer sync checkpoint<%p> to TQ Update (&psTransferContext->asSyncAddrListUpdate[i]=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)psBufferUpdateSyncCheckpoint, (void*)psSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+				/* Append the update (from output fence) */
+				SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+											  1,
+											  &psBufferUpdateSyncCheckpoint);
+				if (!pauiIntUpdateUFOAddress)
+				{
+					pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+				}
+				ui32IntClientUpdateCount++;
+			}
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after buffer_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+#else /* defined(SUPPORT_BUFFER_SYNC) */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Buffer sync not supported but got %u buffers", __func__, ui32SyncPMRCount));
+			PVR_DPF((PVR_DBG_ERROR, "%s:   <--EXIT(%d)", __func__, PVRSRV_ERROR_INVALID_PARAMS));
+			OSLockRelease(psTransferContext->hLock);
+			return PVRSRV_ERROR_INVALID_PARAMS;
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+		}
+
+		/* Create the output fence (if required) */
+		if (bUpdateFence)
+		{
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: calling SyncCheckpointCreateFence (piUpdateFence=%p, iUpdateTimeline=%d,  psTranserContext->psDeviceNode->hSyncCheckpointContext=<%p>)", __func__, piUpdateFence, iUpdateTimeline, (void*)psDeviceNode->hSyncCheckpointContext));
+			eError = SyncCheckpointCreateFence(psDeviceNode,
+			                                   szFenceName,
+			                                   iUpdateTimeline,
+			                                   psDeviceNode->hSyncCheckpointContext,
+			                                   piUpdateFence,
+			                                   puiUpdateFenceUID,
+			                                   ppvUpdateFenceFinaliseData,
+			                                   ppsUpdateSyncCheckpoint,
+			                                   (void*)ppsFenceTimelineUpdateSync,
+			                                   pui32FenceTimelineUpdateValue,
+			                                   ui32PDumpFlags);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+						"%s:   SyncCheckpointCreateFence failed (%s)",
+						__func__,
+						PVRSRVGetErrorString(eError)));
+				goto fail_prepare_loop;
+			}
+
+			CHKPT_DBG((PVR_DBG_ERROR, "%s: returned from SyncCheckpointCreateFence (piUpdateFence=%p)", __func__, piUpdateFence));
+
+			/* Resolve the iExportFenceToSignal (if required) */
+			if (bExportFence)
+			{
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: SyncCheckpointResolveExportFence(iExportFenceToSignal=%d), ui32FenceSyncCheckpointCount=%d", __func__, iExportFenceToSignal, ui32FenceSyncCheckpointCount));
+				eError = SyncCheckpointResolveExportFence(iExportFenceToSignal,
+														  psDeviceNode->hSyncCheckpointContext,
+														  ppsExportFenceSyncCheckpoint,
+														  ui32PDumpFlags);
+				if (eError != PVRSRV_OK)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: ...returned error (%s) psExportFenceSyncCheckpoint=<%p>", __func__, PVRSRVGetErrorString(eError), *ppsExportFenceSyncCheckpoint));
+					goto fail_prepare_loop;
+				}
+
+				/* Check that the export fence was not also included as part of the
+				 * check fence (which is an error and would lead to a stalled kick).
+				 */
+				if (ui32FenceSyncCheckpointCount > 0)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s:   Checking export fence is not part of check fence...", __func__));
+					CHKPT_DBG((PVR_DBG_ERROR, "%s:   ui32FenceSyncCheckpointCount=%d",
+							   __func__, ui32FenceSyncCheckpointCount));
+					if (ui32FenceSyncCheckpointCount > 0)
+					{
+						IMG_UINT32 iii;
+
+						for (iii=0; iii<ui32FenceSyncCheckpointCount; iii++)
+						{
+							CHKPT_DBG((PVR_DBG_ERROR, "%s: apsFenceSyncCheckpoints[%d]=<%p>, FWAddr=0x%x", __func__, iii, apsFenceSyncCheckpoints[iii], SyncCheckpointGetFirmwareAddr(apsFenceSyncCheckpoints[iii])));
+							if (apsFenceSyncCheckpoints[iii] == *ppsExportFenceSyncCheckpoint)
+							{
+								CHKPT_DBG((PVR_DBG_ERROR, "%s: ERROR psExportFenceSyncCheckpoint=<%p>", __func__, *ppsExportFenceSyncCheckpoint));
+								eError = PVRSRV_ERROR_INVALID_PARAMS;
+								PVR_DPF((PVR_DBG_ERROR, " %s - iCheckFence includes iExportFenceToSignal", PVRSRVGetErrorString(eError)));
+								goto fail_prepare_loop;
+							}
+						}
+					}
+				}
+			}
+
+			/* Append the sync prim update for the timeline (if required) */
+			if (*ppsFenceTimelineUpdateSync)
+			{
+				IMG_UINT32 *pui32TimelineUpdateWp = NULL;
+
+				/* Allocate memory to hold the list of update values (including our timeline update) */
+				*ppui32IntAllocatedUpdateValues = OSAllocMem(sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+				if (!*ppui32IntAllocatedUpdateValues)
+				{
+					/* Failed to allocate memory */
+					eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+					goto fail_prepare_loop;
+				}
+				OSCachedMemSet(*ppui32IntAllocatedUpdateValues, 0xbb, sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount+1));
+#if defined(SUPPORT_BUFFER_SYNC)
+				if (psBufferUpdateSyncCheckpoint)
+				{
+					/* Copy the update values into the new memory, then append our timeline update value */
+					OSCachedMemCopy(*ppui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(**ppui32IntAllocatedUpdateValues) * (ui32IntClientUpdateCount-1));
+					pui32TimelineUpdateWp = *ppui32IntAllocatedUpdateValues + (ui32IntClientUpdateCount-1);
+				}
+				else
+#endif
+				{
+					/* Copy the update values into the new memory, then append our timeline update value */
+					OSCachedMemCopy(*ppui32IntAllocatedUpdateValues, paui32IntUpdateValue, sizeof(**ppui32IntAllocatedUpdateValues) * ui32IntClientUpdateCount);
+					pui32TimelineUpdateWp = *ppui32IntAllocatedUpdateValues + ui32IntClientUpdateCount;
+				}
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: Appending the additional update value 0x%x)", __func__, *pui32FenceTimelineUpdateValue));
+				/* Now set the additional update value */
+				*pui32TimelineUpdateWp = *pui32FenceTimelineUpdateValue;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+				if (ui32IntClientUpdateCount > 0)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)*ppui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: *ppui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Now append the timeline sync prim addr to the transfer context update list */
+				SyncAddrListAppendSyncPrim(psSyncAddrListUpdate,
+				                           *ppsFenceTimelineUpdateSync);
+				ui32IntClientUpdateCount++;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+				if (ui32IntClientUpdateCount > 0)
+				{
+					IMG_UINT32 iii;
+					IMG_UINT32 *pui32Tmp = (IMG_UINT32*)*ppui32IntAllocatedUpdateValues;
+
+					for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+					{
+						CHKPT_DBG((PVR_DBG_ERROR, "%s: *ppui32IntAllocatedUpdateValues[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+						pui32Tmp++;
+					}
+				}
+#endif
+				/* Ensure paui32IntUpdateValue is now pointing to our new array of update values */
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: set paui32IntUpdateValue<%p> to point to *ppui32IntAllocatedUpdateValues<%p>", __func__, (void*)paui32IntUpdateValue, (void*)*ppui32IntAllocatedUpdateValues));
+				paui32IntUpdateValue = *ppui32IntAllocatedUpdateValues;
+			}
+		}
+
+		if (bCheckFence && ui32FenceSyncCheckpointCount)
+		{
+			/* Append the checks (from input fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append %d sync checkpoints to TQ Fence (psSyncAddrListFence=<%p>)...", __func__, ui32FenceSyncCheckpointCount, (void*)psSyncAddrListFence));
+			SyncAddrListAppendCheckpoints(psSyncAddrListFence,
+										  ui32FenceSyncCheckpointCount,
+										  apsFenceSyncCheckpoints);
+			if (!pauiIntFenceUFOAddress)
+			{
+				pauiIntFenceUFOAddress = psSyncAddrListFence->pasFWAddrs;
+			}
+			ui32IntClientFenceCount += ui32FenceSyncCheckpointCount;
+
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+			if (ui32IntClientFenceCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntFenceUFOAddress;
+
+				for (iii=0; iii<ui32IntClientFenceCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: psSyncAddrListFence->pasFWAddrs[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+		if (bUpdateFence && *ppsUpdateSyncCheckpoint)
+		{
+			/* Append the update (from output fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to TQ Update (psSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>)...", __func__, (void*)&psTransferContext->asSyncAddrListUpdate , (void*)pauiIntUpdateUFOAddress));
+			SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+										  1,
+										  ppsUpdateSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(TRANSFER_CHECKPOINT_DEBUG)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		if (bExportFence && *ppsExportFenceSyncCheckpoint)
+		{
+			/* Append the update (from export fence) */
+			CHKPT_DBG((PVR_DBG_ERROR, "%s:   Append 1 sync checkpoint to Transfer Update (psSyncAddrListUpdate=<%p>, psExportFenceSyncCheckpoint=<%p>)...", __func__, (void*)psSyncAddrListUpdate , (void*)*ppsExportFenceSyncCheckpoint));
+			SyncAddrListAppendCheckpoints(psSyncAddrListUpdate,
+										  1,
+										  ppsExportFenceSyncCheckpoint);
+			if (!pauiIntUpdateUFOAddress)
+			{
+				pauiIntUpdateUFOAddress = psSyncAddrListUpdate->pasFWAddrs;
+			}
+			ui32IntClientUpdateCount++;
+#if defined(CMP_CHECKPOINT_DEBUG)
+			if (ui32IntClientUpdateCount > 0)
+			{
+				IMG_UINT32 iii;
+				IMG_UINT32 *pui32Tmp = (IMG_UINT32*)pauiIntUpdateUFOAddress;
+
+				CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress=<%p>, pui32Tmp=<%p>, ui32IntClientUpdateCount=%u", __func__, (void*)pauiIntUpdateUFOAddress, (void*)pui32Tmp, ui32IntClientUpdateCount));
+				for (iii=0; iii<ui32IntClientUpdateCount; iii++)
+				{
+					CHKPT_DBG((PVR_DBG_ERROR, "%s: pauiIntUpdateUFOAddress[%d](<%p>) = 0x%x", __func__, iii, (void*)pui32Tmp, *pui32Tmp));
+					pui32Tmp++;
+				}
+			}
+#endif
+		}
+
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   (after pvr_sync) ui32IntClientFenceCount=%d, ui32IntClientUpdateCount=%d", __func__, ui32IntClientFenceCount, ui32IntClientUpdateCount));
+
+#if (ENABLE_TQ_UFO_DUMP == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: dumping TQ fence/updates syncs...", __func__));
+		{
+			IMG_UINT32 ii;
+			PRGXFWIF_UFO_ADDR *psTmpIntFenceUFOAddress = pauiIntFenceUFOAddress;
+			PRGXFWIF_UFO_ADDR *psTmpIntUpdateUFOAddress = pauiIntUpdateUFOAddress;
+			IMG_UINT32 *pui32TmpIntUpdateValue = paui32IntUpdateValue;
+
+			/* Dump Fence syncs and Update syncs */
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TQ fence syncs (&psTransferContext->asSyncAddrListFence=<%p>, pauiIntFenceUFOAddress=<%p>):", __func__, ui32IntClientFenceCount, (void*)&psTransferContext->asSyncAddrListFence, (void*)pauiIntFenceUFOAddress));
+			for (ii=0; ii<ui32IntClientFenceCount; ii++)
+			{
+				PVR_ASSERT(psTmpIntFenceUFOAddress->ui32Addr & 0x1);
+				PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, CheckValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientFenceCount, (void*)psTmpIntFenceUFOAddress, psTmpIntFenceUFOAddress->ui32Addr));
+
+				psTmpIntFenceUFOAddress++;
+			}
+			PVR_DPF((PVR_DBG_ERROR, "%s: Prepared %d TQ update syncs (&psTransferContext->asSyncAddrListUpdate=<%p>, pauiIntUpdateUFOAddress=<%p>):", __func__, ui32IntClientUpdateCount, (void*)&psTransferContext->asSyncAddrListUpdate, (void*)pauiIntUpdateUFOAddress));
+			for (ii=0; ii<ui32IntClientUpdateCount; ii++)
+			{
+				if (psTmpIntUpdateUFOAddress->ui32Addr & 0x1)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=PVRSRV_SYNC_CHECKPOINT_SIGNALLED", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr));
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s:   %d/%d<%p>. FWAddr=0x%x, UpdateValue=%d", __func__, ii+1, ui32IntClientUpdateCount, (void*)psTmpIntUpdateUFOAddress, psTmpIntUpdateUFOAddress->ui32Addr, *pui32TmpIntUpdateValue));
+					pui32TmpIntUpdateValue++;
+				}
+				psTmpIntUpdateUFOAddress++;
+			}
+		}
+#endif
+
+		ui32PreparesDone++;
+
+		/*
+			Create the command helper data for this command
+		*/
+		RGXCmdHelperInitCmdCCB(psDevInfo,
+		                       psClientCCB,
+		                       0,
+		                       ui32IntClientFenceCount,
+		                       pauiIntFenceUFOAddress,
+		                       NULL, /* fence value */
+		                       ui32IntClientUpdateCount,
+		                       pauiIntUpdateUFOAddress,
+		                       paui32IntUpdateValue,
+		                       paui32FWCommandSize[i],
+		                       papaui8FWCommand[i],
+							   &pPreAddr,
+							   &pPostAddr,
+							   &pRMWUFOAddr,
+		                       eType,
+		                       ui32ExtJobRef,
+		                       ui32IntJobRef,
+		                       ui32PDumpFlags,
+		                       NULL,
+		                       pszCommandName,
+		                       bCCBStateOpen,
+		                       psCmdHelper);
+
+	}
+
+	/*
+		Acquire space for all the commands in one go
+	*/
+	if (ui323DCmdCount)
+	{
+		eError = RGXCmdHelperAcquireCmdCCB(ui323DCmdCount,
+										   &pas3DCmdHelper[0]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_cmdacquire;
+		}
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = RGXCmdHelperAcquireCmdCCB(ui322DCmdCount,
+										   &pas2DCmdHelper[0]);
+		if (eError != PVRSRV_OK)
+		{
+			goto fail_cmdacquire;
+		}
+	}
+#endif
+
+	/*
+		We should acquire the kernel CCB(s) space here as the schedule could fail
+		and we would have to roll back all the syncs
+	*/
+
+	eError = PVRSRVPowerLock(psDevInfo->psDeviceNode);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: failed to acquire powerlock (%s)",
+			__func__, PVRSRVGetErrorString(eError)));
+
+		/* If system is found powered OFF, Retry scheduling the command */
+		if (likely(eError == PVRSRV_ERROR_SYSTEM_STATE_POWERED_OFF))
+		{
+			eError = PVRSRV_ERROR_RETRY;
+		}
+		goto fail_acquirepowerlock;
+	}
+
+	if (ui323DCmdCount)
+	{
+		ui323DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->s3DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui323DCmdCount,
+								  &pas3DCmdHelper[0],
+								  "TQ_3D",
+								  FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext).ui32Addr);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		ui322DCmdOffset = RGXGetHostWriteOffsetCCB(FWCommonContextGetClientCCB(psTransferContext->s2DData.psServerCommonContext));
+		RGXCmdHelperReleaseCmdCCB(ui322DCmdCount,
+								  &pas2DCmdHelper[0],
+								  "TQ_2D",
+								  FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext).ui32Addr);
+	}
+#endif
+
+	if (ui323DCmdCount)
+	{
+		RGXFWIF_KCCB_CMD s3DKCCBCmd;
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *ps3DTQCCB = FWCommonContextGetClientCCB(psTransferContext->s3DData.psServerCommonContext);
+
+		/* Take one of the helper data structs and extract the common cmd struct,
+		 * this is used to obtain the frame num. Each command should share the same
+		 * frame number so we can just get the first.
+		 */
+#if defined(PVRSRV_ENABLE_HTB)
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper = &pas3DCmdHelper[0];
+		CMD_COMMON *psTransferCmdCmn = IMG_OFFSET_ADDR(psCmdHelper->pui8DMCmd, 0);
+#endif
+
+		/* Construct the kernel 3D CCB command. */
+		s3DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		s3DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->s3DData.psServerCommonContext);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(ps3DTQCCB);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(ps3DTQCCB);
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+		s3DKCCBCmd.uCmdData.sCmdKickData.ui32WorkEstCmdHeaderOffset = 0;
+#endif
+
+#if defined(PVRSRV_ENABLE_HTB)
+		HTBLOGK(HTB_SF_MAIN_KICK_3D,
+				s3DKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui323DCmdOffset,
+				psTransferCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef
+				);
+#endif
+
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQ3D,
+		                  iCheckFence,
+		                  i3DUpdateFence,
+		                  i3DUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  ui3DUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psDevInfo,
+										RGXFWIF_DM_3D,
+										&s3DKCCBCmd,
+										ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			eError = eError2;
+			goto fail_cmdsubmit;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWCtx, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQ3D);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((ui322DCmdCount) && (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		RGXFWIF_KCCB_CMD s2DKCCBCmd;
+		IMG_UINT32 ui32FWCtx = FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext).ui32Addr;
+		RGX_CLIENT_CCB *ps2DTQCCB = FWCommonContextGetClientCCB(psTransferContext->s2DData.psServerCommonContext);
+
+		/* Take one of the helper data structs and extract the common cmd struct,
+		 * this is used to obtain the frame num. Each command should share the same
+		 * frame number so we can just get the first.
+		 */
+#if defined(PVRSRV_ENABLE_HTB)
+		RGX_CCB_CMD_HELPER_DATA *psCmdHelper = &pas2DCmdHelper[0];
+		CMD_COMMON *psTransferCmdCmn = IMG_OFFSET_ADDR(psCmdHelper->pui8DMCmd, 0);
+#endif
+
+		/* Construct the kernel 2D CCB command. */
+		s2DKCCBCmd.eCmdType = RGXFWIF_KCCB_CMD_KICK;
+		s2DKCCBCmd.uCmdData.sCmdKickData.psContext = FWCommonContextGetFWAddress(psTransferContext->s2DData.psServerCommonContext);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32CWoffUpdate = RGXGetHostWriteOffsetCCB(ps2DTQCCB);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32CWrapMaskUpdate = RGXGetWrapMaskCCB(ps2DTQCCB);
+		s2DKCCBCmd.uCmdData.sCmdKickData.ui32NumCleanupCtl = 0;
+
+#if defined(PVRSRV_ENABLE_HTB)
+		HTBLOGK(HTB_SF_MAIN_KICK_2D,
+				s2DKCCBCmd.uCmdData.sCmdKickData.psContext,
+				ui322DCmdOffset,
+				psTransferCmdCmn->ui32FrameNum,
+				ui32ExtJobRef,
+				ui32IntJobRef);
+#endif
+
+		RGXSRV_HWPERF_ENQ(psTransferContext,
+		                  OSGetCurrentClientProcessIDKM(),
+		                  ui32FWCtx,
+		                  ui32ExtJobRef,
+		                  ui32IntJobRef,
+		                  RGX_HWPERF_KICK_TYPE2_TQ2D,
+		                  iCheckFence,
+		                  i2DUpdateFence,
+		                  i2DUpdateTimeline,
+		                  uiCheckFenceUID,
+		                  ui2DUpdateFenceUID,
+		                  NO_DEADLINE,
+		                  NO_CYCEST);
+
+		LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+		{
+			eError2 = RGXScheduleCommandWithoutPowerLock(psDevInfo,
+										RGXFWIF_DM_2D,
+										&s2DKCCBCmd,
+										ui32PDumpFlags);
+			if (eError2 != PVRSRV_ERROR_RETRY)
+			{
+				break;
+			}
+			OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+		} END_LOOP_UNTIL_TIMEOUT_US();
+
+		if (eError2 != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "PVRSRVRGXSubmitTransferKM failed to schedule kernel CCB command. (0x%x)", eError2));
+			eError = eError2;
+			goto fail_cmdsubmit;
+		}
+
+		PVRGpuTraceEnqueueEvent(psDeviceNode, ui32FWCtx, ui32ExtJobRef,
+		                        ui32IntJobRef, RGX_HWPERF_KICK_TYPE2_TQ2D);
+	}
+#endif
+
+	/*
+	 * Now check eError (which may have returned an error from our earlier calls
+	 * to RGXCmdHelperAcquireCmdCCB) - we needed to process any flush command first
+	 * so we check it now...
+	 */
+	if (eError != PVRSRV_OK )
+	{
+		goto fail_cmdsubmit;
+	}
+
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+
+#if defined(NO_HARDWARE)
+	/* If NO_HARDWARE, signal the output fence's sync checkpoint and sync prim */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (ps2DUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling TLA NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)ps2DUpdateSyncCheckpoint, SyncCheckpointGetId(ps2DUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(ps2DUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(ps2DUpdateSyncCheckpoint);
+	}
+	if (ps2DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating TLA NOHW sync prim<%p> to %d", __func__, (void*)ps2DFenceTimelineUpdateSync, ui322DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps2DFenceTimelineUpdateSync, ui322DFenceTimelineUpdateValue);
+	}
+#endif
+	if (ps3DUpdateSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling TQ3D NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__, (void*)ps3DUpdateSyncCheckpoint, SyncCheckpointGetId(ps3DUpdateSyncCheckpoint), SyncCheckpointGetFirmwareAddr(ps3DUpdateSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(ps3DUpdateSyncCheckpoint);
+	}
+	if (ps3DFenceTimelineUpdateSync)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Updating TQ3D NOHW sync prim<%p> to %d", __func__, (void*)ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue));
+		SyncPrimNoHwUpdate(ps3DFenceTimelineUpdateSync, ui323DFenceTimelineUpdateValue);
+	}
+	SyncCheckpointNoHWUpdateTimelines(NULL);
+
+	if (ppsExportFenceSyncCheckpoint && *ppsExportFenceSyncCheckpoint)
+	{
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   Signalling NOHW sync checkpoint<%p>, ID:%d, FwAddr=0x%x", __func__,
+		           (void*)*ppsExportFenceSyncCheckpoint, SyncCheckpointGetId(*ppsExportFenceSyncCheckpoint),
+		           SyncCheckpointGetFirmwareAddr(*ppsExportFenceSyncCheckpoint)));
+		SyncCheckpointSignalNoHW(*ppsExportFenceSyncCheckpoint);
+		CHKPT_DBG((PVR_DBG_ERROR, "%s:   SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal=%d)", __func__, iExportFenceToSignal));
+		SyncCheckpointNoHWSignalExportFence(iExportFenceToSignal);
+	}
+#endif /* defined(NO_HARDWARE) */
+
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_succeeded(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pi2DUpdateFence)
+	{
+		*pi2DUpdateFence = i2DUpdateFence;
+	}
+#endif
+	if (pi3DUpdateFence)
+	{
+		*pi3DUpdateFence = i3DUpdateFence;
+	}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pv2DUpdateFenceFinaliseData && (i2DUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, i2DUpdateFence, pv2DUpdateFenceFinaliseData,
+		                            ps2DUpdateSyncCheckpoint, szFenceName);
+	}
+#endif
+	if (pv3DUpdateFenceFinaliseData && (i3DUpdateFence != PVRSRV_NO_FENCE))
+	{
+		SyncCheckpointFinaliseFence(psDeviceNode, i3DUpdateFence, pv3DUpdateFenceFinaliseData,
+		                            ps3DUpdateSyncCheckpoint, szFenceName);
+	}
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	OSFreeMem(pas2DCmdHelper);
+#endif
+	OSFreeMem(pas3DCmdHelper);
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+	/* Free memory allocated to hold the internal list of update values */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pui322DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui322DIntAllocatedUpdateValues);
+		pui322DIntAllocatedUpdateValues = NULL;
+	}
+#endif
+	if (pui323DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui323DIntAllocatedUpdateValues);
+		pui323DIntAllocatedUpdateValues = NULL;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+/*
+	No resources are created in this function so there is nothing to free
+	unless we had to merge syncs.
+	If we fail after the client CCB acquire there is still nothing to do
+	as only the client CCB release will modify the client CCB
+*/
+fail_cmdsubmit:
+	PVRSRVPowerUnlock(psDevInfo->psDeviceNode);
+fail_acquirepowerlock:
+fail_cmdacquire:
+fail_prepare_loop:
+
+	PVR_ASSERT(eError != PVRSRV_OK);
+
+	for (i=0;i<ui32PreparesDone;i++)
+	{
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListFence[i]);
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListUpdate[i]);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (ui32PreparesDone > 0)
+	{
+		/* Prevent duplicate rollback in case of buffer sync. */
+		psBufferUpdateSyncCheckpoint = NULL;
+	}
+#endif
+
+	/* Free memory allocated to hold the internal list of update values */
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (pui322DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui322DIntAllocatedUpdateValues);
+		pui322DIntAllocatedUpdateValues = NULL;
+	}
+#endif
+	if (pui323DIntAllocatedUpdateValues)
+	{
+		OSFreeMem(pui323DIntAllocatedUpdateValues);
+		pui323DIntAllocatedUpdateValues = NULL;
+	}
+	if (ppsExportFenceSyncCheckpoint && *ppsExportFenceSyncCheckpoint)
+	{
+		SyncCheckpointRollbackExportFence(iExportFenceToSignal);
+	}
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if (i2DUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(i2DUpdateFence, pv2DUpdateFenceFinaliseData);
+	}
+#endif
+	if (i3DUpdateFence != PVRSRV_NO_FENCE)
+	{
+		SyncCheckpointRollbackFenceData(i3DUpdateFence, pv3DUpdateFenceFinaliseData);
+	}
+#if defined(SUPPORT_BUFFER_SYNC)
+	if (psBufferUpdateSyncCheckpoint)
+	{
+		SyncAddrListRollbackCheckpoints(psDeviceNode, &psTransferContext->asSyncAddrListUpdate[0]);
+	}
+	if (psBufferSyncData)
+	{
+		pvr_buffer_sync_kick_failed(psBufferSyncData);
+	}
+	if (apsBufferFenceSyncCheckpoints)
+	{
+		kfree(apsBufferFenceSyncCheckpoints);
+	}
+fail_resolve_buffersync_input_fence:
+#endif /* defined(SUPPORT_BUFFER_SYNC) */
+
+	/* Drop the references taken on the sync checkpoints in the
+	 * resolved input fence */
+	SyncAddrListDeRefCheckpoints(ui32FenceSyncCheckpointCount,
+								 apsFenceSyncCheckpoints);
+	/* Free the memory that was allocated for the sync checkpoint list returned by ResolveFence() */
+	if (apsFenceSyncCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsFenceSyncCheckpoints);
+	}
+fail_resolve_fencesync_input_fence:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	OSFreeMem(pas2DCmdHelper);
+fail_alloc2dhelper:
+#endif
+	OSFreeMem(pas3DCmdHelper);
+fail_alloc3dhelper:
+
+	OSLockRelease(psTransferContext->hLock);
+	return eError;
+}
+
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDevNode,
+                                                   RGX_SERVER_TQ_CONTEXT *psTransferContext,
+                                                   IMG_INT32 i32Priority)
+{
+	PVRSRV_ERROR eError;
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	PVRSRV_RGXDEV_INFO *psDevInfo = psDevNode->pvDevice;
+#endif
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	OSLockAcquire(psTransferContext->hLock);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+	if ((psTransferContext->s2DData.i32Priority != i32Priority) &&
+			(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+	{
+		eError = ContextSetPriority(psTransferContext->s2DData.psServerCommonContext,
+									psConnection,
+									psTransferContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_2D);
+		if (eError != PVRSRV_OK)
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 2D part of the transfercontext (%s)", __func__, PVRSRVGetErrorString(eError)));
+			}
+			goto fail_2dcontext;
+		}
+		psTransferContext->s2DData.i32Priority = i32Priority;
+	}
+#endif
+
+	if (psTransferContext->s3DData.i32Priority != i32Priority)
+	{
+		eError = ContextSetPriority(psTransferContext->s3DData.psServerCommonContext,
+									psConnection,
+									psTransferContext->psDeviceNode->pvDevice,
+									i32Priority,
+									RGXFWIF_DM_3D);
+		if (eError != PVRSRV_OK)
+		{
+			if (eError != PVRSRV_ERROR_RETRY)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s: Failed to set the priority of the 3D part of the transfercontext (%s)", __func__, PVRSRVGetErrorString(eError)));
+			}
+			goto fail_3dcontext;
+		}
+		psTransferContext->s3DData.i32Priority = i32Priority;
+	}
+
+	OSLockRelease(psTransferContext->hLock);
+	return PVRSRV_OK;
+
+fail_3dcontext:
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+
+fail_2dcontext:
+#endif
+	OSLockRelease(psTransferContext->hLock);
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPropertyKM(RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   RGX_CONTEXT_PROPERTY eContextProperty,
+												   IMG_UINT64 ui64Input,
+												   IMG_UINT64 *pui64Output)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	switch (eContextProperty)
+	{
+		case RGX_CONTEXT_PROPERTY_FLAGS:
+		{
+			IMG_UINT32 ui32ContextFlags = (IMG_UINT32)ui64Input;
+
+			OSLockAcquire(psTransferContext->hLock);
+			eError = FWCommonContextSetFlags(psTransferContext->s2DData.psServerCommonContext,
+			                                 ui32ContextFlags);
+			if (eError == PVRSRV_OK)
+			{
+				eError = FWCommonContextSetFlags(psTransferContext->s3DData.psServerCommonContext,
+			                                     ui32ContextFlags);
+			}
+			OSLockRelease(psTransferContext->hLock);
+			break;
+		}
+
+		default:
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_ERROR_NOT_SUPPORTED - asked to set unknown property (%d)", __func__, eContextProperty));
+			eError = PVRSRV_ERROR_NOT_SUPPORTED;
+		}
+	}
+
+	return eError;
+}
+
+void DumpTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel)
+{
+	DLLIST_NODE *psNode, *psNext;
+
+	OSWRLockAcquireRead(psDevInfo->hTransferCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTransferCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_CONTEXT, sListNode);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			DumpFWCommonContextInfo(psCurrentServerTransferCtx->s2DData.psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+#endif
+
+		if (psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D)
+		{
+			DumpFWCommonContextInfo(psCurrentServerTransferCtx->s3DData.psServerCommonContext,
+			                        pfnDumpDebugPrintf, pvDumpDebugFile, ui32VerbLevel);
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTransferCtxListLock);
+}
+
+IMG_UINT32 CheckForStalledClientTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_UINT32 ui32ContextBitMask = 0;
+
+	OSWRLockAcquireRead(psDevInfo->hTransferCtxListLock);
+
+	dllist_foreach_node(&psDevInfo->sTransferCtxtListHead, psNode, psNext)
+	{
+		RGX_SERVER_TQ_CONTEXT *psCurrentServerTransferCtx =
+			IMG_CONTAINER_OF(psNode, RGX_SERVER_TQ_CONTEXT, sListNode);
+
+#if defined(RGX_FEATURE_TLA_BIT_MASK)
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_2D) &&
+				(NULL != psCurrentServerTransferCtx->s2DData.psServerCommonContext) &&
+				(RGX_IS_FEATURE_SUPPORTED(psDevInfo, TLA)))
+		{
+			if (CheckStalledClientCommonContext(psCurrentServerTransferCtx->s2DData.psServerCommonContext, RGX_KICK_TYPE_DM_TQ2D) == PVRSRV_ERROR_CCCB_STALLED)
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TQ2D;
+			}
+		}
+#endif
+
+		if ((psCurrentServerTransferCtx->ui32Flags & RGX_SERVER_TQ_CONTEXT_FLAGS_3D) && (NULL != psCurrentServerTransferCtx->s3DData.psServerCommonContext))
+		{
+			if ((CheckStalledClientCommonContext(psCurrentServerTransferCtx->s3DData.psServerCommonContext, RGX_KICK_TYPE_DM_TQ3D) == PVRSRV_ERROR_CCCB_STALLED))
+			{
+				ui32ContextBitMask |= RGX_KICK_TYPE_DM_TQ3D;
+			}
+		}
+	}
+
+	OSWRLockReleaseRead(psDevInfo->hTransferCtxListLock);
+	return ui32ContextBitMask;
+}
+
+PVRSRV_ERROR PVRSRVRGXTQGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem)
+{
+	PVRSRVTQAcquireShaders(psDeviceNode, ppsCLIPMRMem);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR PVRSRVRGXTQReleaseSharedMemoryKM(PMR * psPMRMem)
+{
+	PVR_UNREFERENCED_PARAMETER(psPMRMem);
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************//**
+ End of file (rgxtransfer.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxtransfer.h b/drivers/gpu/drm/img-rogue/rgxtransfer.h
new file mode 100644
index 000000000000..3cdc454d3de0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtransfer.h
@@ -0,0 +1,179 @@
+/*************************************************************************/ /*!
+@File
+@Title          RGX Transfer queue Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the RGX Transfer queue Functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXTRANSFER_H)
+#define RGXTRANSFER_H
+
+#include "devicemem.h"
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxfwutils.h"
+#include "rgx_fwif_resetframework.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+
+#include "sync_server.h"
+#include "connection_server.h"
+
+typedef struct _RGX_SERVER_TQ_CONTEXT_ RGX_SERVER_TQ_CONTEXT;
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXCreateTransferContextKM
+
+ @Description
+	Server-side implementation of RGXCreateTransferContext
+
+ @Input pvDeviceNode - device node
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXCreateTransferContextKM(CONNECTION_DATA		*psConnection,
+										   PVRSRV_DEVICE_NODE		*psDeviceNode,
+										   IMG_INT32				i32Priority,
+										   IMG_UINT32				ui32FrameworkCommandSize,
+										   IMG_PBYTE				pabyFrameworkCommand,
+										   IMG_HANDLE				hMemCtxPrivData,
+										   IMG_UINT32				ui32PackedCCBSizeU8888,
+										   IMG_UINT32				ui32ContextFlags,
+										   IMG_UINT64				ui64RobustnessAddress,
+										   RGX_SERVER_TQ_CONTEXT	**ppsTransferContext);
+
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVRGXDestroyTransferContextKM
+
+ @Description
+	Server-side implementation of RGXDestroyTransferContext
+
+ @Input psTransferContext - Transfer context
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXDestroyTransferContextKM(RGX_SERVER_TQ_CONTEXT *psTransferContext);
+
+/*!
+*******************************************************************************
+
+ @Function	PVRSRVSubmitTransferKM
+
+ @Description
+	Schedules one or more 2D or 3D HW commands on the firmware
+
+
+ @Return   PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR PVRSRVRGXSubmitTransferKM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									IMG_UINT32				ui32PrepareCount,
+									IMG_UINT32				*paui32ClientUpdateCount,
+									SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									IMG_UINT32				**papaui32ClientUpdateValue,
+									PVRSRV_FENCE			iCheckFence,
+									PVRSRV_TIMELINE			i2DUpdateTimeline,
+									PVRSRV_FENCE			*pi2DUpdateFence,
+									PVRSRV_TIMELINE			i3DUpdateTimeline,
+									PVRSRV_FENCE			*pi3DUpdateFence,
+									IMG_CHAR				szFenceName[32],
+									IMG_UINT32				*paui32FWCommandSize,
+									IMG_UINT8				**papaui8FWCommand,
+									IMG_UINT32				*pui32TQPrepareFlags,
+									IMG_UINT32				ui32ExtJobRef,
+									IMG_UINT32				ui32SyncPMRCount,
+									IMG_UINT32				*paui32SyncPMRFlags,
+									PMR						**ppsSyncPMRs);
+
+PVRSRV_ERROR PVRSRVRGXSubmitTransfer3KM(RGX_SERVER_TQ_CONTEXT	*psTransferContext,
+									IMG_UINT32				ui32PrepareCount,
+									IMG_UINT32				*paui32ClientUpdateCount,
+									SYNC_PRIMITIVE_BLOCK		***papauiClientUpdateUFODevVarBlock,
+									IMG_UINT32				**papaui32ClientUpdateSyncOffset,
+									IMG_UINT32				**papaui32ClientUpdateValue,
+									PVRSRV_FENCE			iCheckFence,
+									PVRSRV_TIMELINE			i2DUpdateTimeline,
+									PVRSRV_FENCE			*pi2DUpdateFence,
+									PVRSRV_TIMELINE			i3DUpdateTimeline,
+									PVRSRV_FENCE			*pi3DUpdateFence,
+									IMG_CHAR				szFenceName[32],
+									PVRSRV_FENCE			iExportFenceToSignal,
+									IMG_UINT32				*paui32FWCommandSize,
+									IMG_UINT8				**papaui8FWCommand,
+									IMG_UINT32				*pui32TQPrepareFlags,
+									IMG_UINT32				ui32ExtJobRef,
+									IMG_UINT32				ui32SyncPMRCount,
+									IMG_UINT32				*paui32SyncPMRFlags,
+									PMR						**ppsSyncPMRs);
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPriorityKM(CONNECTION_DATA *psConnection,
+                                                   PVRSRV_DEVICE_NODE * psDevNode,
+												   RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   IMG_INT32 i32Priority);
+
+PVRSRV_ERROR PVRSRVRGXSetTransferContextPropertyKM(RGX_SERVER_TQ_CONTEXT *psTransferContext,
+												   RGX_CONTEXT_PROPERTY eContextProperty,
+												   IMG_UINT64 ui64Input,
+												   IMG_UINT64 *pui64Output);
+
+/* Debug - Dump debug info of transfer contexts on this device */
+void DumpTransferCtxtsInfo(PVRSRV_RGXDEV_INFO *psDevInfo,
+                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                           void *pvDumpDebugFile,
+                           IMG_UINT32 ui32VerbLevel);
+
+/* Debug/Watchdog - check if client transfer contexts are stalled */
+IMG_UINT32 CheckForStalledClientTransferCtxt(PVRSRV_RGXDEV_INFO *psDevInfo);
+
+PVRSRV_ERROR PVRSRVRGXTQGetSharedMemoryKM(
+	CONNECTION_DATA           * psConnection,
+	PVRSRV_DEVICE_NODE        * psDeviceNode,
+	PMR                      ** ppsCLIPMRMem);
+
+PVRSRV_ERROR PVRSRVRGXTQReleaseSharedMemoryKM(PMR * psUSCPMRMem);
+
+#endif /* RGXTRANSFER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxtransfer_shader.h b/drivers/gpu/drm/img-rogue/rgxtransfer_shader.h
new file mode 100644
index 000000000000..f261d2d04075
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxtransfer_shader.h
@@ -0,0 +1,69 @@
+/*************************************************************************/ /*!
+@File           rgxtransfer_shader.h
+@Title          TQ binary shader file info
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header holds info about TQ binary shader file generated
+                by the TQ shader factory. This header is need by shader factory
+                when generating the file; by services KM when reading and
+                loading the file into memory; and by services UM when
+                constructing blits using the shaders.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(RGXSHADERHEADER_H)
+#define RGXSHADERHEADER_H
+
+#include "pvrversion.h"
+
+typedef struct _RGX_SHADER_HEADER_
+{
+	IMG_UINT32 ui32Version;
+	IMG_UINT32 ui32NumFragment;
+	IMG_UINT32 ui32SizeFragment;
+	IMG_UINT32 ui32SizeClientMem;
+} RGX_SHADER_HEADER;
+
+/* TQ shaders version is used to check compatibility between the
+   binary TQ shaders file and the DDK. This number should be incremented
+   if a change to the TQ shader factory breaks compatibility. */
+#define RGX_TQ_SHADERS_VERSION 2U
+
+#define RGX_TQ_SHADERS_VERSION_PACK \
+	(((RGX_TQ_SHADERS_VERSION & 0xFFU) << 16) | ((PVRVERSION_MAJ & 0xFFU) << 8) | ((PVRVERSION_MIN & 0xFFU) << 0))
+
+#endif /* RGXSHADERHEADER_H */
diff --git a/drivers/gpu/drm/img-rogue/rgxutils.c b/drivers/gpu/drm/img-rogue/rgxutils.c
new file mode 100644
index 000000000000..4ffc08c07add
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxutils.c
@@ -0,0 +1,306 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific utility routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Device specific functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "rgx_fwif_km.h"
+#include "pdump_km.h"
+#include "osfunc.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "rgxutils.h"
+#include "power.h"
+#include "pvrsrv.h"
+#include "sync_internal.h"
+#include "rgxfwutils.h"
+#include "rgxlayer.h"
+
+PVRSRV_ERROR RGXQueryAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 *pui32State)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	psDevInfo = psDeviceNode->pvDevice;
+	*pui32State = psDevInfo->eActivePMConf;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 ui32State)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+#if !defined(NO_HARDWARE)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+#endif
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (RGX_ACTIVEPM_FORCE_OFF != ui32State)
+	{
+		return PVRSRV_ERROR_NOT_SUPPORTED;
+	}
+
+#if !defined(NO_HARDWARE)
+	psDevInfo = psDeviceNode->pvDevice;
+
+	if (psDevInfo->pvAPMISRData)
+	{
+		psDevInfo->eActivePMConf = RGX_ACTIVEPM_FORCE_OFF;
+		psDevInfo->pvAPMISRData = NULL;
+		eError = PVRSRVSetDeviceDefaultPowerState((PPVRSRV_DEVICE_NODE)psDeviceNode,
+		                                          PVRSRV_DEV_POWER_STATE_ON);
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR RGXQueryPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL *pbDisabled)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	*pbDisabled = !psDevInfo->bPDPEnabled;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL bDisable)
+{
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	PVR_UNREFERENCED_PARAMETER(pvPrivateData);
+
+	if (!psDeviceNode || !psDeviceNode->pvDevice)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psDevInfo = psDeviceNode->pvDevice;
+
+	psDevInfo->bPDPEnabled = !bDisable;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXGetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 *pui32DeviceFlags)
+{
+	if (!pui32DeviceFlags || !psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	*pui32DeviceFlags = psDevInfo->ui32DeviceFlags;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR RGXSetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_BOOL bSetNotClear)
+{
+	if (!psDevInfo)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if ((ui32Config & ~RGXKM_DEVICE_STATE_MASK) != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Bits outside of device state mask set (input: 0x%x, mask: 0x%x)",
+				 __func__, ui32Config, RGXKM_DEVICE_STATE_MASK));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	if (bSetNotClear)
+	{
+		psDevInfo->ui32DeviceFlags |= ui32Config;
+	}
+	else
+	{
+		psDevInfo->ui32DeviceFlags &= ~ui32Config;
+	}
+
+	return PVRSRV_OK;
+}
+
+inline const char * RGXStringifyKickTypeDM(RGX_KICK_TYPE_DM eKickTypeDM)
+{
+	PVR_ASSERT(eKickTypeDM < RGX_KICK_TYPE_DM_LAST);
+
+	switch (eKickTypeDM) {
+		case RGX_KICK_TYPE_DM_GP:
+			return "GP ";
+		case RGX_KICK_TYPE_DM_TDM_2D:
+			return "TDM/2D ";
+		case RGX_KICK_TYPE_DM_TA:
+			return "TA ";
+		case RGX_KICK_TYPE_DM_3D:
+			return "3D ";
+		case RGX_KICK_TYPE_DM_CDM:
+			return "CDM ";
+		case RGX_KICK_TYPE_DM_RTU:
+			return "RTU ";
+		case RGX_KICK_TYPE_DM_SHG:
+			return "SHG ";
+		case RGX_KICK_TYPE_DM_TQ2D:
+			return "TQ2D ";
+		case RGX_KICK_TYPE_DM_TQ3D:
+			return "TQ3D ";
+		default:
+			return "Invalid DM ";
+	}
+}
+
+PHYS_HEAP_POLICY RGXPhysHeapGetLMAPolicy(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags, PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PHYS_HEAP_POLICY ui32Policy;
+
+	if (OSIsMapPhysNonContigSupported())
+	{
+		ui32Policy = PHYS_HEAP_POLICY_ALLOC_ALLOW_NONCONTIG;
+
+		if (BITMASK_ANY(ui32UsageFlags,
+			(PHYS_HEAP_USAGE_FW_SHARED    |
+			 PHYS_HEAP_USAGE_FW_PRIVATE   |
+			 PHYS_HEAP_USAGE_FW_PREMAP_PT |
+			 PHYS_HEAP_USAGE_FW_CODE      |
+			 PHYS_HEAP_USAGE_FW_PRIV_DATA)))
+		{
+			if (PVRSRV_VZ_MODE_IS(GUEST, DEVNODE, psDeviceNode))
+			{
+				/* Guest Firmware heaps are always premapped */
+				ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+			}
+#if defined(RGX_PREMAP_FW_HEAPS)
+			else if (PVRSRV_VZ_MODE_IS(HOST, DEVNODE, psDeviceNode))
+			{
+				/* All Firmware heaps are premapped under AutoVz*/
+				ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+			}
+#endif
+		}
+
+		if (BITMASK_ANY(ui32UsageFlags, PHYS_HEAP_USAGE_FW_PREMAP))
+		{
+			ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+		}
+	}
+	else
+	{
+		ui32Policy = PHYS_HEAP_POLICY_DEFAULT;
+	}
+
+	return ui32Policy;
+}
+
+IMG_BOOL RGXIsErrorAndDeviceRecoverable(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_ERROR *peError)
+{
+	IMG_BOOL bRecoverable = IMG_TRUE;
+
+	if (*peError == PVRSRV_OK)
+	{
+		/* No recovery required */
+		return IMG_FALSE;
+	}
+
+	if (!PVRSRVIsStatusRecoverable(OSAtomicRead(&psDeviceNode->eHealthStatus)))
+	{
+		bRecoverable = IMG_FALSE;
+	}
+	else
+	{
+		RGXUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+
+		if (!PVRSRVIsStatusRecoverable(OSAtomicRead(&psDeviceNode->eHealthStatus)))
+		{
+			bRecoverable = IMG_FALSE;
+		}
+	}
+
+	if (bRecoverable && !PVRSRVIsRetryError(*peError))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device is recoverable. Changing error type (%s) to retry.",
+				 __func__, PVRSRVGetErrorString(*peError)));
+		*peError = PVRSRV_ERROR_RETRY;
+	}
+
+	if (!bRecoverable && PVRSRVIsRetryError(*peError))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: Device is not recoverable. Error type should not be retry (%s).",
+				 __func__, PVRSRVGetErrorString(*peError)));
+		*peError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return bRecoverable;
+}
+
+/******************************************************************************
+ End of file (rgxutils.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/rgxutils.h b/drivers/gpu/drm/img-rogue/rgxutils.h
new file mode 100644
index 000000000000..be6e586e2279
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rgxutils.h
@@ -0,0 +1,236 @@
+/*************************************************************************/ /*!
+@File
+@Title          Device specific utility routines declarations
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Inline functions/structures specific to RGX
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "device.h"
+#include "rgxdevice.h"
+#include "rgxdebug_common.h"
+#include "pvr_notifier.h"
+#include "pvrsrv.h"
+
+/*!
+******************************************************************************
+
+ @Function      RGXQueryAPMState
+
+ @Description   Query the state of the APM configuration
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Output        pui32State   : The APM configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXQueryAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 *pui32State);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetAPMState
+
+ @Description   Set the APM configuration state. Currently only 'OFF' is
+                supported
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         ui32State    : The requested APM configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetAPMState(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_UINT32 ui32State);
+
+/*!
+******************************************************************************
+
+ @Function      RGXQueryPdumpPanicDisable
+
+ @Description   Get the PDump Panic Enable configuration state.
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         pbDisabled    : IMG_TRUE if PDump Panic is disabled
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXQueryPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL *pbDisabled);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetPdumpPanicDisable
+
+ @Description   Set the PDump Panic Enable flag
+
+ @Input         psDeviceNode : The device node
+
+ @Input         pvPrivateData: Unused (required for AppHint callback)
+
+ @Input         bDisable      : The requested configuration state
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetPdumpPanicDisable(const PVRSRV_DEVICE_NODE *psDeviceNode,
+	const void *pvPrivateData,
+	IMG_BOOL bDisable);
+
+/*!
+******************************************************************************
+
+ @Function      RGXGetDeviceFlags
+
+ @Description   Get the device flags for a given device
+
+ @Input         psDevInfo        : The device descriptor query
+
+ @Output        pui32DeviceFlags : The current state of the device flags
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXGetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 *pui32DeviceFlags);
+
+/*!
+******************************************************************************
+
+ @Function      RGXSetDeviceFlags
+
+ @Description   Set the device flags for a given device
+
+ @Input         psDevInfo : The device descriptor to modify
+
+ @Input         ui32Config : The device flags to modify
+
+ @Input         bSetNotClear : Set or clear the specified flags
+
+ @Return        PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RGXSetDeviceFlags(PVRSRV_RGXDEV_INFO *psDevInfo,
+				IMG_UINT32 ui32Config,
+				IMG_BOOL bSetNotClear);
+
+/*!
+******************************************************************************
+
+ @Function    RGXStringifyKickTypeDM
+
+ @Description Gives the kick type DM name stringified
+
+ @Input       Kick type DM
+
+ @Return      Array containing the kick type DM name
+
+******************************************************************************/
+const char* RGXStringifyKickTypeDM(RGX_KICK_TYPE_DM eKickTypeDM);
+
+/*************************************************************************/ /*!
+
+@Function       RGXPhysHeapGetLMAPolicy
+
+@Description    Returns the optimal LMA allocation policy based on a heap's
+                usage flags
+
+@Input          ui32UsageFlags Flags specifying a heap's intended use
+@Input          psDeviceNode The device node.
+
+@Return         PHYS_HEAP_POLICY The recommended LMA policy
+
+*/ /**************************************************************************/
+PHYS_HEAP_POLICY RGXPhysHeapGetLMAPolicy(PHYS_HEAP_USAGE_FLAGS ui32UsageFlags, PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#define RGX_STRINGIFY_KICK_TYPE_DM_IF_SET(bitmask, eKickTypeDM) bitmask & eKickTypeDM ? RGXStringifyKickTypeDM(eKickTypeDM) : ""
+
+/*************************************************************************/ /*!
+@Function       RGXIsErrorAndDeviceRecoverable
+@Description    This function is used to check if device (and firmware) is in
+                a state that can be recovered from without a full reset of the
+                device.
+@Input          psDeviceNode The device node.
+@Input          peError      Pointer to error. Can be changed to retry type.
+@Return         IMG_BOOL   Return true if device is recoverable.
+*/ /**************************************************************************/
+IMG_BOOL RGXIsErrorAndDeviceRecoverable(PVRSRV_DEVICE_NODE *psDeviceNode, PVRSRV_ERROR *peError);
+
+/*
+ * To avoid repeated calls and avoid double frees, the error value is set to PVRSRV_OK
+ * if RGXIsErrorAndDeviceRecoverable is false.
+ */
+#define RGX_RETURN_IF_ERROR_AND_DEVICE_RECOVERABLE(psDeviceNode, eError, cleanupFunc) \
+	do \
+	{ \
+		if (RGXIsErrorAndDeviceRecoverable(psDeviceNode, &eError)) \
+		{ \
+			return eError; \
+		} \
+		else if (eError != PVRSRV_OK) \
+		{ \
+			PVR_LOG(("%s: Unexpected error from " #cleanupFunc "(%s)", \
+					__func__, \
+					PVRSRVGetErrorString(eError))); \
+			/* Device is dead. \
+			 * Change error type to make callers destroy the resource handle. \
+			 * This is to prevent repeated calls to this function. \
+			 */ \
+			eError = PVRSRV_OK; \
+		} \
+	} while (false)
+
+/******************************************************************************
+ End of file (rgxutils.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/ri_server.c b/drivers/gpu/drm/img-rogue/ri_server.c
new file mode 100644
index 000000000000..eec752d26017
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/ri_server.c
@@ -0,0 +1,2571 @@
+/*************************************************************************/ /*!
+@File           ri_server.c
+@Title          Resource Information (RI) server implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) server functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if defined(__linux__) && defined(__KERNEL__)
+ #include <linux/version.h>
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+  #include <linux/stdarg.h>
+ #else
+  #include <stdarg.h>
+ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
+#else
+ #include <stdarg.h>
+#endif /* __linux__ */
+#include "img_defs.h"
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+
+#include "srvkm.h"
+#include "lock.h"
+
+/* services/include */
+#include "pvr_ricommon.h"
+
+/* services/server/include/ */
+#include "ri_server.h"
+
+/* services/include/shared/ */
+#include "hash.h"
+/* services/shared/include/ */
+#include "dllist.h"
+
+#include "pmr.h"
+#include "physheap.h"
+
+/* include/device.h */
+#include "device.h"
+
+#if !defined(RI_UNIT_TEST)
+#include "pvrsrv.h"
+#endif
+
+
+#if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO)
+
+#define USE_RI_LOCK		1
+
+/*
+ * Initial size use for Hash table. (Used to index the RI list entries).
+ */
+#define _RI_INITIAL_HASH_TABLE_SIZE	64
+
+/*
+ * If this define is set to 1, details of the linked lists (addresses,
+ * prev/next ptrs, etc) are also output when function RIDumpList() is called.
+ */
+#define _DUMP_LINKEDLIST_INFO		0
+
+
+typedef IMG_UINT64 _RI_BASE_T;
+
+
+/* No +1 in SIZE macros since sizeof includes \0 byte in size */
+
+#define RI_PROC_BUF_SIZE    16
+#define RI_ANNO_BUF_SIZE    80
+#define RI_ANNO_FRMT_SIZE (sizeof(RI_ANNO_FRMT))
+
+#define RI_DEV_ID_BUF_SIZE  4
+
+#define RI_MEMDESC_SUM_FRMT     "PID:%d %s MEMDESCs Alloc'd:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K) + "\
+                                                  "Imported:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K) = "\
+                                                     "Total:0x%010" IMG_UINT64_FMTSPECx " (%" IMG_UINT64_FMTSPEC "K)\n"
+#define RI_MEMDESC_SUM_BUF_SIZE (sizeof(RI_MEMDESC_SUM_FRMT)+5+RI_PROC_BUF_SIZE+30+60)
+
+
+#define RI_PMR_SUM_FRMT     "PID:%d %s PMRs Alloc'd:0x%010" IMG_UINT64_FMTSPECx ", %" IMG_UINT64_FMTSPEC "K  "\
+                                        "[Physical: 0x%010" IMG_UINT64_FMTSPECx ", %" IMG_UINT64_FMTSPEC "K]\n"
+#define RI_PMR_SUM_BUF_SIZE (sizeof(RI_PMR_SUM_FRMT)+(20+40))
+
+#define RI_FREED_BY_DRIVER "{Freed by KM}"
+
+#define RI_PMR_ENTRY_FRMT      "%%sPID:%%-5d DEV:%%s <%%p>\t%%-%ds\t%%-%ds\t0x%%010" IMG_UINT64_FMTSPECx "\t[0x%%010" IMG_UINT64_FMTSPECx "]\t%%s%%c"
+#define RI_PMR_ENTRY_BUF_SIZE  (sizeof(RI_PMR_ENTRY_FRMT)+(3+5+RI_DEV_ID_BUF_SIZE+16+(PVR_ANNOTATION_MAX_LEN/2)+PHYS_HEAP_NAME_SIZE+10+10)+ sizeof(RI_FREED_BY_DRIVER))
+#define RI_PMR_ENTRY_FRMT_SIZE (sizeof(RI_PMR_ENTRY_FRMT))
+
+/* Use %5d rather than %d so the output aligns in server/kernel.log, debugFS sees extra spaces */
+#define RI_MEMDESC_ENTRY_PROC_FRMT        "[%5d:%s]"
+#define RI_MEMDESC_ENTRY_PROC_BUF_SIZE    (sizeof(RI_MEMDESC_ENTRY_PROC_FRMT)+5+16)
+
+#define RI_SYS_ALLOC_IMPORT_FRMT      "{Import from PID %d}"
+#define RI_SYS_ALLOC_IMPORT_FRMT_SIZE (sizeof(RI_SYS_ALLOC_IMPORT_FRMT)+5)
+static IMG_CHAR g_szSysAllocImport[RI_SYS_ALLOC_IMPORT_FRMT_SIZE];
+
+#define RI_MEMDESC_ENTRY_IMPORT_FRMT     "{Import from PID %d}"
+#define RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE (sizeof(RI_MEMDESC_ENTRY_IMPORT_FRMT)+5)
+
+#define RI_MEMDESC_ENTRY_FRMT      "%%sPID:%%-5d DEV:%%s 0x%%010" IMG_UINT64_FMTSPECx "\t%%-%ds %%s\t0x%%010" IMG_UINT64_FMTSPECx "\t<%%p> %%s%%s %%s%%c"
+#define RI_MEMDESC_ENTRY_BUF_SIZE  (sizeof(RI_MEMDESC_ENTRY_FRMT)+(3+5+RI_DEV_ID_BUF_SIZE+10+PVR_ANNOTATION_MAX_LEN+RI_MEMDESC_ENTRY_PROC_BUF_SIZE+16+\
+                                               RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE+RI_SYS_ALLOC_IMPORT_FRMT_SIZE)) + sizeof(RI_FREED_BY_DRIVER)
+#define RI_MEMDESC_ENTRY_FRMT_SIZE (sizeof(RI_MEMDESC_ENTRY_FRMT))
+
+
+#define RI_FRMT_SIZE_MAX (MAX(RI_MEMDESC_ENTRY_BUF_SIZE,\
+                              MAX(RI_PMR_ENTRY_BUF_SIZE,\
+                                  MAX(RI_MEMDESC_SUM_BUF_SIZE,\
+                                      RI_PMR_SUM_BUF_SIZE))))
+
+typedef struct _RI_PMR_INFO_
+{
+	uintptr_t uiAddr;
+	PHYS_HEAP *psHeap;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+	size_t uiLogicalSize;
+	size_t uiPhysicalSize;
+	IMG_CHAR *pszAnnotation;
+} RI_PMR_INFO;
+
+
+#define RI_DEVID_MASK						0xFF /* Highest value is PVRSRV_HOST_DEVICE_ID[255] */
+
+/* Flags for Memdescs(RI_SUBLIST) */
+#define RI_IMPORT_FLAG						(8)  /* Indicates the PMR is an import */
+#define RI_SUBALLOC_FLAG					(9)  /* Indicates the PMR is sub-allocatable */
+
+/* Flags for PMRs(RI_LIST) */
+#define RI_SYSALLOC_PMR_FLAG				(10) /* Indicates the PMR belongs to the 'system' process */
+#define RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG	(11) /* Indicates size counted prior, to prevent double reads */
+
+/* Shared flags */
+#define RI_RACC_FLAG						(12) /* Freed by Driver after disconnect */
+#define RI_VALID_FLAG						(13) /* Valid on creation, Only invalid after delete */
+#define RI_HAS_PMR_INFO						(14) /* Entry has RI_PMR_INFO object rather than reference to a PMR */
+
+#define GET_DEVICE_ID(entry)			((entry)->ui16Flags & RI_DEVID_MASK)
+#define SET_DEVICE_ID(entry,id)			BITMASK_SET((entry)->ui16Flags, (id & RI_DEVID_MASK))
+#define IS_HOST_DEVICE(entry)			(((entry)->ui16Flags & RI_DEVID_MASK) == PVRSRV_HOST_DEVICE_ID)
+
+#define IS_IMPORT(entry)				BIT_ISSET((entry)->ui16Flags, RI_IMPORT_FLAG)
+#define IS_SUBALLOC(entry)				BIT_ISSET((entry)->ui16Flags, RI_SUBALLOC_FLAG)
+
+#define IS_SYSPMR(entry)				BIT_ISSET((entry)->ui16Flags, RI_SYSALLOC_PMR_FLAG)
+#define IS_COUNTED_BY_DEBUGFS(entry)	BIT_ISSET((entry)->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG)
+
+#define IS_RACC(entry)					BIT_ISSET((entry)->ui16Flags, RI_RACC_FLAG)
+#define IS_VALID(entry)					BIT_ISSET((entry)->ui16Flags, RI_VALID_FLAG)
+#define HAS_PMR_INFO(entry)				BIT_ISSET((entry)->ui16Flags, RI_HAS_PMR_INFO)
+
+/* Structure used to make linked sublist of memory allocations (MEMDESC) */
+struct _RI_SUBLIST_ENTRY_
+{
+	DLLIST_NODE				sListNode;
+	DLLIST_NODE				sProcListNode;
+	CONNECTION_DATA			*psConnection;
+	struct _RI_LIST_ENTRY_	*psRI;
+	IMG_CHAR				*pszTextB; /* Annotation+(NUL)+ProcName+(NUL) */
+	IMG_DEV_VIRTADDR		sVAddr;
+	IMG_UINT64				ui64Offset;
+	IMG_UINT64				ui64Size;
+	IMG_PID					pid;
+	IMG_UINT16				ui16TextBLength; /* Total length of Annotation+(NUL)+ProcName+(NUL) */
+	IMG_UINT16				ui16Flags; /* Refer to above */
+};
+
+/*
+ * Structure used to make linked list of PMRs. Sublists of allocations
+ * (MEMDESCs) made from these PMRs are chained off these entries.
+ */
+struct _RI_LIST_ENTRY_
+{
+	DLLIST_NODE				sListNode;
+	DLLIST_NODE				sSubListFirst;
+	union {
+		PMR *psPMR;
+		RI_PMR_INFO *psPmrInfo;
+	} pmr_info;
+	IMG_PID					pid;
+	IMG_UINT16				ui16SubListCount;
+	IMG_UINT16				ui16Flags; /* Refer to above */
+};
+
+#define GET_ADDR(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (void*)(entry)->pmr_info.psPmrInfo->uiAddr \
+			: (void*)(entry)->pmr_info.psPMR \
+	)
+#define GET_HEAP(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->psHeap \
+			: PMR_PhysHeap((entry)->pmr_info.psPMR) \
+	)
+#define GET_DEVNODE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->psDeviceNode \
+			: (PVRSRV_DEVICE_NODE *) PMR_DeviceNode((entry)->pmr_info.psPMR) \
+	)
+#define GET_LOGICAL_SIZE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiLogicalSize \
+			: PMR_LogicalSize((entry)->pmr_info.psPMR) \
+	)
+#define GET_PHYSICAL_SIZE(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->uiPhysicalSize \
+			: PMR_PhysicalSize((entry)->pmr_info.psPMR) \
+	)
+#define GET_NAME(entry) ( \
+		HAS_PMR_INFO(entry) \
+			? (entry)->pmr_info.psPmrInfo->pszAnnotation \
+			: PMR_GetAnnotation((entry)->pmr_info.psPMR) \
+	)
+
+/* pszTextB = <Memdesc Annotation>,0,<Proc Name>,0 */
+/* Retrieve the string pointer to the ProcName */
+#define GET_PROC(entry) ((IMG_CHAR*) (strchr((entry)->pszTextB, '\0') + 1))
+
+typedef struct _RI_LIST_ENTRY_ RI_LIST_ENTRY;
+typedef struct _RI_SUBLIST_ENTRY_ RI_SUBLIST_ENTRY;
+
+static HASH_TABLE	*g_pPMR2RIListHashTable;
+static HASH_TABLE	*g_PID2RISublistHashTable;
+
+static POS_LOCK		g_hRILock;
+
+/* Linked list of PMR allocations made against the PVR_SYS_ALLOC_PID and lock
+ * to prevent concurrent access to it.
+ */
+static POS_LOCK		g_hSysAllocPidListLock;
+static DLLIST_NODE	g_sSysAllocPidListHead;
+
+/*
+ * Flag used to indicate if RILock should be destroyed when final PMR entry is
+ * deleted, i.e. if RIDeInitKM() has already been called before that point but
+ * the handle manager has deferred deletion of RI entries.
+ */
+static IMG_BOOL		bRIDeInitDeferred = IMG_FALSE;
+
+/*
+ * Used as head of linked-list of PMR RI entries - this is useful when we wish
+ * to iterate all PMR list entries (when we don't have a PMR ref)
+ */
+static DLLIST_NODE	g_sClientsListHead;
+
+/* Function used to produce string containing info for MEMDESC RI entries (used for both debugfs and kernel log output) */
+static void _GenerateMEMDESCEntryString(RI_SUBLIST_ENTRY *psRISubEntry, IMG_BOOL bDebugFs, IMG_UINT16 ui16MaxStrLen, IMG_CHAR *pszEntryString);
+/* Function used to produce string containing info for PMR RI entries (used for both debugfs and kernel log output) */
+static void _GeneratePMREntryString(RI_LIST_ENTRY *psRIEntry, IMG_BOOL bDebugFs, IMG_UINT16 ui16MaxStrLen, IMG_CHAR *pszEntryString);
+
+static PVRSRV_ERROR _DumpAllEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DeleteAllEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DeleteAllProcEntries (uintptr_t k, uintptr_t v, void* pvPriv);
+static PVRSRV_ERROR _DumpList(PMR *psPMR, IMG_PID pid);
+#define _RIOutput(x) PVR_LOG(x)
+
+static IMG_UINT32
+_ProcHashFunc(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen);
+
+static IMG_UINT32
+_ProcHashFunc(size_t uKeySize, void *pKey, IMG_UINT32 uHashTabLen)
+{
+	IMG_UINT32 *p = (IMG_UINT32 *)pKey;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+	IMG_UINT32 uHashKey = 0;
+
+	PVR_UNREFERENCED_PARAMETER(uHashTabLen);
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		IMG_UINT32 uHashPart = *p++;
+
+		uHashPart += (uHashPart << 12);
+		uHashPart ^= (uHashPart >> 22);
+		uHashPart += (uHashPart << 4);
+		uHashPart ^= (uHashPart >> 9);
+		uHashPart += (uHashPart << 10);
+		uHashPart ^= (uHashPart >> 2);
+		uHashPart += (uHashPart << 7);
+		uHashPart ^= (uHashPart >> 12);
+
+		uHashKey += uHashPart;
+	}
+
+	return uHashKey;
+}
+
+static IMG_BOOL
+_ProcHashComp(size_t uKeySize, void *pKey1, void *pKey2);
+
+static IMG_BOOL
+_ProcHashComp(size_t uKeySize, void *pKey1, void *pKey2)
+{
+	IMG_UINT32 *p1 = (IMG_UINT32 *)pKey1;
+	IMG_UINT32 *p2 = (IMG_UINT32 *)pKey2;
+	IMG_UINT32 uKeyLen = uKeySize / sizeof(IMG_UINT32);
+	IMG_UINT32 ui;
+
+	for (ui = 0; ui < uKeyLen; ui++)
+	{
+		if (*p1++ != *p2++)
+			return IMG_FALSE;
+	}
+
+	return IMG_TRUE;
+}
+
+static void _RILock(void)
+{
+#if (USE_RI_LOCK == 1)
+	OSLockAcquire(g_hRILock);
+#endif
+}
+
+static void _RIUnlock(void)
+{
+#if (USE_RI_LOCK == 1)
+	OSLockRelease(g_hRILock);
+#endif
+}
+
+/* This value maintains a count of the number of PMRs attributed to the
+ * PVR_SYS_ALLOC_PID. Access to this value is protected by g_hRILock, so it
+ * does not need to be an ATOMIC_T.
+ */
+static IMG_UINT32 g_ui32SysAllocPMRCount;
+
+static IMG_BOOL RICheckListHandle(RI_HANDLE hRIHandle)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *) hRIHandle, *psTableEntry;
+	void *pPMRHashKey;
+	IMG_BOOL bValid = IMG_FALSE;
+
+	if ((GET_DEVICE_ID(psRIEntry) > PVRSRV_MAX_DEVICES &&
+	     !IS_HOST_DEVICE(psRIEntry)) ||
+	    !g_pPMR2RIListHashTable)
+	{
+		//Shortcut check
+		return IMG_FALSE;
+	}
+
+	pPMRHashKey = GET_ADDR(psRIEntry);
+
+	_RILock();
+
+	/* Look-up psPMR in Hash Table */
+	psTableEntry = (RI_LIST_ENTRY *) HASH_Retrieve_Extended(g_pPMR2RIListHashTable,
+	                                                        &pPMRHashKey);
+	if (psTableEntry != NULL)
+	{
+		bValid = IS_VALID(psTableEntry);
+	}
+
+	_RIUnlock();
+
+	return bValid;
+}
+
+static IMG_BOOL RICheckSubListHandle(RI_HANDLE hRIHandle)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry, *psTableEntry;
+	uintptr_t hashData;
+	IMG_PID pid;
+	IMG_BOOL bValid = IMG_FALSE;
+
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if ((GET_DEVICE_ID(psRISubEntry) > PVRSRV_MAX_DEVICES) || !g_PID2RISublistHashTable)
+	{
+		//Shortcut check
+		return bValid;
+	}
+	pid = psRISubEntry->pid;
+
+	_RILock();
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (hashData)
+	{
+		psTableEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+		bValid = IS_VALID(psTableEntry);
+	}
+
+	/* Release RI Lock */
+	_RIUnlock();
+	return bValid;
+}
+
+PVRSRV_ERROR RIInitKM(void)
+{
+	IMG_INT iCharsWritten;
+	PVRSRV_ERROR eError;
+
+	bRIDeInitDeferred = IMG_FALSE;
+
+	iCharsWritten = OSSNPrintf(g_szSysAllocImport,
+	            RI_SYS_ALLOC_IMPORT_FRMT_SIZE,
+	            RI_SYS_ALLOC_IMPORT_FRMT,
+	            PVR_SYS_ALLOC_PID);
+	PVR_LOG_IF_FALSE((iCharsWritten>0 && iCharsWritten<(IMG_INT32)RI_SYS_ALLOC_IMPORT_FRMT_SIZE),
+			"OSSNPrintf failed to initialise g_szSysAllocImport");
+
+	eError = OSLockCreate(&g_hSysAllocPidListLock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: OSLockCreate (g_hSysAllocPidListLock) failed (returned %d)",
+		         __func__,
+		         eError));
+	}
+	dllist_init(&(g_sSysAllocPidListHead));
+#if (USE_RI_LOCK == 1)
+	eError = OSLockCreate(&g_hRILock);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: OSLockCreate (g_hRILock) failed (returned %d)",
+		         __func__,
+		         eError));
+	}
+#endif
+	return eError;
+}
+
+void RIDeInitKM(void)
+{
+#if (USE_RI_LOCK == 1)
+	if (g_pPMR2RIListHashTable != NULL && HASH_Count(g_pPMR2RIListHashTable) > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s: called with %d entries remaining - deferring OSLockDestroy()",
+		         __func__,
+		         HASH_Count(g_pPMR2RIListHashTable)));
+		bRIDeInitDeferred = IMG_TRUE;
+	}
+	else
+	{
+		OSLockDestroy(g_hRILock);
+		OSLockDestroy(g_hSysAllocPidListLock);
+	}
+#endif
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RILockAcquireKM
+
+ @Description
+            Acquires the RI Lock (which protects the integrity of the RI
+            linked lists). Caller will be suspended until lock is acquired.
+
+ @Return	None
+
+******************************************************************************/
+void RILockAcquireKM(void)
+{
+	_RILock();
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RILockReleaseKM
+
+ @Description
+            Releases the RI Lock (which protects the integrity of the RI
+            linked lists).
+
+ @Return	None
+
+******************************************************************************/
+void RILockReleaseKM(void)
+{
+	_RIUnlock();
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWritePMREntryWithOwnerKM
+
+ @Description
+            Writes a new Resource Information list entry.
+            The new entry will be inserted at the head of the list of
+            PMR RI entries and assigned the values provided.
+
+ @input     psPMR - Reference (handle) to the PMR to which this reference relates
+
+ @input     ui32Owner - PID of the process which owns the allocation. This
+                        may not be the current process (e.g. a request to
+                        grow a buffer may happen in the context of a kernel
+                        thread, or we may import further resource for a
+                        suballocation made from the FW heap which can then
+                        also be utilized by other processes)
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWritePMREntryWithOwnerKM(PMR *psPMR,
+                                        IMG_PID ui32Owner)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PMR *pPMRHashKey = psPMR;
+	RI_LIST_ENTRY *psRIEntry;
+	uintptr_t hashData;
+
+	/* if Hash table has not been created, create it now */
+	if (!g_pPMR2RIListHashTable)
+	{
+		g_pPMR2RIListHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(PMR*), HASH_Func_Default, HASH_Key_Comp_Default);
+		g_PID2RISublistHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(IMG_PID), _ProcHashFunc, _ProcHashComp);
+	}
+	PVR_RETURN_IF_NOMEM(g_pPMR2RIListHashTable);
+	PVR_RETURN_IF_NOMEM(g_PID2RISublistHashTable);
+
+	PVR_RETURN_IF_INVALID_PARAM(psPMR);
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	psRIEntry = (RI_LIST_ENTRY *)hashData;
+	if (psRIEntry == NULL)
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = (PVRSRV_DEVICE_NODE *)PMR_DeviceNode(psPMR);
+
+		/*
+		 * If failed to find a matching existing entry, create a new one
+		 */
+		psRIEntry = (RI_LIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_LIST_ENTRY));
+		if (!psRIEntry)
+		{
+			/* Error - no memory to allocate for new RI entry */
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, exit_);
+		}
+		else
+		{
+			PMR_FLAGS_T uiPMRFlags = PMR_Flags(psPMR);
+			IMG_UINT32 ui32RICount = HASH_Count(g_pPMR2RIListHashTable);
+
+			/*
+			 * Add new RI Entry
+			 */
+			if (ui32RICount == 0)
+			{
+				/* Initialise PMR entry linked-list head */
+				dllist_init(&g_sClientsListHead);
+			}
+			else if (ui32RICount == IMG_UINT32_MAX)
+			{
+				PVR_LOG_GOTO_WITH_ERROR("ui32RICount", eError, PVRSRV_ERROR_REFCOUNT_OVERFLOW, exit_);
+			}
+
+			dllist_init (&(psRIEntry->sListNode));
+			dllist_init (&(psRIEntry->sSubListFirst));
+			BIT_SET(psRIEntry->ui16Flags, RI_VALID_FLAG);
+
+			/* Check if this PMR should be accounted for under the
+			 * PVR_SYS_ALLOC_PID debugFS entry. This should happen if
+			 * we are in the driver init phase, the flags indicate
+			 * this is a FW Main allocation (made from FW heap)
+			 * or the owner PID is PVR_SYS_ALLOC_PID.
+			 * Also record host dev node allocs on the system PID.
+			 */
+			if (psDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+				PVRSRV_CHECK_FW_MAIN(uiPMRFlags) ||
+				ui32Owner == PVR_SYS_ALLOC_PID ||
+				psDeviceNode == PVRSRVGetPVRSRVData()->psHostMemDeviceNode)
+			{
+				BIT_SET(psRIEntry->ui16Flags, RI_SYSALLOC_PMR_FLAG);
+				psRIEntry->pid = PVR_SYS_ALLOC_PID;
+				OSLockAcquire(g_hSysAllocPidListLock);
+				/* Add this psRIEntry to the list of entries for PVR_SYS_ALLOC_PID */
+				dllist_add_to_tail(&g_sSysAllocPidListHead, (PDLLIST_NODE)&(psRIEntry->sListNode));
+				OSLockRelease(g_hSysAllocPidListLock);
+				g_ui32SysAllocPMRCount++;
+			}
+			else
+			{
+				psRIEntry->pid = ui32Owner;
+				dllist_add_to_tail(&g_sClientsListHead, (PDLLIST_NODE)&(psRIEntry->sListNode));
+			}
+		}
+
+		psRIEntry->pmr_info.psPMR = psPMR;
+		SET_DEVICE_ID(psRIEntry, psDeviceNode->sDevId.ui32InternalID);
+
+		/* Create index entry in Hash Table */
+		HASH_Insert_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey, (uintptr_t)psRIEntry);
+
+		/* Store phRIHandle in PMR structure, so it can delete the associated RI entry when it destroys the PMR */
+		PMRStoreRIHandle(psPMR, psRIEntry);
+	}
+exit_:
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWritePMREntryKM
+
+ @Description
+            Writes a new Resource Information list entry.
+            The new entry will be inserted at the head of the list of
+            PMR RI entries and assigned the values provided.
+
+ @input     psPMR - Reference (handle) to the PMR to which this reference relates
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWritePMREntryKM(PMR *psPMR)
+{
+	return RIWritePMREntryWithOwnerKM(psPMR,
+	                                  OSGetCurrentClientProcessIDKM());
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWriteMEMDESCEntryKM
+
+ @Description
+            Writes a new Resource Information sublist entry.
+            The new entry will be inserted at the head of the sublist of
+            the indicated PMR list entry, and assigned the values provided.
+
+ @input     psConnection - Reference to the Services connection
+ @input     psDeviceNode - Reference to the device node
+ @input     psPMR - Reference (handle) to the PMR to which this MEMDESC RI entry relates
+ @input     ui32TextBSize - Length of string provided in psz8TextB parameter
+ @input     psz8TextB - String describing this secondary reference (may be null)
+ @input     ui64Offset - Offset from the start of the PMR at which this allocation begins
+ @input     ui64Size - Size of this allocation
+ @input     uiFlags - Flags indicating if this is a sub-allocation or an import
+ @output    phRIHandle - Handle to the created RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWriteMEMDESCEntryKM(void* psConnection,
+                                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   PMR *psPMR,
+                                   IMG_UINT32 ui32TextBSize,
+                                   const IMG_CHAR *psz8TextB,
+                                   IMG_UINT64 ui64Offset,
+                                   IMG_UINT64 ui64Size,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   RI_HANDLE *phRIHandle)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry;
+	RI_LIST_ENTRY *psRIEntry;
+	PMR *pPMRHashKey = psPMR;
+	uintptr_t hashData;
+	IMG_PID	pid;
+
+	IMG_CHAR szProcName[RI_PROC_BUF_SIZE];
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+
+	/* Check Hash tables have been created (meaning at least one PMR has been defined) */
+	PVR_RETURN_IF_INVALID_PARAM(g_pPMR2RIListHashTable);
+	PVR_RETURN_IF_INVALID_PARAM(g_PID2RISublistHashTable);
+
+	PVR_RETURN_IF_INVALID_PARAM(psPMR);
+	PVR_RETURN_IF_INVALID_PARAM(phRIHandle);
+
+	/* Only allow request for a firmware context that comes from a direct bridge
+	 * (psConnection == NULL). */
+	PVR_RETURN_IF_INVALID_PARAM(!(psConnection != NULL && PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags)));
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	*phRIHandle = NULL;
+
+	/* Look-up psPMR in Hash Table */
+	hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	psRIEntry = (RI_LIST_ENTRY *)hashData;
+	if (!psRIEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_SUBLIST_ENTRY));
+	if (!psRISubEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		/* Error - no memory to allocate for new RI sublist entry */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	else
+	{
+		/*
+		 * Insert new entry in sublist
+		 */
+		PDLLIST_NODE currentNode = dllist_get_next_node(&(psRIEntry->sSubListFirst));
+
+		/*
+		 * Insert new entry before currentNode
+		 */
+		if (!currentNode)
+		{
+			currentNode = &(psRIEntry->sSubListFirst);
+		}
+		dllist_add_to_tail(currentNode, (PDLLIST_NODE)&(psRISubEntry->sListNode));
+
+		psRISubEntry->psRI = psRIEntry;
+
+		/* Increment number of entries in sublist */
+		psRIEntry->ui16SubListCount++;
+		BIT_SET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+	}
+
+	if (PVRSRV_CHECK_RI_IMPORT(uiFlags))
+	{
+		BIT_SET(psRISubEntry->ui16Flags, RI_IMPORT_FLAG);
+	}
+	else
+	{
+		BIT_UNSET(psRISubEntry->ui16Flags, RI_IMPORT_FLAG);
+	}
+
+	if (PVRSRV_CHECK_RI_SUBALLOC(uiFlags))
+	{
+		BIT_SET(psRISubEntry->ui16Flags, RI_SUBALLOC_FLAG);
+	}
+	else
+	{
+		BIT_UNSET(psRISubEntry->ui16Flags, RI_SUBALLOC_FLAG);
+	}
+
+	/* If allocation is made during device or driver initialisation,
+	 * track the MEMDESC entry under PVR_SYS_ALLOC_PID, otherwise use
+	 * the current PID. */
+	{
+		PVRSRV_DEVICE_NODE *psRIDeviceNode = GET_DEVNODE(psRISubEntry->psRI);
+		IMG_INT iRet;
+
+		/* HostMemDevice doesn't update eDevState hence there's no need to test
+		 * for it. */
+		if (psRIDeviceNode == PVRSRVGetPVRSRVData()->psHostMemDeviceNode)
+		{
+			/* Imports on HostMemDev should be attributed to the importing
+			 * process. This way if an import is not freed before disconnect
+			 * it will be outlined in the gpu_mem_area for the given process.
+			 * Otherwise attribute the rest of the records to PVR_SYS_ALLOC_PID. */
+			if (IS_IMPORT(psRISubEntry))
+			{
+				psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+				iRet = OSStringSafeCopy(szProcName, OSGetCurrentClientProcessNameKM(),
+				                        RI_PROC_BUF_SIZE);
+			}
+			else
+			{
+				psRISubEntry->pid = PVR_SYS_ALLOC_PID;
+				iRet = OSStringSafeCopy(szProcName, "SysProc", RI_PROC_BUF_SIZE);
+
+				if (psRISubEntry->pid != psRISubEntry->psRI->pid)
+				{
+					PVR_LOG(("%s(1): current PID = %u, RI entry PID = %u, RI sub-entry PID = %u",
+					         __func__, OSGetCurrentClientProcessIDKM(), psRISubEntry->psRI->pid,
+					         psRISubEntry->pid));
+				}
+			}
+		}
+		else
+		{
+			/* All allocations done during device initialisation or belonging
+			 * to the Firmware should be attributed to PVR_SYS_ALLOC_PID. */
+			if (psRIDeviceNode->eDevState < PVRSRV_DEVICE_STATE_ACTIVE ||
+			    PVRSRV_CHECK_RI_FWKMD_ALLOC(uiFlags))
+			{
+				psRISubEntry->pid = PVR_SYS_ALLOC_PID;
+				iRet = OSStringSafeCopy(szProcName, "SysProc", RI_PROC_BUF_SIZE);
+
+				if (psRISubEntry->pid != psRISubEntry->psRI->pid)
+				{
+					PVR_LOG(("%s(2): current PID = %u, RI entry PID = %u, RI sub-entry PID = %u",
+					         __func__, OSGetCurrentClientProcessIDKM(), psRISubEntry->psRI->pid,
+					         psRISubEntry->pid));
+				}
+			}
+			else
+			{
+				psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+				iRet = OSStringSafeCopy(szProcName, OSGetCurrentClientProcessNameKM(),
+				                        RI_PROC_BUF_SIZE);
+			}
+		}
+		if (iRet < 0)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: process name has been truncated from '%s' to '%s'",
+			         __func__, OSGetCurrentClientProcessNameKM(), szProcName));
+			psRISubEntry->ui16TextBLength = RI_PROC_BUF_SIZE - 1;
+		}
+		else
+		{
+			psRISubEntry->ui16TextBLength = iRet;
+		}
+
+		if (psz8TextB == NULL)
+		{
+			psz8TextB = "";
+			ui32TextBSize = 0;
+		}
+		if (ui32TextBSize > (RI_ANNO_BUF_SIZE))
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s: TextBSize too long (%u). Text will be truncated "
+					 "to %d characters", __func__,
+					 ui32TextBSize, RI_ANNO_BUF_SIZE));
+			ui32TextBSize = RI_ANNO_BUF_SIZE;
+		}
+
+		/* copy TextB field data plus separator char and terminator */
+		psRISubEntry->ui16TextBLength += ui32TextBSize + 2;
+		psRISubEntry->pszTextB = OSAllocZMemNoStats(psRISubEntry->ui16TextBLength);
+		if (!psRISubEntry->pszTextB)
+		{
+			OSFreeMemNoStats(psRISubEntry);
+			_RIUnlock();
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+		/* TextB is formatted as Annotation+NullTerm+ProcName so we can still
+		 * print annotation without proc name.
+		 * If any of the strings is too long it will be truncated. */
+		(void) OSStringSafeCopy(psRISubEntry->pszTextB, psz8TextB, ui32TextBSize + 1);
+		(void) OSStringSafeCopy(psRISubEntry->pszTextB + ui32TextBSize + 1, szProcName,
+		                        psRISubEntry->ui16TextBLength - ui32TextBSize - 1);
+
+		psRISubEntry->ui64Offset = ui64Offset;
+		psRISubEntry->ui64Size = ui64Size;
+		psRISubEntry->psConnection = psConnection;
+		dllist_init (&(psRISubEntry->sProcListNode));
+	}
+
+	/*
+	 *	Now insert this MEMDESC into the proc list
+	 */
+	/* look-up pid in Hash Table */
+	pid = psRISubEntry->pid;
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (!hashData)
+	{
+		/*
+		 * No allocations for this pid yet
+		 */
+		if (!HASH_Insert_Extended(g_PID2RISublistHashTable,
+		                          (void *) &pid,
+		                          (uintptr_t) &psRISubEntry->sProcListNode))
+		{
+			dllist_remove_node(&psRISubEntry->sListNode);
+			psRIEntry->ui16SubListCount--;
+			OSFreeMemNoStats(psRISubEntry->pszTextB);
+			OSFreeMemNoStats(psRISubEntry);
+
+			_RIUnlock();
+			return PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED;
+		}
+	}
+	else
+	{
+		/*
+		 * Insert allocation into pid allocations linked list
+		 */
+		PDLLIST_NODE currentNode = (PDLLIST_NODE)hashData;
+
+		/*
+		 * Insert new entry
+		 */
+		dllist_add_to_tail(currentNode, (PDLLIST_NODE)&(psRISubEntry->sProcListNode));
+	}
+	*phRIHandle = (RI_HANDLE)psRISubEntry;
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIWriteProcListEntryKM
+
+ @Description
+            Write a new entry in the process list directly. We have to do this
+            because there might be no, multiple or changing PMR handles.
+
+            In the common case we have a PMR that will be added to the PMR list
+            and one or several MemDescs that are associated to it in a sub-list.
+            Additionally these MemDescs will be inserted in the per-process list.
+
+            There might be special descriptors from e.g. new user APIs that
+            are associated with no or multiple PMRs and not just one.
+            These can be now added to the per-process list (as RI_SUBLIST_ENTRY)
+            directly with this function and won't be listed in the PMR list (RIEntry)
+            because there might be no PMR.
+
+            To remove entries from the per-process list, just use
+            RIDeleteMEMDESCEntryKM().
+
+ @input     psConnection - Reference to the Services connection
+ @input     psDeviceNode - Reference to the device node
+ @input     psz8TextB - String describing this secondary reference (may be null)
+ @input     ui64Size - Size of this allocation
+ @input     ui64DevVAddr - Virtual address of this entry
+ @output    phRIHandle - Handle to the created RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIWriteProcListEntryKM(void* psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32TextBSize,
+                                    const IMG_CHAR *psz8TextB,
+                                    IMG_UINT64 ui64Size,
+                                    IMG_UINT64 ui64DevVAddr,
+                                    RI_HANDLE *phRIHandle)
+{
+	uintptr_t hashData = 0;
+	IMG_PID		pid;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_INT iRet;
+
+	if (!g_pPMR2RIListHashTable)
+	{
+		g_pPMR2RIListHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(PMR*), HASH_Func_Default, HASH_Key_Comp_Default);
+		g_PID2RISublistHashTable = HASH_Create_Extended(_RI_INITIAL_HASH_TABLE_SIZE, sizeof(IMG_PID), _ProcHashFunc, _ProcHashComp);
+
+		if (!g_pPMR2RIListHashTable || !g_PID2RISublistHashTable)
+		{
+			/* Error - no memory to allocate for Hash table(s) */
+			return PVRSRV_ERROR_OUT_OF_MEMORY;
+		}
+	}
+
+	/* Acquire RI Lock */
+	_RILock();
+
+	*phRIHandle = NULL;
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)OSAllocZMemNoStats(sizeof(RI_SUBLIST_ENTRY));
+	if (!psRISubEntry)
+	{
+		/* Release RI Lock */
+		_RIUnlock();
+		/* Error - no memory to allocate for new RI sublist entry */
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+	BIT_SET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+
+	psRISubEntry->pid = OSGetCurrentClientProcessIDKM();
+	SET_DEVICE_ID(psRISubEntry, psDeviceNode->sDevId.ui32InternalID);
+
+	if (psz8TextB == NULL)
+	{
+		psz8TextB = "";
+		ui32TextBSize = 0;
+	}
+	if (ui32TextBSize > (RI_ANNO_BUF_SIZE))
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: TextBSize too long (%u). Text will be truncated "
+				 "to %d characters", __func__,
+				 ui32TextBSize, RI_ANNO_BUF_SIZE));
+		ui32TextBSize = RI_ANNO_BUF_SIZE;
+	}
+
+	/* copy TextB field data */
+	psRISubEntry->ui16TextBLength = ui32TextBSize + RI_PROC_BUF_SIZE;
+	psRISubEntry->pszTextB = OSAllocZMemNoStats(psRISubEntry->ui16TextBLength);
+	if (!psRISubEntry->pszTextB)
+	{
+		OSFreeMemNoStats(psRISubEntry);
+		_RIUnlock();
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+	/* TextB is formatted as Annotation+NullTerm+ProcName so we can still
+	 * print annotation without proc name.
+	 */
+	(void) OSStringSafeCopy(psRISubEntry->pszTextB, psz8TextB, ui32TextBSize + 1);
+	iRet = OSStringSafeCopy(psRISubEntry->pszTextB + ui32TextBSize + 1,
+	                        OSGetCurrentClientProcessNameKM(),
+	                        psRISubEntry->ui16TextBLength - ui32TextBSize - 1);
+	if (iRet < 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: process name has been truncated from '%s' to '%s'",
+		         __func__, OSGetCurrentClientProcessNameKM(),
+		         psRISubEntry->pszTextB + ui32TextBSize + 1));
+	}
+	psRISubEntry->ui64Size = ui64Size;
+	psRISubEntry->sVAddr.uiAddr = ui64DevVAddr;
+	psRISubEntry->psConnection = psConnection;
+	dllist_init (&(psRISubEntry->sProcListNode));
+
+	/*
+	 *	Now insert this MEMDESC into the proc list
+	 */
+	/* look-up pid in Hash Table */
+	pid = psRISubEntry->pid;
+	hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+	if (!hashData)
+	{
+		/*
+		 * No allocations for this pid yet
+		 */
+		if (!HASH_Insert_Extended(g_PID2RISublistHashTable,
+		                          (void *) &pid,
+		                          (uintptr_t) &psRISubEntry->sProcListNode))
+		{
+			OSFreeMemNoStats(psRISubEntry->pszTextB);
+			OSFreeMemNoStats(psRISubEntry);
+			_RIUnlock();
+			return PVRSRV_ERROR_INSERT_HASH_TABLE_DATA_FAILED;
+		}
+	}
+	else
+	{
+		/*
+		 * Insert allocation into pid allocations linked list
+		 */
+		PDLLIST_NODE currentNode = (PDLLIST_NODE)hashData;
+
+		/*
+		 * Insert new entry
+		 */
+		dllist_add_to_tail(currentNode, (PDLLIST_NODE)&(psRISubEntry->sProcListNode));
+	}
+	*phRIHandle = (RI_HANDLE)psRISubEntry;
+	/* Release RI Lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIUpdateMEMDESCAddrKM
+
+ @Description
+            Update a Resource Information entry.
+
+ @input     hRIHandle - Handle of object whose reference info is to be updated
+ @input     sVAddr - New address for the RI entry
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIUpdateMEMDESCAddrKM(RI_HANDLE hRIHandle,
+								   IMG_DEV_VIRTADDR sVAddr)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry;
+
+	PVR_RETURN_IF_INVALID_PARAM(hRIHandle);
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if (!IS_VALID(psRISubEntry))
+	{
+		/* Pointer does not point to valid structure */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	psRISubEntry->sVAddr.uiAddr = sVAddr.uiAddr;
+
+	/* Release RI lock */
+	_RIUnlock();
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR RIDeletePMREntryUnlocked(RI_LIST_ENTRY *psRIEntry)
+{
+	PMR			*pPMRHashKey;
+	uintptr_t hashValue = 0;
+
+	/* Remove the HASH table index entry */
+	pPMRHashKey = GET_ADDR(psRIEntry);
+	hashValue = HASH_Remove_Extended(g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "RI");
+
+	BIT_UNSET(psRIEntry->ui16Flags, RI_VALID_FLAG);
+
+	/* Remove PMR entry from linked-list of PMR entries */
+	dllist_remove_node((PDLLIST_NODE)&(psRIEntry->sListNode));
+
+	if (IS_SYSPMR(psRIEntry))
+	{
+		g_ui32SysAllocPMRCount--;
+	}
+
+	if (IS_RACC(psRIEntry))
+	{
+		OSFreeMemNoStats(psRIEntry->pmr_info.psPmrInfo);
+	}
+
+	/* Now, free the memory used to store the RI entry */
+	OSFreeMemNoStats(psRIEntry);
+	psRIEntry = NULL;
+
+	/* If the hash table is now empty we can delete the RI hash table */
+	if (HASH_Count(g_pPMR2RIListHashTable) == 0)
+	{
+		HASH_Delete(g_pPMR2RIListHashTable);
+		g_pPMR2RIListHashTable = NULL;
+	}
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _RICreateAndSetPmrInfo(RI_LIST_ENTRY *const psRIEntry)
+{
+	const IMG_CHAR *const pszAnnotation = PMR_GetAnnotation(psRIEntry->pmr_info.psPMR);
+	const IMG_UINT32 uiLength = OSStringNLength(pszAnnotation, DEVMEM_ANNOTATION_MAX_LEN) + 1;
+
+	RI_PMR_INFO *psPmrInfo = OSAllocZMemNoStats(sizeof(*psPmrInfo) + uiLength);
+	PVR_LOG_RETURN_IF_NOMEM(psPmrInfo, "OSAllocZMemNoStats");
+
+	psPmrInfo->uiAddr = (uintptr_t) psRIEntry->pmr_info.psPMR;
+	psPmrInfo->psHeap = PMR_PhysHeap(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->psDeviceNode = (PVRSRV_DEVICE_NODE *) PMR_DeviceNode(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->uiLogicalSize = PMR_LogicalSize(psRIEntry->pmr_info.psPMR);
+	psPmrInfo->uiPhysicalSize = PMR_PhysicalSize(psRIEntry->pmr_info.psPMR);
+
+	psPmrInfo->pszAnnotation = IMG_OFFSET_ADDR(psPmrInfo, sizeof(*psPmrInfo));
+	OSStringSafeCopy(psPmrInfo->pszAnnotation, pszAnnotation, uiLength);
+
+	psRIEntry->pmr_info.psPmrInfo = psPmrInfo;
+	BIT_SET(psRIEntry->ui16Flags, RI_HAS_PMR_INFO);
+
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeletePMREntryKM
+
+ @Description
+            Delete a Resource Information entry.
+
+ @input     hRIHandle - Handle of object whose reference info is to be deleted
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeletePMREntryKM(RI_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_LIST_ENTRY *psRIEntry;
+
+	psRIEntry = (RI_LIST_ENTRY *) hRIHandle;
+
+	if (!RICheckListHandle(hRIHandle))
+	{
+		/* Pointer does not point to valid structure. */
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_INVALID_PARAMS, ErrReturn);
+	}
+	else if (IS_RACC(psRIEntry))
+	{
+		/* Keep this entry so that it can be inspected as a memory leak in the
+		 * gpu_mem_area stats. */
+		return PVRSRV_OK;
+	}
+
+	if (psRIEntry->ui16SubListCount != 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%p not deleted. RIEntry(%s) still has %u allocation(s)",
+		         psRIEntry, GET_NAME(psRIEntry), psRIEntry->ui16SubListCount));
+
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DEVICEMEM_ALLOCATIONS_REMAIN_IN_HEAP,
+		                    ErrCreateAndAssignPmrInfo);
+	}
+
+	_RILock();
+
+	eError = RIDeletePMREntryUnlocked(psRIEntry);
+
+	_RIUnlock();
+
+	/* If deInit has been deferred, we can now destroy the RI Lock */
+	if (bRIDeInitDeferred && (g_pPMR2RIListHashTable == NULL || HASH_Count(g_pPMR2RIListHashTable) == 0))
+	{
+		OSLockDestroy(g_hRILock);
+	}
+
+	return PVRSRV_OK;
+
+ErrCreateAndAssignPmrInfo:
+	{
+		PVRSRV_ERROR eError2 = _RICreateAndSetPmrInfo(psRIEntry);
+		PVR_RETURN_IF_ERROR(eError2);
+	}
+ErrReturn:
+	return eError;
+}
+
+static PVRSRV_ERROR RIDeleteMemdescEntryUnlocked(RI_SUBLIST_ENTRY *psRISubEntry)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	uintptr_t hashData;
+	uintptr_t hashValue = 0;
+	IMG_PID pid;
+
+	/* For entries which do have a parent PMR remove the node from the sublist */
+	if (psRISubEntry->psRI)
+	{
+		psRIEntry = (RI_LIST_ENTRY *)psRISubEntry->psRI;
+
+		/* Now, remove entry from the sublist */
+		dllist_remove_node(&(psRISubEntry->sListNode));
+	}
+
+	BIT_UNSET(psRISubEntry->ui16Flags, RI_VALID_FLAG);
+
+	/* Remove the entry from the proc allocations linked list */
+	pid = psRISubEntry->pid;
+	/* If this is the only allocation for this pid, just remove it from the hash table */
+	if (dllist_get_next_node(&(psRISubEntry->sProcListNode)) == NULL)
+	{
+		hashValue = HASH_Remove_Extended(g_PID2RISublistHashTable, (void *)&pid);
+		PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "PID");
+		/* Delete the hash table if there are now no entries */
+		if (HASH_Count(g_PID2RISublistHashTable) == 0)
+		{
+			HASH_Delete(g_PID2RISublistHashTable);
+			g_PID2RISublistHashTable = NULL;
+		}
+	}
+	else
+	{
+		hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&pid);
+		if ((PDLLIST_NODE)hashData == &(psRISubEntry->sProcListNode))
+		{
+			hashValue = HASH_Remove_Extended(g_PID2RISublistHashTable, (void *)&pid);
+			PVR_LOG_RETURN_IF_INVALID_PARAM(hashValue, "PID");
+			HASH_Insert_Extended (g_PID2RISublistHashTable, (void *)&pid, (uintptr_t)dllist_get_next_node(&(psRISubEntry->sProcListNode)));
+		}
+	}
+	dllist_remove_node(&(psRISubEntry->sProcListNode));
+
+	/* Now, free the memory used to store the sublist entry */
+	OSFreeMemNoStats(psRISubEntry->pszTextB);
+	OSFreeMemNoStats(psRISubEntry);
+	psRISubEntry = NULL;
+
+	/*
+	 * Decrement number of entries in sublist if this MemDesc had a parent entry.
+	 */
+	if (psRIEntry)
+	{
+		psRIEntry->ui16SubListCount--;
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeleteMEMDESCEntryKM
+
+ @Description
+            Delete a Resource Information entry.
+            Entry can be from RIEntry list or ProcList.
+
+ @input     hRIHandle - Handle of object whose reference info is to be deleted
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeleteMEMDESCEntryKM(RI_HANDLE hRIHandle)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_SUBLIST_ENTRY *psRISubEntry;
+
+	psRISubEntry = (RI_SUBLIST_ENTRY *)hRIHandle;
+	if (!RICheckSubListHandle(hRIHandle))
+	{
+		/* Pointer does not point to valid structure */
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	else if (IS_RACC(psRISubEntry))
+	{
+		return PVRSRV_OK;
+	}
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = RIDeleteMemdescEntryUnlocked(psRISubEntry);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	/*
+	 * Make the handle NULL once MEMDESC RI entry is deleted
+	 */
+	hRIHandle = NULL;
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDeleteListKM
+
+ @Description
+            Delete all Resource Information entries and free associated
+            memory.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDeleteListKM(void)
+{
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+
+	_RILock();
+
+	if (g_pPMR2RIListHashTable)
+	{
+		eResult = HASH_Iterate(g_pPMR2RIListHashTable, (HASH_pfnCallback)_DeleteAllEntries, NULL);
+		if (eResult == PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+		{
+			/*
+			 * PVRSRV_ERROR_RESOURCE_UNAVAILABLE is used to stop the Hash iterator when
+			 * the hash table gets deleted as a result of deleting the final PMR entry,
+			 * so this is not a real error condition...
+			 */
+			eResult = PVRSRV_OK;
+		}
+	}
+
+	/* After the run through the RIHashTable that holds the PMR entries there might be
+	 * still entries left in the per-process hash table because they were added with
+	 * RIWriteProcListEntryKM() and have no PMR parent associated.
+	 */
+	if (g_PID2RISublistHashTable)
+	{
+		eResult = HASH_Iterate(g_PID2RISublistHashTable, (HASH_pfnCallback) _DeleteAllProcEntries, NULL);
+		if (eResult == PVRSRV_ERROR_RESOURCE_UNAVAILABLE)
+		{
+			/*
+			 * PVRSRV_ERROR_RESOURCE_UNAVAILABLE is used to stop the Hash iterator when
+			 * the hash table gets deleted as a result of deleting the final PMR entry,
+			 * so this is not a real error condition...
+			 */
+			eResult = PVRSRV_OK;
+		}
+	}
+
+	_RIUnlock();
+
+	return eResult;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpListKM
+
+ @Description
+            Dumps out the contents of the RI List entry for the
+            specified PMR, and all MEMDESC allocation entries
+            in the associated sub linked list.
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @input     psPMR - PMR for which RI entry details are to be output
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpListKM(PMR *psPMR)
+{
+	PVRSRV_ERROR eError;
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpList(psPMR, 0);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIGetListEntryKM
+
+ @Description
+            Returns pointer to a formatted string with details of the specified
+            list entry. If no entry exists (e.g. it may have been deleted
+            since the previous call), NULL is returned.
+
+ @input     pid - pid for which RI entry details are to be output
+ @input     ppHandle - handle to the entry, if NULL, the first entry will be
+                     returned.
+ @output    pszEntryString - string to be output for the entry
+ @output    hEntry - hEntry will be returned pointing to the next entry
+                     (or NULL if there is no next entry)
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+IMG_BOOL RIGetListEntryKM(IMG_PID pid,
+						  IMG_HANDLE **ppHandle,
+						  IMG_CHAR **ppszEntryString)
+{
+	RI_SUBLIST_ENTRY  *psRISubEntry = NULL;
+	RI_LIST_ENTRY  *psRIEntry = NULL;
+	uintptr_t     hashData = 0;
+	IMG_PID       hashKey  = pid;
+
+	static IMG_CHAR acStringBuffer[RI_FRMT_SIZE_MAX];
+
+	static IMG_UINT64 ui64TotalMemdescAlloc;
+	static IMG_UINT64 ui64TotalImport;
+	static IMG_UINT64 ui64TotalPMRAlloc;
+	static IMG_UINT64 ui64TotalPMRBacked;
+	static enum {
+		RI_GET_STATE_MEMDESCS_LIST_START,
+		RI_GET_STATE_MEMDESCS_SUMMARY,
+		RI_GET_STATE_PMR_LIST,
+		RI_GET_STATE_PMR_SUMMARY,
+		RI_GET_STATE_END,
+		RI_GET_STATE_LAST
+	} g_bNextGetState = RI_GET_STATE_MEMDESCS_LIST_START;
+
+	static DLLIST_NODE *psNode;
+	static DLLIST_NODE *psSysAllocNode;
+	static IMG_CHAR szProcName[RI_PROC_BUF_SIZE];
+	static IMG_UINT32 ui32ProcessedSysAllocPMRCount;
+
+	acStringBuffer[0] = '\0';
+
+	switch (g_bNextGetState)
+	{
+	case RI_GET_STATE_MEMDESCS_LIST_START:
+		/* look-up pid in Hash Table, to obtain first entry for pid */
+		hashData = HASH_Retrieve_Extended(g_PID2RISublistHashTable, (void *)&hashKey);
+		if (hashData)
+		{
+			if (*ppHandle)
+			{
+				psRISubEntry = (RI_SUBLIST_ENTRY *)*ppHandle;
+				if (!IS_VALID(psRISubEntry))
+				{
+					psRISubEntry = NULL;
+				}
+			}
+			else
+			{
+				psRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+				if (!IS_VALID(psRISubEntry))
+				{
+					psRISubEntry = NULL;
+				}
+			}
+		}
+
+		if (psRISubEntry)
+		{
+			PDLLIST_NODE psNextProcListNode = dllist_get_next_node(&psRISubEntry->sProcListNode);
+
+			if (IS_IMPORT(psRISubEntry))
+			{
+				ui64TotalImport += psRISubEntry->ui64Size;
+			}
+			else
+			{
+				ui64TotalMemdescAlloc += psRISubEntry->ui64Size;
+			}
+
+			_GenerateMEMDESCEntryString(psRISubEntry,
+										IMG_TRUE,
+										RI_MEMDESC_ENTRY_BUF_SIZE,
+										acStringBuffer);
+
+			if (szProcName[0] == '\0')
+			{
+				OSStringSafeCopy(szProcName, (pid == PVR_SYS_ALLOC_PID) ?
+						PVRSRV_MODNAME : GET_PROC(psRISubEntry), RI_PROC_BUF_SIZE);
+			}
+
+
+			*ppszEntryString = acStringBuffer;
+			*ppHandle        = (IMG_HANDLE)IMG_CONTAINER_OF(psNextProcListNode, RI_SUBLIST_ENTRY, sProcListNode);
+
+			if (psNextProcListNode == NULL ||
+				psNextProcListNode == (PDLLIST_NODE)hashData)
+			{
+				g_bNextGetState = RI_GET_STATE_MEMDESCS_SUMMARY;
+			}
+			/* else continue to list MEMDESCs */
+		}
+		else
+		{
+			if (ui64TotalMemdescAlloc == 0)
+			{
+				acStringBuffer[0] = '\0';
+				*ppszEntryString = acStringBuffer;
+				g_bNextGetState = RI_GET_STATE_MEMDESCS_SUMMARY;
+			}
+			/* else continue to list MEMDESCs */
+		}
+		break;
+
+	case RI_GET_STATE_MEMDESCS_SUMMARY:
+		OSSNPrintf(acStringBuffer,
+		           RI_MEMDESC_SUM_BUF_SIZE,
+		           RI_MEMDESC_SUM_FRMT,
+		           pid,
+		           szProcName,
+		           ui64TotalMemdescAlloc,
+		           ui64TotalMemdescAlloc >> 10,
+		           ui64TotalImport,
+		           ui64TotalImport >> 10,
+		           (ui64TotalMemdescAlloc + ui64TotalImport),
+		           (ui64TotalMemdescAlloc + ui64TotalImport) >> 10);
+
+		*ppszEntryString = acStringBuffer;
+		ui64TotalMemdescAlloc = 0;
+		ui64TotalImport = 0;
+		szProcName[0] = '\0';
+
+		g_bNextGetState = RI_GET_STATE_PMR_LIST;
+		break;
+
+	case RI_GET_STATE_PMR_LIST:
+		if (pid == PVR_SYS_ALLOC_PID)
+		{
+			OSLockAcquire(g_hSysAllocPidListLock);
+			acStringBuffer[0] = '\0';
+			if (!psSysAllocNode)
+			{
+				psSysAllocNode = &g_sSysAllocPidListHead;
+				ui32ProcessedSysAllocPMRCount = 0;
+			}
+			psSysAllocNode = dllist_get_next_node(psSysAllocNode);
+
+			if (szProcName[0] == '\0')
+			{
+				OSStringSafeCopy(szProcName, PVRSRV_MODNAME, RI_PROC_BUF_SIZE);
+			}
+			if (psSysAllocNode != NULL && psSysAllocNode != &g_sSysAllocPidListHead)
+			{
+				psRIEntry = IMG_CONTAINER_OF((PDLLIST_NODE)psSysAllocNode, RI_LIST_ENTRY, sListNode);
+				_GeneratePMREntryString(psRIEntry,
+										IMG_TRUE,
+										RI_PMR_ENTRY_BUF_SIZE,
+										acStringBuffer);
+				ui64TotalPMRAlloc += GET_LOGICAL_SIZE(psRIEntry);
+				ui64TotalPMRBacked += GET_PHYSICAL_SIZE(psRIEntry);
+
+				ui32ProcessedSysAllocPMRCount++;
+				if (ui32ProcessedSysAllocPMRCount > g_ui32SysAllocPMRCount+1)
+				{
+					g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+				}
+				/* else continue to list PMRs */
+			}
+			else
+			{
+				g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+			}
+			*ppszEntryString = (IMG_CHAR *)acStringBuffer;
+			OSLockRelease(g_hSysAllocPidListLock);
+		}
+		else
+		{
+			IMG_BOOL bPMRToDisplay = IMG_FALSE;
+
+			/* Iterate through the 'touched' PMRs and display details */
+			if (!psNode)
+			{
+				psNode = dllist_get_next_node(&g_sClientsListHead);
+			}
+			else
+			{
+				psNode = dllist_get_next_node(psNode);
+			}
+
+			while ((psNode != NULL && psNode != &g_sClientsListHead) &&
+					!bPMRToDisplay)
+			{
+				psRIEntry =	IMG_CONTAINER_OF(psNode, RI_LIST_ENTRY, sListNode);
+				if (psRIEntry->pid == pid)
+				{
+					/* This PMR was 'touched', so display details and unflag it*/
+					_GeneratePMREntryString(psRIEntry,
+											IMG_TRUE,
+											RI_PMR_ENTRY_BUF_SIZE,
+											acStringBuffer);
+					ui64TotalPMRAlloc += GET_LOGICAL_SIZE(psRIEntry);
+					ui64TotalPMRBacked += GET_PHYSICAL_SIZE(psRIEntry);
+
+					/* Remember the name of the process for 1 PMR for the summary */
+					if (szProcName[0] == '\0')
+					{
+						psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)), RI_SUBLIST_ENTRY, sListNode);
+						OSStringSafeCopy(szProcName, GET_PROC(psRISubEntry), RI_PROC_BUF_SIZE);
+					}
+					bPMRToDisplay = IMG_TRUE;
+				}
+				else
+				{
+					psNode = dllist_get_next_node(psNode);
+				}
+			}
+
+			if (psNode == NULL || (psNode == &g_sClientsListHead))
+			{
+				g_bNextGetState = RI_GET_STATE_PMR_SUMMARY;
+			}
+			/* else continue listing PMRs */
+		}
+		break;
+
+	case RI_GET_STATE_PMR_SUMMARY:
+		OSSNPrintf(acStringBuffer,
+		           RI_PMR_SUM_BUF_SIZE,
+		           RI_PMR_SUM_FRMT,
+		           pid,
+		           szProcName,
+		           ui64TotalPMRAlloc,
+		           ui64TotalPMRAlloc >> 10,
+		           ui64TotalPMRBacked,
+		           ui64TotalPMRBacked >> 10);
+
+		*ppszEntryString = acStringBuffer;
+		ui64TotalPMRAlloc = 0;
+		ui64TotalPMRBacked = 0;
+		szProcName[0] = '\0';
+		psSysAllocNode = NULL;
+
+		g_bNextGetState = RI_GET_STATE_END;
+		break;
+
+	default:
+		PVR_DPF((PVR_DBG_ERROR, "%s: Bad %d)",__func__, g_bNextGetState));
+
+		__fallthrough;
+	case RI_GET_STATE_END:
+		/* Reset state ready for the next gpu_mem_area file to display */
+		*ppszEntryString = NULL;
+		*ppHandle        = NULL;
+		psNode = NULL;
+		szProcName[0] = '\0';
+
+		g_bNextGetState = RI_GET_STATE_MEMDESCS_LIST_START;
+		return IMG_FALSE;
+		break;
+	}
+
+	return IMG_TRUE;
+}
+
+/* Function used to produce string containing info for MEMDESC RI entries (used for both debugfs and kernel log output) */
+static void _GenerateMEMDESCEntryString(RI_SUBLIST_ENTRY *psRISubEntry,
+                                        IMG_BOOL bDebugFs,
+                                        IMG_UINT16 ui16MaxStrLen,
+                                        IMG_CHAR *pszEntryString)
+{
+	IMG_CHAR szProc[RI_MEMDESC_ENTRY_PROC_BUF_SIZE];
+	IMG_CHAR szImport[RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE];
+	IMG_CHAR szEntryFormat[RI_MEMDESC_ENTRY_FRMT_SIZE];
+	const IMG_CHAR *pszAnnotationText;
+	IMG_PID uiRIPid = 0;
+	PMR* psRIPMR = NULL;
+	IMG_BOOL bSysPMR = IMG_FALSE;
+	IMG_BOOL bHostDevice = IS_HOST_DEVICE(psRISubEntry);
+	IMG_CHAR szDeviceID[RI_DEV_ID_BUF_SIZE];
+
+	if (psRISubEntry->psRI != NULL)
+	{
+		uiRIPid = psRISubEntry->psRI->pid;
+		psRIPMR = (PMR*)GET_ADDR(psRISubEntry->psRI);
+		bSysPMR = IS_SYSPMR(psRISubEntry->psRI);
+	}
+
+	OSSNPrintf(szEntryFormat,
+			RI_MEMDESC_ENTRY_FRMT_SIZE,
+			RI_MEMDESC_ENTRY_FRMT,
+			DEVMEM_ANNOTATION_MAX_LEN);
+
+	if (!bDebugFs)
+	{
+		/* we don't include process ID info for debugfs output */
+		OSSNPrintf(szProc,
+				RI_MEMDESC_ENTRY_PROC_BUF_SIZE,
+				RI_MEMDESC_ENTRY_PROC_FRMT,
+				psRISubEntry->pid,
+				GET_PROC(psRISubEntry));
+	}
+
+	if (!bHostDevice)
+	{
+		OSSNPrintf(szDeviceID,
+				   sizeof(szDeviceID),
+				   "%-3d",
+				   GET_DEVICE_ID(psRISubEntry));
+	}
+
+	if (IS_IMPORT(psRISubEntry) && psRIPMR)
+	{
+		OSSNPrintf((IMG_CHAR *)&szImport,
+		           RI_MEMDESC_ENTRY_IMPORT_BUF_SIZE,
+		           RI_MEMDESC_ENTRY_IMPORT_FRMT,
+		           uiRIPid);
+		/* Set pszAnnotationText to that of the 'parent' PMR RI entry */
+		pszAnnotationText = GET_NAME(psRISubEntry->psRI);
+	}
+	else if (!IS_SUBALLOC(psRISubEntry) && psRIPMR)
+	{
+		/* Set pszAnnotationText to that of the 'parent' PMR RI entry */
+		pszAnnotationText = GET_NAME(psRISubEntry->psRI);
+	}
+	else
+	{
+		/* Set pszAnnotationText to that of the MEMDESC RI entry */
+		pszAnnotationText = psRISubEntry->pszTextB;
+	}
+
+	/* Don't print memdescs if they are local imports
+	 * (i.e. imported PMRs allocated by this process)
+	 */
+	if (bDebugFs &&
+		((psRISubEntry->sVAddr.uiAddr + psRISubEntry->ui64Offset) == 0) &&
+		(IS_IMPORT(psRISubEntry) && ((psRISubEntry->pid == uiRIPid)
+									 || (psRISubEntry->pid == PVR_SYS_ALLOC_PID))))
+	{
+		/* Don't print this entry */
+		pszEntryString[0] = '\0';
+	}
+	else
+	{
+		OSSNPrintf(pszEntryString,
+				   ui16MaxStrLen,
+				   szEntryFormat,
+				   (bDebugFs ? "" : "   "),
+				   psRISubEntry->pid,
+				   (bHostDevice ? "-  " : szDeviceID),
+				   (psRISubEntry->sVAddr.uiAddr + psRISubEntry->ui64Offset),
+				   pszAnnotationText,
+				   (bDebugFs ? "" : (char *)szProc),
+				   psRISubEntry->ui64Size,
+				   psRIPMR,
+				   (IS_IMPORT(psRISubEntry) ? (char *)&szImport : ""),
+				   (!IS_IMPORT(psRISubEntry) &&
+				    (bSysPMR) &&
+				    (psRISubEntry->pid != PVR_SYS_ALLOC_PID)) ? g_szSysAllocImport : "",
+				   (IS_RACC(psRISubEntry) ? RI_FREED_BY_DRIVER : ""),
+				   (bDebugFs ? '\n' : ' '));
+	}
+}
+
+/* Function used to produce string containing info for PMR RI entries (used for debugfs and kernel log output) */
+static void _GeneratePMREntryString(RI_LIST_ENTRY *psRIEntry,
+                                    IMG_BOOL bDebugFs,
+                                    IMG_UINT16 ui16MaxStrLen,
+                                    IMG_CHAR *pszEntryString)
+{
+	const IMG_CHAR*   pszAnnotationText;
+	const IMG_CHAR*   pszHeapText;
+
+	IMG_CHAR          szEntryFormat[RI_PMR_ENTRY_FRMT_SIZE];
+	IMG_BOOL          bHostDevice = IS_HOST_DEVICE(psRIEntry);
+	IMG_CHAR          szDeviceID[RI_DEV_ID_BUF_SIZE];
+
+	OSSNPrintf(szEntryFormat,
+			RI_PMR_ENTRY_FRMT_SIZE,
+			RI_PMR_ENTRY_FRMT,
+			(DEVMEM_ANNOTATION_MAX_LEN/2),
+			PHYS_HEAP_NAME_SIZE);
+
+	/* Set pszAnnotationText to that PMR RI entry */
+	pszAnnotationText = GET_NAME(psRIEntry);
+
+	/* Acquire PhysHeap Name to that PMR RI entry */
+	pszHeapText = PhysHeapName(GET_HEAP(psRIEntry));
+
+	if (!bHostDevice)
+	{
+		OSSNPrintf(szDeviceID,
+				   sizeof(szDeviceID),
+				   "%-3d",
+				   GET_DEVICE_ID(psRIEntry));
+	}
+
+	OSSNPrintf(pszEntryString,
+	           ui16MaxStrLen,
+	           szEntryFormat,
+	           (bDebugFs ? "" : "   "),
+	           psRIEntry->pid,
+	           (bHostDevice ? "-  " : szDeviceID),
+	           GET_ADDR(psRIEntry),
+	           pszAnnotationText,
+	           pszHeapText,
+	           GET_LOGICAL_SIZE(psRIEntry),
+	           GET_PHYSICAL_SIZE(psRIEntry),
+	           (IS_RACC(psRIEntry) ? RI_FREED_BY_DRIVER : ""),
+	           (bDebugFs ? '\n' : ' '));
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_DumpList
+
+ @Description
+            Dumps out RI List entries according to parameters passed.
+
+ @input     psPMR - If not NULL, function will output the RI entries for
+                   the specified PMR only
+ @input     pid - If non-zero, the function will only output MEMDESC RI
+                  entries made by the process with ID pid.
+                  If zero, all MEMDESC RI entries will be output.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR _DumpList(PMR *psPMR, IMG_PID pid)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_UINT16 ui16SubEntriesParsed = 0;
+	uintptr_t hashData = 0;
+	IMG_PID hashKey;
+	PMR *pPMRHashKey = psPMR;
+	IMG_BOOL bDisplayedThisPMR = IMG_FALSE;
+
+	PVR_RETURN_IF_INVALID_PARAM(psPMR);
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		if (pid != 0)
+		{
+			/* look-up pid in Hash Table */
+			hashKey = pid;
+			hashData = HASH_Retrieve_Extended (g_PID2RISublistHashTable, (void *)&hashKey);
+			if (hashData)
+			{
+				psRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+				if (psRISubEntry)
+				{
+					psRIEntry = psRISubEntry->psRI;
+				}
+			}
+		}
+		else
+		{
+			/* Look-up psPMR in Hash Table */
+			hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+			psRIEntry = (RI_LIST_ENTRY *)hashData;
+		}
+		if (!psRIEntry)
+		{
+			/* No entry found in hash table */
+			return PVRSRV_ERROR_NOT_FOUND;
+		}
+		while (psRIEntry)
+		{
+			bDisplayedThisPMR = IMG_FALSE;
+			/* Output details for RI entry */
+			if (!pid)
+			{
+				_RIOutput (("%s <%p> suballocs:%d size:0x%010" IMG_UINT64_FMTSPECx,
+				            GET_NAME(psRIEntry),
+				            GET_ADDR(psRIEntry),
+				            (IMG_UINT)psRIEntry->ui16SubListCount,
+				            GET_LOGICAL_SIZE(psRIEntry)));
+				bDisplayedThisPMR = IMG_TRUE;
+			}
+			ui16SubEntriesParsed = 0;
+			if (psRIEntry->ui16SubListCount)
+			{
+#if _DUMP_LINKEDLIST_INFO
+				_RIOutput (("RI LIST: {sSubListFirst.psNextNode:0x%p}\n",
+				            psRIEntry->sSubListFirst.psNextNode));
+#endif /* _DUMP_LINKEDLIST_INFO */
+				if (!pid)
+				{
+					psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)),
+					                                RI_SUBLIST_ENTRY, sListNode);
+				}
+				/* Traverse RI sublist and output details for each entry */
+				while (psRISubEntry)
+				{
+					if (psRIEntry)
+					{
+						if ((ui16SubEntriesParsed >= psRIEntry->ui16SubListCount))
+						{
+							break;
+						}
+						if (!bDisplayedThisPMR)
+						{
+							_RIOutput (("%s <%p> suballocs:%d size:0x%010" IMG_UINT64_FMTSPECx,
+								    GET_NAME(psRIEntry),
+								    GET_ADDR(psRIEntry),
+								    (IMG_UINT)psRIEntry->ui16SubListCount,
+								    GET_LOGICAL_SIZE(psRIEntry)));
+							bDisplayedThisPMR = IMG_TRUE;
+						}
+					}
+#if _DUMP_LINKEDLIST_INFO
+					_RIOutput (("RI LIST:    [this subentry:0x%p]\n",psRISubEntry));
+					_RIOutput (("RI LIST:     psRI:0x%p\n",psRISubEntry->psRI));
+#endif /* _DUMP_LINKEDLIST_INFO */
+
+					{
+						IMG_CHAR szEntryString[RI_MEMDESC_ENTRY_BUF_SIZE];
+
+						_GenerateMEMDESCEntryString(psRISubEntry,
+						                            IMG_FALSE,
+						                            RI_MEMDESC_ENTRY_BUF_SIZE,
+						                            szEntryString);
+						_RIOutput (("%s",szEntryString));
+					}
+
+					if (pid)
+					{
+						if ((dllist_get_next_node(&(psRISubEntry->sProcListNode)) == NULL) ||
+							(dllist_get_next_node(&(psRISubEntry->sProcListNode)) == (PDLLIST_NODE)hashData))
+						{
+							psRISubEntry = NULL;
+						}
+						else
+						{
+							psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sProcListNode)),
+							                                RI_SUBLIST_ENTRY, sProcListNode);
+							if (psRISubEntry)
+							{
+								if (psRIEntry != psRISubEntry->psRI)
+								{
+									/*
+									 * The next MEMDESC in the process linked list is in a different PMR
+									 */
+									psRIEntry = psRISubEntry->psRI;
+									bDisplayedThisPMR = IMG_FALSE;
+								}
+							}
+						}
+					}
+					else
+					{
+						ui16SubEntriesParsed++;
+						psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sListNode)),
+						                                RI_SUBLIST_ENTRY, sListNode);
+					}
+				}
+			}
+			if (!pid && psRIEntry)
+			{
+				if (ui16SubEntriesParsed != psRIEntry->ui16SubListCount)
+				{
+					/*
+					 * Output error message as sublist does not contain the
+					 * number of entries indicated by sublist count
+					 */
+					_RIOutput (("RI ERROR: RI sublist contains %d entries, not %d entries\n",
+					            ui16SubEntriesParsed, psRIEntry->ui16SubListCount));
+				}
+				else if (psRIEntry->ui16SubListCount && !dllist_get_next_node(&(psRIEntry->sSubListFirst)))
+				{
+					/*
+					 * Output error message as sublist is empty but sublist count
+					 * is not zero
+					 */
+					_RIOutput (("RI ERROR: ui16SubListCount=%d for empty RI sublist\n",
+					            psRIEntry->ui16SubListCount));
+				}
+			}
+			psRIEntry = NULL;
+		}
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpAllKM
+
+ @Description
+            Dumps out the contents of all RI List entries (i.e. for all
+            MEMDESC allocations for each PMR).
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpAllKM(void)
+{
+	if (g_pPMR2RIListHashTable)
+	{
+		return HASH_Iterate(g_pPMR2RIListHashTable, (HASH_pfnCallback)_DumpAllEntries, NULL);
+	}
+	return PVRSRV_OK;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpProcessKM
+
+ @Description
+            Dumps out the contents of all MEMDESC RI List entries (for every
+            PMR) which have been allocate by the specified process only.
+            At present, output is directed to Kernel log
+            via PVR_DPF.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpProcessKM(IMG_PID pid)
+{
+	PVRSRV_ERROR eError;
+	IMG_UINT32 dummyPMR;
+
+	if (!g_PID2RISublistHashTable)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpList((PMR *)&dummyPMR, pid);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	_TotalAllocsForProcess
+
+ @Description
+            Totals all PMR physical backing for given process.
+
+ @input     pid - ID of process.
+
+ @input     ePhysHeapType - type of Physical Heap for which to total allocs
+
+ @Return	Size of all physical backing for PID's PMRs allocated from the
+            specified heap type (in bytes).
+
+******************************************************************************/
+static IMG_INT32 _TotalAllocsForProcess(const IMG_PID pid, const PHYS_HEAP_TYPE ePhysHeapType)
+{
+	IMG_INT32 i32TotalPhysical = 0;
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		if (pid == PVR_SYS_ALLOC_PID)
+		{
+			DLLIST_NODE *psSysAllocNode = NULL;
+
+			OSLockAcquire(g_hSysAllocPidListLock);
+
+			for (psSysAllocNode = dllist_get_next_node(&g_sSysAllocPidListHead);
+			     psSysAllocNode != NULL && psSysAllocNode != &g_sSysAllocPidListHead;
+			     psSysAllocNode = dllist_get_next_node(psSysAllocNode))
+			{
+				const RI_LIST_ENTRY *const psRIEntry =
+					IMG_CONTAINER_OF((PDLLIST_NODE)psSysAllocNode, RI_LIST_ENTRY, sListNode);
+
+				/* Exclude RACC entries from the stats. This memory should have
+				 * been already freed during connection destruction so they should
+				 * not affect figure showing memory usage. Also RACC entries existence
+				 * is to show unfreed memory in `gpu_mem_area`, not to affect other
+				 * process stats. Finally if the RI entry exists only due to sub-entries
+				 * referencing it but the underlying PMR has been freed don't include
+				 * it in the total stats. */
+				if (PhysHeapGetType(GET_HEAP(psRIEntry)) == ePhysHeapType &&
+				    !IS_RACC(psRIEntry) && !HAS_PMR_INFO(psRIEntry))
+				{
+					IMG_UINT64 ui64PhysicalSize = GET_PHYSICAL_SIZE(psRIEntry);
+
+					if (((IMG_UINT64)i32TotalPhysical + ui64PhysicalSize > IMG_INT32_MAX))
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: i32TotalPhysical exceeding size for i32",
+						         __func__));
+					}
+					i32TotalPhysical += (IMG_INT32)(ui64PhysicalSize & 0x00000000ffffffff);
+				}
+			}
+
+			OSLockRelease(g_hSysAllocPidListLock);
+		}
+		else
+		{
+			RI_LIST_ENTRY *psRIEntry = NULL;
+			RI_SUBLIST_ENTRY *psInitialRISubEntry = NULL, *psRISubEntry = NULL;
+			uintptr_t hashData = 0;
+
+			if (pid != 0)
+			{
+				/* look-up pid in Hash Table */
+				IMG_PID hashKey = pid;
+				hashData = HASH_Retrieve_Extended(g_PID2RISublistHashTable, (void *)&hashKey);
+				if (hashData)
+				{
+					psInitialRISubEntry = IMG_CONTAINER_OF((PDLLIST_NODE)hashData, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psInitialRISubEntry != NULL)
+					{
+						psRISubEntry = psInitialRISubEntry;
+						psRIEntry = psInitialRISubEntry->psRI;
+					}
+				}
+			}
+
+			while (psRISubEntry != NULL && psRIEntry != NULL)
+			{
+				DLLIST_NODE *psNextNode;
+
+				if (!IS_IMPORT(psRISubEntry) &&
+				    !IS_RACC(psRISubEntry) &&
+					!IS_COUNTED_BY_DEBUGFS(psRIEntry) &&
+				    !IS_SYSPMR(psRIEntry) &&
+				    (PhysHeapGetType(GET_HEAP(psRIEntry)) == ePhysHeapType))
+				{
+					IMG_UINT64 ui64PhysicalSize = GET_PHYSICAL_SIZE(psRIEntry);
+
+					if (((IMG_UINT64)i32TotalPhysical + ui64PhysicalSize > IMG_INT32_MAX))
+					{
+						PVR_DPF((PVR_DBG_WARNING, "%s: i32TotalPhysical exceeding size for i32",
+						         __func__));
+					}
+
+					i32TotalPhysical += (IMG_INT32)(ui64PhysicalSize & 0x00000000ffffffff);
+					BIT_SET(psRIEntry->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG);
+				}
+
+				psNextNode = dllist_get_next_node(&(psRISubEntry->sProcListNode));
+				if (psNextNode == NULL || psNextNode == (PDLLIST_NODE)hashData)
+				{
+					psRISubEntry = NULL;
+					psRIEntry = NULL;
+				}
+				else
+				{
+					psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psRISubEntry)
+					{
+						psRIEntry = psRISubEntry->psRI;
+					}
+				}
+			}
+
+			psRISubEntry = psInitialRISubEntry;
+			if (psRISubEntry)
+			{
+				psRIEntry = psRISubEntry->psRI;
+			}
+
+			while (psRISubEntry != NULL && psRIEntry != NULL)
+			{
+				const DLLIST_NODE *const psNextNode =
+				    dllist_get_next_node(&(psRISubEntry->sProcListNode));
+
+				BIT_UNSET(psRIEntry->ui16Flags, RI_PMR_PHYS_COUNTED_BY_DEBUGFS_FLAG);
+
+				if (psNextNode == NULL || psNextNode == (PDLLIST_NODE)hashData)
+				{
+					psRISubEntry = NULL;
+					psRIEntry = NULL;
+				}
+				else
+				{
+					psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY, sProcListNode);
+					if (psRISubEntry)
+					{
+						psRIEntry = psRISubEntry->psRI;
+					}
+				}
+			}
+		}
+	}
+
+	return i32TotalPhysical;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RITotalAllocProcessKM
+
+ @Description
+            Returns the total of allocated GPU memory (backing for PMRs)
+            which has been allocated from the specific heap by the specified
+            process only.
+
+ @Return	Amount of physical backing allocated (in bytes)
+
+******************************************************************************/
+IMG_INT32 RITotalAllocProcessUnlocked(IMG_PID pid, PHYS_HEAP_TYPE ePhysHeapType)
+{
+	IMG_INT32 i32BackingTotal = 0;
+
+	if (g_PID2RISublistHashTable)
+	{
+		i32BackingTotal = _TotalAllocsForProcess(pid, ePhysHeapType);
+	}
+	return i32BackingTotal;
+}
+
+#if defined(DEBUG)
+/*!
+*******************************************************************************
+
+ @Function	_DumpProcessList
+
+ @Description
+            Dumps out RI List entries according to parameters passed.
+
+ @input     psPMR - If not NULL, function will output the RI entries for
+                   the specified PMR only
+ @input     pid - If non-zero, the function will only output MEMDESC RI
+                  entries made by the process with ID pid.
+                  If zero, all MEMDESC RI entries will be output.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+static PVRSRV_ERROR _DumpProcessList(PMR *psPMR,
+									 IMG_PID pid,
+									 IMG_UINT64 ui64Offset,
+									 IMG_DEV_VIRTADDR *psDevVAddr)
+{
+	RI_LIST_ENTRY *psRIEntry = NULL;
+	RI_SUBLIST_ENTRY *psRISubEntry = NULL;
+	IMG_UINT16 ui16SubEntriesParsed = 0;
+	uintptr_t hashData = 0;
+	PMR *pPMRHashKey = psPMR;
+
+	psDevVAddr->uiAddr = 0;
+
+	PVR_RETURN_IF_INVALID_PARAM(psPMR);
+
+	if (g_pPMR2RIListHashTable && g_PID2RISublistHashTable)
+	{
+		PVR_ASSERT(psPMR && pid);
+
+		/* Look-up psPMR in Hash Table */
+		hashData = HASH_Retrieve_Extended (g_pPMR2RIListHashTable, (void *)&pPMRHashKey);
+		psRIEntry = (RI_LIST_ENTRY *)hashData;
+
+		if (!psRIEntry)
+		{
+			/* No entry found in hash table */
+			return PVRSRV_ERROR_NOT_FOUND;
+		}
+
+		if (psRIEntry->ui16SubListCount)
+		{
+			psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)),
+											RI_SUBLIST_ENTRY, sListNode);
+
+			/* Traverse RI sublist and output details for each entry */
+			while (psRISubEntry && (ui16SubEntriesParsed < psRIEntry->ui16SubListCount))
+			{
+				if (pid == psRISubEntry->pid)
+				{
+					IMG_UINT64 ui64StartOffset = psRISubEntry->ui64Offset;
+					IMG_UINT64 ui64EndOffset = psRISubEntry->ui64Offset + psRISubEntry->ui64Size;
+
+					if (ui64Offset >= ui64StartOffset && ui64Offset < ui64EndOffset)
+					{
+						psDevVAddr->uiAddr = psRISubEntry->sVAddr.uiAddr;
+						return PVRSRV_OK;
+					}
+				}
+
+				ui16SubEntriesParsed++;
+				psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRISubEntry->sListNode)),
+												RI_SUBLIST_ENTRY, sListNode);
+			}
+		}
+	}
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+}
+
+/*!
+*******************************************************************************
+
+ @Function	RIDumpProcessListKM
+
+ @Description
+            Dumps out selected contents of all MEMDESC RI List entries (for a
+            PMR) which have been allocate by the specified process only.
+
+ @Return	PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR RIDumpProcessListKM(PMR *psPMR,
+								 IMG_PID pid,
+								 IMG_UINT64 ui64Offset,
+								 IMG_DEV_VIRTADDR *psDevVAddr)
+{
+	PVRSRV_ERROR eError;
+
+	if (!g_PID2RISublistHashTable)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Acquire RI lock*/
+	_RILock();
+
+	eError = _DumpProcessList(psPMR,
+							  pid,
+							  ui64Offset,
+							  psDevVAddr);
+
+	/* Release RI lock*/
+	_RIUnlock();
+
+	return eError;
+}
+#endif
+
+static PVRSRV_ERROR _MarkRACCEntries(uintptr_t k, uintptr_t v, void *psConnection)
+{
+	DLLIST_NODE *psListHead = (DLLIST_NODE *) v;
+	DLLIST_NODE *psNextNode = psListHead;
+
+	PVR_UNREFERENCED_PARAMETER(k);
+
+	do
+	{
+		RI_SUBLIST_ENTRY *psRISubEntry = IMG_CONTAINER_OF(psNextNode, RI_SUBLIST_ENTRY,
+		                                                  sProcListNode);
+
+		if (psRISubEntry->psConnection == psConnection)
+		{
+			RI_LIST_ENTRY *psRIEntry = psRISubEntry->psRI;
+
+			BIT_SET(psRISubEntry->ui16Flags, RI_RACC_FLAG);
+
+			/* RI sub-entry may not have an RI entry associated with it. If that's the
+			 * case just skip processing it. */
+			if (psRIEntry != NULL)
+			{
+				if (!IS_RACC(psRIEntry) &&
+				    !HAS_PMR_INFO(psRIEntry) &&
+				    /* Mark as RACC only if the entry doesn't belong to the system process.
+				     * System process allocations are alive for the whole driver lifetime
+					 * hence they will always exist at a connection closed for every process
+					 * that references them. */
+				    psRIEntry->pid != PVR_SYS_ALLOC_PID)
+				{
+					PVRSRV_ERROR eError;
+
+					eError = _RICreateAndSetPmrInfo(psRIEntry);
+					PVR_LOG_RETURN_IF_ERROR(eError, "_RICreateAndSetPmrInfo");
+
+					BIT_SET(psRIEntry->ui16Flags, RI_RACC_FLAG);
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_MESSAGE, "RIEntry(%s) is already RACC", GET_NAME(psRIEntry)));
+				}
+			}
+		}
+
+		psNextNode = dllist_get_next_node(psNextNode);
+	} while (psNextNode != NULL && psNextNode != psListHead);
+
+	return PVRSRV_OK;
+}
+
+void RIConnectionClosed(void* psConnection)
+{
+	PVRSRV_ERROR eError;
+	if (g_PID2RISublistHashTable)
+	{
+		_RILock();
+		eError = HASH_Iterate(g_PID2RISublistHashTable, (HASH_pfnCallback)_MarkRACCEntries, psConnection);
+		PVR_LOG_IF_FALSE(eError == PVRSRV_OK, "_MarkRACCEntries");
+		_RIUnlock();
+	}
+}
+
+static PVRSRV_ERROR DeleteRACCEntry(RI_SUBLIST_ENTRY *psRISubEntry)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	RI_LIST_ENTRY *psRIEntry = psRISubEntry->psRI;
+
+	PVR_LOG_IF_FALSE_VA(PVR_DBG_WARNING, IS_RACC(psRISubEntry), "Non-RACC entry(%s)",
+	                    psRISubEntry->pszTextB);
+
+	eError = RIDeleteMemdescEntryUnlocked(psRISubEntry);
+	PVR_LOG_IF_ERROR(eError, "RIDeleteMemdescEntryUnlocked");
+
+	/* RI sub-entry may not have an RI entry associated with it. If that's the
+	 * case just skip processing it. */
+	if (psRIEntry != NULL)
+	{
+		if (psRIEntry->ui16SubListCount)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "%p: More than 1 sublist present! (%s)",
+			         psRIEntry, __func__));
+		}
+		else
+		{
+			eError = RIDeletePMREntryUnlocked(psRIEntry);
+			PVR_LOG_IF_ERROR(eError, "RIDeletePMREntryUnlocked");
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR RIDeleteEntriesForPID(IMG_PID pid)
+{
+	DLLIST_NODE *psListHead;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_PID hashKey  = pid;
+
+	_RILock();
+
+	PVR_GOTO_IF_FALSE(g_PID2RISublistHashTable != NULL, ErrHashError);
+
+	psListHead = (DLLIST_NODE *) HASH_Retrieve_Extended(g_PID2RISublistHashTable,
+	                                                    (void *) &hashKey);
+
+	while (psListHead != NULL)
+	{
+		DLLIST_NODE *psNextNode = dllist_get_next_node(psListHead);
+		RI_SUBLIST_ENTRY *psRISubEntry = IMG_CONTAINER_OF(psListHead, RI_SUBLIST_ENTRY,
+		                                                  sProcListNode);
+
+		eError = DeleteRACCEntry(psRISubEntry);
+		PVR_LOG_GOTO_IF_ERROR(eError, "DeleteRACCEntry", ErrUnlockAndReturn);
+
+		psListHead = psNextNode;
+	}
+
+ErrHashError:
+    eError = PVRSRV_ERROR_UNABLE_TO_RETRIEVE_HASH_VALUE;
+
+ErrUnlockAndReturn:
+	_RIUnlock();
+
+	return eError;
+}
+
+static PVRSRV_ERROR _DumpAllEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *)v;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	return RIDumpListKM(GET_ADDR(psRIEntry));
+}
+
+static PVRSRV_ERROR _DeleteAllEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_LIST_ENTRY *psRIEntry = (RI_LIST_ENTRY *)v;
+	RI_SUBLIST_ENTRY *psRISubEntry;
+	PVRSRV_ERROR eResult = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	while ((eResult == PVRSRV_OK) && (psRIEntry->ui16SubListCount > 0))
+	{
+		psRISubEntry = IMG_CONTAINER_OF(dllist_get_next_node(&(psRIEntry->sSubListFirst)), RI_SUBLIST_ENTRY, sListNode);
+		eResult = RIDeleteMEMDESCEntryKM((RI_HANDLE)psRISubEntry);
+	}
+	if (eResult == PVRSRV_OK)
+	{
+		eResult = RIDeletePMREntryKM((RI_HANDLE)psRIEntry);
+		/*
+		 * If we've deleted the Hash table, return
+		 * an error to stop the iterator...
+		 */
+		if (!g_pPMR2RIListHashTable)
+		{
+			eResult = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+		}
+	}
+	return eResult;
+}
+
+static PVRSRV_ERROR _DeleteAllProcEntries (uintptr_t k, uintptr_t v, void* pvPriv)
+{
+	RI_SUBLIST_ENTRY *psRISubEntry = (RI_SUBLIST_ENTRY *)v;
+	PVRSRV_ERROR eResult;
+
+	PVR_UNREFERENCED_PARAMETER (k);
+	PVR_UNREFERENCED_PARAMETER (pvPriv);
+
+	eResult = RIDeleteMEMDESCEntryKM((RI_HANDLE) psRISubEntry);
+	if (eResult == PVRSRV_OK && !g_PID2RISublistHashTable)
+	{
+		/*
+		 * If we've deleted the Hash table, return
+		 * an error to stop the iterator...
+		 */
+		eResult = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
+	}
+
+	return eResult;
+}
+
+#endif /* if defined(PVRSRV_ENABLE_GPU_MEMORY_INFO) */
diff --git a/drivers/gpu/drm/img-rogue/ri_server.h b/drivers/gpu/drm/img-rogue/ri_server.h
new file mode 100644
index 000000000000..a8bcd9f32039
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/ri_server.h
@@ -0,0 +1,115 @@
+/*************************************************************************/ /*!
+@File           ri_server.h
+@Title          Resource Information abstraction
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Resource Information (RI) functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RI_SERVER_H
+#define RI_SERVER_H
+
+#include "img_defs.h"
+#include "ri_typedefs.h"
+#include "pmr.h"
+#include "pvrsrv_error.h"
+#include "physheap.h"
+#include "connection_server.h"
+#include "device.h"
+
+PVRSRV_ERROR RIInitKM(void);
+void RIDeInitKM(void);
+
+void RILockAcquireKM(void);
+void RILockReleaseKM(void);
+
+PVRSRV_ERROR RIWritePMREntryKM(PMR *psPMR);
+
+PVRSRV_ERROR RIWritePMREntryWithOwnerKM(PMR *psPMR,
+                                        IMG_PID ui32Owner);
+
+PVRSRV_ERROR RIWriteMEMDESCEntryKM(void* psConnection,
+                                   PVRSRV_DEVICE_NODE *psDeviceNode,
+                                   PMR *psPMR,
+                                   IMG_UINT32 ui32TextBSize,
+                                   const IMG_CHAR *psz8TextB,
+                                   IMG_UINT64 uiOffset,
+                                   IMG_UINT64 uiSize,
+                                   PVRSRV_MEMALLOCFLAGS_T uiFlags,
+                                   RI_HANDLE *phRIHandle);
+
+PVRSRV_ERROR RIWriteProcListEntryKM(void* psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32TextBSize,
+                                    const IMG_CHAR *psz8TextB,
+                                    IMG_UINT64 ui64Size,
+                                    IMG_UINT64 ui64DevVAddr,
+                                    RI_HANDLE *phRIHandle);
+
+PVRSRV_ERROR RIUpdateMEMDESCAddrKM(RI_HANDLE hRIHandle,
+                                   IMG_DEV_VIRTADDR sVAddr);
+
+PVRSRV_ERROR RIDeletePMREntryKM(RI_HANDLE hRIHandle);
+PVRSRV_ERROR RIDeleteMEMDESCEntryKM(RI_HANDLE hRIHandle);
+
+PVRSRV_ERROR RIDeleteListKM(void);
+
+PVRSRV_ERROR RIDumpListKM(PMR *psPMR);
+
+PVRSRV_ERROR RIDumpAllKM(void);
+
+PVRSRV_ERROR RIDumpProcessKM(IMG_PID pid);
+
+#if defined(DEBUG)
+PVRSRV_ERROR RIDumpProcessListKM(PMR *psPMR,
+                                 IMG_PID pid,
+                                 IMG_UINT64 ui64Offset,
+                                 IMG_DEV_VIRTADDR *psDevVAddr);
+#endif
+
+void RIConnectionClosed(void* psConnection);
+
+PVRSRV_ERROR RIDeleteEntriesForPID(IMG_PID pid);
+
+IMG_BOOL RIGetListEntryKM(IMG_PID pid,
+                          IMG_HANDLE **ppHandle,
+                          IMG_CHAR **ppszEntryString);
+
+IMG_INT32 RITotalAllocProcessUnlocked(IMG_PID pid, PHYS_HEAP_TYPE ePhysHeapType);
+
+#endif /* RI_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/ri_typedefs.h b/drivers/gpu/drm/img-rogue/ri_typedefs.h
new file mode 100644
index 000000000000..77be10e2ab03
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/ri_typedefs.h
@@ -0,0 +1,52 @@
+/*************************************************************************/ /*!
+@File
+@Title          Resource Information (RI) Management
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Client side part of RI management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef RI_TYPEDEFS_H
+#define RI_TYPEDEFS_H
+
+#include "img_types.h"
+
+typedef struct RI_SUBLIST_ENTRY RI_ENTRY;
+typedef RI_ENTRY* RI_HANDLE;
+
+#endif /* #ifndef RI_TYPEDEFS_H */
diff --git a/drivers/gpu/drm/img-rogue/rogue_trace_events.h b/drivers/gpu/drm/img-rogue/rogue_trace_events.h
new file mode 100644
index 000000000000..33bf2f6ecf72
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/rogue_trace_events.h
@@ -0,0 +1,592 @@
+/*************************************************************************/ /*!
+@File
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rogue
+
+#if !defined(ROGUE_TRACE_EVENTS_H) || defined(TRACE_HEADER_MULTI_READ)
+#define ROGUE_TRACE_EVENTS_H
+
+#include <linux/version.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+#include <linux/time.h>
+
+#define show_secs_from_ns(ns) \
+	({ \
+		u64 t = ns + (NSEC_PER_USEC / 2); \
+		do_div(t, NSEC_PER_SEC); \
+		t; \
+	})
+
+#define show_usecs_from_ns(ns) \
+	({ \
+		u64 t = ns + (NSEC_PER_USEC / 2); \
+		do_div(t, NSEC_PER_USEC); \
+		do_div(t, USEC_PER_SEC); \
+	})
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_update_enabled_callback(void);
+#else
+void trace_fence_update_enabled_callback(void);
+#endif
+void trace_fence_update_disabled_callback(void);
+
+TRACE_EVENT_FN(rogue_fence_update,
+
+	TP_PROTO(const char *comm, const char *cmd, const char *dm, u32 gpu_id, u32 ctx_id, u32 offset,
+		u32 sync_fwaddr, u32 sync_value),
+
+	TP_ARGS(comm, cmd, dm, gpu_id, ctx_id, offset, sync_fwaddr, sync_value),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       cmd,            cmd             )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+		__field(        u32,            offset          )
+		__field(        u32,            sync_fwaddr     )
+		__field(        u32,            sync_value      )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm);
+		__assign_str(cmd);
+		__assign_str(dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->offset = offset;
+		__entry->sync_fwaddr = sync_fwaddr;
+		__entry->sync_value = sync_value;
+	),
+
+	TP_printk("comm=%s cmd=%s dm=%s gpu=%lu ctx_id=%lu offset=%lu sync_fwaddr=%#lx sync_value=%#lx",
+		__get_str(comm),
+		__get_str(cmd),
+		__get_str(dm),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->offset,
+		(unsigned long) __entry->sync_fwaddr,
+		(unsigned long) __entry->sync_value
+	),
+
+	trace_fence_update_enabled_callback,
+	trace_fence_update_disabled_callback
+);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_check_enabled_callback(void);
+#else
+void trace_fence_check_enabled_callback(void);
+#endif
+void trace_fence_check_disabled_callback(void);
+
+TRACE_EVENT_FN(rogue_fence_check,
+
+	TP_PROTO(const char *comm, const char *cmd, const char *dm, u32 gpu_id, u32 ctx_id, u32 offset,
+		u32 sync_fwaddr, u32 sync_value),
+
+	TP_ARGS(comm, cmd, dm, gpu_id, ctx_id, offset, sync_fwaddr, sync_value),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       cmd,            cmd             )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+		__field(        u32,            offset          )
+		__field(        u32,            sync_fwaddr     )
+		__field(        u32,            sync_value      )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm);
+		__assign_str(cmd);
+		__assign_str(dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->offset = offset;
+		__entry->sync_fwaddr = sync_fwaddr;
+		__entry->sync_value = sync_value;
+	),
+
+	TP_printk("comm=%s cmd=%s dm=%s gpu=%lu ctx_id=%lu offset=%lu sync_fwaddr=%#lx sync_value=%#lx",
+		__get_str(comm),
+		__get_str(cmd),
+		__get_str(dm),
+		(unsigned long)__entry->gpu_id,
+		(unsigned long)__entry->ctx_id,
+		(unsigned long)__entry->offset,
+		(unsigned long)__entry->sync_fwaddr,
+		(unsigned long)__entry->sync_value
+	),
+
+	trace_fence_check_enabled_callback,
+	trace_fence_check_disabled_callback
+);
+
+TRACE_EVENT(rogue_job_enqueue,
+
+	TP_PROTO(u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         const char *kick_type),
+
+	TP_ARGS(gpu_id, ctx_id, int_id, ext_id, kick_type),
+
+	TP_STRUCT__entry(
+		__field(u32, gpu_id)
+		__field(u32, ctx_id)
+		__field(u32, int_id)
+		__field(u32, ext_id)
+		__string(kick_type, kick_type)
+	),
+
+	TP_fast_assign(
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__assign_str(kick_type);
+	),
+
+	TP_printk("gpu=%lu, ctx_id=%lu int_id=%lu ext_id=%lu kick_type=%s",
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		__get_str(kick_type)
+	)
+);
+
+TRACE_EVENT(rogue_sched_switch,
+
+	TP_PROTO(const char *work_type, u32 switch_type, u64 timestamp, u32 gpu_id, u32 next_ctx_id,
+	         u32 next_prio, u32 next_int_id, u32 next_ext_id),
+
+	TP_ARGS(work_type, switch_type, timestamp, gpu_id, next_ctx_id, next_prio, next_int_id,
+	        next_ext_id),
+
+	TP_STRUCT__entry(
+		__string(work_type, work_type)
+		__field(u32, switch_type)
+		__field(u64, timestamp)
+		__field(u32, gpu_id)
+		__field(u32, next_ctx_id)
+		__field(u32, next_prio)
+		__field(u32, next_int_id)
+		__field(u32, next_ext_id)
+	),
+
+	TP_fast_assign(
+		__assign_str(work_type);
+		__entry->switch_type = switch_type;
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->next_ctx_id = next_ctx_id;
+		__entry->next_prio = next_prio;
+		__entry->next_int_id = next_int_id;
+		__entry->next_ext_id = next_ext_id;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu next_ctx_id=%lu next_int_id=%lu next_ext_id=%lu"
+		" next_prio=%lu work_type=%s switch_type=%s",
+		(unsigned long long) show_secs_from_ns(__entry->timestamp),
+		(unsigned long) show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->next_ctx_id,
+		(unsigned long) __entry->next_int_id,
+		(unsigned long) __entry->next_ext_id,
+		(unsigned long) __entry->next_prio,
+		__get_str(work_type),
+		__print_symbolic(__entry->switch_type,
+			/* These values are from ospvr_gputrace.h. */
+			{ 1, "begin" },
+			{ 2, "end" })
+	)
+);
+
+TRACE_EVENT(rogue_create_fw_context,
+
+	TP_PROTO(const char *comm, const char *dm, u32 gpu_id, u32 ctx_id),
+
+	TP_ARGS(comm, dm, gpu_id, ctx_id),
+
+	TP_STRUCT__entry(
+		__string(       comm,           comm            )
+		__string(       dm,             dm              )
+		__field(        u32,            gpu_id          )
+		__field(        u32,            ctx_id          )
+	),
+
+	TP_fast_assign(
+		__assign_str(comm);
+		__assign_str(dm);
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+	),
+
+	TP_printk("comm=%s dm=%s gpu=%lu ctx_id=%lu",
+		__get_str(comm),
+		__get_str(dm),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id
+	)
+);
+
+void PVRGpuTraceEnableUfoCallback(void);
+void PVRGpuTraceDisableUfoCallback(void);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableUfoCallbackWrapper(void);
+#else
+#define PVRGpuTraceEnableUfoCallbackWrapper \
+		PVRGpuTraceEnableUfoCallback
+#endif
+
+TRACE_EVENT_FN(rogue_ufo_update,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 old_value, u32 new_value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, old_value,
+	        new_value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            old_value   )
+		__field(        u32,            new_value   )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->old_value = old_value;
+		__entry->new_value = new_value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx old_value=%#lx new_value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->old_value,
+		(unsigned long) __entry->new_value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_check_fail,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value, u32 required),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value, required),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+		__field(        u32,            required    )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+		__entry->required = required;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx required=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value,
+		(unsigned long) __entry->required
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_pr_check_fail,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value, u32 required),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value, required),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+		__field(        u32,            required    )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+		__entry->required = required;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx required=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value,
+		(unsigned long) __entry->required
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_check_success,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT_FN(rogue_ufo_pr_check_success,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, u32 ctx_id, u32 int_id, u32 ext_id,
+	         u32 fwaddr, u32 value),
+
+	TP_ARGS(timestamp, gpu_id, ctx_id, int_id, ext_id, fwaddr, value),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp   )
+		__field(        u32,            gpu_id      )
+		__field(        u32,            ctx_id      )
+		__field(        u32,            int_id      )
+		__field(        u32,            ext_id      )
+		__field(        u32,            fwaddr      )
+		__field(        u32,            value       )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__entry->ctx_id = ctx_id;
+		__entry->int_id = int_id;
+		__entry->ext_id = ext_id;
+		__entry->fwaddr = fwaddr;
+		__entry->value = value;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu ctx_id=%lu int_id=%lu ext_id=%lu"
+		" fwaddr=%#lx value=%#lx",
+		(unsigned long long)show_secs_from_ns(__entry->timestamp),
+		(unsigned long)show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		(unsigned long) __entry->ctx_id,
+		(unsigned long) __entry->int_id,
+		(unsigned long) __entry->ext_id,
+		(unsigned long) __entry->fwaddr,
+		(unsigned long) __entry->value
+	),
+
+	PVRGpuTraceEnableUfoCallbackWrapper,
+	PVRGpuTraceDisableUfoCallback
+);
+
+TRACE_EVENT(rogue_events_lost,
+
+	TP_PROTO(u32 event_source, u32 gpu_id, u32 last_ordinal, u32 curr_ordinal),
+
+	TP_ARGS(event_source, gpu_id, last_ordinal, curr_ordinal),
+
+	TP_STRUCT__entry(
+		__field(        u32,            event_source     )
+		__field(        u32,            gpu_id           )
+		__field(        u32,            last_ordinal     )
+		__field(        u32,            curr_ordinal     )
+	),
+
+	TP_fast_assign(
+		__entry->event_source = event_source;
+		__entry->gpu_id = gpu_id;
+		__entry->last_ordinal = last_ordinal;
+		__entry->curr_ordinal = curr_ordinal;
+	),
+
+	TP_printk("event_source=%s gpu=%u last_ordinal=%u curr_ordinal=%u",
+		__print_symbolic(__entry->event_source, {0, "GPU"}, {1, "Host"}),
+		__entry->gpu_id,
+		__entry->last_ordinal,
+		__entry->curr_ordinal)
+);
+
+void PVRGpuTraceEnableFirmwareActivityCallback(void);
+void PVRGpuTraceDisableFirmwareActivityCallback(void);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int PVRGpuTraceEnableFirmwareActivityCallbackWrapper(void);
+#else
+#define PVRGpuTraceEnableFirmwareActivityCallbackWrapper \
+		PVRGpuTraceEnableFirmwareActivityCallback
+#endif
+
+TRACE_EVENT_FN(rogue_firmware_activity,
+
+	TP_PROTO(u64 timestamp, u32 gpu_id, const char *task, u32 fw_event),
+
+	TP_ARGS(timestamp, gpu_id, task, fw_event),
+
+	TP_STRUCT__entry(
+		__field(        u64,            timestamp       )
+		__field(        u32,            gpu_id             )
+		__string(       task,           task            )
+		__field(        u32,            fw_event        )
+	),
+
+	TP_fast_assign(
+		__entry->timestamp = timestamp;
+		__entry->gpu_id = gpu_id;
+		__assign_str(task);
+		__entry->fw_event = fw_event;
+	),
+
+	TP_printk("ts=%llu.%06lu gpu=%lu task=%s event=%s",
+		(unsigned long long) show_secs_from_ns(__entry->timestamp),
+		(unsigned long) show_usecs_from_ns(__entry->timestamp),
+		(unsigned long) __entry->gpu_id,
+		__get_str(task),
+		__print_symbolic(__entry->fw_event,
+			/* These values are from ospvr_gputrace.h. */
+			{ 1, "begin" },
+			{ 2, "end" })),
+
+	PVRGpuTraceEnableFirmwareActivityCallbackWrapper,
+	PVRGpuTraceDisableFirmwareActivityCallback
+);
+
+#undef show_secs_from_ns
+#undef show_usecs_from_ns
+
+#endif /* ROGUE_TRACE_EVENTS_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+
+/* This is needed because the name of this file doesn't match TRACE_SYSTEM. */
+#define TRACE_INCLUDE_FILE rogue_trace_events
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/img-rogue/server_cache_bridge.c b/drivers/gpu/drm/img-rogue/server_cache_bridge.c
new file mode 100644
index 000000000000..b2f5267cc597
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_cache_bridge.c
@@ -0,0 +1,452 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cache
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cache
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "cache_km.h"
+
+#include "common_cache_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(CACHE_BATCH_MAX <= IMG_UINT32_MAX,
+	      "CACHE_BATCH_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeCacheOpQueue(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psCacheOpQueueIN_UI8,
+			 IMG_UINT8 * psCacheOpQueueOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPQUEUE *psCacheOpQueueIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *psCacheOpQueueOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPQUEUE *) IMG_OFFSET_ADDR(psCacheOpQueueOUT_UI8, 0);
+
+	PMR **psPMRInt = NULL;
+	IMG_HANDLE *hPMRInt2 = NULL;
+	IMG_UINT64 *ui64AddressInt = NULL;
+	IMG_DEVMEM_OFFSET_T *uiOffsetInt = NULL;
+	IMG_DEVMEM_SIZE_T *uiSizeInt = NULL;
+	PVRSRV_CACHE_OP *iuCacheOpInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) +
+	    ((IMG_UINT64) psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) + 0;
+
+	if (unlikely(psCacheOpQueueIN->ui32NumCacheOps > CACHE_BATCH_MAX))
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto CacheOpQueue_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psCacheOpQueueOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto CacheOpQueue_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psCacheOpQueueIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psCacheOpQueueIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psCacheOpQueueOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		psPMRInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psPMRInt, 0, psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *));
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PMR *);
+		hPMRInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hPMRInt2, (const void __user *)psCacheOpQueueIN->phPMR,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		ui64AddressInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui64AddressInt, (const void __user *)psCacheOpQueueIN->pui64Address,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_UINT64)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiOffsetInt =
+		    (IMG_DEVMEM_OFFSET_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiOffsetInt, (const void __user *)psCacheOpQueueIN->puiOffset,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_OFFSET_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		uiSizeInt = (IMG_DEVMEM_SIZE_T *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiSizeInt, (const void __user *)psCacheOpQueueIN->puiSize,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(IMG_DEVMEM_SIZE_T)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+	if (psCacheOpQueueIN->ui32NumCacheOps != 0)
+	{
+		iuCacheOpInt =
+		    (PVRSRV_CACHE_OP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP);
+	}
+
+	/* Copy the data over */
+	if (psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, iuCacheOpInt, (const void __user *)psCacheOpQueueIN->piuCacheOp,
+		     psCacheOpQueueIN->ui32NumCacheOps * sizeof(PVRSRV_CACHE_OP)) != PVRSRV_OK)
+		{
+			psCacheOpQueueOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto CacheOpQueue_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+			/* Look up the address from the handle */
+			psCacheOpQueueOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psPMRInt[i],
+						       hPMRInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psCacheOpQueueOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto CacheOpQueue_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpQueueOUT->eError =
+	    CacheOpQueue(psConnection, OSGetDevNode(psConnection),
+			 psCacheOpQueueIN->ui32NumCacheOps,
+			 psPMRInt,
+			 ui64AddressInt,
+			 uiOffsetInt, uiSizeInt, iuCacheOpInt, psCacheOpQueueIN->ui32OpTimeline);
+
+CacheOpQueue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hPMRInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psCacheOpQueueIN->ui32NumCacheOps; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psPMRInt && psPMRInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hPMRInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psCacheOpQueueOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpExec(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psCacheOpExecIN_UI8,
+			IMG_UINT8 * psCacheOpExecOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPEXEC *psCacheOpExecIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPEXEC *psCacheOpExecOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPEXEC *) IMG_OFFSET_ADDR(psCacheOpExecOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpExecIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpExecOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpExecOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpExec_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpExecOUT->eError =
+	    CacheOpValExec(psPMRInt,
+			   psCacheOpExecIN->ui64Address,
+			   psCacheOpExecIN->uiOffset,
+			   psCacheOpExecIN->uiSize, psCacheOpExecIN->iuCacheOp);
+
+CacheOpExec_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeCacheOpLog(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psCacheOpLogIN_UI8,
+		       IMG_UINT8 * psCacheOpLogOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CACHEOPLOG *psCacheOpLogIN =
+	    (PVRSRV_BRIDGE_IN_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CACHEOPLOG *psCacheOpLogOUT =
+	    (PVRSRV_BRIDGE_OUT_CACHEOPLOG *) IMG_OFFSET_ADDR(psCacheOpLogOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psCacheOpLogIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psCacheOpLogOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psCacheOpLogOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto CacheOpLog_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psCacheOpLogOUT->eError =
+	    CacheOpLog(psPMRInt,
+		       psCacheOpLogIN->ui64Address,
+		       psCacheOpLogIN->uiOffset,
+		       psCacheOpLogIN->uiSize,
+		       psCacheOpLogIN->i64StartTime,
+		       psCacheOpLogIN->i64EndTime, psCacheOpLogIN->iuCacheOp);
+
+CacheOpLog_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitCACHEBridge(void);
+void DeinitCACHEBridge(void);
+
+/*
+ * Register all CACHE functions with services
+ */
+PVRSRV_ERROR InitCACHEBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE,
+			      PVRSRVBridgeCacheOpQueue, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPQUEUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPQUEUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC,
+			      PVRSRVBridgeCacheOpExec, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPEXEC),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPEXEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG,
+			      PVRSRVBridgeCacheOpLog, NULL, sizeof(PVRSRV_BRIDGE_IN_CACHEOPLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_CACHEOPLOG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cache functions with services
+ */
+void DeinitCACHEBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPQUEUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPEXEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CACHE, PVRSRV_BRIDGE_CACHE_CACHEOPLOG);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_cmm_bridge.c b/drivers/gpu/drm/img-rogue/server_cmm_bridge.c
new file mode 100644
index 000000000000..cc658c793d3b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_cmm_bridge.c
@@ -0,0 +1,414 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for cmm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for cmm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_server.h"
+
+#include "common_cmm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DevmemIntExportCtxpsContextExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnexportCtx((DEVMEMINT_CTX_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntExportCtx(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntExportCtxIN_UI8,
+			       IMG_UINT8 * psDevmemIntExportCtxOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *psDevmemIntExportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntExportCtxOUT_UI8,
+								     0);
+
+	IMG_HANDLE hContext = psDevmemIntExportCtxIN->hContext;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntExportCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX_EXPORT *psContextExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntExportCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError =
+	    DevmemIntExportCtx(psContextInt, psPMRInt, &psContextExportInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntExportCtxOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntExportCtxOUT->
+								    hContextExport,
+								    (void *)psContextExportInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT,
+								    PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntExportCtxpsContextExportIntRelease);
+	if (unlikely(psDevmemIntExportCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntExportCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntExportCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntExportCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psContextExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			DevmemIntUnexportCtx(psContextExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnexportCtx(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntUnexportCtxIN_UI8,
+				 IMG_UINT8 * psDevmemIntUnexportCtxOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX *) IMG_OFFSET_ADDR(psDevmemIntUnexportCtxIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *psDevmemIntUnexportCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnexportCtxOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnexportCtxOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnexportCtxIN->hContextExport,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX_EXPORT);
+	if (unlikely((psDevmemIntUnexportCtxOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntUnexportCtxOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnexportCtxOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnexportCtx_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnexportCtx_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntAcquireRemoteCtxpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntAcquireRemoteCtx(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxIN_UI8,
+				      IMG_UINT8 * psDevmemIntAcquireRemoteCtxOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *psDevmemIntAcquireRemoteCtxOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX *)
+	    IMG_OFFSET_ADDR(psDevmemIntAcquireRemoteCtxOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevmemIntAcquireRemoteCtxIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_CTX *psContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntAcquireRemoteCtxOUT->hContext = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    DevmemIntAcquireRemoteCtx(psPMRInt, &psContextInt, &hPrivDataInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntAcquireRemoteCtxOUT->hContext,
+				      (void *)psContextInt, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+				      PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntAcquireRemoteCtxpsContextIntRelease);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	psDevmemIntAcquireRemoteCtxOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psDevmemIntAcquireRemoteCtxOUT->hPrivData,
+					 (void *)hPrivDataInt, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psDevmemIntAcquireRemoteCtxOUT->hContext);
+	if (unlikely(psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntAcquireRemoteCtx_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntAcquireRemoteCtx_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntAcquireRemoteCtxOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntAcquireRemoteCtxOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psDevmemIntAcquireRemoteCtxOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DevmemIntCtxDestroy(psContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_CMM_BRIDGE */
+
+#if !defined(EXCLUDE_CMM_BRIDGE)
+PVRSRV_ERROR InitCMMBridge(void);
+void DeinitCMMBridge(void);
+
+/*
+ * Register all CMM functions with services
+ */
+PVRSRV_ERROR InitCMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX,
+			      PVRSRVBridgeDevmemIntExportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX,
+			      PVRSRVBridgeDevmemIntUnexportCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNEXPORTCTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNEXPORTCTX));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX,
+			      PVRSRVBridgeDevmemIntAcquireRemoteCtx, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTACQUIREREMOTECTX),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTACQUIREREMOTECTX));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all cmm functions with services
+ */
+void DeinitCMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTUNEXPORTCTX);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_CMM, PVRSRV_BRIDGE_CMM_DEVMEMINTACQUIREREMOTECTX);
+
+}
+#else /* EXCLUDE_CMM_BRIDGE */
+/* This bridge is conditional on EXCLUDE_CMM_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitCMMBridge() \
+	PVRSRV_OK
+
+#define DeinitCMMBridge()
+
+#endif /* EXCLUDE_CMM_BRIDGE */
diff --git a/drivers/gpu/drm/img-rogue/server_devicememhistory_bridge.c b/drivers/gpu/drm/img-rogue/server_devicememhistory_bridge.c
new file mode 100644
index 000000000000..72c6a86a0ec8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_devicememhistory_bridge.c
@@ -0,0 +1,835 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for devicememhistory
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for devicememhistory
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pmr.h"
+#include "devicemem_history_server.h"
+
+#include "common_devicememhistory_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMap(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevicememHistoryMapIN_UI8,
+				IMG_UINT8 * psDevicememHistoryMapOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *psDevicememHistoryMapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *psDevicememHistoryMapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP *) IMG_OFFSET_ADDR(psDevicememHistoryMapOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryMapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryMapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryMapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryMap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryMapOUT->eError =
+	    DevicememHistoryMapKM(psPMRInt,
+				  psDevicememHistoryMapIN->uiOffset,
+				  psDevicememHistoryMapIN->sDevVAddr,
+				  psDevicememHistoryMapIN->uiSize,
+				  uiTextInt,
+				  psDevicememHistoryMapIN->ui32Log2PageSize,
+				  psDevicememHistoryMapIN->ui32AllocationIndex,
+				  &psDevicememHistoryMapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmap(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevicememHistoryUnmapIN_UI8,
+				  IMG_UINT8 * psDevicememHistoryUnmapOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *psDevicememHistoryUnmapOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistoryUnmapIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmap_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryUnmapIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmap_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmap_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistoryUnmapOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistoryUnmapOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistoryUnmap_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistoryUnmapOUT->eError =
+	    DevicememHistoryUnmapKM(psPMRInt,
+				    psDevicememHistoryUnmapIN->uiOffset,
+				    psDevicememHistoryUnmapIN->sDevVAddr,
+				    psDevicememHistoryUnmapIN->uiSize,
+				    uiTextInt,
+				    psDevicememHistoryUnmapIN->ui32Log2PageSize,
+				    psDevicememHistoryUnmapIN->ui32AllocationIndex,
+				    &psDevicememHistoryUnmapOUT->ui32AllocationIndexOut);
+
+DevicememHistoryUnmap_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryMapVRange(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeIN_UI8,
+				      IMG_UINT8 * psDevicememHistoryMapVRangeOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *psDevicememHistoryMapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryMapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryMapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryMapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDevicememHistoryMapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryMapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryMapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryMapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryMapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryMapVRangeOUT->eError =
+	    DevicememHistoryMapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					psDevicememHistoryMapVRangeIN->sBaseDevVAddr,
+					psDevicememHistoryMapVRangeIN->ui32ui32StartPage,
+					psDevicememHistoryMapVRangeIN->ui32NumPages,
+					psDevicememHistoryMapVRangeIN->uiAllocSize,
+					uiTextInt,
+					psDevicememHistoryMapVRangeIN->ui32Log2PageSize,
+					psDevicememHistoryMapVRangeIN->ui32AllocationIndex,
+					&psDevicememHistoryMapVRangeOUT->ui32AllocationIndexOut);
+
+DevicememHistoryMapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryMapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistoryUnmapVRange(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeIN_UI8,
+					IMG_UINT8 * psDevicememHistoryUnmapVRangeOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *psDevicememHistoryUnmapVRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistoryUnmapVRangeOUT_UI8, 0);
+
+	IMG_CHAR *uiTextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistoryUnmapVRange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistoryUnmapVRangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistoryUnmapVRangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistoryUnmapVRangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistoryUnmapVRange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt, (const void __user *)psDevicememHistoryUnmapVRangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistoryUnmapVRangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistoryUnmapVRange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psDevicememHistoryUnmapVRangeOUT->eError =
+	    DevicememHistoryUnmapVRangeKM(psConnection, OSGetDevNode(psConnection),
+					  psDevicememHistoryUnmapVRangeIN->sBaseDevVAddr,
+					  psDevicememHistoryUnmapVRangeIN->ui32ui32StartPage,
+					  psDevicememHistoryUnmapVRangeIN->ui32NumPages,
+					  psDevicememHistoryUnmapVRangeIN->uiAllocSize,
+					  uiTextInt,
+					  psDevicememHistoryUnmapVRangeIN->ui32Log2PageSize,
+					  psDevicememHistoryUnmapVRangeIN->ui32AllocationIndex,
+					  &psDevicememHistoryUnmapVRangeOUT->
+					  ui32AllocationIndexOut);
+
+DevicememHistoryUnmapVRange_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistoryUnmapVRangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDevicememHistorySparseChange(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psDevicememHistorySparseChangeIN_UI8,
+					 IMG_UINT8 * psDevicememHistorySparseChangeOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *psDevicememHistorySparseChangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE *)
+	    IMG_OFFSET_ADDR(psDevicememHistorySparseChangeOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psDevicememHistorySparseChangeIN->hPMR;
+	PMR *psPMRInt = NULL;
+	IMG_CHAR *uiTextInt = NULL;
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32AllocPageCount *
+	     sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psDevicememHistorySparseChangeIN->ui32FreePageCount *
+	     sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32AllocPageCount >
+	     PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (unlikely
+	    (psDevicememHistorySparseChangeIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DevicememHistorySparseChange_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDevicememHistorySparseChangeIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psDevicememHistorySparseChangeIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDevicememHistorySparseChangeOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DevicememHistorySparseChange_exit;
+			}
+		}
+	}
+
+	{
+		uiTextInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->puiText,
+		     DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+		((IMG_CHAR *) uiTextInt)[(DEVMEM_ANNOTATION_MAX_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32AllocPageIndices,
+		     psDevicememHistorySparseChangeIN->ui32AllocPageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psDevicememHistorySparseChangeIN->pui32FreePageIndices,
+		     psDevicememHistorySparseChangeIN->ui32FreePageCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psDevicememHistorySparseChangeOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DevicememHistorySparseChange_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevicememHistorySparseChangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevicememHistorySparseChangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevicememHistorySparseChange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevicememHistorySparseChangeOUT->eError =
+	    DevicememHistorySparseChangeKM(psPMRInt,
+					   psDevicememHistorySparseChangeIN->uiOffset,
+					   psDevicememHistorySparseChangeIN->sDevVAddr,
+					   psDevicememHistorySparseChangeIN->uiSize,
+					   uiTextInt,
+					   psDevicememHistorySparseChangeIN->ui32Log2PageSize,
+					   psDevicememHistorySparseChangeIN->ui32AllocPageCount,
+					   ui32AllocPageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32FreePageCount,
+					   ui32FreePageIndicesInt,
+					   psDevicememHistorySparseChangeIN->ui32AllocationIndex,
+					   &psDevicememHistorySparseChangeOUT->
+					   ui32AllocationIndexOut);
+
+DevicememHistorySparseChange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDevicememHistorySparseChangeOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pDEVICEMEMHISTORYBridgeLock;
+
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void);
+void DeinitDEVICEMEMHISTORYBridge(void);
+
+/*
+ * Register all DEVICEMEMHISTORY functions with services
+ */
+PVRSRV_ERROR InitDEVICEMEMHISTORYBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pDEVICEMEMHISTORYBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP,
+			      PVRSRVBridgeDevicememHistoryMap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP,
+			      PVRSRVBridgeDevicememHistoryUnmap, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryMapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE,
+			      PVRSRVBridgeDevicememHistoryUnmapVRange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYUNMAPVRANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYUNMAPVRANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+			      PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE,
+			      PVRSRVBridgeDevicememHistorySparseChange, pDEVICEMEMHISTORYBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVICEMEMHISTORYSPARSECHANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVICEMEMHISTORYSPARSECHANGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all devicememhistory functions with services
+ */
+void DeinitDEVICEMEMHISTORYBridge(void)
+{
+	OSLockDestroy(pDEVICEMEMHISTORYBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYUNMAPVRANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DEVICEMEMHISTORY,
+				PVRSRV_BRIDGE_DEVICEMEMHISTORY_DEVICEMEMHISTORYSPARSECHANGE);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_di_bridge.c b/drivers/gpu/drm/img-rogue/server_di_bridge.c
new file mode 100644
index 000000000000..154bccdaae5e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_di_bridge.c
@@ -0,0 +1,652 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for di
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for di
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "di_impl_brg.h"
+
+#include "common_di_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _DICreateContextpsContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DIDestroyContextKM((DI_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDICreateContext(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDICreateContextIN_UI8,
+			    IMG_UINT8 * psDICreateContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DICREATECONTEXT *psDICreateContextIN =
+	    (PVRSRV_BRIDGE_IN_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DICREATECONTEXT *psDICreateContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DICREATECONTEXT *) IMG_OFFSET_ADDR(psDICreateContextOUT_UI8, 0);
+
+	IMG_CHAR *puiStreamNameInt = NULL;
+	DI_CONTEXT *psContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	PVR_UNREFERENCED_PARAMETER(psDICreateContextIN);
+
+	psDICreateContextOUT->puiStreamName = psDICreateContextIN->puiStreamName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDICreateContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DICreateContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDICreateContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDICreateContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDICreateContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DICreateContext_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		puiStreamNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	psDICreateContextOUT->eError = DICreateContextKM(puiStreamNameInt, &psContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		goto DICreateContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDICreateContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psDICreateContextOUT->hContext,
+								 (void *)psContextInt,
+								 PVRSRV_HANDLE_TYPE_DI_CONTEXT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+								 (PFN_HANDLE_RELEASE) &
+								 _DICreateContextpsContextIntRelease);
+	if (unlikely(psDICreateContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DICreateContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamNameInt) && ((PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psDICreateContextOUT->puiStreamName, puiStreamNameInt,
+		      (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psDICreateContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DICreateContext_exit;
+		}
+	}
+
+DICreateContext_exit:
+
+	if (psDICreateContextOUT->eError != PVRSRV_OK)
+	{
+		if (psDICreateContextOUT->hContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDICreateContextOUT->
+							     hContext,
+							     PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psContextInt)
+		{
+			DIDestroyContextKM(psContextInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDICreateContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIDestroyContext(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIDestroyContextIN_UI8,
+			     IMG_UINT8 * psDIDestroyContextOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *psDIDestroyContextIN =
+	    (PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *psDIDestroyContextOUT =
+	    (PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT *) IMG_OFFSET_ADDR(psDIDestroyContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDIDestroyContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDIDestroyContextIN->hContext,
+					      PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	if (unlikely((psDIDestroyContextOUT->eError != PVRSRV_OK) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDIDestroyContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psDIDestroyContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIDestroyContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DIDestroyContext_exit:
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIReadEntry(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psDIReadEntryIN_UI8,
+			IMG_UINT8 * psDIReadEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIREADENTRY *psDIReadEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIREADENTRY *psDIReadEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIREADENTRY *) IMG_OFFSET_ADDR(psDIReadEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIReadEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) + 0;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIReadEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIReadEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIReadEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIReadEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIReadEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIReadEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIReadEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIReadEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIReadEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIReadEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIReadEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIReadEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIReadEntryOUT->eError =
+	    DIReadEntryKM(psContextInt,
+			  uiEntryPathInt, psDIReadEntryIN->ui64Offset, psDIReadEntryIN->ui64Size);
+
+DIReadEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIReadEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+static_assert(DI_IMPL_BRG_PATH_LEN <= IMG_UINT32_MAX,
+	      "DI_IMPL_BRG_PATH_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeDIWriteEntry(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psDIWriteEntryIN_UI8,
+			 IMG_UINT8 * psDIWriteEntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DIWRITEENTRY *psDIWriteEntryIN =
+	    (PVRSRV_BRIDGE_IN_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DIWRITEENTRY *psDIWriteEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_DIWRITEENTRY *) IMG_OFFSET_ADDR(psDIWriteEntryOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIWriteEntryIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+	IMG_CHAR *uiEntryPathInt = NULL;
+	IMG_CHAR *uiValueInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psDIWriteEntryIN->ui32ValueSize > DI_IMPL_BRG_PATH_LEN))
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto DIWriteEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psDIWriteEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto DIWriteEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psDIWriteEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psDIWriteEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psDIWriteEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto DIWriteEntry_exit;
+			}
+		}
+	}
+
+	{
+		uiEntryPathInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiEntryPathInt, (const void __user *)psDIWriteEntryIN->puiEntryPath,
+		     DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiEntryPathInt)[(DI_IMPL_BRG_PATH_LEN * sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+	if (psDIWriteEntryIN->ui32ValueSize != 0)
+	{
+		uiValueInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiValueInt, (const void __user *)psDIWriteEntryIN->puiValue,
+		     psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psDIWriteEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto DIWriteEntry_exit;
+		}
+		((IMG_CHAR *) uiValueInt)[(psDIWriteEntryIN->ui32ValueSize * sizeof(IMG_CHAR)) -
+					  1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIWriteEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIWriteEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIWriteEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIWriteEntryOUT->eError =
+	    DIWriteEntryKM(psContextInt,
+			   uiEntryPathInt, psDIWriteEntryIN->ui32ValueSize, uiValueInt);
+
+DIWriteEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psDIWriteEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDIListAllEntries(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psDIListAllEntriesIN_UI8,
+			     IMG_UINT8 * psDIListAllEntriesOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DILISTALLENTRIES *psDIListAllEntriesIN =
+	    (PVRSRV_BRIDGE_IN_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *psDIListAllEntriesOUT =
+	    (PVRSRV_BRIDGE_OUT_DILISTALLENTRIES *) IMG_OFFSET_ADDR(psDIListAllEntriesOUT_UI8, 0);
+
+	IMG_HANDLE hContext = psDIListAllEntriesIN->hContext;
+	DI_CONTEXT *psContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDIListAllEntriesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psContextInt,
+				       hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT, IMG_TRUE);
+	if (unlikely(psDIListAllEntriesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DIListAllEntries_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDIListAllEntriesOUT->eError = DIListAllEntriesKM(psContextInt);
+
+DIListAllEntries_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hContext, PVRSRV_HANDLE_TYPE_DI_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDIBridge(void);
+void DeinitDIBridge(void);
+
+/*
+ * Register all DI functions with services
+ */
+PVRSRV_ERROR InitDIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT,
+			      PVRSRVBridgeDICreateContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DICREATECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DICREATECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT,
+			      PVRSRVBridgeDIDestroyContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DIDESTROYCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIDESTROYCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY,
+			      PVRSRVBridgeDIReadEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIREADENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIREADENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY,
+			      PVRSRVBridgeDIWriteEntry, NULL, sizeof(PVRSRV_BRIDGE_IN_DIWRITEENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DIWRITEENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES,
+			      PVRSRVBridgeDIListAllEntries, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DILISTALLENTRIES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DILISTALLENTRIES));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all di functions with services
+ */
+void DeinitDIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DICREATECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIDESTROYCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIREADENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DIWRITEENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DI, PVRSRV_BRIDGE_DI_DILISTALLENTRIES);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_dmabuf_bridge.c b/drivers/gpu/drm/img-rogue/server_dmabuf_bridge.c
new file mode 100644
index 000000000000..89aa1477f55a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_dmabuf_bridge.c
@@ -0,0 +1,574 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for dmabuf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for dmabuf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "physmem_dmabuf.h"
+#include "pmr.h"
+
+#include "common_dmabuf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemImportDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemImportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemImportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *psPhysmemImportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemImportDmaBufOUT_UI8,
+								      0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psPhysmemImportDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportDmaBufIN->puiName,
+		     psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportDmaBuf_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(psPhysmemImportDmaBufIN->ui32NameSize * sizeof(IMG_CHAR))
+					 - 1] = '\0';
+	}
+
+	psPhysmemImportDmaBufOUT->eError =
+	    PhysmemImportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPhysmemImportDmaBufIN->ifd,
+				psPhysmemImportDmaBufIN->uiFlags,
+				psPhysmemImportDmaBufIN->ui32NameSize,
+				uiNameInt,
+				&psPMRPtrInt,
+				&psPhysmemImportDmaBufOUT->uiSize,
+				&psPhysmemImportDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportDmaBufOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psPhysmemImportDmaBufOUT->
+								     hPMRPtr, (void *)psPMRPtrInt,
+								     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _PhysmemImportDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportDmaBuf_exit:
+
+	if (psPhysmemImportDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psPhysmemExportDmaBufIN_UI8,
+				IMG_UINT8 * psPhysmemExportDmaBufOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *psPhysmemExportDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF *) IMG_OFFSET_ADDR(psPhysmemExportDmaBufOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMR = psPhysmemExportDmaBufIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportDmaBufOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportDmaBuf_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportDmaBufOUT->eError =
+	    PhysmemExportDmaBuf(psConnection, OSGetDevNode(psConnection),
+				psPMRInt, &psPhysmemExportDmaBufOUT->iFd);
+
+PhysmemExportDmaBuf_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemExportGemHandle(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemExportGemHandleIN_UI8,
+				   IMG_UINT8 * psPhysmemExportGemHandleOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *psPhysmemExportGemHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE *)
+	    IMG_OFFSET_ADDR(psPhysmemExportGemHandleOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPhysmemExportGemHandleIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPhysmemExportGemHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPhysmemExportGemHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemExportGemHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPhysmemExportGemHandleOUT->eError =
+	    PhysmemExportGemHandle(psConnection, OSGetDevNode(psConnection),
+				   psPMRInt, &psPhysmemExportGemHandleOUT->ui32Handle);
+
+PhysmemExportGemHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemImportSparseDmaBufpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemImportSparseDmaBuf(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufIN_UI8,
+				      IMG_UINT8 * psPhysmemImportSparseDmaBufOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *psPhysmemImportSparseDmaBufOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF *)
+	    IMG_OFFSET_ADDR(psPhysmemImportSparseDmaBufOUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiNameInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (unlikely(psPhysmemImportSparseDmaBufIN->ui32NameSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemImportSparseDmaBufIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemImportSparseDmaBufIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemImportSparseDmaBuf_exit;
+			}
+		}
+	}
+
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemImportSparseDmaBufIN->pui32MappingTable,
+		     psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+	}
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize != 0)
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psPhysmemImportSparseDmaBufIN->puiName,
+		     psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psPhysmemImportSparseDmaBufOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemImportSparseDmaBuf_exit;
+		}
+		((IMG_CHAR *)
+		 uiNameInt)[(psPhysmemImportSparseDmaBufIN->ui32NameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PhysmemImportSparseDmaBuf(psConnection, OSGetDevNode(psConnection),
+				      psPhysmemImportSparseDmaBufIN->ifd,
+				      psPhysmemImportSparseDmaBufIN->uiFlags,
+				      psPhysmemImportSparseDmaBufIN->uiChunkSize,
+				      psPhysmemImportSparseDmaBufIN->ui32NumPhysChunks,
+				      psPhysmemImportSparseDmaBufIN->ui32NumVirtChunks,
+				      ui32MappingTableInt,
+				      psPhysmemImportSparseDmaBufIN->ui32NameSize,
+				      uiNameInt,
+				      &psPMRPtrInt,
+				      &psPhysmemImportSparseDmaBufOUT->uiSize,
+				      &psPhysmemImportSparseDmaBufOUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemImportSparseDmaBufOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psPhysmemImportSparseDmaBufOUT->hPMRPtr, (void *)psPMRPtrInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PhysmemImportSparseDmaBufpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemImportSparseDmaBuf_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemImportSparseDmaBuf_exit:
+
+	if (psPhysmemImportSparseDmaBufOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemImportSparseDmaBufOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitDMABUFBridge(void);
+void DeinitDMABUFBridge(void);
+
+/*
+ * Register all DMABUF functions with services
+ */
+PVRSRV_ERROR InitDMABUFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF,
+			      PVRSRVBridgePhysmemImportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF,
+			      PVRSRVBridgePhysmemExportDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTDMABUF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE,
+			      PVRSRVBridgePhysmemExportGemHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMEXPORTGEMHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMEXPORTGEMHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF,
+			      PVRSRVBridgePhysmemImportSparseDmaBuf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMIMPORTSPARSEDMABUF),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMIMPORTSPARSEDMABUF));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all dmabuf functions with services
+ */
+void DeinitDMABUFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTDMABUF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF, PVRSRV_BRIDGE_DMABUF_PHYSMEMEXPORTGEMHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_DMABUF,
+				PVRSRV_BRIDGE_DMABUF_PHYSMEMIMPORTSPARSEDMABUF);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_htbuffer_bridge.c b/drivers/gpu/drm/img-rogue/server_htbuffer_bridge.c
new file mode 100644
index 000000000000..9b730d28e5b4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_htbuffer_bridge.c
@@ -0,0 +1,227 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for htbuffer
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for htbuffer
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "htbserver.h"
+
+#include "common_htbuffer_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "lock.h"
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(HTB_FLAG_NUM_EL <= IMG_UINT32_MAX,
+	      "HTB_FLAG_NUM_EL must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHTBControl(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psHTBControlIN_UI8,
+		       IMG_UINT8 * psHTBControlOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HTBCONTROL *psHTBControlIN =
+	    (PVRSRV_BRIDGE_IN_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HTBCONTROL *psHTBControlOUT =
+	    (PVRSRV_BRIDGE_OUT_HTBCONTROL *) IMG_OFFSET_ADDR(psHTBControlOUT_UI8, 0);
+
+	IMG_UINT32 *ui32GroupEnableInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psHTBControlIN->ui32NumGroups > HTB_FLAG_NUM_EL))
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HTBControl_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHTBControlOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HTBControl_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHTBControlIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHTBControlIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHTBControlOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HTBControl_exit;
+			}
+		}
+	}
+
+	if (psHTBControlIN->ui32NumGroups != 0)
+	{
+		ui32GroupEnableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32GroupEnableInt,
+		     (const void __user *)psHTBControlIN->pui32GroupEnable,
+		     psHTBControlIN->ui32NumGroups * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psHTBControlOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HTBControl_exit;
+		}
+	}
+
+	psHTBControlOUT->eError =
+	    HTBControlKM(psHTBControlIN->ui32NumGroups,
+			 ui32GroupEnableInt,
+			 psHTBControlIN->ui32LogLevel,
+			 psHTBControlIN->ui32EnablePID,
+			 psHTBControlIN->ui32LogMode, psHTBControlIN->ui32OpMode);
+
+HTBControl_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHTBControlOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+static POS_LOCK pHTBUFFERBridgeLock;
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
+
+#if !defined(EXCLUDE_HTBUFFER_BRIDGE)
+PVRSRV_ERROR InitHTBUFFERBridge(void);
+void DeinitHTBUFFERBridge(void);
+
+/*
+ * Register all HTBUFFER functions with services
+ */
+PVRSRV_ERROR InitHTBUFFERBridge(void)
+{
+	PVR_LOG_RETURN_IF_ERROR(OSLockCreate(&pHTBUFFERBridgeLock), "OSLockCreate");
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL,
+			      PVRSRVBridgeHTBControl, pHTBUFFERBridgeLock,
+			      sizeof(PVRSRV_BRIDGE_IN_HTBCONTROL),
+			      sizeof(PVRSRV_BRIDGE_OUT_HTBCONTROL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all htbuffer functions with services
+ */
+void DeinitHTBUFFERBridge(void)
+{
+	OSLockDestroy(pHTBUFFERBridgeLock);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_HTBUFFER, PVRSRV_BRIDGE_HTBUFFER_HTBCONTROL);
+
+}
+#else /* EXCLUDE_HTBUFFER_BRIDGE */
+/* This bridge is conditional on EXCLUDE_HTBUFFER_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitHTBUFFERBridge() \
+	PVRSRV_OK
+
+#define DeinitHTBUFFERBridge()
+
+#endif /* EXCLUDE_HTBUFFER_BRIDGE */
diff --git a/drivers/gpu/drm/img-rogue/server_mm_bridge.c b/drivers/gpu/drm/img-rogue/server_mm_bridge.c
new file mode 100644
index 000000000000..43535d109c8b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_mm_bridge.c
@@ -0,0 +1,3083 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mm
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mm
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "pvrsrv_memalloc_physheap.h"
+#include "devicemem.h"
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "devicemem_utils.h"
+#include "process_stats.h"
+
+#include "common_mm_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR ReleasePMRExport(void *pvData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvData);
+
+	return PVRSRV_OK;
+}
+#endif
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRExportPMRpsPMRExportIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnexportPMR((PMR_EXPORT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRExportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRExportPMRIN_UI8,
+			 IMG_UINT8 * psPMRExportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMREXPORTPMR *psPMRExportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMREXPORTPMR *psPMRExportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMREXPORTPMR *) IMG_OFFSET_ADDR(psPMRExportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRExportPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRExportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PMRExportPMR(psPMRInt,
+			 &psPMRExportInt,
+			 &psPMRExportPMROUT->ui64Size,
+			 &psPMRExportPMROUT->ui32Log2Contig, &psPMRExportPMROUT->ui64Password);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRExportPMR_exit;
+	}
+
+	/*
+	 * For cases where we need a cross process handle we actually allocate two.
+	 *
+	 * The first one is a connection specific handle and it gets given the real
+	 * release function. This handle does *NOT* get returned to the caller. It's
+	 * purpose is to release any leaked resources when we either have a bad or
+	 * abnormally terminated client. If we didn't do this then the resource
+	 * wouldn't be freed until driver unload. If the resource is freed normally,
+	 * this handle can be looked up via the cross process handle and then
+	 * released accordingly.
+	 *
+	 * The second one is a cross process handle and it gets given a noop release
+	 * function. This handle does get returned to the caller.
+	 */
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRExportPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &hPMRExportInt, (void *)psPMRExportInt,
+				      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _PMRExportPMRpsPMRExportIntRelease);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRExportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle creation. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRExportPMROUT->eError = PVRSRVAllocHandleUnlocked(KERNEL_HANDLE_BASE,
+							      &psPMRExportPMROUT->hPMRExport,
+							      (void *)psPMRExportInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      ReleasePMRExport);
+	if (unlikely(psPMRExportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRExportPMR_exit;
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRExportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRExportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRExportPMROUT->hPMRExport)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(KERNEL_HANDLE_BASE);
+
+			eError = PVRSRVDestroyHandleUnlocked(KERNEL_HANDLE_BASE,
+							     (IMG_HANDLE) psPMRExportPMROUT->
+							     hPMRExport,
+							     PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(KERNEL_HANDLE_BASE);
+
+		}
+
+		if (hPMRExportInt)
+		{
+			PVRSRV_ERROR eError;
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+			eError =
+			    PVRSRVDestroyHandleUnlocked(psConnection->psProcessHandleBase->
+							psHandleBase, hPMRExportInt,
+							PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+			if ((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		}
+
+		else if (psPMRExportInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnexportPMR(psPMRExportInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRExportPMR NULL
+#endif
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+
+static IMG_INT
+PVRSRVBridgePMRUnexportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psPMRUnexportPMRIN_UI8,
+			   IMG_UINT8 * psPMRUnexportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *psPMRUnexportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *psPMRUnexportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR *) IMG_OFFSET_ADDR(psPMRUnexportPMROUT_UI8, 0);
+
+	PMR_EXPORT *psPMRExportInt = NULL;
+	IMG_HANDLE hPMRExportInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+				       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_FALSE);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+	/*
+	 * Find the connection specific handle that represents the same data
+	 * as the cross process handle as releasing it will actually call the
+	 * data's real release function (see the function where the cross
+	 * process handle is allocated for more details).
+	 */
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVFindHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				     &hPMRExportInt,
+				     psPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely(psPMRUnexportPMROUT->eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT(psPMRUnexportPMROUT->eError == PVRSRV_OK);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      hPMRExportInt, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+	}
+	PVR_ASSERT((psPMRUnexportPMROUT->eError == PVRSRV_OK) ||
+		   (psPMRUnexportPMROUT->eError == PVRSRV_ERROR_RETRY));
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Lock over handle destruction. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRUnexportPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(KERNEL_HANDLE_BASE,
+					      (IMG_HANDLE) psPMRUnexportPMRIN->hPMRExport,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	if (unlikely((psPMRUnexportPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnexportPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnexportPMROUT->eError)));
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRUnexportPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+PMRUnexportPMR_exit:
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRUnexportPMR NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgePMRGetUID(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psPMRGetUIDIN_UI8,
+		      IMG_UINT8 * psPMRGetUIDOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRGETUID *psPMRGetUIDIN =
+	    (PVRSRV_BRIDGE_IN_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRGETUID *psPMRGetUIDOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRGETUID *) IMG_OFFSET_ADDR(psPMRGetUIDOUT_UI8, 0);
+
+	IMG_HANDLE hPMR = psPMRGetUIDIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRGetUIDOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psPMRGetUIDOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRGetUID_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRGetUIDOUT->eError = PMRGetUID(psPMRInt, &psPMRGetUIDOUT->ui64UID);
+
+PMRGetUID_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PMRMakeLocalImportHandlepsExtMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnmakeLocalImportHandle((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRMakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleIN_UI8,
+				     IMG_UINT8 * psPMRMakeLocalImportHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *psPMRMakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRMakeLocalImportHandleOUT_UI8, 0);
+
+	IMG_HANDLE hBuffer = psPMRMakeLocalImportHandleIN->hBuffer;
+	PMR *psBufferInt = NULL;
+	PMR *psExtMemInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psBufferInt,
+				       hBuffer,
+				       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE, IMG_TRUE);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError = PMRMakeLocalImportHandle(psBufferInt, &psExtMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRMakeLocalImportHandleOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psPMRMakeLocalImportHandleOUT->hExtMem, (void *)psExtMemInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _PMRMakeLocalImportHandlepsExtMemIntRelease);
+	if (unlikely(psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRMakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRMakeLocalImportHandle_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psBufferInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hBuffer, PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psPMRMakeLocalImportHandleOUT->eError != PVRSRV_OK)
+	{
+		if (psExtMemInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnmakeLocalImportHandle(psExtMemInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnmakeLocalImportHandle(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleIN_UI8,
+				       IMG_UINT8 * psPMRUnmakeLocalImportHandleOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *psPMRUnmakeLocalImportHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE *)
+	    IMG_OFFSET_ADDR(psPMRUnmakeLocalImportHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRUnmakeLocalImportHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psPMRUnmakeLocalImportHandleIN->hExtMem,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_OK) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnmakeLocalImportHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psPMRUnmakeLocalImportHandleOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRUnmakeLocalImportHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+PMRUnmakeLocalImportHandle_exit:
+
+	return 0;
+}
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+static PVRSRV_ERROR _PMRImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psPMRImportPMRIN_UI8,
+			 IMG_UINT8 * psPMRImportPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRIMPORTPMR *psPMRImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *psPMRImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRIMPORTPMR *) IMG_OFFSET_ADDR(psPMRImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hPMRExport = psPMRImportPMRIN->hPMRExport;
+	PMR_EXPORT *psPMRExportInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Look up the address from the handle */
+	psPMRImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(KERNEL_HANDLE_BASE,
+				       (void **)&psPMRExportInt,
+				       hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT, IMG_TRUE);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(KERNEL_HANDLE_BASE);
+		goto PMRImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	psPMRImportPMROUT->eError =
+	    PhysmemImportPMR(psConnection, OSGetDevNode(psConnection),
+			     psPMRExportInt,
+			     psPMRImportPMRIN->ui64uiPassword,
+			     psPMRImportPMRIN->ui64uiSize,
+			     psPMRImportPMRIN->ui32uiLog2Contig, &psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psPMRImportPMROUT->hPMR,
+							      (void *)psPMRInt,
+							      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _PMRImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(KERNEL_HANDLE_BASE);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRExportInt)
+	{
+		PVRSRVReleaseHandleUnlocked(KERNEL_HANDLE_BASE,
+					    hPMRExport, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR_EXPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(KERNEL_HANDLE_BASE);
+
+	if (psPMRImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePMRImportPMR NULL
+#endif
+
+static PVRSRV_ERROR _PMRLocalImportPMRpsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePMRLocalImportPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPMRLocalImportPMRIN_UI8,
+			      IMG_UINT8 * psPMRLocalImportPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *psPMRLocalImportPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *psPMRLocalImportPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR *) IMG_OFFSET_ADDR(psPMRLocalImportPMROUT_UI8, 0);
+
+	IMG_HANDLE hExtHandle = psPMRLocalImportPMRIN->hExtHandle;
+	PMR *psExtHandleInt = NULL;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Look up the address from the handle */
+	psPMRLocalImportPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				       (void **)&psExtHandleInt,
+				       hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT, IMG_TRUE);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError =
+	    PMRLocalImportPMR(psExtHandleInt,
+			      &psPMRInt,
+			      &psPMRLocalImportPMROUT->uiSize, &psPMRLocalImportPMROUT->uiAlign);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRLocalImportPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPMRLocalImportPMROUT->hPMR,
+								   (void *)psPMRInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PMRLocalImportPMRpsPMRIntRelease);
+	if (unlikely(psPMRLocalImportPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRLocalImportPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRLocalImportPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psExtHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					    hExtHandle, PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	if (psPMRLocalImportPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgePMRUnrefPMR(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psPMRUnrefPMRIN_UI8,
+			IMG_UINT8 * psPMRUnrefPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PMRUNREFPMR *psPMRUnrefPMRIN =
+	    (PVRSRV_BRIDGE_IN_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PMRUNREFPMR *psPMRUnrefPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PMRUNREFPMR *) IMG_OFFSET_ADDR(psPMRUnrefPMROUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPMRUnrefPMROUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psPMRUnrefPMRIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	if (unlikely((psPMRUnrefPMROUT->eError != PVRSRV_OK) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psPMRUnrefPMROUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psPMRUnrefPMROUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto PMRUnrefPMR_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PMRUnrefPMR_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _PhysmemNewRamBackedPMRpsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysmemNewRamBackedPMR(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMRIN_UI8,
+				   IMG_UINT8 * psPhysmemNewRamBackedPMROUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMRIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *psPhysmemNewRamBackedPMROUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR *)
+	    IMG_OFFSET_ADDR(psPhysmemNewRamBackedPMROUT_UI8, 0);
+
+	IMG_UINT32 *ui32MappingTableInt = NULL;
+	IMG_CHAR *uiAnnotationInt = NULL;
+	PMR *psPMRPtrInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely
+	    (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (unlikely(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysmemNewRamBackedPMRIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysmemNewRamBackedPMRIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysmemNewRamBackedPMR_exit;
+			}
+		}
+	}
+
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks != 0)
+	{
+		ui32MappingTableInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32MappingTableInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->pui32MappingTable,
+		     psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+	}
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength != 0)
+	{
+		uiAnnotationInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiAnnotationInt,
+		     (const void __user *)psPhysmemNewRamBackedPMRIN->puiAnnotation,
+		     psPhysmemNewRamBackedPMRIN->ui32AnnotationLength * sizeof(IMG_CHAR)) !=
+		    PVRSRV_OK)
+		{
+			psPhysmemNewRamBackedPMROUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysmemNewRamBackedPMR_exit;
+		}
+		((IMG_CHAR *)
+		 uiAnnotationInt)[(psPhysmemNewRamBackedPMRIN->ui32AnnotationLength *
+				   sizeof(IMG_CHAR)) - 1] = '\0';
+	}
+
+	psPhysmemNewRamBackedPMROUT->eError =
+	    PhysmemNewRamBackedPMR(psConnection, OSGetDevNode(psConnection),
+				   psPhysmemNewRamBackedPMRIN->uiSize,
+				   psPhysmemNewRamBackedPMRIN->ui32NumPhysChunks,
+				   psPhysmemNewRamBackedPMRIN->ui32NumVirtChunks,
+				   ui32MappingTableInt,
+				   psPhysmemNewRamBackedPMRIN->ui32Log2PageSize,
+				   psPhysmemNewRamBackedPMRIN->uiFlags,
+				   psPhysmemNewRamBackedPMRIN->ui32AnnotationLength,
+				   uiAnnotationInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PID,
+				   &psPMRPtrInt,
+				   psPhysmemNewRamBackedPMRIN->ui32PDumpFlags,
+				   &psPhysmemNewRamBackedPMROUT->uiOutFlags);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemNewRamBackedPMROUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psPhysmemNewRamBackedPMROUT->
+									hPMRPtr,
+									(void *)psPMRPtrInt,
+									PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_PhysmemNewRamBackedPMRpsPMRPtrIntRelease);
+	if (unlikely(psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemNewRamBackedPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemNewRamBackedPMR_exit:
+
+	if (psPhysmemNewRamBackedPMROUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysmemNewRamBackedPMROUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntCtxCreatepsDevMemServerContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntCtxDestroy((DEVMEMINT_CTX *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxCreate(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemIntCtxCreateIN_UI8,
+			       IMG_UINT8 * psDevmemIntCtxCreateOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *psDevmemIntCtxCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE *) IMG_OFFSET_ADDR(psDevmemIntCtxCreateOUT_UI8,
+								     0);
+
+	DEVMEMINT_CTX *psDevMemServerContextInt = NULL;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	psDevmemIntCtxCreateOUT->hDevMemServerContext = NULL;
+
+	psDevmemIntCtxCreateOUT->eError =
+	    DevmemIntCtxCreate(psConnection, OSGetDevNode(psConnection),
+			       psDevmemIntCtxCreateIN->bbKernelMemoryCtx,
+			       &psDevMemServerContextInt,
+			       &hPrivDataInt, &psDevmemIntCtxCreateOUT->ui32CPUCacheLineSize);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								    &psDevmemIntCtxCreateOUT->
+								    hDevMemServerContext,
+								    (void *)
+								    psDevMemServerContextInt,
+								    PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX,
+								    PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								    (PFN_HANDLE_RELEASE) &
+								    _DevmemIntCtxCreatepsDevMemServerContextIntRelease);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	psDevmemIntCtxCreateOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntCtxCreateOUT->
+								       hPrivData,
+								       (void *)hPrivDataInt,
+								       PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       psDevmemIntCtxCreateOUT->
+								       hDevMemServerContext);
+	if (unlikely(psDevmemIntCtxCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxCreate_exit:
+
+	if (psDevmemIntCtxCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemIntCtxCreateOUT->hDevMemServerContext)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psDevmemIntCtxCreateOUT->
+							     hDevMemServerContext,
+							     PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psDevMemServerContextInt)
+		{
+			DevmemIntCtxDestroy(psDevMemServerContextInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntCtxDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntCtxDestroyIN_UI8,
+				IMG_UINT8 * psDevmemIntCtxDestroyOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *psDevmemIntCtxDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY *) IMG_OFFSET_ADDR(psDevmemIntCtxDestroyOUT_UI8,
+								      0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntCtxDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntCtxDestroyIN->
+					      hDevmemServerContext,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	if (unlikely
+	    ((psDevmemIntCtxDestroyOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntCtxDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntCtxDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntCtxDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntCtxDestroy_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntHeapDestroy((DEVMEMINT_HEAP *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapCreate(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psDevmemIntHeapCreateIN_UI8,
+				IMG_UINT8 * psDevmemIntHeapCreateOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *psDevmemIntHeapCreateOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE *) IMG_OFFSET_ADDR(psDevmemIntHeapCreateOUT_UI8,
+								      0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntHeapCreateIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+	DEVMEMINT_HEAP *psDevmemHeapPtrInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntHeapCreateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError =
+	    DevmemIntHeapCreate(psDevmemCtxInt,
+				psDevmemIntHeapCreateIN->ui32HeapConfigIndex,
+				psDevmemIntHeapCreateIN->ui32HeapIndex, &psDevmemHeapPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapCreateOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psDevmemIntHeapCreateOUT->
+								     hDevmemHeapPtr,
+								     (void *)psDevmemHeapPtrInt,
+								     PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _DevmemIntHeapCreatepsDevmemHeapPtrIntRelease);
+	if (unlikely(psDevmemIntHeapCreateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapCreate_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapCreate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntHeapCreateOUT->eError != PVRSRV_OK)
+	{
+		if (psDevmemHeapPtrInt)
+		{
+			DevmemIntHeapDestroy(psDevmemHeapPtrInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntHeapDestroy(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemIntHeapDestroyIN_UI8,
+				 IMG_UINT8 * psDevmemIntHeapDestroyOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY *) IMG_OFFSET_ADDR(psDevmemIntHeapDestroyIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *psDevmemIntHeapDestroyOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY *)
+	    IMG_OFFSET_ADDR(psDevmemIntHeapDestroyOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntHeapDestroyOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntHeapDestroyIN->hDevmemHeap,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	if (unlikely((psDevmemIntHeapDestroyOUT->eError != PVRSRV_OK) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psDevmemIntHeapDestroyOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntHeapDestroyOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntHeapDestroy_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntHeapDestroy_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psDevmemIntMapPMRIN_UI8,
+			    IMG_UINT8 * psDevmemIntMapPMROUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *psDevmemIntMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *psDevmemIntMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntMapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntMapPMROUT->eError = DevmemIntMapPMR(psReservationInt, psPMRInt);
+
+DevmemIntMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnmapPMR(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psDevmemIntUnmapPMRIN_UI8,
+			      IMG_UINT8 * psDevmemIntUnmapPMROUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *psDevmemIntUnmapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR *) IMG_OFFSET_ADDR(psDevmemIntUnmapPMROUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemIntUnmapPMRIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntUnmapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemIntUnmapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnmapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnmapPMROUT->eError = DevmemIntUnmapPMR(psReservationInt);
+
+DevmemIntUnmapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIntReserveRangeIN_UI8,
+				  IMG_UINT8 * psDevmemIntReserveRangeOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *psDevmemIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError =
+	    DevmemIntReserveRange(psDevmemServerHeapInt,
+				  psDevmemIntReserveRangeIN->sAddress,
+				  psDevmemIntReserveRangeIN->uiLength,
+				  psDevmemIntReserveRangeIN->uiFlags, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psDevmemIntReserveRangeOUT->
+								       hReservation,
+								       (void *)psReservationInt,
+								       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _DevmemIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemIntUnreserveRange((DEVMEMINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntReserveRangeAndMapPMR(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMRIN_UI8,
+					   IMG_UINT8 * psDevmemIntReserveRangeAndMapPMROUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMRIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMRIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *psDevmemIntReserveRangeAndMapPMROUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR *)
+	    IMG_OFFSET_ADDR(psDevmemIntReserveRangeAndMapPMROUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemIntReserveRangeAndMapPMRIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	IMG_HANDLE hPMR = psDevmemIntReserveRangeAndMapPMRIN->hPMR;
+	PMR *psPMRInt = NULL;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    DevmemIntReserveRangeAndMapPMR(psDevmemServerHeapInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->sAddress,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiLength,
+					   psPMRInt,
+					   psDevmemIntReserveRangeAndMapPMRIN->uiFlags,
+					   &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntReserveRangeAndMapPMROUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psDevmemIntReserveRangeAndMapPMROUT->hReservation,
+				      (void *)psReservationInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _DevmemIntReserveRangeAndMapPMRpsReservationIntRelease);
+	if (unlikely(psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntReserveRangeAndMapPMR_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntReserveRangeAndMapPMR_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemIntReserveRangeAndMapPMROUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeIN_UI8,
+				    IMG_UINT8 * psDevmemIntUnreserveRangeOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *psDevmemIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+static_assert(PMR_MAX_SUPPORTED_4K_PAGE_COUNT <= IMG_UINT32_MAX,
+	      "PMR_MAX_SUPPORTED_4K_PAGE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeChangeSparseMem(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psChangeSparseMemIN_UI8,
+			    IMG_UINT8 * psChangeSparseMemOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *psChangeSparseMemIN =
+	    (PVRSRV_BRIDGE_IN_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *psChangeSparseMemOUT =
+	    (PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM *) IMG_OFFSET_ADDR(psChangeSparseMemOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AllocPageIndicesInt = NULL;
+	IMG_UINT32 *ui32FreePageIndicesInt = NULL;
+	IMG_HANDLE hReservation = psChangeSparseMemIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psChangeSparseMemIN->ui32AllocPageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (unlikely(psChangeSparseMemIN->ui32FreePageCount > PMR_MAX_SUPPORTED_4K_PAGE_COUNT))
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto ChangeSparseMem_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psChangeSparseMemOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto ChangeSparseMem_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psChangeSparseMemIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psChangeSparseMemIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psChangeSparseMemOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto ChangeSparseMem_exit;
+			}
+		}
+	}
+
+	if (psChangeSparseMemIN->ui32AllocPageCount != 0)
+	{
+		ui32AllocPageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AllocPageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32AllocPageIndices,
+		     psChangeSparseMemIN->ui32AllocPageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+	if (psChangeSparseMemIN->ui32FreePageCount != 0)
+	{
+		ui32FreePageIndicesInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32FreePageIndicesInt,
+		     (const void __user *)psChangeSparseMemIN->pui32FreePageIndices,
+		     psChangeSparseMemIN->ui32FreePageCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psChangeSparseMemOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto ChangeSparseMem_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psChangeSparseMemOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psChangeSparseMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto ChangeSparseMem_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psChangeSparseMemOUT->eError =
+	    DevmemIntChangeSparse(psChangeSparseMemIN->ui32AllocPageCount,
+				  ui32AllocPageIndicesInt,
+				  psChangeSparseMemIN->ui32FreePageCount,
+				  ui32FreePageIndicesInt,
+				  psChangeSparseMemIN->ui32SparseFlags, psReservationInt);
+
+ChangeSparseMem_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psChangeSparseMemOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIsVDevAddrValid(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidIN_UI8,
+				  IMG_UINT8 * psDevmemIsVDevAddrValidOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *psDevmemIsVDevAddrValidOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID *)
+	    IMG_OFFSET_ADDR(psDevmemIsVDevAddrValidOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIsVDevAddrValidIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIsVDevAddrValidOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIsVDevAddrValidOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIsVDevAddrValid_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIsVDevAddrValidOUT->eError =
+	    DevmemIntIsVDevAddrValid(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, psDevmemIsVDevAddrValidIN->sAddress);
+
+DevmemIsVDevAddrValid_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(RGX_FEATURE_FBCDC)
+
+static IMG_INT
+PVRSRVBridgeDevmemInvalidateFBSCTable(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableIN_UI8,
+				      IMG_UINT8 * psDevmemInvalidateFBSCTableOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *psDevmemInvalidateFBSCTableOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE *)
+	    IMG_OFFSET_ADDR(psDevmemInvalidateFBSCTableOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemInvalidateFBSCTableIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemInvalidateFBSCTableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemInvalidateFBSCTable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemInvalidateFBSCTableOUT->eError =
+	    DevmemIntInvalidateFBSCTable(psDevmemCtxInt,
+					 psDevmemInvalidateFBSCTableIN->ui64FBSCEntries);
+
+DevmemInvalidateFBSCTable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeDevmemInvalidateFBSCTable NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigCount(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountIN_UI8,
+				   IMG_UINT8 * psHeapCfgHeapConfigCountOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *psHeapCfgHeapConfigCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigCountOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHeapCfgHeapConfigCountIN);
+
+	psHeapCfgHeapConfigCountOUT->eError =
+	    HeapCfgHeapConfigCount(psConnection, OSGetDevNode(psConnection),
+				   &psHeapCfgHeapConfigCountOUT->ui32NumHeapConfigs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapCount(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psHeapCfgHeapCountIN_UI8,
+			     IMG_UINT8 * psHeapCfgHeapCountOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *psHeapCfgHeapCountOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT *) IMG_OFFSET_ADDR(psHeapCfgHeapCountOUT_UI8, 0);
+
+	psHeapCfgHeapCountOUT->eError =
+	    HeapCfgHeapCount(psConnection, OSGetDevNode(psConnection),
+			     psHeapCfgHeapCountIN->ui32HeapConfigIndex,
+			     &psHeapCfgHeapCountOUT->ui32NumHeaps);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapConfigName(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameIN_UI8,
+				  IMG_UINT8 * psHeapCfgHeapConfigNameOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *psHeapCfgHeapConfigNameOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME *)
+	    IMG_OFFSET_ADDR(psHeapCfgHeapConfigNameOUT_UI8, 0);
+
+	IMG_CHAR *puiHeapConfigNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) +
+	    0;
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	psHeapCfgHeapConfigNameOUT->puiHeapConfigName =
+	    psHeapCfgHeapConfigNameIN->puiHeapConfigName;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapConfigNameIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapConfigNameIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapConfigName_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz != 0)
+	{
+		puiHeapConfigNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapConfigNameOUT->eError =
+	    HeapCfgHeapConfigName(psConnection, OSGetDevNode(psConnection),
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigIndex,
+				  psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz,
+				  puiHeapConfigNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapConfigNameOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapConfigName_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapConfigNameInt) &&
+	    ((psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapConfigNameOUT->puiHeapConfigName,
+		      puiHeapConfigNameInt,
+		      (psHeapCfgHeapConfigNameIN->ui32HeapConfigNameBufSz * sizeof(IMG_CHAR))) !=
+		     PVRSRV_OK))
+		{
+			psHeapCfgHeapConfigNameOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapConfigName_exit;
+		}
+	}
+
+HeapCfgHeapConfigName_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapConfigNameOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(DEVMEM_HEAPNAME_MAXLENGTH <= IMG_UINT32_MAX,
+	      "DEVMEM_HEAPNAME_MAXLENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeHeapCfgHeapDetails(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psHeapCfgHeapDetailsIN_UI8,
+			       IMG_UINT8 * psHeapCfgHeapDetailsOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsIN =
+	    (PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *psHeapCfgHeapDetailsOUT =
+	    (PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS *) IMG_OFFSET_ADDR(psHeapCfgHeapDetailsOUT_UI8,
+								     0);
+
+	IMG_CHAR *puiHeapNameOutInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) + 0;
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz > DEVMEM_HEAPNAME_MAXLENGTH)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	psHeapCfgHeapDetailsOUT->puiHeapNameOut = psHeapCfgHeapDetailsIN->puiHeapNameOut;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psHeapCfgHeapDetailsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psHeapCfgHeapDetailsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto HeapCfgHeapDetails_exit;
+			}
+		}
+	}
+
+	if (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz != 0)
+	{
+		puiHeapNameOutInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR);
+	}
+
+	psHeapCfgHeapDetailsOUT->eError =
+	    HeapCfgHeapDetails(psConnection, OSGetDevNode(psConnection),
+			       psHeapCfgHeapDetailsIN->ui32HeapConfigIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapIndex,
+			       psHeapCfgHeapDetailsIN->ui32HeapNameBufSz,
+			       puiHeapNameOutInt,
+			       &psHeapCfgHeapDetailsOUT->sDevVAddrBase,
+			       &psHeapCfgHeapDetailsOUT->uiHeapLength,
+			       &psHeapCfgHeapDetailsOUT->uiReservedRegionLength,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2DataPageSizeOut,
+			       &psHeapCfgHeapDetailsOUT->ui32Log2ImportAlignmentOut);
+	/* Exit early if bridged call fails */
+	if (unlikely(psHeapCfgHeapDetailsOUT->eError != PVRSRV_OK))
+	{
+		goto HeapCfgHeapDetails_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiHeapNameOutInt) &&
+	    ((psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psHeapCfgHeapDetailsOUT->puiHeapNameOut,
+		      puiHeapNameOutInt,
+		      (psHeapCfgHeapDetailsIN->ui32HeapNameBufSz * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psHeapCfgHeapDetailsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto HeapCfgHeapDetails_exit;
+		}
+	}
+
+HeapCfgHeapDetails_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psHeapCfgHeapDetailsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemIntRegisterPFNotifyKM(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMIN_UI8,
+					IMG_UINT8 * psDevmemIntRegisterPFNotifyKMOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *psDevmemIntRegisterPFNotifyKMOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM *)
+	    IMG_OFFSET_ADDR(psDevmemIntRegisterPFNotifyKMOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemIntRegisterPFNotifyKMIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemIntRegisterPFNotifyKMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemIntRegisterPFNotifyKM_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemIntRegisterPFNotifyKMOUT->eError =
+	    DevmemIntRegisterPFNotifyKM(psDevmemCtxInt, psDevmemIntRegisterPFNotifyKMIN->bRegister);
+
+DevmemIntRegisterPFNotifyKM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PHYS_HEAP_LAST <= IMG_UINT32_MAX,
+	      "PVRSRV_PHYS_HEAP_LAST must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgePhysHeapGetMemInfo(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psPhysHeapGetMemInfoIN_UI8,
+			       IMG_UINT8 * psPhysHeapGetMemInfoOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoIN =
+	    (PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *psPhysHeapGetMemInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO *) IMG_OFFSET_ADDR(psPhysHeapGetMemInfoOUT_UI8,
+								     0);
+
+	PVRSRV_PHYS_HEAP *eaPhysHeapIDInt = NULL;
+	PHYS_HEAP_MEM_STATS *pasapPhysHeapMemStatsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) +
+	    ((IMG_UINT64) psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS)) +
+	    0;
+
+	if (unlikely(psPhysHeapGetMemInfoIN->ui32PhysHeapCount > PVRSRV_PHYS_HEAP_LAST))
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats =
+	    psPhysHeapGetMemInfoIN->pasapPhysHeapMemStats;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psPhysHeapGetMemInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psPhysHeapGetMemInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto PhysHeapGetMemInfo_exit;
+			}
+		}
+	}
+
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		eaPhysHeapIDInt =
+		    (PVRSRV_PHYS_HEAP *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP);
+	}
+
+	/* Copy the data over */
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, eaPhysHeapIDInt,
+		     (const void __user *)psPhysHeapGetMemInfoIN->peaPhysHeapID,
+		     psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PVRSRV_PHYS_HEAP)) !=
+		    PVRSRV_OK)
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+	if (psPhysHeapGetMemInfoIN->ui32PhysHeapCount != 0)
+	{
+		pasapPhysHeapMemStatsInt =
+		    (PHYS_HEAP_MEM_STATS *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS);
+	}
+
+	psPhysHeapGetMemInfoOUT->eError =
+	    PVRSRVPhysHeapGetMemInfoKM(psConnection, OSGetDevNode(psConnection),
+				       psPhysHeapGetMemInfoIN->ui32PhysHeapCount,
+				       eaPhysHeapIDInt, pasapPhysHeapMemStatsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysHeapGetMemInfoOUT->eError != PVRSRV_OK))
+	{
+		goto PhysHeapGetMemInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pasapPhysHeapMemStatsInt) &&
+	    ((psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psPhysHeapGetMemInfoOUT->pasapPhysHeapMemStats,
+		      pasapPhysHeapMemStatsInt,
+		      (psPhysHeapGetMemInfoIN->ui32PhysHeapCount * sizeof(PHYS_HEAP_MEM_STATS))) !=
+		     PVRSRV_OK))
+		{
+			psPhysHeapGetMemInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto PhysHeapGetMemInfo_exit;
+		}
+	}
+
+PhysHeapGetMemInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psPhysHeapGetMemInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDefaultPhysicalHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapIN_UI8,
+				   IMG_UINT8 * psGetDefaultPhysicalHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapIN =
+	    (PVRSRV_BRIDGE_IN_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *psGetDefaultPhysicalHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP *)
+	    IMG_OFFSET_ADDR(psGetDefaultPhysicalHeapOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDefaultPhysicalHeapIN);
+
+	psGetDefaultPhysicalHeapOUT->eError =
+	    PVRSRVGetDefaultPhysicalHeapKM(psConnection, OSGetDevNode(psConnection),
+					   &psGetDefaultPhysicalHeapOUT->eHeap);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemGetFaultAddress(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psDevmemGetFaultAddressIN_UI8,
+				  IMG_UINT8 * psDevmemGetFaultAddressOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *psDevmemGetFaultAddressOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS *)
+	    IMG_OFFSET_ADDR(psDevmemGetFaultAddressOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemCtx = psDevmemGetFaultAddressIN->hDevmemCtx;
+	DEVMEMINT_CTX *psDevmemCtxInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemGetFaultAddressOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemCtxInt,
+				       hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX, IMG_TRUE);
+	if (unlikely(psDevmemGetFaultAddressOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemGetFaultAddress_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemGetFaultAddressOUT->eError =
+	    DevmemIntGetFaultAddress(psConnection, OSGetDevNode(psConnection),
+				     psDevmemCtxInt, &psDevmemGetFaultAddressOUT->sFaultAddress);
+
+DevmemGetFaultAddress_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemCtxInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemCtx, PVRSRV_HANDLE_TYPE_DEVMEMINT_CTX);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PVRSRV_ENABLE_PROCESS_STATS)
+
+static IMG_INT
+PVRSRVBridgePVRSRVStatsUpdateOOMStat(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatIN_UI8,
+				     IMG_UINT8 * psPVRSRVStatsUpdateOOMStatOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatIN =
+	    (PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *psPVRSRVStatsUpdateOOMStatOUT =
+	    (PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT *)
+	    IMG_OFFSET_ADDR(psPVRSRVStatsUpdateOOMStatOUT_UI8, 0);
+
+	psPVRSRVStatsUpdateOOMStatOUT->eError =
+	    PVRSRVStatsUpdateOOMStat(psConnection, OSGetDevNode(psConnection),
+				     psPVRSRVStatsUpdateOOMStatIN->ui32ui32StatType,
+				     psPVRSRVStatsUpdateOOMStatIN->ui32pid);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgePVRSRVStatsUpdateOOMStat NULL
+#endif
+
+static PVRSRV_ERROR _DevmemXIntReserveRangepsReservationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = DevmemXIntUnreserveRange((DEVMEMXINT_RESERVATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntReserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psDevmemXIntReserveRangeIN_UI8,
+				   IMG_UINT8 * psDevmemXIntReserveRangeOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *psDevmemXIntReserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntReserveRangeOUT_UI8, 0);
+
+	IMG_HANDLE hDevmemServerHeap = psDevmemXIntReserveRangeIN->hDevmemServerHeap;
+	DEVMEMINT_HEAP *psDevmemServerHeapInt = NULL;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntReserveRangeOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psDevmemServerHeapInt,
+				       hDevmemServerHeap,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP, IMG_TRUE);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError =
+	    DevmemXIntReserveRange(psDevmemServerHeapInt,
+				   psDevmemXIntReserveRangeIN->sAddress,
+				   psDevmemXIntReserveRangeIN->uiLength, &psReservationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntReserveRangeOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psDevmemXIntReserveRangeOUT->
+									hReservation,
+									(void *)psReservationInt,
+									PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_DevmemXIntReserveRangepsReservationIntRelease);
+	if (unlikely(psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntReserveRange_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntReserveRange_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psDevmemServerHeapInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hDevmemServerHeap, PVRSRV_HANDLE_TYPE_DEVMEMINT_HEAP);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psDevmemXIntReserveRangeOUT->eError != PVRSRV_OK)
+	{
+		if (psReservationInt)
+		{
+			DevmemXIntUnreserveRange(psReservationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnreserveRange(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeIN_UI8,
+				     IMG_UINT8 * psDevmemXIntUnreserveRangeOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *psDevmemXIntUnreserveRangeOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnreserveRangeOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnreserveRangeOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psDevmemXIntUnreserveRangeIN->
+					      hReservation,
+					      PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	if (unlikely
+	    ((psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_OK)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psDevmemXIntUnreserveRangeOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psDevmemXIntUnreserveRangeOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnreserveRange_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+DevmemXIntUnreserveRange_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapPages(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psDevmemXIntMapPagesIN_UI8,
+			       IMG_UINT8 * psDevmemXIntMapPagesOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *psDevmemXIntMapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntMapPagesOUT_UI8,
+								     0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psDevmemXIntMapPagesIN->hPMR;
+	PMR *psPMRInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapPagesOUT->eError =
+	    DevmemXIntMapPages(psReservationInt,
+			       psPMRInt,
+			       psDevmemXIntMapPagesIN->ui32PageCount,
+			       psDevmemXIntMapPagesIN->ui32PhysPageOffset,
+			       psDevmemXIntMapPagesIN->uiFlags,
+			       psDevmemXIntMapPagesIN->ui32VirtPageOffset);
+
+DevmemXIntMapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntUnmapPages(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesIN_UI8,
+				 IMG_UINT8 * psDevmemXIntUnmapPagesOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES *) IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *psDevmemXIntUnmapPagesOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES *)
+	    IMG_OFFSET_ADDR(psDevmemXIntUnmapPagesOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntUnmapPagesIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntUnmapPagesOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntUnmapPagesOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntUnmapPages_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntUnmapPagesOUT->eError =
+	    DevmemXIntUnmapPages(psReservationInt,
+				 psDevmemXIntUnmapPagesIN->ui32VirtPageOffset,
+				 psDevmemXIntUnmapPagesIN->ui32PageCount);
+
+DevmemXIntUnmapPages_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDevmemXIntMapVRangeToBackingPage(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageIN_UI8,
+					     IMG_UINT8 * psDevmemXIntMapVRangeToBackingPageOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageIN =
+	    (PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *psDevmemXIntMapVRangeToBackingPageOUT =
+	    (PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE *)
+	    IMG_OFFSET_ADDR(psDevmemXIntMapVRangeToBackingPageOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psDevmemXIntMapVRangeToBackingPageIN->hReservation;
+	DEVMEMXINT_RESERVATION *psReservationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psDevmemXIntMapVRangeToBackingPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto DevmemXIntMapVRangeToBackingPage_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psDevmemXIntMapVRangeToBackingPageOUT->eError =
+	    DevmemXIntMapVRangeToBackingPage(psReservationInt,
+					     psDevmemXIntMapVRangeToBackingPageIN->ui32PageCount,
+					     psDevmemXIntMapVRangeToBackingPageIN->uiFlags,
+					     psDevmemXIntMapVRangeToBackingPageIN->
+					     ui32VirtPageOffset);
+
+DevmemXIntMapVRangeToBackingPage_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMXINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMBridge(void);
+void DeinitMMBridge(void);
+
+/*
+ * Register all MM functions with services
+ */
+PVRSRV_ERROR InitMMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR,
+			      PVRSRVBridgePMRExportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMREXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMREXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR,
+			      PVRSRVBridgePMRUnexportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNEXPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNEXPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID, PVRSRVBridgePMRGetUID,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRGETUID),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRGETUID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRMakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE,
+			      PVRSRVBridgePMRUnmakeLocalImportHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRUNMAKELOCALIMPORTHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNMAKELOCALIMPORTHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR,
+			      PVRSRVBridgePMRImportPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR,
+			      PVRSRVBridgePMRLocalImportPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PMRLOCALIMPORTPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRLOCALIMPORTPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR,
+			      PVRSRVBridgePMRUnrefPMR, NULL, sizeof(PVRSRV_BRIDGE_IN_PMRUNREFPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PMRUNREFPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR,
+			      PVRSRVBridgePhysmemNewRamBackedPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMNEWRAMBACKEDPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMNEWRAMBACKEDPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE,
+			      PVRSRVBridgeDevmemIntCtxCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY,
+			      PVRSRVBridgeDevmemIntCtxDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTCTXDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTCTXDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE,
+			      PVRSRVBridgeDevmemIntHeapCreate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPCREATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPCREATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY,
+			      PVRSRVBridgeDevmemIntHeapDestroy, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTHEAPDESTROY),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTHEAPDESTROY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR,
+			      PVRSRVBridgeDevmemIntMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR,
+			      PVRSRVBridgeDevmemIntUnmapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE,
+			      PVRSRVBridgeDevmemIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR,
+			      PVRSRVBridgeDevmemIntReserveRangeAndMapPMR, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTRESERVERANGEANDMAPPMR),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTRESERVERANGEANDMAPPMR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM,
+			      PVRSRVBridgeChangeSparseMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_CHANGESPARSEMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_CHANGESPARSEMEM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID,
+			      PVRSRVBridgeDevmemIsVDevAddrValid, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMISVDEVADDRVALID),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMISVDEVADDRVALID));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE,
+			      PVRSRVBridgeDevmemInvalidateFBSCTable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINVALIDATEFBSCTABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINVALIDATEFBSCTABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT,
+			      PVRSRVBridgeHeapCfgHeapConfigCount, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT,
+			      PVRSRVBridgeHeapCfgHeapCount, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCOUNT),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCOUNT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME,
+			      PVRSRVBridgeHeapCfgHeapConfigName, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPCONFIGNAME),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPCONFIGNAME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS,
+			      PVRSRVBridgeHeapCfgHeapDetails, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_HEAPCFGHEAPDETAILS),
+			      sizeof(PVRSRV_BRIDGE_OUT_HEAPCFGHEAPDETAILS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM,
+			      PVRSRVBridgeDevmemIntRegisterPFNotifyKM, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMINTREGISTERPFNOTIFYKM),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMINTREGISTERPFNOTIFYKM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO,
+			      PVRSRVBridgePhysHeapGetMemInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSHEAPGETMEMINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSHEAPGETMEMINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP,
+			      PVRSRVBridgeGetDefaultPhysicalHeap, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEFAULTPHYSICALHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS,
+			      PVRSRVBridgeDevmemGetFaultAddress, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMGETFAULTADDRESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMGETFAULTADDRESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT,
+			      PVRSRVBridgePVRSRVStatsUpdateOOMStat, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PVRSRVSTATSUPDATEOOMSTAT),
+			      sizeof(PVRSRV_BRIDGE_OUT_PVRSRVSTATSUPDATEOOMSTAT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntReserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE,
+			      PVRSRVBridgeDevmemXIntUnreserveRange, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNRESERVERANGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNRESERVERANGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES,
+			      PVRSRVBridgeDevmemXIntMapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES,
+			      PVRSRVBridgeDevmemXIntUnmapPages, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTUNMAPPAGES),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTUNMAPPAGES));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE,
+			      PVRSRVBridgeDevmemXIntMapVRangeToBackingPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DEVMEMXINTMAPVRANGETOBACKINGPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_DEVMEMXINTMAPVRANGETOBACKINGPAGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mm functions with services
+ */
+void DeinitMMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMREXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNEXPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRGETUID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNMAKELOCALIMPORTHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRLOCALIMPORTPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PMRUNREFPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSMEMNEWRAMBACKEDPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTCTXDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPCREATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTHEAPDESTROY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTRESERVERANGEANDMAPPMR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_CHANGESPARSEMEM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMISVDEVADDRVALID);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINVALIDATEFBSCTABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCOUNT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPCONFIGNAME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_HEAPCFGHEAPDETAILS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMINTREGISTERPFNOTIFYKM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PHYSHEAPGETMEMINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_GETDEFAULTPHYSICALHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMGETFAULTADDRESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_PVRSRVSTATSUPDATEOOMSTAT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNRESERVERANGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM, PVRSRV_BRIDGE_MM_DEVMEMXINTUNMAPPAGES);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MM,
+				PVRSRV_BRIDGE_MM_DEVMEMXINTMAPVRANGETOBACKINGPAGE);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_mmextmem_bridge.c b/drivers/gpu/drm/img-rogue/server_mmextmem_bridge.c
new file mode 100644
index 000000000000..538d04cdf0e0
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_mmextmem_bridge.c
@@ -0,0 +1,166 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for mmextmem
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for mmextmem
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "pmr.h"
+#include "devicemem_heapcfg.h"
+#include "physmem.h"
+#include "physmem_extmem.h"
+
+#include "common_mmextmem_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _PhysmemWrapExtMempsPMRPtrIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PMRUnrefPMR((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgePhysmemWrapExtMem(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psPhysmemWrapExtMemIN_UI8,
+			      IMG_UINT8 * psPhysmemWrapExtMemOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemIN =
+	    (PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *psPhysmemWrapExtMemOUT =
+	    (PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM *) IMG_OFFSET_ADDR(psPhysmemWrapExtMemOUT_UI8, 0);
+
+	PMR *psPMRPtrInt = NULL;
+
+	psPhysmemWrapExtMemOUT->eError =
+	    PhysmemWrapExtMem(psConnection, OSGetDevNode(psConnection),
+			      psPhysmemWrapExtMemIN->uiSize,
+			      psPhysmemWrapExtMemIN->ui64CpuVAddr,
+			      psPhysmemWrapExtMemIN->uiFlags, &psPMRPtrInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psPhysmemWrapExtMemOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psPhysmemWrapExtMemOUT->hPMRPtr,
+								   (void *)psPMRPtrInt,
+								   PVRSRV_HANDLE_TYPE_PHYSMEM_PMR,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _PhysmemWrapExtMempsPMRPtrIntRelease);
+	if (unlikely(psPhysmemWrapExtMemOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto PhysmemWrapExtMem_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+PhysmemWrapExtMem_exit:
+
+	if (psPhysmemWrapExtMemOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRPtrInt)
+		{
+			LockHandle(KERNEL_HANDLE_BASE);
+			PMRUnrefPMR(psPMRPtrInt);
+			UnlockHandle(KERNEL_HANDLE_BASE);
+		}
+	}
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitMMEXTMEMBridge(void);
+void DeinitMMEXTMEMBridge(void);
+
+/*
+ * Register all MMEXTMEM functions with services
+ */
+PVRSRV_ERROR InitMMEXTMEMBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM,
+			      PVRSRVBridgePhysmemWrapExtMem, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_PHYSMEMWRAPEXTMEM),
+			      sizeof(PVRSRV_BRIDGE_OUT_PHYSMEMWRAPEXTMEM));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all mmextmem functions with services
+ */
+void DeinitMMEXTMEMBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_MMEXTMEM, PVRSRV_BRIDGE_MMEXTMEM_PHYSMEMWRAPEXTMEM);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_pvrtl_bridge.c b/drivers/gpu/drm/img-rogue/server_pvrtl_bridge.c
new file mode 100644
index 000000000000..9facf794011e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_pvrtl_bridge.c
@@ -0,0 +1,827 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for pvrtl
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for pvrtl
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "tlserver.h"
+
+#include "common_pvrtl_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _TLOpenStreampsSDIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = TLServerCloseStreamKM((TL_STREAM_DESC *) pvData);
+	return eError;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLOpenStream(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psTLOpenStreamIN_UI8,
+			 IMG_UINT8 * psTLOpenStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLOPENSTREAM *psTLOpenStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLOPENSTREAM *psTLOpenStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLOPENSTREAM *) IMG_OFFSET_ADDR(psTLOpenStreamOUT_UI8, 0);
+
+	IMG_CHAR *uiNameInt = NULL;
+	TL_STREAM_DESC *psSDInt = NULL;
+	PMR *psTLPMRInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) + 0;
+
+	psTLOpenStreamOUT->hSD = NULL;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLOpenStreamOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLOpenStream_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLOpenStreamIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLOpenStreamIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLOpenStreamOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLOpenStream_exit;
+			}
+		}
+	}
+
+	{
+		uiNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNameInt, (const void __user *)psTLOpenStreamIN->puiName,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLOpenStreamOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLOpenStream_exit;
+		}
+		((IMG_CHAR *) uiNameInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) - 1] =
+		    '\0';
+	}
+
+	psTLOpenStreamOUT->eError =
+	    TLServerOpenStreamKM(uiNameInt, psTLOpenStreamIN->ui32Mode, &psSDInt, &psTLPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		goto TLOpenStream_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							      &psTLOpenStreamOUT->hSD,
+							      (void *)psSDInt,
+							      PVRSRV_HANDLE_TYPE_PVR_TL_SD,
+							      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+							      (PFN_HANDLE_RELEASE) &
+							      _TLOpenStreampsSDIntRelease);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	psTLOpenStreamOUT->eError = PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+								 &psTLOpenStreamOUT->hTLPMR,
+								 (void *)psTLPMRInt,
+								 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 psTLOpenStreamOUT->hSD);
+	if (unlikely(psTLOpenStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLOpenStream_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLOpenStream_exit:
+
+	if (psTLOpenStreamOUT->eError != PVRSRV_OK)
+	{
+		if (psTLOpenStreamOUT->hSD)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE) psTLOpenStreamOUT->hSD,
+							     PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSDInt)
+		{
+			TLServerCloseStreamKM(psSDInt);
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLOpenStreamOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCloseStream(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLCloseStreamIN_UI8,
+			  IMG_UINT8 * psTLCloseStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCLOSESTREAM *psTLCloseStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *psTLCloseStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCLOSESTREAM *) IMG_OFFSET_ADDR(psTLCloseStreamOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psTLCloseStreamOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psTLCloseStreamIN->hSD,
+					      PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	if (unlikely((psTLCloseStreamOUT->eError != PVRSRV_OK) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psTLCloseStreamOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psTLCloseStreamOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCloseStream_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+TLCloseStream_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLAcquireData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLAcquireDataIN_UI8,
+			  IMG_UINT8 * psTLAcquireDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLACQUIREDATA *psTLAcquireDataIN =
+	    (PVRSRV_BRIDGE_IN_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLACQUIREDATA *psTLAcquireDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLACQUIREDATA *) IMG_OFFSET_ADDR(psTLAcquireDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLAcquireDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLAcquireDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLAcquireDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLAcquireData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLAcquireDataOUT->eError =
+	    TLServerAcquireDataKM(psSDInt,
+				  &psTLAcquireDataOUT->ui32ReadOffset,
+				  &psTLAcquireDataOUT->ui32ReadLen);
+
+TLAcquireData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReleaseData(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psTLReleaseDataIN_UI8,
+			  IMG_UINT8 * psTLReleaseDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRELEASEDATA *psTLReleaseDataIN =
+	    (PVRSRV_BRIDGE_IN_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRELEASEDATA *psTLReleaseDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRELEASEDATA *) IMG_OFFSET_ADDR(psTLReleaseDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReleaseDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReleaseDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReleaseDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReleaseData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReleaseDataOUT->eError =
+	    TLServerReleaseDataKM(psSDInt,
+				  psTLReleaseDataIN->ui32ReadOffset,
+				  psTLReleaseDataIN->ui32ReadLen);
+
+TLReleaseData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PRVSRVTL_MAX_STREAM_NAME_SIZE <= IMG_UINT32_MAX,
+	      "PRVSRVTL_MAX_STREAM_NAME_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLDiscoverStreams(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psTLDiscoverStreamsIN_UI8,
+			      IMG_UINT8 * psTLDiscoverStreamsOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *psTLDiscoverStreamsIN =
+	    (PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *psTLDiscoverStreamsOUT =
+	    (PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS *) IMG_OFFSET_ADDR(psTLDiscoverStreamsOUT_UI8, 0);
+
+	IMG_CHAR *uiNamePatternInt = NULL;
+	IMG_CHAR *puiStreamsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) + 0;
+
+	if (psTLDiscoverStreamsIN->ui32Size > PVRSRVTL_MAX_DISCOVERABLE_STREAMS_BUFFER)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLDiscoverStreams_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psTLDiscoverStreamsOUT->puiStreams = psTLDiscoverStreamsIN->puiStreams;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLDiscoverStreams_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLDiscoverStreamsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLDiscoverStreamsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLDiscoverStreams_exit;
+			}
+		}
+	}
+
+	{
+		uiNamePatternInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiNamePatternInt,
+		     (const void __user *)psTLDiscoverStreamsIN->puiNamePattern,
+		     PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+		((IMG_CHAR *) uiNamePatternInt)[(PRVSRVTL_MAX_STREAM_NAME_SIZE * sizeof(IMG_CHAR)) -
+						1] = '\0';
+	}
+	if (psTLDiscoverStreamsIN->ui32Size != 0)
+	{
+		puiStreamsInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR);
+	}
+
+	psTLDiscoverStreamsOUT->eError =
+	    TLServerDiscoverStreamsKM(uiNamePatternInt,
+				      psTLDiscoverStreamsIN->ui32Size,
+				      puiStreamsInt, &psTLDiscoverStreamsOUT->ui32NumFound);
+	/* Exit early if bridged call fails */
+	if (unlikely(psTLDiscoverStreamsOUT->eError != PVRSRV_OK))
+	{
+		goto TLDiscoverStreams_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((puiStreamsInt) && ((psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psTLDiscoverStreamsOUT->puiStreams, puiStreamsInt,
+		      (psTLDiscoverStreamsIN->ui32Size * sizeof(IMG_CHAR))) != PVRSRV_OK))
+		{
+			psTLDiscoverStreamsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLDiscoverStreams_exit;
+		}
+	}
+
+TLDiscoverStreams_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLDiscoverStreamsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLReserveStream(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psTLReserveStreamIN_UI8,
+			    IMG_UINT8 * psTLReserveStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLRESERVESTREAM *psTLReserveStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *psTLReserveStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLRESERVESTREAM *) IMG_OFFSET_ADDR(psTLReserveStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLReserveStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLReserveStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLReserveStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLReserveStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLReserveStreamOUT->eError =
+	    TLServerReserveStreamKM(psSDInt,
+				    &psTLReserveStreamOUT->ui32BufferOffset,
+				    psTLReserveStreamIN->ui32Size,
+				    psTLReserveStreamIN->ui32SizeMin,
+				    &psTLReserveStreamOUT->ui32Available);
+
+TLReserveStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeTLCommitStream(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psTLCommitStreamIN_UI8,
+			   IMG_UINT8 * psTLCommitStreamOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *psTLCommitStreamIN =
+	    (PVRSRV_BRIDGE_IN_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *psTLCommitStreamOUT =
+	    (PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM *) IMG_OFFSET_ADDR(psTLCommitStreamOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLCommitStreamIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLCommitStreamOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLCommitStreamOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLCommitStream_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLCommitStreamOUT->eError =
+	    TLServerCommitStreamKM(psSDInt, psTLCommitStreamIN->ui32ReqSize);
+
+TLCommitStream_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRVTL_MAX_PACKET_SIZE <= IMG_UINT32_MAX,
+	      "PVRSRVTL_MAX_PACKET_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeTLWriteData(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psTLWriteDataIN_UI8,
+			IMG_UINT8 * psTLWriteDataOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_TLWRITEDATA *psTLWriteDataIN =
+	    (PVRSRV_BRIDGE_IN_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_TLWRITEDATA *psTLWriteDataOUT =
+	    (PVRSRV_BRIDGE_OUT_TLWRITEDATA *) IMG_OFFSET_ADDR(psTLWriteDataOUT_UI8, 0);
+
+	IMG_HANDLE hSD = psTLWriteDataIN->hSD;
+	TL_STREAM_DESC *psSDInt = NULL;
+	IMG_BYTE *ui8DataInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psTLWriteDataIN->ui32Size > PVRSRVTL_MAX_PACKET_SIZE))
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto TLWriteData_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psTLWriteDataOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto TLWriteData_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psTLWriteDataIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psTLWriteDataIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psTLWriteDataOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto TLWriteData_exit;
+			}
+		}
+	}
+
+	if (psTLWriteDataIN->ui32Size != 0)
+	{
+		ui8DataInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DataInt, (const void __user *)psTLWriteDataIN->pui8Data,
+		     psTLWriteDataIN->ui32Size * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psTLWriteDataOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto TLWriteData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psTLWriteDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSDInt,
+				       hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD, IMG_TRUE);
+	if (unlikely(psTLWriteDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto TLWriteData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psTLWriteDataOUT->eError =
+	    TLServerWriteDataKM(psSDInt, psTLWriteDataIN->ui32Size, ui8DataInt);
+
+TLWriteData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSDInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSD, PVRSRV_HANDLE_TYPE_PVR_TL_SD);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psTLWriteDataOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitPVRTLBridge(void);
+void DeinitPVRTLBridge(void);
+
+/*
+ * Register all PVRTL functions with services
+ */
+PVRSRV_ERROR InitPVRTLBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM,
+			      PVRSRVBridgeTLOpenStream, NULL, sizeof(PVRSRV_BRIDGE_IN_TLOPENSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLOPENSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM,
+			      PVRSRVBridgeTLCloseStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCLOSESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCLOSESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA,
+			      PVRSRVBridgeTLAcquireData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLACQUIREDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLACQUIREDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA,
+			      PVRSRVBridgeTLReleaseData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRELEASEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRELEASEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS,
+			      PVRSRVBridgeTLDiscoverStreams, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLDISCOVERSTREAMS),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLDISCOVERSTREAMS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM,
+			      PVRSRVBridgeTLReserveStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLRESERVESTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLRESERVESTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM,
+			      PVRSRVBridgeTLCommitStream, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_TLCOMMITSTREAM),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLCOMMITSTREAM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA,
+			      PVRSRVBridgeTLWriteData, NULL, sizeof(PVRSRV_BRIDGE_IN_TLWRITEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_TLWRITEDATA));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all pvrtl functions with services
+ */
+void DeinitPVRTLBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLOPENSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCLOSESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLACQUIREDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRELEASEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLDISCOVERSTREAMS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLRESERVESTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLCOMMITSTREAM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_PVRTL, PVRSRV_BRIDGE_PVRTL_TLWRITEDATA);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_rgxbreakpoint_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxbreakpoint_bridge.c
new file mode 100644
index 000000000000..78fd2b405e92
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxbreakpoint_bridge.c
@@ -0,0 +1,381 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxbreakpoint
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxbreakpoint
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxbreakpoint.h"
+
+#include "common_rgxbreakpoint_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXBREAKPOINT_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSetBreakpointIN_UI8,
+			     IMG_UINT8 * psRGXSetBreakpointOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT *psRGXSetBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT *) IMG_OFFSET_ADDR(psRGXSetBreakpointIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT *psRGXSetBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT *) IMG_OFFSET_ADDR(psRGXSetBreakpointOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXSetBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXSetBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetBreakpointOUT->eError =
+	    PVRSRVRGXSetBreakpointKM(psConnection, OSGetDevNode(psConnection),
+				     hPrivDataInt,
+				     psRGXSetBreakpointIN->eFWDataMaster,
+				     psRGXSetBreakpointIN->ui64TempSpillingAddr,
+				     psRGXSetBreakpointIN->ui32BreakpointAddr,
+				     psRGXSetBreakpointIN->ui32HandlerAddr,
+				     psRGXSetBreakpointIN->ui32DM);
+
+RGXSetBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXClearBreakpointIN_UI8,
+			       IMG_UINT8 * psRGXClearBreakpointOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT *psRGXClearBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT *) IMG_OFFSET_ADDR(psRGXClearBreakpointIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT *psRGXClearBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT *) IMG_OFFSET_ADDR(psRGXClearBreakpointOUT_UI8,
+								     0);
+
+	IMG_HANDLE hPrivData = psRGXClearBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXClearBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXClearBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXClearBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXClearBreakpointOUT->eError =
+	    PVRSRVRGXClearBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXClearBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXEnableBreakpointIN_UI8,
+				IMG_UINT8 * psRGXEnableBreakpointOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT *psRGXEnableBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXEnableBreakpointIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT *psRGXEnableBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXEnableBreakpointOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPrivData = psRGXEnableBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXEnableBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXEnableBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXEnableBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXEnableBreakpointOUT->eError =
+	    PVRSRVRGXEnableBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXEnableBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableBreakpoint(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXDisableBreakpointIN_UI8,
+				 IMG_UINT8 * psRGXDisableBreakpointOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT *psRGXDisableBreakpointIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT *) IMG_OFFSET_ADDR(psRGXDisableBreakpointIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT *psRGXDisableBreakpointOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT *)
+	    IMG_OFFSET_ADDR(psRGXDisableBreakpointOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXDisableBreakpointIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXDisableBreakpointOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXDisableBreakpointOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDisableBreakpoint_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXDisableBreakpointOUT->eError =
+	    PVRSRVRGXDisableBreakpointKM(psConnection, OSGetDevNode(psConnection), hPrivDataInt);
+
+RGXDisableBreakpoint_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXOverallocateBPRegisters(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXOverallocateBPRegistersIN_UI8,
+				       IMG_UINT8 * psRGXOverallocateBPRegistersOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS *psRGXOverallocateBPRegistersIN =
+	    (PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS *)
+	    IMG_OFFSET_ADDR(psRGXOverallocateBPRegistersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS *psRGXOverallocateBPRegistersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS *)
+	    IMG_OFFSET_ADDR(psRGXOverallocateBPRegistersOUT_UI8, 0);
+
+	psRGXOverallocateBPRegistersOUT->eError =
+	    PVRSRVRGXOverallocateBPRegistersKM(psConnection, OSGetDevNode(psConnection),
+					       psRGXOverallocateBPRegistersIN->ui32TempRegs,
+					       psRGXOverallocateBPRegistersIN->ui32SharedRegs);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
+
+#if !defined(EXCLUDE_RGXBREAKPOINT_BRIDGE)
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void);
+void DeinitRGXBREAKPOINTBridge(void);
+
+/*
+ * Register all RGXBREAKPOINT functions with services
+ */
+PVRSRV_ERROR InitRGXBREAKPOINTBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT,
+			      PVRSRVBridgeRGXSetBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT,
+			      PVRSRVBridgeRGXClearBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCLEARBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT,
+			      PVRSRVBridgeRGXEnableBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXENABLEBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT,
+			      PVRSRVBridgeRGXDisableBreakpoint, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDISABLEBREAKPOINT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEBREAKPOINT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+			      PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS,
+			      PVRSRVBridgeRGXOverallocateBPRegisters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXOVERALLOCATEBPREGISTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXOVERALLOCATEBPREGISTERS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxbreakpoint functions with services
+ */
+void DeinitRGXBREAKPOINTBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXSETBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXCLEARBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXENABLEBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXDISABLEBREAKPOINT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXBREAKPOINT,
+				PVRSRV_BRIDGE_RGXBREAKPOINT_RGXOVERALLOCATEBPREGISTERS);
+
+}
+#else /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXBREAKPOINT_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXBREAKPOINTBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXBREAKPOINTBridge()
+
+#endif /* EXCLUDE_RGXBREAKPOINT_BRIDGE */
diff --git a/drivers/gpu/drm/img-rogue/server_rgxcmp_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxcmp_bridge.c
new file mode 100644
index 000000000000..8910cfb5ba6a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxcmp_bridge.c
@@ -0,0 +1,1580 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxcmp
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxcmp
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxcompute.h"
+
+#include "common_rgxcmp_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateComputeContextpsComputeContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyComputeContextKM((RGX_SERVER_COMPUTE_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_COMPUTECONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_COMPUTECONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXCreateComputeContextIN_UI8,
+				    IMG_UINT8 * psRGXCreateComputeContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *psRGXCreateComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateComputeContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateComputeContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticComputeContextStateInt = NULL;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateComputeContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize >
+	     RGXFWIF_STATIC_COMPUTECONTEXT_SIZE))
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateComputeContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateComputeContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateComputeContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateComputeContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateComputeContextIN->pui8FrameworkCmd,
+		     psRGXCreateComputeContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize != 0)
+	{
+		ui8StaticComputeContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		    sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticComputeContextStateInt,
+		     (const void __user *)psRGXCreateComputeContextIN->
+		     pui8StaticComputeContextState,
+		     psRGXCreateComputeContextIN->ui32StaticComputeContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateComputeContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError =
+	    PVRSRVRGXCreateComputeContextKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXCreateComputeContextIN->i32Priority,
+					    psRGXCreateComputeContextIN->ui32FrameworkCmdSize,
+					    ui8FrameworkCmdInt,
+					    hPrivDataInt,
+					    psRGXCreateComputeContextIN->
+					    ui32StaticComputeContextStateSize,
+					    ui8StaticComputeContextStateInt,
+					    psRGXCreateComputeContextIN->ui32PackedCCBSizeU88,
+					    psRGXCreateComputeContextIN->ui32ContextFlags,
+					    psRGXCreateComputeContextIN->ui64RobustnessAddress,
+					    psRGXCreateComputeContextIN->ui32MaxDeadlineMS,
+					    &psComputeContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateComputeContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psRGXCreateComputeContextOUT->
+									 hComputeContext,
+									 (void *)
+									 psComputeContextInt,
+									 PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _RGXCreateComputeContextpsComputeContextIntRelease);
+	if (unlikely(psRGXCreateComputeContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateComputeContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateComputeContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateComputeContextOUT->eError != PVRSRV_OK)
+	{
+		if (psComputeContextInt)
+		{
+			PVRSRVRGXDestroyComputeContextKM(psComputeContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateComputeContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyComputeContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXDestroyComputeContextIN_UI8,
+				     IMG_UINT8 * psRGXDestroyComputeContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *psRGXDestroyComputeContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyComputeContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXDestroyComputeContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXDestroyComputeContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyComputeContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyComputeContextIN->
+					      hComputeContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyComputeContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyComputeContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyComputeContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyComputeContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyComputeContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFlushComputeData(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXFlushComputeDataIN_UI8,
+				IMG_UINT8 * psRGXFlushComputeDataOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataIN =
+	    (PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *psRGXFlushComputeDataOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA *) IMG_OFFSET_ADDR(psRGXFlushComputeDataOUT_UI8,
+								      0);
+
+	IMG_HANDLE hComputeContext = psRGXFlushComputeDataIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXFlushComputeDataOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXFlushComputeData_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXFlushComputeDataOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXFlushComputeDataOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXFlushComputeData_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXFlushComputeDataOUT->eError = PVRSRVRGXFlushComputeDataKM(psComputeContextInt);
+
+RGXFlushComputeData_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendCancelCmd(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXSendCancelCmdIN_UI8,
+			     IMG_UINT8 * psRGXSendCancelCmdOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *psRGXSendCancelCmdIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *psRGXSendCancelCmdOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD *) IMG_OFFSET_ADDR(psRGXSendCancelCmdOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSendCancelCmdIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSendCancelCmdOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSendCancelCmd_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendCancelCmdOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendCancelCmd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendCancelCmdOUT->eError =
+	    PVRSRVRGXSendCancelCmdKM(psComputeContextInt,
+				     psRGXSendCancelCmdIN->i32FirstIntJobRefToCancel,
+				     psRGXSendCancelCmdIN->i32LastIntJobRefToCancel);
+
+RGXSendCancelCmd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPriorityOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *psRGXSetComputeContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPriorityIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPriorityOUT->eError =
+	    PVRSRVRGXSetComputeContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						 psComputeContextInt,
+						 psRGXSetComputeContextPriorityIN->i32Priority);
+
+RGXSetComputeContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXNotifyComputeWriteOffsetUpdateIN_UI8,
+					      IMG_UINT8 *
+					      psRGXNotifyComputeWriteOffsetUpdateOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *psRGXNotifyComputeWriteOffsetUpdateOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXNotifyComputeWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXNotifyComputeWriteOffsetUpdateIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXNotifyComputeWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXNotifyComputeWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXNotifyComputeWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXNotifyComputeWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXNotifyComputeWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXNotifyComputeWriteOffsetUpdateKM(psComputeContextInt);
+
+RGXNotifyComputeWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetComputeContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyIN_UI8,
+					 IMG_UINT8 * psRGXSetComputeContextPropertyOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *psRGXSetComputeContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetComputeContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXSetComputeContextPropertyIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXSetComputeContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXSetComputeContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetComputeContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetComputeContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetComputeContextPropertyOUT->eError =
+	    PVRSRVRGXSetComputeContextPropertyKM(psComputeContextInt,
+						 psRGXSetComputeContextPropertyIN->ui32Property,
+						 psRGXSetComputeContextPropertyIN->ui64Input,
+						 &psRGXSetComputeContextPropertyOUT->ui64Output);
+
+RGXSetComputeContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetLastDeviceError(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorIN_UI8,
+				  IMG_UINT8 * psRGXGetLastDeviceErrorOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *psRGXGetLastDeviceErrorOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR *)
+	    IMG_OFFSET_ADDR(psRGXGetLastDeviceErrorOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXGetLastDeviceErrorOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXGetLastDeviceError_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetLastDeviceErrorIN);
+
+	psRGXGetLastDeviceErrorOUT->eError =
+	    PVRSRVRGXGetLastDeviceErrorKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetLastDeviceErrorOUT->ui32Error);
+
+RGXGetLastDeviceError_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTimestampQuery(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXKickTimestampQueryIN_UI8,
+				  IMG_UINT8 * psRGXKickTimestampQueryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *psRGXKickTimestampQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY *)
+	    IMG_OFFSET_ADDR(psRGXKickTimestampQueryOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickTimestampQueryIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXKickTimestampQueryIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTimestampQuery_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTimestampQueryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTimestampQueryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTimestampQuery_exit;
+			}
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTimestampQueryIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickTimestampQueryIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickTimestampQueryIN->pui8DMCmd,
+		     psRGXKickTimestampQueryIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTimestampQueryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTimestampQuery_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTimestampQueryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTimestampQuery_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTimestampQueryOUT->eError =
+	    PVRSRVRGXKickTimestampQueryKM(psComputeContextInt,
+					  psRGXKickTimestampQueryIN->hCheckFenceFd,
+					  psRGXKickTimestampQueryIN->hUpdateTimeline,
+					  &psRGXKickTimestampQueryOUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXKickTimestampQueryIN->ui32CmdSize,
+					  ui8DMCmdInt, psRGXKickTimestampQueryIN->ui32ExtJobRef);
+
+RGXKickTimestampQuery_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTimestampQueryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickCDM(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRGXKickCDMIN_UI8,
+		       IMG_UINT8 * psRGXKickCDMOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKCDM *psRGXKickCDMIN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKCDM *psRGXKickCDMOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKCDM *) IMG_OFFSET_ADDR(psRGXKickCDMOUT_UI8, 0);
+
+	IMG_HANDLE hComputeContext = psRGXKickCDMIN->hComputeContext;
+	RGX_SERVER_COMPUTE_CONTEXT *psComputeContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientUpdateOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_BYTE *ui8DMCmdInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickCDMIN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32CmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	if (unlikely(psRGXKickCDMIN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickCDMOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickCDM_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickCDMIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickCDMIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickCDMOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		psClientUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXKickCDMIN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickCDMIN->phClientUpdateUFOSyncPrimBlock,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateOffsetInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateOffset,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32ClientUpdateCount != 0)
+	{
+		ui32ClientUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateValueInt,
+		     (const void __user *)psRGXKickCDMIN->pui32ClientUpdateValue,
+		     psRGXKickCDMIN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickCDMIN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXKickCDMIN->ui32CmdSize != 0)
+	{
+		ui8DMCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8DMCmdInt, (const void __user *)psRGXKickCDMIN->pui8DMCmd,
+		     psRGXKickCDMIN->ui32CmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickCDMIN->pui32SyncPMRFlags,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+	if (psRGXKickCDMIN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0, psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickCDMIN->phSyncPMRs,
+		     psRGXKickCDMIN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickCDMOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickCDM_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickCDMOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psComputeContextInt,
+				       hComputeContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickCDM_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientUpdateUFOSyncPrimBlockInt[i],
+						       hClientUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickCDMOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickCDMOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickCDM_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickCDMOUT->eError =
+	    PVRSRVRGXKickCDMKM(psComputeContextInt,
+			       psRGXKickCDMIN->ui32ClientUpdateCount,
+			       psClientUpdateUFOSyncPrimBlockInt,
+			       ui32ClientUpdateOffsetInt,
+			       ui32ClientUpdateValueInt,
+			       psRGXKickCDMIN->hCheckFenceFd,
+			       psRGXKickCDMIN->hUpdateTimeline,
+			       &psRGXKickCDMOUT->hUpdateFence,
+			       uiUpdateFenceNameInt,
+			       psRGXKickCDMIN->hExportFenceToSignal,
+			       psRGXKickCDMIN->ui32CmdSize,
+			       ui8DMCmdInt,
+			       psRGXKickCDMIN->ui32PDumpFlags,
+			       psRGXKickCDMIN->ui32ExtJobRef,
+			       psRGXKickCDMIN->ui32SyncPMRCount,
+			       ui32SyncPMRFlagsInt,
+			       psSyncPMRsInt,
+			       psRGXKickCDMIN->ui32NumOfWorkgroups,
+			       psRGXKickCDMIN->ui32NumOfWorkitems,
+			       psRGXKickCDMIN->ui64DeadlineInus, &psRGXKickCDMOUT->ui32IntJobRef);
+
+RGXKickCDM_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psComputeContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hComputeContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_COMPUTE_CONTEXT);
+	}
+
+	if (hClientUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientUpdateUFOSyncPrimBlockInt
+			    && psClientUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickCDMIN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickCDMOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXCDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXCDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *psRGXCDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXCDMGetSharedMemoryIN);
+
+	psRGXCDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXCDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXCDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXCDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMGetSharedMemory_exit:
+
+	if (psRGXCDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXCDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXCDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *psRGXCDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXCDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_COMPUTE_BIT_MASK))
+		{
+			psRGXCDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXCDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXCDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXCDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXCDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXCMPBridge(void);
+void DeinitRGXCMPBridge(void);
+
+/*
+ * Register all RGXCMP functions with services
+ */
+PVRSRV_ERROR InitRGXCMPBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT,
+			      PVRSRVBridgeRGXCreateComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATECOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATECOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT,
+			      PVRSRVBridgeRGXDestroyComputeContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYCOMPUTECONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYCOMPUTECONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA,
+			      PVRSRVBridgeRGXFlushComputeData, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFLUSHCOMPUTEDATA),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFLUSHCOMPUTEDATA));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD,
+			      PVRSRVBridgeRGXSendCancelCmd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDCANCELCMD),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDCANCELCMD));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetComputeContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXNotifyComputeWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+			      PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetComputeContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETCOMPUTECONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETCOMPUTECONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR,
+			      PVRSRVBridgeRGXGetLastDeviceError, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETLASTDEVICEERROR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY,
+			      PVRSRVBridgeRGXKickTimestampQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXKICKTIMESTAMPQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTIMESTAMPQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM,
+			      PVRSRVBridgeRGXKickCDM, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKCDM),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKCDM));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXCDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCDMRELEASESHAREDMEMORY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxcmp functions with services
+ */
+void DeinitRGXCMPBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCREATECOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXDESTROYCOMPUTECONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXFLUSHCOMPUTEDATA);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXSENDCANCELCMD);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXNOTIFYCOMPUTEWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXSETCOMPUTECONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXGETLASTDEVICEERROR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKTIMESTAMPQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXKICKCDM);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP, PVRSRV_BRIDGE_RGXCMP_RGXCDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXCMP,
+				PVRSRV_BRIDGE_RGXCMP_RGXCDMRELEASESHAREDMEMORY);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_rgxfwdbg_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxfwdbg_bridge.c
new file mode 100644
index 000000000000..e5646490aa2d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxfwdbg_bridge.c
@@ -0,0 +1,542 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxfwdbg
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxfwdbg
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "devicemem_server.h"
+#include "rgxfwdbg.h"
+#include "pmr.h"
+#include "rgxtimecorr.h"
+
+#include "common_rgxfwdbg_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetFWLog(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugSetFWLogOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *psRGXFWDebugSetFWLogOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG *) IMG_OFFSET_ADDR(psRGXFWDebugSetFWLogOUT_UI8,
+								     0);
+
+	psRGXFWDebugSetFWLogOUT->eError =
+	    PVRSRVRGXFWDebugSetFWLogKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXFWDebugSetFWLogIN->ui32RGXFWLogType);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugDumpFreelistPageList(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListIN_UI8,
+					   IMG_UINT8 * psRGXFWDebugDumpFreelistPageListOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *psRGXFWDebugDumpFreelistPageListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugDumpFreelistPageListOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugDumpFreelistPageListIN);
+
+	psRGXFWDebugDumpFreelistPageListOUT->eError =
+	    PVRSRVRGXFWDebugDumpFreelistPageListKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOff(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXFWDebugPowerOffIN_UI8,
+			       IMG_UINT8 * psRGXFWDebugPowerOffOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *psRGXFWDebugPowerOffOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOffOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOffIN);
+
+	psRGXFWDebugPowerOffOUT->eError =
+	    PVRSRVRGXFWDebugPowerOffKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPowerOn(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXFWDebugPowerOnIN_UI8,
+			      IMG_UINT8 * psRGXFWDebugPowerOnOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *psRGXFWDebugPowerOnOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON *) IMG_OFFSET_ADDR(psRGXFWDebugPowerOnOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXFWDebugPowerOnIN);
+
+	psRGXFWDebugPowerOnOUT->eError =
+	    PVRSRVRGXFWDebugPowerOnKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec(IMG_UINT32 ui32DispatchTableEntry,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8,
+							 IMG_UINT8 *
+							 psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8,
+							 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC
+	    *psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT_UI8, 0);
+
+	psRGXFWDebugSetVzConnectionCooldownPeriodInSecOUT->eError =
+	    PVRSRVRGXFWDebugSetVzConnectionCooldownPeriodInSecKM(psConnection,
+								 OSGetDevNode(psConnection),
+								 psRGXFWDebugSetVzConnectionCooldownPeriodInSecIN->
+								 ui32VzConne);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetHCSDeadline(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineIN_UI8,
+				     IMG_UINT8 * psRGXFWDebugSetHCSDeadlineOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *psRGXFWDebugSetHCSDeadlineOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetHCSDeadlineOUT_UI8, 0);
+
+	psRGXFWDebugSetHCSDeadlineOUT->eError =
+	    PVRSRVRGXFWDebugSetHCSDeadlineKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXFWDebugSetHCSDeadlineIN->ui32RGXHCSDeadline);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityIN_UI8,
+					IMG_UINT8 * psRGXFWDebugSetDriverPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *psRGXFWDebugSetDriverPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverPriorityOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverPriorityOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRGXFWDebugSetDriverPriorityIN->ui32DriverID,
+						psRGXFWDebugSetDriverPriorityIN->ui32Priority);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSlice(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceIN_UI8,
+					 IMG_UINT8 * psRGXFWDebugSetDriverTimeSliceOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *psRGXFWDebugSetDriverTimeSliceOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceKM(psConnection, OSGetDevNode(psConnection),
+						 psRGXFWDebugSetDriverTimeSliceIN->ui32DriverID,
+						 psRGXFWDebugSetDriverTimeSliceIN->
+						 ui32TSPercentage);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval(IMG_UINT32 ui32DispatchTableEntry,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8,
+						 IMG_UINT8 *
+						 psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8,
+						 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL
+	    *psRGXFWDebugSetDriverTimeSliceIntervalOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverTimeSliceIntervalOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverTimeSliceIntervalOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(psConnection, OSGetDevNode(psConnection),
+							 psRGXFWDebugSetDriverTimeSliceIntervalIN->
+							 ui32TSIntervalMs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXFWDebugSetDriverIsolationGroupIN_UI8,
+					      IMG_UINT8 *
+					      psRGXFWDebugSetDriverIsolationGroupOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *psRGXFWDebugSetDriverIsolationGroupOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetDriverIsolationGroupOUT_UI8, 0);
+
+	psRGXFWDebugSetDriverIsolationGroupOUT->eError =
+	    PVRSRVRGXFWDebugSetDriverIsolationGroupKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32DriverID,
+						      psRGXFWDebugSetDriverIsolationGroupIN->
+						      ui32IsolationGroup);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugSetOSNewOnlineState(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateIN_UI8,
+					  IMG_UINT8 * psRGXFWDebugSetOSNewOnlineStateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *psRGXFWDebugSetOSNewOnlineStateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugSetOSNewOnlineStateOUT_UI8, 0);
+
+	psRGXFWDebugSetOSNewOnlineStateOUT->eError =
+	    PVRSRVRGXFWDebugSetOSNewOnlineStateKM(psConnection, OSGetDevNode(psConnection),
+						  psRGXFWDebugSetOSNewOnlineStateIN->ui32DriverID,
+						  psRGXFWDebugSetOSNewOnlineStateIN->
+						  ui32OSNewState);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugMapGuestHeap(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugMapGuestHeapOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *psRGXFWDebugMapGuestHeapOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugMapGuestHeapOUT_UI8, 0);
+
+	psRGXFWDebugMapGuestHeapOUT->eError =
+	    PVRSRVRGXFWDebugMapGuestHeapKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugMapGuestHeapIN->ui32DriverID,
+					   psRGXFWDebugMapGuestHeapIN->ui64ui64GuestHeapBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugPHRConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugPHRConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *psRGXFWDebugPHRConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugPHRConfigureOUT_UI8, 0);
+
+	psRGXFWDebugPHRConfigureOUT->eError =
+	    PVRSRVRGXFWDebugPHRConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugPHRConfigureIN->ui32ui32PHRMode);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXFWDebugWdgConfigure(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureIN_UI8,
+				   IMG_UINT8 * psRGXFWDebugWdgConfigureOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureIN =
+	    (PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *psRGXFWDebugWdgConfigureOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE *)
+	    IMG_OFFSET_ADDR(psRGXFWDebugWdgConfigureOUT_UI8, 0);
+
+	psRGXFWDebugWdgConfigureOUT->eError =
+	    PVRSRVRGXFWDebugWdgConfigureKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXFWDebugWdgConfigureIN->ui32ui32WdgPeriodUs);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCurrentTime(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psRGXCurrentTimeIN_UI8,
+			   IMG_UINT8 * psRGXCurrentTimeOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCURRENTTIME *psRGXCurrentTimeIN =
+	    (PVRSRV_BRIDGE_IN_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *psRGXCurrentTimeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCURRENTTIME *) IMG_OFFSET_ADDR(psRGXCurrentTimeOUT_UI8, 0);
+
+	psRGXCurrentTimeOUT->eError =
+	    PVRSRVRGXCurrentTime(psConnection, OSGetDevNode(psConnection),
+				 psRGXCurrentTimeIN->ui8TimerType, &psRGXCurrentTimeOUT->ui64Time);
+
+	return 0;
+}
+
+#define PVRSRVBridgeRGXFWDebugInjectFault NULL
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXFWDBGBridge(void);
+void DeinitRGXFWDBGBridge(void);
+
+/*
+ * Register all RGXFWDBG functions with services
+ */
+PVRSRV_ERROR InitRGXFWDBGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG,
+			      PVRSRVBridgeRGXFWDebugSetFWLog, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETFWLOG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETFWLOG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST,
+			      PVRSRVBridgeRGXFWDebugDumpFreelistPageList, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGDUMPFREELISTPAGELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF,
+			      PVRSRVBridgeRGXFWDebugPowerOff, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWEROFF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON,
+			      PVRSRVBridgeRGXFWDebugPowerOn, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPOWERON));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC,
+			      PVRSRVBridgeRGXFWDebugSetVzConnectionCooldownPeriodInSec, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC),
+			      sizeof
+			      (PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE,
+			      PVRSRVBridgeRGXFWDebugSetHCSDeadline, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETHCSDEADLINE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETHCSDEADLINE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY,
+			      PVRSRVBridgeRGXFWDebugSetDriverPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSlice, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL,
+			      PVRSRVBridgeRGXFWDebugSetDriverTimeSliceInterval, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP,
+			      PVRSRVBridgeRGXFWDebugSetDriverIsolationGroup, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETDRIVERISOLATIONGROUP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETDRIVERISOLATIONGROUP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+			      PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE,
+			      PVRSRVBridgeRGXFWDebugSetOSNewOnlineState, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGSETOSNEWONLINESTATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGSETOSNEWONLINESTATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP,
+			      PVRSRVBridgeRGXFWDebugMapGuestHeap, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGMAPGUESTHEAP),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGMAPGUESTHEAP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugPHRConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGPHRCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGPHRCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE,
+			      PVRSRVBridgeRGXFWDebugWdgConfigure, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXFWDEBUGWDGCONFIGURE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGWDGCONFIGURE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME,
+			      PVRSRVBridgeRGXCurrentTime, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCURRENTTIME),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCURRENTTIME));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT,
+			      PVRSRVBridgeRGXFWDebugInjectFault, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXFWDEBUGINJECTFAULT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxfwdbg functions with services
+ */
+void DeinitRGXFWDBGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETFWLOG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGDUMPFREELISTPAGELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWEROFF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPOWERON);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETVZCONNECTIONCOOLDOWNPERIODINSEC);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETHCSDEADLINE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERTIMESLICEINTERVAL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETDRIVERISOLATIONGROUP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGSETOSNEWONLINESTATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGMAPGUESTHEAP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGPHRCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGWDGCONFIGURE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG, PVRSRV_BRIDGE_RGXFWDBG_RGXCURRENTTIME);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXFWDBG,
+				PVRSRV_BRIDGE_RGXFWDBG_RGXFWDEBUGINJECTFAULT);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_rgxhwperf_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxhwperf_bridge.c
new file mode 100644
index 000000000000..ea76a5ef2fb1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxhwperf_bridge.c
@@ -0,0 +1,1034 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxhwperf
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxhwperf
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxhwperf.h"
+#include "rgx_fwif_km.h"
+
+#include "common_rgxhwperf_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersIN_UI8,
+					   IMG_UINT8 * psRGXGetConfiguredHWPerfCountersOUT_UI8,
+					   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *psRGXGetConfiguredHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psConfiguredCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters =
+	    psRGXGetConfiguredHWPerfCountersIN->psConfiguredCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredCountersInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+						   psRGXGetConfiguredHWPerfCountersIN->ui32BlockID,
+						   psConfiguredCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfCountersOUT->psConfiguredCounters,
+		      psConfiguredCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetEnabledHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksIN_UI8,
+				      IMG_UINT8 * psRGXGetEnabledHWPerfBlocksOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *psRGXGetEnabledHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXGetEnabledHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT32 *pui32EnabledBlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) + 0;
+
+	psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs =
+	    psRGXGetEnabledHWPerfBlocksIN->pui32EnabledBlockIDs;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetEnabledHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXGetEnabledHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetEnabledHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		pui32EnabledBlockIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32);
+	}
+
+	psRGXGetEnabledHWPerfBlocksOUT->eError =
+	    PVRSRVRGXGetEnabledHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					      psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen,
+					      &psRGXGetEnabledHWPerfBlocksOUT->ui32BlockCount,
+					      pui32EnabledBlockIDsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetEnabledHWPerfBlocksOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetEnabledHWPerfBlocks_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui32EnabledBlockIDsInt) &&
+	    ((psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXGetEnabledHWPerfBlocksOUT->pui32EnabledBlockIDs,
+		      pui32EnabledBlockIDsInt,
+		      (psRGXGetEnabledHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT32))) !=
+		     PVRSRV_OK))
+		{
+			psRGXGetEnabledHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetEnabledHWPerfBlocks_exit;
+		}
+	}
+
+RGXGetEnabledHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetEnabledHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+#if defined(PVRSRV_FORCE_HWPERF_TO_SCHED_CLK)
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfTimeStamp(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampIN_UI8,
+				  IMG_UINT8 * psRGXGetHWPerfTimeStampOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *psRGXGetHWPerfTimeStampOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfTimeStampOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfTimeStampIN);
+
+	psRGXGetHWPerfTimeStampOUT->eError =
+	    PVRSRVRGXGetHWPerfTimeStampKM(psConnection, OSGetDevNode(psConnection),
+					  &psRGXGetHWPerfTimeStampOUT->ui64TimeStamp);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeRGXGetHWPerfTimeStamp NULL
+#endif
+
+static IMG_INT
+PVRSRVBridgeRGXCtrlHWPerf(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXCtrlHWPerfIN_UI8,
+			  IMG_UINT8 * psRGXCtrlHWPerfOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *psRGXCtrlHWPerfIN =
+	    (PVRSRV_BRIDGE_IN_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *psRGXCtrlHWPerfOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF *) IMG_OFFSET_ADDR(psRGXCtrlHWPerfOUT_UI8, 0);
+
+	psRGXCtrlHWPerfOUT->eError =
+	    PVRSRVRGXCtrlHWPerfKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXCtrlHWPerfIN->ui32StreamId,
+				  psRGXCtrlHWPerfIN->bToggle, psRGXCtrlHWPerfIN->ui64Mask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsIN_UI8,
+					 IMG_UINT8 * psRGXGetHWPerfBvncFeatureFlagsOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *psRGXGetHWPerfBvncFeatureFlagsOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS *)
+	    IMG_OFFSET_ADDR(psRGXGetHWPerfBvncFeatureFlagsOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXGetHWPerfBvncFeatureFlagsIN);
+
+	psRGXGetHWPerfBvncFeatureFlagsOUT->eError =
+	    PVRSRVRGXGetHWPerfBvncFeatureFlagsKM(psConnection, OSGetDevNode(psConnection),
+						 &psRGXGetHWPerfBvncFeatureFlagsOUT->sBVNC);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXControlHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksIN_UI8,
+				   IMG_UINT8 * psRGXControlHWPerfBlocksOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *psRGXControlHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXControlHWPerfBlocksOUT_UI8, 0);
+
+	IMG_UINT16 *ui16BlockIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) + 0;
+
+	if (unlikely(psRGXControlHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXControlHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXControlHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXControlHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXControlHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		ui16BlockIDsInt = (IMG_UINT16 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16);
+	}
+
+	/* Copy the data over */
+	if (psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui16BlockIDsInt,
+		     (const void __user *)psRGXControlHWPerfBlocksIN->pui16BlockIDs,
+		     psRGXControlHWPerfBlocksIN->ui32ArrayLen * sizeof(IMG_UINT16)) != PVRSRV_OK)
+		{
+			psRGXControlHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXControlHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXControlHWPerfBlocksOUT->eError =
+	    PVRSRVRGXControlHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXControlHWPerfBlocksIN->bEnable,
+					   psRGXControlHWPerfBlocksIN->ui32ArrayLen,
+					   ui16BlockIDsInt);
+
+RGXControlHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXControlHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigMuxHWPerfCounters(IMG_UINT32 ui32DispatchTableEntry,
+				       IMG_UINT8 * psRGXConfigMuxHWPerfCountersIN_UI8,
+				       IMG_UINT8 * psRGXConfigMuxHWPerfCountersOUT_UI8,
+				       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS *psRGXConfigMuxHWPerfCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigMuxHWPerfCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS *psRGXConfigMuxHWPerfCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigMuxHWPerfCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigMuxHWPerfCounters_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigMuxHWPerfCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigMuxHWPerfCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXConfigMuxHWPerfCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigMuxHWPerfCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigMuxHWPerfCounters_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_MUX_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								     ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+		    sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigMuxHWPerfCountersIN->psBlockConfigs,
+		     psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigMuxHWPerfCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigMuxHWPerfCounters_exit;
+		}
+	}
+
+	psRGXConfigMuxHWPerfCountersOUT->eError =
+	    PVRSRVRGXConfigMuxHWPerfCountersKM(psConnection, OSGetDevNode(psConnection),
+					       psRGXConfigMuxHWPerfCountersIN->ui32ArrayLen,
+					       psBlockConfigsInt);
+
+RGXConfigMuxHWPerfCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigMuxHWPerfCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGX_HWPERF_MAX_CUSTOM_CNTRS <= IMG_UINT32_MAX,
+	      "RGX_HWPERF_MAX_CUSTOM_CNTRS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigCustomCounters(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXConfigCustomCountersIN_UI8,
+				    IMG_UINT8 * psRGXConfigCustomCountersOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS *psRGXConfigCustomCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigCustomCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS *psRGXConfigCustomCountersOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXConfigCustomCountersOUT_UI8, 0);
+
+	IMG_UINT32 *ui32CustomCounterIDsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32)) +
+	    0;
+
+	if (unlikely
+	    (psRGXConfigCustomCountersIN->ui16NumCustomCounters > RGX_HWPERF_MAX_CUSTOM_CNTRS))
+	{
+		psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigCustomCounters_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigCustomCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigCustomCountersIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigCustomCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigCustomCounters_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigCustomCountersIN->ui16NumCustomCounters != 0)
+	{
+		ui32CustomCounterIDsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CustomCounterIDsInt,
+		     (const void __user *)psRGXConfigCustomCountersIN->pui32CustomCounterIDs,
+		     psRGXConfigCustomCountersIN->ui16NumCustomCounters * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXConfigCustomCountersOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigCustomCounters_exit;
+		}
+	}
+
+	psRGXConfigCustomCountersOUT->eError =
+	    PVRSRVRGXConfigCustomCountersKM(psConnection, OSGetDevNode(psConnection),
+					    psRGXConfigCustomCountersIN->ui16CustomBlockID,
+					    psRGXConfigCustomCountersIN->ui16NumCustomCounters,
+					    ui32CustomCounterIDsInt);
+
+RGXConfigCustomCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigCustomCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(RGXFWIF_HWPERF_CTRL_BLKS_MAX <= IMG_UINT32_MAX,
+	      "RGXFWIF_HWPERF_CTRL_BLKS_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXConfigureHWPerfBlocks(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksIN_UI8,
+				     IMG_UINT8 * psRGXConfigureHWPerfBlocksOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksIN =
+	    (PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *psRGXConfigureHWPerfBlocksOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS *)
+	    IMG_OFFSET_ADDR(psRGXConfigureHWPerfBlocksOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_CNTBLK *psBlockConfigsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+	     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) + 0;
+
+	if (unlikely(psRGXConfigureHWPerfBlocksIN->ui32ArrayLen > RGXFWIF_HWPERF_CTRL_BLKS_MAX))
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXConfigureHWPerfBlocks_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXConfigureHWPerfBlocksIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXConfigureHWPerfBlocksIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXConfigureHWPerfBlocks_exit;
+			}
+		}
+	}
+
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen != 0)
+	{
+		psBlockConfigsInt =
+		    (RGX_HWPERF_CONFIG_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK);
+	}
+
+	/* Copy the data over */
+	if (psRGXConfigureHWPerfBlocksIN->ui32ArrayLen * sizeof(RGX_HWPERF_CONFIG_CNTBLK) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, psBlockConfigsInt,
+		     (const void __user *)psRGXConfigureHWPerfBlocksIN->psBlockConfigs,
+		     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen *
+		     sizeof(RGX_HWPERF_CONFIG_CNTBLK)) != PVRSRV_OK)
+		{
+			psRGXConfigureHWPerfBlocksOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXConfigureHWPerfBlocks_exit;
+		}
+	}
+
+	psRGXConfigureHWPerfBlocksOUT->eError =
+	    PVRSRVRGXConfigureHWPerfBlocksKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXConfigureHWPerfBlocksIN->ui32CtrlWord,
+					     psRGXConfigureHWPerfBlocksIN->ui32ArrayLen,
+					     psBlockConfigsInt);
+
+RGXConfigureHWPerfBlocks_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXConfigureHWPerfBlocksOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static_assert(1 <= IMG_UINT32_MAX, "1 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXGetConfiguredHWPerfMuxCounters(IMG_UINT32 ui32DispatchTableEntry,
+					      IMG_UINT8 * psRGXGetConfiguredHWPerfMuxCountersIN_UI8,
+					      IMG_UINT8 *
+					      psRGXGetConfiguredHWPerfMuxCountersOUT_UI8,
+					      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *psRGXGetConfiguredHWPerfMuxCountersIN =
+	    (PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfMuxCountersIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *psRGXGetConfiguredHWPerfMuxCountersOUT
+	    =
+	    (PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS *)
+	    IMG_OFFSET_ADDR(psRGXGetConfiguredHWPerfMuxCountersOUT_UI8, 0);
+
+	RGX_HWPERF_CONFIG_MUX_CNTBLK *psConfiguredMuxCountersInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize = ((IMG_UINT64) 1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) + 0;
+
+	psRGXGetConfiguredHWPerfMuxCountersOUT->psConfiguredMuxCounters =
+	    psRGXGetConfiguredHWPerfMuxCountersIN->psConfiguredMuxCounters;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+		    PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXGetConfiguredHWPerfMuxCounters_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXGetConfiguredHWPerfMuxCountersIN),
+			      sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXGetConfiguredHWPerfMuxCountersIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXGetConfiguredHWPerfMuxCounters_exit;
+			}
+		}
+	}
+
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psConfiguredMuxCountersInt =
+		    (RGX_HWPERF_CONFIG_MUX_CNTBLK *) IMG_OFFSET_ADDR(pArrayArgsBuffer,
+								     ui32NextOffset);
+		ui32NextOffset += 1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK);
+	}
+
+	psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+	    PVRSRVRGXGetConfiguredHWPerfMuxCountersKM(psConnection, OSGetDevNode(psConnection),
+						      psRGXGetConfiguredHWPerfMuxCountersIN->
+						      ui32BlockID, psConfiguredMuxCountersInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXGetConfiguredHWPerfMuxCountersOUT->eError != PVRSRV_OK))
+	{
+		goto RGXGetConfiguredHWPerfMuxCounters_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((psConfiguredMuxCountersInt) && ((1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL,
+		      (void __user *)psRGXGetConfiguredHWPerfMuxCountersOUT->
+		      psConfiguredMuxCounters, psConfiguredMuxCountersInt,
+		      (1 * sizeof(RGX_HWPERF_CONFIG_MUX_CNTBLK))) != PVRSRV_OK))
+		{
+			psRGXGetConfiguredHWPerfMuxCountersOUT->eError =
+			    PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXGetConfiguredHWPerfMuxCounters_exit;
+		}
+	}
+
+RGXGetConfiguredHWPerfMuxCounters_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXGetConfiguredHWPerfMuxCountersOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXHWPERFBridge(void);
+void DeinitRGXHWPERFBridge(void);
+
+/*
+ * Register all RGXHWPERF functions with services
+ */
+PVRSRV_ERROR InitRGXHWPERFBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS,
+			      PVRSRVBridgeRGXGetEnabledHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETENABLEDHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETENABLEDHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP,
+			      PVRSRVBridgeRGXGetHWPerfTimeStamp, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFTIMESTAMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF,
+			      PVRSRVBridgeRGXCtrlHWPerf, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCTRLHWPERF),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCTRLHWPERF));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS,
+			      PVRSRVBridgeRGXGetHWPerfBvncFeatureFlags, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETHWPERFBVNCFEATUREFLAGS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS,
+			      PVRSRVBridgeRGXControlHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONTROLHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONTROLHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS,
+			      PVRSRVBridgeRGXConfigMuxHWPerfCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGMUXHWPERFCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGMUXHWPERFCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS,
+			      PVRSRVBridgeRGXConfigCustomCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGCUSTOMCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGCUSTOMCOUNTERS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS,
+			      PVRSRVBridgeRGXConfigureHWPerfBlocks, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCONFIGUREHWPERFBLOCKS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCONFIGUREHWPERFBLOCKS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+			      PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS,
+			      PVRSRVBridgeRGXGetConfiguredHWPerfMuxCounters, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXGETCONFIGUREDHWPERFMUXCOUNTERS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXGETCONFIGUREDHWPERFMUXCOUNTERS));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxhwperf functions with services
+ */
+void DeinitRGXHWPERFBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETENABLEDHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFTIMESTAMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF, PVRSRV_BRIDGE_RGXHWPERF_RGXCTRLHWPERF);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETHWPERFBVNCFEATUREFLAGS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONTROLHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGMUXHWPERFCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGCUSTOMCOUNTERS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXCONFIGUREHWPERFBLOCKS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXHWPERF,
+				PVRSRV_BRIDGE_RGXHWPERF_RGXGETCONFIGUREDHWPERFMUXCOUNTERS);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_rgxkicksync_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxkicksync_bridge.c
new file mode 100644
index 000000000000..8d943cd536e2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxkicksync_bridge.c
@@ -0,0 +1,593 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxkicksync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxkicksync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxkicksync.h"
+
+#include "common_rgxkicksync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateKickSyncContextpsKickSyncContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyKickSyncContextKM((RGX_SERVER_KICKSYNC_CONTEXT *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateKickSyncContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateKickSyncContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *psRGXCreateKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateKickSyncContextOUT_UI8, 0);
+
+	IMG_HANDLE hPrivData = psRGXCreateKickSyncContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVRGXCreateKickSyncContextKM(psConnection, OSGetDevNode(psConnection),
+					     hPrivDataInt,
+					     psRGXCreateKickSyncContextIN->ui32PackedCCBSizeU88,
+					     psRGXCreateKickSyncContextIN->ui32ContextFlags,
+					     &psKickSyncContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateKickSyncContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateKickSyncContextOUT->hKickSyncContext,
+				      (void *)psKickSyncContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateKickSyncContextpsKickSyncContextIntRelease);
+	if (unlikely(psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateKickSyncContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateKickSyncContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateKickSyncContextOUT->eError != PVRSRV_OK)
+	{
+		if (psKickSyncContextInt)
+		{
+			PVRSRVRGXDestroyKickSyncContextKM(psKickSyncContextInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyKickSyncContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyKickSyncContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *psRGXDestroyKickSyncContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyKickSyncContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyKickSyncContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyKickSyncContextIN->
+					      hKickSyncContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyKickSyncContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyKickSyncContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyKickSyncContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyKickSyncContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyKickSyncContext_exit:
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_DEV_VARS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_DEV_VARS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickSync2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickSync2IN_UI8,
+			 IMG_UINT8 * psRGXKickSync2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *psRGXKickSync2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *psRGXKickSync2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKSYNC2 *) IMG_OFFSET_ADDR(psRGXKickSync2OUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXKickSync2IN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFODevVarBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFODevVarBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateDevVarOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRGXKickSync2IN->ui32ClientUpdateCount > PVRSRV_MAX_DEV_VARS))
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickSync2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickSync2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickSync2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickSync2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickSync2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickSync2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFODevVarBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFODevVarBlockInt, 0,
+			       psRGXKickSync2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFODevVarBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFODevVarBlockInt2,
+		     (const void __user *)psRGXKickSync2IN->phUpdateUFODevVarBlock,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateDevVarOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateDevVarOffsetInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateDevVarOffset,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+	if (psRGXKickSync2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXKickSync2IN->pui32UpdateValue,
+		     psRGXKickSync2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickSync2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickSync2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickSync2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickSync2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickSync2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickSync2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFODevVarBlockInt[i],
+						       hUpdateUFODevVarBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickSync2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickSync2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickSync2OUT->eError =
+	    PVRSRVRGXKickSyncKM(psKickSyncContextInt,
+				psRGXKickSync2IN->ui32ClientUpdateCount,
+				psUpdateUFODevVarBlockInt,
+				ui32UpdateDevVarOffsetInt,
+				ui32UpdateValueInt,
+				psRGXKickSync2IN->hCheckFenceFD,
+				psRGXKickSync2IN->hTimelineFenceFD,
+				&psRGXKickSync2OUT->hUpdateFenceFD,
+				uiUpdateFenceNameInt, psRGXKickSync2IN->ui32ExtJobRef);
+
+RGXKickSync2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+
+	if (hUpdateUFODevVarBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickSync2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFODevVarBlockInt && psUpdateUFODevVarBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFODevVarBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickSync2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetKickSyncContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetKickSyncContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *psRGXSetKickSyncContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetKickSyncContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hKickSyncContext = psRGXSetKickSyncContextPropertyIN->hKickSyncContext;
+	RGX_SERVER_KICKSYNC_CONTEXT *psKickSyncContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psKickSyncContextInt,
+				       hKickSyncContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetKickSyncContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetKickSyncContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetKickSyncContextPropertyOUT->eError =
+	    PVRSRVRGXSetKickSyncContextPropertyKM(psKickSyncContextInt,
+						  psRGXSetKickSyncContextPropertyIN->ui32Property,
+						  psRGXSetKickSyncContextPropertyIN->ui64Input,
+						  &psRGXSetKickSyncContextPropertyOUT->ui64Output);
+
+RGXSetKickSyncContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psKickSyncContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hKickSyncContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_KICKSYNC_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
+
+#if defined(SUPPORT_RGXKICKSYNC_BRIDGE)
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void);
+void DeinitRGXKICKSYNCBridge(void);
+
+/*
+ * Register all RGXKICKSYNC functions with services
+ */
+PVRSRV_ERROR InitRGXKICKSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXCreateKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT,
+			      PVRSRVBridgeRGXDestroyKickSyncContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYKICKSYNCCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYKICKSYNCCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2,
+			      PVRSRVBridgeRGXKickSync2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKSYNC2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKSYNC2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+			      PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetKickSyncContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETKICKSYNCCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETKICKSYNCCONTEXTPROPERTY));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxkicksync functions with services
+ */
+void DeinitRGXKICKSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXCREATEKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXDESTROYKICKSYNCCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC, PVRSRV_BRIDGE_RGXKICKSYNC_RGXKICKSYNC2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXKICKSYNC,
+				PVRSRV_BRIDGE_RGXKICKSYNC_RGXSETKICKSYNCCONTEXTPROPERTY);
+
+}
+#else /* SUPPORT_RGXKICKSYNC_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXKICKSYNC_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXKICKSYNCBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXKICKSYNCBridge()
+
+#endif /* SUPPORT_RGXKICKSYNC_BRIDGE */
diff --git a/drivers/gpu/drm/img-rogue/server_rgxregconfig_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxregconfig_bridge.c
new file mode 100644
index 000000000000..548401470d09
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxregconfig_bridge.c
@@ -0,0 +1,246 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxregconfig
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxregconfig
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxregconfig.h"
+
+#include "common_rgxregconfig_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXSetRegConfigType(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXSetRegConfigTypeIN_UI8,
+				IMG_UINT8 * psRGXSetRegConfigTypeOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *psRGXSetRegConfigTypeOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE *) IMG_OFFSET_ADDR(psRGXSetRegConfigTypeOUT_UI8,
+								      0);
+
+	psRGXSetRegConfigTypeOUT->eError =
+	    PVRSRVRGXSetRegConfigTypeKM(psConnection, OSGetDevNode(psConnection),
+					psRGXSetRegConfigTypeIN->ui8RegPowerIsland);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXAddRegconfig(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRGXAddRegconfigIN_UI8,
+			    IMG_UINT8 * psRGXAddRegconfigOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *psRGXAddRegconfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *psRGXAddRegconfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG *) IMG_OFFSET_ADDR(psRGXAddRegconfigOUT_UI8, 0);
+
+	psRGXAddRegconfigOUT->eError =
+	    PVRSRVRGXAddRegConfigKM(psConnection, OSGetDevNode(psConnection),
+				    psRGXAddRegconfigIN->ui32RegAddr,
+				    psRGXAddRegconfigIN->ui64RegValue,
+				    psRGXAddRegconfigIN->ui64RegMask);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXClearRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXClearRegConfigIN_UI8,
+			      IMG_UINT8 * psRGXClearRegConfigOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *psRGXClearRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *psRGXClearRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG *) IMG_OFFSET_ADDR(psRGXClearRegConfigOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXClearRegConfigIN);
+
+	psRGXClearRegConfigOUT->eError =
+	    PVRSRVRGXClearRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEnableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXEnableRegConfigIN_UI8,
+			       IMG_UINT8 * psRGXEnableRegConfigOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *psRGXEnableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *psRGXEnableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXEnableRegConfigOUT_UI8,
+								     0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEnableRegConfigIN);
+
+	psRGXEnableRegConfigOUT->eError =
+	    PVRSRVRGXEnableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDisableRegConfig(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXDisableRegConfigIN_UI8,
+				IMG_UINT8 * psRGXDisableRegConfigOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *psRGXDisableRegConfigIN =
+	    (PVRSRV_BRIDGE_IN_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *psRGXDisableRegConfigOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG *) IMG_OFFSET_ADDR(psRGXDisableRegConfigOUT_UI8,
+								      0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXDisableRegConfigIN);
+
+	psRGXDisableRegConfigOUT->eError =
+	    PVRSRVRGXDisableRegConfigKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+
+#if !defined(EXCLUDE_RGXREGCONFIG_BRIDGE)
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void);
+void DeinitRGXREGCONFIGBridge(void);
+
+/*
+ * Register all RGXREGCONFIG functions with services
+ */
+PVRSRV_ERROR InitRGXREGCONFIGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE,
+			      PVRSRVBridgeRGXSetRegConfigType, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETREGCONFIGTYPE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETREGCONFIGTYPE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG,
+			      PVRSRVBridgeRGXAddRegconfig, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXADDREGCONFIG),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXADDREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG,
+			      PVRSRVBridgeRGXClearRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCLEARREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG,
+			      PVRSRVBridgeRGXEnableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENABLEREGCONFIG));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+			      PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG,
+			      PVRSRVBridgeRGXDisableRegConfig, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDISABLEREGCONFIG));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxregconfig functions with services
+ */
+void DeinitRGXREGCONFIGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXSETREGCONFIGTYPE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXADDREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXCLEARREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXENABLEREGCONFIG);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXREGCONFIG,
+				PVRSRV_BRIDGE_RGXREGCONFIG_RGXDISABLEREGCONFIG);
+
+}
+#else /* EXCLUDE_RGXREGCONFIG_BRIDGE */
+/* This bridge is conditional on EXCLUDE_RGXREGCONFIG_BRIDGE - when defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXREGCONFIGBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXREGCONFIGBridge()
+
+#endif /* EXCLUDE_RGXREGCONFIG_BRIDGE */
diff --git a/drivers/gpu/drm/img-rogue/server_rgxta3d_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxta3d_bridge.c
new file mode 100644
index 000000000000..141df870f874
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxta3d_bridge.c
@@ -0,0 +1,2531 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxta3d
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxta3d
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxta3d.h"
+
+#include "common_rgxta3d_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetIN_UI8,
+				  IMG_UINT8 * psRGXDestroyHWRTDataSetOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *psRGXDestroyHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXDestroyHWRTDataSetOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyHWRTDataSetOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyHWRTDataSetIN->
+					      hKmHwRTDataSet,
+					      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+	if (unlikely
+	    ((psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyHWRTDataSetOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyHWRTDataSetOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyHWRTDataSet_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyHWRTDataSet_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateZSBufferpssZSBufferKMIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyZSBufferKM((RGX_ZSBUFFER_DATA *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateZSBufferIN_UI8,
+			      IMG_UINT8 * psRGXCreateZSBufferOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *psRGXCreateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *psRGXCreateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXCreateZSBufferOUT_UI8, 0);
+
+	IMG_HANDLE hReservation = psRGXCreateZSBufferIN->hReservation;
+	DEVMEMINT_RESERVATION *psReservationInt = NULL;
+	IMG_HANDLE hPMR = psRGXCreateZSBufferIN->hPMR;
+	PMR *psPMRInt = NULL;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psReservationInt,
+				       hReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRInt,
+				       hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError =
+	    RGXCreateZSBufferKM(psConnection, OSGetDevNode(psConnection),
+				psReservationInt,
+				psPMRInt, psRGXCreateZSBufferIN->uiMapFlags, &pssZSBufferKMInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateZSBufferOUT->
+								   hsZSBufferKM,
+								   (void *)pssZSBufferKMInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateZSBufferpssZSBufferKMIntRelease);
+	if (unlikely(psRGXCreateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hReservation, PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPMRInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMR, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssZSBufferKMInt)
+		{
+			RGXDestroyZSBufferKM(pssZSBufferKMInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyZSBufferIN_UI8,
+			       IMG_UINT8 * psRGXDestroyZSBufferOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *psRGXDestroyZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER *) IMG_OFFSET_ADDR(psRGXDestroyZSBufferOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyZSBufferIN->
+					      hsZSBufferMemDesc,
+					      PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	if (unlikely
+	    ((psRGXDestroyZSBufferOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyZSBuffer_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXPopulateZSBufferpssPopulationIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXUnpopulateZSBufferKM((RGX_POPULATION *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXPopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRGXPopulateZSBufferIN_UI8,
+				IMG_UINT8 * psRGXPopulateZSBufferOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *psRGXPopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER *) IMG_OFFSET_ADDR(psRGXPopulateZSBufferOUT_UI8,
+								      0);
+
+	IMG_HANDLE hsZSBufferKM = psRGXPopulateZSBufferIN->hsZSBufferKM;
+	RGX_ZSBUFFER_DATA *pssZSBufferKMInt = NULL;
+	RGX_POPULATION *pssPopulationInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXPopulateZSBufferOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pssZSBufferKMInt,
+				       hsZSBufferKM,
+				       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError =
+	    RGXPopulateZSBufferKM(pssZSBufferKMInt, &pssPopulationInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXPopulateZSBufferOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRGXPopulateZSBufferOUT->
+								     hsPopulation,
+								     (void *)pssPopulationInt,
+								     PVRSRV_HANDLE_TYPE_RGX_POPULATION,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RGXPopulateZSBufferpssPopulationIntRelease);
+	if (unlikely(psRGXPopulateZSBufferOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXPopulateZSBuffer_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXPopulateZSBuffer_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pssZSBufferKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hsZSBufferKM, PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXPopulateZSBufferOUT->eError != PVRSRV_OK)
+	{
+		if (pssPopulationInt)
+		{
+			RGXUnpopulateZSBufferKM(pssPopulationInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXUnpopulateZSBuffer(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferIN_UI8,
+				  IMG_UINT8 * psRGXUnpopulateZSBufferOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferIN =
+	    (PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *psRGXUnpopulateZSBufferOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER *)
+	    IMG_OFFSET_ADDR(psRGXUnpopulateZSBufferOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXUnpopulateZSBufferOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXUnpopulateZSBufferIN->hsPopulation,
+					      PVRSRV_HANDLE_TYPE_RGX_POPULATION);
+	if (unlikely((psRGXUnpopulateZSBufferOUT->eError != PVRSRV_OK) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXUnpopulateZSBufferOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXUnpopulateZSBufferOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXUnpopulateZSBuffer_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXUnpopulateZSBuffer_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXDestroyFreeListIN_UI8,
+			       IMG_UINT8 * psRGXDestroyFreeListOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *psRGXDestroyFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *psRGXDestroyFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST *) IMG_OFFSET_ADDR(psRGXDestroyFreeListOUT_UI8,
+								     0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyFreeListOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyFreeListIN->hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+	if (unlikely((psRGXDestroyFreeListOUT->eError != PVRSRV_OK) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXDestroyFreeListOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyFreeListOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyFreeList_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyFreeList_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXDestroyRenderContextIN_UI8,
+				    IMG_UINT8 * psRGXDestroyRenderContextOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *psRGXDestroyRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyRenderContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyRenderContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyRenderContextIN->
+					      hCleanupCookie,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyRenderContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyRenderContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyRenderContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyRenderContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyRenderContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSendZSStoreDisable(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXSendZSStoreDisableIN_UI8,
+				  IMG_UINT8 * psRGXSendZSStoreDisableOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableIN =
+	    (PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *psRGXSendZSStoreDisableOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE *)
+	    IMG_OFFSET_ADDR(psRGXSendZSStoreDisableOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSendZSStoreDisableIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSendZSStoreDisableOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSendZSStoreDisable_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSendZSStoreDisableOUT->eError =
+	    PVRSRVRGXSendZSStoreDisableKM(psConnection, OSGetDevNode(psConnection),
+					  psRenderContextInt,
+					  psRGXSendZSStoreDisableIN->bDisableDepthStore,
+					  psRGXSendZSStoreDisableIN->bDisableStencilStore,
+					  psRGXSendZSStoreDisableIN->i32ExtJobRefToDisableZSStore);
+
+RGXSendZSStoreDisable_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPriorityIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPriorityOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *psRGXSetRenderContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPriorityIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPriorityOUT->eError =
+	    PVRSRVRGXSetRenderContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						psRenderContextInt,
+						psRGXSetRenderContextPriorityIN->i32Priority);
+
+RGXSetRenderContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXRenderContextStalled(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psRGXRenderContextStalledIN_UI8,
+				    IMG_UINT8 * psRGXRenderContextStalledOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledIN =
+	    (PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *psRGXRenderContextStalledOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED *)
+	    IMG_OFFSET_ADDR(psRGXRenderContextStalledOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXRenderContextStalledIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXRenderContextStalledOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXRenderContextStalledOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXRenderContextStalled_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXRenderContextStalledOUT->eError = RGXRenderContextStalledKM(psRenderContextInt);
+
+RGXRenderContextStalled_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXKickTA3D2(IMG_UINT32 ui32DispatchTableEntry,
+			 IMG_UINT8 * psRGXKickTA3D2IN_UI8,
+			 IMG_UINT8 * psRGXKickTA3D2OUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *psRGXKickTA3D2IN =
+	    (PVRSRV_BRIDGE_IN_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *psRGXKickTA3D2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXKICKTA3D2 *) IMG_OFFSET_ADDR(psRGXKickTA3D2OUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXKickTA3D2IN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAFenceSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAFenceSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAFenceSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAFenceValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClientTAUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClientTAUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32ClientTAUpdateValueInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psClient3DUpdateSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hClient3DUpdateSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32Client3DUpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32Client3DUpdateValueInt = NULL;
+	IMG_HANDLE hPRFenceUFOSyncPrimBlock = psRGXKickTA3D2IN->hPRFenceUFOSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *psPRFenceUFOSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_CHAR *uiUpdateFenceName3DInt = NULL;
+	IMG_BYTE *ui8TACmdInt = NULL;
+	IMG_BYTE *ui83DPRCmdInt = NULL;
+	IMG_BYTE *ui83DCmdInt = NULL;
+	IMG_HANDLE hKMHWRTDataSet = psRGXKickTA3D2IN->hKMHWRTDataSet;
+	RGX_KM_HW_RT_DATASET *psKMHWRTDataSetInt = NULL;
+	IMG_HANDLE hZSBuffer = psRGXKickTA3D2IN->hZSBuffer;
+	RGX_ZSBUFFER_DATA *psZSBufferInt = NULL;
+	IMG_HANDLE hMSAAScratchBuffer = psRGXKickTA3D2IN->hMSAAScratchBuffer;
+	RGX_ZSBUFFER_DATA *psMSAAScratchBufferInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAFenceCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32ClientTAUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32Client3DUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32TACmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DPRCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui323DCmdSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (unlikely(psRGXKickTA3D2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXKickTA3D2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXKickTA3D2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXKickTA3D2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		psClientTAFenceSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAFenceSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAFenceCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAFenceSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAFenceSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAFenceSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount != 0)
+	{
+		ui32ClientTAFenceValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAFenceValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAFenceValue,
+		     psRGXKickTA3D2IN->ui32ClientTAFenceCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		psClientTAUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClientTAUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32ClientTAUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClientTAUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClientTAUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClientTAUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount != 0)
+	{
+		ui32ClientTAUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientTAUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32ClientTAUpdateValue,
+		     psRGXKickTA3D2IN->ui32ClientTAUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		psClient3DUpdateSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psClient3DUpdateSyncPrimBlockInt, 0,
+			       psRGXKickTA3D2IN->ui32Client3DUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hClient3DUpdateSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hClient3DUpdateSyncPrimBlockInt2,
+		     (const void __user *)psRGXKickTA3D2IN->phClient3DUpdateSyncPrimBlock,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateSyncOffsetInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateSyncOffset,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount != 0)
+	{
+		ui32Client3DUpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32Client3DUpdateValueInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32Client3DUpdateValue,
+		     psRGXKickTA3D2IN->ui32Client3DUpdateCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+
+	{
+		uiUpdateFenceName3DInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceName3DInt,
+		     (const void __user *)psRGXKickTA3D2IN->puiUpdateFenceName3D,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceName3DInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						      1] = '\0';
+	}
+	if (psRGXKickTA3D2IN->ui32TACmdSize != 0)
+	{
+		ui8TACmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8TACmdInt, (const void __user *)psRGXKickTA3D2IN->pui8TACmd,
+		     psRGXKickTA3D2IN->ui32TACmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize != 0)
+	{
+		ui83DPRCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DPRCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DPRCmd,
+		     psRGXKickTA3D2IN->ui323DPRCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui323DCmdSize != 0)
+	{
+		ui83DCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui83DCmdInt, (const void __user *)psRGXKickTA3D2IN->pui83DCmd,
+		     psRGXKickTA3D2IN->ui323DCmdSize * sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXKickTA3D2IN->pui32SyncPMRFlags,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXKickTA3D2IN->phSyncPMRs,
+		     psRGXKickTA3D2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXKickTA3D2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psClientTAFenceSyncPrimBlockInt[i],
+						       hClientTAFenceSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClientTAUpdateSyncPrimBlockInt[i],
+						       hClientTAUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)
+						       &psClient3DUpdateSyncPrimBlockInt[i],
+						       hClient3DUpdateSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPRFenceUFOSyncPrimBlockInt,
+				       hPRFenceUFOSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXKickTA3D2_exit;
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psKMHWRTDataSetInt,
+					       hKMHWRTDataSet,
+					       PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psZSBufferInt,
+					       hZSBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+		/* Look up the address from the handle */
+		psRGXKickTA3D2OUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&psMSAAScratchBufferInt,
+					       hMSAAScratchBuffer,
+					       PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER, IMG_TRUE);
+		if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXKickTA3D2_exit;
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXKickTA3D2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXKickTA3D2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXKickTA3D2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXKickTA3D2OUT->eError =
+	    PVRSRVRGXKickTA3DKM(psRenderContextInt,
+				psRGXKickTA3D2IN->ui32ClientTAFenceCount,
+				psClientTAFenceSyncPrimBlockInt,
+				ui32ClientTAFenceSyncOffsetInt,
+				ui32ClientTAFenceValueInt,
+				psRGXKickTA3D2IN->ui32ClientTAUpdateCount,
+				psClientTAUpdateSyncPrimBlockInt,
+				ui32ClientTAUpdateSyncOffsetInt,
+				ui32ClientTAUpdateValueInt,
+				psRGXKickTA3D2IN->ui32Client3DUpdateCount,
+				psClient3DUpdateSyncPrimBlockInt,
+				ui32Client3DUpdateSyncOffsetInt,
+				ui32Client3DUpdateValueInt,
+				psPRFenceUFOSyncPrimBlockInt,
+				psRGXKickTA3D2IN->ui32PRFenceUFOSyncOffset,
+				psRGXKickTA3D2IN->ui32PRFenceValue,
+				psRGXKickTA3D2IN->hCheckFence,
+				psRGXKickTA3D2IN->hUpdateTimeline,
+				&psRGXKickTA3D2OUT->hUpdateFence,
+				uiUpdateFenceNameInt,
+				psRGXKickTA3D2IN->hCheckFence3D,
+				psRGXKickTA3D2IN->hUpdateTimeline3D,
+				&psRGXKickTA3D2OUT->hUpdateFence3D,
+				uiUpdateFenceName3DInt,
+				psRGXKickTA3D2IN->ui32TACmdSize,
+				ui8TACmdInt,
+				psRGXKickTA3D2IN->ui323DPRCmdSize,
+				ui83DPRCmdInt,
+				psRGXKickTA3D2IN->ui323DCmdSize,
+				ui83DCmdInt,
+				psRGXKickTA3D2IN->ui32ExtJobRef,
+				psRGXKickTA3D2IN->bbKickTA,
+				psRGXKickTA3D2IN->bbKickPR,
+				psRGXKickTA3D2IN->bbKick3D,
+				psRGXKickTA3D2IN->bbAbort,
+				psRGXKickTA3D2IN->ui32PDumpFlags,
+				psKMHWRTDataSetInt,
+				psZSBufferInt,
+				psMSAAScratchBufferInt,
+				psRGXKickTA3D2IN->ui32SyncPMRCount,
+				ui32SyncPMRFlagsInt,
+				psSyncPMRsInt,
+				psRGXKickTA3D2IN->ui32RenderTargetSize,
+				psRGXKickTA3D2IN->ui32NumberOfDrawCalls,
+				psRGXKickTA3D2IN->ui32NumberOfIndices,
+				psRGXKickTA3D2IN->ui32NumberOfMRTs, psRGXKickTA3D2IN->ui64Deadline);
+
+RGXKickTA3D2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+
+	if (hClientTAFenceSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAFenceCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAFenceSyncPrimBlockInt && psClientTAFenceSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAFenceSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClientTAUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32ClientTAUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClientTAUpdateSyncPrimBlockInt && psClientTAUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClientTAUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hClient3DUpdateSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32Client3DUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psClient3DUpdateSyncPrimBlockInt && psClient3DUpdateSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hClient3DUpdateSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psPRFenceUFOSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPRFenceUFOSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+
+	if (psRGXKickTA3D2IN->hKMHWRTDataSet)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psKMHWRTDataSetInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hKMHWRTDataSet,
+						    PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hZSBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psZSBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hZSBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (psRGXKickTA3D2IN->hMSAAScratchBuffer)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (psMSAAScratchBufferInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hMSAAScratchBuffer,
+						    PVRSRV_HANDLE_TYPE_RGX_FWIF_ZSBUFFER);
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXKickTA3D2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXKickTA3D2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetRenderContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXSetRenderContextPropertyIN_UI8,
+					IMG_UINT8 * psRGXSetRenderContextPropertyOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *psRGXSetRenderContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetRenderContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hRenderContext = psRGXSetRenderContextPropertyIN->hRenderContext;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRenderContextInt,
+				       hRenderContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetRenderContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetRenderContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetRenderContextPropertyOUT->eError =
+	    PVRSRVRGXSetRenderContextPropertyKM(psRenderContextInt,
+						psRGXSetRenderContextPropertyIN->ui32Property,
+						psRGXSetRenderContextPropertyIN->ui64Input,
+						&psRGXSetRenderContextPropertyOUT->ui64Output);
+
+RGXSetRenderContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRenderContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRenderContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyHWRTDataSet((RGX_KM_HW_RT_DATASET *) pvData);
+	return eError;
+}
+
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATA_FREELISTS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATA_FREELISTS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_GEOMDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_GEOMDATAS must not be larger than IMG_UINT32_MAX");
+static_assert(RGXMKIF_NUM_RTDATAS <= IMG_UINT32_MAX,
+	      "RGXMKIF_NUM_RTDATAS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateHWRTDataSet(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetIN_UI8,
+				 IMG_UINT8 * psRGXCreateHWRTDataSetOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET *) IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *psRGXCreateHWRTDataSetOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET *)
+	    IMG_OFFSET_ADDR(psRGXCreateHWRTDataSetOUT_UI8, 0);
+
+	IMG_DEV_VIRTADDR *sVHeapTableDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sPMMlistDevVAddrInt = NULL;
+	RGX_FREELIST **psapsFreeListsInt = NULL;
+	IMG_HANDLE *hapsFreeListsInt2 = NULL;
+	IMG_DEV_VIRTADDR *sTailPtrsDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sMacrotileArrayDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRgnHeaderDevVAddrInt = NULL;
+	IMG_DEV_VIRTADDR *sRTCDevVAddrInt = NULL;
+	RGX_KM_HW_RT_DATASET **psKmHwRTDataSetInt = NULL;
+	IMG_HANDLE *hKmHwRTDataSetInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) +
+	    ((IMG_UINT64) RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE)) + 0;
+
+	psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet = psRGXCreateHWRTDataSetIN->phKmHwRTDataSet;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateHWRTDataSetIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateHWRTDataSetIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+
+	{
+		sVHeapTableDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sVHeapTableDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psVHeapTableDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sPMMlistDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sPMMlistDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psPMMlistDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		psapsFreeListsInt =
+		    (RGX_FREELIST **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psapsFreeListsInt, 0,
+			       RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(RGX_FREELIST *);
+		hapsFreeListsInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hapsFreeListsInt2,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->phapsFreeLists,
+		     RGXMKIF_NUM_RTDATA_FREELISTS * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sTailPtrsDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sTailPtrsDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psTailPtrsDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sMacrotileArrayDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sMacrotileArrayDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psMacrotileArrayDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sRgnHeaderDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRgnHeaderDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psRgnHeaderDevVAddr,
+		     RGXMKIF_NUM_RTDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+	{
+		sRTCDevVAddrInt =
+		    (IMG_DEV_VIRTADDR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR);
+	}
+
+	/* Copy the data over */
+	if (RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, sRTCDevVAddrInt,
+		     (const void __user *)psRGXCreateHWRTDataSetIN->psRTCDevVAddr,
+		     RGXMKIF_NUM_GEOMDATAS * sizeof(IMG_DEV_VIRTADDR)) != PVRSRV_OK)
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+	if (IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset) != NULL)
+	{
+		psKmHwRTDataSetInt =
+		    (RGX_KM_HW_RT_DATASET **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psKmHwRTDataSetInt, 0,
+			       RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *));
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *);
+		hKmHwRTDataSetInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += RGXMKIF_NUM_RTDATAS * sizeof(IMG_HANDLE);
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psapsFreeListsInt[i],
+						       hapsFreeListsInt2[i],
+						       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateHWRTDataSetOUT->eError =
+	    RGXCreateHWRTDataSet(psConnection, OSGetDevNode(psConnection),
+				 sVHeapTableDevVAddrInt,
+				 sPMMlistDevVAddrInt,
+				 psapsFreeListsInt,
+				 psRGXCreateHWRTDataSetIN->ui32PPPScreen,
+				 psRGXCreateHWRTDataSetIN->ui64MultiSampleCtl,
+				 psRGXCreateHWRTDataSetIN->ui64FlippedMultiSampleCtl,
+				 psRGXCreateHWRTDataSetIN->ui32TPCStride,
+				 sTailPtrsDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui32TPCSize,
+				 psRGXCreateHWRTDataSetIN->ui32TEScreen,
+				 psRGXCreateHWRTDataSetIN->ui32TEAA,
+				 psRGXCreateHWRTDataSetIN->ui32TEMTILE1,
+				 psRGXCreateHWRTDataSetIN->ui32TEMTILE2,
+				 psRGXCreateHWRTDataSetIN->ui32MTileStride,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeLowerX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeLowerY,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeUpperX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeUpperY,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeScaleX,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMergeScaleY,
+				 sMacrotileArrayDevVAddrInt,
+				 sRgnHeaderDevVAddrInt,
+				 sRTCDevVAddrInt,
+				 psRGXCreateHWRTDataSetIN->ui32RgnHeaderSize,
+				 psRGXCreateHWRTDataSetIN->ui32ISPMtileSize,
+				 psRGXCreateHWRTDataSetIN->ui16MaxRTs, psKmHwRTDataSetInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateHWRTDataSet_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+	if (hKmHwRTDataSetInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+		{
+
+			psRGXCreateHWRTDataSetOUT->eError =
+			    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+						      &hKmHwRTDataSetInt2[i],
+						      (void *)psKmHwRTDataSetInt[i],
+						      PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET,
+						      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+						      (PFN_HANDLE_RELEASE) &
+						      _RGXCreateHWRTDataSetpsKmHwRTDataSetIntRelease);
+			if (unlikely(psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK))
+			{
+				IMG_UINT32 j;
+				/* Ensure the remaining handles are set to NULL. hKmHwRTDataSetInt2[i] was
+				 * zeroed when calling PVRSRVAllocHandleUnlocked, so we start at the next
+				 * element. If it was the last iteration, the loop doesn't run.
+				 */
+				for (j = i + 1; j < RGXMKIF_NUM_RTDATAS; j++)
+				{
+					hKmHwRTDataSetInt2[j] = NULL;
+				}
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXCreateHWRTDataSet_exit;
+			}
+
+		}
+	}
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((hKmHwRTDataSetInt2) && ((RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psRGXCreateHWRTDataSetOUT->phKmHwRTDataSet,
+		      hKmHwRTDataSetInt2,
+		      (RGXMKIF_NUM_RTDATAS * sizeof(RGX_KM_HW_RT_DATASET *))) != PVRSRV_OK))
+		{
+			psRGXCreateHWRTDataSetOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateHWRTDataSet_exit;
+		}
+	}
+
+RGXCreateHWRTDataSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	if (hapsFreeListsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < RGXMKIF_NUM_RTDATA_FREELISTS; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psapsFreeListsInt && psapsFreeListsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hapsFreeListsInt2[i],
+							    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateHWRTDataSetOUT->eError != PVRSRV_OK)
+	{
+		if (hKmHwRTDataSetInt2)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			{
+				IMG_UINT32 idx;
+				for (idx = 0; idx < RGXMKIF_NUM_RTDATAS; idx++)
+				{
+					if (hKmHwRTDataSetInt2[idx])
+					{
+
+						eError =
+						    PVRSRVDestroyHandleUnlocked(psConnection->
+										psHandleBase,
+										hKmHwRTDataSetInt2
+										[idx],
+										PVRSRV_HANDLE_TYPE_RGX_KM_HW_RT_DATASET);
+						if (unlikely
+						    ((eError != PVRSRV_OK)
+						     && (eError != PVRSRV_ERROR_RETRY)))
+						{
+							PVR_DPF((PVR_DBG_ERROR,
+								 "%s: %s",
+								 __func__,
+								 PVRSRVGetErrorString(eError)));
+						}
+						/* Releasing the handle should free/destroy/release the resource.
+						 * This should never fail... */
+						PVR_ASSERT((eError == PVRSRV_OK)
+							   || (eError == PVRSRV_ERROR_RETRY));
+
+					}
+					else if (psKmHwRTDataSetInt[idx])
+					{
+						/* Free/Destroy/Release the resource */
+						RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[idx]);
+					}
+				}
+			}
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psKmHwRTDataSetInt)
+		{
+			IMG_UINT32 i;
+			for (i = 0; i < RGXMKIF_NUM_RTDATAS; i++)
+			{
+				if (psKmHwRTDataSetInt[i])
+				{
+					RGXDestroyHWRTDataSet(psKmHwRTDataSetInt[i]);
+				}
+			}
+		}
+
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateHWRTDataSetOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateFreeListpsCleanupCookieIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RGXDestroyFreeList((RGX_FREELIST *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXCreateFreeList(IMG_UINT32 ui32DispatchTableEntry,
+			      IMG_UINT8 * psRGXCreateFreeListIN_UI8,
+			      IMG_UINT8 * psRGXCreateFreeListOUT_UI8,
+			      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *psRGXCreateFreeListIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *psRGXCreateFreeListOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST *) IMG_OFFSET_ADDR(psRGXCreateFreeListOUT_UI8, 0);
+
+	IMG_HANDLE hMemCtxPrivData = psRGXCreateFreeListIN->hMemCtxPrivData;
+	IMG_HANDLE hMemCtxPrivDataInt = NULL;
+	IMG_HANDLE hsGlobalFreeList = psRGXCreateFreeListIN->hsGlobalFreeList;
+	RGX_FREELIST *pssGlobalFreeListInt = NULL;
+	IMG_HANDLE hFreeListReservation = psRGXCreateFreeListIN->hFreeListReservation;
+	DEVMEMINT_RESERVATION *psFreeListReservationInt = NULL;
+	RGX_FREELIST *psCleanupCookieInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hMemCtxPrivDataInt,
+				       hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+		/* Look up the address from the handle */
+		psRGXCreateFreeListOUT->eError =
+		    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+					       (void **)&pssGlobalFreeListInt,
+					       hsGlobalFreeList,
+					       PVRSRV_HANDLE_TYPE_RGX_FREELIST, IMG_TRUE);
+		if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+		{
+			UnlockHandle(psConnection->psHandleBase);
+			goto RGXCreateFreeList_exit;
+		}
+	}
+
+	/* Look up the address from the handle */
+	psRGXCreateFreeListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psFreeListReservationInt,
+				       hFreeListReservation,
+				       PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION, IMG_TRUE);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError =
+	    RGXCreateFreeList(psConnection, OSGetDevNode(psConnection),
+			      hMemCtxPrivDataInt,
+			      psRGXCreateFreeListIN->ui32MaxFLPages,
+			      psRGXCreateFreeListIN->ui32InitFLPages,
+			      psRGXCreateFreeListIN->ui32GrowFLPages,
+			      psRGXCreateFreeListIN->ui32GrowParamThreshold,
+			      pssGlobalFreeListInt,
+			      psRGXCreateFreeListIN->bbFreeListCheck,
+			      psFreeListReservationInt, &psCleanupCookieInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateFreeListOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								   &psRGXCreateFreeListOUT->
+								   hCleanupCookie,
+								   (void *)psCleanupCookieInt,
+								   PVRSRV_HANDLE_TYPE_RGX_FREELIST,
+								   PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								   (PFN_HANDLE_RELEASE) &
+								   _RGXCreateFreeListpsCleanupCookieIntRelease);
+	if (unlikely(psRGXCreateFreeListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateFreeList_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateFreeList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hMemCtxPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hMemCtxPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+
+	if (psRGXCreateFreeListIN->hsGlobalFreeList)
+	{
+
+		/* Unreference the previously looked up handle */
+		if (pssGlobalFreeListInt)
+		{
+			PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+						    hsGlobalFreeList,
+						    PVRSRV_HANDLE_TYPE_RGX_FREELIST);
+		}
+	}
+
+	/* Unreference the previously looked up handle */
+	if (psFreeListReservationInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hFreeListReservation,
+					    PVRSRV_HANDLE_TYPE_DEVMEMINT_RESERVATION);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateFreeListOUT->eError != PVRSRV_OK)
+	{
+		if (psCleanupCookieInt)
+		{
+			RGXDestroyFreeList(psCleanupCookieInt);
+		}
+	}
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXCreateRenderContextpsRenderContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyRenderContextKM((RGX_SERVER_RENDER_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_STATIC_RENDERCONTEXT_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_STATIC_RENDERCONTEXT_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateRenderContext(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psRGXCreateRenderContextIN_UI8,
+				   IMG_UINT8 * psRGXCreateRenderContextOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *psRGXCreateRenderContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateRenderContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateRenderContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	IMG_BYTE *ui8StaticRenderContextStateInt = NULL;
+	RGX_SERVER_RENDER_CONTEXT *psRenderContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) +
+	    ((IMG_UINT64) psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateRenderContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (unlikely
+	    (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize >
+	     RGXFWIF_STATIC_RENDERCONTEXT_SIZE))
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateRenderContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateRenderContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateRenderContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateRenderContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8FrameworkCmd,
+		     psRGXCreateRenderContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize != 0)
+	{
+		ui8StaticRenderContextStateInt =
+		    (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8StaticRenderContextStateInt,
+		     (const void __user *)psRGXCreateRenderContextIN->pui8StaticRenderContextState,
+		     psRGXCreateRenderContextIN->ui32StaticRenderContextStateSize *
+		     sizeof(IMG_BYTE)) != PVRSRV_OK)
+		{
+			psRGXCreateRenderContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateRenderContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError =
+	    PVRSRVRGXCreateRenderContextKM(psConnection, OSGetDevNode(psConnection),
+					   psRGXCreateRenderContextIN->i32Priority,
+					   psRGXCreateRenderContextIN->sVDMCallStackAddr,
+					   psRGXCreateRenderContextIN->ui32ui32CallStackDepth,
+					   psRGXCreateRenderContextIN->ui32FrameworkCmdSize,
+					   ui8FrameworkCmdInt,
+					   hPrivDataInt,
+					   psRGXCreateRenderContextIN->
+					   ui32StaticRenderContextStateSize,
+					   ui8StaticRenderContextStateInt,
+					   psRGXCreateRenderContextIN->ui32PackedCCBSizeU8888,
+					   psRGXCreateRenderContextIN->ui32ContextFlags,
+					   psRGXCreateRenderContextIN->ui64RobustnessAddress,
+					   psRGXCreateRenderContextIN->ui32MaxTADeadlineMS,
+					   psRGXCreateRenderContextIN->ui32Max3DDeadlineMS,
+					   &psRenderContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateRenderContextOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									&psRGXCreateRenderContextOUT->
+									hRenderContext,
+									(void *)psRenderContextInt,
+									PVRSRV_HANDLE_TYPE_RGX_SERVER_RENDER_CONTEXT,
+									PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									(PFN_HANDLE_RELEASE) &
+									_RGXCreateRenderContextpsRenderContextIntRelease);
+	if (unlikely(psRGXCreateRenderContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateRenderContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateRenderContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateRenderContextOUT->eError != PVRSRV_OK)
+	{
+		if (psRenderContextInt)
+		{
+			PVRSRVRGXDestroyRenderContextKM(psRenderContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateRenderContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTA3DBridge(void);
+void DeinitRGXTA3DBridge(void);
+
+/*
+ * Register all RGXTA3D functions with services
+ */
+PVRSRV_ERROR InitRGXTA3DBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET,
+			      PVRSRVBridgeRGXDestroyHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER,
+			      PVRSRVBridgeRGXCreateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER,
+			      PVRSRVBridgeRGXDestroyZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXPopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER,
+			      PVRSRVBridgeRGXUnpopulateZSBuffer, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXUNPOPULATEZSBUFFER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXUNPOPULATEZSBUFFER));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST,
+			      PVRSRVBridgeRGXDestroyFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT,
+			      PVRSRVBridgeRGXDestroyRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYRENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYRENDERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE,
+			      PVRSRVBridgeRGXSendZSStoreDisable, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSENDZSSTOREDISABLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSENDZSSTOREDISABLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetRenderContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED,
+			      PVRSRVBridgeRGXRenderContextStalled, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXRENDERCONTEXTSTALLED),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXRENDERCONTEXTSTALLED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2,
+			      PVRSRVBridgeRGXKickTA3D2, NULL, sizeof(PVRSRV_BRIDGE_IN_RGXKICKTA3D2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXKICKTA3D2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+			      PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetRenderContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETRENDERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETRENDERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET,
+			      PVRSRVBridgeRGXCreateHWRTDataSet, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEHWRTDATASET),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEHWRTDATASET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST,
+			      PVRSRVBridgeRGXCreateFreeList, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATEFREELIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATEFREELIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT,
+			      PVRSRVBridgeRGXCreateRenderContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATERENDERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATERENDERCONTEXT));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxta3d functions with services
+ */
+void DeinitRGXTA3DBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXUNPOPULATEZSBUFFER);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXDESTROYRENDERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXSENDZSSTOREDISABLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXRENDERCONTEXTSTALLED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXKICKTA3D2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXSETRENDERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEHWRTDATASET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D, PVRSRV_BRIDGE_RGXTA3D_RGXCREATEFREELIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTA3D,
+				PVRSRV_BRIDGE_RGXTA3D_RGXCREATERENDERCONTEXT);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_rgxtimerquery_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxtimerquery_bridge.c
new file mode 100644
index 000000000000..4af3cfc35cba
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxtimerquery_bridge.c
@@ -0,0 +1,171 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtimerquery
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtimerquery
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtimerquery.h"
+
+#include "common_rgxtimerquery_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRGXBeginTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXBeginTimerQueryIN_UI8,
+			       IMG_UINT8 * psRGXBeginTimerQueryOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *psRGXBeginTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY *) IMG_OFFSET_ADDR(psRGXBeginTimerQueryOUT_UI8,
+								     0);
+
+	psRGXBeginTimerQueryOUT->eError =
+	    PVRSRVRGXBeginTimerQueryKM(psConnection, OSGetDevNode(psConnection),
+				       psRGXBeginTimerQueryIN->ui32QueryId);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXEndTimerQuery(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psRGXEndTimerQueryIN_UI8,
+			     IMG_UINT8 * psRGXEndTimerQueryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *psRGXEndTimerQueryIN =
+	    (PVRSRV_BRIDGE_IN_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *psRGXEndTimerQueryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY *) IMG_OFFSET_ADDR(psRGXEndTimerQueryOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psRGXEndTimerQueryIN);
+
+	psRGXEndTimerQueryOUT->eError =
+	    PVRSRVRGXEndTimerQueryKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXQueryTimer(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRGXQueryTimerIN_UI8,
+			  IMG_UINT8 * psRGXQueryTimerOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXQUERYTIMER *psRGXQueryTimerIN =
+	    (PVRSRV_BRIDGE_IN_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *psRGXQueryTimerOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXQUERYTIMER *) IMG_OFFSET_ADDR(psRGXQueryTimerOUT_UI8, 0);
+
+	psRGXQueryTimerOUT->eError =
+	    PVRSRVRGXQueryTimerKM(psConnection, OSGetDevNode(psConnection),
+				  psRGXQueryTimerIN->ui32QueryId,
+				  &psRGXQueryTimerOUT->ui64StartTime,
+				  &psRGXQueryTimerOUT->ui64EndTime);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void);
+void DeinitRGXTIMERQUERYBridge(void);
+
+/*
+ * Register all RGXTIMERQUERY functions with services
+ */
+PVRSRV_ERROR InitRGXTIMERQUERYBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY,
+			      PVRSRVBridgeRGXBeginTimerQuery, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXBEGINTIMERQUERY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXBEGINTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY,
+			      PVRSRVBridgeRGXEndTimerQuery, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXENDTIMERQUERY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+			      PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER, PVRSRVBridgeRGXQueryTimer,
+			      NULL, sizeof(PVRSRV_BRIDGE_IN_RGXQUERYTIMER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXQUERYTIMER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtimerquery functions with services
+ */
+void DeinitRGXTIMERQUERYBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXBEGINTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXENDTIMERQUERY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTIMERQUERY,
+				PVRSRV_BRIDGE_RGXTIMERQUERY_RGXQUERYTIMER);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_rgxtq2_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxtq2_bridge.c
new file mode 100644
index 000000000000..1129b824d850
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxtq2_bridge.c
@@ -0,0 +1,1610 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq2
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq2
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtdmtransfer.h"
+
+#include "common_rgxtq2_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#include "rgx_bvnc_defs_km.h"
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXTDMCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMDestroyTransferContextKM((RGX_SERVER_TQ_TDM_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					IMG_UINT8 * psRGXTDMCreateTransferContextIN_UI8,
+					IMG_UINT8 * psRGXTDMCreateTransferContextOUT_UI8,
+					CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *psRGXTDMCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXTDMCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize *
+	     sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer =
+			    (IMG_BYTE *) (void *)psRGXTDMCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMCreateTransferContextOUT->eError =
+				    PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXTDMCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXTDMCreateTransferContextIN->ui32FrameworkCmdSize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVRGXTDMCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+						psRGXTDMCreateTransferContextIN->i32Priority,
+						psRGXTDMCreateTransferContextIN->
+						ui32FrameworkCmdSize, ui8FrameworkCmdInt,
+						hPrivDataInt,
+						psRGXTDMCreateTransferContextIN->
+						ui32PackedCCBSizeU88,
+						psRGXTDMCreateTransferContextIN->ui32ContextFlags,
+						psRGXTDMCreateTransferContextIN->
+						ui64RobustnessAddress,
+						psRGXTDMCreateTransferContextIN->ui32MaxDeadlineMS,
+						&psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXTDMCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXTDMCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXTDMCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXTDMDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextIN_UI8,
+					 IMG_UINT8 * psRGXTDMDestroyTransferContextOUT_UI8,
+					 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *psRGXTDMDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXTDMDestroyTransferContextOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMDestroyTransferContextOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMDestroyTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	if (unlikely
+	    ((psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXTDMDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__,
+			 PVRSRVGetErrorString(psRGXTDMDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPriorityOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *psRGXTDMSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPriorityOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextPriority_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						     psTransferContextInt,
+						     psRGXTDMSetTransferContextPriorityIN->
+						     i32Priority);
+
+RGXTDMSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateIN_UI8,
+					  IMG_UINT8 * psRGXTDMNotifyWriteOffsetUpdateOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *psRGXTDMNotifyWriteOffsetUpdateOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE *)
+	    IMG_OFFSET_ADDR(psRGXTDMNotifyWriteOffsetUpdateOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMNotifyWriteOffsetUpdateIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMNotifyWriteOffsetUpdateOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMNotifyWriteOffsetUpdate_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMNotifyWriteOffsetUpdateOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMNotifyWriteOffsetUpdate_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMNotifyWriteOffsetUpdateOUT->eError =
+	    PVRSRVRGXTDMNotifyWriteOffsetUpdateKM(psTransferContextInt,
+						  psRGXTDMNotifyWriteOffsetUpdateIN->
+						  ui32PDumpFlags);
+
+RGXTDMNotifyWriteOffsetUpdate_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer2OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *psRGXTDMSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer2OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer2IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer2IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer2IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer2_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer2OUT->eError =
+	    PVRSRVRGXTDMSubmitTransferKM(psTransferContextInt,
+					 psRGXTDMSubmitTransfer2IN->ui32PDumpFlags,
+					 psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount,
+					 psUpdateUFOSyncPrimBlockInt,
+					 ui32UpdateSyncOffsetInt,
+					 ui32UpdateValueInt,
+					 psRGXTDMSubmitTransfer2IN->hCheckFenceFD,
+					 psRGXTDMSubmitTransfer2IN->hUpdateTimeline,
+					 &psRGXTDMSubmitTransfer2OUT->hUpdateFence,
+					 uiUpdateFenceNameInt,
+					 psRGXTDMSubmitTransfer2IN->ui32CommandSize,
+					 ui8FWCommandInt,
+					 psRGXTDMSubmitTransfer2IN->ui32ExternalJobReference,
+					 psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount,
+					 ui32SyncPMRFlagsInt,
+					 psSyncPMRsInt,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic1,
+					 psRGXTDMSubmitTransfer2IN->ui32Characteristic2,
+					 psRGXTDMSubmitTransfer2IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTDMReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryIN_UI8,
+				  IMG_UINT8 * psRGXTDMGetSharedMemoryOUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *psRGXTDMGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMGetSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMGetSharedMemory_exit;
+		}
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTDMGetSharedMemoryIN);
+
+	psRGXTDMGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTDMGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection),
+					  &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								       &psRGXTDMGetSharedMemoryOUT->
+								       hCLIPMRMem,
+								       (void *)psCLIPMRMemInt,
+								       PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								       PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								       (PFN_HANDLE_RELEASE) &
+								       _RGXTDMGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMGetSharedMemory_exit:
+
+	if (psRGXTDMGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTDMReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryIN_UI8,
+				      IMG_UINT8 * psRGXTDMReleaseSharedMemoryOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *psRGXTDMReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTDMReleaseSharedMemoryOUT_UI8, 0);
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMReleaseSharedMemoryOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMReleaseSharedMemory_exit;
+		}
+	}
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTDMReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTDMReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTDMReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTDMReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTDMReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyIN_UI8,
+					     IMG_UINT8 * psRGXTDMSetTransferContextPropertyOUT_UI8,
+					     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *psRGXTDMSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXTDMSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSetTransferContextPropertyOUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSetTransferContextProperty_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXTDMSetTransferContextPropertyKM(psTransferContextInt,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui32Property,
+						     psRGXTDMSetTransferContextPropertyIN->
+						     ui64Input,
+						     &psRGXTDMSetTransferContextPropertyOUT->
+						     ui64Output);
+
+RGXTDMSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXTDMSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3IN_UI8,
+				  IMG_UINT8 * psRGXTDMSubmitTransfer3OUT_UI8,
+				  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *psRGXTDMSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3 *)
+	    IMG_OFFSET_ADDR(psRGXTDMSubmitTransfer3OUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXTDMSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_TDM_CONTEXT *psTransferContextInt = NULL;
+	SYNC_PRIMITIVE_BLOCK **psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE *hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 *ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 *ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT8 *ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+	     sizeof(SYNC_PRIMITIVE_BLOCK *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely
+	    (psRGXTDMSubmitTransfer3IN->ui32CommandSize > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	if (unlikely(psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = OSGetDevNode(psConnection);
+
+		/* Check that device supports the required feature */
+		if ((psDeviceNode->pfnCheckDeviceFeature) &&
+		    !psDeviceNode->pfnCheckDeviceFeature(psDeviceNode,
+							 RGX_FEATURE_FASTRENDER_DM_BIT_MASK))
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_NOT_SUPPORTED;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXTDMSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXTDMSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psUpdateUFOSyncPrimBlockInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+			       sizeof(SYNC_PRIMITIVE_BLOCK *));
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount *
+		    sizeof(SYNC_PRIMITIVE_BLOCK *);
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hUpdateUFOSyncPrimBlockInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phUpdateUFOSyncPrimBlock,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_HANDLE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateSyncOffsetInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateSyncOffset,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount != 0)
+	{
+		ui32UpdateValueInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32UpdateValueInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32UpdateValue,
+		     psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount * sizeof(IMG_UINT32)) !=
+		    PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize != 0)
+	{
+		ui8FWCommandInt = (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FWCommandInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui8FWCommand,
+		     psRGXTDMSubmitTransfer3IN->ui32CommandSize * sizeof(IMG_UINT8)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2,
+		     (const void __user *)psRGXTDMSubmitTransfer3IN->phSyncPMRs,
+		     psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXTDMSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXTDMSubmitTransfer3_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTDMSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psUpdateUFOSyncPrimBlockInt[i],
+						       hUpdateUFOSyncPrimBlockInt2[i],
+						       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+						       IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXTDMSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXTDMSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXTDMSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXTDMSubmitTransfer3OUT->eError =
+	    PVRSRVRGXTDMSubmitTransfer3KM(psTransferContextInt,
+					  psRGXTDMSubmitTransfer3IN->ui32PDumpFlags,
+					  psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount,
+					  psUpdateUFOSyncPrimBlockInt,
+					  ui32UpdateSyncOffsetInt,
+					  ui32UpdateValueInt,
+					  psRGXTDMSubmitTransfer3IN->hCheckFenceFD,
+					  psRGXTDMSubmitTransfer3IN->hUpdateTimeline,
+					  &psRGXTDMSubmitTransfer3OUT->hUpdateFence,
+					  uiUpdateFenceNameInt,
+					  psRGXTDMSubmitTransfer3IN->hExportFenceToSignal,
+					  psRGXTDMSubmitTransfer3IN->ui32CommandSize,
+					  ui8FWCommandInt,
+					  psRGXTDMSubmitTransfer3IN->ui32ExternalJobReference,
+					  psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount,
+					  ui32SyncPMRFlagsInt,
+					  psSyncPMRsInt,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic1,
+					  psRGXTDMSubmitTransfer3IN->ui32Characteristic2,
+					  psRGXTDMSubmitTransfer3IN->ui64DeadlineInus);
+
+RGXTDMSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_TDM_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32ClientUpdateCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hUpdateUFOSyncPrimBlockInt2[i],
+							    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXTDMSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXTDMSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRGXTQ2Bridge(void);
+void DeinitRGXTQ2Bridge(void);
+
+/*
+ * Register all RGXTQ2 functions with services
+ */
+PVRSRV_ERROR InitRGXTQ2Bridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXTDMDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXTDMSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE,
+			      PVRSRVBridgeRGXTDMNotifyWriteOffsetUpdate, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMNOTIFYWRITEOFFSETUPDATE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMNOTIFYWRITEOFFSETUPDATE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXTDMSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTDMReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+			      PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXTDMSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXTDMSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTDMSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTDMSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq2 functions with services
+ */
+void DeinitRGXTQ2Bridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMNOTIFYWRITEOFFSETUPDATE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2,
+				PVRSRV_BRIDGE_RGXTQ2_RGXTDMSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ2, PVRSRV_BRIDGE_RGXTQ2_RGXTDMSUBMITTRANSFER3);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_rgxtq_bridge.c b/drivers/gpu/drm/img-rogue/server_rgxtq_bridge.c
new file mode 100644
index 000000000000..f50c1184676a
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_rgxtq_bridge.c
@@ -0,0 +1,1954 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for rgxtq
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for rgxtq
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "rgxtransfer.h"
+#include "rgx_tq_shared.h"
+
+#include "common_rgxtq_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _RGXCreateTransferContextpsTransferContextIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXDestroyTransferContextKM((RGX_SERVER_TQ_CONTEXT *) pvData);
+	return eError;
+}
+
+static_assert(RGXFWIF_RF_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_RF_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXCreateTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXCreateTransferContextIN_UI8,
+				     IMG_UINT8 * psRGXCreateTransferContextOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT *psRGXCreateTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT *psRGXCreateTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXCreateTransferContextOUT_UI8, 0);
+
+	IMG_BYTE *ui8FrameworkCmdInt = NULL;
+	IMG_HANDLE hPrivData = psRGXCreateTransferContextIN->hPrivData;
+	IMG_HANDLE hPrivDataInt = NULL;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE)) + 0;
+
+	if (unlikely(psRGXCreateTransferContextIN->ui32FrameworkCmdize > RGXFWIF_RF_CMD_SIZE))
+	{
+		psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXCreateTransferContext_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXCreateTransferContext_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXCreateTransferContextIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXCreateTransferContextIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXCreateTransferContext_exit;
+			}
+		}
+	}
+
+	if (psRGXCreateTransferContextIN->ui32FrameworkCmdize != 0)
+	{
+		ui8FrameworkCmdInt = (IMG_BYTE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE);
+	}
+
+	/* Copy the data over */
+	if (psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui8FrameworkCmdInt,
+		     (const void __user *)psRGXCreateTransferContextIN->pui8FrameworkCmd,
+		     psRGXCreateTransferContextIN->ui32FrameworkCmdize * sizeof(IMG_BYTE)) !=
+		    PVRSRV_OK)
+		{
+			psRGXCreateTransferContextOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXCreateTransferContext_exit;
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hPrivDataInt,
+				       hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA, IMG_TRUE);
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateTransferContext_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVRGXCreateTransferContextKM(psConnection, OSGetDevNode(psConnection),
+					     psRGXCreateTransferContextIN->i32Priority,
+					     psRGXCreateTransferContextIN->ui32FrameworkCmdize,
+					     ui8FrameworkCmdInt,
+					     hPrivDataInt,
+					     psRGXCreateTransferContextIN->ui32PackedCCBSizeU8888,
+					     psRGXCreateTransferContextIN->ui32ContextFlags,
+					     psRGXCreateTransferContextIN->ui64RobustnessAddress,
+					     &psTransferContextInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		goto RGXCreateTransferContext_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXCreateTransferContextOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psRGXCreateTransferContextOUT->hTransferContext,
+				      (void *)psTransferContextInt,
+				      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _RGXCreateTransferContextpsTransferContextIntRelease);
+	if (unlikely(psRGXCreateTransferContextOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXCreateTransferContext_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXCreateTransferContext_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hPrivDataInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPrivData, PVRSRV_HANDLE_TYPE_DEV_PRIV_DATA);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRGXCreateTransferContextOUT->eError != PVRSRV_OK)
+	{
+		if (psTransferContextInt)
+		{
+			PVRSRVRGXDestroyTransferContextKM(psTransferContextInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXCreateTransferContextOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXDestroyTransferContext(IMG_UINT32 ui32DispatchTableEntry,
+				      IMG_UINT8 * psRGXDestroyTransferContextIN_UI8,
+				      IMG_UINT8 * psRGXDestroyTransferContextOUT_UI8,
+				      CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT *psRGXDestroyTransferContextIN =
+	    (PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyTransferContextIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT *psRGXDestroyTransferContextOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT *)
+	    IMG_OFFSET_ADDR(psRGXDestroyTransferContextOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXDestroyTransferContextOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXDestroyTransferContextIN->
+					      hTransferContext,
+					      PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	if (unlikely
+	    ((psRGXDestroyTransferContextOUT->eError != PVRSRV_OK)
+	     && (psRGXDestroyTransferContextOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psRGXDestroyTransferContextOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXDestroyTransferContextOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXDestroyTransferContext_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXDestroyTransferContext_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetTransferContextPriority(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetTransferContextPriorityIN_UI8,
+					  IMG_UINT8 * psRGXSetTransferContextPriorityOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY *psRGXSetTransferContextPriorityIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPriorityIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY *psRGXSetTransferContextPriorityOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPriorityOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXSetTransferContextPriorityIN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetTransferContextPriorityOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetTransferContextPriorityOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetTransferContextPriority_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetTransferContextPriorityOUT->eError =
+	    PVRSRVRGXSetTransferContextPriorityKM(psConnection, OSGetDevNode(psConnection),
+						  psTransferContextInt,
+						  psRGXSetTransferContextPriorityIN->i32Priority);
+
+RGXSetTransferContextPriority_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXSubmitTransfer2(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXSubmitTransfer2IN_UI8,
+			       IMG_UINT8 * psRGXSubmitTransfer2OUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2 *psRGXSubmitTransfer2IN =
+	    (PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer2IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2 *psRGXSubmitTransfer2OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer2OUT_UI8,
+								     0);
+
+	IMG_HANDLE hTransferContext = psRGXSubmitTransfer2IN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateCountInt = NULL;
+	SYNC_PRIMITIVE_BLOCK ***psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE **hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 **ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 **ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT32 *ui32CommandSizeInt = NULL;
+	IMG_UINT8 **ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32TQPrepareFlagsInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BYTE *pArrayArgsBuffer2 = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+	IMG_UINT32 ui32BufferSize2 = 0;
+	IMG_UINT32 ui32NextOffset2 = 0;
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount *
+		     sizeof(SYNC_PRIMITIVE_BLOCK **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_HANDLE **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT8 *));
+	}
+
+	if (unlikely(psRGXSubmitTransfer2IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXSubmitTransfer2IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXSubmitTransfer2IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXSubmitTransfer2_exit;
+			}
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32ClientUpdateCountInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateCountInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32ClientUpdateCount,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer for first dimension */
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK ***) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(SYNC_PRIMITIVE_BLOCK **);
+		/* Assigning hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer for first dimension */
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_HANDLE);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateSyncOffsetInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateValueInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateValueInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32CommandSizeInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CommandSizeInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32CommandSize,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui8FWCommandInt to the right offset in the pool buffer for first dimension */
+		ui8FWCommandInt = (IMG_UINT8 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT8 *);
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		ui32TQPrepareFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32TQPrepareFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32TQPrepareFlags,
+		     psRGXSubmitTransfer2IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer2IN->pui32SyncPMRFlags,
+		     psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXSubmitTransfer2IN->phSyncPMRs,
+		     psRGXSubmitTransfer2IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		ui64BufferSize = 0;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] *
+			     sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize += ((IMG_UINT64) ui32CommandSizeInt[i] * sizeof(IMG_UINT8));
+		}
+		if (ui64BufferSize > IMG_UINT32_MAX)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			goto RGXSubmitTransfer2_exit;
+		}
+		ui32BufferSize2 = (IMG_UINT32) ui64BufferSize;
+	}
+
+	if (ui32BufferSize2 != 0)
+	{
+		pArrayArgsBuffer2 = OSAllocMemNoStats(ui32BufferSize2);
+
+		if (!pArrayArgsBuffer2)
+		{
+			psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto RGXSubmitTransfer2_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			if (ui32ClientUpdateCountInt[i] > PVRSRV_MAX_SYNCS)
+			{
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Assigning each psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer (this is the second dimension) */
+			psUpdateUFOSyncPrimBlockInt[i] =
+			    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer2,
+								      ui32NextOffset2);
+			OSCachedMemSet(psUpdateUFOSyncPrimBlockInt[i], 0,
+				       ui32ClientUpdateCountInt[i] *
+				       sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui32NextOffset2 +=
+			    ui32ClientUpdateCountInt[i] * sizeof(SYNC_PRIMITIVE_BLOCK *);
+			/* Assigning each hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer (this is the second dimension) */
+			hUpdateUFOSyncPrimBlockInt2[i] =
+			    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateSyncOffsetInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateSyncOffsetInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateValueInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateValueInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer2IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			if (ui32CommandSizeInt[i] > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE)
+			{
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Assigning each ui8FWCommandInt to the right offset in the pool buffer (this is the second dimension) */
+			ui8FWCommandInt[i] =
+			    (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32CommandSizeInt[i] * sizeof(IMG_UINT8);
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_HANDLE **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->
+			     phUpdateUFOSyncPrimBlock[i], sizeof(IMG_HANDLE **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (hUpdateUFOSyncPrimBlockInt2[i]),
+				     (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui32UpdateSyncOffset[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateSyncOffsetInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui32UpdateValue[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateValueInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT8 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer2IN->pui8FWCommand[i],
+			     sizeof(IMG_UINT8 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer2OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer2_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32CommandSizeInt[i] * sizeof(IMG_UINT8)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui8FWCommandInt[i]), (const void __user *)psPtr,
+				     (ui32CommandSizeInt[i] * sizeof(IMG_UINT8))) != PVRSRV_OK)
+				{
+					psRGXSubmitTransfer2OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSubmitTransfer2OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSubmitTransfer2_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+				/* Look up the address from the handle */
+				psRGXSubmitTransfer2OUT->eError =
+				    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+							       (void **)
+							       &psUpdateUFOSyncPrimBlockInt[i][j],
+							       hUpdateUFOSyncPrimBlockInt2[i][j],
+							       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+							       IMG_TRUE);
+				if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+				{
+					UnlockHandle(psConnection->psHandleBase);
+					goto RGXSubmitTransfer2_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXSubmitTransfer2OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXSubmitTransfer2OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXSubmitTransfer2_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSubmitTransfer2OUT->eError =
+	    PVRSRVRGXSubmitTransferKM(psTransferContextInt,
+				      psRGXSubmitTransfer2IN->ui32PrepareCount,
+				      ui32ClientUpdateCountInt,
+				      psUpdateUFOSyncPrimBlockInt,
+				      ui32UpdateSyncOffsetInt,
+				      ui32UpdateValueInt,
+				      psRGXSubmitTransfer2IN->hCheckFenceFD,
+				      psRGXSubmitTransfer2IN->h2DUpdateTimeline,
+				      &psRGXSubmitTransfer2OUT->h2DUpdateFence,
+				      psRGXSubmitTransfer2IN->h3DUpdateTimeline,
+				      &psRGXSubmitTransfer2OUT->h3DUpdateFence,
+				      uiUpdateFenceNameInt,
+				      ui32CommandSizeInt,
+				      ui8FWCommandInt,
+				      ui32TQPrepareFlagsInt,
+				      psRGXSubmitTransfer2IN->ui32ExtJobRef,
+				      psRGXSubmitTransfer2IN->ui32SyncPMRCount,
+				      ui32SyncPMRFlagsInt, psSyncPMRsInt);
+
+RGXSubmitTransfer2_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+
+				/* Unreference the previously looked up handle */
+				if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i]
+				    && psUpdateUFOSyncPrimBlockInt[i][j])
+				{
+					PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+								    hUpdateUFOSyncPrimBlockInt2[i]
+								    [j],
+								    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+				}
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer2IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer2OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize2 == ui32NextOffset2);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (pArrayArgsBuffer2)
+		OSFreeMemNoStats(pArrayArgsBuffer2);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RGXTQGetSharedMemorypsCLIPMRMemIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVRGXTQReleaseSharedMemoryKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTQGetSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRGXTQGetSharedMemoryIN_UI8,
+				 IMG_UINT8 * psRGXTQGetSharedMemoryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY *psRGXTQGetSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTQGETSHAREDMEMORY *) IMG_OFFSET_ADDR(psRGXTQGetSharedMemoryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY *psRGXTQGetSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQGetSharedMemoryOUT_UI8, 0);
+
+	PMR *psCLIPMRMemInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psRGXTQGetSharedMemoryIN);
+
+	psRGXTQGetSharedMemoryOUT->eError =
+	    PVRSRVRGXTQGetSharedMemoryKM(psConnection, OSGetDevNode(psConnection), &psCLIPMRMemInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		goto RGXTQGetSharedMemory_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTQGetSharedMemoryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRGXTQGetSharedMemoryOUT->
+								      hCLIPMRMem,
+								      (void *)psCLIPMRMemInt,
+								      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RGXTQGetSharedMemorypsCLIPMRMemIntRelease);
+	if (unlikely(psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTQGetSharedMemory_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTQGetSharedMemory_exit:
+
+	if (psRGXTQGetSharedMemoryOUT->eError != PVRSRV_OK)
+	{
+		if (psCLIPMRMemInt)
+		{
+			PVRSRVRGXTQReleaseSharedMemoryKM(psCLIPMRMemInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXTQReleaseSharedMemory(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRGXTQReleaseSharedMemoryIN_UI8,
+				     IMG_UINT8 * psRGXTQReleaseSharedMemoryOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY *psRGXTQReleaseSharedMemoryIN =
+	    (PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQReleaseSharedMemoryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY *psRGXTQReleaseSharedMemoryOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY *)
+	    IMG_OFFSET_ADDR(psRGXTQReleaseSharedMemoryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRGXTQReleaseSharedMemoryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRGXTQReleaseSharedMemoryIN->hPMRMem,
+					      PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE);
+	if (unlikely((psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_OK) &&
+		     (psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRGXTQReleaseSharedMemoryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRGXTQReleaseSharedMemoryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXTQReleaseSharedMemory_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RGXTQReleaseSharedMemory_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRGXSetTransferContextProperty(IMG_UINT32 ui32DispatchTableEntry,
+					  IMG_UINT8 * psRGXSetTransferContextPropertyIN_UI8,
+					  IMG_UINT8 * psRGXSetTransferContextPropertyOUT_UI8,
+					  CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY *psRGXSetTransferContextPropertyIN =
+	    (PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPropertyIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY *psRGXSetTransferContextPropertyOUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY *)
+	    IMG_OFFSET_ADDR(psRGXSetTransferContextPropertyOUT_UI8, 0);
+
+	IMG_HANDLE hTransferContext = psRGXSetTransferContextPropertyIN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSetTransferContextPropertyOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSetTransferContextPropertyOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSetTransferContextProperty_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSetTransferContextPropertyOUT->eError =
+	    PVRSRVRGXSetTransferContextPropertyKM(psTransferContextInt,
+						  psRGXSetTransferContextPropertyIN->ui32Property,
+						  psRGXSetTransferContextPropertyIN->ui64Input,
+						  &psRGXSetTransferContextPropertyOUT->ui64Output);
+
+RGXSetTransferContextProperty_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+static_assert(RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE <= IMG_UINT32_MAX,
+	      "RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE must not be larger than IMG_UINT32_MAX");
+static_assert(PVRSRV_MAX_SYNCS <= IMG_UINT32_MAX,
+	      "PVRSRV_MAX_SYNCS must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRGXSubmitTransfer3(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psRGXSubmitTransfer3IN_UI8,
+			       IMG_UINT8 * psRGXSubmitTransfer3OUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3 *psRGXSubmitTransfer3IN =
+	    (PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer3IN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3 *psRGXSubmitTransfer3OUT =
+	    (PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3 *) IMG_OFFSET_ADDR(psRGXSubmitTransfer3OUT_UI8,
+								     0);
+
+	IMG_HANDLE hTransferContext = psRGXSubmitTransfer3IN->hTransferContext;
+	RGX_SERVER_TQ_CONTEXT *psTransferContextInt = NULL;
+	IMG_UINT32 *ui32ClientUpdateCountInt = NULL;
+	SYNC_PRIMITIVE_BLOCK ***psUpdateUFOSyncPrimBlockInt = NULL;
+	IMG_HANDLE **hUpdateUFOSyncPrimBlockInt2 = NULL;
+	IMG_UINT32 **ui32UpdateSyncOffsetInt = NULL;
+	IMG_UINT32 **ui32UpdateValueInt = NULL;
+	IMG_CHAR *uiUpdateFenceNameInt = NULL;
+	IMG_UINT32 *ui32CommandSizeInt = NULL;
+	IMG_UINT8 **ui8FWCommandInt = NULL;
+	IMG_UINT32 *ui32TQPrepareFlagsInt = NULL;
+	IMG_UINT32 *ui32SyncPMRFlagsInt = NULL;
+	PMR **psSyncPMRsInt = NULL;
+	IMG_HANDLE *hSyncPMRsInt2 = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BYTE *pArrayArgsBuffer2 = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *)) +
+	    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) + 0;
+	IMG_UINT32 ui32BufferSize2 = 0;
+	IMG_UINT32 ui32NextOffset2 = 0;
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount *
+		     sizeof(SYNC_PRIMITIVE_BLOCK **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_HANDLE **));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *));
+		ui64BufferSize +=
+		    ((IMG_UINT64) psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT8 *));
+	}
+
+	if (unlikely(psRGXSubmitTransfer3IN->ui32SyncPMRCount > PVRSRV_MAX_SYNCS))
+	{
+		psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRGXSubmitTransfer3IN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRGXSubmitTransfer3IN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RGXSubmitTransfer3_exit;
+			}
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32ClientUpdateCountInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32ClientUpdateCountInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32ClientUpdateCount,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer for first dimension */
+		psUpdateUFOSyncPrimBlockInt =
+		    (SYNC_PRIMITIVE_BLOCK ***) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset +=
+		    psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(SYNC_PRIMITIVE_BLOCK **);
+		/* Assigning hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer for first dimension */
+		hUpdateUFOSyncPrimBlockInt2 =
+		    (IMG_HANDLE **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_HANDLE);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateSyncOffsetInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateSyncOffsetInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui32UpdateValueInt to the right offset in the pool buffer for first dimension */
+		ui32UpdateValueInt =
+		    (IMG_UINT32 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32 *);
+	}
+
+	{
+		uiUpdateFenceNameInt =
+		    (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiUpdateFenceNameInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->puiUpdateFenceName,
+		     PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+		((IMG_CHAR *) uiUpdateFenceNameInt)[(PVRSRV_SYNC_NAME_LENGTH * sizeof(IMG_CHAR)) -
+						    1] = '\0';
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32CommandSizeInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32CommandSizeInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32CommandSize,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		/* Assigning ui8FWCommandInt to the right offset in the pool buffer for first dimension */
+		ui8FWCommandInt = (IMG_UINT8 **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT8 *);
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		ui32TQPrepareFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32TQPrepareFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32TQPrepareFlags,
+		     psRGXSubmitTransfer3IN->ui32PrepareCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		ui32SyncPMRFlagsInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32SyncPMRFlagsInt,
+		     (const void __user *)psRGXSubmitTransfer3IN->pui32SyncPMRFlags,
+		     psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount != 0)
+	{
+		psSyncPMRsInt = (PMR **) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		OSCachedMemSet(psSyncPMRsInt, 0,
+			       psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *));
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(PMR *);
+		hSyncPMRsInt2 = (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE);
+	}
+
+	/* Copy the data over */
+	if (psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, hSyncPMRsInt2, (const void __user *)psRGXSubmitTransfer3IN->phSyncPMRs,
+		     psRGXSubmitTransfer3IN->ui32SyncPMRCount * sizeof(IMG_HANDLE)) != PVRSRV_OK)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		ui64BufferSize = 0;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] *
+			     sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE *));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize +=
+			    ((IMG_UINT64) ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32));
+			ui64BufferSize += ((IMG_UINT64) ui32CommandSizeInt[i] * sizeof(IMG_UINT8));
+		}
+		if (ui64BufferSize > IMG_UINT32_MAX)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+			goto RGXSubmitTransfer3_exit;
+		}
+		ui32BufferSize2 = (IMG_UINT32) ui64BufferSize;
+	}
+
+	if (ui32BufferSize2 != 0)
+	{
+		pArrayArgsBuffer2 = OSAllocMemNoStats(ui32BufferSize2);
+
+		if (!pArrayArgsBuffer2)
+		{
+			psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto RGXSubmitTransfer3_exit;
+		}
+	}
+
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			if (ui32ClientUpdateCountInt[i] > PVRSRV_MAX_SYNCS)
+			{
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Assigning each psUpdateUFOSyncPrimBlockInt to the right offset in the pool buffer (this is the second dimension) */
+			psUpdateUFOSyncPrimBlockInt[i] =
+			    (SYNC_PRIMITIVE_BLOCK **) IMG_OFFSET_ADDR(pArrayArgsBuffer2,
+								      ui32NextOffset2);
+			OSCachedMemSet(psUpdateUFOSyncPrimBlockInt[i], 0,
+				       ui32ClientUpdateCountInt[i] *
+				       sizeof(SYNC_PRIMITIVE_BLOCK *));
+			ui32NextOffset2 +=
+			    ui32ClientUpdateCountInt[i] * sizeof(SYNC_PRIMITIVE_BLOCK *);
+			/* Assigning each hUpdateUFOSyncPrimBlockInt2 to the right offset in the pool buffer (this is the second dimension) */
+			hUpdateUFOSyncPrimBlockInt2[i] =
+			    (IMG_HANDLE *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateSyncOffsetInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateSyncOffsetInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Assigning each ui32UpdateValueInt to the right offset in the pool buffer (this is the second dimension) */
+			ui32UpdateValueInt[i] =
+			    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32);
+		}
+	}
+	if (psRGXSubmitTransfer3IN->ui32PrepareCount != 0)
+	{
+		IMG_UINT32 i;
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			if (ui32CommandSizeInt[i] > RGXFWIF_DM_INDEPENDENT_KICK_CMD_SIZE)
+			{
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Assigning each ui8FWCommandInt to the right offset in the pool buffer (this is the second dimension) */
+			ui8FWCommandInt[i] =
+			    (IMG_UINT8 *) IMG_OFFSET_ADDR(pArrayArgsBuffer2, ui32NextOffset2);
+			ui32NextOffset2 += ui32CommandSizeInt[i] * sizeof(IMG_UINT8);
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_HANDLE **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->
+			     phUpdateUFOSyncPrimBlock[i], sizeof(IMG_HANDLE **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (hUpdateUFOSyncPrimBlockInt2[i]),
+				     (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_HANDLE))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui32UpdateSyncOffset[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateSyncOffsetInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT32 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui32UpdateValue[i],
+			     sizeof(IMG_UINT32 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui32UpdateValueInt[i]), (const void __user *)psPtr,
+				     (ui32ClientUpdateCountInt[i] * sizeof(IMG_UINT32))) !=
+				    PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+		IMG_UINT8 **psPtr;
+
+		/* Loop over all the pointers in the array copying the data into the kernel */
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			/* Copy the pointer over from the client side */
+			if (OSCopyFromUser
+			    (NULL, &psPtr,
+			     (const void __user *)&psRGXSubmitTransfer3IN->pui8FWCommand[i],
+			     sizeof(IMG_UINT8 **)) != PVRSRV_OK)
+			{
+				psRGXSubmitTransfer3OUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+				goto RGXSubmitTransfer3_exit;
+			}
+
+			/* Copy the data over */
+			if ((ui32CommandSizeInt[i] * sizeof(IMG_UINT8)) > 0)
+			{
+				if (OSCopyFromUser
+				    (NULL, (ui8FWCommandInt[i]), (const void __user *)psPtr,
+				     (ui32CommandSizeInt[i] * sizeof(IMG_UINT8))) != PVRSRV_OK)
+				{
+					psRGXSubmitTransfer3OUT->eError =
+					    PVRSRV_ERROR_INVALID_PARAMS;
+
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRGXSubmitTransfer3OUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psTransferContextInt,
+				       hTransferContext,
+				       PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT, IMG_TRUE);
+	if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RGXSubmitTransfer3_exit;
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+				/* Look up the address from the handle */
+				psRGXSubmitTransfer3OUT->eError =
+				    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+							       (void **)
+							       &psUpdateUFOSyncPrimBlockInt[i][j],
+							       hUpdateUFOSyncPrimBlockInt2[i][j],
+							       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+							       IMG_TRUE);
+				if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+				{
+					UnlockHandle(psConnection->psHandleBase);
+					goto RGXSubmitTransfer3_exit;
+				}
+			}
+		}
+	}
+
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+			/* Look up the address from the handle */
+			psRGXSubmitTransfer3OUT->eError =
+			    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+						       (void **)&psSyncPMRsInt[i],
+						       hSyncPMRsInt2[i],
+						       PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+			if (unlikely(psRGXSubmitTransfer3OUT->eError != PVRSRV_OK))
+			{
+				UnlockHandle(psConnection->psHandleBase);
+				goto RGXSubmitTransfer3_exit;
+			}
+		}
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRGXSubmitTransfer3OUT->eError =
+	    PVRSRVRGXSubmitTransfer3KM(psTransferContextInt,
+				       psRGXSubmitTransfer3IN->ui32PrepareCount,
+				       ui32ClientUpdateCountInt,
+				       psUpdateUFOSyncPrimBlockInt,
+				       ui32UpdateSyncOffsetInt,
+				       ui32UpdateValueInt,
+				       psRGXSubmitTransfer3IN->hCheckFenceFD,
+				       psRGXSubmitTransfer3IN->h2DUpdateTimeline,
+				       &psRGXSubmitTransfer3OUT->h2DUpdateFence,
+				       psRGXSubmitTransfer3IN->h3DUpdateTimeline,
+				       &psRGXSubmitTransfer3OUT->h3DUpdateFence,
+				       uiUpdateFenceNameInt,
+				       psRGXSubmitTransfer3IN->hExportFenceToSignal,
+				       ui32CommandSizeInt,
+				       ui8FWCommandInt,
+				       ui32TQPrepareFlagsInt,
+				       psRGXSubmitTransfer3IN->ui32ExtJobRef,
+				       psRGXSubmitTransfer3IN->ui32SyncPMRCount,
+				       ui32SyncPMRFlagsInt, psSyncPMRsInt);
+
+RGXSubmitTransfer3_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psTransferContextInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hTransferContext,
+					    PVRSRV_HANDLE_TYPE_RGX_SERVER_TQ_CONTEXT);
+	}
+
+	if (hUpdateUFOSyncPrimBlockInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32PrepareCount; i++)
+		{
+			IMG_UINT32 j;
+			for (j = 0; j < ui32ClientUpdateCountInt[i]; j++)
+			{
+
+				/* Unreference the previously looked up handle */
+				if (psUpdateUFOSyncPrimBlockInt && psUpdateUFOSyncPrimBlockInt[i]
+				    && psUpdateUFOSyncPrimBlockInt[i][j])
+				{
+					PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+								    hUpdateUFOSyncPrimBlockInt2[i]
+								    [j],
+								    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+				}
+			}
+		}
+	}
+
+	if (hSyncPMRsInt2)
+	{
+		IMG_UINT32 i;
+
+		for (i = 0; i < psRGXSubmitTransfer3IN->ui32SyncPMRCount; i++)
+		{
+
+			/* Unreference the previously looked up handle */
+			if (psSyncPMRsInt && psSyncPMRsInt[i])
+			{
+				PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+							    hSyncPMRsInt2[i],
+							    PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+			}
+		}
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRGXSubmitTransfer3OUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize2 == ui32NextOffset2);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (pArrayArgsBuffer2)
+		OSFreeMemNoStats(pArrayArgsBuffer2);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+#endif /* SUPPORT_RGXTQ_BRIDGE */
+
+#if defined(SUPPORT_RGXTQ_BRIDGE)
+PVRSRV_ERROR InitRGXTQBridge(void);
+void DeinitRGXTQBridge(void);
+
+/*
+ * Register all RGXTQ functions with services
+ */
+PVRSRV_ERROR InitRGXTQBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT,
+			      PVRSRVBridgeRGXCreateTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXCREATETRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXCREATETRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT,
+			      PVRSRVBridgeRGXDestroyTransferContext, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXDESTROYTRANSFERCONTEXT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXDESTROYTRANSFERCONTEXT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+			      PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY,
+			      PVRSRVBridgeRGXSetTransferContextPriority, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPRIORITY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPRIORITY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2,
+			      PVRSRVBridgeRGXSubmitTransfer2, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER2),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER2));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY,
+			      PVRSRVBridgeRGXTQGetSharedMemory, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTQGETSHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY,
+			      PVRSRVBridgeRGXTQReleaseSharedMemory, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXTQRELEASESHAREDMEMORY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXTQRELEASESHAREDMEMORY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+			      PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY,
+			      PVRSRVBridgeRGXSetTransferContextProperty, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSETTRANSFERCONTEXTPROPERTY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSETTRANSFERCONTEXTPROPERTY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3,
+			      PVRSRVBridgeRGXSubmitTransfer3, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RGXSUBMITTRANSFER3),
+			      sizeof(PVRSRV_BRIDGE_OUT_RGXSUBMITTRANSFER3));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all rgxtq functions with services
+ */
+void DeinitRGXTQBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXCREATETRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXDESTROYTRANSFERCONTEXT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+				PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPRIORITY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER2);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQGETSHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXTQRELEASESHAREDMEMORY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ,
+				PVRSRV_BRIDGE_RGXTQ_RGXSETTRANSFERCONTEXTPROPERTY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RGXTQ, PVRSRV_BRIDGE_RGXTQ_RGXSUBMITTRANSFER3);
+
+}
+#else /* SUPPORT_RGXTQ_BRIDGE */
+/* This bridge is conditional on SUPPORT_RGXTQ_BRIDGE - when not defined,
+ * do not populate the dispatch table with its functions
+ */
+#define InitRGXTQBridge() \
+	PVRSRV_OK
+
+#define DeinitRGXTQBridge()
+
+#endif /* SUPPORT_RGXTQ_BRIDGE */
diff --git a/drivers/gpu/drm/img-rogue/server_ri_bridge.c b/drivers/gpu/drm/img-rogue/server_ri_bridge.c
new file mode 100644
index 000000000000..f8a54d70ef0c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_ri_bridge.c
@@ -0,0 +1,761 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for ri
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for ri
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "ri_server.h"
+
+#include "common_ri_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntry(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psRIWritePMREntryIN_UI8,
+			    IMG_UINT8 * psRIWritePMREntryOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *psRIWritePMREntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *psRIWritePMREntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY *) IMG_OFFSET_ADDR(psRIWritePMREntryOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryOUT->eError = RIWritePMREntryKM(psPMRHandleInt);
+
+RIWritePMREntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteMEMDESCEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIWriteMEMDESCEntryIN_UI8,
+				IMG_UINT8 * psRIWriteMEMDESCEntryOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *psRIWriteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIWriteMEMDESCEntryOUT_UI8,
+								      0);
+
+	IMG_HANDLE hPMRHandle = psRIWriteMEMDESCEntryIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteMEMDESCEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteMEMDESCEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteMEMDESCEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteMEMDESCEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteMEMDESCEntryIN->puiTextB,
+		     psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteMEMDESCEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteMEMDESCEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteMEMDESCEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWriteMEMDESCEntryOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError =
+	    RIWriteMEMDESCEntryKM(psConnection, OSGetDevNode(psConnection),
+				  psPMRHandleInt,
+				  psRIWriteMEMDESCEntryIN->ui32TextBSize,
+				  uiTextBInt,
+				  psRIWriteMEMDESCEntryIN->ui64Offset,
+				  psRIWriteMEMDESCEntryIN->ui64Size,
+				  psRIWriteMEMDESCEntryIN->ui64Flags, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteMEMDESCEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								     &psRIWriteMEMDESCEntryOUT->
+								     hRIHandle,
+								     (void *)psRIHandleInt,
+								     PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								     PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								     (PFN_HANDLE_RELEASE) &
+								     _RIWriteMEMDESCEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteMEMDESCEntry_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psRIWriteMEMDESCEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteMEMDESCEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _RIWriteProcListEntrypsRIHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = RIDeleteMEMDESCEntryKM((RI_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(DEVMEM_ANNOTATION_MAX_LEN <= IMG_UINT32_MAX,
+	      "DEVMEM_ANNOTATION_MAX_LEN must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeRIWriteProcListEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIWriteProcListEntryIN_UI8,
+				 IMG_UINT8 * psRIWriteProcListEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY *) IMG_OFFSET_ADDR(psRIWriteProcListEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *psRIWriteProcListEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY *)
+	    IMG_OFFSET_ADDR(psRIWriteProcListEntryOUT_UI8, 0);
+
+	IMG_CHAR *uiTextBInt = NULL;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psRIWriteProcListEntryIN->ui32TextBSize > DEVMEM_ANNOTATION_MAX_LEN))
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto RIWriteProcListEntry_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psRIWriteProcListEntryIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psRIWriteProcListEntryIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto RIWriteProcListEntry_exit;
+			}
+		}
+	}
+
+	if (psRIWriteProcListEntryIN->ui32TextBSize != 0)
+	{
+		uiTextBInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiTextBInt, (const void __user *)psRIWriteProcListEntryIN->puiTextB,
+		     psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psRIWriteProcListEntryOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto RIWriteProcListEntry_exit;
+		}
+		((IMG_CHAR *)
+		 uiTextBInt)[(psRIWriteProcListEntryIN->ui32TextBSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psRIWriteProcListEntryOUT->eError =
+	    RIWriteProcListEntryKM(psConnection, OSGetDevNode(psConnection),
+				   psRIWriteProcListEntryIN->ui32TextBSize,
+				   uiTextBInt,
+				   psRIWriteProcListEntryIN->ui64Size,
+				   psRIWriteProcListEntryIN->ui64DevVAddr, &psRIHandleInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIWriteProcListEntryOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								      &psRIWriteProcListEntryOUT->
+								      hRIHandle,
+								      (void *)psRIHandleInt,
+								      PVRSRV_HANDLE_TYPE_RI_HANDLE,
+								      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								      (PFN_HANDLE_RELEASE) &
+								      _RIWriteProcListEntrypsRIHandleIntRelease);
+	if (unlikely(psRIWriteProcListEntryOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWriteProcListEntry_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIWriteProcListEntry_exit:
+
+	if (psRIWriteProcListEntryOUT->eError != PVRSRV_OK)
+	{
+		if (psRIHandleInt)
+		{
+			RIDeleteMEMDESCEntryKM(psRIHandleInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psRIWriteProcListEntryOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIUpdateMEMDESCAddr(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrIN_UI8,
+				IMG_UINT8 * psRIUpdateMEMDESCAddrOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrIN =
+	    (PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *psRIUpdateMEMDESCAddrOUT =
+	    (PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR *) IMG_OFFSET_ADDR(psRIUpdateMEMDESCAddrOUT_UI8,
+								      0);
+
+	IMG_HANDLE hRIHandle = psRIUpdateMEMDESCAddrIN->hRIHandle;
+	RI_HANDLE psRIHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psRIHandleInt,
+				       hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE, IMG_TRUE);
+	if (unlikely(psRIUpdateMEMDESCAddrOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIUpdateMEMDESCAddr_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIUpdateMEMDESCAddrOUT->eError =
+	    RIUpdateMEMDESCAddrKM(psRIHandleInt, psRIUpdateMEMDESCAddrIN->sAddr);
+
+RIUpdateMEMDESCAddr_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psRIHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hRIHandle, PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDeleteMEMDESCEntry(IMG_UINT32 ui32DispatchTableEntry,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryIN_UI8,
+				 IMG_UINT8 * psRIDeleteMEMDESCEntryOUT_UI8,
+				 CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryIN =
+	    (PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY *) IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryIN_UI8,
+								      0);
+	PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *psRIDeleteMEMDESCEntryOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY *)
+	    IMG_OFFSET_ADDR(psRIDeleteMEMDESCEntryOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psRIDeleteMEMDESCEntryOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psRIDeleteMEMDESCEntryIN->hRIHandle,
+					      PVRSRV_HANDLE_TYPE_RI_HANDLE);
+	if (unlikely((psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_OK) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psRIDeleteMEMDESCEntryOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psRIDeleteMEMDESCEntryOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDeleteMEMDESCEntry_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+RIDeleteMEMDESCEntry_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpList(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psRIDumpListIN_UI8,
+		       IMG_UINT8 * psRIDumpListOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPLIST *psRIDumpListIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPLIST *psRIDumpListOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPLIST *) IMG_OFFSET_ADDR(psRIDumpListOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIDumpListIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIDumpListOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIDumpListOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIDumpList_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIDumpListOUT->eError = RIDumpListKM(psPMRHandleInt);
+
+RIDumpList_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpAll(IMG_UINT32 ui32DispatchTableEntry,
+		      IMG_UINT8 * psRIDumpAllIN_UI8,
+		      IMG_UINT8 * psRIDumpAllOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPALL *psRIDumpAllIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPALL *psRIDumpAllOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPALL *) IMG_OFFSET_ADDR(psRIDumpAllOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psRIDumpAllIN);
+
+	psRIDumpAllOUT->eError = RIDumpAllKM();
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIDumpProcess(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psRIDumpProcessIN_UI8,
+			  IMG_UINT8 * psRIDumpProcessOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIDUMPPROCESS *psRIDumpProcessIN =
+	    (PVRSRV_BRIDGE_IN_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *psRIDumpProcessOUT =
+	    (PVRSRV_BRIDGE_OUT_RIDUMPPROCESS *) IMG_OFFSET_ADDR(psRIDumpProcessOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psRIDumpProcessOUT->eError = RIDumpProcessKM(psRIDumpProcessIN->ui32Pid);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeRIWritePMREntryWithOwner(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerIN_UI8,
+				     IMG_UINT8 * psRIWritePMREntryWithOwnerOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerIN =
+	    (PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *psRIWritePMREntryWithOwnerOUT =
+	    (PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER *)
+	    IMG_OFFSET_ADDR(psRIWritePMREntryWithOwnerOUT_UI8, 0);
+
+	IMG_HANDLE hPMRHandle = psRIWritePMREntryWithOwnerIN->hPMRHandle;
+	PMR *psPMRHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psPMRHandleInt,
+				       hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR, IMG_TRUE);
+	if (unlikely(psRIWritePMREntryWithOwnerOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto RIWritePMREntryWithOwner_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psRIWritePMREntryWithOwnerOUT->eError =
+	    RIWritePMREntryWithOwnerKM(psPMRHandleInt, psRIWritePMREntryWithOwnerIN->ui32Owner);
+
+RIWritePMREntryWithOwner_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psPMRHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hPMRHandle, PVRSRV_HANDLE_TYPE_PHYSMEM_PMR);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitRIBridge(void);
+void DeinitRIBridge(void);
+
+/*
+ * Register all RI functions with services
+ */
+PVRSRV_ERROR InitRIBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY,
+			      PVRSRVBridgeRIWritePMREntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY,
+			      PVRSRVBridgeRIWriteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY,
+			      PVRSRVBridgeRIWriteProcListEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPROCLISTENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPROCLISTENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR,
+			      PVRSRVBridgeRIUpdateMEMDESCAddr, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIUPDATEMEMDESCADDR),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIUPDATEMEMDESCADDR));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY,
+			      PVRSRVBridgeRIDeleteMEMDESCEntry, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDELETEMEMDESCENTRY),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDELETEMEMDESCENTRY));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST, PVRSRVBridgeRIDumpList,
+			      NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPLIST),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPLIST));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL, PVRSRVBridgeRIDumpAll,
+			      NULL, 0, sizeof(PVRSRV_BRIDGE_OUT_RIDUMPALL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS,
+			      PVRSRVBridgeRIDumpProcess, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIDUMPPROCESS),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIDUMPPROCESS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER,
+			      PVRSRVBridgeRIWritePMREntryWithOwner, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RIWRITEPMRENTRYWITHOWNER),
+			      sizeof(PVRSRV_BRIDGE_OUT_RIWRITEPMRENTRYWITHOWNER));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all ri functions with services
+ */
+void DeinitRIBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPROCLISTENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIUPDATEMEMDESCADDR);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDELETEMEMDESCENTRY);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPLIST);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPALL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIDUMPPROCESS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_RI, PVRSRV_BRIDGE_RI_RIWRITEPMRENTRYWITHOWNER);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_srvcore_bridge.c b/drivers/gpu/drm/img-rogue/server_srvcore_bridge.c
new file mode 100644
index 000000000000..d40e53b244f8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_srvcore_bridge.c
@@ -0,0 +1,1080 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for srvcore
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for srvcore
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "srvcore.h"
+#include "info_page.h"
+#include "proc_stats.h"
+#include "rgx_fwif_alignchecks.h"
+
+#include "common_srvcore_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeConnect(IMG_UINT32 ui32DispatchTableEntry,
+		    IMG_UINT8 * psConnectIN_UI8,
+		    IMG_UINT8 * psConnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_CONNECT *psConnectIN =
+	    (PVRSRV_BRIDGE_IN_CONNECT *) IMG_OFFSET_ADDR(psConnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_CONNECT *psConnectOUT =
+	    (PVRSRV_BRIDGE_OUT_CONNECT *) IMG_OFFSET_ADDR(psConnectOUT_UI8, 0);
+
+	psConnectOUT->eError =
+	    PVRSRVConnectKM(psConnection, OSGetDevNode(psConnection),
+			    psConnectIN->ui32Flags,
+			    psConnectIN->ui32ClientBuildOptions,
+			    psConnectIN->ui32ClientDDKVersion,
+			    psConnectIN->ui32ClientDDKBuild,
+			    &psConnectOUT->ui8KernelArch,
+			    &psConnectOUT->ui32CapabilityFlags, &psConnectOUT->ui64PackedBvnc);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDisconnect(IMG_UINT32 ui32DispatchTableEntry,
+		       IMG_UINT8 * psDisconnectIN_UI8,
+		       IMG_UINT8 * psDisconnectOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DISCONNECT *psDisconnectIN =
+	    (PVRSRV_BRIDGE_IN_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DISCONNECT *psDisconnectOUT =
+	    (PVRSRV_BRIDGE_OUT_DISCONNECT *) IMG_OFFSET_ADDR(psDisconnectOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	PVR_UNREFERENCED_PARAMETER(psDisconnectIN);
+
+	psDisconnectOUT->eError = PVRSRVDisconnectKM();
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireGlobalEventObjecthGlobalEventObjectIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseGlobalEventObjectKM((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psAcquireGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psAcquireGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *psAcquireGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psAcquireGlobalEventObjectOUT_UI8, 0);
+
+	IMG_HANDLE hGlobalEventObjectInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireGlobalEventObjectIN);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAcquireGlobalEventObjectKM(&hGlobalEventObjectInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAcquireGlobalEventObjectOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+				      &psAcquireGlobalEventObjectOUT->hGlobalEventObject,
+				      (void *)hGlobalEventObjectInt,
+				      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) &
+				      _AcquireGlobalEventObjecthGlobalEventObjectIntRelease);
+	if (unlikely(psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AcquireGlobalEventObject_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AcquireGlobalEventObject_exit:
+
+	if (psAcquireGlobalEventObjectOUT->eError != PVRSRV_OK)
+	{
+		if (hGlobalEventObjectInt)
+		{
+			PVRSRVReleaseGlobalEventObjectKM(hGlobalEventObjectInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseGlobalEventObject(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psReleaseGlobalEventObjectIN_UI8,
+				     IMG_UINT8 * psReleaseGlobalEventObjectOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *psReleaseGlobalEventObjectOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT *)
+	    IMG_OFFSET_ADDR(psReleaseGlobalEventObjectOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psReleaseGlobalEventObjectOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psReleaseGlobalEventObjectIN->
+					      hGlobalEventObject,
+					      PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	if (unlikely
+	    ((psReleaseGlobalEventObjectOUT->eError != PVRSRV_OK)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL)
+	     && (psReleaseGlobalEventObjectOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psReleaseGlobalEventObjectOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto ReleaseGlobalEventObject_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+ReleaseGlobalEventObject_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _EventObjectOpenhOSEventIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = OSEventObjectClose((IMG_HANDLE) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectOpen(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectOpenIN_UI8,
+			    IMG_UINT8 * psEventObjectOpenOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *psEventObjectOpenIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *psEventObjectOpenOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN *) IMG_OFFSET_ADDR(psEventObjectOpenOUT_UI8, 0);
+
+	IMG_HANDLE hEventObject = psEventObjectOpenIN->hEventObject;
+	IMG_HANDLE hEventObjectInt = NULL;
+	IMG_HANDLE hOSEventInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectOpenOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hEventObjectInt,
+				       hEventObject,
+				       PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT, IMG_TRUE);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = OSEventObjectOpen(hEventObjectInt, &hOSEventInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		goto EventObjectOpen_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectOpenOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+								 &psEventObjectOpenOUT->hOSEvent,
+								 (void *)hOSEventInt,
+								 PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT,
+								 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+								 (PFN_HANDLE_RELEASE) &
+								 _EventObjectOpenhOSEventIntRelease);
+	if (unlikely(psEventObjectOpenOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectOpen_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectOpen_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hEventObjectInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hEventObject, PVRSRV_HANDLE_TYPE_SHARED_EVENT_OBJECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psEventObjectOpenOUT->eError != PVRSRV_OK)
+	{
+		if (hOSEventInt)
+		{
+			OSEventObjectClose(hOSEventInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWait(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psEventObjectWaitIN_UI8,
+			    IMG_UINT8 * psEventObjectWaitOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *psEventObjectWaitIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *psEventObjectWaitOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT *) IMG_OFFSET_ADDR(psEventObjectWaitOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWait_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitOUT->eError = OSEventObjectWait(hOSEventKMInt);
+
+EventObjectWait_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectClose(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psEventObjectCloseIN_UI8,
+			     IMG_UINT8 * psEventObjectCloseOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *psEventObjectCloseIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *psEventObjectCloseOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE *) IMG_OFFSET_ADDR(psEventObjectCloseOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psEventObjectCloseOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psEventObjectCloseIN->hOSEventKM,
+					      PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	if (unlikely((psEventObjectCloseOUT->eError != PVRSRV_OK) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psEventObjectCloseOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psEventObjectCloseOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectClose_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+EventObjectClose_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeDumpDebugInfo(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psDumpDebugInfoIN_UI8,
+			  IMG_UINT8 * psDumpDebugInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *psDumpDebugInfoIN =
+	    (PVRSRV_BRIDGE_IN_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *psDumpDebugInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO *) IMG_OFFSET_ADDR(psDumpDebugInfoOUT_UI8, 0);
+
+	psDumpDebugInfoOUT->eError =
+	    PVRSRVDumpDebugInfoKM(psConnection, OSGetDevNode(psConnection),
+				  psDumpDebugInfoIN->ui32VerbLevel);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDevClockSpeed(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetDevClockSpeedIN_UI8,
+			     IMG_UINT8 * psGetDevClockSpeedOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *psGetDevClockSpeedIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *psGetDevClockSpeedOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED *) IMG_OFFSET_ADDR(psGetDevClockSpeedOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDevClockSpeedIN);
+
+	psGetDevClockSpeedOUT->eError =
+	    PVRSRVGetDevClockSpeedKM(psConnection, OSGetDevNode(psConnection),
+				     &psGetDevClockSpeedOUT->ui32ClockSpeed);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeHWOpTimeout(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psHWOpTimeoutIN_UI8,
+			IMG_UINT8 * psHWOpTimeoutOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_HWOPTIMEOUT *psHWOpTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *psHWOpTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_HWOPTIMEOUT *) IMG_OFFSET_ADDR(psHWOpTimeoutOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psHWOpTimeoutIN);
+
+	psHWOpTimeoutOUT->eError = PVRSRVHWOpTimeoutKM(psConnection, OSGetDevNode(psConnection));
+
+	return 0;
+}
+
+static_assert(RGXFW_ALIGN_CHECKS_UM_MAX <= IMG_UINT32_MAX,
+	      "RGXFW_ALIGN_CHECKS_UM_MAX must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeAlignmentCheck(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psAlignmentCheckIN_UI8,
+			   IMG_UINT8 * psAlignmentCheckOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *psAlignmentCheckIN =
+	    (PVRSRV_BRIDGE_IN_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *psAlignmentCheckOUT =
+	    (PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK *) IMG_OFFSET_ADDR(psAlignmentCheckOUT_UI8, 0);
+
+	IMG_UINT32 *ui32AlignChecksInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) + 0;
+
+	if (unlikely(psAlignmentCheckIN->ui32AlignChecksSize > RGXFW_ALIGN_CHECKS_UM_MAX))
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto AlignmentCheck_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psAlignmentCheckOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto AlignmentCheck_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psAlignmentCheckIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psAlignmentCheckIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psAlignmentCheckOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto AlignmentCheck_exit;
+			}
+		}
+	}
+
+	if (psAlignmentCheckIN->ui32AlignChecksSize != 0)
+	{
+		ui32AlignChecksInt =
+		    (IMG_UINT32 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32);
+	}
+
+	/* Copy the data over */
+	if (psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, ui32AlignChecksInt,
+		     (const void __user *)psAlignmentCheckIN->pui32AlignChecks,
+		     psAlignmentCheckIN->ui32AlignChecksSize * sizeof(IMG_UINT32)) != PVRSRV_OK)
+		{
+			psAlignmentCheckOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto AlignmentCheck_exit;
+		}
+	}
+
+	psAlignmentCheckOUT->eError =
+	    PVRSRVAlignmentCheckKM(psConnection, OSGetDevNode(psConnection),
+				   psAlignmentCheckIN->ui32AlignChecksSize, ui32AlignChecksInt);
+
+AlignmentCheck_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psAlignmentCheckOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeGetDeviceStatus(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psGetDeviceStatusIN_UI8,
+			    IMG_UINT8 * psGetDeviceStatusOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETDEVICESTATUS *psGetDeviceStatusIN =
+	    (PVRSRV_BRIDGE_IN_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *psGetDeviceStatusOUT =
+	    (PVRSRV_BRIDGE_OUT_GETDEVICESTATUS *) IMG_OFFSET_ADDR(psGetDeviceStatusOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psGetDeviceStatusIN);
+
+	psGetDeviceStatusOUT->eError =
+	    PVRSRVGetDeviceStatusKM(psConnection, OSGetDevNode(psConnection),
+				    &psGetDeviceStatusOUT->ui32DeviceSatus);
+
+	return 0;
+}
+
+static_assert(8 <= IMG_UINT32_MAX, "8 must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeGetMultiCoreInfo(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psGetMultiCoreInfoIN_UI8,
+			     IMG_UINT8 * psGetMultiCoreInfoOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_GETMULTICOREINFO *psGetMultiCoreInfoIN =
+	    (PVRSRV_BRIDGE_IN_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *psGetMultiCoreInfoOUT =
+	    (PVRSRV_BRIDGE_OUT_GETMULTICOREINFO *) IMG_OFFSET_ADDR(psGetMultiCoreInfoOUT_UI8, 0);
+
+	IMG_UINT64 *pui64CapsInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize > 8)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	psGetMultiCoreInfoOUT->pui64Caps = psGetMultiCoreInfoIN->pui64Caps;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto GetMultiCoreInfo_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psGetMultiCoreInfoIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psGetMultiCoreInfoIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto GetMultiCoreInfo_exit;
+			}
+		}
+	}
+
+	if (psGetMultiCoreInfoIN->ui32CapsSize != 0)
+	{
+		pui64CapsInt = (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64);
+	}
+
+	psGetMultiCoreInfoOUT->eError =
+	    PVRSRVGetMultiCoreInfoKM(psConnection, OSGetDevNode(psConnection),
+				     psGetMultiCoreInfoIN->ui32CapsSize,
+				     &psGetMultiCoreInfoOUT->ui32NumCores, pui64CapsInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psGetMultiCoreInfoOUT->eError != PVRSRV_OK))
+	{
+		goto GetMultiCoreInfo_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64CapsInt) && ((psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psGetMultiCoreInfoOUT->pui64Caps, pui64CapsInt,
+		      (psGetMultiCoreInfoIN->ui32CapsSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psGetMultiCoreInfoOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto GetMultiCoreInfo_exit;
+		}
+	}
+
+GetMultiCoreInfo_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psGetMultiCoreInfoOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeEventObjectWaitTimeout(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psEventObjectWaitTimeoutIN_UI8,
+				   IMG_UINT8 * psEventObjectWaitTimeoutOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutIN =
+	    (PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *psEventObjectWaitTimeoutOUT =
+	    (PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT *)
+	    IMG_OFFSET_ADDR(psEventObjectWaitTimeoutOUT_UI8, 0);
+
+	IMG_HANDLE hOSEventKM = psEventObjectWaitTimeoutIN->hOSEventKM;
+	IMG_HANDLE hOSEventKMInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psEventObjectWaitTimeoutOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&hOSEventKMInt,
+				       hOSEventKM,
+				       PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT, IMG_TRUE);
+	if (unlikely(psEventObjectWaitTimeoutOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto EventObjectWaitTimeout_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psEventObjectWaitTimeoutOUT->eError =
+	    OSEventObjectWaitTimeout(hOSEventKMInt, psEventObjectWaitTimeoutIN->ui64uiTimeoutus);
+
+EventObjectWaitTimeout_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (hOSEventKMInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hOSEventKM, PVRSRV_HANDLE_TYPE_EVENT_OBJECT_CONNECT);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+static_assert(PVRSRV_PROCESS_STAT_TYPE_COUNT <= IMG_UINT32_MAX,
+	      "PVRSRV_PROCESS_STAT_TYPE_COUNT must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeFindProcessMemStats(IMG_UINT32 ui32DispatchTableEntry,
+				IMG_UINT8 * psFindProcessMemStatsIN_UI8,
+				IMG_UINT8 * psFindProcessMemStatsOUT_UI8,
+				CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *psFindProcessMemStatsIN =
+	    (PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsIN_UI8,
+								     0);
+	PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *psFindProcessMemStatsOUT =
+	    (PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS *) IMG_OFFSET_ADDR(psFindProcessMemStatsOUT_UI8,
+								      0);
+
+	IMG_UINT64 *pui64MemStatsArrayInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) + 0;
+
+	if (psFindProcessMemStatsIN->ui32ArrSize > PVRSRV_PROCESS_STAT_TYPE_COUNT)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto FindProcessMemStats_exit;
+	}
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psFindProcessMemStatsOUT->pui64MemStatsArray = psFindProcessMemStatsIN->pui64MemStatsArray;
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto FindProcessMemStats_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psFindProcessMemStatsIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psFindProcessMemStatsIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto FindProcessMemStats_exit;
+			}
+		}
+	}
+
+	if (psFindProcessMemStatsIN->ui32ArrSize != 0)
+	{
+		pui64MemStatsArrayInt =
+		    (IMG_UINT64 *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64);
+	}
+
+	psFindProcessMemStatsOUT->eError =
+	    PVRSRVFindProcessMemStatsKM(psFindProcessMemStatsIN->ui32PID,
+					psFindProcessMemStatsIN->ui32ArrSize,
+					psFindProcessMemStatsIN->bbAllProcessStats,
+					pui64MemStatsArrayInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psFindProcessMemStatsOUT->eError != PVRSRV_OK))
+	{
+		goto FindProcessMemStats_exit;
+	}
+
+	/* If dest ptr is non-null and we have data to copy */
+	if ((pui64MemStatsArrayInt) &&
+	    ((psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64)) > 0))
+	{
+		if (unlikely
+		    (OSCopyToUser
+		     (NULL, (void __user *)psFindProcessMemStatsOUT->pui64MemStatsArray,
+		      pui64MemStatsArrayInt,
+		      (psFindProcessMemStatsIN->ui32ArrSize * sizeof(IMG_UINT64))) != PVRSRV_OK))
+		{
+			psFindProcessMemStatsOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto FindProcessMemStats_exit;
+		}
+	}
+
+FindProcessMemStats_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psFindProcessMemStatsOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static PVRSRV_ERROR _AcquireInfoPagepsPMRIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVReleaseInfoPageKM((PMR *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAcquireInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psAcquireInfoPageIN_UI8,
+			    IMG_UINT8 * psAcquireInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *psAcquireInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *psAcquireInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE *) IMG_OFFSET_ADDR(psAcquireInfoPageOUT_UI8, 0);
+
+	PMR *psPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAcquireInfoPageIN);
+
+	psAcquireInfoPageOUT->eError = PVRSRVAcquireInfoPageKM(&psPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psAcquireInfoPageOUT->eError =
+	    PVRSRVAllocHandleUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+				      &psAcquireInfoPageOUT->hPMR, (void *)psPMRInt,
+				      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT,
+				      PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+				      (PFN_HANDLE_RELEASE) & _AcquireInfoPagepsPMRIntRelease);
+	if (unlikely(psAcquireInfoPageOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto AcquireInfoPage_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+AcquireInfoPage_exit:
+
+	if (psAcquireInfoPageOUT->eError != PVRSRV_OK)
+	{
+		if (psPMRInt)
+		{
+			PVRSRVReleaseInfoPageKM(psPMRInt);
+		}
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeReleaseInfoPage(IMG_UINT32 ui32DispatchTableEntry,
+			    IMG_UINT8 * psReleaseInfoPageIN_UI8,
+			    IMG_UINT8 * psReleaseInfoPageOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *psReleaseInfoPageIN =
+	    (PVRSRV_BRIDGE_IN_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *psReleaseInfoPageOUT =
+	    (PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE *) IMG_OFFSET_ADDR(psReleaseInfoPageOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+	psReleaseInfoPageOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psProcessHandleBase->psHandleBase,
+					      (IMG_HANDLE) psReleaseInfoPageIN->hPMR,
+					      PVRSRV_HANDLE_TYPE_DEVMEM_MEM_IMPORT);
+	if (unlikely((psReleaseInfoPageOUT->eError != PVRSRV_OK) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psReleaseInfoPageOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s", __func__, PVRSRVGetErrorString(psReleaseInfoPageOUT->eError)));
+		UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+		goto ReleaseInfoPage_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psProcessHandleBase->psHandleBase);
+
+ReleaseInfoPage_exit:
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSRVCOREBridge(void);
+void DeinitSRVCOREBridge(void);
+
+/*
+ * Register all SRVCORE functions with services
+ */
+PVRSRV_ERROR InitSRVCOREBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT,
+			      PVRSRVBridgeConnect, NULL, sizeof(PVRSRV_BRIDGE_IN_CONNECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_CONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT,
+			      PVRSRVBridgeDisconnect, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_DISCONNECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT,
+			      PVRSRVBridgeAcquireGlobalEventObject, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT,
+			      PVRSRVBridgeReleaseGlobalEventObject, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEGLOBALEVENTOBJECT),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEGLOBALEVENTOBJECT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN,
+			      PVRSRVBridgeEventObjectOpen, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTOPEN),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTOPEN));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT,
+			      PVRSRVBridgeEventObjectWait, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAIT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAIT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE,
+			      PVRSRVBridgeEventObjectClose, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTCLOSE),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTCLOSE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO,
+			      PVRSRVBridgeDumpDebugInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_DUMPDEBUGINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_DUMPDEBUGINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED,
+			      PVRSRVBridgeGetDevClockSpeed, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVCLOCKSPEED));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT,
+			      PVRSRVBridgeHWOpTimeout, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_HWOPTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK,
+			      PVRSRVBridgeAlignmentCheck, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_ALIGNMENTCHECK),
+			      sizeof(PVRSRV_BRIDGE_OUT_ALIGNMENTCHECK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS,
+			      PVRSRVBridgeGetDeviceStatus, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_GETDEVICESTATUS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO,
+			      PVRSRVBridgeGetMultiCoreInfo, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_GETMULTICOREINFO),
+			      sizeof(PVRSRV_BRIDGE_OUT_GETMULTICOREINFO));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT,
+			      PVRSRVBridgeEventObjectWaitTimeout, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_EVENTOBJECTWAITTIMEOUT),
+			      sizeof(PVRSRV_BRIDGE_OUT_EVENTOBJECTWAITTIMEOUT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS,
+			      PVRSRVBridgeFindProcessMemStats, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FINDPROCESSMEMSTATS),
+			      sizeof(PVRSRV_BRIDGE_OUT_FINDPROCESSMEMSTATS));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE,
+			      PVRSRVBridgeAcquireInfoPage, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ACQUIREINFOPAGE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE,
+			      PVRSRVBridgeReleaseInfoPage, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_RELEASEINFOPAGE),
+			      sizeof(PVRSRV_BRIDGE_OUT_RELEASEINFOPAGE));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all srvcore functions with services
+ */
+void DeinitSRVCOREBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_CONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DISCONNECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_ACQUIREGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_RELEASEGLOBALEVENTOBJECT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTOPEN);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAIT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTCLOSE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_DUMPDEBUGINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVCLOCKSPEED);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_HWOPTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ALIGNMENTCHECK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETDEVICESTATUS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_GETMULTICOREINFO);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE,
+				PVRSRV_BRIDGE_SRVCORE_EVENTOBJECTWAITTIMEOUT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_FINDPROCESSMEMSTATS);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_ACQUIREINFOPAGE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SRVCORE, PVRSRV_BRIDGE_SRVCORE_RELEASEINFOPAGE);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_sync_bridge.c b/drivers/gpu/drm/img-rogue/server_sync_bridge.c
new file mode 100644
index 000000000000..a0d9210086c9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_sync_bridge.c
@@ -0,0 +1,755 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for sync
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for sync
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+#include "pdump.h"
+#include "pvrsrv_sync_km.h"
+#include "sync_fallback_server.h"
+#include "sync_checkpoint.h"
+
+#include "common_sync_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static PVRSRV_ERROR _AllocSyncPrimitiveBlockpsSyncHandleIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVFreeSyncPrimitiveBlockKM((SYNC_PRIMITIVE_BLOCK *) pvData);
+	return eError;
+}
+
+static IMG_INT
+PVRSRVBridgeAllocSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockIN_UI8,
+				    IMG_UINT8 * psAllocSyncPrimitiveBlockOUT_UI8,
+				    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *psAllocSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psAllocSyncPrimitiveBlockOUT_UI8, 0);
+
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+	PMR *pshSyncPMRInt = NULL;
+
+	PVR_UNREFERENCED_PARAMETER(psAllocSyncPrimitiveBlockIN);
+
+	psAllocSyncPrimitiveBlockOUT->hSyncHandle = NULL;
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSyncPrimitiveBlockKM(psConnection, OSGetDevNode(psConnection),
+					    &psSyncHandleInt,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimVAddr,
+					    &psAllocSyncPrimitiveBlockOUT->ui32SyncPrimBlockSize,
+					    &pshSyncPMRInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psAllocSyncPrimitiveBlockOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+									 &psAllocSyncPrimitiveBlockOUT->
+									 hSyncHandle,
+									 (void *)psSyncHandleInt,
+									 PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK,
+									 PVRSRV_HANDLE_ALLOC_FLAG_MULTI,
+									 (PFN_HANDLE_RELEASE) &
+									 _AllocSyncPrimitiveBlockpsSyncHandleIntRelease);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	psAllocSyncPrimitiveBlockOUT->eError =
+	    PVRSRVAllocSubHandleUnlocked(psConnection->psHandleBase,
+					 &psAllocSyncPrimitiveBlockOUT->hhSyncPMR,
+					 (void *)pshSyncPMRInt,
+					 PVRSRV_HANDLE_TYPE_PMR_LOCAL_EXPORT_HANDLE,
+					 PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+					 psAllocSyncPrimitiveBlockOUT->hSyncHandle);
+	if (unlikely(psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto AllocSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+AllocSyncPrimitiveBlock_exit:
+
+	if (psAllocSyncPrimitiveBlockOUT->eError != PVRSRV_OK)
+	{
+		if (psAllocSyncPrimitiveBlockOUT->hSyncHandle)
+		{
+			PVRSRV_ERROR eError;
+
+			/* Lock over handle creation cleanup. */
+			LockHandle(psConnection->psHandleBase);
+
+			eError = PVRSRVDestroyHandleUnlocked(psConnection->psHandleBase,
+							     (IMG_HANDLE)
+							     psAllocSyncPrimitiveBlockOUT->
+							     hSyncHandle,
+							     PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+			if (unlikely((eError != PVRSRV_OK) && (eError != PVRSRV_ERROR_RETRY)))
+			{
+				PVR_DPF((PVR_DBG_ERROR,
+					 "%s: %s", __func__, PVRSRVGetErrorString(eError)));
+			}
+			/* Releasing the handle should free/destroy/release the resource.
+			 * This should never fail... */
+			PVR_ASSERT((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_RETRY));
+
+			/* Release now we have cleaned up creation handles. */
+			UnlockHandle(psConnection->psHandleBase);
+
+		}
+
+		else if (psSyncHandleInt)
+		{
+			PVRSRVFreeSyncPrimitiveBlockKM(psSyncHandleInt);
+		}
+
+	}
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeFreeSyncPrimitiveBlock(IMG_UINT32 ui32DispatchTableEntry,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockIN_UI8,
+				   IMG_UINT8 * psFreeSyncPrimitiveBlockOUT_UI8,
+				   CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockIN =
+	    (PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *psFreeSyncPrimitiveBlockOUT =
+	    (PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK *)
+	    IMG_OFFSET_ADDR(psFreeSyncPrimitiveBlockOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psFreeSyncPrimitiveBlockOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psFreeSyncPrimitiveBlockIN->hSyncHandle,
+					      PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	if (unlikely((psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_OK) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psFreeSyncPrimitiveBlockOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psFreeSyncPrimitiveBlockOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto FreeSyncPrimitiveBlock_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+FreeSyncPrimitiveBlock_exit:
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncPrimSet(IMG_UINT32 ui32DispatchTableEntry,
+			IMG_UINT8 * psSyncPrimSetIN_UI8,
+			IMG_UINT8 * psSyncPrimSetOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMSET *psSyncPrimSetIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMSET *psSyncPrimSetOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMSET *) IMG_OFFSET_ADDR(psSyncPrimSetOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimSetIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimSetOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimSetOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimSet_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimSetOUT->eError =
+	    PVRSRVSyncPrimSetKM(psSyncHandleInt,
+				psSyncPrimSetIN->ui32Index, psSyncPrimSetIN->ui32Value);
+
+SyncPrimSet_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDump(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncPrimPDumpIN_UI8,
+			  IMG_UINT8 * psSyncPrimPDumpOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *psSyncPrimPDumpIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *psSyncPrimPDumpOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP *) IMG_OFFSET_ADDR(psSyncPrimPDumpOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDump_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpOUT->eError =
+	    PVRSRVSyncPrimPDumpKM(psSyncHandleInt, psSyncPrimPDumpIN->ui32Offset);
+
+SyncPrimPDump_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDump NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpValue(IMG_UINT32 ui32DispatchTableEntry,
+			       IMG_UINT8 * psSyncPrimPDumpValueIN_UI8,
+			       IMG_UINT8 * psSyncPrimPDumpValueOUT_UI8,
+			       CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *psSyncPrimPDumpValueOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE *) IMG_OFFSET_ADDR(psSyncPrimPDumpValueOUT_UI8,
+								     0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpValueIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpValueOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpValue_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpValueOUT->eError =
+	    PVRSRVSyncPrimPDumpValueKM(psSyncHandleInt,
+				       psSyncPrimPDumpValueIN->ui32Offset,
+				       psSyncPrimPDumpValueIN->ui32Value);
+
+SyncPrimPDumpValue_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpValue NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpPolIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpPolOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *psSyncPrimPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL *) IMG_OFFSET_ADDR(psSyncPrimPDumpPolOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpPolIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpPolOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpPol_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpPolOUT->eError =
+	    PVRSRVSyncPrimPDumpPolKM(psSyncHandleInt,
+				     psSyncPrimPDumpPolIN->ui32Offset,
+				     psSyncPrimPDumpPolIN->ui32Value,
+				     psSyncPrimPDumpPolIN->ui32Mask,
+				     psSyncPrimPDumpPolIN->eOperator,
+				     psSyncPrimPDumpPolIN->uiPDumpFlags);
+
+SyncPrimPDumpPol_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpPol NULL
+#endif
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncPrimPDumpCBP(IMG_UINT32 ui32DispatchTableEntry,
+			     IMG_UINT8 * psSyncPrimPDumpCBPIN_UI8,
+			     IMG_UINT8 * psSyncPrimPDumpCBPOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPIN =
+	    (PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *psSyncPrimPDumpCBPOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP *) IMG_OFFSET_ADDR(psSyncPrimPDumpCBPOUT_UI8, 0);
+
+	IMG_HANDLE hSyncHandle = psSyncPrimPDumpCBPIN->hSyncHandle;
+	SYNC_PRIMITIVE_BLOCK *psSyncHandleInt = NULL;
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&psSyncHandleInt,
+				       hSyncHandle,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncPrimPDumpCBPOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncPrimPDumpCBP_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncPrimPDumpCBPOUT->eError =
+	    PVRSRVSyncPrimPDumpCBPKM(psSyncHandleInt,
+				     psSyncPrimPDumpCBPIN->ui32Offset,
+				     psSyncPrimPDumpCBPIN->uiWriteOffset,
+				     psSyncPrimPDumpCBPIN->uiPacketSize,
+				     psSyncPrimPDumpCBPIN->uiBufferSize);
+
+SyncPrimPDumpCBP_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (psSyncHandleInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hSyncHandle, PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncPrimPDumpCBP NULL
+#endif
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncAllocEvent(IMG_UINT32 ui32DispatchTableEntry,
+			   IMG_UINT8 * psSyncAllocEventIN_UI8,
+			   IMG_UINT8 * psSyncAllocEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *psSyncAllocEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *psSyncAllocEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT *) IMG_OFFSET_ADDR(psSyncAllocEventOUT_UI8, 0);
+
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncAllocEventIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncAllocEvent_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncAllocEventOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncAllocEvent_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncAllocEventIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncAllocEventIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncAllocEventOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncAllocEvent_exit;
+			}
+		}
+	}
+
+	if (psSyncAllocEventIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncAllocEventIN->puiClassName,
+		     psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncAllocEventOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncAllocEvent_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncAllocEventIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	psSyncAllocEventOUT->eError =
+	    PVRSRVSyncAllocEventKM(psConnection, OSGetDevNode(psConnection),
+				   psSyncAllocEventIN->bServerSync,
+				   psSyncAllocEventIN->ui32FWAddr,
+				   psSyncAllocEventIN->ui32ClassNameSize, uiClassNameInt);
+
+SyncAllocEvent_exit:
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncAllocEventOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+static IMG_INT
+PVRSRVBridgeSyncFreeEvent(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncFreeEventIN_UI8,
+			  IMG_UINT8 * psSyncFreeEventOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCFREEEVENT *psSyncFreeEventIN =
+	    (PVRSRV_BRIDGE_IN_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *psSyncFreeEventOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCFREEEVENT *) IMG_OFFSET_ADDR(psSyncFreeEventOUT_UI8, 0);
+
+	psSyncFreeEventOUT->eError =
+	    PVRSRVSyncFreeEventKM(psConnection, OSGetDevNode(psConnection),
+				  psSyncFreeEventIN->ui32FWAddr);
+
+	return 0;
+}
+
+#if defined(PDUMP)
+
+static IMG_INT
+PVRSRVBridgeSyncCheckpointSignalledPDumpPol(IMG_UINT32 ui32DispatchTableEntry,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolIN_UI8,
+					    IMG_UINT8 * psSyncCheckpointSignalledPDumpPolOUT_UI8,
+					    CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolIN =
+	    (PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *psSyncCheckpointSignalledPDumpPolOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL *)
+	    IMG_OFFSET_ADDR(psSyncCheckpointSignalledPDumpPolOUT_UI8, 0);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	psSyncCheckpointSignalledPDumpPolOUT->eError =
+	    PVRSRVSyncCheckpointSignalledPDumpPolKM(psSyncCheckpointSignalledPDumpPolIN->hFence);
+
+	return 0;
+}
+
+#else
+#define PVRSRVBridgeSyncCheckpointSignalledPDumpPol NULL
+#endif
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCBridge(void);
+void DeinitSYNCBridge(void);
+
+/*
+ * Register all SYNC functions with services
+ */
+PVRSRV_ERROR InitSYNCBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeAllocSyncPrimitiveBlock, NULL, 0,
+			      sizeof(PVRSRV_BRIDGE_OUT_ALLOCSYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK,
+			      PVRSRVBridgeFreeSyncPrimitiveBlock, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_FREESYNCPRIMITIVEBLOCK),
+			      sizeof(PVRSRV_BRIDGE_OUT_FREESYNCPRIMITIVEBLOCK));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET,
+			      PVRSRVBridgeSyncPrimSet, NULL, sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMSET),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMSET));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP,
+			      PVRSRVBridgeSyncPrimPDump, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE,
+			      PVRSRVBridgeSyncPrimPDumpValue, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPVALUE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPVALUE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL,
+			      PVRSRVBridgeSyncPrimPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPPOL));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP,
+			      PVRSRVBridgeSyncPrimPDumpCBP, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCPRIMPDUMPCBP),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCPRIMPDUMPCBP));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT,
+			      PVRSRVBridgeSyncAllocEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCALLOCEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCALLOCEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT,
+			      PVRSRVBridgeSyncFreeEvent, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCFREEEVENT),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCFREEEVENT));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+			      PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL,
+			      PVRSRVBridgeSyncCheckpointSignalledPDumpPol, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCCHECKPOINTSIGNALLEDPDUMPPOL),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCCHECKPOINTSIGNALLEDPDUMPPOL));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all sync functions with services
+ */
+void DeinitSYNCBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_ALLOCSYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_FREESYNCPRIMITIVEBLOCK);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMSET);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPVALUE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPPOL);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCPRIMPDUMPCBP);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCALLOCEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC, PVRSRV_BRIDGE_SYNC_SYNCFREEEVENT);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNC,
+				PVRSRV_BRIDGE_SYNC_SYNCCHECKPOINTSIGNALLEDPDUMPPOL);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/server_synctracking_bridge.c b/drivers/gpu/drm/img-rogue/server_synctracking_bridge.c
new file mode 100644
index 000000000000..06757e5ef8d3
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/server_synctracking_bridge.c
@@ -0,0 +1,329 @@
+/*******************************************************************************
+@File
+@Title          Server bridge for synctracking
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side of the bridge for synctracking
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*******************************************************************************/
+
+#include <linux/uaccess.h>
+
+#include "img_defs.h"
+
+#include "sync.h"
+#include "sync_server.h"
+
+#include "common_synctracking_bridge.h"
+
+#include "allocmem.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+#include "srvcore.h"
+#include "handle.h"
+
+#include <linux/slab.h>
+
+/* ***************************************************************************
+ * Server-side bridge entry points
+ */
+
+static IMG_INT
+PVRSRVBridgeSyncRecordRemoveByHandle(IMG_UINT32 ui32DispatchTableEntry,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleIN_UI8,
+				     IMG_UINT8 * psSyncRecordRemoveByHandleOUT_UI8,
+				     CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *psSyncRecordRemoveByHandleOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE *)
+	    IMG_OFFSET_ADDR(psSyncRecordRemoveByHandleOUT_UI8, 0);
+
+	/* Lock over handle destruction. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordRemoveByHandleOUT->eError =
+	    PVRSRVDestroyHandleStagedUnlocked(psConnection->psHandleBase,
+					      (IMG_HANDLE) psSyncRecordRemoveByHandleIN->hhRecord,
+					      PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE);
+	if (unlikely((psSyncRecordRemoveByHandleOUT->eError != PVRSRV_OK) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_KERNEL_CCB_FULL) &&
+		     (psSyncRecordRemoveByHandleOUT->eError != PVRSRV_ERROR_RETRY)))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+			 "%s: %s",
+			 __func__, PVRSRVGetErrorString(psSyncRecordRemoveByHandleOUT->eError)));
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordRemoveByHandle_exit;
+	}
+
+	/* Release now we have destroyed handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordRemoveByHandle_exit:
+
+	return 0;
+}
+
+static PVRSRV_ERROR _SyncRecordAddpshRecordIntRelease(void *pvData)
+{
+	PVRSRV_ERROR eError;
+	eError = PVRSRVSyncRecordRemoveByHandleKM((SYNC_RECORD_HANDLE) pvData);
+	return eError;
+}
+
+static_assert(PVRSRV_SYNC_NAME_LENGTH <= IMG_UINT32_MAX,
+	      "PVRSRV_SYNC_NAME_LENGTH must not be larger than IMG_UINT32_MAX");
+
+static IMG_INT
+PVRSRVBridgeSyncRecordAdd(IMG_UINT32 ui32DispatchTableEntry,
+			  IMG_UINT8 * psSyncRecordAddIN_UI8,
+			  IMG_UINT8 * psSyncRecordAddOUT_UI8, CONNECTION_DATA * psConnection)
+{
+	PVRSRV_BRIDGE_IN_SYNCRECORDADD *psSyncRecordAddIN =
+	    (PVRSRV_BRIDGE_IN_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddIN_UI8, 0);
+	PVRSRV_BRIDGE_OUT_SYNCRECORDADD *psSyncRecordAddOUT =
+	    (PVRSRV_BRIDGE_OUT_SYNCRECORDADD *) IMG_OFFSET_ADDR(psSyncRecordAddOUT_UI8, 0);
+
+	SYNC_RECORD_HANDLE pshRecordInt = NULL;
+	IMG_HANDLE hhServerSyncPrimBlock = psSyncRecordAddIN->hhServerSyncPrimBlock;
+	SYNC_PRIMITIVE_BLOCK *pshServerSyncPrimBlockInt = NULL;
+	IMG_CHAR *uiClassNameInt = NULL;
+
+	IMG_UINT32 ui32NextOffset = 0;
+	IMG_BYTE *pArrayArgsBuffer = NULL;
+	IMG_BOOL bHaveEnoughSpace = IMG_FALSE;
+
+	IMG_UINT32 ui32BufferSize = 0;
+	IMG_UINT64 ui64BufferSize =
+	    ((IMG_UINT64) psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) + 0;
+
+	if (unlikely(psSyncRecordAddIN->ui32ClassNameSize > PVRSRV_SYNC_NAME_LENGTH))
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_ARRAY_SIZE_TOO_BIG;
+		goto SyncRecordAdd_exit;
+	}
+
+	if (ui64BufferSize > IMG_UINT32_MAX)
+	{
+		psSyncRecordAddOUT->eError = PVRSRV_ERROR_BRIDGE_BUFFER_TOO_SMALL;
+		goto SyncRecordAdd_exit;
+	}
+
+	ui32BufferSize = (IMG_UINT32) ui64BufferSize;
+
+	if (ui32BufferSize != 0)
+	{
+		/* Try to use remainder of input buffer for copies if possible, word-aligned for safety. */
+		IMG_UINT32 ui32InBufferOffset =
+		    PVR_ALIGN(sizeof(*psSyncRecordAddIN), sizeof(unsigned long));
+		IMG_UINT32 ui32InBufferExcessSize =
+		    ui32InBufferOffset >=
+		    PVRSRV_MAX_BRIDGE_IN_SIZE ? 0 : PVRSRV_MAX_BRIDGE_IN_SIZE - ui32InBufferOffset;
+
+		bHaveEnoughSpace = ui32BufferSize <= ui32InBufferExcessSize;
+		if (bHaveEnoughSpace)
+		{
+			IMG_BYTE *pInputBuffer = (IMG_BYTE *) (void *)psSyncRecordAddIN;
+
+			pArrayArgsBuffer = &pInputBuffer[ui32InBufferOffset];
+		}
+		else
+		{
+			pArrayArgsBuffer = OSAllocMemNoStats(ui32BufferSize);
+
+			if (!pArrayArgsBuffer)
+			{
+				psSyncRecordAddOUT->eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+				goto SyncRecordAdd_exit;
+			}
+		}
+	}
+
+	if (psSyncRecordAddIN->ui32ClassNameSize != 0)
+	{
+		uiClassNameInt = (IMG_CHAR *) IMG_OFFSET_ADDR(pArrayArgsBuffer, ui32NextOffset);
+		ui32NextOffset += psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR);
+	}
+
+	/* Copy the data over */
+	if (psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR) > 0)
+	{
+		if (OSCopyFromUser
+		    (NULL, uiClassNameInt, (const void __user *)psSyncRecordAddIN->puiClassName,
+		     psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) != PVRSRV_OK)
+		{
+			psSyncRecordAddOUT->eError = PVRSRV_ERROR_INVALID_PARAMS;
+
+			goto SyncRecordAdd_exit;
+		}
+		((IMG_CHAR *)
+		 uiClassNameInt)[(psSyncRecordAddIN->ui32ClassNameSize * sizeof(IMG_CHAR)) - 1] =
+       '\0';
+	}
+
+	/* Lock over handle lookup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Look up the address from the handle */
+	psSyncRecordAddOUT->eError =
+	    PVRSRVLookupHandleUnlocked(psConnection->psHandleBase,
+				       (void **)&pshServerSyncPrimBlockInt,
+				       hhServerSyncPrimBlock,
+				       PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK, IMG_TRUE);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+	/* Release now we have looked up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError =
+	    PVRSRVSyncRecordAddKM(psConnection, OSGetDevNode(psConnection),
+				  &pshRecordInt,
+				  pshServerSyncPrimBlockInt,
+				  psSyncRecordAddIN->ui32ui32FwBlockAddr,
+				  psSyncRecordAddIN->ui32ui32SyncOffset,
+				  psSyncRecordAddIN->bbServerSync,
+				  psSyncRecordAddIN->ui32ClassNameSize, uiClassNameInt);
+	/* Exit early if bridged call fails */
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Lock over handle creation. */
+	LockHandle(psConnection->psHandleBase);
+
+	psSyncRecordAddOUT->eError = PVRSRVAllocHandleUnlocked(psConnection->psHandleBase,
+							       &psSyncRecordAddOUT->hhRecord,
+							       (void *)pshRecordInt,
+							       PVRSRV_HANDLE_TYPE_SYNC_RECORD_HANDLE,
+							       PVRSRV_HANDLE_ALLOC_FLAG_NONE,
+							       (PFN_HANDLE_RELEASE) &
+							       _SyncRecordAddpshRecordIntRelease);
+	if (unlikely(psSyncRecordAddOUT->eError != PVRSRV_OK))
+	{
+		UnlockHandle(psConnection->psHandleBase);
+		goto SyncRecordAdd_exit;
+	}
+
+	/* Release now we have created handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+SyncRecordAdd_exit:
+
+	/* Lock over handle lookup cleanup. */
+	LockHandle(psConnection->psHandleBase);
+
+	/* Unreference the previously looked up handle */
+	if (pshServerSyncPrimBlockInt)
+	{
+		PVRSRVReleaseHandleUnlocked(psConnection->psHandleBase,
+					    hhServerSyncPrimBlock,
+					    PVRSRV_HANDLE_TYPE_SYNC_PRIMITIVE_BLOCK);
+	}
+	/* Release now we have cleaned up look up handles. */
+	UnlockHandle(psConnection->psHandleBase);
+
+	if (psSyncRecordAddOUT->eError != PVRSRV_OK)
+	{
+		if (pshRecordInt)
+		{
+			PVRSRVSyncRecordRemoveByHandleKM(pshRecordInt);
+		}
+	}
+
+	/* Allocated space should be equal to the last updated offset */
+#ifdef PVRSRV_NEED_PVR_ASSERT
+	if (psSyncRecordAddOUT->eError == PVRSRV_OK)
+		PVR_ASSERT(ui32BufferSize == ui32NextOffset);
+#endif /* PVRSRV_NEED_PVR_ASSERT */
+
+	if (!bHaveEnoughSpace && pArrayArgsBuffer)
+		OSFreeMemNoStats(pArrayArgsBuffer);
+
+	return 0;
+}
+
+/* ***************************************************************************
+ * Server bridge dispatch related glue
+ */
+
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void);
+void DeinitSYNCTRACKINGBridge(void);
+
+/*
+ * Register all SYNCTRACKING functions with services
+ */
+PVRSRV_ERROR InitSYNCTRACKINGBridge(void)
+{
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+			      PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE,
+			      PVRSRVBridgeSyncRecordRemoveByHandle, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDREMOVEBYHANDLE),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDREMOVEBYHANDLE));
+
+	SetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING, PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD,
+			      PVRSRVBridgeSyncRecordAdd, NULL,
+			      sizeof(PVRSRV_BRIDGE_IN_SYNCRECORDADD),
+			      sizeof(PVRSRV_BRIDGE_OUT_SYNCRECORDADD));
+
+	return PVRSRV_OK;
+}
+
+/*
+ * Unregister all synctracking functions with services
+ */
+void DeinitSYNCTRACKINGBridge(void)
+{
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDREMOVEBYHANDLE);
+
+	UnsetDispatchTableEntry(PVRSRV_BRIDGE_SYNCTRACKING,
+				PVRSRV_BRIDGE_SYNCTRACKING_SYNCRECORDADD);
+
+}
diff --git a/drivers/gpu/drm/img-rogue/services_kernel_client.h b/drivers/gpu/drm/img-rogue/services_kernel_client.h
new file mode 100644
index 000000000000..3cfa23116a20
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/services_kernel_client.h
@@ -0,0 +1,306 @@
+/*************************************************************************/ /*!
+@File           services_kernel_client.h
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* This file contains a partial redefinition of the PowerVR Services 5
+ * interface for use by components which are checkpatch clean. This
+ * header is included by the unrefined, non-checkpatch clean headers
+ * to ensure that prototype/typedef/macro changes break the build.
+ */
+
+#ifndef __SERVICES_KERNEL_CLIENT__
+#define __SERVICES_KERNEL_CLIENT__
+
+#include "pvrsrv_error.h"
+
+#include <linux/types.h>
+
+#include "pvrsrv_sync_km.h"
+#include "sync_checkpoint_external.h"
+
+/* included for the define PVRSRV_LINUX_DEV_INIT_ON_PROBE */
+#include "pvr_drm.h"
+
+#include "pvr_dma_resv.h"
+
+#ifndef __pvrsrv_defined_struct_enum__
+
+/* sync_external.h */
+
+struct PVRSRV_CLIENT_SYNC_PRIM_TAG {
+	volatile __u32 *pui32LinAddr;
+};
+
+struct PVRSRV_CLIENT_SYNC_PRIM_OP {
+	__u32 ui32Flags;
+	struct pvrsrv_sync_prim *psSync;
+	__u32 ui32FenceValue;
+	__u32 ui32UpdateValue;
+};
+
+#else /* __pvrsrv_defined_struct_enum__ */
+
+struct PVRSRV_CLIENT_SYNC_PRIM_TAG;
+struct PVRSRV_CLIENT_SYNC_PRIM_OP;
+
+#endif /* __pvrsrv_defined_struct_enum__ */
+
+struct _PMR_;
+struct _PVRSRV_DEVICE_NODE_;
+struct dma_buf;
+struct dma_resv;
+struct SYNC_PRIM_CONTEXT_TAG;
+
+/* pvr_notifier.h */
+
+#ifndef CMDCOMPNOTIFY_PFN
+typedef void (*PFN_CMDCOMP_NOTIFY)(void *hCmdCompHandle);
+#define CMDCOMPNOTIFY_PFN
+#endif
+enum PVRSRV_ERROR_TAG PVRSRVRegisterCmdCompleteNotify(void **phNotify,
+	PFN_CMDCOMP_NOTIFY pfnCmdCompleteNotify, void *hPrivData);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterCmdCompleteNotify(void *hNotify);
+void PVRSRVCheckStatus(void *hCmdCompCallerHandle);
+
+#define DEBUG_REQUEST_DC               0
+#define DEBUG_REQUEST_SYNCTRACKING     1
+#define DEBUG_REQUEST_SRV              2
+#define DEBUG_REQUEST_SYS              3
+#define DEBUG_REQUEST_RGX              4
+#define DEBUG_REQUEST_ANDROIDSYNC      5
+#define DEBUG_REQUEST_LINUXFENCE       6
+#define DEBUG_REQUEST_SYNCCHECKPOINT   7
+#define DEBUG_REQUEST_HTB              8
+#define DEBUG_REQUEST_APPHINT          9
+#define DEBUG_REQUEST_FALLBACKSYNC     10
+
+#define DEBUG_REQUEST_VERBOSITY_LOW    0
+#define DEBUG_REQUEST_VERBOSITY_MEDIUM 1
+#define DEBUG_REQUEST_VERBOSITY_HIGH   2
+#define DEBUG_REQUEST_VERBOSITY_MAX    DEBUG_REQUEST_VERBOSITY_HIGH
+
+#define DD_VERB_LVL_ENABLED(_verbLvl, _verbLvlChk) ((_verbLvl) >= (_verbLvlChk))
+
+#ifndef DBGNOTIFY_PFNS
+typedef void (DUMPDEBUG_PRINTF_FUNC)(void *pvDumpDebugFile,
+	const char *fmt, ...) __printf(2, 3);
+typedef void (*PFN_DBGREQ_NOTIFY) (void *hDebugRequestHandle,
+	__u32 ui32VerbLevel,
+	DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	void *pvDumpDebugFile);
+#define DBGNOTIFY_PFNS
+#endif
+enum PVRSRV_ERROR_TAG PVRSRVRegisterDeviceDbgRequestNotify(void **phNotify,
+	struct _PVRSRV_DEVICE_NODE_ *psDevNode,
+	PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	__u32 ui32RequesterID,
+	void *hDbgRequestHandle);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterDeviceDbgRequestNotify(void *hNotify);
+enum PVRSRV_ERROR_TAG PVRSRVRegisterDriverDbgRequestNotify(void **phNotify,
+	PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+	__u32 ui32RequesterID,
+	void *hDbgRequestHandle);
+enum PVRSRV_ERROR_TAG PVRSRVUnregisterDriverDbgRequestNotify(void *hNotify);
+
+/* physmem_dmabuf.h */
+
+struct dma_buf *PhysmemGetDmaBuf(struct _PMR_ *psPMR);
+
+struct dma_resv *PhysmemGetDmaResv(struct _PMR_ *psPMR);
+
+/* pvrsrv.h */
+
+enum PVRSRV_ERROR_TAG PVRSRVAcquireGlobalEventObjectKM(void **phGlobalEventObject);
+enum PVRSRV_ERROR_TAG PVRSRVReleaseGlobalEventObjectKM(void *hGlobalEventObject);
+
+/* sync.h */
+
+enum PVRSRV_ERROR_TAG SyncPrimContextCreate(
+	struct _PVRSRV_DEVICE_NODE_ *psDevConnection,
+	struct SYNC_PRIM_CONTEXT_TAG **phSyncPrimContext);
+void SyncPrimContextDestroy(struct SYNC_PRIM_CONTEXT_TAG *hSyncPrimContext);
+
+enum PVRSRV_ERROR_TAG SyncPrimAlloc(struct SYNC_PRIM_CONTEXT_TAG *hSyncPrimContext,
+	struct PVRSRV_CLIENT_SYNC_PRIM_TAG **ppsSync, const char *pszClassName);
+enum PVRSRV_ERROR_TAG SyncPrimFree(struct PVRSRV_CLIENT_SYNC_PRIM_TAG *psSync);
+enum PVRSRV_ERROR_TAG SyncPrimGetFirmwareAddr(
+	struct PVRSRV_CLIENT_SYNC_PRIM_TAG *psSync,
+	__u32 *sync_addr);
+
+/* osfunc.h */
+enum PVRSRV_ERROR_TAG OSEventObjectWait(void *hOSEventKM);
+enum PVRSRV_ERROR_TAG OSEventObjectOpen(void *hEventObject, void **phOSEventKM);
+enum PVRSRV_ERROR_TAG OSEventObjectClose(void *hOSEventKM);
+__u32 OSGetCurrentClientProcessIDKM(void);
+__u32 OSStringUINT32ToStr(char *pszBuf, size_t uSize, __u32 ui32Num);
+
+/* srvkm.h */
+
+enum PVRSRV_ERROR_TAG PVRSRVCommonDeviceCreate(void *pvOSDevice,
+	int i32KernelDeviceID,
+	struct _PVRSRV_DEVICE_NODE_ **ppsDeviceNode);
+void PVRSRVCommonDeviceDestroy(
+	struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+const char *PVRSRVGetErrorString(enum PVRSRV_ERROR_TAG eError);
+#if (PVRSRV_DEVICE_INIT_MODE == PVRSRV_LINUX_DEV_INIT_ON_PROBE)
+enum PVRSRV_ERROR_TAG PVRSRVCommonDeviceInitialise(
+	struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN)(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext, PVRSRV_FENCE fence, u32 *nr_checkpoints, PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid);
+
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN)(PVRSRV_FENCE export_fence, PSYNC_CHECKPOINT_CONTEXT checkpoint_context, PSYNC_CHECKPOINT *checkpoint_handle);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN)(PVRSRV_FENCE export_fence);
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN)(
+		struct _PVRSRV_DEVICE_NODE_ *device,
+		const char *fence_name,
+		PVRSRV_TIMELINE timeline,
+		PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+		PVRSRV_FENCE *new_fence,
+		u64 *fence_uid,
+		void **fence_finalise_data,
+		PSYNC_CHECKPOINT *new_checkpoint_handle,
+		void **timeline_update_sync,
+		__u32 *timeline_update_value);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN)(PVRSRV_FENCE fence_to_rollback, void *finalise_data);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN)(PVRSRV_FENCE fence_to_finalise, void *finalise_data);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef __u32 (*PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN)(__u32 num_ufos, __u32 *vaddrs);
+#endif
+
+#ifndef CHECKPOINT_PFNS
+typedef bool (*PFN_SYNC_CHECKPOINT_UFO_HAS_SIGNALLED_FN)(
+	__u32 ui32FwAddr, __u32 ui32Value);
+typedef enum PVRSRV_ERROR_TAG (*PFN_SYNC_CHECKPOINT_SIGNAL_WAITERS_FN)(void);
+typedef void(*PFN_SYNC_CHECKPOINT_CHECK_STATE_FN)(void);
+#if defined(PDUMP)
+typedef PVRSRV_ERROR(*PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN)(PVRSRV_FENCE iFence,
+						IMG_UINT32 *puiNumCheckpoints,
+		                                PSYNC_CHECKPOINT **papsCheckpoints);
+#endif
+#endif
+
+/* This is the function that kick code will call in a NO_HARDWARE build only after
+ * sync checkpoints have been manually signalled, to allow the OS native sync
+ * implementation to update its timelines (as the usual callback notification
+ * of signalled checkpoints is not supported for NO_HARDWARE).
+ */
+#ifndef CHECKPOINT_PFNS
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN)(void *private_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN)(PVRSRV_FENCE fence_to_signal);
+typedef void (*PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN)(void *mem_ptr);
+
+#define SYNC_CHECKPOINT_IMPL_MAX_STRLEN 20
+
+typedef struct {
+	PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN pfnFenceResolve;
+	PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN pfnFenceCreate;
+	PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN pfnFenceDataRollback;
+	PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN pfnFenceFinalise;
+	PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN pfnNoHWUpdateTimelines;
+	PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN pfnNoHWSignalExpFence;
+	PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN pfnFreeCheckpointListMem;
+	PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN pfnDumpInfoOnStalledUFOs;
+	char pszImplName[SYNC_CHECKPOINT_IMPL_MAX_STRLEN];
+#if defined(PDUMP)
+	PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN pfnSyncFenceGetCheckpoints;
+#endif
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN pfnExportFenceResolve;
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN pfnExportFenceRollback;
+} PFN_SYNC_CHECKPOINT_STRUCT;
+
+enum PVRSRV_ERROR_TAG SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns);
+
+#define CHECKPOINT_PFNS
+#endif
+
+/* sync_checkpoint.h */
+enum PVRSRV_ERROR_TAG SyncCheckpointContextCreate(struct _PVRSRV_DEVICE_NODE_ *psDevConnection, PSYNC_CHECKPOINT_CONTEXT *phSyncCheckpointContext);
+enum PVRSRV_ERROR_TAG SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext);
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext);
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext);
+enum PVRSRV_ERROR_TAG SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext, PVRSRV_TIMELINE timeline, PVRSRV_FENCE fence, const char *pszCheckpointName, PSYNC_CHECKPOINT *ppsSyncCheckpoint);
+void SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+void SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+bool SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+bool SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint, u32 fence_sync_flags);
+enum PVRSRV_ERROR_TAG SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+enum PVRSRV_ERROR_TAG SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+void SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+void SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+__u32 SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+PVRSRV_TIMELINE SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint);
+const char *SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint);
+#if defined(SUPPORT_NATIVE_FENCE_SYNC)
+struct _PVRSRV_DEVICE_NODE_ *SyncCheckpointGetAssociatedDevice(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+#endif
+
+#endif
+
+#if defined(SUPPORT_NATIVE_FENCE_SYNC) || defined(SUPPORT_BUFFER_SYNC)
+/*************************************************************************/ /*!
+@Function       NativeSyncGetFenceStatusWq
+@Description    Called to get the Foreign Fence status workqueue used in
+                Fence sync and Buffer sync.
+@Return         struct workqueue_struct ptr on success, NULL otherwise.
+*/ /**************************************************************************/
+struct workqueue_struct *NativeSyncGetFenceStatusWq(void);
+/*************************************************************************/ /*!
+@Function       NativeSyncGetFenceCtxDestroyWq
+@Return         struct workqueue_struct ptr on success, NULL otherwise.
+*/ /**************************************************************************/
+struct workqueue_struct *NativeSyncGetFenceCtxDestroyWq(void);
+#endif
+
+#endif /* __SERVICES_KERNEL_CLIENT__ */
diff --git a/drivers/gpu/drm/img-rogue/services_km.h b/drivers/gpu/drm/img-rogue/services_km.h
new file mode 100644
index 000000000000..8903b86a76a9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/services_km.h
@@ -0,0 +1,180 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services API Kernel mode Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Exported services API details
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SERVICES_KM_H
+#define SERVICES_KM_H
+
+#include "img_types.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "virt_validation_defs.h"
+#endif
+
+/*! 4k page size definition */
+#define PVRSRV_4K_PAGE_SIZE					4096U      /*!< Size of a 4K Page */
+#define PVRSRV_4K_PAGE_SIZE_ALIGNSHIFT		12          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 16k page size definition */
+#define PVRSRV_16K_PAGE_SIZE					16384U      /*!< Size of a 16K Page */
+#define PVRSRV_16K_PAGE_SIZE_ALIGNSHIFT		14          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 64k page size definition */
+#define PVRSRV_64K_PAGE_SIZE					65536U      /*!< Size of a 64K Page */
+#define PVRSRV_64K_PAGE_SIZE_ALIGNSHIFT		16          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 256k page size definition */
+#define PVRSRV_256K_PAGE_SIZE					262144U      /*!< Size of a 256K Page */
+#define PVRSRV_256K_PAGE_SIZE_ALIGNSHIFT		18          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 1MB page size definition */
+#define PVRSRV_1M_PAGE_SIZE					1048576U      /*!< Size of a 1M Page */
+#define PVRSRV_1M_PAGE_SIZE_ALIGNSHIFT		20          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+/*! 2MB page size definition */
+#define PVRSRV_2M_PAGE_SIZE					2097152U      /*!< Size of a 2M Page */
+#define PVRSRV_2M_PAGE_SIZE_ALIGNSHIFT		21          /*!< Amount to shift an address by so that
+                                                          it is always page-aligned */
+
+/*!
+ * @AddToGroup SRVConnectInterfaces
+ * @{
+ */
+
+#ifndef PVRSRV_DEV_CONNECTION_TYPEDEF
+#define PVRSRV_DEV_CONNECTION_TYPEDEF
+/*!
+ * Forward declaration (look on connection.h)
+ */
+typedef struct PVRSRV_DEV_CONNECTION_TAG PVRSRV_DEV_CONNECTION;
+#endif
+
+/*!
+ * @Anchor SRV_FLAGS
+ * @Name SRV_FLAGS: Services connection flags
+ * Allows to define per-client policy for Services.
+ * @{
+ */
+
+/*
+ *   Use of the 32-bit connection flags mask
+ *   ( X = taken/in use, - = available/unused )
+ *
+ *   31  27     20             6 4   0
+ *    |   |      |             | |   |
+ *    X---XXXXXXXX-------------XXX----
+ */
+
+#define SRV_NO_HWPERF_CLIENT_STREAM     (1UL << 4)  /*!< Don't create HWPerf for this connection */
+#define SRV_FLAGS_CLIENT_64BIT_COMPAT   (1UL << 5)  /*!< This flags gets set if the client is 64 Bit compatible. */
+#define SRV_FLAGS_CLIENT_SLR_DISABLED   (1UL << 6)  /*!< This flag is set if the client does not want Sync Lockup Recovery (SLR) enabled. */
+#define SRV_FLAGS_PDUMPCTRL             (1UL << 31) /*!< PDump Ctrl client flag */
+
+/*! @} SRV_FLAGS */
+
+/*! @} End of SRVConnectInterfaces */
+
+/*
+ * Bits 20 - 27 are used to pass information needed for validation
+ * of the GPU Virtualisation Validation mechanism. In particular:
+ *
+ * Bits:
+ * [20 - 22]: OSid of the memory region that will be used for allocations
+ * [23 - 25]: OSid that will be emitted by the Firmware for all memory accesses
+ *            regarding that memory context.
+ *      [26]: If the AXI Protection register will be set to secure for that OSid
+ *      [27]: If the Emulator Wrapper Register checking for protection violation
+ *            will be set to secure for that OSid
+ */
+
+#define VIRTVAL_FLAG_OSID_SHIFT        (20)
+#define SRV_VIRTVAL_FLAG_OSID_MASK     (7U << VIRTVAL_FLAG_OSID_SHIFT)
+
+#define VIRTVAL_FLAG_OSIDREG_SHIFT     (23)
+#define SRV_VIRTVAL_FLAG_OSIDREG_MASK  (7U << VIRTVAL_FLAG_OSIDREG_SHIFT)
+
+#define VIRTVAL_FLAG_AXIPREG_SHIFT     (26)
+#define SRV_VIRTVAL_FLAG_AXIPREG_MASK  (1U << VIRTVAL_FLAG_AXIPREG_SHIFT)
+
+#define VIRTVAL_FLAG_AXIPTD_SHIFT      (27)
+#define SRV_VIRTVAL_FLAG_AXIPTD_MASK   (1U << VIRTVAL_FLAG_AXIPTD_SHIFT)
+
+
+/* Size of pointer on a 64 bit machine */
+#define	POINTER_SIZE_64BIT	(8U)
+
+
+/*
+    Pdump flags which are accessible to Services clients
+*/
+#define PDUMP_NONE          0x00000000U /*<! No flags */
+
+#define PDUMP_BLKDATA       0x10000000U /*<! This flag indicates block-mode PDump data to be recorded in
+                                                          Block script stream in addition to Main script stream,
+                                                          if capture mode is set to BLOCKED */
+
+#define PDUMP_CONT          0x40000000U /*<! Output this entry always regardless of framed capture range,
+                                                          used by client applications being dumped. */
+#define PDUMP_PERSIST       0x80000000U /*<! Output this entry always regardless of app and range,
+                                                          used by persistent resources created after
+                                                          driver initialisation that must appear in
+                                                          all PDump captures in that session. */
+
+/* Valid range of values for pdump block length in Block mode of PDump */
+#define PDUMP_BLOCKLEN_MIN          10
+#define PDUMP_BLOCKLEN_MAX          1000
+
+#define PDUMP_FRAME_MIN             0
+#define PDUMP_FRAME_MAX             (IMG_UINT32_MAX - 1)
+#define PDUMP_FRAME_UNSET           IMG_UINT32_MAX
+
+/* Status of the device. */
+typedef IMG_UINT32 PVRSRV_DEVICE_STATUS;
+#define PVRSRV_DEVICE_STATUS_UNKNOWN        0U /* status of the device is unknown */
+#define PVRSRV_DEVICE_STATUS_OK             1U /* the device is operational */
+#define PVRSRV_DEVICE_STATUS_NOT_RESPONDING 2U /* the device is not responding */
+#define PVRSRV_DEVICE_STATUS_DEVICE_ERROR   3U /* the device is not operational */
+
+#endif /* SERVICES_KM_H */
+/**************************************************************************//**
+End of file (services_km.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/servicesext.h b/drivers/gpu/drm/img-rogue/servicesext.h
new file mode 100644
index 000000000000..f2887d48aed7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/servicesext.h
@@ -0,0 +1,134 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services definitions required by external drivers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides services data structures, defines and prototypes
+                required by external drivers
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SERVICESEXT_H)
+#define SERVICESEXT_H
+
+/* include/ */
+#include "pvrsrv_error.h"
+#include "img_types.h"
+#include "img_3dtypes.h"
+#include "pvrsrv_device_types.h"
+
+/*
+ * Lock buffer read/write flags
+ */
+#define PVRSRV_LOCKFLG_READONLY		(1)		/*!< The locking process will only read the locked surface */
+
+/*!
+ *****************************************************************************
+ *	Services State
+ *****************************************************************************/
+typedef enum _PVRSRV_SERVICES_STATE_
+{
+	PVRSRV_SERVICES_STATE_UNDEFINED = 0,
+	PVRSRV_SERVICES_STATE_OK,
+	PVRSRV_SERVICES_STATE_BAD,
+} PVRSRV_SERVICES_STATE;
+
+
+/*!
+ *****************************************************************************
+ *	States for power management
+ *****************************************************************************/
+
+/*!
+  System Power State Enum
+ */
+
+#define _PVRSRV_SYS_POWER_STATES \
+	X(Unspecified, -1) /*!< Unspecified : Uninitialised */ \
+	X(OFF,          0) /*!< Off */ \
+	X(ON,           1) /*!< On */
+
+typedef enum _PVRSRV_SYS_POWER_STATE_
+{
+#define X(name, value) PVRSRV_SYS_POWER_STATE_##name = value,
+	_PVRSRV_SYS_POWER_STATES
+#undef X
+
+	PVRSRV_SYS_POWER_STATE_FORCE_I32 = 0x7fffffff	/*!< Force enum to be at least 32-bits wide */
+
+} PVRSRV_SYS_POWER_STATE, *PPVRSRV_SYS_POWER_STATE; /*!< Typedef for ptr to PVRSRV_SYS_POWER_STATE */
+
+/*!
+  Device Power State Enum
+ */
+typedef IMG_INT32 PVRSRV_DEV_POWER_STATE;
+typedef IMG_INT32 *PPVRSRV_DEV_POWER_STATE;	/*!< Typedef for ptr to PVRSRV_DEV_POWER_STATE */ /* PRQA S 3205 */
+#define PVRSRV_DEV_POWER_STATE_DEFAULT	-1	/*!< Default state for the device */
+#define PVRSRV_DEV_POWER_STATE_OFF		 0	/*!< Unpowered */
+#define PVRSRV_DEV_POWER_STATE_ON		 1	/*!< Running */
+
+/*!
+  Power Flags Enum
+ */
+typedef IMG_UINT32 PVRSRV_POWER_FLAGS;
+#define PVRSRV_POWER_FLAGS_NONE             0U      /*!< No flags */
+#define PVRSRV_POWER_FLAGS_FORCED           (1U << 0) /*!< Power the transition should not fail */
+#define PVRSRV_POWER_FLAGS_OSPM_SUSPEND_REQ (1U << 1) /*!< Indicates an OS Power Management
+                                                           transition (S3/S4) has been requested.
+                                                           Allows system modules to save VRAM */
+#define PVRSRV_POWER_FLAGS_OSPM_RESUME_REQ  (1U << 2) /*!< Indicates an OS Power Management
+                                                           transition has been requested.
+                                                           Allows system modules to load VRAM */
+
+/*!
+ *****************************************************************************
+ * This structure is used for OS independent registry (profile) access
+ *****************************************************************************/
+
+typedef struct PVRSRV_REGISTRY_INFO_TAG
+{
+	IMG_UINT32	ui32DevCookie;
+	IMG_PCHAR	pszKey;
+	IMG_PCHAR	pszValue;
+	IMG_PCHAR	pszBuf;
+	IMG_UINT32	ui32BufSize;
+} PVRSRV_REGISTRY_INFO, *PPVRSRV_REGISTRY_INFO;
+
+#endif /* SERVICESEXT_H */
+/******************************************************************************
+ End of file (servicesext.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/sofunc_pvr.h b/drivers/gpu/drm/img-rogue/sofunc_pvr.h
new file mode 100644
index 000000000000..48bc94d8ef0b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sofunc_pvr.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          SO Interface header file for common PVR functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains SO interface functions. These functions are defined in
+                the common layer and are called from the env layer OS specific
+                implementation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SOFUNC_PVR_H_)
+#define SOFUNC_PVR_H_
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+#include "device.h"
+#include "pvr_notifier.h"
+
+
+/**************************************************************************/ /*!
+ @Function     SOPvrDbgRequestNotifyRegister
+ @Description  SO Interface function called from the OS layer implementation.
+               Register a callback function that is called when a debug request
+               is made via a call PVRSRVDebugRequest. There are a number of
+               verbosity levels ranging from DEBUG_REQUEST_VERBOSITY_LOW up to
+               DEBUG_REQUEST_VERBOSITY_MAX. The callback will be called once
+               for each level up to the highest level specified to
+               PVRSRVDebugRequest.
+@Output        phNotify             On success, points to debug notifier handle
+@Input         psDevNode            Device node for which the debug callback
+                                    should be registered
+@Input         pfnDbgRequestNotify  Function callback
+@Input         ui32RequesterID      Requester ID. This is used to determine
+                                    the order in which callbacks are called,
+                                    see DEBUG_REQUEST_*
+@Input         hDbgReqeustHandle    Data to be passed back to the caller via
+                                    the callback function
+@Return        PVRSRV_ERROR         PVRSRV_OK on success and an error otherwise
+*/ /******************************************************************** ******/
+PVRSRV_ERROR SOPvrDbgRequestNotifyRegister(IMG_HANDLE *phNotify,
+							  PVRSRV_DEVICE_NODE *psDevNode,
+							  PFN_DBGREQ_NOTIFY pfnDbgRequestNotify,
+							  IMG_UINT32 ui32RequesterID,
+							  PVRSRV_DBGREQ_HANDLE hDbgRequestHandle);
+
+/**************************************************************************/ /*!
+ @Function     SOPvrDbgRequestNotifyUnregister
+ @Description  SO Interface function called from the OS layer implementation.
+               Remove and clean up the specified notifier registration so that
+               it does not receive any further callbacks.
+ @Input        hNotify     Handle returned to caller from
+                           SOPvrDbgRequestNotifyRegister().
+ @Return       PVRSRV_ERROR
+*/ /***************************************************************************/
+PVRSRV_ERROR SOPvrDbgRequestNotifyUnregister(IMG_HANDLE hNotify);
+
+
+#endif /* SOFUNC_PVR_H_ */
diff --git a/drivers/gpu/drm/img-rogue/sofunc_rgx.h b/drivers/gpu/drm/img-rogue/sofunc_rgx.h
new file mode 100644
index 000000000000..be9594d95242
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sofunc_rgx.h
@@ -0,0 +1,95 @@
+/*************************************************************************/ /*!
+@File
+@Title          SO Interface header file for devices/RGX functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Contains SO interface functions. These functions are defined in
+                the common devices layer and are called from the env layer OS
+                specific implementation.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SOFUNC_RGX_H_)
+#define SOFUNC_RGX_H_
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+
+#if !defined(NO_HARDWARE)
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsRegister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Initialise data used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument). This function must be called only once for each
+               different user/handle.
+
+ @Input        phGpuUtilUser - Pointer to handle used to identify a user of
+                               RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsRegister(IMG_HANDLE *phGpuUtilUser);
+
+
+/*!
+*******************************************************************************
+
+ @Function     SORgxGpuUtilStatsUnregister
+
+ @Description  SO Interface function called from the OS layer implementation.
+               Free data previously used to compute GPU utilisation statistics
+               for a particular user (identified by the handle passed as
+               argument).
+
+ @Input        hGpuUtilUser - Handle used to identify a user of
+                              RGXGetGpuUtilStats
+
+ @Return       PVRSRV_ERROR
+
+******************************************************************************/
+PVRSRV_ERROR SORgxGpuUtilStatsUnregister(IMG_HANDLE hGpuUtilUser);
+#endif /* !defined(NO_HARDWARE) */
+
+
+
+#endif /* SOFUNC_RGX_H_ */
diff --git a/drivers/gpu/drm/img-rogue/spacemit/devfreq_cooling.h b/drivers/gpu/drm/img-rogue/spacemit/devfreq_cooling.h
new file mode 100644
index 000000000000..331a7e022400
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/spacemit/devfreq_cooling.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * devfreq_cooling: Thermal cooling device implementation for devices using
+ *                  devfreq
+ *
+ * Copyright (C) 2014-2015 ARM Limited
+ *
+ */
+
+#ifndef __DEVFREQ_COOLING_H__
+#define __DEVFREQ_COOLING_H__
+
+#include <linux/devfreq.h>
+#include <linux/thermal.h>
+
+/**
+ * struct devfreq_cooling_power - Devfreq cooling power ops
+ * @get_real_power:	When this is set, the framework uses it to ask the
+ *			device driver for the actual power.
+ *			Some devices have more sophisticated methods
+ *			(like power counters) to approximate the actual power
+ *			that they use.
+ *			This function provides more accurate data to the
+ *			thermal governor. When the driver does not provide
+ *			such function, framework just uses pre-calculated
+ *			table and scale the power by 'utilization'
+ *			(based on 'busy_time' and 'total_time' taken from
+ *			devfreq 'last_status').
+ *			The value returned by this function must be lower
+ *			or equal than the maximum power value
+ *			for the current	state
+ *			(which can be found in power_table[state]).
+ *			When this interface is used, the power_table holds
+ *			max total (static + dynamic) power value for each OPP.
+ */
+struct devfreq_cooling_power {
+	int (*get_real_power)(struct devfreq *df, u32 *power,
+			      unsigned long freq, unsigned long voltage);
+};
+
+#ifdef CONFIG_POWERVR_THERMALS
+
+struct thermal_cooling_device *
+of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df,
+				  struct devfreq_cooling_power *dfc_power);
+struct thermal_cooling_device *
+of_devfreq_cooling_register(struct device_node *np, struct devfreq *df);
+struct thermal_cooling_device *devfreq_cooling_register(struct devfreq *df);
+void devfreq_cooling_unregister(struct thermal_cooling_device *dfc);
+struct thermal_cooling_device *
+devfreq_cooling_em_register(struct devfreq *df,
+			    struct devfreq_cooling_power *dfc_power);
+
+#else /* !CONFIG_POWERVR_THERMALS */
+
+static inline struct thermal_cooling_device *
+of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df,
+				  struct devfreq_cooling_power *dfc_power)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static inline struct thermal_cooling_device *
+of_devfreq_cooling_register(struct device_node *np, struct devfreq *df)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static inline struct thermal_cooling_device *
+devfreq_cooling_register(struct devfreq *df)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static inline struct thermal_cooling_device *
+devfreq_cooling_em_register(struct devfreq *df,
+			    struct devfreq_cooling_power *dfc_power)
+{
+	return ERR_PTR(-EINVAL);
+}
+
+static inline void
+devfreq_cooling_unregister(struct thermal_cooling_device *dfc)
+{
+}
+
+#endif /* CONFIG_POWERVR_THERMALS */
+#endif /* __DEVFREQ_COOLING_H__ */
diff --git a/drivers/gpu/drm/img-rogue/spacemit/spacemit_init.c b/drivers/gpu/drm/img-rogue/spacemit/spacemit_init.c
new file mode 100644
index 000000000000..ea145c94b5f7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/spacemit/spacemit_init.c
@@ -0,0 +1,377 @@
+#if defined(SUPPORT_ION)
+#include "ion_sys.h"
+#endif /* defined(SUPPORT_ION) */
+
+#if defined(SUPPORT_PDVFS)
+#include "rgxpdvfs.h"
+#endif
+
+#include <linux/clkdev.h>
+#include <linux/hardirq.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/version.h>
+#if defined(CONFIG_POWERVR_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+#include <linux/thermal.h>
+#include <linux/devfreq_cooling.h>
+#endif
+#include "power.h"
+#include "sysconfig.h"
+#include "spacemit_init.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include <linux/clk-provider.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_opp.h>
+#include "rgxdevice.h"
+
+static struct st_context *g_platform = NULL;
+
+#if defined(SUPPORT_LINUX_DVFS) && !defined(NO_HARDWARE)
+void stSetFrequency(IMG_HANDLE hSysData, IMG_UINT32 ui32Frequency)
+{
+	int ret = 0;
+	unsigned int old_freq;
+
+	if (NULL == g_platform)
+		panic("oops");
+
+	old_freq = clk_get_rate(g_platform->gpu_clk);
+	mutex_lock(&g_platform->set_power_state);
+	if (g_platform->bEnablePd)
+		ret = clk_set_rate(g_platform->gpu_clk, ui32Frequency);
+	mutex_unlock(&g_platform->set_power_state);
+	PVR_DPF((PVR_DBG_VERBOSE, "gpu clock change frequency: %d",ui32Frequency));
+	if (ret) {
+		PVR_DPF((PVR_DBG_ERROR, "failed to set gpu clock rate: %d", ret));
+		return;
+	}
+}
+
+void stSetVoltage(IMG_HANDLE hSysData, IMG_UINT32 ui32Volt)
+{
+	if (NULL == g_platform)
+		panic("oops");
+	PVR_DPF((PVR_DBG_VERBOSE, "entry %s", __func__));
+}
+#endif
+
+#if defined(CONFIG_POWERVR_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+
+#define FALLBACK_STATIC_TEMPERATURE 55000
+
+static u32 dynamic_coefficient;
+static u32 static_coefficient;
+static s32 ts[4];
+static struct thermal_zone_device *gpu_tz;
+
+static int model_real_power(struct devfreq *df, u32 *power, unsigned long freq, unsigned long voltage)
+{
+	int temperature;
+	unsigned long temp;
+	unsigned long temp_squared, temp_cubed, temp_scaling_factor;
+	const unsigned long voltage_cubed = (voltage * voltage * voltage) >> 10;
+	unsigned long static_power, dynamic_power;
+
+	if (gpu_tz) {
+		int ret;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		ret = thermal_zone_get_temp(gpu_tz, &temperature);
+#else
+		ret = gpu_tz->ops->get_temp(gpu_tz, &temperature);
+#endif
+		if (ret) {
+			pr_warn_ratelimited("Error reading temperature for gpu thermal zone: %d\n",
+					ret);
+			temperature = FALLBACK_STATIC_TEMPERATURE;
+		}
+	} else {
+		temperature = FALLBACK_STATIC_TEMPERATURE;
+	}
+
+	/* Calculate the temperature scaling factor. To be applied to the
+	 * voltage scaled power.
+	 */
+	temp = temperature / 1000;
+	temp_squared = temp * temp;
+	temp_cubed = temp_squared * temp;
+	temp_scaling_factor =
+		  (ts[3] * temp_cubed)
+		+ (ts[2] * temp_squared)
+		+ (ts[1] * temp)
+		+ ts[0];
+
+	static_power = (((static_coefficient * voltage_cubed) >> 20)
+					* temp_scaling_factor)
+					/ 1000000;
+
+	/* Calculate dynamic power */
+	const unsigned long v2 = (voltage * voltage) / 1000; /* m*(V*V) */
+	const unsigned long f_mhz = freq / 1000000; /* MHz */
+	dynamic_power = (dynamic_coefficient * v2 * f_mhz) / 1000000; /* mW */
+
+	*power = static_power + dynamic_power;
+
+	/* Log thermal events for debugging */
+	if (temperature > 80000) { /* Above 80°C */
+		pr_warn_ratelimited("GPU thermal warning: temp=%d°C, power=%umW, freq=%luMHz\n",
+				    temperature / 1000, *power, f_mhz);
+	}
+
+	return 0;
+}
+
+struct devfreq_cooling_power spacemit_power_model_simple_ops = {
+	.get_real_power = model_real_power,
+};
+
+int spacemit_power_model_simple_init(struct device *dev)
+{
+	struct device_node *power_model_node;
+	const char *tz_name;
+
+	power_model_node = of_get_child_by_name(dev->of_node,
+			"power_model");
+	if (!power_model_node) {
+		/* Fallback to reading thermal-zone from device node */
+		if (of_property_read_string(dev->of_node, "thermal-zone", &tz_name)) {
+			dev_err(dev, "could not find power_model node or thermal-zone\n");
+			return -ENODEV;
+		}
+		dev_info(dev, "Using default thermal zone: %s with default power model parameters\n", tz_name);
+
+		/* Use default parameters when no power_model node exists */
+		dynamic_coefficient = 1024;
+		static_coefficient = 512;
+		ts[0] = 10;
+		ts[1] = -100;
+		ts[2] = 500;
+		ts[3] = -200;
+	} else {
+		/* Original logic when power_model node exists */
+		if (!of_device_is_compatible(power_model_node, "img,pvr-simple-power-model")) {
+			dev_err(dev, "power_model incompatible with simple power model\n");
+			of_node_put(power_model_node);
+			return -ENODEV;
+		}
+
+		if (of_property_read_string(power_model_node, "thermal-zone", &tz_name)) {
+			dev_err(dev, "thermal-zone in power_model not available\n");
+			of_node_put(power_model_node);
+			return -EINVAL;
+		}
+
+		if (of_property_read_u32(power_model_node, "dynamic-coefficient", &dynamic_coefficient)) {
+			dev_err(dev, "dynamic-coefficient in power_model not available\n");
+			of_node_put(power_model_node);
+			return -EINVAL;
+		}
+		if (of_property_read_u32(power_model_node, "static-coefficient", &static_coefficient)) {
+			dev_err(dev, "static-coefficient in power_model not available\n");
+			of_node_put(power_model_node);
+			return -EINVAL;
+		}
+		if (of_property_read_u32_array(power_model_node, "ts", (u32 *)ts, 4)) {
+			dev_err(dev, "ts in power_model not available\n");
+			of_node_put(power_model_node);
+			return -EINVAL;
+		}
+
+		of_node_put(power_model_node);
+	}
+
+	/* Initialize thermal zone for both cases */
+	gpu_tz = thermal_zone_get_zone_by_name(tz_name);
+	if (IS_ERR(gpu_tz)) {
+		pr_warn_ratelimited("Error getting gpu thermal zone (%ld), not yet ready?\n",
+				PTR_ERR(gpu_tz));
+		gpu_tz = NULL;
+		return -EPROBE_DEFER;
+	}
+
+	return 0;
+}
+#endif
+static void RgxEnableClock(struct st_context *platform)
+{
+	if (!platform->gpu_clk)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: gpu_clk is null", __func__));
+		return;
+	}
+
+	if (!platform->gpu_active) {
+		clk_prepare_enable(platform->gpu_clk);
+		platform->gpu_active = IMG_TRUE;
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu clock on gpu_active:%d", platform->gpu_active));
+	} else {
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu clock already on!"));
+	}
+}
+
+static void RgxDisableClock(struct st_context *platform)
+{
+
+	if (!platform->gpu_clk) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: gpu_clk is null", __func__));
+		return;
+	}
+
+	if (platform->gpu_active) {
+		clk_disable_unprepare(platform->gpu_clk);
+		platform->gpu_active = IMG_FALSE;
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu clock off gpu_active:%d", platform->gpu_active));
+	} else {
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu clock already off!"));
+	}
+}
+
+static void RgxEnablePower(struct st_context *platform)
+{
+	struct device *dev = (struct device *)platform->dev_config->pvOSDevice;
+	if (!platform->bEnablePd) {
+		pm_runtime_get_sync(dev);
+		platform->bEnablePd = IMG_TRUE;
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu power on bEnablePd:%d", platform->bEnablePd));
+	} else {
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu already power on!"));
+	}
+}
+
+static void RgxDisablePower(struct st_context *platform)
+{
+	struct device *dev = (struct device *)platform->dev_config->pvOSDevice;
+	if (platform->bEnablePd) {
+		pm_runtime_put(dev);
+		platform->bEnablePd = IMG_FALSE;
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu power off bEnablePd:%d", platform->bEnablePd));
+	} else {
+		PVR_DPF((PVR_DBG_VERBOSE, "gpu already power off!"));
+	}
+}
+
+void RgxResume(struct st_context *platform)
+{
+	PVR_DPF((PVR_DBG_VERBOSE, "%s", __func__));
+	RgxEnablePower(platform);
+	RgxEnableClock(platform);
+ }
+
+void RgxSuspend(struct st_context *platform)
+{
+	PVR_DPF((PVR_DBG_VERBOSE, "%s", __func__));
+	RgxDisableClock(platform);
+	RgxDisablePower(platform);
+}
+
+PVRSRV_ERROR STPrePowerState(IMG_HANDLE hSysData,
+							 PVRSRV_SYS_POWER_STATE eNewPowerState,
+							 PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+							 PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	struct st_context *platform = (struct st_context *)hSysData;
+
+	PVR_DPF((PVR_DBG_VERBOSE, "STPrePowerState new:%d, curr:%d", eNewPowerState, eCurrentPowerState));
+	mutex_lock(&platform->set_power_state);
+	if ((PVRSRV_SYS_POWER_STATE_ON == eNewPowerState) &&
+		(PVRSRV_SYS_POWER_STATE_OFF == eCurrentPowerState))
+		RgxResume(platform);
+	mutex_unlock(&platform->set_power_state);
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR STPostPowerState(IMG_HANDLE hSysData,
+							  PVRSRV_SYS_POWER_STATE eNewPowerState,
+							  PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+							  PVRSRV_POWER_FLAGS ePwrFlags)
+{
+	struct st_context *platform = (struct st_context *)hSysData;
+
+	PVR_DPF((PVR_DBG_VERBOSE, "STPostPowerState new:%d, curr:%d", eNewPowerState, eCurrentPowerState));
+	mutex_lock(&platform->set_power_state);
+	if ((PVRSRV_SYS_POWER_STATE_OFF == eNewPowerState) &&
+		(PVRSRV_SYS_POWER_STATE_ON == eCurrentPowerState))
+		RgxSuspend(platform);
+	mutex_unlock(&platform->set_power_state);
+	return PVRSRV_OK;
+}
+
+void RgxStUnInit(struct st_context *platform)
+{
+	struct device *dev = (struct device *)platform->dev_config->pvOSDevice;
+
+	RgxSuspend(platform);
+
+#if defined(CONFIG_POWERVR_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+	/* Unregister thermal zone if it was registered */
+	if (platform->thermal_zone) {
+		thermal_zone_device_unregister(platform->thermal_zone);
+		platform->thermal_zone = NULL;
+	}
+#endif
+
+	if (platform->gpu_clk) {
+		devm_clk_put(dev, platform->gpu_clk);
+		platform->gpu_clk = NULL;
+	}
+
+	pm_runtime_disable(dev);
+	devm_kfree(dev, platform);
+}
+
+struct st_context *RgxStInit(PVRSRV_DEVICE_CONFIG* psDevConfig)
+{
+	struct device *dev = (struct device *)psDevConfig->pvOSDevice;
+	struct st_context *platform;
+	RGX_DATA* psRGXData = (RGX_DATA*)psDevConfig->hDevData;
+
+	platform = devm_kzalloc(dev, sizeof(struct st_context), GFP_KERNEL);
+	if (NULL == platform) {
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to kzalloc rk_context", __func__));
+		return NULL;
+	}
+	g_platform = platform;
+	if (!dev->dma_mask)
+		dev->dma_mask = &dev->coherent_dma_mask;
+
+	platform->dev_config = psDevConfig;
+	platform->gpu_active = IMG_FALSE;
+	platform->bEnablePd = IMG_FALSE;
+
+	pm_runtime_enable(dev);
+	platform->gpu_clk = devm_clk_get(dev, "clk_rgx");
+	if (IS_ERR_OR_NULL(platform->gpu_clk)) {
+		PVR_DPF((PVR_DBG_ERROR, "RgxStInit: Failed to find gpu clk source"));
+		goto fail;
+	}
+
+	clk_set_rate(platform->gpu_clk, RGX_ST_CORE_CLOCK_SPEED);
+
+	if (psRGXData && psRGXData->psRGXTimingInfo)
+	{
+		psRGXData->psRGXTimingInfo->ui32CoreClockSpeed = clk_get_rate(platform->gpu_clk);
+	}
+
+	mutex_init(&platform->set_power_state);
+
+#if defined(CONFIG_POWERVR_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+	/* Initialize power model for thermal cooling */
+	if (spacemit_power_model_simple_init(dev) == 0) {
+		/* Configure power model for thermal cooling */
+		psDevConfig->sDVFS.sDVFSDeviceCfg.psPowerOps = &spacemit_power_model_simple_ops;
+	} else {
+		dev_warn(dev, "Failed to initialize GPU power model, thermal cooling may not work\n");
+	}
+	/* Don't register a separate thermal zone - use the existing thermal_gpu zone from DTS */
+	platform->thermal_zone = NULL;
+#endif
+
+	return platform;
+
+fail:
+	devm_kfree(dev, platform);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/img-rogue/spacemit/spacemit_init.h b/drivers/gpu/drm/img-rogue/spacemit/spacemit_init.h
new file mode 100644
index 000000000000..b5446fe34155
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/spacemit/spacemit_init.h
@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR MIT
+ *
+ *	@File
+ *	@Title          System Configuration
+ *	@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ *	@Description    System Configuration functions
+ *	@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(__DOVE_INIT__)
+#define __DOVE_INIT__
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_device.h"
+#include "servicesext.h"
+#include <linux/version.h>
+
+struct st_context {
+	PVRSRV_DEVICE_CONFIG    *dev_config;
+	struct clk          *gpu_clk;
+	/* mutex protect for set power state */
+	struct mutex set_power_state;
+	IMG_BOOL            gpu_active;
+	IMG_BOOL            bEnablePd;
+#if defined(CONFIG_POWERVR_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+	struct thermal_zone_device *thermal_zone;
+#endif
+};
+
+struct st_context * RgxStInit(PVRSRV_DEVICE_CONFIG* psDevConfig);
+void RgxStUnInit(struct st_context *platform);
+void RgxResume(struct st_context *platform);
+void RgxSuspend(struct st_context *platform);
+PVRSRV_ERROR STPrePowerState(IMG_HANDLE hSysData,
+							 PVRSRV_SYS_POWER_STATE eNewPowerState,
+							 PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+							 PVRSRV_POWER_FLAGS ePwrFlags);
+PVRSRV_ERROR STPostPowerState(IMG_HANDLE hSysData,
+							  PVRSRV_SYS_POWER_STATE eNewPowerState,
+							  PVRSRV_SYS_POWER_STATE eCurrentPowerState,
+							  PVRSRV_POWER_FLAGS ePwrFlags);
+void stSetFrequency(IMG_HANDLE hSysData, IMG_UINT32 ui32Frequency);
+void stSetVoltage(IMG_HANDLE hSysData, IMG_UINT32 ui32Voltage);
+
+#if defined(CONFIG_POWERVR_THERMAL) && defined(SUPPORT_LINUX_DVFS)
+int spacemit_power_model_simple_init(struct device *dev);
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/spacemit/sysconfig.c b/drivers/gpu/drm/img-rogue/spacemit/sysconfig.c
new file mode 100644
index 000000000000..dc7126d409e1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/spacemit/sysconfig.c
@@ -0,0 +1,342 @@
+/*************************************************************************/ /*!
+@File
+@Title          System Configuration
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System Configuration functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+
+#include "interrupt_support.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "sysconfig.h"
+#include "physheap.h"
+#if defined(SUPPORT_ION)
+#include "ion_support.h"
+#endif
+#include "spacemit_init.h"
+#include "vz_vmm_pvz.h"
+
+static RGX_TIMING_INFORMATION	gsRGXTimingInfo;
+static RGX_DATA					gsRGXData;
+static PVRSRV_DEVICE_CONFIG		gsDevices[1];
+static PHYS_HEAP_FUNCTIONS		gsPhysHeapFuncs;
+static PHYS_HEAP_CONFIG			gsPhysHeapConfig[3];
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+
+/*
+ * Fallback OPP table matching device tree configuration.
+ * Used only if DT OPP loading fails in pvr_dvfs_device.c
+ */
+static const IMG_OPP asOPPTable[] = {
+	{ 750000,  409 * 1000 * 1000},  /* 0.75V @ 409MHz - lowest power */
+	{ 800000,  491 * 1000 * 1000},  /* 0.80V @ 491MHz */
+	{ 850000,  614 * 1000 * 1000},  /* 0.85V @ 614MHz */
+	{ 900000,  819 * 1000 * 1000},  /* 0.90V @ 819MHz - highest performance */
+};
+
+#define LEVEL_COUNT (sizeof(asOPPTable) / sizeof(IMG_OPP))
+
+#endif
+
+/*
+	CPU to Device physical address translation
+*/
+static
+void UMAPhysHeapCpuPAddrToDevPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+
+	psDevPAddr[0].uiAddr = psCpuPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psDevPAddr[ui32Idx].uiAddr = psCpuPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+/*
+	Device to CPU physical address translation
+*/
+static
+void UMAPhysHeapDevPAddrToCpuPAddr(IMG_HANDLE hPrivData,
+								   IMG_UINT32 ui32NumOfAddr,
+								   IMG_CPU_PHYADDR *psCpuPAddr,
+								   IMG_DEV_PHYADDR *psDevPAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(hPrivData);
+
+	/* Optimise common case */
+	psCpuPAddr[0].uiAddr = psDevPAddr[0].uiAddr;
+	if (ui32NumOfAddr > 1)
+	{
+		IMG_UINT32 ui32Idx;
+		for (ui32Idx = 1; ui32Idx < ui32NumOfAddr; ++ui32Idx)
+		{
+			psCpuPAddr[ui32Idx].uiAddr = psDevPAddr[ui32Idx].uiAddr;
+		}
+	}
+}
+
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig)
+{
+	IMG_UINT32 ui32NextPhysHeapID = 0;
+	int iIrq;
+	struct resource *psDevMemRes = NULL;
+	struct platform_device *psDev;
+
+	psDev = to_platform_device((struct device *)pvOSDevice);
+
+	if (gsDevices[0].pvOSDevice)
+	{
+		return PVRSRV_ERROR_INVALID_DEVICE;
+	}
+
+	/*
+	 * Setup information about physical memory heap(s) we have
+	 */
+	gsPhysHeapFuncs.pfnCpuPAddrToDevPAddr = UMAPhysHeapCpuPAddrToDevPAddr;
+	gsPhysHeapFuncs.pfnDevPAddrToCpuPAddr = UMAPhysHeapDevPAddrToCpuPAddr;
+
+	gsPhysHeapConfig[0].eType = PHYS_HEAP_TYPE_UMA;
+	gsPhysHeapConfig[0].ui32UsageFlags = PHYS_HEAP_USAGE_GPU_LOCAL;
+	gsPhysHeapConfig[0].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	gsPhysHeapConfig[0].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+	gsPhysHeapConfig[0].uConfig.sUMA.pszHeapName = "uma_gpu_local";
+	gsPhysHeapConfig[0].uConfig.sUMA.hPrivData = NULL;
+
+	ui32NextPhysHeapID += 1;
+	/*
+	 * Setup RGX specific timing data
+	 */
+	gsRGXTimingInfo.ui32CoreClockSpeed        = RGX_ST_CORE_CLOCK_SPEED;
+	gsRGXTimingInfo.bEnableActivePM           = IMG_TRUE;
+	gsRGXTimingInfo.bEnableRDPowIsland        = IMG_FALSE;
+	gsRGXTimingInfo.ui32ActivePMLatencyms     = SYS_RGX_ACTIVE_POWER_LATENCY_MS;
+
+	/*
+	 * Setup RGX specific data
+	 */
+	gsRGXData.psRGXTimingInfo = &gsRGXTimingInfo;
+
+	/*
+	 * Setup RGX device
+	 */
+	gsDevices[0].pvOSDevice             = pvOSDevice;
+	gsDevices[0].pszName                = "spacemit";
+
+	/* Device setup information */
+
+	psDevMemRes = platform_get_resource(psDev, IORESOURCE_MEM, 0);
+	if (psDevMemRes)
+	{
+		gsDevices[0].sRegsCpuPBase.uiAddr = psDevMemRes->start;
+		gsDevices[0].ui32RegsSize         = (unsigned int)(psDevMemRes->end - psDevMemRes->start);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: platform_get_resource failed", __func__));
+		gsDevices[0].sRegsCpuPBase.uiAddr = ST_GPU_PBASE;
+		gsDevices[0].ui32RegsSize         = ST_GPU_SIZE;
+	}
+
+	iIrq = platform_get_irq(psDev, 0);
+	if (iIrq >= 0)
+	{
+		gsDevices[0].ui32IRQ  = (IMG_UINT32) iIrq;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: platform_get_irq failed (%d)", __func__, -iIrq));
+		gsDevices[0].ui32IRQ = ST_IRQ_GPU;
+	}
+
+	gsDevices[0].eCacheSnoopingMode     = PVRSRV_DEVICE_SNOOP_NONE;
+
+	/* Device's physical heaps */
+	gsDevices[0].pasPhysHeaps           = &gsPhysHeapConfig[0];
+	gsDevices[0].ui32PhysHeapCount      = ui32NextPhysHeapID;
+	gsDevices[0].eDefaultHeap = PVRSRV_PHYS_HEAP_GPU_LOCAL;
+
+	/* No power management on ST system */
+	gsDevices[0].pfnPrePowerState       = STPrePowerState;
+	gsDevices[0].pfnPostPowerState      = STPostPowerState;
+
+	/* No clock frequency either */
+	gsDevices[0].pfnClockFreqGet        = NULL;
+
+	//gsDevices[0].pfnCheckMemAllocSize   = NULL;
+
+	gsDevices[0].hDevData               = &gsRGXData;
+
+	gsDevices[0].bHasFBCDCVersion31 = IMG_FALSE;
+	gsDevices[0].bDevicePA0IsValid  = IMG_FALSE;
+
+	/* device error notify callback function */
+	gsDevices[0].pfnSysDevErrorNotify = NULL;
+
+	/* TODO: ST Init */
+	gsDevices[0].hSysData = (IMG_HANDLE)RgxStInit(&gsDevices[0]);
+	if (!gsDevices[0].hSysData)
+	{
+		gsDevices[0].pvOSDevice = NULL;
+		return PVRSRV_ERROR_OUT_OF_MEMORY;
+	}
+
+#if defined(SUPPORT_LINUX_DVFS) || defined(SUPPORT_PDVFS)
+	/*
+	 * Provide fallback OPP table for system layer.
+	 * The actual OPP table will be loaded from device tree by pvr_dvfs_device.c
+	 * If DT loading fails, this hardcoded table will be used as fallback.
+	 */
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pasOPPTable = asOPPTable;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.ui32OPPTableSize = LEVEL_COUNT;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pfnSetFrequency = stSetFrequency;
+	gsDevices[0].sDVFS.sDVFSDeviceCfg.pfnSetVoltage = stSetVoltage;
+#endif
+#if defined(SUPPORT_LINUX_DVFS)
+	/* Load DVFS parameters from device tree with fallback defaults */
+	{
+		struct device_node *np = ((struct device *)pvOSDevice)->of_node;
+		IMG_UINT32 poll_ms = 300;
+		IMG_UINT32 up_threshold = 75;
+		IMG_UINT32 down_differential = 15;
+		IMG_BOOL idle_req = IMG_TRUE;
+		IMG_UINT32 dvfs_config[4];
+
+		/*
+		 * Try to read compact format from DT:
+		 * dvfs-config = <poll_ms up_threshold down_differential idle_req>
+		 */
+		if (np && !of_property_read_u32_array(np, "dvfs-config", dvfs_config, 4)) {
+			poll_ms = dvfs_config[0];
+			up_threshold = dvfs_config[1];
+			down_differential = dvfs_config[2];
+			idle_req = dvfs_config[3] ? IMG_TRUE : IMG_FALSE;
+			PVR_DPF((PVR_DBG_MESSAGE, "DVFS: Loaded config from DT (compact format)"));
+		} else {
+			PVR_DPF((PVR_DBG_MESSAGE, "DVFS: Using default parameters"));
+		}
+
+		/* Apply parameters */
+		gsDevices[0].sDVFS.sDVFSDeviceCfg.ui32PollMs = poll_ms;
+		gsDevices[0].sDVFS.sDVFSDeviceCfg.bIdleReq = idle_req;
+		gsDevices[0].sDVFS.sDVFSGovernorCfg.ui32UpThreshold = up_threshold;
+		gsDevices[0].sDVFS.sDVFSGovernorCfg.ui32DownDifferential = down_differential;
+
+		PVR_DPF((PVR_DBG_MESSAGE, "DVFS config: poll=%ums, up=%u%%, down=%u%%, idle=%s",
+			 poll_ms, up_threshold, down_differential, idle_req ? "Y" : "N"));
+	}
+#endif
+
+	/* Setup other system specific stuff */
+#if defined(SUPPORT_ION)
+	IonInit(NULL);
+#endif
+
+	/* Virtualization support services needs to know which heap ID corresponds to FW */
+	gsPhysHeapConfig[ui32NextPhysHeapID].eType = PHYS_HEAP_TYPE_UMA;
+	gsPhysHeapConfig[ui32NextPhysHeapID].ui32UsageFlags = PHYS_HEAP_USAGE_FW_SHARED;
+	gsPhysHeapConfig[ui32NextPhysHeapID].uConfig.sUMA.pszPDumpMemspaceName = "SYSMEM";
+	gsPhysHeapConfig[ui32NextPhysHeapID].uConfig.sUMA.psMemFuncs = &gsPhysHeapFuncs;
+	gsPhysHeapConfig[ui32NextPhysHeapID].uConfig.sUMA.pszHeapName = "uma_gpu_local";
+	gsPhysHeapConfig[ui32NextPhysHeapID].uConfig.sUMA.hPrivData = NULL;
+	gsDevices[0].ui32PhysHeapCount = ++ui32NextPhysHeapID;
+
+	*ppsDevConfig = &gsDevices[0];
+	return PVRSRV_OK;
+}
+
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+
+	/* TODO:ST UnInit */
+	RgxStUnInit(psDevConfig->hSysData);
+	psDevConfig->hSysData = NULL;
+
+#if defined(SUPPORT_ION)
+	IonDeinit();
+#endif
+
+	psDevConfig->pvOSDevice = NULL;
+}
+
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+	return OSInstallSystemLISR(phLISRData, ui32IRQ, pszName, pfnLISR, pvData,
+							   SYS_IRQ_FLAG_TRIGGER_DEFAULT);
+}
+
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData)
+{
+	return OSUninstallSystemLISR(hLISRData);
+}
+
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+						  DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+						  void *pvDumpDebugFile)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_UNREFERENCED_PARAMETER(pfnDumpDebugPrintf);
+	PVR_UNREFERENCED_PARAMETER(pvDumpDebugFile);
+
+	return PVRSRV_OK;
+}
+
+/******************************************************************************
+ End of file (sysconfig.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/spacemit/sysconfig.h b/drivers/gpu/drm/img-rogue/spacemit/sysconfig.h
new file mode 100644
index 000000000000..f53cba1a12f7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/spacemit/sysconfig.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR MIT
+@File
+@Title          System Description Header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv_device.h"
+#include "rgxdevice.h"
+
+#if !defined(__SYSCCONFIG_H__)
+#define __SYSCCONFIG_H__
+
+#define SYS_RGX_ACTIVE_POWER_LATENCY_MS (100)
+//for st soc
+#if defined(SUPPORT_LINUX_DVFS)
+#define RGX_ST_CORE_CLOCK_SPEED (409 * 1000 * 1000)
+#else
+#define RGX_ST_CORE_CLOCK_SPEED (819 * 1000 * 1000)
+#endif
+#define ST_GPU_PBASE        0xcac00000
+#define ST_GPU_SIZE         0x80000
+#define ST_IRQ_GPU          59
+/*****************************************************************************
+ * system specific data structures
+ *****************************************************************************/
+
+#endif	/* __SYSCCONFIG_H__ */
diff --git a/drivers/gpu/drm/img-rogue/spacemit/sysinfo.h b/drivers/gpu/drm/img-rogue/spacemit/sysinfo.h
new file mode 100644
index 000000000000..010a42cc8756
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/spacemit/sysinfo.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
+
+#if !defined(__SYSINFO_H__)
+#define __SYSINFO_H__
+
+/*!< System specific poll/timeout details */
+#define MAX_HW_TIME_US                           (10000000)
+#define DEVICES_WATCHDOG_POWER_ON_SLEEP_TIMEOUT  (3000)
+#define DEVICES_WATCHDOG_POWER_OFF_SLEEP_TIMEOUT (3600000)
+#define WAIT_TRY_COUNT                           (10000)
+#define EVENT_OBJECT_TIMEOUT_US		(120000000ULL)
+
+#define SYS_RGX_OF_COMPATIBLE "img,rgx"
+
+#if defined(__linux__)
+#define SYS_RGX_DEV_NAME "spacemit"
+#endif
+
+#endif	/* !defined(__SYSINFO_H__) */
diff --git a/drivers/gpu/drm/img-rogue/srvcore.c b/drivers/gpu/drm/img-rogue/srvcore.c
new file mode 100644
index 000000000000..203c89adda15
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/srvcore.c
@@ -0,0 +1,1621 @@
+/*************************************************************************/ /*!
+@File
+@Title          PVR Common Bridge Module (kernel side)
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements core PVRSRV API, server side
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types_check.h"
+#include "pvr_debug.h"
+#include "ra.h"
+#include "pvr_bridge.h"
+#include "connection_server.h"
+#include "device.h"
+#include "htbserver.h"
+
+#include "pdump_km.h"
+
+#include "srvkm.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "log2.h"
+
+#include "srvcore.h"
+#include "pvrsrv.h"
+#include "power.h"
+
+#include "os_apphint.h"
+
+#if defined(SUPPORT_RGX)
+#include "rgxdevice.h"
+#include "rgxinit.h"
+#include "rgx_compat_bvnc.h"
+#endif
+
+#include "rgx_options.h"
+#include "pvrversion.h"
+#include "lock.h"
+#include "osfunc.h"
+#include "device_connection.h"
+#include "process_stats.h"
+#include "pvrsrv_pool.h"
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "physmem_lma.h"
+#include "services_km.h"
+#endif
+
+#include "pvrsrv_tlstreams.h"
+#include "tlstream.h"
+
+#if defined(PVRSRV_MISSING_NO_SPEC_IMPL)
+#pragma message ("There is no implementation of OSConfineArrayIndexNoSpeculation() - see osfunc.h")
+#endif
+
+/* For the purpose of maintainability, it is intended that this file should not
+ * contain any OS specific #ifdefs. Please find a way to add e.g.
+ * an osfunc.c abstraction or override the entire function in question within
+ * env,*,pvr_bridge_k.c
+ */
+
+PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY g_BridgeDispatchTable[BRIDGE_DISPATCH_TABLE_ENTRY_COUNT] = { {.pfFunction = DummyBW,} ,};
+
+#define PVR_DISPATCH_OFFSET_FIRST_FUNC			0
+#define PVR_DISPATCH_OFFSET_LAST_FUNC			1
+#define PVR_DISPATCH_OFFSET_ARRAY_MAX			2
+
+#define PVRSRV_CLIENT_TL_STREAM_SIZE_DEFAULT PVRSRV_APPHINT_HWPERFCLIENTBUFFERSIZE
+
+static IMG_UINT16 g_BridgeDispatchTableStartOffsets[BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT][PVR_DISPATCH_OFFSET_ARRAY_MAX];
+
+
+#define PVRSRV_MAX_POOLED_BRIDGE_BUFFERS 8	/*!< Initial number of pooled bridge buffers */
+
+static PVRSRV_POOL *g_psBridgeBufferPool;	/*! Pool of bridge buffers */
+
+
+#if defined(DEBUG_BRIDGE_KM)
+/* a lock used for protecting bridge call timing calculations
+ * for calls which do not acquire a lock
+ */
+static POS_LOCK g_hStatsLock;
+PVRSRV_BRIDGE_GLOBAL_STATS g_BridgeGlobalStats;
+
+void BridgeGlobalStatsLock(void)
+{
+	OSLockAcquire(g_hStatsLock);
+}
+
+void BridgeGlobalStatsUnlock(void)
+{
+	OSLockRelease(g_hStatsLock);
+}
+#endif
+
+void BridgeDispatchTableStartOffsetsInit(void)
+{
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEFAULT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DEFAULT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEFAULT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DEFAULT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SRVCORE][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SRVCORE_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SRVCORE][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SRVCORE_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED1][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RESERVED1_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED1][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RESERVED1_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED2][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RESERVED2_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RESERVED2][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RESERVED2_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPCTRL][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPCTRL][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMPCTRL_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMPLAT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MMPLAT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMPLAT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MMPLAT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_CMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_CMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMPMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMPMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMPMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMP][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PDUMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PDUMP][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PDUMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMABUF][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DMABUF_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMABUF][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DMABUF_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CACHE][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_CACHE_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_CACHE][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_CACHE_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SMM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SMM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SMM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SMM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PVRTL][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_PVRTL_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_PVRTL][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_PVRTL_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RI][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RI_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RI][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RI_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_VALIDATION][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_VALIDATION_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_VALIDATION][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_TUTILS][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_TUTILS_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_TUTILS][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEVICEMEMHISTORY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DEVICEMEMHISTORY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_HTBUFFER][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_HTBUFFER_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_HTBUFFER][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_HTBUFFER_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DCPLAT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DCPLAT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DCPLAT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DCPLAT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMEXTMEM][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_MMEXTMEM][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_MMEXTMEM_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCTRACKING][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCTRACKING][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNCTRACKING_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCFALLBACK][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_SYNCFALLBACK][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_SYNCFALLBACK_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DI][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DI_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DI][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DI_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMA][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_DMA_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_DMA][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_DMA_DISPATCH_LAST;
+#if defined(SUPPORT_RGX)
+	/* Need a gap here to start next entry at element 128 */
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTQ_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXCMP][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXCMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXCMP][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTA3D][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTA3D_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTA3D][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXBREAKPOINT][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXBREAKPOINT][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXFWDBG][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXFWDBG][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXPDUMP][PVR_DISPATCH_OFFSET_FIRST_FUNC]= PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXPDUMP][PVR_DISPATCH_OFFSET_LAST_FUNC]= PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXHWPERF][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXHWPERF][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXREGCONFIG][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXREGCONFIG][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXKICKSYNC][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXKICKSYNC][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXKICKSYNC_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ2][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTQ2_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTQ2][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTQ2_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTIMERQUERY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXTIMERQUERY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXRAY][PVR_DISPATCH_OFFSET_FIRST_FUNC] = PVRSRV_BRIDGE_RGXRAY_DISPATCH_FIRST;
+	g_BridgeDispatchTableStartOffsets[PVRSRV_BRIDGE_RGXRAY][PVR_DISPATCH_OFFSET_LAST_FUNC] = PVRSRV_BRIDGE_RGXRAY_DISPATCH_LAST;
+#endif
+}
+
+#if defined(DEBUG_BRIDGE_KM)
+
+#if defined(INTEGRITY_OS)
+void PVRSRVPrintBridgeStats(void)
+{
+	IMG_UINT32 ui32Index;
+	IMG_UINT32 ui32Remainder;
+
+	BridgeGlobalStatsLock();
+
+	printf("Total Bridge call count = %u\n"
+		   "Total number of bytes copied via copy_from_user = %u\n"
+		   "Total number of bytes copied via copy_to_user = %u\n"
+		   "Total number of bytes copied via copy_*_user = %u\n\n"
+		   "%3s: %-60s | %-48s | %10s | %20s | %20s | %20s | %20s\n",
+		   g_BridgeGlobalStats.ui32IOCTLCount,
+		   g_BridgeGlobalStats.ui32TotalCopyFromUserBytes,
+		   g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		   g_BridgeGlobalStats.ui32TotalCopyFromUserBytes + g_BridgeGlobalStats.ui32TotalCopyToUserBytes,
+		   "#",
+		   "Bridge Name",
+		   "Wrapper Function",
+		   "Call Count",
+		   "copy_from_user (B)",
+		   "copy_to_user (B)",
+		   "Total Time (us)",
+		   "Max Time (us)");
+
+	/* Is the item asked for (starts at 0) a valid table index? */
+	for ( ui32Index=0; ui32Index < BRIDGE_DISPATCH_TABLE_ENTRY_COUNT; ui32Index++ )
+	{
+		PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY *psEntry = &g_BridgeDispatchTable[ui32Index];
+		printf("%3d: %-60s   %-48s   %-10u   %-20u   %-20u   %-20llu   %-20llu\n",
+			   (IMG_UINT32)(((size_t)psEntry-(size_t)g_BridgeDispatchTable)/sizeof(*g_BridgeDispatchTable)),
+			   psEntry->pszIOCName,
+			   (psEntry->pfFunction != NULL) ? psEntry->pszFunctionName : "(null)",
+			   psEntry->ui32CallCount,
+			   psEntry->ui32CopyFromUserTotalBytes,
+			   psEntry->ui32CopyToUserTotalBytes,
+			   (unsigned long long) OSDivide64r64(psEntry->ui64TotalTimeNS, 1000, &ui32Remainder),
+			   (unsigned long long) OSDivide64r64(psEntry->ui64MaxTimeNS, 1000, &ui32Remainder));
+
+
+	}
+
+	BridgeGlobalStatsUnlock();
+}
+#endif
+
+PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+					IMG_UINT32 ui32DispatchTableEntry,
+					void *pvDest,
+					void __user *pvSrc,
+					IMG_UINT32 ui32Size)
+{
+	BridgeGlobalStatsLock();
+	g_BridgeDispatchTable[ui32DispatchTableEntry].ui32CopyFromUserTotalBytes+=ui32Size;
+	g_BridgeGlobalStats.ui32TotalCopyFromUserBytes+=ui32Size;
+	BridgeGlobalStatsUnlock();
+
+	return OSBridgeCopyFromUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+				  IMG_UINT32 ui32DispatchTableEntry,
+				  void __user *pvDest,
+				  void *pvSrc,
+				  IMG_UINT32 ui32Size)
+{
+	BridgeGlobalStatsLock();
+	g_BridgeDispatchTable[ui32DispatchTableEntry].ui32CopyToUserTotalBytes+=ui32Size;
+	g_BridgeGlobalStats.ui32TotalCopyToUserBytes+=ui32Size;
+	BridgeGlobalStatsUnlock();
+
+	return OSBridgeCopyToUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+#endif
+
+/**************************************************************************/ /*!
+@Function       DeviceDefaultPhysHeapFreeMemCheck
+
+@Description    Check if the required amount of free space is available in the
+                Default PhysHeap for a connection to be made.
+
+@Input          psDeviceNode    The device the connection is being
+                                made on.
+@Input          ui32MinMemInMBs The minimum memory required to be
+                                available in the Default PhysHeap.
+
+@Return         PVRSRV_OK if successful else a PVRSRV_ERROR.
+*/ /***************************************************************************/
+static PVRSRV_ERROR DeviceDefaultPhysHeapFreeMemCheck(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                                      IMG_UINT32 ui32MinMemInMBs)
+{
+	PHYS_HEAP *psDefaultHeap = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psDeviceNode != NULL, "psDeviceNode");
+
+	psDefaultHeap = psDeviceNode->apsPhysHeap[psDeviceNode->psDevConfig->eDefaultHeap];
+	if (psDefaultHeap == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to get device's default PhysHeap"));
+		return PVRSRV_ERROR_INVALID_HEAP;
+	}
+
+	if (PhysHeapGetType(psDefaultHeap) == PHYS_HEAP_TYPE_LMA)
+	{
+		IMG_UINT64 ui64FreePhysHeapMem;
+
+		eError = PhysHeapFreeMemCheck(psDefaultHeap,
+		                              MB2B(ui32MinMemInMBs),
+		                              &ui64FreePhysHeapMem);
+		if (eError == PVRSRV_ERROR_INSUFFICIENT_PHYS_HEAP_MEMORY)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Default PhysHeap contains less than the "
+				"minimum free space required to acquire a connection. "
+				"Free space: %"IMG_UINT64_FMTSPEC"MB "
+				"Minimum required: %uMB",
+				B2MB(ui64FreePhysHeapMem),
+				ui32MinMemInMBs));
+		}
+	}
+
+	return eError;
+}
+
+/**************************************************************************/ /*!
+@Function       CheckConnectionPhysHeapMem
+
+@Description    Check if there is enough memory in the PhysHeaps to allow a
+                connection to be made.
+
+@Input          psConnection    The connection being made.
+
+@Return         PVRSRV_OK if successful else a PVRSRV_ERROR.
+*/ /***************************************************************************/
+static PVRSRV_ERROR CheckConnectionPhysHeapMem(CONNECTION_DATA *psConnection)
+{
+	IMG_UINT32 ui32AppHintDefault = PVRSRV_APPHINT_PHYSHEAPMINMEMONCONNECTION;
+	IMG_UINT32 ui32AppHintPhysHeapMinMemOnConnection = 0;
+	void *pvAppHintState = NULL;
+	PVRSRV_DEVICE_NODE *psDeviceNode = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_INVALID_PARAM(psConnection);
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, PhysHeapMinMemOnConnection,
+		&ui32AppHintDefault, &ui32AppHintPhysHeapMinMemOnConnection);
+	OSFreeAppHintState(pvAppHintState);
+
+	psDeviceNode = OSGetDevNode(psConnection);
+
+	if (ui32AppHintPhysHeapMinMemOnConnection != 0)
+	{
+		eError = DeviceDefaultPhysHeapFreeMemCheck(psDeviceNode,
+		                                           ui32AppHintPhysHeapMinMemOnConnection);
+		PVR_LOG_RETURN_IF_ERROR(eError, "DeviceDefaultPhysHeapFreeMemCheck");
+
+		if (psDeviceNode->pfnCheckForSufficientFWPhysMem != NULL
+		    && RGX_FW_PHYSHEAP_MINMEM_ON_CONNECTION > 0)
+		{
+			eError = psDeviceNode->pfnCheckForSufficientFWPhysMem(psDeviceNode);
+			PVR_LOG_RETURN_IF_ERROR(eError, "pfnCheckForSufficientFWPhysMem");
+		}
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVConnectKM(CONNECTION_DATA *psConnection,
+				PVRSRV_DEVICE_NODE * psDeviceNode,
+				IMG_UINT32 ui32Flags,
+				IMG_UINT32 ui32ClientBuildOptions,
+				IMG_UINT32 ui32ClientDDKVersion,
+				IMG_UINT32 ui32ClientDDKBuild,
+				IMG_UINT8  *pui8KernelArch,
+				IMG_UINT32 *pui32CapabilityFlags,
+				IMG_UINT64 *ui64PackedBvnc)
+{
+	PVRSRV_ERROR		eError = PVRSRV_OK;
+	IMG_UINT32			ui32ServerBuildOptions;
+	IMG_UINT32			ui32DDKVersion, ui32DDKBuild;
+	PVRSRV_DATA			*psSRVData = NULL;
+	IMG_UINT64			ui64ProcessVASpaceSize = OSGetCurrentProcessVASpaceSize();
+	static IMG_BOOL		bIsFirstConnection=IMG_FALSE;
+#if defined(SUPPORT_RGX)
+	PVRSRV_RGXDEV_INFO	*psDevInfo = psDeviceNode->pvDevice;
+#endif
+
+	/* Check the minimum free PhysHeap memory is available before allowing
+	 * the connection to succeed */
+	eError = CheckConnectionPhysHeapMem(psConnection);
+	PVR_RETURN_IF_ERROR(eError);
+
+#if defined(SUPPORT_RGX)
+	/* Gather BVNC information to output to UM */
+
+	*ui64PackedBvnc = rgx_bvnc_pack(psDevInfo->sDevFeatureCfg.ui32B,
+	                       psDevInfo->sDevFeatureCfg.ui32V,
+	                       psDevInfo->sDevFeatureCfg.ui32N,
+	                       psDevInfo->sDevFeatureCfg.ui32C);
+#else
+	*ui64PackedBvnc = 0;
+#endif /* defined(SUPPORT_RGX)*/
+
+	/* Clear the flags */
+	*pui32CapabilityFlags = 0;
+
+	psSRVData = PVRSRVGetPVRSRVData();
+
+	psConnection->ui32ClientFlags = ui32Flags;
+
+	/*Set flags to pass back to the client showing which cache coherency is available.*/
+	/* Is the system snooping of caches emulated in software? */
+	if (PVRSRVSystemSnoopingIsEmulated(psDeviceNode->psDevConfig))
+	{
+		*pui32CapabilityFlags |= PVRSRV_CACHE_COHERENT_EMULATE_FLAG;
+	}
+	else
+	{
+		/*Set flags to pass back to the client showing which cache coherency is available.*/
+		/*Is the system CPU cache coherent?*/
+		if (PVRSRVSystemSnoopingOfCPUCache(psDeviceNode->psDevConfig))
+		{
+			*pui32CapabilityFlags |= PVRSRV_CACHE_COHERENT_DEVICE_FLAG;
+		}
+	}
+
+	/* Has the system device non-mappable local memory?*/
+	if (PVRSRVSystemHasNonMappableLocalMemory(psDeviceNode->psDevConfig))
+	{
+		*pui32CapabilityFlags |= PVRSRV_NONMAPPABLE_MEMORY_PRESENT_FLAG;
+	}
+
+	/* Is system using FBCDC v31? */
+	if (psDeviceNode->pfnHasFBCDCVersion31(psDeviceNode))
+	{
+		*pui32CapabilityFlags |= PVRSRV_FBCDC_V3_1_USED;
+	}
+
+	/* Set flags to indicate shared-virtual-memory (SVM) allocation availability */
+	if (! psDeviceNode->ui64GeneralSVMHeapTopVA || ! ui64ProcessVASpaceSize)
+	{
+		*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_UNSUPPORTED;
+	}
+	else
+	{
+		if (ui64ProcessVASpaceSize <= psDeviceNode->ui64GeneralSVMHeapTopVA)
+		{
+			*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_SUPPORTED;
+		}
+		else
+		{
+			/* This can happen when processor has more virtual address bits
+			   than device (i.e. alloc is not always guaranteed to succeed) */
+			*pui32CapabilityFlags |= PVRSRV_DEVMEM_SVM_ALLOC_CANFAIL;
+		}
+	}
+
+	/* Is the system DMA capable? */
+	if (psDeviceNode->psDevConfig->bHasDma)
+	{
+		*pui32CapabilityFlags |= PVRSRV_SYSTEM_DMA_USED;
+	}
+
+#if defined(SUPPORT_RGX) && defined(RGX_FEATURE_TFBC_LOSSY_37_PERCENT_BIT_MASK)
+	/* For GPUs with lossy TFBC support, is system using lossy control group 1? */
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, TFBC_LOSSY_37_PERCENT))
+	{
+		if (psDeviceNode->pfnGetTFBCLossyGroup(psDeviceNode) == 1)
+		{
+			*pui32CapabilityFlags |= PVRSRV_TFBC_LOSSY_GROUP_1;
+		}
+	}
+#endif
+
+#if defined(SUPPORT_CUSTOM_OSID_EMISSION)
+{
+	IMG_UINT32 ui32OSid = 0, ui32OSidReg = 0;
+	IMG_BOOL   bOSidAxiProtReg = IMG_FALSE;
+
+	ui32OSid    = (ui32Flags & SRV_VIRTVAL_FLAG_OSID_MASK)    >> (VIRTVAL_FLAG_OSID_SHIFT);
+	ui32OSidReg = (ui32Flags & SRV_VIRTVAL_FLAG_OSIDREG_MASK) >> (VIRTVAL_FLAG_OSIDREG_SHIFT);
+
+#if defined(EMULATOR)
+{
+    /* AXI_ACELITE is only supported on rogue cores - volcanic cores all support full ACE
+     * and don't want to compile the code below (RGX_FEATURE_AXI_ACELITE_BIT_MASK is not
+     * defined for volcanic cores).
+     */
+
+     PVRSRV_RGXDEV_INFO *psDevInfo;
+     psDevInfo = (PVRSRV_RGXDEV_INFO *) psDeviceNode->pvDevice;
+
+#if defined(RGX_FEATURE_AXI_ACELITE_BIT_MASK)
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACELITE))
+#else
+	if (RGX_IS_FEATURE_SUPPORTED(psDevInfo, AXI_ACE))
+#endif
+	{
+		IMG_UINT32 ui32OSidAxiProtReg = 0, ui32OSidAxiProtTD = 0;
+
+		ui32OSidAxiProtReg = (ui32Flags & SRV_VIRTVAL_FLAG_AXIPREG_MASK) >> (VIRTVAL_FLAG_AXIPREG_SHIFT);
+		ui32OSidAxiProtTD  = (ui32Flags & SRV_VIRTVAL_FLAG_AXIPTD_MASK)  >> (VIRTVAL_FLAG_AXIPTD_SHIFT);
+
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"[AxiProt & Virt]: Setting bOSidAxiProt of Emulator's Trusted Device for Catbase %d to %s",
+				ui32OSidReg,
+				(ui32OSidAxiProtTD == 1)?"TRUE":"FALSE"));
+
+		bOSidAxiProtReg = ui32OSidAxiProtReg == 1;
+		PVR_DPF((PVR_DBG_MESSAGE,
+				"[AxiProt & Virt]: Setting bOSidAxiProt of FW's Register for Catbase %d to %s",
+				ui32OSidReg,
+				bOSidAxiProtReg?"TRUE":"FALSE"));
+
+		SetAxiProtOSid(psDeviceNode->psDevConfig->hSysData, ui32OSidReg, ui32OSidAxiProtTD);
+	}
+}
+#endif /* defined(EMULATOR) */
+
+	/* We now know the OSid, OSidReg and bOSidAxiProtReg setting for this
+	 * connection. We can access these from wherever we have a connection
+	 * reference and do not need to traverse an arbitrary linked-list to
+	 * obtain them. The settings are process-specific.
+	 */
+	psConnection->ui32OSid = ui32OSid;
+	psConnection->ui32OSidReg = ui32OSidReg;
+	psConnection->bOSidAxiProtReg = bOSidAxiProtReg;
+
+	PVR_DPF((PVR_DBG_MESSAGE,
+	         "[GPU Virtualization Validation]: OSIDs: %d, %d",
+	         ui32OSid,
+	         ui32OSidReg));
+}
+#endif	/* defined(SUPPORT_CUSTOM_OSID_EMISSION) */
+
+#if defined(SUPPORT_WORKLOAD_ESTIMATION)
+	/* Only enabled if enabled in the UM */
+	if (!(ui32ClientBuildOptions & RGX_BUILD_OPTIONS_KM & OPTIONS_WORKLOAD_ESTIMATION_EN))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Workload Estimation disabled. Not enabled in UM",
+				__func__));
+	}
+#endif
+
+#if defined(SUPPORT_PDVFS)
+	/* Only enabled if enabled in the UM */
+	if (!(ui32ClientBuildOptions & RGX_BUILD_OPTIONS_KM & OPTIONS_PDVFS_EN))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Proactive DVFS disabled. Not enabled in UM",
+		         __func__));
+	}
+#endif
+
+	ui32DDKVersion = PVRVERSION_PACK(PVRVERSION_MAJ, PVRVERSION_MIN);
+	ui32DDKBuild = PVRVERSION_BUILD;
+
+	if (ui32Flags & SRV_FLAGS_CLIENT_64BIT_COMPAT)
+	{
+		psSRVData->sDriverInfo.ui8UMSupportedArch |= BUILD_ARCH_64BIT;
+	}
+	else
+	{
+		psSRVData->sDriverInfo.ui8UMSupportedArch |= BUILD_ARCH_32BIT;
+	}
+
+	if (IMG_FALSE == bIsFirstConnection)
+	{
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildOptions = (RGX_BUILD_OPTIONS_KM);
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildOptions = ui32ClientBuildOptions;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildVersion = ui32DDKVersion;
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildVersion = ui32ClientDDKVersion;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildRevision = ui32DDKBuild;
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildRevision = ui32ClientDDKBuild;
+
+		psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildType =
+				((RGX_BUILD_OPTIONS_KM) & OPTIONS_DEBUG_EN) ? BUILD_TYPE_DEBUG : BUILD_TYPE_RELEASE;
+
+		psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildType =
+				(ui32ClientBuildOptions & OPTIONS_DEBUG_EN) ? BUILD_TYPE_DEBUG : BUILD_TYPE_RELEASE;
+
+		if (sizeof(void *) == POINTER_SIZE_64BIT)
+		{
+			psSRVData->sDriverInfo.ui8KMBitArch |= BUILD_ARCH_64BIT;
+		}
+		else
+		{
+			psSRVData->sDriverInfo.ui8KMBitArch |= BUILD_ARCH_32BIT;
+		}
+	}
+
+	/* Masking out every option that is not kernel specific*/
+	ui32ClientBuildOptions &= RGX_BUILD_OPTIONS_MASK_KM;
+
+	/*
+	 * Validate the build options
+	 */
+	ui32ServerBuildOptions = (RGX_BUILD_OPTIONS_KM);
+	if (ui32ServerBuildOptions != ui32ClientBuildOptions)
+	{
+		IMG_UINT32			ui32ServerBuildOptionsMismatch = ui32ServerBuildOptions ^ ui32ClientBuildOptions;
+		IMG_UINT32			ui32ClientBuildOptionsMismatch = ui32ServerBuildOptionsMismatch;
+
+#if !defined(PVRSRV_STRICT_COMPAT_CHECK)
+		/*Mask the debug flag option out as we do support combinations of debug vs release in um & km*/
+		ui32ServerBuildOptionsMismatch &= KM_OPTIONS_STRICT;
+		ui32ClientBuildOptionsMismatch &= UM_OPTIONS_STRICT;
+#endif
+		if ( (ui32ClientBuildOptions & ui32ClientBuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) %s: Mismatch in client-side and KM driver build options; "
+				"extra options present in client-side driver: (0x%x). Please check rgx_options.h",
+				__func__,
+				ui32ClientBuildOptions & ui32ClientBuildOptionsMismatch));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH, chk_exit);
+		}
+
+		if ( (ui32ServerBuildOptions & ui32ServerBuildOptionsMismatch) != 0)
+		{
+			PVR_LOG(("(FAIL) %s: Mismatch in client-side and KM driver build options; "
+				"extra options present in KM driver: (0x%x). Please check rgx_options.h",
+				__func__,
+				ui32ServerBuildOptions & ui32ServerBuildOptionsMismatch ));
+			PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_BUILD_OPTIONS_MISMATCH, chk_exit);
+		}
+		if (IMG_FALSE == bIsFirstConnection)
+		{
+			PVR_LOG(("%s: COMPAT_TEST: Client-side (0x%04x) (%s) and KM driver (0x%04x) (%s) build options differ.",
+																			__func__,
+																			ui32ClientBuildOptions,
+																			(psSRVData->sDriverInfo.sUMBuildInfo.ui32BuildType)?"release":"debug",
+																			ui32ServerBuildOptions,
+																			(psSRVData->sDriverInfo.sKMBuildInfo.ui32BuildType)?"release":"debug"));
+		}else{
+			PVR_DPF((PVR_DBG_WARNING, "%s: COMPAT_TEST: Client-side (0x%04x) and KM driver (0x%04x) build options differ.",
+																		__func__,
+																		ui32ClientBuildOptions,
+																		ui32ServerBuildOptions));
+
+		}
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: Client-side and KM driver build options match. [ OK ]", __func__));
+	}
+
+	/*
+	 * Validate DDK version
+	 */
+	if (ui32ClientDDKVersion != ui32DDKVersion)
+	{
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+		PVR_LOG(("(FAIL) %s: Incompatible driver DDK version (%u.%u) / client DDK version (%u.%u).",
+				__func__,
+				PVRVERSION_MAJ, PVRVERSION_MIN,
+				PVRVERSION_UNPACK_MAJ(ui32ClientDDKVersion),
+				PVRVERSION_UNPACK_MIN(ui32ClientDDKVersion)));
+		PVR_DBG_BREAK;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DDK_VERSION_MISMATCH, chk_exit);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: driver DDK version (%u.%u) and client DDK version (%u.%u) match. [ OK ]",
+				__func__,
+				PVRVERSION_MAJ, PVRVERSION_MIN, PVRVERSION_MAJ, PVRVERSION_MIN));
+	}
+
+	/* Create stream for every connection except for the special clients
+	 * that don't need it e.g.: recipients of HWPerf data. */
+	if (!(psConnection->ui32ClientFlags & SRV_NO_HWPERF_CLIENT_STREAM))
+	{
+		IMG_CHAR acStreamName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+		OSSNPrintf(acStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE,
+		           PVRSRV_TL_HWPERF_HOST_CLIENT_STREAM_FMTSPEC,
+		           psDeviceNode->sDevId.i32KernelDeviceID,
+		           psConnection->pid);
+
+		eError = TLStreamCreate(&psConnection->hClientTLStream,
+		                        acStreamName,
+		                        PVRSRV_CLIENT_TL_STREAM_SIZE_DEFAULT,
+		                        TL_OPMODE_DROP_NEWER |
+		                        TL_FLAG_ALLOCATE_ON_FIRST_OPEN,
+		                        NULL, NULL, NULL, NULL, NULL, NULL);
+		if (eError != PVRSRV_OK && eError != PVRSRV_ERROR_ALREADY_EXISTS)
+		{
+			PVR_LOG_ERROR(eError, "TLStreamCreate");
+			psConnection->hClientTLStream = NULL;
+		}
+		else if (eError == PVRSRV_OK)
+		{
+			/* Set "tlctrl" stream as a notification channel. This channel is
+			 * is used to notify recipients about stream open/close (by writer)
+			 * actions (and possibly other actions in the future). */
+			eError = TLStreamSetNotifStream(psConnection->hClientTLStream,
+			                                psSRVData->hTLCtrlStream);
+			if (eError != PVRSRV_OK)
+			{
+				PVR_LOG_ERROR(eError, "TLStreamSetNotifStream");
+				TLStreamClose(psConnection->hClientTLStream);
+				psConnection->hClientTLStream = NULL;
+			}
+		}
+
+		/* Reset error status. We don't want to propagate any errors from here. */
+		eError = PVRSRV_OK;
+		PVR_DPF((PVR_DBG_MESSAGE, "Created stream \"%s\".", acStreamName));
+	}
+
+	/*
+	 * Validate DDK build
+	 */
+	if (ui32ClientDDKBuild != ui32DDKBuild)
+	{
+		if (!psSRVData->sDriverInfo.bIsNoMatch)
+			psSRVData->sDriverInfo.bIsNoMatch = IMG_TRUE;
+		PVR_DPF((PVR_DBG_WARNING, "%s: Mismatch in driver DDK revision (%d) / client DDK revision (%d).",
+				__func__, ui32DDKBuild, ui32ClientDDKBuild));
+#if defined(PVRSRV_STRICT_COMPAT_CHECK)
+		PVR_DBG_BREAK;
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_DDK_BUILD_MISMATCH, chk_exit);
+#endif
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: COMPAT_TEST: driver DDK revision (%d) and client DDK revision (%d) match. [ OK ]",
+				__func__, ui32DDKBuild, ui32ClientDDKBuild));
+	}
+
+#if defined(PDUMP)
+	/* Success so far so is it the PDump client that is connecting? */
+	if (ui32Flags & SRV_FLAGS_PDUMPCTRL)
+	{
+		if (psDeviceNode->sDevId.ui32InternalID == psSRVData->ui32PDumpBoundDevice)
+		{
+			PDumpConnectionNotify(psDeviceNode);
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_PDUMP_CAPTURE_BOUND_TO_ANOTHER_DEVICE;
+			PVR_DPF((PVR_DBG_ERROR, "%s: PDump requested for device %u but only permitted for device %u",
+					__func__, psDeviceNode->sDevId.ui32InternalID, psSRVData->ui32PDumpBoundDevice));
+			goto chk_exit;
+		}
+	}
+	else
+	{
+		/* Warn if the app is connecting to a device PDump won't be able to capture */
+		if (psDeviceNode->sDevId.ui32InternalID != psSRVData->ui32PDumpBoundDevice)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s: NB. App running on device %d won't be captured by PDump (must be on device %u)",
+					__func__, psDeviceNode->sDevId.ui32InternalID, psSRVData->ui32PDumpBoundDevice));
+		}
+	}
+#endif
+
+	PVR_ASSERT(pui8KernelArch != NULL);
+
+	if (psSRVData->sDriverInfo.ui8KMBitArch & BUILD_ARCH_64BIT)
+	{
+		*pui8KernelArch = 64;
+	}
+	else
+	{
+		*pui8KernelArch = 32;
+	}
+
+	bIsFirstConnection = IMG_TRUE;
+
+#if defined(DEBUG_BRIDGE_KM)
+	{
+		int ii;
+
+		/* dump dispatch table offset lookup table */
+		PVR_DPF((PVR_DBG_MESSAGE, "%s: g_BridgeDispatchTableStartOffsets[0-%lu] entries:", __func__, BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT - 1));
+		for (ii=0; ii < BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT; ii++)
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "g_BridgeDispatchTableStartOffsets[%d]: %u", ii, g_BridgeDispatchTableStartOffsets[ii][PVR_DISPATCH_OFFSET_FIRST_FUNC]));
+		}
+	}
+#endif
+
+#if defined(PDUMP)
+	if (!(ui32Flags & SRV_FLAGS_PDUMPCTRL))
+	{
+		IMG_UINT64 ui64PDumpState = 0;
+
+		PDumpGetStateKM(&ui64PDumpState);
+		if (ui64PDumpState & PDUMP_STATE_CONNECTED)
+		{
+			*pui32CapabilityFlags |= PVRSRV_PDUMP_IS_RECORDING;
+		}
+	}
+#endif
+
+chk_exit:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVDisconnectKM(void)
+{
+#if defined(INTEGRITY_OS) && defined(DEBUG_BRIDGE_KM)
+	PVRSRVPrintBridgeStats();
+#endif
+	/* just return OK, per-process data is cleaned up by resmgr */
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       PVRSRVAcquireGlobalEventObjectKM
+@Description    Acquire the global event object.
+@Output         phGlobalEventObject    On success, points to the global event
+                                       object handle
+@Return         PVRSRV_ERROR           PVRSRV_OK on success or an error
+                                       otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR
+PVRSRVAcquireGlobalEventObjectKM(IMG_HANDLE *phGlobalEventObject)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	*phGlobalEventObject = psPVRSRVData->hGlobalEventObject;
+
+	return PVRSRV_OK;
+}
+
+/**************************************************************************/ /*!
+@Function       PVRSRVReleaseGlobalEventObjectKM
+@Description    Release the global event object.
+@Output         hGlobalEventObject    Global event object handle
+@Return         PVRSRV_ERROR          PVRSRV_OK on success or an error otherwise
+*/ /***************************************************************************/
+PVRSRV_ERROR
+PVRSRVReleaseGlobalEventObjectKM(IMG_HANDLE hGlobalEventObject)
+{
+	PVR_ASSERT(PVRSRVGetPVRSRVData()->hGlobalEventObject == hGlobalEventObject);
+
+	return PVRSRV_OK;
+}
+
+static void _DumpDebugUMReqPrintWrapper(void *pvPriv, const IMG_CHAR *pszFmt, ...)
+{
+	va_list pvArgs;
+	PVR_UNREFERENCED_PARAMETER(pvPriv);
+
+	va_start(pvArgs, pszFmt);
+	PVRSRVReleasePrintfVArgs(pszFmt, pvArgs);
+	va_end(pvArgs);
+}
+
+/*
+	PVRSRVDumpDebugInfoKM
+*/
+PVRSRV_ERROR
+PVRSRVDumpDebugInfoKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDeviceNode,
+					  IMG_UINT32 ui32VerbLevel)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32VerbLevel > DEBUG_REQUEST_VERBOSITY_MAX)
+	{
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+	PVR_LOG(("User requested PVR debug info"));
+
+	PVRSRVDebugRequest(psDeviceNode, ui32VerbLevel, _DumpDebugUMReqPrintWrapper, NULL);
+
+	return PVRSRV_OK;
+}
+
+/*
+	PVRSRVGetDevClockSpeedKM
+*/
+PVRSRV_ERROR
+PVRSRVGetDevClockSpeedKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_PUINT32  pui32RGXClockSpeed)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PVR_ASSERT(psDeviceNode->pfnDeviceClockSpeed != NULL);
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	eError = psDeviceNode->pfnDeviceClockSpeed(psDeviceNode, pui32RGXClockSpeed);
+	PVR_WARN_IF_ERROR(eError, "pfnDeviceClockSpeed");
+
+	return eError;
+}
+
+
+/*
+	PVRSRVHWOpTimeoutKM
+*/
+PVRSRV_ERROR
+PVRSRVHWOpTimeoutKM(CONNECTION_DATA *psConnection,
+					PVRSRV_DEVICE_NODE *psDeviceNode)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(PVRSRV_RESET_ON_HWTIMEOUT)
+	PVR_LOG(("User requested OS reset"));
+	OSPanic();
+#endif
+	PVR_LOG(("HW operation timeout, dump server info"));
+	PVRSRVDebugRequest(psDeviceNode, DEBUG_REQUEST_VERBOSITY_MAX, NULL, NULL);
+	return PVRSRV_OK;
+}
+
+
+IMG_INT
+DummyBW(IMG_UINT32 ui32DispatchTableEntry,
+		IMG_UINT8 *psBridgeIn,
+		IMG_UINT8 *psBridgeOut,
+		CONNECTION_DATA *psConnection)
+{
+	PVR_UNREFERENCED_PARAMETER(psBridgeIn);
+	PVR_UNREFERENCED_PARAMETER(psBridgeOut);
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if defined(DEBUG_BRIDGE_KM)
+	PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE ERROR: ui32DispatchTableEntry %u (%s) mapped to "
+			 "Dummy Wrapper (probably not what you want!)",
+			 __func__, ui32DispatchTableEntry, g_BridgeDispatchTable[ui32DispatchTableEntry].pszIOCName));
+#else
+	PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE ERROR: ui32DispatchTableEntry %u mapped to "
+			 "Dummy Wrapper (probably not what you want!)",
+			 __func__, ui32DispatchTableEntry));
+#endif
+	return PVRSRV_ERROR_BRIDGE_ENOTTY;
+}
+
+PVRSRV_ERROR PVRSRVAlignmentCheckKM(CONNECTION_DATA *psConnection,
+                                    PVRSRV_DEVICE_NODE *psDeviceNode,
+                                    IMG_UINT32 ui32AlignChecksSize,
+                                    IMG_UINT32 aui32AlignChecks[])
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+#if !defined(NO_HARDWARE)
+
+	PVR_ASSERT(psDeviceNode->pfnAlignmentCheck != NULL);
+	return psDeviceNode->pfnAlignmentCheck(psDeviceNode, ui32AlignChecksSize,
+	                                       aui32AlignChecks);
+
+#else
+
+	PVR_UNREFERENCED_PARAMETER(psDeviceNode);
+	PVR_UNREFERENCED_PARAMETER(ui32AlignChecksSize);
+	PVR_UNREFERENCED_PARAMETER(aui32AlignChecks);
+
+	return PVRSRV_OK;
+
+#endif /* !defined(NO_HARDWARE) */
+
+}
+
+PVRSRV_ERROR PVRSRVGetDeviceStatusKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 *pui32DeviceStatus)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	/* First try to update the status. */
+	if (psDeviceNode->pfnUpdateHealthStatus != NULL)
+	{
+		PVRSRV_ERROR eError = psDeviceNode->pfnUpdateHealthStatus(psDeviceNode,
+		                                                          IMG_FALSE);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "PVRSRVGetDeviceStatusKM: Failed to "
+					 "check for device status (%d)", eError));
+
+			/* Return unknown status and error because we don't know what
+			 * happened and if the status is valid. */
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_UNKNOWN;
+			return eError;
+		}
+	}
+
+	switch (OSAtomicRead(&psDeviceNode->eHealthStatus))
+	{
+		case PVRSRV_DEVICE_HEALTH_STATUS_OK:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_OK;
+			return PVRSRV_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_NOT_RESPONDING:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_NOT_RESPONDING;
+			return PVRSRV_OK;
+		case PVRSRV_DEVICE_HEALTH_STATUS_DEAD:
+		case PVRSRV_DEVICE_HEALTH_STATUS_FAULT:
+		case PVRSRV_DEVICE_HEALTH_STATUS_UNDEFINED:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_DEVICE_ERROR;
+			return PVRSRV_OK;
+		default:
+			*pui32DeviceStatus = PVRSRV_DEVICE_STATUS_UNKNOWN;
+			return PVRSRV_ERROR_INTERNAL_ERROR;
+	}
+}
+
+PVRSRV_ERROR PVRSRVGetMultiCoreInfoKM(CONNECTION_DATA *psConnection,
+                                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                                      IMG_UINT32 ui32CapsSize,
+                                      IMG_UINT32 *pui32NumCores,
+                                      IMG_UINT64 *pui64Caps)
+{
+	PVRSRV_ERROR eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	if (ui32CapsSize > 0)
+	{
+		/* Clear the buffer to ensure no uninitialised data is returned to UM
+		 * if the pfn call below does not write to the whole array, or is null.
+		 */
+		memset(pui64Caps, 0x00, (ui32CapsSize * sizeof(IMG_UINT64)));
+	}
+
+	if (psDeviceNode->pfnGetMultiCoreInfo != NULL)
+	{
+		eError = psDeviceNode->pfnGetMultiCoreInfo(psDeviceNode, ui32CapsSize, pui32NumCores, pui64Caps);
+	}
+	return eError;
+}
+
+
+/*!
+ * *****************************************************************************
+ * @brief A wrapper for removing entries in the g_BridgeDispatchTable array.
+ *		  All this does is zero the entry to allow for a full table re-population
+ *		  later.
+ *
+ * @param ui32BridgeGroup
+ * @param ui32Index
+ *
+ * @return
+ ********************************************************************************/
+void
+UnsetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup, IMG_UINT32 ui32Index)
+{
+	ui32Index += g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+
+	g_BridgeDispatchTable[ui32Index].pfFunction = NULL;
+	g_BridgeDispatchTable[ui32Index].hBridgeLock = NULL;
+#if defined(DEBUG_BRIDGE_KM)
+	g_BridgeDispatchTable[ui32Index].pszIOCName = NULL;
+	g_BridgeDispatchTable[ui32Index].pszFunctionName = NULL;
+	g_BridgeDispatchTable[ui32Index].pszBridgeLockName = NULL;
+	g_BridgeDispatchTable[ui32Index].ui32CallCount = 0;
+	g_BridgeDispatchTable[ui32Index].ui32CopyFromUserTotalBytes = 0;
+	g_BridgeDispatchTable[ui32Index].ui64TotalTimeNS = 0;
+	g_BridgeDispatchTable[ui32Index].ui64MaxTimeNS = 0;
+#endif
+}
+
+/*!
+ * *****************************************************************************
+ * @brief A wrapper for filling in the g_BridgeDispatchTable array that does
+ *		  error checking.
+ *
+ * @param ui32Index
+ * @param pszIOCName
+ * @param pfFunction
+ * @param pszFunctionName
+ * @param hBridgeLock
+ * @param pszBridgeLockName
+ * @param ui32InBufferSize
+ * @param ui32OutBufferSize
+ *
+ * @return
+ ********************************************************************************/
+void
+_SetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+					   IMG_UINT32 ui32Index,
+					   const IMG_CHAR *pszIOCName,
+					   BridgeWrapperFunction pfFunction,
+					   const IMG_CHAR *pszFunctionName,
+					   POS_LOCK hBridgeLock,
+					   const IMG_CHAR *pszBridgeLockName,
+					   IMG_UINT32 ui32InBufferSize,
+					   IMG_UINT32 ui32OutBufferSize)
+{
+	static IMG_UINT32 ui32PrevIndex = IMG_UINT32_MAX;		/* -1 */
+
+#if !defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE) && !defined(DEBUG_BRIDGE_KM)
+	PVR_UNREFERENCED_PARAMETER(pszFunctionName);
+	PVR_UNREFERENCED_PARAMETER(pszBridgeLockName);
+#endif
+
+	ui32Index += g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+	/* Enable this to dump out the dispatch table entries */
+	PVR_DPF((PVR_DBG_WARNING, "%s: g_BridgeDispatchTableStartOffsets[%d]=%d", __func__, ui32BridgeGroup, g_BridgeDispatchTableStartOffsets[ui32BridgeGroup][PVR_DISPATCH_OFFSET_FIRST_FUNC]));
+	PVR_DPF((PVR_DBG_WARNING, "%s: %d %s %s %s", __func__, ui32Index, pszIOCName, pszFunctionName, pszBridgeLockName));
+#endif
+
+	/* Any gaps are sub-optimal in-terms of memory usage, but we are mainly
+	 * interested in spotting any large gap of wasted memory that could be
+	 * accidentally introduced.
+	 *
+	 * This will currently flag up any gaps > 5 entries.
+	 *
+	 * NOTE: This shouldn't be debug only since switching from debug->release
+	 * etc is likely to modify the available ioctls and thus be a point where
+	 * mistakes are exposed. This isn't run at a performance critical time.
+	 */
+	if ((ui32PrevIndex != IMG_UINT32_MAX) &&
+		((ui32Index >= ui32PrevIndex + DISPATCH_TABLE_GAP_THRESHOLD) ||
+		 (ui32Index <= ui32PrevIndex)))
+	{
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s: There is a gap in the dispatch table between indices %u (%s) and %u (%s)",
+				 __func__, ui32PrevIndex, g_BridgeDispatchTable[ui32PrevIndex].pszIOCName,
+				 ui32Index, pszIOCName));
+#else
+		PVR_DPF((PVR_DBG_MESSAGE,
+				 "%s: There is a gap in the dispatch table between indices %u and %u (%s)",
+				 __func__, (IMG_UINT)ui32PrevIndex, (IMG_UINT)ui32Index, pszIOCName));
+#endif
+	}
+
+	if (ui32Index >= BRIDGE_DISPATCH_TABLE_ENTRY_COUNT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Index %u (%s) out of range",
+				 __func__, (IMG_UINT)ui32Index, pszIOCName));
+
+#if defined(DEBUG_BRIDGE_KM)
+		PVR_DPF((PVR_DBG_ERROR, "%s: BRIDGE_DISPATCH_TABLE_ENTRY_COUNT = %lu",
+				 __func__, BRIDGE_DISPATCH_TABLE_ENTRY_COUNT));
+#if defined(SUPPORT_RGX)
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXREGCONFIG_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXHWPERF_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXPDUMP_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXFWDBG_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXBREAKPOINT_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTA3D_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXCMP_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTQ_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGXTIMERQUERY_DISPATCH_LAST));
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGX_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGX_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_RGX_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_RGX_LAST));
+#endif
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_DEVICEMEMHISTORY_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_TUTILS_DISPATCH_LAST));
+		PVR_DPF((PVR_DBG_ERROR, "%s: PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST = %lu",
+				 __func__, PVRSRV_BRIDGE_VALIDATION_DISPATCH_LAST));
+#endif
+
+		OSPanic();
+	}
+
+	/* Panic if the previous entry has been overwritten as this is not allowed!
+	 * NOTE: This shouldn't be debug only since switching from debug->release
+	 * etc is likely to modify the available ioctls and thus be a point where
+	 * mistakes are exposed. This isn't run at a performance critical time.
+	 */
+	if (g_BridgeDispatchTable[ui32Index].pfFunction)
+	{
+		if (g_BridgeDispatchTable[ui32Index].pfFunction != pfFunction)
+		{
+#if defined(DEBUG_BRIDGE_KM_DISPATCH_TABLE)
+			PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Adding dispatch table entry for %s clobbers an existing entry for %s (current pfn=<%p>, new pfn=<%p>)",
+				 __func__, pszIOCName, g_BridgeDispatchTable[ui32Index].pszIOCName,
+				 (void*)g_BridgeDispatchTable[ui32Index].pfFunction, (void*)pfFunction));
+#else
+			PVR_DPF((PVR_DBG_ERROR,
+				 "%s: Adding dispatch table entry for %s clobbers an existing entry (index=%u). (current pfn=<%p>, new pfn=<%p>)",
+				 __func__, pszIOCName, ui32Index,
+				 (void*)g_BridgeDispatchTable[ui32Index].pfFunction, (void*)pfFunction));
+			PVR_DPF((PVR_DBG_WARNING, "NOTE: Enabling DEBUG_BRIDGE_KM_DISPATCH_TABLE may help debug this issue."));
+#endif
+			OSPanic();
+		}
+	}
+	else
+	{
+		g_BridgeDispatchTable[ui32Index].pfFunction = pfFunction;
+		g_BridgeDispatchTable[ui32Index].hBridgeLock = hBridgeLock;
+		g_BridgeDispatchTable[ui32Index].ui32InBufferSize = ui32InBufferSize;
+		g_BridgeDispatchTable[ui32Index].ui32OutBufferSize = ui32OutBufferSize;
+#if defined(DEBUG_BRIDGE_KM)
+		g_BridgeDispatchTable[ui32Index].pszIOCName = pszIOCName;
+		g_BridgeDispatchTable[ui32Index].pszFunctionName = pszFunctionName;
+		g_BridgeDispatchTable[ui32Index].pszBridgeLockName = pszBridgeLockName;
+		g_BridgeDispatchTable[ui32Index].ui32CallCount = 0;
+		g_BridgeDispatchTable[ui32Index].ui32CopyFromUserTotalBytes = 0;
+		g_BridgeDispatchTable[ui32Index].ui64TotalTimeNS = 0;
+		g_BridgeDispatchTable[ui32Index].ui64MaxTimeNS = 0;
+#endif
+	}
+
+	ui32PrevIndex = ui32Index;
+}
+
+static PVRSRV_ERROR _BridgeBufferAlloc(void *pvPrivData, void **pvOut)
+{
+	PVR_UNREFERENCED_PARAMETER(pvPrivData);
+
+	*pvOut = OSAllocZMem(PVRSRV_MAX_BRIDGE_IN_SIZE +
+	                     PVRSRV_MAX_BRIDGE_OUT_SIZE);
+	PVR_RETURN_IF_NOMEM(*pvOut);
+
+	return PVRSRV_OK;
+}
+
+static void _BridgeBufferFree(void *pvPrivData, void *pvFreeData)
+{
+	PVR_UNREFERENCED_PARAMETER(pvPrivData);
+
+	OSFreeMem(pvFreeData);
+}
+
+PVRSRV_ERROR BridgeDispatcherInit(void)
+{
+	PVRSRV_ERROR eError;
+
+#if defined(DEBUG_BRIDGE_KM)
+	eError = OSLockCreate(&g_hStatsLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", errorLockCreateFailed);
+#endif
+
+	eError = PVRSRVPoolCreate(_BridgeBufferAlloc,
+	                          _BridgeBufferFree,
+	                          PVRSRV_MAX_POOLED_BRIDGE_BUFFERS,
+	                          "Bridge buffer pool",
+	                          NULL,
+	                          &g_psBridgeBufferPool);
+	PVR_LOG_GOTO_IF_ERROR(eError, "PVRSRVPoolCreate", erroPoolCreateFailed);
+
+	return PVRSRV_OK;
+
+erroPoolCreateFailed:
+#if defined(DEBUG_BRIDGE_KM)
+	OSLockDestroy(g_hStatsLock);
+	g_hStatsLock = NULL;
+errorLockCreateFailed:
+#endif
+	return eError;
+}
+
+void BridgeDispatcherDeinit(void)
+{
+	if (g_psBridgeBufferPool)
+	{
+		PVRSRVPoolDestroy(g_psBridgeBufferPool);
+		g_psBridgeBufferPool = NULL;
+	}
+
+#if defined(DEBUG_BRIDGE_KM)
+	if (g_hStatsLock)
+	{
+		OSLockDestroy(g_hStatsLock);
+		g_hStatsLock = NULL;
+	}
+#endif
+}
+
+PVRSRV_ERROR BridgedDispatchKM(CONNECTION_DATA * psConnection,
+                          PVRSRV_BRIDGE_PACKAGE   * psBridgePackageKM)
+{
+
+	void       * psBridgeIn=NULL;
+	void       * psBridgeOut=NULL;
+	BridgeWrapperFunction pfBridgeHandler;
+	IMG_UINT32   ui32DispatchTableEntry, ui32GroupBoundary;
+	PVRSRV_ERROR err = PVRSRV_OK;
+#if !defined(INTEGRITY_OS)
+	PVRSRV_POOL_TOKEN hBridgeBufferPoolToken = NULL;
+#endif
+	IMG_UINT32 ui32Timestamp = OSClockus();
+#if defined(DEBUG_BRIDGE_KM)
+	IMG_UINT64	ui64TimeStart;
+	IMG_UINT64	ui64TimeEnd;
+	IMG_UINT64	ui64TimeDiff;
+#endif
+	IMG_UINT32	ui32DispatchTableIndex, ui32DispatchTableEntryIndex;
+
+#if defined(DEBUG_BRIDGE_KM_STOP_AT_DISPATCH)
+	PVR_DBG_BREAK;
+#endif
+
+#if !defined(PVRSRV_ENABLE_HTB)
+	PVR_UNREFERENCED_PARAMETER(ui32Timestamp);
+#endif
+
+	if (psBridgePackageKM->ui32BridgeID >= BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Out of range dispatch table group ID: %d",
+		        __func__, psBridgePackageKM->ui32BridgeID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+	ui32DispatchTableIndex = OSConfineArrayIndexNoSpeculation(psBridgePackageKM->ui32BridgeID, BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT);
+
+	ui32DispatchTableEntry = g_BridgeDispatchTableStartOffsets[ui32DispatchTableIndex][PVR_DISPATCH_OFFSET_FIRST_FUNC];
+	ui32GroupBoundary = g_BridgeDispatchTableStartOffsets[ui32DispatchTableIndex][PVR_DISPATCH_OFFSET_LAST_FUNC];
+
+	/* bridge function is not implemented in this build */
+	if (0 == ui32DispatchTableEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Dispatch table entry=%d, boundary = %d, (bridge module %d, function %d)",
+		         __func__,
+		         ui32DispatchTableEntry,
+		         ui32GroupBoundary,
+		         psBridgePackageKM->ui32BridgeID,
+		         psBridgePackageKM->ui32FunctionID));
+		/* this points to DummyBW() which returns PVRSRV_ERROR_ENOTTY */
+		err = g_BridgeDispatchTable[ui32DispatchTableEntry].pfFunction(ui32DispatchTableEntry,
+				  psBridgeIn,
+				  psBridgeOut,
+				  psConnection);
+		goto return_error;
+	}
+	if ((ui32DispatchTableEntry + psBridgePackageKM->ui32FunctionID) > ui32GroupBoundary)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Dispatch table entry=%d, boundary = %d, (bridge module %d, function %d)",
+		         __func__,
+		         ui32DispatchTableEntry,
+		         ui32GroupBoundary,
+		         psBridgePackageKM->ui32BridgeID,
+		         psBridgePackageKM->ui32FunctionID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+	ui32DispatchTableEntry += psBridgePackageKM->ui32FunctionID;
+	ui32DispatchTableEntryIndex = OSConfineArrayIndexNoSpeculation(ui32DispatchTableEntry, ui32GroupBoundary+1);
+	if (BRIDGE_DISPATCH_TABLE_ENTRY_COUNT <= ui32DispatchTableEntry)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Dispatch table entry=%d, entry count = %lu,"
+		        " (bridge module %d, function %d)", __func__,
+		        ui32DispatchTableEntry, BRIDGE_DISPATCH_TABLE_ENTRY_COUNT,
+		        psBridgePackageKM->ui32BridgeID,
+		        psBridgePackageKM->ui32FunctionID));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+#if defined(DEBUG_BRIDGE_KM)
+	BridgeGlobalStatsLock();
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: %s idx:%d mod:%d, func:%d",
+			__func__, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].pszIOCName,
+			ui32DispatchTableEntryIndex, psBridgePackageKM->ui32BridgeID, psBridgePackageKM->ui32FunctionID));
+
+	g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32CallCount++;
+	g_BridgeGlobalStats.ui32IOCTLCount++;
+	BridgeGlobalStatsUnlock();
+#endif
+
+	if (psBridgePackageKM->ui32InBufferSize != g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32InBufferSize ||
+	    psBridgePackageKM->ui32OutBufferSize != g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32OutBufferSize)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Bridge buffer sizes mismatch! "
+		        "In: User(%u), Kernel(%u) - Out: User(%u), Kernel(%u)",
+		        __func__,
+		        psBridgePackageKM->ui32InBufferSize, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32InBufferSize,
+		        psBridgePackageKM->ui32OutBufferSize, g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui32OutBufferSize));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EINVAL, return_error);
+	}
+
+	if (g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock != NULL)
+	{
+		OSLockAcquire(g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock);
+	}
+#if !defined(INTEGRITY_OS)
+	/* try to acquire a bridge buffer from the pool */
+
+	err = PVRSRVPoolGet(g_psBridgeBufferPool,
+			&hBridgeBufferPoolToken,
+			&psBridgeIn);
+	PVR_LOG_GOTO_IF_ERROR(err, "PVRSRVPoolGet", unlock_and_return_error);
+
+	psBridgeOut = ((IMG_BYTE *) psBridgeIn) + PVRSRV_MAX_BRIDGE_IN_SIZE;
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+	ui64TimeStart = OSClockns64();
+#endif
+
+#if !defined(INTEGRITY_OS)
+	if ((CopyFromUserWrapper (psConnection,
+							  ui32DispatchTableEntryIndex,
+							  psBridgeIn,
+							  psBridgePackageKM->pvParamIn,
+							  psBridgePackageKM->ui32InBufferSize) != PVRSRV_OK)
+#if defined(__QNXNTO__)
+/* For Neutrino, the output bridge buffer acts as an input as well */
+					|| (CopyFromUserWrapper(psConnection,
+											ui32DispatchTableEntryIndex,
+											psBridgeOut,
+											(void *)((uintptr_t)psBridgePackageKM->pvParamIn + psBridgePackageKM->ui32InBufferSize),
+											psBridgePackageKM->ui32OutBufferSize) != PVRSRV_OK)
+#endif
+		) /* end of if-condition */
+	{
+		PVR_LOG_GOTO_WITH_ERROR("CopyFromUserWrapper", err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+	}
+#else
+	psBridgeIn = psBridgePackageKM->pvParamIn;
+	psBridgeOut = psBridgePackageKM->pvParamOut;
+#endif
+
+	pfBridgeHandler =
+		(BridgeWrapperFunction)g_BridgeDispatchTable[ui32DispatchTableEntryIndex].pfFunction;
+
+	if (pfBridgeHandler == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui32DispatchTableEntry = %d is not a registered function!",
+				 __func__, ui32DispatchTableEntry));
+		PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+	}
+
+	/* pfBridgeHandler functions do not fail and return an IMG_INT.
+	 * The value returned is either 0 or PVRSRV_OK (0).
+	 * In the event this changes an error may be +ve or -ve,
+	 * so try to return something consistent here.
+	 */
+	if (0 != pfBridgeHandler(ui32DispatchTableEntryIndex,
+						  psBridgeIn,
+						  psBridgeOut,
+						  psConnection)
+		)
+	{
+		PVR_LOG_GOTO_WITH_ERROR("pfBridgeHandler", err, PVRSRV_ERROR_BRIDGE_EPERM, unlock_and_return_error);
+	}
+
+	/*
+	   This should always be true as a.t.m. all bridge calls have to
+	   return an error message, but this could change so we do this
+	   check to be safe.
+	*/
+#if !defined(INTEGRITY_OS)
+	if (psBridgePackageKM->ui32OutBufferSize > 0)
+	{
+		if (CopyToUserWrapper (psConnection,
+						ui32DispatchTableEntryIndex,
+						psBridgePackageKM->pvParamOut,
+						psBridgeOut,
+						psBridgePackageKM->ui32OutBufferSize) != PVRSRV_OK)
+		{
+			PVR_GOTO_WITH_ERROR(err, PVRSRV_ERROR_BRIDGE_EFAULT, unlock_and_return_error);
+		}
+	}
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+	ui64TimeEnd = OSClockns64();
+
+	ui64TimeDiff = ui64TimeEnd - ui64TimeStart;
+
+	BridgeGlobalStatsLock();
+
+	g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64TotalTimeNS += ui64TimeDiff;
+
+	if (ui64TimeDiff > g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64MaxTimeNS)
+	{
+		g_BridgeDispatchTable[ui32DispatchTableEntryIndex].ui64MaxTimeNS = ui64TimeDiff;
+	}
+
+	BridgeGlobalStatsUnlock();
+#endif
+
+unlock_and_return_error:
+
+	if (g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock != NULL)
+	{
+		OSLockRelease(g_BridgeDispatchTable[ui32DispatchTableEntryIndex].hBridgeLock);
+	}
+
+#if !defined(INTEGRITY_OS)
+	if (hBridgeBufferPoolToken != NULL)
+	{
+		err = PVRSRVPoolPut(g_psBridgeBufferPool,
+				hBridgeBufferPoolToken);
+		PVR_LOG_IF_ERROR(err, "PVRSRVPoolPut");
+	}
+#endif
+
+return_error:
+	if (err)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: returning (err = %d)", __func__, err));
+	}
+	/* ignore transport layer bridge to avoid HTB flooding */
+	if (psBridgePackageKM->ui32BridgeID != PVRSRV_BRIDGE_PVRTL)
+	{
+		if (err)
+		{
+			HTBLOGK(HTB_SF_BRG_BRIDGE_CALL_ERR, ui32Timestamp,
+			        psBridgePackageKM->ui32BridgeID,
+			        psBridgePackageKM->ui32FunctionID, err);
+		}
+		else
+		{
+			HTBLOGK(HTB_SF_BRG_BRIDGE_CALL, ui32Timestamp,
+			        psBridgePackageKM->ui32BridgeID,
+			        psBridgePackageKM->ui32FunctionID);
+		}
+	}
+
+	return err;
+}
+
+PVRSRV_ERROR PVRSRVFindProcessMemStatsKM(IMG_PID pid, IMG_UINT32 ui32ArrSize, IMG_BOOL bAllProcessStats, IMG_UINT64 *pui64MemStatArray)
+{
+#if !defined(__QNXNTO__)
+	return PVRSRVFindProcessMemStats(pid,
+					ui32ArrSize,
+					bAllProcessStats,
+					pui64MemStatArray);
+#else
+	PVR_DPF((PVR_DBG_ERROR, "This functionality is not yet implemented for this platform"));
+
+	return PVRSRV_ERROR_NOT_SUPPORTED;
+#endif
+
+}
+
+void PVRSRVBlockIfFrozen(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/* Short-circuit if we're not marked as frozen */
+	if (OSAtomicRead(&psDevNode->eFrozen) == 0)
+	{
+		return;
+	}
+
+	while ((psDevNode->eDevState == PVRSRV_DEVICE_STATE_FROZEN) &&
+	       ((eError == PVRSRV_OK) || (eError == PVRSRV_ERROR_TIMEOUT)))
+	{
+		IMG_HANDLE hEvent;
+
+		eError = OSEventObjectOpen(psDevNode->hDeviceThreadEvObj, &hEvent);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+			        "%s: Failed to open event object (%d)", __func__,
+			        eError));
+
+			/* Continue with the loop by resetting to PVRSRV_OK. This makes it
+			 * a busy wait (with a 10ms delay) if we cannot grab an EvObj
+			 * reference.
+			 */
+			eError = PVRSRV_OK;
+			OSSleepms(10U);		/* Allow failure (OOM etc.) to resolve */
+			continue;
+		}
+
+		if (hEvent != NULL)
+		{
+			/* Register that we've got an interest in the device */
+			(void) OSAtomicIncrement(&psDevNode->iFreezeCount);
+
+			eError = OSEventObjectWait(hEvent);
+
+			OSEventObjectClose(hEvent);
+
+			(void) OSAtomicDecrement(&psDevNode->iFreezeCount);
+		}
+	}
+
+	/* We expect to exit the above loop only when the device is no longer
+	 * FROZEN. If we are still marked as frozen that is an unexpected error
+	 * so log it.
+	 */
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_FROZEN)
+	{
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectWait");
+		if (eError != PVRSRV_OK)
+		{
+			OSDumpStack();
+		}
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/srvcore.h b/drivers/gpu/drm/img-rogue/srvcore.h
new file mode 100644
index 000000000000..8d1385f618d4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/srvcore.h
@@ -0,0 +1,274 @@
+/**************************************************************************/ /*!
+@File
+@Title          PVR Bridge Functionality
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header for the PVR Bridge code
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVCORE_H
+#define SRVCORE_H
+
+#include "lock_types.h"
+#include "connection_server.h"
+#include "pvr_debug.h"
+
+#include "pvr_bridge.h"
+#if defined(SUPPORT_RGX)
+#include "rgx_bridge.h"
+#endif
+
+#if defined(DEBUG_BRIDGE_KM)
+PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+                    IMG_UINT32 ui32DispatchTableEntry,
+                    void *pvDest,
+                    void __user *pvSrc,
+                    IMG_UINT32 ui32Size);
+PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+                  IMG_UINT32 ui32DispatchTableEntry,
+                  void __user *pvDest,
+                  void *pvSrc,
+                  IMG_UINT32 ui32Size);
+#else
+FORCE_INLINE PVRSRV_ERROR
+CopyFromUserWrapper(CONNECTION_DATA *psConnection,
+					IMG_UINT32 ui32DispatchTableEntry,
+					void *pvDest,
+					void __user *pvSrc,
+					IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER (ui32DispatchTableEntry);
+	return OSBridgeCopyFromUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+FORCE_INLINE PVRSRV_ERROR
+CopyToUserWrapper(CONNECTION_DATA *psConnection,
+				  IMG_UINT32 ui32DispatchTableEntry,
+				  void __user *pvDest,
+				  void *pvSrc,
+				  IMG_UINT32 ui32Size)
+{
+	PVR_UNREFERENCED_PARAMETER (ui32DispatchTableEntry);
+	return OSBridgeCopyToUser(psConnection, pvDest, pvSrc, ui32Size);
+}
+#endif
+
+IMG_INT
+DummyBW(IMG_UINT32 ui32DispatchTableEntry,
+        IMG_UINT8 *psBridgeIn,
+        IMG_UINT8 *psBridgeOut,
+        CONNECTION_DATA *psConnection);
+
+typedef PVRSRV_ERROR (*ServerResourceDestroyFunction)(IMG_HANDLE, IMG_HANDLE);
+
+typedef IMG_INT (*BridgeWrapperFunction)(IMG_UINT32 ui32DispatchTableEntry,
+									 IMG_UINT8 *psBridgeIn,
+									 IMG_UINT8 *psBridgeOut,
+									 CONNECTION_DATA *psConnection);
+
+typedef struct _PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY
+{
+	BridgeWrapperFunction pfFunction; /*!< The wrapper function that validates the ioctl
+	                                    arguments before calling into srvkm proper */
+	POS_LOCK hBridgeLock; /*!< The bridge lock which needs to be acquired
+                                   before calling the above wrapper */
+
+	IMG_UINT32 ui32InBufferSize; /*!< The expected size of the in buffer given by the client */
+	IMG_UINT32 ui32OutBufferSize; /*!< The expected size of the out buffer given by the client */
+
+#if defined(DEBUG_BRIDGE_KM)
+	const IMG_CHAR *pszIOCName; /*!< Name of the ioctl: e.g. "PVRSRV_BRIDGE_CONNECT_SERVICES" */
+	const IMG_CHAR *pszFunctionName; /*!< Name of the wrapper function: e.g. "PVRSRVConnectBW" */
+	const IMG_CHAR *pszBridgeLockName; /*!< Name of bridge lock which will be acquired */
+	IMG_UINT32 ui32CallCount; /*!< The total number of times the ioctl has been called */
+	IMG_UINT32 ui32CopyFromUserTotalBytes; /*!< The total number of bytes copied from
+	                                         userspace within this ioctl */
+	IMG_UINT32 ui32CopyToUserTotalBytes; /*!< The total number of bytes copied from
+	                                       userspace within this ioctl */
+	IMG_UINT64 ui64TotalTimeNS; /*!< The total amount of time spent in this bridge function */
+	IMG_UINT64 ui64MaxTimeNS; /*!< The maximum amount of time for a single call to this bridge function */
+#endif
+}PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY;
+
+#if defined(SUPPORT_RGX)
+	#define BRIDGE_DISPATCH_TABLE_ENTRY_COUNT  (PVRSRV_BRIDGE_RGX_DISPATCH_LAST+1)
+	#define BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT  (PVRSRV_BRIDGE_RGX_LAST+1)
+#else
+	#define BRIDGE_DISPATCH_TABLE_ENTRY_COUNT  (PVRSRV_BRIDGE_DISPATCH_LAST+1)
+	#define BRIDGE_DISPATCH_TABLE_START_ENTRY_COUNT  (PVRSRV_BRIDGE_LAST+1)
+#endif
+
+extern PVRSRV_BRIDGE_DISPATCH_TABLE_ENTRY g_BridgeDispatchTable[BRIDGE_DISPATCH_TABLE_ENTRY_COUNT];
+
+void BridgeDispatchTableStartOffsetsInit(void);
+
+void
+_SetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+                       IMG_UINT32 ui32Index,
+                       const IMG_CHAR *pszIOCName,
+                       BridgeWrapperFunction pfFunction,
+                       const IMG_CHAR *pszFunctionName,
+                       POS_LOCK hBridgeLock,
+                       const IMG_CHAR* pszBridgeLockName,
+                       IMG_UINT32 ui32InBufferSize,
+                       IMG_UINT32 ui32OutBufferSize);
+void
+UnsetDispatchTableEntry(IMG_UINT32 ui32BridgeGroup,
+                        IMG_UINT32 ui32Index);
+
+
+/* PRQA S 0884,3410 2*/ /* macro relies on the lack of brackets */
+#define SetDispatchTableEntry(ui32BridgeGroup, ui32Index, pfFunction,\
+                              hBridgeLock, ui32InBufferSize, ui32OutBufferSize) \
+	do \
+	{ \
+		static_assert((ui32InBufferSize) <= PVRSRV_MAX_BRIDGE_IN_SIZE, "Bridge input buffer is too small for bridge function: " #pfFunction); \
+		static_assert((ui32OutBufferSize) <= PVRSRV_MAX_BRIDGE_OUT_SIZE, "Bridge output buffer is too small for bridge function: " #pfFunction); \
+		_SetDispatchTableEntry(ui32BridgeGroup, ui32Index, #ui32Index, (BridgeWrapperFunction)pfFunction, #pfFunction,\
+		                       (POS_LOCK)hBridgeLock, #hBridgeLock, ui32InBufferSize, ui32OutBufferSize); \
+	} while (0)
+
+#define DISPATCH_TABLE_GAP_THRESHOLD 5
+
+
+#if defined(DEBUG_BRIDGE_KM)
+typedef struct _PVRSRV_BRIDGE_GLOBAL_STATS
+{
+	IMG_UINT32 ui32IOCTLCount;
+	IMG_UINT32 ui32TotalCopyFromUserBytes;
+	IMG_UINT32 ui32TotalCopyToUserBytes;
+} PVRSRV_BRIDGE_GLOBAL_STATS;
+
+void BridgeGlobalStatsLock(void);
+void BridgeGlobalStatsUnlock(void);
+
+/* OS specific code may want to report the stats held here and within the
+ * BRIDGE_DISPATCH_TABLE_ENTRYs (E.g. on Linux we report these via a
+ * debugfs entry /(sys/kernel/debug|proc)/pvr/bridge_stats) */
+extern PVRSRV_BRIDGE_GLOBAL_STATS g_BridgeGlobalStats;
+#endif
+
+PVRSRV_ERROR BridgeDispatcherInit(void);
+void BridgeDispatcherDeinit(void);
+
+PVRSRV_ERROR
+BridgedDispatchKM(CONNECTION_DATA * psConnection,
+                  PVRSRV_BRIDGE_PACKAGE   * psBridgePackageKM);
+
+PVRSRV_ERROR
+PVRSRVConnectKM(CONNECTION_DATA *psConnection,
+                PVRSRV_DEVICE_NODE * psDeviceNode,
+                IMG_UINT32 ui32Flags,
+                IMG_UINT32 ui32ClientBuildOptions,
+                IMG_UINT32 ui32ClientDDKVersion,
+                IMG_UINT32 ui32ClientDDKBuild,
+                IMG_UINT8  *pui8KernelArch,
+                IMG_UINT32 *ui32CapabilityFlags,
+                IMG_UINT64 *ui64PackedBvnc);
+
+PVRSRV_ERROR
+PVRSRVDisconnectKM(void);
+
+PVRSRV_ERROR
+PVRSRVAcquireGlobalEventObjectKM(IMG_HANDLE *phGlobalEventObject);
+
+PVRSRV_ERROR
+PVRSRVReleaseGlobalEventObjectKM(IMG_HANDLE hGlobalEventObject);
+
+PVRSRV_ERROR
+PVRSRVDumpDebugInfoKM(CONNECTION_DATA *psConnection,
+                      PVRSRV_DEVICE_NODE *psDeviceNode,
+                      IMG_UINT32 ui32VerbLevel);
+
+PVRSRV_ERROR
+PVRSRVGetDevClockSpeedKM(CONNECTION_DATA * psConnection,
+                         PVRSRV_DEVICE_NODE *psDeviceNode,
+                         IMG_PUINT32  pui32RGXClockSpeed);
+
+PVRSRV_ERROR
+PVRSRVHWOpTimeoutKM(CONNECTION_DATA *psConnection,
+                    PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR PVRSRVAlignmentCheckKM(CONNECTION_DATA *psConnection,
+                                    PVRSRV_DEVICE_NODE * psDeviceNode,
+                                    IMG_UINT32 ui32FWAlignChecksSize,
+                                    IMG_UINT32 aui32FWAlignChecks[]);
+
+PVRSRV_ERROR PVRSRVGetDeviceStatusKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 *pui32DeviceStatus);
+
+PVRSRV_ERROR PVRSRVGetMultiCoreInfoKM(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDeviceNode,
+                                     IMG_UINT32 ui32CapsSize,
+                                     IMG_UINT32 *pui32NumCores,
+                                     IMG_UINT64 *pui64Caps);
+
+PVRSRV_ERROR PVRSRVFindProcessMemStatsKM(IMG_PID pid,
+                                         IMG_UINT32 ui32ArrSize,
+                                         IMG_BOOL bAllProcessStats,
+                                         IMG_UINT64 *pui64MemoryStats);
+
+static INLINE
+PVRSRV_ERROR DestroyServerResource(const SHARED_DEV_CONNECTION hConnection,
+                                   IMG_HANDLE hEvent,
+                                   ServerResourceDestroyFunction pfnDestroyCall,
+                                   IMG_HANDLE hResource)
+{
+    PVR_UNREFERENCED_PARAMETER(hEvent);
+
+    return pfnDestroyCall(GetBridgeHandle(hConnection), hResource);
+}
+
+/*************************************************************************/ /*!
+@Function      PVRSRVBlockIfFrozen
+@Description   Puts caller into a blocking wait (using event-objects) if the
+                specified device is in a FROZEN state. Routine completes and
+                returns control to the caller once the underlying device state
+                clears its FROZEN state.
+@Input         psDeviceNode     Device Node reference
+@Return        Nothing.         Execution blocks if device is frozen.
+*/ /**************************************************************************/
+void PVRSRVBlockIfFrozen(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+#endif /* SRVCORE_H */
+
+/******************************************************************************
+ End of file (srvcore.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/srvinit.h b/drivers/gpu/drm/img-rogue/srvinit.h
new file mode 100644
index 000000000000..48e6863eae1f
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/srvinit.h
@@ -0,0 +1,68 @@
+/*************************************************************************/ /*!
+@File
+@Title          Initialisation server internal header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the connections between the various parts of the
+                initialisation server.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SRVINIT_H
+#define SRVINIT_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "device_connection.h"
+#include "device.h"
+
+#if defined(SUPPORT_RGX)
+PVRSRV_ERROR RGXInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* SRVINIT_H */
+
+/******************************************************************************
+ End of file (srvinit.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/srvkm.h b/drivers/gpu/drm/img-rogue/srvkm.h
new file mode 100644
index 000000000000..29418764daa5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/srvkm.h
@@ -0,0 +1,144 @@
+/**************************************************************************/ /*!
+@File
+@Title          Services kernel module internal header file
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SRVKM_H
+#define SRVKM_H
+
+#include "servicesext.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+struct _PVRSRV_DEVICE_NODE_;
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDriverInit
+@Description  Performs one time driver initialisation of Services Common and
+              Device layers.
+@Return       PVRSRV_ERROR   PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVCommonDriverInit(void);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDriverInit
+@Description  Performs one time driver de-initialisation of Services.
+@Return       void
+*/ /**************************************************************************/
+void PVRSRVCommonDriverDeInit(void);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceCreate
+@Description  Creates and initialises a common layer Services device node
+              for an OS native device. First stage device discovery.
+@Input        pvOSDevice      OS native device
+@Input        i32KernelDeviceID A unique identifier which helps recognise this
+                              Device in the UM space provided by the OS.
+@Output       ppsDeviceNode   Points to the new device node on success
+@Return       PVRSRV_ERROR    PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR
+PVRSRVCommonDeviceCreate(void *pvOSDevice, IMG_INT32 i32KernelDeviceID,
+				   struct _PVRSRV_DEVICE_NODE_ **ppsDeviceNode);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceInitialise
+@Description  Initialises the device layer specifics (e.g. boot FW etc)
+              for the supplied device node, created previously by
+              PVRSRVCommonDeviceCreate. The device is ready for use when this
+              second stage device initialisation returns successfully.
+@Input        psDeviceNode  Device node of the device to be initialised
+@Return       PVRSRV_ERROR  PVRSRV_OK on success and an error otherwise
+*/ /**************************************************************************/
+PVRSRV_ERROR PVRSRVCommonDeviceInitialise(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+/*************************************************************************/ /*!
+@Function     PVRSRVCommonDeviceDestroy
+@Description  Destroys a PVR Services device node.
+@Input        psDeviceNode  Device node to destroy
+*/ /**************************************************************************/
+void
+PVRSRVCommonDeviceDestroy(struct _PVRSRV_DEVICE_NODE_ *psDeviceNode);
+
+/******************
+HIGHER LEVEL MACROS
+*******************/
+
+/*----------------------------------------------------------------------------
+Repeats the body of the loop for a certain minimum time, or until the body
+exits by its own means (break, return, goto, etc.)
+
+Example of usage:
+
+LOOP_UNTIL_TIMEOUT_US(MAX_HW_TIME_US)
+{
+	if (psQueueInfo->ui32ReadOffset == psQueueInfo->ui32WriteOffset)
+	{
+		bTimeout = IMG_FALSE;
+		break;
+	}
+
+	OSWaitus(MAX_HW_TIME_US/WAIT_TRY_COUNT);
+} END_LOOP_UNTIL_TIMEOUT_US();
+
+-----------------------------------------------------------------------------*/
+
+/* uiNotLastLoop will remain at 1 until the timeout has expired, at which time
+ * it will be decremented and the loop executed one final time. This is
+ * necessary when preemption is enabled.
+ */
+/* PRQA S 3411,3431 12 */ /* critical format, leave alone */
+#define LOOP_UNTIL_TIMEOUT_US(TIMEOUT_US) \
+{\
+	IMG_UINT32 uiOffset, uiStart, uiCurrent; \
+	IMG_INT32 iNotLastLoop;					 \
+	for (uiOffset = 0, uiStart = OSClockus(), uiCurrent = uiStart + 1, iNotLastLoop = 1;\
+		((uiCurrent - uiStart + uiOffset) < (TIMEOUT_US)) || iNotLastLoop--;				\
+		uiCurrent = OSClockus(),													\
+		uiOffset = uiCurrent < uiStart ? IMG_UINT32_MAX - uiStart : uiOffset,		\
+		uiStart = uiCurrent < uiStart ? 0 : uiStart)
+
+#define END_LOOP_UNTIL_TIMEOUT_US() \
+}
+
+#endif /* SRVKM_H */
diff --git a/drivers/gpu/drm/img-rogue/sync.c b/drivers/gpu/drm/img-rogue/sync.c
new file mode 100644
index 000000000000..3b6e8914d446
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync.c
@@ -0,0 +1,822 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements client side code for services synchronisation
+                interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "client_sync_bridge.h"
+#include "client_synctracking_bridge.h"
+#include "info_page_client.h"
+#include "pvr_bridge.h"
+#include "allocmem.h"
+#include "osfunc.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pvr_debug.h"
+#include "dllist.h"
+#include "sync.h"
+#include "sync_internal.h"
+#include "lock.h"
+#include "log2.h"
+#if defined(__KERNEL__)
+#include "pvrsrv.h"
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+
+#define SYNC_BLOCK_LIST_CHUNCK_SIZE	10
+
+/*
+	This defines the maximum amount of synchronisation memory
+	that can be allocated per SyncPrim context.
+	In reality this number is meaningless as we would run out
+	of synchronisation memory before we reach this limit, but
+	we need to provide a size to the span RA.
+ */
+#define MAX_SYNC_MEM				(4 * 1024 * 1024)
+
+/* forward declaration */
+static PVRSRV_ERROR
+_SyncPrimSetValue(SYNC_PRIM *psSyncInt, IMG_UINT32 ui32Value);
+
+/*
+	Internal interfaces for management of SYNC_PRIM_CONTEXT
+ */
+static void
+_SyncPrimContextUnref(SYNC_PRIM_CONTEXT *psContext)
+{
+	if (!OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: context already freed", __func__));
+	}
+	else if (0 == OSAtomicDecrement(&psContext->hRefCount))
+	{
+		/* SyncPrimContextDestroy only when no longer referenced */
+		RA_Delete(psContext->psSpanRA);
+		RA_Delete(psContext->psSubAllocRA);
+		OSFreeMem(psContext);
+	}
+}
+
+static void
+_SyncPrimContextRef(SYNC_PRIM_CONTEXT *psContext)
+{
+	if (!OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: context use after free", __func__));
+	}
+	else
+	{
+		OSAtomicIncrement(&psContext->hRefCount);
+	}
+}
+
+/*
+	Internal interfaces for management of synchronisation block memory
+ */
+static PVRSRV_ERROR
+AllocSyncPrimitiveBlock(SYNC_PRIM_CONTEXT *psContext,
+                        SYNC_PRIM_BLOCK **ppsSyncBlock)
+{
+	SYNC_PRIM_BLOCK *psSyncBlk;
+	IMG_HANDLE hSyncPMR;
+	IMG_HANDLE hSyncImportHandle;
+	IMG_DEVMEM_SIZE_T uiImportSize;
+	PVRSRV_ERROR eError;
+
+	psSyncBlk = OSAllocMem(sizeof(SYNC_PRIM_BLOCK));
+	PVR_GOTO_IF_NOMEM(psSyncBlk, eError, fail_alloc);
+
+	psSyncBlk->psContext = psContext;
+
+	/* Allocate sync prim block */
+	eError = BridgeAllocSyncPrimitiveBlock(GetBridgeHandle(psContext->hDevConnection),
+	                                       &psSyncBlk->hServerSyncPrimBlock,
+	                                       &psSyncBlk->ui32FirmwareAddr,
+	                                       &psSyncBlk->ui32SyncBlockSize,
+	                                       &hSyncPMR);
+	PVR_GOTO_IF_ERROR(eError, fail_blockalloc);
+
+	/* Make it mappable by the client */
+	eError = DevmemMakeLocalImportHandle(psContext->hDevConnection,
+	                                     hSyncPMR,
+	                                     &hSyncImportHandle);
+	PVR_GOTO_IF_ERROR(eError, fail_export);
+
+	/* Get CPU mapping of the memory block */
+	eError = DevmemLocalImport(psContext->hDevConnection,
+	                           hSyncImportHandle,
+	                           PVRSRV_MEMALLOCFLAG_CPU_READABLE,
+	                           &psSyncBlk->hMemDesc,
+	                           &uiImportSize,
+	                           "SyncPrimitiveBlock");
+
+	/*
+		Regardless of success or failure we "undo" the export
+	 */
+	DevmemUnmakeLocalImportHandle(psContext->hDevConnection,
+	                              hSyncImportHandle);
+
+	PVR_GOTO_IF_ERROR(eError, fail_import);
+
+	eError = DevmemAcquireCpuVirtAddr(psSyncBlk->hMemDesc,
+	                                  (void **) &psSyncBlk->pui32LinAddr);
+	PVR_GOTO_IF_ERROR(eError, fail_cpuvaddr);
+
+	*ppsSyncBlock = psSyncBlk;
+	return PVRSRV_OK;
+
+fail_cpuvaddr:
+	DevmemFree(psSyncBlk->hMemDesc);
+fail_import:
+fail_export:
+	BridgeFreeSyncPrimitiveBlock(GetBridgeHandle(psContext->hDevConnection),
+	                             psSyncBlk->hServerSyncPrimBlock);
+fail_blockalloc:
+	OSFreeMem(psSyncBlk);
+fail_alloc:
+	return eError;
+}
+
+static void
+FreeSyncPrimitiveBlock(SYNC_PRIM_BLOCK *psSyncBlk)
+{
+	SYNC_PRIM_CONTEXT *psContext = psSyncBlk->psContext;
+
+	DevmemReleaseCpuVirtAddr(psSyncBlk->hMemDesc);
+	DevmemFree(psSyncBlk->hMemDesc);
+	(void) DestroyServerResource(psContext->hDevConnection,
+	                             NULL,
+	                             BridgeFreeSyncPrimitiveBlock,
+	                             psSyncBlk->hServerSyncPrimBlock);
+	OSFreeMem(psSyncBlk);
+}
+
+static PVRSRV_ERROR
+SyncPrimBlockImport(RA_PERARENA_HANDLE hArena,
+                    RA_LENGTH_T uSize,
+                    RA_FLAGS_T uFlags,
+                    RA_LENGTH_T uBaseAlignment,
+                    const IMG_CHAR *pszAnnotation,
+                    RA_IMPORT *psImport)
+{
+	SYNC_PRIM_CONTEXT *psContext = hArena;
+	SYNC_PRIM_BLOCK *psSyncBlock = NULL;
+	RA_LENGTH_T uiSpanSize;
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(uFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	/* Check we've not been called with an unexpected size */
+	PVR_LOG_GOTO_IF_INVALID_PARAM(hArena, eError, e0);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(uSize == sizeof(IMG_UINT32), eError, e0);
+
+	/*
+		Ensure the synprim context doesn't go away while we have sync blocks
+		attached to it
+	 */
+	_SyncPrimContextRef(psContext);
+
+	/* Allocate the block of memory */
+	eError = AllocSyncPrimitiveBlock(psContext, &psSyncBlock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "AllocSyncPrimitiveBlock", fail_syncblockalloc);
+
+	/* Allocate a span for it */
+	eError = RA_Alloc(psContext->psSpanRA,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  pszAnnotation,
+	                  &psSyncBlock->uiSpanBase,
+	                  &uiSpanSize,
+	                  NULL);
+	PVR_GOTO_IF_ERROR(eError, fail_spanalloc);
+
+	/*
+		There is no reason the span RA should return an allocation larger
+		then we request
+	 */
+	PVR_ASSERT(uiSpanSize == psSyncBlock->ui32SyncBlockSize);
+
+	psImport->base = psSyncBlock->uiSpanBase;
+	psImport->uSize = psSyncBlock->ui32SyncBlockSize;
+	psImport->hPriv = psSyncBlock;
+	return PVRSRV_OK;
+
+fail_spanalloc:
+	FreeSyncPrimitiveBlock(psSyncBlock);
+fail_syncblockalloc:
+	_SyncPrimContextUnref(psContext);
+e0:
+	return eError;
+}
+
+static void
+SyncPrimBlockUnimport(RA_PERARENA_HANDLE hArena,
+                      RA_BASE_T uiBase,
+                      RA_PERISPAN_HANDLE hImport)
+{
+	SYNC_PRIM_CONTEXT *psContext = hArena;
+	SYNC_PRIM_BLOCK *psSyncBlock = hImport;
+
+	if (!psContext || !psSyncBlock || uiBase != psSyncBlock->uiSpanBase)
+	{
+		/* Invalid input params */
+		return;
+	}
+
+	/* Free the span this import is using */
+	RA_Free(psContext->psSpanRA, uiBase);
+
+	/* Free the syncpim block */
+	FreeSyncPrimitiveBlock(psSyncBlock);
+
+	/*	Drop our reference to the syncprim context */
+	_SyncPrimContextUnref(psContext);
+}
+
+static INLINE IMG_UINT32 SyncPrimGetOffset(SYNC_PRIM *psSyncInt)
+{
+	IMG_UINT64 ui64Temp = psSyncInt->uiSpanAddr - psSyncInt->psSyncBlock->uiSpanBase;
+
+	PVR_ASSERT(ui64Temp<IMG_UINT32_MAX);
+
+	return TRUNCATE_64BITS_TO_32BITS(ui64Temp);
+}
+
+static void SyncPrimGetCPULinAddr(SYNC_PRIM *psSyncInt)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock = psSyncInt->psSyncBlock;
+
+	psSyncInt->sCommon.pui32LinAddr = psSyncBlock->pui32LinAddr +
+			(SyncPrimGetOffset(psSyncInt)/sizeof(IMG_UINT32));
+}
+
+static void SyncPrimLocalFree(SYNC_PRIM *psSyncInt, IMG_BOOL bFreeFirstSyncPrim)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+#if !defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	PVR_UNREFERENCED_PARAMETER(bFreeFirstSyncPrim);
+#else
+	/* Defer freeing the first allocated sync prim in the sync context */
+	if (psSyncInt != psContext->hFirstSyncPrim || bFreeFirstSyncPrim)
+#endif
+	{
+		PVRSRV_ERROR eError;
+		SHARED_DEV_CONNECTION hDevConnection =
+			psSyncInt->psSyncBlock->psContext->hDevConnection;
+
+		if (GetInfoPageDebugFlags(hDevConnection) & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+		{
+			if (psSyncInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = DestroyServerResource(hDevConnection,
+				                               NULL,
+				                               BridgeSyncRecordRemoveByHandle,
+				                               psSyncInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "BridgeSyncRecordRemoveByHandle");
+			}
+		}
+		else
+		{
+			IMG_UINT32 ui32FWAddr = psSyncBlock->ui32FirmwareAddr +
+					SyncPrimGetOffset(psSyncInt);
+
+			eError = BridgeSyncFreeEvent(GetBridgeHandle(hDevConnection), ui32FWAddr);
+			PVR_LOG_IF_ERROR(eError, "BridgeSyncFreeEvent");
+		}
+#if defined(PVRSRV_ENABLE_SYNC_POISONING)
+		(void) _SyncPrimSetValue(psSyncInt, LOCAL_SYNC_PRIM_POISON_VALUE);
+#else
+		/* reset the sync prim value as it is freed.
+		 * this guarantees the client sync allocated to the client will
+		 * have a value of zero and the client does not need to
+		 * explicitly initialise the sync value to zero.
+		 * the allocation of the backing memory for the sync prim block
+		 * is done with ZERO_ON_ALLOC so the memory is initially all zero.
+		 */
+		(void) _SyncPrimSetValue(psSyncInt, LOCAL_SYNC_PRIM_RESET_VALUE);
+#endif
+
+		RA_Free(psContext->psSubAllocRA, psSyncInt->uiSpanAddr);
+		OSFreeMem(psSyncInt);
+		_SyncPrimContextUnref(psContext);
+	}
+}
+
+static void SyncPrimLocalUnref(SYNC_PRIM *psSyncInt)
+{
+	if (!OSAtomicRead(&psSyncInt->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncPrimLocalUnref sync already freed"));
+	}
+	else if (0 == OSAtomicDecrement(&psSyncInt->hRefCount))
+	{
+		SyncPrimLocalFree(psSyncInt, IMG_FALSE);
+	}
+}
+
+static IMG_UINT32 SyncPrimGetFirmwareAddrLocal(SYNC_PRIM *psSyncInt)
+{
+	SYNC_PRIM_BLOCK *psSyncBlock;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	return psSyncBlock->ui32FirmwareAddr + SyncPrimGetOffset(psSyncInt);
+}
+
+static INLINE IMG_UINT32 _Log2(IMG_UINT32 ui32Align)
+{
+	PVR_ASSERT(IsPower2(ui32Align));
+	return ExactLog2(ui32Align);
+}
+
+/*
+	External interfaces
+ */
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimContextCreate(SHARED_DEV_CONNECTION hDevConnection,
+                      PSYNC_PRIM_CONTEXT *phSyncPrimContext)
+{
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	psContext = OSAllocMem(sizeof(SYNC_PRIM_CONTEXT));
+	PVR_GOTO_IF_NOMEM(psContext, eError, fail_alloc);
+
+	psContext->hDevConnection = hDevConnection;
+
+	OSSNPrintf(psContext->azName, SYNC_PRIM_NAME_SIZE, "Sync Prim RA-%p", psContext);
+	OSSNPrintf(psContext->azSpanName, SYNC_PRIM_NAME_SIZE, "Sync Prim span RA-%p", psContext);
+
+	/*
+		Create the RA for sub-allocations of the SynPrim's
+
+		Note:
+		The import size doesn't matter here as the server will pass
+		back the blocksize when does the import which overrides
+		what we specify here.
+	 */
+
+	psContext->psSubAllocRA = RA_Create(psContext->azName,
+	                                    /* Params for imports */
+	                                    _Log2(sizeof(IMG_UINT32)),
+	                                    RA_LOCKCLASS_3,
+	                                    SyncPrimBlockImport,
+	                                    SyncPrimBlockUnimport,
+	                                    psContext,
+	                                    RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_NOMEM(psContext->psSubAllocRA, eError, fail_suballoc);
+
+	/*
+		Create the span-management RA
+
+		The RA requires that we work with linear spans. For our use
+		here we don't require this behaviour as we're always working
+		within offsets of blocks (imports). However, we need to keep
+		the RA happy so we create the "span" management RA which
+		ensures that all are imports are added to the RA in a linear
+		fashion
+	 */
+	psContext->psSpanRA = RA_Create(psContext->azSpanName,
+	                                /* Params for imports */
+	                                0,
+	                                RA_LOCKCLASS_0,
+	                                NULL,
+	                                NULL,
+	                                NULL,
+	                                RA_POLICY_DEFAULT);
+	PVR_GOTO_IF_NOMEM(psContext->psSpanRA, eError, fail_span);
+
+	if (!RA_Add(psContext->psSpanRA, 0, MAX_SYNC_MEM, 0, NULL))
+	{
+		RA_Delete(psContext->psSpanRA);
+		PVR_GOTO_WITH_ERROR(eError, PVRSRV_ERROR_OUT_OF_MEMORY, fail_span);
+	}
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	psContext->hFirstSyncPrim = NULL;
+#endif
+
+	OSAtomicWrite(&psContext->hRefCount, 1);
+
+	*phSyncPrimContext = psContext;
+	return PVRSRV_OK;
+fail_span:
+	RA_Delete(psContext->psSubAllocRA);
+fail_suballoc:
+	OSFreeMem(psContext);
+fail_alloc:
+	return eError;
+}
+
+IMG_INTERNAL void SyncPrimContextDestroy(PSYNC_PRIM_CONTEXT hSyncPrimContext)
+{
+	SYNC_PRIM_CONTEXT *psContext = hSyncPrimContext;
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	/* Free the first sync prim that was allocated as part of this context */
+	if (psContext->hFirstSyncPrim)
+	{
+		SyncPrimLocalFree((SYNC_PRIM *)psContext->hFirstSyncPrim, IMG_TRUE);
+		psContext->hFirstSyncPrim = NULL;
+	}
+#endif
+
+	if (1 != OSAtomicRead(&psContext->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s attempted with active references, may be the result of a race", __func__));
+	}
+#if defined(PVRSRV_FORCE_UNLOAD_IF_BAD_STATE)
+#if defined(__KERNEL__)
+	if (PVRSRVGetPVRSRVData()->eServicesState != PVRSRV_SERVICES_STATE_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Forcing context destruction due to bad driver state", __func__));
+		OSAtomicWrite(&psContext->hRefCount, 1);
+	}
+#endif
+#endif
+	_SyncPrimContextUnref(psContext);
+}
+
+static PVRSRV_ERROR _SyncPrimAlloc(PSYNC_PRIM_CONTEXT hSyncPrimContext,
+                                   PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+                                   const IMG_CHAR *pszClassName,
+                                   IMG_BOOL bServerSync)
+{
+	SYNC_PRIM_CONTEXT *psContext = hSyncPrimContext;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM *psNewSync;
+	PVRSRV_ERROR eError;
+	RA_BASE_T uiSpanAddr;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(hSyncPrimContext, "hSyncPrimeContext");
+
+	psNewSync = OSAllocMem(sizeof(SYNC_PRIM));
+	PVR_GOTO_IF_NOMEM(psNewSync, eError, fail_alloc);
+
+	eError = RA_Alloc(psContext->psSubAllocRA,
+	                  sizeof(IMG_UINT32),
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  sizeof(IMG_UINT32),
+	                  "Sync_Prim",
+	                  &uiSpanAddr,
+	                  NULL,
+	                  (RA_PERISPAN_HANDLE *) &psSyncBlock);
+	PVR_GOTO_IF_ERROR(eError, fail_raalloc);
+
+	OSAtomicWrite(&psNewSync->hRefCount, 1);
+	psNewSync->uiSpanAddr = uiSpanAddr;
+	psNewSync->psSyncBlock = psSyncBlock;
+	SyncPrimGetCPULinAddr(psNewSync);
+	*ppsSync = &psNewSync->sCommon;
+	_SyncPrimContextRef(psContext);
+#if defined(PVRSRV_ENABLE_SYNC_POISONING)
+	(void) _SyncPrimSetValue(psNewSync, LOCAL_SYNC_PRIM_RESET_VALUE);
+#endif
+
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	/* If this is the first sync prim allocated in the context, keep a handle to it */
+	if (psSyncBlock->uiSpanBase == 0 && psNewSync->uiSpanAddr == 0)
+	{
+		psContext->hFirstSyncPrim = psNewSync;
+	}
+#endif
+
+	if (GetInfoPageDebugFlags(psSyncBlock->psContext->hDevConnection) & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		IMG_CHAR szClassName[PVRSRV_SYNC_NAME_LENGTH];
+		size_t uiSize;
+
+		if (pszClassName)
+		{
+			uiSize = OSStringNLength(pszClassName, PVRSRV_SYNC_NAME_LENGTH);
+			/* Copy the class name annotation into a fixed-size array */
+			OSCachedMemCopy(szClassName, pszClassName, uiSize);
+			if (uiSize == PVRSRV_SYNC_NAME_LENGTH)
+				szClassName[PVRSRV_SYNC_NAME_LENGTH-1] = '\0';
+			else
+				szClassName[uiSize++] = '\0';
+		}
+		else
+		{
+			/* No class name annotation */
+			uiSize = 0;
+			szClassName[0] = '\0';
+		}
+
+		/* record this sync */
+		eError = BridgeSyncRecordAdd(
+				GetBridgeHandle(psSyncBlock->psContext->hDevConnection),
+				&psNewSync->hRecord,
+				psSyncBlock->hServerSyncPrimBlock,
+				psSyncBlock->ui32FirmwareAddr,
+				SyncPrimGetOffset(psNewSync),
+				bServerSync,
+				uiSize,
+				szClassName);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: failed to add SyncRecord \"%s\" (%s)",
+					__func__,
+					szClassName,
+					PVRSRVGETERRORSTRING(eError)));
+			psNewSync->hRecord = NULL;
+		}
+	}
+	else
+	{
+		size_t	uiSize;
+
+		uiSize = OSStringNLength(pszClassName, PVRSRV_SYNC_NAME_LENGTH);
+
+		if (uiSize < PVRSRV_SYNC_NAME_LENGTH)
+			uiSize++;
+		/* uiSize now reflects size used for pszClassName + NUL byte */
+
+		eError = BridgeSyncAllocEvent(GetBridgeHandle(hSyncPrimContext->hDevConnection),
+		                              bServerSync,
+		                              psSyncBlock->ui32FirmwareAddr + SyncPrimGetOffset(psNewSync),
+		                              uiSize,
+		                              pszClassName);
+		PVR_LOG_IF_ERROR(eError, "BridgeSyncAllocEvent");
+	}
+
+	return PVRSRV_OK;
+
+fail_raalloc:
+	OSFreeMem(psNewSync);
+fail_alloc:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimAlloc(PSYNC_PRIM_CONTEXT hSyncPrimContext,
+                                        PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+                                        const IMG_CHAR *pszClassName)
+{
+	return _SyncPrimAlloc(hSyncPrimContext,
+	                      ppsSync,
+	                      pszClassName,
+	                      IMG_FALSE);
+}
+
+static PVRSRV_ERROR
+_SyncPrimSetValue(SYNC_PRIM *psSyncInt, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError;
+
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimSet(GetBridgeHandle(psContext->hDevConnection),
+								psSyncBlock->hServerSyncPrimBlock,
+								SyncPrimGetOffset(psSyncInt)/sizeof(IMG_UINT32),
+								ui32Value);
+
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimFree(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	SyncPrimLocalUnref(psSyncInt);
+
+
+err_out:
+	return eError;
+}
+
+#if defined(NO_HARDWARE)
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimNoHwUpdate(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	eError =  _SyncPrimSetValue(psSyncInt, ui32Value);
+
+err_out:
+	return eError;
+}
+#endif
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimSet(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	eError = _SyncPrimSetValue(psSyncInt, ui32Value);
+
+#if defined(PDUMP)
+	SyncPrimPDump(psSync);
+#endif
+err_out:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimLocalGetHandleAndOffset(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                                          IMG_HANDLE *phBlock,
+                                                          IMG_UINT32 *pui32Offset)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(phBlock, eError, err_out);
+	PVR_LOG_GOTO_IF_INVALID_PARAM(pui32Offset, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	*phBlock = psSyncInt->psSyncBlock->hServerSyncPrimBlock;
+	*pui32Offset = psSyncInt->uiSpanAddr - psSyncInt->psSyncBlock->uiSpanBase;
+
+err_out:
+	return eError;
+}
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimGetFirmwareAddr(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 *pui32FwAddr)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	SYNC_PRIM *psSyncInt;
+
+	*pui32FwAddr = 0;
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psSync, eError, err_out);
+
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+	*pui32FwAddr = SyncPrimGetFirmwareAddrLocal(psSyncInt);
+
+err_out:
+	return eError;
+}
+
+#if defined(PDUMP)
+IMG_INTERNAL void SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDump(GetBridgeHandle(psContext->hDevConnection),
+	                             psSyncBlock->hServerSyncPrimBlock,
+	                             SyncPrimGetOffset(psSyncInt));
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDump");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDumpValue(GetBridgeHandle(psContext->hDevConnection),
+	                                  psSyncBlock->hServerSyncPrimBlock,
+	                                  SyncPrimGetOffset(psSyncInt),
+	                                  ui32Value);
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpValue");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                   IMG_UINT32 ui32Value,
+                                   IMG_UINT32 ui32Mask,
+                                   PDUMP_POLL_OPERATOR eOperator,
+                                   IMG_UINT32 ui32PDumpFlags)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+	eError = BridgeSyncPrimPDumpPol(GetBridgeHandle(psContext->hDevConnection),
+	                                psSyncBlock->hServerSyncPrimBlock,
+	                                SyncPrimGetOffset(psSyncInt),
+	                                ui32Value,
+	                                ui32Mask,
+	                                eOperator,
+	                                ui32PDumpFlags);
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpPol");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+IMG_INTERNAL void SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+                                   IMG_UINT64 uiWriteOffset,
+                                   IMG_UINT64 uiPacketSize,
+                                   IMG_UINT64 uiBufferSize)
+{
+	SYNC_PRIM *psSyncInt;
+	SYNC_PRIM_BLOCK *psSyncBlock;
+	SYNC_PRIM_CONTEXT *psContext;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psSync != NULL);
+	psSyncInt = IMG_CONTAINER_OF(psSync, SYNC_PRIM, sCommon);
+
+	psSyncBlock = psSyncInt->psSyncBlock;
+	psContext = psSyncBlock->psContext;
+
+#if defined(__linux__) && defined(__i386__)
+	PVR_ASSERT(uiWriteOffset<IMG_UINT32_MAX);
+	PVR_ASSERT(uiPacketSize<IMG_UINT32_MAX);
+	PVR_ASSERT(uiBufferSize<IMG_UINT32_MAX);
+#endif
+	eError = BridgeSyncPrimPDumpCBP(GetBridgeHandle(psContext->hDevConnection),
+	                                psSyncBlock->hServerSyncPrimBlock,
+	                                SyncPrimGetOffset(psSyncInt),
+	                                TRUNCATE_64BITS_TO_32BITS(uiWriteOffset),
+	                                TRUNCATE_64BITS_TO_32BITS(uiPacketSize),
+	                                TRUNCATE_64BITS_TO_32BITS(uiBufferSize));
+	PVR_LOG_IF_ERROR(eError, "BridgeSyncPrimPDumpCBP");
+	PVR_ASSERT(eError == PVRSRV_OK);
+}
+
+#endif
diff --git a/drivers/gpu/drm/img-rogue/sync.h b/drivers/gpu/drm/img-rogue/sync.h
new file mode 100644
index 000000000000..f126915060f4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync.h
@@ -0,0 +1,292 @@
+/*************************************************************************/ /*!
+@File
+@Title          Synchronisation interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_H
+#define SYNC_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "sync_prim_internal.h"
+#include "pdumpdefs.h"
+#include "dllist.h"
+#include "pvr_debug.h"
+
+#include "device_connection.h"
+
+#if defined(__KERNEL__) && defined(__linux__) && !defined(__GENKSYMS__)
+#define __pvrsrv_defined_struct_enum__
+#include <services_kernel_client.h>
+#endif
+
+/*************************************************************************/ /*!
+@Function       SyncPrimContextCreate
+
+@Description    Create a new synchronisation context
+
+@Input          hBridge                 Bridge handle
+
+@Input          hDeviceNode             Device node handle
+
+@Output         hSyncPrimContext        Handle to the created synchronisation
+                                        primitive context
+
+@Return         PVRSRV_OK if the synchronisation primitive context was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimContextCreate(SHARED_DEV_CONNECTION hDevConnection,
+                      PSYNC_PRIM_CONTEXT    *hSyncPrimContext);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimContextDestroy
+
+@Description    Destroy a synchronisation context
+
+@Input          hSyncPrimContext        Handle to the synchronisation
+                                        primitive context to destroy
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimContextDestroy(PSYNC_PRIM_CONTEXT hSyncPrimContext);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimAlloc
+
+@Description    Allocate a new synchronisation primitive on the specified
+                synchronisation context
+
+@Input          hSyncPrimContext        Handle to the synchronisation
+                                        primitive context
+
+@Output         ppsSync                 Created synchronisation primitive
+
+@Input          pszClassName            Sync source annotation
+
+@Return         PVRSRV_OK if the synchronisation primitive was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimAlloc(PSYNC_PRIM_CONTEXT      hSyncPrimContext,
+              PVRSRV_CLIENT_SYNC_PRIM **ppsSync,
+              const IMG_CHAR          *pszClassName);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimFree
+
+@Description    Free a synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to free
+
+@Return         PVRSRV_OK if the synchronisation primitive was
+                successfully freed
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimFree(PVRSRV_CLIENT_SYNC_PRIM *psSync);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimSet
+
+@Description    Set the synchronisation primitive to a value
+
+@Input          psSync                  The synchronisation primitive to set
+
+@Input          ui32Value               Value to set it to
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimSet(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+
+#if defined(NO_HARDWARE)
+
+/*************************************************************************/ /*!
+@Function       SyncPrimNoHwUpdate
+
+@Description    Updates the synchronisation primitive value (in NoHardware drivers)
+
+@Input          psSync                  The synchronisation primitive to update
+
+@Input          ui32Value               Value to update it to
+
+@Return         PVRSRV_OK on success
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncPrimNoHwUpdate(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+#endif
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       SyncPrimPDump
+
+@Description    PDump the current value of the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpValue
+
+@Description    PDump the ui32Value as the value of the synchronisation
+                primitive (regardless of the current value).
+
+@Input          psSync          The synchronisation primitive to PDump
+@Input          ui32Value       Value to give to the sync prim on the pdump
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpPol
+
+@Description    Do a PDump poll of the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Input          ui32Value               Value to poll for
+
+@Input          ui32Mask                PDump mask operator
+
+@Input          ui32PDumpFlags          PDump flags
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT32 ui32Value,
+				 IMG_UINT32 ui32Mask,
+				 PDUMP_POLL_OPERATOR eOperator,
+				 IMG_UINT32 ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncPrimPDumpCBP
+
+@Description    Do a PDump CB poll using the synchronisation primitive
+
+@Input          psSync                  The synchronisation primitive to PDump
+
+@Input          uiWriteOffset           Current write offset of buffer
+
+@Input          uiPacketSize            Size of the packet to write into CB
+
+@Input          uiBufferSize            Size of the CB
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT64 uiWriteOffset,
+				 IMG_UINT64 uiPacketSize,
+				 IMG_UINT64 uiBufferSize);
+
+#else
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpValue)
+#endif
+static INLINE void
+SyncPrimPDumpValue(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDump)
+#endif
+static INLINE void
+SyncPrimPDump(PVRSRV_CLIENT_SYNC_PRIM *psSync)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpPol)
+#endif
+static INLINE void
+SyncPrimPDumpPol(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT32 ui32Value,
+				 IMG_UINT32 ui32Mask,
+				 PDUMP_POLL_OPERATOR eOperator,
+				 IMG_UINT32 ui32PDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(SyncPrimPDumpCBP)
+#endif
+static INLINE void
+SyncPrimPDumpCBP(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+				 IMG_UINT64 uiWriteOffset,
+				 IMG_UINT64 uiPacketSize,
+				 IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psSync);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+}
+#endif /* PDUMP */
+#endif /* SYNC_H */
diff --git a/drivers/gpu/drm/img-rogue/sync_checkpoint.c b/drivers/gpu/drm/img-rogue/sync_checkpoint.c
new file mode 100644
index 000000000000..b04d059662ee
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_checkpoint.c
@@ -0,0 +1,3366 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation checkpoint interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Server side code for services synchronisation interface
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "img_defs.h"
+#include "img_types.h"
+#include "allocmem.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "osfunc.h"
+#include "dllist.h"
+#include "sync.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "sync_checkpoint_internal.h"
+#include "sync_checkpoint_init.h"
+#include "lock.h"
+#include "log2.h"
+#include "pvrsrv.h"
+#include "pdump_km.h"
+#include "info_page.h"
+#include "os_apphint.h"
+#include "rgxfwutils.h"
+
+#include "pvrsrv_sync_km.h"
+#include "rgxhwperf.h"
+
+
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+
+/* Enable this to turn on debug relating to the creation and
+   resolution of contexts */
+#define ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG 0
+
+/* Enable this to turn on debug relating to the creation and
+   resolution of fences */
+#define ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint
+   allocation and freeing */
+#define ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint
+   enqueuing and signalling */
+#define ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG 0
+
+/* Enable this to turn on debug relating to the sync checkpoint pool */
+#define ENABLE_SYNC_CHECKPOINT_POOL_DEBUG 0
+
+/* Enable this to turn on debug relating to sync checkpoint UFO
+   lookup */
+#define ENABLE_SYNC_CHECKPOINT_UFO_DEBUG 0
+
+/* Enable this to turn on sync checkpoint deferred cleanup debug
+ * (for syncs we have been told to free but which have some
+ * outstanding FW operations remaining (enqueued in CCBs)
+ */
+#define ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG 0
+
+#else
+
+#define ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_POOL_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_UFO_DEBUG 0
+#define ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG 0
+
+#endif
+
+/* Maximum number of deferred sync checkpoint signal/error received for atomic context */
+#define SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL 500
+
+/* Set the size of the sync checkpoint pool (not used if 0).
+ * A pool will be maintained for each sync checkpoint context.
+ * SYNC_CHECKPOINT_POOL_LIMIT must be a power of 2 (POT),
+ * as the pool wrap mask is calculated using it.
+ */
+#define SYNC_CHECKPOINT_POOL_LIMIT	1024
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT & (SYNC_CHECKPOINT_POOL_LIMIT - 1)) != 0
+#error "SYNC_CHECKPOINT_POOL_LIMIT must be power of 2."
+#endif
+
+/*
+	This defines the maximum amount of synchronisation memory
+	that can be allocated per sync checkpoint context.
+	In reality this number is meaningless as we would run out
+	of synchronisation memory before we reach this limit, but
+	we need to provide a size to the span RA.
+ */
+#define MAX_SYNC_CHECKPOINT_MEM  (4 * 1024 * 1024)
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+/* Flags used to indicate state of pool */
+#define SYNC_CHECKPOINT_POOL_FULL  (1)
+#define SYNC_CHECKPOINT_POOL_VALID (1 << 7)
+#define CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags & SYNC_CHECKPOINT_POOL_FULL
+#define CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags & SYNC_CHECKPOINT_POOL_VALID
+#define SET_CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags |= SYNC_CHECKPOINT_POOL_FULL
+#define SET_CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags |= SYNC_CHECKPOINT_POOL_VALID
+#define CLEAR_CHECKPOINT_POOL_FULL(ctxctl) \
+        ctxctl->ui8PoolStateFlags &= ~SYNC_CHECKPOINT_POOL_FULL
+#define CLEAR_CHECKPOINT_POOL_VALID(ctxctl) \
+        ctxctl->ui8PoolStateFlags &= ~SYNC_CHECKPOINT_POOL_VALID
+#endif
+struct _SYNC_CHECKPOINT_CONTEXT_CTL_
+{
+	SHARED_DEV_CONNECTION					psDeviceNode;
+	/*
+	 *  Used as head of linked-list of sync checkpoints for which
+	 *  SyncCheckpointFree() has been called, but have outstanding
+	 *  FW operations (enqueued in CCBs)
+	 *  This list will be check whenever a SyncCheckpointFree() is
+	 *  called, and when SyncCheckpointContextDestroy() is called.
+	 */
+	DLLIST_NODE								sDeferredCleanupListHead;
+	/* Lock to protect the deferred cleanup list */
+	POS_SPINLOCK							hDeferredCleanupListLock;
+
+	/* Counters to provide stats for number of checkpoints used at any one time */
+	IMG_UINT32                              ui32CurrentInUseSyncCheckpoints;
+	IMG_UINT32                              ui32MaxInUseSyncCheckpoints;
+	/* Lock to protect the checkpoint stats */
+	POS_SPINLOCK							hSyncCheckpointStatsLock;
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	IMG_UINT32								ui32SyncCheckpointPoolSize;    /*! Allocated size of the pool */
+	IMG_UINT32								ui32SyncCheckpointPoolCount;   /*! Number of checkpoints currently in pool */
+	IMG_UINT32								ui32SyncCheckpointPoolWp;      /*! Pool write pointer */
+	IMG_UINT32								ui32SyncCheckpointPoolRp;      /*! Pool read pointer */
+#if defined(PDUMP)
+	SYNC_CHECKPOINT_BLOCK					*psSyncCheckpointPoolBlock;    /*! Block that all pool checkpoints come from */
+	IMG_BOOL								bAllocateFromCheckpointPool;   /*! PDumps only use the pool once it has allocated all its checkpoints */
+#endif
+	POS_SPINLOCK							hSyncCheckpointPoolLock;       /*! Lock to protect access to pool control data */
+	IMG_UINT8								ui8PoolStateFlags;             /*! Flags to indicate state of pool */
+	/*! Array of SYNC_CHECKPOINTs. Must be last member in structure */
+	SYNC_CHECKPOINT *apsSyncCheckpointPool[IMG_FLEX_ARRAY_MEMBER];   /*! The allocated checkpoint pool */
+#endif
+}; /*_SYNC_CHECKPOINT_CONTEXT_CTL is already typedef-ed in sync_checkpoint_internal.h */
+
+struct SYNC_CHECKPOINT_CONTEXT_TAG
+{
+#if defined(PDUMP)
+	DLLIST_NODE						sSyncCheckpointBlockListHead;          /*!< List head for the sync chkpt blocks in this context*/
+	DLLIST_NODE						sListNode;				               /*!< List node for the sync chkpt context list*/
+	POS_LOCK						hSyncCheckpointBlockListLock;          /*!< sync chkpt blocks list lock*/
+#endif
+	RA_ARENA						*psSpanRA;                             /*!< RA used for span management of SubAllocRA */
+	RA_ARENA						*psSubAllocRA;                         /*!< RA context */
+	_PSYNC_CHECKPOINT_CONTEXT_CTL	psContextCtl;
+	ATOMIC_T						hCheckpointCount;                      /*!< Checkpoint count for this context */
+	ATOMIC_T						hRefCount;                             /*!< Ref count for this context */
+}; /*_SYNC_CHECKPOINT_CONTEXT is already typedef-ed in sync_checkpoint_internal.h */
+
+/* this is the max number of sync checkpoint records we will search or dump
+ * at any time.
+ */
+#define SYNC_CHECKPOINT_RECORD_LIMIT 20000
+
+#define DECREMENT_WITH_WRAP(value, sz) ((value) ? ((value) - 1) : ((sz) - 1))
+
+struct SYNC_CHECKPOINT_RECORD
+{
+	SYNC_CHECKPOINT_BLOCK	*psSyncCheckpointBlock;	/*!< handle to SYNC_CHECKPOINT_BLOCK */
+	IMG_UINT32				ui32SyncOffset;			/*!< offset to sync in block */
+	IMG_UINT32				ui32FwBlockAddr;
+	IMG_PID					uiPID;
+	IMG_UINT32				ui32UID;
+	IMG_UINT64				ui64OSTime;
+	DLLIST_NODE				sNode;
+	IMG_CHAR				szClassName[PVRSRV_SYNC_NAME_LENGTH];
+	PSYNC_CHECKPOINT		pSyncCheckpt;
+};
+
+static PFN_SYNC_CHECKPOINT_STRUCT *g_psSyncCheckpointPfnStruct = NULL;
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static SYNC_CHECKPOINT *_GetCheckpointFromPool(_SYNC_CHECKPOINT_CONTEXT *psContext);
+static IMG_BOOL _PutCheckpointInPool(SYNC_CHECKPOINT *psSyncCheckpoint);
+static IMG_UINT32 _CleanCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext);
+#endif
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+static IMG_UINT32 gui32NumSyncCheckpointContexts = 0;
+#endif
+
+/* Defined values to indicate status of sync checkpoint, which is
+ * stored in the memory of the structure */
+#define SYNC_CHECKPOINT_PATTERN_IN_USE 0x1a1aa
+#define SYNC_CHECKPOINT_PATTERN_IN_POOL 0x2b2bb
+#define SYNC_CHECKPOINT_PATTERN_FREED 0x3c3cc
+
+#if defined(SUPPORT_RGX)
+static inline void RGXSRVHWPerfSyncCheckpointUFOIsSignalled(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               SYNC_CHECKPOINT *psSyncCheckpointInt, IMG_UINT32 ui32FenceSyncFlags)
+{
+	if (RGXHWPerfHostIsEventEnabled(psDevInfo, RGX_HWPERF_HOST_UFO)
+	    && !(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		RGX_HWPERF_UFO_EV eEv;
+		RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+		if (psSyncCheckpointInt)
+		{
+			IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+			if ((ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ||
+				(ui32State == PVRSRV_SYNC_CHECKPOINT_ERRORED))
+			{
+				sSyncData.sCheckSuccess.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+				sSyncData.sCheckSuccess.ui32Value = ui32State;
+				eEv = RGX_HWPERF_UFO_EV_CHECK_SUCCESS;
+			}
+			else
+			{
+				sSyncData.sCheckFail.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+				sSyncData.sCheckFail.ui32Value = ui32State;
+				sSyncData.sCheckFail.ui32Required = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+				eEv = RGX_HWPERF_UFO_EV_CHECK_FAIL;
+			}
+			RGXHWPerfHostPostUfoEvent(psDevInfo, eEv, &sSyncData,
+			    (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+		}
+	}
+}
+
+static inline void RGXSRVHWPerfSyncCheckpointUFOUpdate(PVRSRV_RGXDEV_INFO *psDevInfo,
+                               SYNC_CHECKPOINT *psSyncCheckpointInt, IMG_UINT32 ui32FenceSyncFlags)
+{
+	if (RGXHWPerfHostIsEventEnabled(psDevInfo, RGX_HWPERF_HOST_UFO)
+	    && !(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+	{
+		RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+		if (psSyncCheckpointInt)
+		{
+			sSyncData.sUpdate.ui32FWAddr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt);
+			sSyncData.sUpdate.ui32OldValue = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+			sSyncData.sUpdate.ui32NewValue = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+			RGXHWPerfHostPostUfoEvent(psDevInfo, RGX_HWPERF_UFO_EV_UPDATE, &sSyncData,
+			    (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+		}
+	}
+}
+#endif
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordAdd(PSYNC_CHECKPOINT_RECORD_HANDLE *phRecord,
+	                    SYNC_CHECKPOINT_BLOCK *hSyncCheckpointBlock,
+	                    IMG_UINT32 ui32FwBlockAddr,
+	                    IMG_UINT32 ui32SyncOffset,
+	                    IMG_UINT32 ui32UID,
+	                    IMG_UINT32 ui32ClassNameSize,
+	                    const IMG_CHAR *pszClassName, PSYNC_CHECKPOINT pSyncCheckpt);
+static PVRSRV_ERROR
+_SyncCheckpointRecordRemove(PSYNC_CHECKPOINT_RECORD_HANDLE hRecord);
+static void _SyncCheckpointState(PDLLIST_NODE psNode,
+                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                 void *pvDumpDebugFile);
+static void _SyncCheckpointDebugRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                        IMG_UINT32 ui32VerbLevel,
+                                        DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                        void *pvDumpDebugFile);
+static PVRSRV_ERROR _SyncCheckpointRecordListInit(PVRSRV_DEVICE_NODE *psDevNode);
+static void _SyncCheckpointRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+#if defined(PDUMP)
+static void
+MISRHandler_PdumpDeferredSyncSignalPoster(void *pvData);
+static PVRSRV_ERROR _SyncCheckpointAllocPDump(PVRSRV_DEVICE_NODE *psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint);
+static PVRSRV_ERROR _SyncCheckpointUpdatePDump(PPVRSRV_DEVICE_NODE psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint, IMG_UINT32 ui32Status, IMG_UINT32 ui32FenceSyncFlags);
+static PVRSRV_ERROR _SyncCheckpointPDumpTransition(void *pvData, PDUMP_TRANSITION_EVENT eEvent);
+#endif
+
+/* Unique incremental ID assigned to sync checkpoints when allocated */
+static IMG_UINT32 g_SyncCheckpointUID;
+
+static void _CheckDeferredCleanupList(_SYNC_CHECKPOINT_CONTEXT *psContext);
+
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContextInt = (_SYNC_CHECKPOINT_CONTEXT *) psContext;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContextInt->psContextCtl;
+	IMG_UINT32 ui32RefCt = OSAtomicRead(&psContextInt->hRefCount);
+
+	if (ui32RefCt == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_INVALID_CONTEXT,
+		              "SyncCheckpointContextUnref context already freed");
+	}
+	else if (OSAtomicDecrement(&psContextInt->hRefCount) == 0)
+	{
+#if defined(PDUMP)
+		PPVRSRV_DEVICE_NODE psDeviceNode = psCtxCtl->psDeviceNode;
+#endif
+
+		/* SyncCheckpointContextDestroy only when no longer referenced */
+		OSSpinLockDestroy(psCtxCtl->hDeferredCleanupListLock);
+		psCtxCtl->hDeferredCleanupListLock = NULL;
+
+		OSSpinLockDestroy(psCtxCtl->hSyncCheckpointStatsLock);
+		psCtxCtl->hSyncCheckpointStatsLock = NULL;
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+		if (psCtxCtl->ui32SyncCheckpointPoolCount)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+			        "%s called for context<%p> with %d sync checkpoints still"
+			        " in the pool",
+			        __func__,
+			        (void *) psContext,
+			        psCtxCtl->ui32SyncCheckpointPoolCount));
+		}
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+		OSSpinLockDestroy(psCtxCtl->hSyncCheckpointPoolLock);
+		psCtxCtl->hSyncCheckpointPoolLock = NULL;
+#endif
+		OSFreeMem(psContextInt->psContextCtl);
+		RA_Delete(psContextInt->psSpanRA);
+		RA_Delete(psContextInt->psSubAllocRA);
+
+#if defined(PDUMP)
+		PVR_ASSERT(dllist_is_empty(&psContext->sSyncCheckpointBlockListHead));
+
+		OSLockAcquire(psDeviceNode->hSyncCheckpointContextListLock);
+		dllist_remove_node(&psContext->sListNode);
+		OSLockRelease(psDeviceNode->hSyncCheckpointContextListLock);
+
+		OSLockDestroy(psContext->hSyncCheckpointBlockListLock);
+#endif
+
+		OSFreeMem(psContext);
+	}
+}
+
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContextInt = (_SYNC_CHECKPOINT_CONTEXT *)psContext;
+	IMG_UINT32 ui32RefCt = OSAtomicRead(&psContextInt->hRefCount);
+
+	if (ui32RefCt == 0)
+	{
+		PVR_LOG_ERROR(PVRSRV_ERROR_INVALID_CONTEXT,
+		              "SyncCheckpointContextRef context use after free");
+	}
+	else
+	{
+		OSAtomicIncrement(&psContextInt->hRefCount);
+	}
+}
+
+/*
+	Internal interfaces for management of synchronisation block memory
+ */
+static PVRSRV_ERROR
+_AllocSyncCheckpointBlock(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                          SYNC_CHECKPOINT_BLOCK    **ppsSyncBlock)
+{
+	PVRSRV_DEVICE_NODE *psDevNode;
+	SYNC_CHECKPOINT_BLOCK *psSyncBlk;
+	PVRSRV_ERROR eError;
+
+	psSyncBlk = OSAllocMem(sizeof(*psSyncBlk));
+	PVR_LOG_GOTO_IF_NOMEM(psSyncBlk, eError, fail_alloc);
+
+	psSyncBlk->psContext = psContext;
+
+	/* Allocate sync checkpoint block */
+	psDevNode = psContext->psContextCtl->psDeviceNode;
+	PVR_LOG_GOTO_IF_INVALID_PARAM(psDevNode, eError, fail_alloc_ufo_block);
+
+	eError = psDevNode->pfnAllocUFOBlock(psDevNode,
+	                                     SYNC_CHECKPOINT_POOL_LIMIT * sizeof(SYNC_CHECKPOINT_FW_OBJ),
+	                                     &psSyncBlk->hMemDesc,
+	                                     &psSyncBlk->ui32FirmwareAddr,
+	                                     &psSyncBlk->ui32SyncBlockSize);
+	PVR_LOG_GOTO_IF_ERROR(eError, "pfnAllocUFOBlock", fail_alloc_ufo_block);
+
+	eError = DevmemAcquireCpuVirtAddr(psSyncBlk->hMemDesc,
+	                                  (void **) &psSyncBlk->pui32LinAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", fail_devmem_acquire);
+
+	OSAtomicWrite(&psSyncBlk->hRefCount, 1);
+
+	OSLockCreate(&psSyncBlk->hLock);
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Allocated Sync Checkpoint UFO block (FirmwareVAddr = 0x%08x)",
+	                      psSyncBlk->ui32FirmwareAddr);
+#if defined(PDUMP)
+	OSLockAcquire(psContext->hSyncCheckpointBlockListLock);
+	dllist_add_to_tail(&psContext->sSyncCheckpointBlockListHead, &psSyncBlk->sListNode);
+	OSLockRelease(psContext->hSyncCheckpointBlockListLock);
+#endif
+
+	*ppsSyncBlock = psSyncBlk;
+	return PVRSRV_OK;
+
+fail_devmem_acquire:
+	psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->hMemDesc);
+fail_alloc_ufo_block:
+	OSFreeMem(psSyncBlk);
+fail_alloc:
+	return eError;
+}
+
+static void
+_FreeSyncCheckpointBlock(SYNC_CHECKPOINT_BLOCK *psSyncBlk)
+{
+	OSLockAcquire(psSyncBlk->hLock);
+	if (0 == OSAtomicDecrement(&psSyncBlk->hRefCount))
+	{
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncBlk->psContext;
+		PVRSRV_DEVICE_NODE *psDevNode = psContext->psContextCtl->psDeviceNode;
+
+#if defined(PDUMP)
+		OSLockAcquire(psSyncBlk->psContext->hSyncCheckpointBlockListLock);
+		dllist_remove_node(&psSyncBlk->sListNode);
+		OSLockRelease(psSyncBlk->psContext->hSyncCheckpointBlockListLock);
+#endif
+		DevmemReleaseCpuVirtAddr(psSyncBlk->hMemDesc);
+		psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->hMemDesc);
+		OSLockRelease(psSyncBlk->hLock);
+		OSLockDestroy(psSyncBlk->hLock);
+		psSyncBlk->hLock = NULL;
+		OSFreeMem(psSyncBlk);
+	}
+	else
+	{
+		OSLockRelease(psSyncBlk->hLock);
+	}
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointBlockImport(RA_PERARENA_HANDLE hArena,
+                           RA_LENGTH_T uSize,
+                           RA_FLAGS_T uFlags,
+                           RA_LENGTH_T uBaseAlignment,
+                           const IMG_CHAR *pszAnnotation,
+                           RA_IMPORT *psImport)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = hArena;
+	SYNC_CHECKPOINT_BLOCK *psSyncBlock = NULL;
+	RA_LENGTH_T uiSpanSize;
+	PVRSRV_ERROR eError;
+	PVR_UNREFERENCED_PARAMETER(uFlags);
+	PVR_UNREFERENCED_PARAMETER(uBaseAlignment);
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM((hArena != NULL), "hArena");
+
+	/* Check we've not be called with an unexpected size */
+	PVR_LOG_RETURN_IF_INVALID_PARAM((uSize == sizeof(SYNC_CHECKPOINT_FW_OBJ)), "uSize");
+
+	/*
+		Ensure the sync checkpoint context doesn't go away while we have
+		sync blocks attached to it.
+	 */
+	SyncCheckpointContextRef((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+	/* Allocate the block of memory */
+	eError = _AllocSyncCheckpointBlock(psContext, &psSyncBlock);
+	PVR_GOTO_IF_ERROR(eError, fail_syncblockalloc);
+
+	/* Allocate a span for it */
+	eError = RA_Alloc(psContext->psSpanRA,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  RA_NO_IMPORT_MULTIPLIER,
+	                  0,
+	                  psSyncBlock->ui32SyncBlockSize,
+	                  pszAnnotation,
+	                  &psSyncBlock->uiSpanBase,
+	                  &uiSpanSize,
+	                  NULL);
+	PVR_GOTO_IF_ERROR(eError, fail_spanalloc);
+
+	/*
+		There is no reason the span RA should return an allocation larger
+		then we request
+	 */
+	PVR_LOG_IF_FALSE((uiSpanSize == psSyncBlock->ui32SyncBlockSize),
+	                 "uiSpanSize invalid");
+
+	psImport->base = psSyncBlock->uiSpanBase;
+	psImport->uSize = psSyncBlock->ui32SyncBlockSize;
+	psImport->hPriv = psSyncBlock;
+	return PVRSRV_OK;
+
+fail_spanalloc:
+	_FreeSyncCheckpointBlock(psSyncBlock);
+fail_syncblockalloc:
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+	return eError;
+}
+
+static void
+_SyncCheckpointBlockUnimport(RA_PERARENA_HANDLE hArena,
+                             RA_BASE_T uiBase,
+                             RA_PERISPAN_HANDLE hImport)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = hArena;
+	SYNC_CHECKPOINT_BLOCK   *psSyncBlock = hImport;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE((psContext != NULL), "hArena invalid");
+	PVR_LOG_RETURN_VOID_IF_FALSE((psSyncBlock != NULL), "hImport invalid");
+	PVR_LOG_RETURN_VOID_IF_FALSE((uiBase == psSyncBlock->uiSpanBase), "uiBase invalid");
+
+	/* Free the span this import is using */
+	RA_Free(psContext->psSpanRA, uiBase);
+
+	/* Free the sync checkpoint block */
+	_FreeSyncCheckpointBlock(psSyncBlock);
+
+	/*	Drop our reference to the sync checkpoint context */
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+}
+
+static INLINE IMG_UINT32 _SyncCheckpointGetOffset(SYNC_CHECKPOINT *psSyncInt)
+{
+	IMG_UINT64 ui64Temp;
+
+	ui64Temp = (IMG_UINT64)psSyncInt->uiAllocatedAddr -
+	           (IMG_UINT64)psSyncInt->psSyncCheckpointBlock->uiSpanBase;
+	PVR_ASSERT(ui64Temp<IMG_UINT32_MAX);
+	return (IMG_UINT32)ui64Temp;
+}
+
+/* Used by SyncCheckpointContextCreate() below */
+static INLINE IMG_UINT32 _Log2(IMG_UINT32 ui32Align)
+{
+	PVR_ASSERT(IsPower2(ui32Align));
+	return ExactLog2(ui32Align);
+}
+
+/*
+	External interfaces
+ */
+
+PVRSRV_ERROR
+SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns)
+{
+	PVR_ASSERT(psSyncCheckpointPfns != NULL);
+
+	if (g_psSyncCheckpointPfnStruct)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s called but already initialised", __func__));
+		return PVRSRV_ERROR_INIT_FAILURE;
+	}
+
+	g_psSyncCheckpointPfnStruct = psSyncCheckpointPfns;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+SyncCheckpointResolveFence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PVRSRV_FENCE hFence, IMG_UINT32 *pui32NumSyncCheckpoints,
+                           PSYNC_CHECKPOINT **papsSyncCheckpoints,
+                           IMG_UINT64 *pui64FenceUID,
+                           PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 i;
+#if defined(PDUMP)
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+#endif
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceResolve))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_pfnFenceResolve is NULL");
+		return eError;
+	}
+
+	if (papsSyncCheckpoints)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceResolve(
+		                           psSyncCheckpointContext,
+		                           hFence,
+		                           pui32NumSyncCheckpoints,
+		                           papsSyncCheckpoints,
+		                           pui64FenceUID);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceResolve");
+
+#if defined(PDUMP)
+	if (*papsSyncCheckpoints)
+	{
+		for (i = 0; i < *pui32NumSyncCheckpoints; i++)
+		{
+			psSyncCheckpoint = (SYNC_CHECKPOINT *)(*papsSyncCheckpoints)[i];
+			psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	if (*pui32NumSyncCheckpoints > MAX_SYNC_CHECKPOINTS_PER_FENCE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: g_psSyncCheckpointPfnStruct->pfnFenceResolve() returned too many checkpoints (%u > MAX_SYNC_CHECKPOINTS_PER_FENCE=%u)",
+				__func__, *pui32NumSyncCheckpoints, MAX_SYNC_CHECKPOINTS_PER_FENCE));
+
+		/* Free resources after error */
+		if (*papsSyncCheckpoints)
+		{
+			for (i = 0; i < *pui32NumSyncCheckpoints; i++)
+			{
+				SyncCheckpointDropRef((*papsSyncCheckpoints)[i]);
+			}
+
+			SyncCheckpointFreeCheckpointListMem(*papsSyncCheckpoints);
+		}
+
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	{
+		IMG_UINT32 ii;
+
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: g_psSyncCheckpointPfnStruct->pfnFenceResolve() for fence %d returned the following %d checkpoints:",
+				__func__,
+				hFence,
+				*pui32NumSyncCheckpoints));
+
+		for (ii=0; ii<*pui32NumSyncCheckpoints; ii++)
+		{
+			PSYNC_CHECKPOINT psNextCheckpoint = *(*papsSyncCheckpoints +  ii);
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s:   *papsSyncCheckpoints[%d]:<%p>",
+					__func__,
+					ii,
+					(void*)psNextCheckpoint));
+		}
+	}
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointResolveExportFence(PVRSRV_FENCE hExportFence,
+                                 PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                 PSYNC_CHECKPOINT *ppsSyncCheckpoint,
+                                 PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnExportFenceResolve))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+		        __func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "pfnExportFenceResolve is NULL");
+		return eError;
+	}
+
+	if (ppsSyncCheckpoint)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnExportFenceResolve(
+		                           hExportFence,
+		                           psSyncCheckpointContext,
+		                           ppsSyncCheckpoint);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnExportFenceResolve");
+
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s: g_psSyncCheckpointPfnStruct->pfnExportFenceResolve() for fence %d returned the following checkpoint:",
+		        __func__,
+		        hExportFence));
+
+		PVR_DPF((PVR_DBG_WARNING,
+		         "%s:   psSyncCheckpoint:<%p>",
+		         __func__,
+		         (void*)*ppsSyncCheckpoint));
+	}
+#endif
+
+#if defined(PDUMP)
+	if (*ppsSyncCheckpoint)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT *)(*ppsSyncCheckpoint);
+		psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointCreateFence(PVRSRV_DEVICE_NODE *psDevNode,
+                          const IMG_CHAR *pszFenceName,
+                          PVRSRV_TIMELINE hTimeline,
+                          PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                          PVRSRV_FENCE *phNewFence,
+                          IMG_UINT64 *puiUpdateFenceUID,
+                          void **ppvFenceFinaliseData,
+                          PSYNC_CHECKPOINT *psNewSyncCheckpoint,
+                          void **ppvTimelineUpdateSyncPrim,
+                          IMG_UINT32 *pui32TimelineUpdateValue,
+                          PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psDevNode);
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceCreate))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceCreate is NULL");
+	}
+	else
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceCreate(
+		                          psDevNode,
+		                          pszFenceName,
+		                          hTimeline,
+		                          psSyncCheckpointContext,
+		                          phNewFence,
+		                          puiUpdateFenceUID,
+		                          ppvFenceFinaliseData,
+		                          psNewSyncCheckpoint,
+		                          ppvTimelineUpdateSyncPrim,
+		                          pui32TimelineUpdateValue);
+		if (unlikely(eError != PVRSRV_OK))
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					"%s failed to create new fence<%p> for timeline<%d> using "
+					"sync checkpoint context<%p>, psNewSyncCheckpoint=<%p>, eError=%s",
+					__func__,
+					(void*)phNewFence,
+					hTimeline,
+					(void*)psSyncCheckpointContext,
+					(void*)psNewSyncCheckpoint,
+					PVRSRVGetErrorString(eError)));
+		}
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s created new fence<%d> for timeline<%d> using "
+					"sync checkpoint context<%p>, new sync_checkpoint=<%p>",
+					__func__,
+					*phNewFence,
+					hTimeline,
+					(void*)psSyncCheckpointContext,
+					(void*)*psNewSyncCheckpoint));
+		}
+#endif
+
+#if defined(PDUMP)
+		if (eError == PVRSRV_OK)
+		{
+			SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT*)(*psNewSyncCheckpoint);
+			if (psSyncCheckpoint)
+			{
+				psSyncCheckpoint->ui32PDumpFlags = ui32PDumpFlags;
+			}
+		}
+#else
+		PVR_UNREFERENCED_PARAMETER(ui32PDumpFlags);
+#endif
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointRollbackFenceData(PVRSRV_FENCE hFence, void *pvFinaliseData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceDataRollback)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceDataRollback is NULL");
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: called to rollback fence data <%p>",
+				__func__,
+				pvFinaliseData));
+#endif
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceDataRollback(
+		            hFence, pvFinaliseData);
+		PVR_LOG_IF_ERROR(eError,
+		                 "g_psSyncCheckpointPfnStruct->pfnFenceDataRollback returned error");
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointFinaliseFence(PPVRSRV_DEVICE_NODE psDevNode,
+                            PVRSRV_FENCE hFence,
+                            void *pvFinaliseData,
+                            PSYNC_CHECKPOINT psSyncCheckpoint,
+                            const IMG_CHAR *pszName)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnFenceFinalise)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: Warning (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED) (this is permitted)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_FENCE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s: called to finalise fence <%d>",
+				__func__,
+				hFence));
+#endif
+		eError = g_psSyncCheckpointPfnStruct->pfnFenceFinalise(hFence, pvFinaliseData);
+		PVR_LOG_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnFenceFinalise returned error");
+
+		RGXSRV_HWPERF_ALLOC_FENCE(psDevNode, OSGetCurrentClientProcessIDKM(), hFence,
+		                          SyncCheckpointGetFirmwareAddr(psSyncCheckpoint),
+		                          pszName, OSStringLength(pszName));
+	}
+	return eError;
+}
+
+void
+SyncCheckpointFreeCheckpointListMem(void *pvCheckpointListMem)
+{
+	if (g_psSyncCheckpointPfnStruct->pfnFreeCheckpointListMem)
+	{
+		g_psSyncCheckpointPfnStruct->pfnFreeCheckpointListMem(pvCheckpointListMem);
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointNoHWUpdateTimelines(void *pvPrivateData)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines is NULL");
+	}
+	else
+	{
+		g_psSyncCheckpointPfnStruct->pfnNoHWUpdateTimelines(pvPrivateData);
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointNoHWSignalExportFence(PVRSRV_FENCE iExportFenceToSignal)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence is NULL");
+	}
+	else
+	{
+		g_psSyncCheckpointPfnStruct->pfnNoHWSignalExpFence(iExportFenceToSignal);
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointDumpInfoOnStalledUFOs(IMG_UINT32 ui32NumUFOs, IMG_UINT32 *pui32Vaddrs, IMG_UINT32 *pui32NumSyncOwnedUFOs)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_LOG_RETURN_IF_FALSE((pui32NumSyncOwnedUFOs != NULL), "pui32NumSyncOwnedUFOs invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	if (!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs)
+	{
+		*pui32NumSyncOwnedUFOs = 0;
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+				__func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs is NULL");
+	}
+	else
+	{
+		*pui32NumSyncOwnedUFOs = g_psSyncCheckpointPfnStruct->pfnDumpInfoOnStalledUFOs(ui32NumUFOs, pui32Vaddrs);
+		PVR_LOG(("%d sync checkpoint%s owned by %s in stalled context",
+		         *pui32NumSyncOwnedUFOs, *pui32NumSyncOwnedUFOs==1 ? "" : "s",
+		         g_psSyncCheckpointPfnStruct->pszImplName));
+	}
+	return eError;
+}
+
+static PVRSRV_ERROR
+_AllocSyncCheckpoint(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                     SYNC_CHECKPOINT **ppsSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psNewSyncCheckpoint = NULL;
+	PVRSRV_ERROR eError;
+
+	/* Allocate sync checkpoint */
+	psNewSyncCheckpoint = OSAllocMem(sizeof(*psNewSyncCheckpoint));
+	PVR_LOG_RETURN_IF_NOMEM(psNewSyncCheckpoint, "OSAllocMem"); /* Sets OOM error code */
+
+	eError = RA_Alloc(psContext->psSubAllocRA,
+                      sizeof(*psNewSyncCheckpoint->psSyncCheckpointFwObj),
+                      RA_NO_IMPORT_MULTIPLIER,
+                      0,
+                      sizeof(IMG_UINT32),
+                      NULL,
+                      &psNewSyncCheckpoint->uiAllocatedAddr,
+                      NULL,
+                      (RA_PERISPAN_HANDLE *) &psNewSyncCheckpoint->psSyncCheckpointBlock);
+
+	PVR_LOG_GOTO_IF_ERROR(eError, "RA_Alloc", fail_ra_alloc);
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+	        "%s CALLED RA_Alloc(), psSubAllocRA=<%p>, ui32SpanAddr=0x%llx",
+	        __func__,
+	        (void*)psContext->psSubAllocRA,
+	        psNewSyncCheckpoint->uiAllocatedAddr));
+#endif
+	psNewSyncCheckpoint->psSyncCheckpointFwObj =
+	        (volatile SYNC_CHECKPOINT_FW_OBJ*)(void *)(psNewSyncCheckpoint->psSyncCheckpointBlock->pui32LinAddr +
+	                (_SyncCheckpointGetOffset(psNewSyncCheckpoint)/sizeof(IMG_UINT32)));
+	psNewSyncCheckpoint->ui32FWAddr = psNewSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+	                                  _SyncCheckpointGetOffset(psNewSyncCheckpoint) + 1;
+	OSAtomicIncrement(&psNewSyncCheckpoint->psSyncCheckpointBlock->psContext->hCheckpointCount);
+
+#if defined(DEBUG)
+	psNewSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_USE;
+#endif
+	psNewSyncCheckpoint->sListNode.psPrevNode = NULL;
+	psNewSyncCheckpoint->sListNode.psNextNode = NULL;
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s called to allocate new sync checkpoint<%p> for context<%p>",
+	         __func__, (void*)psNewSyncCheckpoint, (void*)psContext));
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s                    psSyncCheckpointFwObj<%p>",
+	         __func__, (void*)psNewSyncCheckpoint->psSyncCheckpointFwObj));
+	PVR_DPF((PVR_DBG_WARNING,
+	         "%s                    psSyncCheckpoint FwAddr=0x%x",
+	         __func__, SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psNewSyncCheckpoint)));
+#endif
+	*ppsSyncCheckpoint = psNewSyncCheckpoint;
+	return PVRSRV_OK;
+
+fail_ra_alloc:
+	OSFreeMem(psNewSyncCheckpoint);
+	return eError;
+}
+
+/* Poisons and frees the checkpoint
+ * Decrements context refcount. */
+static void _FreeSyncCheckpoint(SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+
+	psSyncCheckpoint->sCheckpointUFOAddr.ui32Addr = 0;
+	psSyncCheckpoint->psSyncCheckpointFwObj = NULL;
+#if defined(DEBUG)
+	psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_FREED;
+#endif
+
+	RA_Free(psSyncCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA,
+	        psSyncCheckpoint->uiAllocatedAddr);
+	psSyncCheckpoint->psSyncCheckpointBlock = NULL;
+
+	OSFreeMem(psSyncCheckpoint);
+
+	OSAtomicDecrement(&psContext->hCheckpointCount);
+}
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static PVRSRV_ERROR
+_PrepopulateSyncCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext,
+                               IMG_UINT32 ui32InitPoolSize)
+{
+	IMG_UINT32 ui32SyncCheckpoint;
+	SYNC_CHECKPOINT *psNewSyncCheckpoint;
+	PVRSRV_ERROR eError;
+
+	/* Allocate sync checkpoints and place in the pool */
+	for (ui32SyncCheckpoint=0; ui32SyncCheckpoint<ui32InitPoolSize; ui32SyncCheckpoint++)
+	{
+		eError = _AllocSyncCheckpoint(psContext, &psNewSyncCheckpoint);
+		PVR_LOG_RETURN_IF_ERROR(eError, "_AllocSyncCheckpoint");
+
+#if defined(PDUMP)
+		if (ui32SyncCheckpoint == 0)
+		{
+			psContext->psContextCtl->psSyncCheckpointPoolBlock = psNewSyncCheckpoint->psSyncCheckpointBlock;
+		}
+#endif
+
+		if (!_PutCheckpointInPool(psNewSyncCheckpoint))
+		{
+			_FreeSyncCheckpoint(psNewSyncCheckpoint);
+		}
+	}
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,"%s: Sync checkpoint pool [%d/%d]",
+	         __func__, psContext->psContextCtl->ui32SyncCheckpointPoolCount,
+	         psContext->psContextCtl->ui32SyncCheckpointPoolSize));
+#endif
+	return PVRSRV_OK;
+}
+#endif /* if (SYNC_CHECKPOINT_POOL_LIMIT > 0) */
+
+PVRSRV_ERROR
+SyncCheckpointContextCreate(PPVRSRV_DEVICE_NODE psDevNode,
+                            PSYNC_CHECKPOINT_CONTEXT *ppsSyncCheckpointContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = NULL;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *psContextCtl = NULL;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_CHAR azTempName[PVRSRV_SYNC_NAME_LENGTH] = {0};
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if !defined(PDUMP)
+	void *pvAppHintState = NULL;
+	const IMG_UINT32 ui32DefaultMaxPoolLog2Size = PVRSRV_APPHINT_CHECKPOINTPOOLMAXLOG2;
+	const IMG_UINT32 ui32DefaultInitPoolLog2Size = PVRSRV_APPHINT_CHECKPOINTPOOLINITLOG2;
+#endif
+	IMG_UINT32 ui32MaxPoolLog2Size;
+	IMG_UINT32 ui32InitPoolLog2Size;
+	IMG_UINT32 ui32InitPoolSize = 0;
+#endif
+	IMG_UINT32 ui32MaxPoolSize = 0;
+
+	PVR_LOG_RETURN_IF_FALSE((ppsSyncCheckpointContext != NULL),
+	                  "ppsSyncCheckpointContext invalid",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if defined(PDUMP)
+	/*
+	   Pdumps use the maximum pool size to limit checkpoint address reuse and start with
+	   a minimum pool to avoid extra work allocating checkpoints which might not be used.
+	   Since during a pdump capture the pool is setup once during the pdump init phase
+	   and not emptied / de-initialised between captures we need to map at least one
+	   checkpoint at creation time, so that the pdump records the CPU mapping of the
+	   pool. Not doing this would lead to errors on subsequent pdump captures because
+	   only the first pdump would contain the mapping.
+	*/
+	ui32MaxPoolLog2Size = 10;
+	ui32InitPoolLog2Size = 0;
+	ui32InitPoolSize = 1;
+#else
+	/* Read AppHints to determine the size of the sync checkpoint pool, if specified */
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, SyncCheckpointPoolMaxLog2,
+	                     &ui32DefaultMaxPoolLog2Size, &ui32MaxPoolLog2Size);
+	OSGetAppHintUINT32(APPHINT_NO_DEVICE, pvAppHintState, SyncCheckpointPoolInitLog2,
+	                     &ui32DefaultInitPoolLog2Size, &ui32InitPoolLog2Size);
+	OSFreeAppHintState(pvAppHintState);
+#endif
+
+	if (ui32MaxPoolLog2Size > 0)
+	{
+		ui32MaxPoolSize = 1 << ui32MaxPoolLog2Size;
+		if (ui32MaxPoolSize > SYNC_CHECKPOINT_POOL_LIMIT)
+		{
+			ui32MaxPoolSize = SYNC_CHECKPOINT_POOL_LIMIT;
+		}
+	}
+#endif
+
+	psContext = OSAllocMem(sizeof(*psContext));
+	PVR_LOG_GOTO_IF_NOMEM(psContext, eError, fail_alloc); /* Sets OOM error code */
+
+	/* psContextCtl includes allocation for the sync checkpoint pool) */
+	psContextCtl = OSAllocMem(sizeof(*psContextCtl) + IMG_FLEX_ARRAY_SIZE(sizeof(SYNC_CHECKPOINT*), ui32MaxPoolSize));
+	PVR_LOG_GOTO_IF_NOMEM(psContextCtl, eError, fail_alloc2); /* Sets OOM error code */
+
+	eError = OSSpinLockCreate(&psContextCtl->hDeferredCleanupListLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate:1", fail_create_deferred_cleanup_lock);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	eError = OSSpinLockCreate(&psContextCtl->hSyncCheckpointPoolLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSSpinLockCreate:2", fail_create_pool_lock);
+#endif
+
+	dllist_init(&psContextCtl->sDeferredCleanupListHead);
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	psContextCtl->ui32SyncCheckpointPoolCount = 0;
+	psContextCtl->ui32SyncCheckpointPoolWp = 0;
+	psContextCtl->ui32SyncCheckpointPoolRp = 0;
+#if defined(PDUMP)
+	psContextCtl->psSyncCheckpointPoolBlock = NULL;
+	psContextCtl->bAllocateFromCheckpointPool = IMG_FALSE;
+#endif
+	psContextCtl->ui8PoolStateFlags = SYNC_CHECKPOINT_POOL_VALID;
+#endif
+	psContextCtl->psDeviceNode = (SHARED_DEV_CONNECTION)psDevNode;
+
+	/*
+		Create the RA for sub-allocations of the sync checkpoints
+
+		Note:
+		The import size doesn't matter here as the server will pass
+		back the blocksize when it does the import which overrides
+		what we specify here.
+	 */
+	OSSNPrintf(azTempName, PVRSRV_SYNC_NAME_LENGTH,"Sync Prim RA-%p", psContext);
+	psContext->psSubAllocRA = RA_Create(azTempName,
+	                                    /* Params for imports */
+	                                    _Log2(sizeof(IMG_UINT32)),
+	                                    RA_LOCKCLASS_3,
+	                                    _SyncCheckpointBlockImport,
+	                                    _SyncCheckpointBlockUnimport,
+	                                    psContext,
+	                                    RA_POLICY_DEFAULT);
+	PVR_LOG_GOTO_IF_NOMEM(psContext->psSubAllocRA, eError, fail_suballoc);
+
+	/*
+		Create the span-management RA
+
+		The RA requires that we work with linear spans. For our use
+		here we don't require this behaviour as we're always working
+		within offsets of blocks (imports). However, we need to keep
+		the RA happy so we create the "span" management RA which
+		ensures that all are imports are added to the RA in a linear
+		fashion
+	 */
+	OSSNPrintf(azTempName, PVRSRV_SYNC_NAME_LENGTH, "Sync Prim span RA-%p", psContext);
+	psContext->psSpanRA = RA_Create(azTempName,
+	                                /* Params for imports */
+	                                0,
+	                                RA_LOCKCLASS_1,
+	                                NULL,
+	                                NULL,
+	                                NULL,
+	                                RA_POLICY_DEFAULT);
+	PVR_LOG_GOTO_IF_NOMEM(psContext->psSpanRA, eError, fail_span);
+
+	if (!RA_Add(psContext->psSpanRA, 0, MAX_SYNC_CHECKPOINT_MEM, 0, NULL))
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		PVR_LOG_ERROR(eError, "SyncCheckpointContextCreate call to RA_Add(span) failed");
+		goto fail_span_add;
+	}
+
+	OSAtomicWrite(&psContext->hRefCount, 1);
+	OSAtomicWrite(&psContext->hCheckpointCount, 0);
+
+	psContext->psContextCtl = psContextCtl;
+
+	*ppsSyncCheckpointContext = (PSYNC_CHECKPOINT_CONTEXT)psContext;
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s: created psSyncCheckpointContext=<%p> (%d contexts exist)",
+			__func__,
+			(void*)*ppsSyncCheckpointContext,
+			++gui32NumSyncCheckpointContexts));
+#endif
+
+#if defined(PDUMP)
+	dllist_init(&psContext->sSyncCheckpointBlockListHead);
+
+	eError = OSLockCreate(&psContext->hSyncCheckpointBlockListLock);
+	PVR_GOTO_IF_ERROR(eError, fail_span_add);
+
+	OSLockAcquire(psDevNode->hSyncCheckpointContextListLock);
+	dllist_add_to_tail(&psDevNode->sSyncCheckpointContextListHead, &psContext->sListNode);
+	OSLockRelease(psDevNode->hSyncCheckpointContextListLock);
+#endif
+
+	psContextCtl->ui32CurrentInUseSyncCheckpoints = 0;
+	psContextCtl->ui32MaxInUseSyncCheckpoints = 0;
+	eError = OSSpinLockCreate(&psContextCtl->hSyncCheckpointStatsLock);
+	PVR_GOTO_IF_ERROR(eError, fail_span_stat);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	/* Pre-populate the sync checkpoint pool, if specified */
+	psContextCtl->ui32SyncCheckpointPoolSize = ui32MaxPoolSize;
+
+	/* Ensure ui32MaxPoolSize is a POT and does not exceed SYNC_CHECKPOINT_POOL_LIMIT,
+	 * and ui32InitPoolSize does not exceed ui32MaxPoolSize.
+	 */
+	if (psContextCtl->ui32SyncCheckpointPoolSize > SYNC_CHECKPOINT_POOL_LIMIT)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: AppHint SyncCheckpointPoolMaxLog2(%d) would exceed "
+		                          "SYNC_CHECKPOINT_POOL_LIMIT(%d) - limiting to %d",
+		                          __func__, ui32MaxPoolLog2Size,
+		                          SYNC_CHECKPOINT_POOL_LIMIT, SYNC_CHECKPOINT_POOL_LIMIT));
+		psContextCtl->ui32SyncCheckpointPoolSize = SYNC_CHECKPOINT_POOL_LIMIT;
+	}
+
+	if (ui32InitPoolLog2Size > 0)
+	{
+		ui32InitPoolSize = 1 << ui32InitPoolLog2Size;
+	}
+	if (ui32InitPoolSize > psContextCtl->ui32SyncCheckpointPoolSize)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s: AppHint SyncCheckpointPoolInitLog2(%d) would exceed "
+		                          "ui32SyncCheckpointPoolSize(%d) - limiting to %d",
+		                          __func__, ui32InitPoolLog2Size,
+		                          psContextCtl->ui32SyncCheckpointPoolSize,
+		                          psContextCtl->ui32SyncCheckpointPoolSize));
+		ui32InitPoolSize = psContextCtl->ui32SyncCheckpointPoolSize;
+	}
+
+	if (ui32InitPoolSize > 0)
+	{
+		eError = _PrepopulateSyncCheckpointPool(psContext, ui32InitPoolSize);
+		PVR_LOG_RETURN_IF_ERROR_VA(eError, "_PrepopulateSyncCheckpointPool(%d)", ui32InitPoolSize);
+	}
+#endif
+
+	return PVRSRV_OK;
+fail_span_stat:
+#if defined(PDUMP)
+	OSLockDestroy(psContext->hSyncCheckpointBlockListLock);
+	psContext->hSyncCheckpointBlockListLock = NULL;
+#endif
+fail_span_add:
+	RA_Delete(psContext->psSpanRA);
+fail_span:
+	RA_Delete(psContext->psSubAllocRA);
+fail_suballoc:
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	OSSpinLockDestroy(psContextCtl->hSyncCheckpointPoolLock);
+	psContextCtl->hSyncCheckpointPoolLock = NULL;
+fail_create_pool_lock:
+#endif
+	OSSpinLockDestroy(psContextCtl->hDeferredCleanupListLock);
+	psContextCtl->hDeferredCleanupListLock = NULL;
+fail_create_deferred_cleanup_lock:
+	OSFreeMem(psContextCtl);
+fail_alloc2:
+	OSFreeMem(psContext);
+fail_alloc:
+	return eError;
+}
+
+PVRSRV_ERROR SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	IMG_INT iRf = 0;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncCheckpointContext != NULL),
+	                  "psSyncCheckpointContext invalid",
+	                  PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevNode = (PVRSRV_DEVICE_NODE *)psContext->psContextCtl->psDeviceNode;
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s: destroying psSyncCheckpointContext=<%p> (now have %d contexts)",
+			__func__,
+			(void*)psSyncCheckpointContext,
+			--gui32NumSyncCheckpointContexts));
+#endif
+
+	_CheckDeferredCleanupList(psContext);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+	if (psContext->psContextCtl->ui32SyncCheckpointPoolCount > 0)
+	{
+		IMG_UINT32 ui32NumFreedFromPool = _CleanCheckpointPool(psContext);
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s freed %d sync checkpoints that were still in the pool for context<%p>",
+				__func__,
+				ui32NumFreedFromPool,
+				(void*)psContext));
+#else
+		PVR_UNREFERENCED_PARAMETER(ui32NumFreedFromPool);
+#endif
+	}
+#endif
+
+	iRf = OSAtomicRead(&psContext->hCheckpointCount);
+
+	if (iRf != 0)
+	{
+		OS_SPINLOCK_FLAGS uiFlags;
+
+		/* Note, this is not a permanent error as the caller may retry later */
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s <%p> attempted with active references (iRf=%d), "
+				"may be the result of a race",
+				__func__,
+				(void*)psContext,
+				iRf));
+
+		eError = PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT;
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		{
+			DLLIST_NODE *psNode, *psNext;
+
+			dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+			{
+				SYNC_CHECKPOINT *psSyncCheckpoint = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+				bool bDeferredFree = dllist_node_is_in_list(&psSyncCheckpoint->sDeferredFreeListNode);
+#if defined(DEBUG)
+				IMG_UINT32 ui32State = psSyncCheckpoint->psSyncCheckpointFwObj->ui32State;
+#endif
+
+				/* Line below avoids build error in release builds (where PVR_DPF is not defined) */
+				PVR_UNREFERENCED_PARAMETER(bDeferredFree);
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s syncCheckpoint<%p> ID=%d, %s, refs=%d, state=%s, fwaddr=%#08x, enqCount:%d, FWCount:%d %s",
+						__func__,
+						(void*)psSyncCheckpoint,
+						psSyncCheckpoint->ui32UID,
+						psSyncCheckpoint->azName,
+						OSAtomicRead(&psSyncCheckpoint->hRefCount),
+						ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED ?
+								"PVRSRV_SYNC_CHECKPOINT_SIGNALLED" :
+								ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE ?
+										"PVRSRV_SYNC_CHECKPOINT_ACTIVE" : "PVRSRV_SYNC_CHECKPOINT_ERRORED",
+						psSyncCheckpoint->ui32FWAddr,
+						OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+						psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+						bDeferredFree ? "(deferred free)" : ""));
+
+#if (ENABLE_SYNC_CHECKPOINT_CONTEXT_DEBUG == 1)
+				gui32NumSyncCheckpointContexts++;
+#endif
+			}
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+	else
+	{
+		SyncCheckpointContextUnref(psSyncCheckpointContext);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                    PVRSRV_TIMELINE hTimeline,
+                    PVRSRV_FENCE hFence,
+                    const IMG_CHAR *pszCheckpointName,
+                    PSYNC_CHECKPOINT *ppsSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psNewSyncCheckpoint = NULL;
+	_SYNC_CHECKPOINT_CONTEXT *psSyncContextInt = (_SYNC_CHECKPOINT_CONTEXT*)psSyncContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_ERROR eError;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncContext != NULL), "psSyncContext invalid", PVRSRV_ERROR_INVALID_PARAMS);
+	PVR_LOG_RETURN_IF_FALSE((ppsSyncCheckpoint != NULL), "ppsSyncCheckpoint invalid", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psDevNode = (PVRSRV_DEVICE_NODE *)psSyncContextInt->psContextCtl->psDeviceNode;
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+	PVR_DPF((PVR_DBG_WARNING, "%s Entry, Getting checkpoint from pool",
+			 __func__));
+#endif
+	psNewSyncCheckpoint = _GetCheckpointFromPool(psSyncContextInt);
+	if (!psNewSyncCheckpoint)
+	{
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+		PVR_DPF((PVR_DBG_WARNING,
+				 "%s     checkpoint pool empty - will have to allocate",
+				 __func__));
+#endif
+	}
+#endif
+	/* If pool is empty (or not defined) alloc the new sync checkpoint */
+	if (!psNewSyncCheckpoint)
+	{
+		eError = _AllocSyncCheckpoint(psSyncContextInt, &psNewSyncCheckpoint);
+		PVR_LOG_GOTO_IF_NOMEM(psNewSyncCheckpoint, eError, fail_alloc); /* Sets OOM error code */
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0) && defined(PDUMP)
+		/* If the pool is not yet enabled and we allocated all the block, then the pool is ready to use. */
+		if (!psSyncContextInt->psContextCtl->bAllocateFromCheckpointPool)
+		{
+			RA_USAGE_STATS  sRAUsageStats;
+
+			RA_Get_Usage_Stats(psSyncContextInt->psSubAllocRA, &sRAUsageStats);
+
+			if (sRAUsageStats.ui64FreeArenaSize < sizeof(SYNC_CHECKPOINT_FW_OBJ))
+			{
+				psSyncContextInt->psContextCtl->bAllocateFromCheckpointPool = IMG_TRUE;
+			}
+		}
+#endif
+	}
+
+	OSSpinLockAcquire(psSyncContextInt->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+	if (++psSyncContextInt->psContextCtl->ui32CurrentInUseSyncCheckpoints > psSyncContextInt->psContextCtl->ui32MaxInUseSyncCheckpoints)
+	{
+		psSyncContextInt->psContextCtl->ui32MaxInUseSyncCheckpoints = psSyncContextInt->psContextCtl->ui32CurrentInUseSyncCheckpoints;
+	}
+	OSSpinLockRelease(psSyncContextInt->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+	psNewSyncCheckpoint->hTimeline = hTimeline;
+	OSAtomicWrite(&psNewSyncCheckpoint->hRefCount, 1);
+	OSAtomicWrite(&psNewSyncCheckpoint->hEnqueuedCCBCount, 0);
+	psNewSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount = 0;
+	psNewSyncCheckpoint->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_ACTIVE;
+	psNewSyncCheckpoint->uiProcess = OSGetCurrentClientProcessIDKM();
+	OSCachedMemSet(&psNewSyncCheckpoint->sDeferredFreeListNode, 0, sizeof(psNewSyncCheckpoint->sDeferredFreeListNode));
+
+	if (pszCheckpointName)
+	{
+		/* Copy over the checkpoint name annotation */
+		OSStringSafeCopy(psNewSyncCheckpoint->azName, pszCheckpointName, PVRSRV_SYNC_NAME_LENGTH);
+	}
+	else
+	{
+		/* No sync checkpoint name annotation */
+		psNewSyncCheckpoint->azName[0] = '\0';
+	}
+
+	/* Store sync checkpoint FW address in PRGXFWIF_UFO_ADDR struct */
+	psNewSyncCheckpoint->sCheckpointUFOAddr.ui32Addr = SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psNewSyncCheckpoint);
+
+	/* Assign unique ID to this sync checkpoint */
+	psNewSyncCheckpoint->ui32UID = g_SyncCheckpointUID++;
+
+#if defined(PDUMP)
+	/* Flushing deferred fence signals to pdump */
+	MISRHandler_PdumpDeferredSyncSignalPoster(psDevNode);
+
+	_SyncCheckpointAllocPDump(psDevNode, psNewSyncCheckpoint);
+#endif
+
+	RGXSRV_HWPERF_ALLOC_SYNC_CP(psDevNode, psNewSyncCheckpoint->hTimeline,
+	                            OSGetCurrentClientProcessIDKM(),
+	                            hFence,
+	                            psNewSyncCheckpoint->ui32FWAddr,
+	                            psNewSyncCheckpoint->azName,
+	                            OSStringLength(psNewSyncCheckpoint->azName));
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		IMG_CHAR szChkptName[PVRSRV_SYNC_NAME_LENGTH];
+
+		if (pszCheckpointName)
+		{
+			/* Copy the checkpoint name annotation into a fixed-size array */
+			OSStringSafeCopy(szChkptName, pszCheckpointName, PVRSRV_SYNC_NAME_LENGTH);
+		}
+		else
+		{
+			/* No checkpoint name annotation */
+			szChkptName[0] = 0;
+		}
+		/* record this sync */
+		eError = _SyncCheckpointRecordAdd(&psNewSyncCheckpoint->hRecord,
+		                                 psNewSyncCheckpoint->psSyncCheckpointBlock,
+		                                 psNewSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr,
+		                                 _SyncCheckpointGetOffset(psNewSyncCheckpoint),
+		                                 psNewSyncCheckpoint->ui32UID,
+		                                 OSStringNLength(szChkptName, PVRSRV_SYNC_NAME_LENGTH),
+		                                 szChkptName, (PSYNC_CHECKPOINT)psNewSyncCheckpoint);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync checkpoint record \"%s\" (%s)",
+					__func__,
+					szChkptName,
+					PVRSRVGetErrorString(eError)));
+			psNewSyncCheckpoint->hRecord = NULL;
+			/* note the error but continue without affecting driver operation */
+		}
+	}
+
+	/* Add the sync checkpoint to the device list */
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_add_to_head(&psDevNode->sSyncCheckpointSyncsList,
+	                   &psNewSyncCheckpoint->sListNode);
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+	*ppsSyncCheckpoint = (PSYNC_CHECKPOINT)psNewSyncCheckpoint;
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Exit(Ok), psNewSyncCheckpoint->ui32UID=%d <%p>",
+			__func__,
+			psNewSyncCheckpoint->ui32UID,
+			(void*)psNewSyncCheckpoint));
+#endif
+	return PVRSRV_OK;
+
+fail_alloc:
+	return eError;
+}
+
+static void SyncCheckpointUnref(SYNC_CHECKPOINT *psSyncCheckpointInt)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	psContext = psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+	psDevNode = (PVRSRV_DEVICE_NODE *)psContext->psContextCtl->psDeviceNode;
+
+	/*
+	 * Without this reference, the context may be destroyed as soon
+	 * as _FreeSyncCheckpoint is called, but the context is still
+	 * needed when _CheckDeferredCleanupList is called at the end
+	 * of this function.
+	 */
+	SyncCheckpointContextRef((PSYNC_CHECKPOINT_CONTEXT)psContext);
+
+#if defined(DEBUG)
+	PVR_ASSERT(psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE);
+#endif
+	if (!OSAtomicRead(&psSyncCheckpointInt->hRefCount))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncCheckpointUnref sync checkpoint already freed"));
+	}
+	else if (0 == OSAtomicDecrement(&psSyncCheckpointInt->hRefCount))
+	{
+		/* If the firmware has serviced all enqueued references to the sync checkpoint, free it */
+		if (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount ==
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)))
+		{
+			PVRSRV_ERROR eError;
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s No outstanding FW ops and hRef is zero, deleting SyncCheckpoint..",
+					__func__));
+#endif
+
+			if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+				&& psSyncCheckpointInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = _SyncCheckpointRecordRemove(psSyncCheckpointInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "_SyncCheckpointRecordRemove");
+			}
+
+			/* Remove the sync checkpoint from the global list */
+			OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+			dllist_remove_node(&psSyncCheckpointInt->sListNode);
+			OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+			RGXSRV_HWPERF_FREE(psDevNode, SYNC_CP, psSyncCheckpointInt->ui32FWAddr);
+
+			OSSpinLockAcquire(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+			psContext->psContextCtl->ui32CurrentInUseSyncCheckpoints--;
+			OSSpinLockRelease(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s attempting to return sync checkpoint to the pool",
+					__func__));
+#endif
+			if (!_PutCheckpointInPool(psSyncCheckpointInt))
+#endif
+			{
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if ((ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s pool is full, so just free it",
+						__func__));
+#endif
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+				PVR_DPF((PVR_DBG_WARNING,
+						"%s CALLING RA_Free(psSyncCheckpoint(ID:%d)<%p>), psSubAllocRA=<%p>, ui32AllocatedAddr=0x%llx",
+						__func__,
+						psSyncCheckpointInt->ui32UID,
+						(void*)psSyncCheckpointInt,
+						(void*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext->psSubAllocRA,
+						psSyncCheckpointInt->uiAllocatedAddr));
+#endif
+				_FreeSyncCheckpoint(psSyncCheckpointInt);
+			}
+		}
+		else
+		{
+			OS_SPINLOCK_FLAGS uiFlags;
+#if ((ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1) || (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1))
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s Outstanding FW ops hEnqueuedCCBCount=%d != FwObj->ui32FwRefCount=%d "
+					"- DEFERRING CLEANUP psSyncCheckpoint(ID:%d)<%p>",
+					__func__,
+					OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+					psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount,
+					psSyncCheckpointInt->ui32UID,
+					(void*)psSyncCheckpointInt));
+#endif
+			/* Add the sync checkpoint to the deferred free list */
+			OSSpinLockAcquire(psContext->psContextCtl->hDeferredCleanupListLock, uiFlags);
+			dllist_add_to_tail(&psContext->psContextCtl->sDeferredCleanupListHead,
+			                   &psSyncCheckpointInt->sDeferredFreeListNode);
+			OSSpinLockRelease(psContext->psContextCtl->hDeferredCleanupListLock, uiFlags);
+		}
+	}
+	else
+	{
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s psSyncCheckpoint(ID:%d)<%p>, hRefCount decremented to %d",
+				__func__,
+				psSyncCheckpointInt->ui32UID,
+				(void*)psSyncCheckpointInt,
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount))));
+#endif
+	}
+
+	/* See if any sync checkpoints in the deferred cleanup list can be freed */
+	_CheckDeferredCleanupList(psContext);
+
+	SyncCheckpointContextUnref((PSYNC_CHECKPOINT_CONTEXT)psContext);
+}
+
+void SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Entry,  psSyncCheckpoint(ID:%d)<%p>, hRefCount=%d, psSyncCheckpoint->ui32ValidationCheck=0x%x",
+			__func__,
+			psSyncCheckpointInt->ui32UID,
+			(void*)psSyncCheckpoint,
+			(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount)),
+			psSyncCheckpointInt->ui32ValidationCheck));
+#else
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s Entry,  psSyncCheckpoint(ID:%d)<%p>, hRefCount=%d",
+			__func__,
+			psSyncCheckpointInt->ui32UID,
+			(void*)psSyncCheckpoint,
+			(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hRefCount))));
+#endif
+#endif
+	SyncCheckpointUnref(psSyncCheckpointInt);
+}
+
+void
+SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+			RGXSRVHWPerfSyncCheckpointUFOUpdate(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+
+#if defined(SUPPORT_RGX) && defined(PDUMP)
+			_SyncCheckpointUpdatePDump(psContext->psContextCtl->psDeviceNode, psSyncCheckpointInt, PVRSRV_SYNC_CHECKPOINT_SIGNALLED, ui32FenceSyncFlags);
+#endif
+		}
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "%s asked to set PVRSRV_SYNC_CHECKPOINT_SIGNALLED(%d) for (psSyncCheckpointInt->ui32UID=%d), "
+					 "when value is already %d",
+					 __func__,
+					 PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+					 psSyncCheckpointInt->ui32UID,
+					 ui32State));
+		}
+	}
+}
+
+void
+SyncCheckpointSignalNoHW(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+			RGXSRVHWPerfSyncCheckpointUFOUpdate(psDevInfo, psSyncCheckpointInt, PVRSRV_FENCE_FLAG_NONE);
+#endif
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
+		}
+		else
+		{
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s asked to set PVRSRV_SYNC_CHECKPOINT_SIGNALLED(%d) for (psSyncCheckpointInt->ui32UID=%d), "
+					"when value is already %d",
+					__func__,
+					PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+					psSyncCheckpointInt->ui32UID,
+					ui32State));
+#endif
+		}
+	}
+}
+
+void
+SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+
+		PVR_LOG_IF_FALSE((ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE),
+		                 "psSyncCheckpoint already signalled");
+
+		if (ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+		{
+#if defined(SUPPORT_RGX)
+			_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+			PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+			if (!(ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
+			{
+				RGX_HWPERF_UFO_DATA_ELEMENT sSyncData;
+
+				sSyncData.sUpdate.ui32FWAddr = SyncCheckpointGetFirmwareAddr(psSyncCheckpoint);
+				sSyncData.sUpdate.ui32OldValue = ui32State;
+				sSyncData.sUpdate.ui32NewValue = PVRSRV_SYNC_CHECKPOINT_ERRORED;
+
+				RGXSRV_HWPERF_UFO(psDevInfo, RGX_HWPERF_UFO_EV_UPDATE, &sSyncData,
+				                  (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE);
+			}
+#endif
+
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_ERRORED;
+
+#if defined(SUPPORT_RGX) && defined(PDUMP)
+			_SyncCheckpointUpdatePDump(psContext->psContextCtl->psDeviceNode, psSyncCheckpointInt, PVRSRV_SYNC_CHECKPOINT_ERRORED, ui32FenceSyncFlags);
+#endif
+		}
+	}
+}
+
+IMG_BOOL SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+		IMG_UINT32 ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+#if defined(SUPPORT_RGX)
+		PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+		RGXSRVHWPerfSyncCheckpointUFOIsSignalled(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+
+		switch (ui32State)
+		{
+			case PVRSRV_SYNC_CHECKPOINT_UNDEF:
+			case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			{
+				break;
+			}
+			case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			{
+				bRet = IMG_TRUE;
+				break;
+			}
+			default:
+			{
+
+				PVRSRV_DEVICE_NODE *psDeviceNode = psContext->psContextCtl->psDeviceNode;
+
+#if defined(SUPPORT_RGX)
+				RGXUpdateHealthStatus(psDeviceNode, IMG_FALSE);
+#endif
+				if (psDeviceNode->eDevState == PVRSRV_DEVICE_STATE_PCI_ERROR)
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: PCI error - state=0x%x, ID=%d, %s, fwaddr=%#08x",
+							 __func__, ui32State, psSyncCheckpoint->ui32UID,
+							 psSyncCheckpoint->azName, psSyncCheckpoint->ui32FWAddr));
+					bRet = IMG_TRUE;
+				}
+				else
+				{
+					PVR_DPF((PVR_DBG_ERROR, "%s: unknown state (0x%x) - ID=%d, %s, fwaddr=%#08x",
+							 __func__, ui32State, psSyncCheckpoint->ui32UID,
+							 psSyncCheckpoint->azName, psSyncCheckpoint->ui32FWAddr));
+				}
+				break;
+			}
+
+		}
+
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s called for psSyncCheckpoint<%p>, returning %d",
+				__func__,
+				(void*)psSyncCheckpoint,
+				bRet));
+#endif
+	}
+	return bRet;
+}
+
+IMG_BOOL
+SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bRet = IMG_FALSE;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid");
+
+	if (psSyncCheckpointInt)
+	{
+#if defined(SUPPORT_RGX)
+		_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpointInt->psSyncCheckpointBlock->psContext;
+		PVRSRV_RGXDEV_INFO *psDevInfo = psContext->psContextCtl->psDeviceNode->pvDevice;
+
+		RGXSRVHWPerfSyncCheckpointUFOIsSignalled(psDevInfo, psSyncCheckpointInt, ui32FenceSyncFlags);
+#endif
+		bRet = (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_ERRORED);
+
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+				"%s called for psSyncCheckpoint<%p>, returning %d",
+				__func__,
+				(void*)psSyncCheckpoint,
+				bRet));
+#endif
+	}
+	return bRet;
+}
+
+const IMG_CHAR *
+SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_FALSE((psSyncCheckpoint != NULL), "psSyncCheckpoint invalid", "Null");
+
+	switch (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State)
+	{
+		case PVRSRV_SYNC_CHECKPOINT_SIGNALLED:
+			return "Signalled";
+		case PVRSRV_SYNC_CHECKPOINT_ACTIVE:
+			return "Active";
+		case PVRSRV_SYNC_CHECKPOINT_ERRORED:
+			return "Errored";
+		case PVRSRV_SYNC_CHECKPOINT_UNDEF:
+			return "Undefined";
+		default:
+			return "Unknown";
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	PVRSRV_ERROR eRet = PVRSRV_OK;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncCheckpoint, "psSyncCheckpoint");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount),
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount)+1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	OSAtomicIncrement(&psSyncCheckpointInt->hRefCount);
+
+	return eRet;
+}
+
+PVRSRV_ERROR
+SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	PVRSRV_ERROR eRet = PVRSRV_OK;
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncCheckpoint, "psSyncCheckpoint");
+
+#if (ENABLE_SYNC_CHECKPOINT_ALLOC_AND_FREE_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount),
+			OSAtomicRead(&psSyncCheckpointInt->hRefCount)-1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	SyncCheckpointUnref(psSyncCheckpointInt);
+
+	return eRet;
+}
+
+void
+SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_RETURN_VOID_IF_FALSE(psSyncCheckpointInt != NULL, "psSyncCheckpoint");
+
+#if defined(NO_HARDWARE)
+	PVR_UNREFERENCED_PARAMETER(psSyncCheckpointInt);
+#else /* !defined(NO_HARDWARE) */
+#if (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called for psSyncCheckpoint<%p> %d->%d (FWRef %u)",
+			__func__,
+			(void*)psSyncCheckpointInt,
+			OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+			OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)+1,
+			psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+	OSAtomicIncrement(&psSyncCheckpointInt->hEnqueuedCCBCount);
+#endif
+}
+
+PRGXFWIF_UFO_ADDR*
+SyncCheckpointGetRGXFWIFUFOAddr(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if defined(DEBUG)
+	if (psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE)
+#endif
+	{
+		return &psSyncCheckpointInt->sCheckpointUFOAddr;
+	}
+#if defined(DEBUG)
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called for psSyncCheckpoint<%p>, but ui32ValidationCheck=0x%x",
+				__func__,
+				(void*)psSyncCheckpointInt,
+				psSyncCheckpointInt->ui32ValidationCheck));
+	}
+#endif
+
+invalid_chkpt:
+	return NULL;
+}
+
+IMG_UINT32
+SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	IMG_UINT32 ui32Ret = 0;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if defined(DEBUG)
+	if (psSyncCheckpointInt->ui32ValidationCheck == SYNC_CHECKPOINT_PATTERN_IN_USE)
+#endif
+	{
+		ui32Ret = psSyncCheckpointInt->ui32FWAddr;
+	}
+#if defined(DEBUG)
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s called for psSyncCheckpoint<%p>, but ui32ValidationCheck=0x%x",
+				__func__,
+				(void*)psSyncCheckpointInt,
+				psSyncCheckpointInt->ui32ValidationCheck));
+	}
+#endif
+
+invalid_chkpt:
+	return ui32Ret;
+}
+
+IMG_UINT32
+SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	IMG_UINT32 ui32Ret = 0U;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s returning ID for sync checkpoint<%p>",
+			__func__,
+			(void*)psSyncCheckpointInt));
+#if defined(DEBUG)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s (validationCheck=0x%x)",
+			__func__,
+			psSyncCheckpointInt->ui32ValidationCheck));
+#endif
+#endif
+	ui32Ret = psSyncCheckpointInt->ui32UID;
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s (ui32UID=0x%x)",
+			__func__,
+			psSyncCheckpointInt->ui32UID));
+#endif
+
+invalid_chkpt:
+	return ui32Ret;
+}
+
+PVRSRV_TIMELINE
+SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+
+	PVR_LOG_GOTO_IF_FALSE((psSyncCheckpointInt != NULL), "psSyncCheckpoint invalid", invalid_chkpt);
+
+	return psSyncCheckpointInt->hTimeline;
+
+invalid_chkpt:
+	return 0;
+}
+
+
+IMG_UINT32
+SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount);
+}
+
+IMG_UINT32
+SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return OSAtomicRead(&psSyncCheckpointInt->hRefCount);
+}
+
+IMG_PID
+SyncCheckpointGetCreator(PSYNC_CHECKPOINT psSyncCheckpoint)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt = (SYNC_CHECKPOINT*)psSyncCheckpoint;
+	PVR_LOG_RETURN_IF_FALSE(psSyncCheckpoint != NULL, "psSyncCheckpoint invalid", 0);
+
+	return psSyncCheckpointInt->uiProcess;
+}
+
+IMG_UINT32 SyncCheckpointStateFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt;
+	PDLLIST_NODE psNode, psNext;
+	IMG_UINT32 ui32State = 0;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+	{
+		psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+		if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+		{
+			ui32State = psSyncCheckpointInt->psSyncCheckpointFwObj->ui32State;
+			break;
+		}
+	}
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	return ui32State;
+}
+
+void SyncCheckpointErrorFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr)
+{
+	SYNC_CHECKPOINT *psSyncCheckpointInt;
+	PDLLIST_NODE psNode, psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s called to error UFO with ui32FWAddr=%d",
+			__func__,
+			ui32FwAddr));
+#endif
+
+	OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+	{
+		psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+		if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+		{
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING,
+					"%s calling SyncCheckpointError for sync checkpoint <%p>",
+					__func__,
+					(void*)psSyncCheckpointInt));
+#endif
+			/* Mark as errored */
+			SyncCheckpointError((PSYNC_CHECKPOINT)psSyncCheckpointInt, PVRSRV_FENCE_FLAG_NONE);
+			break;
+		}
+	}
+	OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+}
+
+void SyncCheckpointRollbackFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr)
+{
+#if (ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING,
+			"%s called to rollback UFO with ui32FWAddr=0x%x",
+			__func__,
+			ui32FwAddr));
+#endif
+#if !defined(NO_HARDWARE)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpointInt = NULL;
+		PDLLIST_NODE psNode = NULL, psNext = NULL;
+		OS_SPINLOCK_FLAGS uiFlags;
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+		{
+			psSyncCheckpointInt = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+			if (ui32FwAddr == SyncCheckpointGetFirmwareAddr((PSYNC_CHECKPOINT)psSyncCheckpointInt))
+			{
+#if ((ENABLE_SYNC_CHECKPOINT_UFO_DEBUG == 1)) || (ENABLE_SYNC_CHECKPOINT_ENQ_AND_SIGNAL_DEBUG == 1)
+				PVR_DPF((PVR_DBG_WARNING,
+				        "%s called for psSyncCheckpointInt<%p> %d->%d",
+				        __func__,
+				        (void *) psSyncCheckpointInt,
+				        OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount),
+				        OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount) - 1));
+#endif
+				OSAtomicDecrement(&psSyncCheckpointInt->hEnqueuedCCBCount);
+				break;
+			}
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+#endif
+}
+
+PVRSRV_ERROR SyncCheckpointRollbackExportFence(PVRSRV_FENCE hExportFence)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	if (unlikely(!g_psSyncCheckpointPfnStruct || !g_psSyncCheckpointPfnStruct->pfnExportFenceRollback))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		        "%s: ERROR (eError=PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED)",
+		        __func__));
+		eError = PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED;
+		PVR_LOG_ERROR(eError, "pfnExportFenceRollback is NULL");
+		return eError;
+	}
+
+	if (hExportFence != PVRSRV_NO_FENCE)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnExportFenceRollback(hExportFence);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_psSyncCheckpointPfnStruct->pfnExportFenceRollback");
+
+	return eError;
+}
+
+static void _SyncCheckpointState(PDLLIST_NODE psNode,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	SYNC_CHECKPOINT *psSyncCheckpoint = IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sListNode);
+
+	if (psSyncCheckpoint->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_ACTIVE)
+	{
+		PVR_DUMPDEBUG_LOG("\t- ID = %d, FWAddr = 0x%08x, r%d:e%d:f%d: %s",
+		                   psSyncCheckpoint->ui32UID,
+		                   psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+		                   _SyncCheckpointGetOffset(psSyncCheckpoint),
+		                   OSAtomicRead(&psSyncCheckpoint->hRefCount),
+		                   OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+		                   psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+		                   psSyncCheckpoint->azName);
+	}
+}
+
+static void _SyncCheckpointDebugRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	DLLIST_NODE *psNode, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psDevNode->hSyncCheckpointContext == NULL) return;
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		PVR_DUMPDEBUG_LOG("------[ Active Sync Checkpoints ]------");
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+		PVR_DUMPDEBUG_LOG("(SyncCP Counts: InUse:%d Max:%d)",
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32CurrentInUseSyncCheckpoints,
+		                  psDevNode->hSyncCheckpointContext->psContextCtl->ui32MaxInUseSyncCheckpoints);
+		OSSpinLockRelease(psDevNode->hSyncCheckpointContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointSyncsList, psNode, psNext)
+		{
+			_SyncCheckpointState(psNode, pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+	}
+}
+
+PVRSRV_ERROR
+SyncCheckpointInit(PPVRSRV_DEVICE_NODE psDevNode)
+{
+	PVRSRV_ERROR eError;
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+#endif
+
+	eError = OSSpinLockCreate(&psDevNode->hSyncCheckpointListLock);
+	PVR_RETURN_IF_ERROR(eError);
+
+	dllist_init(&psDevNode->sSyncCheckpointSyncsList);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncCheckpointNotify,
+		                                          psDevNode,
+		                                          _SyncCheckpointDebugRequest,
+		                                          DEBUG_REQUEST_SYNCCHECKPOINT,
+		                                          (PVRSRV_DBGREQ_HANDLE)psDevNode);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		_SyncCheckpointRecordListInit(psDevNode);
+	}
+
+#if defined(PDUMP)
+	eError = OSSpinLockCreate(&psDevInfo->hSyncCheckpointSignalSpinLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+		goto e1;
+	}
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointSignalLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hSyncCheckpointSignalLock = NULL;
+		goto e2;
+	}
+
+	psDevNode->pui8DeferredSyncCPSignal = OSAllocMem(SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL
+	                                                      * sizeof(_SYNC_CHECKPOINT_DEFERRED_SIGNAL));
+	PVR_GOTO_IF_NOMEM(psDevNode->pui8DeferredSyncCPSignal, eError, e3);
+
+	psDevNode->ui16SyncCPWriteIdx = 0;
+	psDevNode->ui16SyncCPReadIdx = 0;
+
+	eError = OSInstallMISR(&psDevNode->pvSyncCPMISR,
+					MISRHandler_PdumpDeferredSyncSignalPoster,
+					psDevNode,
+					"RGX_PdumpDeferredSyncSignalPoster");
+	PVR_GOTO_IF_ERROR(eError, e4);
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointContextListLock);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hSyncCheckpointContextListLock = NULL;
+		goto e5;
+	}
+
+
+	dllist_init(&psDevNode->sSyncCheckpointContextListHead);
+
+	eError = PDumpRegisterTransitionCallbackFenceSync(psDevNode,
+								_SyncCheckpointPDumpTransition,
+								&psDevNode->hTransition);
+	if (eError != PVRSRV_OK)
+	{
+		psDevNode->hTransition = NULL;
+		goto e6;
+	}
+#endif
+
+	return PVRSRV_OK;
+
+#if defined(PDUMP)
+e6:
+	OSLockDestroy(psDevNode->hSyncCheckpointContextListLock);
+	psDevNode->hSyncCheckpointContextListLock = NULL;
+e5:
+	(void) OSUninstallMISR(psDevNode->pvSyncCPMISR);
+	psDevNode->pvSyncCPMISR = NULL;
+e4:
+	if (psDevNode->pui8DeferredSyncCPSignal)
+	{
+		OSFreeMem(psDevNode->pui8DeferredSyncCPSignal);
+		psDevNode->pui8DeferredSyncCPSignal = NULL;
+	}
+e3:
+	OSLockDestroy(psDevNode->hSyncCheckpointSignalLock);
+	psDevNode->hSyncCheckpointSignalLock = NULL;
+e2:
+	OSSpinLockDestroy(psDevInfo->hSyncCheckpointSignalSpinLock);
+	psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+e1:
+	_SyncCheckpointRecordListDeinit(psDevNode);
+#endif
+e0:
+	OSSpinLockDestroy(psDevNode->hSyncCheckpointListLock);
+	psDevNode->hSyncCheckpointListLock = NULL;
+
+	return eError;
+}
+
+void SyncCheckpointDeinit(PPVRSRV_DEVICE_NODE psDevNode)
+{
+#if defined(PDUMP)
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+	PDumpUnregisterTransitionCallbackFenceSync(psDevNode->hTransition);
+	psDevNode->hTransition = NULL;
+
+	if (psDevNode->hSyncCheckpointContextListLock)
+	{
+		OSLockDestroy(psDevNode->hSyncCheckpointContextListLock);
+		psDevNode->hSyncCheckpointContextListLock = NULL;
+	}
+
+	if (psDevNode->pvSyncCPMISR)
+	{
+		(void) OSUninstallMISR(psDevNode->pvSyncCPMISR);
+		psDevNode->pvSyncCPMISR = NULL;
+	}
+
+	if (psDevNode->pui8DeferredSyncCPSignal)
+	{
+		OSFreeMem(psDevNode->pui8DeferredSyncCPSignal);
+		psDevNode->pui8DeferredSyncCPSignal = NULL;
+	}
+	if (psDevNode->hSyncCheckpointSignalLock)
+	{
+		OSLockDestroy(psDevNode->hSyncCheckpointSignalLock);
+		psDevNode->hSyncCheckpointSignalLock = NULL;
+	}
+	if (psDevInfo->hSyncCheckpointSignalSpinLock)
+	{
+		OSSpinLockDestroy(psDevInfo->hSyncCheckpointSignalSpinLock);
+		psDevInfo->hSyncCheckpointSignalSpinLock = NULL;
+	}
+#endif
+
+	PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncCheckpointNotify);
+	psDevNode->hSyncCheckpointNotify = NULL;
+	OSSpinLockDestroy(psDevNode->hSyncCheckpointListLock);
+	psDevNode->hSyncCheckpointListLock = NULL;
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		_SyncCheckpointRecordListDeinit(psDevNode);
+	}
+}
+
+void SyncCheckpointRecordLookup(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr,
+                                IMG_CHAR * pszSyncInfo, size_t len)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (!pszSyncInfo)
+	{
+		return;
+	}
+
+	pszSyncInfo[0] = '\0';
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+	{
+		struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+		if ((psSyncCheckpointRec->ui32FwBlockAddr + psSyncCheckpointRec->ui32SyncOffset + 1) == ui32FwAddr)
+		{
+			SYNC_CHECKPOINT_BLOCK *psSyncCheckpointBlock = psSyncCheckpointRec->psSyncCheckpointBlock;
+			if (psSyncCheckpointBlock && psSyncCheckpointBlock->pui32LinAddr)
+			{
+				void *pSyncCheckpointAddr = IMG_OFFSET_ADDR(psSyncCheckpointBlock->pui32LinAddr,
+													psSyncCheckpointRec->ui32SyncOffset);
+				OSSNPrintf(pszSyncInfo, len, "%s Checkpoint:%05u (%s)",
+				           (*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ?
+				                   "SIGNALLED" :
+				                   ((*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_ERRORED) ?
+				                           "ERRORED" : "ACTIVE"),
+				                           psSyncCheckpointRec->uiPID,
+				                           psSyncCheckpointRec->szClassName);
+			}
+			else
+			{
+				OSSNPrintf(pszSyncInfo, len, "Checkpoint:%05u (%s)",
+				           psSyncCheckpointRec->uiPID,
+				           psSyncCheckpointRec->szClassName);
+			}
+
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	if (!bFound && (psDevNode->ui32SyncCheckpointRecordCountHighWatermark == SYNC_CHECKPOINT_RECORD_LIMIT))
+	{
+		OSSNPrintf(pszSyncInfo, len, "(Record may be lost)");
+	}
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordAdd(
+			PSYNC_CHECKPOINT_RECORD_HANDLE * phRecord,
+			SYNC_CHECKPOINT_BLOCK *hSyncCheckpointBlock,
+			IMG_UINT32 ui32FwBlockAddr,
+			IMG_UINT32 ui32SyncOffset,
+			IMG_UINT32 ui32UID,
+			IMG_UINT32 ui32ClassNameSize,
+			const IMG_CHAR *pszClassName, PSYNC_CHECKPOINT pSyncCheckpt)
+{
+	struct SYNC_CHECKPOINT_RECORD * psSyncRec;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = (_SYNC_CHECKPOINT_CONTEXT*)hSyncCheckpointBlock->psContext;
+	PVRSRV_DEVICE_NODE *psDevNode = psContext->psContextCtl->psDeviceNode;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_RETURN_IF_INVALID_PARAM(phRecord);
+
+	*phRecord = NULL;
+
+	psSyncRec = OSAllocMem(sizeof(*psSyncRec));
+	PVR_LOG_GOTO_IF_NOMEM(psSyncRec, eError, fail_alloc); /* Sets OOM error code */
+
+	psSyncRec->psSyncCheckpointBlock = hSyncCheckpointBlock;
+	psSyncRec->ui32SyncOffset = ui32SyncOffset;
+	psSyncRec->ui32FwBlockAddr = ui32FwBlockAddr;
+	psSyncRec->ui64OSTime = OSClockns64();
+	psSyncRec->uiPID = OSGetCurrentProcessID();
+	psSyncRec->ui32UID = ui32UID;
+	psSyncRec->pSyncCheckpt = pSyncCheckpt;
+	if (pszClassName)
+	{
+		if (ui32ClassNameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			ui32ClassNameSize = PVRSRV_SYNC_NAME_LENGTH;
+		/* Copy over the class name annotation */
+		OSStringSafeCopy(psSyncRec->szClassName, pszClassName, ui32ClassNameSize);
+	}
+	else
+	{
+		/* No class name annotation */
+		psSyncRec->szClassName[0] = 0;
+	}
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	if (psDevNode->ui32SyncCheckpointRecordCount < SYNC_CHECKPOINT_RECORD_LIMIT)
+	{
+		dllist_add_to_head(&psDevNode->sSyncCheckpointRecordList, &psSyncRec->sNode);
+		psDevNode->ui32SyncCheckpointRecordCount++;
+
+		if (psDevNode->ui32SyncCheckpointRecordCount > psDevNode->ui32SyncCheckpointRecordCountHighWatermark)
+		{
+			psDevNode->ui32SyncCheckpointRecordCountHighWatermark = psDevNode->ui32SyncCheckpointRecordCount;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync checkpoint record \"%s\". %u records already exist.",
+				__func__,
+				pszClassName,
+				psDevNode->ui32SyncCheckpointRecordCount));
+		OSFreeMem(psSyncRec);
+		psSyncRec = NULL;
+		eError = PVRSRV_ERROR_TOOMANYBUFFERS;
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	*phRecord = (PSYNC_CHECKPOINT_RECORD_HANDLE)psSyncRec;
+
+fail_alloc:
+	return eError;
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointRecordRemove(PSYNC_CHECKPOINT_RECORD_HANDLE hRecord)
+{
+	struct SYNC_CHECKPOINT_RECORD **ppFreedSync;
+	struct SYNC_CHECKPOINT_RECORD *pSync = (struct SYNC_CHECKPOINT_RECORD*)hRecord;
+	_SYNC_CHECKPOINT_CONTEXT *psContext = pSync->psSyncCheckpointBlock->psContext;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_RETURN_IF_INVALID_PARAM(hRecord);
+
+	psDevNode = psContext->psContextCtl->psDeviceNode;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+
+	dllist_remove_node(&pSync->sNode);
+
+	if (psDevNode->uiSyncCheckpointRecordFreeIdx >= PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: psDevNode->uiSyncCheckpointRecordFreeIdx out of range",
+				__func__));
+		psDevNode->uiSyncCheckpointRecordFreeIdx = 0;
+	}
+	ppFreedSync = &psDevNode->apsSyncCheckpointRecordsFreed[psDevNode->uiSyncCheckpointRecordFreeIdx];
+	psDevNode->uiSyncCheckpointRecordFreeIdx =
+			(psDevNode->uiSyncCheckpointRecordFreeIdx + 1) % PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN;
+
+	if (*ppFreedSync)
+	{
+		OSFreeMem(*ppFreedSync);
+	}
+	pSync->psSyncCheckpointBlock = NULL;
+	pSync->ui64OSTime = OSClockns64();
+	*ppFreedSync = pSync;
+
+	psDevNode->ui32SyncCheckpointRecordCount--;
+
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	return PVRSRV_OK;
+}
+
+#define NS_IN_S (1000000000UL)
+static void _SyncCheckpointRecordPrint(struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec,
+                                       IMG_UINT64 ui64TimeNow,
+                                       DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                       void *pvDumpDebugFile)
+{
+	SYNC_CHECKPOINT *psSyncCheckpoint = (SYNC_CHECKPOINT *)psSyncCheckpointRec->pSyncCheckpt;
+	SYNC_CHECKPOINT_BLOCK *psSyncCheckpointBlock = psSyncCheckpointRec->psSyncCheckpointBlock;
+	IMG_UINT64 ui64DeltaS;
+	IMG_UINT32 ui32DeltaF;
+	IMG_UINT64 ui64Delta = ui64TimeNow - psSyncCheckpointRec->ui64OSTime;
+	ui64DeltaS = OSDivide64(ui64Delta, NS_IN_S, &ui32DeltaF);
+
+	if (psSyncCheckpointBlock && psSyncCheckpointBlock->pui32LinAddr)
+	{
+		void *pSyncCheckpointAddr;
+		pSyncCheckpointAddr = IMG_OFFSET_ADDR(psSyncCheckpointBlock->pui32LinAddr,
+											psSyncCheckpointRec->ui32SyncOffset);
+
+		PVR_DUMPDEBUG_LOG("\t%05u %05" IMG_UINT64_FMTSPEC ".%09u %010u FWAddr=0x%08x (r%d:e%d:f%d) State=%s (%s)",
+		                  psSyncCheckpointRec->uiPID,
+		                  ui64DeltaS, ui32DeltaF, psSyncCheckpointRec->ui32UID,
+		                  (psSyncCheckpointRec->ui32FwBlockAddr+psSyncCheckpointRec->ui32SyncOffset),
+		                  OSAtomicRead(&psSyncCheckpoint->hRefCount),
+	                      OSAtomicRead(&psSyncCheckpoint->hEnqueuedCCBCount),
+	                      psSyncCheckpoint->psSyncCheckpointFwObj->ui32FwRefCount,
+		                  (*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_SIGNALLED) ?
+		                          "SIGNALLED" :
+		                          ((*(IMG_UINT32*)pSyncCheckpointAddr == PVRSRV_SYNC_CHECKPOINT_ERRORED) ?
+		                                  "ERRORED" : "ACTIVE"),
+		                                  psSyncCheckpointRec->szClassName);
+	}
+	else
+	{
+		PVR_DUMPDEBUG_LOG("\t%05u %05" IMG_UINT64_FMTSPEC ".%09u %010u FWAddr=0x%08x State=<null_ptr> (%s)",
+		                  psSyncCheckpointRec->uiPID,
+		                  ui64DeltaS, ui32DeltaF, psSyncCheckpointRec->ui32UID,
+		                  (psSyncCheckpointRec->ui32FwBlockAddr+psSyncCheckpointRec->ui32SyncOffset),
+		                  psSyncCheckpointRec->szClassName
+		);
+	}
+}
+
+static void _SyncCheckpointRecordRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+                                         IMG_UINT32 ui32VerbLevel,
+                                         DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                         void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	IMG_UINT64 ui64TimeNowS;
+	IMG_UINT32 ui32TimeNowF;
+	IMG_UINT64 ui64TimeNow = OSClockns64();
+	DLLIST_NODE *psNode, *psNext;
+
+	ui64TimeNowS = OSDivide64(ui64TimeNow, NS_IN_S, &ui32TimeNowF);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+
+		OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+
+		PVR_DUMPDEBUG_LOG("Dumping allocated sync checkpoints. Allocated: %u High watermark: %u (time ref %05" IMG_UINT64_FMTSPEC ".%09u)",
+		                  psDevNode->ui32SyncCheckpointRecordCount,
+		                  psDevNode->ui32SyncCheckpointRecordCountHighWatermark,
+		                  ui64TimeNowS,
+		                  ui32TimeNowF);
+		if (psDevNode->ui32SyncCheckpointRecordCountHighWatermark == SYNC_CHECKPOINT_RECORD_LIMIT)
+		{
+			PVR_DUMPDEBUG_LOG("Warning: Record limit (%u) was reached. Some sync checkpoints may not have been recorded in the debug information.",
+			                  SYNC_CHECKPOINT_RECORD_LIMIT);
+		}
+		PVR_DUMPDEBUG_LOG("\t%-5s %-15s %-10s %-17s %-14s (%s)",
+		                  "PID", "Time Delta (s)", "UID", "Address", "State", "Annotation");
+
+		dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+		{
+			struct SYNC_CHECKPOINT_RECORD *psSyncCheckpointRec =
+					IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+			_SyncCheckpointRecordPrint(psSyncCheckpointRec, ui64TimeNow,
+			                           pfnDumpDebugPrintf, pvDumpDebugFile);
+		}
+
+		PVR_DUMPDEBUG_LOG("Dumping all recently freed sync checkpoints @ %05" IMG_UINT64_FMTSPEC ".%09u",
+		                  ui64TimeNowS,
+		                  ui32TimeNowF);
+		PVR_DUMPDEBUG_LOG("\t%-5s %-15s %-10s %-17s %-14s (%s)",
+		                  "PID", "Time Delta (s)", "UID", "Address", "State", "Annotation");
+		for (i = DECREMENT_WITH_WRAP(psDevNode->uiSyncCheckpointRecordFreeIdx, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN);
+				i != psDevNode->uiSyncCheckpointRecordFreeIdx;
+				i = DECREMENT_WITH_WRAP(i, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN))
+		{
+			if (psDevNode->apsSyncCheckpointRecordsFreed[i])
+			{
+				_SyncCheckpointRecordPrint(psDevNode->apsSyncCheckpointRecordsFreed[i],
+				                           ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+			else
+			{
+				break;
+			}
+		}
+		OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+	}
+}
+#undef NS_IN_S
+static PVRSRV_ERROR _SyncCheckpointRecordListInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	eError = OSLockCreate(&psDevNode->hSyncCheckpointRecordLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lock_create);
+	dllist_init(&psDevNode->sSyncCheckpointRecordList);
+
+	psDevNode->ui32SyncCheckpointRecordCount = 0;
+	psDevNode->ui32SyncCheckpointRecordCountHighWatermark = 0;
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncCheckpointRecordNotify,
+	                                              psDevNode,
+	                                              _SyncCheckpointRecordRequest,
+	                                              DEBUG_REQUEST_SYNCCHECKPOINT,
+	                                              (PVRSRV_DBGREQ_HANDLE)psDevNode);
+	PVR_GOTO_IF_ERROR(eError, fail_dbg_register);
+
+	return PVRSRV_OK;
+
+fail_dbg_register:
+	OSLockDestroy(psDevNode->hSyncCheckpointRecordLock);
+fail_lock_create:
+	return eError;
+}
+
+static void _SyncCheckpointRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DLLIST_NODE *psNode, *psNext;
+	int i;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncCheckpointRecordList, psNode, psNext)
+	{
+		struct SYNC_CHECKPOINT_RECORD *pSyncCheckpointRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_CHECKPOINT_RECORD, sNode);
+
+		dllist_remove_node(psNode);
+		OSFreeMem(pSyncCheckpointRec);
+	}
+
+	for (i = 0; i < PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN; i++)
+	{
+		if (psDevNode->apsSyncCheckpointRecordsFreed[i])
+		{
+			OSFreeMem(psDevNode->apsSyncCheckpointRecordsFreed[i]);
+			psDevNode->apsSyncCheckpointRecordsFreed[i] = NULL;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncCheckpointRecordLock);
+
+	if (psDevNode->hSyncCheckpointRecordNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncCheckpointRecordNotify);
+	}
+	OSLockDestroy(psDevNode->hSyncCheckpointRecordLock);
+}
+
+#if defined(PDUMP)
+
+static PVRSRV_ERROR
+_SyncCheckpointAllocPDump(PVRSRV_DEVICE_NODE *psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+	                      "Allocated Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+	                      psSyncCheckpoint->azName,
+	                      psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+	                      psSyncCheckpoint->sCheckpointUFOAddr.ui32Addr);
+
+	DevmemPDumpLoadMemValue32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+	                          _SyncCheckpointGetOffset(psSyncCheckpoint),
+	                          PVRSRV_SYNC_CHECKPOINT_ACTIVE,
+	                          PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR
+_SyncCheckpointUpdatePDump(PPVRSRV_DEVICE_NODE psDevNode, SYNC_CHECKPOINT *psSyncCheckpoint, IMG_UINT32 ui32Status, IMG_UINT32 ui32FenceSyncFlags)
+{
+	IMG_BOOL bSleepAllowed = (ui32FenceSyncFlags & PVRSRV_FENCE_FLAG_CTX_ATOMIC) ? IMG_FALSE : IMG_TRUE;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	 */
+	if (bSleepAllowed)
+	{
+		if (ui32Status == PVRSRV_SYNC_CHECKPOINT_ERRORED)
+		{
+			PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+					"Errored Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+					psSyncCheckpoint->azName,
+					psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+					(psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+					_SyncCheckpointGetOffset(psSyncCheckpoint)));
+		}
+		else
+		{
+			PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+					"Signalled Sync Checkpoint %s (ID:%d, TL:%d, FirmwareVAddr = 0x%08x)",
+					psSyncCheckpoint->azName,
+					psSyncCheckpoint->ui32UID, psSyncCheckpoint->hTimeline,
+					(psSyncCheckpoint->psSyncCheckpointBlock->ui32FirmwareAddr +
+					_SyncCheckpointGetOffset(psSyncCheckpoint)));
+		}
+
+		DevmemPDumpLoadMemValue32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+		                          _SyncCheckpointGetOffset(psSyncCheckpoint),
+		                          ui32Status,
+		                          PDUMP_FLAGS_CONTINUOUS);
+	}
+	else
+	{
+		_SYNC_CHECKPOINT_DEFERRED_SIGNAL *psSyncData;
+		OS_SPINLOCK_FLAGS uiFlags;
+		IMG_UINT16 ui16NewWriteIdx;
+
+		OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+
+		ui16NewWriteIdx = GET_CP_CB_NEXT_IDX(psDevNode->ui16SyncCPWriteIdx);
+		if (ui16NewWriteIdx == psDevNode->ui16SyncCPReadIdx)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+				"%s: ERROR Deferred SyncCheckpointSignal CB is full)",
+				__func__));
+		}
+		else
+		{
+			psSyncData = GET_CP_CB_BASE(psDevNode->ui16SyncCPWriteIdx);
+			psSyncData->asSyncCheckpoint = *psSyncCheckpoint;
+			psSyncData->ui32Status = ui32Status;
+			psDevNode->ui16SyncCPWriteIdx = ui16NewWriteIdx;
+		}
+
+		OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+
+		OSScheduleMISR(psDevNode->pvSyncCPMISR);
+	}
+
+	return PVRSRV_OK;
+}
+
+static void
+MISRHandler_PdumpDeferredSyncSignalPoster(void *pvData)
+{
+	PPVRSRV_DEVICE_NODE psDevNode = (PPVRSRV_DEVICE_NODE) pvData;
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT16 ui16ReadIdx, ui16WriteIdx;
+	_SYNC_CHECKPOINT_DEFERRED_SIGNAL *psSyncData;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevInfo = psDevNode->pvDevice;
+
+	OSLockAcquire(psDevNode->hSyncCheckpointSignalLock);
+
+	OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+	/* Snapshot current write and read offset of CB */
+	ui16WriteIdx = psDevNode->ui16SyncCPWriteIdx;
+	ui16ReadIdx = psDevNode->ui16SyncCPReadIdx;
+
+	OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+	/* CB is empty */
+	if (ui16WriteIdx == ui16ReadIdx)
+	{
+		OSLockRelease(psDevNode->hSyncCheckpointSignalLock);
+		return;
+	}
+	do
+	{
+		/* Read item in the CB and flush it to pdump */
+		psSyncData = GET_CP_CB_BASE(ui16ReadIdx);
+		_SyncCheckpointUpdatePDump(psDevNode, &psSyncData->asSyncCheckpoint, psSyncData->ui32Status, PVRSRV_FENCE_FLAG_NONE);
+		ui16ReadIdx = GET_CP_CB_NEXT_IDX(psDevNode->ui16SyncCPReadIdx);
+		/* Increment read offset in CB as one item is flushed to pdump */
+		OSSpinLockAcquire(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+		psDevNode->ui16SyncCPReadIdx = ui16ReadIdx;
+		OSSpinLockRelease(psDevInfo->hSyncCheckpointSignalSpinLock, uiFlags);
+		/* Call to this function will flush all the items present in CB
+		 * when this function is called i.e. use snapshot of WriteOffset
+		 * taken at the beginning in this function and iterate till Write != Read */
+	} while (ui16WriteIdx != ui16ReadIdx);
+
+	OSLockRelease(psDevNode->hSyncCheckpointSignalLock);
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR PVRSRVSyncCheckpointSignalledPDumpPolKM(PVRSRV_FENCE hFence)
+{
+	PVRSRV_ERROR eError;
+	PSYNC_CHECKPOINT *apsCheckpoints = NULL;
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+	IMG_UINT32 i, uiNumCheckpoints = 0;
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+
+	if (hFence != PVRSRV_NO_FENCE)
+	{
+		eError = g_psSyncCheckpointPfnStruct->pfnSyncFenceGetCheckpoints(hFence, &uiNumCheckpoints, &apsCheckpoints);
+	}
+	else
+	{
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "g_pfnFenceGetCheckpoints");
+
+	if (uiNumCheckpoints)
+	{
+		/* Flushing deferred fence signals to pdump */
+		psSyncCheckpoint = (SYNC_CHECKPOINT *)apsCheckpoints[0];
+		psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+		MISRHandler_PdumpDeferredSyncSignalPoster(psContext->psContextCtl->psDeviceNode);
+	}
+
+	for (i=0; i < uiNumCheckpoints; i++)
+	{
+		psSyncCheckpoint = (SYNC_CHECKPOINT *)apsCheckpoints[i];
+		if (psSyncCheckpoint->psSyncCheckpointFwObj->ui32State == PVRSRV_SYNC_CHECKPOINT_SIGNALLED)
+		{
+			psContext = (_SYNC_CHECKPOINT_CONTEXT*)psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+			PDUMPCOMMENTWITHFLAGS(psContext->psContextCtl->psDeviceNode,
+							psSyncCheckpoint->ui32PDumpFlags,
+							"Wait for Fence %s (ID:%d)",
+							psSyncCheckpoint->azName,
+							psSyncCheckpoint->ui32UID);
+
+			eError = DevmemPDumpDevmemPol32(psSyncCheckpoint->psSyncCheckpointBlock->hMemDesc,
+								_SyncCheckpointGetOffset(psSyncCheckpoint),
+								PVRSRV_SYNC_CHECKPOINT_SIGNALLED,
+								0xFFFFFFFF,
+								PDUMP_POLL_OPERATOR_EQUAL,
+								psSyncCheckpoint->ui32PDumpFlags);
+			PVR_LOG_IF_ERROR(eError, "DevmemPDumpDevmemPol32");
+		}
+	}
+
+
+
+	/* Free the memory that was allocated for the sync checkpoint list returned */
+	if (apsCheckpoints)
+	{
+		SyncCheckpointFreeCheckpointListMem(apsCheckpoints);
+	}
+
+	return PVRSRV_OK;
+}
+#endif /* #if defined(PDUMP) */
+
+static PVRSRV_ERROR
+_SyncCheckpointPDumpTransition(void *pvData, PDUMP_TRANSITION_EVENT eEvent)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext;
+	DLLIST_NODE *psNode, *psNext;
+	DLLIST_NODE *psNode1, *psNext1;
+	PPVRSRV_DEVICE_NODE psDevNode = (PPVRSRV_DEVICE_NODE) pvData;
+
+	if ((eEvent == PDUMP_TRANSITION_EVENT_RANGE_ENTERED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		OSLockAcquire(psDevNode->hSyncCheckpointContextListLock);
+		dllist_foreach_node(&psDevNode->sSyncCheckpointContextListHead, psNode, psNext)
+		{
+			psContext = IMG_CONTAINER_OF(psNode, _SYNC_CHECKPOINT_CONTEXT, sListNode);
+
+			OSLockAcquire(psContext->hSyncCheckpointBlockListLock);
+			dllist_foreach_node(&psContext->sSyncCheckpointBlockListHead, psNode1, psNext1)
+			{
+				SYNC_CHECKPOINT_BLOCK *psSyncBlk =
+					IMG_CONTAINER_OF(psNode1, SYNC_CHECKPOINT_BLOCK, sListNode);
+				DevmemPDumpLoadMem(psSyncBlk->hMemDesc,
+							   0,
+							   psSyncBlk->ui32SyncBlockSize,
+							   PDUMP_FLAGS_CONTINUOUS);
+			}
+			OSLockRelease(psContext->hSyncCheckpointBlockListLock);
+		}
+		OSLockRelease(psDevNode->hSyncCheckpointContextListLock);
+	}
+
+	return PVRSRV_OK;
+}
+#endif
+
+static void _CheckDeferredCleanupList(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE*)psCtxCtl->psDeviceNode;
+	DECLARE_DLLIST(sCleanupList);
+	DLLIST_NODE *psNode, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+	PVRSRV_ERROR eError;
+
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s called", __func__));
+#endif
+
+	/* Check the deferred cleanup list and free any sync checkpoints we can */
+	OSSpinLockAcquire(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+
+	if (dllist_is_empty(&psCtxCtl->sDeferredCleanupListHead))
+	{
+		OSSpinLockRelease(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING, "%s: Defer free list is empty", __func__));
+#endif
+		/* if list is empty then we have nothing to do here */
+		return;
+	}
+
+	dllist_foreach_node(&psCtxCtl->sDeferredCleanupListHead, psNode, psNext)
+	{
+		SYNC_CHECKPOINT *psSyncCheckpointInt =
+				IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sDeferredFreeListNode);
+
+		if (psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount ==
+				(IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)))
+		{
+			if ((GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+				&& psSyncCheckpointInt->hRecord)
+			{
+				/* remove this sync record */
+				eError = _SyncCheckpointRecordRemove(psSyncCheckpointInt->hRecord);
+				PVR_LOG_IF_ERROR(eError, "_SyncCheckpointRecordRemove");
+			}
+
+			/* Move the sync checkpoint from the deferred free list to local list */
+			dllist_remove_node(&psSyncCheckpointInt->sDeferredFreeListNode);
+			/* It's not an ideal solution to traverse list of checkpoints-to-free
+			 * twice but it allows us to avoid holding the lock for too long */
+			dllist_add_to_tail(&sCleanupList, &psSyncCheckpointInt->sDeferredFreeListNode);
+		}
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		else
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s psSyncCheckpoint '%s'' (ID:%d)<%p>), "
+			        "still pending (enq=%d,FWRef=%d)", __func__,
+			        psSyncCheckpointInt->azName, psSyncCheckpointInt->ui32UID,
+			        (void*)psSyncCheckpointInt,
+			        (IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)),
+			        psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+		}
+#endif
+	}
+
+	OSSpinLockRelease(psCtxCtl->hDeferredCleanupListLock, uiFlags);
+
+	dllist_foreach_node(&sCleanupList, psNode, psNext) {
+		SYNC_CHECKPOINT *psSyncCheckpointInt =
+				IMG_CONTAINER_OF(psNode, SYNC_CHECKPOINT, sDeferredFreeListNode);
+
+		/* Remove the sync checkpoint from the global list */
+		OSSpinLockAcquire(psDevNode->hSyncCheckpointListLock, uiFlags);
+		dllist_remove_node(&psSyncCheckpointInt->sListNode);
+		OSSpinLockRelease(psDevNode->hSyncCheckpointListLock, uiFlags);
+
+		RGXSRV_HWPERF_FREE(psDevNode, SYNC_CP, psSyncCheckpointInt->ui32FWAddr);
+
+		/* Unref the checkpoint in use */
+		OSSpinLockAcquire(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+		psContext->psContextCtl->ui32CurrentInUseSyncCheckpoints--;
+		OSSpinLockRelease(psContext->psContextCtl->hSyncCheckpointStatsLock, uiFlags);
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s attempting to return sync(ID:%d),%p> to pool",
+		        __func__,
+		        psSyncCheckpointInt->ui32UID,
+		        (void *) psSyncCheckpointInt));
+#endif
+		if (!_PutCheckpointInPool(psSyncCheckpointInt))
+#endif
+		{
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+			PVR_DPF((PVR_DBG_WARNING, "%s pool is full, so just free it",
+			        __func__));
+#endif
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_DEFERRED_CLEANUP_DEBUG == 1)
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+		}
+		else
+		{
+#endif
+			PVR_DPF((PVR_DBG_WARNING,
+			       "%s psSyncCheckpoint '%s'' (ID:%d)<%p>), still pending (enq=%d,FWRef=%d)",
+			        __func__,
+			        psSyncCheckpointInt->azName,
+			        psSyncCheckpointInt->ui32UID,
+			        (void*)psSyncCheckpointInt,
+			        (IMG_UINT32)(OSAtomicRead(&psSyncCheckpointInt->hEnqueuedCCBCount)),
+			        psSyncCheckpointInt->psSyncCheckpointFwObj->ui32FwRefCount));
+#endif
+			_FreeSyncCheckpoint(psSyncCheckpointInt);
+		}
+	}
+}
+
+#if (SYNC_CHECKPOINT_POOL_LIMIT > 0)
+static SYNC_CHECKPOINT *_GetCheckpointFromPool(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	SYNC_CHECKPOINT *psSyncCheckpoint = NULL;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psCtxCtl->ui32SyncCheckpointPoolSize == 0)
+	{
+		goto pool_not_used;
+	}
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* Check if we can allocate from the pool */
+	if (CHECKPOINT_POOL_VALID(psCtxCtl) &&
+#if defined(PDUMP)
+	    psCtxCtl->bAllocateFromCheckpointPool &&
+#endif
+	    (psCtxCtl->ui32SyncCheckpointPoolCount > 0))
+	{
+		/* Get the next sync checkpoint from the pool */
+		psSyncCheckpoint = psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolRp];
+		psCtxCtl->ui32SyncCheckpointPoolRp =
+		        (psCtxCtl->ui32SyncCheckpointPoolRp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount--;
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+#if defined(DEBUG)
+		psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_USE;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s checkpoint(old ID:%d)<-POOL(%d/%d), psContext=<%p>, "
+		        "poolRp=%d, poolWp=%d",
+		        __func__,
+		        psSyncCheckpoint->ui32UID,
+		        psCtxCtl->ui32SyncCheckpointPoolCount,
+		         psCtxCtl->ui32SyncCheckpointPoolSize,
+		        (void *) psContext,
+		        psCtxCtl->ui32SyncCheckpointPoolRp,
+		        psCtxCtl->ui32SyncCheckpointPoolWp));
+#endif
+	}
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+pool_not_used:
+	return psSyncCheckpoint;
+}
+
+static IMG_BOOL _PutCheckpointInPool(SYNC_CHECKPOINT *psSyncCheckpoint)
+{
+	_SYNC_CHECKPOINT_CONTEXT *psContext = psSyncCheckpoint->psSyncCheckpointBlock->psContext;
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	IMG_BOOL bReturnedToPool = IMG_FALSE;
+	OS_SPINLOCK_FLAGS uiFlags;
+
+	if (psCtxCtl->ui32SyncCheckpointPoolSize == 0)
+	{
+		return IMG_FALSE;
+	}
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* Check if pool has space */
+	if (CHECKPOINT_POOL_VALID(psCtxCtl) &&
+#if defined(PDUMP)
+			(psSyncCheckpoint->psSyncCheckpointBlock == psCtxCtl->psSyncCheckpointPoolBlock) &&
+#endif
+			!(CHECKPOINT_POOL_FULL(psCtxCtl)))
+	{
+		/* Put the sync checkpoint into the next write slot in the pool */
+		psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolWp] = psSyncCheckpoint;
+		psCtxCtl->ui32SyncCheckpointPoolWp =
+		        (psCtxCtl->ui32SyncCheckpointPoolWp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount++;
+		/* Update if the checkpoint that was just added filled up the pool */
+		if (psCtxCtl->ui32SyncCheckpointPoolWp == psCtxCtl->ui32SyncCheckpointPoolRp)
+		{
+			SET_CHECKPOINT_POOL_FULL(psCtxCtl);
+#if defined(PDUMP)
+			/* The pool is now fully allocated and ready to use. */
+			psCtxCtl->bAllocateFromCheckpointPool = IMG_TRUE;
+#endif
+		}
+		bReturnedToPool = IMG_TRUE;
+		psSyncCheckpoint->psSyncCheckpointFwObj->ui32State = PVRSRV_SYNC_CHECKPOINT_UNDEF;
+#if defined(DEBUG)
+		psSyncCheckpoint->ui32ValidationCheck = SYNC_CHECKPOINT_PATTERN_IN_POOL;
+#endif
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s checkpoint(ID:%d)->POOL(%d/%d), poolRp=%d, poolWp=%d",
+		        __func__,
+		        psSyncCheckpoint->ui32UID,
+		        psCtxCtl->ui32SyncCheckpointPoolCount,
+		        psCtxCtl->ui32SyncCheckpointPoolSize,
+		        psCtxCtl->ui32SyncCheckpointPoolRp,
+		        psCtxCtl->ui32SyncCheckpointPoolWp));
+#endif
+	}
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	return bReturnedToPool;
+}
+
+static IMG_UINT32 _CleanCheckpointPool(_SYNC_CHECKPOINT_CONTEXT *psContext)
+{
+	_SYNC_CHECKPOINT_CONTEXT_CTL *const psCtxCtl = psContext->psContextCtl;
+	SYNC_CHECKPOINT *psCheckpoint = NULL;
+	DECLARE_DLLIST(sCleanupList);
+	DLLIST_NODE *psThis, *psNext;
+	OS_SPINLOCK_FLAGS uiFlags;
+	IMG_UINT32 ui32ItemsFreed = 0, ui32NullScpCount = 0;
+
+	/* Acquire sync checkpoint pool lock */
+	OSSpinLockAcquire(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* While the pool still contains sync checkpoints, free them */
+	while (CHECKPOINT_POOL_VALID(psCtxCtl) && psCtxCtl->ui32SyncCheckpointPoolCount > 0)
+	{
+		/* Get the sync checkpoint from the next read slot in the pool */
+		psCheckpoint = psCtxCtl->apsSyncCheckpointPool[psCtxCtl->ui32SyncCheckpointPoolRp];
+		psCtxCtl->ui32SyncCheckpointPoolRp =
+		        (psCtxCtl->ui32SyncCheckpointPoolRp + 1) & (psCtxCtl->ui32SyncCheckpointPoolSize-1);
+		psCtxCtl->ui32SyncCheckpointPoolCount--;
+
+		CLEAR_CHECKPOINT_POOL_FULL(psCtxCtl);
+
+		if (psCheckpoint)
+		{
+			PVR_ASSERT(!dllist_node_is_in_list(&psCheckpoint->sListNode));
+			/* before checkpoints are added to the pool they are removed
+			 * from the list so it's safe to use sListNode here */
+			dllist_add_to_head(&sCleanupList, &psCheckpoint->sListNode);
+		}
+		else
+		{
+			ui32NullScpCount++;
+		}
+	}
+
+	/* Release sync checkpoint pool lock */
+	OSSpinLockRelease(psCtxCtl->hSyncCheckpointPoolLock, uiFlags);
+
+	/* go through the local list and free all of the sync checkpoints */
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+	PVR_DPF((PVR_DBG_WARNING, "%s psContext=<%p>, ui8PoolStateFlags=0x%x, "
+	        "uiSyncCheckpointPoolCount=%d", __func__, (void *) psContext,
+	        psCtxCtl->ui8PoolStateFlags, psCtxCtl->ui32SyncCheckpointPoolCount));
+
+	if (ui32NullScpCount > 0)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "%s pool contained %u NULL entries", __func__,
+		        ui32NullScpCount));
+	}
+#endif
+
+	dllist_foreach_node(&sCleanupList, psThis, psNext)
+	{
+		psCheckpoint = IMG_CONTAINER_OF(psThis, SYNC_CHECKPOINT, sListNode);
+
+#if (ENABLE_SYNC_CHECKPOINT_POOL_DEBUG == 1)
+#if defined(DEBUG)
+		if (psCheckpoint->ui32ValidationCheck != SYNC_CHECKPOINT_PATTERN_IN_POOL)
+		{
+			PVR_DPF((PVR_DBG_WARNING, "%s pool contains invalid entry "
+			        "(ui32ValidationCheck=0x%x)", __func__,
+			        psCheckpoint->ui32ValidationCheck));
+		}
+#endif
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint(ID:%d)",
+		        __func__, psCheckpoint->ui32UID));
+#if defined(DEBUG)
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->ui32ValidationCheck=0x%x",
+		        __func__, psCheckpoint->ui32ValidationCheck));
+#endif
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->uiAllocatedAddr=0x%llx",
+		        __func__, psCheckpoint->uiAllocatedAddr));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock->psContext=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock->psContext));
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s psSyncCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA=<%p>",
+		        __func__, (void *) psCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA));
+
+		PVR_DPF((PVR_DBG_WARNING,
+		        "%s CALLING RA_Free(psSyncCheckpoint(ID:%d)<%p>), "
+		        "psSubAllocRA=<%p>, ui32SpanAddr=0x%llx",
+		        __func__,
+		        psCheckpoint->ui32UID,
+		        (void *) psCheckpoint,
+		        (void *) psCheckpoint->psSyncCheckpointBlock->psContext->psSubAllocRA,
+		        psCheckpoint->uiAllocatedAddr));
+#endif
+
+		dllist_remove_node(psThis);
+
+		_FreeSyncCheckpoint(psCheckpoint);
+		ui32ItemsFreed++;
+	}
+
+	return ui32ItemsFreed;
+}
+#endif /* (SYNC_CHECKPOINT_POOL_LIMIT > 0) */
diff --git a/drivers/gpu/drm/img-rogue/sync_checkpoint.h b/drivers/gpu/drm/img-rogue/sync_checkpoint.h
new file mode 100644
index 000000000000..d1f53751dd40
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_checkpoint.h
@@ -0,0 +1,740 @@
+/*************************************************************************/ /*!
+@File
+@Title          Synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the client side interface for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_H
+#define SYNC_CHECKPOINT_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_sync_km.h"
+#include "pdumpdefs.h"
+#include "pdump.h"
+#include "dllist.h"
+#include "pvr_debug.h"
+#include "device_connection.h"
+#include "opaque_types.h"
+
+#ifndef CHECKPOINT_TYPES
+#define CHECKPOINT_TYPES
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG *PSYNC_CHECKPOINT_CONTEXT;
+
+typedef struct SYNC_CHECKPOINT_TAG *PSYNC_CHECKPOINT;
+#endif
+
+/* definitions for functions to be implemented by OS-specific sync - the OS-specific sync code
+   will call SyncCheckpointRegisterFunctions() when initialised, in order to register functions
+   we can then call */
+#ifndef CHECKPOINT_PFNS
+#define CHECKPOINT_PFNS
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN)(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                                             PVRSRV_FENCE fence,
+                                                             IMG_UINT32 *nr_checkpoints,
+                                                             PSYNC_CHECKPOINT **checkpoint_handles,
+                                                             IMG_UINT64 *pui64FenceUID);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN)(PPVRSRV_DEVICE_NODE device,
+                                                            const IMG_CHAR *fence_name,
+                                                            PVRSRV_TIMELINE timeline,
+                                                            PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                                                            PVRSRV_FENCE *new_fence,
+                                                            IMG_UINT64 *pui64FenceUID,
+                                                            void **ppvFenceFinaliseData,
+                                                            PSYNC_CHECKPOINT *new_checkpoint_handle,
+                                                            IMG_HANDLE *timeline_update_sync,
+                                                            IMG_UINT32 *timeline_update_value);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN)(PVRSRV_FENCE fence_to_rollback, void *finalise_data);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN)(PVRSRV_FENCE fence_to_finalise, void *finalise_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN)(void *private_data);
+typedef void (*PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN)(PVRSRV_FENCE fence_to_signal);
+typedef void (*PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN)(void *mem_ptr);
+typedef IMG_UINT32 (*PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN)(IMG_UINT32 num_ufos, IMG_UINT32 *vaddrs);
+#if defined(PDUMP)
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN)(PVRSRV_FENCE iFence,
+									IMG_UINT32 *puiNumCheckpoints,
+									PSYNC_CHECKPOINT **papsCheckpoints);
+#endif
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN)(PVRSRV_FENCE iExportFence,
+                                                             PSYNC_CHECKPOINT_CONTEXT checkpoint_context,
+                                                             PSYNC_CHECKPOINT *checkpoint_handle);
+typedef PVRSRV_ERROR (*PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN)(PVRSRV_FENCE iExportFence);
+
+#define SYNC_CHECKPOINT_IMPL_MAX_STRLEN 20
+
+typedef struct
+{
+	PFN_SYNC_CHECKPOINT_FENCE_RESOLVE_FN pfnFenceResolve;
+	PFN_SYNC_CHECKPOINT_FENCE_CREATE_FN pfnFenceCreate;
+	PFN_SYNC_CHECKPOINT_FENCE_ROLLBACK_DATA_FN pfnFenceDataRollback;
+	PFN_SYNC_CHECKPOINT_FENCE_FINALISE_FN pfnFenceFinalise;
+	PFN_SYNC_CHECKPOINT_NOHW_UPDATE_TIMELINES_FN pfnNoHWUpdateTimelines;
+	PFN_SYNC_CHECKPOINT_NOHW_SIGNAL_EXPORT_FENCE_FN pfnNoHWSignalExpFence;
+	PFN_SYNC_CHECKPOINT_FREE_CHECKPOINT_LIST_MEM_FN pfnFreeCheckpointListMem;
+	PFN_SYNC_CHECKPOINT_DUMP_INFO_ON_STALLED_UFOS_FN pfnDumpInfoOnStalledUFOs;
+	IMG_CHAR pszImplName[SYNC_CHECKPOINT_IMPL_MAX_STRLEN];
+#if defined(PDUMP)
+	PFN_SYNC_CHECKPOINT_FENCE_GETCHECKPOINTS_FN pfnSyncFenceGetCheckpoints;
+#endif
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_RESOLVE_FN pfnExportFenceResolve;
+	PFN_SYNC_CHECKPOINT_EXPORT_FENCE_ROLLBACK_FN pfnExportFenceRollback;
+} PFN_SYNC_CHECKPOINT_STRUCT;
+
+PVRSRV_ERROR SyncCheckpointRegisterFunctions(PFN_SYNC_CHECKPOINT_STRUCT *psSyncCheckpointPfns);
+
+#endif /* ifndef CHECKPOINT_PFNS */
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextCreate
+
+@Description    Create a new synchronisation checkpoint context
+
+@Input          psDevNode                 Device node
+
+@Output         ppsSyncCheckpointContext  Handle to the created synchronisation
+                                          checkpoint context
+
+@Return         PVRSRV_OK if the synchronisation checkpoint context was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointContextCreate(PPVRSRV_DEVICE_NODE psDevNode,
+                            PSYNC_CHECKPOINT_CONTEXT *ppsSyncCheckpointContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextDestroy
+
+@Description    Destroy a synchronisation checkpoint context
+
+@Input          psSyncCheckpointContext  Handle to the synchronisation
+                                         checkpoint context to destroy
+
+@Return         PVRSRV_OK if the synchronisation checkpoint context was
+                successfully destroyed.
+                PVRSRV_ERROR_UNABLE_TO_DESTROY_CONTEXT if the context still
+                has sync checkpoints defined
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointContextDestroy(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextRef
+
+@Description    Takes a reference on a synchronisation checkpoint context
+
+@Input          psContext  Handle to the synchronisation checkpoint context
+                           on which a ref is to be taken
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointContextRef(PSYNC_CHECKPOINT_CONTEXT psContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointContextUnref
+
+@Description    Drops a reference taken on a synchronisation checkpoint
+                context
+
+@Input          psContext  Handle to the synchronisation checkpoint context
+                           on which the ref is to be dropped
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointContextUnref(PSYNC_CHECKPOINT_CONTEXT psContext);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointAlloc
+
+@Description    Allocate a new synchronisation checkpoint on the specified
+                synchronisation checkpoint context
+
+@Input          hSyncCheckpointContext  Handle to the synchronisation
+                                        checkpoint context
+
+@Input          hTimeline               Timeline on which this sync
+                                        checkpoint is being created
+
+@Input          hFence                  Fence as passed into pfnFenceResolve
+                                        API, when the API encounters a non-PVR
+                                        fence as part of its input fence. From
+                                        all other places this argument must be
+                                        PVRSRV_NO_FENCE.
+
+@Input          pszClassName            Sync checkpoint source annotation
+                                        (will be truncated to at most
+                                         PVRSRV_SYNC_NAME_LENGTH chars)
+
+@Output         ppsSyncCheckpoint       Created synchronisation checkpoint
+
+@Return         PVRSRV_OK if the synchronisation checkpoint was
+                successfully created
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointAlloc(PSYNC_CHECKPOINT_CONTEXT psSyncContext,
+                    PVRSRV_TIMELINE hTimeline,
+                    PVRSRV_FENCE hFence,
+                    const IMG_CHAR *pszCheckpointName,
+                    PSYNC_CHECKPOINT *ppsSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFree
+
+@Description    Free a synchronisation checkpoint
+                The reference count held for the synchronisation checkpoint
+                is decremented - if it has becomes zero, it is also freed.
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to free
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointFree(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointSignal
+
+@Description    Signal the synchronisation checkpoint
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to signal
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointSignal(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointSignalNoHW
+
+@Description    Signal the synchronisation checkpoint in NO_HARDWARE build
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to signal
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointSignalNoHW(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointError
+
+@Description    Error the synchronisation checkpoint
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to error
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointError(PSYNC_CHECKPOINT psSyncCheckpoint, IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointStateFromUFO
+
+@Description    Returns the current state of the synchronisation checkpoint
+                which has the given UFO firmware address
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint
+
+@Return         The current state (32-bit value) of the sync checkpoint
+*/
+/*****************************************************************************/
+IMG_UINT32 SyncCheckpointStateFromUFO(PPVRSRV_DEVICE_NODE psDevNode,
+                                IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointErrorFromUFO
+
+@Description    Error the synchronisation checkpoint which has the
+                given UFO firmware address
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to be errored
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to be errored
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointErrorFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackFromUFO
+
+@Description    Drop the enqueued count reference taken on the synchronisation
+                checkpoint on behalf of the firmware.
+                Called in the event of a DM Kick failing.
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to be rolled back
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to be rolled back
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointRollbackFromUFO(PPVRSRV_DEVICE_NODE psDevNode, IMG_UINT32 ui32FwAddr);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointIsSignalled
+
+@Description    Returns IMG_TRUE if the synchronisation checkpoint is
+                signalled or errored
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to test
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_BOOL
+SyncCheckpointIsSignalled(PSYNC_CHECKPOINT psSyncCheckpoint,
+                          IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointIsErrored
+
+@Description    Returns IMG_TRUE if the synchronisation checkpoint is
+                errored
+
+@Input          psSyncCheckpoint        The synchronisation checkpoint to test
+
+@Input          ui32FenceSyncFlags      Flags used for controlling HWPerf behavior
+
+@Return         None
+*/
+/*****************************************************************************/
+IMG_BOOL
+SyncCheckpointIsErrored(PSYNC_CHECKPOINT psSyncCheckpoint,
+                        IMG_UINT32 ui32FenceSyncFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointTakeRef
+
+@Description    Take a reference on a synchronisation checkpoint
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to take a
+                                        reference on
+
+@Return         PVRSRV_OK if a reference was taken on the synchronisation
+                primitive
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointTakeRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDropRef
+
+@Description    Drop a reference on a synchronisation checkpoint
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to drop a
+                                        reference on
+
+@Return         PVRSRV_OK if a reference was dropped on the synchronisation
+                primitive
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointDropRef(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointResolveFence
+
+@Description    Resolve a fence, returning a list of the sync checkpoints
+                that fence contains.
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          psSyncCheckpointContext The sync checkpoint context
+                                        on which checkpoints should be
+                                        created (in the event of the fence
+                                        having a native sync pt with no
+                                        associated sync checkpoint)
+
+@Input          hFence                  The fence to be resolved
+
+@Output         pui32NumSyncCheckpoints The number of sync checkpoints the
+                                        fence contains. Can return 0 if
+                                        passed a null (-1) fence.
+
+@Output         papsSyncCheckpoints     List of sync checkpoints the fence
+                                        contains
+
+@Output         puiFenceUID             Unique ID of the resolved fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointResolveFence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PVRSRV_FENCE hFence,
+                           IMG_UINT32 *pui32NumSyncCheckpoints,
+                           PSYNC_CHECKPOINT **papsSyncCheckpoints,
+                           IMG_UINT64 *puiFenceUID,
+                           PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointResolveExportFence
+
+@Description    Resolve an export fence, returning the sync checkpoint
+                that fence contains.
+                This function in turn calls a function provided by the
+                sync implementation.
+
+@Input          hExportFence            The export fence to be resolved
+
+@Input          psSyncCheckpointContext The context in which to create the
+                                        new sync checkpoint for the export fence
+
+@Output         ppsSyncCheckpoint       The sync checkpoint the fence
+                                        contains
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointResolveExportFence(PVRSRV_FENCE hExportFence,
+                           PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                           PSYNC_CHECKPOINT *ppsSyncCheckpoint,
+                           PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackExportFence
+
+@Description    Rollback an export fence, freeing the sync checkpoint
+                that fence was assigned.
+                This function in turn calls a function provided by the
+                sync implementation.
+
+@Input          hExportFence            The export fence to be rolled back
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointRollbackExportFence(PVRSRV_FENCE hExportFence);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCreateFence
+
+@Description    Create a fence containing a single sync checkpoint.
+                Return the fence and a ptr to sync checkpoint it contains.
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          pszFenceName            String to assign to the new fence
+                                        (for debugging purposes)
+
+@Input          hTimeline               Timeline on which the new fence is
+                                        to be created
+
+@Input          psSyncCheckpointContext Sync checkpoint context to be used
+                                        when creating the new fence
+
+@Output         phNewFence              The newly created fence
+
+@Output         pui64FenceUID           Unique ID of the created fence
+
+@Output         ppvFenceFinaliseData    Any data needed to finalise the fence
+                                        in a later call to the function
+                                        SyncCheckpointFinaliseFence()
+
+@Output         psNewSyncCheckpoint     The sync checkpoint contained in
+                                        the new fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointCreateFence(PPVRSRV_DEVICE_NODE psDeviceNode,
+                          const IMG_CHAR *pszFenceName,
+                          PVRSRV_TIMELINE hTimeline,
+                          PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
+                          PVRSRV_FENCE *phNewFence,
+                          IMG_UINT64 *pui64FenceUID,
+                          void **ppvFenceFinaliseData,
+                          PSYNC_CHECKPOINT *psNewSyncCheckpoint,
+                          void **ppvTimelineUpdateSyncPrim,
+                          IMG_UINT32 *pui32TimelineUpdateValue,
+                          PDUMP_FLAGS_T ui32PDumpFlags);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRollbackFenceData
+
+@Description    'Rolls back' the fence specified (destroys the fence and
+                takes any other required actions to undo the fence
+                creation (eg if the implementation wishes to revert the
+                incrementing of the fence's timeline, etc).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          hFence                  Fence to be 'rolled back'
+
+@Input          pvFinaliseData          Data needed to finalise the
+                                        fence
+
+@Return         PVRSRV_OK if a valid fence was provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointRollbackFenceData(PVRSRV_FENCE hFence, void *pvFinaliseData);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFinaliseFence
+
+@Description    'Finalise' the fence specified (performs any actions the
+                underlying implementation may need to perform just prior
+                to the fence being returned to the client.
+                This function in turn calls a function provided by the
+                OS native sync implementation - if the native sync
+                implementation does not need to perform any actions at
+                this time, this function does not need to be registered.
+
+@Input          psDevNode               Device node
+
+@Input          hFence                  Fence to be 'finalised'
+
+@Input          pvFinaliseData          Data needed to finalise the fence
+
+@Input          psSyncCheckpoint        Base sync checkpoint that this fence
+                                        is formed of
+
+@Input          pszName                 Fence annotation
+
+@Return         PVRSRV_OK if a valid fence and finalise data were provided.
+                PVRSRV_ERROR_INVALID_PARAMS if an invalid fence or finalise
+                data were provided.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function (permitted).
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointFinaliseFence(PPVRSRV_DEVICE_NODE psDevNode,
+                            PVRSRV_FENCE hFence,
+                            void *pvFinaliseData,
+                            PSYNC_CHECKPOINT psSyncCheckpoint,
+                            const IMG_CHAR *pszName);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointFreeCheckpointListMem
+
+@Description    Free memory the memory which was allocated by the sync
+                implementation and used to return the list of sync
+                checkpoints when resolving a fence.
+                to the fence being returned to the client.
+                This function in turn calls a free function registered by
+                the sync implementation (if a function has been registered).
+
+@Input          pvCheckpointListMem     Pointer to the memory to be freed
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointFreeCheckpointListMem(void *pvCheckpointListMem);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointNoHWUpdateTimelines
+
+@Description    Called by the DDK in a NO_HARDWARE build only.
+                After syncs have been manually signalled by the DDK, this
+                function is called to allow the OS native sync implementation
+                to update its timelines (as the usual callback notification
+                of signalled checkpoints is not supported for NO_HARDWARE).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          pvPrivateData            Any data the OS native sync
+                                         implementation might require.
+
+@Return         PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function, otherwise
+                PVRSRV_OK.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointNoHWUpdateTimelines(void *pvPrivateData);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointNoHWSignalExportFence
+
+@Description    Called by the DDK in a NO_HARDWARE build only.
+                After syncs have been manually signalled by the DDK, this
+                function is called to allow the OS native sync implementation
+                to signal the export fence (as the usual callback notification
+                of signalled checkpoints is not supported for NO_HARDWARE).
+                This function in turn calls a function provided by the
+                OS native sync implementation.
+
+@Input          iExportFenceToSignal     The export fence to signal.
+
+@Return         PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function, otherwise
+                PVRSRV_OK.
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointNoHWSignalExportFence(PVRSRV_FENCE iExportFenceToSignal);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDumpInfoOnStalledUFOs
+
+@Description    Called by the DDK in the event of the health check watchdog
+                examining the CCBs and determining that one has failed to
+                progress after 10 second when the GPU is idle due to waiting
+                on one or more UFO fences.
+                The DDK will pass a list of UFOs on which the CCB is waiting
+                and the sync implementation will check them to see if any
+                relate to sync points it has created. If so, the
+                implementation should dump debug information on those sync
+                points to the kernel log or other suitable output (which will
+                allow the unsignalled syncs to be identified).
+                The function shall return the number of syncs in the provided
+                array that were syncs which it had created.
+
+@Input          ui32NumUFOs           The number of UFOs in the array passed
+                                      in the pui32VAddrs parameter.
+                pui32Vaddr            The array of UFOs the CCB is waiting on.
+
+@Output         pui32NumSyncOwnedUFOs The number of UFOs in pui32Vaddr which
+                                      relate to syncs created by the sync
+                                      implementation.
+
+@Return         PVRSRV_OK if a valid pointer is provided in pui32NumSyncOwnedUFOs.
+                PVRSRV_ERROR_INVALID_PARAMS if a NULL value is provided in
+                pui32NumSyncOwnedUFOs.
+                PVRSRV_ERROR_SYNC_NATIVESYNC_NOT_REGISTERED if the OS native
+                sync has not registered a callback function.
+
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointDumpInfoOnStalledUFOs(IMG_UINT32 ui32NumUFOs,
+                                    IMG_UINT32 *pui32Vaddrs,
+                                    IMG_UINT32 *pui32NumSyncOwnedUFOs);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetStateString
+
+@Description    Called to get a string representing the current state of a
+                sync checkpoint.
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get the
+                                        state for.
+
+@Return         The string representing the current state of this checkpoint
+*/
+/*****************************************************************************/
+const IMG_CHAR *
+SyncCheckpointGetStateString(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointRecordLookup
+
+@Description    Returns a debug string with information about the
+                sync checkpoint.
+
+@Input          psDevNode               The device owning the sync
+                                        checkpoint to lookup
+
+@Input          ui32FwAddr              The firmware address of the sync
+                                        checkpoint to lookup
+
+@Input          pszSyncInfo             Character array to write to
+
+@Input          len                     Len of the character array
+
+@Return         None
+*/
+/*****************************************************************************/
+void
+SyncCheckpointRecordLookup(PPVRSRV_DEVICE_NODE psDevNode,
+                           IMG_UINT32 ui32FwAddr,
+                           IMG_CHAR * pszSyncInfo, size_t len);
+
+#if defined(PDUMP)
+/*************************************************************************/ /*!
+@Function       PVRSRVSyncCheckpointFencePDumpPolKM
+
+@Description    Called to insert a poll into the PDump script on a given
+                Fence being signalled or errored.
+
+@Input          hFence        Fence for PDump to poll on
+
+@Return         PVRSRV_OK if a valid sync checkpoint was provided.
+*/
+/*****************************************************************************/
+
+PVRSRV_ERROR PVRSRVSyncCheckpointSignalledPDumpPolKM(PVRSRV_FENCE hFence);
+
+#endif
+
+#endif	/* SYNC_CHECKPOINT_H */
diff --git a/drivers/gpu/drm/img-rogue/sync_checkpoint_external.h b/drivers/gpu/drm/img-rogue/sync_checkpoint_external.h
new file mode 100644
index 000000000000..19b5011aa8b7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_checkpoint_external.h
@@ -0,0 +1,83 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services external synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines synchronisation checkpoint structures that are visible
+				internally and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_EXTERNAL_H
+#define SYNC_CHECKPOINT_EXTERNAL_H
+
+#include "img_types.h"
+
+#ifndef CHECKPOINT_TYPES
+#define CHECKPOINT_TYPES
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG *PSYNC_CHECKPOINT_CONTEXT;
+
+typedef struct SYNC_CHECKPOINT_TAG *PSYNC_CHECKPOINT;
+#endif
+
+/* PVRSRV_SYNC_CHECKPOINT states.
+ * The OS native sync implementation should call pfnIsSignalled() to determine if a
+ * PVRSRV_SYNC_CHECKPOINT has signalled (which will return an IMG_BOOL), but can set the
+ * state for a PVRSRV_SYNC_CHECKPOINT (which is currently in the NOT_SIGNALLED state)
+ * where that PVRSRV_SYNC_CHECKPOINT is representing a foreign sync.
+ */
+typedef IMG_UINT32 PVRSRV_SYNC_CHECKPOINT_STATE;
+
+#define PVRSRV_SYNC_CHECKPOINT_UNDEF         0x000U
+#define PVRSRV_SYNC_CHECKPOINT_ACTIVE        0xac1U  /*!< checkpoint has not signalled */
+#define PVRSRV_SYNC_CHECKPOINT_SIGNALLED     0x519U  /*!< checkpoint has signalled */
+#define PVRSRV_SYNC_CHECKPOINT_ERRORED       0xeffU   /*!< checkpoint has been errored */
+
+
+#define PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR(fwaddr)	(((fwaddr) & 0x1U) != 0U)
+#define PVRSRV_UFO_IS_SYNC_CHECKPOINT(ufoptr)			(PVRSRV_UFO_IS_SYNC_CHECKPOINT_FWADDR((ufoptr)->puiAddrUFO.ui32Addr))
+
+/* Maximum number of sync checkpoints the firmware supports in one fence */
+#define MAX_SYNC_CHECKPOINTS_PER_FENCE 32U
+
+/*!
+ * Define to be used with SyncCheckpointAlloc() to indicate a checkpoint which
+ * represents a foreign sync point or collection of foreign sync points.
+ */
+#define SYNC_CHECKPOINT_FOREIGN_CHECKPOINT ((PVRSRV_TIMELINE) - 2U)
+
+#endif /* SYNC_CHECKPOINT_EXTERNAL_H */
diff --git a/drivers/gpu/drm/img-rogue/sync_checkpoint_init.h b/drivers/gpu/drm/img-rogue/sync_checkpoint_init.h
new file mode 100644
index 000000000000..94f2e000ded9
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_checkpoint_init.h
@@ -0,0 +1,82 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services synchronisation checkpoint initialisation interface
+                header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines synchronisation checkpoint structures that are visible
+                internally and externally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_INIT_H
+#define SYNC_CHECKPOINT_INIT_H
+
+#include "device.h"
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointInit
+
+@Description    Initialise the sync checkpoint driver by giving it the
+                device node (needed to determine the pfnUFOAlloc function
+                to call in order to allocate sync block memory).
+
+@Input          psDevNode               Device for which sync checkpoints
+                                        are being initialised
+
+@Return         PVRSRV_OK               initialised successfully,
+                PVRSRV_ERROR_<error>    otherwise
+*/
+/*****************************************************************************/
+PVRSRV_ERROR
+SyncCheckpointInit(PVRSRV_DEVICE_NODE *psDevNode);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointDeinit
+
+@Description    Deinitialise the sync checkpoint driver.
+                Frees resources allocated during initialisation.
+
+@Input          psDevNode               Device for which sync checkpoints
+                                        are being de-initialised
+
+@Return         None
+*/
+/*****************************************************************************/
+void SyncCheckpointDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+#endif /* SYNC_CHECKPOINT_INIT_H */
diff --git a/drivers/gpu/drm/img-rogue/sync_checkpoint_internal.h b/drivers/gpu/drm/img-rogue/sync_checkpoint_internal.h
new file mode 100644
index 000000000000..2514ff3b6973
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_checkpoint_internal.h
@@ -0,0 +1,273 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation checkpoint interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the internal server interface for services
+                synchronisation checkpoints.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_CHECKPOINT_INTERNAL_H
+#define SYNC_CHECKPOINT_INTERNAL_H
+
+#include "img_types.h"
+#include "opaque_types.h"
+#include "sync_checkpoint_external.h"
+#include "sync_checkpoint.h"
+#include "ra.h"
+#include "dllist.h"
+#include "lock.h"
+#include "devicemem.h"
+#include "rgx_fwif_shared.h"
+
+struct SYNC_CHECKPOINT_RECORD;
+
+/*
+	Private structures
+*/
+
+typedef struct _SYNC_CHECKPOINT_CONTEXT_CTL_ _SYNC_CHECKPOINT_CONTEXT_CTL, *_PSYNC_CHECKPOINT_CONTEXT_CTL;
+
+typedef struct SYNC_CHECKPOINT_CONTEXT_TAG _SYNC_CHECKPOINT_CONTEXT;
+
+typedef struct _SYNC_CHECKPOINT_BLOCK_
+{
+	ATOMIC_T                  hRefCount;                  /*!< Ref count for this sync block */
+	POS_LOCK                  hLock;
+	_SYNC_CHECKPOINT_CONTEXT  *psContext;                 /*!< Our copy of the services connection */
+	IMG_UINT32                ui32SyncBlockSize;          /*!< Size of the sync checkpoint block */
+	IMG_UINT32                ui32FirmwareAddr;           /*!< Firmware address */
+	DEVMEM_MEMDESC            *hMemDesc;                  /*!< DevMem allocation for block */
+	volatile IMG_UINT32       *pui32LinAddr;              /*!< Server-code CPU mapping */
+	RA_BASE_T                 uiSpanBase;                 /*!< Base of this import (FW DevMem) in the span RA */
+#if defined(PDUMP)
+	DLLIST_NODE               sListNode;                  /*!< List node for the sync chkpt blocks */
+#endif
+} SYNC_CHECKPOINT_BLOCK;
+
+typedef struct SYNC_CHECKPOINT_RECORD* PSYNC_CHECKPOINT_RECORD_HANDLE;
+
+typedef struct SYNC_CHECKPOINT_TAG
+{
+	/* A sync checkpoint is assigned a unique ID, to avoid any confusion should
+	 * the same memory be re-used later for a different checkpoint
+	 */
+	IMG_UINT32                      ui32UID;                /*!< Unique ID assigned to sync checkpoint (to distinguish checkpoints if memory is re-used)*/
+	ATOMIC_T                        hRefCount;              /*!< Ref count for this sync */
+	ATOMIC_T                        hEnqueuedCCBCount;      /*!< Num times sync has been put in CCBs */
+	SYNC_CHECKPOINT_BLOCK           *psSyncCheckpointBlock; /*!< Synchronisation block this checkpoint is allocated on */
+	RA_BASE_T                       uiAllocatedAddr;        /*!< Allocated address of the sync */
+	volatile SYNC_CHECKPOINT_FW_OBJ *psSyncCheckpointFwObj; /*!< CPU view of the data held in the sync block */
+	PRGXFWIF_UFO_ADDR               sCheckpointUFOAddr;     /*!< PRGXFWIF_UFO_ADDR struct used to pass update address to FW */
+	IMG_CHAR                        azName[PVRSRV_SYNC_NAME_LENGTH]; /*!< Name of the checkpoint */
+	PVRSRV_TIMELINE                 hTimeline;              /*!< Timeline on which this sync checkpoint was created */
+	IMG_PID                         uiProcess;              /*!< The Process ID of the process which created this sync checkpoint */
+	PSYNC_CHECKPOINT_RECORD_HANDLE  hRecord;                /*!< Sync record handle */
+	DLLIST_NODE                     sListNode;              /*!< List node for the global sync chkpt list */
+	DLLIST_NODE                     sDeferredFreeListNode;  /*!< List node for the deferred free sync chkpt list */
+	IMG_UINT32                      ui32FWAddr;             /*!< FWAddr stored at sync checkpoint alloc time */
+#if defined(PDUMP)
+	PDUMP_FLAGS_T                   ui32PDumpFlags;         /*!< Pdump Capture mode to be used for POL*/
+#endif
+#if defined(DEBUG)
+    IMG_UINT32                      ui32ValidationCheck;    /*!< Structure validity pattern */
+#endif
+} SYNC_CHECKPOINT;
+
+
+typedef struct _SYNC_CHECKPOINT_SIGNAL_
+{
+	SYNC_CHECKPOINT                asSyncCheckpoint;       /*!< Store sync checkpt for deferred signal */
+	IMG_UINT32                      ui32Status;             /*!< sync checkpt status signal/errored */
+} _SYNC_CHECKPOINT_DEFERRED_SIGNAL;
+
+#define GET_CP_CB_NEXT_IDX(_curridx) (((_curridx) + 1) % SYNC_CHECKPOINT_MAX_DEFERRED_SIGNAL)
+#define GET_CP_CB_BASE(_idx)   (IMG_OFFSET_ADDR(psDevNode->pui8DeferredSyncCPSignal, \
+                                                ((_idx) * sizeof(_SYNC_CHECKPOINT_DEFERRED_SIGNAL))))
+
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetFirmwareAddr
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the firmware address of
+
+@Return         The firmware address of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetFirmwareAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointCCBEnqueued
+
+@Description    Increment the CCB enqueued reference count for a
+                synchronisation checkpoint. This indicates how many FW
+                operations (checks/update) have been placed into CCBs for the
+                sync checkpoint.
+                When the FW services these operation, it increments its own
+                reference count. When these two values are equal, we know
+                there are not outstanding FW operating for the checkpoint
+                in any CCB.
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint for which
+                                        to increment the enqueued reference
+                                        count
+
+@Return         None
+
+*/
+/*****************************************************************************/
+void
+SyncCheckpointCCBEnqueued(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetEnqueuedCount
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the enqueued count of
+
+@Return         The enqueued count of the sync checkpoint
+                (i.e. the number of FW operations (checks or updates)
+                 currently enqueued in CCBs for the sync checkpoint)
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetEnqueuedCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetReferenceCount
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the reference count of
+
+@Return         The host reference count of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetReferenceCount(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetCreator
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the creating process of
+
+@Return         The process id of the process which created this sync checkpoint.
+
+*/
+/*****************************************************************************/
+IMG_PID
+SyncCheckpointGetCreator(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetId
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the unique Id of
+
+@Return         The unique Id of the sync checkpoint
+
+*/
+/*****************************************************************************/
+IMG_UINT32
+SyncCheckpointGetId(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetTimeline
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the parent timeline of
+
+@Return         The parent timeline of the sync checkpoint
+
+*/
+/*****************************************************************************/
+PVRSRV_TIMELINE
+SyncCheckpointGetTimeline(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetRGXFWIFUFOAddr
+
+@Description    .
+
+@Input          psSyncCheckpoint        Synchronisation checkpoint to get
+                                        the PRGXFWIF_UFO_ADDR of
+
+@Return         The PRGXFWIF_UFO_ADDR of the sync checkpoint, used when
+                providing the update in server kick code.
+
+*/
+/*****************************************************************************/
+PRGXFWIF_UFO_ADDR*
+SyncCheckpointGetRGXFWIFUFOAddr(PSYNC_CHECKPOINT psSyncCheckpoint);
+
+#if !defined(SUPPORT_NATIVE_FENCE_SYNC)
+/*************************************************************************/ /*!
+@Function       SyncCheckpointGetAssociatedDevice
+
+@Description    .
+
+@Input          psSyncCheckpointContext Synchronisation Checkpoint context
+                                        to get the device node of
+
+@Return         The PVRSRV_DEVICE_NODE of the device on which the sync
+                checkpoint context was created.
+
+*/
+/*****************************************************************************/
+PPVRSRV_DEVICE_NODE
+SyncCheckpointGetAssociatedDevice(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext);
+#endif /* !defined(SUPPORT_NATIVE_FENCE_SYNC) */
+
+#endif /* SYNC_CHECKPOINT_INTERNAL_H */
diff --git a/drivers/gpu/drm/img-rogue/sync_fallback_server.h b/drivers/gpu/drm/img-rogue/sync_fallback_server.h
new file mode 100644
index 000000000000..ac6bd4755b0e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_fallback_server.h
@@ -0,0 +1,204 @@
+/**************************************************************************/ /*!
+@File
+@Title          Fallback sync interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#ifndef SYNC_FALLBACK_SERVER_H
+#define SYNC_FALLBACK_SERVER_H
+
+#include "img_types.h"
+#include "sync_checkpoint.h"
+#include "device.h"
+#include "connection_server.h"
+
+
+typedef struct _PVRSRV_TIMELINE_SERVER_ PVRSRV_TIMELINE_SERVER;
+typedef struct _PVRSRV_FENCE_SERVER_ PVRSRV_FENCE_SERVER;
+typedef struct _PVRSRV_FENCE_EXPORT_ PVRSRV_FENCE_EXPORT;
+
+typedef struct _PVRSRV_SYNC_PT_ PVRSRV_SYNC_PT;
+
+#define SYNC_FB_TIMELINE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+#define SYNC_FB_FENCE_MAX_LENGTH PVRSRV_SYNC_NAME_LENGTH
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         SW SPECIFIC FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbTimelineCreateSW(IMG_UINT32 uiTimelineNameSize,
+                                    const IMG_CHAR *pszTimelineName,
+                                    PVRSRV_TIMELINE_SERVER **ppsTimeline);
+
+PVRSRV_ERROR SyncFbFenceCreateSW(CONNECTION_DATA *psConnection,
+                                 PVRSRV_DEVICE_NODE *psDeviceNode,
+                                 PVRSRV_TIMELINE_SERVER *psTimeline,
+                                 IMG_UINT32 uiFenceNameSize,
+                                 const IMG_CHAR *pszFenceName,
+                                 PVRSRV_FENCE_SERVER **ppsOutputFence,
+                                 IMG_UINT64 *pui64SyncPtIdx);
+PVRSRV_ERROR SyncFbSWTimelineFenceCreateKM(PVRSRV_DEVICE_NODE *psDeviceNode,
+                                           PVRSRV_TIMELINE iSWTimeline,
+                                           const IMG_CHAR *pszFenceName,
+                                           PVRSRV_FENCE *piOutputFence,
+                                           IMG_UINT64* pui64SyncPtIdx);
+
+PVRSRV_ERROR SyncFbTimelineAdvanceSW(PVRSRV_TIMELINE_SERVER *psTimeline,
+                                     IMG_UINT64 *pui64SyncPtIdx);
+PVRSRV_ERROR SyncFbSWTimelineAdvanceKM(void *pvSWTimelineObj,
+                                       IMG_UINT64* pui64SyncPtIdx);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         PVR SPECIFIC FUNCTIONS                            */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbTimelineCreatePVR(IMG_UINT32 uiTimelineNameSize,
+                                     const IMG_CHAR *pszTimelineName,
+                                     PVRSRV_TIMELINE_SERVER **ppsTimeline);
+
+PVRSRV_ERROR SyncFbFenceCreatePVR(PPVRSRV_DEVICE_NODE psDeviceNode,
+                                  const IMG_CHAR *pszName,
+                                  PVRSRV_TIMELINE iTl,
+                                  PSYNC_CHECKPOINT_CONTEXT hSyncCheckpointContext,
+                                  PVRSRV_FENCE *piOutFence,
+                                  IMG_UINT64 *puiFenceUID,
+                                  void **ppvFenceFinaliseData,
+                                  PSYNC_CHECKPOINT *ppsOutCheckpoint,
+                                  void **ppvTimelineUpdateSync,
+                                  IMG_UINT32 *puiTimelineUpdateValue);
+
+PVRSRV_ERROR SyncFbFenceResolvePVR(PSYNC_CHECKPOINT_CONTEXT psContext,
+                                   PVRSRV_FENCE iFence,
+                                   IMG_UINT32 *puiNumCheckpoints,
+                                   PSYNC_CHECKPOINT **papsCheckpoints,
+                                   IMG_UINT64 *puiFenceUID);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                         GENERIC FUNCTIONS                                 */
+/*                                                                           */
+/*****************************************************************************/
+
+PVRSRV_ERROR SyncFbGetFenceObj(PVRSRV_FENCE iFence,
+                               void **ppvFenceObj);
+
+PVRSRV_ERROR SyncFbSWGetTimelineObj(PVRSRV_TIMELINE iSWTimeline,
+                                    void **ppvSWTimelineObj);
+
+PVRSRV_ERROR SyncFbTimelineRelease(PVRSRV_TIMELINE_SERVER *psTl);
+
+PVRSRV_ERROR SyncFbFenceRelease(PVRSRV_FENCE_SERVER *psFence);
+PVRSRV_ERROR SyncFbFenceReleaseKM(void *pvFenceObj);
+
+PVRSRV_ERROR SyncFbFenceDup(PVRSRV_FENCE_SERVER *psInFence,
+                            PVRSRV_FENCE_SERVER **ppsOutFence);
+
+PVRSRV_ERROR SyncFbFenceMerge(PVRSRV_FENCE_SERVER *psInFence1,
+                              PVRSRV_FENCE_SERVER *psInFence2,
+                              IMG_UINT32 uiFenceNameSize,
+                              const IMG_CHAR *pszFenceName,
+                              PVRSRV_FENCE_SERVER **ppsOutFence);
+
+PVRSRV_ERROR SyncFbFenceWait(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiTimeout);
+
+PVRSRV_ERROR SyncFbFenceDump(PVRSRV_FENCE_SERVER *psFence,
+                             IMG_UINT32 uiLine,
+                             IMG_UINT32 uiFileNameLength,
+                             const IMG_CHAR *pszFile,
+                             IMG_UINT32 uiModuleLength,
+                             const IMG_CHAR *pszModule,
+                             IMG_UINT32 uiDescLength,
+                             const IMG_CHAR *pszDesc);
+
+PVRSRV_ERROR SyncFbDumpFenceKM(void *pvSWFenceObj,
+	                           DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+	                           void *pvDumpDebugFile);
+
+PVRSRV_ERROR SyncFbSWDumpTimelineKM(void *pvSWTimelineObj,
+                                    DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+                                    void *pvDumpDebugFile);
+
+PVRSRV_ERROR SyncFbRegisterSyncFunctions(void);
+
+PVRSRV_ERROR SyncFbRegisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+PVRSRV_ERROR SyncFbDeregisterDevice(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+IMG_UINT32 SyncFbDumpInfoOnStalledUFOs(IMG_UINT32 nr_ufos, IMG_UINT32 *vaddrs);
+
+IMG_BOOL SyncFbCheckpointHasSignalled(IMG_UINT32 ui32FwAddr, IMG_UINT32 ui32Value);
+
+/*****************************************************************************/
+/*                                                                           */
+/*                       IMPORT/EXPORT FUNCTIONS                             */
+/*                                                                           */
+/*****************************************************************************/
+
+#if defined(SUPPORT_INSECURE_EXPORT)
+PVRSRV_ERROR SyncFbFenceExportInsecure(PVRSRV_FENCE_SERVER *psFence,
+                                       PVRSRV_FENCE_EXPORT **ppExport);
+
+PVRSRV_ERROR SyncFbFenceExportDestroyInsecure(PVRSRV_FENCE_EXPORT *psExport);
+
+PVRSRV_ERROR SyncFbFenceImportInsecure(CONNECTION_DATA *psConnection,
+                                       PVRSRV_DEVICE_NODE *psDevice,
+                                       PVRSRV_FENCE_EXPORT *psImport,
+                                       PVRSRV_FENCE_SERVER **psFence);
+#endif /* defined(SUPPORT_INSECURE_EXPORT) */
+
+PVRSRV_ERROR SyncFbFenceExportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE * psDevNode,
+                                     PVRSRV_FENCE_SERVER *psFence,
+                                     IMG_SECURE_TYPE *phSecure,
+                                     PVRSRV_FENCE_EXPORT **ppsExport,
+                                     CONNECTION_DATA **ppsSecureConnection);
+
+PVRSRV_ERROR SyncFbFenceExportDestroySecure(PVRSRV_FENCE_EXPORT *psExport);
+
+PVRSRV_ERROR SyncFbFenceImportSecure(CONNECTION_DATA *psConnection,
+                                     PVRSRV_DEVICE_NODE *psDevice,
+                                     IMG_SECURE_TYPE hSecure,
+                                     PVRSRV_FENCE_SERVER **psFence);
+
+#endif /* SYNC_FALLBACK_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/sync_internal.h b/drivers/gpu/drm/img-rogue/sync_internal.h
new file mode 100644
index 000000000000..929b66e0f37c
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_internal.h
@@ -0,0 +1,112 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation interface header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Defines the internal client side interface for services
+                synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_INTERNAL
+#define SYNC_INTERNAL
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "ra.h"
+#include "dllist.h"
+#include "lock.h"
+#include "devicemem.h"
+#include "sync_prim_internal.h"
+
+#define LOCAL_SYNC_PRIM_RESET_VALUE 0
+#define LOCAL_SYNC_PRIM_POISON_VALUE 0xa5a5a5a5u
+
+/*
+	Debug feature to protect against GP DM page faults when
+	sync prims are freed by client before work is completed.
+*/
+#define LOCAL_SYNC_BLOCK_RETAIN_FIRST
+
+/*
+	Private structure's
+*/
+#define SYNC_PRIM_NAME_SIZE		50
+typedef struct SYNC_PRIM_CONTEXT_TAG
+{
+	SHARED_DEV_CONNECTION       hDevConnection;
+	IMG_CHAR					azName[SYNC_PRIM_NAME_SIZE];	/*!< Name of the RA */
+	RA_ARENA					*psSubAllocRA;					/*!< RA context */
+	IMG_CHAR					azSpanName[SYNC_PRIM_NAME_SIZE];/*!< Name of the span RA */
+	RA_ARENA					*psSpanRA;						/*!< RA used for span management of SubAllocRA */
+	ATOMIC_T				hRefCount;	/*!< Ref count for this context */
+#if defined(LOCAL_SYNC_BLOCK_RETAIN_FIRST)
+	IMG_HANDLE					hFirstSyncPrim; /*!< Handle to the first allocated sync prim */
+#endif
+} SYNC_PRIM_CONTEXT;
+
+typedef struct SYNC_PRIM_BLOCK_TAG
+{
+	SYNC_PRIM_CONTEXT	*psContext;				/*!< Our copy of the services connection */
+	IMG_HANDLE			hServerSyncPrimBlock;	/*!< Server handle for this block */
+	IMG_UINT32			ui32SyncBlockSize;		/*!< Size of the sync prim block */
+	IMG_UINT32			ui32FirmwareAddr;		/*!< Firmware address */
+	DEVMEM_MEMDESC		*hMemDesc;				/*!< Host mapping handle */
+	IMG_UINT32 __iomem	*pui32LinAddr;			/*!< User CPU mapping */
+	IMG_UINT64			uiSpanBase;				/*!< Base of this import in the span RA */
+	DLLIST_NODE			sListNode;				/*!< List node for the sync block list */
+} SYNC_PRIM_BLOCK;
+
+typedef struct SYNC_PRIM_TAG
+{
+	PVRSRV_CLIENT_SYNC_PRIM	sCommon;		/*!< Client visible part of the sync prim */
+	ATOMIC_T				hRefCount;		/*!< Ref count for this sync */
+	SYNC_PRIM_BLOCK			*psSyncBlock;	/*!< Synchronisation block this primitive is allocated on */
+	IMG_UINT64				uiSpanAddr;		/*!< Span address of the sync */
+	IMG_HANDLE				hRecord;		/*!< Sync record handle */
+} SYNC_PRIM;
+
+
+IMG_INTERNAL PVRSRV_ERROR
+SyncPrimGetFirmwareAddr(PVRSRV_CLIENT_SYNC_PRIM *psSync, IMG_UINT32 *pui32FwAddr);
+
+IMG_INTERNAL PVRSRV_ERROR SyncPrimLocalGetHandleAndOffset(PVRSRV_CLIENT_SYNC_PRIM *psSync,
+							IMG_HANDLE *phBlock,
+							IMG_UINT32 *pui32Offset);
+
+
+#endif	/* SYNC_INTERNAL */
diff --git a/drivers/gpu/drm/img-rogue/sync_prim_internal.h b/drivers/gpu/drm/img-rogue/sync_prim_internal.h
new file mode 100644
index 000000000000..c4dec49e9ca4
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_prim_internal.h
@@ -0,0 +1,83 @@
+/*************************************************************************/ /*!
+@File
+@Title          Services internal synchronisation typedef header
+@Description    Defines synchronisation types that are used internally
+                only
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef SYNC_INTERNAL_H
+#define SYNC_INTERNAL_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include <powervr/mem_types.h>
+
+/* These are included here as the typedefs are required
+ * internally.
+ */
+
+typedef struct SYNC_PRIM_CONTEXT_TAG *PSYNC_PRIM_CONTEXT;
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_TAG
+{
+	volatile uint32_t __iomem *pui32LinAddr;	/*!< User pointer to the primitive */
+} PVRSRV_CLIENT_SYNC_PRIM;
+
+/*!
+ * Bundled information for a sync prim operation
+ *
+ *   Structure: #PVRSRV_CLIENT_SYNC_PRIM_OP
+ *   Typedef: ::PVRSRV_CLIENT_SYNC_PRIM_OP
+ */
+typedef struct PVRSRV_CLIENT_SYNC_PRIM_OP_TAG
+{
+	#define PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK	(1U << 0)
+	#define PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE	(1U << 1)
+	uint32_t                    ui32Flags;       /*!< Operation flags: PVRSRV_CLIENT_SYNC_PRIM_OP_XXX */
+	PVRSRV_CLIENT_SYNC_PRIM    *psSync;          /*!< Pointer to the client sync primitive */
+	uint32_t                    ui32FenceValue;  /*!< The Fence value (only used if PVRSRV_CLIENT_SYNC_PRIM_OP_CHECK is set) */
+	uint32_t                    ui32UpdateValue; /*!< The Update value (only used if PVRSRV_CLIENT_SYNC_PRIM_OP_UPDATE is set) */
+} PVRSRV_CLIENT_SYNC_PRIM_OP;
+
+#if defined(__cplusplus)
+}
+#endif
+#endif /* SYNC_INTERNAL_H */
diff --git a/drivers/gpu/drm/img-rogue/sync_server.c b/drivers/gpu/drm/img-rogue/sync_server.c
new file mode 100644
index 000000000000..1746d946aca1
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_server.c
@@ -0,0 +1,1230 @@
+/*************************************************************************/ /*!
+@File           sync_server.c
+@Title          Server side synchronisation functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements the server side functions that for synchronisation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "img_types.h"
+#include "img_defs.h"
+#include "sync_server.h"
+#include "allocmem.h"
+#include "device.h"
+#include "devicemem.h"
+#include "devicemem_pdump.h"
+#include "osfunc.h"
+#include "pdump.h"
+#include "pvr_debug.h"
+#include "pvr_notifier.h"
+#include "pdump_km.h"
+#include "sync.h"
+#include "sync_internal.h"
+#include "connection_server.h"
+#include "htbserver.h"
+#include "rgxhwperf.h"
+#include "info_page.h"
+
+#include "sync_checkpoint_internal.h"
+#include "sync_checkpoint.h"
+
+/* Include this to obtain MAX_SYNC_CHECKPOINTS_PER_FENCE */
+#include "sync_checkpoint_external.h"
+
+/* Include this to obtain PVRSRV_MAX_DEV_VARS */
+#include "pvrsrv_devvar.h"
+
+#if defined(SUPPORT_SECURE_EXPORT)
+#include "ossecure_export.h"
+#endif
+
+/* Set this to enable debug relating to the construction and maintenance of the sync address list */
+#define SYNC_ADDR_LIST_DEBUG 0
+
+/* Set maximum number of FWAddrs that can be accommodated in a SYNC_ADDR_LIST.
+ * This should allow for PVRSRV_MAX_DEV_VARS dev vars plus
+ * MAX_SYNC_CHECKPOINTS_PER_FENCE sync checkpoints for check fences.
+ * The same SYNC_ADDR_LIST is also used to hold UFOs for updates. While this
+ * may need to accommodate the additional sync prim update returned by Native
+ * sync implementation (used for timeline debug), the size calculated from
+ * PVRSRV_MAX_DEV_VARS+MAX_SYNC_CHECKPOINTS_PER_FENCE should be ample.
+ */
+#define PVRSRV_MAX_SYNC_ADDR_LIST_SIZE (PVRSRV_MAX_DEV_VARS+MAX_SYNC_CHECKPOINTS_PER_FENCE)
+/* Check that helper functions will not be preparing longer lists of
+ * UFOs than the FW can handle.
+ */
+static_assert(PVRSRV_MAX_SYNC_ADDR_LIST_SIZE <= RGXFWIF_CCB_CMD_MAX_UFOS,
+              "PVRSRV_MAX_SYNC_ADDR_LIST_SIZE > RGXFWIF_CCB_CMD_MAX_UFOS.");
+
+/* Max number of syncs allowed in a sync prim op */
+#define SYNC_PRIM_OP_MAX_SYNCS 1024
+
+struct _SYNC_PRIMITIVE_BLOCK_
+{
+	PVRSRV_DEVICE_NODE	*psDevNode;
+	DEVMEM_MEMDESC		*psMemDesc;
+	IMG_UINT32			*pui32LinAddr;
+	IMG_UINT32			ui32BlockSize;		/*!< Size of the Sync Primitive Block */
+	ATOMIC_T			sRefCount;
+	DLLIST_NODE			sConnectionNode;
+	SYNC_CONNECTION_DATA *psSyncConnectionData;	/*!< Link back to the sync connection data if there is one */
+	PRGXFWIF_UFO_ADDR		uiFWAddr;	/*!< The firmware address of the sync prim block */
+};
+
+struct _SYNC_CONNECTION_DATA_
+{
+	DLLIST_NODE	sListHead;  /*!< list of sync block associated with / created against this connection */
+	ATOMIC_T	sRefCount;  /*!< number of references to this object */
+	POS_LOCK	hLock;      /*!< lock protecting the list of sync blocks */
+};
+
+#define DECREMENT_WITH_WRAP(value, sz) ((value) ? ((value) - 1) : ((sz) - 1))
+
+/* this is the max number of syncs we will search or dump
+ * at any time.
+ */
+#define SYNC_RECORD_LIMIT 20000
+
+enum SYNC_RECORD_TYPE
+{
+	SYNC_RECORD_TYPE_UNKNOWN = 0,
+	SYNC_RECORD_TYPE_CLIENT,
+	SYNC_RECORD_TYPE_SERVER,
+};
+
+struct SYNC_RECORD
+{
+	PVRSRV_DEVICE_NODE		*psDevNode;
+	SYNC_PRIMITIVE_BLOCK	*psServerSyncPrimBlock;	/*!< handle to _SYNC_PRIMITIVE_BLOCK_ */
+	IMG_UINT32				ui32SyncOffset;			/*!< offset to sync in block */
+	IMG_UINT32				ui32FwBlockAddr;
+	IMG_PID					uiPID;
+	IMG_UINT64				ui64OSTime;
+	enum SYNC_RECORD_TYPE	eRecordType;
+	DLLIST_NODE				sNode;
+	IMG_CHAR				szClassName[PVRSRV_SYNC_NAME_LENGTH];
+};
+
+#if defined(SYNC_DEBUG) || defined(REFCOUNT_DEBUG)
+#define SYNC_REFCOUNT_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define SYNC_REFCOUNT_PRINT(fmt, ...)
+#endif
+
+#if defined(SYNC_DEBUG)
+#define SYNC_UPDATES_PRINT(fmt, ...) PVRSRVDebugPrintf(PVR_DBG_WARNING, __FILE__, __LINE__, fmt, __VA_ARGS__)
+#else
+#define SYNC_UPDATES_PRINT(fmt, ...)
+#endif
+
+/*!
+*****************************************************************************
+ @Function      : SyncPrimitiveBlockToFWAddr
+
+ @Description   : Given a pointer to a sync primitive block and an offset,
+                  returns the firmware address of the sync.
+
+ @Input           psSyncPrimBlock : Sync primitive block which contains the sync
+ @Input           ui32Offset      : Offset of sync within the sync primitive block
+ @Output          psAddrOut       : Absolute FW address of the sync is written out through
+                                    this pointer
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncPrimitiveBlockToFWAddr(SYNC_PRIMITIVE_BLOCK *psSyncPrimBlock,
+							IMG_UINT32 ui32Offset,
+						PRGXFWIF_UFO_ADDR *psAddrOut)
+{
+	/* check offset is legal */
+	if (unlikely((ui32Offset >= psSyncPrimBlock->ui32BlockSize) ||
+		(ui32Offset % sizeof(IMG_UINT32))))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "SyncPrimitiveBlockToFWAddr: parameters check failed"));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	psAddrOut->ui32Addr = psSyncPrimBlock->uiFWAddr.ui32Addr + ui32Offset;
+	return PVRSRV_OK;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListGrow
+
+ @Description   : Grow the SYNC_ADDR_LIST so it can accommodate the given
+                  number of syncs, up to a maximum of PVRSRV_MAX_SYNC_PRIMS.
+
+ @Input           psList       : The SYNC_ADDR_LIST to grow
+ @Input           ui32NumSyncs : The number of sync addresses to be able to hold
+ @Return :        PVRSRV_OK on success
+*****************************************************************************/
+
+static PVRSRV_ERROR SyncAddrListGrow(SYNC_ADDR_LIST *psList, IMG_UINT32 ui32NumSyncs)
+{
+	if (unlikely(ui32NumSyncs > PVRSRV_MAX_SYNC_ADDR_LIST_SIZE))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: ui32NumSyncs=%u > PVRSRV_MAX_SYNC_ADDR_LIST_SIZE=%u", __func__, ui32NumSyncs, PVRSRV_MAX_SYNC_ADDR_LIST_SIZE));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s:     Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	if (ui32NumSyncs > psList->ui32NumSyncs)
+	{
+		if (psList->pasFWAddrs == NULL)
+		{
+			psList->pasFWAddrs = OSAllocMem(sizeof(PRGXFWIF_UFO_ADDR) * PVRSRV_MAX_SYNC_ADDR_LIST_SIZE);
+			PVR_RETURN_IF_NOMEM(psList->pasFWAddrs);
+		}
+
+		psList->ui32NumSyncs = ui32NumSyncs;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s:     Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	return PVRSRV_OK;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListInit
+
+ @Description   : Initialise a SYNC_ADDR_LIST structure ready for use
+
+ @Input           psList        : The SYNC_ADDR_LIST structure to initialise
+ @Return        : None
+*****************************************************************************/
+
+void
+SyncAddrListInit(SYNC_ADDR_LIST *psList)
+{
+	psList->ui32NumSyncs = 0;
+	psList->pasFWAddrs   = NULL;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListDeinit
+
+ @Description   : Frees any resources associated with the given SYNC_ADDR_LIST
+
+ @Input           psList        : The SYNC_ADDR_LIST structure to deinitialise
+ @Return        : None
+*****************************************************************************/
+
+void
+SyncAddrListDeinit(SYNC_ADDR_LIST *psList)
+{
+	if (psList->pasFWAddrs != NULL)
+	{
+		OSFreeMem(psList->pasFWAddrs);
+	}
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListPopulate
+
+ @Description   : Populate the given SYNC_ADDR_LIST with the FW addresses
+                  of the syncs given by the SYNC_PRIMITIVE_BLOCKs and sync offsets
+
+ @Input           ui32NumSyncs    : The number of syncs being passed in
+ @Input           apsSyncPrimBlock: Array of pointers to SYNC_PRIMITIVE_BLOCK structures
+                                    in which the syncs are based
+ @Input           paui32SyncOffset: Array of offsets within each of the sync primitive blocks
+                                    where the syncs are located
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncAddrListPopulate(SYNC_ADDR_LIST *psList,
+						IMG_UINT32 ui32NumSyncs,
+						SYNC_PRIMITIVE_BLOCK **apsSyncPrimBlock,
+						IMG_UINT32 *paui32SyncOffset)
+{
+	IMG_UINT32 i;
+	PVRSRV_ERROR eError;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	if (ui32NumSyncs > psList->ui32NumSyncs)
+	{
+		eError = SyncAddrListGrow(psList, ui32NumSyncs);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+	psList->ui32NumSyncs = ui32NumSyncs;
+
+	for (i = 0; i < ui32NumSyncs; i++)
+	{
+		eError = SyncPrimitiveBlockToFWAddr(apsSyncPrimBlock[i],
+								paui32SyncOffset[i],
+								&psList->pasFWAddrs[i]);
+
+		PVR_RETURN_IF_ERROR(eError);
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumSyncs));
+#endif
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+SyncAddrListAppendSyncPrim(SYNC_ADDR_LIST          *psList,
+						   PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32FwAddr = 0;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d)", __func__, (void*)psList, psList->ui32NumSyncs));
+#endif
+	/* Ensure there's room in psList for the additional sync prim update */
+	eError = SyncAddrListGrow(psList, psList->ui32NumSyncs + 1);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	SyncPrimGetFirmwareAddr(psSyncPrim, &ui32FwAddr);
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Appending sync prim <%p> UFO addr (0x%x) to psList[->pasFWAddrss[%d]", __func__, (void*)psSyncPrim, ui32FwAddr, psList->ui32NumSyncs-1));
+#endif
+	psList->pasFWAddrs[psList->ui32NumSyncs-1].ui32Addr = ui32FwAddr;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	{
+		IMG_UINT32 iii;
+
+		PVR_DPF((PVR_DBG_ERROR, "%s: psList->ui32NumSyncs=%d", __func__, psList->ui32NumSyncs));
+		for (iii=0; iii<psList->ui32NumSyncs; iii++)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s: psList->pasFWAddrs[%d].ui32Addr=0x%x", __func__, iii, psList->pasFWAddrs[iii].ui32Addr));
+		}
+	}
+#endif
+e0:
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d", __func__, (void*)psList, psList->ui32NumSyncs));
+#endif
+	return eError;
+}
+
+
+static PVRSRV_ERROR
+_AppendCheckpoints(SYNC_ADDR_LIST *psList,
+				   IMG_UINT32 ui32NumCheckpoints,
+				   PSYNC_CHECKPOINT *apsSyncCheckpoint,
+				   IMG_BOOL bDeRefCheckpoints)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32SyncCheckpointIndex;
+	IMG_UINT32 ui32RollbackSize = psList->ui32NumSyncs;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Entry psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+#endif
+	/* Ensure there's room in psList for the sync checkpoints */
+	eError = SyncAddrListGrow(psList, psList->ui32NumSyncs + ui32NumCheckpoints);
+	if (unlikely(eError != PVRSRV_OK))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: * * * * ERROR * * * * Trying to SyncAddrListGrow(psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+		goto e0;
+	}
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: (ui32NumCheckpoints=%d) (psList->ui32NumSyncs is now %d) array already contains %d FWAddrs:", __func__, ui32NumCheckpoints, psList->ui32NumSyncs, ui32RollbackSize));
+	if (ui32RollbackSize > 0)
+	{
+		{
+			IMG_UINT32 kk;
+			for (kk=0; kk<ui32RollbackSize; kk++)
+			{
+				PVR_DPF((PVR_DBG_ERROR, "%s:    <%p>psList->pasFWAddrs[%d].ui32Addr = %u(0x%x)", __func__,
+						 (void*)&psList->pasFWAddrs[kk], kk,
+						 psList->pasFWAddrs[kk].ui32Addr, psList->pasFWAddrs[kk].ui32Addr));
+			}
+		}
+	}
+	PVR_DPF((PVR_DBG_ERROR, "%s: apsSyncCheckpoint=<%p>, apsSyncCheckpoint[0] = <%p>", __func__, (void*)apsSyncCheckpoint, (void*)apsSyncCheckpoint[0]));
+#endif
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		psList->pasFWAddrs[ui32RollbackSize + ui32SyncCheckpointIndex].ui32Addr = SyncCheckpointGetFirmwareAddr(apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s:  SyncCheckpointCCBEnqueued(<%p>)", __func__, (void*)apsSyncCheckpoint[ui32SyncCheckpointIndex]));
+		PVR_DPF((PVR_DBG_ERROR, "%s:                           ID:%d", __func__, SyncCheckpointGetId((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex])));
+#endif
+		SyncCheckpointCCBEnqueued((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		if (bDeRefCheckpoints)
+		{
+			/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+			SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		}
+	}
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	if (psList->ui32NumSyncs > 0)
+	{
+		IMG_UINT32 kk;
+		for (kk=0; kk<psList->ui32NumSyncs; kk++)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "%s:    <%p>psList->pasFWAddrs[%d].ui32Addr = %u(0x%x)", __func__,
+			         (void*)&psList->pasFWAddrs[kk], kk,
+			         psList->pasFWAddrs[kk].ui32Addr, psList->pasFWAddrs[kk].ui32Addr));
+		}
+	}
+#endif
+	return eError;
+
+e0:
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		if (bDeRefCheckpoints)
+		{
+			/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+			SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+		}
+	}
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: Exit psList=<%p>, psList->ui32NumSyncs=%d, ui32NumCheckpoints=%d)", __func__, (void*)psList, psList->ui32NumSyncs, ui32NumCheckpoints));
+#endif
+	return eError;
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListAppendCheckpoints
+
+ @Description   : Append the FW addresses of the sync checkpoints given in
+                  the PSYNC_CHECKPOINTs array to the given SYNC_ADDR_LIST
+
+ @Input           ui32NumSyncCheckpoints : The number of sync checkpoints
+                                           being passed in
+ @Input           apsSyncCheckpoint : Array of PSYNC_CHECKPOINTs whose details
+                                      are to be appended to the SYNC_ADDR_LIST
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+PVRSRV_ERROR
+SyncAddrListAppendCheckpoints(SYNC_ADDR_LIST *psList,
+								IMG_UINT32 ui32NumCheckpoints,
+								PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	return _AppendCheckpoints(psList, ui32NumCheckpoints, apsSyncCheckpoint, IMG_FALSE);
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListAppendAndDeRefCheckpoints
+
+ @Description   : Append the FW addresses of the sync checkpoints given in
+                  the PSYNC_CHECKPOINTs array to the given SYNC_ADDR_LIST.
+                  A reference is dropped for each of the checkpoints.
+
+ @Input           ui32NumSyncCheckpoints : The number of sync checkpoints
+                                           being passed in
+ @Input           apsSyncCheckpoint : Array of PSYNC_CHECKPOINTs whose details
+                                      are to be appended to the SYNC_ADDR_LIST
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+PVRSRV_ERROR
+SyncAddrListAppendAndDeRefCheckpoints(SYNC_ADDR_LIST *psList,
+									  IMG_UINT32 ui32NumCheckpoints,
+									  PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	return _AppendCheckpoints(psList, ui32NumCheckpoints, apsSyncCheckpoint, IMG_TRUE);
+}
+
+void
+SyncAddrListDeRefCheckpoints(IMG_UINT32 ui32NumCheckpoints,
+							 PSYNC_CHECKPOINT *apsSyncCheckpoint)
+{
+	IMG_UINT32 ui32SyncCheckpointIndex;
+
+	for (ui32SyncCheckpointIndex=0; ui32SyncCheckpointIndex<ui32NumCheckpoints; ui32SyncCheckpointIndex++)
+	{
+		/* Drop the reference that was taken internally by the OS implementation of resolve_fence() */
+		SyncCheckpointDropRef((PSYNC_CHECKPOINT)apsSyncCheckpoint[ui32SyncCheckpointIndex]);
+	}
+}
+
+/*!
+*****************************************************************************
+ @Function      : SyncAddrListRollbackCheckpoints
+
+ @Description   : Rollback the enqueued count of each sync checkpoint in
+                  the given SYNC_ADDR_LIST. This needs to be done in the
+                  event of the kick call failing, so that the reference
+                  taken on each sync checkpoint on the firmware's behalf
+                  is dropped.
+
+ @Input           psList        : The SYNC_ADDR_LIST structure containing
+                                  sync checkpoints to be rolled back
+
+ @Return :        PVRSRV_OK on success. PVRSRV_ERROR_INVALID_PARAMS if input
+                  parameters are invalid.
+*****************************************************************************/
+
+PVRSRV_ERROR
+SyncAddrListRollbackCheckpoints(PVRSRV_DEVICE_NODE *psDevNode, SYNC_ADDR_LIST *psList)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	IMG_UINT32 ui32SyncIndex;
+
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+	PVR_DPF((PVR_DBG_ERROR, "%s: called (psList=<%p>)", __func__, (void*)psList));
+#endif
+	if (psList)
+	{
+#if (SYNC_ADDR_LIST_DEBUG == 1)
+		PVR_DPF((PVR_DBG_ERROR, "%s: psList->ui32NumSyncs=%d", __func__, psList->ui32NumSyncs));
+#endif
+		for (ui32SyncIndex=0; ui32SyncIndex<psList->ui32NumSyncs; ui32SyncIndex++)
+		{
+			if (psList->pasFWAddrs[ui32SyncIndex].ui32Addr & 0x1)
+			{
+				SyncCheckpointRollbackFromUFO(psDevNode, psList->pasFWAddrs[ui32SyncIndex].ui32Addr);
+			}
+		}
+	}
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncRecordAddKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDevNode,
+					  SYNC_RECORD_HANDLE *phRecord,
+					  SYNC_PRIMITIVE_BLOCK *hServerSyncPrimBlock,
+					  IMG_UINT32 ui32FwBlockAddr,
+					  IMG_UINT32 ui32SyncOffset,
+					  IMG_BOOL bServerSync,
+					  IMG_UINT32 ui32ClassNameSize,
+					  const IMG_CHAR *pszClassName)
+{
+	struct SYNC_RECORD * psSyncRec;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+
+	RGXSRV_HWPERF_ALLOC(psDevNode, SYNC,
+	                    ui32FwBlockAddr + ui32SyncOffset,
+	                    pszClassName,
+	                    ui32ClassNameSize);
+
+	PVR_RETURN_IF_INVALID_PARAM(phRecord);
+
+	*phRecord = NULL;
+
+	psSyncRec = OSAllocMem(sizeof(*psSyncRec));
+	PVR_GOTO_IF_NOMEM(psSyncRec, eError, fail_alloc);
+
+	psSyncRec->psDevNode = psDevNode;
+	psSyncRec->psServerSyncPrimBlock = hServerSyncPrimBlock;
+	psSyncRec->ui32SyncOffset = ui32SyncOffset;
+	psSyncRec->ui32FwBlockAddr = ui32FwBlockAddr;
+	psSyncRec->ui64OSTime = OSClockns64();
+	psSyncRec->uiPID = OSGetCurrentProcessID();
+	psSyncRec->eRecordType = bServerSync? SYNC_RECORD_TYPE_SERVER: SYNC_RECORD_TYPE_CLIENT;
+
+	if (pszClassName)
+	{
+		if (ui32ClassNameSize >= PVRSRV_SYNC_NAME_LENGTH)
+			ui32ClassNameSize = PVRSRV_SYNC_NAME_LENGTH;
+		/* Copy over the class name annotation */
+		OSStringSafeCopy(psSyncRec->szClassName, pszClassName, ui32ClassNameSize);
+	}
+	else
+	{
+		/* No class name annotation */
+		psSyncRec->szClassName[0] = 0;
+	}
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	if (psDevNode->ui32SyncServerRecordCount < SYNC_RECORD_LIMIT)
+	{
+		dllist_add_to_head(&psDevNode->sSyncServerRecordList, &psSyncRec->sNode);
+		psDevNode->ui32SyncServerRecordCount++;
+
+		if (psDevNode->ui32SyncServerRecordCount > psDevNode->ui32SyncServerRecordCountHighWatermark)
+		{
+			psDevNode->ui32SyncServerRecordCountHighWatermark = psDevNode->ui32SyncServerRecordCount;
+		}
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Failed to add sync record \"%s\". %u records already exist.",
+											__func__,
+											pszClassName,
+											psDevNode->ui32SyncServerRecordCount));
+		OSFreeMem(psSyncRec);
+		psSyncRec = NULL;
+		eError = PVRSRV_ERROR_TOOMANYBUFFERS;
+	}
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	*phRecord = (SYNC_RECORD_HANDLE)psSyncRec;
+
+fail_alloc:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncRecordRemoveByHandleKM(
+			SYNC_RECORD_HANDLE hRecord)
+{
+	struct SYNC_RECORD **ppFreedSync;
+	struct SYNC_RECORD *pSync = (struct SYNC_RECORD*)hRecord;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	PVR_RETURN_IF_INVALID_PARAM(hRecord);
+
+	psDevNode = pSync->psDevNode;
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+
+	RGXSRV_HWPERF_FREE(psDevNode, SYNC, pSync->ui32FwBlockAddr + pSync->ui32SyncOffset);
+
+	dllist_remove_node(&pSync->sNode);
+
+	if (psDevNode->uiSyncServerRecordFreeIdx >= PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: freed sync record index out of range",
+				 __func__));
+		psDevNode->uiSyncServerRecordFreeIdx = 0;
+	}
+	ppFreedSync = &psDevNode->apsSyncServerRecordsFreed[psDevNode->uiSyncServerRecordFreeIdx];
+	psDevNode->uiSyncServerRecordFreeIdx =
+		(psDevNode->uiSyncServerRecordFreeIdx + 1) % PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN;
+
+	if (*ppFreedSync)
+	{
+		OSFreeMem(*ppFreedSync);
+	}
+	pSync->psServerSyncPrimBlock = NULL;
+	pSync->ui64OSTime = OSClockns64();
+	*ppFreedSync = pSync;
+
+	psDevNode->ui32SyncServerRecordCount--;
+
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncAllocEventKM(CONNECTION_DATA *psConnection,
+			PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_BOOL bServerSync,
+			IMG_UINT32 ui32FWAddr,
+			IMG_UINT32 ui32ClassNameSize,
+			const IMG_CHAR *pszClassName)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	RGXSRV_HWPERF_ALLOC(psDevNode, SYNC, ui32FWAddr, pszClassName, ui32ClassNameSize);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncFreeEventKM(CONNECTION_DATA *psConnection,
+			PVRSRV_DEVICE_NODE *psDevNode,
+			IMG_UINT32 ui32FWAddr)
+{
+	PVR_UNREFERENCED_PARAMETER(psConnection);
+	RGXSRV_HWPERF_FREE(psDevNode, SYNC, ui32FWAddr);
+
+	return PVRSRV_OK;
+}
+
+static
+void _SyncConnectionRef(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicIncrement(&psSyncConnectionData->sRefCount);
+
+	SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+						__func__, psSyncConnectionData, iRefCount);
+	PVR_UNREFERENCED_PARAMETER(iRefCount);
+}
+
+static
+void _SyncConnectionUnref(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	IMG_INT iRefCount = OSAtomicDecrement(&psSyncConnectionData->sRefCount);
+	if (iRefCount == 0)
+	{
+		SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+		                    __func__, psSyncConnectionData, iRefCount);
+
+		PVR_ASSERT(dllist_is_empty(&psSyncConnectionData->sListHead));
+		OSLockDestroy(psSyncConnectionData->hLock);
+		OSFreeMem(psSyncConnectionData);
+	}
+	else
+	{
+		SYNC_REFCOUNT_PRINT("%s: Sync connection %p, refcount = %d",
+		                    __func__, psSyncConnectionData, iRefCount);
+		PVR_ASSERT(iRefCount > 0);
+	}
+}
+
+static
+void _SyncConnectionAddBlock(CONNECTION_DATA *psConnection, SYNC_PRIMITIVE_BLOCK *psBlock)
+{
+	if (psConnection)
+	{
+		SYNC_CONNECTION_DATA *psSyncConnectionData = psConnection->psSyncConnectionData;
+
+		/*
+			Make sure the connection doesn't go away. It doesn't matter that we will release
+			the lock between as the refcount and list don't have to be atomic w.r.t. to each other
+		*/
+		_SyncConnectionRef(psSyncConnectionData);
+
+		OSLockAcquire(psSyncConnectionData->hLock);
+		dllist_add_to_head(&psSyncConnectionData->sListHead, &psBlock->sConnectionNode);
+		OSLockRelease(psSyncConnectionData->hLock);
+		psBlock->psSyncConnectionData = psSyncConnectionData;
+	}
+	else
+	{
+		psBlock->psSyncConnectionData = NULL;
+	}
+}
+
+static
+void _SyncConnectionRemoveBlock(SYNC_PRIMITIVE_BLOCK *psBlock)
+{
+	SYNC_CONNECTION_DATA *psSyncConnectionData = psBlock->psSyncConnectionData;
+
+	if (psBlock->psSyncConnectionData)
+	{
+		OSLockAcquire(psSyncConnectionData->hLock);
+		dllist_remove_node(&psBlock->sConnectionNode);
+		OSLockRelease(psSyncConnectionData->hLock);
+
+		_SyncConnectionUnref(psBlock->psSyncConnectionData);
+	}
+}
+
+static inline
+void _DoPrimBlockFree(SYNC_PRIMITIVE_BLOCK *psSyncBlk)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = psSyncBlk->psDevNode;
+
+	SYNC_REFCOUNT_PRINT("%s: Sync block %p, refcount = %d (remove)",
+	                    __func__, psSyncBlk, OSAtomicRead(&psSyncBlk->sRefCount));
+
+	PVR_ASSERT(OSAtomicRead(&psSyncBlk->sRefCount) == 1);
+
+	_SyncConnectionRemoveBlock(psSyncBlk);
+	DevmemReleaseCpuVirtAddr(psSyncBlk->psMemDesc);
+	psDevNode->pfnFreeUFOBlock(psDevNode, psSyncBlk->psMemDesc);
+	OSFreeMem(psSyncBlk);
+}
+
+PVRSRV_ERROR
+PVRSRVAllocSyncPrimitiveBlockKM(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+								SYNC_PRIMITIVE_BLOCK **ppsSyncBlk,
+								IMG_UINT32 *puiSyncPrimVAddr,
+								IMG_UINT32 *puiSyncPrimBlockSize,
+								PMR        **ppsSyncPMR)
+{
+	SYNC_PRIMITIVE_BLOCK *psNewSyncBlk;
+	PVRSRV_ERROR eError;
+
+	psNewSyncBlk = OSAllocMem(sizeof(SYNC_PRIMITIVE_BLOCK));
+	PVR_GOTO_IF_NOMEM(psNewSyncBlk, eError, e0);
+
+	psNewSyncBlk->psDevNode = psDevNode;
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS, "Allocate UFO block");
+
+	eError = psDevNode->pfnAllocUFOBlock(psDevNode,
+										 sizeof(IMG_UINT32),
+										 &psNewSyncBlk->psMemDesc,
+										 &psNewSyncBlk->uiFWAddr.ui32Addr,
+										 &psNewSyncBlk->ui32BlockSize);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	*puiSyncPrimVAddr = psNewSyncBlk->uiFWAddr.ui32Addr;
+
+	eError = DevmemAcquireCpuVirtAddr(psNewSyncBlk->psMemDesc,
+									  (void **) &psNewSyncBlk->pui32LinAddr);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	eError = DevmemLocalGetImportHandle(psNewSyncBlk->psMemDesc, (void **) ppsSyncPMR);
+
+	PVR_GOTO_IF_ERROR(eError, e3);
+
+	OSAtomicWrite(&psNewSyncBlk->sRefCount, 1);
+
+	/* If there is a connection pointer then add the new block onto it's list */
+	_SyncConnectionAddBlock(psConnection, psNewSyncBlk);
+
+	*ppsSyncBlk = psNewSyncBlk;
+	*puiSyncPrimBlockSize = psNewSyncBlk->ui32BlockSize;
+
+	PDUMPCOMMENTWITHFLAGS(psDevNode, PDUMP_FLAGS_CONTINUOUS,
+						  "Allocated UFO block (FirmwareVAddr = 0x%08x)",
+						  *puiSyncPrimVAddr);
+
+	return PVRSRV_OK;
+
+e3:
+	DevmemReleaseCpuVirtAddr(psNewSyncBlk->psMemDesc);
+e2:
+	psDevNode->pfnFreeUFOBlock(psDevNode, psNewSyncBlk->psMemDesc);
+e1:
+	OSFreeMem(psNewSyncBlk);
+e0:
+	return eError;
+}
+
+PVRSRV_ERROR
+PVRSRVFreeSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk)
+{
+
+	/* This function is an alternative to the above without reference counting.
+	 * With the removal of sync prim ops for server syncs we no longer have to
+	 * reference count prim blocks as the reference will never be incremented /
+	 * decremented by a prim op */
+	_DoPrimBlockFree(psSyncBlk);
+	return PVRSRV_OK;
+}
+
+static INLINE IMG_BOOL _CheckSyncIndex(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+                                       IMG_UINT32 ui32Index)
+{
+	if (psSyncBlk->ui32BlockSize == 0)
+	{
+		return IMG_FALSE;
+	}
+
+	return (ui32Index < psSyncBlk->ui32BlockSize / sizeof(IMG_UINT32));
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimSetKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+                    IMG_UINT32 ui32Index,
+                    IMG_UINT32 ui32Value)
+{
+	PVR_LOG_RETURN_IF_INVALID_PARAM(psSyncBlk != NULL, "psSyncBlk");
+
+	if (_CheckSyncIndex(psSyncBlk, ui32Index))
+	{
+		psSyncBlk->pui32LinAddr[ui32Index] = ui32Value;
+		return PVRSRV_OK;
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+		         "PVRSRVSyncPrimSetKM: Index %u out of range for "
+		         "0x%08X byte sync block (value 0x%08X)",
+		         ui32Index,
+		         psSyncBlk->ui32BlockSize,
+		         ui32Value));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+}
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset, IMG_UINT32 ui32Value)
+{
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	*/
+	DevmemPDumpLoadMemValue32(psSyncBlk->psMemDesc,
+					   ui32Offset,
+					   ui32Value,
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset)
+{
+	/*
+		We might be ask to PDump sync state outside of capture range
+		(e.g. texture uploads) so make this continuous.
+	*/
+	DevmemPDumpLoadMem(psSyncBlk->psMemDesc,
+					   ui32Offset,
+					   sizeof(IMG_UINT32),
+					   PDUMP_FLAGS_CONTINUOUS);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T ui32PDumpFlags)
+{
+	DevmemPDumpDevmemPol32(psSyncBlk->psMemDesc,
+						   ui32Offset,
+						   ui32Value,
+						   ui32Mask,
+						   eOperator,
+						   ui32PDumpFlags);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize)
+{
+	DevmemPDumpCBP(psSyncBlk->psMemDesc,
+				   ui32Offset,
+				   uiWriteOffset,
+				   uiPacketSize,
+				   uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif
+
+/* SyncRegisterConnection */
+PVRSRV_ERROR SyncRegisterConnection(SYNC_CONNECTION_DATA **ppsSyncConnectionData)
+{
+	SYNC_CONNECTION_DATA *psSyncConnectionData;
+	PVRSRV_ERROR eError;
+
+	psSyncConnectionData = OSAllocMem(sizeof(SYNC_CONNECTION_DATA));
+	if (psSyncConnectionData == NULL)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto fail_alloc;
+	}
+
+	eError = OSLockCreate(&psSyncConnectionData->hLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lockcreate);
+	dllist_init(&psSyncConnectionData->sListHead);
+	OSAtomicWrite(&psSyncConnectionData->sRefCount, 1);
+
+	*ppsSyncConnectionData = psSyncConnectionData;
+	return PVRSRV_OK;
+
+fail_lockcreate:
+	OSFreeMem(psSyncConnectionData);
+fail_alloc:
+	PVR_ASSERT(eError != PVRSRV_OK);
+	return eError;
+}
+
+/* SyncUnregisterConnection */
+void SyncUnregisterConnection(SYNC_CONNECTION_DATA *psSyncConnectionData)
+{
+	_SyncConnectionUnref(psSyncConnectionData);
+}
+
+void SyncConnectionPDumpSyncBlocks(PVRSRV_DEVICE_NODE *psDevNode, void *hSyncPrivData, PDUMP_TRANSITION_EVENT eEvent)
+{
+	if ((eEvent == PDUMP_TRANSITION_EVENT_RANGE_ENTERED) || (eEvent == PDUMP_TRANSITION_EVENT_BLOCK_STARTED))
+	{
+		SYNC_CONNECTION_DATA *psSyncConnectionData = hSyncPrivData;
+		DLLIST_NODE *psNode, *psNext;
+
+		OSLockAcquire(psSyncConnectionData->hLock);
+
+		PDUMPCOMMENT(psDevNode, "Dump client Sync Prim state");
+		dllist_foreach_node(&psSyncConnectionData->sListHead, psNode, psNext)
+		{
+			SYNC_PRIMITIVE_BLOCK *psSyncBlock =
+				IMG_CONTAINER_OF(psNode, SYNC_PRIMITIVE_BLOCK, sConnectionNode);
+
+			DevmemPDumpLoadMem(psSyncBlock->psMemDesc,
+					0,
+					psSyncBlock->ui32BlockSize,
+					PDUMP_FLAGS_CONTINUOUS);
+		}
+
+		OSLockRelease(psSyncConnectionData->hLock);
+	}
+}
+
+void SyncRecordLookup(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 ui32FwAddr,
+					  IMG_CHAR * pszSyncInfo, size_t len)
+{
+	DLLIST_NODE *psNode, *psNext;
+	IMG_INT iEnd;
+	IMG_BOOL bFound = IMG_FALSE;
+
+	if (!pszSyncInfo)
+	{
+		return;
+	}
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	pszSyncInfo[0] = '\0';
+
+	dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+	{
+		struct SYNC_RECORD *psSyncRec =
+			IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+		if ((psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset) == ui32FwAddr
+			&& SYNC_RECORD_TYPE_UNKNOWN != psSyncRec->eRecordType
+			&& psSyncRec->psServerSyncPrimBlock
+			&& psSyncRec->psServerSyncPrimBlock->pui32LinAddr
+			)
+		{
+			IMG_UINT32 *pui32SyncAddr;
+			pui32SyncAddr = psSyncRec->psServerSyncPrimBlock->pui32LinAddr
+				+ (psSyncRec->ui32SyncOffset/sizeof(IMG_UINT32));
+			iEnd = OSSNPrintf(pszSyncInfo, len, "Cur=0x%08x %s:%05u (%s)",
+				*pui32SyncAddr,
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				psSyncRec->szClassName
+				);
+			if (iEnd >= 0 && iEnd < len)
+			{
+				pszSyncInfo[iEnd] = '\0';
+			}
+			bFound = IMG_TRUE;
+			break;
+		}
+	}
+
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	if (!bFound && (psDevNode->ui32SyncServerRecordCountHighWatermark == SYNC_RECORD_LIMIT))
+	{
+		OSSNPrintf(pszSyncInfo, len, "(Record may be lost)");
+	}
+}
+
+#define NS_IN_S (1000000000UL)
+static void _SyncRecordPrint(struct SYNC_RECORD *psSyncRec,
+					IMG_UINT64 ui64TimeNow,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	SYNC_PRIMITIVE_BLOCK *psSyncBlock = psSyncRec->psServerSyncPrimBlock;
+
+	if (SYNC_RECORD_TYPE_UNKNOWN != psSyncRec->eRecordType)
+	{
+		IMG_UINT64 ui64DeltaS;
+		IMG_UINT32 ui32DeltaF;
+		IMG_UINT64 ui64Delta = ui64TimeNow - psSyncRec->ui64OSTime;
+		ui64DeltaS = OSDivide64(ui64Delta, NS_IN_S, &ui32DeltaF);
+
+		if (psSyncBlock && psSyncBlock->pui32LinAddr)
+		{
+			IMG_UINT32 *pui32SyncAddr;
+			pui32SyncAddr = psSyncBlock->pui32LinAddr
+				+ (psSyncRec->ui32SyncOffset/sizeof(IMG_UINT32));
+
+			PVR_DUMPDEBUG_LOG("\t%s %05u %05" IMG_UINT64_FMTSPEC ".%09u FWAddr=0x%08x Val=0x%08x (%s)",
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				ui64DeltaS, ui32DeltaF,
+				(psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset),
+				*pui32SyncAddr,
+				psSyncRec->szClassName
+				);
+		}
+		else
+		{
+			PVR_DUMPDEBUG_LOG("\t%s %05u %05" IMG_UINT64_FMTSPEC ".%09u FWAddr=0x%08x Val=<null_ptr> (%s)",
+				((SYNC_RECORD_TYPE_SERVER==psSyncRec->eRecordType)?"Server":"Client"),
+				psSyncRec->uiPID,
+				ui64DeltaS, ui32DeltaF,
+				(psSyncRec->ui32FwBlockAddr+psSyncRec->ui32SyncOffset),
+				psSyncRec->szClassName
+				);
+		}
+	}
+}
+
+static void _SyncRecordRequest(PVRSRV_DBGREQ_HANDLE hDebugRequestHandle,
+					IMG_UINT32 ui32VerbLevel,
+					DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+					void *pvDumpDebugFile)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = (PVRSRV_DEVICE_NODE *)hDebugRequestHandle;
+	IMG_UINT64 ui64TimeNowS;
+	IMG_UINT32 ui32TimeNowF;
+	IMG_UINT64 ui64TimeNow = OSClockns64();
+	DLLIST_NODE *psNode, *psNext;
+
+	ui64TimeNowS = OSDivide64(ui64TimeNow, NS_IN_S, &ui32TimeNowF);
+
+	if (DD_VERB_LVL_ENABLED(ui32VerbLevel, DEBUG_REQUEST_VERBOSITY_MEDIUM))
+	{
+		IMG_UINT32 i;
+		OSLockAcquire(psDevNode->hSyncServerRecordLock);
+
+		PVR_DUMPDEBUG_LOG("Dumping all allocated syncs. Allocated: %u High watermark: %u @ %05" IMG_UINT64_FMTSPEC ".%09u",
+										psDevNode->ui32SyncServerRecordCount,
+										psDevNode->ui32SyncServerRecordCountHighWatermark,
+										ui64TimeNowS,
+										ui32TimeNowF);
+		if (psDevNode->ui32SyncServerRecordCountHighWatermark == SYNC_RECORD_LIMIT)
+		{
+			PVR_DUMPDEBUG_LOG("Warning: Record limit (%u) was reached. Some sync checkpoints may not have been recorded in the debug information.",
+                                                                                                                SYNC_RECORD_LIMIT);
+		}
+
+		PVR_DUMPDEBUG_LOG("\t%-6s %-5s %-15s %-17s %-14s (%s)",
+					"Type", "PID", "Time Delta (s)", "Address", "Value", "Annotation");
+
+		dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+		{
+			struct SYNC_RECORD *psSyncRec =
+				IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+			_SyncRecordPrint(psSyncRec, ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+
+		PVR_DUMPDEBUG_LOG("Dumping all recently freed syncs @ %05" IMG_UINT64_FMTSPEC ".%09u",
+						  ui64TimeNowS, ui32TimeNowF);
+		PVR_DUMPDEBUG_LOG("\t%-6s %-5s %-15s %-17s %-14s (%s)",
+					"Type", "PID", "Time Delta (s)", "Address", "Value", "Annotation");
+		for (i = DECREMENT_WITH_WRAP(psDevNode->uiSyncServerRecordFreeIdx, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN);
+			 i != psDevNode->uiSyncServerRecordFreeIdx;
+			 i = DECREMENT_WITH_WRAP(i, PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN))
+		{
+			if (psDevNode->apsSyncServerRecordsFreed[i])
+			{
+				_SyncRecordPrint(psDevNode->apsSyncServerRecordsFreed[i],
+								 ui64TimeNow, pfnDumpDebugPrintf, pvDumpDebugFile);
+			}
+			else
+			{
+				break;
+			}
+		}
+
+		OSLockRelease(psDevNode->hSyncServerRecordLock);
+	}
+}
+#undef NS_IN_S
+
+static PVRSRV_ERROR SyncRecordListInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	psDevNode->ui32SyncServerRecordCount = 0;
+	psDevNode->ui32SyncServerRecordCountHighWatermark = 0;
+
+	eError = OSLockCreate(&psDevNode->hSyncServerRecordLock);
+	PVR_GOTO_IF_ERROR(eError, fail_lock_create);
+	dllist_init(&psDevNode->sSyncServerRecordList);
+
+	eError = PVRSRVRegisterDeviceDbgRequestNotify(&psDevNode->hSyncServerRecordNotify,
+		                                          psDevNode,
+		                                          _SyncRecordRequest,
+		                                          DEBUG_REQUEST_SYNCTRACKING,
+		                                          psDevNode);
+
+	PVR_GOTO_IF_ERROR(eError, fail_dbg_register);
+
+	return PVRSRV_OK;
+
+fail_dbg_register:
+	OSLockDestroy(psDevNode->hSyncServerRecordLock);
+fail_lock_create:
+	return eError;
+}
+
+static void SyncRecordListDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	DLLIST_NODE *psNode, *psNext;
+	int i;
+
+	OSLockAcquire(psDevNode->hSyncServerRecordLock);
+	dllist_foreach_node(&psDevNode->sSyncServerRecordList, psNode, psNext)
+	{
+		struct SYNC_RECORD *pSyncRec =
+			IMG_CONTAINER_OF(psNode, struct SYNC_RECORD, sNode);
+
+		dllist_remove_node(psNode);
+		OSFreeMem(pSyncRec);
+	}
+
+	for (i = 0; i < PVRSRV_FULL_SYNC_TRACKING_HISTORY_LEN; i++)
+	{
+		if (psDevNode->apsSyncServerRecordsFreed[i])
+		{
+			OSFreeMem(psDevNode->apsSyncServerRecordsFreed[i]);
+			psDevNode->apsSyncServerRecordsFreed[i] = NULL;
+		}
+	}
+	OSLockRelease(psDevNode->hSyncServerRecordLock);
+
+	if (psDevNode->hSyncServerRecordNotify)
+	{
+		PVRSRVUnregisterDeviceDbgRequestNotify(psDevNode->hSyncServerRecordNotify);
+	}
+	OSLockDestroy(psDevNode->hSyncServerRecordLock);
+}
+
+PVRSRV_ERROR SyncServerInit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+	PVRSRV_ERROR eError;
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		eError = SyncRecordListInit(psDevNode);
+		PVR_GOTO_IF_ERROR(eError, fail_record_list);
+	}
+
+	return PVRSRV_OK;
+
+fail_record_list:
+	return eError;
+}
+
+void SyncServerDeinit(PVRSRV_DEVICE_NODE *psDevNode)
+{
+
+	if (GetInfoPageDebugFlagsKM() & DEBUG_FEATURE_FULL_SYNC_TRACKING_ENABLED)
+	{
+		SyncRecordListDeinit(psDevNode);
+	}
+}
diff --git a/drivers/gpu/drm/img-rogue/sync_server.h b/drivers/gpu/drm/img-rogue/sync_server.h
new file mode 100644
index 000000000000..e35682970af6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sync_server.h
@@ -0,0 +1,249 @@
+/**************************************************************************/ /*!
+@File
+@Title          Server side synchronisation interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Describes the server side synchronisation functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv.h"
+#include "device.h"
+#include "devicemem.h"
+#include "pdump.h"
+#include "pvrsrv_error.h"
+#include "connection_server.h"
+#include "pdump_km.h"
+
+#ifndef SYNC_SERVER_H
+#define SYNC_SERVER_H
+
+typedef struct _SYNC_PRIMITIVE_BLOCK_ SYNC_PRIMITIVE_BLOCK;
+typedef struct _SYNC_CONNECTION_DATA_ SYNC_CONNECTION_DATA;
+typedef struct SYNC_RECORD* SYNC_RECORD_HANDLE;
+
+typedef struct _SYNC_ADDR_LIST_
+{
+	IMG_UINT32 ui32NumSyncs;
+	PRGXFWIF_UFO_ADDR *pasFWAddrs;
+} SYNC_ADDR_LIST;
+
+PVRSRV_ERROR
+SyncPrimitiveBlockToFWAddr(SYNC_PRIMITIVE_BLOCK *psSyncPrimBlock,
+						IMG_UINT32 ui32Offset,
+						PRGXFWIF_UFO_ADDR *psAddrOut);
+
+void
+SyncAddrListInit(SYNC_ADDR_LIST *psList);
+
+void
+SyncAddrListDeinit(SYNC_ADDR_LIST *psList);
+
+PVRSRV_ERROR
+SyncAddrListPopulate(SYNC_ADDR_LIST *psList,
+						IMG_UINT32 ui32NumSyncs,
+						SYNC_PRIMITIVE_BLOCK **apsSyncPrimBlock,
+						IMG_UINT32 *paui32SyncOffset);
+
+PVRSRV_ERROR
+SyncAddrListAppendSyncPrim(SYNC_ADDR_LIST          *psList,
+						   PVRSRV_CLIENT_SYNC_PRIM *psSyncPrim);
+PVRSRV_ERROR
+SyncAddrListAppendCheckpoints(SYNC_ADDR_LIST *psList,
+								IMG_UINT32 ui32NumCheckpoints,
+								PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+PVRSRV_ERROR
+SyncAddrListAppendAndDeRefCheckpoints(SYNC_ADDR_LIST *psList,
+									  IMG_UINT32 ui32NumCheckpoints,
+									  PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+void
+SyncAddrListDeRefCheckpoints(IMG_UINT32 ui32NumCheckpoints,
+							 PSYNC_CHECKPOINT *apsSyncCheckpoint);
+
+PVRSRV_ERROR
+SyncAddrListRollbackCheckpoints(PVRSRV_DEVICE_NODE *psDevNode, SYNC_ADDR_LIST *psList);
+
+PVRSRV_ERROR
+PVRSRVAllocSyncPrimitiveBlockKM(CONNECTION_DATA *psConnection,
+                                PVRSRV_DEVICE_NODE * psDevNode,
+								SYNC_PRIMITIVE_BLOCK **ppsSyncBlk,
+								IMG_UINT32 *puiSyncPrimVAddr,
+								IMG_UINT32 *puiSyncPrimBlockSize,
+								PMR        **ppsSyncPMR);
+
+PVRSRV_ERROR
+PVRSRVExportSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk,
+								 DEVMEM_EXPORTCOOKIE **psExportCookie);
+
+PVRSRV_ERROR
+PVRSRVUnexportSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk);
+
+PVRSRV_ERROR
+PVRSRVFreeSyncPrimitiveBlockKM(SYNC_PRIMITIVE_BLOCK *ppsSyncBlk);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimSetKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Index,
+					IMG_UINT32 ui32Value);
+
+PVRSRV_ERROR
+PVRSRVSyncAllocEventKM(CONNECTION_DATA *psConnection,
+					   PVRSRV_DEVICE_NODE *psDevNode,
+					   IMG_BOOL bServerSync,
+                       IMG_UINT32 ui32FWAddr,
+                       IMG_UINT32 ui32ClassNameSize,
+                       const IMG_CHAR *pszClassName);
+
+PVRSRV_ERROR
+PVRSRVSyncFreeEventKM(CONNECTION_DATA *psConnection,
+					   PVRSRV_DEVICE_NODE *psDevNode,
+					   IMG_UINT32 ui32FWAddr);
+
+PVRSRV_ERROR
+PVRSRVSyncRecordAddKM(CONNECTION_DATA *psConnection,
+					  PVRSRV_DEVICE_NODE *psDevNode,
+					  SYNC_RECORD_HANDLE *phRecord,
+					  SYNC_PRIMITIVE_BLOCK *hServerSyncPrimBlock,
+					  IMG_UINT32 ui32FwBlockAddr,
+					  IMG_UINT32 ui32SyncOffset,
+					  IMG_BOOL bServerSync,
+					  IMG_UINT32 ui32ClassNameSize,
+					  const IMG_CHAR *pszClassName);
+
+PVRSRV_ERROR
+PVRSRVSyncRecordRemoveByHandleKM(
+			SYNC_RECORD_HANDLE hRecord);
+void SyncRecordLookup(PVRSRV_DEVICE_NODE *psDevNode, IMG_UINT32 ui32FwAddr,
+					  IMG_CHAR * pszSyncInfo, size_t len);
+
+PVRSRV_ERROR SyncRegisterConnection(SYNC_CONNECTION_DATA **ppsSyncConnectionData);
+void SyncUnregisterConnection(SYNC_CONNECTION_DATA *ppsSyncConnectionData);
+void SyncConnectionPDumpSyncBlocks(PVRSRV_DEVICE_NODE *psDevNode, void *hSyncPrivData, PDUMP_TRANSITION_EVENT eEvent);
+
+/*!
+******************************************************************************
+@Function      SyncServerInit
+
+@Description   Per-device initialisation for the ServerSync module
+******************************************************************************/
+PVRSRV_ERROR SyncServerInit(PVRSRV_DEVICE_NODE *psDevNode);
+void SyncServerDeinit(PVRSRV_DEVICE_NODE *psDevNode);
+
+
+#if defined(PDUMP)
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+							IMG_UINT32 ui32Value);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiDumpFlags);
+
+PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize);
+
+#else	/* PDUMP */
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpValueKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpValueKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+							IMG_UINT32 ui32Value)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpPolKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpPolKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT32 ui32Offset,
+						 IMG_UINT32 ui32Value, IMG_UINT32 ui32Mask,
+						 PDUMP_POLL_OPERATOR eOperator,
+						 PDUMP_FLAGS_T uiDumpFlags)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+	PVR_UNREFERENCED_PARAMETER(ui32Mask);
+	PVR_UNREFERENCED_PARAMETER(eOperator);
+	PVR_UNREFERENCED_PARAMETER(uiDumpFlags);
+	return PVRSRV_OK;
+}
+
+#ifdef INLINE_IS_PRAGMA
+#pragma inline(PVRSRVSyncPrimPDumpCBPKM)
+#endif
+static INLINE PVRSRV_ERROR
+PVRSRVSyncPrimPDumpCBPKM(SYNC_PRIMITIVE_BLOCK *psSyncBlk, IMG_UINT64 ui32Offset,
+						 IMG_UINT64 uiWriteOffset, IMG_UINT64 uiPacketSize,
+						 IMG_UINT64 uiBufferSize)
+{
+	PVR_UNREFERENCED_PARAMETER(psSyncBlk);
+	PVR_UNREFERENCED_PARAMETER(ui32Offset);
+	PVR_UNREFERENCED_PARAMETER(uiWriteOffset);
+	PVR_UNREFERENCED_PARAMETER(uiPacketSize);
+	PVR_UNREFERENCED_PARAMETER(uiBufferSize);
+	return PVRSRV_OK;
+}
+#endif	/* PDUMP */
+#endif	/*SYNC_SERVER_H */
diff --git a/drivers/gpu/drm/img-rogue/syscommon.h b/drivers/gpu/drm/img-rogue/syscommon.h
new file mode 100644
index 000000000000..b7babda19cab
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/syscommon.h
@@ -0,0 +1,175 @@
+/**************************************************************************/ /*!
+@File
+@Title          Common System APIs and structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides common system-specific declarations and
+                macros that are supported by all systems
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /***************************************************************************/
+
+#if !defined(SYSCOMMON_H)
+#define SYSCOMMON_H
+
+#include "img_types.h"
+#include "pvr_notifier.h"
+#include "pvrsrv_device.h"
+#include "pvrsrv_error.h"
+
+/*************************************************************************/ /*!
+@Description    Pointer to a Low-level Interrupt Service Routine (LISR).
+@Input  pvData  Private data provided to the LISR.
+@Return         True if interrupt handled, false otherwise.
+*/ /**************************************************************************/
+typedef IMG_BOOL (*PFN_LISR)(void *pvData);
+
+/**************************************************************************/ /*!
+@Function       SysDevInit
+@Description    System specific device initialisation function.
+@Input          pvOSDevice          pointer to the OS device reference
+@Input          ppsDevConfig        returned device configuration info
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysDevInit(void *pvOSDevice, PVRSRV_DEVICE_CONFIG **ppsDevConfig);
+
+/**************************************************************************/ /*!
+@Function       SysDevDeInit
+@Description    System specific device deinitialisation function.
+@Input          psDevConfig        device configuration info of the device to be
+                                   deinitialised
+@Return         None.
+*/ /***************************************************************************/
+void SysDevDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/**************************************************************************/ /*!
+@Function       SysDebugInfo
+@Description    Dump system specific device debug information.
+@Input          psDevConfig         pointer to device configuration info
+@Input          pfnDumpDebugPrintf  the 'printf' function to be called to
+                                    display the debug info
+@Input          pvDumpDebugFile     optional file identifier to be passed to
+                                    the 'printf' function if required
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysDebugInfo(PVRSRV_DEVICE_CONFIG *psDevConfig,
+				DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
+				void *pvDumpDebugFile);
+
+/**************************************************************************/ /*!
+@Function       SysInstallDeviceLISR
+@Description    Installs the system Low-level Interrupt Service Routine (LISR)
+                which handles low-level processing of interrupts from the device
+                (GPU).
+                The LISR will be invoked when the device raises an interrupt. An
+                LISR may not be descheduled, so code which needs to do so should
+                be placed in an MISR.
+                The installed LISR will schedule any MISRs once it has completed
+                its interrupt processing, by calling OSScheduleMISR().
+@Input          hSysData      pointer to the system data of the device
+@Input          ui32IRQ       the IRQ on which the LISR is to be installed
+@Input          pszName       name of the module installing the LISR
+@Input          pfnLISR       pointer to the function to be installed as the
+                              LISR
+@Input          pvData        private data provided to the LISR
+@Output         phLISRData    handle to the installed LISR (to be used for a
+                              subsequent uninstall)
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysInstallDeviceLISR(IMG_HANDLE hSysData,
+								  IMG_UINT32 ui32IRQ,
+								  const IMG_CHAR *pszName,
+								  PFN_LISR pfnLISR,
+								  void *pvData,
+								  IMG_HANDLE *phLISRData);
+
+/**************************************************************************/ /*!
+@Function       SysUninstallDeviceLISR
+@Description    Uninstalls the system Low-level Interrupt Service Routine (LISR)
+                which handles low-level processing of interrupts from the device
+                (GPU).
+@Input          hLISRData     handle of the LISR to be uninstalled
+@Return         PVRSRV_OK on success, a failure code otherwise.
+*/ /***************************************************************************/
+PVRSRV_ERROR SysUninstallDeviceLISR(IMG_HANDLE hLISRData);
+
+/**************************************************************************/ /*!
+@Function       SysRGXErrorNotify
+@Description    Error reporting callback function, registered as the
+                pfnSysDevErrorNotify member of the PVRSRV_DEVICE_CONFIG
+                struct. System layer will be notified of device errors and
+                resets via this callback.
+                NB. implementers should ensure that the minimal amount of
+                work is done in this callback function, as it will be
+                executed in the main RGX MISR. (e.g. any blocking or lengthy
+                work should be performed by a worker queue/thread instead).
+@Input          hSysData      pointer to the system data of the device
+@Output         psErrorData   structure containing details of the reported error
+@Return         None.
+*/ /***************************************************************************/
+void SysRGXErrorNotify(IMG_HANDLE hSysData,
+                       PVRSRV_ROBUSTNESS_NOTIFY_DATA *psErrorData);
+
+/**************************************************************************/ /*!
+@Function       SysRestrictGpuLocalPhysheap
+@Description    If the Restriction apphint has been set, validate the
+                restriction value and return the new GPU_LOCAL heap size.
+
+@Input          uiHeapSize      Current syslayer detected GPU_LOCAL heap size.
+@Return         IMG_UINT64      New GPU_LOCAL heap size in bytes.
+*/ /***************************************************************************/
+IMG_UINT64 SysRestrictGpuLocalPhysheap(IMG_UINT64 uiHeapSize);
+
+/**************************************************************************/ /*!
+@Function       SysRestrictGpuLocalAddPrivateHeap
+@Description    Determine if the restriction apphint has been set.
+
+@Return         IMG_BOOL        IMG_TRUE if the restriction apphint has been
+                                set.
+*/ /***************************************************************************/
+IMG_BOOL SysRestrictGpuLocalAddPrivateHeap(void);
+
+/**************************************************************************/ /*!
+@Function       SysDefaultToCpuLocalHeap
+@Description    Determine if the Default Heap should be CPU_LOCAL
+                Can only be used on TC_MEMORY_HYBRID systems.
+
+@Return         IMG_BOOL        IMG_TRUE if the Default heap apphint has been
+                                set.
+*/ /***************************************************************************/
+IMG_BOOL SysDefaultToCpuLocalHeap(void);
+
+#endif /* !defined(SYSCOMMON_H) */
diff --git a/drivers/gpu/drm/img-rogue/sysconfig_cmn.c b/drivers/gpu/drm/img-rogue/sysconfig_cmn.c
new file mode 100644
index 000000000000..d8a09182dc04
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sysconfig_cmn.c
@@ -0,0 +1,167 @@
+/*************************************************************************/ /*!
+@File
+@Title          Sysconfig layer common to all platforms
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implements system layer functions common to all platforms
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv.h"
+#include "pvrsrv_device.h"
+#include "syscommon.h"
+#include "pvr_debug.h"
+#include "os_apphint.h"
+//#include "physmem.h"
+
+void SysRGXErrorNotify(IMG_HANDLE hSysData,
+                       PVRSRV_ROBUSTNESS_NOTIFY_DATA *psErrorData)
+{
+	PVR_UNREFERENCED_PARAMETER(hSysData);
+
+#if defined(PVRSRV_NEED_PVR_DPF)
+	{
+		IMG_UINT32 ui32DgbLvl;
+
+		switch (psErrorData->eResetReason)
+		{
+			case RGX_CONTEXT_RESET_REASON_NONE:
+			case RGX_CONTEXT_RESET_REASON_GUILTY_LOCKUP:
+			case RGX_CONTEXT_RESET_REASON_INNOCENT_LOCKUP:
+			case RGX_CONTEXT_RESET_REASON_GUILTY_OVERRUNING:
+			case RGX_CONTEXT_RESET_REASON_INNOCENT_OVERRUNING:
+			case RGX_CONTEXT_RESET_REASON_HARD_CONTEXT_SWITCH:
+			case RGX_CONTEXT_RESET_REASON_GPU_ECC_OK:
+			case RGX_CONTEXT_RESET_REASON_FW_ECC_OK:
+			{
+				ui32DgbLvl = PVR_DBG_MESSAGE;
+				break;
+			}
+			case RGX_CONTEXT_RESET_REASON_GPU_ECC_HWR:
+			case RGX_CONTEXT_RESET_REASON_FW_EXEC_ERR:
+			case RGX_CONTEXT_RESET_REASON_GPU_PARITY_HWR:
+			case RGX_CONTEXT_RESET_REASON_GPU_LATENT_HWR:
+			{
+				ui32DgbLvl = PVR_DBG_WARNING;
+				break;
+			}
+			case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_FW_ECC_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_PTE_PARITY_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_PARITY_ERR:
+			case RGX_CONTEXT_RESET_REASON_FW_WATCHDOG:
+			case RGX_CONTEXT_RESET_REASON_FW_PAGEFAULT:
+			case RGX_CONTEXT_RESET_REASON_HOST_WDG_FW_ERR:
+			case RGX_CONTEXT_PVRIC_SIGNATURE_MISMATCH:
+			case RGX_CONTEXT_RESET_REASON_DCLS_ERR:
+			{
+				ui32DgbLvl = PVR_DBG_ERROR;
+				break;
+			}
+			default:
+			{
+				PVR_ASSERT(false && "Unhandled reset reason");
+				ui32DgbLvl = PVR_DBG_ERROR;
+				break;
+			}
+		}
+
+		if (psErrorData->pid > 0)
+		{
+			PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, " PID %d experienced error %d",
+					 psErrorData->pid, psErrorData->eResetReason);
+		}
+		else
+		{
+			PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, " Device experienced error %d",
+					 psErrorData->eResetReason);
+		}
+
+		switch (psErrorData->eResetReason)
+		{
+			case RGX_CONTEXT_RESET_REASON_WGP_CHECKSUM:
+			case RGX_CONTEXT_RESET_REASON_TRP_CHECKSUM:
+			{
+				PVRSRVDebugPrintf(ui32DgbLvl, __FILE__, __LINE__, "   ExtJobRef 0x%x, DM %d",
+						 psErrorData->uErrData.sChecksumErrData.ui32ExtJobRef,
+						 psErrorData->uErrData.sChecksumErrData.eDM);
+			break;
+			}
+			default:
+			{
+				break;
+			}
+		}
+	}
+#else
+	PVR_UNREFERENCED_PARAMETER(psErrorData);
+#endif /* PVRSRV_NEED_PVR_DPF */
+}
+
+IMG_UINT64 SysRestrictGpuLocalPhysheap(IMG_UINT64 uiHeapSize)
+{
+	return uiHeapSize;
+}
+
+IMG_BOOL SysRestrictGpuLocalAddPrivateHeap(void)
+{
+	return IMG_FALSE;
+}
+
+IMG_BOOL SysDefaultToCpuLocalHeap(void)
+{
+//#if (TC_MEMORY_CONFIG == TC_MEMORY_HYBRID)
+	void *pvAppHintState = NULL;
+	IMG_BOOL bAppHintDefault = IMG_FALSE;
+	IMG_BOOL bSetToCPULocal = IMG_FALSE;
+
+	OSCreateAppHintState(&pvAppHintState);
+	OSGetAppHintBOOL(APPHINT_NO_DEVICE, pvAppHintState,
+			PhysHeapHybridDefault2CpuLocal, &bAppHintDefault, &bSetToCPULocal);
+	OSFreeAppHintState(pvAppHintState);
+
+	return bSetToCPULocal;
+//#else
+//	return IMG_FALSE;
+//#endif
+}
+/******************************************************************************
+ End of file (sysconfig_cmn.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/sysvalidation.h b/drivers/gpu/drm/img-rogue/sysvalidation.h
new file mode 100644
index 000000000000..e3e731e8b810
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/sysvalidation.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File
+@Title          Validation System APIs and structures
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides system-specific declarations and macros
+                needed for hardware validation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(SYSVALIDATION_H)
+#define SYSVALIDATION_H
+
+#if defined(SUPPORT_GPUVIRT_VALIDATION)
+#include "img_types.h"
+#include "rgxdefs_km.h"
+#include "virt_validation_defs.h"
+
+void SysInitFirewall(IMG_HANDLE hSysData,
+                     IMG_UINT64 aui64OSidMin[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS],
+                     IMG_UINT64 aui64OSidMax[GPUVIRT_VALIDATION_NUM_REGIONS][GPUVIRT_VALIDATION_NUM_OS]);
+
+#if defined(EMULATOR)
+void SysSetAxiProtOSid(IMG_HANDLE hSysData, IMG_UINT32 ui32OSid, IMG_BOOL bState);
+void SysSetTrustedDeviceAceEnabled(IMG_HANDLE hSysData);
+#endif
+#endif /* defined(SUPPORT_GPUVIRT_VALIDATION) */
+
+#endif /* !defined(SYSVALIDATION_H) */
diff --git a/drivers/gpu/drm/img-rogue/tlclient.c b/drivers/gpu/drm/img-rogue/tlclient.c
new file mode 100644
index 000000000000..a72f8118483b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlclient.c
@@ -0,0 +1,499 @@
+/*************************************************************************/ /*!
+@File           tlclient.c
+@Title          Services Transport Layer shared API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common API used in both clients and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+/* DESIGN NOTE
+ * This transport layer consumer-role API was created as a shared API when a
+ * client wanted to read the data of a TL stream from within the KM server
+ * driver. This was in addition to the existing clients supported externally
+ * by the UM client library component via PVR API layer.
+ * This shared API is thus used by the PVR TL API in the client library and
+ * by clients internal to the server driver module. It depends on
+ * client entry points of the TL and DEVMEM bridge modules. These entry points
+ * encapsulate from the TL shared API whether a direct bridge or an indirect
+ * (ioctl) bridge is used.
+ * One reason for needing this layer centres around the fact that some of the
+ * API functions make multiple bridge calls and the logic that glues these
+ * together is common regardless of client location. Further this layer has
+ * allowed the defensive coding that checks parameters to move into the PVR
+ * API layer where untrusted clients enter giving a more efficient KM code path.
+ */
+
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvr_debug.h"
+#include "osfunc.h"
+
+#include "allocmem.h"
+#include "devicemem.h"
+
+#include "tlclient.h"
+#include "pvrsrv_tlcommon.h"
+#include "client_pvrtl_bridge.h"
+
+#if defined(__KERNEL__)
+#include "srvcore.h"
+#else
+#include "srvcore_intern.h"
+#endif
+
+/* Defines/Constants
+ */
+
+#define NO_ACQUIRE             0xffffffffU
+
+/* User-side stream descriptor structure.
+ */
+typedef struct _TL_STREAM_DESC_
+{
+	/* Handle on kernel-side stream descriptor*/
+	IMG_HANDLE		hServerSD;
+
+	/* Stream data buffer variables */
+	DEVMEM_MEMDESC*			psUMmemDesc;
+	IMG_PBYTE				pBaseAddr;
+
+	/* Offset in bytes into the circular buffer and valid only after
+	 * an Acquire call and undefined after a release. */
+	IMG_UINT32	uiReadOffset;
+
+	/* Always a positive integer when the Acquire call returns and a release
+	 * is outstanding. Undefined at all other times. */
+	IMG_UINT32	uiReadLen;
+
+	/* Counter indicating how many writes to a stream failed.
+	 * It's used to reduce number of errors in output log. */
+	IMG_UINT32 ui32WritesFailed;
+
+	/* Name of the stream. */
+	IMG_CHAR szName[PRVSRVTL_MAX_STREAM_NAME_SIZE];
+} TL_STREAM_DESC, *PTL_STREAM_DESC;
+
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientOpenStream(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR* pszName,
+		IMG_UINT32   ui32Mode,
+		IMG_HANDLE*  phSD)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	TL_STREAM_DESC *psSD = NULL;
+	IMG_HANDLE hTLPMR;
+	IMG_HANDLE hTLImportHandle;
+	IMG_DEVMEM_SIZE_T uiImportSize;
+	PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(pszName);
+	PVR_ASSERT(phSD);
+	*phSD = NULL;
+
+	/* Allocate memory for the stream descriptor object, initialise with
+	 * "no data read" yet. */
+	psSD = OSAllocZMem(sizeof(TL_STREAM_DESC));
+	PVR_LOG_GOTO_IF_NOMEM(psSD, eError, e0);
+	psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+
+	/* Send open stream request to kernel server to get stream handle and
+	 * buffer cookie so we can get access to the buffer in this process. */
+	eError = BridgeTLOpenStream(GetBridgeHandle(hDevConnection), pszName,
+			ui32Mode, &psSD->hServerSD, &hTLPMR);
+	if (eError != PVRSRV_OK)
+	{
+		if ((ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WAIT) &&
+			(eError == PVRSRV_ERROR_TIMEOUT))
+		{
+			goto e1;
+		}
+		PVR_LOG_GOTO_IF_ERROR(eError, "BridgeTLOpenStream", e1);
+	}
+
+	/* Convert server export cookie into a cookie for use by this client */
+	eError = DevmemMakeLocalImportHandle(hDevConnection,
+			hTLPMR, &hTLImportHandle);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemMakeLocalImportHandle", e2);
+
+	uiMemFlags |= ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	        PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE : 0ULL;
+	/* Now convert client cookie into a client handle on the buffer's
+	 * physical memory region */
+	eError = DevmemLocalImport(hDevConnection,
+	                           hTLImportHandle,
+	                           uiMemFlags,
+	                           &psSD->psUMmemDesc,
+	                           &uiImportSize,
+	                           "TLBuffer");
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemImport", e3);
+
+	/* Now map the memory into the virtual address space of this process. */
+	eError = DevmemAcquireCpuVirtAddr(psSD->psUMmemDesc, (void **)
+															&psSD->pBaseAddr);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", e4);
+
+	/* Ignore error, not much that can be done */
+	(void) DevmemUnmakeLocalImportHandle(hDevConnection,
+			hTLImportHandle);
+
+	/* Copy stream name */
+	OSStringSafeCopy(psSD->szName, pszName, PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	/* Return client descriptor handle to caller */
+	*phSD = psSD;
+	return PVRSRV_OK;
+
+/* Clean up post buffer setup */
+e4:
+	DevmemFree(psSD->psUMmemDesc);
+e3:
+	(void) DevmemUnmakeLocalImportHandle(hDevConnection,
+				&hTLImportHandle);
+/* Clean up post stream open */
+e2:
+	BridgeTLCloseStream(GetBridgeHandle(hDevConnection), psSD->hServerSD);
+
+/* Clean up post allocation of the descriptor object */
+e1:
+	OSFreeMem(psSD);
+
+e0:
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCloseStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	/* Check the caller provided connection is valid */
+	if (!psSD->hServerSD)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: descriptor already "
+				"closed/not open", __func__));
+		return PVRSRV_ERROR_HANDLE_NOT_FOUND;
+	}
+
+	/* Check if acquire is outstanding, perform release if it is, ignore result
+	 * as there is not much we can do if it is an error other than close */
+	if (psSD->uiReadLen != NO_ACQUIRE)
+	{
+		(void) BridgeTLReleaseData(GetBridgeHandle(hDevConnection),
+				psSD->hServerSD, psSD->uiReadOffset, psSD->uiReadLen);
+		psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+	}
+
+	/* Clean up DevMem resources used for this stream in this client */
+	DevmemReleaseCpuVirtAddr(psSD->psUMmemDesc);
+
+	DevmemFree(psSD->psUMmemDesc);
+
+	/* Send close to server to clean up kernel mode resources for this
+	 * handle and release the memory. */
+	eError = DestroyServerResource(hDevConnection,
+	                               NULL,
+	                               BridgeTLCloseStream,
+	                               psSD->hServerSD);
+	PVR_LOG_IF_ERROR(eError, "BridgeTLCloseStream");
+
+	if (psSD->ui32WritesFailed != 0)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s() %u writes failed to stream %s (%c)",
+		        __func__, psSD->ui32WritesFailed, psSD->szName,
+		        psSD->ui32WritesFailed == IMG_UINT32_MAX ? 'T' : 'F'));
+	}
+
+	OSCachedMemSet(psSD, 0x00, sizeof(TL_STREAM_DESC));
+	OSFreeMem(psSD);
+
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientDiscoverStreams(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR *pszNamePattern,
+		IMG_CHAR aszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+		IMG_UINT32 *pui32NumFound)
+{
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(pszNamePattern);
+	PVR_ASSERT(pui32NumFound);
+
+	return BridgeTLDiscoverStreams(GetBridgeHandle(hDevConnection),
+			pszNamePattern,
+			/* we need to treat this as one dimensional array */
+			*pui32NumFound * PRVSRVTL_MAX_STREAM_NAME_SIZE,
+			(IMG_CHAR *) aszStreams,
+			pui32NumFound);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+	IMG_UINT32 ui32BufferOffset, ui32Unused;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppui8Data);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLReserveStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &ui32BufferOffset, ui32Size, ui32Size, &ui32Unused);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppui8Data = psSD->pBaseAddr + ui32BufferOffset;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream2(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size,
+		IMG_UINT32 ui32SizeMin,
+		IMG_UINT32 *pui32Available)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+	IMG_UINT32 ui32BufferOffset;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppui8Data);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLReserveStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &ui32BufferOffset, ui32Size, ui32SizeMin,
+			pui32Available);
+	PVR_RETURN_IF_ERROR(eError);
+
+	*ppui8Data = psSD->pBaseAddr + ui32BufferOffset;
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCommitStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ui32Size);
+
+	eError = BridgeTLCommitStream(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, ui32Size);
+	PVR_RETURN_IF_ERROR(eError);
+
+	return PVRSRV_OK;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientAcquireData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE  hSD,
+		IMG_PBYTE*  ppPacketBuf,
+		IMG_UINT32* pui32BufLen)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ppPacketBuf);
+	PVR_ASSERT(pui32BufLen);
+
+	/* In case of non-blocking acquires, which can return no data, and
+	 * error paths ensure we clear the output parameters first. */
+	*ppPacketBuf = NULL;
+	*pui32BufLen = 0;
+
+	/* Check Acquire has not been called twice in a row without a release */
+	if (psSD->uiReadOffset != NO_ACQUIRE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: acquire already "
+				"outstanding, ReadOffset(%d), ReadLength(%d)",
+				__func__, psSD->uiReadOffset, psSD->uiReadLen));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Ask the kernel server for the next chunk of data to read */
+	eError = BridgeTLAcquireData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, &psSD->uiReadOffset, &psSD->uiReadLen);
+	if (eError != PVRSRV_OK)
+	{
+		/* Mask reporting of the errors seen under normal operation */
+		if ((eError != PVRSRV_ERROR_TIMEOUT) &&
+			(eError != PVRSRV_ERROR_STREAM_READLIMIT_REACHED))
+		{
+			PVR_LOG_ERROR(eError, "BridgeTLAcquireData");
+		}
+		psSD->uiReadOffset = psSD->uiReadLen = NO_ACQUIRE;
+		return eError;
+	}
+	/* else PVRSRV_OK */
+
+	/* Return the data offset and length to the caller if bytes are available
+	 * to be read. Could be zero for non-blocking mode so pass back cleared
+	 * values above */
+	if (psSD->uiReadLen)
+	{
+		*ppPacketBuf = psSD->pBaseAddr + psSD->uiReadOffset;
+		*pui32BufLen = psSD->uiReadLen;
+	}
+
+	return PVRSRV_OK;
+}
+
+static PVRSRV_ERROR _TLClientReleaseDataLen(
+		SHARED_DEV_CONNECTION hDevConnection,
+		TL_STREAM_DESC* psSD,
+		IMG_UINT32 uiReadLen)
+{
+	PVRSRV_ERROR eError;
+
+	/* the previous acquire did not return any data, this is a no-operation */
+	if (psSD->uiReadLen == 0)
+	{
+		return PVRSRV_OK;
+	}
+
+	/* Check release has not been called twice in a row without an acquire */
+	if (psSD->uiReadOffset == NO_ACQUIRE)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: no acquire to release", __func__));
+		return PVRSRV_ERROR_RETRY;
+	}
+
+	/* Inform the kernel to release the data from the buffer */
+	eError = BridgeTLReleaseData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD,
+			psSD->uiReadOffset, uiReadLen);
+	PVR_LOG_IF_ERROR(eError, "BridgeTLReleaseData");
+
+	/* Reset state to indicate no outstanding acquire */
+	psSD->uiReadLen = psSD->uiReadOffset = NO_ACQUIRE;
+
+	return eError;
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD)
+{
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	return _TLClientReleaseDataLen(hDevConnection, psSD, psSD->uiReadLen);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseDataLess(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD, IMG_UINT32 uiActualReadLen)
+{
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+
+	/* Check the specified size is within the size returned by Acquire */
+	if (uiActualReadLen > psSD->uiReadLen)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: no acquire to release", __func__));
+		return PVRSRV_ERROR_INVALID_PARAMS;
+	}
+
+	return _TLClientReleaseDataLen(hDevConnection, psSD, uiActualReadLen);
+}
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientWriteData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size,
+		IMG_BYTE *pui8Data)
+{
+	PVRSRV_ERROR eError;
+	TL_STREAM_DESC* psSD = (TL_STREAM_DESC*) hSD;
+
+	PVR_ASSERT(hDevConnection);
+	PVR_ASSERT(hSD);
+	PVR_ASSERT(ui32Size);
+	PVR_ASSERT(pui8Data);
+
+	eError = BridgeTLWriteData(GetBridgeHandle(hDevConnection),
+			psSD->hServerSD, ui32Size, pui8Data);
+
+	if (eError == PVRSRV_ERROR_STREAM_FULL)
+	{
+		if (psSD->ui32WritesFailed == 0)
+		{
+			PVR_LOG_ERROR(eError, "BridgeTLWriteData");
+		}
+		if (psSD->ui32WritesFailed != IMG_UINT32_MAX)
+		{
+			psSD->ui32WritesFailed++;
+		}
+	}
+	else if (eError != PVRSRV_OK)
+	{
+		PVR_LOG_ERROR(eError, "BridgeTLWriteData");
+	}
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (tlclient.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/tlclient.h b/drivers/gpu/drm/img-rogue/tlclient.h
new file mode 100644
index 000000000000..00f7aa8fc043
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlclient.h
@@ -0,0 +1,257 @@
+/*************************************************************************/ /*!
+@File           tlclient.h
+@Title          Services Transport Layer shared API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport layer common API used in both clients and server
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef TLCLIENT_H
+#define TLCLIENT_H
+
+
+#include "img_defs.h"
+#include "pvrsrv_tlcommon.h"
+#include "pvrsrv_error.h"
+
+
+/* This value is used for the hSrvHandle argument in the client API when
+ * called directly from the kernel which will lead to a direct bridge access.
+ */
+#define DIRECT_BRIDGE_HANDLE	((IMG_HANDLE)0xDEADBEEFU)
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientOpenStream
+ @Description   Open a descriptor onto an existing kernel transport stream.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         pszName         Address of the stream name string, no longer
+                                than PRVSRVTL_MAX_STREAM_NAME_SIZE.
+ @Input         ui32Mode        Unused
+ @Output        phSD            Address of a pointer to an stream object
+ @Return        PVRSRV_ERROR_NOT_FOUND          when named stream not found
+ @Return        PVRSRV_ERROR_ALREADY_OPEN       stream already open by another
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_TIMEOUT            timed out, stream not found
+ @Return        PVRSRV_ERROR                    for other system codes
+*/ /**************************************************************************/
+
+IMG_INTERNAL
+PVRSRV_ERROR TLClientOpenStream(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR* pszName,
+		IMG_UINT32   ui32Mode,
+		IMG_HANDLE*  phSD);
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientCloseStream
+ @Description   Close and release the stream connection to Services kernel
+                server transport layer. Any outstanding Acquire will be
+                released.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle is not known
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCloseStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD);
+
+/*************************************************************************/ /*!
+ @Function      TLClientDiscoverStreams
+ @Description   Finds all streams that's name starts with pszNamePattern and
+                ends with a number.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         pszNamePattern  Name pattern. Must be beginning of a string.
+ @Output        aszStreams      Array of numbers from end of the discovered
+                names.
+ @inOut         pui32NumFound   When input, max number that can fit into
+                                pui32Streams. When output, number of
+                                discovered streams.
+ @Return        PVRSRV_ERROR    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientDiscoverStreams(SHARED_DEV_CONNECTION hDevConnection,
+		const IMG_CHAR *pszNamePattern,
+		IMG_CHAR aszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+		IMG_UINT32 *pui32NumFound);
+
+/*************************************************************************/ /*!
+ @Function      TLClientReserveStream
+ @Description   Reserves a region with given size in the stream. If the stream
+                is already reserved the function will return an error.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Output        ppui8Data       pointer to the buffer
+ @Input         ui32Size        size of the data
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLClientStreamReserve2
+ @Description   Reserves a region with given size in the stream. If the stream
+                is already reserved the function will return an error.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Output        ppui8Data       pointer to the buffer
+ @Input         ui32Size        size of the data
+ @Input         ui32SizeMin     minimum size of the data
+ @Input         ui32Available   available space in buffer
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReserveStream2(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT8 **ppui8Data,
+		IMG_UINT32 ui32Size,
+		IMG_UINT32 ui32SizeMin,
+		IMG_UINT32 *pui32Available);
+
+/*************************************************************************/ /*!
+ @Function      TLClientStreamCommit
+ @Description   Commits previously reserved region in the stream and therefore
+                allows next reserves.
+                This function call has to be preceded by the call to
+                TLClientReserveStream or TLClientReserveStream2.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to close
+ @Input         ui32Size        Size of the data
+ @Return
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientCommitStream(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLClientAcquireData
+ @Description   When there is data available in the stream buffer this call
+                returns with the address and length of the data buffer the
+                client can safely read. This buffer may contain one or more
+                packets of data.
+                If no data is available then this call blocks until it becomes
+                available. However if the stream has been destroyed while
+                waiting then a resource unavailable error will be returned to
+                the caller. Clients must pair this call with a ReleaseData
+                call.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Output        ppPacketBuf     Address of a pointer to an byte buffer. On exit
+                                pointer contains address of buffer to read from
+ @Output        puiBufLen       Pointer to an integer. On exit it is the size
+                                of the data to read from the packet buffer
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              release not called beforehand
+ @Return        PVRSRV_ERROR_TIMEOUT            block timed out, no data
+ @Return        PVRSRV_ERROR                    for other system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientAcquireData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE  hSD,
+		IMG_PBYTE*  ppPacketBuf,
+		IMG_UINT32* puiBufLen);
+
+
+/*************************************************************************/ /*!
+ @Function      TLClientReleaseData
+ @Description   Called after client has read the stream data out of the buffer
+                The data is subsequently flushed from the stream buffer to make
+                room for more data packets from the stream source.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known to TL
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              acquire not called beforehand
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD);
+
+/*************************************************************************/ /*!
+ @Function      TLClientReleaseDataLess
+ @Description   Called after client has read only some data out of the buffer
+                and wishes to complete the read early i.e. does not want to
+                read the full data that the acquire call returned e.g read just
+                one packet from the stream.
+                The data is subsequently flushed from the stream buffer to make
+                room for more data packets from the stream source.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Input         uiActualReadLen Size of data read, in bytes. Must be on a TL
+                                packet boundary.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS     when read length too big
+ @Return        PVRSRV_ERROR_RESOURCE_UNAVAILABLE  when stream no longer exists
+ @Return        PVRSRV_ERROR_HANDLE_NOT_FOUND   when SD handle not known to TL
+ @Return        PVRSRV_ERROR_STREAM_ERROR       internal driver state error
+ @Return        PVRSRV_ERROR_RETRY              acquire not called beforehand
+ @Return        PVRSRV_ERROR                    for system codes
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientReleaseDataLess(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD, IMG_UINT32 uiActualReadLen);
+
+/*************************************************************************/ /*!
+ @Function      TLClientWriteData
+ @Description   Writes data to the stream.
+ @Input         hDevConnection  Address of a pointer to a connection object
+ @Input         hSD             Handle of the stream object to read
+ @Input         ui32Size        Size of the data
+ @Input         pui8Data        Pointer to data
+*/ /**************************************************************************/
+IMG_INTERNAL
+PVRSRV_ERROR TLClientWriteData(SHARED_DEV_CONNECTION hDevConnection,
+		IMG_HANDLE hSD,
+		IMG_UINT32 ui32Size,
+		IMG_BYTE *pui8Data);
+
+
+#endif /* TLCLIENT_H */
+
+/******************************************************************************
+ End of file (tlclient.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/tlintern.c b/drivers/gpu/drm/img-rogue/tlintern.c
new file mode 100644
index 000000000000..3bd6b0af1718
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlintern.c
@@ -0,0 +1,442 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer functions available to driver components in
+                the driver.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "allocmem.h"
+#include "pvrsrv_error.h"
+#include "osfunc.h"
+#include "devicemem.h"
+
+#include "pvrsrv_tlcommon.h"
+#include "tlintern.h"
+
+/*
+ * Make functions
+ */
+PTL_STREAM_DESC
+TLMakeStreamDesc(PTL_SNODE f1, IMG_UINT32 f2, IMG_HANDLE f3)
+{
+	PTL_STREAM_DESC ps = OSAllocZMem(sizeof(TL_STREAM_DESC));
+	if (ps == NULL)
+	{
+		return NULL;
+	}
+	ps->psNode = f1;
+	ps->ui32Flags = f2;
+	ps->hReadEvent = f3;
+	ps->uiRefCount = 1;
+
+	if (f2 & PVRSRV_STREAM_FLAG_READ_LIMIT)
+	{
+		ps->ui32ReadLimit = f1->psStream->ui32Write;
+	}
+	return ps;
+}
+
+PTL_SNODE
+TLMakeSNode(IMG_HANDLE f2, TL_STREAM *f3, TL_STREAM_DESC *f4)
+{
+	PTL_SNODE ps = OSAllocZMem(sizeof(TL_SNODE));
+	if (ps == NULL)
+	{
+		return NULL;
+	}
+	ps->hReadEventObj = f2;
+	ps->psStream = f3;
+	ps->psRDesc = f4;
+	f3->psNode = ps;
+	return ps;
+}
+
+/*
+ * Transport Layer Global top variables and functions
+ */
+static TL_GLOBAL_DATA sTLGlobalData;
+
+TL_GLOBAL_DATA *TLGGD(void) /* TLGetGlobalData() */
+{
+	return &sTLGlobalData;
+}
+
+/* TLInit must only be called once at driver initialisation.
+ * An assert is provided to check this condition on debug builds.
+ */
+PVRSRV_ERROR
+TLInit(void)
+{
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(sTLGlobalData.hTLGDLock == NULL && sTLGlobalData.hTLEventObj == NULL);
+
+	/* Allocate a lock for TL global data, to be used while updating the TL data.
+	 * This is for making TL global data multi-thread safe */
+	eError = OSLockCreate(&sTLGlobalData.hTLGDLock);
+	PVR_GOTO_IF_ERROR(eError, e0);
+
+	/* Allocate the event object used to signal global TL events such as
+	 * a new stream created */
+	eError = OSEventObjectCreate("TLGlobalEventObj", &sTLGlobalData.hTLEventObj);
+	PVR_GOTO_IF_ERROR(eError, e1);
+
+	PVR_DPF_RETURN_OK;
+
+/* Don't allow the driver to start up on error */
+e1:
+	OSLockDestroy (sTLGlobalData.hTLGDLock);
+	sTLGlobalData.hTLGDLock = NULL;
+e0:
+	PVR_DPF_RETURN_RC (eError);
+}
+
+static void RemoveAndFreeStreamNode(PTL_SNODE psRemove)
+{
+	TL_GLOBAL_DATA*  psGD = TLGGD();
+	PTL_SNODE*       last;
+	PTL_SNODE        psn;
+	PVRSRV_ERROR     eError;
+
+	PVR_DPF_ENTERED;
+
+	/* Unlink the stream node from the master list */
+	PVR_ASSERT(psGD->psHead);
+	last = &psGD->psHead;
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn == psRemove)
+		{
+			/* Other calling code may have freed and zeroed the pointers */
+			if (psn->psRDesc)
+			{
+				OSFreeMem(psn->psRDesc);
+				psn->psRDesc = NULL;
+			}
+			if (psn->psStream)
+			{
+				OSFreeMem(psn->psStream);
+				psn->psStream = NULL;
+			}
+			*last = psn->psNext;
+			break;
+		}
+		last = &psn->psNext;
+	}
+
+	/* Release the event list object owned by the stream node */
+	if (psRemove->hReadEventObj)
+	{
+		eError = OSEventObjectDestroy(psRemove->hReadEventObj);
+		PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+		psRemove->hReadEventObj = NULL;
+	}
+
+	/* Release the memory of the stream node */
+	OSFreeMem(psRemove);
+
+	PVR_DPF_RETURN;
+}
+
+static void FreeGlobalData(void)
+{
+	PTL_SNODE psCurrent = sTLGlobalData.psHead;
+	PTL_SNODE psNext;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	/* Clean up the SNODE list */
+	if (psCurrent)
+	{
+		while (psCurrent)
+		{
+			psNext = psCurrent->psNext;
+
+			/* Other calling code may have freed and zeroed the pointers */
+			if (psCurrent->psRDesc)
+			{
+				OSFreeMem(psCurrent->psRDesc);
+				psCurrent->psRDesc = NULL;
+			}
+			if (psCurrent->psStream)
+			{
+				OSFreeMem(psCurrent->psStream);
+				psCurrent->psStream = NULL;
+			}
+
+			/* Release the event list object owned by the stream node */
+			if (psCurrent->hReadEventObj)
+			{
+				eError = OSEventObjectDestroy(psCurrent->hReadEventObj);
+				PVR_LOG_IF_ERROR(eError, "OSEventObjectDestroy");
+
+				psCurrent->hReadEventObj = NULL;
+			}
+
+			OSFreeMem(psCurrent);
+			psCurrent = psNext;
+		}
+
+		sTLGlobalData.psHead = NULL;
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void
+TLDeInit(void)
+{
+	PVR_DPF_ENTERED;
+
+	if (sTLGlobalData.uiClientCnt)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "TLDeInit transport layer but %d client streams are still connected", sTLGlobalData.uiClientCnt));
+		sTLGlobalData.uiClientCnt = 0;
+	}
+
+	FreeGlobalData();
+
+	/* Clean up the TL global event object */
+	if (sTLGlobalData.hTLEventObj)
+	{
+		OSEventObjectDestroy(sTLGlobalData.hTLEventObj);
+		sTLGlobalData.hTLEventObj = NULL;
+	}
+
+	/* Destroy the TL global data lock */
+	if (sTLGlobalData.hTLGDLock)
+	{
+		OSLockDestroy (sTLGlobalData.hTLGDLock);
+		sTLGlobalData.hTLGDLock = NULL;
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void TLAddStreamNode(PTL_SNODE psAdd)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psAdd);
+	psAdd->psNext = TLGGD()->psHead;
+	TLGGD()->psHead = psAdd;
+
+	PVR_DPF_RETURN;
+}
+
+PTL_SNODE TLFindStreamNodeByName(const IMG_CHAR *pszName)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(pszName);
+
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn->psStream && OSStringNCompare(psn->psStream->szName, pszName, PRVSRVTL_MAX_STREAM_NAME_SIZE)==0)
+		{
+			PVR_DPF_RETURN_VAL(psn);
+		}
+	}
+
+	PVR_DPF_RETURN_VAL(NULL);
+}
+
+PTL_SNODE TLFindStreamNodeByDesc(PTL_STREAM_DESC psDesc)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psDesc);
+
+	for (psn = psGD->psHead; psn; psn=psn->psNext)
+	{
+		if (psn->psRDesc == psDesc || psn->psWDesc == psDesc)
+		{
+			PVR_DPF_RETURN_VAL(psn);
+		}
+	}
+	PVR_DPF_RETURN_VAL(NULL);
+}
+
+IMG_UINT32 TLDiscoverStreamNodes(const IMG_CHAR *pszNamePattern,
+                          IMG_CHAR aaszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+                          IMG_UINT32 ui32Max)
+{
+	TL_GLOBAL_DATA *psGD = TLGGD();
+	PTL_SNODE psn;
+	IMG_UINT32 ui32Count = 0;
+	size_t uiLen;
+
+	PVR_ASSERT(pszNamePattern);
+
+	if ((uiLen = OSStringLength(pszNamePattern)) == 0)
+		return 0;
+
+	for (psn = psGD->psHead; psn; psn = psn->psNext)
+	{
+		if (OSStringNCompare(pszNamePattern, psn->psStream->szName, uiLen) != 0)
+			continue;
+
+		/* If aaszStreams is NULL we only count how many string match
+		 * the given pattern. If it's a valid pointer we also return
+		 * the names. */
+		if (aaszStreams != NULL)
+		{
+			if (ui32Count >= ui32Max)
+				break;
+
+			/* all of names are shorter than MAX and null terminated */
+			OSStringSafeCopy(aaszStreams[ui32Count], psn->psStream->szName,
+			              PRVSRVTL_MAX_STREAM_NAME_SIZE);
+		}
+
+		ui32Count++;
+	}
+
+	return ui32Count;
+}
+
+PTL_SNODE TLFindAndGetStreamNodeByDesc(PTL_STREAM_DESC psDesc)
+{
+	PTL_SNODE psn;
+
+	PVR_DPF_ENTERED;
+
+	psn = TLFindStreamNodeByDesc(psDesc);
+	if (psn == NULL)
+		PVR_DPF_RETURN_VAL(NULL);
+
+	PVR_ASSERT(psDesc == psn->psWDesc);
+
+	psn->uiWRefCount++;
+	psDesc->uiRefCount++;
+
+	PVR_DPF_RETURN_VAL(psn);
+}
+
+void TLReturnStreamNode(PTL_SNODE psNode)
+{
+	psNode->uiWRefCount--;
+	psNode->psWDesc->uiRefCount--;
+
+	PVR_ASSERT(psNode->uiWRefCount > 0);
+	PVR_ASSERT(psNode->psWDesc->uiRefCount > 0);
+}
+
+IMG_BOOL TLTryRemoveStreamAndFreeStreamNode(PTL_SNODE psRemove)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psRemove);
+
+	/* If there is a client connected to this stream, defer stream's deletion */
+	if (psRemove->psRDesc != NULL || psRemove->psWDesc != NULL)
+	{
+		PVR_DPF_RETURN_VAL(IMG_FALSE);
+	}
+
+	/* Remove stream from TL_GLOBAL_DATA's list and free stream node */
+	psRemove->psStream = NULL;
+	RemoveAndFreeStreamNode(psRemove);
+
+	PVR_DPF_RETURN_VAL(IMG_TRUE);
+}
+
+IMG_BOOL TLUnrefDescAndTryFreeStreamNode(PTL_SNODE psNodeToRemove,
+                                          PTL_STREAM_DESC psSD)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psNodeToRemove);
+	PVR_ASSERT(psSD);
+
+	/* Decrement reference count. For descriptor obtained by reader it must
+	 * reach 0 (only single reader allowed) and for descriptors obtained by
+	 * writers it must reach value greater or equal to 0 (multiple writers
+	 * model). */
+	psSD->uiRefCount--;
+
+	if (psSD == psNodeToRemove->psRDesc)
+	{
+		PVR_ASSERT(0 == psSD->uiRefCount);
+		/* Remove stream descriptor (i.e. stream reader context) */
+		psNodeToRemove->psRDesc = NULL;
+	}
+	else if (psSD == psNodeToRemove->psWDesc)
+	{
+		PVR_ASSERT(0 <= psSD->uiRefCount);
+
+		psNodeToRemove->uiWRefCount--;
+
+		/* Remove stream descriptor if reference == 0 */
+		if (0 == psSD->uiRefCount)
+		{
+			psNodeToRemove->psWDesc = NULL;
+		}
+	}
+
+	/* Do not Free Stream Node if there is a write reference (a producer
+	 * context) to the stream */
+	if (NULL != psNodeToRemove->psRDesc || NULL != psNodeToRemove->psWDesc ||
+	    0 != psNodeToRemove->uiWRefCount)
+	{
+		PVR_DPF_RETURN_VAL(IMG_FALSE);
+	}
+
+	/* Make stream pointer NULL to prevent it from being destroyed in
+	 * RemoveAndFreeStreamNode. Cleanup of stream should be done by the
+	 * calling context */
+	psNodeToRemove->psStream = NULL;
+	RemoveAndFreeStreamNode(psNodeToRemove);
+
+	PVR_DPF_RETURN_VAL(IMG_TRUE);
+}
diff --git a/drivers/gpu/drm/img-rogue/tlintern.h b/drivers/gpu/drm/img-rogue/tlintern.h
new file mode 100644
index 000000000000..114f90f98548
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlintern.h
@@ -0,0 +1,347 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer internals
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer header used by TL internally
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TLINTERN_H
+#define TLINTERN_H
+
+
+#include "devicemem_typedefs.h"
+#include "pvrsrv_tlcommon.h"
+#include "lock.h"
+#include "tlstream.h"
+
+/* Forward declarations */
+typedef struct _TL_SNODE_* PTL_SNODE;
+
+/* To debug buffer utilisation enable this macro here and define
+ * PVRSRV_NEED_PVR_TRACE in the server pvr_debug.c and in tutils.c
+ * before the inclusion of pvr_debug.h.
+ * Issue pvrtutils 6 on target to see stream buffer utilisation. */
+//#define TL_BUFFER_STATS 1
+
+/*! TL stream structure container.
+ *    pbyBuffer   holds the circular buffer.
+ *    ui32Read    points to the beginning of the buffer, ie to where data to
+ *                  Read begin.
+ *    ui32Write   points to the end of data that have been committed, ie this is
+ *                  where new data will be written.
+ *    ui32Pending number of bytes reserved in last reserve call which have not
+ *                  yet been submitted. Therefore these data are not ready to
+ *                  be transported.
+ *    hStreamWLock - provides atomic protection for the ui32Pending & ui32Write
+ *                   members of the structure for when they are checked and/or
+ *                   updated in the context of a stream writer (producer)
+ *                   calling DoTLStreamReserve() & TLStreamCommit().
+ *                 - Reader context is not multi-threaded, only one client per
+ *                   stream is allowed. Also note the read context may be in an
+ *                   ISR which prevents a design where locks can be held in the
+ *                   AcquireData/ReleaseData() calls. Thus this lock only
+ *                   protects the stream members from simultaneous writers.
+ *
+ *      ui32Read < ui32Write <= ui32Pending
+ *        where < and <= operators are overloaded to make sense in a circular way.
+ */
+typedef struct _TL_STREAM_
+{
+	IMG_CHAR                szName[PRVSRVTL_MAX_STREAM_NAME_SIZE];  /*!< String name identifier */
+	TL_OPMODE               eOpMode;                                /*!< Mode of Operation of TL Buffer */
+
+	IMG_BOOL                bWaitForEmptyOnDestroy;                 /*!< Flag: On destroying a non-empty stream block until
+                                                                         *         stream is drained. */
+	IMG_BOOL                bNoSignalOnCommit;                      /*!< Flag: Used to avoid the TL signalling waiting consumers
+                                                                         *         that new data is available on every commit. Producers
+                                                                         *         using this flag will need to manually signal when
+                                                                         *         appropriate using the TLStreamSync() API */
+
+	void                    (*pfOnReaderOpenCallback)(void *pvArg); /*!< Optional on reader connect callback */
+	void                    *pvOnReaderOpenUserData;                /*!< On reader connect user data */
+	void                    (*pfOnReaderCloseCallback)(void *pvArg);/*!< Optional on reader disconnect callback */
+	void                    *pvOnReaderCloseUserData;               /*!< On reader disconnect user data */
+	void                    (*pfProducerCallback)(void);            /*!< Optional producer callback of type TL_STREAM_SOURCECB */
+	void                    *pvProducerUserData;                    /*!< Producer callback user data */
+
+	struct _TL_STREAM_      *psNotifStream;                         /*!< Pointer to the stream to which notification will be sent */
+
+	volatile IMG_UINT32     ui32Read;                               /*!< Pointer to the beginning of available data */
+	volatile IMG_UINT32     ui32Write;                              /*!< Pointer to already committed data which are ready to be
+                                                                         *   copied to user space */
+	IMG_UINT32              ui32Pending;                            /*!< Count pending bytes reserved in buffer */
+	IMG_UINT32              ui32Size;                               /*!< Buffer size */
+	IMG_UINT32              ui32ThresholdUsageForSignal;            /*!< Buffer usage threshold at which a TL writer signals a blocked/
+	                                                                     *    waiting reader when transitioning from empty->non-empty */
+	IMG_UINT32              ui32MaxPacketSize;                      /*! Max TL packet size */
+	IMG_BYTE                *pbyBuffer;                             /*!< Actual data buffer */
+
+	PTL_SNODE               psNode;                                 /*!< Ptr to parent stream node */
+	DEVMEM_MEMDESC          *psStreamMemDesc;                       /*!< MemDescriptor used to allocate buffer space through PMR */
+
+	IMG_HANDLE              hProducerEvent;	                        /*!< Handle to wait on if there is not enough space */
+	IMG_HANDLE              hProducerEventObj;                      /*!< Handle to signal blocked reserve calls */
+	IMG_BOOL                bSignalPending;                         /*!< Tracks if a "signal" is pending to be sent to a blocked/
+	                                                                     *    waiting reader */
+
+	POS_LOCK                hStreamWLock;                           /*!< Writers Lock for ui32Pending & ui32Write*/
+	POS_LOCK                hReadLock;                              /*!< Readers Lock for bReadPending & ui32Read*/
+	IMG_BOOL                bReadPending;                           /*!< Tracks if a read operation is pending or not*/
+	IMG_BOOL                bNoWrapPermanent;                       /*!< Flag: Prevents buffer wrap and subsequent data loss
+	                                                                     *    as well as resetting the read position on close. */
+
+#if defined(TL_BUFFER_STATS)
+	IMG_UINT32              ui32CntReadFails;                       /*!< Tracks how many times reader failed to acquire read lock */
+	IMG_UINT32              ui32CntReadSuccesses;                   /*!< Tracks how many times reader acquires read lock successfully */
+	IMG_UINT32              ui32CntWriteSuccesses;                  /*!< Tracks how many times writer acquires read lock successfully */
+	IMG_UINT32              ui32CntWriteWaits;                      /*!< Tracks how many times writer had to wait to acquire read lock */
+	IMG_UINT32              ui32CntNumWriteSuccess;	                /*!< Tracks how many write operations were successful*/
+	IMG_UINT32              ui32BufferUt;                           /*!< Buffer utilisation high watermark, see TL_BUFFER_STATS above */
+	IMG_UINT32              ui32MaxReserveWatermark;                /*!< Max stream reserve size that was ever requested by a writer */
+	IMG_UINT32              ui32SignalsSent;                        /*!< Number of signals that were actually sent by the write API */
+	ATOMIC_T                bNoReaderSinceFirstReserve;             /*!< Tracks if a read has been done since the buffer was last found empty */
+	IMG_UINT32              ui32TimeStart;                          /*!< Time at which a write (Reserve call) was done into an empty buffer.
+	                                                                     *    Guarded by hStreamWLock. */
+	IMG_UINT32              ui32MinTimeToFullInUs;                  /*!< Minimum time taken to (nearly) fully fill an empty buffer. Guarded
+	                                                                     *    by hStreamWLock. */
+	/* Behaviour counters, protected by hStreamLock in case of
+	 * multi-threaded access */
+	IMG_UINT32              ui32NumCommits;     /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32SignalNotSent;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32ManSyncs;       /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32ProducerByteCount; /*!< Counters used to analysing stream performance, see ++ loc */
+
+	/* Not protected by the lock, inc in the reader thread which is currently singular */
+	IMG_UINT32              ui32AcquireRead1;   /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32              ui32AcquireRead2;   /*!< Counters used to analysing stream performance, see ++ loc */
+#endif
+
+} TL_STREAM, *PTL_STREAM;
+
+/* there need to be enough space reserved in the buffer for 2 minimal packets
+ * and it needs to be aligned the same way the buffer is or there will be a
+ * compile error.*/
+#define BUFFER_RESERVED_SPACE (2 * PVRSRVTL_PACKET_ALIGNMENT)
+
+/* ensure the space reserved follows the buffer's alignment */
+static_assert(!(BUFFER_RESERVED_SPACE&(PVRSRVTL_PACKET_ALIGNMENT-1)),
+			  "BUFFER_RESERVED_SPACE must be a multiple of PVRSRVTL_PACKET_ALIGNMENT");
+
+/* Define the largest value that a uint that matches the
+ * PVRSRVTL_PACKET_ALIGNMENT size can hold */
+#define MAX_UINT 0xffffFFFF
+
+/*! Defines the value used for TL_STREAM.ui32Pending when no reserve is
+ * outstanding on the stream. */
+#define NOTHING_PENDING IMG_UINT32_MAX
+
+
+/*
+ * Transport Layer Stream Descriptor types/defs
+ */
+typedef struct _TL_STREAM_DESC_
+{
+	PTL_SNODE   psNode;         /*!< Ptr to parent stream node */
+	IMG_UINT32  ui32Flags;      /*!< Flags supplied by client on stream open */
+	IMG_HANDLE  hReadEvent;     /*!< For wait call (only used/set in reader descriptors) */
+	IMG_INT     uiRefCount;     /*!< Reference count to the SD */
+
+#if defined(TL_BUFFER_STATS)
+	/* Behaviour counters, no multi-threading protection need as they are
+	 * incremented in a single thread due to only supporting one reader
+	 * at present */
+	IMG_UINT32  ui32AcquireCount;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32NoData;        /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32NoDataSleep;   /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32Signalled;     /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32TimeoutEmpty;  /*!< Counters used to analysing stream performance, see ++ loc */
+	IMG_UINT32  ui32TimeoutData;   /*!< Counters used to analysing stream performance, see ++ loc */
+#endif
+	IMG_UINT32  ui32ReadLimit;     /*!< Limit buffer reads to data present in the
+                                        buffer at the time of stream open. */
+	IMG_UINT32  ui32ReadLen;       /*!< Size of data returned by initial Acquire */
+} TL_STREAM_DESC, *PTL_STREAM_DESC;
+
+PTL_STREAM_DESC TLMakeStreamDesc(PTL_SNODE f1, IMG_UINT32 f2, IMG_HANDLE f3);
+
+#define TL_STREAM_KM_FLAG_MASK	0xFFFF0000
+#define TL_STREAM_FLAG_TEST		0x10000000
+#define TL_STREAM_FLAG_WRAPREAD	0x00010000
+
+#define TL_STREAM_UM_FLAG_MASK	0x0000FFFF
+
+#if defined(TL_BUFFER_STATS)
+#	define TL_COUNTER_INC(a)    ((a)++)
+#	define TL_COUNTER_ADD(a,b)  ((a) += (b))
+#else
+#	define TL_COUNTER_INC(a)    (void)(0)
+#	define TL_COUNTER_ADD(a,b)  (void)(0)
+#endif
+/*
+ * Transport Layer stream list node
+ */
+typedef struct _TL_SNODE_
+{
+	struct _TL_SNODE_*	psNext;				/*!< Linked list next element */
+	IMG_HANDLE			hReadEventObj;		/*!< Readers 'wait for data' event */
+	PTL_STREAM			psStream;			/*!< TL Stream object */
+	IMG_INT				uiWRefCount;		/*!< Stream writer reference count */
+	PTL_STREAM_DESC		psRDesc;			/*!< Stream reader 0 or ptr only */
+	PTL_STREAM_DESC		psWDesc;			/*!< Stream writer 0 or ptr only */
+} TL_SNODE;
+
+PTL_SNODE TLMakeSNode(IMG_HANDLE f2, TL_STREAM *f3, TL_STREAM_DESC *f4);
+
+/*
+ * Transport Layer global top types and variables
+ * Use access function to obtain pointer.
+ *
+ * hTLGDLock - provides atomicity over read/check/write operations and
+ *             sequence of operations on uiClientCnt, psHead list of SNODEs and
+ *             the immediate members in a list element SNODE structure.
+ *           - This larger scope of responsibility for this lock helps avoid
+ *             the need for a lock in the SNODE structure.
+ *           - Lock held in the client (reader) context when streams are
+ *             opened/closed and in the server (writer) context when streams
+ *             are created/open/closed.
+ */
+typedef struct _TL_GDATA_
+{
+	IMG_HANDLE hTLEventObj;        /* Global TL signal object, new streams, etc */
+
+	IMG_UINT   uiClientCnt;        /* Counter to track the number of client stream connections. */
+	PTL_SNODE  psHead;             /* List of TL streams and associated client handle */
+
+	POS_LOCK   hTLGDLock;          /* Lock for structure AND psHead SNODE list */
+} TL_GLOBAL_DATA, *PTL_GLOBAL_DATA;
+
+/*
+ * Transport Layer Internal Kernel-Mode Server API
+ */
+TL_GLOBAL_DATA* TLGGD(void);		/* TLGetGlobalData() */
+
+PVRSRV_ERROR TLInit(void);
+void TLDeInit(void);
+
+void TLAddStreamNode(PTL_SNODE psAdd);
+PTL_SNODE TLFindStreamNodeByName(const IMG_CHAR *pszName);
+PTL_SNODE TLFindStreamNodeByDesc(PTL_STREAM_DESC psDesc);
+IMG_UINT32 TLDiscoverStreamNodes(const IMG_CHAR *pszNamePattern,
+                          IMG_CHAR aaszStreams[][PRVSRVTL_MAX_STREAM_NAME_SIZE],
+                          IMG_UINT32 ui32Max);
+PTL_SNODE TLFindAndGetStreamNodeByDesc(PTL_STREAM_DESC psDesc);
+void TLReturnStreamNode(PTL_SNODE psNode);
+
+/******************************************************************************
+ Function Name	: TLTryRemoveStreamAndFreeStreamNode
+
+ Inputs		: PTL_SNODE	Pointer to the TL_SNODE whose stream is requested
+			to be removed from TL_GLOBAL_DATA's list
+
+ Return Value	: IMG_TRUE	-	If the stream was made NULL and this
+					TL_SNODE was removed from the
+					TL_GLOBAL_DATA's list
+
+		  IMG_FALSE	-	If the stream wasn't made NULL as there
+					is a client connected to this stream
+
+ Description	: If there is no client currently connected to this stream then,
+			This function removes this TL_SNODE from the
+			TL_GLOBAL_DATA's list. The caller is responsible for the
+			cleanup of the TL_STREAM whose TL_SNODE may be removed
+
+		  Otherwise, this function does nothing
+******************************************************************************/
+IMG_BOOL  TLTryRemoveStreamAndFreeStreamNode(PTL_SNODE psRemove);
+
+/******************************************************************************
+ Function Name	: TLUnrefDescAndTryFreeStreamNode
+
+ Inputs		: PTL_SNODE	Pointer to the TL_SNODE whose descriptor is
+			requested to be removed
+			: PTL_STREAM_DESC	Pointer to the STREAM_DESC
+
+ Return Value	: IMG_TRUE	-	If this	TL_SNODE was removed from the
+					TL_GLOBAL_DATA's list
+
+		  IMG_FALSE	-	Otherwise
+
+ Description	: This function removes the stream descriptor from this TL_SNODE
+			and, if there is no writer (producer context) currently bound to this
+			stream, this function removes this TL_SNODE from the TL_GLOBAL_DATA's
+			list. The caller is responsible for the cleanup of the TL_STREAM
+			whose TL_SNODE may be removed
+******************************************************************************/
+IMG_BOOL TLUnrefDescAndTryFreeStreamNode(PTL_SNODE psRemove, PTL_STREAM_DESC psSD);
+
+/*
+ * Transport Layer stream interface to server part declared here to avoid
+ * circular dependency.
+ */
+IMG_UINT32 TLStreamAcquireReadPos(PTL_STREAM psStream,
+                                  IMG_BOOL bDisableCallback,
+                                  IMG_UINT32* puiReadOffset);
+PVRSRV_ERROR TLStreamAdvanceReadPos(PTL_STREAM psStream,
+                                    IMG_UINT32 uiReadLen,
+                                    IMG_UINT32 uiOrigReadLen);
+void TLStreamResetReadPos(PTL_STREAM psStream);
+
+DEVMEM_MEMDESC* TLStreamGetBufferPointer(PTL_STREAM psStream);
+IMG_BOOL TLStreamOutOfData(IMG_HANDLE psStream);
+
+/******************************************************************************
+ Function Name	: TLStreamDestroy
+
+ Inputs		: PTL_STREAM	Pointer to the TL_STREAM to be destroyed
+
+ Description	: This function performs all the clean-up operations required for
+			destruction of this stream
+******************************************************************************/
+void TLStreamDestroy(PTL_STREAM psStream);
+
+/*
+ * Test related functions
+ */
+PVRSRV_ERROR TUtilsInit(PVRSRV_DEVICE_NODE *psDeviceNode);
+PVRSRV_ERROR TUtilsDeinit(PVRSRV_DEVICE_NODE *psDeviceNode);
+
+
+#endif /* TLINTERN_H */
+/******************************************************************************
+ End of file (tlintern.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/tlserver.c b/drivers/gpu/drm/img-rogue/tlserver.c
new file mode 100644
index 000000000000..5c8359271508
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlserver.c
@@ -0,0 +1,755 @@
+/*************************************************************************/ /*!
+@File
+@Title          KM server Transport Layer implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main bridge APIs for Transport Layer client functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "img_defs.h"
+
+/*#define PVR_DPF_FUNCTION_TRACE_ON 1*/
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "connection_server.h"
+#include "allocmem.h"
+#include "devicemem.h"
+
+#include "tlintern.h"
+#include "tlstream.h"
+#include "tlserver.h"
+
+#include "pvrsrv_tlstreams.h"
+#define NO_STREAM_WAIT_PERIOD_US 2000000ULL
+#define NO_DATA_WAIT_PERIOD_US    500000ULL
+#define NO_ACQUIRE               0xffffffffU
+
+
+/*
+ * Transport Layer Client API Kernel-Mode bridge implementation
+ */
+PVRSRV_ERROR
+TLServerOpenStreamKM(const IMG_CHAR*   pszName,
+                     IMG_UINT32        ui32Mode,
+                     PTL_STREAM_DESC*  ppsSD,
+                     PMR**             ppsTLPMR)
+{
+	PVRSRV_ERROR	eError = PVRSRV_OK;
+	PVRSRV_ERROR	eErrorEO = PVRSRV_OK;
+	PTL_SNODE		psNode;
+	PTL_STREAM		psStream;
+	TL_STREAM_DESC *psNewSD = NULL;
+	IMG_HANDLE		hEvent;
+	IMG_BOOL		bIsWriteOnly = ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	                               IMG_TRUE : IMG_FALSE;
+	IMG_BOOL		bResetOnOpen = ui32Mode & PVRSRV_STREAM_FLAG_RESET_ON_OPEN ?
+	                               IMG_TRUE : IMG_FALSE;
+	IMG_BOOL		bNoOpenCB    = ui32Mode & PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK ?
+	                               IMG_TRUE : IMG_FALSE;
+	PTL_GLOBAL_DATA psGD = TLGGD();
+
+#if defined(PVR_DPF_FUNCTION_TRACE_ON)
+	PVR_DPF((PVR_DBG_CALLTRACE, "--> %s:%d entered (%s, %x)", __func__, __LINE__, pszName, ui32Mode));
+#endif
+
+	PVR_ASSERT(pszName);
+
+	/* Acquire TL_GLOBAL_DATA lock here, as if the following TLFindStreamNodeByName
+	 * returns NON NULL PTL_SNODE, we try updating the global data client count and
+	 * PTL_SNODE's psRDesc and we want to make sure the TL_SNODE is valid (eg. has
+	 * not been deleted) while we are updating it
+	 */
+	OSLockAcquire (psGD->hTLGDLock);
+
+	psNode = TLFindStreamNodeByName(pszName);
+	if ((psNode == NULL) && (ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WAIT))
+	{	/* Blocking code to wait for stream to be created if it does not exist */
+		eError = OSEventObjectOpen(psGD->hTLEventObj, &hEvent);
+		PVR_LOG_GOTO_IF_ERROR (eError, "OSEventObjectOpen", e0);
+
+		do
+		{
+			if ((psNode = TLFindStreamNodeByName(pszName)) == NULL)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "Stream %s does not exist, waiting...", pszName));
+
+				/* Release TL_GLOBAL_DATA lock before sleeping */
+				OSLockRelease (psGD->hTLGDLock);
+
+				/* Will exit OK or with timeout, both cases safe to ignore */
+				eErrorEO = OSEventObjectWaitTimeout(hEvent, NO_STREAM_WAIT_PERIOD_US);
+
+				/* Acquire lock after waking up */
+				OSLockAcquire (psGD->hTLGDLock);
+			}
+		}
+		while ((psNode == NULL) && (eErrorEO == PVRSRV_OK));
+
+		eError = OSEventObjectClose(hEvent);
+		PVR_LOG_GOTO_IF_ERROR (eError, "OSEventObjectClose", e0);
+	}
+
+	/* Make sure we have found a stream node after wait/search */
+	if (psNode == NULL)
+	{
+		/* Did we exit the wait with timeout, inform caller */
+		if (eErrorEO == PVRSRV_ERROR_TIMEOUT)
+		{
+			eError = eErrorEO;
+		}
+		else
+		{
+			eError = PVRSRV_ERROR_NOT_FOUND;
+			PVR_DPF((PVR_DBG_ERROR, "Stream \"%s\" does not exist", pszName));
+		}
+		goto e0;
+	}
+
+	psStream = psNode->psStream;
+
+	/* Allocate memory for the stream. The memory will be allocated with the
+	 * first call. */
+	eError = TLAllocSharedMemIfNull(psStream);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Failed to allocate memory for stream"
+				" \"%s\"", pszName));
+		goto e0;
+	}
+
+	if (bIsWriteOnly)
+	{
+
+		/* If psWDesc == NULL it means that this is the first attempt
+		 * to open stream for write. If yes create the descriptor or increment
+		 * reference count otherwise. */
+		if (psNode->psWDesc == NULL)
+		{
+			psNewSD = TLMakeStreamDesc(psNode, ui32Mode, NULL);
+			psNode->psWDesc = psNewSD;
+		}
+		else
+		{
+			psNewSD = psNode->psWDesc;
+			psNode->psWDesc->uiRefCount++;
+		}
+
+		PVR_LOG_GOTO_IF_NOMEM(psNewSD, eError, e0);
+
+		psNode->uiWRefCount++;
+	}
+	else
+	{
+		/* Only one reader per stream supported */
+		if (psNode->psRDesc != NULL)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Cannot open \"%s\" stream, stream already"
+			        " opened", pszName));
+			eError = PVRSRV_ERROR_ALREADY_OPEN;
+			goto e0;
+		}
+
+		/* Create an event handle for this client to wait on when no data in
+		 * stream buffer. */
+		eError = OSEventObjectOpen(psNode->hReadEventObj, &hEvent);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_LOG_ERROR(eError, "OSEventObjectOpen");
+			eError = PVRSRV_ERROR_UNABLE_TO_CREATE_EVENT;
+			goto e0;
+		}
+
+		psNewSD = TLMakeStreamDesc(psNode, ui32Mode, hEvent);
+		psNode->psRDesc = psNewSD;
+
+		if (!psNewSD)
+		{
+			PVR_DPF((PVR_DBG_ERROR, "Not possible to make a new stream descriptor"));
+			eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+			goto e1;
+		}
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+		        "TLServerOpenStreamKM evList=%p, evObj=%p",
+		        psNode->hReadEventObj,
+		        psNode->psRDesc->hReadEvent));
+	}
+
+	/* Copy the import handle back to the user mode API to enable access to
+	 * the stream buffer from user-mode process. */
+	eError = DevmemLocalGetImportHandle(TLStreamGetBufferPointer(psStream),
+	                                    (void**) ppsTLPMR);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemLocalGetImportHandle", e2);
+
+	psGD->uiClientCnt++;
+
+	/* Global data updated. Now release global lock */
+	OSLockRelease (psGD->hTLGDLock);
+
+	*ppsSD = psNewSD;
+
+	if (bResetOnOpen)
+	{
+		TLStreamReset(psStream);
+	}
+
+	/* This callback is executed only on reader open. There are some actions
+	 * executed on reader open that don't make much sense for writers e.g.
+	 * injection on time synchronisation packet into the stream. */
+	if (!bIsWriteOnly && psStream->pfOnReaderOpenCallback != NULL && !bNoOpenCB)
+	{
+		psStream->pfOnReaderOpenCallback(psStream->pvOnReaderOpenUserData);
+	}
+
+	/* psNode->uiWRefCount is set to '1' on stream create so the first open
+	 * is '2'. */
+	if (bIsWriteOnly && psStream->psNotifStream != NULL &&
+	    psNode->uiWRefCount == 2)
+	{
+		TLStreamMarkStreamOpen(psStream);
+	}
+
+	PVR_DPF((PVR_DBG_MESSAGE, "%s: Stream %s opened for %s", __func__, pszName,
+	        ui32Mode & PVRSRV_STREAM_FLAG_OPEN_WO ? "write" : "read"));
+
+	PVR_DPF_RETURN_OK;
+
+e2:
+	OSFreeMem(psNewSD);
+e1:
+	if (!bIsWriteOnly)
+		OSEventObjectClose(hEvent);
+e0:
+	OSLockRelease (psGD->hTLGDLock);
+	PVR_DPF_RETURN_RC (eError);
+}
+
+PVRSRV_ERROR
+TLServerCloseStreamKM(PTL_STREAM_DESC psSD)
+{
+	PVRSRV_ERROR    eError = PVRSRV_OK;
+	PTL_GLOBAL_DATA psGD = TLGGD();
+	PTL_SNODE		psNode;
+	PTL_STREAM	psStream;
+	IMG_BOOL	bDestroyStream;
+	IMG_BOOL	bIsWriteOnly = psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO ?
+	                           IMG_TRUE : IMG_FALSE;
+	IMG_BOOL	bNoOpenCB    = psSD->ui32Flags & PVRSRV_STREAM_FLAG_IGNORE_OPEN_CALLBACK ?
+	                           IMG_TRUE : IMG_FALSE;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since the descriptor is valid, the stream should not have been made NULL */
+	PVR_ASSERT (psNode->psStream);
+
+	/* Save the stream's reference in-case its destruction is required after this
+	 * client is removed */
+	psStream = psNode->psStream;
+
+	/* Acquire TL_GLOBAL_DATA lock as the following TLRemoveDescAndTryFreeStreamNode
+	 * call will update the TL_SNODE's descriptor value */
+	OSLockAcquire (psGD->hTLGDLock);
+
+	/* Close event handle because event object list might be destroyed in
+	 * TLUnrefDescAndTryFreeStreamNode(). */
+	if (!bIsWriteOnly)
+	{
+		/* Reset the read position on close if the stream requires it. */
+		TLStreamResetReadPos(psStream);
+
+		/* Close and free the event handle resource used by this descriptor */
+		eError = OSEventObjectClose(psSD->hReadEvent);
+		if (eError != PVRSRV_OK)
+		{
+			/* Log error but continue as it seems best */
+			PVR_LOG_ERROR(eError, "OSEventObjectClose");
+			eError = PVRSRV_ERROR_UNABLE_TO_DESTROY_EVENT;
+		}
+	}
+	else if (psNode->uiWRefCount == 2 && psStream->psNotifStream != NULL)
+	{
+		/* psNode->uiWRefCount is set to '1' on stream create so the last close
+		 * before destruction is '2'. */
+		TLStreamMarkStreamClose(psStream);
+	}
+
+	/* Remove descriptor from stream object/list */
+	bDestroyStream = TLUnrefDescAndTryFreeStreamNode (psNode, psSD);
+
+	/* Check the counter is sensible after input data validated. */
+	PVR_ASSERT(psGD->uiClientCnt > 0);
+	psGD->uiClientCnt--;
+
+	/* This callback is executed only on reader close. */
+	if (!bIsWriteOnly && psStream->pfOnReaderCloseCallback != NULL && !bNoOpenCB)
+	{
+		psStream->pfOnReaderCloseCallback(psStream->pvOnReaderCloseUserData);
+	}
+
+	OSLockRelease (psGD->hTLGDLock);
+
+	/* Destroy the stream if its TL_SNODE was removed from TL_GLOBAL_DATA */
+	if (bDestroyStream)
+	{
+		TLStreamDestroy (psStream);
+		psStream = NULL;
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE, "%s: Stream closed", __func__));
+
+	/* Free the descriptor if ref count reaches 0. */
+	if (psSD->uiRefCount == 0)
+	{
+		/* Free the stream descriptor object */
+		OSFreeMem(psSD);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerReserveStreamKM(PTL_STREAM_DESC psSD,
+                        IMG_UINT32* ui32BufferOffset,
+                        IMG_UINT32 ui32Size,
+                        IMG_UINT32 ui32SizeMin,
+                        IMG_UINT32* pui32Available)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+	IMG_UINT8* pui8Buffer = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire the global lock. We have to be sure that no one modifies
+	 * the list while we are looking for our stream. */
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	/* The TL writers that currently land here are at a very low to none risk
+	 * to breach max TL packet size constraint (even if there is no reader
+	 * connected to the TL stream and hence eventually will cause the TL stream
+	 * to be full). Hence no need to know the status of TL stream reader
+	 * connection.
+	 */
+	eError = TLStreamReserve2(psNode->psStream, &pui8Buffer, ui32Size,
+				  ui32SizeMin, pui32Available, NULL);
+	if (eError != PVRSRV_OK)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Failed to reserve %u (%u, %u) bytes in the stream, error %s.",
+				ui32Size, ui32SizeMin, *pui32Available, PVRSRVGETERRORSTRING(eError)));
+	}
+	else if (pui8Buffer == NULL)
+	{
+		PVR_DPF((PVR_DBG_WARNING, "Not enough space in the stream."));
+		eError = PVRSRV_ERROR_STREAM_FULL;
+	}
+	else
+	{
+		*ui32BufferOffset = pui8Buffer - psNode->psStream->pbyBuffer;
+		PVR_ASSERT(*ui32BufferOffset < psNode->psStream->ui32Size);
+	}
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerCommitStreamKM(PTL_STREAM_DESC psSD,
+                       IMG_UINT32 ui32Size)
+{
+	TL_GLOBAL_DATA*	psGD = TLGGD();
+	PTL_SNODE psNode;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire the global lock. We have to be sure that no one modifies
+	 * the list while we are looking for our stream. */
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	eError = TLStreamCommit(psNode->psStream, ui32Size);
+	PVR_LOG_IF_ERROR(eError, "TLStreamCommit");
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLServerDiscoverStreamsKM(const IMG_CHAR *pszNamePattern,
+                          IMG_UINT32 ui32Size,
+                          IMG_CHAR *pszStreams,
+                          IMG_UINT32 *pui32NumFound)
+{
+	PTL_SNODE psNode = NULL;
+	IMG_CHAR (*paszStreams)[PRVSRVTL_MAX_STREAM_NAME_SIZE] =
+			(IMG_CHAR (*)[PRVSRVTL_MAX_STREAM_NAME_SIZE]) (void *)pszStreams;
+
+	if (*pszNamePattern == '\0')
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	if (ui32Size % PRVSRVTL_MAX_STREAM_NAME_SIZE != 0)
+		return PVRSRV_ERROR_INVALID_PARAMS;
+
+	/* Quick exit if there are no streams */
+	if (TLGGD()->psHead == NULL)
+	{
+		*pui32NumFound = 0;
+		return PVRSRV_OK;
+	}
+
+	OSLockAcquire(TLGGD()->hTLGDLock);
+
+	*pui32NumFound = TLDiscoverStreamNodes(pszNamePattern, paszStreams,
+	                                  ui32Size / PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	/* Find "tlctrl" stream and reset it */
+	psNode = TLFindStreamNodeByName(PVRSRV_TL_CTRL_STREAM);
+	if (psNode != NULL)
+		TLStreamReset(psNode->psStream);
+
+	OSLockRelease(TLGGD()->hTLGDLock);
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+TLServerAcquireDataKM(PTL_STREAM_DESC psSD,
+                      IMG_UINT32*     puiReadOffset,
+                      IMG_UINT32*     puiReadLen)
+{
+	PVRSRV_ERROR    eError = PVRSRV_OK;
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	IMG_UINT32      uiTmpOffset;
+	IMG_UINT32      uiTmpLen = 0;
+	PTL_SNODE       psNode;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	TL_COUNTER_INC(psSD->ui32AcquireCount);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* If we are here, the stream will never be made NULL until this context itself
+	 * calls TLRemoveDescAndTryFreeStreamNode(). This is because the producer will
+	 * fail to make the stream NULL (by calling TLTryRemoveStreamAndFreeStreamNode)
+	 * when a valid stream descriptor is present (i.e. a client is connected).
+	 * Hence, no checks for stream being NON NULL are required after this. */
+	PVR_ASSERT (psNode->psStream);
+
+	psSD->ui32ReadLen = 0;	/* Handle NULL read returns */
+
+	do
+	{
+		uiTmpLen = TLStreamAcquireReadPos(psNode->psStream, psSD->ui32Flags & PVRSRV_STREAM_FLAG_DISABLE_PRODUCER_CALLBACK, &uiTmpOffset);
+
+		/* Check we have not already exceeded read limit with just offset
+		 * regardless of data length to ensure the client sees the RC */
+		if (psSD->ui32Flags & PVRSRV_STREAM_FLAG_READ_LIMIT)
+		{
+			/* Check to see if we are reading beyond the read limit */
+			if (uiTmpOffset >= psSD->ui32ReadLimit)
+			{
+				PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_READLIMIT_REACHED);
+			}
+		}
+
+		if (uiTmpLen > 0)
+		{ /* Data found */
+
+			/* Check we have not already exceeded read limit offset+len */
+			if (psSD->ui32Flags & PVRSRV_STREAM_FLAG_READ_LIMIT)
+			{
+				/* Adjust the read length if it goes beyond the read limit
+				 * limit always guaranteed to be on packet */
+				if ((uiTmpOffset + uiTmpLen) >= psSD->ui32ReadLimit)
+				{
+					uiTmpLen = psSD->ui32ReadLimit - uiTmpOffset;
+				}
+			}
+
+			*puiReadOffset = uiTmpOffset;
+			*puiReadLen = uiTmpLen;
+			psSD->ui32ReadLen = uiTmpLen;	/* Save the original data length in the stream desc */
+			PVR_DPF_RETURN_OK;
+		}
+		else if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING))
+		{ /* No data found blocking */
+
+			/* Instead of doing a complete sleep for `NO_DATA_WAIT_PERIOD_US` us, we sleep in chunks
+			 * of 168 ms. In a "deferred" signal scenario from writer, this gives us a chance to
+			 * wake-up (timeout) early and continue reading in-case some data is available */
+			IMG_UINT64 ui64WaitInChunksUs = MIN(NO_DATA_WAIT_PERIOD_US, 168000ULL);
+			IMG_BOOL bDataFound = IMG_FALSE;
+
+			TL_COUNTER_INC(psSD->ui32NoDataSleep);
+
+			LOOP_UNTIL_TIMEOUT_US(NO_DATA_WAIT_PERIOD_US)
+			{
+				eError = OSEventObjectWaitTimeout(psSD->hReadEvent, ui64WaitInChunksUs);
+				if (eError == PVRSRV_OK)
+				{
+					bDataFound = IMG_TRUE;
+					TL_COUNTER_INC(psSD->ui32Signalled);
+					break;
+				}
+				else if (eError == PVRSRV_ERROR_TIMEOUT)
+				{
+					if (TLStreamOutOfData(psNode->psStream))
+					{
+						/* Return on timeout if stream empty, else let while exit and return data */
+						continue;
+					}
+					else
+					{
+						bDataFound = IMG_TRUE;
+						TL_COUNTER_INC(psSD->ui32TimeoutData);
+						PVR_DPF((PVR_DBG_MESSAGE, "%s: Data found at timeout. Current BuffUt = %u",
+												 __func__, TLStreamGetUT(psNode->psStream)));
+						break;
+					}
+				}
+				else
+				{ /* Some other system error with event objects */
+					PVR_DPF_RETURN_RC(eError);
+				}
+			} END_LOOP_UNTIL_TIMEOUT_US();
+
+			if (bDataFound)
+			{
+				continue;
+			}
+			else
+			{
+				TL_COUNTER_INC(psSD->ui32TimeoutEmpty);
+				return PVRSRV_ERROR_TIMEOUT;
+			}
+		}
+		else
+		{ /* No data non-blocking */
+			TL_COUNTER_INC(psSD->ui32NoData);
+
+			/* When no-data in non-blocking mode, uiReadOffset should be set to NO_ACQUIRE
+			 * signifying there's no need of Release call */
+			*puiReadOffset = NO_ACQUIRE;
+			*puiReadLen = 0;
+			PVR_DPF_RETURN_OK;
+		}
+	}
+	while (1);
+}
+
+PVRSRV_ERROR
+TLServerReleaseDataKM(PTL_STREAM_DESC psSD,
+                      IMG_UINT32      uiReadOffset,
+                      IMG_UINT32      uiReadLen)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+
+	PVR_DPF_ENTERED;
+
+	/* Unreferenced in release builds */
+	PVR_UNREFERENCED_PARAMETER(uiReadOffset);
+
+	PVR_ASSERT(psSD);
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	if ((uiReadLen % PVRSRVTL_PACKET_ALIGNMENT != 0))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Check stream still valid */
+	psNode = TLFindStreamNodeByDesc(psSD);
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	PVR_DPF((PVR_DBG_VERBOSE, "TLReleaseDataKM uiReadOffset=%d, uiReadLen=%d", uiReadOffset, uiReadLen));
+
+	/* Move read position on to free up space in stream buffer */
+	PVR_DPF_RETURN_RC(TLStreamAdvanceReadPos(psNode->psStream, uiReadLen, psSD->ui32ReadLen));
+}
+
+PVRSRV_ERROR
+TLServerWriteDataKM(PTL_STREAM_DESC psSD,
+                    IMG_UINT32 ui32Size,
+                    IMG_BYTE* pui8Data)
+{
+	TL_GLOBAL_DATA* psGD = TLGGD();
+	PTL_SNODE psNode;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psSD);
+
+	if (!(psSD->ui32Flags & PVRSRV_STREAM_FLAG_OPEN_WO))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Quick exit if there are no streams */
+	if (psGD->psHead == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	OSLockAcquire(psGD->hTLGDLock);
+	/* Check stream still valid */
+	psNode = TLFindAndGetStreamNodeByDesc(psSD);
+	OSLockRelease(psGD->hTLGDLock);
+
+	if ((psNode == NULL) || (psNode != psSD->psNode))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_HANDLE_NOT_FOUND);
+	}
+
+	/* Since we have a valid stream descriptor, the stream should not have been
+	 * made NULL by any producer context. */
+	PVR_ASSERT (psNode->psStream);
+
+	eError = TLStreamWrite(psNode->psStream, pui8Data, ui32Size);
+	/* propagate error up but don't print anything here */
+
+	OSLockAcquire(psGD->hTLGDLock);
+	TLReturnStreamNode(psNode);
+	OSLockRelease(psGD->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+/******************************************************************************
+ End of file (tlserver.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/tlserver.h b/drivers/gpu/drm/img-rogue/tlserver.h
new file mode 100644
index 000000000000..7ac2958eac1e
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlserver.h
@@ -0,0 +1,97 @@
+/*************************************************************************/ /*!
+@File
+@Title          KM server Transport Layer implementation
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Main bridge APIs for Transport Layer client functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef TLSERVER_H
+#define TLSERVER_H
+
+#include "img_defs.h"
+#include "pvr_debug.h"
+#include "connection_server.h"
+
+#include "tlintern.h"
+
+/*
+ * Transport Layer Client API Kernel-Mode bridge implementation
+ */
+
+PVRSRV_ERROR TLServerConnectKM(CONNECTION_DATA *psConnection);
+PVRSRV_ERROR TLServerDisconnectKM(CONNECTION_DATA *psConnection);
+
+PVRSRV_ERROR TLServerOpenStreamKM(const IMG_CHAR* pszName,
+			   IMG_UINT32 ui32Mode,
+			   PTL_STREAM_DESC* ppsSD,
+			   PMR** ppsTLPMR);
+
+PVRSRV_ERROR TLServerCloseStreamKM(PTL_STREAM_DESC psSD);
+
+PVRSRV_ERROR TLServerDiscoverStreamsKM(const IMG_CHAR *pszNamePattern,
+                          IMG_UINT32 ui32Max,
+                          IMG_CHAR *pszStreams,
+                          IMG_UINT32 *pui32NumFound);
+
+PVRSRV_ERROR TLServerReserveStreamKM(PTL_STREAM_DESC psSD,
+                                     IMG_UINT32* ui32BufferOffset,
+                                     IMG_UINT32 ui32Size,
+                                     IMG_UINT32 ui32SizeMin,
+                                     IMG_UINT32* pui32Available);
+
+PVRSRV_ERROR TLServerCommitStreamKM(PTL_STREAM_DESC psSD,
+                                    IMG_UINT32 ui32Size);
+
+PVRSRV_ERROR TLServerAcquireDataKM(PTL_STREAM_DESC psSD,
+			   IMG_UINT32* puiReadOffset,
+			   IMG_UINT32* puiReadLen);
+
+PVRSRV_ERROR TLServerReleaseDataKM(PTL_STREAM_DESC psSD,
+				 IMG_UINT32 uiReadOffset,
+				 IMG_UINT32 uiReadLen);
+
+PVRSRV_ERROR TLServerWriteDataKM(PTL_STREAM_DESC psSD,
+                                 IMG_UINT32 ui32Size,
+                                 IMG_BYTE *pui8Data);
+
+#endif /* TLSERVER_H */
+
+/******************************************************************************
+ End of file (tlserver.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/tlstream.c b/drivers/gpu/drm/img-rogue/tlstream.c
new file mode 100644
index 000000000000..3d13ccf6b326
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlstream.c
@@ -0,0 +1,1702 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API implementation.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Transport Layer API implementation.
+                These functions are provided to driver components.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+//#define PVR_DPF_FUNCTION_TRACE_ON 1
+#undef PVR_DPF_FUNCTION_TRACE_ON
+#include "pvr_debug.h"
+
+#include "allocmem.h"
+#include "devicemem.h"
+#include "pvrsrv_error.h"
+#include "sysinfo.h"
+#include "osfunc.h"
+#include "log2.h"
+
+#include "tlintern.h"
+#include "tlstream.h"
+
+#include "pvrsrv.h"
+
+/* Prefer CMA memory for TL buffers if possible */
+#define TL_USE_CMA 0
+
+#if !defined(EVENT_OBJECT_TIMEOUT_US)
+#error EVENT_OBJECT_TIMEOUT_US should be defined sysinfo.h
+#endif
+
+#define READ_PENDING_TIMEOUT_US 100000ULL
+
+/*! Compute maximum TL packet size for this stream. Max packet size will be
+ * minimum of PVRSRVTL_MAX_PACKET_SIZE and (BufferSize / 2.5). This computation
+ * is required to avoid a corner case that was observed when TL buffer size is
+ * smaller than twice of TL max packet size and read, write index are positioned
+ * in such a way that the TL packet (write packet + padding packet) size is may
+ * be bigger than the buffer size itself.
+ */
+#define GET_TL_MAX_PACKET_SIZE( bufSize ) PVRSRVTL_ALIGN( MIN( PVRSRVTL_MAX_PACKET_SIZE, ( 2 * bufSize ) / 5 ) )
+
+/* Given the state of the buffer it returns a number of bytes that the client
+ * can use for a successful allocation. */
+static INLINE IMG_UINT32 suggestAllocSize(IMG_UINT32 ui32LRead,
+                                          IMG_UINT32 ui32LWrite,
+                                          IMG_UINT32 ui32CBSize,
+                                          IMG_UINT32 ui32ReqSizeMin,
+                                          IMG_UINT32 ui32MaxPacketSize)
+{
+	IMG_UINT32 ui32AvSpace = 0;
+
+	/* This could be written in fewer lines using the ? operator but it
+		would not be kind to potential readers of this source at all. */
+	if (ui32LRead > ui32LWrite)                          /* Buffer WRAPPED */
+	{
+		if ((ui32LRead - ui32LWrite) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32LRead - ui32LWrite - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+	}
+	else                                                  /* Normal, no wrap */
+	{
+		if ((ui32CBSize - ui32LWrite) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32CBSize - ui32LWrite - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+		else if ((ui32LRead - 0) > (sizeof(PVRSRVTL_PACKETHDR) + ui32ReqSizeMin + (IMG_INT) BUFFER_RESERVED_SPACE))
+		{
+			ui32AvSpace = ui32LRead - sizeof(PVRSRVTL_PACKETHDR) - (IMG_INT) BUFFER_RESERVED_SPACE;
+		}
+	}
+	/* The max size of a TL packet currently is UINT16. adjust accordingly */
+	return MIN(ui32AvSpace, ui32MaxPacketSize);
+}
+
+/* Returns bytes left in the buffer. Negative if there is not any.
+ * two 8b aligned values are reserved, one for the write failed buffer flag
+ * and one to be able to distinguish the buffer full state to the buffer
+ * empty state.
+ * Always returns free space -8 even when the "write failed" packet may be
+ * already in the stream before this write. */
+static INLINE IMG_INT
+circbufSpaceLeft(IMG_UINT32 ui32Read, IMG_UINT32 ui32Write, IMG_UINT32 ui32size)
+{
+	/* We need to reserve 8b (one packet) in the buffer to be able to tell empty
+	 * buffers from full buffers and one more for packet write fail packet */
+	if (ui32Read > ui32Write)
+	{
+		return (IMG_INT)ui32Read - (IMG_INT)ui32Write - (IMG_INT)BUFFER_RESERVED_SPACE;
+	}
+	else
+	{
+		return (IMG_INT)ui32size - ((IMG_INT)ui32Write - (IMG_INT)ui32Read) - (IMG_INT)BUFFER_RESERVED_SPACE;
+	}
+}
+
+IMG_UINT32 TLStreamGetUT(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	IMG_UINT32 ui32LRead = psStream->ui32Read, ui32LWrite = psStream->ui32Write;
+
+	if (ui32LWrite >= ui32LRead)
+	{
+		return (ui32LWrite-ui32LRead);
+	}
+	else
+	{
+		return (psStream->ui32Size-ui32LRead+ui32LWrite);
+	}
+}
+
+PVRSRV_ERROR TLAllocSharedMemIfNull(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	PVRSRV_ERROR eError;
+	PVRSRV_DEVICE_NODE *psDeviceNode;
+
+	/* CPU Local memory used as these buffers are not accessed by the device.
+	 * CPU Uncached write combine memory used to improve write performance,
+	 * memory barrier added in TLStreamCommit to ensure data written to memory
+	 * before CB write point is updated before consumption by the reader.
+	 */
+	IMG_CHAR pszBufferLabel[PRVSRVTL_MAX_STREAM_NAME_SIZE + 20];
+	PVRSRV_MEMALLOCFLAGS_T uiMemFlags = PVRSRV_MEMALLOCFLAG_CPU_READABLE |
+	                                    PVRSRV_MEMALLOCFLAG_CPU_WRITEABLE |
+	                                    PVRSRV_MEMALLOCFLAG_GPU_READABLE |
+	                                    PVRSRV_MEMALLOCFLAG_CPU_UNCACHED_WC |
+	                                    PVRSRV_MEMALLOCFLAG_KERNEL_CPU_MAPPABLE |
+	                                    PVRSRV_MEMALLOCFLAG_PHYS_HEAP_HINT(CPU_LOCAL) |  /* TL for now is only used by host driver, so cpulocal mem suffices */
+	                                    PVRSRV_MEMALLOCFLAG_ZERO_ON_ALLOC;
+
+#if TL_USE_CMA
+	/* TL CTRL stream will be allocated as normal memory, os device node might not
+	 * be setup at this point of init and so we cannot allocate CMA.
+	 */
+	if (PVRSRVGetPVRSRVData()->psDeviceNodeList != NULL)
+	{
+
+		uiMemFlags |= PVRSRV_MEMALLOCFLAG_OS_LINUX_PREFER_CMA;
+		psDeviceNode = PVRSRVGetPVRSRVData()->psDeviceNodeList;
+	}
+	else
+#endif
+	{
+		psDeviceNode = PVRSRVGetPVRSRVData()->psHostMemDeviceNode;
+	}
+
+	/* Exit if memory has already been allocated. */
+	if (psStream->pbyBuffer != NULL)
+		return PVRSRV_OK;
+
+	OSSNPrintf(pszBufferLabel, sizeof(pszBufferLabel), "TLStreamBuf-%s",
+	           psStream->szName);
+
+	eError = DevmemAllocateExportable((IMG_HANDLE)psDeviceNode,
+	                                  (IMG_DEVMEM_SIZE_T) psStream->ui32Size,
+	                                  (IMG_DEVMEM_ALIGN_T) OSGetPageSize(),
+	                                  ExactLog2(OSGetPageSize()),
+	                                  uiMemFlags,
+	                                  pszBufferLabel,
+	                                  &psStream->psStreamMemDesc);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAllocateExportable", e0);
+
+	eError = DevmemAcquireCpuVirtAddr(psStream->psStreamMemDesc,
+	                                  (void**) &psStream->pbyBuffer);
+	PVR_LOG_GOTO_IF_ERROR(eError, "DevmemAcquireCpuVirtAddr", e1);
+
+	return PVRSRV_OK;
+
+e1:
+	DevmemFree(psStream->psStreamMemDesc);
+e0:
+	return eError;
+}
+
+void TLFreeSharedMem(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+
+	if (psStream->pbyBuffer != NULL)
+	{
+		DevmemReleaseCpuVirtAddr(psStream->psStreamMemDesc);
+		psStream->pbyBuffer = NULL;
+	}
+	if (psStream->psStreamMemDesc != NULL)
+	{
+		DevmemFree(psStream->psStreamMemDesc);
+		psStream->psStreamMemDesc = NULL;
+	}
+}
+
+/* Special space left routine for TL_FLAG_PERMANENT_NO_WRAP streams */
+static INLINE IMG_UINT
+bufSpaceLeft(IMG_UINT32 ui32Read, IMG_UINT32 ui32Write, IMG_UINT32 ui32size)
+{
+	 /* buffers from full buffers and one more for packet write fail packet */
+	PVR_ASSERT(ui32Read<=ui32Write);
+	return ui32size - ui32Write;
+}
+
+/*******************************************************************************
+ * TL Server public API implementation.
+ ******************************************************************************/
+PVRSRV_ERROR
+TLStreamCreate(IMG_HANDLE *phStream,
+               const IMG_CHAR *szStreamName,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32StreamFlags,
+               TL_STREAM_ONREADEROPENCB pfOnReaderOpenCB,
+               void *pvOnReaderOpenUD,
+               TL_STREAM_ONREADERCLOSECB pfOnReaderCloseCB,
+               void *pvOnReaderCloseUD,
+               TL_STREAM_SOURCECB pfProducerCB,
+               void *pvProducerUD)
+{
+	PTL_STREAM    psTmp;
+	PVRSRV_ERROR  eError;
+	IMG_HANDLE    hEventList;
+	PTL_SNODE     psn;
+	TL_OPMODE     eOpMode;
+
+	PVR_DPF_ENTERED;
+	/* Parameter checks: non NULL handler required */
+	if (NULL == phStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	if (szStreamName == NULL || *szStreamName == '\0' ||
+	    OSStringLength(szStreamName) >= PRVSRVTL_MAX_STREAM_NAME_SIZE)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eOpMode = ui32StreamFlags & TL_OPMODE_MASK;
+	if (( eOpMode <= TL_OPMODE_UNDEF ) || ( eOpMode >= TL_OPMODE_LAST ))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OpMode for TL stream is invalid"));
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Acquire TL_GLOBAL_DATA lock here because, if the following TLFindStreamNodeByName()
+	 * returns NULL, a new TL_SNODE will be added to TL_GLOBAL_DATA's TL_SNODE list */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Check if there already exists a stream with this name. */
+	psn = TLFindStreamNodeByName( szStreamName );
+	if (NULL != psn)
+	{
+		eError = PVRSRV_ERROR_ALREADY_EXISTS;
+		goto e0;
+	}
+
+	/* Allocate stream structure container (stream struct) for the new stream */
+	psTmp = OSAllocZMem(sizeof(TL_STREAM));
+	if (NULL == psTmp)
+	{
+		eError = PVRSRV_ERROR_OUT_OF_MEMORY;
+		goto e0;
+	}
+
+	OSStringSafeCopy(psTmp->szName, szStreamName, PRVSRVTL_MAX_STREAM_NAME_SIZE);
+
+	if (ui32StreamFlags & TL_FLAG_FORCE_FLUSH)
+	{
+		psTmp->bWaitForEmptyOnDestroy = IMG_TRUE;
+	}
+
+	psTmp->bNoSignalOnCommit = (ui32StreamFlags&TL_FLAG_NO_SIGNAL_ON_COMMIT) ? IMG_TRUE : IMG_FALSE;
+	psTmp->bNoWrapPermanent = (ui32StreamFlags&TL_FLAG_PERMANENT_NO_WRAP) ? IMG_TRUE : IMG_FALSE;
+
+	psTmp->eOpMode = eOpMode;
+	if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+	{
+		/* Only allow drop properties to be mixed with no-wrap type streams
+		 * since space does not become available when reads take place hence
+		 * no point blocking.
+		 */
+		if (psTmp->bNoWrapPermanent)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e1;
+		}
+	}
+
+	/* Additional synchronisation object required for some streams e.g. blocking */
+	eError = OSEventObjectCreate(NULL, &psTmp->hProducerEventObj);
+	PVR_GOTO_IF_ERROR(eError, e1);
+	/* Create an event handle for this kind of stream */
+	eError = OSEventObjectOpen(psTmp->hProducerEventObj, &psTmp->hProducerEvent);
+	PVR_GOTO_IF_ERROR(eError, e2);
+
+	psTmp->pfOnReaderOpenCallback = pfOnReaderOpenCB;
+	psTmp->pvOnReaderOpenUserData = pvOnReaderOpenUD;
+	psTmp->pfOnReaderCloseCallback = pfOnReaderCloseCB;
+	psTmp->pvOnReaderCloseUserData = pvOnReaderCloseUD;
+	/* Remember producer supplied CB and data for later */
+	psTmp->pfProducerCallback = (void(*)(void))pfProducerCB;
+	psTmp->pvProducerUserData = pvProducerUD;
+
+	psTmp->psNotifStream = NULL;
+
+	/* Round the requested bytes to a multiple of array elements' size, eg round 3 to 4 */
+	psTmp->ui32Size = PVRSRVTL_ALIGN(ui32Size);
+
+	/* Signalling from TLStreamCommit is deferred until buffer is slightly (~12%) filled */
+	psTmp->ui32ThresholdUsageForSignal = psTmp->ui32Size >> 3;
+	psTmp->ui32MaxPacketSize = GET_TL_MAX_PACKET_SIZE(psTmp->ui32Size);
+	psTmp->ui32Read = 0;
+	psTmp->ui32Write = 0;
+	psTmp->ui32Pending = NOTHING_PENDING;
+	psTmp->bReadPending = IMG_FALSE;
+	psTmp->bSignalPending = IMG_FALSE;
+
+#if defined(TL_BUFFER_STATS)
+	OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 0);
+	/* Setting MAX possible value for "minimum" time to full,
+	 * helps in the logic which calculates this time */
+	psTmp->ui32MinTimeToFullInUs = IMG_UINT32_MAX;
+#endif
+
+	/* Memory will be allocated on first connect to the stream */
+	if (!(ui32StreamFlags & TL_FLAG_ALLOCATE_ON_FIRST_OPEN))
+	{
+		/* Allocate memory for the circular buffer and export it to user space. */
+		eError = TLAllocSharedMemIfNull(psTmp);
+		PVR_LOG_GOTO_IF_ERROR(eError, "TLAllocSharedMem", e3);
+	}
+
+	/* Synchronisation object to synchronise with user side data transfers. */
+	eError = OSEventObjectCreate(psTmp->szName, &hEventList);
+	PVR_GOTO_IF_ERROR(eError, e4);
+
+	eError = OSLockCreate (&psTmp->hStreamWLock);
+	PVR_GOTO_IF_ERROR(eError, e5);
+
+	eError = OSLockCreate (&psTmp->hReadLock);
+	PVR_GOTO_IF_ERROR(eError, e6);
+
+	/* Now remember the stream in the global TL structures */
+	psn = TLMakeSNode(hEventList, (TL_STREAM *)psTmp, NULL);
+	PVR_GOTO_IF_NOMEM(psn, eError, e7);
+
+	/* Stream node created, now reset the write reference count to 1
+	 * (i.e. this context's reference) */
+	psn->uiWRefCount = 1;
+
+	TLAddStreamNode(psn);
+
+	/* Release TL_GLOBAL_DATA lock as the new TL_SNODE is now added to the list */
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	/* Best effort signal, client wait timeout will ultimately let it find the
+	 * new stream if this fails, acceptable to avoid clean-up as it is tricky
+	 * at this point */
+	(void) OSEventObjectSignal(TLGGD()->hTLEventObj);
+
+	/* Pass the newly created stream handle back to caller */
+	*phStream = (IMG_HANDLE)psTmp;
+	PVR_DPF_RETURN_OK;
+
+e7:
+	OSLockDestroy(psTmp->hReadLock);
+e6:
+	OSLockDestroy(psTmp->hStreamWLock);
+e5:
+	OSEventObjectDestroy(hEventList);
+e4:
+	TLFreeSharedMem(psTmp);
+e3:
+	OSEventObjectClose(psTmp->hProducerEvent);
+e2:
+	OSEventObjectDestroy(psTmp->hProducerEventObj);
+e1:
+	OSFreeMem(psTmp);
+e0:
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+void TLStreamReset(IMG_HANDLE hStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+	IMG_HANDLE hEventWaitForWriterToComplete;
+	PVRSRV_ERROR eError;
+
+	PVR_ASSERT(psStream != NULL);
+
+	OSLockAcquire(psStream->hStreamWLock);
+
+	while (psStream->ui32Pending != NOTHING_PENDING)
+	{
+		/* We're in the middle of a write so we cannot reset the stream.
+		 * We are going to wait until the data is committed. Release lock while
+		 * we're here. */
+		OSLockRelease(psStream->hStreamWLock);
+
+		eError = OSEventObjectOpen(psStream->psNode->hReadEventObj,
+		                           &hEventWaitForWriterToComplete);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectOpen");
+
+		/* Event when psStream->bNoSignalOnCommit is set we can still use
+		 * the timeout capability of event object API (time in us). */
+		eError = OSEventObjectWaitTimeout(hEventWaitForWriterToComplete, 100);
+		if (eError != PVRSRV_ERROR_TIMEOUT && eError != PVRSRV_OK)
+		{
+			PVR_LOG_GOTO_IF_ERROR(eError, "OSEventObjectWaitTimeout", TimeoutError);
+		}
+
+		eError = OSEventObjectClose(hEventWaitForWriterToComplete);
+		PVR_LOG_RETURN_VOID_IF_ERROR(eError, "OSEventObjectClose");
+
+		OSLockAcquire(psStream->hStreamWLock);
+
+		/* Either timeout occurred or the stream has been signalled.
+		 * If former we have to check if the data was committed and if latter
+		 * if the stream hasn't been re-reserved. Either way we have to go
+		 * back to the condition.
+		 * If the stream has been released we'll exit with the lock held so
+		 * we can finally go and reset the stream. */
+	}
+
+	psStream->ui32Read = 0;
+	psStream->ui32Write = 0;
+	/* we know that ui32Pending already has correct value (no need to set) */
+
+	OSLockRelease(psStream->hStreamWLock);
+
+	return;
+
+TimeoutError:
+	eError = OSEventObjectClose(hEventWaitForWriterToComplete);
+	PVR_LOG_IF_ERROR(eError, "OSEventObjectClose");
+}
+
+PVRSRV_ERROR
+TLStreamSetNotifStream(IMG_HANDLE hStream, IMG_HANDLE hNotifStream)
+{
+	PTL_STREAM psStream = (PTL_STREAM) hStream;
+
+	if (hStream == NULL || hNotifStream == NULL)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psStream->psNotifStream = (PTL_STREAM) hNotifStream;
+
+	return PVRSRV_OK;
+}
+
+PVRSRV_ERROR
+TLStreamReconfigure(
+		IMG_HANDLE hStream,
+		IMG_UINT32 ui32StreamFlags)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PTL_STREAM psTmp;
+	TL_OPMODE eOpMode;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eOpMode = ui32StreamFlags & TL_OPMODE_MASK;
+	if (( eOpMode <= TL_OPMODE_UNDEF ) || ( eOpMode >= TL_OPMODE_LAST ))
+	{
+		PVR_DPF((PVR_DBG_ERROR, "OpMode for TL stream is invalid"));
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Prevent the TL Stream buffer from being written to
+	 * while its mode is being reconfigured
+	 */
+	OSLockAcquire (psTmp->hStreamWLock);
+	if (NOTHING_PENDING != psTmp->ui32Pending)
+	{
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_READY);
+	}
+	psTmp->ui32Pending = 0;
+	OSLockRelease (psTmp->hStreamWLock);
+
+	psTmp->eOpMode = eOpMode;
+	if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+	{
+		/* Only allow drop properties to be mixed with no-wrap type streams
+		 * since space does not become available when reads take place hence
+		 * no point blocking.
+		 */
+		if (psTmp->bNoWrapPermanent)
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+			goto e1;
+		}
+	}
+
+	OSLockAcquire (psTmp->hStreamWLock);
+	psTmp->ui32Pending = NOTHING_PENDING;
+	OSLockRelease (psTmp->hStreamWLock);
+e1:
+	PVR_DPF_RETURN_RC(eError);
+}
+
+PVRSRV_ERROR
+TLStreamOpen(IMG_HANDLE     *phStream,
+             const IMG_CHAR *szStreamName)
+{
+	PTL_SNODE psTmpSNode;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == phStream || NULL == szStreamName)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	/* Acquire the TL_GLOBAL_DATA lock first to ensure,
+	 * the TL_STREAM while returned and being modified,
+	 * is not deleted by some other context */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Search for a stream node with a matching stream name */
+	psTmpSNode = TLFindStreamNodeByName(szStreamName);
+
+	if (NULL == psTmpSNode)
+	{
+		OSLockRelease (TLGGD()->hTLGDLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_FOUND);
+	}
+
+	if (psTmpSNode->psStream->psNotifStream != NULL &&
+	    psTmpSNode->uiWRefCount == 1)
+	{
+		TLStreamMarkStreamOpen(psTmpSNode->psStream);
+	}
+
+	/* The TL_SNODE->uiWRefCount governs the presence of this node in the
+	 * TL_GLOBAL_DATA list i.e. when uiWRefCount falls to zero we try removing
+	 * this node from the TL_GLOBAL_DATA list. Hence, is protected using the
+	 * TL_GLOBAL_DATA lock and not TL_STREAM lock */
+	psTmpSNode->uiWRefCount++;
+
+	OSLockRelease (TLGGD()->hTLGDLock);
+
+	/* Return the stream handle to the caller */
+	*phStream = (IMG_HANDLE)psTmpSNode->psStream;
+
+	PVR_DPF_RETURN_VAL(PVRSRV_OK);
+}
+
+void
+TLStreamClose(IMG_HANDLE hStream)
+{
+	PTL_STREAM	psTmp;
+	IMG_BOOL	bDestroyStream;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF((PVR_DBG_WARNING,
+				 "TLStreamClose failed as NULL stream handler passed, nothing done."));
+		PVR_DPF_RETURN;
+	}
+
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Acquire TL_GLOBAL_DATA lock for updating the reference count as this will be required
+	 * in-case this TL_STREAM node is to be deleted */
+	OSLockAcquire (TLGGD()->hTLGDLock);
+
+	/* Decrement write reference counter of the stream */
+	psTmp->psNode->uiWRefCount--;
+
+	if (0 != psTmp->psNode->uiWRefCount)
+	{
+		/* The stream is still being used in other context(s) do not destroy
+		 * anything */
+
+		/* uiWRefCount == 1 means that stream was closed for write. Next
+		 * close is pairing TLStreamCreate(). Send notification to indicate
+		 * that no writer are connected to the stream any more. */
+		if (psTmp->psNotifStream != NULL && psTmp->psNode->uiWRefCount == 1)
+		{
+			TLStreamMarkStreamClose(psTmp);
+		}
+
+		OSLockRelease (TLGGD()->hTLGDLock);
+		PVR_DPF_RETURN;
+	}
+	else
+	{
+		/* Now we try removing this TL_STREAM from TL_GLOBAL_DATA */
+
+		if (psTmp->bWaitForEmptyOnDestroy)
+		{
+			/* We won't require the TL_STREAM lock to be acquired here for accessing its read
+			 * and write offsets. REASON: We are here because there is no producer context
+			 * referencing this TL_STREAM, hence its ui32Write offset won't be changed now.
+			 * Also, the update of ui32Read offset is not protected by locks */
+			while (psTmp->ui32Read != psTmp->ui32Write)
+			{
+				/* Release lock before sleeping */
+				OSLockRelease (TLGGD()->hTLGDLock);
+
+				OSEventObjectWaitTimeout(psTmp->hProducerEvent, EVENT_OBJECT_TIMEOUT_US);
+
+				OSLockAcquire (TLGGD()->hTLGDLock);
+
+				/* Ensure destruction of stream is still required */
+				if (0 != psTmp->psNode->uiWRefCount)
+				{
+					OSLockRelease (TLGGD()->hTLGDLock);
+					PVR_DPF_RETURN;
+				}
+			}
+		}
+
+		/* Try removing the stream from TL_GLOBAL_DATA */
+		bDestroyStream = TLTryRemoveStreamAndFreeStreamNode (psTmp->psNode);
+
+		OSLockRelease (TLGGD()->hTLGDLock);
+
+		if (bDestroyStream)
+		{
+			/* Destroy the stream if it was removed from TL_GLOBAL_DATA */
+			TLStreamDestroy (psTmp);
+			psTmp = NULL;
+		}
+		PVR_DPF_RETURN;
+	}
+}
+
+/*
+ * DoTLSetPacketHeader
+ *
+ * Ensure that whenever we update a Header we always add the RESERVED field
+ */
+static inline void DoTLSetPacketHeader(PVRSRVTL_PPACKETHDR, IMG_UINT32);
+static inline void
+DoTLSetPacketHeader(PVRSRVTL_PPACKETHDR pHdr,
+				IMG_UINT32 ui32Val)
+{
+	PVR_ASSERT(((size_t)pHdr & (size_t)(PVRSRVTL_PACKET_ALIGNMENT - 1)) == 0);
+
+	/* Check that this is a correctly aligned packet header. */
+	if (((size_t)pHdr & (size_t)(PVRSRVTL_PACKET_ALIGNMENT - 1)) != 0)
+	{
+		/* Should return an error because the header is misaligned */
+		PVR_DPF((PVR_DBG_ERROR, "%s: Misaligned header @ %p", __func__, pHdr));
+		pHdr->uiTypeSize = ui32Val;
+	}
+	else
+	{
+		pHdr->uiTypeSize = ui32Val;
+		pHdr->uiReserved = PVRSRVTL_PACKETHDR_RESERVED;
+	}
+}
+
+static PVRSRV_ERROR
+DoTLStreamReserve(IMG_HANDLE hStream,
+				IMG_UINT8 **ppui8Data,
+				IMG_UINT32 ui32ReqSize,
+				IMG_UINT32 ui32ReqSizeMin,
+				PVRSRVTL_PACKETTYPE ePacketType,
+				IMG_UINT32* pui32AvSpace,
+				IMG_UINT32* pui32Flags)
+{
+	PTL_STREAM psTmp;
+	IMG_UINT32 *pui32Buf, ui32LRead, ui32LWrite, ui32LPending, lReqSizeAligned, lReqSizeActual, ui32CreateFreeSpace;
+	IMG_UINT32 ui32InputFlags = 0;
+	IMG_INT pad, iFreeSpace;
+	IMG_UINT8 *pui8IncrRead = NULL;
+	PVRSRVTL_PPACKETHDR pHdr;
+
+	PVR_DPF_ENTERED;
+	if (pui32AvSpace) *pui32AvSpace = 0;
+	if (pui32Flags)
+	{
+		ui32InputFlags = *pui32Flags;
+		*pui32Flags = 0;
+	}
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Assert used as the packet type parameter is currently only provided
+	 * by the TL APIs, not the calling client */
+	PVR_ASSERT((PVRSRVTL_PACKETTYPE_UNDEF < ePacketType) && (PVRSRVTL_PACKETTYPE_LAST >= ePacketType));
+
+	/* The buffer is only used in "rounded" (aligned) chunks */
+	lReqSizeAligned = PVRSRVTL_ALIGN(ui32ReqSize);
+
+	/* Lock the stream before reading it's pending value, because if pending is set
+	 * to NOTHING_PENDING, we update the pending value such that subsequent calls to
+	 * this function from other context(s) fail with PVRSRV_ERROR_NOT_READY */
+	OSLockAcquire (psTmp->hStreamWLock);
+
+#if defined(TL_BUFFER_STATS)
+	/* If writing into an empty buffer, start recording time-to-full */
+	if (psTmp->ui32Read == psTmp->ui32Write)
+	{
+		OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 1);
+		psTmp->ui32TimeStart = OSClockus();
+	}
+
+	if (ui32ReqSize > psTmp->ui32MaxReserveWatermark)
+	{
+		psTmp->ui32MaxReserveWatermark = ui32ReqSize;
+	}
+#endif
+
+	/* Get a local copy of the stream buffer parameters */
+	ui32LRead  = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+	ui32LPending = psTmp->ui32Pending;
+
+	/* Multiple pending reserves are not supported. */
+	if (NOTHING_PENDING != ui32LPending)
+	{
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_NOT_READY);
+	}
+
+	if (psTmp->ui32MaxPacketSize < lReqSizeAligned)
+	{
+		PVR_DPF((PVR_DBG_ERROR, "Requested Size: %u > TL Max Packet size: %u", lReqSizeAligned, psTmp->ui32MaxPacketSize));
+		psTmp->ui32Pending = NOTHING_PENDING;
+		if (pui32AvSpace)
+		{
+			*pui32AvSpace = suggestAllocSize(ui32LRead, ui32LWrite, psTmp->ui32Size, ui32ReqSizeMin, psTmp->ui32MaxPacketSize);
+			if (*pui32AvSpace == 0 && psTmp->eOpMode == TL_OPMODE_DROP_OLDEST)
+			{
+				*pui32AvSpace = psTmp->ui32MaxPacketSize;
+				PVR_DPF((PVR_DBG_MESSAGE, "Opmode is Drop_Oldest, so Available Space changed to: %u", *pui32AvSpace));
+			}
+		}
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED);
+	}
+
+	/* Prevent other threads from entering this region before we are done
+	 * updating the pending value and write offset (in case of padding). This
+	 * is not exactly a lock but a signal for other contexts that there is a
+	 * TLStreamCommit operation pending on this stream */
+	psTmp->ui32Pending = 0;
+
+	OSLockRelease (psTmp->hStreamWLock);
+
+	/* If there is enough contiguous space following the current Write
+	 * position then no padding is required */
+	if (  psTmp->ui32Size
+		< ui32LWrite + lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR) )
+	{
+		pad = psTmp->ui32Size - ui32LWrite;
+	}
+	else
+	{
+		pad = 0;
+	}
+
+	lReqSizeActual = lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR) + pad;
+	if (psTmp->bNoWrapPermanent)
+	{
+		iFreeSpace = bufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+	else
+	{
+		iFreeSpace = circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+
+	if (iFreeSpace < (IMG_INT) lReqSizeActual)
+	{
+		/* If this is a blocking reserve and there is not enough space then wait. */
+		if (psTmp->eOpMode == TL_OPMODE_BLOCK)
+		{
+			/* Stream create should stop us entering here when
+			 * psTmp->bNoWrapPermanent is true as it does not make sense to
+			 * block on permanent data streams. */
+			PVR_ASSERT(psTmp->bNoWrapPermanent == IMG_FALSE);
+			while ( ( circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size)
+				 <(IMG_INT) lReqSizeActual ) )
+			{
+				/* The TL bridge is lockless now, so changing to OSEventObjectWait() */
+				OSEventObjectWait(psTmp->hProducerEvent);
+				// update local copies.
+				ui32LRead  = psTmp->ui32Read;
+				ui32LWrite = psTmp->ui32Write;
+			}
+		}
+		/* Data overwriting, also insert PACKETS_DROPPED flag into existing packet */
+		else if (psTmp->eOpMode == TL_OPMODE_DROP_OLDEST)
+		{
+			OSLockAcquire(psTmp->hReadLock);
+
+			while (psTmp->bReadPending)
+			{
+				PVR_DPF((PVR_DBG_MESSAGE, "Waiting for the pending read operation to complete."));
+				OSLockRelease(psTmp->hReadLock);
+#if defined(TL_BUFFER_STATS)
+				TL_COUNTER_INC(psTmp->ui32CntWriteWaits);
+#endif
+				(void) OSEventObjectWaitTimeout(psTmp->hProducerEvent, READ_PENDING_TIMEOUT_US);
+				OSLockAcquire(psTmp->hReadLock);
+			}
+
+#if defined(TL_BUFFER_STATS)
+			TL_COUNTER_INC(psTmp->ui32CntWriteSuccesses);
+#endif
+			ui32LRead = psTmp->ui32Read;
+
+			if ( circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size)
+			     < (IMG_INT) lReqSizeActual )
+			{
+				ui32CreateFreeSpace = 5 * (psTmp->ui32Size / 100);
+				if (ui32CreateFreeSpace < lReqSizeActual)
+				{
+					ui32CreateFreeSpace = lReqSizeActual;
+				}
+
+				while (ui32CreateFreeSpace > (IMG_UINT32)circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size))
+				{
+					pui8IncrRead = &psTmp->pbyBuffer[ui32LRead];
+					ui32LRead += (sizeof(PVRSRVTL_PACKETHDR) + PVRSRVTL_ALIGN( GET_PACKET_DATA_LEN(pui8IncrRead) ));
+
+					/* Check if buffer needs to wrap */
+					if (ui32LRead >= psTmp->ui32Size)
+					{
+						ui32LRead = 0;
+					}
+				}
+				psTmp->ui32Read = ui32LRead;
+				pui8IncrRead = &psTmp->pbyBuffer[psTmp->ui32Read];
+
+				pHdr = GET_PACKET_HDR(pui8IncrRead);
+				DoTLSetPacketHeader(pHdr, SET_PACKETS_DROPPED(pHdr));
+			}
+			/* else fall through as there is enough space now to write the data */
+
+			OSLockRelease(psTmp->hReadLock);
+			/* If we accepted a flag var set the OVERWRITE bit*/
+			if (pui32Flags) *pui32Flags |= TL_FLAG_OVERWRITE_DETECTED;
+		}
+		/* No data overwriting, insert write_failed flag and return */
+		else if (psTmp->eOpMode == TL_OPMODE_DROP_NEWER)
+		{
+			/* Caller should not try to use ppui8Data,
+			 * NULLify to give user a chance of avoiding memory corruption */
+			*ppui8Data = NULL;
+
+			/* This flag should not be inserted two consecutive times, so
+			 * check the last ui32 in case it was a packet drop packet. */
+			pui32Buf = ui32LWrite
+					  ?
+					    (void *)&psTmp->pbyBuffer[ui32LWrite - sizeof(PVRSRVTL_PACKETHDR)]
+					   : // Previous four bytes are not guaranteed to be a packet header...
+					    (void *)&psTmp->pbyBuffer[psTmp->ui32Size - PVRSRVTL_PACKET_ALIGNMENT];
+
+			pHdr = GET_PACKET_HDR(pui32Buf);
+			if ( PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED
+				 !=
+				 GET_PACKET_TYPE( pHdr ) && (ui32InputFlags & TL_FLAG_NO_WRITE_FAILED) == 0)
+			{
+				/* Insert size-stamped packet header */
+				pui32Buf = (void *)&psTmp->pbyBuffer[ui32LWrite];
+				pHdr = GET_PACKET_HDR(pui32Buf);
+				DoTLSetPacketHeader(pHdr, PVRSRVTL_SET_PACKET_WRITE_FAILED);
+				ui32LWrite += sizeof(PVRSRVTL_PACKETHDR);
+				ui32LWrite %= psTmp->ui32Size;
+				iFreeSpace -= sizeof(PVRSRVTL_PACKETHDR);
+			}
+
+			OSLockAcquire (psTmp->hStreamWLock);
+			psTmp->ui32Write = ui32LWrite;
+			psTmp->ui32Pending = NOTHING_PENDING;
+			OSLockRelease (psTmp->hStreamWLock);
+
+			if (pui32AvSpace)
+			{
+				*pui32AvSpace = suggestAllocSize(ui32LRead, ui32LWrite, psTmp->ui32Size, ui32ReqSizeMin, psTmp->ui32MaxPacketSize);
+			}
+
+			/* Inform call of permanent stream misuse, no space left,
+			 * the size of the stream will need to be increased. */
+			if (psTmp->bNoWrapPermanent)
+			{
+				PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE);
+			}
+
+			PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_FULL);
+		}
+	}
+
+	/* The easy case: buffer has enough space to hold the requested packet (data + header) */
+
+	/* Should we treat the buffer as non-circular buffer? */
+	if (psTmp->bNoWrapPermanent)
+	{
+		iFreeSpace = bufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+	else
+	{
+		iFreeSpace = circbufSpaceLeft(ui32LRead, ui32LWrite, psTmp->ui32Size);
+	}
+
+	if (iFreeSpace >= (IMG_INT) lReqSizeActual)
+	{
+		if (pad)
+		{
+			/* Inserting padding packet. */
+			pui32Buf = (void *)&psTmp->pbyBuffer[ui32LWrite];
+			pHdr = GET_PACKET_HDR(pui32Buf);
+			DoTLSetPacketHeader(pHdr,
+				PVRSRVTL_SET_PACKET_PADDING(pad-sizeof(PVRSRVTL_PACKETHDR)));
+
+			/* CAUTION: the used pad value should always result in a properly
+			 *          aligned ui32LWrite pointer, which in this case is 0 */
+			ui32LWrite = (ui32LWrite + pad) % psTmp->ui32Size;
+			/* Detect unaligned pad value */
+			PVR_ASSERT(ui32LWrite == 0);
+		}
+		/* Insert size-stamped packet header */
+		pui32Buf = (void *) &psTmp->pbyBuffer[ui32LWrite];
+
+		pHdr = GET_PACKET_HDR(pui32Buf);
+		DoTLSetPacketHeader(pHdr,
+			PVRSRVTL_SET_PACKET_HDR(ui32ReqSize, ePacketType));
+
+		/* return the next position in the buffer to the user */
+		*ppui8Data = &psTmp->pbyBuffer[ ui32LWrite+sizeof(PVRSRVTL_PACKETHDR) ];
+
+		/* update pending offset: size stamp + data */
+		ui32LPending = lReqSizeAligned + sizeof(PVRSRVTL_PACKETHDR);
+	}
+	else
+	{
+		OSLockAcquire (psTmp->hStreamWLock);
+		psTmp->ui32Pending = NOTHING_PENDING;
+		OSLockRelease (psTmp->hStreamWLock);
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_ERROR);
+	}
+
+	/* Acquire stream lock for updating stream parameters */
+	OSLockAcquire (psTmp->hStreamWLock);
+	psTmp->ui32Write = ui32LWrite;
+	psTmp->ui32Pending = ui32LPending;
+	OSLockRelease (psTmp->hStreamWLock);
+
+#if defined(TL_BUFFER_STATS)
+	TL_COUNTER_INC(psTmp->ui32CntNumWriteSuccess);
+#endif
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamReserve(IMG_HANDLE hStream,
+				IMG_UINT8 **ppui8Data,
+				IMG_UINT32 ui32Size)
+{
+	return DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32Size, PVRSRVTL_PACKETTYPE_DATA, NULL, NULL);
+}
+
+PVRSRV_ERROR
+TLStreamReserve2(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size,
+                IMG_UINT32 ui32SizeMin,
+                IMG_UINT32* pui32Available,
+                IMG_BOOL* pbIsReaderConnected)
+{
+	PVRSRV_ERROR eError;
+
+	eError = DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32SizeMin, PVRSRVTL_PACKETTYPE_DATA, pui32Available, NULL);
+	if (eError != PVRSRV_OK && pbIsReaderConnected != NULL)
+	{
+		*pbIsReaderConnected = TLStreamIsOpenForReading(hStream);
+	}
+
+	return eError;
+}
+
+PVRSRV_ERROR
+TLStreamReserveReturnFlags(IMG_HANDLE hStream,
+						   IMG_UINT8  **ppui8Data,
+						   IMG_UINT32 ui32Size,
+						   IMG_UINT32* pui32Flags)
+{
+	return DoTLStreamReserve(hStream, ppui8Data, ui32Size, ui32Size, PVRSRVTL_PACKETTYPE_DATA, NULL, pui32Flags);
+}
+
+PVRSRV_ERROR
+TLStreamCommit(IMG_HANDLE hStream, IMG_UINT32 ui32ReqSize)
+{
+	PTL_STREAM psTmp;
+	IMG_UINT32 ui32LRead, ui32OldWrite, ui32LWrite, ui32LPending;
+	PVRSRV_ERROR eError;
+
+#if defined(TL_BUFFER_STATS)
+	IMG_UINT32 ui32UnreadBytes;
+#endif
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)hStream;
+
+	/* Get a local copy of the stream buffer parameters */
+	ui32LRead = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+	ui32LPending = psTmp->ui32Pending;
+
+	ui32OldWrite = ui32LWrite;
+
+	// Space in buffer is aligned
+	ui32ReqSize = PVRSRVTL_ALIGN(ui32ReqSize) + sizeof(PVRSRVTL_PACKETHDR);
+
+	/* Check pending reserver and ReqSize + packet header size. */
+	if ((ui32LPending == NOTHING_PENDING) || (ui32ReqSize > ui32LPending))
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_MISUSE);
+	}
+
+	/* Update pointer to written data. */
+	ui32LWrite = (ui32LWrite + ui32ReqSize) % psTmp->ui32Size;
+
+	/* and reset LPending to 0 since data are now submitted */
+	ui32LPending = NOTHING_PENDING;
+
+#if defined(TL_BUFFER_STATS)
+	/* Calculate new number of bytes unread */
+	if (ui32LWrite > ui32LRead)
+	{
+		ui32UnreadBytes = (ui32LWrite-ui32LRead);
+	}
+	else if (ui32LWrite < ui32LRead)
+	{
+		ui32UnreadBytes = (psTmp->ui32Size-ui32LRead+ui32LWrite);
+	}
+	else
+	{ /* else equal, ignore */
+		ui32UnreadBytes = 0;
+	}
+
+	/* Calculate high water mark for debug purposes */
+	if (ui32UnreadBytes > psTmp->ui32BufferUt)
+	{
+		psTmp->ui32BufferUt = ui32UnreadBytes;
+	}
+#endif
+
+	/* Memory barrier required to ensure prior data written by writer is
+	 * flushed from WC buffer to main memory. */
+	OSWriteMemoryBarrier(NULL);
+
+	/* Acquire stream lock to ensure other context(s) (if any)
+	 * wait on the lock (in DoTLStreamReserve) for consistent values
+	 * of write offset and pending value */
+	OSLockAcquire (psTmp->hStreamWLock);
+
+	/* Update stream buffer parameters to match local copies */
+	psTmp->ui32Write = ui32LWrite;
+	psTmp->ui32Pending = ui32LPending;
+
+	/* Ensure write pointer is flushed */
+	OSWriteMemoryBarrier(&psTmp->ui32Write);
+
+	TL_COUNTER_ADD(psTmp->ui32ProducerByteCount, ui32ReqSize);
+	TL_COUNTER_INC(psTmp->ui32NumCommits);
+
+#if defined(TL_BUFFER_STATS)
+	/* IF there has been no-reader since first reserve on an empty-buffer,
+	 * AND current utilisation is considerably high (90%), calculate the
+	 * time taken to fill up the buffer */
+	if ((OSAtomicRead(&psTmp->bNoReaderSinceFirstReserve) == 1) &&
+	    (TLStreamGetUT(psTmp) >= 90 * psTmp->ui32Size/100))
+	{
+		IMG_UINT32 ui32TimeToFullInUs = OSClockus() - psTmp->ui32TimeStart;
+		if (psTmp->ui32MinTimeToFullInUs > ui32TimeToFullInUs)
+		{
+			psTmp->ui32MinTimeToFullInUs = ui32TimeToFullInUs;
+		}
+		/* Following write ensures ui32MinTimeToFullInUs doesn't lose its
+		 * real (expected) value in case there is no reader until next Commit call */
+		OSAtomicWrite(&psTmp->bNoReaderSinceFirstReserve, 0);
+	}
+#endif
+
+	if (!psTmp->bNoSignalOnCommit)
+	{
+		/* If we have transitioned from an empty buffer to a non-empty buffer, we
+		 * must signal possibly waiting consumer. BUT, let the signal be "deferred"
+		 * until buffer is at least 'ui32ThresholdUsageForSignal' bytes full. This
+		 * avoids a race between OSEventObjectSignal and OSEventObjectWaitTimeout
+		 * (in TLServerAcquireDataKM), where a "signal" might happen before "wait",
+		 * resulting into signal being lost and stream-reader waiting even though
+		 * buffer is no-more empty */
+		if (ui32OldWrite == ui32LRead)
+		{
+			psTmp->bSignalPending = IMG_TRUE;
+		}
+
+		if (psTmp->bSignalPending && (TLStreamGetUT(psTmp) >= psTmp->ui32ThresholdUsageForSignal))
+		{
+			TL_COUNTER_INC(psTmp->ui32SignalsSent);
+			psTmp->bSignalPending = IMG_FALSE;
+
+			/* Signal consumers that may be waiting */
+			eError = OSEventObjectSignal(psTmp->psNode->hReadEventObj);
+			if (eError != PVRSRV_OK)
+			{
+				OSLockRelease (psTmp->hStreamWLock);
+				PVR_DPF_RETURN_RC(eError);
+			}
+		}
+		else
+		{
+			TL_COUNTER_INC(psTmp->ui32SignalNotSent);
+		}
+	}
+	OSLockRelease (psTmp->hStreamWLock);
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamWrite(IMG_HANDLE hStream, IMG_UINT8 *pui8Src, IMG_UINT32 ui32Size)
+{
+	IMG_BYTE *pbyDest = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = TLStreamReserve(hStream, &pbyDest, ui32Size);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	else
+	{
+		OSDeviceMemCopy((void*)pbyDest, (void*)pui8Src, ui32Size);
+		eError = TLStreamCommit(hStream, ui32Size);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF_RETURN_RC(eError);
+		}
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+PVRSRV_ERROR
+TLStreamWriteRetFlags(IMG_HANDLE hStream, IMG_UINT8 *pui8Src, IMG_UINT32 ui32Size, IMG_UINT32 *pui32Flags){
+	IMG_BYTE *pbyDest = NULL;
+	PVRSRV_ERROR eError;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == hStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	eError = TLStreamReserveReturnFlags(hStream, &pbyDest, ui32Size, pui32Flags);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+	else
+	{
+		OSDeviceMemCopy((void*)pbyDest, (void*)pui8Src, ui32Size);
+		eError = TLStreamCommit(hStream, ui32Size);
+		if (PVRSRV_OK != eError)
+		{
+			PVR_DPF_RETURN_RC(eError);
+		}
+	}
+
+	PVR_DPF_RETURN_OK;
+}
+
+void TLStreamInfo(IMG_HANDLE hStream, PTL_STREAM_INFO psInfo)
+{
+	IMG_DEVMEM_SIZE_T actual_req_size;
+	IMG_DEVMEM_ALIGN_T align = 4; /* Low fake value so the real value can be obtained */
+
+	actual_req_size = 2;
+	/* ignore error as OSGetPageShift() should always return correct value */
+	(void) DevmemExportalignAdjustSizeAndAlign(OSGetPageShift(), &actual_req_size, &align);
+
+	psInfo->headerSize = sizeof(PVRSRVTL_PACKETHDR);
+	psInfo->minReservationSize = sizeof(IMG_UINT32);
+	psInfo->pageSize = (IMG_UINT32)(actual_req_size);
+	psInfo->pageAlign = (IMG_UINT32)(align);
+	psInfo->maxTLpacketSize = ((PTL_STREAM)hStream)->ui32MaxPacketSize;
+}
+
+PVRSRV_ERROR
+TLStreamMarkEOS(IMG_HANDLE psStream, IMG_BOOL bRemoveOld)
+{
+	PTL_STREAM   psTmp;
+	PVRSRV_ERROR eError;
+	IMG_UINT8*   pData;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	psTmp = (PTL_STREAM)psStream;
+
+	/* Do not support EOS packets on permanent stream buffers at present,
+	 * EOS is best used with streams where data is consumed. */
+	if (psTmp->bNoWrapPermanent)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_STREAM_MISUSE);
+	}
+
+	if (bRemoveOld)
+	{
+		eError = DoTLStreamReserve(psStream, &pData, 0, 0, PVRSRVTL_PACKETTYPE_MARKER_EOS_REMOVEOLD, NULL, NULL);
+	}
+	else
+	{
+		eError = DoTLStreamReserve(psStream, &pData, 0, 0, PVRSRVTL_PACKETTYPE_MARKER_EOS, NULL, NULL);
+	}
+
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	PVR_DPF_RETURN_RC(TLStreamCommit(psStream, 0));
+}
+
+
+static PVRSRV_ERROR
+_TLStreamMarkOC(IMG_HANDLE hStream, PVRSRVTL_PACKETTYPE ePacketType)
+{
+	PVRSRV_ERROR eError;
+	PTL_STREAM psStream = hStream;
+	IMG_UINT32 ui32Size;
+	IMG_UINT8 *pData;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+
+	if (NULL == psStream->psNotifStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_NOTIF_STREAM);
+	}
+
+	ui32Size = OSStringLength(psStream->szName) + 1;
+
+	eError = DoTLStreamReserve(psStream->psNotifStream, &pData, ui32Size,
+	                           ui32Size, ePacketType, NULL, NULL);
+	if (PVRSRV_OK != eError)
+	{
+		PVR_DPF_RETURN_RC(eError);
+	}
+
+	OSDeviceMemCopy(pData, psStream->szName, ui32Size);
+
+	PVR_DPF_RETURN_RC(TLStreamCommit(psStream->psNotifStream, ui32Size));
+}
+
+PVRSRV_ERROR
+TLStreamMarkStreamOpen(IMG_HANDLE psStream)
+{
+	return _TLStreamMarkOC(psStream, PVRSRVTL_PACKETTYPE_STREAM_OPEN_FOR_WRITE);
+}
+
+PVRSRV_ERROR
+TLStreamMarkStreamClose(IMG_HANDLE psStream)
+{
+	return _TLStreamMarkOC(psStream, PVRSRVTL_PACKETTYPE_STREAM_CLOSE_FOR_WRITE);
+}
+
+PVRSRV_ERROR
+TLStreamSync(IMG_HANDLE psStream)
+{
+	PVRSRV_ERROR eError = PVRSRV_OK;
+	PTL_STREAM   psTmp;
+
+	PVR_DPF_ENTERED;
+
+	if (NULL == psStream)
+	{
+		PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_PARAMS);
+	}
+	psTmp = (PTL_STREAM)psStream;
+
+	/* If read client exists and has opened stream in blocking mode,
+	 * signal when data is available to read. */
+	if (psTmp->psNode->psRDesc &&
+		 (!(psTmp->psNode->psRDesc->ui32Flags & PVRSRV_STREAM_FLAG_ACQUIRE_NONBLOCKING)) &&
+			psTmp->ui32Read != psTmp->ui32Write)
+	{
+		TL_COUNTER_INC(psTmp->ui32ManSyncs);
+		eError = OSEventObjectSignal(psTmp->psNode->hReadEventObj);
+	}
+
+	PVR_DPF_RETURN_RC(eError);
+}
+
+IMG_BOOL
+TLStreamIsOpenForReading(IMG_HANDLE hStream)
+{
+	PTL_STREAM psTmp;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+
+	PVR_DPF_RETURN_VAL(psTmp->psNode->psRDesc != NULL);
+}
+
+IMG_BOOL
+TLStreamOutOfData(IMG_HANDLE hStream)
+{
+	PTL_STREAM psTmp;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+
+	/* If both pointers are equal then the buffer is empty */
+	PVR_DPF_RETURN_VAL(psTmp->ui32Read == psTmp->ui32Write);
+}
+
+
+PVRSRV_ERROR
+TLStreamResetProducerByteCount(IMG_HANDLE hStream, IMG_UINT32 ui32Value)
+{
+	PTL_STREAM   psTmp;
+	IMG_UINT32   ui32LRead, ui32LWrite;
+	PVRSRV_ERROR eErr = PVRSRV_OK;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(hStream);
+	psTmp = (PTL_STREAM)hStream;
+	ui32LRead = psTmp->ui32Read;
+	ui32LWrite = psTmp->ui32Write;
+
+	if (ui32LRead != ui32LWrite)
+	{
+		eErr = PVRSRV_ERROR_STREAM_MISUSE;
+	}
+#if defined(TL_BUFFER_STATS)
+	psTmp->ui32ProducerByteCount = ui32Value;
+#else
+	PVR_UNREFERENCED_PARAMETER(ui32Value);
+#endif
+	PVR_DPF_RETURN_RC(eErr);
+}
+/*
+ * Internal stream APIs to server part of Transport Layer, declared in
+ * header tlintern.h. Direct pointers to stream objects are used here as
+ * these functions are internal.
+ */
+IMG_UINT32
+TLStreamAcquireReadPos(PTL_STREAM psStream,
+                       IMG_BOOL bDisableCallback,
+                       IMG_UINT32* puiReadOffset)
+{
+	IMG_UINT32 uiReadLen = 0;
+	IMG_UINT32 ui32LRead, ui32LWrite;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+	PVR_ASSERT(puiReadOffset);
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		if (!OSTryLockAcquire(psStream->hReadLock))
+		{
+			/*
+			 * This is a normal event when the system is under load.
+			 * An example of how to produce this is to run testrunner /
+			 * regression/ddk_test_seq2_host_fw_mem.conf with HTB / pvrhtbd
+			 * configured as
+			 *
+			 * # pvrdebug -log trace -loggroups main,pow,debug \
+			 * -hostloggroups main,ctrl,sync,brg -hostlogtype dropoldest
+			 *
+			 * # pvrhtbd -hostloggroups main,ctrl,sync,brg
+			 *
+			 * We will see a small number of these collisions but as this is
+			 * an expected calling path, and an expected return code, we drop
+			 * the severity to just be a debug MESSAGE instead of WARNING
+			 */
+			PVR_DPF((PVR_DBG_MESSAGE,
+			    "%s: Read lock on stream '%s' is acquired by some writer, "
+			    "hence reader failed to acquire read lock.", __func__,
+			    psStream->szName));
+#if defined(TL_BUFFER_STATS)
+			TL_COUNTER_INC(psStream->ui32CntReadFails);
+#endif
+			PVR_DPF_RETURN_VAL(0);
+		}
+	}
+
+#if defined(TL_BUFFER_STATS)
+	TL_COUNTER_INC(psStream->ui32CntReadSuccesses);
+#endif
+
+	/* Grab a local copy */
+	ui32LRead = psStream->ui32Read;
+	ui32LWrite = psStream->ui32Write;
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		psStream->bReadPending = IMG_TRUE;
+		OSLockRelease(psStream->hReadLock);
+	}
+
+	/* No data available and CB defined - try and get data */
+	if ((ui32LRead == ui32LWrite) && psStream->pfProducerCallback && !bDisableCallback)
+	{
+		PVRSRV_ERROR eRc;
+		IMG_UINT32   ui32Resp = 0;
+
+		eRc = ((TL_STREAM_SOURCECB)psStream->pfProducerCallback)(psStream, TL_SOURCECB_OP_CLIENT_EOS,
+				&ui32Resp, psStream->pvProducerUserData);
+		PVR_LOG_IF_ERROR(eRc, "TLStream->pfProducerCallback");
+
+		ui32LWrite = psStream->ui32Write;
+	}
+
+	/* No data available... */
+	if (ui32LRead == ui32LWrite)
+	{
+		if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+		{
+			psStream->bReadPending = IMG_FALSE;
+		}
+		PVR_DPF_RETURN_VAL(0);
+	}
+
+#if defined(TL_BUFFER_STATS)
+	/* The moment reader knows it will see a non-zero data, it marks its presence in writer's eyes */
+	OSAtomicWrite (&psStream->bNoReaderSinceFirstReserve, 0);
+#endif
+
+	/* Data is available to read... */
+	*puiReadOffset = ui32LRead;
+
+	/*PVR_DPF((PVR_DBG_VERBOSE,
+	 *		"TLStreamAcquireReadPos Start before: Write:%d, Read:%d, size:%d",
+	 *		ui32LWrite, ui32LRead, psStream->ui32Size));
+	 */
+
+	if (ui32LRead > ui32LWrite)
+	{	/* CB has wrapped around. */
+		PVR_ASSERT(!psStream->bNoWrapPermanent);
+		/* Return the first contiguous piece of memory, ie [ReadLen,EndOfBuffer]
+		 * and let a subsequent AcquireReadPos read the rest of the Buffer */
+		/*PVR_DPF((PVR_DBG_VERBOSE, "TLStreamAcquireReadPos buffer has wrapped"));*/
+		uiReadLen = psStream->ui32Size - ui32LRead;
+		TL_COUNTER_INC(psStream->ui32AcquireRead2);
+	}
+	else
+	{	/* CB has not wrapped */
+		uiReadLen = ui32LWrite - ui32LRead;
+		TL_COUNTER_INC(psStream->ui32AcquireRead1);
+	}
+
+	PVR_DPF_RETURN_VAL(uiReadLen);
+}
+
+PVRSRV_ERROR
+TLStreamAdvanceReadPos(PTL_STREAM psStream,
+                       IMG_UINT32 uiReadLen,
+                       IMG_UINT32 uiOrigReadLen)
+{
+	IMG_UINT32 uiNewReadPos;
+
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	/*
+	 * This API does not use Read lock as 'bReadPending' is sufficient
+	 * to keep Read index safe by preventing a write from updating the
+	 * index and 'bReadPending' itself is safe as it can only be modified
+	 * by readers and there can be only one reader in action at a time.
+	 */
+
+	/* Update the read offset by the length provided in a circular manner.
+	 * Assuming the update to be atomic hence, avoiding use of locks
+	 */
+	uiNewReadPos = (psStream->ui32Read + uiReadLen) % psStream->ui32Size;
+
+	/* Must validate length is on a packet boundary, for
+	 * TLReleaseDataLess calls.
+	 */
+	if (uiReadLen != uiOrigReadLen) /* buffer not empty */
+	{
+		PVRSRVTL_PPACKETHDR psHdr = GET_PACKET_HDR(psStream->pbyBuffer+uiNewReadPos);
+		PVRSRVTL_PACKETTYPE eType = GET_PACKET_TYPE(psHdr);
+
+		if ((psHdr->uiReserved != PVRSRVTL_PACKETHDR_RESERVED) ||
+			(eType == PVRSRVTL_PACKETTYPE_UNDEF) ||
+			(eType >= PVRSRVTL_PACKETTYPE_LAST))
+		{
+			PVR_DPF_RETURN_RC(PVRSRV_ERROR_INVALID_ALIGNMENT);
+		}
+		/* else OK, on a packet boundary */
+	}
+	/* else no check needed */
+
+	psStream->ui32Read = uiNewReadPos;
+
+	if (psStream->eOpMode == TL_OPMODE_DROP_OLDEST)
+	{
+		psStream->bReadPending = IMG_FALSE;
+	}
+
+	/* notify reserves that may be pending */
+	/* The producer event object is used to signal the StreamReserve if the TL
+	 * Buffer is in blocking mode and is full.
+	 * Previously this event was only signalled if the buffer was created in
+	 * blocking mode. Since the buffer mode can now change dynamically the event
+	 * is signalled every time to avoid any potential race where the signal is
+	 * required, but not produced.
+	 */
+	{
+		PVRSRV_ERROR eError;
+		eError = OSEventObjectSignal(psStream->hProducerEventObj);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_WARNING,
+					 "Error in TLStreamAdvanceReadPos: OSEventObjectSignal returned:%u",
+					 eError));
+			/* We've failed to notify the producer event. This means there may
+			 * be a delay in generating more data to be consumed until the next
+			 * Write() generating action occurs.
+			 */
+		}
+	}
+
+	PVR_DPF((PVR_DBG_VERBOSE,
+			 "TLStreamAdvanceReadPos Read now at: %d",
+			psStream->ui32Read));
+	PVR_DPF_RETURN_OK;
+}
+
+void
+TLStreamResetReadPos(PTL_STREAM psStream)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	if (psStream->bNoWrapPermanent)
+	{
+
+		/* Update the read offset by the length provided in a circular manner.
+		 * Assuming the update to be atomic hence, avoiding use of locks */
+		psStream->ui32Read = 0;
+
+		PVR_DPF((PVR_DBG_VERBOSE,
+				 "TLStreamResetReadPos Read now at: %d",
+					psStream->ui32Read));
+	}
+	else
+	{
+		/* else for other stream types this is a no-op */
+		PVR_DPF((PVR_DBG_VERBOSE,
+				"No need to reset read position of circular tlstream"));
+	}
+
+	PVR_DPF_RETURN;
+}
+
+void
+TLStreamDestroy (PTL_STREAM psStream)
+{
+	PVR_ASSERT (psStream);
+
+	OSLockDestroy (psStream->hStreamWLock);
+	OSLockDestroy (psStream->hReadLock);
+
+	OSEventObjectClose(psStream->hProducerEvent);
+	OSEventObjectDestroy(psStream->hProducerEventObj);
+
+	TLFreeSharedMem(psStream);
+	OSFreeMem(psStream);
+}
+
+DEVMEM_MEMDESC*
+TLStreamGetBufferPointer(PTL_STREAM psStream)
+{
+	PVR_DPF_ENTERED;
+
+	PVR_ASSERT(psStream);
+
+	PVR_DPF_RETURN_VAL(psStream->psStreamMemDesc);
+}
+
+/*
+ * Determine the maximum transfer size which will fit into the specified
+ * L2 consumer stream.
+ * This is a point-in-time snapshot which compares the given requested transfer
+ * size and returns the minimum of that vs the available buffer space within
+ * the L2 stream. This is the largest amount of data that can be successfully
+ * copied in the stream without truncation occurring.
+ * Worst-case scenario is that we copy fewer bytes than the maximum
+ * actually available, but we will never copy too much data.
+ */
+IMG_UINT32
+TLStreamGetMaxTransfer(IMG_UINT32 uiXferSize, IMG_HANDLE hConsumerStream)
+{
+	PTL_STREAM  psConsumer = (PTL_STREAM)hConsumerStream;
+	IMG_UINT32  ui32MaxTransfer = 0U;
+
+	IMG_UINT32  ui32Xfer;
+
+	/* Local copies */
+	IMG_UINT32	ui32Read = psConsumer->ui32Read;
+	IMG_UINT32	ui32Write = psConsumer->ui32Write;
+
+	if (ui32Write >= ui32Read)
+	{
+		/* Can transfer Write .. End-of-buffer + Read bytes at start */
+		ui32Xfer = psConsumer->ui32Size - ui32Write + ui32Read;
+	}
+	else
+	{
+		/* Can transfer Read - Write bytes maximum */
+		ui32Xfer = ui32Read - ui32Write;
+	}
+
+	PVR_ASSERT(ui32Xfer <= psConsumer->ui32Size);
+
+	ui32MaxTransfer = MIN(ui32Xfer, uiXferSize);
+
+	return ui32MaxTransfer;
+}
diff --git a/drivers/gpu/drm/img-rogue/tlstream.h b/drivers/gpu/drm/img-rogue/tlstream.h
new file mode 100644
index 000000000000..aa237bba4d79
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/tlstream.h
@@ -0,0 +1,619 @@
+/*************************************************************************/ /*!
+@File
+@Title          Transport Layer kernel side API.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    TL provides driver components with a way to copy data from kernel
+                space to user space (e.g. screen/file).
+
+                Data can be passed to the Transport Layer through the
+                TL Stream (kernel space) API interface.
+
+                The buffer provided to every stream is a modified version of a
+                circular buffer. Which CB version is created is specified by
+                relevant flags when creating a stream. Currently two types
+                of buffer are available:
+                - TL_OPMODE_DROP_NEWER:
+                  When the buffer is full, incoming data are dropped
+                  (instead of overwriting older data) and a marker is set
+                  to let the user know that data have been lost.
+                - TL_OPMODE_BLOCK:
+                  When the circular buffer is full, reserve/write calls block
+                  until enough space is freed.
+                - TL_OPMODE_DROP_OLDEST:
+                  When the circular buffer is full, the oldest packets in the
+                  buffer are dropped and a flag is set in header of next packet
+                  to let the user know that data have been lost.
+
+                All size/space requests are in bytes. However, the actual
+                implementation uses native word sizes (i.e. 4 byte aligned).
+
+                The user does not need to provide space for the stream buffer
+                as the TL handles memory allocations and usage.
+
+                Inserting data to a stream's buffer can be done either:
+                - by using TLReserve/TLCommit: User is provided with a buffer
+                                                 to write data to.
+                - or by using TLWrite:         User provides a buffer with
+                                                 data to be committed. The TL
+                                                 copies the data from the
+                                                 buffer into the stream buffer
+                                                 and returns.
+                Users should be aware that there are implementation overheads
+                associated with every stream buffer. If you find that less
+                data are captured than expected then try increasing the
+                stream buffer size or use TLInfo to obtain buffer parameters
+                and calculate optimum required values at run time.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#ifndef TLSTREAM_H
+#define TLSTREAM_H
+
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_tlcommon.h"
+#include "device.h"
+
+/*! Extract TL stream opmode from the given stream create flags.
+ * Last 3 bits of streamFlag is used for storing opmode, hence
+ * opmode mask is set as following. */
+#define TL_OPMODE_MASK 0x7
+
+/*
+ * NOTE: This enum is used to directly access the HTB_OPMODE_xxx values
+ * within htbserver.c.
+ * As such we *MUST* keep the values matching in order of declaration.
+ */
+/*! Opmode specifying circular buffer behaviour */
+typedef enum
+{
+	/*! Undefined operation mode */
+	TL_OPMODE_UNDEF = 0,
+
+	/*! Reject new data if the buffer is full, producer may then decide to
+	 *    drop the data or retry after some time. */
+	TL_OPMODE_DROP_NEWER,
+
+	/*! When buffer is full, advance the tail/read position to accept the new
+	 * reserve call (size permitting), effectively overwriting the oldest
+	 * data in the circular buffer. Not supported yet. */
+	TL_OPMODE_DROP_OLDEST,
+
+	/*! Block Reserve (subsequently Write) calls if there is not enough space
+	 *    until some space is freed via a client read operation. */
+	TL_OPMODE_BLOCK,
+
+	/*!< For error checking */
+	TL_OPMODE_LAST
+
+} TL_OPMODE;
+
+typedef enum {
+	/* Enum to be used in conjunction with new Flags feature */
+
+	/* Flag set when Drop Oldest is set and packets have been dropped */
+	TL_FLAG_OVERWRITE_DETECTED = (1 << 0),
+	/* Prevents DoTLStreamReserve() from adding from injecting
+	 * PVRSRVTL_PACKETTYPE_MOST_RECENT_WRITE_FAILED */
+	TL_FLAG_NO_WRITE_FAILED = (1 << 1),
+} TL_Flags;
+
+static_assert(TL_OPMODE_LAST <= TL_OPMODE_MASK,
+	      "TL_OPMODE_LAST must not exceed TL_OPMODE_MASK");
+
+/*! Flags specifying stream behaviour */
+/*! Do not destroy stream if there still are data that have not been
+ *     copied in user space. Block until the stream is emptied. */
+#define TL_FLAG_FORCE_FLUSH            (1U<<8)
+/*! Do not signal consumers on commit automatically when the stream buffer
+ * transitions from empty to non-empty. Producer responsible for signal when
+ * it chooses. */
+#define TL_FLAG_NO_SIGNAL_ON_COMMIT    (1U<<9)
+
+/*! When a stream has this property it never wraps around and
+ * overwrites existing data, hence it is a fixed size persistent
+ * buffer, data written is permanent. Producers need to ensure
+ * the buffer is big enough for their needs.
+ * When a stream is opened for reading the client will always
+ * find the read position at the start of the buffer/data. */
+#define TL_FLAG_PERMANENT_NO_WRAP      (1U<<10)
+
+/*! Defer allocation of stream's shared memory until first open. */
+#define TL_FLAG_ALLOCATE_ON_FIRST_OPEN (1U<<11)
+
+/*! Structure used to pass internal TL stream sizes information to users.*/
+typedef struct _TL_STREAM_INFO_
+{
+    IMG_UINT32 headerSize;          /*!< Packet header size in bytes */
+    IMG_UINT32 minReservationSize;  /*!< Minimum data size reserved in bytes */
+    IMG_UINT32 pageSize;            /*!< Page size in bytes */
+    IMG_UINT32 pageAlign;           /*!< Page alignment in bytes */
+    IMG_UINT32 maxTLpacketSize;     /*! Max allowed TL packet size*/
+} TL_STREAM_INFO, *PTL_STREAM_INFO;
+
+/*! Callback operations or notifications that a stream producer may handle
+ * when requested by the Transport Layer.
+ */
+#define TL_SOURCECB_OP_CLIENT_EOS 0x01  /*!< Client has reached end of stream,
+                                         * can anymore data be supplied?
+                                         * ui32Resp ignored in this operation */
+
+/*! Function pointer type for the callback handler into the "producer" code
+ * that writes data to the TL stream.  Producer should handle the notification
+ * or operation supplied in ui32ReqOp on stream hStream. The
+ * Operations and notifications are defined above in TL_SOURCECB_OP */
+typedef PVRSRV_ERROR (*TL_STREAM_SOURCECB)(IMG_HANDLE hStream,
+		IMG_UINT32 ui32ReqOp, IMG_UINT32* ui32Resp, void* pvUser);
+
+typedef void (*TL_STREAM_ONREADEROPENCB)(void *pvArg);
+typedef void (*TL_STREAM_ONREADERCLOSECB)(void *pvArg);
+
+/*************************************************************************/ /*!
+ @Function      TLAllocSharedMemIfNull
+ @Description   Allocates shared memory for the stream.
+ @Input         hStream     Stream handle.
+ @Return        eError      Internal services call returned eError error
+                            number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLAllocSharedMemIfNull(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLFreeSharedMem
+ @Description   Frees stream's shared memory.
+ @Input         phStream    Stream handle.
+*/ /**************************************************************************/
+void
+TLFreeSharedMem(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamCreate
+ @Description   Request the creation of a new stream and open a handle.
+				If creating a stream which should continue to exist after the
+				current context is finished, then TLStreamCreate must be
+				followed by a TLStreamOpen call. On any case, the number of
+				create/open calls must balance with the number of close calls
+				used. This ensures the resources of a stream are released when
+				it is no longer required.
+ @Output        phStream        Pointer to handle to store the new stream.
+ @Input         szStreamName    Name of stream, maximum length:
+                                PRVSRVTL_MAX_STREAM_NAME_SIZE.
+                                If a longer string is provided,creation fails.
+ @Input         ui32Size        Desired buffer size in bytes.
+ @Input         ui32StreamFlags Used to configure buffer behaviour. See above.
+ @Input         pfOnReaderOpenCB    Optional callback called when a client
+                                    opens this stream, may be null.
+ @Input         pvOnReaderOpenUD    Optional user data for pfOnReaderOpenCB,
+                                    may be null.
+ @Input         pfOnReaderCloseCB   Optional callback called when a client
+                                    closes this stream, may be null.
+ @Input         pvOnReaderCloseUD   Optional user data for pfOnReaderCloseCB,
+                                    may be null.
+ @Input         pfProducerCB    Optional callback, may be null.
+ @Input         pvProducerUD    Optional user data for callback, may be null.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle or string name
+                                             exceeded MAX_STREAM_NAME_SIZE
+ @Return        PVRSRV_ERROR_OUT_OF_MEMORY   Failed to allocate space for
+                                             stream handle.
+ @Return        PVRSRV_ERROR_DUPLICATE_VALUE There already exists a stream with
+                                             the same stream name string.
+ @Return        eError                       Internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamCreate(IMG_HANDLE *phStream,
+               const IMG_CHAR *szStreamName,
+               IMG_UINT32 ui32Size,
+               IMG_UINT32 ui32StreamFlags,
+               TL_STREAM_ONREADEROPENCB pfOnReaderOpenCB,
+               void *pvOnReaderOpenUD,
+               TL_STREAM_ONREADERCLOSECB pfOnReaderCloseCB,
+               void *pvOnReaderCloseUD,
+               TL_STREAM_SOURCECB pfProducerCB,
+               void *pvProducerUD);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamOpen
+ @Description   Attach to existing stream that has already been created by a
+                  TLStreamCreate call. A handle is returned to the stream.
+ @Output        phStream        Pointer to handle to store the stream.
+ @Input         szStreamName    Name of stream, should match an already
+                                  existing stream name
+ @Return        PVRSRV_ERROR_NOT_FOUND       None of the streams matched the
+                                             requested stream name.
+				PVRSRV_ERROR_INVALID_PARAMS  Non-NULL pointer to stream
+                                             handler is required.
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamOpen(IMG_HANDLE     *phStream,
+             const IMG_CHAR *szStreamName);
+
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReset
+ @Description   Resets read and write pointers and pending flag.
+ @Output        phStream Pointer to stream's handle
+*/ /**************************************************************************/
+void TLStreamReset(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamSetNotifStream
+ @Description   Registers a "notification stream" which will be used to
+                publish information about state change of the "hStream"
+                stream. Notification can inform about events such as stream
+                open/close, etc.
+ @Input         hStream         Handle to stream to update.
+ @Input         hNotifStream    Handle to the stream which will be used for
+                                publishing notifications.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  If either of the parameters is
+                                             NULL
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamSetNotifStream(IMG_HANDLE hStream, IMG_HANDLE hNotifStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReconfigure
+ @Description   Request the stream flags controlling buffer behaviour to
+                be updated.
+                In the case where TL_OPMODE_BLOCK is to be used,
+                TLStreamCreate should be called without that flag and this
+                function used to change the stream mode once a consumer process
+                has been started. This avoids a deadlock scenario where the
+                TLStreaWrite/TLStreamReserve call will hold the Bridge Lock
+                while blocking if the TL buffer is full.
+                The TL_OPMODE_BLOCK should never drop the Bridge Lock
+                as this leads to another deadlock scenario where the caller to
+                TLStreamWrite/TLStreamReserve has already acquired another lock
+                (e.g. gHandleLock) which is not dropped. This then leads to that
+                thread acquiring locks out of order.
+ @Input         hStream         Handle to stream to update.
+ @Input         ui32StreamFlags Flags that configure buffer behaviour. See above.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle or inconsistent
+                                             stream flags.
+ @Return        PVRSRV_ERROR_NOT_READY       Stream is currently being written to
+                                             try again later.
+ @Return        eError                       Internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReconfigure(IMG_HANDLE hStream,
+                    IMG_UINT32 ui32StreamFlags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamClose
+ @Description   Detach from the stream associated with the given handle. If
+                  the current handle is the last one accessing the stream
+				  (i.e. the number of TLStreamCreate+TLStreamOpen calls matches
+				  the number of TLStreamClose calls) then the stream is also
+				  deleted.
+				On return the handle is no longer valid.
+ @Input         hStream     Handle to stream that will be closed.
+ @Return        None.
+*/ /**************************************************************************/
+void
+TLStreamClose(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserve
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Number of bytes to reserve in buffer.
+ @Return        PVRSRV_INVALID_PARAMS       NULL stream handler.
+ @Return        PVRSRV_ERROR_NOT_READY      There are data previously reserved
+                                              that are pending to be committed.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE  Misusing the stream by trying to
+                                              reserve more space than the
+                                              buffer size.
+ @Return        PVRSRV_ERROR_STREAM_FULL    The reserve size requested
+                                            is larger than the free
+                                            space.
+ @Return         PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED  The reserve size
+                                                            requested is larger
+                                                            than max TL packet size
+ @Return        PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE Permanent stream buffer
+                                                     does not have enough space
+                                                     for the reserve.
+ @Return        PVRSRV_OK                   Success, output arguments valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserve(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserve2
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Ideal number of bytes to reserve in buffer.
+ @Input         ui32SizeMin     Minimum number of bytes to reserve in buffer.
+ @Input         pui32Available  Optional, but when present and the
+                                  RESERVE_TOO_BIG error is returned, a size
+                                  suggestion is returned in this argument which
+                                  the caller can attempt to reserve again for a
+                                  successful allocation.
+ @Output        pbIsReaderConnected Let writing clients know if reader is
+                                    connected or not, in case of error.
+ @Return        PVRSRV_INVALID_PARAMS        NULL stream handler.
+ @Return        PVRSRV_ERROR_NOT_READY       There are data previously reserved
+                                             that are pending to be committed.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE   Misusing the stream by trying to
+                                             reserve more space than the
+                                             buffer size.
+ @Return        PVRSRV_ERROR_STREAM_FULL     The reserve size requested
+                                             is larger than the free
+                                             space.
+                                             Check the pui32Available
+                                             value for the correct
+                                             reserve size to use.
+ @Return         PVRSRV_ERROR_TLPACKET_SIZE_LIMIT_EXCEEDED   The reserve size
+                                                             requested is larger
+                                                             than max TL packet size
+ @Return        PVRSRV_ERROR_STREAM_NOT_ENOUGH_SPACE Permanent stream buffer
+                                                     does not have enough space
+                                                     for the reserve.
+ @Return        PVRSRV_OK                   Success, output arguments valid.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserve2(IMG_HANDLE hStream,
+                IMG_UINT8  **ppui8Data,
+                IMG_UINT32 ui32Size,
+                IMG_UINT32 ui32SizeMin,
+                IMG_UINT32* pui32Available,
+                IMG_BOOL* pbIsReaderConnected);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamReserveReturnFlags
+ @Description   Reserve space in stream buffer. When successful every
+                  TLStreamReserve call must be followed by a matching
+                  TLStreamCommit call. While a TLStreamCommit call is pending
+                  for a stream, subsequent TLStreamReserve calls for this
+                  stream will fail.
+ @Input         hStream         Stream handle.
+ @Output        ppui8Data       Pointer to a pointer to a location in the
+                                  buffer. The caller can then use this address
+                                  in writing data into the stream.
+ @Input         ui32Size        Ideal number of bytes to reserve in buffer.
+ @Output        pui32Flags      Output parameter to return flags generated within
+                                the reserve function.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamReserveReturnFlags(IMG_HANDLE hStream,
+        IMG_UINT8  **ppui8Data,
+        IMG_UINT32 ui32Size,
+		IMG_UINT32* pui32Flags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamGetUT
+ @Description   Returns the current stream utilisation in bytes
+ @Input         hStream     Stream handle.
+ @Return        IMG_UINT32  Stream utilisation
+*/ /**************************************************************************/
+IMG_UINT32 TLStreamGetUT(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamCommit
+ @Description   Notify TL that data have been written in the stream buffer.
+                  Should always follow and match TLStreamReserve call.
+ @Input         hStream         Stream handle.
+ @Input         ui32Size        Number of bytes that have been added to the
+                                  stream.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE   Commit results in more data
+                                             committed than the buffer size,
+                                             the stream is misused.
+ @Return        eError                       Commit was successful but
+                                             internal services call returned
+                                             eError error number.
+ @Return        PVRSRV_OK
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamCommit(IMG_HANDLE hStream,
+               IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamWrite
+ @Description   Combined Reserve/Commit call. This function Reserves space in
+                  the specified stream buffer, copies ui32Size bytes of data
+                  from the array pui8Src points to and Commits in an "atomic"
+                  style operation.
+ @Input         hStream         Stream handle.
+ @Input         pui8Src         Source to read data from.
+ @Input         ui32Size        Number of bytes to copy and commit.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                               Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamWrite(IMG_HANDLE hStream,
+              IMG_UINT8  *pui8Src,
+              IMG_UINT32 ui32Size);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamWriteRetFlags
+ @Description   Combined Reserve/Commit call. This function Reserves space in
+                  the specified stream buffer, copies ui32Size bytes of data
+                  from the array pui8Src points to and Commits in an "atomic"
+                  style operation. Also accepts a pointer to a bit flag value
+                  for returning write status flags.
+ @Input         hStream         Stream handle.
+ @Input         pui8Src         Source to read data from.
+ @Input         ui32Size        Number of bytes to copy and commit.
+ @Output        pui32Flags      Output parameter for write status info
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                               Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamWriteRetFlags(IMG_HANDLE hStream,
+                      IMG_UINT8 *pui8Src,
+					  IMG_UINT32 ui32Size,
+					  IMG_UINT32 *pui32Flags);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamSync
+ @Description   Signal the consumer to start acquiring data from the stream
+                buffer. Called by producers that use the flag
+                TL_FLAG_NO_SIGNAL_ON_COMMIT to manually control when
+                consumers starting reading the stream.
+                Used when multiple small writes need to be batched.
+ @Input         hStream         Stream handle.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS  NULL stream handle.
+ @Return        eError                       Error codes returned by either
+                                             Reserve or Commit.
+ @Return        PVRSRV_OK
+ */ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamSync(IMG_HANDLE hStream);
+
+
+/*************************************************************************/ /*!
+ @Function      TLStreamMarkEOS
+ @Description   Insert a EOS marker packet in the given stream.
+ @Input         hStream         Stream handle.
+ @Input         bRemoveOld      if TRUE, remove old stream record file before
+                                splitting to new file.
+ @Return        PVRSRV_ERROR_INVALID_PARAMS NULL stream handler.
+ @Return        eError                       Error codes returned by either
+                                             Reserve or Commit.
+ @Return        PVRSRV_OK                    Success.
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkEOS(IMG_HANDLE hStream, IMG_BOOL bRemoveOld);
+
+/*************************************************************************/ /*!
+@Function       TLStreamMarkStreamOpen
+@Description    Puts *open* stream packet into hStream's notification stream,
+                if set, error otherwise."
+@Input          hStream Stream handle.
+@Return         PVRSRV_OK on success and error code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkStreamOpen(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+@Function       TLStreamMarkStreamClose
+@Description    Puts *close* stream packet into hStream's notification stream,
+                if set, error otherwise."
+@Input          hStream Stream handle.
+@Return         PVRSRV_OK on success and error code on failure
+*/ /**************************************************************************/
+PVRSRV_ERROR
+TLStreamMarkStreamClose(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamInfo
+ @Description   Run time information about buffer elemental sizes.
+                It sets psInfo members accordingly. Users can use those values
+                to calculate the parameters they use in TLStreamCreate and
+                TLStreamReserve.
+ @Output        psInfo          pointer to stream info structure.
+ @Return        None.
+*/ /**************************************************************************/
+void
+TLStreamInfo(IMG_HANDLE hStream, PTL_STREAM_INFO psInfo);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamIsOpenForReading
+ @Description   Query if a stream has any readers connected.
+ @Input         hStream         Stream handle.
+ @Return        IMG_BOOL        True if at least one reader is connected,
+                                false otherwise
+*/ /**************************************************************************/
+IMG_BOOL
+TLStreamIsOpenForReading(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamOutOfData
+ @Description   Query if the stream is empty (no data waiting to be read).
+ @Input         hStream         Stream handle.
+ @Return        IMG_BOOL        True if read==write, no data waiting,
+                                false otherwise
+*/ /**************************************************************************/
+IMG_BOOL TLStreamOutOfData(IMG_HANDLE hStream);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamResetProducerByteCount
+ @Description   Reset the producer byte counter on the specified stream.
+ @Input         hStream         Stream handle.
+ @Input         ui32Value       Value to reset counter to, often 0.
+ @Return        PVRSRV_OK                   Success.
+ @Return        PVRSRV_ERROR_STREAM_MISUSE  Success but the read and write
+                                            positions did not match,
+                                            stream not empty.
+*/ /**************************************************************************/
+
+PVRSRV_ERROR
+TLStreamResetProducerByteCount(IMG_HANDLE hStream, IMG_UINT32 ui32Value);
+
+/*************************************************************************/ /*!
+ @Function      TLStreamGetMaxTransfer
+ @Description   Obtain the maximum number of bytes that can be submitted to
+                 a given stream.
+ @Input         uiXferSize         Requested transfer size from producer.
+ @Input         hConsumerStream    Stream handle for consumer.
+ @Return        IMG_UINT32         Amount of data that can be submitted to
+                                    the consumer stream.
+*/ /**************************************************************************/
+IMG_UINT32
+TLStreamGetMaxTransfer(IMG_UINT32 uiXferSize, IMG_HANDLE hConsumerStream);
+
+#endif /* TLSTREAM_H */
+/*****************************************************************************
+ End of file (tlstream.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/trace_events.c b/drivers/gpu/drm/img-rogue/trace_events.c
new file mode 100644
index 000000000000..9fdd8598f2ef
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/trace_events.c
@@ -0,0 +1,269 @@
+/*************************************************************************/ /*!
+@Title          Linux trace event helper functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include <linux/version.h>
+#include <linux/sched.h>
+
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+#if !defined(CONFIG_TRACE_GPU_MEM)
+#define CREATE_TRACE_POINTS
+#include <trace/events/gpu_mem.h>
+#undef CREATE_TRACE_POINTS
+#else /* !defined(CONFIG_TRACE_GPU_MEM) */
+#include <trace/events/gpu_mem.h>
+#endif /* !defined(CONFIG_TRACE_GPU_MEM) */
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+
+#include "img_types.h"
+#include "trace_events.h"
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+#include "rogue_trace_events.h"
+#endif
+#include "sync_checkpoint_external.h"
+
+#if defined(PVRSRV_TRACE_ROGUE_EVENTS)
+
+static bool fence_update_event_enabled, fence_check_event_enabled;
+
+bool trace_rogue_are_fence_updates_traced(void)
+{
+	return fence_update_event_enabled;
+}
+
+bool trace_rogue_are_fence_checks_traced(void)
+{
+	return fence_check_event_enabled;
+}
+
+/*
+ * Call backs referenced from rogue_trace_events.h. Note that these are not
+ * thread-safe, however, since running trace code when tracing is not enabled is
+ * simply a no-op, there is no harm in it.
+ */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_update_enabled_callback(void)
+#else
+void trace_fence_update_enabled_callback(void)
+#endif
+{
+	fence_update_event_enabled = true;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return 0;
+#endif
+}
+
+void trace_fence_update_disabled_callback(void)
+{
+	fence_update_event_enabled = false;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+int trace_fence_check_enabled_callback(void)
+#else
+void trace_fence_check_enabled_callback(void)
+#endif
+{
+	fence_check_event_enabled = true;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return 0;
+#endif
+}
+
+void trace_fence_check_disabled_callback(void)
+{
+	fence_check_event_enabled = false;
+}
+
+#if defined(SUPPORT_RGX)
+/* This is a helper that calls trace_rogue_fence_update for each fence in an
+ * array.
+ */
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values)
+{
+	IMG_UINT i;
+	for (i = 0; i < uCount; i++)
+	{
+		trace_rogue_fence_update(current->comm, cmd, dm, ui32GpuId, ui32FWContext, ui32Offset,
+								 pauiAddresses[i].ui32Addr, PVRSRV_SYNC_CHECKPOINT_SIGNALLED);
+	}
+}
+
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values)
+{
+	IMG_UINT i;
+	for (i = 0; i < uCount; i++)
+	{
+		trace_rogue_fence_check(current->comm, cmd, dm, ui32GpuId, ui32FWContext, ui32Offset,
+							  pauiAddresses[i].ui32Addr, PVRSRV_SYNC_CHECKPOINT_SIGNALLED);
+	}
+}
+
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		trace_rogue_ufo_update(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+				ui32IntJobRef,
+				ui32ExtJobRef,
+				puData->sUpdate.ui32FWAddr,
+				puData->sUpdate.ui32OldValue,
+				puData->sUpdate.ui32NewValue);
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sUpdate));
+	}
+}
+
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		if (bPrEvent)
+		{
+			trace_rogue_ufo_pr_check_success(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckSuccess.ui32FWAddr,
+					puData->sCheckSuccess.ui32Value);
+		}
+		else
+		{
+			trace_rogue_ufo_check_success(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckSuccess.ui32FWAddr,
+					puData->sCheckSuccess.ui32Value);
+		}
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sCheckSuccess));
+	}
+}
+
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+	IMG_UINT i;
+	for (i = 0; i < ui32UFOCount; i++)
+	{
+		if (bPrEvent)
+		{
+			trace_rogue_ufo_pr_check_fail(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckFail.ui32FWAddr,
+					puData->sCheckFail.ui32Value,
+					puData->sCheckFail.ui32Required);
+		}
+		else
+		{
+			trace_rogue_ufo_check_fail(ui64OSTimestamp, ui32GpuId, ui32FWCtx,
+					ui32IntJobRef, ui32ExtJobRef,
+					puData->sCheckFail.ui32FWAddr,
+					puData->sCheckFail.ui32Value,
+					puData->sCheckFail.ui32Required);
+		}
+		puData = IMG_OFFSET_ADDR(puData, sizeof(puData->sCheckFail));
+	}
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+
+int PVRGpuTraceEnableUfoCallbackWrapper(void)
+{
+	PVRGpuTraceEnableUfoCallback();
+	return 0;
+}
+
+int PVRGpuTraceEnableFirmwareActivityCallbackWrapper(void)
+{
+	PVRGpuTraceEnableFirmwareActivityCallback();
+	return 0;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) */
+#endif /* defined(SUPPORT_RGX) */
+#endif /* defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size)
+{
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+	trace_gpu_mem_total(ui8GPUId, 0, ui64Size);
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+}
+
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size)
+{
+#if defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT)
+	trace_gpu_mem_total(ui8GPUId, ui32Pid, ui64Size);
+#endif /* defined(CONFIG_TRACE_GPU_MEM) || defined(PVRSRV_ENABLE_GPU_MEM_TRACEPOINT) */
+}
diff --git a/drivers/gpu/drm/img-rogue/trace_events.h b/drivers/gpu/drm/img-rogue/trace_events.h
new file mode 100644
index 000000000000..5a4d0567688b
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/trace_events.h
@@ -0,0 +1,196 @@
+/*************************************************************************/ /*!
+@Title          Linux trace events and event helper functions
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#if !defined(TRACE_EVENTS_H)
+#define TRACE_EVENTS_H
+
+#include "rgx_fwif_km.h"
+#include "rgx_hwperf.h"
+
+/* We need to make these functions do nothing if CONFIG_EVENT_TRACING isn't
+ * enabled, just like the actual trace event functions that the kernel
+ * defines for us.
+ */
+
+#if defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS)
+bool trace_rogue_are_fence_checks_traced(void);
+
+bool trace_rogue_are_fence_updates_traced(void);
+
+#if defined(SUPPORT_RGX)
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values);
+
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values);
+
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData);
+#endif /* if defined(SUPPORT_RGX) */
+#else /* defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+static inline
+bool trace_rogue_are_fence_checks_traced(void)
+{
+	return false;
+}
+
+static inline
+bool trace_rogue_are_fence_updates_traced(void)
+{
+	return false;
+}
+
+#if defined(SUPPORT_RGX)
+static inline
+void trace_rogue_fence_updates(const char *cmd, const char *dm,
+							   IMG_UINT32 ui32GpuId,
+							   IMG_UINT32 ui32FWContext,
+							   IMG_UINT32 ui32Offset,
+							   IMG_UINT uCount,
+							   PRGXFWIF_UFO_ADDR *pauiAddresses,
+							   IMG_UINT32 *paui32Values)
+{
+}
+
+static inline
+void trace_rogue_fence_checks(const char *cmd, const char *dm,
+							  IMG_UINT32 ui32GpuId,
+							  IMG_UINT32 ui32FWContext,
+							  IMG_UINT32 ui32Offset,
+							  IMG_UINT uCount,
+							  PRGXFWIF_UFO_ADDR *pauiAddresses,
+							  IMG_UINT32 *paui32Values)
+{
+}
+
+static inline
+void trace_rogue_ufo_updates(IMG_UINT64 ui64OSTimestamp,
+							 IMG_UINT32 ui32GpuId,
+							 IMG_UINT32 ui32FWCtx,
+							 IMG_UINT32 ui32ExtJobRef,
+							 IMG_UINT32 ui32IntJobRef,
+							 IMG_UINT32 ui32UFOCount,
+							 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+
+static inline
+void trace_rogue_ufo_checks_success(IMG_UINT64 ui64OSTimestamp,
+									IMG_UINT32 ui32GpuId,
+									IMG_UINT32 ui32FWCtx,
+									IMG_UINT32 ui32ExtJobRef,
+									IMG_UINT32 ui32IntJobRef,
+									IMG_BOOL bPrEvent,
+									IMG_UINT32 ui32UFOCount,
+									const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+
+static inline
+void trace_rogue_ufo_checks_fail(IMG_UINT64 ui64OSTimestamp,
+								 IMG_UINT32 ui32GpuId,
+								 IMG_UINT32 ui32FWCtx,
+								 IMG_UINT32 ui32ExtJobRef,
+								 IMG_UINT32 ui32IntJobRef,
+								 IMG_BOOL bPrEvent,
+								 IMG_UINT32 ui32UFOCount,
+								 const RGX_HWPERF_UFO_DATA_ELEMENT *puData)
+{
+}
+#endif /* if defined(SUPPORT_RGX)*/
+#endif /* defined(CONFIG_EVENT_TRACING) && defined(PVRSRV_TRACE_ROGUE_EVENTS) */
+
+#if defined(CONFIG_EVENT_TRACING)
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size);
+
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size);
+#else
+static inline
+void TracepointUpdateGPUMemGlobal(IMG_UINT8 ui8GPUId,
+								  IMG_UINT64 ui64Size)
+{
+}
+
+static inline
+void TracepointUpdateGPUMemPerProcess(IMG_UINT8 ui8GPUId,
+									  IMG_UINT32 ui32Pid,
+									  IMG_UINT64 ui64Size)
+{
+}
+#endif /* defined(CONFIG_EVENT_TRACING) */
+
+#endif /* TRACE_EVENTS_H */
diff --git a/drivers/gpu/drm/img-rogue/uniq_key_splay_tree.c b/drivers/gpu/drm/img-rogue/uniq_key_splay_tree.c
new file mode 100644
index 000000000000..2d4f2fa8ce66
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/uniq_key_splay_tree.c
@@ -0,0 +1,280 @@
+/*************************************************************************/ /*!
+@File
+@Title          Provides splay-trees.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Implementation of splay-trees.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */ /**************************************************************************/
+
+#include "allocmem.h" /* for OSMemAlloc / OSMemFree */
+#include "osfunc.h" /* for OSMemFree */
+#include "pvr_debug.h"
+#include "uniq_key_splay_tree.h"
+
+/**
+ * PVRSRVSplay - perform a simple top down splay
+ * @ui32Flags: flags that must splayed to the root (if possible)
+ * @psTree:    psTree The tree to splay.
+ *
+ * Return the resulting tree after the splay operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVSplay (IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_SPLAY_TREE sTmp1;
+	IMG_PSPLAY_TREE psLeft;
+	IMG_PSPLAY_TREE psRight;
+	IMG_PSPLAY_TREE psTmp2;
+
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	sTmp1.psLeft = NULL;
+	sTmp1.psRight = NULL;
+
+	psLeft = &sTmp1;
+	psRight = &sTmp1;
+
+	for (;;)
+	{
+		if (uiFlags < psTree->uiFlags)
+		{
+			if (psTree->psLeft == NULL)
+			{
+				break;
+			}
+
+			if (uiFlags < psTree->psLeft->uiFlags)
+			{
+				/* if we get to this point, we need to rotate right the tree */
+				psTmp2 = psTree->psLeft;
+				psTree->psLeft = psTmp2->psRight;
+				psTmp2->psRight = psTree;
+				psTree = psTmp2;
+				if (psTree->psLeft == NULL)
+				{
+					break;
+				}
+			}
+
+			/* if we get to this point, we need to link right */
+			psRight->psLeft = psTree;
+			psRight = psTree;
+			psTree = psTree->psLeft;
+		}
+		else
+		{
+			if (uiFlags > psTree->uiFlags)
+			{
+				if (psTree->psRight == NULL)
+				{
+					break;
+				}
+
+				if (uiFlags > psTree->psRight->uiFlags)
+				{
+					/* if we get to this point, we need to rotate left the tree */
+					psTmp2 = psTree->psRight;
+					psTree->psRight = psTmp2->psLeft;
+					psTmp2->psLeft = psTree;
+					psTree = psTmp2;
+					if (psTree->psRight == NULL)
+					{
+						break;
+					}
+				}
+
+				/* if we get to this point, we need to link left */
+				psLeft->psRight = psTree;
+				psLeft = psTree;
+				psTree = psTree->psRight;
+			}
+			else
+			{
+				break;
+			}
+		}
+	}
+
+	/* at this point re-assemble the tree */
+	psLeft->psRight = psTree->psLeft;
+	psRight->psLeft = psTree->psRight;
+	psTree->psLeft = sTmp1.psRight;
+	psTree->psRight = sTmp1.psLeft;
+	return psTree;
+}
+
+
+/**
+ * PVRSRVInsert - insert a node into the Tree (unless it is already present, in
+ * which case it is equivalent to performing only a splay operation
+ * @ui32Flags: the key of the new node
+ * @psTree:    tree into which one wants to add a new node
+ *
+ * Return the resulting tree after the splay operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVInsert(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_PSPLAY_TREE psNew;
+
+	if (psTree != NULL)
+	{
+		psTree = PVRSRVSplay(uiFlags, psTree);
+		if (psTree->uiFlags == uiFlags)
+		{
+			return psTree;
+		}
+	}
+
+	psNew = (IMG_PSPLAY_TREE) OSAllocMem(sizeof(IMG_SPLAY_TREE));
+	if (psNew == NULL)
+	{
+		PVR_DPF ((PVR_DBG_ERROR, "Error: failed to allocate memory to add a node to the splay tree."));
+		return NULL;
+	}
+
+	psNew->uiFlags = uiFlags;
+	OSCachedMemSet(&(psNew->buckets[0]), 0, sizeof(psNew->buckets));
+
+#if defined(PVR_CTZLL)
+	psNew->bHasEltsMapping = ~(((IMG_ELTS_MAPPINGS) 1 << (sizeof(psNew->buckets) / (sizeof(psNew->buckets[0])))) - 1);
+#endif
+
+	if (psTree == NULL)
+	{
+		psNew->psLeft  = NULL;
+		psNew->psRight = NULL;
+		return psNew;
+	}
+
+	if (uiFlags < psTree->uiFlags)
+	{
+		psNew->psLeft  = psTree->psLeft;
+		psNew->psRight = psTree;
+		psTree->psLeft = NULL;
+	}
+	else
+	{
+		psNew->psRight  = psTree->psRight;
+		psNew->psLeft   = psTree;
+		psTree->psRight = NULL;
+	}
+
+	return psNew;
+}
+
+
+/**
+ * PVRSRVDelete - delete a node from the tree (unless it is not there, in which
+ * case it is equivalent to a splay operation)
+ * @ui32Flags: value of the node to remove
+ * @psTree:    tree into which the node must be removed
+ *
+ * Return the resulting tree.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVDelete(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	IMG_PSPLAY_TREE psTmp;
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	psTree = PVRSRVSplay(uiFlags, psTree);
+	if (uiFlags == psTree->uiFlags)
+	{
+		/* The value was present in the tree */
+		if (psTree->psLeft == NULL)
+		{
+			psTmp = psTree->psRight;
+		}
+		else
+		{
+			psTmp = PVRSRVSplay(uiFlags, psTree->psLeft);
+			psTmp->psRight = psTree->psRight;
+		}
+		OSFreeMem(psTree);
+		return psTmp;
+	}
+
+	/* The value was not present in the tree, so just return it as is
+	 * (after the splay) */
+	return psTree;
+}
+
+/**
+ * PVRSRVFindNode - pick up the appropriate node for the given flags
+ * @ui32Flags: flags that must associated with the node
+ * @psTree:    current splay tree node
+ *
+ * Return the resulting tree node after the search operation.
+ */
+IMG_INTERNAL
+IMG_PSPLAY_TREE PVRSRVFindNode(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree)
+{
+	if (psTree == NULL)
+	{
+		return NULL;
+	}
+
+	while (psTree)
+	{
+		if (uiFlags == psTree->uiFlags)
+		{
+			return psTree;
+		}
+
+		if (uiFlags < psTree->uiFlags)
+		{
+			psTree = psTree->psLeft;
+			continue;
+		}
+
+		if (uiFlags > psTree->uiFlags)
+		{
+			psTree = psTree->psRight;
+			continue;
+		}
+	}
+
+	return NULL;
+}
diff --git a/drivers/gpu/drm/img-rogue/uniq_key_splay_tree.h b/drivers/gpu/drm/img-rogue/uniq_key_splay_tree.h
new file mode 100644
index 000000000000..75ec9297e9c2
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/uniq_key_splay_tree.h
@@ -0,0 +1,90 @@
+/*************************************************************************/ /*!
+@File
+@Title          Splay trees interface
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Provides debug functionality
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef UNIQ_KEY_SPLAY_TREE_H_
+#define UNIQ_KEY_SPLAY_TREE_H_
+
+#include "img_types.h"
+#include "pvr_intrinsics.h"
+
+#if defined(PVR_CTZLL)
+  /* map the is_bucket_n_free to an int.
+   * This way, the driver can find the first non empty without loop
+   */
+  typedef IMG_UINT64 IMG_ELTS_MAPPINGS;
+#endif
+
+typedef IMG_UINT64 IMG_PSPLAY_FLAGS_T;
+
+/* head of list of free boundary tags for indexed by pvr_log2 of the
+   boundary tag size */
+
+#define FREE_TABLE_LIMIT 40
+
+struct _BT_;
+
+typedef struct img_splay_tree
+{
+	/* left child/subtree */
+    struct img_splay_tree * psLeft;
+
+	/* right child/subtree */
+    struct img_splay_tree * psRight;
+
+    /* Flags to match on this span, used as the key. */
+    IMG_PSPLAY_FLAGS_T uiFlags;
+#if defined(PVR_CTZLL)
+	/* each bit of this int is a boolean telling if the corresponding
+	   bucket is empty or not */
+    IMG_ELTS_MAPPINGS bHasEltsMapping;
+#endif
+	struct _BT_ * buckets[FREE_TABLE_LIMIT];
+} IMG_SPLAY_TREE, *IMG_PSPLAY_TREE;
+
+IMG_PSPLAY_TREE PVRSRVSplay (IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVInsert(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVDelete(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+IMG_PSPLAY_TREE PVRSRVFindNode(IMG_PSPLAY_FLAGS_T uiFlags, IMG_PSPLAY_TREE psTree);
+
+
+#endif /* !UNIQ_KEY_SPLAY_TREE_H_ */
diff --git a/drivers/gpu/drm/img-rogue/vmm_impl.h b/drivers/gpu/drm/img-rogue/vmm_impl.h
new file mode 100644
index 000000000000..928a3f6bc9ac
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vmm_impl.h
@@ -0,0 +1,206 @@
+/*************************************************************************/ /*!
+@File           vmm_impl.h
+@Title          Common VM manager API
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides common VM manager definitions that need to
+                be shared by system virtualization layer itself and modules that
+                implement the actual VM manager types.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_IMPL_H
+#define VMM_IMPL_H
+
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "pvrsrv_device.h"
+
+typedef enum _VMM_CONF_PARAM_
+{
+	VMM_CONF_PRIO_DRV0 = 0,
+	VMM_CONF_PRIO_DRV1 = 1,
+	VMM_CONF_PRIO_DRV2 = 2,
+	VMM_CONF_PRIO_DRV3 = 3,
+	VMM_CONF_PRIO_DRV4 = 4,
+	VMM_CONF_PRIO_DRV5 = 5,
+	VMM_CONF_PRIO_DRV6 = 6,
+	VMM_CONF_PRIO_DRV7 = 7,
+	VMM_CONF_HCS_DEADLINE = 8,
+	VMM_CONF_ISOLATION_GROUP_DRV0 = 9,
+	VMM_CONF_ISOLATION_GROUP_DRV1 = 10,
+	VMM_CONF_ISOLATION_GROUP_DRV2 = 11,
+	VMM_CONF_ISOLATION_GROUP_DRV3 = 12,
+	VMM_CONF_ISOLATION_GROUP_DRV4 = 13,
+	VMM_CONF_ISOLATION_GROUP_DRV5 = 14,
+	VMM_CONF_ISOLATION_GROUP_DRV6 = 15,
+	VMM_CONF_ISOLATION_GROUP_DRV7 = 16,
+	VMM_CONF_TIME_SLICE_DRV0 = 17,
+	VMM_CONF_TIME_SLICE_DRV1 = 18,
+	VMM_CONF_TIME_SLICE_DRV2 = 19,
+	VMM_CONF_TIME_SLICE_DRV3 = 20,
+	VMM_CONF_TIME_SLICE_DRV4 = 21,
+	VMM_CONF_TIME_SLICE_DRV5 = 22,
+	VMM_CONF_TIME_SLICE_DRV6 = 23,
+	VMM_CONF_TIME_SLICE_DRV7 = 24,
+	VMM_CONF_TIME_SLICE_INTERVAL = 25,
+	VMM_CONF_VZ_CONNECTION_COOLDOWN_PERIOD = 26,
+} VMM_CONF_PARAM;
+
+/*
+	Virtual machine manager (hypervisor) para-virtualization (PVZ) connection:
+		- Type is implemented by host and guest drivers
+			- Assumes synchronous function call semantics
+			- Unidirectional semantics
+				- For Host  (vmm -> host)
+				- For Guest (guest -> vmm)
+			- Parameters can be IN/OUT/INOUT
+
+		- Host pvz entries are pre-implemented by IMG
+			- For host implementation, see vmm_pvz_server.c
+			- Called by host side hypercall handler or VMM
+
+		- Guest pvz entries are supplied by 3rd-party
+			- These are specific to hypervisor (VMM) type
+			- These implement the actual hypercalls mechanism
+
+	Para-virtualization (PVZ) call runtime sequence:
+		1 - Guest driver in guest VM calls PVZ function
+		1.1 - Guest PVZ connection calls
+		1.2 - Guest VM Manager type which
+		1.2.1 - Performs any pre-processing like parameter packing, etc.
+		1.2.2 - Issues hypercall (blocking synchronous call)
+
+		2 - VM Manager (hypervisor) receives hypercall
+		2.1 - Hypercall handler:
+		2.1.1 - Performs any pre-processing
+		2.1.2 - If call terminates in VM Manager: perform action and return from hypercall
+		2.1.3 - Otherwise forward to host driver (implementation specific call)
+
+		3 - Host driver receives call from VM Manager
+		3.1 - Host VM manager type:
+		3.1.1 - Performs any pre-processing like parameter unpacking, etc.
+		3.1.2 - Acquires host driver PVZ handler and calls the appropriate entry
+		3.2 - Host PVZ connection calls corresponding host system virtualisation layer
+		3.3 - Host driver system virtualisation layer:
+		3.3.1 - Perform action requested by guest driver
+		3.3.2 - Return to host VM Manager type
+		3.4 - Host VM Manager type:
+		3.4.1 - Prepare to return from hypercall
+		3.4.2 - Perform any post-processing like result packing, etc.
+		3.4.3 - Issue return from hypercall
+
+		4 - VM Manager (hypervisor)
+		4.1 - Perform any post-processing
+		4.2 - Return control to guest driver
+
+		5 - Guest driver in guest VM
+		5.1 - Perform any post-processing like parameter unpacking, etc.
+		5.2 - Continue execution in guest VM
+ */
+typedef struct _VMM_PVZ_CONNECTION_
+{
+	struct {
+		/*
+		   This pair must be implemented if the guest is responsible
+		   for allocating the physical heap that backs its firmware
+		   allocations, this is the default configuration. The physical
+		   heap is allocated within the guest VM IPA space and this
+		   IPA Addr/Size must be translated into the host's IPA space
+		   by the VM manager before forwarding request to host.
+		   If not implemented, return PVRSRV_ERROR_NOT_IMPLEMENTED.
+		 */
+		PVRSRV_ERROR (*pfnMapDevPhysHeap)(IMG_UINT64 ui64Size,
+										  IMG_UINT64 ui64PAddr);
+
+		PVRSRV_ERROR (*pfnUnmapDevPhysHeap)(void);
+	} sClientFuncTab;
+
+	struct {
+		/*
+			Corresponding server side entries to handle guest PVZ calls
+			NOTE:
+				 - Additional PVZ function ui32DriverID parameter
+					 - Driver ID determination is responsibility of VM manager
+					 - Actual Driver ID value must be supplied by VM manager
+						- This can be done either in client/VMM/host side
+					 - Must be done before host pvz function(s) are called
+					 - Host pvz function validates incoming Driver ID values
+		 */
+		PVRSRV_ERROR (*pfnMapDevPhysHeap)(IMG_UINT32 ui32DriverID,
+										  IMG_UINT32 ui32DevID,
+										  IMG_UINT64 ui64Size,
+										  IMG_UINT64 ui64PAddr);
+
+		PVRSRV_ERROR (*pfnUnmapDevPhysHeap)(IMG_UINT32 ui32DriverID,
+											IMG_UINT32 ui32DevID);
+	} sServerFuncTab;
+
+	struct {
+		/*
+		   This is used by the VM manager to report pertinent runtime guest VM
+		   information to the host; these events may in turn be forwarded to
+		   the firmware
+		 */
+		PVRSRV_ERROR (*pfnOnVmOnline)(IMG_UINT32 ui32DriverID,
+									  IMG_UINT32 ui32DevID);
+
+		PVRSRV_ERROR (*pfnOnVmOffline)(IMG_UINT32 ui32DriverID,
+									   IMG_UINT32 ui32DevID);
+
+		PVRSRV_ERROR (*pfnVMMConfigure)(VMM_CONF_PARAM eVMMParamType,
+										IMG_UINT32 ui32ParamValue,
+										IMG_UINT32 ui32DevID);
+
+	} sVmmFuncTab;
+} VMM_PVZ_CONNECTION;
+
+/*!
+*******************************************************************************
+ @Function      VMMCreatePvzConnection() and VMMDestroyPvzConnection()
+ @Description   Both the guest and VM manager call this in order to obtain a
+                PVZ connection to the VM and host respectively; that is, guest
+                calls it to obtain connection to VM, VM calls it to obtain a
+                connection to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig);
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+#endif /* VMM_IMPL_H */
diff --git a/drivers/gpu/drm/img-rogue/vmm_pvz_client.c b/drivers/gpu/drm/img-rogue/vmm_pvz_client.c
new file mode 100644
index 000000000000..8ea86d9a0f0d
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vmm_pvz_client.c
@@ -0,0 +1,131 @@
+/*************************************************************************/ /*!
+@File			vmm_pvz_client.c
+@Title          VM manager client para-virtualization
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides VMM client para-virtualization APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+
+#include "vmm_impl.h"
+#include "vz_vmm_pvz.h"
+#include "vmm_pvz_client.h"
+
+
+static inline void
+PvzClientLockAcquire(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockAcquire(psPVRSRVData->hPvzConnectionLock);
+}
+
+static inline void
+PvzClientLockRelease(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockRelease(psPVRSRVData->hPvzConnectionLock);
+}
+
+/*
+ * ===========================================================
+ *  The following client para-virtualization (pvz) functions
+ *  are exclusively called by guests to initiate a pvz call
+ *  to the host via hypervisor (guest -> vm manager -> host)
+ * ===========================================================
+ */
+
+PVRSRV_ERROR
+PvzClientMapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	IMG_DEV_PHYADDR sDevPAddr;
+	VMM_PVZ_CONNECTION *psVmmPvz;
+	PHYS_HEAP *psFwPhysHeap = psDevConfig->psDevNode->apsPhysHeap[FIRST_PHYSHEAP_MAPPED_TO_FW_MAIN_DEVMEM];
+
+	eError = PhysHeapGetDevPAddr(psFwPhysHeap, &sDevPAddr);
+
+#if defined(PVR_PMR_TRANSLATE_UMA_ADDRESSES)
+{
+	IMG_DEV_PHYADDR sDevPAddrTranslated;
+
+	/* If required, perform a software translation between CPU and Device physical addresses. */
+	PhysHeapCpuPAddrToDevPAddr(psFwPhysHeap, 1, &sDevPAddrTranslated, (IMG_CPU_PHYADDR *)&sDevPAddr);
+	sDevPAddr.uiAddr = sDevPAddrTranslated.uiAddr;
+}
+#endif
+
+	PVR_LOG_RETURN_IF_ERROR(eError, "PhysHeapGetDevPAddr");
+	PVR_LOG_RETURN_IF_FALSE((sDevPAddr.uiAddr != 0), "PhysHeapGetDevPAddr", PVRSRV_ERROR_INVALID_PARAMS);
+
+	psVmmPvz = PvzConnectionAcquire();
+	PvzClientLockAcquire();
+
+	eError = psVmmPvz->sClientFuncTab.pfnMapDevPhysHeap(RGX_FIRMWARE_RAW_HEAP_SIZE, sDevPAddr.uiAddr);
+
+	PvzClientLockRelease();
+	PvzConnectionRelease(psVmmPvz);
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzClientUnmapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+	VMM_PVZ_CONNECTION *psVmmPvz = PvzConnectionAcquire();
+	PVR_ASSERT(psVmmPvz);
+
+	PvzClientLockAcquire();
+
+	PVR_ASSERT(psVmmPvz->sClientFuncTab.pfnUnmapDevPhysHeap);
+
+	eError = psVmmPvz->sClientFuncTab.pfnUnmapDevPhysHeap();
+
+	PvzClientLockRelease();
+	PvzConnectionRelease(psVmmPvz);
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (vmm_pvz_client.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vmm_pvz_client.h b/drivers/gpu/drm/img-rogue/vmm_pvz_client.h
new file mode 100644
index 000000000000..205dc4ae93b5
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vmm_pvz_client.h
@@ -0,0 +1,76 @@
+/*************************************************************************/ /*!
+@File           vmm_pvz_client.h
+@Title          Guest VM manager client para-virtualization routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides guest VMM client para-virtualization APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_PVZ_CLIENT_H
+#define VMM_PVZ_CLIENT_H
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "vmm_impl.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzClientMapDevPhysHeap
+ @Description   The guest front-end to initiate a pfnMapDevPhysHeap PVZ call
+                to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzClientMapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+*******************************************************************************
+ @Function      PvzClientUnmapDevPhysHeap
+ @Description   The guest front-end to initiate a pfnUnmapDevPhysHeap PVZ call
+                to the host.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzClientUnmapDevPhysHeap(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+#endif /* VMM_PVZ_CLIENT_H */
+
+/******************************************************************************
+ End of file (vmm_pvz_client.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vmm_pvz_server.c b/drivers/gpu/drm/img-rogue/vmm_pvz_server.c
new file mode 100644
index 000000000000..7e67d397a908
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vmm_pvz_server.c
@@ -0,0 +1,251 @@
+/*************************************************************************/ /*!
+@File			vmm_pvz_server.c
+@Title          VM manager server para-virtualization handlers
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides VMM server para-virtz handler APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxfwutils.h"
+
+#include "vz_vm.h"
+#include "vmm_impl.h"
+#include "vz_vmm_pvz.h"
+#include "vmm_pvz_server.h"
+
+static inline void
+PvzServerLockAcquire(void)
+{
+#if !defined(FPGA)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockAcquire(psPVRSRVData->hPvzConnectionLock);
+#endif
+}
+
+static inline void
+PvzServerLockRelease(void)
+{
+#if !defined(FPGA)
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	OSLockRelease(psPVRSRVData->hPvzConnectionLock);
+#endif
+}
+
+#define VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID) do {							\
+	if ((ui32DriverID >= RGX_NUM_DRIVERS_SUPPORTED) ||								\
+		(ui32DriverID < RGXFW_GUEST_DRIVER_ID_START))								\
+	{																				\
+		PVR_DPF((PVR_DBG_ERROR,														\
+					"%s: Invalid OSID %u. Supported Guest OSID range: %u - %u",		\
+					__func__,														\
+					ui32DriverID,													\
+					RGXFW_GUEST_DRIVER_ID_START,									\
+					RGX_NUM_DRIVERS_SUPPORTED-1));									\
+		return PVRSRV_ERROR_INVALID_PARAMS;											\
+	}																				\
+	if (PVRSRVGetDeviceInstance(ui32DevID) == NULL)									\
+	{																				\
+		PVR_DPF((PVR_DBG_ERROR,														\
+					"%s: Invalid Device ID %u.",									\
+					__func__,														\
+					ui32DevID));													\
+		return PVRSRV_ERROR_INVALID_PARAMS;											\
+	}																				\
+} while (false);
+
+/*
+ * ===========================================================
+ *  The following server para-virtualization (pvz) functions
+ *  are exclusively called by the VM manager (hypervisor) on
+ *  behalf of guests to complete guest pvz calls
+ *  (guest -> vm manager -> host)
+ * ===========================================================
+ */
+
+PVRSRV_ERROR
+PvzServerMapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						IMG_UINT32 ui32DevID,
+						IMG_UINT64 ui64Size,
+						IMG_UINT64 ui64PAddr)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && !defined(FPGA)
+		/*
+		 * Reject hypercall if called on a system configured at build time to
+		 * preallocate the Guest's firmware heaps from static carveout memory.
+		 */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Host PVZ config: Does not match with Guest PVZ config."
+		         " Host preallocates the Guest's FW physheap from static memory carveouts at startup.", __func__));
+		return PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+#else
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+
+	PvzServerLockAcquire();
+
+#if defined(SUPPORT_RGX)
+	if (IsVmOnline(ui32DriverID, ui32DevID))
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstance(ui32DevID);
+		IMG_DEV_PHYADDR sDevPAddr = {ui64PAddr};
+		IMG_UINT32 sync;
+
+		eError = RGXFwRawHeapAllocMap(psDeviceNode, ui32DriverID, sDevPAddr, ui64Size);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXFwRawHeapAllocMap", e0);
+
+		/* Invalidate MMU cache in preparation for a kick from this Guest */
+		eError = psDeviceNode->pfnMMUCacheInvalidateKick(psDeviceNode, &sync);
+		PVR_LOG_GOTO_IF_ERROR(eError, "MMUCacheInvalidateKick", e0);
+
+		/* Everything is ready for the firmware to start interacting with this OS */
+		eError = RGXFWSetFwOsState(psDeviceNode->pvDevice, ui32DriverID, RGXFWIF_OS_ONLINE);
+	}
+	else
+	{
+		PVR_DPF((PVR_DBG_ERROR, "%s: Paravirtualized request to map Guest%u's Firmware heap"
+								" rejected: Guest not allowed to run.", __func__, ui32DriverID));
+		eError = PVRSRV_ERROR_PVZ_OSID_IS_OFFLINE;
+	}
+e0:
+#endif /* defined(SUPPORT_RGX) */
+	PvzServerLockRelease();
+
+	return eError;
+#endif
+}
+
+PVRSRV_ERROR
+PvzServerUnmapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						  IMG_UINT32 ui32DevID)
+{
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS) && !defined(FPGA)
+		/*
+		 * Reject hypercall if called on a system configured at built time to
+		 * preallocate the Guest's firmware heaps from static carveout memory.
+		 */
+		PVR_DPF((PVR_DBG_ERROR,
+		         "%s: Host PVZ config: Does not match with Guest PVZ config."
+		         " Host preallocates the Guest's FW physheap from static memory carveouts at startup.", __func__));
+		return PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+#else
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+
+	PvzServerLockAcquire();
+
+#if defined(SUPPORT_RGX)
+	if (IsVmOnline(ui32DriverID, ui32DevID))
+	{
+		PVRSRV_DEVICE_NODE *psDeviceNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+		/* Order firmware to offload this OS' data and stop accepting commands from it */
+		eError = RGXFWSetFwOsState(psDeviceNode->pvDevice, ui32DriverID, RGXFWIF_OS_OFFLINE);
+		PVR_LOG_GOTO_IF_ERROR(eError, "RGXFWSetFwOsState", e0);
+
+		/* it is now safe to remove the Guest's memory mappings */
+		RGXFwRawHeapUnmapFree(psDeviceNode, ui32DriverID);
+	}
+e0:
+#endif
+
+	PvzServerLockRelease();
+
+	return eError;
+#endif
+}
+
+/*
+ * ============================================================
+ *  The following server para-virtualization (pvz) functions
+ *  are exclusively called by the VM manager (hypervisor) to
+ *  pass side band information to the host (vm manager -> host)
+ * ============================================================
+ */
+
+PVRSRV_ERROR
+PvzServerOnVmOnline(IMG_UINT32 ui32DriverID,
+					IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzOnVmOnline(ui32DriverID, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzServerOnVmOffline(IMG_UINT32 ui32DriverID,
+					 IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(ui32DriverID, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzOnVmOffline(ui32DriverID, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+PVRSRV_ERROR
+PvzServerVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+					  IMG_UINT32 ui32ParamValue,
+					  IMG_UINT32 ui32DevID)
+{
+	PVRSRV_ERROR eError;
+
+	VALIDATE_DRVID_DEVID(RGXFW_GUEST_DRIVER_ID_START, ui32DevID);
+	PvzServerLockAcquire();
+	eError = PvzVMMConfigure(eVMMParamType, ui32ParamValue, ui32DevID);
+	PvzServerLockRelease();
+
+	return eError;
+}
+
+/******************************************************************************
+ End of file (vmm_pvz_server.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vmm_pvz_server.h b/drivers/gpu/drm/img-rogue/vmm_pvz_server.h
new file mode 100644
index 000000000000..02e2683c0c19
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vmm_pvz_server.h
@@ -0,0 +1,121 @@
+/*************************************************************************/ /*!
+@File           vmm_pvz_server.h
+@Title          VM manager para-virtualization interface helper routines
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Header provides API(s) available to VM manager, this must be
+                called to close the loop during guest para-virtualization calls.
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VMM_PVZ_SERVER_H
+#define VMM_PVZ_SERVER_H
+
+#include "vmm_impl.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzServerMapDevPhysHeap
+ @Description   The VM manager calls this in response to guest PVZ interface
+                call pfnMapDevPhysHeap.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerMapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						IMG_UINT32 ui32DevID,
+						IMG_UINT64 ui64Size,
+						IMG_UINT64 ui64PAddr);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerUnmapDevPhysHeap
+ @Description   The VM manager calls this in response to guest PVZ interface
+                call pfnUnmapDevPhysHeap.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerUnmapDevPhysHeap(IMG_UINT32 ui32DriverID,
+						  IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerOnVmOnline
+ @Description   The VM manager calls this when guest VM machine comes online.
+                The host driver will initialize the FW if it has not done so
+                already.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerOnVmOnline(IMG_UINT32 ui32DriverID,
+					IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerOnVmOffline
+ @Description   The VM manager calls this when a guest VM machine is about to
+                go offline. The VM manager might have unmapped the GPU kick
+                register for such VM but not the GPU memory until the call
+                returns. Once the function returns, the FW does not hold any
+                reference for such VM and no workloads from it are running in
+                the GPU and it is safe to remove the memory for such VM.
+ @Return        PVRSRV_OK on success. PVRSRV_ERROR_TIMEOUT if for some reason
+                the FW is taking too long to clean-up the resources of the
+                DriverID. Otherwise, a PVRSRV_ERROR code.
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerOnVmOffline(IMG_UINT32 ui32DriverID,
+					 IMG_UINT32 ui32DevID);
+
+/*!
+*******************************************************************************
+ @Function      PvzServerVMMConfigure
+ @Description   The VM manager calls this to configure several parameters like
+                HCS or isolation.
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR
+PvzServerVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+					  IMG_UINT32 ui32ParamValue,
+					  IMG_UINT32 ui32DevID);
+
+#endif /* VMM_PVZ_SERVER_H */
+
+/******************************************************************************
+ End of file (vmm_pvz_server.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vmm_type_stub.c b/drivers/gpu/drm/img-rogue/vmm_type_stub.c
new file mode 100644
index 000000000000..7733ec65ca89
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vmm_type_stub.c
@@ -0,0 +1,116 @@
+/*************************************************************************/ /*!
+@File			vmm_type_stub.c
+@Title          Stub VM manager type
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Sample stub (no-operation) VM manager implementation
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "pvrsrv.h"
+#include "img_types.h"
+#include "img_defs.h"
+#include "pvrsrv_error.h"
+#include "rgxheapconfig.h"
+
+#include "vmm_impl.h"
+#include "vmm_pvz_server.h"
+
+static PVRSRV_ERROR
+StubVMMMapDevPhysHeap(IMG_UINT64 ui64Size,
+					  IMG_UINT64 ui64Addr)
+{
+	PVR_UNREFERENCED_PARAMETER(ui64Size);
+	PVR_UNREFERENCED_PARAMETER(ui64Addr);
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static PVRSRV_ERROR
+StubVMMUnmapDevPhysHeap(void)
+{
+	return PVRSRV_ERROR_NOT_IMPLEMENTED;
+}
+
+static VMM_PVZ_CONNECTION gsStubVmmPvz =
+{
+	.sClientFuncTab = {
+		/* pfnMapDevPhysHeap */
+		.pfnMapDevPhysHeap = &StubVMMMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		.pfnUnmapDevPhysHeap = &StubVMMUnmapDevPhysHeap
+	},
+
+	.sServerFuncTab = {
+		/* pfnMapDevPhysHeap */
+		.pfnMapDevPhysHeap = &PvzServerMapDevPhysHeap,
+
+		/* pfnUnmapDevPhysHeap */
+		.pfnUnmapDevPhysHeap = &PvzServerUnmapDevPhysHeap
+	},
+
+	.sVmmFuncTab = {
+		/* pfnOnVmOnline */
+		.pfnOnVmOnline = &PvzServerOnVmOnline,
+
+		/* pfnOnVmOffline */
+		.pfnOnVmOffline = &PvzServerOnVmOffline,
+
+		/* pfnVMMConfigure */
+		.pfnVMMConfigure = &PvzServerVMMConfigure
+	}
+};
+
+PVRSRV_ERROR VMMCreatePvzConnection(VMM_PVZ_CONNECTION **psPvzConnection,
+									PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_RETURN_IF_FALSE((NULL != psPvzConnection), "VMMCreatePvzConnection", PVRSRV_ERROR_INVALID_PARAMS);
+	*psPvzConnection = &gsStubVmmPvz;
+	PVR_DPF((PVR_DBG_ERROR, "Using a stub VM manager type, no runtime VZ support"));
+	return PVRSRV_OK;
+}
+
+void VMMDestroyPvzConnection(VMM_PVZ_CONNECTION *psPvzConnection,
+							 PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_LOG_IF_FALSE((NULL != psPvzConnection), "VMMDestroyPvzConnection");
+}
+
+/******************************************************************************
+ End of file (vmm_type_stub.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vz_vm.h b/drivers/gpu/drm/img-rogue/vz_vm.h
new file mode 100644
index 000000000000..ee4a7fb5b3a7
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vz_vm.h
@@ -0,0 +1,63 @@
+/*************************************************************************/ /*!
+@File			vz_vm.h
+@Title          System virtualization VM support APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides VM management support APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VZ_VM_H
+#define VZ_VM_H
+
+#include "vmm_impl.h"
+
+bool IsVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzOnVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzOnVmOffline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID);
+
+PVRSRV_ERROR PvzVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+							 IMG_UINT32 ui32ParamValue,
+							 IMG_UINT32 ui32DevID);
+
+#endif /* VZ_VM_H */
+
+/*****************************************************************************
+ End of file (vz_vm.h)
+*****************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vz_vmm_pvz.c b/drivers/gpu/drm/img-rogue/vz_vmm_pvz.c
new file mode 100644
index 000000000000..bb7c8f6426c6
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vz_vmm_pvz.c
@@ -0,0 +1,201 @@
+/*************************************************************************/ /*!
+@File           vz_vmm_pvz.c
+@Title          VM manager para-virtualization APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    VM manager para-virtualization management
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#include "pvrsrv.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv_error.h"
+#include "allocmem.h"
+#include "pvrsrv.h"
+#include "vz_vmm_pvz.h"
+
+#if (RGX_NUM_DRIVERS_SUPPORTED > 1)
+static PVRSRV_ERROR
+PvzConnectionValidate(void)
+{
+	VMM_PVZ_CONNECTION *psVmmPvz;
+	PVRSRV_ERROR eError = PVRSRV_OK;
+
+	/*
+	 * Acquire the underlying VM manager PVZ connection & validate it.
+	 */
+	psVmmPvz = PvzConnectionAcquire();
+	if (psVmmPvz == NULL)
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				"%s: Unable to acquire PVZ connection", __func__));
+		eError = PVRSRV_ERROR_INVALID_PVZ_CONFIG;
+		goto e0;
+	}
+
+	/* Log which PVZ setup type is being used by driver */
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/*
+	 *  Static PVZ bootstrap setup
+	 *
+	 *  This setup uses carve-out memory, has no hypercall mechanism & does not support
+	 *  out-of-order initialisation of host/guest VMs/drivers. The host driver has all
+	 *  the information needed to initialize all Drivers firmware state when it's loaded
+	 *  and its PVZ layer must mark all guest Drivers as being online as part of its PVZ
+	 *  initialisation. Having no out-of-order initialisation support, the guest driver
+	 *  can only submit a workload to the device after the host driver has completely
+	 *  initialized the firmware, the VZ hypervisor/VM setup must guarantee this.
+	 */
+	PVR_LOG(("Using static PVZ bootstrap setup"));
+#else
+	/*
+	 *  Dynamic PVZ bootstrap setup
+	 *
+	 *  This setup uses guest memory, has PVZ hypercall mechanism & supports out-of-order
+	 *  initialisation of host/guest VMs/drivers. The host driver initializes only its
+	 *  own Driver-0 firmware state when its loaded and each guest driver will use its PVZ
+	 *  interface to hypercall to the host driver to both synchronise its initialisation
+	 *  so it does not submit any workload to the firmware before the host driver has
+	 *  had a chance to initialize the firmware and to also initialize its own Driver-x
+	 *  firmware state.
+	 */
+	PVR_LOG(("Using dynamic PVZ bootstrap setup"));
+#endif
+
+	PvzConnectionRelease(psVmmPvz);
+e0:
+	return eError;
+}
+#endif /* (RGX_NUM_DRIVERS_SUPPORTED > 1) */
+
+PVRSRV_ERROR PvzConnectionInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_ERROR eError;
+
+#if (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVR_UNREFERENCED_PARAMETER(psDevConfig);
+	PVR_DPF((PVR_DBG_ERROR, "This kernel driver does not support virtualization. Please rebuild with RGX_NUM_DRIVERS_SUPPORTED > 1"));
+	eError = PVRSRV_ERROR_NOT_SUPPORTED;
+	goto e0;
+#else
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+
+	if ((psPVRSRVData->hPvzConnection != NULL) &&
+		(psPVRSRVData->hPvzConnectionLock != NULL))
+	{
+		eError = PVRSRV_OK;
+		PVR_DPF((PVR_DBG_MESSAGE, "PVzConnection already initialised."));
+		goto e0;
+	}
+
+	/* Create para-virtualization connection lock */
+	eError = OSLockCreate(&psPVRSRVData->hPvzConnectionLock);
+	PVR_LOG_GOTO_IF_ERROR(eError, "OSLockCreate", e0);
+
+	/* Create VM manager para-virtualization connection */
+	eError = VMMCreatePvzConnection((VMM_PVZ_CONNECTION **)&psPVRSRVData->hPvzConnection, psDevConfig);
+	if (eError != PVRSRV_OK)
+	{
+		OSLockDestroy(psPVRSRVData->hPvzConnectionLock);
+		psPVRSRVData->hPvzConnectionLock = NULL;
+
+		PVR_LOG_ERROR(eError, "VMMCreatePvzConnection");
+		goto e0;
+	}
+
+	/* Ensure pvz connection is configured correctly */
+	eError = PvzConnectionValidate();
+	PVR_LOG_RETURN_IF_ERROR(eError, "PvzConnectionValidate");
+#endif
+e0:
+	return eError;
+}
+
+void PvzConnectionDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVRSRV_DEVICE_NODE *psDN;
+	IMG_BOOL bCanDestroyPvzData = IMG_TRUE;
+
+	OSWRLockAcquireRead(psPVRSRVData->hDeviceNodeListLock);
+	for (psDN = psPVRSRVData->psDeviceNodeList; psDN != NULL; psDN = psDN->psNext)
+	{
+		if ((psDN->psDevConfig != psDevConfig) &&
+			(!PVRSRV_VZ_MODE_IS(NATIVE, DEVNODE, psDN)))
+		{
+			/* if any other virtual devices are present keep the pvz data */
+			bCanDestroyPvzData = IMG_FALSE;
+			break;
+		}
+	}
+	OSWRLockReleaseRead(psPVRSRVData->hDeviceNodeListLock);
+
+	if (bCanDestroyPvzData)
+	{
+		if ((psPVRSRVData->hPvzConnection == NULL) &&
+			(psPVRSRVData->hPvzConnectionLock == NULL))
+		{
+			PVR_DPF((PVR_DBG_MESSAGE, "PVzConnection already deinitialised."));
+			return;
+		}
+
+		VMMDestroyPvzConnection(psPVRSRVData->hPvzConnection, psDevConfig);
+		psPVRSRVData->hPvzConnection = NULL;
+
+		OSLockDestroy(psPVRSRVData->hPvzConnectionLock);
+		psPVRSRVData->hPvzConnectionLock = NULL;
+	}
+}
+
+VMM_PVZ_CONNECTION* PvzConnectionAcquire(void)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	PVR_ASSERT(psPVRSRVData->hPvzConnection != NULL);
+	return psPVRSRVData->hPvzConnection;
+}
+
+void PvzConnectionRelease(VMM_PVZ_CONNECTION *psParaVz)
+{
+	PVRSRV_DATA *psPVRSRVData = PVRSRVGetPVRSRVData();
+	/* Nothing to do, just validate the pointer we're passed back */
+	PVR_ASSERT(psParaVz == psPVRSRVData->hPvzConnection);
+}
+
+/******************************************************************************
+ End of file (vz_vmm_pvz.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vz_vmm_pvz.h b/drivers/gpu/drm/img-rogue/vz_vmm_pvz.h
new file mode 100644
index 000000000000..03d5525936b8
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vz_vmm_pvz.h
@@ -0,0 +1,79 @@
+/*************************************************************************/ /*!
+@File           vz_vmm_pvz.h
+@Title          System virtualization VM manager management APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    This header provides VM manager para-virtz management APIs
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef VZ_VMM_PVZ_H
+#define VZ_VMM_PVZ_H
+
+#include "img_types.h"
+#include "vmm_impl.h"
+
+/*!
+*******************************************************************************
+ @Function      PvzConnectionInit() and PvzConnectionDeInit()
+ @Description   PvzConnectionInit initializes the VM manager para-virt
+                which is used subsequently for communication between guest and
+                host; depending on the underlying VM setup, this could either
+                be a hyper-call or cross-VM call
+ @Return        PVRSRV_OK on success. Otherwise, a PVRSRV error code
+******************************************************************************/
+PVRSRV_ERROR PvzConnectionInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+void PvzConnectionDeInit(PVRSRV_DEVICE_CONFIG *psDevConfig);
+
+/*!
+*******************************************************************************
+ @Function      PvzConnectionAcquire() and PvzConnectionRelease()
+ @Description   These are to acquire/release a handle to the VM manager
+                para-virtz connection to make a pvz call; on the client, use it
+                to make the actual pvz call and on the server handler /
+                VM manager, use it to complete the processing for the pvz call
+                or make a VM manager to host pvzbridge call
+@Return         VMM_PVZ_CONNECTION* on success. Otherwise NULL
+******************************************************************************/
+VMM_PVZ_CONNECTION* PvzConnectionAcquire(void);
+void PvzConnectionRelease(VMM_PVZ_CONNECTION *psPvzConnection);
+
+#endif /* VZ_VMM_PVZ_H */
+
+/******************************************************************************
+ End of file (vz_vmm_pvz.h)
+******************************************************************************/
diff --git a/drivers/gpu/drm/img-rogue/vz_vmm_vm.c b/drivers/gpu/drm/img-rogue/vz_vmm_vm.c
new file mode 100644
index 000000000000..bc737f5acd81
--- /dev/null
+++ b/drivers/gpu/drm/img-rogue/vz_vmm_vm.c
@@ -0,0 +1,294 @@
+/*************************************************************************/ /*!
+@File			vz_vmm_vm.c
+@Title          System virtualization VM support APIs
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    System virtualization VM support functions
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+#include "osfunc.h"
+#include "pvrsrv.h"
+#include "img_defs.h"
+#include "img_types.h"
+#include "pvrsrv.h"
+#include "pvrsrv_error.h"
+#include "vz_vm.h"
+#include "rgxfwutils.h"
+#include "rgxfwdbg.h"
+
+bool IsVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+	PVRSRV_DEVICE_NODE *psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		return false;
+	}
+	else
+	{
+		return BIT_ISSET(psDevNode->ui32VmState, ui32DriverID);
+	}
+}
+
+PVRSRV_ERROR PvzOnVmOnline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVRSRV_ERROR       eError          = PVRSRV_ERROR_INVALID_PARAMS;
+#else
+	PVRSRV_ERROR       eError          = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psDevNode;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+	else if (BIT_ISSET(psDevNode->ui32VmState, ui32DriverID))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: DriverID %u on Device %u is already enabled.",
+				 __func__, ui32DriverID, ui32DevID));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	if (psDevNode->eDevState == PVRSRV_DEVICE_STATE_CREATED)
+	{
+		/* Firmware not initialized yet, do it here */
+		eError = PVRSRVCommonDeviceInitialise(psDevNode);
+		if (eError != PVRSRV_OK)
+		{
+			PVR_DPF((PVR_DBG_ERROR,
+					 "%s: failed to initialize firmware (%s)",
+					 __func__, PVRSRVGetErrorString(eError)));
+			goto e0;
+		}
+	}
+
+	eError = RGXFWHealthCheckCmd(psDevNode->pvDevice);
+	if (eError != PVRSRV_OK)
+	{
+		goto e0;
+	}
+
+	BIT_SET(psDevNode->ui32VmState, ui32DriverID);
+
+#if defined(RGX_VZ_STATIC_CARVEOUT_FW_HEAPS)
+	/* Everything is ready for the firmware to start interacting with this OS */
+	eError = RGXFWSetFwOsState(psDevNode->pvDevice, ui32DriverID, RGXFWIF_OS_ONLINE);
+#endif
+
+e0:
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PvzOnVmOffline(IMG_UINT32 ui32DriverID, IMG_UINT32 ui32DevID)
+{
+#if !defined(RGX_NUM_DRIVERS_SUPPORTED) || (RGX_NUM_DRIVERS_SUPPORTED == 1)
+	PVRSRV_ERROR       eError          = PVRSRV_ERROR_INVALID_PARAMS;
+#else
+	PVRSRV_ERROR      eError          = PVRSRV_OK;
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+	else if (!BIT_ISSET(psDevNode->ui32VmState, ui32DriverID))
+	{
+		PVR_DPF((PVR_DBG_ERROR,
+				 "%s: DriverID %u on Device %u is already disabled.",
+				 __func__, ui32DriverID, ui32DevID));
+		eError = PVRSRV_ERROR_INVALID_PARAMS;
+		goto e0;
+	}
+
+	psDevInfo = psDevNode->pvDevice;
+	if (psDevInfo == NULL)
+	{
+		eError = PVRSRV_ERROR_INVALID_DEVINFO;
+		goto e0;
+	}
+
+	eError = RGXFWSetFwOsState(psDevInfo, ui32DriverID, RGXFWIF_OS_OFFLINE);
+	if (eError == PVRSRV_OK)
+	{
+		BIT_UNSET(psDevNode->ui32VmState, ui32DriverID);
+	}
+
+e0:
+#endif
+	return eError;
+}
+
+PVRSRV_ERROR PvzVMMConfigure(VMM_CONF_PARAM eVMMParamType,
+							 IMG_UINT32 ui32ParamValue,
+							 IMG_UINT32 ui32DevID)
+{
+#if defined(SUPPORT_RGX)
+	PVRSRV_DEVICE_NODE *psDevNode;
+	PVRSRV_RGXDEV_INFO *psDevInfo;
+	PVRSRV_ERROR eError;
+
+	psDevNode = PVRSRVGetDeviceInstance(ui32DevID);
+	if (psDevNode == NULL)
+	{
+		eError = PVRSRV_ERROR_NO_DEVICENODE_FOUND;
+		goto e0;
+	}
+
+	psDevInfo = psDevNode->pvDevice;
+	if (psDevInfo == NULL)
+	{
+		eError = PVRSRV_ERROR_INVALID_DEVINFO;
+		goto e0;
+	}
+
+	switch (eVMMParamType)
+	{
+		case VMM_CONF_PRIO_DRV0:
+		case VMM_CONF_PRIO_DRV1:
+		case VMM_CONF_PRIO_DRV2:
+		case VMM_CONF_PRIO_DRV3:
+		case VMM_CONF_PRIO_DRV4:
+		case VMM_CONF_PRIO_DRV5:
+		case VMM_CONF_PRIO_DRV6:
+		case VMM_CONF_PRIO_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType;
+			IMG_UINT32 ui32Prio = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverPriorityKM(NULL, psDevNode, ui32DriverID, ui32Prio);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_HCS_DEADLINE:
+		{
+			IMG_UINT32 ui32HCSDeadline = ui32ParamValue;
+			eError = RGXFWSetHCSDeadline(psDevInfo, ui32HCSDeadline);
+			break;
+		}
+		case VMM_CONF_ISOLATION_GROUP_DRV0:
+		case VMM_CONF_ISOLATION_GROUP_DRV1:
+		case VMM_CONF_ISOLATION_GROUP_DRV2:
+		case VMM_CONF_ISOLATION_GROUP_DRV3:
+		case VMM_CONF_ISOLATION_GROUP_DRV4:
+		case VMM_CONF_ISOLATION_GROUP_DRV5:
+		case VMM_CONF_ISOLATION_GROUP_DRV6:
+		case VMM_CONF_ISOLATION_GROUP_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType - VMM_CONF_ISOLATION_GROUP_DRV0;
+			IMG_UINT32 ui32IsolationGroup = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverIsolationGroupKM(NULL, psDevNode, ui32DriverID, ui32IsolationGroup);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_TIME_SLICE_DRV0:
+		case VMM_CONF_TIME_SLICE_DRV1:
+		case VMM_CONF_TIME_SLICE_DRV2:
+		case VMM_CONF_TIME_SLICE_DRV3:
+		case VMM_CONF_TIME_SLICE_DRV4:
+		case VMM_CONF_TIME_SLICE_DRV5:
+		case VMM_CONF_TIME_SLICE_DRV6:
+		case VMM_CONF_TIME_SLICE_DRV7:
+		{
+			IMG_UINT32 ui32DriverID = eVMMParamType - VMM_CONF_TIME_SLICE_DRV0;
+			IMG_UINT32 ui32TSPercentage = ui32ParamValue;
+
+			if (ui32DriverID < RGX_NUM_DRIVERS_SUPPORTED)
+			{
+				eError = PVRSRVRGXFWDebugSetDriverTimeSliceKM(NULL, psDevNode, ui32DriverID, ui32TSPercentage);
+			}
+			else
+			{
+				eError = PVRSRV_ERROR_INVALID_PARAMS;
+			}
+			break;
+		}
+		case VMM_CONF_TIME_SLICE_INTERVAL:
+		{
+			IMG_UINT32 ui32TSIntervalMs = ui32ParamValue;
+
+			eError = PVRSRVRGXFWDebugSetDriverTimeSliceIntervalKM(NULL, psDevNode, ui32TSIntervalMs);
+			break;
+		}
+		case VMM_CONF_VZ_CONNECTION_COOLDOWN_PERIOD:
+		{
+			IMG_UINT32 ui32VzConnectionCooldownPeriodInSec = ui32ParamValue;
+			eError = RGXFWSetVzConnectionCooldownPeriod(psDevInfo, ui32VzConnectionCooldownPeriodInSec);
+			break;
+		}
+		default:
+		{
+			eError = PVRSRV_ERROR_INVALID_PARAMS;
+		}
+	}
+
+e0:
+	return eError;
+#else
+	PVR_UNREFERENCED_PARAMETER(eVMMParamType);
+	PVR_UNREFERENCED_PARAMETER(ui32ParamValue);
+	PVR_UNREFERENCED_PARAMETER(ui32DevID);
+
+	return PVRSRV_ERROR_INVALID_PARAMS;
+#endif
+}
+
+/******************************************************************************
+ End of file (vz_vmm_vm.c)
+******************************************************************************/
diff --git a/drivers/gpu/drm/nulldisp/Kconfig b/drivers/gpu/drm/nulldisp/Kconfig
new file mode 100644
index 000000000000..b912475b3c22
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/Kconfig
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+config POWERVR_ROGUE_NULLDRMDISP
+	tristate "null drm disp"
+	default n
+	select DRM_KMS_HELPER
+	depends on POWERVR_ROGUE
+        help
+          This enables powervr null drm display driver
diff --git a/drivers/gpu/drm/nulldisp/Makefile b/drivers/gpu/drm/nulldisp/Makefile
new file mode 100644
index 000000000000..0c9a9c8e226c
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+ccflags-y += -I$(srctree)/drivers/gpu/drm/img-rogue/
+
+obj-y += drm_nulldisp.o
+drm_nulldisp-y := drm_nulldisp_drv.o drm_nulldisp_netlink.o drm_netlink_gem.o drm_nulldisp_gem.o
diff --git a/drivers/gpu/drm/nulldisp/drm_netlink_gem.c b/drivers/gpu/drm/nulldisp/drm_netlink_gem.c
new file mode 100644
index 000000000000..e67f5f6abe4e
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_netlink_gem.c
@@ -0,0 +1,106 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#endif
+
+#include "drm_netlink_gem.h"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <drm/drm_vma_manager.h>
+#endif
+
+#include <linux/capability.h>
+
+#include "kernel_compatibility.h"
+
+static int netlink_gem_mmap_capsys(struct file *file,
+				   struct vm_area_struct *vma)
+{
+	struct drm_file *file_priv = file->private_data;
+	struct drm_device *dev = file_priv->minor->dev;
+	struct drm_vma_offset_node *node;
+	struct drm_gem_object *obj = NULL;
+	int err;
+
+	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
+	node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
+						  vma->vm_pgoff,
+						  vma_pages(vma));
+	if (node) {
+		obj = container_of(node, struct drm_gem_object, vma_node);
+
+		/* Don't mmap an object that is being destroyed */
+		if (!kref_get_unless_zero(&obj->refcount))
+			obj = NULL;
+	}
+	drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
+
+	if (!obj)
+		return -EINVAL;
+
+	err = drm_vma_node_allow(node, file_priv);
+	if (!err) {
+		err = drm_gem_mmap(file, vma);
+
+		drm_vma_node_revoke(node, file_priv);
+	}
+
+	drm_gem_object_put(obj);
+
+	return err;
+}
+
+int netlink_gem_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	int err;
+
+	err = drm_gem_mmap(file, vma);
+	if (!!err && capable(CAP_SYS_RAWIO))
+		err = netlink_gem_mmap_capsys(file, vma);
+
+	return err;
+}
diff --git a/drivers/gpu/drm/nulldisp/drm_netlink_gem.h b/drivers/gpu/drm/nulldisp/drm_netlink_gem.h
new file mode 100644
index 000000000000..d70e20676b3c
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_netlink_gem.h
@@ -0,0 +1,59 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__DRM_NETLINK_GEM_H__)
+#define __DRM_NETLINK_GEM_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+struct file;
+struct vm_area_struct;
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_gem.h>
+
+int netlink_gem_mmap(struct file *file, struct vm_area_struct *vma);
+
+#endif	/* !defined(__DRM_NETLINK_GEM_H__) */
diff --git a/drivers/gpu/drm/nulldisp/drm_nulldisp_drv.c b/drivers/gpu/drm/nulldisp/drm_nulldisp_drv.c
new file mode 100644
index 000000000000..462de20d5451
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_nulldisp_drv.c
@@ -0,0 +1,2645 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#include <linux/atomic.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/jiffies.h>
+#include "pvr_linux_fence.h"
+#include <linux/workqueue.h>
+#include <linux/dma-mapping.h>
+#include <linux/mutex.h>
+#include <linux/capability.h>
+#include <linux/completion.h>
+#include <linux/dma-buf.h>
+#if defined(CONFIG_OF)
+#include <linux/of_device.h>
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_vblank.h>
+#include <linux/platform_device.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_plane_helper.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+#include <drm/drm_atomic_helper.h>
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0))
+#include <drm/drm_probe_helper.h>
+#endif
+
+#include "pvr_dma_resv.h"
+
+#include "img_drm_fourcc_internal.h"
+#include <pvrversion.h>
+
+#include <drm/drm_fourcc.h>
+
+#include "drm_nulldisp_drv.h"
+#if defined(LMA)
+#if !defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+#include "tc_drv.h"
+#endif
+#include "drm_pdp_gem.h"
+#include "pdp_drm.h"
+#else
+#include "drm_nulldisp_gem.h"
+#endif
+#include "nulldisp_drm.h"
+#include "drm_netlink_gem.h"
+#include "drm_nulldisp_netlink.h"
+
+#if defined(NULLDISP_USE_ATOMIC)
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0))
+#include <drm/drm_gem_atomic_helper.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0) */
+#endif
+
+#include "kernel_compatibility.h"
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+MODULE_IMPORT_NS("DMA_BUF");
+#else
+MODULE_IMPORT_NS(DMA_BUF);
+#endif
+
+#define DRIVER_NAME "nulldisp"
+#define DRIVER_DESC "Imagination Technologies Null DRM Display Driver"
+#define DRIVER_DATE "20150612"
+
+#if defined(NULLDISP_USE_ATOMIC)
+#define NULLDISP_DRIVER_ATOMIC DRIVER_ATOMIC
+#else
+#define NULLDISP_DRIVER_ATOMIC 0
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#define	NULLDISP_DRIVER_PRIME 0
+#else
+#define	NULLDISP_DRIVER_PRIME DRIVER_PRIME
+#endif
+
+#define NULLDISP_FB_WIDTH_MIN 0
+#define NULLDISP_FB_WIDTH_MAX 8192
+#define NULLDISP_FB_HEIGHT_MIN 0
+#define NULLDISP_FB_HEIGHT_MAX 8192
+
+#if defined(ANDROID)
+#define NULLDISP_DEFAULT_WIDTH PVR_ANDROID_VNC_WIDTH
+#define NULLDISP_DEFAULT_HEIGHT PVR_ANDROID_VNC_HEIGHT
+#else
+#define NULLDISP_DEFAULT_WIDTH 640
+#define NULLDISP_DEFAULT_HEIGHT 480
+#endif
+#define NULLDISP_DEFAULT_REFRESH_RATE 60
+
+#define NULLDISP_MAX_PLANES 3
+
+#if defined(NULLDISP_USE_ATOMIC)
+#define NULLDISP_NETLINK_TIMEOUT 5
+#else
+#define NULLDISP_NETLINK_TIMEOUT 30
+#endif
+#define NULLDISP_NETLINK_TIMEOUT_MAX 300
+#define NULLDISP_NETLINK_TIMEOUT_MIN 1
+
+enum nulldisp_crtc_flip_status {
+	NULLDISP_CRTC_FLIP_STATUS_NONE = 0,
+#if !defined(NULLDISP_USE_ATOMIC)
+	NULLDISP_CRTC_FLIP_STATUS_PENDING,
+#endif
+	NULLDISP_CRTC_FLIP_STATUS_DONE,
+};
+
+struct nulldisp_flip_data {
+	struct dma_fence_cb base;
+	struct drm_crtc *crtc;
+	struct dma_fence *wait_fence;
+};
+
+struct nulldisp_crtc {
+	struct drm_crtc base;
+	struct delayed_work vb_work;
+#if defined(NULLDISP_USE_ATOMIC)
+	struct drm_framebuffer *fb;
+	struct completion flip_done;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0))
+	struct completion copy_done;
+#endif
+#else
+	struct work_struct flip_work;
+	struct delayed_work flip_to_work;
+	struct delayed_work copy_to_work;
+
+	struct completion flip_scheduled;
+	struct completion copy_done;
+#endif
+
+	/* Reuse the drm_device event_lock to protect these */
+	atomic_t flip_status;
+	struct drm_pending_vblank_event *flip_event;
+#if !defined(NULLDISP_USE_ATOMIC)
+	struct drm_framebuffer *old_fb;
+	struct nulldisp_flip_data *flip_data;
+#endif
+	bool flip_async;
+};
+
+#if !defined(NULLDISP_USE_ATOMIC)
+struct nulldisp_framebuffer {
+	struct drm_framebuffer base;
+	struct drm_gem_object *obj[NULLDISP_MAX_PLANES];
+};
+
+#define to_nulldisp_framebuffer(framebuffer) \
+	container_of(framebuffer, struct nulldisp_framebuffer, base)
+#endif
+
+struct nulldisp_module_params {
+	unsigned int hdisplay;
+	unsigned int vdisplay;
+	unsigned int vrefresh;
+	unsigned int updateto;
+};
+
+#define to_nulldisp_crtc(crtc) \
+	container_of(crtc, struct nulldisp_crtc, base)
+
+#if defined(LMA)
+#define	obj_to_resv(obj) pdp_gem_get_resv(obj)
+#else
+#define	obj_to_resv(obj) nulldisp_gem_get_resv(obj)
+#endif
+
+/*
+ * The order of this array helps determine the order in which EGL configs are
+ * returned to an application using eglGetConfigs. As such, RGB 8888 formats
+ * should appear first, followed by RGB 565 configs. YUV configs should appear
+ * last.
+ */
+static const uint32_t nulldisp_modeset_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_ABGR2101010,
+#ifdef DRM_FORMAT_AXBXGXRX106106106106
+	DRM_FORMAT_AXBXGXRX106106106106,
+#endif
+#ifdef DRM_FORMAT_ABGR16161616F
+	DRM_FORMAT_ABGR16161616F,
+#endif
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV21,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_YUV444,
+	DRM_FORMAT_YUV420,
+	DRM_FORMAT_YVU420,
+};
+
+/*
+ * Note that nulldisp, being a no-hardware display controller driver,
+ * "supports" a number different decompression hardware
+ * versions (V0, V1, V2 ...). Real, hardware display controllers are
+ * likely to support only a single version.
+ */
+static const uint64_t nulldisp_primary_plane_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0_FIX,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V1,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V2,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V3,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V7,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V8,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V10,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_8x8_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_8x8_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_8x8_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_8x8_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_8x8_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0_FIX,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V1,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V2,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V3,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V7,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V8,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V10,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V13,
+	DRM_FORMAT_MOD_PVR_FBCDC_16x4_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_16x4_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_16x4_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_16x4_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_16x4_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_32x2_V1,
+	DRM_FORMAT_MOD_PVR_FBCDC_32x2_V3,
+	DRM_FORMAT_MOD_PVR_FBCDC_32x2_V8,
+	DRM_FORMAT_MOD_PVR_FBCDC_32x2_V10,
+	DRM_FORMAT_MOD_PVR_FBCDC_32x2_V12,
+	DRM_FORMAT_MOD_PVR_FBCDC_32x2_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY25_32x2_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY37_32x2_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY50_32x2_V14,
+	DRM_FORMAT_MOD_PVR_FBCDC_LOSSY75_32x2_V14,
+	DRM_FORMAT_MOD_INVALID
+};
+
+static struct nulldisp_module_params module_params = {
+	.hdisplay = NULLDISP_DEFAULT_WIDTH,
+	.vdisplay = NULLDISP_DEFAULT_HEIGHT,
+	.vrefresh = NULLDISP_DEFAULT_REFRESH_RATE,
+	.updateto = NULLDISP_NETLINK_TIMEOUT,
+};
+
+static int updateto_param_set(const char *val, const struct kernel_param *kp);
+
+static const struct kernel_param_ops updateto_ops = {
+	.set = updateto_param_set,
+	.get = param_get_uint,
+};
+
+module_param_named(width, module_params.hdisplay, uint, 0444);
+module_param_named(height, module_params.vdisplay, uint, 0444);
+module_param_named(refreshrate, module_params.vrefresh, uint, 0444);
+module_param_cb(updateto, &updateto_ops, &module_params.updateto, 0644);
+
+MODULE_PARM_DESC(width, "Preferred display width in pixels");
+MODULE_PARM_DESC(height, "Preferred display height in pixels");
+MODULE_PARM_DESC(refreshrate, "Preferred display refresh rate");
+MODULE_PARM_DESC(updateto, "Preferred remote update timeout (in seconds)");
+
+/*
+ * Please use this function to obtain the module parameters instead of
+ * accessing the global "module_params" structure directly.
+ */
+static inline const struct nulldisp_module_params *
+nulldisp_get_module_params(void)
+{
+	return &module_params;
+}
+
+static int updateto_param_set(const char *val, const struct kernel_param *kp)
+{
+	unsigned int updateto;
+	int err;
+
+	err = kstrtouint(val, 10, &updateto);
+	if (err)
+		return err;
+
+	if (updateto < NULLDISP_NETLINK_TIMEOUT_MIN ||
+	    updateto > NULLDISP_NETLINK_TIMEOUT_MAX)
+		return -EINVAL;
+
+	return param_set_uint(val, kp);
+}
+
+static unsigned long nulldisp_netlink_timeout(void)
+{
+	const struct nulldisp_module_params *module_params =
+		nulldisp_get_module_params();
+	unsigned int updateto;
+
+	kernel_param_lock(THIS_MODULE);
+
+	updateto = module_params->updateto;
+
+	kernel_param_unlock(THIS_MODULE);
+
+	return msecs_to_jiffies(updateto * 1000);
+}
+
+/******************************************************************************
+ * Linux compatibility functions
+ ******************************************************************************/
+static inline void
+nulldisp_drm_fb_set_format(struct drm_framebuffer *fb,
+			   u32 pixel_format)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	fb->format = drm_format_info(pixel_format);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	const struct drm_format_info *format = drm_format_info(pixel_format);
+
+	fb->pixel_format = pixel_format;
+
+	fb->depth = format->depth;
+	fb->bits_per_pixel = format->depth ? (format->cpp[0] * 8) : 0;
+#else
+	fb->pixel_format = pixel_format;
+
+	switch (pixel_format) {
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YUV420:
+	case DRM_FORMAT_YVU420:
+		/* Unused for YUV formats */
+		fb->depth = 0;
+		fb->bits_per_pixel = 0;
+		break;
+
+	default: /* RGB */
+		drm_fb_get_bpp_depth(pixel_format,
+				     &fb->depth,
+				     &fb->bits_per_pixel);
+	}
+#endif
+}
+
+static inline void nulldisp_drm_fb_set_modifier(struct drm_framebuffer *fb,
+						uint64_t value)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	fb->modifier = value;
+#else
+	/* FB modifier values must be the same for all planes */
+	fb->modifier[0] = value;
+	fb->modifier[1] = value;
+	fb->modifier[2] = value;
+	fb->modifier[3] = value;
+#endif
+}
+
+/******************************************************************************
+ * Plane functions
+ ******************************************************************************/
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+static bool nulldisp_primary_format_mod_supported(struct drm_plane *plane,
+						  uint32_t format,
+						  uint64_t modifier)
+{
+	/*
+	 * All 'nulldisp_modeset_formats' are supported for every modifier
+	 * in the 'nulldisp_primary_plane_modifiers' array.
+	 */
+	return true;
+}
+#endif
+
+#if defined(NULLDISP_USE_ATOMIC)
+static int nulldisp_plane_helper_atomic_check(struct drm_plane *plane,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0))
+					      struct drm_plane_state *state)
+{
+#else
+					      struct drm_atomic_state *astate)
+{
+	struct drm_plane_state *state =
+			drm_atomic_get_new_plane_state(astate, plane);
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0) */
+	struct drm_crtc_state *crtc_new_state;
+
+	if (!state->crtc)
+		return 0;
+
+	crtc_new_state = drm_atomic_get_new_crtc_state(state->state,
+						       state->crtc);
+
+	return drm_atomic_helper_check_plane_state(state, crtc_new_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, true);
+}
+
+static void
+nulldisp_plane_helper_atomic_update(struct drm_plane *plane,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0))
+				  struct drm_plane_state *old_state)
+#else
+				  struct drm_atomic_state *astate)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0) */
+{
+	struct drm_plane_state *state = plane->state;
+
+	if (state->crtc) {
+		struct nulldisp_crtc *nulldisp_crtc =
+					to_nulldisp_crtc(state->crtc);
+
+		nulldisp_crtc->fb = state->fb;
+	}
+}
+
+static const struct drm_plane_helper_funcs nulldisp_plane_helper_funcs = {
+	.prepare_fb =  drm_gem_plane_helper_prepare_fb,
+	.atomic_check = nulldisp_plane_helper_atomic_check,
+	.atomic_update = nulldisp_plane_helper_atomic_update,
+};
+
+static const struct drm_plane_funcs nulldisp_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = drm_plane_helper_destroy,
+	.reset = drm_atomic_helper_plane_reset,
+	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	.format_mod_supported = nulldisp_primary_format_mod_supported,
+};
+#else	/* defined(NULLDISP_USE_ATOMIC) */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+static int nulldisp_primary_helper_update(struct drm_plane *plane,
+					  struct drm_crtc *crtc,
+					  struct drm_framebuffer *fb,
+					  int crtc_x, int crtc_y,
+					  unsigned int crtc_w,
+					  unsigned int crtc_h,
+					  uint32_t src_x, uint32_t src_y,
+					  uint32_t src_w, uint32_t src_h,
+					  struct drm_modeset_acquire_ctx *ctx)
+{
+	struct nulldisp_display_device *nulldisp_dev = crtc->dev->dev_private;
+	struct drm_plane_state plane_state = {
+		.plane = plane,
+		.crtc = crtc,
+		.fb = fb,
+		.crtc_x = crtc_x,
+		.crtc_y = crtc_y,
+		.crtc_w = crtc_w,
+		.crtc_h = crtc_h,
+		.src_x = src_x,
+		.src_y = src_y,
+		.src_w = src_w,
+		.src_h = src_h,
+		.alpha = DRM_BLEND_ALPHA_OPAQUE,
+		.rotation = DRM_MODE_ROTATE_0,
+	};
+	struct drm_crtc_state crtc_state = {
+		.crtc = crtc,
+		.enable = crtc->enabled,
+		.adjusted_mode = crtc->mode,
+		.mode = crtc->mode,
+	};
+	struct drm_mode_set set = {
+		.fb = fb,
+		.crtc = crtc,
+		.mode = &crtc->mode,
+		.x = src_x >> 16, /* convert from fixed point */
+		.y = src_y >> 16, /* convert from fixed point */
+		.connectors = &nulldisp_dev->connector,
+		.num_connectors = 1,
+	};
+	int err;
+
+	BUG_ON(nulldisp_dev->connector->encoder == NULL);
+	BUG_ON(nulldisp_dev->connector->encoder->crtc != crtc);
+
+	err = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, false);
+	if (err)
+		return err;
+
+	if (!plane_state.visible)
+		return -EINVAL;
+
+	return crtc->funcs->set_config(&set, ctx);
+}
+
+static int nulldisp_primary_helper_disable(struct drm_plane *plane,
+					   struct drm_modeset_acquire_ctx *ctx)
+{
+	return -EINVAL;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)) */
+
+static const struct drm_plane_funcs nulldisp_plane_funcs = {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+	.update_plane = nulldisp_primary_helper_update,
+	.disable_plane = nulldisp_primary_helper_disable,
+#else
+	.update_plane = drm_primary_helper_update,
+	.disable_plane = drm_primary_helper_disable,
+#endif
+	.destroy = drm_primary_helper_destroy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	.format_mod_supported = nulldisp_primary_format_mod_supported,
+#endif
+};
+#endif	/* defined(NULLDISP_USE_ATOMIC) */
+
+/******************************************************************************
+ * CRTC functions
+ ******************************************************************************/
+
+static bool
+nulldisp_crtc_helper_mode_fixup(struct drm_crtc *crtc,
+				const struct drm_display_mode *mode,
+				struct drm_display_mode *adjusted_mode)
+{
+	/*
+	 * Fix up mode so that it's compatible with the hardware. The results
+	 * should be stored in adjusted_mode (i.e. mode should be untouched).
+	 */
+	return true;
+}
+
+static void nulldisp_crtc_helper_disable(struct drm_crtc *crtc)
+{
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+
+#if !defined(NULLDISP_USE_ATOMIC)
+	if (atomic_read(&nulldisp_crtc->flip_status) ==
+	    NULLDISP_CRTC_FLIP_STATUS_PENDING)
+		wait_for_completion(&nulldisp_crtc->flip_scheduled);
+
+	/*
+	 * Flush any outstanding page flip related work. The order this
+	 * is done is important, to ensure there are no outstanding
+	 * page flips.
+	 */
+	flush_work(&nulldisp_crtc->flip_work);
+	flush_delayed_work(&nulldisp_crtc->flip_to_work);
+#endif
+	flush_delayed_work(&nulldisp_crtc->vb_work);
+
+	drm_crtc_vblank_off(crtc);
+	flush_delayed_work(&nulldisp_crtc->vb_work);
+
+	/*
+	 * Vblank has been disabled, so the vblank handler shouldn't be
+	 * able to reschedule itself.
+	 */
+	BUG_ON(cancel_delayed_work(&nulldisp_crtc->vb_work));
+
+	BUG_ON(atomic_read(&nulldisp_crtc->flip_status) !=
+	       NULLDISP_CRTC_FLIP_STATUS_NONE);
+
+#if !defined(NULLDISP_USE_ATOMIC)
+	/* Flush any remaining dirty FB work */
+	flush_delayed_work(&nulldisp_crtc->copy_to_work);
+#endif
+}
+
+static void nulldisp_crtc_flip_complete(struct drm_crtc *crtc)
+{
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+	unsigned long flags;
+
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
+
+	/* The flipping process has been completed so reset the flip state */
+	atomic_set(&nulldisp_crtc->flip_status, NULLDISP_CRTC_FLIP_STATUS_NONE);
+	nulldisp_crtc->flip_async = false;
+
+#if !defined(NULLDISP_USE_ATOMIC)
+	if (nulldisp_crtc->flip_data) {
+		dma_fence_put(nulldisp_crtc->flip_data->wait_fence);
+		kfree(nulldisp_crtc->flip_data);
+		nulldisp_crtc->flip_data = NULL;
+	}
+#endif
+	if (nulldisp_crtc->flip_event) {
+		drm_crtc_send_vblank_event(crtc, nulldisp_crtc->flip_event);
+		nulldisp_crtc->flip_event = NULL;
+	}
+
+	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+}
+
+#if defined(NULLDISP_USE_ATOMIC)
+static void nulldisp_crtc_helper_mode_set_nofb(struct drm_crtc *crtc)
+{
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void nulldisp_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+					      struct drm_crtc_state *old_state)
+{
+#else
+static void nulldisp_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+					      struct drm_atomic_state *state)
+{
+	struct drm_crtc_state *old_state = drm_atomic_get_new_crtc_state(state, crtc);
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+
+	if (!crtc->state->active || !old_state->active)
+		return;
+
+	if (nulldisp_crtc->fb) {
+		struct nulldisp_display_device *nulldisp_dev =
+							crtc->dev->dev_private;
+
+		reinit_completion(&nulldisp_crtc->flip_done);
+
+		if (!nlpvrdpy_send_flip(nulldisp_dev->nlpvrdpy,
+				       nulldisp_crtc->fb,
+				       &nulldisp_crtc->fb->obj[0])) {
+			unsigned long res;
+
+			res = wait_for_completion_timeout(
+					&nulldisp_crtc->flip_done,
+					nulldisp_netlink_timeout());
+
+			if (!res)
+				DRM_ERROR(
+				    "timed out waiting for remote update\n");
+		}
+
+		nulldisp_crtc->fb = NULL;
+	}
+
+	if (crtc->state->event) {
+		unsigned long flags;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+		nulldisp_crtc->flip_async = crtc->state->async_flip;
+#else
+		nulldisp_crtc->flip_async = !!(crtc->state->pageflip_flags
+					       & DRM_MODE_PAGE_FLIP_ASYNC);
+#endif
+		if (nulldisp_crtc->flip_async)
+			WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		nulldisp_crtc->flip_event = crtc->state->event;
+		crtc->state->event = NULL;
+
+		atomic_set(&nulldisp_crtc->flip_status,
+			   NULLDISP_CRTC_FLIP_STATUS_DONE);
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+		if (nulldisp_crtc->flip_async)
+			nulldisp_crtc_flip_complete(crtc);
+	}
+}
+
+static void nulldisp_crtc_set_enabled(struct drm_crtc *crtc, bool enable)
+{
+	if (enable)
+		drm_crtc_vblank_on(crtc);
+	else
+		nulldisp_crtc_helper_disable(crtc);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void
+nulldisp_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_crtc_state)
+#else
+static void
+nulldisp_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_atomic_state *state)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+{
+	nulldisp_crtc_set_enabled(crtc, true);
+
+	if (crtc->state->event) {
+		struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+		unsigned long flags;
+
+		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		nulldisp_crtc->flip_event = crtc->state->event;
+		crtc->state->event = NULL;
+
+		atomic_set(&nulldisp_crtc->flip_status,
+			   NULLDISP_CRTC_FLIP_STATUS_DONE);
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+	}
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+static void
+nulldisp_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_crtc_state)
+#else
+static void
+nulldisp_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_atomic_state *state)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+{
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+
+	nulldisp_crtc_set_enabled(crtc, false);
+
+	nulldisp_crtc->fb = NULL;
+
+	if (crtc->state->event) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+	}
+}
+#else	/* defined(NULLDISP_USE_ATOMIC) */
+static void nulldisp_crtc_helper_dpms(struct drm_crtc *crtc,
+				      int mode)
+{
+	/*
+	 * Change the power state of the display/pipe/port/etc. If the mode
+	 * passed in is unsupported, the provider must use the next lowest
+	 * power level.
+	 */
+}
+
+static void nulldisp_crtc_helper_prepare(struct drm_crtc *crtc)
+{
+	drm_crtc_vblank_off(crtc);
+
+	/*
+	 * Prepare the display/pipe/port/etc for a mode change e.g. put them
+	 * in a low power state/turn them off
+	 */
+}
+
+static void nulldisp_crtc_helper_commit(struct drm_crtc *crtc)
+{
+	/* Turn the display/pipe/port/etc back on */
+
+	drm_crtc_vblank_on(crtc);
+}
+
+static int
+nulldisp_crtc_helper_mode_set_base_atomic(struct drm_crtc *crtc,
+					  struct drm_framebuffer *fb,
+					  int x, int y,
+					  enum mode_set_atomic atomic)
+{
+	/* Set the display base address or offset from the base address */
+	return 0;
+}
+
+static int nulldisp_crtc_helper_mode_set_base(struct drm_crtc *crtc,
+					      int x, int y,
+					      struct drm_framebuffer *old_fb)
+{
+	return nulldisp_crtc_helper_mode_set_base_atomic(crtc,
+							 crtc->primary->fb,
+							 x,
+							 y,
+							 0);
+}
+
+static int
+nulldisp_crtc_helper_mode_set(struct drm_crtc *crtc,
+			      struct drm_display_mode *mode,
+			      struct drm_display_mode *adjusted_mode,
+			      int x, int y,
+			      struct drm_framebuffer *old_fb)
+{
+	/* Setup the new mode and/or framebuffer */
+	return nulldisp_crtc_helper_mode_set_base(crtc, x, y, old_fb);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+static void nulldisp_crtc_helper_load_lut(struct drm_crtc *crtc)
+{
+}
+#endif
+#endif	/* defined(NULLDISP_USE_ATOMIC) */
+
+static void nulldisp_crtc_destroy(struct drm_crtc *crtc)
+{
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+
+	DRM_DEBUG_DRIVER("[CRTC:%d]\n", crtc->base.id);
+
+	drm_crtc_cleanup(crtc);
+
+	BUG_ON(atomic_read(&nulldisp_crtc->flip_status) !=
+	       NULLDISP_CRTC_FLIP_STATUS_NONE);
+
+	kfree(nulldisp_crtc);
+}
+
+#if !defined(NULLDISP_USE_ATOMIC)
+static void nulldisp_crtc_flip_done(struct nulldisp_crtc *nulldisp_crtc)
+{
+	struct drm_crtc *crtc = &nulldisp_crtc->base;
+
+	struct drm_framebuffer *old_fb;
+
+	WARN_ON(atomic_read(&nulldisp_crtc->flip_status) !=
+		NULLDISP_CRTC_FLIP_STATUS_PENDING);
+
+	old_fb = nulldisp_crtc->old_fb;
+	nulldisp_crtc->old_fb = NULL;
+
+	(void) nulldisp_crtc_helper_mode_set_base(crtc, crtc->x, crtc->y,
+						  old_fb);
+
+	atomic_set(&nulldisp_crtc->flip_status, NULLDISP_CRTC_FLIP_STATUS_DONE);
+
+	if (nulldisp_crtc->flip_async)
+		nulldisp_crtc_flip_complete(crtc);
+}
+
+static bool nulldisp_set_flip_to(struct nulldisp_crtc *nulldisp_crtc)
+{
+	struct drm_crtc *crtc = &nulldisp_crtc->base;
+	struct nulldisp_display_device *nulldisp_dev = crtc->dev->dev_private;
+
+	/* Returns false if work already queued, else true */
+	return queue_delayed_work(nulldisp_dev->workqueue,
+				  &nulldisp_crtc->flip_to_work,
+				  nulldisp_netlink_timeout());
+}
+
+static bool nulldisp_set_copy_to(struct nulldisp_crtc *nulldisp_crtc)
+{
+	struct drm_crtc *crtc = &nulldisp_crtc->base;
+	struct nulldisp_display_device *nulldisp_dev = crtc->dev->dev_private;
+
+	/* Returns false if work already queued, else true */
+	return queue_delayed_work(nulldisp_dev->workqueue,
+				  &nulldisp_crtc->copy_to_work,
+				  nulldisp_netlink_timeout());
+}
+
+static void nulldisp_flip_to_work(struct work_struct *w)
+{
+	struct delayed_work *dw =
+		container_of(w, struct delayed_work, work);
+	struct nulldisp_crtc *nulldisp_crtc =
+		container_of(dw, struct nulldisp_crtc, flip_to_work);
+
+	if (atomic_read(&nulldisp_crtc->flip_status) ==
+	    NULLDISP_CRTC_FLIP_STATUS_PENDING)
+		nulldisp_crtc_flip_done(nulldisp_crtc);
+}
+
+static void nulldisp_copy_to_work(struct work_struct *w)
+{
+	struct delayed_work *dw =
+		container_of(w, struct delayed_work, work);
+	struct nulldisp_crtc *nulldisp_crtc =
+		container_of(dw, struct nulldisp_crtc, copy_to_work);
+
+	complete(&nulldisp_crtc->copy_done);
+}
+
+static void nulldisp_flip_work(struct work_struct *w)
+{
+	struct nulldisp_crtc *nulldisp_crtc =
+		container_of(w, struct nulldisp_crtc, flip_work);
+	struct drm_crtc *crtc = &nulldisp_crtc->base;
+	struct drm_device *dev = crtc->dev;
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+	struct nulldisp_framebuffer *nulldisp_fb =
+		to_nulldisp_framebuffer(crtc->primary->fb);
+
+	/*
+	 * To prevent races with disconnect requests from user space,
+	 * set the timeout before sending the flip request.
+	 */
+	nulldisp_set_flip_to(nulldisp_crtc);
+
+	if (nlpvrdpy_send_flip(nulldisp_dev->nlpvrdpy,
+			       &nulldisp_fb->base,
+			       &nulldisp_fb->obj[0]))
+		goto fail_cancel;
+
+	return;
+
+fail_cancel:
+	/*
+	 * We can't flush the work, as we are running on the same
+	 * single threaded workqueue as the work to be flushed.
+	 */
+	cancel_delayed_work(&nulldisp_crtc->flip_to_work);
+
+	nulldisp_crtc_flip_done(nulldisp_crtc);
+}
+
+static void nulldisp_crtc_flip_cb(struct dma_fence *fence,
+				  struct dma_fence_cb *cb)
+{
+	struct nulldisp_flip_data *flip_data =
+		container_of(cb, struct nulldisp_flip_data, base);
+	struct drm_crtc *crtc = flip_data->crtc;
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	(void) queue_work(nulldisp_dev->workqueue,
+			  &nulldisp_crtc->flip_work);
+
+	complete_all(&nulldisp_crtc->flip_scheduled);
+}
+
+static void nulldisp_crtc_flip_schedule_cb(struct dma_fence *fence,
+					   struct dma_fence_cb *cb)
+{
+	struct nulldisp_flip_data *flip_data =
+		container_of(cb, struct nulldisp_flip_data, base);
+	int err = 0;
+
+	if (flip_data->wait_fence)
+		err = dma_fence_add_callback(flip_data->wait_fence,
+					     &flip_data->base,
+					     nulldisp_crtc_flip_cb);
+
+	if (!flip_data->wait_fence || err) {
+		if (err && err != -ENOENT)
+			DRM_ERROR("flip failed to wait on old buffer\n");
+		nulldisp_crtc_flip_cb(flip_data->wait_fence, &flip_data->base);
+	}
+}
+
+static int nulldisp_crtc_flip_schedule(struct drm_crtc *crtc,
+				       struct drm_gem_object *obj,
+				       struct drm_gem_object *old_obj)
+{
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+	struct dma_resv *resv = obj_to_resv(obj);
+	struct dma_resv *old_resv = obj_to_resv(old_obj);
+	struct nulldisp_flip_data *flip_data;
+	struct dma_fence *fence;
+	int err;
+
+	flip_data = kmalloc(sizeof(*flip_data), GFP_KERNEL);
+	if (!flip_data)
+		return -ENOMEM;
+
+	flip_data->crtc = crtc;
+
+	ww_mutex_lock(&old_resv->lock, NULL);
+	flip_data->wait_fence =
+		dma_fence_get(dma_resv_get_excl(old_resv));
+
+	if (old_resv != resv) {
+		ww_mutex_unlock(&old_resv->lock);
+		ww_mutex_lock(&resv->lock, NULL);
+	}
+
+	fence = dma_fence_get(dma_resv_get_excl(resv));
+	ww_mutex_unlock(&resv->lock);
+
+	nulldisp_crtc->flip_data = flip_data;
+	reinit_completion(&nulldisp_crtc->flip_scheduled);
+	atomic_set(&nulldisp_crtc->flip_status,
+		   NULLDISP_CRTC_FLIP_STATUS_PENDING);
+
+	if (fence) {
+		err = dma_fence_add_callback(fence, &flip_data->base,
+					     nulldisp_crtc_flip_schedule_cb);
+		dma_fence_put(fence);
+		if (err && err != -ENOENT)
+			goto err_set_flip_status_none;
+	}
+
+	if (!fence || err == -ENOENT) {
+		nulldisp_crtc_flip_schedule_cb(fence, &flip_data->base);
+		err = 0;
+	}
+
+	return err;
+
+err_set_flip_status_none:
+	atomic_set(&nulldisp_crtc->flip_status, NULLDISP_CRTC_FLIP_STATUS_NONE);
+	dma_fence_put(flip_data->wait_fence);
+	kfree(flip_data);
+	return err;
+}
+
+static int nulldisp_crtc_page_flip(struct drm_crtc *crtc,
+				   struct drm_framebuffer *fb,
+				   struct drm_pending_vblank_event *event,
+				   uint32_t page_flip_flags
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0))
+				   , struct drm_modeset_acquire_ctx *ctx
+#endif
+				   )
+{
+	struct nulldisp_crtc *nulldisp_crtc = to_nulldisp_crtc(crtc);
+	struct nulldisp_framebuffer *nulldisp_fb = to_nulldisp_framebuffer(fb);
+	struct nulldisp_framebuffer *nulldisp_old_fb =
+		to_nulldisp_framebuffer(crtc->primary->fb);
+	enum nulldisp_crtc_flip_status status;
+	unsigned long flags;
+	int err;
+
+	spin_lock_irqsave(&crtc->dev->event_lock, flags);
+	status = atomic_read(&nulldisp_crtc->flip_status);
+	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+
+	if (status != NULLDISP_CRTC_FLIP_STATUS_NONE)
+		return -EBUSY;
+
+	if (!(page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC)) {
+		err = drm_crtc_vblank_get(crtc);
+		if (err)
+			return err;
+	}
+
+	nulldisp_crtc->old_fb = crtc->primary->fb;
+	nulldisp_crtc->flip_event = event;
+	nulldisp_crtc->flip_async = !!(page_flip_flags &
+				       DRM_MODE_PAGE_FLIP_ASYNC);
+
+	/* Set the crtc to point to the new framebuffer */
+	crtc->primary->fb = fb;
+
+	err = nulldisp_crtc_flip_schedule(crtc, nulldisp_fb->obj[0],
+					  nulldisp_old_fb->obj[0]);
+	if (err) {
+		crtc->primary->fb = nulldisp_crtc->old_fb;
+		nulldisp_crtc->old_fb = NULL;
+		nulldisp_crtc->flip_event = NULL;
+		nulldisp_crtc->flip_async = false;
+
+		DRM_ERROR("failed to schedule flip (err=%d)\n", err);
+		goto err_vblank_put;
+	}
+
+	return 0;
+
+err_vblank_put:
+	if (!(page_flip_flags & DRM_MODE_PAGE_FLIP_ASYNC))
+		drm_crtc_vblank_put(crtc);
+	return err;
+}
+#endif /* !defined(NULLDISP_USE_ATOMIC) */
+
+static bool nulldisp_queue_vblank_work(struct nulldisp_crtc *nulldisp_crtc)
+{
+	struct drm_crtc *crtc = &nulldisp_crtc->base;
+	struct nulldisp_display_device *nulldisp_dev = crtc->dev->dev_private;
+	int vrefresh;
+	const int vrefresh_default = 60;
+
+	vrefresh = drm_mode_vrefresh(&crtc->hwmode);
+	if (!vrefresh) {
+		vrefresh = vrefresh_default;
+		DRM_INFO_ONCE(
+			"vertical refresh rate is zero, defaulting to %d\n",
+			vrefresh);
+	}
+
+	/* Returns false if work already queued, else true */
+	return queue_delayed_work(nulldisp_dev->workqueue,
+				  &nulldisp_crtc->vb_work,
+				  usecs_to_jiffies(1000000/vrefresh));
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+static int nulldisp_enable_vblank(struct drm_crtc *crtc)
+#else
+static int nulldisp_enable_vblank(struct drm_device *dev, unsigned int pipe)
+#endif
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	struct drm_device *dev = crtc->dev;
+	unsigned int pipe      = drm_crtc_index(crtc);
+#endif
+
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	switch (pipe) {
+	case 0:
+		break;
+	default:
+		DRM_ERROR("invalid crtc %u\n", pipe);
+		return -EINVAL;
+	}
+
+	if (!nulldisp_queue_vblank_work(nulldisp_dev->nulldisp_crtc)) {
+		DRM_ERROR("work already queued\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+static void nulldisp_disable_vblank(struct drm_crtc *crtc)
+#else
+static void nulldisp_disable_vblank(struct drm_device *dev, unsigned int pipe)
+#endif
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	struct drm_device *dev = crtc->dev;
+	unsigned int pipe      = drm_crtc_index(crtc);
+#endif
+
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	switch (pipe) {
+	case 0:
+		break;
+	default:
+		DRM_ERROR("invalid crtc %u\n", pipe);
+		return;
+	}
+
+	/*
+	 * Vblank events may be disabled from within the vblank handler,
+	 * so don't wait for the work to complete.
+	 */
+	(void) cancel_delayed_work(&nulldisp_dev->nulldisp_crtc->vb_work);
+}
+
+static const struct drm_crtc_helper_funcs nulldisp_crtc_helper_funcs = {
+	.mode_fixup = nulldisp_crtc_helper_mode_fixup,
+#if defined(NULLDISP_USE_ATOMIC)
+	.mode_set_nofb = nulldisp_crtc_helper_mode_set_nofb,
+	.atomic_flush = nulldisp_crtc_helper_atomic_flush,
+	.atomic_enable = nulldisp_crtc_helper_atomic_enable,
+	.atomic_disable = nulldisp_crtc_helper_atomic_disable,
+#else
+	.dpms = nulldisp_crtc_helper_dpms,
+	.prepare = nulldisp_crtc_helper_prepare,
+	.commit = nulldisp_crtc_helper_commit,
+	.mode_set = nulldisp_crtc_helper_mode_set,
+	.mode_set_base = nulldisp_crtc_helper_mode_set_base,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	.load_lut = nulldisp_crtc_helper_load_lut,
+#endif
+	.mode_set_base_atomic = nulldisp_crtc_helper_mode_set_base_atomic,
+	.disable = nulldisp_crtc_helper_disable,
+#endif	/* defined(NULLDISP_USE_ATOMIC) */
+};
+
+static const struct drm_crtc_funcs nulldisp_crtc_funcs = {
+	.destroy = nulldisp_crtc_destroy,
+#if defined(NULLDISP_USE_ATOMIC)
+	.reset = drm_atomic_helper_crtc_reset,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+#else
+	.reset = NULL,
+	.cursor_set = NULL,
+	.cursor_move = NULL,
+	.gamma_set = NULL,
+	.set_config = drm_crtc_helper_set_config,
+	.page_flip = nulldisp_crtc_page_flip,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
+	.enable_vblank  = nulldisp_enable_vblank,
+	.disable_vblank = nulldisp_disable_vblank,
+#endif
+};
+
+static void nulldisp_handle_vblank(struct work_struct *w)
+{
+	struct delayed_work *dw =
+		container_of(w, struct delayed_work, work);
+	struct nulldisp_crtc *nulldisp_crtc =
+		container_of(dw, struct nulldisp_crtc, vb_work);
+	struct drm_crtc *crtc = &nulldisp_crtc->base;
+	struct drm_device *dev = crtc->dev;
+	enum nulldisp_crtc_flip_status status;
+
+	/*
+	 * Reschedule the handler, if necessary. This is done before
+	 * calling drm_crtc_vblank_put, so that the work can be cancelled
+	 * if vblank events are disabled.
+	 */
+	if (drm_handle_vblank(dev, 0))
+		(void) nulldisp_queue_vblank_work(nulldisp_crtc);
+
+	status = atomic_read(&nulldisp_crtc->flip_status);
+	if (status == NULLDISP_CRTC_FLIP_STATUS_DONE) {
+		if (!nulldisp_crtc->flip_async)
+			nulldisp_crtc_flip_complete(crtc);
+#if !defined(NULLDISP_USE_ATOMIC)
+		drm_crtc_vblank_put(crtc);
+#endif
+	}
+
+}
+
+static struct nulldisp_crtc *
+nulldisp_crtc_create(struct nulldisp_display_device *nulldisp_dev)
+{
+	struct nulldisp_crtc *nulldisp_crtc;
+	struct drm_crtc *crtc;
+	struct drm_plane *primary;
+
+	nulldisp_crtc = kzalloc(sizeof(*nulldisp_crtc), GFP_KERNEL);
+	if (!nulldisp_crtc)
+		goto err_return;
+
+	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
+	if (!primary)
+		goto err_free_crtc;
+
+	crtc = &nulldisp_crtc->base;
+
+	atomic_set(&nulldisp_crtc->flip_status, NULLDISP_CRTC_FLIP_STATUS_NONE);
+#if defined(NULLDISP_USE_ATOMIC)
+	init_completion(&nulldisp_crtc->flip_done);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0))
+	init_completion(&nulldisp_crtc->copy_done);
+#endif
+#else
+	init_completion(&nulldisp_crtc->flip_scheduled);
+	init_completion(&nulldisp_crtc->copy_done);
+#endif
+
+	if (drm_universal_plane_init(nulldisp_dev->dev, primary, 0,
+				     &nulldisp_plane_funcs,
+				     nulldisp_modeset_formats,
+				     ARRAY_SIZE(nulldisp_modeset_formats),
+				     nulldisp_primary_plane_modifiers,
+				     DRM_PLANE_TYPE_PRIMARY, NULL)) {
+		goto err_free_primary;
+	}
+
+#if defined(NULLDISP_USE_ATOMIC)
+	drm_plane_helper_add(primary, &nulldisp_plane_helper_funcs);
+#endif
+
+	if (drm_crtc_init_with_planes(nulldisp_dev->dev, crtc, primary,
+				      NULL, &nulldisp_crtc_funcs, NULL)) {
+		goto err_cleanup_plane;
+	}
+
+	drm_crtc_helper_add(crtc, &nulldisp_crtc_helper_funcs);
+
+	INIT_DELAYED_WORK(&nulldisp_crtc->vb_work, nulldisp_handle_vblank);
+#if !defined(NULLDISP_USE_ATOMIC)
+	INIT_WORK(&nulldisp_crtc->flip_work, nulldisp_flip_work);
+	INIT_DELAYED_WORK(&nulldisp_crtc->copy_to_work, nulldisp_copy_to_work);
+	INIT_DELAYED_WORK(&nulldisp_crtc->flip_to_work, nulldisp_flip_to_work);
+#endif
+
+	DRM_DEBUG_DRIVER("[CRTC:%d]\n", crtc->base.id);
+
+	return nulldisp_crtc;
+
+err_cleanup_plane:
+	drm_plane_cleanup(primary);
+err_free_primary:
+	kfree(primary);
+err_free_crtc:
+	kfree(nulldisp_crtc);
+err_return:
+	return NULL;
+}
+
+
+/******************************************************************************
+ * Connector functions
+ ******************************************************************************/
+
+static int
+nulldisp_validate_module_parameters(void)
+{
+	const struct nulldisp_module_params *module_params =
+		nulldisp_get_module_params();
+
+	if (!module_params->hdisplay ||
+	    !module_params->vdisplay ||
+	    !module_params->vrefresh ||
+	    (module_params->hdisplay > NULLDISP_FB_WIDTH_MAX) ||
+	    (module_params->vdisplay > NULLDISP_FB_HEIGHT_MAX))
+		return -EINVAL;
+
+	return 0;
+}
+
+static bool
+nulldisp_set_preferred_mode(struct drm_connector *connector,
+			    uint32_t hdisplay,
+			    uint32_t vdisplay,
+			    uint32_t vrefresh)
+{
+	struct drm_display_mode *mode;
+
+	/*
+	 * Mark the first mode, matching the hdisplay, vdisplay and
+	 * vrefresh, preferred.
+	 */
+	list_for_each_entry(mode, &connector->probed_modes, head)
+		if (mode->hdisplay == hdisplay &&
+		    mode->vdisplay == vdisplay &&
+		    drm_mode_vrefresh(mode) == vrefresh) {
+			mode->type |= DRM_MODE_TYPE_PREFERRED;
+			return true;
+		}
+
+	return false;
+}
+
+static bool
+nulldisp_connector_add_preferred_mode(struct drm_connector *connector,
+				      uint32_t hdisplay,
+				      uint32_t vdisplay,
+				      uint32_t vrefresh)
+{
+	struct drm_display_mode *preferred_mode;
+
+	preferred_mode = drm_cvt_mode(connector->dev,
+				      hdisplay, vdisplay, vrefresh,
+				      false, false, false);
+	if (!preferred_mode) {
+		DRM_DEBUG_DRIVER("[CONNECTOR:%s]:create mode %dx%d@%d failed\n",
+				 connector->name,
+				 hdisplay,
+				 vdisplay,
+				 vrefresh);
+
+		return false;
+	}
+
+	preferred_mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER;
+
+	drm_mode_probed_add(connector, preferred_mode);
+
+	return true;
+}
+
+/*
+ * Gather modes. Here we can get the EDID data from the monitor and
+ * turn it into drm_display_mode structures.
+ */
+static int
+nulldisp_connector_helper_get_modes(struct drm_connector *connector)
+{
+	int modes_count;
+	struct drm_device *dev = connector->dev;
+	const struct nulldisp_module_params *module_params =
+		nulldisp_get_module_params();
+	uint32_t hdisplay = module_params->hdisplay;
+	uint32_t vdisplay = module_params->vdisplay;
+	uint32_t vrefresh = module_params->vrefresh;
+
+	/* Add common modes */
+	modes_count = drm_add_modes_noedid(connector,
+					   dev->mode_config.max_width,
+					   dev->mode_config.max_height);
+
+	/*
+	 * Check if any of the connector modes match the preferred mode
+	 * criteria specified by the module parameters. If the mode is
+	 * found - flag it as preferred. Otherwise create the preferred
+	 * mode based on the module parameters criteria, and flag it as
+	 * preferred.
+	 */
+	if (!nulldisp_set_preferred_mode(connector,
+					 hdisplay,
+					 vdisplay,
+					 vrefresh))
+		if (nulldisp_connector_add_preferred_mode(connector,
+							  hdisplay,
+							  vdisplay,
+							  vrefresh))
+			modes_count++;
+
+	/* Sort the connector modes by relevance */
+	drm_mode_sort(&connector->probed_modes);
+
+	return modes_count;
+}
+
+static enum drm_mode_status
+nulldisp_connector_helper_mode_valid(struct drm_connector *connector,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+				     const struct drm_display_mode *mode)
+#else
+				     struct drm_display_mode *mode)
+#endif
+{
+	/*
+	 * This function is called on each gathered mode (e.g. via EDID)
+	 * and gives the driver a chance to reject it if the hardware
+	 * cannot support it.
+	 */
+	return MODE_OK;
+}
+
+#if !defined(NULLDISP_USE_ATOMIC)
+static struct drm_encoder *
+nulldisp_connector_helper_best_encoder(struct drm_connector *connector)
+{
+	/* Pick the first encoder we find */
+	if (connector->encoder_ids[0] != 0) {
+		struct drm_encoder *encoder;
+
+		encoder = drm_encoder_find(connector->dev,
+					   NULL,
+					   connector->encoder_ids[0]);
+		if (encoder) {
+			DRM_DEBUG_DRIVER(
+				"[ENCODER:%d:%s] best for [CONNECTOR:%d:%s]\n",
+				 encoder->base.id,
+				 encoder->name,
+				 connector->base.id,
+				 connector->name);
+			return encoder;
+		}
+	}
+
+	return NULL;
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+static enum drm_connector_status
+nulldisp_connector_detect(struct drm_connector *connector,
+			  bool force)
+{
+	/* Return whether or not a monitor is attached to the connector */
+	return connector_status_connected;
+}
+#endif
+
+static void nulldisp_connector_destroy(struct drm_connector *connector)
+{
+	DRM_DEBUG_DRIVER("[CONNECTOR:%d:%s]\n",
+			 connector->base.id,
+			 connector->name);
+
+	drm_connector_update_edid_property(connector, NULL);
+	drm_connector_cleanup(connector);
+
+	kfree(connector);
+}
+
+static void nulldisp_connector_force(struct drm_connector *connector)
+{
+}
+
+static const struct drm_connector_helper_funcs
+nulldisp_connector_helper_funcs = {
+	.get_modes = nulldisp_connector_helper_get_modes,
+	.mode_valid = nulldisp_connector_helper_mode_valid,
+	/*
+	 * For atomic, don't set atomic_best_encoder or best_encoder. This will
+	 * cause the DRM core to fallback to drm_atomic_helper_best_encoder().
+	 * This is fine as we only have a single connector and encoder.
+	 */
+#if !defined(NULLDISP_USE_ATOMIC)
+	.best_encoder = nulldisp_connector_helper_best_encoder,
+#endif
+};
+
+static const struct drm_connector_funcs nulldisp_connector_funcs = {
+#if defined(NULLDISP_USE_ATOMIC)
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+#else
+	.dpms = drm_helper_connector_dpms,
+	.reset = NULL,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
+	.detect = nulldisp_connector_detect,
+#endif
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = nulldisp_connector_destroy,
+	.force = nulldisp_connector_force,
+};
+
+static struct drm_connector *
+nulldisp_connector_create(struct nulldisp_display_device *nulldisp_dev,
+			  int type)
+{
+	struct drm_connector *connector;
+
+	connector = kzalloc(sizeof(*connector), GFP_KERNEL);
+	if (!connector)
+		return NULL;
+
+	drm_connector_init(nulldisp_dev->dev,
+			   connector,
+			   &nulldisp_connector_funcs,
+			   type);
+	drm_connector_helper_add(connector, &nulldisp_connector_helper_funcs);
+
+	connector->dpms = DRM_MODE_DPMS_OFF;
+	connector->interlace_allowed = false;
+	connector->doublescan_allowed = false;
+	connector->display_info.subpixel_order = SubPixelUnknown;
+
+	DRM_DEBUG_DRIVER("[CONNECTOR:%d:%s]\n",
+			 connector->base.id,
+			 connector->name);
+
+	return connector;
+}
+
+
+/******************************************************************************
+ * Encoder functions
+ ******************************************************************************/
+
+static void nulldisp_encoder_helper_dpms(struct drm_encoder *encoder,
+					 int mode)
+{
+	/*
+	 * Set the display power state or active encoder based on the mode. If
+	 * the mode passed in is unsupported, the provider must use the next
+	 * lowest power level.
+	 */
+}
+
+static bool
+nulldisp_encoder_helper_mode_fixup(struct drm_encoder *encoder,
+				   const struct drm_display_mode *mode,
+				   struct drm_display_mode *adjusted_mode)
+{
+	/*
+	 * Fix up mode so that it's compatible with the hardware. The results
+	 * should be stored in adjusted_mode (i.e. mode should be untouched).
+	 */
+	return true;
+}
+
+static void nulldisp_encoder_helper_prepare(struct drm_encoder *encoder)
+{
+	/*
+	 * Prepare the encoder for a mode change e.g. set the active encoder
+	 * accordingly/turn the encoder off
+	 */
+}
+
+static void nulldisp_encoder_helper_commit(struct drm_encoder *encoder)
+{
+	/* Turn the encoder back on/set the active encoder */
+}
+
+static void
+nulldisp_encoder_helper_mode_set(struct drm_encoder *encoder,
+				 struct drm_display_mode *mode,
+				 struct drm_display_mode *adjusted_mode)
+{
+	/* Setup the encoder for the new mode */
+}
+
+static void nulldisp_encoder_destroy(struct drm_encoder *encoder)
+{
+	DRM_DEBUG_DRIVER("[ENCODER:%d:%s]\n", encoder->base.id, encoder->name);
+
+	drm_encoder_cleanup(encoder);
+	kfree(encoder);
+}
+
+static const struct drm_encoder_helper_funcs nulldisp_encoder_helper_funcs = {
+	.dpms = nulldisp_encoder_helper_dpms,
+	.mode_fixup = nulldisp_encoder_helper_mode_fixup,
+	.prepare = nulldisp_encoder_helper_prepare,
+	.commit = nulldisp_encoder_helper_commit,
+	.mode_set = nulldisp_encoder_helper_mode_set,
+	.detect = NULL,
+	.disable = NULL,
+};
+
+static const struct drm_encoder_funcs nulldisp_encoder_funcs = {
+	.reset = NULL,
+	.destroy = nulldisp_encoder_destroy,
+};
+
+static struct drm_encoder *
+nulldisp_encoder_create(struct nulldisp_display_device *nulldisp_dev,
+			int type)
+{
+	struct drm_encoder *encoder;
+	int err;
+
+	encoder = kzalloc(sizeof(*encoder), GFP_KERNEL);
+	if (!encoder)
+		return ERR_PTR(-ENOMEM);
+
+	err = drm_encoder_init(nulldisp_dev->dev,
+			       encoder,
+			       &nulldisp_encoder_funcs,
+			       type,
+			       NULL);
+	if (err) {
+		DRM_ERROR("Failed to initialise encoder\n");
+		return ERR_PTR(err);
+	}
+	drm_encoder_helper_add(encoder, &nulldisp_encoder_helper_funcs);
+
+	/*
+	 * This is a bit field that's used to determine which
+	 * CRTCs can drive this encoder.
+	 */
+	encoder->possible_crtcs = 0x1;
+
+	DRM_DEBUG_DRIVER("[ENCODER:%d:%s]\n", encoder->base.id, encoder->name);
+
+	return encoder;
+}
+
+
+/******************************************************************************
+ * Framebuffer functions
+ ******************************************************************************/
+
+#if defined(NULLDISP_USE_ATOMIC)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0))
+static int
+nulldisp_framebuffer_dirty(struct drm_framebuffer *framebuffer,
+			   struct drm_file *file_priv,
+			   unsigned int flags,
+			   unsigned int color,
+			   struct drm_clip_rect *clips,
+			   unsigned int num_clips)
+{
+	struct nulldisp_display_device *nulldisp_dev =
+		framebuffer->dev->dev_private;
+	struct nulldisp_crtc *nulldisp_crtc = nulldisp_dev->nulldisp_crtc;
+
+	reinit_completion(&nulldisp_crtc->copy_done);
+
+	if (!nlpvrdpy_send_copy(nulldisp_dev->nlpvrdpy,
+				framebuffer,
+				&framebuffer->obj[0])) {
+		unsigned long res;
+
+		res = wait_for_completion_timeout(&nulldisp_crtc->copy_done,
+						  nulldisp_netlink_timeout());
+
+		if (!res)
+			DRM_ERROR("timed out waiting for remote update\n");
+	}
+
+	return 0;
+}
+
+static const struct drm_framebuffer_funcs nulldisp_framebuffer_funcs = {
+	.destroy = drm_gem_fb_destroy,
+	.create_handle = drm_gem_fb_create_handle,
+	.dirty = nulldisp_framebuffer_dirty,
+};
+
+static struct drm_framebuffer *
+nulldisp_fb_create(struct drm_device *dev, struct drm_file *file,
+		   const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	return drm_gem_fb_create_with_funcs(dev, file, mode_cmd,
+					    &nulldisp_framebuffer_funcs);
+}
+#else /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)) */
+#define nulldisp_fb_create drm_gem_fb_create_with_dirty
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)) */
+#else /* defined(NULLDISP_USE_ATOMIC) */
+static void nulldisp_framebuffer_destroy(struct drm_framebuffer *framebuffer)
+{
+	struct nulldisp_framebuffer *nulldisp_framebuffer =
+		to_nulldisp_framebuffer(framebuffer);
+	int i;
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", framebuffer->base.id);
+
+	drm_framebuffer_cleanup(framebuffer);
+
+	for (i = 0; i < nulldisp_drm_fb_num_planes(framebuffer); i++)
+		drm_gem_object_put(nulldisp_framebuffer->obj[i]);
+
+	kfree(nulldisp_framebuffer);
+}
+
+static int
+nulldisp_framebuffer_create_handle(struct drm_framebuffer *framebuffer,
+				   struct drm_file *file_priv,
+				   unsigned int *handle)
+{
+	struct nulldisp_framebuffer *nulldisp_framebuffer =
+		to_nulldisp_framebuffer(framebuffer);
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", framebuffer->base.id);
+
+	return drm_gem_handle_create(file_priv,
+				     nulldisp_framebuffer->obj[0],
+				     handle);
+}
+
+static int
+nulldisp_framebuffer_dirty(struct drm_framebuffer *framebuffer,
+			   struct drm_file *file_priv,
+			   unsigned int flags,
+			   unsigned int color,
+			   struct drm_clip_rect *clips,
+			   unsigned int num_clips)
+{
+	struct nulldisp_framebuffer *nulldisp_fb =
+		to_nulldisp_framebuffer(framebuffer);
+	struct nulldisp_display_device *nulldisp_dev =
+		framebuffer->dev->dev_private;
+	struct nulldisp_crtc *nulldisp_crtc = nulldisp_dev->nulldisp_crtc;
+
+	/*
+	 * To prevent races with disconnect requests from user space,
+	 * set the timeout before sending the copy request.
+	 */
+	nulldisp_set_copy_to(nulldisp_crtc);
+
+	if (nlpvrdpy_send_copy(nulldisp_dev->nlpvrdpy,
+			       &nulldisp_fb->base,
+			       &nulldisp_fb->obj[0]))
+		goto fail_flush;
+
+	wait_for_completion(&nulldisp_crtc->copy_done);
+
+	return 0;
+
+fail_flush:
+	flush_delayed_work(&nulldisp_crtc->copy_to_work);
+
+	wait_for_completion(&nulldisp_crtc->copy_done);
+
+	return 0;
+
+}
+
+static const struct drm_framebuffer_funcs nulldisp_framebuffer_funcs = {
+	.destroy = nulldisp_framebuffer_destroy,
+	.create_handle = nulldisp_framebuffer_create_handle,
+	.dirty = nulldisp_framebuffer_dirty,
+};
+
+static int
+nulldisp_framebuffer_init(struct drm_device *dev,
+			  const struct drm_mode_fb_cmd2 *mode_cmd,
+			  struct nulldisp_framebuffer *nulldisp_framebuffer,
+			  struct drm_gem_object **obj)
+{
+	struct drm_framebuffer *fb = &nulldisp_framebuffer->base;
+	int err;
+	int i;
+
+	fb->dev = dev;
+
+	nulldisp_drm_fb_set_format(fb, mode_cmd->pixel_format);
+
+	fb->width        = mode_cmd->width;
+	fb->height       = mode_cmd->height;
+	fb->flags        = mode_cmd->flags;
+
+	nulldisp_drm_fb_set_modifier(fb, mode_cmd->modifier[0]);
+
+	for (i = 0; i < nulldisp_drm_fb_num_planes(fb); i++) {
+		fb->pitches[i]  = mode_cmd->pitches[i];
+		fb->offsets[i]  = mode_cmd->offsets[i];
+
+		nulldisp_framebuffer->obj[i] = obj[i];
+	}
+
+	err = drm_framebuffer_init(dev, fb, &nulldisp_framebuffer_funcs);
+	if (err) {
+		DRM_ERROR("failed to initialise framebuffer structure (%d)\n",
+			  err);
+		return err;
+	}
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", fb->base.id);
+
+	return 0;
+}
+
+static struct drm_framebuffer *
+nulldisp_fb_create(struct drm_device *dev,
+		   struct drm_file *file_priv,
+		   const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_gem_object *obj[NULLDISP_MAX_PLANES];
+	struct nulldisp_framebuffer *nulldisp_framebuffer;
+	int err;
+	int i;
+
+	nulldisp_framebuffer = kzalloc(sizeof(*nulldisp_framebuffer),
+				       GFP_KERNEL);
+	if (!nulldisp_framebuffer) {
+		err = -ENOMEM;
+		goto fail_exit;
+	}
+
+	for (i = 0; i < drm_format_num_planes(mode_cmd->pixel_format); i++) {
+		obj[i] = drm_gem_object_lookup(file_priv, mode_cmd->handles[i]);
+		if (!obj[i]) {
+			DRM_ERROR("failed to find buffer with handle %u\n",
+				  mode_cmd->handles[i]);
+			err = -ENOENT;
+			goto fail_unreference;
+		}
+	}
+
+	err = nulldisp_framebuffer_init(dev,
+					mode_cmd,
+					nulldisp_framebuffer,
+					obj);
+	if (err)
+		goto fail_unreference;
+
+	DRM_DEBUG_DRIVER("[FB:%d]\n", nulldisp_framebuffer->base.base.id);
+
+	return &nulldisp_framebuffer->base;
+
+fail_unreference:
+	kfree(nulldisp_framebuffer);
+
+	while (i--)
+		drm_gem_object_put(obj[i]);
+
+fail_exit:
+	return ERR_PTR(err);
+}
+#endif	/* defined(NULLDISP_USE_ATOMIC) */
+
+static const struct drm_mode_config_funcs nulldisp_mode_config_funcs = {
+	.fb_create = nulldisp_fb_create,
+	//.output_poll_changed = NULL,
+#if defined(NULLDISP_USE_ATOMIC)
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+#endif
+};
+
+static int nulldisp_nl_flipped_cb(void *data)
+{
+	struct nulldisp_crtc *nulldisp_crtc = data;
+
+#if defined(NULLDISP_USE_ATOMIC)
+	complete(&nulldisp_crtc->flip_done);
+#else
+	flush_delayed_work(&nulldisp_crtc->flip_to_work);
+#endif
+	flush_delayed_work(&nulldisp_crtc->vb_work);
+
+	return 0;
+}
+
+static int nulldisp_nl_copied_cb(void *data)
+{
+#if defined(NULLDISP_USE_ATOMIC)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0))
+	struct nulldisp_crtc *nulldisp_crtc = data;
+
+	complete(&nulldisp_crtc->copy_done);
+#endif
+#else
+	struct nulldisp_crtc *nulldisp_crtc = data;
+
+	flush_delayed_work(&nulldisp_crtc->copy_to_work);
+#endif
+	return 0;
+}
+
+static void nulldisp_nl_disconnect_cb(void *data)
+{
+	struct nulldisp_crtc *nulldisp_crtc = data;
+
+#if defined(NULLDISP_USE_ATOMIC)
+	complete(&nulldisp_crtc->flip_done);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0))
+	complete(&nulldisp_crtc->copy_done);
+#endif
+#else
+	flush_delayed_work(&nulldisp_crtc->flip_to_work);
+	flush_delayed_work(&nulldisp_crtc->copy_to_work);
+#endif
+}
+
+#if defined(LMA)
+struct pdp_gem_private *pdp_gem_get_private(struct drm_device *dev)
+{
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	return nulldisp_dev->pdp_gem_priv;
+}
+#endif
+
+static int nulldisp_early_load(struct drm_device *dev, unsigned int instance)
+{
+	struct nulldisp_display_device *nulldisp_dev;
+	struct drm_connector *connector;
+	struct drm_encoder *encoder;
+	int err;
+
+	platform_set_drvdata(to_platform_device(dev->dev), dev);
+
+	nulldisp_dev = kzalloc(sizeof(*nulldisp_dev), GFP_KERNEL);
+	if (!nulldisp_dev)
+		return -ENOMEM;
+
+	dev->dev_private = nulldisp_dev;
+	nulldisp_dev->dev = dev;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_init(&nulldisp_dev->struct_mutex);
+#endif
+	drm_mode_config_init(dev);
+
+	dev->mode_config.funcs = (void *)&nulldisp_mode_config_funcs;
+	dev->mode_config.min_width = NULLDISP_FB_WIDTH_MIN;
+	dev->mode_config.max_width = NULLDISP_FB_WIDTH_MAX;
+	dev->mode_config.min_height = NULLDISP_FB_HEIGHT_MIN;
+	dev->mode_config.max_height = NULLDISP_FB_HEIGHT_MAX;
+	dev->mode_config.async_page_flip = true;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+	dev->mode_config.allow_fb_modifiers = true;
+#endif
+
+	nulldisp_dev->nulldisp_crtc = nulldisp_crtc_create(nulldisp_dev);
+	if (!nulldisp_dev->nulldisp_crtc) {
+		DRM_ERROR("failed to create a CRTC.\n");
+
+		err = -ENOMEM;
+		goto err_config_cleanup;
+	}
+
+	connector = nulldisp_connector_create(nulldisp_dev,
+					      DRM_MODE_CONNECTOR_Unknown);
+	if (!connector) {
+		DRM_ERROR("failed to create a connector.\n");
+
+		err = -ENOMEM;
+		goto err_config_cleanup;
+	}
+	encoder = nulldisp_encoder_create(nulldisp_dev,
+					  DRM_MODE_ENCODER_NONE);
+	if (IS_ERR(encoder)) {
+		DRM_ERROR("failed to create an encoder.\n");
+
+		err = PTR_ERR(encoder);
+		goto err_config_cleanup;
+	}
+
+	err = drm_connector_attach_encoder(connector, encoder);
+	if (err) {
+		DRM_ERROR("failed to attach [ENCODER:%d:%s] to [CONNECTOR:%d:%s] (err=%d)\n",
+			  encoder->base.id,
+			  encoder->name,
+			  connector->base.id,
+			  connector->name,
+			  err);
+		goto err_config_cleanup;
+	}
+
+#if defined(LMA)
+	nulldisp_dev->pdp_gem_priv = pdp_gem_init(dev, instance);
+	if (!nulldisp_dev->pdp_gem_priv) {
+		err = -ENOMEM;
+		goto err_config_cleanup;
+	}
+#endif
+	nulldisp_dev->workqueue =
+		create_singlethread_workqueue(DRIVER_NAME);
+	if (!nulldisp_dev->workqueue) {
+		DRM_ERROR("failed to create work queue\n");
+		goto err_gem_cleanup;
+	}
+
+	err = drm_vblank_init(nulldisp_dev->dev, 1);
+	if (err) {
+		DRM_ERROR("failed to complete vblank init (err=%d)\n", err);
+		goto err_workqueue_cleanup;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0))
+	dev->irq_enabled = true;
+#endif
+
+	nulldisp_dev->nlpvrdpy = nlpvrdpy_create(dev,
+						 nulldisp_nl_disconnect_cb,
+						 nulldisp_dev->nulldisp_crtc,
+						 nulldisp_nl_flipped_cb,
+						 nulldisp_dev->nulldisp_crtc,
+						 nulldisp_nl_copied_cb,
+						 nulldisp_dev->nulldisp_crtc);
+	if (!nulldisp_dev->nlpvrdpy) {
+		DRM_ERROR("Netlink initialisation failed (err=%d)\n", err);
+		goto err_vblank_cleanup;
+	}
+
+	return 0;
+
+err_vblank_cleanup:
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	/* Called by drm_dev_fini in Linux 4.11.0 and later */
+	drm_vblank_cleanup(dev);
+#endif
+err_workqueue_cleanup:
+	destroy_workqueue(nulldisp_dev->workqueue);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0))
+	dev->irq_enabled = false;
+#endif
+err_gem_cleanup:
+#if defined(LMA)
+	pdp_gem_cleanup(nulldisp_dev->pdp_gem_priv);
+#endif
+err_config_cleanup:
+	drm_mode_config_cleanup(dev);
+	kfree(nulldisp_dev);
+	return err;
+}
+
+static int nulldisp_late_load(struct drm_device *dev)
+{
+	drm_mode_config_reset(dev);
+
+	return 0;
+}
+
+static void nulldisp_early_unload(struct drm_device *dev)
+{
+}
+
+static void nulldisp_late_unload(struct drm_device *dev)
+{
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	nlpvrdpy_send_disconnect(nulldisp_dev->nlpvrdpy);
+	nlpvrdpy_destroy(nulldisp_dev->nlpvrdpy);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	/* Called by drm_dev_fini in Linux 4.11.0 and later */
+	drm_vblank_cleanup(dev);
+#endif
+	destroy_workqueue(nulldisp_dev->workqueue);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0))
+	dev->irq_enabled = false;
+#endif
+
+#if defined(LMA)
+	pdp_gem_cleanup(nulldisp_dev->pdp_gem_priv);
+#endif
+	drm_mode_config_cleanup(dev);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_destroy(&nulldisp_dev->struct_mutex);
+#endif
+	kfree(nulldisp_dev);
+}
+
+
+
+static void nulldisp_lastclose(struct drm_device *dev)
+{
+#if defined(NULLDISP_USE_ATOMIC)
+	drm_atomic_helper_shutdown(dev);
+#else
+	struct drm_crtc *crtc;
+
+	drm_modeset_lock_all(dev);
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		if (crtc->primary->fb) {
+			struct drm_mode_set mode_set = { .crtc = crtc };
+			int err;
+
+			err = drm_mode_set_config_internal(&mode_set);
+			if (err)
+				DRM_ERROR(
+				    "failed to disable crtc %p (err=%d)\n",
+				    crtc, err);
+		}
+	}
+	drm_modeset_unlock_all(dev);
+#endif
+}
+
+#if defined(LMA)
+static const struct vm_operations_struct nulldisp_gem_vm_ops = {
+	.fault	= pdp_gem_object_vm_fault,
+	.open	= drm_gem_vm_open,
+	.close	= drm_gem_vm_close,
+};
+
+static int pdp_gem_dumb_create(struct drm_file *file,
+			       struct drm_device *dev,
+			       struct drm_mode_create_dumb *args)
+{
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	return pdp_gem_dumb_create_priv(file,
+					dev,
+					nulldisp_dev->pdp_gem_priv,
+					args);
+}
+
+static int nulldisp_gem_object_create_ioctl(struct drm_device *dev,
+					    void *data,
+					    struct drm_file *file)
+{
+	struct drm_nulldisp_gem_create *args = data;
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+	struct drm_pdp_gem_create pdp_args;
+	int err;
+
+	if (args->flags) {
+		DRM_ERROR("invalid flags: %#08x\n", args->flags);
+		return -EINVAL;
+	}
+
+	if (args->handle) {
+		DRM_ERROR("invalid handle (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Remapping of nulldisp create args to pdp create args.
+	 *
+	 * Note: even though the nulldisp and pdp args are identical
+	 * in this case, they may potentially change in future.
+	 */
+	pdp_args.size = args->size;
+	pdp_args.flags = args->flags;
+	pdp_args.handle = args->handle;
+
+	err = pdp_gem_object_create_ioctl_priv(dev,
+					       nulldisp_dev->pdp_gem_priv,
+					       &pdp_args,
+					       file);
+	if (!err)
+		args->handle = pdp_args.handle;
+
+	return err;
+}
+
+static int nulldisp_gem_object_mmap_ioctl(struct drm_device *dev,
+					  void *data,
+					  struct drm_file *file)
+{
+	struct drm_nulldisp_gem_mmap *args = data;
+	struct drm_pdp_gem_mmap pdp_args;
+	int err;
+
+	pdp_args.handle = args->handle;
+	pdp_args.pad = args->pad;
+	pdp_args.offset = args->offset;
+
+	err = pdp_gem_object_mmap_ioctl(dev, &pdp_args, file);
+
+	if (!err)
+		args->offset = pdp_args.offset;
+
+	return err;
+}
+
+static int nulldisp_gem_object_cpu_prep_ioctl(struct drm_device *dev,
+					      void *data,
+					      struct drm_file *file)
+{
+	struct drm_nulldisp_gem_cpu_prep *args =
+		(struct drm_nulldisp_gem_cpu_prep *)data;
+	struct drm_pdp_gem_cpu_prep pdp_args;
+
+	pdp_args.handle = args->handle;
+	pdp_args.flags = args->flags;
+
+	return pdp_gem_object_cpu_prep_ioctl(dev, &pdp_args, file);
+}
+
+static int nulldisp_gem_object_cpu_fini_ioctl(struct drm_device *dev,
+				       void *data,
+				       struct drm_file *file)
+{
+	struct drm_nulldisp_gem_cpu_fini *args =
+		(struct drm_nulldisp_gem_cpu_fini *)data;
+	struct drm_pdp_gem_cpu_fini pdp_args;
+
+	pdp_args.handle = args->handle;
+	pdp_args.pad = args->pad;
+
+	return pdp_gem_object_cpu_fini_ioctl(dev, &pdp_args, file);
+}
+
+void pdp_gem_object_free(struct drm_gem_object *obj)
+{
+	struct nulldisp_display_device *nulldisp_dev = obj->dev->dev_private;
+
+	pdp_gem_object_free_priv(nulldisp_dev->pdp_gem_priv, obj);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+const struct drm_gem_object_funcs nulldisp_gem_funcs = {
+	.free = pdp_gem_object_free,
+	.export = pdp_gem_prime_export,
+	.vm_ops = &nulldisp_gem_vm_ops,
+};
+#endif
+#endif /* defined(LMA) */
+
+static const struct drm_ioctl_desc nulldisp_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(NULLDISP_GEM_CREATE,
+			  nulldisp_gem_object_create_ioctl,
+			  DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NULLDISP_GEM_MMAP,
+			  nulldisp_gem_object_mmap_ioctl,
+			  DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NULLDISP_GEM_CPU_PREP,
+			  nulldisp_gem_object_cpu_prep_ioctl,
+			  DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NULLDISP_GEM_CPU_FINI,
+			  nulldisp_gem_object_cpu_fini_ioctl,
+			  DRM_AUTH),
+};
+
+static int nulldisp_gem_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	int err = netlink_gem_mmap(file, vma);
+#if !defined(LMA)
+	if (!err) {
+		struct drm_file *file_priv = file->private_data;
+		struct drm_device *dev = file_priv->minor->dev;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+#endif
+		struct drm_gem_object *obj;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		mutex_lock(&nulldisp_dev->struct_mutex);
+#else
+		mutex_lock(&dev->struct_mutex);
+#endif
+		obj = vma->vm_private_data;
+
+		if (obj->import_attach)
+			err = dma_buf_mmap(obj->dma_buf, vma, 0);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+		else
+			err = nulldisp_gem_object_get_pages(obj);
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+		mutex_unlock(&dev->struct_mutex);
+#endif
+	}
+#endif
+	return err;
+}
+
+static const struct file_operations nulldisp_driver_fops = {
+	.owner		= THIS_MODULE,
+	.open		= drm_open,
+	.release	= drm_release,
+	.unlocked_ioctl	= drm_ioctl,
+	.mmap		= nulldisp_gem_mmap,
+	.poll		= drm_poll,
+	.read		= drm_read,
+	.llseek		= noop_llseek,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= drm_compat_ioctl,
+#endif
+	.fop_flags              = FOP_UNSIGNED_OFFSET,
+};
+
+static struct drm_driver nulldisp_drm_driver = {
+	.load				= NULL,
+	.unload				= nulldisp_lastclose,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+	.get_vblank_counter		= drm_vblank_no_hw_counter,
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 7, 0))
+	.enable_vblank			= nulldisp_enable_vblank,
+	.disable_vblank			= nulldisp_disable_vblank,
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	.prime_handle_to_fd		= drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle		= drm_gem_prime_fd_to_handle,
+#endif
+
+#if defined(LMA)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_free_object		= pdp_gem_object_free,
+	.gem_prime_export		= pdp_gem_prime_export,
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) */
+	.gem_prime_import		= pdp_gem_prime_import,
+	.gem_prime_import_sg_table	= pdp_gem_prime_import_sg_table,
+
+	.dumb_create			= pdp_gem_dumb_create,
+	.dumb_map_offset		= pdp_gem_dumb_map_offset,
+#else
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_prime_pin			= nulldisp_gem_prime_pin,
+	.gem_prime_unpin		= nulldisp_gem_prime_unpin,
+	.gem_prime_get_sg_table = nulldisp_gem_prime_get_sg_table,
+	.gem_prime_vmap			= nulldisp_gem_prime_vmap,
+	.gem_prime_vunmap		= nulldisp_gem_prime_vunmap,
+	.gem_free_object		= nulldisp_gem_object_free,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+	.gem_prime_export		= nulldisp_gem_prime_export,
+#else
+	.gem_prime_export		= drm_gem_prime_export,
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) */
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) */
+	.gem_prime_import_sg_table	= nulldisp_gem_prime_import_sg_table,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_prime_mmap			= nulldisp_gem_prime_mmap,
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+	.gem_prime_mmap			= drm_gem_prime_mmap,
+#endif
+	.gem_prime_import		= drm_gem_prime_import,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+	.gem_prime_res_obj		= nulldisp_gem_prime_res_obj,
+#endif
+	.dumb_create			= nulldisp_gem_dumb_create,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	.dumb_map_offset		= nulldisp_gem_dumb_map_offset,
+#endif
+#endif /* defined(LMA) */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	.dumb_destroy			= drm_gem_dumb_destroy,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+	.gem_vm_ops			= &nulldisp_gem_vm_ops,
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) */
+	.name				= DRIVER_NAME,
+	.desc				= DRIVER_DESC,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0))
+	.date				= DRIVER_DATE,
+#endif
+	.major				= PVRVERSION_MAJ,
+	.minor				= PVRVERSION_MIN,
+	.patchlevel			= PVRVERSION_BUILD,
+
+	.driver_features		= DRIVER_GEM |
+					  DRIVER_MODESET |
+					  NULLDISP_DRIVER_PRIME |
+					  NULLDISP_DRIVER_ATOMIC,
+	.ioctls				= nulldisp_ioctls,
+	.num_ioctls			= ARRAY_SIZE(nulldisp_ioctls),
+	.fops				= &nulldisp_driver_fops,
+};
+
+static int nulldisp_probe(struct platform_device *pdev)
+{
+	unsigned int instance = 0;
+	struct drm_device *ddev;
+	int ret;
+
+	ddev = drm_dev_alloc(&nulldisp_drm_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	/*
+	 * The load callback, called from drm_dev_register, is deprecated,
+	 * because of potential race conditions.
+	 */
+	BUG_ON(nulldisp_drm_driver.load != NULL);
+
+#if defined(CONFIG_OF)
+	/* The device is not spawned from a device tree: arch_setup_dma_ops
+	 * was not called and the device has dummy DMA ops. Call of_dma_configure()
+	 * to setup proper dma ops.
+	 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0))
+	of_dma_configure(ddev->dev, NULL);
+#else
+	of_dma_configure(ddev->dev, NULL, true);
+#endif
+#endif
+
+	ret = nulldisp_early_load(ddev, instance);
+	if (ret)
+		goto err_drm_dev_put;
+
+	ret = drm_dev_register(ddev, 0);
+	if (ret)
+		goto err_drm_dev_late_unload;
+
+	ret = nulldisp_late_load(ddev);
+	if (ret)
+		goto err_drm_dev_unregister;
+
+	drm_mode_config_reset(ddev);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+	DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
+		nulldisp_drm_driver.name,
+		nulldisp_drm_driver.major,
+		nulldisp_drm_driver.minor,
+		nulldisp_drm_driver.patchlevel,
+		nulldisp_drm_driver.date,
+		ddev->primary->index);
+#endif
+	return 0;
+
+err_drm_dev_unregister:
+	drm_dev_unregister(ddev);
+err_drm_dev_late_unload:
+	nulldisp_late_unload(ddev);
+err_drm_dev_put:
+	drm_dev_put(ddev);
+	return	ret;
+}
+
+static void nulldisp_remove(struct platform_device *pdev)
+{
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+
+	/*
+	 * The unload callback, called from drm_dev_unregister, is
+	 * deprecated.
+	 */
+	BUG_ON(nulldisp_drm_driver.unload != NULL);
+
+	nulldisp_early_unload(ddev);
+
+	drm_dev_unregister(ddev);
+
+	nulldisp_late_unload(ddev);
+
+	drm_dev_put(ddev);
+
+	return;
+}
+
+static void nulldisp_shutdown(struct platform_device *pdev)
+{
+}
+
+static struct platform_device_id nulldisp_platform_device_id_table[] = {
+#if defined(LMA) && !defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+	{ .name = APOLLO_DEVICE_NAME_PDP, .driver_data = 0 },
+	{ .name = ODN_DEVICE_NAME_PDP, .driver_data = 0 },
+#else
+	{ .name = "nulldisp", .driver_data = 0 },
+#endif
+	{ },
+};
+
+static struct platform_driver nulldisp_platform_driver = {
+	.probe		= nulldisp_probe,
+	.remove		= nulldisp_remove,
+	.shutdown	= nulldisp_shutdown,
+	.driver		= {
+		.owner  = THIS_MODULE,
+		.name	= DRIVER_NAME,
+	},
+	.id_table	= nulldisp_platform_device_id_table,
+};
+
+
+#if !defined(LMA) || defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+static struct platform_device_info nulldisp_device_info = {
+	.name		= "nulldisp",
+	.id		= -1,
+#if defined(LMA)
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wshift-count-overflow"
+#endif /* defined(__clang__) */
+	/* No restriction, the GPU can access all on-card memory */
+	.dma_mask	= DMA_BIT_MASK(64),
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif /* defined(__clang__) */
+#elif defined(NULLDISP_PHYS_BUS_WIDTH)
+	.dma_mask	= DMA_BIT_MASK(NULLDISP_PHYS_BUS_WIDTH),
+#elif defined(NO_HARDWARE)
+	/*
+	 * Not all cores have 40 bit physical support, but this
+	 * will work unless > 32 bit address is returned on those cores.
+	 * In the future this will be fixed properly.
+	 */
+	.dma_mask	= DMA_BIT_MASK(40),
+#else
+	.dma_mask	= DMA_BIT_MASK(32),
+#endif
+};
+
+static struct platform_device *nulldisp_dev;
+#endif
+
+static int __init nulldisp_init(void)
+{
+	int err;
+
+	err = nulldisp_validate_module_parameters();
+	if (err) {
+		DRM_ERROR("invalid module parameters (err=%d)\n", err);
+		return err;
+	}
+
+	err = nlpvrdpy_register();
+	if (err) {
+		DRM_ERROR("failed to register with netlink (err=%d)\n", err);
+		return err;
+	}
+
+#if !defined(LMA) || defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+	nulldisp_dev = platform_device_register_full(&nulldisp_device_info);
+	if (IS_ERR(nulldisp_dev)) {
+		err = PTR_ERR(nulldisp_dev);
+		nulldisp_dev = NULL;
+		goto err_unregister_family;
+	}
+#endif
+	err = platform_driver_register(&nulldisp_platform_driver);
+	if (err)
+		goto err_unregister_family;
+
+	return 0;
+
+err_unregister_family:
+		(void) nlpvrdpy_unregister();
+		return err;
+}
+
+static void __exit nulldisp_exit(void)
+{
+	int err;
+
+	err = nlpvrdpy_unregister();
+	BUG_ON(err);
+
+#if !defined(LMA) || defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+	if (nulldisp_dev)
+		platform_device_unregister(nulldisp_dev);
+#endif
+	platform_driver_unregister(&nulldisp_platform_driver);
+}
+
+module_init(nulldisp_init);
+module_exit(nulldisp_exit);
+
+#if defined(LMA) && !defined(SUPPORT_EXTERNAL_PHYSHEAP_INTERFACE)
+/*
+ * For Test Chip, this module relies on a memory heap created in another
+ * module. There is no explicit dependency on the other module, as the heap
+ * information is passed in device data. This lack of a dependency would
+ * allow the other module to be unloaded whilst the heap is still in use,
+ * resulting in the PDP platform device being removed whilst there are
+ * resources still associated with it. Avoid the problem by creating a
+ * dependency on the other module, by referencing a symbol in it.
+ */
+static int (*nulldisp_parent_mod_ref)(struct device *) __used = tc_enable;
+#endif
+
+MODULE_AUTHOR("Imagination Technologies Ltd. <gpl-support@imgtec.com>");
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/nulldisp/drm_nulldisp_drv.h b/drivers/gpu/drm/nulldisp/drm_nulldisp_drv.h
new file mode 100644
index 000000000000..f132294e65a6
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_nulldisp_drv.h
@@ -0,0 +1,118 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __DRM_NULLDISP_DRV_H__
+#define __DRM_NULLDISP_DRV_H__
+
+#include <linux/version.h>
+#include <linux/mutex.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+#include <drm/drm_fourcc.h>
+#endif
+
+struct nulldisp_display_device {
+	struct drm_device *dev;
+
+	struct workqueue_struct *workqueue;
+	struct nulldisp_crtc *nulldisp_crtc;
+	struct nlpvrdpy *nlpvrdpy;
+#if defined(LMA)
+	struct pdp_gem_private *pdp_gem_priv;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+	struct drm_connector *connector;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	/*
+	 * Currently, struct_mutex is only used by the nulldisp driver as a replacement
+	 * for BKL.
+	 *
+	 * For this reason, it is no longer part of struct drm_device.
+	 */
+	struct mutex struct_mutex;
+#endif
+};
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+#define NULLDISP_USE_ATOMIC
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0))
+#include <drm/drm_framebuffer.h>
+#else
+struct drm_framebuffer;
+#endif
+
+/******************************************************************************
+ * Linux compatibility functions
+ ******************************************************************************/
+static inline u32 nulldisp_drm_fb_format(struct drm_framebuffer *fb)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	return fb->format->format;
+#else	/* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) */
+	return fb->pixel_format;
+#endif	/* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) */
+}
+
+static inline u64 nulldisp_drm_fb_modifier(struct drm_framebuffer *fb)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	return fb->modifier;
+#else
+	return fb->modifier[0];
+#endif
+}
+
+/******************************************************************************
+ * DRM framebuffer support functions
+ ******************************************************************************/
+static inline int nulldisp_drm_fb_num_planes(struct drm_framebuffer *fb)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	return fb->format->num_planes;
+#else
+	return drm_format_num_planes(nulldisp_drm_fb_format(fb));
+#endif
+}
+#endif /* __DRM_NULLDISP_DRV_H__ */
diff --git a/drivers/gpu/drm/nulldisp/drm_nulldisp_gem.c b/drivers/gpu/drm/nulldisp/drm_nulldisp_gem.c
new file mode 100644
index 000000000000..336d253f3ed4
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_nulldisp_gem.c
@@ -0,0 +1,789 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/atomic.h>
+#include <linux/mm_types.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/pagemap.h>
+#include <linux/mutex.h>
+#include <linux/capability.h>
+
+#include "pvr_dma_resv.h"
+#include "drm_nulldisp_gem.h"
+#include "nulldisp_drm.h"
+#include "kernel_compatibility.h"
+#include "drm_nulldisp_drv.h"
+
+struct nulldisp_gem_object {
+	struct drm_gem_object base;
+
+	atomic_t pg_refcnt;
+	struct page **pages;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	struct dma_resv _resv;
+#endif
+	struct dma_resv *resv;
+
+	bool cpu_prep;
+	struct sg_table *import_sgt;
+};
+
+#define to_nulldisp_obj(obj) \
+	container_of(obj, struct nulldisp_gem_object, base)
+
+int nulldisp_gem_object_get_pages(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+#endif
+	struct nulldisp_gem_object *nulldisp_obj = to_nulldisp_obj(obj);
+
+	if (WARN_ON(obj->import_attach))
+		return -EEXIST;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	WARN_ON(!mutex_is_locked(&nulldisp_dev->struct_mutex));
+#else
+	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+#endif
+
+	if (atomic_inc_return(&nulldisp_obj->pg_refcnt) == 1) {
+		struct page **pages = drm_gem_get_pages(obj);
+		if (IS_ERR(pages)) {
+			atomic_dec(&nulldisp_obj->pg_refcnt);
+			return PTR_ERR(pages);
+		}
+
+		nulldisp_obj->pages = pages;
+	}
+
+	return 0;
+}
+
+static void nulldisp_gem_object_put_pages(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+#endif
+	struct nulldisp_gem_object *nulldisp_obj = to_nulldisp_obj(obj);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	WARN_ON(!mutex_is_locked(&nulldisp_dev->struct_mutex));
+#else
+	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+#endif
+
+	if (WARN_ON(atomic_read(&nulldisp_obj->pg_refcnt) == 0))
+		return;
+
+	if (atomic_dec_and_test(&nulldisp_obj->pg_refcnt)) {
+		drm_gem_put_pages(obj, nulldisp_obj->pages, true, true);
+		nulldisp_obj->pages = NULL;
+	}
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0))
+typedef int vm_fault_t;
+#endif
+static vm_fault_t nulldisp_gem_object_vm_fault(struct vm_fault *vmf)
+#else
+static int nulldisp_gem_object_vm_fault(struct vm_area_struct *vma,
+				 struct vm_fault *vmf)
+#endif
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	struct vm_area_struct *vma = vmf->vma;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0))
+	unsigned long addr = vmf->address;
+#else
+	unsigned long addr = (unsigned long)vmf->virtual_address;
+#endif
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct nulldisp_gem_object *nulldisp_obj = to_nulldisp_obj(obj);
+	unsigned long pg_off;
+	struct page *page;
+
+	/*
+	 * nulldisp_gem_object_get_pages should have been called in
+	 * nulldisp_gem_mmap so there's no need to do it here.
+	 */
+	if (WARN_ON(atomic_read(&nulldisp_obj->pg_refcnt) == 0))
+		return VM_FAULT_SIGBUS;
+
+	pg_off = (addr - vma->vm_start) >> PAGE_SHIFT;
+	page = nulldisp_obj->pages[pg_off];
+
+	get_page(page);
+	vmf->page = page;
+
+	return 0;
+}
+
+static void nulldisp_gem_vm_open(struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj = vma->vm_private_data;
+
+	drm_gem_vm_open(vma);
+
+	if (!obj->import_attach) {
+		struct drm_device *dev = obj->dev;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+		mutex_lock(&nulldisp_dev->struct_mutex);
+		(void)nulldisp_gem_object_get_pages(obj);
+		mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+		mutex_lock(&dev->struct_mutex);
+		(void) nulldisp_gem_object_get_pages(obj);
+		mutex_unlock(&dev->struct_mutex);
+#endif
+	}
+}
+
+static void nulldisp_gem_vm_close(struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj = vma->vm_private_data;
+
+	if (!obj->import_attach) {
+		struct drm_device *dev = obj->dev;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+		struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+		mutex_lock(&nulldisp_dev->struct_mutex);
+		(void)nulldisp_gem_object_put_pages(obj);
+		mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+		mutex_lock(&dev->struct_mutex);
+		(void) nulldisp_gem_object_put_pages(obj);
+		mutex_unlock(&dev->struct_mutex);
+#endif
+	}
+
+	drm_gem_vm_close(vma);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static
+#endif
+void nulldisp_gem_object_free(struct drm_gem_object *obj)
+{
+	struct nulldisp_gem_object *nulldisp_obj = to_nulldisp_obj(obj);
+
+	WARN_ON(atomic_read(&nulldisp_obj->pg_refcnt) != 0);
+
+	if (obj->import_attach) {
+		kfree(nulldisp_obj->pages);
+		drm_gem_free_mmap_offset(obj);
+		drm_prime_gem_destroy(obj, nulldisp_obj->import_sgt);
+	} else {
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+		dma_resv_fini(&nulldisp_obj->_resv);
+#endif
+		drm_gem_object_release(obj);
+	}
+
+	kfree(nulldisp_obj);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static
+#endif
+int nulldisp_gem_prime_pin(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+	int err;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	mutex_lock(&nulldisp_dev->struct_mutex);
+	err = nulldisp_gem_object_get_pages(obj);
+	mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_lock(&dev->struct_mutex);
+	err = nulldisp_gem_object_get_pages(obj);
+	mutex_unlock(&dev->struct_mutex);
+#endif
+
+	return err;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static
+#endif
+void nulldisp_gem_prime_unpin(struct drm_gem_object *obj)
+{
+	struct drm_device *dev = obj->dev;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	mutex_lock(&nulldisp_dev->struct_mutex);
+	nulldisp_gem_object_put_pages(obj);
+	mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_lock(&dev->struct_mutex);
+	nulldisp_gem_object_put_pages(obj);
+	mutex_unlock(&dev->struct_mutex);
+#endif
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static
+#endif
+struct sg_table *
+nulldisp_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct nulldisp_gem_object *nulldisp_obj = to_nulldisp_obj(obj);
+	int nr_pages = obj->size >> PAGE_SHIFT;
+
+	/*
+	 * nulldisp_gem_prime_pin should have been called in which case we don't
+	 * need to call nulldisp_gem_object_get_pages.
+	 */
+	if (WARN_ON(atomic_read(&nulldisp_obj->pg_refcnt) == 0))
+		return NULL;
+
+	return drm_prime_pages_to_sg(obj->dev, nulldisp_obj->pages, nr_pages);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+struct dma_buf *nulldisp_gem_prime_export(
+					  struct drm_device *dev,
+					  struct drm_gem_object *obj,
+					  int flags)
+{
+	return drm_gem_prime_export(dev, obj, flags);
+}
+#endif
+
+static void *nulldisp_gem_vmap(struct drm_gem_object *obj)
+{
+	struct nulldisp_gem_object *nulldisp_obj = to_nulldisp_obj(obj);
+	int nr_pages = obj->size >> PAGE_SHIFT;
+
+	/*
+	 * nulldisp_gem_prime_pin should have been called in which case we don't
+	 * need to call nulldisp_gem_object_get_pages.
+	 */
+	if (WARN_ON(atomic_read(&nulldisp_obj->pg_refcnt) == 0))
+		return NULL;
+
+
+	return vmap(nulldisp_obj->pages, nr_pages, VM_MAP, PAGE_KERNEL);
+}
+
+static void nulldisp_gem_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+	vunmap(vaddr);
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static
+#endif
+void *nulldisp_gem_prime_vmap(struct drm_gem_object *obj)
+{
+	return nulldisp_gem_vmap(obj);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static
+#endif
+void nulldisp_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+	nulldisp_gem_vunmap(obj, vaddr);
+}
+#else
+static
+int nulldisp_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
+{
+	void *vaddr = nulldisp_gem_vmap(obj);
+
+	iosys_map_set_vaddr_iomem(map, vaddr);
+	return (vaddr == NULL) ? -ENOMEM : 0;
+}
+
+static
+void nulldisp_gem_prime_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
+{
+	nulldisp_gem_vunmap(obj, map->vaddr);
+	iosys_map_clear(map);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static
+#endif
+const struct vm_operations_struct nulldisp_gem_vm_ops = {
+	.fault	= nulldisp_gem_object_vm_fault,
+	.open	= nulldisp_gem_vm_open,
+	.close	= nulldisp_gem_vm_close,
+};
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+int nulldisp_gem_prime_mmap(struct drm_gem_object *obj,
+			    struct vm_area_struct *vma)
+{
+	int err;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct drm_device *dev = obj->dev;
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	mutex_lock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_lock(&obj->dev->struct_mutex);
+#endif
+
+	err = nulldisp_gem_object_get_pages(obj);
+	if (!err)
+		err = drm_gem_mmap_obj(obj, obj->size, vma);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_unlock(&obj->dev->struct_mutex);
+
+#endif
+	return err;
+}
+#else
+static int nulldisp_gem_obj_mmap(struct drm_gem_object *obj,
+				 struct vm_area_struct *vma)
+{
+	int err;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct drm_device *dev = obj->dev;
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	mutex_lock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_lock(&obj->dev->struct_mutex);
+#endif
+	err = nulldisp_gem_object_get_pages(obj);
+	/* Required in documentation for drm_gem_object_funcs::mmap */
+	vma->vm_ops = &nulldisp_gem_vm_ops;
+	/* This is what used to be done in drm_gem_mmap_obj() before */
+	pvr_vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
+	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_unlock(&obj->dev->struct_mutex);
+#endif
+
+	return err;
+}
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+static const struct drm_gem_object_funcs nulldisp_gem_funcs = {
+	.export = drm_gem_prime_export,
+	.pin = nulldisp_gem_prime_pin,
+	.unpin = nulldisp_gem_prime_unpin,
+	.get_sg_table = nulldisp_gem_prime_get_sg_table,
+	.vmap = nulldisp_gem_prime_vmap,
+	.vunmap = nulldisp_gem_prime_vunmap,
+	.free = nulldisp_gem_object_free,
+	.mmap = nulldisp_gem_obj_mmap,
+};
+#endif
+
+struct drm_gem_object *
+nulldisp_gem_prime_import_sg_table(struct drm_device *dev,
+				   struct dma_buf_attachment *attach,
+				   struct sg_table *sgt)
+{
+	struct nulldisp_gem_object *nulldisp_obj;
+	struct drm_gem_object *obj;
+	struct page **pages;
+	unsigned int npages;
+
+	nulldisp_obj = kzalloc(sizeof(*nulldisp_obj), GFP_KERNEL);
+	if (!nulldisp_obj)
+		return NULL;
+
+	nulldisp_obj->resv = attach->dmabuf->resv;
+
+	obj = &nulldisp_obj->base;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0))
+	obj->resv = nulldisp_obj->resv;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	obj->funcs = &nulldisp_gem_funcs;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0) */
+
+	drm_gem_private_object_init(dev, obj, attach->dmabuf->size);
+
+	npages = obj->size >> PAGE_SHIFT;
+
+	pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
+	if (!pages)
+		goto exit_free_arrays;
+
+	if (drm_prime_sg_to_page_array(sgt, pages, npages))
+		goto exit_free_arrays;
+
+	nulldisp_obj->import_sgt = sgt;
+	nulldisp_obj->pages = pages;
+
+	return obj;
+
+exit_free_arrays:
+	kfree(pages);
+	drm_prime_gem_destroy(obj, sgt);
+	kfree(nulldisp_obj);
+	return NULL;
+}
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+struct dma_resv *
+nulldisp_gem_prime_res_obj(struct drm_gem_object *obj)
+{
+	struct nulldisp_gem_object *nulldisp_obj = to_nulldisp_obj(obj);
+
+	return nulldisp_obj->resv;
+}
+#endif
+
+int nulldisp_gem_object_mmap_ioctl(struct drm_device *dev, void *data,
+				   struct drm_file *file)
+{
+	struct drm_nulldisp_gem_mmap *args =
+		(struct drm_nulldisp_gem_mmap *)data;
+
+	if (args->pad) {
+		DRM_ERROR("invalid pad (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	if (args->offset) {
+		DRM_ERROR("invalid offset (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	return nulldisp_gem_dumb_map_offset(file, dev, args->handle,
+					    &args->offset);
+}
+
+int nulldisp_gem_object_cpu_prep_ioctl(struct drm_device *dev, void *data,
+				       struct drm_file *file)
+{
+	struct drm_nulldisp_gem_cpu_prep *args =
+		(struct drm_nulldisp_gem_cpu_prep *)data;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+#endif
+
+	struct drm_gem_object *obj;
+	struct nulldisp_gem_object *nulldisp_obj;
+	bool write = !!(args->flags & NULLDISP_GEM_CPU_PREP_WRITE);
+	bool wait = !(args->flags & NULLDISP_GEM_CPU_PREP_NOWAIT);
+	int err;
+
+	if (args->flags & ~(NULLDISP_GEM_CPU_PREP_READ |
+			    NULLDISP_GEM_CPU_PREP_WRITE |
+			    NULLDISP_GEM_CPU_PREP_NOWAIT)) {
+		DRM_ERROR("invalid flags: %#08x\n", args->flags);
+		return -EINVAL;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_lock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_lock(&dev->struct_mutex);
+#endif
+
+	obj = drm_gem_object_lookup(file, args->handle);
+	if (!obj) {
+		err = -ENOENT;
+		goto exit_unlock;
+	}
+
+	nulldisp_obj = to_nulldisp_obj(obj);
+
+	if (nulldisp_obj->cpu_prep) {
+		err = -EBUSY;
+		goto exit_unref;
+	}
+
+	if (wait) {
+		long lerr;
+
+		lerr = dma_resv_wait_timeout(nulldisp_obj->resv,
+						 write,
+						 true,
+						 30 * HZ);
+
+		/* Remap return value (0 indicates busy state, > 0 success) */
+		if (lerr > 0)
+			err = 0;
+		else if (!lerr)
+			err = -EBUSY;
+		else
+			err = lerr;
+	} else {
+		/*
+		 * Remap return value (false indicates busy state,
+		 * true success).
+		 */
+		if (!dma_resv_test_signaled(nulldisp_obj->resv,
+						write))
+			err = -EBUSY;
+		else
+			err = 0;
+	}
+
+	if (!err)
+		nulldisp_obj->cpu_prep = true;
+exit_unref:
+	drm_gem_object_put(obj);
+exit_unlock:
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_unlock(&dev->struct_mutex);
+#endif
+	return err;
+}
+
+int nulldisp_gem_object_cpu_fini_ioctl(struct drm_device *dev, void *data,
+				       struct drm_file *file)
+{
+	struct drm_nulldisp_gem_cpu_fini *args =
+		(struct drm_nulldisp_gem_cpu_fini *)data;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+#endif
+	struct drm_gem_object *obj;
+	struct nulldisp_gem_object *nulldisp_obj;
+	int err;
+
+	if (args->pad) {
+		DRM_ERROR("invalid pad (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_lock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_lock(&dev->struct_mutex);
+#endif
+
+	obj = drm_gem_object_lookup(file, args->handle);
+	if (!obj) {
+		err = -ENOENT;
+		goto exit_unlock;
+	}
+
+	nulldisp_obj = to_nulldisp_obj(obj);
+
+	if (!nulldisp_obj->cpu_prep) {
+		err = -EINVAL;
+		goto exit_unref;
+	}
+
+	nulldisp_obj->cpu_prep = false;
+	err = 0;
+exit_unref:
+	drm_gem_object_put(obj);
+exit_unlock:
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_unlock(&dev->struct_mutex);
+#endif
+	return err;
+}
+
+static int nulldisp_gem_object_create_priv(struct drm_file *file,
+					   struct drm_device *dev,
+					   u64 size,
+					   u32 *handle)
+{
+	struct nulldisp_gem_object *nulldisp_obj;
+	struct drm_gem_object *obj;
+	struct address_space *mapping;
+	int err;
+
+	nulldisp_obj = kzalloc(sizeof(*nulldisp_obj), GFP_KERNEL);
+	if (!nulldisp_obj)
+		return -ENOMEM;
+
+	obj = &nulldisp_obj->base;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	obj->funcs = &nulldisp_gem_funcs;
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0) */
+
+	err = drm_gem_object_init(dev, obj, size);
+	if (err) {
+		kfree(nulldisp_obj);
+		return err;
+	}
+
+	mapping = file_inode(obj->filp)->i_mapping;
+	mapping_set_gfp_mask(mapping, GFP_USER |
+//#if !defined(NULLDISP_PHYS_BUS_WIDTH) || NULLDISP_PHYS_BUS_WIDTH <= 32
+//				      __GFP_DMA32 |
+//#endif
+				      __GFP_NORETRY);
+
+	err = drm_gem_handle_create(file, obj, handle);
+	if (err)
+		goto exit;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	dma_resv_init(&nulldisp_obj->_resv);
+	nulldisp_obj->resv = &nulldisp_obj->_resv;
+#else
+	nulldisp_obj->resv = nulldisp_obj->base.resv;
+#endif
+
+exit:
+	drm_gem_object_put(obj);
+	return err;
+}
+
+int nulldisp_gem_object_create_ioctl(struct drm_device *dev,
+				     void *data,
+				     struct drm_file *file)
+{
+	struct drm_nulldisp_gem_create *args = data;
+	u32 handle;
+	int err;
+	u64 aligned_size;
+
+	if (args->flags) {
+		DRM_ERROR("invalid flags: %#08x\n", args->flags);
+		return -EINVAL;
+	}
+
+	if (args->handle) {
+		DRM_ERROR("invalid handle (this should always be 0)\n");
+		return -EINVAL;
+	}
+
+	aligned_size = PAGE_ALIGN(args->size);
+
+	err = nulldisp_gem_object_create_priv(file, dev, aligned_size, &handle);
+	if (!err)
+		args->handle = handle;
+
+	return err;
+}
+
+int nulldisp_gem_dumb_create(struct drm_file *file,
+			     struct drm_device *dev,
+			     struct drm_mode_create_dumb *args)
+{
+	u32 handle;
+	u32 pitch;
+	size_t size;
+	int err;
+
+	pitch = args->width * (ALIGN(args->bpp, 8) >> 3);
+	size = PAGE_ALIGN(pitch * args->height);
+
+	err = nulldisp_gem_object_create_priv(file, dev, size, &handle);
+	if (!err) {
+		args->handle = handle;
+		args->pitch = pitch;
+		args->size = size;
+	}
+
+	return err;
+}
+
+int nulldisp_gem_dumb_map_offset(struct drm_file *file,
+				 struct drm_device *dev,
+				 uint32_t handle,
+				 uint64_t *offset)
+{
+	struct drm_gem_object *obj;
+	int err;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	struct nulldisp_display_device *nulldisp_dev = dev->dev_private;
+
+	mutex_lock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_lock(&dev->struct_mutex);
+#endif
+
+	obj = drm_gem_object_lookup(file, handle);
+	if (!obj) {
+		err = -ENOENT;
+		goto exit_unlock;
+	}
+
+	err = drm_gem_create_mmap_offset(obj);
+	if (err)
+		goto exit_obj_unref;
+
+	*offset = drm_vma_node_offset_addr(&obj->vma_node);
+
+exit_obj_unref:
+	drm_gem_object_put(obj);
+exit_unlock:
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0))
+	mutex_unlock(&nulldisp_dev->struct_mutex);
+#else
+	mutex_unlock(&dev->struct_mutex);
+#endif
+	return err;
+}
+
+struct dma_resv *nulldisp_gem_get_resv(struct drm_gem_object *obj)
+{
+	return (to_nulldisp_obj(obj)->resv);
+}
diff --git a/drivers/gpu/drm/nulldisp/drm_nulldisp_gem.h b/drivers/gpu/drm/nulldisp/drm_nulldisp_gem.h
new file mode 100644
index 000000000000..77835187e4ea
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_nulldisp_gem.h
@@ -0,0 +1,129 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__DRM_NULLDISP_H__)
+#define __DRM_NULLDISP_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_device.h>
+#include <drm/drm_file.h>
+#include <linux/types.h>
+#include <linux/vmalloc.h>
+
+struct dma_buf_attachment;
+struct vm_area_struct;
+struct vm_fault;
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_gem.h>
+
+#include "kernel_compatibility.h"
+
+int nulldisp_gem_object_get_pages(struct drm_gem_object *obj);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+extern const struct vm_operations_struct nulldisp_gem_vm_ops;
+int nulldisp_gem_prime_mmap(struct drm_gem_object *obj,
+			    struct vm_area_struct *vma);
+
+void nulldisp_gem_object_free(struct drm_gem_object *obj);
+
+int nulldisp_gem_prime_pin(struct drm_gem_object *obj);
+
+void nulldisp_gem_prime_unpin(struct drm_gem_object *obj);
+
+struct sg_table *nulldisp_gem_prime_get_sg_table(struct drm_gem_object *obj);
+
+void *nulldisp_gem_prime_vmap(struct drm_gem_object *obj);
+
+void nulldisp_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+#endif
+
+struct drm_gem_object *
+nulldisp_gem_prime_import_sg_table(struct drm_device *dev,
+				   struct dma_buf_attachment *attach,
+				   struct sg_table *sgt);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+struct dma_buf *nulldisp_gem_prime_export(struct drm_device *dev,
+					  struct drm_gem_object *obj,
+					  int flags);
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+struct dma_resv *
+nulldisp_gem_prime_res_obj(struct drm_gem_object *obj);
+#endif
+
+int nulldisp_gem_dumb_create(struct drm_file *file,
+			     struct drm_device *dev,
+			     struct drm_mode_create_dumb *args);
+
+int nulldisp_gem_dumb_map_offset(struct drm_file *file,
+				 struct drm_device *dev,
+				 uint32_t handle,
+				 uint64_t *offset);
+
+/* internal interfaces */
+struct dma_resv *nulldisp_gem_get_resv(struct drm_gem_object *obj);
+
+int nulldisp_gem_object_mmap_ioctl(struct drm_device *dev,
+				   void *data,
+				   struct drm_file *file);
+
+int nulldisp_gem_object_cpu_prep_ioctl(struct drm_device *dev,
+				       void *data,
+				       struct drm_file *file);
+
+int nulldisp_gem_object_cpu_fini_ioctl(struct drm_device *dev,
+				       void *data,
+				       struct drm_file *file);
+
+int nulldisp_gem_object_create_ioctl(struct drm_device *dev,
+				     void *data,
+				     struct drm_file *file);
+
+#endif	/* !defined(__DRM_NULLDISP_H__) */
diff --git a/drivers/gpu/drm/nulldisp/drm_nulldisp_netlink.c b/drivers/gpu/drm/nulldisp/drm_nulldisp_netlink.c
new file mode 100644
index 000000000000..e4bb42e9b2ea
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_nulldisp_netlink.c
@@ -0,0 +1,691 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0))
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_vma_manager.h>
+#else
+#include <drm/drmP.h>
+#endif
+
+#include <drm/drm_crtc.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/types.h>
+#include <net/netlink.h>
+#include <net/genetlink.h>
+#include <linux/bug.h>
+
+#include "drm_netlink_gem.h"
+#include "drm_nulldisp_drv.h"
+#include "drm_nulldisp_netlink.h"
+#include "kernel_compatibility.h"
+
+#include "netlink.h"
+
+struct nlpvrdpy {
+	atomic_t connected;
+	struct net *net;
+	u32 dst_portid;
+	struct drm_device *dev;
+	nlpvrdpy_disconnect_cb disconnect_cb;
+	void *disconnect_cb_data;
+	nlpvrdpy_flipped_cb flipped_cb;
+	void *flipped_cb_data;
+	nlpvrdpy_copied_cb copied_cb;
+	void *copied_cb_data;
+	struct mutex mutex;
+	struct list_head nl_list;
+	bool gem_names_required;
+};
+#define NLPVRDPY_MINOR(nlpvrdpy) \
+	((unsigned int)((nlpvrdpy)->dev->primary->index))
+
+/* Command internal flags */
+#define	NLPVRDPY_CIF_NLPVRDPY_NOT_CONNECTED	0x00000001
+#define	NLPVRDPY_CIF_NLPVRDPY			0x00000002
+
+static LIST_HEAD(nlpvrdpy_list);
+static DEFINE_MUTEX(nlpvrdpy_list_mutex);
+
+static inline void nlpvrdpy_lock(struct nlpvrdpy *nlpvrdpy)
+{
+	mutex_lock(&nlpvrdpy->mutex);
+}
+
+static inline void nlpvrdpy_unlock(struct nlpvrdpy *nlpvrdpy)
+{
+	mutex_unlock(&nlpvrdpy->mutex);
+}
+
+struct nlpvrdpy *nlpvrdpy_create(struct drm_device *dev,
+					nlpvrdpy_disconnect_cb disconnect_cb,
+					void *disconnect_cb_data,
+					nlpvrdpy_flipped_cb flipped_cb,
+					void *flipped_cb_data,
+					nlpvrdpy_copied_cb copied_cb,
+					void *copied_cb_data)
+{
+	struct nlpvrdpy *nlpvrdpy = kzalloc(sizeof(*nlpvrdpy), GFP_KERNEL);
+
+	if (!nlpvrdpy)
+		return NULL;
+
+	mutex_init(&nlpvrdpy->mutex);
+	INIT_LIST_HEAD(&nlpvrdpy->nl_list);
+
+	atomic_set(&nlpvrdpy->connected, 0);
+
+	nlpvrdpy->dev = dev;
+	nlpvrdpy->disconnect_cb = disconnect_cb;
+	nlpvrdpy->disconnect_cb_data = disconnect_cb_data;
+	nlpvrdpy->flipped_cb = flipped_cb;
+	nlpvrdpy->flipped_cb_data = flipped_cb_data;
+	nlpvrdpy->copied_cb = copied_cb;
+	nlpvrdpy->copied_cb_data = copied_cb_data;
+
+	mutex_lock(&nlpvrdpy_list_mutex);
+	list_add_tail(&nlpvrdpy->nl_list, &nlpvrdpy_list);
+	mutex_unlock(&nlpvrdpy_list_mutex);
+
+	return nlpvrdpy;
+}
+
+void nlpvrdpy_destroy(struct nlpvrdpy *nlpvrdpy)
+{
+	if (!nlpvrdpy)
+		return;
+
+	mutex_lock(&nlpvrdpy_list_mutex);
+	nlpvrdpy_lock(nlpvrdpy);
+	list_del(&nlpvrdpy->nl_list);
+	nlpvrdpy_unlock(nlpvrdpy);
+	mutex_unlock(&nlpvrdpy_list_mutex);
+
+	mutex_destroy(&nlpvrdpy->mutex);
+
+	kfree(nlpvrdpy);
+}
+
+static struct nlpvrdpy *nlpvrdpy_lookup(u32 minor)
+{
+	struct nlpvrdpy *nlpvrdpy = NULL;
+	struct nlpvrdpy *iter;
+
+	mutex_lock(&nlpvrdpy_list_mutex);
+	list_for_each_entry(iter, &nlpvrdpy_list, nl_list) {
+		if (NLPVRDPY_MINOR(iter) == minor) {
+			nlpvrdpy = iter;
+			nlpvrdpy_lock(nlpvrdpy);
+			break;
+		}
+	}
+	mutex_unlock(&nlpvrdpy_list_mutex);
+
+	return nlpvrdpy;
+}
+
+static int nlpvrdpy_pre_cmd(const struct genl_split_ops *ops,
+				struct sk_buff *skb,
+				struct genl_info *info)
+{
+	struct nlattr **attrs = info->attrs;
+	struct nlpvrdpy *nlpvrdpy = NULL;
+	int ret;
+
+	if (ops->internal_flags & NLPVRDPY_CIF_NLPVRDPY_NOT_CONNECTED) {
+		if (!(ops->flags & GENL_ADMIN_PERM))
+			return -EINVAL;
+	}
+
+	if (ops->internal_flags & (NLPVRDPY_CIF_NLPVRDPY_NOT_CONNECTED |
+					NLPVRDPY_CIF_NLPVRDPY)) {
+		u32 minor;
+
+		if (!attrs[NLPVRDPY_ATTR_MINOR])
+			return -EINVAL;
+
+		minor = nla_get_u32(attrs[NLPVRDPY_ATTR_MINOR]);
+
+		nlpvrdpy = nlpvrdpy_lookup(minor);
+		if (!nlpvrdpy)
+			return -ENODEV;
+
+		if (ops->internal_flags & NLPVRDPY_CIF_NLPVRDPY) {
+			if (!atomic_read(&nlpvrdpy->connected)) {
+				ret = -ENOTCONN;
+				goto err_unlock;
+			}
+			if ((nlpvrdpy->net != genl_info_net(info)) ||
+				(nlpvrdpy->dst_portid != info->snd_portid)) {
+				ret = -EPROTO;
+				goto err_unlock;
+			}
+		}
+
+		info->user_ptr[0] = nlpvrdpy;
+	}
+
+	ret = 0;
+
+err_unlock:
+	nlpvrdpy_unlock(nlpvrdpy);
+	return ret;
+}
+
+static struct genl_family nlpvrdpy_family = {
+	.name = "nlpvrdpy",
+	.version = 1,
+	.maxattr = NLPVRDPY_ATTR_MAX,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0))
+	.policy = nlpvrdpy_policy,
+#endif
+	.pre_doit = &nlpvrdpy_pre_cmd,
+};
+
+/* Must be called with the struct nlpvrdpy mutex held */
+static int nlpvrdpy_send_msg_locked(struct nlpvrdpy *nlpvrdpy,
+					struct sk_buff *msg)
+{
+	int err;
+
+	if (atomic_read(&nlpvrdpy->connected)) {
+		err = genlmsg_unicast(nlpvrdpy->net, msg, nlpvrdpy->dst_portid);
+		if (err == -ECONNREFUSED)
+			atomic_set(&nlpvrdpy->connected, 0);
+	} else {
+		err = -ENOTCONN;
+		nlmsg_free(msg);
+	}
+
+	return err;
+}
+
+static int nlpvrdpy_send_msg(struct nlpvrdpy *nlpvrdpy, struct sk_buff *msg)
+{
+	int err;
+
+	nlpvrdpy_lock(nlpvrdpy);
+	err = nlpvrdpy_send_msg_locked(nlpvrdpy, msg);
+	nlpvrdpy_unlock(nlpvrdpy);
+
+	return err;
+}
+
+void nlpvrdpy_send_disconnect(struct nlpvrdpy *nlpvrdpy)
+{
+	struct sk_buff *msg;
+	void *hdr;
+	int err;
+
+	if (!atomic_read(&nlpvrdpy->connected))
+		return;
+
+	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+	if (!msg)
+		return;
+
+	hdr = genlmsg_put(msg, nlpvrdpy->dst_portid, 0,
+				&nlpvrdpy_family, 0, NLPVRDPY_CMD_DISCONNECT);
+	if (!hdr)
+		goto err_msg_free;
+
+	err = nla_put_u32(msg, NLPVRDPY_ATTR_MINOR, NLPVRDPY_MINOR(nlpvrdpy));
+	if (err)
+		goto err_msg_free;
+
+	genlmsg_end(msg, hdr);
+
+	nlpvrdpy_lock(nlpvrdpy);
+
+	(void) nlpvrdpy_send_msg_locked(nlpvrdpy, msg);
+
+	atomic_set(&nlpvrdpy->connected, 0);
+	nlpvrdpy->net = NULL;
+	nlpvrdpy->dst_portid = 0;
+
+	nlpvrdpy_unlock(nlpvrdpy);
+
+	return;
+
+err_msg_free:
+	nlmsg_free(msg);
+}
+
+static int nlpvrdpy_get_offsets_and_sizes(struct drm_framebuffer *fb,
+					  struct drm_gem_object **objs,
+					  u64 *addr, u64 *size)
+{
+	int i;
+
+	for (i = 0; i < nulldisp_drm_fb_num_planes(fb); i++) {
+		int err;
+		struct drm_gem_object *obj = objs[i];
+
+		err = drm_gem_create_mmap_offset(obj);
+		if (err) {
+			DRM_ERROR(
+			    "Failed to get mmap offset for buffer[%d] = %p\n",
+			    i, obj);
+			return err;
+		}
+
+		addr[i] = drm_vma_node_offset_addr(&obj->vma_node);
+		size[i] = obj->size;
+	}
+
+	return 0;
+}
+
+static int nlpvrdpy_put_fb_attributes(struct sk_buff *msg,
+				      struct drm_framebuffer *fb,
+				      struct nlpvrdpy *nlpvrdpy,
+				      struct drm_gem_object **objs)
+{
+	int i, err;
+	const int num_planes = nulldisp_drm_fb_num_planes(fb);
+	u64 plane_addr[NLPVRDPY_MAX_NUM_PLANES],
+	    plane_size[NLPVRDPY_MAX_NUM_PLANES];
+
+	err = nlpvrdpy_get_offsets_and_sizes(fb, objs, &plane_addr[0], &plane_size[0]);
+	if (err) {
+		pr_err("%s: nlpvrdpy_get_offsets_and_sizes failed", __func__);
+		return err;
+	}
+
+	err = nla_put_u32(msg, NLPVRDPY_ATTR_MINOR, NLPVRDPY_MINOR(nlpvrdpy));
+	if (err) {
+		pr_err("%s: nla_put_u32 NLPVRDPY_ATTR_MINOR failed", __func__);
+		return err;
+	}
+
+	err = nla_put_u8(msg, NLPVRDPY_ATTR_NUM_PLANES, num_planes);
+	if (err) {
+		pr_err("%s: nla_put_u8 NLPVRDPY_ATTR_NUM_PLANES failed", __func__);
+		return err;
+	}
+
+	err = nla_put_u32(msg, NLPVRDPY_ATTR_WIDTH, fb->width);
+	if (err) {
+		pr_err("%s: nla_put_u32 NLPVRDPY_ATTR_WIDTH failed", __func__);
+		return err;
+	}
+
+	err = nla_put_u32(msg, NLPVRDPY_ATTR_HEIGHT, fb->height);
+	if (err) {
+		pr_err("%s: nla_put_u32 NLPVRDPY_ATTR_HEIGHT failed", __func__);
+		return err;
+	}
+
+	err = nla_put_u32(msg, NLPVRDPY_ATTR_PIXFMT, nulldisp_drm_fb_format(fb));
+	if (err) {
+		pr_err("%s: nla_put_u32 NLPVRDPY_ATTR_PIXFMT failed",
+		       __func__);
+		return err;
+	}
+
+	err = nla_put_u64_64bit(msg, NLPVRDPY_ATTR_FB_MODIFIER,
+				nulldisp_drm_fb_modifier(fb), NLPVRDPY_ATTR_PAD);
+	if (err) {
+		pr_err("%s: nla_put_u64_64bit NLPVRDPY_ATTR_FB_MODIFIER "
+		       "NLPVRDPY_ATTR_PAD failed", __func__);
+		return err;
+	}
+
+	/* IMG_COLORSPACE_BT601_CONFORMANT_RANGE */
+	err = nla_put_u8(msg, NLPVRDPY_ATTR_YUV_CSC, 1);
+	if (err) {
+		pr_err("%s: nla_put_u8 NLPVRDPY_ATTR_YUV_CSC 1 failed", __func__);
+		return err;
+	}
+
+	/* 8-bit per sample */
+	err = nla_put_u8(msg, NLPVRDPY_ATTR_YUV_BPP, 8);
+	if (err) {
+		pr_err("%s: nla_put_u8 NLPVRDPY_ATTR_YUV_BPP 8 failed", __func__);
+		return err;
+	}
+
+	for (i = 0; i < num_planes; i++) {
+		err = nla_put_u64_64bit(msg, NLPVRDPY_ATTR_PLANE(i, ADDR),
+					plane_addr[i], NLPVRDPY_ATTR_PAD);
+		if (err) {
+			pr_err("%s: nla_put_u64_64bit NLPVRDPY_ATTR_PLANE(%d, ADDR)"
+			       " NLPVRDPY_ATTR_PAD failed", __func__, i);
+			return err;
+		}
+
+		err = nla_put_u64_64bit(msg, NLPVRDPY_ATTR_PLANE(i, SIZE),
+					plane_size[i], NLPVRDPY_ATTR_PAD);
+		if (err) {
+			pr_err("%s: nla_put_u64_64bit NLPVRDPY_ATTR_PLANE(%d, SIZE)"
+			       " NLPVRDPY_ATTR_PAD failed", __func__, i);
+			return err;
+		}
+
+		err = nla_put_u64_64bit(msg, NLPVRDPY_ATTR_PLANE(i, OFFSET),
+					fb->offsets[i], NLPVRDPY_ATTR_PAD);
+		if (err) {
+			pr_err("%s: nla_put_u64_64bit NLPVRDPY_ATTR_PLANE(%d, OFFSET)"
+			       " NLPVRDPY_ATTR_PAD failed", __func__, i);
+			return err;
+		}
+
+		err = nla_put_u64_64bit(msg, NLPVRDPY_ATTR_PLANE(i, PITCH),
+					fb->pitches[i], NLPVRDPY_ATTR_PAD);
+		if (err) {
+			pr_err("%s: nla_put_u64_64bit NLPVRDPY_ATTR_PLANE(%d, PITCH)"
+			       " NLPVRDPY_ATTR_PAD failed", __func__, i);
+			return err;
+		}
+
+		err = nla_put_u32(msg, NLPVRDPY_ATTR_PLANE(i, GEM_OBJ_NAME), (u32)objs[0]->name);
+		if (err) {
+			pr_err("%s: nla_put_u32 NLPVRDPY_ATTR_PLANE(%d, GEM_OBJ_NAME)"
+			       " failed", __func__, i);
+			return err;
+		}
+	}
+
+	WARN_ONCE(num_planes > NLPVRDPY_MAX_NUM_PLANES,
+		  "NLPVRDPY_MAX_NUM_PLANES = [%d], num_planes = [%d]\n",
+		  NLPVRDPY_MAX_NUM_PLANES, num_planes);
+
+	return 0;
+}
+
+static int nlpvrdpy_name_gem_obj(struct drm_device *dev,
+				 struct drm_gem_object *obj)
+{
+	int ret;
+
+	mutex_lock(&dev->object_name_lock);
+	if (!obj->name) {
+		ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);
+		if (ret < 0)
+			goto exit_unlock;
+
+		obj->name = ret;
+	}
+
+	ret = 0;
+
+exit_unlock:
+	mutex_unlock(&dev->object_name_lock);
+	return ret;
+}
+
+static int nlpvrdpy_name_gem_objs(struct drm_framebuffer *fb,
+				  struct drm_gem_object **objs)
+{
+	int i;
+	struct drm_device *dev = fb->dev;
+
+	for (i = 0; i < nulldisp_drm_fb_num_planes(fb); i++) {
+		int err = nlpvrdpy_name_gem_obj(dev, objs[i]);
+
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+int nlpvrdpy_send_flip(struct nlpvrdpy *nlpvrdpy,
+		       struct drm_framebuffer *fb,
+		       struct drm_gem_object **objs)
+{
+	struct sk_buff *msg;
+	void *hdr;
+	int err;
+
+	if (!atomic_read(&nlpvrdpy->connected))
+		return -ENOTCONN;
+
+	if (nlpvrdpy->gem_names_required) {
+		err = nlpvrdpy_name_gem_objs(fb, objs);
+		if (err)
+			return err;
+	}
+
+	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_put(msg, nlpvrdpy->dst_portid, 0,
+				&nlpvrdpy_family, 0, NLPVRDPY_CMD_FLIP);
+	if (!hdr) {
+		err = -ENOMEM;
+		goto err_msg_free;
+	}
+
+	err = nlpvrdpy_put_fb_attributes(msg, fb, nlpvrdpy, objs);
+	if (err)
+		goto err_msg_free;
+
+	genlmsg_end(msg, hdr);
+
+	return nlpvrdpy_send_msg(nlpvrdpy, msg);
+
+err_msg_free:
+	nlmsg_free(msg);
+	return err;
+}
+
+int nlpvrdpy_send_copy(struct nlpvrdpy *nlpvrdpy,
+		       struct drm_framebuffer *fb,
+		       struct drm_gem_object **objs)
+{
+	struct sk_buff *msg;
+	void *hdr;
+	int err;
+
+	if (!atomic_read(&nlpvrdpy->connected))
+		return -ENOTCONN;
+
+	if (nlpvrdpy->gem_names_required) {
+		err = nlpvrdpy_name_gem_objs(fb, objs);
+		if (err)
+			return err;
+	}
+
+	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_put(msg, nlpvrdpy->dst_portid, 0,
+				&nlpvrdpy_family, 0, NLPVRDPY_CMD_COPY);
+	if (!hdr) {
+		err = -ENOMEM;
+		goto err_msg_free;
+	}
+
+	err = nlpvrdpy_put_fb_attributes(msg, fb, nlpvrdpy, objs);
+	if (err)
+		goto err_msg_free;
+
+	genlmsg_end(msg, hdr);
+
+	return nlpvrdpy_send_msg(nlpvrdpy, msg);
+
+err_msg_free:
+	nlmsg_free(msg);
+	return err;
+}
+
+static int nlpvrdpy_cmd_connect(struct sk_buff *skb, struct genl_info *info)
+{
+	struct nlpvrdpy *nlpvrdpy = info->user_ptr[0];
+	struct sk_buff *msg;
+	void *hdr;
+	int err;
+
+	if (info->attrs[NLPVRDPY_ATTR_NAMING_REQUIRED])
+		nlpvrdpy->gem_names_required = true;
+
+	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_put_reply(msg, info, &nlpvrdpy_family,
+						0, NLPVRDPY_CMD_CONNECTED);
+	if (!hdr) {
+		err = -ENOMEM;
+		goto err_msg_free;
+	}
+
+	err = nla_put_string(msg, NLPVRDPY_ATTR_NAME,
+				nlpvrdpy->dev->driver->name);
+	if (err)
+		goto err_msg_free;
+
+	genlmsg_end(msg, hdr);
+
+	err = genlmsg_reply(msg, info);
+
+	if (!err) {
+		nlpvrdpy_lock(nlpvrdpy);
+
+		nlpvrdpy->net = genl_info_net(info);
+		nlpvrdpy->dst_portid = info->snd_portid;
+		atomic_set(&nlpvrdpy->connected, 1);
+
+		nlpvrdpy_unlock(nlpvrdpy);
+	}
+
+	return err;
+
+err_msg_free:
+	nlmsg_free(msg);
+	return err;
+}
+
+static int nlpvrdpy_cmd_disconnect(struct sk_buff *skb, struct genl_info *info)
+{
+	struct nlpvrdpy *nlpvrdpy = info->user_ptr[0];
+
+	atomic_set(&nlpvrdpy->connected, 0);
+
+	if (nlpvrdpy->disconnect_cb)
+		nlpvrdpy->disconnect_cb(nlpvrdpy->disconnect_cb_data);
+
+	return 0;
+}
+
+static int nlpvrdpy_cmd_flipped(struct sk_buff *skb, struct genl_info *info)
+{
+	struct nlpvrdpy *nlpvrdpy = info->user_ptr[0];
+
+	return (nlpvrdpy->flipped_cb) ?
+			nlpvrdpy->flipped_cb(nlpvrdpy->flipped_cb_data) :
+			0;
+}
+
+static int nlpvrdpy_cmd_copied(struct sk_buff *skb, struct genl_info *info)
+{
+	struct nlpvrdpy *nlpvrdpy = info->user_ptr[0];
+
+	return (nlpvrdpy->copied_cb) ?
+			nlpvrdpy->copied_cb(nlpvrdpy->copied_cb_data) :
+			0;
+}
+
+static struct genl_ops nlpvrdpy_ops[] = {
+	{
+		.cmd = NLPVRDPY_CMD_CONNECT,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+		.policy = nlpvrdpy_policy,
+#endif
+		.doit = nlpvrdpy_cmd_connect,
+		.flags = GENL_ADMIN_PERM,
+		.internal_flags = NLPVRDPY_CIF_NLPVRDPY_NOT_CONNECTED
+	},
+	{
+		.cmd = NLPVRDPY_CMD_DISCONNECT,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+		.policy = nlpvrdpy_policy,
+#endif
+		.doit = nlpvrdpy_cmd_disconnect,
+		.flags = 0,
+		.internal_flags = NLPVRDPY_CIF_NLPVRDPY
+	},
+	{
+		.cmd = NLPVRDPY_CMD_FLIPPED,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+		.policy = nlpvrdpy_policy,
+#endif
+		.doit = nlpvrdpy_cmd_flipped,
+		.flags = 0,
+		.internal_flags = NLPVRDPY_CIF_NLPVRDPY
+	},
+	{
+		.cmd = NLPVRDPY_CMD_COPIED,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+		.policy = nlpvrdpy_policy,
+#endif
+		.doit = nlpvrdpy_cmd_copied,
+		.flags = 0,
+		.internal_flags = NLPVRDPY_CIF_NLPVRDPY
+	}
+};
+
+int nlpvrdpy_register(void)
+{
+	nlpvrdpy_family.module = THIS_MODULE;
+	nlpvrdpy_family.ops = nlpvrdpy_ops;
+	nlpvrdpy_family.n_ops = ARRAY_SIZE(nlpvrdpy_ops);
+
+	return genl_register_family(&nlpvrdpy_family);
+}
+
+int nlpvrdpy_unregister(void)
+{
+	return genl_unregister_family(&nlpvrdpy_family);
+}
diff --git a/drivers/gpu/drm/nulldisp/drm_nulldisp_netlink.h b/drivers/gpu/drm/nulldisp/drm_nulldisp_netlink.h
new file mode 100644
index 000000000000..1cea24085bff
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/drm_nulldisp_netlink.h
@@ -0,0 +1,77 @@
+/*
+ * @File
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __DRM_NULLDISP_NETLINK_H__
+#define __DRM_NULLDISP_NETLINK_H__
+
+#include <linux/types.h>
+#include <linux/version.h>
+
+typedef void (*nlpvrdpy_disconnect_cb)(void *data);
+typedef int (*nlpvrdpy_flipped_cb)(void *data);
+typedef int (*nlpvrdpy_copied_cb)(void *data);
+
+struct nlpvrdpy *nlpvrdpy_create(struct drm_device *dev,
+					nlpvrdpy_disconnect_cb disconnect_cb,
+					void *disconnect_cb_data,
+					nlpvrdpy_flipped_cb flipped_cb,
+					void *flipped_cb_data,
+					nlpvrdpy_copied_cb copied_cb,
+					void *copied_cb_data);
+
+void nlpvrdpy_destroy(struct nlpvrdpy *nlpvrdpy);
+
+int nlpvrdpy_send_flip(struct nlpvrdpy *nlpvrdpy,
+		       struct drm_framebuffer *fb,
+		       struct drm_gem_object **objs);
+
+int nlpvrdpy_send_copy(struct nlpvrdpy *nlpvrdpy,
+		       struct drm_framebuffer *fb,
+		       struct drm_gem_object **objs);
+
+void nlpvrdpy_send_disconnect(struct nlpvrdpy *nlpvrdpy);
+
+int nlpvrdpy_register(void);
+
+int nlpvrdpy_unregister(void);
+
+#endif /* __DRM_NULLDISP_NETLINK_H__ */
diff --git a/drivers/gpu/drm/nulldisp/img_drm_fourcc_internal.h b/drivers/gpu/drm/nulldisp/img_drm_fourcc_internal.h
new file mode 100644
index 000000000000..ee88e90cde1e
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/img_drm_fourcc_internal.h
@@ -0,0 +1,94 @@
+/*************************************************************************/ /*!
+@File
+@Title          Wrapper around drm_fourcc.h
+@Description    FourCCs and the DRM framebuffer modifiers should be added here
+                unless they are used by kernel code or a known user outside of
+                the DDK. If FourCCs or DRM framebuffer modifiers are required
+                outside of the DDK, they shall be moved to the corresponding
+                public header.
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef IMG_DRM_FOURCC_INTERNAL_H
+#define IMG_DRM_FOURCC_INTERNAL_H
+
+#include <powervr/img_drm_fourcc.h>
+
+/*
+ * Modifier names are structured using the following convention,
+ * with underscores (_) between items:
+ * - prefix: DRM_FORMAT_MOD
+ * - identifier for our driver: PVR
+ * - category: FBCDC
+ *   - compression tile dimension: 8x8, 16x4, 32x2
+ *   - FBDC version: V0, V1, V2, V3, V7, V8, V10, V12
+ */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0      fourcc_mod_code(PVR, 1)
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V0_FIX  fourcc_mod_code(PVR, 2) /* Fix for HW_BRN_37464 */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V1 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V2      fourcc_mod_code(PVR, 4)
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V3      fourcc_mod_code(PVR, 5)
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V7 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_8x8_V8      fourcc_mod_code(PVR, 18)
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V10 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V12 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_LOSSY25_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_LOSSY50_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_8x8_LOSSY75_V13 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0     fourcc_mod_code(PVR, 7)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V0_FIX fourcc_mod_code(PVR, 8) /* Fix for HW_BRN_37464 */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V1 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V2     fourcc_mod_code(PVR, 10)
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V3     fourcc_mod_code(PVR, 11)
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V7 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_16x4_V8     fourcc_mod_code(PVR, 19)
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V10 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V12 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_LOSSY25_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_LOSSY50_V13 - moved to the public header */
+/* DRM_FORMAT_MOD_PVR_FBCDC_16x4_LOSSY75_V13 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V1     fourcc_mod_code(PVR, 13)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V3     fourcc_mod_code(PVR, 14)
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V8     fourcc_mod_code(PVR, 20)
+/* DRM_FORMAT_MOD_PVR_FBCDC_32x2_V10 - moved to the public header */
+#define DRM_FORMAT_MOD_PVR_FBCDC_32x2_V12    fourcc_mod_code(PVR, 17)
+
+#endif /* IMG_DRM_FOURCC_INTERNAL_H */
diff --git a/drivers/gpu/drm/nulldisp/kernel_compatibility.h b/drivers/gpu/drm/nulldisp/kernel_compatibility.h
new file mode 100644
index 000000000000..048f4449ae68
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/kernel_compatibility.h
@@ -0,0 +1,404 @@
+/*************************************************************************/ /*!
+@Title          Kernel versions compatibility macros
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __KERNEL_COMPATIBILITY_H__
+#define __KERNEL_COMPATIBILITY_H__
+
+#include <linux/version.h>
+#include <linux/compiler.h>
+
+/* Explicitly error out if DDK is built against out-of-support Linux kernel */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0))
+#error Linux kernels older than 4.9.0 are not supported
+#endif
+
+/*
+ * Stop supporting an old kernel? Remove the top block.
+ * New incompatible kernel?       Append a new block at the bottom.
+ *
+ * Please write your version test as `VERSION < X.Y`, and use the earliest
+ * possible version :)
+ *
+ * If including this header file in other files, this should always be the
+ * last file included, as it can affect definitions/declarations in files
+ * included after it.
+ */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0))
+#define refcount_read(r) atomic_read(r)
+#define drm_mm_insert_node(mm, node, size) drm_mm_insert_node(mm, node, size, 0, DRM_MM_SEARCH_DEFAULT)
+
+#define drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd) drm_helper_mode_fill_fb_struct(fb, mode_cmd)
+
+/*
+ * In Linux Kernels >= 4.12 for x86 another level of page tables has been
+ * added. The added level (p4d) sits between pgd and pud, so when it
+ * doesn`t exist, pud_offset function takes pgd as a parameter instead
+ * of p4d.
+ */
+#define p4d_t pgd_t
+#define p4d_offset(pgd, address) (pgd)
+#define p4d_none(p4d) (0)
+#define p4d_bad(p4d) (0)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+
+#define drm_mode_object_get(obj)          drm_mode_object_reference(obj)
+#define drm_mode_object_put(obj)          drm_mode_object_unreference(obj)
+#define drm_connector_get(obj)            drm_connector_reference(obj)
+#define drm_connector_put(obj)            drm_connector_unreference(obj)
+#define drm_framebuffer_get(obj)          drm_framebuffer_reference(obj)
+#define drm_framebuffer_put(obj)          drm_framebuffer_unreference(obj)
+#define drm_gem_object_get(obj)           drm_gem_object_reference(obj)
+#define drm_gem_object_put_locked(obj)    drm_gem_object_unreference(obj)
+#define __drm_gem_object_put(obj)         __drm_gem_object_unreference(obj)
+#define drm_property_blob_get(obj)        drm_property_reference_blob(obj)
+#define drm_property_blob_put(obj)        drm_property_unreference_blob(obj)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+
+#define drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, format_modifiers, type, name, ...) \
+	({ (void) format_modifiers; drm_universal_plane_init(dev, plane, possible_crtcs, funcs, formats, format_count, type, name, ##__VA_ARGS__); })
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0))
+
+#define drm_dev_put(dev) drm_dev_unref(dev)
+
+#define drm_mode_object_find(dev, file_priv, id, type) drm_mode_object_find(dev, id, type)
+#define drm_encoder_find(dev, file_priv, id) drm_encoder_find(dev, id)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_plane_helper_check_state(plane_state, \
+												 &__clip, \
+												 min_scale, max_scale, \
+												 can_position, \
+												 can_update_disabled); \
+		__ret; \
+	})
+
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0))
+
+#define drm_atomic_helper_check_plane_state(plane_state, crtc_state, \
+											min_scale, max_scale, \
+											can_position, can_update_disabled) \
+	({ \
+		const struct drm_rect __clip = { \
+			.x2 = crtc_state->crtc->mode.hdisplay, \
+			.y2 = crtc_state->crtc->mode.vdisplay, \
+		}; \
+		int __ret = drm_atomic_helper_check_plane_state(plane_state, \
+														crtc_state, \
+														&__clip, \
+														min_scale, max_scale, \
+														can_position, \
+														can_update_disabled); \
+		__ret; \
+	})
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+
+#define drm_connector_attach_encoder(connector, encoder) \
+	drm_mode_connector_attach_encoder(connector, encoder)
+
+#define drm_connector_update_edid_property(connector, edid) \
+	drm_mode_connector_update_edid_property(connector, edid)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) */
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
+
+/*
+ * Work around architectures, e.g. MIPS, that define copy_from_user and
+ * copy_to_user as macros that call access_ok, as this gets redefined below.
+ * As of kernel 4.12, these functions are no longer defined per-architecture
+ * so this work around isn't needed.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#if defined(copy_from_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_from_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+	return copy_from_user(to, from, n);
+}
+
+#undef copy_from_user
+#define copy_from_user(to, from, n) __copy_from_user(to, from, n)
+#endif
+
+#if defined(copy_to_user)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around copy_to_user being defined as a macro that calls access_ok.
+  */
+static inline int
+__pvr_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+	return copy_to_user(to, from, n);
+}
+
+#undef copy_to_user
+#define copy_to_user(to, from, n) __copy_to_user(to, from, n)
+#endif
+#endif
+
+/*
+ * Linux 5.0 dropped the type argument.
+ *
+ * This is unused in at least Linux 3.4 and above for all architectures other
+ * than 'um' (User Mode Linux), which stopped using it in 4.2.
+ */
+#if defined(access_ok)
+ /*
+  * NOTE: This function should not be called directly as it exists simply to
+  * work around access_ok being defined as a macro.
+  */
+static inline int
+__pvr_access_ok_compat(int type, const void __user * addr, unsigned long size)
+{
+	return access_ok(type, addr, size);
+}
+
+#undef access_ok
+#define access_ok(addr, size) __pvr_access_ok_compat(0, addr, size)
+#else
+#define access_ok(addr, size) access_ok(0, addr, size)
+#endif
+
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0))
+#define MODULE_IMPORT_NS(ns)
+#endif
+
+/*
+ * Before v5.8, the "struct mm" has a semaphore named "mmap_sem" which is
+ * renamed to "mmap_lock" in v5.8. Moreover, new APIs are provided to
+ * access this lock starting from v5.8.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+
+#define mmap_write_lock(mm)   down_write(&mm->mmap_sem)
+#define mmap_write_unlock(mm) up_write(&mm->mmap_sem)
+
+#define mmap_read_lock(mm)    down_read(&mm->mmap_sem)
+#define mmap_read_unlock(mm)  up_read(&mm->mmap_sem)
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0))
+#define drm_gem_object_put(obj) drm_gem_object_unreference_unlocked(obj)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
+#define drm_gem_object_put(obj) drm_gem_object_put_unlocked(obj)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+
+#define drm_prime_pages_to_sg(dev, pages, nr_pages) \
+	drm_prime_pages_to_sg(pages, nr_pages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0))
+
+struct dma_buf_map {
+	void *vaddr;
+};
+
+#define dma_buf_vmap(dmabuf, map) \
+	({ \
+		(map)->vaddr = dma_buf_vmap(dmabuf); \
+		(map)->vaddr ? 0 : ((dmabuf) && (dmabuf)->ops->vmap) ? -ENOMEM : -EINVAL; \
+	})
+
+#define dma_buf_vunmap(dmabuf, map) \
+	({ \
+		dma_buf_vunmap(dmabuf, (map)->vaddr); \
+		(map)->vaddr = NULL; \
+	})
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0))
+
+#define drm_prime_sg_to_page_array(sgt, pages, npages) \
+	drm_prime_sg_to_page_addr_arrays(sgt, pages, NULL, npages)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0))
+
+#define drm_gem_plane_helper_prepare_fb drm_gem_fb_prepare_fb
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 13, 0)) */
+
+/*
+ * Linux 5.11 renames the privileged uaccess routines for arm64 and Android
+ * kernel v5.10 merges the change as well. These routines are only used for
+ * arm64 so CONFIG_ARM64 testing can be ignored.
+ */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)) || \
+	((LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) && !defined(ANDROID))
+#define uaccess_enable_privileged() uaccess_enable()
+#define uaccess_disable_privileged() uaccess_disable()
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0))
+#define pde_data PDE_DATA
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+#define kthread_complete_and_exit(comp, ret) complete_and_exit(comp, ret);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0))
+#define iosys_map dma_buf_map
+#define iosys_map_set_vaddr dma_buf_map_set_vaddr
+#define iosys_map_set_vaddr_iomem dma_buf_map_set_vaddr_iomem
+#define iosys_map_clear dma_buf_map_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0))
+
+#define register_shrinker(shrinker, name) \
+	register_shrinker(shrinker)
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 0, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0))
+#define DRM_PLANE_NO_SCALING DRM_PLANE_HELPER_NO_SCALING
+#define drm_plane_helper_destroy drm_primary_helper_destroy
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0))
+#define genl_split_ops genl_ops
+#define COMPAT_FB_INFO fbdev
+#define drm_fb_helper_alloc_info drm_fb_helper_alloc_fbi
+#define drm_fb_helper_unregister_info drm_fb_helper_unregister_fbi
+#else
+#define COMPAT_FB_INFO info
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) || \
+        ((LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) && !defined(ANDROID))
+static inline void pvr_vm_flags_set(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags |= flags;
+}
+static inline void pvr_vm_flags_init(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags = flags;
+}
+static inline void pvr_vm_flags_clear(struct vm_area_struct *vma,
+				vm_flags_t flags)
+{
+	vma->vm_flags &= ~flags;
+}
+#else
+#define pvr_vm_flags_set  vm_flags_set
+#define pvr_vm_flags_init vm_flags_init
+#define pvr_vm_flags_clear vm_flags_clear
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0))
+#define pvr_class_create(name) class_create(THIS_MODULE, name)
+#else
+#define pvr_class_create(name) class_create(name)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0))
+#define thermal_tripless_zone_device_register(type, devdata, ops, tzp) \
+	thermal_zone_device_register((type), 0, 0, (devdata), (ops), (tzp), 0, 0)
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0)) */
+
+#if defined(__GNUC__)
+#define GCC_VERSION_AT_LEAST(major, minor) \
+	(__GNUC__ > (major) || \
+	(__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#else
+#define GCC_VERSION_AT_LEAST(major, minor) 0
+#endif
+
+#if defined(__clang__)
+#define CLANG_VERSION_AT_LEAST(major) \
+	(__clang_major__ >= (major))
+#else
+#define CLANG_VERSION_AT_LEAST(major) 0
+#endif
+
+#if !defined(__fallthrough)
+	#if GCC_VERSION_AT_LEAST(7, 0) || CLANG_VERSION_AT_LEAST(10)
+		#define __fallthrough __attribute__((__fallthrough__))
+	#else
+		#define __fallthrough
+	#endif
+#endif
+
+#endif /* __KERNEL_COMPATIBILITY_H__ */
diff --git a/drivers/gpu/drm/nulldisp/netlink.h b/drivers/gpu/drm/nulldisp/netlink.h
new file mode 100644
index 000000000000..7b0a71fbaab3
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/netlink.h
@@ -0,0 +1,147 @@
+/*
+ * @File
+ * @Title       Nulldisp/Netlink interface definition
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+
+#ifndef __NETLINK_H__
+#define __NETLINK_H__
+
+/* For multi-plane pixel formats */
+#define NLPVRDPY_MAX_NUM_PLANES 3
+
+enum nlpvrdpy_cmd {
+	__NLPVRDPY_CMD_INVALID,
+	NLPVRDPY_CMD_CONNECT,
+	NLPVRDPY_CMD_CONNECTED,
+	NLPVRDPY_CMD_DISCONNECT,
+	NLPVRDPY_CMD_FLIP,
+	NLPVRDPY_CMD_FLIPPED,
+	NLPVRDPY_CMD_COPY,
+	NLPVRDPY_CMD_COPIED,
+	__NLPVRDPY_CMD_MAX
+};
+#define NLPVRDPY_CMD_MAX (__NLPVRDPY_CMD_MAX - 1)
+
+enum nlpvrdpy_attr {
+	__NLPVRDPY_ATTR_INVALID,
+	NLPVRDPY_ATTR_NAME,
+	NLPVRDPY_ATTR_MINOR,
+	NLPVRDPY_ATTR_NUM_PLANES,
+	NLPVRDPY_ATTR_WIDTH,
+	NLPVRDPY_ATTR_HEIGHT,
+	NLPVRDPY_ATTR_PIXFMT,
+	NLPVRDPY_ATTR_YUV_CSC,
+	NLPVRDPY_ATTR_YUV_BPP,
+	NLPVRDPY_ATTR_PLANE0_ADDR,
+	NLPVRDPY_ATTR_PLANE0_SIZE,
+	NLPVRDPY_ATTR_PLANE0_OFFSET,
+	NLPVRDPY_ATTR_PLANE0_PITCH,
+	NLPVRDPY_ATTR_PLANE0_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_PLANE1_ADDR,
+	NLPVRDPY_ATTR_PLANE1_SIZE,
+	NLPVRDPY_ATTR_PLANE1_OFFSET,
+	NLPVRDPY_ATTR_PLANE1_PITCH,
+	NLPVRDPY_ATTR_PLANE1_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_PLANE2_ADDR,
+	NLPVRDPY_ATTR_PLANE2_SIZE,
+	NLPVRDPY_ATTR_PLANE2_OFFSET,
+	NLPVRDPY_ATTR_PLANE2_PITCH,
+	NLPVRDPY_ATTR_PLANE2_GEM_OBJ_NAME,
+	NLPVRDPY_ATTR_FB_MODIFIER,
+	NLPVRDPY_ATTR_NAMING_REQUIRED,
+	NLPVRDPY_ATTR_PAD,
+	__NLPVRDPY_ATTR_MAX
+};
+#define NLPVRDPY_ATTR_MAX  (__NLPVRDPY_ATTR_MAX - 1)
+
+static struct nla_policy __attribute__((unused))
+nlpvrdpy_policy[NLPVRDPY_ATTR_MAX + 1] = {
+	[NLPVRDPY_ATTR_NAME]                = { .type = NLA_STRING },
+	[NLPVRDPY_ATTR_MINOR]               = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_NUM_PLANES]          = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_WIDTH]               = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_HEIGHT]              = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PIXFMT]              = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_YUV_CSC]             = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_YUV_BPP]             = { .type = NLA_U8  },
+	[NLPVRDPY_ATTR_PLANE0_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE0_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PLANE1_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE1_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_PLANE2_ADDR]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_SIZE]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_OFFSET]       = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_PITCH]        = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_PLANE2_GEM_OBJ_NAME] = { .type = NLA_U32 },
+	[NLPVRDPY_ATTR_FB_MODIFIER]         = { .type = NLA_U64 },
+	[NLPVRDPY_ATTR_NAMING_REQUIRED]     = { .type = NLA_FLAG },
+};
+
+#define NLPVRDPY_ATTR_PLANE(index, type)				\
+	({								\
+		enum nlpvrdpy_attr __retval;				\
+									\
+		switch (index) {					\
+		case 0:							\
+			__retval = NLPVRDPY_ATTR_PLANE0_ ## type;	\
+			break;						\
+		case 1:							\
+			__retval = NLPVRDPY_ATTR_PLANE1_ ## type;	\
+			break;						\
+		case 2:							\
+			__retval = NLPVRDPY_ATTR_PLANE2_ ## type;	\
+			break;						\
+		default:						\
+			__retval = __NLPVRDPY_ATTR_INVALID;		\
+			break;						\
+		};							\
+									\
+		__retval;						\
+	})
+
+#endif /* __NETLINK_H__ */
diff --git a/drivers/gpu/drm/nulldisp/nulldisp_drm.h b/drivers/gpu/drm/nulldisp/nulldisp_drm.h
new file mode 100644
index 000000000000..8403fb56f1a5
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/nulldisp_drm.h
@@ -0,0 +1,105 @@
+/*
+ * @File
+ * @Title       Nulldisp DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__NULLDISP_DRM_H__)
+#define __NULLDISP_DRM_H__
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+struct drm_nulldisp_gem_create {
+	__u64 size;   /* in */
+	__u32 flags;  /* in */
+	__u32 handle; /* out */
+};
+
+struct drm_nulldisp_gem_mmap {
+	__u32 handle; /* in */
+	__u32 pad;
+	__u64 offset; /* out */
+};
+
+#define NULLDISP_GEM_CPU_PREP_READ   (1 << 0)
+#define NULLDISP_GEM_CPU_PREP_WRITE  (1 << 1)
+#define NULLDISP_GEM_CPU_PREP_NOWAIT (1 << 2)
+
+struct drm_nulldisp_gem_cpu_prep {
+	__u32 handle; /* in */
+	__u32 flags;  /* in */
+};
+
+struct drm_nulldisp_gem_cpu_fini {
+	__u32 handle; /* in */
+	__u32 pad;
+};
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+#define DRM_NULLDISP_GEM_CREATE   0x00
+#define DRM_NULLDISP_GEM_MMAP     0x01
+#define DRM_NULLDISP_GEM_CPU_PREP 0x02
+#define DRM_NULLDISP_GEM_CPU_FINI 0x03
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define DRM_IOCTL_NULLDISP_GEM_CREATE \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CREATE, \
+		 struct drm_nulldisp_gem_create)
+
+#define DRM_IOCTL_NULLDISP_GEM_MMAP \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_MMAP, \
+		 struct drm_nulldisp_gem_mmap)
+
+#define DRM_IOCTL_NULLDISP_GEM_CPU_PREP \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CPU_PREP, \
+		struct drm_nulldisp_gem_cpu_prep)
+
+#define DRM_IOCTL_NULLDISP_GEM_CPU_FINI \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_NULLDISP_GEM_CPU_FINI, \
+		struct drm_nulldisp_gem_cpu_fini)
+
+#endif /* defined(__NULLDISP_DRM_H__) */
diff --git a/drivers/gpu/drm/nulldisp/pdp_drm.h b/drivers/gpu/drm/nulldisp/pdp_drm.h
new file mode 100644
index 000000000000..f5d747d3ad10
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/pdp_drm.h
@@ -0,0 +1,105 @@
+/*
+ * @File
+ * @Title       PDP DRM definitions shared between kernel and user space.
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PDP_DRM_H__)
+#define __PDP_DRM_H__
+
+#if defined(__KERNEL__)
+#include <drm/drm.h>
+#else
+#include <drm.h>
+#endif
+
+struct drm_pdp_gem_create {
+	__u64 size;	/* in */
+	__u32 flags;	/* in */
+	__u32 handle;	/* out */
+};
+
+struct drm_pdp_gem_mmap {
+	__u32 handle;	/* in */
+	__u32 pad;
+	__u64 offset;	/* out */
+};
+
+#define PDP_GEM_CPU_PREP_READ	(1 << 0)
+#define PDP_GEM_CPU_PREP_WRITE	(1 << 1)
+#define PDP_GEM_CPU_PREP_NOWAIT	(1 << 2)
+
+struct drm_pdp_gem_cpu_prep {
+	__u32 handle;	/* in */
+	__u32 flags;	/* in */
+};
+
+struct drm_pdp_gem_cpu_fini {
+	__u32 handle;	/* in */
+	__u32 pad;
+};
+
+/*
+ * DRM command numbers, relative to DRM_COMMAND_BASE.
+ * These defines must be prefixed with "DRM_".
+ */
+#define DRM_PDP_GEM_CREATE		0x00
+#define DRM_PDP_GEM_MMAP		0x01
+#define DRM_PDP_GEM_CPU_PREP		0x02
+#define DRM_PDP_GEM_CPU_FINI		0x03
+
+/* These defines must be prefixed with "DRM_IOCTL_". */
+#define DRM_IOCTL_PDP_GEM_CREATE \
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_CREATE, \
+		 struct drm_pdp_gem_create)
+
+#define DRM_IOCTL_PDP_GEM_MMAP\
+	DRM_IOWR(DRM_COMMAND_BASE + DRM_PDP_GEM_MMAP, \
+		 struct drm_pdp_gem_mmap)
+
+#define DRM_IOCTL_PDP_GEM_CPU_PREP \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_PREP, \
+		struct drm_pdp_gem_cpu_prep)
+
+#define DRM_IOCTL_PDP_GEM_CPU_FINI \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_PDP_GEM_CPU_FINI, \
+		struct drm_pdp_gem_cpu_fini)
+
+#endif /* defined(__PDP_DRM_H__) */
diff --git a/drivers/gpu/drm/nulldisp/pvr_dma_resv.h b/drivers/gpu/drm/nulldisp/pvr_dma_resv.h
new file mode 100644
index 000000000000..a51c9de84ada
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/pvr_dma_resv.h
@@ -0,0 +1,80 @@
+/*************************************************************************/ /*!
+@Title          Kernel reservation object compatibility header
+@Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+@Description    Per-version macros to allow code to seamlessly use older kernel
+@License        Dual MIT/GPLv2
+
+The contents of this file are subject to the MIT license as set out below.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+Alternatively, the contents of this file may be used under the terms of
+the GNU General Public License Version 2 ("GPL") in which case the provisions
+of GPL are applicable instead of those above.
+
+If you wish to allow use of your version of this file only under the terms of
+GPL, and not to allow others to use your version of this file under the terms
+of the MIT license, indicate your decision by deleting the provisions above
+and replace them with the notice and other provisions required by GPL as set
+out in the file called "GPL-COPYING" included in this distribution. If you do
+not delete the provisions above, a recipient may use your version of this file
+under the terms of either the MIT license or GPL.
+
+This License is also included in this distribution in the file called
+"MIT-COPYING".
+
+EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/ /**************************************************************************/
+
+#ifndef __PVR_DMA_RESV_H__
+#define __PVR_DMA_RESV_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <linux/dma-resv.h>
+#else
+#include <linux/reservation.h>
+
+/* Reservation object types */
+#define dma_resv			reservation_object
+#define dma_resv_list			reservation_object_list
+
+/* Reservation object functions */
+#define dma_resv_add_excl_fence		reservation_object_add_excl_fence
+#define dma_resv_add_shared_fence	reservation_object_add_shared_fence
+#define dma_resv_fini			reservation_object_fini
+#define dma_resv_get_excl		reservation_object_get_excl
+#define dma_resv_get_list		reservation_object_get_list
+#define dma_resv_held			reservation_object_held
+#define dma_resv_init			reservation_object_init
+#define dma_resv_reserve_shared		reservation_object_reserve_shared
+#define dma_resv_test_signaled_rcu	reservation_object_test_signaled_rcu
+#define dma_resv_wait_timeout_rcu	reservation_object_wait_timeout_rcu
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0))
+
+#define dma_resv_shared_list   dma_resv_get_list
+#define dma_resv_excl_fence    dma_resv_get_excl
+#define dma_resv_wait_timeout  dma_resv_wait_timeout_rcu
+#define dma_resv_test_signaled dma_resv_test_signaled_rcu
+#define dma_resv_get_fences    dma_resv_get_fences_rcu
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)) */
+
+#endif /* __PVR_DMA_RESV_H__ */
diff --git a/drivers/gpu/drm/nulldisp/pvr_linux_fence.h b/drivers/gpu/drm/nulldisp/pvr_linux_fence.h
new file mode 100644
index 000000000000..b9c542a26a65
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/pvr_linux_fence.h
@@ -0,0 +1,103 @@
+/*
+ * @File
+ * @Title       PowerVR Linux fence compatibility header
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if !defined(__PVR_LINUX_FENCE_H__)
+#define __PVR_LINUX_FENCE_H__
+
+#include <linux/version.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+#include <linux/fence.h>
+#else
+#include <linux/dma-fence.h>
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) && \
+	!defined(CHROMIUMOS_KERNEL_HAS_DMA_FENCE)
+/* Structures */
+#define	dma_fence fence
+#define dma_fence_array fence_array
+#define	dma_fence_cb fence_cb
+#define	dma_fence_ops fence_ops
+
+/* Defines and Enums */
+#define DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT FENCE_FLAG_ENABLE_SIGNAL_BIT
+#define DMA_FENCE_FLAG_SIGNALED_BIT FENCE_FLAG_SIGNALED_BIT
+#define DMA_FENCE_FLAG_USER_BITS FENCE_FLAG_USER_BITS
+
+#define DMA_FENCE_ERR FENCE_ERR
+#define	DMA_FENCE_TRACE FENCE_TRACE
+#define DMA_FENCE_WARN FENCE_WARN
+
+/* Functions */
+#define dma_fence_add_callback fence_add_callback
+#define dma_fence_context_alloc fence_context_alloc
+#define dma_fence_default_wait fence_default_wait
+#define dma_fence_is_signaled fence_is_signaled
+#define dma_fence_enable_sw_signaling fence_enable_sw_signaling
+#define dma_fence_free fence_free
+#define dma_fence_get fence_get
+#define dma_fence_get_rcu fence_get_rcu
+#define dma_fence_init fence_init
+#define dma_fence_is_array fence_is_array
+#define dma_fence_put fence_put
+#define dma_fence_signal fence_signal
+#define dma_fence_wait fence_wait
+#define to_dma_fence_array to_fence_array
+
+static inline signed long
+dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
+{
+	signed long lret;
+
+	lret = fence_wait_timeout(fence, intr, timeout);
+	if (lret || timeout)
+		return lret;
+
+	return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) ? 1 : 0;
+}
+
+#endif
+
+#endif /* !defined(__PVR_LINUX_FENCE_H__) */
diff --git a/drivers/gpu/drm/nulldisp/pvr_vmap.h b/drivers/gpu/drm/nulldisp/pvr_vmap.h
new file mode 100644
index 000000000000..6424b20e764c
--- /dev/null
+++ b/drivers/gpu/drm/nulldisp/pvr_vmap.h
@@ -0,0 +1,83 @@
+/*
+ * @File        pvr_vmap.h
+ * @Title       Utility functions for virtual memory mapping
+ * @Codingstyle LinuxKernel
+ * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
+ * @License     Dual MIT/GPLv2
+ *
+ * The contents of this file are subject to the MIT license as set out below.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * the GNU General Public License Version 2 ("GPL") in which case the provisions
+ * of GPL are applicable instead of those above.
+ *
+ * If you wish to allow use of your version of this file only under the terms of
+ * GPL, and not to allow others to use your version of this file under the terms
+ * of the MIT license, indicate your decision by deleting the provisions above
+ * and replace them with the notice and other provisions required by GPL as set
+ * out in the file called "GPL-COPYING" included in this distribution. If you do
+ * not delete the provisions above, a recipient may use your version of this file
+ * under the terms of either the MIT license or GPL.
+ *
+ * This License is also included in this distribution in the file called
+ * "MIT-COPYING".
+ *
+ * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
+ * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+ * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef PVR_VMAP_H
+#define PVR_VMAP_H
+
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+
+static inline void *pvr_vmap(struct page **pages,
+			     unsigned int count,
+			     __maybe_unused unsigned long flags,
+			     pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	return vmap(pages, count, flags, prot);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	return vm_map_ram(pages, count, -1, prot);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		return vm_map_ram(pages, count, -1);
+	else
+		return vmap(pages, count, flags, prot);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+static inline void pvr_vunmap(const void *vaddr,
+			      __maybe_unused unsigned int count,
+			      __maybe_unused pgprot_t prot)
+{
+#if !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS)
+	vunmap(vaddr);
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	vm_unmap_ram(vaddr, count);
+#else
+	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
+		vm_unmap_ram(vaddr, count);
+	else
+		vunmap(vaddr);
+#endif /* !defined(CONFIG_64BIT) || defined(PVRSRV_FORCE_SLOWER_VMAP_ON_64BIT_BUILDS) */
+}
+
+#endif /* PVR_VMAP_H */
diff --git a/drivers/gpu/drm/spacemit/Kconfig b/drivers/gpu/drm/spacemit/Kconfig
new file mode 100644
index 000000000000..a31e478f9625
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/Kconfig
@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config DRM_SPACEMIT
+	tristate "DRM Support for Spacemit"
+	depends on DRM
+	select DRM_CLIENT_SELECTION
+	select DRM_KMS_HELPER
+	select DRM_GEM_CMA_HELPER
+	select DRM_KMS_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
+	select DRM_FBDEV_EMULATION
+	select DRM_MIPI_DSI
+	select DRM_PANEL
+	select VIDEOMODE_HELPERS
+	select BACKLIGHT_CLASS_DEVICE
+	select GKI_FIX_WORKAROUND if DRM_SPACEMIT=m
+	default n
+	help
+	  Choose this option if you have a SPACEMIT chipsets.
+	  If M is selected the module will be called spacemit-drm.
+
+config SPACEMIT_INNO_DP
+	tristate "Spacemit specific extensions for inno dp/edp"
+	depends on DRM_SPACEMIT
+	select DRM_DISPLAY_HELPER
+	select DRM_DISPLAY_DP_HELPER
+	select DRM_DISPLAY_DP_AUX_BUS
+	default n
+	help
+	  Support for Spacemit SoC specific extensions for
+	  the Innosilicon dp/edp driver.
+
+config SPACEMIT_MIPI_PANEL
+	tristate "mipi panel support for SPACEMIT SOCs platform"
+	depends on DRM_SPACEMIT
+
+config DRM_RASPBERRYPI_TOUCHSCREEN
+	tristate "Raspberrypi Touchscreen"
+	default y
+	depends on DRM_SPACEMIT
+	select DRM_KMS_HELPER
+	select REGMAP_I2C
+	select DRM_PANEL
+	select DRM_MIPI_DSI
+
+	help
+	  Support for Raspberrypi Touchscreen driver.
+
+source "drivers/gpu/drm/spacemit/backlight/Kconfig"
diff --git a/drivers/gpu/drm/spacemit/Makefile b/drivers/gpu/drm/spacemit/Makefile
new file mode 100644
index 000000000000..97c30989b287
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/Makefile
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: GPL-2.0
+
+spacemit-drm-y := 	spacemit_drm.o \
+				spacemit_crtc.o \
+				spacemit_cmdlist.o \
+				spacemit_wb.o \
+				spacemit_planes.o \
+				spacemit_dsi.o \
+				spacemit_dphy.o \
+				spacemit_lib.o \
+				spacemit_gem.o \
+				spacemit_dmmu.o \
+				spacemit_bootloader.o \
+				common/spacemit_drm_notifier.o \
+				sysfs/sysfs_class.o \
+				sysfs/sysfs_dpu.o \
+				sysfs/sysfs_dsi.o \
+				sysfs/sysfs_dphy.o \
+				sysfs/sysfs_mipi_panel.o \
+				dpu/dpu_debug.o \
+				dpu/dpu_saturn.o \
+				dpu/post_process_hee.o \
+				dpu/dpu_saturn_hee.o \
+				dpu/saturn_fbcmem.o \
+				dsi/spacemit_dptc_drv.o \
+				dsi/spacemit_dsi_drv.o \
+				dphy/spacemit_dphy_drv.o \
+
+obj-$(CONFIG_DRM_SPACEMIT) += spacemit-drm.o
+obj-$(CONFIG_SPACEMIT_INNO_DP) += spacemit_inno_dp.o
+obj-$(CONFIG_SPACEMIT_MIPI_PANEL) += spacemit_mipi_panel.o
+obj-$(CONFIG_DRM_SPACEMIT_BACKLIGHT) += backlight/
+obj-$(CONFIG_DRM_SPACEMIT_SELFTEST) += selftest/
+obj-$(CONFIG_DRM_RASPBERRYPI_TOUCHSCREEN) += raspberrypi_touchscreen.o
diff --git a/drivers/gpu/drm/spacemit/backlight/Kconfig b/drivers/gpu/drm/spacemit/backlight/Kconfig
new file mode 100644
index 000000000000..eb2af90d0433
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/backlight/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+config DRM_SPACEMIT_BACKLIGHT
+	tristate "Support for DRM SPACEMIT backlight"
+	select BACKLIGHT_CLASS_DEVICE
+	default n
+	help
+	  Choose this option if you have an SPACEMIT Soc with SPACEMIT backlight.
+
+
diff --git a/drivers/gpu/drm/spacemit/backlight/Makefile b/drivers/gpu/drm/spacemit/backlight/Makefile
new file mode 100644
index 000000000000..3158e1232a62
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/backlight/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_DRM_SPACEMIT_BACKLIGHT) += spacemit-backlight.o
+
diff --git a/drivers/gpu/drm/spacemit/backlight/spacemit-backlight.c b/drivers/gpu/drm/spacemit/backlight/spacemit-backlight.c
new file mode 100644
index 000000000000..d7e7115a9e3e
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/backlight/spacemit-backlight.c
@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include "spacemit-backlight.h"
+
+#define DEV_NAME			"spacemit-backlight"
+
+struct spacemit_bl_device {
+	struct spacemit_bl_ops *ops;
+	void *devdata;
+	int max_brightness;
+	int brightness;
+
+	struct platform_device pdev;
+};
+
+static ssize_t spacemit_bl_get_brightness(struct device *dev,
+			       struct device_attribute *attr,
+			       char *buf)
+{
+	struct spacemit_bl_device *bld = dev_get_platdata(dev);
+	int brightness = 0;
+
+	if (bld->ops && bld->ops->get_bl)
+		brightness = bld->ops->get_bl(bld->devdata);
+	else
+		brightness = bld->brightness;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", brightness);
+}
+
+static ssize_t spacemit_bl_get_max_brightness(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct spacemit_bl_device *bld = dev_get_platdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", bld->max_brightness);
+}
+
+static ssize_t spacemit_bl_set_brightness(struct device *dev,
+				struct device_attribute *attr,
+				const char *buf,
+				size_t count)
+{
+	struct spacemit_bl_device *bld = dev_get_platdata(dev);
+	int old_brightness;
+	int new_brightness;
+	int ret;
+
+	if (!bld->ops || !bld->ops->set_bl)
+		return -ENODEV;
+
+	ret = sscanf(buf, "%u\n", &new_brightness);
+	if (ret != 1 || new_brightness > bld->max_brightness) {
+		pr_err("Wrong parameter! please echo 0 ~ %d.\n", bld->max_brightness);
+		return -EINVAL;
+	}
+
+	if (bld->ops->get_bl)
+		old_brightness = bld->ops->get_bl(bld->devdata);
+	else
+		old_brightness = bld->brightness;
+
+	if (old_brightness != new_brightness) {
+		bld->brightness = new_brightness;
+		bld->ops->set_bl(bld->devdata, new_brightness);
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR(brightness, S_IRUGO | S_IWUSR, spacemit_bl_get_brightness, spacemit_bl_set_brightness);
+static DEVICE_ATTR(max_brightness, S_IRUGO, spacemit_bl_get_max_brightness, NULL);
+
+static int spacemit_backlight_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	ret = device_create_file(dev, &dev_attr_brightness);
+	if (ret)
+		pr_err("failed to create device file: brightness\n");
+	else
+		pr_debug("create device file brightness\n");
+
+	ret = device_create_file(dev, &dev_attr_max_brightness);
+	if (ret)
+		pr_err("failed to create device file: max_brightness\n");
+	else
+		pr_debug("create device file max_brightness\n");
+
+	return 0;
+}
+
+static void spacemit_backlight_remove(struct platform_device *pdev)
+{
+}
+
+static void spacemit_backlight_shutdown(struct platform_device *pdev)
+{
+	struct spacemit_bl_device *bld = container_of(pdev, struct spacemit_bl_device, pdev);
+
+	if (bld->ops && bld->ops->set_bl) {
+		bld->brightness = 0;
+		bld->ops->set_bl(bld->devdata, 0);
+	}
+}
+
+int spacemit_bl_device_register(struct device *parent, void *devdata, struct spacemit_bl_ops *ops, int max)
+{
+	struct spacemit_bl_device *bld;
+	int ret;
+
+	bld = kzalloc(sizeof(struct spacemit_bl_device), GFP_KERNEL);
+	if (!bld)
+		return -ENOMEM;
+
+	bld->pdev.dev.parent = NULL;
+	bld->pdev.dev.platform_data = bld;
+	bld->pdev.name = DEV_NAME;
+	bld->devdata = devdata;
+	bld->max_brightness = max;
+	bld->ops = ops;
+
+	ret = platform_device_register(&bld->pdev);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+EXPORT_SYMBOL(spacemit_bl_device_register);
+
+static const struct of_device_id spacemit_backlight_dt_match[] = {
+	{ .compatible = "spacemit,backlight", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, spacemit_backlight_dt_match);
+
+static struct platform_driver spacemit_backlight_driver = {
+	.driver = {
+		.name = DEV_NAME,
+		.of_match_table = of_match_ptr(spacemit_backlight_dt_match),
+	},
+	.probe = spacemit_backlight_probe,
+	.remove = spacemit_backlight_remove,
+	.shutdown = spacemit_backlight_shutdown,
+};
+module_platform_driver(spacemit_backlight_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SPACEMIT Backlight Driver");
diff --git a/drivers/gpu/drm/spacemit/backlight/spacemit-backlight.h b/drivers/gpu/drm/spacemit/backlight/spacemit-backlight.h
new file mode 100644
index 000000000000..0a2b4f118012
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/backlight/spacemit-backlight.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_BACKLIGHT_H_
+#define _SPACEMIT_BACKLIGHT_H_
+
+struct spacemit_bl_ops {
+	int (*get_bl)(void *devdata);
+	int (*set_bl)(void *devdata, int brightness);
+};
+
+int spacemit_bl_device_register(struct device *parent, void *devdata, struct spacemit_bl_ops *ops, int max);
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/common/spacemit_drm_notifier.c b/drivers/gpu/drm/spacemit/common/spacemit_drm_notifier.c
new file mode 100644
index 000000000000..33aa15bb9641
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/common/spacemit_drm_notifier.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/notifier.h>
+
+#include "spacemit_drm_notifier.h"
+
+BLOCKING_NOTIFIER_HEAD(drm_notifier_list);
+
+/*	spacemit_drm_register_client - register a client notifier  */
+int spacemit_drm_notifier_register_client(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&drm_notifier_list, nb);
+}
+EXPORT_SYMBOL_GPL(spacemit_drm_notifier_register_client);
+
+/*  spacemit_drm_unregister_client - unregister a client notifier  */
+int spacemit_drm_notifier_unregister_client(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_unregister(&drm_notifier_list, nb);
+}
+EXPORT_SYMBOL_GPL(spacemit_drm_notifier_unregister_client);
+
+/*  spacemit_drm_notifier_call_chain - notify clients of drm_events  */
+int spacemit_drm_notifier_call_chain(unsigned long val, void *v)
+{
+	return blocking_notifier_call_chain(&drm_notifier_list, val, v);
+}
+EXPORT_SYMBOL_GPL(spacemit_drm_notifier_call_chain);
diff --git a/drivers/gpu/drm/spacemit/common/spacemit_drm_notifier.h b/drivers/gpu/drm/spacemit/common/spacemit_drm_notifier.h
new file mode 100644
index 000000000000..ea1f86060fac
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/common/spacemit_drm_notifier.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_DRM_NOTIFIER_H_
+#define _SPACEMIT_DRM_NOTIFIER_H_
+
+#include <linux/notifier.h>
+
+enum {
+	DRM_PANEL_EARLY_EVENT_BLANK = 0,
+	DRM_PANEL_EVENT_BLANK,
+	DRM_PANEL_BLANK_UNBLANK,
+	DRM_PANEL_BLANK_POWERDOWN,
+	DRM_PANEL_TOUCH_INT,
+};
+
+struct spacemit_drm_notifier {
+	int blank;
+};
+
+int spacemit_drm_notifier_register_client(struct notifier_block *nb);
+
+int spacemit_drm_notifier_unregister_client(struct notifier_block *nb);
+
+int spacemit_drm_notifier_call_chain(unsigned long val, void *v);
+
+extern struct blocking_notifier_head drm_notifier_list;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dphy/spacemit_dphy_drv.c b/drivers/gpu/drm/spacemit/dphy/spacemit_dphy_drv.c
new file mode 100644
index 000000000000..3e8938cfee81
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dphy/spacemit_dphy_drv.c
@@ -0,0 +1,440 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+
+#include "../dsi/spacemit_dsi_hw.h"
+#include "../spacemit_dphy.h"
+#include "../dsi/spacemit_dptc_drv.h"
+
+static unsigned int spacemit_dphy_lane[5] = {0, 0x1, 0x3, 0x7, 0xf};
+
+static void dphy_ana_reset(void __iomem *base_addr)
+{
+	dsi_clear_bits(base_addr, DSI_PHY_ANA_PWR_CTRL, CFG_DPHY_ANA_RESET);
+	udelay(5);
+	dsi_set_bits(base_addr, DSI_PHY_ANA_PWR_CTRL, CFG_DPHY_ANA_RESET);
+}
+
+static void dphy_set_power(void __iomem *base_addr, bool poweron)
+{
+	if (poweron) {
+		dsi_set_bits(base_addr, DSI_PHY_ANA_PWR_CTRL, CFG_DPHY_ANA_RESET);
+		dsi_set_bits(base_addr, DSI_PHY_ANA_PWR_CTRL, CFG_DPHY_ANA_PU);
+	} else {
+		dsi_clear_bits(base_addr, DSI_PHY_ANA_PWR_CTRL, CFG_DPHY_ANA_PU);
+		dsi_clear_bits(base_addr, DSI_PHY_ANA_PWR_CTRL, CFG_DPHY_ANA_RESET);
+	}
+	udelay(5);
+}
+
+static void dphy_set_cont_clk(void __iomem *base_addr, bool cont_clk)
+{
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	uint32_t tmp;
+
+	if (cont_clk) {
+		tmp = dptc_dsi_read(0x04);
+		tmp |= CFG_DPHY_CONT_CLK;
+		//dptc_dsi_write(0x04, tmp);
+	} else {
+		tmp = dptc_dsi_read(0x04);
+		tmp &= (~CFG_DPHY_CONT_CLK);
+		//dptc_dsi_write(0x04, tmp);
+	}
+	dptc_dsi_write(0x04, 0x30001);
+#else
+	if (cont_clk)
+		dsi_set_bits(base_addr, DSI_PHY_CTRL_1, CFG_DPHY_CONT_CLK);
+	else
+		dsi_clear_bits(base_addr, DSI_PHY_CTRL_1, CFG_DPHY_CONT_CLK);
+
+	dsi_set_bits(base_addr, DSI_PHY_CTRL_1, CFG_DPHY_ADD_VALID);
+	dsi_set_bits(base_addr, DSI_PHY_CTRL_1, CFG_DPHY_VDD_VALID);
+#endif
+}
+
+static void dphy_set_lane_num(void __iomem *base_addr, uint32_t lane_num)
+{
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	uint32_t tmp;
+
+	tmp = dptc_dsi_read(0x08);
+	tmp &= ~CFG_DPHY_LANE_EN_MASK;
+	tmp |= spacemit_dphy_lane[lane_num] << CFG_DPHY_LANE_EN_SHIFT;
+	dptc_dsi_write(0x08, 0);
+	tmp = dptc_dsi_read(0x08);
+	dptc_dsi_write(0x08, 0x30);
+#endif
+	dsi_write_bits(base_addr, DSI_PHY_CTRL_2,
+		CFG_DPHY_LANE_EN_MASK, spacemit_dphy_lane[lane_num] << CFG_DPHY_LANE_EN_SHIFT);
+}
+
+static void dphy_set_bit_clk_src(void __iomem *base_addr, uint32_t bit_clk_src,
+	uint32_t half_pll5)
+{
+	if (bit_clk_src >= DPHY_BIT_CLK_SRC_MAX) {
+		DRM_ERROR("%s: Invalid bit clk src (%d)\n", __func__, bit_clk_src);
+		return;
+	}
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	uint32_t tmp;
+	tmp = dptc_dsi_read(0x68);
+	tmp &= ~CFG_CLK_SEL;
+	dptc_dsi_write(0x68, tmp);
+
+	tmp = dptc_dsi_read(0x68);
+	tmp &= ~CFG_CLK_DIV2;
+	dptc_dsi_write(0x68, tmp);
+#endif
+}
+
+static void dphy_set_timing(struct spacemit_dphy_ctx *dphy_ctx)
+{
+	uint32_t bitclk;
+	int ui, wakeup, reg;
+	int hs_prep, hs_zero, hs_trail, hs_exit, ck_zero, ck_trail, ck_exit, lpx_clk, lpx_time, ta_get, ta_go;
+	int esc_clk, esc_clk_t;
+	struct spacemit_dphy_timing *phy_timing;
+
+	if (dphy_ctx == NULL) {
+		DRM_ERROR("%s: Invalid param!\n", __func__);
+		return;
+	}
+
+	phy_timing = &(dphy_ctx->dphy_timing);
+
+	esc_clk = dphy_ctx->esc_clk/1000;
+	esc_clk_t = 1000/esc_clk;
+
+	bitclk = dphy_ctx->phy_freq / 1000;
+	ui = 1000/bitclk + 1;
+
+	lpx_clk = (phy_timing->lpx_constant + phy_timing->lpx_ui * ui) / esc_clk_t + 1;
+	lpx_time = lpx_clk * esc_clk_t;
+
+	ta_get = lpx_time * 5 / esc_clk_t - 1;
+	ta_go = lpx_time * 4 / esc_clk_t - 1;
+
+	wakeup = phy_timing->wakeup_constant;
+	wakeup = wakeup / esc_clk_t + 1;
+
+	hs_prep = phy_timing->hs_prep_constant + phy_timing->hs_prep_ui * ui;
+	hs_prep = hs_prep / esc_clk_t + 1;
+
+	/* Our hardware added 3-byte clk automatically.
+	 * 3-byte 3 * 8 * ui.
+	 */
+	hs_zero = phy_timing->hs_zero_constant + phy_timing->hs_zero_ui * ui -
+		(hs_prep + 1) * esc_clk_t;
+	hs_zero = (hs_zero - (3 * ui << 3)) / esc_clk_t + 4;
+	if (hs_zero < 0)
+		hs_zero = 0;
+
+	hs_trail = phy_timing->hs_trail_constant + phy_timing->hs_trail_ui * ui;
+	hs_trail = ((8 * ui) >= hs_trail) ? (8 * ui) : hs_trail;
+	hs_trail = hs_trail / esc_clk_t + 1;
+	if (hs_trail > 3)
+		hs_trail -= 3;
+	else
+		hs_trail = 0;
+
+	hs_exit = phy_timing->hs_exit_constant + phy_timing->hs_exit_ui * ui;
+	hs_exit = hs_exit / esc_clk_t + 1;
+
+	ck_zero = phy_timing->ck_zero_constant + phy_timing->ck_zero_ui * ui -
+		(hs_prep + 1) * esc_clk_t;
+	ck_zero = ck_zero / esc_clk_t + 1;
+
+	ck_trail = phy_timing->ck_trail_constant + phy_timing->ck_trail_ui * ui;
+	ck_trail = ck_trail / esc_clk_t + 1;
+
+	ck_exit = hs_exit;
+
+	reg = (hs_exit << CFG_DPHY_TIME_HS_EXIT_SHIFT)
+		| (hs_trail << CFG_DPHY_TIME_HS_TRAIL_SHIFT)
+		| (hs_zero << CFG_DPHY_TIME_HS_ZERO_SHIFT)
+		| (hs_prep << CFG_DPHY_TIME_HS_PREP_SHIFT);
+
+	DRM_DEBUG("%s: dphy timing 0 :0x%x\n", __func__, reg);
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	dptc_dsi_write(0x40, 0x01000000);
+#else
+	dsi_write(dphy_ctx->base_addr, DSI_PHY_TIME_0, reg);
+#endif
+	reg = (ta_get << CFG_DPHY_TIME_TA_GET_SHIFT)
+		| (ta_go << CFG_DPHY_TIME_TA_GO_SHIFT)
+		| (wakeup << CFG_DPHY_TIME_WAKEUP_SHIFT);
+
+	DRM_DEBUG("%s: dphy timing 1 :0x%x\n", __func__, reg);
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	dptc_dsi_write(0x44, 0x0403001F);
+#else
+	dsi_write(dphy_ctx->base_addr, DSI_PHY_TIME_1, reg);
+#endif
+	reg = (ck_exit << CFG_DPHY_TIME_CLK_EXIT_SHIFT)
+		| (ck_trail << CFG_DPHY_TIME_CLK_TRAIL_SHIFT)
+		| (ck_zero << CFG_DPHY_TIME_CLK_ZERO_SHIFT)
+		| (lpx_clk << CFG_DPHY_TIME_CLK_LPX_SHIFT);
+
+	DRM_DEBUG("%s: dphy timing 2 :0x%x\n", __func__, reg);
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	dptc_dsi_write(0x48, 0x02010500);
+#else
+	dsi_write(dphy_ctx->base_addr, DSI_PHY_TIME_2, reg);
+#endif
+	reg = (lpx_clk << CFG_DPHY_TIME_LPX_SHIFT)
+		| phy_timing->req_ready << CFG_DPHY_TIME_REQRDY_SHIFT;
+
+	DRM_DEBUG("%s: dphy timing 3 :0x%x\n", __func__, reg);
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	dptc_dsi_write(0x4c, 0x001F);
+#else
+	dsi_write(dphy_ctx->base_addr, DSI_PHY_TIME_3, reg);
+#endif
+	/* calculated timing on brownstone:
+	 * DSI_PHY_TIME_0 0x06080204
+	 * DSI_PHY_TIME_1 0x6d2bfff0
+	 * DSI_PHY_TIME_2 0x603130a
+	 * DSI_PHY_TIME_3 0xa3c
+	 */
+}
+
+static void dphy_get_setting(struct spacemit_dphy_ctx *dphy_ctx, struct device_node *np)
+{
+	struct spacemit_dphy_timing *dphy_timing = &dphy_ctx->dphy_timing;
+	int ret;
+
+	if (dphy_timing == NULL) {
+		DRM_ERROR("%s: Invalid param\n", __func__);
+		return;
+	}
+
+	ret = of_property_read_u32(np, "hs_prep_constant", &dphy_timing->hs_prep_constant);
+	if (ret != 0)
+		dphy_timing->hs_prep_constant = HS_PREP_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "hs_prep_ui", &dphy_timing->hs_prep_ui);
+	if (ret != 0)
+		dphy_timing->hs_prep_ui = HS_PREP_UI_DEFAULT;
+
+	ret = of_property_read_u32(np, "hs_zero_constant", &dphy_timing->hs_zero_constant);
+	if (ret != 0)
+		dphy_timing->hs_zero_constant = HS_ZERO_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "hs_zero_ui", &dphy_timing->hs_zero_ui);
+	if (ret != 0)
+		dphy_timing->hs_zero_ui = HS_ZERO_UI_DEFAULT;
+
+	ret = of_property_read_u32(np, "hs_trail_constant", &dphy_timing->hs_trail_constant);
+	if (ret != 0)
+		dphy_timing->hs_trail_constant = HS_TRAIL_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "hs_trail_ui", &dphy_timing->hs_trail_ui);
+	if (ret != 0)
+		dphy_timing->hs_trail_ui = HS_TRAIL_UI_DEFAULT;
+
+	ret = of_property_read_u32(np, "hs_exit_constant", &dphy_timing->hs_exit_constant);
+	if (ret != 0)
+		dphy_timing->hs_exit_constant = HS_EXIT_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "hs_exit_ui", &dphy_timing->hs_exit_ui);
+	if (ret != 0)
+		dphy_timing->hs_exit_ui = HS_EXIT_UI_DEFAULT;
+
+	ret = of_property_read_u32(np, "ck_zero_constant", &dphy_timing->ck_zero_constant);
+	if (ret != 0)
+		dphy_timing->ck_zero_constant = CK_ZERO_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "ck_zero_ui", &dphy_timing->ck_zero_ui);
+	if (ret != 0)
+		dphy_timing->ck_zero_ui = CK_ZERO_UI_DEFAULT;
+
+	ret = of_property_read_u32(np, "ck_trail_constant", &dphy_timing->ck_trail_constant);
+	if (ret != 0)
+		dphy_timing->ck_trail_constant = CK_TRAIL_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "ck_zero_ui", &dphy_timing->ck_zero_ui);
+	if (ret != 0)
+		dphy_timing->ck_zero_ui = CK_TRAIL_UI_DEFAULT;
+
+	ret = of_property_read_u32(np, "req_ready", &dphy_timing->req_ready);
+	if (ret != 0)
+		dphy_timing->req_ready = REQ_READY_DEFAULT;
+
+	ret = of_property_read_u32(np, "wakeup_constant", &dphy_timing->wakeup_constant);
+	if (ret != 0)
+		dphy_timing->wakeup_constant = WAKEUP_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "wakeup_ui", &dphy_timing->wakeup_ui);
+	if (ret != 0)
+		dphy_timing->wakeup_ui = WAKEUP_UI_DEFAULT;
+
+	ret = of_property_read_u32(np, "lpx_constant", &dphy_timing->lpx_constant);
+	if (ret != 0)
+		dphy_timing->lpx_constant = LPX_CONSTANT_DEFAULT;
+
+	ret = of_property_read_u32(np, "lpx_ui", &dphy_timing->lpx_ui);
+	if (ret != 0)
+		dphy_timing->lpx_ui = LPX_UI_DEFAULT;
+}
+
+
+static void spacemit_dphy_core_get_status(struct spacemit_dphy_ctx *dphy_ctx)
+{
+	DRM_DEBUG("%s\n", __func__);
+
+	if (dphy_ctx == NULL) {
+		DRM_ERROR("%s: Invalid param\n", __func__);
+		return;
+	}
+
+	dphy_ctx->dphy_status0 = dsi_read(dphy_ctx->base_addr, DSI_PHY_STATUS_0);
+	dphy_ctx->dphy_status1 = dsi_read(dphy_ctx->base_addr, DSI_PHY_STATUS_1);
+	dphy_ctx->dphy_status2 = dsi_read(dphy_ctx->base_addr, DSI_PHY_STATUS_2);
+	DRM_DEBUG("%s: dphy_status0 = 0x%x\n", __func__, dphy_ctx->dphy_status0);
+	DRM_DEBUG("%s: dphy_status1 = 0x%x\n", __func__, dphy_ctx->dphy_status1);
+	DRM_DEBUG("%s: dphy_status2 = 0x%x\n", __func__, dphy_ctx->dphy_status2);
+}
+
+static void spacemit_dphy_core_reset(struct spacemit_dphy_ctx *dphy_ctx)
+{
+	DRM_DEBUG("%s\n", __func__);
+
+	if (dphy_ctx == NULL) {
+		DRM_ERROR("%s: Invalid param\n", __func__);
+		return;
+	}
+
+	dphy_ana_reset(dphy_ctx->base_addr);
+}
+
+/**
+ * spacemit_dphy_init - int spacemit dphy
+ *
+ * @dphy_ctx: pointer to the spacemit_dphy_ctx
+ *
+ * This function will be called by the dsi driver in order to init the dphy
+ * This function will do phy power on, enable continous clk, set dphy timing
+ * and set lane number.
+ *
+ * This function has no return value.
+ *
+ */
+static void spacemit_dphy_core_init(struct spacemit_dphy_ctx *dphy_ctx)
+{
+	DRM_DEBUG("%s\n", __func__);
+
+	if (dphy_ctx == NULL) {
+		DRM_ERROR("%s: Invalid param\n", __func__);
+		return;
+	}
+
+	if (dphy_ctx->status != DPHY_STATUS_UNINIT) {
+		DRM_WARN("%s: dphy_ctx has been initialized (%d)\n",
+			__func__, dphy_ctx->status);
+		return;
+	}
+
+	/*use DPHY_BIT_CLK_SRC_MUX as default clk src*/
+	dphy_set_bit_clk_src(dphy_ctx->base_addr, dphy_ctx->clk_src, dphy_ctx->half_pll5);
+
+	/* digital and analog power on */
+	dphy_set_power(dphy_ctx->base_addr, true);
+
+	/* turn on DSI continuous clock for HS */
+	dphy_set_cont_clk(dphy_ctx->base_addr, true);
+
+	/* set dphy */
+	dphy_set_timing(dphy_ctx);
+
+	/* enable data lanes */
+	dphy_set_lane_num(dphy_ctx->base_addr, dphy_ctx->lane_num);
+
+	dphy_ctx->status = DPHY_STATUS_INIT;
+}
+
+/**
+ * spacemit_dphy_uninit - unint spacemit dphy
+ *
+ * @dphy_ctx: pointer to the spacemit_dphy_ctx
+ *
+ * This function will be called by the dsi driver in order to unint the dphy
+ * This function will disable continous clk, reset dephy, power down dphy
+ *
+ * This function has no return value.
+ *
+ */
+static void spacemit_dphy_core_uninit(struct spacemit_dphy_ctx *dphy_ctx)
+{
+	DRM_DEBUG("%s\n", __func__);
+
+	if (dphy_ctx == NULL) {
+		DRM_ERROR("%s: Invalid param\n", __func__);
+		return;
+	}
+
+	if (dphy_ctx->status != DPHY_STATUS_INIT) {
+		DRM_WARN("%s: dphy_ctx has not been initialized (%d)\n",
+			__func__, dphy_ctx->status);
+		return;
+	}
+
+	dphy_set_cont_clk(dphy_ctx->base_addr, false);
+	dphy_ana_reset(dphy_ctx->base_addr);
+	dphy_set_power(dphy_ctx->base_addr, false);
+
+	dphy_ctx->status = DPHY_STATUS_UNINIT;
+}
+
+static int spacemit_dphy_core_parse_dt(struct spacemit_dphy_ctx *dphy_ctx, struct device_node *np)
+{
+	if (!dphy_ctx) {
+		DRM_ERROR("%s: Param is NULL\n", __func__);
+		return -1;
+	}
+
+	dphy_get_setting(dphy_ctx, np);
+
+	return 0;
+}
+
+
+static struct dphy_core_ops dphy_core_ops = {
+	.parse_dt = spacemit_dphy_core_parse_dt,
+	.init = spacemit_dphy_core_init,
+	.uninit = spacemit_dphy_core_uninit,
+	.reset = spacemit_dphy_core_reset,
+	.get_status = spacemit_dphy_core_get_status,
+};
+
+static struct ops_entry entry = {
+	.ver = "spacemit-dphy",
+	.ops = &dphy_core_ops,
+};
+
+#ifndef MODULE
+static int __init dphy_core_register(void)
+{
+	return dphy_core_ops_register(&entry);
+}
+
+subsys_initcall(dphy_core_register);
+#else
+int dphy_core_register(void)
+{
+	return dphy_core_ops_register(&entry);
+}
+
+#endif
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/dpu/dpu_debug.c b/drivers/gpu/drm/spacemit/dpu/dpu_debug.c
new file mode 100644
index 000000000000..b7b5ae4d98a2
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/dpu_debug.c
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/io.h>
+#include <linux/trace_events.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_framebuffer.h>
+#include "dpu_debug.h"
+#include "dpu_trace.h"
+#include "../spacemit_dpu_reg.h"
+#include "../spacemit_drm.h"
+
+#if IS_ENABLED(CONFIG_GKI_FIX_WORKAROUND)
+static struct file *gki_filp_open(const char *filename, int flags, umode_t mode)
+{
+	return 0;
+}
+static ssize_t gki_kernel_write(struct file *file, const void *buf, size_t count,
+			    loff_t *pos)
+{
+	return 0;
+}
+#endif
+#define DPU_BUFFER_DUMP_FILE "/mnt/dpu_buffer_dump"
+int dpu_buffer_dump(struct drm_plane *plane)
+{
+	unsigned int buffer_size = 0;
+	int i = 0;
+	void *mmu_tbl_vaddr = NULL;
+	phys_addr_t dpu_buffer_paddr = 0;
+	void __iomem *dpu_buffer_vaddr = NULL;
+	loff_t pos = 0;
+	static int dump_once = true;
+	struct file *filep = NULL;
+	struct spacemit_plane_state *spacemit_pstate = to_spacemit_plane_state(plane->state);
+
+	if (!dump_once)
+		return 0;
+
+	mmu_tbl_vaddr = spacemit_pstate->mmu_tbl.va;
+	buffer_size = plane->state->fb->obj[0]->size >> PAGE_SHIFT;
+
+#if IS_ENABLED(CONFIG_GKI_FIX_WORKAROUND)
+	filep = gki_filp_open(DPU_BUFFER_DUMP_FILE, O_RDWR | O_APPEND | O_CREAT, 0644);
+#else
+	filep = filp_open(DPU_BUFFER_DUMP_FILE, O_RDWR | O_APPEND | O_CREAT, 0644);
+#endif
+
+	if (IS_ERR(filep)) {
+		pr_err("Open file %s error\n", DPU_BUFFER_DUMP_FILE);
+		return -EINVAL;
+	}
+	for (i = 0; i < buffer_size; i++) {
+		dpu_buffer_paddr = *(volatile u32 __force *)mmu_tbl_vaddr;
+		dpu_buffer_paddr = dpu_buffer_paddr << PAGE_SHIFT;
+		dpu_buffer_vaddr = phys_to_virt((unsigned long)dpu_buffer_paddr);
+		mmu_tbl_vaddr += 4;
+#if IS_ENABLED(CONFIG_GKI_FIX_WORKAROUND)
+		gki_kernel_write(filep, (void *)dpu_buffer_vaddr, PAGE_SIZE, &pos);
+#else
+		kernel_write(filep, (void *)dpu_buffer_vaddr, PAGE_SIZE, &pos);
+#endif
+	}
+
+	filp_close(filep, NULL);
+	filep = NULL;
+
+	dump_once = false;
+
+	return 0;
+}
+
+void dpu_dump_fps(struct spacemit_crtc *a_crtc)
+{
+	struct timespec64 cur_tm, tmp_tm;
+
+	if (!a_crtc->enable_dump_fps)
+		return;
+
+	ktime_get_real_ts64(&cur_tm);
+	tmp_tm = timespec64_sub(cur_tm, a_crtc->last_tm);
+	a_crtc->last_tm.tv_sec = cur_tm.tv_sec;
+	a_crtc->last_tm.tv_nsec = cur_tm.tv_nsec;
+	if (tmp_tm.tv_sec == 0)
+		trace_printk("fps: %ld\n", 1000000000 / (tmp_tm.tv_nsec / 1000));
+}
+
+//static unsigned long last_exec_time = 0;
+void dpu_trace(struct work_struct *work)
+{
+	// unsigned long current_time = jiffies;
+	// if (time_after(current_time, last_exec_time + (60 * HZ))) {
+	// 	last_exec_time = current_time;
+	// 	logger_noti_helper(LOG_DUMP_NOTI_DPU);
+	// }
+}
diff --git a/drivers/gpu/drm/spacemit/dpu/dpu_debug.h b/drivers/gpu/drm/spacemit/dpu/dpu_debug.h
new file mode 100644
index 000000000000..adb5b518598d
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/dpu_debug.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _DPU_DEBUG_H_
+#define _DPU_DEBUG_H_
+
+#include <linux/types.h>
+//#include "saturn_regs/reg_map.h"
+#include "../spacemit_crtc.h"
+
+void dpu_dump_fps(struct spacemit_crtc *a_crtc);
+int dpu_buffer_dump(struct drm_plane *plane);
+void dpu_trace(struct work_struct *work);
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/dpu_saturn.c b/drivers/gpu/drm/spacemit/dpu/dpu_saturn.c
new file mode 100644
index 000000000000..94a9199385a5
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/dpu_saturn.c
@@ -0,0 +1,2006 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_qos.h>
+#include <linux/regmap.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/types.h>
+#include <linux/math64.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_writeback.h>
+#include "dpu_saturn.h"
+#include "saturn_fbcmem.h"
+#include "../spacemit_cmdlist.h"
+#include "../spacemit_dmmu.h"
+#include "../spacemit_dpu_reg.h"
+#include "../spacemit_drm.h"
+#include "../spacemit_wb.h"
+#include "post_process_hee.h"
+#include <video/display_timing.h>
+#include <dt-bindings/display/spacemit_dpu.h>
+#include "saturn_regs/ops_hee.h"
+#if IS_ENABLED(CONFIG_SPACEMIT_NI700)
+#include <soc/spacemit/ni700.h>
+#endif
+
+#define CREATE_TRACE_POINTS
+#include "dpu_trace.h"
+#include "dpu_debug.h"
+#include "../spacemit_bootloader.h"
+
+/* DPU MCLK */
+#define DPU_MCLK_MAX				409600000
+
+static void dpu_rpm_resume(struct spacemit_crtc *a_crtc);
+static void dpu_rpm_suspend(struct spacemit_crtc *a_crtc);
+
+enum {
+	DITHER_MODE_ARM = 0,
+	DITHER_MODE_PATTERN,
+	DITHER_MODE_MAX_LIMITED
+};
+
+#define TOTAL_RDMA_MEMSIZE	(68 * 1024) /* 68KB */
+#define DSC_COMP_COEF	(3) /* dsc compression coefficient */
+#define DSI_PHY_ANA_CTRL1 0x1E8
+//RDMA_FMT_YUV_420_P1_8, RDMA_FMT_YUV_420_P1_10 not support by hardware, has checked with asic
+//rdma hardware support RDMA_FMT_BGRA_16161616/RDMA_FMT_RGBA_16161616 formats are not  support by fourcc
+static const struct dpu_format_id primary_fmts[] = {
+	{ DRM_FORMAT_ABGR2101010, 1, 32 }, //RDMA_FMT_ABGR_2101010
+	{ DRM_FORMAT_ARGB8888, 4, 32 }, //RDMA_FMT_ARGB_8888
+	{ DRM_FORMAT_ABGR8888, 5, 32 }, //RDMA_FMT_ABGR_8888
+	{ DRM_FORMAT_RGBA8888, 6, 32 }, //RDMA_FMT_RGBA_8888
+	{ DRM_FORMAT_BGRA8888, 7, 32 }, //RDMA_FMT_BGRA_8888
+	{ DRM_FORMAT_XRGB8888, 8, 32 }, //RDMA_FMT_XRGB_8888
+	{ DRM_FORMAT_XBGR8888, 9, 32 }, //RDMA_FMT_XBGR_8888
+	{ DRM_FORMAT_RGBX8888, 10, 32 }, //RDMA_FMT_RGBX_8888
+	{ DRM_FORMAT_BGRX8888, 11, 32 }, //RDMA_FMT_BGRX_8888
+	{ DRM_FORMAT_RGB565, 22, 16 }, //RDMA_FMT_RGB_565
+	{ DRM_FORMAT_BGR565, 23, 16 }, //RDMA_FMT_BGR_565
+	{ DRM_FORMAT_RGB888, 12, 24 }, //RDMA_FMT_RGB_888
+	{ DRM_FORMAT_BGR888, 13, 24 }, //RDMA_FMT_BGR_888
+	/*
+	{ DRM_FORMAT_ARGB2101010,	 0 }, //RDMA_FMT_ARGB_2101010
+	{ DRM_FORMAT_RGBA1010102,	 2 }, //RDMA_FMT_RGBA_2101010
+	{ DRM_FORMAT_BGRA1010102,	 3 }, //RDMA_FMT_BGRA_2101010
+	{ DRM_FORMAT_RGB888,		12 }, //RDMA_FMT_RGB_888
+	{ DRM_FORMAT_BGR888,        13 }, //RDMA_FMT_BGR_888
+	{ DRM_FORMAT_RGBA5551,      14 }, //RDMA_FMT_RGBA_5551
+	{ DRM_FORMAT_BGRA5551,      15 }, //RDMA_FMT_BGRA_5551
+	{ DRM_FORMAT_ABGR1555,      16 }, //RDMA_FMT_ABGR_1555
+	{ DRM_FORMAT_ARGB1555,      17 }, //RDMA_FMT_ARGB_1555
+	{ DRM_FORMAT_RGBX5551,      18 }, //RDMA_FMT_RGBX_5551
+	{ DRM_FORMAT_BGRX5551,      19 }, //RDMA_FMT_BGRX_5551
+	{ DRM_FORMAT_XBGR1555,      20 }, //RDMA_FMT_XBGR_1555
+	{ DRM_FORMAT_XRGB1555,      21 }, //RDMA_FMT_XRGB_1555
+	{ DRM_FORMAT_ARGB16161616F, 24 }, //RDMA_FMT_ARGB_16161616
+	{ DRM_FORMAT_ABGR16161616F, 25 }, //RDMA_FMT_ABGR_16161616
+	{ DRM_FORMAT_XYUV8888,      32 }, //RDMA_FMT_XYUV_444_P1_8, uv_swap has no corresponding fourcc format
+	{ DRM_FORMAT_Y410,          33 }, //RDMA_FMT_XYUV_444_P1_10, uv_swap has no corresponding fourcc format
+	{ DRM_FORMAT_YUYV,          34 }, //RDMA_FMT_VYUY_422_P1_8
+	{ DRM_FORMAT_YVYU,          34 }, //RDMA_FMT_VYUY_422_P1_8, uv_swap = 1
+	{ DRM_FORMAT_UYVY,          35 }, //RDMA_FMT_YVYU_422_P1_8
+	{ DRM_FORMAT_VYUY,          35 }, //RDMA_FMT_YVYU_422_P1_8, uv_swap = 1
+	*/
+	{ DRM_FORMAT_YUV420_8BIT,   37, 12 }, //DRM_FORMAT_YUV420_8BIT for AFBC
+	{ DRM_FORMAT_NV12,          37, 12 }, //RDMA_FMT_YUV_420_P2_8
+	{ DRM_FORMAT_YVU420,        38, 12 }, //4CC: YV12
+
+	/*
+	{ DRM_FORMAT_NV21,          37 }, //RDMA_FMT_YUV_420_P2_8, uv_swap = 1
+	{ DRM_FORMAT_YUV420,        38 }, //RDMA_FMT_YUV_420_P3_8
+	{ DRM_FORMAT_YVU420,        38 }, //RDMA_FMT_YUV_420_P3_8, uv_swap = 1
+	{ DRM_FORMAT_YUV420_10BIT,  39 }, //RDMA_FMT_YUV_420_P1_10 //DPU not support, DO NOT use
+	*/
+	{ DRM_FORMAT_P010,          40, 24 }, //RDMA_FMT_YUV_420_P2_10
+	/*
+	{ DRM_FORMAT_Q410,          41 }, //DPU not support
+	{ DRM_FORMAT_Q401,          41 }, //DPU not support
+	*/
+};
+
+/* hee */
+const struct spacemit_hw_rdma saturn_hee_rdmas[] = {
+	/* TODO: set max_yuv_height to 1088 instead of 1080 due to vpu fw issue */
+	{FORMAT_RGB | FORMAT_AFBC, ROTATE_COMMON},
+	{FORMAT_RGB | FORMAT_AFBC | FORMAT_RAW_YUV | FORMAT_AFBC, ROTATE_COMMON | ROTATE_AFBC_90_270},
+	{FORMAT_RGB | FORMAT_AFBC, ROTATE_COMMON},
+	{FORMAT_RGB | FORMAT_AFBC, ROTATE_COMMON},
+};
+
+const u32 saturn_hee_fbcmem_sizes[] = {
+	89600,	//87.5k
+	15360,	//16k
+};
+EXPORT_SYMBOL(saturn_hee_fbcmem_sizes);
+
+struct spacemit_hw_device spacemit_dp_devices[SPACEMIT_DP_MAX_DEVICES] = {
+	[SATURN_HEE] = {
+		.base = NULL,		/* Parsed by dts */
+		.phy_addr = 0x0,	/* Parsed by dts */
+		.plane_nums = 16,
+		.offline_plane_nums = 1,
+		.crtc_nums = 2,
+		.rdma_nums = ARRAY_SIZE(saturn_hee_rdmas),
+		.rdmas = saturn_hee_rdmas,
+		.n_formats = ARRAY_SIZE(primary_fmts),
+		.formats = primary_fmts,
+		.n_fbcmems = ARRAY_SIZE(saturn_hee_fbcmem_sizes),
+		.fbcmem_sizes = saturn_hee_fbcmem_sizes,
+		.solid_color_shift = 0,
+		.hdr_coef_size = 135,
+		.hor_scale_coef_size = 48,
+		.ver_scale_coef_size = 48,
+		.scaler_num = 1,
+		.gamma_size = 257,
+		.etm_size = 65,
+		.acad_num = 149,
+		.is_acad_on = false,
+		.is_bl_save_on = false,
+		.dpu_version = SATURN_HEE,
+		.conf_dpuctrl_color_matrix = saturn_hee_conf_dpuctrl_color_matrix,
+		.check_end_matrix = saturn_hee_check_end_matrix,
+		.conf_dpuctrl_acad = saturn_hee_conf_dpuctrl_acad,
+		.conf_ee = saturn_hee_conf_dpuctrl_ee,
+		.update_csc_matrix = saturn_hee_update_csc_matrix,
+		.update_hdr_matrix = saturn_hee_update_hdr_matrix,
+		.conf_scaler_coefs = saturn_hee_conf_scaler_coefs,
+		.conf_scaler_x = saturn_hee_conf_scaler_x,
+		.enable_vsync = saturn_hee_enable_vsync,
+		.enable_cfg_irq = saturn_hee_enable_cfg_irq,
+		.cfg_ready = saturn_hee_cfg_ready,
+		.sw_start = saturn_hee_sw_start,
+		.irq_enable = saturn_hee_irq_enable,
+		.dpu_init = saturn_hee_dpu_init,
+		.conf_gamma_table = saturn_hee_conf_dpuctrl_pp_gamma,
+		.conf_matrix = saturn_hee_dpuctrl_color_temp,
+		.plane_update_hw_channel = saturn_hee_plane_update_hw_channel,
+		.plane_disable_hw_channel = saturn_hee_plane_disable_hw_channel,
+		.conf_dpuctrl = saturn_hee_conf_dpuctrl,
+		.wb_config = saturn_hee_wb_config,
+		.wb_disable = saturn_hee_wb_disable,
+		.is_wb_en = saturn_hee_is_wb_en,
+		.get_cfg_rdy = saturn_hee_get_cfg_rdy,
+		.get_int_sts = saturn_hee_get_int_sts,
+		.get_irq_bit = saturn_hee_get_irq_bit,
+		.clr_int_sts = saturn_hee_clr_int_sts,
+		.dpu_disable = saturn_hee_dpu_disable,
+		.dpu_restart = saturn_hee_dpu_restart,
+		.dpu_stop_check = saturn_hee_dpu_stop_check,
+		.enable_cmdlist = saturn_hee_enable_cmdlist,
+		.cfg_cmdlist = saturn_hee_cfg_cmdlist,
+		.rdma_contig_mem = saturn_hee_rdma_contig_mem,
+		.rdma_dmmu = saturn_hee_rdma_dmmu,
+		.wb_dmmu = saturn_hee_wb_dmmu,
+		.get_cl_rdma_buf = saturn_hee_get_cl_rdma_buf,
+		.cmdlist_fill_data_row = saturn_hee_cmdlist_fill_data_row,
+		.cmdlist_fill_conf_row = saturn_hee_cmdlist_fill_conf_row,
+		.wb_cmdlist = saturn_hee_wb_cmdlist,
+		.cmdlist_dump_node = saturn_hee_cmdlist_dump_node,
+#ifdef CONFIG_SPACEMIT_DEBUG
+		.dpu_dump_rdma_status = hee_dpu_dump_rdma_status,
+		.dpu_dump_reg = hee_dpu_dump_reg,
+#endif
+	},
+	[SATURN_EDP] = {
+		.base = NULL,		/* Parsed by dts */
+		.phy_addr = 0x0,	/* Parsed by dts */
+		.plane_nums = 16,
+		.offline_plane_nums = 1,
+		.crtc_nums = 2,
+		.rdma_nums = ARRAY_SIZE(saturn_hee_rdmas),
+		.rdmas = saturn_hee_rdmas,
+		.n_formats = ARRAY_SIZE(primary_fmts),
+		.formats = primary_fmts,
+		.n_fbcmems = ARRAY_SIZE(saturn_hee_fbcmem_sizes),
+		.fbcmem_sizes = saturn_hee_fbcmem_sizes,
+		.solid_color_shift = 0,
+		.hdr_coef_size = 135,
+		.hor_scale_coef_size = 48,
+		.ver_scale_coef_size = 48,
+		.scaler_num = 1,
+		.gamma_size = 257,
+		.etm_size = 65,
+		.acad_num = 149,
+		.is_acad_on = false,
+		.is_edp = true,
+		.is_bl_save_on = false,
+		.dpu_version = SATURN_HEE,
+		.conf_dpuctrl_color_matrix = saturn_hee_conf_dpuctrl_color_matrix,
+		.check_end_matrix = saturn_hee_check_end_matrix,
+		.conf_dpuctrl_acad = saturn_hee_conf_dpuctrl_acad,
+		.conf_ee = saturn_hee_conf_dpuctrl_ee,
+		.update_csc_matrix = saturn_hee_update_csc_matrix,
+		.update_hdr_matrix = saturn_hee_update_hdr_matrix,
+		.conf_scaler_coefs = saturn_hee_conf_scaler_coefs,
+		.conf_scaler_x = saturn_hee_conf_scaler_x,
+		.enable_vsync = saturn_hee_enable_vsync,
+		.enable_cfg_irq = saturn_hee_enable_cfg_irq,
+		.cfg_ready = saturn_hee_cfg_ready,
+		.sw_start = saturn_hee_sw_start,
+		.irq_enable = saturn_hee_irq_enable,
+		.dpu_init = saturn_hee_dpu_init,
+		.conf_gamma_table = saturn_hee_conf_dpuctrl_pp_gamma,
+		.conf_matrix = saturn_hee_dpuctrl_color_temp,
+		.plane_update_hw_channel = saturn_hee_plane_update_hw_channel,
+		.plane_disable_hw_channel = saturn_hee_plane_disable_hw_channel,
+		.conf_dpuctrl = saturn_hee_conf_dpuctrl,
+		.wb_config = saturn_hee_wb_config,
+		.wb_disable = saturn_hee_wb_disable,
+		.is_wb_en = saturn_hee_is_wb_en,
+		.get_cfg_rdy = saturn_hee_get_cfg_rdy,
+		.get_int_sts = saturn_hee_get_int_sts,
+		.get_irq_bit = saturn_hee_get_irq_bit,
+		.clr_int_sts = saturn_hee_clr_int_sts,
+		.dpu_disable = saturn_hee_dpu_disable,
+		.dpu_restart = saturn_hee_dpu_restart,
+		.dpu_stop_check = saturn_hee_dpu_stop_check,
+		.enable_cmdlist = saturn_hee_enable_cmdlist,
+		.cfg_cmdlist = saturn_hee_cfg_cmdlist,
+		.rdma_contig_mem = saturn_hee_rdma_contig_mem,
+		.rdma_dmmu = saturn_hee_rdma_dmmu,
+		.wb_dmmu = saturn_hee_wb_dmmu,
+		.get_cl_rdma_buf = saturn_hee_get_cl_rdma_buf,
+		.cmdlist_fill_data_row = saturn_hee_cmdlist_fill_data_row,
+		.cmdlist_fill_conf_row = saturn_hee_cmdlist_fill_conf_row,
+		.wb_cmdlist = saturn_hee_wb_cmdlist,
+		.cmdlist_dump_node = saturn_hee_cmdlist_dump_node,
+#ifdef CONFIG_SPACEMIT_DEBUG
+		.dpu_dump_rdma_status = hee_dpu_dump_rdma_status,
+		.dpu_dump_reg = hee_dpu_dump_reg,
+#endif
+	},
+};
+EXPORT_SYMBOL(spacemit_dp_devices);
+
+void saturn_enable_irq_mask(struct spacemit_crtc *a_crtc, bool enable, u32 offset, u32 mask)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	//unsigned long flags;
+	u32 irq_bit = hwdev->get_irq_bit(INT_VSYNC, a_crtc->dev_id) |
+		hwdev->get_irq_bit(INT_CFG_RDY, a_crtc->dev_id);
+
+	if (!hwdev->base) {
+		DRM_ERROR("hwdev->base NULL %s\n", __func__);
+		return;
+	}
+
+	if (!a_crtc->power_on)
+		return;
+
+	//spin_lock_irqsave(&a_crtc->irq_msk_lock, flags);
+	if (enable) {
+		//uboot -> kernel, clear online vsync status first
+		if ((a_crtc->is_offline_mode == 0) && (unlikely(a_crtc->logo_booton))) {
+			//write 1 to clr irq status
+			hwdev->clr_int_sts(a_crtc, irq_bit, a_crtc->dev_id);
+		}
+		dpu_set_bit(hwdev->base, offset, mask);
+	} else {
+		dpu_clr_bit(hwdev->base, offset, mask);
+	}
+
+	//spin_unlock_irqrestore(&a_crtc->irq_msk_lock, flags);
+}
+EXPORT_SYMBOL(saturn_enable_irq_mask);
+
+static void dpu_get_dsc_pxclk(struct spacemit_crtc *a_crtc, unsigned int dsc_bpc,
+	unsigned int dsc_bpp, struct drm_display_mode *mode, unsigned int fps)
+{
+	unsigned int width;
+
+	if (!mode || !dsc_bpc)
+		return;
+
+	width = mode->hdisplay;
+	width = width * dsc_bpp / DSC_COMP_COEF / dsc_bpc;  /* dsc_hout */
+	a_crtc->dsc_pxclk = (mode->htotal - mode->hdisplay + width) * mode->vtotal * fps;
+
+}
+
+static void dpu_get_dsc_info(struct spacemit_crtc *a_crtc, struct device_node *np)
+{
+	int bytes, ret, i;
+	unsigned int dsc_bpc;
+	unsigned int dsc_bpp;
+	unsigned int val;
+	const void *p;
+	struct drm_display_mode mode;
+
+	ret = of_property_read_u32(np, "dsc-enable", &val);
+	if (ret || !val)
+		return;
+
+	ret = of_property_read_u32(np, "dsc-bpc", &dsc_bpc);
+	if (ret) {
+		DRM_ERROR("of get mipi dsc-bpc failed\n");
+		return;
+	}
+
+	ret = of_property_read_u32(np, "dsc-bpp", &dsc_bpp);
+	if (ret) {
+		DRM_ERROR("of get mipi dsc-bpp failed\n");
+		return;
+	}
+
+	of_get_drm_display_mode(np, &mode, 0, 0);
+	val = mode.clock * 1000 / mode.htotal / mode.vtotal;
+	dpu_get_dsc_pxclk(a_crtc, dsc_bpc, dsc_bpp, &mode, val);
+
+	a_crtc->dsc_clk = 2 * mode.hdisplay * mode.vtotal * val;
+	a_crtc->dsc_x_slice = SPACEMIT_DSC_DEFALUT_X_SLICE;
+	p = of_get_property(np, "dsc-regs", &bytes);
+	if (p) {
+		a_crtc->dsc_regs = devm_kmalloc(a_crtc->dev, bytes, GFP_KERNEL);
+		if (IS_ERR(a_crtc->dsc_regs)) {
+			DRM_ERROR("alloc dsc reg buffer(size:0x%x) failed\n", bytes);
+			return;
+		}
+		a_crtc->dsc_regs_len = bytes / sizeof(u32);
+		for (i = 0; i < a_crtc->dsc_regs_len; i++)
+			a_crtc->dsc_regs[i] = be32_to_cpup((u32 *)p + i);
+	} else {
+		DRM_ERROR("can't find dsc-regs property\n");
+		a_crtc->dsc_regs = NULL;
+		a_crtc->dsc_regs_len = 0;
+		return;
+	}
+
+	ret = of_property_read_u32(np, "dsc-x-slice", &a_crtc->dsc_x_slice);
+	if (ret || a_crtc->dsc_x_slice > SPACEMIT_DSC_MAX_X_SLICE || !a_crtc->dsc_x_slice) {
+		DRM_ERROR("of get dsc-x-slice failed\n");
+		a_crtc->dsc_x_slice = SPACEMIT_DSC_DEFALUT_X_SLICE;
+		return;
+	}
+
+	if (a_crtc->dsc_regs && a_crtc->dsc_x_slice)
+		a_crtc->dsc_clk /= a_crtc->dsc_x_slice;
+}
+
+void spacemit_dpu_power_enable(struct spacemit_crtc *a_crtc, bool enable)
+{
+	if (enable) {
+		if (a_crtc->power_on)
+			return;
+
+		pm_runtime_get_sync(a_crtc->dev);
+		a_crtc->power_on = true;
+	} else {
+		if (!a_crtc->power_on)
+			return;
+
+		a_crtc->power_on = false;
+		pm_runtime_put_sync(a_crtc->dev);
+
+	}
+
+}
+
+#ifdef CONFIG_PM
+static void dpu_lpm_work_func(struct work_struct *work)
+{
+	struct spacemit_crtc *a_crtc = container_of(work, struct spacemit_crtc,
+						lpm_qos_work.work);
+	dpu_rpm_suspend(a_crtc);
+}
+#endif
+
+static void dpu_get_pipe_out_node(struct spacemit_crtc *a_crtc)
+{
+	struct device_node *ports, *port, *ep;
+	struct device_node *remote; /* dsi */
+	struct device_node *child;  /* panel candidate */
+	u32 reg;
+
+	a_crtc->dsi_node = NULL;
+	a_crtc->panel_node = NULL;
+
+	ports = of_get_child_by_name(a_crtc->dev->of_node, "ports");
+	if (!ports) {
+		DRM_ERROR("no ports under %pOF\n", a_crtc->dev->of_node);
+		return;
+	}
+
+	for_each_child_of_node(ports, port) {
+		if (of_property_read_u32(port, "reg", &reg))
+			continue;
+		if (reg != 0)
+			continue;
+
+		ep = of_graph_get_endpoint_by_regs(
+				a_crtc->dev->of_node, reg, 0);
+		if (!ep)
+			continue;
+
+		remote = of_graph_get_remote_port_parent(ep);
+		if (!remote || !of_device_is_available(remote)) {
+			of_node_put(ep);
+			of_node_put(remote);
+			continue;
+		}
+
+		a_crtc->dsi_node = remote;
+
+		DRM_DEBUG(" found dsi node: %pOF\n", remote);
+
+		for_each_child_of_node(remote, child) {
+
+			if (of_node_name_eq(child, "ports"))
+				continue;
+
+			if (!of_device_is_available(child))
+				continue;
+
+			if (!of_property_read_bool(child, "compatible"))
+				continue;
+
+			DRM_DEBUG(" found panel node: %pOF\n", child);
+			a_crtc->panel_node = child;
+			break;
+		}
+
+		of_node_put(ep);
+
+		if (!a_crtc->panel_node) {
+			DRM_ERROR("no panel found under dsi %pOF\n", remote);
+		} else {
+			DRM_DEBUG("SUCCESS: crtc %pOF -> dsi %pOF -> panel %pOF\n",
+				 a_crtc->dev->of_node,
+				 a_crtc->dsi_node,
+				 a_crtc->panel_node);
+		}
+		return;
+	}
+
+	DRM_ERROR("failed to get dsi/panel node\n");
+}
+
+static void dpu_parse_panel_dt(struct spacemit_crtc *a_crtc)
+{
+	struct device_node *lcd_node = NULL;
+	const char *str;
+	char lcd_path[60];
+	uint32_t value;
+	int rc;
+
+	dpu_get_pipe_out_node(a_crtc);
+	if (!a_crtc->panel_node)
+		return;
+
+	rc = of_property_read_string(a_crtc->panel_node, "force-attached", &str);
+	if (rc) {
+		DRM_WARN("get panel string failed.\n");
+		goto node_put;
+	}
+
+	sprintf(lcd_path, "/lcds/%s", str);
+	lcd_node = of_find_node_by_path(lcd_path);
+	if (!lcd_node) {
+		DRM_WARN("find lcd node failed.\n");
+		goto node_put;
+	}
+
+	a_crtc->dsc_clk = DPU_DSCCLK_DEFAULT;
+	dpu_get_dsc_info(a_crtc, lcd_node);
+
+	if (of_property_read_u32(lcd_node, "work-mode", &a_crtc->out_mode))
+		a_crtc->out_mode = DPU_OUT_MODE_VIDEO; /* default is video mode */
+
+	if (of_property_read_bool(lcd_node, "oled"))
+		a_crtc->is_oled = true;
+	else
+		a_crtc->is_oled = false;
+
+	DRM_INFO("is_oled: %d\n", a_crtc->is_oled);
+
+	rc = of_property_read_string(lcd_node, "dsi-color-format", &str);
+	if (rc)
+		a_crtc->out_format = OUTFMT_RGB888;
+	else if (!strcmp(str, "rgb888"))
+		a_crtc->out_format = OUTFMT_RGB888;
+	else if (!strcmp(str, "rgb666"))
+		a_crtc->out_format = OUTFMT_RGB666;
+	else if (!strcmp(str, "rgb666_packed"))
+		a_crtc->out_format = OUTFMT_RGB666;
+	else if (!strcmp(str, "rgb565"))
+		a_crtc->out_format = OUTFMT_RGB565;
+	else
+		DRM_ERROR("dsi-color-format (%s) is not supported\n", str);
+
+	rc = of_property_read_u32(lcd_node, "dither-mode", &value);
+	if (!rc && value < DITHER_MODE_MAX_LIMITED)
+		a_crtc->dither_mode = value;
+	else
+		a_crtc->dither_mode = DITHER_MODE_ARM;
+
+	if (!of_property_read_u32(lcd_node, "spacemit-dpu-min-mclk", &value))
+		a_crtc->min_mclk = value;
+
+	if (!of_property_read_u32(lcd_node, "spacemit-dsi-escclk", &value))
+		a_crtc->escclk = value;
+
+	if (!of_property_read_u32(lcd_node, "split-enable", &value))
+		a_crtc->split_en = value;
+
+	if (a_crtc->dsipll_valid) {
+		if (of_property_read_u32(lcd_node, "spacemit-dpu-dsipll-reg0", &a_crtc->dsipll_reg0))
+			a_crtc->dsipll_reg0 = DPU_DSIPLL_REG0_DEFAULT;
+		if (of_property_read_u32(lcd_node, "spacemit-dpu-dsipll-reg1", &a_crtc->dsipll_reg1))
+			a_crtc->dsipll_reg1 = DPU_DSIPLL_REG1_DEFAULT;
+		if (of_property_read_u32(lcd_node, "spacemit-dpu-dsipll-reg2", &a_crtc->dsipll_reg2))
+			a_crtc->dsipll_reg2 = DPU_DSIPLL_REG2_DEFAULT;
+	}
+
+node_put:
+	of_node_put(a_crtc->panel_node);
+}
+
+static int dpu_parse_dt(struct spacemit_crtc *a_crtc, struct device_node *np)
+{
+	struct dpu_clk_context *clk_ctx = &a_crtc->clk_ctx;
+	struct resource *r;
+	struct platform_device *pdev = to_platform_device(a_crtc->dev);
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	return 0;
+#endif
+	clk_ctx->pxclk = of_clk_get_by_name(np, "pxclk");
+	if (IS_ERR(clk_ctx->pxclk)) {
+		pr_err("%s, read pxclk failed from dts!\n", __func__);
+		return PTR_ERR(clk_ctx->pxclk);
+	}
+
+	clk_ctx->mclk = of_clk_get_by_name(np, "mclk");
+	if (IS_ERR(clk_ctx->mclk)) {
+		pr_err("%s, read mclk failed from dts!\n", __func__);
+		return PTR_ERR(clk_ctx->mclk);
+	}
+
+	clk_ctx->hclk = of_clk_get_by_name(np, "hclk");
+	if (IS_ERR(clk_ctx->hclk)) {
+		clk_ctx->hclk = NULL;
+		pr_err("%s, read hclk failed from dts!\n", __func__);
+		// return PTR_ERR(clk_ctx->hclk);
+	}
+
+	if (!a_crtc->is_edp) {
+		clk_ctx->escclk = of_clk_get_by_name(np, "escclk");
+		if (IS_ERR(clk_ctx->escclk)) {
+			pr_err("%s, read escclk failed from dts!\n", __func__);
+			return PTR_ERR(clk_ctx->escclk);
+		}
+		clk_ctx->bitclk = of_clk_get_by_name(np, "bitclk");
+		if (IS_ERR(clk_ctx->bitclk)) {
+			clk_ctx->bitclk = NULL;
+			DRM_INFO("%s, read bitclk failed from dts!\n", __func__);
+		}
+		if (of_property_read_u32(np, "spacemit-dpu-bitclk", &a_crtc->bitclk))
+			a_crtc->bitclk = DPU_BITCLK_DEFAULT;
+		if (of_property_read_u32(np, "spacemit-dsi-escclk", &a_crtc->escclk))
+			a_crtc->escclk = DPU_ESCCLK_DEFAULT;
+	} else {
+		clk_ctx->escclk = of_clk_get_by_name(np, "escclk");
+		if (IS_ERR(clk_ctx->escclk)) {
+			pr_err("%s, read escclk failed from dts!\n", __func__);
+			clk_ctx->escclk = NULL;
+			// return PTR_ERR(clk_ctx->escclk);
+		}
+		clk_ctx->bitclk = NULL;
+	}
+
+	clk_ctx->aclk = of_clk_get_by_name(np, "aclk");
+	if (IS_ERR(clk_ctx->aclk)) {
+		clk_ctx->aclk = NULL;
+		DRM_INFO("%s, read aclk failed from dts!\n", __func__);
+	}
+
+	clk_ctx->dscclk = of_clk_get_by_name(np, "dscclk");
+	if (IS_ERR(clk_ctx->dscclk)) {
+		clk_ctx->dscclk = NULL;
+		DRM_INFO("%s, read dscclk failed from dts!\n", __func__);
+	}
+
+	if (of_property_read_u32(np, "spacemit-dpu-min-mclk", &a_crtc->min_mclk))
+		a_crtc->min_mclk = DPU_MCLK_DEFAULT;
+
+	if (of_property_read_u32(np, "spacemit-dpu-max-mclk", &a_crtc->max_mclk) ||
+			(a_crtc->max_mclk > DPU_MCLK_MAX))
+		a_crtc->max_mclk = DPU_MCLK_MAX;
+
+	// if (of_property_read_bool(np, "spacemit-dpu-auto-fc"))
+	// 	a_crtc->enable_auto_fc = 1;
+
+	if (of_property_read_bool(np, "spacemit-dpu-dsipll"))
+		a_crtc->dsipll_valid = true;
+
+	//crtc is_offline_mode dts
+	if (a_crtc->dsipll_valid || a_crtc->is_edp)
+		a_crtc->is_offline_mode = 0;
+	else
+		a_crtc->is_offline_mode = 1;
+
+	if (of_property_read_u32(np, "spacemit-dpu-aclk", &a_crtc->aclk))
+		a_crtc->aclk = DPU_AXICLK_DEFAULT;
+
+	if (of_property_read_u64(np, "spacemit-dpu-bw-margin", &a_crtc->bw_margin))
+		a_crtc->bw_margin = 0;
+	a_crtc->max_bw = DPU_MAX_QOS_REQ - a_crtc->bw_margin;
+
+#ifdef CONFIG_PM
+	if (of_property_read_u32(np, "lpm-commit-qos", &a_crtc->lpm_commit_qos)) {
+		DRM_INFO("can not get lcd lpm commit qos value\n");
+		a_crtc->lpm_commit_qos = 0;
+	}
+
+	if (!a_crtc->is_offline_mode) {
+		if (of_property_read_u32(np, "lpm-bl-qos", &a_crtc->lpm_bl_qos)) {
+			DRM_INFO("can not get lcd bl lpm qos value\n");
+			a_crtc->lpm_bl_qos = 0;
+		}
+	}
+
+	if (of_property_read_u32(np, "lpm-period", &a_crtc->lpm_period)) {
+		DRM_INFO("can not get lcd lpm period value\n");
+		a_crtc->lpm_period = SPACEMIT_DPU_LPM_PERIOD_MIN_MS;
+	}
+
+	if (a_crtc->lpm_period) {
+		INIT_DELAYED_WORK(&a_crtc->lpm_qos_work, dpu_lpm_work_func);
+		a_crtc->lpm_work_pending = false;
+	}
+#endif
+
+	if (!a_crtc->is_offline_mode && !a_crtc->is_edp)
+		/* must be the last in parse process */
+		dpu_parse_panel_dt(a_crtc);
+
+	if (a_crtc->dsipll_valid) {
+		r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		a_crtc->dsipll_base = devm_ioremap_resource(&pdev->dev, r);
+		if (IS_ERR(a_crtc->dsipll_base))
+			return PTR_ERR(a_crtc->dsipll_base);
+
+		if (a_crtc->split_en) {
+			r = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+			a_crtc->dsi1pll_base = devm_ioremap_resource(&pdev->dev, r);
+			if (IS_ERR(a_crtc->dsi1pll_base))
+				return PTR_ERR(a_crtc->dsi1pll_base);
+
+		}
+	}
+
+	return 0;
+}
+
+static unsigned int dpu_get_bpp(u32 format)
+{
+	unsigned int i = 0;
+
+	for (i = 0; i < ARRAY_SIZE(primary_fmts); i++) {
+		if (format == primary_fmts[i].format)
+			return primary_fmts[i].bpp;
+	}
+
+	DRM_ERROR("format 0x%x is not supported!\n", format);
+	return SPACEMIT_DPU_INVALID_FORMAT_ID;
+}
+
+
+static int dpu_calc_plane_mclk_bw(struct drm_plane *plane,
+		struct drm_plane_state *new_state)
+{
+	/* For some platform without aclk, mclk = max(aclk, mclk) */
+	uint64_t calc_mclk = 0;
+	uint64_t Fpixclk_hblk = 0;
+	struct drm_plane_state *state = NULL;
+	struct drm_crtc *crtc = new_state->crtc;
+	struct drm_display_mode *mode = NULL;
+	struct spacemit_plane_state *spacemit_plane_state = NULL;
+	uint64_t tmp;
+	uint64_t hact = 0;
+	uint64_t fps = 0;
+	bool scl_en = 0;
+	unsigned int C = 0;
+	uint64_t width_ratio = 0;	/* (scl_in_width/hact) * 1000000 */
+	uint64_t height_ratio = 0;	/* roundup(scl_in_height/scl_out_height) */
+	uint64_t scl_in_width, scl_in_height, scl_out_width, scl_out_height;
+	uint64_t fscl_in = 0;
+
+	struct drm_framebuffer *fb = new_state->fb;
+	const struct drm_format_info *format = fb->format;
+	unsigned int bpp = dpu_get_bpp(format->format);
+	uint64_t calc_bandwidth = 0;
+	unsigned long img_width = 0;
+
+	struct spacemit_crtc *a_crtc = NULL;
+
+	state = new_state;
+	if (!crtc)
+		return 0;
+
+	a_crtc = to_spacemit_crtc(crtc);
+	if (!a_crtc->enable_auto_fc)
+		return 0;
+
+	/* prepare calc mclk and bw */
+	mode = &crtc->mode;
+	hact = mode->hdisplay;
+	fps = mode->clock * 1000 / (mode->htotal * (u16)mode->vtotal);
+	spacemit_plane_state = to_spacemit_plane_state(state);
+	scl_en = spacemit_plane_state->use_scl;
+	Fpixclk_hblk = hact * (mode->vtotal) * fps; /* MHZ */
+
+	trace_dpu_fpixclk_hblk(hact, fps, (uint64_t)mode->vtotal, Fpixclk_hblk);
+
+	/* calculate no_scl and scl_up mclk and bw */
+	if (is_rot_90_270(state->rotation))
+		img_width = state->src_h >> 16;
+	else
+		img_width = state->src_w >> 16;
+
+	trace_u64_data("img_width", img_width);
+	//calc_mclk = ( hact + 32 ) * MHZ2HZ / hact * Fpixclk_hblk / 2 / MHZ2HZ * 1.1;
+	tmp = (hact + 32) * MHZ2HZ * 110;
+	do_div(tmp, hact);
+	tmp = tmp * Fpixclk_hblk;
+	tmp = tmp >> 1;
+	do_div(tmp, MHZ2HZ);
+	do_div(tmp, 100);
+	calc_mclk = tmp;
+	if (calc_mclk > a_crtc->max_mclk) {
+		DRM_INFO("plane:%d mclk too large %lld\n", state->zpos, calc_mclk);
+		DRM_INFO("img_width = %ld\n", img_width);
+		return -EINVAL;
+	}
+	//calc_bandwidth = Fpixclk_hblk * bpp * img_width / hact / 8;
+	tmp = Fpixclk_hblk * bpp * img_width;
+	do_div(tmp, hact);
+	do_div(tmp, 8);
+	calc_bandwidth = tmp;
+	if (calc_bandwidth > (a_crtc->max_bw * MHZ2KHZ)) {
+		DRM_INFO("plane:%d bandwidth too large %lld\n", state->zpos, calc_bandwidth);
+		DRM_INFO("img_width = %ld\n", img_width);
+		return -EINVAL;
+	}
+
+	/* calculate scl_en mclk and bandwidth */
+	if (scl_en) {
+		if (is_rot_90_270(state->rotation)) {
+			scl_in_width  = state->src_h >> 16;
+			scl_in_height = state->src_w >> 16;
+		} else {
+			scl_in_width  = state->src_w >> 16;
+			scl_in_height = state->src_h >> 16;
+		}
+
+		scl_out_width = state->crtc_w;
+		scl_out_height = state->crtc_h;
+
+		C = min(2 * scl_in_width / scl_out_width, 4ULL);	/* sclaer after rdma */
+		spacemit_plane_state->afbc_effc = 4 / C;
+
+		//width_ratio = scl_in_width * MHZ2HZ / hact;
+		tmp = scl_in_width * MHZ2HZ;
+		do_div(tmp, hact);
+		width_ratio = tmp;
+		height_ratio = DIV_ROUND_UP_ULL(scl_in_height, scl_out_height) * MHZ2HZ;
+
+		/* if scale down */
+		if ((width_ratio > MHZ2HZ) || (height_ratio > MHZ2HZ)) {
+			/*
+			 * fscl_in = Fpixclk_hblk * max(width_ratio, MHZ2HZ) / MHZ2HZ * max(height_ratio, MHZ2HZ) / MHZ2HZ / C * 1.16
+			 * fscl_out = (hact+32)/hact * Fpixclk_hblk/2*1.1, same as current calc_mclk.
+			 * calc_mclk = max(fscl_in, fscl_out);
+			 */
+			tmp = Fpixclk_hblk * max(width_ratio, MHZ2HZ) * 116;
+			do_div(tmp, MHZ2HZ);
+			tmp = tmp * max(height_ratio, MHZ2HZ);
+			do_div(tmp, MHZ2HZ);
+			do_div(tmp, C);
+			do_div(tmp, 100);
+			fscl_in = tmp;
+			calc_mclk = max(fscl_in, calc_mclk);
+
+			if (calc_mclk > a_crtc->max_mclk) {
+				DRM_INFO("plane:%d mclk too large %lld\n", state->zpos, calc_mclk);
+				DRM_INFO("hact = %lld, fps = %lld, vtotal = %d, Fpixclk_hblk = %lld\n", hact, fps, (u16)mode->vtotal, Fpixclk_hblk);
+				DRM_INFO("scl_in_width = %lld, scl_in_height = %lld, scl_out_width = %lld, scl_out_height = %lld, bpp = %d, C = %d\n", scl_in_width, scl_in_height, scl_out_width, scl_out_height, bpp, C);
+				return -EINVAL;
+			}
+
+			//calc_bandwidth = Fpixclk_hblk * bpp * width_ratio / MHZ2HZ * height_ratio / MHZ2HZ / 8;
+			tmp = Fpixclk_hblk * bpp * width_ratio;
+			do_div(tmp, MHZ2HZ);
+			tmp = tmp * height_ratio;
+			do_div(tmp, MHZ2HZ);
+			tmp = tmp >> 3;
+			calc_bandwidth = tmp;
+			if (calc_bandwidth > (a_crtc->max_bw * MHZ2KHZ)) {
+				DRM_INFO("plane:%d bandwidth too large %lld\n", state->zpos, calc_bandwidth);
+				DRM_INFO("hact = %lld, fps = %lld, vtotal = %d, Fpixclk_hblk = %lld\n", hact, fps, (u16)mode->vtotal, Fpixclk_hblk);
+				DRM_INFO("scl_in_width = %lld, scl_in_height = %lld, scl_out_width = %lld, scl_out_height = %lld, bpp = %d, C = %d\n", scl_in_width, scl_in_height, scl_out_width, scl_out_height, bpp, C);
+				return -EINVAL;
+			}
+			trace_dpu_mclk_scl(scl_in_width, scl_in_height, scl_out_width, scl_out_height, bpp, C, width_ratio, height_ratio);
+		}
+	}
+
+	a_crtc = to_spacemit_crtc(crtc);
+	if (calc_mclk < a_crtc->min_mclk)
+		calc_mclk = a_crtc->min_mclk;
+
+	if (calc_bandwidth < DPU_MIN_QOS_REQ)
+		calc_bandwidth = DPU_MIN_QOS_REQ;
+
+	spacemit_plane_state->mclk = calc_mclk;
+
+	/* add some buffer for MMU and AFBC Header */
+	//calc_bandwidth = calc_bandwidth * 115 / 100;
+	tmp = calc_bandwidth * 115;
+	do_div(tmp, 100);
+	calc_bandwidth = tmp;
+	spacemit_plane_state->bw = calc_bandwidth;
+
+	trace_u64_data("plane calc_mclk", calc_mclk);
+	trace_u64_data("plane calc_bw", calc_bandwidth);
+	return 0;
+}
+
+static int dpu_update_clocks(struct spacemit_crtc *a_crtc, uint64_t mclk)
+{
+	struct dpu_clk_context *clk_ctx = &a_crtc->clk_ctx;
+	uint64_t cur_mclk = 0;
+	int ret = 0;
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	return 0;
+#endif
+	//offline does not change clk
+	if (a_crtc->is_offline_mode || a_crtc->is_edp || !clk_ctx->mclk)
+		return 0;
+
+	trace_u64_data("update mclk", mclk);
+	cur_mclk = clk_get_rate(clk_ctx->mclk);
+	if (cur_mclk == mclk)
+		return 0;
+
+	ret = clk_set_rate(clk_ctx->mclk, mclk);
+	if (unlikely(ret)) {
+		trace_u64_data("Failed to set mclk", mclk);
+		DRM_ERROR("Failed to set DPU MCLK %llu %d\n", mclk, ret);
+		WARN_ON_ONCE(ret);
+	} else {
+		a_crtc->cur_mclk = clk_get_rate(clk_ctx->mclk);
+		trace_u64_data("Pass to set mclk", mclk);
+	}
+
+	return ret;
+}
+
+static int dpu_update_bw(struct spacemit_crtc *a_crtc, uint64_t bw)
+{
+	uint64_t __maybe_unused tmp;
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	return 0;
+#endif
+	trace_u64_data("update bw", bw);
+	if (a_crtc->cur_bw == bw)
+		return 0;
+#ifdef CONFIG_PM
+#if IS_ENABLED(CONFIG_SPACEMIT_DDR_FC)
+	tmp = bw;
+	do_div(tmp, MHZ2KHZ);
+	update_spacemit_ddr_bw_read_req(a_crtc->ddr_qos_cons, tmp);
+	a_crtc->cur_bw = bw;
+#endif
+#endif
+
+	return 0;
+}
+
+static const struct pll_freq_range_t pll_freq_table[32] = {
+	{780,  1010, 1090},
+	{1090, 1170, 1250},
+	{1250, 1330, 1410},
+	{1410, 1490, 1570},
+	{1570, 1650, 1730},
+	{1730, 1810, 1890},
+	{1890, 1970, 2050},
+	{2050, 2130, 2210},
+	{2210, 2290, 2370},
+	{2370, 2450, 2530},
+	{2530, 2610, 2695},
+	{2695, 2780, 2860},
+	{2860, 2940, 3020},
+	{3020, 3100, 3185},
+	{3185, 3270, 3350},
+	{3350, 3430, 3510},
+
+	{3510, 3590, 3675},
+	{3675, 3760, 3840},
+	{3840, 3920, 4000},
+	{4000, 4080, 4165},
+	{4165, 4250, 4330},
+	{4330, 4410, 4485},
+	{4485, 4560, 4650},
+	{4650, 4740, 4825},
+	{4825, 4910, 4990},
+	{4990, 5070, 5150},
+	{5150, 5230, 5315},
+	{5315, 5400, 5480},
+	{5480, 5560, 5640},
+	{5640, 5720, 5805},
+	{5805, 5890, 5970},
+	{5970, 6050, 6500}
+};
+
+static int pll_get_rate_sel(uint32_t vco_freq)
+{
+	if (vco_freq < 2000)
+		return 0;
+	else if (vco_freq < 4000)
+		return 1;
+	else if (vco_freq <= 6500)
+		return 2;
+	else
+		return -1;
+}
+
+static int pll_get_range_index(uint32_t vco_freq)
+{
+	for (int i = 0; i < 32; i++) {
+		if (vco_freq > pll_freq_table[i].low && vco_freq <= pll_freq_table[i].high)
+			return i;
+	}
+	return -1;
+}
+
+/* pll_reg5 / pll_reg6 */
+static uint8_t pll_make_range_reg(int range_idx)
+{
+	return (uint8_t)((0b100 << 5) | (range_idx & 0x1F));
+}
+
+/* pll_reg7 */
+static uint8_t pll_make_rate_reg(int rate_sel)
+{
+	if (rate_sel < 0)
+		return 0xFF;
+
+	/* 01 | rate_sel | 0101 */
+	return (uint8_t)((0x01 << 6) |
+			(rate_sel << 4) |
+			0x05);
+}
+
+/* ===================== ctrl_reg0 ===================== */
+static u32 pll_make_ctrl_reg0(u32 vco_freq, int rate_sel)
+{
+	u32 denom = PLL_REF_FREQ_MHZ;
+	u32 div_int;
+	u64 frac22;
+	u32 remainder;
+
+	if (rate_sel >= 0)
+		denom = PLL_REF_FREQ_MHZ  * (rate_sel + 1);
+
+	/* integer part */
+	div_int = vco_freq / denom;
+
+	/* fractional part: round((vco % denom) / denom * 2^22) */
+	remainder = vco_freq % denom;
+	frac22 = (u64)remainder << 22;
+	frac22 += denom / 2;
+	do_div(frac22, denom);   /* frac22 is 22-bit */
+
+	return ((div_int & 0xff) << 24) | ((u32)frac22 & 0x00ffffff);
+}
+
+/* ===================== ctrl_reg1 ===================== */
+static uint32_t pll_make_ctrl_reg1(uint32_t vco_freq, int pllmode, int rate_sel)
+{
+	uint8_t pll_reg4, pll_reg5, pll_reg6, pll_reg7;
+
+	int range_idx = pll_get_range_index(vco_freq);
+	if (range_idx < 0)
+		return 0xFFFFFFFF;
+
+	pll_reg4 = (uint8_t)(3 + pllmode * 8);
+	pll_reg5 = pll_make_range_reg(range_idx);
+	pll_reg6 = pll_reg5;
+	pll_reg7 = pll_make_rate_reg(rate_sel);
+
+	return (pll_reg7 << 24) | (pll_reg6 << 16) |
+		(pll_reg5 << 8) | pll_reg4;
+}
+
+/*
+ * PLL_CTRL_REG2[30:29] div_sel encoding:
+ *   0b00: pll_div2  → VCO / 2  (VCO = bitclock × 2)
+ *   0b01: pll_div2  → VCO / 2  (VCO = bitclock × 2, same as 0b00)
+ *   0b10: pll_div8  → VCO / 8  (VCO = bitclock × 8)
+ *   0b11: pll_divm  → VCO / m  (not used)
+ */
+static const struct {
+	uint32_t mult;     /* VCO = bitclock * mult */
+	uint32_t div_sel;  /* PLL_CTRL_REG2[30:29] */
+} pll_vco_candidates[] = {
+	{  2, 0 }, /* pll_div2 (0b00) */
+	{  2, 1 }, /* pll_div2 (0b01) */
+	{  8, 2 }, /* pll_div8 (0b10) */
+};
+
+static int spacemit_calc_pll_regs(uint32_t bitclock, uint32_t *pll_ctrl_reg0,
+					uint32_t *pll_ctrl_reg1, uint32_t *div_sel_out)
+{
+	const int pllmode = PLL_MODE_5G_PLL;
+	uint32_t vco_freq = 0;
+	uint32_t div_sel = 0;
+	int rate_sel;
+
+	/* If bitclock itself is already within the PLL VCO range,
+	 * use it directly without any post-divider multiplication.
+	 * NOTE: div_sel for "no division" case needs hardware clarification.
+	 */
+	if (pll_get_rate_sel(bitclock) >= 0 && pll_get_range_index(bitclock) >= 0) {
+		vco_freq = bitclock;
+		div_sel  = 0;
+		DRM_DEBUG("pll vco: bitclock=%u MHz already in range, no division needed\n",
+			 bitclock);
+	} else {
+		/* Try each multiplier in order until we find a VCO that falls in range */
+		for (int i = 0; i < ARRAY_SIZE(pll_vco_candidates); i++) {
+			uint32_t vco = bitclock * pll_vco_candidates[i].mult;
+
+			if (pll_get_rate_sel(vco) >= 0 && pll_get_range_index(vco) >= 0) {
+				vco_freq = vco;
+				div_sel  = pll_vco_candidates[i].div_sel;
+				DRM_INFO("pll vco selected: bitclock=%u MHz, mult=%u, vco=%u MHz, div_sel=%u\n",
+					 bitclock, pll_vco_candidates[i].mult, vco_freq, div_sel);
+				break;
+			}
+		}
+	}
+
+	if (!vco_freq) {
+		DRM_ERROR("no valid VCO config found for bitclock=%u MHz\n", bitclock);
+		return -EINVAL;
+	}
+
+	rate_sel = pll_get_rate_sel(vco_freq);
+
+	*pll_ctrl_reg0 = pll_make_ctrl_reg0(vco_freq, rate_sel);
+	*pll_ctrl_reg1 = pll_make_ctrl_reg1(vco_freq, pllmode, rate_sel);
+	*div_sel_out = div_sel;
+
+	return 0;
+}
+
+static void dpu_disable_dsipll(struct spacemit_crtc *a_crtc)
+{
+	if (a_crtc->split_en)
+		return;
+
+	/* disable pu */
+	writel(0, a_crtc->dsipll_base + PLL_CTRL_REG2);
+}
+
+static void dpu_enable_dsipll(struct spacemit_crtc *a_crtc)
+{
+	u32 pll_ctrl_reg0;
+	u32 pll_ctrl_reg1;
+	u32 div_sel;
+	u32 value;
+	unsigned int timeout = 100;
+
+	void __iomem *dsi1_base = NULL;
+
+	if (a_crtc->split_en) {
+		dsi1_base = ioremap(DSI1_BASE_ADDR, 0x200);
+		if (!dsi1_base) {
+			DRM_ERROR("%s: failed to ioremap dsi1 base\n", __func__);
+			return;
+		}
+	}
+
+	int ret;
+	u32 bitclock = a_crtc->bitclk/1000000; /* MHz */
+
+	/* not touch reg if running */
+	if (readl(a_crtc->dsipll_base + PLL_CTRL_REG2) & PLL_UP)
+		return;
+
+	ret = spacemit_calc_pll_regs(bitclock,
+				     &pll_ctrl_reg0,
+				     &pll_ctrl_reg1,
+				     &div_sel);
+	if (ret) {
+		DRM_ERROR("DSI PLL calc failed, bitclock=%u\n", bitclock);
+		return;
+	}
+
+	a_crtc->dsipll_reg0 = pll_ctrl_reg0;
+	a_crtc->dsipll_reg1 = pll_ctrl_reg1;
+
+	writel(pll_ctrl_reg0, a_crtc->dsipll_base + PLL_CTRL_REG0);
+
+	writel(pll_ctrl_reg1, a_crtc->dsipll_base + PLL_CTRL_REG1);
+
+	/* Apply div_sel to bits[30:29], keep other bits from dsipll_reg2 */
+	value = (a_crtc->dsipll_reg2 & ~(3U << 29)) | (div_sel << 29);
+	writel(value, a_crtc->dsipll_base + PLL_CTRL_REG2);
+	if (a_crtc->split_en) {
+		writel(pll_ctrl_reg0, a_crtc->dsi1pll_base + PLL_CTRL_REG0);
+		writel(pll_ctrl_reg1, a_crtc->dsi1pll_base + PLL_CTRL_REG1);
+		writel(value, a_crtc->dsi1pll_base + PLL_CTRL_REG2);
+	}
+
+	if ((a_crtc->dsipll_reg1 & (BIT(29) | BIT(30))) != 0)
+		dpu_set_bit(a_crtc->dsipll_base, DSI_PHY_ANA_CTRL1, BIT(28));
+	else
+		dpu_clr_bit(a_crtc->dsipll_base, DSI_PHY_ANA_CTRL1, BIT(28));
+
+	/* pu */
+	value |= PLL_UP;
+	writel(value, a_crtc->dsipll_base + PLL_CTRL_REG2);
+	if (a_crtc->split_en)
+		writel(value, a_crtc->dsi1pll_base + PLL_CTRL_REG2);
+
+	/* wait pll lock */
+	while (true) {
+		value = readl(a_crtc->dsipll_base + PLL_CTRL_REG3);
+
+		if (value & PLL_LK) {
+			DRM_DEBUG("bitclk pll locked\n");
+			break;
+		}
+
+		if (timeout == 0) {
+			DRM_ERROR("failed to got bitclk pll lock 0x%x\n", value);
+			break;
+		}
+
+		timeout--;
+
+		udelay(10);
+	}
+}
+
+/* dove: hclk, pxclk, mclk, escclk, bitclk
+ * lark: hclk, pxclk, mclk, escclk, dsipll, aclk
+ * dovenr/larkpro: hclk, pxclk, mclk, escclk, dsipll, aclk, dscclk
+ */
+static int dpu_enable_clocks(struct spacemit_crtc *a_crtc)
+{
+	struct dpu_clk_context *clk_ctx = &a_crtc->clk_ctx;
+	struct drm_crtc *crtc = &a_crtc->crtc;
+	struct drm_display_mode *mode = &crtc->mode;
+	uint64_t clk_val;
+	uint64_t set_clk_val;
+	bool dpu_online_enabled = false;
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	return 0;
+#endif
+	if (a_crtc->is_offline_mode)
+		dpu_online_enabled = __clk_is_enabled(clk_ctx->pxclk);
+
+	if (clk_ctx->hclk)
+		clk_prepare_enable(clk_ctx->hclk);
+
+	if (clk_ctx->pxclk) {
+		clk_prepare_enable(clk_ctx->pxclk);
+
+		if (dpu_online_enabled == false) {
+			if (a_crtc->dsc_regs)
+				set_clk_val = a_crtc->dsc_pxclk ;  /* dsc mode pxclk */
+			else
+				set_clk_val = mode->clock * 1000;
+
+			if (set_clk_val) {
+				set_clk_val = clk_round_rate(clk_ctx->pxclk, set_clk_val);
+				clk_val = clk_get_rate(clk_ctx->pxclk);
+				if (clk_val != set_clk_val) {
+					clk_set_rate(clk_ctx->pxclk, set_clk_val);
+					DRM_DEBUG("pxclk=%lld\n", clk_val);
+				}
+			}
+		}
+	}
+
+	if (clk_ctx->mclk) {
+		clk_prepare_enable(clk_ctx->mclk);
+
+		clk_val = clk_get_rate(clk_ctx->mclk);
+		if (clk_val != DPU_MCLK_DEFAULT) {
+			clk_val = clk_round_rate(clk_ctx->mclk, DPU_MCLK_DEFAULT);
+			clk_set_rate(clk_ctx->mclk, clk_val);
+			DRM_DEBUG("mclk=%lld\n", clk_val);
+		}
+	}
+
+	if (clk_ctx->escclk) {
+		clk_prepare_enable(clk_ctx->escclk);
+
+		clk_val = clk_get_rate(clk_ctx->escclk);
+		set_clk_val = a_crtc->escclk;
+		if (clk_val != set_clk_val) {
+			clk_val = clk_round_rate(clk_ctx->escclk, set_clk_val);
+			clk_set_rate(clk_ctx->escclk, clk_val);
+			DRM_DEBUG("escclk=%lld\n", clk_val);
+		}
+	}
+
+	if (clk_ctx->bitclk) {
+		clk_prepare_enable(clk_ctx->bitclk);
+
+		clk_val = clk_get_rate(clk_ctx->bitclk);
+		set_clk_val = a_crtc->bitclk;
+		if (clk_val != set_clk_val) {
+			clk_val = clk_round_rate(clk_ctx->bitclk, set_clk_val);
+			clk_set_rate(clk_ctx->bitclk, clk_val);
+			DRM_DEBUG("bitclk=%lld\n", clk_val);
+		}
+	}
+
+	if (clk_ctx->aclk) {
+		clk_prepare_enable(clk_ctx->aclk);
+
+		clk_val = clk_get_rate(clk_ctx->aclk);
+		set_clk_val = a_crtc->aclk;
+		DRM_INFO("Current aclk rate: %llu, Target: %llu\n", clk_val, set_clk_val);
+
+		if (clk_val != set_clk_val) {
+			clk_val = clk_round_rate(clk_ctx->aclk, set_clk_val);
+			clk_set_rate(clk_ctx->aclk, clk_val);
+			DRM_DEBUG("aclk=%lld\n", clk_val);
+		}
+	}
+
+	if (a_crtc->dsipll_valid)
+		dpu_enable_dsipll(a_crtc);
+
+	if (clk_ctx->dscclk) {
+		clk_prepare_enable(clk_ctx->dscclk);
+
+		if (a_crtc->dsc_regs && a_crtc->dsc_x_slice && a_crtc->dsc_clk) {
+			clk_val = clk_get_rate(clk_ctx->dscclk);
+			if (clk_val != a_crtc->dsc_clk) {
+				clk_val = clk_round_rate(clk_ctx->dscclk, a_crtc->dsc_clk);
+				clk_set_rate(clk_ctx->dscclk, clk_val);
+				DRM_DEBUG("dscclk=%lld\n", clk_val);
+			}
+		}
+	}
+
+#if IS_ENABLED(CONFIG_SPACEMIT_NI700)
+	ni700_power_on(SPACEMIT_NI700_LCD);
+#endif
+
+	trace_dpu_enable_clocks(a_crtc->dev_id);
+	return 0;
+}
+
+static int dpu_disable_clocks(struct spacemit_crtc *a_crtc)
+{
+	struct dpu_clk_context *clk_ctx = &a_crtc->clk_ctx;
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	return 0;
+#endif
+	trace_dpu_disable_clocks(a_crtc->dev_id);
+
+#if IS_ENABLED(CONFIG_SPACEMIT_NI700)
+	ni700_power_off(SPACEMIT_NI700_LCD);
+#endif
+
+	if (clk_ctx->hclk)
+		clk_disable_unprepare(clk_ctx->hclk);
+
+	if (clk_ctx->pxclk)
+		clk_disable_unprepare(clk_ctx->pxclk);
+
+	if (clk_ctx->mclk)
+		clk_disable_unprepare(clk_ctx->mclk);
+
+	if (clk_ctx->escclk)
+		clk_disable_unprepare(clk_ctx->escclk);
+
+	if (clk_ctx->bitclk)
+		clk_disable_unprepare(clk_ctx->bitclk);
+
+	if (a_crtc->dsipll_valid)
+		dpu_disable_dsipll(a_crtc);
+
+	if (clk_ctx->aclk)
+		clk_disable_unprepare(clk_ctx->aclk);
+
+	if (clk_ctx->dscclk)
+		clk_disable_unprepare(clk_ctx->dscclk);
+
+	return 0;
+}
+
+u8 spacemit_plane_hw_get_format_id(u32 format)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(primary_fmts); i++) {
+		if (primary_fmts[i].format == format)
+			return primary_fmts[i].id;
+	}
+
+	return SPACEMIT_DPU_INVALID_FORMAT_ID;
+}
+
+void spacemit_get_afbc_modifier(uint64_t modifier, struct spacemit_afbc_state *afbc_state)
+{
+	uint64_t super_block_size = modifier & AFBC_FORMAT_MOD_BLOCK_SIZE_MASK;
+
+	if (super_block_size == AFBC_FORMAT_MOD_BLOCK_SIZE_16x16)
+		afbc_state->block_size = 0;
+	else if (super_block_size == AFBC_FORMAT_MOD_BLOCK_SIZE_32x8)
+		afbc_state->block_size = 1;
+
+	if (modifier & AFBC_FORMAT_MOD_TILED)
+		afbc_state->tile_type = 1;
+	else
+		afbc_state->tile_type = 0;
+
+	if (modifier & AFBC_FORMAT_MOD_YTR)
+		afbc_state->yuv_transform = 1;
+	else
+		afbc_state->yuv_transform = 0;
+
+	if (modifier & AFBC_FORMAT_MOD_SPLIT)
+		afbc_state->split_mode = 1;
+	else
+		afbc_state->split_mode = 0;
+
+	if (modifier & AFBC_FORMAT_MOD_CBR)
+		afbc_state->copy_mode = 1;
+	else
+		afbc_state->copy_mode = 0;
+
+}
+
+static void saturn_enable_vsync(struct spacemit_crtc *a_crtc, bool enable)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	trace_saturn_enable_vsync("vsync", enable);
+	hwdev->enable_vsync(a_crtc, hwdev, enable);
+}
+
+void spacemit_cfg_rdy_timer_handler(struct timer_list *t)
+{
+	struct spacemit_crtc *a_crtc = timer_container_of(a_crtc, t, cfg_rdy_timer);
+#ifdef CONFIG_SPACEMIT_DEBUG
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+#endif
+
+	DRM_ERROR("CFG_RDY ERROR:%s\n", __func__);
+	trace_saturn_ctrl_cfg_ready_timer(a_crtc->dev_id);
+	//flip_done is used for online
+	if (!a_crtc->is_offline_mode)
+		a_crtc->flip_done = false;
+#ifdef CONFIG_SPACEMIT_DEBUG
+	hwdev->dpu_dump_reg(a_crtc);
+	//logger_noti_helper(3);
+#endif
+}
+
+static void saturn_ctrl_cfg_ready(struct spacemit_crtc *a_crtc, bool enable)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	if (a_crtc->is_stopped)
+		return;
+
+	trace_saturn_ctrl_cfg_ready(a_crtc->dev_id, enable);
+	mod_timer(&a_crtc->cfg_rdy_timer, jiffies + msecs_to_jiffies(3000));
+	hwdev->cfg_ready(a_crtc, hwdev);
+
+}
+
+static void saturn_ctrl_sw_start(struct spacemit_crtc *a_crtc, bool enable)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	if (a_crtc->is_offline_mode == 1)
+		return;
+
+	trace_saturn_ctrl_sw_start(a_crtc->dev_id, enable);
+
+	hwdev->sw_start(a_crtc, hwdev);
+}
+
+static u32 dpu_get_version(struct spacemit_crtc *a_crtc)
+{
+	return 0;
+}
+
+static void saturn_ctrl_cmd_update(struct spacemit_crtc *a_crtc, bool enable)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	if (a_crtc->out_mode == DPU_OUT_MODE_CMD) {
+		trace_saturn_ctl_cmd_update(a_crtc->dev_id, enable);
+		if (hwdev->cmd_update)
+			hwdev->cmd_update(a_crtc, hwdev);
+	}
+}
+
+static int dpu_init(struct spacemit_crtc *a_crtc)
+{
+	unsigned int timeout = 1000;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	if (!a_crtc->power_on)
+		return 0;
+
+	trace_dpu_init(a_crtc->dev_id);
+
+	while (timeout) {
+		if (hwdev->get_cfg_rdy(a_crtc, hwdev) == 0)
+			break;
+		udelay(100);
+		timeout--;
+	};
+	if (timeout == 0)
+		DRM_ERROR("%s wait cfg ready done timeout\n", __func__);
+
+	hwdev->dpu_init(a_crtc);
+
+	return 0;
+}
+
+static void dpu_uninit(struct spacemit_crtc *a_crtc)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	if (!a_crtc->power_on)
+		return;
+
+	trace_dpu_uninit(a_crtc->dev_id);
+	hwdev->irq_enable(a_crtc, false);
+}
+
+static void dpu_pm_update_commit_qos(struct spacemit_crtc *a_crtc)
+{
+#ifdef CONFIG_PM
+	if (a_crtc->out_mode == DPU_OUT_MODE_CMD && a_crtc->lpm_commit_qos) {
+		if (a_crtc->lpm_period) {
+			/* put lpm qos in delay a_crtc->lpm_period time */
+			schedule_delayed_work(&a_crtc->lpm_qos_work,
+					msecs_to_jiffies(a_crtc->lpm_period));
+			a_crtc->lpm_work_pending = true;
+		} else {
+			dpu_rpm_suspend(a_crtc);
+		}
+	}
+#endif
+}
+
+static inline void dpu_isr_vblank(struct spacemit_crtc *a_crtc, bool *flip)
+{
+	struct drm_crtc *crtc = &a_crtc->crtc;
+
+	drm_crtc_handle_vblank(crtc);
+	if (!*flip) {
+		struct drm_device *drm = a_crtc->crtc.dev;
+		struct drm_crtc_state *state;
+		struct drm_pending_vblank_event *event = NULL;
+
+		*flip = true;
+		spin_lock(&drm->event_lock);
+
+		state = crtc->state;
+		if (state)
+			event = state->event;
+
+		if (event) {
+			/*
+			* Set event to NULL first to ensure event is consumed
+			* before drm_atomic_helper_commit_hw_done.
+			*/
+			state->event = NULL;
+			drm_crtc_send_vblank_event(crtc, event);
+		}
+
+		spin_unlock(&drm->event_lock);
+		drm_crtc_vblank_put(crtc);
+		dpu_pm_update_commit_qos(a_crtc);
+	}
+}
+
+u32 saturn_conf_dpuctrl_rdma(struct spacemit_crtc *a_crtc)
+{
+	struct drm_crtc *crtc = &a_crtc->crtc;
+	struct drm_plane *plane;
+	u32 rdma_en = 0;
+
+	/* Find out the active rdmas */
+	drm_atomic_crtc_for_each_plane(plane, crtc) {
+		u32 rdma_id = to_spacemit_plane_state(plane->state)->rdma_id;
+
+		if (rdma_id != RDMA_INVALID_ID)
+			rdma_en |= (1 << rdma_id);
+	}
+
+	trace_dpuctrl("rdma_en", rdma_en);
+
+	return rdma_en;
+}
+
+static uint32_t dpu_online_isr(struct spacemit_crtc *a_crtc)
+{
+	uint32_t irq_raw, irq_bit, irq_ur_bit, irq_dbg_sts;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	static bool ur_done;
+	struct drm_writeback_connector *wb_conn = priv->wb_connector[a_crtc->wb_id];
+	int dev_id = a_crtc->dev_id;
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+	unsigned long flags;
+#endif
+
+	trace_dpu_isr(a_crtc->dev_id);
+
+	irq_raw = hwdev->get_int_sts(hwdev, dev_id);
+	trace_dpu_isr_status("ONLINE", irq_raw);
+	/* underrun */
+	irq_bit = hwdev->get_irq_bit(INT_UNDERRUN, dev_id);
+	irq_ur_bit = irq_raw & irq_bit;
+	if (irq_ur_bit && !ur_done) {
+		hwdev->clr_int_sts(a_crtc, irq_ur_bit, dev_id);
+		trace_dpu_isr_status("Under Run!", irq_ur_bit);
+		trace_dpu_isr_ul_data("DPU Mclk", a_crtc->cur_mclk);
+		trace_dpu_isr_ul_data("DPU BW", a_crtc->cur_bw);
+		queue_work(a_crtc->dpu_trace_wq, &a_crtc->work_dpu_trace);
+#ifdef CONFIG_SPACEMIT_DEBUG
+		hwdev->dpu_dump_reg(a_crtc);
+#endif
+		DRM_ERROR_RATELIMITED("Under Run! DPU_Mclk = %ld, DPU BW = %ld\n", (unsigned long)a_crtc->cur_mclk, (unsigned long)a_crtc->cur_bw);
+#if IS_ENABLED(CONFIG_SPACEMIT_CORE_CLK_DOVE) || IS_ENABLED(CONFIG_SPACEMIT_CORE_CLK_LARK)
+		DRM_ERROR_RATELIMITED("DDR Freq = %d\n", ddr_get_freq_lv());
+#endif
+#ifdef CONFIG_SPACEMIT_DEBUG
+		hwdev->dpu_dump_rdma_status(priv);
+#endif
+		ur_done = true;
+	}
+	/* cfg ready clear */
+	irq_bit = hwdev->get_irq_bit(INT_CFG_RDY, dev_id);
+	if (irq_raw & irq_bit) {
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+		trace_dpu_isr_status("cfg_rdy_clr", irq_raw & irq_bit);
+		a_crtc->flip_done = false;
+		timer_delete(&a_crtc->cfg_rdy_timer);
+		ur_done = false;
+		if (hwdev->enable_cfg_irq)
+			hwdev->enable_cfg_irq(a_crtc, hwdev, false);
+		trace_u64_data("irq crtc mclk cur", a_crtc->cur_mclk);
+		trace_u64_data("irq crtc mclk new", a_crtc->new_mclk);
+		trace_u64_data("irq crtc bw cur", a_crtc->cur_bw);
+		trace_u64_data("irq crtc bw new", a_crtc->new_bw);
+		if (a_crtc->enable_auto_fc && a_crtc->new_mclk < a_crtc->cur_mclk && a_crtc->fix_max_mclk == false) {
+			trace_u64_data("run wq to set mclk", a_crtc->new_mclk);
+			queue_work(system_wq, &a_crtc->work_update_clk);
+		}
+		if (a_crtc->enable_auto_fc && a_crtc->new_bw < a_crtc->cur_bw) {
+			trace_u64_data("run wq to set bw", a_crtc->new_bw);
+			queue_work(system_wq, &a_crtc->work_update_bw);
+		}
+	}
+	/* vsync */
+	irq_bit = hwdev->get_irq_bit(INT_VSYNC, dev_id);
+	if (irq_raw & irq_bit) {
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+		trace_dpu_isr_status("vsync", irq_raw & irq_bit);
+		dpu_isr_vblank(a_crtc, &a_crtc->flip_done);
+		dpu_dump_fps(a_crtc);
+#ifdef CONFIG_SPACEMIT_DEBUG
+		spin_lock_irqsave(&priv->ur_dump_lock, flags);
+		priv->underrun_debug = false;
+		priv->old_state = NULL;
+		spin_unlock_irqrestore(&priv->ur_dump_lock, flags);
+#endif
+	}
+	/* wb done */
+	irq_bit = hwdev->get_irq_bit(INT_WB_DONE, dev_id);
+	if (irq_raw & irq_bit) {
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+
+		if (hwdev->is_wb_en(a_crtc, hwdev)) {
+			trace_dpu_isr_status("wb", irq_raw & irq_bit);
+			hwdev->wb_disable(a_crtc);
+			drm_writeback_signal_completion(wb_conn, 0);
+		}
+	}
+	/* rest irq status */
+	irq_bit = hwdev->get_irq_bit(INT_REST, dev_id) | irq_ur_bit;
+	if (irq_raw & irq_bit)
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+
+	/* vsync update clear */
+	irq_bit = hwdev->get_irq_bit(INT_VSYNC_UPDATE, dev_id);
+	if (irq_raw & irq_bit)
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+
+	if (hwdev->get_rdma_dbg_sts) {
+		irq_dbg_sts = hwdev->get_rdma_dbg_sts(a_crtc, dev_id);
+		if (irq_dbg_sts) {
+			trace_dpu_isr_status("ONLINE INT DBG STS", irq_dbg_sts);
+			queue_work(a_crtc->dpu_trace_wq, &a_crtc->work_dpu_trace);
+		}
+	}
+
+	return 0;
+}
+
+static uint32_t dpu_offline_isr(struct spacemit_crtc *a_crtc)
+{
+	uint32_t irq_raw, irq_bit;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	static bool wb_flip_done;
+	struct drm_writeback_connector *wb_conn = priv->wb_connector[a_crtc->wb_id];
+	int dev_id = a_crtc->dev_id;
+
+	trace_dpu_isr(a_crtc->dev_id);
+	irq_raw = hwdev->get_int_sts(hwdev, dev_id);
+	trace_dpu_isr_status("OFFLINE", irq_raw);
+	/* cfg ready clear */
+	irq_bit = hwdev->get_irq_bit(OFF_CFG_RDY, dev_id);
+	if (irq_raw & irq_bit) {
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+		trace_dpu_isr_status("cfg_rdy_clr", irq_raw & irq_bit);
+		wb_flip_done = false;
+		timer_delete(&a_crtc->cfg_rdy_timer);
+
+		if (hwdev->enable_cfg_irq)
+			hwdev->enable_cfg_irq(a_crtc, hwdev, false);
+		if (saturn_conf_dpuctrl_rdma(a_crtc) == 0)
+			dpu_isr_vblank(a_crtc, &wb_flip_done);
+		trace_u64_data("irq crtc mclk cur", a_crtc->cur_mclk);
+		trace_u64_data("irq crtc mclk new", a_crtc->new_mclk);
+		trace_u64_data("irq crtc bw cur", a_crtc->cur_bw);
+		trace_u64_data("irq crtc bw new", a_crtc->new_bw);
+		if (a_crtc->enable_auto_fc && a_crtc->new_mclk < a_crtc->cur_mclk && a_crtc->fix_max_mclk == false) {
+			trace_u64_data("run wq to set mclk", a_crtc->new_mclk);
+			queue_work(system_wq, &a_crtc->work_update_clk);
+		}
+		if (a_crtc->enable_auto_fc && a_crtc->new_bw < a_crtc->cur_bw) {
+			trace_u64_data("run wq to set bw", a_crtc->new_bw);
+			queue_work(system_wq, &a_crtc->work_update_bw);
+		}
+	}
+	/* wb done */
+	irq_bit = hwdev->get_irq_bit(OFF_WB_DONE, dev_id);
+	if (irq_raw & irq_bit) {
+		trace_dpu_isr_status("wb", irq_raw & irq_bit);
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+		dpu_isr_vblank(a_crtc, &wb_flip_done);
+		drm_writeback_signal_completion(wb_conn, 0);
+	}
+	/* rest irq status */
+	irq_bit = hwdev->get_irq_bit(OFF_REST, dev_id);
+	if (irq_raw & irq_bit)
+		hwdev->clr_int_sts(a_crtc, irq_bit, dev_id);
+	return 0;
+}
+
+static void dpu_run(struct drm_crtc *crtc,
+		    struct drm_crtc_state *old_state)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct spacemit_dsi_vrr_param vrr_param = {0};
+
+	trace_dpu_run(a_crtc->dev_id);
+
+	/* config dpuctrl modules */
+	if (hwdev->conf_dpuctrl)
+		hwdev->conf_dpuctrl(crtc, old_state);
+
+	if (!a_crtc->is_offline_mode &&
+		a_crtc->dsi && a_crtc->dsi->core) {
+		if (a_crtc->dsi->core->dsi_update_vrr) {
+			vrr_param.vrr_vfp = a_crtc->vrr_vfp;
+			a_crtc->dsi->core->dsi_update_vrr(&a_crtc->dsi->ctx, &vrr_param);
+		}
+	}
+
+	//dsb(sy);
+	mb();
+
+	drm_crtc_vblank_get(crtc);
+	if (hwdev->enable_cfg_irq)
+		hwdev->enable_cfg_irq(a_crtc, hwdev, true);
+
+	saturn_ctrl_cfg_ready(a_crtc, true);
+
+	saturn_ctrl_cmd_update(a_crtc, true);
+
+	if (unlikely(a_crtc->is_1st_f)) {
+		a_crtc->is_1st_f = false;
+		saturn_ctrl_sw_start(a_crtc, true);
+		DRM_INFO("DPU %d Start!\n", a_crtc->dpu_id);
+	}
+#ifdef CONFIG_ARM64
+	__iomb();
+#else
+	dma_rmb();
+#endif
+}
+
+static void dpu_esd_restart(struct spacemit_crtc *a_crtc)
+{
+	unsigned int timeout = DPU_STOP_TIMEOUT;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_crtc *crtc = &a_crtc->crtc;
+
+	DRM_INFO("%s()\n", __func__);
+
+	while (timeout) {
+		if (hwdev->get_cfg_rdy(a_crtc, hwdev) == 0)
+			break;
+		udelay(10);
+		timeout--;
+	};
+	if (timeout == 0)
+		DRM_ERROR("%s dpu %d wait cfg ready done timeout\n", __func__, a_crtc->dev_id);
+	else
+		DRM_DEBUG("%s dpu %d wait cfg ready done %d\n", __func__, timeout, a_crtc->dev_id);
+
+	hwdev->dpu_restart(a_crtc, hwdev);
+	drm_crtc_vblank_get(crtc);
+	if (hwdev->enable_cfg_irq)
+		hwdev->enable_cfg_irq(a_crtc, hwdev, true);
+	//dsb(sy);
+	mb();
+	hwdev->cfg_ready(a_crtc, hwdev);
+	hwdev->sw_start(a_crtc, hwdev);
+#ifdef CONFIG_ARM64
+	__iomb();
+#else
+	dma_rmb();
+#endif
+
+	timeout = hwdev->reboot_flag ? DPU_STOP_REBOOT_TIMEOUT : DPU_STOP_TIMEOUT;
+	while (timeout) {
+		if (hwdev->get_cfg_rdy(a_crtc, hwdev) & 1) {
+			udelay(10);
+			timeout--;
+			continue;
+		} else
+			break;
+	};
+	if (timeout == 0)
+		DRM_ERROR("%s esd_restart timeout!!!\n", __func__);
+	else
+		DRM_DEBUG("%s esd_restart done %d\n", __func__, timeout);
+	hwdev->irq_enable(a_crtc, 1);
+}
+
+static void dpu_stop(struct spacemit_crtc *a_crtc)
+{
+	unsigned int timeout = DPU_STOP_TIMEOUT;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	int ret = 0;
+
+	if (!a_crtc->power_on)
+		return;
+
+	DRM_INFO("%s()\n", __func__);
+
+	trace_dpu_stop(a_crtc->dev_id);
+
+	hwdev->irq_enable(a_crtc, 0);
+	while (timeout) {
+		if (hwdev->get_cfg_rdy(a_crtc, hwdev) == 0)
+			break;
+		udelay(10);
+		timeout--;
+	};
+	if (timeout == 0) {
+		DRM_ERROR("%s dpu %d wait cfg ready done timeout\n", __func__, a_crtc->dev_id);
+		a_crtc->stop_to++;
+	} else
+		DRM_DEBUG("%s dpu %d wait cfg ready done %d\n", __func__, timeout, a_crtc->dev_id);
+
+	hwdev->dpu_disable(a_crtc, hwdev);
+	//dsb(sy);
+	mb();
+	hwdev->cfg_ready(a_crtc, hwdev);
+#ifdef CONFIG_ARM64
+	__iomb();
+#else
+	dma_rmb();
+#endif
+
+	timeout = hwdev->reboot_flag ? DPU_STOP_REBOOT_TIMEOUT : DPU_STOP_TIMEOUT;
+	while (timeout) {
+		if (hwdev->get_cfg_rdy(a_crtc, hwdev) & 1) {
+			udelay(10);
+			timeout--;
+			continue;
+		} else
+			break;
+	};
+	ret = hwdev->dpu_stop_check(a_crtc, hwdev);
+	if (ret) {
+		DRM_ERROR("dpu %d stop check failed!!!\n", a_crtc->dev_id);
+		a_crtc->stop_to++;
+	}
+	if (timeout == 0) {
+		DRM_ERROR("%s dpu %d stop timeout!!!\n", __func__, a_crtc->dev_id);
+		a_crtc->stop_to++;
+		pm_stay_awake(a_crtc->dev);
+	} else {
+		DRM_DEBUG("%s dpu %d stop Done %d\n", __func__, a_crtc->dev_id, timeout);
+		if (a_crtc->stop_to) {
+			pm_relax(a_crtc->dev);
+			a_crtc->stop_to = 0;
+		}
+	}
+
+}
+
+static int dpu_modeset(struct spacemit_crtc *a_crtc, struct drm_mode_modeinfo *mode)
+{
+	return 0;
+}
+
+static void dpu_enable_vsync(struct spacemit_crtc *a_crtc)
+{
+	saturn_enable_vsync(a_crtc, true);
+}
+
+static void dpu_disable_vsync(struct spacemit_crtc *a_crtc)
+{
+	saturn_enable_vsync(a_crtc, false);
+}
+
+static void __maybe_unused dpu_rpm_suspend(struct spacemit_crtc *a_crtc)
+{
+	if (!a_crtc->rpm_status)
+		return;
+
+	DRM_DEBUG("%s\n", __func__);
+
+	dpu_stop(a_crtc);
+
+	if (!a_crtc->is_offline_mode &&
+		a_crtc->dsi && a_crtc->dsi->core) {
+		mutex_lock(&a_crtc->dsi->disable_lock);
+		if (a_crtc->dsi->core->dsi_close_datatx)
+			a_crtc->dsi->core->dsi_close_datatx(&a_crtc->dsi->ctx);
+
+		if (a_crtc->dsi->core->dsi_close)
+			a_crtc->dsi->core->dsi_close(&a_crtc->dsi->ctx);
+		mutex_unlock(&a_crtc->dsi->disable_lock);
+	}
+
+	dpu_uninit(a_crtc);
+
+	drm_crtc_vblank_off(&a_crtc->crtc);
+
+	if (!a_crtc->is_offline_mode)
+		spacemit_dpu_power_enable(a_crtc, false);
+
+	a_crtc->rpm_status = false;
+}
+
+static void __maybe_unused dpu_rpm_resume(struct spacemit_crtc *a_crtc)
+{
+#ifdef CONFIG_PM
+	if (a_crtc->lpm_work_pending && a_crtc->lpm_period) {
+		cancel_delayed_work_sync(&a_crtc->lpm_qos_work);
+		a_crtc->lpm_work_pending = false;
+	}
+#endif
+	if (a_crtc->rpm_status)
+		return;
+
+	DRM_DEBUG("%s\n", __func__);
+
+	if (!a_crtc->is_offline_mode)
+		spacemit_dpu_power_enable(a_crtc, true);
+
+	drm_crtc_vblank_on(&a_crtc->crtc);
+
+	dpu_init(a_crtc);
+	a_crtc->is_1st_f = true;
+
+	if (!a_crtc->is_offline_mode &&
+		a_crtc->dsi && a_crtc->dsi->core) {
+		if (a_crtc->dsi->core->dsi_open)
+			a_crtc->dsi->core->dsi_open(&a_crtc->dsi->ctx, false);
+
+		if (a_crtc->dsi->core->dsi_ready_for_datatx)
+			a_crtc->dsi->core->dsi_ready_for_datatx(&a_crtc->dsi->ctx);
+	}
+
+	a_crtc->rpm_status = true;
+}
+
+static void dpu_begin(struct spacemit_crtc *a_crtc)
+{
+#ifdef CONFIG_PM
+	if (a_crtc->out_mode == DPU_OUT_MODE_CMD && a_crtc->lpm_commit_qos) {
+		dpu_rpm_resume(a_crtc);
+	}
+#endif
+}
+
+static struct dpu_core_ops dpu_saturn_ops = {
+	.parse_dt = dpu_parse_dt,
+	.version = dpu_get_version,
+	.init = dpu_init,
+	.uninit = dpu_uninit,
+	.run = dpu_run,
+	.stop = dpu_stop,
+	.esd_restart = dpu_esd_restart,
+	.online_isr = dpu_online_isr,
+	.offline_isr = dpu_offline_isr,
+	.modeset = dpu_modeset,
+	.enable_clk = dpu_enable_clocks,
+	.disable_clk = dpu_disable_clocks,
+	.update_clk = dpu_update_clocks,
+	.update_bw = dpu_update_bw,
+	.enable_vsync = dpu_enable_vsync,
+	.disable_vsync = dpu_disable_vsync,
+	.cal_layer_fbcmem_size = saturn_cal_layer_fbcmem_size,
+	.calc_plane_mclk_bw = dpu_calc_plane_mclk_bw,
+	.adjust_rdma_fbcmem = saturn_adjust_rdma_fbcmem,
+	.begin = dpu_begin,
+};
+
+static struct ops_entry entry = {
+	.ver = "spacemit-saturn",
+	.ops = &dpu_saturn_ops,
+};
+
+#ifndef MODULE
+static int __init dpu_core_register(void)
+#else
+int dpu_core_register(void)
+#endif
+{
+	return dpu_core_ops_register(&entry);
+}
+#ifndef MODULE
+subsys_initcall(dpu_core_register);
+#endif
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/dpu/dpu_saturn.h b/drivers/gpu/drm/spacemit/dpu/dpu_saturn.h
new file mode 100644
index 000000000000..ed5dedf47249
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/dpu_saturn.h
@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _DPU_SATURN_H_
+#define _DPU_SATURN_H_
+
+//#include "saturn_regs/reg_map.h"
+#include "../spacemit_crtc.h"
+
+#ifndef min
+#define min(x, y) (((x) < (y)) ? (x) : (y))
+#endif
+
+#ifndef max
+#define max(x, y) (((x) > (y)) ? (x) : (y))
+#endif
+
+#ifndef clip
+#define clip(x, a, b) (max(a, min(x, b)))
+#endif
+
+#ifndef is_rot_90_270
+#define is_rot_90_270(x) (x & (DRM_MODE_ROTATE_90 | DRM_MODE_ROTATE_270))
+#endif
+
+#define DOVE_SCALER_MAX			4
+#define DOVE_DMA_CHANNEL_MAX		4
+#define DOVE_DMA_LAYER_MAX		16
+#define DOVE_COMPOSER_MAX		2
+#define DOVE_COMPOSER_LAYER_MAX	16
+#define DOVE_PANEL_MAX			1
+#define DOVE_OUTCTRL_MAX			4
+#define DOVE_WRITEBACK_MAX		1
+#define DOVE_DISPLAY_MAX			2
+#define DOVE_LUT3D_MAX			0
+#define DOVE_CMDLIST_MAX			5
+
+#define SPACEMIT_DSC_MAX_X_SLICE	(2)
+#define SPACEMIT_DSC_DEFALUT_X_SLICE	(1)
+
+/* Supported variants of the hardware
+ * saturn_hw_version must match with HWC
+ * so can NOT update current IP id
+ */
+enum saturn_hw_version {
+	SATURN_LEC = 2,			//lark L
+	SATURN_LED,			//lark M
+	SATURN_LEE,			//dove NR
+	SATURN_MEA,			//lark pro
+	SATURN_HEE,			//1st dpu
+	SATURN_EDP,			//2rd dpu
+	/* keep the next entry last */
+	SPACEMIT_DP_MAX_DEVICES
+};
+
+struct dpu_format_id {
+	u32 format;		/* DRM fourcc */
+	u8 id;			/* used internally */
+	u8 bpp;			/* bit per pixel */
+};
+#define SPACEMIT_DPU_INVALID_FORMAT_ID	0xff
+
+enum format_features {
+	FORMAT_RGB  = BIT(0),
+	FORMAT_RAW_YUV  = BIT(1),
+	FORMAT_AFBC = BIT(2),
+};
+
+enum out_mode {
+	DPU_OUT_MODE_VIDEO,
+	DPU_OUT_MODE_CMD,
+	DPU_OUT_MODE_MAX
+};
+
+enum rotation_features {
+	ROTATE_COMMON = BIT(0),		/* supports rotation on raw and afbc 0/180, x and y */
+	ROTATE_RAW_90_270 = BIT(1),	/* supports rotation on raw 90/270 */
+	ROTATE_AFBC_90_270 = BIT(2),	/* supports rotation on afbc 90/270 */
+};
+
+struct spacemit_hw_rdma {
+	u16 formats;
+	u16 rots;
+};
+
+struct drm_dpu_hw_version_blob {
+	u16 dpu_version;
+	u16 rdma_nums;
+	u16 scaler_num;
+	u16 ver_scaler_num;
+	u16 hor_scaler_num;
+	u16 acad_num;
+	u16 rdma_offset;
+	u64 dpu_cfg;
+	u64 reserved;
+} __attribute__((packed));
+
+struct spacemit_afbc_state {
+	uint8_t block_size;
+	uint8_t tile_type;
+	uint8_t yuv_transform;
+	uint8_t split_mode;
+	uint8_t copy_mode;
+};
+
+struct pll_freq_range_t{
+	u16 low;
+	u16 mid;
+	u16 high;
+};
+
+#define PLL_MODE_LWG_PLL 0
+#define PLL_MODE_5G_PLL  1
+#define PLL_REF_FREQ_MHZ 24
+/* DSI bitclk pll control regs */
+#define PLL_CTRL_REG0   0xc0
+#define PLL_CTRL_REG1   0xc4
+#define PLL_CTRL_REG2   0xc8
+#define PLL_CTRL_REG3   0xcc
+#define PLL_CTRL_STATUS 0x230
+
+#define PLL_LK          BIT(29)
+#define PLL_UP          BIT(31)
+#define PLL_DIV_EN      (0xf << 4)
+
+#define SPACEMIT_MAX_SCALE_FACTOR 4
+
+extern const u32 saturn_fbcmem_sizes[2];
+extern const u32 saturn_le_fbcmem_sizes[2];
+
+u8 spacemit_plane_hw_get_format_id(u32 format);
+u32 saturn_conf_dpuctrl_rdma(struct spacemit_crtc *a_crtc);
+extern struct spacemit_hw_device spacemit_dp_devices[SPACEMIT_DP_MAX_DEVICES];
+void spacemit_get_afbc_modifier(uint64_t modifier, struct spacemit_afbc_state *afbc_state);
+void spacemit_dpu_power_enable(struct spacemit_crtc *a_crtc, bool enable);
+void saturn_enable_irq_mask(struct spacemit_crtc *a_crtc, bool enable, u32 offset, u32 mask);
+void spacemit_cfg_rdy_timer_handler(struct timer_list *t);
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/dpu_saturn_hee.c b/drivers/gpu/drm/spacemit/dpu/dpu_saturn_hee.c
new file mode 100644
index 000000000000..ee87d70fd255
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/dpu_saturn_hee.c
@@ -0,0 +1,1968 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/pm_qos.h>
+#include <linux/regmap.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_blend.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_writeback.h>
+#include "dpu_saturn.h"
+#include "saturn_fbcmem.h"
+#include "../spacemit_cmdlist.h"
+#include "../spacemit_dmmu.h"
+#include "../spacemit_dpu_reg.h"
+#include "../spacemit_wb.h"
+#include <video/display_timing.h>
+#include <dt-bindings/display/spacemit_dpu.h>
+#include "saturn_regs/ops_hee.h"
+#include "saturn_regs/reg_map_hee.h"
+
+#include "dpu_trace.h"
+#include "dpu_debug.h"
+#include "../spacemit_bootloader.h"
+
+const u32 PREPIPE_SCAL_BASE_ADDR[] = {
+	PREPIPE_SCAL0_BASE_ADDR,
+	PREPIPE_SCAL1_BASE_ADDR,
+	PREPIPE_SCAL2_BASE_ADDR,
+	PREPIPE_SCAL3_BASE_ADDR,
+};
+
+enum RDMA_ID {
+	DPU_RDMA0,
+	DPU_RDMA1,
+	DPU_RDMA2,
+	DPU_RDMA3,
+	DPU_RDMA_MAX,
+	DPU_RDMA_INVALID,
+};
+
+/* hee:
+ * dpu_write:  uniform api for both cpu and cmdlist config dpu registers
+ */
+
+#define SPACEMIT_YUV2RGB_COEFFS 12
+#define ROT_MODE_MIRROR	4
+#define ROT_MODE_FLIP	5
+#define ROT_MODE_MIRROR_90	6
+#define ROT_MODE_FLIP_90	7
+#define hee_SCL_SEL_ID_INVALID  9
+#define POST_SCALER_POS	6
+#define POST_SCL 2
+
+static const s16
+	spacemit_rgb2yuv_coefs[DRM_COLOR_ENCODING_MAX][DRM_COLOR_RANGE_MAX][SPACEMIT_RGB2YUV_COEFFS] = {
+		[DRM_COLOR_YCBCR_BT601][DRM_COLOR_YCBCR_LIMITED_RANGE] = {
+			263,   516,  100,   16,
+			-152,  -298,  450,  128,
+			450,  -377,  -73,  128,
+		},
+		[DRM_COLOR_YCBCR_BT601][DRM_COLOR_YCBCR_FULL_RANGE] = {
+			306,   601,  117,    0,
+			-173,  -339,  512,  128,
+			512,  -429,  -83,  128,
+		},
+		[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE] = {
+			187,  629,   63,   16,
+			-103, -347,  450,  128,
+			450,  -409,  -41,  128,
+		},
+		[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_FULL_RANGE] = {
+			218,   732,   74,    0,
+			-117,  -395,  512,  128,
+			512,  -465,  -47,  128,
+		},
+		[DRM_COLOR_YCBCR_BT2020][DRM_COLOR_YCBCR_LIMITED_RANGE] = {
+			231,   596,   52,   64,
+			-126,  -324,  450,  512,
+			450,  -414,  -36,  512,
+		},
+		[DRM_COLOR_YCBCR_BT2020][DRM_COLOR_YCBCR_FULL_RANGE] = {
+			269,  694,   61,    0,
+			-143, -369,  512,  512,
+			512, -471,  -41,  512,
+		}
+	};
+
+static bool ur_enabhee = true;
+
+static const s16
+spacemit_yuv2rgb_coefs[DRM_COLOR_ENCODING_MAX][DRM_COLOR_RANGE_MAX][SPACEMIT_YUV2RGB_COEFFS] = {
+	[DRM_COLOR_YCBCR_BT601][DRM_COLOR_YCBCR_LIMITED_RANGE] = {
+		1192,    0, 1635, -223,
+		1192, -403, -833, 136,
+		1192, 2065,    0, -277,
+	},
+	[DRM_COLOR_YCBCR_BT601][DRM_COLOR_YCBCR_FULL_RANGE] = {
+		1024,    0, 1436, -179,
+		1024, -354, -732,  136,
+		1024, 1814,    0, -227,
+	},
+	[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE] = {
+		1192,    0, 1836, -248,
+		1192, -218, -546,   77,
+		1192, 2163,    0, -289,
+	},
+	[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_FULL_RANGE] = {
+		1024,	 0, 1613, -202,
+		1024, -192, -479,   84,
+		1024, 1900,    0, -238,
+	},
+	[DRM_COLOR_YCBCR_BT2020][DRM_COLOR_YCBCR_LIMITED_RANGE] = {
+		1196,	 0, 1724,  -937,
+		1196, -192, -668,   355,
+		1196, 2200,    0, -1175,
+	},
+	[DRM_COLOR_YCBCR_BT2020][DRM_COLOR_YCBCR_FULL_RANGE] = {
+		1024,    0, 1510, -755,
+		1024, -168, -585,  377,
+		1024, 1927,    0, -963,
+	}
+};
+
+static dpu_hee_reg_dump_t __maybe_unused dpu_reg_dump_array[] = {
+	{E_DPU_TOP_REG, "DPU_TOP", DPU_TOP_BASE_ADDR, 45},
+	{E_DMA_TOP_CTRL_REG, "DPU_TOP_CTRL", DPU_CTRL_BASE_ADDR, 51},
+	{E_DPU_SCENE_CTRL1_REG, "SCENE_CTRL1", DPU_SCENE_CTRL1_BASE_ADDR, 15},
+	{E_DPU_SCENE_CTRL2_REG, "SCENE_CTRL1", DPU_SCENE_CTRL2_BASE_ADDR, 15},
+	{E_DPU_CMDLIST_REG, "DPU_CMDLIST", CMDLIST_BASE_ADDR, 128},
+	{E_DPU_INT_REG, "DPU_INT", DPU_INT_BASE_ADDR, 50},
+	{E_DMA_TOP_CTRL_REG, "DPU_DMA_TOP", DMA_TOP_BASE_ADDR, 25},
+	{E_RDMA_LAYER0_REG, "RDMA_LAYER0", RDMA0_BASE_ADDR, 60},
+	{E_RDMA_LAYER1_REG, "RDMA_LAYER1", RDMA1_BASE_ADDR, 60},
+	{E_RDMA_LAYER2_REG, "RDMA_LAYER2", RDMA2_BASE_ADDR, 60},
+	{E_RDMA_LAYER3_REG, "RDMA_LAYER3", RDMA3_BASE_ADDR, 60},
+	{E_MMU_TBU0_REG, "MMU_TBU0", TBU0_ADDR, 21},
+	{E_MMU_TBU2_REG, "MMU_TBU2", TBU2_ADDR, 21},
+	{E_MMU_TBU4_REG, "MMU_TBU4", TBU4_ADDR, 21},
+	{E_MMU_TBU12_REG, "MMU_TBU12", WB_TBU_ADDR, 21},
+	{E_MMU_TOP_REG, "MMU_TOP", MMU_TOP_BASE_ADDR, 23},
+	{E_COMPOSER1_REG, "COMPOSER2", CMP1_BASE_ADDR, 146},
+	{E_COMPOSER2_REG, "COMPOSER3", CMP2_BASE_ADDR, 146},
+	{E_SCALER0_REG, "SCALER0", PREPIPE_SCAL0_BASE_ADDR, 129},
+	{E_SCALER1_REG, "SCALER1", PREPIPE_SCAL1_BASE_ADDR, 129},
+	{E_SCALER2_REG, "SCALER2", PREPIPE_SCAL2_BASE_ADDR, 129},
+	{E_SCALER3_REG, "SCALER3", PREPIPE_SCAL3_BASE_ADDR, 129},
+	{E_OUTCTRL0_REG, "OUTCTRL2", TMG0_BASE_ADDR, 28},
+	{E_PP0_REG, "PP0_REG", WB0_TOP_BASE_ADDR, 50},
+	{E_WB_TOP_0_REG, "WB0_TOP", WB0_TOP_BASE_ADDR, 55},
+};
+
+static dpu_hee_reg_enum __maybe_unused SATURN_hee_DPU_REG_ENUM_LISTS[] = {
+	E_DPU_TOP_REG,
+	E_DPU_SCENE_CTRL1_REG,
+	E_DPU_SCENE_CTRL2_REG,
+	E_DPU_CMDLIST_REG,
+	E_DPU_INT_REG,
+	E_DMA_TOP_CTRL_REG,
+	E_RDMA_LAYER0_REG,
+	E_RDMA_LAYER1_REG,
+	E_RDMA_LAYER2_REG,
+	E_RDMA_LAYER3_REG,
+	E_RDMA_LAYER4_REG,
+	E_RDMA_LAYER5_REG,
+	E_MMU_TBU0_REG,
+	E_MMU_TBU2_REG,
+	E_MMU_TBU4_REG,
+	E_MMU_TBU6_REG,
+	E_MMU_TBU8_REG,
+	E_MMU_TBU10_REG,
+	E_MMU_TBU12_REG,
+	E_MMU_TOP_REG,
+	E_COMPOSER1_REG,
+	E_COMPOSER2_REG,
+	E_SCALER0_REG,
+	E_SCALER1_REG,
+	E_OUTCTRL0_REG,
+	E_PP0_REG,
+	E_WB_TOP_0_REG,
+};
+
+#define CONFIG_TBU_REGS(hwdev, base, reg_id, cl_tbu) \
+{\
+	dpu_write(hwdev, MMU_TBU_REG, base, tbu_base_addr##reg_id##_low, \
+		  tbu->ttb_pa[reg_id] & 0xFFFFFFFF, cl_tbu, 1 + 2 * reg_id); \
+	dpu_write(hwdev, MMU_TBU_REG, base, tbu_base_addr##reg_id##_high, \
+		  (tbu->ttb_pa[reg_id] >> 32) & 0xFFF, cl_tbu, 2 + 2 * reg_id); \
+	dpu_write(hwdev, MMU_TBU_REG, base, tbu_va##reg_id, \
+		 (tbu->tbu_va[reg_id] >> 12), cl_tbu, 7 + reg_id); \
+	dpu_write(hwdev, MMU_TBU_REG, base, \
+		  tbu_size##reg_id, tbu->ttb_size[reg_id], cl_tbu, 10 + reg_id); \
+	dpu_write(hwdev, MMU_TBU_REG, base, \
+		  vsync_update_en, 1, cl_tbu, 13); \
+}
+
+#define CONFIG_RDMA_ADDR_REG(hwdev, reg_id, rdma_id, addr, cl_rdma) \
+{ \
+	if (hwdev) { \
+		dpu_write(hwdev, RDMA_PATH_X_REG, RDMA_BASE_ADDR[rdma_id], \
+			  base_addr##reg_id##_low_ly0, addr & 0xFFFFFFFF, \
+			  cl_rdma, 9 + 2 * reg_id); \
+		dpu_write(hwdev, RDMA_PATH_X_REG, RDMA_BASE_ADDR[rdma_id], \
+			  base_addr##reg_id##_high_ly0, (addr >> 32) & 0xFFF, \
+			  cl_rdma, 10 + 2 * reg_id); \
+	} \
+}
+
+static const u32 RDMA_BASE_ADDR[] = {
+	RDMA0_BASE_ADDR,
+	RDMA1_BASE_ADDR,
+	RDMA2_BASE_ADDR,
+	RDMA3_BASE_ADDR,
+};
+
+#define CONFIG_WB_ADDR_REG(hwdev, reg_id, addr, cl) \
+{ \
+	struct spacemit_hw_device *hwdev_p = (struct spacemit_hw_device *)hwdev; \
+	dpu_write(hwdev_p, WB_REG, WB0_TOP_BASE_ADDR, \
+			wb_wdma_base_addr##reg_id##_low, addr & 0xFFFFFFFF, cl, 5 + 2 * reg_id); \
+	dpu_write(hwdev_p, WB_REG, WB0_TOP_BASE_ADDR, \
+			wb_wdma_base_addr##reg_id##_high, (addr >> 32) & 0x3, cl, 6 + 2 * reg_id); \
+}
+
+
+static const u32 CMP_BASE_ADDR[] = {
+	CMP0_BASE_ADDR,
+	CMP1_BASE_ADDR,
+	CMP2_BASE_ADDR,
+};
+
+static const u32 TMG_BASE_ADDR[] = {
+	0,
+	TMG0_BASE_ADDR,
+	0,
+};
+
+static const u32 MMU_TBU_BASE_ADDR_ARRAY[] = {
+	TBU0_ADDR, TBU1_ADDR, TBU2_ADDR, TBU3_ADDR,
+	TBU4_ADDR, TBU5_ADDR, TBU6_ADDR, TBU7_ADDR,
+	WB_TBU_ADDR,
+};
+
+void saturn_hee_update_hdr_matrix(struct drm_plane *plane, struct spacemit_plane_state *spacemit_pstate)
+{
+}
+
+void saturn_hee_update_csc_matrix(struct drm_plane *plane, struct drm_plane_state *old_state)
+{
+	struct spacemit_plane_state *spacemit_plane_state = to_spacemit_plane_state(plane->state);
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u32 rdma_id = spacemit_plane_state->rdma_id;
+	u32 module_base;
+	int color_encoding = plane->state->color_encoding;
+	int color_range = plane->state->color_range;
+	int value;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(plane->state->crtc);
+	struct cmdlist_regs *cl_rdma = a_crtc->cl_rdma;
+
+	if (rdma_id == RDMA_INVALID_ID)
+		return;
+
+	module_base = RDMA_BASE_ADDR[rdma_id];
+
+	if ((color_encoding != old_state->color_encoding) || (color_range != old_state->color_range)) {
+		int i, index = 33;
+
+		for (i = 0; i < 12; i += 2, index++) {
+			value = (spacemit_yuv2rgb_coefs[color_encoding][color_range][i] & 0x3FFF) |
+				((spacemit_yuv2rgb_coefs[color_encoding][color_range][i + 1] & 0x3FFF) << 16);
+			dpu_write(hwdev, RDMA_PATH_X_REG, module_base, value32[index], value, cl_rdma, index);
+		}
+		if (saturn_hee_is_wb_en(a_crtc, hwdev)) {
+			hwdev->color_encoding = color_encoding;
+			hwdev->color_range = color_range;
+		}
+
+	}
+}
+
+void saturn_hee_conf_scaler_x(struct drm_plane_state *state, struct cmdlist_regs *cl)
+{
+	struct spacemit_plane_state *spacemit_plane_state = to_spacemit_plane_state(state);
+	struct drm_plane *plane = state->plane;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u32 in_width, in_height, out_width, out_height;
+	uint32_t hor_delta_phase, ver_delta_phase;
+	int64_t hor_init_phase, ver_init_phase;
+	uint32_t hor_init_phase_h1b, ver_init_phase_h1b;
+	bool hor_enable = false, ver_enable = false;
+	u32 module_base;
+	struct cmdlist_regs *cl_scl = NULL;
+	cl_scl = alloc_cmdlist_regs(SCALER_X_REG);
+
+	trace_saturn_conf_scaler_x(spacemit_plane_state);
+
+	if (is_rot_90_270(state->rotation)) {
+		in_width  = state->src_h >> 16;
+		in_height = state->src_w >> 16;
+	} else {
+		in_width  = state->src_w >> 16;
+		in_height = state->src_h >> 16;
+	}
+
+	out_width = state->crtc_w;
+	out_height = state->crtc_h;
+
+	hor_enable = out_width == in_width ? false : true;
+	ver_enable = out_height == in_height ? false : true;
+	module_base = PREPIPE_SCAL_BASE_ADDR[spacemit_plane_state->rdma_id];
+
+	hor_delta_phase = in_width * 65536 / out_width;
+	ver_delta_phase = in_height * 65536 / out_height;
+
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_hor_enable, hor_enable, cl_scl, 1);
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_ver_enable, ver_enable, cl_scl, 1);
+
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_hor_delta_phase, hor_delta_phase, cl_scl, 7);
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_ver_delta_phase, ver_delta_phase, cl_scl, 8);
+
+	hor_init_phase = ((int64_t)hor_delta_phase - 65536) >> 1;
+	ver_init_phase = ((int64_t)ver_delta_phase - 65536) >> 1;
+
+	DRM_DEBUG("hor_delta:%d ver_delta:%d\n", hor_delta_phase, ver_delta_phase);
+	DRM_DEBUG("hor_init_phase:%lld ver_init_phase:%lld\n", hor_init_phase, ver_init_phase);
+
+	if (hor_init_phase >= 0) {
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_hor_init_phase_l32b, hor_init_phase & 0x00000000ffffffff, cl_scl, 3);
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_hor_init_phase_h1b, 0, cl_scl, 4);
+	} else { //convert to positive value if negative
+		hor_init_phase += 8589934592;
+		hor_init_phase_h1b = (uint32_t)((hor_init_phase & 0x0000000100000000ULL) >> 32);
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_hor_init_phase_l32b, hor_init_phase & 0x00000000ffffffff, cl_scl, 3);
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_hor_init_phase_h1b, hor_init_phase_h1b, cl_scl, 4);
+	}
+
+	if (ver_init_phase >= 0) {
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_ver_init_phase_l32b, ver_init_phase & 0x00000000ffffffff, cl_scl, 5);
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_ver_init_phase_h1b, 0, cl_scl, 6);
+	} else { //convert to positive value if negative
+		ver_init_phase += 8589934592;
+		ver_init_phase_h1b = (uint32_t)((ver_init_phase & 0x0000000100000000ULL) >> 32);
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_ver_init_phase_l32b, ver_init_phase & 0x00000000ffffffff, cl_scl, 5);
+		dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_ver_init_phase_h1b, ver_init_phase_h1b, cl_scl, 6);
+	}
+
+	// enable both ver and hor
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_hor_enable, 0x1, cl_scl, 0);
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_ver_enable, 0x1, cl_scl, 0);
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_input_width, in_width, cl_scl, 1);
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_input_height, in_height, cl_scl, 1);
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_output_width, out_width, cl_scl, 2);
+	dpu_write(hwdev, SCALER_X_REG, module_base, b.m_nscl_output_height, out_height, cl_scl, 2);
+
+	cmdlist_regs_packing(plane_to_cl(plane), CMDLIST_MOD_SCL, cl_scl);
+	free_cmdlist_regs(cl_scl);
+}
+
+void saturn_hee_conf_scaler_coefs(struct drm_plane *plane, struct spacemit_plane_state *spacemit_pstate,
+				  struct cmdlist_regs *cl_scl)
+{
+	struct drm_property_blob *blob = spacemit_pstate->scale_coefs_blob_prop;
+	u32 module_base;
+	int val;
+	int *coef_data;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	int hor_scale_num = hwdev->hor_scale_coef_size;
+	int ver_scale_num = hwdev->ver_scale_coef_size;
+
+	struct cmdlist_regs *scl_cl = NULL;
+	scl_cl = alloc_cmdlist_regs(SCALER_X_REG);
+
+	module_base = PREPIPE_SCAL_BASE_ADDR[spacemit_pstate->rdma_id];
+
+	if (blob) {
+		int i, index = 9;
+
+		/* should never happen */
+		if (unlikely(blob->length != (hor_scale_num + ver_scale_num) * sizeof(int)))
+			DRM_ERROR("The blob length %zu is not correct scale_num %lu\n", blob->length, (hor_scale_num + ver_scale_num) * sizeof(int));
+
+		coef_data = (int *)blob->data;
+		if (unlikely(coef_data == NULL)) {
+			DRM_ERROR("The coef data is NULL\n");
+			return;
+		}
+
+		for (i = 0; i < hor_scale_num; i += 2, index++) {
+			val  = (coef_data[i] & 0xFFFF) | coef_data[i + 1] << 16;
+			dpu_write(hwdev, SCALER_X_REG, module_base, value32[index], val, scl_cl, index);
+		}
+
+		index = 65;
+		for (; i < ver_scale_num + hor_scale_num; i += 2, index++) {
+			val  = (coef_data[i] & 0xFFFF) | coef_data[i + 1] << 16;
+			dpu_write(hwdev, SCALER_X_REG, module_base, value32[index], val, scl_cl, index);
+		}
+	}
+
+	cmdlist_regs_packing(plane_to_cl(plane), CMDLIST_MOD_RDMA, scl_cl);
+	free_cmdlist_regs(scl_cl);
+}
+
+void saturn_hee_enable_vsync(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, bool enable)
+{
+	u32 offset = 0, mask = 0;
+
+	offset = a_crtc->is_offline_mode ? DPU_INT_BASE_ADDR + DPU_OFFLINE_IRQ_MSK : DPU_INT_BASE_ADDR + DPU_ONLINE_IRQ_MSK;
+	mask = a_crtc->is_offline_mode ? saturn_hee_get_irq_bit(OFF_WB_DONE, a_crtc->dev_id) : saturn_hee_get_irq_bit(INT_VSYNC, a_crtc->dev_id);
+
+	saturn_enable_irq_mask(a_crtc, enable, offset, mask);
+}
+
+void saturn_hee_enable_cfg_irq(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, bool enable)
+{
+	u32 offset = 0, mask = 0;
+
+	offset = a_crtc->is_offline_mode ? DPU_INT_BASE_ADDR + DPU_OFFLINE_IRQ_MSK : DPU_INT_BASE_ADDR + DPU_ONLINE_IRQ_MSK;
+	mask = a_crtc->is_offline_mode ? saturn_hee_get_irq_bit(OFF_CFG_RDY, a_crtc->dev_id) : saturn_hee_get_irq_bit(INT_CFG_RDY, a_crtc->dev_id);
+
+	saturn_enable_irq_mask(a_crtc, enable, offset, mask);
+}
+
+void saturn_hee_cfg_ready(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev)
+{
+	u32 base = DPU_SCENE_CTL_ADDR(a_crtc->dev_id);
+
+	if (a_crtc->is_offline_mode == 0) {
+		saturn_enable_irq_mask(a_crtc, true, DPU_INT_BASE_ADDR + DPU_ONLINE_IRQ_MSK, saturn_hee_get_irq_bit(INT_UNDERRUN, a_crtc->dev_id));
+		ur_enabhee = true;
+	}
+	dpu_write(hwdev, DPU_CTL_REG, base, both_cfg_rdy, 1);
+}
+
+void saturn_hee_sw_start(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev)
+{
+	u32 base;
+	base = DPU_SCENE_CTL_ADDR(a_crtc->dev_id);
+	dpu_write_reg(hwdev, DPU_CTL_REG, base, sw_start, 1);
+}
+
+static void saturn_hee_wb_fmt_cvt_matrix(struct spacemit_wb *wb, struct spacemit_hw_device *hwdev, int drm_color_encoding, int drm_color_range, struct cmdlist_regs *cl_wb)
+{
+	u32 wb_base = 0;
+	int index = 13;
+	int val;
+
+	wb_base = WB0_TOP_BASE_ADDR;
+
+	if (wb->format == SPACEMIT_WB_FORMAT_YUV420_P2_8_VU || wb->format == SPACEMIT_WB_FORMAT_YUV420_P2_10_VU) {
+		for (int i = 0; i < SPACEMIT_RGB2YUV_COEFFS; i += 2, index++) {
+			val  = (spacemit_rgb2yuv_coefs[drm_color_encoding][drm_color_range][i] & 0x3FFF) | (spacemit_rgb2yuv_coefs[drm_color_encoding][drm_color_range][i + 1] & 0x3FFF) << 16;
+			dpu_write(hwdev, WB_REG, wb_base, value32[index], val, cl_wb, index);
+		}
+	}
+}
+
+static void saturn_init_csc(struct spacemit_crtc *a_crtc)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u32 module_base;
+	int i = 0;
+
+	hwdev->color_encoding = DRM_COLOR_YCBCR_BT709;
+	hwdev->color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+
+	for (i = 0; i < hwdev->rdma_nums; i++) {
+		module_base = RDMA_BASE_ADDR[i];
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix00, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][0] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix01, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][1] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix02, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][2] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix03, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][3] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix10, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][4] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix11, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][5] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix12, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][6] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix13, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][7] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix20, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][8] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix21, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][9] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix22, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][10] & 0x3FFF);
+		dpu_write(hwdev, RDMA_PATH_X_REG, module_base, csc_matrix23, spacemit_yuv2rgb_coefs[DRM_COLOR_YCBCR_BT709][DRM_COLOR_YCBCR_LIMITED_RANGE][11] & 0x3FFF);
+	}
+}
+
+static void saturn_init_tmg(struct spacemit_crtc *a_crtc)
+{
+	u32 base = 0;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_crtc *crtc = &a_crtc->crtc;
+	struct drm_display_mode *mode = &crtc->mode;
+	u16 vfp, vbp, vsync, hfp, hbp, hsync;
+	void __iomem *tmg_addr;
+	u32 value;
+
+	if (a_crtc->dpu_id == 1)
+		tmg_addr= (void __iomem *)ioremap(0xc0491200, 0x100);
+	else
+		tmg_addr= (void __iomem *)ioremap(0xc0391200, 0x100);
+
+	hsync = mode->hsync_end - mode->hsync_start;
+	hbp = mode->htotal - mode->hsync_end;
+	hfp = mode->hsync_start - mode->hdisplay;
+	vsync = mode->vsync_end - mode->vsync_start;
+	vbp = mode->vtotal - mode->vsync_end;
+	vfp = mode->vsync_start - mode->vdisplay;
+	trace_drm_display_mode_info(mode);
+
+	base = TMG_BASE_ADDR[a_crtc->dev_id];
+	if (base) {
+		dpu_write(hwdev, TMG_REG, base, disp_ready_man_en, 0);
+		dpu_write(hwdev, TMG_REG, base, background_r, 0);
+		dpu_write(hwdev, TMG_REG, base, background_g, 0x0);
+		dpu_write(hwdev, TMG_REG, base, background_b, 0xff);
+		dpu_write(hwdev, TMG_REG, base, eof_1st_ln_dly_num, vfp - 7);
+		dpu_write(hwdev, TMG_REG, base, eof_2nd_ln_dly_num, vfp - 6);
+		dpu_write(hwdev, TMG_REG, base, split_en, 0);
+		dpu_write(hwdev, TMG_REG, base, cmd_screen, 0);
+		dpu_write(hwdev, TMG_REG, base, cmd_wait_en, 0);
+		dpu_write(hwdev, TMG_REG, base, cmd_wait_te, 0);
+		dpu_write(hwdev, TMG_REG, base, sof_pre_ln_num, 0);
+		dpu_write(hwdev, TMG_REG, base, hfp, hfp);
+		dpu_write(hwdev, TMG_REG, base, hbp, hbp);
+		dpu_write(hwdev, TMG_REG, base, vfp, vfp);
+		dpu_write(hwdev, TMG_REG, base, vbp, vbp);
+		dpu_write(hwdev, TMG_REG, base, hsync_width, hsync);
+		dpu_write(hwdev, TMG_REG, base, vsync_width, vsync);
+		dpu_write(hwdev, TMG_REG, base, hsp, 1);
+		dpu_write(hwdev, TMG_REG, base, vsp, 1);
+		dpu_write(hwdev, TMG_REG, base, h_active, mode->hdisplay);
+		dpu_write(hwdev, TMG_REG, base, v_active, mode->vdisplay);
+		dpu_write(hwdev, TMG_REG, base, fm_timing_en, 1);
+		dpu_write(hwdev, TMG_REG, base, user, a_crtc->out_format);
+	}
+
+	if (a_crtc->split_en) {
+		dpu_write(hwdev, TMG_REG, base, split_en, 1);
+		dpu_write(hwdev, TMG_REG, base, split_overlap, 0);
+		dpu_write(hwdev, TMG_REG, base, h_active, (mode->hdisplay / 2));
+	}
+
+	/* set vfp */
+	value = readl(tmg_addr + 0x14);
+	value &= 0xFFFF0000;
+	value |= vfp;
+	writel(value, tmg_addr + 0x14);
+
+	if (a_crtc->is_edp) {
+		/* set vsync align with hsync */
+		value = readl(tmg_addr + 0x3c);
+		value |= 0x01;
+		writel(value, tmg_addr + 0x3c);
+	}
+
+	iounmap(tmg_addr);
+}
+
+static void saturn_init_regs(struct spacemit_crtc *a_crtc)
+{
+	u32 base = 0;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	base = CMP_BASE_ADDR[a_crtc->dev_id];
+	/* set bg color to black */
+	dpu_write(hwdev, CMPS_X_REG, base, bg_color_b, 0x0);
+	dpu_write(hwdev, CMPS_X_REG, base, bg_color_r, 0x0);
+	dpu_write(hwdev, CMPS_X_REG, base, bg_color_g, 0x0);
+	dpu_write(hwdev, CMPS_X_REG, base, bg_color_a, 0xFF);
+	dpu_write(hwdev, CMPS_X_REG, base, module_enable, 1);
+	dpu_write(hwdev, CMPS_X_REG, base, layer00_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer01_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer02_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer03_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer04_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer05_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer06_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer07_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer08_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer09_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer10_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer11_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer12_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer13_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer14_en, 0);
+	dpu_write(hwdev, CMPS_X_REG, base, layer15_en, 0);
+
+	base = DPU_TOP_BASE_ADDR;
+
+	base = EE_ADDR;
+	dpu_write(hwdev, EE_REG, base, m_benable, 0);
+
+	if (a_crtc->is_offline_mode == 0)
+		saturn_init_tmg(a_crtc);
+}
+
+static void saturn_setup_dma_top(struct spacemit_crtc *a_crtc)
+{
+	u32 base = DMA_TOP_BASE_ADDR;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+//	dpu_write(hwdev, DMA_TOP_REG, base, dbg_en, 0x1);
+#endif
+	dpu_write(hwdev, DMA_TOP_REG, base, image_rr_ratio, 0x10);
+//	dpu_write(hwdev, DMA_TOP_REG, base, round_robin_mode, 0);
+	dpu_write(hwdev, DMA_TOP_REG, base, pixel_num_th, 4);
+
+	//regnum: 2, offset: 0x08
+	dpu_write(hwdev, DMA_TOP_REG, base, rdma_timeout_limit, 0xFFFE);
+	dpu_write(hwdev, DMA_TOP_REG, base, wdma_timeout_limit, 0xFFFF);
+}
+
+static void saturn_setup_mmu_top(struct spacemit_crtc *a_crtc)
+{
+	unsigned int rd_outs_num = 0;
+	u32 base = MMU_TOP_BASE_ADDR;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	rd_outs_num = RD_OUTS_NUM / 2;
+
+	/* MMU TOP init setting */
+	dpu_write(hwdev, MMU_TOP_REG, base, rdma_timelimit, RDMA_TIMELIMIT);
+
+	dpu_write(hwdev, MMU_TOP_REG, base, sram0_tlb_axi_port_sel, 0);
+	dpu_write(hwdev, MMU_TOP_REG, base, sram1_tlb_axi_port_sel, 0);
+	dpu_write(hwdev, MMU_TOP_REG, base, dmac0_rd_outs_num, rd_outs_num);
+	dpu_write(hwdev, MMU_TOP_REG, base, dmac1_rd_outs_num, rd_outs_num);
+}
+
+void saturn_hee_irq_enable(struct spacemit_crtc *a_crtc, bool enable)
+{
+	u32 base = DPU_INT_BASE_ADDR, mask = 0, offset = 0;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	if (a_crtc->is_offline_mode == 0) {
+		trace_saturn_irq_enable("irq online", enable);
+		//enable irq mask except vsync
+		offset = DPU_INT_BASE_ADDR + DPU_ONLINE_IRQ_MSK;
+		mask = saturn_hee_get_irq_bit(INT_UNDERRUN, a_crtc->dev_id) | saturn_hee_get_irq_bit(INT_CFG_RDY, a_crtc->dev_id) | saturn_hee_get_irq_bit(INT_WB_DONE, a_crtc->dev_id);
+		saturn_enable_irq_mask(a_crtc, enable, offset, mask);
+		ur_enabhee = true;
+	} else {
+		trace_saturn_irq_enable("irq offline", enable);
+		dpu_write(hwdev, DPU_INTP_REG, base, b.offl0_cfg_rdy_clr_int_msk, enable ? 1 : 0);
+		dpu_write(hwdev, DPU_INTP_REG, base, b.offl0_wb_frm_done_int_msk, enable ? 1 : 0);
+	}
+	/*
+	 * dpu_write(hwdev, DPU_INTP_REG, base, b.cmb_frm_timing_eof_int_msk, enable ? 1 : 0);
+	 * dpu_write(hwdev, DPU_INTP_REG, base, b.cmb_cmdlist_ch_frm_cfg_done_int_msk, enable ? 0xF : 0);
+	 */
+}
+
+void saturn_hee_dpu_init(struct spacemit_crtc *a_crtc)
+{
+	saturn_init_regs(a_crtc);
+	saturn_setup_dma_top(a_crtc);
+	saturn_setup_mmu_top(a_crtc);
+	saturn_hee_irq_enable(a_crtc, true);
+	saturn_init_csc(a_crtc);
+}
+
+static void spacemit_set_afbc_info(struct spacemit_plane_state *spacemit_plane_state, struct spacemit_hw_device *hwdev, uint64_t modifier, u32 module_base, struct cmdlist_regs *cl_rdma)
+{
+	spacemit_plane_state->afbc_state = kzalloc(sizeof(struct spacemit_afbc_state), GFP_KERNEL);
+	if (!spacemit_plane_state->afbc_state) {
+		DRM_ERROR("Faild to set afbc plane\n");
+		return;
+	}
+
+	spacemit_get_afbc_modifier(modifier, spacemit_plane_state->afbc_state);
+
+	dpu_write(hwdev, RDMA_PATH_X_REG, module_base, fbc_tile_type, spacemit_plane_state->afbc_state->tile_type, cl_rdma, 30);
+	dpu_write(hwdev, RDMA_PATH_X_REG, module_base, fbc_yuv_transform, spacemit_plane_state->afbc_state->yuv_transform, cl_rdma, 30);
+	dpu_write(hwdev, RDMA_PATH_X_REG, module_base, fbc_split_mode, spacemit_plane_state->afbc_state->split_mode, cl_rdma, 30);
+	dpu_write(hwdev, RDMA_PATH_X_REG, module_base, fbc_sb_layout, spacemit_plane_state->afbc_state->block_size, cl_rdma, 30);
+
+	kfree(spacemit_plane_state->afbc_state);
+}
+
+#define CONFIG_HW_COMPOSER_LAYER(id_name, id) \
+{\
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _colorkey_en, 0, cl_cmp, id * 7 + 13); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _blend_mode, blend_mode, cl_cmp, id * 7 + 13); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _blend_sel, alpha_sel, cl_cmp, id * 7 + 14); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _layer_alpha, alpha, cl_cmp, id * 7 + 14); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _alpha_ratio, 0, cl_cmp, id * 7 + 14); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _area_left, crtc_x, cl_cmp, id * 7 + 8); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _area_top, crtc_y, cl_cmp, id * 7 + 9); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _area_right, crtc_x + crtc_w - 1,cl_cmp, id * 7 + 8); \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _area_bottom, crtc_y + crtc_h - 1,cl_cmp, id * 7 + 9); \
+	if (unlikely(solid_en)) { \
+		dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _solid_color_r, solid_r, cl_cmp, id * 7 + 10); \
+		dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _solid_color_g, solid_g, cl_cmp, id * 7 + 11); \
+		dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _solid_color_b, solid_b, cl_cmp, id * 7 + 12); \
+		dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _solid_color_a, solid_a, cl_cmp, id * 7 + 13); \
+		dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _solid_en, 1, cl_cmp, id * 7 + 14); \
+		DRM_DEBUG("solid_r:0x%x, solid_g:0x%x, solid_b:0x%x, solid_a:0x%x\n", \
+			solid_r, solid_g, solid_b, solid_a); \
+	} else { \
+		dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _solid_en, 0, cl_cmp, id * 7 + 14); \
+		dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _dma_id, rdma_id, cl_cmp, id * 7 + 14); \
+	} \
+	dpu_write(hwdev, CMPS_X_REG, base, layer ## id_name ## _en, 1, cl_cmp, id * 7 + 14); \
+}
+
+static void saturn_write_fbcmem_regs(struct spacemit_hw_device *hwdev, struct drm_plane_state *state, u32 rdma_id,
+				     u32 base, struct cmdlist_regs *cl_rdma)
+{
+	struct drm_crtc_state *crtc_state = state->crtc->state;
+	const struct spacemit_crtc_rdma *rdmas = to_spacemit_crtc_state(crtc_state)->rdmas;
+	u32 size  = rdmas[rdma_id].fbcmem.size;// / FBCMEM_UNIT;
+	u32 start = rdmas[rdma_id].fbcmem.start;// / FBCMEM_UNIT;
+	bool map   = rdmas[rdma_id].fbcmem.map;
+
+	dpu_write(hwdev, RDMA_PATH_X_REG, base, value32[31], map << 28 | start << 16 | size, cl_rdma, 31);
+}
+
+static void dpu_saturn_scaler_reuse_en(struct drm_plane *plane, bool enable)
+{
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct cmdlist_regs *cl_dpuctl = NULL;
+	struct drm_plane_state *state = plane->state;
+	struct spacemit_plane_state *spacemit_plane_state = to_spacemit_plane_state(state);
+	u32 rdma_id = spacemit_plane_state->rdma_id;
+
+	cl_dpuctl = alloc_cmdlist_regs(DPU_CTL_TOP_REG);
+	DRM_DEBUG("silvie dpu_saturn_scaler_reuse_en rch id %d\n",rdma_id);
+
+	switch (rdma_id){
+		case DPU_RDMA0:
+			dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR, nml_scl0_reuse_en, enable, cl_dpuctl, 49);
+			break;
+		case DPU_RDMA1:
+			dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR, nml_scl1_reuse_en, enable, cl_dpuctl, 50);
+			break;
+		case DPU_RDMA2:
+			dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR, nml_scl2_reuse_en, enable, cl_dpuctl, 51);
+			break;
+		case DPU_RDMA3:
+			dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR, nml_scl3_reuse_en, enable, cl_dpuctl, 52);
+			break;
+		default:
+			DRM_ERROR("Unsupported RDMA id for scaler!\n");
+	}
+
+	cmdlist_regs_packing(plane_to_cl(plane), CMDLIST_MOD_RDMA, cl_dpuctl);
+	free_cmdlist_regs(cl_dpuctl);
+}
+
+void saturn_hee_plane_update_hw_channel(struct drm_plane *plane, int slice_id)
+{
+	struct drm_plane_state *state = plane->state;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(state->crtc);
+	struct drm_crtc_state *crtc_state = a_crtc->crtc.state;
+	struct spacemit_crtc_state *spacemit_crtc_state = to_spacemit_crtc_state(crtc_state);
+	struct spacemit_plane *spacemit_plane = to_spacemit_plane(plane);
+	struct drm_framebuffer *fb = plane->state->fb;
+	struct spacemit_plane_state *spacemit_plane_state = to_spacemit_plane_state(state);
+	struct cmdlist_regs *cl_cmp = NULL;
+	u32 rdma_id = spacemit_plane_state->rdma_id;
+	u8 alpha = state->alpha >> 8;
+	u16 pixel_alpha = state->pixel_blend_mode;
+	u32 src_w, src_h, src_x, src_y;
+	u32 crtc_w, crtc_h, crtc_x, crtc_y;
+	u32 alpha_sel, blend_mode = 0;
+	u8 uv_swap = 0;
+	uint32_t fccf = fb->format->format;
+	struct drm_display_mode *mode = &state->crtc->mode;
+
+	u32 base;
+	u32 val;
+	bool solid_en = false;
+	u32 solid_a, solid_r, solid_g, solid_b;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	bool is_afbc = (fb->modifier > 0);
+	struct cmdlist_regs *cl_rdma = a_crtc->cl_rdma;
+	int i = 0;
+
+	trace_spacemit_plane_update_hw_channel("rdma_id", rdma_id);
+
+	if (a_crtc->is_slice_mode == 1) {
+		src_w = a_crtc->slice_wb[slice_id].rdma_w;
+		src_h = a_crtc->slice_wb[slice_id].rdma_h;
+		src_x = a_crtc->slice_wb[slice_id].rdma_x;
+		src_y = a_crtc->slice_wb[slice_id].rdma_y;
+
+		crtc_w = a_crtc->slice_wb[slice_id].crtc_w;
+		crtc_h = a_crtc->slice_wb[slice_id].crtc_h;
+		crtc_x = a_crtc->slice_wb[slice_id].crtc_x;
+		crtc_y = a_crtc->slice_wb[slice_id].crtc_y;
+	} else {
+		src_w = state->src_w >> 16;
+		src_h = state->src_h >> 16;
+		src_x = state->src_x >> 16;
+		src_y = state->src_y >> 16;
+
+		crtc_w = state->crtc_w;
+		crtc_h = state->crtc_h;
+		crtc_x = state->crtc_x;
+		crtc_y = state->crtc_y;
+	}
+
+	DRM_DEBUG("crtc_x %u crtc_x %u\n", crtc_x, crtc_y);
+
+	if (rdma_id == RDMA_INVALID_ID)
+		solid_en = true;
+
+
+	trace_dpu_plane_info(state, fb, rdma_id, alpha,
+			     state->rotation, is_afbc);
+
+	/* For solid color both src_w and src_h are 0 */
+	if (solid_en == false) {
+		/* Set RDMA regs */
+		base = RDMA_BASE_ADDR[rdma_id];
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, layer_mode, is_afbc ? 1 : 0, cl_rdma, 0);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, layer_cmpsr_id, a_crtc->dev_id, cl_rdma, 0);
+		if (spacemit_plane_state->is_offline) {
+			dpu_write(hwdev, RDMA_PATH_X_REG, base, is_offline, 1, cl_rdma, 1);
+		} else {
+			dpu_write(hwdev, RDMA_PATH_X_REG, base, is_offline, 0, cl_rdma, 1);
+		}
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, is_two_layers, 0, cl_rdma, 1);
+
+		// dpu_write(hwdev, RDMA_PATH_X_REG, base, cmpsr_y_offset0, crtc_y, cl_rdma, 2);
+		// dpu_writel(hwdev->base, base + 0x40, fb->width | fb->height << 16);
+		// dpu_writel(hwdev->base, base + 0x44, src_x | src_x << 16);
+		// dpu_writel(hwdev->base, base + 0x48, (src_x + src_w - 1) | (src_y + src_h - 1) << 16);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, img_width_ly0 , fb->width, cl_rdma, 16);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, img_height_ly0 , fb->height, cl_rdma, 16);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, bbox_start_x_ly0 , src_x, cl_rdma, 17);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, bbox_start_y_ly0 , src_y, cl_rdma, 17);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, bbox_end_x_ly0 , src_x + src_w - 1, cl_rdma, 18);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, bbox_end_y_ly0 , src_y + src_h - 1, cl_rdma, 18);
+
+		saturn_write_fbcmem_regs(hwdev, state, rdma_id, base, cl_rdma);
+
+		val = 0;
+		/* setup the rotation and axis flip bits */
+		if (state->rotation & DRM_MODE_ROTATE_MASK)
+			val = ilog2(plane->state->rotation & DRM_MODE_ROTATE_MASK) & 0x3;
+		if (state->rotation & DRM_MODE_REFLECT_MASK) {
+			if (state->rotation & DRM_MODE_ROTATE_90) {
+				if (state->rotation & DRM_MODE_REFLECT_X)
+					val = ROT_MODE_FLIP_90;
+				if (state->rotation & DRM_MODE_REFLECT_Y)
+					val = ROT_MODE_MIRROR_90;
+			} else {
+				if (state->rotation & DRM_MODE_REFLECT_X)
+					val |= ROT_MODE_MIRROR;
+				if (state->rotation & DRM_MODE_REFLECT_Y)
+					val |= ROT_MODE_FLIP;
+			}
+		}
+
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, rot_mode_ly0, val, cl_rdma, 29);
+		if (fccf == DRM_FORMAT_YVYU || fccf == DRM_FORMAT_VYUY || fccf == DRM_FORMAT_NV21 || fccf == DRM_FORMAT_YVU420 || fccf == DRM_FORMAT_Q401)
+			uv_swap = 1;
+
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, uv_swap, uv_swap, cl_rdma, 29);
+		dpu_write(hwdev, RDMA_PATH_X_REG, base, pixel_format, spacemit_plane_state->format, cl_rdma, 29);
+
+		if (fb->modifier)
+			spacemit_set_afbc_info(spacemit_plane_state, hwdev, fb->modifier, base, cl_rdma);
+
+		if (spacemit_plane_state->use_scl) {
+			hwdev->conf_scaler_x(state, NULL);
+
+			dpu_saturn_scaler_reuse_en(plane, true);
+
+			/*scaling mismatch gpu render result without these coefs*/
+			hwdev->conf_scaler_coefs(plane, spacemit_plane_state, NULL);
+		} else {
+			for (i = 0; i < MAX_SCALER_NUMS; i++) {
+				if (spacemit_crtc_state->scl_rdma_id[i] == rdma_id) {
+					dpu_saturn_scaler_reuse_en(plane, false);
+					break;
+				}
+			}
+		}
+	} else {
+		solid_r = (u8)spacemit_plane_state->solid_color;
+		solid_g = (u8)(spacemit_plane_state->solid_color >> 8);
+		solid_b = (u8)(spacemit_plane_state->solid_color >> 16);
+
+		solid_r = (0x3ff * solid_r) / 0xff;
+		solid_g = (0x3ff * solid_g) / 0xff;
+		solid_b = (0x3ff * solid_b) / 0xff;
+
+		solid_a = (u8)(spacemit_plane_state->solid_color >> 24);
+		solid_r = solid_r << hwdev->solid_color_shift;
+		solid_g = solid_g << hwdev->solid_color_shift;
+		solid_b = solid_b << hwdev->solid_color_shift;
+	}
+
+	/* Set layer regs for blend mode */
+	switch (pixel_alpha) {
+		// Pixel blend mode: coverage blend
+	case DRM_MODE_BLEND_COVERAGE:
+		blend_mode = 0x0;
+		break;
+	// Pixel blend mode: premultiplied blend
+	case DRM_MODE_BLEND_PREMULTI:
+		blend_mode = 0x1;
+		break;
+	// Pixel blend mode: blend none
+	case DRM_MODE_BLEND_PIXEL_NONE:
+		blend_mode = 0x0;
+		break;
+	default:
+		DRM_ERROR("Unsupported blend mode for pixel alpha!\n");
+	}
+
+	/* Set layer regs for alpha mode */
+	if (state->fb->format && (state->fb->format->has_alpha) && (pixel_alpha != DRM_MODE_BLEND_PIXEL_NONE)) {
+		// Exist pixel alpha if come to here.
+		if (alpha != 0xff) {
+			//Pixel alpha + layer alpha(combined alpha)
+			alpha_sel = 0x2;
+		} else {
+			//Pixel alpha
+			alpha_sel = 0x1;
+		}
+	} else {
+		// None pixel alpha if come to here.
+		blend_mode = 0x0;
+
+		// Layer alpha
+		alpha_sel = 0x0;
+	}
+
+	cl_cmp = alloc_cmdlist_regs(CMPS_X_REG);
+
+	/* enable composer and bind RDMA */
+	base = CMP_BASE_ADDR[a_crtc->dev_id];
+	if (a_crtc->is_slice_mode == 1) {
+		dpu_write(hwdev, CMPS_X_REG, base, dst_w, crtc_w, cl_cmp, 1);
+		dpu_write(hwdev, CMPS_X_REG, base, dst_h, crtc_h, cl_cmp, 1);
+	} else {
+		dpu_write(hwdev, CMPS_X_REG, base, dst_w, spacemit_crtc_state->post_scl_on ? spacemit_crtc_state->post_scaler_w : mode->hdisplay, cl_cmp, 1);
+		dpu_write(hwdev, CMPS_X_REG, base, dst_h, spacemit_crtc_state->post_scl_on ? spacemit_crtc_state->post_scaler_h : mode->vdisplay, cl_cmp, 1);
+		// dpu_writel(hwdev->base, base + 0x4, mode->hdisplay | mode->vdisplay << 16);
+	}
+
+	// dpu_writel(hwdev->base, base + 0x1c4, 0 | ((crtc_x + crtc_w - 1) << 16));
+	// dpu_writel(hwdev->base, base + 0x1c8, 0 | ((crtc_y + crtc_h - 1) << 16));
+	switch (spacemit_plane->hw_pid) {
+	case 0:
+		CONFIG_HW_COMPOSER_LAYER(00, 0);
+		break;
+	case 1:
+		CONFIG_HW_COMPOSER_LAYER(01, 1);
+		break;
+	case 2:
+		CONFIG_HW_COMPOSER_LAYER(02, 2);
+		break;
+	case 3:
+		CONFIG_HW_COMPOSER_LAYER(03, 3);
+		break;
+	case 4:
+		CONFIG_HW_COMPOSER_LAYER(04, 4);
+		break;
+	case 5:
+		CONFIG_HW_COMPOSER_LAYER(05, 5);
+		break;
+	case 6:
+		CONFIG_HW_COMPOSER_LAYER(06, 6);
+		break;
+	case 7:
+		CONFIG_HW_COMPOSER_LAYER(07, 7);
+		break;
+	case 8:
+		CONFIG_HW_COMPOSER_LAYER(08, 8);
+		break;
+	case 9:
+		CONFIG_HW_COMPOSER_LAYER(09, 9);
+		break;
+	case 10:
+		CONFIG_HW_COMPOSER_LAYER(10, 10);
+		break;
+	case 11:
+		CONFIG_HW_COMPOSER_LAYER(11, 11);
+		break;
+	case 12:
+		CONFIG_HW_COMPOSER_LAYER(12, 12);
+		break;
+	case 13:
+		CONFIG_HW_COMPOSER_LAYER(13, 13);
+		break;
+	case 14:
+		CONFIG_HW_COMPOSER_LAYER(14, 14);
+		break;
+	case 15:
+		CONFIG_HW_COMPOSER_LAYER(15, 15);
+		break;
+	default:
+		DRM_ERROR("%s unsupported zpos:%d\n", __func__, state->zpos);
+		break;
+	}
+	cmdlist_regs_packing(crtc_to_cl(plane->state->crtc), CMDLIST_MOD_COMP, cl_cmp);
+	free_cmdlist_regs(cl_cmp);
+}
+
+void saturn_hee_plane_disable_hw_channel(struct drm_plane *plane, struct drm_plane_state *old_state)
+{
+	struct spacemit_plane *p = to_spacemit_plane(plane);
+	u8 dev_id = to_spacemit_crtc(old_state->crtc)->dev_id;
+	u32 base = CMP_BASE_ADDR[dev_id];
+	u32 rdma_id = to_spacemit_plane_state(old_state)->rdma_id;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct cmdlist_regs *cl_cmp = NULL;
+
+	trace_spacemit_plane_disable_hw_channel(p->hw_pid, rdma_id);
+	cl_cmp = alloc_cmdlist_regs(CMPS_X_REG);
+	switch (p->hw_pid) {
+	case 0:
+		dpu_write(hwdev, CMPS_X_REG, base, layer00_en, 0, cl_cmp, 14);
+		break;
+	case 1:
+		dpu_write(hwdev, CMPS_X_REG, base, layer01_en, 0, cl_cmp, 21);
+		break;
+	case 2:
+		dpu_write(hwdev, CMPS_X_REG, base, layer02_en, 0, cl_cmp, 28);
+		break;
+	case 3:
+		dpu_write(hwdev, CMPS_X_REG, base, layer03_en, 0, cl_cmp, 35);
+		break;
+	case 4:
+		dpu_write(hwdev, CMPS_X_REG, base, layer04_en, 0,  cl_cmp, 42);
+		break;
+	case 5:
+		dpu_write(hwdev, CMPS_X_REG, base, layer05_en, 0,  cl_cmp, 49);
+		break;
+	case 6:
+		dpu_write(hwdev, CMPS_X_REG, base, layer06_en, 0,  cl_cmp, 56);
+		break;
+	case 7:
+		dpu_write(hwdev, CMPS_X_REG, base, layer07_en, 0, cl_cmp, 63);
+		break;
+	case 8:
+		dpu_write(hwdev, CMPS_X_REG, base, layer08_en, 0, cl_cmp, 70);
+		break;
+	case 9:
+		dpu_write(hwdev, CMPS_X_REG, base, layer09_en, 0, cl_cmp, 77);
+		break;
+	case 10:
+		dpu_write(hwdev, CMPS_X_REG, base, layer10_en, 0, cl_cmp, 84);
+		break;
+	case 11:
+		dpu_write(hwdev, CMPS_X_REG, base, layer11_en, 0, cl_cmp, 91);
+		break;
+	case 12:
+		dpu_write(hwdev, CMPS_X_REG, base, layer12_en, 0, cl_cmp, 98);
+		break;
+	case 13:
+		dpu_write(hwdev, CMPS_X_REG, base, layer13_en, 0, cl_cmp, 105);
+		break;
+	case 14:
+		dpu_write(hwdev, CMPS_X_REG, base, layer14_en, 0, cl_cmp, 112);
+		break;
+	case 15:
+		dpu_write(hwdev, CMPS_X_REG, base, layer15_en, 0, cl_cmp, 119);
+		break;
+	default:
+		DRM_ERROR("%s unsupported zpos:%d %d\n", __func__, old_state->zpos, p->hw_pid);
+		break;
+	}
+
+	cmdlist_regs_packing(crtc_to_cl(old_state->crtc), CMDLIST_MOD_COMP, cl_cmp);
+	free_cmdlist_regs(cl_cmp);
+}
+
+
+static u32 saturn_conf_dpuctrl_scaling(struct spacemit_crtc *a_crtc)
+{
+	struct spacemit_crtc_scaler *scaler = NULL;
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(a_crtc->crtc.state);
+	u32 scl_en = 0;
+	u32 i;
+
+	for (i = 0; i < MAX_SCALER_NUMS; i++) {
+		scaler = &(ac->scalers[i]);
+		DRM_DEBUG("scaler%d: in_use:0x%x rdma_id:%d\n", i, scaler->in_use, scaler->rdma_id);
+		trace_dpuctrl_scaling_setting(i, scaler->in_use, scaler->rdma_id);
+		if (scaler->in_use)
+			scl_en |= 1 << scaler->rdma_id;
+	}
+
+	DRM_DEBUG("scl_en:%d ac->post_scl_on %d\n", scl_en, ac->post_scl_on);
+	trace_dpuctrl_scaling("scl_en", scl_en);
+	return scl_en;
+}
+
+void saturn_hee_conf_dpuctrl(struct drm_crtc *crtc,
+		    struct drm_crtc_state *old_state)
+{
+	u32 scl_en = 0;
+	u32 rdma_en = 0;
+	u32 base;
+	u32 pp_base = POST_PIPE_ADDR;
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct drm_display_mode *mode = &crtc->mode;
+	struct drm_crtc_state *crtc_state = a_crtc->crtc.state;
+	struct spacemit_crtc_state *spacemit_crtc_state = to_spacemit_crtc_state(crtc_state);
+	struct cmdlist *cl = NULL;
+	struct cmdlist_regs *pp_cl = NULL;
+
+	base = DPU_SCENE_CTL_ADDR(a_crtc->dev_id);
+	rdma_en = saturn_conf_dpuctrl_rdma(a_crtc);
+	scl_en = saturn_conf_dpuctrl_scaling(a_crtc);
+
+	if (a_crtc->is_offline_mode == 0) {
+		pp_cl = alloc_cmdlist_regs(POSTPIPE_REG);
+		//postpipe should be configed no matter whether pq function is on or off
+		if (a_crtc->split_en) {
+			dpu_write(hwdev, POSTPIPE_REG, pp_base, value32[0], 0x20, pp_cl, 0);
+		} else {
+			dpu_write(hwdev, POSTPIPE_REG, pp_base, value32[0], 0, pp_cl, 0);
+		}
+		// dpu_writel(hwdev->base, pp_base + 0x34, mode->hdisplay | mode->vdisplay << 16);
+		dpu_write(hwdev, POSTPIPE_REG, pp_base, m_inwidth, spacemit_crtc_state->post_scl_on ? spacemit_crtc_state->post_scaler_w : mode->hdisplay, pp_cl, 13);
+		dpu_write(hwdev, POSTPIPE_REG, pp_base, m_inheight, spacemit_crtc_state->post_scl_on ? spacemit_crtc_state->post_scaler_h : mode->vdisplay, pp_cl, 13);
+
+		cmdlist_regs_packing(crtc_to_cl(crtc), CMDLIST_MOD_COMP, pp_cl);
+		free_cmdlist_regs(pp_cl);
+	}
+
+	cmdlist_sort_by_group(crtc);
+	cmdlist_atomic_commit(crtc, old_state);
+
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_scl_en, scl_en);
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_rch_en, rdma_en);
+	dpu_write(hwdev, DPU_CTL_REG, base, timing_inter0, 8);
+	dpu_write(hwdev, DPU_CTL_REG, base, timing_inter1, 8);
+	if (a_crtc->is_offline_mode == 0) {
+		dpu_write(hwdev, DPU_CTL_REG, base, video_mod, 1);
+	} else {
+		dpu_write(hwdev, DPU_CTL_REG, base, video_mod, 0);
+		dpu_write(hwdev, DPU_CTL_REG, base, nml_cmd_updt_en, 1);
+		if (rdma_en == 0) {
+			dpu_write(hwdev, DPU_CTL_REG, base, nml_wb_en, 0);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_en, 0);
+		}
+	}
+	dpu_write(hwdev, DPU_CTL_REG, base, dbg_mod, 0);
+
+	if (a_crtc->is_offline_mode == 0) {
+		a_crtc->dpu_online_nml_scl_en = scl_en;
+		a_crtc->dpu_online_nml_rch_en = rdma_en;
+		a_crtc->dpu_online_nml_outctl_en = 0x1;
+		dpu_write(hwdev, DPU_CTL_REG, base, nml_outctl_en, 0x1);
+		dpu_write(hwdev, DPU_CTL_REG, base, nml_frm_timing_en, 2);
+		if (!a_crtc->is_edp) {
+			dpu_write(hwdev, TMG_REG, TMG_BASE_ADDR[a_crtc->dev_id], vfp, a_crtc->vrr_vfp);
+			dpu_write(hwdev, TMG_REG, TMG_BASE_ADDR[a_crtc->dev_id], eof_1st_ln_dly_num, a_crtc->vrr_vfp - 7);
+			dpu_write(hwdev, TMG_REG, TMG_BASE_ADDR[a_crtc->dev_id], eof_2nd_ln_dly_num, a_crtc->vrr_vfp - 6);
+		}
+	} else {
+		dpu_write(hwdev, DPU_CTL_REG, base, nml_outctl_en, 0);
+		dpu_write(hwdev, DPU_CTL_REG, base, nml_frm_timing_en, 0);
+	}
+	cl = &spacemit_crtc_state->cl[0];
+	if (cl->va != NULL) {
+		wb_cmdlist_sort_by_group((void *)a_crtc);
+		wb_cmdlist_atomic_commit((void *)a_crtc);
+	}
+}
+
+void saturn_hee_wb_disable(struct spacemit_crtc *a_crtc)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u32 base = DPU_SCENE_CTL_ADDR(a_crtc->dev_id);
+
+	if (a_crtc->is_offline_mode == 0) {
+		dpu_write(hwdev, DPU_CTL_REG, base, nml_wb_en, 0x0);
+		dpu_write(hwdev, DPU_CTL_REG, base, both_cfg_rdy, 1);
+	} else {
+		dpu_write(hwdev, DPU_INTP_REG, DPU_INT_BASE_ADDR, b.offl0_wb_frm_done_int_msk, 0);
+	}
+}
+
+int saturn_hee_is_wb_en(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev)
+{
+	int ctl_base = DPU_SCENE_CTL_ADDR(a_crtc->dev_id);
+
+	return dpu_read_reg(hwdev, DPU_CTL_REG, ctl_base, nml_wb_en);
+}
+
+static int wb_stide_cal(int format)
+{
+	int stride = 0;
+
+	switch (format) {
+	case SPACEMIT_WB_FORMAT_ARGB2101010:
+	case SPACEMIT_WB_FORMAT_ABGR2101010:
+	case SPACEMIT_WB_FORMAT_RGBA1010102:
+	case SPACEMIT_WB_FORMAT_BGRA1010102:
+	case SPACEMIT_WB_FORMAT_ARGB8888:
+	case SPACEMIT_WB_FORMAT_ABGR8888:
+	case SPACEMIT_WB_FORMAT_RGBA8888:
+	case SPACEMIT_WB_FORMAT_BGRA8888:
+	case SPACEMIT_WB_FORMAT_XRGB8888:
+	case SPACEMIT_WB_FORMAT_XBGR8888:
+	case SPACEMIT_WB_FORMAT_RGBX8888:
+	case SPACEMIT_WB_FORMAT_BGRX8888:
+		stride = 4;
+		break;
+	case SPACEMIT_WB_FORMAT_RGB888:
+	case SPACEMIT_WB_FORMAT_BGR888:
+		stride = 3;
+		break;
+	case SPACEMIT_WB_FORMAT_BGRA5551:
+	case SPACEMIT_WB_FORMAT_ABGR1555:
+	case SPACEMIT_WB_FORMAT_ARGB1555:
+	case SPACEMIT_WB_FORMAT_RGBX5551:
+	case SPACEMIT_WB_FORMAT_BGRX5551:
+	case SPACEMIT_WB_FORMAT_XBGR1555:
+	case SPACEMIT_WB_FORMAT_XRGB1555:
+	case SPACEMIT_WB_FORMAT_RGB565:
+	case SPACEMIT_WB_FORMAT_BGR565:
+		stride = 2;
+		break;
+	default:
+		stride = 1;
+	}
+	return stride;
+
+}
+
+static void saturn_wb_scaler_config(struct spacemit_wb *wb, struct spacemit_hw_device *hwdev)
+{
+#if 0
+	u8 wb_id = wb->ctx.id;
+	u32 wb_base = 0;
+	u32 out_w, out_h;
+	u32 wb_scl_base = 0;
+	u32 hor_delta_phase = 0;
+	u32 ver_delta_phase = 0;
+	int64_t hor_init_phase = 0;
+	int64_t ver_init_phase = 0;
+
+	DRM_DEBUG("%s(%d)\n", __func__, wb_id);
+	return;
+	if (wb_id == 0) {
+		wb_base = WB0_TOP_BASE_ADDR;
+		wb_scl_base = WB0_SCALER_BASE_ADDR;
+	} else {
+		wb_base = WB1_TOP_BASE_ADDR;
+		wb_scl_base = WB1_SCALER_BASE_ADDR;
+	}
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_scl_en, 1);
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_hor_enable, 1);
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_ver_enable, 1);
+
+	hor_delta_phase = wb->in_w * 65536 / wb->out_w;
+	ver_delta_phase = wb->in_h * 65536 / wb->out_h;
+	hor_init_phase = ((((int64_t)hor_delta_phase) - 65536) >> 1);
+	ver_init_phase = ((((int64_t)ver_delta_phase) - 65536) >> 1);
+	if (hor_init_phase >= 0) {
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_hor_init_phase_l32b, hor_init_phase & 0x00000000ffffffff);
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_hor_init_phase_h1b, 0);
+	} else {
+		hor_init_phase += 8589934592;
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_hor_init_phase_l32b, hor_init_phase & 0x00000000ffffffff);
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_hor_init_phase_h1b, (u32)((hor_init_phase & 0x0000000100000000) >> 32));
+	}
+
+	if (ver_init_phase >= 0) {
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_ver_init_phase_l32b, ver_init_phase & 0x00000000ffffffff);
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_ver_init_phase_h1b, 0);
+	} else {
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_ver_init_phase_l32b, ver_init_phase & 0x00000000ffffffff);
+		dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_ver_init_phase_h1b, (u32)((ver_init_phase & 0x0000000100000000) >> 32));
+	}
+
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_hor_delta_phase, hor_delta_phase);
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_ver_delta_phase, ver_delta_phase);
+
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_input_width, wb->in_w);
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_input_height, wb->in_h);
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_output_width, wb->out_w);
+	dpu_write_reg(hwdev, SCALER_8X6TAP_REG, wb_scl_base, m_nscl_output_height, wb->out_h);
+
+	if (is_rot_90_270(wb->rotation)) {
+		out_w = wb->out_h;
+		out_h = wb->out_w;
+	} else {
+		out_w = wb->out_w;
+		out_h = wb->out_h;
+	}
+	int stride = wb_stide_cal(wb->format);
+
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_out_ori_width, out_w);
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_out_ori_height, out_h);
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_out_subframe_width, out_w);
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_out_subframe_height, out_h);
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_wdma_stride, out_w * stride);
+#endif
+}
+
+void saturn_hee_wb_config(struct spacemit_crtc *a_crtc, struct spacemit_wb *wb, struct drm_framebuffer *fb, struct cmdlist_regs *cl_wb, int slice_id)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	unsigned int wb_pos = SPACEMIT_WB_COMP0;
+	unsigned int wb_format = SPACEMIT_WB_FORMAT_ARGB8888;
+	unsigned int wb_en = 1 << a_crtc->wb_id;
+	u32 wb_base = WB0_TOP_BASE_ADDR;
+	u32 val;
+	struct drm_crtc *crtc = &a_crtc->crtc;
+	struct drm_display_mode *mode = &crtc->mode;
+	u32 wb_width = mode->hdisplay;
+	u32 wb_height = mode->vdisplay;
+	int afbc_en, stride;
+
+	DRM_DEBUG("SPACEMIT_WB(%d) FB: w %d, h %d, stride %d, modifier %llu", a_crtc->wb_id, fb->width, fb->height, fb->pitches[0], fb->modifier);
+	DRM_DEBUG("SPACEMIT_WB(%d) mode: w %d, h %d", a_crtc->wb_id, wb_width, wb_height);
+	DRM_DEBUG("SPACEMIT_WB scl: %d %d %d %d %d %d %d %d\n", wb->in_x, wb->in_y, wb->in_w, wb->in_h, wb->out_x, wb->out_y, wb->out_w, wb->out_h);
+
+	if (a_crtc->is_offline_mode == 1)
+		wb_pos = OFFLINE_COMPOSE_WB_POS;
+	else
+		wb_pos = a_crtc->wb_pos;
+
+	if (wb->format != SPACEMIT_WB_INVALID_FORMAT_ID)
+		wb_format = wb->format;
+
+	if (a_crtc->wb_id == 0) {
+		wb_base = WB0_TOP_BASE_ADDR;
+		dpu_write_reg(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR, wb0_sel_id, wb_pos);
+	}
+	dpu_write_reg(hwdev, DPU_CTL_REG, DPU_SCENE_CTL_ADDR(a_crtc->dev_id), nml_wb_en, wb_en);
+
+	afbc_en = (fb->modifier > 0);
+	DRM_DEBUG("SPACEMIT_WB%d: wb_en %x, wb_pos %d, wb_base = 0x%x\n", a_crtc->wb_id, wb_en, wb_pos, wb_base);
+	dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, m_nwb_proc_en, 0, cl_wb, 0);
+	dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, m_nmatrix_en, 0, cl_wb, 0);
+	dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, m_noetf_en, 0, cl_wb, 0);
+	dpu_write(hwdev, WB_REG, wb_base, wb_dither_en, 0, cl_wb, 0);
+	dpu_write(hwdev, WB_REG, wb_base, wb_out_format, wb_format, cl_wb, 0);
+	dpu_write(hwdev, WB_REG, wb_base, wb_in_crop_en, 0, cl_wb, 0);
+	dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR,  wb_wdma_burst_len, 0x10, cl_wb, 9);
+	dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_wdma_outstanding_num, 0x10, cl_wb, 8);
+
+	if (afbc_en) {
+		wb->afbc_state = kzalloc(sizeof(struct spacemit_afbc_state), GFP_KERNEL);
+		if (!wb->afbc_state)
+			DRM_ERROR("Faild to set wb afbc\n");
+
+		spacemit_get_afbc_modifier(fb->modifier, wb->afbc_state);
+
+		dpu_write(hwdev, WB_REG, wb_base, wb_wdma_fbc_en, afbc_en, cl_wb, 0);
+		dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_fbc_mode, wb->afbc_state->block_size, cl_wb, 0);
+		dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, fbc_tile_hd_mode_en, wb->afbc_state->tile_type, cl_wb, 10);
+		dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, fbc_split_en, wb->afbc_state->split_mode, cl_wb, 10);
+		dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, yuv_transform_en, wb->afbc_state->yuv_transform, cl_wb, 10);
+		dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, default_color_enable, 1, cl_wb, 10);
+		dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, copy_mode_enable, wb->afbc_state->copy_mode, cl_wb, 10);
+
+		kfree(wb->afbc_state);
+	}
+
+	/* rot */
+	val = ilog2(wb->rotation & DRM_MODE_ROTATE_MASK) & 0x3;
+	dpu_write(hwdev, WB_REG, wb_base, wb_rot_mode, val, cl_wb, 0);
+
+	if (wb->in_w)
+		wb_width = wb->in_w;
+	if (wb->in_h)
+		wb_height = wb->in_h;
+
+
+	if (a_crtc->is_slice_mode) {
+		dpu_write(hwdev, WB_REG, wb_base, wb_in_width, a_crtc->slice_wb[slice_id].wb_in_w, cl_wb, 1);
+		dpu_write(hwdev, WB_REG, wb_base, wb_in_height, a_crtc->slice_wb[slice_id].wb_in_h, cl_wb, 1);
+	} else {
+		dpu_write(hwdev, WB_REG, wb_base, wb_in_width, wb_width, cl_wb, 1);
+		dpu_write(hwdev, WB_REG, wb_base, wb_in_height, wb_height, cl_wb, 1);
+		// dpu_writel(hwdev->base, WB0_TOP_BASE_ADDR + 0x04, wb_height | wb_width << 16);
+	}
+
+	if ((wb->in_w != wb->out_w) || (wb->in_h != wb->out_h))
+		wb->use_scl = true;
+	else
+		wb->use_scl = false;
+
+	stride = wb_stide_cal(wb_format);
+	DRM_DEBUG("SPACEMIT_WB format %x\n", wb_format);
+	DRM_DEBUG("SPACEMIT_WB stride %x\n", stride);
+
+	dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_format, wb_format, cl_wb, 0); /* ABGR8888 */
+	dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_en, 1, cl_wb, 0);
+
+	if (wb->use_scl) {
+		saturn_wb_scaler_config(wb, hwdev);
+	} else {
+		if (is_rot_90_270(wb->rotation))
+			swap(wb_width, wb_height);
+		dpu_write(hwdev, WB_REG, wb_base, wb_scl_en, 0, cl_wb, 0);
+		dpu_write(hwdev, WB_REG, wb_base, wb_wdma_stride, wb_width*stride, cl_wb, 8);
+		if (a_crtc->is_slice_mode == 0) {
+			// dpu_writel(hwdev->base, WB0_TOP_BASE_ADDR + 0x08, wb_height | wb_width << 16);
+			// dpu_writel(hwdev->base, WB0_TOP_BASE_ADDR + 0x10, wb_height | wb_width << 16);
+			// dpu_writel(hwdev->base, WB0_TOP_BASE_ADDR + 0x0c, 0);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_ori_width, wb_width, cl_wb, 2);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_ori_height, wb_height, cl_wb, 2);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_width, wb_width, cl_wb, 4);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_height, wb_height, cl_wb, 4);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_ltopx, 0, cl_wb, 3);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_ltopy, 0, cl_wb, 3);
+		} else {
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_ori_width, wb_width, cl_wb, 2);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_ori_height, wb_height, cl_wb, 2);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_width, a_crtc->slice_wb[slice_id].wb_out_w, cl_wb, 4);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_height, a_crtc->slice_wb[slice_id].wb_out_h, cl_wb, 4);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_ltopx, a_crtc->slice_wb[slice_id].wb_out_x, cl_wb, 3);
+			dpu_write(hwdev, WB_REG, WB0_TOP_BASE_ADDR, wb_out_crop_ltopy, a_crtc->slice_wb[slice_id].wb_out_y, cl_wb, 3);
+		}
+	}
+
+	saturn_hee_wb_fmt_cvt_matrix(wb, hwdev, hwdev->color_encoding, hwdev->color_range, cl_wb);
+
+#ifdef SPACEMIT_WB_CONTINUE_MEM
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_wdma_base_addr0_low, 0x16400000);
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_wdma_base_addr0_high, 0x1);
+	dpu_write_reg(hwdev, WB_REG, wb_base, wb_wdma_stride, fb->pitches[0]);
+#endif
+}
+
+uint32_t saturn_hee_get_cfg_rdy(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev)
+{
+	u32 base = DPU_SCENE_CTL_ADDR(a_crtc->dev_id);
+
+	return dpu_read_reg(hwdev, DPU_CTL_REG, base, value32[2]);
+}
+
+uint32_t saturn_hee_get_int_sts(struct spacemit_hw_device *hwdev, int dev_id)
+{
+	if (dev_id == COMPOSER1)
+		return dpu_read_reg(hwdev, DPU_INTP_REG, DPU_INT_BASE_ADDR, value32[12]);
+	else if (dev_id == COMPOSER2)
+		return dpu_read_reg(hwdev, DPU_INTP_REG, DPU_INT_BASE_ADDR, value32[15]);
+	else
+		return 0;
+}
+
+static struct spacemit_crtc_irq_bitfield {
+	enum spacemit_dpu_irq irq_id;
+	u32 irq_bit;
+} irq_list[] = {
+	[INT_UNDERRUN] = {INT_UNDERRUN, DPU_INT_FRM_TIMING_UNFLOW},
+	[INT_CFG_RDY] = {INT_CFG_RDY, DPU_INT_CFG_RDY_CLR},
+	[INT_VSYNC] = {INT_VSYNC, DPU_INT_FRM_TIMING_VSYNC},
+	[INT_EOF] = {INT_EOF, DPU_INT_FRM_TIMING_EOF},
+	[INT_WB_DONE] = {INT_WB_DONE, DPU_INT_WB_DONE},
+	[INT_REST] = {INT_REST, DPU_REST_INT_BITS},
+	[OFF_CFG_RDY] = {OFF_CFG_RDY, DPU_INT_OFF_CFG_RDY_CLR},
+	[OFF_WB_DONE] = {OFF_WB_DONE, DPU_INT_OFF_WB_DONE},
+	[OFF_REST] = {OFF_REST, DPU_OFF_REST_INT_BITS},
+};
+
+static unsigned int irq_num = ARRAY_SIZE(irq_list);
+
+uint32_t saturn_hee_get_irq_bit(enum spacemit_dpu_irq irq_id, int dev_id)
+{
+	if (irq_id >= irq_num)
+		return 0;
+
+	if (dev_id == COMPOSER1 || dev_id == COMPOSER2)
+		return irq_list[irq_id].irq_bit;
+	else
+		return 0; //TODO
+}
+
+void saturn_hee_clr_int_sts(struct spacemit_crtc *a_crtc, enum spacemit_dpu_irq data, int dev_id)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	if (dev_id == COMPOSER1) {
+		dpu_writel(hwdev->base, DPU_INT_BASE_ADDR + DPU_ONLINE_IRQ_STS, data);
+		if ((saturn_hee_get_irq_bit(INT_UNDERRUN, dev_id) & data) && ur_enabhee)
+			saturn_enable_irq_mask(a_crtc, false, DPU_INT_BASE_ADDR + DPU_ONLINE_IRQ_MSK, saturn_hee_get_irq_bit(INT_UNDERRUN, dev_id));
+		return;
+	} else if (dev_id == COMPOSER2) {
+		dpu_write_reg(hwdev, DPU_INTP_REG, DPU_INT_BASE_ADDR, value32[15], data);
+		return;
+	}
+
+	if (dev_id == COMPOSER3)
+		return; //TODO
+
+}
+
+int saturn_hee_dpu_stop_check(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev)
+{
+	return 0;
+}
+
+void saturn_hee_dpu_disable(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev)
+{
+	u8 dev_id = a_crtc->dev_id;
+	u32 base = DPU_SCENE_CTL_ADDR(dev_id);
+	u32 tmg_base = TMG_BASE_ADDR[dev_id];
+
+	int i = 0;
+
+	/* disable dpu all modules */
+	if (tmg_base) {
+		a_crtc->outctrl_reg = dpu_read_reg(hwdev, TMG_REG, tmg_base, fm_timing_en);
+		dpu_write(hwdev, TMG_REG, tmg_base, fm_timing_en, 0);
+	}
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_rch_en, 0x0);
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_scl_en, 0x0);
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_outctl_en, 0x0);
+	if (saturn_hee_is_wb_en(a_crtc, hwdev))
+		dpu_write(hwdev, DPU_CTL_REG, base, nml_wb_en, 0x0);
+
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_frm_timing_en, 0);
+	if (a_crtc->is_offline_mode == 0) {
+		for (i = 0; i < (hwdev->rdma_nums); i++) {
+			dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR,
+					dpu_ctl_top_reg_0[i].cmdlist_rch_en, 0);
+		}
+	} else {
+		dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR,
+				dpu_ctl_top_reg_0[3].cmdlist_rch_en, 0);
+	}
+}
+
+void saturn_hee_dpu_restart(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev)
+{
+	u8 dev_id = a_crtc->dev_id;
+	u32 base = DPU_SCENE_CTL_ADDR(dev_id);
+	u32 tmg_base = TMG_BASE_ADDR[dev_id];
+	int i = 0;
+
+	dpu_write(hwdev, TMG_REG, tmg_base, fm_timing_en, a_crtc->outctrl_reg);
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_rch_en, a_crtc->dpu_online_nml_rch_en);
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_scl_en, a_crtc->dpu_online_nml_scl_en);
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_outctl_en, a_crtc->dpu_online_nml_outctl_en);
+	for (i = 0; i < (hwdev->rdma_nums); i++) {
+		dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR,
+				dpu_ctl_top_reg_0[i].cmdlist_rch_en, a_crtc->dpuctrl_ctl_nml_cmdlist_rch_en[i]);
+	}
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_frm_timing_en, 2);
+	dpu_write(hwdev, CMPS_X_REG, CMP_BASE_ADDR[dev_id], module_enable, 1);
+}
+
+void saturn_hee_enable_cmdlist(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, int id, bool enable)
+{
+	u32 val = 0;
+	u32 base = 0;
+	struct drm_crtc *crtc = &a_crtc->crtc;
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct spacemit_crtc_state *spacemit_state = to_spacemit_crtc_state(crtc_state);
+	struct spacemit_crtc_rdma *rdmas = spacemit_state->rdmas;
+
+	base = DPU_SCENE_CTL_ADDR(a_crtc->dev_id);
+
+	dpu_write(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR,
+		      dpu_ctl_top_reg_0[id].cmdlist_rch_en, enable ? 1 : 0);
+	val = dpu_read_reg(hwdev, DPU_CTL_REG, base, nml_rch_vrt_reuse);
+	if (enable && rdmas[id].use_cnt > 1)
+		val |= (1 << id);
+	else
+		val &= ~(1 << id);
+
+	dpu_write(hwdev, DPU_CTL_REG, base, nml_rch_vrt_reuse, val);
+}
+
+void saturn_hee_cfg_cmdlist(struct spacemit_hw_device *hwdev, int id, u32 chy, u32 addrl, u32 addrh)
+{
+	dpu_write(hwdev, CMDLIST_REG, CMDLIST_BASE_ADDR, cmdlist_reg_48[id].cmdlist_ch_y_first, chy);
+	dpu_write(hwdev, CMDLIST_REG, CMDLIST_BASE_ADDR, cmdlist_reg_0[id].cmdlist_ch_start_addrl, addrl);
+	dpu_write(hwdev, CMDLIST_REG, CMDLIST_BASE_ADDR, cmdlist_reg_16[id].cmdlist_ch_start_addrh, addrh);
+}
+
+void saturn_hee_rdma_contig_mem(struct spacemit_hw_device *hwdev, u8 tbu_id, phys_addr_t contig_pa, struct drm_framebuffer *fb, struct cmdlist_regs *cl_rdma, struct drm_plane *plane)
+{
+	u8 rdma_id = tbu_id / 2;
+	u32 base = RDMA_BASE_ADDR[rdma_id];
+
+	//TODO: disable tbu
+	CONFIG_RDMA_ADDR_REG(hwdev, 0, rdma_id, contig_pa, cl_rdma);
+	CONFIG_RDMA_ADDR_REG(hwdev, 1, rdma_id, (contig_pa + fb->offsets[1]), cl_rdma);
+	CONFIG_RDMA_ADDR_REG(hwdev, 2, rdma_id, (contig_pa + fb->offsets[2]), cl_rdma);
+	dpu_write(hwdev, RDMA_PATH_X_REG, base, rdma_stride0_layer0, fb->pitches[0], cl_rdma, 15);
+	dpu_write(hwdev, RDMA_PATH_X_REG, base, rdma_stride1_layer0, fb->pitches[1], cl_rdma, 15);
+}
+
+void saturn_hee_rdma_dmmu(struct spacemit_hw_device *hwdev, u8 tbu_id, struct tbu_instance *tbu, struct drm_framebuffer *fb, u32 val, struct cmdlist_regs *cl_rdma, struct drm_plane *plane)
+{
+	u32 base;
+	u8 rdma_id = tbu_id / 2;
+	struct cmdlist_regs *cl_tbu = NULL;
+
+	CONFIG_RDMA_ADDR_REG(hwdev, 0, rdma_id, tbu->tbu_va[0], cl_rdma);
+	CONFIG_RDMA_ADDR_REG(hwdev, 1, rdma_id, tbu->tbu_va[1], cl_rdma);
+	CONFIG_RDMA_ADDR_REG(hwdev, 2, rdma_id, tbu->tbu_va[2], cl_rdma);
+
+	base = RDMA_BASE_ADDR[rdma_id];
+	// dpu_writel(hwdev->base, base + 0x3c, fb->pitches[0] | fb->pitches[1] << 16);
+	dpu_write(hwdev, RDMA_PATH_X_REG, base, rdma_stride0_layer0, fb->pitches[0], cl_rdma, 15);
+	dpu_write(hwdev, RDMA_PATH_X_REG, base, rdma_stride1_layer0, fb->pitches[1], cl_rdma, 15);
+
+	cl_tbu = alloc_cmdlist_regs(MMU_TBU_REG);
+	base = MMU_TBU_BASE_ADDR_ARRAY[tbu_id];
+	CONFIG_TBU_REGS(hwdev, base, 0, cl_tbu);
+	CONFIG_TBU_REGS(hwdev, base, 1, cl_tbu);
+	CONFIG_TBU_REGS(hwdev, base, 2, cl_tbu);
+
+	dpu_write(hwdev, MMU_TBU_REG, base, value32[0], val, cl_tbu, 0);
+	cmdlist_regs_packing(plane_to_cl(plane), CMDLIST_MOD_DMMU, cl_tbu);
+	free_cmdlist_regs(cl_tbu);
+}
+
+void saturn_hee_wb_dmmu(struct spacemit_hw_device *hwdev, u8 tbu_id, struct tbu_instance *tbu, struct drm_framebuffer *fb, u32 val, u8 fbc_mode, int header_size, struct spacemit_wb *wb)
+{
+	u32 base;
+	struct cmdlist_regs *cl_tbu = NULL;
+	struct drm_crtc *crtc = &wb->a_crtc->crtc;
+	cl_tbu = alloc_cmdlist_regs(MMU_TBU_REG);
+	CONFIG_WB_ADDR_REG(hwdev, 0, tbu->tbu_va[0], wb->cl_wb);
+	if (fbc_mode) {
+		CONFIG_WB_ADDR_REG(hwdev, 1, (tbu->tbu_va[0] + header_size), wb->cl_wb);
+	} else {
+		CONFIG_WB_ADDR_REG(hwdev, 1, tbu->tbu_va[1], wb->cl_wb);
+	}
+
+	base = MMU_TBU_BASE_ADDR_ARRAY[tbu_id];
+	CONFIG_TBU_REGS(hwdev, base, 0, cl_tbu);
+	CONFIG_TBU_REGS(hwdev, base, 1, cl_tbu);
+	CONFIG_TBU_REGS(hwdev, base, 2, cl_tbu);
+	dpu_write(hwdev, MMU_TBU_REG, base, value32[0], val, cl_tbu, 0);
+	cmdlist_regs_packing(crtc_to_cl(crtc), CMDLIST_MOD_WB0, cl_tbu);
+	free_cmdlist_regs(cl_tbu);
+}
+
+static u32 saturn_get_cl_chy_addr(void)
+{
+	return CMDLIST_BASE_ADDR + CMDLIST_CH_Y;
+}
+
+static u32 saturn_get_cl_start_cmps_y_addr(void)
+{
+	return DPU_CTRL_BASE_ADDR + CMDLIST_CH_START_CMPS_Y;
+}
+
+static u32 saturn_get_cl_cfg_rdy_addr(void)
+{
+	return DPU_CTRL_BASE_ADDR + CMDLIST_CFG_READY;
+}
+
+int saturn_hee_get_cl_rdma_buf(struct spacemit_crtc *a_crtc)
+{
+	a_crtc->cl_rdma = alloc_cmdlist_regs(RDMA_PATH_X_REG);
+	if (!a_crtc->cl_rdma)
+		return -ENOMEM;
+	else
+		return 0;
+}
+
+/* cmdlist v1 support */
+struct cmdlist_header {
+	uint64_t next_list_addr : 39;
+	// reserved
+	uint32_t: 1;
+	uint32_t nod_len : 16;
+	/*1: the last cmdlist node, 2: pending node, otherwise 0 */
+	uint32_t nod_type : 2;
+	uint32_t next_nod_secu : 1;
+	uint64_t wait_event_low    : 5;
+	uint64_t wait_event    : 59;
+	// reserved
+	uint32_t: 5;
+};
+
+struct cmdlist_row {
+	uint32_t module_cfg_addr   : 19;
+	uint32_t module_cfg_strobe : 12;
+	//the last row tag = 1, else tag = 0.
+	uint32_t row_eof_tag       : 1;
+	uint32_t module_regs[3];
+};
+
+#define CL_HEADER_SZ sizeof(struct cmdlist_header)
+#define CL_ROW_SZ    sizeof(struct cmdlist_row)
+
+#define CMDLIST_RDMA_CFG_RDY(ch) (1 << ch)
+#define CMDLIST_PREPQ_CFG_RDY(ch) (1 << (ch + 10))
+#define CMDLIST_CMPS_CFG_RDY(ch) (1 << (ch + 20))
+#define CMDLIST_WB_CFG_RDY(ch) (1 << (ch + 23))
+
+#define CMDLIST_WAIT_EVENT_BIT_RDMA_RELOAD BIT(4)
+#define CMDLIST_WAIT_EVENT_BIT_PREPQ_RELOAD BIT(5)
+#define CMDLIST_WAIT_EVENT_BIT_WB0_RELOAD BIT(4)
+#define CMDLIST_WAIT_EVENT_BIT_WB1_RELOAD BIT(5)
+#define CMDLIST_WAIT_EVENT_BIT_CMPS_RELOAD BIT(6)
+
+void saturn_hee_cmdlist_fill_data_row(struct cmdlist *cl, u32 strobe, u32 offset, u32 value[])
+{
+	struct cmdlist_row *row;
+	u8 i;
+
+	row = (struct cmdlist_row *)((char *)cl->va + CL_HEADER_SZ) + cl->nod_len;
+	row->module_cfg_addr = offset >> 2;
+	row->module_cfg_strobe = strobe;
+
+	for (i = 0; i < CMDLIST_ROW_REGS; i++)
+		if (strobe & CMDLIST_REG_STROBE(i))
+			row->module_regs[i] = value[i];
+
+	print_row((u32 *)row);
+}
+
+void saturn_hee_cmdlist_fill_conf_row(struct cmdlist *cl, struct spacemit_hw_device *hwdev, u8 dev_id)
+{
+	struct cmdlist_header *header;
+	struct cmdlist_row *row;
+	struct spacemit_plane_state *spacemit_pstate = cl_to_spacemit_pstate(cl);
+	u8 rch_id = spacemit_pstate->rdma_id;
+	unsigned int zpos = spacemit_pstate->state.zpos;
+	u32 size = 0, wait_event_bits = 0, cfg_rdy_bits = 0;
+	int cur_crtc_y, cur_crtc_h, next_crtc_y;
+
+	cur_crtc_y = cur_crtc_h = next_crtc_y = -1;
+
+	header = (struct cmdlist_header *)(cl->va);
+	// fill ch_y_other row;
+	row = (struct cmdlist_row *)((char *)header + CL_HEADER_SZ + cl->nod_len * CL_ROW_SZ);
+	if (cl->next) {
+
+		if (dev_id == CMDLIST_CMP_INVALID) {
+			trace_u64_data("rch_id", rch_id);
+			row->module_cfg_addr = ((saturn_get_cl_chy_addr() + rch_id * 4) >> 2);
+		} else {
+			trace_u64_data("dev_id", dev_id);
+			row->module_cfg_addr = ((saturn_get_cl_chy_addr() + (5 + dev_id) * 4) >> 2);
+		}
+		row->module_cfg_strobe = CMDLIST_REG_STROBE(0);
+		if (cl->cmdlist_ch_y_other != 0xDEADBEEF)
+			next_crtc_y = cl->cmdlist_ch_y_other;
+		else
+			next_crtc_y = cl_to_spacemit_pstate(cl->next)->state.crtc_y;
+		row->module_regs[0] = next_crtc_y;
+		cl->nod_len++;
+		row++;
+	}
+
+	if (cl->type == CMDLIST_PLANE) {
+		// fill start cmps chy row;
+		if (cl->rch_start_cmps_y != 0xDEADBEEF)
+			cur_crtc_y = cl->rch_start_cmps_y;
+		else
+			cur_crtc_y = cl_to_spacemit_pstate(cl)->state.crtc_y;
+		cur_crtc_h = cl_to_spacemit_pstate(cl)->state.crtc_h;
+		/* the cmdlist nodes don't belong to the same plane */
+		if (next_crtc_y != -1 && cl_to_spacemit_pstate(cl->next) != spacemit_pstate &&
+				(cur_crtc_y + cur_crtc_h + 24) > next_crtc_y) {
+			DRM_ERROR("Invalid gap: cur_crtc_y:%d cur_crtc_h:%d next_crtc_y:%d\n",
+					cur_crtc_y, cur_crtc_h, next_crtc_y);
+		}
+		row->module_cfg_addr = (saturn_get_cl_start_cmps_y_addr() + rch_id * 4) >> 2;
+		row->module_cfg_strobe = CMDLIST_REG_STROBE(0);
+		row->module_regs[0] = (cur_crtc_y >= 24) ? (cur_crtc_y - 24) : 0;
+		cl->nod_len++;
+		row++;
+	}
+
+	// fill last_row
+	if (cl->mode_mask & CMDLIST_MOD_RDMA) {
+		wait_event_bits |= CMDLIST_WAIT_EVENT_BIT_RDMA_RELOAD;
+		cfg_rdy_bits |= CMDLIST_RDMA_CFG_RDY(rch_id);
+	}
+	if (cl->mode_mask & CMDLIST_MOD_LP) {
+		wait_event_bits |= CMDLIST_WAIT_EVENT_BIT_PREPQ_RELOAD;
+		cfg_rdy_bits |= CMDLIST_PREPQ_CFG_RDY(rch_id);
+	}
+	if (cl->mode_mask & CMDLIST_MOD_COMP) {
+		wait_event_bits |= CMDLIST_WAIT_EVENT_BIT_CMPS_RELOAD;
+		cfg_rdy_bits |= CMDLIST_CMPS_CFG_RDY(dev_id);
+	}
+	if (cl->mode_mask & CMDLIST_MOD_WB0) {
+		wait_event_bits |= CMDLIST_WAIT_EVENT_BIT_WB0_RELOAD;
+		cfg_rdy_bits |= CMDLIST_WB_CFG_RDY(0);
+	}
+	if (cl->mode_mask & CMDLIST_MOD_WB1) {
+		wait_event_bits |= CMDLIST_WAIT_EVENT_BIT_WB1_RELOAD;
+		cfg_rdy_bits |= CMDLIST_WB_CFG_RDY(1);
+	}
+	row->module_cfg_addr = (saturn_get_cl_cfg_rdy_addr()) >> 2;
+	row->module_cfg_strobe = CMDLIST_REG_STROBE(0);
+	row->module_regs[0] = cfg_rdy_bits;
+	row->row_eof_tag = 1;
+	cl->nod_len++;
+	row++;
+
+	// fill header
+	if (cl->next)
+		header->next_list_addr = cl->next->pa;
+	else
+		header->nod_type = 1;
+	header->nod_len = cl->nod_len + 1; //include header
+	header->wait_event_low = wait_event_bits & 0x1F; /* low 5 bits */
+	header->wait_event = wait_event_bits >> 5;
+
+	size = (u64)row - (u64)header;
+	if (size > PER_CMDLIST_SIZE)
+		DRM_ERROR("plane%d cmdlist occupies %d bytes!\n", zpos, size);
+}
+
+void saturn_hee_wb_cmdlist(struct cmdlist *cl, struct spacemit_hw_device *hwdev, struct spacemit_drm_private *priv, u8 crtc_id, u8 dev_id)
+{
+	u32 val = 0;
+
+	dpu_write_reg(hwdev, CMDLIST_REG, CMDLIST_BASE_ADDR, cmdlist_reg_48[hwdev->rdma_nums + 2 + crtc_id].cmdlist_ch_y_first, val);
+
+	val = ((priv->cmdlist_groups[hwdev->rdma_nums + crtc_id]->pa) & CMDLIST_ADDRL_ALIGN_MASK) >> CMDLIST_ADDRL_ALIGN_BITS;
+	dpu_write_reg(hwdev, CMDLIST_REG, CMDLIST_BASE_ADDR, cmdlist_reg_0[hwdev->rdma_nums + 2 + crtc_id].cmdlist_ch_start_addrl, val);
+	val = (priv->cmdlist_groups[hwdev->rdma_nums + crtc_id]->pa) >> 32;
+	dpu_write_reg(hwdev, CMDLIST_REG, CMDLIST_BASE_ADDR, cmdlist_reg_16[hwdev->rdma_nums + 2 + crtc_id].cmdlist_ch_start_addrh, val);
+	dpu_write_reg(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR,
+					dpu_ctl_top_reg_10[dev_id].cmdlist_cmps_top_en, 1);
+	dpu_write_reg(hwdev, DPU_CTL_TOP_REG, DPU_CTRL_BASE_ADDR,
+			dpu_ctl_top_reg_10[dev_id].cmdlist_cmps_other_en, 1);
+
+	priv->cmdlist_groups[hwdev->rdma_nums + crtc_id] = NULL;
+}
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+void hee_dump_dpu_regs_by_enum(void __iomem *io_base, phys_addr_t phy_base, dpu_hee_reg_enum reg_enum, u8 trace_dump)
+{
+	int i;
+	int j;
+	uint32_t reg_num;
+	void __iomem *io_addr;
+	phys_addr_t phy_addr;
+
+	dpu_hee_reg_dump_t *tmp = &dpu_reg_dump_array[0];
+
+	for (i = 0; i < ARRAY_SIZE(dpu_reg_dump_array); i++) {
+		if (tmp->index == reg_enum) {
+			reg_num = tmp->dump_reg_num;
+			io_addr = io_base + tmp->module_offset;
+			phy_addr = phy_base + tmp->module_offset;
+			if (trace_dump) {
+				trace_dpu_reg_info(tmp->module_name, phy_addr, reg_num);
+				for (j = 0; j < reg_num; j++)
+					trace_dpu_reg_dump(phy_addr + j * 4, readl(io_addr + j * 4));
+
+			} else {
+				printk(KERN_DEBUG "%d-%s, address:0x%08llx, num:%d\n", tmp->index, tmp->module_name, phy_addr, reg_num);
+				for (j = 0; j < reg_num; j++)
+					printk(KERN_DEBUG "0x%08llx: 0x%08x\n", (phy_addr + j * 4), readl(io_addr + j * 4));
+			}
+		}
+		tmp++;
+	}
+}
+
+bool hee_dpu_reg_enum_valid(dpu_hee_reg_enum reg_enum)
+{
+	int size = ARRAY_SIZE(SATURN_hee_DPU_REG_ENUM_LISTS);
+	int i = 0;
+
+	for (i = 0; i < size; i++) {
+		if (reg_enum == SATURN_hee_DPU_REG_ENUM_LISTS[i])
+			return true;
+	}
+
+	return false;
+}
+
+void hee_dump_dpu_regs(struct spacemit_crtc *a_crtc, dpu_hee_reg_enum reg_enum, u8 trace_dump)
+{
+	dpu_hee_reg_enum tmp = E_DPU_TOP_REG;
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	void __iomem *reg_io_base = hwdev->base;
+	phys_addr_t reg_phy_base = hwdev->phy_addr;
+
+	if (reg_enum > E_DPU_DUMP_ALL) {
+		pr_err("invalid dump regsiter enum\n");
+		return;
+	} else if (reg_enum == E_DPU_DUMP_ALL) {
+		for (; tmp < E_DPU_DUMP_ALL; tmp++) {
+			if (hee_dpu_reg_enum_valid(tmp))
+				hee_dump_dpu_regs_by_enum(reg_io_base, reg_phy_base, tmp, trace_dump);
+		}
+	} else {
+		hee_dump_dpu_regs_by_enum(reg_io_base, reg_phy_base, reg_enum, trace_dump);
+	}
+}
+
+
+void hee_dpu_dump_reg(struct spacemit_crtc *a_crtc)
+{
+	if (!a_crtc->enable_dump_reg)
+		return;
+
+	hee_dump_dpu_regs(a_crtc, E_DPU_DUMP_ALL, 1);
+}
+
+void hee_dpu_dump_rdma_status(struct spacemit_drm_private *priv)
+{
+	return;
+}
+#endif
+
+#define DUMP_BUF_LEN	100
+void saturn_hee_cmdlist_dump_node(struct cmdlist *cl)
+{
+	struct cmdlist_header *header = (struct cmdlist_header *)(cl->va);
+	ssize_t ret = 0;
+	char tmp[DUMP_BUF_LEN] = {0x0};
+
+	ret += sprintf(tmp + ret, "header: ");
+	ret += sprintf(tmp + ret, "next_addr:0x%llx ", (unsigned long long)header->next_list_addr);
+	ret += sprintf(tmp + ret, "rows:%d ", header->nod_len);
+	ret += sprintf(tmp + ret, "type:%d ", header->nod_type);
+	ret += sprintf(tmp + ret, "event_low:0x%llx ", (unsigned long long)header->wait_event_low);
+	ret += sprintf(tmp + ret, "event:0x%llx ", (unsigned long long)header->wait_event);
+	trace_plat_cmdlist_dump_node(tmp);
+
+	/* header is already dumped*/
+	for (int i = 0; i < header->nod_len - 1; i++) {
+		struct cmdlist_row *row = (struct cmdlist_row *)((char *)header + CL_HEADER_SZ) + i;
+		u32 addr = row->module_cfg_addr << 2;
+
+		ret = 0;
+		ret += sprintf(tmp + ret, "row[%4d:tag:%d]: ", i, row->row_eof_tag);
+
+		for (int j = 0; j < CMDLIST_ROW_REGS; j++) {
+			if (row->module_cfg_strobe & CMDLIST_REG_STROBE(j))
+				ret += sprintf(tmp + ret, "[%#10x:%#10x]", addr + j * 4, row->module_regs[j]);
+			else
+				ret += sprintf(tmp + ret, "%#11x:%#11x", addr + j * 4, row->module_regs[j]);
+		}
+		trace_plat_cmdlist_dump_node(tmp);
+	}
+	return;
+}
diff --git a/drivers/gpu/drm/spacemit/dpu/dpu_trace.h b/drivers/gpu/drm/spacemit/dpu/dpu_trace.h
new file mode 100644
index 000000000000..934849ad01cd
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/dpu_trace.h
@@ -0,0 +1,566 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM dpu
+
+#if !defined(_DPU_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _DPU_TRACE_H_
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#include "dpu_saturn.h"
+
+#undef __assign_str
+#define __assign_str(dst, src)                                          \
+	strcpy(__get_str(dst), (src) ? (const char *)(src) : "(null)");
+
+TRACE_EVENT(drm_display_mode_info,
+	TP_PROTO(struct drm_display_mode *mode),
+	TP_ARGS(mode),
+	TP_STRUCT__entry(
+		__field(u16, hdisplay)
+		__field(u16, hbp)
+		__field(u16, hfp)
+		__field(u16, hsync)
+		__field(u16, vdisplay)
+		__field(u16, vbp)
+		__field(u16, vfp)
+		__field(u16, vsync)
+	),
+	TP_fast_assign(
+		__entry->hdisplay = mode->hdisplay;
+		__entry->hbp = mode->htotal - mode->hsync_end;
+		__entry->hfp = mode->hsync_start - mode->hdisplay;
+		__entry->hsync = mode->hsync_end - mode->hsync_start;
+		__entry->vdisplay = mode->vdisplay;
+		__entry->vbp = mode->vtotal - mode->vsync_end;
+		__entry->vfp = mode->vsync_start - mode->vdisplay;
+		__entry->vsync = mode->vsync_end - mode->vsync_start;
+	),
+	TP_printk("drm_display_mode: hdisplay=%d vdisplay=%d hsync=%d vsync=%d hbp=%d hfp=%d vbp=%d vfp=%d",
+		  __entry->hdisplay, __entry->vdisplay,  __entry->hsync, __entry->vsync,
+		  __entry->hbp, __entry->hfp,  __entry->vbp, __entry->vfp
+	)
+);
+
+TRACE_EVENT(dpu_plane_info,
+	TP_PROTO(struct drm_plane_state *state, struct drm_framebuffer *fb,
+		u32 rdma_id, u32 alpha, u32 rotation, u32 afbc),
+	TP_ARGS(state, fb, rdma_id, alpha, rotation, afbc),
+	TP_STRUCT__entry(
+			__field(u32, rdma_id)
+			__field(u32, src_w)
+			__field(u32, src_h)
+			__field(u32, src_x)
+			__field(u32, src_y)
+			__field(u32, crtc_w)
+			__field(u32, crtc_h)
+			__field(u32, crtc_x)
+			__field(u32, crtc_y)
+			__field(u32, width)
+			__field(u32, height)
+			__field(u32, format)
+			__field(u32, blend_mode)
+			__field(u32, alpha)
+			__field(u32, zpos)
+			__field(u32, rotation)
+			__field(u32, afbc)
+	),
+	TP_fast_assign(
+			__entry->rdma_id = rdma_id;
+			__entry->src_w = state->src_w >> 16;
+			__entry->src_h = state->src_h >> 16;
+			__entry->src_x = state->src_x >> 16;
+			__entry->src_y = state->src_y >> 16;
+			__entry->crtc_w = state->crtc_w;
+			__entry->crtc_h = state->crtc_h;
+			__entry->crtc_x = state->crtc_x;
+			__entry->crtc_y = state->crtc_y;
+			__entry->width = fb->width;
+			__entry->height = fb->height;
+			__entry->format = fb->format->format;
+			__entry->blend_mode = state->pixel_blend_mode;
+			__entry->alpha = alpha;
+			__entry->zpos = state->zpos;
+			__entry->rotation = rotation;
+			__entry->afbc = afbc;
+	),
+	TP_printk("rdma_id=%d src: w=%d h=%d x=%d y=%d crtc: w=%d h=%d x=%d y=%d "
+		  "width=%d height=%d fmt=0x%x blend=%d alpha:%d "
+		  "zpos=%d rot:%d afbc:%d",
+		  __entry->rdma_id, __entry->src_w,
+		  __entry->src_h, __entry->src_x,
+		  __entry->src_y, __entry->crtc_w,
+		  __entry->crtc_h, __entry->crtc_x,
+		  __entry->crtc_y, __entry->width,
+		  __entry->height, __entry->format,
+		  __entry->blend_mode, __entry->alpha,
+		  __entry->zpos, __entry->rotation,
+		  __entry->afbc)
+);
+
+DECLARE_EVENT_CLASS(dpu_status_template,
+	TP_PROTO(const char *name, int status),
+	TP_ARGS(name, status),
+	TP_STRUCT__entry(
+		__string(name_str, name)
+		__field(int, status)
+	),
+	TP_fast_assign(
+		__assign_str(name_str, name);
+		__entry->status = status;
+	),
+	TP_printk("%s: 0x%x",
+		  __get_str(name_str), __entry->status)
+);
+
+DEFINE_EVENT(dpu_status_template, dpuctrl_scaling,
+	TP_PROTO(const char *name, int status),
+	TP_ARGS(name, status)
+);
+DEFINE_EVENT(dpu_status_template, dpuctrl,
+	TP_PROTO(const char *name, int status),
+	TP_ARGS(name, status)
+);
+DEFINE_EVENT(dpu_status_template, spacemit_plane_update_hw_channel,
+	TP_PROTO(const char *name, int status),
+	TP_ARGS(name, status)
+);
+DEFINE_EVENT(dpu_status_template, saturn_irq_enable,
+	TP_PROTO(const char *name, int status),
+	TP_ARGS(name, status)
+);
+DEFINE_EVENT(dpu_status_template, saturn_enable_vsync,
+	TP_PROTO(const char *name, int status),
+	TP_ARGS(name, status)
+);
+DEFINE_EVENT(dpu_status_template, dpu_isr_status,
+	TP_PROTO(const char *name, int status),
+	TP_ARGS(name, status)
+);
+
+DECLARE_EVENT_CLASS(dpu_ul_data_template,
+	TP_PROTO(const char *name, unsigned long ul_data),
+	TP_ARGS(name, ul_data),
+	TP_STRUCT__entry(
+		__string(name_str, name)
+		__field(unsigned long, ul_data)
+	),
+	TP_fast_assign(
+		__assign_str(name_str, name);
+		__entry->ul_data = ul_data;
+	),
+	TP_printk("%s: %ld",
+		  __get_str(name_str), __entry->ul_data)
+);
+
+DEFINE_EVENT(dpu_ul_data_template, dpu_isr_ul_data,
+	TP_PROTO(const char *name, unsigned long ul_data),
+	TP_ARGS(name, ul_data)
+);
+
+DECLARE_EVENT_CLASS(dpu_uint64_t_data_template,
+	TP_PROTO(const char *name, uint64_t data),
+	TP_ARGS(name, data),
+	TP_STRUCT__entry(
+		__string(name_str, name)
+		__field(uint64_t, data)
+	),
+	TP_fast_assign(
+		__assign_str(name_str, name);
+		__entry->data = data;
+	),
+	TP_printk("%s: %lld",
+		  __get_str(name_str), __entry->data)
+);
+
+DEFINE_EVENT(dpu_uint64_t_data_template, u64_data,
+	TP_PROTO(const char *name, uint64_t data),
+	TP_ARGS(name, data)
+);
+
+TRACE_EVENT(spacemit_dpu_reg_write,
+	TP_PROTO(uint32_t data, uint32_t base, const char *name),
+	TP_ARGS(data, base, name),
+	TP_STRUCT__entry(
+		__field(uint32_t, data)
+		__field(uint32_t, base)
+		__string(name_str, name)
+	),
+	TP_fast_assign(
+		__entry->data = data;
+		__entry->base = base;
+		__assign_str(name_str, name);
+	),
+	TP_printk("dpu write 0x%x to addr 0x%x field %s\n",
+		__entry->data, __entry->base, __get_str(name_str))
+);
+
+TRACE_EVENT(spacemit_plane_disable_hw_channel,
+	TP_PROTO(uint32_t layer_id, uint32_t rdma_id),
+	TP_ARGS(layer_id, rdma_id),
+	TP_STRUCT__entry(
+		__field(uint32_t, layer_id)
+		__field(uint32_t, rdma_id)
+	),
+	TP_fast_assign(
+		__entry->layer_id = layer_id;
+		__entry->rdma_id = rdma_id;
+	),
+	TP_printk("layer_id:%d rdma_id:%d",
+		  __entry->layer_id, __entry->rdma_id)
+);
+
+TRACE_EVENT(dpu_mclk_scl,
+	TP_PROTO(uint64_t in_width, uint64_t in_height, uint64_t out_width, uint64_t out_height, uint64_t bpp, uint64_t C, uint64_t width_ratio, uint64_t height_ratio),
+	TP_ARGS(in_width, in_height, out_width, out_height, bpp, C, width_ratio, height_ratio),
+	TP_STRUCT__entry(
+		__field(uint64_t, in_width)
+		__field(uint64_t, in_height)
+		__field(uint64_t, out_width)
+		__field(uint64_t, out_height)
+		__field(uint64_t, bpp)
+		__field(uint64_t, C)
+		__field(uint64_t, width_ratio)
+		__field(uint64_t, height_ratio)
+	),
+	TP_fast_assign(
+		__entry->in_width = in_width;
+		__entry->in_height = in_height;
+		__entry->out_width = out_width;
+		__entry->out_height = out_height;
+		__entry->bpp = bpp;
+		__entry->C = C;
+		__entry->width_ratio = width_ratio;
+		__entry->height_ratio = height_ratio;
+	),
+	TP_printk("scl_in_width = %llu, scl_in_height = %llu, scl_out_width = %llu, scl_out_height = %llu, bpp = %llu, C = %llu, width_ratio = %llu, height_ratio = %llu\n",
+		__entry->in_width, __entry->in_height, __entry->out_width,
+		__entry->out_height, __entry->bpp, __entry->C,
+		__entry->width_ratio, __entry->height_ratio)
+);
+
+TRACE_EVENT(dpu_fpixclk_hblk,
+	TP_PROTO(uint64_t hact, uint64_t fps, uint64_t vtotal, uint64_t fpixclk_hblk),
+	TP_ARGS(hact, fps, vtotal, fpixclk_hblk),
+	TP_STRUCT__entry(
+		__field(uint64_t, hact)
+		__field(uint64_t, fps)
+		__field(uint64_t, vtotal)
+		__field(uint64_t, fpixclk_hblk)
+	),
+	TP_fast_assign(
+		__entry->hact = hact;
+		__entry->fps = fps;
+		__entry->vtotal = vtotal;
+		__entry->fpixclk_hblk = fpixclk_hblk;
+	),
+	TP_printk("hact = %lld, fps = %lld, vtotal = %lld, Fpixclk_hblk = %lld\n",
+		  __entry->hact, __entry->fps, __entry->vtotal, __entry->fpixclk_hblk)
+);
+
+TRACE_EVENT(dpuctrl_scaling_setting,
+	TP_PROTO(uint32_t id, int in_use, int rdma_id),
+	TP_ARGS(id, in_use, rdma_id),
+	TP_STRUCT__entry(
+		__field(uint32_t, id)
+		__field(int, in_use)
+		__field(int, rdma_id)
+	),
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->in_use = in_use;
+		__entry->rdma_id = rdma_id;
+	),
+	TP_printk("scaler%d: in_use:0x%x rdma_id:%d",
+		  __entry->id, __entry->in_use, __entry->rdma_id)
+);
+
+TRACE_EVENT(dpu_reg_info,
+	TP_PROTO(u8 *mod_name, phys_addr_t p_addr, uint32_t reg_num),
+	TP_ARGS(mod_name, p_addr, reg_num),
+	TP_STRUCT__entry(
+		__string(name_str, mod_name)
+		__field(phys_addr_t, p_addr)
+		__field(uint32_t, reg_num)
+	),
+	TP_fast_assign(
+		__assign_str(name_str, mod_name);
+		__entry->p_addr = p_addr;
+		__entry->reg_num = reg_num;
+	),
+	TP_printk("%s(0x%08llx), num:%d",
+		__get_str(name_str), __entry->p_addr, __entry->reg_num)
+);
+
+TRACE_EVENT(dpu_reg_dump,
+	TP_PROTO(uint32_t reg, uint32_t val),
+	TP_ARGS(reg, val),
+	TP_STRUCT__entry(
+		__field(uint32_t, reg)
+		__field(uint32_t, val)
+	),
+	TP_fast_assign(
+		__entry->reg = reg;
+		__entry->val = val;
+	),
+	TP_printk("0x%08x: 0x%08x",
+		  __entry->reg, __entry->val)
+);
+
+DECLARE_EVENT_CLASS(dpu_func_template,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id),
+	TP_STRUCT__entry(
+		__field(uint32_t, dev_id)
+	),
+	TP_fast_assign(
+		__entry->dev_id = dev_id;
+	),
+	TP_printk("dev_id=%d", __entry->dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_plane_check_rdma,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_plane_atomic_check,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_plane_atomic_update,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_plane_reset,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_plane_atomic_duplicate_state,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_plane_atomic_destroy_state,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_plane_init,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, dpu_enable_clocks,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, dpu_disable_clocks,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, saturn_wait_for_cfg_ready,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, saturn_wait_for_eof,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, saturn_wait_for_wb,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, saturn_run_display,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, dpu_init,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, dpu_uninit,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, dpu_isr,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, dpu_run,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, dpu_stop,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_mode_set_nofb,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_atomic_enable,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_atomic_disable,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_atomic_check,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_atomic_begin,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_atomic_flush,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_enable_vblank,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_disable_vblank,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_run,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_stop,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_init,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, spacemit_crtc_uninit,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DEFINE_EVENT(dpu_func_template, saturn_ctrl_cfg_ready_timer,
+	TP_PROTO(uint32_t dev_id),
+	TP_ARGS(dev_id)
+);
+DECLARE_EVENT_CLASS(dpu_ctrl_template,
+	TP_PROTO(int id, int enable),
+	TP_ARGS(id, enable),
+	TP_STRUCT__entry(
+		__field(uint32_t, id)
+		__field(uint32_t, enable)
+	),
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->enable = enable;
+	),
+	TP_printk("id=%d enable:%d",
+		  __entry->id, __entry->enable)
+);
+DEFINE_EVENT(dpu_ctrl_template, saturn_ctrl_cfg_ready,
+	TP_PROTO(int id, int enable),
+	TP_ARGS(id, enable)
+);
+
+DEFINE_EVENT(dpu_ctrl_template, saturn_ctrl_sw_start,
+	TP_PROTO(int id, int enable),
+	TP_ARGS(id, enable)
+);
+
+DEFINE_EVENT(dpu_ctrl_template, saturn_ctl_cmd_update,
+	TP_PROTO(int id, int enable),
+	TP_ARGS(id, enable)
+);
+
+TRACE_EVENT(saturn_conf_scaler_x,
+	TP_PROTO(struct spacemit_plane_state *state),
+	TP_ARGS(state),
+	TP_STRUCT__entry(
+		__field(int, id)
+	),
+	TP_fast_assign(
+		__entry->id = state->scaler_id;
+	),
+	TP_printk("scaler_id:%d", __entry->id)
+);
+
+TRACE_EVENT(plat_cmdlist_dump_node,
+	TP_PROTO(char *dump),
+	TP_ARGS(dump),
+	TP_STRUCT__entry(
+		__string(dump_str, dump)
+	),
+	TP_fast_assign(
+		__assign_str(dump_str, dump);
+	),
+	TP_printk("%s", __get_str(dump_str))
+);
+
+TRACE_EVENT(csc_dump_state,
+	TP_PROTO(int new_encoding, int new_range, int old_encoding, int old_range, int rdma_id),
+	TP_ARGS(new_encoding, new_range, old_encoding, old_range, rdma_id),
+	TP_STRUCT__entry(
+		__field(int, new_encoding)
+		__field(int, new_range)
+		__field(int, old_encoding)
+		__field(int, old_range)
+		__field(int, rdma_id)
+	),
+	TP_fast_assign(
+		__entry->new_encoding = new_encoding;
+		__entry->new_range = new_range;
+		__entry->old_encoding = old_encoding;
+		__entry->old_range = old_range;
+		__entry->rdma_id = rdma_id;
+	),
+
+	TP_printk("csc_dump_state new_encoding %d new_range %d old_encoding %d old_range %d rdma_id %d", \
+		__entry->new_encoding,      \
+		__entry->new_range,    \
+		__entry->old_encoding,    \
+		__entry->old_range,    \
+		__entry->rdma_id)
+);
+
+TRACE_EVENT(cmdlist_dump_node,
+	TP_PROTO(char *type, u8 index, int rdma_id, u8 dev_id, int zpos, u16 nod_len),
+	TP_ARGS(type, index, rdma_id, dev_id, zpos, nod_len),
+	TP_STRUCT__entry(
+		__string(type_str, type)
+		__field(u8, index)
+		__field(int, rdma_id)
+		__field(u8, dev_id)
+		__field(int, zpos)
+		__field(u16, nod_len)
+	),
+	TP_fast_assign(
+		__assign_str(type_str, type);
+		__entry->index = index;
+		__entry->rdma_id = rdma_id;
+		__entry->dev_id = dev_id;
+		__entry->zpos = zpos;
+		__entry->nod_len = nod_len;
+	),
+	TP_printk("cmdlist[%s]: index:%d rdma_id:%d crtc:%d zpos:%d rows:%d", \
+		__get_str(type_str), \
+		__entry->index,      \
+		__entry->rdma_id,    \
+		__entry->dev_id,    \
+		__entry->zpos,       \
+		__entry->nod_len)
+);
+#endif /* _DPU_TRACE_H_ */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_FILE
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/spacemit/dpu/
+
+#define TRACE_INCLUDE_FILE dpu_trace
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/spacemit/dpu/post_process_hee.c b/drivers/gpu/drm/spacemit/dpu/post_process_hee.c
new file mode 100644
index 000000000000..a27908fba6e1
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/post_process_hee.c
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include "post_process_hee.h"
+#include "../spacemit_dpu_reg.h"
+#include "saturn_regs/reg_map_hee.h"
+
+void saturn_hee_conf_dpuctrl_color_matrix(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_crtc_state *state = a_crtc->crtc.state;
+	struct spacemit_crtc_state *spacemit_state = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = spacemit_state->color_matrix_blob_prop;
+	int *color_matrix;
+	struct cmdlist_regs *cmd_regs = NULL;
+
+	/*
+	 * For color matrix, if no update from user space,
+	 * we keep the original configuration, do not change the value of any color matrix register
+	 */
+	if (blob) {
+		color_matrix = (int *)blob->data;
+
+		cmd_regs = alloc_cmdlist_regs(LTM_REG);
+		if (!cmd_regs) {
+			DRM_ERROR("cmd_regs for color matrix failed\n");
+			return;
+		}
+
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_npost_proc_en, 1);
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_nendmatrix_en, 1);
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_ngain_to_full_en, 0);
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_nmatrix_en, 0);
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_nfront_tmootf_en, 0);
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_nend_tmootf_en, 0);
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_neotf_en, 0);
+		dpu_write_reg(hwdev, LTM_REG, LTM_BASE_ADDR, m_noetf_en, 0);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table0, color_matrix[0] & 0xFFFF, cmd_regs, 78);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table1, color_matrix[1] & 0xFFFF, cmd_regs, 78);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table2, color_matrix[2] & 0xFFFF, cmd_regs, 79);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table3, color_matrix[3] & 0xFFFF, cmd_regs, 79);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table4, color_matrix[4] & 0xFFFF, cmd_regs, 80);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table5, color_matrix[5] & 0xFFFF, cmd_regs, 80);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table6, color_matrix[6] & 0xFFFF, cmd_regs, 81);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table7, color_matrix[7] & 0xFFFF, cmd_regs, 81);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_table8, color_matrix[8] & 0xFFFF, cmd_regs, 82);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_offset0, color_matrix[9] & 0xFFFF, cmd_regs, 83);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_offset1, color_matrix[10] & 0xFFFF, cmd_regs, 84);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pendmatrix_offset2, color_matrix[11] & 0xFFFF, cmd_regs, 85);
+
+		cmdlist_regs_packing(crtc_to_cl(&a_crtc->crtc), CMDLIST_MOD_COMP, cmd_regs);
+		free_cmdlist_regs(cmd_regs);
+	}
+
+}
+
+int saturn_hee_check_end_matrix(struct drm_crtc_state *state)
+{
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = ac->color_matrix_blob_prop;
+	int *data;
+	int n;
+
+	if (blob) {
+		data = (int *)blob->data;
+		for (n = 0; n < 9; n++) {
+			if ((data[n] > 8191) || (data[n] < -8192)) {
+				DRM_DEBUG("The value of color matrix coeffs is invalid: value %d, n %d\n", data[n], n);
+				return -EINVAL;
+			}
+			data[n] = data[n] & 0xFFFF;
+		}
+
+		for (n = 9; n < 12; n++) {
+			//due to silicon limitation, LARK L offset value should be -1024 ~ 1023
+			if (data[n] > 1023) {
+				DRM_DEBUG("The value of color matrix offset : value %d, n %d\n", data[n], n);
+				data[n] = 1023;
+			}
+			if ((data[n] < -1024)) {
+				DRM_DEBUG("The value of color matrix offset : value %d, n %d\n", data[n], n);
+				data[n] = -1024;
+			}
+			//13 bit REG fihee for LARK M, althogh the range is from -1024 ~ 1023
+			data[n] = data[n] & 0x1FFF;
+		}
+	}
+
+	return 0;
+}
+
+void saturn_hee_conf_dpuctrl_ee(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_crtc_state *state = a_crtc->crtc.state;
+	struct spacemit_crtc_state *spacemit_state = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = spacemit_state->ee_blob_prop;
+	int i = 0;
+	uint32_t *ee_coef;
+
+	if (blob) {
+		ee_coef = (uint32_t *)blob->data;
+
+		for (i = 0; i < 45; i++)
+			dpu_write_reg(hwdev, EE_REG, EE_ADDR, value32[i], ee_coef[i]);
+	}
+}
+
+void saturn_hee_conf_dpuctrl_acad(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_crtc_state *state = a_crtc->crtc.state;
+	struct spacemit_crtc_state *spacemit_state = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = spacemit_state->pp_acad_blob_prop;
+	u32 acad_base = ACAD_ADDR;
+	u32 value;
+	int i;
+	struct cmdlist_regs *ad_cl = NULL;
+	int *acad_coef = NULL;
+
+	if (a_crtc->is_offline_mode == 1)
+		return;
+
+	ad_cl = alloc_cmdlist_regs(ACAD_REG);
+	if (hwdev->is_acad_on) {
+		if (blob) {
+			acad_coef = (int *)blob->data;
+			for (i = 0; i < hwdev->acad_num; i++) {
+				value = acad_coef[i];
+				dpu_write(hwdev, ACAD_REG, acad_base, value32[i], value, ad_cl, i);
+			}
+		}
+	} else {
+		dpu_write(hwdev, ACAD_REG, acad_base, acad_en, 0, ad_cl, 0);
+	}
+
+	cmdlist_regs_packing(crtc_to_cl(&a_crtc->crtc), CMDLIST_MOD_COMP, ad_cl);
+	free_cmdlist_regs(ad_cl);
+
+}
+
+void saturn_hee_conf_dpuctrl_pp_gamma(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	int i, j;
+	struct drm_crtc *crtc = &a_crtc->crtc;
+	struct drm_crtc_state *state = crtc->state;
+	struct spacemit_crtc_state *spacemit_state = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = spacemit_state->gamma_table_blob_prop;
+	u32 value;
+	u32 pp_base = PP1_BASE_ADDR;
+	u32 gamma_base = GAMMA_BASE_ADDR;
+	u32 base = 0x8;
+
+	if (blob) {
+		//first frame, gamma table is set
+		if (!hwdev->gamma_table) {
+			hwdev->gamma_table = kzalloc(blob->length, GFP_KERNEL);
+			if (!hwdev->gamma_table)
+				return;
+			memcpy(hwdev->gamma_table, blob->data, blob->length);
+		}
+	}
+
+	//set hwdev->gamma_table when power on
+	if (!hwdev->gamma_table)
+		return;
+
+	dpu_write_reg(hwdev, LTM_REG, pp_base, m_npost_proc_en, 1);
+	dpu_write_reg(hwdev, USR_GMA_REG, gamma_base, usr_gma_en, 1);
+	for (i = 0; i < 3; i++) {
+		for (j = 0; j < 256; j += 2) {
+			value = ((hwdev->gamma_table[i * 257 + j] >> 2) & 0xFFF) |
+					(((hwdev->gamma_table[i * 257 + j + 1] >> 2) & 0xFFF) << 16);
+			dpu_write_reg(hwdev, USR_GMA_REG, gamma_base, value32[(base + j * 2 + i * 0x204) / 4], value);
+		}
+		value = (hwdev->gamma_table[i * 257 + 256] >> 2) & 0xFFF;
+		dpu_write_reg(hwdev, USR_GMA_REG, gamma_base, value32[(base + 256 * 2 + i * 0x204) / 4], value);
+	}
+
+	dpu_write_reg(hwdev, USR_GMA_REG, gamma_base, usr_gma_cfg_done, 1);
+}
+
+void saturn_hee_dpuctrl_color_temp(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state)
+{
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_crtc_state *state = a_crtc->crtc.state;
+	struct spacemit_crtc_state *spacemit_state = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = spacemit_state->pp_color_temperature_blob_property;
+	int *color_temp;
+	struct cmdlist_regs *cmd_regs = NULL;
+
+	if (blob) {
+		color_temp = (int *)blob->data;
+
+		cmd_regs = alloc_cmdlist_regs(LTM_REG);
+
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_npost_proc_en, 1, cmd_regs, 0);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_nmatrix_en, 1, cmd_regs, 1);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_ngain_to_full_en, 0, cmd_regs, 2);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_nendmatrix_en, 0, cmd_regs, 3);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_nfront_tmootf_en, 1, cmd_regs, 4);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_nend_tmootf_en, 0, cmd_regs, 5);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_neotf_en, 0, cmd_regs, 6);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_noetf_en, 0, cmd_regs, 7);
+
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table0, color_temp[0] & 0xFFFF, cmd_regs, 65);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table1, color_temp[1] & 0xFFFF, cmd_regs, 66);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table2, color_temp[2] & 0xFFFF, cmd_regs, 67);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table3, color_temp[3] & 0xFFFF, cmd_regs, 68);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table4, color_temp[4] & 0xFFFF, cmd_regs, 69);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table5, color_temp[5] & 0xFFFF, cmd_regs, 70);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table6, color_temp[6] & 0xFFFF, cmd_regs, 71);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table7, color_temp[7] & 0xFFFF, cmd_regs, 72);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_table8, color_temp[8] & 0xFFFF, cmd_regs, 73);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_offset0, color_temp[9] & 0x1FFFFFF, cmd_regs, 74);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_offset1, color_temp[10] & 0x1FFFFFF, cmd_regs, 75);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_pmatrix_offset2, color_temp[11] & 0x1FFFFFF, cmd_regs, 76);
+
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_tmootf_weightYr, color_temp[12] & 0xFFF, cmd_regs, 157);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_tmootf_weightYg, color_temp[13] & 0xFFF, cmd_regs, 157);
+		dpu_write(hwdev, LTM_REG, LTM_BASE_ADDR, m_tmootf_weightYb, color_temp[14] & 0xFFF, cmd_regs, 158);
+
+		cmdlist_regs_packing(crtc_to_cl(&a_crtc->crtc), CMDLIST_MOD_COMP, cmd_regs);
+
+		free_cmdlist_regs(cmd_regs);
+	}
+
+	return;
+}
diff --git a/drivers/gpu/drm/spacemit/dpu/post_process_hee.h b/drivers/gpu/drm/spacemit/dpu/post_process_hee.h
new file mode 100644
index 000000000000..dee526b2814b
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/post_process_hee.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SATURN_hee_POST_H_
+#define _SATURN_hee_POST_H_
+#include "../spacemit_crtc.h"
+
+int saturn_hee_check_end_matrix(struct drm_crtc_state *state);
+void saturn_hee_conf_dpuctrl_pp_gamma(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+void saturn_hee_conf_dpuctrl_color_matrix(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+void saturn_hee_conf_dpuctrl_acad(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+void saturn_hee_conf_dpuctrl_ee(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+void saturn_hee_dpuctrl_color_temp(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_fbcmem.c b/drivers/gpu/drm/spacemit/dpu/saturn_fbcmem.c
new file mode 100644
index 000000000000..54f056f718a3
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_fbcmem.c
@@ -0,0 +1,342 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/stddef.h>
+#include <linux/export.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_mode.h>
+#include <drm/drm_print.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_fourcc.h>
+#include "../spacemit_cmdlist.h"
+#include "saturn_fbcmem.h"
+
+static int get_fbc_block_size_by_modifier(uint64_t modifier, u32 *out_fbc_block_size)
+{
+	int ret_val = 0;
+	uint64_t super_block_size = modifier & AFBC_FORMAT_MOD_BLOCK_SIZE_MASK;
+
+	if (super_block_size == AFBC_FORMAT_MOD_BLOCK_SIZE_16x16) {
+		*out_fbc_block_size = FBC_BLOCK_SIZE_16x16;
+	} else if (super_block_size == AFBC_FORMAT_MOD_BLOCK_SIZE_32x8) {
+		*out_fbc_block_size = FBC_BLOCK_SIZE_32x8;
+	} else {
+		DRM_ERROR("FBC_MEM: failed to get fbc block size info for modifier = 0x%llx\n", super_block_size);
+		ret_val = -1;
+	}
+
+	return ret_val;
+}
+
+static bool is_bpp32_in_fbc_mem_cal(uint32_t drm_4cc_fmt)
+{
+	bool is_taken_as_bpp32 = false;
+	const struct drm_format_info *info = NULL;
+
+	info = drm_format_info(drm_4cc_fmt);
+	if (info->num_planes == 1 && info->cpp[0] == 4) {
+		is_taken_as_bpp32 = true;
+	} else {
+		switch (drm_4cc_fmt) {
+		case DRM_FORMAT_RGB888:   //RDMA_FMT_RGB_888:
+		case DRM_FORMAT_BGR888:   //RDMA_FMT_BGR_888:
+			is_taken_as_bpp32 = true;
+			break;
+		default:
+			break;
+		}
+	}
+
+	return is_taken_as_bpp32;
+}
+
+static u32 adjust_afbc_layer_mem_size(u8 rdma_work_mode, u32 drm_4cc_fmt, u32 fbc_block_size, bool rot_90_or_270, u32 fbcmem_size)
+{
+	u32 ret_size = fbcmem_size;
+	bool is_bpp_32 = is_bpp32_in_fbc_mem_cal(drm_4cc_fmt);
+
+	//if left_right mode fbc_layout 32x8, is_bpp_32 and not rotation 90/270, hw request double mem_size.
+	if ((rdma_work_mode == LEFT_RIGHT) && (fbc_block_size == FBC_BLOCK_SIZE_32x8) && is_bpp_32 && (rot_90_or_270 == false))
+		ret_size *= 2;
+	DRM_DEBUG("FBC_MEM: %s, rdma_mode = %u, is_bpp_32 = %d, fbc_block_size = %u, rot_90_or_270 = %d, input = %u, ret = %u\n",
+		__func__, rdma_work_mode, is_bpp_32, fbc_block_size, rot_90_or_270, fbcmem_size, ret_size);
+
+	return ret_size;
+}
+
+int get_raw_data_plane_rdma_mem_size(u32 drm_4cc_fmt, bool rot_90_or_270, u32 plane_crop_width, u32 *output_mem_size)
+{
+	u8 index = 0;
+	u32 ret_mem_size = 0;
+	u32 data_plane_mem_size[3] = {0}; //max 3 plane, YUV data
+	const struct drm_format_info *info = NULL;
+
+	if (rot_90_or_270) {
+		ret_mem_size = 1024; //dpu hardware request 32K fix size, 32 byte unit has considered
+	} else {
+		info = drm_format_info(drm_4cc_fmt);
+		if (info->cpp[0] == 0) {
+			DRM_ERROR("FBC_MEM: not support format %p4cc\n", &drm_4cc_fmt);
+			return -1;
+		}
+		if (info->num_planes == 1 && info->is_yuv == false) {
+			data_plane_mem_size[0] = plane_crop_width * info->cpp[0];
+		} else if (info->num_planes >= 2 && info->is_yuv) {
+			data_plane_mem_size[0] = plane_crop_width * info->cpp[0];
+			data_plane_mem_size[1] = plane_crop_width * info->cpp[1]/info->hsub;
+			if (info->num_planes == 3)
+				data_plane_mem_size[2] = plane_crop_width * info->cpp[2]/info->vsub;
+		} else {
+			DRM_ERROR("FBC_MEM: not considered drm format %p4cc\n", &drm_4cc_fmt);
+			return -1;
+		}
+		for (index = 0; index < info->num_planes; index++) { //max 3 plane, YUV data
+			data_plane_mem_size[index] = roundup(data_plane_mem_size[index], 64); //dpu hardware request
+			ret_mem_size += data_plane_mem_size[index];
+		}
+		ret_mem_size = roundup(ret_mem_size, 64);
+		ret_mem_size = ret_mem_size/32;
+	}
+	if (output_mem_size) {
+		*output_mem_size = ret_mem_size;
+		DRM_DEBUG("FBC_MEM: raw layer, fmt = %p4cc, rot_90_270 = %d, crop_w = %u, cal memsize = %u\n",
+		&drm_4cc_fmt, rot_90_or_270, plane_crop_width, *output_mem_size);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(get_raw_data_plane_rdma_mem_size);
+
+int get_afbc_data_plane_min_rdma_mem_size(u8 rdma_work_mode, u32 drm_4cc_fmt,
+		u32 crop_start_x, u32 crop_start_y, u32 crop_width, u32 crop_height,
+		u32 fbc_block_size, bool rot_90_or_270, u8 min_lines, u32 *output_mem_size)
+{
+	bool is_bpp_32 = false;
+	uint32_t ret_mem_size = 0;
+	uint32_t crop_start_align = 0;
+	uint32_t crop_end_align = 0;
+	uint32_t bbox_width = 0;
+	uint32_t num_addr_per_line = 0;
+	uint32_t num_addr_frac = 0;
+	uint32_t num_addr_line = 0;
+	uint32_t align_num = (fbc_block_size == FBC_BLOCK_SIZE_16x16) ? 16 : 32;
+
+	DRM_DEBUG("FBC_MEM: afbc layer, fmt = %p4cc, \
+		crop_x = %u, crop_y = %u, crop_w = %u, crop_h = %u, \
+		fbc_block_size = %u, rot_90_270 = %u, min_lines = %u, rdma_mode = %u\n",
+		&drm_4cc_fmt, crop_start_x, crop_start_y, crop_width,
+		crop_height, fbc_block_size, rot_90_or_270, min_lines, rdma_work_mode);
+
+	if (rot_90_or_270) {
+		crop_start_align = rounddown(crop_start_y, align_num);
+		crop_end_align = roundup(crop_start_y + crop_height, align_num) - 1;
+	} else {
+		crop_start_align = rounddown(crop_start_x, align_num);
+		crop_end_align = roundup(crop_start_x + crop_width, align_num) - 1;
+	}
+
+	bbox_width = crop_end_align - crop_start_align + 1;
+	is_bpp_32 = is_bpp32_in_fbc_mem_cal(drm_4cc_fmt);
+	num_addr_per_line = is_bpp_32 ? bbox_width/8 : bbox_width/16;
+
+	if (min_lines == 4) {
+		num_addr_frac = (num_addr_per_line % 8 == 0) ? num_addr_per_line/8 : (num_addr_per_line/8 + 1);
+		num_addr_line = num_addr_frac * 8 * 4;
+		ret_mem_size = num_addr_line;
+	} else if (min_lines == 8) {
+		num_addr_frac = (num_addr_per_line % 16 == 0) ? num_addr_per_line/16 : (num_addr_per_line/16 + 1);
+		num_addr_line = num_addr_frac * 16 * 8;
+		ret_mem_size = num_addr_line;
+	} else if (min_lines == 16) {
+		num_addr_frac = (num_addr_per_line % 32 == 0) ? num_addr_per_line/32 : (num_addr_per_line/32 + 1);
+		num_addr_line = num_addr_frac * 32 * 16;
+		ret_mem_size = num_addr_line;
+	}
+
+	ret_mem_size = adjust_afbc_layer_mem_size(rdma_work_mode, drm_4cc_fmt, fbc_block_size, rot_90_or_270, ret_mem_size);
+	*output_mem_size = ret_mem_size;
+
+	DRM_DEBUG("FBC_MEM: afbc layer, fmt = %p4cc, \
+		crop_x = %u, crop_y = %u, crop_w = %u, crop_h = %u, \
+		fbc_block_size = %u, rot_90_270 = %u, min_lines = %u, rdma_mode = %u, cal memsize = %u\n",
+		&drm_4cc_fmt, crop_start_x, crop_start_y, crop_width,
+		crop_height, fbc_block_size, rot_90_or_270, min_lines, rdma_work_mode, *output_mem_size);
+
+	return 0;
+}
+EXPORT_SYMBOL(get_afbc_data_plane_min_rdma_mem_size);
+
+int saturn_cal_layer_fbcmem_size(struct drm_plane *plane,
+				   struct drm_plane_state *state)
+{
+	int ret = 0;
+	//u32 adjust_fbcmem_size = 0;
+	struct spacemit_plane_state *pstate = to_spacemit_plane_state(state);
+	uint64_t modifier = pstate->state.fb->modifier;
+
+	u32 drm_4cc_fmt = pstate->state.fb->format->format;
+	unsigned int rotation = pstate->state.rotation;
+	bool rot_90_or_270 = is_rot_90_270(rotation);
+	u32 crop_x = pstate->state.src_x >> 16; //check
+	u32 crop_y = pstate->state.src_y >> 16; //check
+	u32 crop_w = pstate->state.src_w >> 16; //check
+	u32 crop_h = pstate->state.src_h >> 16; //check
+	u32 fbc_block_size = FBC_BLOCK_SIZE_LIMIT;
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state->state, state->crtc);
+	const struct spacemit_crtc_rdma *rdmas = to_spacemit_crtc_state(crtc_state)->rdmas;
+	u8 rdma_work_mode = rdmas[pstate->rdma_id].mode;
+	const struct drm_format_info *info = NULL;
+	u8 min_lines = pstate->dec_lines ? pstate->dec_lines : 4;
+
+	info = drm_format_info(drm_4cc_fmt);
+
+	DRM_DEBUG("min_lines %d ac->dec_lines %d\n", min_lines, pstate->dec_lines);
+
+	if (modifier == 0) { //raw data layer
+		ret = get_raw_data_plane_rdma_mem_size(drm_4cc_fmt, rot_90_or_270, crop_w, &(pstate->fbcmem_size));
+	} else { //afbc data layer
+		ret = get_fbc_block_size_by_modifier(modifier, &fbc_block_size);
+		if (ret < 0) {
+			DRM_ERROR("FBC_MEM: failed to get fbc block size info for modifier = 0x%llx\n", modifier);
+			return -1;
+		}
+		if (modifier && info->is_yuv) { //video layer
+			if (rot_90_or_270)
+				min_lines = 16; //dpu hardware request 16 line at least
+			else
+				min_lines = 8; //dpu hardware request 8 line at least
+		} else if (modifier && rot_90_or_270) {
+			min_lines = 8;
+		}
+
+		ret = get_afbc_data_plane_min_rdma_mem_size(rdma_work_mode, drm_4cc_fmt, crop_x, crop_y,
+			crop_w, crop_h, fbc_block_size, rot_90_or_270, min_lines, &(pstate->fbcmem_size));
+	}
+	if (ret < 0) {
+		DRM_ERROR("FBC_MEM: faied to get plane mem size for plane: src_x = %u, scr_y = %u, src_h = %u, src_w = %u\n",
+				pstate->state.src_x, pstate->state.src_y, pstate->state.src_h, pstate->state.src_w);
+	}
+
+	return ret;
+}
+
+int saturn_adjust_rdma_fbcmem(struct spacemit_hw_device *hwdev,
+				     struct spacemit_crtc_rdma *rdmas)
+{
+	int ret = -1;
+	u8 index = 0;
+	u8 sec_fbcmem_index = 0;
+	u32 pri_fbcmem_size = 0;
+	u32 sec_fbcmem_size = 0;
+	u32 *fbc_mems_left = NULL;
+	u32 cur_rdma_fbcmem_size = 0;
+	u8 rdma_nums = hwdev->rdma_nums;
+	bool pre_odd_rdma_use_shared_fbc_mem = false;
+
+	struct spacemit_crtc_fbcmem *fbcmem = NULL;
+
+	for (index = 0; index < rdma_nums; index++) {
+		fbcmem = &(rdmas[index].fbcmem);
+		DRM_DEBUG("input rdmas[%u/%u]: mode = %d, start = %d, size = %d, map = %d\n",
+			index, rdma_nums, rdmas[index].mode, fbcmem->start, fbcmem->size, fbcmem->map);
+	}
+
+	fbc_mems_left = kzalloc(sizeof(u32) * (rdma_nums/2), GFP_KERNEL);
+	if (fbc_mems_left == NULL) {
+		DRM_ERROR("FBC_MEM: No memory left!\n");
+		goto free;
+	}
+	for (index = 0; index < rdma_nums/2; index++) {
+		fbc_mems_left[index] = hwdev->fbcmem_sizes[index] / FBCMEM_UNIT;
+		DRM_DEBUG("fbcmem_sizes[%u/%u] = %u, total fbc_mems_left = %u\n",
+			index, rdma_nums/2, hwdev->fbcmem_sizes[index], fbc_mems_left[index]);
+		if (fbc_mems_left[index] == 0) {
+			DRM_ERROR("FBC_MEM: error fbcmem_sizes[%d] = %u\n", index, hwdev->fbcmem_sizes[index]);
+			goto free;
+		}
+	}
+
+	for (index = 0; index < rdma_nums; index++) {
+		cur_rdma_fbcmem_size = rdmas[index].fbcmem.size;
+		if (cur_rdma_fbcmem_size == 0) {
+			DRM_DEBUG("return directly for rmda %u fbcmem size is 0\n", index);
+			continue;
+		}
+		pri_fbcmem_size = hwdev->fbcmem_sizes[index/2] / FBCMEM_UNIT;
+
+		if (index % 2 == 0) { //dma_id is even: 0, 2, 4, 6...
+			DRM_DEBUG("rdma%u, cur_rdma_fbcmem_size = %u, pri_fbcmem_size = %u\n",
+									index, cur_rdma_fbcmem_size, pri_fbcmem_size);
+			if (cur_rdma_fbcmem_size > pri_fbcmem_size) {
+				DRM_ERROR("FBC_MEM: rdma %d use %d byte, excess the size %d\n",
+					index, cur_rdma_fbcmem_size, pri_fbcmem_size);
+				goto free;
+			}
+			if (index > 0) {
+				pre_odd_rdma_use_shared_fbc_mem = rdmas[index - 1].fbcmem.map;
+				if (pre_odd_rdma_use_shared_fbc_mem) {
+					DRM_INFO_ONCE("FBC_MEM: both rdma %d %d use fbc memory\n", index, index - 1);
+					goto free;
+				}
+			}
+			rdmas[index].fbcmem.start = 0;
+			rdmas[index].fbcmem.map = true;
+			fbc_mems_left[index/2] -= cur_rdma_fbcmem_size;
+			DRM_DEBUG("FBC_MEM: rdma: id = %d, size = %u, start = %u, map = %u, fbc_mems_left[%u] = %d\n",
+				index, cur_rdma_fbcmem_size, rdmas[index].fbcmem.start, rdmas[index].fbcmem.map, index/2, fbc_mems_left[index/2]);
+		} else { //odd rdma: rdma 1, 3, 5...
+			if (cur_rdma_fbcmem_size <= fbc_mems_left[index/2]) { //not share fbc mem
+				DRM_DEBUG("rdma%u, cur_rdma_fbcmem_size = %u, fbc_mems_left[%u] = %u\n",
+					index, cur_rdma_fbcmem_size, index/2, fbc_mems_left[index/2]);
+				rdmas[index].fbcmem.map = false;
+				rdmas[index].fbcmem.start = pri_fbcmem_size - fbc_mems_left[index/2];
+				rdmas[index].fbcmem.size = fbc_mems_left[index/2];//use all the mem left
+				fbc_mems_left[index/2] = 0;
+				DRM_DEBUG("FBC_MEM: rdma: id = %d, size = %u, actually size = %u, start = %u, map = %u, fbc_mems_left[%u] = 0\n",
+					index, cur_rdma_fbcmem_size, rdmas[index].fbcmem.size, rdmas[index].fbcmem.start, rdmas[index].fbcmem.map, index/2);
+			} else { //need share fbc mem
+				sec_fbcmem_index = (index/2 + 1)%(rdma_nums/2);
+				sec_fbcmem_size = hwdev->fbcmem_sizes[sec_fbcmem_index] / FBCMEM_UNIT;
+				DRM_DEBUG("rdma%u, cur_rdma_fbcmem_size = %u, fbc_mems_left[%u] = %d, sec_fbcmem_size = %u\n",
+					index, cur_rdma_fbcmem_size, index/2, fbc_mems_left[index/2], sec_fbcmem_size);
+				if (cur_rdma_fbcmem_size > fbc_mems_left[index/2] + sec_fbcmem_size) {
+					DRM_ERROR("FBC_MEM: rdma %d use %d mem size, but left %d\n",
+						index, cur_rdma_fbcmem_size, fbc_mems_left[index/2] + sec_fbcmem_size);
+					goto free;
+				} else {
+					if (index == rdma_nums - 1) { //last rdma id
+						if (fbc_mems_left[0] != hwdev->fbcmem_sizes[0] / FBCMEM_UNIT) { //fbc mem0 has used
+							DRM_ERROR("FBC_MEM: rdma %d can not use fbc mem 0 for it has been used\n", index);
+							goto free;
+						} else {
+							rdmas[index].fbcmem.size = fbc_mems_left[index/2] + sec_fbcmem_size;
+							cur_rdma_fbcmem_size = rdmas[index].fbcmem.size;
+						}
+					}
+					fbc_mems_left[sec_fbcmem_index] -= (cur_rdma_fbcmem_size - fbc_mems_left[index/2]);
+					DRM_DEBUG("rmda%u, fbc_mems_left[%u] = %d, fbc_mems_left[%u] = %u\n",
+						index, index/2, fbc_mems_left[index/2], sec_fbcmem_index, fbc_mems_left[sec_fbcmem_index]);
+					rdmas[index].fbcmem.start = pri_fbcmem_size - fbc_mems_left[index/2];
+					rdmas[index].fbcmem.map = true;
+					fbc_mems_left[index/2] = 0;
+
+					fbcmem = &(rdmas[index].fbcmem);
+					DRM_DEBUG("FBC_MEM: rdma: id = %d, actually size = %u, start = %u, map = %u\n",
+						index, fbcmem->size, fbcmem->start, fbcmem->map);
+				}
+			}
+		}
+	}
+	ret = 0;
+
+free:
+	if (fbc_mems_left) {
+		kfree(fbc_mems_left);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(saturn_adjust_rdma_fbcmem);
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_fbcmem.h b/drivers/gpu/drm/spacemit/dpu/saturn_fbcmem.h
new file mode 100644
index 000000000000..3a35577e6932
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_fbcmem.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _SATURN_FBCMEM_H_
+#define _SATURN_FBCMEM_H_
+
+#include <drm/drm_plane.h>
+#include "../spacemit_crtc.h"
+#include "../spacemit_dpu_reg.h"
+#include "dpu_saturn.h"
+
+#define FBCMEM_UNIT	(32)	//fbcmem is 32 bytes per unit
+
+typedef enum FBC_BLOCK_SIZE_E {
+	FBC_BLOCK_SIZE_16x16, //0
+	FBC_BLOCK_SIZE_32x8,  //1
+	FBC_BLOCK_SIZE_LIMIT, //error block size
+} FBC_BLOCK_SIZE_T;
+
+int get_raw_data_plane_rdma_mem_size(u32 drm_4cc_fmt, bool rot_90_or_270,
+				u32 plane_crop_width, u32 *output_mem_size);
+
+int get_afbc_data_plane_min_rdma_mem_size(u8 rdma_work_mode, u32 drm_4cc_fmt,
+	u32 crop_start_x, u32 crop_start_y, u32 crop_width, u32 crop_height,
+	u32 fbc_block_size, bool rot_90_or_270, u8 min_lines, u32 *output_mem_size);
+
+int saturn_cal_layer_fbcmem_size(struct drm_plane *plane,
+				 struct drm_plane_state *state);
+
+int saturn_adjust_rdma_fbcmem(struct spacemit_hw_device *hwdev,
+			      struct spacemit_crtc_rdma *rdmas);
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/acad.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/acad.h
new file mode 100644
index 000000000000..382294f65425
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/acad.h
@@ -0,0 +1,672 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef ACAD_REG_H
+#define ACAD_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_acad_with_curve_reg_0
+	UINT32 acad_en                 : 1;
+	UINT32 m_n_hist_mode           : 1;
+	UINT32 m_n_curve_mode          : 1;
+	UINT32 m_n_etm_mode            : 1;
+	UINT32:28;
+
+	//REGISTER saturn_acad_with_curve_reg_1
+	UINT32 acad_restart            : 1;
+	UINT32:31;
+
+	//REGISTER saturn_acad_with_curve_reg_2
+	UINT32:32;
+
+	//REGISTER saturn_acad_with_curve_reg_3
+	UINT32 m_n_roi_x               : 13;
+	UINT32:3;
+	UINT32 m_n_roi_y               : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_4
+	UINT32 m_n_roi_w               : 13;
+	UINT32:3;
+	UINT32 m_n_roi_h               : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_5
+	UINT32 m_n_blk_width           : 9;
+	UINT32 m_n_blk_num_hori        : 6;
+	UINT32:1;
+	UINT32 m_n_blk_height          : 9;
+	UINT32 m_n_blk_num_vert        : 6;
+	UINT32:1;
+
+	//REGISTER saturn_acad_with_curve_reg_6
+	UINT32 m_n_weight_scaler_x     : 11;
+	UINT32:5;
+	UINT32 m_n_weight_scaler_y     : 11;
+	UINT32:5;
+
+	//REGISTER saturn_acad_with_curve_reg_7
+	UINT32 m_n_sub_roi0_x          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi0_y          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_8
+	UINT32 m_n_sub_roi0_w          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi0_h          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_9
+	UINT32 m_n_sub_roi1_x          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi1_y          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_10
+	UINT32 m_n_sub_roi1_w          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi1_h          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_11
+	UINT32 m_n_sub_roi2_x          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi2_y          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_12
+	UINT32 m_n_sub_roi2_w          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi2_h          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_13
+	UINT32 m_n_sub_roi3_x          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi3_y          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_14
+	UINT32 m_n_sub_roi3_w          : 13;
+	UINT32:3;
+	UINT32 m_n_sub_roi3_h          : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_15
+	UINT32 tp_alpha                : 9;
+	UINT32:7;
+	UINT32 m_n_curve_alpha         : 13;
+	UINT32:3;
+
+	//REGISTER saturn_acad_with_curve_reg_16
+	UINT32 m_n_offset0             : 11;
+	UINT32:5;
+	UINT32 m_n_offset1             : 11;
+	UINT32:5;
+
+	//REGISTER saturn_acad_with_curve_reg_17
+	UINT32 m_n_dst_alpha           : 8;
+	UINT32 m_n_alpha_step          : 8;
+	UINT32 m_n_skip_frames         : 8;
+	UINT32 m_p_segment_thr0        : 8;
+
+	//REGISTER saturn_acad_with_curve_reg_18
+	UINT32 m_p_segment_thr1        : 8;
+	UINT32 m_p_segment_thr2        : 8;
+	UINT32 m_p_segment_thr3        : 8;
+	UINT32 m_p_segment_thr4        : 8;
+
+	//REGISTER saturn_acad_with_curve_reg_19
+	UINT32 m_p_contrast_curve0_0   : 16;
+	UINT32 m_p_contrast_curve1_0   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_20
+	UINT32 m_p_contrast_curve0_1   : 16;
+	UINT32 m_p_contrast_curve1_1   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_21
+	UINT32 m_p_contrast_curve0_2   : 16;
+	UINT32 m_p_contrast_curve1_2   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_22
+	UINT32 m_p_contrast_curve0_3   : 16;
+	UINT32 m_p_contrast_curve1_3   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_23
+	UINT32 m_p_contrast_curve0_4   : 16;
+	UINT32 m_p_contrast_curve1_4   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_24
+	UINT32 m_p_contrast_curve0_5   : 16;
+	UINT32 m_p_contrast_curve1_5   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_25
+	UINT32 m_p_contrast_curve0_6   : 16;
+	UINT32 m_p_contrast_curve1_6   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_26
+	UINT32 m_p_contrast_curve0_7   : 16;
+	UINT32 m_p_contrast_curve1_7   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_27
+	UINT32 m_p_contrast_curve0_8   : 16;
+	UINT32 m_p_contrast_curve1_8   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_28
+	UINT32 m_p_contrast_curve0_9   : 16;
+	UINT32 m_p_contrast_curve1_9   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_29
+	UINT32 m_p_contrast_curve0_10  : 16;
+	UINT32 m_p_contrast_curve1_10  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_30
+	UINT32 m_p_contrast_curve0_11  : 16;
+	UINT32 m_p_contrast_curve1_11  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_31
+	UINT32 m_p_contrast_curve0_12  : 16;
+	UINT32 m_p_contrast_curve1_12  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_32
+	UINT32 m_p_contrast_curve0_13  : 16;
+	UINT32 m_p_contrast_curve1_13  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_33
+	UINT32 m_p_contrast_curve0_14  : 16;
+	UINT32 m_p_contrast_curve1_14  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_34
+	UINT32 m_p_contrast_curve0_15  : 16;
+	UINT32 m_p_contrast_curve1_15  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_35
+	UINT32 m_p_contrast_curve0_16  : 16;
+	UINT32 m_p_contrast_curve1_16  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_36
+	UINT32 m_p_contrast_curve0_17  : 16;
+	UINT32 m_p_contrast_curve1_17  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_37
+	UINT32 m_p_contrast_curve0_18  : 16;
+	UINT32 m_p_contrast_curve1_18  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_38
+	UINT32 m_p_contrast_curve0_19  : 16;
+	UINT32 m_p_contrast_curve1_19  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_39
+	UINT32 m_p_contrast_curve0_20  : 16;
+	UINT32 m_p_contrast_curve1_20  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_40
+	UINT32 m_p_contrast_curve0_21  : 16;
+	UINT32 m_p_contrast_curve1_21  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_41
+	UINT32 m_p_contrast_curve0_22  : 16;
+	UINT32 m_p_contrast_curve1_22  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_42
+	UINT32 m_p_contrast_curve0_23  : 16;
+	UINT32 m_p_contrast_curve1_23  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_43
+	UINT32 m_p_contrast_curve0_24  : 16;
+	UINT32 m_p_contrast_curve1_24  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_44
+	UINT32 m_p_contrast_curve0_25  : 16;
+	UINT32 m_p_contrast_curve1_25  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_45
+	UINT32 m_p_contrast_curve0_26  : 16;
+	UINT32 m_p_contrast_curve1_26  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_46
+	UINT32 m_p_contrast_curve0_27  : 16;
+	UINT32 m_p_contrast_curve1_27  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_47
+	UINT32 m_p_contrast_curve0_28  : 16;
+	UINT32 m_p_contrast_curve1_28  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_48
+	UINT32 m_p_contrast_curve0_29  : 16;
+	UINT32 m_p_contrast_curve1_29  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_49
+	UINT32 m_p_contrast_curve0_30  : 16;
+	UINT32 m_p_contrast_curve1_30  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_50
+	UINT32 m_p_contrast_curve0_31  : 16;
+	UINT32 m_p_contrast_curve1_31  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_51
+	UINT32 m_p_contrast_curve0_32  : 16;
+	UINT32 m_p_contrast_curve1_32  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_52
+	UINT32 m_p_contrast_curve0_33  : 16;
+	UINT32 m_p_contrast_curve1_33  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_53
+	UINT32 m_p_contrast_curve0_34  : 16;
+	UINT32 m_p_contrast_curve1_34  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_54
+	UINT32 m_p_contrast_curve0_35  : 16;
+	UINT32 m_p_contrast_curve1_35  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_55
+	UINT32 m_p_contrast_curve0_36  : 16;
+	UINT32 m_p_contrast_curve1_36  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_56
+	UINT32 m_p_contrast_curve0_37  : 16;
+	UINT32 m_p_contrast_curve1_37  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_57
+	UINT32 m_p_contrast_curve0_38  : 16;
+	UINT32 m_p_contrast_curve1_38  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_58
+	UINT32 m_p_contrast_curve0_39  : 16;
+	UINT32 m_p_contrast_curve1_39  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_59
+	UINT32 m_p_contrast_curve0_40  : 16;
+	UINT32 m_p_contrast_curve1_40  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_60
+	UINT32 m_p_contrast_curve0_41  : 16;
+	UINT32 m_p_contrast_curve1_41  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_61
+	UINT32 m_p_contrast_curve0_42  : 16;
+	UINT32 m_p_contrast_curve1_42  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_62
+	UINT32 m_p_contrast_curve0_43  : 16;
+	UINT32 m_p_contrast_curve1_43  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_63
+	UINT32 m_p_contrast_curve0_44  : 16;
+	UINT32 m_p_contrast_curve1_44  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_64
+	UINT32 m_p_contrast_curve0_45  : 16;
+	UINT32 m_p_contrast_curve1_45  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_65
+	UINT32 m_p_contrast_curve0_46  : 16;
+	UINT32 m_p_contrast_curve1_46  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_66
+	UINT32 m_p_contrast_curve0_47  : 16;
+	UINT32 m_p_contrast_curve1_47  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_67
+	UINT32 m_p_contrast_curve0_48  : 16;
+	UINT32 m_p_contrast_curve1_48  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_68
+	UINT32 m_p_contrast_curve0_49  : 16;
+	UINT32 m_p_contrast_curve1_49  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_69
+	UINT32 m_p_contrast_curve0_50  : 16;
+	UINT32 m_p_contrast_curve1_50  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_70
+	UINT32 m_p_contrast_curve0_51  : 16;
+	UINT32 m_p_contrast_curve1_51  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_71
+	UINT32 m_p_contrast_curve0_52  : 16;
+	UINT32 m_p_contrast_curve1_52  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_72
+	UINT32 m_p_contrast_curve0_53  : 16;
+	UINT32 m_p_contrast_curve1_53  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_73
+	UINT32 m_p_contrast_curve0_54  : 16;
+	UINT32 m_p_contrast_curve1_54  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_74
+	UINT32 m_p_contrast_curve0_55  : 16;
+	UINT32 m_p_contrast_curve1_55  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_75
+	UINT32 m_p_contrast_curve0_56  : 16;
+	UINT32 m_p_contrast_curve1_56  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_76
+	UINT32 m_p_contrast_curve0_57  : 16;
+	UINT32 m_p_contrast_curve1_57  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_77
+	UINT32 m_p_contrast_curve0_58  : 16;
+	UINT32 m_p_contrast_curve1_58  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_78
+	UINT32 m_p_contrast_curve0_59  : 16;
+	UINT32 m_p_contrast_curve1_59  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_79
+	UINT32 m_p_contrast_curve0_60  : 16;
+	UINT32 m_p_contrast_curve1_60  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_80
+	UINT32 m_p_contrast_curve0_61  : 16;
+	UINT32 m_p_contrast_curve1_61  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_81
+	UINT32 m_p_contrast_curve0_62  : 16;
+	UINT32 m_p_contrast_curve1_62  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_82
+	UINT32 m_p_contrast_curve0_63  : 16;
+	UINT32 m_p_contrast_curve1_63  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_83
+	UINT32 m_p_contrast_curve0_64  : 16;
+	UINT32 m_p_contrast_curve1_64  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_84
+	UINT32 m_p_contrast_curve2_0   : 16;
+	UINT32 m_p_contrast_curve3_0   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_85
+	UINT32 m_p_contrast_curve2_1   : 16;
+	UINT32 m_p_contrast_curve3_1   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_86
+	UINT32 m_p_contrast_curve2_2   : 16;
+	UINT32 m_p_contrast_curve3_2   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_87
+	UINT32 m_p_contrast_curve2_3   : 16;
+	UINT32 m_p_contrast_curve3_3   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_88
+	UINT32 m_p_contrast_curve2_4   : 16;
+	UINT32 m_p_contrast_curve3_4   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_89
+	UINT32 m_p_contrast_curve2_5   : 16;
+	UINT32 m_p_contrast_curve3_5   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_90
+	UINT32 m_p_contrast_curve2_6   : 16;
+	UINT32 m_p_contrast_curve3_6   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_91
+	UINT32 m_p_contrast_curve2_7   : 16;
+	UINT32 m_p_contrast_curve3_7   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_92
+	UINT32 m_p_contrast_curve2_8   : 16;
+	UINT32 m_p_contrast_curve3_8   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_93
+	UINT32 m_p_contrast_curve2_9   : 16;
+	UINT32 m_p_contrast_curve3_9   : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_94
+	UINT32 m_p_contrast_curve2_10  : 16;
+	UINT32 m_p_contrast_curve3_10  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_95
+	UINT32 m_p_contrast_curve2_11  : 16;
+	UINT32 m_p_contrast_curve3_11  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_96
+	UINT32 m_p_contrast_curve2_12  : 16;
+	UINT32 m_p_contrast_curve3_12  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_97
+	UINT32 m_p_contrast_curve2_13  : 16;
+	UINT32 m_p_contrast_curve3_13  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_98
+	UINT32 m_p_contrast_curve2_14  : 16;
+	UINT32 m_p_contrast_curve3_14  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_99
+	UINT32 m_p_contrast_curve2_15  : 16;
+	UINT32 m_p_contrast_curve3_15  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_100
+	UINT32 m_p_contrast_curve2_16  : 16;
+	UINT32 m_p_contrast_curve3_16  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_101
+	UINT32 m_p_contrast_curve2_17  : 16;
+	UINT32 m_p_contrast_curve3_17  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_102
+	UINT32 m_p_contrast_curve2_18  : 16;
+	UINT32 m_p_contrast_curve3_18  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_103
+	UINT32 m_p_contrast_curve2_19  : 16;
+	UINT32 m_p_contrast_curve3_19  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_104
+	UINT32 m_p_contrast_curve2_20  : 16;
+	UINT32 m_p_contrast_curve3_20  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_105
+	UINT32 m_p_contrast_curve2_21  : 16;
+	UINT32 m_p_contrast_curve3_21  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_106
+	UINT32 m_p_contrast_curve2_22  : 16;
+	UINT32 m_p_contrast_curve3_22  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_107
+	UINT32 m_p_contrast_curve2_23  : 16;
+	UINT32 m_p_contrast_curve3_23  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_108
+	UINT32 m_p_contrast_curve2_24  : 16;
+	UINT32 m_p_contrast_curve3_24  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_109
+	UINT32 m_p_contrast_curve2_25  : 16;
+	UINT32 m_p_contrast_curve3_25  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_110
+	UINT32 m_p_contrast_curve2_26  : 16;
+	UINT32 m_p_contrast_curve3_26  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_111
+	UINT32 m_p_contrast_curve2_27  : 16;
+	UINT32 m_p_contrast_curve3_27  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_112
+	UINT32 m_p_contrast_curve2_28  : 16;
+	UINT32 m_p_contrast_curve3_28  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_113
+	UINT32 m_p_contrast_curve2_29  : 16;
+	UINT32 m_p_contrast_curve3_29  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_114
+	UINT32 m_p_contrast_curve2_30  : 16;
+	UINT32 m_p_contrast_curve3_30  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_115
+	UINT32 m_p_contrast_curve2_31  : 16;
+	UINT32 m_p_contrast_curve3_31  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_116
+	UINT32 m_p_contrast_curve2_32  : 16;
+	UINT32 m_p_contrast_curve3_32  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_117
+	UINT32 m_p_contrast_curve2_33  : 16;
+	UINT32 m_p_contrast_curve3_33  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_118
+	UINT32 m_p_contrast_curve2_34  : 16;
+	UINT32 m_p_contrast_curve3_34  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_119
+	UINT32 m_p_contrast_curve2_35  : 16;
+	UINT32 m_p_contrast_curve3_35  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_120
+	UINT32 m_p_contrast_curve2_36  : 16;
+	UINT32 m_p_contrast_curve3_36  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_121
+	UINT32 m_p_contrast_curve2_37  : 16;
+	UINT32 m_p_contrast_curve3_37  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_122
+	UINT32 m_p_contrast_curve2_38  : 16;
+	UINT32 m_p_contrast_curve3_38  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_123
+	UINT32 m_p_contrast_curve2_39  : 16;
+	UINT32 m_p_contrast_curve3_39  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_124
+	UINT32 m_p_contrast_curve2_40  : 16;
+	UINT32 m_p_contrast_curve3_40  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_125
+	UINT32 m_p_contrast_curve2_41  : 16;
+	UINT32 m_p_contrast_curve3_41  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_126
+	UINT32 m_p_contrast_curve2_42  : 16;
+	UINT32 m_p_contrast_curve3_42  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_127
+	UINT32 m_p_contrast_curve2_43  : 16;
+	UINT32 m_p_contrast_curve3_43  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_128
+	UINT32 m_p_contrast_curve2_44  : 16;
+	UINT32 m_p_contrast_curve3_44  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_129
+	UINT32 m_p_contrast_curve2_45  : 16;
+	UINT32 m_p_contrast_curve3_45  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_130
+	UINT32 m_p_contrast_curve2_46  : 16;
+	UINT32 m_p_contrast_curve3_46  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_131
+	UINT32 m_p_contrast_curve2_47  : 16;
+	UINT32 m_p_contrast_curve3_47  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_132
+	UINT32 m_p_contrast_curve2_48  : 16;
+	UINT32 m_p_contrast_curve3_48  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_133
+	UINT32 m_p_contrast_curve2_49  : 16;
+	UINT32 m_p_contrast_curve3_49  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_134
+	UINT32 m_p_contrast_curve2_50  : 16;
+	UINT32 m_p_contrast_curve3_50  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_135
+	UINT32 m_p_contrast_curve2_51  : 16;
+	UINT32 m_p_contrast_curve3_51  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_136
+	UINT32 m_p_contrast_curve2_52  : 16;
+	UINT32 m_p_contrast_curve3_52  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_137
+	UINT32 m_p_contrast_curve2_53  : 16;
+	UINT32 m_p_contrast_curve3_53  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_138
+	UINT32 m_p_contrast_curve2_54  : 16;
+	UINT32 m_p_contrast_curve3_54  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_139
+	UINT32 m_p_contrast_curve2_55  : 16;
+	UINT32 m_p_contrast_curve3_55  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_140
+	UINT32 m_p_contrast_curve2_56  : 16;
+	UINT32 m_p_contrast_curve3_56  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_141
+	UINT32 m_p_contrast_curve2_57  : 16;
+	UINT32 m_p_contrast_curve3_57  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_142
+	UINT32 m_p_contrast_curve2_58  : 16;
+	UINT32 m_p_contrast_curve3_58  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_143
+	UINT32 m_p_contrast_curve2_59  : 16;
+	UINT32 m_p_contrast_curve3_59  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_144
+	UINT32 m_p_contrast_curve2_60  : 16;
+	UINT32 m_p_contrast_curve3_60  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_145
+	UINT32 m_p_contrast_curve2_61  : 16;
+	UINT32 m_p_contrast_curve3_61  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_146
+	UINT32 m_p_contrast_curve2_62  : 16;
+	UINT32 m_p_contrast_curve3_62  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_147
+	UINT32 m_p_contrast_curve2_63  : 16;
+	UINT32 m_p_contrast_curve3_63  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_148
+	UINT32 m_p_contrast_curve2_64  : 16;
+	UINT32 m_p_contrast_curve3_64  : 16;
+
+	//REGISTER saturn_acad_with_curve_reg_149
+	UINT32 force_update_en         : 1;
+	UINT32 vsync_update_en         : 1;
+	UINT32 shadow_read_en          : 1;
+	UINT32:29;
+
+	//REGISTER saturn_acad_with_curve_reg_150
+	UINT32 force_update_pulse      : 1;
+	UINT32:31;
+
+	//REGISTER saturn_acad_with_curve_reg_151
+	UINT32 icg_override            : 1;
+	UINT32:31;
+
+	//REGISTER saturn_acad_with_curve_reg_152
+	UINT32 trigger                 : 1;
+	UINT32:31;
+
+	//REGISTER saturn_acad_with_curve_reg_153
+	UINT32 trigger2                : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[154];
+
+} ACAD_REG;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/cmdlist_top.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/cmdlist_top.h
new file mode 100644
index 000000000000..f7bcb467e065
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/cmdlist_top.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef CMDLIST_TOP_REG_H
+#define CMDLIST_TOP_REG_H
+
+typedef union {
+	struct {
+	//REGISTER cmdlist_reg_0
+	struct {
+	UINT32:4;
+	UINT32 cmdlist_ch_start_addrl          : 28;
+	} cmdlist_reg_0[13];
+
+	//REGISTER cmdlist_reg_13
+	struct {
+	UINT32:32;
+	} cmdlist_reg_13[3];
+
+	//REGISTER cmdlist_reg_16
+	struct {
+	UINT32 cmdlist_ch_start_addrh          : 6;
+	UINT32:26;
+	} cmdlist_reg_16[13];
+
+	//REGISTER cmdlist_reg_29
+	struct {
+	UINT32:32;
+	} cmdlist_reg_29[3];
+
+	//REGISTER cmdlist_reg_32
+	struct {
+	UINT32 cmdlist_ch_secu_en              : 1;
+	UINT32 cmdlist_ch_user_id              : 4;
+	UINT32:27;
+	} cmdlist_reg_32[13];
+
+	//REGISTER cmdlist_reg_45
+	UINT32 secu_cfg_icg_override           : 1;
+	UINT32:31;
+
+	//REGISTER cmdlist_reg_46
+	struct {
+	UINT32:32;
+	} cmdlist_reg_46[2];
+
+	//REGISTER cmdlist_reg_48
+	struct {
+	UINT32 cmdlist_ch_arcache              : 4;
+	UINT32 cmdlist_ch_arqos                : 4;
+	UINT32 cmdlist_ch_arregion             : 4;
+	UINT32 cmdlist_ch_y_first              : 13;
+	UINT32 cmdlist_ch_osd_dep              : 4;
+	UINT32:3;
+	} cmdlist_reg_48[13];
+
+	//REGISTER cmdlist_reg_61
+	struct {
+	UINT32:32;
+	} cmdlist_reg_61[3];
+
+	//REGISTER cmdlist_reg_64
+	struct {
+	UINT32:4;
+	UINT32 cmdlist_ch_cont_addrl           : 28;
+	} cmdlist_reg_64[13];
+
+	//REGISTER cmdlist_reg_77
+	struct {
+	UINT32:32;
+	} cmdlist_reg_77[3];
+
+	//REGISTER cmdlist_reg_80
+	struct {
+	UINT32 cmdlist_ch_cont_addrh           : 6;
+	UINT32:26;
+	} cmdlist_reg_80[13];
+
+	//REGISTER cmdlist_reg_93
+	struct {
+	UINT32:32;
+	} cmdlist_reg_93[3];
+
+	//REGISTER cmdlist_reg_96
+	UINT32 cmdlist_burst_len               : 5;
+	UINT32 axi_port_sel                    : 2;
+	UINT32 onl_arb_ratio                   : 3;
+	UINT32 reorder_burst_split_en          : 1;
+	UINT32 reorder_burst_split_mode        : 1;
+	UINT32 reorder_osd_num                 : 6;
+	UINT32:14;
+
+	//REGISTER cmdlist_reg_97
+	UINT32:32;
+
+	//REGISTER cmdlist_reg_98
+	struct {
+	UINT32 cmdlist_ch_y_other              : 13;
+	UINT32:19;
+	} cmdlist_reg_98[13];
+
+	//REGISTER cmdlist_reg_111
+	struct {
+	UINT32:32;
+	} cmdlist_reg_111[3];
+
+	//REGISTER cmdlist_reg_114
+	UINT32:16;
+	UINT32 cmdlist_clr_timeout_th          : 16;
+
+	//REGISTER cmdlist_reg_115
+	UINT32 cmdlist_pslverr_int_msk         : 1;
+	UINT32:15;
+	UINT32 cmdlist_ch_clr_timeout_int_msk  : 8;
+	UINT32:8;
+
+	//REGISTER cmdlist_reg_116
+	UINT32 cmdlist_pslverr_ints            : 1;
+	UINT32:15;
+	UINT32 cmdlist_ch_clr_timeout_ints     : 13;
+	UINT32:3;
+
+	//REGISTER cmdlist_reg_117
+	UINT32 cmdlist_pslverr_int_raw         : 1;
+	UINT32:15;
+	UINT32 cmdlist_ch_clr_timeout_int_raw  : 13;
+	UINT32:3;
+
+	//REGISTER cmdlist_reg_118
+	struct {
+	UINT32 cmdlist_ch_dbg0                 : 11;
+	UINT32:5;
+	UINT32 cmdlist_ch_dbg1                 : 11;
+	UINT32:5;
+	} cmdlist_reg_118[8];
+
+	//REGISTER cmdlist_reg_126
+	UINT32 cmdlist_dbg                     : 6;
+	UINT32:26;
+
+	//REGISTER cmdlist_reg_127
+	UINT32 mem_lp_auto_en                  : 2;
+	UINT32:30;
+
+	};
+
+	INT32 value32[128];
+
+} CMDLIST_TOP_REG;
+
+#endif
+
+typedef CMDLIST_TOP_REG CMDLIST_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/composer_x.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/composer_x.h
new file mode 100644
index 000000000000..68a70058973c
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/composer_x.h
@@ -0,0 +1,783 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef COMPOSER_X_REG_H
+#define COMPOSER_X_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_cmps_reg_0
+	UINT32 module_enable            : 1;
+	UINT32:31;
+
+	//REGISTER saturn_cmps_reg_1
+	UINT32 dst_w                    : 16;
+	UINT32 dst_h                    : 16;
+
+	//REGISTER saturn_cmps_reg_2
+	UINT32 bg_color_r               : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_3
+	UINT32 bg_color_g               : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_4
+	UINT32 bg_color_b               : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_5
+	UINT32 bg_color_a               : 8;
+	UINT32:24;
+
+	//REGISTER saturn_cmps_reg_6
+	struct {
+	UINT32:32;
+	} saturn_cmps_reg_6[2];
+
+	//REGISTER saturn_cmps_reg_8
+	UINT32 layer00_area_left        : 16;
+	UINT32 layer00_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_9
+	UINT32 layer00_area_top         : 16;
+	UINT32 layer00_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_10
+	UINT32 layer00_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_11
+	UINT32 layer00_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_12
+	UINT32 layer00_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_13
+	UINT32 layer00_solid_color_a    : 8;
+	UINT32 layer00_rsv00            : 1;
+	UINT32 layer00_rsv01            : 1;
+	UINT32 layer00_rsv02            : 1;
+	UINT32 layer00_rsv03            : 1;
+	UINT32 layer00_rsv04            : 1;
+	UINT32 layer00_rsv05            : 1;
+	UINT32 layer00_blend_mode       : 2;
+	UINT32 layer00_solid_area_mode  : 1;
+	UINT32 layer00_spe_alpha_mode   : 1;
+	UINT32 layer00_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_14
+	UINT32 layer00_en               : 1;
+	UINT32 layer00_blend_sel        : 2;
+	UINT32 layer00_solid_en         : 1;
+	UINT32 layer00_dma_id           : 4;
+	UINT32 layer00_layer_alpha      : 8;
+	UINT32 layer00_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_15
+	UINT32 layer01_area_left        : 16;
+	UINT32 layer01_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_16
+	UINT32 layer01_area_top         : 16;
+	UINT32 layer01_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_17
+	UINT32 layer01_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_18
+	UINT32 layer01_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_19
+	UINT32 layer01_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_20
+	UINT32 layer01_solid_color_a    : 8;
+	UINT32 layer01_rsv00            : 1;
+	UINT32 layer01_rsv01            : 1;
+	UINT32 layer01_rsv02            : 1;
+	UINT32 layer01_rsv03            : 1;
+	UINT32 layer01_rsv04            : 1;
+	UINT32 layer01_rsv05            : 1;
+	UINT32 layer01_blend_mode       : 2;
+	UINT32 layer01_solid_area_mode  : 1;
+	UINT32 layer01_spe_alpha_mode   : 1;
+	UINT32 layer01_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_21
+	UINT32 layer01_en               : 1;
+	UINT32 layer01_blend_sel        : 2;
+	UINT32 layer01_solid_en         : 1;
+	UINT32 layer01_dma_id           : 4;
+	UINT32 layer01_layer_alpha      : 8;
+	UINT32 layer01_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_22
+	UINT32 layer02_area_left        : 16;
+	UINT32 layer02_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_23
+	UINT32 layer02_area_top         : 16;
+	UINT32 layer02_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_24
+	UINT32 layer02_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_25
+	UINT32 layer02_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_26
+	UINT32 layer02_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_27
+	UINT32 layer02_solid_color_a    : 8;
+	UINT32 layer02_rsv00            : 1;
+	UINT32 layer02_rsv01            : 1;
+	UINT32 layer02_rsv02            : 1;
+	UINT32 layer02_rsv03            : 1;
+	UINT32 layer02_rsv04            : 1;
+	UINT32 layer02_rsv05            : 1;
+	UINT32 layer02_blend_mode       : 2;
+	UINT32 layer02_solid_area_mode  : 1;
+	UINT32 layer02_spe_alpha_mode   : 1;
+	UINT32 layer02_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_28
+	UINT32 layer02_en               : 1;
+	UINT32 layer02_blend_sel        : 2;
+	UINT32 layer02_solid_en         : 1;
+	UINT32 layer02_dma_id           : 4;
+	UINT32 layer02_layer_alpha      : 8;
+	UINT32 layer02_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_29
+	UINT32 layer03_area_left        : 16;
+	UINT32 layer03_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_30
+	UINT32 layer03_area_top         : 16;
+	UINT32 layer03_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_31
+	UINT32 layer03_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_32
+	UINT32 layer03_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_33
+	UINT32 layer03_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_34
+	UINT32 layer03_solid_color_a    : 8;
+	UINT32 layer03_rsv00            : 1;
+	UINT32 layer03_rsv01            : 1;
+	UINT32 layer03_rsv02            : 1;
+	UINT32 layer03_rsv03            : 1;
+	UINT32 layer03_rsv04            : 1;
+	UINT32 layer03_rsv05            : 1;
+	UINT32 layer03_blend_mode       : 2;
+	UINT32 layer03_solid_area_mode  : 1;
+	UINT32 layer03_spe_alpha_mode   : 1;
+	UINT32 layer03_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_35
+	UINT32 layer03_en               : 1;
+	UINT32 layer03_blend_sel        : 2;
+	UINT32 layer03_solid_en         : 1;
+	UINT32 layer03_dma_id           : 4;
+	UINT32 layer03_layer_alpha      : 8;
+	UINT32 layer03_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_36
+	UINT32 layer04_area_left        : 16;
+	UINT32 layer04_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_37
+	UINT32 layer04_area_top         : 16;
+	UINT32 layer04_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_38
+	UINT32 layer04_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_39
+	UINT32 layer04_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_40
+	UINT32 layer04_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_41
+	UINT32 layer04_solid_color_a    : 8;
+	UINT32 layer04_rsv00            : 1;
+	UINT32 layer04_rsv01            : 1;
+	UINT32 layer04_rsv02            : 1;
+	UINT32 layer04_rsv03            : 1;
+	UINT32 layer04_rsv04            : 1;
+	UINT32 layer04_rsv05            : 1;
+	UINT32 layer04_blend_mode       : 2;
+	UINT32 layer04_solid_area_mode  : 1;
+	UINT32 layer04_spe_alpha_mode   : 1;
+	UINT32 layer04_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_42
+	UINT32 layer04_en               : 1;
+	UINT32 layer04_blend_sel        : 2;
+	UINT32 layer04_solid_en         : 1;
+	UINT32 layer04_dma_id           : 4;
+	UINT32 layer04_layer_alpha      : 8;
+	UINT32 layer04_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_43
+	UINT32 layer05_area_left        : 16;
+	UINT32 layer05_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_44
+	UINT32 layer05_area_top         : 16;
+	UINT32 layer05_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_45
+	UINT32 layer05_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_46
+	UINT32 layer05_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_47
+	UINT32 layer05_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_48
+	UINT32 layer05_solid_color_a    : 8;
+	UINT32 layer05_rsv00            : 1;
+	UINT32 layer05_rsv01            : 1;
+	UINT32 layer05_rsv02            : 1;
+	UINT32 layer05_rsv03            : 1;
+	UINT32 layer05_rsv04            : 1;
+	UINT32 layer05_rsv05            : 1;
+	UINT32 layer05_blend_mode       : 2;
+	UINT32 layer05_solid_area_mode  : 1;
+	UINT32 layer05_spe_alpha_mode   : 1;
+	UINT32 layer05_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_49
+	UINT32 layer05_en               : 1;
+	UINT32 layer05_blend_sel        : 2;
+	UINT32 layer05_solid_en         : 1;
+	UINT32 layer05_dma_id           : 4;
+	UINT32 layer05_layer_alpha      : 8;
+	UINT32 layer05_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_50
+	UINT32 layer06_area_left        : 16;
+	UINT32 layer06_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_51
+	UINT32 layer06_area_top         : 16;
+	UINT32 layer06_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_52
+	UINT32 layer06_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_53
+	UINT32 layer06_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_54
+	UINT32 layer06_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_55
+	UINT32 layer06_solid_color_a    : 8;
+	UINT32 layer06_rsv00            : 1;
+	UINT32 layer06_rsv01            : 1;
+	UINT32 layer06_rsv02            : 1;
+	UINT32 layer06_rsv03            : 1;
+	UINT32 layer06_rsv04            : 1;
+	UINT32 layer06_rsv05            : 1;
+	UINT32 layer06_blend_mode       : 2;
+	UINT32 layer06_solid_area_mode  : 1;
+	UINT32 layer06_spe_alpha_mode   : 1;
+	UINT32 layer06_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_56
+	UINT32 layer06_en               : 1;
+	UINT32 layer06_blend_sel        : 2;
+	UINT32 layer06_solid_en         : 1;
+	UINT32 layer06_dma_id           : 4;
+	UINT32 layer06_layer_alpha      : 8;
+	UINT32 layer06_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_57
+	UINT32 layer07_area_left        : 16;
+	UINT32 layer07_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_58
+	UINT32 layer07_area_top         : 16;
+	UINT32 layer07_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_59
+	UINT32 layer07_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_60
+	UINT32 layer07_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_61
+	UINT32 layer07_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_62
+	UINT32 layer07_solid_color_a    : 8;
+	UINT32 layer07_rsv00            : 1;
+	UINT32 layer07_rsv01            : 1;
+	UINT32 layer07_rsv02            : 1;
+	UINT32 layer07_rsv03            : 1;
+	UINT32 layer07_rsv04            : 1;
+	UINT32 layer07_rsv05            : 1;
+	UINT32 layer07_blend_mode       : 2;
+	UINT32 layer07_solid_area_mode  : 1;
+	UINT32 layer07_spe_alpha_mode   : 1;
+	UINT32 layer07_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_63
+	UINT32 layer07_en               : 1;
+	UINT32 layer07_blend_sel        : 2;
+	UINT32 layer07_solid_en         : 1;
+	UINT32 layer07_dma_id           : 4;
+	UINT32 layer07_layer_alpha      : 8;
+	UINT32 layer07_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_64
+	UINT32 layer08_area_left        : 16;
+	UINT32 layer08_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_65
+	UINT32 layer08_area_top         : 16;
+	UINT32 layer08_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_66
+	UINT32 layer08_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_67
+	UINT32 layer08_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_68
+	UINT32 layer08_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_69
+	UINT32 layer08_solid_color_a    : 8;
+	UINT32 layer08_rsv00            : 1;
+	UINT32 layer08_rsv01            : 1;
+	UINT32 layer08_rsv02            : 1;
+	UINT32 layer08_rsv03            : 1;
+	UINT32 layer08_rsv04            : 1;
+	UINT32 layer08_rsv05            : 1;
+	UINT32 layer08_blend_mode       : 2;
+	UINT32 layer08_solid_area_mode  : 1;
+	UINT32 layer08_spe_alpha_mode   : 1;
+	UINT32 layer08_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_70
+	UINT32 layer08_en               : 1;
+	UINT32 layer08_blend_sel        : 2;
+	UINT32 layer08_solid_en         : 1;
+	UINT32 layer08_dma_id           : 4;
+	UINT32 layer08_layer_alpha      : 8;
+	UINT32 layer08_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_71
+	UINT32 layer09_area_left        : 16;
+	UINT32 layer09_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_72
+	UINT32 layer09_area_top         : 16;
+	UINT32 layer09_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_73
+	UINT32 layer09_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_74
+	UINT32 layer09_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_75
+	UINT32 layer09_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_76
+	UINT32 layer09_solid_color_a    : 8;
+	UINT32 layer09_rsv00            : 1;
+	UINT32 layer09_rsv01            : 1;
+	UINT32 layer09_rsv02            : 1;
+	UINT32 layer09_rsv03            : 1;
+	UINT32 layer09_rsv04            : 1;
+	UINT32 layer09_rsv05            : 1;
+	UINT32 layer09_blend_mode       : 2;
+	UINT32 layer09_solid_area_mode  : 1;
+	UINT32 layer09_spe_alpha_mode   : 1;
+	UINT32 layer09_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_77
+	UINT32 layer09_en               : 1;
+	UINT32 layer09_blend_sel        : 2;
+	UINT32 layer09_solid_en         : 1;
+	UINT32 layer09_dma_id           : 4;
+	UINT32 layer09_layer_alpha      : 8;
+	UINT32 layer09_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_78
+	UINT32 layer10_area_left        : 16;
+	UINT32 layer10_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_79
+	UINT32 layer10_area_top         : 16;
+	UINT32 layer10_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_80
+	UINT32 layer10_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_81
+	UINT32 layer10_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_82
+	UINT32 layer10_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_83
+	UINT32 layer10_solid_color_a    : 8;
+	UINT32 layer10_rsv00            : 1;
+	UINT32 layer10_rsv01            : 1;
+	UINT32 layer10_rsv02            : 1;
+	UINT32 layer10_rsv03            : 1;
+	UINT32 layer10_rsv04            : 1;
+	UINT32 layer10_rsv05            : 1;
+	UINT32 layer10_blend_mode       : 2;
+	UINT32 layer10_solid_area_mode  : 1;
+	UINT32 layer10_spe_alpha_mode   : 1;
+	UINT32 layer10_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_84
+	UINT32 layer10_en               : 1;
+	UINT32 layer10_blend_sel        : 2;
+	UINT32 layer10_solid_en         : 1;
+	UINT32 layer10_dma_id           : 4;
+	UINT32 layer10_layer_alpha      : 8;
+	UINT32 layer10_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_85
+	UINT32 layer11_area_left        : 16;
+	UINT32 layer11_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_86
+	UINT32 layer11_area_top         : 16;
+	UINT32 layer11_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_87
+	UINT32 layer11_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_88
+	UINT32 layer11_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_89
+	UINT32 layer11_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_90
+	UINT32 layer11_solid_color_a    : 8;
+	UINT32 layer11_rsv00            : 1;
+	UINT32 layer11_rsv01            : 1;
+	UINT32 layer11_rsv02            : 1;
+	UINT32 layer11_rsv03            : 1;
+	UINT32 layer11_rsv04            : 1;
+	UINT32 layer11_rsv05            : 1;
+	UINT32 layer11_blend_mode       : 2;
+	UINT32 layer11_solid_area_mode  : 1;
+	UINT32 layer11_spe_alpha_mode   : 1;
+	UINT32 layer11_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_91
+	UINT32 layer11_en               : 1;
+	UINT32 layer11_blend_sel        : 2;
+	UINT32 layer11_solid_en         : 1;
+	UINT32 layer11_dma_id           : 4;
+	UINT32 layer11_layer_alpha      : 8;
+	UINT32 layer11_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_92
+	UINT32 layer12_area_left        : 16;
+	UINT32 layer12_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_93
+	UINT32 layer12_area_top         : 16;
+	UINT32 layer12_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_94
+	UINT32 layer12_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_95
+	UINT32 layer12_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_96
+	UINT32 layer12_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_97
+	UINT32 layer12_solid_color_a    : 8;
+	UINT32 layer12_rsv00            : 1;
+	UINT32 layer12_rsv01            : 1;
+	UINT32 layer12_rsv02            : 1;
+	UINT32 layer12_rsv03            : 1;
+	UINT32 layer12_rsv04            : 1;
+	UINT32 layer12_rsv05            : 1;
+	UINT32 layer12_blend_mode       : 2;
+	UINT32 layer12_solid_area_mode  : 1;
+	UINT32 layer12_spe_alpha_mode   : 1;
+	UINT32 layer12_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_98
+	UINT32 layer12_en               : 1;
+	UINT32 layer12_blend_sel        : 2;
+	UINT32 layer12_solid_en         : 1;
+	UINT32 layer12_dma_id           : 4;
+	UINT32 layer12_layer_alpha      : 8;
+	UINT32 layer12_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_99
+	UINT32 layer13_area_left        : 16;
+	UINT32 layer13_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_100
+	UINT32 layer13_area_top         : 16;
+	UINT32 layer13_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_101
+	UINT32 layer13_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_102
+	UINT32 layer13_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_103
+	UINT32 layer13_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_104
+	UINT32 layer13_solid_color_a    : 8;
+	UINT32 layer13_rsv00            : 1;
+	UINT32 layer13_rsv01            : 1;
+	UINT32 layer13_rsv02            : 1;
+	UINT32 layer13_rsv03            : 1;
+	UINT32 layer13_rsv04            : 1;
+	UINT32 layer13_rsv05            : 1;
+	UINT32 layer13_blend_mode       : 2;
+	UINT32 layer13_solid_area_mode  : 1;
+	UINT32 layer13_spe_alpha_mode   : 1;
+	UINT32 layer13_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_105
+	UINT32 layer13_en               : 1;
+	UINT32 layer13_blend_sel        : 2;
+	UINT32 layer13_solid_en         : 1;
+	UINT32 layer13_dma_id           : 4;
+	UINT32 layer13_layer_alpha      : 8;
+	UINT32 layer13_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_106
+	UINT32 layer14_area_left        : 16;
+	UINT32 layer14_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_107
+	UINT32 layer14_area_top         : 16;
+	UINT32 layer14_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_108
+	UINT32 layer14_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_109
+	UINT32 layer14_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_110
+	UINT32 layer14_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_111
+	UINT32 layer14_solid_color_a    : 8;
+	UINT32 layer14_rsv00            : 1;
+	UINT32 layer14_rsv01            : 1;
+	UINT32 layer14_rsv02            : 1;
+	UINT32 layer14_rsv03            : 1;
+	UINT32 layer14_rsv04            : 1;
+	UINT32 layer14_rsv05            : 1;
+	UINT32 layer14_blend_mode       : 2;
+	UINT32 layer14_solid_area_mode  : 1;
+	UINT32 layer14_spe_alpha_mode   : 1;
+	UINT32 layer14_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_112
+	UINT32 layer14_en               : 1;
+	UINT32 layer14_blend_sel        : 2;
+	UINT32 layer14_solid_en         : 1;
+	UINT32 layer14_dma_id           : 4;
+	UINT32 layer14_layer_alpha      : 8;
+	UINT32 layer14_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_113
+	UINT32 layer15_area_left        : 16;
+	UINT32 layer15_area_right       : 16;
+
+	//REGISTER saturn_cmps_reg_114
+	UINT32 layer15_area_top         : 16;
+	UINT32 layer15_area_bottom      : 16;
+
+	//REGISTER saturn_cmps_reg_115
+	UINT32 layer15_solid_color_r    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_116
+	UINT32 layer15_solid_color_g    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_117
+	UINT32 layer15_solid_color_b    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_cmps_reg_118
+	UINT32 layer15_solid_color_a    : 8;
+	UINT32 layer15_rsv00            : 1;
+	UINT32 layer15_rsv01            : 1;
+	UINT32 layer15_rsv02            : 1;
+	UINT32 layer15_rsv03            : 1;
+	UINT32 layer15_rsv04            : 1;
+	UINT32 layer15_rsv05            : 1;
+	UINT32 layer15_blend_mode       : 2;
+	UINT32 layer15_solid_area_mode  : 1;
+	UINT32 layer15_spe_alpha_mode   : 1;
+	UINT32 layer15_colorkey_en      : 1;
+	UINT32:13;
+
+	//REGISTER saturn_cmps_reg_119
+	UINT32 layer15_en               : 1;
+	UINT32 layer15_blend_sel        : 2;
+	UINT32 layer15_solid_en         : 1;
+	UINT32 layer15_dma_id           : 4;
+	UINT32 layer15_layer_alpha      : 8;
+	UINT32 layer15_alpha_ratio      : 8;
+	UINT32:8;
+
+	//REGISTER saturn_cmps_reg_120
+	UINT32 layer_data_not_ready     : 32;
+
+	//REGISTER saturn_cmps_reg_121
+	struct {
+	UINT32 layer_proc_location      : 32;
+	} saturn_cmps_reg_121[16];
+
+	//REGISTER saturn_cmps_reg_137
+	UINT32 cmps_proc_location       : 32;
+
+	//REGISTER saturn_cmps_reg_138
+	UINT32 cfg_se                   : 1;
+	UINT32:31;
+
+	//REGISTER saturn_cmps_reg_139
+	UINT32 force_update_en          : 1;
+	UINT32 vsync_update_en          : 1;
+	UINT32 shadow_read_en           : 1;
+	UINT32:29;
+
+	//REGISTER saturn_cmps_reg_140
+	UINT32 force_update_pulse       : 1;
+	UINT32:31;
+
+	//REGISTER saturn_cmps_reg_141
+	UINT32 force_update_en_se       : 1;
+	UINT32 vsync_update_en_se       : 1;
+	UINT32 shadow_read_en_se        : 1;
+	UINT32:29;
+
+	//REGISTER saturn_cmps_reg_142
+	UINT32 force_update_pulse_se    : 1;
+	UINT32:31;
+
+	//REGISTER saturn_cmps_reg_143
+	UINT32 icg_override             : 1;
+	UINT32:31;
+
+	//REGISTER saturn_cmps_reg_144
+	UINT32 trigger                  : 1;
+	UINT32:31;
+
+	//REGISTER saturn_cmps_reg_145
+	UINT32 trigger2                 : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[146];
+
+} COMPOSER_X_REG;
+
+typedef COMPOSER_X_REG CMPS_X_REG;
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_ctl_top.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_ctl_top.h
new file mode 100644
index 000000000000..97059db615c5
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_ctl_top.h
@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef DPU_CTL_TOP_REG_H
+#define DPU_CTL_TOP_REG_H
+
+typedef union {
+	struct {
+	//REGISTER dpu_ctl_top_reg_0
+	struct {
+	UINT32 cmdlist_rch_en                    : 1;
+	UINT32:31;
+	} dpu_ctl_top_reg_0[10];
+
+	//REGISTER dpu_ctl_top_reg_10
+	struct {
+	UINT32 cmdlist_cmps_other_en             : 1;
+	UINT32 cmdlist_cmps_top_en               : 1;
+	UINT32:30;
+	} dpu_ctl_top_reg_10[3];
+
+	//REGISTER dpu_ctl_top_reg_13
+	UINT32 cmdlist_rdma_cfg_rdy              : 10;
+	UINT32 cmdlist_prepq_cfg_rdy             : 10;
+	UINT32 cmdlist_cmps_cfg_rdy              : 3;
+	UINT32 cmdlist_wb_cfg_rdy                : 2;
+	UINT32:7;
+
+	//REGISTER dpu_ctl_top_reg_14
+	UINT32 wb0_sel_id                        : 5;
+	UINT32:27;
+
+	//REGISTER dpu_ctl_top_reg_15
+	UINT32 wb1_sel_id                        : 5;
+	UINT32:27;
+
+	//REGISTER dpu_ctl_top_reg_16
+	UINT32 reuse_rdma_act                    : 10;
+	UINT32 reuse_prepq_act                   : 10;
+	UINT32 reuse_scl_act0                    : 2;
+	UINT32 reuse_cmps_act                    : 3;
+	UINT32 reuse_wb_act                      : 2;
+	UINT32 reuse_scl_act1                    : 2;
+	UINT32:3;
+
+	//REGISTER dpu_ctl_top_reg_17
+	UINT32 rch_conflict_ints                 : 10;
+	UINT32 acad_timeout_ints                 : 1;
+	UINT32 wb_timeout_ints                   : 2;
+	UINT32 cmdlist_rdma_cfg_timeout_ints     : 10;
+	UINT32:9;
+
+	//REGISTER dpu_ctl_top_reg_18
+	UINT32 rch_conflict_int_en               : 10;
+	UINT32 acad_timeout_int_en               : 1;
+	UINT32 wb_timeout_int_en                 : 2;
+	UINT32 cmdlist_rdma_cfg_timeout_int_en   : 10;
+	UINT32:9;
+
+	//REGISTER dpu_ctl_top_reg_19
+	UINT32 rch_conflict_int_raw              : 10;
+	UINT32 acad_timeout_int_raw              : 1;
+	UINT32 wb_timeout_int_raw                : 2;
+	UINT32 cmdlist_rdma_cfg_timeout_int_raw  : 10;
+	UINT32:9;
+
+	//REGISTER dpu_ctl_top_reg_20
+	UINT32 wb_sel_secu                       : 2;
+	UINT32 secu_cfg_icg_override             : 1;
+	UINT32 scl_sel_secu                      : 2;
+	UINT32:27;
+
+	//REGISTER dpu_ctl_top_reg_21
+	UINT32 pslverr_addr                      : 8;
+	UINT32:24;
+
+	//REGISTER dpu_ctl_top_reg_22
+	struct {
+	UINT32 cmdlist_ch_sw_event               : 4;
+	UINT32:28;
+	} dpu_ctl_top_reg_22[13];
+
+	//REGISTER dpu_ctl_top_reg_35
+	UINT32 ctl_rd_shadow                     : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_top_reg_36
+	struct {
+	UINT32 rch_start_cmps_y                  : 16;
+	UINT32:16;
+	} dpu_ctl_top_reg_36[10];
+
+	//REGISTER dpu_ctl_top_reg_46
+	UINT32 wb0_slice_cnt                     : 5;
+	UINT32 wb1_slice_cnt                     : 5;
+	UINT32 wb_busy                           : 2;
+	UINT32 acad_busy                         : 2;
+	UINT32 dscw_busy                         : 2;
+	UINT32:16;
+
+	//REGISTER dpu_ctl_top_reg_47
+	UINT32:32;
+
+	//REGISTER dpu_ctl_top_reg_48
+	UINT32:32;
+
+	//REGISTER dpu_ctl_top_reg_49
+	UINT32 nml_scl0_reuse_en                 : 1;
+	UINT32:31;
+
+
+	//REGISTER dpu_ctl_top_reg_50
+	UINT32 nml_scl1_reuse_en                 : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_top_reg_51
+	UINT32 nml_scl2_reuse_en                 : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_top_reg_52
+	UINT32 nml_scl3_reuse_en                 : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[53];
+
+} DPU_CTL_TOP_REG;
+
+#endif
+
+typedef DPU_CTL_TOP_REG DPU_CTL_TOP_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_int.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_int.h
new file mode 100644
index 000000000000..16d983731827
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_int.h
@@ -0,0 +1,389 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef DPU_INT_REG_H
+#define DPU_INT_REG_H
+
+#define DPU_INT_CMDLIST_CH_FRM_CFG_DONE_MASK	(0xff)
+#define DPU_INT_CMDLIST_CH_FRM_CFG_DONE		(DPU_INT_CMDLIST_CH_FRM_CFG_DONE_MASK << 11)
+#define DPU_INT_WB_OVFLOW			BIT(9)
+#define DPU_INT_FRM_TIMING_UNFLOW		BIT(8)
+#define DPU_INT_WB_DONE				(BIT(6) | BIT(24))
+#define DPU_INT_CFG_RDY_CLR			BIT(5)
+#define DPU_INT_FRM_TIMING_CFG_LINE		BIT(4)
+#define DPU_INT_FRM_TIMING_CFG_EOF		(BIT(2) | BIT(3))
+#define DPU_INT_FRM_TIMING_EOF			BIT(1)
+#define DPU_INT_FRM_TIMING_VSYNC		BIT(0)
+#define DPU_INT_RDMA_DEBUG	(BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5))
+#define DPU_REST_INT_BITS			(DPU_INT_FRM_TIMING_CFG_EOF | \
+						 DPU_INT_FRM_TIMING_CFG_LINE | \
+						 DPU_INT_FRM_TIMING_EOF | \
+						 DPU_INT_CMDLIST_CH_FRM_CFG_DONE)
+/* Offline */
+#define DPU_INT_OFF_CFG_RDY_CLR			BIT(0)
+#define DPU_INT_OFF_WB_DONE			BIT(1)
+#define DPU_INT_OFF_SLICE_DONE			BIT(3)
+#define DPU_INT_OFF_CL_CH_DONE_MASK		(0xff)
+#define DPU_INT_OFF_CMDLIST_CH_DONE		(DPU_INT_OFF_CL_CH_DONE_MASK << 5)
+#define DPU_OFF_REST_INT_BITS			(DPU_INT_OFF_CMDLIST_CH_DONE | \
+						 DPU_INT_OFF_SLICE_DONE)
+
+typedef union {
+	struct {
+	//REGISTER dpu_int_reg_0
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_1
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_2
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_3
+	UINT32 cmb_frm_timing_vsync_int_msk              : 1;
+	UINT32 cmb_frm_timing_eof_int_msk                : 1;
+	UINT32 cmb_frm_timing_cfg_eof_int_msk            : 2;
+	UINT32 cmb_frm_timing_cfg_line_int_msk           : 1;
+	UINT32 cmb_cfg_rdy_clr_int_msk                   : 1;
+	UINT32 cmb_wb_frm_done_int_msk                   : 1;
+	UINT32:1;
+	UINT32 cmb_frm_timing_unflow_int_msk             : 1;
+	UINT32 cmb_wb_ovflow_int_msk                     : 1;
+	UINT32:1;
+	UINT32 cmb_cmdlist_ch_frm_cfg_done_int_msk       : 8;
+	UINT32:5;
+	UINT32 cmb_wb_slice_done_int_msk                 : 1;
+	UINT32:1;
+	UINT32 video_mode_enter_idle_int_msk             : 1;
+	UINT32 acad_eof_int_msk                          : 1;
+	UINT32 cmb_te_int_msk                            : 1;
+	UINT32 cmb_vsync_updt_int_msk                    : 1;
+	UINT32 cmb_vsync_done_free_int_msk               : 1;
+	UINT32:1;
+
+	//REGISTER dpu_int_reg_4
+	UINT32 cmb_cmdlist_ch_enter_pend_int_msk         : 8;
+	UINT32 cmb_arb_dec_pslverr_int_msk               : 1;
+	UINT32 cmb_cfg_eof_free_int_msk                  : 2;
+	UINT32 cmb_eof_free_int_msk                      : 1;
+	UINT32 cmb_sof_free_int_msk                      : 1;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_5
+	UINT32 cmb_dma_dbg_int_msk                       : 16;
+	UINT32 cmb_outctl_dbg_int_msk                    : 1;
+	UINT32 cmb_ctl_dbg_int_msk                       : 1;
+	UINT32 cmb_cmdlist_dbg_int_msk                   : 1;
+	UINT32:13;
+
+	//REGISTER dpu_int_reg_6
+	UINT32 offl0_cfg_rdy_clr_int_msk                 : 1;
+	UINT32 offl0_wb_frm_done_int_msk                 : 1;
+	UINT32:1;
+	UINT32 offl0_wb_slice_done_int_msk               : 1;
+	UINT32:1;
+	UINT32 offl0_cmdlist_ch_frm_cfg_done_int_msk     : 8;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_7
+	UINT32 offl0_cmdlist_ch_enter_pend_int_msk       : 8;
+	UINT32 offl0_arb_dec_pslverr_int_msk             : 1;
+	UINT32:23;
+
+	//REGISTER dpu_int_reg_8
+	UINT32 offl0_nml_dma_dbg_int_msk                 : 16;
+	UINT32 offl0_nml_ctl_dbg_int_msk                 : 1;
+	UINT32 offl0_nml_cmdlist_dbg_int_msk             : 1;
+	UINT32:14;
+
+	//REGISTER dpu_int_reg_9
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_10
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_11
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_12
+	UINT32 cmb_frm_timing_vsync_ints                 : 1;
+	UINT32 cmb_frm_timing_eof_ints                   : 1;
+	UINT32 cmb_frm_timing_cfg_eof_ints               : 2;
+	UINT32 cmb_frm_timing_cfg_line_ints              : 1;
+	UINT32 cmb_cfg_rdy_clr_ints                      : 1;
+	UINT32 cmb_wb_frm_done_ints                      : 1;
+	UINT32:1;
+	UINT32 cmb_frm_timing_unflow_ints                : 1;
+	UINT32 cmb_wb_ovflow_ints                        : 1;
+	UINT32:1;
+	UINT32 cmb_cmdlist_ch_frm_cfg_done_ints          : 8;
+	UINT32:5;
+	UINT32 cmb_wb_slice_done_ints                    : 1;
+	UINT32:1;
+	UINT32 video_mode_enter_idle_ints                : 1;
+	UINT32 acad_eof_ints                             : 1;
+	UINT32 cmb_te_ints                               : 1;
+	UINT32 cmb_vsync_updt_ints                       : 1;
+	UINT32 cmb_vsync_done_free_ints                  : 1;
+	UINT32:1;
+
+	//REGISTER dpu_int_reg_13
+	UINT32 cmb_cmdlist_ch_enter_pend_ints            : 8;
+	UINT32 cmb_arb_dec_pslverr_ints                  : 1;
+	UINT32 cmb_cfg_eof_free_ints                     : 2;
+	UINT32 cmb_eof_free_ints                         : 1;
+	UINT32 cmb_sof_free_ints                         : 1;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_14
+	UINT32 cmb_dma_dbg_ints                          : 16;
+	UINT32 cmb_outctl_dbg_ints                       : 1;
+	UINT32 cmb_ctl_dbg_ints                          : 1;
+	UINT32 cmb_cmdlist_dbg_ints                      : 1;
+	UINT32:13;
+
+	//REGISTER dpu_int_reg_15
+	UINT32 offl0_cfg_rdy_clr_ints                    : 1;
+	UINT32 offl0_wb_frm_done_ints                    : 1;
+	UINT32:1;
+	UINT32 offl0_wb_slice_done_ints                  : 1;
+	UINT32:1;
+	UINT32 offl0_cmdlist_ch_frm_cfg_done_ints        : 8;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_16
+	UINT32 off0_cmdlist_ch_enter_pend_ints           : 8;
+	UINT32 off0_arb_dec_pslverr_ints                 : 1;
+	UINT32:23;
+
+	//REGISTER dpu_int_reg_17
+	UINT32 offl0_nml_dma_dbg_ints                    : 16;
+	UINT32 offl0_nml_ctl_dbg_ints                    : 1;
+	UINT32 offl0_nml_cmdlist_dbg_ints                : 1;
+	UINT32:14;
+
+	//REGISTER dpu_int_reg_18
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_19
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_20
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_21
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_22
+	UINT32 cmb_frm_timing_vsync_int_raw              : 1;
+	UINT32 cmb_frm_timing_eof_int_raw                : 1;
+	UINT32 cmb_frm_timing_cfg_eof_int_raw            : 2;
+	UINT32 cmb_frm_timing_cfg_line_int_raw           : 1;
+	UINT32 cmb_cfg_rdy_clr_int_raw                   : 1;
+	UINT32 cmb_wb_frm_done_int_raw                   : 1;
+	UINT32:1;
+	UINT32 cmb_frm_timing_unflow_int_raw             : 1;
+	UINT32 cmb_wb_ovflow_int_raw                     : 1;
+	UINT32:1;
+	UINT32 cmb_cmdlist_ch_frm_cfg_done_int_raw       : 8;
+	UINT32:5;
+	UINT32 cmb_wb_slice_done_int_raw                 : 1;
+	UINT32:1;
+	UINT32 video_mode_enter_idle_int_raw             : 1;
+	UINT32 acad_eof_int_raw                          : 1;
+	UINT32 cmb_te_int_raw                            : 1;
+	UINT32 cmb_vsync_updt_int_raw                    : 1;
+	UINT32 cmb_vsync_done_free_int_raw               : 1;
+	UINT32:1;
+
+	//REGISTER dpu_int_reg_23
+	UINT32 cmb_cmdlist_ch_enter_pend_int_raw         : 8;
+	UINT32 cmb_arb_dec_pslverr_int_raw               : 1;
+	UINT32 cmb_cfg_eof_free_int_raw                  : 2;
+	UINT32 cmb_eof_free_int_raw                      : 1;
+	UINT32 cmb_sof_free_int_raw                      : 1;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_24
+	UINT32 cmb_dma_dbg_int_raw                       : 16;
+	UINT32 cmb_outctl_dbg_int_raw                    : 1;
+	UINT32 cmb_ctl_dbg_int_raw                       : 1;
+	UINT32 cmb_cmdlist_dbg_int_raw                   : 1;
+	UINT32:13;
+
+	//REGISTER dpu_int_reg_25
+	UINT32 cmb_arb_dec_paddr                         : 21;
+	UINT32:11;
+
+	//REGISTER dpu_int_reg_26
+	UINT32 offl0_cfg_rdy_clr_int_raw                 : 1;
+	UINT32 offl0_wb_frm_done_int_raw                 : 1;
+	UINT32:1;
+	UINT32 offl0_wb_slice_done_int_raw               : 1;
+	UINT32:1;
+	UINT32 offl0_cmdlist_ch_frm_cfg_done_int_raw     : 8;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_27
+	UINT32 off0_cmdlist_ch_enter_pend_int_raw        : 8;
+	UINT32 off0_arb_dec_pslverr_int_raw              : 1;
+	UINT32:23;
+
+	//REGISTER dpu_int_reg_28
+	UINT32 offl0_nml_dma_dbg_int_raw                 : 16;
+	UINT32 offl0_nml_ctl_dbg_int_raw                 : 1;
+	UINT32 offl0_nml_cmdlist_dbg_int_raw             : 1;
+	UINT32:14;
+
+	//REGISTER dpu_int_reg_29
+	UINT32 offl0_arb_dec_paddr                       : 21;
+	UINT32:11;
+
+	//REGISTER dpu_int_reg_30
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_31
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_32
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_33
+	UINT32 cmb_frm_timing_vsync_secu_int_msk         : 1;
+	UINT32 cmb_frm_timing_eof_secu_int_msk           : 1;
+	UINT32 cmb_frm_timing_cfg_eof_secu_int_msk       : 2;
+	UINT32 cmb_frm_timing_cfg_line_secu_int_msk      : 1;
+	UINT32 cmb_cfg_rdy_clr_secu_int_msk              : 1;
+	UINT32 cmb_wb_frm_done_secu_int_msk              : 1;
+	UINT32:1;
+	UINT32 cmb_frm_timing_unflow_secu_int_msk        : 1;
+	UINT32 cmb_wb_ovflow_secu_int_msk                : 1;
+	UINT32:1;
+	UINT32 cmb_cmdlist_ch_frm_cfg_done_secu_int_msk  : 8;
+	UINT32:5;
+	UINT32 cmb_wb_slice_done_secu_int_msk            : 1;
+	UINT32:1;
+	UINT32 video_mode_enter_idle_secu_int_msk        : 1;
+	UINT32 acad_eof_secu_int_msk                     : 1;
+	UINT32 cmb_te_secu_int_msk                       : 1;
+	UINT32 cmb_vsync_updt_secu_int_msk               : 1;
+	UINT32 cmb_vsync_done_free_secu_int_msk          : 1;
+	UINT32:1;
+
+	//REGISTER dpu_int_reg_34
+	UINT32 cmb_cmdlist_ch_enter_pend_secu_int_msk    : 8;
+	UINT32 cmb_arb_dec_pslverr_secu_int_msk          : 1;
+	UINT32 cmb_cfg_eof_free_secu_int_msk             : 2;
+	UINT32 cmb_eof_free_secu_int_msk                 : 1;
+	UINT32 cmb_sof_free_secu_int_msk                 : 1;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_35
+	UINT32 cmb_dma_dbg_secu_int_msk                  : 16;
+	UINT32 cmb_outctl_dbg_secu_int_msk               : 1;
+	UINT32 cmb_ctl_dbg_secu_int_msk                  : 1;
+	UINT32 cmb_cmdlist_dbg_secu_int_msk              : 1;
+	UINT32:13;
+
+	//REGISTER dpu_int_reg_36
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_37
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_38
+	UINT32:32;
+
+	//REGISTER dpu_int_reg_39
+	UINT32 cmb_frm_timing_vsync_secu_ints            : 1;
+	UINT32 cmb_frm_timing_eof_secu_ints              : 1;
+	UINT32 cmb_frm_timing_cfg_eof_secu_ints          : 2;
+	UINT32 cmb_frm_timing_cfg_line_secu_ints         : 1;
+	UINT32 cmb_cfg_rdy_clr_secu_ints                 : 1;
+	UINT32 cmb_wb_frm_done_secu_ints                 : 1;
+	UINT32:1;
+	UINT32 cmb_frm_timing_unflow_secu_ints           : 1;
+	UINT32 cmb_wb_ovflow_secu_ints                   : 1;
+	UINT32:1;
+	UINT32 cmb_cmdlist_ch_frm_cfg_done_secu_ints     : 8;
+	UINT32:5;
+	UINT32 cmb_wb_slice_done_secu_ints               : 1;
+	UINT32:1;
+	UINT32 video_mode_enter_idle_secu_ints           : 1;
+	UINT32 acad_eof_secu_ints                        : 1;
+	UINT32 cmb_te_secu_ints                          : 1;
+	UINT32 cmb_vsync_updt_secu_ints                  : 1;
+	UINT32 cmb_vsync_done_free_secu_ints             : 1;
+	UINT32:1;
+
+	//REGISTER dpu_int_reg_40
+	UINT32 cmb_cmdlist_ch_enter_pend_secu_ints       : 8;
+	UINT32 cmb_arb_dec_pslverr_secu_ints             : 1;
+	UINT32 cmb_cfg_eof_free_secu_ints                : 2;
+	UINT32 cmb_eof_free_secu_ints                    : 1;
+	UINT32 cmb_sof_free_secu_ints                    : 1;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_41
+	UINT32 cmb_dma_dbg_secu_ints                     : 16;
+	UINT32 cmb_outctl_dbg_secu_ints                  : 1;
+	UINT32 cmb_ctl_dbg_secu_ints                     : 1;
+	UINT32 cmb_cmdlist_dbg_secu_ints                 : 1;
+	UINT32:13;
+
+	//REGISTER dpu_int_reg_42
+	struct {
+	UINT32:32;
+	} dpu_int_reg_42[4];
+
+	//REGISTER dpu_int_reg_46
+	UINT32 cmb_frm_timing_vsync_secu_int_raw         : 1;
+	UINT32 cmb_frm_timing_eof_secu_int_raw           : 1;
+	UINT32 cmb_frm_timing_cfg_eof_secu_int_raw       : 2;
+	UINT32 cmb_frm_timing_cfg_line_secu_int_raw      : 1;
+	UINT32 cmb_cfg_rdy_clr_secu_int_raw              : 1;
+	UINT32 cmb_wb_frm_done_secu_int_raw              : 1;
+	UINT32:1;
+	UINT32 cmb_frm_timing_unflow_secu_int_raw        : 1;
+	UINT32 cmb_wb_ovflow_secu_int_raw                : 1;
+	UINT32:1;
+	UINT32 cmb_cmdlist_ch_frm_cfg_done_secu_int_raw  : 8;
+	UINT32:5;
+	UINT32 cmb_wb_slice_done_secu_int_raw            : 1;
+	UINT32:1;
+	UINT32 video_mode_enter_idle_secu_int_raw        : 1;
+	UINT32 acad_eof_secu_int_raw                     : 1;
+	UINT32 cmb_te_secu_int_raw                       : 1;
+	UINT32 cmb_vsync_updt_secu_int_raw               : 1;
+	UINT32 cmb_vsync_done_free_secu_int_raw          : 1;
+	UINT32:1;
+
+	//REGISTER dpu_int_reg_47
+	UINT32 cmb_cmdlist_ch_enter_pend_secu_int_raw    : 8;
+	UINT32 cmb_arb_dec_pslverr_secu_int_raw          : 1;
+	UINT32 cmb_cfg_eof_free_secu_int_raw             : 2;
+	UINT32 cmb_eof_free_secu_int_raw                 : 1;
+	UINT32 cmb_sof_free_secu_int_raw                 : 1;
+	UINT32:19;
+
+	//REGISTER dpu_int_reg_48
+	UINT32 cmb_dma_dbg_secu_int_raw                  : 16;
+	UINT32 cmb_outctl_dbg_secu_int_raw               : 1;
+	UINT32 cmb_ctl_dbg_secu_int_raw                  : 1;
+	UINT32 cmb_cmdlist_dbg_secu_int_raw              : 1;
+	UINT32:13;
+
+	//REGISTER dpu_int_reg_49
+	UINT32 cmb_secu_arb_dec_paddr                    : 21;
+	UINT32:11;
+
+	} b;
+
+	INT32 value32[50];
+
+} DPU_INT_REG;
+typedef DPU_INT_REG DPU_INTP_REG;
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_scene_ctl.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_scene_ctl.h
new file mode 100644
index 000000000000..eac581a11f31
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_scene_ctl.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef DPU_SCENE_CTL_REG_H
+#define DPU_SCENE_CTL_REG_H
+
+typedef union {
+	struct {
+	//REGISTER dpu_ctl_reg_0
+	UINT32 nml_rch_en             : 10;
+	UINT32 nml_scl_en             : 4;
+	UINT32:2;
+	UINT32 nml_wb_en              : 2;
+	UINT32 nml_outctl_en          : 1;
+	UINT32 nml_dscw_en            : 2;
+	UINT32 nml_dscr_en            : 2;
+	UINT32 nml_frm_timing_en      : 3;
+	UINT32:6;
+
+	//REGISTER dpu_ctl_reg_1
+	UINT32 nml_cmd_updt_en        : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_reg_2
+	UINT32 both_cfg_rdy           : 1;
+	UINT32 sw_clr                 : 1;
+	UINT32 sw_clr_st              : 1;
+	UINT32 nml_cfg_rdy            : 1;
+	UINT32:28;
+
+	//REGISTER dpu_ctl_reg_3
+	UINT32 video_mod              : 1;
+	UINT32 dbg_mod                : 1;
+	UINT32 timing_inter0          : 6;
+	UINT32 timing_inter1          : 6;
+	UINT32:18;
+
+	//REGISTER dpu_ctl_reg_4
+	UINT32 sw_start               : 1;
+	UINT32 dbg_clr                : 1;
+	UINT32:30;
+
+	//REGISTER dpu_ctl_reg_5
+	UINT32 reg_nml_value_rst      : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_reg_6
+	UINT32 nml_init_en            : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_reg_7
+	UINT32 nml_rch_vrt_reuse      : 10;
+	UINT32 cmd_auto_refresh_en    : 1;
+	UINT32:21;
+
+	//REGISTER dpu_ctl_reg_8
+	UINT32 secu_rch_en            : 10;
+	UINT32 secu_rch_vrt_reuse     : 10;
+	UINT32 secu_scl_en            : 4;
+	UINT32:8;
+
+	//REGISTER dpu_ctl_reg_9
+	UINT32 secu_cmd_updt_en       : 1;
+	UINT32 secu_cfg_icg_override  : 1;
+	UINT32:30;
+
+	//REGISTER dpu_ctl_reg_10
+	UINT32 secu_cfg_rdy           : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_reg_11
+	UINT32 secu_dscw_disable      : 2;
+	UINT32 secu_wb_disable        : 2;
+	UINT32:28;
+
+	//REGISTER dpu_ctl_reg_12
+	UINT32 reg_secu_value_rst     : 1;
+	UINT32:31;
+
+	//REGISTER dpu_ctl_reg_13
+	UINT32 scene_ctl_dbg0         : 29;
+	UINT32:3;
+
+	//REGISTER dpu_ctl_reg_14
+	UINT32 scene_ctl_dbg1         : 32;
+
+	};
+
+	INT32 value32[15];
+
+} DPU_SCENE_CTL_REG;
+
+#endif
+typedef DPU_SCENE_CTL_REG DPU_CTL_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_top.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_top.h
new file mode 100644
index 000000000000..4ddc1efc7c02
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dpu_top.h
@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef DPU_TOP_REG_H
+#define DPU_TOP_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_led_top_reg_0
+	UINT32 Minor_number           : 8;
+	UINT32 Major_number           : 8;
+	UINT32 Product_ID             : 16;
+
+	//REGISTER saturn_led_top_reg_1
+	struct {
+	UINT32:32;
+	} saturn_led_top_reg_1[3];
+
+	//REGISTER saturn_led_top_reg_4
+	UINT32 rd_sdw_reg_en          : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_5
+	UINT32:32;
+
+	//REGISTER saturn_led_top_reg_6
+	UINT32 shut_down_num          : 8;
+	UINT32:24;
+
+	//REGISTER saturn_led_top_reg_7
+	UINT32 deep_slep_num          : 8;
+	UINT32:24;
+
+	//REGISTER saturn_led_top_reg_8
+	struct {
+	UINT32:32;
+	} saturn_led_top_reg_8[3];
+
+	//REGISTER saturn_led_top_reg_11
+	UINT32 rdma_pclk_cg_en        : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_12
+	UINT32 pre_ly_pclk_cg_en      : 6;
+	UINT32:26;
+
+	//REGISTER saturn_led_top_reg_13
+	UINT32 cmps_pclk_cg_en        : 3;
+	UINT32:29;
+
+	//REGISTER saturn_led_top_reg_14
+	UINT32 scale_pclk_cg_en       : 4;
+	UINT32:28;
+
+	//REGISTER saturn_led_top_reg_15
+	UINT32 postpipe_pclk_cg_en    : 8;
+	UINT32:24;
+
+	//REGISTER saturn_led_top_reg_16
+	UINT32 wb_pclk_cg_en          : 2;
+	UINT32:30;
+
+	//REGISTER saturn_led_top_reg_17
+	struct {
+	UINT32:32;
+	} saturn_led_top_reg_17[5];
+
+	//REGISTER saturn_led_top_reg_22
+	UINT32 top_clk_auto_en        : 9;
+	UINT32:23;
+
+	//REGISTER saturn_led_top_reg_23
+	UINT32 ctl_clk_auto_en        : 2;
+	UINT32:30;
+
+	//REGISTER saturn_led_top_reg_24
+	UINT32 cmdlist_clk_auto_en    : 2;
+	UINT32:30;
+
+	//REGISTER saturn_led_top_reg_25
+	UINT32 rdma_aclk_auto_en      : 4;
+	UINT32:28;
+
+	//REGISTER saturn_led_top_reg_26
+	UINT32 pre_ly_mclk_auto_en    : 6;
+	UINT32:26;
+
+	//REGISTER saturn_led_top_reg_27
+	UINT32 cmps_clk_auto_en       : 3;
+	UINT32:29;
+
+	//REGISTER saturn_led_top_reg_28
+	UINT32 scale_mclk_auto_en     : 4;
+	UINT32:28;
+
+	//REGISTER saturn_led_top_reg_29
+	UINT32 postpipe_clk_auto_en   : 17;
+	UINT32:15;
+
+	//REGISTER saturn_led_top_reg_30
+	UINT32 tmg_clk_auto_en        : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_31
+	UINT32 wb_clk_auto_en         : 4;
+	UINT32:28;
+
+	//REGISTER saturn_led_top_reg_32
+	UINT32 dma_mem_lp_en          : 20;
+	UINT32:12;
+
+	//REGISTER saturn_led_top_reg_33
+	UINT32 postpipe_mem_lp_en     : 10;
+	UINT32:22;
+
+	//REGISTER saturn_led_top_reg_34
+	UINT32 tmg_mem_lp_en          : 1;
+	UINT32 wb_mem_lp_en           : 1;
+	UINT32 scale_mem_lp_en        : 1;
+	UINT32 scale_obuf_mem_lp_en   : 1;
+	UINT32:28;
+
+	//REGISTER saturn_led_top_reg_35
+	UINT32:32;
+
+	//REGISTER saturn_led_top_reg_36
+	UINT32 prepipe0_valid         : 1;
+	UINT32 prepipe0_ready         : 1;
+	UINT32 prepipe1_valid         : 1;
+	UINT32 prepipe1_ready         : 1;
+	UINT32 prepipe2_valid         : 1;
+	UINT32 prepipe2_ready         : 1;
+	UINT32 prepipe3_valid         : 1;
+	UINT32 prepipe3_ready         : 1;
+	UINT32 prepipe4_valid         : 1;
+	UINT32 prepipe4_ready         : 1;
+	UINT32 prepipe5_valid         : 1;
+	UINT32 prepipe5_ready         : 1;
+	UINT32 compser0_valid         : 1;
+	UINT32 compser0_ready         : 1;
+	UINT32 compser1_valid         : 1;
+	UINT32 compser1_ready         : 1;
+	UINT32 pospipe_valid          : 1;
+	UINT32 postpipe_ready         : 1;
+	UINT32 scl_1d_valid           : 1;
+	UINT32 scl_1d_ready           : 1;
+	UINT32 scl_2d_valid           : 1;
+	UINT32 scl_2d_ready           : 1;
+	UINT32:10;
+
+	//REGISTER saturn_led_top_reg_37
+	UINT32 cfg_se                 : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_38
+	UINT32 force_update_en        : 1;
+	UINT32 vsync_update_en        : 1;
+	UINT32 shadow_read_en         : 1;
+	UINT32:29;
+
+	//REGISTER saturn_led_top_reg_39
+	UINT32 force_update_pulse     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_40
+	UINT32 force_update_en_se     : 1;
+	UINT32 vsync_update_en_se     : 1;
+	UINT32 shadow_read_en_se      : 1;
+	UINT32:29;
+
+	//REGISTER saturn_led_top_reg_41
+	UINT32 force_update_pulse_se  : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_42
+	UINT32 icg_override           : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_43
+	UINT32 trigger                : 1;
+	UINT32:31;
+
+	//REGISTER saturn_led_top_reg_44
+	UINT32 trigger2               : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[45];
+
+} DPU_TOP_REG;
+
+#endif
+typedef DPU_TOP_REG DPU_TOP_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/dsc_enc_top.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dsc_enc_top.h
new file mode 100644
index 000000000000..d5be76da8456
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/dsc_enc_top.h
@@ -0,0 +1,215 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef DSC_ENC_TOP_REG_H
+#define DSC_ENC_TOP_REG_H
+
+typedef union {
+	struct {
+	//REGISTER dsc_enc_reg_0
+	UINT32 dsc_support_version         : 4;
+	UINT32 dsc_enable                  : 1;
+	UINT32 full_ich_err_precision      : 1;
+	UINT32:2;
+	UINT32 pps_identifier              : 8;
+	UINT32 linebuf_depth               : 4;
+	UINT32 bits_per_component          : 4;
+	UINT32:8;
+
+	//REGISTER dsc_enc_reg_1
+	UINT32 bits_per_pixel              : 10;
+	UINT32 vbr_enable                  : 1;
+	UINT32 simple_422                  : 1;
+	UINT32 convert_rgb                 : 1;
+	UINT32 block_pred_enable           : 1;
+	UINT32:2;
+	UINT32 pic_height                  : 16;
+
+	//REGISTER dsc_enc_reg_2
+	UINT32 pic_width                   : 16;
+	UINT32 slice_height                : 16;
+
+	//REGISTER dsc_enc_reg_3
+	UINT32 slice_width                 : 16;
+	UINT32 chunk_size                  : 16;
+
+	//REGISTER dsc_enc_reg_4
+	UINT32 initial_xmit_delay          : 10;
+	UINT32:6;
+	UINT32 initial_dec_delay           : 16;
+
+	//REGISTER dsc_enc_reg_5
+	UINT32 initial_scale_value         : 6;
+	UINT32:2;
+	UINT32 scale_increment_interval    : 16;
+	UINT32:8;
+
+	//REGISTER dsc_enc_reg_6
+	UINT32 scale_decrement_interval    : 12;
+	UINT32:4;
+	UINT32 first_line_bpg_offset       : 5;
+	UINT32:11;
+
+	//REGISTER dsc_enc_reg_7
+	UINT32 nfl_bpg_offset              : 16;
+	UINT32 slice_bpg_offset            : 16;
+
+	//REGISTER dsc_enc_reg_8
+	UINT32 initial_offset              : 16;
+	UINT32 final_offset                : 16;
+
+	//REGISTER dsc_enc_reg_9
+	UINT32 flatness_min_qp             : 5;
+	UINT32:3;
+	UINT32 flatness_max_qp             : 5;
+	UINT32:3;
+	UINT32 rc_model_size               : 16;
+
+	//REGISTER dsc_enc_reg_10
+	UINT32 rc_edge_factor              : 4;
+	UINT32:4;
+	UINT32 rc_quant_incr_limit0        : 5;
+	UINT32:3;
+	UINT32 rc_quant_incr_limit1        : 5;
+	UINT32:3;
+	UINT32 rc_tgt_offset_lo            : 4;
+	UINT32 rc_tgt_offset_hi            : 4;
+
+	//REGISTER dsc_enc_reg_11
+	struct {
+	UINT32 rc_buf_thresh               : 8;
+	UINT32:24;
+	} dsc_enc_reg_11[14];
+
+	//REGISTER dsc_enc_reg_25
+	struct {
+	UINT32 range_min_qp                : 5;
+	UINT32:27;
+	} dsc_enc_reg_25[15];
+
+	//REGISTER dsc_enc_reg_40
+	struct {
+	UINT32 range_max_qp                : 5;
+	UINT32:27;
+	} dsc_enc_reg_40[15];
+
+	//REGISTER dsc_enc_reg_55
+	struct {
+	UINT32 range_bpg_offset            : 6;
+	UINT32:26;
+	} dsc_enc_reg_55[15];
+
+	//REGISTER dsc_enc_reg_70
+	UINT32 native_422                  : 1;
+	UINT32 native_420                  : 1;
+	UINT32 second_line_bpg_offset      : 5;
+	UINT32:1;
+	UINT32 nsl_bpg_offset              : 16;
+	UINT32:8;
+
+	//REGISTER dsc_enc_reg_71
+	UINT32 second_line_offset_adj      : 16;
+	UINT32:16;
+
+	//REGISTER dsc_enc_reg_72
+	struct {
+	UINT32:32;
+	} dsc_enc_reg_72[9];
+
+	//REGISTER dsc_enc_reg_81
+	UINT32 dbg_path_start_enable       : 1;
+	UINT32 dbg_dsc_i_ready             : 1;
+	UINT32 dbg_dsc_o_ready             : 1;
+	UINT32 dbg_path_core_ready         : 4;
+	UINT32:1;
+	UINT32 dbg_path_ratebuf_ready      : 4;
+	UINT32:20;
+
+	//REGISTER dsc_enc_reg_82
+	UINT32 dbg_path_in_pxl_cnt         : 16;
+	UINT32 dbg_path_in_line_cnt        : 16;
+
+	//REGISTER dsc_enc_reg_83
+	struct {
+	UINT32 dbg_dsc_slice_in_group_cnt  : 16;
+	UINT32:16;
+	} dsc_enc_reg_83[4];
+
+	//REGISTER dsc_enc_reg_87
+	struct {
+	UINT32 dbg_dsc_slice_in_line_cnt   : 16;
+	UINT32:16;
+	} dsc_enc_reg_87[4];
+
+	//REGISTER dsc_enc_reg_91
+	struct {
+	UINT32 dbg_dsc_slice_out_byte_cnt  : 16;
+	UINT32:16;
+	} dsc_enc_reg_91[4];
+
+	//REGISTER dsc_enc_reg_95
+	struct {
+	UINT32 dbg_dsc_slice_out_line_cnt  : 16;
+	UINT32:16;
+	} dsc_enc_reg_95[4];
+
+	//REGISTER dsc_enc_reg_99
+	UINT32 dbg_dsc_output_byte_cnt     : 16;
+	UINT32 dbg_dsc_output_line_cnt     : 16;
+
+	//REGISTER dsc_enc_reg_100
+	UINT32 dsc_dbg_irq_raw             : 32;
+
+	//REGISTER dsc_enc_reg_101
+	UINT32 dsc_dbg_irq_mask            : 32;
+
+	//REGISTER dsc_enc_reg_102
+	UINT32 dsc_dbg_irq_status          : 32;
+
+	//REGISTER dsc_enc_reg_103
+	UINT32 cfg_sec                     : 1;
+	UINT32:31;
+
+	//REGISTER dsc_enc_reg_104
+	UINT32 force_update_en             : 1;
+	UINT32 vsync_update_en             : 1;
+	UINT32 shadow_read_en              : 1;
+	UINT32:29;
+
+	//REGISTER dsc_enc_reg_105
+	UINT32 force_update_pulse          : 1;
+	UINT32:31;
+
+	//REGISTER dsc_enc_reg_106
+	UINT32 force_update_en_se          : 1;
+	UINT32 vsync_update_en_se          : 1;
+	UINT32 shadow_read_en_se           : 1;
+	UINT32:29;
+
+	//REGISTER dsc_enc_reg_107
+	UINT32 force_update_pulse_se       : 1;
+	UINT32:31;
+
+	//REGISTER dsc_enc_reg_108
+	UINT32 icg_override                : 1;
+	UINT32:31;
+
+	//REGISTER dsc_enc_reg_109
+	UINT32 trigger                     : 1;
+	UINT32:31;
+
+	//REGISTER dsc_enc_reg_110
+	UINT32 trigger2                    : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[111];
+
+} DSC_ENC_TOP_REG;
+
+#endif
+typedef DSC_ENC_TOP_REG DSC_ENC_TOP_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/ee.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/ee.h
new file mode 100644
index 000000000000..66921075f61a
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/ee.h
@@ -0,0 +1,345 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef EE_REG_H
+#define EE_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_ee_reg_0
+	UINT32 m_benable              : 1;
+	UINT32:31;
+
+	//REGISTER saturn_ee_reg_1
+	UINT32 m_plpf1_0              : 8;
+	UINT32 m_plpf1_1              : 8;
+	UINT32 m_plpf2_0              : 8;
+	UINT32 m_plpf2_1              : 8;
+
+	//REGISTER saturn_ee_reg_2
+	UINT32 m_nfreq_exp_level1     : 6;
+	UINT32:2;
+	UINT32 m_nfreq_exp_level2     : 6;
+	UINT32:2;
+	UINT32 m_nnode_freq           : 5;
+	UINT32:11;
+
+	//REGISTER saturn_ee_reg_3
+	UINT32 m_nsingular_p1         : 13;
+	UINT32:3;
+	UINT32 m_nsingular_p2         : 13;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_4
+	UINT32 m_pcurve_freq_tx_p_0   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_1   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_2   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_3   : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_5
+	UINT32 m_pcurve_freq_tx_p_4   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_5   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_6   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_7   : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_6
+	UINT32 m_pcurve_freq_tx_p_8   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_9   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_10  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_11  : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_7
+	UINT32 m_pcurve_freq_tx_p_12  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_13  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_14  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_p_15  : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_8
+	UINT32 m_pcurve_freq_tx_p_16  : 5;
+	UINT32:27;
+
+	//REGISTER saturn_ee_reg_9
+	UINT32 m_pcurve_freq_tx_n_0   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_1   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_2   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_3   : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_10
+	UINT32 m_pcurve_freq_tx_n_4   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_5   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_6   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_7   : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_11
+	UINT32 m_pcurve_freq_tx_n_8   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_9   : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_10  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_11  : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_12
+	UINT32 m_pcurve_freq_tx_n_12  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_13  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_14  : 5;
+	UINT32:3;
+	UINT32 m_pcurve_freq_tx_n_15  : 5;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_13
+	UINT32 m_pcurve_freq_tx_n_16  : 5;
+	UINT32:27;
+
+	//REGISTER saturn_ee_reg_14
+	UINT32 m_nfreq_offset_tx      : 8;
+	UINT32:24;
+
+	//REGISTER saturn_ee_reg_15
+	UINT32 m_pcurve_luma_p_0      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_1      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_2      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_3      : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_16
+	UINT32 m_pcurve_luma_p_4      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_5      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_6      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_7      : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_17
+	UINT32 m_pcurve_luma_p_8      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_9      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_10     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_11     : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_18
+	UINT32 m_pcurve_luma_p_12     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_13     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_14     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_p_15     : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_19
+	UINT32 m_pcurve_luma_p_16     : 7;
+	UINT32:25;
+
+	//REGISTER saturn_ee_reg_20
+	UINT32 m_pcurve_luma_n_0      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_1      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_2      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_3      : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_21
+	UINT32 m_pcurve_luma_n_4      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_5      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_6      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_7      : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_22
+	UINT32 m_pcurve_luma_n_8      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_9      : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_10     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_11     : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_23
+	UINT32 m_pcurve_luma_n_12     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_13     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_14     : 7;
+	UINT32:1;
+	UINT32 m_pcurve_luma_n_15     : 7;
+	UINT32:1;
+
+	//REGISTER saturn_ee_reg_24
+	UINT32 m_pcurve_luma_n_16     : 7;
+	UINT32:25;
+
+	//REGISTER saturn_ee_reg_25
+	UINT32 m_ntx_clip_p12         : 13;
+	UINT32:3;
+	UINT32 m_ntx_clip_p10         : 13;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_26
+	UINT32 m_ntx_clip_n12         : 13;
+	UINT32:3;
+	UINT32 m_ntx_clip_n10         : 13;
+	UINT32:3;
+
+	//REGISTER saturn_ee_reg_27
+	UINT32 m_ntx_thrd_p12         : 9;
+	UINT32:7;
+	UINT32 m_ntx_thrd_p10         : 9;
+	UINT32:7;
+
+	//REGISTER saturn_ee_reg_28
+	UINT32 m_ntx_thrd_n12         : 9;
+	UINT32:7;
+	UINT32 m_ntx_thrd_n10         : 9;
+	UINT32:7;
+
+	//REGISTER saturn_ee_reg_29
+	UINT32 m_nclip_pos12          : 12;
+	UINT32:4;
+	UINT32 m_nclip_pos10          : 12;
+	UINT32:4;
+
+	//REGISTER saturn_ee_reg_30
+	UINT32 m_nclip_neg12          : 12;
+	UINT32:4;
+	UINT32 m_nclip_neg10          : 12;
+	UINT32:4;
+
+	//REGISTER saturn_ee_reg_31
+	UINT32 m_nhc_strength         : 6;
+	UINT32:2;
+	UINT32 m_pcoff_w_0            : 8;
+	UINT32 m_pcoff_w_1            : 8;
+	UINT32:8;
+
+	//REGISTER saturn_ee_reg_32
+	UINT32 m_ncoff_w_curve        : 11;
+	UINT32:21;
+
+	//REGISTER saturn_ee_reg_33
+	UINT32 rgb2yuv_matrix00       : 16;
+	UINT32 rgb2yuv_matrix01       : 16;
+
+	//REGISTER saturn_ee_reg_34
+	UINT32 rgb2yuv_matrix02       : 16;
+	UINT32 rgb2yuv_matrix03       : 16;
+
+	//REGISTER saturn_ee_reg_35
+	UINT32 rgb2yuv_matrix10       : 16;
+	UINT32 rgb2yuv_matrix11       : 16;
+
+	//REGISTER saturn_ee_reg_36
+	UINT32 rgb2yuv_matrix12       : 16;
+	UINT32 rgb2yuv_matrix13       : 16;
+
+	//REGISTER saturn_ee_reg_37
+	UINT32 rgb2yuv_matrix20       : 16;
+	UINT32 rgb2yuv_matrix21       : 16;
+
+	//REGISTER saturn_ee_reg_38
+	UINT32 rgb2yuv_matrix22       : 16;
+	UINT32 rgb2yuv_matrix23       : 16;
+
+	//REGISTER saturn_ee_reg_39
+	UINT32 yuv2rgb_matrix00       : 16;
+	UINT32 yuv2rgb_matrix01       : 16;
+
+	//REGISTER saturn_ee_reg_40
+	UINT32 yuv2rgb_matrix02       : 16;
+	UINT32 yuv2rgb_matrix03       : 16;
+
+	//REGISTER saturn_ee_reg_41
+	UINT32 yuv2rgb_matrix10       : 16;
+	UINT32 yuv2rgb_matrix11       : 16;
+
+	//REGISTER saturn_ee_reg_42
+	UINT32 yuv2rgb_matrix12       : 16;
+	UINT32 yuv2rgb_matrix13       : 16;
+
+	//REGISTER saturn_ee_reg_43
+	UINT32 yuv2rgb_matrix20       : 16;
+	UINT32 yuv2rgb_matrix21       : 16;
+
+	//REGISTER saturn_ee_reg_44
+	UINT32 yuv2rgb_matrix22       : 16;
+	UINT32 yuv2rgb_matrix23       : 16;
+
+	//REGISTER saturn_ee_reg_45
+	UINT32 cfg_se                 : 1;
+	UINT32:31;
+
+	//REGISTER saturn_ee_reg_46
+	UINT32 force_update_en        : 1;
+	UINT32 vsync_update_en        : 1;
+	UINT32 shadow_read_en         : 1;
+	UINT32:29;
+
+	//REGISTER saturn_ee_reg_47
+	UINT32 force_update_pulse     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_ee_reg_48
+	UINT32 icg_override           : 1;
+	UINT32:31;
+
+	//REGISTER saturn_ee_reg_49
+	UINT32 trigger                : 1;
+	UINT32:31;
+
+	//REGISTER saturn_ee_reg_50
+	UINT32 trigger2               : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[51];
+
+} EE_REG;
+
+#endif
+
+typedef EE_REG EE_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/ltm.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/ltm.h
new file mode 100644
index 000000000000..d94f0fe823f2
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/ltm.h
@@ -0,0 +1,693 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef LTM_REG_H
+#define LTM_REG_H
+
+typedef union {
+	struct {
+	//REGISTER outctl_postproc_reg_0
+	UINT32 m_npost_proc_en               : 1;
+	UINT32 m_ngain_to_full_en            : 1;
+	UINT32 m_nmatrix_en                  : 1;
+	UINT32 m_nendmatrix_en               : 1;
+	UINT32 m_nfront_tmootf_en            : 1;
+	UINT32 m_nend_tmootf_en              : 1;
+	UINT32 m_neotf_en                    : 1;
+	UINT32 m_noetf_en                    : 1;
+	UINT32:24;
+
+	//REGISTER outctl_postproc_reg_1
+	UINT32 m_neotf_mode                  : 3;
+	UINT32:29;
+
+	//REGISTER outctl_postproc_reg_2
+	UINT32 m_noetf_mode                  : 3;
+	UINT32:13;
+	UINT32 m_noetf_max                   : 10;
+	UINT32:6;
+
+	//REGISTER outctl_postproc_reg_3
+	UINT32 m_pfront_tmootf_gain_table0   : 16;
+	UINT32 m_pfront_tmootf_gain_table1   : 16;
+
+	//REGISTER outctl_postproc_reg_4
+	UINT32 m_pfront_tmootf_gain_table2   : 16;
+	UINT32 m_pfront_tmootf_gain_table3   : 16;
+
+	//REGISTER outctl_postproc_reg_5
+	UINT32 m_pfront_tmootf_gain_table4   : 16;
+	UINT32 m_pfront_tmootf_gain_table5   : 16;
+
+	//REGISTER outctl_postproc_reg_6
+	UINT32 m_pfront_tmootf_gain_table6   : 16;
+	UINT32 m_pfront_tmootf_gain_table7   : 16;
+
+	//REGISTER outctl_postproc_reg_7
+	UINT32 m_pfront_tmootf_gain_table8   : 16;
+	UINT32 m_pfront_tmootf_gain_table9   : 16;
+
+	//REGISTER outctl_postproc_reg_8
+	UINT32 m_pfront_tmootf_gain_table10  : 16;
+	UINT32 m_pfront_tmootf_gain_table11  : 16;
+
+	//REGISTER outctl_postproc_reg_9
+	UINT32 m_pfront_tmootf_gain_table12  : 16;
+	UINT32 m_pfront_tmootf_gain_table13  : 16;
+
+	//REGISTER outctl_postproc_reg_10
+	UINT32 m_pfront_tmootf_gain_table14  : 16;
+	UINT32 m_pfront_tmootf_gain_table15  : 16;
+
+	//REGISTER outctl_postproc_reg_11
+	UINT32 m_pfront_tmootf_gain_table16  : 16;
+	UINT32 m_pfront_tmootf_gain_table17  : 16;
+
+	//REGISTER outctl_postproc_reg_12
+	UINT32 m_pfront_tmootf_gain_table18  : 16;
+	UINT32 m_pfront_tmootf_gain_table19  : 16;
+
+	//REGISTER outctl_postproc_reg_13
+	UINT32 m_pfront_tmootf_gain_table20  : 16;
+	UINT32 m_pfront_tmootf_gain_table21  : 16;
+
+	//REGISTER outctl_postproc_reg_14
+	UINT32 m_pfront_tmootf_gain_table22  : 16;
+	UINT32 m_pfront_tmootf_gain_table23  : 16;
+
+	//REGISTER outctl_postproc_reg_15
+	UINT32 m_pfront_tmootf_gain_table24  : 16;
+	UINT32 m_pfront_tmootf_gain_table25  : 16;
+
+	//REGISTER outctl_postproc_reg_16
+	UINT32 m_pfront_tmootf_gain_table26  : 16;
+	UINT32 m_pfront_tmootf_gain_table27  : 16;
+
+	//REGISTER outctl_postproc_reg_17
+	UINT32 m_pfront_tmootf_gain_table28  : 16;
+	UINT32 m_pfront_tmootf_gain_table29  : 16;
+
+	//REGISTER outctl_postproc_reg_18
+	UINT32 m_pfront_tmootf_gain_table30  : 16;
+	UINT32 m_pfront_tmootf_gain_table31  : 16;
+
+	//REGISTER outctl_postproc_reg_19
+	UINT32 m_pfront_tmootf_gain_table32  : 16;
+	UINT32 m_pfront_tmootf_gain_table33  : 16;
+
+	//REGISTER outctl_postproc_reg_20
+	UINT32 m_pfront_tmootf_gain_table34  : 16;
+	UINT32 m_pfront_tmootf_gain_table35  : 16;
+
+	//REGISTER outctl_postproc_reg_21
+	UINT32 m_pfront_tmootf_gain_table36  : 16;
+	UINT32 m_pfront_tmootf_gain_table37  : 16;
+
+	//REGISTER outctl_postproc_reg_22
+	UINT32 m_pfront_tmootf_gain_table38  : 16;
+	UINT32 m_pfront_tmootf_gain_table39  : 16;
+
+	//REGISTER outctl_postproc_reg_23
+	UINT32 m_pfront_tmootf_gain_table40  : 16;
+	UINT32 m_pfront_tmootf_gain_table41  : 16;
+
+	//REGISTER outctl_postproc_reg_24
+	UINT32 m_pfront_tmootf_gain_table42  : 16;
+	UINT32 m_pfront_tmootf_gain_table43  : 16;
+
+	//REGISTER outctl_postproc_reg_25
+	UINT32 m_pfront_tmootf_gain_table44  : 16;
+	UINT32 m_pfront_tmootf_gain_table45  : 16;
+
+	//REGISTER outctl_postproc_reg_26
+	UINT32 m_pfront_tmootf_gain_table46  : 16;
+	UINT32 m_pfront_tmootf_gain_table47  : 16;
+
+	//REGISTER outctl_postproc_reg_27
+	UINT32 m_pfront_tmootf_gain_table48  : 16;
+	UINT32 m_pfront_tmootf_gain_table49  : 16;
+
+	//REGISTER outctl_postproc_reg_28
+	UINT32 m_pfront_tmootf_gain_table50  : 16;
+	UINT32 m_pfront_tmootf_gain_table51  : 16;
+
+	//REGISTER outctl_postproc_reg_29
+	UINT32 m_pfront_tmootf_gain_table52  : 16;
+	UINT32 m_pfront_tmootf_gain_table53  : 16;
+
+	//REGISTER outctl_postproc_reg_30
+	UINT32 m_pfront_tmootf_gain_table54  : 16;
+	UINT32 m_pfront_tmootf_gain_table55  : 16;
+
+	//REGISTER outctl_postproc_reg_31
+	UINT32 m_pfront_tmootf_gain_table56  : 16;
+	UINT32 m_pfront_tmootf_gain_table57  : 16;
+
+	//REGISTER outctl_postproc_reg_32
+	UINT32 m_pfront_tmootf_gain_table58  : 16;
+	UINT32 m_pfront_tmootf_gain_table59  : 16;
+
+	//REGISTER outctl_postproc_reg_33
+	UINT32 m_pfront_tmootf_gain_table60  : 16;
+	UINT32 m_pfront_tmootf_gain_table61  : 16;
+
+	//REGISTER outctl_postproc_reg_34
+	UINT32 m_pfront_tmootf_gain_table62  : 16;
+	UINT32 m_pfront_tmootf_gain_table63  : 16;
+
+	//REGISTER outctl_postproc_reg_35
+	UINT32 m_pfront_tmootf_gain_table64  : 16;
+	UINT32 m_nfront_tmootf_shift_bits    : 5;
+	UINT32 m_nfront_tmootf_rgb_mode      : 2;
+	UINT32:9;
+
+	//REGISTER outctl_postproc_reg_36
+	UINT32 m_pend_tmootf_gain_table0     : 16;
+	UINT32 m_pend_tmootf_gain_table1     : 16;
+
+	//REGISTER outctl_postproc_reg_37
+	UINT32 m_pend_tmootf_gain_table2     : 16;
+	UINT32 m_pend_tmootf_gain_table3     : 16;
+
+	//REGISTER outctl_postproc_reg_38
+	UINT32 m_pend_tmootf_gain_table4     : 16;
+	UINT32 m_pend_tmootf_gain_table5     : 16;
+
+	//REGISTER outctl_postproc_reg_39
+	UINT32 m_pend_tmootf_gain_table6     : 16;
+	UINT32 m_pend_tmootf_gain_table7     : 16;
+
+	//REGISTER outctl_postproc_reg_40
+	UINT32 m_pend_tmootf_gain_table8     : 16;
+	UINT32 m_pend_tmootf_gain_table9     : 16;
+
+	//REGISTER outctl_postproc_reg_41
+	UINT32 m_pend_tmootf_gain_table10    : 16;
+	UINT32 m_pend_tmootf_gain_table11    : 16;
+
+	//REGISTER outctl_postproc_reg_42
+	UINT32 m_pend_tmootf_gain_table12    : 16;
+	UINT32 m_pend_tmootf_gain_table13    : 16;
+
+	//REGISTER outctl_postproc_reg_43
+	UINT32 m_pend_tmootf_gain_table14    : 16;
+	UINT32 m_pend_tmootf_gain_table15    : 16;
+
+	//REGISTER outctl_postproc_reg_44
+	UINT32 m_pend_tmootf_gain_table16    : 16;
+	UINT32 m_pend_tmootf_gain_table17    : 16;
+
+	//REGISTER outctl_postproc_reg_45
+	UINT32 m_pend_tmootf_gain_table18    : 16;
+	UINT32 m_pend_tmootf_gain_table19    : 16;
+
+	//REGISTER outctl_postproc_reg_46
+	UINT32 m_pend_tmootf_gain_table20    : 16;
+	UINT32 m_pend_tmootf_gain_table21    : 16;
+
+	//REGISTER outctl_postproc_reg_47
+	UINT32 m_pend_tmootf_gain_table22    : 16;
+	UINT32 m_pend_tmootf_gain_table23    : 16;
+
+	//REGISTER outctl_postproc_reg_48
+	UINT32 m_pend_tmootf_gain_table24    : 16;
+	UINT32 m_pend_tmootf_gain_table25    : 16;
+
+	//REGISTER outctl_postproc_reg_49
+	UINT32 m_pend_tmootf_gain_table26    : 16;
+	UINT32 m_pend_tmootf_gain_table27    : 16;
+
+	//REGISTER outctl_postproc_reg_50
+	UINT32 m_pend_tmootf_gain_table28    : 16;
+	UINT32 m_pend_tmootf_gain_table29    : 16;
+
+	//REGISTER outctl_postproc_reg_51
+	UINT32 m_pend_tmootf_gain_table30    : 16;
+	UINT32 m_pend_tmootf_gain_table31    : 16;
+
+	//REGISTER outctl_postproc_reg_52
+	UINT32 m_pend_tmootf_gain_table32    : 16;
+	UINT32 m_pend_tmootf_gain_table33    : 16;
+
+	//REGISTER outctl_postproc_reg_53
+	UINT32 m_pend_tmootf_gain_table34    : 16;
+	UINT32 m_pend_tmootf_gain_table35    : 16;
+
+	//REGISTER outctl_postproc_reg_54
+	UINT32 m_pend_tmootf_gain_table36    : 16;
+	UINT32 m_pend_tmootf_gain_table37    : 16;
+
+	//REGISTER outctl_postproc_reg_55
+	UINT32 m_pend_tmootf_gain_table38    : 16;
+	UINT32 m_pend_tmootf_gain_table39    : 16;
+
+	//REGISTER outctl_postproc_reg_56
+	UINT32 m_pend_tmootf_gain_table40    : 16;
+	UINT32 m_pend_tmootf_gain_table41    : 16;
+
+	//REGISTER outctl_postproc_reg_57
+	UINT32 m_pend_tmootf_gain_table42    : 16;
+	UINT32 m_pend_tmootf_gain_table43    : 16;
+
+	//REGISTER outctl_postproc_reg_58
+	UINT32 m_pend_tmootf_gain_table44    : 16;
+	UINT32 m_pend_tmootf_gain_table45    : 16;
+
+	//REGISTER outctl_postproc_reg_59
+	UINT32 m_pend_tmootf_gain_table46    : 16;
+	UINT32 m_pend_tmootf_gain_table47    : 16;
+
+	//REGISTER outctl_postproc_reg_60
+	UINT32 m_pend_tmootf_gain_table48    : 16;
+	UINT32 m_pend_tmootf_gain_table49    : 16;
+
+	//REGISTER outctl_postproc_reg_61
+	UINT32 m_pend_tmootf_gain_table50    : 16;
+	UINT32 m_pend_tmootf_gain_table51    : 16;
+
+	//REGISTER outctl_postproc_reg_62
+	UINT32 m_pend_tmootf_gain_table52    : 16;
+	UINT32 m_pend_tmootf_gain_table53    : 16;
+
+	//REGISTER outctl_postproc_reg_63
+	UINT32 m_pend_tmootf_gain_table54    : 16;
+	UINT32 m_pend_tmootf_gain_table55    : 16;
+
+	//REGISTER outctl_postproc_reg_64
+	UINT32 m_pend_tmootf_gain_table56    : 16;
+	UINT32 m_pend_tmootf_gain_table57    : 16;
+
+	//REGISTER outctl_postproc_reg_65
+	UINT32 m_pend_tmootf_gain_table58    : 16;
+	UINT32 m_pend_tmootf_gain_table59    : 16;
+
+	//REGISTER outctl_postproc_reg_66
+	UINT32 m_pend_tmootf_gain_table60    : 16;
+	UINT32 m_pend_tmootf_gain_table61    : 16;
+
+	//REGISTER outctl_postproc_reg_67
+	UINT32 m_pend_tmootf_gain_table62    : 16;
+	UINT32 m_pend_tmootf_gain_table63    : 16;
+
+	//REGISTER outctl_postproc_reg_68
+	UINT32 m_pend_tmootf_gain_table64    : 16;
+	UINT32 m_nend_tmootf_shift_bits      : 5;
+	UINT32 m_nend_tmootf_rgb_mode        : 2;
+	UINT32:9;
+
+	//REGISTER outctl_postproc_reg_69
+	UINT32 m_pmatrix_table0              : 16;
+	UINT32 m_pmatrix_table1              : 16;
+
+	//REGISTER outctl_postproc_reg_70
+	UINT32 m_pmatrix_table2              : 16;
+	UINT32 m_pmatrix_table3              : 16;
+
+	//REGISTER outctl_postproc_reg_71
+	UINT32 m_pmatrix_table4              : 16;
+	UINT32 m_pmatrix_table5              : 16;
+
+	//REGISTER outctl_postproc_reg_72
+	UINT32 m_pmatrix_table6              : 16;
+	UINT32 m_pmatrix_table7              : 16;
+
+	//REGISTER outctl_postproc_reg_73
+	UINT32 m_pmatrix_table8              : 16;
+	UINT32:16;
+
+	//REGISTER outctl_postproc_reg_74
+	UINT32 m_pmatrix_offset0             : 25;
+	UINT32:7;
+
+	//REGISTER outctl_postproc_reg_75
+	UINT32 m_pmatrix_offset1             : 25;
+	UINT32:7;
+
+	//REGISTER outctl_postproc_reg_76
+	UINT32 m_pmatrix_offset2             : 25;
+	UINT32:7;
+
+	//REGISTER outctl_postproc_reg_77
+	UINT32 m_ngain_to_full               : 16;
+	UINT32:16;
+
+	//REGISTER outctl_postproc_reg_78
+	UINT32 m_pendmatrix_table0           : 16;
+	UINT32 m_pendmatrix_table1           : 16;
+
+	//REGISTER outctl_postproc_reg_79
+	UINT32 m_pendmatrix_table2           : 16;
+	UINT32 m_pendmatrix_table3           : 16;
+
+	//REGISTER outctl_postproc_reg_80
+	UINT32 m_pendmatrix_table4           : 16;
+	UINT32 m_pendmatrix_table5           : 16;
+
+	//REGISTER outctl_postproc_reg_81
+	UINT32 m_pendmatrix_table6           : 16;
+	UINT32 m_pendmatrix_table7           : 16;
+
+	//REGISTER outctl_postproc_reg_82
+	UINT32 m_pendmatrix_table8           : 16;
+	UINT32:16;
+
+	//REGISTER outctl_postproc_reg_83
+	UINT32 m_pendmatrix_offset0          : 11;
+	UINT32:21;
+
+	//REGISTER outctl_postproc_reg_84
+	UINT32 m_pendmatrix_offset1          : 11;
+	UINT32:21;
+
+	//REGISTER outctl_postproc_reg_85
+	UINT32 m_pendmatrix_offset2          : 11;
+	UINT32:21;
+
+	//REGISTER outctl_postproc_reg_86
+	UINT32 lut4_group_0                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_87
+	UINT32 lut4_group_1                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_88
+	UINT32 lut4_group_2                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_89
+	UINT32 lut4_group_3                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_90
+	UINT32 lut4_group_4                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_91
+	UINT32 lut4_group_5                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_92
+	UINT32 lut4_group_6                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_93
+	UINT32 lut4_group_7                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_94
+	UINT32 lut4_group_8                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_95
+	UINT32 lut4_group_9                  : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_96
+	UINT32 lut4_group_10                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_97
+	UINT32 lut4_group_11                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_98
+	UINT32 lut4_group_12                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_99
+	UINT32 lut4_group_13                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_100
+	UINT32 lut4_group_14                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_101
+	UINT32 lut4_group_15                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_102
+	UINT32 lut4_group_16                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_103
+	UINT32 lut4_group_17                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_104
+	UINT32 lut4_group_18                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_105
+	UINT32 lut4_group_19                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_106
+	UINT32 lut4_group_20                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_107
+	UINT32 lut4_group_21                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_108
+	UINT32 lut4_group_22                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_109
+	UINT32 lut4_group_23                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_110
+	UINT32 lut4_group_24                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_111
+	UINT32 lut4_group_25                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_112
+	UINT32 lut4_group_26                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_113
+	UINT32 lut4_group_27                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_114
+	UINT32 lut4_group_28                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_115
+	UINT32 lut4_group_29                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_116
+	UINT32 lut4_group_30                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_117
+	UINT32 lut4_group_31                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_118
+	UINT32 lut4_group_32                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_119
+	UINT32 lut4_group_33                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_120
+	UINT32 lut4_group_34                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_121
+	UINT32 lut4_group_35                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_122
+	UINT32 lut4_group_36                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_123
+	UINT32 lut4_group_37                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_124
+	UINT32 lut4_group_38                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_125
+	UINT32 lut4_group_39                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_126
+	UINT32 lut4_group_40                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_127
+	UINT32 lut4_group_41                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_128
+	UINT32 lut4_group_42                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_129
+	UINT32 lut4_group_43                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_130
+	UINT32 lut4_group_44                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_131
+	UINT32 lut4_group_45                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_132
+	UINT32 lut4_group_46                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_133
+	UINT32 lut4_group_47                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_134
+	UINT32 lut4_group_48                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_135
+	UINT32 lut4_group_49                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_136
+	UINT32 lut4_group_50                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_137
+	UINT32 lut4_group_51                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_138
+	UINT32 lut4_group_52                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_139
+	UINT32 lut4_group_53                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_140
+	UINT32 lut4_group_54                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_141
+	UINT32 lut4_group_55                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_142
+	UINT32 lut4_group_56                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_143
+	UINT32 lut4_group_57                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_144
+	UINT32 lut4_group_58                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_145
+	UINT32 lut4_group_59                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_146
+	UINT32 lut4_group_60                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_147
+	UINT32 lut4_group_61                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_148
+	UINT32 lut4_group_62                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_149
+	UINT32 lut4_group_63                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_150
+	UINT32 lut4_group_64                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_151
+	UINT32 lut4_group_65                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_152
+	UINT32 lut4_group_66                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_153
+	UINT32 lut4_group_67                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_154
+	UINT32 lut4_group_68                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_155
+	UINT32 lut4_group_69                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_156
+	UINT32 lut4_group_70                 : 24;
+	UINT32:8;
+
+	//REGISTER outctl_postproc_reg_157
+	UINT32 m_tmootf_weightYr             : 12;
+	UINT32:4;
+	UINT32 m_tmootf_weightYg             : 12;
+	UINT32:4;
+
+	//REGISTER outctl_postproc_reg_158
+	UINT32 m_tmootf_weightYb             : 12;
+	UINT32:20;
+
+	//REGISTER outctl_postproc_reg_159
+	UINT32 force_update_en               : 1;
+	UINT32 vsync_update_en               : 1;
+	UINT32 shadow_read_en                : 1;
+	UINT32:29;
+
+	//REGISTER outctl_postproc_reg_160
+	UINT32 force_update_pulse            : 1;
+	UINT32:31;
+
+	//REGISTER outctl_postproc_reg_161
+	UINT32 icg_override                  : 1;
+	UINT32:31;
+
+	//REGISTER outctl_postproc_reg_162
+	UINT32 trigger                       : 1;
+	UINT32:31;
+
+	//REGISTER outctl_postproc_reg_163
+	UINT32 trigger2                      : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[164];
+
+} LTM_REG;
+
+#endif
+
+typedef LTM_REG LTM_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/lut_3d.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/lut_3d.h
new file mode 100644
index 000000000000..7cac3704701f
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/lut_3d.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef LUT_3D_REG_H
+#define LUT_3D_REG_H
+
+typedef union {
+	struct {
+	//REGISTER Lut_3d_reg_0
+	UINT32 lut_3d_en        : 1;
+	UINT32:31;
+
+	//REGISTER Lut_3d_reg_1
+	UINT32 lut_3d_cfg_done  : 1;
+	UINT32:31;
+
+	//REGISTER Lut_3d_reg_2
+	UINT32 m_pLut0_r        : 11;
+	UINT32:5;
+	UINT32 m_pLut0_g        : 11;
+	UINT32:5;
+
+	//REGISTER Lut_3d_reg_3
+	UINT32 m_pLut0_b        : 11;
+	UINT32:5;
+	UINT32 m_pLut1_r        : 11;
+	UINT32:5;
+
+	//REGISTER Lut_3d_reg_4
+	UINT32 m_pLut1_g        : 11;
+	UINT32:5;
+	UINT32 m_pLut1_b        : 11;
+	UINT32:5;
+
+	//REGISTER lut_3d_reg_5to7804
+	struct {
+	UINT32 m_pLut2_r        : 11;
+	UINT32:5;
+	UINT32 m_pLut2_g        : 11;
+	UINT32:5;
+	} lut_3d_reg_5to7804[7800];
+
+	};
+
+	INT32 value32[7805];
+
+} LUT_3D_REG;
+
+#endif
+
+typedef LUT_3D_REG LUT_3D_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/mmu_tbu_x.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/mmu_tbu_x.h
new file mode 100644
index 000000000000..84ac6b1e7698
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/mmu_tbu_x.h
@@ -0,0 +1,107 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef MMU_TBU_X_REG_H
+#define MMU_TBU_X_REG_H
+
+typedef union {
+	struct {
+	//REGISTER mmu_tbu_reg_0
+	UINT32 tbu_en                 : 1;
+	UINT32 tbu_fbc_mode           : 1;
+	UINT32 tbu_plane_num          : 2;
+	UINT32 tbu_burst_limit_en     : 1;
+	UINT32 tlb_fetch_active_en    : 1;
+	UINT32:2;
+	UINT32 tbu_qos                : 4;
+	UINT32:20;
+
+	//REGISTER mmu_tbu_reg_1
+	UINT32 tbu_base_addr0_low     : 32;
+
+	//REGISTER mmu_tbu_reg_2
+	UINT32 tbu_base_addr0_high    : 12;
+	UINT32:20;
+
+	//REGISTER mmu_tbu_reg_3
+	UINT32 tbu_base_addr1_low     : 32;
+
+	//REGISTER mmu_tbu_reg_4
+	UINT32 tbu_base_addr1_high    : 12;
+	UINT32:20;
+
+	//REGISTER mmu_tbu_reg_5
+	UINT32 tbu_base_addr2_low     : 32;
+
+	//REGISTER mmu_tbu_reg_6
+	UINT32 tbu_base_addr2_high    : 12;
+	UINT32:20;
+
+	//REGISTER mmu_tbu_reg_7
+	UINT32 tbu_va0                : 32;
+
+	//REGISTER mmu_tbu_reg_8
+	UINT32 tbu_va1                : 32;
+
+	//REGISTER mmu_tbu_reg_9
+	UINT32 tbu_va2                : 32;
+
+	//REGISTER mmu_tbu_reg_10
+	UINT32 tbu_size0              : 16;
+	UINT32:16;
+
+	//REGISTER mmu_tbu_reg_11
+	UINT32 tbu_size1              : 16;
+	UINT32:16;
+
+	//REGISTER mmu_tbu_reg_12
+	UINT32 tbu_size2              : 16;
+	UINT32:16;
+
+	//REGISTER mmu_tbu_reg_13
+	UINT32 force_update_en        : 1;
+	UINT32 vsync_update_en        : 1;
+	UINT32 shadow_read_en         : 1;
+	UINT32:29;
+
+	//REGISTER mmu_tbu_reg_14
+	UINT32 force_update_pulse     : 1;
+	UINT32:31;
+
+	//REGISTER mmu_tbu_reg_15
+	UINT32 force_update_en_se     : 1;
+	UINT32 vsync_update_en_se     : 1;
+	UINT32 shadow_read_en_se      : 1;
+	UINT32:29;
+
+	//REGISTER mmu_tbu_reg_16
+	UINT32 force_update_pulse_se  : 1;
+	UINT32:31;
+
+	//REGISTER mmu_tbu_reg_17
+	UINT32 icg_override           : 1;
+	UINT32:31;
+
+	//REGISTER mmu_tbu_reg_18
+	UINT32 regbank_reset          : 1;
+	UINT32:31;
+
+	//REGISTER mmu_tbu_reg_19
+	UINT32 regbank_reset2         : 1;
+	UINT32:31;
+
+	//REGISTER mmu_tbu_reg_20
+	UINT32 tbu_nsaid              : 16;
+	UINT32:16;
+
+	};
+
+	INT32 value32[21];
+
+} MMU_TBU_X_REG;
+
+#endif
+typedef MMU_TBU_X_REG MMU_TBU_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/mmu_top.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/mmu_top.h
new file mode 100644
index 000000000000..3cedf7926986
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/mmu_top.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef MMU_TOP_REG_H
+#define MMU_TOP_REG_H
+
+typedef union {
+	struct {
+	//REGISTER mmu_top_reg_0
+	UINT32 burst_split_mode        : 1;
+	UINT32 burst_split_en          : 1;
+	UINT32:14;
+	UINT32 rdma_timelimit          : 16;
+
+	//REGISTER mmu_top_reg_1
+	UINT32 sram0_tlb_axi_port_sel  : 2;
+	UINT32 sram1_tlb_axi_port_sel  : 2;
+	UINT32:28;
+
+	//REGISTER mmu_top_reg_2
+	UINT32 dmac0_rd_outs_num       : 8;
+	UINT32 dmac0_arregion          : 4;
+	UINT32 dmac0_arcache           : 4;
+	UINT32 dmac0_burst_length      : 3;
+	UINT32:13;
+
+	//REGISTER mmu_top_reg_3
+	UINT32 dmac1_rd_outs_num       : 8;
+	UINT32 dmac1_arregion          : 4;
+	UINT32 dmac1_arcache           : 4;
+	UINT32 dmac1_burst_length      : 3;
+	UINT32:13;
+
+	//REGISTER mmu_top_reg_4
+	UINT32 axi_s0_ar_debug         : 32;
+
+	//REGISTER mmu_top_reg_5
+	UINT32 axi_m0_ar_debug         : 32;
+
+	//REGISTER mmu_top_reg_6
+	UINT32 axi_s0_aw_debug         : 32;
+
+	//REGISTER mmu_top_reg_7
+	UINT32 axi_m0_aw_debug         : 32;
+
+	//REGISTER mmu_top_reg_8
+	UINT32 axi_s1_ar_debug         : 32;
+
+	//REGISTER mmu_top_reg_9
+	UINT32 axi_m1_ar_debug         : 32;
+
+	//REGISTER mmu_top_reg_10
+	UINT32 axi_s1_aw_debug         : 32;
+
+	//REGISTER mmu_top_reg_11
+	UINT32 axi_m1_aw_debug         : 32;
+
+	//REGISTER mmu_top_reg_12
+	UINT32 tlb_cmd_num             : 20;
+	UINT32 tlb_miss_num            : 12;
+
+	//REGISTER mmu_top_reg_13
+	UINT32 tlb_wait_cycle          : 16;
+	UINT32:16;
+
+	//REGISTER mmu_top_reg_14
+	UINT32 tlb_wait_cycle_sel      : 7;
+	UINT32:1;
+	UINT32 tlb_cmd_num_sel         : 7;
+	UINT32:1;
+	UINT32 tlb_miss_num_sel        : 7;
+	UINT32 tlb_miss_num_clr        : 1;
+	UINT32:8;
+
+	//REGISTER mmu_top_reg_15
+	UINT32 tlb_wait_cycle_total    : 32;
+
+	//REGISTER mmu_top_reg_16
+	UINT32 tlb_cmd_num_total       : 32;
+
+	//REGISTER mmu_top_reg_17
+	UINT32 tlb_miss_num_total      : 32;
+
+	//REGISTER mmu_top_reg_18
+	UINT32 icg_override            : 1;
+	UINT32:31;
+
+	//REGISTER mmu_top_reg_19
+	UINT32 regbank_reset           : 1;
+	UINT32:31;
+
+	//REGISTER mmu_top_reg_20
+	UINT32 regbank_reset2          : 1;
+	UINT32:31;
+
+	//REGISTER mmu_top_reg_21
+	UINT32 axi_safty_addr_low      : 32;
+
+	//REGISTER mmu_top_reg_22
+	UINT32 axi_safty_addr_high     : 12;
+	UINT32:20;
+
+	};
+
+	INT32 value32[23];
+
+} MMU_TOP_REG;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/ops_hee.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/ops_hee.h
new file mode 100644
index 000000000000..bb4b07a24fc8
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/ops_hee.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SATURN_hee_OPS_H_
+#define _SATURN_hee_OPS_H_
+
+#include "../../spacemit_crtc.h"
+
+void saturn_hee_update_hdr_matrix(struct drm_plane *plane, struct spacemit_plane_state *spacemit_pstate);
+void saturn_hee_update_csc_matrix(struct drm_plane *plane, struct drm_plane_state *old_state);
+void saturn_hee_conf_dpuctrl_color_matrix(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+void saturn_hee_conf_scaler_x(struct drm_plane_state *state, struct cmdlist_regs *cl_scl);
+void saturn_hee_conf_scaler_coefs(struct drm_plane *plane, struct spacemit_plane_state *spacemit_pstate, struct cmdlist_regs
+		*cl_scl);
+void saturn_hee_enable_vsync(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, bool enable);
+void saturn_hee_enable_cfg_irq(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, bool enable);
+void saturn_hee_cfg_ready(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+void saturn_hee_sw_start(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+void saturn_hee_dpu_init(struct spacemit_crtc *a_crtc);
+void saturn_hee_plane_update_hw_channel(struct drm_plane *plane, int slice_id);
+void saturn_hee_plane_disable_hw_channel(struct drm_plane *plane, struct drm_plane_state *old_state);
+void saturn_hee_irq_enable(struct spacemit_crtc *a_crtc, bool enable);
+void saturn_hee_conf_dpuctrl(struct drm_crtc *crtc, struct drm_crtc_state *old_state);
+void saturn_hee_wb_config(struct spacemit_crtc *a_crtc, struct spacemit_wb *wb, struct drm_framebuffer *fb, struct cmdlist_regs *cl_wb, int slice_id);
+void saturn_hee_wb_disable(struct spacemit_crtc *a_crtc);
+int saturn_hee_is_wb_en(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+uint32_t saturn_hee_get_cfg_rdy(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+uint32_t saturn_hee_get_int_sts(struct spacemit_hw_device *hwdev, int dev_id);
+uint32_t saturn_hee_get_irq_bit(enum spacemit_dpu_irq irq_id, int dev_id);
+void saturn_hee_clr_int_sts(struct spacemit_crtc *a_crtc, enum spacemit_dpu_irq irq, int dev_id);
+void saturn_hee_dpu_disable(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+void saturn_hee_dpu_restart(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+int saturn_hee_dpu_stop_check(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+void saturn_hee_enable_cmdlist(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, int id, bool enable);
+void saturn_hee_cfg_cmdlist(struct spacemit_hw_device *hwdev, int id, u32 chy, u32 addrl, u32 addrh);
+void saturn_hee_rdma_contig_mem(struct spacemit_hw_device *hwdev, u8 tbu_id, phys_addr_t contig_pa, struct drm_framebuffer *fb, struct cmdlist_regs *cl_rdma, struct drm_plane *plane);
+void saturn_hee_rdma_dmmu(struct spacemit_hw_device *hwdev, u8 tbu_id, struct tbu_instance *tbu, struct drm_framebuffer *fb, u32 val, struct cmdlist_regs *cl_rdma, struct drm_plane *plane);
+void saturn_hee_wb_dmmu(struct spacemit_hw_device *hwdev, u8 tbu_id, struct tbu_instance *tbu, struct drm_framebuffer *fb, u32 val, u8 fbc_mode, int header_size, struct spacemit_wb *wb);
+int saturn_hee_get_cl_rdma_buf(struct spacemit_crtc *a_crtc);
+void saturn_hee_cmdlist_fill_data_row(struct cmdlist *cl, u32 strobe, u32 offset, u32 value[]);
+void saturn_hee_cmdlist_fill_conf_row(struct cmdlist *cl, struct spacemit_hw_device *hwdev, u8 crtc_id);
+void saturn_hee_wb_cmdlist(struct cmdlist *cl, struct spacemit_hw_device *hwdev, struct spacemit_drm_private *priv, u8 crtc_id, u8 dev_id);
+void saturn_hee_cmdlist_dump_node(struct cmdlist *cl);
+#ifdef CONFIG_SPACEMIT_DEBUG
+void hee_dpu_dump_reg(struct spacemit_crtc *a_crtc);
+void hee_dpu_dump_rdma_status(struct spacemit_drm_private *priv);
+#endif
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/postpipe.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/postpipe.h
new file mode 100644
index 000000000000..ac02143987ac
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/postpipe.h
@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef POSTPIPE_REG_H
+#define POSTPIPE_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_postpipe_reg_0
+	UINT32 dither_en           : 1;
+	UINT32 postpq_en           : 1;
+	UINT32 dither_mem_lp_en    : 1;
+	UINT32 rgb2yuv_en          : 1;
+	UINT32 yuv2rgb_en          : 1;
+	UINT32 split_en            : 1;
+	UINT32:26;
+
+	//REGISTER saturn_postpipe_reg_1
+	UINT32 rgb2yuv_matrix00    : 16;
+	UINT32 rgb2yuv_matrix01    : 16;
+
+	//REGISTER saturn_postpipe_reg_2
+	UINT32 rgb2yuv_matrix02    : 16;
+	UINT32 rgb2yuv_matrix03    : 16;
+
+	//REGISTER saturn_postpipe_reg_3
+	UINT32 rgb2yuv_matrix10    : 16;
+	UINT32 rgb2yuv_matrix11    : 16;
+
+	//REGISTER saturn_postpipe_reg_4
+	UINT32 rgb2yuv_matrix12    : 16;
+	UINT32 rgb2yuv_matrix13    : 16;
+
+	//REGISTER saturn_postpipe_reg_5
+	UINT32 rgb2yuv_matrix20    : 16;
+	UINT32 rgb2yuv_matrix21    : 16;
+
+	//REGISTER saturn_postpipe_reg_6
+	UINT32 rgb2yuv_matrix22    : 16;
+	UINT32 rgb2yuv_matrix23    : 16;
+
+	//REGISTER saturn_postpipe_reg_7
+	UINT32 yuv2rgb_matrix00    : 16;
+	UINT32 yuv2rgb_matrix01    : 16;
+
+	//REGISTER saturn_postpipe_reg_8
+	UINT32 yuv2rgb_matrix02    : 16;
+	UINT32 yuv2rgb_matrix03    : 16;
+
+	//REGISTER saturn_postpipe_reg_9
+	UINT32 yuv2rgb_matrix10    : 16;
+	UINT32 yuv2rgb_matrix11    : 16;
+
+	//REGISTER saturn_postpipe_reg_10
+	UINT32 yuv2rgb_matrix12    : 16;
+	UINT32 yuv2rgb_matrix13    : 16;
+
+	//REGISTER saturn_postpipe_reg_11
+	UINT32 yuv2rgb_matrix20    : 16;
+	UINT32 yuv2rgb_matrix21    : 16;
+
+	//REGISTER saturn_postpipe_reg_12
+	UINT32 yuv2rgb_matrix22    : 16;
+	UINT32 yuv2rgb_matrix23    : 16;
+
+	//REGISTER saturn_postpipe_reg_13
+	UINT32 m_inwidth           : 16;
+	UINT32 m_inheight          : 16;
+
+	//REGISTER saturn_postpipe_reg_14
+	UINT32 split_overlap       : 10;
+	UINT32:22;
+
+	//REGISTER saturn_postpipe_reg_15
+	struct {
+	UINT32:32;
+	} saturn_postpipe_reg_15[4];
+
+	//REGISTER saturn_postpipe_reg_19
+	UINT32 dither_mode         : 1;
+	UINT32 dither_auto_temp    : 1;
+	UINT32 dither_rotate_mode  : 2;
+	UINT32 dither_out_dpth0    : 4;
+	UINT32 dither_out_dpth1    : 4;
+	UINT32 dither_out_dpth2    : 4;
+	UINT32 dither_temp_value   : 8;
+	UINT32 dither_pattern_bit  : 3;
+	UINT32:5;
+
+	//REGISTER saturn_postpipe_reg_20
+	UINT32 dither_bayer_map00  : 8;
+	UINT32 dither_bayer_map01  : 8;
+	UINT32 dither_bayer_map02  : 8;
+	UINT32 dither_bayer_map03  : 8;
+
+	//REGISTER saturn_postpipe_reg_21
+	UINT32 dither_bayer_map04  : 8;
+	UINT32 dither_bayer_map05  : 8;
+	UINT32 dither_bayer_map06  : 8;
+	UINT32 dither_bayer_map07  : 8;
+
+	//REGISTER saturn_postpipe_reg_22
+	UINT32 dither_bayer_map10  : 8;
+	UINT32 dither_bayer_map11  : 8;
+	UINT32 dither_bayer_map12  : 8;
+	UINT32 dither_bayer_map13  : 8;
+
+	//REGISTER saturn_postpipe_reg_23
+	UINT32 dither_bayer_map14  : 8;
+	UINT32 dither_bayer_map15  : 8;
+	UINT32 dither_bayer_map16  : 8;
+	UINT32 dither_bayer_map17  : 8;
+
+	//REGISTER saturn_postpipe_reg_24
+	UINT32 dither_bayer_map20  : 8;
+	UINT32 dither_bayer_map21  : 8;
+	UINT32 dither_bayer_map22  : 8;
+	UINT32 dither_bayer_map23  : 8;
+
+	//REGISTER saturn_postpipe_reg_25
+	UINT32 dither_bayer_map24  : 8;
+	UINT32 dither_bayer_map25  : 8;
+	UINT32 dither_bayer_map26  : 8;
+	UINT32 dither_bayer_map27  : 8;
+
+	//REGISTER saturn_postpipe_reg_26
+	UINT32 dither_bayer_map30  : 8;
+	UINT32 dither_bayer_map31  : 8;
+	UINT32 dither_bayer_map32  : 8;
+	UINT32 dither_bayer_map33  : 8;
+
+	//REGISTER saturn_postpipe_reg_27
+	UINT32 dither_bayer_map34  : 8;
+	UINT32 dither_bayer_map35  : 8;
+	UINT32 dither_bayer_map36  : 8;
+	UINT32 dither_bayer_map37  : 8;
+
+	//REGISTER saturn_postpipe_reg_28
+	UINT32 dither_bayer_map40  : 8;
+	UINT32 dither_bayer_map41  : 8;
+	UINT32 dither_bayer_map42  : 8;
+	UINT32 dither_bayer_map43  : 8;
+
+	//REGISTER saturn_postpipe_reg_29
+	UINT32 dither_bayer_map44  : 8;
+	UINT32 dither_bayer_map45  : 8;
+	UINT32 dither_bayer_map46  : 8;
+	UINT32 dither_bayer_map47  : 8;
+
+	//REGISTER saturn_postpipe_reg_30
+	UINT32 dither_bayer_map50  : 8;
+	UINT32 dither_bayer_map51  : 8;
+	UINT32 dither_bayer_map52  : 8;
+	UINT32 dither_bayer_map53  : 8;
+
+	//REGISTER saturn_postpipe_reg_31
+	UINT32 dither_bayer_map54  : 8;
+	UINT32 dither_bayer_map55  : 8;
+	UINT32 dither_bayer_map56  : 8;
+	UINT32 dither_bayer_map57  : 8;
+
+	//REGISTER saturn_postpipe_reg_32
+	UINT32 dither_bayer_map60  : 8;
+	UINT32 dither_bayer_map61  : 8;
+	UINT32 dither_bayer_map62  : 8;
+	UINT32 dither_bayer_map63  : 8;
+
+	//REGISTER saturn_postpipe_reg_33
+	UINT32 dither_bayer_map64  : 8;
+	UINT32 dither_bayer_map65  : 8;
+	UINT32 dither_bayer_map66  : 8;
+	UINT32 dither_bayer_map67  : 8;
+
+	//REGISTER saturn_postpipe_reg_34
+	UINT32 dither_bayer_map70  : 8;
+	UINT32 dither_bayer_map71  : 8;
+	UINT32 dither_bayer_map72  : 8;
+	UINT32 dither_bayer_map73  : 8;
+
+	//REGISTER saturn_postpipe_reg_35
+	UINT32 dither_bayer_map74  : 8;
+	UINT32 dither_bayer_map75  : 8;
+	UINT32 dither_bayer_map76  : 8;
+	UINT32 dither_bayer_map77  : 8;
+
+	//REGISTER saturn_postpipe_reg_36
+	UINT32 preproc_loc         : 32;
+
+	//REGISTER saturn_postpipe_reg_37
+	UINT32 dither_proc_loc     : 32;
+
+	//REGISTER saturn_postpipe_reg_38
+	UINT32 split_location      : 32;
+
+	//REGISTER saturn_postpipe_reg_39
+	UINT32 p0_out_location     : 32;
+
+	//REGISTER saturn_postpipe_reg_40
+	UINT32 p1_out_location     : 32;
+
+	//REGISTER saturn_postpipe_reg_41
+	UINT32 pospq_valid         : 1;
+	UINT32 pospq_ready         : 1;
+	UINT32 acad_valid          : 1;
+	UINT32 acad_ready          : 1;
+	UINT32 hsv_valid           : 1;
+	UINT32 hsv_ready           : 1;
+	UINT32 endmatrix_valid     : 1;
+	UINT32 endmatrix_ready     : 1;
+	UINT32 to_scl_valid        : 1;
+	UINT32 scl_ready           : 1;
+	UINT32 lut3d_valid         : 1;
+	UINT32 lut3d_ready         : 1;
+	UINT32 ee_valid            : 1;
+	UINT32 ee_ready            : 1;
+	UINT32 usrgamma_valid      : 1;
+	UINT32 usrgamma_ready      : 1;
+	UINT32 dither_valid        : 1;
+	UINT32 dither_ready        : 1;
+	UINT32 rc_valid            : 1;
+	UINT32 rc_ready            : 1;
+	UINT32 split_valid         : 1;
+	UINT32 split_ready         : 1;
+	UINT32 dsc_p0_valid        : 1;
+	UINT32 dsc_p0_ready        : 1;
+	UINT32 dsc_p1_valid        : 1;
+	UINT32 dsc_p1_ready        : 1;
+	UINT32 postpipe_p0_valid   : 1;
+	UINT32 postpipe_p0_ready   : 1;
+	UINT32 postpipe_p1_valid   : 1;
+	UINT32 postpipe_p1_ready   : 1;
+	UINT32:2;
+
+	//REGISTER saturn_postpipe_reg_42
+	struct {
+	UINT32:32;
+	} saturn_postpipe_reg_42[3];
+
+	//REGISTER saturn_postpipe_reg_45
+	UINT32 force_update_en     : 1;
+	UINT32 vsync_update_en     : 1;
+	UINT32 shadow_read_en      : 1;
+	UINT32:29;
+
+	//REGISTER saturn_postpipe_reg_46
+	UINT32 force_update_pulse  : 1;
+	UINT32:31;
+
+	//REGISTER saturn_postpipe_reg_47
+	UINT32 icg_override        : 1;
+	UINT32:31;
+
+	//REGISTER saturn_postpipe_reg_48
+	UINT32 trigger             : 1;
+	UINT32:31;
+
+	//REGISTER saturn_postpipe_reg_49
+	UINT32 trigger2            : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[50];
+
+} POSTPIPE_REG;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/prepipe_x.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/prepipe_x.h
new file mode 100644
index 000000000000..4b32900e4c3d
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/prepipe_x.h
@@ -0,0 +1,93 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef PREPIPE_X_REG_H
+#define PREPIPE_X_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_prepipe_top_reg_0
+	UINT32:5;
+	UINT32 m_ncolor_key_en        : 1;
+	UINT32:26;
+
+	//REGISTER saturn_prepipe_top_reg_1
+	struct {
+	UINT32:32;
+	} saturn_prepipe_top_reg_1[76];
+
+	//REGISTER saturn_prepipe_top_reg_77
+	UINT32:16;
+	UINT32 m_pcolor_key_R_thr0    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_prepipe_top_reg_78
+	UINT32 m_pcolor_key_R_thr1    : 10;
+	UINT32:6;
+	UINT32 m_pcolor_key_G_thr0    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_prepipe_top_reg_79
+	UINT32 m_pcolor_key_G_thr1    : 10;
+	UINT32:6;
+	UINT32 m_pcolor_key_B_thr0    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_prepipe_top_reg_80
+	UINT32 m_pcolor_key_B_thr1    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_prepipe_top_reg_81
+	UINT32 prepq_secu_en          : 1;
+	UINT32:31;
+
+	//REGISTER saturn_prepipe_top_reg_82
+	UINT32 DITHER_EN              : 1;
+	UINT32:31;
+
+	//REGISTER saturn_prepipe_top_reg_83
+	UINT32 DITHER_CG2_AUTO_EN     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_prepipe_top_reg_84
+	UINT32 force_update_en        : 1;
+	UINT32 vsync_update_en        : 1;
+	UINT32 shadow_read_en         : 1;
+	UINT32:29;
+
+	//REGISTER saturn_prepipe_top_reg_85
+	UINT32 force_update_pulse     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_prepipe_top_reg_86
+	UINT32 force_update_en_se     : 1;
+	UINT32 vsync_update_en_se     : 1;
+	UINT32 shadow_read_en_se      : 1;
+	UINT32:29;
+
+	//REGISTER saturn_prepipe_top_reg_87
+	UINT32 force_update_pulse_se  : 1;
+	UINT32:31;
+
+	//REGISTER saturn_prepipe_top_reg_88
+	UINT32 icg_override           : 1;
+	UINT32:31;
+
+	//REGISTER saturn_prepipe_top_reg_89
+	UINT32 trigger                : 1;
+	UINT32:31;
+
+	//REGISTER saturn_prepipe_top_reg_90
+	UINT32 trigger2               : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[91];
+
+} PREPIPE_X_REG;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/rc.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/rc.h
new file mode 100644
index 000000000000..dcb96a6f9333
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/rc.h
@@ -0,0 +1,196 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef RC_REG_H
+#define RC_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_rc_full_reg_0
+	UINT32 module_enable          : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_1
+	UINT32 m_nImgWidth            : 16;
+	UINT32 m_nImgHeight           : 16;
+
+	//REGISTER saturn_rc_full_reg_2
+	UINT32 enbuf_cfg_done         : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_3
+	struct {
+	UINT32:32;
+	} saturn_rc_full_reg_3[4];
+
+	//REGISTER saturn_rc_full_reg_7
+	UINT32 m_nRCEn_0              : 1;
+	UINT32:15;
+	UINT32 m_nRCStartX_0          : 16;
+
+	//REGISTER saturn_rc_full_reg_8
+	UINT32 m_nRCStartY_0          : 16;
+	UINT32 m_nRCWidth_0           : 10;
+	UINT32:6;
+
+	//REGISTER saturn_rc_full_reg_9
+	UINT32 m_nRCHeight_0          : 10;
+	UINT32 m_nRCSymmetric0        : 1;
+	UINT32:5;
+	UINT32 m_nRCSymmStarX0        : 16;
+
+	//REGISTER saturn_rc_full_reg_10
+	UINT32 m_nRCOffset0           : 16;
+	UINT32:16;
+
+	//REGISTER saturn_rc_full_reg_11
+	UINT32 m_nRCEn_1              : 1;
+	UINT32:15;
+	UINT32 m_nRCStartX_1          : 16;
+
+	//REGISTER saturn_rc_full_reg_12
+	UINT32 m_nRCStartY_1          : 16;
+	UINT32 m_nRCWidth_1           : 10;
+	UINT32:6;
+
+	//REGISTER saturn_rc_full_reg_13
+	UINT32 m_nRCHeight_1          : 10;
+	UINT32 m_nRCSymmetric1        : 1;
+	UINT32:5;
+	UINT32 m_nRCSymmStarX1        : 16;
+
+	//REGISTER saturn_rc_full_reg_14
+	UINT32 m_nRCOffset1           : 16;
+	UINT32:16;
+
+	//REGISTER saturn_rc_full_reg_15
+	UINT32 m_nRCEn_2              : 1;
+	UINT32:15;
+	UINT32 m_nRCStartX_2          : 16;
+
+	//REGISTER saturn_rc_full_reg_16
+	UINT32 m_nRCStartY_2          : 16;
+	UINT32 m_nRCWidth_2           : 10;
+	UINT32:6;
+
+	//REGISTER saturn_rc_full_reg_17
+	UINT32 m_nRCHeight_2          : 10;
+	UINT32 m_nRCSymmetric2        : 1;
+	UINT32:5;
+	UINT32 m_nRCSymmStarX2        : 16;
+
+	//REGISTER saturn_rc_full_reg_18
+	UINT32 m_nRCOffset2           : 16;
+	UINT32:16;
+
+	//REGISTER saturn_rc_full_reg_19
+	UINT32 m_nRCEn_3              : 1;
+	UINT32:15;
+	UINT32 m_nRCStartX_3          : 16;
+
+	//REGISTER saturn_rc_full_reg_20
+	UINT32 m_nRCStartY_3          : 16;
+	UINT32 m_nRCWidth_3           : 10;
+	UINT32:6;
+
+	//REGISTER saturn_rc_full_reg_21
+	UINT32 m_nRCHeight_3          : 10;
+	UINT32 m_nRCSymmetric3        : 1;
+	UINT32:5;
+	UINT32 m_nRCSymmStarX3        : 16;
+
+	//REGISTER saturn_rc_full_reg_22
+	UINT32 m_nRCOffset3           : 16;
+	UINT32:16;
+
+	//REGISTER saturn_rc_full_reg_23
+	UINT32 m_nRCEn_4              : 1;
+	UINT32:15;
+	UINT32 m_nRCStartX_4          : 16;
+
+	//REGISTER saturn_rc_full_reg_24
+	UINT32 m_nRCStartY_4          : 16;
+	UINT32 m_nRCWidth_4           : 10;
+	UINT32:6;
+
+	//REGISTER saturn_rc_full_reg_25
+	UINT32 m_nRCHeight_4          : 10;
+	UINT32 m_nRCSymmetric4        : 1;
+	UINT32:5;
+	UINT32 m_nRCSymmStarX4        : 16;
+
+	//REGISTER saturn_rc_full_reg_26
+	UINT32 m_nRCOffset4           : 16;
+	UINT32:16;
+
+	//REGISTER saturn_rc_full_reg_27
+	UINT32 m_nRCEn_5              : 1;
+	UINT32:15;
+	UINT32 m_nRCStartX_5          : 16;
+
+	//REGISTER saturn_rc_full_reg_28
+	UINT32 m_nRCStartY_5          : 16;
+	UINT32 m_nRCWidth_5           : 10;
+	UINT32:6;
+
+	//REGISTER saturn_rc_full_reg_29
+	UINT32 m_nRCHeight_5          : 10;
+	UINT32 m_nRCSymmetric5        : 1;
+	UINT32:5;
+	UINT32 m_nRCSymmStarX5        : 16;
+
+	//REGISTER saturn_rc_full_reg_30
+	UINT32 m_nRCOffset5           : 16;
+	UINT32:16;
+
+	//REGISTER saturn_rc_full_reg_31
+	UINT32 cfg_se                 : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_32
+	UINT32 force_update_en        : 1;
+	UINT32 vsync_update_en        : 1;
+	UINT32 shadow_read_en         : 1;
+	UINT32:29;
+
+	//REGISTER saturn_rc_full_reg_33
+	UINT32 force_update_pulse     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_34
+	UINT32 force_update_en_se     : 1;
+	UINT32 vsync_update_en_se     : 1;
+	UINT32 shadow_read_en_se      : 1;
+	UINT32:29;
+
+	//REGISTER saturn_rc_full_reg_35
+	UINT32 force_update_pulse_se  : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_36
+	UINT32 icg_override           : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_37
+	UINT32 trigger                : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_38
+	UINT32 trigger2               : 1;
+	UINT32:31;
+
+	//REGISTER saturn_rc_full_reg_39
+	struct {
+	UINT32 compress_data          : 32;
+	} saturn_rc_full_reg_39[4096];
+
+	};
+
+	INT32 value32[4135];
+
+} RC_REG;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/rdma_path.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/rdma_path.h
new file mode 100644
index 000000000000..effce4d42623
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/rdma_path.h
@@ -0,0 +1,246 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef RDMA_PATH_REG_H
+#define RDMA_PATH_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_dma_path_reg_0
+	UINT32 layer_mode             : 2;
+	UINT32 outstanding_num        : 5;
+	UINT32:1;
+	UINT32 req_conti_num          : 5;
+	UINT32 layer_cmpsr_id         : 3;
+	UINT32 axi_port_sel           : 2;
+	UINT32:6;
+	UINT32 rdma_burst_len         : 8;
+
+	//REGISTER saturn_dma_path_reg_1
+	UINT32 is_two_layers          : 1;
+	UINT32 is_offline             : 1;
+	UINT32 hdr_osd_num            : 4;
+	UINT32:26;
+
+	//REGISTER saturn_dma_path_reg_2
+	UINT32 cmpsr_y_offset0        : 16;
+	UINT32:16;
+
+	//REGISTER saturn_dma_path_reg_3
+	struct {
+	UINT32:32;
+	} saturn_dma_path_reg_3[6];
+
+	//REGISTER saturn_dma_path_reg_9
+	UINT32 base_addr0_low_ly0     : 32;
+
+	//REGISTER saturn_dma_path_reg_10
+	UINT32 base_addr0_high_ly0    : 2;
+	UINT32:30;
+
+	//REGISTER saturn_dma_path_reg_11
+	UINT32 base_addr1_low_ly0     : 32;
+
+	//REGISTER saturn_dma_path_reg_12
+	UINT32 base_addr1_high_ly0    : 2;
+	UINT32:30;
+
+	//REGISTER saturn_dma_path_reg_13
+	UINT32 base_addr2_low_ly0     : 32;
+
+	//REGISTER saturn_dma_path_reg_14
+	UINT32 base_addr2_high_ly0    : 2;
+	UINT32:30;
+
+	//REGISTER saturn_dma_path_reg_15
+	UINT32 rdma_stride0_layer0    : 16;
+	UINT32 rdma_stride1_layer0    : 16;
+
+	//REGISTER saturn_dma_path_reg_16
+	UINT32 img_width_ly0          : 16;
+	UINT32 img_height_ly0         : 16;
+
+	//REGISTER saturn_dma_path_reg_17
+	UINT32 bbox_start_x_ly0       : 16;
+	UINT32 bbox_start_y_ly0       : 16;
+
+	//REGISTER saturn_dma_path_reg_18
+	UINT32 bbox_end_x_ly0         : 16;
+	UINT32 bbox_end_y_ly0         : 16;
+
+	//REGISTER saturn_dma_path_reg_19
+	struct {
+	UINT32:32;
+	} saturn_dma_path_reg_19[10];
+
+	//REGISTER saturn_dma_path_reg_29
+	UINT32 pixel_format           : 6;
+	UINT32 uv_swap                : 1;
+	UINT32:1;
+	UINT32 rot_mode_ly0           : 3;
+	UINT32 rot_mode_ly1           : 3;
+	UINT32:18;
+
+	//REGISTER saturn_dma_path_reg_30
+	UINT32 fbc_split_mode         : 1;
+	UINT32 fbc_yuv_transform      : 1;
+	UINT32 fbc_sb_layout          : 1;
+	UINT32 fbc_tile_type          : 1;
+	UINT32:28;
+
+	//REGISTER saturn_dma_path_reg_31
+	UINT32 fbc_mem_size           : 16;
+	UINT32 fbc_mem_base_addr      : 12;
+	UINT32 fbc_mem_map            : 1;
+	UINT32 dec_line_num_sw        : 1;
+	UINT32 sw_dec_line_num        : 2;
+
+	//REGISTER saturn_dma_path_reg_32
+	UINT32 nsaid                  : 4;
+	UINT32:28;
+
+	//REGISTER saturn_dma_path_reg_33
+	UINT32 csc_matrix00           : 14;
+	UINT32:2;
+	UINT32 csc_matrix01           : 14;
+	UINT32:2;
+
+	//REGISTER saturn_dma_path_reg_34
+	UINT32 csc_matrix02           : 14;
+	UINT32:2;
+	UINT32 csc_matrix03           : 14;
+	UINT32:2;
+
+	//REGISTER saturn_dma_path_reg_35
+	UINT32 csc_matrix10           : 14;
+	UINT32:2;
+	UINT32 csc_matrix11           : 14;
+	UINT32:2;
+
+	//REGISTER saturn_dma_path_reg_36
+	UINT32 csc_matrix12           : 14;
+	UINT32:2;
+	UINT32 csc_matrix13           : 14;
+	UINT32:2;
+
+	//REGISTER saturn_dma_path_reg_37
+	UINT32 csc_matrix20           : 14;
+	UINT32:2;
+	UINT32 csc_matrix21           : 14;
+	UINT32:2;
+
+	//REGISTER saturn_dma_path_reg_38
+	UINT32 csc_matrix22           : 14;
+	UINT32:2;
+	UINT32 csc_matrix23           : 14;
+	UINT32:2;
+
+	//REGISTER saturn_dma_path_reg_39
+	UINT32 alpha0_ly0             : 12;
+	UINT32:4;
+	UINT32 alpha1_ly0             : 12;
+	UINT32:4;
+
+	//REGISTER saturn_dma_path_reg_40
+	UINT32 alpha2_ly0             : 12;
+	UINT32:4;
+	UINT32 alpha3_ly0             : 12;
+	UINT32:4;
+
+	//REGISTER saturn_dma_path_reg_41
+	UINT32 rdma_urgent_up         : 8;
+	UINT32 rdma_urgent_down       : 8;
+	UINT32:16;
+
+	//REGISTER saturn_dma_path_reg_42
+	UINT32 sw_qos_rsv0            : 1;
+	UINT32 sw_urgent_en           : 1;
+	UINT32 sw_urgent              : 1;
+	UINT32:5;
+	UINT32 offl_prefetch_lines    : 8;
+	UINT32:16;
+
+	//REGISTER saturn_dma_path_reg_43
+	UINT32 rdma_dbg_irq_raw       : 32;
+
+	//REGISTER saturn_dma_path_reg_44
+	UINT32 rdma_dbg_irq_mask      : 32;
+
+	//REGISTER saturn_dma_path_reg_45
+	UINT32 rdma_dbg_irq_status    : 32;
+
+	//REGISTER saturn_dma_path_reg_46
+	UINT32 prefetch_osd_num       : 5;
+	UINT32:3;
+	UINT32 prefetch_hdr_osd_num   : 4;
+	UINT32 osd_sub_step           : 2;
+	UINT32:2;
+	UINT32 osd_num_min            : 5;
+	UINT32:3;
+	UINT32 hdr_osd_num_min        : 4;
+	UINT32 pp_buf_switch_num      : 3;
+	UINT32:1;
+
+	//REGISTER saturn_dma_path_reg_47
+	UINT32 rdma_dbg_info0         : 32;
+
+	//REGISTER saturn_dma_path_reg_48
+	UINT32 rdma_dbg_info1         : 32;
+
+	//REGISTER saturn_dma_path_reg_49
+	UINT32 rdma_dbg_info2         : 32;
+
+	//REGISTER saturn_dma_path_reg_50
+	UINT32 rdma_dbg_info3         : 32;
+
+	//REGISTER saturn_dma_path_reg_51
+	UINT32 rdma_dbg_info4         : 32;
+
+	//REGISTER saturn_dma_path_reg_52
+	UINT32 secu_en                : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_path_reg_53
+	UINT32 force_update_en        : 1;
+	UINT32 vsync_update_en        : 1;
+	UINT32 shadow_read_en         : 1;
+	UINT32:29;
+
+	//REGISTER saturn_dma_path_reg_54
+	UINT32 force_update_pulse     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_path_reg_55
+	UINT32 force_update_en_se     : 1;
+	UINT32 vsync_update_en_se     : 1;
+	UINT32 shadow_read_en_se      : 1;
+	UINT32:29;
+
+	//REGISTER saturn_dma_path_reg_56
+	UINT32 force_update_pulse_se  : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_path_reg_57
+	UINT32 icg_override           : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_path_reg_58
+	UINT32 regbank_reset          : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_path_reg_59
+	UINT32 regbank_reset2         : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[60];
+
+} RDMA_PATH_REG;
+
+#endif
+
+typedef RDMA_PATH_REG RDMA_PATH_X_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/rdma_top.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/rdma_top.h
new file mode 100644
index 000000000000..6ed29916f0ea
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/rdma_top.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef RDMA_TOP_REG_H
+#define RDMA_TOP_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_dma_top_reg_0
+	UINT32 dbg_en                  : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_top_reg_1
+	UINT32 image_rr_ratio          : 8;
+	UINT32 round_robin_mode        : 1;
+	UINT32 pixel_num_th            : 6;
+	UINT32:17;
+
+	//REGISTER saturn_dma_top_reg_2
+	UINT32 rdma_timeout_limit      : 16;
+	UINT32 wdma_timeout_limit      : 16;
+
+	//REGISTER saturn_dma_top_reg_3
+	UINT32 dmac0_rstn_pwr          : 1;
+	UINT32 dmac0_rst_req           : 1;
+	UINT32 dmac0_burst_length      : 3;
+	UINT32:3;
+	UINT32 dmac0_arcache           : 4;
+	UINT32 dmac0_awcache           : 4;
+	UINT32 dmac0_arregion          : 4;
+	UINT32 dmac0_awregion          : 4;
+	UINT32:8;
+
+	//REGISTER saturn_dma_top_reg_4
+	UINT32 dmac1_rstn_pwr          : 1;
+	UINT32 dmac1_rst_req           : 1;
+	UINT32 dmac1_burst_length      : 3;
+	UINT32:3;
+	UINT32 dmac1_arcache           : 4;
+	UINT32 dmac1_awcache           : 4;
+	UINT32 dmac1_arregion          : 4;
+	UINT32 dmac1_awregion          : 4;
+	UINT32:8;
+
+	//REGISTER saturn_dma_top_reg_5
+	UINT32 dmac2_rstn_pwr          : 1;
+	UINT32 dmac2_rst_req           : 1;
+	UINT32 dmac2_burst_length      : 3;
+	UINT32:3;
+	UINT32 dmac2_arcache           : 4;
+	UINT32 dmac2_awcache           : 4;
+	UINT32 dmac2_arregion          : 4;
+	UINT32 dmac2_awregion          : 4;
+	UINT32:8;
+
+	//REGISTER saturn_dma_top_reg_6
+	UINT32 dmac3_rstn_pwr          : 1;
+	UINT32 dmac3_rst_req           : 1;
+	UINT32 dmac3_burst_length      : 3;
+	UINT32:3;
+	UINT32 dmac3_arcache           : 4;
+	UINT32 dmac3_awcache           : 4;
+	UINT32 dmac3_arregion          : 4;
+	UINT32 dmac3_awregion          : 4;
+	UINT32:8;
+
+	//REGISTER saturn_dma_top_reg_7
+	UINT32 online_rqos             : 4;
+	UINT32 offline_rqos            : 4;
+	UINT32 online_wqos             : 4;
+	UINT32 offline_wqos            : 4;
+	UINT32 cmdlist_rqos            : 4;
+	UINT32:12;
+
+	//REGISTER saturn_dma_top_reg_8
+	UINT32 dmac0_rd_oust_num       : 8;
+	UINT32 dmac0_wr_oust_num       : 8;
+	UINT32:16;
+
+	//REGISTER saturn_dma_top_reg_9
+	UINT32 dmac1_rd_oust_num       : 8;
+	UINT32 dmac1_wr_oust_num       : 8;
+	UINT32:16;
+
+	//REGISTER saturn_dma_top_reg_10
+	UINT32 dmac2_rd_oust_num       : 8;
+	UINT32 dmac2_wr_oust_num       : 8;
+	UINT32:16;
+
+	//REGISTER saturn_dma_top_reg_11
+	UINT32 dmac3_rd_oust_num       : 8;
+	UINT32 dmac3_wr_oust_num       : 8;
+	UINT32:16;
+
+	//REGISTER saturn_dma_top_reg_12
+	UINT32 cmdlist_dbg_irq_raw     : 32;
+
+	//REGISTER saturn_dma_top_reg_13
+	UINT32 cmdlist_dbg_irq_mask    : 32;
+
+	//REGISTER saturn_dma_top_reg_14
+	UINT32 cmdlist_dbg_irq_status  : 32;
+
+	//REGISTER saturn_dma_top_reg_15
+	UINT32 wb0_dbg_irq_raw         : 32;
+
+	//REGISTER saturn_dma_top_reg_16
+	UINT32 wb0_dbg_irq_mask        : 32;
+
+	//REGISTER saturn_dma_top_reg_17
+	UINT32 wb0_dbg_irq_status      : 32;
+
+	//REGISTER saturn_dma_top_reg_18
+	UINT32 arb_debug_info_axi0     : 32;
+
+	//REGISTER saturn_dma_top_reg_19
+	UINT32 arb_debug_info_axi1     : 32;
+
+	//REGISTER saturn_dma_top_reg_20
+	UINT32 arb_debug_info_axi2     : 32;
+
+	//REGISTER saturn_dma_top_reg_21
+	UINT32 arb_debug_info_axi3     : 32;
+
+	//REGISTER saturn_dma_top_reg_22
+	UINT32 icg_override            : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_top_reg_23
+	UINT32 regbank_reset           : 1;
+	UINT32:31;
+
+	//REGISTER saturn_dma_top_reg_24
+	UINT32 regbank_reset2          : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[25];
+
+} RDMA_TOP_REG;
+
+#endif
+
+typedef RDMA_TOP_REG DMA_TOP_REG;
\ No newline at end of file
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/reg_map.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/reg_map.h
new file mode 100644
index 000000000000..d07a0f7d9998
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/reg_map.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SATURN_REG_MAP_H_
+#define _SATURN_REG_MAP_H_
+
+#ifndef UINT32
+typedef uint32_t UINT32;
+#endif
+
+#ifndef INT32
+typedef uint32_t INT32;
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/reg_map_hee.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/reg_map_hee.h
new file mode 100644
index 000000000000..271de757ef74
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/reg_map_hee.h
@@ -0,0 +1,256 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SATURN_REG_MAP_hee_H_
+#define _SATURN_REG_MAP_hee_H_
+
+/* Base address, then bit definitions */
+
+#ifdef ZYNC_FPGA
+#define SATURN_REG_BASE 0x80000000
+#define DSI_A_BASE 0x8421A000
+#define I2C_NO	1
+#else
+#define SATURN_REG_BASE 0xc0340000
+#define DSI_A_BASE 0xd421A000
+#define I2C_NO 0
+#endif
+
+#define SATURN_REG_SIZE 0x60000
+
+#define DPU_TOP_BASE_ADDR 0x0
+#define DPU_CTRL_BASE_ADDR 0x3c0 /* dpu_ctl_top start address */
+#define CMDLIST_BASE_ADDR 0x500
+#define DPU_SCENE_CTRL1_BASE_ADDR 0x340
+#define DPU_SCENE_CTRL2_BASE_ADDR 0x380
+#define DPU_INT_BASE_ADDR 0x700
+#define DPU_DSC_ENC_TOP_BASE_ADDR 0x41100
+
+#define DPU_ONLINE_IRQ_MSK	(3 * 4)
+#define DPU_ONLINE_IRQ_STS	(12 * 4)
+#define DPU_OFFLINE_IRQ_MSK	(6 * 4)
+#define DPU_OFFLINE_IRQ_STS	(15 * 4)
+
+#define RDMA_SIZE 0x400
+#define DMA_TOP_BASE_ADDR (RDMA0_BASE_ADDR + 0x280)
+
+#define RDMA0_BASE_ADDR (0x00001000)
+#define RDMA1_BASE_ADDR (RDMA0_BASE_ADDR + 0x1000)
+#define RDMA2_BASE_ADDR (RDMA0_BASE_ADDR + 0xa000)
+#define RDMA3_BASE_ADDR (RDMA0_BASE_ADDR + 0xb000)
+// #define RDMA4_BASE_ADDR (RDMA0_BASE_ADDR + 0xc000)
+// #define RDMA5_BASE_ADDR (RDMA0_BASE_ADDR + 0xd000)
+
+// #define LP_SIZE 0x600
+// #define LP0_BASE_ADDR (0x1000 + 0x0400 )
+// #define LP1_BASE_ADDR (0x1000 + 0x1400 )
+// #define LP2_BASE_ADDR (0x1000 + 0xa400 )
+// #define LP3_BASE_ADDR (0x1000 + 0xb400 )
+// #define LP4_BASE_ADDR (0x1000 + 0xc400 )
+// #define LP5_BASE_ADDR (0x1000 + 0xd400 )
+
+#define PREPIPE1_USR_GAMMA 0x2600
+#define PREPIPE3_USR_GAMMA 0xb600
+
+//TODO MMU NOT checked
+#define MMU_BASE_ADDR (0x1100)
+#define MMU_TOP_BASE_ADDR (MMU_BASE_ADDR + 0x100)
+#define MMU_TBU_BASE_ADDR (MMU_BASE_ADDR)
+
+#define TBU0_ADDR 0x1100
+#define TBU1_ADDR 0x1180
+#define TBU2_ADDR 0x2100
+#define TBU3_ADDR 0x2180
+#define TBU4_ADDR 0xb100
+#define TBU5_ADDR 0xb180
+#define TBU6_ADDR 0xc100
+#define TBU7_ADDR 0xc180
+#define WB_TBU_ADDR 0x50400
+
+extern unsigned int tbu_offset_array[];
+extern unsigned int tbu_top_offset_array[];
+#define TBU_NUM 13
+#define TBU_CMDLIST_NUM 13  /*Number of tpus available for cmdList configuration*/
+#define EXTRA_ENTRY 60
+#define TBU_TOP_NUM 1
+#define TBU_CTRL 0x0
+#define TBU_VALIDE_SIZE (sizeof(MMU_TBU_X_REG))
+#define MMU_TOP_VALID_SIZE (sizeof(MMU_TOP_REG))
+/* return the ith TBU offset comparing to TBU base */
+#define TBU_OFFSET(i) \
+	tbu_offset_array[i]
+#define WB0_HANDLE_POS 12
+#define WB1_HANDLE_POS 23
+#define TBU_RDMA_START 0
+#define TBU_RDMA_END   7
+#define TBU_WB_POSITION_START  12
+#define TBU_WB_POSITION_END    12
+#define TBU_QOS                4
+#define TBU_TOP_ADDR(i)  tbu_top_offset_array[i]
+#define TBU_BASE_ADDR0_HIGH_MSK 0xFFF
+#define TBU_BASE_ADDR0_LOW_MSK 0xFFFFFFFF
+#define TBU_BASE_ADDR1_HIGH_MSK 0xFFF
+#define TBU_BASE_ADDR1_LOW_MSK 0xFFFFFFFF
+#define TBU_BASE_ADDR2_HIGH_MSK 0xFFF
+#define TBU_BASE_ADDR2_LOW_MSK 0xFFFFFFFF
+#define TBU_VA0_MSK 0xFFFFFFFF
+#define TBU_VA1_MSK 0xFFFFFFFF
+#define TBU_VA2_MSK 0xFFFFFFFF
+
+#define CMP_SIZE 0x18000
+#define CMP0_BASE_ADDR 0x00018000
+#define CMP1_BASE_ADDR (CMP0_BASE_ADDR + 0x18000)
+#define CMP2_BASE_ADDR (CMP0_BASE_ADDR + 0x30000)
+
+#define PREPIPE_SCAL0_BASE_ADDR 0x14000
+#define PREPIPE_SCAL1_BASE_ADDR 0x14400
+#define PREPIPE_SCAL2_BASE_ADDR 0x14800
+#define PREPIPE_SCAL3_BASE_ADDR 0x14c00
+
+#define POST_PQ_TOP_ADDR (CMP0_BASE_ADDR + 0x400)
+#define POST_PIPE_ADDR (CMP1_BASE_ADDR + 0x300)
+
+#define GAMMA_BASE_ADDR (CMP1_BASE_ADDR + 0x8900)
+#define LTM_BASE_ADDR (CMP1_BASE_ADDR + 0x600)
+
+#define RC_BASE 0x39100
+
+#define PP1_BASE_ADDR LTM_BASE_ADDR
+
+#define WB_CORE_VALID_SIZE (sizeof(WB_ASIC_REG))
+#define WB_TOP_BASE_ADDR (0x00050000 + 0x100)
+#define WB0_TOP_BASE_ADDR (0x00050000 + 0x200)
+#define WB0_SCALER_BASE_ADDR (WB0_TOP_BASE_ADDR + 0x500)
+#define WB1_TOP_BASE_ADDR (0x00050800 + 0x200)
+#define WB1_SCALER_BASE_ADDR (WB1_TOP_BASE_ADDR + 0x500)
+
+#define DPU_SCENE_CTL_BASE_ADDR (0x300)
+#define DPU_SCENE_CTL_SIZE 0x40
+#define DPU_SCENE_CTL_ADDR(i) (DPU_SCENE_CTL_BASE_ADDR + i * DPU_SCENE_CTL_SIZE)
+
+#define TMG0_BASE_ADDR (0x00051000 + 0x200)
+#define TMG1_BASE_ADDR (0x00054000 + 0x200)
+#define TMG2_BASE_ADDR (0x00057000)
+
+#define POST_LUT3D_ADDR (CMP1_BASE_ADDR + 0xf00)
+#define ACAD_ADDR (CMP1_BASE_ADDR + 0xa00)
+#define EE_ADDR (CMP1_BASE_ADDR + 0xd00)
+
+//#define OUTCTL_SIZE 0x8800
+
+/* for onl0 */
+#define INT_CTRL0_FUNC_STS 0x24
+#define INT_CTRL0_FUNC_RAW 0x48
+/* for cmb */
+#define INT_CTRL1_FUNC_STS 0x30
+#define INT_CTRL1_FUNC_RAW 0x58
+/* for offline */
+#define INT_CTRL2_FUNC_STS 0x3c
+#define INT_CTRL2_FUNC_RAW 0x68
+
+#define ONLINE_EOF BIT(1)
+#define ONLINE_FRM_TIMING_CFG_EOF (BIT(2) | BIT(3))
+#define ONLINE_WB0_FRAME_DONE BIT(6)
+#define ONLINE_WB1_FRAME_DONE BIT(7)
+#define ONLINE_FRAME_TIMING_UNFLOW BIT(8)
+#define ONLINE_CMDLIST_CH_FRM_CFG_DONE (0x1ff << 11)
+
+#define OFFLINE_WB0_FRAME_DONE BIT(1)
+#define OFFLINE_WB1_FRAME_DONE BIT(2)
+#define CMB_WB_SLICE_DONE  (BIT(24) | BIT(25))
+#define OFF_WB_SLICE_DONE  (BIT(3) | BIT(4))
+#define OFFLINE_CMDLIST_CH_FRM_CFG_DONE (0x1ff << 5)
+
+#define LUT3D_CFG_DONE (0x4)
+
+#define WB_ADDR_HIGH_BIT_MASK 0xFFF
+#define BG_COLOR_SHIFT 2
+
+#define CMDLIST_CH_Y                 0x188
+#define CMDLIST_CH_START_CMPS_Y      0x90
+#define CMDLIST_CFG_READY            0x34
+
+#define OFFLINE_COMPOSE_WB_POS 22
+
+typedef enum {
+	E_DPU_TOP_REG = 0,
+	E_DPU_SCENE_CTRL1_REG,
+	E_DPU_SCENE_CTRL2_REG,
+	E_DPU_CMDLIST_REG,
+	E_DPU_INT_REG,
+	E_DMA_TOP_CTRL_REG,
+	E_RDMA_LAYER0_REG,
+	E_RDMA_LAYER1_REG,
+	E_RDMA_LAYER2_REG,
+	E_RDMA_LAYER3_REG,
+	E_RDMA_LAYER4_REG,
+	E_RDMA_LAYER5_REG,
+	E_MMU_TBU0_REG,
+	E_MMU_TBU1_REG,
+	E_MMU_TBU2_REG,
+	E_MMU_TBU3_REG,
+	E_MMU_TBU4_REG,
+	E_MMU_TBU5_REG,
+	E_MMU_TBU6_REG,
+	E_MMU_TBU7_REG,
+	E_MMU_TBU8_REG,
+	E_MMU_TBU9_REG,
+	E_MMU_TBU10_REG,
+	E_MMU_TBU11_REG,
+	E_MMU_TBU12_REG,
+	E_MMU_TBU13_REG,
+	E_MMU_TOP_REG,
+	E_LP0_REG,
+	E_LP1_REG,
+	E_LP2_REG,
+	E_LP3_REG,
+	E_LP4_REG,
+	E_LP5_REG,
+	E_COMPOSER1_REG,
+	E_COMPOSER2_REG,
+	E_SCALER0_REG,
+	E_SCALER2_REG,
+	E_SCALER3_REG,
+	E_SCALER1_REG,
+	E_OUTCTRL0_REG,
+	E_PP0_REG,
+	E_WB_TOP_0_REG,
+	E_DPU_DUMP_ALL
+} dpu_hee_reg_enum;
+
+
+typedef struct dpu_reg_dump {
+	dpu_hee_reg_enum	index;
+	u8 *module_name;
+	uint32_t	module_offset;
+	uint32_t	dump_reg_num;
+} dpu_hee_reg_dump_t;
+
+#include "dpu_ctl_top.h"
+#include "dpu_scene_ctl.h"
+#include "dpu_int.h"
+#include "dpu_top.h"
+#include "mmu_tbu_x.h"
+#include "mmu_top.h"
+#include "ltm.h" /* outctrl_proc_x.h in dove */
+#include "usr_gma.h"
+#include "tmg.h"
+#include "rdma_path.h"
+#include "lut_3d.h"
+#include "acad.h"
+
+#include "composer_x.h"
+
+#include "cmdlist_top.h"
+#include "ee.h"
+#include "postpipe.h"
+#include "prepipe_x.h"
+#include "rc.h"
+#include "rdma_top.h"
+#include "scale_x.h"
+#include "wb.h"
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/scale_x.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/scale_x.h
new file mode 100644
index 000000000000..005f368c24c4
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/scale_x.h
@@ -0,0 +1,549 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef SCALE_X_REG_H
+#define SCALE_X_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_scaler_8x4tap_reg_0
+	UINT32 m_nscl_hor_enable           : 1;
+	UINT32 m_nscl_ver_enable           : 1;
+	UINT32 m_nyuv_narrow               : 1;
+	UINT32:29;
+
+	//REGISTER saturn_scaler_8x4tap_reg_1
+	UINT32 m_nscl_input_width          : 16;
+	UINT32 m_nscl_input_height         : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_2
+	UINT32 m_nscl_output_width         : 16;
+	UINT32 m_nscl_output_height        : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_3
+	UINT32 m_nscl_hor_init_phase_l32b  : 32;
+
+	//REGISTER saturn_scaler_8x4tap_reg_4
+	UINT32 m_nscl_hor_init_phase_h1b   : 1;
+	UINT32:31;
+
+	//REGISTER saturn_scaler_8x4tap_reg_5
+	UINT32 m_nscl_ver_init_phase_l32b  : 32;
+
+	//REGISTER saturn_scaler_8x4tap_reg_6
+	UINT32 m_nscl_ver_init_phase_h1b   : 1;
+	UINT32:31;
+
+	//REGISTER saturn_scaler_8x4tap_reg_7
+	UINT32 m_nscl_hor_delta_phase      : 20;
+	UINT32:12;
+
+	//REGISTER saturn_scaler_8x4tap_reg_8
+	UINT32 m_nscl_ver_delta_phase      : 20;
+	UINT32:12;
+
+	//REGISTER saturn_scaler_8x4tap_reg_9
+	UINT32 m_nscl_hor_coef0            : 16;
+	UINT32 m_nscl_hor_coef1            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_10
+	UINT32 m_nscl_hor_coef2            : 16;
+	UINT32 m_nscl_hor_coef3            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_11
+	UINT32 m_nscl_hor_coef4            : 16;
+	UINT32 m_nscl_hor_coef5            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_12
+	UINT32 m_nscl_hor_coef6            : 16;
+	UINT32 m_nscl_hor_coef7            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_13
+	UINT32 m_nscl_hor_coef8            : 16;
+	UINT32 m_nscl_hor_coef9            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_14
+	UINT32 m_nscl_hor_coef10           : 16;
+	UINT32 m_nscl_hor_coef11           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_15
+	UINT32 m_nscl_hor_coef12           : 16;
+	UINT32 m_nscl_hor_coef13           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_16
+	UINT32 m_nscl_hor_coef14           : 16;
+	UINT32 m_nscl_hor_coef15           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_17
+	UINT32 m_nscl_hor_coef16           : 16;
+	UINT32 m_nscl_hor_coef17           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_18
+	UINT32 m_nscl_hor_coef18           : 16;
+	UINT32 m_nscl_hor_coef19           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_19
+	UINT32 m_nscl_hor_coef20           : 16;
+	UINT32 m_nscl_hor_coef21           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_20
+	UINT32 m_nscl_hor_coef22           : 16;
+	UINT32 m_nscl_hor_coef23           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_21
+	UINT32 m_nscl_hor_coef24           : 16;
+	UINT32 m_nscl_hor_coef25           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_22
+	UINT32 m_nscl_hor_coef26           : 16;
+	UINT32 m_nscl_hor_coef27           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_23
+	UINT32 m_nscl_hor_coef28           : 16;
+	UINT32 m_nscl_hor_coef29           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_24
+	UINT32 m_nscl_hor_coef30           : 16;
+	UINT32 m_nscl_hor_coef31           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_25
+	UINT32 m_nscl_hor_coef32           : 16;
+	UINT32 m_nscl_hor_coef33           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_26
+	UINT32 m_nscl_hor_coef34           : 16;
+	UINT32 m_nscl_hor_coef35           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_27
+	UINT32 m_nscl_hor_coef36           : 16;
+	UINT32 m_nscl_hor_coef37           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_28
+	UINT32 m_nscl_hor_coef38           : 16;
+	UINT32 m_nscl_hor_coef39           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_29
+	UINT32 m_nscl_hor_coef40           : 16;
+	UINT32 m_nscl_hor_coef41           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_30
+	UINT32 m_nscl_hor_coef42           : 16;
+	UINT32 m_nscl_hor_coef43           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_31
+	UINT32 m_nscl_hor_coef44           : 16;
+	UINT32 m_nscl_hor_coef45           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_32
+	UINT32 m_nscl_hor_coef46           : 16;
+	UINT32 m_nscl_hor_coef47           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_33
+	UINT32 m_nscl_hor_coef48           : 16;
+	UINT32 m_nscl_hor_coef49           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_34
+	UINT32 m_nscl_hor_coef50           : 16;
+	UINT32 m_nscl_hor_coef51           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_35
+	UINT32 m_nscl_hor_coef52           : 16;
+	UINT32 m_nscl_hor_coef53           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_36
+	UINT32 m_nscl_hor_coef54           : 16;
+	UINT32 m_nscl_hor_coef55           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_37
+	UINT32 m_nscl_hor_coef56           : 16;
+	UINT32 m_nscl_hor_coef57           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_38
+	UINT32 m_nscl_hor_coef58           : 16;
+	UINT32 m_nscl_hor_coef59           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_39
+	UINT32 m_nscl_hor_coef60           : 16;
+	UINT32 m_nscl_hor_coef61           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_40
+	UINT32 m_nscl_hor_coef62           : 16;
+	UINT32 m_nscl_hor_coef63           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_41
+	UINT32 m_nscl_hor_coef64           : 16;
+	UINT32 m_nscl_hor_coef65           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_42
+	UINT32 m_nscl_hor_coef66           : 16;
+	UINT32 m_nscl_hor_coef67           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_43
+	UINT32 m_nscl_hor_coef68           : 16;
+	UINT32 m_nscl_hor_coef69           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_44
+	UINT32 m_nscl_hor_coef70           : 16;
+	UINT32 m_nscl_hor_coef71           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_45
+	UINT32 m_nscl_hor_coef72           : 16;
+	UINT32 m_nscl_hor_coef73           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_46
+	UINT32 m_nscl_hor_coef74           : 16;
+	UINT32 m_nscl_hor_coef75           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_47
+	UINT32 m_nscl_hor_coef76           : 16;
+	UINT32 m_nscl_hor_coef77           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_48
+	UINT32 m_nscl_hor_coef78           : 16;
+	UINT32 m_nscl_hor_coef79           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_49
+	UINT32 m_nscl_hor_coef80           : 16;
+	UINT32 m_nscl_hor_coef81           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_50
+	UINT32 m_nscl_hor_coef82           : 16;
+	UINT32 m_nscl_hor_coef83           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_51
+	UINT32 m_nscl_hor_coef84           : 16;
+	UINT32 m_nscl_hor_coef85           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_52
+	UINT32 m_nscl_hor_coef86           : 16;
+	UINT32 m_nscl_hor_coef87           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_53
+	UINT32 m_nscl_hor_coef88           : 16;
+	UINT32 m_nscl_hor_coef89           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_54
+	UINT32 m_nscl_hor_coef90           : 16;
+	UINT32 m_nscl_hor_coef91           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_55
+	UINT32 m_nscl_hor_coef92           : 16;
+	UINT32 m_nscl_hor_coef93           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_56
+	UINT32 m_nscl_hor_coef94           : 16;
+	UINT32 m_nscl_hor_coef95           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_57
+	UINT32 m_nscl_hor_coef96           : 16;
+	UINT32 m_nscl_hor_coef97           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_58
+	UINT32 m_nscl_hor_coef98           : 16;
+	UINT32 m_nscl_hor_coef99           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_59
+	UINT32 m_nscl_hor_coef100          : 16;
+	UINT32 m_nscl_hor_coef101          : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_60
+	UINT32 m_nscl_hor_coef102          : 16;
+	UINT32 m_nscl_hor_coef103          : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_61
+	UINT32 m_nscl_hor_coef104          : 16;
+	UINT32 m_nscl_hor_coef105          : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_62
+	UINT32 m_nscl_hor_coef106          : 16;
+	UINT32 m_nscl_hor_coef107          : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_63
+	UINT32 m_nscl_hor_coef108          : 16;
+	UINT32 m_nscl_hor_coef109          : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_64
+	UINT32 m_nscl_hor_coef110          : 16;
+	UINT32 m_nscl_hor_coef111          : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_65
+	UINT32 m_nscl_ver_coef0            : 16;
+	UINT32 m_nscl_ver_coef1            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_66
+	UINT32 m_nscl_ver_coef2            : 16;
+	UINT32 m_nscl_ver_coef3            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_67
+	UINT32 m_nscl_ver_coef4            : 16;
+	UINT32 m_nscl_ver_coef5            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_68
+	UINT32 m_nscl_ver_coef6            : 16;
+	UINT32 m_nscl_ver_coef7            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_69
+	UINT32 m_nscl_ver_coef8            : 16;
+	UINT32 m_nscl_ver_coef9            : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_70
+	UINT32 m_nscl_ver_coef10           : 16;
+	UINT32 m_nscl_ver_coef11           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_71
+	UINT32 m_nscl_ver_coef12           : 16;
+	UINT32 m_nscl_ver_coef13           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_72
+	UINT32 m_nscl_ver_coef14           : 16;
+	UINT32 m_nscl_ver_coef15           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_73
+	UINT32 m_nscl_ver_coef16           : 16;
+	UINT32 m_nscl_ver_coef17           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_74
+	UINT32 m_nscl_ver_coef18           : 16;
+	UINT32 m_nscl_ver_coef19           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_75
+	UINT32 m_nscl_ver_coef20           : 16;
+	UINT32 m_nscl_ver_coef21           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_76
+	UINT32 m_nscl_ver_coef22           : 16;
+	UINT32 m_nscl_ver_coef23           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_77
+	UINT32 m_nscl_ver_coef24           : 16;
+	UINT32 m_nscl_ver_coef25           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_78
+	UINT32 m_nscl_ver_coef26           : 16;
+	UINT32 m_nscl_ver_coef27           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_79
+	UINT32 m_nscl_ver_coef28           : 16;
+	UINT32 m_nscl_ver_coef29           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_80
+	UINT32 m_nscl_ver_coef30           : 16;
+	UINT32 m_nscl_ver_coef31           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_81
+	UINT32 m_nscl_ver_coef32           : 16;
+	UINT32 m_nscl_ver_coef33           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_82
+	UINT32 m_nscl_ver_coef34           : 16;
+	UINT32 m_nscl_ver_coef35           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_83
+	UINT32 m_nscl_ver_coef36           : 16;
+	UINT32 m_nscl_ver_coef37           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_84
+	UINT32 m_nscl_ver_coef38           : 16;
+	UINT32 m_nscl_ver_coef39           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_85
+	UINT32 m_nscl_ver_coef40           : 16;
+	UINT32 m_nscl_ver_coef41           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_86
+	UINT32 m_nscl_ver_coef42           : 16;
+	UINT32 m_nscl_ver_coef43           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_87
+	UINT32 m_nscl_ver_coef44           : 16;
+	UINT32 m_nscl_ver_coef45           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_88
+	UINT32 m_nscl_ver_coef46           : 16;
+	UINT32 m_nscl_ver_coef47           : 16;
+
+	//REGISTER saturn_scaler_8x4tap_reg_89
+	struct {
+	UINT32:32;
+	} saturn_scaler_8x4tap_reg_89[32];
+
+	//REGISTER saturn_scaler_8x4tap_reg_121
+	UINT32 cfg_se                      : 1;
+	UINT32:31;
+
+	//REGISTER saturn_scaler_8x4tap_reg_122
+	UINT32 force_update_en             : 1;
+	UINT32 vsync_update_en             : 1;
+	UINT32 shadow_read_en              : 1;
+	UINT32:29;
+
+	//REGISTER saturn_scaler_8x4tap_reg_123
+	UINT32 force_update_pulse          : 1;
+	UINT32:31;
+
+	//REGISTER saturn_scaler_8x4tap_reg_124
+	UINT32 force_update_en_se          : 1;
+	UINT32 vsync_update_en_se          : 1;
+	UINT32 shadow_read_en_se           : 1;
+	UINT32:29;
+
+	//REGISTER saturn_scaler_8x4tap_reg_125
+	UINT32 force_update_pulse_se       : 1;
+	UINT32:31;
+
+	//REGISTER saturn_scaler_8x4tap_reg_126
+	UINT32 icg_override                : 1;
+	UINT32:31;
+
+	//REGISTER saturn_scaler_8x4tap_reg_127
+	UINT32 trigger                     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_scaler_8x4tap_reg_128
+	UINT32 trigger2                    : 1;
+	UINT32:31;
+
+	} b;
+
+	INT32 value32[129];
+
+	struct {
+		UINT32 saturn_scaler_reg_0;
+		UINT32 saturn_scaler_reg_1;
+		UINT32 saturn_scaler_reg_2;
+		UINT32 saturn_scaler_reg_3;
+		UINT32 saturn_scaler_reg_4;
+		UINT32 saturn_scaler_reg_5;
+		UINT32 saturn_scaler_reg_6;
+		UINT32 saturn_scaler_reg_7;
+		UINT32 saturn_scaler_reg_8;
+		UINT32 saturn_scaler_reg_9;
+		UINT32 saturn_scaler_reg_10;
+		UINT32 saturn_scaler_reg_11;
+		UINT32 saturn_scaler_reg_12;
+		UINT32 saturn_scaler_reg_13;
+		UINT32 saturn_scaler_reg_14;
+		UINT32 saturn_scaler_reg_15;
+		UINT32 saturn_scaler_reg_16;
+		UINT32 saturn_scaler_reg_17;
+		UINT32 saturn_scaler_reg_18;
+		UINT32 saturn_scaler_reg_19;
+		UINT32 saturn_scaler_reg_20;
+		UINT32 saturn_scaler_reg_21;
+		UINT32 saturn_scaler_reg_22;
+		UINT32 saturn_scaler_reg_23;
+		UINT32 saturn_scaler_reg_24;
+		UINT32 saturn_scaler_reg_25;
+		UINT32 saturn_scaler_reg_26;
+		UINT32 saturn_scaler_reg_27;
+		UINT32 saturn_scaler_reg_28;
+		UINT32 saturn_scaler_reg_29;
+		UINT32 saturn_scaler_reg_30;
+		UINT32 saturn_scaler_reg_31;
+		UINT32 saturn_scaler_reg_32;
+		UINT32 saturn_scaler_reg_33;
+		UINT32 saturn_scaler_reg_34;
+		UINT32 saturn_scaler_reg_35;
+		UINT32 saturn_scaler_reg_36;
+		UINT32 saturn_scaler_reg_37;
+		UINT32 saturn_scaler_reg_38;
+		UINT32 saturn_scaler_reg_39;
+		UINT32 saturn_scaler_reg_40;
+		UINT32 saturn_scaler_reg_41;
+		UINT32 saturn_scaler_reg_42;
+		UINT32 saturn_scaler_reg_43;
+		UINT32 saturn_scaler_reg_44;
+		UINT32 saturn_scaler_reg_45;
+		UINT32 saturn_scaler_reg_46;
+		UINT32 saturn_scaler_reg_47;
+		UINT32 saturn_scaler_reg_48;
+		UINT32 saturn_scaler_reg_49;
+		UINT32 saturn_scaler_reg_50;
+		UINT32 saturn_scaler_reg_51;
+		UINT32 saturn_scaler_reg_52;
+		UINT32 saturn_scaler_reg_53;
+		UINT32 saturn_scaler_reg_54;
+		UINT32 saturn_scaler_reg_55;
+		UINT32 saturn_scaler_reg_56;
+		UINT32 saturn_scaler_reg_57;
+		UINT32 saturn_scaler_reg_58;
+		UINT32 saturn_scaler_reg_59;
+		UINT32 saturn_scaler_reg_60;
+		UINT32 saturn_scaler_reg_61;
+		UINT32 saturn_scaler_reg_62;
+		UINT32 saturn_scaler_reg_63;
+		UINT32 saturn_scaler_reg_64;
+		UINT32 saturn_scaler_reg_65;
+		UINT32 saturn_scaler_reg_66;
+		UINT32 saturn_scaler_reg_67;
+		UINT32 saturn_scaler_reg_68;
+		UINT32 saturn_scaler_reg_69;
+		UINT32 saturn_scaler_reg_70;
+		UINT32 saturn_scaler_reg_71;
+		UINT32 saturn_scaler_reg_72;
+		UINT32 saturn_scaler_reg_73;
+		UINT32 saturn_scaler_reg_74;
+		UINT32 saturn_scaler_reg_75;
+		UINT32 saturn_scaler_reg_76;
+		UINT32 saturn_scaler_reg_77;
+		UINT32 saturn_scaler_reg_78;
+		UINT32 saturn_scaler_reg_79;
+		UINT32 saturn_scaler_reg_80;
+		UINT32 saturn_scaler_reg_81;
+		UINT32 saturn_scaler_reg_82;
+		UINT32 saturn_scaler_reg_83;
+		UINT32 saturn_scaler_reg_84;
+		UINT32 saturn_scaler_reg_85;
+		UINT32 saturn_scaler_reg_86;
+		UINT32 saturn_scaler_reg_87;
+		UINT32 saturn_scaler_reg_88;
+		UINT32 saturn_scaler_reg_89;
+		UINT32 saturn_scaler_reg_90;
+		UINT32 saturn_scaler_reg_91;
+		UINT32 saturn_scaler_reg_92;
+		UINT32 saturn_scaler_reg_93;
+		UINT32 saturn_scaler_reg_94;
+		UINT32 saturn_scaler_reg_95;
+		UINT32 saturn_scaler_reg_96;
+		UINT32 saturn_scaler_reg_97;
+		UINT32 saturn_scaler_reg_98;
+		UINT32 saturn_scaler_reg_99;
+		UINT32 saturn_scaler_reg_100;
+		UINT32 saturn_scaler_reg_101;
+		UINT32 saturn_scaler_reg_102;
+		UINT32 saturn_scaler_reg_103;
+		UINT32 saturn_scaler_reg_104;
+		UINT32 saturn_scaler_reg_105;
+		UINT32 saturn_scaler_reg_106;
+		UINT32 saturn_scaler_reg_107;
+		UINT32 saturn_scaler_reg_108;
+		UINT32 saturn_scaler_reg_109;
+		UINT32 saturn_scaler_reg_110;
+		UINT32 saturn_scaler_reg_111;
+		UINT32 saturn_scaler_reg_112;
+		UINT32 saturn_scaler_reg_113;
+		UINT32 saturn_scaler_reg_114;
+		UINT32 saturn_scaler_reg_115;
+		UINT32 saturn_scaler_reg_116;
+		UINT32 saturn_scaler_reg_117;
+		UINT32 saturn_scaler_reg_118;
+		UINT32 saturn_scaler_reg_119;
+		UINT32 saturn_scaler_reg_120;
+		UINT32 saturn_scaler_reg_121;
+		UINT32 saturn_scaler_reg_122;
+		UINT32 saturn_scaler_reg_123;
+		UINT32 saturn_scaler_reg_124;
+		UINT32 saturn_scaler_reg_125;
+		UINT32 saturn_scaler_reg_126;
+		UINT32 saturn_scaler_reg_127;
+		UINT32 saturn_scaler_reg_128;
+	} v;
+
+} SCALE_X_REG;
+
+#endif
+
+typedef SCALE_X_REG SCALER_X_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/tmg.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/tmg.h
new file mode 100644
index 000000000000..5ee58492874b
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/tmg.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef TMG_REG_H
+#define TMG_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_tmg_reg_0
+	UINT32 split_en                     : 1;
+	UINT32 cmd_screen                   : 1;
+	UINT32 fm_timing_en                 : 1;
+	UINT32 cmd_wait_en                  : 1;
+	UINT32 cmd_wait_te                  : 1;
+	UINT32 disp_ready_man_en            : 1;
+	UINT32 ready_1_non_active           : 1;
+	UINT32 hsp                          : 1;
+	UINT32 vsp                          : 1;
+	UINT32 fm_cmd_tmout_eq_eof          : 1;
+	UINT32 skip_frm_en                  : 1;
+	UINT32 swap_rgb_en                  : 1;
+	UINT32:20;
+
+	//REGISTER saturn_tmg_reg_1
+	UINT32 dsc_enable                   : 1;
+	UINT32 dsc_one_line_byte_remainder  : 2;
+	UINT32:29;
+
+	//REGISTER saturn_tmg_reg_2
+	UINT32 drift_timeout                : 12;
+	UINT32 user                         : 4;
+	UINT32 active_frm_num               : 8;
+	UINT32 inactive_frm_num             : 8;
+
+	//REGISTER saturn_tmg_reg_3
+	UINT32 split_overlap                : 10;
+	UINT32:6;
+	UINT32 hfp                          : 12;
+	UINT32:4;
+
+	//REGISTER saturn_tmg_reg_4
+	UINT32 hsync_width                  : 10;
+	UINT32:6;
+	UINT32 hbp                          : 12;
+	UINT32:4;
+
+	//REGISTER saturn_tmg_reg_5
+	UINT32 vfp                          : 16;
+	UINT32 vsync_width                  : 10;
+	UINT32:6;
+
+	//REGISTER saturn_tmg_reg_6
+	UINT32 vbp                          : 12;
+	UINT32:4;
+	UINT32 h_active                     : 14;
+	UINT32:2;
+
+	//REGISTER saturn_tmg_reg_7
+	UINT32 v_active                     : 14;
+	UINT32:18;
+
+	//REGISTER saturn_tmg_reg_8
+	UINT32 fm_cmd_tmout_num             : 24;
+	UINT32:8;
+
+	//REGISTER saturn_tmg_reg_9
+	UINT32 background_r                 : 12;
+	UINT32:4;
+	UINT32 background_g                 : 12;
+	UINT32:4;
+
+	//REGISTER saturn_tmg_reg_10
+	UINT32 background_b                 : 12;
+	UINT32:20;
+
+	//REGISTER saturn_tmg_reg_11
+	UINT32 eof_1st_ln_dly_num           : 16;
+	UINT32 eof_2nd_ln_dly_num           : 16;
+
+	//REGISTER saturn_tmg_reg_12
+	UINT32 cfg_ln_num_intp              : 17;
+	UINT32:15;
+
+	//REGISTER saturn_tmg_reg_13
+	UINT32 frame_te_dly_num             : 32;
+
+	//REGISTER saturn_tmg_reg_14
+	UINT32 sof_pre_ln_num               : 17;
+	UINT32:15;
+
+	//REGISTER saturn_tmg_reg_15
+	UINT32 vsync_align_with_hsync       : 1;
+	UINT32:31;
+
+	//REGISTER saturn_tmg_reg_16
+	UINT32 dfc_low_thre                 : 12;
+	UINT32:4;
+	UINT32 dfc_high_thre                : 12;
+	UINT32:4;
+
+	//REGISTER saturn_tmg_reg_17
+	UINT32 urgent_vld_en                : 1;
+	UINT32:7;
+	UINT32 urgent_l_thre                : 12;
+	UINT32:12;
+
+	//REGISTER saturn_tmg_reg_18
+	UINT32 urgent_h_thre                : 12;
+	UINT32:20;
+
+	//REGISTER saturn_tmg_reg_19
+	UINT32 dbg_disp_ready_i             : 2;
+	UINT32:30;
+
+	//REGISTER saturn_tmg_reg_20
+	UINT32 dbg_tmg_cur_state_i          : 32;
+
+	//REGISTER saturn_tmg_reg_21
+	UINT32 dfc_low_hold_cycle           : 32;
+
+	//REGISTER saturn_tmg_reg_22
+	UINT32 dbg_dfc_mode                 : 1;
+	UINT32:31;
+
+	//REGISTER saturn_tmg_reg_23
+	UINT32 force_update_en              : 1;
+	UINT32 vsync_update_en              : 1;
+	UINT32 shadow_read_en               : 1;
+	UINT32:29;
+
+	//REGISTER saturn_tmg_reg_24
+	UINT32 force_update_pulse           : 1;
+	UINT32:31;
+
+	//REGISTER saturn_tmg_reg_25
+	UINT32 icg_override                 : 1;
+	UINT32:31;
+
+	//REGISTER saturn_tmg_reg_26
+	UINT32 trigger                      : 1;
+	UINT32:31;
+
+	//REGISTER saturn_tmg_reg_27
+	UINT32 trigger2                     : 1;
+	UINT32:31;
+
+	//REGISTER saturn_tmg_reg_28
+	UINT32 mem0_wr_cnt                  : 32;
+
+	//REGISTER saturn_tmg_reg_29
+	UINT32 mem1_wr_cnt                  : 32;
+
+	};
+
+	INT32 value32[30];
+
+} TMG_REG;
+
+#endif
+typedef TMG_REG TMG_REG;
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/usr_gma.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/usr_gma.h
new file mode 100644
index 000000000000..8a78ed63a0a4
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/usr_gma.h
@@ -0,0 +1,2346 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef USR_GMA_REG_H
+#define USR_GMA_REG_H
+
+typedef union {
+	struct {
+	//REGISTER saturn_usrgamma_reg_0
+	UINT32 usr_gma_en        : 1;
+	UINT32:31;
+
+	//REGISTER saturn_usrgamma_reg_1
+	UINT32 mem_lp_auto_en    : 1;
+	UINT32:31;
+
+	//REGISTER saturn_usrgamma_reg_2
+	UINT32 m_ncurve_a_000    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_001    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_3
+	UINT32 m_ncurve_a_002    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_003    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_4
+	UINT32 m_ncurve_a_004    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_005    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_5
+	UINT32 m_ncurve_a_006    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_007    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_6
+	UINT32 m_ncurve_a_008    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_009    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_7
+	UINT32 m_ncurve_a_010    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_011    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_8
+	UINT32 m_ncurve_a_012    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_013    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_9
+	UINT32 m_ncurve_a_014    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_015    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_10
+	UINT32 m_ncurve_a_016    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_017    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_11
+	UINT32 m_ncurve_a_018    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_019    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_12
+	UINT32 m_ncurve_a_020    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_021    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_13
+	UINT32 m_ncurve_a_022    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_023    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_14
+	UINT32 m_ncurve_a_024    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_025    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_15
+	UINT32 m_ncurve_a_026    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_027    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_16
+	UINT32 m_ncurve_a_028    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_029    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_17
+	UINT32 m_ncurve_a_030    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_031    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_18
+	UINT32 m_ncurve_a_032    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_033    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_19
+	UINT32 m_ncurve_a_034    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_035    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_20
+	UINT32 m_ncurve_a_036    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_037    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_21
+	UINT32 m_ncurve_a_038    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_039    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_22
+	UINT32 m_ncurve_a_040    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_041    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_23
+	UINT32 m_ncurve_a_042    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_043    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_24
+	UINT32 m_ncurve_a_044    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_045    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_25
+	UINT32 m_ncurve_a_046    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_047    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_26
+	UINT32 m_ncurve_a_048    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_049    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_27
+	UINT32 m_ncurve_a_050    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_051    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_28
+	UINT32 m_ncurve_a_052    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_053    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_29
+	UINT32 m_ncurve_a_054    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_055    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_30
+	UINT32 m_ncurve_a_056    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_057    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_31
+	UINT32 m_ncurve_a_058    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_059    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_32
+	UINT32 m_ncurve_a_060    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_061    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_33
+	UINT32 m_ncurve_a_062    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_063    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_34
+	UINT32 m_ncurve_a_064    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_065    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_35
+	UINT32 m_ncurve_a_066    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_067    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_36
+	UINT32 m_ncurve_a_068    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_069    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_37
+	UINT32 m_ncurve_a_070    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_071    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_38
+	UINT32 m_ncurve_a_072    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_073    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_39
+	UINT32 m_ncurve_a_074    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_075    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_40
+	UINT32 m_ncurve_a_076    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_077    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_41
+	UINT32 m_ncurve_a_078    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_079    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_42
+	UINT32 m_ncurve_a_080    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_081    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_43
+	UINT32 m_ncurve_a_082    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_083    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_44
+	UINT32 m_ncurve_a_084    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_085    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_45
+	UINT32 m_ncurve_a_086    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_087    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_46
+	UINT32 m_ncurve_a_088    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_089    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_47
+	UINT32 m_ncurve_a_090    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_091    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_48
+	UINT32 m_ncurve_a_092    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_093    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_49
+	UINT32 m_ncurve_a_094    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_095    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_50
+	UINT32 m_ncurve_a_096    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_097    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_51
+	UINT32 m_ncurve_a_098    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_099    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_52
+	UINT32 m_ncurve_a_100    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_101    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_53
+	UINT32 m_ncurve_a_102    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_103    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_54
+	UINT32 m_ncurve_a_104    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_105    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_55
+	UINT32 m_ncurve_a_106    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_107    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_56
+	UINT32 m_ncurve_a_108    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_109    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_57
+	UINT32 m_ncurve_a_110    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_111    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_58
+	UINT32 m_ncurve_a_112    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_113    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_59
+	UINT32 m_ncurve_a_114    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_115    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_60
+	UINT32 m_ncurve_a_116    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_117    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_61
+	UINT32 m_ncurve_a_118    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_119    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_62
+	UINT32 m_ncurve_a_120    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_121    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_63
+	UINT32 m_ncurve_a_122    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_123    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_64
+	UINT32 m_ncurve_a_124    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_125    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_65
+	UINT32 m_ncurve_a_126    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_127    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_66
+	UINT32 m_ncurve_a_128    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_129    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_67
+	UINT32 m_ncurve_a_130    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_131    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_68
+	UINT32 m_ncurve_a_132    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_133    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_69
+	UINT32 m_ncurve_a_134    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_135    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_70
+	UINT32 m_ncurve_a_136    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_137    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_71
+	UINT32 m_ncurve_a_138    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_139    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_72
+	UINT32 m_ncurve_a_140    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_141    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_73
+	UINT32 m_ncurve_a_142    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_143    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_74
+	UINT32 m_ncurve_a_144    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_145    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_75
+	UINT32 m_ncurve_a_146    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_147    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_76
+	UINT32 m_ncurve_a_148    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_149    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_77
+	UINT32 m_ncurve_a_150    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_151    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_78
+	UINT32 m_ncurve_a_152    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_153    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_79
+	UINT32 m_ncurve_a_154    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_155    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_80
+	UINT32 m_ncurve_a_156    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_157    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_81
+	UINT32 m_ncurve_a_158    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_159    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_82
+	UINT32 m_ncurve_a_160    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_161    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_83
+	UINT32 m_ncurve_a_162    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_163    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_84
+	UINT32 m_ncurve_a_164    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_165    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_85
+	UINT32 m_ncurve_a_166    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_167    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_86
+	UINT32 m_ncurve_a_168    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_169    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_87
+	UINT32 m_ncurve_a_170    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_171    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_88
+	UINT32 m_ncurve_a_172    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_173    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_89
+	UINT32 m_ncurve_a_174    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_175    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_90
+	UINT32 m_ncurve_a_176    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_177    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_91
+	UINT32 m_ncurve_a_178    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_179    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_92
+	UINT32 m_ncurve_a_180    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_181    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_93
+	UINT32 m_ncurve_a_182    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_183    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_94
+	UINT32 m_ncurve_a_184    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_185    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_95
+	UINT32 m_ncurve_a_186    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_187    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_96
+	UINT32 m_ncurve_a_188    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_189    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_97
+	UINT32 m_ncurve_a_190    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_191    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_98
+	UINT32 m_ncurve_a_192    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_193    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_99
+	UINT32 m_ncurve_a_194    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_195    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_100
+	UINT32 m_ncurve_a_196    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_197    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_101
+	UINT32 m_ncurve_a_198    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_199    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_102
+	UINT32 m_ncurve_a_200    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_201    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_103
+	UINT32 m_ncurve_a_202    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_203    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_104
+	UINT32 m_ncurve_a_204    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_205    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_105
+	UINT32 m_ncurve_a_206    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_207    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_106
+	UINT32 m_ncurve_a_208    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_209    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_107
+	UINT32 m_ncurve_a_210    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_211    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_108
+	UINT32 m_ncurve_a_212    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_213    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_109
+	UINT32 m_ncurve_a_214    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_215    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_110
+	UINT32 m_ncurve_a_216    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_217    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_111
+	UINT32 m_ncurve_a_218    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_219    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_112
+	UINT32 m_ncurve_a_220    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_221    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_113
+	UINT32 m_ncurve_a_222    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_223    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_114
+	UINT32 m_ncurve_a_224    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_225    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_115
+	UINT32 m_ncurve_a_226    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_227    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_116
+	UINT32 m_ncurve_a_228    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_229    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_117
+	UINT32 m_ncurve_a_230    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_231    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_118
+	UINT32 m_ncurve_a_232    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_233    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_119
+	UINT32 m_ncurve_a_234    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_235    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_120
+	UINT32 m_ncurve_a_236    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_237    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_121
+	UINT32 m_ncurve_a_238    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_239    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_122
+	UINT32 m_ncurve_a_240    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_241    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_123
+	UINT32 m_ncurve_a_242    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_243    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_124
+	UINT32 m_ncurve_a_244    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_245    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_125
+	UINT32 m_ncurve_a_246    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_247    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_126
+	UINT32 m_ncurve_a_248    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_249    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_127
+	UINT32 m_ncurve_a_250    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_251    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_128
+	UINT32 m_ncurve_a_252    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_253    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_129
+	UINT32 m_ncurve_a_254    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_a_255    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_130
+	UINT32 m_ncurve_a_256    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_usrgamma_reg_131
+	UINT32 m_ncurve_b_000    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_001    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_132
+	UINT32 m_ncurve_b_002    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_003    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_133
+	UINT32 m_ncurve_b_004    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_005    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_134
+	UINT32 m_ncurve_b_006    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_007    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_135
+	UINT32 m_ncurve_b_008    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_009    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_136
+	UINT32 m_ncurve_b_010    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_011    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_137
+	UINT32 m_ncurve_b_012    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_013    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_138
+	UINT32 m_ncurve_b_014    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_015    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_139
+	UINT32 m_ncurve_b_016    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_017    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_140
+	UINT32 m_ncurve_b_018    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_019    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_141
+	UINT32 m_ncurve_b_020    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_021    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_142
+	UINT32 m_ncurve_b_022    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_023    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_143
+	UINT32 m_ncurve_b_024    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_025    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_144
+	UINT32 m_ncurve_b_026    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_027    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_145
+	UINT32 m_ncurve_b_028    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_029    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_146
+	UINT32 m_ncurve_b_030    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_031    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_147
+	UINT32 m_ncurve_b_032    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_033    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_148
+	UINT32 m_ncurve_b_034    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_035    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_149
+	UINT32 m_ncurve_b_036    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_037    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_150
+	UINT32 m_ncurve_b_038    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_039    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_151
+	UINT32 m_ncurve_b_040    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_041    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_152
+	UINT32 m_ncurve_b_042    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_043    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_153
+	UINT32 m_ncurve_b_044    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_045    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_154
+	UINT32 m_ncurve_b_046    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_047    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_155
+	UINT32 m_ncurve_b_048    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_049    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_156
+	UINT32 m_ncurve_b_050    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_051    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_157
+	UINT32 m_ncurve_b_052    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_053    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_158
+	UINT32 m_ncurve_b_054    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_055    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_159
+	UINT32 m_ncurve_b_056    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_057    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_160
+	UINT32 m_ncurve_b_058    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_059    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_161
+	UINT32 m_ncurve_b_060    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_061    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_162
+	UINT32 m_ncurve_b_062    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_063    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_163
+	UINT32 m_ncurve_b_064    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_065    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_164
+	UINT32 m_ncurve_b_066    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_067    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_165
+	UINT32 m_ncurve_b_068    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_069    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_166
+	UINT32 m_ncurve_b_070    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_071    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_167
+	UINT32 m_ncurve_b_072    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_073    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_168
+	UINT32 m_ncurve_b_074    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_075    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_169
+	UINT32 m_ncurve_b_076    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_077    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_170
+	UINT32 m_ncurve_b_078    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_079    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_171
+	UINT32 m_ncurve_b_080    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_081    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_172
+	UINT32 m_ncurve_b_082    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_083    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_173
+	UINT32 m_ncurve_b_084    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_085    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_174
+	UINT32 m_ncurve_b_086    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_087    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_175
+	UINT32 m_ncurve_b_088    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_089    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_176
+	UINT32 m_ncurve_b_090    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_091    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_177
+	UINT32 m_ncurve_b_092    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_093    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_178
+	UINT32 m_ncurve_b_094    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_095    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_179
+	UINT32 m_ncurve_b_096    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_097    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_180
+	UINT32 m_ncurve_b_098    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_099    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_181
+	UINT32 m_ncurve_b_100    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_101    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_182
+	UINT32 m_ncurve_b_102    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_103    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_183
+	UINT32 m_ncurve_b_104    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_105    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_184
+	UINT32 m_ncurve_b_106    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_107    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_185
+	UINT32 m_ncurve_b_108    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_109    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_186
+	UINT32 m_ncurve_b_110    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_111    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_187
+	UINT32 m_ncurve_b_112    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_113    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_188
+	UINT32 m_ncurve_b_114    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_115    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_189
+	UINT32 m_ncurve_b_116    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_117    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_190
+	UINT32 m_ncurve_b_118    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_119    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_191
+	UINT32 m_ncurve_b_120    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_121    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_192
+	UINT32 m_ncurve_b_122    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_123    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_193
+	UINT32 m_ncurve_b_124    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_125    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_194
+	UINT32 m_ncurve_b_126    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_127    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_195
+	UINT32 m_ncurve_b_128    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_129    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_196
+	UINT32 m_ncurve_b_130    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_131    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_197
+	UINT32 m_ncurve_b_132    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_133    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_198
+	UINT32 m_ncurve_b_134    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_135    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_199
+	UINT32 m_ncurve_b_136    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_137    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_200
+	UINT32 m_ncurve_b_138    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_139    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_201
+	UINT32 m_ncurve_b_140    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_141    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_202
+	UINT32 m_ncurve_b_142    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_143    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_203
+	UINT32 m_ncurve_b_144    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_145    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_204
+	UINT32 m_ncurve_b_146    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_147    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_205
+	UINT32 m_ncurve_b_148    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_149    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_206
+	UINT32 m_ncurve_b_150    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_151    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_207
+	UINT32 m_ncurve_b_152    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_153    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_208
+	UINT32 m_ncurve_b_154    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_155    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_209
+	UINT32 m_ncurve_b_156    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_157    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_210
+	UINT32 m_ncurve_b_158    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_159    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_211
+	UINT32 m_ncurve_b_160    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_161    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_212
+	UINT32 m_ncurve_b_162    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_163    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_213
+	UINT32 m_ncurve_b_164    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_165    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_214
+	UINT32 m_ncurve_b_166    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_167    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_215
+	UINT32 m_ncurve_b_168    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_169    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_216
+	UINT32 m_ncurve_b_170    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_171    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_217
+	UINT32 m_ncurve_b_172    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_173    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_218
+	UINT32 m_ncurve_b_174    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_175    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_219
+	UINT32 m_ncurve_b_176    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_177    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_220
+	UINT32 m_ncurve_b_178    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_179    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_221
+	UINT32 m_ncurve_b_180    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_181    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_222
+	UINT32 m_ncurve_b_182    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_183    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_223
+	UINT32 m_ncurve_b_184    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_185    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_224
+	UINT32 m_ncurve_b_186    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_187    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_225
+	UINT32 m_ncurve_b_188    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_189    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_226
+	UINT32 m_ncurve_b_190    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_191    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_227
+	UINT32 m_ncurve_b_192    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_193    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_228
+	UINT32 m_ncurve_b_194    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_195    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_229
+	UINT32 m_ncurve_b_196    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_197    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_230
+	UINT32 m_ncurve_b_198    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_199    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_231
+	UINT32 m_ncurve_b_200    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_201    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_232
+	UINT32 m_ncurve_b_202    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_203    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_233
+	UINT32 m_ncurve_b_204    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_205    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_234
+	UINT32 m_ncurve_b_206    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_207    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_235
+	UINT32 m_ncurve_b_208    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_209    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_236
+	UINT32 m_ncurve_b_210    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_211    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_237
+	UINT32 m_ncurve_b_212    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_213    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_238
+	UINT32 m_ncurve_b_214    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_215    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_239
+	UINT32 m_ncurve_b_216    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_217    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_240
+	UINT32 m_ncurve_b_218    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_219    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_241
+	UINT32 m_ncurve_b_220    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_221    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_242
+	UINT32 m_ncurve_b_222    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_223    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_243
+	UINT32 m_ncurve_b_224    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_225    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_244
+	UINT32 m_ncurve_b_226    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_227    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_245
+	UINT32 m_ncurve_b_228    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_229    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_246
+	UINT32 m_ncurve_b_230    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_231    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_247
+	UINT32 m_ncurve_b_232    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_233    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_248
+	UINT32 m_ncurve_b_234    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_235    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_249
+	UINT32 m_ncurve_b_236    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_237    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_250
+	UINT32 m_ncurve_b_238    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_239    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_251
+	UINT32 m_ncurve_b_240    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_241    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_252
+	UINT32 m_ncurve_b_242    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_243    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_253
+	UINT32 m_ncurve_b_244    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_245    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_254
+	UINT32 m_ncurve_b_246    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_247    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_255
+	UINT32 m_ncurve_b_248    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_249    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_256
+	UINT32 m_ncurve_b_250    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_251    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_257
+	UINT32 m_ncurve_b_252    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_253    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_258
+	UINT32 m_ncurve_b_254    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_b_255    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_259
+	UINT32 m_ncurve_b_256    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_usrgamma_reg_260
+	UINT32 m_ncurve_c_000    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_001    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_261
+	UINT32 m_ncurve_c_002    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_003    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_262
+	UINT32 m_ncurve_c_004    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_005    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_263
+	UINT32 m_ncurve_c_006    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_007    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_264
+	UINT32 m_ncurve_c_008    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_009    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_265
+	UINT32 m_ncurve_c_010    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_011    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_266
+	UINT32 m_ncurve_c_012    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_013    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_267
+	UINT32 m_ncurve_c_014    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_015    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_268
+	UINT32 m_ncurve_c_016    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_017    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_269
+	UINT32 m_ncurve_c_018    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_019    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_270
+	UINT32 m_ncurve_c_020    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_021    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_271
+	UINT32 m_ncurve_c_022    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_023    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_272
+	UINT32 m_ncurve_c_024    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_025    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_273
+	UINT32 m_ncurve_c_026    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_027    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_274
+	UINT32 m_ncurve_c_028    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_029    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_275
+	UINT32 m_ncurve_c_030    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_031    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_276
+	UINT32 m_ncurve_c_032    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_033    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_277
+	UINT32 m_ncurve_c_034    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_035    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_278
+	UINT32 m_ncurve_c_036    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_037    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_279
+	UINT32 m_ncurve_c_038    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_039    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_280
+	UINT32 m_ncurve_c_040    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_041    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_281
+	UINT32 m_ncurve_c_042    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_043    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_282
+	UINT32 m_ncurve_c_044    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_045    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_283
+	UINT32 m_ncurve_c_046    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_047    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_284
+	UINT32 m_ncurve_c_048    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_049    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_285
+	UINT32 m_ncurve_c_050    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_051    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_286
+	UINT32 m_ncurve_c_052    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_053    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_287
+	UINT32 m_ncurve_c_054    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_055    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_288
+	UINT32 m_ncurve_c_056    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_057    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_289
+	UINT32 m_ncurve_c_058    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_059    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_290
+	UINT32 m_ncurve_c_060    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_061    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_291
+	UINT32 m_ncurve_c_062    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_063    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_292
+	UINT32 m_ncurve_c_064    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_065    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_293
+	UINT32 m_ncurve_c_066    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_067    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_294
+	UINT32 m_ncurve_c_068    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_069    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_295
+	UINT32 m_ncurve_c_070    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_071    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_296
+	UINT32 m_ncurve_c_072    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_073    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_297
+	UINT32 m_ncurve_c_074    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_075    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_298
+	UINT32 m_ncurve_c_076    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_077    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_299
+	UINT32 m_ncurve_c_078    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_079    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_300
+	UINT32 m_ncurve_c_080    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_081    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_301
+	UINT32 m_ncurve_c_082    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_083    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_302
+	UINT32 m_ncurve_c_084    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_085    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_303
+	UINT32 m_ncurve_c_086    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_087    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_304
+	UINT32 m_ncurve_c_088    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_089    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_305
+	UINT32 m_ncurve_c_090    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_091    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_306
+	UINT32 m_ncurve_c_092    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_093    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_307
+	UINT32 m_ncurve_c_094    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_095    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_308
+	UINT32 m_ncurve_c_096    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_097    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_309
+	UINT32 m_ncurve_c_098    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_099    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_310
+	UINT32 m_ncurve_c_100    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_101    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_311
+	UINT32 m_ncurve_c_102    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_103    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_312
+	UINT32 m_ncurve_c_104    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_105    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_313
+	UINT32 m_ncurve_c_106    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_107    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_314
+	UINT32 m_ncurve_c_108    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_109    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_315
+	UINT32 m_ncurve_c_110    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_111    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_316
+	UINT32 m_ncurve_c_112    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_113    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_317
+	UINT32 m_ncurve_c_114    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_115    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_318
+	UINT32 m_ncurve_c_116    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_117    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_319
+	UINT32 m_ncurve_c_118    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_119    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_320
+	UINT32 m_ncurve_c_120    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_121    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_321
+	UINT32 m_ncurve_c_122    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_123    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_322
+	UINT32 m_ncurve_c_124    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_125    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_323
+	UINT32 m_ncurve_c_126    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_127    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_324
+	UINT32 m_ncurve_c_128    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_129    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_325
+	UINT32 m_ncurve_c_130    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_131    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_326
+	UINT32 m_ncurve_c_132    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_133    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_327
+	UINT32 m_ncurve_c_134    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_135    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_328
+	UINT32 m_ncurve_c_136    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_137    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_329
+	UINT32 m_ncurve_c_138    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_139    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_330
+	UINT32 m_ncurve_c_140    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_141    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_331
+	UINT32 m_ncurve_c_142    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_143    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_332
+	UINT32 m_ncurve_c_144    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_145    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_333
+	UINT32 m_ncurve_c_146    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_147    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_334
+	UINT32 m_ncurve_c_148    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_149    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_335
+	UINT32 m_ncurve_c_150    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_151    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_336
+	UINT32 m_ncurve_c_152    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_153    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_337
+	UINT32 m_ncurve_c_154    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_155    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_338
+	UINT32 m_ncurve_c_156    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_157    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_339
+	UINT32 m_ncurve_c_158    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_159    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_340
+	UINT32 m_ncurve_c_160    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_161    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_341
+	UINT32 m_ncurve_c_162    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_163    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_342
+	UINT32 m_ncurve_c_164    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_165    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_343
+	UINT32 m_ncurve_c_166    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_167    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_344
+	UINT32 m_ncurve_c_168    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_169    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_345
+	UINT32 m_ncurve_c_170    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_171    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_346
+	UINT32 m_ncurve_c_172    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_173    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_347
+	UINT32 m_ncurve_c_174    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_175    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_348
+	UINT32 m_ncurve_c_176    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_177    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_349
+	UINT32 m_ncurve_c_178    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_179    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_350
+	UINT32 m_ncurve_c_180    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_181    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_351
+	UINT32 m_ncurve_c_182    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_183    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_352
+	UINT32 m_ncurve_c_184    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_185    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_353
+	UINT32 m_ncurve_c_186    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_187    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_354
+	UINT32 m_ncurve_c_188    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_189    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_355
+	UINT32 m_ncurve_c_190    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_191    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_356
+	UINT32 m_ncurve_c_192    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_193    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_357
+	UINT32 m_ncurve_c_194    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_195    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_358
+	UINT32 m_ncurve_c_196    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_197    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_359
+	UINT32 m_ncurve_c_198    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_199    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_360
+	UINT32 m_ncurve_c_200    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_201    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_361
+	UINT32 m_ncurve_c_202    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_203    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_362
+	UINT32 m_ncurve_c_204    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_205    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_363
+	UINT32 m_ncurve_c_206    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_207    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_364
+	UINT32 m_ncurve_c_208    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_209    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_365
+	UINT32 m_ncurve_c_210    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_211    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_366
+	UINT32 m_ncurve_c_212    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_213    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_367
+	UINT32 m_ncurve_c_214    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_215    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_368
+	UINT32 m_ncurve_c_216    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_217    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_369
+	UINT32 m_ncurve_c_218    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_219    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_370
+	UINT32 m_ncurve_c_220    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_221    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_371
+	UINT32 m_ncurve_c_222    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_223    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_372
+	UINT32 m_ncurve_c_224    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_225    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_373
+	UINT32 m_ncurve_c_226    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_227    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_374
+	UINT32 m_ncurve_c_228    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_229    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_375
+	UINT32 m_ncurve_c_230    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_231    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_376
+	UINT32 m_ncurve_c_232    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_233    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_377
+	UINT32 m_ncurve_c_234    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_235    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_378
+	UINT32 m_ncurve_c_236    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_237    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_379
+	UINT32 m_ncurve_c_238    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_239    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_380
+	UINT32 m_ncurve_c_240    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_241    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_381
+	UINT32 m_ncurve_c_242    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_243    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_382
+	UINT32 m_ncurve_c_244    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_245    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_383
+	UINT32 m_ncurve_c_246    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_247    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_384
+	UINT32 m_ncurve_c_248    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_249    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_385
+	UINT32 m_ncurve_c_250    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_251    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_386
+	UINT32 m_ncurve_c_252    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_253    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_387
+	UINT32 m_ncurve_c_254    : 10;
+	UINT32:6;
+	UINT32 m_ncurve_c_255    : 10;
+	UINT32:6;
+
+	//REGISTER saturn_usrgamma_reg_388
+	UINT32 m_ncurve_c_256    : 10;
+	UINT32:22;
+
+	//REGISTER saturn_usrgamma_reg_389
+	UINT32 usr_gma_cfg_done  : 1;
+	UINT32:31;
+
+	};
+
+	INT32 value32[390];
+
+} USR_GMA_REG;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dpu/saturn_regs/wb.h b/drivers/gpu/drm/spacemit/dpu/saturn_regs/wb.h
new file mode 100644
index 000000000000..0c06645f376d
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dpu/saturn_regs/wb.h
@@ -0,0 +1,320 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef WB_REG_H
+#define WB_REG_H
+
+typedef union {
+	struct {
+	//REGISTER wb_top_reg_0
+	UINT32 wb_in_crop_en            : 1;
+	UINT32 m_nwb_proc_en            : 1;
+	UINT32 m_nmatrix_en             : 1;
+	UINT32 m_noetf_en               : 1;
+	UINT32 wb_en                    : 1;
+	UINT32 wb_scl_en                : 1;
+	UINT32:2;
+	UINT32 wb_rot_mode              : 3;
+	UINT32 wb_wdma_fbc_en           : 1;
+	UINT32 wb_fbc_mode              : 1;
+	UINT32:3;
+	UINT32 wb_out_format            : 5;
+	UINT32:11;
+
+	//REGISTER wb_top_reg_1
+	UINT32 wb_in_height             : 16;
+	UINT32 wb_in_width              : 16;
+
+	//REGISTER wb_top_reg_2
+	UINT32 wb_out_ori_height        : 16;
+	UINT32 wb_out_ori_width         : 16;
+
+	//REGISTER wb_top_reg_3
+	UINT32 wb_out_crop_ltopx        : 16;
+	UINT32 wb_out_crop_ltopy        : 16;
+
+	//REGISTER wb_top_reg_4
+	UINT32 wb_out_crop_height       : 16;
+	UINT32 wb_out_crop_width        : 16;
+
+	//REGISTER wb_top_reg_5
+	UINT32 wb_wdma_base_addr0_low   : 32;
+
+	//REGISTER wb_top_reg_6
+	UINT32 wb_wdma_base_addr0_high  : 2;
+	UINT32:30;
+
+	//REGISTER wb_top_reg_7
+	UINT32 wb_wdma_base_addr1_low   : 32;
+
+	//REGISTER wb_top_reg_8
+	UINT32 wb_wdma_base_addr1_high  : 2;
+	UINT32:6;
+	UINT32 wb_wdma_stride           : 16;
+	UINT32 wb_wdma_outstanding_num  : 5;
+	UINT32:3;
+
+	//REGISTER wb_top_reg_9
+	UINT32 wb_wdma_burst_len        : 5;
+	UINT32 wb_wdma_axi_port         : 2;
+	UINT32:5;
+	UINT32 wb_wdma_cache            : 4;
+	UINT32 wb_wdma_region           : 4;
+	UINT32 wb_wdma_qos              : 4;
+	UINT32:8;
+
+	//REGISTER wb_top_reg_10
+	UINT32 copy_mode_enable         : 1;
+	UINT32 default_color_enable     : 1;
+	UINT32 yuv_transform_en         : 1;
+	UINT32 fbc_split_en             : 1;
+	UINT32 fbc_tile_hd_mode_en      : 1;
+	UINT32:27;
+
+	//REGISTER wb_top_reg_11
+	UINT32 fmt_cvt_A0               : 16;
+	UINT32 fmt_cvt_A1               : 16;
+
+	//REGISTER wb_top_reg_12
+	UINT32 fmt_cvt_A2               : 16;
+	UINT32 fmt_cvt_narrow_mode      : 1;
+	UINT32:15;
+
+	//REGISTER wb_top_reg_13
+	UINT32 fmt_cvt_matrix00         : 14;
+	UINT32:2;
+	UINT32 fmt_cvt_matrix01         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_14
+	UINT32 fmt_cvt_matrix02         : 14;
+	UINT32:2;
+	UINT32 fmt_cvt_matrix03         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_15
+	UINT32 fmt_cvt_matrix10         : 14;
+	UINT32:2;
+	UINT32 fmt_cvt_matrix11         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_16
+	UINT32 fmt_cvt_matrix12         : 14;
+	UINT32:2;
+	UINT32 fmt_cvt_matrix13         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_17
+	UINT32 fmt_cvt_matrix20         : 14;
+	UINT32:2;
+	UINT32 fmt_cvt_matrix21         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_18
+	UINT32 fmt_cvt_matrix22         : 14;
+	UINT32:2;
+	UINT32 fmt_cvt_matrix23         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_19
+	UINT32 wb_dither_en             : 1;
+	UINT32 wb_dither_auto_temp_en   : 1;
+	UINT32 wb_dither_rot_mode       : 2;
+	UINT32 wb_dither_mode           : 1;
+	UINT32:3;
+	UINT32 wb_dither_out_dpth0      : 4;
+	UINT32 wb_dither_out_dpth1      : 4;
+	UINT32 wb_dither_out_dpth2      : 4;
+	UINT32:4;
+	UINT32 wb_dither_tmp_value      : 8;
+
+	//REGISTER wb_top_reg_20
+	UINT32 wb_dither_pattern_bits   : 3;
+	UINT32:29;
+
+	//REGISTER wb_top_reg_21
+	UINT32 wb_dither_bayer_map0     : 8;
+	UINT32 wb_dither_bayer_map1     : 8;
+	UINT32 wb_dither_bayer_map2     : 8;
+	UINT32 wb_dither_bayer_map3     : 8;
+
+	//REGISTER wb_top_reg_22
+	UINT32 wb_dither_bayer_map4     : 8;
+	UINT32 wb_dither_bayer_map5     : 8;
+	UINT32 wb_dither_bayer_map6     : 8;
+	UINT32 wb_dither_bayer_map7     : 8;
+
+	//REGISTER wb_top_reg_23
+	UINT32 wb_dither_bayer_map8     : 8;
+	UINT32 wb_dither_bayer_map9     : 8;
+	UINT32 wb_dither_bayer_map10    : 8;
+	UINT32 wb_dither_bayer_map11    : 8;
+
+	//REGISTER wb_top_reg_24
+	UINT32 wb_dither_bayer_map12    : 8;
+	UINT32 wb_dither_bayer_map13    : 8;
+	UINT32 wb_dither_bayer_map14    : 8;
+	UINT32 wb_dither_bayer_map15    : 8;
+
+	//REGISTER wb_top_reg_25
+	UINT32 wb_dither_bayer_map16    : 8;
+	UINT32 wb_dither_bayer_map17    : 8;
+	UINT32 wb_dither_bayer_map18    : 8;
+	UINT32 wb_dither_bayer_map19    : 8;
+
+	//REGISTER wb_top_reg_26
+	UINT32 wb_dither_bayer_map20    : 8;
+	UINT32 wb_dither_bayer_map21    : 8;
+	UINT32 wb_dither_bayer_map22    : 8;
+	UINT32 wb_dither_bayer_map23    : 8;
+
+	//REGISTER wb_top_reg_27
+	UINT32 wb_dither_bayer_map24    : 8;
+	UINT32 wb_dither_bayer_map25    : 8;
+	UINT32 wb_dither_bayer_map26    : 8;
+	UINT32 wb_dither_bayer_map27    : 8;
+
+	//REGISTER wb_top_reg_28
+	UINT32 wb_dither_bayer_map28    : 8;
+	UINT32 wb_dither_bayer_map29    : 8;
+	UINT32 wb_dither_bayer_map30    : 8;
+	UINT32 wb_dither_bayer_map31    : 8;
+
+	//REGISTER wb_top_reg_29
+	UINT32 wb_dither_bayer_map32    : 8;
+	UINT32 wb_dither_bayer_map33    : 8;
+	UINT32 wb_dither_bayer_map34    : 8;
+	UINT32 wb_dither_bayer_map35    : 8;
+
+	//REGISTER wb_top_reg_30
+	UINT32 wb_dither_bayer_map36    : 8;
+	UINT32 wb_dither_bayer_map37    : 8;
+	UINT32 wb_dither_bayer_map38    : 8;
+	UINT32 wb_dither_bayer_map39    : 8;
+
+	//REGISTER wb_top_reg_31
+	UINT32 wb_dither_bayer_map40    : 8;
+	UINT32 wb_dither_bayer_map41    : 8;
+	UINT32 wb_dither_bayer_map42    : 8;
+	UINT32 wb_dither_bayer_map43    : 8;
+
+	//REGISTER wb_top_reg_32
+	UINT32 wb_dither_bayer_map44    : 8;
+	UINT32 wb_dither_bayer_map45    : 8;
+	UINT32 wb_dither_bayer_map46    : 8;
+	UINT32 wb_dither_bayer_map47    : 8;
+
+	//REGISTER wb_top_reg_33
+	UINT32 wb_dither_bayer_map48    : 8;
+	UINT32 wb_dither_bayer_map49    : 8;
+	UINT32 wb_dither_bayer_map50    : 8;
+	UINT32 wb_dither_bayer_map51    : 8;
+
+	//REGISTER wb_top_reg_34
+	UINT32 wb_dither_bayer_map52    : 8;
+	UINT32 wb_dither_bayer_map53    : 8;
+	UINT32 wb_dither_bayer_map54    : 8;
+	UINT32 wb_dither_bayer_map55    : 8;
+
+	//REGISTER wb_top_reg_35
+	UINT32 wb_dither_bayer_map56    : 8;
+	UINT32 wb_dither_bayer_map57    : 8;
+	UINT32 wb_dither_bayer_map58    : 8;
+	UINT32 wb_dither_bayer_map59    : 8;
+
+	//REGISTER wb_top_reg_36
+	UINT32 wb_dither_bayer_map60    : 8;
+	UINT32 wb_dither_bayer_map61    : 8;
+	UINT32 wb_dither_bayer_map62    : 8;
+	UINT32 wb_dither_bayer_map63    : 8;
+
+	//REGISTER wb_top_reg_37
+	UINT32 wb_in_crop_ltopx         : 16;
+	UINT32 wb_in_crop_ltopy         : 16;
+
+	//REGISTER wb_top_reg_38
+	UINT32 wb_in_crop_rbotx         : 16;
+	UINT32 wb_in_crop_rboty         : 16;
+
+	//REGISTER wb_top_reg_39
+	UINT32 m_noetf_mode             : 3;
+	UINT32:5;
+	UINT32 m_noetf_max              : 12;
+	UINT32:12;
+
+	//REGISTER wb_top_reg_40
+	UINT32 m_pmatrix_table0         : 14;
+	UINT32:2;
+	UINT32 m_pmatrix_table1         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_41
+	UINT32 m_pmatrix_table2         : 14;
+	UINT32:2;
+	UINT32 m_pmatrix_table3         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_42
+	UINT32 m_pmatrix_table4         : 14;
+	UINT32:2;
+	UINT32 m_pmatrix_table5         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_43
+	UINT32 m_pmatrix_table6         : 14;
+	UINT32:2;
+	UINT32 m_pmatrix_table7         : 14;
+	UINT32:2;
+
+	//REGISTER wb_top_reg_44
+	UINT32 m_pmatrix_table8         : 14;
+	UINT32:18;
+
+	//REGISTER wb_top_reg_45
+	UINT32 m_pmatrix_offset0        : 25;
+	UINT32:7;
+
+	//REGISTER wb_top_reg_46
+	UINT32 m_pmatrix_offset1        : 25;
+	UINT32:7;
+
+	//REGISTER wb_top_reg_47
+	UINT32 m_pmatrix_offset2        : 25;
+	UINT32:7;
+
+	//REGISTER wb_top_reg_48
+	UINT32 dbug_bus0                : 32;
+
+	//REGISTER wb_top_reg_49
+	UINT32 dbug_bus1                : 32;
+
+	//REGISTER wb_top_reg_50
+	UINT32 dbug_bus2                : 32;
+
+	//REGISTER wb_top_reg_51
+	UINT32 dbug_bus3                : 32;
+
+	//REGISTER wb_top_reg_52
+	UINT32 dbug_bus4                : 32;
+
+	//REGISTER wb_top_reg_53
+	UINT32 dbug_bus5                : 32;
+
+	//REGISTER wb_top_reg_54
+	UINT32 wb_wdma_nsaid            : 4;
+	UINT32:28;
+
+	//REGISTER wb_top_reg_55
+	UINT32 lbuf_low_lvl             : 16;
+	UINT32 lbuf_high_lvl            : 16;
+
+	};
+
+	INT32 value32[56];
+
+} WB_REG;
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/dsi/spacemit_dptc_drv.c b/drivers/gpu/drm/spacemit/dsi/spacemit_dptc_drv.c
new file mode 100644
index 000000000000..0137e6c5e974
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dsi/spacemit_dptc_drv.c
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of_device.h>
+#include <linux/i2c.h>
+#include <linux/delay.h>
+#include "spacemit_dptc_drv.h"
+#define I2C_NO 0
+#define TOP_SLAVE_ADDR 0x30
+#define DSI_SLAVE_ADDR 0x33
+
+static struct twsi_data dptc_i2c_data;
+static DEFINE_MUTEX(cmd_mutex);
+static int twsi_write_i2c(struct twsi_data *data)
+{
+	int ret = 0;
+	struct i2c_adapter *adapter;
+	struct i2c_msg msg;
+	u8 val[8];
+	int i, j = 0;
+
+	if (!data || !data->addr || !data->reg_len || !data->val_len) {
+		pr_err("Error: %s, %d", __func__, __LINE__);
+		return -EINVAL;
+	}
+	msg.addr = data->addr;
+	msg.flags = 0;
+	msg.len = data->reg_len + data->val_len;
+	msg.buf = val;
+	adapter = i2c_get_adapter(data->twsi_no);
+	if (adapter == NULL)
+		return -1;
+	mutex_lock(&cmd_mutex);
+	for (i = 0; i < data->reg_len; i++)
+		val[j++] = ((u8 *)(&data->reg))[i];
+	for (i = 0; i < data->val_len; i++)
+		val[j++] = ((u8 *)(&data->val))[i];
+	ret = i2c_transfer(adapter, &msg, 1);
+	if (ret < 0) {
+		mutex_unlock(&cmd_mutex);
+		return ret;
+	}
+	mutex_unlock(&cmd_mutex);
+	return ret;
+}
+
+static int twsi_read_i2c(struct twsi_data *data)
+{
+	struct i2c_adapter *adapter;
+	struct i2c_msg msg;
+	int ret = 0;
+	u8 val[4];
+
+	if (!data || !data->addr || !data->reg_len || !data->val_len) {
+		pr_err("%s, error param", __func__);
+		return -EINVAL;
+	}
+	msg.addr = data->addr;
+	msg.flags = 0;
+	msg.len = data->reg_len;
+	msg.buf = val;
+	adapter = i2c_get_adapter(data->twsi_no);
+	if (adapter == NULL)
+		return -1;
+	mutex_lock(&cmd_mutex);
+	if (data->reg_len == I2C_8BIT)
+		val[0] = data->reg & 0xff;
+	else if (data->reg_len == I2C_16BIT) {
+		val[0] = (data->reg >> 8) & 0xff;
+		val[1] = data->reg & 0xff;
+	}
+	msg.len = data->reg_len;
+	ret = i2c_transfer(adapter, &msg, 1);
+	if (ret < 0) {
+		mutex_unlock(&cmd_mutex);
+		goto err;
+	}
+	msg.flags = I2C_M_RD;
+	msg.len = data->val_len;
+	ret = i2c_transfer(adapter, &msg, 1);
+	if (ret < 0) {
+		mutex_unlock(&cmd_mutex);
+		goto err;
+	}
+	if (data->val_len == I2C_8BIT)
+		data->val = val[0];
+	else if (data->val_len == I2C_16BIT)
+		data->val = (val[0] << 8) + val[1];
+	else if (data->val_len == I2C_32BIT)
+		data->val = (val[3] << 24) + (val[2] << 16) + (val[1] << 8) + val[0];
+	//pr_info("twsi_read_i2c: val[0]=0x%x,val[1]=0x%x,val[2]=0x%x,val[3]=0x%x\n",val[0],val[1],val[2],val[3]);
+	mutex_unlock(&cmd_mutex);
+	return 0;
+err:
+	pr_info("Failed reading register 0x%02x!", data->reg);
+	return ret;
+}
+
+static void TWSI_Init(I2C_FAST_MODE mode, unsigned int i2c_no)
+{
+	dptc_i2c_data.twsi_no = i2c_no;
+	dptc_i2c_data.addr = 0x6c;
+	dptc_i2c_data.reg_len = 1;//I2C_8BIT;
+	dptc_i2c_data.val_len = 4;//I2C_32BIT;
+}
+
+static int TWSI_REG_WRITE_DPTC(uint8_t i2c_no, uint8_t slaveaddress, uint8_t addr, uint32_t data)
+{
+	int ret = 0;
+
+	dptc_i2c_data.twsi_no = i2c_no;
+	dptc_i2c_data.addr = slaveaddress;
+	dptc_i2c_data.reg = addr;
+	dptc_i2c_data.val = data;
+	ret = twsi_write_i2c(&dptc_i2c_data);
+	if (ret == 0)
+		return 1;
+	else
+		return -1;
+}
+
+static int TWSI_REG_READ_DPTC(uint8_t i2c_no, uint8_t slaveaddress, uint8_t addr)
+{
+	int ret = 0;
+
+	dptc_i2c_data.twsi_no = i2c_no;
+	dptc_i2c_data.addr = slaveaddress;
+	dptc_i2c_data.reg = addr;
+	dptc_i2c_data.val = 0x00;
+	ret = twsi_read_i2c(&dptc_i2c_data);
+	if (ret == 0)
+		return dptc_i2c_data.val;
+	else
+		return -1;
+}
+
+static void dptc_top_write(uint32_t reg, uint32_t data)
+{
+	int ret = 0;
+	uint32_t rd_data;
+
+	ret = TWSI_REG_WRITE_DPTC(I2C_NO, TOP_SLAVE_ADDR, reg >> 2, data);
+	rd_data = TWSI_REG_READ_DPTC(I2C_NO, TOP_SLAVE_ADDR, reg >> 2);
+	if (rd_data != data)
+		pr_err("spacemitfb: %s failed,  [0x%x] = 0x%x ret %d\n", __func__, reg, data, ret);
+}
+
+static uint32_t dptc_top_read(uint32_t reg)
+{
+	uint32_t data = 0;
+
+	data = TWSI_REG_READ_DPTC(I2C_NO, TOP_SLAVE_ADDR, reg >> 2);
+	pr_debug("spacemitfb: %s [0x%x] = 0x%x\n", __func__, reg, data);
+	return data;
+}
+
+void dptc_dsi_write(uint32_t reg, uint32_t data)
+{
+	int ret = 0;
+	uint32_t rd_data;
+
+	ret = TWSI_REG_WRITE_DPTC(I2C_NO, DSI_SLAVE_ADDR, reg >> 2, data);
+	rd_data = TWSI_REG_READ_DPTC(I2C_NO, DSI_SLAVE_ADDR, reg >> 2);
+	if (rd_data != data)
+		pr_err("spacemitfb: %s failed,  [0x%x] = 0x%x ret%d\n", __func__, reg, data, ret);
+}
+
+uint32_t dptc_dsi_read(uint32_t reg)
+{
+	uint32_t data = 0;
+
+	data = TWSI_REG_READ_DPTC(I2C_NO, DSI_SLAVE_ADDR, reg >> 2);
+	pr_debug("spacemitfb: %s [0x%x] = 0x%x\n", __func__, reg, data);
+	return data;
+}
+
+void dptc_board_init(void)
+{
+	uint32_t data;
+
+	TWSI_Init(STANDARD_MODE, I2C_NO);
+	msleep(100);
+	data = dptc_top_read(0x04);    // enter function 4
+	data |= 0x4;
+	dptc_top_write(0x4, data);
+	data = dptc_top_read(0x8);    // pll control
+	data |= 0x1;
+	dptc_top_write(0x8, data);
+	data = dptc_top_read(0x18);    // clk reset
+	data |= 0x4;
+	dptc_top_write(0x18, data);
+	dptc_top_write(0x0c, 0x3e627627);// setting pll   1.62G // success
+	dptc_top_write(0x10, 0x428D4420);
+	data = dptc_top_read(0x14);    // enable ck_dsi
+	data |= (0xff00);            // ???
+	dptc_top_write(0x14, data);
+	data = dptc_top_read(0x08);    // power up PLL
+	data |= 0x02;
+	dptc_top_write(0x08, data);
+}
diff --git a/drivers/gpu/drm/spacemit/dsi/spacemit_dptc_drv.h b/drivers/gpu/drm/spacemit/dsi/spacemit_dptc_drv.h
new file mode 100644
index 000000000000..a2f6ee8f4afe
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dsi/spacemit_dptc_drv.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _DPTC_DRV_H_
+#define _DPTC_DRV_H_
+typedef enum {
+	STANDARD_MODE = 0,  /*100Kbps*/
+	FAST_MODE = 1,      /*400Kbps*/
+	HS_MODE = 2,        /*3.4 Mbps slave/3.3 Mbps master,standard mode when not doing a high speed transfer*/
+	HS_MODE_FAST = 3,   /*3.4 Mbps slave/3.3 Mbps master,fast mode when not doing a high speed transfer*/
+} I2C_FAST_MODE;
+
+struct twsi_data {
+	u8 twsi_no;
+	u8 reg_len;/* byte num*/
+	u8 val_len;/* byte num*/
+	u8 addr; /* 7 bit i2c address*/
+	u16 reg;
+	u32 val;
+};
+
+enum i2c_len {
+	I2C_8BIT = 1,
+	I2C_16BIT = 2,
+	I2C_24BIT = 3,
+	I2C_32BIT = 4,
+};
+
+void dptc_dsi_write(uint32_t reg, uint32_t data);
+uint32_t dptc_dsi_read(uint32_t reg);
+void dptc_board_init(void);
+#endif
diff --git a/drivers/gpu/drm/spacemit/dsi/spacemit_dsi_drv.c b/drivers/gpu/drm/spacemit/dsi/spacemit_dsi_drv.c
new file mode 100644
index 000000000000..3cb2bd5d0579
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dsi/spacemit_dsi_drv.c
@@ -0,0 +1,1433 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/delay.h>
+#include <linux/clk-provider.h>
+
+#include "spacemit_dsi_hw.h"
+#include "../spacemit_dphy.h"
+#include "../spacemit_dsi.h"
+#include "../dsi/spacemit_dptc_drv.h"
+
+#define SPACEMIT_DSI_MAX_TX_FIFO_BYTES	256
+#define SPACEMIT_DSI_MAX_RX_FIFO_BYTES	64
+
+static struct spacemit_dsi_info g_spacemit_dsi_info[DSI_VERSION_MAX] = {
+	[DSI_VERSION_1] = {
+		.cmd_pkt_max_size = 0x400, /* to check */
+		.cmd_pkt_default_size = 0x400,
+		.cmd_fifo_max_level = 0x400, /* to check */
+		.cmd_fifo_default_level = 0x400,
+		.vpn_burst_mode_shift = 2,
+	},
+	[DSI_VERSION_2] = {
+		.cmd_pkt_max_size = 0x167E,
+		.cmd_pkt_default_size = 0x167E,
+		.cmd_fifo_max_level = 0x1680,
+		.cmd_fifo_default_level = 0x1680,
+		.vpn_burst_mode_shift = 3,
+	},
+};
+
+#define HIGH_BRUSH_V	3000
+
+#define to_dsi_bcnt(timing, bpp)        (((timing) * (bpp)) >> 3)
+
+static unsigned int spacemit_dsi_lane[5] = {0, 0x1, 0x3, 0x7, 0xf};
+
+static unsigned char dsi_bit(unsigned int index, unsigned char *pdata)
+{
+	unsigned char ret;
+	unsigned int cindex, bindex;
+
+	cindex = index / 8;
+	bindex = index % 8;
+
+	if (pdata[cindex] & (0x1 << bindex))
+		ret = (unsigned char) 0x1;
+	else
+		ret = (unsigned char) 0x0;
+
+	return ret;
+}
+
+static unsigned char calculate_ecc(unsigned char *pdata)
+{
+	unsigned char ret;
+	unsigned char p[8];
+
+	p[7] = (unsigned char) 0x0;
+	p[6] = (unsigned char) 0x0;
+
+	p[5] = (unsigned char) (
+	(
+		dsi_bit(10, pdata) ^
+		dsi_bit(11, pdata) ^
+		dsi_bit(12, pdata) ^
+		dsi_bit(13, pdata) ^
+		dsi_bit(14, pdata) ^
+		dsi_bit(15, pdata) ^
+		dsi_bit(16, pdata) ^
+		dsi_bit(17, pdata) ^
+		dsi_bit(18, pdata) ^
+		dsi_bit(19, pdata) ^
+		dsi_bit(21, pdata) ^
+		dsi_bit(22, pdata) ^
+		dsi_bit(23, pdata)
+		)
+	);
+	p[4] = (unsigned char) (
+		dsi_bit(4, pdata) ^
+		dsi_bit(5, pdata) ^
+		dsi_bit(6, pdata) ^
+		dsi_bit(7, pdata) ^
+		dsi_bit(8, pdata) ^
+		dsi_bit(9, pdata) ^
+		dsi_bit(16, pdata) ^
+		dsi_bit(17, pdata) ^
+		dsi_bit(18, pdata) ^
+		dsi_bit(19, pdata) ^
+		dsi_bit(20, pdata) ^
+		dsi_bit(22, pdata) ^
+		dsi_bit(23, pdata)
+	);
+	p[3] = (unsigned char) (
+	(
+		dsi_bit(1, pdata) ^
+		dsi_bit(2, pdata) ^
+		dsi_bit(3, pdata) ^
+		dsi_bit(7, pdata) ^
+		dsi_bit(8, pdata) ^
+		dsi_bit(9, pdata) ^
+		dsi_bit(13, pdata) ^
+		dsi_bit(14, pdata) ^
+		dsi_bit(15, pdata) ^
+		dsi_bit(19, pdata) ^
+		dsi_bit(20, pdata) ^
+		dsi_bit(21, pdata) ^
+		dsi_bit(23, pdata)
+		)
+	);
+	p[2] = (unsigned char) (
+	(
+		dsi_bit(0, pdata) ^
+		dsi_bit(2, pdata) ^
+		dsi_bit(3, pdata) ^
+		dsi_bit(5, pdata) ^
+		dsi_bit(6, pdata) ^
+		dsi_bit(9, pdata) ^
+		dsi_bit(11, pdata) ^
+		dsi_bit(12, pdata) ^
+		dsi_bit(15, pdata) ^
+		dsi_bit(18, pdata) ^
+		dsi_bit(20, pdata) ^
+		dsi_bit(21, pdata) ^
+		dsi_bit(22, pdata)
+		)
+	);
+	p[1] = (unsigned char) (
+		(
+		dsi_bit(0, pdata) ^
+		dsi_bit(1, pdata) ^
+		dsi_bit(3, pdata) ^
+		dsi_bit(4, pdata) ^
+		dsi_bit(6, pdata) ^
+		dsi_bit(8, pdata) ^
+		dsi_bit(10, pdata) ^
+		dsi_bit(12, pdata) ^
+		dsi_bit(14, pdata) ^
+		dsi_bit(17, pdata) ^
+		dsi_bit(20, pdata) ^
+		dsi_bit(21, pdata) ^
+		dsi_bit(22, pdata) ^
+		dsi_bit(23, pdata)
+		)
+	);
+	p[0] = (unsigned char) (
+		(
+		dsi_bit(0, pdata) ^
+		dsi_bit(1, pdata) ^
+		dsi_bit(2, pdata) ^
+		dsi_bit(4, pdata) ^
+		dsi_bit(5, pdata) ^
+		dsi_bit(7, pdata) ^
+		dsi_bit(10, pdata) ^
+		dsi_bit(11, pdata) ^
+		dsi_bit(13, pdata) ^
+		dsi_bit(16, pdata) ^
+		dsi_bit(20, pdata) ^
+		dsi_bit(21, pdata) ^
+		dsi_bit(22, pdata) ^
+		dsi_bit(23, pdata)
+		)
+	);
+	ret = (unsigned char)(
+		p[0] |
+		(p[1] << 0x1) |
+		(p[2] << 0x2) |
+		(p[3] << 0x3) |
+		(p[4] << 0x4) |
+		(p[5] << 0x5)
+	);
+	return   ret;
+}
+
+static unsigned short gs_crc16_generation_code = 0x8408;
+static unsigned short calculate_crc16(unsigned char *pdata, unsigned
+		short count)
+{
+	unsigned short byte_counter;
+	unsigned char bit_counter;
+	unsigned char data;
+	unsigned short crc16_result = 0xFFFF;
+
+	if (count > 0) {
+		for (byte_counter = 0; byte_counter < count;
+			byte_counter++) {
+			data = *(pdata + byte_counter);
+			for (bit_counter = 0; bit_counter < 8; bit_counter++) {
+				if (((crc16_result & 0x0001) ^ ((0x0001 *
+					data) & 0x0001)) > 0)
+					crc16_result = ((crc16_result >> 1)
+					& 0x7FFF) ^ gs_crc16_generation_code;
+				else
+					crc16_result = (crc16_result >> 1)
+					& 0x7FFF;
+				data = (data >> 1) & 0x7F;
+			}
+		}
+	}
+	return crc16_result;
+}
+
+#define DSI_GLB_RESET	CFG_SOFT_RST
+#define DSI_DEBUG_RESET	(CFG_SOFT_RST | CFG_SOFT_RST_REG | CFG_CLR_PHY_FIFO | CFG_RST_TXLP | \
+			 CFG_RST_CPU | CFG_RST_CPN | CFG_RST_VPN | CFG_DSI_PHY_RST)
+
+static void dsi_reset(void __iomem *base_addr)
+{
+	uint32_t reg;
+
+	reg = DSI_GLB_RESET;
+
+	/* software reset DSI module */
+	dsi_write(base_addr, DSI_CTRL_0, reg);
+	/* Note: there need some delay after set CFG_SOFT_RST */
+	udelay(1000);
+	dsi_write(base_addr, DSI_CTRL_0, 0);
+
+	dsi_write(base_addr, DSI_IRQ_ST, 0xFFFFFFFF);
+}
+
+static void dsi_enable_video_mode(void __iomem *base_addr, bool enable)
+{
+	if (enable)
+		dsi_set_bits(base_addr, DSI_CTRL_0, CFG_VPN_TX_EN | CFG_VPN_SLV | CFG_VPN_EN);
+	else
+		dsi_clear_bits(base_addr, DSI_CTRL_0, CFG_VPN_TX_EN | CFG_VPN_EN);
+}
+
+static void dsi_enable_cmd_mode(void __iomem *base_addr, bool enable)
+{
+	if (enable)
+		dsi_set_bits(base_addr, DSI_CTRL_0, CFG_CPN_EN);
+	else
+		dsi_clear_bits(base_addr, DSI_CTRL_0, CFG_CPN_EN);
+}
+
+static void dsi_enable_eotp(void __iomem *base_addr, bool enable)
+{
+	if (enable)
+		dsi_set_bits(base_addr, DSI_CTRL_1, CFG_EOTP_EN);
+	else
+		dsi_clear_bits(base_addr, DSI_CTRL_1, CFG_EOTP_EN);
+}
+
+static void dsi_enable_lptx_lanes(void __iomem *base_addr, uint32_t lane_num)
+{
+	dsi_write_bits(base_addr, DSI_CPU_CMD_1,
+		CFG_TXLP_LPDT_MASK, lane_num << CFG_TXLP_LPDT_SHIFT);
+}
+
+static void dsi_enable_split_mode(void __iomem *base_addr, bool splite_mode)
+{
+	if (splite_mode)
+		dsi_set_bits(base_addr, DSI_LCD_BDG_CTRL0, CFG_SPLIT_EN);
+	else
+		dsi_clear_bits(base_addr, DSI_LCD_BDG_CTRL0, CFG_SPLIT_EN);
+}
+
+static void dsi_enable_irq(void __iomem *base_addr, bool enable)
+{
+	uint32_t irq_mask = DSI_IRQ_MASK_BITS;
+
+	//if (cpu_is_dovenr())
+	//	irq_mask &= ~DSI_IRQ_VPN_BF_OVERRUN_ERR;
+
+	if (enable)
+		dsi_set_bits(base_addr, DSI_IRQ_MASK, irq_mask);
+	else
+		dsi_clear_bits(base_addr, DSI_IRQ_MASK, irq_mask);
+}
+
+static int dsi_write_cmd(void __iomem *base_addr, uint8_t *parameter, uint8_t count, bool lp)
+{
+	uint32_t send_data = 0, reg, timeout, tmp, i;
+	bool turnaround;
+	uint32_t len;
+
+	if (lp)
+		pr_debug("%s: %d packet data will be write in lp mode\n", __func__, count);
+	else
+		pr_debug("%s: %d data will be write in hs mode\n", __func__, count);
+
+	/* write all packet bytes to packet data buffer */
+	for (i = 0; i < count; i++) {
+		send_data |= parameter[i] << ((i % 4) * 8);
+		if (((i + 1) % 4) == 0) {
+			dsi_write(base_addr, DSI_CPU_WDAT, send_data);
+			reg = CFG_CPU_DAT_REQ | CFG_CPU_DAT_RW | ((i - 3) << CFG_CPU_DAT_ADDR_SHIFT);
+			dsi_write(base_addr, DSI_CPU_CMD_3, reg);
+			/* wait write operation done */
+			timeout = 1000;
+			do {
+				timeout--;
+				tmp = dsi_read(base_addr, DSI_CPU_CMD_3);
+			} while ((tmp & CFG_CPU_DAT_REQ) && timeout);
+			if (timeout == 0)
+				pr_err("DSI write data to the packet data buffer not done.\n");
+			send_data = 0;
+		}
+	}
+
+	/* handle last none 4Byte align data */
+	if (i % 4 != 0) {
+		dsi_write(base_addr, DSI_CPU_WDAT, send_data);
+		reg = CFG_CPU_DAT_REQ | CFG_CPU_DAT_RW | ((4 * (i / 4)) << CFG_CPU_DAT_ADDR_SHIFT);
+		dsi_write(base_addr, DSI_CPU_CMD_3, reg);
+		/* wait write operation done */
+		timeout = 1000;
+		do {
+			timeout--;
+			tmp = dsi_read(base_addr, DSI_CPU_CMD_3);
+		} while ((tmp & CFG_CPU_DAT_REQ) && timeout);
+		if (timeout == 0)
+			pr_err("DSI write data to the packet data buffer not done.\n");
+		send_data = 0;
+	}
+
+	if (parameter[0] == SPACEMIT_DSI_DCS_READ ||
+		parameter[0] == SPACEMIT_DSI_GENERIC_READ1)
+		turnaround = true;
+	else
+		turnaround = false;
+
+	len = count;
+#if 0
+	/* The packet length should contain  CRC_bytes_length in Aquilac_DSI version */
+	if ((parameter[0] == SPACEMIT_DSI_DCS_LWRITE ||
+		parameter[0] == SPACEMIT_DSI_GENERIC_LWRITE) && !lp)
+		len = count - 6;
+#endif
+	reg = CFG_CPU_CMD_REQ |
+		((count == 4) ? CFG_CPU_SP : 0) |
+		(turnaround ? CFG_CPU_TURN : 0) |
+		(lp ? CFG_CPU_TXLP : 0) |
+		(len << CFG_CPU_WC_SHIFT);
+
+	/* send out the packet */
+	dsi_write(base_addr, DSI_CPU_CMD_0, reg);
+	/* wait packet be sent out */
+	timeout = 1000;
+	do {
+		timeout--;
+		tmp = dsi_read(base_addr, DSI_CPU_CMD_0);
+		udelay(50);
+	} while ((tmp & CFG_CPU_CMD_REQ) && timeout);
+	if (timeout == 0) {
+		pr_info("%s: DSI send out packet maybe failed.\n", __func__);
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dsi_config_video_mode(struct spacemit_dsi_device *dsi_ctx, struct spacemit_mipi_info *mipi_info)
+{
+	uint32_t hsync_b, hbp_b, hact_b, hex_b, hfp_b, httl_b;
+	uint32_t hsync, hbp, hact, httl, v_total;
+	uint32_t hsa_wc, hbp_wc, hact_wc, hex_wc, hfp_wc, hlp_wc;
+	uint32_t bpp, hss_bcnt = 4, hse_bct = 4, lgp_over_head = 6, reg;
+	uint32_t slot_cnt0, slot_cnt1;
+	uint32_t dsi_ex_pixel_cnt = 0;
+	uint32_t dsi_hex_en = 0;
+	uint32_t width, lane_number, height_val;
+	void __iomem *base_addr = dsi_ctx->base_addr;
+	struct spacemit_dsi_advanced_setting *adv_setting = &dsi_ctx->adv_setting;
+
+	switch (mipi_info->rgb_mode) {
+	case DSI_INPUT_DATA_RGB_MODE_565:
+		bpp = 16;
+		break;
+	case DSI_INPUT_DATA_RGB_MODE_666PACKET:
+		bpp = 18;
+		break;
+	case DSI_INPUT_DATA_RGB_MODE_666UNPACKET:
+		bpp = 18;
+		break;
+	case DSI_INPUT_DATA_RGB_MODE_888:
+		bpp = 24;
+		break;
+	default:
+		bpp = 24;
+	}
+
+	if (mipi_info->dsc_enable) {
+		mipi_info->rgb_mode = DSI_INPUT_DATA_RGB_MODE_DSC;
+		bpp = 24; //Jessica: for dsc, this always be 24.
+	}
+
+	height_val = mipi_info->height + mipi_info->h_extra;
+
+	if (dsi_ctx->version >= DSI_VERSION_2)
+		v_total = height_val + mipi_info->vfp + mipi_info->vbp + mipi_info->vsync;
+	else
+		v_total = height_val + mipi_info->vfp + mipi_info->vbp + mipi_info->vsync + HIGH_BRUSH_V;
+
+	mipi_info->vrr_param.vrr_vfp = mipi_info->vfp;
+	if (mipi_info->split_enable) {
+		if (((mipi_info->width & 0x1) != 0) || ((mipi_info->lane_number & 0x1) != 0)) {
+			pr_err("%s: warning:Invalid split config(lane = %d, width = %d)\n",
+				__func__, mipi_info->lane_number, mipi_info->width);
+		}
+		width = mipi_info->width >> 1;
+		lane_number = mipi_info->lane_number >> 1;
+	} else {
+		width = mipi_info->width;
+		lane_number = mipi_info->lane_number;
+	}
+
+	if (mipi_info->dsc_enable)
+		hact_b = to_dsi_bcnt(width, mipi_info->dsc_bpc);
+	else
+		hact_b = to_dsi_bcnt(width, bpp);
+
+	hfp_b = to_dsi_bcnt(mipi_info->hfp, bpp);
+	hbp_b = to_dsi_bcnt(mipi_info->hbp, bpp);
+	hsync_b = to_dsi_bcnt(mipi_info->hsync, bpp);
+	hex_b = to_dsi_bcnt(dsi_ex_pixel_cnt, bpp);
+	httl_b = hact_b + hsync_b + hfp_b + hbp_b + hex_b;
+	slot_cnt0 = (httl_b - hact_b) / lane_number + 3;
+	slot_cnt1 = slot_cnt0;
+
+	hact = hact_b / lane_number;
+	hbp = hbp_b / lane_number;
+	hsync = hsync_b / lane_number;
+	httl = (hact_b + hfp_b + hbp_b + hsync_b) / lane_number;
+
+	/* word count in the unit of byte */
+	hsa_wc = (mipi_info->burst_mode == DSI_BURST_MODE_NON_BURST_SYNC_PULSE) ?
+		(hsync_b - hss_bcnt - lgp_over_head) : 0;
+
+	/* Hse is with backporch */
+	hbp_wc = (mipi_info->burst_mode == DSI_BURST_MODE_NON_BURST_SYNC_PULSE) ?
+		(hbp_b - hse_bct - lgp_over_head)
+		: (hsync_b + hbp_b - hss_bcnt - lgp_over_head);
+
+	hfp_wc = ((mipi_info->burst_mode == DSI_BURST_MODE_BURST) && (dsi_hex_en == 0)) ?
+		(hfp_b + hex_b - lgp_over_head - lgp_over_head) :
+		(hfp_b - lgp_over_head - lgp_over_head);
+
+	if (mipi_info->dsc_enable)
+		hact_wc = to_dsi_bcnt(width, mipi_info->dsc_bpc);
+	else
+		hact_wc = to_dsi_bcnt(width, bpp);
+
+	/* disable Hex currently */
+	hex_wc = 0;
+
+	/*  There is no hlp with active data segment.  */
+	hlp_wc = (mipi_info->burst_mode == DSI_BURST_MODE_NON_BURST_SYNC_PULSE) ?
+		(httl_b - hsync_b - hse_bct - lgp_over_head) :
+		(httl_b - hss_bcnt - lgp_over_head);
+
+	/* FIXME - need to double check the (*3) is bytes_per_pixel
+	 * from input data or output to panel
+	 */
+	DRM_DEBUG("silvie version %d %d %d\n", dsi_ctx->version, mipi_info->vpn_dly_cnt, mipi_info->vpn_tx_dly_cnt);
+	if (dsi_ctx->version >= DSI_VERSION_2)
+		if (mipi_info->vpn_dly_cnt && mipi_info->vpn_tx_dly_cnt)
+			dsi_write(base_addr, DSI_VPN_CTRL_0, ((mipi_info->vpn_dly_cnt & 0xFFFF) << CFG_VPN_DLY_CNT_SHIFT) |
+				CFG_VPN_HSYNC_GRANTEE_EN | CFG_VPN_1LN_DLY | (mipi_info->vpn_tx_dly_cnt & 0xFF));
+		else
+			dsi_write(base_addr, DSI_VPN_CTRL_0, (0x50 << CFG_VPN_DLY_CNT_SHIFT) | 0x508);
+	else
+		dsi_write(base_addr, DSI_VPN_CTRL_0, (0x50 << CFG_VPN_DLY_CNT_SHIFT) | 0xc08);
+
+	if (mipi_info->split_enable)
+		dsi_write(base_addr, DSI_VPN_CTRL_0, (0x258 << CFG_VPN_DLY_CNT_SHIFT) | 0x54f);
+
+    /* SET UP LCD1 TIMING REGISTERS FOR DSI BUS */
+	dsi_write(base_addr, DSI_VPN_TIMING_0, (hact << 16) | httl);
+	dsi_write(base_addr, DSI_VPN_TIMING_1, (hsync << 16) | hbp);
+	dsi_write(base_addr, DSI_VPN_TIMING_2, ((height_val)<<16) | (v_total));
+	dsi_write(base_addr, DSI_VPN_TIMING_3, ((mipi_info->vsync) << 16) | (mipi_info->vbp));
+
+    /* SET UP LCD1 WORD COUNT REGISTERS FOR DSI BUS */
+	dsi_write(base_addr, DSI_VPN_WC_0, (hbp_wc << 16) | hsa_wc);
+	dsi_write(base_addr, DSI_VPN_WC_1, (hfp_wc << 16) | hact_wc);
+	dsi_write(base_addr, DSI_VPN_WC_2, (hex_wc << 16) | hlp_wc);
+
+	dsi_write(base_addr, DSI_VPN_SLOT_CNT_0, (slot_cnt0 << 16) | slot_cnt0);
+	dsi_write(base_addr, DSI_VPN_SLOT_CNT_1, (slot_cnt1 << 16) | slot_cnt1);
+
+    /* Configure LCD control register 1 FOR DSI BUS */
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	adv_setting->hact_wc_en = 0;
+	adv_setting->timing_check_dis = 1;
+	adv_setting->auto_dly_dis = 1;
+#endif
+
+	reg = adv_setting->vsync_rst_en << CFG_VPN_VSYNC_RST_EN_SHIFT |
+			adv_setting->auto_wc_dis << CFG_VPN_AUTO_WC_DIS_SHIFT |
+			adv_setting->hact_wc_en << CFG_VPN_HACT_WC_EN_SHIFT |
+			adv_setting->timing_check_dis << CFG_VPN_TIMING_CHECK_DIS_SHIFT |
+			adv_setting->auto_dly_dis << CFG_VPN_AUTO_DLY_DIS_SHIFT |
+			adv_setting->hlp_pkt_en << CFG_VPN_HLP_PKT_EN_SHIFT |
+			adv_setting->hex_pkt_en << CFG_VPN_HEX_PKT_EN_SHIFT |
+			adv_setting->hfp_pkt_en << CFG_VPN_HFP_PKT_EN_SHIFT |
+			adv_setting->hbp_pkt_en << CFG_VPN_HBP_PKT_EN_SHIFT |
+			adv_setting->hse_pkt_en << CFG_VPN_HSE_PKT_EN_SHIFT |
+			adv_setting->hsa_pkt_en << CFG_VPN_HSA_PKT_EN_SHIFT |
+			adv_setting->hex_slot_en << CFG_VPN_HEX_SLOT_EN_SHIFT |
+			adv_setting->last_line_turn << CFG_VPN_LAST_LINE_TURN_SHIFT |
+			adv_setting->lpm_frame_en << CFG_VPN_LPM_FRAME_EN_SHIFT |
+			mipi_info->burst_mode << g_spacemit_dsi_info[dsi_ctx->version].vpn_burst_mode_shift |
+			mipi_info->rgb_mode << CFG_VPN_RGB_TYPE_SHIFT;
+	dsi_write(base_addr, DSI_VPN_CTRL_1, reg);
+
+	dsi_write_bits(base_addr, DSI_LCD_BDG_CTRL0, CFG_VPN_FIFO_AFULL_CNT_MASK,
+		0 << CFG_VPN_FIFO_AFULL_CNT_SHIT);
+//#ifdef CONFIG_ESD_SUPPORT
+	dsi_set_bits(base_addr, DSI_LCD_BDG_CTRL0, CFG_VPN_FIFO_AFULL_BYPASS);
+//#else
+	//dsi_clear_bits(base_addr, DSI_LCD_BDG_CTRL0, CFG_VPN_FIFO_AFULL_BYPASS);
+//#endif
+	dsi_set_bits(base_addr, DSI_LCD_BDG_CTRL0, CFG_PIXEL_SWAP);
+
+	dsi_enable_cmd_mode(base_addr, false);
+	dsi_enable_video_mode(base_addr, true);
+}
+
+static void spacemit_dsi_config_cmd_pkg(struct spacemit_dsi_device *dsi_ctx, struct spacemit_mipi_info *mipi_info)
+{
+	struct spacemit_dsi_info *dsi_info = g_spacemit_dsi_info;
+	u32 cmd_pkt_size = dsi_info[dsi_ctx->version].cmd_pkt_default_size;
+	u32 cmd_fifo_level = dsi_info[dsi_ctx->version].cmd_fifo_default_level;
+
+	if (mipi_info->cmd_pkg_ctrl > 0) {
+		cmd_pkt_size = mipi_info->cmd_pkg_ctrl >> 16;
+		cmd_fifo_level = mipi_info->cmd_pkg_ctrl & 0xffff;
+	} else if (mipi_info->cmd_pkg_ctrl == 0) {
+		switch (mipi_info->rgb_mode) {
+		case DSI_INPUT_DATA_RGB_MODE_565:
+			cmd_pkt_size = mipi_info->width * 2 + 1;
+			break;
+		case DSI_INPUT_DATA_RGB_MODE_666UNPACKET:
+		case DSI_INPUT_DATA_RGB_MODE_888:
+		default:
+			cmd_pkt_size = mipi_info->width * 3 + 1;
+		}
+		cmd_fifo_level = dsi_info[dsi_ctx->version].cmd_fifo_default_level;
+	}
+
+	while (cmd_pkt_size > dsi_info[dsi_ctx->version].cmd_pkt_max_size)
+		cmd_pkt_size = cmd_pkt_size / 2;
+
+	mipi_info->cmd_pkt_size = cmd_pkt_size;
+	mipi_info->cmd_fifo_level = cmd_fifo_level < dsi_info[dsi_ctx->version].cmd_fifo_max_level ?
+				cmd_fifo_level : dsi_info[dsi_ctx->version].cmd_fifo_max_level;
+}
+
+static void dsi_config_cmd_mode(struct spacemit_dsi_device *dsi_ctx, struct spacemit_mipi_info *mipi_info)
+{
+	int reg;
+	int rgb_mode, bpp;
+
+	switch (mipi_info->rgb_mode) {
+	case DSI_INPUT_DATA_RGB_MODE_565:
+		bpp = 16;
+		rgb_mode = 2;
+		break;
+	case DSI_INPUT_DATA_RGB_MODE_666UNPACKET:
+		bpp = 18;
+		rgb_mode = 1;
+		break;
+	case DSI_INPUT_DATA_RGB_MODE_888:
+		bpp = 24;
+		rgb_mode = 0;
+		break;
+	default:
+		pr_err("%s: unsupported rgb format!\n", __func__);
+		bpp = 24;
+		rgb_mode = 0;
+	}
+
+	pr_debug("bpp %d\n", bpp);
+
+	reg = mipi_info->te_enable << CFG_CPN_TE_EN_SHIFT |
+			rgb_mode << CFG_CPN_RGB_TYPE_SHIFT |
+			1 << CFG_CPN_BURST_MODE_SHIFT |
+			0 << CFG_CPN_DMA_DIS_SHIFT |
+			0 << CFG_CPN_ADDR0_EN_SHIFT;
+	dsi_write(dsi_ctx->base_addr, DSI_CPN_CMD, reg);
+
+	spacemit_dsi_config_cmd_pkg(dsi_ctx, mipi_info);
+	reg = mipi_info->cmd_pkt_size << CFG_CPN_PKT_CNT_SHIFT |
+			mipi_info->cmd_fifo_level << CFG_CPN_FIFO_FULL_LEVEL_SHIFT;
+	dsi_write(dsi_ctx->base_addr, DSI_CPN_CTRL_1, reg);
+
+	dsi_write_bits(dsi_ctx->base_addr, DSI_LCD_BDG_CTRL0, CFG_CPN_TE_EDGE_MASK,
+		mipi_info->te_pol << CFG_CPN_TE_EDGE_SHIFT);
+	dsi_write_bits(dsi_ctx->base_addr, DSI_LCD_BDG_CTRL0, CFG_CPN_VSYNC_EDGE_MASK,
+		mipi_info->vsync_pol << CFG_CPN_VSYNC_EDGE_SHIFT);
+	dsi_write_bits(dsi_ctx->base_addr, DSI_LCD_BDG_CTRL0, CFG_CPN_TE_MODE_MASK,
+		mipi_info->te_mode << CFG_CPN_TE_MODE_SHIFT);
+
+	reg = 0x80 << CFG_CPN_TE_DLY_CNT_SHIFT |
+			0 << CFG_CPN_TE_LINE_CNT_SHIFT;
+	dsi_write(dsi_ctx->base_addr, DSI_LCD_BDG_CTRL1, reg);
+
+	if (dsi_ctx->version >= DSI_VERSION_2)
+		dsi_write(dsi_ctx->base_addr, DSI_PHY_CTRL_2, CFG_DPHY_LANE_EN | CFG_DPHY_HSTX_RX);
+
+	dsi_enable_video_mode(dsi_ctx->base_addr, false);
+	dsi_enable_cmd_mode(dsi_ctx->base_addr, true);
+}
+
+static int dsi_write_cmd_array(struct spacemit_dsi_device *dsi_ctx,
+					struct spacemit_dsi_cmd_desc *cmds, int count)
+{
+	struct spacemit_dsi_cmd_desc cmd_line;
+	uint8_t type, parameter[SPACEMIT_DSI_MAX_TX_FIFO_BYTES], len;
+	uint32_t crc, loop;
+	int ret = 0;
+
+	pr_debug("%s: %d cmd will be write\n", __func__, count);
+
+	if (dsi_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	for (loop = 0; loop < count; loop++) {
+		cmd_line = cmds[loop];
+		type = cmd_line.cmd_type;
+		len = cmd_line.length;
+		memset(parameter, 0x00, len + 6);
+		parameter[0] = type & 0xff;
+		switch (type) {
+		case SPACEMIT_DSI_DCS_SWRITE:
+		case SPACEMIT_DSI_DCS_SWRITE1:
+		case SPACEMIT_DSI_DCS_READ:
+		case SPACEMIT_DSI_GENERIC_READ1:
+		case SPACEMIT_DSI_SET_MAX_PKT_SIZE:
+			memcpy(&parameter[1], cmd_line.data, len);
+			len = 4;
+			break;
+		case SPACEMIT_DSI_GENERIC_LWRITE:
+		case SPACEMIT_DSI_DCS_LWRITE:
+		case SPACEMIT_DSI_PICTURE_PARAMETER_SET:
+		case SPACEMIT_DSI_COMPRESSION_MODE:
+			parameter[1] = len & 0xff;
+			parameter[2] = 0;
+			memcpy(&parameter[4], cmd_line.data, len);
+			crc = calculate_crc16(&parameter[4], len);
+			parameter[len + 4] = crc & 0xff;
+			parameter[len + 5] = (crc >> 8) & 0xff;
+			len += 6;
+			break;
+		default:
+			pr_err("%s: data type not supported 0x%8x\n", __func__, type);
+			break;
+		}
+
+		parameter[3] = calculate_ecc(parameter);
+
+		/* send dsi commands */
+		ret = dsi_write_cmd(dsi_ctx->base_addr, parameter, len, cmd_line.lp);
+		if (ret)
+			return -1;
+
+	//	if (0 != cmd_line.delay)
+	//		msleep(cmd_line.delay);
+	}
+	return 0;
+}
+
+static int dsi_read_cmd_array(struct spacemit_dsi_device *dsi_ctx, struct spacemit_dsi_rx_buf *dbuf,
+					struct spacemit_dsi_cmd_desc *cmds, int count)
+{
+	uint8_t parameter[SPACEMIT_DSI_MAX_RX_FIFO_BYTES];
+	uint32_t i, rx_reg, timeout, tmp, packet,
+	    data_pointer, byte_count;
+
+	pr_debug("%s: %d cmds will be write\n", __func__, count);
+
+	if (dsi_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	memset(dbuf, 0x0, sizeof(struct spacemit_dsi_rx_buf));
+	dsi_write_cmd_array(dsi_ctx, cmds, count);
+
+	timeout = 1000;
+	do {
+		timeout--;
+		tmp = dsi_read(dsi_ctx->base_addr, DSI_IRQ_ST);
+	} while (((tmp & IRQ_RX_PKT) == 0) && timeout);
+	if (timeout == 0) {
+		pr_err("%s: dsi didn't receive packet, irq status 0x%x\n", __func__, tmp);
+		return -1;
+	}
+
+	if (tmp & IRQ_RX_TRG3)
+		pr_err("%s: not defined package is received\n", __func__);
+	if (tmp & IRQ_RX_TRG2)
+		pr_err("%s: ACK package is received\n", __func__);
+	if (tmp & IRQ_RX_TRG1)
+		pr_err("%s: TE trigger is received\n", __func__);
+	if (tmp & IRQ_RX_ERR) {
+		tmp = dsi_read(dsi_ctx->base_addr, DSI_RX_PKT_HDR_0);
+		pr_err("%s: error: ACK with error report (0x%x)\n", __func__, tmp);
+	}
+
+	packet = dsi_read(dsi_ctx->base_addr, DSI_RX_PKT_ST_0);
+
+	data_pointer = (packet & CFG_RX_PKT0_PTR_MASK) >> CFG_RX_PKT0_PTR_SHIFT;
+	tmp = dsi_read(dsi_ctx->base_addr, DSI_RX_PKT_CTRL_1);
+	byte_count = tmp & CFG_RX_PKT_BCNT_MASK;
+
+	memset(parameter, 0x00, byte_count);
+	for (i = data_pointer; i < data_pointer + byte_count; i++) {
+		rx_reg = dsi_read(dsi_ctx->base_addr, DSI_RX_PKT_CTRL);
+		rx_reg &= ~CFG_RX_PKT_RD_PTR_MASK;
+		rx_reg |= CFG_RX_PKT_RD_REQ | (i << CFG_RX_PKT_RD_PTR_SHIFT);
+		dsi_write(dsi_ctx->base_addr, DSI_RX_PKT_CTRL, rx_reg);
+		count = 10000;
+		do {
+			count--;
+			rx_reg = dsi_read(dsi_ctx->base_addr, DSI_RX_PKT_CTRL);
+		} while (rx_reg & CFG_RX_PKT_RD_REQ && count);
+		if (count == 0)
+			pr_err("%s: error: read Rx packet FIFO error\n", __func__);
+		parameter[i - data_pointer] = rx_reg & 0xff;
+	}
+	switch (parameter[0]) {
+	case SPACEMIT_DSI_ACK_ERR_RESP:
+		pr_err("%s: error: Acknowledge with error report\n", __func__);
+		break;
+	case SPACEMIT_DSI_EOTP:
+		pr_err("%s: error: End of Transmission packet\n", __func__);
+		break;
+	case SPACEMIT_DSI_GEN_READ1_RESP:
+	case SPACEMIT_DSI_DCS_READ1_RESP:
+		dbuf->data_type = parameter[0];
+		dbuf->length = 1;
+		memcpy(dbuf->data, &parameter[1], dbuf->length);
+		break;
+	case SPACEMIT_DSI_GEN_READ2_RESP:
+	case SPACEMIT_DSI_DCS_READ2_RESP:
+		dbuf->data_type = parameter[0];
+		dbuf->length = 2;
+		memcpy(dbuf->data, &parameter[1], dbuf->length);
+		break;
+	case SPACEMIT_DSI_GEN_LREAD_RESP:
+	case SPACEMIT_DSI_DCS_LREAD_RESP:
+		dbuf->data_type = parameter[0];
+		dbuf->length = (parameter[2] << 8) | parameter[1];
+		memcpy(dbuf->data, &parameter[4], dbuf->length);
+		pr_debug("%s: read %d data: 0x%x, 0x%x, 0x%x, 0x%x, 0x%x\n", __func__, dbuf->length,
+			parameter[4], parameter[5], parameter[6], parameter[7], parameter[8]);
+		break;
+	}
+	return 0;
+}
+
+static void dsi_open_dphy(struct spacemit_dsi_device *device_ctx, bool ready)
+{
+	struct spacemit_dphy *spacemit_dphy = device_ctx->phy;
+	struct spacemit_dphy_ctx *dphy_config = &spacemit_dphy->ctx;
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+
+	dphy_config->base_addr = device_ctx->base_addr;
+
+	if (mipi_info->split_enable)
+		dphy_config->lane_num = mipi_info->lane_number >> 1;
+	else
+		dphy_config->lane_num = mipi_info->lane_number;
+	dphy_config->status = DPHY_STATUS_UNINIT;
+
+	if (ready) {
+		dphy_config->status = DPHY_STATUS_INIT;
+		return;
+	}
+
+	spacemit_dphy_resume(spacemit_dphy);
+}
+
+static void dsi_close_dphy(struct spacemit_dphy *spacemit_dphy)
+{
+	spacemit_dphy_suspend(spacemit_dphy);
+}
+
+static void dsi_ready_dphy(struct spacemit_dsi_device *device_ctx)
+{
+}
+
+static void spacemit_dsi_sw_sleep(struct spacemit_dsi_device *device_ctx, bool sleep)
+{
+	uint32_t reg;
+	void *base;
+	base = ioremap(0xD4282800, 0x400);
+
+	reg = readl(base + APMU_LCD_CLK_RES_CTRL1);
+	if (sleep)
+		reg |= APMU_LCD_SW_SLEEP;
+	else
+		reg &= ~(APMU_LCD_SW_SLEEP);
+	writel(reg, base + APMU_LCD_CLK_RES_CTRL1);
+	udelay(100);
+
+	pr_debug("%s, %d sleep:%d\n", __func__, __LINE__, sleep);
+}
+
+/**
+ * spacemit_dsi_update_vrr - update variable refresh rate parameters
+ * @vrr_param: variable refresh rate parameters, for example：vrr_vfp.
+ */
+static void spacemit_dsi0_update_vrr(struct spacemit_dsi_device *device_ctx, struct spacemit_dsi_vrr_param *vrr_param)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+	void __iomem *base_addr = device_ctx->base_addr;
+	uint32_t v_total;
+
+	if (!vrr_param)
+		return;
+
+	if (device_ctx->version < DSI_VERSION_2 ||
+			vrr_param->vrr_vfp == mipi_info->vrr_param.vrr_vfp) {
+		pr_debug("%s, dsi version_%d: vrr_vfp = %d, mipi vrr vfp = %d.\n",
+				__func__, device_ctx->version, vrr_param->vrr_vfp, mipi_info->vrr_param.vrr_vfp);
+		return;
+	}
+
+	v_total = mipi_info->height + vrr_param->vrr_vfp + mipi_info->vbp + mipi_info->vsync;
+	dsi_write(base_addr, DSI_VPN_TIMING_2, ((mipi_info->height) << 16) | (v_total));
+
+	mipi_info->vrr_param.vrr_vfp = vrr_param->vrr_vfp;
+}
+
+static void spacemit_dsi1_update_vrr(struct spacemit_dsi_device *device_ctx, struct spacemit_dsi_vrr_param *vrr_param)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+	void __iomem *base_addr = device_ctx->base_addr;
+	uint32_t v_total;
+
+	if (!vrr_param)
+		return;
+
+	if (device_ctx->version < DSI_VERSION_2 ||
+			vrr_param->vrr_vfp == mipi_info->vrr_param.vrr_vfp) {
+		pr_debug("%s, dsi version_%d: vrr_vfp = %d, mipi vrr vfp = %d.\n",
+				__func__, device_ctx->version, vrr_param->vrr_vfp, mipi_info->vrr_param.vrr_vfp);
+		return;
+	}
+
+	v_total = mipi_info->height + vrr_param->vrr_vfp + mipi_info->vbp + mipi_info->vsync;
+	dsi_write(base_addr, DSI_VPN_TIMING_2, ((mipi_info->height) << 16) | (v_total));
+
+	mipi_info->vrr_param.vrr_vfp = vrr_param->vrr_vfp;
+}
+
+static void spacemit_dsi_update_vrr(struct spacemit_dsi_device *device_ctx, struct spacemit_dsi_vrr_param *vrr_param)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+
+	spacemit_dsi0_update_vrr(device_ctx, vrr_param);
+	if (mipi_info->split_enable)
+		spacemit_dsi1_update_vrr(device_ctx, vrr_param);
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+}
+
+static int spacemit_dsi0_open(struct spacemit_dsi_device* device_ctx, bool ready)
+{
+	int lane_number;
+	struct spacemit_mipi_info *mipi_info;
+	uint32_t irq_st;
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	mipi_info = &device_ctx->mipi_info;
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	if (!ready)
+		dptc_board_init();
+#endif
+	if (device_ctx->status == DSI_STATUS_OPENED)
+		return 0;
+
+	pr_debug("%s: dsi(%d) Enter, ready = %d\n", __func__, device_ctx->id, ready);
+
+	if (mipi_info->split_enable)
+		lane_number = mipi_info->lane_number >> 1;
+	else
+		lane_number = mipi_info->lane_number;
+
+	if (!ready)
+		dsi_reset(device_ctx->base_addr);
+
+	dsi_open_dphy(device_ctx, ready);
+	if (!ready) {
+		dsi_enable_split_mode(device_ctx->base_addr, mipi_info->split_enable);
+		dsi_enable_lptx_lanes(device_ctx->base_addr, spacemit_dsi_lane[lane_number]);
+		dsi_enable_eotp(device_ctx->base_addr, mipi_info->eotp_enable);
+	}
+
+	/*clear interrupt*/
+	irq_st = dsi_read(device_ctx->base_addr, DSI_IRQ_ST);
+	dsi_write(device_ctx->base_addr, DSI_IRQ_ST, irq_st);
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	dsi_enable_irq(device_ctx->base_addr, false);
+#else
+	dsi_enable_irq(device_ctx->base_addr, true);
+#endif
+	if (device_ctx->version >= DSI_VERSION_2)
+		spacemit_dsi_sw_sleep(device_ctx, false);
+
+	device_ctx->status = DSI_STATUS_OPENED;
+	return 0;
+}
+
+static int spacemit_dsi1_open(struct spacemit_dsi_device* device_ctx, bool ready)
+{
+	int lane_number;
+	struct spacemit_mipi_info *mipi_info;
+	uint32_t irq_st;
+
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	mipi_info = &device_ctx->mipi_info;
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	if (!ready)
+		dptc_board_init();
+#endif
+
+	pr_debug("%s: dsi(%d) Enter, ready = %d\n", __func__, device_ctx->id, ready);
+
+	if (mipi_info->split_enable)
+		lane_number = mipi_info->lane_number >> 1;
+	else
+		lane_number = mipi_info->lane_number;
+
+	if (!ready)
+		dsi_reset(device_ctx->base_addr);
+
+	dsi_open_dphy(device_ctx, ready);
+	if (!ready) {
+		dsi_enable_split_mode(device_ctx->base_addr, mipi_info->split_enable);
+		dsi_enable_lptx_lanes(device_ctx->base_addr, spacemit_dsi_lane[lane_number]);
+		dsi_enable_eotp(device_ctx->base_addr, mipi_info->eotp_enable);
+	}
+
+	/*clear interrupt*/
+	irq_st = dsi_read(device_ctx->base_addr, DSI_IRQ_ST);
+	dsi_write(device_ctx->base_addr, DSI_IRQ_ST, irq_st);
+#ifdef CONFIG_SOC_SPACEMIT_K3_FPGA
+	dsi_enable_irq(device_ctx->base_addr, false);
+#else
+	dsi_enable_irq(device_ctx->base_addr, true);
+#endif
+
+	if (device_ctx->version >= DSI_VERSION_2)
+		spacemit_dsi_sw_sleep(device_ctx, false);
+
+	return 0;
+}
+
+static int spacemit_dsi_open(struct spacemit_dsi_device *device_ctx, bool ready)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	int ret;
+
+	ret = spacemit_dsi0_open(device_ctx, ready);
+	if (ret)
+		return ret;
+
+	if (mipi_info->split_enable) {
+		ret = spacemit_dsi1_open(device_ctx, ready);
+		if (ret)
+			return ret;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	return 0;
+}
+
+static int spacemit_dsi0_close(struct spacemit_dsi_device* device_ctx)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	if (device_ctx->status == DSI_STATUS_UNINIT)
+		return 0;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	dsi_enable_irq(device_ctx->base_addr, false);
+
+	dsi_close_dphy(device_ctx->phy);
+
+	if (device_ctx->version >= DSI_VERSION_2)
+		spacemit_dsi_sw_sleep(device_ctx, true);
+
+	device_ctx->status = DSI_STATUS_UNINIT;
+
+	pr_debug("%s: dsi(%d) leave\n", __func__, device_ctx->id);
+	return 0;
+}
+
+static int spacemit_dsi1_close(struct spacemit_dsi_device* device_ctx)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	dsi_enable_irq(device_ctx->base_addr, false);
+
+	dsi_close_dphy(device_ctx->phy);
+
+	if (device_ctx->version >= DSI_VERSION_2)
+		spacemit_dsi_sw_sleep(device_ctx, true);
+
+	pr_debug("%s: dsi(%d) leave\n", __func__, device_ctx->id);
+	return 0;
+}
+
+static int spacemit_dsi_close(struct spacemit_dsi_device *device_ctx)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	int ret;
+
+	ret = spacemit_dsi0_close(device_ctx);
+	if (ret)
+		return ret;
+
+	if (mipi_info->split_enable) {
+		ret = spacemit_dsi1_close(device_ctx);
+		if (ret)
+			return ret;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	return 0;
+}
+
+static int spacemit_dsi0_ready_for_datatx(struct spacemit_dsi_device* device_ctx)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+	struct spacemit_mipi_info *mipi_info;
+
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	mipi_info = &device_ctx->mipi_info;
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	if (device_ctx->status == DSI_STATUS_UNINIT)
+		return 0;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	if (mipi_info->work_mode == SPACEMIT_DSI_MODE_CMD)
+		dsi_config_cmd_mode(device_ctx, mipi_info);
+	else
+		dsi_config_video_mode(device_ctx, mipi_info);
+
+	dsi_ready_dphy(device_ctx);
+	return 0;
+}
+
+static int spacemit_dsi1_ready_for_datatx(struct spacemit_dsi_device* device_ctx)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+	struct spacemit_mipi_info *mipi_info;
+
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid device_ctx param\n", __func__);
+		return -1;
+	}
+
+	mipi_info = &device_ctx->mipi_info;
+
+	if (mipi_info == NULL) {
+		pr_err("%s: Invalid mipi_info\n", __func__);
+		return -1;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	if (mipi_info->work_mode == SPACEMIT_DSI_MODE_CMD)
+		dsi_config_cmd_mode(device_ctx, mipi_info);
+	else
+		dsi_config_video_mode(device_ctx, mipi_info);
+
+	dsi_ready_dphy(device_ctx);
+	return 0;
+}
+
+static int spacemit_dsi_ready_for_datatx(struct spacemit_dsi_device *device_ctx)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	int ret;
+
+	ret = spacemit_dsi0_ready_for_datatx(device_ctx);
+	if (ret)
+		return ret;
+
+	if (mipi_info->split_enable) {
+		ret = spacemit_dsi1_ready_for_datatx(device_ctx);
+		if (ret)
+			return ret;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	return 0;
+}
+
+static void spacemit_dsi0_enable_irq(struct spacemit_dsi_device *device_ctx, bool enable)
+{
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	dsi_enable_irq(device_ctx->base_addr, enable);
+}
+
+static void spacemit_dsi1_enable_irq(struct spacemit_dsi_device *device_ctx, bool enable)
+{
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+
+	dsi_enable_irq(device_ctx->base_addr, enable);
+}
+
+static void spacemit_dsi_enable_irq(struct spacemit_dsi_device *device_ctx, bool enable)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	spacemit_dsi0_enable_irq(device_ctx, enable);
+
+	if (mipi_info->split_enable)
+		spacemit_dsi1_enable_irq(device_ctx, enable);
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+}
+
+static int spacemit_dsi0_close_datatx(struct spacemit_dsi_device* device_ctx)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	if (device_ctx->status == DSI_STATUS_UNINIT)
+		return 0;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	dsi_enable_cmd_mode(device_ctx->base_addr, false);
+	dsi_enable_video_mode(device_ctx->base_addr, false);
+
+	return 0;
+}
+
+static int spacemit_dsi1_close_datatx(struct spacemit_dsi_device *device_ctx)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+	if (device_ctx == NULL) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	dsi_enable_cmd_mode(device_ctx->base_addr, false);
+	dsi_enable_video_mode(device_ctx->base_addr, false);
+
+	return 0;
+}
+
+static int spacemit_dsi_close_datatx(struct spacemit_dsi_device *device_ctx)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	int ret;
+
+	ret = spacemit_dsi0_close_datatx(device_ctx);
+	if (ret)
+		return ret;
+
+	if (mipi_info->split_enable) {
+		ret = spacemit_dsi1_close_datatx(device_ctx);
+		if (ret)
+			return ret;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	return 0;
+}
+
+static int spacemit_dsi0_write_cmds(struct spacemit_dsi_device *device_ctx, struct spacemit_dsi_cmd_desc *cmds, int count)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+
+	if ((device_ctx == NULL) || (cmds == NULL)) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	if (device_ctx->status == DSI_STATUS_UNINIT)
+		return 0;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	return dsi_write_cmd_array(device_ctx, cmds, count);
+}
+
+static int spacemit_dsi1_write_cmds(struct spacemit_dsi_device *device_ctx, struct spacemit_dsi_cmd_desc *cmds, int count)
+{
+
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+	if ((device_ctx == NULL) || (cmds == NULL)) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+
+	if (device_ctx->status == DSI_STATUS_UNINIT)
+		return 0;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+
+	return dsi_write_cmd_array(device_ctx, cmds, count);
+}
+
+static int spacemit_dsi_write_cmds(struct spacemit_dsi_device *device_ctx,
+					struct spacemit_dsi_cmd_desc *cmds, int count)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	int ret;
+
+	ret = spacemit_dsi0_write_cmds(device_ctx, cmds, count);
+	if (ret)
+		return ret;
+
+	if (mipi_info->split_enable) {
+		ret = spacemit_dsi1_write_cmds(device_ctx, cmds, count);
+		if (ret)
+			return ret;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	return 0;
+}
+
+static int spacemit_dsi_read_cmds(struct spacemit_dsi_device *device_ctx, struct spacemit_dsi_rx_buf *dbuf,
+								struct spacemit_dsi_cmd_desc *cmds, int count)
+{
+#ifdef LCD_IS_READY
+	return 0;
+#endif
+
+	if ((device_ctx == NULL) || (cmds == NULL)) {
+		pr_err("%s: Invalid param\n", __func__);
+		return -1;
+	}
+
+	if (device_ctx->status == DSI_STATUS_UNINIT)
+		return 0;
+
+	pr_debug("%s: dsi(%d) Enter\n", __func__, device_ctx->id);
+	return dsi_read_cmd_array(device_ctx, dbuf, cmds, count);
+}
+
+static int spacemit_dsi_parse_dt(struct spacemit_dsi_device *device_ctx, struct device_node *np)
+{
+	return 0;
+}
+
+static int spacemit_dsi0_isr(struct spacemit_dsi_device* device_ctx)
+{
+	uint32_t irq_st;
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	irq_st = dsi_read(device_ctx->base_addr, DSI_IRQ_ST);
+
+	/*clear interrupt*/
+	dsi_write(device_ctx->base_addr, DSI_IRQ_ST, irq_st);
+
+	if (irq_st & DSI_IRQ_PHY_FIFO_UNDERRUN)
+		pr_err_ratelimited("DSI: DSI_IRQ_PHY_FIFO_UNDERRUN\n");
+
+	if (irq_st & DSI_IRQ_VPN_BF_UNDERRUN_ERR)
+		pr_err_ratelimited("DSI: DSI_IRQ_VPN_BF_UNDERRUN_ERR\n");
+
+	if (irq_st & DSI_IRQ_VPN_BF_OVERRUN_ERR)
+		pr_err_ratelimited("DSI: DSI_IRQ_VPN_BF_OVERRUN_ERR\n");
+
+	return 0;
+}
+
+static int spacemit_dsi1_isr(struct spacemit_dsi_device* device_ctx)
+{
+	uint32_t irq_st;
+	device_ctx->base_addr = device_ctx->base_addr_dsi1;
+
+	irq_st = dsi_read(device_ctx->base_addr, DSI_IRQ_ST);
+
+	/*clear interrupt*/
+	dsi_write(device_ctx->base_addr, DSI_IRQ_ST, irq_st);
+
+	if (irq_st & DSI_IRQ_PHY_FIFO_UNDERRUN)
+		pr_err_ratelimited("DSI: DSI_IRQ_PHY_FIFO_UNDERRUN\n");
+
+	if (irq_st & DSI_IRQ_VPN_BF_UNDERRUN_ERR)
+		pr_err_ratelimited("DSI: DSI_IRQ_VPN_BF_UNDERRUN_ERR\n");
+
+	if (irq_st & DSI_IRQ_VPN_BF_OVERRUN_ERR)
+		pr_err_ratelimited("DSI: DSI_IRQ_VPN_BF_OVERRUN_ERR\n");
+
+	return 0;
+}
+
+static int spacemit_dsi_isr(struct spacemit_dsi_device *device_ctx)
+{
+	struct spacemit_mipi_info *mipi_info = &device_ctx->mipi_info;
+	int ret;
+
+	ret = spacemit_dsi0_isr(device_ctx);
+	if (ret)
+		return ret;
+
+	if (mipi_info->split_enable) {
+		ret = spacemit_dsi1_isr(device_ctx);
+		if (ret)
+			return ret;
+	}
+
+	device_ctx->base_addr = device_ctx->base_addr_dsi0;
+
+	return 0;
+}
+
+static struct dsi_core_ops dsi_core_ops = {
+	.parse_dt = spacemit_dsi_parse_dt,
+	.isr = spacemit_dsi_isr,
+	.dsi_open = spacemit_dsi_open,
+	.dsi_close = spacemit_dsi_close,
+	.dsi_write_cmds = spacemit_dsi_write_cmds,
+	.dsi_read_cmds = spacemit_dsi_read_cmds,
+	.dsi_ready_for_datatx = spacemit_dsi_ready_for_datatx,
+	.dsi_close_datatx = spacemit_dsi_close_datatx,
+	.dsi_enable_irq = spacemit_dsi_enable_irq,
+	.dsi_update_vrr = spacemit_dsi_update_vrr,
+};
+
+static struct ops_entry entry = {
+	.ver = "synopsys-dhost",
+	.ops = &dsi_core_ops,
+};
+#ifndef MODULE
+static int __init dsi_core_register(void)
+{
+	return dsi_core_ops_register(&entry);
+}
+
+subsys_initcall(dsi_core_register);
+#else
+int dsi_core_register(void)
+{
+	return dsi_core_ops_register(&entry);
+}
+
+#endif
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/dsi/spacemit_dsi_hw.h b/drivers/gpu/drm/spacemit/dsi/spacemit_dsi_hw.h
new file mode 100644
index 000000000000..aec9b2b0cd05
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/dsi/spacemit_dsi_hw.h
@@ -0,0 +1,367 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_DSI_HW_H_
+#define _SPACEMIT_DSI_HW_H_
+
+#include <linux/io.h>
+#include <drm/drm_print.h>
+
+#define APMU_LCD_CLK_RES_CTRL1		(0x44)
+#define APMU_LCD_SW_SLEEP		BIT(7)
+
+#define DSI_CTRL_0 0x0
+#define DSI_CTRL_1 0x4
+#define DSI_IRQ_ST1  0x8
+#define DSI_IRQ_MASK1 0xC
+#define DSI_IRQ_ST 0x10
+#define DSI_IRQ_MASK 0x14
+#define DSI_IRQ_PHY_FIFO_UNDERRUN	BIT(23)
+#define DSI_IRQ_VPN_BF_UNDERRUN_ERR	BIT(19)
+#define DSI_IRQ_VPN_BF_OVERRUN_ERR	BIT(17)
+#define DSI_IRQ_MASK_BITS		(DSI_IRQ_PHY_FIFO_UNDERRUN | \
+					DSI_IRQ_VPN_BF_UNDERRUN_ERR | \
+					DSI_IRQ_VPN_BF_OVERRUN_ERR)
+
+#ifdef CONFIG_SPACEMIT_FPGA
+#define DSI_FPGA_PHY_CTRL_0 0x18
+#define DSI_FPGA_PHY_CTRL_1 0x1C
+#endif
+
+#define DSI_CPU_CMD_0 0x20
+#define DSI_CPU_CMD_1 0x24
+#define DSI_CPU_CMD_3 0x2C
+#define DSI_CPU_WDAT 0x30
+#define DSI_CPU_STATUS_0 0x34
+#define DSI_CPU_STATUS_1 0x38
+#define DSI_CPU_STATUS_2 0x3C
+#define DSI_CPU_STATUS_3 0x40
+#define DSI_CPU_STATUS_4 0x44
+
+#define DSI_CPN_STATUS_1 0x4C
+#define DSI_CPN_CMD 0x50
+#define DSI_CPN_CTRL_0 0x54
+#define DSI_CPN_CTRL_1 0x58
+#define DSI_CPN_STATUS_0 0x5C
+
+#define DSI_RX_PKT_ST_0 0x60
+#define DSI_RX_PKT_HDR_0 0x64
+#define DSI_RX_PKT_ST_1 0x68
+#define DSI_RX_PKT_HDR_1 0x6C
+#define DSI_RX_PKT_CTRL 0x70
+#define DSI_RX_PKT_CTRL_1 0x74
+#define DSI_RX_PKT_ST_2 0x78
+#define DSI_RX_PKT_HDR_2 0x7C
+
+#define DSI_LCD_BDG_CTRL0 0x84
+#define DSI_LCD_BDG_CTRL1 0x88
+
+#define DSI_TX_TIMER 0xE4
+#define DSI_RX_TIMER 0xE8
+#define DSI_TURN_TIMER 0xEC
+
+#define DSI_VPN_CTRL_0 0x100
+#define DSI_VPN_CTRL_1 0x104
+#define DSI_VPN_TIMING_0 0x110
+#define DSI_VPN_TIMING_1 0x114
+#define DSI_VPN_TIMING_2 0x118
+#define DSI_VPN_TIMING_3 0x11C
+#define DSI_VPN_WC_0 0x120
+#define DSI_VPN_WC_1 0x124
+#define DSI_VPN_WC_2 0x128
+#define DSI_VPN_SLOT_CNT_0 0x130
+#define DSI_VPN_SLOT_CNT_1 0x134
+#define DSI_VPN_SYNC_CODE 0x138
+#define DSI_VPN_STATUS_0 0x140
+#define DSI_VPN_STATUS_1 0x144
+#define DSI_VPN_STATUS_2 0x148
+#define DSI_VPN_STATUS_3 0x14C
+#define DSI_VPN_STATUS_4 0x150
+
+#define DSI_PHY_CTRL_0 0x180
+#define DSI_PHY_CTRL_1 0x184
+#define DSI_PHY_CTRL_2 0x188
+#define DSI_PHY_CTRL_3 0x18C
+#define DSI_PHY_STATUS_0 0x190
+#define DSI_PHY_STATUS_1 0x194
+#define DSI_PHY_LPRX_0 0x198
+#define DSI_PHY_LPRX_1 0x19C
+#define DSI_PHY_LPTX_0 0x1A0
+#define DSI_PHY_LPTX_1 0x1A4
+#define DSI_PHY_LPTX_2 0x1A8
+#define DSI_PHY_STATUS_2 0x1AC
+#define DSI_PHY_TIME_0 0x1C0
+#define DSI_PHY_TIME_1 0x1C4
+#define DSI_PHY_TIME_2 0x1C8
+#define DSI_PHY_TIME_3 0x1CC
+#define DSI_PHY_CODE_0 0x1D0
+#define DSI_PHY_CODE_1 0x1D4
+#define DSI_PHY_ANA_PWR_CTRL 0x1E0
+#define DSI_PHY_ANA_CTRL0 0x1E4
+#define DSI_PHY_ANA_CTRL1 0x1E8
+
+//DSI_CTRL_0 0x0
+#define CFG_SOFT_RST BIT(31)
+#define CFG_SOFT_RST_REG BIT(30)
+#define CFG_CLR_PHY_FIFO BIT(29)
+#define CFG_RST_TXLP BIT(28)
+#define CFG_RST_CPU BIT(27)
+#define CFG_RST_CPN BIT(26)
+#define CFG_RST_VPN BIT(24)
+#define CFG_DSI_PHY_RST BIT(23)
+#define CFG_VPN_TX_EN BIT(8)
+#define CFG_VPN_SLV BIT(4)
+#define CFG_CPN_EN BIT(2)
+#define CFG_VPN_EN BIT(0)
+
+//DSI_CTRL_1 0x4
+#define CFG_EOTP_EN BIT(8)
+
+//DSI_IRQ_ST 0x10
+#define IRQ_RX_ERR BIT(25)
+#define IRQ_RX_TRG3 BIT(7)
+#define IRQ_RX_TRG2 BIT(6)
+#define IRQ_RX_TRG1 BIT(5)
+#define IRQ_RX_TRG0 BIT(4)
+#define IRQ_RX_PKT BIT(2)
+
+#ifdef CONFIG_SPACEMIT_FPGA
+//DSI_FPGA_PHY_CTRL_0 0x18
+#define CFG_DPHY_RSETZ 0
+#define CFG_DPHY_SHUTDOWN 1
+#define CFG_DPHY_RSTZCAL 2
+#define CFG_DPHY_TXRXZ 3
+#define CFG_DPHY_MASSLVZ 4
+#define CFG_DPHY_ENABLE0 5
+#define CFG_DPHY_ENABLE1 6
+#define CFG_DPHY_ENABLECLK 7
+#define CFG_DPHY_HSREQ_LANECLK 8
+#define CFG_DPHY_HSREQ_LANE0 9
+#define CFG_DPHY_HSREQ_LANE1 10
+#define CFG_DPHY_HSREQ_LANE2 11
+#define CFG_DPHY_HSREQ_LANE3 12
+#define CFG_DPHY_TXRX_BYTECLK_REV 13
+#define CFG_DPHY_FCLK_REV 14
+
+//DSI_FPGA_PHY_CTRL_1 0x1C
+#define CFG_DPHY_TESTCLK 0
+#define CFG_DPHY_TESTCLR 1
+#define CFG_DPHY_TESTEN 2
+#define CFG_DPHY_TXRXZ 3
+#define CFG_DPHY_TESTDIN 8
+#define CFG_DPHY_TESTDOUT 16
+#define CFG_DPHY_LOCK 24
+#endif
+
+//DSI_CPU_CMD_0 0x20
+#define CFG_CPU_CMD_REQ BIT(31)
+#define CFG_CPU_SP BIT(30)
+#define CFG_CPU_TURN BIT(29)
+#define CFG_CPU_TXLP BIT(27)
+#define CFG_CPU_WC_SHIFT 0
+
+//DSI_CPU_CMD_1 0x24
+#define CFG_TXLP_LPDT_SHIFT 20
+
+#define CFG_TXLP_LPDT_MASK (0xF << CFG_TXLP_LPDT_SHIFT)
+
+//DSI_CPU_CMD_3 0x2C
+#define CFG_CPU_DAT_REQ BIT(31)
+#define CFG_CPU_DAT_RW BIT(30)
+#define CFG_CPU_DAT_ADDR_SHIFT 16
+
+//DSI_CPN_CMD 0x50
+#define CFG_CPN_TE_EN_SHIFT 28
+#define CFG_CPN_RGB_TYPE_SHIFT 24
+#define CFG_CPN_BURST_MODE_SHIFT 3
+#define CFG_CPN_FIRSTP_SEL_SHIFT 2
+#define CFG_CPN_DMA_DIS_SHIFT 1
+#define CFG_CPN_ADDR0_EN_SHIFT 0
+
+//DSI_CPN_CTRL_1 0X58
+#define CFG_CPN_PKT_CNT_SHIFT 	16
+#define CFG_CPN_FIFO_FULL_LEVEL_SHIFT 0
+
+//DSI_RX_PKT_ST_0 0x60
+#define CFG_RX_PKT0_PTR_SHIFT 16
+#define CFG_RX_PKT0_PTR_MASK (0x3F << CFG_RX_PKT0_PTR_SHIFT)
+
+//DSI_RX_PKT_CTRL 0x70
+#define CFG_RX_PKT_RD_REQ BIT(31)
+#define CFG_RX_PKT_RD_PTR_SHIFT 16
+#define CFG_RX_PKT_RD_PTR_MASK (0x3F << CFG_RX_PKT_RD_PTR_SHIFT)
+#define CFG_RX_PKT_RD_DATA_SHIFT 0
+#define CFG_RX_PKT_RD_DATA_MASK (0x3F << CFG_RX_PKT_RD_DATA_SHIFT)
+
+//DSI_RX_PKT_CTRL_1 0x74
+#define CFG_RX_PKT_BCNT_SHIFT 0
+#define CFG_RX_PKT_BCNT_MASK (0xff << CFG_RX_PKT_BCNT_SHIFT)
+
+//DSI_LCD_BDG_CTRL0 0x84
+#define CFG_VPN_FIFO_AFULL_CNT_SHIT 16
+#define CFG_VPN_FIFO_AFULL_CNT_MASK (0xfff << CFG_VPN_FIFO_AFULL_CNT_SHIT)
+#define CFG_VPN_FIFO_AFULL_BYPASS BIT(6)
+#define CFG_CPN_VSYNC_EDGE_SHIFT 5
+#define CFG_CPN_VSYNC_EDGE_MASK (1 << CFG_CPN_VSYNC_EDGE_SHIFT)
+#define CFG_CPN_TE_EDGE_SHIFT 4
+#define CFG_CPN_TE_EDGE_MASK (1 << CFG_CPN_TE_EDGE_SHIFT)
+#define CFG_CPN_TE_MODE_SHIFT 2
+#define CFG_CPN_TE_MODE_MASK (3 << CFG_CPN_TE_MODE_SHIFT)
+#define CFG_PIXEL_SWAP BIT(1)
+#define CFG_SPLIT_EN BIT(0)
+
+//DSI_LCD_BDG_CTRL1 0x88
+#define CFG_CPN_TE_DLY_CNT_SHIFT 16
+#define CFG_CPN_TE_LINE_CNT_SHIFT 0
+
+//DSI_VPN_CTRL_0 0x100
+#define CFG_VPN_DLY_CNT_SHIFT 16
+#define CFG_VPN_HSYNC_GRANTEE_EN_SHIFT 10
+#define CFG_VPN_HSYNC_GRANTEE_EN (1 << CFG_VPN_HSYNC_GRANTEE_EN_SHIFT)
+#define CFG_VPN_1LN_DLY_SHIFT 8
+#define CFG_VPN_1LN_DLY (1 << CFG_VPN_1LN_DLY_SHIFT)
+#define CFG_VPN_TX_DLY_CNT_SHIFT 0
+
+//DSI_VPN_CTRL_1 0x104
+#define CFG_VPN_VSYNC_RST_EN_SHIFT 31
+#define CFG_VPN_AUTO_WC_DIS_SHIFT 27
+#define CFG_VPN_HACT_WC_EN_SHIFT 26
+#define CFG_VPN_TIMING_CHECK_DIS_SHIFT 25
+#define CFG_VPN_AUTO_DLY_DIS_SHIFT 24
+#define CFG_VPN_HLP_PKT_EN_SHIFT 22
+#define CFG_VPN_HEX_PKT_EN_SHIFT 21
+#define CFG_VPN_HFP_PKT_EN_SHIFT 20
+#define CFG_VPN_HBP_PKT_EN_SHIFT 18
+#define CFG_VPN_HSE_PKT_EN_SHIFT 17
+#define CFG_VPN_HSA_PKT_EN_SHIFT 16
+#define CFG_VPN_HEX_SLOT_EN_SHIFT 14
+#define CFG_VPN_LAST_LINE_TURN_SHIFT 10
+#define CFG_VPN_LPM_FRAME_EN_SHIFT 9
+#define CFG_VPN_BURST_MODE_MASK (0x3 << CFG_VPN_BURST_MODE_SHIFT)
+#define CFG_VPN_RGB_TYPE_SHIFT 0
+#define CFG_VPN_RGB_TYPE_MASK (0x3 << CFG_VPN_RGB_TYPE_SHIFT)
+
+//DSI_PHY_CTRL_1 0x184
+#define CFG_DPHY_ADD_VALID BIT(17)
+#define CFG_DPHY_VDD_VALID BIT(16)
+#define CFG_DPHY_ULPS_DATA BIT(2)
+#define CFG_DPHY_ULPS_CLK BIT(1)
+#define CFG_DPHY_CONT_CLK BIT(0)
+
+//DSI_PHY_CTRL_2 0x188
+#define CFG_DPHY_HSTX_RX BIT(14)
+#define CFG_DPHY_LANE_MAP_SHIFT 12
+#define CFG_DPHY_LANE_EN_SHIFT 4
+#define CFG_DPHY_LANE_EN BIT(4)
+#define CFG_DPHY_FORCE_BTA BIT(0)
+
+#define CFG_DPHY_LANE_MAP_MASK (0x3 << CFG_DPHY_LANE_MAP_SHIFT)
+#define CFG_DPHY_LANE_EN_MASK (0xF << CFG_DPHY_LANE_EN_SHIFT)
+
+//DSI_PHY_TIME_0 0x1C0
+#define CFG_DPHY_TIME_HS_EXIT_SHIFT 24
+#define CFG_DPHY_TIME_HS_TRAIL_SHIFT 16
+#define CFG_DPHY_TIME_HS_ZERO_SHIFT 8
+#define CFG_DPHY_TIME_HS_PREP_SHIFT 0
+
+#define CFG_DPHY_TIME_HS_EXIT_MASK (0xFF << CFG_DPHY_TIME_HS_EXIT_SHIFT)
+#define CFG_DPHY_TIME_HS_TRAIL_MASK (0xFF << CFG_DPHY_TIME_HS_TRAIL_SHIFT)
+#define CFG_DPHY_TIME_HS_ZERO_MASK (0xFF << CFG_DPHY_TIME_HS_ZERO_SHIFT)
+#define CFG_DPHY_TIME_HS_PREP_MASK (0xFF << CFG_DPHY_TIME_HS_PREP_SHIFT)
+
+//DSI_PHY_TIME_1 0x1C4
+#define CFG_DPHY_TIME_TA_GET_SHIFT 24
+#define CFG_DPHY_TIME_TA_GO_SHIFT 16
+#define CFG_DPHY_TIME_WAKEUP_SHIFT 0
+
+#define CFG_DPHY_TIME_TA_GET_MASK (0xFF << CFG_DPHY_TIME_TA_GET_SHIFT)
+#define CFG_DPHY_TIME_TA_GO_MASK (0xFF << CFG_DPHY_TIME_TA_GO_SHIFT)
+#define CFG_DPHY_TIME_WAKEUP_MASK (0xFFFF << CFG_DPHY_TIME_WAKEUP_SHIFT)
+
+//DSI_PHY_TIME_2 0x1C8
+#define CFG_DPHY_TIME_CLK_EXIT_SHIFT 24
+#define CFG_DPHY_TIME_CLK_TRAIL_SHIFT 16
+#define CFG_DPHY_TIME_CLK_ZERO_SHIFT 8
+#define CFG_DPHY_TIME_CLK_LPX_SHIFT 0
+
+#define CFG_DPHY_TIME_CLK_EXIT_MASK (0xFF << CFG_DPHY_TIME_CLK_EXIT_SHIFT)
+#define CFG_DPHY_TIME_CLK_TRAIL_MASK (0xFF << CFG_DPHY_TIME_CLK_TRAIL_SHIFT)
+#define CFG_DPHY_TIME_CLK_ZERO_MASK (0xFF << CFG_DPHY_TIME_CLK_ZERO_SHIFT)
+#define CFG_DPHY_TIME_CLK_LPX_MASK (0xFF << CFG_DPHY_TIME_CLK_LPX_SHIFT)
+
+//DSI_PHY_TIME_3 0x1CC
+#define CFG_DPHY_TIME_LPX_SHIFT 8
+#define CFG_DPHY_TIME_REQRDY_SHIFT 0
+
+#define CFG_DPHY_TIME_LPX_MASK (0xFF << CFG_DPHY_TIME_LPX_SHIFT)
+#define CFG_DPHY_TIME_REQRDY_MASK (0xFF << CFG_DPHY_TIME_REQRDY_SHIFT)
+
+//DSI_PHY_ANA_PWR_CTRL 0x1E0
+#define CFG_DPHY_ANA_RESET BIT(8)
+#define CFG_DPHY_ANA_PU BIT(0)
+
+//DSI_PHY_ANA_CTRL1 0x1E8
+#ifdef CONFIG_SPACEMIT_FPGA
+#define CFG_CLK_SEL BIT(23)
+#else
+#define CFG_CLK_SEL BIT(21)
+#endif
+#define CFG_CLK_DIV2 BIT(11)
+
+
+/*dphy timming*/
+#define HS_PREP_CONSTANT_DEFAULT 40
+#define HS_PREP_UI_DEFAULT 4
+#define HS_ZERO_CONSTANT_DEFAULT 145
+#define HS_ZERO_UI_DEFAULT 10
+#define HS_TRAIL_CONSTANT_DEFAULT 60
+#define HS_TRAIL_UI_DEFAULT 4
+#define HS_EXIT_CONSTANT_DEFAULT 100
+#define HS_EXIT_UI_DEFAULT 0
+#define CK_ZERO_CONSTANT_DEFAULT 300
+#define CK_ZERO_UI_DEFAULT 0
+#define CK_TRAIL_CONSTANT_DEFAULT 60
+#define CK_TRAIL_UI_DEFAULT 0
+#define REQ_READY_DEFAULT 0x3C
+#define WAKEUP_CONSTANT_DEFAULT 1000000
+#define WAKEUP_UI_DEFAULT 0
+#define LPX_CONSTANT_DEFAULT 60
+#define LPX_UI_DEFAULT 0
+
+
+static inline uint32_t dsi_read(void __iomem *addr, uint32_t offset)
+{
+	DRM_DEBUG("DSI_READ [0x%x] = 0x%x\n", offset, __raw_readl(addr + offset));
+	return __raw_readl(addr + offset);
+}
+
+static inline void dsi_write(void __iomem *addr, uint32_t offset, uint32_t data)
+{
+	DRM_DEBUG("DSI_WRITE [0x%x] = 0x%x\n", 0xd421a800 + offset, data);
+	__raw_writel(data, (addr + offset));
+}
+
+static inline void dsi_set_bits(void __iomem *addr, uint32_t offset, uint32_t bits)
+{
+	dsi_write(addr, offset, (dsi_read(addr, offset) | bits));
+}
+
+static inline void dsi_clear_bits(void __iomem *addr, uint32_t offset, uint32_t bits)
+{
+	dsi_write(addr, offset, (dsi_read(addr, offset) & ~bits));
+}
+
+static inline void dsi_write_bits(void __iomem *addr, uint32_t offset, uint32_t mask, uint32_t value)
+{
+	uint32_t tmp = 0;
+
+	tmp = dsi_read(addr, offset);
+	tmp &= ~mask;
+	tmp |= value;
+	dsi_write(addr, offset, tmp);
+}
+
+#endif /*_SPACEMIT_DSI_HW_H_*/
diff --git a/drivers/gpu/drm/spacemit/raspberrypi_touchscreen.c b/drivers/gpu/drm/spacemit/raspberrypi_touchscreen.c
new file mode 100644
index 000000000000..ee81febd97a6
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/raspberrypi_touchscreen.c
@@ -0,0 +1,585 @@
+/*
+ * Copyright © 2016-2017 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Portions of this file (derived from panel-simple.c) are:
+ *
+ * Copyright (C) 2013, NVIDIA Corporation.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sub license,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * Raspberry Pi 7" touchscreen panel driver.
+ *
+ * The 7" touchscreen consists of a DPI LCD panel, a Toshiba
+ * TC358762XBG DSI-DPI bridge, and an I2C-connected Atmel ATTINY88-MUR
+ * controlling power management, the LCD PWM, and initial register
+ * setup of the Tohsiba.
+ *
+ * This driver controls the TC358762 and ATTINY88, presenting a DSI
+ * device with a drm_panel.
+ */
+
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/fb.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <linux/pm.h>
+#include <linux/gpio/consumer.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_device.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+#include <linux/media-bus-format.h>
+#include <drm/drm_print.h>
+
+#define RPI_DSI_DRIVER_NAME "rpi-ts-dsi"
+
+/* I2C registers of the Atmel microcontroller. */
+enum REG_ADDR {
+	REG_ID = 0x80,
+	REG_PORTA, /* BIT(2) for horizontal flip, BIT(3) for vertical flip */
+	REG_PORTB,
+	REG_PORTC,
+	REG_PORTD,
+	REG_POWERON,
+	REG_PWM,
+	REG_DDRA,
+	REG_DDRB,
+	REG_DDRC,
+	REG_DDRD,
+	REG_TEST,
+	REG_WR_ADDRL,
+	REG_WR_ADDRH,
+	REG_READH,
+	REG_READL,
+	REG_WRITEH,
+	REG_WRITEL,
+	REG_ID2,
+};
+
+/* DSI D-PHY Layer Registers */
+#define D0W_DPHYCONTTX		0x0004
+#define CLW_DPHYCONTRX		0x0020
+#define D0W_DPHYCONTRX		0x0024
+#define D1W_DPHYCONTRX		0x0028
+#define COM_DPHYCONTRX		0x0038
+#define CLW_CNTRL		0x0040
+#define D0W_CNTRL		0x0044
+#define D1W_CNTRL		0x0048
+#define DFTMODE_CNTRL		0x0054
+
+/* DSI PPI Layer Registers */
+#define PPI_STARTPPI		0x0104
+#define PPI_BUSYPPI		0x0108
+#define PPI_LINEINITCNT		0x0110
+#define PPI_LPTXTIMECNT		0x0114
+#define PPI_CLS_ATMR		0x0140
+#define PPI_D0S_ATMR		0x0144
+#define PPI_D1S_ATMR		0x0148
+#define PPI_D0S_CLRSIPOCOUNT	0x0164
+#define PPI_D1S_CLRSIPOCOUNT	0x0168
+#define CLS_PRE			0x0180
+#define D0S_PRE			0x0184
+#define D1S_PRE			0x0188
+#define CLS_PREP		0x01A0
+#define D0S_PREP		0x01A4
+#define D1S_PREP		0x01A8
+#define CLS_ZERO		0x01C0
+#define D0S_ZERO		0x01C4
+#define D1S_ZERO		0x01C8
+#define PPI_CLRFLG		0x01E0
+#define PPI_CLRSIPO		0x01E4
+#define HSTIMEOUT		0x01F0
+#define HSTIMEOUTENABLE		0x01F4
+
+/* DSI Protocol Layer Registers */
+#define DSI_STARTDSI		0x0204
+#define DSI_BUSYDSI		0x0208
+#define DSI_LANEENABLE		0x0210
+# define DSI_LANEENABLE_CLOCK		BIT(0)
+# define DSI_LANEENABLE_D0		BIT(1)
+# define DSI_LANEENABLE_D1		BIT(2)
+
+#define DSI_LANESTATUS0		0x0214
+#define DSI_LANESTATUS1		0x0218
+#define DSI_INTSTATUS		0x0220
+#define DSI_INTMASK		0x0224
+#define DSI_INTCLR		0x0228
+#define DSI_LPTXTO		0x0230
+#define DSI_MODE		0x0260
+#define DSI_PAYLOAD0		0x0268
+#define DSI_PAYLOAD1		0x026C
+#define DSI_SHORTPKTDAT		0x0270
+#define DSI_SHORTPKTREQ		0x0274
+#define DSI_BTASTA		0x0278
+#define DSI_BTACLR		0x027C
+
+/* DSI General Registers */
+#define DSIERRCNT		0x0300
+#define DSISIGMOD		0x0304
+
+/* DSI Application Layer Registers */
+#define APLCTRL			0x0400
+#define APLSTAT			0x0404
+#define APLERR			0x0408
+#define PWRMOD			0x040C
+#define RDPKTLN			0x0410
+#define PXLFMT			0x0414
+#define MEMWRCMD		0x0418
+
+/* LCDC/DPI Host Registers */
+#define LCDCTRL			0x0420
+#define HSR			0x0424
+#define HDISPR			0x0428
+#define VSR			0x042C
+#define VDISPR			0x0430
+#define VFUEN			0x0434
+
+/* DBI-B Host Registers */
+#define DBIBCTRL		0x0440
+
+/* SPI Master Registers */
+#define SPICMR			0x0450
+#define SPITCR			0x0454
+
+/* System Controller Registers */
+#define SYSSTAT			0x0460
+#define SYSCTRL			0x0464
+#define SYSPLL1			0x0468
+#define SYSPLL2			0x046C
+#define SYSPLL3			0x0470
+#define SYSPMCTRL		0x047C
+
+/* GPIO Registers */
+#define GPIOC			0x0480
+#define GPIOO			0x0484
+#define GPIOI			0x0488
+
+/* I2C Registers */
+#define I2CCLKCTRL		0x0490
+
+/* Chip/Rev Registers */
+#define IDREG			0x04A0
+
+/* Debug Registers */
+#define WCMDQUEUE		0x0500
+#define RCMDQUEUE		0x0504
+
+struct rpi_touchscreen {
+	struct drm_panel base;
+	struct mipi_dsi_device *dsi;
+	struct i2c_client *i2c;
+
+	struct gpio_desc *enable_gpio;
+};
+
+static const struct drm_display_mode rpi_touchscreen_modes[] = {
+	{
+		/* Modeline comes from the Raspberry Pi firmware, with HFP=1
+		 * plugged in and clock re-computed from that.
+		 */
+		.clock = 25000000 / 1000,
+		.hdisplay = 800,
+		.hsync_start = 800 + 1,
+		.hsync_end = 800 + 1 + 2,
+		.htotal = 800 + 1 + 2 + 46,
+		.vdisplay = 480,
+		.vsync_start = 480 + 7,
+		.vsync_end = 480 + 7 + 2,
+		.vtotal = 480 + 7 + 2 + 21,
+	},
+};
+
+static struct rpi_touchscreen *panel_to_ts(struct drm_panel *panel)
+{
+	return container_of(panel, struct rpi_touchscreen, base);
+}
+
+static int rpi_touchscreen_i2c_read(struct rpi_touchscreen *ts, u8 reg)
+{
+	return i2c_smbus_read_byte_data(ts->i2c, reg);
+}
+
+static void rpi_touchscreen_i2c_write(struct rpi_touchscreen *ts,
+				      u8 reg, u8 val)
+{
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(ts->i2c, reg, val);
+	if (ret)
+		dev_err(&ts->i2c->dev, "I2C write failed: %d\n", ret);
+}
+
+static int rpi_touchscreen_write(struct rpi_touchscreen *ts, u16 reg, u32 val)
+{
+	u8 msg[] = {
+		reg,
+		reg >> 8,
+		val,
+		val >> 8,
+		val >> 16,
+		val >> 24,
+	};
+
+	mipi_dsi_generic_write(ts->dsi, msg, sizeof(msg));
+
+	return 0;
+}
+
+static int rpi_touchscreen_disable(struct drm_panel *panel)
+{
+	DRM_INFO("%s()\n", __func__);
+	struct rpi_touchscreen *ts = panel_to_ts(panel);
+
+	rpi_touchscreen_i2c_write(ts, REG_PWM, 0);
+
+	rpi_touchscreen_i2c_write(ts, REG_POWERON, 0);
+	udelay(1);
+
+	return 0;
+}
+
+static int rpi_touchscreen_noop(struct drm_panel *panel)
+{
+	return 0;
+}
+
+static int rpi_touchscreen_prepare(struct drm_panel *panel)
+{
+	struct rpi_touchscreen *ts = panel_to_ts(panel);
+	int i;
+
+	rpi_touchscreen_i2c_write(ts, REG_POWERON, 1);
+	/* Wait for nPWRDWN to go low to indicate poweron is done. */
+	for (i = 0; i < 100; i++) {
+		if (rpi_touchscreen_i2c_read(ts, REG_PORTB) & 1)
+			break;
+	}
+
+	rpi_touchscreen_write(ts, DSI_LANEENABLE, 0x03);
+	rpi_touchscreen_write(ts, PPI_D0S_CLRSIPOCOUNT, 0x05);
+	rpi_touchscreen_write(ts, PPI_D1S_CLRSIPOCOUNT, 0x05);
+	rpi_touchscreen_write(ts, PPI_D0S_ATMR, 0x00);
+	rpi_touchscreen_write(ts, PPI_D1S_ATMR, 0x00);
+	rpi_touchscreen_write(ts, PPI_LPTXTIMECNT, 0x03);
+
+	rpi_touchscreen_write(ts, SPICMR, 0x00);
+	rpi_touchscreen_write(ts, LCDCTRL, 0x00100150);
+	rpi_touchscreen_write(ts, SYSCTRL, 0x040f);
+	msleep(100);
+
+	rpi_touchscreen_write(ts, PPI_STARTPPI, 0x01);
+	rpi_touchscreen_write(ts, DSI_STARTDSI, 0x01);
+	msleep(100);
+
+	return 0;
+}
+
+static int rpi_touchscreen_enable(struct drm_panel *panel)
+{
+	DRM_INFO("%s()\n", __func__);
+	struct rpi_touchscreen *ts = panel_to_ts(panel);
+	/* Turn on the backlight. */
+	rpi_touchscreen_i2c_write(ts, REG_PWM, 255);
+
+	/* Default to the same orientation as the closed source
+	 * firmware used for the panel.  Runtime rotation
+	 * configuration will be supported using VC4's plane
+	 * orientation bits.
+	 */
+	rpi_touchscreen_i2c_write(ts, REG_PORTA, BIT(2));
+
+	return 0;
+}
+
+static int rpi_touchscreen_get_modes(struct drm_panel *panel,
+				     struct drm_connector *connector)
+{
+	unsigned int i, num = 0;
+	static const u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+	DRM_DEBUG("%s()\n", __func__);
+
+	for (i = 0; i < ARRAY_SIZE(rpi_touchscreen_modes); i++) {
+		const struct drm_display_mode *m = &rpi_touchscreen_modes[i];
+		struct drm_display_mode *mode;
+
+		mode = drm_mode_duplicate(connector->dev, m);
+		if (!mode) {
+			dev_err(panel->dev, "failed to add mode %ux%u@%u\n",
+				m->hdisplay, m->vdisplay,
+				drm_mode_vrefresh(m));
+			continue;
+		}
+
+		mode->type |= DRM_MODE_TYPE_DRIVER;
+
+		if (i == 0)
+			mode->type |= DRM_MODE_TYPE_PREFERRED;
+
+		drm_mode_set_name(mode);
+
+		drm_mode_probed_add(connector, mode);
+		num++;
+	}
+
+	connector->display_info.bpc = 8;
+	connector->display_info.width_mm = 95;
+	connector->display_info.height_mm = 53;
+	drm_display_info_set_bus_formats(&connector->display_info,
+					 &bus_format, 1);
+
+	return num;
+}
+
+static const struct drm_panel_funcs rpi_touchscreen_funcs = {
+	.disable = rpi_touchscreen_disable,
+	.unprepare = rpi_touchscreen_noop,
+	.prepare = rpi_touchscreen_prepare,
+	.enable = rpi_touchscreen_enable,
+	.get_modes = rpi_touchscreen_get_modes,
+};
+
+static int rpi_touchscreen_probe(struct i2c_client *i2c)
+{
+	struct device *dev = &i2c->dev;
+	struct rpi_touchscreen *ts;
+	struct device_node *endpoint = NULL;
+	struct device_node *remote_ep = NULL;
+	struct device_node *dsi_host_node = NULL;
+	struct device_node *node = NULL;
+	struct mipi_dsi_host *host;
+	int ver, rc, ret;
+
+	struct device_node *lcd_node;
+	const char *lcd_name = NULL;
+	char lcd_path[64];
+
+	struct mipi_dsi_device_info info = {
+		.type = RPI_DSI_DRIVER_NAME,
+		.channel = 0,
+		.node = NULL,
+	};
+
+	DRM_INFO("%s\n", __func__);
+
+	if (!dev->of_node) {
+		DRM_ERROR("%s: no of_node\n", __func__);
+		return -ENODEV;
+	}
+
+	ts = devm_kzalloc(dev, sizeof(*ts), GFP_KERNEL);
+	if (!ts) {
+		DRM_ERROR("%s: kzalloc failed\n", __func__);
+		return -ENOMEM;
+	}
+
+	i2c_set_clientdata(i2c, ts);
+	ts->i2c = i2c;
+
+	/* Enable GPIO */
+	ts->enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(ts->enable_gpio)) {
+		DRM_ERROR("%s: failed to get enable gpio\n", __func__);
+		return PTR_ERR(ts->enable_gpio);
+	}
+
+	mdelay(200);
+
+	/* Read controller ID */
+	ver = rpi_touchscreen_i2c_read(ts, REG_ID);
+	if (ver < 0) {
+		DRM_ERROR("%s: I2C read failed: %d\n", __func__, ver);
+		return -ENODEV;
+	}
+
+	DRM_INFO("%s: touchscreen ver = 0x%x\n", __func__, ver);
+
+	if (ver != 0xde && ver != 0xc3) {
+		DRM_ERROR("%s: unknown firmware 0x%x\n", __func__, ver);
+		return -ENODEV;
+	}
+
+	rpi_touchscreen_i2c_write(ts, REG_POWERON, 0);
+
+	endpoint = of_graph_get_next_endpoint(dev->of_node, NULL);
+	if (!endpoint) {
+		DRM_ERROR("%s: no endpoint found under %pOF\n",
+			  __func__, dev->of_node);
+		return -ENODEV;
+	}
+
+	remote_ep = of_graph_get_remote_endpoint(endpoint);
+	if (!remote_ep) {
+		DRM_ERROR("%s: no remote-endpoint\n", __func__);
+		goto err;
+	}
+
+	dsi_host_node = of_graph_get_port_parent(remote_ep);
+	if (!dsi_host_node) {
+		DRM_ERROR("%s: failed to get DSI host node\n", __func__);
+		goto err;
+	}
+
+	node = of_find_node_by_name(NULL, "panel0");
+	if (node) {
+		rc = of_property_read_string(node, "force-attached", &lcd_name);
+		if (rc) {
+			DRM_ERROR("%s: no force-attached property in %pOF\n",
+				__func__, node);
+			goto err;
+		}
+	} else {
+		DRM_ERROR("can not find panel0 node\n");
+	}
+
+	snprintf(lcd_path, sizeof(lcd_path), "/lcds/%s", lcd_name);
+
+	lcd_node = of_find_node_by_path(lcd_path);
+	if (!lcd_node) {
+		DRM_ERROR("%s: lcd node %s not found\n",
+			  __func__, lcd_path);
+		goto err;
+	}
+
+	DRM_INFO("%s: found lcd node %pOF\n", __func__, lcd_node);
+
+	host = of_find_mipi_dsi_host_by_node(dsi_host_node);
+	if (!host) {
+		DRM_INFO("%s: DSI host not ready, defer probe\n", __func__);
+		of_node_put(lcd_node);
+		return -EPROBE_DEFER;
+	}
+
+	drm_panel_init(&ts->base, dev,
+		       &rpi_touchscreen_funcs,
+		       DRM_MODE_CONNECTOR_DSI);
+
+	drm_panel_add(&ts->base);
+
+	ts->base.dev->of_node = lcd_node;
+
+	info.node = of_node_get(remote_ep);
+	if (!info.node) {
+		DRM_ERROR("%s: failed to get info.node\n", __func__);
+		goto err;
+	}
+
+	ts->dsi = mipi_dsi_device_register_full(host, &info);
+	if (IS_ERR(ts->dsi)) {
+		DRM_ERROR("%s: mipi_dsi_device_register_full failed: %ld\n",
+			  __func__, PTR_ERR(ts->dsi));
+		return PTR_ERR(ts->dsi);
+	}
+
+	ret = mipi_dsi_attach(ts->dsi);
+	if (ret)
+		dev_err(&ts->dsi->dev, "failed to attach dsi to host: %d\n", ret);
+
+	return ret;
+
+err:
+	of_node_put(node);
+	of_node_put(endpoint);
+	of_node_put(remote_ep);
+	of_node_put(dsi_host_node);
+	return -ENODEV;
+}
+
+
+static void rpi_touchscreen_remove(struct i2c_client *i2c)
+{
+	struct rpi_touchscreen *ts = i2c_get_clientdata(i2c);
+
+	mipi_dsi_detach(ts->dsi);
+
+	drm_panel_remove(&ts->base);
+
+	mipi_dsi_device_unregister(ts->dsi);
+
+	return;
+}
+
+static int rpi_touchscreen_dsi_probe(struct mipi_dsi_device *dsi)
+{
+	dsi->mode_flags = (MIPI_DSI_MODE_VIDEO |
+			   MIPI_DSI_MODE_VIDEO_BURST |
+			   MIPI_DSI_MODE_LPM);
+	DRM_INFO("%s\n", __func__);
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->lanes = 1;
+
+	return 0;
+}
+
+static const struct of_device_id rpi_dpi_of_match[] = {
+	{ .compatible = "raspberrypi,dpi-panel" },
+	{ /* sentinel */ }
+};
+
+static struct mipi_dsi_driver rpi_touchscreen_dsi_driver = {
+	.probe = rpi_touchscreen_dsi_probe,
+	.driver = {
+		.name = RPI_DSI_DRIVER_NAME,
+		.of_match_table = rpi_dpi_of_match,
+	},
+};
+
+static const struct of_device_id rpi_touchscreen_of_ids[] = {
+	{ .compatible = "raspberrypi,7inch-touchscreen-panel" },
+	{ } /* sentinel */
+};
+MODULE_DEVICE_TABLE(of, rpi_touchscreen_of_ids);
+
+static struct i2c_driver rpi_touchscreen_driver = {
+	.driver = {
+		.name = "rpi_touchscreen",
+		.of_match_table = rpi_touchscreen_of_ids,
+	},
+	.probe = rpi_touchscreen_probe,
+	.remove = rpi_touchscreen_remove,
+};
+
+static int __init rpi_touchscreen_init(void)
+{
+	mipi_dsi_driver_register(&rpi_touchscreen_dsi_driver);
+	return i2c_add_driver(&rpi_touchscreen_driver);
+}
+module_init(rpi_touchscreen_init);
+
+static void __exit rpi_touchscreen_exit(void)
+{
+	i2c_del_driver(&rpi_touchscreen_driver);
+	mipi_dsi_driver_unregister(&rpi_touchscreen_dsi_driver);
+}
+module_exit(rpi_touchscreen_exit);
+
+MODULE_AUTHOR("Eric Anholt <eric@anholt.net>");
+MODULE_DESCRIPTION("Raspberry Pi 7-inch touchscreen driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/selftest/Kconfig b/drivers/gpu/drm/spacemit/selftest/Kconfig
new file mode 100644
index 000000000000..a6b89e36800d
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/selftest/Kconfig
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+config DRM_SPACEMIT_SELFTEST
+	tristate "SPACEMIT self test"
+	depends on DRM_SPACEMIT
+	default n
+	help
+		Choose this option if you want to test fbc mem size cal API.
diff --git a/drivers/gpu/drm/spacemit/selftest/Makefile b/drivers/gpu/drm/spacemit/selftest/Makefile
new file mode 100644
index 000000000000..e186903859cc
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/selftest/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+spacemit_selftest-y := test-spacemit_fbcmem.o
+
+obj-$(CONFIG_DRM_SPACEMIT_SELFTEST) += spacemit_selftest.o
diff --git a/drivers/gpu/drm/spacemit/selftest/test-spacemit_fbcmem.c b/drivers/gpu/drm/spacemit/selftest/test-spacemit_fbcmem.c
new file mode 100644
index 000000000000..aa5f93d10ba6
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/selftest/test-spacemit_fbcmem.c
@@ -0,0 +1,463 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/moduleparam.h>
+#include "../dpu/dpu_saturn.h"
+#include "../dpu/saturn_fbcmem.h"
+#include "../../selftests/test-drm_modeset_common.h"
+
+static uint dpu_id = 1;
+module_param(dpu_id, uint, 0644);
+
+struct raw_layer_fbcmem_size_test {
+	u32 drm_4cc_fmt;
+	bool rot_90_or_270;
+	u32 plane_crop_width;
+	u32 output_mem_size;
+	u32 expected_mem_size;
+};
+
+struct afbc_layer_fbcmem_size_test {
+	u8 rdma_work_mode;
+	u32 drm_4cc_fmt;
+	u32 crop_start_x;
+	u32 crop_start_y;
+	u32 crop_width;
+	u32 crop_height;
+	u32 fbc_block_size;
+	bool rot_90_or_270;
+	u8 min_lines;
+	u32 output_mem_size;
+	u32 expected_mem_size;
+};
+
+struct spacemit_hw_device hwdev_res[] = {
+	{//SATURN
+		.fbcmem_sizes = saturn_fbcmem_sizes,
+	},
+	{//SATURN_LE
+		.fbcmem_sizes = saturn_le_fbcmem_sizes,
+	}
+};
+
+struct single_rdma_test {
+	struct spacemit_crtc_rdma dpu_rdma;
+	u32 expect_start;
+	u32 expect_size;
+	bool expect_map;
+};
+
+struct dpu_rdma_test {
+	int expect_result;
+	struct single_rdma_test *dpu_rdmas;
+};
+
+struct concrete_rdma_test {
+	size_t test_counter;
+	struct dpu_rdma_test *rdmas_test;
+};
+
+struct single_rdma_test le_rdma_test[][4] = { //4 rdmas
+	{//test 0: odd rdma not use sec fbcmem, use all the primary fbc mem left
+		{ //rdma0
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 100, }
+			},
+			.expect_start = 0,
+			.expect_size = 100,
+			.expect_map = true,
+		},
+		{ //rdma1
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 40, }
+			},
+			.expect_start = 100,
+			.expect_size = 1292,
+			.expect_map = false,
+		},
+		{ //rdma2
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 200, }
+			},
+			.expect_start = 0,
+			.expect_size = 200,
+			.expect_map = true,
+		},
+		{ //rdma3
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 60, }
+			},
+			.expect_start = 200,
+			.expect_size = 916,
+			.expect_map = false,
+		},
+	},
+	{//test 1: rdma1 use primary and second fbc mem
+		{ //rdma0
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 1024, }
+			},
+			.expect_start = 0,
+			.expect_size = 1024,
+			.expect_map = true,
+		},
+		{ //rdma1
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 1024, }
+			},
+			.expect_start = 1024,
+			.expect_size = 1024,
+			.expect_map = true,
+		},
+		{ //rdma2
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 0, }
+			},
+			.expect_start = 0,
+			.expect_size = 0,
+			.expect_map = false,
+		},
+		{ //rdma3
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 100, }
+			},
+			.expect_start = 656,
+			.expect_size = 460, //use all the left
+			.expect_map = false,
+		},
+	},
+	{//test 2: rdma3 use fbcmem0
+		{ //rdma0
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 0, }
+			},
+			.expect_start = 0,
+			.expect_size = 0,
+			.expect_map = false,
+		},
+		{ //rdma1
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 0, }
+			},
+			.expect_start = 0,
+			.expect_size = 0,
+			.expect_map = false,
+		},
+		{ //rdma2
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 600, }
+			},
+			.expect_start = 0,
+			.expect_size = 600,
+			.expect_map = true,
+		},
+		{ //rdma3
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 600, }
+			},
+			.expect_start = 600,
+			.expect_size = 1908, //use all the left
+			.expect_map = true,
+		},
+	},
+	{//test 3: error case, rdma3 use fbcmem0 which has been used
+		{ //rdma0
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 100, }
+			},
+			.expect_start = 0,
+			.expect_size = 100,
+			.expect_map = 0,
+		},
+		{ //rdma1
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 0, }
+			},
+			.expect_start = 0,
+			.expect_size = 0,
+			.expect_map = false,
+		},
+		{ //rdma2
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 600, }
+			},
+			.expect_start = 0,
+			.expect_size = 600,
+			.expect_map = true,
+		},
+		{ //rdma3
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 600, }
+			},
+			.expect_start = 600,
+			.expect_size = 1908, //use fbcmem0 but rdma0 use already
+			.expect_map = true,
+		},
+	},
+	{//test 4: error case: rdma1 and rdma2 both use fbcmem1
+		{ //rdma0
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 600, }
+			},
+			.expect_start = 0,
+			.expect_size = 600,
+			.expect_map = true,
+		},
+		{ //rdma1
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 800, }
+			},
+			.expect_start = 600,
+			.expect_size = 800,
+			.expect_map = true,
+		},
+		{ //rdma2
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 600, }
+			},
+			.expect_start = 0,
+			.expect_size = 600,
+			.expect_map = true,
+		},
+		{ //rdma3
+			.dpu_rdma = {
+				.mode = 0,
+				.fbcmem = { .size = 0, }
+			},
+			.expect_start = 0,
+			.expect_size = 0, //use fbcmem0 but rdma0 use already
+			.expect_map = false,
+		},
+	},
+};
+
+struct dpu_rdma_test saturn_le_rdma_tests[] = { //4 rdmas
+	{//test 0: odd rdma not use sec fbcmem, use all the primary fbc mem left
+		.expect_result = 0,
+		.dpu_rdmas = le_rdma_test[0],
+	},
+	{//test 1: rdma1 use primary and second fbc mem
+		.expect_result = 0,
+		.dpu_rdmas = le_rdma_test[1],
+	},
+	{//test 2: rdma3 use fbcmem0
+		.expect_result = 0,
+		.dpu_rdmas = le_rdma_test[2],
+	},
+	{//test 3: error case
+		.expect_result = -1,
+		.dpu_rdmas = le_rdma_test[3],
+	},
+	{//test 3: error case
+		.expect_result = -1,
+		.dpu_rdmas = le_rdma_test[4],
+	},
+};
+
+struct concrete_rdma_test concrete_rdma_tests[] = {
+	{ //SATURN
+		.rdmas_test = NULL,
+		.test_counter = 0,
+	},
+	{ //SATURN_LE
+		.rdmas_test = (struct dpu_rdma_test *)saturn_le_rdma_tests,
+		.test_counter = ARRAY_SIZE(saturn_le_rdma_tests),
+	},
+};
+
+int execute_rdma_fbcmem_cal(struct spacemit_hw_device *hw_dev, struct single_rdma_test *dpu_rdmas, size_t rdma_num)
+{
+	int ret = 0;
+	size_t index = 0;
+
+	struct spacemit_crtc_rdma *dpu_rdmas_ptr = kzalloc(
+		sizeof(struct spacemit_crtc_rdma) * rdma_num, GFP_KERNEL);
+
+	if (dpu_rdmas_ptr == NULL) {
+		DRM_ERROR("in %s, error for not enough memory left\n", __func__);
+		return -ENOMEM;
+	}
+
+	memset(dpu_rdmas_ptr, 0, sizeof(struct spacemit_crtc_rdma) * rdma_num);
+	for (index = 0; index < rdma_num; index++) { //copy param
+		memcpy(&(dpu_rdmas_ptr[index]), &(dpu_rdmas[index].dpu_rdma), sizeof(struct spacemit_crtc_rdma));
+	}
+	ret = saturn_adjust_rdma_fbcmem(hw_dev, dpu_rdmas_ptr);
+	for (index = 0; index < rdma_num; index++) { //copy result back
+		memcpy(&(dpu_rdmas[index].dpu_rdma), &(dpu_rdmas_ptr[index]), sizeof(struct spacemit_crtc_rdma));
+	}
+
+	kfree(dpu_rdmas_ptr);
+
+	return ret;
+}
+
+static struct raw_layer_fbcmem_size_test raw_layer_tests[] = {
+	{
+		.drm_4cc_fmt = DRM_FORMAT_RGBA8888,
+		.rot_90_or_270 = true,
+		.plane_crop_width = 1920,
+		.expected_mem_size = 1024, //checked
+	},
+	{
+		.drm_4cc_fmt = DRM_FORMAT_NV12,
+		.rot_90_or_270 = true,
+		.plane_crop_width = 1920,
+		.expected_mem_size = 1024, //checked
+	},
+	{
+		.drm_4cc_fmt = DRM_FORMAT_ABGR8888,
+		.rot_90_or_270 = false,
+		.plane_crop_width = 720,
+		.expected_mem_size = 90, //checked
+	}
+};
+
+static struct afbc_layer_fbcmem_size_test afbc_layer_tests[] = {
+	{
+		.rdma_work_mode = 0,
+		.drm_4cc_fmt = DRM_FORMAT_NV12,
+		.crop_start_x = 0,
+		.crop_start_y = 0,
+		.crop_width = 1920,
+		.crop_height = 1080,
+		.fbc_block_size = 0,
+		.rot_90_or_270 = false,
+		.min_lines = 4,
+		.expected_mem_size = 480, //checked
+	},
+	{
+		.rdma_work_mode = 0,
+		.drm_4cc_fmt = DRM_FORMAT_NV12,
+		.crop_start_x = 0,
+		.crop_start_y = 0,
+		.crop_width = 1920,
+		.crop_height = 1080,
+		.fbc_block_size = 0,
+		.rot_90_or_270 = true,
+		.min_lines = 4,
+		.expected_mem_size = 288, //checked
+	},
+};
+
+static int execute_raw_fbcmem_size_cal(struct raw_layer_fbcmem_size_test *test)
+{
+	int ret = get_raw_data_plane_rdma_mem_size(test->drm_4cc_fmt,
+		test->rot_90_or_270, test->plane_crop_width, &(test->output_mem_size));
+
+	return ret;
+}
+
+static int execute_afbc_fbcmem_size_cal(struct afbc_layer_fbcmem_size_test *test)
+{
+	int ret = get_afbc_data_plane_min_rdma_mem_size(test->rdma_work_mode,
+			test->drm_4cc_fmt, test->crop_start_x, test->crop_start_y,
+			test->crop_width, test->crop_height, test->fbc_block_size,
+			test->rot_90_or_270, test->min_lines, &(test->output_mem_size));
+	return ret;
+}
+
+int spacemit_fbcmem_size_test(void)
+{
+	int ret = 0;
+	size_t i = 0;
+	size_t j = 0;
+	struct dpu_rdma_test *temp = NULL;
+	struct spacemit_hw_device *hw_device = NULL;
+	struct concrete_rdma_test *test = NULL;
+
+	for (i = 0; i < ARRAY_SIZE(raw_layer_tests); i++) {
+		ret = execute_raw_fbcmem_size_cal(&raw_layer_tests[i]);
+		FAIL(ret != 0, "fbcmem raw_layer_tests[%d] return %d but expect %d\n",
+			i, ret, 0);
+		FAIL(raw_layer_tests[i].output_mem_size !=
+			raw_layer_tests[i].expected_mem_size,
+			"fbcmem raw_layer_tests[%d] cal size = %d, expect size = %d\n",
+			i, raw_layer_tests[i].expected_mem_size,
+			raw_layer_tests[i].output_mem_size);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(afbc_layer_tests); i++) {
+		ret = execute_afbc_fbcmem_size_cal(&(afbc_layer_tests[i]));
+		FAIL(ret != 0, "fbcmem afbc_layer_tests[%d] return %d but expect %d\n");
+		FAIL(afbc_layer_tests[i].output_mem_size !=
+			afbc_layer_tests[i].expected_mem_size,
+			"fbcmem afbc_layer_tests[%d] cal size = %d, expect size = %d\n",
+			i, afbc_layer_tests[i].output_mem_size,
+			afbc_layer_tests[i].expected_mem_size);
+	}
+
+	FAIL(dpu_id >= SPACEMIT_DP_MAX_DEVICES, "dpu_id(%d) should < %d\n", dpu_id, SPACEMIT_DP_MAX_DEVICES);
+	FAIL(dpu_id != 1, "only dpu_id = 1 is support now\n");
+	hw_device = &(hwdev_res[dpu_id]);
+	hw_device->rdma_nums = spacemit_dp_devices[dpu_id].rdma_nums;
+	test = &concrete_rdma_tests[dpu_id];
+
+	for (i = 0; i < test->test_counter; i++) {
+		DRM_DEBUG("\ntest[%u/%u] start\n", i, test->test_counter);
+		ret = execute_rdma_fbcmem_cal(hw_device, test->rdmas_test[i].dpu_rdmas, hw_device->rdma_nums);
+		//printk("test_counter[%u/%u] test result = %d\n", i, test->test_counter, ret);
+		temp = &(test->rdmas_test[i]);
+		FAIL(ret != temp->expect_result,
+			"rdma fbcmem test[%d] test failed, cal result = %d, expect = %d\n", ret, temp->expect_result);
+		if (ret == 0) { //check details
+			DRM_DEBUG("rmda[%u][%u] cal result = %d, fbcmem: start = %u, size = %u, map = %u\n",
+				i, j, temp->expect_result, temp->dpu_rdmas[j].dpu_rdma.fbcmem.start,
+				temp->dpu_rdmas[j].dpu_rdma.fbcmem.size, temp->dpu_rdmas[j].dpu_rdma.fbcmem.map);
+			for (j = 0; j < hw_device->rdma_nums; j++) {
+				FAIL(temp->dpu_rdmas[j].dpu_rdma.fbcmem.start != temp->dpu_rdmas[j].expect_start,
+					"rdma fbcmem test[%d][%d] cal start = %u, expect start = %u\n",
+					i, j, temp->dpu_rdmas[j].dpu_rdma.fbcmem.start, temp->dpu_rdmas[j].expect_start);
+				FAIL(temp->dpu_rdmas[j].dpu_rdma.fbcmem.size != temp->dpu_rdmas[j].expect_size,
+					"rdma fbcmem test[%d][%d] cal size = %u, expect size = %u\n",
+					i, j, temp->dpu_rdmas[j].dpu_rdma.fbcmem.size, temp->dpu_rdmas[j].expect_size);
+				FAIL(temp->dpu_rdmas[j].dpu_rdma.fbcmem.map != temp->dpu_rdmas[j].expect_map,
+					"rdma fbcmem test[%d][%d] cal map = %u, expect_map = %u\n",
+					i, j, temp->dpu_rdmas[j].dpu_rdma.fbcmem.map, temp->dpu_rdmas[j].expect_map);
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int __init test_spacemit_fbcmem_size_init(void)
+{
+	int err = spacemit_fbcmem_size_test();
+
+	return err >= 0 ? 0 : err;
+}
+
+static void __exit test_spacemit_fbcmem_size_exit(void)
+{
+}
+
+module_init(test_spacemit_fbcmem_size_init);
+module_exit(test_spacemit_fbcmem_size_exit);
+
+MODULE_AUTHOR("SPACEMIT Corporation");
+MODULE_LICENSE("GPL");
+
diff --git a/drivers/gpu/drm/spacemit/spacemit_bootloader.c b/drivers/gpu/drm/spacemit/spacemit_bootloader.c
new file mode 100644
index 000000000000..baa889158374
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_bootloader.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include "spacemit_bootloader.h"
+
+static struct reserved_mem *bootloader_mem;
+static struct reserved_mem bootloader_mem_copy;
+static struct work_struct work_free_bootloader_mem;
+static DEFINE_MUTEX(bootloader_mem_lock);
+static bool bootloader_mem_release_queued;
+static unsigned int bootloader_mem_release_target = 1;
+static unsigned int bootloader_mem_release_count;
+
+static void __free_bootloader_mem(struct work_struct *work)
+{
+	struct reserved_mem rmem;
+	struct page *page;
+	phys_addr_t size;
+
+	mutex_lock(&bootloader_mem_lock);
+	if (!bootloader_mem) {
+		bootloader_mem_release_queued = false;
+		mutex_unlock(&bootloader_mem_lock);
+		return;
+	}
+
+	rmem = *bootloader_mem;
+	bootloader_mem = NULL;
+	bootloader_mem_release_queued = false;
+	mutex_unlock(&bootloader_mem_lock);
+
+	/* Give back reserved framebuffer pages to buddy system */
+	for (size = 0; size < rmem.size; size += PAGE_SIZE) {
+		page = phys_to_page(rmem.base + size);
+		free_reserved_page(page);
+	}
+
+	pr_debug("released reserved framebuffer memory at %pa, size %ld MB\n",
+		&rmem.base, (unsigned long)rmem.size / SZ_1M);
+}
+
+void spacemit_dpu_free_bootloader_mem(void)
+{
+	unsigned int target, count;
+
+	/*
+	 * Freeing pages to buddy system may take several milliseconds.
+	 * Use workqueue here for drm performance consideration.
+	 */
+	mutex_lock(&bootloader_mem_lock);
+	if (!bootloader_mem || bootloader_mem_release_queued) {
+		mutex_unlock(&bootloader_mem_lock);
+		return;
+	}
+
+	bootloader_mem_release_count++;
+	target = bootloader_mem_release_target;
+	count = bootloader_mem_release_count;
+	if (count < target) {
+		mutex_unlock(&bootloader_mem_lock);
+		pr_debug("defer bootloader framebuffer release (%u/%u)\n",
+			count, target);
+		return;
+	}
+
+	bootloader_mem_release_queued = true;
+	mutex_unlock(&bootloader_mem_lock);
+
+	pr_debug("bootloader framebuffer release threshold reached (%u/%u)\n",
+		count, target);
+	queue_work(system_wq, &work_free_bootloader_mem);
+}
+
+void spacemit_dpu_set_bootloader_mem_release_target(unsigned int count)
+{
+	mutex_lock(&bootloader_mem_lock);
+	bootloader_mem_release_target = count ? count : 1;
+	mutex_unlock(&bootloader_mem_lock);
+}
+
+int spacemit_dpu_bootloader_mem_setup(struct reserved_mem *rmem)
+{
+	mutex_lock(&bootloader_mem_lock);
+	if (bootloader_mem &&
+	    bootloader_mem->base == rmem->base &&
+	    bootloader_mem->size == rmem->size) {
+		mutex_unlock(&bootloader_mem_lock);
+		return 0;
+	}
+
+	bootloader_mem_copy = *rmem;
+	bootloader_mem = &bootloader_mem_copy;
+	bootloader_mem_release_queued = false;
+	bootloader_mem_release_count = 0;
+	INIT_WORK(&work_free_bootloader_mem, __free_bootloader_mem);
+	mutex_unlock(&bootloader_mem_lock);
+
+	pr_info("Reserved memory: detected framebuffer at %pa, size %ld MB\n",
+		&rmem->base, (unsigned long)rmem->size / SZ_1M);
+
+	return 0;
+}
+
+#ifndef MODULE
+static int __init spacemit_dpu_bootloader_reserved_mem_setup(struct reserved_mem *rmem)
+{
+	return spacemit_dpu_bootloader_mem_setup(rmem);
+}
+
+RESERVEDMEM_OF_DECLARE(framebuffer, "framebuffer",
+		       spacemit_dpu_bootloader_reserved_mem_setup);
+#endif
diff --git a/drivers/gpu/drm/spacemit/spacemit_bootloader.h b/drivers/gpu/drm/spacemit/spacemit_bootloader.h
new file mode 100644
index 000000000000..2e79048f14ee
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_bootloader.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_BOOTLOADER_H_
+#define _SPACEMIT_BOOTLOADER_H_
+
+#include <linux/of_reserved_mem.h>
+
+void spacemit_dpu_free_bootloader_mem(void);
+void spacemit_dpu_set_bootloader_mem_release_target(unsigned int count);
+int spacemit_dpu_bootloader_mem_setup(struct reserved_mem *rmem);
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/spacemit_cmdlist.c b/drivers/gpu/drm/spacemit/spacemit_cmdlist.c
new file mode 100644
index 000000000000..cded764db679
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_cmdlist.c
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/sort.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_plane_helper.h>
+#include "spacemit_cmdlist.h"
+#include "spacemit_crtc.h"
+#include "spacemit_dpu_reg.h"
+#include "spacemit_drm.h"
+#include "dpu/dpu_saturn.h"
+#include "dpu/dpu_trace.h"
+
+inline struct spacemit_plane_state *cl_to_spacemit_pstate(const struct cmdlist *cl)
+{
+	return container_of(cl, struct spacemit_plane_state, cl[cl->index]);
+}
+
+inline struct spacemit_crtc_state *cl_to_spacemit_cstate(const struct cmdlist *cl)
+{
+	return container_of(cl, struct spacemit_crtc_state, cl[cl->index]);
+}
+
+struct cmdlist *plane_to_cl(struct drm_plane *plane)
+{
+	struct spacemit_plane_state *spacemit_pstate = to_spacemit_plane_state(plane->state);
+	u8 cur_cl = spacemit_pstate->cur_cl;
+
+	return (struct cmdlist *)(&spacemit_pstate->cl[cur_cl]);
+}
+
+struct cmdlist *crtc_to_cl(struct drm_crtc *crtc)
+{
+	struct spacemit_crtc_state *spacemit_cstate = to_spacemit_crtc_state(crtc->state);
+	u8 cur_cl = spacemit_cstate->cur_cl;
+
+	return (struct cmdlist *)(&spacemit_cstate->cl[cur_cl]);
+}
+
+void print_row(u32 *row)
+{
+	u32 *p = row;
+
+	DRM_DEBUG("%s: 0x%02x, 0x%02x, 0x%02x, 0x%02x", __func__, *p, *(p+1), *(p+2), *(p+3));
+}
+
+static void cmdlist_dump_node(struct cmdlist *cl)
+{
+	struct spacemit_drm_private *priv;
+	struct spacemit_hw_device *hwdev;
+	struct spacemit_crtc *a_crtc;
+	char *type;
+	int zpos = -1, rdma_id = -1;
+
+	if (cl == NULL)
+		return;
+
+	if (cl->type == CMDLIST_PLANE) {
+		struct spacemit_plane_state *spacemit_pstate = cl_to_spacemit_pstate(cl);
+
+		a_crtc = to_spacemit_crtc(spacemit_pstate->state.crtc);
+		rdma_id = spacemit_pstate->rdma_id;
+		zpos = (int)(spacemit_pstate->state.zpos);
+		type = "RDMA";
+	} else if (cl->type == CMDLIST_CRTC) {
+		struct spacemit_crtc_state *spacemit_cstate = cl_to_spacemit_cstate(cl);
+
+		a_crtc = to_spacemit_crtc(spacemit_cstate->base.crtc);
+		type = "CRTC";
+	} else {
+		trace_printk("Dump Unknown cmdlist group\n");
+		return;
+	}
+
+	trace_cmdlist_dump_node(type, cl->index, rdma_id, a_crtc->dev_id, zpos, cl->nod_len);
+
+	priv = a_crtc->crtc.dev->dev_private;
+	hwdev = priv->hwdev;
+	hwdev->cmdlist_dump_node(cl);
+
+}
+
+void cmdlist_regs_packing(struct cmdlist *cl, cmdlist_mode_type_t mod, struct cmdlist_regs *cl_regs)
+{
+	struct spacemit_crtc *a_crtc;
+	struct spacemit_drm_private *priv;
+	struct spacemit_hw_device *hwdev;
+	u32 *reg_base;
+	int i;
+	u32 index;
+
+	if (cl_regs == NULL)
+		return;
+
+	if (cl->type == CMDLIST_PLANE) {
+		struct spacemit_plane_state *spacemit_pstate = cl_to_spacemit_pstate(cl);
+
+		a_crtc = to_spacemit_crtc(spacemit_pstate->state.crtc);
+
+		DRM_DEBUG("-----%s----- rch_id = %d, ch_y = %u\n", __func__,
+			  spacemit_pstate->rdma_id, spacemit_pstate->state.crtc_y);
+	} else if (cl->type == CMDLIST_CRTC) {
+		struct spacemit_crtc_state *spacemit_cstate = cl_to_spacemit_cstate(cl);
+
+		a_crtc = to_spacemit_crtc(spacemit_cstate->base.crtc);
+	} else {
+		DRM_ERROR("Unsupported cmdlist type:%d\n", cl->type);
+		return;
+	}
+
+	priv = a_crtc->crtc.dev->dev_private;
+	hwdev = priv->hwdev;
+	reg_base = (u32 *)(cl_regs->module);
+
+	cl->mode_mask |= mod;
+	if (cl->va == NULL) {
+		cl->size = PER_CMDLIST_SIZE;
+		cl->va = dma_alloc_coherent(a_crtc->dev, cl->size, &cl->pa, GFP_KERNEL | __GFP_ZERO);
+		if (cl->va == NULL) {
+			DRM_ERROR("Failed to allocate %d bytes for dpu type:%d cmdlist buffer\n",
+				  PER_CMDLIST_SIZE, cl->type);
+			return;
+		}
+	}
+
+	for (i = 0; i < cl_regs->size;) {
+		u32 value[CMDLIST_ROW_REGS] = {0x0};
+		u32 offset, strobe = 0;
+		u8 regs_in_row = 0;
+
+		if (cl_regs->flags[i]) {
+			offset = cl_regs->base + i * sizeof(u32);
+			for (u8 j = 0; j < CMDLIST_ROW_REGS; j++) {
+				index = i + j;
+				if (likely(index < cl_regs->size)) {
+					if (cl_regs->flags[index]) {
+						value[j] = reg_base[index];
+						regs_in_row++;
+						strobe |= CMDLIST_REG_STROBE(j);
+					}
+				} else
+					break;
+			}
+			hwdev->cmdlist_fill_data_row(cl, strobe, offset, value);
+			i += regs_in_row;
+			cl->nod_len++;
+		} else
+			i++;
+	}
+
+	DRM_DEBUG("-----%s----- row_num = %d\n", __func__, cl->nod_len);
+}
+
+void cmdlist_sort_by_group(struct drm_crtc *crtc)
+{
+	struct cmdlist *first_cl;
+	struct cmdlist *last_cl;
+	struct cmdlist *p;
+	struct cmdlist *prev;
+	struct drm_plane *plane;
+	struct spacemit_crtc_rdma *rdmas = to_spacemit_crtc_state(crtc->state)->rdmas;
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+
+#ifndef CONFIG_DRM_SPACEMIT_CMDLIST
+	return;
+#endif
+	drm_atomic_crtc_for_each_plane(plane, crtc) {
+		struct spacemit_plane_state *spacemit_pstate = to_spacemit_plane_state(plane->state);
+		u32 rdma_id = spacemit_pstate->rdma_id;
+
+		first_cl = (struct cmdlist *)(&(spacemit_pstate->cl[0]));
+
+		if (!first_cl->va)
+			continue;
+		else {
+			int i;
+			struct cmdlist *tmp_cl = first_cl;
+
+			/* Add a plane's cmdlist items into a list */
+			for (i = 1; i < MAX_CL_NUM; i++) {
+				if (spacemit_pstate->cl[i].va) {
+					tmp_cl->next = (struct cmdlist *)(&(spacemit_pstate->cl[i]));
+					tmp_cl = tmp_cl->next;
+				}
+			}
+			last_cl = tmp_cl;
+			last_cl->next = NULL;
+		}
+
+		rdmas[rdma_id].in_use = true;
+		if (priv->cmdlist_groups[rdma_id]) {
+			p = priv->cmdlist_groups[rdma_id];
+			prev = NULL;
+			while (p) {
+				if (cl_to_spacemit_pstate(p)->state.crtc_y < spacemit_pstate->state.crtc_y) {
+					prev = p;
+					p = p->next;
+				} else
+					break;
+			}
+			if (!prev) {
+				priv->cmdlist_groups[rdma_id] = first_cl;
+				last_cl->next = p;
+			} else {
+				prev->next = first_cl;
+				last_cl->next = p;
+			}
+		} else
+			priv->cmdlist_groups[rdma_id] = first_cl;
+	}
+}
+
+void cmdlist_atomic_commit(struct drm_crtc *crtc,
+			   struct drm_crtc_state *old_state)
+{
+	int i;
+	struct cmdlist *cur_cl, *first_cl;
+	struct spacemit_crtc *a_crtc = NULL;
+	u32 chy, addrl, addrh;
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct spacemit_crtc_rdma *cur_rdmas = to_spacemit_crtc_state(crtc->state)->rdmas;
+	struct spacemit_crtc_rdma *old_rdmas = to_spacemit_crtc_state(old_state)->rdmas;
+
+#ifndef CONFIG_DRM_SPACEMIT_CMDLIST
+	return;
+#endif
+	a_crtc = to_spacemit_crtc(crtc);
+
+	for (i = 0; i < hwdev->rdma_nums; i++) {
+		/* Shut down the rdma used in previous frame first */
+		if (old_rdmas[i].in_use) {
+			if (a_crtc->is_stopped == false)
+				hwdev->enable_cmdlist(a_crtc, hwdev, i, false);
+			a_crtc->dpuctrl_ctl_nml_cmdlist_rch_en[i] = 0;
+		}
+
+		if (cur_rdmas[i].in_use) {
+			DRM_DEBUG("+++++%s+++++ cmdlist group = %d\n", __func__, i);
+			cur_cl = priv->cmdlist_groups[i];
+			first_cl = cur_cl;
+			while (cur_cl) {
+				hwdev->cmdlist_fill_conf_row(cur_cl, hwdev, CMDLIST_CMP_INVALID);
+				cmdlist_dump_node(cur_cl);
+				cur_cl = cur_cl->next;
+			}
+			chy = cl_to_spacemit_pstate(first_cl)->state.crtc_y;
+			addrl = ((priv->cmdlist_groups[i]->pa) & CMDLIST_ADDRL_ALIGN_MASK) >> CMDLIST_ADDRL_ALIGN_BITS;
+
+			addrh = (priv->cmdlist_groups[i]->pa) >> 32;
+			hwdev->cfg_cmdlist(hwdev, i, chy, addrl, addrh);
+			if (a_crtc->is_stopped == false)
+				hwdev->enable_cmdlist(a_crtc, hwdev, i, true);
+			a_crtc->dpuctrl_ctl_nml_cmdlist_rch_en[i] = 1;
+			priv->cmdlist_groups[i] = NULL;
+		}
+	}
+}
+
+void wb_cmdlist_sort_by_group(void *data)
+{
+	struct cmdlist *first_cl;
+	struct cmdlist *last_cl;
+	struct spacemit_crtc *a_crtc = (struct spacemit_crtc *)data;
+	struct drm_crtc *crtc = &(a_crtc->crtc);
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_crtc_state *crtc_state = a_crtc->crtc.state;
+	struct spacemit_crtc_state *spacemit_crtc_state = to_spacemit_crtc_state(crtc_state);
+	u8 crtc_id = drm_crtc_index(crtc);
+	int i;
+	struct cmdlist *tmp_cl;
+
+	first_cl =  (struct cmdlist *)(&(spacemit_crtc_state->cl[0]));
+
+	tmp_cl = first_cl;
+
+	for (i = 1; i < MAX_CL_NUM; i++) {
+		if (spacemit_crtc_state->cl[i].va) {
+			tmp_cl->next = (struct cmdlist *)(&(spacemit_crtc_state->cl[i]));
+			tmp_cl = tmp_cl->next;
+		} else
+			break;
+	}
+	last_cl = tmp_cl;
+	last_cl->next = NULL;
+	priv->cmdlist_groups[hwdev->rdma_nums + crtc_id] = first_cl;
+}
+
+void wb_cmdlist_atomic_commit(void *data)
+{
+	struct cmdlist *cur_cl;
+	struct spacemit_crtc *a_crtc = (struct spacemit_crtc *)data;
+	struct drm_crtc *crtc = &(a_crtc->crtc);
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u8 crtc_id = drm_crtc_index(crtc);
+	u8 dev_id = a_crtc->dev_id;
+
+	DRM_DEBUG("+++++cmdlist_atomic_commit+++++ cmdlist group = %d\n", hwdev->rdma_nums + crtc_id);
+	cur_cl = priv->cmdlist_groups[hwdev->rdma_nums + crtc_id];
+	while (cur_cl) {
+		hwdev->cmdlist_fill_conf_row(cur_cl, hwdev, dev_id);
+		cmdlist_dump_node(cur_cl);
+		cur_cl = cur_cl->next;
+	}
+	hwdev->wb_cmdlist(cur_cl, hwdev, priv, crtc_id, dev_id);
+}
diff --git a/drivers/gpu/drm/spacemit/spacemit_cmdlist.h b/drivers/gpu/drm/spacemit/spacemit_cmdlist.h
new file mode 100644
index 000000000000..6d3d9305c1f6
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_cmdlist.h
@@ -0,0 +1,125 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SATURN_FW_CMDLIST_H_
+#define _SATURN_FW_CMDLIST_H_
+
+#include <linux/stddef.h>
+#include <linux/types.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_plane.h>
+
+/*
+ * cmdlist v0 : lea, le, lec, lee
+ * cmdlist v1 : led
+ */
+#define PER_CMDLIST_SIZE             4096
+#define CMDLIST_ROW_REGS	(3)  //number of regs in each row
+#define CMDLIST_CMP_INVALID	(0xff)
+
+enum cmdlist_type {
+	CMDLIST_PLANE = 1,
+	CMDLIST_CRTC,
+};
+/* compatible for v0 and v1 */
+struct cmdlist {
+	u16 nod_len;        //the number of cmdlist row.
+	u32 size;
+	u32 mode_mask;		//for cmdlist_v1
+	void *va;
+	dma_addr_t pa;
+	struct cmdlist *next;
+	u8 index;	//cmdlist index
+	u8 type;
+	u32 rch_start_cmps_y;
+	u32 cmdlist_ch_y_other;
+};
+
+typedef enum {
+	CMDLIST_MOD_RDMA = BIT(0),
+	CMDLIST_MOD_LP = BIT(1),
+	CMDLIST_MOD_SCL = BIT(2),
+	CMDLIST_MOD_COMP = BIT(3),
+	CMDLIST_MOD_WB0 = BIT(4),
+	CMDLIST_MOD_WB1 = BIT(5),
+	CMDLIST_MOD_DMMU = BIT(6),
+} cmdlist_mode_type_t;
+
+/* reg0: 0xf reg1: 0xf0 reg2: 0xf00*/
+#define CMDLIST_REG_STROBE(index)	(0xf << index * 4)
+
+#define CMDLIST_ADDRL_ALIGN_BITS               (4) //From cmdlist_reg_0[] in CMDLIST_REG
+#define CMDLIST_ADDRL_ALIGN_MASK               ((u32)(~(BIT(CMDLIST_ADDRL_ALIGN_BITS) - 1)))
+#define CONFIG_DRM_SPACEMIT_CMDLIST 1
+
+#ifdef CONFIG_DRM_SPACEMIT_CMDLIST
+#define alloc_cmdlist_regs(module_name) \
+({ \
+	struct cmdlist_regs *cl = kzalloc(sizeof(struct cmdlist_regs), GFP_KERNEL); \
+	if (cl == NULL) \
+		DRM_ERROR("Failed to allocate struct cmdlist_regs!\n"); \
+	cl->size = sizeof(module_name) / sizeof(INT32); \
+	cl->module = kzalloc(sizeof(module_name), GFP_KERNEL); \
+	if (cl->module == NULL) \
+		DRM_ERROR("Failed to allocate module regs!\n"); \
+	cl->flags = kzalloc(cl->size, GFP_KERNEL); \
+	if (cl->flags == NULL) \
+		DRM_ERROR("Failed to allocate module flags!\n"); \
+	cl; \
+})
+
+#define free_cmdlist_regs(cl_p) \
+{ \
+	struct cmdlist_regs *cl = (struct cmdlist_regs *)cl_p; \
+	if (unlikely(cl == NULL)) \
+		DRM_ERROR("NULL cmdlist_regs pointer!\n"); \
+	else { \
+		kfree(cl->module); \
+		kfree(cl->flags); \
+		kfree(cl); \
+	} \
+}
+
+#define cmdlist_write_reg(hwdev, module_name, module_base, field, data, cl_regs, offset) \
+{ \
+	struct cmdlist_regs *cl_p = (struct cmdlist_regs *)cl_regs; \
+	volatile module_name *module = (module_name *)(cl_p->module); \
+	if (cl_p->flags[offset] == 0) { \
+		volatile module_name *module_hw = ((volatile module_name *)(module_base + \
+						  ((struct spacemit_hw_device *)hwdev)->base)); \
+		module->value32[offset] = module_hw->value32[offset]; \
+		cl_p->base = module_base; \
+		cl_p->flags[offset] = 1; \
+	} \
+	module->field = data; \
+}
+#else
+#define alloc_cmdlist_regs(module_name) NULL
+#define free_cmdlist_regs(cl_p) {}
+#define cmdlist_write_reg(hwdev, module_name, module_base, field, data, cl_regs, offset) {}
+#endif	/* CONFIG_DRM_SPACEMIT_CMDLIST */
+
+struct cmdlist_regs {
+	u32 base;
+	u16 size;
+	void *module;
+	u8 *flags;
+};
+
+void cmdlist_regs_packing(struct cmdlist *cl, cmdlist_mode_type_t mod, \
+			  struct cmdlist_regs *cl_regs);
+void cmdlist_sort_by_group(struct drm_crtc *crtc);
+void cmdlist_atomic_commit(struct drm_crtc *crtc,
+			   struct drm_crtc_state *old_state);
+void print_row(u32 *row);
+struct spacemit_plane_state *cl_to_spacemit_pstate(const struct cmdlist *cl);
+void wb_cmdlist_sort_by_group(void *a_crtc);
+void wb_cmdlist_atomic_commit(void *a_crtc);
+struct spacemit_crtc_state *cl_to_spacemit_cstate(const struct cmdlist *cl);
+struct cmdlist *plane_to_cl(struct drm_plane *plane);
+struct cmdlist *crtc_to_cl(struct drm_crtc *crtc);
+
+#endif /* _SATURN_FW_CMDLIST_H_ */
diff --git a/drivers/gpu/drm/spacemit/spacemit_crtc.c b/drivers/gpu/drm/spacemit/spacemit_crtc.c
new file mode 100644
index 000000000000..2faeb81d24a4
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_crtc.c
@@ -0,0 +1,1844 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/component.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_qos.h>
+#include <linux/mutex.h>
+#include <linux/string.h>
+#include <linux/trace_events.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <dt-bindings/display/spacemit_dpu.h>
+#include <linux/reset.h>
+#include "spacemit_cmdlist.h"
+#include "spacemit_dmmu.h"
+#include "spacemit_drm.h"
+#include "spacemit_crtc.h"
+#include "spacemit_wb.h"
+#include "spacemit_gem.h"
+#include "spacemit_lib.h"
+#include "spacemit_bootloader.h"
+#include "dpu/dpu_saturn.h"
+#include "dpu/dpu_debug.h"
+#include "sysfs/sysfs_display.h"
+#include "dpu/dpu_trace.h"
+
+LIST_HEAD(dpu_core_head);
+
+static int spacemit_crtc_init(struct spacemit_crtc *a_crtc);
+static int spacemit_crtc_uninit(struct spacemit_crtc *a_crtc);
+static int dpu_pm_suspend(struct device *dev);
+static int dpu_pm_resume(struct device *dev);
+static unsigned int spacemit_dpu_get_bootlogo_total_count(void);
+static DEFINE_MUTEX(spacemit_bootloader_mem_setup_lock);
+static bool spacemit_bootloader_mem_setup_done = false;
+
+static struct device_node *spacemit_dpu_find_bootloader_mem_node(void)
+{
+	struct device_node *rmem_np, *child;
+
+	rmem_np = of_find_node_by_path("/reserved-memory");
+	if (!rmem_np)
+		return NULL;
+
+	for_each_child_of_node(rmem_np, child) {
+		if (of_device_is_compatible(child, "framebuffer") ||
+		    !strncmp(child->name, "framebuffer",
+			     sizeof("framebuffer") - 1)) {
+			of_node_put(rmem_np);
+			return child;
+		}
+	}
+
+	of_node_put(rmem_np);
+	return NULL;
+}
+
+static void spacemit_dpu_setup_bootloader_mem(struct device *dev)
+{
+	struct device_node *np;
+	struct resource rsrv_mem;
+	struct reserved_mem rmem;
+	int ret;
+
+	mutex_lock(&spacemit_bootloader_mem_setup_lock);
+	if (spacemit_bootloader_mem_setup_done) {
+		mutex_unlock(&spacemit_bootloader_mem_setup_lock);
+		return;
+	}
+
+	np = spacemit_dpu_find_bootloader_mem_node();
+	if (!np) {
+		mutex_unlock(&spacemit_bootloader_mem_setup_lock);
+		return;
+	}
+
+	ret = of_address_to_resource(np, 0, &rsrv_mem);
+	if (ret < 0) {
+		DRM_INFO("no bootloader reserved memory resource found\n");
+		of_node_put(np);
+		mutex_unlock(&spacemit_bootloader_mem_setup_lock);
+		return;
+	}
+
+	rmem.base = rsrv_mem.start;
+	rmem.size = resource_size(&rsrv_mem);
+	spacemit_dpu_set_bootloader_mem_release_target(spacemit_dpu_get_bootlogo_total_count());
+	ret = spacemit_dpu_bootloader_mem_setup(&rmem);
+	if (ret)
+		DRM_INFO("failed to setup bootloader reserved memory: %d\n", ret);
+	else
+		spacemit_bootloader_mem_setup_done = true;
+
+	of_node_put(np);
+	mutex_unlock(&spacemit_bootloader_mem_setup_lock);
+}
+
+static unsigned int spacemit_dpu_get_bootlogo_total_count(void)
+{
+	struct device_node *np;
+	unsigned int count = 0;
+
+	for_each_compatible_node(np, NULL, "spacemit,dpu-saturn") {
+		if (of_device_is_available(np))
+			count++;
+	}
+
+	return count ? count : 1;
+}
+
+static atomic_t mclk_cnt = ATOMIC_INIT(0);
+bool dpu_mclk_exclusive_get(void)
+{
+	if (atomic_cmpxchg(&mclk_cnt, 0, 1) == 0)
+		return true;
+	else
+		return false;
+}
+EXPORT_SYMBOL(dpu_mclk_exclusive_get);
+
+void dpu_mclk_exclusive_put(void)
+{
+	atomic_set(&mclk_cnt, 0);
+}
+EXPORT_SYMBOL(dpu_mclk_exclusive_put);
+
+static BLOCKING_NOTIFIER_HEAD(dpu_max_mclk_notifier_list);
+int dpu_max_mclk_notifier_register(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&dpu_max_mclk_notifier_list, nb);
+}
+EXPORT_SYMBOL(dpu_max_mclk_notifier_register);
+
+int dpu_max_mclk_notifier_unregister(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_unregister(&dpu_max_mclk_notifier_list, nb);
+}
+EXPORT_SYMBOL(dpu_max_mclk_notifier_unregister);
+
+int dpu_max_mclk_notifier_call_chain(unsigned int mclk_rate)
+{
+	return blocking_notifier_call_chain(&dpu_max_mclk_notifier_list, (unsigned long)mclk_rate, NULL);
+}
+EXPORT_SYMBOL(dpu_max_mclk_notifier_call_chain);
+
+static int spacemit_crtc_atomic_check_color_matrix(struct drm_crtc *crtc,
+					  struct drm_crtc_state *state)
+{
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	int ret = 0;
+
+	if (hwdev->check_end_matrix)
+		ret = hwdev->check_end_matrix(state);
+
+	return ret;
+}
+
+static int spacemit_crtc_atomic_check_color_temp(struct drm_crtc *crtc,
+					  struct drm_crtc_state *state)
+{
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = ac->pp_color_temperature_blob_property;
+	int *coef_data;
+	int n;
+
+	if (blob){
+		coef_data = (int *)blob->data;
+		for (n = 0; n < 9; n++){
+			if ((coef_data[n] > 32767) || (coef_data[n] < -32768)){
+				DRM_ERROR("color temp table is invalid %d\n", coef_data[n]);
+				return -EINVAL;
+			}
+		}
+
+		for (n = 9; n < 12; n++){
+			if ((coef_data[n] > 16777215) || (coef_data[n] < -16777216)){
+				DRM_ERROR("color temp offset is invalid %d\n", coef_data[n]);
+				return -EINVAL;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int spacemit_crtc_atomic_check_gamma_table(struct drm_crtc *crtc,
+					  struct drm_crtc_state *state)
+{
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = ac->gamma_table_blob_prop;
+	uint16_t *data;
+	int idx;
+	int len;
+
+	if (blob) {
+		data = (uint16_t *)blob->data;
+		len = blob->length / sizeof(uint16_t);
+		for (idx = 0; idx < len; idx++) {
+			if ((data[idx] > 4095) || (data[idx] < 0)) {
+				DRM_DEBUG("The value of gamma table is invalid: value %d, n %d\n", data[idx], idx);
+				return -EINVAL;
+			}
+		}
+	}
+	return 0;
+}
+
+static int spacemit_crtc_atomic_check_end_tone_mapping(struct drm_crtc *crtc,
+						struct drm_crtc_state *state)
+{
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(state);
+	struct drm_property_blob *blob = ac->end_tone_mapping_blob_prop;
+	uint16_t *data;
+	int idx, len;
+
+	if (blob) {
+		data = (uint16_t *)blob->data;
+		len = blob->length / sizeof(uint16_t);
+		for (idx = 0; idx < len; idx++) {
+			if (data[idx] > 65535 || data[idx] < 0) {
+				DRM_DEBUG("The value of end tone mapping is invalid: value %d, n %d\n", data[idx], idx);
+				return -EINVAL;
+			}
+		}
+	}
+	return 0;
+}
+
+static int spacemit_crtc_atomic_check_scaling(struct drm_crtc *crtc,
+					    struct drm_crtc_state *crtc_state)
+{
+	struct drm_plane *plane;
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(crtc_state);
+	const struct drm_plane_state *pstate;
+	struct spacemit_crtc_scaler *scaler = NULL;
+	struct spacemit_plane_state *spacemit_pstate;
+	u32 i;
+
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		spacemit_pstate = to_spacemit_plane_state(pstate);
+
+		if (spacemit_pstate->use_scl) {
+			for (i = 0; i < MAX_SCALER_NUMS; i++) {
+				scaler = &(ac->scalers[i]);
+				if (scaler->in_use == 0x0 || scaler->rdma_id == spacemit_pstate->rdma_id) {
+					scaler->in_use |= (1 << plane->index);
+					scaler->rdma_id = spacemit_pstate->rdma_id;
+					break;
+				}
+			}
+
+			if (i == MAX_SCALER_NUMS) {
+				DRM_ERROR("Exceeds the max scaler number\n");
+				return -EINVAL;
+			}
+		}
+	}
+
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		spacemit_pstate = to_spacemit_plane_state(pstate);
+
+		if (spacemit_pstate->rdma_id != RDMA_INVALID_ID) {
+			for (i = 0; i < MAX_SCALER_NUMS; i++) {
+				scaler = &(ac->scalers[i]);
+				if (scaler->rdma_id == spacemit_pstate->rdma_id && scaler->in_use)
+					spacemit_pstate->scaler_id = i;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/* crtc actual mclk depends on max(mclk, aclk) */
+static int spacemit_crtc_atomic_update_mclk(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_state)
+{
+	struct spacemit_crtc_state *new_ac = to_spacemit_crtc_state(crtc->state);
+
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct dpu_clk_context *clk_ctx = NULL;
+
+	if (a_crtc == NULL)
+		return 0;
+
+	if (!a_crtc->enable_auto_fc)
+		return 0;
+
+	/* when shutdown, all planes disabled, crtc still work, need keep cur mclk */
+	if (new_ac->real_mclk == 0) {
+		trace_u64_data("new_ac->real_mclk", new_ac->real_mclk);
+		return 0;
+	}
+
+	clk_ctx = &a_crtc->clk_ctx;
+	cancel_work_sync(&a_crtc->work_update_clk); /* incase mclk fq doing */
+
+	a_crtc->cur_mclk = clk_get_rate(clk_ctx->mclk);
+
+	if (a_crtc->fix_max_mclk)
+		a_crtc->new_mclk = clk_round_rate(clk_ctx->mclk, a_crtc->max_mclk);
+	else
+		a_crtc->new_mclk = clk_round_rate(clk_ctx->mclk, new_ac->real_mclk);
+
+	trace_u64_data("crtc mclk cur", a_crtc->cur_mclk);
+	trace_u64_data("crtc mclk new", a_crtc->new_mclk);
+	if (a_crtc->core && a_crtc->core->update_clk && (a_crtc->new_mclk > a_crtc->cur_mclk)) {
+		trace_u64_data("mclk increase to", a_crtc->new_mclk);
+		a_crtc->core->update_clk(a_crtc, a_crtc->new_mclk);
+	}
+
+	cancel_work_sync(&a_crtc->work_update_bw);
+
+	a_crtc->new_bw = new_ac->bw;
+
+	trace_u64_data("crtc bw cur", a_crtc->cur_bw);
+	trace_u64_data("crtc bw new", a_crtc->new_bw);
+
+	if (a_crtc->core && a_crtc->core->update_bw && (a_crtc->new_bw > a_crtc->cur_bw)) {
+		trace_u64_data("bw increase to", a_crtc->new_bw);
+		a_crtc->core->update_bw(a_crtc, a_crtc->new_bw);
+	}
+
+	return 0;
+}
+
+#define  RDMA_NUM_MAX   16
+/* crtc aclk depends on sum of all rdma bw */
+static int spacemit_crtc_atomic_check_aclk(struct drm_crtc *crtc,
+					    struct drm_crtc_state *crtc_state)
+{
+	const struct drm_plane_state *pstate;
+	struct drm_plane *plane;
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(crtc_state); /* new state */
+	bool scl_en = false;
+	uint64_t afbc_effc = ~0ULL;
+	uint64_t tmp, tmp1;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	uint32_t rdma_num = priv->hwdev->rdma_nums;
+	uint64_t rdma_bw[RDMA_NUM_MAX] = { 0 };
+
+	if (ARRAY_SIZE(rdma_bw) < rdma_num) {
+		DRM_ERROR("size of rdma_bw(%lu) too small, rdma_num: %d", ARRAY_SIZE(rdma_bw), rdma_num);
+		return -EINVAL;
+	}
+
+	if (!a_crtc->enable_auto_fc)
+		return 0;
+
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		uint64_t tmp_bw = to_spacemit_plane_state(pstate)->bw;
+		bool tmp_scl_en = to_spacemit_plane_state(pstate)->use_scl;
+		uint32_t rdma_id = to_spacemit_plane_state(pstate)->rdma_id;
+
+		if (rdma_id < rdma_num)
+			rdma_bw[rdma_id] = max(tmp_bw, rdma_bw[rdma_id]);
+
+		if (tmp_scl_en) {
+			scl_en = true;
+			if (afbc_effc > to_spacemit_plane_state(pstate)->afbc_effc)
+				afbc_effc = to_spacemit_plane_state(pstate)->afbc_effc;
+		}
+	}
+
+	for (int i = 0; i < rdma_num; i++)
+		ac->bw += rdma_bw[i];
+
+	if (ac->bw == 0)
+		ac->bw = DPU_MIN_QOS_REQ;
+
+	if (ac->bw > (a_crtc->max_bw * MHZ2KHZ)) {
+		DRM_INFO("crtc:%lld bandwidth too large\n", ac->bw);
+		return -EINVAL;
+	}
+
+	if (scl_en) {
+		//ac->aclk = max(ac->bw / 16 * 10 / 5, ac->mclk / afbc_effc);
+		tmp = ac->bw;
+		do_div(tmp, 16);
+		tmp = tmp * 100;
+		do_div(tmp, 50);
+		tmp1 = ac->mclk;
+		do_div(tmp1, afbc_effc);
+		ac->aclk = max(tmp, tmp1);
+	} else {
+		//ac->aclk = ac->bw / 16 * 10 / 5;
+		tmp = ac->bw;
+		do_div(tmp, 16);
+		tmp = tmp * 100;
+		do_div(tmp, 50);
+		ac->aclk = tmp;
+	}
+
+	trace_u64_data("crtc calc bw", ac->bw);
+	if (a_crtc->bw_margin)
+		ac->bw += a_crtc->bw_margin * MHZ2KHZ;
+
+	ac->real_mclk = max(ac->mclk, ac->aclk);
+
+	if (ac->real_mclk < a_crtc->min_mclk)
+		ac->real_mclk = a_crtc->min_mclk;
+
+	if (ac->real_mclk > a_crtc->max_mclk) {
+		pr_err_ratelimited("real_mclk = %llu, max_mclk = %d\n", ac->real_mclk, a_crtc->max_mclk);
+		return -EINVAL;
+	}
+
+	if (a_crtc->bw_margin)
+		trace_u64_data("crtc fixed bw", ac->bw);
+	trace_u64_data("crtc calc aclk", ac->aclk);
+	trace_u64_data("crtc calc mclk", ac->mclk);
+	trace_u64_data("crtc calc real_mclk", ac->real_mclk);
+
+	return 0;
+}
+
+/* crtc mclk depends on max of all rdma mclk */
+static int spacemit_crtc_atomic_check_mclk(struct drm_crtc *crtc,
+					    struct drm_crtc_state *crtc_state)
+{
+	const struct drm_plane_state *pstate;
+	struct drm_plane *plane;
+	struct spacemit_crtc_state *ac = to_spacemit_crtc_state(crtc_state);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+
+	if (!a_crtc->enable_auto_fc)
+		return 0;
+
+	ac->mclk = DPU_MCLK_MIN;
+
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		uint64_t tmp_mclk = to_spacemit_plane_state(pstate)->mclk;
+
+		if (ac->mclk < tmp_mclk)
+			ac->mclk = tmp_mclk;
+	}
+
+	return 0;
+}
+
+static void saturn_check_dpuctrl_scl_reuse(struct drm_crtc *crtc,
+					    struct drm_crtc_state *crtc_state)
+{
+	struct drm_plane *plane;
+	struct spacemit_crtc_state *spacemit_state = to_spacemit_crtc_state(crtc_state);
+	struct spacemit_crtc_rdma *rdmas = spacemit_state->rdmas;
+	u32 rdma_id = 0;
+	u32 scl_rdma_id = 0;
+	struct spacemit_crtc_scaler *scaler = NULL;
+	const struct drm_plane_state *pstate;
+	int i = 0;
+
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		rdma_id = to_spacemit_plane_state(pstate)->rdma_id;
+		if (rdma_id != RDMA_INVALID_ID)
+			rdmas[rdma_id].use_cnt++;
+	}
+
+	//TODO: SCALER_NUMS for different IP
+	for (i = 0; i < MAX_SCALER_NUMS; i++) {
+		scaler = &(spacemit_state->scalers[i]);
+
+		if (scaler->in_use)
+			spacemit_state->scl_rdma_id[i] = scaler->rdma_id;
+		else
+			spacemit_state->scl_rdma_id[i] = RDMA_INVALID_ID;
+
+		if (rdmas[scl_rdma_id].use_cnt > 1)
+			spacemit_state->scl_rdma_reuse[i] = true;
+	}
+}
+
+static int spacemit_crtc_atomic_check_fbmem(struct drm_crtc *crtc,
+					    struct drm_crtc_state *crtc_state)
+{
+	struct spacemit_crtc_rdma *rdmas = to_spacemit_crtc_state(crtc_state)->rdmas;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	const struct drm_plane_state *pstate;
+	struct drm_plane *plane;
+	/* Calc each rdma required fbc mem size */
+	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
+		u32 rdma_id = to_spacemit_plane_state(pstate)->rdma_id;
+		u32 layer_fbcmem_size = to_spacemit_plane_state(pstate)->fbcmem_size;
+
+		if (rdma_id != RDMA_INVALID_ID) {
+			if (rdmas[rdma_id].mode == UP_DOWN)
+				rdmas[rdma_id].fbcmem.size = max(layer_fbcmem_size,
+								 rdmas[rdma_id].fbcmem.size);
+			else
+				rdmas[rdma_id].fbcmem.size += layer_fbcmem_size;
+		}
+	}
+
+	/* Adjust each rdma's fbcmem layout */
+	return a_crtc->core->adjust_rdma_fbcmem(hwdev, rdmas);
+
+}
+
+static void spacemit_crtc_mode_set_nofb(struct drm_crtc *crtc)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct videomode vm;
+
+	DRM_INFO("%s()\n", __func__);
+	trace_spacemit_crtc_mode_set_nofb(a_crtc->dev_id);
+	drm_display_mode_to_videomode(&crtc->mode, &vm);
+}
+
+static void spacemit_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_atomic_state *old_state)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+
+	DRM_INFO("%s(power on)\n", __func__);
+	trace_spacemit_crtc_atomic_enable(a_crtc->dev_id);
+
+	/* If bootloader framebuffer is active, release its resources first */
+	if (unlikely(a_crtc->logo_booton)) {
+		pm_runtime_enable(a_crtc->dev);
+		spacemit_dpu_power_enable(a_crtc, true);
+		dpu_pm_resume(a_crtc->dev);
+		dpu_pm_suspend(a_crtc->dev);
+		spacemit_dpu_power_enable(a_crtc, false);
+		spacemit_dpu_free_bootloader_mem();
+		a_crtc->logo_booton = false;
+		msleep(10);
+	}
+
+	if (!a_crtc->power_on) {
+		spacemit_dpu_power_enable(a_crtc, true);
+		dpu_pm_resume(a_crtc->dev);
+	}
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+	a_crtc->is_working = true;
+#endif
+	drm_crtc_vblank_on(&a_crtc->crtc);
+
+	spacemit_crtc_init(a_crtc);
+	a_crtc->rpm_status = true;
+}
+
+static void spacemit_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_atomic_state *old_state)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct drm_device *drm = a_crtc->crtc.dev;
+
+	DRM_INFO("%s(power off)\n", __func__);
+	trace_spacemit_crtc_atomic_disable(a_crtc->dev_id);
+
+	if (a_crtc->is_offline_mode)
+		spacemit_crtc_stop(a_crtc);
+	spacemit_crtc_uninit(a_crtc);
+
+	drm_crtc_vblank_off(&a_crtc->crtc);
+#ifdef CONFIG_SPACEMIT_DEBUG
+	a_crtc->is_working = false;
+#endif
+
+	dpu_pm_suspend(a_crtc->dev);
+	spacemit_dpu_power_enable(a_crtc, false);
+
+	spin_lock_irq(&drm->event_lock);
+	if (crtc->state->event) {
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+	}
+	spin_unlock_irq(&drm->event_lock);
+	a_crtc->rpm_status = false;
+}
+
+void spacemit_crtc_fill_slice(struct spacemit_crtc *a_crtc, u32 width, u32 height)
+{
+	a_crtc->slice_wb[a_crtc->slice_num].wb_in_w = width;
+	a_crtc->slice_wb[a_crtc->slice_num].wb_in_h = height;
+	a_crtc->slice_wb[a_crtc->slice_num].wb_out_w = width;
+	a_crtc->slice_wb[a_crtc->slice_num].wb_out_h = height;
+	a_crtc->slice_wb[a_crtc->slice_num].wb_out_x =
+		a_crtc->slice_num ? a_crtc->slice_wb[a_crtc->slice_num - 1].wb_out_x + a_crtc->slice_wb[a_crtc->slice_num - 1].wb_out_w : 0;
+	a_crtc->slice_wb[a_crtc->slice_num].wb_out_y = 0;
+	a_crtc->slice_wb[a_crtc->slice_num].rdma_w = width;
+	a_crtc->slice_wb[a_crtc->slice_num].rdma_h = height;
+	a_crtc->slice_wb[a_crtc->slice_num].rdma_x =
+		a_crtc->slice_num ? a_crtc->slice_wb[a_crtc->slice_num - 1].rdma_x + a_crtc->slice_wb[a_crtc->slice_num - 1].rdma_w : 0;
+	a_crtc->slice_wb[a_crtc->slice_num].rdma_y = 0;
+	a_crtc->slice_wb[a_crtc->slice_num].crtc_w = width;
+	a_crtc->slice_wb[a_crtc->slice_num].crtc_h = height;
+	a_crtc->slice_wb[a_crtc->slice_num].crtc_x = 0;
+	a_crtc->slice_wb[a_crtc->slice_num].crtc_y = 0;
+}
+
+int spacemit_crtc_calc_slices(struct spacemit_crtc *a_crtc, u32 width, u32 height)
+{
+	int default_slice_width = DEFAULT_SLICE_WIDTH;
+	int slice_width;
+	int left_width;
+	int total_slice_width;
+
+	DRM_DEBUG("%s() %d\n", __func__, __LINE__);
+	if (default_slice_width > width)
+		default_slice_width = width;
+
+	DRM_DEBUG("wb slice: default_slice_width: %d\n", default_slice_width);
+
+	//analyze slices
+	total_slice_width = 0;
+	a_crtc->slice_num = 0;
+	while (total_slice_width < width) {
+		slice_width = default_slice_width;
+		left_width = width - total_slice_width;
+		if (left_width <= slice_width)
+			slice_width = left_width;
+		else if (left_width < slice_width + 16)
+			slice_width -= 16;
+
+		DRM_DEBUG("slice%d: use slice_width:%d\n", a_crtc->slice_num, slice_width);
+		total_slice_width += slice_width;
+		spacemit_crtc_fill_slice(a_crtc, slice_width, height);
+		a_crtc->slice_num++;
+	}
+	return a_crtc->slice_num;
+}
+
+static int spacemit_crtc_atomic_check(struct drm_crtc *crtc,
+						struct drm_atomic_state *atomic_state)
+{
+	struct drm_crtc_state *state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	int ret = 0;
+	struct drm_display_mode *mode = NULL;
+
+	DRM_DEBUG("%s()\n", __func__);
+	trace_spacemit_crtc_atomic_check(a_crtc->dev_id);
+
+	ret = spacemit_crtc_atomic_check_scaling(crtc, state);
+	if (ret) {
+		DRM_ERROR("crtc_id=%u check_scaling FAILED ret=%d\n",
+			  crtc->base.id, ret);
+		return -EINVAL;
+	}
+
+	if (spacemit_crtc_atomic_check_color_temp(crtc, state)){
+		DRM_ERROR("The value of color temperature is invalid\n");
+		return -EINVAL;
+	}
+
+	if (spacemit_crtc_atomic_check_color_matrix(crtc, state)) {
+		DRM_ERROR("The value of color matrix is invalid\n");
+		return -EINVAL;
+	}
+
+	if (spacemit_crtc_atomic_check_gamma_table(crtc, state)) {
+		DRM_ERROR("The value of gamma table is invalid!\n");
+		return -EINVAL;
+	}
+
+	if (spacemit_crtc_atomic_check_end_tone_mapping(crtc, state)) {
+		DRM_ERROR("The value of end tone mapping is invalid!\n");
+		return -EINVAL;
+	}
+
+	if (spacemit_crtc_atomic_check_fbmem(crtc, state)) {
+		DRM_ERROR("Failed to satisfy fbcmem size for all rdmas!\n");
+		return -EINVAL;
+	}
+
+	if (spacemit_crtc_atomic_check_mclk(crtc, state)) {
+		DRM_ERROR("Failed to satisfy mclk for all rdmas!\n");
+		return -EINVAL;
+	}
+
+	if (spacemit_crtc_atomic_check_aclk(crtc, state)) {
+		DRM_ERROR("Failed to satisfy aclk for all rdmas!\n");
+		return -EINVAL;
+	}
+
+	saturn_check_dpuctrl_scl_reuse(crtc, state);
+
+	if (a_crtc->is_offline_mode) {
+		mode = &state->adjusted_mode;
+		if (mode->hdisplay > MAX_WIDTH) {
+			DRM_INFO("Frame width %d over max %d!\n", mode->hdisplay, MAX_WIDTH);
+			return -EINVAL;
+		}
+		if (mode->hdisplay > DEFAULT_SLICE_WIDTH) {
+			spacemit_crtc_calc_slices(a_crtc, mode->hdisplay, mode->vdisplay);
+			a_crtc->is_slice_mode = 1;
+		}
+	}
+
+	return ret;
+}
+
+static void spacemit_crtc_atomic_begin(struct drm_crtc *crtc,
+				  struct drm_atomic_state *state)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct videomode vm;
+
+	DRM_DEBUG("%s()\n", __func__);
+	trace_spacemit_crtc_atomic_begin(a_crtc->dev_id);
+	drm_display_mode_to_videomode(&crtc->mode, &vm);
+	if (a_crtc->vrr_vfp != vm.vfront_porch)
+		a_crtc->vrr_vfp = vm.vfront_porch;
+
+	if (a_crtc->core && a_crtc->core->begin)
+		a_crtc->core->begin(a_crtc);
+}
+
+#define VSYNC_PERIOD_VARIANCE_NS		2000000
+
+static void spacemit_wait_earliest_process_time(int32_t vrefresh, uint64_t expected_present_time)
+{
+
+	ktime_t now;
+	int64_t present_time_adjust, delay_until_process;
+	int64_t vsync_period_ns = mult_frac(1000, 1000 * 1000, vrefresh);
+
+	present_time_adjust = expected_present_time - (vsync_period_ns - VSYNC_PERIOD_VARIANCE_NS);
+	if (present_time_adjust <= 0) {
+		// Don't need to wait
+		return;
+	}
+
+	now = ktime_get();
+	delay_until_process = (int64_t)ktime_us_delta(present_time_adjust, now);
+	if (delay_until_process > 0) {
+		int32_t max_delay_us = (10 * vsync_period_ns) / 1000;
+
+		if (delay_until_process > max_delay_us) {
+			delay_until_process = max_delay_us;
+			pr_warn("expected present time seems incorrect(now %llu, earliest %llu)\n",
+					now, present_time_adjust);
+		}
+		usleep_range(delay_until_process, delay_until_process + 10);
+	}
+}
+
+static void spacemit_crtc_atomic_flush(struct drm_crtc *crtc,
+				  struct drm_atomic_state *state)
+
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct spacemit_crtc_state *new_state = to_spacemit_crtc_state(crtc->state);
+	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state, crtc);
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	DRM_DEBUG("%s()\n", __func__);
+	trace_spacemit_crtc_atomic_flush(a_crtc->dev_id);
+	hwdev->conf_dpuctrl_color_matrix(a_crtc, old_state);
+	if (hwdev->conf_dpuctrl_acad)
+		hwdev->conf_dpuctrl_acad(a_crtc, old_state);
+	if (hwdev->conf_ee)
+		hwdev->conf_ee(a_crtc, old_state);
+	if (hwdev->conf_gamma_table)
+		hwdev->conf_gamma_table(a_crtc, old_state);
+	if (hwdev->conf_end_tone_mapping)
+		hwdev->conf_end_tone_mapping(a_crtc, old_state);
+	if (hwdev->conf_matrix)
+		hwdev->conf_matrix(a_crtc, old_state);
+	spacemit_crtc_atomic_update_mclk(crtc, old_state);
+
+	if (new_state->expected_present_time != 0) {
+		int32_t vrefresh = drm_mode_vrefresh(&old_state->mode);
+
+		if (vrefresh == 0) {
+			/* decon just be enabled */
+			vrefresh = drm_mode_vrefresh(&crtc->state->mode);
+		}
+
+		spacemit_wait_earliest_process_time(vrefresh, new_state->expected_present_time);
+	}
+
+	spacemit_crtc_run(crtc, old_state);
+}
+
+static struct drm_crtc_state *spacemit_crtc_duplicate_state(struct drm_crtc *crtc)
+{
+	struct spacemit_crtc_state *state;
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u8 n_rdma, i;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+
+	if (WARN_ON(!crtc->state))
+		return NULL;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &state->base);
+	memset(&state->scalers, 0x0, sizeof(struct spacemit_crtc_scaler) * MAX_SCALER_NUMS);
+
+	n_rdma = hwdev->rdma_nums;
+	state->rdmas = kzalloc(sizeof(struct spacemit_crtc_rdma) * n_rdma, GFP_KERNEL);
+	if (!state->rdmas) {
+		kfree(state);
+		return NULL;
+	}
+
+	/* Rdma use UP_DOWN mode by default */
+	for (i = 0; i < n_rdma; i++) {
+		state->rdmas[i].mode = UP_DOWN;
+		state->rdmas[i].in_use = false;
+		if (a_crtc->is_offline_mode)
+			state->rdmas[i].is_offline_mode = true;
+	}
+
+	if (state->color_matrix_blob_prop)
+		drm_property_blob_get(state->color_matrix_blob_prop);
+
+	if (state->gamma_table_blob_prop)
+		drm_property_blob_get(state->gamma_table_blob_prop);
+
+	if (state->pp_color_temperature_blob_property)
+		drm_property_blob_get(state->pp_color_temperature_blob_property);
+
+	if (state->end_tone_mapping_blob_prop)
+		drm_property_blob_get(state->end_tone_mapping_blob_prop);
+
+	if (state->pp_acad_blob_prop)
+		drm_property_blob_get(state->pp_acad_blob_prop);
+
+	for (i = 0; i < MAX_CL_NUM; i++) {
+		state->cl[i].index = i;
+		state->cl[i].type = CMDLIST_CRTC;
+	}
+
+	return &state->base;
+}
+
+static void spacemit_crtc_destroy_state(struct drm_crtc *crtc,
+				struct drm_crtc_state *state)
+{
+	struct spacemit_crtc_state *spacemit_state = NULL;
+	struct spacemit_crtc *a_crtc = NULL;
+	int i = 0;
+
+	if (state) {
+		spacemit_state = to_spacemit_crtc_state(state);
+		__drm_atomic_helper_crtc_destroy_state(state);
+		a_crtc = to_spacemit_crtc(crtc);
+		if (spacemit_state->wb_mmu_tbl.va)
+			dma_free_coherent(a_crtc->dev, spacemit_state->wb_mmu_tbl.size,
+				spacemit_state->wb_mmu_tbl.va, spacemit_state->wb_mmu_tbl.pa);
+
+		for (i = 0; i < MAX_CL_NUM; i++) {
+			if (spacemit_state->cl[i].va)
+				dma_free_coherent(a_crtc->dev, spacemit_state->cl[i].size,
+					spacemit_state->cl[i].va, spacemit_state->cl[i].pa);
+		}
+		if (spacemit_state->color_matrix_blob_prop)
+			drm_property_blob_put(spacemit_state->color_matrix_blob_prop);
+
+		if (spacemit_state->gamma_table_blob_prop)
+			drm_property_blob_put(spacemit_state->gamma_table_blob_prop);
+
+		if (spacemit_state->pp_color_temperature_blob_property)
+			drm_property_blob_put(spacemit_state->pp_color_temperature_blob_property);
+
+		if (spacemit_state->pp_acad_blob_prop)
+			drm_property_blob_put(spacemit_state->pp_acad_blob_prop);
+
+		if (spacemit_state->end_tone_mapping_blob_prop)
+			drm_property_blob_put(spacemit_state->end_tone_mapping_blob_prop);
+
+		kfree(spacemit_state->rdmas);
+		kfree(spacemit_state);
+	}
+}
+
+static void spacemit_crtc_reset(struct drm_crtc *crtc)
+{
+	struct spacemit_crtc_state *state =
+		kzalloc(sizeof(*state), GFP_KERNEL);
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u8 n_rdma;
+
+	if (crtc->state)
+		spacemit_crtc_destroy_state(crtc, crtc->state);
+
+	__drm_atomic_helper_crtc_reset(crtc, &state->base);
+
+	n_rdma = hwdev->rdma_nums;
+	state->rdmas = kzalloc(sizeof(struct spacemit_crtc_rdma) * n_rdma, GFP_KERNEL);
+	if (!state->rdmas) {
+		DRM_ERROR("Failed to allocate memory of struct spacemit_crtc_rdma!\n");
+		return;
+	}
+
+}
+
+static int spacemit_crtc_enable_vblank(struct drm_crtc *crtc)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+
+	DRM_DEBUG("%s()\n", __func__);
+	trace_spacemit_crtc_enable_vblank(a_crtc->dev_id);
+
+	if (a_crtc->core && a_crtc->core->enable_vsync)
+		a_crtc->core->enable_vsync(a_crtc);
+
+	return 0;
+}
+
+static void spacemit_crtc_disable_vblank(struct drm_crtc *crtc)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+
+	DRM_DEBUG("%s()\n", __func__);
+	trace_spacemit_crtc_disable_vblank(a_crtc->dev_id);
+
+	if (a_crtc->core && a_crtc->core->disable_vsync)
+		a_crtc->core->disable_vsync(a_crtc);
+}
+
+static int spacemit_crtc_atomic_set_property(struct drm_crtc *crtc,
+				   struct drm_crtc_state *state,
+				   struct drm_property *property,
+				   uint64_t val)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct spacemit_crtc_state *s = to_spacemit_crtc_state(state);
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	bool replaced = false;
+	int ret = 0;
+
+	DRM_DEBUG("%s() name = %s, val = %llu\n",
+		  __func__, property->name, val);
+
+	if (property == a_crtc->color_matrix_property) {
+		ret = spacemit_atomic_replace_property_blob_from_id(crtc->dev,
+					&s->color_matrix_blob_prop,
+					val,
+					-1,
+					sizeof(int),
+					&replaced);
+		return ret;
+	} else if (property == a_crtc->expected_present_time) {
+		s->expected_present_time = val;
+		return 0;
+	} else if (property == a_crtc->offline_mode_property) {
+		a_crtc->is_offline_mode = val;
+		return 0;
+	} else if (property == a_crtc->pp_color_temperature_property){
+		ret = spacemit_atomic_replace_property_blob_from_id(crtc->dev,
+					&s->pp_color_temperature_blob_property,
+					val,
+					-1,
+					sizeof(int),
+					&replaced);
+		return ret;
+	} else if (property == a_crtc->wb_property) {
+		a_crtc->wb_pos = val;
+		return 0;
+	} else if (property == a_crtc->post_scaler_property) {
+		s->post_scl_on = val;
+		return 0;
+	} else if (property == a_crtc->gamma_table_property) {
+		ret = spacemit_atomic_replace_property_blob_from_id(crtc->dev,
+					&s->gamma_table_blob_prop,
+					val,
+					-1,
+					sizeof(uint16_t),
+					&replaced);
+		return ret;
+	} else if (property == a_crtc->ee_property) {
+		ret = spacemit_atomic_replace_property_blob_from_id(crtc->dev,
+					&s->ee_blob_prop,
+					val,
+					-1,
+					sizeof(uint32_t),
+					&replaced);
+		return ret;
+	} else if (property == a_crtc->acad_property) {
+		ret = spacemit_atomic_replace_property_blob_from_id(crtc->dev,
+					&s->pp_acad_blob_prop,
+					val,
+					-1,
+					sizeof(int),
+					&replaced);
+		return ret;
+	} else if (property == a_crtc->end_tone_mapping_property) {
+		ret = spacemit_atomic_replace_property_blob_from_id(crtc->dev,
+					&s->end_tone_mapping_blob_prop,
+					val,
+					-1,
+					sizeof(uint16_t),
+					&replaced);
+		return ret;
+	} else if (property == a_crtc->acad_status_property) {
+		hwdev->is_acad_on = val;
+		return 0;
+	} else if (property == a_crtc->bl_save_status_property) {
+		hwdev->is_bl_save_on = val;
+		return 0;
+	} else {
+		DRM_ERROR("property %s is invalid\n", property->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int spacemit_crtc_atomic_get_property(struct drm_crtc *crtc,
+					  const struct drm_crtc_state *state,
+					  struct drm_property *property,
+					  u64 *val)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct spacemit_crtc_state *s = to_spacemit_crtc_state(state);
+	struct spacemit_drm_private *priv = a_crtc->crtc.dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	DRM_DEBUG("%s() name = %s\n", __func__, property->name);
+
+	if (property == a_crtc->color_matrix_property) {
+		if (s->color_matrix_blob_prop)
+			*val = (s->color_matrix_blob_prop) ? s->color_matrix_blob_prop->base.id : 0;
+	} else if (property == a_crtc->hw_info_property) {
+		*val = a_crtc->hw_info_property->base.id;
+	} else if (property == a_crtc->offline_mode_property) {
+		*val = a_crtc->is_offline_mode;
+	} else if (property == a_crtc->gamma_table_property) {
+		*val = (s->gamma_table_blob_prop) ? s->gamma_table_blob_prop->base.id : 0;
+	} else if (property == a_crtc->ee_property) {
+		*val = (s->ee_blob_prop) ? s->ee_blob_prop->base.id : 0;
+	} else if (property == a_crtc->post_scaler_property) {
+		*val = s->post_scl_on;
+	} else if (property == a_crtc->acad_property) {
+		*val = (s->pp_acad_blob_prop) ? s->pp_acad_blob_prop->base.id : 0;
+	} else if (property == a_crtc->pp_color_temperature_property){
+		if (s->pp_color_temperature_blob_property)
+			*val = (s->pp_color_temperature_blob_property) ? s->pp_color_temperature_blob_property->base.id : 0;
+	} else if (property == a_crtc->end_tone_mapping_property) {
+		*val = (s->end_tone_mapping_blob_prop) ? s->end_tone_mapping_blob_prop->base.id : 0;
+	} else if (property == a_crtc->expected_present_time) {
+		*val = (a_crtc->expected_present_time) ? a_crtc->expected_present_time->base.id : 0;
+	} else if (property == a_crtc->wb_property) {
+		*val =  a_crtc->wb_pos;
+	} else if (property == a_crtc->acad_status_property) {
+		*val = hwdev->is_acad_on ? 1 : 0;
+	} else if (property == a_crtc->bl_save_status_property) {
+		*val = hwdev->is_bl_save_on ? 1 : 0;
+	} else {
+		DRM_ERROR("property %s is invalid\n", property->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct drm_crtc_helper_funcs spacemit_crtc_helper_funcs = {
+	.mode_set_nofb = spacemit_crtc_mode_set_nofb,
+	.atomic_check = spacemit_crtc_atomic_check,
+	.atomic_begin = spacemit_crtc_atomic_begin,
+	.atomic_flush = spacemit_crtc_atomic_flush,
+	.atomic_enable = spacemit_crtc_atomic_enable,
+	.atomic_disable = spacemit_crtc_atomic_disable,
+};
+
+static const struct drm_crtc_funcs spacemit_crtc_funcs = {
+	.atomic_get_property = spacemit_crtc_atomic_get_property,
+	.atomic_set_property = spacemit_crtc_atomic_set_property,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.reset = spacemit_crtc_reset,
+	.atomic_duplicate_state = spacemit_crtc_duplicate_state,
+	.atomic_destroy_state = spacemit_crtc_destroy_state,
+	.enable_vblank = spacemit_crtc_enable_vblank,
+	.disable_vblank = spacemit_crtc_disable_vblank,
+};
+
+static inline struct spacemit_hw_rdma *
+modifiers_ptr(struct drm_dpu_hw_version_blob *blob)
+{
+	return (struct spacemit_hw_rdma *)(((char *)blob) + blob->rdma_offset);
+}
+
+static int create_hwinfo_blob(struct drm_crtc *crtc)
+{
+	struct drm_property_blob *blob;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	struct drm_property *prop;
+	struct drm_dpu_hw_version_blob *blob_data;
+	struct spacemit_drm_private *priv = crtc->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct spacemit_hw_rdma *hw_rdma;
+	u8 rdma_nums = hwdev->rdma_nums;
+	size_t blob_size = 0;
+
+	prop = drm_property_create(crtc->dev,
+			DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_BLOB,
+			"hw_info", 0);
+	if (!prop)
+		return -ENOMEM;
+
+	blob_size = sizeof(struct drm_dpu_hw_version_blob);
+	blob_size += rdma_nums * sizeof(struct spacemit_crtc_rdma);
+
+	blob = drm_property_create_blob(crtc->dev, blob_size, NULL);
+	if (IS_ERR(blob))
+		return -1;
+
+	blob_data = blob->data;
+	blob_data->rdma_offset = sizeof(struct drm_dpu_hw_version_blob);
+	blob_data->rdma_nums = rdma_nums;
+	blob_data->scaler_num = hwdev->scaler_num;
+	blob_data->ver_scaler_num = hwdev->ver_scale_coef_size;
+	blob_data->hor_scaler_num = hwdev->hor_scale_coef_size;
+	blob_data->acad_num = hwdev->acad_num;
+	blob_data->dpu_version = hwdev->dpu_version;
+
+	hw_rdma = modifiers_ptr(blob_data);
+	memcpy(hw_rdma, hwdev->rdmas, sizeof(struct spacemit_hw_rdma) * rdma_nums);
+
+	drm_object_attach_property(&crtc->base, prop, blob->base.id);
+	a_crtc->hw_info_property = prop;
+
+	return 0;
+}
+
+static int spacemit_crtc_create_properties(struct drm_crtc *crtc)
+{
+	struct drm_property *prop;
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+	int ret = 0;
+
+	DRM_DEBUG("%s()\n", __func__);
+	/* create rotation property */
+
+	prop = drm_property_create(crtc->dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"color_matrix_coef", 0);
+	if (!prop) {
+		DRM_ERROR("create color_matrix_coef faild!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->color_matrix_property = prop;
+
+	prop = drm_property_create(crtc->dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"ee_coef", 0);
+
+	if (!prop) {
+		DRM_ERROR("create ee_property faild!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->ee_property = prop;
+
+	ret = create_hwinfo_blob(crtc);
+	if (ret)
+		DRM_ERROR("create_hwinfo_blob failed %d\n", ret);
+
+	prop = drm_property_create_range(crtc->dev, 0,
+			"expected_present_time", 0, ULLONG_MAX);
+	if (!prop) {
+		DRM_ERROR("expected_present_time failed %d\n", ret);
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->expected_present_time = prop;
+
+	prop = drm_property_create_bool(crtc->dev, DRM_MODE_PROP_ATOMIC,
+			"offline_mode");
+	if (!prop) {
+		DRM_ERROR("offline_mode failed %d\n", ret);
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->offline_mode_property = prop;
+
+	prop = drm_property_create_bool(crtc->dev, DRM_MODE_PROP_ATOMIC,
+			"pos_scl");
+	if (!prop) {
+		DRM_ERROR("pos_scl %d\n", ret);
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->post_scaler_property = prop;
+
+	prop = drm_property_create(crtc->dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"pp_cct", 0);
+	if (!prop) {
+		DRM_ERROR("create cct_prop failed!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->pp_color_temperature_property = prop;
+
+	prop = drm_property_create_range(crtc->dev, DRM_MODE_PROP_RANGE,
+			"WB_POS", SPACEMIT_WB_RDMA0, SPACEMIT_WB_POST2);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&crtc->base, prop, SPACEMIT_WB_COMP0);
+	a_crtc->wb_property = prop;
+	prop = drm_property_create(crtc->dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"pp_gamma", 0);
+
+	if (!prop) {
+		DRM_ERROR("create gamma_table_prop faild!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->gamma_table_property = prop;
+
+	prop = drm_property_create(crtc->dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"acad_coef", 0);
+
+	if (!prop) {
+		DRM_ERROR("create acad_property faild!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->acad_property = prop;
+
+	prop = drm_property_create(crtc->dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"pp_etm", 0);
+	if (!prop) {
+		DRM_ERROR("create end_tone_mapping_prop faild!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->end_tone_mapping_property = prop;
+
+	prop = drm_property_create_bool(crtc->dev, DRM_MODE_PROP_ATOMIC,
+			"acad_status");
+	if (!prop) {
+		DRM_ERROR("create acad_status failed!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->acad_status_property = prop;
+
+	prop = drm_property_create_bool(crtc->dev, DRM_MODE_PROP_ATOMIC,
+			"bl_save_status");
+	if (!prop) {
+		DRM_ERROR("create bl_save_status failed!");
+		return -ENOMEM;
+	}
+	drm_object_attach_property(&crtc->base, prop, 0);
+	a_crtc->bl_save_status_property = prop;
+	return 0;
+}
+
+static int spacemit_crtc_bind_init(struct drm_device *drm, struct drm_crtc *crtc,
+			 struct drm_plane *primary, struct device_node *port)
+{
+	int err;
+
+	/*
+	 * set crtc port so that drm_of_find_possible_crtcs call works
+	 */
+	of_node_put(port);
+	crtc->port = port;
+
+	err = drm_crtc_init_with_planes(drm, crtc, primary, NULL,
+					&spacemit_crtc_funcs, NULL);
+	if (err) {
+		DRM_ERROR("failed to init crtc.\n");
+		return err;
+	}
+
+	drm_mode_crtc_set_gamma_size(crtc, 256);
+
+	drm_crtc_helper_add(crtc, &spacemit_crtc_helper_funcs);
+
+	spacemit_crtc_create_properties(crtc);
+
+	DRM_INFO("%s() ok\n", __func__);
+	return 0;
+}
+
+int spacemit_crtc_run(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_state)
+{
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(crtc);
+
+	DRM_DEBUG("%s()\n", __func__);
+	trace_spacemit_crtc_run(a_crtc->dev_id);
+
+	if (a_crtc->core && a_crtc->core->run)
+		a_crtc->core->run(crtc, old_state);
+
+	return 0;
+}
+
+int spacemit_dpu_esd_restart(struct spacemit_crtc *a_crtc)
+{
+	if (a_crtc->core && a_crtc->core->esd_restart)
+		a_crtc->core->esd_restart(a_crtc);
+	return 0;
+}
+EXPORT_SYMBOL(spacemit_dpu_esd_restart);
+
+int spacemit_crtc_stop(struct spacemit_crtc *a_crtc)
+{
+	trace_spacemit_crtc_stop(a_crtc->dev_id);
+
+	if (a_crtc->core && a_crtc->core->stop)
+		a_crtc->core->stop(a_crtc);
+
+	drm_crtc_handle_vblank(&a_crtc->crtc);
+
+	return 0;
+}
+EXPORT_SYMBOL(spacemit_crtc_stop);
+
+static int spacemit_crtc_init(struct spacemit_crtc *a_crtc)
+{
+	trace_spacemit_crtc_init(a_crtc->dev_id);
+#ifdef CONFIG_PM
+#if IS_ENABLED(CONFIG_SPACEMIT_DDR_FC)
+	update_spacemit_ddr_bw_read_req(a_crtc->ddr_qos_cons, DPU_QOS_REQ);
+#endif
+#endif
+	if (a_crtc->core && a_crtc->core->init)
+		a_crtc->core->init(a_crtc);
+
+	a_crtc->is_1st_f = true;
+
+	return 0;
+}
+
+static int spacemit_crtc_uninit(struct spacemit_crtc *a_crtc)
+{
+	trace_spacemit_crtc_uninit(a_crtc->dev_id);
+
+	if (a_crtc->core && a_crtc->core->uninit)
+		a_crtc->core->uninit(a_crtc);
+#ifdef CONFIG_PM
+#if IS_ENABLED(CONFIG_SPACEMIT_DDR_FC)
+	update_spacemit_ddr_bw_read_req(a_crtc->ddr_qos_cons, PM_QOS_CPUIDLE_BLOCK_DEFAULT_VALUE);
+#endif
+#endif
+	return 0;
+}
+
+static irqreturn_t spacemit_dpu_isr(int irq, void *data)
+{
+	struct spacemit_crtc *a_crtc = data;
+
+	if (a_crtc->core) {
+		if (a_crtc->is_offline_mode == 0) {
+			if (a_crtc->core->online_isr)
+				a_crtc->core->online_isr(a_crtc);
+		} else {
+			if (a_crtc->core->offline_isr)
+				a_crtc->core->offline_isr(a_crtc);
+		}
+
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int spacemit_dpu_irqs_init(struct spacemit_crtc *a_crtc,
+				struct device_node *np, struct platform_device *pdev)
+{
+	int err;
+
+	int irq_online, irq_offline;
+
+	DRM_INFO("%s()\n", __func__);
+	/*request irq*/
+	irq_online = platform_get_irq_byname(pdev, "ONLINE_IRQ");
+	irq_offline = platform_get_irq_byname(pdev, "OFFLINE_IRQ");
+	if ((irq_online < 0) && (irq_offline < 0)) {
+		DRM_ERROR("failed to get ONLINE irq number %d\n", irq_online);
+		DRM_ERROR("failed to get OFFLINE irq number %d\n", irq_offline);
+		// return -EINVAL;
+	}
+	DRM_DEBUG("dpu online_irq = %d\n", irq_online);
+	DRM_DEBUG("dpu offline_irq = %d\n", irq_offline);
+
+	if (irq_online > 0) {
+		err = request_irq(irq_online, spacemit_dpu_isr, 0, "DPU_ONLINE", a_crtc);
+		if (err) {
+			DRM_ERROR("error: dpu request online irq failed\n");
+			return -EINVAL;
+		}
+	}
+
+	if (irq_offline > 0) {
+		err = request_irq(irq_offline, spacemit_dpu_isr, 0, "DPU_OFFLINE", a_crtc);
+		if (err) {
+			DRM_ERROR("error: dpu request offline irq failed\n");
+			free_irq(irq_online, a_crtc);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static void dpu_wq_update_bw(struct work_struct *work)
+{
+	struct spacemit_crtc *a_crtc =
+		container_of(work, struct spacemit_crtc, work_update_bw);
+
+	if (a_crtc->core && a_crtc->core->update_bw) {
+		trace_u64_data("bw decrease to", a_crtc->new_bw);
+		a_crtc->core->update_bw(a_crtc, a_crtc->new_bw);
+	}
+}
+
+static void dpu_wq_update_clk(struct work_struct *work)
+{
+	struct spacemit_crtc *a_crtc =
+		container_of(work, struct spacemit_crtc, work_update_clk);
+
+	if (a_crtc->core && a_crtc->core->update_clk) {
+		trace_u64_data("mclk decrease to", a_crtc->new_mclk);
+		a_crtc->core->update_clk(a_crtc, a_crtc->new_mclk);
+	}
+}
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+static bool check_dpu_running_status(struct spacemit_crtc *a_crtc)
+{
+	return a_crtc->is_working;
+}
+
+#define to_dpuinfo(_nb) container_of(_nb, struct spacemit_crtc, nb)
+static int dpu_clkoffdet_notifier_handler(struct notifier_block *nb,
+			    unsigned long msg, void *data)
+{
+	struct clk_notifier_data *cnd = data;
+	struct spacemit_crtc *a_crtc = to_dpuinfo(nb);
+
+	if ((__clk_is_enabled(cnd->clk)) && (msg & PRE_RATE_CHANGE) && (cnd->new_rate == 0) && (cnd->old_rate != 0)) {
+		if (a_crtc->is_dpu_running(a_crtc))
+			return NOTIFY_BAD;
+	}
+
+	return NOTIFY_OK;
+}
+#endif
+#define max_mclk_nb_to_dpuinfo(_nb) container_of(_nb, struct spacemit_crtc, max_mclk_nb)
+static int dpu_max_mclk_notifier_hander(struct notifier_block *nb,
+		unsigned long fix_mclk, void *data)
+{
+	struct spacemit_crtc *a_crtc = max_mclk_nb_to_dpuinfo(nb);
+	struct drm_device *drm = a_crtc->crtc.dev;
+	char *cam_on_event[] = { "CAMERA=1", NULL };
+	char *cam_off_event[] = { "CAMERA=0", NULL };
+
+
+
+	if (fix_mclk) {
+		a_crtc->fix_max_mclk = true;
+		if (a_crtc->cur_mclk < a_crtc->max_mclk && a_crtc->core && a_crtc->core->update_clk) {
+			cancel_work_sync(&a_crtc->work_update_clk);
+			trace_u64_data("mclk increase to", a_crtc->max_mclk);
+			a_crtc->core->update_clk(a_crtc, a_crtc->max_mclk);
+			kobject_uevent_env(&drm->primary->kdev->kobj, KOBJ_CHANGE, cam_on_event);
+
+		}
+	} else {
+		a_crtc->fix_max_mclk = false;
+		kobject_uevent_env(&drm->primary->kdev->kobj, KOBJ_CHANGE, cam_off_event);
+	}
+
+	return NOTIFY_OK;
+}
+
+static void dpu_parse_dsi_ops(struct spacemit_crtc *a_crtc)
+{
+	struct platform_device *dsi_pdev;
+
+	if (!a_crtc->dsi_node)
+		return;
+
+	dsi_pdev = of_find_device_by_node(a_crtc->dsi_node);
+	if (!dsi_pdev) {
+		DRM_ERROR("Failed to find dsi platform device\n");
+		goto node_put;
+	}
+
+	a_crtc->dsi = platform_get_drvdata(dsi_pdev);
+	if (!a_crtc->dsi) {
+		DRM_ERROR("Failed to get dsi drvdata\n");
+		goto node_put;
+	}
+
+node_put:
+	of_node_put(a_crtc->dsi_node);
+}
+
+static int spacemit_dpu_bind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm_dev = data;
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+	struct device_node *np = dev->of_node;
+	struct device_node *ports, *port;
+	struct drm_plane *plane;
+	int ret;
+#ifdef CONFIG_SPACEMIT_DEBUG
+	struct dpu_clk_context *clk_ctx = NULL;
+#endif
+	DRM_INFO("%s()\n", __func__);
+	if (a_crtc->is_offline_mode){
+		dpu_pm_suspend(a_crtc->dev);
+		spacemit_dpu_power_enable(a_crtc, false);
+	} else if (!a_crtc->is_edp) {
+		dpu_parse_dsi_ops(a_crtc);
+	}
+
+	ret = spacemit_dpu_irqs_init(a_crtc, np, pdev);
+	if (ret)
+		return ret;
+
+	ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		DRM_ERROR("dma_set_mask_and_coherent failed (%d)\n", ret);
+		return ret;
+	}
+
+	ret = of_reserved_mem_device_init(a_crtc->dev);
+	if (ret) {
+		DRM_ERROR("Failed to reserve dpu memory, ret:%d\n", ret);
+		return ret;
+	}
+
+#if IS_ENABLED(CONFIG_SPACEMIT_DDR_FC) && defined(CONFIG_PM)
+	a_crtc->ddr_qos_cons = register_spacemit_ddr_bw_cons("DPU", PM_QOS_CPUIDLE_BLOCK_DEFAULT_VALUE, PM_QOS_CPUIDLE_BLOCK_DEFAULT_VALUE);
+#endif
+
+	a_crtc->dpu_trace_wq = create_singlethread_workqueue("dpu_trace_wq");
+	if (!a_crtc->dpu_trace_wq) {
+		DRM_ERROR("%s: failed to create wq.\n", __func__);
+		ret = -ESRCH;
+		goto alloc_fail;
+	}
+
+	timer_setup(&a_crtc->cfg_rdy_timer, spacemit_cfg_rdy_timer_handler, 0);
+	INIT_WORK(&a_crtc->work_dpu_trace, dpu_trace);
+	INIT_WORK(&a_crtc->work_update_clk, dpu_wq_update_clk);
+	INIT_WORK(&a_crtc->work_update_bw, dpu_wq_update_bw);
+
+	plane = spacemit_plane_init(drm_dev, a_crtc);
+	if (IS_ERR_OR_NULL(plane)) {
+		ret = PTR_ERR(plane);
+		goto err_destroy_workqueue;
+	}
+
+	ports = of_get_child_by_name(np, "ports");
+	if (!ports) {
+		DRM_ERROR("CRTC %pOF has no ports node\n", np);
+		ret = -EINVAL;
+		goto err_destroy_workqueue;
+	}
+
+	port = of_get_child_by_name(ports, "port");
+	if (!port) {
+		DRM_ERROR("CRTC %pOF has no port@X node\n", np);
+		of_node_put(ports);
+		ret = -EINVAL;
+		goto err_destroy_workqueue;
+	}
+
+	ret = spacemit_crtc_bind_init(drm_dev, &a_crtc->crtc, plane, port);
+	if (ret)
+		goto err_destroy_workqueue;
+
+	spacemit_dpu_sysfs_init(dev);
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+	clk_ctx = &a_crtc->clk_ctx;
+	a_crtc->is_dpu_running = check_dpu_running_status;
+	a_crtc->nb.notifier_call = dpu_clkoffdet_notifier_handler;
+	clk_notifier_register(clk_ctx->mclk, &a_crtc->nb);
+#endif
+	a_crtc->max_mclk_nb.notifier_call = dpu_max_mclk_notifier_hander;
+	dpu_max_mclk_notifier_register(&a_crtc->max_mclk_nb);
+
+	DRM_INFO("dpu driver probe success\n");
+
+	return 0;
+
+err_destroy_workqueue:
+	destroy_workqueue(a_crtc->dpu_trace_wq);
+alloc_fail:
+	of_reserved_mem_device_release(a_crtc->dev);
+	return ret;
+}
+
+static void spacemit_dpu_unbind(struct device *dev, struct device *master,
+	void *data)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+#ifdef CONFIG_SPACEMIT_DEBUG
+	struct dpu_clk_context *clk_ctx = &a_crtc->clk_ctx;
+#endif
+
+	DRM_INFO("%s()\n", __func__);
+
+#if IS_ENABLED(CONFIG_SPACEMIT_DDR_FC) && defined(CONFIG_PM)
+	rm_spacemit_ddr_bw_cons(a_crtc->ddr_qos_cons);
+#endif
+	pm_runtime_disable(dev);
+	of_reserved_mem_device_release(a_crtc->dev);
+	drm_crtc_cleanup(&a_crtc->crtc);
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+	clk_notifier_unregister(clk_ctx->mclk, &a_crtc->nb);
+#endif
+
+}
+
+static const struct component_ops dpu_component_ops = {
+	.bind = spacemit_dpu_bind,
+	.unbind = spacemit_dpu_unbind,
+};
+
+static int spacemit_dpu_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct spacemit_crtc *a_crtc;
+	struct device_node *np = dev->of_node;
+	const char *str;
+	u32 dpu_id;
+	u32 pipeline_id;
+	u32 is_edp;
+	u32 dpu_out_format;
+	DRM_INFO("%s()\n", __func__);
+	if (!dev->of_node) {
+		DRM_DEV_ERROR(dev, "can't find dpu devices\n");
+		return -ENODEV;
+	}
+
+	a_crtc = devm_kzalloc(dev, sizeof(*a_crtc), GFP_KERNEL);
+	if (!a_crtc)
+		return -ENOMEM;
+	a_crtc->dev = dev;
+	a_crtc->power_on = false;
+	a_crtc->logo_booton = true;
+	a_crtc->rpm_status = false;
+	dev_set_drvdata(dev, a_crtc);
+
+	a_crtc->mclk_reset = devm_reset_control_get_optional_shared(&pdev->dev, "mclk_reset");
+	if (IS_ERR_OR_NULL(a_crtc->mclk_reset))
+		DRM_DEV_DEBUG(dev, "not found mclk_reset\n");
+	a_crtc->esc_reset = devm_reset_control_get_optional_shared(&pdev->dev, "esc_reset");
+	if (IS_ERR_OR_NULL(a_crtc->esc_reset))
+		DRM_DEV_DEBUG(dev, "not found esc_reset\n");
+	a_crtc->lcd_reset = devm_reset_control_get_optional_shared(&pdev->dev, "lcd_reset");
+	if (IS_ERR_OR_NULL(a_crtc->lcd_reset))
+		DRM_DEV_DEBUG(dev, "not found lcd_reset\n");
+	a_crtc->aclk_reset = devm_reset_control_get_optional_shared(&pdev->dev, "aclk_reset");
+	if (IS_ERR_OR_NULL(a_crtc->aclk_reset))
+		DRM_DEV_DEBUG(dev, "not found aclk_reset\n");
+	a_crtc->dsc_reset = devm_reset_control_get_optional_shared(&pdev->dev, "dsc_reset");
+	if (IS_ERR_OR_NULL(a_crtc->dsc_reset))
+		DRM_DEV_DEBUG(dev, "not found dsc_reset\n");
+
+	if (of_property_read_u32(np, "pipeline-id", &pipeline_id))
+		return -EINVAL;
+	a_crtc->dev_id = pipeline_id;
+
+	if (of_property_read_u32(np, "dpu-id", &dpu_id)) {
+		DRM_INFO("%s() dpu id was not found\n", __func__);
+		a_crtc->dpu_id = 0;
+	} else
+		a_crtc->dpu_id = dpu_id;
+
+	if (of_property_read_u32(np, "is_edp", &is_edp))
+		return -EINVAL;
+	a_crtc->is_edp = is_edp ? true : false;
+
+	if (of_property_read_u32(np, "out-format", &dpu_out_format))
+		dpu_out_format = OUTFMT_RGB888;
+	a_crtc->out_format = dpu_out_format;
+
+	if (!of_property_read_string(np, "ip", &str))
+		a_crtc->core = dpu_core_ops_attach(str);
+	else
+		DRM_WARN("ip was not found\n");
+
+	/* Clk dts nodes must be parsed in head of pm_runtime_xxx */
+	if (a_crtc->core && a_crtc->core->parse_dt)
+		a_crtc->core->parse_dt(a_crtc, np);
+
+	spacemit_dpu_setup_bootloader_mem(dev);
+
+	return component_add(dev, &dpu_component_ops);
+}
+
+static void spacemit_dpu_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &dpu_component_ops);
+}
+
+static int __maybe_unused dpu_pm_suspend(struct device *dev)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+	int result;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	if (a_crtc->core && a_crtc->core->disable_clk)
+		a_crtc->core->disable_clk(a_crtc);
+
+	if (!IS_ERR_OR_NULL(a_crtc->lcd_reset)) {
+		result = reset_control_assert(a_crtc->lcd_reset);
+		if (result < 0)
+			DRM_INFO("Failed to assert lcd_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->esc_reset)) {
+		result = reset_control_assert(a_crtc->esc_reset);
+		if (result < 0)
+			DRM_INFO("Failed to assert esc_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->mclk_reset)) {
+		result = reset_control_assert(a_crtc->mclk_reset);
+		if (result < 0)
+			DRM_INFO("Failed to assert mclk_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->aclk_reset)) {
+		result = reset_control_assert(a_crtc->aclk_reset);
+		if (result < 0)
+			DRM_INFO("Failed to assert aclk_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->dsc_reset)) {
+			result = reset_control_assert(a_crtc->dsc_reset);
+			if (result < 0)
+				DRM_INFO("Failed to assert dsc_reset: %d\n", result);
+	}
+
+	return 0;
+}
+
+static int __maybe_unused dpu_pm_resume(struct device *dev)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+	int result;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	if (!IS_ERR_OR_NULL(a_crtc->mclk_reset)) {
+		result = reset_control_deassert(a_crtc->mclk_reset);
+		if (result < 0)
+			DRM_INFO("Failed to deassert mclk_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->esc_reset)) {
+		result = reset_control_deassert(a_crtc->esc_reset);
+		if (result < 0)
+			DRM_INFO("Failed to deassert esc_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->lcd_reset)) {
+		result = reset_control_deassert(a_crtc->lcd_reset);
+		if (result < 0)
+			DRM_INFO("Failed to deassert lcd_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->aclk_reset)) {
+		result = reset_control_deassert(a_crtc->aclk_reset);
+		if (result < 0)
+			DRM_INFO("Failed to deassert aclk_reset: %d\n", result);
+	}
+	if (!IS_ERR_OR_NULL(a_crtc->dsc_reset)) {
+			result = reset_control_deassert(a_crtc->dsc_reset);
+			if (result < 0)
+				DRM_INFO("Failed to deassert dsc_reset: %d\n", result);
+	}
+
+	if (a_crtc->core && a_crtc->core->enable_clk)
+		a_crtc->core->enable_clk(a_crtc);
+
+	return 0;
+}
+
+static int __maybe_unused dpu_rt_pm_suspend(struct device *dev)
+{
+	// struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+
+	DRM_DEBUG("%s() \n", __func__);
+
+	return 0;
+}
+
+static int __maybe_unused dpu_rt_pm_resume(struct device *dev)
+{
+	// struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+
+	DRM_DEBUG("%s() \n", __func__);
+
+	return 0;
+}
+
+static const struct dev_pm_ops dpu_pm_ops = {
+	SET_RUNTIME_PM_OPS(dpu_rt_pm_suspend,
+			dpu_rt_pm_resume,
+			NULL)
+};
+
+static const struct of_device_id dpu_match_table[] = {
+	{ .compatible = "spacemit,dpu-saturn" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dpu_match_table);
+
+struct platform_driver spacemit_dpu_driver = {
+	.probe = spacemit_dpu_probe,
+	.remove = spacemit_dpu_remove,
+	.driver = {
+		.name = "spacemit-dpu-drv",
+		.of_match_table = dpu_match_table,
+		.pm = &dpu_pm_ops,
+	},
+};
+
+MODULE_DESCRIPTION("SPACEMIT Display Controller Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/spacemit_crtc.h b/drivers/gpu/drm/spacemit/spacemit_crtc.h
new file mode 100644
index 000000000000..9e8ca8a7af28
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_crtc.h
@@ -0,0 +1,495 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_CRTC_H_
+#define _SPACEMIT_CRTC_H_
+
+#include <linux/delay.h>
+#include <linux/string.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/bug.h>
+#include <video/videomode.h>
+#include <linux/workqueue.h>
+
+#include <drm/drm_atomic_uapi.h>
+#include <drm/drm_print.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_vblank.h>
+#include <drm/drm_writeback.h>
+#include <dt-bindings/display/spacemit_dpu.h>
+#include "dpu/saturn_regs/reg_map.h"
+#include "spacemit_cmdlist.h"
+#include "spacemit_drm.h"
+#include "spacemit_lib.h"
+#include "../drm_crtc_internal.h"
+#include <linux/spinlock.h>
+#include "spacemit_dsi.h"
+
+#define SPACEMIT_DPU_LPM_PERIOD_MIN_MS	(50)
+#define SPACEMIT_DPU_LPM_PERIOD_MAX_MS	(5000)
+
+#define DMA_TOP_ARB_DEBUG_INFO0_DFT	0x20003000
+#define CMDLIST_CH_DBG_CH_CS_STS	0x11
+#define CMDLIST_CH_DBG_CH_CS_MASK	0xff
+#define CMDLIST_CH_DBG_SIZE		0x7
+
+#define N_SCALER_MAX			5
+#define N_DMA_CHANNEL_MAX		12
+#define N_DMA_LAYER_MAX			16
+#define N_COMPOSER_MAX			4
+#define N_COMPOSER_LAYER_MAX		16
+#define N_PANEL_MAX			3
+#define N_OUTCTRL_MAX			3
+#define N_WRITEBACK_MAX			2
+#define N_DISPLAY_MAX			5
+#define N_LUT3D_MAX			3
+#define N_CMDLIST_MAX			14
+
+#define MHZ2HZ	1000000ULL
+#define MHZ2KHZ	1000ULL
+#define DPU_QOS_REQ			1100000
+#define DPU_MIN_QOS_REQ			331070976
+#define DPU_MAX_QOS_REQ			(3900000ULL)
+#define RDMA_INVALID_ID			(~0)
+#define SCALER_INVALID_ID		((u8)(~0))
+
+#define DPU_STOP_TIMEOUT		(2000)
+#define DPU_STOP_REBOOT_TIMEOUT		(150000)
+#define DPU_CTRL_MAX_TIMING_INTER1	(0xf)
+
+#define DPU_DSCCLK_DEFAULT	307200000
+#define DPU_PXCLK_DEFAULT	98000000
+#define DPU_MCLK_DEFAULT	307200000
+#define DPU_MCLK_MIN		40960000
+#define DPU_AXICLK_DEFAULT	409000000
+#define DPU_ESCCLK_DEFAULT	52000000
+#define DPU_BITCLK_DEFAULT	624000000
+#define DPU_DSIPLL_REG0_DEFAULT	0x8010c563
+#define DPU_DSIPLL_REG1_DEFAULT	0x0bcec4ec
+#define DPU_DSIPLL_REG2_DEFAULT	0x800078a0
+
+#define CMDLIST_ADDRL_ALIGN_BITS		(4) //From cmdlist_reg_0[] in CMDLIST_REG
+#define CMDLIST_ADDRL_ALIGN_MASK		((u32)(~(BIT(CMDLIST_ADDRL_ALIGN_BITS) - 1)))
+
+#define MAX_SCALER_NUMS		4
+struct spacemit_crtc_scaler {
+	u32 rdma_id;
+	u32 in_use;
+};
+
+enum spacemit_dpu_irq {
+	INT_UNDERRUN,
+	INT_CFG_RDY,
+	INT_VSYNC,
+	INT_EOF,
+	INT_WB_DONE,
+	INT_REST,
+	OFF_CFG_RDY,
+	OFF_WB_DONE,
+	OFF_REST,
+	INT_VSYNC_UPDATE,
+	INT_DEBUG,
+};
+
+enum rdma_mode {
+	UP_DOWN = BIT(0),
+	LEFT_RIGHT = BIT(1),
+};
+
+struct spacemit_crtc_fbcmem {
+	u32 start;
+	u32 size;
+	bool map;
+};
+
+struct spacemit_crtc_rdma {
+	enum rdma_mode mode;
+	struct spacemit_crtc_fbcmem fbcmem;
+	bool in_use;
+	bool is_offline_mode;
+	u32 use_cnt;
+};
+
+struct dpu_mmu_tbl {
+	u32 size;
+	void *va;
+	dma_addr_t pa;
+};
+
+struct dpu_clk_context {
+	struct clk *pxclk;
+	struct clk *mclk;
+	struct clk *hclk;
+	struct clk *escclk;
+	struct clk *bitclk;
+	struct clk *aclk;
+	struct clk *dscclk;
+};
+
+#define DEFAULT_SLICE_WIDTH	512
+#define MAX_WIDTH		4096
+#define MAX_SLICE	(MAX_WIDTH / DEFAULT_SLICE_WIDTH)
+#define DPU_NO_SLICE (0xFF)
+struct slice_cfg {
+	u32 wb_in_w;
+	u32 wb_in_h;
+	u32 wb_out_x;
+	u32 wb_out_y;
+	u32 wb_out_w;
+	u32 wb_out_h;
+	u32 rdma_w;
+	u32 rdma_h;
+	u32 rdma_x;
+	u32 rdma_y;
+	u32 crtc_w;
+	u32 crtc_h;
+	u32 crtc_x;
+	u32 crtc_y;
+};
+
+struct spacemit_crtc {
+	spinlock_t irq_msk_lock;
+	struct device *dev;
+	struct drm_crtc crtc;
+	struct dpu_core_ops *core;
+	struct workqueue_struct *dpu_trace_wq;
+	struct work_struct work_dpu_trace;
+	struct work_struct work_update_clk;
+	struct work_struct work_update_bw;
+	struct dpu_mmu_tbl mmu_tbl;
+#ifdef CONFIG_PM
+#if IS_ENABLED(CONFIG_SPACEMIT_DDR_FC)
+	struct spacemit_bw_con *ddr_qos_cons;
+#endif
+	s32 lpm_commit_qos;
+	u32 lpm_period;
+	struct delayed_work lpm_qos_work;
+	bool lpm_work_pending;
+	s32 lpm_bl_qos;
+#endif
+	int dev_id;
+	bool is_edp;
+	int dpu_id;
+	struct timer_list cfg_rdy_timer;
+	int wb_id;
+	struct slice_cfg slice_wb[MAX_SLICE];
+	bool enable_dump_reg;
+	bool enable_dump_fps;
+	bool enable_auto_fc;
+	struct timespec64 last_tm;
+
+	bool is_1st_f;
+	bool logo_booton;
+	uint32_t vrr_vfp;
+	struct dpu_clk_context clk_ctx;
+	uint64_t new_mclk;		/* new frame mclk */
+	uint64_t cur_mclk;		/* current frame mclk */
+	unsigned int min_mclk;		/* min_mclk of board panel resolution */
+	unsigned int max_mclk;		/* max_mclk of board panel resolution */
+	uint64_t bw_margin;		/* bandwidth margin */
+	uint64_t max_bw;		/* max bandwidth */
+	bool fix_max_mclk;
+	uint64_t new_bw;
+	uint64_t cur_bw;
+	struct drm_property *color_matrix_property;
+	struct drm_property *gamma_table_property;
+	struct drm_property *end_tone_mapping_property;
+	struct drm_property *acad_property;
+	struct drm_property *ee_property;
+
+	struct drm_property *hw_info_property;
+	struct drm_property *wb_property;
+	uint32_t bitclk;
+	uint32_t dsipll_reg0;
+	uint32_t dsipll_reg1;
+	uint32_t dsipll_reg2;
+	void __iomem *dsipll_base;
+	void __iomem *dsi1pll_base;
+	bool dsipll_valid;
+	uint32_t aclk;
+	uint32_t escclk;
+	uint32_t out_format;
+	uint32_t out_mode; /* cmd or video */
+	struct notifier_block max_mclk_nb;
+	uint32_t stop_to;
+	bool is_stopped;		/* skip dpu key regs when dpu_stopped */
+	uint32_t outctrl_reg;
+	uint32_t dpu_online_nml_rch_en;
+	uint32_t dpu_online_nml_scl_en;
+	uint32_t dpu_online_nml_outctl_en;
+	uint32_t dpuctrl_ctl_nml_cmdlist_rch_en[N_CMDLIST_MAX];
+
+	struct reset_control *mclk_reset;
+	struct reset_control *lcd_reset;
+	struct reset_control *esc_reset;
+	struct reset_control *aclk_reset;
+	struct reset_control *dsc_reset;
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+	bool (*is_dpu_running)(struct spacemit_crtc *a_crtc);
+	struct notifier_block nb;
+	bool is_working;
+#endif
+    // Added for Android14(HWC3.x)
+	struct drm_property *expected_present_time;
+	/* add for cmdlist_v1 (led) */
+	struct cmdlist_regs *cl_rdma;
+	struct cmdlist_regs *cl_tbu;
+	struct cmdlist_regs *cl_pq;
+
+	unsigned int wb_pos;
+	unsigned int wb_format;
+	unsigned int is_offline_mode;
+	unsigned int is_slice_mode;
+	unsigned int slice_num;
+	unsigned int split_en;
+	struct drm_property *offline_mode_property;
+	struct drm_property *post_scaler_property;
+	struct drm_property *pp_color_temperature_property;
+	struct drm_property *acad_status_property;
+	struct drm_property *bl_save_status_property;
+
+	/* dsc parameters */
+	unsigned int dsc_pxclk;
+	unsigned int dsc_clk;
+	unsigned int dsc_x_slice;
+	u32 *dsc_regs;
+	int dsc_regs_len;
+
+	u32 dither_mode;
+
+	bool flip_done;
+	bool is_oled;
+
+	bool rpm_status;
+	bool power_on;
+	struct device_node *panel_node;
+	struct device_node *dsi_node;
+	struct spacemit_dsi *dsi;
+};
+
+extern struct list_head dpu_core_head;
+
+static inline struct spacemit_crtc *to_spacemit_crtc(struct drm_crtc *crtc)
+{
+	return crtc ? container_of(crtc, struct spacemit_crtc, crtc) : NULL;
+}
+
+struct spacemit_hw_device;
+struct dpu_core_ops {
+	int (*parse_dt)(struct spacemit_crtc *a_crtc, struct device_node *np);
+	u32 (*version)(struct spacemit_crtc *a_crtc);
+	int (*init)(struct spacemit_crtc *a_crtc);
+	void (*uninit)(struct spacemit_crtc *a_crtc);
+	void (*run)(struct drm_crtc *crtc,
+		    struct drm_crtc_state *old_state);
+	void (*stop)(struct spacemit_crtc *a_crtc);
+	void (*esd_restart)(struct spacemit_crtc *a_crtc);
+	void (*flip)(struct spacemit_crtc *a_crtc);
+	void (*disable_vsync)(struct spacemit_crtc *a_crtc);
+	void (*enable_vsync)(struct spacemit_crtc *a_crtc);
+	u32 (*online_isr)(struct spacemit_crtc *a_crtc);
+	u32 (*offline_isr)(struct spacemit_crtc *a_crtc);
+	int (*modeset)(struct spacemit_crtc *a_crtc, struct drm_mode_modeinfo *mode);
+	int (*enable_clk)(struct spacemit_crtc *a_crtc);
+	int (*disable_clk)(struct spacemit_crtc *a_crtc);
+	int (*cal_layer_fbcmem_size)(struct drm_plane *plane,
+				     struct drm_plane_state *state);
+	int (*adjust_rdma_fbcmem)(struct spacemit_hw_device *hwdev,
+				 struct spacemit_crtc_rdma *rdmas);
+	int (*calc_plane_mclk_bw)(struct drm_plane *plane,
+			struct drm_plane_state *state);
+	int (*update_clk)(struct spacemit_crtc *a_crtc, uint64_t mclk);
+	int (*update_bw)(struct spacemit_crtc *a_crtc, uint64_t bw);
+	void (*begin)(struct spacemit_crtc *a_crtc);
+};
+
+#define dpu_core_ops_register(entry) \
+	disp_ops_register(entry, &dpu_core_head)
+#define dpu_core_ops_attach(str) \
+	disp_ops_attach(str, &dpu_core_head)
+
+int spacemit_crtc_run(struct drm_crtc *crtc,
+		struct drm_crtc_state *old_state);
+int spacemit_crtc_stop(struct spacemit_crtc *a_crtc);
+int spacemit_dpu_esd_restart(struct spacemit_crtc *a_crtc);
+bool dpu_mclk_exclusive_get(void);
+void dpu_mclk_exclusive_put(void);
+int dpu_max_mclk_notifier_register(struct notifier_block *nb);
+int dpu_max_mclk_notifier_unregister(struct notifier_block *nb);
+int dpu_max_mclk_notifier_call_chain(unsigned int mclk_rate);
+void spacemit_crtc_fill_slice(struct spacemit_crtc *a_crtc, u32 width, u32 height);
+int spacemit_crtc_calc_slices(struct spacemit_crtc *a_crtc, u32 width, u32 height);
+
+struct spacemit_plane {
+	struct drm_plane plane;
+	struct spacemit_hw_device *hwdev;
+	struct drm_property *rdma_id_property;
+	struct drm_property *solid_color_property;
+	struct drm_property *dec_lines_property;
+	struct drm_property *hdr_coef_property;
+	struct drm_property *scale_coef_property;
+	u32 hw_pid;
+};
+
+#define MAX_CL_NUM	MAX_SLICE
+
+struct spacemit_plane_state {
+	struct drm_plane_state state;
+	u32 rdma_id;
+	u32 solid_color;
+	u32 dec_lines;
+	/*
+	 * scaler id. As long as its rdma channel is identical with any
+	 * one uses the scaler, it's set to the scaler's id, even if it
+	 * doesn't really use the scaler.
+	 */
+	u8 scaler_id;
+	/* hw format */
+	u8 format;
+	bool use_scl;
+	bool is_offline; //to indicate rdma is offline
+	bool right_image;
+	u32 fbcmem_size;
+	uint64_t mclk;	//DPU MCLK = MAX(Mclk, Aclk) of all planes
+	uint64_t bw;	//BandWidth = SUM(BW_single) * 1.08
+	struct dpu_mmu_tbl mmu_tbl;
+	struct cmdlist cl[MAX_CL_NUM];
+	u8 cur_cl;
+	uint64_t afbc_effc;
+	struct drm_property_blob *hdr_coefs_blob_prop;
+	struct drm_property_blob *scale_coefs_blob_prop;
+	struct spacemit_afbc_state *afbc_state;
+};
+
+struct spacemit_plane *to_spacemit_plane(struct drm_plane *plane);
+
+static inline struct
+spacemit_plane_state *to_spacemit_plane_state(const struct drm_plane_state *state)
+{
+	return container_of(state, struct spacemit_plane_state, state);
+}
+
+struct spacemit_crtc_state {
+	struct drm_crtc_state base;
+	uint32_t post_scaler_w;
+	uint32_t post_scaler_h;
+	bool post_scl_on;
+	struct spacemit_crtc_scaler scalers[MAX_SCALER_NUMS];
+	struct spacemit_crtc_rdma *rdmas;
+	uint64_t mclk;	/* max of all rdma mclk */
+	uint64_t bw;	/* sum of all rdma bw*/
+	uint64_t aclk;	/* based on bw */
+	uint64_t real_mclk; /* real_mclk = max(mclk, aclk) */
+	uint64_t expected_present_time;
+	struct drm_property_blob *color_matrix_blob_prop;
+	bool scl_rdma_reuse[MAX_SCALER_NUMS];
+	u32 scl_rdma_id[MAX_SCALER_NUMS];
+	struct dpu_mmu_tbl wb_mmu_tbl;
+	struct cmdlist cl[MAX_CL_NUM];
+	int cur_cl;
+	struct drm_property_blob *gamma_table_blob_prop;
+	struct drm_property_blob *pp_acad_blob_prop;
+	struct drm_property_blob *end_tone_mapping_blob_prop;
+	struct drm_property_blob *ee_blob_prop;
+	struct drm_property_blob *pp_color_temperature_blob_property;
+};
+
+#define to_spacemit_crtc_state(x) container_of(x, struct spacemit_crtc_state, base)
+
+struct drm_plane *spacemit_plane_init(struct drm_device *drm,
+					struct spacemit_crtc *a_crtc);
+
+struct spacemit_plane_state;
+struct spacemit_crtc;
+struct tbu_instance;
+struct dpu_mmu_tbl;
+struct spacemit_wb;
+enum spacemit_dpu_irq;
+struct spacemit_hw_device {
+	void __iomem *base;
+	phys_addr_t phy_addr;
+	/* lcd top reg, used for cmd mode */
+	void __iomem *top_base;
+	phys_addr_t top_phy_addr;
+	u8 plane_nums;
+	u8 offline_plane_nums;
+	u8 rdma_nums;
+	u8 crtc_nums;
+	const struct spacemit_hw_rdma *rdmas;
+	u8 n_formats;
+	const struct dpu_format_id *formats;
+	u8 n_fbcmems;
+	const u32 *fbcmem_sizes;
+	u32 solid_color_shift;
+	int hdr_coef_size;
+	int hor_scale_coef_size;
+	int ver_scale_coef_size;
+	u8 reboot_flag;
+	u16 scaler_num;
+	u16 acad_num;
+	u16 dpu_version;
+	u8 crtc_num;
+	u16 *gamma_table;
+	int gamma_size;
+	enum drm_color_encoding color_encoding;
+	enum drm_color_range color_range;
+	int etm_size;
+	bool is_edp;
+	bool is_acad_on;
+	bool is_bl_save_on;
+	void (*conf_ee)(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+	void (*conf_dpuctrl_color_matrix)(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+	void (*update_prepipe_gamma)(struct drm_plane *plane, struct spacemit_plane_state *spacemit_pstate);
+	void (*conf_gamma_table)(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+	int (*check_end_matrix)(struct drm_crtc_state *state);
+	void (*conf_end_tone_mapping)(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+	void (*conf_matrix)(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+	void (*update_hdr_matrix)(struct drm_plane *plane, struct spacemit_plane_state *spacemit_pstate);
+	void (*update_csc_matrix)(struct drm_plane *plane, struct drm_plane_state *old_state);
+	void (*conf_dpuctrl_acad)(struct spacemit_crtc *a_crtc, struct drm_crtc_state *old_state);
+	void (*conf_scaler_x)(struct drm_plane_state *state, struct cmdlist_regs *cl_scl);
+	void (*conf_scaler_coefs)(struct drm_plane *plane, struct spacemit_plane_state *spacemit_pstate, struct cmdlist_regs *cl_scl);
+	void (*enable_vsync)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, bool enable);
+	void (*enable_cfg_irq)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, bool enable);
+	void (*cfg_ready)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	void (*sw_start)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	void (*cmd_update)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	void (*dpu_init)(struct spacemit_crtc *a_crtc);
+	void (*irq_enable)(struct spacemit_crtc *a_crtc, bool enable);
+	void (*plane_update_hw_channel)(struct drm_plane *plane, int slice_id);
+	void (*plane_disable_hw_channel)(struct drm_plane *plane, struct drm_plane_state *old_state);
+	void (*conf_dpuctrl)(struct drm_crtc *crtc, struct drm_crtc_state *old_state);
+	void (*wb_config)(struct spacemit_crtc *a_crtc, struct spacemit_wb *wb, struct drm_framebuffer *fb, struct cmdlist_regs *cl_wb, int slice_id);
+	void (*wb_disable)(struct spacemit_crtc *a_crtc);
+	int (*is_wb_en)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	uint32_t (*get_cfg_rdy)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	uint32_t (*get_irq_bit)(enum spacemit_dpu_irq irq_id, int dev_id);
+	uint32_t (*get_int_sts)(struct spacemit_hw_device *hwdev, int dev_id);
+	uint32_t (*get_rdma_dbg_sts)(struct spacemit_crtc *a_crtc, int dev_id);
+	void (*clr_int_sts)(struct spacemit_crtc *a_crtc, u32 data, int dev_id);
+	void (*dpu_disable)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	void (*dpu_restart)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	int (*dpu_stop_check)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev);
+	void (*enable_cmdlist)(struct spacemit_crtc *a_crtc, struct spacemit_hw_device *hwdev, int id, bool enable);
+	void (*cfg_cmdlist)(struct spacemit_hw_device *hwdev, int id, u32 chy, u32 addrl, u32 addrh);
+	void (*rdma_contig_mem)(struct spacemit_hw_device *hwdev, u8 tbu_id, phys_addr_t contig_pa, struct drm_framebuffer *fb, struct cmdlist_regs *cl_rdma, struct drm_plane *plane);
+	void (*rdma_dmmu)(struct spacemit_hw_device *hwdev, u8 tbu_id, struct tbu_instance *tbu, struct drm_framebuffer *fb, u32 val, struct cmdlist_regs *cl_rdma, struct drm_plane *plane);
+	void (*wb_dmmu)(struct spacemit_hw_device *hwdev, u8 tbu_id, struct tbu_instance *tbu, struct drm_framebuffer *fb, u32 val, u8 fbc_mode, int header_size, struct spacemit_wb *wb);
+	int (*get_cl_rdma_buf)(struct spacemit_crtc *a_crtc);
+	void (*cmdlist_fill_data_row)(struct cmdlist *cl, u32 strobe, u32 offset, u32 value[]);
+	void (*cmdlist_fill_conf_row)(struct cmdlist *cl, struct spacemit_hw_device *hwdev, u8 dev_id);
+	void (*wb_cmdlist)(struct cmdlist *cl, struct spacemit_hw_device *hwdev, struct spacemit_drm_private *priv, u8 crtc_id, u8 dev_id);
+#ifdef CONFIG_SPACEMIT_DEBUG
+	void (*dpu_dump_reg)(struct spacemit_crtc *a_crtc);
+	void (*dpu_dump_rdma_status)(struct spacemit_drm_private *priv);
+#endif
+	void (*cmdlist_dump_node)(struct cmdlist *cl);
+};
+#endif
diff --git a/drivers/gpu/drm/spacemit/spacemit_dmmu.c b/drivers/gpu/drm/spacemit/spacemit_dmmu.c
new file mode 100644
index 000000000000..ee0f0e1ad6da
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_dmmu.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2023 Spacemit Co., Ltd.
+ *
+ */
+
+#include "linux/compiler.h"
+#include <linux/stddef.h>
+#include <linux/gfp.h>
+#include <linux/dma-mapping.h>
+#include <drm/drm_framebuffer.h>
+
+#include "spacemit_cmdlist.h"
+#include "spacemit_dmmu.h"
+#include "spacemit_gem.h"
+#include "spacemit_drm.h"
+#include "spacemit_crtc.h"
+#include "spacemit_dpu_reg.h"
+#include "spacemit_wb.h"
+
+static inline void spacemit_dmmu_fill_pgtable(uint32_t *ttbr, struct sg_table *sgt)
+{
+	struct sg_dma_page_iter dma_iter;
+	uint32_t i = 0, temp = 0;
+
+	for_each_sgtable_dma_page(sgt, &dma_iter, 0)
+		*ttbr++ = (sg_page_iter_dma_address(&dma_iter) >> PAGE_SHIFT) & 0x3FFFFFF;
+	/*
+	 * Due to slicon's alignment access requirement, fill more mmu entries
+	 * to avoid ddrc filter error. We simply use the last valid ttbr value.
+	 */
+	temp = *(ttbr - 1);
+	for (i = 0; i < HW_ALIGN_TTB_NUM; i++)
+		*ttbr++ = temp;
+}
+
+int spacemit_dmmu_map(struct drm_framebuffer *fb, struct dpu_mmu_tbl *mmu_tbl,
+		 u8 tbu_id, struct spacemit_wb *wb, struct cmdlist_regs *cl_rdma,
+		 struct drm_plane *plane)
+{
+	struct spacemit_drm_private *priv = fb->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	const struct drm_format_info *format = NULL;
+	struct sg_table *sgt = NULL;
+	uint32_t total_size, offset1, offset2;
+	struct tbu_instance tbu = {0x0};
+	u8 fbc_mode, plane_num;
+	u32 val;
+	int header_size;
+
+	format = fb->format;
+	sgt = to_spacemit_obj(fb->obj[0])->sgt;
+
+	if (!wb && priv->contig_mem) {
+		phys_addr_t contig_pa = sg_dma_address(sgt->sgl);
+
+		hwdev->rdma_contig_mem(hwdev, tbu_id, contig_pa, fb, cl_rdma, plane);
+		return 0;
+	}
+
+	plane_num = format->num_planes;
+	fbc_mode = (fb->modifier > 0);
+
+	total_size = roundup(fb->obj[0]->size, PAGE_SIZE);
+
+	if (fbc_mode && wb)
+		header_size = spacemit_wb_get_afbc_header_size(wb, fb, fb->width, fb->height);
+
+	offset1 = plane_num > 1 ? fb->offsets[1] : total_size;
+	offset2 = plane_num > 2 ? fb->offsets[2] : total_size;
+
+	switch (plane_num) {
+	case 3:
+		tbu.ttb_pa[2] = mmu_tbl->pa + (offset2 >> PAGE_SHIFT) * 4;
+		tbu.tbu_va[2] = TBU_BASE_VA(tbu_id) + offset2;
+		tbu.ttb_size[2] = PAGE_ALIGN(total_size - rounddown(offset2, PAGE_SIZE)) >> PAGE_SHIFT;
+		fallthrough;
+	case 2:
+		tbu.ttb_pa[1] = mmu_tbl->pa + (offset1 >> PAGE_SHIFT) * 4;
+		tbu.tbu_va[1] = TBU_BASE_VA(tbu_id) + offset1;
+		tbu.ttb_size[1] = PAGE_ALIGN(offset2 - rounddown(offset1, PAGE_SIZE)) >> PAGE_SHIFT;
+		fallthrough;
+	case 1:
+		tbu.ttb_pa[0] = mmu_tbl->pa;
+		tbu.tbu_va[0] = TBU_BASE_VA(tbu_id);
+		tbu.ttb_size[0] = PAGE_ALIGN(offset1) >> PAGE_SHIFT;
+		fallthrough;
+	default:
+		break;
+	}
+	/* Special handling for afbc */
+	if (fbc_mode) {
+		tbu.tbu_va[1] = tbu.tbu_va[0];
+		tbu.ttb_pa[1] = tbu.ttb_pa[0];
+		tbu.ttb_size[1] = tbu.ttb_size[0];
+	}
+
+	spacemit_dmmu_fill_pgtable((uint32_t *)(mmu_tbl->va), sgt);
+
+	val = 0x1 | (fbc_mode << 1) | ((fbc_mode ? 1 : plane_num - 1) << 2);
+	/* tbu_burst_limit_en = 1, tlb_fetch_active_en = 1 */
+	val |= BIT(4) | BIT(5);
+	/* Config hw regs */
+	if (wb) {
+		val = val | (DPU_QOS_LOW << 8);
+		hwdev->wb_dmmu(hwdev, tbu_id, &tbu, fb, val, fbc_mode, header_size, wb);
+	} else {
+		val = val | (DPU_QOS_URGENT << 8);
+		hwdev->rdma_dmmu(hwdev, tbu_id, &tbu, fb, val, cl_rdma, plane);
+	}
+
+	return 0;
+}
+
+void spacemit_dmmu_unmap(struct drm_plane *plane)
+{
+	return;
+}
diff --git a/drivers/gpu/drm/spacemit/spacemit_dmmu.h b/drivers/gpu/drm/spacemit/spacemit_dmmu.h
new file mode 100644
index 000000000000..557f81c79d1c
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_dmmu.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_DMMU_H_
+#define _SPACEMIT_DMMU_H_
+
+#include <linux/types.h>
+
+#include <drm/drm_file.h>
+//#include "dpu/dpu_saturn.h"
+#include "spacemit_cmdlist.h"
+
+#define DPU_QOS_URGENT	4
+#define DPU_QOS_NORMAL	3
+#define DPU_QOS_LOW	2
+/*
+ * In worst case, the tlb alignment requires src_x * 16 * pixel_bytes.
+ * The maximum src_x currently support is 4096. So we need to fill
+ * extra 60 (4096*15*4) entries into mmu page table.
+ */
+#define HW_ALIGN_TTB_NUM	60
+
+#define RD_OUTS_NUM             16
+#define RDMA_TIMELIMIT  0xFFFF
+
+#define BASE_VA         0x10000000ULL
+#define VA_STEP_PER_TBU 0x40000000ULL
+
+#define TBU_BASE_VA(tbu_id) ((uint64_t)BASE_VA + (uint64_t)VA_STEP_PER_TBU * tbu_id)
+
+struct tbu_instance {
+	uint64_t ttb_pa[3];
+	uint64_t tbu_va[3];
+	uint32_t ttb_size[3];
+};
+
+struct dpu_mmu_tbl;
+struct spacemit_wb;
+int spacemit_dmmu_map(struct drm_framebuffer *fb, struct dpu_mmu_tbl *mmu_tbl,
+			u8 tbu_id, struct spacemit_wb *wb, struct cmdlist_regs *cl_rdma,
+			struct drm_plane *plane);
+void spacemit_dmmu_unmap(struct drm_plane *plane);
+
+#endif /* _SPACEMIT_DMMU_H_ */
diff --git a/drivers/gpu/drm/spacemit/spacemit_dphy.c b/drivers/gpu/drm/spacemit/spacemit_dphy.c
new file mode 100644
index 000000000000..059458d979f7
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_dphy.c
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "spacemit_dphy.h"
+#include "sysfs/sysfs_display.h"
+
+LIST_HEAD(dphy_core_head);
+
+int spacemit_dphy_resume(struct spacemit_dphy *dphy)
+{
+	if (dphy->core && dphy->core->init)
+		dphy->core->init(&dphy->ctx);
+
+	DRM_INFO("dphy resume OK\n");
+	return 0;
+}
+
+int spacemit_dphy_suspend(struct spacemit_dphy *dphy)
+{
+	if (dphy->core && dphy->core->uninit)
+		dphy->core->uninit(&dphy->ctx);
+
+	DRM_INFO("dphy suspend OK\n");
+	return 0;
+}
+
+int spacemit_dphy_reset(struct spacemit_dphy *dphy)
+{
+	if (dphy->core && dphy->core->reset)
+		dphy->core->reset(&dphy->ctx);
+
+	DRM_INFO("dphy reset OK\n");
+	return 0;
+}
+
+int spacemit_dphy_get_status(struct spacemit_dphy *dphy)
+{
+	if (dphy->core && dphy->core->get_status)
+		dphy->core->get_status(&dphy->ctx);
+
+	DRM_INFO("dphy get status OK\n");
+	return 0;
+}
+
+static int spacemit_dphy_device_create(struct spacemit_dphy *dphy,
+				   struct device *parent)
+{
+	int ret;
+
+	dphy->dev.class = display_class;
+	dphy->dev.parent = parent;
+	dphy->dev.of_node = parent->of_node;
+	dev_set_name(&dphy->dev, "dphy%d", dphy->ctx.id);
+	dev_set_drvdata(&dphy->dev, dphy);
+
+	ret = device_register(&dphy->dev);
+	if (ret)
+		DRM_ERROR("dphy device register failed\n");
+
+	return ret;
+}
+
+static int spacemit_dphy_context_init(struct spacemit_dphy *dphy,
+				  struct device_node *np)
+{
+	struct resource r;
+	u32 tmp;
+
+	if (dphy->core && dphy->core->parse_dt)
+		dphy->core->parse_dt(&dphy->ctx, np);
+
+	if (!of_address_to_resource(np, 0, &r)) {
+		dphy->ctx.base_addr = (void __iomem *)ioremap(r.start, resource_size(&r));
+		if (dphy->ctx.base_addr == NULL) {
+			DRM_ERROR("dphy ctrlbase ioremap failed\n");
+			return -EFAULT;
+		}
+	} else {
+		DRM_ERROR("parse dphy ctrl reg base failed\n");
+		return -EINVAL;
+	}
+
+	if (!of_property_read_u32(np, "dev-id", &tmp))
+		dphy->ctx.id = tmp;
+
+	return 0;
+}
+
+static int spacemit_dphy_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct spacemit_dphy *dphy;
+	const char *str;
+	int ret;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL);
+	if (!dphy)
+		return -ENOMEM;
+
+	if (!of_property_read_string(np, "ip", &str))
+		dphy->core = dphy_core_ops_attach(str);
+	else
+		DRM_WARN("dphy pll ops parse failed\n");
+
+	ret = spacemit_dphy_context_init(dphy, pdev->dev.of_node);
+	if (ret)
+		return ret;
+
+	spacemit_dphy_device_create(dphy, &pdev->dev);
+	spacemit_dphy_sysfs_init(&dphy->dev);
+	platform_set_drvdata(pdev, dphy);
+
+	DRM_INFO("dphy driver probe success\n");
+
+	return 0;
+}
+
+
+static const struct of_device_id dt_ids[] = {
+	{ .compatible = "spacemit,dsi-phy", },
+	{},
+};
+
+struct platform_driver spacemit_dphy_driver = {
+	.probe = spacemit_dphy_probe,
+	.driver = {
+		.name = "spacemit-dphy-drv",
+		.of_match_table = of_match_ptr(dt_ids),
+	}
+};
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Spacemit MIPI DSI PHY driver");
diff --git a/drivers/gpu/drm/spacemit/spacemit_dphy.h b/drivers/gpu/drm/spacemit/spacemit_dphy.h
new file mode 100644
index 000000000000..e39debac731f
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_dphy.h
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_DPHY_H_
+#define _SPACEMIT_DPHY_H_
+
+#include <asm/types.h>
+#include <drm/drm_print.h>
+#include <linux/device.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+
+#include "spacemit_lib.h"
+
+#define DPHY_BITCLK_DEFAULT	614400000
+#define DPHY_ESCCLK_DEFAULT	51200000
+
+enum spacemit_dphy_lane_map {
+	DPHY_LANE_MAP_0123 = 0,
+	DPHY_LANE_MAP_0312 = 1,
+	DPHY_LANE_MAP_0231 = 2,
+	DPHY_LANE_MAP_MAX
+};
+
+enum spacemit_dphy_status {
+	DPHY_STATUS_UNINIT = 0,
+	DPHY_STATUS_INIT = 1,
+	DPHY_STATUS_MAX
+};
+
+enum spacemit_dphy_bit_clk_src {
+	DPHY_BIT_CLK_SRC_PLL5 = 1,
+	DPHY_BIT_CLK_SRC_MUX = 2,
+	DPHY_BIT_CLK_SRC_MAX
+};
+
+struct spacemit_dphy_timing {
+	uint32_t hs_prep_constant;    /* Unit: ns. */
+	uint32_t hs_prep_ui;
+	uint32_t hs_zero_constant;
+	uint32_t hs_zero_ui;
+	uint32_t hs_trail_constant;
+	uint32_t hs_trail_ui;
+	uint32_t hs_exit_constant;
+	uint32_t hs_exit_ui;
+	uint32_t ck_zero_constant;
+	uint32_t ck_zero_ui;
+	uint32_t ck_trail_constant;
+	uint32_t ck_trail_ui;
+	uint32_t req_ready;
+	uint32_t wakeup_constant;
+	uint32_t wakeup_ui;
+	uint32_t lpx_constant;
+	uint32_t lpx_ui;
+};
+
+struct spacemit_dphy_ctx {
+	u8 id;
+	void __iomem *base_addr;
+	uint32_t phy_freq; /*kHz*/
+	uint32_t lane_num;
+	uint32_t esc_clk; /*kHz*/
+	uint32_t half_pll5;
+	enum spacemit_dphy_bit_clk_src clk_src;
+	struct spacemit_dphy_timing dphy_timing;
+	uint32_t dphy_status0; /*status0 reg*/
+	uint32_t dphy_status1; /*status1 reg*/
+	uint32_t dphy_status2; /*status2 reg*/
+
+	enum spacemit_dphy_status status;
+};
+
+struct dphy_core_ops {
+	int (*parse_dt)(struct spacemit_dphy_ctx *ctx, struct device_node *np);
+	void (*init)(struct spacemit_dphy_ctx *ctx);
+	void (*uninit)(struct spacemit_dphy_ctx *ctx);
+	void (*reset)(struct spacemit_dphy_ctx *ctx);
+	void (*get_status)(struct spacemit_dphy_ctx *ctx);
+};
+
+struct spacemit_dphy {
+	struct device dev;
+	struct spacemit_dphy_ctx ctx;
+	struct dphy_core_ops *core;
+};
+
+extern struct list_head dphy_core_head;
+
+#define dphy_core_ops_register(entry) \
+	disp_ops_register(entry, &dphy_core_head)
+#define dphy_core_ops_attach(str) \
+	disp_ops_attach(str, &dphy_core_head)
+
+int spacemit_dphy_resume(struct spacemit_dphy *dphy);
+int spacemit_dphy_suspend(struct spacemit_dphy *dphy);
+int spacemit_dphy_reset(struct spacemit_dphy *dphy);
+int spacemit_dphy_get_status(struct spacemit_dphy *dphy);
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/spacemit_dpu_reg.h b/drivers/gpu/drm/spacemit/spacemit_dpu_reg.h
new file mode 100644
index 000000000000..b41637bd090f
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_dpu_reg.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_DPU_REG_H_
+#define _SPACEMIT_DPU_REG_H_
+#include "dpu/dpu_trace.h"
+
+enum REG_FILED_TYPE {
+	FIELD_DEF,  //reg field is rw
+	FIELD_W1C,  //reg field is write 1 cleared
+	FIELD_W1S,  //reg field is write 1 set
+	FIELD_W0C,  //reg field is write 0 cleared
+};
+
+static inline void dpu_writel(void __iomem *addr, uint32_t offset, uint32_t data)
+{
+	trace_spacemit_dpu_reg_write((u32)data, offset, NULL);
+	writel(data, (addr + offset));
+}
+
+static inline void dpu_set_bit(void __iomem *addr, uint32_t offset, uint32_t mask)
+{
+	u32 value = 0;
+	value = readl(addr + offset) | mask;
+	trace_spacemit_dpu_reg_write(value, offset, NULL);
+	writel(value, (addr + offset));
+}
+
+static inline void dpu_clr_bit(void __iomem *addr, uint32_t offset, uint32_t mask)
+{
+	u32 value = 0;
+	value = readl(addr + offset) & (~mask);
+	trace_spacemit_dpu_reg_write(value, offset, NULL);
+	writel(value, (addr + offset));
+}
+
+#define dpu_read_reg(hwdev, module_name, module_base, field) \
+	({ u32 __v = (((volatile module_name *)(module_base + hwdev->base))->field); \
+	__v; }) \
+
+#define dpu_write_reg_common(hwdev, module_name, module_base, field, data, type) \
+{ \
+	volatile module_name *module = (module_name *)(module_base + hwdev->base); \
+		module->field = data; \
+		trace_spacemit_dpu_reg_write((u32)data, module_base, #field); \
+}
+
+/*
+ * This API unifies cpu and cmdlist io operations.
+ * For cpu io, parameters cl_regs and offset are not required;
+ * For cmdlist io,
+ *     cl_regs: got from alloc_cmdlist_regs before any io operations to this module;
+ *     offset:  the offset of the field in this module.
+ */
+#define _dpu_write_reg(hwdev, module_name, module_base, field, data, cl_regs, offset, ...) \
+{ \
+	if (cl_regs != NULL) \
+		cmdlist_write_reg(hwdev, module_name, module_base, field, data, cl_regs, offset) \
+	else \
+		dpu_write_reg_common(hwdev, module_name, module_base, field, data, FIELD_DEF) \
+}
+
+#define dpu_write(...) _dpu_write_reg(__VA_ARGS__, NULL, -1)
+
+#define dpu_write_reg(hwdev, module_name, module_base, field, data) \
+	dpu_write_reg_common(hwdev, module_name, module_base, field, data, FIELD_DEF)
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/spacemit_drm.c b/drivers/gpu/drm/spacemit/spacemit_drm.c
new file mode 100644
index 000000000000..560440ec389e
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_drm.c
@@ -0,0 +1,643 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_debugfs.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_of.h>
+#include <linux/aperture.h>
+#include <linux/component.h>
+#include <linux/mutex.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/kernel.h>
+#include <drm/clients/drm_client_setup.h>
+#include <drm/drm_fbdev_dma.h>
+#include <drm/drm_gem_dma_helper.h>
+
+#include "spacemit_drm.h"
+#include "spacemit_dmmu.h"
+#include "spacemit_gem.h"
+#include "dpu/dpu_saturn.h"
+
+#define DRIVER_NAME	"spacemit"
+#define DRIVER_DESC	"Spacemit SoCs' DRM Driver"
+#define DRIVER_MAJOR	1
+#define DRIVER_MINOR	0
+
+static void spacemit_drm_atomic_commit_tail(struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = old_state->dev;
+#ifdef CONFIG_SPACEMIT_DEBUG
+	struct spacemit_drm_private *priv = dev->dev_private;
+	unsigned long flags;
+#endif
+
+	drm_atomic_helper_commit_modeset_disables(dev, old_state);
+
+	drm_atomic_helper_commit_modeset_enables(dev, old_state);
+
+	drm_atomic_helper_commit_planes(dev, old_state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
+#ifdef CONFIG_SPACEMIT_DEBUG
+	spin_lock_irqsave(&priv->ur_dump_lock, flags);
+	priv->old_state = old_state;
+	priv->underrun_debug = true;
+	spin_unlock_irqrestore(&priv->ur_dump_lock, flags);
+#endif
+
+	drm_atomic_helper_wait_for_flip_done(dev, old_state);
+
+	drm_atomic_helper_commit_hw_done(old_state);
+
+	drm_atomic_helper_cleanup_planes(dev, old_state);
+}
+
+static const struct drm_mode_config_helper_funcs spacemit_drm_mode_config_helper = {
+	.atomic_commit_tail = spacemit_drm_atomic_commit_tail,
+};
+
+/* timgings except vstart, vend and vtotal */
+static bool spacemit_drm_mode_match_timings_1(const struct drm_display_mode *mode1,
+				   const struct drm_display_mode *mode2)
+{
+	return mode1->hdisplay == mode2->hdisplay &&
+		mode1->hsync_start == mode2->hsync_start &&
+		mode1->hsync_end == mode2->hsync_end &&
+		mode1->htotal == mode2->htotal &&
+		mode1->hskew == mode2->hskew &&
+		mode1->vdisplay == mode2->vdisplay &&
+		mode1->vscan == mode2->vscan;
+}
+
+static bool spacemit_drm_mode_match_timings_2(const struct drm_display_mode *mode1,
+				   const struct drm_display_mode *mode2)
+{
+	return mode1->vsync_start == mode2->vsync_start &&
+		mode1->vsync_end == mode2->vsync_end &&
+		mode1->vtotal == mode2->vtotal;
+}
+
+/* check whether only timing vfp not equal, based on drm_mode_equal */
+static bool spacemit_drm_mode_equal(const struct drm_display_mode *mode1,
+		const struct drm_display_mode *mode2)
+{
+	int vfp1, vbp1, vfp2, vbp2;
+	bool ret;
+
+	ret = drm_mode_match(mode1, mode2,
+			      DRM_MODE_MATCH_CLOCK |
+			      DRM_MODE_MATCH_FLAGS |
+			      DRM_MODE_MATCH_3D_FLAGS|
+			      DRM_MODE_MATCH_ASPECT_RATIO);
+
+	if (ret == false)
+		return false;
+
+	if (!spacemit_drm_mode_match_timings_1(mode1, mode2))
+		return false;
+
+	vfp1 = mode1->vsync_start - mode1->vdisplay;
+	vbp1 = mode1->vtotal - mode1->vsync_end;
+	vfp2 = mode2->vsync_start - mode2->vdisplay;
+	vbp2 = mode2->vtotal - mode2->vsync_end;
+	if (!spacemit_drm_mode_match_timings_2(mode1, mode2)) {
+		if ((vbp1 == vbp2) && (vfp1 != vfp2))	/* Just vfp change for fps update */
+			return true;
+		else
+			return false;
+	}
+
+	return true;
+}
+
+/* based on drm_atomic_helper_check */
+static int spacemit_drm_atomic_helper_check(struct drm_device *dev, struct drm_atomic_state *state)
+{
+	int ret;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	int i;
+
+	/* original drm api */
+	ret = drm_atomic_helper_check(dev, state);
+
+	/* if new crtc mode only update vfp, no need to disable and enable crtc and connector */
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		if (new_crtc_state->mode_changed) {
+			if (spacemit_drm_mode_equal(&old_crtc_state->mode, &new_crtc_state->mode))
+				new_crtc_state->mode_changed = false;
+		}
+	}
+
+	return ret;
+}
+
+static const struct drm_mode_config_funcs spacemit_drm_mode_config_funcs = {
+	.fb_create = drm_gem_fb_create,
+	.atomic_check = spacemit_drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static void spacemit_drm_mode_config_init(struct drm_device *drm)
+{
+	drm_mode_config_init(drm);
+
+	/* HW has no limitation of min width and min height,
+	 * even for YUV format, which is limitated in plane check
+	 */
+	drm->mode_config.min_width = 1;
+	drm->mode_config.min_height = 1;
+	drm->mode_config.max_width = 4096;
+	drm->mode_config.max_height = 4096;
+
+	drm->mode_config.funcs = &spacemit_drm_mode_config_funcs;
+	drm->mode_config.helper_private = &spacemit_drm_mode_config_helper;
+}
+
+#ifdef CONFIG_DEBUG_FS
+
+#define FRAMEBUFFER_DUMP_PATH "/tmp"
+static int spacemit_framebuffer_dump(struct drm_plane *plane)
+{
+	unsigned int buffer_size = 0;
+	int i, j;
+	void *mmu_tbl_vaddr = NULL;
+	phys_addr_t dpu_buffer_paddr = 0;
+	void __iomem *dpu_buffer_vaddr = NULL;
+	loff_t pos = 0;
+	struct file *filep = NULL;
+	struct drm_framebuffer *fb;
+	char file_name[128];
+	struct spacemit_plane_state *spacemit_pstate = to_spacemit_plane_state(plane->state);
+
+	fb = plane->state->fb;
+	if (!fb)
+		return -EINVAL;
+
+	for (i = 0; i < fb->format->num_planes; i++) {
+		if (fb->obj[i]) {
+			char format[5];
+
+			format[0] = (fb->format->format & 0xFF);
+			format[1] = ((fb->format->format >> 8) & 0xFF);
+			format[2] = ((fb->format->format >> 16) & 0xFF);
+			format[3] = ((fb->format->format >> 24) & 0xFF);
+			format[4] = '\0';
+
+			if (fb->format->is_yuv) {
+				snprintf(file_name, 100, "%s/plane%d_fb%d_%s_planes%d_%dx%d.%s", FRAMEBUFFER_DUMP_PATH, plane->base.id, fb->base.id,
+				format, i, fb->width, fb->height, "yuv");
+			} else {
+				snprintf(file_name, 100, "%s/plane%d_fb%d_%s_planes%d_%dx%d.%s", FRAMEBUFFER_DUMP_PATH, plane->base.id, fb->base.id,
+				format, i, fb->width, fb->height, "rgb");
+			}
+
+			mmu_tbl_vaddr = spacemit_pstate->mmu_tbl.va;
+			buffer_size = plane->state->fb->obj[i]->size >> PAGE_SHIFT;
+			filep = filp_open(file_name, O_RDWR | O_APPEND | O_CREAT, 0644);
+
+			if (IS_ERR(filep)) {
+				DRM_ERROR("Open file %s failed!\n", file_name);
+				return -EINVAL;
+			}
+			for (j = 0; j < buffer_size; j++) {
+				dpu_buffer_paddr = *(volatile u32 __force *)mmu_tbl_vaddr;
+				dpu_buffer_paddr = dpu_buffer_paddr << PAGE_SHIFT;
+				dpu_buffer_vaddr = phys_to_virt((unsigned long)dpu_buffer_paddr);
+				mmu_tbl_vaddr += 4;
+				kernel_write(filep, (void *)dpu_buffer_vaddr, PAGE_SIZE, &pos);
+			}
+			filp_close(filep, NULL);
+			filep = NULL;
+			DRM_INFO("dump framebuffer: %s\n", file_name);
+		}
+
+	}
+
+	return 0;
+}
+
+static int spacemit_drm_dump_show(struct seq_file *s, void *data)
+{
+	struct drm_info_node *node = s->private;
+	struct drm_minor *minor = node->minor;
+	struct drm_device *drm_dev = minor->dev;
+	struct drm_plane *plane;
+	struct drm_framebuffer *fb;
+
+	DRM_INFO("%s()\n", __func__);
+	mutex_lock(&drm_dev->mode_config.fb_lock);
+	drm_for_each_fb(fb, drm_dev) {
+		DRM_INFO("framebuffer[%u]\n", fb->base.id);
+		drm_for_each_plane(plane, drm_dev) {
+			if (plane->state->fb != fb)
+				continue;
+
+			spacemit_framebuffer_dump(plane);
+		}
+	}
+	mutex_unlock(&drm_dev->mode_config.fb_lock);
+
+	return 0;
+}
+
+static const struct drm_info_list spacemit_debugfs_files[] = {
+	{ "dump", spacemit_drm_dump_show, 0 },
+};
+
+static void spacemit_drm_debugfs_init(struct drm_minor *minor)
+{
+	DRM_DEBUG("%s()\n", __func__);
+	drm_debugfs_create_files(spacemit_debugfs_files,
+						ARRAY_SIZE(spacemit_debugfs_files),
+						minor->debugfs_root,
+						minor);
+}
+#endif
+
+static const struct file_operations spacemit_drm_fops = {
+	.owner		= THIS_MODULE,
+	.open		= drm_open,
+	.release		= drm_release,
+	.unlocked_ioctl	= drm_ioctl,
+	.compat_ioctl	= drm_compat_ioctl,
+	.poll			= drm_poll,
+	.read			= drm_read,
+	// .llseek		= no_llseek,
+	.mmap		= spacemit_gem_mmap,
+	.fop_flags = FOP_UNSIGNED_OFFSET,
+};
+
+static struct drm_driver spacemit_drm_drv = {
+	.driver_features = DRIVER_GEM | DRIVER_MODESET |
+					DRIVER_ATOMIC | DRIVER_HAVE_IRQ,
+	.fops = &spacemit_drm_fops,
+
+	.dumb_create = spacemit_gem_dumb_create,
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.gem_prime_import_sg_table = spacemit_gem_prime_import_sg_table,
+	DRM_FBDEV_DMA_DRIVER_OPS,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init		= spacemit_drm_debugfs_init,
+#endif
+
+	.name		= DRIVER_NAME,
+	.desc			= DRIVER_DESC,
+	.major		= DRIVER_MAJOR,
+	.minor		= DRIVER_MINOR,
+};
+
+static int spacemit_drm_bind(struct device *dev)
+{
+	struct drm_device *drm;
+	struct spacemit_drm_private *priv;
+	int err;
+
+	DRM_DEBUG("%s()\n", __func__);
+	/* Remove existing drivers that may own the framebuffer memory. */
+	err = aperture_remove_all_conflicting_devices(spacemit_drm_drv.name);
+	if (err) {
+		DRM_ERROR("Failed to remove existing framebuffers - %d.\n", err);
+		return err;
+	}
+
+	drm = drm_dev_alloc(&spacemit_drm_drv, dev);
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
+
+	priv = dev_get_drvdata(dev);
+	priv->ddev = drm;
+	drm->dev_private = priv;
+	priv->dev = dev;
+
+	spacemit_drm_mode_config_init(drm);
+
+	/* bind and init sub drivers */
+	err = component_bind_all(drm->dev, drm);
+	if (err) {
+		DRM_ERROR("failed to bind all component.\n");
+		goto err_dc_cleanup;
+	}
+
+	/* vblank init */
+	err = drm_vblank_init(drm, drm->mode_config.num_crtc);
+	if (err) {
+		DRM_ERROR("failed to initialize vblank.\n");
+		goto err_unbind_all;
+	}
+
+	/* reset all the states of crtc/plane/encoder/connector */
+	drm_mode_config_reset(drm);
+
+	/* init kms poll for handling hpd */
+	drm_kms_helper_poll_init(drm);
+
+	/* force detection after connectors init */
+	drm_helper_hpd_irq_event(drm);
+
+	err = drm_dev_register(drm, 0);
+	if (err < 0)
+		goto err_kms_helper_poll_fini;
+	drm_client_setup(drm, NULL);
+
+	return 0;
+
+err_kms_helper_poll_fini:
+	drm_kms_helper_poll_fini(drm);
+err_unbind_all:
+	component_unbind_all(drm->dev, drm);
+err_dc_cleanup:
+	drm_mode_config_cleanup(drm);
+	drm_dev_put(drm);
+	return err;
+}
+
+static void spacemit_drm_put_dev(struct drm_device *dev)
+{
+	DRM_DEBUG("\n");
+
+	if (!dev) {
+		DRM_ERROR("cleanup called no dev\n");
+		return;
+	}
+
+	drm_dev_unregister(dev);
+	drm_dev_put(dev);
+}
+
+static void spacemit_drm_unbind(struct device *dev)
+{
+	DRM_DEBUG("%s()\n", __func__);
+	spacemit_drm_put_dev(dev_get_drvdata(dev));
+}
+
+static const struct component_master_ops drm_component_ops = {
+	.bind = spacemit_drm_bind,
+	.unbind = spacemit_drm_unbind,
+};
+
+static int compare_of(struct device *dev, void *data)
+{
+	struct device_node *np = data;
+
+	DRM_DEBUG("compare %s\n", np->full_name);
+
+	return dev->of_node == np;
+}
+
+static int spacemit_drm_of_component_probe(struct device *dev,
+			   int (*compare_of)(struct device *, void *),
+			   const struct component_master_ops *m_ops)
+{
+	struct device_node *crtc_np;
+	struct device_node *ports, *port, *ep, *remote;
+	struct component_match *match = NULL;
+	int i;
+
+	if (!dev->of_node)
+		return -EINVAL;
+
+	/*
+	 * Bind the crtc's ports first, so that drm_of_find_possible_crtcs()
+	 * called from encoder's .bind callbacks works as expected
+	 */
+	for (i = 0; ; i++) {
+		crtc_np = of_parse_phandle(dev->of_node, "ports", i);
+		if (!crtc_np)
+			break;
+
+		if (of_device_is_available(crtc_np))
+			drm_of_component_match_add(dev, &match,
+						   compare_of, crtc_np);
+
+		of_node_put(crtc_np);
+	}
+
+	if (i == 0) {
+		dev_err(dev, "missing 'ports' property\n");
+		return -ENODEV;
+	}
+
+	if (!match) {
+		dev_err(dev, "no available port\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * For bound crtcs, bind the encoders attached to their remote endpoint
+	 */
+	for (i = 0; ; i++) {
+		crtc_np = of_parse_phandle(dev->of_node, "ports", i);
+		if (!crtc_np)
+			break;
+
+		if (!of_device_is_available(crtc_np))
+			goto next_crtc;
+
+		ports = of_get_child_by_name(crtc_np, "ports");
+		if (!ports)
+			goto next_crtc;
+
+		for_each_child_of_node(ports, port) {
+			if (!of_node_name_eq(port, "port"))
+				continue;
+			for_each_child_of_node(port, ep) {
+				if (!of_node_name_eq(ep, "endpoint"))
+					continue;
+				remote = of_graph_get_remote_port_parent(ep);
+				if (!remote)
+					continue;
+				if (of_device_is_available(remote)) {
+					drm_of_component_match_add(dev, &match, compare_of,
+								remote);
+				}
+				of_node_put(remote);
+			}
+		}
+		of_node_put(ports);
+
+next_crtc:
+		of_node_put(crtc_np);
+	}
+
+	return component_master_add_with_match(dev, m_ops, match);
+}
+
+static int spacemit_drm_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct spacemit_drm_private *priv;
+	struct device_node *np = pdev->dev.of_node;
+	struct resource *r;
+
+	DRM_INFO("%s()\n", __func__);
+
+	ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret)
+		DRM_ERROR("dma_set_mask_and_coherent failed (%d)\n", ret);
+
+	/* Allocate and initialize the driver private structure. */
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev_set_drvdata(&pdev->dev, priv);
+
+	if (of_property_read_u32(np, "hw_ver", &priv->hw_ver))
+		return -EINVAL;
+
+	priv->contig_mem = false;
+
+	priv->hwdev = (struct spacemit_hw_device *)of_device_get_match_data(&pdev->dev);
+
+	priv->cmdlist_groups = kzalloc(sizeof(struct cmdlist *) *
+			(priv->hwdev->rdma_nums + priv->hwdev->crtc_nums), GFP_KERNEL);
+	if (!priv->cmdlist_groups)
+		return -ENOMEM;
+
+	priv->wb_connector = kzalloc(sizeof(struct drm_writeback_connector *) *
+			N_WRITEBACK_MAX, GFP_KERNEL);
+	if (!priv->wb_connector)
+		return -ENOMEM;
+
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->hwdev->phy_addr = r->start;
+	priv->hwdev->base = devm_ioremap_resource(&pdev->dev, r);
+	if (IS_ERR(priv->hwdev->base))
+		return PTR_ERR(priv->hwdev->base);
+
+	/* for cmd panel */
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (r) {
+		priv->hwdev->top_phy_addr = r->start;
+		priv->hwdev->top_base = devm_ioremap_resource(&pdev->dev, r);
+		if (IS_ERR(priv->hwdev->top_base))
+			DRM_ERROR("get top_base fail!\n");
+	}
+
+#ifdef CONFIG_SPACEMIT_DEBUG
+	spin_lock_init(&priv->ur_dump_lock);
+#endif
+
+	return spacemit_drm_of_component_probe(&pdev->dev, compare_of,  &drm_component_ops);
+}
+
+static void spacemit_drm_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &drm_component_ops);
+}
+
+static void spacemit_drm_shutdown(struct platform_device *pdev)
+{
+	struct spacemit_drm_private *priv = dev_get_drvdata(&pdev->dev);
+	struct drm_device *drm = priv->ddev;
+
+	if (!drm) {
+		DRM_WARN("drm device is not available, no shutdown\n");
+		return;
+	}
+
+	priv->hwdev->reboot_flag = 1;
+	drm_atomic_helper_shutdown(drm);
+}
+
+static const struct of_device_id drm_match_table[] = {
+	{
+		.compatible = "spacemit,saturn-hee",
+		.data = &spacemit_dp_devices[SATURN_HEE],
+	},
+	{
+		.compatible = "spacemit,saturn-edp",
+		.data = &spacemit_dp_devices[SATURN_EDP],
+	},
+	{},
+
+};
+MODULE_DEVICE_TABLE(of, drm_match_table);
+
+static int __maybe_unused spacemit_drm_suspend(struct device *dev)
+{
+	struct drm_device *drm;
+	struct spacemit_drm_private *priv;
+
+	priv = dev_get_drvdata(dev);
+	drm = priv->ddev;
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int __maybe_unused spacemit_drm_resume(struct device *dev)
+{
+	struct drm_device *drm;
+	struct spacemit_drm_private *priv;
+
+	priv = dev_get_drvdata(dev);
+	drm = priv->ddev;
+
+	return drm_mode_config_helper_resume(drm);
+}
+
+static SIMPLE_DEV_PM_OPS(spacemit_drm_pm_ops, spacemit_drm_suspend,
+			 spacemit_drm_resume);
+
+
+static struct platform_driver spacemit_drm_driver = {
+	.probe = spacemit_drm_probe,
+	.remove = spacemit_drm_remove,
+	.shutdown = spacemit_drm_shutdown,
+	.driver = {
+		.name = "spacemit-drm-drv",
+		.of_match_table = drm_match_table,
+		.pm = &spacemit_drm_pm_ops,
+	},
+};
+
+static struct platform_driver * const spacemit_drm_drivers[] = {
+	&spacemit_drm_driver,
+	&spacemit_dpu_driver,
+	&spacemit_dsi_driver,
+	&spacemit_wb_driver,
+	&spacemit_dphy_driver,
+};
+
+#ifdef MODULE
+extern int dsi_core_register(void);
+extern int dpu_core_register(void);
+extern int dphy_core_register(void);
+extern int display_class_init(void);
+void drm_core_register(void)
+{
+	display_class_init();
+	dsi_core_register();
+	dpu_core_register();
+	dphy_core_register();
+}
+#endif
+static int __init spacemit_drm_drivers_init(void)
+{
+#ifdef MODULE
+	drm_core_register();
+#endif
+	return platform_register_drivers(spacemit_drm_drivers,
+					 ARRAY_SIZE(spacemit_drm_drivers));
+}
+
+static void __exit spacemit_drm_drivers_exit(void)
+{
+	platform_unregister_drivers(spacemit_drm_drivers,
+				    ARRAY_SIZE(spacemit_drm_drivers));
+}
+
+module_init(spacemit_drm_drivers_init);
+module_exit(spacemit_drm_drivers_exit);
+
+MODULE_DESCRIPTION("Spacemit DRM KMS Master Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/spacemit_drm.h b/drivers/gpu/drm/spacemit/spacemit_drm.h
new file mode 100644
index 000000000000..077bd3476a83
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_drm.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_DRM_H_
+#define _SPACEMIT_DRM_H_
+
+#include <drm/drm_print.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_vblank.h>
+#include <drm/drm_probe_helper.h>
+#include <linux/dma-mapping.h>
+#include "spacemit_dmmu.h"
+
+struct spacemit_drm_private {
+	struct drm_device *ddev;
+	struct device *dev;
+	struct spacemit_hw_device *hwdev;
+	int hw_ver;
+	bool contig_mem;
+	struct cmdlist **cmdlist_groups;
+	struct drm_writeback_connector **wb_connector;
+#ifdef CONFIG_SPACEMIT_DEBUG
+	bool underrun_debug;
+	struct drm_atomic_state *old_state;
+	spinlock_t ur_dump_lock;
+#endif
+};
+
+extern struct platform_driver spacemit_dpu_driver;
+extern struct platform_driver spacemit_dphy_driver;
+extern struct platform_driver spacemit_wb_driver;
+extern struct platform_driver spacemit_dsi_driver;
+
+#endif /* _SPACEMIT_DRM_H_ */
diff --git a/drivers/gpu/drm/spacemit/spacemit_dsi.c b/drivers/gpu/drm/spacemit/spacemit_dsi.c
new file mode 100644
index 000000000000..8bdf388b6019
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_dsi.c
@@ -0,0 +1,929 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_of.h>
+#include <linux/component.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_graph.h>
+#include <linux/pm_runtime.h>
+#include <video/mipi_display.h>
+#include <linux/regmap.h>
+
+#include "spacemit_lib.h"
+#include "spacemit_crtc.h"
+#include "spacemit_dsi.h"
+#include "sysfs/sysfs_display.h"
+#include "spacemit_bootloader.h"
+#include "spacemit_mipi_panel.h"
+
+LIST_HEAD(dsi_core_head);
+
+static void spacemit_dsi_encoder_enable(struct drm_encoder *encoder)
+{
+	struct spacemit_dsi *dsi = encoder_to_dsi(encoder);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(encoder->crtc);
+
+	DRM_INFO("%s()\n", __func__);
+
+	if (dsi->panel == NULL)
+		return;
+
+	if (!dsi->core || !dsi->core->dsi_open) {
+		DRM_ERROR("%s(), dsi->core is null!\n", __func__);
+		return;
+	}
+
+	dsi->core->dsi_open(&dsi->ctx, false);
+
+	if (dsi->panel) {
+		drm_panel_prepare(dsi->panel);
+		drm_panel_enable(dsi->panel);
+	}
+
+	if (dsi->core && dsi->core->dsi_ready_for_datatx)
+		dsi->core->dsi_ready_for_datatx(&dsi->ctx);
+
+	a_crtc->is_stopped = false;
+	dsi->is_disabled = false;
+}
+
+static void spacemit_dsi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct spacemit_dsi *dsi = encoder_to_dsi(encoder);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(encoder->crtc);
+	int ret = 0;
+
+	if (dsi->panel == NULL)
+		return;
+
+	mutex_lock(&dsi->disable_lock);
+	DRM_INFO("%s()\n", __func__);
+
+	if (dsi->panel && dsi->panel->backlight) {
+		ret = backlight_disable(dsi->panel->backlight);
+		if (ret < 0)
+			DRM_DEV_INFO(dsi->panel->dev, "failed to disable backlight: %d\n", ret);
+	}
+
+	a_crtc->is_stopped = true;
+	if (!a_crtc->is_offline_mode)
+		spacemit_crtc_stop(a_crtc);
+
+	if (dsi->core && dsi->core->dsi_close_datatx)
+		dsi->core->dsi_close_datatx(&dsi->ctx);
+
+	if (dsi->panel) {
+		drm_panel_disable(dsi->panel);
+		drm_panel_unprepare(dsi->panel);
+	}
+
+	if (dsi->core && dsi->core->dsi_close)
+		dsi->core->dsi_close(&dsi->ctx);
+	dsi->is_disabled = true;
+	mutex_unlock(&dsi->disable_lock);
+}
+
+static void spacemit_dsi_encoder_mode_set(struct drm_encoder *encoder,
+				 struct drm_display_mode *mode,
+				 struct drm_display_mode *adj_mode)
+{
+	struct spacemit_dsi *dsi = encoder_to_dsi(encoder);
+
+	DRM_INFO("%s()\n", __func__);
+
+	drm_display_mode_to_videomode(mode, &dsi->ctx.vm);
+}
+
+static int spacemit_dsi_encoder_atomic_check(struct drm_encoder *encoder,
+				    struct drm_crtc_state *crtc_state,
+				    struct drm_connector_state *conn_state)
+{
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs spacemit_encoder_helper_funcs = {
+	.atomic_check = spacemit_dsi_encoder_atomic_check,
+	.mode_set = spacemit_dsi_encoder_mode_set,
+	.enable = spacemit_dsi_encoder_enable,
+	.disable = spacemit_dsi_encoder_disable,
+};
+
+static const struct drm_encoder_funcs spacemit_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static int spacemit_dsi_encoder_init(struct drm_device *drm, struct spacemit_dsi *dsi)
+{
+	struct drm_encoder *encoder = &dsi->encoder;
+	struct device *dev = dsi->host.dev;
+	u32 crtc_mask;
+	int ret;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	crtc_mask = drm_of_find_possible_crtcs(drm, dev->of_node);
+	if (!crtc_mask) {
+		DRM_ERROR("failed to find crtc mask\n");
+		return -EINVAL;
+	}
+	DRM_INFO("find possible crtcs: 0x%08x\n", crtc_mask);
+
+	encoder->possible_crtcs = crtc_mask;
+	ret = drm_encoder_init(drm, encoder, &spacemit_encoder_funcs,
+			       DRM_MODE_ENCODER_DSI, NULL);
+	if (ret) {
+		DRM_ERROR("failed to init dsi encoder\n");
+		return ret;
+	}
+
+	drm_encoder_helper_add(encoder, &spacemit_encoder_helper_funcs);
+
+	return 0;
+}
+
+static int spacemit_dsi_find_panel(struct spacemit_dsi *dsi)
+{
+	struct device *dev = dsi->host.dev;
+	struct device_node *child, *lcds_node;
+	struct drm_panel *panel;
+
+	/* search /lcds child node first */
+	lcds_node = of_find_node_by_path("/lcds");
+	for_each_child_of_node(lcds_node, child) {
+		panel = of_drm_find_panel(child);
+		if (!IS_ERR(panel)) {
+			dsi->panel = panel;
+			of_node_put(child);
+			return 0;
+		}
+	}
+
+	/*
+	 * If /lcds child node search failed, we search
+	 * the child of dsi host node.
+	 */
+	for_each_child_of_node(dev->of_node, child) {
+		panel = of_drm_find_panel(child);
+		if (panel) {
+			dsi->panel = panel;
+			of_node_put(child);
+			return 0;
+		}
+	}
+
+	DRM_ERROR("of_drm_find_panel() failed\n");
+	return -ENODEV;
+}
+
+static int spacemit_dsi_phy_attach(struct spacemit_dsi *dsi)
+{
+	struct device *dev;
+
+	dev = spacemit_disp_pipe_get_output(&dsi->dev);
+	if (!dev)
+		return -ENODEV;
+
+	dsi->ctx.phy = dev_get_drvdata(dev);
+	if (!dsi->ctx.phy) {
+		DRM_ERROR("dsi attach phy failed\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int spacemit_dsi_get_dsc_info(struct device_node *lcd_node, struct spacemit_mipi_info *mipi_info)
+{
+	int ret;
+	u32 val;
+
+	mipi_info->dsc_enable = 0;
+	ret = of_property_read_u32(lcd_node, "dsc-enable", &val);
+	if (ret)
+		return 0;
+	mipi_info->dsc_enable = val;
+
+	if (!mipi_info->dsc_enable)
+		return 0;
+
+	ret = of_property_read_u32(lcd_node, "dsc-bpc", &val);
+	if (ret || (val != DSC_BPC_8 && val != DSC_BPC_10)) {
+		DRM_ERROR("of get mipi dsc-bpc failed\n");
+		return -1;
+	}
+	mipi_info->dsc_bpc = val;
+
+	ret = of_property_read_u32(lcd_node, "dsc-bpp", &val);
+	if (ret || val < DSC_MIN_BPP || val > DSC_MAX_BPP) {
+		DRM_ERROR("of get mipi dsc-bpp failed\n");
+		return -1;
+	}
+	mipi_info->dsc_bpp = val;
+
+	return 0;
+}
+
+static void spacemit_dsi_get_mipi_info(struct device_node *lcd_node, struct spacemit_mipi_info *mipi_info)
+{
+	int ret;
+	u32 val;
+
+	ret = of_property_read_u32(lcd_node, "height", &val);
+	if (ret)
+		DRM_ERROR("of get mipi height failed\n");
+	else
+		mipi_info->height = val;
+
+	ret = of_property_read_u32(lcd_node, "width", &val);
+	if (ret)
+		DRM_ERROR("of get mipi width failed\n");
+	else
+		mipi_info->width = val;
+
+	ret = of_property_read_u32(lcd_node, "hfp", &val);
+	if (ret)
+		DRM_ERROR("of get mipi hfp failed\n");
+	else
+		mipi_info->hfp = val;
+
+	ret = of_property_read_u32(lcd_node, "hbp", &val);
+	if (ret)
+		DRM_ERROR("of get mipi hbp failed\n");
+	else
+		mipi_info->hbp = val;
+
+	ret = of_property_read_u32(lcd_node, "hsync", &val);
+	if (ret)
+		DRM_ERROR("of get mipi hsync failed\n");
+	else
+		mipi_info->hsync = val;
+
+	ret = of_property_read_u32(lcd_node, "vfp", &val);
+	if (ret)
+		DRM_ERROR("of get mipi vfp failed\n");
+	else
+		mipi_info->vfp = val;
+
+	ret = of_property_read_u32(lcd_node, "vbp", &val);
+	if (ret)
+		DRM_ERROR("of get mipi vbp failed\n");
+	else
+		mipi_info->vbp = val;
+
+	ret = of_property_read_u32(lcd_node, "vsync", &val);
+	if (ret)
+		DRM_ERROR("of get mipi vsync failed\n");
+	else
+		mipi_info->vsync = val;
+
+	ret = of_property_read_u32(lcd_node, "fps", &val);
+	if (ret)
+		DRM_ERROR("of get mipi fps failed\n");
+	else
+		mipi_info->fps = val;
+
+	ret = of_property_read_u32(lcd_node, "work-mode", &val);
+	if (ret)
+		DRM_ERROR("of get mipi work-mode failed\n");
+	else
+		mipi_info->work_mode = val;
+
+	ret = of_property_read_u32(lcd_node, "rgb-mode", &val);
+	if (ret)
+		DRM_ERROR("of get mipi rgb-mode failed\n");
+	else
+		mipi_info->rgb_mode = val;
+
+	ret = of_property_read_u32(lcd_node, "lane-number", &val);
+	if (ret)
+		DRM_ERROR("of get mipi lane-number failed\n");
+	else
+		mipi_info->lane_number = val;
+
+	ret = of_property_read_u32(lcd_node, "phy-freq", &val);
+	if (ret)
+		DRM_ERROR("of get mipi phy-freq failed\n");
+	else
+		mipi_info->phy_freq = val;
+
+	ret = of_property_read_u32(lcd_node, "split-enable", &val);
+	if (ret)
+		DRM_ERROR("of get mipi split-enable failed\n");
+	else
+		mipi_info->split_enable = val;
+
+	ret = of_property_read_u32(lcd_node, "eotp-enable", &val);
+	if (ret)
+		DRM_ERROR("of get mipi eotp-enable failed\n");
+	else
+		mipi_info->eotp_enable = val;
+
+	ret = of_property_read_u32(lcd_node, "burst-mode", &val);
+	if (ret)
+		DRM_ERROR("of get mipi burst-mode failed\n");
+	else
+		mipi_info->burst_mode = val;
+
+	if (mipi_info->work_mode == SPACEMIT_DSI_MODE_CMD) {
+		ret = of_property_read_u32(lcd_node, "ctrl-flag", &val);
+		if (ret)
+			mipi_info->cmd_pkg_ctrl = -1;
+		else
+			mipi_info->cmd_pkg_ctrl = val;
+	}
+
+	ret = of_property_read_u32(lcd_node, "vpn-tx-dly-cnt", &val);
+	if (ret) {
+		DRM_DEBUG("of get mipi vpn-tx-dly-cnt failed\n");
+		mipi_info->vpn_tx_dly_cnt = 0;
+	} else
+		mipi_info->vpn_tx_dly_cnt = val;
+
+	ret = of_property_read_u32(lcd_node, "vpn-dly-cnt", &val);
+	if (ret) {
+		DRM_DEBUG("of get mipi vpn-dly-cnt failed\n");
+		mipi_info->vpn_dly_cnt = 0;
+	} else
+		mipi_info->vpn_dly_cnt = val;
+
+	ret = spacemit_dsi_get_dsc_info(lcd_node, mipi_info);
+	if (ret)
+		mipi_info->dsc_enable = 0;
+
+	ret = of_property_read_u32(lcd_node, "h_extra", &val);
+	if (ret)
+		mipi_info->h_extra = 0;
+	else
+		mipi_info->h_extra = val;
+}
+
+static void spacemit_dsi_get_phy_clock(struct device_node *np, struct spacemit_dsi *dsi)
+{
+	struct spacemit_dsi_device *ctx = &dsi->ctx;
+	struct spacemit_dphy *spacemit_dphy = ctx->phy;
+	struct spacemit_dphy_ctx *dphy_ctx = &spacemit_dphy->ctx;
+	int ret = 0;
+
+	if (np) {
+		ret = of_property_read_u32(np, "phy-freq", &dphy_ctx->phy_freq);
+		if (ret) {
+			DRM_ERROR("of get mipi phy-freq failed\n");
+			dphy_ctx->phy_freq = DSI_DEFAULT_PHY_FREQ;
+		}
+
+		ret = of_property_read_u32(np, "phy-escape-clock", &dphy_ctx->esc_clk);
+		if (ret) {
+			DRM_ERROR("of get mipi phy-escape-clock failed\n");
+			dphy_ctx->esc_clk = DSI_DEFAULT_ESCAPE_FREQ;
+		}
+	}
+}
+
+static void spacemit_dsi_get_advanced_info(struct spacemit_dsi *dsi, struct spacemit_mipi_info *mipi_info)
+{
+	struct spacemit_dsi_device *ctx = &dsi->ctx;
+	struct spacemit_dsi_advanced_setting *adv_setting = &ctx->adv_setting;
+	struct device_node *np = dsi->dev.of_node;
+	int ret;
+
+	/*advanced_setting*/
+	ret = of_property_read_u32(np, "lpm_frame_en", &adv_setting->lpm_frame_en);
+	if (ret)
+		adv_setting->lpm_frame_en = LPM_FRAME_EN_DEFAULT;
+
+	ret = of_property_read_u32(np, "last_line_turn", &adv_setting->last_line_turn);
+	if (ret)
+		adv_setting->last_line_turn = LAST_LINE_TURN_DEFAULT;
+
+	ret = of_property_read_u32(np, "hex_slot_en", &adv_setting->hex_slot_en);
+	if (ret)
+		adv_setting->hex_slot_en = HEX_SLOT_EN_DEFAULT;
+
+	ret = of_property_read_u32(np, "hsa_pkt_en", &adv_setting->hsa_pkt_en);
+	if (ret) {
+		if (mipi_info->burst_mode == DSI_BURST_MODE_NON_BURST_SYNC_PULSE)
+			adv_setting->hsa_pkt_en = HSA_PKT_EN_DEFAULT_SYNC_PULSE;
+		else
+			adv_setting->hsa_pkt_en = HSA_PKT_EN_DEFAULT_OTHER;
+	}
+
+	ret = of_property_read_u32(np, "hse_pkt_en", &adv_setting->hse_pkt_en);
+	if (ret) {
+		if (mipi_info->burst_mode == DSI_BURST_MODE_NON_BURST_SYNC_PULSE)
+			adv_setting->hse_pkt_en = HSE_PKT_EN_DEFAULT_SYNC_PULSE;
+		else
+			adv_setting->hse_pkt_en = HSE_PKT_EN_DEFAULT_OTHER;
+	}
+
+	ret = of_property_read_u32(np, "hbp_pkt_en", &adv_setting->hbp_pkt_en);
+	if (ret)
+		adv_setting->hbp_pkt_en = HBP_PKT_EN_DEFAULT;
+
+	ret = of_property_read_u32(np, "hfp_pkt_en", &adv_setting->hfp_pkt_en);
+	if (ret)
+		adv_setting->hfp_pkt_en = HFP_PKT_EN_DEFAULT;
+
+	ret = of_property_read_u32(np, "hex_pkt_en", &adv_setting->hex_pkt_en);
+	if (ret)
+		adv_setting->hex_pkt_en = HEX_PKT_EN_DEFAULT;
+
+	ret = of_property_read_u32(np, "hlp_pkt_en", &adv_setting->hlp_pkt_en);
+	if (ret)
+		adv_setting->hlp_pkt_en = HLP_PKT_EN_DEFAULT;
+
+	ret = of_property_read_u32(np, "auto_dly_dis", &adv_setting->auto_dly_dis);
+	if (ret)
+		adv_setting->auto_dly_dis = AUTO_DLY_DIS_DEFAULT;
+
+	ret = of_property_read_u32(np, "timing_check_dis", &adv_setting->timing_check_dis);
+	if (ret)
+		adv_setting->timing_check_dis = TIMING_CHECK_DIS_DEFAULT;
+
+	ret = of_property_read_u32(np, "hact_wc_en", &adv_setting->hact_wc_en);
+	if (ret)
+		adv_setting->hact_wc_en = HACT_WC_EN_DEFAULT;
+
+	ret = of_property_read_u32(np, "auto_wc_dis", &adv_setting->auto_wc_dis);
+	if (ret)
+		adv_setting->auto_wc_dis = AUTO_WC_DIS_DEFAULT;
+
+	ret = of_property_read_u32(np, "vsync_rst_en", &adv_setting->vsync_rst_en);
+	if (ret)
+		adv_setting->vsync_rst_en = VSYNC_RST_EN_DEFAULT;
+}
+
+
+static int create_panel_name_blob(struct drm_connector *connector)
+{
+	struct drm_property_blob *blob;
+	struct spacemit_dsi *dsi = connector_to_dsi(connector);
+	struct drm_property *prop;
+	char *blob_data;
+	size_t blob_size = 0;
+	struct device_node *node;
+	int ret = 0;
+	char *default_panel = "default_panel";
+
+	node = of_find_node_by_name(NULL, "panel0");
+	if (node) {
+		ret = of_property_read_string(node, "force-attached", &dsi->panel_name);
+		if (ret)
+			DRM_ERROR("bad panel node\n");
+		of_node_put(node);
+	}
+
+	prop = drm_property_create(connector->dev,
+			DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_BLOB,
+			"panel_name", 0);
+	if (!prop)
+		return -ENOMEM;
+
+	if (!dsi->panel_name) {
+		DRM_ERROR("dsi->panel_name null\n");
+		memcpy((void *)dsi->panel_name, default_panel, strlen(dsi->panel_name));
+	}
+
+	blob_size = strlen(dsi->panel_name);
+	blob = drm_property_create_blob(connector->dev, blob_size, NULL);
+	if (IS_ERR(blob))
+		return -1;
+
+	blob_data = blob->data;
+	memcpy(blob_data, dsi->panel_name, blob_size);
+
+	drm_object_attach_property(&connector->base, prop, blob->base.id);
+	dsi->panel_name_prop = prop;
+
+	return 0;
+}
+
+static int spacemit_dsi_connector_get_modes(struct drm_connector *connector)
+{
+	struct spacemit_dsi *dsi = connector_to_dsi(connector);
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	return drm_panel_get_modes(dsi->panel, connector);
+}
+
+static enum drm_mode_status
+spacemit_dsi_connector_mode_valid(struct drm_connector *connector,
+			 const struct drm_display_mode *mode)
+{
+	DRM_DEBUG("%s()\n", __func__);
+
+	return MODE_OK;
+}
+
+static struct drm_encoder *
+spacemit_dsi_connector_best_encoder(struct drm_connector *connector)
+{
+	struct spacemit_dsi *dsi = connector_to_dsi(connector);
+
+	DRM_DEBUG("%s()\n", __func__);
+	return &dsi->encoder;
+}
+
+static struct drm_connector_helper_funcs spacemit_dsi_connector_helper_funcs = {
+	.get_modes = spacemit_dsi_connector_get_modes,
+	.mode_valid = spacemit_dsi_connector_mode_valid,
+	.best_encoder = spacemit_dsi_connector_best_encoder,
+};
+
+static enum drm_connector_status
+spacemit_dsi_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct spacemit_dsi *dsi = connector_to_dsi(connector);
+
+	DRM_DEBUG("%s() dsi subconnector type %d status %d\n", __func__, dsi->ctx.dsi_subconnector, dsi->ctx.connector_status);
+
+	if (dsi->ctx.dsi_subconnector == SPACEMIT_DSI_SUBCONNECTOR_DP)
+		return dsi->ctx.connector_status;
+	else
+		return connector_status_connected;
+}
+
+static void spacemit_dsi_connector_destroy(struct drm_connector *connector)
+{
+	DRM_INFO("%s()\n", __func__);
+
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static int spacemit_conn_atomic_get_property(struct drm_connector *connector,
+				   const struct drm_connector_state *state,
+				   struct drm_property *property,
+				   uint64_t *val)
+{
+	struct spacemit_dsi *dsi = connector_to_dsi(connector);
+
+	if (property == dsi->panel_name_prop)
+		*val = dsi->panel_name_prop->base.id;
+
+	return 0;
+}
+
+static const struct drm_connector_funcs spacemit_dsi_atomic_connector_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.detect = spacemit_dsi_connector_detect,
+	.destroy = spacemit_dsi_connector_destroy,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.atomic_get_property = spacemit_conn_atomic_get_property,
+};
+
+static int spacemit_dsi_connector_init(struct drm_device *drm, struct spacemit_dsi *dsi)
+{
+	struct drm_encoder *encoder = &dsi->encoder;
+	struct drm_connector *connector = &dsi->connector;
+	int ret;
+
+	connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+	ret = drm_connector_init(drm, connector,
+				 &spacemit_dsi_atomic_connector_funcs,
+				 DRM_MODE_CONNECTOR_DSI);
+	if (ret) {
+		DRM_ERROR("drm_connector_init() failed\n");
+		return ret;
+	}
+
+	drm_connector_helper_add(connector,
+				 &spacemit_dsi_connector_helper_funcs);
+
+	//drm_mode_connector_attach_encoder(connector, encoder);
+	drm_connector_attach_encoder(connector, encoder);
+
+	return 0;
+}
+
+static int __maybe_unused spacemit_dsi_bridge_attach(struct spacemit_dsi *dsi)
+{
+	struct drm_encoder *encoder = &dsi->encoder;
+	struct drm_bridge *bridge = dsi->bridge;
+	struct device *dev = dsi->host.dev;
+	struct device_node *bridge_node;
+	int ret;
+
+	bridge_node = of_graph_get_remote_node(dev->of_node, 2, 0);
+	if (!bridge_node)
+		return 0;
+
+	bridge = of_drm_find_bridge(bridge_node);
+	if (!bridge) {
+		DRM_ERROR("of_drm_find_bridge() failed\n");
+		return -ENODEV;
+	}
+	dsi->bridge = bridge;
+
+	ret = drm_bridge_attach(encoder, bridge, NULL, 0);
+	if (ret) {
+		DRM_ERROR("failed to attach external bridge\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static irqreturn_t spacemit_dsi_isr(int irq, void *data)
+{
+	struct spacemit_dsi *dsi = data;
+
+	DRM_DEBUG("%s: dsi Enter\n", __func__);
+
+	if (dsi->core && dsi->core->isr)
+		dsi->core->isr(&dsi->ctx);
+
+	return IRQ_HANDLED;
+}
+
+static int spacemit_dsi_irq_request(struct spacemit_dsi *dsi)
+{
+	int ret;
+	int irq;
+
+	irq = irq_of_parse_and_map(dsi->host.dev->of_node, 0);
+	if (irq) {
+		DRM_DEBUG("dsi irq num = %d\n", irq);
+		ret = request_irq(irq, spacemit_dsi_isr, 0, "DSI", dsi);
+		if (ret) {
+			DRM_ERROR("dsi failed to request irq int0!\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int spacemit_dsi_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm = data;
+	struct spacemit_dsi *dsi = dev_get_drvdata(dev);
+	struct drm_connector *connector = &dsi->connector;
+	int ret;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	ret = spacemit_dsi_encoder_init(drm, dsi);
+	if (ret)
+		goto cleanup_host;
+
+	ret = spacemit_dsi_connector_init(drm, dsi);
+	if (ret)
+		goto cleanup_encoder;
+
+	ret = spacemit_dsi_irq_request(dsi);
+	if (ret)
+		goto cleanup_connector;
+
+	dsi->is_disabled = false;
+
+	create_panel_name_blob(connector);
+
+	return 0;
+
+cleanup_connector:
+	drm_connector_cleanup(&dsi->connector);
+cleanup_encoder:
+	drm_encoder_cleanup(&dsi->encoder);
+cleanup_host:
+	mipi_dsi_host_unregister(&dsi->host);
+	return ret;
+}
+
+static void spacemit_dsi_unbind(struct device *dev,
+			struct device *master, void *data)
+{
+	/* do nothing */
+	DRM_INFO("%s()\n", __func__);
+
+}
+
+static const struct component_ops dsi_component_ops = {
+	.bind	= spacemit_dsi_bind,
+	.unbind = spacemit_dsi_unbind,
+};
+
+static int spacemit_dsi_host_attach(struct mipi_dsi_host *host,
+			   struct mipi_dsi_device *slave)
+{
+	struct spacemit_dsi *dsi = host_to_dsi(host);
+	struct spacemit_dsi_device *ctx = &dsi->ctx;
+	struct spacemit_mipi_info *mipi_info = &ctx->mipi_info;
+	struct device_node *lcd_node;
+	int ret;
+
+	DRM_INFO("%s()\n", __func__);
+
+	dsi->slave = slave;
+
+	ret = spacemit_dsi_phy_attach(dsi);
+	if (ret)
+		return ret;
+
+	ret = spacemit_dsi_find_panel(dsi);
+	if (ret)
+		return ret;
+
+	lcd_node = dsi->panel->dev->of_node;
+
+	spacemit_dsi_get_mipi_info(lcd_node, mipi_info);
+	spacemit_dsi_get_advanced_info(dsi, mipi_info);
+	spacemit_dsi_get_phy_clock(lcd_node, dsi);
+
+	ret = component_add(host->dev, &dsi_component_ops);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static int spacemit_dsi_host_detach(struct mipi_dsi_host *host,
+			   struct mipi_dsi_device *slave)
+{
+	DRM_INFO("%s()\n", __func__);
+	component_del(host->dev, &dsi_component_ops);
+	return 0;
+}
+
+static ssize_t spacemit_dsi_host_transfer(struct mipi_dsi_host *host,
+				const struct mipi_dsi_msg *msg)
+{
+	struct spacemit_dsi *dsi = host_to_dsi(host);
+	struct spacemit_dsi_cmd_desc cmd;
+	struct spacemit_dsi_rx_buf dbuf = {0x0};
+
+	cmd.cmd_type = msg->type;
+	cmd.length = msg->tx_len;
+	memcpy(cmd.data, msg->tx_buf, cmd.length);
+
+	if (msg->flags & MIPI_DSI_MSG_USE_LPM)
+		cmd.lp = 1;
+	else
+		cmd.lp = 0;
+
+	if (msg->rx_buf && msg->rx_len) {
+		if (dsi->core && dsi->core->dsi_read_cmds)
+			dsi->core->dsi_read_cmds(&dsi->ctx, &dbuf, &cmd, 1);
+		memcpy(msg->rx_buf, dbuf.data, 1);
+		return 0;
+	}
+
+	if (msg->tx_buf && msg->tx_len) {
+		if (dsi->core && dsi->core->dsi_write_cmds)
+			dsi->core->dsi_write_cmds(&dsi->ctx, &cmd, 1);
+	}
+
+	return 0;
+}
+
+static const struct mipi_dsi_host_ops spacemit_dsi_host_ops = {
+	.attach = spacemit_dsi_host_attach,
+	.detach = spacemit_dsi_host_detach,
+	.transfer = spacemit_dsi_host_transfer,
+};
+
+static int spacemit_dsi_host_init(struct device *dev, struct spacemit_dsi *dsi)
+{
+	int ret;
+
+	dsi->host.dev = dev;
+	dsi->host.ops = &spacemit_dsi_host_ops;
+
+	ret = mipi_dsi_host_register(&dsi->host);
+	if (ret)
+		DRM_ERROR("failed to register dsi host\n");
+
+	return ret;
+}
+
+static int spacemit_dsi_device_create(struct spacemit_dsi *dsi, struct device *parent)
+{
+	int ret;
+
+	dsi->dev.class = display_class;
+	dsi->dev.parent = parent;
+	dsi->dev.of_node = parent->of_node;
+	dev_set_name(&dsi->dev, "dsi%d", dsi->ctx.id);
+	dev_set_drvdata(&dsi->dev, dsi);
+
+	ret = device_register(&dsi->dev);
+	if (ret)
+		DRM_ERROR("dsi device register failed\n");
+
+	return ret;
+}
+
+static int spacemit_dsi_context_init(struct spacemit_dsi *dsi, struct device_node *np)
+{
+	struct spacemit_dsi_device *ctx = &dsi->ctx;
+	struct resource r;
+	u32 tmp;
+
+	if (dsi->core && dsi->core->parse_dt)
+		dsi->core->parse_dt(&dsi->ctx, np);
+
+	if (of_address_to_resource(np, 0, &r)) {
+		DRM_ERROR("parse dsi ctrl reg base failed\n");
+		return -ENODEV;
+	}
+
+	ctx->base_addr_dsi0 = (void __iomem *)ioremap(r.start, resource_size(&r));
+	if (ctx->base_addr_dsi0 == NULL) {
+		DRM_ERROR("dsi0 ctrl reg base ioremap failed\n");
+		return -ENODEV;
+	}
+	ctx->base_addr = ctx->base_addr_dsi0;
+
+	ctx->base_addr_dsi1 = (void __iomem *)ioremap(DSI1_BASE_ADDR, 200);
+	if (ctx->base_addr_dsi1 == NULL) {
+		DRM_ERROR("dsi1 ctrl reg base ioremap failed\n");
+	}
+
+	ctx->dsi_subconnector = SPACEMIT_DSI_SUBCONNECTOR_MIPI_DSI;
+	ctx->previous_connector_status = connector_status_connected;
+	ctx->connector_status = connector_status_connected;
+
+	if (!of_property_read_u32(np, "dev-id", &tmp))
+		ctx->id = tmp;
+
+	ctx->version = DSI_VERSION_2;
+	if (!of_property_read_u32(np, "dsi-version", &tmp))
+		ctx->version = tmp;
+
+	// if (ctx->version >= DSI_VERSION_2) {
+	//	ctx->apmu_base = spacemit_syscon_regmap_lookup_by_phandle("apmu-phandle");
+	//	if (IS_ERR(ctx->apmu_base))
+	//	{
+	//		DRM_ERROR("BUG: failed to get apmu base\n");
+	//		return -ENODEV;
+	//	}
+	// }
+
+	return 0;
+}
+
+static int spacemit_dsi_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct spacemit_dsi *dsi;
+	const char *str;
+	int ret;
+	DRM_INFO("%s()\n", __func__);
+	dsi = devm_kzalloc(&pdev->dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi) {
+		DRM_ERROR("failed to allocate dsi data.\n");
+		return -ENOMEM;
+	}
+
+	if (!of_property_read_string(np, "ip", &str))
+		dsi->core = dsi_core_ops_attach(str);
+	else
+		DRM_WARN("error: 'ip' was not found\n");
+
+	ret = spacemit_dsi_context_init(dsi, np);
+	if (ret)
+		return -EINVAL;
+
+	spacemit_dsi_device_create(dsi, &pdev->dev);
+	spacemit_dsi_sysfs_init(&dsi->dev);
+	platform_set_drvdata(pdev, dsi);
+
+	ret = spacemit_dsi_host_init(&pdev->dev, dsi);
+	if (ret)
+		return ret;
+
+	mutex_init(&dsi->disable_lock);
+
+	return 0;
+}
+
+static void spacemit_dsi_remove(struct platform_device *pdev)
+{
+}
+
+static const struct of_device_id spacemit_dsi_of_match[] = {
+	{ .compatible = "spacemit,dsi-host" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, spacemit_dsi_of_match);
+
+struct platform_driver spacemit_dsi_driver = {
+	.probe = spacemit_dsi_probe,
+	.remove = spacemit_dsi_remove,
+	.driver = {
+		.name = "spacemit-dsi-drv",
+		.of_match_table = spacemit_dsi_of_match,
+	},
+};
+
+MODULE_DESCRIPTION("Spacemit MIPI DSI HOST Controller Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/spacemit_dsi.h b/drivers/gpu/drm/spacemit/spacemit_dsi.h
new file mode 100644
index 000000000000..6d7c7ffd5bfc
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_dsi.h
@@ -0,0 +1,300 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_DSI_H_
+#define _SPACEMIT_DSI_H_
+
+#include <linux/of.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <video/videomode.h>
+
+#include <drm/drm_print.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_panel.h>
+
+#include "spacemit_lib.h"
+#include "spacemit_dphy.h"
+
+/* define LCD_IS_READY to bypass kernel dsi driver */
+/* #define LCD_IS_READY */
+
+/*advanced setting*/
+#define LPM_FRAME_EN_DEFAULT 0
+#define LAST_LINE_TURN_DEFAULT 0
+#define HEX_SLOT_EN_DEFAULT 0
+#define HSA_PKT_EN_DEFAULT_SYNC_PULSE 1
+#define HSA_PKT_EN_DEFAULT_OTHER 0
+#define HSE_PKT_EN_DEFAULT_SYNC_PULSE 1
+#define HSE_PKT_EN_DEFAULT_OTHER 0
+#define HBP_PKT_EN_DEFAULT 1
+#define HFP_PKT_EN_DEFAULT 0
+#define HEX_PKT_EN_DEFAULT 0
+#define HLP_PKT_EN_DEFAULT 0
+#define AUTO_DLY_DIS_DEFAULT 0
+#define TIMING_CHECK_DIS_DEFAULT 0
+#define HACT_WC_EN_DEFAULT 1
+#define AUTO_WC_DIS_DEFAULT 0
+#define VSYNC_RST_EN_DEFAULT 1
+#define DSI_DEFAULT_PHY_FREQ 911000
+#define DSI_DEFAULT_ESCAPE_FREQ 52000
+
+#define MAX_TX_CMD_COUNT 300
+#define MAX_RX_DATA_COUNT 64
+
+#define DSC_MIN_BPP	(6)
+#define DSC_MAX_BPP	(15)
+
+#define DSI1_BASE_ADDR	0xD421AA00
+
+enum spacemit_mipi_burst_mode {
+	DSI_BURST_MODE_NON_BURST_SYNC_PULSE = 0,
+	DSI_BURST_MODE_NON_BURST_SYNC_EVENT = 1,
+	DSI_BURST_MODE_BURST = 2,
+	DSI_BURST_MODE_MAX
+};
+
+enum spacemit_mipi_input_data_mode {
+	DSI_INPUT_DATA_RGB_MODE_565 = 0,
+	DSI_INPUT_DATA_RGB_MODE_666PACKET = 1,
+	DSI_INPUT_DATA_RGB_MODE_666UNPACKET = 2,
+	DSI_INPUT_DATA_RGB_MODE_888 = 3,
+	DSI_INPUT_DATA_RGB_MODE_101010 = 4,
+	DSI_INPUT_DATA_RGB_MODE_DSC = 5,
+	DSI_INPUT_DATA_RGB_MODE_MAX
+};
+
+enum spacemit_dsi_work_mode {
+	SPACEMIT_DSI_MODE_VIDEO,
+	SPACEMIT_DSI_MODE_CMD,
+	SPACEMIT_DSI_MODE_MAX
+};
+
+enum spacemit_dsi_cmd_type {
+	SPACEMIT_DSI_DCS_SWRITE = 0x5,
+	SPACEMIT_DSI_DCS_SWRITE1 = 0x15,
+	SPACEMIT_DSI_DCS_LWRITE = 0x39,
+	SPACEMIT_DSI_DCS_READ = 0x6,
+	SPACEMIT_DSI_GENERIC_LWRITE = 0x29,
+	SPACEMIT_DSI_GENERIC_READ1 = 0x14,
+	SPACEMIT_DSI_SET_MAX_PKT_SIZE = 0x37,
+	SPACEMIT_DSI_PICTURE_PARAMETER_SET = 0x0a,
+	SPACEMIT_DSI_COMPRESSION_MODE = 0x07,
+};
+
+enum spacemit_dsi_tx_mode {
+	SPACEMIT_DSI_HS_MODE = 0,
+	SPACEMIT_DSI_LP_MODE = 1,
+};
+
+enum spacemit_dsi_rx_data_type {
+	SPACEMIT_DSI_ACK_ERR_RESP = 0x2,
+	SPACEMIT_DSI_EOTP = 0x8,
+	SPACEMIT_DSI_GEN_READ1_RESP = 0x11,
+	SPACEMIT_DSI_GEN_READ2_RESP = 0x12,
+	SPACEMIT_DSI_GEN_LREAD_RESP = 0x1A,
+	SPACEMIT_DSI_DCS_READ1_RESP = 0x21,
+	SPACEMIT_DSI_DCS_READ2_RESP = 0x22,
+	SPACEMIT_DSI_DCS_LREAD_RESP = 0x1C,
+};
+
+enum spacemit_dsi_polarity {
+	SPACEMIT_DSI_POLARITY_POS = 0,
+	SPACEMIT_DSI_POLARITY_NEG,
+	SPACEMIT_DSI_POLARITY_MAX
+};
+
+enum spacemit_dsi_te_mode {
+	SPACEMIT_DSI_TE_MODE_NO = 0,
+	SPACEMIT_DSI_TE_MODE_A,
+	SPACEMIT_DSI_TE_MODE_B,
+	SPACEMIT_DSI_TE_MODE_C,
+	SPACEMIT_DSI_TE_MODE_MAX,
+};
+
+enum spacemit_dsi_event_id {
+	SPACEMIT_DSI_EVENT_ERROR,
+	SPACEMIT_DSI_EVENT_MAX,
+};
+
+enum spacemit_dsi_status {
+	DSI_STATUS_UNINIT = 0,
+	DSI_STATUS_OPENED = 1,
+	DSI_STATUS_INIT = 2,
+	DSI_STATUS_MAX
+};
+
+enum spacemit_dsc_bpc {
+	DSC_BPC_8 = 8,  /* FORMAT_RGB888 */
+	DSC_BPC_10 = 10, /* FORMAT_ARGB2101010 */
+};
+
+enum spacemit_dsi_version {
+	DSI_VERSION_1 = 0,
+	DSI_VERSION_2,
+	DSI_VERSION_MAX
+};
+
+struct spacemit_dsi_vrr_param {
+	uint32_t vrr_vfp;
+};
+
+struct spacemit_dsi_info {
+	unsigned int cmd_pkt_max_size;
+	unsigned int cmd_pkt_default_size;
+	unsigned int cmd_fifo_max_level;
+	unsigned int cmd_fifo_default_level;
+	unsigned int vpn_burst_mode_shift;
+};
+
+struct spacemit_dsi_advanced_setting {
+	uint32_t lpm_frame_en; /*return to LP mode every frame*/
+	uint32_t last_line_turn;
+	uint32_t hex_slot_en;
+	uint32_t hsa_pkt_en;
+	uint32_t hse_pkt_en;
+	uint32_t hbp_pkt_en; /*bit:18*/
+	uint32_t hfp_pkt_en; /*bit:20*/
+	uint32_t hex_pkt_en;
+	uint32_t hlp_pkt_en; /*bit:22*/
+	uint32_t auto_dly_dis;
+	uint32_t timing_check_dis;
+	uint32_t hact_wc_en;
+	uint32_t auto_wc_dis;
+	uint32_t vsync_rst_en;
+};
+
+struct spacemit_mipi_info {
+	unsigned int height;
+	unsigned int width;
+	unsigned int hfp; /*pixel*/
+	unsigned int hbp;
+	unsigned int hsync;
+	unsigned int vfp; /*line*/
+	unsigned int vbp;
+	unsigned int vsync;
+	unsigned int fps;
+	struct spacemit_dsi_vrr_param vrr_param; /* for variable refresh rate */
+
+	unsigned int work_mode; /*command_mode, video_mode*/
+	unsigned int rgb_mode;
+	unsigned int lane_number;
+	unsigned int phy_freq;
+	unsigned int split_enable;
+	unsigned int eotp_enable;
+
+	/*for video mode*/
+	unsigned int burst_mode;
+
+	/*for cmd mode*/
+	unsigned int te_enable;
+	unsigned int vsync_pol;
+	unsigned int te_pol;
+	unsigned int te_mode;
+
+	/* dsc parameters */
+	unsigned int dsc_enable;
+	unsigned int dsc_bpc;
+	unsigned int dsc_bpp;
+
+	unsigned int cmd_pkg_ctrl;
+	unsigned int cmd_pkt_size;
+	unsigned int cmd_fifo_level;
+	/*The following fields need not be set by panel*/
+	unsigned int real_fps;
+
+	/* the following dsi version need to set: dsi version2 */
+	unsigned int vpn_tx_dly_cnt;
+	unsigned int vpn_dly_cnt;
+
+	unsigned int h_extra;
+};
+
+struct spacemit_dsi_cmd_desc {
+	enum spacemit_dsi_cmd_type cmd_type;
+	uint8_t lp;
+	uint32_t delay;	/* time to delay */
+	uint32_t length;	/* cmds length */
+	uint8_t data[MAX_TX_CMD_COUNT];
+};
+
+struct spacemit_dsi_rx_buf {
+	enum spacemit_dsi_rx_data_type data_type;
+	uint32_t length; /* cmds length */
+	uint8_t data[MAX_RX_DATA_COUNT];
+};
+
+enum spacemit_dsi_subconnector {
+	SPACEMIT_DSI_SUBCONNECTOR_MIPI_DSI   = 0,
+	SPACEMIT_DSI_SUBCONNECTOR_HDMI       = 1,
+	SPACEMIT_DSI_SUBCONNECTOR_DP         = 2,
+	SPACEMIT_DSI_SUBCONNECTOR_eDP        = 3,
+};
+
+struct spacemit_dsi_device {
+	uint16_t id;
+	uint16_t version; /* lark：DSI_VERSION_1, dovenr:DSI_VERSION_2 */
+	void __iomem *base_addr;
+	void __iomem *base_addr_dsi0;
+	void __iomem *base_addr_dsi1;
+	struct regmap *apmu_base;
+	struct spacemit_dsi_advanced_setting adv_setting;
+	struct spacemit_mipi_info mipi_info;
+	struct videomode vm;
+	struct spacemit_dphy *phy;
+	enum spacemit_dsi_status status;
+	enum drm_connector_status previous_connector_status;
+	enum drm_connector_status connector_status;
+	enum spacemit_dsi_subconnector dsi_subconnector;
+};
+
+struct dsi_core_ops {
+	int (*parse_dt)(struct spacemit_dsi_device *ctx, struct device_node *np);
+	int (*isr)(struct spacemit_dsi_device *ctx);
+	int (*dsi_open)(struct spacemit_dsi_device *ctx, bool ready);
+	int (*dsi_close)(struct spacemit_dsi_device *ctx);
+	int (*dsi_write_cmds)(struct spacemit_dsi_device *ctx, struct spacemit_dsi_cmd_desc *cmds, int count);
+	int (*dsi_read_cmds)(struct spacemit_dsi_device *ctx, struct spacemit_dsi_rx_buf *dbuf,
+								struct spacemit_dsi_cmd_desc *cmds, int count);
+	int (*dsi_ready_for_datatx)(struct spacemit_dsi_device *ctx);
+	int (*dsi_close_datatx)(struct spacemit_dsi_device *ctx);
+	void (*dsi_enable_irq)(struct spacemit_dsi_device *ctx, bool enable);
+	void (*dsi_update_vrr)(struct spacemit_dsi_device *ctx, struct spacemit_dsi_vrr_param *vrr_param);
+};
+
+struct spacemit_dsi {
+	struct device dev;
+	struct mipi_dsi_host host;
+	struct mipi_dsi_device *slave;
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
+	struct dsi_core_ops *core;
+	struct spacemit_dsi_device ctx;
+	struct mutex disable_lock;	/* serialize dsi disable and esd recovery */
+	bool is_disabled;		/* abort esd when dsi(panel) disabled */
+	struct drm_property *panel_name_prop;
+	const char *panel_name;
+};
+
+extern struct list_head dsi_core_head;
+
+#define dsi_core_ops_register(entry) \
+	disp_ops_register(entry, &dsi_core_head)
+#define dsi_core_ops_attach(str) \
+	disp_ops_attach(str, &dsi_core_head)
+
+#define encoder_to_dsi(encoder) \
+	container_of(encoder, struct spacemit_dsi, encoder)
+#define host_to_dsi(host) \
+	container_of(host, struct spacemit_dsi, host)
+#define connector_to_dsi(connector) \
+	container_of(connector, struct spacemit_dsi, connector)
+#endif /* _SPACEMIT_DSI_H_ */
diff --git a/drivers/gpu/drm/spacemit/spacemit_gem.c b/drivers/gpu/drm/spacemit/spacemit_gem.c
new file mode 100644
index 000000000000..f267c5907be6
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_gem.c
@@ -0,0 +1,479 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-map-ops.h>
+#include <drm/drm_prime.h>
+#include <linux/vmalloc.h>
+
+#include "spacemit_gem.h"
+#include "spacemit_dmmu.h"
+#include "spacemit_drm.h"
+
+MODULE_IMPORT_NS("DMA_BUF");
+static const unsigned int orders[] = {8, 4, 0};
+static const int num_orders = ARRAY_SIZE(orders);
+
+static struct page *__alloc_largest_available(size_t size,
+				unsigned int max_order)
+{
+	struct page *page;
+	unsigned int order;
+	gfp_t gfp_flags;
+	int i;
+
+	for (i = 0; i < num_orders; i++) {
+		order = orders[i];
+
+		if (size < (PAGE_SIZE << order))
+			continue;
+		if (max_order < order)
+			continue;
+
+		if (order > 0)
+			gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
+					__GFP_COMP | __GFP_NORETRY) & ~__GFP_RECLAIM;
+		else
+			gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
+					__GFP_COMP);
+
+		page = alloc_pages(gfp_flags, order);
+		if (!page)
+			continue;
+
+		return page;
+	}
+
+	return NULL;
+}
+
+static int spacemit_gem_sysmem_alloc(struct drm_device *drm,
+				struct spacemit_gem_object *spacemit_obj,
+				size_t size)
+{
+	struct sg_table *sgt;
+	struct scatterlist *sg;
+	struct list_head pages;
+	struct page *page, *tmp_page;
+	size_t size_remaining = size;
+	unsigned int max_order = orders[0];
+	int i = 0;
+	u32 page_nums = PAGE_ALIGN(size) >> PAGE_SHIFT;
+	struct spacemit_drm_private *priv = drm->dev_private;
+
+
+	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&pages);
+	if (priv->contig_mem) {
+#if IS_ENABLED(CONFIG_GKI_FIX_WORKAROUND)
+		kfree(sgt);
+		return -ENOMEM;
+#else
+		page = dma_alloc_from_contiguous(drm->dev, page_nums, 0, 0);
+		if (!page) {
+			kfree(sgt);
+			DRM_ERROR("Failed to alloc %zd bytes continous mem\n", size);
+			return -ENOMEM;
+		}
+		list_add_tail(&page->lru, &pages);
+		i++;
+#endif
+	} else {
+		while (size_remaining > 0) {
+			if (fatal_signal_pending(current))
+				goto free_pages;
+
+			page = __alloc_largest_available(size_remaining, max_order);
+			if (!page)
+				goto free_pages;
+
+			list_add_tail(&page->lru, &pages);
+			size_remaining -= page_size(page);
+			max_order = compound_order(page);
+			i++;
+		}
+	}
+
+	if (sg_alloc_table(sgt, i, GFP_KERNEL))
+		goto free_pages;
+
+	sg = sgt->sgl;
+	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
+		if (priv->contig_mem)
+			sg_set_page(sg, page, page_nums << PAGE_SHIFT, 0);
+		else
+			sg_set_page(sg, page, page_size(page), 0);
+		sg = sg_next(sg);
+		list_del(&page->lru);
+	}
+
+	dma_map_sgtable(drm->dev, sgt, DMA_BIDIRECTIONAL, 0);
+	dma_unmap_sgtable(drm->dev, sgt, DMA_BIDIRECTIONAL, 0);
+
+	spacemit_obj->sgt = sgt;
+
+	return 0;
+
+free_pages:
+	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
+		if (priv->contig_mem)
+#if IS_ENABLED(CONFIG_GKI_FIX_WORKAROUND)
+			nop();
+#else
+			dma_release_from_contiguous(drm->dev, page, page_nums);
+#endif
+		else
+			__free_pages(page, compound_order(page));
+	}
+	kfree(sgt);
+
+	return -ENOMEM;
+}
+
+static void spacemit_gem_sysmem_free(struct spacemit_gem_object *spacemit_obj)
+{
+	struct sg_table *sgt = spacemit_obj->sgt;
+	struct scatterlist *sg;
+	int i;
+	struct drm_device *drm = spacemit_obj->base.dev;
+	struct spacemit_drm_private *priv = drm->dev_private;
+
+	if (!sgt)
+		return;
+
+	for_each_sgtable_sg(sgt, sg, i) {
+		struct page *page = sg_page(sg);
+
+		if (priv->contig_mem)
+#if IS_ENABLED(CONFIG_GKI_FIX_WORKAROUND)
+			nop();
+#else
+			dma_release_from_contiguous(drm->dev, page, sg->length >> PAGE_SHIFT);
+#endif
+		else
+			__free_pages(page, compound_order(page));
+	}
+
+	sg_free_table(sgt);
+	kfree(sgt);
+}
+
+static int spacemit_gem_alloc_buf(struct drm_device *drm,
+				struct spacemit_gem_object *spacemit_obj,
+				size_t size)
+{
+	int ret;
+
+	ret = spacemit_gem_sysmem_alloc(drm, spacemit_obj, size);
+	if (ret) {
+		DRM_ERROR("SPACEMIT_GEM: failed to allocate %zu byte buffer", size);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void spacemit_gem_free_buf(struct spacemit_gem_object *spacemit_obj)
+{
+	spacemit_gem_sysmem_free(spacemit_obj);
+}
+
+static void spacemit_gem_free_object(struct drm_gem_object *gem_obj)
+{
+	struct spacemit_gem_object *spacemit_obj = to_spacemit_obj(gem_obj);
+
+	if (gem_obj->import_attach)
+		drm_prime_gem_destroy(gem_obj, spacemit_obj->sgt);
+	else
+		spacemit_gem_free_buf(spacemit_obj);
+
+	drm_gem_object_release(gem_obj);
+
+	kfree(spacemit_obj);
+}
+
+static struct sg_table *__dup_sg_table(struct sg_table *sgt)
+{
+	struct sg_table *new_sgt;
+	struct scatterlist *sg, *new_sg;
+	int ret, i;
+
+	new_sgt = kzalloc(sizeof(*new_sgt), GFP_KERNEL);
+	if (!new_sgt)
+		return ERR_PTR(-ENOMEM);
+
+	ret = sg_alloc_table(new_sgt, sgt->orig_nents, GFP_KERNEL);
+	if (ret) {
+		kfree(new_sgt);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_sg = new_sgt->sgl;
+	for_each_sgtable_sg(sgt, sg, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
+		new_sg = sg_next(new_sg);
+	}
+
+	return new_sgt;
+}
+
+static struct sg_table *spacemit_gem_prime_get_sg_table(struct drm_gem_object *gem_obj)
+{
+	struct spacemit_gem_object *spacemit_obj = to_spacemit_obj(gem_obj);
+
+	return __dup_sg_table(spacemit_obj->sgt);
+}
+
+static int spacemit_gem_prime_vmap(struct drm_gem_object *gem_obj, struct iosys_map *map)
+{
+	struct spacemit_gem_object *spacemit_obj = to_spacemit_obj(gem_obj);
+	struct sg_page_iter piter;
+	int ret = 0;
+	void *vaddr;
+	int npages = PAGE_ALIGN(gem_obj->size) >> PAGE_SHIFT;
+	struct page **pages, **tmp;
+	pgprot_t pgprot = pgprot_writecombine(PAGE_KERNEL);
+
+	mutex_lock(&spacemit_obj->vmap_lock);
+
+	if (spacemit_obj->vmap_cnt) {
+		vaddr = spacemit_obj->vaddr;
+		goto vmap_success;
+	}
+
+	if (gem_obj->import_attach) {
+		ret = dma_buf_vmap(gem_obj->import_attach->dmabuf, map);
+		if (!ret) {
+			if (WARN_ON(map->is_iomem)) {
+				dma_buf_vunmap(gem_obj->import_attach->dmabuf, map);
+				ret = -EIO;
+				goto vmap_fail;
+			}
+			spacemit_obj->vaddr = map->vaddr;
+		}
+	} else {
+		pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
+		tmp = pages;
+		if (!pages) {
+			ret = -ENOMEM;
+			goto vmap_fail;
+		}
+
+		for_each_sgtable_page(spacemit_obj->sgt, &piter, 0) {
+			WARN_ON(tmp - pages >= npages);
+			*(tmp++) = sg_page_iter_page(&piter);
+		}
+
+		vaddr = vmap(pages, npages, VM_MAP, pgprot);
+		kvfree(pages);
+		if (!vaddr) {
+			ret = -ENOMEM;
+			goto vmap_fail;
+		}
+		spacemit_obj->vaddr = vaddr;
+	}
+
+	if (ret) {
+		DRM_DEBUG_KMS("Failed to vmap pages, error %d\n", ret);
+		goto vmap_fail;
+	}
+vmap_success:
+	spacemit_obj->vmap_cnt++;
+	iosys_map_set_vaddr(map, vaddr);
+	mutex_unlock(&spacemit_obj->vmap_lock);
+	return 0;
+
+vmap_fail:
+	mutex_unlock(&spacemit_obj->vmap_lock);
+	return ret;
+}
+
+static void spacemit_gem_prime_vunmap(struct drm_gem_object *gem_obj, struct iosys_map *map)
+{
+	struct spacemit_gem_object *spacemit_obj = to_spacemit_obj(gem_obj);
+
+	mutex_lock(&spacemit_obj->vmap_lock);
+	spacemit_obj->vmap_cnt--;
+	if (!spacemit_obj->vmap_cnt) {
+		if (gem_obj->import_attach)
+			dma_buf_vunmap(gem_obj->import_attach->dmabuf, spacemit_obj->vaddr);
+		else
+			vunmap(spacemit_obj->vaddr);
+		spacemit_obj->vaddr = NULL;
+	}
+	mutex_unlock(&spacemit_obj->vmap_lock);
+}
+
+const struct vm_operations_struct vm_ops = {
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+static const struct drm_gem_object_funcs spacemit_gem_funcs = {
+	.free = spacemit_gem_free_object,
+	.get_sg_table = spacemit_gem_prime_get_sg_table,
+	.vmap = spacemit_gem_prime_vmap,
+	.vunmap = spacemit_gem_prime_vunmap,
+	.vm_ops = &vm_ops
+};
+
+static struct drm_gem_object *
+__spacemit_gem_create_object(struct drm_device *drm, size_t size)
+{
+	struct spacemit_gem_object *spacemit_obj;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	spacemit_obj = kzalloc(sizeof(*spacemit_obj), GFP_KERNEL);
+	if (!spacemit_obj)
+		return ERR_PTR(-ENOMEM);
+
+	gem_obj = &spacemit_obj->base;
+
+	gem_obj->funcs = &spacemit_gem_funcs;
+
+	drm_gem_private_object_init(drm, gem_obj, size);
+
+	ret = drm_gem_create_mmap_offset(gem_obj);
+	if (ret) {
+		drm_gem_object_release(gem_obj);
+		kfree(spacemit_obj);
+		return ERR_PTR(ret);
+	}
+
+	spacemit_obj->sgt = NULL;
+	spacemit_obj->vaddr = NULL;
+	spacemit_obj->vmap_cnt = 0;
+	mutex_init(&spacemit_obj->vmap_lock);
+
+	return gem_obj;
+}
+
+static struct drm_gem_object *
+spacemit_gem_create_object(struct drm_device *drm, size_t size)
+{
+	struct spacemit_gem_object *spacemit_obj;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	size = PAGE_ALIGN(size);
+
+	gem_obj = __spacemit_gem_create_object(drm, size);
+	if (IS_ERR(gem_obj))
+		return gem_obj;
+
+	spacemit_obj = to_spacemit_obj(gem_obj);
+	ret = spacemit_gem_alloc_buf(drm, spacemit_obj, size);
+	if (ret) {
+		drm_gem_object_put(gem_obj);
+		return ERR_PTR(ret);
+	}
+
+	return gem_obj;
+}
+
+static struct drm_gem_object *
+spacemit_gem_create_with_handle(struct drm_file *file_priv, struct drm_device *drm,
+						size_t size, unsigned int *handle)
+{
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	gem_obj = spacemit_gem_create_object(drm, size);
+	if (IS_ERR(gem_obj))
+		return gem_obj;
+
+	ret = drm_gem_handle_create(file_priv, gem_obj, handle);
+	drm_gem_object_put(gem_obj);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return gem_obj;
+}
+
+int spacemit_gem_dumb_create(struct drm_file *file_priv,
+				struct drm_device *drm,
+				struct drm_mode_create_dumb *args)
+{
+	struct drm_gem_object *gem_obj;
+
+	args->pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
+	args->size = args->pitch * args->height;
+
+	gem_obj = spacemit_gem_create_with_handle(file_priv, drm, args->size, &args->handle);
+
+	return PTR_ERR_OR_ZERO(gem_obj);
+}
+
+static int spacemit_gem_object_mmap(struct drm_gem_object *gem_obj,
+				struct vm_area_struct *vma)
+{
+	struct spacemit_gem_object *spacemit_obj = to_spacemit_obj(gem_obj);
+	unsigned long addr = vma->vm_start;
+	struct sg_page_iter piter;
+	int ret;
+
+	for_each_sgtable_page(spacemit_obj->sgt, &piter, vma->vm_pgoff) {
+		struct page *page = sg_page_iter_page(&piter);
+
+		ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
+						vma->vm_page_prot);
+		if (ret)
+			return ret;
+
+		addr += PAGE_SIZE;
+		if (addr >= vma->vm_end)
+			return 0;
+	}
+
+	return 0;
+}
+
+int spacemit_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	gem_obj = vma->vm_private_data;
+
+	vma->vm_pgoff -= drm_vma_node_start(&gem_obj->vma_node);
+
+	if (gem_obj->import_attach) {
+		drm_gem_object_put(gem_obj);
+		vma->vm_private_data = NULL;
+
+		return dma_buf_mmap(gem_obj->dma_buf, vma, vma->vm_pgoff);
+	}
+
+	return spacemit_gem_object_mmap(gem_obj, vma);
+}
+
+struct drm_gem_object *
+spacemit_gem_prime_import_sg_table(struct drm_device *drm,
+				struct dma_buf_attachment *attach,
+				struct sg_table *sgt)
+{
+	struct spacemit_gem_object *spacemit_obj;
+	struct drm_gem_object *gem_obj;
+	size_t size = PAGE_ALIGN(attach->dmabuf->size);
+
+	gem_obj = __spacemit_gem_create_object(drm, size);
+	if (IS_ERR(gem_obj))
+		return gem_obj;
+
+	spacemit_obj = to_spacemit_obj(gem_obj);
+	spacemit_obj->sgt = sgt;
+
+	return gem_obj;
+}
diff --git a/drivers/gpu/drm/spacemit/spacemit_gem.h b/drivers/gpu/drm/spacemit/spacemit_gem.h
new file mode 100644
index 000000000000..5a6be651b97c
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_gem.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_GEM_H_
+#define _SPACEMIT_GEM_H_
+
+#include <linux/scatterlist.h>
+#include <drm/drm_device.h>
+#include <drm/drm_gem.h>
+#include "spacemit_dmmu.h"
+
+struct spacemit_gem_object {
+	struct drm_gem_object base;
+	struct sg_table *sgt;
+	void *vaddr;
+	int vmap_cnt;
+	struct mutex vmap_lock;
+	struct list_head ttb_entry;
+};
+
+static inline struct spacemit_gem_object *
+to_spacemit_obj(struct drm_gem_object *gem_obj)
+{
+	return container_of(gem_obj, struct spacemit_gem_object, base);
+}
+
+struct drm_gem_object *
+spacemit_gem_prime_import_sg_table(struct drm_device *drm, struct dma_buf_attachment *attch,
+				  struct sg_table *sgt);
+int spacemit_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm,
+			struct drm_mode_create_dumb *args);
+int spacemit_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+
+#endif /* _SPACEMIT_GEM_H_ */
diff --git a/drivers/gpu/drm/spacemit/spacemit_inno_dp.c b/drivers/gpu/drm/spacemit/spacemit_inno_dp.c
new file mode 100644
index 000000000000..35a17a0586d9
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_inno_dp.c
@@ -0,0 +1,3176 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/component.h>
+#include <linux/clk.h>
+#include <linux/gpio.h>
+#include <linux/delay.h>
+#include <linux/iopoll.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+#include <linux/backlight.h>
+#include <drm/drm_of.h>
+#include <drm/drm_device.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_atomic_state_helper.h>
+#include <drm/display/drm_dp_aux_bus.h>
+#include <drm/display/drm_dp.h>
+#include <drm/display/drm_dp_helper.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+
+#include "spacemit_inno_dp.h"
+
+#define INVALID_GPIO	0xFFFFFFFF
+
+#define ACTIVATE_DO_DIV	1
+#define HPD_BYPASS	0
+
+#define HOT_PLUG_THREAD_ENABLED 1
+#define HPD_POLL_INTERVAL_MS    200
+
+#define SOC_DP_SWING_MAX  2
+#define SOC_DP_PREEMP_MAX 2
+#define SOC_DP_AUX_MAX_RETRIES 3
+
+#define SOC_DP_EDID_CHUNK_SIZE 16
+#define SOC_DP_DDC_SEGMENT_ADDR 0x30
+#define SOC_DP_SINK_READY_DELAY_MS 120
+#define SOC_DP_SINK_READY_RETRIES 3
+
+#define SOC_DP_APMU_CLK_CTRL	0x23c
+#define SOC_DP_QOS_BASE		0xd4282c00
+#define SOC_DP_QOS_SIZE		0x200
+#define SOC_DP_QOS_MUX_CTRL	0x12c
+
+#ifdef CONFIG_SOC_DP_DRIVER_QEMU
+#include <linux/proc_fs.h>
+#else
+#include <linux/clk.h>
+#include <linux/reset.h>
+#endif
+
+#if HOT_PLUG_THREAD_ENABLED
+#include <linux/workqueue.h>
+#endif
+
+#if ACTIVATE_DO_DIV
+#include <linux/math64.h>
+#endif
+
+/*
+ * Local definitions for Link Configuration.
+ * Decoupled from <drm/drm_dp_helper.h> to facilitate bare-metal porting.
+ */
+enum soc_dp_link_rate {
+	SOC_DP_LINK_RATE_1_62 = 1620000, /* 1.62 Gbps */
+	SOC_DP_LINK_RATE_2_70 = 2700000, /* 2.70 Gbps */
+	SOC_DP_LINK_RATE_5_40 = 5400000, /* 5.40 Gbps */
+	SOC_DP_LINK_RATE_8_10 = 8100000, /* 8.10 Gbps */
+};
+
+enum soc_dp_lane_count {
+	SOC_DP_LANE_1 = 1,
+	SOC_DP_LANE_2 = 2,
+	SOC_DP_LANE_4 = 4,
+};
+
+enum soc_video_format {
+	SOC_VIDEO_RGB_6BIT = 0,
+	SOC_VIDEO_RGB_8BIT = 1,
+	SOC_VIDEO_RGB_10BIT = 2,
+	SOC_VIDEO_RGB_12BIT = 3,
+	SOC_VIDEO_RGB_16BIT = 4,
+	SOC_VIDEO_YUV444_8BIT = 5,
+	SOC_VIDEO_YUV444_10BIT = 6,
+	SOC_VIDEO_YUV444_12BIT = 7,
+	SOC_VIDEO_YUV444_16BIT = 8,
+	SOC_VIDEO_YUV422_8BIT = 9,
+	SOC_VIDEO_YUV422_10BIT = 10,
+	SOC_VIDEO_YUV422_12BIT = 11,
+	SOC_VIDEO_YUV422_16BIT = 12,
+};
+
+enum soc_dp_ref_clk {
+	SOC_DP_REF_CLK_24M = 24000,
+	SOC_DP_REF_CLK_50M = 50000,
+};
+
+static const struct soc_dp_link_config {
+	enum soc_dp_link_rate rate;
+	enum soc_dp_lane_count lanes;
+} soc_dp_link_priority_table[] = {
+	/* --- Tier 1: Low Bandwidth (< 4 Gbps) --- */
+	{SOC_DP_LINK_RATE_1_62, SOC_DP_LANE_1}, /* 1.62 Gbps */
+	{SOC_DP_LINK_RATE_2_70, SOC_DP_LANE_1}, /* 2.70 Gbps */
+	{SOC_DP_LINK_RATE_1_62, SOC_DP_LANE_2}, /* 3.24 Gbps */
+
+	/* --- Tier 2: Medium Bandwidth (~5-6 Gbps) --- */
+	{SOC_DP_LINK_RATE_2_70, SOC_DP_LANE_2}, /* 5.40 Gbps */
+	{SOC_DP_LINK_RATE_1_62, SOC_DP_LANE_4}, /* 6.48 Gbps */
+
+	/* --- Tier 3: High Bandwidth (~10 Gbps) --- */
+	{SOC_DP_LINK_RATE_2_70, SOC_DP_LANE_4}, /* 10.8 Gbps */
+	{SOC_DP_LINK_RATE_5_40, SOC_DP_LANE_2}, /* 10.8 Gbps */
+
+	/* --- Tier 4: Ultra High Bandwidth (> 17 Gbps) --- */
+	{SOC_DP_LINK_RATE_5_40, SOC_DP_LANE_4}, /* 21.6 Gbps */
+};
+
+static const struct soc_format_info {
+	uint8_t bpp; /* Bits Per Pixel */
+} format_info_table[] = {
+	[SOC_VIDEO_RGB_6BIT]      = { .bpp = 18 },
+	[SOC_VIDEO_RGB_8BIT]      = { .bpp = 24 },
+	[SOC_VIDEO_RGB_10BIT]     = { .bpp = 30 },
+	[SOC_VIDEO_RGB_12BIT]     = { .bpp = 36 },
+	[SOC_VIDEO_RGB_16BIT]     = { .bpp = 48 },
+
+	[SOC_VIDEO_YUV444_8BIT]   = { .bpp = 24 },
+	[SOC_VIDEO_YUV444_10BIT]  = { .bpp = 30 },
+	[SOC_VIDEO_YUV444_12BIT]  = { .bpp = 36 },
+	[SOC_VIDEO_YUV444_16BIT]  = { .bpp = 48 },
+
+	[SOC_VIDEO_YUV422_8BIT]   = { .bpp = 16 },
+	[SOC_VIDEO_YUV422_10BIT]  = { .bpp = 20 },
+	[SOC_VIDEO_YUV422_12BIT]  = { .bpp = 24 },
+	[SOC_VIDEO_YUV422_16BIT]  = { .bpp = 32 },
+};
+
+static int soc_dp_get_bpp(uint32_t format)
+{
+	if (format >= ARRAY_SIZE(format_info_table)) {
+		pr_warn("DP: Invalid color format index %d, defaulting to RGB888\n", format);
+		return 24;
+	}
+	return format_info_table[format].bpp;
+}
+
+#define SOC_DP_VCO_MIN_KHZ        1000000
+#define SOC_DP_VCO_MAX_KHZ        3000000
+#define SOC_DP_PLL_FRAC_MOD       16777216  /* 2^24 */
+#define SOC_DP_PLL_ERR_TOLERANCE  10
+
+/* Data structure for Core PLL results */
+struct soc_dp_core_pll_cfg {
+	uint32_t target_rate_kbps;
+	uint32_t vco_freq_khz;
+	uint8_t prediv;
+	uint16_t fbdiv;
+	uint32_t frac;
+	uint8_t postdiv_reg;
+	uint8_t frac_pd;
+	uint8_t vcoclk_div8_en;
+	uint8_t postdiv_en;
+	uint8_t clk_16mdiv;
+	uint32_t actual_rate_khz;
+	bool valid;
+};
+
+static struct soc_dp_core_pll_cfg core_pll_cfg_table[] = {
+	/* LinkRate 1.62Gbps */
+	{
+		.target_rate_kbps = 1620000,
+		.vco_freq_khz     = 1620000,
+		.prediv           = 0x02,
+		.fbdiv            = 0x87,
+		.frac             = 0x0,
+		.postdiv_reg      = 0x0,
+		.frac_pd          = 0x3,
+		.vcoclk_div8_en   = 0x1,
+		.postdiv_en       = 0x1,
+		.clk_16mdiv       = 12,
+		.actual_rate_khz  = 1620000,
+		.valid            = true,
+	},
+	/* LinkRate 2.7Gbps */
+	{
+		.target_rate_kbps = 2700000,
+		.vco_freq_khz     = 2700000,
+		.prediv           = 0x02,
+		.fbdiv            = 0xe1,
+		.frac             = 0x0,
+		.postdiv_reg      = 0x0,
+		.frac_pd          = 0x3,
+		.vcoclk_div8_en   = 0x1,
+		.postdiv_en       = 0x1,
+		.clk_16mdiv       = 21,
+		.actual_rate_khz  = 2700000,
+		.valid            = true,
+	},
+	/* LinkRate 5.4Gbps */
+	{
+		.target_rate_kbps = 5400000,
+		.vco_freq_khz     = 2700000,
+		.prediv           = 0x02,
+		.fbdiv            = 0xe1,
+		.frac             = 0x0,
+		.postdiv_reg      = 0x0,
+		.frac_pd          = 0x3,
+		.vcoclk_div8_en   = 0x0,
+		.postdiv_en       = 0x0,
+		.clk_16mdiv       = 42,
+		.actual_rate_khz  = 5400000,
+		.valid            = true,
+	},
+};
+
+/* Data structure for Pixel PLL results */
+struct soc_dp_pixel_pll_cfg {
+	uint32_t target_pclk_khz;
+	uint32_t vco_freq_khz;
+	uint8_t prediv;
+	uint16_t fbdiv;
+	uint32_t frac_pd;
+	uint32_t frac;
+	uint8_t div5_en;
+	uint8_t divm;
+	uint8_t divaux;
+	uint8_t divp;
+	uint32_t actual_pclk_khz;
+	bool valid;
+};
+
+static const struct soc_dp_pixel_pll_cfg pixel_pll_cfg_table[] = {
+	{ 614400, 2460000, 0x05, 512,  0x3, 0x0, 0x0, 0x1, 0x01, 0x1, 614400, true },
+	{ 594000, 2376000, 0x01, 99,   0x3, 0x0, 0x0, 0x1, 0x01, 0x1, 594000, true },
+	{ 551040, 2760000, 0x05, 574,  0x3, 0x0, 0x1, 0x0, 0x00, 0x0, 551040, true },
+	{ 533250, 2130000, 0x08, 711,  0x3, 0x0, 0x0, 0x1, 0x01, 0x1, 533250, true },
+	{ 443250, 1770000, 0x08, 591,  0x3, 0x0, 0x0, 0x1, 0x01, 0x1, 443250, true },
+	{ 375000, 3000000, 0x01, 125,  0x3, 0x0, 0x0, 0x0, 0x04, 0x1, 375000, true },
+	{ 372000, 2980000, 0x01, 124,  0x3, 0x0, 0x0, 0x0, 0x04, 0x1, 372000, true },
+	{ 348500, 2790000, 0x06, 697,  0x3, 0x0, 0x0, 0x0, 0x04, 0x1, 348500, true },
+	{ 307200, 1540000, 0x01, 64,   0x3, 0x0, 0x1, 0x0, 0x00, 0x0, 307200, true },
+	{ 297000, 2376000, 0x01, 99,   0x3, 0x0, 0x0, 0x0, 0x04, 0x1, 297000, true },
+	{ 280000, 1676000, 0x01, 70,   0x3, 0x0, 0x0, 0xa, 0x01, 0x1, 280000, true },
+	{ 277440, 2770000, 0x05, 578,  0x3, 0x0, 0x0, 0x3, 0x01, 0x1, 277440, true },
+	{ 245760, 2457600, 0x05, 512,  0x3, 0x0, 0x0, 0x3, 0x01, 0x1, 245760, true },
+	{ 241500, 1932000, 0x02, 161,  0x3, 0x0, 0x0, 0x0, 0x04, 0x1, 241500, true },
+	{ 236000, 2830000, 0x01, 118,  0x3, 0x0, 0x0, 0x0, 0x06, 0x1, 236000, true },
+	{ 204800, 2048000, 0x03, 256,  0x3, 0x0, 0x0, 0x3, 0x01, 0x1, 204800, true },
+	{ 193250, 2320000, 0x08, 773,  0x3, 0x0, 0x0, 0x0, 0x06, 0x1, 193250, true },
+	{ 189000, 1510000, 0x01,  63,  0x3, 0x0, 0x0, 0x0, 0x04, 0x1, 189000, true },
+	{ 187500, 3000000, 0x01, 125,  0x3, 0x0, 0x0, 0x0, 0x08, 0x1, 187500, true },
+	{ 162000, 2592000, 0x01, 108,  0x3, 0x0, 0x0, 0x0, 0x08, 0x1, 162000, true },
+	{ 156000, 2810000, 0x01, 117,  0x3, 0x0, 0x0, 0x0, 0x09, 0x1, 156000, true },
+	{ 150000, 3000000, 0x01, 125,  0x3, 0x0, 0x0, 0x0, 0x0a, 0x1, 150000, true },
+	{ 148500, 2376000, 0x01, 99,   0x3, 0x0, 0x0, 0x0, 0x08, 0x1, 148500, true },
+	{ 146000, 1750000, 0x01, 73,   0x3, 0x0, 0x0, 0x0, 0x06, 0x1, 146000, true },
+	{ 142860, 2860000, 0x14, 2381, 0x3, 0x0, 0x0, 0x0, 0x0a, 0x1, 142860, true },
+	{ 140000, 2520000, 0x01, 105,  0x3, 0x0, 0x0, 0x0, 0x09, 0x1, 140000, true },
+	{ 138500, 2220000, 0x03, 277,  0x3, 0x0, 0x0, 0x0, 0x08, 0x1, 138500, true },
+	{ 122000, 2930000, 0x01, 122,  0x3, 0x0, 0x0, 0x0, 0x0c, 0x1, 122000, true },
+	{ 121750, 2920000, 0x04, 487,  0x3, 0x0, 0x0, 0x0, 0x0c, 0x1, 121750, true },
+	{ 108000, 2810000, 0x01, 117,  0x3, 0x0, 0x0, 0x0, 0x0d, 0x1, 108000, true },
+	{ 106500, 1700000, 0x01,  71,  0x3, 0x0, 0x0, 0x0, 0x08, 0x1, 106500, true },
+	{ 83500,  2000000, 0x02, 167,  0x3, 0x0, 0x0, 0x0, 0x0c, 0x1, 83500,  true },
+	{ 79500,  2540000, 0x01, 106,  0x3, 0x0, 0x0, 0x0, 0x10, 0x1, 79500,  true },
+	{ 75000,  3000000, 0x01, 125,  0x3, 0x0, 0x0, 0x0, 0x14, 0x1, 75000,  true },
+	{ 74250,  2376000, 0x01,  99,  0x3, 0x0, 0x0, 0x0, 0x10, 0x1, 74250,  true },
+	{ 65000,  1560000, 0x01,  65,  0x3, 0x0, 0x0, 0x0, 0x0c, 0x1, 65000,  true },
+	{ 40000,  2880000, 0x01, 120,  0x3, 0x0, 0x0, 0x0, 0x12, 0x2, 40000,  true },
+	{ 27000,  2810000, 0x01, 117,  0x3, 0x0, 0x0, 0x0, 0x1a, 0x2, 27000,  true },
+	{ 25600,  2300000, 0x01,  96,  0x3, 0x0, 0x0, 0x0, 0x0f, 0x3, 25600,  true },
+	{ 25200,  2520000, 0x01, 105,  0x3, 0x0, 0x0, 0x0, 0x19, 0x2, 25200,  true },
+};
+
+struct soc_dp_dev {
+	struct device *dev;
+
+	struct drm_device *drm;
+	struct drm_encoder encoder;
+	struct drm_connector connector;
+
+	enum drm_connector_status connector_status;
+	struct drm_display_mode mode;
+	struct backlight_device *backlight;
+
+	void __iomem *regs;
+	struct regmap *apmu;
+	struct regmap *qos;
+	struct drm_dp_aux aux;
+
+	/* Buffer to store raw DPCD data */
+	uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
+	int lane_count;
+	uint32_t link_rate;
+
+	/* Structure to store negotiated link parameters */
+	struct {
+		uint8_t revision;
+		uint8_t enhanced_framing;
+		uint32_t max_rate;
+		uint32_t max_num_lanes;
+	} link;
+
+	struct reset_control *reset;
+	struct clk *pxclk;
+
+	u32 gpio_bl;
+	u32 gpio_power;
+	u32 gpio_enable;
+
+	bool edp_mode;
+	int dpu_id;
+	bool use_ext_pixel_clock;
+	int pixel_clock;
+	struct mutex mode_lock;
+	bool suspended;
+
+	uint32_t ref_clk;
+	uint32_t color_format;
+
+#ifdef CONFIG_SOC_DP_DRIVER_QEMU
+	struct proc_dir_entry *proc_irq;
+#endif
+
+#if HOT_PLUG_THREAD_ENABLED
+	struct delayed_work hpd_work;
+#else
+	int irq;
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC)
+	uint32_t aud_mode;
+	bool card_instantiated;
+	bool aud_registered;
+#endif
+};
+
+static const struct regmap_config soc_dp_qos_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = SOC_DP_QOS_SIZE - 4,
+};
+
+static struct regmap *soc_dp_qos_init_regmap(struct device *dev,
+					      resource_size_t base,
+					      resource_size_t size,
+					      const struct regmap_config *config)
+{
+	void __iomem *regs;
+
+	regs = devm_ioremap(dev, base, size);
+	if (!regs)
+		return ERR_PTR(-ENOMEM);
+
+	return devm_regmap_init_mmio(dev, regs, config);
+}
+
+static int soc_dp_reg_write(struct soc_dp_dev *dp,
+		uint32_t offset, uint32_t bit_wide, uint32_t mask, uint32_t val)
+{
+	uint32_t reg_val;
+
+	reg_val = (uint32_t)readl((char *)dp->regs + offset);
+	reg_val &= ~mask;
+	reg_val |= val & mask;
+	DRM_DEBUG("%s() [W] 0x%x 0x%x\n", __func__, offset, reg_val);
+	writel(reg_val, (char *)dp->regs + offset);
+
+	return 0;
+}
+
+static int soc_dp_reg_read(struct soc_dp_dev *dp,
+		uint32_t offset, uint32_t bit_wide, uint32_t mask, uint32_t *val)
+{
+	*val = ((uint32_t)readl((char *)dp->regs + offset)) & mask;
+	DRM_DEBUG("%s() [R] 0x%x 0x%x\n", __func__, offset, *val);
+
+	return 0;
+}
+
+static int soc_dp_reg_write_range(struct soc_dp_dev *dp,
+		uint32_t offset, uint32_t high, uint32_t low, uint32_t val)
+{
+	uint32_t mask;
+
+	mask = (uint32_t)(((((uint64_t)1) << (high - low + 1)) - 1) << low);
+	return soc_dp_reg_write(dp, offset, 32, mask, (val << low) & mask);
+}
+
+static int soc_dp_reg_only_write_range(struct soc_dp_dev *dp,
+		uint32_t offset, uint32_t high, uint32_t low, uint32_t val)
+	{
+	uint32_t mask;
+
+	mask = (uint32_t)(((((uint64_t)1) << (high - low + 1)) - 1) << low);
+	writel((val << low) & mask, (char *)dp->regs + offset);
+	return 0;
+}
+
+static int soc_dp_reg_read_range(struct soc_dp_dev *dp,
+		uint32_t offset, uint32_t high, uint32_t low, uint32_t *val)
+{
+	int ret;
+	uint32_t mask;
+
+	mask = (uint32_t)(((((uint64_t)1) << (high - low + 1)) - 1) << low);
+	ret = soc_dp_reg_read(dp, offset, 32, mask, val);
+	*val = *val >> low;
+
+	return ret;
+}
+
+#if IS_ENABLED(CONFIG_SND_SOC)
+static int inno_dp_dai_probe(struct snd_soc_dai *dai)
+{
+	struct soc_dp_dev *dp = snd_soc_dai_get_drvdata(dai);
+
+	if (!dp) {
+		dev_err(dp->dev, "Failed to get soc_dp_dev from dai probe\n");
+		return -EINVAL;
+	}
+	dp->card_instantiated = true;
+
+	return 0;
+}
+
+static int inno_dp_dai_remove(struct snd_soc_dai *dai)
+{
+	struct soc_dp_dev *dp = snd_soc_dai_get_drvdata(dai);
+
+	if (!dp) {
+		dev_err(dp->dev, "Failed to get soc_dp_dev from dai remove\n");
+		return -EINVAL;
+	}
+	dp->card_instantiated = false;
+
+	return 0;
+}
+
+static int inno_dp_dai_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct soc_dp_dev *dp = snd_soc_dai_get_drvdata(dai);
+	uint32_t mode = 0x00;
+
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		mode = 0x00;
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		mode = 0x01;
+		break;
+	case SND_SOC_DAIFMT_RIGHT_J:
+		mode = 0x02;
+		break;
+	default:
+		return -EINVAL;
+	}
+	dp->aud_mode = mode;
+	soc_dp_reg_write_range(dp, SOC_DPTX_I2S_AUDIO_MODE, mode);
+	return 0;
+}
+
+static int inno_dp_dai_pcm_hw_params(struct snd_pcm_substream *substream,
+				     struct snd_pcm_hw_params *params,
+				     struct snd_soc_dai *dai)
+{
+	struct soc_dp_dev *dp = snd_soc_dai_get_drvdata(dai);
+	unsigned int data_bits = 0;
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		data_bits = 0x10;
+		break;
+	case SNDRV_PCM_FORMAT_S20_3LE:
+		data_bits = 0x14;
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		data_bits = 0x18;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_DATA_WIDTH, data_bits);
+	soc_dp_reg_write_range(dp, SOC_DPTX_I2S_AUDIO_MODE, dp->aud_mode);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_RESET, 1);
+
+	return 0;
+}
+
+static int inno_dp_dai_mute(struct snd_soc_dai *dai, int mute, int direction)
+{
+	struct soc_dp_dev *dp = snd_soc_dai_get_drvdata(dai);
+
+	if (mute)
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_MUTE, 1);
+	else
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_MUTE, 0);
+
+	return 0;
+}
+
+static int inno_dp_dai_trigger(struct snd_pcm_substream *substream,
+				int cmd, struct snd_soc_dai *dai)
+{
+	struct soc_dp_dev *dp = snd_soc_dai_get_drvdata(dai);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_RESET, 0);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_RESET, 1);
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+const struct snd_soc_dai_ops inno_dp_dai_ops = {
+	.probe = inno_dp_dai_probe,
+	.remove = inno_dp_dai_remove,
+	.hw_params = inno_dp_dai_pcm_hw_params,
+	.set_fmt = inno_dp_dai_set_dai_fmt,
+	.trigger = inno_dp_dai_trigger,
+	.mute_stream = inno_dp_dai_mute,
+	.no_capture_mute = 0,
+};
+
+struct snd_soc_dai_driver inno_dp_dai_driver = {
+	.name = "dp audio",
+	.playback = {
+		.stream_name = "Playback",
+		.channels_min = 2,
+		.channels_max = 2,
+		.rates = SNDRV_PCM_RATE_8000_48000,
+		.formats = SNDRV_PCM_FMTBIT_S16_LE
+			   | SNDRV_PCM_FMTBIT_S20_3LE
+			   | SNDRV_PCM_FMTBIT_S24_LE,
+		},
+	.ops = &inno_dp_dai_ops,
+};
+
+const struct snd_soc_component_driver soc_component_inno_dp = {
+	.name = "inno-dp-audio",
+};
+
+static int inno_dp_audio_register(struct device *dev)
+{
+	return snd_soc_register_component(dev,
+					  &soc_component_inno_dp,
+					  &inno_dp_dai_driver, 1);
+}
+
+static void inno_dp_audio_unregister(struct device *dev)
+{
+	snd_soc_unregister_component(dev);
+}
+#endif
+
+static uint32_t soc_dp_aux_get_cmd(struct drm_dp_aux_msg *msg)
+{
+	switch (msg->request & ~DP_AUX_I2C_MOT) {
+	case DP_AUX_NATIVE_WRITE:
+	case DP_AUX_I2C_WRITE:
+	case DP_AUX_I2C_WRITE_STATUS_UPDATE:
+		return msg->request;
+	case DP_AUX_NATIVE_READ:
+	case DP_AUX_I2C_READ:
+		return msg->request;
+	default:
+		return 0;
+	}
+
+	return 0;
+}
+
+static void soc_dp_aux_hw_reset(struct soc_dp_dev *dp)
+{
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_RESET, 0x1);
+	usleep_range(2000, 2500);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_RESET, 0x0);
+	usleep_range(2000, 2500);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_REPLY_EVENT_INT_STA, 1);
+}
+
+static bool soc_dp_dpcd_caps_valid(const u8 *dpcd)
+{
+	u8 max_bw = dpcd[DP_MAX_LINK_RATE];
+	u8 max_lanes = dpcd[DP_MAX_LANE_COUNT] & DP_MAX_LANE_COUNT_MASK;
+
+	if (!dpcd[DP_DPCD_REV])
+		return false;
+
+	switch (max_bw) {
+	case DP_LINK_BW_1_62:
+	case DP_LINK_BW_2_7:
+	case DP_LINK_BW_5_4:
+	case DP_LINK_BW_8_1:
+		break;
+	default:
+		return false;
+	}
+
+	return max_lanes == 1 || max_lanes == 2 || max_lanes == 4;
+}
+
+static ssize_t soc_dp_aux_transfer(struct drm_dp_aux *aux,
+				   struct drm_dp_aux_msg *msg)
+{
+	int ret, i, retries = 0;
+	unsigned long timeout;
+
+	struct soc_dp_dev *dp = container_of(aux, struct soc_dp_dev, aux);
+	uint32_t cmd, len, val, status;
+	uint32_t data[4] = {0};
+	uint8_t *buf = msg->buffer;
+	bool is_read = (msg->request & DP_AUX_I2C_READ) ||
+		((msg->request & DP_AUX_NATIVE_READ) == DP_AUX_NATIVE_READ);
+
+	/* 1. Check message validity */
+	if (msg->size > 16)
+		return -EINVAL;
+
+	cmd = soc_dp_aux_get_cmd(msg);
+
+retry_eio:
+	/* 2. Prepare Data for Write (if applicable) */
+	if (!is_read) {
+		/* Pack bytes into 32-bit words (Little Endian packing) */
+		for (i = 0; i < msg->size; i++) {
+			data[i / 4] |= buf[i] << ((i % 4) * 8);
+		}
+
+		/* Write data to registers: DATA1(LSB)..DATA4(MSB) */
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUX_DATA1, data[0]);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUX_DATA2, data[1]);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUX_DATA3, data[2]);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUX_DATA4, data[3]);
+	}
+
+	/* 3. Configure Command, Address, Length */
+	/* HW expects Length - 1 */
+	len = msg->size > 0 ? msg->size - 1 : 0;
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_LENGTH, len);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_ADDR, msg->address);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_CMD_TYPE, cmd);
+
+	/* 4. Trigger Transfer */
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_START, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_START, 1);
+
+	/* 5. Wait for Completion */
+	timeout = jiffies + msecs_to_jiffies(200);
+	ret = -ETIMEDOUT;
+
+	while (1) {
+		soc_dp_reg_read_range(dp, SOC_DPTX_AUX_REPLY_EVENT_INT_STA, &val);
+		if (val) {
+			ret = 0;
+			break;
+		}
+		if (time_after(jiffies, timeout))
+			break;
+		usleep_range(100, 110);
+	}
+
+	if (ret) {
+		if (retries < SOC_DP_AUX_MAX_RETRIES) {
+			retries++;
+			soc_dp_aux_hw_reset(dp);
+			dev_dbg(dp->dev,
+				"AUX timeout retry, cmd: 0x%x, addr: 0x%x, size %zu, retries %d/%d\n",
+				cmd, msg->address, msg->size, retries, SOC_DP_AUX_MAX_RETRIES);
+			goto retry_eio;
+		}
+
+		dev_err(dp->dev,
+			"AUX transfer timeout, req: 0x%x, cmd: 0x%x, addr: 0x%x, size %zu\n",
+			msg->request, cmd, msg->address, msg->size);
+		return ret;
+	}
+
+	/* 6. Read Status */
+	soc_dp_reg_read_range(dp, SOC_DPTX_AUX_STATUS, &status);
+
+	/* 7. Clear Interrupt Status (W1C) */
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_REPLY_EVENT_INT_STA, 1);
+
+	/* Map HW status to DRM reply codes */
+	switch (status) {
+	case 0: /* ACK */
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+		break;
+	case 1: /* NACK */
+		msg->reply = DP_AUX_NATIVE_REPLY_NACK;
+		return 0; /* Standard says return 0 on NACK for upper layer retry */
+	case 2: /* DEFER */
+		msg->reply = DP_AUX_NATIVE_REPLY_DEFER;
+		return 0;
+	default:
+		/* Check error code if status is weird */
+		soc_dp_reg_read_range(dp, SOC_DPTX_AUX_REPLY_ERR_CODE, &val);
+		if (retries < SOC_DP_AUX_MAX_RETRIES) {
+			retries++;
+			dev_dbg(dp->dev,
+				"AUX status retry, cmd: 0x%x, addr: 0x%x, size %zu, retries %d/%d\n",
+				cmd, msg->address, msg->size, retries, SOC_DP_AUX_MAX_RETRIES);
+			usleep_range(1000, 1100);
+			goto retry_eio;
+		}
+		dev_err(dp->dev, "AUX status error, cmd: 0x%x, address: 0x%x, size %zu, status: 0x%x, code: 0x%x\n",
+			cmd, msg->address, msg->size, status, val);
+		return -EIO;
+	}
+
+	/* 8. Read Data (if Read operation and ACK) */
+	if (is_read && msg->size > 0) {
+		soc_dp_reg_read_range(dp, SOC_DPTX_AUX_DATA1, &data[0]);
+		soc_dp_reg_read_range(dp, SOC_DPTX_AUX_DATA2, &data[1]);
+		soc_dp_reg_read_range(dp, SOC_DPTX_AUX_DATA3, &data[2]);
+		soc_dp_reg_read_range(dp, SOC_DPTX_AUX_DATA4, &data[3]);
+
+		/* Unpack 32-bit words back to bytes */
+		for (i = 0; i < msg->size; i++) {
+			buf[i] = (data[i / 4] >> ((i % 4) * 8)) & 0xFF;
+		}
+	}
+
+	return msg->size;
+}
+
+static void soc_dp_aux_init(struct soc_dp_dev *dp)
+{
+	DRM_INFO("%s() \n", __func__);
+
+	dp->aux.name = "soc-dp-aux";
+	dp->aux.dev = dp->dev;
+	dp->aux.drm_dev = dp->drm;
+	dp->aux.transfer = soc_dp_aux_transfer;
+	dp->aux.no_zero_sized = true;
+
+	drm_dp_aux_register(&dp->aux);
+}
+
+static uint64_t soc_dp_abs_diff(uint64_t a, uint64_t b)
+{
+	return (a > b) ? (a - b) : (b - a);
+}
+
+static uint32_t soc_dp_div64(uint64_t *n, uint32_t base)
+{
+#if ACTIVATE_DO_DIV
+	return do_div(*n, base);
+#else
+	uint32_t rem = *n % base;
+	*n = *n / base;
+	return rem;
+#endif
+}
+
+static uint32_t soc_dp_get_rate_khz(uint8_t pre, uint16_t fb, uint32_t frac,
+				    uint32_t ref_clk_khz, uint32_t total_div)
+{
+	uint64_t vco_hz;
+	uint64_t ref_hz = (uint64_t)ref_clk_khz * 1000;
+	uint64_t int_part;
+	uint64_t frac_part;
+
+	/* Integer part: (Ref * FB) / Pre */
+	int_part = ref_hz * fb;
+	int_part += (pre / 2);
+	soc_dp_div64(&int_part, pre);
+
+	/* Fractional part: (Ref * Frac) / (Pre * 2^24) */
+	frac_part = ref_hz * frac;
+
+	frac_part += (pre / 2);
+	soc_dp_div64(&frac_part, pre);
+
+	frac_part += (SOC_DP_PLL_FRAC_MOD / 2);
+	soc_dp_div64(&frac_part, SOC_DP_PLL_FRAC_MOD);
+
+	vco_hz = int_part + frac_part;
+
+	/* Final Rate = VCO / total_div */
+	vco_hz += (total_div / 2); // Rounding before final division
+	soc_dp_div64(&vco_hz, total_div);
+
+	vco_hz += 500; // Rounding for 1000
+	soc_dp_div64(&vco_hz, 1000);
+
+	return (uint32_t)vco_hz;
+}
+
+static int soc_dp_get_pixel_pll_div_total(const struct soc_dp_pixel_pll_cfg *cfg,
+					  uint32_t *div_total)
+{
+	static const uint8_t divm_factors[] = { 1, 2, 3, 5 };
+
+	if (!cfg || !div_total || !cfg->valid || !cfg->divp)
+		return -EINVAL;
+
+	if (cfg->div5_en) {
+		*div_total = 5;
+		return 0;
+	}
+
+	if (cfg->divaux == 1) {
+		if (cfg->divm >= ARRAY_SIZE(divm_factors))
+			return -EINVAL;
+
+		*div_total = 2 * divm_factors[cfg->divm] * cfg->divp;
+		return 0;
+	}
+
+	if (cfg->divaux < 2)
+		return -EINVAL;
+
+	*div_total = 2 * cfg->divaux * cfg->divp;
+	return 0;
+}
+
+static bool soc_dp_pixel_pll_cfg_matches(const struct soc_dp_pixel_pll_cfg *cfg,
+					 uint32_t target_pclk_khz,
+					 uint32_t ref_clk_khz)
+{
+	uint32_t div_total;
+	uint32_t actual_pclk_khz;
+
+	if (soc_dp_get_pixel_pll_div_total(cfg, &div_total))
+		return false;
+
+	actual_pclk_khz = soc_dp_get_rate_khz(cfg->prediv, cfg->fbdiv, cfg->frac,
+					      ref_clk_khz, div_total);
+
+	if (cfg->actual_pclk_khz &&
+	    soc_dp_abs_diff(actual_pclk_khz, cfg->actual_pclk_khz) > SOC_DP_PLL_ERR_TOLERANCE)
+		return false;
+
+	return soc_dp_abs_diff(actual_pclk_khz, target_pclk_khz) <= SOC_DP_PLL_ERR_TOLERANCE;
+}
+
+static const struct soc_dp_pixel_pll_cfg* find_pixel_pll_cfg(uint32_t pclk_khz,
+							      uint32_t ref_clk_khz) {
+	const struct soc_dp_pixel_pll_cfg *best_match = NULL;
+	uint32_t min_diff = 0xFFFFFFFF;
+	int num_configs = sizeof(pixel_pll_cfg_table) / sizeof(pixel_pll_cfg_table[0]);
+
+	for (int i = 0; i < num_configs; i++) {
+		uint32_t current_target = pixel_pll_cfg_table[i].target_pclk_khz;
+		uint32_t diff = (pclk_khz > current_target) ? (pclk_khz - current_target) : (current_target - pclk_khz);
+
+		if (diff == 0 && soc_dp_pixel_pll_cfg_matches(&pixel_pll_cfg_table[i],
+							      pclk_khz, ref_clk_khz)) {
+			return &pixel_pll_cfg_table[i];
+		}
+
+		if ((pclk_khz / 100) == (current_target / 100) &&
+		    soc_dp_pixel_pll_cfg_matches(&pixel_pll_cfg_table[i],
+						 pclk_khz, ref_clk_khz)) {
+			return &pixel_pll_cfg_table[i];
+		}
+
+		if (diff < min_diff && diff < 500 &&
+		    soc_dp_pixel_pll_cfg_matches(&pixel_pll_cfg_table[i],
+						 pclk_khz, ref_clk_khz)) {
+			min_diff = diff;
+			best_match = &pixel_pll_cfg_table[i];
+		}
+	}
+
+	return best_match;
+
+}
+
+static int update_edp_config(struct soc_dp_dev *dp, bool enable)
+{
+	uint8_t value;
+	int ret;
+
+	ret = drm_dp_dpcd_read(&dp->aux, DP_EDP_CONFIGURATION_SET, &value, 1);
+	if (ret < 0) {
+		dev_err(dp->dev, "Failed to read DP_EDP_CONFIGURATION_SET, ret: %d\n", ret);
+		return ret;
+	}
+
+	if (enable)
+		value |= 0x01;
+	else
+		value &= ~0x01;
+
+	ret = drm_dp_dpcd_write(&dp->aux, DP_EDP_CONFIGURATION_SET, &value, 1);
+	if (ret < 0) {
+		dev_err(dp->dev, "Failed to write DP_EDP_CONFIGURATION_SET, ret: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Read the Sink's DPCD capability information.
+ * Note: EDID is parsed separately. This function focuses solely on
+ * Link Layer capabilities (Rate, Lanes, etc.).
+ */
+static int soc_dp_hw_read_sink_caps(struct soc_dp_dev *dp)
+{
+#ifdef CONFIG_SOC_DP_DRIVER_QEMU
+	dp->link.revision = 0x14; // DP 1.4
+	dp->link.max_rate = SOC_DP_LINK_RATE_5_40;
+	dp->link.max_num_lanes = SOC_DP_LANE_4;
+	dp->link.enhanced_framing = 1;
+#else
+	ssize_t ret;
+	uint8_t max_bw;
+	int retry;
+
+	for (retry = 0; retry < SOC_DP_SINK_READY_RETRIES; retry++) {
+		if (retry) {
+			soc_dp_aux_hw_reset(dp);
+			msleep(SOC_DP_SINK_READY_DELAY_MS);
+		}
+
+		/* Read DPCD Receiver Capability fields (0x00000 - 0x0000F) */
+		ret = drm_dp_dpcd_read(&dp->aux, DP_DPCD_REV, dp->dpcd,
+				       DP_RECEIVER_CAP_SIZE);
+		if (ret < 0)
+			continue;
+
+		if (ret != DP_RECEIVER_CAP_SIZE || !soc_dp_dpcd_caps_valid(dp->dpcd)) {
+			dev_dbg(dp->dev,
+				"DPCD caps not ready: rev=0x%02x bw=0x%02x lanes=0x%02x\n",
+				dp->dpcd[DP_DPCD_REV], dp->dpcd[DP_MAX_LINK_RATE], dp->dpcd[DP_MAX_LANE_COUNT]);
+			ret = -EAGAIN;
+			continue;
+		}
+
+		break;
+	}
+
+	if (ret < 0) {
+		dev_err(dp->dev, "Failed to read DPCD after %d retries: %zd\n",
+			SOC_DP_SINK_READY_RETRIES, ret);
+		dp->link.revision = 0x14;
+		dp->link.max_rate = SOC_DP_LINK_RATE_5_40;
+		dp->link.max_num_lanes = SOC_DP_LANE_2;
+		dp->link.enhanced_framing = 1;
+		return ret;
+	}
+
+	/* 2. Parse DP Revision */
+	dp->link.revision = dp->dpcd[DP_DPCD_REV];
+
+	/*
+	 * 3. Parse and determine Link Rate.
+	 * Get the maximum link rate supported by the Sink.
+	 * Note: During link training, we usually start from min(Sink_Max, Source_Max).
+	 */
+	max_bw = dp->dpcd[DP_MAX_LINK_RATE];
+	switch (max_bw) {
+	case DP_LINK_BW_1_62:
+		dp->link.max_rate = SOC_DP_LINK_RATE_1_62;
+		break;
+	case DP_LINK_BW_2_7:
+		dp->link.max_rate = SOC_DP_LINK_RATE_2_70;
+		break;
+	case DP_LINK_BW_5_4:
+		dp->link.max_rate = SOC_DP_LINK_RATE_5_40;
+		break;
+	case DP_LINK_BW_8_1:
+		dp->link.max_rate = SOC_DP_LINK_RATE_8_10;
+		break;
+	default:
+		dev_warn(dp->dev, "Unknown DPCD Max Rate: 0x%x, defaulting to 5.40G\n", max_bw);
+		dp->link.revision = 0x14;
+		dp->link.max_rate = SOC_DP_LINK_RATE_5_40;
+		dp->link.max_num_lanes = SOC_DP_LANE_2;
+		dp->link.enhanced_framing = 1;
+		return -1;
+	}
+
+	/* 4. Parse and determine Lane Count */
+	dp->link.max_num_lanes = dp->dpcd[DP_MAX_LANE_COUNT] & DP_MAX_LANE_COUNT_MASK;
+
+	/* 5. Check for Enhanced Framing support */
+	dp->link.enhanced_framing =
+		(dp->dpcd[DP_MAX_LANE_COUNT] & DP_ENHANCED_FRAME_CAP);
+#endif
+
+	dev_info(dp->dev, "DPCD: Rev %x.%x, MaxRate %d kHz, MaxLanes %d, EnhFrame %d\n",
+		 dp->link.revision >> 4, dp->link.revision & 0xF,
+		 dp->link.max_rate,
+		 dp->link.max_num_lanes,
+		 dp->link.enhanced_framing);
+
+	return 0;
+}
+
+static int soc_dp_is_better_config(bool new_valid, bool new_is_int, uint8_t new_pre, uint32_t new_vco,
+				   bool best_valid, bool best_is_int, uint8_t best_pre, uint32_t best_vco)
+{
+	if (!new_valid) return 0;
+	if (!best_valid) return 1;
+
+	if (new_is_int && !best_is_int) return 1;
+	if (!new_is_int && best_is_int) return 0;
+
+	if (new_pre < best_pre) return 1;
+	if (new_pre > best_pre) return 0;
+
+	if (new_vco > best_vco) return 1;
+
+	return 0;
+}
+
+static int soc_dp_solve_pll_frac(uint32_t target_vco_khz, uint32_t ref_clk_khz,
+				 uint8_t *best_pre, uint16_t *best_fb, uint32_t *best_frac)
+{
+	uint64_t min_err = ~0ULL;
+	int found = 0;
+	bool best_is_int = false;
+	int pre;
+
+	/* Iterate pre-divider 1 to 63 to find best PFD frequency */
+	for (pre = 1; pre <= 63; pre++) {
+		uint64_t ref_clk_hz = (uint64_t)ref_clk_khz * 1000;
+		uint64_t target_vco_hz = (uint64_t)target_vco_khz * 1000;
+
+		/* Calculate required multiplier: Mult = (TargetVCO * Pre) / Ref */
+		uint64_t num = target_vco_hz * pre;
+		uint64_t den = ref_clk_hz;
+		uint64_t remainder;
+		uint64_t fb_val;
+		uint64_t frac_val;
+		uint64_t actual_vco;
+		uint64_t diff;
+		bool current_is_int;
+
+		fb_val = num;
+		remainder = soc_dp_div64(&fb_val, (uint32_t)den);
+
+		if (fb_val > 4095) continue;
+
+		/* Frac = (Remainder * 2^24 + Ref/2) / Ref */
+		frac_val = remainder * SOC_DP_PLL_FRAC_MOD;
+		frac_val += (den / 2);
+		soc_dp_div64(&frac_val, (uint32_t)den);
+
+		if (frac_val > 0xFFFFFF)
+			frac_val = 0xFFFFFF;
+
+		/*
+		* Calculate actual VCO for error checking.
+		* VCO = (Ref * FB / Pre) + (Ref * Frac / (Pre * 2^24))
+		*/
+		{
+			uint64_t vco_int, vco_frac;
+			/* Integer part: (Ref * FB) / Pre */
+			vco_int = ref_clk_hz * fb_val;
+			soc_dp_div64(&vco_int, pre);
+			/* Fractional part: (Ref * Frac) / (Pre * 2^24) */
+			vco_frac = ref_clk_hz * frac_val;
+			soc_dp_div64(&vco_frac, pre);
+			soc_dp_div64(&vco_frac, SOC_DP_PLL_FRAC_MOD);
+
+			actual_vco = vco_int + vco_frac;
+		}
+
+		diff = soc_dp_abs_diff(actual_vco, target_vco_hz);
+		current_is_int = (frac_val == 0);
+
+		if (diff < min_err) {
+			min_err = diff;
+			*best_pre = pre;
+			*best_fb = (uint16_t)fb_val;
+			*best_frac = (uint32_t)frac_val;
+			best_is_int = current_is_int;
+			found = 1;
+		} else if (diff == min_err) {
+			if (current_is_int && !best_is_int) {
+			*best_pre = pre;
+			*best_fb = (uint16_t)fb_val;
+			*best_frac = (uint32_t)frac_val;
+			best_is_int = true;
+			found = 1;
+			}
+		}
+	}
+
+	return found ? 0 : -1;
+}
+
+static int soc_dp_calc_pixel_pll(uint32_t target_pclk_khz, uint32_t ref_clk_khz, struct soc_dp_pixel_pll_cfg *cfg)
+{
+	struct soc_dp_pixel_pll_cfg best = {0};
+	int pclk_div;
+
+	/* Strategy 1: Div5 Path (VCO = PCLK * 5) */
+	{
+		struct soc_dp_pixel_pll_cfg curr = {0};
+		uint32_t div_total = 5;
+		uint32_t target_vco = target_pclk_khz * div_total;
+
+		if (target_vco >= SOC_DP_VCO_MIN_KHZ && target_vco <= SOC_DP_VCO_MAX_KHZ) {
+			uint8_t pre;
+			uint16_t fb;
+			uint32_t frac;
+
+			if (soc_dp_solve_pll_frac(target_vco, ref_clk_khz, &pre, &fb, &frac) == 0) {
+			uint32_t actual_pclk = soc_dp_get_rate_khz(pre, fb, frac, ref_clk_khz, div_total);
+
+			if (soc_dp_abs_diff(actual_pclk, target_pclk_khz) <= SOC_DP_PLL_ERR_TOLERANCE) {
+				curr.valid = true;
+				curr.vco_freq_khz = target_vco;
+				curr.actual_pclk_khz = actual_pclk;
+				curr.prediv = pre; curr.fbdiv = fb; curr.frac = frac;
+
+				curr.frac_pd = (frac == 0) ? 3 : 0;
+
+				curr.div5_en = 1;
+				curr.divaux = 0; curr.divm = 0; curr.divp = 0;
+
+				if (soc_dp_is_better_config(curr.valid, (curr.frac==0), curr.prediv, curr.vco_freq_khz,
+						best.valid, (best.frac==0), best.prediv, best.vco_freq_khz)) {
+					best = curr;
+				}
+			}
+			}
+		}
+	}
+
+	/* Strategy 2 & 3: Iterate PclkDiv (1 to 31) */
+	for (pclk_div = 1; pclk_div <= 31; pclk_div++) {
+
+		/* Strategy 2: DivM Path (DivAux = 1) */
+		int divm_factors[] = {1, 2, 3, 5};
+		int divm_regs[]    = {0, 1, 2, 3};
+		int i;
+
+		for (i = 0; i < 4; i++) {
+			struct soc_dp_pixel_pll_cfg curr = {0};
+			int m_val = divm_factors[i];
+			uint32_t div_total = 2 * m_val * pclk_div;
+			uint32_t target_vco = target_pclk_khz * div_total;
+			uint8_t pre;
+			uint16_t fb;
+			uint32_t frac;
+
+			if (target_vco < SOC_DP_VCO_MIN_KHZ || target_vco > SOC_DP_VCO_MAX_KHZ) continue;
+
+			if (soc_dp_solve_pll_frac(target_vco, ref_clk_khz, &pre, &fb, &frac) == 0) {
+			uint32_t actual_pclk = soc_dp_get_rate_khz(pre, fb, frac, ref_clk_khz, div_total);
+
+			if (soc_dp_abs_diff(actual_pclk, target_pclk_khz) > SOC_DP_PLL_ERR_TOLERANCE) continue;
+
+			curr.valid = true;
+			curr.vco_freq_khz = target_vco;
+			curr.actual_pclk_khz = actual_pclk;
+			curr.prediv = pre; curr.fbdiv = fb; curr.frac = frac;
+
+			curr.frac_pd = (frac == 0) ? 3 : 0;
+
+			curr.div5_en = 0;
+			curr.divaux = 1;         /* Must be 1 to enable DivM logic */
+			curr.divm = divm_regs[i];
+			curr.divp = pclk_div;
+
+			if (soc_dp_is_better_config(curr.valid, (curr.frac==0), curr.prediv, curr.vco_freq_khz,
+					best.valid, (best.frac==0), best.prediv, best.vco_freq_khz)) {
+				best = curr;
+			}
+			}
+		}
+
+		/* Strategy 3: DivAux Path (DivAux > 1) */
+		{
+			int aux;
+			for (aux = 2; aux <= 31; aux++) {
+			struct soc_dp_pixel_pll_cfg curr = {0};
+			uint32_t div_total = 2 * aux * pclk_div;
+			uint32_t target_vco = target_pclk_khz * div_total;
+			uint8_t pre;
+			uint16_t fb;
+			uint32_t frac;
+
+			if (target_vco < SOC_DP_VCO_MIN_KHZ || target_vco > SOC_DP_VCO_MAX_KHZ) continue;
+
+			if (soc_dp_solve_pll_frac(target_vco, ref_clk_khz, &pre, &fb, &frac) == 0) {
+				uint32_t actual_pclk = soc_dp_get_rate_khz(pre, fb, frac, ref_clk_khz, div_total);
+
+				if (soc_dp_abs_diff(actual_pclk, target_pclk_khz) > SOC_DP_PLL_ERR_TOLERANCE) continue;
+
+				curr.valid = true;
+				curr.vco_freq_khz = target_vco;
+				curr.actual_pclk_khz = actual_pclk;
+				curr.prediv = pre; curr.fbdiv = fb; curr.frac = frac;
+
+				curr.frac_pd = (frac == 0) ? 3 : 0;
+
+				curr.div5_en = 0;
+				curr.divaux = aux;
+				curr.divm = 0; /* Ignored when divaux != 1 */
+				curr.divp = pclk_div;
+
+				if (soc_dp_is_better_config(curr.valid, (curr.frac==0), curr.prediv, curr.vco_freq_khz,
+						best.valid, (best.frac==0), best.prediv, best.vco_freq_khz)) {
+				best = curr;
+				}
+			}
+			}
+		}
+	}
+
+	if (!best.valid)
+		return -EINVAL;
+
+	*cfg = best;
+	return 0;
+}
+
+static int soc_dp_check_pll_lock(struct soc_dp_dev *dp)
+{
+	uint32_t pll_locked;
+
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+	soc_dp_reg_read_range(dp, SOC_DPTX_AD_LOCK_PIXELPLL, &pll_locked);
+#else
+	pll_locked = 1;
+#endif
+	if (!pll_locked) {
+		dev_err(dp->dev, "Pre_pll unlocked.\n");
+		return -EINVAL;
+	}
+
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+	soc_dp_reg_read_range(dp, SOC_DPTX_AD_LOCK_COREPLL, &pll_locked);
+#else
+	pll_locked = 1;
+#endif
+	if (!pll_locked) {
+		dev_err(dp->dev, "Post_pll unlocked.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void soc_dp_calc_core_pll_to_reg(struct soc_dp_dev *dp, struct soc_dp_core_pll_cfg *cfg)
+{
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_PD, 1);
+	mdelay(2);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_PREDIV, cfg->prediv);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_FBDIV_LBIT, cfg->fbdiv & 0xFF);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_FBDIV_HBIT, (cfg->fbdiv >> 8) & 0xF);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_DACPD, (cfg->frac_pd >> 1) & 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_DSMPD, cfg->frac_pd & 0x1);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_FRAC_LBIT, cfg->frac & 0xFF);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_FRAC_MBIT, (cfg->frac >> 8) & 0xFF);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_FRAC_HBIT, (cfg->frac >> 16) & 0xFF);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_POSTDIV, cfg->postdiv_reg);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_POSTDIVEN, cfg->postdiv_en);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_VCOCLK_DIV8_EN, cfg->vcoclk_div8_en);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_CLKDIV_16M, cfg->clk_16mdiv);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_LOCK_BYPEN, 1);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_PD, 0);
+	mdelay(2);
+}
+
+static void soc_dp_calc_pixel_pll_to_reg(struct soc_dp_dev *dp, const struct soc_dp_pixel_pll_cfg *cfg)
+{
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PD, 1);
+	mdelay(2);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PREDIV, cfg->prediv);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_FBDIV2_LBIT, cfg->fbdiv & 0xFF);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_FBDIV2_HBIT, (cfg->fbdiv >> 8) & 0xF);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_DACPD, (cfg->frac_pd >> 1) & 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_DSMPD, cfg->frac_pd & 0x1);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_FRAC2_LBIT, cfg->frac & 0xFF);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_FRAC2_MBIT, (cfg->frac >> 8) & 0xFF);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_FRAC2_HBIT, (cfg->frac >> 16) & 0xFF);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PRECLK_DIVM, cfg->divm);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PRECLK_DIVAUX, cfg->divaux);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PCLKDIV5_EN, cfg->div5_en);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PCLK_DIVAUX, cfg->divp);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_REG_PCLK_OUTPUT_NORMAL, 1);
+	mdelay(2);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PD, 0);
+	mdelay(2);
+}
+
+static int soc_dp_hw_set_pll(struct soc_dp_dev *dp, enum soc_dp_link_rate rate, uint32_t pclk)
+{
+	const struct soc_dp_pixel_pll_cfg *pixel_pll_cfg;
+	int ret;
+
+	dev_info(dp->dev, "Setting PLL to Rate %d kHz, Pclk %d kHz\n", rate, pclk);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_DP_EN, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_HDMI_EN, 0);
+
+	if (rate == SOC_DP_LINK_RATE_1_62) {
+		soc_dp_calc_core_pll_to_reg(dp, &core_pll_cfg_table[0]);
+	} else if (rate == SOC_DP_LINK_RATE_2_70 ) {
+		soc_dp_calc_core_pll_to_reg(dp, &core_pll_cfg_table[1]);
+	} else if (rate == SOC_DP_LINK_RATE_5_40 ) {
+		soc_dp_calc_core_pll_to_reg(dp, &core_pll_cfg_table[2]);
+	} else {
+		dev_err(dp->dev, "Unsupported link rate %d\n", rate);
+		return -EINVAL;
+	}
+
+	pixel_pll_cfg = find_pixel_pll_cfg(pclk, dp->ref_clk);
+	if (pixel_pll_cfg) {
+		soc_dp_calc_pixel_pll_to_reg(dp, pixel_pll_cfg);
+	} else {
+		struct soc_dp_pixel_pll_cfg pll_cfg;
+
+		memset(&pll_cfg, 0, sizeof(pll_cfg));
+		ret = soc_dp_calc_pixel_pll(pclk, dp->ref_clk, &pll_cfg);
+		if (ret) {
+			dev_err(dp->dev, "Unsupported pixel clock %d\n", pclk);
+			return ret;
+		}
+		dev_info(dp->dev, "pixel pll config: prediv %d, fbdiv %d, frac_pd %d, frac %d, div5_en %d, divm %d, divaux %d, divp %d\n",
+			 pll_cfg.prediv, pll_cfg.fbdiv, pll_cfg.frac_pd, pll_cfg.frac, pll_cfg.div5_en, pll_cfg.divm, pll_cfg.divaux, pll_cfg.divp);
+		soc_dp_calc_pixel_pll_to_reg(dp, &pll_cfg);
+	}
+
+	return 0;
+}
+
+static void soc_dp_phy_config_lanes(struct soc_dp_dev *dp, enum soc_dp_lane_count lanes)
+{
+	uint32_t phy_lanes_val;
+
+	switch (lanes) {
+	case SOC_DP_LANE_1:
+		phy_lanes_val = 0;   /* Register value for 1 Lane */
+		break;
+	case SOC_DP_LANE_2:
+		phy_lanes_val = 1;   /* Register value for 2 Lanes */
+		break;
+	case SOC_DP_LANE_4:
+	default:
+		phy_lanes_val = 2;   /* Register value for 4 Lanes */
+		break;
+	}
+
+	dev_dbg(dp->dev, "Configuring PHY Lane Count: %d (Reg: %d)\n",
+		 lanes, phy_lanes_val);
+
+	/* Set the number of active lanes */
+	soc_dp_reg_write_range(dp, SOC_DPTX_PHY_NUM_LANES, phy_lanes_val);
+	dp->lane_count = lanes;
+}
+
+/*
+ * Check Hot Plug Detect (HPD) Status
+ */
+static enum drm_connector_status soc_dp_hw_detect_hpd(struct soc_dp_dev *dp)
+{
+#if defined(CONFIG_SOC_DP_DRIVER_QEMU) || HPD_BYPASS
+	/* QEMU Environment: Always simulate as Connected */
+	return connector_status_connected;
+#else
+	uint32_t hpd_status;
+	enum drm_connector_status connector_status = connector_status_disconnected;
+
+	soc_dp_reg_read_range(dp, SOC_DPTX_HPD_IN_STATUS, &hpd_status);
+	if (hpd_status)
+		connector_status = connector_status_connected;
+	else
+		connector_status = connector_status_disconnected;
+
+	return connector_status;
+#endif
+}
+
+/*
+ * Clean Hot Plug Detect (HPD) Status
+ */
+static void soc_dp_hw_clean_hpd(struct soc_dp_dev *dp)
+{
+#if defined(CONFIG_SOC_DP_DRIVER_QEMU) || HPD_BYPASS
+	return;
+#else
+	uint32_t plug_event, unplug_event;
+
+	soc_dp_reg_read_range(dp, SOC_DPTX_HOT_PLUG_EVENT, &plug_event);
+	soc_dp_reg_read_range(dp, SOC_DPTX_HOT_UNPLUG_EVENT, &unplug_event);
+
+	if (plug_event)
+		soc_dp_reg_only_write_range(dp, SOC_DPTX_HOT_PLUG_EVENT, 0x1);
+
+	if (unplug_event)
+		soc_dp_reg_only_write_range(dp, SOC_DPTX_HOT_UNPLUG_EVENT, 0x1);
+#endif
+}
+
+static int soc_dp_phy_power_on(struct soc_dp_dev *dp)
+{
+	int ret, retry;
+	uint32_t lane_en;
+
+	switch (dp->lane_count) {
+	case SOC_DP_LANE_1:
+		lane_en = 0x1;
+		break;
+	case SOC_DP_LANE_2:
+		lane_en = 0x3;
+		break;
+	case SOC_DP_LANE_4:
+		default:
+		lane_en = 0xF;
+		break;
+	}
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_PD, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PD, 0);
+	mdelay(2);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_XMIT_ENABLE, lane_en);
+	mdelay(2);
+
+	for (retry = 0; retry < 3; retry++) {
+		ret = soc_dp_check_pll_lock(dp);
+		if (!ret)
+			break;
+		mdelay(2);
+	}
+
+	return ret;
+}
+
+static int soc_dp_phy_power_off(struct soc_dp_dev *dp)
+{
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_STREAM_ENABLE, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_XMIT_ENABLE, 0);
+	mdelay(2);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PD, 1);
+	mdelay(2);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_RESET, 0x1);
+	mdelay(5);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_RESET, 0x0);
+	mdelay(2);
+
+	return 0;
+}
+
+static int soc_dp_set_training_pattern(struct soc_dp_dev *dp, uint8_t pattern)
+{
+	uint32_t tps_sel = 0;
+	uint8_t dpcd_pattern = pattern;
+	int ret;
+
+	if (pattern != DP_TRAINING_PATTERN_DISABLE)
+		dpcd_pattern |= DP_LINK_SCRAMBLING_DISABLE;
+
+	/* Configure PHY Pattern */
+	switch (pattern) {
+	case DP_TRAINING_PATTERN_DISABLE:
+		tps_sel = 0;
+		soc_dp_reg_write_range(dp, SOC_DPTX_SCRAMBLER_DISABLE, 0);
+		break;
+	case DP_TRAINING_PATTERN_1:
+		tps_sel = 1;
+		soc_dp_reg_write_range(dp, SOC_DPTX_SCRAMBLER_DISABLE, 1);
+		break;
+	case DP_TRAINING_PATTERN_2:
+		tps_sel = 2;
+		soc_dp_reg_write_range(dp, SOC_DPTX_SCRAMBLER_DISABLE, 1);
+		break;
+	case DP_TRAINING_PATTERN_3:
+		tps_sel = 3;
+		soc_dp_reg_write_range(dp, SOC_DPTX_SCRAMBLER_DISABLE, 1);
+		break;
+	default:
+		dev_err(dp->dev, "Unsupported training pattern: 0x%x\n", pattern);
+		return -EINVAL;
+	}
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_TPS_SEL, tps_sel);
+
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+	/* Configure DPCD Pattern */
+	ret = drm_dp_dpcd_writeb(&dp->aux, DP_TRAINING_PATTERN_SET, dpcd_pattern);
+	if (ret < 0) {
+		dev_err(dp->dev, "Failed to set DPCD training pattern: %d\n", ret);
+		return ret;
+	}
+#else
+	ret = 0;
+	return ret;
+#endif
+
+	return 0;
+}
+
+/*
+ * Configure PHY Rate Register
+ * This must be called before Link Training.
+ */
+static void soc_dp_phy_config_rate(struct soc_dp_dev *dp, enum soc_dp_link_rate rate)
+{
+	uint32_t rate_val = 0;
+
+	switch (rate) {
+	case SOC_DP_LINK_RATE_1_62:
+		rate_val = 0;
+		break;
+	case SOC_DP_LINK_RATE_2_70:
+		rate_val = 1;
+		break;
+	case SOC_DP_LINK_RATE_5_40:
+		rate_val = 2;
+		break;
+	case SOC_DP_LINK_RATE_8_10:
+		rate_val = 3;
+		break;
+	default:
+		dev_err(dp->dev, "Invalid Link Rate: %d\n", rate);
+		rate_val = 0;
+		break;
+	}
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_PHY_RATE, rate_val);
+	dp->link_rate = rate;
+}
+
+static int soc_dp_link_train_clock_recovery(struct soc_dp_dev *dp, enum soc_dp_link_rate rate, enum soc_dp_lane_count lanes)
+{
+	uint8_t link_status[DP_LINK_STATUS_SIZE];
+	uint8_t training_set[4] = {0};
+	int retries = 0;
+	int i, ret;
+
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+	ret = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET,
+				training_set, lanes);
+	if (ret < 0)
+		return ret;
+#endif
+
+	ret = soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_1);
+	if (ret < 0) {
+		soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+		return ret;
+	}
+
+	while (retries < 8) {
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+		drm_dp_link_train_clock_recovery_delay(&dp->aux, dp->dpcd);
+
+		ret = drm_dp_dpcd_read_link_status(&dp->aux, link_status);
+		if (ret < 0) {
+			soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+			return ret;
+		}
+
+		if (drm_dp_clock_recovery_ok(link_status, lanes))
+#else
+		if (1)
+#endif
+			return 0;
+
+		/* Update settings based on Sink request */
+		for (i = 0; i < lanes; i++) {
+			uint8_t v = drm_dp_get_adjust_request_voltage(link_status, i);
+			uint8_t p = drm_dp_get_adjust_request_pre_emphasis(link_status, i);
+
+			if (v >= SOC_DP_SWING_MAX) {
+				v = SOC_DP_SWING_MAX;
+				v |= DP_TRAIN_MAX_SWING_REACHED;
+			}
+
+			if (p >= SOC_DP_PREEMP_MAX) {
+				p = SOC_DP_PREEMP_MAX;
+				v |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
+			}
+
+			training_set[i] = v | (p << DP_TRAIN_PRE_EMPHASIS_SHIFT);
+		}
+
+		ret = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET,
+					training_set, lanes);
+		if (ret < 0) {
+			soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+			return ret;
+		}
+
+		retries++;
+	}
+
+	dev_err(dp->dev, "Link Training Clock Recovery Failed\n");
+	soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+	return -ETIMEDOUT;
+}
+
+static int soc_dp_link_train_channel_eq(struct soc_dp_dev *dp, enum soc_dp_link_rate rate, enum soc_dp_lane_count lanes)
+{
+	uint8_t link_status[DP_LINK_STATUS_SIZE];
+	uint8_t training_set[4] = {0};
+	uint8_t training_pattern = DP_TRAINING_PATTERN_2;
+	int retries = 0;
+	int i, ret;
+
+	if (dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED) {
+		training_pattern = DP_TRAINING_PATTERN_3;
+		dev_info(dp->dev, "Link Training: Using TPS3\n");
+	} else {
+		dev_info(dp->dev, "Link Training: Using TPS2\n");
+	}
+
+	ret = soc_dp_set_training_pattern(dp, training_pattern);
+	if (ret < 0) {
+		soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+		return ret;
+	}
+
+	while (retries < 8) {
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+		drm_dp_link_train_channel_eq_delay(&dp->aux, dp->dpcd);
+
+		ret = drm_dp_dpcd_read_link_status(&dp->aux, link_status);
+		if (ret < 0) {
+			soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+			return ret;
+		}
+
+		if (drm_dp_channel_eq_ok(link_status, lanes)) {
+#else
+		if (1) {
+#endif
+			soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+			return 0;
+		}
+
+		/* Update settings based on Sink request */
+		for (i = 0; i < lanes; i++) {
+			uint8_t v = drm_dp_get_adjust_request_voltage(link_status, i);
+			uint8_t p = drm_dp_get_adjust_request_pre_emphasis(link_status, i);
+
+			if (v >= SOC_DP_SWING_MAX) {
+				v = SOC_DP_SWING_MAX;
+				v |= DP_TRAIN_MAX_SWING_REACHED;
+			}
+
+			if (p >= SOC_DP_PREEMP_MAX) {
+				p = SOC_DP_PREEMP_MAX;
+				v |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
+			}
+
+			training_set[i] = v | (p << DP_TRAIN_PRE_EMPHASIS_SHIFT);
+		}
+
+		ret = drm_dp_dpcd_write(&dp->aux, DP_TRAINING_LANE0_SET,
+					training_set, lanes);
+		if (ret < 0) {
+			soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+			return ret;
+		}
+
+		retries++;
+	}
+
+	dev_err(dp->dev, "Link Training Channel EQ Failed\n");
+	soc_dp_set_training_pattern(dp, DP_TRAINING_PATTERN_DISABLE);
+	return -ETIMEDOUT;
+}
+
+/*
+ * Main Link Training Function
+ */
+static int soc_dp_link_train(struct soc_dp_dev *dp, enum soc_dp_link_rate rate, enum soc_dp_lane_count lanes)
+{
+	int ret;
+	uint8_t link_config[2];
+	uint8_t bw_code;
+
+	/* Map Link Rate Enum to DPCD Bandwidth Code */
+	switch (rate) {
+	case SOC_DP_LINK_RATE_1_62:
+		bw_code = DP_LINK_BW_1_62;
+		break;
+	case SOC_DP_LINK_RATE_2_70:
+		bw_code = DP_LINK_BW_2_7;
+		break;
+	case SOC_DP_LINK_RATE_5_40:
+		bw_code = DP_LINK_BW_5_4;
+		break;
+	case SOC_DP_LINK_RATE_8_10:
+		bw_code = DP_LINK_BW_8_1;
+		break;
+	default:
+		bw_code = DP_LINK_BW_1_62;
+		break;
+	}
+
+	/* Configure DPCD Link Rate and Lane Count */
+	link_config[0] = bw_code;
+	link_config[1] = lanes;
+	if (dp->link.enhanced_framing)
+		link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
+
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+	ret = drm_dp_dpcd_write(&dp->aux, DP_LINK_BW_SET, link_config, 2);
+	if (ret < 0) {
+		dev_err(dp->dev, "Failed to configure DPCD\n");
+		return ret;
+	}
+#endif
+
+	ret = soc_dp_link_train_clock_recovery(dp, rate, lanes);
+	if (ret)
+		return ret;
+
+	ret = soc_dp_link_train_channel_eq(dp, rate, lanes);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void soc_dp_hw_set_msa_and_enable_video(struct soc_dp_dev *dp, const struct drm_display_mode *mode,
+		enum soc_dp_link_rate rate, enum soc_dp_lane_count lanes)
+{
+	uint64_t hb_num;
+	uint32_t link_rate;
+	uint32_t fp; // Pixel clock in MHz
+	uint32_t bpp, misc0;
+	uint32_t tu, tu_frac, tu_int, rd_thres;
+	uint32_t hsync_len;
+
+	// 1. Prepare basic parameters
+	// mode->clock unit is kHz, fp unit is MHz
+	if (dp->use_ext_pixel_clock)
+		fp = dp->pixel_clock / 1000;
+	else
+		fp = mode->clock / 1000;
+
+	if (fp == 0) fp = 1; // Prevent division by zero
+
+	// Get BPP
+	bpp = soc_dp_get_bpp(dp->color_format);
+
+	// Calculate MISC0
+	// bit0: 0 (Sync Clock)
+	// bits1-7: Color Format (000=RGB, 001=YCbCr422, 010=YCbCr444)
+	// bits5-7: BPC (001=8bpc, 010=10bpc, etc)
+	switch (dp->color_format) {
+	case SOC_VIDEO_RGB_6BIT:      misc0 = 0x00; break;
+	case SOC_VIDEO_RGB_8BIT:      misc0 = 0x20; break;
+	case SOC_VIDEO_RGB_10BIT:     misc0 = 0x40; break;
+	case SOC_VIDEO_RGB_12BIT:     misc0 = 0x60; break;
+	case SOC_VIDEO_RGB_16BIT:     misc0 = 0x80; break;
+	case SOC_VIDEO_YUV422_8BIT:   misc0 = 0x22; break;
+	case SOC_VIDEO_YUV422_10BIT:  misc0 = 0x42; break;
+	case SOC_VIDEO_YUV422_12BIT:  misc0 = 0x62; break;
+	case SOC_VIDEO_YUV422_16BIT:  misc0 = 0x82; break;
+	case SOC_VIDEO_YUV444_8BIT:   misc0 = 0x24; break;
+	case SOC_VIDEO_YUV444_10BIT:  misc0 = 0x44; break;
+	case SOC_VIDEO_YUV444_12BIT:  misc0 = 0x64; break;
+	case SOC_VIDEO_YUV444_16BIT:  misc0 = 0x84; break;
+	default:                      misc0 = 0x20; break;
+	}
+
+	// 2. Calculate HBlank Interval (hb_num)
+	// (htotal - hactive) * (LinkSymbolClock / 4) / PixelClock
+	// LinkSymbolClock = LinkRate * 100 (e.g., 1.62G -> 162MHz)
+	// rate unit is kHz (e.g., 1620000)
+	// link_rate = rate / 10000 (e.g., 162)
+	link_rate = rate / 10000;
+
+	// Formula: hb_num = hblank * (link_rate / 4) / fp
+	// To avoid floating point arithmetic, multiply first then divide
+	hb_num = (uint64_t)(mode->htotal - mode->hdisplay) * link_rate;
+	do_div(hb_num, 4 * fp);
+
+	// 3. Calculate TU (Transfer Unit)
+	// tu = fp * bpp * 640 / (8 * num_cnt * link_rate)
+	// Here link_rate also refers to 162, 270 etc.
+#if ACTIVATE_DO_DIV
+	{
+		uint64_t temp_tu = (uint64_t)fp * bpp * 640;
+		uint32_t den = 8 * lanes * link_rate;
+		do_div(temp_tu, den);
+		tu = temp_tu;
+	}
+#else
+	tu = (uint64_t)fp * bpp * 640 / (8 * lanes * link_rate);
+#endif
+	tu_frac = tu % 10;
+	tu_int  = tu / 10;
+
+	// 4. Calculate FIFO read threshold
+	if (tu_int < 6) {
+		rd_thres = 32;
+	} else if ((mode->htotal - mode->hdisplay) < 80) {
+		rd_thres = 12;
+	} else {
+		rd_thres = 16;
+	}
+
+	dev_info(dp->dev, "MSA: %dx%d, Rate:%d kHz, Lanes:%d, BPP:%d, TU:%d.%d\n",
+		 mode->hdisplay, mode->vdisplay, rate, lanes, bpp, tu_int, tu_frac);
+
+	// 5. Video mapping format
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_MAPPING, dp->color_format);
+
+	// Polarity configuration
+	if (mode->flags & DRM_MODE_FLAG_PHSYNC)
+		soc_dp_reg_write_range(dp, SOC_DPTX_HSYNC_IN_POLARITY, 1);
+	else
+		soc_dp_reg_write_range(dp, SOC_DPTX_HSYNC_IN_POLARITY, 0);
+
+	if (mode->flags & DRM_MODE_FLAG_PVSYNC)
+		soc_dp_reg_write_range(dp, SOC_DPTX_VSYNC_IN_POLARITY, 1);
+	else
+		soc_dp_reg_write_range(dp, SOC_DPTX_VSYNC_IN_POLARITY, 0);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_HSYNC_IN_POLARITY, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VSYNC_IN_POLARITY, 1);
+
+	// Basic timing
+	soc_dp_reg_write_range(dp, SOC_DPTX_HACTIVE, mode->hdisplay);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VACTIVE, mode->vdisplay);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HBLANK, mode->htotal - mode->hdisplay);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VBLANK, mode->vtotal - mode->vdisplay);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_HSTART, mode->htotal - mode->hsync_end + (mode->hsync_end - mode->hsync_start));
+	soc_dp_reg_write_range(dp, SOC_DPTX_VSTART, mode->vtotal - mode->vsync_end + (mode->vsync_end - mode->vsync_start));
+
+	hsync_len = mode->hsync_end - mode->hsync_start;
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_H_SYNC_WIDTH, hsync_len);
+	soc_dp_reg_write_range(dp, SOC_DPTX_V_SYNC_WIDTH, mode->vsync_end - mode->vsync_start);
+	soc_dp_reg_write_range(dp, SOC_DPTX_H_FRONT_PORCH, mode->hsync_start - mode->hdisplay);
+	soc_dp_reg_write_range(dp, SOC_DPTX_V_FRONT_PORCH, mode->vsync_start - mode->vdisplay);
+
+	// MSA and MISC
+	soc_dp_reg_write_range(dp, SOC_DPTX_MISC0, misc0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_MISC1, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_NVID, 0);
+
+	// Link layer parameters
+	soc_dp_reg_write_range(dp, SOC_DPTX_HBLANK_INTERVAL, (uint32_t)hb_num);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AVERAGE_BYTES_PER_TU, tu_int);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AVERAGE_BYTES_PER_TU_FRAC, tu_frac);
+	soc_dp_reg_write_range(dp, SOC_DPTX_INIT_THRESHOLD, rd_thres);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_PHY_SSC_DIS, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_REG_VID_CLK_SEL, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VID_BIST_EN, 0);
+
+	// 6. Enable video stream
+	dev_dbg(dp->dev, "Enabling Video Stream...\n");
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_STREAM_ENABLE, 1);
+}
+
+static void soc_dp_hw_disable(struct soc_dp_dev *dp)
+{
+	dev_info(dp->dev, "Disabling Video & PHY\n");
+
+	/* 1. Disable Video Stream */
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_STREAM_ENABLE, 0);
+
+	/* 2. Disable Transmitters */
+	soc_dp_reg_write_range(dp, SOC_DPTX_XMIT_ENABLE, 0);
+
+	/* Disable LDO */
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_LDO_D0, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_LDO_D1, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_LDO_D2, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_LDO_D3, 0);
+
+	/* Disable Driver */
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_DRV_D0, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_DRV_D1, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_DRV_D2, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_DRV_D3, 0);
+
+	/* Disable P2S */
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_P2S_D0, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_P2S_D1, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_P2S_D2, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_EN_P2S_D3, 0);
+
+	/*Disable BG_EN */
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_BG_EN, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_BG_EN_RCAL, 0);
+
+	/* 3. Power Down PLLs (MPLL and PREPLL) */
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_PD, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PD, 1);
+}
+
+/* Calculate required bandwidth in kbps (Pixel Clock * Bits Per Pixel) */
+static uint32_t soc_dp_calc_required_bw(const struct drm_display_mode *mode, int bpp)
+{
+	return mode->clock * bpp;
+}
+
+/* Calculate available link capacity in kbps (taking 8b/10b overhead into account) */
+static uint32_t soc_dp_calc_link_capacity(enum soc_dp_link_rate rate, enum soc_dp_lane_count lanes)
+{
+	/* Capacity = Rate(kHz) * Lanes * 0.8 */
+	return (rate * lanes * 8) / 10;
+}
+
+static enum drm_connector_status soc_dp_conn_detect(struct drm_connector *connector, bool force)
+{
+	struct soc_dp_dev *dp = container_of(connector, struct soc_dp_dev, connector);
+
+	enum drm_connector_status status;
+
+	mutex_lock(&dp->mode_lock);
+	status = dp->connector_status;
+	mutex_unlock(&dp->mode_lock);
+
+	return status;
+}
+
+static int
+soc_dp_conn_probe_single_connector_modes(struct drm_connector *connector,
+				       uint32_t maxX, uint32_t maxY)
+{
+	return drm_helper_probe_single_connector_modes(connector, 3840, 2160);
+}
+
+static const struct drm_connector_funcs soc_dp_connector_funcs = {
+	.fill_modes = soc_dp_conn_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.detect = soc_dp_conn_detect,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int soc_dp_aux_i2c_write(struct soc_dp_dev *dp, u32 address,
+				const u8 *buf, size_t len)
+{
+	struct drm_dp_aux_msg msg = {
+		.address = address,
+		.request = DP_AUX_I2C_WRITE,
+		.buffer = (u8 *)buf,
+		.size = len,
+	};
+	int ret;
+
+	ret = soc_dp_aux_transfer(&dp->aux, &msg);
+	if (ret < 0)
+		return ret;
+
+	return ret == len ? 0 : -EIO;
+}
+
+static int soc_dp_aux_i2c_read(struct soc_dp_dev *dp, u32 address,
+			       u8 *buf, size_t len)
+{
+	struct drm_dp_aux_msg msg = {
+		.address = address,
+		.request = DP_AUX_I2C_READ,
+		.buffer = buf,
+		.size = len,
+	};
+	int ret;
+
+	ret = soc_dp_aux_transfer(&dp->aux, &msg);
+	if (ret < 0)
+		return ret;
+
+	return ret == len ? 0 : -EIO;
+}
+
+static int soc_dp_conn_get_edid_block(void *data, u8 *buf,
+				      unsigned int block, size_t len)
+{
+	struct soc_dp_dev *dp = data;
+	unsigned int start = block * EDID_LENGTH;
+	u8 segment = block >> 1;
+	int ret, retry;
+	size_t offset;
+
+	if (segment) {
+		for (retry = 0; retry < 3; retry++) {
+			ret = soc_dp_aux_i2c_write(dp, SOC_DP_DDC_SEGMENT_ADDR,
+						   &segment, 1);
+			if (!ret)
+				break;
+		}
+
+		if (ret) {
+			dev_err(dp->dev,
+				"[EDID] segment write failed, block %u segment %u ret %d\n",
+				block, segment, ret);
+			return ret;
+		}
+	}
+
+	for (offset = 0; offset < len; offset += SOC_DP_EDID_CHUNK_SIZE) {
+		u8 edid_offset = (start + offset) & 0xff;
+		size_t chunk = min_t(size_t, SOC_DP_EDID_CHUNK_SIZE, len - offset);
+
+		for (retry = 0; retry < 3; retry++) {
+			ret = soc_dp_aux_i2c_write(dp, DDC_ADDR, &edid_offset, 1);
+			if (ret)
+				continue;
+
+			ret = soc_dp_aux_i2c_read(dp, DDC_ADDR, buf + offset, chunk);
+			if (!ret)
+				break;
+		}
+
+		if (ret) {
+			dev_err(dp->dev,
+				"[EDID] read failed, block %u offset 0x%02x len %zu ret %d\n",
+				block, edid_offset, chunk, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int soc_dp_conn_get_modes(struct drm_connector *connector)
+{
+	int count;
+	uint32_t max_mode_pixels = 0;
+	const struct drm_edid *edid;
+	struct drm_display_mode *mode, *tmp;
+	struct drm_device *dev = connector->dev;
+	struct soc_dp_dev *dp = container_of(connector, struct soc_dp_dev, connector);
+	struct drm_display_mode *preferred_mode = NULL;
+	struct drm_display_mode *edid_preferred_mode = NULL;
+	int retry;
+
+	mutex_lock(&dp->mode_lock);
+
+	for (retry = 0; retry < SOC_DP_SINK_READY_RETRIES; retry++) {
+		if (retry) {
+			soc_dp_aux_hw_reset(dp);
+			msleep(SOC_DP_SINK_READY_DELAY_MS);
+		}
+
+		edid = drm_edid_read_custom(connector, soc_dp_conn_get_edid_block, dp);
+		if (edid)
+			break;
+	}
+
+	if (!edid) {
+		mutex_unlock(&dp->mode_lock);
+		dev_err(dp->dev, "Failed to read EDID\n");
+		return drm_add_modes_noedid(connector, 1920, 1080);
+	}
+
+	drm_edid_connector_update(connector, edid);
+	count = drm_edid_connector_add_modes(connector);
+
+	list_for_each_entry_safe(mode, tmp, &connector->probed_modes, head) {
+		if (mode->hdisplay == 2560) {
+			if (drm_mode_vrefresh(mode) > 90) {
+				list_del(&mode->head);
+				drm_mode_destroy(dev, mode);
+				count--;
+			}
+		}
+	}
+
+	if (dp->link.max_num_lanes > SOC_DP_LANE_2) {
+		list_for_each_entry_safe(mode, tmp, &connector->probed_modes, head) {
+			if (mode->hdisplay == 3840) {
+				if (drm_mode_vrefresh(mode) > 60) {
+					list_del(&mode->head);
+					drm_mode_destroy(dev, mode);
+					count--;
+				}
+			}
+		}
+	} else {
+		list_for_each_entry_safe(mode, tmp, &connector->probed_modes, head) {
+			if (mode->hdisplay == 3840) {
+				if (drm_mode_vrefresh(mode) > 30) {
+					list_del(&mode->head);
+					drm_mode_destroy(dev, mode);
+					count--;
+				}
+			}
+		}
+	}
+
+	if (count > 1) {
+		list_for_each_entry(mode, &connector->probed_modes, head) {
+			u32 mode_pixels = mode->hdisplay * mode->vdisplay;
+
+			if (mode_pixels > max_mode_pixels)
+				max_mode_pixels = mode_pixels;
+
+			if (!edid_preferred_mode &&
+			    (mode->type & DRM_MODE_TYPE_PREFERRED)) {
+				edid_preferred_mode = mode;
+			}
+		}
+
+		if (!dp->edp_mode && max_mode_pixels > 1920 * 1080 &&
+		    (!edid_preferred_mode ||
+		     edid_preferred_mode->hdisplay != 1920 ||
+		     edid_preferred_mode->vdisplay != 1080)) {
+			list_for_each_entry_safe(mode, tmp, &connector->probed_modes, head) {
+				mode->type &= ~DRM_MODE_TYPE_PREFERRED;
+
+				if (!preferred_mode && mode->hdisplay == 1920 &&
+				    mode->vdisplay == 1080 &&
+				    drm_mode_vrefresh(mode) == 60) {
+					preferred_mode = mode;
+				}
+			}
+
+			if (preferred_mode) {
+				preferred_mode->type |= DRM_MODE_TYPE_PREFERRED;
+				list_move(&preferred_mode->head, &connector->probed_modes);
+			} else {
+				mode = list_first_entry(&connector->probed_modes,
+							struct drm_display_mode, head);
+				mode->type |= DRM_MODE_TYPE_PREFERRED;
+			}
+		}
+	}
+
+	drm_edid_free(edid);
+
+	mutex_unlock(&dp->mode_lock);
+
+	return count;
+}
+
+static enum drm_mode_status soc_dp_conn_mode_valid(struct drm_connector *connector,
+					       const struct drm_display_mode *mode)
+{
+	return MODE_OK;
+}
+
+static const struct drm_connector_helper_funcs soc_dp_conn_helper_funcs = {
+	.get_modes = soc_dp_conn_get_modes,
+	.mode_valid = soc_dp_conn_mode_valid,
+};
+
+static const struct drm_encoder_funcs soc_dp_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static void soc_dp_encoder_enable(struct drm_encoder *encoder)
+{
+	int ret;
+	int i;
+	struct soc_dp_dev *dp = container_of(encoder, struct soc_dp_dev, encoder);
+
+	uint32_t req_bw;
+	int bpp;
+	const struct soc_dp_link_config *cfg;
+	struct drm_display_mode *adjusted_mode = &dp->mode;
+	uint64_t clk_val;
+	uint64_t set_clk_val;
+
+	DRM_INFO("%s()\n", __func__);
+
+	mutex_lock(&dp->mode_lock);
+
+	if (dp->use_ext_pixel_clock && dp->pxclk) {
+		set_clk_val = adjusted_mode->clock * 1000;
+		if (set_clk_val) {
+			set_clk_val = clk_round_rate(dp->pxclk, set_clk_val);
+			clk_val = clk_get_rate(dp->pxclk);
+			if(clk_val != set_clk_val){
+				clk_set_rate(dp->pxclk, set_clk_val);
+				DRM_INFO("set dp pxclk=%lld\n", set_clk_val);
+			}
+		}
+		clk_val = clk_get_rate(dp->pxclk);
+		dp->pixel_clock = clk_val / 1000;
+	}
+
+	/* use DP pixel clock */
+	if (!dp->use_ext_pixel_clock && dp->dpu_id == 0) {
+		ret = regmap_update_bits(dp->apmu, SOC_DP_APMU_CLK_CTRL,
+					 BIT(2), BIT(2));
+		if (ret)
+			dev_err(dp->dev, "Failed to enable dpu0 DP/eDP pixel clock mux: %d\n", ret);
+	} else if (!dp->use_ext_pixel_clock && dp->dpu_id == 1) {
+		ret = regmap_update_bits(dp->apmu, SOC_DP_APMU_CLK_CTRL,
+					 BIT(18), BIT(18));
+		if (ret)
+			dev_err(dp->dev, "Failed to enable dpu1 DP/eDP pixel clock mux: %d\n", ret);
+	}
+
+	bpp = soc_dp_get_bpp(dp->color_format);
+	req_bw = soc_dp_calc_required_bw(adjusted_mode, bpp);
+
+	for (i = 0; i < ARRAY_SIZE(soc_dp_link_priority_table); i++) {
+		uint32_t capacity;
+
+		cfg = &soc_dp_link_priority_table[i];
+
+		/* Filter 1: Check HW Capabilities (Source & Sink limits) */
+		if (cfg->rate > dp->link.max_rate || cfg->lanes > dp->link.max_num_lanes)
+			continue;
+
+		/* Filter 2: Check Bandwidth Requirement */
+		capacity = soc_dp_calc_link_capacity(cfg->rate, cfg->lanes);
+		if (capacity < req_bw)
+			continue;
+
+		dev_info(dp->dev, "DP: Attempting Config: R=%d, L=%d (Cap: %d > Req: %d)\n",
+			cfg->rate, cfg->lanes, capacity, req_bw);
+
+		soc_dp_phy_power_off(dp);
+
+		/* Apply Hardware Settings */
+		if (dp->use_ext_pixel_clock) {
+			if (soc_dp_hw_set_pll(dp, cfg->rate, dp->pixel_clock))
+				continue;
+		} else {
+			if (soc_dp_hw_set_pll(dp, cfg->rate, adjusted_mode->clock))
+				continue;
+		}
+
+		soc_dp_phy_config_lanes(dp, cfg->lanes);
+		soc_dp_phy_config_rate(dp, cfg->rate);
+
+		if (soc_dp_phy_power_on(dp))
+			continue;
+
+		if (!dp->edp_mode && soc_dp_hw_detect_hpd(dp) == connector_status_disconnected) {
+			mutex_unlock(&dp->mode_lock);
+			dev_warn(dp->dev, "DP: Training failed for the connector is disconnected\n");
+			return;
+		}
+
+		if (dp->edp_mode) {
+			soc_dp_reg_write_range(dp, SOC_DPTX_ENABLE_EDP, 0x1);
+			soc_dp_reg_write_range(dp, SOC_DPTX_STREAM_ENC_EN, 0x1);
+			update_edp_config(dp, true);
+		}
+
+		/* Execute Link Training */
+		if (soc_dp_link_train(dp, cfg->rate, cfg->lanes) == 0) {
+			dev_info(dp->dev, "DP: Training successful for R:%d L:%d.\n",
+					cfg->rate, cfg->lanes);
+			break;
+		}
+
+		dev_warn(dp->dev, "DP: Training failed for R:%d L:%d. Upgrading...\n",
+			cfg->rate, cfg->lanes);
+	}
+
+	soc_dp_hw_set_msa_and_enable_video(dp, adjusted_mode, dp->link_rate, dp->lane_count);
+
+	mutex_unlock(&dp->mode_lock);
+	if (dp->backlight)
+		backlight_enable(dp->backlight);
+}
+
+static void soc_dp_encoder_disable(struct drm_encoder *encoder)
+{
+	struct soc_dp_dev *dp = container_of(encoder, struct soc_dp_dev, encoder);
+
+	DRM_INFO("%s()\n", __func__);
+
+	if (dp->backlight)
+		backlight_disable(dp->backlight);
+
+	mutex_lock(&dp->mode_lock);
+
+	/* Disable Video Stream */
+	soc_dp_phy_power_off(dp);
+
+	mutex_unlock(&dp->mode_lock);
+}
+
+static int soc_dp_encoder_atomic_check(struct drm_encoder *encoder,
+		struct drm_crtc_state *crtc_state, struct drm_connector_state *conn_state)
+{
+	return 0;
+}
+
+/*
+ * soc_dp_mode_set - Main DP Configuration Entry Point
+ * Logic:
+ * 1. Read Sink Capabilities.
+ * 2. Iterate through link configurations (Rate/Lane combinations).
+ * 3. Strategy: Ascending Bandwidth Order (Upgrade Logic).
+ * - Start with the lowest config that satisfies bandwidth.
+ * - Priority: Maximize Lanes first, then increase Rate (Stability over raw speed).
+ * 4. Perform Link Training. If failed, upgrade to next config.
+ * 5. Enable Video Stream.
+ */
+static void soc_dp_mode_set(struct drm_encoder *encoder,
+		struct drm_display_mode *mode,
+		struct drm_display_mode *adjusted_mode)
+{
+	struct soc_dp_dev *dp = container_of(encoder, struct soc_dp_dev, encoder);
+
+	mutex_lock(&dp->mode_lock);
+	drm_mode_copy(&dp->mode, adjusted_mode);
+	dev_dbg(dp->dev, "DP: Mode Set %dx%d (PCLK: %d kHz) flags 0x%x\n",
+		adjusted_mode->hdisplay, adjusted_mode->vdisplay, adjusted_mode->clock, adjusted_mode->flags);
+	mutex_unlock(&dp->mode_lock);
+}
+
+static const struct drm_encoder_helper_funcs soc_dp_encoder_helper_funcs = {
+	.enable = soc_dp_encoder_enable,
+	.disable = soc_dp_encoder_disable,
+	.atomic_check = soc_dp_encoder_atomic_check,
+	.mode_set = soc_dp_mode_set,
+};
+
+#ifdef CONFIG_SOC_DP_DRIVER_QEMU
+/* Debug interface for simulating Hotplug events in QEMU/Simulation */
+static ssize_t soc_dp_irq_proc_write(struct file *filp, const char __user *buf, size_t count, loff_t *ppos)
+{
+	struct soc_dp_dev *dp = PDE_DATA(file_inode(filp));
+	char write_status[2] = {0};
+
+	if (copy_from_user(write_status, buf, 1))
+		write_status[0] = '1';
+
+	if (write_status[0] == '1')
+		dp->connector_status = connector_status_connected;
+	else if (write_status[0] == '0')
+		dp->connector_status = connector_status_disconnected;
+	else
+		return -EINVAL;
+
+	drm_kms_helper_hotplug_event(dp->drm);
+	return count;
+}
+
+static const struct proc_ops soc_dp_irq_proc_ops = {
+	.proc_flags = PROC_ENTRY_PERMANENT,
+	.proc_write = soc_dp_irq_proc_write,
+};
+
+static void soc_dp_proc_irq_debug_init(struct soc_dp_dev *dp)
+{
+	dp->proc_irq = proc_create_data(dp->connector.name,
+			S_IWUSR, NULL, &soc_dp_irq_proc_ops, dp);
+}
+
+static void soc_dp_proc_irq_debug_exit(struct soc_dp_dev *dp)
+{
+	if (dp->proc_irq)
+		proc_remove(dp->proc_irq);
+	dp->proc_irq = NULL;
+}
+#endif
+
+#if HOT_PLUG_THREAD_ENABLED
+static void soc_dp_hpd_poll_work(struct work_struct *work)
+{
+	struct soc_dp_dev *dp = container_of(work, struct soc_dp_dev, hpd_work.work);
+	enum drm_connector_status old_status, new_status;
+	int interval_ms = HPD_POLL_INTERVAL_MS;
+
+	mutex_lock(&dp->mode_lock);
+
+	old_status = dp->connector_status;
+
+	if (!dp->suspended) {
+		new_status = soc_dp_hw_detect_hpd(dp);
+		soc_dp_hw_clean_hpd(dp);
+	} else {
+		mutex_unlock(&dp->mode_lock);
+		return;
+	}
+
+	if (new_status != old_status) {
+		dp->connector_status = new_status;
+
+		if (dp->connector_status == connector_status_connected)
+			soc_dp_hw_read_sink_caps(dp);
+		else
+			interval_ms = 3000;
+
+		mutex_unlock(&dp->mode_lock);
+		DRM_INFO("%s() hot plug event\n", __func__);
+		drm_kms_helper_hotplug_event(dp->drm);
+
+#if IS_ENABLED(CONFIG_SND_SOC)
+		if (!dp->edp_mode) {
+			if (dp->connector_status == connector_status_connected) {
+				if (inno_dp_audio_register(dp->dev))
+					DRM_INFO("%s() failed to register dp audio component\n", __func__);
+				else
+					dp->aud_registered = true;
+			} else {
+				inno_dp_audio_unregister(dp->dev);
+				dp->aud_registered = false;
+			}
+		}
+#endif
+	} else {
+		mutex_unlock(&dp->mode_lock);
+#if IS_ENABLED(CONFIG_SND_SOC)
+		if (!dp->edp_mode) {
+			if (dp->aud_registered && new_status == connector_status_disconnected) {
+				if (dp->card_instantiated) {
+					inno_dp_audio_unregister(dp->dev);
+					dp->aud_registered = false;
+				}
+			} else if (!dp->aud_registered && new_status == connector_status_connected) {
+				if (inno_dp_audio_register(dp->dev))
+					DRM_INFO("%s() failed to register dp audio component\n", __func__);
+				else
+					dp->aud_registered = true;
+			}
+		}
+	}
+#endif
+
+	schedule_delayed_work(&dp->hpd_work, msecs_to_jiffies(interval_ms));
+}
+#else
+static irqreturn_t soc_dp_irq_handler(int irq, void *data)
+{
+	struct soc_dp_dev *dp = data;
+	enum drm_connector_status old_status, new_status;
+	uint32_t hpd_status;
+
+	old_status = dp->connector_status;
+
+	if (!dp->suspended) {
+		new_status = soc_dp_hw_detect_hpd(dp);
+		soc_dp_hw_clean_hpd(dp);
+	} else {
+		new_status = connector_status_disconnected;
+	}
+
+	if (new_status != old_status) {
+
+		soc_dp_reg_read_range(dp, SOC_DPTX_HPD_IN_STATUS, &hpd_status);
+		DRM_INFO("%s() hpd status 0x%x\n", __func__, hpd_status);
+
+		dp->connector_status = new_status;
+		return IRQ_WAKE_THREAD; // Call hotplug_event
+	}
+
+	return IRQ_NONE;
+}
+
+static irqreturn_t soc_dp_hotplug_event_handler(int irq, void *data)
+{
+	struct soc_dp_dev *dp = data;
+
+	drm_kms_helper_hotplug_event(dp->drm);
+
+	return IRQ_HANDLED;
+}
+#endif
+
+static int soc_dp_resource_init(struct soc_dp_dev *dp, struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	uint32_t dp_id, edp_id;
+	int ret;
+
+	if (of_property_read_u32(pdev->dev.of_node, "dp-id", &dp_id))
+		dp_id = -1;
+
+	if (of_property_read_u32(pdev->dev.of_node, "edp-id", &edp_id))
+		edp_id = -1;
+
+#ifndef CONFIG_SOC_DP_DRIVER_QEMU
+	dp->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(dp->regs)) {
+		dev_err(dev, "Failed to map registers\n");
+		return PTR_ERR(dp->regs);
+	}
+#else
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to obtain dp resource.\n");
+		return -EINVAL;
+	}
+
+	dp->regs = devm_kmalloc(dev, res->end - res->start + 1, GFP_KERNEL);
+	if (!dp->regs) {
+		dev_err(dev, "Failed to map registers\n");
+		return -ENOMEM;
+	}
+#endif
+
+	if (dp_id == 0 || edp_id == 0) {
+		dp->dpu_id = 0;
+		dp->qos = soc_dp_qos_init_regmap(dev, SOC_DP_QOS_BASE,
+						SOC_DP_QOS_SIZE,
+						&soc_dp_qos_regmap_config);
+		if (IS_ERR(dp->qos)) {
+			dev_err(dev, "Failed to regmap QoS\n");
+			return PTR_ERR(dp->qos);
+		}
+
+		ret = regmap_update_bits(dp->qos, SOC_DP_QOS_MUX_CTRL,
+					 BIT(8), BIT(8));
+		if (ret) {
+			dev_err(dev, "Failed to mux dp0/edp0 %d\n", ret);
+			return ret;
+		}
+	} else
+		dp->dpu_id = 1;
+
+	dp->apmu = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "spacemit,apmu");
+	if (IS_ERR(dp->apmu)) {
+		dev_err(dev, "Failed to lookup APMU\n");
+		return PTR_ERR(dp->apmu);
+	}
+
+	/* use external pixel clock */
+	dp->use_ext_pixel_clock = true;
+
+	/* use DP pixel clock */
+	if (dp->dpu_id == 0 ) {
+		ret = regmap_update_bits(dp->apmu, SOC_DP_APMU_CLK_CTRL,
+					 BIT(2), BIT(2));
+		if (ret) {
+			dev_err(dev, "Failed to select dp0 internal pixel clock mux: %d\n", ret);
+			return ret;
+		}
+		dp->use_ext_pixel_clock = false;
+	} else if (dp->dpu_id == 1) {
+		ret = regmap_update_bits(dp->apmu, SOC_DP_APMU_CLK_CTRL,
+					 BIT(18), BIT(18));
+		if (ret) {
+			dev_err(dev, "Failed to select dp1 internal pixel clock mux: %d\n", ret);
+			return ret;
+		}
+		dp->use_ext_pixel_clock = false;
+	}
+
+#if HOT_PLUG_THREAD_ENABLED
+	INIT_DELAYED_WORK(&dp->hpd_work, soc_dp_hpd_poll_work);
+#else
+	dp->irq = platform_get_irq(pdev, 0);
+	if (dp->irq < 0) {
+		dev_err(dev, "Failed to get IRQ\n");
+		return dp->irq;
+	}
+	dev_info(dev, "irq %d\n", dp->irq);
+#endif
+
+	return 0;
+}
+
+static int soc_dp_dev_init(struct soc_dp_dev *dp)
+{
+	int ret;
+	uint32_t m_isel = 0x5, m_mainsel = 0x19;
+	uint32_t m_pre = 0x0, m_post = 0x2;
+	uint32_t tx_mode = 0x1, tx_pre = 0x0;
+	uint32_t clk_div = 24 * 1000 / 100;
+
+	dp->ref_clk = SOC_DP_REF_CLK_24M;
+	dp->color_format = SOC_VIDEO_RGB_8BIT;
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_XMIT_ENABLE, 0);
+	mdelay(2);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_PD, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_PREPLL_PD, 1);
+	mdelay(2);
+
+	// Reset Controller and PHY
+	soc_dp_reg_write_range(dp, SOC_DPTX_CONTROLLER_RESET, 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_PHY_RESET, 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HDCP_RESET, 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_RESET, 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_RESET, 0x1);
+	mdelay(5);
+
+	// Clear Video Reset
+	soc_dp_reg_write_range(dp, SOC_DPTX_CONTROLLER_RESET, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_PHY_RESET, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HDCP_RESET, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_RESET, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_RESET, 0x0);
+	mdelay(2);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_REPLY_EVENT_INT_STA, 1);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_DEFAULT_FAST_LINK_TRAIN_EN, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_SCRAMBLER_DISABLE, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_SCALE_DOWN_MODE, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_XMIT_ENABLE, 0);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_STREAM_ENABLE, 0);
+
+	// Unmask Interrupts
+	soc_dp_reg_write_range(dp, SOC_DPTX_AUX_REPLY_EVENT_INT_STA_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HDCP_INT_STA_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ILLEGAL_AUX_CMD_INT_STA_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_TYPE_C_EVENT_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_DSC_EVENT_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_SDP_INT_STA_S3_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_SDP_INT_STA_S2_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_SDP_INT_STA_S1_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_SDP_INT_STA_S0_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S3_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S2_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S1_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S0_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_SINK_IRQ_EVENT_MSK, 0x0);
+#if HPD_BYPASS || HOT_PLUG_THREAD_ENABLED
+	soc_dp_reg_write_range(dp, SOC_DPTX_HPD_INT_STA_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HOT_PLUG_EVENT_MSK, 0x0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HOT_UNPLUG_EVENT_MSK, 0x0);
+#else
+	soc_dp_reg_write_range(dp, SOC_DPTX_HPD_INT_STA_MSK, 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HOT_PLUG_EVENT_MSK, 0x1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_HOT_UNPLUG_EVENT_MSK, 0x1);
+#endif
+	soc_dp_reg_write_range(dp, SOC_DPTX_SINK_UNPLUG_ERROR_EVENT_MSK, 0x0);
+	mdelay(2);
+
+	// Disable PHY SSC (Spread Spectrum Clocking)
+	// soc_dp_reg_write_range(dp, SOC_DPTX_ANA_MPLL_DISABLE_SSCG, 0x1);
+
+	// Bypass PHY busy state
+	soc_dp_reg_write_range(dp, SOC_DPTX_PHY_BUSY_BYP, 0x1);
+
+	// Enable Enhance Framing and Scale Down Mode
+	soc_dp_reg_write_range(dp, SOC_DPTX_ENHANCE_FRAMING_EN, 0x1);
+
+	// Configure PLL and Lanes
+	if (dp->use_ext_pixel_clock) {
+		ret = soc_dp_hw_set_pll(dp, SOC_DP_LINK_RATE_2_70, 150000);
+		if (ret)
+			return ret;
+	} else {
+		ret = soc_dp_hw_set_pll(dp, SOC_DP_LINK_RATE_2_70, 148500);
+		if (ret)
+			return ret;
+	}
+
+	soc_dp_phy_config_lanes(dp, SOC_DP_LANE_2);
+	soc_dp_phy_config_rate(dp, SOC_DP_LINK_RATE_2_70);
+
+	ret = soc_dp_phy_power_on(dp);
+	if (ret)
+		return ret;
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D0, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D1, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D2, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D3, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_RTCAL_FREQDIV_HBIT, (clk_div >> 8) & 0x7f);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_RTCAL_BYPASS, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_RTCAL_FREQDIV_LBIT, clk_div & 0xff);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_BG_RCAL_SEL, 0);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_RTM_D3, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_RTM_D2, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_RTM_D1, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_RTM_D0, 0);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_RTCAL_BYPASS, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_RTCAL_BYPASS, 0);
+	msleep(100);
+
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_PRE_D3, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_PRE_D2, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_PRE_D1, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_PRE_D0, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_DE_D3, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_DE_D2, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_DE_D1, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_DE_D0, 1);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_PRE_D3, tx_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_PRE_D2, tx_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_PRE_D1, tx_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_PRE_D0, tx_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_ISEL_DRV_D3, m_isel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_ISEL_DRV_D2, m_isel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MAINSEL_D2, m_mainsel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MAINSEL_D3, m_mainsel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_ISEL_DRV_D1, m_isel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_ISEL_DRV_D0, m_isel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_D1, m_post);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_D0, m_post);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_D3, m_post);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_POSTSEL_D2, m_post);
+	soc_dp_reg_write_range(dp, SOC_DPTX_DA_TX_MAINSEL_D0_4_0, m_mainsel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MAINSEL_D1, m_mainsel);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_PRESEL_D1, m_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_PRESEL_D0, m_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_PRESEL_D3, m_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_PRESEL_D2, m_pre);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D3, tx_mode);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D2, tx_mode);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D1, tx_mode);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_MODE_D0, tx_mode);
+	soc_dp_reg_write_range(dp, SOC_DPTX_ANA_TX_AUX_RX_VSEL, 0x0);
+
+#if IS_ENABLED(CONFIG_SND_SOC)
+	// init Audio
+	if (!dp->edp_mode) {
+		dev_info(dp->dev, "init audio\n");
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUD_STREAM_VERTICAL_EN, 1);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUD_STREAM_HORIZONTAL_EN, 1);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUD_TIMESTAMP_VERTICAL_EN, 1);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUD_TIMESTAMP_HORIZONTAL_EN, 1);
+
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_INF_SELECT, 0);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUD_ADJUST_SEL, 1);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_NUM_CHANNELS, 0x1);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_DATA_IN_EN, 0x1);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_DATA_WIDTH, 0x10);
+		soc_dp_reg_write_range(dp, SOC_DPTX_I2S_AUDIO_MODE, 0x01);
+
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_PACKET_ID, 0);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_TIMESTAMP_VERSION_NUM, 0x12);
+
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_MUTE, 0);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_RESET, 1);
+		udelay(1000);
+		soc_dp_reg_write_range(dp, SOC_DPTX_AUDIO_RESET, 0);
+	}
+#endif
+
+	return 0;
+}
+
+static int soc_dp_bind(struct device *dev, struct device *master, void *data)
+{
+	int ret;
+	struct soc_dp_dev *dp;
+	struct drm_device *drm = (struct drm_device *)data;
+	struct platform_device *pdev = to_platform_device(dev);
+	uint32_t dp_id;
+
+	DRM_INFO("%s()\n", __func__);
+
+	dp = devm_kmalloc(dev, sizeof(*dp), GFP_KERNEL);
+	if (!dp)
+		return -ENOMEM;
+	memset(dp, 0, sizeof(*dp));
+
+	dp->dev = dev;
+	dp->drm = drm;
+	dp->connector_status = connector_status_disconnected;
+	mutex_init(&dp->mode_lock);
+	dp->suspended = false;
+
+#ifdef CONFIG_SOC_DP_DRIVER_QEMU
+	dp->proc_irq = NULL;
+#endif
+
+	dp->reset = devm_reset_control_get_optional_shared(&pdev->dev, "reset");
+	if (IS_ERR_OR_NULL(dp->reset)) {
+		DRM_INFO("Failed to found reset\n");
+	}
+
+	dp->pxclk = of_clk_get_by_name(dev->of_node, "pxclk");
+	if (IS_ERR(dp->pxclk)) {
+		dp->pxclk = NULL;
+		DRM_INFO("Failed to found pxclk\n");
+	}
+
+	ret = of_property_read_u32(dev->of_node, "gpios-bl", &dp->gpio_bl);
+	if (ret || !gpio_is_valid(dp->gpio_bl)) {
+		dev_dbg(dev, "missing dt property: gpios-bl\n");
+		dp->gpio_bl = INVALID_GPIO;
+	} else {
+		ret = gpio_request(dp->gpio_bl, NULL);
+		if (ret) {
+			pr_err("gpio_bl request fail\n");
+		}
+	}
+
+	ret = of_property_read_u32(dev->of_node, "gpios-enable", &dp->gpio_enable);
+	if (ret || !gpio_is_valid(dp->gpio_enable)) {
+		dev_dbg(dev, "missing dt property: gpios-enable\n");
+		dp->gpio_enable = INVALID_GPIO;
+	} else {
+		ret = gpio_request(dp->gpio_enable, NULL);
+		if (ret) {
+			pr_err("gpio_enable request fail\n");
+		}
+	}
+
+	ret = of_property_read_u32(dev->of_node, "gpios-power", &dp->gpio_power);
+	if (ret || !gpio_is_valid(dp->gpio_power)) {
+		dev_dbg(dev, "missing dt property: gpios-power\n");
+		dp->gpio_power = INVALID_GPIO;
+	} else {
+		ret = gpio_request(dp->gpio_power, NULL);
+		if (ret) {
+			pr_err("gpio_power request fail\n");
+		}
+	}
+
+	if (of_property_read_u32(pdev->dev.of_node, "dp-id", &dp_id))
+		dp->edp_mode = true;
+	else
+		dp->edp_mode = false;
+
+	if (!IS_ERR_OR_NULL(dp->reset)) {
+		ret = reset_control_deassert(dp->reset);
+		if (ret < 0) {
+			DRM_INFO("Failed to deassert reset\n");
+		}
+	}
+
+	if (dp->pxclk)
+		clk_prepare_enable(dp->pxclk);
+
+	if(INVALID_GPIO != dp->gpio_power)
+		gpio_direction_output(dp->gpio_power, 1);
+	if(INVALID_GPIO != dp->gpio_enable)
+		gpio_direction_output(dp->gpio_enable, 1);
+	if(INVALID_GPIO != dp->gpio_bl)
+		gpio_direction_output(dp->gpio_bl, 1);
+
+	/* Init Connector */
+	if (dp->edp_mode) {
+		ret = drm_connector_init(drm, &dp->connector,
+				&soc_dp_connector_funcs, DRM_MODE_CONNECTOR_eDP);
+		if (ret) {
+			dev_err(dev, "Failed to init connector\n");
+			return ret;
+		}
+	} else {
+		ret = drm_connector_init(drm, &dp->connector,
+				&soc_dp_connector_funcs, DRM_MODE_CONNECTOR_DisplayPort);
+		if (ret) {
+			dev_err(dev, "Failed to init connector\n");
+			return ret;
+		}
+	}
+	drm_connector_helper_add(&dp->connector, &soc_dp_conn_helper_funcs);
+
+	if (dp->edp_mode) {
+		dp->backlight = devm_of_find_backlight(dp->dev);
+		if (IS_ERR(dp->backlight))
+			dev_err(dev, "Failed to find backlight\n");
+	} else {
+		dp->backlight = NULL;
+	}
+
+	/* Init Encoder */
+	ret = drm_encoder_init(drm, &dp->encoder,
+			&soc_dp_encoder_funcs, DRM_MODE_ENCODER_TMDS, NULL);
+	if (ret) {
+		dev_err(dev, "Failed to init encoder\n");
+		drm_connector_cleanup(&dp->connector);
+		return ret;
+	}
+	drm_encoder_helper_add(&dp->encoder, &soc_dp_encoder_helper_funcs);
+
+	dp->encoder.possible_crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
+	drm_connector_attach_encoder(&dp->connector, &dp->encoder);
+
+	platform_set_drvdata(pdev, dp);
+
+#ifdef CONFIG_SOC_DP_DRIVER_QEMU
+	soc_dp_proc_irq_debug_init(dp);
+#endif
+
+	soc_dp_aux_init(dp);
+	dp->connector.ddc = &dp->aux.ddc;
+
+	ret = soc_dp_resource_init(dp, pdev);
+	if (ret) {
+		drm_connector_cleanup(&dp->connector);
+		return ret;
+	}
+
+	ret = soc_dp_dev_init(dp);
+	if (ret) {
+		drm_connector_cleanup(&dp->connector);
+		return ret;
+	}
+
+	soc_dp_phy_power_off(dp);
+
+#if IS_ENABLED(CONFIG_SND_SOC)
+	if (!dp->edp_mode) {
+		ret = inno_dp_audio_register(dp->dev);
+		if (ret)
+			dev_err(dev, "failed to register dp audio component\n");
+		else
+			dp->aud_registered = true;
+	}
+#endif
+
+#if HPD_BYPASS
+	soc_dp_reg_write_range(dp, SOC_DPTX_FORCE_HPD, 0x1);
+	mdelay(5);
+#endif
+	dp->connector_status = soc_dp_hw_detect_hpd(dp);
+	soc_dp_hw_clean_hpd(dp);
+
+	if (dp->connector_status == connector_status_connected)
+		soc_dp_hw_read_sink_caps(dp);
+
+	drm_kms_helper_hotplug_event(dp->drm);
+
+#if HOT_PLUG_THREAD_ENABLED
+	dev_info(dp->dev, "Starting HPD Polling Thread...\n");
+	schedule_delayed_work(&dp->hpd_work, msecs_to_jiffies(HPD_POLL_INTERVAL_MS));
+#else
+	ret = devm_request_threaded_irq(dp->dev, dp->irq, soc_dp_irq_handler,
+			soc_dp_hotplug_event_handler, 0, dev_name(dp->dev), dp);
+	if (ret) {
+		dev_err(dp->dev, "Failure requesting irq %d: %d.\n", dp->irq, ret);
+		return ret;
+	}
+#endif
+
+	return 0;
+}
+
+static void soc_dp_unbind(struct device *dev, struct device *master, void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct soc_dp_dev *dp = platform_get_drvdata(pdev);
+	int ret;
+
+	DRM_INFO("%s()\n", __func__);
+
+#if IS_ENABLED(CONFIG_SND_SOC)
+	if (!dp->edp_mode) {
+		inno_dp_audio_unregister(dp->dev);
+		dp->aud_registered = false;
+	}
+#endif
+
+#if HOT_PLUG_THREAD_ENABLED
+	cancel_delayed_work_sync(&dp->hpd_work);
+#endif
+
+	drm_dp_aux_unregister(&dp->aux);
+
+	soc_dp_hw_disable(dp);
+
+#ifdef CONFIG_SOC_DP_DRIVER_QEMU
+	soc_dp_proc_irq_debug_exit(dp);
+#endif
+	drm_encoder_cleanup(&dp->encoder);
+	drm_connector_cleanup(&dp->connector);
+
+	mutex_destroy(&dp->mode_lock);
+
+	if(INVALID_GPIO != dp->gpio_bl)
+		gpio_direction_output(dp->gpio_bl, 0);
+	if(INVALID_GPIO != dp->gpio_enable)
+		gpio_direction_output(dp->gpio_enable, 0);
+	if(INVALID_GPIO != dp->gpio_power)
+		gpio_direction_output(dp->gpio_power, 0);
+
+	if (dp->pxclk)
+		clk_disable_unprepare(dp->pxclk);
+
+	if (!IS_ERR_OR_NULL(dp->reset)) {
+		ret = reset_control_assert(dp->reset);
+		if (ret < 0) {
+			DRM_INFO("Failed to assert reset\n");
+		}
+	}
+}
+
+static const struct component_ops soc_dp_ops = {
+	.bind = soc_dp_bind,
+	.unbind = soc_dp_unbind,
+};
+
+static int inno_dp_probe(struct platform_device *pdev)
+{
+	DRM_INFO("%s()\n", __func__);
+	return component_add(&pdev->dev, &soc_dp_ops);
+}
+
+static void inno_dp_remove(struct platform_device *pdev)
+{
+	DRM_INFO("%s()\n", __func__);
+	component_del(&pdev->dev, &soc_dp_ops);
+}
+
+static void inno_dp_shutdown(struct platform_device *pdev)
+{
+	struct soc_dp_dev *dp = platform_get_drvdata(pdev);
+
+#if HOT_PLUG_THREAD_ENABLED
+	cancel_delayed_work_sync(&dp->hpd_work);
+#endif
+	soc_dp_hw_disable(dp);
+}
+
+#ifdef CONFIG_PM_SLEEP
+
+static int inno_dp_drv_pm_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct soc_dp_dev *dp = platform_get_drvdata(pdev);
+
+	DRM_INFO("%s()\n", __func__);
+
+#if HOT_PLUG_THREAD_ENABLED
+	cancel_delayed_work_sync(&dp->hpd_work);
+#endif
+
+	mutex_lock(&dp->mode_lock);
+	dp->suspended = true;
+	mutex_unlock(&dp->mode_lock);
+
+	return 0;
+}
+
+static int inno_dp_drv_pm_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct soc_dp_dev *dp = platform_get_drvdata(pdev);
+
+	DRM_INFO("%s()\n", __func__);
+
+	mutex_lock(&dp->mode_lock);
+	dp->suspended = false;
+	mutex_unlock(&dp->mode_lock);
+
+#if HPD_BYPASS
+	soc_dp_reg_write_range(dp, SOC_DPTX_FORCE_HPD, 0x1);
+	mdelay(5);
+#endif
+	dp->connector_status = soc_dp_hw_detect_hpd(dp);
+	soc_dp_hw_clean_hpd(dp);
+
+	if (dp->connector_status == connector_status_connected)
+		soc_dp_hw_read_sink_caps(dp);
+
+	drm_kms_helper_hotplug_event(dp->drm);
+
+#if HOT_PLUG_THREAD_ENABLED
+	schedule_delayed_work(&dp->hpd_work, msecs_to_jiffies(HPD_POLL_INTERVAL_MS));
+#endif
+
+	return 0;
+}
+
+static int inno_dp_drv_pm_suspend_late(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct soc_dp_dev *dp = platform_get_drvdata(pdev);
+	int ret;
+
+	DRM_INFO("%s()\n", __func__);
+
+	mutex_lock(&dp->mode_lock);
+
+	soc_dp_hw_disable(dp);
+
+	if (dp->pxclk)
+		clk_disable_unprepare(dp->pxclk);
+
+	if (!IS_ERR_OR_NULL(dp->reset)) {
+		ret = reset_control_assert(dp->reset);
+		if (ret < 0) {
+			DRM_INFO("Failed to assert reset\n");
+		}
+	}
+
+	mutex_unlock(&dp->mode_lock);
+
+	return 0;
+}
+
+static int inno_dp_drv_pm_resume_early(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct soc_dp_dev *dp = platform_get_drvdata(pdev);
+	int ret;
+
+	DRM_INFO("%s()\n", __func__);
+
+	mutex_lock(&dp->mode_lock);
+
+	if (!IS_ERR_OR_NULL(dp->reset)) {
+		ret = reset_control_deassert(dp->reset);
+		if (ret < 0) {
+			DRM_INFO("Failed to deassert reset\n");
+		}
+	}
+	if (dp->pxclk)
+		clk_prepare_enable(dp->pxclk);
+
+	soc_dp_dev_init(dp);
+
+	mutex_unlock(&dp->mode_lock);
+
+	return 0;
+}
+
+#endif
+
+static const struct dev_pm_ops inno_dp_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(inno_dp_drv_pm_suspend,
+				inno_dp_drv_pm_resume)
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(inno_dp_drv_pm_suspend_late,
+				     inno_dp_drv_pm_resume_early)
+};
+
+static const struct of_device_id soc_dp_match[] = {
+	{ .compatible = "spacemit,inno-dp0" },
+	{ .compatible = "spacemit,inno-dp1" },
+	{ .compatible = "spacemit,inno-edp0" },
+	{ .compatible = "spacemit,inno-edp1" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, soc_dp_match);
+
+struct platform_driver inno_dp_driver = {
+	.probe = inno_dp_probe,
+	.remove = inno_dp_remove,
+	.shutdown = inno_dp_shutdown,
+	.driver = {
+		.name = "spacemit-inno-dp-drv",
+		.of_match_table = soc_dp_match,
+		.pm = &inno_dp_pm_ops,
+	},
+};
+
+// module_platform_driver(inno_dp_driver);
+static int inno_dp_driver_init(void)
+{
+	return platform_driver_register(&inno_dp_driver);
+}
+late_initcall(inno_dp_driver_init);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/spacemit/spacemit_inno_dp.h b/drivers/gpu/drm/spacemit/spacemit_inno_dp.h
new file mode 100644
index 000000000000..e4c50e26da44
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_inno_dp.h
@@ -0,0 +1,326 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef __SPACEMIT_INNO_DP_H__
+#define __SPACEMIT_INNO_DP_H__
+
+// SOC_DPTX_CORE_CONTROL: 0x0018
+#define SOC_DPTX_SCRAMBLER_DISABLE                    0x0018,31,31 // Scramble_disable;0: Scramble enabled;1: Disable scramble
+#define SOC_DPTX_ENHANCE_FRAMING_EN                   0x0018,30,30 // Enhance_framing_enable;0:disable;1:enable
+#define SOC_DPTX_DEFAULT_FAST_LINK_TRAIN_EN           0x0018,29,29 // DEFAULT_FAST_LINK_TRAIN_EN;Default fast link training state.
+#define SOC_DPTX_SCALE_DOWN_MODE                      0x0018,28,28 // SCALE_DOWN_MODE;scaling hpd counter for fast simulation;0: normal;1: scaling
+#define SOC_DPTX_FORCE_HPD                            0x0018,27,27 // FORCE_HPD;Force HPD to 1'b1.
+#define SOC_DPTX_ENABLE_EDP                           0x0018,4,4   // ENABLE_EDP;EDP mode Enable.
+
+// SOC_DPTX_SOFT_RESET: 0x001c
+#define SOC_DPTX_CONTROLLER_RESET                     0x001c,31,31 // CONTROLLER_RESET;0: normal;1: reset
+#define SOC_DPTX_PHY_RESET                            0x001c,30,30 // PHY_RESET;0: normal;1: reset
+#define SOC_DPTX_HDCP_RESET                           0x001c,29,29 // HDCP_RESET;0: normal;1: reset
+#define SOC_DPTX_AUDIO_RESET                          0x001c,28,28 // AUDIO_RESET;0: normal;1: reset
+#define SOC_DPTX_AUX_RESET                            0x001c,27,27 // AUX_RESET;0: normal;1:reset
+#define SOC_DPTX_VIDEO_RESET                          0x001c,3,0   // VIDEO_RESET;video soft reset,up to 4 streams.;0: noraml;1:reset
+
+// SOC_DPTX_CLK_INV_MUX: 0x0028
+#define SOC_DPTX_REG_VID_CLK_SEL                      0x0028,0,0   // REG_VID_CLK_SEL;0: external pixel clock;1: internal pixel clock
+
+// SOC_DPTX_GENERAL_INTERRUPT: 0x0080
+#define SOC_DPTX_AUX_REPLY_EVENT_INT_STA              0x0080,16,16 // AUX__REPLY_EVENT_INT_STA;AUX reply event,can be clear by RWite 1’b1
+
+// SOC_DPTX_GENERAL_INTERRUPT_MASK: 0x0084
+#define SOC_DPTX_HPD_INT_STA_MSK                      0x0084,17,17 // HPD_INT_STA_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_AUX_REPLY_EVENT_INT_STA_MSK          0x0084,16,16 // AUX_REPLY_EVENT_INT_STA_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_HDCP_INT_STA_MSK                     0x0084,15,15 // HDCP_INT_STA_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_ILLEGAL_AUX_CMD_INT_STA_MSK          0x0084,14,14 // ILLEGAL_AUX_CMD_INT_STA_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_TYPE_C_EVENT_MSK                     0x0084,13,13 // TYPE_C_EVENT_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_DSC_EVENT_MSK                        0x0084,12,12 // DSC_EVENT_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_SDP_INT_STA_S3_MSK                   0x0084,11,11 // SDP_INT_STA_S3_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_SDP_INT_STA_S2_MSK                   0x0084,10,10 // SDP_INT_STA_S2_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_SDP_INT_STA_S1_MSK                   0x0084,9,9   // SDP_INT_STA_S1_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_SDP_INT_STA_S0_MSK                   0x0084,8,8   // SDP_INT_STA_S0_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S3_MSK   0x0084,7,7   // VIDEO_FIFO_OVERFLOW_INT_STA_S3_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S2_MSK   0x0084,6,6   // VIDEO_FIFO_OVERFLOW_INT_STA_S2_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S1_MSK   0x0084,5,5   // VIDEO_FIFO_OVERFLOW_INT_STA_S1_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_VIDEO_FIFO_OVERFLOW_INT_STA_S0_MSK   0x0084,4,4   // VIDEO_FIFO_OVERFLOW_INT_STA_S0_MSK;0: disable interrupt;1: enable interrupt
+
+// SOC_DPTX_HPD_INTERRUPT_STATUS: 0x0088
+#define SOC_DPTX_HOT_PLUG_EVENT                       0x0088,29,29 // HOT_PLUG_EVENT;HPD Plug interrupt status,can be clear by RWite 1’b1.
+#define SOC_DPTX_HOT_UNPLUG_EVENT                     0x0088,28,28 // HOT_UNPLUG_EVENT;HPD unplug interrupt status,can be clear by RWite 1'b1.
+#define SOC_DPTX_HPD_IN_STATUS                        0x0088,26,26 // HPD_IN_STATUS;HPD plug in status.
+
+// SOC_DPTX_HPD_INTERRUPT_ENABLE: 0x008c
+#define SOC_DPTX_SINK_IRQ_EVENT_MSK                   0x008c,31,31 // SINK_IRQ_EVENT_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_HOT_PLUG_EVENT_MSK                   0x008c,29,29 // HOT_PLUG_EVENT_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_HOT_UNPLUG_EVENT_MSK                 0x008c,28,28 // HOT_UNPLUG_EVENT_MSK;0: disable interrupt;1: enable interrupt
+#define SOC_DPTX_SINK_UNPLUG_ERROR_EVENT_MSK          0x008c,27,27 // SINK_UNPLUG_ERROR_EVENT_MSK;0: disable interrupt;1: enable interrupt
+
+// SOC_DPTX_PHYIF_CTRL_ADDR: 0x0100
+#define SOC_DPTX_PHY_BUSY_BYP                         0x0100,31,31 // PHY_BUSY_BYP;lane phybusy status bypass.
+#define SOC_DPTX_TPS_SEL                              0x0100,28,25 // TPS_SEL;selects the training pattern.;0: No training pattern. Normal stream data is transmitted instead.;1: TPS1;2: TPS2;3: TPS3
+#define SOC_DPTX_XMIT_ENABLE                          0x0100,20,17 // Transmit enable;enable transmitter on the per lane.;Bit17:lane 0;Bit18:lane 1;Bit19:lane 2;Bit20:lane 3
+#define SOC_DPTX_PHY_POWERDOWN                        0x0100,16,13 // PHY_POWERDOWN;0x0: Powered on;0xc: PHY is powered down,reference clock can be stopped.
+#define SOC_DPTX_PHY_SSC_DIS                          0x0100,8,8   // PHY_SSC_DIS;PHY SSC disable.;0: SSC enable;1: SSC disable
+#define SOC_DPTX_PHY_NUM_LANES                        0x0100,6,5   // PHY_NUM_LANES;number of lanes active :;2’b00:1 lane;2’b01:2 lanes;2’b10:4 lanes
+#define SOC_DPTX_PHY_RATE                             0x0100,1,0   // PHY_RATE;rate set for the phy.;2’b00:RBR 1.62G;2’b01:HBR 2.7G;2’b10: HBR2 5.4G;2’b11: Reserved
+
+// SOC_DPTX_PHYIF_TX_EQ_ADDR: 0x0104
+#define SOC_DPTX_PHY_LANE3_TX_VSWING                  0x0104,21,20 // PHY_LANE3_TX_VSWING;lane3 vswing.
+#define SOC_DPTX_PHY_LANE3_TX_PREEMP                  0x0104,19,18 // PHY_LANE3_TX_PREEMP;lane3 preemp.
+#define SOC_DPTX_PHY_LANE2_TX_VSWING                  0x0104,15,14 // PHY_LANE2_TX_VSWING;lane2 vswing.
+#define SOC_DPTX_PHY_LANE2_TX_PREEMP                  0x0104,13,12 // PHY_LANE2_TX_PREEMP;lane2 preemp.
+#define SOC_DPTX_PHY_LANE1_TX_VSWING                  0x0104,9,8   // PHY_LANE1_TX_VSWING;lane1 vswing.
+#define SOC_DPTX_PHY_LANE1_TX_PREEMP                  0x0104,7,6   // PHY_LANE1_TX_PREEMP;lane1 preemp.
+#define SOC_DPTX_PHY_LANE0_TX_VSWING                  0x0104,3,2   // PHY_LANE0_TX_VSWING;lan0 vswing.
+#define SOC_DPTX_PHY_LANE0_TX_PREEMP                  0x0104,1,0   // PHY_LANE0_TX_PREEMP;lane0 preemp.
+
+// SOC_DPTX_MPLL_CTRL0: 0x0180
+#define SOC_DPTX_ANA_MPLL_FBDIV_LBIT                  0x0180,31,24 // ANA_MPLL_FBDIV_LBIT;reg_mpll_fbdiv[7:0],The integer part of CORE PLL feedback divide value
+#define SOC_DPTX_ANA_MPLL_SEL_EXTWAVE                 0x0180,22,22 // ANA_MPLL_SEL_EXTWAVE;MPLL SSC wave table maxaddr value.;0: default value,127;1: software configure,ANA_MPLL_SSC_EXT_MAXADDR.
+#define SOC_DPTX_ANA_MPLL_DISABLE_SSCG                0x0180,21,21 // ANA_MPLL_DISABLE_SSCG;0: enable SSC;1: disable SSC
+#define SOC_DPTX_ANA_MPLL_DOWNSPREAD                  0x0180,20,20 // ANA_MPLL_DOWNSPREAD;0: center spread;1: down spread
+#define SOC_DPTX_ANA_MPLL_FBDIV_HBIT                  0x0180,19,16 // ANA_MPLL_FBDIV_HBIT;reg_mpll_fbdiv[11:8],The integer part of CORE PLL feedback divide value
+#define SOC_DPTX_ANA_MPLL_OBSEN                       0x0180,15,15 // ANA_MPLL_OBSEN;TBD
+#define SOC_DPTX_ANA_MPLL_OBSSEL                      0x0180,14,14 // ANA_MPLL_OBSSEL;TBD
+#define SOC_DPTX_ANA_MPLL_PREDIV                      0x0180,13,8  // ANA_MPLL_PREDIV;CORE PLL reference divide value
+#define SOC_DPTX_AD_LOCK_COREPLL                      0x0180,7,7   // ad_lock_corepll;1: core pll is locked;0: core pll is not locked
+#define SOC_DPTX_ANA_MPLL_DACPD                       0x0180,5,5   // da_mpll_frac_pd[1];Fractional divider control register
+#define SOC_DPTX_ANA_MPLL_DSMPD                       0x0180,4,4   // da_mpll_frac_pd[0];Fractional divider control register
+#define SOC_DPTX_ANA_MPLL_PD                          0x0180,0,0   // ANA_MPLL_PD;0: power up;1: power down
+
+// SOC_DPTX_MPLL_CTRL1: 0x0184
+#define SOC_DPTX_ANA_MPLL_FRAC_LBIT                   0x0184,23,16 // ANA_MPLL_FRAC_LBIT;reg_mpll_frac[7:0],The fractional part of CORE PLL feedback divide value
+#define SOC_DPTX_ANA_MPLL_FRAC_MBIT                   0x0184,15,8  // ANA_MPLL_FRAC_MBIT;reg_mpll_frac[15:8],The fractional part of CORE PLL feedback divide value
+#define SOC_DPTX_ANA_MPLL_FRAC_HBIT                   0x0184,7,0   // ANA_MPLL_FRAC_HBIT;reg_mpll_frac[23:16],The fractional part of CORE PLL feedback divide value
+
+// SOC_DPTX_MPLL_CTRL2: 0x0188
+#define SOC_DPTX_ANA_MPLL_VCOCLK_DIV8_EN              0x0188,16,16 // ANA_MPLL_VCOCLK_DIV8_EN
+#define SOC_DPTX_ANA_MPLL_POSTDIVEN                   0x0188,11,11 // ANA_MPLL_POSTDIVEN
+#define SOC_DPTX_ANA_MPLL_POSTDIV                     0x0188,10,8  // ANA_MPLL_POSTDIV;CORE PLL reference divide value;3’bxx0: Divided by 2;3’b001: Divided by 4;3’b011: Divided by 8;3’b101: Divided by 16;3’b111: Divided by 32
+
+// SOC_DPTX_PREPLL_CTRL0: 0x0190
+#define SOC_DPTX_ANA_PREPLL_FBDIV2_LBIT               0x0190,31,24 // ANA_PREPLL_FBDIV2_LBIT;da_prepll_fbdiv2[7:0],The integer part of PIXEL PLL feedback divide value
+#define SOC_DPTX_ANA_PREPLL_FBDIV2_HBIT               0x0190,19,16 // ANA_PREPLL_FBDIV2_HBIT;da_prepll_fbdiv2[11:8],The integer part of PIXEL PLL feedback divide value
+#define SOC_DPTX_ANA_PREPLL_PREDIV                    0x0190,13,8  // ANA_PREPLL_PREDIV;PIXEL PLL reference divide value
+#define SOC_DPTX_AD_LOCK_PIXELPLL                     0x0190,7,7   // ad_lock_pixelpll ;1: pixel pll is locked;0: pixel pll is not locked
+#define SOC_DPTX_ANA_PREPLL_DACPD                     0x0190,5,5   // da_prepll_frac_pd[1];Fractional divider control register
+#define SOC_DPTX_ANA_PREPLL_DSMPD                     0x0190,4,4   // da_prepll_frac_pd[0];Fractional divider control register
+#define SOC_DPTX_REG_PCLK_OUTPUT_NORMAL               0x0190,1,1   // REG_PCLK_OUTPUT_NORMAL;1: pin_vid_clk_out output is from prepll;0: pin_vid_clk_out output is pin_ref_clk
+#define SOC_DPTX_ANA_PREPLL_PD                        0x0190,0,0   // ANA_PREPLL_PD;0: power up;1: power down pixel pll
+
+
+// SOC_DPTX_PREPLL_CTRL1: 0x0194
+#define SOC_DPTX_ANA_PREPLL_PRECLK_DIVM               0x0194,17,16 // ANA_PREPLL_PRECLK_DIVM;PIXEL PLL main clock divider control:;2’b00: Divided by 1;2’b01: Divided by 2;2’b10: Divided by 3;2’b11: Divided by 5
+#define SOC_DPTX_ANA_PREPLL_PRECLK_DIVAUX             0x0194,12,8  // ANA_PREPLL_PRECLK_DIVAUX;PIXEL PLL auxiliary clock divider control:;5’h00: Divided by 1;5’h01: Divided by 1;5’h02: Divided by 2;...;5’h3f: Divided by 31
+
+// SOC_DPTX_PREPLL_CTRL2: 0x0198
+#define SOC_DPTX_ANA_PREPLL_PCLKDIV5_EN               0x0198,31,31 // ANA_PREPLL_PCLKDIV5_EN;PREPLL pclkdiv5 enable.;0:disable;1:enable
+#define SOC_DPTX_ANA_PREPLL_PCLK_DIVAUX               0x0198,28,24 // ANA_PREPLL_PCLK_DIVAUX;PIXEL PLL pixel clock divider control:;5’h00: Divided by 1;5’h01: Divided by 1;5’h02: Divided by 2;...;5’h3f: Divided by 31
+#define SOC_DPTX_ANA_PREPLL_LOCK_BYPEN                0x0198,17,17 // ANA_PREPLL_LOCK_BYPEN;prepll lock bypass.;0: prepll lock.;1: bypass,lock always 1.
+#define SOC_DPTX_ANA_PREPLL_LOWFRE_EN                 0x0198,16,16 // ANA_PREPLL_LOWFRE_EN
+#define SOC_DPTX_ANA_PREPLL_TESTEN                    0x0198,15,15 // ANA_PREPLL_TESTEN
+#define SOC_DPTX_ANA_PREPLL_TESTSEL                   0x0198,14,12 // ANA_PREPLL_TESTSEL
+#define SOC_DPTX_ANA_PREPLL_HDMI_EN                   0x0198,9,9   // ANA_PREPLL_HDMI_EN
+#define SOC_DPTX_ANA_PREPLL_DP_EN                     0x0198,8,8   // ANA_PREPLL_DP_EN
+
+// SOC_DPTX_PREPLL_SSC_CTRL: 0x019c
+#define SOC_DPTX_ANA_PREPLL_FRAC2_LBIT                0x019c,23,16 // ANA_PREPLL_FRAC2_LBIT;da_prepll_frac2[7:0],The fractional part of PIXEL PLL feedback divide value
+#define SOC_DPTX_ANA_PREPLL_FRAC2_MBIT                0x019c,15,8  // ANA_PREPLL_FRAC2_MBIT;da_prepll_frac2[15:8],The fractional part of PIXEL PLL feedback divide value
+#define SOC_DPTX_ANA_PREPLL_FRAC2_HBIT                0x019c,7,0   // ANA_PREPLL_FRAC2_HBIT;da_prepll_frac2[23:16],The fractional part of PIXEL PLL feedback divide value
+
+// SOC_DPTX_ANA_TX_CTRL0: 0x01a0
+#define SOC_DPTX_ANA_MPLL_CLKDIV_16M                  0x01a0,13,8  // ANA_MPLL_CLKDIV_16M;MPLL CLK_16M postdiv.
+
+// SOC_DPTX_ANA_TX_ENABLE: 0x01a4
+#define SOC_DPTX_ANA_TX_ISEL_DRV_D3                   0x01a4,31,28 // da_tx_isel_drv_d3[3:0];Current bias control register of lane3.;Ibias = 160uA + 40uA * isel[3:0]
+#define SOC_DPTX_ANA_TX_ISEL_DRV_D2                   0x01a4,27,24 // da_tx_isel_drv_d2[3:0];Current bias control register of lane2.;Ibias = 160uA + 40uA * isel[3:0]
+#define SOC_DPTX_AD_TX_ESDSENSE_D3                    0x01a4,23,22 // AD_TX_ESDSENSE_D3
+#define SOC_DPTX_AD_TX_ESDSENSE_D2                    0x01a4,21,20 // AD_TX_ESDSENSE_D2
+#define SOC_DPTX_AD_TX_ESDSENSE_D1                    0x01a4,19,18 // AD_TX_ESDSENSE_D1
+#define SOC_DPTX_AD_TX_ESDSENSE_D0                    0x01a4,17,16 // AD_TX_ESDSENSE_D0
+#define SOC_DPTX_ANA_TX_EN_LDO_D3                     0x01a4,11,11 // reg_tx_en_ldo_d3;LDO enable control of data lane3;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_LDO_D2                     0x01a4,10,10 // reg_tx_en_ldo_d2;LDO enable control of data lane2;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_LDO_D1                     0x01a4,9,9   // reg_tx_en_ldo_d1;LDO enable control of data lane1;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_LDO_D0                     0x01a4,8,8   // reg_tx_en_ldo_d0;LDO enable control of data lane0;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_BYP_D3                     0x01a4,7,7   // ANA_TX_EN_BYP_D3
+#define SOC_DPTX_ANA_TX_EN_BYP_D2                     0x01a4,6,6   // ANA_TX_EN_BYP_D2
+#define SOC_DPTX_ANA_TX_EN_BYP_D1                     0x01a4,5,5   // ANA_TX_EN_BYP_D1
+#define SOC_DPTX_ANA_TX_EN_BYP_D0                     0x01a4,4,4   // ANA_TX_EN_BYP_D0
+#define SOC_DPTX_ANA_TX_EN_DRV_D3                     0x01a4,3,3   // reg_tx_en_drv_d3;Output driver enable control of data lane3;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_DRV_D2                     0x01a4,2,2   // reg_tx_en_drv_d2;Output driver enable control of data lane2;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_DRV_D1                     0x01a4,1,1   // reg_tx_en_drv_d1;Output driver enable control of data lane1;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_DRV_D0                     0x01a4,0,0   // reg_tx_en_drv_d0;Output driver enable control of data lane0;1’b0: disable;1’b1: enable
+
+// SOC_DPTX_ANA_TX_CTRL1: 0x01a8
+#define SOC_DPTX_ANA_TX_PHASE_D2                      0x01a8,31,30 //
+#define SOC_DPTX_ANA_TX_MAINSEL_D2                    0x01a8,28,24 // da_tx_mainsel_d2[4:0];Output voltage level control registers of lane2.;Imain = Ibias * mainsel[4:0]
+#define SOC_DPTX_ANA_TX_PHASE_D3                      0x01a8,23,22 //
+#define SOC_DPTX_ANA_TX_MAINSEL_D3                    0x01a8,20,16 // da_tx_mainsel_d3[4:0];Output voltage level control registers of lane3.;Imain = Ibias * mainsel[4:0]
+#define SOC_DPTX_ANA_TX_ISEL_LDO_D3                   0x01a8,11,11 //
+#define SOC_DPTX_ANA_TX_ISEL_LDO_D2                   0x01a8,10,10 //
+#define SOC_DPTX_ANA_TX_ISEL_LDO_D1                   0x01a8,9,9 //
+#define SOC_DPTX_ANA_TX_ISEL_LDO_D0                   0x01a8,8,8 //
+#define SOC_DPTX_ANA_TX_ISEL_DRV_D1                   0x01a8,7,4   // da_tx_isel_drv_d1[3:0];Current bias control register of lane1.;Ibias = 160uA + 40uA * isel[3:0]
+#define SOC_DPTX_ANA_TX_ISEL_DRV_D0                   0x01a8,3,0   // da_tx_isel_drv_d0[3:0];Current bias control register of lane0.;Ibias = 160uA + 40uA * isel[3:0]
+
+// SOC_DPTX_ANA_TX_CTRL2: 0x01ac
+#define SOC_DPTX_ANA_TX_POSTSEL_D1                    0x01ac,31,28 // da_tx_postsel_d1[3:0];Post-cursor pre-emphasis level control registers of lane1.;Ipost = Ibias * postsel[3:0]
+#define SOC_DPTX_ANA_TX_POSTSEL_D0                    0x01ac,27,24 // da_tx_postsel_d0[3:0];Post-cursor pre-emphasis level control registers of lane0.;Ipost = Ibias * postsel[3:0]
+#define SOC_DPTX_ANA_TX_POSTSEL_D3                    0x01ac,23,20 // da_tx_postsel_d3[3:0];Post-cursor pre-emphasis level control registers of lane3.;Ipost = Ibias * postsel[3:0]
+#define SOC_DPTX_ANA_TX_POSTSEL_D2                    0x01ac,19,16 // da_tx_postsel_d2[3:0];Post-cursor pre-emphasis level control registers of lane2.;Ipost = Ibias * postsel[3:0]
+#define SOC_DPTX_ANA_TX_PHASE_D0                      0x01ac,15,14 //
+#define SOC_DPTX_DA_TX_MAINSEL_D0_4_0                 0x01ac,12,8  // da_tx_mainsel_d0[4:0];Output voltage level control registers of lane0.;Imain = Ibias * mainsel[4:0]
+#define SOC_DPTX_ANA_TX_PHASE_D1                      0x01ac,7,6   //
+#define SOC_DPTX_ANA_TX_MAINSEL_D1                    0x01ac,4,0   // da_tx_mainsel_d1[4:0];Output voltage level control registers of lane1.;Imain = Ibias * mainsel[4:0]
+
+// SOC_DPTX_ANA_TX_CTRL3: 0x01b0
+#define SOC_DPTX_ANA_TX_EN_P2S_D3                     0x01b0,31,31 // reg_tx_en_p2s_d3/2/1/0;Serializer enable control of 4 data lanes;1’b0: disable;1’b1: enable
+#define SOC_DPTX_ANA_TX_EN_P2S_D2                     0x01b0,30,30 //
+#define SOC_DPTX_ANA_TX_EN_P2S_D1                     0x01b0,29,29 //
+#define SOC_DPTX_ANA_TX_EN_P2S_D0                     0x01b0,28,28 //
+#define SOC_DPTX_ANA_TX_MODE_D3                       0x01b0,27,27 // reg_tx_mode_d3/2/1/0;Output driver mode control of 4 data lanes;1’b0: select voltage mode driver;1’b1: enable current mode driver;
+#define SOC_DPTX_ANA_TX_MODE_D2                       0x01b0,26,26 //
+#define SOC_DPTX_ANA_TX_MODE_D1                       0x01b0,25,25 //
+#define SOC_DPTX_ANA_TX_MODE_D0                       0x01b0,24,24 //
+#define SOC_DPTX_ANA_TX_PRESEL_D1                     0x01b0,14,12 // da_tx_presel_d1[2:0];Pre-cursor pre-emphasis level control registers of lane1.;Ipre = Ibias * presel[2:0]
+#define SOC_DPTX_ANA_TX_PRESEL_D0                     0x01b0,10,8  // da_tx_presel_d0[2:0];Pre-cursor pre-emphasis level control registers of lane0.;Ipre = Ibias * presel[2:0]
+#define SOC_DPTX_ANA_TX_PRESEL_D3                     0x01b0,6,4   // da_tx_presel_d3[2:0];Pre-cursor pre-emphasis level control registers of lane3.;Ipre = Ibias * presel[2:0]
+#define SOC_DPTX_ANA_TX_PRESEL_D2                     0x01b0,2,0   // da_tx_presel_d2[2:0];Pre-cursor pre-emphasis level control registers of lan2.;Ipre = Ibias * presel[2:0]
+
+// SOC_DPTX_ANA_TX_CTRL6: 0x01bc
+#define SOC_DPTX_ANA_BG_VREF5                         0x01bc,15,14 // ANA_BG_VREF5
+#define SOC_DPTX_ANA_BG_VREF4                         0x01bc,13,12 // ANA_BG_VREF4
+#define SOC_DPTX_ANA_BG_VREF3                         0x01bc,11,10 // ANA_BG_VREF3
+#define SOC_DPTX_ANA_BG_VREF2                         0x01bc,9,8   // ANA_BG_VREF2
+#define SOC_DPTX_ANA_BG_VREF                          0x01bc,7,6   // ANA_BG_VREF
+#define SOC_DPTX_ANA_BG_VREF1                         0x01bc,5,4   // ANA_BG_VREF1
+#define SOC_DPTX_ANA_BG_EN_CHOP                       0x01bc,3,3   // ANA_BG_EN_CHOP
+#define SOC_DPTX_ANA_BG_EN                            0x01bc,2,2   // ANA_BG_EN
+#define SOC_DPTX_ANA_BG_ISEL                          0x01bc,1,1   // ANA_BG_ISEL
+#define SOC_DPTX_ANA_BG_VSEL                          0x01bc,0,0   // ANA_BG_VSEL
+
+// SOC_DPTX_ANA_TX_CTRL7: 0x01c0
+#define SOC_DPTX_ANA_BG_EN_RCAL                       0x01c0,27,27 // ANA_BG_EN_RCAL;Termination resistor calibration enable control in Bias;1’b0: disable;1’b1: enable;W: software configure value.only ANA_RTCAL_BYPASS is 1'b1 valid.;R: analog real value,software configure or hardware generate.
+#define SOC_DPTX_ANA_BG_RCAL_SEL                      0x01c0,26,25 // ANA_BG_RCAL_SEL;The resistance calibration,adjust the termination resistor to approach the off-chip reference resistor.;W: software configure value.only ANA_RTCAL_BYPASS is 1'b1 valid.;R: analog real value,software configure or hardware generate.
+#define SOC_DPTX_AD_BG_RCAL_OUT                       0x01c0,24,24 // AD_BG_RCAL_OUT;The resistance calibration result.
+#define SOC_DPTX_ANA_RTCAL_FREQDIV_LBIT               0x01c0,23,16 // ANA_RTCAL_FREQDIV_LBIT;reg_rtcal_freqdiv[7:0],The resistance calibration clock divider.
+#define SOC_DPTX_ANA_RTCAL_BYPASS                     0x01c0,15,15 // ANA_RTCAL_BYPASS;Control the configuration method for termination resistance.;0: hardware adjust.;1: bypass,software configure.
+#define SOC_DPTX_ANA_RTCAL_FREQDIV_HBIT               0x01c0,14,8  // ANA_RTCAL_FREQDIV_HBIT;reg_rtcal_freqdiv[14:8],The resistance calibration clock divider.
+#define SOC_DPTX_ANA_BG_RCAL_VAL                      0x01c0,5,0   // ANA_BG_RCAL_VAL;The resistance calibration software configure value.;W: software configure value.only ANA_RTCAL_BYPASS is 1'b1 valid.;R: analog real value,software configure or hardware generate.
+
+// SOC_DPTX_ANA_TX_CTRL8: 0x01c4
+#define SOC_DPTX_ANA_TX_RTM_D3                        0x01c4,29,24 // ANA_TX_RTM_D3;Differential termination resistance control registers of lane3,RT = 4000 / rtm[5:0];W: software configure value.only ANA_RTCAL_BYPASS is 1'b1 valid.;R: analog real value,software configure or hardware generate.
+#define SOC_DPTX_ANA_TX_RTM_D2                        0x01c4,21,16 // reg_tx_rtm_d2[5:0];Differential termination resistance control registers of lane2,RT = 4000 / rtm[5:0];W: software configure value.only ANA_RTCAL_BYPASS is 1'b1 valid.;R: analog real value,software configure or hardware generate.
+#define SOC_DPTX_ANA_TX_RTM_D1                        0x01c4,13,8  // reg_tx_rtm_d1[5:0];Differential termination resistance control registers of lane1,RT = 4000 / rtm[5:0];W: software configure value.only ANA_RTCAL_BYPASS is 1'b1 valid.;R: analog real value,software configure or hardware generate.
+#define SOC_DPTX_ANA_TX_RTM_D0                        0x01c4,5,0   // ANA_TX_RTM_D0;Differential termination resistance control registers of lane0,RT = 4000 / rtm[5:0];W: software configure value.only ANA_RTCAL_BYPASS is 1'b1 valid.;R: analog real value,software configure or hardware generate.
+
+// SOC_DPTX_ANA_TX_AUX_CTRL: 0x01d0
+#define SOC_DPTX_ANA_TX_AUX_RX_VSEL                   0x01d0,13,12 //
+
+// SOC_DPTX_ANA_LANE_EMPH: 0x01dc
+#define SOC_DPTX_ANA_TX_POSTSEL_PRE_D3                0x01dc,23,20 //
+#define SOC_DPTX_ANA_TX_POSTSEL_PRE_D2                0x01dc,19,16 //
+#define SOC_DPTX_ANA_TX_POSTSEL_PRE_D1                0x01dc,15,12 //
+#define SOC_DPTX_ANA_TX_POSTSEL_PRE_D0                0x01dc,11,8  //
+#define SOC_DPTX_ANA_TX_MODE_DE_D3                    0x01dc,7,7   //
+#define SOC_DPTX_ANA_TX_MODE_DE_D2                    0x01dc,6,6   //
+#define SOC_DPTX_ANA_TX_MODE_DE_D1                    0x01dc,5,5   //
+#define SOC_DPTX_ANA_TX_MODE_DE_D0                    0x01dc,4,4   //
+#define SOC_DPTX_ANA_TX_MODE_PRE_D3                   0x01dc,3,3   //
+#define SOC_DPTX_ANA_TX_MODE_PRE_D2                   0x01dc,2,2   //
+#define SOC_DPTX_ANA_TX_MODE_PRE_D1                   0x01dc,1,1   //
+#define SOC_DPTX_ANA_TX_MODE_PRE_D0                   0x01dc,0,0   //
+
+// SOC_DPTX_VIDEO_VSAMPLE_REG: 0x0200
+#define SOC_DPTX_VIDEO_STREAM_ENABLE                  0x0200,28,28 // Video_stream_en;0:video stream disable;1: video stream enable
+#define SOC_DPTX_VIDEO_MAPPING                        0x0200,26,22 // VIDEO_MAPPING;the bit width of each color;0: RGB 6bits;1: RGB 8bits;2: RGB 10bits;3: RGB 12bits;4: RGB 16bits;5: YCbCR4:4:4 8bits;6: YCbCr4:4:4 10bitss;7: YCbCr4:4:4 12bits;8: YCbCr4:4:4 16bits;9: YCbCR4:2:2 8bits;10: YCbCr4:2:2 10bits;11: YCbCr4:2:2 12bits;12: YCbCr4:2:2 16bits;13: YCbCr 4:2:0 8bits ;14: YCbCr 4:2:0 10bits ;15: YCbCr 4:2:0 12bits ;16: YCbCr 4:2:0 16bits;others: Reserved
+#define SOC_DPTX_STREAM_ENC_EN                        0x0200,18,18 // STREAM_ENC_EN;alternate_scrambler_reset_capable,a setting of 1 indicates that this is an DP device that can use the DP alternate scrambler reset value of FFFEh
+
+// SOC_DPTX_VINPUT_POLARITY: 0x020c
+#define SOC_DPTX_HSYNC_IN_POLARITY                    0x020c,29,29 // reg_tx_polarity[1:0];polarity of hsync and vsync.;bit1 : Vsync polarity ;bit0 : Hsync polarity;if the vsync/hsync of simulation video source is active low,then the corresponding bit in reg_tx_polarity should set 0; or active high to set 1.
+#define SOC_DPTX_VSYNC_IN_POLARITY                    0x020c,28,28 //
+
+// SOC_DPTX_VIDEO_CONFIG2: 0x0210
+#define SOC_DPTX_VACTIVE                              0x0210,31,16 // VACTIVE;reg_tx_vactive [15:0],Active vertical pixels per line
+#define SOC_DPTX_VBLANK                               0x0210,15,0  // VBLANK;reg_tx_vblank [15:0],vtotal – vactive
+
+// SOC_DPTX_VIDEO_CONFIG1: 0x0214
+#define SOC_DPTX_HBLANK                               0x0214,29,16 // HBLANK;reg_tx_hblank [13:0],htotal – hactive
+#define SOC_DPTX_HACTIVE                              0x0214,15,0  // HACTIVE;reg_tx_hactive [15:0],Active horizontal pixels per
+
+// SOC_DPTX_VIDEO_CONFIG5: 0x0218
+#define SOC_DPTX_AVERAGE_BYTES_PER_TU                 0x0218,31,25 // reg_avg_per_tu_int;integer portion of average valid symbol per Transfer Unit,calculate formula refer to DP spec.;For example,pixel clock is 27M RGB888,@2.7Gbps 4lanes,then average valid symbol per Transfer Unit  = 4.8,configure reg_avg_per_tu_int to 4
+#define SOC_DPTX_AVERAGE_BYTES_PER_TU_FRAC            0x0218,24,21 // reg_avg_per_tu_frac;fractional portion of average valid symbol per Transfer Unit,calculate formula refer to DP spec.;For example,pixel clock is 27M RGB888,@2.7Gbps 4lanes,then average valid symbol per Transfer Unit  = 4.8,configure reg_avg_per_tu_frac to 8
+#define SOC_DPTX_INIT_THRESHOLD                       0x0218,18,10 // reg_line_rd_thres;An asynchronous fifo needed to deal with video data from pixel clock domain to link clock domain ,reg_line_rd_thres is related to read enable of fifo. According to video format and main link rate,this value may changed as  following description:;reg_line_rd_thres = ; (average valid symbol per TU)  < 6 ? 7'd32 :;(hblank < 80) ? 7'd12 : 7'd16;
+
+// SOC_DPTX_VIDEO_CONFIG3: 0x021c
+#define SOC_DPTX_H_FRONT_PORCH                        0x021c,31,16 // H_FRONT_PORCH;reg_tx_h_front_porch [15:0],htotal – hactive – hstart
+#define SOC_DPTX_H_SYNC_WIDTH                         0x021c,15,0  // H_SYNC_WIDTH;reg_tx_hswidth [15:0],Hsync width
+
+// SOC_DPTX_VIDEO_CONFIG4: 0x0220
+#define SOC_DPTX_V_FRONT_PORCH                        0x0220,31,16 // V_FRONT_PORCH;reg_tx_v_front_porch [15:0],vtotal – vactive – vstart
+#define SOC_DPTX_V_SYNC_WIDTH                         0x0220,15,0  // V_SYNC_WIDTH;reg_tx_vswidth [15:0],Vsync width
+
+// SOC_DPTX_VIDEO_MSA1: 0x0224
+#define SOC_DPTX_HSTART                               0x0224,31,16 // reg_tx_hstart [15:0];Hsync width + Hback porch width
+#define SOC_DPTX_VSTART                               0x0224,15,0  // reg_tx_vstart[15:0];Vsync width + Vback porch width
+
+// SOC_DPTX_VIDEO_MSA2: 0x0228
+#define SOC_DPTX_MISC0                                0x0228,7,0   // reg_tx_msa_misc0[7:0];MSA Miscellaneous0 field;bit0: 1’b0,IP Link clock and main video stream clock asynchronous. ;tips: the bit must set 1’b0.;bit1~bit7: color encoding format ,please refer to DP spec;for example: 6’b0000000,6’b 0010000,;6’b 0100000,6’b 0110000,6’b 1000000 ;(6,8,10,12,16 bits/color respectively)
+
+// SOC_DPTX_VIDEO_MSA3: 0x022c
+#define SOC_DPTX_NVID                                 0x022c,31,8  // NVID
+#define SOC_DPTX_MISC1                                0x022c,7,0   // reg_tx_msa_misc1[7:0];MSA Miscellaneous1 field;please refer to DP spec
+
+// SOC_DPTX_VIDEO_HBLANK_INTERVAL: 0x0230
+#define SOC_DPTX_HBLANK_INTERVAL                      0x0230,31,16 // hblank_link_cyc;How many link clock cycles of hblank .;hblank *link_clk/pixel_clk,;For example,pixel clock is 27M hblank 138,@2.7Gbps 4lanes,hblank_link_cyc; = 138*67.5/27. link_clk = symbol clock /4
+
+// SOC_DPTX_VIDEO_COLORBAR_CFG: 0x0238
+#define SOC_DPTX_VID_BIST_EN                          0x0238,0,0   // VID_BIST_EN;video bist mode enable.
+
+// SOC_DPTX_AUDIO_CONFIG1: 0x0300
+#define SOC_DPTX_AUDIO_INF_SELECT                     0x0300,31,31 // AUDIO_INF_SELECT;1’b0:select i2S as input(default);1’b1:select spdif as input
+#define SOC_DPTX_AUDIO_MUTE                           0x0300,30,30 // AUDIO_MUTE;1’b0: Clears the AudioMute_Flag in VB-ID.;1’b1: Sets the AudioMute_Flag in VB-ID.
+#define SOC_DPTX_HBR_MODE_ENABLE                      0x0300,29,29 // HBR_MODE_ENABLE
+#define SOC_DPTX_AUDIO_DATA_IN_EN                     0x0300,28,25 // AUDIO_DATA_IN_EN;Indicates whether the input data is vaild.;Bit 1:indicates whether the channels 1,2  input data is vaild.;Bit 2:indicates whether the channels 3,4  input data is vaild.;Bit 3:indicates whether the channels 5,6  input data is vaild.;Bit 4:indicates whether the channels 7,8  input data is vaild.;tips: must be consistent with audio_channel_num(0x0300[16:14])
+#define SOC_DPTX_I2S_AUDIO_MODE                       0x0300,24,23 // I2S_AUDIO_MODE ;1: left adjusted.;2: right adjusted.;others: standard mode.
+#define SOC_DPTX_AUD_ADJUST_SEL                       0x0300,22,22 // AUD_ADJUST_SEL;0: I2S origin data;1: I2S format adjust data
+#define SOC_DPTX_AUDIO_DATA_WIDTH                     0x0300,21,17 // AUDIO_DATA_WIDTH;Indicates the bit width of the data samples at input.;16: 16 bits;17: 17 bits;18: 18 bits;19: 19 bits;20: 20 bits;21: 21 bits;22: 22 bits;23: 23 bits;others: 24 bits
+#define SOC_DPTX_AUDIO_NUM_CHANNELS                   0x0300,16,14 // AUDIO_NUM_CHANNELS;Number of audio channels.;0: 1 channel;1: 2 channel;Others:8 channel
+#define SOC_DPTX_AUDIO_PACKET_ID                      0x0300,13,6  // AUDIO_PACKET_ID;audio packrt packet ID.
+#define SOC_DPTX_AUDIO_TIMESTAMP_VERSION_NUM          0x0300,5,0   // AUDIO_TIMESTAMP_VERSION_NUM;audio packet version num.
+
+// SOC_DPTX_AUX_STS_REG: 0x0400
+#define SOC_DPTX_AUX_STATUS                           0x0400,31,24 // Aux_reply_cmd;4’b0000:ack;4’b0001:nack
+#define SOC_DPTX_AUX_REPLY_ERR_CODE                   0x0400,6,4   // AUX_REPLY_ERR_CODE;0: no error;1: AUX DATA PAYLOAD detect less than 8.;4: AUX STOP_L STATE detect greater than 2.;5: AUX SYNC_H STATE detect greater than 2.;6: AUX SYNC_L STATE detect greater than 2.;7: AUX STOP_H STATE detect greater than 2.;others: reserved
+
+// SOC_DPTX_AUX_CMD_REG: 0x0404
+#define SOC_DPTX_AUX_LENGTH                           0x0404,27,24 // AUX_LENGTH;length,the number of bytes to be transfer by AUX requester,equal to (bytes of data )-1,please refer to Native AUX syntax of spec.
+#define SOC_DPTX_AUX_ADDR                             0x0404,23,4  // AUX_ADDR;addr[19:0],the address to be transfer by AUX requester,please refer to Native AUX syntax of spec.
+#define SOC_DPTX_AUX_CMD_TYPE                         0x0404,3,0   // AUX_CMD_TYPE;AUX cmd[3:0],the command to be transfer by AUX requester.;4’b1000->Native AUX Wite; ;4’b1001->Native AUX read; ;Please refer to Native AUX syntax of DP spec.
+
+// SOC_DPTX_AUX_START_REG: 0x0408
+#define SOC_DPTX_AUX_START                            0x0408,31,31 // AUX_START;the trigger signal to start a Wite/read request by AUX requester(Source Devive),wirte to 1.
+
+// SOC_DPTX_AUX_DATA4_REG: 0x040c
+#define SOC_DPTX_AUX_DATA4                            0x040c,31,0  // cfg_phy_aux_data[127:96],cfg_phy_aux_data composed of 0x0418,0x0414,0x0410,0x040C,totally 16bytes,including data(8bits*16),payload for Wite request or read request. ;tips: for a RWite request ,filled attached data needs to be transmitted;for a read reply ,stored the replied data.
+
+// SOC_DPTX_AUX_DATA3_REG: 0x0410
+#define SOC_DPTX_AUX_DATA3                            0x0410,31,0  // cfg_phy_aux_data[95:64],cfg_phy_aux_data composed of 0x0418,0x0414,0x0410,0x040C,totally 16bytes,including data(8bits*16),payload for RWite request or read request.;tips: for a RWite request ,filled attached data needs to be transmitted;for a read reply ,stored the replied data.
+
+// SOC_DPTX_AUX_DATA2_REG: 0x0414
+#define SOC_DPTX_AUX_DATA2                            0x0414,31,0  // cfg_phy_aux_data[63:32],cfg_phy_aux_data composed of 0x0418,0x0414,0x0410,0x040C,totally 16bytes,including data(8bits*16),payload for RWite request or read request.;tips: for a RWite request ,filled attached data needs to be transmitted;for a read reply ,stored the replied data.
+
+// SOC_DPTX_AUX_DATA1_REG: 0x0418
+#define SOC_DPTX_AUX_DATA1                            0x0418,31,0  // cfg_phy_aux_data[31:0],cfg_phy_aux_data composed of 0x0418,0x0414,0x0410,0x040C,totally 16bytes,including data(8bits*16),payload for RWite request or read request.;tips: for a RWite request ,filled attached data needs to be transmitted;for a read reply ,stored the replied data.
+
+// SOC_DPTX_SDP_HORIZONTAL_CTRL: 0x0420
+#define SOC_DPTX_AUD_STREAM_HORIZONTAL_EN             0x0420,13,13 // AUD_STREAM_HORIZONTAL_EN;1’b0:disable audio stream is transmitted during horizontal blanking period.;1’b1:Enable audio stream is transmitted during horizontal blanking period.
+#define SOC_DPTX_AUD_TIMESTAMP_HORIZONTAL_EN          0x0420,12,12 // AUD_TIMESTAMP_HORIZONTAL_EN;1’b0:disable send audio timestamp SDP once every video frame during horizontal blanking period.;1’b1:Enable send audio timestamp SDP once every video frame during horizontal blanking period.
+
+// SOC_DPTX_SDP_VERTICAL_CTRL: 0x0424
+#define SOC_DPTX_AUD_STREAM_VERTICAL_EN               0x0424,13,13 // AUD_STREAM_VERTICAL_EN;1’b0:disable audio stream is transmitted during vertical blanking period.;1’b1:Enable audio stream is transmitted during vertical blanking period.
+#define SOC_DPTX_AUD_TIMESTAMP_VERTICAL_EN            0x0424,12,12 // AUD_TIMESTAMP_VERTICAL_EN;1’b0:disable send audio timestamp SDP once every video frame during vertical blanking period.;1’b1:Enable send audio timestamp SDP once every video frame during vertical blanking period.
+
+#endif /* __SPACEMIT_INNO_DP_H__ */
diff --git a/drivers/gpu/drm/spacemit/spacemit_lib.c b/drivers/gpu/drm/spacemit/spacemit_lib.c
new file mode 100644
index 000000000000..346861ae1995
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_lib.c
@@ -0,0 +1,248 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#define pr_fmt(__fmt) "[drm][%20s] "__fmt, __func__
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/of_graph.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/of.h>
+
+#include "spacemit_lib.h"
+
+struct bmp_header {
+	u16 magic;
+	u32 size;
+	u32 unused;
+	u32 start;
+} __attribute__((__packed__));
+
+struct dib_header {
+	u32 size;
+	u32 width;
+	u32 height;
+	u16 planes;
+	u16 bpp;
+	u32 compression;
+	u32 data_size;
+	u32 h_res;
+	u32 v_res;
+	u32 colours;
+	u32 important_colours;
+	u32 red_mask;
+	u32 green_mask;
+	u32 blue_mask;
+	u32 alpha_mask;
+	u32 colour_space;
+	u32 unused[12];
+} __attribute__((__packed__));
+
+int str_to_u32_array(const char *p, u32 base, u32 array[])
+{
+	const char *start = p;
+	char str[12];
+	int length = 0;
+	int i, ret;
+
+	pr_info("input: %s", p);
+
+	for (i = 0 ; i < 255; i++) {
+		while (*p == ' ')
+			p++;
+		if (*p == '\0')
+			break;
+		start = p;
+
+		while ((*p != ' ') && (*p != '\0'))
+			p++;
+
+		if ((p - start) >= sizeof(str))
+			break;
+
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, p - start);
+
+		ret = kstrtou32(str, base, &array[i]);
+		if (ret) {
+			DRM_ERROR("input format error\n");
+			break;
+		}
+
+		length++;
+	}
+
+	return length;
+}
+EXPORT_SYMBOL_GPL(str_to_u32_array);
+
+int str_to_u8_array(const char *p, u32 base, u8 array[])
+{
+	const char *start = p;
+	char str[12];
+	int length = 0;
+	int i, ret;
+
+	pr_info("input: %s", p);
+
+	for (i = 0 ; i < 255; i++) {
+		while (*p == ' ')
+			p++;
+		if (*p == '\0')
+			break;
+		start = p;
+
+		while ((*p != ' ') && (*p != '\0'))
+			p++;
+
+		if ((p - start) >= sizeof(str))
+			break;
+
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, p - start);
+
+		ret = kstrtou8(str, base, &array[i]);
+		if (ret) {
+			DRM_ERROR("input format error\n");
+			break;
+		}
+
+		length++;
+	}
+
+	return length;
+}
+EXPORT_SYMBOL_GPL(str_to_u8_array);
+
+#if 0//IS_ENABLED(CONFIG_GKI_FIX_WORKAROUND)
+static struct file *gki_filp_open(const char *filename, int flags, umode_t mode)
+{
+	return 0;
+}
+static ssize_t gki_vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
+{
+	return 0;
+}
+#endif
+
+void *disp_ops_attach(const char *str, struct list_head *head)
+{
+	struct ops_list *list;
+	const char *ver;
+
+	list_for_each_entry(list, head, head) {
+		ver = list->entry->ver;
+		if (!strcmp(str, ver))
+			return list->entry->ops;
+	}
+
+	DRM_ERROR("attach disp ops %s failed\n", str);
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(disp_ops_attach);
+
+int disp_ops_register(struct ops_entry *entry, struct list_head *head)
+{
+	struct ops_list *list;
+
+	list = kzalloc(sizeof(struct ops_list), GFP_KERNEL);
+	if (!list)
+		return -ENOMEM;
+
+	list->entry = entry;
+	list_add(&list->head, head);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(disp_ops_register);
+
+struct device *spacemit_disp_pipe_get_by_port(struct device *dev, int port)
+{
+	struct device_node *np = dev->of_node;
+	struct device_node *endpoint;
+	struct device_node *remote_node;
+	struct platform_device *remote_pdev;
+
+	endpoint = of_graph_get_endpoint_by_regs(np, port, 0);
+	if (!endpoint) {
+		DRM_ERROR("%s/port%d/endpoint0 was not found\n",
+			  np->full_name, port);
+		return NULL;
+	}
+
+	remote_node = of_graph_get_remote_port_parent(endpoint);
+	if (!remote_node) {
+		DRM_ERROR("device node was not found by endpoint0\n");
+		return NULL;
+	}
+
+	remote_pdev = of_find_device_by_node(remote_node);
+	if (remote_pdev == NULL) {
+		DRM_ERROR("find %s platform device failed\n",
+			  remote_node->full_name);
+		return NULL;
+	}
+
+	return &remote_pdev->dev;
+}
+EXPORT_SYMBOL_GPL(spacemit_disp_pipe_get_by_port);
+
+struct device *spacemit_disp_pipe_get_input(struct device *dev)
+{
+	return spacemit_disp_pipe_get_by_port(dev, 1);
+}
+EXPORT_SYMBOL_GPL(spacemit_disp_pipe_get_input);
+
+struct device *spacemit_disp_pipe_get_output(struct device *dev)
+{
+	return spacemit_disp_pipe_get_by_port(dev, 0);
+}
+EXPORT_SYMBOL_GPL(spacemit_disp_pipe_get_output);
+
+/*
+ * copy from drm opensource, change name from drm_atomic_replace_property_blob_from_id
+ * to spacemit_atomic_replace_property_blob_from_id
+ */
+int spacemit_atomic_replace_property_blob_from_id(struct drm_device *dev,
+					 struct drm_property_blob **blob,
+					 uint64_t blob_id,
+					 ssize_t expected_size,
+					 ssize_t expected_elem_size,
+					 bool *replaced)
+{
+	struct drm_property_blob *new_blob = NULL;
+
+	if (blob_id != 0) {
+		new_blob = drm_property_lookup_blob(dev, blob_id);
+		if (new_blob == NULL)
+			return -EINVAL;
+
+		if (expected_size > 0 &&
+		    new_blob->length != expected_size) {
+			drm_property_blob_put(new_blob);
+			return -EINVAL;
+		}
+		if (expected_elem_size > 0 &&
+		    new_blob->length % expected_elem_size != 0) {
+			drm_property_blob_put(new_blob);
+			return -EINVAL;
+		}
+	}
+
+	*replaced |= drm_property_replace_blob(blob, new_blob);
+	drm_property_blob_put(new_blob);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(spacemit_atomic_replace_property_blob_from_id);
+
+MODULE_DESCRIPTION("display common API library");
+MODULE_LICENSE("GPL");
+
diff --git a/drivers/gpu/drm/spacemit/spacemit_lib.h b/drivers/gpu/drm/spacemit/spacemit_lib.h
new file mode 100644
index 000000000000..ad3bca732153
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_lib.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_LIB_H_
+#define _SPACEMIT_LIB_H_
+
+#include <drm/drm_print.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_mode_object.h>
+#include <linux/list.h>
+
+struct ops_entry {
+	const char *ver;
+	void *ops;
+};
+
+struct ops_list {
+	struct list_head head;
+	struct ops_entry *entry;
+};
+
+int str_to_u32_array(const char *p, u32 base, u32 array[]);
+int str_to_u8_array(const char *p, u32 base, u8 array[]);
+int dump_bmp32(const char *p, u32 width, u32 height,
+		bool bgra, const char *filename);
+
+void *disp_ops_attach(const char *str, struct list_head *head);
+int disp_ops_register(struct ops_entry *entry, struct list_head *head);
+
+struct device *spacemit_disp_pipe_get_by_port(struct device *dev, int port);
+struct device *spacemit_disp_pipe_get_input(struct device *dev);
+struct device *spacemit_disp_pipe_get_output(struct device *dev);
+int spacemit_atomic_replace_property_blob_from_id(struct drm_device *dev,
+					 struct drm_property_blob **blob,
+					 uint64_t blob_id,
+					 ssize_t expected_size,
+					 ssize_t expected_elem_size,
+					 bool *replaced);
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/spacemit_mipi_panel.c b/drivers/gpu/drm/spacemit/spacemit_mipi_panel.c
new file mode 100644
index 000000000000..d6b1aa1aeb3c
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_mipi_panel.c
@@ -0,0 +1,1131 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <linux/atomic.h>
+#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/delay.h>
+#include <video/mipi_display.h>
+#include <video/of_display_timing.h>
+#include <video/videomode.h>
+
+#include "spacemit_bootloader.h"
+#include "spacemit_mipi_panel.h"
+#include "spacemit_dsi.h"
+#include "spacemit_crtc.h"
+#include "dpu/dpu_saturn.h"
+#include "sysfs/sysfs_display.h"
+#include "./common/spacemit_drm_notifier.h"
+#include "./backlight/spacemit-backlight.h"
+
+const char *lcd_name;
+
+static void _spacemit_dsi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct spacemit_dsi *dsi = encoder_to_dsi(encoder);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(encoder->crtc);
+	int ret = 0;
+
+	DRM_INFO("%s()\n", __func__);
+
+	if (dsi->panel && dsi->panel->backlight) {
+		ret = backlight_disable(dsi->panel->backlight);
+		if (ret < 0)
+			DRM_DEV_INFO(dsi->panel->dev, "failed to disable backlight: %d\n", ret);
+	}
+
+	a_crtc->is_stopped = true;
+	spacemit_crtc_stop(a_crtc);
+
+	if (dsi->core && dsi->core->dsi_close_datatx)
+		dsi->core->dsi_close_datatx(&dsi->ctx);
+
+	if (dsi->panel) {
+		drm_panel_disable(dsi->panel);
+		drm_panel_unprepare(dsi->panel);
+	}
+
+	if (dsi->core && dsi->core->dsi_close)
+		dsi->core->dsi_close(&dsi->ctx);
+}
+
+static void _spacemit_dsi_encoder_enable(struct drm_encoder *encoder)
+{
+	struct spacemit_dsi *dsi = encoder_to_dsi(encoder);
+	struct spacemit_panel *panel = container_of(dsi->panel, struct spacemit_panel, base);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(encoder->crtc);
+	void __iomem *addr = (void __iomem *)ioremap(0xd421a1a8, 100);
+
+	if (!dsi->core || !dsi->core->dsi_open) {
+		DRM_ERROR("%s(), dsi->core is null!\n", __func__);
+		return;
+	}
+
+	if (panel->encoder == NULL)
+		panel->encoder = encoder;
+
+	/* Dove dsi online setup */
+	writel(0x40000001, addr);
+	writel(0x1, addr + 0x10);
+
+	dsi->core->dsi_open(&dsi->ctx, false);
+
+	if (dsi->panel) {
+		drm_panel_prepare(dsi->panel);
+		drm_panel_enable(dsi->panel);
+		if (panel->info.bl_enable_delay)
+			backlight_disable(dsi->panel->backlight);
+	}
+
+	if (dsi->core && dsi->core->dsi_ready_for_datatx)
+		dsi->core->dsi_ready_for_datatx(&dsi->ctx);
+
+	if (panel->esd_restarting)
+		spacemit_dpu_esd_restart(a_crtc);
+	a_crtc->is_stopped = false;
+}
+
+#if IS_ENABLED(CONFIG_DRM_SPACEMIT_BACKLIGHT)
+static int spacemit_mipi_panel_set_brightness(void *devdata, int value)
+{
+	struct spacemit_panel *panel = (struct spacemit_panel *)devdata;
+	int ret = 0;
+	int brightness = value;
+
+	if (!atomic_read(&panel->prepare_refcnt)) {
+		panel->info.brightness = brightness;
+		return 0;
+	}
+
+	if (panel && (brightness != panel->info.brightness)) {
+		panel->slave->mode_flags &= ~MIPI_DSI_MODE_LPM;
+		ret = mipi_dsi_dcs_set_display_brightness(panel->slave, (u16)brightness);
+		panel->slave->mode_flags |= MIPI_DSI_MODE_LPM;
+		panel->info.brightness = brightness;
+	}
+
+	return ret;
+}
+
+static int spacemit_mipi_panel_get_brightness(void *devdata)
+{
+	struct spacemit_panel *panel = (struct spacemit_panel *)devdata;
+	int value = 0;
+
+	if (panel)
+		value = panel->info.brightness;
+
+	return value;
+}
+
+static struct spacemit_bl_ops spacemit_panel_bl_ops = {
+	.set_bl = spacemit_mipi_panel_set_brightness,
+	.get_bl = spacemit_mipi_panel_get_brightness,
+};
+#endif
+
+static ssize_t mipi_dsi_device_transfer(struct mipi_dsi_device *dsi,
+					struct mipi_dsi_msg *msg)
+{
+	const struct mipi_dsi_host_ops *ops = dsi->host->ops;
+
+	if (!ops || !ops->transfer)
+		return -ENOSYS;
+
+	if (dsi->mode_flags & MIPI_DSI_MODE_LPM)
+		msg->flags |= MIPI_DSI_MSG_USE_LPM;
+	//msg->flags |= MIPI_DSI_MSG_LASTCOMMAND;
+
+	return ops->transfer(dsi->host, msg);
+}
+
+static ssize_t mipi_dsi_spacemit_write(struct mipi_dsi_device *dsi, const void *payload,
+			       uint8_t cmd_type, size_t size)
+{
+	struct mipi_dsi_msg msg = {
+		.channel = dsi->channel,
+		.type = cmd_type,
+		.tx_buf = payload,
+		.tx_len = size
+	};
+
+	return mipi_dsi_device_transfer(dsi, &msg);
+}
+
+static void spacemit_mipi_wq_reset_panel(struct work_struct *work)
+{
+	struct spacemit_panel *panel = container_of(work, struct spacemit_panel, work_reset_panel);
+	struct drm_encoder *encoder = NULL;
+	struct spacemit_dsi *dsi = NULL;
+
+	if (panel->gpio_te_irq) {
+		enable_irq(panel->gpio_te_irq);
+		usleep_range(16000, 18000);
+		disable_irq(panel->gpio_te_irq);
+
+		if (panel->gpio_te_cnt) {
+			panel->gpio_te_cnt = 0;
+			return;
+		}
+	}
+
+	DRM_INFO("%s()\n", __func__);
+	encoder = panel->encoder;
+	if (encoder == NULL)
+		return;
+
+	dsi = encoder_to_dsi(encoder);
+
+	mutex_lock(&dsi->disable_lock);
+	/* hwc send a diable commit */
+	if (dsi->is_disabled == true) {
+		DRM_INFO("esd abort since lcd disabled\n");
+		mutex_unlock(&dsi->disable_lock);
+		return;
+	}
+
+	DRM_INFO("====== esd recovery start ========\n");
+	panel->esd_restarting = 1;
+	_spacemit_dsi_encoder_disable(encoder);
+	msleep(10);
+	_spacemit_dsi_encoder_enable(encoder);
+	panel->esd_restarting = 0;
+	DRM_INFO("======= esd recovery end =========\n");
+	mutex_unlock(&dsi->disable_lock);
+}
+
+static void spacemit_mipi_te_esd_timer_handler(struct timer_list *t)
+{
+	struct spacemit_panel *panel = timer_container_of(panel, t, te_esd_timer);
+
+	mod_timer(&panel->te_esd_timer,
+		 jiffies + msecs_to_jiffies(2000));
+	queue_work(system_wq, &panel->work_reset_panel);
+}
+
+static int __maybe_unused spacemit_panel_send_cmds(struct mipi_dsi_device *dsi,
+				const void *data, int size)
+{
+	struct spacemit_panel *panel;
+	struct spacemit_dsi_cmd_desc *cmds = NULL;
+	u16 len;
+	int data_off = 0;
+	int i = 0;
+	unsigned char *tmp = NULL;
+
+	if (dsi == NULL)
+		return -EINVAL;
+
+	panel = mipi_dsi_get_drvdata(dsi);
+	cmds  = devm_kzalloc(&dsi->dev, sizeof(struct spacemit_dsi_cmd_desc), GFP_KERNEL);
+	if (!cmds)
+		return -ENOMEM;
+
+	while (size > 0) {
+		cmds->cmd_type = *(unsigned char *)(data + data_off++);
+		cmds->lp = *(unsigned char *)(data + data_off++);
+		cmds->delay = *(unsigned char *)(data + data_off++);
+		cmds->length = *(unsigned char *)(data + data_off++);
+		for (i = 0; i < cmds->length; i++) {
+			tmp = (unsigned char *)data + data_off++;
+			cmds->data[i] = *(unsigned char *)tmp;
+		}
+
+		len = cmds->length;
+
+		if (cmds->lp)
+			dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+		else
+			dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+		if (panel->info.use_dcs)
+			mipi_dsi_dcs_write_buffer(dsi, cmds->data, len);
+		else if (panel->info.use_spacemit)
+			mipi_dsi_spacemit_write(dsi, cmds->data, cmds->cmd_type, len);
+		else
+			mipi_dsi_generic_write(dsi, cmds->data, len);
+
+		if (cmds->delay)
+			msleep(cmds->delay);
+		size -= (len + 4);
+	}
+
+	devm_kfree(&dsi->dev, cmds);
+
+	return 0;
+}
+
+static int spacemit_panel_unprepare(struct drm_panel *p)
+{
+	struct spacemit_panel *panel = to_spacemit_panel(p);
+	struct spacemit_drm_notifier noti_blank;
+
+	/* do nothing before spacemit_panel_prepare been called */
+	int blank = DRM_PANEL_BLANK_POWERDOWN;
+
+	noti_blank.blank = blank;
+
+	if (!atomic_read(&panel->prepare_refcnt))
+		return 0;
+
+	DRM_INFO("mipi: POWERDOWN!!\n");
+	spacemit_drm_notifier_call_chain(DRM_PANEL_EARLY_EVENT_BLANK, &noti_blank);
+
+	DRM_INFO("%s()\n", __func__);
+	gpio_direction_output(panel->gpio_reset, panel->info.reset_on_state);
+
+	if (panel->gpio_bl != INVALID_GPIO)
+		gpio_direction_output(panel->gpio_bl, 0);
+	msleep(150);
+
+	if (panel->gpio_dc[0] != INVALID_GPIO &&
+		panel->gpio_dc[1] != INVALID_GPIO) {
+		gpio_direction_output(panel->gpio_dc[0], 0);
+		gpio_direction_output(panel->gpio_dc[1], 0);
+	}
+
+	if (panel->vdd_1v2)
+		regulator_disable(panel->vdd_1v2);
+	if (panel->vdd_1v8)
+		regulator_disable(panel->vdd_1v8);
+
+	if (panel->vdd_2v8)
+		regulator_disable(panel->vdd_2v8);
+
+	atomic_set(&panel->prepare_refcnt, 0);
+	return 0;
+}
+
+static void spacemit_prepare_regulator(struct spacemit_panel *panel)
+{
+	int ret = 0;
+
+	if (panel->vdd_2v8 != NULL) {
+		ret = regulator_enable(panel->vdd_2v8);
+		if (ret)
+			DRM_ERROR("enable lcd regulator vdd_2v8 failed\n");
+	}
+
+	if (panel->vdd_1v8 != NULL) {
+		ret = regulator_enable(panel->vdd_1v8);
+		if (ret)
+			DRM_ERROR("enable lcd regulator vdd_1v8 failed\n");
+	}
+
+	if (panel->vdd_1v2 != NULL) {
+		ret = regulator_enable(panel->vdd_1v2);
+		if (ret)
+			DRM_ERROR("enable lcd regulator vdd_1v2 failed\n");
+	}
+}
+
+static int spacemit_panel_prepare(struct drm_panel *p)
+{
+	struct spacemit_panel *panel = to_spacemit_panel(p);
+	struct spacemit_drm_notifier noti_blank;
+	int i = 0;
+
+	int blank_ = DRM_PANEL_BLANK_UNBLANK;
+
+	/* prevent this function been called twice */
+	if (atomic_read(&panel->prepare_refcnt))
+		return 0;
+
+	DRM_INFO("%s()\n", __func__);
+
+	spacemit_prepare_regulator(panel);
+
+	if (panel->gpio_dc[0] != INVALID_GPIO &&
+		panel->gpio_dc[1] != INVALID_GPIO) {
+		gpio_direction_output(panel->gpio_dc[0], 1);
+		gpio_direction_output(panel->gpio_dc[1], 1);
+	}
+
+	if (panel->gpio_bl != INVALID_GPIO)
+		gpio_direction_output(panel->gpio_bl, 1);
+
+	noti_blank.blank = blank_;
+
+	spacemit_drm_notifier_call_chain(DRM_PANEL_EVENT_BLANK, &noti_blank);
+	gpio_direction_output(panel->gpio_reset, 1);
+	for (; i < panel->reset_toggle_cnt; i++) {
+		msleep(10);
+		gpio_direction_output(panel->gpio_reset, 0);
+		msleep(10);
+		gpio_direction_output(panel->gpio_reset, 1);
+	}
+	msleep(panel->delay_after_reset);
+	spacemit_drm_notifier_call_chain(DRM_PANEL_TOUCH_INT, &noti_blank);
+
+	DRM_INFO("mipi: UNBLANK!!\n");
+
+	/* update refcnt */
+	atomic_set(&panel->prepare_refcnt, 1);
+	return 0;
+}
+
+static int spacemit_panel_disable(struct drm_panel *p)
+{
+	struct spacemit_panel *panel = to_spacemit_panel(p);
+
+	if (!atomic_read(&panel->enable_refcnt))
+		return 0;
+
+	DRM_INFO("%s()\n", __func__);
+
+	if (panel->esd_work_pending) {
+		cancel_delayed_work_sync(&panel->esd_work);
+		panel->esd_work_pending = false;
+	}
+
+	cancel_delayed_work_sync(&panel->bl_work);
+
+	if (panel->gpio_te_irq)
+		timer_delete(&panel->te_esd_timer);
+
+	spacemit_panel_send_cmds(panel->slave,
+			panel->info.cmds[CMD_CODE_SLEEP_IN],
+			panel->info.cmds_len[CMD_CODE_SLEEP_IN]);
+
+	atomic_set(&panel->enable_refcnt, 0);
+	return 0;
+}
+
+static int spacemit_panel_enable(struct drm_panel *p)
+{
+	struct spacemit_panel *panel = to_spacemit_panel(p);
+
+	if (atomic_read(&panel->enable_refcnt))
+		return 0;
+
+	DRM_INFO("%s()\n", __func__);
+
+	if (panel->gpio_te_irq)
+		mod_timer(&panel->te_esd_timer,
+			jiffies + msecs_to_jiffies(2000));
+
+	spacemit_panel_send_cmds(panel->slave,
+			panel->info.cmds[CMD_CODE_INIT],
+			panel->info.cmds_len[CMD_CODE_INIT]);
+
+	if (panel->info.cmds[CMD_CODE_DSC_PPS]) {
+		spacemit_panel_send_cmds(panel->slave,
+				panel->info.cmds[CMD_CODE_DSC_PPS],
+				panel->info.cmds_len[CMD_CODE_DSC_PPS]);
+	}
+
+	if (panel->info.cmds[CMD_CODE_COMPRESS_ON]) {
+		spacemit_panel_send_cmds(panel->slave,
+				panel->info.cmds[CMD_CODE_COMPRESS_ON],
+				panel->info.cmds_len[CMD_CODE_COMPRESS_ON]);
+	}
+
+	if (panel->info.esd_check_en) {
+		schedule_delayed_work(&panel->esd_work,
+				      msecs_to_jiffies(1000));
+		panel->esd_work_pending = true;
+	}
+
+	if (panel->info.bl_enable_delay) {
+		schedule_delayed_work(&panel->bl_work,
+			msecs_to_jiffies(panel->info.bl_enable_delay));
+	}
+	atomic_set(&panel->enable_refcnt, 1);
+	return 0;
+}
+
+static int spacemit_panel_get_modes(struct drm_panel *p, struct drm_connector *connector)
+{
+	struct drm_display_mode *mode;
+	struct spacemit_panel *panel = to_spacemit_panel(p);
+	int index = 0;
+	int cnt = 0;
+
+	DRM_INFO("%s()\n", __func__);
+
+	for (; index < panel->info.mode_num; index++) {
+		mode = drm_mode_duplicate(connector->dev, &panel->info.mode[index]);
+		if (!mode) {
+			DRM_ERROR("failed to add mode %ux%ux@%u\n",
+					panel->info.mode[index].hdisplay,
+					panel->info.mode[index].vdisplay,
+					drm_mode_vrefresh(mode));
+			return -ENOMEM;
+		}
+
+		drm_mode_set_name(mode);
+		// if (index)
+		// 	snprintf(mode->name, DRM_DISPLAY_MODE_LEN, "%s_%d", mode->name, index);
+
+		if (drm_mode_vrefresh(mode) == 60)
+			mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+		else
+			mode->type = DRM_MODE_TYPE_DRIVER;
+		drm_mode_probed_add(connector, mode);
+		cnt++;
+	}
+
+	connector->display_info.width_mm = panel->info.mode[0].width_mm;
+	connector->display_info.height_mm = panel->info.mode[0].height_mm;
+
+
+	return cnt;
+}
+
+static const struct drm_panel_funcs spacemit_panel_funcs = {
+	.get_modes = spacemit_panel_get_modes,
+	.enable = spacemit_panel_enable,
+	.disable = spacemit_panel_disable,
+	.prepare = spacemit_panel_prepare,
+	.unprepare = spacemit_panel_unprepare,
+};
+
+static int spacemit_mipi_dsi_set_maximum_return_packet_size(struct mipi_dsi_device *dsi,
+					    u16 value)
+{
+	u8 tx[2] = { value & 0xff, value >> 8 };
+	struct mipi_dsi_msg msg = {
+		.channel = dsi->channel,
+		.type = MIPI_DSI_SET_MAXIMUM_RETURN_PACKET_SIZE,
+		.tx_len = sizeof(tx),
+		.tx_buf = tx,
+	};
+	int ret = mipi_dsi_device_transfer(dsi, &msg);
+
+	return (ret < 0) ? ret : 0;
+}
+
+static int spacemit_panel_esd_check(struct spacemit_panel *panel)
+{
+	struct panel_info *info = &panel->info;
+	u8 read_val = 0;
+
+	spacemit_mipi_dsi_set_maximum_return_packet_size(panel->slave, 1);
+	mipi_dsi_dcs_read(panel->slave, info->esd_check_reg,
+			  &read_val, 1);
+
+	if (read_val != info->esd_check_val) {
+		DRM_ERROR("esd check failed, read value = 0x%02x\n",
+			  read_val);
+		return -EINVAL;
+	}
+	DRM_INFO("esd check, read value = 0x%02x\n", read_val);
+
+	return 0;
+}
+
+static void spacemit_panel_enable_backlight_work_func(struct work_struct *work)
+{
+	struct spacemit_panel *panel = container_of(work, struct spacemit_panel,
+						bl_work.work);
+	backlight_enable(panel->base.backlight);
+}
+
+static void spacemit_panel_esd_work_func(struct work_struct *work)
+{
+	struct spacemit_panel *panel = container_of(work, struct spacemit_panel,
+						esd_work.work);
+	struct panel_info *info = &panel->info;
+	int ret;
+
+	ret = spacemit_panel_esd_check(panel);
+	if (ret) {
+		/*
+		const struct drm_encoder_helper_funcs *funcs;
+		struct drm_encoder *encoder;
+
+		encoder = panel->base.connector->encoder;
+		funcs = encoder->helper_private;
+		panel->esd_work_pending = false;
+
+		DRM_INFO("====== esd recovery start ========\n");
+		funcs->disable(encoder);
+		funcs->enable(encoder);
+		DRM_INFO("======= esd recovery end =========\n");
+		*/
+	} else
+		schedule_delayed_work(&panel->esd_work,
+			msecs_to_jiffies(info->esd_check_period));
+}
+
+static int spacemit_panel_parse_dt(struct device_node *np, struct spacemit_panel *panel)
+{
+	u32 val;
+	struct device_node *lcd_node;
+	struct panel_info *info = &panel->info;
+	int bytes, rc;
+	const void *p;
+	const char *str;
+	char lcd_path[60];
+	int index = 0;
+
+	rc = of_property_read_string(np, "force-attached", &str);
+	if (!rc)
+		lcd_name = str;
+
+	sprintf(lcd_path, "/lcds/%s", lcd_name);
+	lcd_node = of_find_node_by_path(lcd_path);
+	if (!lcd_node) {
+		DRM_ERROR("%pOF: could not find %s node\n", np, lcd_name);
+		return -ENODEV;
+	}
+	info->of_node = lcd_node;
+
+	rc = of_property_read_u32(lcd_node, "dsi-work-mode", &val);
+	if (!rc) {
+		if (val == DSI_MODE_CMD)
+			info->mode_flags = 0;
+		else if (val == DSI_MODE_VIDEO_BURST)
+			info->mode_flags = MIPI_DSI_MODE_VIDEO |
+					   MIPI_DSI_MODE_VIDEO_BURST;
+		else if (val == DSI_MODE_VIDEO_SYNC_PULSE)
+			info->mode_flags = MIPI_DSI_MODE_VIDEO |
+					   MIPI_DSI_MODE_VIDEO_SYNC_PULSE;
+		else if (val == DSI_MODE_VIDEO_SYNC_EVENT)
+			info->mode_flags = MIPI_DSI_MODE_VIDEO;
+	} else {
+		DRM_ERROR("dsi work mode is not found! use video mode\n");
+		info->mode_flags = MIPI_DSI_MODE_VIDEO |
+				   MIPI_DSI_MODE_VIDEO_BURST;
+	}
+
+	if (of_property_read_bool(lcd_node, "dsi-non-continuous-clock"))
+		info->mode_flags |= MIPI_DSI_CLOCK_NON_CONTINUOUS;
+
+	rc = of_property_read_u32(lcd_node, "dsi-lane-number", &val);
+	if (!rc)
+		info->lanes = val;
+	else
+		info->lanes = 4;
+
+	rc = of_property_read_string(lcd_node, "dsi-color-format", &str);
+	if (rc)
+		info->format = MIPI_DSI_FMT_RGB888;
+	else if (!strcmp(str, "rgb888"))
+		info->format = MIPI_DSI_FMT_RGB888;
+	else if (!strcmp(str, "rgb666"))
+		info->format = MIPI_DSI_FMT_RGB666;
+	else if (!strcmp(str, "rgb666_packed"))
+		info->format = MIPI_DSI_FMT_RGB666_PACKED;
+	else if (!strcmp(str, "rgb565"))
+		info->format = MIPI_DSI_FMT_RGB565;
+	else
+		DRM_ERROR("dsi-color-format (%s) is not supported\n", str);
+
+	rc = of_property_read_u32(lcd_node, "width-mm", &val);
+	if (!rc)
+		info->mode[0].width_mm = val;
+	else
+		info->mode[0].width_mm = 68;
+
+	rc = of_property_read_u32(lcd_node, "height-mm", &val);
+	if (!rc)
+		info->mode[0].height_mm = val;
+	else
+		info->mode[0].height_mm = 121;
+
+	rc = of_property_read_u32(lcd_node, "esd-check-enable", &val);
+	if (!rc)
+		info->esd_check_en = val;
+
+	rc = of_property_read_u32(lcd_node, "esd-check-mode", &val);
+	if (!rc)
+		info->esd_check_mode = val;
+	else
+		info->esd_check_mode = 1;
+
+	rc = of_property_read_u32(lcd_node, "esd-check-period", &val);
+	if (!rc)
+		info->esd_check_period = val;
+	else
+		info->esd_check_period = 1000;
+
+	rc = of_property_read_u32(lcd_node, "esd-check-register", &val);
+	if (!rc)
+		info->esd_check_reg = val;
+	else
+		info->esd_check_reg = 0x0A;
+
+	rc = of_property_read_u32(lcd_node, "esd-check-value", &val);
+	if (!rc)
+		info->esd_check_val = val;
+	else
+		info->esd_check_val = 0x9C;
+
+
+	rc = of_property_read_u32(lcd_node, "bl-enable-delay", &val);
+	if (!rc)
+		info->bl_enable_delay = val;
+	else
+		info->bl_enable_delay = 0;
+
+	if (of_property_read_bool(lcd_node, "use-dcs-write"))
+		info->use_dcs = true;
+	else
+		info->use_dcs = false;
+
+	if (of_property_read_bool(lcd_node, "use-spacemit-write"))
+		info->use_spacemit = true;
+	else
+		info->use_spacemit = false;
+
+	if (of_property_read_bool(lcd_node, "oled"))
+		info->is_oled = true;
+	else
+		info->is_oled = false;
+
+	if (info->is_oled) {
+		rc = of_property_read_u32(lcd_node, "brightness", &val);
+		if (!rc)
+			info->brightness = val;
+		else
+			info->brightness = 255;
+		rc = of_property_read_u32(lcd_node, "brightness-max", &val);
+		if (!rc)
+			info->brightness_max = val;
+		else
+			info->brightness_max = 255;
+	}
+
+	p = of_get_property(lcd_node, "read-id-command", &bytes);
+	if (p) {
+		info->cmds[CMD_CODE_READ_ID] = p;
+		info->cmds_len[CMD_CODE_READ_ID] = bytes;
+	} else
+		DRM_ERROR("can't find read-id property\n");
+
+	p = of_get_property(lcd_node, "initial-command", &bytes);
+	if (p) {
+		info->cmds[CMD_CODE_INIT] = p;
+		info->cmds_len[CMD_CODE_INIT] = bytes;
+	} else
+		DRM_ERROR("can't find initial-command property\n");
+
+	p = of_get_property(lcd_node, "sleep-in-command", &bytes);
+	if (p) {
+		info->cmds[CMD_CODE_SLEEP_IN] = p;
+		info->cmds_len[CMD_CODE_SLEEP_IN] = bytes;
+	} else
+		DRM_ERROR("can't find sleep-in-command property\n");
+
+	p = of_get_property(lcd_node, "sleep-out-command", &bytes);
+	if (p) {
+		info->cmds[CMD_CODE_SLEEP_OUT] = p;
+		info->cmds_len[CMD_CODE_SLEEP_OUT] = bytes;
+	} else
+		DRM_ERROR("can't find sleep-out-command property\n");
+
+	rc = of_property_read_u32(lcd_node, "dsc-enable", &val);
+	if (rc)
+		val = 0;
+
+	info->cmds[CMD_CODE_DSC_PPS] = NULL;
+	info->cmds_len[CMD_CODE_DSC_PPS] = 0;
+	info->cmds[CMD_CODE_COMPRESS_ON] = NULL;
+	info->cmds_len[CMD_CODE_COMPRESS_ON] = 0;
+	if (val) {
+		p = of_get_property(lcd_node, "dsc-pps-command", &bytes);
+		if (p) {
+			info->cmds[CMD_CODE_DSC_PPS] = p;
+			info->cmds_len[CMD_CODE_DSC_PPS] = bytes;
+		} else {
+			DRM_ERROR("can't find dsc-pps-command property\n");
+		}
+
+		p = of_get_property(lcd_node, "compress-on-command", &bytes);
+		if (p) {
+			info->cmds[CMD_CODE_COMPRESS_ON] = p;
+			info->cmds_len[CMD_CODE_COMPRESS_ON] = bytes;
+		} else {
+			DRM_ERROR("can't find compress-on-command property\n");
+		}
+	}
+
+	for (; index < LCD_MODE_MAX; index++) {
+		rc = of_get_drm_display_mode(lcd_node, &info->mode[index], 0, index);
+		if (rc) {
+			DRM_DEBUG("get display timing done\n");
+			info->mode_num = index--;
+			break;
+		}
+	}
+
+	rc = of_property_read_u32(lcd_node, "te-esd-en", &val);
+	if (!rc)
+		info->te_esd_en = val;
+	else
+		info->te_esd_en = 0;
+
+	rc = of_property_read_u32(lcd_node, "reset-on-state", &val);
+	if (!rc)
+		info->reset_on_state = val;
+	else
+		info->reset_on_state = 1; /* defalt reset on state is 1 */
+
+	return 0;
+}
+
+static int spacemit_panel_device_create(struct device *parent,
+				    struct spacemit_panel *panel)
+{
+	panel->dev.class = display_class;
+	panel->dev.parent = parent;
+	panel->dev.of_node = panel->info.of_node;
+	dev_set_name(&panel->dev, "panel%d", panel->id);
+	dev_set_drvdata(&panel->dev, panel);
+
+	return device_register(&panel->dev);
+}
+
+static irqreturn_t spacemit_mipi_gpio_te_irq_handler(int irq, void *dev_id)
+{
+	struct spacemit_panel *panel = (struct spacemit_panel *)dev_id;
+
+	panel->gpio_te_cnt++;
+
+	return IRQ_HANDLED;
+
+}
+
+/* based on of_node_put */
+static void spacemit_of_node_put(struct device_node *node)
+{
+	if (node)
+		kobject_put(&node->kobj);
+}
+
+/* based on of_find_backlight */
+static struct backlight_device *spacemit_of_find_backlight(struct device *dev)
+{
+	struct backlight_device *bd = NULL;
+	struct device_node *np;
+
+	if (!dev)
+		return NULL;
+
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		np = of_parse_phandle(dev->of_node, "backlight", 0);
+		if (np) {
+			bd = of_find_backlight_by_node(np);
+			spacemit_of_node_put(np);
+			if (!bd)
+				return ERR_PTR(-EPROBE_DEFER);
+		}
+	}
+
+	return bd;
+}
+
+/* based on __devm_add_action_or_reset */
+static inline int spacemit__devm_add_action_or_reset(struct device *dev, void (*action)(void *),
+				  void *data, const char *name)
+{
+	int ret;
+
+	ret = __devm_add_action(dev, action, data, name);
+	if (ret)
+		action(data);
+
+	return ret;
+}
+#define spacemit_devm_add_action_or_reset(release, action, data) \
+	spacemit__devm_add_action_or_reset(release, action, data, #action)
+
+/* based on devm_backlight_release */
+static void spacemit_devm_backlight_release(void *data)
+{
+	struct backlight_device *bd = data;
+
+	put_device(&bd->dev);
+}
+
+
+/* based on devm_of_find_backlight */
+static struct backlight_device *spacemit_devm_of_find_backlight(struct device *dev)
+{
+	struct backlight_device *bd;
+	int ret;
+
+	bd = spacemit_of_find_backlight(dev);
+	if (IS_ERR_OR_NULL(bd))
+		return bd;
+	ret = spacemit_devm_add_action_or_reset(dev, spacemit_devm_backlight_release, bd);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return bd;
+}
+
+/* based on drm_panel_of_backlight */
+static int spacemit_drm_panel_of_backlight(struct drm_panel *panel, struct device *dev)
+{
+	struct backlight_device *backlight;
+
+	if (!panel || !panel->dev)
+		return -EINVAL;
+
+	backlight = spacemit_devm_of_find_backlight(dev);
+
+	if (IS_ERR(backlight))
+		return PTR_ERR(backlight);
+	panel->backlight = backlight;
+	return 0;
+}
+
+static int spacemit_panel_probe(struct mipi_dsi_device *slave)
+{
+	int ret;
+	struct spacemit_panel *panel;
+	struct device *dev = &slave->dev;
+	u32 tmp;
+	struct gpio_desc *gpiod;
+
+	panel = devm_kzalloc(&slave->dev, sizeof(*panel), GFP_KERNEL);
+	if (!panel)
+		return -ENOMEM;
+
+	if (!of_property_read_u32(dev->of_node, "id", &tmp))
+		panel->id = tmp;
+
+	panel->vdd_2v8 = devm_regulator_get(&slave->dev, "vdd_2v8");
+	if (IS_ERR(panel->vdd_2v8)) {
+		DRM_DEBUG("get lcd regulator vdd_2v8 failed\n");
+		panel->vdd_2v8 = NULL;
+	} else {
+		regulator_set_voltage(panel->vdd_2v8, 2800000, 2800000);
+		ret = regulator_enable(panel->vdd_2v8);
+		if (ret)
+			DRM_ERROR("enable lcd regulator vdd_2v8 failed\n");
+	}
+
+	panel->vdd_1v8 = devm_regulator_get(&slave->dev, "vdd_1v8");
+	if (IS_ERR(panel->vdd_1v8)) {
+		DRM_DEBUG("get lcd regulator vdd_1v8 failed\n");
+		panel->vdd_1v8 = NULL;
+	} else {
+		regulator_set_voltage(panel->vdd_1v8, 1800000, 1800000);
+		ret = regulator_enable(panel->vdd_1v8);
+		if (ret)
+			DRM_ERROR("enable lcd regulator vdd_1v8 failed\n");
+	}
+
+	panel->vdd_1v2 = devm_regulator_get(&slave->dev, "vdd_1v2");
+	if (IS_ERR(panel->vdd_1v2)) {
+		DRM_DEBUG("get regulator vdd_1v2 failed\n");
+		panel->vdd_1v2 = NULL;
+	} else {
+		regulator_set_voltage(panel->vdd_1v2, 1200000, 1200000);
+		ret = regulator_enable(panel->vdd_1v2);
+		if (ret)
+			DRM_ERROR("enable lcd regulator vdd_1v2 failed\n");
+	}
+
+	gpiod = devm_gpiod_get(dev, "reset", GPIOD_ASIS);
+	if (IS_ERR(gpiod)) {
+		ret = PTR_ERR(gpiod);
+		if (ret != -ENOENT) {
+			dev_err(dev, "Failed to get reset-gpios: %d\n", ret);
+			return ret;
+		}
+		dev_warn(dev, "reset-gpios not specified\n");
+		panel->gpio_reset = INVALID_GPIO;
+	} else {
+		panel->gpio_reset = desc_to_gpio(gpiod);
+	}
+
+	gpiod = devm_gpiod_get(dev, "bl", GPIOD_ASIS);
+	if (IS_ERR(gpiod)) {
+		ret = PTR_ERR(gpiod);
+		if (ret != -ENOENT) {
+			dev_err(dev, "Failed to get bl-gpios: %d\n", ret);
+			return ret;
+		}
+		panel->gpio_bl = INVALID_GPIO;
+	} else {
+		panel->gpio_bl = desc_to_gpio(gpiod);
+	}
+
+	gpiod = devm_gpiod_get(dev, "dc0", GPIOD_ASIS);
+	if (IS_ERR(gpiod)) {
+		ret = PTR_ERR(gpiod);
+		if (ret != -ENOENT) {
+			dev_err(dev, "Failed to get dc0-gpios: %d\n", ret);
+			return ret;
+		}
+		panel->gpio_dc[0] = INVALID_GPIO;
+	} else {
+		panel->gpio_dc[0] = desc_to_gpio(gpiod);
+	}
+
+	gpiod = devm_gpiod_get(dev, "dc1", GPIOD_ASIS);
+	if (IS_ERR(gpiod)) {
+		ret = PTR_ERR(gpiod);
+		if (ret != -ENOENT) {
+			dev_err(dev, "Failed to get dc1-gpios: %d\n", ret);
+			return ret;
+		}
+		panel->gpio_dc[1] = INVALID_GPIO;
+	} else {
+		panel->gpio_dc[1] = desc_to_gpio(gpiod);
+	}
+
+	if (of_property_read_u32(dev->of_node, "reset-toggle-cnt", &panel->reset_toggle_cnt))
+		panel->reset_toggle_cnt = LCD_PANEL_RESET_CNT;
+
+	if (of_property_read_u32(dev->of_node, "delay-after-reset", &panel->delay_after_reset))
+		panel->delay_after_reset = LCD_DELAY_AFTER_RESET;
+
+	ret = spacemit_panel_parse_dt(slave->dev.of_node, panel);
+	if (ret) {
+		DRM_ERROR("parse panel info failed\n");
+		return ret;
+	}
+
+	if (panel->info.te_esd_en) {
+		ret = of_property_read_u32(dev->of_node, "gpios-te", &panel->gpio_te);
+		if (ret || !gpio_is_valid(panel->gpio_te)) {
+			dev_info(dev, "Not support gpios-te\n");
+		} else {
+			ret = gpio_request(panel->gpio_te, NULL);
+			if (ret) {
+				pr_err("gpio_te request fail\n");
+				return ret;
+			}
+			panel->gpio_te_irq = gpio_to_irq(panel->gpio_te);
+			if (panel->gpio_te_irq) {
+				timer_setup(&panel->te_esd_timer, spacemit_mipi_te_esd_timer_handler, 0);
+				INIT_WORK(&panel->work_reset_panel, spacemit_mipi_wq_reset_panel);
+				ret = devm_request_threaded_irq(dev, panel->gpio_te_irq, NULL,
+								spacemit_mipi_gpio_te_irq_handler,
+								IRQF_TRIGGER_RISING|IRQF_ONESHOT,
+								"lcd_panel_te", panel);
+				if (ret != 0) {
+					dev_err(dev, "Failed to request IRQ: %d\n",
+							panel->gpio_te_irq);
+					return ret;
+				}
+				disable_irq(panel->gpio_te_irq);
+			}
+		}
+	}
+
+	ret = spacemit_panel_device_create(&slave->dev, panel);
+	if (ret) {
+		DRM_ERROR("panel device create failed\n");
+		return ret;
+	}
+#if IS_ENABLED(CONFIG_DRM_SPACEMIT_BACKLIGHT)
+	if (panel->info.is_oled) {
+		ret = spacemit_bl_device_register(&slave->dev,
+			panel, &spacemit_panel_bl_ops, panel->info.brightness_max);
+		if (ret) {
+			DRM_ERROR("panel backlight device register failed\n");
+			return ret;
+		}
+	}
+#endif
+	panel->base.dev = &panel->dev;
+	panel->base.funcs = &spacemit_panel_funcs;
+	drm_panel_init(&panel->base, &panel->dev, &spacemit_panel_funcs, DRM_MODE_CONNECTOR_DSI);
+
+	ret = drm_panel_of_backlight(&panel->base);
+	if (ret || panel->base.backlight == NULL) {
+		ret = spacemit_drm_panel_of_backlight(&panel->base, &slave->dev);
+		if (ret || panel->base.backlight == NULL) {
+			DRM_INFO("panel device can not get backlight\n");
+		}
+	}
+
+	drm_panel_add(&panel->base);
+
+	backlight_enable(panel->base.backlight);
+
+	slave->lanes = panel->info.lanes;
+	slave->format = panel->info.format;
+	slave->mode_flags = panel->info.mode_flags;
+
+	panel->slave = slave;
+
+	spacemit_mipi_panel_sysfs_init(&panel->dev);
+	mipi_dsi_set_drvdata(slave, panel);
+
+	/*do esd init work*/
+	if (panel->info.esd_check_en) {
+		INIT_DELAYED_WORK(&panel->esd_work, spacemit_panel_esd_work_func);
+		/*
+		 * schedule_delayed_work(&panel->esd_work,
+				      msecs_to_jiffies(2000));
+		panel->esd_work_pending = true;
+		*/
+	}
+
+	INIT_DELAYED_WORK(&panel->bl_work, spacemit_panel_enable_backlight_work_func);
+
+	atomic_set(&panel->enable_refcnt, 0);
+	atomic_set(&panel->prepare_refcnt, 0);
+
+	ret = mipi_dsi_attach(slave);
+	if (ret) {
+		DRM_ERROR("failed to attach dsi panel to host\n");
+		drm_panel_remove(&panel->base);
+		return ret;
+	}
+	DRM_INFO("panel driver probe success\n");
+
+	return 0;
+}
+
+static void spacemit_panel_remove(struct mipi_dsi_device *slave)
+{
+	struct spacemit_panel *panel = NULL;
+	int ret;
+
+	DRM_INFO("%s()\n", __func__);
+
+	if (!slave) {
+		DRM_ERROR("%s fail\n", __func__);
+		return;
+	}
+
+	panel = mipi_dsi_get_drvdata(slave);
+	if (!panel) {
+		DRM_ERROR("null panel\n");
+		return;
+	}
+
+	backlight_disable(panel->base.backlight);
+	spacemit_panel_disable(&panel->base);
+	spacemit_panel_unprepare(&panel->base);
+
+	ret = mipi_dsi_detach(slave);
+	if (ret < 0)
+		DRM_ERROR("failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&panel->base);
+
+}
+
+static const struct of_device_id panel_of_match[] = {
+	{ .compatible = "spacemit,mipi-panel", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, panel_of_match);
+
+static struct mipi_dsi_driver spacemit_panel_driver = {
+	.driver = {
+		.name = "spacemit-mipi-panel-drv",
+		.of_match_table = panel_of_match,
+	},
+	.probe = spacemit_panel_probe,
+	.remove = spacemit_panel_remove,
+};
+module_mipi_dsi_driver(spacemit_panel_driver);
+
+MODULE_DESCRIPTION("Spacemit MIPI Panel Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/spacemit_mipi_panel.h b/drivers/gpu/drm/spacemit/spacemit_mipi_panel.h
new file mode 100644
index 000000000000..b298f27932f1
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_mipi_panel.h
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_MIPI_PANEL_H_
+#define _SPACEMIT_MIPI_PANEL_H_
+
+#include <drm/drm_print.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <linux/backlight.h>
+#include <linux/notifier.h>
+#include <linux/of.h>
+#include <linux/atomic.h>
+#include <linux/regulator/consumer.h>
+#include <linux/workqueue.h>
+
+#define INVALID_GPIO 0xFFFFFFFF
+
+#define LCD_PANEL_RESET_CNT 4
+#define LCD_DELAY_AFTER_RESET 50
+#define LCD_MODE_MAX	5
+
+enum {
+	CMD_CODE_INIT = 0,
+	CMD_CODE_SLEEP_IN,
+	CMD_CODE_SLEEP_OUT,
+	CMD_CODE_READ_ID,
+	CMD_CODE_READ_POWER,
+	CMD_CODE_DSC_PPS,
+	CMD_CODE_COMPRESS_ON,
+#if IS_ENABLED(CONFIG_TOUCHSCREEN_OMNIVISION_TCM_FACE_DETECT) || IS_ENABLED(CONFIG_TOUCHSCREEN_SITRONIX_FACE_DETECT)
+	CMD_FACE_IN,
+	CMD_FACE_OUT,
+#endif
+	CMD_CODE_MAX,
+};
+
+enum {
+	DSI_MODE_CMD = 0,
+	DSI_MODE_VIDEO_BURST,
+	DSI_MODE_VIDEO_SYNC_PULSE,
+	DSI_MODE_VIDEO_SYNC_EVENT,
+};
+
+struct panel_info {
+	/* common parameters */
+	struct device_node *of_node;
+	struct drm_display_mode mode[LCD_MODE_MAX];
+	u8 mode_num;
+	const void *cmds[CMD_CODE_MAX];
+	int cmds_len[CMD_CODE_MAX];
+
+	/* esd check parameters*/
+	bool esd_check_en;
+	u8 esd_check_mode;
+	u16 esd_check_period;
+	u32 esd_check_reg;
+	u32 esd_check_val;
+
+	/* MIPI DSI specific parameters */
+	u32 format;
+	u32 lanes;
+	u32 mode_flags;
+	bool use_dcs;
+	bool use_spacemit;
+	u32 bl_enable_delay;
+
+	u32 te_esd_en;
+	u32 reset_on_state;
+
+	int brightness;
+	int brightness_max;
+	bool is_oled;
+};
+struct spacemit_panel {
+	int id;
+	struct device dev;
+	struct drm_panel base;
+	struct drm_encoder *encoder;
+	struct mipi_dsi_device *slave;
+	struct panel_info info;
+
+	struct delayed_work esd_work;
+	struct delayed_work bl_work;
+	bool esd_work_pending;
+
+	struct regulator *vdd_1v2;
+	struct regulator *vdd_1v8;
+	struct regulator *vdd_2v8;
+	u32 gpio_reset;
+	u32 gpio_bl;
+	u32 gpio_dc[2];
+	u32 gpio_te;
+	u32 gpio_te_irq;
+	u32 gpio_te_cnt;
+	u32 esd_restarting;
+	struct work_struct work_reset_panel;
+	struct timer_list te_esd_timer;
+	atomic_t enable_refcnt;
+	atomic_t prepare_refcnt;
+	u32 reset_toggle_cnt;
+	u32 delay_after_reset;
+#if IS_ENABLED(CONFIG_TOUCHSCREEN_OMNIVISION_TCM_FACE_DETECT) || IS_ENABLED(CONFIG_TOUCHSCREEN_SITRONIX_FACE_DETECT)
+	struct work_struct work_facereset_panel;
+	struct work_struct work_facerecover_panel;
+	bool tp_ps_enabled;
+	bool tp_ps_neared;
+	bool user_panel_disabled;
+	u32 tp_ps_restarting;
+	struct mutex face_lock;
+#endif
+};
+
+static inline struct spacemit_panel *to_spacemit_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct spacemit_panel, base);
+}
+#endif
diff --git a/drivers/gpu/drm/spacemit/spacemit_planes.c b/drivers/gpu/drm/spacemit/spacemit_planes.c
new file mode 100644
index 000000000000..a3f741e0891c
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_planes.c
@@ -0,0 +1,825 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_blend.h>
+#include <drm/drm_atomic_uapi.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_color_mgmt.h>
+#include <linux/component.h>
+#include <linux/dma-mapping.h>
+#include "spacemit_cmdlist.h"
+#include "spacemit_dmmu.h"
+#include "spacemit_drm.h"
+#include "spacemit_gem.h"
+#include "spacemit_lib.h"
+#include "dpu/dpu_saturn.h"
+#include "dpu/dpu_trace.h"
+
+static const uint64_t supported_format_modifiers[] = {
+	// Added for raw format
+	DRM_FORMAT_MOD_LINEAR,
+
+	// Added for AFBC(rgb*)
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPLIT | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPLIT | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_SC),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPLIT | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPLIT | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_SC),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_CBR | AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC),
+
+	/* All RGB formats (except XRGB, RGBX, XBGR, BGRX) */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPARSE),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_YTR),
+
+	/* All RGB formats > 16bpp (except XRGB, RGBX, XBGR, BGRX) */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_SPLIT),
+
+	/* All 8 or 10 bit YUV 444 formats. */
+	/* In DP550, 10 bit YUV 420 format also supported */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_SPLIT),
+
+	/* YUV 420, 422 P1 8 bit and YUV 444 8 bit/10 bit formats */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_CBR),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_TILED),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_SPARSE),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16),
+
+	/* YUV 420, 422 P1 8, 10 bit formats */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_CBR | AFBC_FORMAT_MOD_SPARSE),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | AFBC_FORMAT_MOD_CBR),
+
+	// For BG16/RG16 (3rd app will use this config, for example: games, nenamark etc ...)
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_32x8 | AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPARSE),
+
+	DRM_FORMAT_MOD_INVALID
+};
+
+struct spacemit_plane *to_spacemit_plane(struct drm_plane *plane)
+{
+	return container_of(plane, struct spacemit_plane, plane);
+}
+
+static int spacemit_plane_check_rdma(const struct spacemit_hw_rdma *rdma, u32 rdma_id, struct drm_plane_state *state)
+{
+	unsigned int rot = state->rotation;
+	unsigned int zpos = state->zpos;
+	u32 format = state->fb->format->format;
+	u16 hw_formats = rdma[rdma_id].formats;
+	u16 hw_rots = rdma[rdma_id].rots;
+	bool afbc = (state->fb->modifier > 0);
+	const struct drm_format_info *info = drm_format_info(format);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(state->crtc);
+
+	trace_spacemit_plane_check_rdma(a_crtc->dev_id);
+	if ((info->is_yuv) && !afbc && ((hw_formats & FORMAT_RAW_YUV) == 0)) {
+		DRM_DEBUG("rdma%d doesn't support RAW YUV format with zpos%d!\n", rdma_id, zpos);
+		return -EINVAL;
+	}
+
+	if (afbc && ((hw_formats & FORMAT_AFBC) == 0)) {
+		DRM_DEBUG("rdma%d doesn't support AFBC format with zpos%d!\n", rdma_id, zpos);
+		return -EINVAL;
+	}
+
+	if (is_rot_90_270(rot)) {
+		if (afbc) {
+			if ((hw_rots & ROTATE_AFBC_90_270) == 0) {
+				DRM_DEBUG("rdma%d doesn't support AFBC 90/270 rotation with zpos%d\n", rdma_id, zpos);
+				return -EINVAL;
+			}
+		} else {
+			if ((hw_rots & ROTATE_RAW_90_270) == 0) {
+				DRM_DEBUG("rdma%d doesn't support RAW 90/270 rotation with zpos%d!\n", rdma_id, zpos);
+				return -EINVAL;
+			}
+		}
+	} else {
+		if ((hw_rots & ROTATE_COMMON) == 0) {
+			DRM_DEBUG("rdma%d doesn't support common rotation with zpos%d!\n", rdma_id, zpos);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int spacemit_plane_atomic_check_hdr_coefs(struct drm_plane *plane,
+					  struct drm_plane_state *state)
+{
+	struct spacemit_plane_state *apstate = to_spacemit_plane_state(state);
+	struct drm_property_blob *blob;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	int size = hwdev->hdr_coef_size;
+	int *coef_data;
+	int n = 0;
+
+	if (apstate->hdr_coefs_blob_prop) {
+		blob = apstate->hdr_coefs_blob_prop;
+		coef_data = (int *)blob->data;
+
+		for (n = 0; n < size; n++) {
+			if ((coef_data[n] > 65535) || (coef_data[n] < 0)) {
+				DRM_ERROR("HDR coef is invalid %d\n", coef_data[n]);
+				return -EINVAL;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int spacemit_plane_atomic_check_scale_coefs(struct drm_plane *plane,
+					  struct drm_plane_state *state)
+{
+	struct spacemit_plane_state *apstate = to_spacemit_plane_state(state);
+	struct drm_property_blob *blob;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	int size = hwdev->hor_scale_coef_size + hwdev->ver_scale_coef_size;
+	int *coef_data;
+	int n = 0;
+
+	if (apstate->scale_coefs_blob_prop) {
+		blob = apstate->scale_coefs_blob_prop;
+		coef_data = (int *)blob->data;
+
+		for (n = 0; n < size; n++) {
+			if ((coef_data[n] > 32766) || (coef_data[n] < -32767)) {
+				DRM_ERROR("scale coef is invalid %d\n", coef_data[n]);
+				return -EINVAL;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int spacemit_plane_atomic_check(struct drm_plane *plane,
+				  struct drm_atomic_state *atomic_state)
+{
+	struct drm_plane_state *state = drm_atomic_get_new_plane_state(atomic_state, plane);
+	struct drm_framebuffer *fb = state->fb;
+	u32 src_w, src_h, src_x, src_y, scl_in_width, scl_in_height;
+	u32 crtc_w, crtc_h;
+	struct spacemit_plane_state *cur_state = to_spacemit_plane_state(state);
+	u32 cur_rdma_id = cur_state->rdma_id;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	const struct spacemit_hw_rdma *rdmas = hwdev->rdmas;
+	struct spacemit_crtc *a_crtc = NULL;
+
+	if (!state->crtc || WARN_ON(!state->fb))
+		return 0;
+
+	a_crtc = to_spacemit_crtc(state->crtc);
+	trace_spacemit_plane_atomic_check(a_crtc->dev_id);
+
+	src_x = state->src_x >> 16;
+	src_y = state->src_y >> 16;
+	src_w = state->src_w >> 16;
+	src_h = state->src_h >> 16;
+	crtc_w = state->crtc_w;
+	crtc_h = state->crtc_h;
+
+	/* For multi planes, only support planes with its offsets is set */
+	if (fb->format->num_planes > 3 ||
+	   (fb->format->num_planes > 2 && fb->offsets[2] == 0) ||
+	   (fb->format->num_planes > 1 && fb->offsets[1] == 0)) {
+		DRM_ERROR("%s, Unsupported plane format: plane_num:%d offsets[1]:%d offsets[2]:%d\n",
+			  __func__, fb->format->num_planes, fb->offsets[1], fb->offsets[2]);
+		return -EINVAL;
+	}
+
+	if (fb->format->num_planes > 1 && fb->modifier) {
+		DRM_ERROR("%s, Unsupported afbc with plane_num:%d\n", __func__, fb->format->num_planes);
+		return -EINVAL;
+	}
+
+	if (fb->format->format == DRM_FORMAT_NV12 || fb->format->format == DRM_FORMAT_YUV420_8BIT) {
+		if (src_x % 2 || src_y % 2 || src_w % 2 || src_h % 2) {
+			DRM_ERROR("YUV420 coordinations must be even! src_x:%d src_y:%d src_w:%d src_h:%d\n",
+				  src_x, src_y, src_w, src_h);
+			return -EINVAL;
+		}
+	}
+
+	/* adjust rdma id */
+	if (src_w == 0 && src_h == 0)
+		cur_rdma_id = RDMA_INVALID_ID;
+	else if (src_w == 0 || src_h == 0) {
+		/* check whether plane is invalid when not solid */
+		DRM_ERROR("%s, Unsupported src_w %d src_h %d\n", __func__, src_w, src_h);
+		return -EINVAL;
+	} else {
+		/* In case the userspace hasn't set rdma id */
+		if (cur_rdma_id == RDMA_INVALID_ID)
+			cur_rdma_id = state->zpos;
+	}
+	cur_state->rdma_id = cur_rdma_id;
+
+	/* Skip solid color */
+	if (cur_rdma_id != RDMA_INVALID_ID) {
+		if (a_crtc->is_offline_mode)
+			cur_state->is_offline = 1;
+
+		if (is_rot_90_270(state->rotation)) {
+			scl_in_width  = src_h;
+			scl_in_height = src_w;
+		} else {
+			scl_in_width  = src_w;
+			scl_in_height = src_h;
+		}
+
+		if (scl_in_width != crtc_w || scl_in_height != crtc_h) {
+			if ((scl_in_width * SPACEMIT_MAX_SCALE_FACTOR < crtc_w) ||
+				   (crtc_w * SPACEMIT_MAX_SCALE_FACTOR < scl_in_width)) {
+				DRM_ERROR("Width scaling exceeds %dx limit: %d -> %d\n",
+					SPACEMIT_MAX_SCALE_FACTOR, scl_in_width, crtc_w);
+				return -EINVAL;
+			}
+			if ((scl_in_height * SPACEMIT_MAX_SCALE_FACTOR < crtc_h) ||
+				   (crtc_h * SPACEMIT_MAX_SCALE_FACTOR < scl_in_height)) {
+				DRM_ERROR("Height scaling exceeds %dx limit: %d -> %d\n",
+					SPACEMIT_MAX_SCALE_FACTOR, scl_in_height, crtc_h);
+				return -EINVAL;
+			}
+			cur_state->use_scl = true;
+		} else {
+			cur_state->use_scl = false;
+		}
+		if (cur_rdma_id < hwdev->rdma_nums) {
+			if (spacemit_plane_check_rdma(rdmas, cur_rdma_id, state))
+				return -EINVAL;
+		} else {
+			DRM_ERROR("Invalid rdma id:%d\n", cur_rdma_id);
+			return -EINVAL;
+		}
+
+		if (a_crtc->core->cal_layer_fbcmem_size(plane, state)) {
+			DRM_ERROR("plane:%d Invalid fbcmem size\n", state->zpos);
+			return -EINVAL;
+		}
+
+		if (a_crtc->core->calc_plane_mclk_bw(plane, state)) {
+			DRM_INFO("plane:%d unsupported mclk or bandwidth\n", state->zpos);
+			return -EINVAL;
+		}
+	}
+
+	if (spacemit_plane_atomic_check_hdr_coefs(plane, state)) {
+		DRM_ERROR("The value of hdr coef is invalid\n");
+		return -EINVAL;
+	}
+
+	if (spacemit_plane_atomic_check_scale_coefs(plane, state)) {
+		DRM_ERROR("The value of scale coef is invalid\n");
+		return -EINVAL;
+	}
+
+	cur_state->format = spacemit_plane_hw_get_format_id(fb->format->format);
+	if (cur_state->format == SPACEMIT_DPU_INVALID_FORMAT_ID) {
+		DRM_ERROR("Can't support format:%p4cc\n", &fb->format->format);
+		return -EINVAL;
+	}
+
+	/* Use default values as they are not actually used now  */
+	cur_state->right_image = 0;
+	return 0;
+}
+
+static void spacemit_plane_atomic_update(struct drm_plane *plane,
+				    struct drm_atomic_state *state)
+{
+	int ret = 0;
+	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
+	struct spacemit_crtc *a_crtc = to_spacemit_crtc(plane->state->crtc);
+	struct spacemit_plane_state *spacemit_pstate = to_spacemit_plane_state(plane->state);
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u32 rdma_id = spacemit_pstate->rdma_id;
+	int i = 0;
+	struct drm_crtc_state *crtc_state;
+	struct spacemit_crtc_state *spacemit_crtc_state;
+	struct cmdlist *cl = NULL;
+
+	DRM_DEBUG("%s()\n", __func__);
+	trace_spacemit_plane_atomic_update(a_crtc->dev_id);
+
+	spacemit_pstate->cur_cl = 0;
+
+	crtc_state = a_crtc->crtc.state;
+	spacemit_crtc_state = to_spacemit_crtc_state(crtc_state);
+	spacemit_crtc_state->cur_cl = 0;
+
+	hwdev->get_cl_rdma_buf(a_crtc);
+	//cl_rdma free is at the end of the function and early return
+
+	hwdev->update_hdr_matrix(plane, spacemit_pstate);
+	hwdev->update_csc_matrix(plane, old_state);
+
+	if (hwdev->update_prepipe_gamma)
+		hwdev->update_prepipe_gamma(plane, spacemit_pstate);
+
+	if (a_crtc->is_slice_mode) {
+		for (i = 0; i < a_crtc->slice_num; i++) {
+			hwdev->plane_update_hw_channel(plane, i);
+
+			/* slice 0 config dmmu*/
+			if (i == 0) {
+				u8 tbu_id;
+
+				spacemit_pstate->mmu_tbl.size = ((PAGE_ALIGN(plane->state->fb->obj[0]->size) >> PAGE_SHIFT) +
+						HW_ALIGN_TTB_NUM) * 4;
+				spacemit_pstate->mmu_tbl.va = dma_alloc_coherent(a_crtc->dev, spacemit_pstate->mmu_tbl.size,
+						&spacemit_pstate->mmu_tbl.pa, GFP_KERNEL | __GFP_ZERO);
+				if (spacemit_pstate->mmu_tbl.va == NULL) {
+					DRM_ERROR("Failed to allocate %d bytes for dpu plane%d mmu table\n",
+							spacemit_pstate->mmu_tbl.size, plane->state->zpos);
+					free_cmdlist_regs(a_crtc->cl_rdma);
+					return;
+				}
+				tbu_id = !spacemit_pstate->right_image ? (rdma_id * 2) : (rdma_id * 2 + 1);
+				ret = spacemit_dmmu_map(plane->state->fb, &spacemit_pstate->mmu_tbl, tbu_id, NULL, a_crtc->cl_rdma, plane);
+				if (!ret)
+					cmdlist_regs_packing(plane_to_cl(plane), CMDLIST_MOD_RDMA, a_crtc->cl_rdma);
+				else
+					DRM_ERROR("%s failed to map plane with ret = %d\n", __func__, ret);
+			} else {
+				cmdlist_regs_packing(plane_to_cl(plane), CMDLIST_MOD_RDMA, a_crtc->cl_rdma);
+			}
+			cl = &spacemit_pstate->cl[spacemit_pstate->cur_cl];
+			cl->rch_start_cmps_y = a_crtc->slice_wb[0].crtc_h * i;
+			cl->cmdlist_ch_y_other = a_crtc->slice_wb[0].crtc_h * (i + 1);
+
+			spacemit_pstate->cur_cl++;
+			spacemit_crtc_state->cur_cl++;
+		}
+	} else {
+		hwdev->plane_update_hw_channel(plane, DPU_NO_SLICE);
+
+		/* No need for solid color layer */
+		if (rdma_id < hwdev->rdma_nums && spacemit_pstate->cur_cl == 0) {
+			u8 tbu_id;
+
+			spacemit_pstate->mmu_tbl.size = ((PAGE_ALIGN(plane->state->fb->obj[0]->size) >> PAGE_SHIFT) +
+					HW_ALIGN_TTB_NUM) * 4;
+			spacemit_pstate->mmu_tbl.va = dma_alloc_coherent(a_crtc->dev, spacemit_pstate->mmu_tbl.size,
+					&spacemit_pstate->mmu_tbl.pa, GFP_KERNEL | __GFP_ZERO);
+			if (spacemit_pstate->mmu_tbl.va == NULL) {
+				DRM_ERROR("Failed to allocate %d bytes for dpu plane%d mmu table\n",
+						spacemit_pstate->mmu_tbl.size, plane->state->zpos);
+				free_cmdlist_regs(a_crtc->cl_rdma);
+				return;
+			}
+			tbu_id = !spacemit_pstate->right_image ? (rdma_id * 2) : (rdma_id * 2 + 1);
+			ret = spacemit_dmmu_map(plane->state->fb, &spacemit_pstate->mmu_tbl, tbu_id, NULL, a_crtc->cl_rdma, plane);
+			if (!ret)
+				cmdlist_regs_packing(plane_to_cl(plane), CMDLIST_MOD_RDMA, a_crtc->cl_rdma);
+			else
+				DRM_ERROR("%s failed to map plane with ret = %d\n", __func__, ret);
+		}
+
+		cl = &spacemit_pstate->cl[spacemit_pstate->cur_cl];
+		cl->rch_start_cmps_y = 0xDEADBEEF;
+		cl->cmdlist_ch_y_other = 0xDEADBEEF;
+	}
+	free_cmdlist_regs(a_crtc->cl_rdma);
+}
+
+static void spacemit_plane_atomic_disable(struct drm_plane *plane,
+				     struct drm_atomic_state *state)
+{
+	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	spacemit_dmmu_unmap(plane);
+	hwdev->plane_disable_hw_channel(plane, old_state);
+}
+
+static void spacemit_plane_reset(struct drm_plane *plane)
+{
+	struct spacemit_plane *p = to_spacemit_plane(plane);
+	struct spacemit_plane_state *s;
+	struct spacemit_drm_private *priv = plane->dev->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	if (plane->state) {
+		s = to_spacemit_plane_state(plane->state);
+		__drm_atomic_helper_plane_destroy_state(plane->state);
+		kfree(s);
+		plane->state = NULL;
+	}
+
+	s = kzalloc(sizeof(*s), GFP_KERNEL);
+	if (s) {
+		__drm_atomic_helper_plane_reset(plane, &s->state);
+		s->state.zpos = hwdev->plane_nums - p->hw_pid - 1;
+		s->rdma_id = RDMA_INVALID_ID;
+		s->is_offline = 0;
+		s->scaler_id = SCALER_INVALID_ID;
+	}
+}
+
+static struct drm_plane_state *
+spacemit_plane_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct spacemit_plane_state *s;
+	struct spacemit_plane_state *old_state = to_spacemit_plane_state(plane->state);
+	struct spacemit_crtc *a_crtc = NULL;
+	u8 i = 0;
+
+	if (plane->crtc) {
+		a_crtc = to_spacemit_crtc(plane->crtc);
+		trace_spacemit_plane_atomic_duplicate_state(a_crtc->dev_id);
+	}
+	DRM_DEBUG("%s()\n", __func__);
+
+	s = kzalloc(sizeof(*s), GFP_KERNEL);
+	if (!s)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &s->state);
+
+	WARN_ON(s->state.plane != plane);
+
+	for (i = 0; i < MAX_CL_NUM; i++) {
+		s->cl[i].index = i;
+		s->cl[i].type = CMDLIST_PLANE;
+	}
+
+	s->is_offline = old_state->is_offline;
+	s->rdma_id = old_state->rdma_id;
+	s->format = old_state->format;
+	s->right_image = old_state->right_image;
+	s->scaler_id = SCALER_INVALID_ID;
+	s->use_scl = false;
+	s->fbcmem_size = 0;
+
+	if (s->hdr_coefs_blob_prop)
+		drm_property_blob_get(s->hdr_coefs_blob_prop);
+	if (s->scale_coefs_blob_prop)
+		drm_property_blob_get(s->scale_coefs_blob_prop);
+
+	return &s->state;
+}
+
+static void spacemit_plane_atomic_destroy_state(struct drm_plane *plane,
+					    struct drm_plane_state *state)
+{
+	int i = 0;
+	struct spacemit_plane_state *spacemit_pstate = to_spacemit_plane_state(state);
+	struct spacemit_crtc *a_crtc = NULL;
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	if (state->crtc) {
+		a_crtc = to_spacemit_crtc(state->crtc);
+
+		if (spacemit_pstate->mmu_tbl.va)
+			dma_free_coherent(a_crtc->dev, spacemit_pstate->mmu_tbl.size,
+					  spacemit_pstate->mmu_tbl.va, spacemit_pstate->mmu_tbl.pa);
+
+		for (i = 0; i < MAX_CL_NUM; i++) {
+			if (spacemit_pstate->cl[i].va)
+				dma_free_coherent(a_crtc->dev, spacemit_pstate->cl[i].size,
+						  spacemit_pstate->cl[i].va, spacemit_pstate->cl[i].pa);
+		}
+		trace_spacemit_plane_atomic_destroy_state(a_crtc->dev_id);
+	}
+	__drm_atomic_helper_plane_destroy_state(state);
+
+	if (spacemit_pstate->hdr_coefs_blob_prop)
+		drm_property_blob_put(spacemit_pstate->hdr_coefs_blob_prop);
+
+	if (spacemit_pstate->scale_coefs_blob_prop)
+		drm_property_blob_put(spacemit_pstate->scale_coefs_blob_prop);
+
+	kfree(to_spacemit_plane_state(state));
+}
+
+static int spacemit_plane_atomic_set_property(struct drm_plane *plane,
+					  struct drm_plane_state *state,
+					  struct drm_property *property,
+					  u64 val)
+{
+	struct spacemit_plane *p = to_spacemit_plane(plane);
+	struct spacemit_plane_state *s = to_spacemit_plane_state(state);
+	bool replaced = false;
+	int ret = 0;
+
+	DRM_DEBUG("%s() name = %s, val = %llu\n",
+		  __func__, property->name, val);
+
+	if (property == p->rdma_id_property)
+		s->rdma_id = val;
+	else if (property == p->solid_color_property)
+		s->solid_color = val;
+	else if (property == p->dec_lines_property)
+		s->dec_lines = val;
+	else if (property == p->hdr_coef_property) {
+		ret = spacemit_atomic_replace_property_blob_from_id(plane->dev,
+					&s->hdr_coefs_blob_prop,
+					val,
+					-1,
+					sizeof(int),
+					&replaced);
+		return ret;
+	} else if (property == p->scale_coef_property) {
+		ret = spacemit_atomic_replace_property_blob_from_id(plane->dev,
+					&s->scale_coefs_blob_prop,
+					val,
+					-1,
+					sizeof(int),
+					&replaced);
+		return ret;
+	} else {
+		DRM_ERROR("property %s is invalid\n", property->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int spacemit_plane_atomic_get_property(struct drm_plane *plane,
+					  const struct drm_plane_state *state,
+					  struct drm_property *property,
+					  u64 *val)
+{
+	struct spacemit_plane *p = to_spacemit_plane(plane);
+	const struct spacemit_plane_state *s = to_spacemit_plane_state(state);
+
+	DRM_DEBUG("%s() name = %s\n", __func__, property->name);
+
+	if (property == p->rdma_id_property)
+		*val = s->rdma_id;
+	else if (property == p->solid_color_property)
+		*val = s->solid_color;
+	else if (property == p->dec_lines_property)
+		*val = s->dec_lines;
+	else if (property == p->hdr_coef_property) {
+		if (s->hdr_coefs_blob_prop)
+			*val = (s->hdr_coefs_blob_prop) ? s->hdr_coefs_blob_prop->base.id : 0;
+	} else if (property == p->scale_coef_property) {
+		if (s->scale_coefs_blob_prop)
+			*val = (s->scale_coefs_blob_prop) ? s->scale_coefs_blob_prop->base.id : 0;
+	} else {
+		DRM_ERROR("property %s is invalid\n", property->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const char * const color_encoding_name[] = {
+	[DRM_COLOR_YCBCR_BT601] = "ITU-R BT.601 YCbCr",
+	[DRM_COLOR_YCBCR_BT709] = "ITU-R BT.709 YCbCr",
+	[DRM_COLOR_YCBCR_BT2020] = "ITU-R BT.2020 YCbCr",
+};
+
+static const char * const color_range_name[] = {
+	[DRM_COLOR_YCBCR_FULL_RANGE] = "YCbCr full range",
+	[DRM_COLOR_YCBCR_LIMITED_RANGE] = "YCbCr limited range",
+};
+
+static int spacemit_drm_plane_create_color_properties(struct drm_plane *plane,
+				      u32 supported_encodings,
+				      u32 supported_ranges,
+				      enum drm_color_encoding default_encoding,
+				      enum drm_color_range default_range)
+{
+	struct drm_device *dev = plane->dev;
+	struct drm_property *prop;
+	struct drm_prop_enum_list enum_list[MAX_T(int, DRM_COLOR_ENCODING_MAX,
+						       DRM_COLOR_RANGE_MAX)];
+	int i, len;
+
+	if (WARN_ON(supported_encodings == 0 ||
+		    (supported_encodings & -BIT(DRM_COLOR_ENCODING_MAX)) != 0 ||
+		    (supported_encodings & BIT(default_encoding)) == 0))
+		return -EINVAL;
+
+	if (WARN_ON(supported_ranges == 0 ||
+		    (supported_ranges & -BIT(DRM_COLOR_RANGE_MAX)) != 0 ||
+		    (supported_ranges & BIT(default_range)) == 0))
+		return -EINVAL;
+
+	len = 0;
+	for (i = 0; i < DRM_COLOR_ENCODING_MAX; i++) {
+		if ((supported_encodings & BIT(i)) == 0)
+			continue;
+
+		enum_list[len].type = i;
+		enum_list[len].name = color_encoding_name[i];
+		len++;
+	}
+
+	prop = drm_property_create_enum(dev, 0, "COLOR_ENCODING",
+					enum_list, len);
+	if (!prop)
+		return -ENOMEM;
+	plane->color_encoding_property = prop;
+	drm_object_attach_property(&plane->base, prop, default_encoding);
+	if (plane->state)
+		plane->state->color_encoding = default_encoding;
+
+	len = 0;
+	for (i = 0; i < DRM_COLOR_RANGE_MAX; i++) {
+		if ((supported_ranges & BIT(i)) == 0)
+			continue;
+
+		enum_list[len].type = i;
+		enum_list[len].name = color_range_name[i];
+		len++;
+	}
+
+	prop = drm_property_create_enum(dev, 0,	"COLOR_RANGE",
+					enum_list, len);
+	if (!prop)
+		return -ENOMEM;
+	plane->color_range_property = prop;
+	drm_object_attach_property(&plane->base, prop, default_range);
+	if (plane->state)
+		plane->state->color_range = default_range;
+
+	return 0;
+}
+
+static int spacemit_plane_create_properties(struct spacemit_plane *p, int index)
+{
+	struct drm_property *prop;
+	unsigned int support_modes = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+		BIT(DRM_MODE_BLEND_PREMULTI) |
+		BIT(DRM_MODE_BLEND_COVERAGE);
+	int ret = 0;
+
+	DRM_DEBUG("%s()\n", __func__);
+	/* create rotation property */
+	drm_plane_create_rotation_property(&p->plane,
+					   DRM_MODE_ROTATE_0,
+					   DRM_MODE_ROTATE_MASK |
+					   DRM_MODE_REFLECT_MASK);
+
+	/* create zpos property */
+	drm_plane_create_zpos_immutable_property(&p->plane, index);
+
+	/* create layer alpha property */
+	drm_plane_create_alpha_property(&p->plane);
+
+	/* create blend mode property */
+	drm_plane_create_blend_mode_property(&p->plane, support_modes);
+
+	prop = drm_property_create_range(p->plane.dev, 0,
+			"RDMA_ID", 0, ULLONG_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&p->plane.base, prop, 0);
+	p->rdma_id_property = prop;
+
+	prop = drm_property_create_range(p->plane.dev, 0,
+			"dec_lines", 0, 16);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&p->plane.base, prop, 0);
+	p->dec_lines_property = prop;
+
+	prop = drm_property_create_range(p->plane.dev, 0,
+			"SOLID_COLOR", 0, ULLONG_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&p->plane.base, prop, 0);
+	p->solid_color_property = prop;
+
+	prop = drm_property_create(p->plane.dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"hdr_coefs", 0);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&p->plane.base, prop, 0);
+	p->hdr_coef_property = prop;
+
+	prop = drm_property_create(p->plane.dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"scale_coefs", 0);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&p->plane.base, prop, 0);
+	p->scale_coef_property = prop;
+
+	ret = spacemit_drm_plane_create_color_properties(&p->plane,
+					BIT(DRM_COLOR_YCBCR_BT601) |
+					BIT(DRM_COLOR_YCBCR_BT709) |
+					BIT(DRM_COLOR_YCBCR_BT2020),
+					BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) |
+					BIT(DRM_COLOR_YCBCR_FULL_RANGE),
+					DRM_COLOR_YCBCR_BT601, DRM_COLOR_YCBCR_LIMITED_RANGE);
+	if (ret)
+		DRM_ERROR("Failed to create color properties %d\n", ret);
+
+	return 0;
+}
+
+static const struct drm_plane_helper_funcs spacemit_plane_helper_funcs = {
+	.atomic_check = spacemit_plane_atomic_check,
+	.atomic_update = spacemit_plane_atomic_update,
+	.atomic_disable = spacemit_plane_atomic_disable,
+};
+
+static const struct drm_plane_funcs spacemit_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane	= drm_atomic_helper_disable_plane,
+	.destroy = drm_plane_cleanup,
+	.reset = spacemit_plane_reset,
+	.atomic_duplicate_state = spacemit_plane_atomic_duplicate_state,
+	.atomic_destroy_state = spacemit_plane_atomic_destroy_state,
+	.atomic_set_property = spacemit_plane_atomic_set_property,
+	.atomic_get_property = spacemit_plane_atomic_get_property,
+};
+
+struct drm_plane *spacemit_plane_init(struct drm_device *drm,
+					struct spacemit_crtc *a_crtc)
+{
+	struct drm_plane *primary = NULL;
+	struct spacemit_plane *p = NULL;
+	enum drm_plane_type plane_type;
+	int err, i, j;
+	u32 *formats;
+	struct spacemit_drm_private *priv = drm->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	u8 n_planes = a_crtc->is_offline_mode == 1 ? hwdev->offline_plane_nums : hwdev->plane_nums;
+	u8 n_formats = hwdev->n_formats;
+	u8 n_fbcmems = hwdev->n_fbcmems;
+	u8 n_rdmas = hwdev->rdma_nums;
+	u32 plane_crtc_mask;
+
+	trace_spacemit_plane_init(a_crtc->dev_id);
+	if (n_fbcmems * 2 != n_rdmas) {
+		DRM_ERROR("Unmatched rdma and fbcmem numbers, n_rdmas:%d n_fbcmems:%d!\n",
+			   n_rdmas, n_fbcmems);
+		err = -EINVAL;
+		return ERR_PTR(err);
+	}
+
+	formats = kcalloc(n_formats, sizeof(*formats), GFP_KERNEL);
+	if (!formats) {
+		err = -ENOMEM;
+		return ERR_PTR(err);
+	}
+
+	/* Create planes for CRTC(a_crtc->dev_id) */
+	plane_crtc_mask = 1 << hwdev->crtc_num++;
+
+	for (i = 0; i < n_planes; i++) {
+		p = devm_kzalloc(drm->dev, sizeof(*p), GFP_KERNEL);
+		if (!p) {
+			kfree(formats);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		/* build the list of DRM supported formats based on the map */
+		for (j = 0; j < n_formats; j++)
+			formats[j] = hwdev->formats[j].format;
+
+		plane_type = (i == 0)
+			   ? DRM_PLANE_TYPE_PRIMARY
+			   : DRM_PLANE_TYPE_OVERLAY;
+
+		err = drm_universal_plane_init(drm, &p->plane, plane_crtc_mask,
+					       &spacemit_plane_funcs, formats,
+					       n_formats,  supported_format_modifiers,
+					       plane_type, NULL);
+		if (err) {
+			DRM_ERROR("fail to init %s plane%d\n", (i == 0) ? "primary" : "overlay", i);
+			return ERR_PTR(err);
+		}
+
+		drm_plane_helper_add(&p->plane, &spacemit_plane_helper_funcs);
+
+		spacemit_plane_create_properties(p, i);
+
+		p->hwdev = hwdev;
+		p->hw_pid = n_planes - i - 1;
+		if (i == 0)
+			primary = &p->plane;
+	}
+
+	kfree(formats);
+
+	if (p)
+		DRM_INFO("dpu plane init ok\n");
+
+	return primary;
+}
diff --git a/drivers/gpu/drm/spacemit/spacemit_wb.c b/drivers/gpu/drm/spacemit/spacemit_wb.c
new file mode 100644
index 000000000000..5b9cc954cca9
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_wb.c
@@ -0,0 +1,667 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_writeback.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_of.h>
+#include <drm/drm_edid.h>
+#include <linux/component.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/of_graph.h>
+#include "spacemit_lib.h"
+#include "spacemit_wb.h"
+//#include "spacemit_drm.h"
+#include "spacemit_dmmu.h"
+#include "sysfs/sysfs_display.h"
+#include "dpu/dpu_saturn.h"
+#include "dpu/saturn_regs/wb.h"
+
+static const u32 spacemit_wb_formats[] = {
+	DRM_FORMAT_XRGB8888,
+};
+
+int spacemit_wb_get_afbc_header_size(struct spacemit_wb *wb, struct drm_framebuffer *fb, int width, int height)
+{
+	int raw_width = 0;
+	int raw_heght = 0;
+	int fbc_out_width = 0;
+	int fbc_out_height = 0;
+	int w_block_num = 0;
+	int h_block_num = 0;
+	int fbc_block_size = 0;
+	int header_size = 0;
+	int payload_size = 0;
+
+	int bpp = 0;
+	int divide = 1;
+
+	if (fb->format->format == DRM_FORMAT_YUV420_8BIT) {
+		DRM_DEBUG("fb->format: DRM_FORMAT_YU08\n\r");
+		bpp = 3;
+		divide = 2;
+	} else if (fb->format->format == DRM_FORMAT_YUV420_10BIT) {
+		DRM_DEBUG("fb->format: DRM_FORMAT_YUV420_10BIT\n\r");
+		bpp = 3;
+	} else if (fb->format->format == DRM_FORMAT_ABGR8888) {
+		DRM_DEBUG("fb->format: DRM_FORMAT_ABGR8888\n\r");
+		bpp = 4;
+	} else if (fb->format->format == DRM_FORMAT_BGR888) {
+		DRM_DEBUG("fb->format: DRM_FORMAT_BGR8888\n\r");
+		bpp = 3;
+	} else if (fb->format->format == DRM_FORMAT_BGR565) {
+		DRM_DEBUG("fb->format: DRM_FORMAT_BGR565\n\r");
+		bpp = 2;
+	}
+
+	wb->afbc_state = kzalloc(sizeof(struct spacemit_afbc_state), GFP_KERNEL);
+	if (!wb->afbc_state) {
+		DRM_ERROR("Faild to set wb afbc\n");
+		return 0;
+	}
+	spacemit_get_afbc_modifier(fb->modifier, wb->afbc_state);
+
+	raw_width = width;
+	raw_heght = height;
+	if (wb->afbc_state->block_size == 1) {
+		if (wb->afbc_state->tile_type) {
+			fbc_out_width = ((raw_width + 255) >> 8) << 8;
+			fbc_out_height = ((raw_heght + 63) >> 6) << 6;
+		} else {
+			fbc_out_width = ((raw_width + 31) >> 5) << 5;
+			fbc_out_height = ((raw_heght + 7) >> 3) << 3;
+		}
+		w_block_num = fbc_out_width >> 5;
+		h_block_num = fbc_out_height >> 3;
+	} else {
+		if (wb->afbc_state->tile_type) {
+			fbc_out_width = ((raw_width + 127) >> 7) << 7;
+			fbc_out_height = ((raw_heght + 127) >> 7) << 7;
+		} else {
+			fbc_out_width = ((raw_width + 15) >> 4) << 4;
+			fbc_out_height = ((raw_heght + 15) >> 4) << 4;
+		}
+		w_block_num = fbc_out_width >> 4;
+		h_block_num = fbc_out_height >> 4;
+	}
+	kfree(wb->afbc_state);
+
+	fbc_block_size = w_block_num * h_block_num;
+	header_size = fbc_block_size * 16;// each block 16 byte header
+	payload_size = w_block_num * h_block_num * 256 * bpp/divide;
+	if (fbc_block_size == 0 || payload_size == 0) {
+		DRM_ERROR("Failed to cal afbc head/payload size, w_block_num = %d, h_block_num = %d, bpp = %d, divide = %d, payload_size = %d\n",
+				w_block_num, h_block_num, bpp, divide, payload_size);
+	}
+	return header_size;
+}
+
+static const struct spacemit_format_fb2wb spacemit_wb_format_fb2wb[] = {
+	{ DRM_FORMAT_ABGR2101010, SPACEMIT_WB_FORMAT_ABGR2101010},
+	{ DRM_FORMAT_ARGB8888, SPACEMIT_WB_FORMAT_ARGB8888},
+	{ DRM_FORMAT_ABGR8888, SPACEMIT_WB_FORMAT_ABGR8888},
+	{ DRM_FORMAT_RGBA8888, SPACEMIT_WB_FORMAT_RGBA8888},
+	{ DRM_FORMAT_BGRA8888, SPACEMIT_WB_FORMAT_BGRA8888},
+	{ DRM_FORMAT_XRGB8888, SPACEMIT_WB_FORMAT_XRGB8888},
+	{ DRM_FORMAT_XBGR8888, SPACEMIT_WB_FORMAT_XBGR8888},
+	{ DRM_FORMAT_RGBX8888, SPACEMIT_WB_FORMAT_RGBX8888},
+	{ DRM_FORMAT_BGRX8888, SPACEMIT_WB_FORMAT_BGRX8888},
+	{ DRM_FORMAT_RGB565, SPACEMIT_WB_FORMAT_RGB565},
+	{ DRM_FORMAT_BGR565, SPACEMIT_WB_FORMAT_BGR565},
+	{ DRM_FORMAT_RGB888, SPACEMIT_WB_FORMAT_RGB888},
+	{ DRM_FORMAT_BGR888, SPACEMIT_WB_FORMAT_BGR888},
+	{ DRM_FORMAT_YUV420_8BIT, SPACEMIT_WB_FORMAT_YUV420_P2_8_VU},
+	{ DRM_FORMAT_NV12, SPACEMIT_WB_FORMAT_YUV420_P2_8_VU},
+	{ DRM_FORMAT_P010, SPACEMIT_WB_FORMAT_YUV420_P2_10_VU},
+};
+
+static int spacemit_wb_get_format(u32 format)
+{
+	unsigned int i = 0;
+
+	for (i = 0; i < ARRAY_SIZE(spacemit_wb_format_fb2wb); i++) {
+		if (format == spacemit_wb_format_fb2wb[i].format)
+			return spacemit_wb_format_fb2wb[i].wb_id;
+	}
+
+	DRM_ERROR("format 0x%x is not supported in wb!\n", format);
+	return SPACEMIT_WB_INVALID_FORMAT_ID;
+}
+
+static void spacemit_wb_connector_atomic_commit(struct drm_connector *conn, struct drm_atomic_state *state)
+{
+	struct drm_device *drm = conn->dev;
+	struct spacemit_drm_private *priv = drm->dev_private;
+	struct spacemit_hw_device *hwdev = priv->hwdev;
+	struct drm_writeback_connector *wb_conn;
+	struct drm_framebuffer *fb;
+	struct spacemit_wb *wb = container_of(conn, struct spacemit_wb, wb_connector.base);
+	u8 tbu_id = hwdev->rdma_nums * 2 + wb->ctx.id;
+	struct drm_connector_state *conn_state = NULL;
+
+	struct cmdlist_regs *cl_wb;
+	cmdlist_mode_type_t cl_mode_type;
+	int ret;
+
+	struct drm_crtc *crtc = NULL;
+	struct spacemit_crtc *a_crtc = NULL;
+	struct drm_crtc_state *crtc_state;
+	struct spacemit_crtc_state *spacemit_crtc_state;
+	struct cmdlist *cl = NULL;
+	int i = 0;
+
+	cl_wb = NULL;
+	cl_mode_type = CMDLIST_MOD_WB0;
+	ret = 0;
+
+	wb_conn = priv->wb_connector[wb->ctx.id];
+	conn_state = wb_conn->base.state;
+	if (WARN_ON(!conn_state->writeback_job))
+		return;
+	cl_wb = alloc_cmdlist_regs(WB_REG);
+
+	crtc = conn_state->crtc;
+	a_crtc = to_spacemit_crtc(crtc);
+
+	crtc_state = a_crtc->crtc.state;
+	spacemit_crtc_state = to_spacemit_crtc_state(crtc_state);
+
+	fb = conn_state->writeback_job->fb;
+	DRM_DEBUG("WB: %s w %d, h %d, stride %d", __func__, fb->width, fb->height, fb->pitches[0]);
+	wb->format = spacemit_wb_get_format(fb->format->format);
+	/* init mmu_tbl */
+	spacemit_crtc_state->wb_mmu_tbl.size = ((PAGE_ALIGN(fb->obj[0]->size) >> PAGE_SHIFT) +
+			HW_ALIGN_TTB_NUM) * 4;
+	spacemit_crtc_state->wb_mmu_tbl.va = dma_alloc_coherent(a_crtc->dev, spacemit_crtc_state->wb_mmu_tbl.size,
+			&spacemit_crtc_state->wb_mmu_tbl.pa, GFP_KERNEL | __GFP_ZERO);
+	if (spacemit_crtc_state->wb_mmu_tbl.va == NULL) {
+		DRM_ERROR("Failed to allocate %d bytes for wb mmu table\n", spacemit_crtc_state->wb_mmu_tbl.size);
+		free_cmdlist_regs(cl_wb);
+		return;
+	}
+
+	wb_conn = priv->wb_connector[wb->ctx.id];
+	drm_writeback_queue_job(wb_conn, wb_conn->base.state);
+
+	wb->a_crtc = a_crtc;
+	wb->cl_wb = cl_wb;
+	a_crtc->wb_id = wb->ctx.id;
+	if (a_crtc->wb_id == 1)
+		cl_mode_type = CMDLIST_MOD_WB1;
+
+	ret = spacemit_dmmu_map(fb, &spacemit_crtc_state->wb_mmu_tbl, tbu_id, wb, NULL, NULL);
+	if (ret) {
+		DRM_ERROR("%s failed to map wb with ret = %d\n", __func__, ret);
+		free_cmdlist_regs(cl_wb);
+		if (spacemit_crtc_state->wb_mmu_tbl.va)
+			dma_free_coherent(a_crtc->dev, spacemit_crtc_state->wb_mmu_tbl.size,
+				spacemit_crtc_state->wb_mmu_tbl.va, spacemit_crtc_state->wb_mmu_tbl.pa);
+		return;
+	}
+
+	if (a_crtc->is_slice_mode) {
+		for (i = 0; i < a_crtc->slice_num; i++) {
+			hwdev->wb_config(a_crtc, wb, fb, wb->cl_wb, i);
+			cmdlist_regs_packing(crtc_to_cl(crtc), cl_mode_type, cl_wb);
+			cl = &spacemit_crtc_state->cl[spacemit_crtc_state->cur_cl];
+			cl->rch_start_cmps_y = 0xDEADBEEF;
+			cl->cmdlist_ch_y_other = a_crtc->slice_wb[0].crtc_h * (i + 1);
+			spacemit_crtc_state->cur_cl++;
+		}
+	} else {
+		hwdev->wb_config(a_crtc, wb, fb, wb->cl_wb, i);
+		cmdlist_regs_packing(crtc_to_cl(crtc), cl_mode_type, cl_wb);
+		cl = &spacemit_crtc_state->cl[spacemit_crtc_state->cur_cl];
+		cl->rch_start_cmps_y = 0xDEADBEEF;
+		cl->cmdlist_ch_y_other = 0xDEADBEEF;
+		spacemit_crtc_state->cur_cl++;
+	}
+
+
+	free_cmdlist_regs(cl_wb);
+}
+
+static int spacemit_wb_encoder_atomic_check(struct drm_encoder *encoder,
+				    struct drm_crtc_state *crtc_state,
+				    struct drm_connector_state *conn_state)
+{
+	DRM_DEBUG("%s()\n", __func__);
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs spacemit_wb_encoder_helper_funcs = {
+	.atomic_check = spacemit_wb_encoder_atomic_check,
+};
+
+static int spacemit_wb_connector_get_modes(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_connector *c;
+	struct drm_connector_list_iter conn_iter;
+	struct drm_display_mode *mode, *dup;
+	int cnt = 0;
+	struct drm_display_mode mode_256x600 = {
+			DRM_MODE("256x600", DRM_MODE_TYPE_DRIVER, 20276, 256, 360,
+			364, 474, 0, 600, 650, 654, 704, 0,
+			DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) };
+
+	DRM_DEBUG("%s()\n", __func__);
+
+	/*
+	 * Mirror modes from all non-writeback connectors (e.g. DSI panel).
+	 * drm_add_modes_noedid() only covers standard VESA resolutions and
+	 * misses panel-specific ones, so we copy them here.
+	 */
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(c, &conn_iter) {
+		if (c == connector)
+			continue;
+		if (c->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+		list_for_each_entry(mode, &c->modes, head) {
+			dup = drm_mode_duplicate(dev, mode);
+			if (!dup) {
+				DRM_ERROR("wb: failed to dup mode %s\n", mode->name);
+				continue;
+			}
+			drm_mode_probed_add(connector, dup);
+			DRM_INFO("wb: mirrored mode %s from connector %s\n",
+				 dup->name, c->name);
+			cnt++;
+		}
+	}
+	drm_connector_list_iter_end(&conn_iter);
+
+	/* Add standard VESA modes as additional options */
+	cnt += drm_add_modes_noedid(connector, dev->mode_config.max_width,
+				    dev->mode_config.max_height);
+
+	/* Add hardware-specific 256x600 mode */
+	dup = drm_mode_duplicate(connector->dev, &mode_256x600);
+	if (dup) {
+		drm_mode_set_name(dup);
+		drm_mode_probed_add(connector, dup);
+		cnt++;
+	}
+
+	return cnt;
+}
+
+static enum drm_mode_status
+spacemit_wb_connector_mode_valid(struct drm_connector *connector,
+			 const struct drm_display_mode *mode)
+{
+	enum drm_mode_status mode_status = MODE_OK;
+
+	DRM_DEBUG("%s(%s)\n", __func__, mode->name);
+
+	return mode_status;
+}
+
+static enum drm_connector_status
+spacemit_wb_connector_detect(struct drm_connector *connector, bool force)
+{
+	return connector_status_connected;
+}
+
+static void spacemit_wb_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static const struct drm_connector_helper_funcs spacemit_wb_connector_helper_funcs = {
+	.get_modes = spacemit_wb_connector_get_modes,
+	.mode_valid = spacemit_wb_connector_mode_valid,
+	.atomic_commit = spacemit_wb_connector_atomic_commit,
+};
+
+static int spacemit_wb_atomic_get_property(struct drm_connector *connector,
+				   const struct drm_connector_state *state,
+				   struct drm_property *property,
+				   uint64_t *val)
+{
+	struct spacemit_wb *wb = container_of(connector, struct spacemit_wb, wb_connector.base);
+
+	DRM_DEBUG("%s() name = %s\n", __func__, property->name);
+
+	if (property == wb->rotation_property) {
+		*val = wb->rotation;
+	} else if (property == wb->in_x_property) {
+		*val = wb->in_x;
+	} else if (property == wb->in_y_property) {
+		*val = wb->in_y;
+	} else if (property == wb->in_w_property) {
+		*val = wb->in_w;
+	} else if (property == wb->in_h_property) {
+		*val = wb->in_h;
+	} else if (property == wb->out_x_property) {
+		*val = wb->out_x;
+	} else if (property == wb->out_y_property) {
+		*val = wb->out_y;
+	} else if (property == wb->out_w_property) {
+		*val = wb->out_w;
+	} else if (property == wb->out_h_property) {
+		*val = wb->out_h;
+	} else {
+		DRM_ERROR("property %s is invalid\n", property->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int spacemit_wb_atomic_set_property(struct drm_connector *connector,
+				   struct drm_connector_state *state,
+				   struct drm_property *property,
+				   uint64_t val)
+{
+	struct spacemit_wb *wb = container_of(connector, struct spacemit_wb, wb_connector.base);
+
+	DRM_DEBUG("%s() name = %s, val = %llu\n",
+		__func__, property->name, val);
+
+	if (property == wb->rotation_property) {
+		if (!is_power_of_2(val & DRM_MODE_ROTATE_MASK)) {
+			DRM_DEBUG_ATOMIC("[WB:%d] bad rotation bitmask: 0x%llx\n",
+					wb->ctx.id, val);
+			return -EINVAL;
+		}
+		wb->rotation = val;
+	} else if (property == wb->in_x_property) {
+		wb->in_x = val;
+	} else if (property == wb->in_y_property) {
+		wb->in_y = val;
+	} else if (property == wb->in_w_property) {
+		wb->in_w = val;
+	} else if (property == wb->in_h_property) {
+		wb->in_h = val;
+	} else if (property == wb->out_x_property) {
+		wb->out_x = val;
+	} else if (property == wb->out_y_property) {
+		wb->out_y = val;
+	} else if (property == wb->out_w_property) {
+		wb->out_w = val;
+	} else if (property == wb->out_h_property) {
+		wb->out_h = val;
+	} else {
+		DRM_ERROR("property %s is invalid\n", property->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct drm_connector_funcs spacemit_wb_connector_funcs = {
+	.atomic_set_property = spacemit_wb_atomic_set_property,
+	.atomic_get_property = spacemit_wb_atomic_get_property,
+	.reset = drm_atomic_helper_connector_reset,
+	.detect = spacemit_wb_connector_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = spacemit_wb_connector_destroy,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int spacemit_wb_device_create(struct spacemit_wb *wb, struct device *parent)
+{
+	int ret;
+
+	wb->dev.class = display_class;
+	wb->dev.parent = parent;
+	wb->dev.of_node = parent->of_node;
+	dev_set_name(&wb->dev, "wb%d", wb->ctx.id);
+	dev_set_drvdata(&wb->dev, wb);
+
+	ret = device_register(&wb->dev);
+	if (ret)
+		DRM_ERROR("wb device register failed\n");
+
+	return ret;
+}
+
+static int spacemit_wb_context_init(struct spacemit_wb *wb, struct device_node *np)
+{
+	struct spacemit_wb_device *ctx = &wb->ctx;
+	u32 tmp;
+	u32 crtc_mask = drm_of_find_possible_crtcs(wb->ddev, np);
+
+	if (!crtc_mask) {
+		DRM_ERROR("failed to find crtc mask\n");
+		return -EINVAL;
+	}
+	DRM_INFO("wb find possible crtcs: 0x%08x\n", crtc_mask);
+
+	wb->wb_connector.encoder.possible_crtcs = crtc_mask;
+
+	if (!of_property_read_u32(np, "dev-id", &tmp))
+		ctx->id = tmp;
+
+	if (ctx->id >= N_WRITEBACK_MAX) {
+		DRM_ERROR("Current id = %d, Max id = %d\n", ctx->id, N_WRITEBACK_MAX - 1);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int spacemit_wb_create_properties(struct drm_connector *conn)
+{
+	struct drm_property *prop;
+	struct drm_device *drm = conn->dev;
+	struct spacemit_wb *wb = container_of(conn, struct spacemit_wb, wb_connector.base);
+
+	static const struct drm_prop_enum_list props[] = {
+		{ __builtin_ffs(DRM_MODE_ROTATE_0) - 1,   "rotate-0" },
+		{ __builtin_ffs(DRM_MODE_ROTATE_90) - 1,  "rotate-90" },
+		{ __builtin_ffs(DRM_MODE_ROTATE_180) - 1, "rotate-180" },
+		{ __builtin_ffs(DRM_MODE_ROTATE_270) - 1, "rotate-270" },
+	};
+	unsigned int rotation = DRM_MODE_ROTATE_0;
+	unsigned int supported_rotations = DRM_MODE_ROTATE_MASK;
+
+	DRM_DEBUG("%s()\n", __func__);
+	WARN_ON((supported_rotations & DRM_MODE_ROTATE_MASK) == 0);
+	WARN_ON(!is_power_of_2(rotation & DRM_MODE_ROTATE_MASK));
+	WARN_ON(rotation & ~supported_rotations);
+
+	prop = drm_property_create_bitmask(drm, 0, "wb_rotation",
+					props, ARRAY_SIZE(props),
+
+					supported_rotations);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, rotation);
+	wb->rotation_property = prop;
+	wb->rotation = rotation;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_IN_X", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->in_x_property = prop;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_IN_Y", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->in_y_property = prop;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_IN_W", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->in_w_property = prop;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_IN_H", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->in_h_property = prop;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_OUT_X", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->out_x_property = prop;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_OUT_Y", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->out_y_property = prop;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_OUT_W", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->out_w_property = prop;
+
+	prop = drm_property_create_range(drm, DRM_MODE_PROP_ATOMIC,
+			"WB_OUT_H", 0, UINT_MAX);
+	if (!prop)
+		return -ENOMEM;
+	drm_object_attach_property(&conn->base, prop, 0);
+	wb->out_h_property = prop;
+
+	return 0;
+}
+
+static int spacemit_wb_bind(struct device *dev, struct device *master, void *data)
+{
+	struct drm_device *drm = data;
+	struct spacemit_drm_private *priv = drm->dev_private;
+	struct spacemit_wb *wb = dev_get_drvdata(dev);
+	struct device_node *np = wb->pdev->dev.of_node;
+	struct drm_encoder *enc;
+	struct drm_encoder *wb_enc;
+	u32 all_clones;
+	int id = -1;
+	int ret;
+
+	wb->ddev = drm;
+	ret = spacemit_wb_context_init(wb, np);
+	if (ret)
+		return -EINVAL;
+
+	id = wb->ctx.id;
+	if (id < 0)
+		return -EINVAL;
+
+	spacemit_wb_device_create(wb, &wb->pdev->dev);
+
+	priv->wb_connector[id] = &wb->wb_connector;
+
+	drm_connector_helper_add(&priv->wb_connector[id]->base,
+				 &spacemit_wb_connector_helper_funcs);
+
+	ret = drm_writeback_connector_init(drm, priv->wb_connector[id],
+					   &spacemit_wb_connector_funcs,
+					   &spacemit_wb_encoder_helper_funcs,
+					   spacemit_wb_formats,
+					   ARRAY_SIZE(spacemit_wb_formats),
+					   wb->wb_connector.encoder.possible_crtcs);
+	if (ret) {
+		DRM_ERROR("drm_connector_init() failed\n");
+		return ret;
+	}
+
+	/*
+	 * drm_mode_config_validate() auto-fills possible_clones=0 to
+	 * drm_encoder_mask(self). So DSI ends up with 0x1, WB with 0x2.
+	 * drm_atomic_check_valid_clones() then rejects the commit because
+	 * DSI's possible_clones(0x1) doesn't include WB(0x2).
+	 * Fix: set both to the full encoder mask so they can share a CRTC.
+	 */
+	wb_enc = &priv->wb_connector[id]->encoder;
+	all_clones = drm_encoder_mask(wb_enc);
+
+	/* First pass: collect mask of all display encoders */
+	drm_for_each_encoder(enc, drm) {
+		if (enc != wb_enc)
+			all_clones |= drm_encoder_mask(enc);
+	}
+
+	/* Second pass: set possible_clones = all, for everyone */
+	drm_for_each_encoder(enc, drm) {
+		enc->possible_clones = all_clones;
+		DRM_INFO("wb: encoder %u possible_clones = 0x%x\n",
+			 enc->base.id, enc->possible_clones);
+	}
+
+	spacemit_wb_create_properties(&wb->wb_connector.base);
+
+	return 0;
+}
+
+static void spacemit_wb_unbind(struct device *dev,
+		       struct device *master, void *data)
+{
+	/* do nothing */
+	DRM_INFO("%s()\n", __func__);
+}
+
+static const struct component_ops spacemit_wb_component_ops = {
+	.bind = spacemit_wb_bind,
+	.unbind = spacemit_wb_unbind,
+};
+
+static int spacemit_wb_probe(struct platform_device *pdev)
+{
+	struct spacemit_wb *wb;
+
+	wb = devm_kzalloc(&pdev->dev, sizeof(*wb), GFP_KERNEL);
+	if (!wb) {
+		DRM_ERROR("failed to allocate wb data.\n");
+		return -ENOMEM;
+	}
+
+	wb->pdev = pdev;
+
+	platform_set_drvdata(pdev, wb);
+
+	return component_add(&pdev->dev, &spacemit_wb_component_ops);
+}
+
+static void spacemit_wb_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &spacemit_wb_component_ops);
+}
+
+static const struct of_device_id spacemit_wb_of_match[] = {
+	{.compatible = "spacemit,wb0"},
+	{.compatible = "spacemit,wb1"},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, spacemit_wb_of_match);
+
+struct platform_driver spacemit_wb_driver = {
+	.probe = spacemit_wb_probe,
+	.remove = spacemit_wb_remove,
+	.driver = {
+		.name = "spacemit-wb-drv",
+		.of_match_table = spacemit_wb_of_match,
+	},
+};
+
+MODULE_DESCRIPTION("Spacemit WB Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/spacemit_wb.h b/drivers/gpu/drm/spacemit/spacemit_wb.h
new file mode 100644
index 000000000000..2bf29ee6a42f
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/spacemit_wb.h
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SPACEMIT_MB_H_
+#define _SPACEMIT_MB_H_
+
+#include <linux/of.h>
+#include <linux/device.h>
+#include <video/videomode.h>
+
+#include <drm/drm_print.h>
+#include <drm/drm_writeback.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_bridge.h>
+
+#include "spacemit_cmdlist.h"
+#include "spacemit_lib.h"
+#include "spacemit_crtc.h"
+
+#define SPACEMIT_RGB2YUV_COEFFS	12
+#define SPACEMIT_WB_INVALID_FORMAT_ID	0xff
+
+struct dpu_mmu_tbl;
+struct spacemit_crtc;
+
+
+
+
+enum spacemit_wb_loc {
+	SPACEMIT_WB_RDMA0 = 0,
+	SPACEMIT_WB_PREPQ0,
+	SPACEMIT_WB_RDMA1,
+	SPACEMIT_WB_PREPQ1,
+	SPACEMIT_WB_RDMA2,
+	SPACEMIT_WB_PREPQ2,
+	SPACEMIT_WB_RDMA3,
+	SPACEMIT_WB_PREPQ3,
+	SPACEMIT_WB_RDMA4,
+	SPACEMIT_WB_PREPQ4,
+	SPACEMIT_WB_RDMA5,
+	SPACEMIT_WB_PREPQ5,
+	SPACEMIT_WB_RDMA6,
+	SPACEMIT_WB_PREPQ6,
+	SPACEMIT_WB_RDMA7,
+	SPACEMIT_WB_PREPQ7,
+	SPACEMIT_WB_RDMA8,
+	SPACEMIT_WB_PREPQ8,
+	SPACEMIT_WB_RDMA9,
+	SPACEMIT_WB_PREPQ9,
+	SPACEMIT_WB_RDMA10,
+	SPACEMIT_WB_PREPQ10,
+	SPACEMIT_WB_RDMA11,
+	SPACEMIT_WB_PREPQ11,
+	SPACEMIT_WB_COMP0,
+	SPACEMIT_WB_COMP1,
+	SPACEMIT_WB_COMP2,
+	SPACEMIT_WB_POST0,
+	SPACEMIT_WB_POST1,
+	SPACEMIT_WB_POST2,
+};
+
+enum spacemit_wb_format {
+	SPACEMIT_WB_FORMAT_ARGB2101010 = 0,
+	SPACEMIT_WB_FORMAT_ABGR2101010,
+	SPACEMIT_WB_FORMAT_RGBA1010102,
+	SPACEMIT_WB_FORMAT_BGRA1010102,
+	SPACEMIT_WB_FORMAT_ARGB8888,
+	SPACEMIT_WB_FORMAT_ABGR8888,
+	SPACEMIT_WB_FORMAT_RGBA8888,
+	SPACEMIT_WB_FORMAT_BGRA8888,
+	SPACEMIT_WB_FORMAT_XRGB8888,
+	SPACEMIT_WB_FORMAT_XBGR8888,
+	SPACEMIT_WB_FORMAT_RGBX8888,
+	SPACEMIT_WB_FORMAT_BGRX8888,
+	SPACEMIT_WB_FORMAT_RGB888,
+	SPACEMIT_WB_FORMAT_BGR888,
+	SPACEMIT_WB_FORMAT_RGBA5551,
+	SPACEMIT_WB_FORMAT_BGRA5551,
+	SPACEMIT_WB_FORMAT_ABGR1555,
+	SPACEMIT_WB_FORMAT_ARGB1555,
+	SPACEMIT_WB_FORMAT_RGBX5551,
+	SPACEMIT_WB_FORMAT_BGRX5551,
+	SPACEMIT_WB_FORMAT_XBGR1555,
+	SPACEMIT_WB_FORMAT_XRGB1555,
+	SPACEMIT_WB_FORMAT_RGB565,
+	SPACEMIT_WB_FORMAT_BGR565,
+	SPACEMIT_WB_FORMAT_YUV420_P2_8_UV,
+	SPACEMIT_WB_FORMAT_YUV420_P2_8_VU,
+	SPACEMIT_WB_FORMAT_YUV420_P2_10_UV,
+	SPACEMIT_WB_FORMAT_YUV420_P2_10_VU,
+};
+
+struct spacemit_format_fb2wb {
+	u32 format;             /* DRM fourcc */
+	int wb_id;		/* WB_FORMAT*/
+};
+
+struct spacemit_wb_device {
+	uint32_t id;
+	struct videomode vm;
+	int status;
+};
+
+struct spacemit_wb {
+	struct platform_device *pdev;
+	struct device dev;
+	struct drm_device *ddev;
+	struct drm_encoder encoder;
+	struct drm_writeback_connector wb_connector;
+	struct spacemit_wb_device ctx;
+	struct spacemit_crtc *a_crtc;
+	struct cmdlist_regs *cl_wb;
+	struct drm_property *rotation_property;
+	struct drm_property *in_x_property;
+	struct drm_property *in_y_property;
+	struct drm_property *in_w_property;
+	struct drm_property *in_h_property;
+	struct drm_property *out_x_property;
+	struct drm_property *out_y_property;
+	struct drm_property *out_w_property;
+	struct drm_property *out_h_property;
+	struct drm_property *scale_smooth_prop;
+	unsigned int rotation;
+	unsigned int in_x;
+	unsigned int in_y;
+	unsigned int in_w;
+	unsigned int in_h;
+	unsigned int out_x;
+	unsigned int out_y;
+	unsigned int out_w;
+	unsigned int out_h;
+	unsigned int format;
+	bool use_scl;
+	struct spacemit_afbc_state *afbc_state;
+};
+
+struct spacemit_connector_state {
+	struct drm_connector_state base;
+};
+
+int spacemit_wb_get_afbc_header_size(struct spacemit_wb *wb, struct drm_framebuffer *fb, int width, int height);
+
+#define to_spacemit_conn_state(x) container_of(x, struct spacemit_connector_state, base)
+
+void saturn_wb_config(struct spacemit_crtc *a_crtc, struct spacemit_wb *wb, struct drm_framebuffer *fb, struct cmdlist_regs *cl_wb);
+#endif
diff --git a/drivers/gpu/drm/spacemit/sysfs/sysfs_class.c b/drivers/gpu/drm/spacemit/sysfs/sysfs_class.c
new file mode 100644
index 000000000000..75abe345875f
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/sysfs/sysfs_class.c
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include "sysfs_display.h"
+
+struct class *display_class;
+EXPORT_SYMBOL_GPL(display_class);
+
+#ifndef MODULE
+static int __init display_class_init(void)
+#else
+int display_class_init(void)
+#endif
+{
+	pr_info("display class register\n");
+
+	display_class = class_create("display");
+	if (IS_ERR(display_class)) {
+		pr_err("Unable to create display class\n");
+		return PTR_ERR(display_class);
+	}
+
+	return 0;
+}
+
+#ifndef MODULE
+postcore_initcall(display_class_init);
+#endif
+MODULE_DESCRIPTION("Provide display class for hardware driver");
+MODULE_LICENSE("GPL v2");
+
diff --git a/drivers/gpu/drm/spacemit/sysfs/sysfs_display.h b/drivers/gpu/drm/spacemit/sysfs/sysfs_display.h
new file mode 100644
index 000000000000..f4866a60aac8
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/sysfs/sysfs_display.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#ifndef _SYSFS_DISPLAY_H_
+#define _SYSFS_DISPLAY_H_
+
+#include <linux/device.h>
+
+extern struct class *display_class;
+
+int spacemit_dpu_sysfs_init(struct device *dev);
+int spacemit_dsi_sysfs_init(struct device *dev);
+int spacemit_dphy_sysfs_init(struct device *dev);
+int spacemit_mipi_panel_sysfs_init(struct device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/spacemit/sysfs/sysfs_dphy.c b/drivers/gpu/drm/spacemit/sysfs/sysfs_dphy.c
new file mode 100644
index 000000000000..dee3d82e0f7b
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/sysfs/sysfs_dphy.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/platform_device.h>
+#include <linux/sysfs.h>
+
+#include "../spacemit_lib.h"
+#include "../spacemit_dphy.h"
+#include "sysfs_display.h"
+
+
+
+int spacemit_dphy_sysfs_init(struct device *dev)
+{
+	int rc = 0;
+/*
+	rc = sysfs_create_groups(&dev->kobj, dphy_groups);
+	if (rc)
+		pr_err("create dphy attr node failed, rc=%d\n", rc);
+*/
+	return rc;
+}
+EXPORT_SYMBOL(spacemit_dphy_sysfs_init);
+
+MODULE_DESCRIPTION("Add dphy attribute nodes for userspace");
+MODULE_LICENSE("GPL v2");
+
diff --git a/drivers/gpu/drm/spacemit/sysfs/sysfs_dpu.c b/drivers/gpu/drm/spacemit/sysfs/sysfs_dpu.c
new file mode 100644
index 000000000000..13c3438aca5f
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/sysfs/sysfs_dpu.c
@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/sysfs.h>
+#include <linux/timer.h>
+#include <linux/timex.h>
+#include <linux/rtc.h>
+
+#include "../spacemit_lib.h"
+#include "../spacemit_crtc.h"
+#include "../spacemit_mipi_panel.h"
+#include "sysfs_display.h"
+
+#ifdef CONFIG_PM
+static ssize_t spacemit_dpu_get_lpm_period(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "lpm peroid = %d\n", a_crtc->lpm_period);
+}
+
+static ssize_t spacemit_dpu_set_lpm_period(struct device *dev,
+					   struct device_attribute *attr,
+					   const char *buf,
+					   size_t count)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+	int period = 0;
+	int ret = 0;
+
+	ret = sscanf(buf, "%d\n", &period);
+	if ((ret != 1) || (period < SPACEMIT_DPU_LPM_PERIOD_MIN_MS)
+		|| (period > SPACEMIT_DPU_LPM_PERIOD_MAX_MS)) {
+		pr_err("Wrong parameter! Please echo peroid in[%d, %d]\n",
+			SPACEMIT_DPU_LPM_PERIOD_MIN_MS, SPACEMIT_DPU_LPM_PERIOD_MAX_MS);
+		return -EINVAL;
+	}
+
+	a_crtc->lpm_period = period;
+
+	return count;
+}
+
+static ssize_t spacemit_dpu_get_enable_auto_fc(struct device *dev,
+					       struct device_attribute *attr,
+					       char *buf)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "enable_auto_fc = %d %d\n", a_crtc->enable_auto_fc, a_crtc->dev->power.runtime_status);
+}
+
+static ssize_t spacemit_dpu_set_enable_auto_fc(struct device *dev,
+					       struct device_attribute *attr,
+					       const char *buf,
+					       size_t count)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+	int enable_auto_fc = 0;
+	int ret = 0;
+
+	if (a_crtc->dev->power.runtime_status != RPM_SUSPENDED) {
+		pr_err("set dpu_enable_auto_fc only support when screen off!\n");
+		return -EINVAL;
+	}
+
+	ret = sscanf(buf, "%d\n", &enable_auto_fc);
+	if ((ret != 1) || (enable_auto_fc < 0) || (enable_auto_fc > 1)) {
+		pr_err("Wrong parameter! Please echo 0 or 1\n");
+		return -EINVAL;
+	}
+
+	if (enable_auto_fc == 0) {
+		a_crtc->new_mclk = DPU_MCLK_DEFAULT;
+		if (a_crtc->core && a_crtc->core->update_clk)
+			a_crtc->core->update_clk(a_crtc, a_crtc->new_mclk);
+	}
+
+	a_crtc->enable_auto_fc = enable_auto_fc;
+
+	return count;
+}
+#endif
+
+static ssize_t spacemit_dpu_get_enable_dump_reg(struct device *dev,
+						struct device_attribute *attr,
+						char *buf)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "enable_dump_reg = %d\n", a_crtc->enable_dump_reg);
+}
+
+static ssize_t spacemit_dpu_set_enable_dump_reg(struct device *dev,
+						struct device_attribute *attr,
+						const char *buf,
+						size_t count)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+	int enable_dump_reg = 0;
+	int ret = 0;
+
+	ret = sscanf(buf, "%d\n", &enable_dump_reg);
+	if ((ret != 1) || (enable_dump_reg < 0) || (enable_dump_reg > 1)) {
+		pr_err("Wrong parameter! Please echo 0 or 1\n");
+		return -EINVAL;
+	}
+
+	a_crtc->enable_dump_reg = enable_dump_reg;
+
+	return count;
+}
+
+static ssize_t spacemit_dpu_get_enable_dump_fps(struct device *dev,
+						struct device_attribute *attr,
+						char *buf)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+
+	return snprintf(buf, PAGE_SIZE, "enable_dump_fps = %d\n", a_crtc->enable_dump_fps);
+}
+
+static ssize_t spacemit_dpu_set_enable_dump_fps(struct device *dev,
+						struct device_attribute *attr,
+						const char *buf,
+						size_t count)
+{
+	struct spacemit_crtc *a_crtc = dev_get_drvdata(dev);
+	int enable_dump_fps = 0;
+	int ret = 0;
+
+	ret = sscanf(buf, "%d\n", &enable_dump_fps);
+	if ((ret != 1) || (enable_dump_fps < 0) || (enable_dump_fps > 1)) {
+		pr_err("Wrong parameter! Please echo 0 or 1\n");
+		return -EINVAL;
+	}
+
+	a_crtc->enable_dump_fps = enable_dump_fps;
+
+	return count;
+}
+
+static DEVICE_ATTR(dpu_enable_dump_fps, S_IRUGO | S_IWUSR, spacemit_dpu_get_enable_dump_fps, spacemit_dpu_set_enable_dump_fps);
+static DEVICE_ATTR(dpu_enable_dump_reg, S_IRUGO | S_IWUSR, spacemit_dpu_get_enable_dump_reg, spacemit_dpu_set_enable_dump_reg);
+#ifdef CONFIG_PM
+static DEVICE_ATTR(dpu_enable_auto_fc, S_IRUGO | S_IWUSR, spacemit_dpu_get_enable_auto_fc, spacemit_dpu_set_enable_auto_fc);
+static DEVICE_ATTR(dpu_set_lpm_period, S_IRUGO | S_IWUSR, spacemit_dpu_get_lpm_period, spacemit_dpu_set_lpm_period);
+#endif
+
+int spacemit_dpu_sysfs_init(struct device *dev)
+{
+
+	int ret = 0;
+
+	ret = device_create_file(dev, &dev_attr_dpu_enable_dump_reg);
+	if (ret)
+		DRM_ERROR("failed to create device file: enable_dump_reg\n");
+	else
+		DRM_INFO("create device file enable_dump_reg\n");
+
+	ret = device_create_file(dev, &dev_attr_dpu_enable_dump_fps);
+	if (ret)
+		DRM_ERROR("failed to create device file: enable_dump_fps\n");
+	else
+		DRM_INFO("create device file enable_dump_fps\n");
+#ifdef CONFIG_PM
+	ret = device_create_file(dev, &dev_attr_dpu_enable_auto_fc);
+	if (ret)
+		DRM_ERROR("failed to create device file: enable_auto_fc\n");
+	else
+		DRM_INFO("create device file enable_auto_fc\n");
+
+	ret = device_create_file(dev, &dev_attr_dpu_set_lpm_period);
+	if (ret)
+		DRM_ERROR("failed to create device file: dpu_set_lpm_period\n");
+	else
+		DRM_INFO("create device file dpu_set_lpm_period\n");
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(spacemit_dpu_sysfs_init);
+
+MODULE_DESCRIPTION("Add dpu attribute nodes for userspace");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/spacemit/sysfs/sysfs_dsi.c b/drivers/gpu/drm/spacemit/sysfs/sysfs_dsi.c
new file mode 100644
index 000000000000..eed870bcc04a
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/sysfs/sysfs_dsi.c
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sysfs.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include "../spacemit_mipi_panel.h"
+#include "../spacemit_dsi.h"
+#include "../spacemit_crtc.h"
+#include "sysfs_display.h"
+
+
+int spacemit_dsi_sysfs_init(struct device *dev)
+{
+
+	return 0;
+}
+EXPORT_SYMBOL(spacemit_dsi_sysfs_init);
+
+MODULE_DESCRIPTION("Add dsi attribute nodes for userspace");
+MODULE_LICENSE("GPL v2");
+
diff --git a/drivers/gpu/drm/spacemit/sysfs/sysfs_mipi_panel.c b/drivers/gpu/drm/spacemit/sysfs/sysfs_mipi_panel.c
new file mode 100644
index 000000000000..04241d0d71e3
--- /dev/null
+++ b/drivers/gpu/drm/spacemit/sysfs/sysfs_mipi_panel.c
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Spacemit Co., Ltd.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sysfs.h>
+#include <linux/pm_runtime.h>
+#include <linux/platform_device.h>
+#include <video/videomode.h>
+
+#include "../spacemit_lib.h"
+#include "../spacemit_mipi_panel.h"
+#include "sysfs_display.h"
+
+
+int spacemit_mipi_panel_sysfs_init(struct device *dev)
+{
+
+	return 0;
+}
+EXPORT_SYMBOL(spacemit_mipi_panel_sysfs_init);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Add panel attribute nodes for userspace");
+
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index acbbca9d5c92..2a8777d1f2b8 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -307,6 +307,14 @@ pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res,
 		caching = res->bus.caching;
 	}
 
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
+	/* Downgrade cached mapping for non-snooping devices */
+	if (!bo->bdev->dma_coherent && caching == ttm_cached)
+		caching = ttm_write_combined;
+#endif
+
 	return ttm_prot_from_caching(caching, tmp);
 }
 EXPORT_SYMBOL(ttm_io_prot);
@@ -360,6 +368,11 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 		return ret;
 
 	if (num_pages == 1 && ttm->caching == ttm_cached &&
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
+	    bo->bdev->dma_coherent &&
+#endif
 	    !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) {
 		/*
 		 * We're mapping a single page, and the desired
diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
index c3e2fcbdd2cc..1758d94b3d44 100644
--- a/drivers/gpu/drm/ttm/ttm_device.c
+++ b/drivers/gpu/drm/ttm/ttm_device.c
@@ -246,6 +246,12 @@ int ttm_device_init(struct ttm_device *bdev, const struct ttm_device_funcs *func
 	list_add_tail(&bdev->device_list, &glob->device_list);
 	mutex_unlock(&ttm_global_mutex);
 
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
+	bdev->dma_coherent = dev->dma_coherent;
+#endif
+
 	return 0;
 }
 EXPORT_SYMBOL(ttm_device_init);
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
index b3fffe7b5062..aa137ead5cc5 100644
--- a/drivers/gpu/drm/ttm/ttm_module.c
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -74,7 +74,8 @@ pgprot_t ttm_prot_from_caching(enum ttm_caching caching, pgprot_t tmp)
 #endif /* CONFIG_UML */
 #endif /* __i386__ || __x86_64__ */
 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
-	defined(__powerpc__) || defined(__mips__) || defined(__loongarch__)
+	defined(__powerpc__) || defined(__mips__) || defined(__loongarch__) || \
+	defined(__riscv)
 	if (caching == ttm_write_combined)
 		tmp = pgprot_writecombine(tmp);
 	else
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index e2c82ad07eb4..8058e93dac65 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -827,8 +827,17 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 			     struct ttm_device *bdev,
 			     struct ttm_resource *mem)
 {
+	enum ttm_caching caching = mem->bus.caching;
 	int ret;
 
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
+	/* Downgrade cached mapping for non-snooping devices */
+	if (!bdev->dma_coherent && caching == ttm_cached)
+		caching = ttm_write_combined;
+#endif
+
 	ret = ttm_mem_io_reserve(bdev, mem);
 	if (ret)
 		goto out_err;
@@ -843,11 +852,11 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 	} else {
 		iter_io->needs_unmap = true;
 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
-		if (mem->bus.caching == ttm_write_combined)
+		if (caching == ttm_write_combined)
 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
 						  ioremap_wc(mem->bus.offset,
 							     mem->size));
-		else if (mem->bus.caching == ttm_cached)
+		else if (caching == ttm_cached)
 			iosys_map_set_vaddr(&iter_io->dmap,
 					    memremap(mem->bus.offset, mem->size,
 						     MEMREMAP_WB |
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 506e257dfba8..8338deb3330f 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -154,6 +154,14 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm,
 			       enum ttm_caching caching,
 			       unsigned long extra_pages)
 {
+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
+	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
+	/* Downgrade cached mapping for non-snooping devices */
+	if (!bo->bdev->dma_coherent && caching == ttm_cached)
+		caching = ttm_write_combined;
+#endif
+
 	ttm->num_pages = (PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT) + extra_pages;
 	ttm->page_flags = page_flags;
 	ttm->dma_address = NULL;
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 2760feb9f83b..c75b7516ffd9 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -537,6 +537,17 @@ config SENSORS_CROS_EC
 	  This driver can also be built as a module. If so, the module
 	  will be called cros_ec_hwmon.
 
+config SENSORS_CTF2301
+	tristate "Sensylink CTF2301"
+	depends on I2C
+	select REGMAP
+	help
+	  If you say yes here you get support for Sensylink CTF2301
+	  sensor chip.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called ctf2301.
+
 config SENSORS_DRIVETEMP
 	tristate "Hard disk drives with temperature sensors"
 	depends on SCSI && ATA
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 73b2abdcc6dd..ff000f5931ee 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -65,6 +65,7 @@ obj-$(CONFIG_SENSORS_CORETEMP)	+= coretemp.o
 obj-$(CONFIG_SENSORS_CORSAIR_CPRO) += corsair-cpro.o
 obj-$(CONFIG_SENSORS_CORSAIR_PSU) += corsair-psu.o
 obj-$(CONFIG_SENSORS_CROS_EC)	+= cros_ec_hwmon.o
+obj-$(CONFIG_SENSORS_CTF2301)	+= ctf2301.o
 obj-$(CONFIG_SENSORS_DA9052_ADC)+= da9052-hwmon.o
 obj-$(CONFIG_SENSORS_DA9055)+= da9055-hwmon.o
 obj-$(CONFIG_SENSORS_DELL_SMM)	+= dell-smm-hwmon.o
diff --git a/drivers/hwmon/ctf2301.c b/drivers/hwmon/ctf2301.c
new file mode 100644
index 000000000000..04414ea39fd1
--- /dev/null
+++ b/drivers/hwmon/ctf2301.c
@@ -0,0 +1,421 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Driver for CTF2301 system-level thermal management solution chip
+ * Datasheet: https://www.sensylink.com/upload/1/net.sensylink.portal/1689557281035.pdf
+ *
+ * Copyright (C) 2025 Troy Mitchell <troy.mitchell@linux.dev>
+ */
+
+#include <linux/hwmon.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+#include <linux/thermal.h>
+
+#define PWM_PARENT_CLOCK			360000
+
+#define CTF2301_LOCAL_TEMP_MSB			0x00
+#define CTF2301_RMT_TEMP_MSB			0x01
+#define CTF2301_ALERT_STATUS			0x02
+#define CTF2301_GLOBAL_CFG			0x03
+#define CTF2301_RMT_TEMP_LSB			0x10
+#define CTF2301_LOCAL_TEMP_LSB			0x15
+#define	CTF2301_ENHANCED_CFG			0x45
+#define CTF2301_TACH_COUNT_LSB			0x46
+#define CTF2301_TACH_COUNT_MSB			0x47
+#define CTF2301_PWM_AND_TACH_CFG		0x4a
+#define CTF2301_PWM_VALUE			0x4c
+#define CTF2301_PWM_FREQ			0x4d
+#define CTF2301_RMT_DIODE_TEMP_FILTER		0xbf
+
+/* remote diode fault alarm */
+#define ALERT_STATUS_RDFA			BIT(2)
+
+/* alert interrupts enable  */
+#define GLOBAL_CFG_ALERT_MASK			BIT(7)
+/* tach input enable  */
+#define GLOBAL_CFG_TACH_SEL			BIT(2)
+
+/* enables signed format for high and t_crit setpoints */
+#define ENHANGCED_CFG_USF			BIT(3)
+
+/* PWM Programming enable */
+#define PWM_AND_TACH_CFG_PWPGM			BIT(5)
+
+#define PWM_DEFAULT_FREQ_CODE			0x17
+
+
+struct ctf2301 {
+	struct i2c_client *client;
+
+	struct regmap *regmap;
+	struct thermal_cooling_device *cdev;
+
+	unsigned int pwm_freq_code;
+	bool temp_signed;
+};
+
+static int ctf2301_read_temp(struct device *dev, u32 attr, int channel, long *val)
+{
+	int regval[2], raw, err, flag = 1, shift = 4, scale = 625;
+	struct ctf2301 *ctf2301 = dev_get_drvdata(dev);
+	unsigned int reg_msb = CTF2301_LOCAL_TEMP_MSB,
+		     reg_lsb = CTF2301_LOCAL_TEMP_LSB;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		if (channel != 0 && channel != 1)
+			return -EOPNOTSUPP;
+
+		if (channel == 1) {
+			err = regmap_read(ctf2301->regmap, CTF2301_ALERT_STATUS, regval);
+			if (err)
+				return err;
+
+			if (regval[0] & ALERT_STATUS_RDFA)
+				return -ENODEV;
+
+			shift = 5;
+			scale = 1250;
+			reg_msb = CTF2301_RMT_TEMP_MSB;
+			reg_lsb = CTF2301_RMT_TEMP_LSB;
+		}
+
+		err = regmap_read(ctf2301->regmap, reg_msb, regval);
+		if (err)
+			return err;
+
+		err = regmap_read(ctf2301->regmap, reg_lsb, regval + 1);
+		if (err)
+			return err;
+
+		raw = (s16)((regval[0] << 8) | regval[1]);
+
+		raw >>= shift;
+
+		*val = raw * scale * flag;
+
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int ctf2301_read_fan(struct device *dev, u32 attr, long *val)
+{
+	struct ctf2301 *ctf2301 = dev_get_drvdata(dev);
+	int regval[2], err, speed;
+
+	switch (attr) {
+	case hwmon_fan_input:
+		err = regmap_read(ctf2301->regmap, CTF2301_TACH_COUNT_MSB, regval);
+		if (err)
+			return err;
+
+		err = regmap_read(ctf2301->regmap, CTF2301_TACH_COUNT_LSB, regval + 1);
+		if (err)
+			return err;
+
+		speed = (regval[0] << 8) | regval[1];
+
+		*val = speed == 0 ? 0 : (unsigned int)(1 * (5400000 / speed));
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int ctf2301_update_pwm(struct ctf2301 *data, long val)
+{
+	int map_val;
+
+	val = clamp_val(val, 0, 255);
+
+	map_val = (val * data->pwm_freq_code * 2) / 255;
+
+	return regmap_write(data->regmap, CTF2301_PWM_VALUE, map_val);
+}
+
+static int ctf2301_write_pwm(struct device *dev, u32 attr, long val)
+{
+	struct ctf2301 *ctf2301 = dev_get_drvdata(dev);
+	int err;
+
+	switch (attr) {
+	case hwmon_pwm_input:
+		return ctf2301_update_pwm(ctf2301, val);
+	case hwmon_pwm_freq:
+		ctf2301->pwm_freq_code = DIV_ROUND_UP(PWM_PARENT_CLOCK, val) / 2;
+
+		err = regmap_write(ctf2301->regmap, CTF2301_PWM_FREQ, ctf2301->pwm_freq_code);
+		if (err)
+			return err;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int ctf2301_read_pwm(struct device *dev, u32 attr, long *val)
+{
+	struct ctf2301 *ctf2301 = dev_get_drvdata(dev);
+	int err;
+	unsigned int reg_val;
+	unsigned int freq_code;
+
+	switch (attr) {
+	case hwmon_pwm_input:
+		err = regmap_read(ctf2301->regmap, CTF2301_PWM_VALUE, &reg_val);
+		if (err)
+			return err;
+
+		freq_code = ctf2301->pwm_freq_code;
+
+		if (freq_code == 0)
+			freq_code = 1;
+
+		*val = (reg_val * 255) / (freq_code * 2);
+
+		if (*val > 255)
+			*val = 255;
+		break;
+
+	case hwmon_pwm_freq:
+		err = regmap_read(ctf2301->regmap, CTF2301_PWM_FREQ, &reg_val);
+		if (err)
+			return err;
+
+		if (reg_val == 0)
+			reg_val = 1;
+
+		*val = PWM_PARENT_CLOCK / (2 * reg_val);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static umode_t ctf2301_is_visible(const void *drvdata,
+				 enum hwmon_sensor_types type,
+				 u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			return 0444;
+		default:
+			return 0;
+		}
+	case hwmon_fan:
+		switch (attr) {
+		case hwmon_fan_input:
+			return 0444;
+		default:
+			return 0;
+		}
+	case hwmon_pwm:
+		switch (attr) {
+		case hwmon_pwm_input:
+		case hwmon_pwm_freq:
+			return 0644;
+		default:
+			return 0;
+		}
+	default:
+		return 0;
+	}
+}
+
+static int ctf2301_read(struct device *dev, enum hwmon_sensor_types type,
+		       u32 attr, int channel, long *val)
+{
+	switch (type) {
+	case hwmon_temp:
+		return ctf2301_read_temp(dev, attr, channel, val);
+	case hwmon_fan:
+		return ctf2301_read_fan(dev, attr, val);
+	case hwmon_pwm:
+		return ctf2301_read_pwm(dev, attr, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
+
+static int ctf2301_write(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long val)
+{
+	switch (type) {
+	case hwmon_pwm:
+		return ctf2301_write_pwm(dev, attr, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
+
+static int ctf2301_cdev_get_max_state(struct thermal_cooling_device *cdev,
+				 unsigned long *state)
+{
+	*state = 255;
+	return 0;
+}
+
+static int ctf2301_cdev_get_cur_state(struct thermal_cooling_device *cdev,
+                                 unsigned long *state)
+{
+	struct ctf2301 *data = cdev->devdata;
+	unsigned int reg_val;
+	long val;
+	int err;
+
+	err = regmap_read(data->regmap, CTF2301_PWM_VALUE, &reg_val);
+	if (err)
+		return err;
+
+	val = (reg_val * 255) / (data->pwm_freq_code * 2);
+
+	*state = clamp_val(val, 0, 255);
+
+	return 0;
+}
+
+static int ctf2301_cdev_set_cur_state(struct thermal_cooling_device *cdev,
+                                 unsigned long state)
+{
+	struct ctf2301 *data = cdev->devdata;
+
+	return ctf2301_update_pwm(data, state);
+}
+
+static const struct thermal_cooling_device_ops ctf2301_cooling_ops = {
+	.get_max_state = ctf2301_cdev_get_max_state,
+	.get_cur_state = ctf2301_cdev_get_cur_state,
+	.set_cur_state = ctf2301_cdev_set_cur_state,
+};
+
+static const struct hwmon_channel_info * const ctf2301_info[] = {
+	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT, HWMON_T_INPUT),
+	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT | HWMON_PWM_FREQ),
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT),
+	NULL
+};
+
+static const struct hwmon_ops ctf2301_hwmon_ops = {
+	.is_visible = ctf2301_is_visible,
+	.read = ctf2301_read,
+	.write = ctf2301_write
+};
+
+static const struct hwmon_chip_info ctf2301_chip_info = {
+	.ops = &ctf2301_hwmon_ops,
+	.info = ctf2301_info,
+};
+
+static const struct regmap_config ctf2301_regmap_config = {
+	.max_register = CTF2301_RMT_DIODE_TEMP_FILTER,
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+static void ctf2301_parse_dt(struct device_node *np, struct ctf2301 *ctf2301)
+{
+	int ret;
+	u32 pwm_freq;
+
+	ret = of_property_read_u32(np, "sensylink,pwm-freq", &pwm_freq);
+	if (ret)
+		ctf2301->pwm_freq_code = PWM_DEFAULT_FREQ_CODE;
+	else
+		ctf2301->pwm_freq_code = DIV_ROUND_UP(PWM_PARENT_CLOCK, pwm_freq) / 2;
+
+	ctf2301->temp_signed = of_property_read_bool(np, "sensylink,temp-signed");
+}
+
+static int ctf2301_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct device *hwmon_dev;
+	struct ctf2301 *ctf2301;
+	int err;
+
+	ctf2301 = devm_kzalloc(dev, sizeof(*ctf2301), GFP_KERNEL);
+	if (!ctf2301)
+		return -ENOMEM;
+
+	ctf2301->client = client;
+
+	ctf2301_parse_dt(client->dev.of_node, ctf2301);
+
+	ctf2301->regmap = devm_regmap_init_i2c(client, &ctf2301_regmap_config);
+	if (IS_ERR(ctf2301->regmap))
+		return dev_err_probe(dev, PTR_ERR(ctf2301->regmap),
+				     "failed to allocate register map");
+
+	err = regmap_write(ctf2301->regmap, CTF2301_GLOBAL_CFG,
+			   GLOBAL_CFG_ALERT_MASK | GLOBAL_CFG_TACH_SEL);
+	if (err)
+		return dev_err_probe(dev, err,
+				     "failed to write CTF2301_GLOBAL_CFG register");
+
+	err = regmap_write(ctf2301->regmap, CTF2301_ENHANCED_CFG, ENHANGCED_CFG_USF);
+	if (err)
+		return dev_err_probe(dev, err,
+				     "failed to write CTF2301_ENHANCED_CFG");
+
+	err = regmap_write(ctf2301->regmap, CTF2301_PWM_AND_TACH_CFG, PWM_AND_TACH_CFG_PWPGM);
+	if (err)
+		return dev_err_probe(dev, err,
+				     "failed to write CTF2301_PWM_AND_TACH_CFG");
+
+	/* default to enable the fan */
+	ctf2301_update_pwm(ctf2301, 255);
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, ctf2301,
+							 &ctf2301_chip_info,
+							 NULL);
+	if (IS_ERR(hwmon_dev))
+		return dev_err_probe(dev, PTR_ERR(hwmon_dev),
+				     "failed to register hwmon device");
+
+	if (IS_ENABLED(CONFIG_THERMAL)) {
+		ctf2301->cdev = devm_thermal_of_cooling_device_register(dev,
+									dev->of_node,
+									"ctf2301_fan",
+									ctf2301,
+									&ctf2301_cooling_ops);
+		if (IS_ERR(ctf2301->cdev))
+			dev_warn(dev, "failed to register cooling device");
+	}
+
+	return 0;
+}
+
+static const struct of_device_id ctf2301_of_match[] = {
+	{ .compatible = "sensylink,ctf2301", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ctf2301_of_match);
+
+static struct i2c_driver ctf2301_driver = {
+	.driver = {
+		.name	= "ctf2301",
+		.of_match_table = of_match_ptr(ctf2301_of_match),
+	},
+	.probe		= ctf2301_probe,
+};
+module_i2c_driver(ctf2301_driver);
+
+MODULE_AUTHOR("Troy Mitchell <troy.mitchell@linux.dev>");
+MODULE_DESCRIPTION("ctf2301 driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwspinlock/Kconfig b/drivers/hwspinlock/Kconfig
index 3874d15b0e9b..68f31b9f2432 100644
--- a/drivers/hwspinlock/Kconfig
+++ b/drivers/hwspinlock/Kconfig
@@ -63,4 +63,10 @@ config HSEM_U8500
 
 	  If unsure, say N.
 
+config HWSPINLOCK_SPACEMIT
+	tristate "Spacemit Hardware Spinlock device"
+	depends on SOC_SPACEMIT_K3
+	help
+	  Say y here to support the Spacemit Hardware Spinlock device.
+
 endif # HWSPINLOCK
diff --git a/drivers/hwspinlock/Makefile b/drivers/hwspinlock/Makefile
index a0f16c9aaa82..cc13b104140e 100644
--- a/drivers/hwspinlock/Makefile
+++ b/drivers/hwspinlock/Makefile
@@ -10,3 +10,4 @@ obj-$(CONFIG_HWSPINLOCK_SPRD)		+= sprd_hwspinlock.o
 obj-$(CONFIG_HWSPINLOCK_STM32)		+= stm32_hwspinlock.o
 obj-$(CONFIG_HWSPINLOCK_SUN6I)		+= sun6i_hwspinlock.o
 obj-$(CONFIG_HSEM_U8500)		+= u8500_hsem.o
+obj-$(CONFIG_HWSPINLOCK_SPACEMIT)	+= spacemit_hwspinlock.o
diff --git a/drivers/hwspinlock/spacemit_hwspinlock.c b/drivers/hwspinlock/spacemit_hwspinlock.c
new file mode 100644
index 000000000000..508f49d6f3bf
--- /dev/null
+++ b/drivers/hwspinlock/spacemit_hwspinlock.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * spacemit hwspinlock driver
+ *
+ * Copyright (C) 2025 Spacemit
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/debugfs.h>
+#include <linux/errno.h>
+#include <linux/hwspinlock.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+#include <linux/delay.h>
+#include "hwspinlock_internal.h"
+
+#define SPINLOCK_LOCK(x)	(0x0 + 0x4 * x)
+#define SPINLOCK_VER		0x100
+#define SPINLOCK_SSTAT		0x104
+#define SPINLOCK_STATUS		0x108
+#define SPINLOCK_IRQ_EN		0x110
+#define SPINLOCK_IRQ_STA	0x114
+#define SPINLOCK_NOTTAKEN	0
+#define SPINLOCK_BASE_ID	0
+
+struct spacemit_hwspinlock_data {
+	struct hwspinlock_device *bank;
+	void __iomem *base;
+	int num;
+};
+
+static int spacemit_hwspinlock_trylock(struct hwspinlock *lock)
+{
+	void __iomem *reg = lock->priv;
+
+	return (readl(reg) == SPINLOCK_NOTTAKEN);
+}
+
+static void spacemit_hwspinlock_unlock(struct hwspinlock *lock)
+{
+	void __iomem *reg = lock->priv;
+
+	writel(SPINLOCK_NOTTAKEN, reg);
+}
+
+static const struct hwspinlock_ops spacemit_hwspinlock_ops = {
+	.trylock	= spacemit_hwspinlock_trylock,
+	.unlock		= spacemit_hwspinlock_unlock,
+};
+
+static void spacemit_hwspinlock_disable(void *data)
+{
+	return;
+}
+
+static int spacemit_hwspinlock_probe(struct platform_device *pdev)
+{
+	struct spacemit_hwspinlock_data *data;
+	struct hwspinlock *lock;
+	int err, i;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->base = devm_platform_ioremap_resource(pdev, SPINLOCK_BASE_ID);
+	if (IS_ERR(data->base))
+		return PTR_ERR(data->base);
+
+	data->num = readl(data->base + SPINLOCK_SSTAT);
+	data->bank = devm_kzalloc(&pdev->dev, struct_size(data->bank, lock, data->num),
+				GFP_KERNEL);
+	if (!data->bank) {
+		err = -ENOMEM;
+		goto bank_fail;
+	}
+
+	for (i = 0; i < data->num; i++) {
+		lock = &data->bank->lock[i];
+		lock->priv = data->base + SPINLOCK_LOCK(i);
+	}
+
+	err = devm_add_action_or_reset(&pdev->dev, spacemit_hwspinlock_disable, data);
+	if (err) {
+		dev_err(&pdev->dev, "failed to add hwspinlock disable action\n");
+		goto bank_fail;
+	}
+
+	platform_set_drvdata(pdev, data);
+
+	return devm_hwspin_lock_register(&pdev->dev, data->bank, &spacemit_hwspinlock_ops,
+					 SPINLOCK_BASE_ID, data->num);
+
+bank_fail:
+	return err;
+}
+
+static const struct of_device_id spacemit_hwspinlock_ids[] = {
+	{ .compatible = "spacemit,hwspinlock", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, spacemit_hwspinlock_ids);
+
+static struct platform_driver spacemit_hwspinlock_driver = {
+	.probe = spacemit_hwspinlock_probe,
+	.driver = {
+		.name		= "spacemit_hwspinlock",
+		.of_match_table = spacemit_hwspinlock_ids,
+	},
+};
+module_platform_driver(spacemit_hwspinlock_driver);
diff --git a/drivers/hwtracing/Kconfig b/drivers/hwtracing/Kconfig
index 911ee977103c..167ff172dd72 100644
--- a/drivers/hwtracing/Kconfig
+++ b/drivers/hwtracing/Kconfig
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menu "HW tracing support"
 
+source "drivers/hwtracing/coresight/Kconfig"
+
 source "drivers/hwtracing/stm/Kconfig"
 
 source "drivers/hwtracing/intel_th/Kconfig"
diff --git a/drivers/hwtracing/coresight/Kconfig b/drivers/hwtracing/coresight/Kconfig
index 6a4239ebb582..850de905fdb4 100644
--- a/drivers/hwtracing/coresight/Kconfig
+++ b/drivers/hwtracing/coresight/Kconfig
@@ -4,7 +4,7 @@
 #
 menuconfig CORESIGHT
 	tristate "CoreSight Tracing Support"
-	depends on ARM || ARM64
+	depends on ARM || ARM64 || RISCV
 	depends on OF || ACPI
 	select ARM_AMBA
 	select PERF_EVENTS
@@ -280,4 +280,51 @@ config CORESIGHT_TNOC
 	  To compile this driver as a module, choose M here: the module will be
 	  called coresight-tnoc.
 
+config RVTRACE
+	tristate "RISC-V Trace Support"
+	help
+	  This enables support for the RISC-V trace drivers. drivers
+	  (including Trace Encoder, Trace Funnel and  ATB Bridge) are
+	  dynamically aggregated with CoreSight trace infrastructure
+	  at run time to form a complete trace path.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called rvtrace.
+
+config RVTRACE_ATBBRIDGE
+	tristate "RISCV Trace ATB Bridge driver"
+	depends on RVTRACE
+	help
+	  This driver provides support for the ATB Bridge driver. The ATB
+	  Bridge allows sending RISC-V trace to Arm CoreSight infrastructure
+	  (instead of RISC-V compliant sink) as an ATB initiator.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called rvtrace-atbbridge.
+
+config RVTRACE_FUNNEL
+	tristate "RISCV Trace Funnel driver"
+	depends on RVTRACE
+	help
+	  his driver provides support for the Trace Funnel driver. The Funnel
+	  aggregates the trace from each of its inputs and sends the combined
+	  trace stream to its designated Trace Sink or ATB Bridge.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called rvtrace-funnel.
+
+config RVTRACE_ENCODER
+	tristate "RISCV Trace Encoder driver"
+	depends on RVTRACE
+	select RVTRACE_FUNNEL
+	select RVTRACE_ATBBRIDGE
+	help
+	  This driver provides support for the Trace Encoder module, tracing
+	  the instructions that a processor is executing. This is primarily
+	  useful for instruction level tracing. Depending on the implemented
+	  version data tracing may also be available.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called rvtrace-encoder.
+
 endif
diff --git a/drivers/hwtracing/coresight/Makefile b/drivers/hwtracing/coresight/Makefile
index ab16d06783a5..fbfa8432eb37 100644
--- a/drivers/hwtracing/coresight/Makefile
+++ b/drivers/hwtracing/coresight/Makefile
@@ -57,3 +57,9 @@ obj-$(CONFIG_CORESIGHT_DUMMY) += coresight-dummy.o
 obj-$(CONFIG_CORESIGHT_CTCU) += coresight-ctcu.o
 coresight-ctcu-y := coresight-ctcu-core.o
 obj-$(CONFIG_CORESIGHT_KUNIT_TESTS) += coresight-kunit-tests.o
+obj-$(CONFIG_RVTRACE) += rvtrace.o
+rvtrace-y := rvtrace-core.o rvtrace-timestamp.o
+obj-$(CONFIG_RVTRACE_FUNNEL) += rvtrace-funnel.o
+obj-$(CONFIG_RVTRACE_ATBBRIDGE) += rvtrace-atbbridge.o
+obj-$(CONFIG_RVTRACE_ENCODER) += rvtrace-encoder.o
+rvtrace-encoder-y := rvtrace-encoder-core.o rvtrace-encoder-sysfs.o
diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c
index 3267192f0c1c..8c8370461f2f 100644
--- a/drivers/hwtracing/coresight/coresight-core.c
+++ b/drivers/hwtracing/coresight/coresight-core.c
@@ -141,7 +141,11 @@ static void coresight_set_self_claim_tag_unlocked(struct coresight_device *csdev
 {
 	csdev_access_relaxed_write32(&csdev->access, CORESIGHT_CLAIM_SELF_HOSTED,
 				     CORESIGHT_CLAIMSET);
+#if defined(__riscv)
+	local_flush_icache_all();
+#else
 	isb();
+#endif
 }
 
 void coresight_clear_self_claim_tag(struct csdev_access *csa)
@@ -158,7 +162,11 @@ void coresight_clear_self_claim_tag_unlocked(struct csdev_access *csa)
 {
 	csdev_access_relaxed_write32(csa, CORESIGHT_CLAIM_SELF_HOSTED,
 				     CORESIGHT_CLAIMCLR);
+#if defined(__riscv)
+	local_flush_icache_all();
+#else
 	isb();
+#endif
 }
 EXPORT_SYMBOL_GPL(coresight_clear_self_claim_tag_unlocked);
 
diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c
index 0db64c5f4995..261ba6a75b86 100644
--- a/drivers/hwtracing/coresight/coresight-platform.c
+++ b/drivers/hwtracing/coresight/coresight-platform.c
@@ -14,7 +14,6 @@
 #include <linux/amba/bus.h>
 #include <linux/coresight.h>
 #include <linux/cpumask.h>
-#include <asm/smp_plat.h>
 
 #include "coresight-priv.h"
 
diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c
index 36599c431be6..623d93fe74e0 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-core.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-core.c
@@ -32,6 +32,11 @@
 #include "coresight-priv.h"
 #include "coresight-tmc.h"
 
+static int sysfs_etb_activated, sysfs_etf_activated, sysfs_etr_activated;
+module_param_named(sysfs_etb_activated, sysfs_etb_activated, int, S_IRUGO);
+module_param_named(sysfs_etf_activated, sysfs_etf_activated, int, S_IRUGO);
+module_param_named(sysfs_etr_activated, sysfs_etr_activated, int, S_IRUGO);
+
 DEFINE_CORESIGHT_DEVLIST(etb_devs, "tmc_etb");
 DEFINE_CORESIGHT_DEVLIST(etf_devs, "tmc_etf");
 DEFINE_CORESIGHT_DEVLIST(etr_devs, "tmc_etr");
@@ -876,6 +881,11 @@ static int __tmc_probe(struct device *dev, struct resource *res)
 		goto out;
 	}
 
+	if ((drvdata->config_type == TMC_CONFIG_TYPE_ETB && sysfs_etb_activated) ||
+		(drvdata->config_type == TMC_CONFIG_TYPE_ETR && sysfs_etr_activated) ||
+		(drvdata->config_type == TMC_CONFIG_TYPE_ETF && sysfs_etf_activated))
+		drvdata->csdev->sysfs_sink_activated = true;
+
 	drvdata->miscdev.name = desc.name;
 	drvdata->miscdev.minor = MISC_DYNAMIC_MINOR;
 	drvdata->miscdev.fops = &tmc_fops;
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index 0f45ab5e5249..82fe22344560 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -659,7 +659,11 @@ static int tmc_panic_sync_etf(struct coresight_device *csdev)
 	 * Make sure all previous writes are ordered,
 	 * before we mark valid
 	 */
+#if defined(__riscv)
+	__mb();
+#else
 	dmb(sy);
+#endif
 	mdata->valid = true;
 	/*
 	 * Below order need to maintained, since crc of metadata
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 60b0e0a6da05..ebe4d074e783 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -1882,7 +1882,11 @@ static int tmc_panic_sync_etr(struct coresight_device *csdev)
 	 * Make sure all previous writes are ordered,
 	 * before we mark valid
 	 */
+#if defined(__riscv)
+	__mb();
+#else
 	dmb(sy);
+#endif
 	mdata->valid = true;
 	/*
 	 * Below order need to maintained, since crc of metadata
diff --git a/drivers/hwtracing/coresight/rvtrace-atbbridge.c b/drivers/hwtracing/coresight/rvtrace-atbbridge.c
new file mode 100644
index 000000000000..81d87c12d9cc
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-atbbridge.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/coresight.h>
+#include <linux/platform_device.h>
+#include <linux/bitfield.h>
+#include <linux/rvtrace.h>
+
+#include "coresight-priv.h"
+#include "coresight-trace-id.h"
+
+#define RVTRACE_ATBBRIDGE_CONTROL_ID_MASK	GENMASK(14, 8)
+/**
+ * struct atbbridge_data - specifics associated to a ATB bridge component
+ * @csdev:      Component vitals needed by the framework.
+ * @spinlock:   Only one at a time pls.
+ * @traceid:    Value of the current ID for this component.
+ */
+struct atbbridge_data {
+	struct coresight_device	*csdev;
+	spinlock_t		spinlock;
+	u8			traceid;
+};
+
+DEFINE_CORESIGHT_DEVLIST(atbbridge_devs, "atbbridge");
+
+static int atbbridge_enable(struct coresight_device *csdev,
+			    struct coresight_connection *in,
+			    struct coresight_connection *out)
+{
+	u32 control;
+	int ret = 0;
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+	struct atbbridge_data *atbbridge_data = rvtrace_component_data(comp);
+	unsigned long flags;
+
+	spin_lock_irqsave(&atbbridge_data->spinlock, flags);
+	if (!comp->was_reset) {
+		ret = rvtrace_component_reset(comp);
+		if (ret)
+			goto done;
+	}
+
+	control = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	control |= FIELD_PREP(RVTRACE_ATBBRIDGE_CONTROL_ID_MASK, atbbridge_data->traceid);
+	writel_relaxed(control, comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+
+	ret = rvtrace_enable_component(comp);
+	if (ret)
+		goto done;
+
+	spin_unlock_irqrestore(&atbbridge_data->spinlock, flags);
+
+	dev_dbg(&csdev->dev, "Trace ATB bridge enabled\n");
+done:
+	return ret;
+}
+
+static void atbbridge_disable(struct coresight_device *csdev,
+			      struct coresight_connection *in,
+			      struct coresight_connection *out)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+	struct atbbridge_data *atbbridge_data = rvtrace_component_data(comp);
+	unsigned long flags;
+
+	spin_lock_irqsave(&atbbridge_data->spinlock, flags);
+
+	if (rvtrace_disable_component(comp))
+		dev_err(&csdev->dev,
+			"timeout while waiting for Trace ATB Bridge to be disabled\n");
+	/*
+	 * if (rvtrace_comp_is_empty(comp))
+	 * 	dev_err(&csdev->dev,
+	 * 		"timeout while waiting for Trace ATB Bridge internal buffers become empty\n");
+	 */
+
+	spin_unlock_irqrestore(&atbbridge_data->spinlock, flags);
+
+	dev_dbg(&csdev->dev, "Trace ATB bridge disabled\n");
+}
+
+static int atbbridge_link_trace_id(struct coresight_device *csdev, __maybe_unused enum cs_mode mode,
+				   __maybe_unused struct coresight_device *sink)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+	struct atbbridge_data *atbbridge_data = rvtrace_component_data(comp);
+
+	return atbbridge_data->traceid;
+}
+
+static const struct coresight_ops_link atbbridge_link_ops = {
+	.enable		= atbbridge_enable,
+	.disable	= atbbridge_disable,
+};
+
+static const struct coresight_ops atbbridge_cs_ops = {
+	.trace_id	= atbbridge_link_trace_id,
+	.link_ops       = &atbbridge_link_ops,
+};
+
+static ssize_t reset_store(struct device *dev,
+			   struct device_attribute *attr,
+			   const char *buf, size_t size)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct atbbridge_data *atbbridge_data = rvtrace_component_data(comp);
+	unsigned long flags;
+
+	if (kstrtoul(buf, 16, &val))
+		return -EINVAL;
+
+	if (val) {
+		if (rvtrace_component_reset(comp)) {
+			comp->was_reset = false;
+			spin_unlock_irqrestore(&atbbridge_data->spinlock, flags);
+			return -EINVAL;
+		}
+	}
+
+	spin_unlock_irqrestore(&atbbridge_data->spinlock, flags);
+
+	return size;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t cpu_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+
+	val = comp->cpu;
+	return scnprintf(buf, PAGE_SIZE, "%#lx\n", val);
+}
+static DEVICE_ATTR_RO(cpu);
+
+static ssize_t traceid_show(struct device *dev,
+			 struct device_attribute *attr, char *buf)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct atbbridge_data *atbbridge_data = rvtrace_component_data(comp);
+
+	val = atbbridge_data->traceid;
+	return scnprintf(buf, PAGE_SIZE, "%#lx\n", val);
+}
+static DEVICE_ATTR_RO(traceid);
+
+static struct attribute *trace_atbbridge_attrs[] = {
+	coresight_simple_reg32(control, RVTRACE_COMPONENT_CTRL_OFFSET),
+	coresight_simple_reg32(impl, RVTRACE_COMPONENT_IMPL_OFFSET),
+        &dev_attr_reset.attr,
+        &dev_attr_cpu.attr,
+	&dev_attr_traceid.attr,
+        NULL,
+};
+ATTRIBUTE_GROUPS(trace_atbbridge);
+
+static int atbbridge_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct coresight_platform_data *pdata;
+	struct atbbridge_data *atbbridge_data;
+	struct rvtrace_component *comp;
+	struct coresight_desc desc = { 0 };
+	int trace_id = 0;
+
+	comp = rvtrace_register_component(pdev);
+	if (IS_ERR(comp))
+		return PTR_ERR(comp);
+
+	atbbridge_data = devm_kzalloc(dev, sizeof(*atbbridge_data), GFP_KERNEL);
+	if (!atbbridge_data)
+		return -ENOMEM;
+
+	spin_lock_init(&atbbridge_data->spinlock);
+
+	pdata = coresight_get_platform_data(dev);
+	if (IS_ERR(pdata))
+		return PTR_ERR(pdata);
+	pdev->dev.platform_data = pdata;
+
+	platform_set_drvdata(pdev, comp);
+
+	desc.name = coresight_alloc_device_name(&atbbridge_devs, dev);
+	desc.access = CSDEV_ACCESS_IOMEM(comp->base);
+        desc.type = CORESIGHT_DEV_TYPE_LINK;
+        desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO;
+	desc.ops = &atbbridge_cs_ops;
+        desc.pdata = pdata;
+        desc.dev = dev;
+        desc.groups = trace_atbbridge_groups;
+        atbbridge_data->csdev = coresight_register(&desc);
+        if (IS_ERR(atbbridge_data->csdev))
+                return PTR_ERR(atbbridge_data->csdev);
+
+	trace_id = coresight_trace_id_get_system_id();
+	if (trace_id < 0)
+		return -EINVAL;
+	atbbridge_data->traceid = (u8)trace_id;
+
+	comp->id.data = atbbridge_data;
+
+	dev_dbg(dev, "Trace ATB Bridge initialized\n");
+
+	return 0;
+}
+
+static void atbbridge_remove(struct platform_device *pdev)
+{
+	struct rvtrace_component *comp = platform_get_drvdata(pdev);
+	struct atbbridge_data *atbbridge_data = rvtrace_component_data(comp);
+
+	coresight_trace_id_put_system_id(atbbridge_data->traceid);
+	coresight_unregister(atbbridge_data->csdev);
+}
+
+static const struct of_device_id atbbridge_match[] = {
+	{.compatible = "riscv,trace-atbbridge"},
+	{},
+};
+
+static struct platform_driver atbbridge_driver = {
+	.probe = atbbridge_probe,
+	.remove = atbbridge_remove,
+	.driver = {
+		.name	= "trace-atbbridge",
+		.of_match_table = atbbridge_match,
+	},
+};
+
+module_platform_driver(atbbridge_driver);
+
+MODULE_DESCRIPTION("RISC-V Trace ATB Bridge driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwtracing/coresight/rvtrace-core.c b/drivers/hwtracing/coresight/rvtrace-core.c
new file mode 100644
index 000000000000..00893b1577c1
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-core.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/percpu.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/rvtrace.h>
+
+int rvtrace_poll_bit(struct rvtrace_component *comp, int offset,
+                     int bit, int bitval)
+{
+	int i = RVTRACE_TIMEOUT_US;
+	u32 val;
+
+	while (i--) {
+		val = readl_relaxed(comp->base + offset);
+		if (((val >> bit) & 0x1) == bitval)
+			break;
+		udelay(1);
+	}
+
+	return (i < 0) ? -ETIMEDOUT : 0;
+}
+EXPORT_SYMBOL_GPL(rvtrace_poll_bit);
+
+int rvtrace_enable_component(struct rvtrace_component *comp)
+{
+	u32 val;
+
+	val = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	val |= BIT(RVTRACE_COMPONENT_CTRL_ENABLE_SHIFT);
+	writel_relaxed(val, comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	return rvtrace_poll_bit(comp, RVTRACE_COMPONENT_CTRL_OFFSET,
+				RVTRACE_COMPONENT_CTRL_ENABLE_SHIFT, 1);
+}
+EXPORT_SYMBOL_GPL(rvtrace_enable_component);
+
+int rvtrace_disable_component(struct rvtrace_component *comp)
+{
+	u32 val;
+
+	val = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	val &= ~BIT(RVTRACE_COMPONENT_CTRL_ENABLE_SHIFT);
+	writel_relaxed(val, comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	return rvtrace_poll_bit(comp, RVTRACE_COMPONENT_CTRL_OFFSET,
+				RVTRACE_COMPONENT_CTRL_ENABLE_SHIFT, 0);
+}
+EXPORT_SYMBOL_GPL(rvtrace_disable_component);
+
+int rvtrace_component_reset(struct rvtrace_component *comp)
+{
+	int ret;
+
+	writel_relaxed(0, comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	ret = rvtrace_poll_bit(comp, RVTRACE_COMPONENT_CTRL_OFFSET,
+			       RVTRACE_COMPONENT_CTRL_ACTIVE_SHIFT, 0);
+	if (ret)
+		return ret;
+
+	writel_relaxed(RVTRACE_COMPONENT_CTRL_ACTIVE_MASK,
+			comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	return rvtrace_poll_bit(comp, RVTRACE_COMPONENT_CTRL_OFFSET,
+				RVTRACE_COMPONENT_CTRL_ACTIVE_SHIFT, 1);
+}
+EXPORT_SYMBOL_GPL(rvtrace_component_reset);
+
+struct rvtrace_component *rvtrace_register_component(struct platform_device *pdev)
+{
+	int ret;
+	void __iomem *base;
+	struct device *dev = &pdev->dev;
+	struct rvtrace_component *comp;
+	struct resource *res;
+	struct device_node *node;
+	u32 impl, type, major, minor;
+
+	comp = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL);
+	if (!comp) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base)) {
+		ret = -EINVAL;
+		goto err_out;
+	}
+	comp->base = base;
+
+	comp->cpu = -1;
+	for (int i = 0; ; i++) {
+		node = of_parse_phandle(dev->of_node, "cpus", i);
+		if (!node)
+			break;
+
+		ret = of_cpu_node_to_id(node);
+		of_node_put(node);
+		if (ret >= 0 && cpu_online(ret)) {
+			comp->cpu = ret;
+			break;
+		}
+	}
+
+	if (comp->cpu < 0) {
+		dev_err(dev, "No valid CPU found in 'cpus' property\n");
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	ret = rvtrace_component_reset(comp);
+	if (ret)
+		goto err_out;
+	comp->was_reset = true;
+
+	impl = readl_relaxed(comp->base + RVTRACE_COMPONENT_IMPL_OFFSET);
+	type = (impl >> RVTRACE_COMPONENT_IMPL_TYPE_SHIFT) &
+		RVTRACE_COMPONENT_IMPL_TYPE_MASK;
+	major = (impl >> RVTRACE_COMPONENT_IMPL_VERMAJOR_SHIFT) &
+		RVTRACE_COMPONENT_IMPL_VERMAJOR_MASK;
+	minor = (impl >> RVTRACE_COMPONENT_IMPL_VERMINOR_SHIFT) &
+		RVTRACE_COMPONENT_IMPL_VERMINOR_MASK;
+
+	comp->id.type = type;
+	comp->id.version = rvtrace_component_mkversion(major, minor);
+
+	return comp;
+
+err_out:
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(rvtrace_register_component);
diff --git a/drivers/hwtracing/coresight/rvtrace-encoder-core.c b/drivers/hwtracing/coresight/rvtrace-encoder-core.c
new file mode 100644
index 000000000000..6975f24a158b
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-encoder-core.c
@@ -0,0 +1,562 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#include <linux/moduleparam.h>
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/err.h>
+#include <linux/bitfield.h>
+#include <linux/smp.h>
+#include <linux/coresight.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/rvtrace.h>
+
+#include "rvtrace-encoder.h"
+#include "rvtrace-timestamp.h"
+#include "coresight-etm-perf.h"
+
+static int boot_enable;
+module_param_named(boot_enable, boot_enable, int, S_IRUGO);
+
+static struct rvtrace_component *rvtrace_cpu_encoder[NR_CPUS];
+
+static int encoder_cpu_id(struct coresight_device *csdev)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+
+	return comp->cpu;
+}
+
+static void encoder_set_config(struct rvtrace_component *comp)
+{
+	u32 val;
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	struct device *dev = &encoder_data->csdev->dev;
+	struct encoder_config *config = &encoder_data->config;
+
+	/* Configure Trace Encoder features register */
+	val = (FIELD_PREP(RVTRACE_ENCODER_INST_NO_ADDR_DIFF, config->inst_na_diff) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_NO_TRAP_ADDR, config->inst_nt_addr) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_EN_SEQUENTIAL_JUMP, config->seq_jump) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_EN_IMPLICIT_RETURN, config->impl_ret) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_EN_BRANCH_PREDICTION, config->branch_pred) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_IMPLICIT_RETURN_MODE, config->impl_ret_mode) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_EN_REPEQTED_HISTORT, config->rep_hist) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_EN_ALL_JUMPS, config->all_jumps) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_EXTEND_ADDR_MSB, config->ext_msb) |
+		   FIELD_PREP(RVTRACE_ENCODER_SRCID, config->srcid) |
+		   FIELD_PREP(RVTRACE_ENCODER_SRCBITS, config->srcb));
+
+	writel_relaxed(val, comp->base + RVTRACE_ENCODER_INST_FTRS_OFFSET);
+
+	/* Check Trace Encoder features register WARL bits could be set */
+	val = readl_relaxed(comp->base + RVTRACE_ENCODER_INST_FTRS_OFFSET);
+	if (BMVAL(val, 0, 0) != config->inst_na_diff) {
+		dev_warn(dev, "trTeInstNoAddrDiff %#x is not supported\n",
+			 config->inst_na_diff);
+		config->inst_na_diff = BMVAL(val, 0, 0);
+	}
+
+	if (BMVAL(val, 1, 1) != config->inst_nt_addr) {
+		dev_warn(dev, "trTeInstNoTrapAddr %#x is not supported\n",
+			 config->inst_nt_addr);
+		config->inst_nt_addr = BMVAL(val, 1, 1);
+	}
+
+	if (BMVAL(val, 2, 2) != config->seq_jump) {
+		dev_warn(dev, "trTeInstEnSequentialJump %#x is not supported\n",
+			 config->seq_jump);
+		config->seq_jump = BMVAL(val, 2, 2);
+	}
+
+	if (BMVAL(val, 3, 3) != config->impl_ret) {
+		dev_warn(dev, "trTeInstEnImplicitReturn %#x is not supported\n",
+			 config->impl_ret);
+		config->impl_ret = BMVAL(val, 3, 3);
+	}
+
+	if (BMVAL(val, 4, 4) != config->branch_pred) {
+		dev_warn(dev, "trTeInstEnBranchPrediction %#x is not supported\n",
+			 config->branch_pred);
+		config->branch_pred = BMVAL(val, 4, 4);
+	}
+
+	if (BMVAL(val, 5, 5) != config->jump_target_cache) {
+		dev_warn(dev, "trTeInstEnJumpTargetCache %#x is not supported\n",
+			 config->jump_target_cache);
+		config->jump_target_cache = BMVAL(val, 5, 5);
+	}
+
+	if (BMVAL(val, 6, 7) != config->impl_ret_mode) {
+		dev_warn(dev, "trTeInstImplicitReturnMode %#x is not supported\n",
+			 config->impl_ret_mode);
+		config->impl_ret_mode = BMVAL(val, 6, 7);
+	}
+
+	if (BMVAL(val, 8, 8) != config->rep_hist) {
+		dev_warn(dev, "trTeInstEnRepeatedHistory %#x is not supported\n",
+			 config->rep_hist);
+		config->rep_hist = BMVAL(val, 8, 8);
+	}
+
+	if (BMVAL(val, 9, 9) != config->all_jumps) {
+		dev_warn(dev, "trTeInstEnAllJumps %#x is not supported\n",
+			 config->all_jumps);
+		config->all_jumps = BMVAL(val, 9, 9);
+	}
+
+	if (BMVAL(val, 10, 10) != config->ext_msb) {
+		dev_warn(dev, " trTeInstExtendAddrMSB %#x is not supported\n",
+			 config->ext_msb);
+		config->ext_msb = BMVAL(val, 10, 10);
+	}
+
+	if (BMVAL(val, 16, 27) != config->srcid) {
+		dev_warn(dev, "trTeSrcID %#x is not supported\n",
+			 config->srcid);
+		config->srcid = BMVAL(val, 16, 27);
+	}
+
+	if (BMVAL(val, 28, 31) != config->srcb) {
+		dev_warn(dev, "trTeSrcBits %#x is not supported\n",
+			 config->srcb);
+		config->srcb = BMVAL(val, 28, 31);
+	}
+
+	/* Configure Trace Encoder control register */
+	val = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	val |= (FIELD_PREP(RVTRACE_ENCODER_INSTMODE, ENCODER_INSTMODE_HTM) |
+		   FIELD_PREP(RVTRACE_ENCODER_CONTEXT, config->context) |
+		   FIELD_PREP(RVTRACE_ENCODER_INSTTRIGEN, config->inst_trigger) |
+		   FIELD_PREP(RVTRACE_ENCODER_INST_STALL_EN, config->inst_stall) |
+		   FIELD_PREP(RVTRACE_ENCODER_INHBSRC, config->inhb_src) |
+		   FIELD_PREP(RVTRACE_ENCODER_INSTSYNC_MODE, config->inst_syncmode) |
+		   FIELD_PREP(RVTRACE_ENCODER_INSTSYNC_MAX, config->inst_syncmax));
+
+
+	writel_relaxed(val, comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+
+	/* Check Trace Encoder control register WARL bits could be set */
+	val = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+
+	if (BMVAL(val, 9, 9) != config->context) {
+		dev_warn(dev, "trTeContext %#x is not supported\n",
+			 config->context);
+		config->context = BMVAL(val, 9, 9);
+	}
+
+	if (BMVAL(val, 11, 11) != config->inst_trigger) {
+		dev_warn(dev, "trTeInstTrigEnable %#x is not supported\n",
+			 config->inst_trigger);
+		config->inst_trigger = BMVAL(val, 11, 11);
+	}
+
+	if (BMVAL(val, 13, 13) != config->inst_stall) {
+		dev_warn(dev, "trTeInstStallEna %#x is not supported\n",
+			 config->inst_stall);
+		config->inst_stall = BMVAL(val, 13, 13);
+	}
+
+	if (BMVAL(val, 15, 15) != config->inhb_src) {
+		dev_warn(dev, "trTeInhibitSrc %#x is not supported\n",
+			 config->inhb_src);
+		config->inhb_src = BMVAL(val, 15, 15);
+	}
+
+	if (BMVAL(val, 20, 23) != config->inst_syncmax) {
+		dev_warn(dev, "trTeInstSyncMax %#x is not supported\n",
+			 config->inst_syncmax);
+		config->inst_syncmax = BMVAL(val, 20, 23);
+	}
+
+	/* Configure timestamp only if encoder has timestamp component */
+	if (encoder_data->has_timestamp && encoder_data->ts_ctrl)
+		timestamp_set_config(comp, dev, &encoder_data->ts_config);
+}
+
+static int encoder_enable_hw(struct rvtrace_component *comp)
+{
+	u32 val;
+	int ret;
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+
+	if (!comp->was_reset) {
+		ret = rvtrace_component_reset(comp);
+		if (ret)
+			goto done;
+	}
+
+	encoder_set_config(comp);
+
+	ret = rvtrace_enable_component(comp);
+	if (ret)
+		goto done;
+
+	/* Enable timestamp only if encoder has timestamp component */
+	if (encoder_data->has_timestamp && encoder_data->ts_ctrl) {
+		ret = timestamp_enable(comp);
+		if (ret) {
+			dev_warn(&encoder_data->csdev->dev,
+				 "Failed to enable timestamp\n");
+			ret = 0;  /* Don't fail encoder enable if timestamp fails */
+		}
+	}
+
+	val = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	val |= RVTRACE_ENCODER_ITRACE;
+	writel_relaxed(val, comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	ret = rvtrace_poll_bit(comp, RVTRACE_COMPONENT_CTRL_OFFSET,
+				RVTRACE_COMPONENT_CTRL_ITRACE_SHIFT, 1);
+
+done:
+	dev_dbg(&encoder_data->csdev->dev, "cpu: %d enable smp call done: %d\n",
+		comp->cpu, ret);
+	return ret;
+}
+
+static void encoder_enable_hw_smp_call(void *info)
+{
+	struct component_enable_arg *arg = info;
+
+	if (WARN_ON(!arg))
+		return;
+	arg->rc = encoder_enable_hw(arg->comp);
+}
+
+static int encoder_enable_perf(struct coresight_device *csdev)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+
+	if (WARN_ON_ONCE(comp->cpu != smp_processor_id()))
+		return -EINVAL;
+
+	return encoder_enable_hw(comp);
+}
+
+static int encoder_enable_sysfs(struct coresight_device *csdev)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	struct component_enable_arg arg = { };
+	int ret;
+
+	spin_lock(&encoder_data->spinlock);
+
+	/*
+	 * Executing encoder_enable_hw on the cpu whose trace encoder is being
+	 * enabled ensures that register writes occur when cpu is powered.
+	 */
+	arg.comp = comp;
+	ret = smp_call_function_single(comp->cpu,
+				       encoder_enable_hw_smp_call, &arg, 1);
+	if (!ret)
+		ret = arg.rc;
+	if (!ret)
+		encoder_data->sticky_enable = true;
+
+	spin_unlock(&encoder_data->spinlock);
+
+	if (!ret)
+		dev_dbg(&csdev->dev, "Trace Encoder tracing enabled\n");
+	return ret;
+}
+
+static int encoder_enable(struct coresight_device *csdev, struct perf_event *event,
+			  enum cs_mode mode, __maybe_unused struct coresight_path *path)
+{
+	int ret;
+
+	if (!coresight_take_mode(csdev, mode)) {
+		/* Someone is already using the tracer */
+		return -EBUSY;
+	}
+
+	switch (mode) {
+	case CS_MODE_SYSFS:
+		ret = encoder_enable_sysfs(csdev);
+		break;
+	case CS_MODE_PERF:
+		ret = encoder_enable_perf(csdev);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	/* The tracer didn't start */
+	if (ret)
+		coresight_set_mode(csdev, CS_MODE_DISABLED);
+
+	return ret;
+}
+
+static void encoder_disable_hw(struct rvtrace_component *comp)
+{
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+
+	/* Disable timestamp only if encoder has timestamp component */
+	if (encoder_data->has_timestamp && encoder_data->ts_ctrl)
+		timestamp_disable(comp);
+
+	if (rvtrace_disable_component(comp))
+		dev_err(&encoder_data->csdev->dev,
+			"timeout while waiting for Trace Encoder become disabled\n");
+
+	if (rvtrace_comp_is_empty(comp))
+		dev_err(&encoder_data->csdev->dev,
+			"timeout while waiting for all generated trace have been emitted\n");
+
+	dev_dbg(&encoder_data->csdev->dev, "cpu: %d disable smp call done\n", comp->cpu);
+}
+
+static void encoder_disable_sysfs_smp_call(void *info)
+{
+	struct rvtrace_component *comp = info;
+
+	encoder_disable_hw(comp);
+}
+
+static void encoder_disable_sysfs(struct coresight_device *csdev)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+
+        /*
+         * Taking hotplug lock here protects from clocks getting disabled
+         * with tracing being left on (crash scenario) if user disable occurs
+         * after cpu online mask indicates the cpu is offline but before the
+         * DYING hotplug callback is serviced by the trace encoder driver.
+         */
+        cpus_read_lock();
+        spin_lock(&encoder_data->spinlock);
+
+        /*
+         * Executing encoder_disable_hw on the cpu whose trace encoder is being
+         * disabled ensures that register writes occur when cpu is powered.
+         */
+	smp_call_function_single(comp->cpu, encoder_disable_sysfs_smp_call, comp, 1);
+
+	spin_unlock(&encoder_data->spinlock);
+	cpus_read_unlock();
+
+	dev_dbg(&csdev->dev, "Trace Encoder tracing disabled\n");
+}
+
+static void encoder_disable_perf(struct coresight_device *csdev)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+
+	if (WARN_ON_ONCE(comp->cpu != smp_processor_id()))
+		return;
+
+	encoder_disable_hw(comp);
+}
+
+static void encoder_disable(struct coresight_device *csdev,
+			    struct perf_event *event)
+{
+	enum cs_mode mode;
+
+	/*
+	 * For as long as the tracer isn't disabled another entity can't
+	 * change its status.  As such we can read the status here without
+	 * fearing it will change under us.
+	 */
+	mode = coresight_get_mode(csdev);
+
+	switch (mode) {
+	case CS_MODE_DISABLED:
+		break;
+	case CS_MODE_SYSFS:
+		encoder_disable_sysfs(csdev);
+		break;
+	case CS_MODE_PERF:
+		encoder_disable_perf(csdev);
+		break;
+	default:
+		WARN_ON_ONCE(mode);
+		return;
+	}
+
+	if (mode)
+		coresight_set_mode(csdev, CS_MODE_DISABLED);
+}
+
+static const struct coresight_ops_source encoder_source_ops = {
+	.cpu_id		= encoder_cpu_id,
+	.enable         = encoder_enable,
+	.disable        = encoder_disable,
+};
+
+static const struct coresight_ops encoder_cs_ops = {
+	.source_ops     = &encoder_source_ops,
+};
+
+void encoder_set_default(struct rvtrace_component *comp)
+{
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	if (WARN_ON_ONCE(!encoder_data))
+		return;
+
+	struct encoder_config *config = &encoder_data->config;
+
+	/* Enable sending trace messages/fields with scontext/mcontext values
+	 * and/or privilege levels
+	 */
+	config->context = true;
+
+	/* Allows trTeInstTracing to be set or cleared by Trace-on and Trace-
+	 * off signals generated by the corresponding trigger module.
+	 */
+	config->inst_trigger = true;
+
+	/* Enable periodic instruction trace synchronization */
+	config->inst_syncmode = ENCODER_SYNCMODE_CLOCK;
+	config->inst_syncmax = 0x6;
+
+	/* trace source ID*/
+	config->srcid = encoder_cpu_id(encoder_data->csdev);
+	config->srcb = 0xc;
+}
+
+static int encoder_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+	struct coresight_platform_data *pdata;
+	struct encoder_data *encoder_data;
+	struct rvtrace_component *comp;
+	struct coresight_desc desc = { 0 };
+
+	comp = rvtrace_register_component(pdev);
+	if (IS_ERR(comp))
+		return PTR_ERR(comp);
+
+	encoder_data = devm_kzalloc(dev, sizeof(*encoder_data), GFP_KERNEL);
+	if (!encoder_data)
+		return -ENOMEM;
+
+	spin_lock_init(&encoder_data->spinlock);
+
+	pdata = coresight_get_platform_data(dev);
+	if (IS_ERR(pdata))
+		return PTR_ERR(pdata);
+	pdev->dev.platform_data = pdata;
+
+	platform_set_drvdata(pdev, comp);
+
+	/* Check if encoder has timestamp component from device tree early */
+	encoder_data->has_timestamp = fwnode_property_present(dev->fwnode, "riscv,timestamp-present");
+	if (encoder_data->has_timestamp) {
+		if (rvtrace_init_timestamp(comp, &encoder_data->ts_config)) {
+			dev_err(dev, "Timestamp initialization failed\n");
+			return -EINVAL;
+		}
+
+		/* TODO: Default to enabling timestamp control if present, as
+		 * encoder_data->ts_ctrl can be configured via sysfs attribute,
+		 * but not through perf_event at this time. Future versions may
+		 * add support for configuring timestamps via perf_event.
+		 */
+		encoder_data->ts_ctrl = true;
+	}
+
+	/* Set component data before registration so is_visible callbacks can access it */
+	comp->id.data = encoder_data;
+
+	desc.name = devm_kasprintf(dev, GFP_KERNEL, "encoder%d", comp->cpu);
+	if (!desc.name)
+		return -ENOMEM;
+
+	desc.access = CSDEV_ACCESS_IOMEM(comp->base);
+	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
+	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
+	desc.ops = &encoder_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = trace_encoder_groups;
+	encoder_data->csdev = coresight_register(&desc);
+	if (IS_ERR(encoder_data->csdev))
+		return PTR_ERR(encoder_data->csdev);
+
+	ret = etm_perf_symlink(encoder_data->csdev, true);
+	if (ret) {
+		coresight_unregister(encoder_data->csdev);
+		return ret;
+	}
+
+	rvtrace_cpu_encoder[comp->cpu] = comp;
+
+	encoder_set_default(comp);
+
+	dev_dbg(dev, "CPU%d: Trace Encoder initialized\n", comp->cpu);
+
+	if (boot_enable) {
+		coresight_enable_sysfs(encoder_data->csdev);
+		encoder_data->boot_enable = true;
+	}
+
+	return 0;
+}
+
+static void clear_encodata(void *info)
+{
+	int cpu = *(int *)info;
+
+	rvtrace_cpu_encoder[cpu]->id.data = NULL;
+	rvtrace_cpu_encoder[cpu] = NULL;
+}
+
+static void encoder_remove(struct platform_device *pdev)
+{
+	struct rvtrace_component *comp = platform_get_drvdata(pdev);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	struct device *dev = &pdev->dev;
+
+	etm_perf_symlink(encoder_data->csdev, false);
+
+	/*
+	 * Taking hotplug lock here to avoid racing between encoder_remove and
+	 * CPU hotplug call backs.
+	 */
+	cpus_read_lock();
+	/*
+	 * The readers for encodata[] are CPU hotplug call backs
+	 * and PM notification call backs. Change encodata[i] on
+	 * CPU i ensures these call backs has consistent view
+	 * inside one call back function.
+	 */
+	if (smp_call_function_single(comp->cpu, clear_encodata, &comp->cpu, 1)) {
+		rvtrace_cpu_encoder[comp->cpu]->id.data = NULL;
+		rvtrace_cpu_encoder[comp->cpu] = NULL;
+	}
+
+	cpus_read_unlock();
+
+	coresight_unregister(encoder_data->csdev);
+}
+
+static const struct of_device_id encoder_match[] = {
+	{.compatible = "riscv,trace-encoder"},
+	{},
+};
+
+static struct platform_driver encoder_driver = {
+	.probe = encoder_probe,
+	.remove = encoder_remove,
+	.driver = {
+		.name = "trace-encoder",
+		.of_match_table = encoder_match,
+	},
+};
+
+module_platform_driver(encoder_driver);
+
+MODULE_DESCRIPTION("RISC-V Trace Encoder Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwtracing/coresight/rvtrace-encoder-sysfs.c b/drivers/hwtracing/coresight/rvtrace-encoder-sysfs.c
new file mode 100644
index 000000000000..19fd8936f5c7
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-encoder-sysfs.c
@@ -0,0 +1,361 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#include <linux/sysfs.h>
+#include <linux/rvtrace.h>
+#include "rvtrace-encoder.h"
+#include "rvtrace-timestamp.h"
+#include "coresight-priv.h"
+
+static ssize_t cpu_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+
+	val = comp->cpu;
+	return scnprintf(buf, PAGE_SIZE, "%#lx\n", val);
+}
+static DEVICE_ATTR_RO(cpu);
+
+static ssize_t reset_store(struct device *dev,
+			   struct device_attribute *attr,
+			   const char *buf, size_t size)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+
+	if (kstrtoul(buf, 16, &val))
+		return -EINVAL;
+
+	spin_lock(&encoder_data->spinlock);
+
+	if (val) {
+		encoder_set_default(comp);
+		if (rvtrace_component_reset(comp)) {
+			comp->was_reset = false;
+			spin_unlock(&encoder_data->spinlock);
+			return -EINVAL;
+		}
+	}
+
+	spin_unlock(&encoder_data->spinlock);
+
+	return size;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t ts_ctrl_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", encoder_data->ts_ctrl);
+}
+
+static ssize_t ts_ctrl_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t size)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+
+	if (kstrtoul(buf, 10, &val))
+		return -EINVAL;
+
+	spin_lock(&encoder_data->spinlock);
+	encoder_data->ts_ctrl = !!val;
+	spin_unlock(&encoder_data->spinlock);
+
+	return size;
+}
+static DEVICE_ATTR_RW(ts_ctrl);
+
+static struct attribute *trace_encoder_attrs[] = {
+	&dev_attr_cpu.attr,
+	&dev_attr_reset.attr,
+	&dev_attr_ts_ctrl.attr,
+	NULL,
+};
+
+static struct attribute *trace_encoder_mgmt_attrs[] = {
+	coresight_simple_reg32(control, RVTRACE_COMPONENT_CTRL_OFFSET),
+	coresight_simple_reg32(impl, RVTRACE_COMPONENT_IMPL_OFFSET),
+	coresight_simple_reg32(features, RVTRACE_ENCODER_INST_FTRS_OFFSET),
+	NULL,
+};
+
+#define encoder_simple_rw(name)						    \
+static ssize_t name##_show(struct device *dev,				    \
+			   struct device_attribute *attr, char *buf)	    \
+{									    \
+	unsigned long val;						    \
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);	    \
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);   \
+	struct encoder_config *config = &encoder_data->config;		    \
+									    \
+	val = config->name;						    \
+	return scnprintf(buf, PAGE_SIZE, "%#lx\n", val);		    \
+}									    \
+									    \
+static ssize_t name##_store(struct device *dev,				    \
+			    struct device_attribute *attr,		    \
+			    const char *buf, size_t size)		    \
+{									    \
+	unsigned long val;						    \
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);	    \
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);   \
+	struct encoder_config *config = &encoder_data->config;		    \
+									    \
+	if (kstrtoul(buf, 16, &val))					    \
+		return -EINVAL;						    \
+									    \
+	spin_lock(&encoder_data->spinlock);	    			    \
+	config->name = val;						    \
+	spin_unlock(&encoder_data->spinlock);				    \
+									    \
+	return size;							    \
+}									    \
+static DEVICE_ATTR_RW(name);
+
+static const char *const instmodes_str[] = {
+	[ENCODER_INSTMODE_OFF]	    = "off",
+	[ENCODER_INSTMODE_RESV1]    = "resv1",
+	[ENCODER_INSTMODE_BTM]	    = "btm",
+	[ENCODER_INSTMODE_RESV4]    = "resv4",
+	[ENCODER_INSTMODE_RESV5]    = "resv5",
+	[ENCODER_INSTMODE_HTM]	    = "htm",
+	[ENCODER_INSTMODE_RESV7]    = "resv7",
+};
+
+static ssize_t instmode_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+
+	val = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	val = BMVAL(val, 4, 6);
+
+	return sysfs_emit(buf, "%s\n", instmodes_str[val]);
+}
+static DEVICE_ATTR_RO(instmode);
+
+static const char *const inst_syncmodes_str[] = {
+	[ENCODER_SYNCMODE_OFF]		= "off",
+	[ENCODER_SYNCMODE_MESSAGES]	= "messages",
+	[ENCODER_SYNCMODE_CLOCK]	= "clock",
+	[ENCODER_SYNCMODE_INSTRUCTION]	= "instruction"
+};
+
+static ssize_t inst_syncmodes_available_show(struct device *dev,
+					     struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%s %s %s %s\n",
+			    inst_syncmodes_str[ENCODER_SYNCMODE_OFF],
+			    inst_syncmodes_str[ENCODER_SYNCMODE_MESSAGES],
+			    inst_syncmodes_str[ENCODER_SYNCMODE_CLOCK],
+			    inst_syncmodes_str[ENCODER_SYNCMODE_INSTRUCTION]);
+}
+static DEVICE_ATTR_RO(inst_syncmodes_available);
+
+static ssize_t inst_syncmode_preferred_show(struct device *dev,
+					    struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	struct encoder_config *config = &encoder_data->config;
+
+        return sysfs_emit(buf, "%s\n", inst_syncmodes_str[config->inst_syncmode]);
+}
+
+static ssize_t inst_syncmode_preferred_store(struct device *dev,
+					     struct device_attribute *attr,
+					     const char *buf, size_t size)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	struct encoder_config *config = &encoder_data->config;
+
+	if (sysfs_streq(buf, inst_syncmodes_str[ENCODER_SYNCMODE_OFF])) {
+		config->inst_syncmode = ENCODER_SYNCMODE_OFF;
+	} else if (sysfs_streq(buf, inst_syncmodes_str[ENCODER_SYNCMODE_MESSAGES])) {
+		config->inst_syncmode = ENCODER_SYNCMODE_MESSAGES;
+	} else if (sysfs_streq(buf, inst_syncmodes_str[ENCODER_SYNCMODE_CLOCK])) {
+		config->inst_syncmode = ENCODER_SYNCMODE_CLOCK;
+	} else if (sysfs_streq(buf, inst_syncmodes_str[ENCODER_SYNCMODE_INSTRUCTION])) {
+		config->inst_syncmode = ENCODER_SYNCMODE_INSTRUCTION;
+	} else {
+		return -EINVAL;
+	}
+	return size;
+}
+static DEVICE_ATTR_RW(inst_syncmode_preferred);
+
+static ssize_t format_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+
+	val = readl_relaxed(comp->base + RVTRACE_COMPONENT_CTRL_OFFSET);
+	val = BMVAL(val, 24, 26);
+
+	return scnprintf(buf, PAGE_SIZE, "%#lx\n", val);
+}
+static DEVICE_ATTR_RO(format);
+
+encoder_simple_rw(context);
+encoder_simple_rw(inst_trigger);
+encoder_simple_rw(inst_stall);
+encoder_simple_rw(inhb_src);
+encoder_simple_rw(inst_syncmax);
+
+static struct attribute *trace_encoder_control_attrs[] = {
+	&dev_attr_instmode.attr,
+	&dev_attr_context.attr,
+	&dev_attr_inst_trigger.attr,
+	&dev_attr_inst_stall.attr,
+	&dev_attr_inhb_src.attr,
+	&dev_attr_inst_syncmodes_available.attr,
+	&dev_attr_inst_syncmode_preferred.attr,
+	&dev_attr_inst_syncmax.attr,
+	&dev_attr_format.attr,
+	NULL,
+};
+
+static const char *const inst_implicit_return_modes_str[] = {
+	[ENCODER_IMPLICITRETURNMODE_NOT_SUPPORTED]	= "off",
+	[ENCODER_IMPLICITRETURNMODE_SIMPLE_COUNTING]	= "simple",
+	[ENCODER_IMPLICITRETURNMODE_PARTIAL_ADDRESS]	= "partial",
+	[ENCODER_IMPLICITRETURNMODE_FULL_ADDRESS]	= "full"
+};
+
+static ssize_t inst_implicit_return_modes_available_show(struct device *dev,
+							 struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%s %s %s %s\n",
+			    inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_NOT_SUPPORTED],
+			    inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_SIMPLE_COUNTING],
+			    inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_PARTIAL_ADDRESS],
+			    inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_FULL_ADDRESS]);
+}
+static DEVICE_ATTR_RO(inst_implicit_return_modes_available);
+
+
+static ssize_t inst_implicit_return_mode_preferred_show(struct device *dev,
+					    struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	struct encoder_config *config = &encoder_data->config;
+
+	return sysfs_emit(buf, "%s\n", inst_implicit_return_modes_str[config->impl_ret_mode]);
+}
+
+static ssize_t inst_implicit_return_mode_preferred_store(struct device *dev,
+					     struct device_attribute *attr,
+					     const char *buf, size_t size)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+	struct encoder_config *config = &encoder_data->config;
+
+	if (sysfs_streq(buf, inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_NOT_SUPPORTED])) {
+		config->impl_ret_mode = ENCODER_IMPLICITRETURNMODE_NOT_SUPPORTED;
+	} else if (sysfs_streq(buf, inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_SIMPLE_COUNTING])) {
+		config->impl_ret_mode = ENCODER_IMPLICITRETURNMODE_SIMPLE_COUNTING;
+	} else if (sysfs_streq(buf, inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_PARTIAL_ADDRESS])) {
+		config->impl_ret_mode = ENCODER_IMPLICITRETURNMODE_PARTIAL_ADDRESS;
+	} else if (sysfs_streq(buf, inst_implicit_return_modes_str[ENCODER_IMPLICITRETURNMODE_FULL_ADDRESS])) {
+		config->impl_ret_mode = ENCODER_IMPLICITRETURNMODE_FULL_ADDRESS;
+	} else {
+		return -EINVAL;
+	}
+	return size;
+}
+static DEVICE_ATTR_RW(inst_implicit_return_mode_preferred);
+
+encoder_simple_rw(inst_na_diff);
+encoder_simple_rw(inst_nt_addr);
+encoder_simple_rw(seq_jump);
+encoder_simple_rw(impl_ret);
+encoder_simple_rw(branch_pred);
+encoder_simple_rw(jump_target_cache);
+encoder_simple_rw(rep_hist);
+encoder_simple_rw(all_jumps);
+encoder_simple_rw(ext_msb);
+encoder_simple_rw(srcid);
+encoder_simple_rw(srcb);
+
+static struct attribute *trace_encoder_features_attrs[] = {
+	&dev_attr_inst_na_diff.attr,
+	&dev_attr_inst_nt_addr.attr,
+	&dev_attr_seq_jump.attr,
+	&dev_attr_impl_ret.attr,
+	&dev_attr_branch_pred.attr,
+	&dev_attr_jump_target_cache.attr,
+	&dev_attr_inst_implicit_return_modes_available.attr,
+	&dev_attr_inst_implicit_return_mode_preferred.attr,
+	&dev_attr_rep_hist.attr,
+	&dev_attr_all_jumps.attr,
+	&dev_attr_ext_msb.attr,
+	&dev_attr_srcid.attr,
+	&dev_attr_srcb.attr,
+	NULL,
+};
+
+static umode_t timestamp_attr_is_visible(struct kobject *kobj,
+					 struct attribute *attr, int idx)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct encoder_data *encoder_data = rvtrace_component_data(comp);
+
+	if (encoder_data->has_timestamp)
+		return attr->mode;
+
+	return 0;
+}
+
+static const struct attribute_group trace_encoder_group = {
+	.attrs = trace_encoder_attrs,
+};
+
+static const struct attribute_group trace_encoder_mgmt_group = {
+	.attrs = trace_encoder_mgmt_attrs,
+	.name = "mgmt",
+};
+
+static const struct attribute_group trace_encoder_control_group = {
+	.attrs = trace_encoder_control_attrs,
+	.name = "control",
+};
+
+static const struct attribute_group trace_encoder_features_group = {
+	.attrs = trace_encoder_features_attrs,
+	.name = "features",
+};
+
+extern const struct attribute *timestamp_attrs[];
+
+static const struct attribute_group trace_encoder_timestamp_group = {
+	.attrs = (struct attribute **)timestamp_attrs,
+	.name = "timestamp",
+	.is_visible = timestamp_attr_is_visible,
+};
+
+const struct attribute_group *trace_encoder_groups[] = {
+	&trace_encoder_group,
+	&trace_encoder_mgmt_group,
+	&trace_encoder_control_group,
+	&trace_encoder_features_group,
+	&trace_encoder_timestamp_group,
+	NULL,
+};
diff --git a/drivers/hwtracing/coresight/rvtrace-encoder.h b/drivers/hwtracing/coresight/rvtrace-encoder.h
new file mode 100644
index 000000000000..82f699207cda
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-encoder.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#ifndef _RVTRACE_ENCODER_H
+#define _RVTRACE_ENCODER_H
+
+#include <asm/local.h>
+#include <linux/spinlock.h>
+#include "coresight-priv.h"
+#include "rvtrace-timestamp.h"
+
+/* Trace Encoder Control Register */
+#define RVTRACE_ENCODER_ITRACE				    BIT(2)
+#define RVTRACE_ENCODER_INSTMODE			    GENMASK(6, 4)
+#define RVTRACE_ENCODER_CONTEXT			   	    BIT(9)
+#define RVTRACE_ENCODER_INSTTRIGEN			    BIT(11)
+#define RVTRACE_ENCODER_INST_STALL_OR_OVERFLOW   	    BIT(12)
+#define RVTRACE_ENCODER_INST_STALL_EN			    BIT(13)
+#define RVTRACE_ENCODER_INHBSRC				    BIT(15)
+#define RVTRACE_ENCODER_INSTSYNC_MODE			    GENMASK(17, 16)
+#define RVTRACE_ENCODER_INSTSYNC_MAX			    GENMASK(23, 20)
+
+#define RVTRACE_COMPONENT_CTRL_ITRACE_SHIFT		    2
+
+/* Trace Encoder Implementation Register */
+
+/* Trace Instruction Features Register */
+#define RVTRACE_ENCODER_INST_FTRS_OFFSET		    0x008
+#define RVTRACE_ENCODER_INST_NO_ADDR_DIFF		    BIT(0)
+#define RVTRACE_ENCODER_INST_NO_TRAP_ADDR		    BIT(1)
+#define RVTRACE_ENCODER_INST_EN_SEQUENTIAL_JUMP		    BIT(2)
+#define RVTRACE_ENCODER_INST_EN_IMPLICIT_RETURN		    BIT(3)
+#define RVTRACE_ENCODER_INST_EN_BRANCH_PREDICTION	    BIT(4)
+#define RVTRACE_ENCODER_INST_JUMP_TARGET_CACHE		    BIT(5)
+#define RVTRACE_ENCODER_INST_IMPLICIT_RETURN_MODE	    GENMASK(7, 6)
+#define RVTRACE_ENCODER_INST_EN_REPEQTED_HISTORT	    BIT(8)
+#define RVTRACE_ENCODER_INST_EN_ALL_JUMPS		    BIT(9)
+#define RVTRACE_ENCODER_INST_EXTEND_ADDR_MSB		    BIT(10)
+#define RVTRACE_ENCODER_SRCID				    GENMASK(27, 16)
+#define RVTRACE_ENCODER_SRCBITS				    GENMASK(31, 28)
+
+enum encoder_format {
+	ENCODER_FORMAT_ETRACE,
+	ENCODER_FORMAT_NTRACE,
+	ENCODER_FORMAT_UNKNOWN = 7
+};
+
+enum encoder_instmode {
+	ENCODER_INSTMODE_OFF,
+	ENCODER_INSTMODE_RESV1,
+	ENCODER_INSTMODE_RESV2,
+	ENCODER_INSTMODE_BTM,
+	ENCODER_INSTMODE_RESV4,
+	ENCODER_INSTMODE_RESV5,
+	ENCODER_INSTMODE_HTM,
+	ENCODER_INSTMODE_RESV7
+};
+
+enum encoder_syncmode {
+        ENCODER_SYNCMODE_OFF,
+        ENCODER_SYNCMODE_MESSAGES,
+        ENCODER_SYNCMODE_CLOCK,
+        ENCODER_SYNCMODE_INSTRUCTION
+};
+
+enum encoder_implicit_return_mode {
+        ENCODER_IMPLICITRETURNMODE_NOT_SUPPORTED,
+        ENCODER_IMPLICITRETURNMODE_SIMPLE_COUNTING,
+        ENCODER_IMPLICITRETURNMODE_PARTIAL_ADDRESS,
+        ENCODER_IMPLICITRETURNMODE_FULL_ADDRESS
+};
+
+/**
+ * struct encoder_config - configuration information related to an trace encoder
+ * @context:		Controls sending trace messages/fields with scontext/mcontext
+ *			values and/or privilege levels.
+ * @inst_trig:		Controls trTeInstTracing to be set or cleared by Trace-on and
+ *			Trace-off signals generated by the corresponding trigger module.
+ * @inst_stall:		Controls whether hart is stalled until TE can send a message.
+ * @inhb_src:		Controls whether include source field in trace messages/packets.
+ * @inst_syncmode:	Periodic instruction trace synchronization message/packet
+ *			generation mechanism.
+ * @inst_syncmax:	The maximum interval between instruction trace synchronization
+ *			messages/packets.
+ * @trc_fmt:		Trace recording/protocol format.
+ * @inst_na_diff:	Controls whether trace messages/packets always carry a full address.
+ * @inst_nt_addr:	Controls whether include trap handler address in trap messages/packets.
+ * @seq_jump:		Controls the sequential jump optimization.
+ * @impl_ret:		Controls the implicit return optimization.
+ * @branch_pred:	Branch Predictor based compression.
+ * @jump_target_cache:	Jump Target Cache based compression.
+ * @impl_ret_mode:	Controls how the decoder is handling stack of return addresses.
+ * @rep_hist:		Controls the repeated history optimization.
+ * @all_jumps:		Controls the emitting of trace message or add history/map bit for
+ *			direct unconditional/inferable control flow changes.
+ * @ext_msb:		Controls the extended handing of MSB address bits.
+ * @srcid:		Trace source ID assigned to this trace encoder.
+ * @srcb:		The number of bits in the trace source field, unless disabled by
+ *			inhibit_src.
+ */
+struct encoder_config {
+	bool				    context;
+	bool				    inst_trigger;
+	bool				    inst_stall;
+	bool				    inhb_src;
+	u8				    inst_syncmode;
+	u8				    inst_syncmax;
+	bool				    inst_na_diff;
+	bool				    inst_nt_addr;
+	bool				    seq_jump;
+	bool				    impl_ret;
+	bool				    branch_pred;
+	bool				    jump_target_cache;
+	u8				    impl_ret_mode;
+	bool				    rep_hist;
+	bool				    all_jumps;
+	bool				    ext_msb;
+	u16				    srcid;
+	u8				    srcb;
+};
+
+/**
+ * struct encoder_drvdata - specifics associated to an Trace Encoder component
+ * @csdev:	    Component vitals needed by the framework.
+ * @spinlock:	    Only one at a time pls.
+ * @sticky_enable:  True if trace encoder base configuration has been done.
+ * @boot_enable:    True if we should start tracing at boot time.
+ * @has_timestamp:  True if this encoder has timestamp component.
+ * @ts_ctrl:        Controls the insertion of global timestamps in the trace streams.
+ * @config:	    Structure holding configuration parameters.
+ * @ts_config:	    Timestamp configuration.
+ */
+
+struct encoder_data {
+	struct coresight_device		    *csdev;
+	spinlock_t                          spinlock;
+	bool                                sticky_enable;
+	bool                                boot_enable;
+	bool                                has_timestamp;
+	bool                                ts_ctrl;
+	struct encoder_config		    config;
+	struct timestamp_config		    ts_config;
+};
+
+extern const struct attribute_group *trace_encoder_groups[];
+void encoder_set_default(struct rvtrace_component *comp);
+
+#endif
diff --git a/drivers/hwtracing/coresight/rvtrace-funnel.c b/drivers/hwtracing/coresight/rvtrace-funnel.c
new file mode 100644
index 000000000000..6b743cc0d0fd
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-funnel.c
@@ -0,0 +1,336 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/coresight.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/rvtrace.h>
+
+#include "coresight-priv.h"
+#include "rvtrace-timestamp.h"
+#include "rvtrace-funnel.h"
+
+DEFINE_CORESIGHT_DEVLIST(funnel_devs, "rvtrace_funnel");
+
+static int funnel_enable_hw(struct rvtrace_component *comp, int port)
+{
+	u32 disinput;
+	int ret = 0;
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+
+	if (!comp->was_reset) {
+		ret = rvtrace_component_reset(comp);
+		if (ret)
+			goto done;
+	}
+
+	disinput = readl_relaxed(comp->base + RVTRACE_FUNNEL_DISINPUT_OFFSET);
+	disinput &= ~(1 << port);
+	writel_relaxed(disinput, comp->base + RVTRACE_FUNNEL_DISINPUT_OFFSET);
+	ret = rvtrace_poll_bit(comp, RVTRACE_FUNNEL_DISINPUT_OFFSET, port, 0);
+	if (ret)
+		goto done;
+
+	if (!funnel_data->was_enabled) {
+		ret = rvtrace_enable_component(comp);
+		if (ret)
+			goto done;
+
+		/* Enable timestamp only if funnel has timestamp component */
+		if (funnel_data->has_timestamp && funnel_data->ts_ctrl) {
+			ret = timestamp_enable(comp);
+			if (ret) {
+				dev_warn(&funnel_data->csdev->dev,
+					 "Failed to enable timestamp\n");
+				ret = 0;  /* Don't fail funnel enable if timestamp fails */
+			}
+		}
+	}
+
+done:
+	return ret;
+}
+
+static int funnel_enable(struct coresight_device *csdev,
+			 struct coresight_connection *in,
+			 struct coresight_connection *out)
+{
+	int ret = 0;
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+	unsigned long flags;
+	bool first_enable = false;
+
+	spin_lock_irqsave(&funnel_data->spinlock, flags);
+	if (in->dest_refcnt == 0) {
+		ret = funnel_enable_hw(comp, in->dest_port);
+		if (!ret)
+			first_enable = true;
+	}
+	if (!ret) {
+		in->dest_refcnt++;
+		funnel_data->input_refcnt++;
+	}
+
+	if (first_enable)
+		dev_dbg(&csdev->dev, "Trace funnel inport %d enabled\n",
+			in->dest_port);
+
+	spin_unlock_irqrestore(&funnel_data->spinlock, flags);
+
+	return ret;
+}
+
+static void funnel_disable_hw(struct rvtrace_component *comp, int inport)
+{
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+	if (--funnel_data->input_refcnt != 0)
+		return;
+
+	/* Disable timestamp only if funnel has timestamp component */
+	if (funnel_data->has_timestamp && funnel_data->ts_ctrl)
+		timestamp_disable(comp);
+
+	writel_relaxed(RVTRACE_FUNNEL_DISINPUT_MASK, comp->base + RVTRACE_FUNNEL_DISINPUT_OFFSET);
+
+	if (rvtrace_disable_component(comp))
+		dev_err(&funnel_data->csdev->dev,
+			"timeout while waiting for Trace Funnel to be disabled\n");
+
+	if (rvtrace_comp_is_empty(comp))
+		dev_err(&funnel_data->csdev->dev,
+			"timeout while waiting for Trace Funnel internal buffers become empty\n");
+}
+
+static void funnel_disable(struct coresight_device *csdev,
+			   struct coresight_connection *in,
+			   struct coresight_connection *out)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(csdev->dev.parent);
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+	unsigned long flags;
+	bool last_disable = false;
+
+	spin_lock_irqsave(&funnel_data->spinlock, flags);
+	if (--in->dest_refcnt == 0) {
+		funnel_disable_hw(comp, in->dest_port);
+		last_disable = true;
+	}
+	spin_unlock_irqrestore(&funnel_data->spinlock, flags);
+
+	if (last_disable)
+		dev_dbg(&csdev->dev, "Trace funnel inport %d disabled\n",
+			in->dest_port);
+
+	if (funnel_data->input_refcnt == 0)
+		dev_dbg(&csdev->dev, "Trace funnel disabled\n");
+}
+
+static const struct coresight_ops_link funnel_link_ops = {
+	.enable		= funnel_enable,
+	.disable	= funnel_disable,
+};
+
+static const struct coresight_ops funnel_cs_ops = {
+        .link_ops       = &funnel_link_ops,
+};
+
+static ssize_t reset_store(struct device *dev,
+			   struct device_attribute *attr,
+			   const char *buf, size_t size)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+	unsigned long flags;
+
+	if (kstrtoul(buf, 16, &val))
+		return -EINVAL;
+
+	spin_lock_irqsave(&funnel_data->spinlock, flags);
+
+	if (val) {
+		if (rvtrace_component_reset(comp)) {
+			comp->was_reset = false;
+			spin_unlock_irqrestore(&funnel_data->spinlock, flags);
+			return -EINVAL;
+		}
+	}
+
+	spin_unlock_irqrestore(&funnel_data->spinlock, flags);
+
+	return size;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t cpu_show(struct device *dev,
+			struct device_attribute *attr, char *buf)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+
+	val = comp->cpu;
+	return scnprintf(buf, PAGE_SIZE, "%#lx\n", val);
+}
+static DEVICE_ATTR_RO(cpu);
+
+static ssize_t ts_ctrl_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", funnel_data->ts_ctrl);
+}
+
+static ssize_t ts_ctrl_store(struct device *dev,
+			    struct device_attribute *attr,
+			    const char *buf, size_t size)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+
+	if (kstrtoul(buf, 10, &val))
+		return -EINVAL;
+
+	spin_lock(&funnel_data->spinlock);
+	funnel_data->ts_ctrl = !!val;
+	spin_unlock(&funnel_data->spinlock);
+
+	return size;
+}
+static DEVICE_ATTR_RW(ts_ctrl);
+
+static struct attribute *trace_funnel_attrs[] = {
+	coresight_simple_reg32(control, RVTRACE_COMPONENT_CTRL_OFFSET),
+	coresight_simple_reg32(impl, RVTRACE_COMPONENT_IMPL_OFFSET),
+	coresight_simple_reg32(disinput, RVTRACE_FUNNEL_DISINPUT_OFFSET),
+	&dev_attr_reset.attr,
+	&dev_attr_cpu.attr,
+	&dev_attr_ts_ctrl.attr,
+	NULL,
+};
+
+static umode_t timestamp_attr_is_visible(struct kobject *kobj,
+					 struct attribute *attr, int idx)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+
+	if (funnel_data->has_timestamp)
+		return attr->mode;
+
+	return 0;
+}
+
+extern const struct attribute *timestamp_attrs[];
+
+static struct attribute_group trace_funnel_group = {
+	.attrs = trace_funnel_attrs,
+};
+
+static struct attribute_group trace_funnel_timestamp_group = {
+	.attrs = (struct attribute **)timestamp_attrs,
+	.name = "timestamp",
+	.is_visible = timestamp_attr_is_visible,
+};
+
+const struct attribute_group *trace_funnel_groups[] = {
+	&trace_funnel_group,
+	&trace_funnel_timestamp_group,
+	NULL,
+};
+
+static int funnel_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct coresight_platform_data *pdata;
+	struct funnel_data *funnel_data;
+	struct rvtrace_component *comp;
+	struct coresight_desc desc = { 0 };
+
+	comp = rvtrace_register_component(pdev);
+	if (IS_ERR(comp))
+		return PTR_ERR(comp);
+
+	funnel_data = devm_kzalloc(dev, sizeof(*funnel_data), GFP_KERNEL);
+	if (!funnel_data)
+		return -ENOMEM;
+
+	spin_lock_init(&funnel_data->spinlock);
+
+	pdata = coresight_get_platform_data(dev);
+	if (IS_ERR(pdata))
+		return PTR_ERR(pdata);
+	pdev->dev.platform_data = pdata;
+
+	platform_set_drvdata(pdev, comp);
+
+	/* Check if funnel has timestamp component from device tree */
+	funnel_data->has_timestamp = fwnode_property_present(dev->fwnode, "riscv,timestamp-present");
+	if (funnel_data->has_timestamp) {
+		if (rvtrace_init_timestamp(comp, &funnel_data->ts_config)) {
+			dev_err(dev, "Timestamp initialization failed\n");
+			return -EINVAL;
+		}
+
+		/* TODO: Default to enabling timestamp control if present, as
+		 * encoder_data->ts_ctrl can be configured via sysfs attribute,
+		 * but not through perf_event at this time. Future versions may
+		 * add support for configuring timestamps via perf_event.
+		 */
+		funnel_data->ts_ctrl = true;
+	}
+
+	/* Set component data before registration so is_visible callbacks can access it */
+	comp->id.data = funnel_data;
+
+	desc.name = coresight_alloc_device_name(&funnel_devs, dev);
+	desc.access = CSDEV_ACCESS_IOMEM(comp->base);
+	desc.type = CORESIGHT_DEV_TYPE_LINK;
+	desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_MERG;
+	desc.ops = &funnel_cs_ops;
+	desc.pdata = pdata;
+	desc.dev = dev;
+	desc.groups = trace_funnel_groups;
+	funnel_data->csdev = coresight_register(&desc);
+	if (IS_ERR(funnel_data->csdev))
+		return PTR_ERR(funnel_data->csdev);
+
+	dev_dbg(dev, "Trace Funnel initialized\n");
+
+	return 0;
+}
+
+static void funnel_remove(struct platform_device *pdev)
+{
+	struct rvtrace_component *comp = platform_get_drvdata(pdev);
+	struct funnel_data *funnel_data = rvtrace_component_data(comp);
+
+	coresight_unregister(funnel_data->csdev);
+}
+
+static const struct of_device_id funnel_match[] = {
+	{.compatible = "riscv,trace-funnel"},
+	{},
+};
+
+static struct platform_driver funnel_driver = {
+	.probe = funnel_probe,
+	.remove = funnel_remove,
+	.driver = {
+		.name	= "trace-funnel",
+		.of_match_table = funnel_match,
+	},
+};
+
+module_platform_driver(funnel_driver);
+
+MODULE_DESCRIPTION("RISC-V Trace funnel driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwtracing/coresight/rvtrace-funnel.h b/drivers/hwtracing/coresight/rvtrace-funnel.h
new file mode 100644
index 000000000000..5bf896a831a0
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-funnel.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#ifndef _RVTRACE_FUNNEL_H
+#define _RVTRACE_FUNNEL_H
+
+#include <linux/spinlock.h>
+#include <linux/coresight.h>
+#include "rvtrace-timestamp.h"
+
+/* Disable Individual Funnel Inputs */
+#define RVTRACE_FUNNEL_DISINPUT_OFFSET			0x008
+#define RVTRACE_FUNNEL_DISINPUT_MASK			0xffff
+
+/**
+ * struct funnel_data - specifics associated to a Trace Funnel component
+ * @csdev:        Component vitals needed by the framework.
+ * @spinlock:     Only one at a time pls.
+ * @was_enabled:  Flag showing whether the Trace Funnel was enabled.
+ * @input_refcnt: Record the number of funnel inputs
+ * @has_timestamp: True if this funnel has timestamp component.
+ * @ts_ctrl:      Controls the insertion of global timestamps in the trace streams.
+ * @ts_config:    Timestamp configuration.
+ */
+struct funnel_data {
+	struct coresight_device	*csdev;
+	spinlock_t		spinlock;
+	bool			was_enabled;
+	u32			input_refcnt;
+	u32			disintput;
+	bool			has_timestamp;
+	bool			ts_ctrl;
+	struct timestamp_config	ts_config;
+};
+
+#endif
diff --git a/drivers/hwtracing/coresight/rvtrace-timestamp.c b/drivers/hwtracing/coresight/rvtrace-timestamp.c
new file mode 100644
index 000000000000..6aa63f4891a4
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-timestamp.c
@@ -0,0 +1,277 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/bitfield.h>
+#include <linux/sysfs.h>
+#include <linux/rvtrace.h>
+
+#include "rvtrace-timestamp.h"
+#include "rvtrace-encoder.h"
+#include "rvtrace-funnel.h"
+#include "coresight-priv.h"
+
+int timestamp_enable(struct rvtrace_component *comp)
+{
+	u32 val;
+
+	val = readl_relaxed(comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+	val |= (RVTRACE_TIMESTAMP_ENABLE | RVTRACE_TIMESTAMP_COUNT);
+
+	writel_relaxed(val, comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+
+	return rvtrace_poll_bit(comp, RVTRACE_TIMESTAMP_CTRL_OFFSET,
+				RVTRACE_TIMESTAMP_ENABLE_SHIFT, 1);
+}
+EXPORT_SYMBOL_GPL(timestamp_enable);
+
+void timestamp_disable(struct rvtrace_component *comp)
+{
+	u32 val;
+
+	val = readl_relaxed(comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+	val &= ~(RVTRACE_TIMESTAMP_ENABLE | RVTRACE_TIMESTAMP_COUNT);
+	writel_relaxed(val, comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+}
+EXPORT_SYMBOL_GPL(timestamp_disable);
+
+struct timestamp_config *timestamp_get_config(struct rvtrace_component *comp)
+{
+	if (comp->id.type == RVTRACE_COMPONENT_TYPE_ENCODER) {
+		struct encoder_data *encoder_data = rvtrace_component_data(comp);
+		return &encoder_data->ts_config;
+	} else if (comp->id.type == RVTRACE_COMPONENT_TYPE_FUNNEL) {
+		struct funnel_data *funnel_data = rvtrace_component_data(comp);
+		return &funnel_data->ts_config;
+	} else {
+		return NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(timestamp_get_config);
+
+void timestamp_set_config(struct rvtrace_component *comp, struct device *dev,
+			  struct timestamp_config *config)
+{
+	u32 val;
+
+	val = readl_relaxed(comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+
+	val |= (FIELD_PREP(RVTRACE_TIMESTAMP_RUN_IN_DEBUG, config->run_in_debug) |
+		FIELD_PREP(RVTRACE_TIMESTAMP_MODE, config->mode) |
+		FIELD_PREP(RVTRACE_TIMESTAMP_PRESCALE, config->prescale));
+
+	writel_relaxed(val, comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+
+	/* Verify configuration was applied */
+	val = readl_relaxed(comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+
+	if (BMVAL(val, 3, 3) != config->run_in_debug) {
+		dev_warn(dev, "timestamp run_in_debug %#x not supported\n",
+			 config->run_in_debug);
+		config->run_in_debug = BMVAL(val, 3, 3);
+	}
+
+	if (BMVAL(val, 4, 6) != config->mode) {
+		dev_warn(dev, "timestamp mode %#x not supported\n",
+			 config->mode);
+		config->mode = BMVAL(val, 4, 6);
+	}
+
+	if (BMVAL(val, 8, 9) != config->prescale) {
+		dev_warn(dev, "timestamp prescale %#x not supported\n",
+			 config->prescale);
+		config->prescale = BMVAL(val, 8, 9);
+	}
+}
+EXPORT_SYMBOL_GPL(timestamp_set_config);
+
+static int rvtrace_timestamp_reset(struct rvtrace_component *comp)
+{
+	int ret;
+
+	writel_relaxed(0, comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+	ret = rvtrace_poll_bit(comp, RVTRACE_TIMESTAMP_CTRL_OFFSET,
+			       RVTRACE_TIMESTAMP_ACTIVE_SHIFT, 0);
+
+	if (ret)
+		return ret;
+
+	writel_relaxed(RVTRACE_TIMESTAMP_ACTIVE,
+			comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+	return rvtrace_poll_bit(comp, RVTRACE_TIMESTAMP_CTRL_OFFSET,
+				RVTRACE_TIMESTAMP_ACTIVE_SHIFT, 1);
+}
+
+int rvtrace_init_timestamp(struct rvtrace_component *comp,
+			   struct timestamp_config *config)
+{
+	u32 val;
+	int ret;
+
+	val = readl_relaxed(comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+	if (!FIELD_GET(RVTRACE_TIMESTAMP_ACTIVE, val)) {
+		ret = rvtrace_timestamp_reset(comp);
+		if (ret)
+			return ret;
+	}
+
+	val = readl_relaxed(comp->base + RVTRACE_TIMESTAMP_CTRL_OFFSET);
+
+	config->run_in_debug = !!(val & RVTRACE_TIMESTAMP_RUN_IN_DEBUG);
+	config->mode = BMVAL(val, 4, 6);
+	config->prescale = BMVAL(val, 8, 9);
+	config->width = BMVAL(val, 24, 29);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rvtrace_init_timestamp);
+
+static ssize_t run_in_debug_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct timestamp_config *config = timestamp_get_config(comp);
+
+	if (!config)
+		return -EINVAL;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", config->run_in_debug);
+}
+
+static ssize_t run_in_debug_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t size)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct timestamp_config *config = timestamp_get_config(comp);
+	unsigned long val;
+
+	if (!config)
+		return -EINVAL;
+
+	if (kstrtoul(buf, 10, &val))
+		return -EINVAL;
+
+	config->run_in_debug = !!val;
+
+	return size;
+}
+static DEVICE_ATTR_RW(run_in_debug);
+
+static const char * const modes_str[] = {
+	[TIMESTAMP_MODE_NONE] = "none",
+	[TIMESTAMP_MODE_EXTERNAL] = "external",
+	[TIMESTAMP_MODE_INTERNAL_SYSTEM] = "internal_system",
+	[TIMESTAMP_MODE_INTERNAL_CORE] = "internal_core",
+	[TIMESTAMP_MODE_SHARED] = "shared",
+};
+
+static ssize_t modes_available_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%s %s %s %s %s\n",
+			    modes_str[TIMESTAMP_MODE_NONE],
+			    modes_str[TIMESTAMP_MODE_EXTERNAL],
+			    modes_str[TIMESTAMP_MODE_INTERNAL_SYSTEM],
+			    modes_str[TIMESTAMP_MODE_INTERNAL_CORE],
+			    modes_str[TIMESTAMP_MODE_SHARED]);
+}
+static DEVICE_ATTR_RO(modes_available);
+
+static ssize_t mode_preferred_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct timestamp_config *config = timestamp_get_config(comp);
+
+	if (!config)
+		return -EINVAL;
+
+	return scnprintf(buf, PAGE_SIZE, "%s\n", modes_str[config->mode]);
+}
+
+static ssize_t mode_preferred_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t size)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct timestamp_config *config = timestamp_get_config(comp);
+
+	if (!config)
+		return -EINVAL;
+
+	if (sysfs_streq(buf, modes_str[TIMESTAMP_MODE_NONE]))
+		config->mode = TIMESTAMP_MODE_NONE;
+	else if (sysfs_streq(buf, modes_str[TIMESTAMP_MODE_EXTERNAL]))
+		config->mode = TIMESTAMP_MODE_EXTERNAL;
+	else if (sysfs_streq(buf, modes_str[TIMESTAMP_MODE_INTERNAL_SYSTEM]))
+		config->mode = TIMESTAMP_MODE_INTERNAL_SYSTEM;
+	else if (sysfs_streq(buf, modes_str[TIMESTAMP_MODE_INTERNAL_CORE]))
+		config->mode = TIMESTAMP_MODE_INTERNAL_CORE;
+	else if (sysfs_streq(buf, modes_str[TIMESTAMP_MODE_SHARED]))
+		config->mode = TIMESTAMP_MODE_SHARED;
+	else
+		return -EINVAL;
+	return size;
+}
+static DEVICE_ATTR_RW(mode_preferred);
+
+static ssize_t prescale_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct timestamp_config *config = timestamp_get_config(comp);
+
+	if (!config)
+		return -EINVAL;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", config->prescale);
+}
+
+static ssize_t prescale_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf, size_t size)
+{
+	unsigned long val;
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct timestamp_config *config = timestamp_get_config(comp);
+
+	if (!config)
+		return -EINVAL;
+
+	if (kstrtoul(buf, 16, &val))
+		return -EINVAL;
+
+	config->prescale = val;
+
+	return size;
+}
+static DEVICE_ATTR_RW(prescale);
+
+static ssize_t width_show(struct device *dev,
+			  struct device_attribute *attr, char *buf)
+{
+	struct rvtrace_component *comp = dev_get_drvdata(dev->parent);
+	struct timestamp_config *config = timestamp_get_config(comp);
+
+	if (!config)
+		return -EINVAL;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", config->width);
+}
+static DEVICE_ATTR_RO(width);
+
+const struct attribute *timestamp_attrs[] = {
+	&dev_attr_run_in_debug.attr,
+	&dev_attr_modes_available.attr,
+	&dev_attr_mode_preferred.attr,
+	&dev_attr_prescale.attr,
+	&dev_attr_width.attr,
+	NULL,
+};
+EXPORT_SYMBOL_GPL(timestamp_attrs);
diff --git a/drivers/hwtracing/coresight/rvtrace-timestamp.h b/drivers/hwtracing/coresight/rvtrace-timestamp.h
new file mode 100644
index 000000000000..13cecddbd82c
--- /dev/null
+++ b/drivers/hwtracing/coresight/rvtrace-timestamp.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright(C) 2026 Spacemit Limited. All rights reserved.
+ * Author: liangzhen <zhen.liang@spacemit.com>
+ */
+
+#ifndef _RVTRACE_TIMESTAMP_H
+#define _RVTRACE_TIMESTAMP_H
+
+#include <linux/types.h>
+
+/* Timestamp Control Register */
+#define RVTRACE_TIMESTAMP_CTRL_OFFSET			0x040
+#define RVTRACE_TIMESTAMP_ACTIVE			BIT(0)
+#define RVTRACE_TIMESTAMP_COUNT				BIT(1)
+#define RVTRACE_TIMESTAMP_RESET				BIT(2)
+#define RVTRACE_TIMESTAMP_RUN_IN_DEBUG			BIT(3)
+#define RVTRACE_TIMESTAMP_MODE				GENMASK(6, 4)
+#define RVTRACE_TIMESTAMP_PRESCALE			GENMASK(9, 8)
+#define RVTRACE_TIMESTAMP_ENABLE			BIT(15)
+#define RVTRACE_TIMESTAMP_WIDTH				GENMASK(29, 24)
+
+#define RVTRACE_TIMESTAMP_ACTIVE_SHIFT			0
+#define RVTRACE_TIMESTAMP_ENABLE_SHIFT			15
+
+/* Timestamp Counter Lower Bits */
+#define RVTRACE_TIMESTAMP_COUNTER_LOW			0x048
+/* Timestamp Counter Upper Bits */
+#define RVTRACE_TIMESTAMP_COUNTER_HIGH			0x04C
+
+enum timestamp_mode {
+	TIMESTAMP_MODE_NONE = 0,
+	TIMESTAMP_MODE_EXTERNAL = 1,
+	TIMESTAMP_MODE_INTERNAL_SYSTEM = 2,
+	TIMESTAMP_MODE_INTERNAL_CORE = 3,
+	TIMESTAMP_MODE_SHARED = 4
+};
+
+/**
+ * struct timestamp_config - timestamp configuration for encoder/funnel
+ * @run_in_debug:       Continue timestamp counting in debug mode
+ * @mode:		Timestamp generation mode (periodic, event-triggered)
+ * @prescale:		Clock prescale factor (1, 4, 16, 64)
+ * @width:		Timestamp counter width in bits (0-63)
+ */
+struct timestamp_config {
+	bool				run_in_debug;
+	u8				mode;
+	u8				prescale;
+	u8				width;
+};
+
+struct rvtrace_component;
+
+/* Timestamp control functions */
+int timestamp_enable(struct rvtrace_component *comp);
+void timestamp_disable(struct rvtrace_component *comp);
+struct timestamp_config *timestamp_get_config(struct rvtrace_component *comp);
+void timestamp_set_config(struct rvtrace_component *comp, struct device *dev,
+			  struct timestamp_config *config);
+int rvtrace_init_timestamp(struct rvtrace_component *comp,
+			   struct timestamp_config *config);
+
+#endif
diff --git a/drivers/i2c/Kconfig b/drivers/i2c/Kconfig
index c232054fddd6..2953c1f63211 100644
--- a/drivers/i2c/Kconfig
+++ b/drivers/i2c/Kconfig
@@ -134,6 +134,13 @@ config I2C_SLAVE_TESTUNIT
 	  multi-master, SMBus Host Notify, etc. Please read
 	  Documentation/i2c/slave-testunit-backend.rst for further details.
 
+config I2C_SPACEMIT_SLAVE
+	tristate "I2C spacemit slave driver"
+	depends on THERMAL || THERMAL=n
+	depends on CPU_FREQ || CPU_FREQ=n
+	help
+	  This backend is used by spacemit's K3 Cluster Server project.
+
 endif
 
 config I2C_DEBUG_CORE
diff --git a/drivers/i2c/Makefile b/drivers/i2c/Makefile
index d27de18de46f..6b9c3375368b 100644
--- a/drivers/i2c/Makefile
+++ b/drivers/i2c/Makefile
@@ -19,5 +19,6 @@ obj-y				+= algos/ busses/ muxes/
 obj-$(CONFIG_I2C_STUB)		+= i2c-stub.o
 obj-$(CONFIG_I2C_SLAVE_EEPROM)	+= i2c-slave-eeprom.o
 obj-$(CONFIG_I2C_SLAVE_TESTUNIT)	+= i2c-slave-testunit.o
+obj-$(CONFIG_I2C_SPACEMIT_SLAVE)	+= i2c-slave-spacemit-k3.o
 
 ccflags-$(CONFIG_I2C_DEBUG_CORE) := -DDEBUG
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index fd81e49638aa..fedf5d31f903 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -798,7 +798,7 @@ config I2C_JZ4780
 config I2C_K1
 	tristate "SpacemiT K1 I2C adapter"
 	depends on ARCH_SPACEMIT || COMPILE_TEST
-	depends on OF
+	depends on OF && COMMON_CLK
 	help
 	  This option enables support for the I2C interface on the SpacemiT K1
 	  platform.
diff --git a/drivers/i2c/busses/i2c-k1.c b/drivers/i2c/busses/i2c-k1.c
index d42c03ef5db5..de390c70f43c 100644
--- a/drivers/i2c/busses/i2c-k1.c
+++ b/drivers/i2c/busses/i2c-k1.c
@@ -4,17 +4,24 @@
  */
 
 #include <linux/bitfield.h>
- #include <linux/clk.h>
- #include <linux/i2c.h>
- #include <linux/iopoll.h>
- #include <linux/module.h>
- #include <linux/of_address.h>
- #include <linux/platform_device.h>
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/i2c.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/pm_runtime.h>
 
 /* spacemit i2c registers */
 #define SPACEMIT_ICR		 0x0		/* Control register */
 #define SPACEMIT_ISR		 0x4		/* Status register */
 #define SPACEMIT_IDBR		 0xc		/* Data buffer register */
+#define SPACEMIT_ILCR		 0x10		/* Load Count Register */
+#define SPACEMIT_IWCR		 0x14		/* Wait Count Register */
 #define SPACEMIT_IRCR		 0x18		/* Reset cycle counter */
 #define SPACEMIT_IBMR		 0x1c		/* Bus monitor register */
 
@@ -87,6 +94,13 @@
 #define SPACEMIT_BMR_SDA         BIT(0)		/* SDA line level */
 #define SPACEMIT_BMR_SCL         BIT(1)		/* SCL line level */
 
+#define SPACEMIT_LCR_LV_STANDARD_SHIFT		0
+#define SPACEMIT_LCR_LV_FAST_SHIFT		9
+#define SPACEMIT_LCR_LV_STANDARD_MASK		GENMASK(8, 0)
+#define SPACEMIT_LCR_LV_FAST_MASK		GENMASK(17, 9)
+#define SPACEMIT_LCR_LV_STANDARD_MAX_VALUE	FIELD_MAX(SPACEMIT_LCR_LV_STANDARD_MASK)
+#define SPACEMIT_LCR_LV_FAST_MAX_VALUE		FIELD_MAX(SPACEMIT_LCR_LV_FAST_MASK)
+
 /* i2c bus recover timeout: us */
 #define SPACEMIT_I2C_BUS_BUSY_TIMEOUT		100000
 
@@ -97,6 +111,19 @@
 
 #define SPACEMIT_BUS_RESET_CLK_CNT_MAX		9
 
+/* slave-related registers */
+#define SPACEMIT_SAR				0x8	   /* Slave Address Register */
+
+#define SPACEMIT_CR_SADIE			BIT(23)	   /* slave address detected int enable */
+#define SPACEMIT_CR_SSDIE			BIT(24)	   /* slave STOP detected int enable */
+
+#define SPACEMIT_SR_RWM				BIT(13)	   /* read/write mode */
+
+#define SPACEMIT_I2C_SLAVE_CRINIT		(SPACEMIT_CR_IUE | SPACEMIT_CR_ALDIE | \
+						 SPACEMIT_CR_DTEIE | SPACEMIT_CR_DRFIE | \
+						 SPACEMIT_CR_GCD | SPACEMIT_CR_BEIE | \
+						 SPACEMIT_CR_SADIE | SPACEMIT_CR_SSDIE)
+
 enum spacemit_i2c_state {
 	SPACEMIT_STATE_IDLE,
 	SPACEMIT_STATE_START,
@@ -104,15 +131,27 @@ enum spacemit_i2c_state {
 	SPACEMIT_STATE_WRITE,
 };
 
+enum spacemit_i2c_mode {
+	SPACEMIT_MODE_STANDARD,
+	SPACEMIT_MODE_FAST
+};
+
 /* i2c-spacemit driver's main struct */
 struct spacemit_i2c_dev {
 	struct device *dev;
 	struct i2c_adapter adapt;
 
+	struct clk_hw scl_clk_hw;
+	struct clk *scl_clk;
+	struct clk *func_clk;
+	struct clk *bus_clk;
+	enum spacemit_i2c_mode mode;
+
 	/* hardware resources */
 	void __iomem *base;
 	int irq;
 	u32 clock_freq;
+	struct reset_control *resets;
 
 	struct i2c_msg *msgs;
 	u32 msg_num;
@@ -127,6 +166,132 @@ struct spacemit_i2c_dev {
 	bool read;
 	struct completion complete;
 	u32 status;
+
+	/* Controls whether to bypass the controller's SDA glitch fix logic. */
+	bool sda_glitch_nofix;
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	struct i2c_client *slave;
+	bool is_slave_xfer;
+#endif
+};
+
+static int spacemit_i2c_prepare_enable_clks(struct spacemit_i2c_dev *i2c)
+{
+	int ret;
+
+	ret = clk_prepare_enable(i2c->func_clk);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(i2c->bus_clk);
+	if (ret) {
+		clk_disable_unprepare(i2c->func_clk);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void spacemit_i2c_disable_unprepare_clks(struct spacemit_i2c_dev *i2c)
+{
+	clk_disable_unprepare(i2c->bus_clk);
+	clk_disable_unprepare(i2c->func_clk);
+}
+
+static int spacemit_i2c_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long parent_rate)
+{
+	struct spacemit_i2c_dev *i2c = container_of(hw, struct spacemit_i2c_dev, scl_clk_hw);
+	u32 lv, lcr, mask, shift, max_lv;
+	u32 denom;
+
+	/*
+	 * Controller timing (from vendor formula):
+	 * - standard mode: SCL = FCLK / (2 * SLV + 0x8)
+	 * - fast mode:     SCL = FCLK / (2 * (FLV + 1) + 8)
+	 */
+	denom = DIV_ROUND_UP(parent_rate, rate);
+
+	if (i2c->mode == SPACEMIT_MODE_STANDARD) {
+		mask = SPACEMIT_LCR_LV_STANDARD_MASK;
+		shift = SPACEMIT_LCR_LV_STANDARD_SHIFT;
+		max_lv = SPACEMIT_LCR_LV_STANDARD_MAX_VALUE;
+		/*
+		 * SLV >= (denom - 8) / 2
+		 * Allow SLV=0 (max SCL = FCLK/8).
+		 */
+		lv = (denom <= 8) ? 0 : DIV_ROUND_UP(denom - 8, 2);
+	} else if (i2c->mode == SPACEMIT_MODE_FAST) {
+		mask = SPACEMIT_LCR_LV_FAST_MASK;
+		shift = SPACEMIT_LCR_LV_FAST_SHIFT;
+		max_lv = SPACEMIT_LCR_LV_FAST_MAX_VALUE;
+		/*
+		 * FLV >= (denom - 10) / 2
+		 * Allow FLV=0 (max SCL = FCLK/10).
+		 */
+		lv = (denom <= 10) ? 0 : DIV_ROUND_UP(denom - 10, 2);
+	} else {
+		return -EINVAL;
+	}
+
+	if (lv > max_lv) {
+		dev_err(i2c->dev, "set scl clock failed: lv 0x%x", lv);
+		return -EINVAL;
+	}
+
+	lcr = readl(i2c->base + SPACEMIT_ILCR);
+	lcr &= ~mask;
+	lcr |= lv << shift;
+	writel(lcr, i2c->base + SPACEMIT_ILCR);
+
+	return 0;
+}
+
+static long spacemit_i2c_clk_round_rate(struct clk_hw *hw, unsigned long rate,
+					unsigned long *parent_rate)
+{
+	struct spacemit_i2c_dev *i2c = container_of(hw, struct spacemit_i2c_dev, scl_clk_hw);
+	u32 lv, freq, denom;
+
+	denom = DIV_ROUND_UP(*parent_rate, rate);
+	if (i2c->mode == SPACEMIT_MODE_STANDARD) {
+		lv = (denom <= 8) ? 0 : DIV_ROUND_UP(denom - 8, 2);
+		freq = DIV_ROUND_UP(*parent_rate, lv * 2 + 8);
+	} else if (i2c->mode == SPACEMIT_MODE_FAST) {
+		lv = (denom <= 10) ? 0 : DIV_ROUND_UP(denom - 10, 2);
+		freq = DIV_ROUND_UP(*parent_rate, lv * 2 + 10);
+	} else {
+		return 0;
+	}
+
+	return freq;
+}
+
+static unsigned long spacemit_i2c_clk_recalc_rate(struct clk_hw *hw,
+						  unsigned long parent_rate)
+{
+	struct spacemit_i2c_dev *i2c = container_of(hw, struct spacemit_i2c_dev, scl_clk_hw);
+	u32 lcr, lv = 0;
+
+	lcr = readl(i2c->base + SPACEMIT_ILCR);
+
+	if (i2c->mode == SPACEMIT_MODE_STANDARD) {
+		lv = FIELD_GET(SPACEMIT_LCR_LV_STANDARD_MASK, lcr);
+		return DIV_ROUND_UP(parent_rate, lv * 2 + 8);
+	} else if (i2c->mode == SPACEMIT_MODE_FAST) {
+		lv = FIELD_GET(SPACEMIT_LCR_LV_FAST_MASK, lcr);
+		return DIV_ROUND_UP(parent_rate, lv * 2 + 10);
+	} else {
+		return 0;
+	}
+
+}
+
+static const struct clk_ops spacemit_i2c_clk_ops = {
+	.set_rate = spacemit_i2c_clk_set_rate,
+	.round_rate = spacemit_i2c_clk_round_rate,
+	.recalc_rate = spacemit_i2c_clk_recalc_rate,
 };
 
 static void spacemit_i2c_enable(struct spacemit_i2c_dev *i2c)
@@ -147,11 +312,45 @@ static void spacemit_i2c_disable(struct spacemit_i2c_dev *i2c)
 	writel(val, i2c->base + SPACEMIT_ICR);
 }
 
+static struct clk *spacemit_i2c_register_scl_clk(struct spacemit_i2c_dev *i2c,
+						 struct clk *parent)
+{
+	struct clk_init_data init = {};
+	char name[32];
+
+	snprintf(name, sizeof(name), "%s_scl_clk", dev_name(i2c->dev));
+
+	init.name = name;
+	init.ops = &spacemit_i2c_clk_ops;
+	init.parent_data = (struct clk_parent_data[]) {
+		{ .fw_name = "func" },
+	};
+	init.num_parents = 1;
+
+	i2c->scl_clk_hw.init = &init;
+
+	return devm_clk_register(i2c->dev, &i2c->scl_clk_hw);
+}
+
 static void spacemit_i2c_reset(struct spacemit_i2c_dev *i2c)
 {
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u32 slave_value;
+#endif
 	writel(SPACEMIT_CR_UR, i2c->base + SPACEMIT_ICR);
 	udelay(5);
 	writel(0, i2c->base + SPACEMIT_ICR);
+
+	writel(0x0000142A, i2c->base + SPACEMIT_IWCR);
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (i2c->slave) {
+		slave_value = SPACEMIT_I2C_SLAVE_CRINIT;
+		if (i2c->mode == SPACEMIT_MODE_FAST)
+			slave_value |= SPACEMIT_CR_MODE_FAST;
+		writel(slave_value, i2c->base + SPACEMIT_ICR);
+	}
+#endif
 }
 
 static int spacemit_i2c_handle_err(struct spacemit_i2c_dev *i2c)
@@ -257,7 +456,7 @@ static void spacemit_i2c_init(struct spacemit_i2c_dev *i2c)
 	 */
 	val |= SPACEMIT_CR_DRFIE;
 
-	if (i2c->clock_freq == SPACEMIT_I2C_MAX_FAST_MODE_FREQ)
+	if (i2c->mode == SPACEMIT_MODE_FAST)
 		val |= SPACEMIT_CR_MODE_FAST;
 
 	/* disable response to general call */
@@ -272,11 +471,13 @@ static void spacemit_i2c_init(struct spacemit_i2c_dev *i2c)
 	writel(val, i2c->base + SPACEMIT_ICR);
 
 	/*
-	 * The glitch fix in the K1 I2C controller introduces a delay
-	 * on restart signals, so we disable the fix here.
+	 * The K1 I2C controller has an SDA glitch fix which can suppress short
+	 * pulses on SDA, but it may also introduce a small delay on restart
+	 * (repeated-start) signals on some systems.
 	 */
 	val = readl(i2c->base + SPACEMIT_IRCR);
-	val |= SPACEMIT_RCR_SDA_GLITCH_NOFIX;
+	if (i2c->sda_glitch_nofix)
+		val |= SPACEMIT_RCR_SDA_GLITCH_NOFIX;
 	writel(val, i2c->base + SPACEMIT_IRCR);
 
 	spacemit_i2c_clear_int_status(i2c, SPACEMIT_I2C_INT_STATUS_MASK);
@@ -348,6 +549,19 @@ static bool spacemit_i2c_is_last_msg(struct spacemit_i2c_dev *i2c)
 
 static void spacemit_i2c_handle_write(struct spacemit_i2c_dev *i2c)
 {
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u8 slave_value;
+	if (i2c->is_slave_xfer) {
+		/* to confirm its not NACK */
+		if (i2c->status & SPACEMIT_SR_ACKNAK)
+			return;
+
+		i2c_slave_event(i2c->slave, I2C_SLAVE_READ_PROCESSED, &slave_value);
+		writel(slave_value, i2c->base + SPACEMIT_IDBR);
+		return;
+	}
+#endif
+
 	/* if transfer completes, SPACEMIT_ISR will handle it */
 	if (i2c->status & SPACEMIT_SR_MSD)
 		return;
@@ -365,6 +579,18 @@ static void spacemit_i2c_handle_write(struct spacemit_i2c_dev *i2c)
 
 static void spacemit_i2c_handle_read(struct spacemit_i2c_dev *i2c)
 {
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u8 slave_value;
+	if (i2c->is_slave_xfer) {
+		/* to confirm that it's triggered by IRF */
+		if (i2c->status & SPACEMIT_SR_IRF) {
+			slave_value = readl(i2c->base + SPACEMIT_IDBR);
+			i2c_slave_event(i2c->slave, I2C_SLAVE_WRITE_RECEIVED, &slave_value);
+		}
+		return;
+	}
+#endif
+
 	if (i2c->unprocessed) {
 		*i2c->msg_buf++ = readl(i2c->base + SPACEMIT_IDBR);
 		i2c->unprocessed--;
@@ -385,6 +611,20 @@ static void spacemit_i2c_handle_read(struct spacemit_i2c_dev *i2c)
 
 static void spacemit_i2c_handle_start(struct spacemit_i2c_dev *i2c)
 {
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u8 value;
+	if (i2c->is_slave_xfer) {
+		if (i2c->status & SPACEMIT_SR_RWM) {
+			i2c->state = SPACEMIT_STATE_WRITE;
+			i2c_slave_event(i2c->slave, I2C_SLAVE_READ_REQUESTED, &value);
+			writel(value, i2c->base + SPACEMIT_IDBR);
+		} else {
+			i2c->state = SPACEMIT_STATE_READ;
+			i2c_slave_event(i2c->slave, I2C_SLAVE_WRITE_REQUESTED, &value);
+		}
+		return;
+	}
+#endif
 	i2c->state = i2c->read ? SPACEMIT_STATE_READ : SPACEMIT_STATE_WRITE;
 	if (i2c->state == SPACEMIT_STATE_WRITE)
 		spacemit_i2c_handle_write(i2c);
@@ -415,13 +655,69 @@ static void spacemit_i2c_err_check(struct spacemit_i2c_dev *i2c)
 	spacemit_i2c_clear_int_status(i2c, SPACEMIT_I2C_INT_STATUS_MASK);
 
 	i2c->state = SPACEMIT_STATE_IDLE;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	i2c->is_slave_xfer = false;
+#endif
 	complete(&i2c->complete);
 }
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+static int spacemit_i2c_reg_slave(struct i2c_client *slave)
+{
+	struct spacemit_i2c_dev *i2c = i2c_get_adapdata(slave->adapter);
+	u32 val;
+	int ret;
+
+	if (i2c->slave)
+		return -EBUSY;
+
+	if (slave->flags & I2C_CLIENT_TEN)
+		return -EAFNOSUPPORT;
+
+	ret = pm_runtime_resume_and_get(i2c->dev);
+	if (ret < 0)
+		return ret;
+
+	i2c->slave = slave;
+
+	writel(slave->addr, i2c->base + SPACEMIT_SAR);
+
+	val = SPACEMIT_I2C_SLAVE_CRINIT;
+	if (i2c->mode == SPACEMIT_MODE_FAST)
+		val |= SPACEMIT_CR_MODE_FAST;
+
+	writel(val, i2c->base + SPACEMIT_ICR);
+
+	return 0;
+}
+
+static int spacemit_i2c_unreg_slave(struct i2c_client *slave)
+{
+	struct spacemit_i2c_dev *i2c = i2c_get_adapdata(slave->adapter);
+	int ret = 0;
+
+	if (!i2c->slave) {
+		dev_err(i2c->dev, "no slave registered\n");
+		ret = -EINVAL;
+	}
+
+	writel(0, i2c->base + SPACEMIT_ICR);
+	writel(0, i2c->base + SPACEMIT_SAR);
+
+	i2c->slave = NULL;
+	pm_runtime_put_autosuspend(i2c->dev);
+
+	return ret;
+}
+#endif
+
 static irqreturn_t spacemit_i2c_irq_handler(int irq, void *devid)
 {
 	struct spacemit_i2c_dev *i2c = devid;
 	u32 status, val;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u8 slave_value;
+#endif
 
 	status = readl(i2c->base + SPACEMIT_ISR);
 	if (!status)
@@ -437,6 +733,14 @@ static irqreturn_t spacemit_i2c_irq_handler(int irq, void *devid)
 	val = readl(i2c->base + SPACEMIT_ICR);
 	val &= ~(SPACEMIT_CR_TB | SPACEMIT_CR_ACKNAK | SPACEMIT_CR_STOP | SPACEMIT_CR_START);
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (i2c->slave && (status & SPACEMIT_SR_SAD)) {
+		/* slave address detected */
+		i2c->is_slave_xfer = true;
+		i2c->state = SPACEMIT_STATE_START;
+	}
+#endif
+
 	switch (i2c->state) {
 	case SPACEMIT_STATE_START:
 		spacemit_i2c_handle_start(i2c);
@@ -451,10 +755,23 @@ static irqreturn_t spacemit_i2c_irq_handler(int irq, void *devid)
 		break;
 	}
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (i2c->slave && (status & SPACEMIT_SR_SSD)) {
+		i2c_slave_event(i2c->slave, I2C_SLAVE_STOP, &slave_value);
+		i2c->state = SPACEMIT_STATE_IDLE;
+		i2c->is_slave_xfer = false;
+	}
+#endif
+
 	if (i2c->state != SPACEMIT_STATE_IDLE) {
 		val |= SPACEMIT_CR_TB | SPACEMIT_CR_ALDIE;
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+		/* Do not trigger master STOP generation if we are in a slave transfer */
+		if (!i2c->is_slave_xfer && spacemit_i2c_is_last_msg(i2c)) {
+#else
 		if (spacemit_i2c_is_last_msg(i2c)) {
+#endif
 			/* trigger next byte with stop */
 			val |= SPACEMIT_CR_STOP;
 
@@ -489,8 +806,52 @@ static void spacemit_i2c_calc_timeout(struct spacemit_i2c_dev *i2c)
 static int spacemit_i2c_xfer(struct i2c_adapter *adapt, struct i2c_msg *msgs, int num)
 {
 	struct spacemit_i2c_dev *i2c = i2c_get_adapdata(adapt);
+	bool clk_directly = false;
 	int ret;
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (i2c->slave) {
+		dev_err(i2c->dev, "working as slave mode here\n");
+		return -EBUSY;
+	}
+#endif
+
+	ret = pm_runtime_get_sync(i2c->dev);
+	if (ret < 0) {
+		/*
+		 * During system suspend_late to system resume_early stage,
+		 * if PM runtime is suspended, we will get -EACCES return
+		 * value, so we need to enable clock directly, and disable after
+		 * i2c transfer is finished. During this stage, pmic onkey ISR
+		 * that invoked in an irq thread may use i2c interface if we have
+		 * onkey press action.
+		 */
+		if (ret == -EACCES) {
+			ret = clk_prepare_enable(i2c->func_clk);
+			if (ret) {
+				dev_err(i2c->dev,
+					"failed to enable func clock directly: %d\n",
+					ret);
+				pm_runtime_put_noidle(i2c->dev);
+				return ret;
+			}
+			ret = clk_prepare_enable(i2c->bus_clk);
+			if (ret) {
+				dev_err(i2c->dev,
+					"failed to enable bus clock directly: %d\n",
+					ret);
+				clk_disable_unprepare(i2c->func_clk);
+				pm_runtime_put_noidle(i2c->dev);
+				return ret;
+			}
+			clk_directly = true;
+		} else {
+			dev_err(i2c->dev, "pm runtime sync error: %d\n", ret);
+			pm_runtime_put_noidle(i2c->dev);
+			return ret;
+		}
+	}
+
 	i2c->msgs = msgs;
 	i2c->msg_num = num;
 
@@ -515,17 +876,35 @@ static int spacemit_i2c_xfer(struct i2c_adapter *adapt, struct i2c_msg *msgs, in
 		dev_err(i2c->dev, "i2c transfer failed, ret %d err 0x%lx\n",
 			  ret, i2c->status & SPACEMIT_SR_ERR);
 
+	if (clk_directly) {
+		/* If clocks are enabled directly, here disable them */
+		clk_disable_unprepare(i2c->bus_clk);
+		clk_disable_unprepare(i2c->func_clk);
+	}
+
+	pm_runtime_mark_last_busy(i2c->dev);
+	pm_runtime_put_autosuspend(i2c->dev);
+
 	return ret < 0 ? ret : num;
 }
 
 static u32 spacemit_i2c_func(struct i2c_adapter *adap)
 {
-	return I2C_FUNC_I2C | (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK);
+	u32 flags = I2C_FUNC_I2C | (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK);
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	flags |= I2C_FUNC_SLAVE;
+#endif
+	return flags;
 }
 
 static const struct i2c_algorithm spacemit_i2c_algo = {
 	.xfer = spacemit_i2c_xfer,
 	.functionality = spacemit_i2c_func,
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	.reg_slave = spacemit_i2c_reg_slave,
+	.unreg_slave = spacemit_i2c_unreg_slave,
+#endif
 };
 
 static int spacemit_i2c_probe(struct platform_device *pdev)
@@ -540,23 +919,27 @@ static int spacemit_i2c_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
+	i2c->dev = &pdev->dev;
+
 	ret = of_property_read_u32(of_node, "clock-frequency", &i2c->clock_freq);
 	if (ret && ret != -EINVAL)
 		dev_warn(dev, "failed to read clock-frequency property: %d\n", ret);
 
+	i2c->sda_glitch_nofix = of_property_read_bool(of_node, "spacemit,sda-glitch-nofix");
+
+	i2c->dev = &pdev->dev;
 	/* For now, this driver doesn't support high-speed. */
-	if (!i2c->clock_freq || i2c->clock_freq > SPACEMIT_I2C_MAX_FAST_MODE_FREQ) {
-		dev_warn(dev, "unsupported clock frequency %u; using %u\n",
-			 i2c->clock_freq, SPACEMIT_I2C_MAX_FAST_MODE_FREQ);
+	if (i2c->clock_freq > SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ &&
+	    i2c->clock_freq <= SPACEMIT_I2C_MAX_FAST_MODE_FREQ) {
+		i2c->mode = SPACEMIT_MODE_FAST;
+	} else if (i2c->clock_freq && i2c->clock_freq <= SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ) {
+		i2c->mode = SPACEMIT_MODE_STANDARD;
+	} else {
+		dev_warn(i2c->dev, "invalid clock-frequency, fallback to fast mode");
+		i2c->mode = SPACEMIT_MODE_FAST;
 		i2c->clock_freq = SPACEMIT_I2C_MAX_FAST_MODE_FREQ;
-	} else if (i2c->clock_freq < SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ) {
-		dev_warn(dev, "unsupported clock frequency %u; using %u\n",
-			 i2c->clock_freq,  SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ);
-		i2c->clock_freq = SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ;
 	}
 
-	i2c->dev = &pdev->dev;
-
 	i2c->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->base))
 		return dev_err_probe(dev, PTR_ERR(i2c->base), "failed to do ioremap");
@@ -570,13 +953,47 @@ static int spacemit_i2c_probe(struct platform_device *pdev)
 	if (ret)
 		return dev_err_probe(dev, ret, "failed to request irq");
 
-	clk = devm_clk_get_enabled(dev, "func");
+	clk = devm_clk_get(dev, "func");
 	if (IS_ERR(clk))
-		return dev_err_probe(dev, PTR_ERR(clk), "failed to enable func clock");
+		return dev_err_probe(dev, PTR_ERR(clk), "failed to get func clock");
+
+	i2c->func_clk = clk;
+
+	i2c->scl_clk = spacemit_i2c_register_scl_clk(i2c, clk);
+	if (IS_ERR(i2c->scl_clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(i2c->scl_clk),
+				     "failed to register scl clock\n");
 
-	clk = devm_clk_get_enabled(dev, "bus");
+	clk = devm_clk_get(dev, "bus");
 	if (IS_ERR(clk))
-		return dev_err_probe(dev, PTR_ERR(clk), "failed to enable bus clock");
+		return dev_err_probe(dev, PTR_ERR(clk), "failed to get bus clock");
+
+	i2c->bus_clk = clk;
+
+	ret = spacemit_i2c_prepare_enable_clks(i2c);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "failed to prepare and enable controller clocks");
+
+	/* Get reset control */
+	i2c->resets = devm_reset_control_get_optional(dev, NULL);
+	if (IS_ERR(i2c->resets)) {
+		spacemit_i2c_disable_unprepare_clks(i2c);
+		return dev_err_probe(dev, PTR_ERR(i2c->resets), "failed to get reset control");
+	}
+
+	/* Reset the I2C controller */
+	if (i2c->resets) {
+		reset_control_assert(i2c->resets);
+		udelay(200);
+		reset_control_deassert(i2c->resets);
+	}
+
+	ret = clk_set_rate(i2c->scl_clk, i2c->clock_freq);
+	if (ret) {
+		spacemit_i2c_disable_unprepare_clks(i2c);
+		return dev_err_probe(&pdev->dev, ret, "failed to set rate for SCL clock");
+	}
 
 	spacemit_i2c_reset(i2c);
 
@@ -594,9 +1011,19 @@ static int spacemit_i2c_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, i2c);
 
+	spacemit_i2c_disable_unprepare_clks(i2c);
+
+	pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_suspended(dev);
+	pm_suspend_ignore_children(dev, 1);
+	pm_runtime_enable(dev);
+
 	ret = i2c_add_numbered_adapter(&i2c->adapt);
-	if (ret)
+	if (ret) {
+		pm_runtime_disable(dev);
 		return dev_err_probe(&pdev->dev, ret, "failed to add i2c adapter");
+	}
 
 	return 0;
 }
@@ -606,6 +1033,11 @@ static void spacemit_i2c_remove(struct platform_device *pdev)
 	struct spacemit_i2c_dev *i2c = platform_get_drvdata(pdev);
 
 	i2c_del_adapter(&i2c->adapt);
+
+	pm_runtime_force_suspend(i2c->dev);
+
+	if (i2c->resets)
+		reset_control_assert(i2c->resets);
 }
 
 static const struct of_device_id spacemit_i2c_of_match[] = {
diff --git a/drivers/i2c/i2c-slave-spacemit-k3.c b/drivers/i2c/i2c-slave-spacemit-k3.c
new file mode 100644
index 000000000000..015fb9c5ad65
--- /dev/null
+++ b/drivers/i2c/i2c-slave-spacemit-k3.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * I2C Slave Backend for Spacemit K3 Cluster Server
+ * * I2C Slave Protocol Description:
+ * Master writes 2 bytes to request specific data, then reads the response:
+ * - 0x2E, 0x03 : Request 4-byte CPU temperature (Little Endian).
+ * - 0x2E, 0x04 : Request full sys_info_pkt (approx 60 bytes).
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/i2c.h>
+#include <linux/workqueue.h>
+#include <linux/compiler.h>
+#include <linux/fs.h>
+#include <linux/string.h>
+#include <linux/of.h>
+#include <linux/sysinfo.h>
+#include <linux/cpumask.h>
+#include <linux/namei.h>
+#include <linux/statfs.h>
+#include <linux/byteorder/generic.h>
+#include <linux/cpufreq.h>
+#include <linux/thermal.h>
+#include <linux/spinlock.h>
+
+#define MIN_VALID_DISK_SIZE	2
+
+/* little endian */
+struct spacemit_sys_info {
+	char model[32];
+	u8 cpu_cores;
+	__le32 cpu_freq_khz;
+	__le32 ram_mb;
+	__le32 disk_gb;
+	char serial_number[16];
+	u8 reserved[3];
+} __packed;
+
+struct spacemit_slave_data {
+	struct i2c_client *client;
+
+	u8 rx_buf[2];
+	int rx_idx;
+	u8 tx_buf[sizeof(struct spacemit_sys_info)];
+	int tx_idx;
+	int current_tx_len;
+
+	u32 current_temp_mc;
+	struct delayed_work temp_poll_work;
+
+	struct spacemit_sys_info sys_info_pkt;
+	spinlock_t info_lock;
+
+	bool fs_info_loaded;
+	int vfs_retry_count;
+
+	struct thermal_zone_device *tz;
+};
+
+static void poll_cpu_temp_work_func(struct work_struct *work)
+{
+	struct spacemit_slave_data *data = container_of(work, struct spacemit_slave_data, temp_poll_work.work);
+	const char *model_str = NULL, *serial_str = NULL;
+	u32 freq_khz = 0, disk_gb = 0, ram_mb = 0;
+	struct spacemit_sys_info tmp_pkt = {0};
+	bool freq_ok = false, disk_ok = false;
+	struct kstatfs st;
+	struct sysinfo i;
+	struct path path;
+	unsigned long flags;
+
+	if (!data->fs_info_loaded && data->vfs_retry_count < 10) {
+
+		/* model */
+		if (of_property_read_string(of_root, "model", &model_str) == 0)
+			strscpy(tmp_pkt.model, model_str, sizeof(tmp_pkt.model));
+		else
+			strscpy(tmp_pkt.model, "Unknown", sizeof(tmp_pkt.model));
+
+		/* serial number */
+		if (of_property_read_string(of_root, "serial-number", &serial_str) == 0) {
+			strncpy(tmp_pkt.serial_number, serial_str, sizeof(tmp_pkt.serial_number));
+		} else {
+			strncpy(tmp_pkt.serial_number, "Unknown", sizeof(tmp_pkt.serial_number));
+		}
+
+		/* cpu core number */
+		tmp_pkt.cpu_cores = (u8)num_possible_cpus();
+
+		/* ram */
+		si_meminfo(&i);
+		ram_mb = (u32)(((u64)i.totalram * i.mem_unit) / (1024 * 1024));
+		tmp_pkt.ram_mb = cpu_to_le32(ram_mb);
+
+		/* cpu frequency */
+		freq_khz = cpufreq_quick_get_max(0);
+		if (freq_khz) {
+			tmp_pkt.cpu_freq_khz = cpu_to_le32(freq_khz);
+			freq_ok = true;
+		}
+
+		/* disk size mounted in / */
+		if (kern_path("/", LOOKUP_FOLLOW, &path) == 0) {
+			if (vfs_statfs(&path, &st) == 0) {
+				disk_gb = (u32)(((u64)st.f_blocks * st.f_frsize) >> 30);
+				if (disk_gb > MIN_VALID_DISK_SIZE) {
+					tmp_pkt.disk_gb = cpu_to_le32(disk_gb);
+					disk_ok = true;
+				}
+			}
+			path_put(&path);
+		}
+
+		spin_lock_irqsave(&data->info_lock, flags);
+		memcpy(&data->sys_info_pkt, &tmp_pkt, sizeof(tmp_pkt));
+		spin_unlock_irqrestore(&data->info_lock, flags);
+
+		if (freq_ok && disk_ok)
+			data->fs_info_loaded = true;
+		else
+			data->vfs_retry_count++;
+	}
+
+	/* cpu temperature */
+	if (data->tz) {
+		int temp_mc = 0;
+		if (thermal_zone_get_temp(data->tz, &temp_mc) == 0) {
+			WRITE_ONCE(data->current_temp_mc, (u32)temp_mc);
+		}
+	}
+
+	/* run again in 1s */
+	schedule_delayed_work(&data->temp_poll_work, HZ);
+}
+
+static int spacemit_slave_callback(struct i2c_client *client,
+				   enum i2c_slave_event event, u8 *val)
+{
+	struct spacemit_slave_data *data = i2c_get_clientdata(client);
+	__le32 le_temp;
+	u32 temp_val;
+
+	switch (event) {
+	case I2C_SLAVE_WRITE_REQUESTED:
+		data->rx_idx = 0;
+		break;
+
+	case I2C_SLAVE_WRITE_RECEIVED:
+		if (data->rx_idx < 2) {
+			data->rx_buf[data->rx_idx++] = *val;
+		}
+		break;
+
+	case I2C_SLAVE_READ_REQUESTED:
+		data->tx_idx = 0;
+
+		if (data->rx_idx == 2 && data->rx_buf[0] == 0x2E) {
+			if (data->rx_buf[1] == 0x03) {
+				data->current_tx_len = 4;
+				temp_val = READ_ONCE(data->current_temp_mc);
+
+				le_temp = cpu_to_le32(temp_val);
+				memcpy(data->tx_buf, &le_temp, 4);
+			} else if (data->rx_buf[1] == 0x04) {
+				data->current_tx_len = sizeof(struct spacemit_sys_info);
+
+				spin_lock(&data->info_lock);
+				memcpy(data->tx_buf, &data->sys_info_pkt, data->current_tx_len);
+				spin_unlock(&data->info_lock);
+			} else {
+				data->current_tx_len = 4;
+				memset(data->tx_buf, 0xFF, 4);
+			}
+		} else {
+			data->current_tx_len = 4;
+			memset(data->tx_buf, 0xFF, 4);
+		}
+
+		*val = data->tx_buf[data->tx_idx++];
+		break;
+
+	case I2C_SLAVE_READ_PROCESSED:
+		if (data->tx_idx < data->current_tx_len) {
+			*val = data->tx_buf[data->tx_idx++];
+		} else {
+			*val = 0xFF;
+		}
+		break;
+
+	case I2C_SLAVE_STOP:
+		data->rx_idx = 0;
+		data->tx_idx = 0;
+		break;
+
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int spacemit_slave_probe(struct i2c_client *client)
+{
+	const char *tz_name = "thermal_top";
+	struct spacemit_slave_data *data;
+	int ret;
+
+	dev_info(&client->dev, "Initializing\n");
+	data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->client = client;
+	i2c_set_clientdata(client, data);
+
+	spin_lock_init(&data->info_lock);
+
+	data->tz = thermal_zone_get_zone_by_name(tz_name);
+	if (IS_ERR(data->tz)) {
+		dev_warn(&client->dev, "Could not find thermal zone '%s', temp tracking disabled.\n", tz_name);
+		data->tz = NULL;
+	}
+
+	client->flags |= I2C_CLIENT_SLAVE;
+	ret = i2c_slave_register(client, spacemit_slave_callback);
+	if (ret)
+		return ret;
+
+	INIT_DELAYED_WORK(&data->temp_poll_work, poll_cpu_temp_work_func);
+	schedule_delayed_work(&data->temp_poll_work, msecs_to_jiffies(100));
+
+	dev_info(&client->dev, "Spacemit Slave Loaded.\n");
+	return 0;
+}
+
+static void spacemit_slave_remove(struct i2c_client *client)
+{
+	struct spacemit_slave_data *data = i2c_get_clientdata(client);
+
+	cancel_delayed_work_sync(&data->temp_poll_work);
+	i2c_slave_unregister(client);
+}
+
+static const struct of_device_id spacemit_slave_of_match[] = {
+	{ .compatible = "spacemit,i2c-slave" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, spacemit_slave_of_match);
+
+static const struct i2c_device_id spacemit_slave_id[] = {
+	{ "spacemit-i2c-slave" },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, spacemit_slave_id);
+
+static struct i2c_driver spacemit_slave_driver = {
+	.driver = {
+		.name = "spacemit_i2c_slave",
+		.of_match_table = spacemit_slave_of_match,
+	},
+	.probe = spacemit_slave_probe,
+	.remove = spacemit_slave_remove,
+	.id_table = spacemit_slave_id,
+};
+
+module_i2c_driver(spacemit_slave_driver);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("I2C Slave Backend for Spacemit K3 System Info");
diff --git a/drivers/iio/light/cm32181.c b/drivers/iio/light/cm32181.c
index bb90f738312a..b5cba8fd48a8 100644
--- a/drivers/iio/light/cm32181.c
+++ b/drivers/iio/light/cm32181.c
@@ -201,6 +201,7 @@ static int cm32181_reg_init(struct cm32181_chip *cm32181)
 		break;
 	case 0x81: /* CM32181 */
 	case 0x82: /* CM32182, fully compat. with CM32181 */
+	case 0x83: /* CM32183 */
 		cm32181->num_als_it = ARRAY_SIZE(cm32181_als_it_bits);
 		cm32181->als_it_bits = cm32181_als_it_bits;
 		cm32181->als_it_values = cm32181_als_it_values;
@@ -522,6 +523,7 @@ static DEFINE_SIMPLE_DEV_PM_OPS(cm32181_pm_ops, cm32181_suspend, cm32181_resume)
 static const struct of_device_id cm32181_of_match[] = {
 	{ .compatible = "capella,cm3218" },
 	{ .compatible = "capella,cm32181" },
+	{ .compatible = "capella,cm32183" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, cm32181_of_match);
diff --git a/drivers/input/keyboard/cros_ec_keyb.c b/drivers/input/keyboard/cros_ec_keyb.c
index 1c6b0461dc35..917698f595ce 100644
--- a/drivers/input/keyboard/cros_ec_keyb.c
+++ b/drivers/input/keyboard/cros_ec_keyb.c
@@ -25,10 +25,23 @@
 #include <linux/sysrq.h>
 #include <linux/input/matrix_keypad.h>
 #include <linux/platform_data/cros_ec_commands.h>
+#include <linux/platform_data/cros_ec_oem_commands.h>
 #include <linux/platform_data/cros_ec_proto.h>
+#include <linux/workqueue.h>
 
 #include <linux/unaligned.h>
 
+/* Maximum size of the normal key matrix, this is limited by the host command
+ * key_matrix field defined in ec_response_get_next_data_v3
+ */
+#define CROS_EC_KEYBOARD_COLS_MAX 18
+
+static_assert(EC_CMD_OEM_KB_LOCK_LED <
+	      EC_CMD_PASSTHRU_OFFSET(CROS_EC_DEV_PD_INDEX));
+
+#define CROS_EC_KB_LED_RETRY_DELAY_MS	100
+#define CROS_EC_KB_LED_MAX_RETRIES	5
+
 /**
  * struct cros_ec_keyb - Structure representing EC keyboard device
  *
@@ -43,7 +56,13 @@
  * @idev: The input device for the matrix keys.
  * @bs_idev: The input device for non-matrix buttons and switches (or NULL).
  * @notifier: interrupt event notifier for transport devices
+ * @led_sync_disabled: EC does not support the board-specific lock LED command
  * @vdata: vivaldi function row data
+ * @has_fn_map: whether the driver uses an fn function-map layer
+ * @normal_key_status: active normal keys map
+ * @fn_key_status: active function keys map
+ * @fn_key_pressed: tracks the function key status
+ * @fn_key_triggered: tracks where any function key fired
  */
 struct cros_ec_keyb {
 	unsigned int rows;
@@ -60,7 +79,22 @@ struct cros_ec_keyb {
 	struct input_dev *bs_idev;
 	struct notifier_block notifier;
 
+	struct delayed_work led_sync_work;
+	spinlock_t led_sync_lock;
+	u8 desired_lock_led_state;
+	u8 synced_lock_led_state;
+	u8 led_sync_retry_count;
+	bool led_sync_valid;
+	bool led_sync_disabled;
+
 	struct vivaldi_data vdata;
+
+	bool has_fn_map;
+	u8 normal_key_status[CROS_EC_KEYBOARD_COLS_MAX];
+	u8 fn_key_status[CROS_EC_KEYBOARD_COLS_MAX];
+	bool fn_key_pressed;
+	bool fn_key_triggered;
+	bool fn_locked;
 };
 
 /**
@@ -166,16 +200,138 @@ static bool cros_ec_keyb_has_ghosting(struct cros_ec_keyb *ckdev, uint8_t *buf)
 	return false;
 }
 
+/*
+ * Process a function key state change, send an event report if appropriate.
+ */
+static void cros_ec_keyb_process_fn_key(struct cros_ec_keyb *ckdev,
+					int row, int col, bool state)
+{
+	struct input_dev *idev = ckdev->idev;
+	int pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
+
+	ckdev->fn_key_pressed = state;
+	dev_dbg(ckdev->dev, "FN key %s (r=%d c=%d)\n",
+			     state ? "pressed" : "released", row, col);
+
+	if (state) {
+		ckdev->fn_key_triggered = false;
+	} else if (!ckdev->fn_key_triggered) {
+		/*
+		 * Send the original code if nothing else has been pressed
+		 * together with Fn.
+		 */
+		input_event(idev, EV_MSC, MSC_SCAN, pos);
+		input_report_key(idev, KEY_FN, true);
+		input_sync(idev);
+
+		input_event(idev, EV_MSC, MSC_SCAN, pos);
+		input_report_key(idev, KEY_FN, false);
+	}
+}
+
+/*
+ * Return the Fn code for a normal key row, col combination, optionally set a
+ * position code too.
+ */
+static unsigned int cros_ec_keyb_fn_code(struct cros_ec_keyb *ckdev,
+					 int row, int col, int *pos)
+{
+	struct input_dev *idev = ckdev->idev;
+	const unsigned short *keycodes = idev->keycode;
+	int fn_pos = MATRIX_SCAN_CODE(row + ckdev->rows, col, ckdev->row_shift);
+
+	if (pos)
+		*pos = fn_pos;
+
+	return keycodes[fn_pos];
+}
+
+/*
+ * Process the new state for a single key.
+ */
+static void cros_ec_keyb_process_one(struct cros_ec_keyb *ckdev,
+				     int row, int col, bool state)
+{
+	struct input_dev *idev = ckdev->idev;
+	const unsigned short *keycodes = idev->keycode;
+	int pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
+	unsigned int normal_code = keycodes[pos];
+	unsigned int code = normal_code;
+	bool effective_fn;
+	bool is_f_key = (normal_code >= KEY_F1 && normal_code <= KEY_F12);
+
+	dev_dbg(ckdev->dev, "changed: [r%d c%d]: byte %02x\n", row, col, state);
+
+	if (ckdev->has_fn_map) {
+		if (code == KEY_FN) {
+			cros_ec_keyb_process_fn_key(ckdev, row, col, state);
+			return;
+		}
+
+		if (state && ckdev->fn_key_pressed && normal_code == KEY_ESC) {
+			ckdev->fn_locked = !ckdev->fn_locked;
+			ckdev->fn_key_triggered = true;
+			dev_dbg(ckdev->dev, "Fn Lock: %d\n", ckdev->fn_locked);
+			return;
+		}
+
+		if (!state) {
+			if (ckdev->fn_key_status[col] & BIT(row)) {
+				code = cros_ec_keyb_fn_code(ckdev, row, col, &pos);
+
+				ckdev->fn_key_status[col] &= ~BIT(row);
+			} else if (ckdev->normal_key_status[col] & BIT(row)) {
+				ckdev->normal_key_status[col] &= ~BIT(row);
+			} else {
+				/* Discard, key press code was not sent */
+				return;
+			}
+		} else {
+			if (is_f_key)
+				effective_fn = ckdev->fn_key_pressed ^ ckdev->fn_locked;
+			else
+				effective_fn = ckdev->fn_key_pressed;
+
+			if (effective_fn) {
+				code = cros_ec_keyb_fn_code(ckdev, row, col, &pos);
+
+				if (!code && !ckdev->fn_key_pressed) {
+					code = normal_code;
+					effective_fn = false;
+					pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
+				}
+
+				ckdev->fn_key_triggered = true;
+				if (!code)
+					return;
+
+				if (effective_fn) {
+					ckdev->fn_key_status[col] |= BIT(row);
+					dev_dbg(
+						ckdev->dev,
+						"FN map: r=%d c=%d state=%d normal=%u -> fn=%u\n",
+						row, col, state, normal_code, code);
+				} else {
+					ckdev->normal_key_status[col] |= BIT(row);
+				}
+			} else {
+				ckdev->normal_key_status[col] |= BIT(row);
+			}
+		}
+	}
+
+	input_event(idev, EV_MSC, MSC_SCAN, pos);
+	input_report_key(idev, code, state);
+}
 
 /*
  * Compares the new keyboard state to the old one and produces key
- * press/release events accordingly.  The keyboard state is 13 bytes (one byte
+ * press/release events accordingly.  The keyboard state is one byte
  * per column)
  */
 static void cros_ec_keyb_process(struct cros_ec_keyb *ckdev,
 			 uint8_t *kb_state, int len)
 {
-	struct input_dev *idev = ckdev->idev;
 	int col, row;
 	int new_state;
 	int old_state;
@@ -192,26 +348,178 @@ static void cros_ec_keyb_process(struct cros_ec_keyb *ckdev,
 
 	for (col = 0; col < ckdev->cols; col++) {
 		for (row = 0; row < ckdev->rows; row++) {
-			int pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
-			const unsigned short *keycodes = idev->keycode;
-
 			new_state = kb_state[col] & (1 << row);
 			old_state = ckdev->old_kb_state[col] & (1 << row);
-			if (new_state != old_state) {
-				dev_dbg(ckdev->dev,
-					"changed: [r%d c%d]: byte %02x\n",
-					row, col, new_state);
-
-				input_event(idev, EV_MSC, MSC_SCAN, pos);
-				input_report_key(idev, keycodes[pos],
-						 new_state);
-			}
+
+			if (new_state == old_state)
+				continue;
+
+			cros_ec_keyb_process_one(ckdev, row, col, new_state);
 		}
 		ckdev->old_kb_state[col] = kb_state[col];
 	}
 	input_sync(ckdev->idev);
 }
 
+static u8 cros_ec_keyb_get_lock_led_state(struct input_dev *idev)
+{
+	u8 state = 0;
+
+	if (test_bit(LED_SCROLLL, idev->led))
+		state |= EC_OEM_KB_LOCK_LED_SCROLL;
+	if (test_bit(LED_NUML, idev->led))
+		state |= EC_OEM_KB_LOCK_LED_NUM;
+	if (test_bit(LED_CAPSL, idev->led))
+		state |= EC_OEM_KB_LOCK_LED_CAPS;
+
+	return state;
+}
+
+static int cros_ec_keyb_send_lock_led_cmd(struct cros_ec_keyb *ckdev, u8 lock_leds)
+{
+	DEFINE_RAW_FLEX(struct cros_ec_command, msg, data,
+			sizeof(struct ec_params_oem_kb_lock_led));
+	struct ec_params_oem_kb_lock_led *params =
+		(struct ec_params_oem_kb_lock_led *)msg->data;
+
+	msg->command = EC_CMD_OEM_KB_LOCK_LED;
+	msg->version = 0;
+	msg->outsize = sizeof(*params);
+	params->lock_leds = lock_leds;
+
+	return cros_ec_cmd_xfer_status(ckdev->ec, msg);
+}
+
+static bool cros_ec_keyb_led_sync_should_retry(int ret)
+{
+	switch (ret) {
+	case -EMSGSIZE:
+	case -EINVAL:
+	case -EOPNOTSUPP:
+	case -ENOPROTOOPT:
+		return false;
+	default:
+		return true;
+	}
+}
+
+static bool cros_ec_keyb_led_sync_is_unsupported(int ret)
+{
+	switch (ret) {
+	case -EOPNOTSUPP:
+	case -ENOPROTOOPT:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static void cros_ec_keyb_queue_led_sync(struct cros_ec_keyb *ckdev, bool force)
+{
+	unsigned long flags;
+
+	if (!ckdev->idev)
+		return;
+
+	spin_lock_irqsave(&ckdev->led_sync_lock, flags);
+	if (ckdev->led_sync_disabled) {
+		spin_unlock_irqrestore(&ckdev->led_sync_lock, flags);
+		return;
+	}
+	ckdev->desired_lock_led_state = cros_ec_keyb_get_lock_led_state(ckdev->idev);
+	if (force)
+		ckdev->led_sync_valid = false;
+	spin_unlock_irqrestore(&ckdev->led_sync_lock, flags);
+
+	mod_delayed_work(system_wq, &ckdev->led_sync_work, 0);
+}
+
+static void cros_ec_keyb_led_sync_work(struct work_struct *work)
+{
+	struct cros_ec_keyb *ckdev = container_of(to_delayed_work(work),
+						   struct cros_ec_keyb,
+						   led_sync_work);
+	unsigned long flags;
+	u8 desired_state;
+	u8 synced_state;
+	bool is_synced_valid;
+	bool is_disabled;
+	bool retry = false;
+	int ret;
+
+	if (!ckdev->idev)
+		return;
+
+	spin_lock_irqsave(&ckdev->led_sync_lock, flags);
+	desired_state = ckdev->desired_lock_led_state;
+	synced_state = ckdev->synced_lock_led_state;
+	is_synced_valid = ckdev->led_sync_valid;
+	is_disabled = ckdev->led_sync_disabled;
+	spin_unlock_irqrestore(&ckdev->led_sync_lock, flags);
+
+	if (is_disabled)
+		return;
+
+	if (is_synced_valid && desired_state == synced_state)
+		return;
+
+	ret = cros_ec_keyb_send_lock_led_cmd(ckdev, desired_state);
+
+	spin_lock_irqsave(&ckdev->led_sync_lock, flags);
+	if (!ret) {
+		ckdev->synced_lock_led_state = desired_state;
+		ckdev->led_sync_valid = true;
+		ckdev->led_sync_retry_count = 0;
+	} else if (cros_ec_keyb_led_sync_is_unsupported(ret)) {
+		ckdev->led_sync_disabled = true;
+		ckdev->led_sync_retry_count = 0;
+	} else if (cros_ec_keyb_led_sync_should_retry(ret) &&
+		   ckdev->led_sync_retry_count < CROS_EC_KB_LED_MAX_RETRIES) {
+		ckdev->led_sync_retry_count++;
+		retry = true;
+	} else {
+		ckdev->led_sync_retry_count = 0;
+	}
+	spin_unlock_irqrestore(&ckdev->led_sync_lock, flags);
+
+	if (ret) {
+		if (cros_ec_keyb_led_sync_is_unsupported(ret)) {
+			dev_dbg(ckdev->dev,
+				"lock LED sync unsupported by EC, disabling\n");
+			return;
+		}
+
+		dev_warn_ratelimited(ckdev->dev,
+				     "failed to sync lock LEDs to EC: %d\n",
+				     ret);
+		if (retry) {
+			mod_delayed_work(system_wq, &ckdev->led_sync_work,
+					 msecs_to_jiffies(CROS_EC_KB_LED_RETRY_DELAY_MS));
+		}
+	}
+}
+
+static int cros_ec_keyb_event(struct input_dev *idev, unsigned int type,
+			      unsigned int code, int value)
+{
+	struct cros_ec_keyb *ckdev = input_get_drvdata(idev);
+
+	if (type != EV_LED)
+		return 0;
+
+	switch (code) {
+	case LED_CAPSL:
+	case LED_NUML:
+	case LED_SCROLLL:
+		cros_ec_keyb_queue_led_sync(ckdev, false);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
 /**
  * cros_ec_keyb_report_bs - Report non-matrixed buttons or switches
  *
@@ -437,9 +745,15 @@ static int cros_ec_keyb_query_switches(struct cros_ec_keyb *ckdev)
 static int cros_ec_keyb_resume(struct device *dev)
 {
 	struct cros_ec_keyb *ckdev = dev_get_drvdata(dev);
+	int ret;
+
+	if (ckdev->bs_idev) {
+		ret = cros_ec_keyb_query_switches(ckdev);
+		if (ret)
+			return ret;
+	}
 
-	if (ckdev->bs_idev)
-		return cros_ec_keyb_query_switches(ckdev);
+	cros_ec_keyb_queue_led_sync(ckdev, true);
 
 	return 0;
 }
@@ -582,6 +896,48 @@ static void cros_ec_keyb_parse_vivaldi_physmap(struct cros_ec_keyb *ckdev)
 	ckdev->vdata.num_function_row_keys = n_physmap;
 }
 
+/* Returns true if there is a KEY_FN code defined in the normal keymap */
+static bool cros_ec_keyb_has_fn_key(struct cros_ec_keyb *ckdev)
+{
+	struct input_dev *idev = ckdev->idev;
+	const unsigned short *keycodes = idev->keycode;
+	int row;
+	int col;
+
+	for (row = 0; row < ckdev->rows; row++) {
+		for (col = 0; col < ckdev->cols; col++) {
+			int pos = MATRIX_SCAN_CODE(row, col, ckdev->row_shift);
+
+			if (keycodes[pos] == KEY_FN)
+				return true;
+		}
+	}
+
+	return false;
+}
+
+/*
+ * Returns true if there is a KEY_FN defined and at least one key in the fn
+ * layer keymap
+ */
+static bool cros_ec_keyb_has_fn_map(struct cros_ec_keyb *ckdev)
+{
+	int row;
+	int col;
+
+	if (!cros_ec_keyb_has_fn_key(ckdev))
+		return false;
+
+	for (row = 0; row < ckdev->rows; row++) {
+		for (col = 0; col < ckdev->cols; col++) {
+			if (cros_ec_keyb_fn_code(ckdev, row, col, NULL) != 0)
+				return true;
+		}
+	}
+
+	return false;
+}
+
 /**
  * cros_ec_keyb_register_matrix - Register matrix keys
  *
@@ -603,6 +959,12 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev)
 	if (err)
 		return err;
 
+	if (ckdev->cols > CROS_EC_KEYBOARD_COLS_MAX) {
+		dev_err(dev, "keypad,num-columns too large: %d (max: %d)\n",
+			ckdev->cols, CROS_EC_KEYBOARD_COLS_MAX);
+		return -EINVAL;
+	}
+
 	ckdev->valid_keys = devm_kzalloc(dev, ckdev->cols, GFP_KERNEL);
 	if (!ckdev->valid_keys)
 		return -ENOMEM;
@@ -635,7 +997,7 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev)
 	ckdev->ghost_filter = device_property_read_bool(dev,
 					"google,needs-ghost-filter");
 
-	err = matrix_keypad_build_keymap(NULL, NULL, ckdev->rows, ckdev->cols,
+	err = matrix_keypad_build_keymap(NULL, NULL, ckdev->rows * 2, ckdev->cols,
 					 NULL, idev);
 	if (err) {
 		dev_err(dev, "cannot build key matrix\n");
@@ -645,10 +1007,15 @@ static int cros_ec_keyb_register_matrix(struct cros_ec_keyb *ckdev)
 	ckdev->row_shift = get_count_order(ckdev->cols);
 
 	input_set_capability(idev, EV_MSC, MSC_SCAN);
+	input_set_capability(idev, EV_LED, LED_NUML);
+	input_set_capability(idev, EV_LED, LED_CAPSL);
+	input_set_capability(idev, EV_LED, LED_SCROLLL);
+	idev->event = cros_ec_keyb_event;
 	input_set_drvdata(idev, ckdev);
 	ckdev->idev = idev;
 	cros_ec_keyb_compute_valid_keys(ckdev);
 	cros_ec_keyb_parse_vivaldi_physmap(ckdev);
+	ckdev->has_fn_map = cros_ec_keyb_has_fn_map(ckdev);
 
 	err = input_register_device(ckdev->idev);
 	if (err) {
@@ -724,7 +1091,10 @@ static int cros_ec_keyb_probe(struct platform_device *pdev)
 
 	ckdev->ec = ec;
 	ckdev->dev = dev;
+	ckdev->fn_locked = true;
 	dev_set_drvdata(dev, ckdev);
+	spin_lock_init(&ckdev->led_sync_lock);
+	INIT_DELAYED_WORK(&ckdev->led_sync_work, cros_ec_keyb_led_sync_work);
 
 	if (!buttons_switches_only) {
 		err = cros_ec_keyb_register_matrix(ckdev);
@@ -750,6 +1120,7 @@ static int cros_ec_keyb_probe(struct platform_device *pdev)
 	}
 
 	device_init_wakeup(ckdev->dev, true);
+	cros_ec_keyb_queue_led_sync(ckdev, true);
 	return 0;
 }
 
@@ -757,6 +1128,7 @@ static void cros_ec_keyb_remove(struct platform_device *pdev)
 {
 	struct cros_ec_keyb *ckdev = dev_get_drvdata(&pdev->dev);
 
+	cancel_delayed_work_sync(&ckdev->led_sync_work);
 	blocking_notifier_chain_unregister(&ckdev->ec->event_notifier,
 					   &ckdev->notifier);
 }
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index cc2558630797..019259629ddb 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -992,4 +992,11 @@ config INPUT_STPMIC1_ONKEY
 	  To compile this driver as a module, choose M here: the
 	  module will be called stpmic1_onkey.
 
+config INPUT_DRV_RPMI_PWRKEY
+	tristate "Pwrkey driver based on RISC-V RPMI"
+	depends on SOC_SPACEMIT && RISCV_SBI_MPXY_MBOX
+	help
+	   Support for pwrkey based on the pwrkey service group expansioned by
+	   Spacemit.
+
 endif
diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile
index f5ebfa9d9983..1e82a6dd8de6 100644
--- a/drivers/input/misc/Makefile
+++ b/drivers/input/misc/Makefile
@@ -95,3 +95,4 @@ obj-$(CONFIG_INPUT_WM831X_ON)		+= wm831x-on.o
 obj-$(CONFIG_INPUT_XEN_KBDDEV_FRONTEND)	+= xen-kbdfront.o
 obj-$(CONFIG_INPUT_YEALINK)		+= yealink.o
 obj-$(CONFIG_INPUT_IDEAPAD_SLIDEBAR)	+= ideapad_slidebar.o
+obj-$(CONFIG_INPUT_DRV_RPMI_PWRKEY)	+= pwrkey-rpmi.o
diff --git a/drivers/input/misc/pwrkey-rpmi.c b/drivers/input/misc/pwrkey-rpmi.c
new file mode 100644
index 000000000000..0d8bc76668c9
--- /dev/null
+++ b/drivers/input/misc/pwrkey-rpmi.c
@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RISC-V MPXY Based PWRKEY Driver
+ *
+ * Copyright (C) 2025 Spacemit.
+ */
+
+#include <linux/rtc.h>
+#include <linux/err.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/input.h>
+#include <linux/pm_wakeirq.h>
+#include <linux/notifier.h>
+#include <linux/suspend.h>
+#include <linux/platform_device.h>
+#include <linux/mailbox/riscv-rpmi-message.h>
+
+static int report_event, fall_triggered;
+static struct notifier_block   pm_notify;
+static spinlock_t pm_lock;
+
+/** RPMI pwrkey service IDs */
+enum rpmi_pwrkey_service_id {
+	RPMI_PWRKEY_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_PWRKEY_SRV_QUERY_PENDING = 0x02,
+	RPMI_PWRKEY_SRV_CLR_PENDING = 0x03,
+	RPMI_PWRKEY_SRV_ID_MAX_COUNT,
+};
+
+struct rpmi_pwrkey_query_pending_req {
+	u32 dummy;
+};
+
+struct rpmi_pwrkey_query_pending_resp {
+#define RPMI_PWRKEY_RELEASE_OFFSET	(1 << 0)
+#define RPMI_PWRKEY_PRESS_OFFSET	(1 << 1)
+	s32 status;
+};
+
+struct rpmi_pwrkey_clear_pending_req {
+#define RPMI_PWRKEY_RELEASE_OFFSET	(1 << 0)
+#define RPMI_PWRKEY_PRESS_OFFSET	(1 << 1)
+	u32 clear;
+};
+
+struct rpmi_pwrkey_clear_pending_resp {
+	s32 status;
+};
+
+struct rpmi_pwrkey_context {
+	int virt_irq;
+	struct device *dev;
+	struct input_dev *input;
+	struct mbox_chan *chan;
+	struct mbox_client client;
+	u32 max_msg_data_size;
+};
+
+static int pwrk_pm_notify(struct notifier_block *notify_block,
+			unsigned long mode, void *unused)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&pm_lock, flags);
+
+	switch (mode) {
+	case PM_SUSPEND_PREPARE:
+		/* don't report power-key when enter suspend */
+		report_event = 0;
+		break;
+
+	case PM_POST_SUSPEND:
+		/* restore report power-key */
+		report_event = 1;
+		break;
+	}
+
+	spin_unlock_irqrestore(&pm_lock, flags);
+
+	return 0;
+}
+
+static irqreturn_t mpxy_pwrkey_irq_event(int irq, void *dev_id)
+{
+	/* We only have MSI for notification so just wakeup IRQ thread */
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t mpxy_pwrkey_irq_thread(int irq, void *dev_id)
+{
+	int ret;
+	unsigned long flags;
+	struct rpmi_mbox_message msg;
+	struct rpmi_pwrkey_context *context = dev_id;
+	struct rpmi_pwrkey_query_pending_req alarmtx;
+	struct rpmi_pwrkey_query_pending_resp alarmrx;
+	struct rpmi_pwrkey_clear_pending_req alarmctx;
+	struct rpmi_pwrkey_clear_pending_resp alarmcrx;
+
+	rpmi_mbox_init_send_with_response(&msg, RPMI_PWRKEY_SRV_QUERY_PENDING,
+					  &alarmtx, sizeof(alarmtx), &alarmrx, sizeof(alarmrx));
+	ret = rpmi_mbox_send_message(context->chan, &msg);
+	if (ret)
+		return IRQ_HANDLED;
+
+	if (alarmrx.status) {
+		spin_lock_irqsave(&pm_lock, flags);
+
+		/* update the power key event */
+		if (alarmrx.status & RPMI_PWRKEY_PRESS_OFFSET) {
+			if (report_event) {
+				input_report_key(context->input, KEY_POWER, 1);
+				input_sync(context->input);
+				fall_triggered = 1;
+			}
+		}
+
+		if (alarmrx.status & RPMI_PWRKEY_RELEASE_OFFSET) {
+			if (fall_triggered) {
+				input_report_key(context->input, KEY_POWER, 0);
+				input_sync(context->input);
+				fall_triggered = 0;
+			}
+		}
+
+		spin_unlock_irqrestore(&pm_lock, flags);
+
+		pm_wakeup_event(context->dev, 0);
+
+		/* clear pending */
+		alarmctx.clear = alarmrx.status;
+		rpmi_mbox_init_send_with_response(&msg, RPMI_PWRKEY_SRV_CLR_PENDING,
+					  &alarmctx, sizeof(alarmctx), &alarmcrx, sizeof(alarmcrx));
+		ret = rpmi_mbox_send_message(context->chan, &msg);
+		if (ret)
+			return IRQ_HANDLED;
+
+		if (alarmcrx.status)
+			return IRQ_HANDLED;
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int rpmi_pwrkey_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct device *dev = &pdev->dev;
+	struct rpmi_pwrkey_context *context;
+	struct rpmi_mbox_message msg;
+
+	/* Allocate RPMI rtc context */
+	context = devm_kzalloc(dev, sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return -ENOMEM;
+	context->dev = dev;
+	platform_set_drvdata(pdev, context);
+
+	/* Setup mailbox client */
+	context->client.dev		= context->dev;
+	context->client.rx_callback	= NULL;
+	context->client.tx_block	= false;
+	context->client.knows_txdone	= true;
+	context->client.tx_tout		= 0;
+
+	/* Request mailbox channel */
+	context->chan = mbox_request_channel(&context->client, 0);
+	if (IS_ERR(context->chan))
+		return PTR_ERR(context->chan);
+
+	/* Validate RPMI specification version */
+	rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_SPEC_VERSION);
+	ret = rpmi_mbox_send_message(context->chan, &msg);
+	if (ret) {
+		dev_err_probe(dev, ret, "Failed to get spec version\n");
+		goto fail_free_channel;
+	}
+	if (msg.attr.value < RPMI_MKVER(1, 0)) {
+		ret = dev_err_probe(dev, -EINVAL,
+				    "msg protocol version mismatch, expected 0x%x, found 0x%x\n",
+				    RPMI_MKVER(1, 0), msg.attr.value);
+		goto fail_free_channel;
+	}
+
+	/* Save the maximum message data size of mailbox channel */
+	rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_MAX_MSG_DATA_SIZE);
+	ret = rpmi_mbox_send_message(context->chan, &msg);
+	if (ret) {
+		dev_err_probe(dev, ret, "Failed to get max message data size\n");
+		goto fail_free_channel;
+	}
+	context->max_msg_data_size = msg.attr.value;
+
+	/* register input event */
+	context->input = devm_input_allocate_device(&pdev->dev);
+	if (context->input == NULL) {
+		dev_err_probe(dev, -EINVAL, "Failed to allocate input device\n");
+		goto fail_free_channel;
+	}
+
+        context->input->name = "rpmi pwrkey";
+        context->input->phys = "rpmi-pwrkey/input0";
+        context->input->id.bustype = BUS_HOST;
+        input_set_capability(context->input, EV_KEY, KEY_POWER);
+
+	ret = input_register_device(context->input);
+	if (ret) {
+		dev_err(dev, "failed to register input device\n");
+		goto fail_free_channel;
+	}
+
+	spin_lock_init(&pm_lock);
+
+	context->virt_irq = platform_get_irq_byname(pdev, "rpmi pwrkey");
+	ret = request_threaded_irq(context->virt_irq,
+				  mpxy_pwrkey_irq_event,
+				  mpxy_pwrkey_irq_thread,
+				  IRQF_SHARED | IRQF_ONESHOT, dev_name(dev), context);
+	if (ret) {
+		dev_err(dev, "failed to request MPXY channel IRQ\n");
+		goto fail_free_channel;
+	}
+
+        dev_pm_set_wake_irq(&pdev->dev, context->virt_irq);
+        device_init_wakeup(&pdev->dev, true);
+
+	pm_notify.notifier_call = pwrk_pm_notify;
+	ret = register_pm_notifier(&pm_notify);
+	if (ret) {
+		dev_err(&pdev->dev, "Register pm notifier failed\n");
+		free_irq(context->virt_irq, context);
+		goto fail_free_channel;
+	}
+
+	return 0;
+
+fail_free_channel:
+	mbox_free_channel(context->chan);
+	return ret;
+}
+
+static const struct of_device_id rpmi_pwrkey_of_match[] = {
+	{ .compatible = "riscv,rpmi-pwrkey" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, rpmi_pwrkey_of_match);
+
+static struct platform_driver rpmi_pwrkey_driver = {
+	.driver = {
+		.name = "riscv-rpmi-pwrkey",
+		.of_match_table = rpmi_pwrkey_of_match,
+	},
+	.probe = rpmi_pwrkey_probe,
+};
+module_platform_driver(rpmi_pwrkey_driver);
+
+MODULE_AUTHOR("xianbin.zhu <xianbin.zhu@linux.spacemit.com>");
+MODULE_DESCRIPTION("PWRKEY Driver based on RPMI message protocol");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig
index 7d5b72ee07fa..1f48f1aaa0bd 100644
--- a/drivers/input/touchscreen/Kconfig
+++ b/drivers/input/touchscreen/Kconfig
@@ -817,6 +817,12 @@ config TOUCHSCREEN_RASPBERRYPI_FW
 	  To compile this driver as a module, choose M here: the
 	  module will be called raspberrypi-ts.
 
+config TOUCHSCREEN_RASPITS_FT5426
+	tristate "Raspberry Pi 7 inch touchscreen touch control IC FT5426"
+	depends on I2C
+	help
+	  Control FT5426 touch IC.
+
 config TOUCHSCREEN_MIGOR
 	tristate "Renesas MIGO-R touchscreen"
 	depends on (SH_MIGOR || COMPILE_TEST) && I2C
diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile
index ab9abd151078..a26f9f609fb7 100644
--- a/drivers/input/touchscreen/Makefile
+++ b/drivers/input/touchscreen/Makefile
@@ -116,6 +116,7 @@ obj-$(CONFIG_TOUCHSCREEN_ZFORCE)	+= zforce_ts.o
 obj-$(CONFIG_TOUCHSCREEN_COLIBRI_VF50)	+= colibri-vf50-ts.o
 obj-$(CONFIG_TOUCHSCREEN_ROHM_BU21023)	+= rohm_bu21023.o
 obj-$(CONFIG_TOUCHSCREEN_RASPBERRYPI_FW)	+= raspberrypi-ts.o
+obj-$(CONFIG_TOUCHSCREEN_RASPITS_FT5426)	+= raspits_ft5426.o
 obj-$(CONFIG_TOUCHSCREEN_IQS5XX)	+= iqs5xx.o
 obj-$(CONFIG_TOUCHSCREEN_IQS7211)	+= iqs7211.o
 obj-$(CONFIG_TOUCHSCREEN_ZINITIX)	+= zinitix.o
diff --git a/drivers/input/touchscreen/raspits_ft5426.c b/drivers/input/touchscreen/raspits_ft5426.c
new file mode 100644
index 000000000000..c76841358173
--- /dev/null
+++ b/drivers/input/touchscreen/raspits_ft5426.c
@@ -0,0 +1,307 @@
+/*
+ *
+ * Raspberry Pi 7 inch Touchscreen FT5426 touch driver.
+ *
+ * Copyright (c) 2016 ASUSTek Computer Inc.
+ * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/input.h>
+#include <linux/input/mt.h>
+#include <linux/module.h>
+#include <linux/workqueue.h>
+#include "raspits_ft5426.h"
+
+static int fts_i2c_read(struct i2c_client *client, char *writebuf,
+			   int writelen, char *readbuf, int readlen)
+{
+	int ret;
+
+	if (writelen > 0) {
+		struct i2c_msg msgs[] = {
+			{
+				 .addr = client->addr,
+				 .flags = 0,
+				 .len = writelen,
+				 .buf = writebuf,
+			 },
+			{
+				 .addr = client->addr,
+				 .flags = I2C_M_RD,
+				 .len = readlen,
+				 .buf = readbuf,
+			 },
+		};
+		ret = i2c_transfer(client->adapter, msgs, 2);
+		if (ret < 0)
+			LOG_INFO("i2c read issue, please check touchscreen connection, %d\n", ret);
+	} else {
+		struct i2c_msg msgs[] = {
+			{
+				 .addr = client->addr,
+				 .flags = I2C_M_RD,
+				 .len = readlen,
+				 .buf = readbuf,
+			 },
+		};
+		ret = i2c_transfer(client->adapter, msgs, 1);
+		if (ret < 0)
+			LOG_INFO("i2c read issue, please check touchscreen connection, %d\n", ret);
+	}
+
+	return ret;
+}
+
+static int fts_read_reg(struct i2c_client *client, u8 addr, u8 *val)
+{
+	return fts_i2c_read(client, &addr, 1, val, 1);
+}
+
+static int fts_check_fw_ver(struct i2c_client *client)
+{
+	u8 reg_addr, fw_ver[3];
+	int ret;
+
+	reg_addr = FT_REG_FW_VER;
+	ret = fts_i2c_read(client, &reg_addr, 1, &fw_ver[0], 1);
+	if (ret < 0)
+		goto error;
+
+	reg_addr = FT_REG_FW_MIN_VER;
+	ret = fts_i2c_read(client, &reg_addr, 1, &fw_ver[1], 1);
+	if (ret < 0)
+		goto error;
+
+	reg_addr = FT_REG_FW_SUB_MIN_VER;
+	ret = fts_i2c_read(client, &reg_addr, 1, &fw_ver[2], 1);
+	if (ret < 0)
+		goto error;
+
+	LOG_INFO("Firmware version = %d.%d.%d\n", fw_ver[0], fw_ver[1], fw_ver[2]);
+	return 0;
+
+error:
+	return ret;
+}
+
+static int fts_read_td_status(struct raspits_ft5426_data *ts_data)
+{
+	u8 td_status;
+	int ret = -1;
+	ret = fts_read_reg(ts_data->client, FT_TD_STATUS_REG, &td_status);
+	if (ret < 0) {
+		LOG_ERR("get reg td_status failed, %d\n", ret);
+		return ret;
+	}
+	return (int)td_status;
+}
+
+static int fts_read_touchdata(struct raspits_ft5426_data *ts_data)
+{
+	struct ts_event *event = &ts_data->event;
+	int ret = -1, i;
+	u8 buf[FT_ONE_TCH_LEN-2] = { 0 };
+	u8 reg_addr, pointid = FT_MAX_ID;
+
+	for (i = 0; i < event->touch_point && i < MAX_TOUCH_POINTS; i++) {
+		reg_addr = FT_TOUCH_X_H_REG + (i * FT_ONE_TCH_LEN);
+		ret = fts_i2c_read(ts_data->client, &reg_addr, 1, buf, FT_ONE_TCH_LEN-2);
+		if (ret < 0) {
+			LOG_ERR("read touchdata failed.\n");
+			return ret;
+		}
+
+		pointid = (buf[FT_TOUCH_ID]) >> 4;
+		if (pointid >= MAX_TOUCH_POINTS)
+			break;
+		event->au8_finger_id[i] = pointid;
+		event->au16_x[i] = (s16) (buf[FT_TOUCH_X_H] & 0x0F) << 8 | (s16) buf[FT_TOUCH_X_L];
+		event->au16_y[i] = (s16) (buf[FT_TOUCH_Y_H] & 0x0F) << 8 | (s16) buf[FT_TOUCH_Y_L];
+		event->au8_touch_event[i] = buf[FT_TOUCH_EVENT] >> 6;
+
+#if XY_REVERSE
+		event->au16_x[i] = SCREEN_WIDTH - event->au16_x[i] - 1;
+		event->au16_y[i] = SCREEN_HEIGHT - event->au16_y[i] - 1;
+#endif
+	}
+	event->pressure = FT_PRESS;
+
+	return 0;
+}
+
+static void fts_report_value(struct raspits_ft5426_data *ts_data)
+{
+	struct ts_event *event = &ts_data->event;
+	int i, modified_ids = 0, released_ids;
+
+	for (i = 0; i < event->touch_point && i < MAX_TOUCH_POINTS; i++) {
+		if (event->au8_touch_event[i]== FT_TOUCH_DOWN
+				|| event->au8_touch_event[i] == FT_TOUCH_CONTACT)
+		{
+			modified_ids |= 1 << event->au8_finger_id[i];
+			input_mt_slot(ts_data->input_dev, event->au8_finger_id[i]);
+			input_mt_report_slot_state(ts_data->input_dev, MT_TOOL_FINGER,
+				true);
+			input_report_abs(ts_data->input_dev, ABS_MT_TOUCH_MAJOR,
+					event->pressure);
+			input_report_abs(ts_data->input_dev, ABS_MT_POSITION_X,
+					event->au16_x[i]);
+			input_report_abs(ts_data->input_dev, ABS_MT_POSITION_Y,
+					event->au16_y[i]);
+
+			if(!((1 << event->au8_finger_id[i]) & ts_data->known_ids))
+				LOG_DBG("Touch id-%d: x = %d, y = %d\n",
+					event->au8_finger_id[i], event->au16_x[i], event->au16_y[i]);
+		}
+	}
+
+	released_ids = ts_data->known_ids & ~modified_ids;
+	for(i = 0; released_ids && i < MAX_TOUCH_POINTS; i++) {
+		if(released_ids & (1<<i)) {
+			LOG_DBG("Release id-%d, known = %x modified = %x\n", i, ts_data->known_ids, modified_ids);
+			input_mt_slot(ts_data->input_dev, i);
+			input_mt_report_slot_state(ts_data->input_dev, MT_TOOL_FINGER, false);
+			modified_ids &= ~(1 << i);
+		}
+	}
+	ts_data->known_ids = modified_ids;
+	input_mt_report_pointer_emulation(ts_data->input_dev, true);
+	input_sync(ts_data->input_dev);
+}
+
+static void raspits_ft5426_work(struct work_struct *work)
+{
+	struct raspits_ft5426_data *ts_data
+			= container_of(work, struct raspits_ft5426_data, ft5426_work);
+	struct ts_event *event = &ts_data->event;
+	int ret = 0, count = 8, td_status;
+
+	while(count > 0) {
+		ret = fts_check_fw_ver(ts_data->client);
+		if (ret == 0)
+			break;
+		LOG_INFO("checking touch ic, countdown: %d\n", count);
+		msleep(1000);
+		count--;
+	}
+	if (!count) {
+		LOG_ERR("checking touch ic timeout, %d\n", ret);
+		return;
+	}
+
+	//polling 60fps
+	while(1) {
+		td_status = fts_read_td_status(ts_data);
+		if (td_status < VALID_TD_STATUS_VAL+1 && (td_status > 0 || ts_data->known_ids != 0)) {
+			memset(event, -1, sizeof(struct ts_event));
+			event->touch_point = td_status;
+			ret = fts_read_touchdata(ts_data);
+			if (ret == 0)
+				fts_report_value(ts_data);
+		}
+		msleep_interruptible(17);
+	}
+}
+
+static int raspits_ft5426_probe(struct i2c_client *client)
+{
+	struct raspits_ft5426_data *ts_data;
+	struct input_dev *input_dev;
+	int ret = 0;//, timeout = 10;
+
+	LOG_INFO("address = 0x%x\n", client->addr);
+
+	ts_data = kzalloc(sizeof(struct raspits_ft5426_data), GFP_KERNEL);
+	if (ts_data == NULL) {
+		LOG_ERR("no memory for device\n");
+		return -ENOMEM;
+	}
+
+	ts_data->client = client;
+	i2c_set_clientdata(client, ts_data);
+
+	input_dev = input_allocate_device();
+	if (!input_dev) {
+		LOG_ERR("failed to allocate input device\n");
+		goto input_allocate_failed;
+	}
+	input_dev->name = "fts_ts";
+	input_dev->id.bustype = BUS_I2C;
+	input_dev->dev.parent = &ts_data->client->dev;
+
+	ts_data->input_dev = input_dev;
+	input_set_drvdata(input_dev, ts_data);
+
+	__set_bit(EV_SYN, input_dev->evbit);
+	__set_bit(EV_KEY, input_dev->evbit);
+	__set_bit(EV_ABS, input_dev->evbit);
+	__set_bit(BTN_TOUCH, input_dev->keybit);
+
+	input_mt_init_slots(input_dev, MAX_TOUCH_POINTS, 0);
+	input_set_abs_params(input_dev, ABS_MT_POSITION_X, 0,
+			     SCREEN_WIDTH, 0, 0);
+	input_set_abs_params(input_dev, ABS_MT_POSITION_Y, 0,
+			     SCREEN_HEIGHT, 0, 0);
+
+	ret = input_register_device(input_dev);
+	if (ret) {
+		LOG_ERR("Input device registration failed\n");
+		goto input_register_failed;
+	}
+
+	INIT_WORK(&ts_data->ft5426_work, raspits_ft5426_work);
+	schedule_work(&ts_data->ft5426_work);
+
+	return 0;
+
+input_register_failed:
+	input_free_device(input_dev);
+input_allocate_failed:
+	kfree(ts_data);
+	return ret;
+}
+
+static void raspits_ft5426_remove(struct i2c_client *client)
+{
+	struct raspits_ft5426_data *ts_data = i2c_get_clientdata(client);
+
+	cancel_work_sync(&ts_data->ft5426_work);
+	if (ts_data->input_dev) {
+		input_unregister_device(ts_data->input_dev);
+		input_free_device(ts_data->input_dev);
+	}
+	kfree(ts_data);
+	return;
+}
+
+static const struct i2c_device_id raspits_ft5426_id[] = {
+	{ "raspits_ft5426", 0 },
+	{ },
+};
+
+static struct i2c_driver raspits_ft5426_driver = {
+	.driver = {
+		.name = "raspits_ft5426",
+	},
+	.probe = raspits_ft5426_probe,
+	.remove = raspits_ft5426_remove,
+	.id_table = raspits_ft5426_id,
+};
+module_i2c_driver(raspits_ft5426_driver);
+
+MODULE_DESCRIPTION("Raspberry Pi 7 inch Touchscreen FT5426 Touch driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/touchscreen/raspits_ft5426.h b/drivers/input/touchscreen/raspits_ft5426.h
new file mode 100644
index 000000000000..fa98445e1c58
--- /dev/null
+++ b/drivers/input/touchscreen/raspits_ft5426.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _RASPITS_FT5426_H_
+#define _RASPITS_FT5426_H_
+
+#define LOG_DBG(fmt,arg...) pr_debug("raspits-ft5426: %s: "fmt, __func__, ##arg);
+#define LOG_INFO(fmt,arg...) pr_info("raspits-ft5426: %s: "fmt, __func__, ##arg);
+#define LOG_ERR(fmt,arg...) pr_err("raspits-ft5426: %s: "fmt, __func__, ##arg);
+
+#define XY_REVERSE 1
+
+#define SCREEN_WIDTH	800
+#define SCREEN_HEIGHT	480
+
+#define FT_ONE_TCH_LEN	6
+
+#define FT_REG_FW_VER			0xA6
+#define FT_REG_FW_MIN_VER		0xB2
+#define FT_REG_FW_SUB_MIN_VER	0xB3
+
+#define VALID_TD_STATUS_VAL		10
+#define MAX_TOUCH_POINTS		1
+
+#define FT_PRESS			0x7F
+#define FT_MAX_ID			0x0F
+
+#define FT_TOUCH_X_H	0
+#define FT_TOUCH_X_L	1
+#define FT_TOUCH_Y_H	2
+#define FT_TOUCH_Y_L	3
+#define FT_TOUCH_EVENT	0
+#define FT_TOUCH_ID		2
+
+#define FT_TOUCH_X_H_REG	3
+#define FT_TOUCH_X_L_REG	4
+#define FT_TOUCH_Y_H_REG	5
+#define FT_TOUCH_Y_L_REG	6
+#define FT_TD_STATUS_REG	2
+#define FT_TOUCH_EVENT_REG	3
+#define FT_TOUCH_ID_REG		5
+
+#define FT_TOUCH_DOWN		0
+#define FT_TOUCH_CONTACT	2
+
+struct ts_event {
+	u16 au16_x[MAX_TOUCH_POINTS]; /*x coordinate */
+	u16 au16_y[MAX_TOUCH_POINTS]; /*y coordinate */
+	u8 au8_touch_event[MAX_TOUCH_POINTS]; /*touch event: 0:down; 1:up; 2:contact */
+	u8 au8_finger_id[MAX_TOUCH_POINTS]; /*touch ID */
+	u16 pressure;
+	u8 touch_point;
+	u8 point_num;
+};
+
+struct raspits_ft5426_data {
+	struct device *dev;
+	struct i2c_client *client;
+	struct input_dev *input_dev;
+	struct ts_event event;
+	struct work_struct ft5426_work;
+
+	int known_ids;
+};
+
+#endif
diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig
index c071816f59a6..fea37936adf8 100644
--- a/drivers/iommu/riscv/Kconfig
+++ b/drivers/iommu/riscv/Kconfig
@@ -18,3 +18,10 @@ config RISCV_IOMMU_PCI
 	def_bool y if RISCV_IOMMU && PCI_MSI
 	help
 	  Support for the PCIe implementation of RISC-V IOMMU architecture.
+
+config RISCV_IOMMU_SPACEMIT_K3
+	bool "SpacemiT K3 IOMMU Support"
+	depends on RISCV_IOMMU
+	default n
+	help
+	  Support for the RISC-V IOMMU on SpacemiT K3 SoCs.
diff --git a/drivers/iommu/riscv/Makefile b/drivers/iommu/riscv/Makefile
index b5929f9f23e6..046aadbbaebd 100644
--- a/drivers/iommu/riscv/Makefile
+++ b/drivers/iommu/riscv/Makefile
@@ -1,3 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-y += iommu.o iommu-platform.o
 obj-$(CONFIG_RISCV_IOMMU_PCI) += iommu-pci.o
+obj-$(CONFIG_RISCV_IOMMU_SPACEMIT_K3) += iommu-spacemit-k3.o
diff --git a/drivers/iommu/riscv/iommu-spacemit-k3.c b/drivers/iommu/riscv/iommu-spacemit-k3.c
new file mode 100644
index 000000000000..418651ac2e9f
--- /dev/null
+++ b/drivers/iommu/riscv/iommu-spacemit-k3.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RISC-V IOMMU as a platform device
+ *
+ * Copyright © 2023 FORTH-ICS/CARV
+ * Copyright © 2023-2024 Rivos Inc.
+ * Copyright © 2026 SpacemiT
+ *
+ * Authors
+ *	Nick Kossifidis <mick@ics.forth.gr>
+ *	Tomasz Jeznach <tjeznach@rivosinc.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/irqchip/riscv-imsic.h>
+#include <linux/kernel.h>
+#include <linux/msi.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+#include "iommu-bits.h"
+#include "iommu.h"
+
+struct k3_iommu_device {
+	struct riscv_iommu_device iommu;
+
+	struct clk *clk;
+};
+
+static void riscv_iommu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
+{
+	struct device *dev = msi_desc_to_dev(desc);
+	struct riscv_iommu_device *iommu = dev_get_drvdata(dev);
+	u16 idx = desc->msi_index;
+	u64 addr;
+
+	addr = ((u64)msg->address_hi << 32) | msg->address_lo;
+
+	if (addr != (addr & RISCV_IOMMU_MSI_CFG_TBL_ADDR)) {
+		dev_err_once(dev,
+			     "uh oh, the IOMMU can't send MSIs to 0x%llx, sending to 0x%llx instead\n",
+			     addr, addr & RISCV_IOMMU_MSI_CFG_TBL_ADDR);
+	}
+
+	addr &= RISCV_IOMMU_MSI_CFG_TBL_ADDR;
+
+	riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_ADDR(idx), addr);
+	riscv_iommu_writel(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_DATA(idx), msg->data);
+	riscv_iommu_writel(iommu, RISCV_IOMMU_REG_MSI_CFG_TBL_CTRL(idx), 0);
+}
+
+static int k3_iommu_platform_probe(struct platform_device *pdev)
+{
+	enum riscv_iommu_igs_settings igs;
+	struct device *dev = &pdev->dev;
+	struct k3_iommu_device *k3_iommu = NULL;
+	struct riscv_iommu_device *iommu = NULL;
+	struct irq_domain *msi_domain;
+	struct resource *res = NULL;
+	int vec, ret;
+
+	k3_iommu = devm_kzalloc(dev, sizeof(*k3_iommu), GFP_KERNEL);
+	if (!k3_iommu)
+		return -ENOMEM;
+
+	iommu = &k3_iommu->iommu;
+
+	iommu->dev = dev;
+	iommu->reg = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
+	if (IS_ERR(iommu->reg))
+		return dev_err_probe(dev, PTR_ERR(iommu->reg),
+				     "could not map register region\n");
+
+	k3_iommu->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(k3_iommu->clk))
+		return dev_err_probe(dev, PTR_ERR(k3_iommu->clk),
+				     "could not get clk\n");
+	ret = clk_prepare_enable(k3_iommu->clk);
+	if (ret)
+		return dev_err_probe(dev, -ENODEV,
+				     "clk_prepare_enable failed\n");
+
+	dev_set_drvdata(dev, iommu);
+
+	/* Check device reported capabilities / features. */
+	iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES);
+	iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
+
+	iommu->irqs_count = platform_irq_count(pdev);
+	if (iommu->irqs_count <= 0)
+		return dev_err_probe(dev, -ENODEV,
+				     "no IRQ resources provided\n");
+	if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT)
+		iommu->irqs_count = RISCV_IOMMU_INTR_COUNT;
+
+	igs = FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps);
+	switch (igs) {
+	case RISCV_IOMMU_CAPABILITIES_IGS_BOTH:
+	case RISCV_IOMMU_CAPABILITIES_IGS_MSI:
+		if (is_of_node(dev_fwnode(dev))) {
+			of_msi_configure(dev, to_of_node(dev->fwnode));
+		} else {
+			msi_domain = irq_find_matching_fwnode(imsic_acpi_get_fwnode(dev),
+							      DOMAIN_BUS_PLATFORM_MSI);
+			dev_set_msi_domain(dev, msi_domain);
+		}
+
+		if (!dev_get_msi_domain(dev)) {
+			dev_warn(dev, "failed to find an MSI domain\n");
+			goto msi_fail;
+		}
+
+		ret = platform_device_msi_init_and_alloc_irqs(dev, iommu->irqs_count,
+							      riscv_iommu_write_msi_msg);
+		if (ret) {
+			dev_warn(dev, "failed to allocate MSIs\n");
+			goto msi_fail;
+		}
+
+		for (vec = 0; vec < iommu->irqs_count; vec++)
+			iommu->irqs[vec] = msi_get_virq(dev, vec);
+
+		/* Enable message-signaled interrupts, fctl.WSI */
+		if (iommu->fctl & RISCV_IOMMU_FCTL_WSI) {
+			iommu->fctl ^= RISCV_IOMMU_FCTL_WSI;
+			riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
+		}
+
+		dev_info(dev, "using MSIs\n");
+		break;
+
+msi_fail:
+		if (igs != RISCV_IOMMU_CAPABILITIES_IGS_BOTH) {
+			return dev_err_probe(dev, -ENODEV,
+					     "unable to use wire-signaled interrupts\n");
+		}
+
+		fallthrough;
+
+	case RISCV_IOMMU_CAPABILITIES_IGS_WSI:
+		for (vec = 0; vec < iommu->irqs_count; vec++)
+			iommu->irqs[vec] = platform_get_irq(pdev, vec);
+
+		/* Enable wire-signaled interrupts, fctl.WSI */
+		if (!(iommu->fctl & RISCV_IOMMU_FCTL_WSI)) {
+			iommu->fctl |= RISCV_IOMMU_FCTL_WSI;
+			riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL, iommu->fctl);
+		}
+		dev_info(dev, "using wire-signaled interrupts\n");
+		break;
+	default:
+		return dev_err_probe(dev, -ENODEV, "invalid IGS\n");
+	}
+
+	return riscv_iommu_init(iommu);
+};
+
+static void k3_iommu_platform_remove(struct platform_device *pdev)
+{
+	struct riscv_iommu_device *iommu = dev_get_drvdata(&pdev->dev);
+	struct k3_iommu_device *k3_iommu = container_of(iommu, struct k3_iommu_device, iommu);
+	bool msi = !(iommu->fctl & RISCV_IOMMU_FCTL_WSI);
+
+	clk_disable_unprepare(k3_iommu->clk);
+
+	riscv_iommu_remove(iommu);
+
+	if (msi)
+		platform_device_msi_free_irqs_all(&pdev->dev);
+};
+
+static void k3_iommu_platform_shutdown(struct platform_device *pdev)
+{
+	riscv_iommu_disable(dev_get_drvdata(&pdev->dev));
+};
+
+static const struct of_device_id k3_iommu_of_match[] = {
+	{.compatible = "spacemit,k3-iommu",},
+	{},
+};
+
+static struct platform_driver k3_iommu_platform_driver = {
+	.probe = k3_iommu_platform_probe,
+	.remove = k3_iommu_platform_remove,
+	.shutdown = k3_iommu_platform_shutdown,
+	.driver = {
+		.name = "spacemit,k3-iommu",
+		.of_match_table = k3_iommu_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+builtin_platform_driver(k3_iommu_platform_driver);
diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index ebb22979075d..a7e1a7758b4c 100644
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -12,6 +12,10 @@
 
 #define pr_fmt(fmt) "riscv-iommu: " fmt
 
+#ifdef CONFIG_SOC_SPACEMIT_K3
+#define IOMMU_IS_NON_COHERENT
+#endif
+
 #include <linux/acpi.h>
 #include <linux/acpi_rimt.h>
 #include <linux/compiler.h>
@@ -19,9 +23,15 @@
 #include <linux/init.h>
 #include <linux/iommu.h>
 #include <linux/iopoll.h>
+#include <linux/irqchip/riscv-imsic.h>
+#include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
 
+#ifdef IOMMU_IS_NON_COHERENT
+#include <linux/dma-map-ops.h>
+#endif
+
 #include "../iommu-pages.h"
 #include "iommu-bits.h"
 #include "iommu.h"
@@ -428,6 +438,10 @@ static unsigned int riscv_iommu_queue_send(struct riscv_iommu_queue *queue,
 	 *    completed and visible before signaling the tail doorbell to fetch
 	 *    the next command. 'fence ow, ow'
 	 */
+#ifdef IOMMU_IS_NON_COHERENT
+	arch_sync_dma_for_device(queue->phys + Q_ITEM(queue, prod) * entry_size,
+				 entry_size, DMA_TO_DEVICE);
+#endif
 	dma_wmb();
 	riscv_iommu_writel(queue->iommu, Q_TAIL(queue), Q_ITEM(queue, prod + 1));
 
@@ -511,17 +525,34 @@ static void riscv_iommu_cmd_sync(struct riscv_iommu_device *iommu,
  * IOMMU Fault/Event queue chapter 3.2
  */
 
+static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iommu,
+						 unsigned int devid);
 static void riscv_iommu_fault(struct riscv_iommu_device *iommu,
 			      struct riscv_iommu_fq_record *event)
 {
 	unsigned int err = FIELD_GET(RISCV_IOMMU_FQ_HDR_CAUSE, event->hdr);
 	unsigned int devid = FIELD_GET(RISCV_IOMMU_FQ_HDR_DID, event->hdr);
+	unsigned int pid = FIELD_GET(RISCV_IOMMU_FQ_HDR_PID, event->hdr);
+	unsigned int pv = FIELD_GET(RISCV_IOMMU_FQ_HDR_PV, event->hdr);
+	unsigned int priv = FIELD_GET(RISCV_IOMMU_FQ_HDR_PRIV, event->hdr);
+	unsigned int ttyp = FIELD_GET(RISCV_IOMMU_FQ_HDR_TTYP, event->hdr);
+	struct riscv_iommu_dc *dc = riscv_iommu_get_dc(iommu, devid);
 
 	/* Placeholder for future fault handling implementation, report only. */
-	if (err)
+	if (err) {
 		dev_warn_ratelimited(iommu->dev,
-				     "Fault %d devid: 0x%x iotval: %llx iotval2: %llx\n",
-				     err, devid, event->iotval, event->iotval2);
+				     "Fault %d, pid: 0x%x, pv: 0x%x, priv: 0x%x, ttyp: 0x%x, "
+				     "devid: 0x%x, iotval: %llx, iotval2: %llx\n",
+				     err, pid, pv, priv, ttyp, devid, event->iotval, event->iotval2);
+		dev_warn_ratelimited(iommu->dev, "(%s:%d) devid:0x%x, dc: \n", __func__, __LINE__, devid);
+		dev_warn_ratelimited(iommu->dev, "        tc:            0x%llx\n", dc->tc);
+		dev_warn_ratelimited(iommu->dev, "        iohgatp:       0x%llx\n", dc->iohgatp);
+		dev_warn_ratelimited(iommu->dev, "        ta:            0x%llx\n", dc->ta);
+		dev_warn_ratelimited(iommu->dev, "        fsc:           0x%llx\n", dc->fsc);
+		dev_warn_ratelimited(iommu->dev, "        msiptp:        0x%llx\n", dc->msiptp);
+		dev_warn_ratelimited(iommu->dev, "        msi_addr_mask: 0x%llx\n", dc->msi_addr_mask);
+		dev_warn_ratelimited(iommu->dev, "        msi_addr_patt: 0x%llx\n", dc->msi_addr_pattern);
+	}
 }
 
 /* Fault queue interrupt handler thread function */
@@ -540,6 +571,11 @@ static irqreturn_t riscv_iommu_fltq_process(int irq, void *data)
 
 	do {
 		cnt = riscv_iommu_queue_consume(queue, &idx);
+#ifdef IOMMU_IS_NON_COHERENT
+		arch_sync_dma_for_cpu(__pa(&events[Q_ITEM(queue, idx)]),
+				      sizeof(struct riscv_iommu_fq_record) * cnt,
+				      DMA_FROM_DEVICE);
+#endif
 		for (len = 0; len < cnt; idx++, len++)
 			riscv_iommu_fault(iommu, &events[Q_ITEM(queue, idx)]);
 		riscv_iommu_queue_release(queue, cnt);
@@ -629,6 +665,9 @@ static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iomm
 			old = cmpxchg_relaxed((unsigned long *)ddtp, ddt, new);
 
 			if (old == ddt) {
+#ifdef IOMMU_IS_NON_COHERENT
+				arch_sync_dma_for_device(__pa(ddtp), sizeof(new), DMA_TO_DEVICE);
+#endif
 				ddtp = (u64 *)ptr;
 				break;
 			}
@@ -1026,6 +1065,9 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
 			continue;
 
 		WRITE_ONCE(dc->tc, tc & ~RISCV_IOMMU_DC_TC_V);
+#ifdef IOMMU_IS_NON_COHERENT
+		arch_sync_dma_for_device(__pa(dc), sizeof(*dc), DMA_TO_DEVICE);
+#endif
 
 		/* Invalidate device context cached values */
 		riscv_iommu_cmd_iodir_inval_ddt(&cmd);
@@ -1051,11 +1093,24 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
 		/* Update device context, write TC.V as the last step. */
 		dma_wmb();
 		WRITE_ONCE(dc->tc, tc);
+#ifdef IOMMU_IS_NON_COHERENT
+		arch_sync_dma_for_device(__pa(dc), sizeof(*dc), DMA_TO_DEVICE);
+#endif
 
 		/* Invalidate device context after update */
 		riscv_iommu_cmd_iodir_inval_ddt(&cmd);
 		riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]);
 		riscv_iommu_cmd_send(iommu, &cmd);
+
+		dev_dbg(iommu->dev, "(%s:%d) devid[%d]:0x%x, dc:\n",
+				__func__, __LINE__, i, fwspec->ids[i]);
+		dev_dbg(iommu->dev, "	     tc:	    0x%llx\n", dc->tc);
+		dev_dbg(iommu->dev, "	     iohgatp:	    0x%llx\n", dc->iohgatp);
+		dev_dbg(iommu->dev, "	     ta:	    0x%llx\n", dc->ta);
+		dev_dbg(iommu->dev, "	     fsc:	    0x%llx\n", dc->fsc);
+		dev_dbg(iommu->dev, "	     msiptp:	    0x%llx\n", dc->msiptp);
+		dev_dbg(iommu->dev, "	     msi_addr_mask: 0x%llx\n", dc->msi_addr_mask);
+		dev_dbg(iommu->dev, "	     msi_addr_patt: 0x%llx\n", dc->msi_addr_pattern);
 	}
 
 	riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);
@@ -1156,6 +1211,9 @@ static unsigned long *riscv_iommu_pte_alloc(struct riscv_iommu_domain *domain,
 				iommu_free_pages(addr);
 				goto pte_retry;
 			}
+#ifdef IOMMU_IS_NON_COHERENT
+			arch_sync_dma_for_device(__pa(ptr), sizeof(pte), DMA_TO_DEVICE);
+#endif
 		}
 		ptr = (unsigned long *)pfn_to_virt(__page_val_to_pfn(pte));
 	} while (level-- > 0);
@@ -1217,6 +1275,9 @@ static int riscv_iommu_map_pages(struct iommu_domain *iommu_domain,
 		pte = _io_pte_entry(phys_to_pfn(phys), pte_prot);
 		if (cmpxchg_relaxed(ptr, old, pte) != old)
 			continue;
+#ifdef IOMMU_IS_NON_COHERENT
+		arch_sync_dma_for_device(__pa(ptr), sizeof(pte), DMA_TO_DEVICE);
+#endif
 
 		riscv_iommu_pte_free(domain, old, &freelist);
 
@@ -1320,6 +1381,51 @@ static bool riscv_iommu_pt_supported(struct riscv_iommu_device *iommu, int pgd_m
 	return false;
 }
 
+static int riscv_iommu_map_msi_bypass(struct riscv_iommu_device *iommu,
+				      struct riscv_iommu_domain *domain)
+{
+	const struct imsic_global_config *imsic_global;
+	const int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
+	size_t mapped, stride;
+	phys_addr_t base;
+	int i;
+
+	/*
+	 * MSI bypass is required only if:
+	 * 1) IMSIC interrupt controller is enabled and configured
+	 * 2) IOMMU hardware supported interrupt remapping is not enabled
+	 *
+	 * Primary first stage protection domain should provide identity
+	 * mapping for all available supervisor IMSIC doorbell pages.
+	 *
+	 * Note: IOMMU MSI remapping is not supported by the driver yet.
+	 */
+	imsic_global = imsic_get_global_config();
+	if (!imsic_global || !imsic_global->nr_ids)
+		return 0;
+
+	base = imsic_global->base_addr;
+	stride = IMSIC_MMIO_PAGE_SZ << imsic_global->guest_index_bits;
+	for (i = 0; i < BIT(imsic_global->hart_index_bits); i++) {
+		if (riscv_iommu_map_pages(&domain->domain, base, base,
+					  IMSIC_MMIO_PAGE_SZ, 1, prot,
+					  GFP_KERNEL_ACCOUNT, &mapped)) {
+			/* unroll mapping */
+			do {
+				riscv_iommu_unmap_pages(&domain->domain, base,
+							IMSIC_MMIO_PAGE_SZ, 1,
+							NULL);
+				base -= stride;
+			} while (i-- > 0);
+
+			return -ENOMEM;
+		}
+		base += stride;
+	}
+
+	return 0;
+}
+
 static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain,
 					    struct device *dev)
 {
@@ -1331,6 +1437,10 @@ static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain,
 	if (!riscv_iommu_pt_supported(iommu, domain->pgd_mode))
 		return -ENODEV;
 
+	/* FIXME: Run riscv_iommu_map_msi_bypass() only once. */
+	if (riscv_iommu_map_msi_bypass(iommu, domain))
+		return -ENOMEM;
+
 	fsc = FIELD_PREP(RISCV_IOMMU_PC_FSC_MODE, domain->pgd_mode) |
 	      FIELD_PREP(RISCV_IOMMU_PC_FSC_PPN, virt_to_pfn(domain->pgd_root));
 	ta = FIELD_PREP(RISCV_IOMMU_PC_TA_PSCID, domain->pscid) |
@@ -1466,6 +1576,25 @@ static struct iommu_domain riscv_iommu_identity_domain = {
 	}
 };
 
+static void riscv_iommu_get_resv_regions(struct device *dev,
+					 struct list_head *head)
+{
+	const struct imsic_global_config *imsic_global;
+	struct iommu_resv_region *reg;
+	size_t size;
+
+	/* We assume target MSI controller to be RISC-V AIA IMSIC controller. */
+	imsic_global = imsic_get_global_config();
+	if (imsic_global && imsic_global->nr_ids) {
+		size = IMSIC_MMIO_PAGE_SZ << (imsic_global->guest_index_bits +
+					      imsic_global->hart_index_bits);
+		reg = iommu_alloc_resv_region(imsic_global->base_addr, size,
+					      0, IOMMU_RESV_MSI, GFP_KERNEL);
+		if (reg)
+			list_add_tail(&reg->list, head);
+	}
+}
+
 static struct iommu_group *riscv_iommu_device_group(struct device *dev)
 {
 	if (dev_is_pci(dev))
@@ -1473,6 +1602,17 @@ static struct iommu_group *riscv_iommu_device_group(struct device *dev)
 	return generic_device_group(dev);
 }
 
+static bool riscv_iommu_capable(struct device *dev, enum iommu_cap cap)
+{
+	switch (cap) {
+	case IOMMU_CAP_CACHE_COHERENCY:
+		/* The RISC-V IOMMU is always DMA cache coherent. */
+		return true;
+	default:
+		return false;
+	}
+}
+
 static int riscv_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
 {
 	return iommu_fwspec_add_ids(dev, args->args, 1);
@@ -1484,6 +1624,7 @@ static struct iommu_device *riscv_iommu_probe_device(struct device *dev)
 	struct riscv_iommu_device *iommu;
 	struct riscv_iommu_info *info;
 	struct riscv_iommu_dc *dc;
+	struct irq_domain *irqdomain;
 	u64 tc;
 	int i;
 
@@ -1511,6 +1652,7 @@ static struct iommu_device *riscv_iommu_probe_device(struct device *dev)
 	tc = 0;
 	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD)
 		tc |= RISCV_IOMMU_DC_TC_SADE;
+
 	for (i = 0; i < fwspec->num_ids; i++) {
 		dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);
 		if (!dc) {
@@ -1522,6 +1664,11 @@ static struct iommu_device *riscv_iommu_probe_device(struct device *dev)
 		WRITE_ONCE(dc->tc, tc);
 	}
 
+	/* FIXME: A hack to set the IRQ_DOMAIN_FLAG_ISOLATED_MSI flag. */
+	irqdomain = dev_get_msi_domain(dev);
+	if (irqdomain)
+		irqdomain->flags |= IRQ_DOMAIN_FLAG_ISOLATED_MSI;
+
 	dev_iommu_priv_set(dev, info);
 
 	return &iommu->iommu;
@@ -1536,10 +1683,12 @@ static void riscv_iommu_release_device(struct device *dev)
 
 static const struct iommu_ops riscv_iommu_ops = {
 	.of_xlate = riscv_iommu_of_xlate,
+	.capable = riscv_iommu_capable,
 	.identity_domain = &riscv_iommu_identity_domain,
 	.blocked_domain = &riscv_iommu_blocking_domain,
 	.release_domain = &riscv_iommu_blocking_domain,
 	.domain_alloc_paging = riscv_iommu_alloc_paging_domain,
+	.get_resv_regions = riscv_iommu_get_resv_regions,
 	.device_group = riscv_iommu_device_group,
 	.probe_device = riscv_iommu_probe_device,
 	.release_device	= riscv_iommu_release_device,
@@ -1677,3 +1826,12 @@ int riscv_iommu_init(struct riscv_iommu_device *iommu)
 	riscv_iommu_queue_disable(&iommu->cmdq);
 	return rc;
 }
+
+static __init int riscv_iommu_request_acs(void)
+{
+#ifdef CONFIG_PCI
+       pci_request_acs();
+#endif
+       return 0;
+}
+arch_initcall(riscv_iommu_request_acs);
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index a61c6dc63c29..50289127c2e7 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -610,11 +610,6 @@ config TI_PRUSS_INTC
 	  The PRUSS INTC enables various interrupts to be routed to multiple
 	  different processors within the SoC.
 
-config RISCV_INTC
-	bool
-	depends on RISCV
-	select IRQ_DOMAIN_HIERARCHY
-
 config RISCV_APLIC
 	bool
 	depends on RISCV
@@ -626,6 +621,11 @@ config RISCV_APLIC_MSI
 	select GENERIC_MSI_IRQ
 	default RISCV_APLIC
 
+config RISCV_INTC
+	bool
+	depends on RISCV
+	select IRQ_DOMAIN_HIERARCHY
+
 config RISCV_IMSIC
 	bool
 	depends on RISCV
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 3de083f5484c..1466fbe4f544 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -103,9 +103,9 @@ obj-$(CONFIG_QCOM_MPM)			+= irq-qcom-mpm.o
 obj-$(CONFIG_CSKY_MPINTC)		+= irq-csky-mpintc.o
 obj-$(CONFIG_CSKY_APB_INTC)		+= irq-csky-apb-intc.o
 obj-$(CONFIG_RISCV_INTC)		+= irq-riscv-intc.o
+obj-$(CONFIG_RISCV_IMSIC)		+= irq-riscv-imsic-state.o irq-riscv-imsic-early.o irq-riscv-imsic-platform.o
 obj-$(CONFIG_RISCV_APLIC)		+= irq-riscv-aplic-main.o irq-riscv-aplic-direct.o
 obj-$(CONFIG_RISCV_APLIC_MSI)		+= irq-riscv-aplic-msi.o
-obj-$(CONFIG_RISCV_IMSIC)		+= irq-riscv-imsic-state.o irq-riscv-imsic-early.o irq-riscv-imsic-platform.o
 obj-$(CONFIG_RISCV_RPMI_SYSMSI)		+= irq-riscv-rpmi-sysmsi.o
 obj-$(CONFIG_SIFIVE_PLIC)		+= irq-sifive-plic.o
 obj-$(CONFIG_STARFIVE_JH8100_INTC)	+= irq-starfive-jh8100-intc.o
diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c
index 2c4c682627b8..3cc9208fac6c 100644
--- a/drivers/irqchip/irq-riscv-imsic-early.c
+++ b/drivers/irqchip/irq-riscv-imsic-early.c
@@ -153,6 +153,11 @@ static int imsic_starting_cpu(unsigned int cpu)
 
 static int imsic_dying_cpu(unsigned int cpu)
 {
+#ifdef CONFIG_SOC_SPACEMIT_K3
+	/* sync irqs on this cpu */
+	imsic_local_sync_all(true);
+#endif
+
 	/* Cleanup IPIs */
 	imsic_ipi_dying_cpu();
 
diff --git a/drivers/irqchip/irq-riscv-imsic-platform.c b/drivers/irqchip/irq-riscv-imsic-platform.c
index 643c8e459611..9b12aec5d49f 100644
--- a/drivers/irqchip/irq-riscv-imsic-platform.c
+++ b/drivers/irqchip/irq-riscv-imsic-platform.c
@@ -13,12 +13,16 @@
 #include <linux/irq.h>
 #include <linux/irqchip.h>
 #include <linux/irqdomain.h>
+#include <linux/log2.h>
 #include <linux/module.h>
 #include <linux/msi.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 #include <linux/smp.h>
+#ifdef CONFIG_SPACEMIT_HMP
+#include <linux/soc/spacemit/spacemit-hmp.h>
+#endif
 
 #include <linux/irqchip/irq-msi-lib.h>
 #include "irq-riscv-imsic-state.h"
@@ -226,20 +230,58 @@ static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 				  unsigned int nr_irqs, void *args)
 {
 	struct imsic_vector *vec;
+	struct cpumask cpu_mask;
+	unsigned int i;
+
+#ifdef CONFIG_SPACEMIT_HMP
+	/* try allocate irq domain from online regular cpus as default */
+	if (!hmp_get_cpumask(&cpu_mask, HMP_REGULAR))
+		cpumask_and(&cpu_mask, cpu_online_mask, &cpu_mask);
+	else
+		cpumask_copy(&cpu_mask, cpu_online_mask);
+#else
+	cpumask_copy(&cpu_mask, cpu_online_mask);
+#endif
 
-	/* Multi-MSI is not supported yet. */
-	if (nr_irqs > 1)
-		return -EOPNOTSUPP;
-
-	vec = imsic_vector_alloc(virq, cpu_online_mask);
-	if (!vec)
-		return -ENOSPC;
-
-	irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec,
-			    handle_edge_irq, NULL, NULL);
-	irq_set_noprobe(virq);
-	irq_set_affinity(virq, cpu_online_mask);
-	irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu));
+	if (nr_irqs > 1) {
+		unsigned int align_order = ilog2(nr_irqs);
+
+		/*
+		 * Multi-MSI requires consecutive, naturally aligned
+		 * interrupt IDs (PCIe spec requirement).
+		 */
+		if (!is_power_of_2(nr_irqs))
+			align_order++;
+
+		vec = imsic_vector_alloc_range(virq, nr_irqs, align_order,
+					       &cpu_mask);
+		if (!vec)
+			return -ENOSPC;
+
+		for (i = 0; i < nr_irqs; i++) {
+			struct imsic_vector *v = vec + i;
+
+			irq_domain_set_info(domain, virq + i, virq + i,
+					    &imsic_irq_base_chip, v,
+					    handle_edge_irq, NULL, NULL);
+			irq_set_noprobe(virq + i);
+			irq_set_affinity(virq + i, &cpu_mask);
+			irq_data_update_effective_affinity(
+				irq_get_irq_data(virq + i),
+				cpumask_of(v->cpu));
+		}
+	} else {
+		vec = imsic_vector_alloc(virq, &cpu_mask);
+		if (!vec)
+			return -ENOSPC;
+
+		irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip,
+				    vec, handle_edge_irq, NULL, NULL);
+		irq_set_noprobe(virq);
+		irq_set_affinity(virq, &cpu_mask);
+		irq_data_update_effective_affinity(
+			irq_get_irq_data(virq), cpumask_of(vec->cpu));
+	}
 
 	return 0;
 }
@@ -247,9 +289,14 @@ static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
 				  unsigned int nr_irqs)
 {
-	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
+	unsigned int i;
 
-	imsic_vector_free(irq_data_get_irq_chip_data(d));
+	for (i = 0; i < nr_irqs; i++) {
+		struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
+
+		if (d)
+			imsic_vector_free(irq_data_get_irq_chip_data(d));
+	}
 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
 }
 
@@ -295,7 +342,8 @@ static bool imsic_init_dev_msi_info(struct device *dev, struct irq_domain *domai
 
 static const struct msi_parent_ops imsic_msi_parent_ops = {
 	.supported_flags	= MSI_GENERIC_FLAGS_MASK |
-				  MSI_FLAG_PCI_MSIX,
+				  MSI_FLAG_PCI_MSIX |
+				  MSI_FLAG_MULTI_PCI_MSI,
 	.required_flags		= MSI_FLAG_USE_DEF_DOM_OPS |
 				  MSI_FLAG_USE_DEF_CHIP_OPS |
 				  MSI_FLAG_PCI_MSI_MASK_PARENT,
diff --git a/drivers/irqchip/irq-riscv-imsic-state.c b/drivers/irqchip/irq-riscv-imsic-state.c
index dc95ad856d80..7f270f5ad670 100644
--- a/drivers/irqchip/irq-riscv-imsic-state.c
+++ b/drivers/irqchip/irq-riscv-imsic-state.c
@@ -468,6 +468,46 @@ struct imsic_vector *imsic_vector_alloc(unsigned int irq, const struct cpumask *
 	return vec;
 }
 
+/**
+ * imsic_vector_alloc_range - Allocate a range of consecutive aligned vectors
+ * @base_irq:	Base Linux IRQ number
+ * @nr_irqs:	Number of consecutive vectors to allocate
+ * @align_order: Alignment order (2^align_order)
+ * @mask:	CPU mask to allocate from
+ *
+ * Returns pointer to the first vector on success, NULL on failure.
+ * All vectors in the range are on the same CPU with consecutive local_ids.
+ */
+struct imsic_vector *imsic_vector_alloc_range(unsigned int base_irq,
+					      unsigned int nr_irqs,
+					      unsigned int align_order,
+					      const struct cpumask *mask)
+{
+	struct imsic_local_priv *lpriv;
+	struct imsic_vector *vec;
+	unsigned long flags;
+	unsigned int cpu, i;
+	int local_id;
+
+	raw_spin_lock_irqsave(&imsic->matrix_lock, flags);
+	local_id = irq_matrix_alloc_range(imsic->matrix, mask, false,
+					  nr_irqs, align_order, &cpu);
+	raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags);
+	if (local_id < 0)
+		return NULL;
+
+	lpriv = per_cpu_ptr(imsic->lpriv, cpu);
+	for (i = 0; i < nr_irqs; i++) {
+		vec = &lpriv->vectors[local_id + i];
+		vec->irq = base_irq + i;
+		vec->enable = false;
+		vec->move_next = NULL;
+		vec->move_prev = NULL;
+	}
+
+	return &lpriv->vectors[local_id];
+}
+
 void imsic_vector_free(struct imsic_vector *vec)
 {
 	unsigned long flags;
diff --git a/drivers/irqchip/irq-riscv-imsic-state.h b/drivers/irqchip/irq-riscv-imsic-state.h
index 57f951952b0c..b03fc312b5a6 100644
--- a/drivers/irqchip/irq-riscv-imsic-state.h
+++ b/drivers/irqchip/irq-riscv-imsic-state.h
@@ -98,6 +98,10 @@ void imsic_vector_move(struct imsic_vector *old_vec, struct imsic_vector *new_ve
 struct imsic_vector *imsic_vector_from_local_id(unsigned int cpu, unsigned int local_id);
 
 struct imsic_vector *imsic_vector_alloc(unsigned int irq, const struct cpumask *mask);
+struct imsic_vector *imsic_vector_alloc_range(unsigned int base_irq,
+					      unsigned int nr_irqs,
+					      unsigned int align_order,
+					      const struct cpumask *mask);
 void imsic_vector_free(struct imsic_vector *vector);
 
 void imsic_vector_debug_show(struct seq_file *m, struct imsic_vector *vec, int ind);
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index 29f16f220384..69d85d17072f 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -390,4 +390,24 @@ config RISCV_SBI_MPXY_MBOX
 	  or HS-mode hypervisor). Say Y here if you want to have this support.
 	  If unsure say N.
 
+config SPACEMIT_MAILBOX
+        tristate "Spacemit Message Box"
+        depends on SOC_SPACEMIT
+        help
+          Mailbox implementation for the hardware message box present in
+          various Spacemit SoCs. This mailbox is used for communication
+          between the application CPUs and the power management coprocessor.
+
+if SPACEMIT_MAILBOX
+
+config K3_MAILBOX
+        tristate "Spacemit K3 Message Box"
+        depends on SOC_SPACEMIT
+        help
+          Mailbox implementation for the hardware message box present in
+          various Spacemit SoCs. This mailbox is used for communication
+          between the application CPUs and the power management coprocessor.
+endif
+
+
 endif
diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile
index 81820a4f5528..44a0fa9f568f 100644
--- a/drivers/mailbox/Makefile
+++ b/drivers/mailbox/Makefile
@@ -82,3 +82,5 @@ obj-$(CONFIG_CIX_MBOX)	+= cix-mailbox.o
 obj-$(CONFIG_BCM74110_MAILBOX)	+= bcm74110-mailbox.o
 
 obj-$(CONFIG_RISCV_SBI_MPXY_MBOX)	+= riscv-sbi-mpxy-mbox.o
+
+obj-$(CONFIG_SPACEMIT_MAILBOX)		+= spacemit/
diff --git a/drivers/mailbox/riscv-sbi-mpxy-mbox.c b/drivers/mailbox/riscv-sbi-mpxy-mbox.c
index 7c9c006b7244..2e950d18319b 100644
--- a/drivers/mailbox/riscv-sbi-mpxy-mbox.c
+++ b/drivers/mailbox/riscv-sbi-mpxy-mbox.c
@@ -511,6 +511,9 @@ static int mpxy_mbox_send_data(struct mbox_chan *chan, void *data)
 	if (mchan->attrs.msg_proto_id == SBI_MPXY_MSGPROTO_RPMI_ID) {
 		mpxy_mbox_send_rpmi_data(mchan, data);
 		return 0;
+	} else if (mchan->attrs.msg_proto_id == SBI_MPXY_MSGPROTO_TEE_ID) {
+		mpxy_mbox_send_rpmi_data(mchan, data);
+		return 0;
 	}
 
 	return -EOPNOTSUPP;
diff --git a/drivers/mailbox/spacemit/Makefile b/drivers/mailbox/spacemit/Makefile
new file mode 100644
index 000000000000..fc7a197102fa
--- /dev/null
+++ b/drivers/mailbox/spacemit/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_K3_MAILBOX) += k3-mailbox.o
diff --git a/drivers/mailbox/spacemit/k3-mailbox.c b/drivers/mailbox/spacemit/k3-mailbox.c
new file mode 100644
index 000000000000..6b584168a25c
--- /dev/null
+++ b/drivers/mailbox/spacemit/k3-mailbox.c
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_controller.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/spinlock.h>
+#include <linux/of_address.h>
+#include "../mailbox.h"
+#include "k3_mailbox.h"
+
+#define mbox_dbg(mbox, ...)	dev_dbg((mbox)->controller.dev, __VA_ARGS__)
+
+static irqreturn_t spacemit_mbox_irq(int irq, void *dev_id)
+{
+	struct spacemit_mailbox *mbox = dev_id;
+	struct mbox_chan *chan;
+	u32 status, msgs[SPACEMIT_NUM_CHANNELS] = {};
+	u32 txdone_mask = 0, rxdata_mask = 0;
+	int i, j;
+	mbox_msg_status_t mstatus;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mbox->lock, flags);
+
+	status = readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_status) &
+		 readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_set);
+
+	if (!(status & 0xff)) {
+		spin_unlock_irqrestore(&mbox->lock, flags);
+		return IRQ_HANDLED;
+	}
+
+	for (i = 0; i < SPACEMIT_NUM_CHANNELS; ++i) {
+		chan = &mbox->controller.chans[i];
+
+		/* not full irq */
+		if (status & (1 << (i * 2 + 1))) {
+			/* disable not full irq */
+			j = readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_clr);
+			j |= (1 << (i * 2 + 1));
+			writel(j, (void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_clr);
+
+			j = readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_status_clr);
+			j |= (1 << (i * 2 + 1));
+			writel(j, (void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_status_clr);
+
+			if (chan->txdone_method & TXDONE_BY_IRQ)
+				txdone_mask |= BIT(i);
+		}
+
+		/* new msg irq */
+		if (status & (1 << (i * 2))) {
+
+			/* clear the fifo */
+			while (1) {
+				msgs[i] = readl((void *)&mbox->regs->mbox_msg[i]);
+				mstatus.val = readl((void *)&mbox->regs->msg_status[i]);
+				if (mstatus.bits.num_msg == 0)
+					break;
+			}
+			rxdata_mask |= BIT(i);
+
+#if 0
+			/* disable the new msg irq */
+			j = readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_clr);
+			j |= (1 << (i * 2));
+			writel(j, (void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_clr);
+#endif
+			/* clear the irq pending */
+			j = readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_status_clr);
+			j |= (1 << (i * 2));
+			writel(j, (void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_status_clr);
+		}
+	}
+
+	spin_unlock_irqrestore(&mbox->lock, flags);
+
+	/*
+	 * Call framework callbacks without holding mbox->lock. mbox_chan_txdone
+	 * acquires chan->lock, while the send path holds chan->lock when calling
+	 * into this driver (which takes mbox->lock), causing lock inversion.
+	 */
+	for (i = 0; i < SPACEMIT_NUM_CHANNELS; ++i) {
+		chan = &mbox->controller.chans[i];
+		if (txdone_mask & BIT(i))
+			mbox_chan_txdone(chan, 0);
+		if (rxdata_mask & BIT(i))
+			mbox_chan_received_data(chan, &msgs[i]);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int spacemit_chan_send_data(struct mbox_chan *chan, void *data)
+{
+	int j;
+	unsigned long flags;
+	struct spacemit_mailbox *mbox = chan->con_priv;
+	u32 chan_num = chan - mbox->controller.chans;
+
+	spin_lock_irqsave(&mbox->lock, flags);
+
+        /* send data */
+	writel(data ? *(u32 *)data : 0, (void *)&mbox->regs->mbox_msg[chan_num]);
+
+	if (!mbox->is_remote) {
+		/* set not full thresh */
+		j = readl((void *)&mbox->regs->mbox_thresh[USER0_MBOX_OFFSET].thresh0);
+		j |= 1 << (chan_num * 8 + 4);
+		writel(j, (void *)&mbox->regs->mbox_thresh[USER0_MBOX_OFFSET].thresh0);
+
+		/* enable not full irq */
+		j = readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_set);
+		j |= (1 << (chan_num * 2 + 1));
+		writel(j, (void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_set);
+	}
+
+	spin_unlock_irqrestore(&mbox->lock, flags);
+
+	return 0;
+}
+
+static int spacemit_chan_startup(struct mbox_chan *chan)
+{
+	struct spacemit_mailbox *mbox = chan->con_priv;
+	u32 chan_num = chan - mbox->controller.chans;
+	u32 msg, j;
+	mbox_msg_status_t mstatus;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mbox->lock, flags);
+
+	/* clear the fifo */
+	while (1) {
+		mstatus.val = readl((void *)&mbox->regs->msg_status[chan_num]);
+		msg = readl((void *)&mbox->regs->mbox_msg[chan_num]);
+		if (mstatus.bits.num_msg == 0)
+			break;
+	}
+
+	/* Enable USER0 new-msg IRQ for local mailbox to receive cross-die messages */
+	if (!mbox->is_remote) {
+		j = readl((void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_set);
+		j |= (1 << (chan_num * 2));
+		writel(j, (void *)&mbox->regs->mbox_irq[USER0_MBOX_OFFSET].irq_en_set);
+	}
+
+	spin_unlock_irqrestore(&mbox->lock, flags);
+
+        return 0;
+}
+
+static void spacemit_chan_shutdown(struct mbox_chan *chan)
+{
+	struct spacemit_mailbox *mbox = chan->con_priv;
+	u32 chan_num = chan - mbox->controller.chans;
+	u32 msg, j;
+	mbox_msg_status_t mstatus;
+	unsigned long flags;
+
+	if (chan->cl->tx_prepare != NULL)
+		return;
+
+	spin_lock_irqsave(&mbox->lock, flags);
+
+	/* disable new msg irq */
+	j = readl((void *)&mbox->regs->mbox_irq[USER1_MBOX_OFFSET].irq_en_clr);
+	j |= (1 << (chan_num * 2));
+	writel(j, (void *)&mbox->regs->mbox_irq[USER1_MBOX_OFFSET].irq_en_clr);
+
+	/* flush the fifo */
+	while (1) {
+		mstatus.val = readl((void *)&mbox->regs->msg_status[chan_num]);
+		msg = readl((void *)&mbox->regs->mbox_msg[chan_num]);
+		if (mstatus.bits.num_msg == 0)
+			break;
+	}
+
+	/* clear pending */
+	j = readl((void *)&mbox->regs->mbox_irq[USER1_MBOX_OFFSET].irq_status_clr);
+	j |= (1 << (chan_num * 2));
+	writel(j, (void *)&mbox->regs->mbox_irq[USER1_MBOX_OFFSET].irq_status_clr);
+
+	spin_unlock_irqrestore(&mbox->lock, flags);
+}
+
+static bool spacemit_chan_last_tx_done(struct mbox_chan *chan)
+{
+	/* TODO */
+	return true;
+}
+
+static bool spacemit_chan_peek_data(struct mbox_chan *chan)
+{
+	struct spacemit_mailbox *mbox = chan->con_priv;
+	u32 chan_num = chan - mbox->controller.chans;
+
+	return readl((void *)&mbox->regs->msg_status[chan_num]);
+}
+
+static const struct mbox_chan_ops spacemit_chan_ops = {
+	.send_data    = spacemit_chan_send_data,
+	.startup      = spacemit_chan_startup,
+	.shutdown     = spacemit_chan_shutdown,
+	.last_tx_done = spacemit_chan_last_tx_done,
+	.peek_data    = spacemit_chan_peek_data,
+};
+
+static int spacemit_mailbox_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mbox_chan *chans;
+	struct spacemit_mailbox *mbox;
+	int i, ret;
+
+	mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
+	if (!mbox)
+		return -ENOMEM;
+
+	chans = devm_kcalloc(dev, SPACEMIT_NUM_CHANNELS, sizeof(*chans), GFP_KERNEL);
+	if (!chans)
+		return -ENOMEM;
+
+	for (i = 0; i < SPACEMIT_NUM_CHANNELS; ++i)
+		chans[i].con_priv = mbox;
+
+	mbox->regs = (mbox_reg_desc_t *)of_iomap(pdev->dev.of_node, 0);
+	if (IS_ERR(mbox->regs)) {
+		ret = PTR_ERR(mbox->regs);
+		dev_err(dev, "Failed to map MMIO resource: %d\n", ret);
+		return -EINVAL;
+	}
+
+	mbox->ap_communicate = of_property_read_bool(pdev->dev.of_node, "ap-communicate");
+	mbox->is_remote = (bool)(uintptr_t)of_device_get_match_data(dev);
+
+	/* request irq only for local mailbox */
+	if (!mbox->is_remote) {
+		ret = devm_request_irq(dev, platform_get_irq(pdev, 0),
+				       spacemit_mbox_irq, 0, dev_name(dev), mbox);
+		if (ret) {
+			dev_err(dev, "Failed to register IRQ handler: %d\n", ret);
+			return ret;
+		}
+	}
+
+	/* register the mailbox controller */
+	mbox->controller.dev = dev;
+	mbox->controller.ops = &spacemit_chan_ops;
+	mbox->controller.chans = chans;
+	mbox->controller.num_chans = SPACEMIT_NUM_CHANNELS;
+
+	if (mbox->is_remote) {
+		/* Remote mailbox: txdone IRQ fires on the remote CPU,
+		 * not locally. Use polling with last_tx_done() == true
+		 * to free the channel immediately.
+		 */
+		mbox->controller.txdone_irq = false;
+		mbox->controller.txdone_poll = true;
+		mbox->controller.txpoll_period = 1;
+	} else {
+		mbox->controller.txdone_irq = true;
+		mbox->controller.txdone_poll = false;
+		mbox->controller.txpoll_period = 5;
+	}
+
+	spin_lock_init(&mbox->lock);
+	platform_set_drvdata(pdev, mbox);
+
+	ret = mbox_controller_register(&mbox->controller);
+	if (ret) {
+		dev_err(dev, "Failed to register controller: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void spacemit_mailbox_remove(struct platform_device *pdev)
+{
+	struct spacemit_mailbox *mbox = platform_get_drvdata(pdev);
+
+	mbox_controller_unregister(&mbox->controller);
+}
+
+static const struct of_device_id spacemit_mailbox_of_match[] = {
+	{ .compatible = "spacemit,k3-mailbox", .data = (void *)false },
+	{ .compatible = "spacemit,k3-rmailbox", .data = (void *)true },
+	{},
+};
+MODULE_DEVICE_TABLE(of, spacemit_mailbox_of_match);
+
+static struct platform_driver spacemit_mailbox_driver = {
+	.driver = {
+		.name = "spacemit-mailbox",
+		.of_match_table = spacemit_mailbox_of_match,
+	},
+	.probe  = spacemit_mailbox_probe,
+	.remove = spacemit_mailbox_remove,
+};
+module_platform_driver(spacemit_mailbox_driver);
+
+MODULE_DESCRIPTION("spacemit Message Box driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mailbox/spacemit/k3_mailbox.h b/drivers/mailbox/spacemit/k3_mailbox.h
new file mode 100644
index 000000000000..9676da56bea8
--- /dev/null
+++ b/drivers/mailbox/spacemit/k3_mailbox.h
@@ -0,0 +1,155 @@
+#ifndef __SPACEMIT_MAILBOX_H__
+#define __SPACEMIT_MAILBOX_H__
+
+#include <linux/kernel.h>
+#include <linux/mailbox_controller.h>
+#include <linux/spinlock.h>
+
+#define SPACEMIT_NUM_CHANNELS	4
+
+/* mailbox register description */
+typedef union mbox_msg {
+	u32 val;
+	struct {
+		u32 msg:32;
+	} bits;
+} mbox_msg_t;
+
+typedef union mbox_fifo_status {
+	u32 val;
+	struct {
+		u32 is_full:1;
+		u32 is_empty:1;
+		u32 reserved0:26;
+		u32 writefull:1;
+		u32 readempty:1;
+		u32 reserved1:2;
+	} bits;
+} mbox_fifo_status_t;
+
+
+typedef union mbox_msg_status {
+	u32 val;
+	struct {
+		u32 num_msg:4;
+		u32 reserved0:24;
+		u32 writefull:1;
+		u32 readempty:1;
+		u32 reserved1:2;
+	} bits;
+} mbox_msg_status_t;
+
+typedef union mbox_irq_status {
+	u32 val;
+	struct {
+		u32 new_msg0_status:1;
+		u32 not_msg0_full:1;
+		u32 new_msg1_status:1;
+		u32 not_msg1_full:1;
+		u32 new_msg2_status:1;
+		u32 not_msg2_full:1;
+		u32 new_msg3_status:1;
+		u32 not_msg3_full:1;
+		u32 reserved:24;
+	} bits;
+} mbox_irq_status_t;
+
+typedef union mbox_irq_status_clr {
+	u32 val;
+	struct {
+		u32 new_msg0_clr:1;
+		u32 not_msg0_full_clr:1;
+		u32 new_msg1_clr:1;
+		u32 not_msg1_full_clr:1;
+		u32 new_msg2_clr:1;
+		u32 not_msg2_full_clr:1;
+		u32 new_msg3_clr:1;
+		u32 not_msg3_full_clr:1;
+		u32 reserved:24;
+	} bits;
+} mbox_irq_status_clr_t;
+
+typedef union mbox_irq_enable_set {
+	u32 val;
+	struct {
+		u32 new_msg0_irq_en:1;
+		u32 not_msg0_full_irq_en:1;
+		u32 new_msg1_irq_en:1;
+		u32 not_msg1_full_irq_en:1;
+		u32 new_msg2_irq_en:1;
+		u32 not_msg2_full_irq_en:1;
+		u32 new_msg3_irq_en:1;
+		u32 not_msg3_full_irq_en:1;
+		u32 reserved:24;
+	} bits;
+} mbox_irq_enable_set_t;
+
+typedef union mbox_irq_enable_clr {
+	u32 val;
+	struct {
+		u32 new_msg0_irq_clr:1;
+		u32 not_msg0_full_irq_clr:1;
+		u32 new_msg1_irq_clr:1;
+		u32 not_msg1_full_irq_clr:1;
+		u32 new_msg2_irq_clr:1;
+		u32 not_msg2_full_irq_clr:1;
+		u32 new_msg3_irq_clr:1;
+		u32 not_msg3_full_irq_clr:1;
+		u32 reserved:24;
+	} bits;
+} mbox_irq_enable_clr_t;
+
+typedef struct mbox_irqthresh {
+	u32 val;
+	struct {
+		u32 mbox0_new_msg_thresh:4;
+		u32 mbox0_not_full_thresh:4;
+		u32 mbox1_new_msg_thresh:4;
+		u32 mbox1_not_full_thresh:4;
+		u32 mbox2_new_msg_thresh:4;
+		u32 mbox2_not_full_thresh:4;
+		u32 mbox3_new_msg_thresh:4;
+		u32 mbox3_not_full_thresh:4;
+	} bits;
+} mbox_irqthresh_t;
+
+typedef struct mbox_thresh {
+	mbox_irqthresh_t thresh0;
+	u32 reserved[3];
+} mbox_thresh_t;
+
+typedef struct mbox_irq {
+	mbox_irq_status_t irq_status;
+	mbox_irq_status_clr_t irq_status_clr;
+	mbox_irq_enable_set_t irq_en_set;
+	mbox_irq_enable_clr_t irq_en_clr;
+} mbox_irq_t;
+
+typedef struct mbox_reg_desc {
+	u32 mbox_version; /* 0x00 */
+	u32 reseved0[3]; /* 0x04 ~ 0x0c */
+	u32 mbox_sysconfig; /* 0x10 */
+	u32 reserved1[11]; /* 0x14 ~ 0x3c */
+	mbox_msg_t mbox_msg[4]; /* 0x40 ~ 0x4c */
+	u32 reserved2[12]; /* 0x50 ~ 0x7c */
+	mbox_fifo_status_t fifo_status[4]; /* 0x80 ~ 0x8c */
+	u32 reserved3[12]; /* 0x90 ~ 0xbc */
+	mbox_msg_status_t msg_status[4]; /* 0xc0 ~ 0xcc */
+	u32 reserved4[12]; /* 0xd0 ~ 0xfc */
+	mbox_irq_t mbox_irq[2]; /* 0x100 ~ 0x11c */
+	u32 reserved5[24]; /* 0x120 ~ 0x17c */
+	mbox_thresh_t mbox_thresh[2]; /* 0x180 */
+} mbox_reg_desc_t;
+
+struct spacemit_mailbox {
+	struct mbox_controller controller;
+	mbox_reg_desc_t *regs;
+	spinlock_t lock;
+	bool ap_communicate;
+	bool is_remote;
+};
+
+#define USER0_MBOX_OFFSET ((mbox->ap_communicate) ? 1 : 0)
+#define USER1_MBOX_OFFSET ((mbox->ap_communicate) ? 0 : 1)
+
+#endif /* __SPACEMIT_MAILBOX_H__ */
diff --git a/drivers/mailbox/spacemit/mailbox-test.c b/drivers/mailbox/spacemit/mailbox-test.c
new file mode 100644
index 000000000000..455efa035ec0
--- /dev/null
+++ b/drivers/mailbox/spacemit/mailbox-test.c
@@ -0,0 +1,223 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2015 ST Microelectronics
+ *
+ * Author: Lee Jones <lee.jones@linaro.org>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/err.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+#include <linux/sched/signal.h>
+#include <linux/freezer.h>
+#include <linux/kthread.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+
+struct mbox_test_device {
+	struct device		*dev;
+	/* system0 channels */
+	struct mbox_chan	*s0s_channel;
+	struct mbox_chan	*s0r_channel;
+	/* system1 channels */
+	struct mbox_chan	*s1s_channel;
+	struct mbox_chan	*s1r_channel;
+
+	struct task_struct	*s0r_thread;
+	struct task_struct	*s1r_thread;
+
+	/* system0 clint */
+	struct mbox_client	*s0r_clint;
+	/* system1 clint */
+	struct mbox_client	*s1r_clint;
+	struct completion	user0_comp;
+	struct completion	user1_comp;
+	bool			user0_thread_run;
+	bool			user1_thread_run;
+};
+
+static unsigned long long user0_count;
+static unsigned long long user1_count;
+
+static void user0_mbox_test_receive_message(struct mbox_client *client, void *message)
+{
+	struct mbox_test_device *tdev;
+
+	tdev = dev_get_drvdata(client->dev);
+
+	++user0_count;
+
+	if (user0_count != user1_count)
+		pr_err("testing wrong: asymmetric sending and receiving\n");
+
+	complete(&tdev->user0_comp);
+}
+
+static void user1_mbox_test_receive_message(struct mbox_client *client, void *message)
+{
+	struct mbox_test_device *tdev;
+
+	tdev = dev_get_drvdata(client->dev);
+
+	++user1_count;
+
+	complete(&tdev->user1_comp);
+}
+
+static int __user0_process_theread(void *arg)
+{
+	int ret;
+	char c = 'c';
+	struct mbox_client *cl = arg;
+	struct mbox_test_device *tdev = dev_get_drvdata(cl->dev);
+	struct sched_param param = {.sched_priority = 0 };
+
+	tdev->user0_thread_run = true;
+	ret = sched_setscheduler(current, SCHED_FIFO, &param);
+	set_freezable();
+
+	do {
+		try_to_freeze();
+		wait_for_completion(&tdev->user0_comp);
+		msleep(200);
+		/* send message again */
+		mbox_send_message(tdev->s0s_channel, &c);
+	} while (!kthread_should_stop());
+
+	tdev->user0_thread_run = true;
+
+	return 0;
+}
+
+static int __user1_process_theread(void *arg)
+{
+	int ret;
+	char c = 'c';
+	struct mbox_client *cl = arg;
+	struct mbox_test_device *tdev = dev_get_drvdata(cl->dev);
+	struct sched_param param = {.sched_priority = 0 };
+
+	tdev->user1_thread_run = true;
+	ret = sched_setscheduler(current, SCHED_FIFO, &param);
+	set_freezable();
+
+	do {
+		try_to_freeze();
+		wait_for_completion(&tdev->user1_comp);
+		msleep(200);
+		/* send message again */
+		mbox_send_message(tdev->s1s_channel, &c);
+	} while (!kthread_should_stop());
+
+	tdev->user0_thread_run = true;
+
+	return 0;
+}
+
+static struct mbox_chan *mbox_test_request_channel(struct platform_device *pdev, const char *name)
+{
+	struct mbox_client *client;
+	struct mbox_chan *channel;
+	struct mbox_test_device *tdev = platform_get_drvdata(pdev);
+
+	client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
+	if (!client)
+		return ERR_PTR(-ENOMEM);
+
+	if (strcmp(name, "s0r") == 0) {
+		client->rx_callback	= user0_mbox_test_receive_message;
+		init_completion(&tdev->user0_comp);
+		tdev->s0r_clint = client;
+	} else if (strcmp(name, "s1r") == 0) {
+		client->rx_callback	= user1_mbox_test_receive_message;
+		init_completion(&tdev->user1_comp);
+		tdev->s1r_clint = client;
+	}
+
+	client->dev = &pdev->dev;
+	client->tx_block = true;
+
+	channel = mbox_request_channel_byname(client, name);
+	if (IS_ERR(channel)) {
+		dev_warn(&pdev->dev, "Failed to request %s channel\n", name);
+		return NULL;
+	}
+
+	return channel;
+}
+
+static int mbox_test_probe(struct platform_device *pdev)
+{
+	struct mbox_test_device *tdev;
+	char c = 'c';
+
+	tdev = devm_kzalloc(&pdev->dev, sizeof(*tdev), GFP_KERNEL);
+	if (!tdev)
+		return -ENOMEM;
+
+	tdev->dev = &pdev->dev;
+	platform_set_drvdata(pdev, tdev);
+
+	tdev->s0s_channel = mbox_test_request_channel(pdev, "s0s");
+	tdev->s0r_channel = mbox_test_request_channel(pdev, "s0r");
+	if (IS_ERR_OR_NULL(tdev->s0s_channel) && IS_ERR_OR_NULL(tdev->s0r_channel))
+		return -EPROBE_DEFER;
+
+	tdev->s1s_channel = mbox_test_request_channel(pdev, "s1s");
+	tdev->s1r_channel = mbox_test_request_channel(pdev, "s1r");
+	if (IS_ERR_OR_NULL(tdev->s1s_channel) && IS_ERR_OR_NULL(tdev->s1r_channel))
+		return -EPROBE_DEFER;
+
+
+	tdev->s0r_thread = kthread_run(__user0_process_theread, (void *)tdev->s0r_clint, "user0_test_thread"); 
+	tdev->s1r_thread = kthread_run(__user1_process_theread, (void *)tdev->s1r_clint, "user1_test_thread"); 
+
+	msleep(100);
+
+	/* trigger the test */
+	mbox_send_message(tdev->s0s_channel, &c);
+
+	dev_info(&pdev->dev, "Successfully registered\n");
+
+	return 0;
+}
+
+static void mbox_test_remove(struct platform_device *pdev)
+{
+	struct mbox_test_device *tdev = platform_get_drvdata(pdev);
+
+	mbox_free_channel(tdev->s0s_channel);
+	mbox_free_channel(tdev->s0r_channel);
+	mbox_free_channel(tdev->s1s_channel);
+	mbox_free_channel(tdev->s1r_channel);
+}
+
+static const struct of_device_id mbox_test_match[] = {
+	{ .compatible = "spacemit,mailbox_test" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, mbox_test_match);
+
+static struct platform_driver mbox_test_driver = {
+	.driver = {
+		.name = "mailbox_test",
+		.of_match_table = mbox_test_match,
+	},
+	.probe  = mbox_test_probe,
+	.remove_new = mbox_test_remove,
+};
+module_platform_driver(mbox_test_driver);
+
+MODULE_DESCRIPTION("spacemit Message Box test driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index 9287faafdce5..23e229c36459 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -90,5 +90,6 @@ source "drivers/media/platform/ti/Kconfig"
 source "drivers/media/platform/verisilicon/Kconfig"
 source "drivers/media/platform/via/Kconfig"
 source "drivers/media/platform/xilinx/Kconfig"
+source "drivers/media/platform/spacemit/Kconfig"
 
 endif # MEDIA_PLATFORM_DRIVERS
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 6fd7db0541c7..415663535f48 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -33,6 +33,7 @@ obj-y += ti/
 obj-y += verisilicon/
 obj-y += via/
 obj-y += xilinx/
+obj-y += spacemit/
 
 # Please place here only ancillary drivers that aren't SoC-specific
 # Please keep it alphabetically sorted by Kconfig name
diff --git a/drivers/media/platform/spacemit/Kconfig b/drivers/media/platform/spacemit/Kconfig
new file mode 100644
index 000000000000..97f595a9db10
--- /dev/null
+++ b/drivers/media/platform/spacemit/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+comment "Spacemit media platform drivers"
+
+source "drivers/media/platform/spacemit/vpu_k3/Kconfig"
+
+source "drivers/media/platform/spacemit/cam_ccic/Kconfig"
+
+source "drivers/media/platform/spacemit/cam_sensor/Kconfig"
diff --git a/drivers/media/platform/spacemit/Makefile b/drivers/media/platform/spacemit/Makefile
new file mode 100644
index 000000000000..cb6a1e1bae32
--- /dev/null
+++ b/drivers/media/platform/spacemit/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-y += vpu_k3/
+obj-y += cam_ccic/
+obj-y += cam_sensor/
diff --git a/drivers/media/platform/spacemit/cam_ccic/Kconfig b/drivers/media/platform/spacemit/cam_ccic/Kconfig
new file mode 100644
index 000000000000..c09822b49bae
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/Kconfig
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# spacemit camera configuration
+#
+
+comment "SPACEMIT K3 Camera And Video"
+
+config SPACEMIT_K3_CCIC
+    tristate "SPACEMIT K3 camera CCIC support"
+    select VIDEOBUF2_CORE
+    select VIDEOBUF2_V4L2
+    select VIDEOBUF2_DMA_CONTIG
+    select VIDEOBUF2_DMA_SG
+    depends on VIDEO_DEV
+    help
+      Enable support for camera ccic
+
+config SPACEMIT_K3_CCIC_IOMMU
+	bool "SPACEMIT K3 camera CCIC IOMMU support"
+	depends on SPACEMIT_K3_CCIC
+    help
+      Enable support for camera ccic iommu
diff --git a/drivers/media/platform/spacemit/cam_ccic/Makefile b/drivers/media/platform/spacemit/cam_ccic/Makefile
new file mode 100644
index 000000000000..1ff7f4236918
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/Makefile
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0
+subdir-ccflags-y := -Werror -Wno-maybe-uninitialized # zmc: to be removed
+
+obj-$(CONFIG_SPACEMIT_K3_CCIC) += spacemit_ccic.o
+spacemit_ccic-objs += ccic_hwreg.o
+spacemit_ccic-objs += csiphy.o
+spacemit_ccic-objs += ccic_drv.o
+spacemit_ccic-objs += dptc_drv.o
+spacemit_ccic-objs += ccic_vdev.o
+spacemit_ccic-objs += ccic_iommu.o
+spacemit_ccic-objs += ccic_dma.o
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_dma.c b/drivers/media/platform/spacemit/cam_ccic/ccic_dma.c
new file mode 100644
index 000000000000..d2434e01a2be
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_dma.c
@@ -0,0 +1,473 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for CCIC DMA
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/errno.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/interrupt.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-dev.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-dma-sg.h>
+#include <linux/media-bus-format.h>
+#include <linux/delay.h>
+#include <linux/atomic.h>
+#include "ccic_drv.h"
+#include "ccic_hwreg.h"
+#include "csiphy.h"
+#include "ccic_vdev.h"
+#include "ccic_dma.h"
+
+#define DEBUG 0 /* for pr_debug() */
+
+#define CCIC_DMA_DRV_NAME "spacemit_ccic_dma"
+extern irqreturn_t ccic_dma_irq_handler(int irq, void *data);
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+extern irqreturn_t ccic_mmu_irq_handler(int irq, void *data);
+#endif
+struct ccic_dma *g_ccic_dma = NULL;
+
+/*
+ * Device register I/O
+ */
+static inline u32 dma_reg_read(struct ccic_dma *ccic_dma, unsigned int reg)
+{
+	return ioread32(ccic_dma->base + reg);
+}
+
+static inline void dma_reg_write(struct ccic_dma *ccic_dma, unsigned int reg,
+				 u32 val)
+{
+	iowrite32(val, ccic_dma->base + reg);
+}
+
+static inline void dma_reg_write_mask(struct ccic_dma *ccic_dma,
+				      unsigned int reg, u32 val, u32 mask)
+{
+	u32 v = dma_reg_read(ccic_dma, reg);
+
+	v = (v & ~mask) | (val & mask);
+	dma_reg_write(ccic_dma, reg, v);
+}
+
+static inline void dma_reg_set_bit(struct ccic_dma *ccic_dma, unsigned int reg,
+				   u32 val)
+{
+	dma_reg_write_mask(ccic_dma, reg, val, val);
+}
+
+static inline void dma_reg_clear_bit(struct ccic_dma *ccic_dma,
+				     unsigned int reg, u32 val)
+{
+	dma_reg_write_mask(ccic_dma, reg, 0, val);
+}
+
+static int ccic_dma_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct ccic_dma *ccic_dma = NULL;
+	int ret = 0;
+
+	ret = of_property_read_u32(np, "cell-index", &pdev->id);
+	if (ret < 0) {
+		dev_err(dev, "failed to get alias id, errno %d\n", ret);
+		return ret;
+	}
+
+	ccic_dma = devm_kzalloc(dev, sizeof(*ccic_dma), GFP_KERNEL);
+	if (!ccic_dma) {
+		dev_err(dev, "could not allocate ccic dma\n");
+		return -ENOMEM;
+	}
+
+	/* get mem */
+	ccic_dma->mem = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						     "ccic-dma-regs");
+	if (!ccic_dma->mem) {
+		dev_err(dev, "no mem resource");
+		return -ENODEV;
+	}
+	ccic_dma->base = devm_ioremap(dev, ccic_dma->mem->start,
+				      resource_size(ccic_dma->mem));
+	if (!ccic_dma->base) {
+		dev_err(dev, "fail to remap iomem\n");
+		return -ENOMEM;
+	}
+
+	/* get irqs */
+	ccic_dma->irq = platform_get_irq_byname(pdev, "ccic-dma-irq");
+	if (ccic_dma->irq < 0) {
+		dev_err(dev, "failed to get ccic dma irq\n");
+		return ccic_dma->irq;
+	}
+	ret = devm_request_irq(dev, ccic_dma->irq, ccic_dma_irq_handler,
+			       IRQF_SHARED, CCIC_DMA_DRV_NAME, ccic_dma);
+	if (ret) {
+		dev_err(dev, "fail to request ccic dma irq\n");
+		return ret;
+	}
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+	ccic_dma->mmu_irq = platform_get_irq_byname(pdev, "ccic-dma-mmu-irq");
+	if (ccic_dma->mmu_irq < 0) {
+		dev_err(dev, "failed to get ccic mmu irq\n");
+		return ccic_dma->mmu_irq;
+	}
+	ret = devm_request_irq(dev, ccic_dma->mmu_irq, ccic_mmu_irq_handler,
+			       IRQF_SHARED, CCIC_DMA_DRV_NAME, ccic_dma);
+	if (ret) {
+		dev_err(dev, "fail to request ccic dma irq\n");
+		return ret;
+	}
+#endif
+	ccic_dma->dev = &pdev->dev;
+	atomic_set(&ccic_dma->busy_cnt, 0);
+	/* initialize lock protecting open/release and register writes */
+	spin_lock_init(&ccic_dma->dev_lock);
+	g_ccic_dma = ccic_dma;
+	pr_info("%s probed", dev_name(&pdev->dev));
+	return 0;
+}
+
+static void ccic_dma_remove(struct platform_device *pdev)
+{
+	g_ccic_dma = NULL;
+}
+
+static const struct of_device_id ccic_dt_match[] = {
+	{
+		.compatible = "spacemit,ccic-dma",
+		.data = NULL,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, ccic_dt_match);
+
+struct platform_driver ccic_dma_driver = {
+    .driver = {
+        .name = CCIC_DMA_DRV_NAME,
+        .of_match_table = of_match_ptr(ccic_dt_match),
+    },
+    .probe = ccic_dma_probe,
+    .remove = ccic_dma_remove,
+};
+
+void ccic_dma_open(struct ccic_dma *ccic_dma)
+{
+	struct device *dev = ccic_dma->dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ccic_dma->dev_lock, flags);
+	if (atomic_inc_return(&ccic_dma->busy_cnt) == 1) {
+		dev_info(dev, "ccic dma open\n");
+		dma_reg_write(ccic_dma, 0x80, 0x00);
+		dma_reg_write(ccic_dma, 0x7c, 0x00);
+		/* dma overflow irq */
+		dma_reg_write(ccic_dma, 0x128, 0xf0000000);
+	}
+	spin_unlock_irqrestore(&ccic_dma->dev_lock, flags);
+}
+
+void ccic_dma_release(struct ccic_dma *ccic_dma)
+{
+	struct device *dev = ccic_dma->dev;
+	int v = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ccic_dma->dev_lock, flags);
+	v = atomic_dec_return(&ccic_dma->busy_cnt);
+	if (v == 0) {
+		dev_info(dev, "ccic dma release\n");
+		dma_reg_write(ccic_dma, 0x128, 0x00000000);
+	} else if (v < 0) {
+		dev_err(dev, "invalid ccic dma release\n");
+		atomic_inc(&ccic_dma->busy_cnt);
+	}
+	spin_unlock_irqrestore(&ccic_dma->dev_lock, flags);
+}
+
+int ccic_dma_ch_set_fmt(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			unsigned int width, unsigned int height,
+			unsigned int h_offset, unsigned int v_offset,
+			unsigned int pix_fmt)
+{
+	struct device *dev = NULL;
+	unsigned int data_fmt = C0_DF_BAYER, imgsz_w = 0, imgsz_h = 0;
+	unsigned int stride = 0;
+	unsigned int reg_addr = 0, reg_val = 0;
+
+	if (!ccic_dma) {
+		pr_err("ccic_dma is null\n");
+		return -EINVAL;
+	}
+	dev = ccic_dma->dev;
+	if (dma_ch >= MAX_CCIC_DMA_CNT) {
+		dev_err(dev, "invalid dma_ch %u\n", dma_ch);
+		return -EINVAL;
+	}
+	switch (pix_fmt) {
+	case CSI_DUMP_FMT_YUV422:
+		data_fmt = C0_DF_YUV;
+		imgsz_w = width * 2;
+		imgsz_h = height;
+		break;
+	case CSI_DUMP_FMT_RAW8:
+		data_fmt = C0_DF_BAYER;
+		imgsz_w = width;
+		imgsz_h = height;
+		break;
+	case CSI_DUMP_FMT_RAW10:
+		data_fmt = C0_DF_BAYER;
+		imgsz_w = CAM_ALIGN(width, 4);
+		imgsz_w = imgsz_w * 5 / 4;
+		imgsz_h = height;
+		h_offset = CAM_ALIGN(h_offset, 4);
+		h_offset = h_offset * 5 / 4;
+		break;
+	case CSI_DUMP_FMT_RAW12:
+		data_fmt = C0_DF_BAYER;
+		imgsz_w = CAM_ALIGN(width, 2);
+		imgsz_w = imgsz_w * 3 / 2;
+		imgsz_h = height;
+		h_offset = CAM_ALIGN(h_offset, 2);
+		h_offset = h_offset * 3 / 2;
+		break;
+	default:
+		pr_err("%s failed: invalid pixfmt %d\n", __func__, pix_fmt);
+		return -1;
+	}
+
+	stride = CAM_ALIGN(imgsz_w, 4);
+
+	dev_info(dev, "dma_ch=%u stride=0x%x, width=%u height=%u\n", dma_ch,
+		 stride, width, height);
+
+	//pitch
+	reg_addr = 0x98 + dma_ch * 0x4;
+	reg_val = stride & 0xffff;
+	reg_val |= (stride << 16);
+	dma_reg_write(ccic_dma, reg_addr, reg_val);
+
+	//ch size
+	reg_addr = 0xd0 + dma_ch * 0x4;
+	reg_val = imgsz_w & 0xffff;
+	reg_val |= (height << 16);
+	dma_reg_write_mask(ccic_dma, reg_addr, reg_val, 0x7fffffff);
+	return 0;
+}
+
+int ccic_dma_ch_src_sel(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			unsigned int src_sel)
+{
+	struct device *dev = NULL;
+	unsigned int reg_val = 0, reg_mask = 0;
+
+	if (!ccic_dma) {
+		pr_err("ccic_dma is null\n");
+		return -EINVAL;
+	}
+	dev = ccic_dma->dev;
+	if (dma_ch >= MAX_CCIC_DMA_CNT) {
+		dev_err(dev, "invalid dma_ch %u\n", dma_ch);
+		return -EINVAL;
+	}
+	if (src_sel >= PATH_NUM_MAX) {
+		dev_err(dev, "invalid src_sel %u\n", src_sel);
+		return -EINVAL;
+	}
+	if (dma_ch < 4) {
+		reg_mask = 0xf0000 << (dma_ch * 4);
+		reg_val = (src_sel << 16) << (dma_ch * 4);
+		dma_reg_write_mask(ccic_dma, 0x7c, reg_val, reg_mask);
+	} else {
+		reg_mask = 0xf << ((dma_ch - 4) * 4);
+		reg_val = src_sel << ((dma_ch - 4) * 4);
+		dma_reg_write_mask(ccic_dma, 0x80, reg_val, reg_mask);
+	}
+
+	return 0;
+}
+
+int ccic_dma_ch_enable(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+		       unsigned int enable)
+{
+	struct device *dev = NULL;
+	unsigned int reg_val = 0, reg_mask = 0;
+
+	if (!ccic_dma) {
+		pr_err("ccic_dma is null\n");
+		return -EINVAL;
+	}
+	dev = ccic_dma->dev;
+	if (dma_ch >= MAX_CCIC_DMA_CNT) {
+		dev_err(dev, "invalid dma_ch %u\n", dma_ch);
+		return -EINVAL;
+	}
+	reg_mask = 1 << (dma_ch * 1);
+	if (enable) {
+		reg_val = reg_mask;
+	} else {
+		reg_val = 0;
+	}
+	dma_reg_write_mask(ccic_dma, 0x7c, reg_val, reg_mask);
+
+	return 0;
+}
+
+int ccic_dma_ch_set_addr(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			 unsigned long phy_addr)
+{
+	struct device *dev = NULL;
+	unsigned int reg_addr = 0;
+
+	if (!ccic_dma) {
+		pr_err("ccic_dma is null\n");
+		return -EINVAL;
+	}
+	dev = ccic_dma->dev;
+	if (dma_ch >= MAX_CCIC_DMA_CNT) {
+		dev_err(dev, "invalid dma_ch %u\n", dma_ch);
+		return -EINVAL;
+	}
+	reg_addr = dma_ch * 0x8;
+	dma_reg_write(ccic_dma, reg_addr, phy_addr);
+	if (dma_ch > 9) {
+		dma_reg_write(ccic_dma, reg_addr + 0x04, phy_addr);
+	}
+
+	return 0;
+}
+
+int ccic_dma_ch_shadow_ready(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			     unsigned int ready)
+{
+	struct device *dev = NULL;
+	unsigned int reg_addr = 0, reg_val = 0;
+
+	if (!ccic_dma) {
+		pr_err("ccic_dma is null\n");
+		return -EINVAL;
+	}
+	dev = ccic_dma->dev;
+	if (dma_ch >= MAX_CCIC_DMA_CNT) {
+		dev_err(dev, "invalid dma_ch %u\n", dma_ch);
+		return -EINVAL;
+	}
+	reg_addr = 0xd0 + dma_ch * 0x4;
+	if (ready) {
+		reg_val = BIT(31);
+	} else {
+		reg_val = 0;
+	}
+	dma_reg_write_mask(ccic_dma, reg_addr, reg_val, BIT(31));
+
+	return 0;
+}
+
+int ccic_dma_ch_irq_enable(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			   unsigned int enable)
+{
+	struct device *dev = NULL;
+	unsigned int reg_val = 0, reg_mask = 0;
+
+	if (!ccic_dma) {
+		pr_err("ccic_dma is null\n");
+		return -EINVAL;
+	}
+	dev = ccic_dma->dev;
+	if (dma_ch >= MAX_CCIC_DMA_CNT) {
+		dev_err(dev, "invalid dma_ch %u\n", dma_ch);
+		return -EINVAL;
+	}
+
+	if (dma_ch < 10) {
+		reg_mask = 0x7 << (dma_ch * 3);
+		if (enable) {
+			reg_val = reg_mask;
+		} else {
+			reg_val = 0;
+		}
+		dma_reg_write_mask(ccic_dma, 0x124, reg_val, reg_mask);
+	} else if (dma_ch == 10) {
+		reg_mask = 0x3 << 30;
+		if (enable) {
+			reg_val = reg_mask;
+		} else {
+			reg_val = 0;
+		}
+		dma_reg_write_mask(ccic_dma, 0x124, reg_val, reg_mask);
+
+		reg_mask = 0x1 << 0;
+		if (enable) {
+			reg_val = reg_mask;
+		} else {
+			reg_val = 0;
+		}
+		dma_reg_write_mask(ccic_dma, 0x128, reg_val, reg_mask);
+	} else {
+		reg_mask = 0x7 << 1;
+		if (enable) {
+			reg_val = reg_mask;
+		} else {
+			reg_val = 0;
+		}
+		dma_reg_write_mask(ccic_dma, 0x128, reg_val, reg_mask);
+	}
+	return 0;
+}
+
+unsigned int ccic_dma_get_irq0(struct ccic_dma *ccic_dma)
+{
+	BUG_ON(!ccic_dma);
+	return dma_reg_read(ccic_dma, 0x114);
+}
+
+void ccic_dma_clear_irq0(struct ccic_dma *ccic_dma, unsigned int clear)
+{
+	BUG_ON(!ccic_dma);
+	dma_reg_write(ccic_dma, 0x114, clear);
+}
+
+unsigned int ccic_dma_get_irq1(struct ccic_dma *ccic_dma)
+{
+	BUG_ON(!ccic_dma);
+	return dma_reg_read(ccic_dma, 0x118);
+}
+
+void ccic_dma_clear_irq1(struct ccic_dma *ccic_dma, unsigned int clear)
+{
+	BUG_ON(!ccic_dma);
+	dma_reg_write(ccic_dma, 0x118, clear);
+}
+
+unsigned int ccic_dma_ch_irq_analyze(unsigned int dma_ch, unsigned int irq0,
+				     unsigned int irq1)
+{
+	unsigned int ret = 0;
+
+	if (dma_ch < 10) {
+		ret = irq0 >> (dma_ch * 3);
+	} else if (dma_ch == 10) {
+		ret = irq0 >> (dma_ch * 3);
+		if (irq1 & BIT(0)) {
+			ret |= BIT(2);
+		}
+	} else {
+		ret = irq1 >> 1;
+	}
+	ret &= 0x07;
+
+	return ret;
+}
+
+struct ccic_dma *get_ccic_dma(void)
+{
+	return g_ccic_dma;
+}
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_dma.h b/drivers/media/platform/spacemit/cam_ccic/ccic_dma.h
new file mode 100644
index 000000000000..0fce736f4fc2
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_dma.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ccic_dma.h - Driver for CCIC DMA
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef _CCIC_DMA_H_
+#define _CCIC_DMA_H_
+
+#define MAX_CCIC_DMA_CNT (12)
+
+struct ccic_dma {
+	int index;
+	struct device *dev;
+	struct platform_device *pdev;
+	int irq;
+#if defined(CONFIG_SPACEMIT_K3_CCIC_IOMMU)
+	int mmu_irq;
+#endif
+	struct resource *mem;
+	void __iomem *base;
+	struct mutex ops_mutex;
+	spinlock_t dev_lock;
+	//struct ccic_dma_ops *ops;
+	//void *vnode;
+	atomic_t busy_cnt;
+};
+
+struct ccic_dma *get_ccic_dma(void);
+void ccic_dma_open(struct ccic_dma *ccic_dma);
+void ccic_dma_release(struct ccic_dma *ccic_dma);
+int ccic_dma_clock_enable(struct ccic_dma *ccic_dma);
+int ccic_dma_ch_set_fmt(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			unsigned int width, unsigned int height,
+			unsigned int h_offset, unsigned int v_offset,
+			unsigned int pix_fmt);
+int ccic_dma_ch_src_sel(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			unsigned int src_sel);
+int ccic_dma_ch_enable(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+		       unsigned int enable);
+int ccic_dma_ch_set_addr(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			 unsigned long phy_addr);
+int ccic_dma_ch_shadow_ready(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			     unsigned int ready);
+int ccic_dma_ch_irq_enable(struct ccic_dma *ccic_dma, unsigned int dma_ch,
+			   unsigned int enable);
+unsigned int ccic_dma_get_irq0(struct ccic_dma *ccic_dma);
+unsigned int ccic_dma_get_irq1(struct ccic_dma *ccic_dma);
+void ccic_dma_clear_irq0(struct ccic_dma *ccic_dma, unsigned int clear);
+void ccic_dma_clear_irq1(struct ccic_dma *ccic_dma, unsigned int clear);
+#define DMA_IRQ_SOF BIT(0)
+#define DMA_IRQ_DONE BIT(1)
+#define DMA_IRQ_ERR BIT(2)
+unsigned int ccic_dma_ch_irq_analyze(unsigned int dma_ch, unsigned int irq0,
+				     unsigned int irq1);
+#endif
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_drv.c b/drivers/media/platform/spacemit/cam_ccic/ccic_drv.c
new file mode 100644
index 000000000000..cab2f045c364
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_drv.c
@@ -0,0 +1,1025 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Spacemit IPE MODULE
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/errno.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/interrupt.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-dev.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-dma-sg.h>
+#include <linux/media-bus-format.h>
+#include <media/spacemit/ccic_uapi.h>
+#include <linux/delay.h>
+#include <linux/atomic.h>
+#include <linux/clk.h>
+#include "ccic_drv.h"
+#include "ccic_hwreg.h"
+#include "csiphy.h"
+#include "ccic_vdev.h"
+#include "ccic_dma.h"
+
+#define DEBUG 0 /* for pr_debug() */
+
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+#include "ccic_iommu.h"
+#include <linux/dma-mapping.h>
+#define MMU_RESERVED_MEM_SIZE (4096)
+#define MMU_REG_BASE (0xd420fc00)
+#endif
+
+#define CCIC_DRV_NAME "spacemit_ccic"
+
+#define CAM_ALIGN(a, b)                                   \
+	({                                                \
+		unsigned int ___tmp1 = (a);               \
+		unsigned int ___tmp2 = (b);               \
+		unsigned int ___tmp3 = ___tmp1 % ___tmp2; \
+		___tmp1 /= ___tmp2;                       \
+		if (___tmp3)                              \
+			___tmp1++;                        \
+		___tmp1 *= ___tmp2;                       \
+		___tmp1;                                  \
+	})
+
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+struct ccic_iommu_device *mmu_dev = NULL;
+dma_addr_t trans_tab_dma_addr = 0;
+void *trans_tab_cpu_addr = NULL;
+size_t total_trans_tab_sz = 0;
+dma_addr_t rsvd_phy_addr = 0;
+unsigned char *rsvd_vaddr = NULL;
+struct mmu_ctx mmu_ctx[CCIC_IOMMU_CH_NUM];
+#endif
+
+static LIST_HEAD(ccic_devices);
+static DEFINE_MUTEX(list_lock);
+static void ccic_dma_bh_handler(struct ccic_dma_work_struct *ccic_dma_work);
+
+static void ccic_irqmask(struct ccic_ctrl *ctrl, int on)
+{
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+
+	if (on) {
+		//ccic_reg_write(ccic_dev, REG_IRQSTAT, ALLIRQS);
+		ccic_reg_set_bit(ccic_dev, REG_IRQMASK, ALLIRQS);
+	}
+}
+
+static int ccic_config_csi2(struct ccic_dev *ccic_dev, struct mipi_csi2 *csi,
+			    int enable)
+{
+	unsigned int ctrl0_val = 0;
+	int lanes = csi->dphy_desc.nr_lane;
+
+	if (!ccic_dev->csiphy)
+		return -EINVAL;
+	if (enable) {
+		csiphy_start(ccic_dev->csiphy, csi);
+		ctrl0_val = ccic_reg_read(ccic_dev, REG_CSI2_CTRL0);
+		ctrl0_val &= ~(CSI2_C0_LANE_NUM_MASK);
+		ctrl0_val |= CSI2_C0_LANE_NUM(lanes);
+		ctrl0_val |= CSI2_C0_ENABLE;
+		ctrl0_val &= ~(CSI2_C0_VLEN_MASK);
+		ctrl0_val |= CSI2_C0_VLEN;
+		ccic_reg_write(ccic_dev, REG_CSI2_CTRL0, ctrl0_val);
+	} else {
+		csiphy_stop(ccic_dev->csiphy);
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_CTRL0,
+				   CSI2_C0_ENABLE); //csi off
+	}
+	return 0;
+}
+
+static int ccic_config_csi2_dphy(struct ccic_ctrl *ctrl, struct mipi_csi2 *csi,
+				 int enable)
+{
+	int ret = 0;
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+
+	ret = ccic_config_csi2(ccic_dev, csi, enable);
+	if (ret) {
+		dev_err(ccic_dev->dev, "csi2 config failed\n");
+		goto out;
+	}
+
+out:
+	return ret;
+}
+
+static int ccic_config_csi2_vc(struct ccic_ctrl *ctrl, int md,
+			       unsigned int dt_en)
+{
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+
+	return ccic_csi2_vc_ctrl(ccic_dev, md, dt_en);
+}
+
+static int __maybe_unused ccic_enable_csi2idi(struct ccic_ctrl *ctrl)
+{
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+
+	ccic_csi2idi_reset(ccic_dev, 0);
+
+	return 0;
+}
+
+static int ccic_clk_set_rate(struct ccic_ctrl *ctrl_dev, int mode)
+{
+	unsigned long clk_val;
+	struct ccic_dev *ccic_dev = ctrl_dev->ccic_dev;
+
+	//clk_val = clk_round_rate(ccic_dev->csi_clk, 1000000000);
+	clk_val = clk_round_rate(ccic_dev->csi_clk, 500000000);
+	clk_set_rate(ccic_dev->csi_clk, clk_val);
+	clk_val = clk_get_rate(ccic_dev->csi_clk);
+	pr_info("cam clk[csi_func]: %ld\n", clk_val);
+
+	//clk_val = clk_round_rate(ccic_dev->clk4x, 1000000000);
+	clk_val = clk_round_rate(ccic_dev->clk4x, 500000000);
+	clk_set_rate(ccic_dev->clk4x, clk_val);
+	clk_val = clk_get_rate(ccic_dev->clk4x);
+	pr_info("cam clk[ccic_func]: %ld\n", clk_val);
+
+	clk_val = clk_round_rate(ccic_dev->ahb_clk, 307200000);
+	clk_set_rate(ccic_dev->ahb_clk, clk_val);
+	clk_val = clk_get_rate(ccic_dev->ahb_clk);
+	pr_info("cam clk[ahb_func]: %ld\n", clk_val);
+
+	clk_val = clk_round_rate(ccic_dev->axi_clk, 409600000);
+	clk_set_rate(ccic_dev->axi_clk, clk_val);
+	clk_val = clk_get_rate(ccic_dev->axi_clk);
+	pr_info("cam clk[axi_func]: %ld\n", clk_val);
+	return 0;
+}
+
+static int ccic_clk_enable(struct ccic_ctrl *ctrl, int en)
+{
+	int ret = 0, v = 0;
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+	struct device *dev = &ccic_dev->pdev->dev;
+
+	if (en) {
+		mutex_lock(&ctrl->lock);
+		if (atomic_inc_return(&ctrl->usr_cnt) == 1) {
+			reset_control_deassert(ccic_dev->sc2_hclk_reset);
+			reset_control_deassert(ccic_dev->ccic_4x_reset);
+			reset_control_deassert(ccic_dev->csi_reset);
+			reset_control_deassert(ccic_dev->isp_cibus_reset);
+
+			clk_prepare_enable(ccic_dev->ahb_clk);
+			clk_prepare_enable(ccic_dev->clk4x);
+			clk_prepare_enable(ccic_dev->csi_clk);
+			clk_prepare_enable(ccic_dev->axi_clk);
+
+			ret = ccic_clk_set_rate(ctrl, SC2_MODE_ISP);
+			if (ret < 0) {
+				atomic_dec(&ctrl->usr_cnt);
+				return ret;
+			}
+
+			ccic_csi2idi_reset(ccic_dev, 0);
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+			//set mmu timeout default addr
+			mmu_dev->ops->set_timeout_default_addr(mmu_dev, (uint64_t)rsvd_phy_addr);
+#endif
+			dev_info(dev, "power on\n");
+		}
+		mutex_unlock(&ctrl->lock);
+	} else {
+		mutex_lock(&ctrl->lock);
+		v = atomic_dec_return(&ctrl->usr_cnt);
+		if (v == 0) {
+			ccic_csi2idi_reset(ccic_dev, 1);
+			clk_disable_unprepare(ccic_dev->axi_clk);
+			reset_control_assert(ccic_dev->isp_cibus_reset);
+			clk_disable_unprepare(ccic_dev->csi_clk);
+			reset_control_assert(ccic_dev->csi_reset);
+			clk_disable_unprepare(ccic_dev->clk4x);
+			reset_control_assert(ccic_dev->ccic_4x_reset);
+			clk_disable_unprepare(ccic_dev->ahb_clk);
+			reset_control_assert(ccic_dev->sc2_hclk_reset);
+			dev_info(dev, "power off\n");
+		} else if (v < 0) {
+			atomic_inc(&ctrl->usr_cnt);
+			dev_err(dev, "invalid power off\n");
+		}
+		mutex_unlock(&ctrl->lock);
+	}
+
+	return ret;
+}
+
+static int ccic_config_csi2_mbus(struct ccic_ctrl *ctrl, int lanes,
+				 int dphy_freq)
+{
+	int ret;
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+	struct mipi_csi2 csi2para;
+
+	csi2para.calc_dphy = 0;
+	csi2para.dphy[0] = 0x00000001;
+	csi2para.dphy[1] = 0x5ada5ada; //for lark
+	//csi2para.dphy[2] = 0x0000201a;//0x0000201a: 1.0G ~ 1.5G
+	csi2para.dphy[2] = 0x00001500;
+	csi2para.dphy[3] = 0x000000ff;
+	csi2para.dphy[4] = 0x1000;
+	csi2para.dphy_desc.nr_lane = lanes;
+	ret = ccic_config_csi2_dphy(ctrl, &csi2para, !!lanes);
+	if (ret)
+		return ret;
+
+	ret = ccic_dphy_hssettle_set(ccic_dev->index, dphy_freq);
+	if (ret)
+		return ret;
+	pr_debug("ccic%d csi2 %s", ccic_dev->index,
+		 lanes ? "enabled" : "disabled");
+
+	return ret;
+}
+
+static int ccic_config_csi_path_dt_filter(struct ccic_ctrl *ctrl, int path_id,
+					  int filter0_en, u32 filter0,
+					  int filter1_en, u32 filter1)
+{
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+
+	if (path_id == 0) {
+		ccic_set_path0_dt_filter(ccic_dev, filter0_en, filter0,
+					 filter1_en, filter1);
+	} else if (path_id == 1) {
+		ccic_set_path1_dt_filter(ccic_dev, filter0_en, filter0,
+					 filter1_en, filter1);
+	} else if (path_id == 2) {
+		ccic_set_path2_dt_filter(ccic_dev, filter0_en, filter0,
+					 filter1_en, filter1);
+	} else if (path_id == 3) {
+		ccic_set_path3_dt_filter(ccic_dev, filter0_en, filter0,
+					 filter1_en, filter1);
+	} else {
+		dev_err(ccic_dev->dev, "%s invalid path_id %d\n", __func__,
+			path_id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ccic_config_csi_path_vc(struct ccic_ctrl *ctrl, int path_id, u32 vc)
+{
+	struct ccic_dev *ccic_dev = ctrl->ccic_dev;
+
+	if (path_id < 0 || path_id >= 4) {
+		dev_err(ccic_dev->dev, "%s invalid path_id %d\n", __func__,
+			path_id);
+		return -EINVAL;
+	}
+	ccic_set_path_vc(ccic_dev, path_id, vc);
+	return 0;
+}
+
+static struct ccic_ctrl_ops ccic_ctrl_ops = {
+	.irq_mask = ccic_irqmask,
+	.clk_enable = ccic_clk_enable,
+	.config_csi2_mbus = ccic_config_csi2_mbus,
+	.config_csi2_mode = ccic_config_csi2_vc,
+	.config_csi_path_dt_filter = ccic_config_csi_path_dt_filter,
+	.config_csi_path_vc = ccic_config_csi_path_vc,
+};
+
+static int ccic_init_clk(struct ccic_dev *dev)
+{
+	dev->csi_reset = devm_reset_control_get_optional_shared(&dev->pdev->dev,
+								"csi_reset");
+	if (IS_ERR(dev->csi_reset))
+		return PTR_ERR(dev->csi_reset);
+
+	dev->ccic_4x_reset = devm_reset_control_get_optional_shared(
+		&dev->pdev->dev, "ccic_4x_reset");
+	if (IS_ERR(dev->ccic_4x_reset))
+		return PTR_ERR(dev->ccic_4x_reset);
+
+	dev->sc2_hclk_reset = devm_reset_control_get_optional_shared(
+		&dev->pdev->dev, "sc2_hclk_reset");
+	if (IS_ERR(dev->sc2_hclk_reset))
+		return PTR_ERR(dev->sc2_hclk_reset);
+
+	dev->isp_cibus_reset = devm_reset_control_get_optional_shared(
+		&dev->pdev->dev, "isp_cibus_reset");
+	if (IS_ERR(dev->isp_cibus_reset))
+		return PTR_ERR(dev->isp_cibus_reset);
+
+	dev->axi_clk = devm_clk_get(&dev->pdev->dev, "sc2_axi");
+	if (IS_ERR(dev->axi_clk))
+		return PTR_ERR(dev->axi_clk);
+
+	dev->ahb_clk = devm_clk_get(&dev->pdev->dev, "sc2_ahb");
+	if (IS_ERR(dev->ahb_clk))
+		return PTR_ERR(dev->ahb_clk);
+
+	dev->csi_clk = devm_clk_get(&dev->pdev->dev, "csi_func");
+	if (IS_ERR(dev->csi_clk))
+		return PTR_ERR(dev->csi_clk);
+
+	dev->clk4x = devm_clk_get(&dev->pdev->dev, "ccic_func");
+	return PTR_ERR_OR_ZERO(dev->clk4x);
+}
+
+static int ccic_device_register(struct ccic_dev *ccic_dev)
+{
+	struct ccic_dev *other;
+
+	mutex_lock(&list_lock);
+	list_for_each_entry(other, &ccic_devices, list) {
+		if (other->index == ccic_dev->index) {
+			dev_warn(ccic_dev->dev, "ccic%d already registered\n",
+				 ccic_dev->index);
+			mutex_unlock(&list_lock);
+			return -EBUSY;
+		}
+	}
+
+	list_add_tail(&ccic_dev->list, &ccic_devices);
+	mutex_unlock(&list_lock);
+	return 0;
+}
+
+static int ccic_device_unregister(struct ccic_dev *ccic_dev)
+{
+	mutex_lock(&list_lock);
+	list_del(&ccic_dev->list);
+	mutex_unlock(&list_lock);
+	return 0;
+}
+
+int ccic_dphy_hssettle_set(unsigned int ccic_id, unsigned int dphy_freq)
+{
+	u32 reg_settle = 0x00002b00;
+	struct ccic_dev *ccic_dev = NULL;
+	struct ccic_dev *tmp;
+
+	if (dphy_freq < 80) //dphy_clock uint: MHZ
+		return -EINVAL;
+	// RX_Tsettle > TX_HSprepare; reg uint: (1 / (dphy_clock / 2))
+	reg_settle =
+		(HS_PREP_ZERO_MIN + HS_PREP_MAX) * dphy_freq / (2 * 8 * 1000);
+	reg_settle = reg_settle << CSI2_DPHY3_HS_SETTLE_SHIFT;
+
+	mutex_lock(&list_lock);
+	list_for_each_entry(tmp, &ccic_devices, list) {
+		if (tmp->index == ccic_id) {
+			ccic_dev = tmp;
+			break;
+		}
+	}
+	if (!ccic_dev) {
+		pr_err("ccic%d not found", ccic_id);
+		mutex_unlock(&list_lock);
+		return -ENODEV;
+	}
+
+	csiphy_timming_setting(ccic_dev->csiphy, reg_settle);
+	mutex_unlock(&list_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ccic_dphy_hssettle_set);
+
+int ccic_ctrl_get(struct ccic_ctrl **ctrl_host, int id)
+{
+	struct ccic_dev *ccic_dev = NULL;
+	struct ccic_dev *tmp;
+	struct ccic_ctrl *ctrl = NULL;
+
+	list_for_each_entry(tmp, &ccic_devices, list) {
+		if (tmp->index == id) {
+			ccic_dev = tmp;
+			break;
+		}
+	}
+	if (!ccic_dev) {
+		pr_err("ccic%d not found", id);
+		return -ENODEV;
+	}
+
+	ctrl = ccic_dev->ctrl;
+	*ctrl_host = ctrl;
+	pr_debug("acquire ccic%d ctrl dev succeed\n", id);
+
+	return 0;
+}
+EXPORT_SYMBOL(ccic_ctrl_get);
+
+static void ipe_error_irq_handler(struct ccic_dev *ccic, u32 ipestatus,
+				  u32 csi2status)
+{
+	static DEFINE_RATELIMIT_STATE(rs, 5 * HZ, 20);
+
+	if (__ratelimit(&rs)) {
+		pr_err("CCIC%d: interrupt status 0x%08x, csi2 status 0x%08x\n",
+		       ccic->index, ipestatus, csi2status);
+	} else {
+		/* aovid soft lockup due to high frequency interrupt */
+		ccic_reg_clear_bit(ccic, REG_IRQMASK, CSI2PHYERRS);
+		pr_err("CCIC%d: too many interrupt errors, mask\n", ccic->index);
+	}
+}
+
+static int ccic_put_dma_work(struct ccic_dma_context *dma_ctx,
+			     struct ccic_dma_work_struct *ccic_dma_work)
+{
+	unsigned long flags = 0;
+
+	spin_lock_irqsave(&dma_ctx->slock, flags);
+	list_del_init(&ccic_dma_work->busy_list_entry);
+	list_add(&ccic_dma_work->idle_list_entry, &dma_ctx->dma_work_idle_list);
+	spin_unlock_irqrestore(&dma_ctx->slock, flags);
+
+	return 0;
+}
+
+static int ccic_get_dma_work(struct ccic_dma_context *dma_ctx,
+			     struct ccic_dma_work_struct **ccic_dma_work)
+{
+	unsigned long flags = 0;
+
+	spin_lock_irqsave(&dma_ctx->slock, flags);
+	*ccic_dma_work = list_first_entry_or_null(&dma_ctx->dma_work_idle_list,
+						  struct ccic_dma_work_struct,
+						  idle_list_entry);
+	if (NULL == *ccic_dma_work) {
+		spin_unlock_irqrestore(&dma_ctx->slock, flags);
+		return -1;
+	}
+	list_del_init(&((*ccic_dma_work)->idle_list_entry));
+	list_add(&((*ccic_dma_work)->busy_list_entry),
+		 &dma_ctx->dma_work_busy_list);
+	spin_unlock_irqrestore(&dma_ctx->slock, flags);
+
+	return 0;
+}
+
+static void ccic_dma_bh_handler(struct ccic_dma_work_struct *ccic_dma_work)
+{
+	struct ccic_vnode *vnode = ccic_dma_work->sc_vnode;
+	struct device *dev = vnode->ccic_dev->dev;
+	struct ccic_dma_context *dma_ctx = &vnode->dma_ctx;
+	struct ccic_vbuffer *n = NULL, *pos = NULL;
+	//unsigned int irq_status = ccic_dma_work->irq_status;
+	LIST_HEAD(export_list);
+	unsigned long flags = 0;
+	unsigned long wq_flags = 0;
+
+	spin_lock_irqsave(&vnode->waitq_head.lock, wq_flags);
+	vnode->in_tasklet = 1;
+	if (vnode->in_streamoff || !vnode->is_streaming) {
+		wake_up_locked(&vnode->waitq_head);
+		spin_unlock_irqrestore(&vnode->waitq_head.lock, wq_flags);
+		goto dma_tasklet_finish;
+	}
+	wake_up_locked(&vnode->waitq_head);
+	spin_unlock_irqrestore(&vnode->waitq_head.lock, wq_flags);
+	spin_lock_irqsave(&vnode->slock, flags);
+	list_for_each_entry_safe(pos, n, &vnode->busy_list, list_entry) {
+		if (pos->flags &
+		    (BUF_FLAG_HW_ERR | BUF_FLAG_SW_ERR | BUF_FLAG_DONE_TOUCH)) {
+			list_del_init(&(pos->list_entry));
+			atomic_dec(&vnode->busy_buf_cnt);
+			list_add_tail(&(pos->list_entry), &export_list);
+		}
+	}
+	spin_unlock_irqrestore(&vnode->slock, flags);
+	list_for_each_entry_safe(pos, n, &export_list, list_entry) {
+		if (!(pos->flags & BUF_FLAG_SOF_TOUCH)) {
+			dev_warn(
+				dev,
+				"%s export buf index=%u frameid=%u without sof touch\n",
+				vnode->name, pos->vb2_v4l2_buf.vb2_buf.index,
+				pos->vb2_v4l2_buf.sequence);
+		}
+		if (pos->flags & BUF_FLAG_HW_ERR) {
+			//pos->vb2_v4l2_buf.flags |= V4L2_BUF_FLAG_ERROR_HW;
+			dev_warn(
+				dev,
+				"%s export buf index=%u frameid=%u with hw error\n",
+				vnode->name, pos->vb2_v4l2_buf.vb2_buf.index,
+				pos->vb2_v4l2_buf.sequence);
+			cvdev_export_ccic_vbuffer(pos, 1);
+			vnode->hw_err_frm++;
+		} else if (pos->flags & BUF_FLAG_SW_ERR) {
+			//pos->vb2_v4l2_buf.flags |= V4L2_BUF_FLAG_ERROR_SW;
+			dev_warn(
+				dev,
+				"%s export buf index=%u frameid=%u with sw error\n",
+				vnode->name, pos->vb2_v4l2_buf.vb2_buf.index,
+				pos->vb2_v4l2_buf.sequence);
+			cvdev_export_ccic_vbuffer(pos, 1);
+			vnode->sw_err_frm++;
+		} else if (pos->flags & BUF_FLAG_DONE_TOUCH) {
+			cvdev_export_ccic_vbuffer(pos, 0);
+			vnode->ok_frm++;
+		}
+	}
+dma_tasklet_finish:
+	if (vnode) {
+		spin_lock_irqsave(&vnode->waitq_head.lock, wq_flags);
+		vnode->in_tasklet = 0;
+		wake_up_locked(&vnode->waitq_head);
+		spin_unlock_irqrestore(&vnode->waitq_head.lock, wq_flags);
+	}
+	ccic_put_dma_work(dma_ctx, ccic_dma_work);
+}
+
+static void ccic_dma_tasklet_handler(unsigned long param)
+{
+	struct ccic_dma_work_struct *ccic_dma_work =
+		(struct ccic_dma_work_struct *)param;
+	ccic_dma_bh_handler(ccic_dma_work);
+}
+
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+irqreturn_t ccic_mmu_irq_handler(int irq, void *data)
+{
+	unsigned int mmu_irq_status = 0;
+	int i = 0;
+
+	mmu_irq_status = ccic_mmu_call(mmu_dev, irq_status);
+	if (mmu_irq_status & MMU_RD_TIMEOUT) {
+		pr_err("iommu RD_Timeout_error_IRQ\n");
+		ccic_mmu_call(mmu_dev, dump_channel_regs, 0);
+	}
+	if (mmu_irq_status & MMU_WR_TIMEOUT) {
+		pr_err("iommu WR_Timeout_error_IRQ\n");
+		ccic_mmu_call(mmu_dev, dump_channel_regs, 0);
+	}
+	for (i = 0; i < 16; i++) {
+		if (mmu_irq_status & (0x1 << i)) {
+			pr_err("iommu tbu%d/%d dma err\n", 2 * i, 2 * i + 1);
+			ccic_mmu_call(mmu_dev, dump_channel_regs, 2 * i);
+			ccic_mmu_call(mmu_dev, dump_channel_regs, 2 * i + 1);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+#endif
+
+irqreturn_t ccic_dma_irq_handler(int irq, void *data)
+{
+	struct ccic_dma *ccic_dma = (struct ccic_dma *)data;
+	struct ccic_dev *ccic_dev = NULL;
+	struct ccic_vnode *vnode = NULL;
+	struct ccic_vbuffer *pos = NULL, *vb = NULL;
+	struct ccic_dma_work_struct *ccic_dma_work = NULL;
+	struct ccic_dma_context *dma_ctx = NULL;
+	unsigned long wq_flags = 0;
+	unsigned int irq0 = 0, irq1 = 0, dma_ch = 0, irq_status = 0;
+	unsigned int tmp = 0;
+	int i = 0, ret = 0;
+
+	irq0 = ccic_dma_get_irq0(ccic_dma);
+	if (irq0) {
+		ccic_dma_clear_irq0(ccic_dma, irq0);
+	}
+	irq1 = ccic_dma_get_irq1(ccic_dma);
+	if (irq1) {
+		ccic_dma_clear_irq1(ccic_dma, irq1);
+	}
+
+	if (irq1 & BIT(31)) {
+		pr_err("dma_vsync_overflow0\n");
+	}
+	if (irq1 & BIT(30)) {
+		pr_err("dma_data_overflow0\n");
+	}
+	if (irq1 & BIT(29)) {
+		pr_err("txpath_snr1_overflow\n");
+	}
+	if (irq1 & BIT(28)) {
+		pr_err("txpath_snr0_overflow\n");
+	}
+
+	list_for_each_entry(ccic_dev, &ccic_devices, list) {
+		for (i = 0; i < PATH_NUM_PER_DEV; i++) {
+			vnode = (struct ccic_vnode *)ccic_dev->path_vnode[i];
+			dma_ch = vnode->dma_ctx.dma_ch;
+			dma_ctx = &vnode->dma_ctx;
+			if (dma_ch >= MAX_CCIC_DMA_CNT) {
+				continue;
+			}
+			spin_lock_irqsave(&vnode->waitq_head.lock, wq_flags);
+			vnode->in_irq = 1;
+			if (vnode->in_streamoff) {
+				vnode->in_irq = 0;
+				wake_up_locked(&vnode->waitq_head);
+				spin_unlock_irqrestore(&vnode->waitq_head.lock, wq_flags);
+				continue;
+			}
+			spin_unlock_irqrestore(&vnode->waitq_head.lock, wq_flags);
+			do {
+				irq_status = ccic_dma_ch_irq_analyze(dma_ch, irq0, irq1);
+				if (!irq_status) {
+					break;
+				}
+				if (irq_status & DMA_IRQ_SOF) {
+					vnode->frame_id++;
+					vnode->total_frm++;
+				}
+				if (!vnode->is_streaming)
+					break;
+				tmp = irq_status;
+				spin_lock(&(vnode->slock));
+				list_for_each_entry(pos, &(vnode->busy_list),
+						    list_entry) {
+					if (!tmp)
+						break;
+					if (tmp & DMA_IRQ_ERR) {
+						if (!(pos->flags &
+						      BUF_FLAG_SOF_TOUCH)) {
+							dev_info(
+								ccic_dev->dev,
+								"path %d dma_ch %u dma err without sof, drop it\n",
+								i, dma_ch);
+							tmp &= ~DMA_IRQ_ERR;
+						} else if (!(pos->flags &
+							     BUF_FLAG_HW_ERR)) {
+							pos->flags |=
+								BUF_FLAG_HW_ERR;
+							dev_info(
+								ccic_dev->dev,
+								"path %d dma_ch %u dma err\n",
+								i, dma_ch);
+							tmp &= ~DMA_IRQ_ERR;
+						}
+					}
+					if (tmp & DMA_IRQ_DONE) {
+						if (!(pos->flags &
+						      BUF_FLAG_SOF_TOUCH)) {
+							dev_info(
+								ccic_dev->dev,
+								"path %d dma_ch %u dma done without sof, drop it\n",
+								i, dma_ch);
+							tmp &= ~DMA_IRQ_DONE;
+						} else if (!(pos->flags &
+							     BUF_FLAG_DONE_TOUCH)) {
+							pos->flags |=
+								BUF_FLAG_DONE_TOUCH;
+							pos->vb2_v4l2_buf
+								.sequence =
+								vnode->frame_id -
+								1;
+							pos->vb2_v4l2_buf
+								.vb2_buf
+								.timestamp =
+								ktime_get_ns();
+							pos->vb2_v4l2_buf
+								.vb2_buf
+								.planes[0]
+								.bytesused =
+								ccic_reg_read(
+									ccic_dev,
+									REG_FRAME_CNT);
+							tmp &= ~DMA_IRQ_DONE;
+							dev_dbg(ccic_dev->dev,
+								"path %d dma_ch %u dma done\n",
+								i, dma_ch);
+						}
+						if (vnode->wait_done_flush) {
+							complete(
+								&vnode->flush_complete);
+							vnode->wait_done_flush =
+								0;
+						}
+					}
+					if (tmp & DMA_IRQ_SOF) {
+						if (pos->flags &
+						    BUF_FLAG_SOF_TOUCH) {
+							if (!(pos->flags &
+							      (BUF_FLAG_DONE_TOUCH |
+							       BUF_FLAG_HW_ERR |
+							       BUF_FLAG_SW_ERR))) {
+								dev_warn(
+									ccic_dev->dev,
+									"path %d dma_ch %u next sof arrived without dma done or err\n",
+									i,
+									dma_ch);
+								pos->flags |=
+									BUF_FLAG_SW_ERR;
+							}
+						} else {
+							pos->flags |=
+								(BUF_FLAG_SOF_TOUCH |
+								 BUF_FLAG_TIMESTAMPED);
+							tmp &= ~DMA_IRQ_SOF;
+						}
+					}
+				}
+				spin_unlock(&(vnode->slock));
+				ret = ccic_get_dma_work(dma_ctx,
+							&ccic_dma_work);
+				if (ret) {
+					dev_warn(
+						ccic_dev->dev,
+						"dma work idle list was null\n");
+				} else {
+					ccic_dma_work->irq_status = irq_status;
+					tasklet_schedule(
+						&(ccic_dma_work->dma_tasklet));
+				}
+				if (irq_status & DMA_IRQ_SOF) {
+					cvdev_dq_idle_vbuffer(vnode, &vb);
+					if (vb) {
+						cvdev_q_busy_vbuffer(vnode, vb);
+						ccic_update_dma_addr(vnode, vb,
+								     0);
+						ccic_dma_ch_shadow_ready(
+							ccic_dma, dma_ch, 1);
+					}
+				}
+			} while (0);
+			spin_lock_irqsave(&vnode->waitq_head.lock, wq_flags);
+			vnode->in_irq = 0;
+			wake_up_locked(&vnode->waitq_head);
+			spin_unlock_irqrestore(&vnode->waitq_head.lock, wq_flags);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t ccic_isr(int irq, void *data)
+{
+	struct ccic_dev *ccic_dev = data;
+	uint32_t csi2status = 0, irqs = 0;
+
+	irqs = ccic_reg_read(ccic_dev, REG_IRQSTAT);
+	if (irqs) {
+		ccic_reg_write(ccic_dev, REG_IRQSTAT, irqs);
+	}
+
+	csi2status = ccic_reg_read(ccic_dev, 0x108);
+	if (irqs & CSI2PHYERRS)
+		ipe_error_irq_handler(ccic_dev, irqs, csi2status);
+
+	if (irqs & IRQ_DMA_PRO_LINE)
+		pr_debug("CCIC%d: IRQ_DMA_PRO_LINE\n", ccic_dev->index);
+
+	//if (irqs & IRQ_IDI_PRO_LINE)
+	//	pr_debug("CCIC%d: IRQ_IDI_PRO_LINE\n", ccic_dev->index);
+
+	if (irqs & IRQ_CSI2IDI_FLUSH)
+		pr_debug("CCIC%d: IRQ_CSI2IDI_FLUSH\n", ccic_dev->index);
+
+	if (irqs & IRQ_CSI2IDI_HBLK2HSYNC)
+		pr_debug("CCIC%d: IRQ_CSI2IDI_HBLK2HSYNC\n", ccic_dev->index);
+
+	if (irqs & IRQ_DPHY_RX_CLKULPS_ACTIVE)
+		pr_debug("CCIC%d: IRQ_DPHY_RX_CLKULPS_ACTIVE\n",
+			 ccic_dev->index);
+
+	if (irqs & IRQ_DPHY_RX_CLKULPS)
+		pr_debug("CCIC%d: IRQ_DPHY_RX_CLKULPS\n", ccic_dev->index);
+
+	if (irqs & IRQ_DPHY_LN_ULPS_ACTIVE)
+		pr_debug("CCIC%d: IRQ_DPHY_LN_ULPS_ACTIVE\n", ccic_dev->index);
+
+	if (irqs & IRQ_CSI_SOF) {
+		pr_debug("CCIC%d: IRQ_CSI_SOF\n", ccic_dev->index);
+	}
+	if (irqs & IRQ_CSI_EOF) {
+		pr_debug("CCIC%d: IRQ_CSI_EOF\n", ccic_dev->index);
+	}
+	return IRQ_HANDLED;
+}
+
+static int ccic_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct ccic_dev *ccic_dev;
+	struct ccic_ctrl *ccic_ctrl;
+	struct device *dev = &pdev->dev;
+	int ret = 0, i = 0;
+	char buf[32];
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+	size_t tabs_size = 0, tab_offset = 0;
+	int j = 0;
+	void __iomem *mmu_reg_base = NULL;
+#endif
+
+	ret = of_property_read_u32(np, "cell-index", &pdev->id);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to get alias id, errno %d\n", ret);
+		return ret;
+	}
+
+	ccic_dev = devm_kzalloc(&pdev->dev, sizeof(*ccic_dev), GFP_KERNEL);
+	if (!ccic_dev) {
+		dev_err(&pdev->dev, "camera: Could not allocate ccic dev\n");
+		return -ENOMEM;
+	}
+
+	ccic_ctrl = devm_kzalloc(&pdev->dev, sizeof(*ccic_ctrl), GFP_KERNEL);
+	if (!ccic_ctrl) {
+		dev_err(&pdev->dev, "camera: Could not allocate ctrl dev\n");
+		return -ENOMEM;
+	}
+
+	/* get mem */
+	ccic_dev->mem =
+		platform_get_resource_byname(pdev, IORESOURCE_MEM, "ccic-regs");
+	if (!ccic_dev->mem) {
+		dev_err(&pdev->dev, "no mem resource");
+		return -ENODEV;
+	}
+	ccic_dev->base = devm_ioremap(&pdev->dev, ccic_dev->mem->start,
+				      resource_size(ccic_dev->mem));
+	if (IS_ERR(ccic_dev->base)) {
+		dev_err(&pdev->dev, "fail to remap iomem\n");
+		return PTR_ERR(ccic_dev->base);
+	}
+
+	/* get irqs */
+	ccic_dev->irq = platform_get_irq_byname(pdev, "ccic-irq");
+	if (ccic_dev->irq < 0)
+		return ccic_dev->irq;
+	ret = devm_request_irq(&pdev->dev, ccic_dev->irq, ccic_isr,
+			       IRQF_SHARED, CCIC_DRV_NAME, ccic_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "fail to request irq\n");
+		return ret;
+	}
+
+	/* ccic device and ctrl init */
+	ccic_ctrl->ccic_dev = ccic_dev;
+	ccic_ctrl->index = pdev->id;
+	ccic_ctrl->ops = &ccic_ctrl_ops;
+	atomic_set(&ccic_ctrl->usr_cnt, 0);
+	mutex_init(&ccic_ctrl->lock);
+
+	ccic_dev->csiphy =
+		csiphy_lookup_by_phandle(&pdev->dev, "spacemit,csiphy");
+	if (!ccic_dev->csiphy) {
+		dev_err(&pdev->dev, "fail to acquire csiphy\n");
+		return -EPROBE_DEFER;
+	}
+
+	ccic_dev->index = pdev->id;
+	ccic_dev->pdev = pdev;
+	ccic_dev->dev = &pdev->dev;
+	ccic_dev->ctrl = ccic_ctrl;
+	ccic_dev->interrupt_mask_value = CSI2PHYERRS | FRAMEIRQS;
+	dev_set_drvdata(dev, ccic_dev);
+
+	ccic_init_clk(ccic_dev);
+
+	ccic_device_register(ccic_dev);
+
+	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(33));
+	ret = v4l2_device_register(&pdev->dev, &ccic_dev->v4l2_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to register v4l2 dev\n");
+		return ret;
+	}
+	snprintf(ccic_dev->name, 32, "csi%d", ccic_ctrl->index);
+	for (i = 0; i < 4; i++) {
+		snprintf(buf, 32, "csi%d_path%d", ccic_ctrl->index, i);
+		ccic_dev->path_vnode[i] = cvdev_create_vnode(buf, ccic_dev->index * 4 + i,
+							  &ccic_dev->v4l2_dev,
+							  &pdev->dev,
+							  ccic_dev,
+							  ccic_dma_tasklet_handler,
+							  0);
+		if (NULL == ccic_dev->path_vnode[i]) {
+			dev_err(&pdev->dev,
+				"failed to create ccic path vnode %d\n", i);
+			return -EPROBE_DEFER;
+		}
+	}
+
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+	if (!mmu_dev) {
+		rsvd_vaddr =
+			devm_kmalloc(dev, MMU_RESERVED_MEM_SIZE, GFP_KERNEL);
+		if (!rsvd_vaddr) {
+			pr_err("failed to alloc mem for mmu reserved");
+			return -EPROBE_DEFER;
+		}
+		rsvd_phy_addr = virt_to_phys(rsvd_vaddr);
+		pr_info("rsvd_phy_addr=0x%llx size=%d", (uint64_t)rsvd_phy_addr,
+			MMU_RESERVED_MEM_SIZE);
+		memset(rsvd_vaddr, 0xff, MMU_RESERVED_MEM_SIZE);
+		mmu_reg_base = ioremap(MMU_REG_BASE, PAGE_SIZE);
+		if (!mmu_reg_base) {
+			pr_err("failed to ioremap mmu reg base");
+			return -EPROBE_DEFER;
+		}
+		mmu_dev = ccic_iommu_create(dev, (unsigned long)mmu_reg_base);
+		if (!mmu_dev) {
+			pr_err("failed to create iommu device");
+			return -EPROBE_DEFER;
+		}
+		tabs_size = 2 * IOMMU_TRANS_TAB_MAX_NUM * sizeof(uint32_t) * CCIC_IOMMU_TBU_NUM;
+		trans_tab_cpu_addr = dmam_alloc_coherent(dev,
+							tabs_size,
+							&trans_tab_dma_addr, GFP_KERNEL);
+		if (!trans_tab_cpu_addr) {
+			pr_err("%s alloc page tables failed", __func__);
+			return -EPROBE_DEFER;
+		}
+		total_trans_tab_sz = tabs_size;
+		tab_offset = 0;
+		for (i = 0; i < CCIC_IOMMU_CH_NUM; i++) {
+			for (j = 0; j < 2; j++) {
+				mmu_ctx[i].tt_addr[j][0] = trans_tab_dma_addr + tab_offset;
+				mmu_ctx[i].tt_base[j][0] = trans_tab_cpu_addr + tab_offset;
+				tab_offset += IOMMU_TRANS_TAB_MAX_NUM * sizeof(uint32_t);
+			}
+		}
+		pr_info("ccic mmu tbl size alloc:%lu needed:%lu", tabs_size,
+			tab_offset);
+	}
+#endif
+
+	atomic_set(&ccic_dev->open_cnt, 0);
+
+	pr_info("%s probed", dev_name(&pdev->dev));
+
+	return ret;
+}
+
+static void ccic_remove(struct platform_device *pdev)
+{
+	struct ccic_dev *ccic_dev;
+	int i = 0;
+
+	ccic_dev = dev_get_drvdata(&pdev->dev);
+
+	for (i = 0; i < 4; i++) {
+		cvdev_destroy_vnode(
+			(struct ccic_vnode *)ccic_dev->path_vnode[i]);
+	}
+
+	v4l2_device_unregister(&ccic_dev->v4l2_dev);
+	ccic_device_unregister(ccic_dev);
+}
+
+static const struct of_device_id ccic_dt_match[] = {
+	{
+		.compatible = "spacemit,ccic",
+		.data = NULL,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, ccic_dt_match);
+
+struct platform_driver ccic_driver = {
+	.driver = {
+		.name = CCIC_DRV_NAME,
+		.of_match_table = of_match_ptr(ccic_dt_match),
+	},
+	.probe = ccic_probe,
+	.remove = ccic_remove,
+};
+
+extern struct platform_driver ccic_dma_driver;
+
+static int __init ccic_driver_init(void)
+{
+	int ret;
+
+	ret = ccic_csiphy_register();
+	if (ret < 0)
+		return ret;
+
+	ret = platform_driver_register(&ccic_driver);
+	if (ret == 0) {
+		ret = platform_driver_register(&ccic_dma_driver);
+	}
+
+	if (ret < 0)
+		ccic_csiphy_unregister();
+
+	return ret;
+}
+
+static void __exit ccic_driver_exit(void)
+{
+	platform_driver_unregister(&ccic_dma_driver);
+	platform_driver_unregister(&ccic_driver);
+
+	ccic_csiphy_unregister();
+}
+
+module_init(ccic_driver_init);
+module_exit(ccic_driver_exit);
+
+MODULE_DESCRIPTION("SPACEMIT CCIC Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_drv.h b/drivers/media/platform/spacemit/cam_ccic/ccic_drv.h
new file mode 100644
index 000000000000..82f7dedd182e
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_drv.h
@@ -0,0 +1,224 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ccic_drv.h - Driver for ccic
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef _CCIC_DRV_H_
+#define _CCIC_DRV_H_
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of_device.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-v4l2.h>
+#include <linux/reset.h>
+//#include "cam_dbg.h"
+
+#define SC2_MODE_CCIC 1
+#define SC2_MODE_ISP 2
+
+#define MHZ 1000000
+/*
+ * the min/max is for calcuting DPHY, the unit is ns.
+ */
+#define D_TERMEN_MAX (35)
+#define HS_PREP_MIN (40)
+#define HS_PREP_MAX (85)
+#define HS_PREP_ZERO_MIN (145)
+#define NS_TO_PS(nsec) ((nsec) * 1000)
+#define CCIC_CSI_DEV_MAX (4)
+#define PATH_NUM_PER_DEV (4)
+#define PATH_NUM_MAX (CCIC_CSI_DEV_MAX * PATH_NUM_PER_DEV)
+
+/* MIPI related */
+/* Sensor MIPI behavior descriptor, sensor driver should pass it to controller
+ * driver, and let controller driver decide how to config its PHY registers */
+struct csi_dphy_desc {
+	u32 clk_mul;
+	u32 clk_div; /* clock_lane_freq = input_clock * clk_mul / clk_div */
+	u32 clk_freq;
+	u32 cl_prepare; /* cl_* describes clock lane timing in the unit of ns */
+	u32 cl_zero;
+	u32 hs_prepare; /* hs_* describes data LP to HS transition timing */
+	u32 hs_zero; /* in the unit of clock lane period(DDR period) */
+	u32 nr_lane; /* When set to 0, S/W will try to figure out a value */
+};
+
+struct mipi_csi2 {
+	int dphy_type; /* 0: DPHY on chip, 1: DPTC off chip */
+	u32 dphy[5]; /* DPHY:  CSI2_DPHY1, CSI2_DPHY2, CSI2_DPHY3, CSI2_DPHY5, CSI2_DPHY6 */
+	int calc_dphy;
+	struct csi_dphy_desc dphy_desc;
+	int enable_dpcm;
+};
+
+#define HS_SETTLE_POS_MAX (100)
+struct csi_dphy_calc {
+	char name[16];
+	int hs_termen_pos;
+	int hs_settle_pos; /* 0~100 */
+};
+
+struct csi_dphy_reg {
+	u16 cl_termen;
+	u16 cl_settle;
+	u16 cl_miss;
+	u16 hs_termen;
+	u16 hs_settle;
+	u16 hs_rx_to;
+	u16 lane; /* When set to 0, S/W will try to figure out a value */
+	u16 vc; /* Virtual channel */
+	u16 dt1; /* Data type 1: For video or main data type */
+	u16 dt2; /* Data type 2: For thumbnail or auxiliry data type */
+};
+
+struct ccic_ctrl {
+	int index;
+	atomic_t usr_cnt;
+	struct mipi_csi2 csi;
+	struct ccic_dev *ccic_dev;
+	struct ccic_ctrl_ops *ops;
+	struct mutex lock;
+	//irqreturn_t (*handler)(struct ccic_ctrl *, u32);
+};
+
+enum ccic_idi {
+	CCIC_CSI2IDI0 = 0,
+	CCIC_CSI2IDI1,
+};
+
+enum ccic_idi_sel {
+	CCIC_IDI_SEL_NONE = 0,
+	CCIC_IDI_SEL_DPCM,
+	CCIC_IDI_SEL_REPACK,
+	CCIC_IDI_SEL_PARALLEL,
+	CCIC_IDI_SEL_AHB,
+	CCIC_IDI_RELEASE_RESET,
+};
+
+enum ccic_idi_mux {
+	CCIC_IDI_MUX_LOCAL_MAIN = 0,
+	CCIC_IDI_MUX_IPE2_VCDT,
+	CCIC_IDI_MUX_IPE2_MAIN,
+	CCIC_IDI_MUX_REMOTE_VCDT,
+};
+
+enum ccic_csi2vc_mode {
+	CCIC_CSI2VC_NM = 0,
+	CCIC_CSI2VC_VC,
+	CCIC_CSI2VC_DT,
+};
+
+enum ccic_csi2vc_chnl {
+	CCIC_CSI2VC_MAIN = 0,
+	CCIC_CSI2VC_VCDT,
+};
+
+enum csi_dma_dump_format {
+	CSI_DUMP_FMT_YUV422 = 0,
+	CSI_DUMP_FMT_RAW8,
+	CSI_DUMP_FMT_RAW10,
+	CSI_DUMP_FMT_RAW12,
+	CSI_DUMP_FMT_MAX,
+};
+
+struct ccic_ctrl_ops {
+	void (*irq_mask)(struct ccic_ctrl *ctrl, int on);
+	int (*clk_enable)(struct ccic_ctrl *ctrl, int en);
+	int (*config_csi2_mode)(struct ccic_ctrl *ctrl, int mode,
+				unsigned int dt_en);
+	int (*config_csi2_mbus)(struct ccic_ctrl *ctrl, int lanes,
+				int mipi_m_bps);
+	int (*config_csi_path_dt_filter)(struct ccic_ctrl *ctrl, int path_id,
+					 int filter0_en, u32 filter0,
+					 int filter1_en, u32 filter1);
+	int (*config_csi_path_vc)(struct ccic_ctrl *ctrl, int path_id, u32 vc);
+};
+
+//todo: to delete
+enum ccic_dma_sel {
+	CCIC_DMA_SEL_LOCAL_MAIN = 0,
+	CCIC_DMA_SEL_LOCAL_VCDT,
+	CCIC_DMA_SEL_REMOTE_MAIN,
+	CCIC_DMA_SEL_REMOTE_VCDT,
+};
+
+struct ccic_dev {
+	//struct miscdevice mdev;
+	char name[32];
+	int index;
+	struct device *dev;
+	struct platform_device *pdev;
+	struct list_head list;
+	int irq;
+	struct resource *mem;
+	void __iomem *base;
+	struct clk *csi_clk;
+	struct clk *clk4x;
+	struct clk *ahb_clk;
+	struct clk *axi_clk;
+
+	struct reset_control *csi_reset;
+	struct reset_control *ccic_4x_reset;
+	struct reset_control *sc2_hclk_reset;
+	struct reset_control *isp_cibus_reset;
+	int dma_burst;
+	spinlock_t ccic_lock; /* protect the struct members and HW */
+	u32 interrupt_mask_value;
+
+	/* object for ccic csi part */
+	struct ccic_ctrl *ctrl;
+	/* object for csiphy part */
+	struct csiphy_device *csiphy;
+	struct v4l2_device v4l2_dev;
+	void *vnode;
+	void *path_vnode[PATH_NUM_PER_DEV];
+
+	atomic_t open_cnt;
+};
+
+/*
+ * Device register I/O
+ */
+static inline u32 ccic_reg_read(struct ccic_dev *ccic_dev, unsigned int reg)
+{
+	return ioread32(ccic_dev->base + reg);
+}
+
+static inline void ccic_reg_write(struct ccic_dev *ccic_dev, unsigned int reg,
+				  u32 val)
+{
+	iowrite32(val, ccic_dev->base + reg);
+}
+
+static inline void ccic_reg_write_mask(struct ccic_dev *ccic_dev,
+				       unsigned int reg, u32 val, u32 mask)
+{
+	u32 v = ccic_reg_read(ccic_dev, reg);
+
+	v = (v & ~mask) | (val & mask);
+	ccic_reg_write(ccic_dev, reg, v);
+}
+
+static inline void ccic_reg_set_bit(struct ccic_dev *ccic_dev, unsigned int reg,
+				    u32 val)
+{
+	ccic_reg_write_mask(ccic_dev, reg, val, val);
+}
+
+static inline void ccic_reg_clear_bit(struct ccic_dev *ccic_dev,
+				      unsigned int reg, u32 val)
+{
+	ccic_reg_write_mask(ccic_dev, reg, 0, val);
+}
+
+int ccic_ctrl_get(struct ccic_ctrl **ctrl_host, int id);
+int ccic_dphy_hssettle_set(unsigned int ccic_id, unsigned int dphy_freg);
+irqreturn_t ccic_mmu_irq_handler(int irq, void *data);
+irqreturn_t ccic_dma_irq_handler(int irq, void *data);
+#endif
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_hwreg.c b/drivers/media/platform/spacemit/cam_ccic/ccic_hwreg.c
new file mode 100644
index 000000000000..50bc9d55ba14
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_hwreg.c
@@ -0,0 +1,527 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Spacemit CCIC driver
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#include <media/v4l2-dev.h>
+#include "ccic_drv.h"
+#include "ccic_hwreg.h"
+
+#if 0
+/* TODO: ccic config for raw dump on ctest, hard coding */
+int mipi_sensor_TASK(struct ccic_dev *dev, uint8_t RAW_type,
+		uint32_t sensor_width, uint32_t sensor_height)
+{
+	uint32_t ipe1_yuvinfmt, ipe1_yuvoutend, ipe1_yuvoutfmt;
+	uint32_t ipe1_rgboutend, ipe1_rgbinoutfmt, ipe1_rgbendfmt, ipe1_doutfmt, ipe1_dinfmt, ipe1_420sp;
+	uint32_t ipe1_sifmode, sensor_pclk, sensor_vsync, sensor_hsync, sensor_vsync_edge;
+	uint32_t ipe1_image_width, ipe1_image_height, ipe1_y_mem_pitch, ipe1_uv_mem_pitch;
+
+	uint32_t ipe1_donesel, ipe1_widsel, ipe1_srampdwn, ipe1_pwrdnen, ipe1_dropsel420, ipe1_dmaburstsel,
+		 dma_burst_length_sel, ipe1_hsynccnt_ctrl, ipe1_sensorclkgate;
+
+	uint32_t lgcy, frame_done_irq_sel, ipe1_linebufnum, ipe1_linebufen ;
+
+	/* 0xb4: 0x00000030 */
+	ccic_reg_write(dev, 0xb4, 0x30);
+
+	/* 0x34: 0x05a00c80 */
+	if (RAW_type == 12) {
+		ipe1_image_width = sensor_width * 3 / 2; /* Image Width in bytes, raw12 */
+	} else if (RAW_type == 10) {
+		ipe1_image_width = sensor_width * 5 / 4; /* Image Width in bytes, raw10 */
+	} else {
+		pr_err("%s: invalid raw type %d\n", __func__, RAW_type);
+		return -EINVAL;
+	}
+	ipe1_image_height = sensor_height << 16; /* Image Length in scanline */
+	ccic_reg_write(dev, REG_IMGSIZE, ipe1_image_width | ipe1_image_height);
+
+	/* 0x24: 0x00000c80 */
+	ipe1_y_mem_pitch = ipe1_image_width;  /* This field is the distance between two vertical adjacent pixels, need 8-bytes aligned */
+	ipe1_uv_mem_pitch = 0 << 16; /* This field is the distance between two vertical adjacent pixels, need 8-bytes aligned */
+	ccic_reg_write(dev, REG_IMGPITCH, ipe1_y_mem_pitch | ipe1_uv_mem_pitch);
+
+	/* 0x38: 0x00000000 */
+	ccic_reg_write(dev, REG_IMGOFFSET, 0x0);
+
+	/* 0x3c: 0x00008001 */
+	ipe1_sifmode = 0 << 30; /* Master mode with or without hsync/vsync */
+	sensor_pclk = 0 << 26; /* Parallel CMOS Sensor VCLK Polarity */
+	sensor_vsync = 0 << 25; /* Parallel CMOS Sensor VSYNC Polarity */
+	sensor_hsync = 0 << 24; /* Parallel CMOS Sensor HSYNC Polarity */
+	sensor_vsync_edge = 0 << 23; /* Vsync Edge Control */
+	ipe1_yuvinfmt = 0 << 18; /* YUV Format */
+	ipe1_yuvoutend = 0 << 16; /* YCbCr Endianness Format */
+	ipe1_yuvoutfmt = 4 << 13; /* YCbCr Output Format */
+	ipe1_rgboutend = 0 << 12; /* don't care */
+	ipe1_rgbinoutfmt = 0 << 9; /* don't care */
+	ipe1_doutfmt = 1 << 8; /* CCIC's Output Data Format */
+	ipe1_dinfmt = 1 << 6; /* CCIC's Input Data Format */
+	ipe1_420sp = 0 << 4;
+	ipe1_rgbendfmt = 0 << 2; /* don't care */
+	ccic_reg_write(dev, REG_CTRL0,
+			ipe1_sifmode | sensor_pclk | sensor_vsync | sensor_hsync |
+			sensor_vsync_edge | ipe1_yuvinfmt | ipe1_yuvoutend |
+			ipe1_yuvoutfmt | ipe1_rgboutend | ipe1_rgbinoutfmt |
+			ipe1_rgbendfmt | ipe1_doutfmt | ipe1_dinfmt | 1 << 21);
+
+	/* 0x40: 0x8402003c */
+	ipe1_donesel = 1 << 31; /* 0: WLAST_dly; 1: bvalid_reg */
+	ipe1_widsel = 0 << 30; /* 0: always posted; 1: non-posted at fe_to_fs */
+	ipe1_widsel = 0 << 30; /* 0: always posted; 1: non-posted at fe_to_fs */
+	ipe1_srampdwn = 0 << 29; /* SRAM Power Down */
+	ipe1_pwrdnen = 0 << 28; /* Power Down Enable */
+	ipe1_dmaburstsel = 2 << 25; /* 0: 64-byte; 1: 128-byte; 2: 256-byte */
+	ipe1_dropsel420 = 0 << 24;
+	dma_burst_length_sel = 1 << 17;
+	ipe1_hsynccnt_ctrl =
+		0x1e << 1; /* controls line_end indication when sensorclkgate = 1 */
+	ipe1_sensorclkgate =
+		0; /* 0: parallel sensor output clock is always running; 1: parallel sensor output clock is gated when not used */
+	ccic_reg_write(dev, REG_CTRL1,
+			ipe1_donesel | ipe1_widsel | ipe1_srampdwn | ipe1_pwrdnen |
+			ipe1_dmaburstsel | ipe1_dropsel420 |
+			dma_burst_length_sel | ipe1_hsynccnt_ctrl |
+			ipe1_sensorclkgate);
+
+	/* 0x44: 0xc0000000 */
+	lgcy = 0x3 << 30;
+	frame_done_irq_sel = 0 << 29;
+	ipe1_linebufnum = 0 << 17;
+	ipe1_linebufen = 0 << 16;
+	ccic_reg_write(dev, REG_CTRL2, lgcy | frame_done_irq_sel | ipe1_linebufnum | ipe1_linebufen);
+
+	/* 0x48: 0xc0000000 */
+	ccic_reg_write(dev, REG_CTRL3, 0xc0000000);
+
+	/* 0x88: 0x00000003 */
+	ccic_reg_write(dev, 0x88, 0x3);
+
+	/* 0x3c: enable ccic dma */
+	ccic_reg_set_bit(dev, REG_CTRL0, BIT(0));
+
+	return 0;
+}
+#endif
+#if 0
+	int ccic_csi2_config_dphy(struct ccic_dev *ccic_dev, int lanes,
+				int enable)
+	{
+	unsigned int dphy2_val = 0xa2848888;
+	unsigned int dphy3_val = 0x00001500;
+	/* unsigned int dphy4_val = 0x00000000; */
+	unsigned int dphy5_val = 0x000000ff; /* 4lanes */
+	unsigned int dphy6_val = 0x1001;
+
+	if (!enable) {
+		ccic_reg_write(ccic_dev, REG_CSI2_DPHY5, 0x00);
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DPHY1,
+					CSI2_DHPY1_ANA_PU); /* analog power off */
+		return 0;
+	}
+
+	if (lanes < 1 || lanes > 4)
+		return -EINVAL;
+
+	dphy5_val = CSI2_DPHY5_LANE_ENA(lanes);
+	dphy5_val = dphy5_val | (dphy5_val << CSI2_DPHY5_LANE_RESC_ENA_SHIFT);
+
+	ccic_reg_write(ccic_dev, REG_CSI2_DPHY2, dphy2_val);
+	ccic_reg_write(ccic_dev, REG_CSI2_DPHY3, dphy3_val);
+	/* ccic_reg_write(ccic_dev, REG_CSI2_DPHY4, dphy4_val); */
+	ccic_reg_write(ccic_dev, REG_CSI2_DPHY5, dphy5_val);
+	ccic_reg_write(ccic_dev, REG_CSI2_DPHY6, dphy6_val);
+	ccic_reg_set_bit(ccic_dev, REG_CSI2_DPHY1,
+				CSI2_DHPY1_ANA_PU); /* analog power on */
+
+	return 0;
+	}
+#endif
+int ccic_csi2_lanes_enable(struct ccic_dev *ccic_dev, int lanes)
+{
+	unsigned int ctrl0_val = 0;
+
+	if (lanes < 0 || lanes > 4)
+		return -EINVAL;
+
+	if (!lanes) { /* Disable MIPI CSI2 Interface */
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_CTRL0,
+				   CSI2_C0_ENABLE); /* csi off */
+		return 0;
+	}
+
+	ctrl0_val = ccic_reg_read(ccic_dev, REG_CSI2_CTRL0);
+	ctrl0_val &= ~(CSI2_C0_LANE_NUM_MASK);
+	ctrl0_val |= CSI2_C0_LANE_NUM(lanes);
+	ctrl0_val |= CSI2_C0_ENABLE;
+	ctrl0_val &= ~(CSI2_C0_VLEN_MASK);
+	ctrl0_val |= CSI2_C0_VLEN;
+
+	ccic_reg_write(ccic_dev, REG_CSI2_CTRL0, ctrl0_val);
+
+	return 0;
+}
+
+void ccic_set_path_vc(struct ccic_dev *ccic_dev, int path_id, u32 vc)
+{
+	if (path_id == 0) {
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_VCCTRL, vc << 14,
+				    CSI2_VCCTRL_VC0_MASK);
+	} else if (path_id == 1) {
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_VCCTRL, vc << 22,
+				    CSI2_VCCTRL_VC1_MASK);
+	} else if (path_id == 2) {
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_VCCTRL2, vc << 6,
+				    CSI2_VCCTRL_VC2_MASK);
+	} else if (path_id == 3) {
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_VCCTRL2, vc << 14,
+				    CSI2_VCCTRL_VC3_MASK);
+	}
+}
+
+int ccic_csi2_vc_ctrl(struct ccic_dev *ccic_dev, int md, unsigned int dt_en)
+{
+	int ret = 0;
+
+	switch (md) {
+	case CCIC_CSI2VC_NM: /* Normal mode */
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_VCCTRL,
+				    CSI2_VCCTRL_MD_NORMAL, CSI2_VCCTRL_MD_MASK);
+		ccic_en_dt_match_pass_mode(ccic_dev, 0);
+		break;
+	case CCIC_CSI2VC_VC: /* Virtual Channel mode */
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_VCCTRL,
+				    CSI2_VCCTRL_MD_VC, CSI2_VCCTRL_MD_MASK);
+		ccic_en_dt_match_pass_mode(ccic_dev, dt_en);
+		break;
+	case CCIC_CSI2VC_DT: /* TODO: Data-Type Interleaving */
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_VCCTRL,
+				    CSI2_VCCTRL_MD_DT, CSI2_VCCTRL_MD_MASK);
+		ccic_en_dt_match_pass_mode(ccic_dev, dt_en);
+		pr_info("csi2 vc mode %d todo\n", md);
+		break;
+	default:
+		pr_err("%s: invalid csi2 vc mode %d\n", __func__, md);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+void ccic_en_dt_match_pass_mode(struct ccic_dev *ccic_dev, int en)
+{
+	if (en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_VCCTRL,
+				 CSI2_VCCTRL_DT_PASS_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_VCCTRL,
+				   CSI2_VCCTRL_DT_PASS_MASK);
+	}
+}
+
+void ccic_set_path0_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data)
+{
+	if (flt0_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT,
+				 CSI2_DT_FLT0_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT, flt0_data,
+				    CSI2_DT_FLT0_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT,
+				   CSI2_DT_FLT0_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT, 0,
+				    CSI2_DT_FLT0_PATTERN_MASK);
+	}
+
+	if (flt1_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT,
+				 CSI2_DT_FLT1_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT, (flt1_data << 8),
+				    CSI2_DT_FLT1_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT,
+				   CSI2_DT_FLT1_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT, 0,
+				    CSI2_DT_FLT1_PATTERN_MASK);
+	}
+}
+
+void ccic_set_path1_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data)
+{
+	if (flt0_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT,
+				 CSI2_DT_FLT2_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT,
+				    (flt0_data << 16),
+				    CSI2_DT_FLT2_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT,
+				   CSI2_DT_FLT2_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT, 0,
+				    CSI2_DT_FLT2_PATTERN_MASK);
+	}
+
+	if (flt1_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT,
+				 CSI2_DT_FLT3_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT,
+				    (flt1_data << 24),
+				    CSI2_DT_FLT3_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT,
+				   CSI2_DT_FLT3_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT, 0,
+				    CSI2_DT_FLT3_PATTERN_MASK);
+	}
+}
+
+void ccic_set_path2_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data)
+{
+	if (flt0_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				 CSI2_DT_FLT4_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2, flt0_data,
+				    CSI2_DT_FLT4_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				   CSI2_DT_FLT4_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2, 0,
+				    CSI2_DT_FLT4_PATTERN_MASK);
+	}
+
+	if (flt1_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				 CSI2_DT_FLT5_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2,
+				    (flt1_data << 8),
+				    CSI2_DT_FLT5_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				   CSI2_DT_FLT5_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2, 0,
+				    CSI2_DT_FLT5_PATTERN_MASK);
+	}
+}
+
+void ccic_set_path3_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data)
+{
+	if (flt0_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				 CSI2_DT_FLT6_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2,
+				    (flt0_data << 16),
+				    CSI2_DT_FLT6_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				   CSI2_DT_FLT6_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2, 0,
+				    CSI2_DT_FLT6_PATTERN_MASK);
+	}
+
+	if (flt1_en) {
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				 CSI2_DT_FLT7_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2,
+				    (flt1_data << 24),
+				    CSI2_DT_FLT7_PATTERN_MASK);
+	} else {
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_DT_FLT2,
+				   CSI2_DT_FLT7_EN_MASK);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_DT_FLT2, 0,
+				    CSI2_DT_FLT7_PATTERN_MASK);
+	}
+}
+
+int ccic_dma_set_burst(struct ccic_dev *ccic_dev)
+{
+	u32 dma_burst;
+
+	/* setup the DMA burst */
+	switch (ccic_dev->dma_burst) {
+	case 128:
+		dma_burst = C1_DMAB128;
+		break;
+	case 256:
+		dma_burst = C1_DMAB256;
+		break;
+	default:
+		dma_burst = C1_DMAB64;
+		break;
+	}
+	ccic_reg_write_mask(ccic_dev, REG_CTRL1, dma_burst, C1_DMAB_MASK);
+	ccic_reg_set_bit(ccic_dev, REG_CTRL1, C1_DMAB_LENSEL);
+	ccic_reg_set_bit(ccic_dev, REG_CTRL1, BIT(31));
+
+	/* ccic_reg_set_bit(ccic_dev, REG_CTRL2, C2_LGCY_LNNUM); */
+	/* ccic_reg_set_bit(ccic_dev, REG_CTRL2, C2_LGCY_HBLANK); */
+	return 0;
+}
+
+int ccic_csi2idi_src_sel(struct ccic_dev *ccic_dev, int sel)
+{
+	switch (sel) {
+	case CCIC_IDI_SEL_NONE:
+		/* ccic_reg_clear_bit(ccic_dev, REG_IDI_CTRL, IDI_RELEASE_RESET); */
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_CTRL2, CSI2_C2_REPACK_RST);
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_CTRL2,
+				   CSI2_C2_REPACK_ENA);
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_CTRL2, CSI2_C2_DPCM_ENA);
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_VCCTRL,
+				   CSI2_VCCTRL_MD_VC);
+		ccic_reg_write_mask(ccic_dev, REG_CSI2_CTRL2,
+				    CSI2_C2_MUX_SEL_LOCAL_MAIN,
+				    CSI2_C2_MUX_SEL_MASK);
+		break;
+	case CCIC_IDI_SEL_REPACK:
+		ccic_reg_write_mask(ccic_dev, REG_IDI_CTRL, IDI_SEL_DPCM_REPACK,
+				    IDI_SEL_MASK);
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_CTRL2,
+				   CSI2_C2_IDI_MUX_SEL_DPCM);
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_CTRL2,
+				   CSI2_C2_REPACK_RST);
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_CTRL2, CSI2_C2_REPACK_ENA);
+		break;
+	case CCIC_IDI_SEL_DPCM:
+		ccic_reg_write_mask(ccic_dev, REG_IDI_CTRL, IDI_SEL_DPCM_REPACK,
+				    IDI_SEL_MASK);
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_CTRL2,
+				 CSI2_C2_IDI_MUX_SEL_DPCM);
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_CTRL2, CSI2_C2_DPCM_ENA);
+		break;
+	case CCIC_IDI_SEL_PARALLEL:
+		ccic_reg_write_mask(ccic_dev, REG_IDI_CTRL, IDI_SEL_PARALLEL,
+				    IDI_SEL_MASK);
+		break;
+	default:
+		pr_err("%s: IDI source is error %d\n", __func__, sel);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void ccic_csi2idi_reset(struct ccic_dev *ccic_dev, int reset)
+{
+	if (reset) {
+		ccic_reg_clear_bit(ccic_dev, REG_IDI_CTRL, IDI_RELEASE_RESET);
+		/* assert reset to Repack module */
+		ccic_reg_set_bit(ccic_dev, REG_CSI2_CTRL2, CSI2_C2_REPACK_RST);
+	} else {
+		ccic_reg_set_bit(ccic_dev, REG_IDI_CTRL, IDI_RELEASE_RESET);
+		/* Deassert reset to Repack module */
+		ccic_reg_clear_bit(ccic_dev, REG_CSI2_CTRL2,
+				   CSI2_C2_REPACK_RST);
+	}
+}
+
+/* dump register in order to debug */
+void ccic_hw_dump_regs(struct ccic_dev *ccic_dev)
+{
+	unsigned int ret;
+
+	pr_info("CCIC%d regs dump:\n", ccic_dev->index);
+	/*
+	 * CCIC IRQ REG
+	 */
+	ret = ccic_reg_read(ccic_dev, REG_IRQSTAT);
+	pr_info("CCIC: REG_IRQSTAT[0x%02x] is 0x%08x\n", REG_IRQSTAT, ret);
+	ret = ccic_reg_read(ccic_dev, REG_IRQSTATRAW);
+	pr_info("CCIC: REG_IRQSTATRAW[0x%02x] is 0x%08x\n", REG_IRQSTATRAW,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_IRQMASK);
+	pr_info("CCIC: REG_IRQMASK[0x%02x] is 0x%08x\n\n", REG_IRQMASK, ret);
+
+	/*
+	 * CCIC IMG REG
+	 */
+	ret = ccic_reg_read(ccic_dev, REG_IMGPITCH);
+	pr_info("CCIC: REG_IMGPITCH[0x%02x] is 0x%08x\n", REG_IMGPITCH, ret);
+	ret = ccic_reg_read(ccic_dev, REG_IMGSIZE);
+	pr_info("CCIC: REG_IMGSIZE[0x%02x] is 0x%08x\n", REG_IMGSIZE, ret);
+	ret = ccic_reg_read(ccic_dev, REG_IMGOFFSET);
+	pr_info("CCIC: REG_IMGOFFSET[0x%02x] is 0x%08x\n\n", REG_IMGOFFSET,
+		ret);
+
+	/*
+	 * CCIC CTRL REG
+	 */
+	ret = ccic_reg_read(ccic_dev, REG_CTRL0);
+	pr_info("CCIC: REG_CTRL0[0x%02x] is 0x%08x\n", REG_CTRL0, ret);
+	ret = ccic_reg_read(ccic_dev, REG_CTRL1);
+	pr_info("CCIC: REG_CTRL1[0x%02x] is 0x%08x\n", REG_CTRL1, ret);
+	ret = ccic_reg_read(ccic_dev, REG_CTRL2);
+	pr_info("CCIC: REG_CTRL2[0x%02x] is 0x%08x\n", REG_CTRL2, ret);
+	ret = ccic_reg_read(ccic_dev, REG_CTRL3);
+	pr_info("CCIC: REG_CTRL3[0x%02x] is 0x%08x\n", REG_CTRL3, ret);
+	ret = ccic_reg_read(ccic_dev, REG_IDI_CTRL);
+	pr_info("CCIC: REG_IDI_CTRL[0x%02x] is 0x%08x\n\n", REG_IDI_CTRL, ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_VCCTRL);
+	pr_info("CCIC: REG_CSI2_VCCTRL[0x%02x] is 0x%08x\n\n", REG_CSI2_VCCTRL,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_LNNUM);
+	pr_info("CCIC: REG_LNNUM[0x%02x] is 0x%08x\n", REG_LNNUM, ret);
+	ret = ccic_reg_read(ccic_dev, REG_FRAME_CNT);
+	pr_info("CCIC: REG_FRAME_CNT[0x%02x] is 0x%08x\n", REG_FRAME_CNT, ret);
+
+	/*
+	 * CCIC CSI2 REG
+	 */
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_DPHY1);
+	pr_info("CCIC: REG_CSI2_DPHY1[0x%02x] is 0x%08x\n", REG_CSI2_DPHY1,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_DPHY2);
+	pr_info("CCIC: REG_CSI2_DPHY2[0x%02x] is 0x%08x\n", REG_CSI2_DPHY2,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_DPHY3);
+	pr_info("CCIC: REG_CSI2_DPHY3[0x%02x] is 0x%08x\n", REG_CSI2_DPHY3,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_DPHY4);
+	pr_info("CCIC: REG_CSI2_DPHY4[0x%02x] is 0x%08x\n", REG_CSI2_DPHY4,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_DPHY5);
+	pr_info("CCIC: REG_CSI2_DPHY5[0x%02x] is 0x%08x\n", REG_CSI2_DPHY5,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_DPHY6);
+	pr_info("CCIC: REG_CSI2_DPHY6[0x%02x] is 0x%08x\n", REG_CSI2_DPHY6,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_CTRL0);
+	pr_info("CCIC: REG_CSI2_CTRL0[0x%02x] is 0x%08x\n", REG_CSI2_CTRL0,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_CTRL2);
+	pr_info("CCIC: REG_CSI2_CTRL2[0x%02x] is 0x%08x\n", REG_CSI2_CTRL2,
+		ret);
+	ret = ccic_reg_read(ccic_dev, REG_CSI2_CTRL3);
+	pr_info("CCIC: REG_CSI2_CTRL3[0x%02x] is 0x%08x\n", REG_CSI2_CTRL3,
+		ret);
+
+	/*
+	 * CCIC YUV REG
+	 */
+	ret = ccic_reg_read(ccic_dev, REG_Y0BAR);
+	pr_info("CCIC: REG_Y0BAR[0x%02x] 0x%08x\n", REG_Y0BAR, ret);
+	ret = ccic_reg_read(ccic_dev, REG_U0BAR);
+	pr_info("CCIC: REG_U0BAR[0x%02x] 0x%08x\n", REG_U0BAR, ret);
+	ret = ccic_reg_read(ccic_dev, REG_V0BAR);
+	pr_info("CCIC: REG_V0BAR[0x%02x] 0x%08x\n\n", REG_V0BAR, ret);
+
+#if 0
+	/*
+	 * CCIC APMU REG
+	 */
+	ret = __raw_readl(get_apmu_base_va() + REG_CLK_CCIC_RES);
+	pr_info("CCIC: APMU_CCIC_RES[0x%02x] is 0x%08x\n", REG_CLK_CCIC_RES, ret);
+	ret = __raw_readl(get_apmu_base_va() + REG_CLK_CCIC2_RES);
+	pr_info("CCIC: APMU_CCIC2_RES[0x%02x] is 0x%08x\n", REG_CLK_CCIC2_RES, ret);
+#endif
+}
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_hwreg.h b/drivers/media/platform/spacemit/cam_ccic/ccic_hwreg.h
new file mode 100644
index 000000000000..ecff6c60e921
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_hwreg.h
@@ -0,0 +1,278 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ccic_hwreg.h - hw register for ccic
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef _CCIC_HWREG_H_
+#define _CCIC_HWREG_H_
+
+#define REG_Y0BAR 0x00
+#define REG_U0BAR 0x0c
+#define REG_V0BAR 0x18
+
+#define REG_IMGPITCH 0x24 /* Image pitch register */
+#define IMGP_YP_SHFT 0 /* Y pitch params */
+#define IMGP_YP_MASK 0x00003fff /* Y pitch field */
+#define IMGP_UVP_SHFT 16 /* UV pitch (planar) */
+#define IMGP_UVP_MASK 0x3fff0000
+
+#define REG_IRQSTATRAW 0x28 /* RAW IRQ Status */
+#define REG_IRQMASK 0x2c /* IRQ mask - same bits as IRQSTAT */
+#define REG_IRQSTAT 0x30 /* IRQ status / clear */
+#define IRQ_DMA_EOF (BIT(0)) /* DMA EOF IRQ */
+#define IRQ_DMA_SOF (BIT(1)) /* DMA SOF IRQ */
+#define IRQ_CSI_EOF (BIT(2)) /* CSI EOF IRQ */
+#define IRQ_CSI_SOF (BIT(3)) /* CSI SOF IRQ */
+#define IRQ_DMA_NOT_DONE (BIT(4)) /* DMA not done at frame start IRQ */
+#define IRQ_SHADOW_NOT_RDY \
+	(BIT(5)) /* Shadow bit not ready at frame start IRQ */
+#define IRQ_DMA_OVERFLOW (BIT(6)) /* FIFO full IRQ*/
+#define IRQ_DMA_PRO_LINE (BIT(7)) /* CCIC Programmable Line IRQ */
+#define IRQ_IDI_PRO_LINE (BIT(8)) /* IDI Programmable Line IRQ */
+#define IRQ_CSI2IDI_FLUSH (BIT(9)) /* CSI2IDI DATA FLUSH IRQ */
+#define IRQ_CSI2IDI_HBLK2HSYNC \
+	(BIT(10)) /* HBLK_TO_HSYNC IRQ, CSI2IDI module detects Hblank to Hsync gap too small */
+#define IRQ_DMA_WR_ERR (BIT(11)) /* AXI Write Error IRQ */
+#define IRQ_DPHY_RX_CLKULPS_ACTIVE (BIT(12)) /* DPHY Rx CLKULPS Active IRQ */
+#define IRQ_DPHY_RX_CLKULPS (BIT(13)) /* DPHY Rx CLKULPS IRQ */
+#define IRQ_DPHY_LN_ULPS_ACTIVE (BIT(14)) /* DPHY Lane ULPS Active IRQ */
+#define IRQ_DPHY_LN_ERR_CTL (BIT(15)) /* DPHY Lane Error Control IRQ */
+#define IRQ_DPHY_LN_TX_SYNC_ERR \
+	(BIT(16)) /* DPHY Lane Start of Transmission Sync Error IRQ */
+#define IRQ_DPHY_LN_TX_ERR \
+	(BIT(17)) /* DPHY Lane Start of Transmission Error IRQ */
+#define IRQ_DPHY_LN_RX_ERR (BIT(18)) /* DPHY receiver Line Error IRQ */
+#define IRQ_DPCM_REPACK_ERR (BIT(19)) /* DPCM/Repack IRQ */
+
+#define IRQ_CSI2PACKET_ERR (BIT(23))
+#define IRQ_CSI2CRC_ERR (BIT(24))
+#define IRQ_CSI2ECC2BIT_ERR (BIT(25))
+#define IRQ_CSI2PATIRY_ERR (BIT(26))
+#define IRQ_CSI2ECCCORRECTABLE_ERR (BIT(27))
+#define IRQ_CSI2LANEFIFOOVERRUN_ERR (BIT(28))
+#define IRQ_CSI2PARSE_ERR (BIT(29))
+#define IRQ_CSI2GENSHORTPACKVALID (BIT(30))
+#define IRQ_CSI2GENSHORTPACK_ERR (BIT(31))
+#define FRAMEIRQS (IRQ_CSI_SOF)
+#define CSI2PHYERRS (0xff0b0840)
+#define ALLIRQS (FRAMEIRQS | CSI2PHYERRS | IRQ_CSI2IDI_HBLK2HSYNC)
+
+#define REG_IMGSIZE 0x34 /* Image size */
+#define IMGSZ_V_MASK 0x1fff0000
+#define IMGSZ_V_SHIFT 16
+#define IMGSZ_H_MASK 0x00003fff
+#define IMGSZ_H_SHIFT 0
+
+#define REG_IMGOFFSET 0x38 /* IMage offset */
+
+#define REG_CTRL0 0x3c /* Control 0 */
+#define C0_ENABLE 0x00000001 /* Makes the whole thing go */
+/* Mask for all the format bits */
+#define C0_DF_MASK 0x08dffffc
+/* RGB ordering */
+#define C0_RGB4_RGBX 0x00000000
+#define C0_RGB4_XRGB 0x00000004
+#define C0_RGB4_BGRX 0x00000008
+#define C0_RGB4_XBGR 0x0000000c
+#define C0_RGB5_RGGB 0x00000000
+#define C0_RGB5_GRBG 0x00000004
+#define C0_RGB5_GBRG 0x00000008
+#define C0_RGB5_BGGR 0x0000000c
+/* YUV4222 to 420 Semi-Planar Enable */
+#define C0_YUV420SP 0x00000010
+/* Spec has two fields for DIN and DOUT, but they must match, so
+   combine them here. */
+#define C0_DF_YUV 0x00000000 /* Data is YUV */
+#define C0_DF_RGB 0x000000a0 /* Data is RGB */
+#define C0_DF_BAYER 0x00000140 /* Data is Bayer */
+/* 8-8-8 must be missing from the below - ask */
+#define C0_RGBF_565 0x00000000
+#define C0_RGBF_444 0x00000800
+#define C0_RGB_BGR 0x00001000 /* Blue comes first */
+#define C0_YUV_PLANAR 0x00000000 /* YUV 422 planar format */
+#define C0_YUV_PACKED 0x00008000 /* YUV 422 packed format */
+#define C0_YUV_420PL 0x0000a000 /* YUV 420 planar format */
+/* Think that 420 packed must be 111 - ask */
+#define C0_YUVE_YUYV 0x00000000 /* Y1CbY0Cr */
+#define C0_YUVE_YVYU 0x00010000 /* Y1CrY0Cb */
+#define C0_YUVE_VYUY 0x00020000 /* CrY1CbY0 */
+#define C0_YUVE_UYVY 0x00030000 /* CbY1CrY0 */
+#define C0_YUVE_XYUV 0x00000000 /* 420: .YUV */
+#define C0_YUVE_XYVU 0x00010000 /* 420: .YVU */
+#define C0_YUVE_XUVY 0x00020000 /* 420: .UVY */
+#define C0_YUVE_XVUY 0x00030000 /* 420: .VUY */
+/* Bayer bits 18,19 if needed */
+#define C0_HPOL_LOW 0x01000000 /* HSYNC polarity active low */
+#define C0_VPOL_LOW 0x02000000 /* VSYNC polarity active low */
+#define C0_VCLK_LOW 0x04000000 /* VCLK on falling edge */
+#define C0_420SP_UVSWAP 0x08000000 /* YUV420SP U/V Swap */
+#define C0_SIFM_MASK 0xc0000000 /* SIF mode bits */
+#define C0_SIF_HVSYNC 0x00000000 /* Use H/VSYNC */
+#define C0_SOF_NOSYNC 0x40000000 /* Use inband active signaling */
+#define C0_EOF_VSYNC 0x00400000 /* Generate EOF by VSYNC */
+#define C0_VEDGE_CTRL 0x00800000 /* Detecting falling edge of VSYNC */
+/* bit 21: fifo overrun auto-recovery */
+#define C0_EOFFLUSH 0x00200000
+/* bit 27 YUV420SP_UV_SWAP */
+#define C0_YUV420SP_UV_SWAP 0x08000000
+
+#define REG_CTRL1 0x40 /* Control 1 */
+#define C1_SENCLKGATE 0x00000001 /* Sensor Clock Gate */
+#define C1_RESVZ 0x0001fffe
+#define C1_DMAB_LENSEL 0x00020000 /* set 1, coupled CCICx */
+#define C1_444ALPHA 0x00f00000 /* Alpha field in RGB444 */
+#define C1_ALPHA_SHFT 20
+#define C1_AWCACHE 0x00100000 /* set 1. coupled CCICx */
+#define C1_DMAB64 0x00000000 /* 64-byte DMA burst */
+#define C1_DMAB128 0x02000000 /* 128-byte DMA burst */
+#define C1_DMAB256 0x04000000 /* 256-byte DMA burst */
+#define C1_DMAB_MASK 0x06000000
+#define C1_SHADOW_RDY 0x08000000 /* set it 1 when BAR is set*/
+#define C1_PWRDWN 0x10000000 /* Power down */
+#define C1_DMAPOSTED 0x40000000 /* DMA Posted Select */
+
+#define REG_CTRL2 0x44 /* Control 2 */
+/* recommend set 1 to disable legacy calc DMA line num */
+#define C2_LGCY_LNNUM 0x80000000
+/* recommend set 1 to disaable legacy CSI2 hblank */
+#define C2_LGCY_HBLANK 0x40000000
+
+#define REG_CTRL3 0x48 /* Control 2 */
+
+#define REG_LNNUM 0x60 /* Lines num DMA filled */
+
+#define CLK_DIV_MASK 0x0000ffff /* Upper bits RW "reserved" */
+#define REG_FRAME_CNT 0x23C
+
+/* MIPI */
+#define REG_CSI2_CTRL0 0x100
+#define CSI2_C0_ENABLE 0x01
+#define CSI2_C0_LANE_NUM(n) (((n) - 1) << 1)
+#define CSI2_C0_LANE_NUM_MASK 0x06
+#define CSI2_C0_EXT_TIM_ENA (0x1 << 3)
+#define CSI2_C0_VLEN (0x4 << 4)
+#define CSI2_C0_VLEN_MASK (0xf << 4)
+#define CSI2_C0_VCDC_SEL (0x1 << 13)
+#define REG_CSI2_VCCTRL 0x114
+#define CSI2_VCCTRL_MD_MASK (0x3 << 0)
+#define CSI2_VCCTRL_MD_NORMAL (0x0 << 0)
+#define CSI2_VCCTRL_MD_VC (0x1 << 0)
+#define CSI2_VCCTRL_MD_DT (0x2 << 0)
+#define CSI2_VCCTRL_VC0_MASK (0x3 << 14)
+#define CSI2_VCCTRL_DT1_MASK (0x3 << 16)
+#define CSI2_VCCTRL_VC1_MASK (0x3 << 22)
+#define CSI2_VCCTRL_DT_PASS_MASK (0x1 << 24)
+#define REG_CSI2_VCCTRL2 0x118
+#define CSI2_VCCTRL_DT2_MASK (0x3f << 0)
+#define CSI2_VCCTRL_VC2_MASK (0x3 << 6)
+#define CSI2_VCCTRL_DT3_MASK (0x3f << 8)
+#define CSI2_VCCTRL_VC3_MASK (0x3 << 14)
+#define REG_CSI2_DT_FLT 0x11c
+#define CSI2_DT_FLT0_EN_MASK (0x1 << 6)
+#define CSI2_DT_FLT0_PATTERN_MASK (0x3f << 0)
+#define CSI2_DT_FLT1_EN_MASK (0x1 << 14)
+#define CSI2_DT_FLT1_PATTERN_MASK (0x3f << 8)
+#define CSI2_DT_FLT2_EN_MASK (0x1 << 22)
+#define CSI2_DT_FLT2_PATTERN_MASK (0x3f << 16)
+#define CSI2_DT_FLT3_EN_MASK (0x1 << 30)
+#define CSI2_DT_FLT3_PATTERN_MASK (0x3f << 24)
+#define REG_CSI2_DT_FLT2 0x120
+#define CSI2_DT_FLT4_EN_MASK (0x1 << 6)
+#define CSI2_DT_FLT4_PATTERN_MASK (0x3f << 0)
+#define CSI2_DT_FLT5_EN_MASK (0x1 << 14)
+#define CSI2_DT_FLT5_PATTERN_MASK (0x3f << 8)
+#define CSI2_DT_FLT6_EN_MASK (0x1 << 22)
+#define CSI2_DT_FLT6_PATTERN_MASK (0x3f << 16)
+#define CSI2_DT_FLT7_EN_MASK (0x1 << 30)
+#define CSI2_DT_FLT7_PATTERN_MASK (0x3f << 24)
+#define REG_CSI2_DPHY1 0x124
+#define CSI2_DHPY1_ANA_PU (0x1 << 0)
+#define CSI2_DHPY1_BIF_EN (0x1 << 1)
+#define REG_CSI2_DPHY2 0x128
+#define CSI2_DPHY2_SEL_IREF(n) ((n & 0x03) << 30)
+#define CSI2_DPHY2_VTH_LPRX_H(n) ((n & 0x07) << 27)
+#define CSI2_DPHY2_VTH_LPRX_L(n) ((n & 0x07) << 24)
+#define CSI2_DPHY2_CK_ENA 0x00800000
+#define CSI2_DPHY2_CK_DELAY(n) ((n & 0x07) << 20)
+#define CSI2_DPHY2_LPRX_CTL 0x00080000
+#define CSI2_DPHY2_HSRX_TERM(n) ((n & 0x07) << 16)
+#define CSI2_DPHY2_CH2_ENA 0x00008000
+#define CSI2_DPHY2_CH2_DELAY(n) ((n & 0x07) << 12)
+#define CSI2_DPHY2_CH3_ENA 0x00000800
+#define CSI2_DPHY2_CH3_DELAY(n) ((n & 0x07) << 8)
+#define CSI2_DPHY2_CH0_ENA 0x00000080
+#define CSI2_DPHY2_CH0_DELAY(n) ((n & 0x07) << 4)
+#define CSI2_DPHY2_CH1_ENA 0x00000008
+#define CSI2_DPHY2_CH1_DELAY(n) ((n & 0x07) << 0)
+#define REG_CSI2_DPHY3 0x12c
+#define CSI2_DPHY3_HS_SETTLE_SHIFT 8
+#define REG_CSI2_DPHY4 0x130
+#define CSI2_DHPY4_BIF_EN (0x1 << 23)
+#define CSI2_DPHY4_CHK_ENA 0x00000040
+#define CSI2_DPHY4_ERR_OUT_SEL 0x00000020
+#define CSI2_DPHY4_ERR_CLEAR 0x00000010
+#define CSI2_DPHY4_LANE_SEL(n) ((n & 0x03) << 2)
+#define CSI2_DPHY4_PATTERN_SEL(n) ((n & 0x03) << 0)
+#define REG_CSI2_DPHY5 0x134
+#define CSI2_DPHY5_LANE_RESC_ENA_SHIFT 4
+#define CSI2_DPHY5_LANE_ENA(n) ((1 << (n)) - 1)
+#define REG_CSI2_DPHY6 0x138
+#define CSI2_DPHY6_CK_SETTLE_SHIFT 8
+#define REG_CSI2_CTRL2 0x140
+#define CSI2_C2_MUX_SEL_MASK 0x06000000
+#define CSI2_C2_MUX_SEL_LOCAL_MAIN 0x00000000
+#define CSI2_C2_MUX_SEL_IPE2_VCDT 0x02000000
+#define CSI2_C2_MUX_SEL_IPE2_MAIN 0x04000000
+#define CSI2_C2_MUX_SEL_REMOTE_VCDT 0x06000000
+#define CSI2_C2_IDI_MUX_SEL_DPCM 0x01000000
+#define CSI2_C2_REPACK_RST 0x00200000
+#define CSI2_C2_REPACK_ENA 0x00010000
+#define CSI2_C2_DPCM_ENA 0x00000001
+#define REG_CSI2_CTRL3 0x144
+#define REG_CSI2_DPHY9 0x32c
+#define CSI2_DHPY9_EN_BIF2 (0x1 << 23)
+
+/* IDI */
+#define REG_IDI_CTRL 0x310
+#define IDI_SEL_MASK 0x06
+#define IDI_SEL_DPCM_REPACK 0x00
+#define IDI_SEL_PARALLEL 0x02
+#define IDI_SEL_AHB 0x04
+#define IDI_RELEASE_RESET 0x01
+
+#define REG_IDI_TRIG_LINE_NUM 0x330
+#define REG_CSI_LANE_STATE_DBG 0x334
+
+/* APMU */
+#define REG_CLK_CCIC_RES 0x50
+#define REG_CLK_CCIC2_RES 0x24
+
+#define CF_SINGLE_BUF 0
+#define CF_FRAME_SOF0 1
+#define CF_FRAME_OVERFLOW 2
+
+int ccic_csi2_config_dphy(struct ccic_dev *ccic_dev, int lanes, int enable);
+int ccic_csi2_lanes_enable(struct ccic_dev *ccic_dev, int lanes);
+int ccic_csi2_vc_ctrl(struct ccic_dev *ccic_dev, int md, unsigned int dt_en);
+int ccic_dma_set_burst(struct ccic_dev *ccic_dev);
+//void ccic_dma_enable(struct ccic_dev *ccic_dev, int en);
+int ccic_csi2idi_src_sel(struct ccic_dev *ccic_dev, int sel);
+void ccic_csi2idi_reset(struct ccic_dev *ccic_dev, int reset);
+void ccic_hw_dump_regs(struct ccic_dev *ccic_dev);
+void ccic_en_dt_match_pass_mode(struct ccic_dev *ccic_dev, int en);
+void ccic_set_path_vc(struct ccic_dev *ccic_dev, int path_id, u32 vc);
+void ccic_set_path0_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data);
+void ccic_set_path1_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data);
+void ccic_set_path2_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data);
+void ccic_set_path3_dt_filter(struct ccic_dev *ccic_dev, int flt0_en,
+			      uint32_t flt0_data, int flt1_en,
+			      uint32_t flt1_data);
+#endif
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_iommu.c b/drivers/media/platform/spacemit/cam_ccic/ccic_iommu.c
new file mode 100644
index 000000000000..4b8a08338bca
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_iommu.c
@@ -0,0 +1,398 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ccic_iommu.c - Driver for CCIC IOMMU
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <asm/io.h>
+#include <linux/printk.h>
+#include "ccic_reg_iommu.h"
+#include "ccic_iommu.h"
+
+#define DEBUG 0
+
+#define read32(a) readl((volatile void __iomem *)(a))
+#define write32(a, v) writel((v), (volatile void __iomem *)(a))
+
+static void ccic_iommu_set_sva(struct ccic_iommu_device *mmu_dev);
+
+static inline uint32_t iommu_reg_read(struct ccic_iommu_device *mmu_dev,
+				      uint32_t reg)
+{
+	return read32(mmu_dev->regs_base + reg);
+}
+
+static inline void iommu_reg_write(struct ccic_iommu_device *mmu_dev,
+				   uint32_t reg, uint32_t val)
+{
+	write32(mmu_dev->regs_base + reg, val);
+}
+
+static inline void iommu_reg_write_mask(struct ccic_iommu_device *mmu_dev,
+					uint32_t reg, uint32_t val,
+					uint32_t mask)
+{
+	uint32_t v;
+
+	v = iommu_reg_read(mmu_dev, reg);
+	v = (v & ~mask) | (val & mask);
+	iommu_reg_write(mmu_dev, reg, v);
+}
+
+static inline void iommu_reg_set_bit(struct ccic_iommu_device *mmu_dev,
+				     uint32_t reg, uint32_t val)
+{
+	iommu_reg_write_mask(mmu_dev, reg, val, val);
+}
+
+static inline void iommu_reg_clr_bit(struct ccic_iommu_device *mmu_dev,
+				     uint32_t reg, uint32_t val)
+{
+	iommu_reg_write_mask(mmu_dev, reg, 0, val);
+}
+
+static void iommu_enable_tbu(struct ccic_iommu_device *mmu_dev, int tbu)
+{
+	iommu_reg_set_bit(mmu_dev, REG_IOMMU_TCR0(tbu), 0x1);
+}
+
+static void iommu_disable_tbu(struct ccic_iommu_device *mmu_dev, int tbu)
+{
+	iommu_reg_clr_bit(mmu_dev, REG_IOMMU_TCR0(tbu), 0x1);
+}
+
+static void iommu_set_tbu_ttaddr(struct ccic_iommu_device *mmu_dev, int tbu,
+				 uint64_t addr)
+{
+	iommu_reg_write(mmu_dev, REG_IOMMU_TTBL(tbu), addr & 0xffffffff);
+	iommu_reg_write(mmu_dev, REG_IOMMU_TTBH(tbu), (addr >> 32) & 0x1);
+}
+
+static void iommu_set_tbu_ttsize(struct ccic_iommu_device *mmu_dev, int tbu,
+				 int size)
+{
+	/* version 2 */
+#if 0
+	iommu_reg_write_mask(mmu_dev, REG_IOMMU_TCR0(tbu),
+			     ((size - 1) & 0x1fff) << 16, 0x1fff << 16);
+#else
+	iommu_reg_write_mask(mmu_dev, REG_IOMMU_TCR0(tbu),
+			     ((size - 1) & 0xffff) << 16, 0xffff << 16);
+#endif
+}
+
+static void __maybe_unused iommu_set_tbu_qos(struct ccic_iommu_device *mmu_dev, int tbu,
+			      int qos)
+{
+	iommu_reg_write_mask(mmu_dev, REG_IOMMU_TCR0(tbu), (qos & 0xf) << 4,
+			     0xf << 4);
+}
+
+static void iommu_enable_irqs(struct ccic_iommu_device *mmu_dev)
+{
+	iommu_reg_write_mask(mmu_dev, REG_IOMMU_GIRQ_ENA, 0xffffffff, 0xffffffff);
+}
+
+static inline uint32_t iommu_bva_low(struct ccic_iommu_device *mmu_dev)
+{
+	return iommu_reg_read(mmu_dev, REG_IOMMU_BVAL);
+}
+
+static inline uint32_t iommu_bva_high(struct ccic_iommu_device *mmu_dev)
+{
+	return iommu_reg_read(mmu_dev, REG_IOMMU_BVAH);
+}
+
+static int tid_to_tbu(struct ccic_iommu_device *mmu_dev, uint32_t tid)
+{
+	int i;
+
+	for (i = 0; i < CCIC_IOMMU_CH_NUM; ++i) {
+		if (mmu_dev->ch_matrix[i] == tid)
+			return i;
+	}
+
+	return -1;
+}
+
+static int ccic_iommu_acquire_channel(struct ccic_iommu_device *mmu_dev,
+				     uint32_t tid)
+{
+	int tbu;
+	unsigned long flags;
+
+	tbu = tid_to_tbu(mmu_dev, tid);
+	if (tbu < 0) {
+		pr_debug("no such channel %x to acquire\n", tid);
+		return -ENODEV;
+	}
+
+	spin_lock_irqsave(&mmu_dev->ops_lock, flags);
+	if (test_bit(tbu, &mmu_dev->ch_map)) {
+		spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+		pr_err("channel %x not free\n", tid);
+		return -EBUSY;
+	}
+	set_bit(tbu, &mmu_dev->ch_map);
+	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+
+	return 0;
+}
+
+static int ccic_iommu_release_channel(struct ccic_iommu_device *mmu_dev,
+				     uint32_t tid)
+{
+	int tbu;
+	unsigned long flags;
+
+	tbu = tid_to_tbu(mmu_dev, tid);
+	if (tbu < 0) {
+		pr_err("no such channel %x to release\n", tid);
+		return -ENODEV;
+	}
+
+	spin_lock_irqsave(&mmu_dev->ops_lock, flags);
+	clear_bit(tbu, &mmu_dev->ch_map);
+	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+
+	return 0;
+}
+
+static int ccic_iommu_enable_channel(struct ccic_iommu_device *mmu_dev,
+				    uint32_t tid)
+{
+	int tbu;
+	unsigned long flags;
+
+	ccic_iommu_set_sva(mmu_dev);
+	tbu = tid_to_tbu(mmu_dev, tid);
+	if (tbu < 0) {
+		pr_err("no such channel %x to enable\n", tid);
+		return -ENODEV;
+	}
+
+	spin_lock_irqsave(&mmu_dev->ops_lock, flags);
+	/* if (!test_bit(tbu, &mmu_dev->ch_map)) {
+	 *	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+	 *	return -EPERM;
+	 * }
+	 */
+
+	iommu_enable_tbu(mmu_dev, tbu);
+	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+
+	return 0;
+}
+
+static int ccic_iommu_disable_channel(struct ccic_iommu_device *mmu_dev,
+				     uint32_t tid)
+{
+	int tbu;
+	unsigned long flags;
+
+	tbu = tid_to_tbu(mmu_dev, tid);
+	if (tbu < 0) {
+		pr_err("no such channel %x to disable\n", tid);
+		return -ENODEV;
+	}
+
+	spin_lock_irqsave(&mmu_dev->ops_lock, flags);
+	/* if (!test_bit(tbu, &mmu_dev->ch_map)) {
+	 *	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+	 *	return -EPERM;
+	 * }
+	 */
+
+	iommu_disable_tbu(mmu_dev, tbu);
+	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+
+	return 0;
+}
+
+static int ccic_iommu_config_channel(struct ccic_iommu_device *mmu_dev,
+				    uint32_t tid, uint64_t ttAddr,
+				    uint32_t ttSize)
+{
+	int tbu;
+	unsigned long flags;
+
+	tbu = tid_to_tbu(mmu_dev, tid);
+	if (tbu < 0) {
+		pr_err("no such channel %x to configure\n", tid);
+		return -ENODEV;
+	}
+
+	spin_lock_irqsave(&mmu_dev->ops_lock, flags);
+	/* if (!test_bit(tbu, &mmu_dev->ch_map)) {
+	 *	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+	 *	return -EPERM;
+	 * }
+	 */
+
+	/* iommu_set_tbu_qos(mmu_dev, tbu, 4); */
+	iommu_set_tbu_ttaddr(mmu_dev, tbu, ttAddr);
+	iommu_set_tbu_ttsize(mmu_dev, tbu, ttSize);
+	/* iommu_update_trans_table(mmu_dev, tbu); */
+	iommu_enable_irqs(mmu_dev);
+	spin_unlock_irqrestore(&mmu_dev->ops_lock, flags);
+
+	return 0;
+}
+
+static const uint64_t IOMMU_VADDR_BASE = 0x00000000;
+static uint64_t ccic_iommu_get_sva(struct ccic_iommu_device *mmu_dev,
+				  uint32_t tid, uint32_t offset)
+{
+	int tbu;
+	uint64_t svAddr = 0, bva = 0;
+
+	tbu = tid_to_tbu(mmu_dev, tid);
+	if (tbu < 0) {
+		pr_err("no such channel %x to get sva\n", tid);
+		return -ENODEV;
+	}
+	bva = iommu_bva_high(mmu_dev);
+	bva <<= 32;
+	bva += iommu_bva_low(mmu_dev);
+	/* version 2 */
+#if 0
+	svAddr = bva + 0x2000000 * (uint64_t)tbu + (offset & 0xfff);
+#else
+	svAddr = bva + 0x10000000 * (uint64_t)tbu + (offset & 0xfff);
+#endif
+	return svAddr;
+}
+
+static void ccic_iommu_set_sva(struct ccic_iommu_device *mmu_dev)
+{
+	iommu_reg_write(mmu_dev, REG_IOMMU_BVAL, IOMMU_VADDR_BASE);
+	iommu_reg_write(mmu_dev, REG_IOMMU_BVAH, 0x0);
+}
+
+static unsigned int ccic_iommu_irq_status(struct ccic_iommu_device *mmu_dev)
+{
+	unsigned int status = 0;
+	status = iommu_reg_read(mmu_dev, REG_IOMMU_GIRQ_STAT);
+	if (status) {
+		iommu_reg_write(mmu_dev, REG_IOMMU_GIRQ_STAT, status);
+	}
+	return status;
+}
+
+static int ccic_iommu_dump_regs(struct ccic_iommu_device *mmu_dev,
+				uint32_t ch_id)
+{
+#if 0
+	int ret = 0;
+	unsigned int status = 0, tlb_size = 0;
+	uint64_t addr1 = 0, addr2 = 0, addr3 = 0;
+
+	pr_info("**************start dump ccic iommu ch%d regs:\n", ch_id);
+	addr1 = iommu_reg_read(mmu_dev, REG_IOMMU_LVAL);
+	status = iommu_reg_read(mmu_dev, REG_IOMMU_LVAH);
+	if (status)
+		addr1 = addr1 | (1ULL << 32);
+
+	addr2 = iommu_reg_read(mmu_dev, REG_IOMMU_LPAL);
+	status = iommu_reg_read(mmu_dev, REG_IOMMU_LPAH);
+	if (status)
+		addr2 |= (1ULL << 32);
+
+	addr3 = iommu_reg_read(mmu_dev, REG_IOMMU_TVAL);
+	status = iommu_reg_read(mmu_dev, REG_IOMMU_TVAH);
+	if (status)
+		addr3 = addr3 | (1ULL << 32);
+	pr_info("ccic mmu: last virtual addr=0x%llx,last phy addr=0x%llx, timeout addr=0x%llx\n", addr1, addr2, addr3);
+
+	addr1 = iommu_reg_read(mmu_dev, REG_IOMMU_TTBL(ch_id));
+	status = iommu_reg_read(mmu_dev, REG_IOMMU_TTBH(ch_id));
+	if (status)
+		addr1 |= (1ULL << 32);
+	status = iommu_reg_read(mmu_dev, REG_IOMMU_TCR0(ch_id));
+	tlb_size = (status & 0x1fff0000) >> 16;
+	pr_info("ccic mmu ch%d: tlb addr=0x%llx,tcr0=0x%x, tlb size=%d\n", ch_id, addr1, status, tlb_size);
+
+	return ret;
+#else
+	return 0;
+#endif
+}
+
+static void ccic_iommu_set_timeout_default_addr(struct ccic_iommu_device *mmu_dev, uint64_t timeout_default_addr)
+{
+	unsigned int high = 0, low = 0;
+
+	low = timeout_default_addr & 0xffffffffULL;
+	high = (timeout_default_addr >> 32) & 0xffffffffULL;
+	iommu_reg_write(mmu_dev, REG_IOMMU_TIMEOUT_ADDR_LOW, low);
+	iommu_reg_write(mmu_dev, REG_IOMMU_TIMEOUT_ADDR_HIGH, high);
+}
+
+static struct ccic_iommu_ops mmu_ops = {
+	.acquire_channel = ccic_iommu_acquire_channel,
+	.release_channel = ccic_iommu_release_channel,
+	.enable_channel = ccic_iommu_enable_channel,
+	.disable_channel = ccic_iommu_disable_channel,
+	.config_channel = ccic_iommu_config_channel,
+	.get_sva = ccic_iommu_get_sva,
+	.irq_status = ccic_iommu_irq_status,
+	.dump_channel_regs = ccic_iommu_dump_regs,
+	.set_timeout_default_addr = ccic_iommu_set_timeout_default_addr,
+};
+
+static const uint32_t iommu_ch_dmac_mapping[CCIC_IOMMU_CH_NUM] = {
+	MMU_TID(0), /* dma0 TBU0 */
+	MMU_TID(1), /* dma1 TBU1 */
+	MMU_TID(2), /* dma2 TBU2 */
+	MMU_TID(3), /* dma3 TBU3 */
+	MMU_TID(4), /* dma4 TBU4 */
+	MMU_TID(5), /* dma5 TBU5 */
+	MMU_TID(6), /* dma6 TBU6 */
+	MMU_TID(7), /* dma7 TBU7 */
+	MMU_TID(8), /* dma8 TBU8 */
+	MMU_TID(9), /* dma9 TBU9 */
+	MMU_TID(10), /* dma10 TBU10 */
+	MMU_TID(11), /* dma11 TBU11 */
+};
+
+struct ccic_iommu_device *ccic_iommu_create(struct device *dev,
+										unsigned long regs_base)
+{
+	struct ccic_iommu_device *mmu_dev = NULL;
+
+	mmu_dev = devm_kzalloc(dev,
+			       sizeof(struct ccic_iommu_device), GFP_KERNEL);
+	if (!mmu_dev)
+		return NULL;
+
+	mmu_dev->regs_base = regs_base;
+	mmu_dev->ops = &mmu_ops;
+	memcpy(mmu_dev->ch_matrix, iommu_ch_dmac_mapping,
+	       sizeof(iommu_ch_dmac_mapping));
+
+	spin_lock_init(&mmu_dev->ops_lock);
+	mmu_dev->dev = dev;
+
+	pr_debug("%s X\n", __func__);
+
+	return mmu_dev;
+}
+
+void ccic_iommu_unregister(struct ccic_iommu_device *mmu_dev)
+{
+	struct device *dev = mmu_dev->dev;
+	devm_kfree(dev, mmu_dev);
+
+	pr_debug("%s X\n", __func__);
+}
+
+#endif
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_iommu.h b/drivers/media/platform/spacemit/cam_ccic/ccic_iommu.h
new file mode 100644
index 000000000000..ee6f78a89c49
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_iommu.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ccic_iommu.h - Driver for CCIC IOMMU
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef _CCIC_IOMMU_H_
+#define _CCIC_IOMMU_H_
+
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include "ccic_reg_iommu.h"
+
+#define CCIC_IOMMU_TBU_NUM (12)
+#define CCIC_IOMMU_CH_NUM (CCIC_IOMMU_TBU_NUM)
+//#define IOMMU_TRANS_TAB_MAX_NUM (8192)
+//#define IOMMU_TRANS_TAB_MAX_NUM (65536)
+#define IOMMU_TRANS_TAB_MAX_NUM (32768)
+#define MMU_TID(dma_ch) (dma_ch)
+
+#define ccic_mmu_call(mmu_dev, f, args...)             \
+    ({                              \
+        struct ccic_iommu_device *__mmu_dev = (mmu_dev);            \
+        int __result;                       \
+        if (!__mmu_dev)                      \
+            __result = -ENODEV;             \
+        else if (!(__mmu_dev->ops && __mmu_dev->ops->f))        \
+            __result = -ENOIOCTLCMD;            \
+        else                            \
+            __result = __mmu_dev->ops->f(__mmu_dev, ##args);   \
+        __result;                       \
+    })
+
+struct mmu_ctx {
+	dma_addr_t tt_addr[2][2];
+	unsigned int *tt_base[2][2];
+	unsigned int tbu_update_cnt[2];
+};
+
+struct iommu_ch_info {
+	uint32_t tid;
+	uint64_t ttAddr;
+	uint32_t ttSize;
+};
+
+struct ccic_iommu_device {
+	struct device *dev;
+	unsigned long regs_base;
+	unsigned long ch_map;
+	uint32_t ch_matrix[CCIC_IOMMU_CH_NUM];
+	struct iommu_ch_info info[CCIC_IOMMU_CH_NUM];
+	spinlock_t ops_lock;
+	struct ccic_iommu_ops *ops;
+};
+
+struct ccic_iommu_ops {
+	int (*acquire_channel)(struct ccic_iommu_device *mmu_dev, uint32_t tid);
+	int (*release_channel)(struct ccic_iommu_device *mmu_dev, uint32_t tid);
+	int (*enable_channel)(struct ccic_iommu_device *mmu_dev, uint32_t tid);
+	int (*disable_channel)(struct ccic_iommu_device *mmu_dev, uint32_t tid);
+	int (*config_channel)(struct ccic_iommu_device *mmu_dev, uint32_t tid,
+			      uint64_t ttAddr, uint32_t ttSize);
+	uint64_t (*get_sva)(struct ccic_iommu_device *mmu_dev, uint32_t tid,
+			    uint32_t offset);
+	unsigned int (*irq_status)(struct ccic_iommu_device *mmu_dev);
+	int (*dump_channel_regs)(struct ccic_iommu_device *mmu_dev, uint32_t tid);
+	void (*set_timeout_default_addr)(struct ccic_iommu_device *mmu_dev, uint64_t timeout_default_addr);
+};
+
+struct ccic_iommu_device* ccic_iommu_create(struct device *dev, unsigned long regs_base);
+void ccic_iommu_unregister(struct ccic_iommu_device *mmu_dev);
+#endif /* ifndef __CCIC_IOMMU_H__ */
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_reg_iommu.h b/drivers/media/platform/spacemit/cam_ccic/ccic_reg_iommu.h
new file mode 100644
index 000000000000..c0ed4a30811e
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_reg_iommu.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * hw_reg_iommu.h
+ *
+ * register for iommu
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef _CCIC_REG_IOMMU_H_
+#define _CCIC_REG_IOMMU_H_
+
+/* TBU(n) registers */
+#define REG_IOMMU_TTBL(n)		(0x080 + 0x20 * (n))
+#define REG_IOMMU_TTBH(n)		(0x084 + 0x20 * (n))
+#define REG_IOMMU_TCR0(n)		(0x088 + 0x20 * (n))
+#define REG_IOMMU_TCR1(n)		(0x08c + 0x20 * (n))
+#define REG_IOMMU_STAT(n)		(0x090 + 0x20 * (n))
+
+/* TOP registers */
+#define REG_IOMMU_BVAL		(0x000)
+#define REG_IOMMU_BVAH		(0x004)
+#define REG_IOMMU_TVAL		(0x020)
+#define REG_IOMMU_TVAH		(0x024)
+#define REG_IOMMU_GIRQ_STAT	(0x018)
+#define REG_IOMMU_GIRQ_ENA	(0x01c)
+#define REG_IOMMU_TIMEOUT	(0x010)
+#define REG_IOMMU_ERR_CLR	(0x014)
+#define REG_IOMMU_RD_LVAL	(0x028)
+#define REG_IOMMU_RD_LVAH	(0x02c)
+#define REG_IOMMU_RD_LPAL	(0x030)
+#define REG_IOMMU_RD_LPAH	(0x034)
+#define REG_IOMMU_WR_LVAL	(0x038)
+#define REG_IOMMU_WR_LVAH	(0x03c)
+#define REG_IOMMU_WR_LPAL	(0x040)
+#define REG_IOMMU_WR_LPAH	(0x044)
+//#define REG_IOMMU_LVAL		(0x12220)
+//#define REG_IOMMU_LVAH		(0x12224)
+//#define REG_IOMMU_LPAL		(0x12228)
+//#define REG_IOMMU_LPAH		(0x1222c)
+#define REG_IOMMU_TIMEOUT_ADDR_LOW	(0x008)
+#define REG_IOMMU_TIMEOUT_ADDR_HIGH	(0x00c)
+//#define REG_IOMMU_VER		(0x1223c)
+
+#define MMU_RD_TIMEOUT	(1 << 16)
+#define MMU_WR_TIMEOUT	(1 << 17)
+
+#endif /* ifndef __REGS_ISP_IOMMU_H__ */
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_vdev.c b/drivers/media/platform/spacemit/cam_ccic/ccic_vdev.c
new file mode 100644
index 000000000000..346334dd2fb7
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_vdev.c
@@ -0,0 +1,1439 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * vdev.c - video divece functions
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#include <media/v4l2-dev.h>
+#include <media/media-entity.h>
+#include <media/media-device.h>
+#include <media/v4l2-subdev.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-ioctl.h>
+#include <linux/media-bus-format.h>
+#include <linux/compat.h>
+#include <media/spacemit/ccic_uapi.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-dma-contig.h>
+#include <media/videobuf2-dma-sg.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_qos.h>
+#include "ccic_vdev.h"
+#include "ccic_dma.h"
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+#include "ccic_iommu.h"
+#endif
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+extern struct ccic_iommu_device *mmu_dev;
+extern struct mmu_ctx mmu_ctx[CCIC_IOMMU_CH_NUM];
+#endif
+
+static struct {
+	__u32 pixelformat;
+	__u8 num_planes;
+	__u32 pixel_width_align;
+	__u32 pixel_height_align;
+	__u32 plane_bytes_align[VIDEO_MAX_PLANES];
+	struct {
+		__u32 num;
+		__u32 den;
+	}plane_bpp[VIDEO_MAX_PLANES];
+	struct {
+		__u32 num;
+		__u32 den;
+	}height_subsampling[VIDEO_MAX_PLANES];
+} ccic_formats_table[] = {
+	/* bayer raw8 */
+	{
+		.pixelformat = V4L2_PIX_FMT_SBGGR8,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 8,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SGBRG8,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 8,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SGRBG8,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 8,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SRGGB8,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 8,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	/* bayer raw10 */
+	{
+		.pixelformat = V4L2_PIX_FMT_SBGGR10P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 10,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SGBRG10P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 10,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SGRBG10P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 10,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SRGGB10P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 10,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	/* bayer raw12 */
+	{
+		.pixelformat = V4L2_PIX_FMT_SBGGR12P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 12,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SGBRG12P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 12,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SGRBG12P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 12,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	{
+		.pixelformat = V4L2_PIX_FMT_SRGGB12P,
+		.num_planes = 1,
+		.pixel_width_align = 1,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 12,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	/* yuv */
+	/* YUYV YUV422 */
+	{
+		.pixelformat = V4L2_PIX_FMT_YUYV,
+		.num_planes = 1,
+		.pixel_width_align = 2,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 16,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+	/* YVYU YUV422 */
+	{
+		.pixelformat = V4L2_PIX_FMT_YVYU,
+		.num_planes = 1,
+		.pixel_width_align = 2,
+		.plane_bytes_align = {
+			[0] = 1,
+		},
+		.plane_bpp = {
+			[0] = {
+				.num = 16,
+				.den = 1,
+			},
+		},
+		.height_subsampling = {
+			[0] = {
+				.num = 1,
+				.den = 1,
+			},
+		},
+	},
+};
+
+static int cvdev_lookup_formats_table(struct v4l2_format *f, int *bit_depth)
+{
+	struct v4l2_pix_format_mplane *pix_fmt = &f->fmt.pix_mp;
+	int loop = 0;
+
+	for (loop = 0; loop < ARRAY_SIZE(ccic_formats_table); loop++) {
+		if (ccic_formats_table[loop].pixelformat == pix_fmt->pixelformat) {
+			*bit_depth = ccic_formats_table[loop].plane_bpp[0].num;
+			break;
+		}
+	}
+	if (loop >= ARRAY_SIZE(ccic_formats_table))
+		return -1;
+
+	return 0;
+}
+
+void cvdev_fill_v4l2_format(struct v4l2_format *f)
+{
+	int loop = 0, plane = 0;
+	unsigned int width = 0, height = 0, stride = 0;
+	struct v4l2_plane_pix_format *plane_fmt = NULL;
+
+	for (loop = 0; loop < ARRAY_SIZE(ccic_formats_table); loop++) {
+		if (f->fmt.pix_mp.pixelformat == ccic_formats_table[loop].pixelformat) {
+			width = CAM_ALIGN(f->fmt.pix_mp.width, ccic_formats_table[loop].pixel_width_align);
+			if (0 == ccic_formats_table[loop].pixel_height_align)
+				ccic_formats_table[loop].pixel_height_align = 1;
+			height = CAM_ALIGN(f->fmt.pix_mp.height, ccic_formats_table[loop].pixel_height_align);
+			pr_debug("%s width=%u, width_align=%u",__func__ ,width, ccic_formats_table[loop].pixel_width_align);
+			f->fmt.pix_mp.num_planes = ccic_formats_table[loop].num_planes;
+			for (plane = 0; plane < f->fmt.pix_mp.num_planes; plane++) {
+				plane_fmt = &f->fmt.pix_mp.plane_fmt[plane];
+				stride = CAM_ALIGN((width * ccic_formats_table[loop].plane_bpp[plane].num) / (ccic_formats_table[loop].plane_bpp[plane].den * 8),
+								ccic_formats_table[loop].plane_bytes_align[plane]);
+				plane_fmt->sizeimage =
+					height * stride * ccic_formats_table[loop].height_subsampling[plane].num / ccic_formats_table[loop].height_subsampling[plane].den;
+				plane_fmt->bytesperline = stride;
+				pr_debug("plane%d stride=%u", plane, stride);
+			}
+			break;
+		}
+	}
+}
+
+static int cvdev_queue_setup(struct vb2_queue *q,
+				unsigned int *num_buffers,
+				unsigned int *num_planes,
+				unsigned int sizes[],
+				struct device *alloc_devs[])
+{
+	struct ccic_vnode *sc_vnode = container_of(q, struct ccic_vnode, buf_queue);
+	int loop = 0;
+
+	if (num_buffers && num_planes) {
+		*num_planes = sc_vnode->cur_fmt.fmt.pix_mp.num_planes;
+		pr_debug("%s num_buffers=%d num_planes=%d ", __func__, *num_buffers, *num_planes);
+		for (loop = 0; loop < *num_planes; loop++) {
+			sizes[loop] = sc_vnode->cur_fmt.fmt.pix_mp.plane_fmt[loop].sizeimage;
+			pr_debug("plane%d size=%u ", loop, sizes[loop]);
+		}
+	}
+	else {
+		pr_err("%s NULL num_buffers or num_planes\n", __func__);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void cvdev_wait_prepare(struct vb2_queue *q)
+{
+	/* going to wait sleep, release all locks that may block any vb2 buf/stream functions */
+	struct ccic_vnode *sc_vnode = container_of(q, struct ccic_vnode, buf_queue);
+	mutex_unlock(&sc_vnode->mlock);
+}
+
+static void cvdev_wait_finish(struct vb2_queue *q)
+{
+	/* wakeup from wait sleep, reacquire all locks */
+	struct ccic_vnode *sc_vnode = container_of(q, struct ccic_vnode, buf_queue);
+	mutex_lock(&sc_vnode->mlock);
+}
+
+static int cvdev_buf_init(struct vb2_buffer *vb)
+{
+	struct ccic_vbuffer *sc_vb = to_ccic_vbuffer(vb);
+
+	INIT_LIST_HEAD(&sc_vb->list_entry);
+	sc_vb->reset_flag = 0;
+	return 0;
+}
+
+static int cvdev_buf_prepare(struct vb2_buffer *vb)
+{
+	struct ccic_vbuffer *sc_vb = to_ccic_vbuffer(vb);
+	struct ccic_vnode *sc_vnode = container_of(vb->vb2_queue, struct ccic_vnode, buf_queue);
+
+	sc_vb->flags = 0;
+	memset(sc_vb->reserved, 0, BUF_RESERVED_DATA_LEN);
+	/* sc_vb->vb2_v4l2_buf.flags &= ~V4L2_BUF_FLAG_IGNOR; */
+	sc_vb->vb2_v4l2_buf.flags = 0;
+	sc_vb->sc_vnode = sc_vnode;
+	return 0;
+}
+
+static void cvdev_buf_finish(struct vb2_buffer *vb)
+{
+}
+
+static void cvdev_buf_cleanup(struct vb2_buffer *vb)
+{
+
+}
+
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+static uint32_t cvdev_fill_trans_tab_by_sg(uint32_t *tt_base, struct sg_table *sgt, uint32_t offset, uint32_t length)
+{
+	struct scatterlist *sg = NULL;
+	size_t temp_size = 0, temp_offset = 0, temp_length = 0;
+	dma_addr_t start_addr = 0, end_addr = 0, dmad = 0;
+	int i = 0;
+	uint32_t tt_size = 0;
+
+	sg = sgt->sgl;
+	for (i = 0; i < sgt->nents; ++i, sg = sg_next(sg)) {
+		pr_debug("sg%d: addr 0x%llx, size 0x%x", i, sg_phys(sg),
+			sg_dma_len(sg));
+		temp_size += sg_dma_len(sg);
+		if (temp_size <= offset) {
+			continue;
+		}
+
+		if (offset > temp_size - sg_dma_len(sg)) {
+			temp_offset =
+				offset - temp_size + sg_dma_len(sg);
+		} else {
+			temp_offset = 0;
+		}
+		start_addr = ((sg_phys(sg) + temp_offset) >> 12) << 12;
+
+		temp_length = temp_size - offset;
+		if (temp_length >= length) {
+			temp_offset = sg_dma_len(sg) - temp_length + length;
+		} else {
+			temp_offset = sg_dma_len(sg);
+		}
+		end_addr = ((sg_phys(sg) + temp_offset + 0xfff) >> 12) << 12;
+
+		for (dmad = start_addr; dmad < end_addr; dmad += 0x1000) {
+			tt_base[tt_size++] = (dmad >> 12) & 0x3fffff;
+		}
+
+		if (temp_length >= length)
+			break;
+	}
+
+	return tt_size;
+}
+
+dma_addr_t vb2_buf_paddr(struct vb2_buffer *vb, unsigned int plane_no)
+{
+	unsigned int offset = 0, length = 0, tt_size = 0, tid = 0;
+	int index = 0;
+	uint32_t *tt_base = NULL;
+	dma_addr_t tt_addr = 0;
+	dma_addr_t paddr = 0;
+	struct ccic_vbuffer *sc_vb = NULL;
+	struct ccic_vnode *sc_vnode = NULL;
+	/* struct scatterlist *sg = NULL; */
+	struct sg_table *sgt = NULL;
+	unsigned int dma_ch = 0;
+
+	sc_vb = vb2_buffer_to_ccic_vbuffer(vb);
+	sc_vnode = sc_vb->sc_vnode;
+	BUG_ON(!sc_vnode);
+	sgt = (struct sg_table*)vb2_plane_cookie(vb, plane_no);
+	offset = sc_vnode->planes_offset[vb->index][plane_no];
+	length = vb->planes[plane_no].length;
+	BUG_ON(sc_vnode->dma_ctx.dma_ch >= MAX_CCIC_DMA_CNT);
+	dma_ch = sc_vnode->dma_ctx.dma_ch;
+        index = (mmu_ctx[dma_ch].tbu_update_cnt[plane_no])++ & 0x1;
+        tt_base = mmu_ctx[dma_ch].tt_base[index][plane_no];
+        tt_addr = mmu_ctx[dma_ch].tt_addr[index][plane_no];
+        tid = MMU_TID(dma_ch);
+	tt_size = cvdev_fill_trans_tab_by_sg(tt_base, sgt, offset, length);
+	ccic_mmu_call(mmu_dev, config_channel, tid, tt_addr, tt_size);
+	ccic_mmu_call(mmu_dev, enable_channel, tid);
+	paddr = (dma_addr_t)mmu_dev->ops->get_sva(mmu_dev, tid, offset);
+	return paddr;
+}
+#endif
+
+static int cvdev_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+	struct ccic_vnode *sc_vnode = container_of(q, struct ccic_vnode, buf_queue);
+	struct ccic_dma *ccic_dma = get_ccic_dma();
+	struct device *dev = sc_vnode->ccic_dev->dev;
+	struct ccic_vbuffer *sc_vb = NULL;
+	int ret = 0;
+
+	pr_debug("%s(%s)", __func__, sc_vnode->name);
+	sc_vnode->total_frm = 0;
+	sc_vnode->sw_err_frm = 0;
+	sc_vnode->hw_err_frm = 0;
+	sc_vnode->ok_frm = 0;
+	sc_vnode->frame_id = 0;
+
+	ret = ccic_dma_ch_irq_enable(ccic_dma, sc_vnode->dma_ctx.dma_ch, 1);
+	if (ret < 0) {
+		dev_err(dev, "%s(%s) ccic_dma_ch_irq_enable failed ret=%d\n", __func__, sc_vnode->name, ret);
+		return ret;
+	}
+	ret = ccic_dma_ch_enable(ccic_dma, sc_vnode->dma_ctx.dma_ch, 1);
+	if (ret < 0) {
+		dev_err(dev, "%s(%s) ccic_dma_ch_enable failed ret=%d\n", __func__, sc_vnode->name, ret);
+		return ret;
+	}
+
+	ret = cvdev_dq_idle_vbuffer(sc_vnode, &sc_vb);
+	if (ret) {
+		dev_info(dev, "%s(%s) no initial buffer available\n", __func__, sc_vnode->name);
+	} else {
+		cvdev_q_busy_vbuffer(sc_vnode, sc_vb);
+	}
+	if (sc_vb) {
+		ccic_update_dma_addr(sc_vnode, sc_vb, 0);
+		ccic_dma_ch_shadow_ready(ccic_dma, sc_vnode->dma_ctx.dma_ch, 1);
+	}
+	sc_vnode->is_streaming = 1;
+	return 0;
+}
+
+static void cvdev_stop_streaming(struct vb2_queue *q)
+{
+	struct ccic_vnode *sc_vnode = container_of(q, struct ccic_vnode, buf_queue);
+	struct ccic_dma *ccic_dma = get_ccic_dma();
+	unsigned long flags = 0;
+
+	pr_debug("%s(%s) enter", __func__, sc_vnode->name);
+
+	pr_notice("%s total_frm(%u) sw_err_frm(%u) hw_err_frm(%u) ok_frm(%u)\n",
+			sc_vnode->name, sc_vnode->total_frm, sc_vnode->sw_err_frm, sc_vnode->hw_err_frm, sc_vnode->ok_frm);
+
+	spin_lock_irqsave(&(sc_vnode->waitq_head.lock), flags);
+	wait_event_interruptible_locked_irq(sc_vnode->waitq_head, !sc_vnode->in_irq && !sc_vnode->in_tasklet);
+	sc_vnode->in_streamoff = 1;
+	spin_unlock_irqrestore(&(sc_vnode->waitq_head.lock), flags);
+
+	ccic_dma_ch_irq_enable(ccic_dma, sc_vnode->dma_ctx.dma_ch, 0);
+	ccic_dma_ch_enable(ccic_dma, sc_vnode->dma_ctx.dma_ch, 0);
+
+	sc_vnode->is_streaming = 0;
+	sc_vnode->dma_ctx.dma_ch = MAX_CCIC_DMA_CNT;
+	spin_lock_irqsave(&(sc_vnode->waitq_head.lock), flags);
+	sc_vnode->in_streamoff = 0;
+	spin_unlock_irqrestore(&(sc_vnode->waitq_head.lock), flags);
+	pr_debug("%s(%s) leave", __func__, sc_vnode->name);
+}
+
+static void cvdev_buf_queue(struct vb2_buffer *vb)
+{
+	unsigned long flags = 0;
+	struct ccic_vbuffer *sc_vb = to_ccic_vbuffer(vb);
+	struct vb2_queue *buf_queue = vb->vb2_queue;
+	struct ccic_vnode *sc_vnode = container_of(buf_queue, struct ccic_vnode, buf_queue);
+	/* struct ccic_dma *ccic_dma = sc_vnode->ccic_dev->dma; */
+	/* unsigned int v4l2_buf_flags = sc_vnode->v4l2_buf_flags[vb->index]; */
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	atomic_inc(&sc_vnode->queued_buf_cnt);
+	list_add_tail(&sc_vb->list_entry, &sc_vnode->queued_list);
+	/* if (sc_vnode->is_streaming) {
+		if (__cvdev_busy_list_empty(sc_vnode)) {
+			__cvdev_dq_idle_vbuffer(sc_vnode, &sc_vb);
+			if (sc_vb) {
+				__cvdev_q_busy_vbuffer(sc_vnode, sc_vb);
+				ccic_update_dma_addr(sc_vnode, sc_vb, 0);
+				ccic_dma->ops->shadow_ready(ccic_dma);
+			}
+		}
+	} */
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+}
+
+static struct vb2_ops ccic_vb2_ops = {
+	.queue_setup = cvdev_queue_setup,
+	.wait_prepare = cvdev_wait_prepare,
+	.wait_finish = cvdev_wait_finish,
+	.buf_init = cvdev_buf_init,
+	.buf_prepare = cvdev_buf_prepare,
+	.buf_finish = cvdev_buf_finish,
+	.buf_cleanup = cvdev_buf_cleanup,
+	.start_streaming = cvdev_start_streaming,
+	.stop_streaming = cvdev_stop_streaming,
+	.buf_queue = cvdev_buf_queue,
+};
+
+static void cvdev_cancel_all_buffers(struct ccic_vnode *sc_vnode)
+{
+	unsigned long flags = 0;
+	struct ccic_vbuffer *pos = NULL, *n = NULL;
+	struct vb2_buffer *vb2_buf = NULL;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	list_for_each_entry_safe(pos, n, &sc_vnode->queued_list, list_entry) {
+		vb2_buf = &(pos->vb2_v4l2_buf.vb2_buf);
+		vb2_buffer_done(vb2_buf, VB2_BUF_STATE_ERROR);
+		list_del_init(&pos->list_entry);
+		atomic_dec(&sc_vnode->queued_buf_cnt);
+	}
+	list_for_each_entry_safe(pos, n, &sc_vnode->busy_list, list_entry) {
+		vb2_buf = &(pos->vb2_v4l2_buf.vb2_buf);
+		vb2_buffer_done(vb2_buf, VB2_BUF_STATE_ERROR);
+		list_del_init(&pos->list_entry);
+		atomic_dec(&sc_vnode->busy_buf_cnt);
+	}
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+}
+
+static void cvdev_flush_all_buffers(struct ccic_vnode *sc_vnode)
+{
+	unsigned long flags = 0;
+	struct ccic_vbuffer *pos = NULL, *n = NULL;
+	struct vb2_buffer *vb2_buf = NULL;
+	struct ccic_dma *ccic_dma = get_ccic_dma();
+	unsigned long timeout = 0;
+	int ret, wait_done = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+
+	/* cancel next buffer output by dma. */
+	if (atomic_read(&sc_vnode->busy_buf_cnt) >= 1) {
+		wait_done = 1;
+		sc_vnode->wait_done_flush = 1;
+		if (atomic_read(&sc_vnode->busy_buf_cnt) > 1) {
+			ccic_dma_ch_shadow_ready(ccic_dma, sc_vnode->dma_ctx.dma_ch, 0);
+			pos = list_last_entry(&sc_vnode->busy_list, struct ccic_vbuffer, list_entry);
+			vb2_buf = &(pos->vb2_v4l2_buf.vb2_buf);
+			vb2_buffer_done(vb2_buf, VB2_BUF_STATE_ERROR);
+			list_del_init(&pos->list_entry);
+			atomic_dec(&sc_vnode->busy_buf_cnt);
+		}
+	}
+
+	list_for_each_entry_safe(pos, n, &sc_vnode->queued_list, list_entry) {
+		vb2_buf = &(pos->vb2_v4l2_buf.vb2_buf);
+		vb2_buffer_done(vb2_buf, VB2_BUF_STATE_ERROR);
+		list_del_init(&pos->list_entry);
+		atomic_dec(&sc_vnode->queued_buf_cnt);
+	}
+
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+
+	if (wait_done) {
+		timeout = msecs_to_jiffies(300);
+		ret = wait_for_completion_timeout(&sc_vnode->flush_complete, timeout);
+		if (0 == ret) {
+			pr_warn("csi flush buffer wait completion timeout.");
+		}
+	}
+}
+
+static int cvdev_vidioc_reqbufs(struct file *file, void *fh, struct v4l2_requestbuffers *b)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+
+	mutex_lock(&sc_vnode->mlock);
+	ret = vb2_reqbufs(&sc_vnode->buf_queue, b);
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *b)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+
+	mutex_lock(&sc_vnode->mlock);
+	ret = vb2_querybuf(&sc_vnode->buf_queue, b);
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_qbuf(struct file *file, void *fh, struct v4l2_buffer *b)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+	unsigned int i = 0;
+
+	sc_vnode->v4l2_buf_flags[b->index] = b->flags;
+	if (!b->m.planes) {
+		return -EINVAL;
+	}
+	for (i = 0; i < b->length; i++) {
+		sc_vnode->planes_offset[b->index][i] = b->m.planes[i].data_offset;
+	}
+	mutex_lock(&sc_vnode->mlock);
+	ret = vb2_qbuf(&sc_vnode->buf_queue, vnode->v4l2_dev->mdev, b);
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_expbuf(struct file *file, void *fh, struct v4l2_exportbuffer *e)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+
+	mutex_lock(&sc_vnode->mlock);
+#ifndef MODULE
+	ret = vb2_expbuf(&sc_vnode->buf_queue, e);
+#endif
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *b)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+
+	mutex_lock(&sc_vnode->mlock);
+	ret = vb2_dqbuf(&sc_vnode->buf_queue, b, file->f_flags & O_NONBLOCK);
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_create_bufs(struct file *file, void *fh, struct v4l2_create_buffers *b)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+
+	mutex_lock(&sc_vnode->mlock);
+#ifndef MODULE
+	ret = vb2_create_bufs(&sc_vnode->buf_queue, b);
+#endif
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_prepare_buf(struct file *file, void *fh, struct v4l2_buffer *b)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+
+	mutex_lock(&sc_vnode->mlock);
+	ret = vb2_prepare_buf(&sc_vnode->buf_queue, vnode->v4l2_dev->mdev, b);
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_streamon(struct file *file, void *fn, enum v4l2_buf_type i)
+{
+	int ret = 0;
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	mutex_lock(&sc_vnode->mlock);
+	ret = vb2_streamon(&sc_vnode->buf_queue, i);
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_streamoff(struct file *file, void *fn, enum v4l2_buf_type i)
+{
+	int ret = 0;
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	unsigned long flags = 0;
+
+	pr_debug("%s(%s) enter", __func__, sc_vnode->name);
+	pr_debug("%s(%s) queued_buf_cnt=%d busy_buf_cnt=%d.", __func__, sc_vnode->name, atomic_read(&sc_vnode->queued_buf_cnt), atomic_read(&sc_vnode->busy_buf_cnt));
+	spin_lock_irqsave(&sc_vnode->waitq_head.lock, flags);
+	wait_event_interruptible_locked_irq(sc_vnode->waitq_head, !sc_vnode->in_tasklet && !sc_vnode->in_irq);
+	sc_vnode->in_streamoff = 1;
+	spin_unlock_irqrestore(&sc_vnode->waitq_head.lock, flags);
+	pr_debug("%s tasklet clean", sc_vnode->name);
+	mutex_lock(&sc_vnode->mlock);
+	pr_debug("%s cancel all buffers", sc_vnode->name);
+	cvdev_cancel_all_buffers(sc_vnode);
+	pr_debug("%s streamoff", sc_vnode->name);
+	ret = vb2_streamoff(&sc_vnode->buf_queue, i);
+	mutex_unlock(&sc_vnode->mlock);
+	spin_lock_irqsave(&sc_vnode->waitq_head.lock, flags);
+	sc_vnode->in_streamoff = 0;
+	spin_unlock_irqrestore(&sc_vnode->waitq_head.lock, flags);
+	pr_debug("%s(%s) leave", __func__, sc_vnode->name);
+	return ret;
+}
+
+static int cvdev_vidioc_querycap(struct file *file, void *fh, struct v4l2_capability *cap)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+
+	strncpy(cap->driver, sc_vnode->name, 16);
+	cap->capabilities = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE_MPLANE;
+	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE_MPLANE;
+	return 0;
+}
+
+/*
+ * static int cvdev_vidioc_enum_fmt_vid_cap_mplane(struct file *file, void *fh, struct v4l2_fmtdesc *f)
+ * {
+ *     return 0;
+ * }
+ */
+
+static int cvdev_vidioc_g_fmt_vid_cap_mplane(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+
+	pr_debug("get format fourcc code[0x%08x] (%dx%d)",
+			sc_vnode->cur_fmt.fmt.pix_mp.pixelformat,
+			sc_vnode->cur_fmt.fmt.pix_mp.width,
+			sc_vnode->cur_fmt.fmt.pix_mp.height);
+	*f = sc_vnode->cur_fmt;
+	return 0;
+}
+
+static int __cvdev_vidioc_s_fmt_vid_cap_mplane(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	struct vb2_queue *vb2_queue = &sc_vnode->buf_queue;
+	struct ccic_dma *ccic_dma = get_ccic_dma();
+	struct device *dev = &vnode->dev;
+	int ret = 0, bit_depth = 0;
+	unsigned int fmt_code = 0, width = 0, height = 0, h_offset, v_offset;
+
+	pr_debug("set format fourcc code[0x%08x] (%dx%d)",
+			f->fmt.pix_mp.pixelformat, f->fmt.pix_mp.width, f->fmt.pix_mp.height);
+	width = f->fmt.pix_mp.width;
+	height = f->fmt.pix_mp.height;
+	h_offset = f->fmt.pix_mp.reserved[0];
+	v_offset = f->fmt.pix_mp.reserved[1];
+
+	ret = cvdev_lookup_formats_table(f, &bit_depth);
+	if (ret) {
+		pr_err("%s failed to lookup formats table fourcc code[0x%08x]\n", __func__, f->fmt.pix_mp.pixelformat);
+		return ret;
+	}
+
+	if (bit_depth == 8) {
+		fmt_code = CSI_DUMP_FMT_RAW8;
+	} else if (bit_depth == 10) {
+		fmt_code = CSI_DUMP_FMT_RAW10;
+	} else if (bit_depth == 12) {
+		fmt_code = CSI_DUMP_FMT_RAW12;
+	} else if (bit_depth == 16) {
+		fmt_code = CSI_DUMP_FMT_YUV422;
+		/* width *= 2; */
+	} else {
+		dev_err(dev, "unknown bit_depth=%d\n", bit_depth);
+		return -EINVAL;
+	}
+	if (vb2_is_streaming(vb2_queue)) {
+		/* ret = ccic_dma->ops->change_window_size(ccic_dma, width, height, h_offset, v_offset, fmt_code); */
+		/*if (ret) {
+			dev_err(dev, "%s resize windwo(%ux%u offset:%ux%u) failed\n", __func__,
+							width, height, h_offset, v_offset);
+			return ret;
+		} */
+	} else {
+		ret = ccic_dma_ch_set_fmt(ccic_dma, sc_vnode->dma_ctx.dma_ch,
+								width, height, h_offset, v_offset, fmt_code);
+		if (ret) {
+			dev_err(dev, "%s set fmt(%ux%u code:0x%08x) failed\n", __func__,
+					width, height, fmt_code);
+			return ret;
+		}
+	}
+	cvdev_fill_v4l2_format(f);
+	sc_vnode->cur_fmt = *f;
+
+	return 0;
+}
+
+static int cvdev_vidioc_s_fmt_vid_cap_mplane(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	int ret = 0;
+
+	mutex_lock(&sc_vnode->mlock);
+	ret = __cvdev_vidioc_s_fmt_vid_cap_mplane(file, fh, f);
+	mutex_unlock(&sc_vnode->mlock);
+	return ret;
+}
+
+static int cvdev_vidioc_try_fmt_vid_cap_mplane(struct file *file, void *fh, struct v4l2_format *f)
+{
+	return 0;
+}
+
+static int cvdev_vidioc_s_parm(struct file *file, void *fh, struct v4l2_streamparm *a)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	struct csi_out_path_param *out_path_param = (struct csi_out_path_param*)(&a->parm.raw_data[sizeof(struct v4l2_captureparm)]);
+	struct ccic_ctrl *ccic_ctrl = sc_vnode->ccic_dev->ctrl;
+	struct device *dev = &vnode->dev;
+	struct ccic_dma *ccic_dma = get_ccic_dma();
+	unsigned int path_id = sc_vnode->idx % PATH_NUM_PER_DEV;
+	int ret = 0;
+
+	if (out_path_param->mode_param.mode >= CSI_WORK_MODE_MAX) {
+		dev_err(dev, "%s(%s) invalid csi work mode %u\n", __func__, sc_vnode->name,
+				out_path_param->mode_param.mode);
+		return -EINVAL;
+	}
+	if (out_path_param->dma_channel >= MAX_CCIC_DMA_CNT) {
+		dev_err(dev, "%s(%s) invalid dma channel %u\n", __func__, sc_vnode->name,
+				out_path_param->dma_channel);
+		return -EINVAL;
+	}
+	ret = ccic_ctrl->ops->config_csi2_mode(ccic_ctrl, out_path_param->mode_param.mode,
+										out_path_param->mode_param.dt_filter_en);
+	if (ret < 0) {
+		dev_err(dev, "%s(%s) config csi work mode %u failed\n", __func__, sc_vnode->name,
+				out_path_param->mode_param.mode);
+		return ret;
+	}
+	if (out_path_param->mode_param.mode == CSI_WORK_MODE_VC) {
+		ret = ccic_ctrl->ops->config_csi_path_vc(ccic_ctrl, path_id, out_path_param->vc);
+		if (ret < 0) {
+			dev_err(dev, "%s(%s) config vc %u failed\n", __func__, sc_vnode->name,
+					out_path_param->vc);
+			return ret;
+		}
+		if (out_path_param->mode_param.dt_filter_en) {
+			ret = ccic_ctrl->ops->config_csi_path_dt_filter(ccic_ctrl, path_id,
+							out_path_param->dt_filter0_en,
+							out_path_param->filter0_pattern,
+							out_path_param->dt_filter1_en,
+							out_path_param->filter1_pattern);
+			if (ret < 0) {
+				dev_err(dev, "%s(%s) config dt filters failed\n", __func__, sc_vnode->name);
+				return ret;
+			}
+		}
+	}
+	ret = ccic_ctrl->ops->config_csi2_mbus(ccic_ctrl,
+							out_path_param->phy_param.lane_num,
+							out_path_param->phy_param.mipi_mbps);
+	if (ret < 0) {
+		dev_err(dev, "%s(%s) config mbus lanes %u mipi_mbps %u failed\n", __func__, sc_vnode->name,
+				out_path_param->phy_param.lane_num,
+				out_path_param->phy_param.mipi_mbps);
+		return ret;
+	}
+	ret = ccic_dma_ch_src_sel(ccic_dma, out_path_param->dma_channel, sc_vnode->idx);
+	if (ret < 0) {
+		dev_err(dev, "%s(%s) attach dma ch %u failed\n", __func__, sc_vnode->name, out_path_param->dma_channel);
+		return ret;
+	}
+	dev_info(dev,
+			"%s mode %u dt_en %u vc %u filter0_en %u filter0 %u filter1_en %u filter1 %u lanes %u mipi_bps %u\n",
+			sc_vnode->name, out_path_param->mode_param.mode, out_path_param->mode_param.dt_filter_en,
+			out_path_param->vc, out_path_param->dt_filter0_en, out_path_param->filter0_pattern,
+			out_path_param->dt_filter1_en, out_path_param->filter1_pattern,
+			out_path_param->phy_param.lane_num, out_path_param->phy_param.mipi_mbps);
+	dev_info(dev, "%s set dma ch %u\n", sc_vnode->name, out_path_param->dma_channel);
+	sc_vnode->dma_ctx.dma_ch =  out_path_param->dma_channel;
+	return 0;
+}
+
+static long cvdev_vidioc_default(struct file *file,
+					void *fh,
+					bool valid_prio,
+					unsigned int cmd,
+					void *arg)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	struct device *dev = &vnode->dev;
+	struct csi_path_info *path_info = NULL;
+
+	switch (cmd) {
+	case CSI_VIDIOC_G_PATH_INFO:
+		path_info = (struct csi_path_info*)arg;
+		path_info->id = sc_vnode->idx;
+		path_info->csi_id = sc_vnode->ccic_dev->index;
+		path_info->path_id = sc_vnode->idx % PATH_NUM_PER_DEV;
+		break;
+	case CSI_VIDIOC_FLUSH_BUF:
+		cvdev_flush_all_buffers(sc_vnode);
+		break;
+	default:
+		dev_err(dev, "%s(%s) unknown ioctl cmd(%d)\n", __func__, sc_vnode->name, cmd);
+		return -ENOIOCTLCMD;
+	}
+	return 0;
+}
+
+static struct v4l2_ioctl_ops ccic_v4l2_ioctl_ops = {
+	/* VIDIOC_QUERYCAP handler */
+	.vidioc_querycap = cvdev_vidioc_querycap,
+	/* VIDIOC_ENUM_FMT handlers */
+	/* .vidioc_enum_fmt_vid_cap = cvdev_vidioc_enum_fmt_vid_cap_mplane, */
+	/* VIDIOC_G_FMT handlers */
+	.vidioc_g_fmt_vid_cap_mplane = cvdev_vidioc_g_fmt_vid_cap_mplane,
+	/* VIDIOC_S_FMT handlers */
+	.vidioc_s_fmt_vid_cap_mplane = cvdev_vidioc_s_fmt_vid_cap_mplane,
+	/* VIDIOC_TRY_FMT handlers */
+	.vidioc_try_fmt_vid_cap_mplane = cvdev_vidioc_try_fmt_vid_cap_mplane,
+	/* Buffer handlers */
+	.vidioc_reqbufs = cvdev_vidioc_reqbufs,
+	.vidioc_querybuf = cvdev_vidioc_querybuf,
+	.vidioc_qbuf = cvdev_vidioc_qbuf,
+	.vidioc_expbuf = cvdev_vidioc_expbuf,
+	.vidioc_dqbuf = cvdev_vidioc_dqbuf,
+	.vidioc_create_bufs = cvdev_vidioc_create_bufs,
+	.vidioc_prepare_buf = cvdev_vidioc_prepare_buf,
+	.vidioc_streamon = cvdev_vidioc_streamon,
+	.vidioc_streamoff = cvdev_vidioc_streamoff,
+	/* int (*vidioc_s_parm)(struct file *file, void *fh, struct v4l2_streamparm *a); */
+	.vidioc_s_parm = cvdev_vidioc_s_parm,
+	.vidioc_default = cvdev_vidioc_default,
+};
+
+static int cvdev_open(struct file *file)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	struct ccic_ctrl *ccic_ctrl = sc_vnode->ccic_dev->ctrl;
+	struct v4l2_fh *fh;
+
+	if (atomic_inc_return(&sc_vnode->ref_cnt) != 1) {
+		pr_err("vnode(%s - %s) was already openned.\n", sc_vnode->name, video_device_node_name(vnode));
+		atomic_dec(&sc_vnode->ref_cnt);
+		return -EBUSY;
+	}
+
+	/* fix v4l2_fh allocation failure issue */
+	fh = kzalloc(sizeof(*fh), GFP_KERNEL);
+	if (!fh) {
+		atomic_dec(&sc_vnode->ref_cnt);
+		return -ENOMEM;
+	}
+
+	v4l2_fh_init(fh, vnode);
+	v4l2_fh_add(fh, file);
+	file->private_data = fh;
+
+	ccic_ctrl->ops->clk_enable(ccic_ctrl, 1);
+	ccic_ctrl->ops->irq_mask(ccic_ctrl, 1);
+	pr_debug("open vnode(%s - %s).", sc_vnode->name, video_device_node_name(vnode));
+	return 0;
+}
+
+static void __cvdev_close(struct ccic_vnode *sc_vnode)
+{
+	unsigned long flags = 0;
+
+	pr_debug("%s(%s) enter", __func__, sc_vnode->name);
+	pr_debug("%s(%s) queued_buf_cnt=%d busy_buf_cnt=%d.", __func__, sc_vnode->name, atomic_read(&sc_vnode->queued_buf_cnt), atomic_read(&sc_vnode->busy_buf_cnt));
+	spin_lock_irqsave(&sc_vnode->waitq_head.lock, flags);
+	sc_vnode->in_streamoff = 1;
+	wait_event_interruptible_locked_irq(sc_vnode->waitq_head, !sc_vnode->in_tasklet && !sc_vnode->in_irq);
+	spin_unlock_irqrestore(&sc_vnode->waitq_head.lock, flags);
+	pr_debug("%s tasklet clean", sc_vnode->name);
+	mutex_lock(&sc_vnode->mlock);
+	pr_debug("%s cancel all buffers", sc_vnode->name);
+	cvdev_cancel_all_buffers(sc_vnode);
+	pr_debug("%s queue release", sc_vnode->name);
+	vb2_queue_release(&sc_vnode->buf_queue);
+	sc_vnode->buf_queue.owner = NULL;
+	sc_vnode->is_streaming = 0;
+	mutex_unlock(&sc_vnode->mlock);
+	spin_lock_irqsave(&sc_vnode->waitq_head.lock, flags);
+	sc_vnode->in_streamoff = 0;
+	spin_unlock_irqrestore(&sc_vnode->waitq_head.lock, flags);
+	reinit_completion(&sc_vnode->flush_complete);
+	sc_vnode->wait_done_flush = 0;
+
+	pr_debug("%s(%s) leave", __func__, sc_vnode->name);
+}
+
+static int cvdev_close(struct file *file)
+{
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+	struct ccic_ctrl *ccic_ctrl = sc_vnode->ccic_dev->ctrl;
+
+	if (atomic_dec_and_test(&sc_vnode->ref_cnt)) {
+		__cvdev_close(sc_vnode);
+		ccic_ctrl->ops->clk_enable(ccic_ctrl, 0);
+	}
+
+	return v4l2_fh_release(file);
+}
+
+static __poll_t cvdev_poll(struct file *file, struct poll_table_struct *wait)
+{
+	__poll_t ret;
+	struct video_device *vnode = video_devdata(file);
+	struct ccic_vnode *sc_vnode = container_of(vnode, struct ccic_vnode, vnode);
+
+	ret = vb2_poll(&sc_vnode->buf_queue, file, wait);
+
+	return ret;
+}
+
+static struct v4l2_file_operations ccic_file_operations = {
+	.owner = THIS_MODULE,
+	.poll = cvdev_poll,
+	.unlocked_ioctl = video_ioctl2,
+	.open = cvdev_open,
+	.release = cvdev_close,
+};
+
+static void cvdev_release(struct video_device *vdev)
+{
+	struct ccic_vnode *sc_vnode = container_of(vdev, struct ccic_vnode, vnode);
+
+	pr_debug("%s(%s %s) enter.", __func__, sc_vnode->name, video_device_node_name(&sc_vnode->vnode));
+	mutex_destroy(&sc_vnode->mlock);
+}
+/*
+static void cvdev_block_release(struct ccic_block *b)
+{
+	struct ccic_vnode *sc_vnode = container_of(b, struct ccic_vnode, ac_block);
+
+	pr_debug("%s(%s %s) enter.", __func__, sc_vnode->name, video_device_node_name(&sc_vnode->vnode));
+	vb2_queue_release(&sc_vnode->buf_queue);
+	video_unregister_device(&sc_vnode->vnode);
+}
+*/
+
+void cvdev_destroy_vnode(struct ccic_vnode *sc_vnode)
+{
+	video_unregister_device(&sc_vnode->vnode);
+}
+
+struct ccic_vnode* cvdev_create_vnode(const char *name,
+							unsigned int idx,
+							struct v4l2_device *v4l2_dev,
+							struct device *alloc_dev,
+							struct ccic_dev *ccic_dev,
+							void (*dma_tasklet_handler)(unsigned long),
+							unsigned int min_buffers_needed)
+{
+	int ret = 0, i = 0;
+	struct ccic_vnode *sc_vnode = NULL;
+	struct ccic_dma_context *dma_ctx = NULL;
+	struct ccic_dma_work_struct *ccic_dma_work = NULL;
+
+	if (NULL == name || NULL == v4l2_dev || NULL == alloc_dev || NULL == ccic_dev) {
+		pr_err("%s invalid arguments.\n", __func__);
+		return NULL;
+	}
+	sc_vnode = devm_kzalloc(alloc_dev, sizeof(*sc_vnode), GFP_KERNEL);
+	if (NULL == sc_vnode) {
+		pr_err("%s failed to alloc mem for ccic_vnode(%s).\n", __func__, name);
+		return NULL;
+	}
+	dma_ctx = &sc_vnode->dma_ctx;
+	dma_ctx->sc_vnode = sc_vnode;
+	dma_ctx->dma_ch = MAX_CCIC_DMA_CNT;
+	INIT_LIST_HEAD(&dma_ctx->dma_work_idle_list);
+	INIT_LIST_HEAD(&dma_ctx->dma_work_busy_list);
+	spin_lock_init(&dma_ctx->slock);
+	for (i = 0; i < CCIC_DMA_WORK_MAX_CNT; i++) {
+		ccic_dma_work = devm_kzalloc(alloc_dev, sizeof(*ccic_dma_work), GFP_KERNEL);
+		if (!ccic_dma_work) {
+			dev_err(alloc_dev, "%s not enough mem\n", __func__);
+			return NULL;
+		}
+		tasklet_init(&ccic_dma_work->dma_tasklet, dma_tasklet_handler, (unsigned long)ccic_dma_work);
+		INIT_LIST_HEAD(&ccic_dma_work->idle_list_entry);
+		INIT_LIST_HEAD(&ccic_dma_work->busy_list_entry);
+		ccic_dma_work->sc_vnode = sc_vnode;
+		list_add(&ccic_dma_work->idle_list_entry, &dma_ctx->dma_work_idle_list);
+	}
+	sc_vnode->csi2vc = CCIC_CSI2VC_MAIN;
+	sc_vnode->src_sel = CCIC_DMA_SEL_LOCAL_MAIN;
+	sc_vnode->lane_num = 1;
+	sc_vnode->in_streamoff = 0;
+	sc_vnode->in_irq = 0;
+	sc_vnode->in_tasklet = 0;
+	INIT_LIST_HEAD(&sc_vnode->queued_list);
+	INIT_LIST_HEAD(&sc_vnode->busy_list);
+	atomic_set(&sc_vnode->queued_buf_cnt, 0);
+	atomic_set(&sc_vnode->busy_buf_cnt, 0);
+	spin_lock_init(&sc_vnode->slock);
+	init_completion(&sc_vnode->flush_complete);
+	sc_vnode->wait_done_flush = 0;
+	mutex_init(&sc_vnode->mlock);
+	init_waitqueue_head(&sc_vnode->waitq_head);
+	sc_vnode->idx = idx;
+	sc_vnode->buf_queue.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC | V4L2_BUF_FLAG_TSTAMP_SRC_SOE;
+	sc_vnode->buf_queue.buf_struct_size = sizeof(struct ccic_vbuffer);
+	sc_vnode->buf_queue.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	sc_vnode->buf_queue.io_modes = VB2_DMABUF;
+	sc_vnode->buf_queue.ops = &ccic_vb2_ops;
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+	sc_vnode->buf_queue.mem_ops = &vb2_dma_sg_memops;
+#else
+	sc_vnode->buf_queue.mem_ops = &vb2_dma_contig_memops;
+#endif
+	/* sc_vnode->buf_queue.min_buffers_needed = min_buffers_needed; */
+	sc_vnode->buf_queue.dev = alloc_dev;
+	ret = vb2_queue_init(&sc_vnode->buf_queue);
+	if (ret) {
+		pr_err("%s vb2_queue_init failed for ccic_vnode(%s).\n", __func__, name);
+		goto queue_init_fail;
+	}
+
+	strncpy(sc_vnode->vnode.name, name, 32);
+	strncpy(sc_vnode->name, name, 32);
+	sc_vnode->ccic_dev = ccic_dev;
+	sc_vnode->vnode.queue = &sc_vnode->buf_queue;
+	sc_vnode->vnode.fops = &ccic_file_operations;
+	sc_vnode->vnode.ioctl_ops = &ccic_v4l2_ioctl_ops;
+	sc_vnode->vnode.release = cvdev_release;
+	sc_vnode->vnode.device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE_MPLANE;
+	sc_vnode->vnode.v4l2_dev = v4l2_dev;
+	set_bit(V4L2_FL_USES_V4L2_FH, &sc_vnode->vnode.flags);
+	ret = __video_register_device(&sc_vnode->vnode, VFL_TYPE_VIDEO, -1, 1, THIS_MODULE);
+	if (ret) {
+		pr_err("%s video dev register failed for ccic_vnode(%s).\n", __func__, name);
+		goto vdev_register_fail;
+	}
+	ccic_dev->vnode = sc_vnode;
+	pr_debug("create vnode(%s - %s) successfully.", name, video_device_node_name(&sc_vnode->vnode));
+	return sc_vnode;
+vdev_register_fail:
+	vb2_queue_release(&sc_vnode->buf_queue);
+queue_init_fail:
+	devm_kfree(alloc_dev, sc_vnode);
+	return NULL;
+}
+
+int __cvdev_dq_idle_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	*sc_vb = list_first_entry_or_null(&sc_vnode->queued_list, struct ccic_vbuffer, list_entry);
+	if (NULL == *sc_vb)
+		return -1;
+	list_del_init(&(*sc_vb)->list_entry);
+	atomic_dec(&sc_vnode->queued_buf_cnt);
+	return 0;
+}
+
+int __cvdev_q_idle_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer *sc_vb)
+{
+	list_add_tail(&sc_vb->list_entry, &sc_vnode->queued_list);
+	atomic_inc(&sc_vnode->queued_buf_cnt);
+	return 0;
+}
+
+int cvdev_dq_idle_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	unsigned long flags = 0;
+	int ret = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	ret = __cvdev_dq_idle_vbuffer(sc_vnode, sc_vb);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+	return ret;
+}
+
+int cvdev_q_idle_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer *sc_vb)
+{
+	unsigned long flags = 0;
+	int ret = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	ret = __cvdev_q_idle_vbuffer(sc_vnode, sc_vb);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+
+	return ret;
+}
+
+int cvdev_pick_idle_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	unsigned long flags = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	*sc_vb = list_first_entry_or_null(&sc_vnode->queued_list, struct ccic_vbuffer, list_entry);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+	if (NULL == *sc_vb) {
+		return -1;
+	}
+	return 0;
+}
+
+int __cvdev_pick_idle_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	*sc_vb = list_first_entry_or_null(&sc_vnode->queued_list, struct ccic_vbuffer, list_entry);
+	if (NULL == *sc_vb) {
+		return -1;
+	}
+	return 0;
+}
+
+int __cvdev_dq_busy_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	*sc_vb = list_first_entry_or_null(&sc_vnode->busy_list, struct ccic_vbuffer, list_entry);
+	if (NULL == *sc_vb)
+		return -1;
+	list_del_init(&(*sc_vb)->list_entry);
+	atomic_dec(&sc_vnode->busy_buf_cnt);
+	return 0;
+}
+
+int __cvdev_q_busy_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer *sc_vb)
+{
+	list_add_tail(&sc_vb->list_entry, &sc_vnode->busy_list);
+	atomic_inc(&sc_vnode->busy_buf_cnt);
+	return 0;
+}
+
+int cvdev_dq_busy_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	unsigned long flags = 0;
+	int ret = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	ret = __cvdev_dq_busy_vbuffer(sc_vnode, sc_vb);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+	return ret;
+}
+
+int cvdev_pick_busy_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	unsigned long flags = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	*sc_vb = list_first_entry_or_null(&sc_vnode->busy_list, struct ccic_vbuffer, list_entry);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+	if (NULL == *sc_vb)
+		return -1;
+
+	return 0;
+}
+
+int __cvdev_pick_busy_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer **sc_vb)
+{
+	*sc_vb = list_first_entry_or_null(&sc_vnode->busy_list, struct ccic_vbuffer, list_entry);
+	if (NULL == *sc_vb)
+		return -1;
+
+	return 0;
+}
+
+int cvdev_q_busy_vbuffer(struct ccic_vnode *sc_vnode, struct ccic_vbuffer *sc_vb)
+{
+	unsigned long flags = 0;
+	int ret = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	ret = __cvdev_q_busy_vbuffer(sc_vnode, sc_vb);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+	return ret;
+}
+
+int cvdev_export_ccic_vbuffer(struct ccic_vbuffer *sc_vb, int with_error)
+{
+	struct vb2_buffer *vb = &sc_vb->vb2_v4l2_buf.vb2_buf;
+	if (with_error)
+		vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
+	else
+		vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
+	return 0;
+}
+
+int __cvdev_busy_list_empty(struct ccic_vnode *sc_vnode)
+{
+	return list_empty(&sc_vnode->busy_list);
+}
+
+int cvdev_busy_list_empty(struct ccic_vnode *sc_vnode)
+{
+	unsigned long flags = 0;
+	int ret = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	ret = __cvdev_busy_list_empty(sc_vnode);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+	return ret;
+}
+
+int __cvdev_idle_list_empty(struct ccic_vnode *sc_vnode)
+{
+	return list_empty(&sc_vnode->queued_list);
+}
+
+int cvdev_idle_list_empty(struct ccic_vnode *sc_vnode)
+{
+	unsigned long flags = 0;
+	int ret = 0;
+
+	spin_lock_irqsave(&sc_vnode->slock, flags);
+	ret = __cvdev_idle_list_empty(sc_vnode);
+	spin_unlock_irqrestore(&sc_vnode->slock, flags);
+	return ret;
+}
diff --git a/drivers/media/platform/spacemit/cam_ccic/ccic_vdev.h b/drivers/media/platform/spacemit/cam_ccic/ccic_vdev.h
new file mode 100644
index 000000000000..a5fd9ee12e5f
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/ccic_vdev.h
@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ccic_vdev.h - video device functions
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef _CCIC_VDEV_H_
+#define _CCIC_VDEV_H_
+#include <media/v4l2-dev.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-v4l2.h>
+#include <linux/notifier.h>
+#include "ccic_drv.h"
+#include "ccic_dma.h"
+
+#define CCIC_DMA_WORK_MAX_CNT (16)
+
+struct ccic_vbuffer;
+struct ccic_vnode;
+
+struct ccic_dma_context {
+	struct list_head dma_work_idle_list;
+	struct list_head dma_work_busy_list;
+	spinlock_t slock;
+	struct ccic_vnode *sc_vnode;
+	unsigned int dma_ch;
+};
+
+struct ccic_vnode {
+	struct video_device vnode;
+	char name[32];
+	struct vb2_queue buf_queue;
+	struct list_head queued_list;
+	struct list_head busy_list;
+	struct ccic_dma_context dma_ctx;
+	atomic_t queued_buf_cnt;
+	atomic_t busy_buf_cnt;
+	atomic_t ref_cnt;
+	spinlock_t slock;
+	unsigned char wait_done_flush;
+	struct completion flush_complete;
+	struct mutex mlock;
+	struct v4l2_format cur_fmt;
+	struct wait_queue_head waitq_head;
+	int in_streamoff;
+	int in_tasklet;
+	int in_irq;
+	int is_streaming;
+	unsigned int idx;
+	unsigned int total_frm;
+	unsigned int sw_err_frm;
+	unsigned int hw_err_frm;
+	unsigned int ok_frm;
+	unsigned int planes_offset[VB2_MAX_FRAME][VB2_MAX_PLANES];
+	unsigned int v4l2_buf_flags[VB2_MAX_FRAME];
+	struct ccic_dev *ccic_dev;
+	int csi2vc;
+	int src_sel;
+	int lane_num;
+	int mipi_m_bps;
+	int csi_src; /*csi1 can select dphy0 or dphy1, csi2 can select dphy2 or dphy3 (only for larkM) */
+	int ccic_mode;
+	int ch_mode;
+	unsigned int main_vc;
+	unsigned int sub_vc;
+	uint64_t frame_id;
+	void *usr_data;
+};
+
+struct ccic_dma_work_struct {
+	struct tasklet_struct dma_tasklet;
+	struct list_head idle_list_entry;
+	struct list_head busy_list_entry;
+	unsigned int irq_status;
+	struct ccic_vnode *sc_vnode;
+};
+
+#define BUF_FLAG_SOF_TOUCH (1 << 0)
+#define BUF_FLAG_DONE_TOUCH (1 << 1)
+#define BUF_FLAG_HW_ERR (1 << 2)
+#define BUF_FLAG_SW_ERR (1 << 3)
+#define BUF_FLAG_TIMESTAMPED (1 << 4)
+#define BUF_FLAG_CCIC_TOUCH (1 << 5)
+
+#define BUF_RESERVED_DATA_LEN (32)
+struct ccic_vbuffer {
+	struct vb2_v4l2_buffer vb2_v4l2_buf;
+	struct list_head list_entry;
+	unsigned int reset_flag;
+	unsigned int flags;
+	struct ccic_vnode *sc_vnode;
+	unsigned char reserved[BUF_RESERVED_DATA_LEN];
+};
+
+#define vb2_buffer_to_ccic_vbuffer(vb) ((struct ccic_vbuffer *)(vb))
+
+#define CAM_ALIGN(a, b)                                   \
+	({                                                \
+		unsigned int ___tmp1 = (a);               \
+		unsigned int ___tmp2 = (b);               \
+		unsigned int ___tmp3 = ___tmp1 % ___tmp2; \
+		___tmp1 /= ___tmp2;                       \
+		if (___tmp3)                              \
+			___tmp1++;                        \
+		___tmp1 *= ___tmp2;                       \
+		___tmp1;                                  \
+	})
+
+#define is_vnode_streaming(vnode) ((vnode)->buf_queue.streaming)
+
+#ifdef CONFIG_SPACEMIT_K3_CCIC_IOMMU
+dma_addr_t vb2_buf_paddr(struct vb2_buffer *vb, unsigned int plane_no);
+#else
+static inline dma_addr_t vb2_buf_paddr(struct vb2_buffer *vb,
+				       unsigned int plane_no)
+{
+	unsigned int offset = 0;
+	dma_addr_t paddr = 0;
+	struct ccic_vbuffer *sc_vb = vb2_buffer_to_ccic_vbuffer(vb);
+	struct ccic_vnode *sc_vnode = sc_vb->sc_vnode;
+	dma_addr_t *dma_addr = (dma_addr_t *)vb2_plane_cookie(vb, plane_no);
+
+	BUG_ON(!sc_vnode);
+	offset = sc_vnode->planes_offset[vb->index][plane_no];
+	paddr = *dma_addr + offset;
+	return paddr;
+}
+#endif
+
+static inline void ccic_update_dma_addr(struct ccic_vnode *sc_vnode,
+					struct ccic_vbuffer *sc_vbuf,
+					unsigned int offset)
+{
+	dma_addr_t p0 = 0;
+	struct ccic_dma *ccic_dma = get_ccic_dma();
+	struct vb2_buffer *vb2_buf = &(sc_vbuf->vb2_v4l2_buf.vb2_buf);
+
+	p0 = vb2_buf_paddr(vb2_buf, 0) + offset;
+	ccic_dma_ch_set_addr(ccic_dma, sc_vnode->dma_ctx.dma_ch, p0);
+}
+
+static inline void *sc_vnode_get_usrdata(struct ccic_vnode *sc_vnode)
+{
+	return sc_vnode->usr_data;
+}
+
+static inline struct ccic_vbuffer *to_ccic_vbuffer(struct vb2_buffer *vb2)
+{
+	struct vb2_v4l2_buffer *vb2_v4l2_buf = to_vb2_v4l2_buffer(vb2);
+	return container_of(vb2_v4l2_buf, struct ccic_vbuffer, vb2_v4l2_buf);
+}
+
+struct ccic_vnode *
+cvdev_create_vnode(const char *name, unsigned int idx,
+		   struct v4l2_device *v4l2_dev, struct device *alloc_dev,
+		   struct ccic_dev *ccic_dev,
+		   void (*dma_tasklet_handler)(unsigned long),
+		   unsigned int min_buffers_needed);
+void cvdev_destroy_vnode(struct ccic_vnode *sc_vnode);
+
+int cvdev_busy_list_empty(struct ccic_vnode *sc_vnode);
+int __cvdev_busy_list_empty(struct ccic_vnode *sc_vnode);
+int cvdev_idle_list_empty(struct ccic_vnode *sc_vnode);
+int __cvdev_idle_list_empty(struct ccic_vnode *sc_vnode);
+int cvdev_dq_idle_vbuffer(struct ccic_vnode *sc_vnode,
+			  struct ccic_vbuffer **sc_vb);
+int cvdev_pick_idle_vbuffer(struct ccic_vnode *sc_vnode,
+			    struct ccic_vbuffer **sc_vb);
+int __cvdev_pick_idle_vbuffer(struct ccic_vnode *sc_vnode,
+			      struct ccic_vbuffer **sc_vb);
+int cvdev_q_idle_vbuffer(struct ccic_vnode *sc_vnode,
+			 struct ccic_vbuffer *sc_vb);
+int __cvdev_dq_idle_vbuffer(struct ccic_vnode *sc_vnode,
+			    struct ccic_vbuffer **sc_vb);
+int __cvdev_q_idle_vbuffer(struct ccic_vnode *sc_vnode,
+			   struct ccic_vbuffer *sc_vb);
+int cvdev_dq_busy_vbuffer(struct ccic_vnode *sc_vnode,
+			  struct ccic_vbuffer **sc_vb);
+int cvdev_pick_busy_vbuffer(struct ccic_vnode *sc_vnode,
+			    struct ccic_vbuffer **sc_vb);
+int __cvdev_pick_busy_vbuffer(struct ccic_vnode *sc_vnode,
+			      struct ccic_vbuffer **sc_vb);
+int cvdev_q_busy_vbuffer(struct ccic_vnode *sc_vnode,
+			 struct ccic_vbuffer *sc_vb);
+int __cvdev_dq_busy_vbuffer(struct ccic_vnode *sc_vnode,
+			    struct ccic_vbuffer **sc_vb);
+int __cvdev_q_busy_vbuffer(struct ccic_vnode *sc_vnode,
+			   struct ccic_vbuffer *sc_vb);
+int cvdev_export_ccic_vbuffer(struct ccic_vbuffer *sc_vb, int with_error);
+void cvdev_fill_v4l2_format(struct v4l2_format *f);
+#endif
diff --git a/drivers/media/platform/spacemit/cam_ccic/csiphy.c b/drivers/media/platform/spacemit/cam_ccic/csiphy.c
new file mode 100644
index 000000000000..89b34931b0a3
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/csiphy.c
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Spacemit CCIC MIPI D-PHY MODULE
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/printk.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/atomic.h>
+#include <linux/regmap.h>
+#include <linux/bitfield.h>
+#include <linux/mfd/syscon.h>
+#include "ccic_drv.h"
+#include "ccic_hwreg.h"
+#include "csiphy.h"
+
+#define DEBUG 0 /* for pr_debug() */
+
+#define CSIPHY_DRV_NAME "spacemit-csi-phy"
+
+#define CAMERA_ENABLE_BIT BIT(1)
+
+struct csiphy_device {
+	struct list_head list;
+	struct platform_device *pdev;
+	struct regmap *apmu;
+	u32 ctrl_off;
+	struct device *dev;
+	struct resource *mem;
+	void __iomem *base;
+	struct clk *csiphy_clk;
+	struct reset_control *csi_dphy_reset;
+	atomic_t usecnt;
+	int is_bifmode;
+};
+
+/*
+ * Device register I/O
+ */
+static inline u32 csiphy_readl(struct csiphy_device *csiphy, unsigned int reg)
+{
+	return readl(csiphy->base + reg);
+}
+
+static inline void csiphy_writel(struct csiphy_device *csiphy, unsigned int reg,
+				 u32 val)
+{
+	writel(val, csiphy->base + reg);
+}
+
+static inline void csiphy_mask_writel(struct csiphy_device *csiphy,
+				      unsigned int reg, u32 val, u32 mask)
+{
+	u32 v = csiphy_readl(csiphy, reg);
+
+	v = (v & ~mask) | (val & mask);
+	csiphy_writel(csiphy, reg, v);
+}
+
+static inline void csiphy_set_bit(struct csiphy_device *csiphy,
+				  unsigned int reg, u32 val)
+{
+	csiphy_mask_writel(csiphy, reg, val, val);
+}
+
+static inline void csiphy_clear_bit(struct csiphy_device *csiphy,
+				    unsigned int reg, u32 val)
+{
+	csiphy_mask_writel(csiphy, reg, 0, val);
+}
+
+static int csiphy_camera_sys_enable(struct csiphy_device *csiphy_dev,
+				    bool enable)
+{
+	unsigned int val;
+	int ret;
+
+	val = enable ? 1 : 0;
+
+	ret = regmap_update_bits(csiphy_dev->apmu, csiphy_dev->ctrl_off,
+				 CAMERA_ENABLE_BIT, val << 1);
+	if (ret < 0) {
+		dev_err(csiphy_dev->dev, "failed to update camera sys enable, ret=%d\n", ret);
+	} else {
+		dev_info(csiphy_dev->dev, "camera sys enable success:%d\n", val);
+	}
+	return ret;
+}
+
+/**
+ * csiphy_set_power - Power on/off csiphy module
+ *
+ * @csiphy_dev: csiphy device
+ * @on: requested power state
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+static int csiphy_set_power(struct csiphy_device *csiphy_dev, int on)
+{
+	int ret;
+	unsigned long clk_val;
+
+	if (on) {
+		clk_val = clk_round_rate(csiphy_dev->csiphy_clk, 104000000);
+		clk_set_rate(csiphy_dev->csiphy_clk, clk_val);
+		clk_val = clk_get_rate(csiphy_dev->csiphy_clk);
+		dev_info(csiphy_dev->dev, "csiphy clk:%ld\n", clk_val);
+		ret = clk_prepare_enable(csiphy_dev->csiphy_clk);
+		if (ret < 0) {
+			dev_err(csiphy_dev->dev,
+				"enable dphy clk failed, err=%d\n", ret);
+			return ret;
+		}
+		ret = reset_control_deassert(csiphy_dev->csi_dphy_reset);
+		if (ret < 0) {
+			dev_err(csiphy_dev->dev,
+				"deassert dphy reset failed, err=%d\n", ret);
+			return ret;
+		}
+		ret = csiphy_camera_sys_enable(csiphy_dev, 1);
+		if (ret < 0) {
+			dev_err(csiphy_dev->dev,
+				"enable camera sys failed, err=%d\n", ret);
+			return ret;
+		}
+	} else {
+		clk_disable_unprepare(csiphy_dev->csiphy_clk);
+		reset_control_assert(csiphy_dev->csi_dphy_reset);
+		csiphy_camera_sys_enable(csiphy_dev, 0);
+	}
+
+	return 0;
+}
+
+/**
+ * csiphy_set_2to2dphy
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+static int csiphy_set_2to2dphy(struct csiphy_device *csiphy_dev, int enable)
+{
+	if (enable) {
+		/* REG_CSI2_DPHY1[1]: analog bif mode on */
+		csiphy_set_bit(csiphy_dev, REG_CSI2_DPHY1, CSI2_DHPY1_BIF_EN);
+		/* ccic3:ccic_csi2_dphy9[23]: dphy3 2+2 lane mux */
+		csiphy_set_bit(csiphy_dev, REG_CSI2_DPHY9, CSI2_DHPY9_EN_BIF2);
+	} else {
+		csiphy_clear_bit(csiphy_dev, REG_CSI2_DPHY1, CSI2_DHPY1_BIF_EN);
+		csiphy_clear_bit(csiphy_dev, REG_CSI2_DPHY9, CSI2_DHPY9_EN_BIF2);
+	}
+
+	return 0;
+}
+
+static struct csi_dphy_calc dphy_calc_profiles[] = {
+	{
+		.hs_termen_pos = 0,
+		.hs_settle_pos = 50,
+	},
+};
+
+static int ccic_calc_dphy(struct csi_dphy_desc *desc,
+			  struct csi_dphy_calc *algo, struct csi_dphy_reg *reg)
+{
+	u32 ps_period, ps_ui, ps_termen_max, ps_prep_max, ps_prep_min;
+	u32 ps_sot_min, ps_termen, ps_settle;
+
+	ps_period = MHZ * 1000 / (desc->clk_freq / 1000);
+	ps_ui = ps_period / 2;
+	ps_termen_max = NS_TO_PS(D_TERMEN_MAX) + 4 * ps_ui;
+	ps_prep_min = NS_TO_PS(HS_PREP_MIN) + 4 * ps_ui;
+	ps_prep_max = NS_TO_PS(HS_PREP_MAX) + 6 * ps_ui;
+	ps_sot_min = NS_TO_PS(HS_PREP_ZERO_MIN) + 10 * ps_ui;
+	ps_termen = ps_termen_max + algo->hs_termen_pos * ps_period;
+	ps_settle = NS_TO_PS(desc->hs_prepare + desc->hs_zero *
+							algo->hs_settle_pos /
+							HS_SETTLE_POS_MAX);
+
+	reg->cl_termen = 0x01;
+	reg->cl_settle = 0x10;
+	reg->cl_miss = 0x00;
+	/* term_en = round_up(ps_termen / ps_period) - 1 */
+	reg->hs_termen = (ps_termen + ps_period - 1) / ps_period - 1;
+	/* For Marvell DPHY, Ths-settle started from HS-0, not VILmax */
+	ps_settle -= (reg->hs_termen + 1) * ps_period;
+	/* DE recommend this value reset to zero */
+	reg->hs_termen = 0x0;
+	/* round_up(ps_settle / ps_period) - 1 */
+	reg->hs_settle = (ps_settle + ps_period - 1) / ps_period - 1;
+	reg->hs_rx_to = 0xFFFF;
+	reg->lane = desc->nr_lane;
+	return 0;
+}
+
+int csiphy_stop(struct csiphy_device *csiphy_dev)
+{
+	if (csiphy_dev->is_bifmode)
+		csiphy_set_2to2dphy(csiphy_dev, 0);
+
+	csiphy_clear_bit(csiphy_dev, REG_CSI2_CTRL0, CSI2_C0_ENABLE);
+	csiphy_writel(csiphy_dev, REG_CSI2_DPHY5, 0x00);
+	csiphy_clear_bit(csiphy_dev, REG_CSI2_DPHY1,
+			 CSI2_DHPY1_ANA_PU); /* analog power off */
+
+	if (atomic_dec_if_positive(&csiphy_dev->usecnt) == 0)
+		csiphy_set_power(csiphy_dev, 0);
+
+	return 0;
+}
+
+int csiphy_start(struct csiphy_device *csiphy_dev, struct mipi_csi2 *csi)
+{
+	unsigned int dphy2_val = 0;
+	unsigned int dphy3_val = 0;
+	unsigned int dphy5_val = 0;
+	unsigned int dphy6_val = 0;
+	unsigned int ctrl0_val = 0;
+	struct csi_dphy_reg dphy_reg;
+	int lanes = csi->dphy_desc.nr_lane;
+
+	if (lanes < 1 || lanes > (csiphy_dev->is_bifmode ? 2 : 4)) {
+		dev_err(csiphy_dev->dev, "wrong lanes num %d\n", lanes);
+		return -EINVAL;
+	}
+
+	if (atomic_inc_return(&csiphy_dev->usecnt) == 1)
+		csiphy_set_power(csiphy_dev, 1);
+
+	if (csi->calc_dphy) {
+		ccic_calc_dphy(&csi->dphy_desc, dphy_calc_profiles, &dphy_reg);
+		dphy3_val = dphy_reg.hs_settle & 0xFF;
+		dphy3_val = dphy_reg.hs_termen |
+			    (dphy3_val << CSI2_DPHY3_HS_SETTLE_SHIFT);
+		dphy6_val = dphy_reg.cl_settle & 0xFF;
+		dphy6_val = dphy_reg.cl_termen |
+			    (dphy6_val << CSI2_DPHY6_CK_SETTLE_SHIFT);
+	} else {
+		dphy2_val = csi->dphy[1];
+		dphy3_val = csi->dphy[2];
+		dphy6_val = csi->dphy[4];
+	}
+
+	dphy5_val = CSI2_DPHY5_LANE_ENA(lanes);
+	dphy5_val = dphy5_val | (dphy5_val << CSI2_DPHY5_LANE_RESC_ENA_SHIFT);
+
+	ctrl0_val = csiphy_readl(csiphy_dev, REG_CSI2_CTRL0);
+	ctrl0_val &= ~(CSI2_C0_LANE_NUM_MASK);
+	ctrl0_val |= CSI2_C0_LANE_NUM(lanes);
+	ctrl0_val |= CSI2_C0_ENABLE;
+	ctrl0_val &= ~(CSI2_C0_VLEN_MASK);
+	ctrl0_val |= CSI2_C0_VLEN;
+
+	/* CSI2_DPHY1 must set bit,otherwise may cover someone else's config. */
+	csiphy_set_bit(csiphy_dev, REG_CSI2_DPHY1,
+		       CSI2_DHPY1_ANA_PU); /* analog power on */
+	csiphy_writel(csiphy_dev, REG_CSI2_DPHY2, dphy2_val);
+	if (!csiphy_readl(csiphy_dev, REG_CSI2_DPHY3))
+		csiphy_writel(csiphy_dev, REG_CSI2_DPHY3, dphy3_val);
+	csiphy_writel(csiphy_dev, REG_CSI2_DPHY5, dphy5_val);
+	csiphy_writel(csiphy_dev, REG_CSI2_DPHY6, dphy6_val);
+	csiphy_writel(csiphy_dev, REG_CSI2_CTRL0, ctrl0_val);
+
+	/* for lark */
+	csiphy_writel(csiphy_dev, REG_CSI2_DPHY4, 0x0013a35a);
+	csiphy_writel(csiphy_dev, REG_CSI2_DPHY9, 0x1a280090);
+
+	if (csiphy_dev->is_bifmode)
+		csiphy_set_2to2dphy(csiphy_dev, 1);
+
+	return 0;
+}
+
+void csiphy_timming_setting(struct csiphy_device *csiphy_dev, unsigned int settle)
+{
+	csiphy_writel(csiphy_dev, REG_CSI2_DPHY3, settle);
+}
+
+static DEFINE_MUTEX(csiphy_list_mutex);
+static LIST_HEAD(csiphy_list);
+struct csiphy_device *csiphy_lookup_by_phandle(struct device *dev,
+					       const char *name)
+{
+	struct device_node *csiphy_node =
+		of_parse_phandle(dev->of_node, name, 0);
+	struct csiphy_device *csiphy_dev;
+
+	mutex_lock(&csiphy_list_mutex);
+	list_for_each_entry(csiphy_dev, &csiphy_list, list) {
+		if (csiphy_node == csiphy_dev->dev->of_node) {
+			mutex_unlock(&csiphy_list_mutex);
+			of_node_put(csiphy_node);
+			return csiphy_dev;
+		}
+	}
+	mutex_unlock(&csiphy_list_mutex);
+
+	of_node_put(csiphy_node);
+	return NULL;
+}
+
+static int csiphy_probe(struct platform_device *pdev)
+{
+	struct csiphy_device *csiphy_dev;
+	struct device_node *np = pdev->dev.of_node;
+	int ret;
+
+	csiphy_dev = devm_kzalloc(&pdev->dev, sizeof(struct csiphy_device),
+				  GFP_KERNEL);
+	if (!csiphy_dev) {
+		dev_err(&pdev->dev, "no enough memory\n");
+		return -ENOMEM;
+	}
+
+	ret = of_property_read_u32(np, "cell-index", &pdev->id);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to get alias id, errno %d\n", ret);
+		return ret;
+	}
+
+	/* get mem */
+	csiphy_dev->mem = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						       "csiphy-regs");
+	if (!csiphy_dev->mem) {
+		dev_err(&pdev->dev, "no mem resource\n");
+		return -ENODEV;
+	}
+
+	csiphy_dev->base = devm_ioremap(&pdev->dev, csiphy_dev->mem->start,
+					resource_size(csiphy_dev->mem));
+	if (IS_ERR(csiphy_dev->base)) {
+		dev_err(&pdev->dev, "fail to remap iomem\n");
+		return PTR_ERR(csiphy_dev->base);
+	}
+
+	/* get clock(s) */
+	csiphy_dev->csi_dphy_reset =
+		devm_reset_control_get(&pdev->dev, "csi_dphy_reset");
+	if (IS_ERR(csiphy_dev->csi_dphy_reset)) {
+		ret = PTR_ERR(csiphy_dev->csi_dphy_reset);
+		dev_err(&pdev->dev, "failed to get csi_dphy reset: %d\n", ret);
+		return ret;
+	}
+	csiphy_dev->csiphy_clk = devm_clk_get(&pdev->dev, "csi_dphy");
+	if (IS_ERR(csiphy_dev->csiphy_clk)) {
+		ret = PTR_ERR(csiphy_dev->csiphy_clk);
+		dev_err(&pdev->dev, "failed to get csiphy clock: %d\n", ret);
+		return ret;
+	}
+
+	csiphy_dev->apmu = syscon_regmap_lookup_by_phandle(np, "spacemit,apmu");
+	if (IS_ERR(csiphy_dev->apmu)) {
+		ret = PTR_ERR(csiphy_dev->apmu);
+		dev_err(&pdev->dev, "spacemit,apmu lookup failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = of_property_read_u32(np, "spacemit,ctrl-offset",
+				   &csiphy_dev->ctrl_off);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to get ctrl-offset, errno %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = of_property_read_s32(np, "spacemit,bifmode-enable",
+				   &csiphy_dev->is_bifmode);
+	if (ret < 0) {
+		dev_warn(&pdev->dev, "failed to get bifmode, ret=%d\n", ret);
+		csiphy_dev->is_bifmode = 0;
+	}
+
+	atomic_set(&csiphy_dev->usecnt, 0);
+	csiphy_dev->pdev = pdev;
+	csiphy_dev->dev = &pdev->dev;
+	platform_set_drvdata(pdev, csiphy_dev);
+
+	mutex_lock(&csiphy_list_mutex);
+	list_add(&csiphy_dev->list, &csiphy_list);
+	mutex_unlock(&csiphy_list_mutex);
+
+	pr_info("%s probed", dev_name(&pdev->dev));
+
+	return ret;
+}
+
+static void csiphy_remove(struct platform_device *pdev)
+{
+	struct csiphy_device *csiphy_dev;
+
+	csiphy_dev = platform_get_drvdata(pdev);
+	if (!csiphy_dev) {
+		dev_err(&pdev->dev, "csiphy device is NULL\n");
+		return;
+	}
+
+	devm_kfree(&pdev->dev, csiphy_dev);
+	pr_info("%s removed", dev_name(&pdev->dev));
+
+	return;
+}
+
+static const struct of_device_id csiphy_dt_match[] = {
+	{
+		.compatible = "spacemit,csi-dphy",
+		.data = NULL
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, csiphy_dt_match);
+
+struct platform_driver csiphy_driver = {
+	.driver = {
+		.name = CSIPHY_DRV_NAME,
+		.of_match_table = of_match_ptr(csiphy_dt_match),
+	},
+	.probe = csiphy_probe,
+	.remove = csiphy_remove,
+};
+
+int ccic_csiphy_register(void)
+{
+	return platform_driver_register(&csiphy_driver);
+}
+
+void ccic_csiphy_unregister(void)
+{
+	platform_driver_unregister(&csiphy_driver);
+}
+/* module_platform_driver(csiphy_driver); */
+
+MODULE_DESCRIPTION("SPACEMIT CSI-PHY Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/spacemit/cam_ccic/csiphy.h b/drivers/media/platform/spacemit/cam_ccic/csiphy.h
new file mode 100644
index 000000000000..3429fa400390
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/csiphy.h
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for SPACEMIT CCIC MIPI D-PHY MODULE
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#ifndef _CSIPHY_H_
+#define _CSIPHY_H_
+
+#include "ccic_drv.h"
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/list.h>
+#include <linux/reset.h>
+
+struct csiphy_device;
+struct mipi_csi2;
+
+struct csiphy_device *csiphy_lookup_by_phandle(struct device *dev,
+					       const char *name);
+int csiphy_stop(struct csiphy_device *csiphy_dev);
+int csiphy_start(struct csiphy_device *csiphy_dev,
+			    struct mipi_csi2 *csi);
+
+void csiphy_timming_setting(struct csiphy_device *csiphy_dev, unsigned int settle);
+
+int ccic_csiphy_register(void);
+void ccic_csiphy_unregister(void);
+#endif /* ifndef __CSIPHY_H__ */
diff --git a/drivers/media/platform/spacemit/cam_ccic/dptc_drv.c b/drivers/media/platform/spacemit/cam_ccic/dptc_drv.c
new file mode 100644
index 000000000000..e594b791c7f0
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/dptc_drv.c
@@ -0,0 +1,756 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * dptc_drv.c - Driver for dptc
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifdef CONFIG_ARCH_ZYNQMP
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of_device.h>
+#include <linux/i2c.h>
+#include <linux/delay.h>
+
+#include "dptc_drv.h"
+#include "dptc_pll_setting.h"
+
+#define DPTCDELAY_US(us) usleep_range(us, us)
+#define DPTCDELAY_MS(ms) DPTCDELAY_US(ms * 1000)
+
+u8 g_current_function = DPTC_FUNC_LIMIT;
+
+static int dptc_ports[2] = {DPTC_I2C_PORT, DPTC_I2C_PORT_1};
+
+struct s_pll_setting DP_pll_configs[PLL_SSC_LIMIT][PLL_RATE_LIMIT] = {
+	{ /*PLL_SSC_0*/
+		{0x27, 0x76, 0x62, 0x3E, 0x20, 0x44, 0x85, 0x42, 0x0}, /*PLL_RATE_1620*/
+		{0x76, 0x62, 0x7F, 0x34, 0x20, 0x4B, 0x95, 0x42, 0x0}, /*PLL_RATE_2700*/
+		{0x9E, 0xD8, 0x61, 0x45, 0x20, 0x5B, 0xA5, 0x42, 0x0}, /*PLL_RATE_5400*/
+	},
+	{ /*PLL_SSC_5000*/
+		{0x27, 0x76, 0x62, 0x3E, 0x2A, 0x44, 0x85, 0x42, 0x0}, /*PLL_RATE_1620*/
+		{0x76, 0x62, 0x7F, 0x34, 0x2A, 0x4B, 0x95, 0x42, 0x0}, /*PLL_RATE_2700*/
+		{0x9E, 0xD8, 0x61, 0x45, 0x2A, 0x5B, 0xA5, 0x42, 0x0}, /*PLL_RATE_5400*/
+	}
+};
+
+typedef enum {
+	STANDARD_MODE = 0, /*100Kbps*/
+	FAST_MODE, /*400Kbps*/
+	HS_MODE, /*3.4 Mbps slave/3.3 Mbps master,standard mode when not doing a high speed transfer*/
+	HS_MODE_FAST, /*3.4 Mbps slave/3.3 Mbps master,fast mode when not doing a high speed transfer*/
+} I2C_FAST_MODE;
+
+static struct twsi_data dptc_i2c_data[2];
+static DEFINE_MUTEX(cmd_mutex);
+
+u32 swap_n(u32 addr, unsigned int cmdlen)
+{
+	u32 res;
+
+	switch (cmdlen) {
+	case 1:
+		res = addr;
+		break;
+	case 2:
+		res = (addr & 0xff) << 8 | ((addr >> 8) & 0xff);
+		break;
+	case 3:
+		res = (addr & 0xff) << 16 | (addr & 0xff00) |
+		      ((addr >> 16) & 0xff);
+		break;
+	case 4:
+		res = (addr & 0xff) << 24 | ((addr & 0xff00) << 8) |
+		      ((addr & 0xff0000) >> 8) | ((addr >> 24) & 0xff);
+		break;
+	default:
+		res = addr;
+		break;
+	}
+
+	return res;
+}
+
+int twsi_write_i2c(struct twsi_data *data)
+{
+	int ret = 0;
+	struct i2c_adapter *adapter;
+	struct i2c_msg msg;
+	u8 val[8];
+	int i, j = 0;
+
+	if (!data || !data->addr || !data->reg_len || !data->val_len) {
+		pr_err("Error: %s, %d", __func__, __LINE__);
+		return -EINVAL;
+	}
+
+	msg.addr = data->addr;
+	msg.flags = 0;
+	msg.len = data->reg_len + data->val_len;
+	msg.buf = val;
+
+	adapter = i2c_get_adapter(data->twsi_no);
+	if (!adapter)
+		return -1;
+
+	mutex_lock(&cmd_mutex);
+	for (i = 0; i < data->reg_len; i++)
+		val[j++] = ((u8 *)(&data->reg))[i];
+	for (i = 0; i < data->val_len; i++)
+		val[j++] = ((u8 *)(&data->val))[i];
+	ret = i2c_transfer(adapter, &msg, 1);
+	if (ret < 0) {
+		mutex_unlock(&cmd_mutex);
+		return ret;
+	}
+	mutex_unlock(&cmd_mutex);
+
+	return ret;
+}
+
+int twsi_read_i2c(struct twsi_data *data)
+{
+	struct i2c_adapter *adapter;
+	struct i2c_msg msg;
+	int ret = 0;
+	u8 val[4];
+
+	if (!data || !data->addr || !data->reg_len || !data->val_len) {
+		pr_err("%s, error param", __func__);
+		return -EINVAL;
+	}
+
+	msg.addr = data->addr;
+	msg.flags = 0;
+	msg.len = data->reg_len;
+	msg.buf = val;
+
+	adapter = i2c_get_adapter(data->twsi_no);
+	if (!adapter) {
+		return -1;
+	}
+
+	mutex_lock(&cmd_mutex);
+	if (data->reg_len == I2C_8BIT) {
+		val[0] = data->reg & 0xff;
+	} else if (data->reg_len == I2C_16BIT) {
+		val[0] = (data->reg >> 8) & 0xff;
+		val[1] = data->reg & 0xff;
+	}
+	msg.len = data->reg_len;
+	ret = i2c_transfer(adapter, &msg, 1);
+	if (ret < 0) {
+		mutex_unlock(&cmd_mutex);
+		goto err;
+	}
+
+	msg.flags = I2C_M_RD;
+	msg.len = data->val_len;
+	ret = i2c_transfer(adapter, &msg, 1);
+	if (ret < 0) {
+		mutex_unlock(&cmd_mutex);
+		goto err;
+	}
+
+	if (data->val_len == I2C_8BIT)
+		data->val = val[0];
+	else if (data->val_len == I2C_16BIT)
+		data->val = (val[0] << 8) + val[1];
+	else if (data->val_len == I2C_32BIT)
+		data->val = (val[3] << 24) + (val[2] << 16) + (val[1] << 8) + val[0];
+	/* pr_info("twsi_read_i2c: val[0]=0x%x,val[1]=0x%x,val[2]=0x%x,val[3]=0x%x\n",val[0],val[1],val[2],val[3]); */
+	mutex_unlock(&cmd_mutex);
+
+	return 0;
+
+err:
+	pr_info("Failed reading register 0x%02x!", data->reg);
+	return ret;
+}
+
+void TWSI_Init(I2C_FAST_MODE mode, unsigned int id)
+{
+	dptc_i2c_data[id].twsi_no = dptc_ports[id];
+	dptc_i2c_data[id].addr = 0x6c;
+	dptc_i2c_data[id].reg_len = 1; /* I2C_8BIT; */
+	dptc_i2c_data[id].val_len = 4; /* I2C_32BIT; */
+}
+
+int TWSI_REG_READ_DPTC(u32 id, u8 slaveaddress, u8 addr)
+{
+	int ret = 0;
+
+	dptc_i2c_data[id].twsi_no = dptc_ports[id];
+	dptc_i2c_data[id].addr = slaveaddress;
+	dptc_i2c_data[id].reg = addr;
+	dptc_i2c_data[id].val = 0x00;
+	ret = twsi_read_i2c(&dptc_i2c_data[id]);
+	if (ret == 0)
+		return dptc_i2c_data[id].val;
+	else
+		return -1;
+}
+
+int TWSI_REG_WRITE_DPTC(u32 id, u8 slaveaddress, u8 addr, u32 data)
+{
+	int ret = 0;
+
+	dptc_i2c_data[id].twsi_no = dptc_ports[id];
+	dptc_i2c_data[id].addr = slaveaddress;
+	dptc_i2c_data[id].reg = addr;
+	dptc_i2c_data[id].val = data;
+	ret = twsi_write_i2c(&dptc_i2c_data[id]);
+	if (ret == 0)
+		return 1;
+	else
+		return -1;
+}
+
+int TWSI_REG_WRITE_CAM(u32 id, u8 i2c_no, u8 slaveaddress, u16 addr, u8 data)
+{
+#if 0
+	u32 twsi_param = 0x21;	/*2 byte address, 1 byte data*/
+	int ret = 0;
+
+	ret = TWSIput(i2c_no, slaveaddress, addr, data, &twsi_param);
+	if (ret == 0)
+		return 1;
+	else
+		return -1;
+#else
+	return -1;
+#endif
+}
+
+int TWSI_REG_READ_CAM(u32 id, u8 i2c_no, u8 slaveaddress, u16 addr)
+{
+#if 0
+	u32 twsi_param = 0x21; /*2 byte address, 1 byte data*/
+	int ret = 0;
+
+	ret = TWSIget(i2c_no, slaveaddress, addr, &twsi_param);
+	if (ret == 0)
+		return twsi_param;
+	else
+		return -1;
+#else
+	return -1;
+#endif
+}
+
+static u32 top_read(u32 id, u8 reg)
+{
+	return	TWSI_REG_READ_DPTC(id, TOP_SLAVE_ADDR, reg >> 2);
+}
+
+static void top_write(u32 id, u8 reg, u32 data)
+{
+	/* pr_info("DPTC:top_write: 0x%x = 0x%x\r\n", reg, data); */
+	TWSI_REG_WRITE_DPTC(id, TOP_SLAVE_ADDR, reg >> 2, data);
+}
+
+static void top_set_bits(u32 id, u8 reg, u32 bits)
+{
+	top_write(id, reg, top_read(id, reg) | bits);
+}
+
+static void top_clear_bits(u32 id, u8 reg, u32 bits)
+{
+	top_write(id, reg, top_read(id, reg) & ~bits);
+}
+
+static void top_write_bits(u32 id, u8 reg, u32 value, u8 shifts, u8 width)
+{
+	u32 reg_val;
+	u32 mask = 0;
+
+	mask = (1 << width) - 1;
+	reg_val = top_read(id, reg);
+	reg_val &= ~(mask << shifts);
+	reg_val |= (value & mask) << shifts;
+	top_write(id, reg, reg_val);
+}
+
+u32 top_getverion(u32 id)
+{
+	u32 reg = 0;
+	u32 version = 0;
+
+	pr_info("DPTC:top_getverion ++\r\n");
+	reg = top_read(id, VERSION_REG);
+	pr_info("DPTC:top_getverion 0x0 = 0x%x\r\n", reg);
+
+	version = reg & 0xFF;
+	pr_info("DPTC:top_getverion version is 0x%x\r\n", version);
+
+	return version;
+}
+
+int32_t top_powerup(u32 id)
+{
+	u32 reg = 0;
+	u32 timeout = 1000;
+
+	pr_info("DPTC:top_powerup ++\r\n");
+
+	reg = top_read(id, PLL_CTRL);
+	while (0  == (reg & (RCAL_DONE | RCAL_TIMEOUT)) && timeout > 0) {
+		timeout--;
+		DPTCDELAY_US(10);
+		reg = top_read(id, PLL_CTRL);
+	}
+	if (timeout == 0) {
+		pr_info("DPTC:top_powerup timeout(0x%x)!\r\n", reg);
+		return -1;
+	}
+	pr_info("DPTC:top_powerup OK (0x%x)!\r\n", reg);
+	return 0;
+}
+
+int32_t top_init(u32 id, u8 func)
+{
+	pr_info("DPTC:top_init ++ (%d)\r\n", func);
+
+	g_current_function = func;
+
+	switch (func) {
+	case DPTC_FUNC_0: /*DP PHY TEST MODE*/
+		top_write(id, FUNCTION_SEL, 0);
+		top_clear_bits(id, PLL_CTRL, PLL_CTRL_SEL); /*DP control pll*/
+		top_write(id, CLK_RESET, 0); /*reset devices*/
+		DPTCDELAY_MS(5);
+		top_write(id, CLK_RESET, DPC_SW_RST); /*release DP*/
+		break;
+	case DPTC_FUNC_1: /*DP CONTROLLER + PHY TEST MODE*/
+		top_write(id, FUNCTION_SEL, 1);
+		top_clear_bits(id, PLL_CTRL, PLL_CTRL_SEL); /*DP control pll*/
+		top_write(id, CLK_RESET, 0); /*reset devices*/
+		DPTCDELAY_MS(5);
+		top_write(id, CLK_RESET, DPC_SW_RST); /*release DP*/
+		break;
+	case DPTC_FUNC_2: /*CSI + DP TEST MODE*/
+		top_write(id, FUNCTION_SEL, 2);
+		top_clear_bits(id, PLL_CTRL, PLL_CTRL_SEL); /*DP control pll*/
+		top_write(id, CLK_RESET, 0); /*reset devices*/
+		DPTCDELAY_MS(5);
+		top_write(id, CLK_RESET, DPC_SW_RST | CSI_SW_RST); /*release DP & CSI*/
+		break;
+	case DPTC_FUNC_3: /*CSI PHY TEST MODE*/
+		top_write(id, FUNCTION_SEL, 3);
+		top_set_bits(id, PLL_CTRL, PLL_CTRL_SEL); /*TOP control pll*/
+		top_write(id, CLK_RESET, 0); /*reset devices*/
+		DPTCDELAY_US(5);
+		top_write(id, CLK_RESET, CSI_SW_RST | (FROM_PLL << 4)); /*release CSI*/
+		break;
+	case DPTC_FUNC_4: /*DSI PHY TEST MODE*/
+		top_write(id, FUNCTION_SEL, 4);
+		top_set_bits(id, PLL_CTRL, PLL_CTRL_SEL); /*TOP control pll*/
+		top_write(id, CLK_RESET, 0); /*reset devices*/
+		DPTCDELAY_US(5);
+		top_write(id, CLK_RESET, DSI_SW_RST); /*release DP & DSI*/
+		break;
+	default:
+		g_current_function = DPTC_FUNC_LIMIT;
+		pr_info("DPTC:top_sel_func Invalid param(%d)!\r\n", func);
+		return -1;
+	}
+	return 0;
+}
+
+static void top_enable_clks(u32 id, u32 clks, u32 enable)
+{
+	u32 reg = 0;
+
+	pr_info("top_enable_clks ++ (0x%x, enable=%d)\r\n", clks, enable);
+
+	if (0 != (clks & DP_CLK_DSI))
+		reg |= PLL_CLK_DSI;
+	if (0 != (clks & DP_CLK_CSI))
+		reg |= PLL_CLK_CSI;
+	if (0 != (clks & DP_CLK_LS))
+		reg |= PLL_CLK_LS;
+	if (0 != (clks & DP_CLK_ESC))
+		reg |= PLL_CLK_ESC;
+	if (0 != (clks & DP_CLK_20X_40X))
+		reg |= PLL_CLK_20X_40X;
+
+	if (enable)
+		top_write_bits(id, PLL_CFG2, reg, 8, 8);
+	else
+		top_clear_bits(id, PLL_CFG2, reg << 8);
+}
+
+static int top_set_pllrate(u32 id, u8 freq, u8 ssc)
+{
+	struct s_pll_setting *pll = NULL;
+	u32 reg0 = 0, reg1 = 0, reg2 = 0;
+	u32 reg = 0;
+	u32 timeout = 50;
+
+	pr_info("DP:top_set_pllrate ++ (%d)\r\n", freq);
+
+	if (freq >= PLL_RATE_LIMIT || ssc >= PLL_SSC_LIMIT) {
+		pr_info("DP:top_set_pllrate Invalid param(%d)!\r\n", freq);
+		return -1;
+	}
+
+	pll = &DP_pll_configs[ssc][freq];
+
+	reg0 = (pll->reg3 << 24) | (pll->reg2 << 16) | (pll->reg1 << 8) | pll->reg0;
+	reg1 = (pll->reg7 << 24) | (pll->reg6 << 16) | (pll->reg5 << 8) | pll->reg4;
+	reg2 = pll->reg8;
+
+	/*power down pll*/
+	top_clear_bits(id, PLL_CTRL, PLL_POWER_UP);
+
+	/*config pll*/
+	top_write(id, PLL_CFG0, reg0);
+	top_write(id, PLL_CFG1, reg1);
+	top_write_bits(id, PLL_CFG2, reg2, 0, 8);
+
+	/*power up pll*/
+	top_set_bits(id, PLL_CTRL, PLL_POWER_UP);
+
+	/*wait for pll_lk*/
+	reg = top_read(id, PLL_CTRL);
+	while (((reg & PLL_LOCK) != PLL_LOCK) && timeout > 0) {
+		timeout--;
+		DPTCDELAY_US(10);
+		reg = top_read(id, PLL_CTRL);
+	}
+
+	if (timeout == 0) {
+		pr_info("DP:top_set_pllrate pll lock timeout(0x%x)!\r\n", reg);
+		return -1;
+	}
+	pr_info("DP:top_set_pllrate pll locked at %d KHz\r\n", freq);
+
+	return 0;
+}
+
+int32_t top_pll_init(u32 id, u32 func)
+{
+	int ret = 0;
+
+	pr_info("top_pll_init (func: %d)+++!\r\n", func);
+
+	switch (func) {
+	case DPTC_FUNC_0: /*DP PHY TEST MODE*/
+		break;
+	case DPTC_FUNC_1: /*DP CONTROLLER + PHY TEST MODE*/
+		break;
+	case DPTC_FUNC_2: /*CSI + DP TEST MODE*/
+		break;
+	case DPTC_FUNC_3: /*CSI PHY TEST MODE*/
+		top_enable_clks(id, DP_CLK_FUNC3, 1);
+		ret = top_set_pllrate(id, PLL_RATE_5400, PLL_SSC_0); /* PLL_SSC_5000 */
+		break;
+	case DPTC_FUNC_4:/*DSI PHY TEST MODE*/
+		break;
+	default:
+		g_current_function = DPTC_FUNC_LIMIT;
+		pr_info("top_pll_init Invalid param(%d)!\r\n", func);
+		return -1;
+	}
+
+	if (ret != 0)
+		pr_info("top_pll_init fail (%d)!\r\n", ret);
+
+	return ret;
+}
+
+static u32 dp_read(u32 id, u8 reg)
+{
+	return	TWSI_REG_READ_DPTC(id, DP_SLAVE_ADDR, reg >> 2);
+}
+
+static void dp_write(u32 id, u8 reg, u32 data)
+{
+	/* pr_info("DPTC:top_write: 0x%x = 0x%x\r\n", reg, data); */
+	TWSI_REG_WRITE_DPTC(id, DP_SLAVE_ADDR, reg >> 2, data);
+}
+
+static void dp_set_bits(u32 id, u8 reg, u32 bits)
+{
+	dp_write(id, reg, dp_read(id, reg) | bits);
+}
+
+static void dp_clear_bits(u32 id, u8 reg, u32 bits)
+{
+	dp_write(id, reg, dp_read(id, reg) & ~bits);
+}
+
+static void dp_write_bits(u32 id, u8 reg, u32 value, u8 shifts, u8 width)
+{
+	u32 reg_val;
+	u32 mask = 0;
+
+	mask = (1 << width) - 1;
+	reg_val = dp_read(id, reg);
+	reg_val &= ~(mask << shifts);
+	reg_val |= (value & mask) << shifts;
+	dp_write(id, reg, reg_val);
+}
+
+static void dp_enable_clks(u32 id, u32 clks, u32 enable)
+{
+	u32 reg = 0;
+
+	pr_info("DP:DP_enable_clks ++ (0x%x, enable=%d)\r\n", clks, enable);
+
+	if (0 != (clks & DP_CLK_DSI))
+		reg |= PLL_CLK_DSI;
+	if (0 != (clks & DP_CLK_CSI))
+		reg |= PLL_CLK_CSI;
+	if (0 != (clks & DP_CLK_LS))
+		reg |= PLL_CLK_LS;
+	if (0 != (clks & DP_CLK_ESC))
+		reg |= PLL_CLK_ESC;
+	if (0 != (clks & DP_CLK_20X_40X))
+		reg |= PLL_CLK_20X_40X;
+
+	if (enable)
+		dp_write_bits(id, PHY_CTRL2, reg, 8, 6);
+	else
+		dp_clear_bits(id, PHY_CTRL2, reg << 8);
+}
+
+static int dp_set_pllrate(u32 id, u8 freq, u8 ssc)
+{
+	struct s_pll_setting *pll = NULL;
+	u32 reg0 = 0, reg1 = 0, reg2 = 0;
+	u32 reg = 0;
+	u32 timeout = 50;
+
+	pr_info("DP:DP_set_pllrate ++ (%d)\r\n", freq);
+
+	if (freq >= PLL_RATE_LIMIT || ssc >= PLL_SSC_LIMIT) {
+		pr_info("DP:DP_set_pllrate Invalid param(%d)!\r\n", freq);
+		return -1;
+	}
+
+	pll = &DP_pll_configs[ssc][freq];
+	reg0 = (pll->reg3 << 24) | (pll->reg2 << 16) | (pll->reg1 << 8) | pll->reg0;
+	reg1 = (pll->reg7 << 24) | (pll->reg6 << 16) | (pll->reg5 << 8) | pll->reg4;
+	reg2 = pll->reg8;
+
+	/*power down pll*/
+	dp_clear_bits(id, PHY_PU_CTRL, PU_HPD | PU_AUX | PU_LANE1 | PU_LANE0 | PU_PLL);
+
+	/*config pll*/
+	dp_write(id, PHY_CTRL0, reg0);
+	dp_write(id, PHY_CTRL1, reg1);
+	dp_write_bits(id, PHY_CTRL2, reg2, 0, 8);
+
+	/*power up pll, aux, hpd*/
+	dp_set_bits(id, PHY_PU_CTRL, PU_HPD | PU_AUX | PU_PLL);
+
+	/*wait for pll_lk*/
+	reg = dp_read(id, PHY_PU_CTRL);
+	while (((reg & PLL_LK) != PLL_LK) && timeout > 0) {
+		timeout--;
+		DPTCDELAY_US(10);
+		reg = dp_read(id, PHY_PU_CTRL);
+	}
+
+	if (timeout == 0) {
+		pr_info("DP:DP_set_pllrate pll lock timeout(0x%x)!\r\n", reg);
+		return -1;
+	}
+	pr_info("DP:DP_set_pllrate pll locked at %d KHz\r\n", freq);
+
+	return 0;
+}
+
+int32_t dp_init(u32 id, u32 func)
+{
+	int ret = 0;
+
+	pr_info("DP:DP_init (func: %d)+++!\r\n", func);
+
+	switch (func) {
+	case DPTC_FUNC_0: /*DP PHY TEST MODE*/
+		break;
+	case DPTC_FUNC_1: /*DP CONTROLLER + PHY TEST MODE*/
+		break;
+	case DPTC_FUNC_2: /*CSI + DP TEST MODE*/
+		dp_enable_clks(id, DP_CLK_ALL, 1);
+		ret = dp_set_pllrate(id, PLL_RATE_5400, PLL_SSC_5000);
+		break;
+	case DPTC_FUNC_3: /*CSI PHY TEST MODE*/
+		dp_enable_clks(id, DP_CLK_FUNC3, 1);
+		ret = dp_set_pllrate(id, PLL_RATE_5400, PLL_SSC_5000);
+		break;
+	case DPTC_FUNC_4:/*DSI PHY TEST MODE*/
+		break;
+	default:
+		g_current_function = DPTC_FUNC_LIMIT;
+		pr_info("EDP:top_sel_func Invalid param(%d)!\r\n", func);
+		return -1;
+	}
+
+	if (ret != 0)
+		pr_info("DP:DP_init fail (%d)!\r\n", ret);
+
+	return ret;
+}
+
+static u32 dptc_csi_read(u32 id, u8 reg)
+{
+	return	TWSI_REG_READ_DPTC(id, CSI_SLAVE_ADDR, reg >> 2);
+}
+
+static void dptc_csi_write(u32 id, u8 reg, u32 data)
+{
+	/* pr_info("DPTC:top_write: 0x%x = 0x%x\r\n", reg, data); */
+	TWSI_REG_WRITE_DPTC(id, CSI_SLAVE_ADDR, reg >> 2, data);
+}
+
+#if 0
+static void dptc_csi_set_bits(u8 reg, u32 bits)
+{
+	dptc_csi_write(reg, dptc_csi_read(reg) | bits);
+}
+
+static void dptc_csi_clear_bits(u8 reg, u32 bits)
+{
+	dptc_csi_write(reg, dptc_csi_read(reg) & ~bits);
+}
+#endif
+static void dptc_csi_write_bits(u32 id, u8 reg, u32 value, u8 shifts, u8 width)
+{
+	u32 reg_val;
+	u32 mask = 0;
+
+	mask = (1 << width) - 1;
+	reg_val = top_read(id, reg);
+	reg_val &= ~(mask << shifts);
+	reg_val |= (value & mask) << shifts;
+	dptc_csi_write(id, reg, reg_val);
+}
+
+void dptc_csi_dphy_setting(u32 id, u8 lane_num)
+{
+	dptc_csi_write_bits(id, dptc_csi_phy_ctrl_adr, 0x01, 8, 1); /* analog power on:bit8 */
+	/* bit[31:30] adjust the 25uA bias current for HSRX.  10b:def, 11b:max */
+	/* bit[18:16] hsrx termination adjustment. */
+	dptc_csi_write(id, dptc_csi_phy_ana_cfg0_adr, 0xa28c8888); /* 0xa2848888 */ /* 0xe2848888 */ /* 0xa2848888 */
+	dptc_csi_write(id, dptc_csi_phy_timing_adr, 0x04040404); /* thomaszhang dptc_csi_write_bits(dptc_csi_phy_timing_adr, 0x00000820, 0, 16); 0x00001501 bit[15:8]:HS_SETTLE, bit[7:0]:HS_TERM_ENA xUI */
+
+	if (lane_num == 1) /* bit[3:0]:Lane Enable,Each bit controls each of the MIPI lane */
+		dptc_csi_write_bits(id, dptc_csi_phy_ctrl_adr, 0x11, 0, 8);
+	else if (lane_num == 2)
+		dptc_csi_write_bits(id, dptc_csi_phy_ctrl_adr, 0x33, 0, 8);
+	else if (lane_num == 4)
+		dptc_csi_write_bits(id, dptc_csi_phy_ctrl_adr, 0xff, 0, 8);
+}
+
+void dptc_csi_dphy_timing_setting(u32 id, u32 settle)
+{
+	dptc_csi_write(id, dptc_csi_phy_timing_adr, settle);
+}
+
+void dptc_csi_reg_setting(u32 id, u8 RAW_type, u32 sensor_width, u32 sensor_height, u8 csi_lanes)
+{
+	u32 csi2_laneend  = 0;
+	u32 csi2_eccend   = 0;
+	u32 csi2_eot   = 0;
+	u32 csi2_parse_error   = 0;
+	u32 csi2_vsynclen = 4;
+	u32 csi2_hsynclen = 4;
+	/* u32 csi2_ana_pu = 1; */
+	u32 csi2_hs_settle = 0x8;
+	u32 csi2_hs_termen = 0x20;
+	u32 csi2_ck_settle = 0x10;
+	u32 csi2_ck_termen = 0x1;
+	/* u32 csi2_dsi_mode = 0x0; */ /* 1: dsi mode; 0:csi mode */
+	/* u32 csi2_packet_code = 0x2b; */
+	/* u32 csi2_hsync_start_code = 0x02; */
+	/* u32 csi2_vsync_start_code = 0x00; */
+
+	dptc_csi_write(id, dptc_csi_ctrl_adr, ((csi2_hsynclen << 10) + (csi2_vsynclen << 6) + (csi2_parse_error << 5) + (csi2_eccend << 4) + (csi2_eot << 3) + (csi2_laneend << 2) + (csi_lanes - 1)));
+	dptc_csi_write(id, dptc_csi_image_size_adr, (sensor_height << 16) + sensor_width);
+
+	dptc_csi_write(id, dptc_csi_status_adr, 0xffffffff);
+
+	/* dptc_csi_write(id, dptc_csi_phy_ctrl_adr, ((csi2_ana_pu << 8) + (CSI2_DPHY5_LANE_ENA(csi_lanes) << 4) + CSI2_DPHY5_LANE_ENA(csi_lanes))); */
+	dptc_csi_write(id, dptc_csi_phy_timing_adr, ((csi2_ck_settle << 24) + (csi2_ck_termen << 16) + (csi2_hs_settle << 8) + csi2_hs_termen));
+	dptc_csi_write(id, dptc_csi_phy_ana_cfg0_adr, 0xa28c8888);
+	dptc_csi_write(id, dptc_csi_phy_ana_cfg1_adr, 0x00000000);
+	dptc_csi_write(id, dptc_csi_status_adr, 0xffffffff);
+	dptc_csi_write_bits(id, dptc_csi_phy_status_adr, 0x7f, 1, 7); /* dptc_csi_write(dptc_csi_phy_status_adr, 0x0000000f); */
+	/* dptc_csi_write(id, dptc_csi_sync_code_adr, (csi2_dsi_mode << 24) + (csi2_packet_code << 16) + (csi2_hsync_start_code << 8) + csi2_vsync_start_code); */
+	dptc_csi_dphy_setting(id, csi_lanes);
+}
+
+int DPTC_func3_open(u32 id)
+{
+	s32 ret = 0;
+
+	if (id > 1) {
+		pr_info("invalid dptc id %d [0, 1]\n", id);
+		return -1;
+	}
+
+	pr_info("DPTC %d: DPTC_func3_open enter! DPTC_I2C_PORT: %d\n", id, dptc_ports[id]);
+
+	TWSI_Init(STANDARD_MODE, id); /* STANDARD_MODE */
+	DPTCDELAY_US(100); /* thomaszhang DPTCDELAY_MS(100); */
+
+	ret = top_powerup(id);
+	if (ret != 0) {
+		pr_info("DPTC: DPTC_func3_open: top_powerup fail!\r\n");
+		return -1;
+	}
+	top_getverion(id);
+	ret = top_init(id, DPTC_FUNC_3);
+	if (ret != 0) {
+		pr_info("DPTC: DPTC_func3_open: top_init fail!\r\n");
+		return -1;
+	}
+	ret = top_pll_init(id, DPTC_FUNC_3);
+	if (ret != 0) {
+		pr_info("DPTC: DPTC_func3_open: top_pll_init fail!\r\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+int DPTC_func3_close(u32 id)
+{
+	return 0;
+}
+
+void dptc_csi_status_handler(u32 id)
+{
+	u32 irqs, i;
+
+	DPTCDELAY_MS(1000);
+
+	for (i = 0; i <= 0x28; i += 4) {
+		irqs = dptc_csi_read(id, i);
+		pr_info("dptc_csi_status_adr: 0x%x = 0x%08x\n", i, irqs);
+	}
+
+	for (i = 0; i <= 0x18; i += 4) {
+		irqs = top_read(id, i);
+		pr_info("top_read: 0x%x = 0x%08x\n", i, irqs);
+	}
+	dptc_csi_write(id, dptc_csi_status_adr, 0xffffffff);
+	dptc_csi_write_bits(id, dptc_csi_phy_status_adr, 0x7f, 1, 7);
+}
+
+void dptc_csi_status_read(u32 id)
+{
+	u32 irqs;
+
+	DPTCDELAY_MS(1000);
+
+	irqs = dptc_csi_read(id, dptc_csi_status_adr);
+	pr_info("dptc_csi_status_adr: 0x%x = 0x%08x\n",
+		dptc_csi_status_adr, irqs);
+}
+#endif
diff --git a/drivers/media/platform/spacemit/cam_ccic/dptc_drv.h b/drivers/media/platform/spacemit/cam_ccic/dptc_drv.h
new file mode 100644
index 000000000000..705f94cbec10
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/dptc_drv.h
@@ -0,0 +1,113 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * dptc_drv.h
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef __DPTC_DRV_H__
+#define __DPTC_DRV_H__
+
+//#define DPTC_I2C_PORT  1 //8
+#define DPTC_I2C_PORT 2 //for zynq
+#define DPTC_I2C_PORT_1 4 //for zynq
+#define TOP_SLAVE_ADDR 0x30 /*7-bit, 8-bit: 0x60 */
+#define DP_SLAVE_ADDR 0x31 /*7-bit, 8-bit: 0x62 */
+#define CSI_SLAVE_ADDR 0x32 /*7-bit, 8-bit: 0x64 */
+
+/* top reg */
+#define VERSION_REG 0x0
+#define FUNCTION_SEL 0x4
+#define PLL_CTRL 0x8
+#define PLL_CFG0 0xC
+#define PLL_CFG1 0x10
+#define PLL_CFG2 0x14
+#define CLK_RESET 0x18
+
+/*0x8*/
+#define PLL_CTRL_SEL BIT(0)
+#define PLL_POWER_UP BIT(1)
+#define RCAL_POWER_UP BIT(2)
+#define PLL_LOCK BIT(3)
+#define RCAL_DONE BIT(4)
+#define RCAL_TIMEOUT BIT(5)
+
+/*0x18*/
+#define DSI_SW_RST BIT(2)
+#define CSI_SW_RST BIT(1)
+#define DPC_SW_RST BIT(0)
+
+/* DP PLL */
+#define PHY_SWING_CTRL 0x8C
+#define PHY_PU_CTRL 0x90
+#define PHY_CTRL0 0x94
+#define PHY_CTRL1 0x98
+#define PHY_CTRL2 0x9C
+
+/*0x90*/
+#define PHY_PN_SWAP BIT(30)
+#define PHY_LANE_SWAP BIT(29)
+#define PLL_LK BIT(28)
+#define AUX_SINGLE_END BIT(27)
+#define PU_HPD BIT(26)
+#define PU_AUX BIT(25)
+#define PU_PLL BIT(24)
+#define PU_LANE1 BIT(21)
+#define PU_LANE0 BIT(20)
+#define EN_20B_MODE BIT(19)
+
+/* csi register */
+#define dptc_csi_ctrl_adr 0x00
+#define dptc_csi_image_size_adr 0x04
+#define dptc_csi_gate_ctrl_adr 0x08
+#define dptc_csi_status_adr 0x0c
+#define dptc_csi_phy_ctrl_adr 0x10
+#define dptc_csi_phy_timing_adr 0x14
+#define dptc_csi_phy_ana_cfg0_adr 0x18
+#define dptc_csi_phy_ana_cfg1_adr 0x1c
+#define dptc_csi_phy_status_adr 0x20
+#define dptc_csi_sync_code_adr 0x24
+#define dptc_csi_mem_cfg_adr 0x28
+
+#define CSI2_DPHY5_LANE_ENA(n) ((1 << (n)) - 1)
+
+enum {
+	DPTC_FUNC_0, /*DP PHY TEST MODE*/
+	DPTC_FUNC_1, /*DP CONTROLLER + PHY TEST MODE*/
+	DPTC_FUNC_2, /*CSI + DP TEST MODE*/
+	DPTC_FUNC_3, /*CSI PHY TEST MODE*/
+	DPTC_FUNC_4, /*DSI PHY TEST MODE*/
+	DPTC_FUNC_LIMIT,
+};
+
+typedef enum {
+	FROM_PLL = 0,
+	FROM_PAD = 1,
+	FROM_REF_CLK = 2,
+	FROM_RESERVE,
+} CSI_ESC_CLK_SEL;
+
+struct twsi_data {
+	u8 twsi_no;
+	u8 reg_len; /* byte num*/
+	u8 val_len; /* byte num*/
+	u8 addr; /* 7 bit i2c address*/
+	u16 reg;
+	u32 val;
+};
+
+enum sensor_i2c_len {
+	I2C_8BIT = 1,
+	I2C_16BIT = 2,
+	I2C_24BIT = 3,
+	I2C_32BIT = 4,
+};
+
+u32 top_getverion(u32 id);
+int DPTC_func3_open(u32 id);
+int DPTC_func3_close(u32 id);
+void dptc_csi_status_handler(u32 id);
+void dptc_csi_reg_setting(u32 id, u8 RAW_type, u32 sensor_width,
+			  u32 sensor_height, u8 csi_lanes);
+void dptc_csi_dphy_timing_setting(u32 id, u32 settle);
+#endif /* ifndef __DPTC_DRV_H__ */
diff --git a/drivers/media/platform/spacemit/cam_ccic/dptc_pll_setting.h b/drivers/media/platform/spacemit/cam_ccic/dptc_pll_setting.h
new file mode 100644
index 000000000000..7e1993ee9e65
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_ccic/dptc_pll_setting.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * dptc_pll_setting.h - pll setting for dptc
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+
+#ifndef DPTC_PLL_SETTING_H_
+#define DPTC_PLL_SETTING_H_
+
+enum {
+	PLL_RATE_1620,
+	PLL_RATE_2700,
+	PLL_RATE_5400,
+	PLL_RATE_LIMIT
+};
+
+enum {
+	PLL_SSC_0,
+	PLL_SSC_5000,
+	PLL_SSC_LIMIT,
+};
+
+enum {
+	DP_CLK_DSI = 0x1,
+	DP_CLK_CSI = 0x2,
+	DP_CLK_LS = 0x4,
+	DP_CLK_ESC = 0x8,
+	DP_CLK_20X_40X = 0x10,
+	DP_CLK_LIMIT
+};
+
+#define DP_CLK_ALL (DP_CLK_DSI|DP_CLK_CSI|DP_CLK_LS|DP_CLK_ESC|DP_CLK_20X_40X)
+#define DP_CLK_FUNC3 DP_CLK_ESC//(DP_CLK_CSI|DP_CLK_LS|DP_CLK_ESC)//(DP_CLK_CSI|DP_CLK_ESC)
+
+
+#define PLL_CLK_20X_40X BIT(1)
+#define PLL_CLK_ESC BIT(2)
+#define PLL_CLK_LS BIT(3)
+#define PLL_CLK_CSI BIT(4)
+#define PLL_CLK_DSI BIT(5)
+
+struct s_pll_setting{
+	uint8_t reg0;
+	uint8_t reg1;
+	uint8_t reg2;
+	uint8_t reg3;
+	uint8_t reg4;
+	uint8_t reg5;
+	uint8_t reg6;
+	uint8_t reg7;
+	uint8_t reg8;
+	uint8_t reserved1;
+	uint16_t reserved2;
+};
+
+#endif /*DPTC_PLL_SETTING_H_*/
diff --git a/drivers/media/platform/spacemit/cam_sensor/Kconfig b/drivers/media/platform/spacemit/cam_sensor/Kconfig
new file mode 100644
index 000000000000..14e1467a0111
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_sensor/Kconfig
@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# spacemit sensor configuration
+#
+
+comment "SPACEMIT K3 Camera Sensor"
+
+config SPACEMIT_K3_CAM_IMX219
+	tristate "Sony IMX219 sensor support for K3"
+	depends on SPACEMIT_K3_CCIC && I2C
+	help
+	  This is a i2c sensor driver for the Sony
+	  IMX219 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx219.
+
+config SPACEMIT_K3_CAM_IMX415
+	tristate "Sony IMX415 sensor support for K3"
+	depends on SPACEMIT_K3_CCIC && I2C
+	help
+	  This is a i2c sensor driver for the Sony
+	  IMX415 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx219.
+
+config SPACEMIT_K3_CAM_OV5647
+	tristate "OmniVision OV5647 sensor support"
+	depends on SPACEMIT_K3_CCIC && I2C
+	help
+	  This is a i2c sensor driver for the OmniVision
+	  OV5647 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ov5647.
+
+config SPACEMIT_K3_CAM_OV2735
+	tristate "OmniVision OV2735 sensor support"
+	depends on SPACEMIT_K3_CCIC && I2C
+	help
+	  This is a i2c sensor driver for the OmniVision
+	  OV2735 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called OV2735.
+
+config SPACEMIT_K3_CAM_OV5640_MAX96724_MAX9295
+	tristate "OV5640 over MAX96724/MAX9295 GMSL module support"
+	depends on SPACEMIT_K3_CCIC && I2C
+	help
+	  This is a simplified I2C/ioctl sensor driver for the OV5640 sensor
+	  connected through a MAX96724 deserializer and MAX9295 serializer.
+
+	  It provides power, register initialization, stream control and
+	  GMSL bring-up logic for the Spacemit K3 camera pipeline.
\ No newline at end of file
diff --git a/drivers/media/platform/spacemit/cam_sensor/Makefile b/drivers/media/platform/spacemit/cam_sensor/Makefile
new file mode 100644
index 000000000000..0b135a4c11cc
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_sensor/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_SPACEMIT_K3_CAM_IMX219) += imx219_sensor.o
+obj-$(CONFIG_SPACEMIT_K3_CAM_IMX415) += imx415_sensor.o
+obj-$(CONFIG_SPACEMIT_K3_CAM_OV5647) += ov5647_sensor.o
+obj-$(CONFIG_SPACEMIT_K3_CAM_OV2735) += ov2735_sensor.o
+obj-$(CONFIG_SPACEMIT_K3_CAM_OV5640_MAX96724_MAX9295) += ov5640_max96724_max9295_sensor.o
\ No newline at end of file
diff --git a/drivers/media/platform/spacemit/cam_sensor/imx219_sensor.c b/drivers/media/platform/spacemit/cam_sensor/imx219_sensor.c
new file mode 100644
index 000000000000..0228f8903a8d
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_sensor/imx219_sensor.c
@@ -0,0 +1,508 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Simplified I2C driver for Sony IMX219
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/ioctl.h>
+#include <linux/regulator/consumer.h>
+
+/*
+ * Sensor Configuration: 1920x1080 @ 30fps, 2-lane MIPI
+ *
+ * MCLK:           24MHz
+ * Resolution:     1920x1080
+ * Bit Depth:      10bit
+ * FPS:            30fps
+ * HTS:            3448 (0x0d78) - registers 0x0162:0x0163
+ * VTS:            1766 (0x06e6) - registers 0x0160:0x0161
+ * PCLK:           182.675MHz
+ * MIPI Data Rate: 913.375Mbps/Lane
+ * MIPI Clock:     914MHz
+ * Htime:          18.875us
+ */
+
+/* IOCTL interface for user-space control */
+#define IMX219_IOC_MAGIC		'I'
+#define IMX219_IOCTL_POWER_ON		_IO(IMX219_IOC_MAGIC, 1)
+#define IMX219_IOCTL_POWER_OFF		_IO(IMX219_IOC_MAGIC, 2)
+#define IMX219_IOCTL_INIT_REGS		_IO(IMX219_IOC_MAGIC, 3)
+#define IMX219_IOCTL_STREAM_ON		_IO(IMX219_IOC_MAGIC, 4)
+#define IMX219_IOCTL_STREAM_OFF		_IO(IMX219_IOC_MAGIC, 5)
+#define IMX219_IOCTL_DETECT		_IO(IMX219_IOC_MAGIC, 6)
+
+static struct imx219 *global_imx219;
+
+struct imx219 {
+	struct i2c_client *client;
+	struct gpio_desc *pwdn;
+	struct gpio_desc *i2c_mux;
+	struct mutex lock;
+	struct regulator *vdd;
+	struct miscdevice miscdev;
+};
+
+struct regval_list {
+	u16 addr;
+	u8 data;
+};
+
+static struct regval_list imx219_1080p_regs[] = {
+	{0x30EB, 0x05},
+	{0x30EB, 0x0C},
+	{0x300A, 0xFF},
+	{0x300B, 0xFF},
+	{0x30EB, 0x05},
+	{0x30EB, 0x09},
+	{0x0114, 0x01},
+	{0x0128, 0x00},
+	{0x012A, 0x18},
+	{0x012B, 0x00},
+	{0x0160, 0x06},
+	{0x0161, 0xE6},
+	{0x0162, 0x0D},
+	{0x0163, 0x78},
+	{0x0164, 0x02},
+	{0x0165, 0xA8},
+	{0x0166, 0x0A},
+	{0x0167, 0x27},
+	{0x0168, 0x02},
+	{0x0169, 0xB4},
+	{0x016A, 0x06},
+	{0x016B, 0xEB},
+	{0x016C, 0x07},
+	{0x016D, 0x80},
+	{0x016E, 0x04},
+	{0x016F, 0x38},
+	{0x0170, 0x01},
+	{0x0171, 0x01},
+	{0x0174, 0x00},
+	{0x0175, 0x00},
+	{0x018C, 0x0A},
+	{0x018D, 0x0A},
+	{0x0301, 0x05},
+	{0x0303, 0x01},
+	{0x0304, 0x03},
+	{0x0305, 0x03},
+	{0x0306, 0x00},
+	{0x0307, 0x39},
+	{0x0309, 0x0A},
+	{0x030B, 0x01},
+	{0x030C, 0x00},
+	{0x030D, 0x72},
+	{0x455E, 0x00},
+	{0x471E, 0x4B},
+	{0x4767, 0x0F},
+	{0x4750, 0x14},
+	{0x4540, 0x00},
+	{0x47B4, 0x14},
+	{0x0100, 0x00},
+};
+
+static int imx219_write(struct imx219 *sensor, u16 reg, u8 val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msg;
+	u8 data[3];
+	int ret;
+
+	data[0] = (reg >> 8) & 0xff;
+	data[1] = reg & 0xff;
+	data[2] = val & 0xff;
+
+	msg.addr = sensor->client->addr;
+	msg.flags = 0;
+	msg.len = sizeof(data);
+	msg.buf = data;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, &msg, 1);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 1) {
+		dev_err(&sensor->client->dev,
+			"imx219-test: I2C write failed, reg=0x%x, val=0x%x, ret=%d\n",
+			reg, val, ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	return 0;
+}
+
+static int imx219_read(struct imx219 *sensor, u16 reg, u8 *val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msgs[2];
+	u8 reg_buf[2];
+	u8 data_buf[1];
+	int ret;
+
+	reg_buf[0] = (reg >> 8) & 0xff;
+	reg_buf[1] = reg & 0xff;
+
+	msgs[0].addr = sensor->client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = sizeof(reg_buf);
+	msgs[0].buf = reg_buf;
+
+	msgs[1].addr = sensor->client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = sizeof(data_buf);
+	msgs[1].buf = data_buf;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, msgs, 2);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 2) {
+		dev_err(&sensor->client->dev,
+			"imx219-test: I2C read failed, reg=0x%x, ret=%d\n", reg,
+			ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	*val = data_buf[0];
+	return 0;
+}
+
+static int imx219_stream_on(struct imx219 *sensor)
+{
+	return imx219_write(sensor, 0x0100, 0x01);
+}
+
+static int imx219_stream_off(struct imx219 *sensor)
+{
+	return imx219_write(sensor, 0x0100, 0x00);
+}
+
+static int imx219_write_init_regs(struct imx219 *sensor);
+static int imx219_power_on(struct imx219 *sensor);
+static void imx219_power_off(struct imx219 *sensor);
+static int imx219_detect(struct imx219 *sensor);
+
+static long imx219_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct imx219 *sensor = file->private_data;
+	int ret = 0;
+
+	if (!sensor)
+		return -ENODEV;
+
+	switch (cmd) {
+	case IMX219_IOCTL_POWER_ON:
+		ret = imx219_power_on(sensor);
+		break;
+	case IMX219_IOCTL_POWER_OFF:
+		imx219_power_off(sensor);
+		ret = 0;
+		break;
+	case IMX219_IOCTL_INIT_REGS:
+		ret = imx219_write_init_regs(sensor);
+		break;
+	case IMX219_IOCTL_STREAM_ON:
+		ret = imx219_stream_on(sensor);
+		break;
+	case IMX219_IOCTL_STREAM_OFF:
+		ret = imx219_stream_off(sensor);
+		break;
+	case IMX219_IOCTL_DETECT:
+		ret = imx219_detect(sensor);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int imx219_dev_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *misc = file->private_data;
+	struct imx219 *sensor;
+
+	if (!misc)
+		return -ENODEV;
+
+	sensor = container_of(misc, struct imx219, miscdev);
+	/* replace private_data with sensor pointer for use in ioctl */
+	file->private_data = sensor;
+	return 0;
+}
+
+static const struct file_operations imx219_fops = {
+	.owner = THIS_MODULE,
+	.open = imx219_dev_open,
+	.unlocked_ioctl = imx219_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = imx219_ioctl,
+#endif
+};
+
+static int imx219_write_init_regs(struct imx219 *sensor)
+{
+	int i, ret;
+
+	dev_info(&sensor->client->dev,
+		 "imx219-test: write init regs, total=%ld\n",
+		 ARRAY_SIZE(imx219_1080p_regs));
+	for (i = 0; i < ARRAY_SIZE(imx219_1080p_regs); i++) {
+		const struct regval_list *reg = &imx219_1080p_regs[i];
+
+		ret = imx219_write(sensor, reg->addr, reg->data);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev,
+				"imx219-test: write init reg failed, idx=%d, reg=0x%x, val=0x%x, ret=%d\n",
+				i, reg->addr, reg->data, ret);
+			return ret;
+		}
+	}
+
+	dev_info(&sensor->client->dev, "imx219-test: init regs written\n");
+	return 0;
+}
+
+static int imx219_detect(struct imx219 *sensor)
+{
+	u8 hi, lo;
+	int ret;
+
+	dev_info(&sensor->client->dev, "imx219-test: detect start\n");
+
+	ret = imx219_read(sensor, 0x0000, &hi);
+	if (ret < 0)
+		return ret;
+	if ((hi & 0x0f) != 0x02) {
+		dev_err(&sensor->client->dev,
+			"imx219-test: ID high mismatch: 0x%x\n", hi);
+		return -ENODEV;
+	}
+	ret = imx219_read(sensor, 0x0001, &lo);
+	if (ret < 0)
+		return ret;
+	if (lo != 0x19) {
+		dev_err(&sensor->client->dev,
+			"imx219-test: ID low mismatch: 0x%x\n", lo);
+		return -ENODEV;
+	}
+
+	dev_info(&sensor->client->dev,
+		 "imx219-test: detected IMX219 (id %02x%02x)\n", hi, lo);
+	return 0;
+}
+
+static int imx219_power_on(struct imx219 *sensor)
+{
+	int ret;
+
+	dev_info(&sensor->client->dev, "imx219-test: power_on enter\n");
+
+	/* Set I2C mux to select this sensor */
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "imx219-test: set i2c-mux high\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 1);
+	}
+
+	dev_info(&sensor->client->dev, "imx219-test: get vdd regulator\n");
+	sensor->vdd = devm_regulator_get(&sensor->client->dev, "vdd");
+	if (IS_ERR(sensor->vdd)) {
+		dev_err(&sensor->client->dev, "imx219-test: Failed to get vdd regulator: %ld\n",
+			PTR_ERR(sensor->vdd));
+		return PTR_ERR(sensor->vdd);
+	}
+
+	if (sensor->vdd) {
+		ret = regulator_enable(sensor->vdd);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev,
+				"imx219-test: failed to enable vdd: %d\n", ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "imx219-test: enable vdd\n");
+
+		ret = regulator_set_voltage(sensor->vdd, 3300000, 3300000);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev, "imx219-test: failed to set vdd voltage: %d\n",
+				ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "imx219-test: vdd set to 3.3V\n");
+	}
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev, "imx219-test: drive PWDN low\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	}
+
+	usleep_range(10000, 12000);
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev,
+			 "imx219-test: drive PWDN high\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 1);
+	}
+
+	usleep_range(30000, 31000);
+
+	dev_info(&sensor->client->dev, "imx219-test: power_on done\n");
+
+	return 0;
+}
+
+static void imx219_power_off(struct imx219 *sensor)
+{
+	dev_info(&sensor->client->dev, "imx219-test: power_off enter\n");
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev, "imx219-test: drive PWDN low\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	}
+
+	if (sensor->vdd) {
+		dev_info(&sensor->client->dev, "imx219-test: disable vdd\n");
+		regulator_disable(sensor->vdd);
+	}
+
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "imx219-test: set i2c-mux low\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 0);
+	}
+
+	dev_info(&sensor->client->dev, "imx219-test: power_off done\n");
+}
+
+static int imx219_probe(struct i2c_client *client)
+{
+	struct imx219 *sensor;
+	struct device *dev = &client->dev;
+	int ret;
+
+	dev_info(dev, "imx219-test: probe enter, client addr=0x%x\n",
+		 client->addr);
+
+	sensor = devm_kzalloc(dev, sizeof(*sensor), GFP_KERNEL);
+	if (!sensor)
+		return -ENOMEM;
+
+	sensor->client = client;
+	mutex_init(&sensor->lock);
+	i2c_set_clientdata(client, sensor);
+
+	dev_info(dev, "imx219-test: get PWDN gpio\n");
+	sensor->pwdn = devm_gpiod_get_optional(
+		dev, "pwdn", GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->pwdn)) {
+		dev_err(dev, "imx219-test: Failed to get PWDN GPIO\n");
+		return PTR_ERR(sensor->pwdn);
+	}
+
+	sensor->i2c_mux = devm_gpiod_get_optional(dev, "i2c-mux",
+		GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->i2c_mux)) {
+		dev_warn(dev, "imx219-test: Failed to get i2c-mux GPIO, continuing without it\n");
+		sensor->i2c_mux = NULL;
+	}
+
+	ret = imx219_power_on(sensor);
+	if (ret) {
+		dev_err(dev, "imx219-test: power on failed: %d\n", ret);
+		goto err_power_on;
+	}
+
+	ret = imx219_detect(sensor);
+	if (ret) {
+		dev_err(dev, "imx219-test: sensor detect failed: %d\n", ret);
+		goto err_detect;
+	}
+
+	imx219_power_off(sensor);
+
+	sensor->miscdev.minor = MISC_DYNAMIC_MINOR;
+	sensor->miscdev.fops = &imx219_fops;
+	sensor->miscdev.parent = dev;
+	if (client->dev.of_node) {
+		u32 csi_id;
+		if (of_property_read_u32(client->dev.of_node, "csi-id",
+					 &csi_id) == 0) {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "imx219-%u", csi_id);
+			dev_info(dev, "imx219-test: imx219-%u \n", csi_id);
+		} else {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "imx219-%02x", client->addr);
+			dev_info(dev, "imx219-test: imx219-%02x \n",
+				 client->addr);
+		}
+	} else {
+		sensor->miscdev.name = devm_kasprintf(
+			dev, GFP_KERNEL, "imx219-%02x", client->addr);
+	}
+
+	ret = misc_register(&sensor->miscdev);
+	if (ret) {
+		dev_err(dev,
+			"imx219-test: failed to register misc device: %d\n",
+			ret);
+		goto err_misc_register;
+	}
+
+	global_imx219 = sensor;
+	dev_info(dev, "imx219-test: probe successful, ioctl device /dev/%s\n",
+		 sensor->miscdev.name);
+	return 0;
+
+err_misc_register:
+	imx219_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+	return ret;
+
+err_detect:
+	imx219_power_off(sensor);
+err_power_on:
+	mutex_destroy(&sensor->lock);
+	return ret;
+}
+
+static void imx219_remove(struct i2c_client *client)
+{
+	struct imx219 *sensor = i2c_get_clientdata(client);
+
+	dev_info(&client->dev, "imx219-test: remove\n");
+	if (global_imx219 == sensor)
+		global_imx219 = NULL;
+
+	misc_deregister(&sensor->miscdev);
+	imx219_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+}
+
+static const struct of_device_id imx219_of_match[] = {
+	{ .compatible = "sony,imx219" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, imx219_of_match);
+
+static struct i2c_driver imx219_driver = {
+	.driver = {
+		.name		= "imx219-simple",
+		.of_match_table	= of_match_ptr(imx219_of_match),
+	},
+	.probe		= imx219_probe,
+	.remove		= imx219_remove,
+};
+
+module_i2c_driver(imx219_driver)
+
+MODULE_DESCRIPTION("Simplified I2C driver for IMX219");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/spacemit/cam_sensor/imx415_sensor.c b/drivers/media/platform/spacemit/cam_sensor/imx415_sensor.c
new file mode 100755
index 000000000000..ff7301d78594
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_sensor/imx415_sensor.c
@@ -0,0 +1,1047 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Simplified I2C driver for Sony IMX415
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ *
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/ioctl.h>
+#include <linux/gpio.h>
+#include <linux/regulator/consumer.h>
+
+/*
+ * IMX415 Configuration: 1920x1080 @ 60fps, 4-lane MIPI
+ *
+ * MCLK:           24MHz
+ * Resolution:     1920x1080
+ * Bit Depth:      12bit
+ * FPS:            60fps
+ * HTS:            365 (0x16D)
+ * VTS:            2892
+ * PCLK:           72MHz
+ * MIPI Data Rate: 1485Mbps/Lane
+ * MIPI Clock:     742.5MHz
+ */
+
+/* IOCTL interface for user-space control */
+#define IMX415_IOC_MAGIC		'I'
+#define IMX415_IOCTL_POWER_ON		_IO(IMX415_IOC_MAGIC, 1)
+#define IMX415_IOCTL_POWER_OFF		_IO(IMX415_IOC_MAGIC, 2)
+#define IMX415_IOCTL_INIT_REGS		_IO(IMX415_IOC_MAGIC, 3)
+#define IMX415_IOCTL_STREAM_ON		_IO(IMX415_IOC_MAGIC, 4)
+#define IMX415_IOCTL_STREAM_OFF		_IO(IMX415_IOC_MAGIC, 5)
+#define IMX415_IOCTL_DETECT		_IO(IMX415_IOC_MAGIC, 6)
+
+
+#define SENSOR_REG_END 0xFFFF
+#define SENSOR_REG_DELAY 0xFFFE
+
+static struct imx415 *global_imx415;
+
+struct imx415 {
+	struct i2c_client *client;
+	struct gpio_desc *pwdn;
+	struct gpio_desc *i2c_mux;
+	struct mutex lock;
+	struct regulator *vdd;
+	struct miscdevice miscdev;
+};
+
+struct regval_list {
+	u16 addr;
+	u8 data;
+};
+
+__maybe_unused static struct regval_list imx415_1920x1080_10bit_112fps_tab[] = {
+	// @@1920x1080 crop 112fps 1485Mbps/Lane
+	// MCLK: 37.125Mhz
+	// resolution: 1920x1080
+	// Mipi : 4 lane
+	// Mipi data rate: 1485Mbps/Lane
+	// FPS      :112fps
+	// HTS      :538 -> 0x021A(0x3028:0x3029)
+	// VTS      :1232 -> 0x04d0(0x3024:0x3025)
+	// PCLK/system clk     : 74235392
+	// Htime= 7.201 us; Vblank= ? ms
+	{0x3000, 0x01},  // STANDBY
+	{0x3001, 0x00},  // REGHOLD
+	{0x3002, 0x01},  // XMSTA
+	{0x3003, 0x00},  // XMASTER
+	{0x3008, 0x7F},  // BCWAIT_TIME[9:0]
+	{0x3009, 0x00},  // BCWAIT_TIME[9:0]
+	{0x300A, 0x5B},  // CPWAIT_TIME[9:0]
+	{0x300B, 0xA0},  // CPWAIT_TIME[9:0]
+	{0x301C, 0x04},  // WINMODE[3:0]
+	{0x301D, 0x08},  // -
+	{0x3020, 0x00},  // HADD
+	{0x3021, 0x00},  // VADD
+	{0x3022, 0x00},  // ADDMODE[1:0]
+	{0x3023, 0x01},  // VCMODE
+	{0x3024, 0xD0},  // VMAX[19:0]
+	{0x3025, 0x04},  // VMAX[19:0]
+	{0x3026, 0x00},  // VMAX[19:0]
+	{0x3028, 0x1A},  // HMAX[15:0]
+	{0x3029, 0x02},  // HMAX[15:0]
+	{0x302C, 0x00},  // WDMODE[1:0]
+	{0x302D, 0x00},  // WDSEL[1:0]
+	{0x3030, 0x00},  // HREVERSE
+	{0x3031, 0x00},  // ADBIT[1:0]
+	{0x3032, 0x00},  // MDBIT
+	{0x3033, 0x08},  // SYS_MODE[3:0]
+	{0x3040, 0xCC},  // PIX_HST[12:0]
+	{0x3041, 0x03},  // PIX_HST[12:0]
+	{0x3042, 0x80},  // PIX_HWIDTH[12:0]
+	{0x3043, 0x07},  // PIX_HWIDTH[12:0]
+	{0x3044, 0x58},  // PIX_VST[12:0]
+	{0x3045, 0x04},  // PIX_VST[12:0]
+	{0x3046, 0x70},  // PIX_VWIDTH[12:0]
+	{0x3047, 0x08},  // PIX_VWIDTH[12:0]
+	{0x3050, 0x08},  // SHR0[19:0]
+	{0x3051, 0x00},  // SHR0[19:0]
+	{0x3052, 0x00},  // SHR0[19:0]
+	{0x3054, 0x19},  // SHR1[19:0]
+	{0x3055, 0x00},  // SHR1[19:0]
+	{0x3056, 0x00},  // SHR1[19:0]
+	{0x3058, 0x3E},  // SHR2[19:0]
+	{0x3059, 0x00},  // SHR2[19:0]
+	{0x305A, 0x00},  // SHR2[19:0]
+	{0x305C, 0x66},  // SHR3[19:0]
+	{0x305D, 0x00},  // SHR3[19:0]
+	{0x305E, 0x00},  // SHR3[19:0]
+	{0x3060, 0x25},  // RHS1[19:0]
+	{0x3061, 0x00},  // RHS1[19:0]
+	{0x3062, 0x00},  // RHS1[19:0]
+	{0x3064, 0x4A},  // RHS2[19:0]
+	{0x3065, 0x00},  // RHS2[19:0]
+	{0x3066, 0x00},  // RHS2[19:0]
+	{0x3090, 0x00},  // GAIN_PGC_0[8:0]
+	{0x3091, 0x00},  // GAIN_PGC_0[8:0]
+	{0x3092, 0x00},  // GAIN_PGC_1[8:0]
+	{0x3093, 0x00},  // GAIN_PGC_1[8:0]
+	{0x3094, 0x00},  // GAIN_PGC_2[8:0]
+	{0x3095, 0x00},  // GAIN_PGC_2[8:0]
+	{0x3096, 0x00},  // GAIN_PGC_3[8:0]
+	{0x3097, 0x00},  // GAIN_PGC_3[8:0]
+	{0x30C0, 0x2A},  // XVSOUTSEL[1:0]
+	{0x30C1, 0x00},  // XVS_DRV[1:0]
+	{0x30CC, 0x00},  // -
+	{0x30CD, 0x00},  // -
+	{0x30CF, 0x00},  // XVSMSKCNT_INT[1:0]
+	{0x30D9, 0x06},  // DIG_CLP_VSTART[4:0]
+	{0x30DA, 0x02},  // DIG_CLP_VNUM[1:0]
+	{0x30E2, 0x32},  // BLKELVEL[9:0]
+	{0x30E3, 0x00},  // BLKELVEL[9:0]
+	{0x3115, 0x00},  // INCKSEL1[7:0]
+	{0x3116, 0x24},  // INCKSEL2[7:0]
+	{0x3118, 0xA0},  // INCKSEL3[10:0]
+	{0x3119, 0x00},  // INCKSEL3[10:0]
+	{0x311A, 0xE0},  // INCKSEL4[10:0]
+	{0x311B, 0x00},  // INCKSEL4[10:0]
+	{0x311E, 0x24},  // INCKSEL5[7:0]
+	{0x3260, 0x01},  // GAIN_PGC_FIDMD
+	{0x32C8, 0x01},  // -
+	{0x32D4, 0x21},  // -
+	{0x32EC, 0xA1},  // -
+	{0x344C, 0x2B},  // -
+	{0x344D, 0x01},  // -
+	{0x344E, 0xED},  // -
+	{0x344F, 0x01},  // -
+	{0x3450, 0xF6},  // -
+	{0x3451, 0x02},  // -
+	{0x3452, 0x7F},  // -
+	{0x3453, 0x03},  // -
+	{0x358A, 0x04},  // -
+	{0x35A1, 0x02},  // -
+	{0x35EC, 0x27},  // -
+	{0x35EE, 0x8D},  // -
+	{0x35F0, 0x8D},  // -
+	{0x35F2, 0x29},  // -
+	{0x36BC, 0x0C},  // -
+	{0x36CC, 0x53},  // -
+	{0x36CD, 0x00},  // -
+	{0x36CE, 0x3C},  // -
+	{0x36D0, 0x8C},  // -
+	{0x36D1, 0x00},  // -
+	{0x36D2, 0x71},  // -
+	{0x36D4, 0x3C},  // -
+	{0x36D6, 0x53},  // -
+	{0x36D7, 0x00},  // -
+	{0x36D8, 0x71},  // -
+	{0x36DA, 0x8C},  // -
+	{0x36DB, 0x00},  // -
+	{0x3701, 0x00},  // ADBIT1[7:0]
+	{0x3720, 0x00},  // -
+	{0x3724, 0x02},  // -
+	{0x3726, 0x02},  // -
+	{0x3732, 0x02},  // -
+	{0x3734, 0x03},  // -
+	{0x3736, 0x03},  // -
+	{0x3742, 0x03},  // -
+	{0x3862, 0xE0},  // -
+	{0x38CC, 0x30},  // -
+	{0x38CD, 0x2F},  // -
+	{0x395C, 0x0C},  // -
+	{0x39A4, 0x07},  // -
+	{0x39A8, 0x32},  // -
+	{0x39AA, 0x32},  // -
+	{0x39AC, 0x32},  // -
+	{0x39AE, 0x32},  // -
+	{0x39B0, 0x32},  // -
+	{0x39B2, 0x2F},  // -
+	{0x39B4, 0x2D},  // -
+	{0x39B6, 0x28},  // -
+	{0x39B8, 0x30},  // -
+	{0x39BA, 0x30},  // -
+	{0x39BC, 0x30},  // -
+	{0x39BE, 0x30},  // -
+	{0x39C0, 0x30},  // -
+	{0x39C2, 0x2E},  // -
+	{0x39C4, 0x2B},  // -
+	{0x39C6, 0x25},  // -
+	{0x3A42, 0xD1},  // -
+	{0x3A4C, 0x77},  // -
+	{0x3AE0, 0x02},  // -
+	{0x3AEC, 0x0C},  // -
+	{0x3B00, 0x2E},  // -
+	{0x3B06, 0x29},  // -
+	{0x3B98, 0x25},  // -
+	{0x3B99, 0x21},  // -
+	{0x3B9B, 0x13},  // -
+	{0x3B9C, 0x13},  // -
+	{0x3B9D, 0x13},  // -
+	{0x3B9E, 0x13},  // -
+	{0x3BA1, 0x00},  // -
+	{0x3BA2, 0x06},  // -
+	{0x3BA3, 0x0B},  // -
+	{0x3BA4, 0x10},  // -
+	{0x3BA5, 0x14},  // -
+	{0x3BA6, 0x18},  // -
+	{0x3BA7, 0x1A},  // -
+	{0x3BA8, 0x1A},  // -
+	{0x3BA9, 0x1A},  // -
+	{0x3BAC, 0xED},  // -
+	{0x3BAD, 0x01},  // -
+	{0x3BAE, 0xF6},  // -
+	{0x3BAF, 0x02},  // -
+	{0x3BB0, 0xA2},  // -
+	{0x3BB1, 0x03},  // -
+	{0x3BB2, 0xE0},  // -
+	{0x3BB3, 0x03},  // -
+	{0x3BB4, 0xE0},  // -
+	{0x3BB5, 0x03},  // -
+	{0x3BB6, 0xE0},  // -
+	{0x3BB7, 0x03},  // -
+	{0x3BB8, 0xE0},  // -
+	{0x3BBA, 0xE0},  // -
+	{0x3BBC, 0xDA},  // -
+	{0x3BBE, 0x88},  // -
+	{0x3BC0, 0x44},  // -
+	{0x3BC2, 0x7B},  // -
+	{0x3BC4, 0xA2},  // -
+	{0x3BC8, 0xBD},  // -
+	{0x3BCA, 0xBD},  // -
+	{0x4000, 0x10},  // -
+	{0x4001, 0x03},  // LANEMODE[2:0]
+	{0x4004, 0x48},  // TXCLKESC_FREQ[15:0]
+	{0x4005, 0x09},  // TXCLKESC_FREQ[15:0]
+	{0x400C, 0x01},  // INCKSEL6
+	{0x4018, 0xA7},  // TCLKPOST[15:0]
+	{0x4019, 0x00},  // TCLKPOST[15:0]
+	{0x401A, 0x57},  // TCLKPREPARE[15:0]
+	{0x401B, 0x00},  // TCLKPREPARE[15:0]
+	{0x401C, 0x5F},  // TCLKTRAIL[15:0]
+	{0x401D, 0x00},  // TCLKTRAIL[15:0]
+	{0x401E, 0x97},  // TCLKZERO[15:0]
+	{0x401F, 0x01},  // TCLKZERO[15:0]
+	{0x4020, 0x5F},  // THSPREPARE[15:0] smaller
+	{0x4021, 0x00},  // THSPREPARE[15:0]
+	{0x4022, 0xAF},  // THSZERO[15:0]	bigger
+	{0x4023, 0x00},  // THSZERO[15:0]
+	{0x4024, 0x5F},  // THSTRAIL[15:0]	bigger
+	{0x4025, 0x00},  // THSTRAIL[15:0]
+	{0x4026, 0x9F},  // THSEXIT[15:0]
+	{0x4027, 0x00},  // THSEXIT[15:0]
+	{0x4028, 0x4F},  // TLPX[15:0]
+	{0x4029, 0x00},  // TLPX[15:0]
+	{0x4074, 0x00},  // INCKSEL7[2:0]
+};
+
+static struct regval_list imx415_1920x1080_12bit_112fps_tab[] = {
+	// @@1920x1080 crop 112fps 1485Mbps/Lane
+	// MCLK: 37.125Mhz
+	// resolution: 1920x1080
+	// Mipi : 4 lane
+	// Mipi data rate: 1485Mbps/Lane
+	// FPS      :112fps
+	// HTS      :538 -> 0x021A(0x3028:0x3029)
+	// VTS      :1232 -> 0x04d0(0x3024:0x3025)
+	// PCLK/system clk     : 74235392
+	// Htime= 7.201 us; Vblank= ? ms
+	{0x3000, 0x01},  // STANDBY
+	{0x3001, 0x00},  // REGHOLD
+	{0x3002, 0x01},  // XMSTA
+	{0x3003, 0x00},  // XMASTER
+	{0x3008, 0x7F},  // BCWAIT_TIME[9:0]
+	{0x3009, 0x00},  // BCWAIT_TIME[9:0]
+	{0x300A, 0x5B},  // CPWAIT_TIME[9:0]
+	{0x300B, 0xA0},  // CPWAIT_TIME[9:0]
+	{0x301C, 0x04},  // WINMODE[3:0]
+	{0x301D, 0x08},  // -
+	{0x3020, 0x00},  // HADD
+	{0x3021, 0x00},  // VADD
+	{0x3022, 0x00},  // ADDMODE[1:0]
+	{0x3023, 0x01},  // VCMODE
+	{0x3024, 0xD0},  // VMAX[19:0]
+	{0x3025, 0x04},  // VMAX[19:0]
+	{0x3026, 0x00},  // VMAX[19:0]
+	{0x3028, 0x1A},  // HMAX[15:0]
+	{0x3029, 0x02},  // HMAX[15:0]
+	{0x302C, 0x00},  // WDMODE[1:0]
+	{0x302D, 0x00},  // WDSEL[1:0]
+	{0x3030, 0x00},  // HREVERSE
+	{0x3031, 0x01},  // ADBIT[1:0]
+	{0x3032, 0x01},  // MDBIT
+	{0x3033, 0x08},  // SYS_MODE[3:0]
+	{0x3040, 0xCC},  // PIX_HST[12:0]
+	{0x3041, 0x03},  // PIX_HST[12:0]
+	{0x3042, 0x80},  // PIX_HWIDTH[12:0]
+	{0x3043, 0x07},  // PIX_HWIDTH[12:0]
+	{0x3044, 0x58},  // PIX_VST[12:0]
+	{0x3045, 0x04},  // PIX_VST[12:0]
+	{0x3046, 0x70},  // PIX_VWIDTH[12:0]
+	{0x3047, 0x08},  // PIX_VWIDTH[12:0]
+	{0x3050, 0x08},  // SHR0[19:0]
+	{0x3051, 0x00},  // SHR0[19:0]
+	{0x3052, 0x00},  // SHR0[19:0]
+	{0x3054, 0x19},  // SHR1[19:0]
+	{0x3055, 0x00},  // SHR1[19:0]
+	{0x3056, 0x00},  // SHR1[19:0]
+	{0x3058, 0x3E},  // SHR2[19:0]
+	{0x3059, 0x00},  // SHR2[19:0]
+	{0x305A, 0x00},  // SHR2[19:0]
+	{0x305C, 0x66},  // SHR3[19:0]
+	{0x305D, 0x00},  // SHR3[19:0]
+	{0x305E, 0x00},  // SHR3[19:0]
+	{0x3060, 0x25},  // RHS1[19:0]
+	{0x3061, 0x00},  // RHS1[19:0]
+	{0x3062, 0x00},  // RHS1[19:0]
+	{0x3064, 0x4A},  // RHS2[19:0]
+	{0x3065, 0x00},  // RHS2[19:0]
+	{0x3066, 0x00},  // RHS2[19:0]
+	{0x3090, 0x00},  // GAIN_PGC_0[8:0]
+	{0x3091, 0x00},  // GAIN_PGC_0[8:0]
+	{0x3092, 0x00},  // GAIN_PGC_1[8:0]
+	{0x3093, 0x00},  // GAIN_PGC_1[8:0]
+	{0x3094, 0x00},  // GAIN_PGC_2[8:0]
+	{0x3095, 0x00},  // GAIN_PGC_2[8:0]
+	{0x3096, 0x00},  // GAIN_PGC_3[8:0]
+	{0x3097, 0x00},  // GAIN_PGC_3[8:0]
+	{0x30C0, 0x2A},  // XVSOUTSEL[1:0]
+	{0x30C1, 0x00},  // XVS_DRV[1:0]
+	{0x30CC, 0x00},  // -
+	{0x30CD, 0x00},  // -
+	{0x30CF, 0x00},  // XVSMSKCNT_INT[1:0]
+	{0x30D9, 0x06},  // DIG_CLP_VSTART[4:0]
+	{0x30DA, 0x02},  // DIG_CLP_VNUM[1:0]
+	{0x30E2, 0x32},  // BLKELVEL[9:0]
+	{0x30E3, 0x00},  // BLKELVEL[9:0]
+	{0x3115, 0x00},  // INCKSEL1[7:0]
+	{0x3116, 0x24},  // INCKSEL2[7:0]
+	{0x3118, 0xA0},  // INCKSEL3[10:0]
+	{0x3119, 0x00},  // INCKSEL3[10:0]
+	{0x311A, 0xE0},  // INCKSEL4[10:0]
+	{0x311B, 0x00},  // INCKSEL4[10:0]
+	{0x311E, 0x24},  // INCKSEL5[7:0]
+	{0x3260, 0x01},  // GAIN_PGC_FIDMD
+	{0x32C8, 0x01},  // -
+	{0x32D4, 0x21},  // -
+	{0x32EC, 0xA1},  // -
+	{0x344C, 0x2B},  // -
+	{0x344D, 0x01},  // -
+	{0x344E, 0xED},  // -
+	{0x344F, 0x01},  // -
+	{0x3450, 0xF6},  // -
+	{0x3451, 0x02},  // -
+	{0x3452, 0x7F},  // -
+	{0x3453, 0x03},  // -
+	{0x358A, 0x04},  // -
+	{0x35A1, 0x02},  // -
+	{0x35EC, 0x27},  // -
+	{0x35EE, 0x8D},  // -
+	{0x35F0, 0x8D},  // -
+	{0x35F2, 0x29},  // -
+	{0x36BC, 0x0C},  // -
+	{0x36CC, 0x53},  // -
+	{0x36CD, 0x00},  // -
+	{0x36CE, 0x3C},  // -
+	{0x36D0, 0x8C},  // -
+	{0x36D1, 0x00},  // -
+	{0x36D2, 0x71},  // -
+	{0x36D4, 0x3C},  // -
+	{0x36D6, 0x53},  // -
+	{0x36D7, 0x00},  // -
+	{0x36D8, 0x71},  // -
+	{0x36DA, 0x8C},  // -
+	{0x36DB, 0x00},  // -
+	{0x3701, 0x00},  // ADBIT1[7:0]
+	{0x3720, 0x00},  // -
+	{0x3724, 0x02},  // -
+	{0x3726, 0x02},  // -
+	{0x3732, 0x02},  // -
+	{0x3734, 0x03},  // -
+	{0x3736, 0x03},  // -
+	{0x3742, 0x03},  // -
+	{0x3862, 0xE0},  // -
+	{0x38CC, 0x30},  // -
+	{0x38CD, 0x2F},  // -
+	{0x395C, 0x0C},  // -
+	{0x39A4, 0x07},  // -
+	{0x39A8, 0x32},  // -
+	{0x39AA, 0x32},  // -
+	{0x39AC, 0x32},  // -
+	{0x39AE, 0x32},  // -
+	{0x39B0, 0x32},  // -
+	{0x39B2, 0x2F},  // -
+	{0x39B4, 0x2D},  // -
+	{0x39B6, 0x28},  // -
+	{0x39B8, 0x30},  // -
+	{0x39BA, 0x30},  // -
+	{0x39BC, 0x30},  // -
+	{0x39BE, 0x30},  // -
+	{0x39C0, 0x30},  // -
+	{0x39C2, 0x2E},  // -
+	{0x39C4, 0x2B},  // -
+	{0x39C6, 0x25},  // -
+	{0x3A42, 0xD1},  // -
+	{0x3A4C, 0x77},  // -
+	{0x3AE0, 0x02},  // -
+	{0x3AEC, 0x0C},  // -
+	{0x3B00, 0x2E},  // -
+	{0x3B06, 0x29},  // -
+	{0x3B98, 0x25},  // -
+	{0x3B99, 0x21},  // -
+	{0x3B9B, 0x13},  // -
+	{0x3B9C, 0x13},  // -
+	{0x3B9D, 0x13},  // -
+	{0x3B9E, 0x13},  // -
+	{0x3BA1, 0x00},  // -
+	{0x3BA2, 0x06},  // -
+	{0x3BA3, 0x0B},  // -
+	{0x3BA4, 0x10},  // -
+	{0x3BA5, 0x14},  // -
+	{0x3BA6, 0x18},  // -
+	{0x3BA7, 0x1A},  // -
+	{0x3BA8, 0x1A},  // -
+	{0x3BA9, 0x1A},  // -
+	{0x3BAC, 0xED},  // -
+	{0x3BAD, 0x01},  // -
+	{0x3BAE, 0xF6},  // -
+	{0x3BAF, 0x02},  // -
+	{0x3BB0, 0xA2},  // -
+	{0x3BB1, 0x03},  // -
+	{0x3BB2, 0xE0},  // -
+	{0x3BB3, 0x03},  // -
+	{0x3BB4, 0xE0},  // -
+	{0x3BB5, 0x03},  // -
+	{0x3BB6, 0xE0},  // -
+	{0x3BB7, 0x03},  // -
+	{0x3BB8, 0xE0},  // -
+	{0x3BBA, 0xE0},  // -
+	{0x3BBC, 0xDA},  // -
+	{0x3BBE, 0x88},  // -
+	{0x3BC0, 0x44},  // -
+	{0x3BC2, 0x7B},  // -
+	{0x3BC4, 0xA2},  // -
+	{0x3BC8, 0xBD},  // -
+	{0x3BCA, 0xBD},  // -
+	{0x4000, 0x10},  // -
+	{0x4001, 0x03},  // LANEMODE[2:0]
+	{0x4004, 0x48},  // TXCLKESC_FREQ[15:0]
+	{0x4005, 0x09},  // TXCLKESC_FREQ[15:0]
+	{0x400C, 0x01},  // INCKSEL6
+	{0x4018, 0xA7},  // TCLKPOST[15:0]
+	{0x4019, 0x00},  // TCLKPOST[15:0]
+	{0x401A, 0x57},  // TCLKPREPARE[15:0]
+	{0x401B, 0x00},  // TCLKPREPARE[15:0]
+	{0x401C, 0x5F},  // TCLKTRAIL[15:0]
+	{0x401D, 0x00},  // TCLKTRAIL[15:0]
+	{0x401E, 0x97},  // TCLKZERO[15:0]
+	{0x401F, 0x01},  // TCLKZERO[15:0]
+	{0x4020, 0x5F},  // THSPREPARE[15:0] smaller
+	{0x4021, 0x00},  // THSPREPARE[15:0]
+	{0x4022, 0xAF},  // THSZERO[15:0]	bigger
+	{0x4023, 0x00},  // THSZERO[15:0]
+	{0x4024, 0x5F},  // THSTRAIL[15:0]	bigger
+	{0x4025, 0x00},  // THSTRAIL[15:0]
+	{0x4026, 0x9F},  // THSEXIT[15:0]
+	{0x4027, 0x00},  // THSEXIT[15:0]
+	{0x4028, 0x4F},  // TLPX[15:0]
+	{0x4029, 0x00},  // TLPX[15:0]
+	{0x4074, 0x00},  // INCKSEL7[2:0]
+    };
+    
+/*
+ * 1920x1080@60fps 12bit 4-lane MIPI configuration
+ * From sensor_init_regs_1920_1080_60fps_mipi[]
+ *
+ * Key registers:
+ * - 0x3033: 0x08 (SYS_MODE[3:0])
+ * - 0x3024: VMAX[19:0] = 0x0B4C (2892)
+ * - 0x3028: HMAX[15:0] = 0x016D (365)
+ */
+__maybe_unused static struct regval_list imx415_1920x1080_60fps_12bpp_4lane_regs[] = {
+	{0x3008, 0x7F},	/* BCWAIT_TIME[9:0] */
+	{0x300A, 0x5B},	/* CPWAIT_TIME[9:0] */
+	{0x3020, 0x01},	/* HADD */
+	{0x3021, 0x01},	/* VADD */
+	{0x3022, 0x01},	/* ADDMODE[1:0] */
+	{0x3024, 0x4C},	/* VMAX[19:0] LSB */
+	{0x3025, 0x0B},	/* VMAX[19:0] MSB */
+	{0x3028, 0x6D},	/* HMAX[15:0] LSB */
+	{0x3029, 0x01},	/* HMAX[15:0] MSB */
+	{0x3031, 0x00},	/* ADBIT[1:0] */
+	{0x3033, 0x08},	/* SYS_MODE[3:0] - MIPI mode */
+	{0x3050, 0x08},	/* SHR0[19:0] LSB */
+	{0x30C1, 0x00},	/* XVS_DRV[1:0] */
+	{0x30D9, 0x02},	/* DIG_CLP_VSTART[4:0] */
+	{0x30DA, 0x01},	/* DIG_CLP_VNUM[1:0] */
+	{0x3116, 0x24},	/* INCKSEL2[7:0] */
+	{0x3118, 0xA0},	/* INCKSEL3[10:0] LSB */
+	{0x311E, 0x24},	/* INCKSEL5[7:0] */
+	{0x32D4, 0x21},	/* */
+	{0x32EC, 0xA1},	/* */
+	{0x344C, 0x2B},	/* */
+	{0x344D, 0x01},	/* */
+	{0x344E, 0xED},	/* */
+	{0x344F, 0x01},	/* */
+	{0x3450, 0xF6},	/* */
+	{0x3451, 0x02},	/* */
+	{0x3452, 0x7F},	/* */
+	{0x3453, 0x03},	/* */
+	{0x358A, 0x04},	/* */
+	{0x35A1, 0x02},	/* */
+	{0x35EC, 0x27},	/* */
+	{0x35EE, 0x8D},	/* */
+	{0x35F0, 0x8D},	/* */
+	{0x35F2, 0x29},	/* */
+	{0x36BC, 0x0C},	/* */
+	{0x36CC, 0x53},	/* */
+	{0x36CD, 0x00},	/* */
+	{0x36CE, 0x3C},	/* */
+	{0x36D0, 0x8C},	/* */
+	{0x36D1, 0x00},	/* */
+	{0x36D2, 0x71},	/* */
+	{0x36D4, 0x3C},	/* */
+	{0x36D6, 0x53},	/* */
+	{0x36D7, 0x00},	/* */
+	{0x36D8, 0x71},	/* */
+	{0x36DA, 0x8C},	/* */
+	{0x36DB, 0x00},	/* */
+	{0x3701, 0x00},	/* ADBIT1[7:0] */
+	{0x3720, 0x00},	/* */
+	{0x3724, 0x02},	/* */
+	{0x3726, 0x02},	/* */
+	{0x3732, 0x02},	/* */
+	{0x3734, 0x03},	/* */
+	{0x3736, 0x03},	/* */
+	{0x3742, 0x03},	/* */
+	{0x3862, 0xE0},	/* */
+	{0x38CC, 0x30},	/* */
+	{0x38CD, 0x2F},	/* */
+	{0x395C, 0x0C},	/* */
+	{0x39A4, 0x07},	/* */
+	{0x39A8, 0x32},	/* */
+	{0x39AA, 0x32},	/* */
+	{0x39AC, 0x32},	/* */
+	{0x39AE, 0x32},	/* */
+	{0x39B0, 0x32},	/* */
+	{0x39B2, 0x2F},	/* */
+	{0x39B4, 0x2D},	/* */
+	{0x39B6, 0x28},	/* */
+	{0x39B8, 0x30},	/* */
+	{0x39BA, 0x30},	/* */
+	{0x39BC, 0x30},	/* */
+	{0x39BE, 0x30},	/* */
+	{0x39C0, 0x30},	/* */
+	{0x39C2, 0x2E},	/* */
+	{0x39C4, 0x2B},	/* */
+	{0x39C6, 0x25},	/* */
+	{0x3A42, 0xD1},	/* */
+	{0x3A4C, 0x77},	/* */
+	{0x3AE0, 0x02},	/* */
+	{0x3AEC, 0x0C},	/* */
+	{0x3B00, 0x2E},	/* */
+	{0x3B06, 0x29},	/* */
+	{0x3B98, 0x25},	/* */
+	{0x3B99, 0x21},	/* */
+	{0x3B9B, 0x13},	/* */
+	{0x3B9C, 0x13},	/* */
+	{0x3B9D, 0x13},	/* */
+	{0x3B9E, 0x13},	/* */
+	{0x3BA1, 0x00},	/* */
+	{0x3BA2, 0x06},	/* */
+	{0x3BA3, 0x0B},	/* */
+	{0x3BA4, 0x10},	/* */
+	{0x3BA5, 0x14},	/* */
+	{0x3BA6, 0x18},	/* */
+	{0x3BA7, 0x1A},	/* */
+	{0x3BA8, 0x1A},	/* */
+	{0x3BA9, 0x1A},	/* */
+	{0x3BAC, 0xED},	/* */
+	{0x3BAD, 0x01},	/* */
+	{0x3BAE, 0xF6},	/* */
+	{0x3BAF, 0x02},	/* */
+	{0x3BB0, 0xA2},	/* */
+	{0x3BB1, 0x03},	/* */
+	{0x3BB2, 0xE0},	/* */
+	{0x3BB3, 0x03},	/* */
+	{0x3BB4, 0xE0},	/* */
+	{0x3BB5, 0x03},	/* */
+	{0x3BB6, 0xE0},	/* */
+	{0x3BB7, 0x03},	/* */
+	{0x3BB8, 0xE0},	/* */
+	{0x3BBA, 0xE0},	/* */
+	{0x3BBC, 0xDA},	/* */
+	{0x3BBE, 0x88},	/* */
+	{0x3BC0, 0x44},	/* */
+	{0x3BC2, 0x7B},	/* */
+	{0x3BC4, 0xA2},	/* */
+	{0x3BC8, 0xBD},	/* */
+	{0x3BCA, 0xBD},	/* */
+	{0x4004, 0x48},	/* TXCLKESC_FREQ[15:0] */
+	{0x4005, 0x09},	/* */
+	{0x4018, 0xA7},	/* TCLKPOST[15:0] */
+	{0x401A, 0x57},	/* TCLKPREPARE[15:0] */
+	{0x401C, 0x5F},	/* TCLKTRAIL[15:0] */
+	{0x401E, 0x97},	/* TCLKZERO[15:0] */
+	{0x4020, 0x5F},	/* THSPREPARE[15:0] */
+	{0x4022, 0xAF},	/* THSZERO[15:0] */
+	{0x4024, 0x5F},	/* THSTRAIL[15:0] */
+	{0x4026, 0x9F},	/* THSEXIT[15:0] */
+	{0x4028, 0x4F},	/* TLPX[15:0] */
+	{0x3000, 0x00},	/* STANDBY */
+};
+
+static int imx415_write(struct imx415 *sensor, u16 reg, u8 val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msg;
+	u8 data[3];
+	int ret;
+
+	data[0] = (reg >> 8) & 0xFF;
+	data[1] = reg & 0xFF;
+	data[2] = val & 0xFF;
+
+	msg.addr = sensor->client->addr;
+	msg.flags = 0;
+	msg.len = 3;
+	msg.buf = data;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, &msg, 1);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 1) {
+		dev_err(&sensor->client->dev,
+			"imx415-test: I2C write failed, reg=0x%04x, val=0x%02x, ret=%d\n",
+			reg, val, ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	return 0;
+}
+
+static int imx415_read(struct imx415 *sensor, u16 reg, u8 *val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msgs[2];
+	u8 reg_buf[2];
+	u8 data_buf[1];
+	int ret;
+
+	reg_buf[0] = (reg >> 8) & 0xFF;
+	reg_buf[1] = reg & 0xFF;
+
+	msgs[0].addr = sensor->client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 2;
+	msgs[0].buf = reg_buf;
+
+	msgs[1].addr = sensor->client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = 1;
+	msgs[1].buf = data_buf;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, msgs, 2);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 2) {
+		dev_err(&sensor->client->dev,
+			"imx415-test: I2C read failed, reg=0x%04x, ret=%d\n",
+			reg, ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	*val = data_buf[0];
+	return 0;
+}
+
+static int imx415_write_array(struct imx415 *sensor, struct regval_list *vals)
+{
+	int ret;
+
+	while (vals->addr != SENSOR_REG_END) {
+		if (vals->addr == SENSOR_REG_DELAY) {
+			msleep(vals->data);
+		} else {
+			ret = imx415_write(sensor, vals->addr, vals->data);
+			if (ret < 0)
+				return ret;
+		}
+		vals++;
+	}
+
+	return 0;
+}
+
+static int imx415_stream_on(struct imx415 *sensor)
+{
+	imx415_write(sensor, 0x3000, 0x00);
+	imx415_write(sensor, 0x3002, 0x00);
+	return 0;
+}
+
+static int imx415_stream_off(struct imx415 *sensor)
+{
+	return imx415_write(sensor, 0x3000, 0x01);
+}
+
+static int imx415_detect(struct imx415 *sensor)
+{
+	u8 lo, hi;
+	u16 chip_id;
+	int ret;
+
+	dev_info(&sensor->client->dev, "imx415-test: detect start\n");
+
+	/*
+	 * The sensor info register (0x3F12, 16-bit LE) can only be read when
+	 * the sensor is NOT in standby mode. Wake the sensor up first.
+	 */
+	ret = imx415_write(sensor, 0x3000, 0x00); /* STANDBY -> Operating */
+	if (ret < 0)
+		return ret;
+	/* Wait for sensor to wake up */
+	usleep_range(30000, 40000);
+
+	/* Read SENSOR_INFO register at 0x3F12 (16-bit, little-endian) */
+	ret = imx415_read(sensor, 0x3f12, &lo);
+	if (ret < 0) {
+		dev_err(&sensor->client->dev,
+			"imx415-test: failed to read sensor info low byte\n");
+		goto done;
+	}
+
+	ret = imx415_read(sensor, 0x3f13, &hi);
+	if (ret < 0) {
+		dev_err(&sensor->client->dev,
+			"imx415-test: failed to read sensor info high byte\n");
+		goto done;
+	}
+
+	chip_id = ((u16)hi << 8) | lo;
+	chip_id &= 0x0FFF; /* IMX415_SENSOR_INFO_MASK */
+
+	if (chip_id != 0x0514) { /* IMX415_CHIP_ID */
+		dev_err(&sensor->client->dev,
+			"imx415-test: chip ID mismatch: expected 0x0514, got 0x%04x\n",
+			chip_id);
+		ret = -ENODEV;
+		goto done;
+	}
+
+	dev_info(&sensor->client->dev,
+		 "imx415-test: Detected IMX415 image sensor (chip_id=0x%04x)\n",
+		 chip_id);
+	usleep_range(10000, 20000);
+	ret = 0;
+
+done:
+	/* Put sensor back to standby */
+	imx415_write(sensor, 0x3000, 0x01);
+	return ret;
+}
+
+static int imx415_power_on(struct imx415 *sensor)
+{
+	int ret = 0;
+
+	dev_info(&sensor->client->dev, "imx415-test: power_on enter\n");
+
+	/* Set I2C mux to select this sensor */
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "imx415-test: set i2c-mux high\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 1);
+	}
+
+	dev_info(&sensor->client->dev, "imx415-test: get vdd regulator\n");
+	sensor->vdd = devm_regulator_get(&sensor->client->dev, "vdd");
+	if (IS_ERR(sensor->vdd)) {
+		dev_err(&sensor->client->dev, "imx415-test: Failed to get vdd regulator: %ld\n",
+			PTR_ERR(sensor->vdd));
+		return PTR_ERR(sensor->vdd);
+	}
+
+	if (sensor->vdd) {
+		ret = regulator_enable(sensor->vdd);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev,
+				"imx415-test: failed to enable vdd: %d\n", ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "imx415-test: enable vdd\n");
+
+		ret = regulator_set_voltage(sensor->vdd, 3300000, 3300000);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev, "imx415-test: failed to set vdd voltage: %d\n",
+				ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "imx415-test: vdd set to 3.3V\n");
+	}
+
+	if (sensor->pwdn) {
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+		usleep_range(10000, 20000);
+		gpiod_set_value_cansleep(sensor->pwdn, 1);
+		usleep_range(10000, 20000);
+	}
+
+	dev_info(&sensor->client->dev, "imx415-test: power_on done\n");
+
+	return 0;
+}
+
+static void imx415_power_off(struct imx415 *sensor)
+{
+	dev_info(&sensor->client->dev, "imx415-test: power_off enter\n");
+
+	if (sensor->pwdn) {
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	}
+
+	if (sensor->vdd) {
+		dev_info(&sensor->client->dev, "imx415-test: disable vdd\n");
+		regulator_disable(sensor->vdd);
+	}
+
+	/* Clear I2C mux */
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "imx415-test: set i2c-mux low\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 0);
+	}
+
+	dev_info(&sensor->client->dev, "imx415-test: power_off done\n");
+}
+
+static long imx415_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct imx415 *sensor = file->private_data;
+	int ret = 0;
+
+	if (!sensor)
+		return -ENODEV;
+
+	switch (cmd) {
+	case IMX415_IOCTL_POWER_ON:
+		ret = imx415_power_on(sensor);
+		break;
+	case IMX415_IOCTL_POWER_OFF:
+		imx415_power_off(sensor);
+		ret = 0;
+		break;
+	case IMX415_IOCTL_INIT_REGS:
+		ret = imx415_write_array(
+			sensor, imx415_1920x1080_12bit_112fps_tab);
+		break;
+	case IMX415_IOCTL_STREAM_ON:
+		ret = imx415_stream_on(sensor);
+		break;
+	case IMX415_IOCTL_STREAM_OFF:
+		ret = imx415_stream_off(sensor);
+		break;
+	case IMX415_IOCTL_DETECT:
+		ret = imx415_detect(sensor);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int imx415_dev_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *misc = file->private_data;
+	struct imx415 *sensor;
+
+	if (!misc)
+		return -ENODEV;
+
+	sensor = container_of(misc, struct imx415, miscdev);
+	file->private_data = sensor;
+	return 0;
+}
+
+static const struct file_operations imx415_fops = {
+	.owner = THIS_MODULE,
+	.open = imx415_dev_open,
+	.unlocked_ioctl = imx415_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = imx415_ioctl,
+#endif
+};
+
+static int imx415_probe(struct i2c_client *client)
+{
+	struct imx415 *sensor;
+	struct device *dev = &client->dev;
+	int ret;
+
+	dev_info(dev, "imx415-test: probe enter, client addr=0x%02x\n",
+		 client->addr);
+
+	sensor = devm_kzalloc(dev, sizeof(*sensor), GFP_KERNEL);
+	if (!sensor)
+		return -ENOMEM;
+
+	sensor->client = client;
+	mutex_init(&sensor->lock);
+	i2c_set_clientdata(client, sensor);
+
+	/* Get power-down GPIO */
+	sensor->pwdn = devm_gpiod_get_optional(
+		dev, "pwdn", GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->pwdn)) {
+		dev_err(dev, "imx415-test: Failed to get pwdn GPIO\n");
+		goto err_pwdn;
+	}
+
+	/* Get I2C mux GPIO (optional) */
+	sensor->i2c_mux = devm_gpiod_get_optional(dev, "i2c-mux",
+		GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->i2c_mux)) {
+		dev_warn(dev, "imx415-test: Failed to get i2c-mux GPIO, continuing without it\n");
+		sensor->i2c_mux = NULL;
+	}
+
+	/* Power on and detect sensor before creating device node */
+	ret = imx415_power_on(sensor);
+	if (ret) {
+		dev_err(dev, "imx415-test: power on failed: %d\n", ret);
+		goto err_pwdn;
+	}
+
+	ret = imx415_detect(sensor);
+	if (ret) {
+		dev_err(dev, "imx415-test: sensor detect failed: %d\n", ret);
+		goto err_detect;
+	}
+
+	imx415_power_off(sensor);
+
+	sensor->miscdev.minor = MISC_DYNAMIC_MINOR;
+	sensor->miscdev.fops = &imx415_fops;
+	sensor->miscdev.parent = dev;
+
+	/* Generate device name from CSI ID or I2C address */
+	if (client->dev.of_node) {
+		u32 csi_id;
+		if (of_property_read_u32(client->dev.of_node, "csi-id",
+					 &csi_id) == 0) {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "imx415-%u", csi_id);
+			dev_info(dev, "imx415-test: imx415-%u\n", csi_id);
+		} else {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "imx415-%02x", client->addr);
+			dev_info(dev, "imx415-test: imx415-%02x\n",
+				 client->addr);
+		}
+	} else {
+		sensor->miscdev.name = devm_kasprintf(
+			dev, GFP_KERNEL, "imx415-%02x", client->addr);
+	}
+
+	ret = misc_register(&sensor->miscdev);
+	if (ret) {
+		dev_err(dev,
+			"imx415-test: failed to register misc device: %d\n",
+			ret);
+		goto err_misc_register;
+	}
+
+	global_imx415 = sensor;
+	dev_info(dev, "imx415-test: probe successful, ioctl device /dev/%s\n",
+		 sensor->miscdev.name);
+	return 0;
+
+err_misc_register:
+	imx415_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+	return ret;
+
+err_detect:
+	imx415_power_off(sensor);
+err_pwdn:
+	mutex_destroy(&sensor->lock);
+	return ret;
+}
+
+static void imx415_remove(struct i2c_client *client)
+{
+	struct imx415 *sensor = i2c_get_clientdata(client);
+
+	dev_info(&client->dev, "imx415-test: remove\n");
+	if (global_imx415 == sensor)
+		global_imx415 = NULL;
+
+	misc_deregister(&sensor->miscdev);
+	imx415_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+}
+
+static const struct of_device_id imx415_of_match[] = {
+	{ .compatible = "sony,imx415" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, imx415_of_match);
+
+static struct i2c_driver imx415_driver = {
+	.driver = {
+		.name		= "imx415-simple",
+		.of_match_table	= of_match_ptr(imx415_of_match),
+	},
+	.probe		= imx415_probe,
+	.remove		= imx415_remove,
+};
+
+module_i2c_driver(imx415_driver)
+
+MODULE_DESCRIPTION("Simplified I2C driver for IMX415");
+MODULE_LICENSE("GPL v2");
+
diff --git a/drivers/media/platform/spacemit/cam_sensor/ov2735_sensor.c b/drivers/media/platform/spacemit/cam_sensor/ov2735_sensor.c
new file mode 100755
index 000000000000..5dadf68addcc
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_sensor/ov2735_sensor.c
@@ -0,0 +1,625 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Simplified I2C driver for OmniVision OV2735
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/ioctl.h>
+#include <linux/regulator/consumer.h>
+
+/*
+ * Sensor Configuration: 1920x1080 @ 30fps, 1-lane MIPI
+ *
+ * MCLK:           24MHz
+ * Resolution:     1920x1080
+ * Bit Depth:      10bit
+ * FPS:            30fps
+ * HTS:            1053 - registers P1:0x8C 0x8D
+ * VTS:            1329 - registers P1:0x4E 0x4F
+ * MIPI Data Rate: 105Mbps/Lane
+ */
+
+/* IOCTL interface for user-space control */
+#define OV2735_IOC_MAGIC		'I'
+#define OV2735_IOCTL_POWER_ON		_IO(OV2735_IOC_MAGIC, 1)
+#define OV2735_IOCTL_POWER_OFF		_IO(OV2735_IOC_MAGIC, 2)
+#define OV2735_IOCTL_INIT_REGS		_IO(OV2735_IOC_MAGIC, 3)
+#define OV2735_IOCTL_STREAM_ON		_IO(OV2735_IOC_MAGIC, 4)
+#define OV2735_IOCTL_STREAM_OFF		_IO(OV2735_IOC_MAGIC, 5)
+#define OV2735_IOCTL_DETECT		_IO(OV2735_IOC_MAGIC, 6)
+
+static struct ov2735 *global_ov2735;
+
+struct ov2735 {
+	struct i2c_client *client;
+	struct gpio_desc *pwdn;
+	struct gpio_desc *i2c_mux;
+	struct clk *mclk;
+	struct mutex lock;
+	struct regulator *vdd;
+	struct miscdevice miscdev;
+};
+
+struct regval_list {
+	u8 addr;
+	u8 data;
+};
+struct regval_list ov2735_spm_1920x1080_10bit_30fps_1lane_tab[] = {
+	{0xfd, 0x00}, /* page flag */
+	/* {0x20, 0x00}, soft reset */
+	/* {0xfd, 0x00}, page flag */
+	{0x2f, 0x10}, /* PLL_CTRL_BUF */
+	{0x34, 0x00},
+	{0x30, 0x15},
+	{0x33, 0x01},
+	{0x35, 0x20},
+
+	{0xfd, 0x01},
+	{0x0d, 0x00},
+	{0x30, 0x00},
+	{0x03, 0x01},
+	{0x04, 0x8f},
+	{0x01, 0x01},
+	{0x09, 0x00},
+	{0x0a, 0x20},
+	{0x06, 0x0a},
+	{0x24, 0x10},
+	{0x01, 0x01},
+	{0xfb, 0x73},
+	{0x01, 0x01},
+
+	{0xfd, 0x01},
+	{0x1a, 0x6b},
+	{0x1c, 0xea},
+	{0x16, 0x0c},
+	{0x21, 0x00},
+	{0x11, 0xe8}, /* RST_NUM_8LSB dac */
+	{0x19, 0xc3},
+	{0x26, 0xda}, /* ANALOG CTRL4 reference voltage */
+	{0x29, 0x01},
+	{0x33, 0x6f},
+	{0x2a, 0xd2},
+	{0x2c, 0x40},
+	{0xd0, 0x02},
+	{0xd1, 0x01},
+	{0xd2, 0x20},
+	{0xd3, 0x03}, /* B4_NUM_3MSB */
+	{0xd4, 0xa4}, /* B4_NUM_8LSB */
+	{0x50, 0x00},
+	{0x51, 0x2c},
+	{0x52, 0x29},
+	{0x53, 0x00},
+	{0x55, 0x44},
+	{0x58, 0x29},
+	{0x5a, 0x00},
+	{0x5b, 0x00},
+	{0x5d, 0x00},
+	{0x64, 0x2f},
+	{0x66, 0x62},
+	{0x68, 0x5b},
+	{0x75, 0x46},
+	{0x76, 0xf0}, /* P34 Cycle for Pixel Timing */
+	{0x77, 0x4f},
+	{0x78, 0xef},
+	{0x72, 0xcf},
+	{0x73, 0x36},
+	{0x7d, 0x0d},
+	{0x7e, 0x0d},
+	{0x8a, 0x77},
+	{0x8b, 0x77},
+
+	{0xfd, 0x01},
+	{0xb1, 0x83},
+	{0xb3, 0x0b},
+	{0xb4, 0x14},
+	{0x9d, 0x40},
+	{0xa1, 0x05},
+	{0x94, 0x44},
+	{0x95, 0x33},
+	{0x96, 0x1f},
+	{0x98, 0x45},
+	{0x9c, 0x10},
+	{0xb5, 0x70},
+	/* {0xa0, 0x01}, MIPI EN BUF */
+	{0x25, 0xe0},
+	{0x20, 0x7b},
+	{0x8f, 0x88},
+	{0x91, 0x40},
+
+	{0xfd, 0x02}, /* PAGE FLAG */
+	{0x5e, 0x03},
+	{0xa1, 0x04},
+	{0xa3, 0x40},
+	{0xa5, 0x02},
+	{0xa7, 0xc4},
+
+	{0xfd, 0x01},
+	{0x86, 0x77},
+	{0x89, 0x77},
+	{0x87, 0x74},
+	{0x88, 0x74},
+	{0xfc, 0xe0},
+	{0xfe, 0xe0},
+	{0xf0, 0x40},
+	{0xf1, 0x40},
+	{0xf2, 0x40},
+	{0xf3, 0x40},
+	{0xb2, 0x00}, /* MIPI CTRL4 single lane mipi LP mode */
+
+	/* crop to 1920x1080 */
+	{0xfd, 0x02},
+	{0xa0, 0x00},
+	{0xa1, 0x08},
+	{0xa2, 0x04},
+	{0xa3, 0x38}, /* image vertical size 1088 */
+	{0xa4, 0x00},
+	{0xa5, 0x04},
+	{0xa6, 0x03},
+	{0xa7, 0xc0}, /* image half horizontal size 964 */
+
+	{0xfd, 0x01},
+	{0x8e, 0x07},
+	{0x8f, 0x80},
+	{0x90, 0x04},
+	{0x91, 0x38},
+
+	{0xfd, 0x03},
+	{0xc0, 0x01},
+	{0xfd, 0x04},
+	{0x21, 0x14},
+	{0x22, 0x14},
+	{0x23, 0x14},
+
+	{0xfd, 0x01},
+	{0x06, 0xe0},
+	{0x01, 0x01},
+	/* {0xa0, 0x01}, MIPI enable, stream on */
+};
+
+static int ov2735_write(struct ov2735 *sensor, u8 reg, u8 val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msg;
+	u8 data[2];
+	int ret;
+
+	data[0] = reg;
+	data[1] = val;
+
+	msg.addr = sensor->client->addr;
+	msg.flags = 0;
+	msg.len = sizeof(data);
+	msg.buf = data;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, &msg, 1);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 1) {
+		dev_err(&sensor->client->dev,
+			"ov2735-test: I2C write failed, reg=0x%x, val=0x%x, ret=%d\n",
+			reg, val, ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	return 0;
+}
+
+static int ov2735_read(struct ov2735 *sensor, u8 reg, u8 *val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msgs[2];
+	u8 reg_buf[1];
+	u8 data_buf[1];
+	int ret;
+
+	reg_buf[0] = reg;
+
+	msgs[0].addr = sensor->client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = sizeof(reg_buf);
+	msgs[0].buf = reg_buf;
+
+	msgs[1].addr = sensor->client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = sizeof(data_buf);
+	msgs[1].buf = data_buf;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, msgs, 2);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 2) {
+		dev_err(&sensor->client->dev,
+			"ov2735-test: I2C read failed, reg=0x%x, ret=%d\n", reg,
+			ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	*val = data_buf[0];
+	return 0;
+}
+
+static int ov2735_stream_on(struct ov2735 *sensor)
+{
+	int ret;
+
+	ret = ov2735_write(sensor, 0xfd, 0x01); /* page 1 */
+	if (ret < 0)
+		return ret;
+	return ov2735_write(sensor, 0xa0, 0x01); /* MIPI enable, stream on */
+}
+
+static int ov2735_stream_off(struct ov2735 *sensor)
+{
+	int ret;
+
+	ret = ov2735_write(sensor, 0xfd, 0x01); /* page 1 */
+	if (ret < 0)
+		return ret;
+	return ov2735_write(sensor, 0xa0, 0x00); /* MIPI disable, stream off */
+}
+
+static int ov2735_write_init_regs(struct ov2735 *sensor);
+static int ov2735_power_on(struct ov2735 *sensor);
+static void ov2735_power_off(struct ov2735 *sensor);
+static int ov2735_detect(struct ov2735 *sensor);
+
+static long ov2735_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct ov2735 *sensor = file->private_data;
+	int ret = 0;
+
+	if (!sensor)
+		return -ENODEV;
+
+	switch (cmd) {
+	case OV2735_IOCTL_POWER_ON:
+		ret = ov2735_power_on(sensor);
+		break;
+	case OV2735_IOCTL_POWER_OFF:
+		ov2735_power_off(sensor);
+		ret = 0;
+		break;
+	case OV2735_IOCTL_INIT_REGS:
+		ret = ov2735_write_init_regs(sensor);
+		break;
+	case OV2735_IOCTL_STREAM_ON:
+		ret = ov2735_stream_on(sensor);
+		break;
+	case OV2735_IOCTL_STREAM_OFF:
+		ret = ov2735_stream_off(sensor);
+		break;
+	case OV2735_IOCTL_DETECT:
+		ret = ov2735_detect(sensor);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int ov2735_dev_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *misc = file->private_data;
+	struct ov2735 *sensor;
+
+	if (!misc)
+		return -ENODEV;
+
+	sensor = container_of(misc, struct ov2735, miscdev);
+	/* Replace private_data with sensor pointer for use in ioctl */
+	file->private_data = sensor;
+	return 0;
+}
+
+static const struct file_operations ov2735_fops = {
+	.owner = THIS_MODULE,
+	.open = ov2735_dev_open,
+	.unlocked_ioctl = ov2735_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = ov2735_ioctl,
+#endif
+};
+
+static int ov2735_write_init_regs(struct ov2735 *sensor)
+{
+	int i, ret;
+
+	dev_info(&sensor->client->dev,
+		 "ov2735-test: write init regs, total=%zu\n",
+		 ARRAY_SIZE(ov2735_spm_1920x1080_10bit_30fps_1lane_tab));
+	for (i = 0; i < ARRAY_SIZE(ov2735_spm_1920x1080_10bit_30fps_1lane_tab); i++) {
+		const struct regval_list *reg = &ov2735_spm_1920x1080_10bit_30fps_1lane_tab[i];
+
+		ret = ov2735_write(sensor, reg->addr, reg->data);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev,
+				"ov2735-test: write init reg failed, idx=%d, reg=0x%x, val=0x%x, ret=%d\n",
+				i, reg->addr, reg->data, ret);
+			return ret;
+		}
+	}
+
+	dev_info(&sensor->client->dev, "ov2735-test: init regs written\n");
+	return 0;
+}
+
+static int ov2735_detect(struct ov2735 *sensor)
+{
+	u8 hi, lo;
+	int ret;
+
+	dev_info(&sensor->client->dev, "ov2735-test: detect start\n");
+	ret = ov2735_write(sensor, 0xfd, 0x00);
+	if (ret < 0)
+		return ret;
+	ret = ov2735_read(sensor, 0x02, &hi);
+	if (ret < 0)
+		return ret;
+	if (hi != 0x27) {
+		dev_err(&sensor->client->dev,
+			"ov2735-test: ID high mismatch: 0x%x\n", hi);
+		return -ENODEV;
+	}
+	ret = ov2735_read(sensor, 0x03, &lo);
+	if (ret < 0)
+		return ret;
+	if (lo != 0x35) {
+		dev_err(&sensor->client->dev,
+			"ov2735-test: ID low mismatch: 0x%x\n", lo);
+		return -ENODEV;
+	}
+
+	dev_info(&sensor->client->dev,
+		 "ov2735-test: detected OV2735 (id %02x%02x)\n", hi, lo);
+	return 0;
+}
+
+static int ov2735_power_on(struct ov2735 *sensor)
+{
+	int ret;
+
+	/* Set I2C mux to select this sensor */
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "ov2735-test: set i2c-mux high\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 1);
+	}
+
+	sensor->mclk = devm_clk_get(&sensor->client->dev, "cam_mclk");
+	if (IS_ERR(sensor->mclk)) {
+		dev_err(&sensor->client->dev,
+			"ov2735-test: Failed to get cam_mclk clock: %ld\n",
+			PTR_ERR(sensor->mclk));
+		return PTR_ERR(sensor->mclk);
+	}
+
+	/* Set cam_mclk frequency to 24MHz */
+	ret = clk_set_rate(sensor->mclk, 24000000);
+	if (ret < 0) {
+		dev_err(&sensor->client->dev,
+			"ov2735-test: Failed to set cam_mclk rate to 24MHz: %d\n",
+			ret);
+		return ret;
+	}
+
+	/* Enable cam_mclk clock */
+	ret = clk_prepare_enable(sensor->mclk);
+	if (ret < 0) {
+		dev_err(&sensor->client->dev,
+			"ov2735-test: Failed to enable cam_mclk: %d\n", ret);
+		return ret;
+	}
+	dev_info(&sensor->client->dev,
+		 "ov2735-test: cam_mclk enabled, rate=%lu Hz\n",
+		 clk_get_rate(sensor->mclk));
+
+	dev_info(&sensor->client->dev, "ov2735-test: power_on enter\n");
+
+	dev_info(&sensor->client->dev, "ov2735-test: get vdd regulator\n");
+	sensor->vdd = devm_regulator_get(&sensor->client->dev, "vdd");
+	if (IS_ERR(sensor->vdd)) {
+		dev_err(&sensor->client->dev, "ov2735-test: Failed to get vdd regulator: %ld\n",
+			PTR_ERR(sensor->vdd));
+		return PTR_ERR(sensor->vdd);
+	}
+
+	if (sensor->vdd) {
+		ret = regulator_enable(sensor->vdd);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev,
+				"ov2735-test: failed to enable vdd: %d\n", ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "ov2735-test: enable vdd\n");
+
+		ret = regulator_set_voltage(sensor->vdd, 3300000, 3300000);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev, "ov2735-test: failed to set vdd voltage: %d\n",
+				ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "ov2735-test: vdd set to 3.3V\n");
+	}
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev, "ov2735-test: drive PWDN low\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	}
+
+	usleep_range(10000, 12000);
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev,
+			 "ov2735-test: drive PWDN high\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 1);
+	}
+
+	usleep_range(30000, 31000);
+
+	dev_info(&sensor->client->dev, "ov2735-test: power_on done\n");
+
+	return 0;
+}
+
+static void ov2735_power_off(struct ov2735 *sensor)
+{
+	dev_info(&sensor->client->dev, "ov2735-test: power_off enter\n");
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev, "ov2735-test: drive PWDN low\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	}
+
+	if (sensor->vdd) {
+		dev_info(&sensor->client->dev, "ov2735-test: disable vdd\n");
+		regulator_disable(sensor->vdd);
+	}
+
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "ov2735-test: set i2c-mux low\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 0);
+	}
+
+	dev_info(&sensor->client->dev, "ov2735-test: power_off done\n");
+}
+
+static int ov2735_probe(struct i2c_client *client)
+{
+	struct ov2735 *sensor;
+	struct device *dev = &client->dev;
+	int ret;
+
+	dev_info(dev, "ov2735-test: probe enter, client addr=0x%x\n",
+		 client->addr);
+
+	sensor = devm_kzalloc(dev, sizeof(*sensor), GFP_KERNEL);
+	if (!sensor)
+		return -ENOMEM;
+
+	sensor->client = client;
+	mutex_init(&sensor->lock);
+	i2c_set_clientdata(client, sensor);
+
+	dev_info(dev, "ov2735-test: get PWDN gpio\n");
+	sensor->pwdn = devm_gpiod_get_optional(
+		dev, "pwdn", GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->pwdn)) {
+		dev_err(dev, "ov2735-test: Failed to get PWDN GPIO\n");
+		goto err_pwdn;
+	}
+
+	sensor->i2c_mux = devm_gpiod_get_optional(dev, "i2c-mux",
+		GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->i2c_mux)) {
+		dev_warn(dev, "ov2735-test: Failed to get i2c-mux GPIO, continuing without it\n");
+		sensor->i2c_mux = NULL;
+	}
+
+	/* Power on and detect sensor before creating device node */
+	ret = ov2735_power_on(sensor);
+	if (ret) {
+		dev_err(dev, "ov2735-test: power on failed: %d\n", ret);
+		goto err_power_on;
+	}
+
+	ret = ov2735_detect(sensor);
+	if (ret) {
+		dev_err(dev, "ov2735-test: sensor detect failed: %d\n", ret);
+		goto err_detect;
+	}
+
+	ov2735_power_off(sensor);
+
+	sensor->miscdev.minor = MISC_DYNAMIC_MINOR;
+	sensor->miscdev.fops = &ov2735_fops;
+	sensor->miscdev.parent = dev;
+	if (client->dev.of_node) {
+		u32 csi_id;
+		if (of_property_read_u32(client->dev.of_node, "csi-id",
+					 &csi_id) == 0) {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "ov2735-%u", csi_id);
+			dev_info(dev, "ov2735-test: ov2735-%u \n", csi_id);
+		} else {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "ov2735-%02x", client->addr);
+			dev_info(dev, "ov2735-test: ov2735-%02x \n",
+				 client->addr);
+		}
+	} else {
+		sensor->miscdev.name = devm_kasprintf(
+			dev, GFP_KERNEL, "ov2735-%02x", client->addr);
+	}
+
+	ret = misc_register(&sensor->miscdev);
+	if (ret) {
+		dev_err(dev,
+			"ov2735-test: failed to register misc device: %d\n",
+			ret);
+		goto err_misc_register;
+	}
+
+	global_ov2735 = sensor;
+	dev_info(dev, "ov2735-test: probe successful, ioctl device /dev/%s\n",
+		 sensor->miscdev.name);
+	return 0;
+
+err_misc_register:
+	ov2735_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+	return ret;
+
+err_detect:
+	ov2735_power_off(sensor);
+err_power_on:
+err_pwdn:
+	mutex_destroy(&sensor->lock);
+	return ret;
+}
+
+static void ov2735_remove(struct i2c_client *client)
+{
+	struct ov2735 *sensor = i2c_get_clientdata(client);
+
+	dev_info(&client->dev, "ov2735-test: remove\n");
+	if (global_ov2735 == sensor)
+		global_ov2735 = NULL;
+
+	misc_deregister(&sensor->miscdev);
+	ov2735_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+}
+
+static const struct of_device_id ov2735_of_match[] = {
+	{ .compatible = "ovti,ov2735" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ov2735_of_match);
+
+static struct i2c_driver ov2735_driver = {
+	.driver = {
+		.name		= "ov2735-simple",
+		.of_match_table	= of_match_ptr(ov2735_of_match),
+	},
+	.probe		= ov2735_probe,
+	.remove		= ov2735_remove,
+};
+
+module_i2c_driver(ov2735_driver)
+
+MODULE_DESCRIPTION("Simplified I2C driver for OV2735");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/spacemit/cam_sensor/ov5640_max96724_max9295_sensor.c b/drivers/media/platform/spacemit/cam_sensor/ov5640_max96724_max9295_sensor.c
new file mode 100755
index 000000000000..a4583a518701
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_sensor/ov5640_max96724_max9295_sensor.c
@@ -0,0 +1,1102 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ov5640_max96724_max9295_sensor.c - OV5640 GMSL sensor driver
+ *
+ * Copyright (C) 2026 Spacemit Ltd.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/ioctl.h>
+#include <linux/miscdevice.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+
+#define OV5640_IOC_MAGIC			'V'
+#define OV5640_IOCTL_POWER_ON			_IO(OV5640_IOC_MAGIC, 1)
+#define OV5640_IOCTL_POWER_OFF			_IO(OV5640_IOC_MAGIC, 2)
+#define OV5640_IOCTL_INIT_REGS			_IO(OV5640_IOC_MAGIC, 3)
+#define OV5640_IOCTL_STREAM_ON			_IO(OV5640_IOC_MAGIC, 4)
+#define OV5640_IOCTL_STREAM_OFF			_IO(OV5640_IOC_MAGIC, 5)
+#define OV5640_IOCTL_DETECT			_IO(OV5640_IOC_MAGIC, 6)
+#define OV5640_IOCTL_SETUP_GMSL			_IO(OV5640_IOC_MAGIC, 7)
+
+#define MAX96724_DEF_ADDR		0x27
+#define MAX9295_BASE_ADDR		0x41
+#define MAX9295_BROADCAST_ADDR		0x40
+
+#define MAX96724_PHY_CLK		(0x20 | 12)
+#define MAX96724_TX11_PIPE_X_EN_ADDR	0x090b
+#define MAX96724_TX45_PIPE_X_DST_CTRL_ADDR	0x092d
+#define MAX96724_PIPE_X_SRC_0_MAP_ADDR	0x090d
+#define MAX9295_GPIO_FRAME_TRIGGER	0x02d3
+#define OV5640_XCLK_FREQ		24000000
+
+struct regval_list {
+	u16 addr;
+	u8 data;
+	u8 mask;
+	u16 delay_ms;
+};
+
+struct ov5640_gmsl {
+	struct i2c_client *client;
+	struct mutex lock;
+	struct gpio_desc *pwdn;
+	struct gpio_desc *reset;
+	struct gpio_desc *i2c_mux;
+	struct clk *xclk;
+	struct regulator *vdd;
+	struct miscdevice miscdev;
+	u8 des_addr;
+	u8 ser_base_addr;
+	u8 link_lock_status;
+	bool gmsl_inited;
+	bool powered;
+	bool streaming;
+};
+
+static struct ov5640_gmsl *global_ov5640_gmsl;
+
+static const struct regval_list ov5640_init_setting[] = {
+	{0x3103, 0x11, 0, 0},
+	{0x3008, 0x82, 0, 10},
+	{0x3008, 0x42, 0, 0},
+	{0x3103, 0x03, 0, 0},
+	{0x3017, 0x00, 0, 0},
+	{0x3018, 0x00, 0, 0},
+	{0x3034, 0x18, 0, 0},
+	{0x3035, 0x11, 0, 0},
+	{0x3036, 0x54, 0, 0},
+	{0x3037, 0x13, 0, 0},
+	{0x3108, 0x01, 0, 0},
+	{0x3630, 0x36, 0, 0},
+	{0x3631, 0x0e, 0, 0},
+	{0x3632, 0xe2, 0, 0},
+	{0x3633, 0x12, 0, 0},
+	{0x3621, 0xe0, 0, 0},
+	{0x3704, 0xa0, 0, 0},
+	{0x3703, 0x5a, 0, 0},
+	{0x3715, 0x78, 0, 0},
+	{0x3717, 0x01, 0, 0},
+	{0x370b, 0x60, 0, 0},
+	{0x3705, 0x1a, 0, 0},
+	{0x3905, 0x02, 0, 0},
+	{0x3906, 0x10, 0, 0},
+	{0x3901, 0x0a, 0, 0},
+	{0x3731, 0x12, 0, 0},
+	{0x3600, 0x08, 0, 0},
+	{0x3601, 0x33, 0, 0},
+	{0x302d, 0x60, 0, 0},
+	{0x3620, 0x52, 0, 0},
+	{0x371b, 0x20, 0, 0},
+	{0x471c, 0x50, 0, 0},
+	{0x3a13, 0x43, 0, 0},
+	{0x3a18, 0x00, 0, 0},
+	{0x3a19, 0xf8, 0, 0},
+	{0x3635, 0x13, 0, 0},
+	{0x3636, 0x03, 0, 0},
+	{0x3634, 0x40, 0, 0},
+	{0x3622, 0x01, 0, 0},
+	{0x3c01, 0x34, 0, 0},
+	{0x3c04, 0x28, 0, 0},
+	{0x3c05, 0x98, 0, 0},
+	{0x3c06, 0x00, 0, 0},
+	{0x3c07, 0x07, 0, 0},
+	{0x3c08, 0x00, 0, 0},
+	{0x3c09, 0x1c, 0, 0},
+	{0x3c0a, 0x9c, 0, 0},
+	{0x3c0b, 0x40, 0, 0},
+	{0x3820, 0x40, 0, 0},
+	{0x3821, 0x06, 0, 0},
+	{0x3814, 0x11, 0, 0},
+	{0x3815, 0x11, 0, 0},
+	{0x3800, 0x01, 0, 0},
+	{0x3801, 0x50, 0, 0},
+	{0x3802, 0x01, 0, 0},
+	{0x3803, 0xb2, 0, 0},
+	{0x3804, 0x08, 0, 0},
+	{0x3805, 0xef, 0, 0},
+	{0x3806, 0x05, 0, 0},
+	{0x3807, 0xf1, 0, 0},
+	{0x3808, 0x07, 0, 0},
+	{0x3809, 0x80, 0, 0},
+	{0x380a, 0x04, 0, 0},
+	{0x380b, 0x38, 0, 0},
+	{0x380c, 0x09, 0, 0},
+	{0x380d, 0xc4, 0, 0},
+	{0x380e, 0x04, 0, 0},
+	{0x380f, 0x60, 0, 0},
+	{0x3810, 0x00, 0, 0},
+	{0x3811, 0x10, 0, 0},
+	{0x3812, 0x00, 0, 0},
+	{0x3813, 0x04, 0, 0},
+	{0x3618, 0x04, 0, 0},
+	{0x3612, 0x2b, 0, 0},
+	{0x3708, 0x64, 0, 0},
+	{0x3709, 0x12, 0, 0},
+	{0x370c, 0x00, 0, 0},
+	{0x3a02, 0x04, 0, 0},
+	{0x3a03, 0x60, 0, 0},
+	{0x3a08, 0x01, 0, 0},
+	{0x3a09, 0x50, 0, 0},
+	{0x3a0a, 0x01, 0, 0},
+	{0x3a0b, 0x18, 0, 0},
+	{0x3a0e, 0x03, 0, 0},
+	{0x3a0d, 0x04, 0, 0},
+	{0x3a14, 0x04, 0, 0},
+	{0x3a15, 0x60, 0, 0},
+	{0x4001, 0x02, 0, 0},
+	{0x4004, 0x06, 0, 0},
+	{0x3000, 0x00, 0, 0},
+	{0x3002, 0x1c, 0, 0},
+	{0x3004, 0xff, 0, 0},
+	{0x3006, 0xc3, 0, 0},
+	{0x300e, 0x45, 0, 0},
+	{0x302e, 0x08, 0, 0},
+	{0x4300, 0x32, 0, 0},
+	{0x501f, 0x00, 0, 0},
+	{0x4713, 0x02, 0, 0},
+	{0x4407, 0x04, 0, 0},
+	{0x440e, 0x00, 0, 0},
+	{0x460b, 0x37, 0, 0},
+	{0x460c, 0x20, 0, 0},
+	{0x4837, 0x0a, 0, 0},
+	{0x3824, 0x04, 0, 0},
+	{0x5000, 0xa7, 0, 0},
+	{0x5001, 0x83, 0, 0},
+	{0x5180, 0xff, 0, 0},
+	{0x5181, 0xf2, 0, 0},
+	{0x5182, 0x00, 0, 0},
+	{0x5183, 0x14, 0, 0},
+	{0x5184, 0x25, 0, 0},
+	{0x5185, 0x24, 0, 0},
+	{0x5186, 0x09, 0, 0},
+	{0x5187, 0x09, 0, 0},
+	{0x5188, 0x09, 0, 0},
+	{0x5189, 0x75, 0, 0},
+	{0x518a, 0x54, 0, 0},
+	{0x518b, 0xe0, 0, 0},
+	{0x518c, 0xb2, 0, 0},
+	{0x518d, 0x42, 0, 0},
+	{0x518e, 0x3d, 0, 0},
+	{0x518f, 0x56, 0, 0},
+	{0x5190, 0x46, 0, 0},
+	{0x5191, 0xf8, 0, 0},
+	{0x5192, 0x04, 0, 0},
+	{0x5193, 0x70, 0, 0},
+	{0x5194, 0xf0, 0, 0},
+	{0x5195, 0xf0, 0, 0},
+	{0x5196, 0x03, 0, 0},
+	{0x5197, 0x01, 0, 0},
+	{0x5198, 0x04, 0, 0},
+	{0x5199, 0x12, 0, 0},
+	{0x519a, 0x04, 0, 0},
+	{0x519b, 0x00, 0, 0},
+	{0x519c, 0x06, 0, 0},
+	{0x519d, 0x82, 0, 0},
+	{0x519e, 0x38, 0, 0},
+	{0x5381, 0x1e, 0, 0},
+	{0x5382, 0x5b, 0, 0},
+	{0x5383, 0x08, 0, 0},
+	{0x5384, 0x0a, 0, 0},
+	{0x5385, 0x7e, 0, 0},
+	{0x5386, 0x88, 0, 0},
+	{0x5387, 0x7c, 0, 0},
+	{0x5388, 0x6c, 0, 0},
+	{0x5389, 0x10, 0, 0},
+	{0x538a, 0x01, 0, 0},
+	{0x538b, 0x98, 0, 0},
+	{0x5300, 0x08, 0, 0},
+	{0x5301, 0x30, 0, 0},
+	{0x5302, 0x10, 0, 0},
+	{0x5303, 0x00, 0, 0},
+	{0x5304, 0x08, 0, 0},
+	{0x5305, 0x30, 0, 0},
+	{0x5306, 0x08, 0, 0},
+	{0x5307, 0x16, 0, 0},
+	{0x5309, 0x08, 0, 0},
+	{0x530a, 0x30, 0, 0},
+	{0x530b, 0x04, 0, 0},
+	{0x530c, 0x06, 0, 0},
+	{0x5480, 0x01, 0, 0},
+	{0x5580, 0x02, 0, 0},
+	{0x5583, 0x40, 0, 0},
+	{0x5584, 0x10, 0, 0},
+	{0x5589, 0x10, 0, 0},
+	{0x558a, 0x00, 0, 0},
+	{0x558b, 0xf8, 0, 0},
+	{0x5800, 0x23, 0, 0},
+	{0x5801, 0x14, 0, 0},
+	{0x5802, 0x0f, 0, 0},
+	{0x5803, 0x0f, 0, 0},
+	{0x5804, 0x12, 0, 0},
+	{0x5805, 0x26, 0, 0},
+	{0x5806, 0x0c, 0, 0},
+	{0x5807, 0x08, 0, 0},
+	{0x5808, 0x05, 0, 0},
+	{0x5809, 0x05, 0, 0},
+	{0x580a, 0x08, 0, 0},
+	{0x580b, 0x0d, 0, 0},
+	{0x580c, 0x08, 0, 0},
+	{0x580d, 0x03, 0, 0},
+	{0x580e, 0x00, 0, 0},
+	{0x580f, 0x00, 0, 0},
+	{0x5810, 0x03, 0, 0},
+	{0x5811, 0x09, 0, 0},
+	{0x5812, 0x07, 0, 0},
+	{0x5813, 0x03, 0, 0},
+	{0x5814, 0x00, 0, 0},
+	{0x5815, 0x01, 0, 0},
+	{0x5816, 0x03, 0, 0},
+	{0x5817, 0x08, 0, 0},
+	{0x5818, 0x0d, 0, 0},
+	{0x5819, 0x08, 0, 0},
+	{0x581a, 0x05, 0, 0},
+	{0x581b, 0x06, 0, 0},
+	{0x581c, 0x08, 0, 0},
+	{0x581d, 0x0e, 0, 0},
+	{0x581e, 0x29, 0, 0},
+	{0x581f, 0x17, 0, 0},
+	{0x5820, 0x11, 0, 0},
+	{0x5821, 0x11, 0, 0},
+	{0x5822, 0x15, 0, 0},
+	{0x5823, 0x28, 0, 0},
+	{0x5824, 0x46, 0, 0},
+	{0x5825, 0x26, 0, 0},
+	{0x5826, 0x08, 0, 0},
+	{0x5827, 0x26, 0, 0},
+	{0x5828, 0x64, 0, 0},
+	{0x5829, 0x26, 0, 0},
+	{0x582a, 0x24, 0, 0},
+	{0x582b, 0x22, 0, 0},
+	{0x582c, 0x24, 0, 0},
+	{0x582d, 0x24, 0, 0},
+	{0x582e, 0x06, 0, 0},
+	{0x582f, 0x22, 0, 0},
+	{0x5830, 0x40, 0, 0},
+	{0x5831, 0x42, 0, 0},
+	{0x5832, 0x24, 0, 0},
+	{0x5833, 0x26, 0, 0},
+	{0x5834, 0x24, 0, 0},
+	{0x5835, 0x22, 0, 0},
+	{0x5836, 0x22, 0, 0},
+	{0x5837, 0x26, 0, 0},
+	{0x5838, 0x44, 0, 0},
+	{0x5839, 0x24, 0, 0},
+	{0x583a, 0x26, 0, 0},
+	{0x583b, 0x28, 0, 0},
+	{0x583c, 0x42, 0, 0},
+	{0x583d, 0xce, 0, 0},
+	{0x5025, 0x00, 0, 0},
+	{0x3a0f, 0x30, 0, 0},
+	{0x3a10, 0x28, 0, 0},
+	{0x3a1b, 0x30, 0, 0},
+	{0x3a1e, 0x26, 0, 0},
+	{0x3a11, 0x60, 0, 0},
+	{0x3a1f, 0x14, 0, 5},
+};
+
+static const struct regval_list ov5640_start_stream[] = {
+	{0x3008, 0x02, 0, 10},
+};
+
+static const struct regval_list ov5640_stop_stream[] = {
+	{0x3008, 0x42, 0, 10},
+};
+
+static int ov5640_i2c_write(struct ov5640_gmsl *sensor, u16 reg, u8 val)
+{
+	struct i2c_msg msg;
+	u8 buf[3];
+	int ret;
+
+	buf[0] = reg >> 8;
+	buf[1] = reg & 0xff;
+	buf[2] = val;
+
+	msg.addr = sensor->client->addr;
+	msg.flags = 0;
+	msg.len = sizeof(buf);
+	msg.buf = buf;
+
+	ret = i2c_transfer(sensor->client->adapter, &msg, 1);
+	if (ret != 1) {
+		dev_err(&sensor->client->dev,
+			"ov5640-gmsl: i2c write failed slave=0x%02x reg=0x%04x val=0x%02x ret=%d\n",
+			sensor->client->addr, reg, val, ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	return 0;
+}
+
+static int ov5640_i2c_read(struct ov5640_gmsl *sensor, u16 reg, u8 *val)
+{
+	struct i2c_msg msgs[2];
+	u8 reg_buf[2];
+	u8 data_buf[1];
+	int ret;
+
+	reg_buf[0] = reg >> 8;
+	reg_buf[1] = reg & 0xff;
+
+	msgs[0].addr = sensor->client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = sizeof(reg_buf);
+	msgs[0].buf = reg_buf;
+
+	msgs[1].addr = sensor->client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = sizeof(data_buf);
+	msgs[1].buf = data_buf;
+
+	ret = i2c_transfer(sensor->client->adapter, msgs, 2);
+	if (ret != 2) {
+		dev_err(&sensor->client->dev,
+			"ov5640-gmsl: i2c read failed slave=0x%02x reg=0x%04x ret=%d\n",
+			sensor->client->addr, reg, ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	*val = data_buf[0];
+	return 0;
+}
+
+static int ov5640_i2c_update_bits(struct ov5640_gmsl *sensor, u16 reg,
+				  u8 mask, u8 val)
+{
+	u8 tmp;
+	int ret;
+
+	ret = ov5640_i2c_read(sensor, reg, &tmp);
+	if (ret)
+		return ret;
+
+	tmp = (tmp & ~mask) | (val & mask);
+	return ov5640_i2c_write(sensor, reg, tmp);
+}
+
+static int ov5640_write_array(struct ov5640_gmsl *sensor,
+			      const struct regval_list *regs, int num)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < num; i++) {
+		if (regs[i].mask)
+			ret = ov5640_i2c_update_bits(sensor, regs[i].addr,
+						 regs[i].mask, regs[i].data);
+		else
+			ret = ov5640_i2c_write(sensor, regs[i].addr, regs[i].data);
+		if (ret)
+			return ret;
+		if (regs[i].delay_ms)
+			msleep(regs[i].delay_ms);
+	}
+
+	return 0;
+}
+
+static int ov5640_check_des_version(struct ov5640_gmsl *sensor)
+{
+	static const u16 ver_addrs[2] = { 0x0316, 0x0319 };
+	u8 ver = 0;
+	u8 tmp;
+	int i;
+	int ret;
+
+	sensor->client->addr = sensor->des_addr;
+	for (i = 0; i < ARRAY_SIZE(ver_addrs); i++) {
+		ret = ov5640_i2c_read(sensor, ver_addrs[i], &tmp);
+		if (ret)
+			return ret;
+		ver |= ((tmp & 0x08) >> 3) << i;
+	}
+
+	dev_info(&sensor->client->dev,
+		 "ov5640-gmsl: %s des version bits=0x%02x\n", __func__, ver);
+
+	return ver == 0x01 ? 0 : -ENODEV;
+}
+
+static int ov5640_get_link_lock_status(struct ov5640_gmsl *sensor)
+{
+	u8 val;
+	int ret;
+
+	dev_dbg(&sensor->client->dev, "ov5640-gmsl: enter %s\n", __func__);
+
+	sensor->client->addr = sensor->des_addr;
+	sensor->link_lock_status = 0;
+	ret = ov5640_i2c_read(sensor, 0x001a, &val);
+	if (ret)
+		return ret;
+	sensor->link_lock_status |= (val & 0x08) >> 3;
+
+	ret = ov5640_i2c_read(sensor, 0x000a, &val);
+	if (ret)
+		return ret;
+	sensor->link_lock_status |= (val & 0x08) >> 2;
+
+	ret = ov5640_i2c_read(sensor, 0x000b, &val);
+	if (ret)
+		return ret;
+	sensor->link_lock_status |= (val & 0x08) >> 1;
+
+	ret = ov5640_i2c_read(sensor, 0x000c, &val);
+	if (ret)
+		return ret;
+	sensor->link_lock_status |= (val & 0x08) >> 0;
+
+	dev_dbg(&sensor->client->dev,
+		 "ov5640-gmsl: %s link_lock_status=0x%02x\n",
+		 __func__, sensor->link_lock_status);
+
+	return 0;
+}
+
+static int ov5640_monopolize_link(struct ov5640_gmsl *sensor, int link)
+{
+	u8 value;
+	int ret;
+
+	sensor->client->addr = sensor->des_addr;
+	ret = ov5640_i2c_read(sensor, 0x0006, &value);
+	if (ret)
+		return ret;
+
+	value = (value & 0xf0) | BIT(link);
+	ret = ov5640_i2c_write(sensor, 0x0006, value);
+	if (!ret)
+		msleep(150);
+
+	if (!ret)
+		dev_dbg(&sensor->client->dev,
+			 "ov5640-gmsl: monopolize link%d success, reg0x0006=0x%02x\n",
+			 link, value);
+
+	return ret;
+}
+
+static int ov5640_restore_links(struct ov5640_gmsl *sensor)
+{
+	int ret;
+
+	dev_dbg(&sensor->client->dev, "ov5640-gmsl: enter %s lock_status=0x%02x\n",
+		 __func__, sensor->link_lock_status);
+
+	sensor->client->addr = sensor->des_addr;
+	ret = ov5640_i2c_write(sensor, 0x0006,
+			      0xf0 | sensor->link_lock_status);
+	if (!ret)
+		msleep(150);
+
+	if (!ret)
+		dev_dbg(&sensor->client->dev,
+			 "ov5640-gmsl: restore links success reg0x0006=0x%02x\n",
+			 0xf0 | sensor->link_lock_status);
+
+	return ret;
+}
+
+static int ov5640_setup_gmsl(struct ov5640_gmsl *sensor)
+{
+	u16 addrbackup = sensor->client->addr;
+	u8 val;
+	int i;
+	int ret;
+
+	dev_dbg(&sensor->client->dev,
+		 "ov5640-gmsl: enter %s des=0x%02x ser_base=0x%02x\n",
+		 __func__, sensor->des_addr, sensor->ser_base_addr);
+
+	sensor->client->addr = sensor->des_addr;
+	ret = ov5640_i2c_read(sensor, 0x0000, &val);
+	if (ret)
+		goto out_restore_addr;
+	if (val != 0x4e) {
+		dev_err(&sensor->client->dev,
+			"ov5640-gmsl: max96724 not detected, id=0x%02x\n", val);
+		ret = -ENODEV;
+		goto out_restore_addr;
+	}
+
+	ret = ov5640_check_des_version(sensor);
+	if (ret)
+		goto out_restore_addr;
+
+	ov5640_i2c_write(sensor, 0x0013, 0x40);
+	msleep(100);
+	ov5640_i2c_write(sensor, 0x0013, 0x00);
+
+	ov5640_i2c_write(sensor, 0x0005, 0x88);
+	ov5640_i2c_write(sensor, 0x0310, 0x90);
+	ov5640_i2c_write(sensor, 0x0309, 0x90);
+
+	ret = ov5640_get_link_lock_status(sensor);
+	if (ret)
+		goto out_restore_addr;
+	if (!sensor->link_lock_status) {
+		dev_err(&sensor->client->dev, "ov5640-gmsl: no locked GMSL links\n");
+		ret = -ENOLINK;
+		goto out_restore_addr;
+	}
+
+	dev_dbg(&sensor->client->dev,
+		 "ov5640-gmsl: detected link lock bitmap=0x%02x\n",
+		 sensor->link_lock_status);
+
+	ret = ov5640_i2c_read(sensor, 0x0006, &val);
+	if (ret)
+		goto out_restore_addr;
+	ov5640_i2c_write(sensor, 0x0006, (val & 0xf0) | (sensor->link_lock_status & 0x0f));
+
+	ov5640_i2c_write(sensor, 0x090a, 0xc0);
+	ov5640_i2c_write(sensor, 0x094a, 0xc0);
+	ov5640_i2c_write(sensor, 0x098a, 0xc0);
+	ov5640_i2c_write(sensor, 0x09ca, 0xc0);
+
+	ov5640_i2c_write(sensor, 0x08a3, 0xe4);
+	ov5640_i2c_write(sensor, 0x08a4, 0xe4);
+	ov5640_i2c_write(sensor, 0x08a0, 0x24);
+
+	ov5640_i2c_write(sensor, 0x0415, MAX96724_PHY_CLK);
+	ov5640_i2c_write(sensor, 0x0418, MAX96724_PHY_CLK);
+	ov5640_i2c_write(sensor, 0x041b, MAX96724_PHY_CLK);
+	ov5640_i2c_write(sensor, 0x041e, MAX96724_PHY_CLK);
+
+	ov5640_i2c_write(sensor, 0x00f0, 0x62);
+	ov5640_i2c_write(sensor, 0x00f1, 0xea);
+	ov5640_i2c_write(sensor, 0x00f4, 0x00);
+
+	ov5640_i2c_write(sensor, 0x0300, 0x83);
+	ov5640_i2c_write(sensor, 0x0301, 0xaa);
+	ov5640_i2c_write(sensor, 0x0337, 0x2a);
+	ov5640_i2c_write(sensor, 0x036d, 0x2a);
+	ov5640_i2c_write(sensor, 0x03a4, 0x2a);
+
+	ov5640_i2c_write(sensor, 0x0027, 0x00);
+	ov5640_i2c_write(sensor, 0x0029, 0xf0);
+
+	for (i = 0; i < 4; i++) {
+		u8 ser_addr;
+
+		if (!(sensor->link_lock_status & BIT(i)))
+			continue;
+
+		dev_dbg(&sensor->client->dev,
+			 "ov5640-gmsl: start setup link%d\n", i);
+
+		sensor->client->addr = sensor->des_addr;
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_TX11_PIPE_X_EN_ADDR, 0x1f);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_TX45_PIPE_X_DST_CTRL_ADDR, 0xaa);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_TX45_PIPE_X_DST_CTRL_ADDR + 1, 0x02);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 0, 0x1e);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 1, 0x1e | (i << 6));
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 2, 0x00);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 3, 0x00 | (i << 6));
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 4, 0x01);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 5, 0x01 | (i << 6));
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 6, 0x02);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 7, 0x02 | (i << 6));
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 8, 0x03);
+		ov5640_i2c_write(sensor, (0x40 * i) + MAX96724_PIPE_X_SRC_0_MAP_ADDR + 9, 0x03 | (i << 6));
+
+		ret = ov5640_monopolize_link(sensor, i);
+		if (ret)
+			goto out_restore_addr;
+
+		sensor->client->addr = MAX9295_BROADCAST_ADDR;
+		ov5640_i2c_write(sensor, 0x0000, (sensor->ser_base_addr + i) << 1);
+		msleep(75);
+
+		ser_addr = sensor->ser_base_addr + i;
+		sensor->client->addr = ser_addr;
+		dev_dbg(&sensor->client->dev,
+			 "ov5640-gmsl: link%d serializer remapped to 0x%02x\n",
+			 i, ser_addr);
+		ret = ov5640_i2c_read(sensor, 0x000d, &val);
+		if (ret) {
+			sensor->link_lock_status &= ~BIT(i);
+			dev_err(&sensor->client->dev,
+				"ov5640-gmsl: read serializer id failed on link %d addr=0x%02x ret=%d\n",
+				i, ser_addr, ret);
+			goto restore_links;
+		}
+		if (val != 0x91) {
+			dev_err(&sensor->client->dev,
+				"ov5640-gmsl: max9295 on link %d not detected, id=0x%02x\n",
+				i, val);
+			sensor->link_lock_status &= ~BIT(i);
+			ret = -ENODEV;
+			goto restore_links;
+		}
+
+		dev_dbg(&sensor->client->dev,
+			 "ov5640-gmsl: link%d serializer detected, addr=0x%02x id=0x%02x\n",
+			 i, ser_addr, val);
+
+		ov5640_i2c_write(sensor, 0x02bf, 0x60);
+		ov5640_i2c_write(sensor, 0x02be, 0x10);
+		ov5640_i2c_write(sensor, 0x0311, 0x40);
+		ov5640_i2c_write(sensor, 0x0308, 0x64);
+		ov5640_i2c_write(sensor, 0x0002, 0x43);
+		ov5640_i2c_write(sensor, 0x0331, 0x11);
+		ov5640_i2c_write(sensor, 0x0334, 0x70);
+		ov5640_i2c_write(sensor, 0x0335, 0x07);
+
+		ov5640_i2c_write(sensor, 0x007b, 0x30 + i);
+		ov5640_i2c_write(sensor, 0x0083, 0x30 + i);
+		ov5640_i2c_write(sensor, 0x0093, 0x30 + i);
+		ov5640_i2c_write(sensor, 0x009b, 0x30 + i);
+		ov5640_i2c_write(sensor, 0x00a3, 0x30 + i);
+		ov5640_i2c_write(sensor, 0x00ab, 0x30 + i);
+		ov5640_i2c_write(sensor, 0x008b, 0x30 + i);
+
+		dev_dbg(&sensor->client->dev,
+			 "ov5640-gmsl: link%d setup done serializer=0x%02x\n",
+			 i, ser_addr);
+	}
+
+	restore_links:
+	ret = ov5640_restore_links(sensor);
+	if (ret)
+		goto out_restore_addr;
+
+	if (!sensor->link_lock_status)
+		ret = -ENOLINK;
+	else
+		sensor->gmsl_inited = true;
+
+	out_restore_addr:
+	sensor->client->addr = addrbackup;
+	if (!ret)
+		dev_dbg(&sensor->client->dev, "ov5640-gmsl: %s done\n", __func__);
+	return ret;
+}
+
+static int __maybe_unused ov5640_sensor_detect(struct ov5640_gmsl *sensor)
+{
+	u8 hi;
+	u8 lo;
+	int ret;
+
+	ret = ov5640_i2c_read(sensor, 0x300a, &hi);
+	if (ret)
+		return ret;
+	ret = ov5640_i2c_read(sensor, 0x300b, &lo);
+	if (ret)
+		return ret;
+
+	if (hi != 0x56 || lo != 0x40) {
+		dev_err(&sensor->client->dev,
+			"ov5640-gmsl: ov5640 id mismatch %02x%02x\n", hi, lo);
+		return -ENODEV;
+	}
+
+	dev_dbg(&sensor->client->dev,
+		 "ov5640-gmsl: sensor detected id=%02x%02x\n", hi, lo);
+
+	return 0;
+}
+
+static int ov5640_power_on(struct ov5640_gmsl *sensor)
+{
+	int i;
+	int ret;
+
+	if (sensor->powered)
+		return 0;
+
+	if (sensor->i2c_mux)
+		gpiod_set_value_cansleep(sensor->i2c_mux, 1);
+
+	if (sensor->xclk) {
+		ret = clk_prepare_enable(sensor->xclk);
+		if (ret)
+			return ret;
+	}
+
+	sensor->vdd = devm_regulator_get(&sensor->client->dev, "vdd");
+	if (IS_ERR(sensor->vdd)) {
+		ret = PTR_ERR(sensor->vdd);
+		goto err_clk;
+	}
+
+	ret = regulator_enable(sensor->vdd);
+	if (ret)
+		goto err_clk;
+
+	ret = regulator_set_voltage(sensor->vdd, 3300000, 3300000);
+	if (ret)
+		goto err_regulator;
+
+	if (sensor->reset)
+		gpiod_set_value_cansleep(sensor->reset, 0);
+	if (sensor->pwdn)
+		gpiod_set_value_cansleep(sensor->pwdn, 1);
+	usleep_range(5000, 10000);
+	if (sensor->pwdn)
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	usleep_range(5000, 10000);
+	if (sensor->reset)
+		gpiod_set_value_cansleep(sensor->reset, 1);
+	usleep_range(1000, 2000);
+	if (sensor->reset)
+		gpiod_set_value_cansleep(sensor->reset, 0);
+	usleep_range(20000, 25000);
+
+	for (i = 0; i < 3; i++)
+		dev_dbg(&sensor->client->dev, "ov5640-gmsl: power sequence step %d done\n", i);
+
+	sensor->powered = true;
+	dev_info(&sensor->client->dev, "ov5640-gmsl: %s done\n", __func__);
+	return 0;
+
+err_clk:
+	if (sensor->xclk)
+		clk_disable_unprepare(sensor->xclk);
+	if (sensor->i2c_mux)
+		gpiod_set_value_cansleep(sensor->i2c_mux, 0);
+	return ret;
+
+err_regulator:
+	regulator_disable(sensor->vdd);
+	goto err_clk;
+}
+
+static void ov5640_power_off(struct ov5640_gmsl *sensor)
+{
+	if (!sensor->powered)
+		return;
+
+	if (sensor->pwdn)
+		gpiod_set_value_cansleep(sensor->pwdn, 1);
+	if (sensor->reset)
+		gpiod_set_value_cansleep(sensor->reset, 0);
+	if (sensor->vdd)
+		regulator_disable(sensor->vdd);
+	if (sensor->xclk)
+		clk_disable_unprepare(sensor->xclk);
+	if (sensor->i2c_mux)
+		gpiod_set_value_cansleep(sensor->i2c_mux, 0);
+
+	sensor->powered = false;
+	sensor->streaming = false;
+	dev_info(&sensor->client->dev, "ov5640-gmsl: %s done\n", __func__);
+}
+
+static int ov5640_write_init_regs(struct ov5640_gmsl *sensor)
+{
+	int ret;
+
+	dev_dbg(&sensor->client->dev, "ov5640-gmsl: enter %s\n", __func__);
+
+	if (!sensor->gmsl_inited) {
+		ret = ov5640_setup_gmsl(sensor);
+		if (ret)
+			return ret;
+	}
+
+	ret = ov5640_write_array(sensor,
+				 ov5640_init_setting,
+				 ARRAY_SIZE(ov5640_init_setting));
+	if (!ret)
+		dev_dbg(&sensor->client->dev, "ov5640-gmsl: sensor init registers written\n");
+
+	return ret;
+}
+
+static int ov5640_stream_on(struct ov5640_gmsl *sensor)
+{
+	int i;
+	u16 addrbackup = sensor->client->addr;
+
+	dev_dbg(&sensor->client->dev, "ov5640-gmsl: enter %s link_status=0x%02x\n",
+		 __func__, sensor->link_lock_status);
+
+	if (sensor->streaming)
+		return 0;
+
+	ov5640_write_array(sensor,
+				 ov5640_start_stream,
+				 ARRAY_SIZE(ov5640_start_stream));
+
+	sensor->client->addr = sensor->des_addr;
+
+	ov5640_i2c_write(sensor, 0x00f4, sensor->link_lock_status);
+	ov5640_i2c_write(sensor, 0x0018, sensor->link_lock_status);
+	ov5640_i2c_write(sensor, 0x0943, 0x10);
+	ov5640_i2c_write(sensor, 0x0943, 0x33);
+	ov5640_i2c_write(sensor, 0x0983, 0x10);
+	ov5640_i2c_write(sensor, 0x0983, 0x33);
+
+	msleep(100);
+
+	for (i = 0; i < 4; i++) {
+		u8 ser_addr;
+
+		if (!(sensor->link_lock_status & BIT(i)))
+			continue;
+
+		ser_addr = sensor->ser_base_addr + i;
+		sensor->client->addr = ser_addr;
+		dev_dbg(&sensor->client->dev,
+			 "ov5640-gmsl: stream on trigger link%d serializer=0x%02x\n",
+			 i, ser_addr);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER + 1, 0xa0);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER, 0x00);
+		msleep(50);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER, 0x10);
+		msleep(50);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER, 0x00);
+		msleep(50);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER, 0x10);
+		msleep(50);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER, 0x04);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER + 1, 0x6a);
+		ov5640_i2c_write(sensor, MAX9295_GPIO_FRAME_TRIGGER + 2, 0x0a);
+	}
+
+	sensor->client->addr = addrbackup;
+
+	sensor->streaming = true;
+	dev_info(&sensor->client->dev, "ov5640-gmsl: %s done\n", __func__);
+	return 0;
+}
+
+static int ov5640_stream_off(struct ov5640_gmsl *sensor)
+{
+	int ret;
+
+	if (!sensor->streaming)
+		return 0;
+
+	ret = ov5640_write_array(sensor,
+				 ov5640_stop_stream,
+				 ARRAY_SIZE(ov5640_stop_stream));
+	if (!ret)
+		sensor->streaming = false;
+
+	if (!ret)
+		dev_info(&sensor->client->dev, "ov5640-gmsl: %s done\n", __func__);
+
+	return ret;
+}
+
+static long ov5640_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct ov5640_gmsl *sensor = file->private_data;
+	long ret = 0;
+
+	if (!sensor)
+		return -ENODEV;
+
+	mutex_lock(&sensor->lock);
+	switch (cmd) {
+	case OV5640_IOCTL_POWER_ON:
+		/* ret = ov5640_power_on(sensor); */
+		break;
+	case OV5640_IOCTL_POWER_OFF:
+		ov5640_stream_off(sensor);
+		ov5640_power_off(sensor);
+		break;
+	case OV5640_IOCTL_INIT_REGS:
+		/* ret = ov5640_write_init_regs(sensor); */
+		break;
+	case OV5640_IOCTL_STREAM_ON:
+		ret = ov5640_stream_on(sensor);
+		break;
+	case OV5640_IOCTL_STREAM_OFF:
+		ret = ov5640_stream_off(sensor);
+		break;
+	case OV5640_IOCTL_DETECT:
+		/* ret = ov5640_sensor_detect(sensor); */
+		break;
+	case OV5640_IOCTL_SETUP_GMSL:
+		/* ret = ov5640_setup_gmsl(sensor); */
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	mutex_unlock(&sensor->lock);
+
+	return ret;
+}
+
+static int ov5640_dev_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *misc = file->private_data;
+	struct ov5640_gmsl *sensor;
+
+	if (!misc)
+		return -ENODEV;
+
+	sensor = container_of(misc, struct ov5640_gmsl, miscdev);
+	file->private_data = sensor;
+
+	return 0;
+}
+
+static const struct file_operations ov5640_fops = {
+	.owner = THIS_MODULE,
+	.open = ov5640_dev_open,
+	.unlocked_ioctl = ov5640_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = ov5640_ioctl,
+#endif
+};
+
+static int ov5640_probe(struct i2c_client *client)
+{
+	struct ov5640_gmsl *sensor;
+	struct device *dev = &client->dev;
+	int ret;
+	u32 val;
+
+	dev_dbg(dev, "ov5640-gmsl: enter %s client=0x%02x\n", __func__, client->addr);
+
+	sensor = devm_kzalloc(dev, sizeof(*sensor), GFP_KERNEL);
+	if (!sensor)
+		return -ENOMEM;
+
+	sensor->client = client;
+	sensor->des_addr = MAX96724_DEF_ADDR;
+	sensor->ser_base_addr = MAX9295_BASE_ADDR;
+	mutex_init(&sensor->lock);
+	i2c_set_clientdata(client, sensor);
+
+	sensor->pwdn = devm_gpiod_get_optional(dev, "pwdn",
+				       GPIOD_OUT_HIGH | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->pwdn)) {
+		dev_warn(dev, "ov5640-gmsl: failed to get pwdn gpio: %ld\n",
+			 PTR_ERR(sensor->pwdn));
+		sensor->pwdn = NULL;
+	}
+
+	sensor->reset = devm_gpiod_get_optional(dev, "reset",
+				        GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->reset)) {
+		dev_warn(dev, "ov5640-gmsl: failed to get reset gpio: %ld\n",
+			 PTR_ERR(sensor->reset));
+		sensor->reset = NULL;
+	}
+
+	sensor->i2c_mux = devm_gpiod_get_optional(dev, "i2c-mux",
+				          GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->i2c_mux)) {
+		dev_warn(dev, "ov5640-gmsl: failed to get i2c-mux gpio: %ld\n",
+			 PTR_ERR(sensor->i2c_mux));
+		sensor->i2c_mux = NULL;
+	}
+
+	sensor->xclk = devm_clk_get_optional(dev, "xclk");
+	if (IS_ERR(sensor->xclk)) {
+		ret = PTR_ERR(sensor->xclk);
+		goto err_mutex;
+	}
+
+	if (sensor->xclk) {
+		ret = clk_set_rate(sensor->xclk, OV5640_XCLK_FREQ);
+		if (ret) {
+			dev_err(dev, "ov5640-gmsl: failed to set xclk to %u Hz: %d\n",
+				OV5640_XCLK_FREQ, ret);
+			goto err_mutex;
+		}
+	}
+
+	if (!of_property_read_u32(dev->of_node, "deserializer-addr", &val))
+		sensor->des_addr = val;
+	if (!of_property_read_u32(dev->of_node, "serializer-base-addr", &val))
+		sensor->ser_base_addr = val;
+
+
+	ret = ov5640_setup_gmsl(sensor);
+	if (ret)
+		goto err_power;
+
+	ret = ov5640_power_on(sensor);
+	if (ret)
+		goto err_mutex;
+
+        ret = ov5640_write_init_regs(sensor);
+        if (ret) 
+                goto err_power;
+
+	sensor->miscdev.minor = MISC_DYNAMIC_MINOR;
+	sensor->miscdev.fops = &ov5640_fops;
+	sensor->miscdev.parent = dev;
+	if (client->dev.of_node &&
+	    of_property_read_u32(client->dev.of_node, "csi-id", &val) == 0)
+		sensor->miscdev.name = devm_kasprintf(dev, GFP_KERNEL,
+					     "ov5640-gmsl-%u", val);
+	else
+		sensor->miscdev.name = devm_kasprintf(dev, GFP_KERNEL,
+					     "ov5640-gmsl-%02x", client->addr);
+	if (!sensor->miscdev.name) {
+		ret = -ENOMEM;
+		goto err_mutex;
+	}
+
+	ret = misc_register(&sensor->miscdev);
+	if (ret)
+		goto err_mutex;
+
+	global_ov5640_gmsl = sensor;
+	dev_info(dev, "ov5640-gmsl: probe ok, /dev/%s\n", sensor->miscdev.name);
+
+        return 0;
+
+err_power:
+	ov5640_power_off(sensor);
+err_mutex:
+	mutex_destroy(&sensor->lock);
+	return ret;
+}
+
+static void ov5640_remove(struct i2c_client *client)
+{
+	struct ov5640_gmsl *sensor = i2c_get_clientdata(client);
+
+	if (!sensor)
+		return;
+
+	if (global_ov5640_gmsl == sensor)
+		global_ov5640_gmsl = NULL;
+
+	misc_deregister(&sensor->miscdev);
+	ov5640_stream_off(sensor);
+	ov5640_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+}
+
+static const struct of_device_id ov5640_of_match[] = {
+	{ .compatible = "spacemit,ov5640-max96724-max9295" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ov5640_of_match);
+
+static struct i2c_driver ov5640_driver = {
+	.driver = {
+		.name = "ov5640-max96724-max9295",
+		.of_match_table = of_match_ptr(ov5640_of_match),
+	},
+	.probe = ov5640_probe,
+	.remove = ov5640_remove,
+};
+
+module_i2c_driver(ov5640_driver);
+
+MODULE_DESCRIPTION("Simplified OV5640 MAX96724/MAX9295 driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/spacemit/cam_sensor/ov5647_sensor.c b/drivers/media/platform/spacemit/cam_sensor/ov5647_sensor.c
new file mode 100644
index 000000000000..2b48af9a4812
--- /dev/null
+++ b/drivers/media/platform/spacemit/cam_sensor/ov5647_sensor.c
@@ -0,0 +1,853 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Simplified I2C driver for OmniVision OV5647
+ *
+ * Copyright (C) 2025 Spacemit Ltd.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/ioctl.h>
+#include <linux/gpio.h>
+#include <linux/regulator/consumer.h>
+
+/*
+ * Sensor Configuration: 1920x1080 @ 30fps, 2-lane MIPI
+ *
+ * MCLK:           24MHz
+ * Resolution:     1920x1080
+ * Bit Depth:      10bit
+ * FPS:            30fps
+ * HTS:            2416 (0x0970) - registers 0x380c:0x380d
+ * VTS:            1104 (0x0450) - registers 0x380e:0x380f
+ * PCLK:           81.667MHz
+ * MIPI Data Rate: 408.334Mbps/Lane
+ * MIPI Clock:     409MHz
+ * Htime:          29.59us
+ */
+
+/* IOCTL interface for user-space control */
+#define OV5647_IOC_MAGIC		'O'
+#define OV5647_IOCTL_POWER_ON		_IO(OV5647_IOC_MAGIC, 1)
+#define OV5647_IOCTL_POWER_OFF		_IO(OV5647_IOC_MAGIC, 2)
+#define OV5647_IOCTL_INIT_REGS		_IO(OV5647_IOC_MAGIC, 3)
+#define OV5647_IOCTL_STREAM_ON		_IO(OV5647_IOC_MAGIC, 4)
+#define OV5647_IOCTL_STREAM_OFF		_IO(OV5647_IOC_MAGIC, 5)
+#define OV5647_IOCTL_DETECT		_IO(OV5647_IOC_MAGIC, 6)
+#define OV5647_IOCTL_INIT_REGS_1LANE	_IO(OV5647_IOC_MAGIC, 7)
+#define OV5647_IOCTL_INIT_RAW8		_IO(OV5647_IOC_MAGIC, 8)
+
+static struct ov5647 *global_ov5647;
+
+struct ov5647 {
+	struct i2c_client *client;
+	struct gpio_desc *pwdn;
+	struct gpio_desc *i2c_mux;
+	struct mutex lock;
+	struct regulator *vdd;
+	struct miscdevice miscdev;
+};
+
+struct regval_list {
+	u16 addr;
+	u8 data;
+};
+
+/* 1080p register (bringup) */
+static struct regval_list ov5647_1080p_30fps_10bpp_2lane_regs[] = {
+	{0x0100, 0x00},
+	{0x0103, 0x01},
+	{0x3034, 0x1a},
+	{0x3035, 0x21},
+	{0x3036, 0x62},
+	{0x303c, 0x11},
+	{0x3106, 0xf5},
+	{0x3821, 0x06},
+	{0x3820, 0x00},
+	{0x3827, 0xec},
+	{0x370c, 0x03},
+	{0x3612, 0x5b},
+	{0x3618, 0x04},
+	{0x5000, 0x06},
+	{0x5002, 0x41},
+	{0x5003, 0x08},
+	{0x5a00, 0x08},
+	{0x3000, 0x00},
+	{0x3001, 0x00},
+	{0x3002, 0x00},
+	/* 2x io drive */
+	{0x3011, 0x42},
+	/* end */
+	{0x3016, 0x08},
+	{0x3017, 0xe0},
+	{0x3018, 0x44},
+	{0x301c, 0xf8},
+	{0x301d, 0xf0},
+	{0x3a18, 0x00},
+	{0x3a19, 0xf8},
+	{0x3c01, 0x80},
+	{0x3b07, 0x0c},
+	{0x380c, 0x09},
+	{0x380d, 0x70},
+	{0x3814, 0x11},
+	{0x3815, 0x11},
+	{0x3708, 0x64},
+	{0x3709, 0x12},
+	{0x3808, 0x07},
+	{0x3809, 0x80},
+	{0x380a, 0x04},
+	{0x380b, 0x38},
+	{0x3800, 0x01},
+	{0x3801, 0x5c},
+	{0x3802, 0x01},
+	{0x3803, 0xb2},
+	{0x3804, 0x08},
+	{0x3805, 0xe3},
+	{0x3806, 0x05},
+	{0x3807, 0xf1},
+	{0x3811, 0x04},
+	{0x3813, 0x02},
+	{0x3630, 0x2e},
+	{0x3632, 0xe2},
+	{0x3633, 0x23},
+	{0x3634, 0x44},
+	{0x3636, 0x06},
+	{0x3620, 0x64},
+	{0x3621, 0xe0},
+	{0x3600, 0x37},
+	{0x3704, 0xa0},
+	{0x3703, 0x5a},
+	{0x3715, 0x78},
+	{0x3717, 0x01},
+	{0x3731, 0x02},
+	{0x370b, 0x60},
+	{0x3705, 0x1a},
+	{0x3f05, 0x02},
+	{0x3f06, 0x10},
+	{0x3f01, 0x0a},
+	{0x3a08, 0x01},
+	{0x3a09, 0x4b},
+	{0x3a0a, 0x01},
+	{0x3a0b, 0x13},
+	{0x3a0d, 0x04},
+	{0x3a0e, 0x03},
+	{0x3a0f, 0x58},
+	{0x3a10, 0x50},
+	{0x3a1b, 0x58},
+	{0x3a1e, 0x50},
+	{0x3a11, 0x60},
+	{0x3a1f, 0x28},
+	{0x4001, 0x02},
+	{0x4004, 0x04},
+	{0x4000, 0x09},
+	{0x4837, 0x19},
+	{0x4800, 0x34},
+	{0x3503, 0x03},
+	{0x0100, 0x00},
+};
+
+static struct regval_list ov5647_1080p_30fps_10bpp_1lane_regs[] = {
+	{0x0100, 0x00},
+	{0x0103, 0x01},
+	{0x3034, 0x1a},
+	{0x3035, 0x21},
+	{0x3036, 0x62},
+	{0x303c, 0x11},
+	{0x3106, 0xf5},
+	{0x3821, 0x06},
+	{0x3820, 0x00},
+	{0x3827, 0xec},
+	{0x370c, 0x03},
+	{0x3612, 0x5b},
+	{0x3618, 0x04},
+	{0x5000, 0x06},
+	{0x5002, 0x41},
+	{0x5003, 0x08},
+	{0x5a00, 0x08},
+	{0x3000, 0x00},
+	{0x3001, 0x00},
+	{0x3002, 0x00},
+	/* 2x io drive */
+	{0x3011, 0x42},
+	/* end */
+	{0x3016, 0x08},
+	{0x3017, 0xe0},
+	{0x3018, 0x24},
+	{0x301c, 0xf8},
+	{0x301d, 0xf0},
+	{0x3a18, 0x00},
+	{0x3a19, 0xf8},
+	{0x3c01, 0x80},
+	{0x3b07, 0x0c},
+	{0x380c, 0x09},
+	{0x380d, 0x70},
+	{0x3814, 0x11},
+	{0x3815, 0x11},
+	{0x3708, 0x64},
+	{0x3709, 0x12},
+	{0x3808, 0x07},
+	{0x3809, 0x80},
+	{0x380a, 0x04},
+	{0x380b, 0x38},
+	{0x3800, 0x01},
+	{0x3801, 0x5c},
+	{0x3802, 0x01},
+	{0x3803, 0xb2},
+	{0x3804, 0x08},
+	{0x3805, 0xe3},
+	{0x3806, 0x05},
+	{0x3807, 0xf1},
+	{0x3811, 0x04},
+	{0x3813, 0x02},
+	{0x3630, 0x2e},
+	{0x3632, 0xe2},
+	{0x3633, 0x23},
+	{0x3634, 0x44},
+	{0x3636, 0x06},
+	{0x3620, 0x64},
+	{0x3621, 0xe0},
+	{0x3600, 0x37},
+	{0x3704, 0xa0},
+	{0x3703, 0x5a},
+	{0x3715, 0x78},
+	{0x3717, 0x01},
+	{0x3731, 0x02},
+	{0x370b, 0x60},
+	{0x3705, 0x1a},
+	{0x3f05, 0x02},
+	{0x3f06, 0x10},
+	{0x3f01, 0x0a},
+	{0x3a08, 0x01},
+	{0x3a09, 0x4b},
+	{0x3a0a, 0x01},
+	{0x3a0b, 0x13},
+	{0x3a0d, 0x04},
+	{0x3a0e, 0x03},
+	{0x3a0f, 0x58},
+	{0x3a10, 0x50},
+	{0x3a1b, 0x58},
+	{0x3a1e, 0x50},
+	{0x3a11, 0x60},
+	{0x3a1f, 0x28},
+	{0x4001, 0x02},
+	{0x4004, 0x04},
+	{0x4000, 0x09},
+	{0x4837, 0x19},
+	{0x4800, 0x34},
+	{0x3503, 0x03},
+	{0x0100, 0x00},
+};
+
+
+/*
+ * 800x640 RAW8 2-lane MIPI configuration
+ * Based on ov5647_input_24M_MIPI_2lane_raw8_800x640_50fps from ov5647_settings_ori.h
+ *
+ * MCLK:           24MHz
+ * Resolution:     800x640
+ * Bit Depth:      8bit
+ * FPS:            50fps
+ * HTS:            1896
+ * VTS:            984
+ * Lane:           2
+ * MIPI BPS:       374
+ */
+static struct regval_list ov5647_800x640_50fps_2lane_8bpp_regs[] = {
+	{0x0100, 0x00},
+	{0x0103, 0x01},
+	{0x3034, 0x18}, // set RAW format
+	{0x3035, 0x41}, // system clk div
+	{0x3036, 0x80},
+	{0x303c, 0x11},
+	{0x3106, 0xf5},
+	{0x3821, 0x01},
+	{0x3820, 0x41},
+	{0x3827, 0xec},
+	{0x370c, 0x0f},
+	{0x3612, 0x59},
+	{0x3618, 0x00},
+	{0x5000, 0x06},
+	{0x5002, 0x41},
+	{0x5003, 0x08},
+	{0x5a00, 0x08},
+	{0x3000, 0x00},
+	{0x3001, 0x00},
+	{0x3002, 0x00},
+	{0x3016, 0x08},
+	{0x3017, 0xe0},
+	{0x3018, 0x44},
+	{0x301c, 0xf8},
+	{0x301d, 0xf0},
+	{0x3a18, 0x00},
+	{0x3a19, 0xf8},
+	{0x3c01, 0x80},
+	{0x3b07, 0x0c},
+	//HTS line exposure time in # of pixels
+	{0x380c, (1896 >> 8) & 0x1F},
+	{0x380d, 1896 & 0xFF},
+	//VTS frame exposure time in # lines
+	{0x380e, (984 >> 8) & 0xFF},
+	{0x380f, 984 & 0xFF},
+	{0x3814, 0x31},
+	{0x3815, 0x31},
+	{0x3708, 0x64},
+	{0x3709, 0x52},
+	//[3:0]=0 X address start high byte
+	{0x3800, (500 >> 8) & 0x0F},
+	//[7:0]=0 X address start low byte
+	{0x3801, 500 & 0xFF},
+	//[2:0]=0 Y address start high byte
+	{0x3802, (0 >> 8) & 0x07},
+	//[7:0]=0 Y address start low byte
+	{0x3803, 0 & 0xFF},
+	//[3:0] X address end high byte
+	{0x3804, ((2624 - 1) >> 8) & 0x0F},
+	//[7:0] X address end low byte
+	{0x3805, (2624 - 1) & 0xFF},
+	//[2:0] Y address end high byte
+	{0x3806, ((1954 - 1) >> 8) & 0x07},
+	//[7:0] Y address end low byte
+	{0x3807, (1954 - 1) & 0xFF},
+	//[3:0] Output horizontal width high byte
+	{0x3808, (800 >> 8) & 0x0F},
+	//[7:0] Output horizontal width low byte
+	{0x3809, 800 & 0xFF},
+	//[2:0] Output vertical height high byte
+	{0x380a, (800 >> 8) & 0x7F},
+	//[7:0] Output vertical height low byte
+	{0x380b, 800 & 0xFF},
+	//[3:0]=0 timing hoffset high byte
+	{0x3810, (8 >> 8) & 0x0F},
+	//[7:0]=0 timing hoffset low byte
+	{0x3811, 8 & 0xFF},
+	//[2:0]=0 timing voffset high byte
+	{0x3812, (0 >> 8) & 0x07},
+	//[7:0]=0 timing voffset low byte
+	{0x3813, 0 & 0xFF},
+	{0x3630, 0x2e},
+	{0x3632, 0xe2},
+	{0x3633, 0x23},
+	{0x3634, 0x44},
+	{0x3636, 0x06},
+	{0x3620, 0x64},
+	{0x3621, 0xe0},
+	{0x3600, 0x37},
+	{0x3704, 0xa0},
+	{0x3703, 0x5a},
+	{0x3715, 0x78},
+	{0x3717, 0x01},
+	{0x3731, 0x02},
+	{0x370b, 0x60},
+	{0x3705, 0x1a},
+	{0x3f05, 0x02},
+	{0x3f06, 0x10},
+	{0x3f01, 0x0a},
+	{0x3a08, 0x01},
+	{0x3a09, 0x27},
+	{0x3a0a, 0x00},
+	{0x3a0b, 0xf6},
+	{0x3a0d, 0x04},
+	{0x3a0e, 0x03},
+	{0x3a0f, 0x58},
+	{0x3a10, 0x50},
+	{0x3a1b, 0x58},
+	{0x3a1e, 0x50},
+	{0x3a11, 0x60},
+	{0x3a1f, 0x28},
+	{0x4001, 0x02},
+	{0x4004, 0x02},
+	{0x4000, 0x09},
+	{0x4837, 0x14},
+	{0x4050, 0x6e},
+	{0x4051, 0x8f},
+	{0x0100, 0x00},
+};
+
+
+static int ov5647_write(struct ov5647 *sensor, u16 reg, u8 val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msg;
+	u8 data[3];
+	int ret;
+
+	data[0] = (reg >> 8) & 0xff;
+	data[1] = reg & 0xff;
+	data[2] = val & 0xff;
+
+	msg.addr = sensor->client->addr;
+	msg.flags = 0;
+	msg.len = sizeof(data);
+	msg.buf = data;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, &msg, 1);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 1) {
+		dev_err(&sensor->client->dev,
+			"ov5647-test: I2C write failed, reg=0x%x, val=0x%x, ret=%d\n",
+			reg, val, ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	return 0;
+}
+
+static int ov5647_read(struct ov5647 *sensor, u16 reg, u8 *val)
+{
+	struct i2c_adapter *adapter = sensor->client->adapter;
+	struct i2c_msg msgs[2];
+	u8 reg_buf[2];
+	u8 data_buf[1];
+	int ret;
+
+	reg_buf[0] = (reg >> 8) & 0xff;
+	reg_buf[1] = reg & 0xff;
+
+	msgs[0].addr = sensor->client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = sizeof(reg_buf);
+	msgs[0].buf = reg_buf;
+
+	msgs[1].addr = sensor->client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = sizeof(data_buf);
+	msgs[1].buf = data_buf;
+
+	mutex_lock(&sensor->lock);
+	ret = i2c_transfer(adapter, msgs, 2);
+	mutex_unlock(&sensor->lock);
+
+	if (ret != 2) {
+		dev_err(&sensor->client->dev,
+			"ov5647-test: I2C read failed, reg=0x%x, ret=%d\n", reg,
+			ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	*val = data_buf[0];
+	return 0;
+}
+
+static int ov5647_stream_on(struct ov5647 *sensor)
+{
+	return ov5647_write(sensor, 0x0100, 0x01);
+}
+
+static int ov5647_stream_off(struct ov5647 *sensor)
+{
+	return ov5647_write(sensor, 0x0100, 0x00);
+}
+
+static int ov5647_set_virtual_channel(struct ov5647 *sensor, int channel)
+{
+	u8 channel_id;
+	int ret;
+
+	ret = ov5647_read(sensor, 0x4814, &channel_id);
+	if (ret < 0)
+		return ret;
+
+	channel_id &= ~(3 << 6);
+
+	return ov5647_write(sensor, 0x4814, channel_id | (channel << 6));
+}
+
+static int ov5647_write_init_regs(struct ov5647 *sensor, int lane_num);
+static int ov5647_write_raw8_regs(struct ov5647 *sensor);
+static int ov5647_power_on(struct ov5647 *sensor);
+static void ov5647_power_off(struct ov5647 *sensor);
+static int ov5647_detect(struct ov5647 *sensor);
+static int ov5647_set_virtual_channel(struct ov5647 *sensor, int channel);
+
+static long ov5647_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct ov5647 *sensor = file->private_data;
+	int ret = 0;
+
+	if (!sensor)
+		return -ENODEV;
+
+	switch (cmd) {
+	case OV5647_IOCTL_POWER_ON:
+		ret = ov5647_power_on(sensor);
+		break;
+	case OV5647_IOCTL_POWER_OFF:
+		ov5647_power_off(sensor);
+		ret = 0;
+		break;
+	case OV5647_IOCTL_INIT_REGS:
+		ret = ov5647_write_init_regs(sensor, 2);
+		break;
+	case OV5647_IOCTL_INIT_REGS_1LANE:
+		ret = ov5647_write_init_regs(sensor, 1);
+		break;
+	case OV5647_IOCTL_INIT_RAW8:
+		ret = ov5647_write_raw8_regs(sensor);
+		break;
+	case OV5647_IOCTL_STREAM_ON:
+		ret = ov5647_stream_on(sensor);
+		break;
+	case OV5647_IOCTL_STREAM_OFF:
+		ret = ov5647_stream_off(sensor);
+		break;
+	case OV5647_IOCTL_DETECT:
+		ret = ov5647_detect(sensor);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+
+static int ov5647_dev_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *misc = file->private_data;
+	struct ov5647 *sensor;
+
+	if (!misc)
+		return -ENODEV;
+
+	sensor = container_of(misc, struct ov5647, miscdev);
+	/* replace private_data with sensor pointer for use in ioctl */
+	file->private_data = sensor;
+	return 0;
+}
+
+static const struct file_operations ov5647_fops = {
+	.owner = THIS_MODULE,
+	.open = ov5647_dev_open,
+	.unlocked_ioctl = ov5647_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = ov5647_ioctl,
+#endif
+};
+
+static int ov5647_write_init_regs(struct ov5647 *sensor, int lane_num)
+{
+	int i, ret;
+
+	if (lane_num == 2) {
+		dev_info(&sensor->client->dev,
+			 "ov5647-test: write 2lane init regs, total=%ld\n",
+			 ARRAY_SIZE(ov5647_1080p_30fps_10bpp_2lane_regs));
+		for (i = 0; i < ARRAY_SIZE(ov5647_1080p_30fps_10bpp_2lane_regs); i++) {
+			const struct regval_list *reg =
+				&ov5647_1080p_30fps_10bpp_2lane_regs[i];
+
+			ret = ov5647_write(sensor, reg->addr, reg->data);
+			if (ret < 0) {
+				dev_err(&sensor->client->dev,
+					"ov5647-test: write 2lane init reg failed, idx=%d, reg=0x%x, val=0x%x, ret=%d\n",
+					i, reg->addr, reg->data, ret);
+				return ret;
+			}
+		}
+	} else {
+		dev_info(&sensor->client->dev,
+			 "ov5647-test: write 1lane init regs, total=%ld\n",
+			 ARRAY_SIZE(ov5647_1080p_30fps_10bpp_1lane_regs));
+		for (i = 0; i < ARRAY_SIZE(ov5647_1080p_30fps_10bpp_1lane_regs); i++) {
+			const struct regval_list *reg = &ov5647_1080p_30fps_10bpp_1lane_regs[i];
+
+			ret = ov5647_write(sensor, reg->addr, reg->data);
+			if (ret < 0) {
+				dev_err(&sensor->client->dev,
+					"ov5647-test: write 1lane init reg failed, idx=%d, reg=0x%x, val=0x%x, ret=%d\n",
+					i, reg->addr, reg->data, ret);
+				return ret;
+			}
+		}
+	}
+
+	ret = ov5647_set_virtual_channel(sensor, 0);
+	if (ret < 0) {
+		dev_err(&sensor->client->dev,
+			"ov5647-test: set virtual channel failed, ret=%d\n",
+			ret);
+		return ret;
+	}
+
+	dev_info(&sensor->client->dev, "ov5647-test: init regs written\n");
+	return 0;
+}
+
+static int ov5647_write_raw8_regs(struct ov5647 *sensor)
+{
+	int i, ret;
+
+	dev_info(&sensor->client->dev,
+		 "ov5647-test: write 800x640 RAW8 2lane regs, total=%ld\n",
+		 ARRAY_SIZE(ov5647_800x640_50fps_2lane_8bpp_regs));
+
+	for (i = 0; i < ARRAY_SIZE(ov5647_800x640_50fps_2lane_8bpp_regs); i++) {
+		const struct regval_list *reg = &ov5647_800x640_50fps_2lane_8bpp_regs[i];
+
+		ret = ov5647_write(sensor, reg->addr, reg->data);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev,
+				"ov5647-test: write RAW8 reg failed, idx=%d, reg=0x%x, val=0x%x, ret=%d\n",
+				i, reg->addr, reg->data, ret);
+			return ret;
+		}
+	}
+
+	ret = ov5647_set_virtual_channel(sensor, 0);
+	if (ret < 0) {
+		dev_err(&sensor->client->dev,
+			"ov5647-test: set virtual channel failed, ret=%d\n",
+			ret);
+		return ret;
+	}
+
+	dev_info(&sensor->client->dev, "ov5647-test: RAW8 regs written\n");
+	return 0;
+}
+
+static int ov5647_detect(struct ov5647 *sensor)
+{
+	u8 hi, lo;
+	int ret;
+
+	dev_info(&sensor->client->dev, "ov5647-test: detect start\n");
+
+	/* Read sensor ID: registers 0x300a and 0x300b (as in ov5647.c) */
+	ret = ov5647_read(sensor, 0x300a, &hi);
+	if (ret < 0)
+		return ret;
+	if (hi != 0x56) {
+		dev_err(&sensor->client->dev,
+			"ov5647-test: ID high mismatch: 0x%x\n", hi);
+		return -ENODEV;
+	}
+	ret = ov5647_read(sensor, 0x300b, &lo);
+	if (ret < 0)
+		return ret;
+	if (lo != 0x47) {
+		dev_err(&sensor->client->dev,
+			"ov5647-test: ID low mismatch: 0x%x\n", lo);
+		return -ENODEV;
+	}
+
+	dev_info(&sensor->client->dev,
+		 "ov5647-test: detected OV5647 (id %02x%02x)\n", hi, lo);
+	return 0;
+}
+
+static int ov5647_power_on(struct ov5647 *sensor)
+{
+	int ret = 0;
+
+	dev_info(&sensor->client->dev, "ov5647-test: power_on enter\n");
+
+	/* Set I2C mux to select this sensor */
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "ov5647-test: set i2c-mux high\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 1);
+	}
+
+	dev_info(&sensor->client->dev, "ov5647-test: get vdd regulator\n");
+	sensor->vdd = devm_regulator_get(&sensor->client->dev, "vdd");
+	if (IS_ERR(sensor->vdd)) {
+		dev_err(&sensor->client->dev, "ov5647-test: Failed to get vdd regulator: %ld\n",
+			PTR_ERR(sensor->vdd));
+		return PTR_ERR(sensor->vdd);
+	}
+
+	if (sensor->vdd) {
+		ret = regulator_enable(sensor->vdd);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev,
+				"ov5647-test: failed to enable vdd: %d\n", ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "ov5647-test: enable vdd\n");
+
+		ret = regulator_set_voltage(sensor->vdd, 3300000, 3300000);
+		if (ret < 0) {
+			dev_err(&sensor->client->dev, "ov5647-test: failed to set vdd voltage: %d\n",
+				ret);
+			return ret;
+		}
+		dev_info(&sensor->client->dev, "ov5647-test: vdd set to 3.3V\n");
+	}
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev, "ov5647-test: drive PWDN low\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	}
+
+	usleep_range(20000, 21000); /* OV5647 needs 20ms after PWDN goes low */
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev,
+			 "ov5647-test: drive PWDN high\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 1);
+	}
+
+	usleep_range(20000, 21000);  /* OV5647 needs another 20ms after RESETB goes high */
+
+	dev_info(&sensor->client->dev, "ov5647-test: power_on done\n");
+
+	return 0;
+}
+
+static void ov5647_power_off(struct ov5647 *sensor)
+{
+	dev_info(&sensor->client->dev, "ov5647-test: power_off enter\n");
+
+	if (sensor->pwdn) {
+		dev_info(&sensor->client->dev, "ov5647-test: drive PWDN low\n");
+		gpiod_set_value_cansleep(sensor->pwdn, 0);
+	}
+
+	if (sensor->vdd) {
+		dev_info(&sensor->client->dev, "ov5647-test: disable vdd\n");
+		regulator_disable(sensor->vdd);
+	}
+
+	if (sensor->i2c_mux) {
+		dev_info(&sensor->client->dev, "ov5647-test: set i2c-mux low\n");
+		gpiod_set_value_cansleep(sensor->i2c_mux, 0);
+	}
+
+	dev_info(&sensor->client->dev, "ov5647-test: power_off done\n");
+}
+
+static int ov5647_probe(struct i2c_client *client)
+{
+	struct ov5647 *sensor;
+	struct device *dev = &client->dev;
+	int ret;
+
+	dev_info(dev, "ov5647-test: probe enter, client addr=0x%x\n",
+		 client->addr);
+
+	sensor = devm_kzalloc(dev, sizeof(*sensor), GFP_KERNEL);
+	if (!sensor)
+		return -ENOMEM;
+
+	sensor->client = client;
+	mutex_init(&sensor->lock);
+	i2c_set_clientdata(client, sensor);
+
+	dev_info(dev, "ov5647-test: get PWDN gpio\n");
+	sensor->pwdn = devm_gpiod_get_optional(
+		dev, "pwdn", GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->pwdn)) {
+		dev_err(dev, "ov5647-test: Failed to get PWDN GPIO\n");
+		goto err_pwdn;
+	}
+
+	sensor->i2c_mux = devm_gpiod_get_optional(dev, "i2c-mux",
+		GPIOD_OUT_LOW | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
+	if (IS_ERR(sensor->i2c_mux)) {
+		dev_warn(dev, "ov5647-test: Failed to get i2c-mux GPIO, continuing without it\n");
+		sensor->i2c_mux = NULL;
+	}
+
+	ret = ov5647_power_on(sensor);
+	if (ret) {
+		dev_err(dev, "ov5647-test: power on failed: %d\n", ret);
+		goto err_power_on;
+	}
+
+	ret = ov5647_detect(sensor);
+	if (ret) {
+		dev_err(dev, "ov5647-test: sensor detect failed: %d\n", ret);
+		goto err_detect;
+	}
+
+	ov5647_power_off(sensor);
+
+	sensor->miscdev.minor = MISC_DYNAMIC_MINOR;
+	sensor->miscdev.fops = &ov5647_fops;
+	sensor->miscdev.parent = dev;
+	if (client->dev.of_node) {
+		u32 csi_id;
+		if (of_property_read_u32(client->dev.of_node, "csi-id",
+					 &csi_id) == 0) {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "ov5647-%u", csi_id);
+			dev_info(dev, "ov5647-test: ov5647-%u \n", csi_id);
+		} else {
+			sensor->miscdev.name = devm_kasprintf(
+				dev, GFP_KERNEL, "ov5647-%02x", client->addr);
+			dev_info(dev, "ov5647-test: ov5647-%02x \n",
+				 client->addr);
+		}
+	} else {
+		sensor->miscdev.name = devm_kasprintf(
+			dev, GFP_KERNEL, "ov5647-%02x", client->addr);
+	}
+
+	ret = misc_register(&sensor->miscdev);
+	if (ret) {
+		dev_err(dev,
+			"ov5647-test: failed to register misc device: %d\n",
+			ret);
+		goto err_misc_register;
+	}
+
+	global_ov5647 = sensor;
+	dev_info(dev, "ov5647-test: probe successful, ioctl device /dev/%s\n",
+		 sensor->miscdev.name);
+	return 0;
+
+err_misc_register:
+	ov5647_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+	return ret;
+
+err_detect:
+	ov5647_power_off(sensor);
+err_power_on:
+err_pwdn:
+	mutex_destroy(&sensor->lock);
+	return ret;
+}
+
+static void ov5647_remove(struct i2c_client *client)
+{
+	struct ov5647 *sensor = i2c_get_clientdata(client);
+
+	dev_info(&client->dev, "ov5647-test: remove\n");
+	if (global_ov5647 == sensor)
+		global_ov5647 = NULL;
+
+	misc_deregister(&sensor->miscdev);
+	ov5647_power_off(sensor);
+	mutex_destroy(&sensor->lock);
+}
+
+static const struct of_device_id ov5647_of_match[] = {
+	{ .compatible = "ovti,ov5647" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ov5647_of_match);
+
+static struct i2c_driver ov5647_driver = {
+	.driver = {
+		.name		= "ov5647-simple",
+		.of_match_table	= of_match_ptr(ov5647_of_match),
+	},
+	.probe		= ov5647_probe,
+	.remove		= ov5647_remove,
+};
+
+module_i2c_driver(ov5647_driver)
+
+MODULE_DESCRIPTION("Simplified I2C driver for OV5647");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/spacemit/vpu_k3/Kbuild b/drivers/media/platform/spacemit/vpu_k3/Kbuild
new file mode 100644
index 000000000000..e5f0d1934a46
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/Kbuild
@@ -0,0 +1,58 @@
+###########################################################
+# Set the include-path according to the defined interface.
+###########################################################
+
+ccflags-y += -I$(srctree)/drivers/media/platform/spacemit/vpu_k3
+ccflags-y += -I$(srctree)/drivers/media/platform/spacemit/vpu_k3/if
+ccflags-y += -I$(srctree)/drivers/media/platform/spacemit/vpu_k3/dev
+ccflags-y += -I$(srctree)/drivers/media/platform/spacemit/vpu_k3/if/v4l2
+ccflags-y += -I$(srctree)/drivers/media/platform/spacemit/vpu_k3/external
+
+ccflags-$(CONFIG_VIDEO_LINLON_K3_FTRACE) += -DMVX_LOG_FTRACE_ENABLE
+ccflags-$(CONFIG_VIDEO_LINLON_K3_PRINT_FILE) += -DMVX_LOG_PRINT_FILE_ENABLE
+ccflags-y += $(EXTRA_CCFLAGS)
+
+###########################################################
+# Define build targets and what files to include.
+###########################################################
+
+# Amvx module
+obj-$(CONFIG_VIDEO_LINLON_K3) := amvx.o
+
+# Add objects for if module.
+if-y := if/mvx_if.o \
+	if/mvx_buffer.o \
+	if/mvx_firmware_cache.o \
+	if/mvx_firmware.o \
+	if/mvx_firmware_v2.o \
+	if/mvx_firmware_v3.o \
+	if/mvx_mmu.o \
+	if/mvx_secure.o \
+	if/mvx_session.o
+
+# Add external interface.
+if-y += if/v4l2/mvx_ext_v4l2.o \
+	if/v4l2/mvx_v4l2_buffer.o \
+	if/v4l2/mvx_v4l2_session.o \
+	if/v4l2/mvx_v4l2_vidioc.o \
+	if/v4l2/mvx_v4l2_fops.o \
+	if/v4l2/mvx_v4l2_ctrls.o
+
+# Add objects for dev module.
+dev-y := dev/mvx_dev.o \
+	 dev/mvx_hwreg.o \
+	 dev/mvx_hwreg_v500.o \
+	 dev/mvx_hwreg_v550.o \
+	 dev/mvx_hwreg_v61.o \
+	 dev/mvx_hwreg_v52_v76.o \
+	 dev/mvx_lsid.o \
+	 dev/mvx_scheduler.o \
+	 mvx_pm_runtime.o
+
+# Add driver objects.
+amvx-y := mvx_driver.o \
+	  mvx_seq.o \
+	  mvx_log.o \
+	  mvx_log_group.o \
+          mvx_dvfs.o \
+	  $(if-y) $(dev-y)
diff --git a/drivers/media/platform/spacemit/vpu_k3/Kconfig b/drivers/media/platform/spacemit/vpu_k3/Kconfig
new file mode 100644
index 000000000000..a1bfe899e66a
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/Kconfig
@@ -0,0 +1,26 @@
+config VIDEO_LINLON_K3
+	tristate "Linlon VPU support."
+	depends on VIDEO_DEV
+#    depends on VIDEO_V4L2
+    select VIDEOBUF2_VMALLOC
+    select VIDEOBUF2_CORE
+    select DMA_SHARED_BUFFER
+    select VIDEOBUF2_MEMOPS
+    select VIDEOBUF2_DMA_SG
+	default n
+	help
+	  This enables support for the Linlon VPU family.
+
+config VIDEO_LINLON_FTRACE_K3
+	depends on VIDEO_LINLON_K3
+	bool "Send kernel space logs to ftrace."
+	default n
+	help
+		Send kernel space logs to ftrace.
+
+config VIDEO_LINLON_PRINT_FILE_K3
+	depends on VIDEO_LINLON_K3
+	bool "Append file and line number to kernel space log messages."
+	default y
+	help
+		Append file and line number to kernel space log messages.
diff --git a/drivers/media/platform/spacemit/vpu_k3/Makefile b/drivers/media/platform/spacemit/vpu_k3/Makefile
new file mode 100644
index 000000000000..7861be84b1bf
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/Makefile
@@ -0,0 +1,11 @@
+all: mono_v4l2
+
+mono_v4l2:
+	@env CONFIG_VIDEO_LINLON=m CONFIG_VIDEO_LINLON_MONO=y CONFIG_VIDEO_LINLON_IF_V4L2=y $(MAKE) -C $(KDIR) M=$(CURDIR) modules
+
+clean:
+	@rm -rf *.ko
+	@find . -type f -name '*.o' -delete
+	@rm -rf *.mod.c
+	@rm -f Module.symvers
+	@rm -f modules.order
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_dev.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_dev.c
new file mode 100644
index 000000000000..c02d82493f1d
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_dev.c
@@ -0,0 +1,733 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/bitops.h>
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/pm_runtime.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/printk.h>
+#include <linux/workqueue.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/version.h>
+#include "mvx_bitops.h"
+#include "mvx_dev.h"
+#include "mvx_if.h"
+#include "mvx_session.h"
+#include "mvx_log_group.h"
+#include "mvx_pm_runtime.h"
+#include "mvx_dvfs.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+/**
+ * Name of the MVx dev device.
+ */
+#define MVX_DEV_NAME "amvx_dev"
+
+#define MVX_PCI_VENDOR 0x13b5
+#define MVX_PCI_DEVICE 0x0001
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+/**
+ * struct mvx_client_session - Device session.
+ *
+ * When the if module registers a session this structure is returned.
+ */
+struct mvx_client_session {
+	struct mvx_dev_ctx *ctx;
+	struct mvx_sched_session session;
+};
+/****************************************************************************
+ * Static variables and functions
+ ****************************************************************************/
+
+static struct mvx_dev_ctx *client_ops_to_ctx(struct mvx_client_ops *client)
+{
+	return container_of(client, struct mvx_dev_ctx, client_ops);
+}
+
+static void get_hw_ver(struct mvx_client_ops *client, struct mvx_hw_ver *hw_ver)
+{
+	struct mvx_dev_ctx *ctx = client_ops_to_ctx(client);
+
+	hw_ver->id = ctx->hw_id;
+	hw_ver->revision = ctx->hw_revision;
+	hw_ver->patch = ctx->hw_patch;
+}
+
+static void get_formats(struct mvx_client_ops *client, enum mvx_direction direction, uint64_t *formats)
+{
+	struct mvx_dev_ctx *ctx = client_ops_to_ctx(client);
+	uint32_t fuses;
+
+	*formats = 0;
+
+	ctx->hwreg.ops.get_formats(direction, formats);
+
+	/* Remove formats based on fuses. */
+	fuses = ctx->fuses;
+	if (fuses & MVX_HWREG_FUSE_DISABLE_AFBC) {
+		mvx_clear_bit(MVX_FORMAT_YUV420_AFBC_8, formats);
+		mvx_clear_bit(MVX_FORMAT_YUV420_AFBC_10, formats);
+		mvx_clear_bit(MVX_FORMAT_YUV422_AFBC_8, formats);
+		mvx_clear_bit(MVX_FORMAT_YUV422_AFBC_10, formats);
+	}
+
+	if (fuses & MVX_HWREG_FUSE_DISABLE_REAL)
+		mvx_clear_bit(MVX_FORMAT_RV, formats);
+
+	if (fuses & MVX_HWREG_FUSE_DISABLE_VPX) {
+		mvx_clear_bit(MVX_FORMAT_VP8, formats);
+		mvx_clear_bit(MVX_FORMAT_VP9, formats);
+	}
+
+	if (fuses & MVX_HWREG_FUSE_DISABLE_HEVC)
+		mvx_clear_bit(MVX_FORMAT_HEVC, formats);
+}
+
+static unsigned int get_ncores(struct mvx_client_ops *client)
+{
+	struct mvx_dev_ctx *ctx = client_ops_to_ctx(client);
+	uint32_t ncores;
+
+	ncores = ctx->ncores;
+
+	return ncores;
+}
+
+static struct mvx_client_session *register_session(struct mvx_client_ops *client, struct mvx_if_session *isession)
+{
+	struct mvx_dev_ctx *ctx = client_ops_to_ctx(client);
+	struct mvx_client_session *csession;
+	int ret;
+
+	csession = devm_kzalloc(ctx->dev, sizeof(*csession), GFP_KERNEL);
+	if (csession == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	csession->ctx = ctx;
+
+	ret = mvx_pm_runtime_get_sync(ctx->dev);
+	if (ret < 0)
+		goto free_session;
+
+	ret = mvx_sched_session_construct(&csession->session, isession);
+	if (ret != 0)
+		goto runtime_put;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO,
+		      "Register client session. csession=0x%p, isession=0x%p.",
+		      csession, isession);
+
+	return csession;
+
+runtime_put:
+	mvx_pm_runtime_put_sync(csession->ctx->dev);
+free_session:
+	devm_kfree(ctx->dev, csession);
+
+	return ERR_PTR(ret);
+}
+
+static void unregister_session(struct mvx_client_session *csession)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "%p Unregister client session. csession=0x%p.",
+		      mvx_if_session_to_session(csession->session.isession), csession);
+
+	mvx_sched_terminate(&csession->ctx->scheduler, &csession->session);
+	mvx_sched_session_destruct(&csession->session);
+
+	mvx_pm_runtime_put_sync(csession->ctx->dev);
+
+	devm_kfree(csession->ctx->dev, csession);
+}
+
+static int switch_in(struct mvx_client_session *csession)
+{
+	struct mvx_dev_ctx *ctx = csession->ctx;
+	int ret;
+
+	ret = mvx_sched_switch_in(&ctx->scheduler, &csession->session);
+
+	return ret;
+}
+
+static int send_irq(struct mvx_client_session *csession)
+{
+	struct mvx_dev_ctx *ctx = csession->ctx;
+	int ret;
+
+	ret = mvx_sched_send_irq(&ctx->scheduler, &csession->session);
+
+	return ret;
+}
+
+static int flush_mmu(struct mvx_client_session *csession)
+{
+	struct mvx_dev_ctx *ctx = csession->ctx;
+	int ret;
+
+	ret = mvx_sched_flush_mmu(&ctx->scheduler, &csession->session);
+
+	return ret;
+}
+
+static void print_debug(struct mvx_client_session *csession)
+{
+	struct mvx_dev_ctx *ctx = csession->ctx;
+
+	mvx_sched_print_debug(&ctx->scheduler, &csession->session);
+}
+
+static void wait_session_idle(struct mvx_client_session *csession)
+{
+	struct mvx_dev_ctx *ctx = csession->ctx;
+
+	mvx_sched_wait_session_idle(&ctx->scheduler, &csession->session);
+}
+
+static struct mvx_dev_ctx *work_to_ctx(struct work_struct *work)
+{
+	return container_of(work, struct mvx_dev_ctx, work);
+}
+
+/**
+ * irq_bottom() - Handle IRQ bottom.
+ * @work:	Work struct that is part of the context structure.
+ *
+ * This function is called from a work queue and id doing the actual work of
+ * handling the interrupt.
+ */
+static void irq_bottom(struct work_struct *work)
+{
+	struct mvx_dev_ctx *ctx = work_to_ctx(work);
+	uint32_t nlsid;
+	uint32_t i;
+
+	nlsid = ctx->scheduler.nlsid;
+	for (i = 0; i < nlsid; i++)
+		if (test_and_clear_bit(i, &ctx->irqve))
+			mvx_sched_handle_irq(&ctx->scheduler, i);
+}
+
+/**
+ * irq_top() - Handle IRQ top.
+ * @irq:	IRQ number.
+ * @dev_id:	Pointer to context.
+ *
+ * This function is called in interrupt context. It should be short and must not
+ * block.
+ *
+ * Return: IRQ status if the IRQ was handled or not.
+ */
+static irqreturn_t irq_top(int irq, void *dev_id)
+{
+	struct mvx_dev_ctx *ctx = dev_id;
+	uint32_t nlsid;
+	uint32_t irqve;
+	int ret = IRQ_NONE;
+
+	nlsid = mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_NLSID);
+	irqve = mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_IRQVE);
+	while (nlsid-- > 0)
+		if ((irqve >> nlsid) & 0x1) {
+			mvx_hwreg_write_lsid(&ctx->hwreg, nlsid, MVX_HWREG_LIRQVE, 0);
+			wmb(); // make sure the LIRQVE is cleared.
+			set_bit(nlsid, &ctx->irqve);
+			ret = IRQ_HANDLED;
+		}
+
+	queue_work(ctx->work_queue, &ctx->work);
+
+	return ret;
+}
+
+static int mvx_dev_probe(struct device *dev, struct resource *iores, struct resource *irqres)
+{
+	struct mvx_dev_ctx *ctx;
+	int ret;
+
+	/* Create device context and store pointer in device private data. */
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (ctx == NULL)
+		return -EINVAL;
+
+	ctx->dev = dev;
+	ctx->clock = devm_clk_get(dev, NULL);
+	mutex_init(&ctx->pm_mutex);
+
+	dev_set_drvdata(dev, ctx);
+	mvx_dvfs_init(dev);
+
+	ret = mvx_pm_runtime_get_sync(ctx->dev);
+	if (ret < 0)
+		goto free_ctx;
+
+	/*handle reset for k1x*/
+	ctx->rst = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(ctx->rst))
+		goto exit_reset;
+
+	reset_control_deassert(ctx->rst);
+
+	/* Setup client ops callbacks. */
+	ctx->client_ops.get_hw_ver = get_hw_ver;
+	ctx->client_ops.get_formats = get_formats;
+	ctx->client_ops.get_ncores = get_ncores;
+	ctx->client_ops.register_session = register_session;
+	ctx->client_ops.unregister_session = unregister_session;
+	ctx->client_ops.switch_in = switch_in;
+	ctx->client_ops.send_irq = send_irq;
+	ctx->client_ops.flush_mmu = flush_mmu;
+	ctx->client_ops.print_debug = print_debug;
+	ctx->client_ops.wait_session_idle = wait_session_idle;
+
+	clk_set_rate(ctx->clock, 1000000000);
+
+	/* Create if context. */
+	ctx->if_ops = mvx_if_create(dev, &ctx->client_ops, ctx);
+	if (IS_ERR(ctx->if_ops))
+		goto runtime_put;
+
+	/* Create debugfs entry */
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		char name[20];
+
+		scnprintf(name, sizeof(name), "%s%u", MVX_DEV_NAME, dev->id);
+		ctx->dentry = debugfs_create_dir(name, NULL);
+		if (IS_ERR_OR_NULL(ctx->dentry)) {
+			ret = -EINVAL;
+			goto destroy_if;
+		}
+	}
+
+	/* Construct hw register context. */
+	ret = mvx_hwreg_construct(&ctx->hwreg, dev, iores, ctx->dentry);
+	if (ret != 0)
+		goto destruct_dentry;
+
+	ret = mvx_sched_construct(&ctx->scheduler, dev, ctx->if_ops, &ctx->hwreg, ctx->dentry);
+	if (ret != 0)
+		goto destruct_hwreg;
+
+	ctx->fuses = mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_FUSE);
+	ctx->ncores = mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_NCORES);
+
+	/* Create work queue for IRQ handler. */
+	ctx->work_queue = alloc_workqueue(dev_name(dev), WQ_UNBOUND, 1);
+	if (ctx->work_queue == NULL) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "Failed to create work queue.");
+		ret = -EINVAL;
+		goto destruct_sched;
+	}
+
+	INIT_WORK(&ctx->work, irq_bottom);
+
+	/* Request IRQ handler. */
+	ctx->irq = irqres->start;
+	ret = request_irq(ctx->irq, irq_top, IRQF_SHARED | (irqres->flags & IRQF_TRIGGER_MASK), dev_name(dev), ctx);
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "Failed to request IRQ. irq=%u, ret=%d.", ctx->irq, ret);
+		goto workqueue_destroy;
+	}
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+		      "Linlon v%x identified. cores=%u, nlsid=%u, id=%u, fuse=%08x, hw_id=%08x, hw_rev=%08x",
+		      mvx_hwreg_get_hw_id(&ctx->hwreg, NULL, NULL), mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_NCORES),
+		      mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_NLSID), dev->id, ctx->fuses, ctx->hw_id, ctx->hw_revision);
+
+	ret = mvx_pm_runtime_put_sync(ctx->dev);
+	if (ret < 0)
+		goto irq_free;
+
+	return 0;
+
+irq_free:
+	free_irq(ctx->irq, ctx);
+
+workqueue_destroy:
+	destroy_workqueue(ctx->work_queue);
+
+destruct_sched:
+	mvx_sched_destruct(&ctx->scheduler);
+
+destruct_hwreg:
+	mvx_hwreg_destruct(&ctx->hwreg);
+
+destruct_dentry:
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(ctx->dentry);
+
+destroy_if:
+	mvx_if_destroy(ctx->if_ops);
+
+runtime_put:
+	pm_runtime_put_sync(ctx->dev);
+
+exit_reset:
+	reset_control_assert(ctx->rst);
+
+free_ctx:
+	devm_kfree(dev, ctx);
+
+	return ret;
+}
+
+static int mvx_dev_remove(struct mvx_dev_ctx *ctx)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "remove");
+	mvx_dvfs_deinit(ctx->dev);
+
+	mvx_if_destroy(ctx->if_ops);
+	free_irq(ctx->irq, ctx);
+	destroy_workqueue(ctx->work_queue);
+	mvx_sched_destruct(&ctx->scheduler);
+	mvx_hwreg_destruct(&ctx->hwreg);
+	dev_set_drvdata(ctx->dev, NULL);
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(ctx->dentry);
+
+	reset_control_assert(ctx->rst);
+
+	devm_kfree(ctx->dev, ctx);
+
+	return 0;
+}
+
+/****************************************************************************
+ * Platform driver
+ ****************************************************************************/
+
+static int mvx_pdev_probe(struct platform_device *pdev)
+{
+	struct resource iores;
+	struct resource irqres;
+	int irq;
+	int ret;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "probe");
+
+	/* Get resource. */
+	ret = of_address_to_resource(pdev->dev.of_node, 0, &iores);
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "Failed to get address of resource. ret=%d.", ret);
+		return ret;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+
+	if (irq < 0) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "Failed to get IRQ resource.");
+		return -EINVAL;
+	}
+
+	irqres.start = irq;
+	irqres.end = irq;
+	irqres.flags = 0;
+
+	pm_runtime_enable(&pdev->dev);
+
+	ret = mvx_dev_probe(&pdev->dev, &iores, &irqres);
+	if (ret != 0)
+		pm_runtime_disable(&pdev->dev);
+
+	return ret;
+}
+
+#if (KERNEL_VERSION(6, 10, 0) <= LINUX_VERSION_CODE)
+static void mvx_pdev_remove(struct platform_device *pdev)
+#else
+static int mvx_pdev_remove(struct platform_device *pdev)
+#endif
+{
+	struct mvx_dev_ctx *ctx = platform_get_drvdata(pdev);
+	int ret;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "pdev remove");
+
+	ret = mvx_dev_remove(ctx);
+
+	pm_runtime_disable(&pdev->dev);
+
+#if (KERNEL_VERSION(6, 10, 0) <= LINUX_VERSION_CODE)
+	return;
+#else
+	return ret;
+#endif
+}
+
+#ifdef CONFIG_PM
+static void reset_hw(struct device *dev)
+{
+	uint32_t ncores;
+	uint32_t corelsid_mask;
+	struct mvx_dev_ctx *ctx;
+	ctx = dev_get_drvdata(dev);
+
+	if (ctx->hwreg.registers != NULL) {
+		ncores = mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_NCORES);
+		corelsid_mask = 0;
+		for (; ncores > 0; --ncores) {
+			corelsid_mask = (corelsid_mask << 4) | 0xF;
+		}
+		mvx_hwreg_write(&ctx->hwreg, MVX_HWREG_RESET, 1);
+		while (corelsid_mask != mvx_hwreg_read(&ctx->hwreg, MVX_HWREG_CORELSID)) {
+		}
+		mvx_hwreg_write(&ctx->hwreg, MVX_HWREG_CLKFORCE, 0);
+	}
+}
+
+static void mvx_pm_disable_clk(struct device *dev)
+{
+	struct clk *clock;
+	struct mvx_dev_ctx *ctx;
+
+	ctx = dev_get_drvdata(dev);
+	clock = ctx->clock;
+
+	if (!IS_ERR_OR_NULL(clock)) {
+		clk_disable_unprepare(clock);
+	}
+}
+
+static void mvx_pm_enable_clk(struct device *dev)
+{
+	struct clk *clock;
+	struct mvx_dev_ctx *ctx;
+
+	ctx = dev_get_drvdata(dev);
+	clock = ctx->clock;
+
+	if (!IS_ERR_OR_NULL(clock)) {
+		clk_prepare_enable(clock);
+	}
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int mvx_pm_poweron(struct device *dev)
+{
+	struct mvx_dev_ctx *ctx = dev_get_drvdata(dev);
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_poweron");
+	mvx_pm_enable_clk(dev);
+	reset_hw(dev);
+	mvx_sched_resume(&ctx->scheduler);
+
+	return 0;
+}
+
+static int mvx_pm_poweroff(struct device *dev)
+{
+	struct mvx_dev_ctx *ctx = dev_get_drvdata(dev);
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_poweroff");
+	mvx_sched_suspend(&ctx->scheduler);
+	mvx_pm_disable_clk(dev);
+
+	return 0;
+}
+
+static int b_backto_active;
+static int mvx_pm_suspend(struct device *dev)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_suspend start. b_backto_active=%d", b_backto_active);
+
+	if (!pm_runtime_status_suspended(dev)) {
+		mvx_pm_poweroff(dev);
+		b_backto_active = true;
+	} else {
+		b_backto_active = false;
+	}
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_suspend exit. b_backto_active=%d", b_backto_active);
+
+	return 0;
+}
+
+static int mvx_pm_resume(struct device *dev)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_resume start. b_backto_active=%d", b_backto_active);
+
+	if (b_backto_active) {
+		mvx_pm_poweron(dev);
+	}
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_resume exit. b_backto_active=%d", b_backto_active);
+	return 0;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static int mvx_pm_runtime_suspend(struct device *dev)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_runtime_suspend");
+	mvx_pm_disable_clk(dev);
+	return 0;
+}
+
+static int mvx_pm_runtime_resume(struct device *dev)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_runtime_resume");
+	mvx_pm_enable_clk(dev);
+	reset_hw(dev);
+
+	return 0;
+}
+
+static int mvx_pm_runtime_idle(struct device *dev)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_pm_runtime_idle");
+	return 0;
+}
+
+static const struct dev_pm_ops mvx_dev_pm_ops = {
+	SET_RUNTIME_PM_OPS(mvx_pm_runtime_suspend, mvx_pm_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(mvx_pm_suspend, mvx_pm_resume)
+	.runtime_idle = mvx_pm_runtime_idle,
+};
+#endif /* CONFIG_PM */
+
+static const struct of_device_id mvx_dev_match_table[] = {
+	{ .compatible = "arm,mali-mve"  },
+	{ .compatible = "arm,mali-v500" },
+	{ .compatible = "arm,mali-v550" },
+	{ .compatible = "arm,mali-v61"  },
+	{ .compatible = "arm china,linlon-v5"  },
+	{ .compatible = "arm china,linlon-v7"  },
+	{ { 0 } }
+};
+
+static struct platform_driver mvx_dev_driver = {
+	.probe                  = mvx_pdev_probe,
+	.remove                 = mvx_pdev_remove,
+	.driver                 = {
+		.name           = MVX_DEV_NAME,
+		.owner          = THIS_MODULE,
+		.of_match_table = mvx_dev_match_table,
+#ifdef CONFIG_PM
+		.pm             = &mvx_dev_pm_ops
+#endif /* CONFIG_PM */
+	}
+};
+
+/****************************************************************************
+ * PCI driver
+ ****************************************************************************/
+
+/* LCOV_EXCL_START */
+static int mvx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	static unsigned int dev_id;
+	struct resource irqres = {
+		.start = pdev->irq,
+		.end   = pdev->irq,
+		.flags = 0
+	};
+	pdev->dev.id = dev_id++;
+	return mvx_dev_probe(&pdev->dev, &pdev->resource[1], &irqres);
+}
+
+static void mvx_pci_remove(struct pci_dev *pdev)
+{
+	struct mvx_dev_ctx *ctx = pci_get_drvdata(pdev);
+
+	mvx_dev_remove(ctx);
+}
+
+static struct pci_device_id mvx_pci_device_id[] = {
+	{ PCI_DEVICE(MVX_PCI_VENDOR, MVX_PCI_DEVICE) },
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, mvx_pci_device_id);
+
+static struct pci_driver mvx_pci_driver = {
+	.name     = MVX_DEV_NAME,
+	.id_table = mvx_pci_device_id,
+	.probe    = mvx_pci_probe,
+	.remove   = mvx_pci_remove
+};
+/* LCOV_EXCL_STOP */
+
+/****************************************************************************
+ * Exported variables and functions
+ ****************************************************************************/
+
+int mvx_dev_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&mvx_dev_driver);
+	if (ret != 0) {
+		pr_err("mvx_dev: Failed to register driver.\n");
+		return ret;
+	}
+
+	/* LCOV_EXCL_START */
+	ret = pci_register_driver(&mvx_pci_driver);
+	if (ret != 0) {
+		pr_err("mvx_dev: Failed to register PCI driver.\n");
+		goto unregister_driver;
+	}
+
+	/* LCOV_EXCL_STOP */
+
+	return 0;
+
+unregister_driver:
+	platform_driver_unregister(&mvx_dev_driver); /* LCOV_EXCL_LINE */
+
+	return ret;
+}
+
+void mvx_dev_exit(void)
+{
+	pci_unregister_driver(&mvx_pci_driver); /* LCOV_EXCL_LINE */
+	platform_driver_unregister(&mvx_dev_driver);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_dev.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_dev.h
new file mode 100644
index 000000000000..3ce4df515c62
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_dev.h
@@ -0,0 +1,77 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_DEV_H_
+#define _MVX_DEV_H_
+
+#include <linux/reset.h>
+#include "mvx_hwreg.h"
+#include "mvx_scheduler.h"
+
+/**
+ * struct mvx_dev_ctx - Private context for the MVx dev device.
+ */
+struct mvx_dev_ctx {
+	struct device *dev;
+	struct mvx_if_ops *if_ops;
+	struct mvx_client_ops client_ops;
+	struct mvx_hwreg hwreg;
+	struct mvx_sched scheduler;
+	unsigned int irq;
+	struct workqueue_struct *work_queue;
+	struct work_struct work;
+	unsigned long irqve;
+	struct dentry *dentry;
+	struct clk *clock;
+	struct mutex pm_mutex;
+	uint32_t fuses;
+	uint32_t ncores;
+	enum mvx_hw_id hw_id;
+	uint32_t hw_revision;
+	uint32_t hw_patch;
+	struct reset_control *rst;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_dev_init() - Initialize the dev device.
+ */
+int mvx_dev_init(void);
+
+/**
+ * mvx_dev_exit() - Remove and exit the dev device.
+ */
+void mvx_dev_exit(void);
+
+#endif /* _MVX_DEV_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg.c
new file mode 100644
index 000000000000..1a9ae0c65d85
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg.c
@@ -0,0 +1,405 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/device.h>
+#include <linux/of_address.h>
+#include <linux/clk.h>
+#include "mvx_log_group.h"
+#include "mvx_hwreg.h"
+#include "mvx_hwreg_v500.h"
+#include "mvx_hwreg_v550.h"
+#include "mvx_hwreg_v61.h"
+#include "mvx_hwreg_v52_v76.h"
+#include "mvx_pm_runtime.h"
+#include "mvx_dev.h"
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+static unsigned int get_offset(enum mvx_hwreg_what what)
+{
+	switch (what) {
+	case MVX_HWREG_HARDWARE_ID:
+		return 0x0;
+	case MVX_HWREG_ENABLE:
+		return 0x4;
+	case MVX_HWREG_NCORES:
+		return 0x8;
+	case MVX_HWREG_NLSID:
+		return 0xc;
+	case MVX_HWREG_CORELSID:
+		return 0x10;
+	case MVX_HWREG_JOBQUEUE:
+		return 0x14;
+	case MVX_HWREG_IRQVE:
+		return 0x18;
+	case MVX_HWREG_CLKFORCE:
+		return 0x24;
+	case MVX_HWREG_FUSE:
+		return 0x34;
+	case MVX_HWREG_CONFIG:
+		return 0x38;
+	case MVX_HWREG_PROTCTRL:
+		return 0x40;
+	case MVX_HWREG_RESET:
+		return 0x50;
+	default:
+		return 0;
+	}
+}
+
+static unsigned int get_lsid_offset(unsigned int lsid, enum mvx_hwreg_lsid what)
+{
+	unsigned int offset = 0x0200 + 0x40 * lsid;
+
+	switch (what) {
+	case MVX_HWREG_CTRL:
+		offset += 0x0;
+		break;
+	case MVX_HWREG_MMU_CTRL:
+		offset += 0x4;
+		break;
+	case MVX_HWREG_NPROT:
+		offset += 0x8;
+		break;
+	case MVX_HWREG_ALLOC:
+		offset += 0xc;
+		break;
+	case MVX_HWREG_FLUSH_ALL:
+		offset += 0x10;
+		break;
+	case MVX_HWREG_SCHED:
+		offset += 0x14;
+		break;
+	case MVX_HWREG_TERMINATE:
+		offset += 0x18;
+		break;
+	case MVX_HWREG_LIRQVE:
+		offset += 0x1c;
+		break;
+	case MVX_HWREG_IRQHOST:
+		offset += 0x20;
+		break;
+	case MVX_HWREG_INTSIG:
+		offset += 0x24;
+		break;
+	case MVX_HWREG_STREAMID:
+		offset += 0x2c;
+		break;
+	case MVX_HWREG_BUSATTR_0:
+		offset += 0x30;
+		break;
+	case MVX_HWREG_BUSATTR_1:
+		offset += 0x34;
+		break;
+	case MVX_HWREG_BUSATTR_2:
+		offset += 0x38;
+		break;
+	case MVX_HWREG_BUSATTR_3:
+		offset += 0x3c;
+		break;
+	default:
+		return 0;
+	}
+
+	return offset;
+}
+
+static enum mvx_hw_id get_hw_id(void *registers, uint32_t *revision, uint32_t *patch)
+{
+	uint32_t value;
+
+	value = readl(registers);
+
+	if (revision != NULL)
+		*revision = (value >> 8) & 0xff;
+
+	if (patch != NULL)
+		*patch = value & 0xff;
+
+	switch (value >> 16) {
+	case 0x5650:
+		return MVE_v500;
+	case 0x5655:
+		return MVE_v550;
+	case 0x5660:
+	case 0x5661:
+		return MVE_v61;
+	case 0x5662:
+	case 0x5663:
+		return MVE_v52_v76;
+	default:
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "Unknown hardware version. version=0x%08x.", value);
+		break;
+	}
+
+	return MVE_Unknown;
+}
+
+static int regs_show(struct seq_file *s, void *v)
+{
+	struct mvx_hwreg *hwreg = (struct mvx_hwreg *)s->private;
+	int ret;
+
+	ret = mvx_pm_runtime_get_sync(hwreg->dev);
+	if (ret < 0)
+		return 0;
+
+	seq_printf(s, "HARDWARE_ID = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_HARDWARE_ID));
+	seq_printf(s, "ENABLE = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_ENABLE));
+	seq_printf(s, "NCORES = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_NCORES));
+	seq_printf(s, "NLSID = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_NLSID));
+	seq_printf(s, "CORELSID = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_CORELSID));
+	seq_printf(s, "JOBQUEUE = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_JOBQUEUE));
+	seq_printf(s, "IRQVE = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_IRQVE));
+	seq_printf(s, "CLKFORCE = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_CLKFORCE));
+	seq_printf(s, "FUSE = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_FUSE));
+	seq_printf(s, "CONFIG = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_CONFIG));
+	seq_printf(s, "PROTCTRL = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_PROTCTRL));
+	seq_printf(s, "RESET = 0x%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_RESET));
+	seq_puts(s, "\n");
+
+	mvx_pm_runtime_put_sync(hwreg->dev);
+
+	return 0;
+}
+
+static int regs_open(struct inode *inode, struct file *file) { return single_open(file, regs_show, inode->i_private); }
+
+static const struct file_operations regs_fops = {
+	.open = regs_open, .read = seq_read, .llseek = seq_lseek, .release = single_release};
+
+static int regs_debugfs_init(struct mvx_hwreg *hwreg, struct dentry *parent)
+{
+	struct dentry *dentry;
+
+	dentry = debugfs_create_file("regs", 0400, parent, hwreg, &regs_fops);
+	if (IS_ERR_OR_NULL(dentry))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int lsid_regs_show(struct seq_file *s, void *v)
+{
+	struct mvx_lsid_hwreg *lsid_hwreg = (struct mvx_lsid_hwreg *)s->private;
+	struct mvx_hwreg *hwreg = lsid_hwreg->hwreg;
+	int lsid = lsid_hwreg->lsid;
+	int ret;
+
+	ret = mvx_pm_runtime_get_sync(hwreg->dev);
+	if (ret < 0)
+		return 0;
+
+	seq_printf(s, "CTRL = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_CTRL));
+	seq_printf(s, "MMU_CTRL = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_MMU_CTRL));
+	seq_printf(s, "NPROT = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_NPROT));
+	seq_printf(s, "ALLOC = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_ALLOC));
+	seq_printf(s, "FLUSH_ALL = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_FLUSH_ALL));
+	seq_printf(s, "SCHED = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_SCHED));
+	seq_printf(s, "TERMINATE = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_TERMINATE));
+	seq_printf(s, "LIRQVE = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_LIRQVE));
+	seq_printf(s, "IRQHOST = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_IRQHOST));
+	seq_printf(s, "INTSIG = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_INTSIG));
+	seq_printf(s, "STREAMID = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_STREAMID));
+	seq_printf(s, "BUSATTR_0 = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_BUSATTR_0));
+	seq_printf(s, "BUSATTR_1 = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_BUSATTR_1));
+	seq_printf(s, "BUSATTR_2 = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_BUSATTR_2));
+	seq_printf(s, "BUSATTR_3 = 0x%08x\n", mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_BUSATTR_3));
+	seq_puts(s, "\n");
+
+	mvx_pm_runtime_put_sync(hwreg->dev);
+
+	return 0;
+}
+
+static int lsid_regs_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, lsid_regs_show, inode->i_private);
+}
+
+static const struct file_operations lsid_regs_fops = {
+	.open    = lsid_regs_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = single_release
+};
+
+static int lsid_regs_debugfs_init(struct mvx_lsid_hwreg *lsid_hwreg, struct dentry *parent)
+{
+	struct dentry *dentry;
+	char name[20];
+
+	scnprintf(name, sizeof(name), "lsid%u_regs", lsid_hwreg->lsid);
+
+	dentry = debugfs_create_file(name, 0400, parent, lsid_hwreg, &lsid_regs_fops);
+	if (IS_ERR_OR_NULL(dentry))
+		return -ENOMEM;
+
+	return 0;
+}
+
+int debugfs_init(struct mvx_hwreg *hwreg, struct dentry *parent)
+{
+	int ret;
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		int lsid;
+
+		ret = regs_debugfs_init(hwreg, parent);
+		if (ret != 0)
+			return ret;
+
+		for (lsid = 0; lsid < MVX_LSID_MAX; ++lsid) {
+			ret = lsid_regs_debugfs_init(&hwreg->lsid_hwreg[lsid], parent);
+			if (ret != 0)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_hwreg_construct(struct mvx_hwreg *hwreg, struct device *dev, struct resource *res, struct dentry *parent)
+{
+	char const *name = dev_name(dev);
+	enum mvx_hw_id hw_id;
+	int ret;
+	int lsid;
+	struct mvx_dev_ctx *ctx = dev_get_drvdata(dev);
+
+	hwreg->dev = dev;
+
+	hwreg->res = request_mem_region(res->start, resource_size(res), name);
+	if (hwreg->res == NULL) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "Failed to request mem region. start=0x%llx, size=0x%llx.",
+			      res->start, resource_size(res));
+		return -EINVAL;
+	}
+
+	hwreg->registers = ioremap(res->start, resource_size(res));
+	if (hwreg->registers == NULL) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "Failed to iomap region. start=0x%llx, size=0x%llx.",
+			      res->start, resource_size(res));
+		ret = -ENOMEM;
+		goto release_mem;
+	}
+
+	hw_id = get_hw_id(hwreg->registers, &ctx->hw_revision, &ctx->hw_patch);
+	switch (hw_id) {
+	case MVE_v500:
+		hwreg->ops.get_formats = mvx_hwreg_get_formats_v500;
+		break;
+	case MVE_v550:
+		hwreg->ops.get_formats = mvx_hwreg_get_formats_v550;
+		break;
+	case MVE_v61:
+		hwreg->ops.get_formats = mvx_hwreg_get_formats_v61;
+		break;
+	case MVE_v52_v76:
+		hwreg->ops.get_formats = mvx_hwreg_get_formats_v52_v76;
+		break;
+	default:
+		ret = -EINVAL;
+		goto unmap_io;
+	}
+
+	ctx->hw_id = hw_id;
+
+	for (lsid = 0; lsid < MVX_LSID_MAX; ++lsid) {
+		hwreg->lsid_hwreg[lsid].hwreg = hwreg;
+		hwreg->lsid_hwreg[lsid].lsid = lsid;
+	}
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ret = debugfs_init(hwreg, parent);
+		if (ret != 0)
+			goto unmap_io;
+	}
+
+	return 0;
+
+unmap_io:
+	iounmap(hwreg->registers);
+
+release_mem:
+	release_mem_region(res->start, resource_size(res));
+
+	return ret;
+}
+
+void mvx_hwreg_destruct(struct mvx_hwreg *hwreg)
+{
+	iounmap(hwreg->registers);
+	release_mem_region(hwreg->res->start, resource_size(hwreg->res));
+}
+
+uint32_t mvx_hwreg_read(struct mvx_hwreg *hwreg, enum mvx_hwreg_what what)
+{
+	unsigned int offset = get_offset(what);
+
+	return readl(hwreg->registers + offset);
+}
+
+void mvx_hwreg_write(struct mvx_hwreg *hwreg, enum mvx_hwreg_what what, uint32_t value)
+{
+	unsigned int offset = get_offset(what);
+
+	writel(value, hwreg->registers + offset);
+}
+
+uint32_t mvx_hwreg_read_lsid(struct mvx_hwreg *hwreg, unsigned int lsid, enum mvx_hwreg_lsid what)
+{
+	unsigned int offset = get_lsid_offset(lsid, what);
+
+	return readl(hwreg->registers + offset);
+}
+
+void mvx_hwreg_write_lsid(struct mvx_hwreg *hwreg, unsigned int lsid, enum mvx_hwreg_lsid what, uint32_t value)
+{
+	unsigned int offset = get_lsid_offset(lsid, what);
+
+	writel(value, hwreg->registers + offset);
+}
+
+enum mvx_hw_id mvx_hwreg_get_hw_id(struct mvx_hwreg *hwreg, uint32_t *revision, uint32_t *patch)
+{
+	return get_hw_id(hwreg->registers, revision, patch);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg.h
new file mode 100644
index 000000000000..cdc692fab4cf
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg.h
@@ -0,0 +1,218 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_HW_REG_
+#define _MVX_HW_REG_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/debugfs.h>
+#include <linux/types.h>
+#include <linux/ioport.h>
+#include "mvx_if.h"
+#include "mvx_lsid.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define MVX_HWREG_FUSE_DISABLE_AFBC     (1 << 0)
+#define MVX_HWREG_FUSE_DISABLE_REAL     (1 << 1)
+#define MVX_HWREG_FUSE_DISABLE_VPX      (1 << 2)
+#define MVX_HWREG_FUSE_DISABLE_HEVC     (1 << 3)
+
+#define MVE_JOBQUEUE_JOB_BITS           8
+#define MVE_JOBQUEUE_JOB_MASK           ((1 << MVE_JOBQUEUE_JOB_BITS) - 1)
+#define MVE_JOBQUEUE_JOB_INVALID        0xf
+#define MVE_JOBQUEUE_NJOBS              4
+#define MVE_JOBQUEUE_LSID_SHIFT         0
+#define MVE_JOBQUEUE_LSID_BITS          4
+#define MVE_JOBQUEUE_LSID_MASK          ((1 << MVE_JOBQUEUE_LSID_BITS) - 1)
+#define MVE_JOBQUEUE_NCORES_SHIFT       4
+#define MVE_JOBQUEUE_NCORES_BITS        4
+
+#define MVE_CORELSID_LSID_BITS          4
+#define MVX_CORELSID_LSID_MASK          ((1 << MVE_CORELSID_LSID_BITS) - 1)
+
+#define MVE_CTRL_DISALLOW_SHIFT         0
+#define MVE_CTRL_DISALLOW_BITS          8
+#define MVE_CTRL_DISALLOW_MASK          ((1 << MVE_CTRL_DISALLOW_BITS) - 1)
+#define MVE_CTRL_MAXCORES_SHIFT         8
+#define MVE_CTRL_MAXCORES_BITS          4
+#define MVE_CTRL_MAXCORES_MASK          ((1 << MVE_CTRL_MAXCORES_BITS) - 1)
+
+#define MVE_ALLOC_FREE                  0
+#define MVE_ALLOC_NON_PROTECTED         1
+#define MVE_ALLOC_PROTECTED             2
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+
+/**
+ * enum mvx_hwreg_what - Hardware registers that can be read or written.
+ */
+enum mvx_hwreg_what {
+	MVX_HWREG_HARDWARE_ID,
+	MVX_HWREG_ENABLE,
+	MVX_HWREG_NCORES,
+	MVX_HWREG_NLSID,
+	MVX_HWREG_CORELSID,
+	MVX_HWREG_JOBQUEUE,
+	MVX_HWREG_IRQVE,
+	MVX_HWREG_CLKFORCE,
+	MVX_HWREG_FUSE,
+	MVX_HWREG_PROTCTRL,
+	MVX_HWREG_RESET,
+	MVX_HWREG_CONFIG,
+	MVX_HWREG_WHAT_MAX
+};
+
+/**
+ * enum mvx_hwreg_lsid - Hardware registers per LSID.
+ */
+enum mvx_hwreg_lsid {
+	MVX_HWREG_CTRL,
+	MVX_HWREG_MMU_CTRL,
+	MVX_HWREG_NPROT,
+	MVX_HWREG_ALLOC,
+	MVX_HWREG_FLUSH_ALL,
+	MVX_HWREG_SCHED,
+	MVX_HWREG_TERMINATE,
+	MVX_HWREG_LIRQVE,
+	MVX_HWREG_IRQHOST,
+	MVX_HWREG_INTSIG,
+	MVX_HWREG_STREAMID,
+	MVX_HWREG_BUSATTR_0,
+	MVX_HWREG_BUSATTR_1,
+	MVX_HWREG_BUSATTR_2,
+	MVX_HWREG_BUSATTR_3,
+	MVX_HWREG_LSID_MAX
+};
+
+struct mvx_hwreg;
+
+/**
+ * struct mvx_lsid_hwreg - Helper struct used for debugfs reading of lsid
+ *			   dependent registers.
+ */
+struct mvx_lsid_hwreg {
+	struct mvx_hwreg *hwreg;
+	unsigned int lsid;
+};
+
+/**
+ * struct mvx_hwreg - Context class for the hardware register interface.
+ */
+struct mvx_hwreg {
+	struct device *dev;
+	struct resource *res;
+	void *registers;
+	struct mvx_lsid_hwreg lsid_hwreg[MVX_LSID_MAX];
+	struct {
+		void (*get_formats)(enum mvx_direction direction, uint64_t *formats);
+	} ops;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_hwreg_construct() - Construct the hardware register object.
+ * @hwreg:	Pointer to hwreg object.
+ * @dev:	Pointer to device struct.
+ * @res:	Memory resource.
+ * @parent:	Parent debugfs directory entry.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_hwreg_construct(struct mvx_hwreg *hwreg, struct device *dev, struct resource *res, struct dentry *parent);
+
+/**
+ * mvx_hwreg_destruct() - Destroy the hardware register object.
+ * @hwreg:	Pointer to hwreg object.
+ */
+void mvx_hwreg_destruct(struct mvx_hwreg *hwreg);
+
+/**
+ * mvx_hwreg_read() - Read hardware register.
+ * @hwreg:	Pointer to hwreg object.
+ * @what:	Which register to read.
+ *
+ * Return: Value of register.
+ */
+uint32_t mvx_hwreg_read(struct mvx_hwreg *hwreg, enum mvx_hwreg_what what);
+
+/**
+ * mvx_hwreg_write() - Write hardware register.
+ * @hwreg:	Pointer to hwreg object.
+ * @what:	Which register to write.
+ * @value:	Value to write.
+ */
+void mvx_hwreg_write(struct mvx_hwreg *hwreg, enum mvx_hwreg_what what, uint32_t value);
+
+/**
+ * mvx_hwreg_read_lsid() - Read LSID hardware register.
+ * @hwreg:	Pointer to hwreg object.
+ * @lsid:	LSID register index.
+ * @what:	Which register to read.
+ *
+ * Return: Value of register.
+ */
+uint32_t mvx_hwreg_read_lsid(struct mvx_hwreg *hwreg, unsigned int lsid, enum mvx_hwreg_lsid what);
+
+/**
+ * mvx_hwreg_write_lsid() - Write LSID hardware register.
+ * @hwreg:	Pointer to hwreg object.
+ * @lsid:	LSID register index.
+ * @what:	Which register to write.
+ * @value:	Value to write.
+ */
+void mvx_hwreg_write_lsid(struct mvx_hwreg *hwreg, unsigned int lsid, enum mvx_hwreg_lsid what, uint32_t value);
+
+/**
+ * mvx_hwreg_get_hw_id() - Get hardware id.
+ * @hwreg:	Pointer to hwreg object.
+ * @revision:	Hardware revision.
+ * @patch:	Hardware patch revision.
+ *
+ * Return: Hardware id.
+ */
+enum mvx_hw_id mvx_hwreg_get_hw_id(struct mvx_hwreg *hwreg, uint32_t *revision, uint32_t *patch);
+
+int debugfs_init(struct mvx_hwreg *hwreg, struct dentry *parent);
+
+#endif /* _MVX_HW_REG_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v500.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v500.c
new file mode 100644
index 000000000000..09fd54efb433
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v500.c
@@ -0,0 +1,64 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_bitops.h"
+#include "mvx_hwreg_v500.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v500(enum mvx_direction direction, uint64_t *formats)
+{
+	if (direction == MVX_DIR_INPUT) {
+		mvx_set_bit(MVX_FORMAT_H263, formats);
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG2, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG4, formats);
+		mvx_set_bit(MVX_FORMAT_RV, formats);
+		mvx_set_bit(MVX_FORMAT_VC1, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+	} else {
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_HEVC, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+	}
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v500.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v500.h
new file mode 100644
index 000000000000..a347dc75e03e
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v500.h
@@ -0,0 +1,47 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_HWREG_V500_H_
+#define _MVX_HWREG_V500_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_if.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v500(enum mvx_direction direction, uint64_t *formats);
+
+#endif /* _MVX_HWREG_V500_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v52_v76.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v52_v76.c
new file mode 100644
index 000000000000..669ef18f9b45
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v52_v76.c
@@ -0,0 +1,96 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/io.h>
+#include "mvx_bitops.h"
+#include "mvx_hwreg_v52_v76.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v52_v76(enum mvx_direction direction, uint64_t *formats)
+{
+	if (direction == MVX_DIR_INPUT) {
+		mvx_set_bit(MVX_FORMAT_AVS, formats);
+		mvx_set_bit(MVX_FORMAT_AVS2, formats);
+		mvx_set_bit(MVX_FORMAT_H263, formats);
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_HEVC, formats);
+		mvx_set_bit(MVX_FORMAT_JPEG, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG2, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG4, formats);
+		mvx_set_bit(MVX_FORMAT_RV, formats);
+		mvx_set_bit(MVX_FORMAT_VC1, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_VP9, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV21, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_P010, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_Y0L2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AQB1, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_YUY2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_UYVY, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_Y210, formats);
+		mvx_set_bit(MVX_FORMAT_RGBA_8888, formats);
+		mvx_set_bit(MVX_FORMAT_BGRA_8888, formats);
+		mvx_set_bit(MVX_FORMAT_ARGB_8888, formats);
+		mvx_set_bit(MVX_FORMAT_ABGR_8888, formats);
+	} else {
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_HEVC, formats);
+		mvx_set_bit(MVX_FORMAT_JPEG, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_VP9, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV21, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_P010, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_Y0L2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AQB1, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_YUY2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_UYVY, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_Y210, formats);
+	}
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v52_v76.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v52_v76.h
new file mode 100644
index 000000000000..331ca2010707
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v52_v76.h
@@ -0,0 +1,48 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_HWREG_V52_V76_H_
+#define _MVX_HWREG_V52_V76_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_hwreg.h"
+#include "mvx_if.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v52_v76(enum mvx_direction direction, uint64_t *formats);
+
+#endif /* _MVX_HWREG_V52_V76_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v550.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v550.c
new file mode 100644
index 000000000000..92b1a08a4185
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v550.c
@@ -0,0 +1,81 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_bitops.h"
+#include "mvx_hwreg_v550.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v550(enum mvx_direction direction, uint64_t *formats)
+{
+	if (direction == MVX_DIR_INPUT) {
+		mvx_set_bit(MVX_FORMAT_H263, formats);
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_HEVC, formats);
+		mvx_set_bit(MVX_FORMAT_JPEG, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG2, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG4, formats);
+		mvx_set_bit(MVX_FORMAT_RV, formats);
+		mvx_set_bit(MVX_FORMAT_VC1, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV21, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_P010, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_Y0L2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AQB1, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_YUY2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_UYVY, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_Y210, formats);
+	} else {
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV21, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_P010, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_Y0L2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AQB1, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_YUY2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_UYVY, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_Y210, formats);
+	}
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v550.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v550.h
new file mode 100644
index 000000000000..bc035bc007d1
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v550.h
@@ -0,0 +1,47 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_HWREG_V550_H_
+#define _MVX_HWREG_V550_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_if.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v550(enum mvx_direction direction, uint64_t *formats);
+
+#endif /* _MVX_HWREG_V550_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v61.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v61.c
new file mode 100644
index 000000000000..6e300f796f8d
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v61.c
@@ -0,0 +1,94 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/io.h>
+#include "mvx_bitops.h"
+#include "mvx_hwreg_v61.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v61(enum mvx_direction direction, uint64_t *formats)
+{
+	if (direction == MVX_DIR_INPUT) {
+		mvx_set_bit(MVX_FORMAT_H263, formats);
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_HEVC, formats);
+		mvx_set_bit(MVX_FORMAT_JPEG, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG2, formats);
+		mvx_set_bit(MVX_FORMAT_MPEG4, formats);
+		mvx_set_bit(MVX_FORMAT_RV, formats);
+		mvx_set_bit(MVX_FORMAT_VC1, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_VP9, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV21, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_P010, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_Y0L2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AQB1, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_YUY2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_UYVY, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_Y210, formats);
+		mvx_set_bit(MVX_FORMAT_RGBA_8888, formats);
+		mvx_set_bit(MVX_FORMAT_BGRA_8888, formats);
+		mvx_set_bit(MVX_FORMAT_ARGB_8888, formats);
+		mvx_set_bit(MVX_FORMAT_ABGR_8888, formats);
+	} else {
+		mvx_set_bit(MVX_FORMAT_H264, formats);
+		mvx_set_bit(MVX_FORMAT_HEVC, formats);
+		mvx_set_bit(MVX_FORMAT_JPEG, formats);
+		mvx_set_bit(MVX_FORMAT_VP8, formats);
+		mvx_set_bit(MVX_FORMAT_VP9, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_8, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_AFBC_10, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_I420, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV12, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_NV21, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_P010, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_Y0L2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV420_AQB1, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_YUY2, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_UYVY, formats);
+		mvx_set_bit(MVX_FORMAT_YUV422_Y210, formats);
+	}
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v61.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v61.h
new file mode 100644
index 000000000000..0f3addc49bdb
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_hwreg_v61.h
@@ -0,0 +1,48 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_HWREG_V61_H_
+#define _MVX_HWREG_V61_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_hwreg.h"
+#include "mvx_if.h"
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+void mvx_hwreg_get_formats_v61(enum mvx_direction direction, uint64_t *formats);
+
+#endif /* _MVX_HWREG_V61_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_lsid.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_lsid.c
new file mode 100644
index 000000000000..d5d4f25a42c1
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_lsid.c
@@ -0,0 +1,300 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/device.h>
+#include <linux/of.h>
+#include "mvx_if.h"
+#include "mvx_hwreg.h"
+#include "mvx_lsid.h"
+#include "mvx_log_group.h"
+#include "mvx_session.h"
+
+/****************************************************************************
+ * Private functions
+ ****************************************************************************/
+
+static bool is_alloc(struct mvx_lsid *lsid)
+{
+	uint32_t alloc;
+
+	alloc = mvx_hwreg_read_lsid(lsid->hwreg, lsid->lsid, MVX_HWREG_ALLOC);
+
+	return alloc != MVE_ALLOC_FREE;
+}
+
+static uint32_t get_core_lsid(uint32_t reg, unsigned int core)
+{
+	return (reg >> (MVE_CORELSID_LSID_BITS * core)) & MVX_CORELSID_LSID_MASK;
+}
+
+static uint32_t get_jobqueue_job(uint32_t reg, unsigned int nr)
+{
+	return (reg >> (MVE_JOBQUEUE_JOB_BITS * nr)) & MVE_JOBQUEUE_JOB_MASK;
+}
+
+static uint32_t set_jobqueue_job(uint32_t reg, unsigned int nr, uint32_t job)
+{
+	reg &= ~(MVE_JOBQUEUE_JOB_MASK << (nr * MVE_JOBQUEUE_JOB_BITS));
+	reg |= job << (MVE_JOBQUEUE_JOB_BITS * nr);
+	return reg;
+}
+
+static uint32_t get_jobqueue_lsid(uint32_t reg, unsigned int nr)
+{
+	return (reg >> (MVE_JOBQUEUE_JOB_BITS * nr + MVE_JOBQUEUE_LSID_SHIFT)) & MVE_JOBQUEUE_LSID_MASK;
+}
+
+static uint32_t set_lsid_ncores(uint32_t reg, unsigned int nr, unsigned int lsid, unsigned int ncores)
+{
+	reg &= ~(MVE_JOBQUEUE_JOB_MASK << (nr * MVE_JOBQUEUE_JOB_BITS));
+	reg |= ((lsid << MVE_JOBQUEUE_LSID_SHIFT) | ((ncores - 1) << MVE_JOBQUEUE_NCORES_SHIFT))
+	       << (nr * MVE_JOBQUEUE_JOB_BITS);
+
+	return reg;
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_lsid_construct(struct mvx_lsid *lsid, struct device *dev, struct mvx_hwreg *hwreg, unsigned int id)
+{
+	lsid->dev = dev;
+	lsid->hwreg = hwreg;
+	lsid->session = NULL;
+	lsid->lsid = id;
+
+	return 0;
+}
+
+void mvx_lsid_destruct(struct mvx_lsid *lsid) {}
+
+int mvx_lsid_map(struct mvx_lsid *lsid, struct mvx_lsid_pcb *pcb)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+	uint32_t alloc;
+	uint32_t busattr[4];
+	int ret;
+
+	/* Check that the LSID is not already allocated. */
+	if (is_alloc(lsid)) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+			      "Failed to map session to LSID. LSID already allocated. lsid=%u.", lsid->lsid);
+		return -EFAULT;
+	}
+
+	/* Allocate LSID. */
+	alloc = pcb->nprot == 0 ? MVE_ALLOC_PROTECTED : MVE_ALLOC_NON_PROTECTED;
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_ALLOC, alloc);
+
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_TERMINATE, 1);
+	do {
+		ret = mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_TERMINATE);
+	} while (ret != 0);
+
+	/* Configure number of cores to use and which to cores to disable. */
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_CTRL, pcb->ctrl);
+
+	/* Configure MMU L0 entry and flush MMU tables. */
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_MMU_CTRL, pcb->mmu_ctrl);
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_FLUSH_ALL, 0);
+
+	if (of_property_read_u32_array(lsid->dev->of_node, "busattr", busattr, ARRAY_SIZE(busattr))) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_VERBOSE, "busattr in of_node is not available.");
+
+		/* We apply default values in this case. */
+		busattr[0] = 0;
+		busattr[1] = 0;
+		busattr[2] = 0x33;
+		busattr[3] = 0x33;
+	} else {
+		int i;
+
+		for (i = 0; i < ARRAY_SIZE(busattr); i++)
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_VERBOSE, "busattr[%d] = 0x%x.", i, busattr[i]);
+	}
+
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_BUSATTR_0, busattr[0]);
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_BUSATTR_1, busattr[1]);
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_BUSATTR_2, busattr[2]);
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_BUSATTR_3, busattr[3]);
+
+	/* Restore interrupt registers. */
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_LIRQVE, 0);
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_IRQHOST, pcb->irqhost);
+
+	/*
+	 * Make sure all register writes have completed before scheduling is
+	 * enabled.
+	 */
+	wmb();
+
+	/* Enable scheduling. */
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_SCHED, 1);
+
+	return 0;
+}
+
+void mvx_lsid_unmap(struct mvx_lsid *lsid, struct mvx_lsid_pcb *pcb)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+
+	if (!is_alloc(lsid)) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "LSID was not allocated. lsid=%u.", lsid->lsid);
+		return;
+	}
+
+	/* Disable scheduling. */
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_SCHED, 0);
+
+	/* Store registers in process control block. */
+	pcb->ctrl = mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_CTRL);
+	pcb->mmu_ctrl = mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_MMU_CTRL);
+	pcb->irqhost = mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_IRQHOST);
+	pcb->nprot = mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_NPROT);
+
+	/* Deallocate LSID. */
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_ALLOC, MVE_ALLOC_FREE);
+}
+
+int mvx_lsid_jobqueue_add(struct mvx_lsid *lsid, unsigned int ncores)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+	uint32_t jobqueue;
+	int i;
+
+	/* Disable scheduling. */
+	mvx_hwreg_write(hwreg, MVX_HWREG_ENABLE, 0);
+
+	jobqueue = mvx_hwreg_read(hwreg, MVX_HWREG_JOBQUEUE);
+
+	/* Search if the LSID is already in the job queue. */
+	for (i = 0; i < MVE_JOBQUEUE_NJOBS; i++)
+		if (get_jobqueue_lsid(jobqueue, i) == lsid->lsid)
+			goto jobqueue_enable;
+
+	/* Search for a free slot in the job queue. */
+	for (i = 0; i < MVE_JOBQUEUE_NJOBS; i++)
+		if (get_jobqueue_lsid(jobqueue, i) == MVE_JOBQUEUE_JOB_INVALID) {
+			jobqueue = set_lsid_ncores(jobqueue, i, lsid->lsid, ncores);
+			mvx_hwreg_write(hwreg, MVX_HWREG_JOBQUEUE, jobqueue);
+			break;
+		}
+
+jobqueue_enable:
+	/* Reenable scheduling. */
+	mvx_hwreg_write(hwreg, MVX_HWREG_ENABLE, 1);
+
+	return i < MVE_JOBQUEUE_NJOBS ? 0 : -EAGAIN;
+}
+
+void mvx_lsid_send_irq(struct mvx_lsid *lsid)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_IRQHOST, 1);
+}
+
+void mvx_lsid_flush_mmu(struct mvx_lsid *lsid)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_FLUSH_ALL, 0);
+}
+
+void mvx_lsid_terminate(struct mvx_lsid *lsid)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+	uint32_t ret;
+
+	mvx_hwreg_write_lsid(hwreg, lsid->lsid, MVX_HWREG_TERMINATE, 1);
+
+	do {
+		ret = mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_TERMINATE);
+	} while (ret != 0);
+}
+
+void mvx_lsid_jobqueue_remove(struct mvx_lsid *lsid)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+	uint32_t jobqueue;
+	int i;
+	int j;
+	uint32_t ncores = mvx_hwreg_read(hwreg, MVX_HWREG_NCORES);
+
+	/* Disable scheduling. */
+	mvx_hwreg_write(hwreg, MVX_HWREG_ENABLE, 0);
+
+	jobqueue = mvx_hwreg_read(hwreg, MVX_HWREG_JOBQUEUE);
+
+	/* Copy job entries that do not match the LSID to be removed. */
+	for (i = 0, j = 0; i < MVE_JOBQUEUE_NJOBS; i++)
+		if (get_jobqueue_lsid(jobqueue, i) != lsid->lsid)
+			jobqueue = set_jobqueue_job(jobqueue, j++, get_jobqueue_job(jobqueue, i));
+
+	/* Blank out remaining job entries. */
+	for (; j < MVE_JOBQUEUE_NJOBS; j++)
+		jobqueue = set_lsid_ncores(jobqueue, j, MVE_JOBQUEUE_JOB_INVALID, ncores);
+
+	mvx_hwreg_write(hwreg, MVX_HWREG_JOBQUEUE, jobqueue);
+
+	/* Reenable scheduling. */
+	mvx_hwreg_write(hwreg, MVX_HWREG_ENABLE, 1);
+}
+
+bool mvx_lsid_idle(struct mvx_lsid *lsid)
+{
+	struct mvx_hwreg *hwreg = lsid->hwreg;
+	uint32_t jobqueue;
+	uint32_t corelsid;
+	uint32_t ncores;
+	uint32_t i;
+
+	jobqueue = mvx_hwreg_read(hwreg, MVX_HWREG_JOBQUEUE);
+	corelsid = mvx_hwreg_read(hwreg, MVX_HWREG_CORELSID);
+	ncores = mvx_hwreg_read(hwreg, MVX_HWREG_NCORES);
+
+	/* Check if LSID is found in job queue. */
+	for (i = 0; i < MVE_JOBQUEUE_NJOBS; i++)
+		if (get_jobqueue_lsid(jobqueue, i) == lsid->lsid)
+			return false;
+
+	/* Check if LSID is found in core lsid. */
+	for (i = 0; i < ncores; i++)
+		if (get_core_lsid(corelsid, i) == lsid->lsid)
+			return false;
+
+	return true;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_lsid.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_lsid.h
new file mode 100644
index 000000000000..be32604f04a1
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_lsid.h
@@ -0,0 +1,160 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_LSID_H_
+#define _MVX_LSID_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/types.h>
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define MVX_LSID_MAX    4
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct mvx_hwreg;
+struct mvx_sched_session;
+
+/**
+ * struct mvx_lsid_pcb - LSID process control block.
+ *
+ * This structure is used to store the register map when a session is unmapped
+ * from a LSID, so it can be restored again when the session is remapped.
+ */
+struct mvx_lsid_pcb {
+	uint32_t ctrl;
+	uint32_t mmu_ctrl;
+	uint32_t irqhost;
+	uint32_t nprot;
+};
+
+/**
+ * struct mvx_lsid - LSID class.
+ */
+struct mvx_lsid {
+	struct device *dev;
+	struct mvx_hwreg *hwreg;
+	struct mvx_sched_session *session;
+	unsigned int lsid;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_lsid_construct() - Construct the LSID object.
+ * @lsid:	Pointer to LSID object.
+ * @dev:	Pointer to device.
+ * @hwreg:	Pointer to hwreg object.
+ * @id:		LSID number.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_lsid_construct(struct mvx_lsid *lsid, struct device *dev, struct mvx_hwreg *hwreg, unsigned int id);
+
+/**
+ * mvx_lsid_destruct() - Destruct the LSID object.
+ * @lsid:	Pointer to LSID object.
+ */
+void mvx_lsid_destruct(struct mvx_lsid *lsid);
+
+/**
+ * mvx_lsid_map() - Map a session to this LSID.
+ * @lsid:	Pointer to LSID object.
+ * @pcb:	Process control block to be restored.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_lsid_map(struct mvx_lsid *lsid, struct mvx_lsid_pcb *pcb);
+
+/**
+ * mvx_lsid_unmap() - Unmap session from LSID.
+ * @lsid:	Pointer to LSID object.
+ * @pcb:	Process control block where the registers are stored.
+ *
+ * A LSID must not be unmapped if it is present in the job queue or core LSID.
+ * It is the responsibility of the scheduler to guarantee that the LSID is idle
+ * before it is unmapped.
+ */
+void mvx_lsid_unmap(struct mvx_lsid *lsid, struct mvx_lsid_pcb *pcb);
+
+/**
+ * mvx_lsid_jobqueue_add() - Add LSID to job queue.
+ * @lsid:	Pointer to LSID object.
+ * @ncores:	Number of cores to request.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_lsid_jobqueue_add(struct mvx_lsid *lsid, unsigned int ncores);
+
+/**
+ * mvx_lsid_send_irq() - Send IRQ to firmware.
+ * @lsid:	Pointer to LSID object.
+ */
+void mvx_lsid_send_irq(struct mvx_lsid *lsid);
+
+/**
+ * mvx_lsid_flush_mmu() - Flush MMU tables.
+ * @lsid:	Pointer to LSID object.
+ */
+void mvx_lsid_flush_mmu(struct mvx_lsid *lsid);
+
+/**
+ * mvx_lsid_terminate() - Terminate the LSID.
+ * @lsid:	Pointer to LSID object.
+ */
+void mvx_lsid_terminate(struct mvx_lsid *lsid);
+
+/**
+ * mvx_lsid_jobqueue_remove() - Remove LSID from job queue.
+ * @lsid:	Pointer to LSID object.
+ */
+void mvx_lsid_jobqueue_remove(struct mvx_lsid *lsid);
+
+/**
+ * mvx_lsid_idle() - Check if LSID is idle.
+ * @lsid:	Pointer to LSID object.
+ *
+ * Return: true if LSID is idle, else false.
+ */
+bool mvx_lsid_idle(struct mvx_lsid *lsid);
+
+#endif /* _MVX_LSID_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_scheduler.c b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_scheduler.c
new file mode 100644
index 000000000000..795d84b1b0ce
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_scheduler.c
@@ -0,0 +1,873 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/errno.h>
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/workqueue.h>
+#include <linux/delay.h>
+#include "mvx_if.h"
+#include "mvx_hwreg.h"
+#include "mvx_mmu.h"
+#include "mvx_scheduler.h"
+#include "mvx_session.h"
+#include "mvx_seq.h"
+#include "mvx_pm_runtime.h"
+#include "mvx_log_group.h"
+#include "mvx_dvfs.h"
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+static struct mvx_lsid *find_free_lsid(struct mvx_sched *sched)
+{
+	unsigned int i;
+
+	for (i = 0; i < sched->nlsid; i++)
+		if (sched->lsid[i].session == NULL)
+			return &sched->lsid[i];
+
+	return NULL;
+}
+
+static struct mvx_lsid *find_idle_lsid(struct mvx_sched *sched)
+{
+	unsigned int i;
+
+	for (i = 0; i < sched->nlsid; i++) {
+		bool idle;
+
+		idle = mvx_lsid_idle(&sched->lsid[i]);
+		if (idle != false)
+			return &sched->lsid[i];
+	}
+
+	return NULL;
+}
+
+static int map_session(struct mvx_sched *sched, struct mvx_sched_session *session, struct mvx_lsid *lsid)
+{
+	int ret;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "%p Map LSID. lsid=%u, jobqueue=%08x, corelsid=%08x.",
+		      mvx_if_session_to_session(session->isession), lsid->lsid,
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_JOBQUEUE),
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_CORELSID));
+
+	ret = mvx_lsid_map(lsid, &session->pcb);
+	if (ret != 0)
+		return ret;
+
+	session->lsid = lsid;
+	lsid->session = session;
+
+	return 0;
+}
+
+static void unmap_session(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	struct mvx_lsid *lsid = session->lsid;
+
+	if (lsid == NULL)
+		return;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "%p Unmap LSID. lsid=%u, jobqueue=%08x, corelsid=%08x.",
+		      mvx_if_session_to_session(session->isession), lsid->lsid,
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_JOBQUEUE),
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_CORELSID));
+
+	mvx_lsid_unmap(lsid, &session->pcb);
+	session->lsid = NULL;
+	lsid->session = NULL;
+}
+
+static struct list_head *list_find_node(struct list_head *list, struct list_head *node)
+{
+	struct list_head *i;
+
+	list_for_each(i, list) {
+		if (i == node)
+			return i;
+	}
+
+	return NULL;
+}
+
+/**
+ * pending list is only updated when sched is locked.
+ * a session can only be added once
+ *
+ * notify_list = []
+ * lock_sched
+ * for pending in pending_list:
+ *      if is_mapped(pending):
+ *              jobqueue.add(pending)
+ *              pending_list.remove(pending)
+ *              continue
+ *
+ *      l = free_lsid
+ *      if l is Nul:
+ *              l = idle_lsid
+ *              if l is Nul:
+ *                      break
+ *      if is_mapped(l):
+ *              s = session[l]
+ *              unmap(s)
+ *              notify_list.add(s)
+ *
+ *      map(pending)
+ *      jobqueue.add(pending)
+ *      pending_list.remove(pending)
+ * unlock_sched
+ *
+ * for s in notify_list:
+ *      session_notify(s)
+ *      notify_list.remove(s)
+ */
+static void sched_task(struct work_struct *ws)
+{
+	struct mvx_sched *sched = container_of(ws, struct mvx_sched, sched_task);
+	struct mvx_sched_session *pending;
+	struct mvx_sched_session *unmapped;
+	struct mvx_sched_session *tmp;
+	struct mvx_session *m_session;
+	LIST_HEAD(notify_list);
+	int ret;
+
+	mvx_pm_runtime_get_sync(sched->dev);
+	ret = mutex_lock_interruptible(&sched->mutex);
+	if (ret != 0) {
+		mvx_pm_runtime_put_sync(sched->dev);
+		return;
+	}
+
+	/*
+	 * Try to map sessions from pending queue while possible.
+	 */
+	list_for_each_entry_safe(pending, tmp, &sched->pending, pending) {
+		struct mvx_lsid *lsid;
+
+		/*
+		 * This session is already mapped to LSID.
+		 * Just make sure it is scheduled.
+		 */
+		if (pending->lsid != NULL) {
+			m_session = mvx_if_session_to_session(pending->isession);
+			if (sched->is_suspend == true && m_session != NULL && m_session->switched_in == false) {
+				continue;
+			}
+
+			ret = mvx_lsid_jobqueue_add(pending->lsid, pending->isession->ncores);
+			if (ret != 0) {
+				MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+					      "Cannot add pending session to job queue. csession=%p, mvx_session=%p",
+					      pending, mvx_if_session_to_session(pending->isession));
+				continue;
+			}
+
+			pending->in_pending = false;
+			list_del(&pending->pending);
+			continue;
+		} else if (sched->is_suspend == true) {
+			continue;
+		}
+
+		/* Find LSID to be used for the pending session. */
+		lsid = find_free_lsid(sched);
+		if (lsid == NULL)
+			lsid = find_idle_lsid(sched);
+
+		if (lsid == NULL)
+			break;
+
+		/*
+		 * This LSID was mapped to some session. We have to notify
+		 * the session about an irq in case there are messages in
+		 * a message queue.
+		 *
+		 * Notifications are done after pending list is processed.
+		 */
+		if (lsid->session != NULL) {
+			struct mvx_sched_session *unmapped = lsid->session;
+
+			unmap_session(sched, unmapped);
+
+			/*
+			 * If the reference count is 0, then the session is
+			 * about to be removed and should be ignored.
+			 */
+			ret = kref_get_unless_zero(&unmapped->isession->kref);
+			if (ret != 0) {
+				if (list_find_node(&notify_list, &unmapped->notify))
+					/*
+					 * Consider a situation when a session
+					 * that was unmapped from LSID and added
+					 * notify_list was also present in the
+					 * pending_list. It is possible that
+					 * such a session will be mapped to the
+					 * new LSID, executed by the hardware
+					 * and switched to idle state while
+					 * this function is still looping
+					 * through pending list.
+					 *
+					 * If it happens, then this session
+					 * might be unmapped again in order to
+					 * make a room for another pending
+					 * session. As a result we will try to
+					 * add this session to notify_list
+					 * again. This will break notify list
+					 * and could lead to crashes or hangs.
+					 *
+					 * However, it is safe just to skip
+					 * adding the session to notify_list if
+					 * it is already there, because it will
+					 * be processed anyway.
+					 */
+					kref_put(&unmapped->isession->kref, unmapped->isession->release);
+				else
+					list_add_tail(&unmapped->notify, &notify_list);
+			} else {
+				MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "Ref is zero. csession=%p", unmapped);
+			}
+		}
+
+		ret = map_session(sched, pending, lsid);
+		if (ret != 0) {
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+				      "Cannot map pending session. csession=%p, mvx_session=%p", pending,
+				      mvx_if_session_to_session(pending->isession));
+			break;
+		}
+
+		ret = mvx_lsid_jobqueue_add(lsid, pending->isession->ncores);
+		if (ret != 0) {
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+				      "Cannot add pending session to job queue. csession=%p, mvx_session=%p", pending,
+				      mvx_if_session_to_session(pending->isession));
+			continue;
+		}
+
+		pending->in_pending = false;
+		list_del(&pending->pending);
+	}
+
+	/*
+	 * It is important that the scheduler mutex is released before the
+	 * callbacks to the if-module are invoked. The if-module may issue
+	 * requests to the dev-module (for example switch_in()) that would
+	 * otherwise deadlock.
+	 */
+	mutex_unlock(&sched->mutex);
+
+	list_for_each_entry_safe(unmapped, tmp, &notify_list, notify) {
+		struct mvx_if_session *iunmapped = unmapped->isession;
+
+		list_del(&unmapped->notify);
+
+		mutex_lock(iunmapped->mutex);
+		sched->if_ops->irq(iunmapped);
+		ret = kref_put(&iunmapped->kref, iunmapped->release);
+		if (ret == 0)
+			mutex_unlock(iunmapped->mutex);
+	}
+
+	mvx_pm_runtime_put_sync(sched->dev);
+}
+
+static void sched_session_print(struct seq_file *s, struct mvx_sched_session *session, struct mvx_hwreg *hwreg, int ind)
+{
+	struct mvx_lsid *lsid;
+
+	if (session == NULL)
+		return;
+
+	mvx_seq_printf(s, "Client session", ind, "%p\n", session->isession);
+	mvx_seq_printf(s, "Dev session", ind, "%p\n", session);
+	mvx_seq_printf(s, "MVX session", ind, "%p\n", mvx_if_session_to_session(session->isession));
+
+	lsid = session->lsid;
+	if (lsid == NULL)
+		return;
+
+	mvx_seq_printf(s, "IRQ host", ind, "%d\n", mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_IRQHOST));
+	mvx_seq_printf(s, "IRQ MVE", ind, "%d\n", mvx_hwreg_read_lsid(hwreg, lsid->lsid, MVX_HWREG_LIRQVE));
+}
+
+static int sched_show(struct seq_file *s, void *v)
+{
+	struct mvx_sched *sched = (struct mvx_sched *)s->private;
+	struct mvx_hwreg *hwreg = sched->hwreg;
+	struct mvx_sched_session *session;
+	int i;
+	int ret;
+
+	ret = mvx_pm_runtime_get_sync(hwreg->dev);
+	if (ret < 0)
+		return 0;
+
+	ret = mutex_lock_interruptible(&sched->mutex);
+	if (ret != 0) {
+		mvx_pm_runtime_put_sync(hwreg->dev);
+		return ret;
+	}
+
+	mvx_seq_printf(s, "Core LSID", 0, "%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_CORELSID));
+	mvx_seq_printf(s, "Job queue", 0, "%08x\n", mvx_hwreg_read(hwreg, MVX_HWREG_JOBQUEUE));
+	seq_puts(s, "\n");
+
+	seq_puts(s, "scheduled:\n");
+	for (i = 0; i < sched->nlsid; ++i) {
+		mvx_seq_printf(s, "LSID", 1, "%d\n", i);
+		session = sched->lsid[i].session;
+		sched_session_print(s, session, hwreg, 2);
+	}
+
+	seq_puts(s, "pending:\n");
+	i = 0;
+	list_for_each_entry(session, &sched->pending, pending) {
+		char tmp[10];
+
+		scnprintf(tmp, sizeof(tmp), "%d", i++);
+		mvx_seq_printf(s, tmp, 1, "\n");
+		sched_session_print(s, session, hwreg, 2);
+	}
+
+	mutex_unlock(&sched->mutex);
+	mvx_pm_runtime_put_sync(hwreg->dev);
+
+	return 0;
+}
+
+static int sched_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, sched_show, inode->i_private);
+}
+
+static const struct file_operations sched_fops = {
+	.open    = sched_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = single_release
+};
+
+int sched_debugfs_init(struct mvx_sched *sched, struct dentry *parent)
+{
+	struct dentry *dentry;
+
+	dentry = debugfs_create_file("sched", 0400, parent, sched, &sched_fops);
+	if (IS_ERR_OR_NULL(dentry))
+		return -ENOMEM;
+
+	return 0;
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_sched_construct(struct mvx_sched *sched, struct device *dev, struct mvx_if_ops *if_ops, struct mvx_hwreg *hwreg,
+			struct dentry *parent)
+{
+	unsigned int lsid;
+	int ret;
+
+	sched->dev = dev;
+	sched->hwreg = hwreg;
+	sched->if_ops = if_ops;
+	sched->is_suspend = false;
+	mutex_init(&sched->mutex);
+	INIT_LIST_HEAD(&sched->pending);
+	INIT_WORK(&sched->sched_task, sched_task);
+	sched->sched_queue = create_singlethread_workqueue("mvx_sched");
+	if (!sched->sched_queue) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "Cannot create work queue");
+		return -EINVAL;
+	}
+
+	sched->nlsid = mvx_hwreg_read(hwreg, MVX_HWREG_NLSID);
+
+	for (lsid = 0; lsid < sched->nlsid; lsid++) {
+		ret = mvx_lsid_construct(&sched->lsid[lsid], dev, hwreg, lsid);
+		if (ret != 0)
+			goto destruct_lsid;
+	}
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ret = sched_debugfs_init(sched, parent);
+		if (ret != 0)
+			goto destruct_lsid;
+	}
+
+	mvx_hwreg_write(hwreg, MVX_HWREG_RESET, 1);
+	mvx_hwreg_write(hwreg, MVX_HWREG_CLKFORCE, 0);
+
+	return 0;
+
+destruct_lsid:
+	while (lsid-- > 0)
+		mvx_lsid_destruct(&sched->lsid[lsid]);
+
+	return ret;
+}
+
+void mvx_sched_destruct(struct mvx_sched *sched)
+{
+	destroy_workqueue(sched->sched_queue);
+
+	while (sched->nlsid-- > 0)
+		mvx_lsid_destruct(&sched->lsid[sched->nlsid]);
+}
+
+int mvx_sched_session_construct(struct mvx_sched_session *session, struct mvx_if_session *isession)
+{
+	uint32_t disallow;
+	uint32_t maxcores;
+
+	session->isession = isession;
+	INIT_LIST_HEAD(&session->pending);
+	INIT_LIST_HEAD(&session->notify);
+	session->lsid = NULL;
+	session->in_pending = false;
+
+	memset(&session->pcb, 0, sizeof(session->pcb));
+
+	disallow = (0xffffffff << isession->ncores) & MVE_CTRL_DISALLOW_MASK;
+	maxcores = isession->ncores & MVE_CTRL_MAXCORES_MASK;
+	session->pcb.ctrl = (disallow << MVE_CTRL_DISALLOW_SHIFT) | (maxcores << MVE_CTRL_MAXCORES_SHIFT);
+
+	session->pcb.mmu_ctrl = isession->l0_pte;
+	session->pcb.nprot = isession->securevideo == false;
+
+	return 0;
+}
+
+void mvx_sched_session_destruct(struct mvx_sched_session *session) {}
+
+int mvx_sched_switch_in(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	int ret;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "%p Switch in session. jobqueue=%08x, coreslid=%08x.",
+		      mvx_if_session_to_session(session->isession), mvx_hwreg_read(sched->hwreg, MVX_HWREG_JOBQUEUE),
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_CORELSID));
+
+	ret = mutex_lock_interruptible(&sched->mutex);
+	if (ret != 0)
+		return ret;
+
+	if (session->in_pending) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_VERBOSE, "Session is already in pending state.");
+		goto unlock_mutex;
+	}
+
+	session->in_pending = true;
+	list_add_tail(&session->pending, &sched->pending);
+	queue_work(sched->sched_queue, &sched->sched_task);
+
+unlock_mutex:
+	mutex_unlock(&sched->mutex);
+	return 0;
+}
+
+int mvx_sched_send_irq(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	mutex_lock(&sched->mutex);
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_DEBUG, "%p Send irq. lsid=%d, jobqueue=%08x, corelsid=%08x.",
+		      mvx_if_session_to_session(session->isession), session->lsid == NULL ? -1 : session->lsid->lsid,
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_JOBQUEUE),
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_CORELSID));
+
+	if (session->lsid == NULL)
+		session->pcb.irqhost = 1;
+	else
+		mvx_lsid_send_irq(session->lsid);
+
+	mutex_unlock(&sched->mutex);
+
+	return 0;
+}
+
+int mvx_sched_flush_mmu(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	mutex_lock(&sched->mutex);
+
+	if (session->lsid != NULL)
+		mvx_lsid_flush_mmu(session->lsid);
+
+	mutex_unlock(&sched->mutex);
+
+	return 0;
+}
+
+static void print_session(struct mvx_sched *sched, struct mvx_sched_session *session, struct mvx_session *s)
+{
+	int lsid = -1;
+	uint32_t irqve = 0;
+	uint32_t irqhost = 0;
+
+	if (session != NULL && session->lsid != NULL) {
+		struct mvx_hwreg *hwreg = sched->hwreg;
+
+		lsid = session->lsid->lsid;
+		irqve = mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_LIRQVE);
+		irqhost = mvx_hwreg_read_lsid(hwreg, lsid, MVX_HWREG_IRQHOST);
+	}
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "%p    session=%p, lsid=%d, irqve=%08x, irqhost=%08x", s,
+		      mvx_if_session_to_session(session->isession), lsid, irqve, irqhost);
+}
+
+void mvx_sched_wait_session_idle(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	bool is_idle = 0;
+	int wait_count = 20;
+
+	mutex_lock(&sched->mutex);
+
+	if (session->lsid != NULL) {
+		is_idle = mvx_lsid_idle(session->lsid);
+		if (is_idle == 0) {
+			do {
+				mutex_unlock(&sched->mutex);
+				msleep(50);
+				mutex_lock(&sched->mutex);
+				if (session->lsid != NULL) {
+					is_idle = mvx_lsid_idle(session->lsid);
+				} else {
+					is_idle = 1;
+				}
+			} while (wait_count-- && is_idle != 1);
+		}
+
+		if (is_idle == 0) {
+			mvx_lsid_terminate(session->lsid);
+		}
+	}
+	mutex_unlock(&sched->mutex);
+}
+
+void mvx_sched_print_debug(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	struct mvx_hwreg *hwreg = sched->hwreg;
+	struct mvx_sched_session *pending;
+	struct mvx_sched_session *tmp;
+	struct mvx_session *s = mvx_if_session_to_session(session->isession);
+	unsigned int i;
+	int ret;
+
+	mvx_pm_runtime_get_sync(sched->dev);
+
+	ret = mutex_lock_interruptible(&sched->mutex);
+	if (ret != 0) {
+		mvx_pm_runtime_put_sync(sched->dev);
+		return;
+	}
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "%p Current session:", s);
+	print_session(sched, session, s);
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "%p Pending queue:", s);
+	list_for_each_entry_safe(pending, tmp, &sched->pending, pending) { print_session(sched, pending, s); }
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "%p Print register:", s);
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+		      "%p     jobqueue=%08x, corelsid=%08x, "
+		      "irqve=%08x,HARDWARE_ID=0x%08x,ENABLE=0x%08x,NCORES=0x%08x,NLSID=0x%08x,CLKFORCE=0x%08x,PROTCTRL="
+		      "0x%08x,RESET=0x%08x,FUSE=0x%08x",
+		      s, mvx_hwreg_read(hwreg, MVX_HWREG_JOBQUEUE), mvx_hwreg_read(hwreg, MVX_HWREG_CORELSID),
+		      mvx_hwreg_read(hwreg, MVX_HWREG_IRQVE), mvx_hwreg_read(hwreg, MVX_HWREG_HARDWARE_ID),
+		      mvx_hwreg_read(hwreg, MVX_HWREG_ENABLE), mvx_hwreg_read(hwreg, MVX_HWREG_NCORES),
+		      mvx_hwreg_read(hwreg, MVX_HWREG_NLSID), mvx_hwreg_read(hwreg, MVX_HWREG_CLKFORCE),
+		      mvx_hwreg_read(hwreg, MVX_HWREG_PROTCTRL), mvx_hwreg_read(hwreg, MVX_HWREG_RESET),
+		      mvx_hwreg_read(hwreg, MVX_HWREG_FUSE));
+
+	for (i = 0; i < sched->nlsid; i++) {
+		struct mvx_sched_session *ss = sched->lsid[i].session;
+		struct mvx_session *ls = NULL;
+
+		if (ss != NULL)
+			ls = mvx_if_session_to_session(ss->isession);
+
+		MVX_LOG_PRINT(
+			&mvx_log_dev, MVX_LOG_WARNING,
+			"%p     lsid=%u, session=%p, irqve=%08x, "
+			"irqhost=%08x,CTRL=0x%08x,MMU_CTRL=0x%08x,NPROT=0x%08x,ALLOC=0x%08x,FLUSH_ALL=0x%08x,SCHED=0x%"
+			"08x,TERMINATE=0x%08x,INTSIG=0x%08x,STREAMID=0x%08x",
+			s, i, ls, mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_LIRQVE),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_IRQHOST), mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_CTRL),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_MMU_CTRL),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_NPROT), mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_ALLOC),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_FLUSH_ALL),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_SCHED),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_TERMINATE),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_INTSIG),
+			mvx_hwreg_read_lsid(hwreg, i, MVX_HWREG_STREAMID));
+	}
+
+	mutex_unlock(&sched->mutex);
+
+	mvx_pm_runtime_put_sync(sched->dev);
+}
+
+void mvx_sched_handle_irq(struct mvx_sched *sched, unsigned int lsid)
+{
+	struct mvx_sched_session *session;
+	struct mvx_if_session *isession = NULL;
+	int ret;
+
+	ret = mutex_lock_interruptible(&sched->mutex);
+	if (ret != 0)
+		return;
+
+	/*
+	 * If a session has been terminated/unmapped just before the IRQ bottom
+	 * handler has been executed, then the session pointer will be NULL or
+	 * may even point at a different session. This is an unharmful
+	 * situation.
+	 *
+	 * If the reference count is 0, then the session is about to be removed
+	 * and should be ignored.
+	 */
+	session = sched->lsid[lsid].session;
+	if (session != NULL) {
+		ret = kref_get_unless_zero(&session->isession->kref);
+		if (ret != 0)
+			isession = session->isession;
+	}
+
+	/*
+	 * It is important that the scheduler mutex is released before the
+	 * callbacks to the if-module are invoked. The if-module may issue
+	 * requests to the dev-module (for example switch_in()) that would
+	 * otherwise deadlock.
+	 */
+	mutex_unlock(&sched->mutex);
+
+	/* Inform if-session that an IRQ was received. */
+	if (isession != NULL) {
+		mutex_lock(isession->mutex);
+		sched->if_ops->irq(isession);
+		ret = kref_put(&isession->kref, isession->release);
+
+		if (ret == 0)
+			mutex_unlock(isession->mutex);
+	}
+
+	ret = mutex_lock_interruptible(&sched->mutex);
+	if (ret != 0)
+		return;
+
+	if (!list_empty(&sched->pending)) {
+		queue_work(sched->sched_queue, &sched->sched_task);
+	}
+	mutex_unlock(&sched->mutex);
+}
+
+void mvx_sched_terminate(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	struct list_head *head;
+	struct list_head *tmp;
+
+	mutex_lock(&sched->mutex);
+
+	if (session->lsid != NULL) {
+		mvx_lsid_jobqueue_remove(session->lsid);
+		mvx_lsid_terminate(session->lsid);
+		unmap_session(sched, session);
+	}
+
+	list_for_each_safe(head, tmp, &sched->pending) {
+		if (head == &session->pending) {
+			list_del(head);
+			break;
+		}
+	}
+
+	mutex_unlock(&sched->mutex);
+}
+
+static void suspend_session(struct mvx_sched *sched, struct mvx_sched_session *session)
+{
+	struct mvx_lsid *lsid = session->lsid;
+
+	if (lsid == NULL)
+		return;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "%p suspend_session LSID. lsid=%u, jobqueue=%08x, corelsid=%08x.",
+		      mvx_if_session_to_session(session->isession), lsid->lsid,
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_JOBQUEUE),
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_CORELSID));
+
+	mvx_lsid_unmap(lsid, &session->pcb);
+}
+
+static int resume_session(struct mvx_sched *sched, struct mvx_sched_session *session, struct mvx_lsid *lsid)
+{
+	int ret;
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "%p resume_session LSID. lsid=%u, jobqueue=%08x, corelsid=%08x.",
+		      mvx_if_session_to_session(session->isession), lsid->lsid,
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_JOBQUEUE),
+		      mvx_hwreg_read(sched->hwreg, MVX_HWREG_CORELSID));
+
+	ret = mvx_lsid_map(lsid, &session->pcb);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+static void switch_out_session(struct mvx_sched *sched, unsigned int lsid_id)
+{
+	int i;
+	int ret;
+	struct mvx_session *session = NULL;
+	struct mvx_if_session *isession = NULL;
+	struct mvx_sched_session *sched_session = sched->lsid[lsid_id].session;
+	int wait_count = 20;
+
+	if (sched_session != NULL) {
+		ret = kref_get_unless_zero(&sched_session->isession->kref);
+		if (ret != 0)
+			isession = sched_session->isession;
+	}
+
+	mutex_unlock(&sched->mutex);
+
+	if (isession != NULL) {
+		session = mvx_if_session_to_session(sched_session->isession);
+		mutex_lock(isession->mutex);
+		if (session != NULL) {
+			session->is_suspend = true;
+		}
+		ret = kref_put(&isession->kref, isession->release);
+
+		if (ret == 0)
+			mutex_unlock(isession->mutex);
+	}
+
+	mutex_lock(&sched->mutex);
+	for (i = 0; i < wait_count; i++) {
+		if (sched->lsid[lsid_id].session != NULL && session != NULL && session->switched_in == false) {
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "%p finish switch_out session LSID. lsid=%u.",
+				      session, sched_session->lsid->lsid);
+			break;
+		}
+		mutex_unlock(&sched->mutex);
+		msleep(10);
+
+		mutex_lock(&sched->mutex);
+
+		if (sched->lsid[lsid_id].session != NULL && session != NULL) {
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+				      "%p wait switch_out session LSID. lsid=%u. loop=%d, switch_in=%d", session,
+				      sched_session->lsid->lsid, i, session->switched_in);
+		}
+	}
+}
+
+static int switch_in_session(struct mvx_sched_session *sched_session)
+{
+	int ret;
+	struct mvx_session *session;
+	session = mvx_if_session_to_session(sched_session->isession);
+	if (session == NULL) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "session is null when switch in.");
+		return -EINVAL;
+	}
+
+	session->is_suspend = false;
+	if (session->fw.msg_pending > 0) {
+		session->idle_count = 0;
+
+		if (session->switched_in != false)
+			return 0;
+
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "switch_in_session.");
+
+		ret = session->client_ops->switch_in(session->csession);
+		if (ret != 0) {
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "Failed to switch in session.");
+			session->error = ret;
+			wake_up(&session->waitq);
+			session->event(session, MVX_SESSION_EVENT_ERROR, (void *)session->error);
+			return ret;
+		}
+
+		session->switched_in = true;
+	}
+	return 0;
+}
+
+void mvx_sched_suspend(struct mvx_sched *sched)
+{
+	unsigned int i;
+
+	mutex_lock(&sched->mutex);
+	sched->is_suspend = true;
+	mvx_dvfs_suspend_session();
+
+	for (i = 0; i < sched->nlsid; i++) {
+		if (sched->lsid[i].session != NULL) {
+			switch_out_session(sched, i);
+			if (sched->lsid[i].session != NULL) {
+				mvx_lsid_terminate(sched->lsid[i].session->lsid);
+				suspend_session(sched, sched->lsid[i].session);
+			}
+		}
+	}
+
+	mutex_unlock(&sched->mutex);
+}
+
+void mvx_sched_resume(struct mvx_sched *sched)
+{
+	unsigned int i;
+
+	mutex_lock(&sched->mutex);
+
+	for (i = 0; i < sched->nlsid; i++) {
+		if (sched->lsid[i].session != NULL) {
+			resume_session(sched, sched->lsid[i].session, &(sched->lsid[i]));
+			switch_in_session(sched->lsid[i].session);
+		}
+	}
+	sched->is_suspend = false;
+	mvx_dvfs_resume_session();
+	mutex_unlock(&sched->mutex);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/dev/mvx_scheduler.h b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_scheduler.h
new file mode 100644
index 000000000000..6cf8e8c6e9a5
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/dev/mvx_scheduler.h
@@ -0,0 +1,193 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_SCHEDULER_H_
+#define _MVX_SCHEDULER_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include "mvx_lsid.h"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct mvx_if_ops;
+struct mvx_hwreg;
+
+/**
+ * struct mvx_sched - Scheduler class.
+ * @dev:	Pointer to device.
+ * @if_ops:	Pointer to if module operations.
+ * @hwreg:	Pointer to hwreg.
+ * @mutex:	Mutex protecting the scheduler.
+ * @pending:	List if sessions pending scheduling.
+ * @nlsid:	Number of LSID.
+ * @lsid:	Array of LSID instances.
+ */
+struct mvx_sched {
+	struct device *dev;
+	struct mvx_if_ops *if_ops;
+	struct mvx_hwreg *hwreg;
+	struct mutex mutex;
+	struct list_head pending;
+	unsigned int nlsid;
+	struct mvx_lsid lsid[MVX_LSID_MAX];
+	struct work_struct sched_task;
+	struct workqueue_struct *sched_queue;
+	bool is_suspend;
+};
+
+/**
+ * struct mvx_sched_session - Client session class.
+ * @isession:	Pointer to if session.
+ * @head:	List head used to insert session into scheduler pending list.
+ * @lsid:	Pointer to LSID the session is mapped to.
+ * @pcb:	LSID pcb.
+ *
+ * This struct is used to keep track of sessions specific information.
+ */
+struct mvx_sched_session {
+	struct mvx_if_session *isession;
+	struct list_head pending;
+	struct list_head notify;
+	struct mvx_lsid *lsid;
+	struct mvx_lsid_pcb pcb;
+	bool in_pending;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_sched_construct() - Construct the scheduler object.
+ * @sched:	Pointer to scheduler object.
+ * @dev:	Pointer to device.
+ * @if_ops:	Pointer to if ops.
+ * @hwreg:	Pointer to hwreg.
+ * @parent:	Pointer to parent debugfs directory entry.
+ *
+ * Return: 0 on success, else errorr code.
+ */
+int mvx_sched_construct(struct mvx_sched *sched, struct device *dev, struct mvx_if_ops *if_ops, struct mvx_hwreg *hwreg,
+			struct dentry *parent);
+
+/**
+ * mvx_sched_destruct() - Destruct the scheduler object.
+ * @sched:	Pointer to scheduler object.
+ */
+void mvx_sched_destruct(struct mvx_sched *sched);
+
+/**
+ * mvx_sched_session_construct() - Construct the scheduler session object.
+ * @if_ops:	If module operations.
+ * @session:	Pointer to session object.
+ * @isession:	Pointer to if session.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_sched_session_construct(struct mvx_sched_session *session, struct mvx_if_session *isession);
+
+/**
+ * mvx_sched_session_destruct() - Destruct the scheduler session object.
+ * @session:	Pointer to session object.
+ *
+ * The client must make sure the session is terminated before the destructor
+ * is called.
+ */
+void mvx_sched_session_destruct(struct mvx_sched_session *session);
+
+/**
+ * mvx_sched_switch_in() - Switch in a session.
+ * @sched:	Pointer to scheduler object.
+ * @session:	Pointer to session object.
+ *
+ * Map a session to a LSID and schedule session for execution. If no LSID
+ * is available the session is placed in the pending queue.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_sched_switch_in(struct mvx_sched *sched, struct mvx_sched_session *session);
+
+/**
+ * mvx_sched_send_irq() - Send IRQ to session.
+ * @sched:	Pointer to scheduler object.
+ * @session:	Pointer to session object.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_sched_send_irq(struct mvx_sched *sched, struct mvx_sched_session *session);
+
+/**
+ * mvx_sched_flush_mmu() - Flush MMU tables.
+ * @sched:	Pointer to scheduler object.
+ * @session:	Pointer to session object.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_sched_flush_mmu(struct mvx_sched *sched, struct mvx_sched_session *session);
+
+/**
+ * mvx_sched_handle_irq() - Handle interrupt for a LSID.
+ * @sched:	Pointer to scheduler object.
+ * @lsid:	LSID number.
+ */
+void mvx_sched_handle_irq(struct mvx_sched *sched, unsigned int lsid);
+
+/**
+ * mvx_sched_terminate() - Terminate a session.
+ * @sched:	Pointer to scheduler object.
+ * @session:	Pointer to session object.
+ */
+void mvx_sched_terminate(struct mvx_sched *sched, struct mvx_sched_session *session);
+
+/**
+ * mvx_sched_print_debug() - Print debug information.
+ * @sched:	Pointer to scheduler object.
+ * @session:	Pointer to session object.
+ */
+void mvx_sched_print_debug(struct mvx_sched *sched, struct mvx_sched_session *session);
+
+void mvx_sched_suspend(struct mvx_sched *sched);
+
+void mvx_sched_resume(struct mvx_sched *sched);
+
+void mvx_sched_wait_session_idle(struct mvx_sched *sched, struct mvx_sched_session *session);
+
+int sched_debugfs_init(struct mvx_sched *sched, struct dentry *parent);
+
+#endif /* _MVX_SCHEDULER_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/external/fw_v2/mve_protocol_def.h b/drivers/media/platform/spacemit/vpu_k3/external/fw_v2/mve_protocol_def.h
new file mode 100644
index 000000000000..770aa99a959d
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/external/fw_v2/mve_protocol_def.h
@@ -0,0 +1,1621 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/*
+ * Copyright:
+ * ----------------------------------------------------------------------------
+ * This confidential and proprietary software may be used only as authorized
+ * by a licensing agreement from Arm Technology (China) Co., Ltd.
+ *      (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ * The entire notice above must be reproduced on all authorized copies and
+ * copies may only be made to the extent permitted by a licensing agreement
+ * from Arm Technology (China) Co., Ltd.
+ * ----------------------------------------------------------------------------
+ */
+#ifndef __FW_INCLUDE__MVE_PROTOCOL_DEF_H__
+#define __FW_INCLUDE__MVE_PROTOCOL_DEF_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+#endif
+
+/*****************************************************************************
+ *
+ * Communication protocol between the host/driver and the MVE firmware,
+ * the 'host interface'.
+ *
+ * MVE == LINLON Video Engine
+ *
+ * Protocol version 2.5
+ *
+ * Note: Message structs may be expanded in the future; the host should
+ * use the 'size' of the message to determine how many bytes to
+ * read from the message queue, rather than a sizeof(struct).
+ *
+ ****************************************************************************/
+
+/*****************************************************************************
+ *
+ * Virtual memory regions
+ *
+ * ..._ADDR_BEGIN gives the starting virtual address of the region,
+ * and ..._ADDR_END the (non-inclusive) ending address, such that
+ * the size of the region is obtained with the subtraction
+ * (..._ADDR_END - ..._ADDR_BEGIN).
+ *
+ ****************************************************************************/
+
+/* Memory region for first firmware instance */
+#define MVE_MEM_REGION_FW_INSTANCE0_ADDR_BEGIN (0x00000000u)
+#define MVE_MEM_REGION_FW_INSTANCE0_ADDR_END (0x000FFFFFu + 1)
+
+/*
+ * Areas for communication between host and MVE are placed in the interval
+ * 0x10079000 - 0x1007FFFF, see special defines further down.
+ */
+
+/* PROTECTED virtual memory region */
+#define MVE_MEM_REGION_PROTECTED_ADDR_BEGIN (0x20000000u)
+#define MVE_MEM_REGION_PROTECTED_ADDR_END (0x4FFFFFFFu + 1)
+
+/* FRAMEBUF virtual memory region */
+#define MVE_MEM_REGION_FRAMEBUF_ADDR_BEGIN (0x50000000u)
+#define MVE_MEM_REGION_FRAMEBUF_ADDR_END (0x7FFFFFFFu + 1)
+
+/* Memory regions for other firmware instances */
+#define MVE_MEM_REGION_FW_INSTANCE1_ADDR_BEGIN (0x80000000u)
+#define MVE_MEM_REGION_FW_INSTANCE1_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE1_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE2_ADDR_BEGIN (0x90000000u)
+#define MVE_MEM_REGION_FW_INSTANCE2_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE2_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE3_ADDR_BEGIN (0xA0000000u)
+#define MVE_MEM_REGION_FW_INSTANCE3_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE3_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE4_ADDR_BEGIN (0xB0000000u)
+#define MVE_MEM_REGION_FW_INSTANCE4_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE4_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE5_ADDR_BEGIN (0xC0000000u)
+#define MVE_MEM_REGION_FW_INSTANCE5_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE5_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE6_ADDR_BEGIN (0xD0000000u)
+#define MVE_MEM_REGION_FW_INSTANCE6_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE6_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE7_ADDR_BEGIN (0xE0000000u)
+#define MVE_MEM_REGION_FW_INSTANCE7_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE7_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+/* 0xF0000000 - 0xFFFFFFFF is used internally in MVE */
+
+/*****************************************************************************
+ *
+ * Communication queues between HOST/DRIVER and MVE
+ *
+ * Address for queue for messages in to MVE,
+ * one struct mve_comm_area_host located here
+ *
+ ****************************************************************************/
+
+#define MVE_COMM_MSG_INQ_ADDR (0x10079000u)
+
+/* Address for queue for messages out from MVE,
+ * one struct mve_comm_area_mve located here
+ */
+#define MVE_COMM_MSG_OUTQ_ADDR (0x1007A000u)
+
+/* Address for queue for input buffers in to MVE,
+ * one struct mve_comm_area_host located here
+ */
+#define MVE_COMM_BUF_INQ_ADDR (0x1007B000u)
+
+/* Address for queue for input buffers returned from MVE,
+ * one struct mve_comm_area_mve located here
+ */
+#define MVE_COMM_BUF_INRQ_ADDR (0x1007C000u)
+
+/* Address for queue for output buffers in to MVE,
+ * one struct mve_comm_area_host located here
+ */
+#define MVE_COMM_BUF_OUTQ_ADDR (0x1007D000u)
+
+/* Address for queue for output buffers returned from MVE,
+ * one struct mve_comm_area_mve located here
+ */
+#define MVE_COMM_BUF_OUTRQ_ADDR (0x1007E000u)
+
+/* One struct mve_rpc_communication_area located here */
+#define MVE_COMM_RPC_ADDR (0x1007F000u)
+
+/* Address for ram_print buffer in FW */
+#define MVE_FW_PRINT_RAM_ADDR (0x10100000u)
+#define MVE_FW_PRINT_RAM_SIZE (0x80000u)
+
+/* One page of memory (4 kB) is used for each queue,
+ * so maximum 1024 words, but need room for some counters as well,
+ * see structs mve_comm_area_mve and mve_comm_area_host below.
+ */
+#define MVE_COMM_QUEUE_SIZE_IN_WORDS 1020
+
+/* This is the part of the message area that is written by host. */
+struct mve_comm_area_host {
+	volatile uint16_t out_rpos;
+	volatile uint16_t in_wpos;
+	volatile uint32_t reserved[3];
+	/*
+	 * Queue of messages to MVE, each block of data prefixed with
+	 * a mve_msg_header
+	 */
+	volatile uint32_t in_data[MVE_COMM_QUEUE_SIZE_IN_WORDS];
+};
+
+/* This is the part of the message area that is written by MVE. */
+struct mve_comm_area_mve {
+	volatile uint16_t out_wpos;
+	volatile uint16_t in_rpos;
+	volatile uint32_t reserved[3];
+	/*
+	 * Queue of messages to host, each block of data prefixed with
+	 * a mve_msg_header
+	 */
+	volatile uint32_t out_data[MVE_COMM_QUEUE_SIZE_IN_WORDS];
+};
+
+#define MVE_RPC_AREA_SIZE_IN_WORDS 256
+#define MVE_RPC_DATA_SIZE_IN_WORDS (MVE_RPC_AREA_SIZE_IN_WORDS - 3)
+union mve_rpc_params {
+	volatile uint32_t data[MVE_RPC_DATA_SIZE_IN_WORDS];
+	struct {
+		char string[MVE_RPC_DATA_SIZE_IN_WORDS * 4];
+	} debug_print;
+	struct {
+		uint32_t size;
+		uint32_t max_size;
+		uint8_t region; /* Memory region selection */
+#define MVE_MEM_REGION_PROTECTED (0)
+#define MVE_MEM_REGION_OUTBUF (1)
+#define MVE_MEM_REGION_FRAMEBUF (MVE_MEM_REGION_OUTBUF)
+
+		/* The newly allocated memory must be placed
+		 * on (at least) a 2^(log2_alignment) boundary
+		 */
+		uint8_t log2_alignment;
+	} mem_alloc;
+	struct {
+		uint32_t ve_pointer;
+		uint32_t new_size;
+	} mem_resize;
+	struct {
+		uint32_t ve_pointer;
+	} mem_free;
+};
+
+struct mve_rpc_communication_area {
+	volatile uint32_t state;
+#define MVE_RPC_STATE_FREE (0)
+#define MVE_RPC_STATE_PARAM (1)
+#define MVE_RPC_STATE_RETURN (2)
+	volatile uint32_t call_id;
+#define MVE_RPC_FUNCTION_DEBUG_PRINTF (1)
+#define MVE_RPC_FUNCTION_MEM_ALLOC (2)
+#define MVE_RPC_FUNCTION_MEM_RESIZE (3)
+#define MVE_RPC_FUNCTION_MEM_FREE (4)
+	volatile uint32_t size;
+	union mve_rpc_params params;
+};
+
+struct mve_fw_ram_print_head_aera {
+	volatile uint32_t rd_cnt;
+	volatile uint32_t reserved0[15];
+
+	volatile uint32_t flag;
+	volatile uint32_t index;
+	volatile uint32_t wr_cnt;
+	volatile uint32_t reserved1[13];
+};
+
+/*********************************************************************
+ *
+ *   Message codes
+ *
+ *********************************************************************/
+
+/* Messages consist of one struct mve_msg_header, possibly followed
+ * by extra data.
+ */
+struct mve_msg_header {
+	uint16_t code;
+/* REQUESTs are messages from the
+ * host/driver to the firmware:               Code:     Extra data in message: */
+#define MVE_REQUEST_CODE_GO (1001)		  /* no extra data */
+#define MVE_REQUEST_CODE_STOP (1002)		  /* no extra data */
+#define MVE_REQUEST_CODE_INPUT_FLUSH (1003)	  /* no extra data */
+#define MVE_REQUEST_CODE_OUTPUT_FLUSH (1004)	  /* no extra data */
+#define MVE_REQUEST_CODE_SWITCH (1005)		  /* no extra data */
+#define MVE_REQUEST_CODE_PING (1006)		  /* no extra data */
+#define MVE_REQUEST_CODE_DUMP (1008)		  /* no extra data */
+#define MVE_REQUEST_CODE_JOB (1009)		  /* struct mve_request_job */
+#define MVE_REQUEST_CODE_SET_OPTION (1010)	  /* struct mve_request_set_option (variable size) */
+#define MVE_REQUEST_CODE_RELEASE_REF_FRAME (1011) /* struct mve_request_release_ref_frame */
+#define MVE_REQUEST_CODE_IDLE_ACK (1012)	  /* no extra data */
+#define MVE_REQUEST_CODE_DEBUG (1013)		  /* level: 0 for disable, refer to fw_log_level */
+/* RESPONSEs are messages from
+ * the firmware to the host: */
+#define MVE_RESPONSE_CODE_SWITCHED_IN (2001)	     /* struct mve_response_switched_in */
+#define MVE_RESPONSE_CODE_SWITCHED_OUT (2002)	     /* struct mve_response_switched_out */
+#define MVE_RESPONSE_CODE_SET_OPTION_CONFIRM (2003)  /* no extra data */
+#define MVE_RESPONSE_CODE_JOB_DEQUEUED (2004)	     /* struct mve_response_job_dequeued */
+#define MVE_RESPONSE_CODE_INPUT (2005)		     /* no extra data, but buffer placed in buffer queue */
+#define MVE_RESPONSE_CODE_OUTPUT (2006)		     /* no extra data, but buffer placed in buffer queue */
+#define MVE_RESPONSE_CODE_INPUT_FLUSHED (2007)	     /* no extra data */
+#define MVE_RESPONSE_CODE_OUTPUT_FLUSHED (2008)	     /* no extra data */
+#define MVE_RESPONSE_CODE_PONG (2009)		     /* no extra data */
+#define MVE_RESPONSE_CODE_ERROR (2010)		     /* struct mve_response_error */
+#define MVE_RESPONSE_CODE_STATE_CHANGE (2011)	     /* struct mve_response_state_change */
+#define MVE_RESPONSE_CODE_DUMP (2012)		     /* no extra data */
+#define MVE_RESPONSE_CODE_IDLE (2013)		     /* no extra data */
+#define MVE_RESPONSE_CODE_FRAME_ALLOC_PARAM (2014)   /* struct mve_response_frame_alloc_parameters */
+#define MVE_RESPONSE_CODE_SEQUENCE_PARAMETERS (2015) /* struct mve_response_sequence_parameters */
+#define MVE_RESPONSE_CODE_EVENT (2016)		     /* struct mve_response_event (variable size) */
+#define MVE_RESPONSE_CODE_SET_OPTION_FAIL (2017)     /* struct mve_response_set_option_failed */
+#define MVE_RESPONSE_CODE_REF_FRAME_UNUSED (2018)    /* struct mve_response_ref_frame_unused */
+#define MVE_RESPONSE_CODE_DEBUG (2019)		     /* no extra data */
+/* BUFFERs are sent from host to firmware,
+ * and then return at some time: */
+#define MVE_BUFFER_CODE_FRAME (3001)	 /* struct mve_buffer_frame */
+#define MVE_BUFFER_CODE_BITSTREAM (3002) /* struct mve_buffer_bitstream */
+#define MVE_BUFFER_CODE_PARAM (3003)	 /* struct mve_buffer_param */
+#define MVE_BUFFER_CODE_GENERAL (3004)	 /* struct mve_buffer_general */
+
+	uint16_t size; /* size in bytes of trailing data, 0 if none */
+};
+
+/*********************************************************************
+ *
+ *   REQUESTs are messages from the host to the firmware
+ *
+ *   Some of the MVE_REQUEST_CODE_ codes are followed by one of the
+ *   structs below.
+ *
+ *********************************************************************/
+
+struct mve_request_job {
+	uint16_t cores;	 /* >= 1, number of cores to use, must match request to HW scheduler */
+	uint16_t frames; /* number of frames to process, zero means infinite */
+	uint32_t flags;	 /* can be zero */
+#define MVE_JOB_FLAG_DISABLE_BNDMGR (0x01)
+};
+
+struct mve_request_set_option {
+	uint32_t index;
+#define MVE_SET_OPT_INDEX_NALU_FORMAT (1)	   /* see arg, MVE_OPT_NALU_FORMAT_ */
+#define MVE_SET_OPT_INDEX_STREAM_ESCAPING (2)	   /* arg=1 to enable (default), arg=0 to disable */
+#define MVE_SET_OPT_INDEX_PROFILE_LEVEL (3)	   /* data.profile_level */
+#define MVE_SET_OPT_INDEX_HOST_PROTOCOL_PRINTS (4) /* arg=1 to enable, arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_PROFILING (5)		   /* arg=1 to enable, arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_DISABLE_FEATURES (6)	   /* see arg, MVE_OPT_DISABLE_FEATURE_ */
+#define MVE_SET_OPT_INDEX_IGNORE_STREAM_HEADERS                                                                        \
+	(7) /* decode, arg=1 to enable,                                                                                \
+	     * arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_FRAME_REORDERING                                                                             \
+	(8) /* decode, arg=1 to enable (default),                                                                      \
+	     * arg=0 to disable */
+#define MVE_SET_OPT_INDEX_INTBUF_SIZE                                                                                  \
+	(9)					/* decode, arg = suggested limit of intermediate                       \
+						 * buffer allocation */
+#define MVE_SET_OPT_INDEX_ENC_P_FRAMES (16)	/* encode, arg = nPFrames */
+#define MVE_SET_OPT_INDEX_ENC_B_FRAMES (17)	/* encode, arg = number of B frames */
+#define MVE_SET_OPT_INDEX_GOP_TYPE (18)		/* encode, see arg */
+#define MVE_SET_OPT_INDEX_INTRA_MB_REFRESH (19) /* encode, arg */
+#define MVE_SET_OPT_INDEX_ENC_CONSTR_IPRED (20) /* encode, arg = 0 or 1 */
+#define MVE_SET_OPT_INDEX_ENC_ENTROPY_SYNC (21) /* encode, arg = 0 or 1 */
+#define MVE_SET_OPT_INDEX_ENC_TEMPORAL_MVP (22) /* encode, arg = 0 or 1 */
+#define MVE_SET_OPT_INDEX_TILES (23)		/* encode, data.tiles */
+#define MVE_SET_OPT_INDEX_ENC_MIN_LUMA_CB_SIZE                                                                         \
+	(24)					   /* HEVC encode, arg = 8 or 16,                                      \
+						    * for sizes 8x8 or 16x16 */
+#define MVE_SET_OPT_INDEX_ENC_MB_TYPE_ENABLE (25)  /* encode, see arg */
+#define MVE_SET_OPT_INDEX_ENC_MB_TYPE_DISABLE (26) /* encode, see arg */
+#define MVE_SET_OPT_INDEX_ENC_H264_CABAC (27)	   /* encode, arg = 0 or 1, enabled by default */
+#define MVE_SET_OPT_INDEX_ENC_SLICE_SPACING                                                                            \
+	(28)					   /* encode, arg = suggested number of                                \
+						    * CTUs/macroblocks in a slice */
+#define MVE_SET_OPT_INDEX_ENC_VP9_PROB_UPDATE (30) /* VP9 encode, see arg */
+#define MVE_SET_OPT_INDEX_RESYNC_INTERVAL                                                                              \
+	(31)					     /* JPEG encode, arg = nRestartInterval                            \
+						      * = nResynchMarkerSpacing */
+#define MVE_SET_OPT_INDEX_HUFFMAN_TABLE (32)	     /* JPEG encode, data.huffman_table */
+#define MVE_SET_OPT_INDEX_QUANT_TABLE (33)	     /* JPEG encode, data.quant_table */
+#define MVE_SET_OPT_INDEX_ENC_EXPOSE_REF_FRAMES (34) /* encode only, disabled by default */
+#define MVE_SET_OPT_INDEX_MBINFO_OUTPUT                                                                                \
+	(35)				       /* encode, arg=1 to enable,                                             \
+						* arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_MV_SEARCH_RANGE (36) /* encode, data,motion_vector_search_range */
+#define MVE_SET_OPT_INDEX_ENC_STREAM_BITDEPTH                                                                          \
+	(38) /* encode, data.bitdepth, to set other bitdepth                                                           \
+	      * of encoded stream than of input frames */
+#define MVE_SET_OPT_INDEX_ENC_STREAM_CHROMA_FORMAT                                                                     \
+	(39) /* encode, arg, to set other chroma format of                                                             \
+	      * encoded stream than of input frames */
+#define MVE_SET_OPT_INDEX_ENC_RGB_TO_YUV_MODE                                                                          \
+	(40) /* encode, arg, select which way RGB is converted                                                         \
+	      * to YUV before encoding */
+#define MVE_SET_OPT_INDEX_ENC_BANDWIDTH_LIMIT                                                                          \
+	(41)						/* encode, arg, the maxium bandwidth limit defined             \
+							 * by host */
+#define MVE_SET_OPT_INDEX_WATCHDOG_TIMEOUT (42)		/* arg=timeout, arg=0 to disable */
+#define MVE_SET_OPT_INDEX_ENC_CABAC_INIT_IDC (43)	/* encode, arg; 0,1,2 for H264; 0,1 for HEVC */
+#define MVE_SET_OPT_INDEX_ENC_ADPTIVE_QUANTISATION (44) /* encode (h264 and hevc) */
+#define MVE_SET_OPT_INDEX_QP_DELTA_I_P (45)
+#define MVE_SET_OPT_INDEX_QP_DELTA_I_B_REF (46)
+#define MVE_SET_OPT_INDEX_QP_DELTA_I_B_NONREF (47)
+#define MVE_SET_OPT_INDEX_CB_QP_OFFSET (48)
+#define MVE_SET_OPT_INDEX_CR_QP_OFFSET (49)
+#define MVE_SET_OPT_INDEX_LAMBDA_SCALE (50)	 /* encode, data.lambda_scale */
+#define MVE_SET_OPT_INDEX_ENC_MAX_NUM_CORES (51) /* maximum number of cores */
+#define MVE_SET_OPT_INDEX_ENC_EXTRA_REFS (52)	 /* configure number of extra ref buffers */
+#define MVE_SET_OPT_INDEX_QP_DELTA_RAW_I_P (53)
+#define MVE_SET_OPT_INDEX_QP_DELTA_RAW_I_B_REF (54)
+#define MVE_SET_OPT_INDEX_QP_DELTA_RAW_I_B_NONREF (55)
+#define MVE_SET_OPT_INDEX_ENC_FIXED_QP (56)
+/* ARBITRARY_DOWNSCALE */
+#define MVE_SET_OPT_INDEX_DEC_DOWNSCALE (57) /* decode, set downscaled width and height */
+#define MVE_SET_OPT_INDEX_FLUSHLESS_REFBANK                                                                            \
+	(58) /* configure AFBC ref bank for individual buffer                                                          \
+	      * allocation. Forced internally for flushless                                                            \
+	      * resolution change codecs */
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_LEFT (62)
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_RIGHT (63)
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_TOP (64)
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_BOTTOM (65)
+/* LONG_TERM_REFERENCE */
+#define MVE_SET_OPT_INDEX_ENC_LTR_MODE (66)
+#define MVE_SET_OPT_INDEX_ENC_LTR_PERIOD (67)
+#define MVE_SET_OPT_INDEX_DEC_DOWNSCALE_POS_MODE (69)
+
+	union {
+		uint32_t arg; /* Most options only need a uint32_t as argument */
+/* For option MVE_SET_OPT_INDEX_NALU_FORMAT, arg should
+ * be one of these: */
+#define MVE_OPT_NALU_FORMAT_START_CODES (1)
+#define MVE_OPT_NALU_FORMAT_ONE_NALU_PER_BUFFER (2)
+#define MVE_OPT_NALU_FORMAT_ONE_BYTE_LENGTH_FIELD (4)
+#define MVE_OPT_NALU_FORMAT_TWO_BYTE_LENGTH_FIELD (8)
+#define MVE_OPT_NALU_FORMAT_FOUR_BYTE_LENGTH_FIELD (16)
+/* For option MVE_SET_OPT_INDEX_GOP_TYPE, arg should
+ * be one of these: */
+#define MVE_OPT_GOP_TYPE_BIDIRECTIONAL (1)
+#define MVE_OPT_GOP_TYPE_LOW_DELAY (2)
+#define MVE_OPT_GOP_TYPE_PYRAMID (3)
+/* For option MVE_SET_OPT_INDEX_ENC_VP9_PROB_UPDATE,
+ * arg should be one of these: */
+#define MVE_OPT_VP9_PROB_UPDATE_DISABLED (0)
+#define MVE_OPT_VP9_PROB_UPDATE_IMPLICIT (1)
+#define MVE_OPT_VP9_PROB_UPDATE_EXPLICIT (2)
+/* For option MVE_SET_OPT_INDEX_DISABLE_FEATURES, arg
+ * should be a bitmask with features to disable: */
+#define MVE_OPT_DISABLE_FEATURE_AFBC_COMP (0x00000001)	/* VDMA AFBC Compression   */
+#define MVE_OPT_DISABLE_FEATURE_REF_CACHE (0x00000002)	/* REF caching             */
+#define MVE_OPT_DISABLE_FEATURE_DEBLOCK (0x00000004)	/* Deblocking              */
+#define MVE_OPT_DISABLE_FEATURE_SAO (0x00000008)	/* SAO                     */
+#define MVE_OPT_DISABLE_FEATURE_PIC_OUTPUT (0x00000020) /* Picture Output Removal  */
+#define MVE_OPT_DISABLE_FEATURE_PIPE (0x00000040)	/* Pipe (i.e. parser-only) */
+#define MVE_OPT_DISABLE_FEATURE_SLEEP                                                                                  \
+	(0x00000080) /* Clock gating                                                                                   \
+		      * (SOC_SYSCTRL.SLEEP bit) */
+#define MVE_OPT_DISABLE_FEATURE_AFBC_LEGACY_REF                                                                        \
+	(0x00000100) /* Enables tiled AFBC format in                                                                   \
+		      * reference buffers. Ignored                                                                     \
+		      * for decode AFBC output */
+#define MVE_OPT_DISABLE_FEATURE_REF_PICS                                                                               \
+	(0x00000400)					  /* Forces use of static 16x16                                \
+							   * reference pics */
+#define MVE_OPT_DISABLE_FEATURE_CHNG_RECT_WA (0x00000800) /* Disables workaround */
+#define MVE_OPT_DISABLE_FEATURE_REFSZ_LIMIT (0x00001000)  /* Disable REFSZ bw limit */
+/* For options MVE_SET_OPT_INDEX_ENC_MB_TYPE_ENABLE
+ * and MVE_SET_OPT_INDEX_ENC_MB_TYPE_DISABLE, arg
+ * should be a bitmask of MVE_MBTYPEs: */
+#define MVE_MBTYPE_4x4 (0x00000001)    /* 4x4 inter     */
+#define MVE_MBTYPE_4x8 (0x00000002)    /* 4x8 inter     */
+#define MVE_MBTYPE_8x4 (0x00000004)    /* 8x4 inter     */
+#define MVE_MBTYPE_8x8 (0x00000008)    /* 8x8 inter     */
+#define MVE_MBTYPE_8x16 (0x00000010)   /* 8x16 inter    */
+#define MVE_MBTYPE_16x8 (0x00000020)   /* 16x8 inter    */
+#define MVE_MBTYPE_16x16 (0x00000040)  /* 16x16 inter   */
+#define MVE_MBTYPE_PSKIP (0x00000080)  /* P Skip inter  */
+#define MVE_MBTYPE_I4x4 (0x00000100)   /* 4x4 intra     */
+#define MVE_MBTYPE_I8x8 (0x00000200)   /* 8x8 intra     */
+#define MVE_MBTYPE_I16x16 (0x00000400) /* 16x16 intra   */
+#define MVE_MBTYPE_I32x32 (0x00000800) /* 32x32 intra   */
+#define MVE_MBTYPE_16x32 (0x00001000)  /* 16x32 inter   */
+#define MVE_MBTYPE_32x16 (0x00002000)  /* 32x16 inter   */
+#define MVE_MBTYPE_32x32 (0x00004000)  /* 32x32 inter   */
+/* For option MVE_SET_OPT_INDEX_ENC_RGB_TO_YUV_MODE,
+ * arg should be one of these: */
+#define MVE_OPT_RGB_TO_YUV_BT601_STUDIO (0)
+#define MVE_OPT_RGB_TO_YUV_BT601_FULL (1)
+#define MVE_OPT_RGB_TO_YUV_BT709_STUDIO (2)
+#define MVE_OPT_RGB_TO_YUV_BT709_FULL (3)
+/* For option MVE_SET_OPT_INDEX_ENC_EXPOSE_REF_FRAMES,
+ * arg should be one of: */
+#define MVE_OPT_REF_OUTPUT_NONE (0)	    /* No REF output */
+#define MVE_OPT_REF_OUTPUT_USED_FOR_REF (1) /* Output reference frames */
+#define MVE_OPT_REF_OUTPUT_ALL (2)	    /* Output/reconstruct all frames */
+		struct {
+			uint16_t profile;
+/* AVC/H.264 profiles */
+#define MVE_OPT_PROFILE_H264_BASELINE (1)
+#define MVE_OPT_PROFILE_H264_MAIN (2)
+#define MVE_OPT_PROFILE_H264_HIGH (3)
+/* HEVC/H.265 profiles */
+#define MVE_OPT_PROFILE_H265_MAIN (1)
+#define MVE_OPT_PROFILE_H265_MAIN_STILL (2)
+#define MVE_OPT_PROFILE_H265_MAIN_INTRA (3)
+#define MVE_OPT_PROFILE_H265_MAIN_10 (4)
+/* VC-1 profiles */
+#define MVE_OPT_PROFILE_VC1_SIMPLE (1)
+#define MVE_OPT_PROFILE_VC1_MAIN (2)
+#define MVE_OPT_PROFILE_VC1_ADVANCED (3)
+/* VP8 profiles */
+#define MVE_OPT_PROFILE_VP8_MAIN (1)
+			uint16_t level;
+/* AVC/H.264 levels */
+#define MVE_OPT_LEVEL_H264_1 (1)
+#define MVE_OPT_LEVEL_H264_1b (2)
+#define MVE_OPT_LEVEL_H264_11 (3)
+#define MVE_OPT_LEVEL_H264_12 (4)
+#define MVE_OPT_LEVEL_H264_13 (5)
+#define MVE_OPT_LEVEL_H264_2 (6)
+#define MVE_OPT_LEVEL_H264_21 (7)
+#define MVE_OPT_LEVEL_H264_22 (8)
+#define MVE_OPT_LEVEL_H264_3 (9)
+#define MVE_OPT_LEVEL_H264_31 (10)
+#define MVE_OPT_LEVEL_H264_32 (11)
+#define MVE_OPT_LEVEL_H264_4 (12)
+#define MVE_OPT_LEVEL_H264_41 (13)
+#define MVE_OPT_LEVEL_H264_42 (14)
+#define MVE_OPT_LEVEL_H264_5 (15)
+#define MVE_OPT_LEVEL_H264_51 (16)
+#define MVE_OPT_LEVEL_H264_52 (17)
+#define MVE_OPT_LEVEL_H264_6 (18)
+#define MVE_OPT_LEVEL_H264_61 (19)
+#define MVE_OPT_LEVEL_H264_62 (20)
+#define MVE_OPT_LEVEL_H264_USER_SUPPLIED_BASE (32)
+/* The value (MVE_OPT_LEVEL_H264_USER_SUPPLIED_BASE + level_idc) encodes a user
+ * supplied level_idc value in the range 0 to 255 inclusive. If the host supplies a level_idc
+ * value by this method then the encoder will encode this level_idc value in the bitstream
+ * without checking the validity of the level_idc value
+ */
+#define MVE_OPT_LEVEL_H264_USER_SUPPLIED_MAX (MVE_OPT_LEVEL_H264_USER_SUPPLIED_BASE + 255)
+/* HEVC/H.265 levels */
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_1 (1)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_1 (2)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_2 (3)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_2 (4)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_21 (5)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_21 (6)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_3 (7)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_3 (8)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_31 (9)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_31 (10)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_4 (11)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_4 (12)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_41 (13)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_41 (14)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_5 (15)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_5 (16)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_51 (17)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_51 (18)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_52 (19)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_52 (20)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_6 (21)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_6 (22)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_61 (23)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_61 (24)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_62 (25)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_62 (26)
+		} profile_level;
+		struct {
+			int32_t mv_search_range_x;
+			int32_t mv_search_range_y;
+		} motion_vector_search_range;
+		struct {
+			uint32_t type;
+#define MVE_OPT_HUFFMAN_TABLE_DC_LUMA (1)
+#define MVE_OPT_HUFFMAN_TABLE_AC_LUMA (2)
+#define MVE_OPT_HUFFMAN_TABLE_DC_CHROMA (3)
+#define MVE_OPT_HUFFMAN_TABLE_AC_CHROMA (4)
+			uint8_t number_of_huffman_of_code_length[16];
+			uint8_t table[162]; /* 12 are used for DC, 162 for AC */
+		} huffman_table;
+		struct {
+			uint32_t type;
+#define MVE_OPT_QUANT_TABLE_LUMA (1)
+#define MVE_OPT_QUANT_TABLE_CHROMA (2)
+			uint8_t matrix[64];
+		} quant_table;
+		struct {
+			/* For HEVC, tile_cols must be zero. For VP9, tile_rows
+			 * and tile_cols must be powers of 2. */
+			uint16_t tile_rows;
+			uint16_t tile_cols;
+		} tiles;
+		struct {
+			uint16_t luma_bitdepth;
+			uint16_t chroma_bitdepth;
+		} bitdepth;
+		struct {
+			/* Scale factors, and their square roots, for the lambda
+			 * coefficients used by the encoder, in unsigned Q8 fixed-point
+			 * format. Default (no scaling) is 1.0 (so 0x0100 in hex).
+			 */
+			uint16_t lambda_scale_i_q8;
+			uint16_t lambda_scale_sqrt_i_q8;
+			uint16_t lambda_scale_p_q8;
+			uint16_t lambda_scale_sqrt_p_q8;
+			uint16_t lambda_scale_b_ref_q8;
+			uint16_t lambda_scale_sqrt_b_ref_q8;
+			uint16_t lambda_scale_b_nonref_q8;
+			uint16_t lambda_scale_sqrt_b_nonref_q8;
+		} lambda_scale;
+		/* ARBITRARY_DOWNSCALE */
+		struct {
+			uint16_t width;
+			uint16_t height;
+		} downscaled_frame;
+		struct {
+			uint32_t mode;
+		} dsl_pos;
+	} data;
+};
+
+struct mve_request_release_ref_frame {
+	/* Decode: For a frame buffer that MVE has returned
+	 * marked as _REF_FRAME, the host can send this message
+	 * to ask the MVE to release the buffer as soon as it is
+	 * no longer used as reference anymore. (Otherwise, in
+	 * normal operation, the host would re-enqueue the buffer
+	 * to the MVE when it has been displayed and can be over-
+	 * written with a new frame.)
+	 *
+	 * Note: When a frame is no longer used as reference depends
+	 * on the stream being decoded, and there is no way to
+	 * guarantee a short response time, the response may not
+	 * come until the end of the stream.
+	 *
+	 * Encode: Return this reference buffer to the firmware
+	 * so it can be reused. This is only useful when the
+	 * MVE_SET_OPT_INDEX_ENC_EXPOSE_REF_FRAMES is used and reference
+	 * frames are reported by events (and must be returned).
+	 */
+	uint32_t buffer_address;
+};
+
+/*********************************************************************
+ *
+ *   RESPONSEs are messages from the host to the firmware
+ *
+ *   Some of the MVE_RESPONSE_CODE_ codes are followed by one of the
+ *   structs below.
+ *
+ *********************************************************************/
+
+/* Sent when firmware has booted.
+ */
+struct mve_response_switched_in {
+	uint32_t core;
+};
+
+/* Sent when last core in a session has switched out.
+ */
+struct mve_response_switched_out {
+	uint32_t core;
+	uint32_t reason;
+	uint32_t sub_reason;
+};
+
+/* Response confirming state transition after either GO or STOP
+ * command from host.
+ */
+struct mve_response_state_change {
+	uint32_t new_state;
+#define MVE_STATE_STOPPED (0)
+#define MVE_STATE_RUNNING (2)
+};
+
+/* Message sent when the all cores in the session have dequeued a
+ * job from the firmware job queue.
+ */
+struct mve_response_job_dequeued {
+	uint32_t valid_job;
+};
+
+/* Fatal error message from firmware, if sent then no further
+ * operation is possible.
+ */
+struct mve_response_error {
+	uint32_t error_code;
+#define MVE_ERROR_ABORT (1)
+#define MVE_ERROR_OUT_OF_MEMORY (2)
+#define MVE_ERROR_ASSERT (3)
+#define MVE_ERROR_UNSUPPORTED (4)
+#define MVE_ERROR_INVALID_BUFFER (6)
+#define MVE_ERROR_INVALID_STATE (8)
+#define MVE_ERROR_WATCHDOG (9)
+
+#define MVE_MAX_ERROR_MESSAGE_SIZE (128)
+	char message[MVE_MAX_ERROR_MESSAGE_SIZE];
+};
+
+/* When a set-option succeeds, a confirmation message is
+ * sent, including the index-code for that particular option.
+ */
+struct mve_response_set_option_confirm {
+	uint32_t index; /* Same as 'index' in struct mve_request_set_option */
+};
+
+/* If a set-option request fails, this message is returned.
+ * This is not a fatal error. The set-option had no effect,
+ * and the session is still alive.
+ * For example, trying to set an option with a too large
+ * or small parameter would result in this message.
+ * The included text string is meant for development and
+ * debugging purposes only.
+ * (When a set-option succeeds the set-option-confirm
+ * message code is sent instead.)
+ */
+struct mve_response_set_option_fail {
+	uint32_t index; /* Same as 'index' in struct mve_request_set_option */
+	char message[MVE_MAX_ERROR_MESSAGE_SIZE];
+};
+
+/* Decode only: This message is sent from MVE to the host so that it can
+ * allocate large enough output buffers. Output buffers that are to small
+ * will be returned to the host marked as 'rejected'.
+ */
+struct mve_response_frame_alloc_parameters {
+	/* Please note that the below information is a hint
+	 * for what buffers to allocate, it does not say
+	 * what actual resolution an output picture has.
+	 */
+
+	/* To use if allocating PLANAR YUV output buffers: */
+	uint16_t planar_alloc_frame_width;
+	uint16_t planar_alloc_frame_height;
+
+	/* To use if allocating AFBC output buffers
+	 * (if interlace, each field needs this size):
+	 */
+	uint32_t afbc_alloc_bytes;
+
+	/* For situations where downscaled AFBC is supported,
+	 * this number of bytes is needed for the downscaled frame.
+	 */
+	uint32_t afbc_alloc_bytes_downscaled;
+
+	/* When the host allocates an AFBC frame buffer, it should normally set
+	 * the the afbc_width_in_superblocks to be at least this recommended value.
+	 * Buffers with smaller values are likely to be returned rejected by the MVE.
+	 * See also comments above for afbc_alloc_bytes and
+	 * afbc_alloc_bytes_downscaled, they describe the situations where the
+	 * different values are used.
+	 */
+	uint16_t afbc_width_in_superblocks;
+	uint16_t afbc_width_in_superblocks_downscaled;
+
+	/* For PLANAR YUV output, every plane's address need to be adjusted to get
+	 * optimal AXI bursts when the pixel data is written, the values below may
+	 * be used to calculate address offsets.
+	 */
+	uint16_t cropx;
+	uint16_t cropy;
+
+	uint32_t mbinfo_alloc_bytes; /* Only for debugging */
+
+	/* downscaled frame width/height for decode */
+	/* ARBITRARY_DOWNSCALE */
+	uint16_t dsl_frame_width;
+	uint16_t dsl_frame_height;
+	uint16_t dsl_pos_mode;
+	uint8_t ctu_size; /* EXPORT_SEQ_INFO */
+};
+
+/* Decode only: This message is sent from MVE to the host so that it can
+ * allocate suitable output buffers. The needed size of the buffer is sent
+ * in a separate message (above).
+ * When MVE sends the message below, it enters a waiting-state and will not
+ * make any progress until the host sends an output-flush command, upon
+ * which MVE will return all output buffers, followed by a message saying
+ * that the output has been flushed. Only then should the host start
+ * enqueueing new output buffers.
+ */
+struct mve_response_sequence_parameters {
+	/* Other stream parameters affecting buffer allocation,
+	 * any change in these values will trigger a flush.
+	 */
+	uint8_t interlace; /* 0 or 1 */
+	uint8_t chroma_format;
+#define MVE_CHROMA_FORMAT_MONO (0x0)
+#define MVE_CHROMA_FORMAT_420 (0x1)
+#define MVE_CHROMA_FORMAT_422 (0x2)
+#define MVE_CHROMA_FORMAT_440 (0x3)
+#define MVE_CHROMA_FORMAT_ARGB (0x4)
+	uint8_t bitdepth_luma;	       /* 8, 9 or 10 */
+	uint8_t bitdepth_chroma;       /* 8, 9 or 10 */
+	uint8_t num_buffers_planar;    /* number of planar buffers needed */
+	uint8_t num_buffers_afbc;      /* number of AFBC buffers needed, for
+					* AFBC output more buffers are needed
+					* (for planar output, the firmware
+					* will allocate extra memory via RPC)
+					*/
+	uint8_t range_mapping_enabled; /* VC-1 AP specific feature, if enabled
+					* then AFBC buffers may need special
+					* filtering before they can be
+					* displayed correctly. If the host is
+					* not able to do that, then planar output
+					* should be used, for which MVE
+					* automatically performs the filtering.
+					*/
+	uint8_t reserved0;
+};
+
+struct mve_response_ref_frame_unused {
+	/* Decode only: If requested by the host with the message
+	 * MVE_REQUEST_CODE_RELEASE_REF_FRAME, the MVE will respond
+	 * with this message when (if ever) the buffer is no longer
+	 * used.
+	 */
+	uint32_t unused_buffer_address;
+};
+
+/* This message is only for debugging and performance profiling.
+ * Is sent by the firmware if the corresponding options is enabled.
+ */
+struct mve_event_processed {
+	uint8_t pic_format;
+	uint8_t qp;
+	uint8_t pad0;
+	uint8_t pad1;
+	uint32_t parse_start_time; /* Timestamp, absolute time */
+	uint32_t parse_end_time;   /* Timestamp, absolute time */
+	uint32_t parse_idle_time;  /* Definition of idle here is waiting for in/out buffers or available RAM */
+
+	uint32_t pipe_start_time; /* Timestamp */
+	uint32_t pipe_end_time;	  /* Timestamp, end-start = process time. Idle time while in a frame is
+				   * not measured. */
+	uint32_t pipe_idle_time;  /* Always 0 in decode, */
+
+	uint32_t parser_coreid; /* Core used to parse this frame */
+	uint32_t pipe_coreid;	/* Core used to pipe this frame */
+
+	uint32_t bitstream_bits; /* Number of bitstream bits used for this frame. */
+
+	uint32_t intermediate_buffer_size; /* Size of intermediate (mbinfo/residuals) buffer after this frame was
+					    * parsed. */
+	uint32_t total_memory_allocated;   /* after the frame was parsed. Including reference frames. */
+
+	uint32_t bus_read_bytes;  /* bus read bytes */
+	uint32_t bus_write_bytes; /* bus written bytes */
+
+	uint32_t afbc_bytes; /* afbc data transferred */
+
+	uint32_t slice0_end_time;   /* Timestamp, absolute time */
+	uint32_t stream_start_time; /* Timestamp, absolute stream start time */
+	uint32_t stream_open_time;  /* Timestamp, absolute stream open time */
+};
+
+/* This message is sent by the firmware if the option
+ * MVE_SET_OPT_INDEX_ENC_EXPOSE_REF_FRAMES is enabled
+ */
+struct mve_event_ref_frame {
+	uint32_t ref_addr;	    /* MVE virtual address of AFBC reference frame */
+	uint32_t ref_width;	    /* Width of display area in luma pixels        */
+	uint32_t ref_height;	    /* Height of display area in luma pixels       */
+	uint32_t ref_mb_width;	    /* Width in macroblocks                        */
+	uint32_t ref_mb_height;	    /* Height in macroblocks                       */
+	uint32_t ref_left_crop;	    /* Left crop in luma pixels                    */
+	uint32_t ref_top_crop;	    /* Top crop in luma pixels                     */
+	uint32_t ref_frame_size;    /* Total AFBC frame size in bytes              */
+	uint32_t ref_display_order; /* Display picture order count                 */
+	uint16_t bit_width;	    /* Bit width of the YUV either 8 or 10         */
+	uint16_t tiled_headers;	    /* AFBC format is tiled                        */
+	uint64_t user_data_tag;	    /* User data tag of corresponding input buffer */
+};
+
+/* This message is only for debugging, is sent by the firmware if event tracing
+ * is enabled.
+ */
+struct mve_event_trace_buffers {
+	uint16_t reserved;
+	uint8_t num_cores;
+	uint8_t rasc_mask;
+#define MVE_MAX_TRACE_BUFFERS 40
+	/* this array will contain one buffer per rasc in rasc_mask per num_core */
+	struct {
+		uint32_t rasc_addr; /* rasc address of the buffer */
+		uint32_t size;	    /* size of the buffer in bytes */
+	} buffers[MVE_MAX_TRACE_BUFFERS];
+};
+
+/* 'Events' are informative messages, the host is not required to react in
+ * any particular way.
+ */
+struct mve_response_event {
+	uint32_t event_code;
+#define MVE_EVENT_ERROR_STREAM_CORRUPT (1)	 /* message, text string */
+#define MVE_EVENT_ERROR_STREAM_NOT_SUPPORTED (2) /* message, text string */
+#define MVE_EVENT_PROCESSED (3)			 /* struct mve_event_processed */
+#define MVE_EVENT_REF_FRAME (4)			 /* struct mve_event_ref_frame */
+#define MVE_EVENT_TRACE_BUFFERS (5)		 /* struct mve_event_trace_buffers */
+	union {
+		struct mve_event_processed event_processed;
+		struct mve_event_ref_frame event_ref_frame;
+		struct mve_event_trace_buffers event_trace_buffers;
+		char message[MVE_MAX_ERROR_MESSAGE_SIZE];
+	} event_data;
+} __attribute__((packed));
+
+/*********************************************************************
+ *
+ *   BUFFERs are sent both ways, from host to firmware and back again
+ *
+ *   Each MVE_BUFFER_CODE_ code is followed by one of the structs
+ *   below.
+ *
+ *********************************************************************/
+
+/* Flags in mve_buffer_frame::frame_flags:
+ *                                     Set by whom?    Meaning:
+ *                                     DECODE: ENCODE:
+ * MVE_BUFFER_FRAME_FLAG_INTERLACE      host    -      Buffer is interlaced (both top and
+ *                                                     bottom fields are allocated)
+ * MVE_BUFFER_FRAME_FLAG_BOT_FIRST      fw      -      Bottom field should be displayed
+ *                                                     first (only if interlaced)
+ * MVE_BUFFER_FRAME_FLAG_TOP_PRESENT    fw      host   Top field present (or full frame if
+ *                                                     not interlaced)
+ * MVE_BUFFER_FRAME_FLAG_BOT_PRESENT    fw      -      Bottom present (only if interlaced)
+ *
+ * MVE_BUFFER_FRAME_FLAG_ROTATION_*     host    host   Decode: MVE will rotate the output frame
+ *                                                             according to this setting.
+ *                                                     Encode: MVE will rotate the input frame
+ *                                                             according to this setting before
+ *                                                             encoding them.
+ * MVE_BUFFER_FRAME_FLAG_SCALING_MASK   host    -      Output pictures should be downscaled
+ *
+ * MVE_BUFFER_FRAME_FLAG_MIRROR_*       -       host   Input frame should be mirrored before encoding
+ *
+ * MVE_BUFFER_FRAME_FLAG_REJECTED       fw      -      Buffer was too small, host should re-allocate
+ *
+ * MVE_BUFFER_FRAME_FLAG_CORRUPT        fw      -      Frame contains visual corruption
+ *
+ * MVE_BUFFER_FRAME_FLAG_DECODE_ONLY    fw      -      Frame should not be displayed
+ *
+ * MVE_BUFFER_FRAME_FLAG_REF_FRAME      fw      -      Frame is used by MVE as reference, host must
+ *                                                     not change, just re-enqueue when displayed
+ * MVE_BUFFER_FRAME_FLAG_EOS            fw      host   This is the last frame in the stream.
+ */
+
+/* mve_buffer_frame_planar stores uncompressed YUV pictures.
+ *       ________________________________________
+ *      |  ^                          |          |  ^
+ *      |<-:--visible_frame_width---->|          |  :
+ *      |  :                          |          |  :
+ *      |  :                          |          |  :
+ *      | visible_frame_height        |          | max_frame_height
+ *      |  :                          |          |  :
+ *      |  :                          |          |  :
+ *      |__v__________________________|          |  :
+ *      |                                        |  :
+ *      |<-------------max_frame_width---------->|  :
+ *      |________________________________________|  v
+ *
+ */
+struct mve_buffer_frame_planar {
+	/* Y,Cb,Cr top field */
+	uint32_t plane_top[3];
+
+	/* Y,Cb,Cr bottom field (interlace only) */
+	uint32_t plane_bot[3];
+
+	/* Stride between rows, in bytes */
+	int32_t stride[3];
+
+	/* Size of largest frame allowed to put in this buffer */
+	uint16_t max_frame_width;
+	uint16_t max_frame_height;
+};
+
+/* mve_buffer_frame_afbc stores AFBC compressed content that is also used
+ * as the reference frame. Out of loop processing (crop, rotation,
+ * range reduction) must be supported by the user of this buffer and
+ * the parameters are signaled within the buffer descriptor below.
+ *       ________________________________________
+ *      |         ^                              |
+ *      |       cropy                            |
+ *      |         v_____________________________ |
+ *      |<-cropx->|  ^                          ||
+ *      |         |<-:--visible_frame_width---->||
+ *      |         |  :                          ||
+ *      |         |  :                          ||
+ *      |         | visible_frame_height        ||
+ *      |         |  :                          ||
+ *      |         |  :                          ||
+ *      |         |__v__________________________||
+ *      |________________________________________|
+ *
+ *       <----- superblock_width --------------->
+ *              * afbc_width_in_superblocks
+ *
+ * Note that the sizes and cropping values need not be multiples of 16.
+ *
+ * For interlaced streams, the values refer to a full frame,
+ * while the output is actually separated into fields. Thus for fields,
+ * cropy and visible_frame_height should be divided by two.
+ *
+ * For dual-downscaled AFBC output (not supported for interlace),
+ * then the cropx, cropy, visible_frame_width and visible_frame_height
+ * should be divided by two for the downscaled plane.
+ */
+struct mve_buffer_frame_afbc {
+	uint32_t plane[2];				/* Addresses for up to two AFBC planes:
+							 * Top and bottom fields for interlace,
+							 * or standard and optional downscaled output. */
+	uint32_t alloc_bytes[2];			/* Size of allocation for each plane */
+	uint16_t cropx;					/* Luma x crop */
+	uint16_t cropy;					/* Luma y crop */
+	uint16_t afbc_width_in_superblocks[2];		/* Width of AFBC frame buffer, in units
+							 * of superblock width (32 or 16).
+							 * If dual-downscaled output is chosen,
+							 * this width can be different for the
+							 * two planes.
+							 * For first plane:
+							 * (cropx + frame_width)
+							 *      <= superblock_width  * afbc_width...
+							 */
+	uint32_t afbc_params;				/* AFBC parameters */
+#define MVE_BUFFER_FRAME_AFBC_TILED_BODY (0x00000001)	/* Output body blocks should be tiled */
+#define MVE_BUFFER_FRAME_AFBC_TILED_HEADER (0x00000002) /* Output headers should be tiled */
+#define MVE_BUFFER_FRAME_AFBC_32X8_SUPERBLOCK                                                                          \
+	(0x00000004) /* Super block is 32x8, default is 16x16,                                                         \
+		      * (only supported as input for encode) */
+#define MVE_BUFFER_FRAME_AFBC_DN_FORCE_8BIT                                                                            \
+	(0x00000008) /* For downscaled AFBC plane: It shall                                                            \
+		      * be 8-bit, even if full-scale is 10-bit */
+#define MVE_BUFFER_FRAME_AFBC_DN_FORCE_420                                                                             \
+	(0x00000010) /* For downscaled AFBC plane: It shall                                                            \
+		      * be 4:2:0, even if full-scale is 4:2:2 */
+#define MVE_BUFFER_FRAME_AFBC_STRIDE_SET_BY_MVE                                                                        \
+	(0x00000020) /* Decode only: By default, the host should                                                       \
+			set the afbc_width_in_superblocks. If the                                                      \
+			value is zero, or if this bit is set, then                                                     \
+			the MVE sets an appropriate value. */
+#define MVE_BUFFER_FRAME_AFBC_BLOCK_SPLIT                                                                              \
+	(0x00000040) /* For Superblock layout, block_split mode                                                        \
+			should be enabled*/
+};
+
+/*
+ * The FRAME buffer stores the common information for PLANAR and AFBC buffers,
+ * and a union of PLANAR and AFBC specific information.
+ */
+struct mve_buffer_frame {
+	/* For identification of the buffer, this is not changed by
+	 * the firmware. */
+	uint64_t host_handle;
+
+	/* For matching input buffer with output buffers, the firmware
+	 * copies these values between frame buffers and bitstream buffers. */
+	uint64_t user_data_tag;
+
+	/* Frame buffer flags, see commentary above */
+	uint32_t frame_flags;
+#define MVE_BUFFER_FRAME_FLAG_INTERLACE (0x00000001)
+#define MVE_BUFFER_FRAME_FLAG_BOT_FIRST (0x00000002)
+#define MVE_BUFFER_FRAME_FLAG_TOP_PRESENT (0x00000004)
+#define MVE_BUFFER_FRAME_FLAG_BOT_PRESENT (0x00000008)
+#define MVE_BUFFER_FRAME_FLAG_ROTATION_90 (0x00000010)
+#define MVE_BUFFER_FRAME_FLAG_ROTATION_180 (0x00000020)
+#define MVE_BUFFER_FRAME_FLAG_ROTATION_270 (0x00000030)
+#define MVE_BUFFER_FRAME_FLAG_SCALING_MASK (0x000000C0)
+#define MVE_BUFFER_FRAME_FLAG_MIRROR_HORI (0x00000100)
+#define MVE_BUFFER_FRAME_FLAG_MIRROR_VERT (0x00000200)
+#define MVE_BUFFER_FRAME_FLAG_REJECTED (0x00001000)
+#define MVE_BUFFER_FRAME_FLAG_CORRUPT (0x00002000)
+#define MVE_BUFFER_FRAME_FLAG_DECODE_ONLY (0x00004000)
+#define MVE_BUFFER_FRAME_FLAG_REF_FRAME (0x00008000)
+#define MVE_BUFFER_FRAME_FLAG_EOS (0x00010000)
+/*ARBITRARY_DOWNSCALE*/
+#define MVE_BUFFER_FRAME_FLAG_SCALING_MASKX (0xFF000000) // 8bit
+#define MVE_BUFFER_FRAME_FLAG_SCALING_MASKY (0x00FE0000) // 7bit
+
+	/* Height (in luma samples) of visible part of frame,
+	 * may be smaller than allocated frame size. */
+	uint16_t visible_frame_height;
+
+	/* Width (in luma samples) of visible part of frame,
+	 * may be smaller than allocated frame size. */
+	uint16_t visible_frame_width;
+
+	/* Color format of buffer */
+	uint16_t format;
+/* format bitfield: */
+#define MVE_FORMAT_BF_C (0)  /* 3 bits, chroma subsampling   */
+#define MVE_FORMAT_BF_B (4)  /* 4 bits, max bitdepth minus 8 */
+#define MVE_FORMAT_BF_N (8)  /* 2 bits, number of planes     */
+#define MVE_FORMAT_BF_V (12) /* 2 bits, format variant       */
+#define MVE_FORMAT_BF_A (15) /* 1 bit,  AFBC bit             */
+/* formats: */
+#define MVE_FORMAT_YUV420_AFBC_8                                                                                       \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV420_AFBC_10                                                                                      \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV422_AFBC_8                                                                                       \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV422_AFBC_10                                                                                      \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV420_I420                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (3 << MVE_FORMAT_BF_N) |          \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_NV12                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (2 << MVE_FORMAT_BF_N) |          \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_NV21                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (2 << MVE_FORMAT_BF_N) |          \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_P010                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((16 - 8) << MVE_FORMAT_BF_B) | (2 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_Y0L2                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_AQB1                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV422_YUY2                                                                                         \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |          \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV422_UYVY                                                                                         \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |          \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV422_Y210                                                                                         \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((16 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_RGBA_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_BGRA_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_ARGB_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (2 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_ABGR_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (3 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_MBINFO (0x0001) /* only used for debugging */
+
+#define MVE_FORMAT_UNUSED (0x0000)
+
+	uint16_t reserved0; /* force 'data' to be 4-byte aligned */
+
+	union {
+		struct mve_buffer_frame_planar planar;
+		struct mve_buffer_frame_afbc afbc;
+	} data;
+
+	uint32_t reserved1; /* force size to be multiple of 8 bytes */
+};
+
+/* The bitstream buffer stores a number of bitstream bytes */
+struct mve_buffer_bitstream {
+	/* For identification of the buffer, this is not changed by
+	 * the firmware. */
+	uint64_t host_handle;
+
+	/* For matching input buffer with output buffers, the firmware
+	 * copies these values between frame buffers and bitstream buffers. */
+	uint64_t user_data_tag;
+
+	/* BufferFlags */
+	uint32_t bitstream_flags;
+#define MVE_BUFFER_BITSTREAM_FLAG_EOS (0x00000001)
+#define MVE_BUFFER_BITSTREAM_FLAG_ENDOFFRAME (0x00000010)
+#define MVE_BUFFER_BITSTREAM_FLAG_SYNCFRAME (0x00000020)
+#define MVE_BUFFER_BITSTREAM_FLAG_CODECCONFIG (0x00000080)
+#define MVE_BUFFER_BITSTREAM_FLAG_ENDOFSUBFRAME (0x00000400)
+
+	/* Length of allocated buffer */
+	uint32_t bitstream_alloc_bytes;
+
+	/* Byte offset from start to first byte */
+	uint32_t bitstream_offset;
+
+	/* Number of bytes in the buffer */
+	uint32_t bitstream_filled_len;
+
+	/* Pointer to buffer start */
+	uint32_t bitstream_buf_addr;
+
+	/* frame_type. 0:I, 1:P, 2:B, 3:b */
+	uint8_t frame_type;
+
+	/* Pad to force 8-byte alignment */
+	// uint32_t reserved;
+	uint8_t reserved[3];
+};
+
+/*
+ * Define a region in 16x16 units
+ *
+ * The region is macroblock positions (x,y) in the range
+ * mbx_left <= x < mbx_right
+ * mby_top  <= y < mby_bottom
+ */
+struct mve_buffer_param_region {
+	uint16_t mbx_left;   /* macroblock x left edge   (inclusive) */
+	uint16_t mbx_right;  /* macroblock x right edge  (exclusive) */
+	uint16_t mby_top;    /* macroblock y top edge    (inclusive) */
+	uint16_t mby_bottom; /* macroblock y bottom edge (exclusive) */
+	int16_t qp_delta;    /* QP delta value for this region, this
+			      * delta applies to QP values in the ranges:
+			      *   H264: 0-51
+			      *   HEVC: 0-51
+			      *   VP9:  0-255 */
+	uint16_t reserved;
+};
+
+/* input for encoder,
+ * the mve_buffer_param_regions buffer stores the information for FRAME buffers,
+ * and the information for regions of interest.
+ */
+struct mve_buffer_param_regions {
+	uint8_t n_regions; /* Number of regions */
+	uint8_t reserved[3];
+#define MVE_MAX_FRAME_REGIONS 16
+	struct mve_buffer_param_region region[MVE_MAX_FRAME_REGIONS];
+};
+
+/* the block parameter record specifies the various properties of a quad */
+struct mve_block_param_record {
+	uint16_t qp_delta;
+/* Bitset of four 4-bit QP delta values for a quad.
+ * For H.264 and HEVC these are qp delta values in the range -8 to +7.
+ * For Vp9 these are segment map values in the range 0 to 7.
+ */
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_LEFT_16X16 (0)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_LEFT_16X16_SZ (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_RIGHT_16X16 (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_RIGHT_16X16_SZ (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_LEFT_16X16 (8)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_LEFT_16X16_SZ (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_RIGHT_16X16 (12)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_RIGHT_16X16_SZ (4)
+
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_LEFT_16X16 (0)
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_LEFT_16X16_SZ (3)
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_RIGHT_16X16 (4)
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_RIGHT_16X16_SZ (3)
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_LEFT_16X16 (8)
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_LEFT_16X16_SZ (3)
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_RIGHT_16X16 (12)
+#define MVE_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_RIGHT_16X16_SZ (3)
+
+	uint8_t force;
+#define MVE_BLOCK_PARAM_RECORD_FORCE_NONE (0x00)
+#define MVE_BLOCK_PARAM_RECORD_FORCE_QP (0x01)
+#define MVE_BLOCK_PARAM_RECORD_FORCE_32X32 (0x02)
+#define MVE_BLOCK_PARAM_RECORD_FORCE_RB (0x04)
+
+	uint8_t reserved;
+};
+
+/* block configuration uncompressed rows header. this configures the size of the
+ * uncompressed body. */
+struct mve_buffer_general_rows_uncomp_hdr {
+	uint8_t n_cols_minus1; /* number of quad cols in picture minus 1 */
+	uint8_t n_rows_minus1; /* number of quad rows in picture minus 1 */
+	uint8_t reserved[2];
+};
+
+/* block configuration uncompressed rows body. this structure contains an array
+ * of block parameter records whose length is (n_cols_minus1 + 1) * (n_rows_minus1 + 1)
+ * elements. therefore the allocation of this structure needs to be dynamic and
+ * a pointer to the allocated memory should then be assigned to the general
+ * purpose buffer data pointer
+ */
+struct mve_buffer_general_rows_uncomp_body {
+	/* the size of this array is variable and not necessarily equal to 1.
+	 * therefore the sizeof operator should not be used
+	 */
+	struct mve_block_param_record bpr[1];
+};
+
+/* input for encoder, block level configurations.
+ * the row based block configurations can be defined in different formats. they
+ * are stored in the blk_cfgs union and identified by the blk_cfg_type member.
+ * these definitions consist of a header and body pair. the header part contains
+ * configuration information for the body. the body part describes the actual
+ * layout of the data buffer pointed to by the mve_buffer_general_hdr buffer_ptr.
+ */
+struct mve_buffer_general_block_configs {
+	uint8_t blk_cfg_type;
+#define MVE_BLOCK_CONFIGS_TYPE_NONE (0x00)
+#define MVE_BLOCK_CONFIGS_TYPE_ROW_UNCOMP (0xff)
+	uint8_t reserved[3];
+	union {
+		struct mve_buffer_general_rows_uncomp_hdr rows_uncomp;
+	} blk_cfgs;
+};
+
+/* for the mve_buffer_general_encoder_stats buffer type the body data
+ * is an array of records of the following format, one record for each
+ * 32x32 pixel block of the picture. Values that are marked "per CTU"
+ * are only valid in the first 32x32 block of the 64x64 CTU for HEVC or VP9.
+ * For H.264 the "per CTU" values are the sum of four 16x16 macroblocks.
+ * Each record is 12 bytes and must be 32-bit aligned.
+ */
+struct mve_block_stats_record_full {
+	uint8_t intra_count;   // number of 8x8 blocks in the CTU that are intra
+	uint8_t reserved0;     // not used
+	uint16_t bit_estimate; // bit estimate for the CTU
+	uint16_t luma_mean;    // luminance mean
+	uint16_t luma_cplx;    // luma complexity measure (0=flat)
+	int16_t rmv_x;	       // rough x motion vector in pixels
+	int16_t rmv_y;	       // rough y motion vector in pixels
+};
+
+/* input for encoder, block level statistics
+ * this buffer is added to the encoder input frame queue, before the
+ * input frame for which statistics are to be gathered. The buffer is
+ * filled duing frame analysis and returned in the input buffer return
+ * queue before the input buffer is returned.
+ */
+struct mve_buffer_general_encoder_stats {
+	uint8_t encoder_stats_type;
+#define MVE_ENCODER_STATS_TYPE_FULL (0x01)
+	uint8_t frame_type; // See MVE_FRAME_TYPE_*
+#define MVE_FRAME_TYPE_I 0
+#define MVE_FRAME_TYPE_P 1
+#define MVE_FRAME_TYPE_B 2
+	uint8_t used_as_reference; // 0=No, 1=Yes
+	uint8_t qp;		   // base quantizer used for the frame
+				   // HEVC, H.264: 0-51. VP9: 0-63
+	uint32_t picture_count;	   // display order picture count
+	uint16_t num_cols;	   // number of columns (each 32 pixels wide)
+	uint16_t num_rows;	   // number of rows    (each 32 pixels high)
+	uint32_t ref_pic_count[2]; // display order picture count of references
+				   // unused values are set to zero
+};
+
+/* input for encoder */
+struct mve_buffer_param_qp {
+	/* QP (quantization parameter) for encode.
+	 *
+	 * When used to set fixed QP for encode, with rate control
+	 * disabled, then the valid ranges are:
+	 *   H264: 0-51
+	 *   HEVC: 0-51
+	 *   VP8:  0-63
+	 *   VP9:  0-63
+	 * Note: The QP must be set separately for I, P and B frames.
+	 *
+	 * But when this message is used with the regions-feature,
+	 * then the valid ranges are the internal bitstream ranges:
+	 *   H264: 0-51
+	 *   HEVC: 0-51
+	 *   VP8:  0-127
+	 *   VP9:  0-255
+	 */
+	int32_t qp;
+};
+
+/* output from decoder */
+struct mve_buffer_param_display_size {
+	uint16_t display_width;
+	uint16_t display_height;
+};
+
+/* output from decoder, colour information needed for hdr */
+struct mve_buffer_param_colour_description {
+	uint32_t flags;
+#define MVE_BUFFER_PARAM_COLOUR_FLAG_MASTERING_DISPLAY_DATA_VALID (1)
+#define MVE_BUFFER_PARAM_COLOUR_FLAG_CONTENT_LIGHT_DATA_VALID (2)
+
+	uint8_t range; /* Unspecified=0, Limited=1, Full=2 */
+#define MVE_BUFFER_PARAM_COLOUR_RANGE_UNSPECIFIED (0)
+#define MVE_BUFFER_PARAM_COLOUR_RANGE_LIMITED (1)
+#define MVE_BUFFER_PARAM_COLOUR_RANGE_FULL (2)
+
+	uint8_t colour_primaries;	  /* see hevc spec. E.3.1 */
+	uint8_t transfer_characteristics; /* see hevc spec. E.3.1 */
+	uint8_t matrix_coeff;		  /* see hevc spec. E.3.1 */
+
+	uint16_t mastering_display_primaries_x[3]; /* see hevc spec. D.3.27 */
+	uint16_t mastering_display_primaries_y[3]; /* see hevc spec. D.3.27 */
+	uint16_t mastering_white_point_x;	   /* see hevc spec. D.3.27 */
+	uint16_t mastering_white_point_y;	   /* see hevc spec. D.3.27 */
+	uint32_t max_display_mastering_luminance;  /* see hevc spec. D.3.27 */
+	uint32_t min_display_mastering_luminance;  /* see hevc spec. D.3.27 */
+
+	uint32_t max_content_light_level; /* see hevc spec. D.3.35 */
+	uint32_t avg_content_light_level; /* see hevc spec. D.3.35 */
+
+	uint8_t video_format_present_flag;
+	uint8_t video_format;
+	uint8_t aspect_ratio_info_present_flag;
+	uint8_t aspect_ratio_idc;
+	uint8_t timing_flag_info_present_flag;
+	uint16_t sar_width;
+	uint16_t sar_height;
+	uint32_t num_units_in_tick;
+	uint32_t time_scale;
+
+	uint32_t reserved[2];
+};
+
+struct mve_buffer_param_sei_user_data_unregistered {
+	uint8_t flags;
+#define MVE_BUFFER_PARAM_USER_DATA_UNREGISTERED_VALID (1)
+	uint8_t uuid[16];
+	char user_data[256 - 35];
+	uint8_t user_data_len;
+
+	uint8_t reserved[5];
+};
+
+/* output from decoder see hevc spec. D.3.3 */
+struct mve_buffer_param_frame_field_info {
+	uint8_t pic_struct;
+	uint8_t source_scan_type;
+	uint8_t duplicate_flag;
+	uint8_t reserved;
+};
+
+/* output from decoder, VC-1 specific feature only relevant
+ * if using AFBC output
+ */
+struct mve_buffer_param_range_map {
+	uint8_t luma_map_enabled;
+	uint8_t luma_map_value;
+	uint8_t chroma_map_enabled;
+	uint8_t chroma_map_value;
+};
+
+/* input for encoder */
+struct mve_buffer_param_rate_control {
+	uint32_t rate_control_mode;
+#define MVE_OPT_RATE_CONTROL_MODE_OFF (0)
+#define MVE_OPT_RATE_CONTROL_MODE_STANDARD (1)
+#define MVE_OPT_RATE_CONTROL_MODE_VARIABLE (2)
+#define MVE_OPT_RATE_CONTROL_MODE_CONSTANT (3)
+#define MVE_OPT_RATE_CONTROL_MODE_C_VARIABLE (4)
+	uint32_t target_bitrate;  /* in bits per second */
+	uint32_t maximum_bitrate; /* in bits per second */
+};
+
+/* input for encoder */
+struct mve_buffer_param_rate_control_qp_range {
+	int32_t qp_min;
+	int32_t qp_max;
+};
+
+/* input for encoder, see hevc spec. D.3.16 */
+struct mve_buffer_param_frame_packing {
+	uint32_t flags;
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_QUINCUNX_SAMPLING (1)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_SPATIAL_FLIPPING (2)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_FRAME0_FLIPPED (4)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_FIELD_VIEWS (8)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_CURRENT_FRAME_IS_FRAME0 (16)
+
+	uint8_t frame_packing_arrangement_type;
+	uint8_t content_interpretation_type;
+
+	uint8_t frame0_grid_position_x;
+	uint8_t frame0_grid_position_y;
+	uint8_t frame1_grid_position_x;
+	uint8_t frame1_grid_position_y;
+
+	uint8_t reserved[2];
+};
+
+struct mve_buffer_param_rectangle {
+	uint16_t x_left;   /* pixel x left edge   (inclusive) */
+	uint16_t x_right;  /* pixel x right edge  (exclusive) */
+	uint16_t y_top;	   /* pixel y top edge    (inclusive) */
+	uint16_t y_bottom; /* pixel y bottom edge (exclusive) */
+};
+
+/* input for encoder,
+ * indicate which parts of the source picture has changed.
+ * The encoder can (optionally) use this information to
+ * reduce memory bandwidth.
+ *
+ * n_rectangles=0 indicates the source picture is unchanged.
+ *
+ * This parameter only applies to the picture that immediately
+ * follows (and not to subsequent ones).
+ */
+struct mve_buffer_param_change_rectangles {
+	uint8_t n_rectangles; /* Number of rectangles */
+	uint8_t reserved[3];
+#define MVE_MAX_FRAME_CHANGE_RECTANGLES 2
+	struct mve_buffer_param_rectangle rectangles[MVE_MAX_FRAME_CHANGE_RECTANGLES];
+};
+
+/* input for VP9 encoder,
+ * specify the qp deltas for each segment map index.
+ * These are intended to be used with block configs only.
+ */
+struct mve_buffer_param_vp9_segmap {
+#define VP9SPEC_MAX_SEGMENTS 8
+	int8_t qp_delta[VP9SPEC_MAX_SEGMENTS]; /* Qp delta to use for each segment map region */
+	int8_t num_segments;		       /* Number of active segments (to set coding probability) */
+};
+
+/* Parameters that are sent in the same communication channels
+ * as the buffers. A parameter applies to all subsequent buffers.
+ * Some types are only valid for decode, and some only for encode.
+ */
+struct mve_buffer_param {
+	uint32_t type;			       /* Extra data: */
+#define MVE_BUFFER_PARAM_TYPE_QP (2)	       /* qp */
+#define MVE_BUFFER_PARAM_TYPE_REGIONS (3)      /* regions */
+#define MVE_BUFFER_PARAM_TYPE_DISPLAY_SIZE (5) /* display_size */
+#define MVE_BUFFER_PARAM_TYPE_RANGE_MAP (6)    /* range_map */
+#define MVE_BUFFER_PARAM_TYPE_FRAME_RATE                                                                               \
+	(9)					      /* arg, in frames per second, as a                               \
+						       * fixed point Q16 value, for example                            \
+						       * 0x001e0000 == 30.0 fps */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL (10)	      /* rate_control */
+#define MVE_BUFFER_PARAM_TYPE_QP_I (12)		      /* qp for I frames, when no rate control */
+#define MVE_BUFFER_PARAM_TYPE_QP_P (13)		      /* qp for P frames, when no rate control */
+#define MVE_BUFFER_PARAM_TYPE_QP_B (14)		      /* qp for B frames, when no rate control */
+#define MVE_BUFFER_PARAM_TYPE_COLOUR_DESCRIPTION (15) /* colour_description */
+#define MVE_BUFFER_PARAM_TYPE_FRAME_PACKING (16)      /* frame_packing */
+#define MVE_BUFFER_PARAM_TYPE_FRAME_FIELD_INFO (17)   /* frame_field_info */
+#define MVE_BUFFER_PARAM_TYPE_GOP_RESET (18)	      /* no extra data */
+#define MVE_BUFFER_PARAM_TYPE_DPB_HELD_FRAMES                                                                          \
+	(19)						     /* arg, number of output buffers that are                 \
+							      * complete and held by firmware in the                   \
+							      * DPB for reordering purposes.                           \
+							      * Valid after the next frame is output */
+#define MVE_BUFFER_PARAM_TYPE_CHANGE_RECTANGLES (20)	     /* change rectangles */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_QP_RANGE (21)     /* rate_control_qp_range */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_HRD_BUF_SIZE (23) /* arg */
+#define MVE_BUFFER_PARAM_TYPE_VP9_SEGMAP (24)		     /* VP9 segment map settings */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_QP_RANGE_I                                                                  \
+	(25)						      /* special range for I frames,                           \
+							       * rate_control_qp_range */
+#define MVE_BUFFER_PARAM_TYPE_SEI_USER_DATA_UNREGISTERED (26) /* sei user_data_unregistered */
+
+	union {
+		uint32_t arg; /* some parameters only need a uint32_t as argument */
+		struct mve_buffer_param_qp qp;
+		struct mve_buffer_param_regions regions;
+		struct mve_buffer_param_display_size display_size;
+		struct mve_buffer_param_range_map range_map;
+		struct mve_buffer_param_rate_control rate_control;
+		struct mve_buffer_param_rate_control_qp_range rate_control_qp_range;
+		struct mve_buffer_param_colour_description colour_description;
+		struct mve_buffer_param_frame_packing frame_packing;
+		struct mve_buffer_param_frame_field_info frame_field_info;
+		struct mve_buffer_param_change_rectangles change_rectangles;
+		struct mve_buffer_param_vp9_segmap vp9_segmap;
+		struct mve_buffer_param_sei_user_data_unregistered user_data_unregistered;
+	} data;
+};
+
+/* The general purpose buffer header stores the common fields of an
+ * mve_buffer_general. it contains the pointer to the data buffer that contains
+ * the general purpose data
+ */
+struct mve_buffer_general_hdr {
+	/* For identification of the buffer, this is not changed by the firmware. */
+	uint64_t host_handle;
+
+	/* this depends upon the type of the general purpose buffer */
+	uint64_t user_data_tag;
+
+	/* pointer to the buffer containing the general purpose data. the format
+	 * of this data is defined by the configuration in the mve_buffer_general */
+	uint32_t buffer_ptr;
+
+	/* size of the buffer pointed to by buffer_ptr */
+	uint32_t buffer_size;
+
+	/* selects the type of semantics to use for the general purpose buffer. it
+	 * tags (or discriminates) the union config member in mve_buffer_general
+	 */
+	uint16_t type;				  /* Extra data: */
+#define MVE_BUFFER_GENERAL_TYPE_BLOCK_CONFIGS (1) /* block_configs */
+#define MVE_BUFFER_GENERAL_TYPE_ENCODER_STATS (4) /* encoder_stats */
+
+	/* size of the mve_buffer_general config member */
+	uint16_t config_size;
+
+	/* pad to force 8-byte alignment */
+	uint32_t reserved;
+};
+
+/* The general purpose buffer consists of a header and a configuration. The
+ * header contains a pointer to a buffer whose format is described by the
+ * configuration. The type of configuration is indicated by the type value in
+ * the header. N.B. In use, the size of the config part of this structure is
+ * defined in the header and is not necessarily equal to that returned by the
+ * sizeof() operator. This allows a more size efficient communication between
+ * the host and firmware.
+ */
+struct mve_buffer_general {
+	struct mve_buffer_general_hdr header;
+
+	/* used to describe the configuration of the general purpose buffer data
+	 * pointed to be buffer_ptr
+	 */
+	union {
+		struct mve_buffer_general_block_configs block_configs;
+		struct mve_buffer_general_encoder_stats encoder_stats;
+	} config;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __FW_INCLUDE__MVE_PROTOCOL_DEF_H__ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/external/fw_v3/mve_protocol_def.h b/drivers/media/platform/spacemit/vpu_k3/external/fw_v3/mve_protocol_def.h
new file mode 100644
index 000000000000..ee5d3ff041e3
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/external/fw_v3/mve_protocol_def.h
@@ -0,0 +1,1579 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/*
+ * Copyright:
+ * ----------------------------------------------------------------------------
+ * This confidential and proprietary software may be used only as authorized
+ * by a licensing agreement from Arm Technology (China) Co., Ltd.
+ *      (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ * The entire notice above must be reproduced on all authorized copies and
+ * copies may only be made to the extent permitted by a licensing agreement
+ * from Arm Technology (China) Co., Ltd.
+ * ----------------------------------------------------------------------------
+ */
+#ifndef __FW_INCLUDE__MVE_PROTOCOL_DEF_H__
+#define __FW_INCLUDE__MVE_PROTOCOL_DEF_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdint.h>
+#endif
+
+/*****************************************************************************
+ *
+ * Communication protocol between the host/driver and the MVE firmware,
+ * the 'host interface'.
+ *
+ * MVE == LINLON Video Engine
+ *
+ * Protocol version 3.3
+ *
+ * Note: Message structs may be expanded in the future; the host should
+ * use the 'size' of the message to determine how many bytes to
+ * read from the message queue, rather than a sizeof(struct).
+ *
+ ****************************************************************************/
+
+/*****************************************************************************
+ *
+ * Virtual memory regions
+ *
+ * ..._ADDR_BEGIN gives the starting virtual address of the region,
+ * and ..._ADDR_END the (non-inclusive) ending address, such that
+ * the size of the region is obtained with the subtraction
+ * (..._ADDR_END - ..._ADDR_BEGIN).
+ *
+ ****************************************************************************/
+
+/* Memory region for first firmware instance */
+#define MVE_MEM_REGION_FW_INSTANCE0_ADDR_BEGIN (0x00000000u)
+#define MVE_MEM_REGION_FW_INSTANCE0_ADDR_END (0x000FFFFFu + 1)
+
+/* Memory regions for other firmware instances */
+#define MVE_MEM_REGION_FW_INSTANCE1_ADDR_BEGIN (0x01000000u)
+#define MVE_MEM_REGION_FW_INSTANCE1_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE1_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE2_ADDR_BEGIN (0x02000000u)
+#define MVE_MEM_REGION_FW_INSTANCE2_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE2_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE3_ADDR_BEGIN (0x03000000u)
+#define MVE_MEM_REGION_FW_INSTANCE3_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE3_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE4_ADDR_BEGIN (0x04000000u)
+#define MVE_MEM_REGION_FW_INSTANCE4_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE4_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE5_ADDR_BEGIN (0x05000000u)
+#define MVE_MEM_REGION_FW_INSTANCE5_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE5_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE6_ADDR_BEGIN (0x06000000u)
+#define MVE_MEM_REGION_FW_INSTANCE6_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE6_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+#define MVE_MEM_REGION_FW_INSTANCE7_ADDR_BEGIN (0x07000000u)
+#define MVE_MEM_REGION_FW_INSTANCE7_ADDR_END                                                                           \
+	(MVE_MEM_REGION_FW_INSTANCE7_ADDR_BEGIN + MVE_MEM_REGION_FW_INSTANCE0_ADDR_END)
+
+/*
+ * Areas for communication between host and MVE are placed in the interval
+ * 0x10079000 - 0x1007FFFF, see special defines further down.
+ */
+
+/* PROTECTED virtual memory region */
+#define MVE_MEM_REGION_PROTECTED_ADDR_BEGIN (0x20000000u)
+#define MVE_MEM_REGION_PROTECTED_ADDR_END (0x6FFFFFFFu + 1)
+
+/* FRAMEBUF virtual memory region */
+#define MVE_MEM_REGION_FRAMEBUF_ADDR_BEGIN (0x70000000u)
+#define MVE_MEM_REGION_FRAMEBUF_ADDR_END (0xEFFFFFFFu + 1)
+
+/* 0xF0000000 - 0xFFFFFFFF is used internally in MVE */
+
+/*****************************************************************************
+ *
+ * Communication queues between HOST/DRIVER and MVE
+ *
+ * Address for queue for messages in to MVE,
+ * one struct mve_comm_area_host located here
+ *
+ ****************************************************************************/
+
+#define MVE_COMM_MSG_INQ_ADDR (0x10079000u)
+
+/* Address for queue for messages out from MVE,
+ * one struct mve_comm_area_mve located here
+ */
+#define MVE_COMM_MSG_OUTQ_ADDR (0x1007A000u)
+
+/* Address for queue for input buffers in to MVE,
+ * one struct mve_comm_area_host located here
+ */
+#define MVE_COMM_BUF_INQ_ADDR (0x1007B000u)
+
+/* Address for queue for input buffers returned from MVE,
+ * one struct mve_comm_area_mve located here
+ */
+#define MVE_COMM_BUF_INRQ_ADDR (0x1007C000u)
+
+/* Address for queue for output buffers in to MVE,
+ * one struct mve_comm_area_host located here
+ */
+#define MVE_COMM_BUF_OUTQ_ADDR (0x1007D000u)
+
+/* Address for queue for output buffers returned from MVE,
+ * one struct mve_comm_area_mve located here
+ */
+#define MVE_COMM_BUF_OUTRQ_ADDR (0x1007E000u)
+
+/* One struct mve_rpc_communication_area located here */
+#define MVE_COMM_RPC_ADDR (0x1007F000u)
+
+/* Address for ram_print buffer in FW */
+#define MVE_FW_PRINT_RAM_ADDR (0x10100000u)
+#define MVE_FW_PRINT_RAM_SIZE (0x80000u)
+
+/* One page of memory (4 kB) is used for each queue,
+ * so maximum 1024 words, but need room for some counters as well,
+ * see structs mve_comm_area_mve and mve_comm_area_host below.
+ */
+#define MVE_COMM_QUEUE_SIZE_IN_WORDS 1020
+
+/* This is the part of the message area that is written by host. */
+struct mve_comm_area_host {
+	volatile uint16_t out_rpos;
+	volatile uint16_t in_wpos;
+	volatile uint32_t reserved[3];
+	/*
+	 * Queue of messages to MVE, each block of data prefixed with
+	 * a mve_msg_header
+	 */
+	volatile uint32_t in_data[MVE_COMM_QUEUE_SIZE_IN_WORDS];
+};
+
+/* This is the part of the message area that is written by MVE. */
+struct mve_comm_area_mve {
+	volatile uint16_t out_wpos;
+	volatile uint16_t in_rpos;
+	volatile uint32_t reserved[3];
+	/*
+	 * Queue of messages to host, each block of data prefixed with
+	 * a mve_msg_header
+	 */
+	volatile uint32_t out_data[MVE_COMM_QUEUE_SIZE_IN_WORDS];
+};
+
+#define MVE_RPC_AREA_SIZE_IN_WORDS 256
+#define MVE_RPC_DATA_SIZE_IN_WORDS (MVE_RPC_AREA_SIZE_IN_WORDS - 3)
+union mve_rpc_params {
+	volatile uint32_t data[MVE_RPC_DATA_SIZE_IN_WORDS];
+	struct {
+		char string[MVE_RPC_DATA_SIZE_IN_WORDS * 4];
+	} debug_print;
+	struct {
+		uint32_t size;
+		uint32_t max_size;
+		uint8_t region; /* Memory region selection */
+#define MVE_MEM_REGION_PROTECTED (0)
+#define MVE_MEM_REGION_OUTBUF (1)
+#define MVE_MEM_REGION_FRAMEBUF (MVE_MEM_REGION_OUTBUF)
+
+		/* The newly allocated memory must be placed
+		 * on (at least) a 2^(log2_alignment) boundary
+		 */
+		uint8_t log2_alignment;
+	} mem_alloc;
+	struct {
+		uint32_t ve_pointer;
+		uint32_t new_size;
+	} mem_resize;
+	struct {
+		uint32_t ve_pointer;
+	} mem_free;
+};
+
+struct mve_rpc_communication_area {
+	volatile uint32_t state;
+#define MVE_RPC_STATE_FREE (0)
+#define MVE_RPC_STATE_PARAM (1)
+#define MVE_RPC_STATE_RETURN (2)
+	volatile uint32_t call_id;
+#define MVE_RPC_FUNCTION_DEBUG_PRINTF (1)
+#define MVE_RPC_FUNCTION_MEM_ALLOC (2)
+#define MVE_RPC_FUNCTION_MEM_RESIZE (3)
+#define MVE_RPC_FUNCTION_MEM_FREE (4)
+	volatile uint32_t size;
+	union mve_rpc_params params;
+};
+
+struct mve_fw_ram_print_head_aera {
+	volatile uint32_t rd_cnt;
+	volatile uint32_t reserved0[15];
+
+	volatile uint32_t flag;
+	volatile uint32_t index;
+	volatile uint32_t wr_cnt;
+	volatile uint32_t reserved1[13];
+};
+
+/*********************************************************************
+ *
+ *   Message codes
+ *
+ *********************************************************************/
+
+/* Messages consist of one struct mve_msg_header, possibly followed
+ * by extra data.
+ */
+struct mve_msg_header {
+	uint16_t code;
+/* REQUESTs are messages from the
+ * host/driver to the firmware:               Code:     Extra data in message: */
+#define MVE_REQUEST_CODE_GO (1001)		  /* no extra data */
+#define MVE_REQUEST_CODE_STOP (1002)		  /* no extra data */
+#define MVE_REQUEST_CODE_INPUT_FLUSH (1003)	  /* no extra data */
+#define MVE_REQUEST_CODE_OUTPUT_FLUSH (1004)	  /* no extra data */
+#define MVE_REQUEST_CODE_SWITCH (1005)		  /* no extra data */
+#define MVE_REQUEST_CODE_PING (1006)		  /* no extra data */
+#define MVE_REQUEST_CODE_DUMP (1008)		  /* no extra data */
+#define MVE_REQUEST_CODE_JOB (1009)		  /* struct mve_request_job */
+#define MVE_REQUEST_CODE_SET_OPTION (1010)	  /* struct mve_request_set_option (variable size) */
+#define MVE_REQUEST_CODE_RELEASE_REF_FRAME (1011) /* struct mve_request_release_ref_frame */
+#define MVE_REQUEST_CODE_IDLE_ACK (1012)	  /* no extra data */
+#define MVE_REQUEST_CODE_DEBUG (1013)		  /* level: 0 for disable, refer to fw_log_level */
+/* RESPONSEs are messages from
+ * the firmware to the host: */
+#define MVE_RESPONSE_CODE_SWITCHED_IN (2001)	     /* struct mve_response_switched_in */
+#define MVE_RESPONSE_CODE_SWITCHED_OUT (2002)	     /* struct mve_response_switched_out */
+#define MVE_RESPONSE_CODE_SET_OPTION_CONFIRM (2003)  /* no extra data */
+#define MVE_RESPONSE_CODE_JOB_DEQUEUED (2004)	     /* struct mve_response_job_dequeued */
+#define MVE_RESPONSE_CODE_INPUT (2005)		     /* no extra data, but buffer placed in buffer queue */
+#define MVE_RESPONSE_CODE_OUTPUT (2006)		     /* no extra data, but buffer placed in buffer queue */
+#define MVE_RESPONSE_CODE_INPUT_FLUSHED (2007)	     /* no extra data */
+#define MVE_RESPONSE_CODE_OUTPUT_FLUSHED (2008)	     /* no extra data */
+#define MVE_RESPONSE_CODE_PONG (2009)		     /* no extra data */
+#define MVE_RESPONSE_CODE_ERROR (2010)		     /* struct mve_response_error */
+#define MVE_RESPONSE_CODE_STATE_CHANGE (2011)	     /* struct mve_response_state_change */
+#define MVE_RESPONSE_CODE_DUMP (2012)		     /* no extra data */
+#define MVE_RESPONSE_CODE_IDLE (2013)		     /* no extra data */
+#define MVE_RESPONSE_CODE_FRAME_ALLOC_PARAM (2014)   /* struct mve_response_frame_alloc_parameters */
+#define MVE_RESPONSE_CODE_SEQUENCE_PARAMETERS (2015) /* struct mve_response_sequence_parameters */
+#define MVE_RESPONSE_CODE_EVENT (2016)		     /* struct mve_response_event (variable size) */
+#define MVE_RESPONSE_CODE_SET_OPTION_FAIL (2017)     /* struct mve_response_set_option_failed */
+#define MVE_RESPONSE_CODE_REF_FRAME_UNUSED (2018)    /* struct mve_response_ref_frame_unused */
+#define MVE_RESPONSE_CODE_DEBUG (2019)		     /* no extra data */
+/* BUFFERs are sent from host to firmware,
+ * and then return at some time: */
+#define MVE_BUFFER_CODE_FRAME (3001)	 /* struct mve_buffer_frame */
+#define MVE_BUFFER_CODE_BITSTREAM (3002) /* struct mve_buffer_bitstream */
+#define MVE_BUFFER_CODE_PARAM (3003)	 /* struct mve_buffer_param */
+#define MVE_BUFFER_CODE_GENERAL (3004)	 /* struct mve_buffer_general */
+
+	uint16_t size; /* size in bytes of trailing data, 0 if none */
+};
+
+enum fw_log_level { FW_LOG_ERROR = 1, FW_LOG_WARNING, FW_LOG_INFO, FW_LOG_DEBUG, FW_LOG_VERBOSE, FW_LOG_MAX };
+
+/*********************************************************************
+ *
+ *   REQUESTs are messages from the host to the firmware
+ *
+ *   Some of the MVE_REQUEST_CODE_ codes are followed by one of the
+ *   structs below.
+ *
+ *********************************************************************/
+
+struct mve_request_job {
+	uint16_t cores;	 /* >= 1, number of cores to use, must match request to HW scheduler */
+	uint16_t frames; /* number of frames to process, zero means infinite */
+	uint32_t flags;	 /* can be zero */
+#define MVE_JOB_FLAG_DISABLE_BNDMGR (0x01)
+};
+
+struct mve_request_set_option {
+	uint32_t index;
+#define MVE_SET_OPT_INDEX_NALU_FORMAT (1)	   /* see arg, MVE_OPT_NALU_FORMAT_ */
+#define MVE_SET_OPT_INDEX_STREAM_ESCAPING (2)	   /* arg=1 to enable (default), arg=0 to disable */
+#define MVE_SET_OPT_INDEX_PROFILE_LEVEL (3)	   /* data.profile_level */
+#define MVE_SET_OPT_INDEX_HOST_PROTOCOL_PRINTS (4) /* arg=1 to enable, arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_PROFILING (5)		   /* arg=1 to enable, arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_DISABLE_FEATURES (6)	   /* see arg, MVE_OPT_DISABLE_FEATURE_ */
+#define MVE_SET_OPT_INDEX_IGNORE_STREAM_HEADERS                                                                        \
+	(7) /* decode, arg=1 to enable,                                                                                \
+	     * arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_FRAME_REORDERING                                                                             \
+	(8) /* decode, arg=1 to enable (default),                                                                      \
+	     * arg=0 to disable */
+#define MVE_SET_OPT_INDEX_INTBUF_SIZE                                                                                  \
+	(9)					/* decode, arg = suggested limit of intermediate                       \
+						 * buffer allocation */
+#define MVE_SET_OPT_INDEX_ENC_P_FRAMES (16)	/* encode, arg = nPFrames */
+#define MVE_SET_OPT_INDEX_ENC_B_FRAMES (17)	/* encode, arg = number of B frames */
+#define MVE_SET_OPT_INDEX_GOP_TYPE (18)		/* encode, see arg */
+#define MVE_SET_OPT_INDEX_INTRA_MB_REFRESH (19) /* encode, arg */
+#define MVE_SET_OPT_INDEX_ENC_CONSTR_IPRED (20) /* encode, arg = 0 or 1 */
+#define MVE_SET_OPT_INDEX_ENC_ENTROPY_SYNC (21) /* encode, arg = 0 or 1 */
+#define MVE_SET_OPT_INDEX_ENC_TEMPORAL_MVP (22) /* encode, arg = 0 or 1 */
+#define MVE_SET_OPT_INDEX_TILES (23)		/* encode, data.tiles */
+#define MVE_SET_OPT_INDEX_ENC_MIN_LUMA_CB_SIZE                                                                         \
+	(24)					   /* HEVC encode, arg = 8 or 16,                                      \
+						    * for sizes 8x8 or 16x16 */
+#define MVE_SET_OPT_INDEX_ENC_MB_TYPE_ENABLE (25)  /* encode, see arg */
+#define MVE_SET_OPT_INDEX_ENC_MB_TYPE_DISABLE (26) /* encode, see arg */
+#define MVE_SET_OPT_INDEX_ENC_H264_CABAC (27)	   /* encode, arg = 0 or 1, enabled by default */
+#define MVE_SET_OPT_INDEX_ENC_SLICE_SPACING                                                                            \
+	(28)					   /* encode, arg = suggested number of                                \
+						    * CTUs/macroblocks in a slice */
+#define MVE_SET_OPT_INDEX_ENC_VP9_PROB_UPDATE (30) /* VP9 encode, see arg */
+#define MVE_SET_OPT_INDEX_RESYNC_INTERVAL                                                                              \
+	(31)				     /* JPEG encode, arg = nRestartInterval                                    \
+					      * = nResynchMarkerSpacing */
+#define MVE_SET_OPT_INDEX_HUFFMAN_TABLE (32) /* JPEG encode, data.huffman_table */
+#define MVE_SET_OPT_INDEX_QUANT_TABLE (33)   /* JPEG encode, data.quant_table */
+#define MVE_SET_OPT_INDEX_ENC_EXPOSE_REF_FRAMES                                                                        \
+	(34) /* encode debug, arg = 0 or 1,                                                                            \
+	      * disabled by default */
+#define MVE_SET_OPT_INDEX_MBINFO_OUTPUT                                                                                \
+	(35)				       /* encode, arg=1 to enable,                                             \
+						* arg=0 to disable (default) */
+#define MVE_SET_OPT_INDEX_MV_SEARCH_RANGE (36) /* encode, data,motion_vector_search_range */
+#define MVE_SET_OPT_INDEX_ENC_STREAM_BITDEPTH                                                                          \
+	(38) /* encode, data.bitdepth, to set other bitdepth                                                           \
+	      * of encoded stream than of input frames */
+#define MVE_SET_OPT_INDEX_ENC_STREAM_CHROMA_FORMAT                                                                     \
+	(39) /* encode, arg, to set other chroma format of                                                             \
+	      * encoded stream than of input frames */
+#define MVE_SET_OPT_INDEX_ENC_RGB_TO_YUV_MODE                                                                          \
+	(40) /* encode, arg, select which way RGB is converted                                                         \
+	      * to YUV before encoding */
+#define MVE_SET_OPT_INDEX_ENC_BANDWIDTH_LIMIT                                                                          \
+	(41)						/* encode, arg, the maxium bandwidth limit defined             \
+							 * by host */
+#define MVE_SET_OPT_INDEX_WATCHDOG_TIMEOUT (42)		/* arg=timeout, arg=0 to disable */
+#define MVE_SET_OPT_INDEX_ENC_CABAC_INIT_IDC (43)	/* encode, arg; 0,1,2 for H264; 0,1 for HEVC */
+#define MVE_SET_OPT_INDEX_ENC_ADPTIVE_QUANTISATION (44) /* encode (h264 and hevc) */
+#define MVE_SET_OPT_INDEX_QP_DELTA_I_P (45)
+#define MVE_SET_OPT_INDEX_QP_DELTA_I_B_REF (46)
+#define MVE_SET_OPT_INDEX_QP_DELTA_I_B_NONREF (47)
+#define MVE_SET_OPT_INDEX_CB_QP_OFFSET (48)
+#define MVE_SET_OPT_INDEX_CR_QP_OFFSET (49)
+#define MVE_SET_OPT_INDEX_LAMBDA_SCALE (50)	 /* encode, data.lambda_scale */
+#define MVE_SET_OPT_INDEX_ENC_MAX_NUM_CORES (51) /* maximum number of cores */
+#define MVE_SET_OPT_INDEX_ENC_FIXED_QP (56)
+/* ARBITRARY_DOWNSCALE */
+#define MVE_SET_OPT_INDEX_DEC_DOWNSCALE (57) /* decode, set downscaled width and height */
+/* SAO_EVAL */
+#define MVE_SET_OPT_INDEX_ENC_SAO_PENALTY (58)
+#define MVE_SET_OPT_INDEX_ENC_SAO_EN (59) /* do not use now */
+#define MVE_SET_OPT_INDEX_ENC_SAO_LUMA_EN (60)
+#define MVE_SET_OPT_INDEX_ENC_SAO_CHROMA_EN (61)
+/* MVE_ENCODE_CROP_FEATURE */
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_LEFT (62)
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_RIGHT (63)
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_TOP (64)
+#define MVE_SET_OPT_INDEX_ENC_CROP_RARAM_BOTTOM (65)
+/* LONG_TERM_REFERENCE */
+#define MVE_SET_OPT_INDEX_ENC_LTR_MODE (66)
+#define MVE_SET_OPT_INDEX_ENC_LTR_PERIOD (67)
+#define MVE_SET_OPT_INDEX_DEC_DOWNSCALE_POS_MODE (69)
+
+	union {
+		uint32_t arg; /* Most options only need a uint32_t as argument */
+/* For option MVE_SET_OPT_INDEX_NALU_FORMAT, arg should
+ * be one of these: */
+#define MVE_OPT_NALU_FORMAT_START_CODES (1)
+#define MVE_OPT_NALU_FORMAT_ONE_NALU_PER_BUFFER (2)
+#define MVE_OPT_NALU_FORMAT_ONE_BYTE_LENGTH_FIELD (4)
+#define MVE_OPT_NALU_FORMAT_TWO_BYTE_LENGTH_FIELD (8)
+#define MVE_OPT_NALU_FORMAT_FOUR_BYTE_LENGTH_FIELD (16)
+#define MVE_OPT_NALU_FORMAT_MBINFO (32) /* only used for debugging */
+/* For option MVE_SET_OPT_INDEX_GOP_TYPE, arg should
+ * be one of these: */
+#define MVE_OPT_GOP_TYPE_BIDIRECTIONAL (1)
+#define MVE_OPT_GOP_TYPE_LOW_DELAY (2)
+#define MVE_OPT_GOP_TYPE_PYRAMID (3)
+/* For option MVE_SET_OPT_INDEX_ENC_VP9_PROB_UPDATE,
+ * arg should be one of these: */
+#define MVE_OPT_VP9_PROB_UPDATE_DISABLED (0)
+#define MVE_OPT_VP9_PROB_UPDATE_IMPLICIT (1)
+#define MVE_OPT_VP9_PROB_UPDATE_EXPLICIT (2)
+/* For option MVE_SET_OPT_INDEX_DISABLE_FEATURES, arg
+ * should be a bitmask with features to disable: */
+#define MVE_OPT_DISABLE_FEATURE_AFBC_COMP (0x00000001)	/* VDMA AFBC Compression   */
+#define MVE_OPT_DISABLE_FEATURE_REF_CACHE (0x00000002)	/* REF caching             */
+#define MVE_OPT_DISABLE_FEATURE_DEBLOCK (0x00000004)	/* Deblocking              */
+#define MVE_OPT_DISABLE_FEATURE_SAO (0x00000008)	/* SAO                     */
+#define MVE_OPT_DISABLE_FEATURE_PIC_OUTPUT (0x00000020) /* Picture Output Removal  */
+#define MVE_OPT_DISABLE_FEATURE_PIPE (0x00000040)	/* Pipe (i.e. parser-only) */
+#define MVE_OPT_DISABLE_FEATURE_SLEEP                                                                                  \
+	(0x00000080) /* Clock gating                                                                                   \
+		      * (SOC_SYSCTRL.SLEEP bit) */
+#define MVE_OPT_DISABLE_FEATURE_AFBC_LEGACY_REF                                                                        \
+	(0x00000100) /* Enables tiled AFBC format in                                                                   \
+		      * reference buffers. Ignored                                                                     \
+		      * for decode AFBC output */
+#define MVE_OPT_DISABLE_FEATURE_REF_PICS                                                                               \
+	(0x00000400)					  /* Forces use of static 16x16                                \
+							   * reference pics */
+#define MVE_OPT_DISABLE_FEATURE_CHNG_RECT_WA (0x00000800) /* Disables workaround */
+#define MVE_OPT_DISABLE_FEATURE_REFSZ_LIMIT (0x00001000)  /* Disable REFSZ bw limit */
+/* For options MVE_SET_OPT_INDEX_ENC_MB_TYPE_ENABLE
+ * and MVE_SET_OPT_INDEX_ENC_MB_TYPE_DISABLE, arg
+ * should be a bitmask of MVE_MBTYPEs: */
+#define MVE_MBTYPE_4x4 (0x00000001)    /* 4x4 inter     */
+#define MVE_MBTYPE_4x8 (0x00000002)    /* 4x8 inter     */
+#define MVE_MBTYPE_8x4 (0x00000004)    /* 8x4 inter     */
+#define MVE_MBTYPE_8x8 (0x00000008)    /* 8x8 inter     */
+#define MVE_MBTYPE_8x16 (0x00000010)   /* 8x16 inter    */
+#define MVE_MBTYPE_16x8 (0x00000020)   /* 16x8 inter    */
+#define MVE_MBTYPE_16x16 (0x00000040)  /* 16x16 inter   */
+#define MVE_MBTYPE_PSKIP (0x00000080)  /* P Skip inter  */
+#define MVE_MBTYPE_I4x4 (0x00000100)   /* 4x4 intra     */
+#define MVE_MBTYPE_I8x8 (0x00000200)   /* 8x8 intra     */
+#define MVE_MBTYPE_I16x16 (0x00000400) /* 16x16 intra   */
+#define MVE_MBTYPE_I32x32 (0x00000800) /* 32x32 intra   */
+#define MVE_MBTYPE_16x32 (0x00001000)  /* 16x32 inter   */
+#define MVE_MBTYPE_32x16 (0x00002000)  /* 32x16 inter   */
+#define MVE_MBTYPE_32x32 (0x00004000)  /* 32x32 inter   */
+/* For option MVE_SET_OPT_INDEX_ENC_RGB_TO_YUV_MODE,
+ * arg should be one of these: */
+#define MVE_OPT_RGB_TO_YUV_BT601_STUDIO (0)
+#define MVE_OPT_RGB_TO_YUV_BT601_FULL (1)
+#define MVE_OPT_RGB_TO_YUV_BT709_STUDIO (2)
+#define MVE_OPT_RGB_TO_YUV_BT709_FULL (3)
+		struct {
+			uint16_t profile;
+/* AVC/H.264 profiles */
+#define MVE_OPT_PROFILE_H264_BASELINE (1)
+#define MVE_OPT_PROFILE_H264_MAIN (2)
+#define MVE_OPT_PROFILE_H264_HIGH (3)
+#define MVE_OPT_PROFILE_H264_HIGH_10 (4)
+/* HEVC/H.265 profiles */
+#define MVE_OPT_PROFILE_H265_MAIN (1)
+#define MVE_OPT_PROFILE_H265_MAIN_STILL (2)
+#define MVE_OPT_PROFILE_H265_MAIN_INTRA (3)
+#define MVE_OPT_PROFILE_H265_MAIN_10 (4)
+/* VC-1 profiles */
+#define MVE_OPT_PROFILE_VC1_SIMPLE (1)
+#define MVE_OPT_PROFILE_VC1_MAIN (2)
+#define MVE_OPT_PROFILE_VC1_ADVANCED (3)
+/* VP8 profiles */
+#define MVE_OPT_PROFILE_VP8_MAIN (1)
+			uint16_t level;
+/* AVC/H.264 levels */
+#define MVE_OPT_LEVEL_H264_1 (1)
+#define MVE_OPT_LEVEL_H264_1b (2)
+#define MVE_OPT_LEVEL_H264_11 (3)
+#define MVE_OPT_LEVEL_H264_12 (4)
+#define MVE_OPT_LEVEL_H264_13 (5)
+#define MVE_OPT_LEVEL_H264_2 (6)
+#define MVE_OPT_LEVEL_H264_21 (7)
+#define MVE_OPT_LEVEL_H264_22 (8)
+#define MVE_OPT_LEVEL_H264_3 (9)
+#define MVE_OPT_LEVEL_H264_31 (10)
+#define MVE_OPT_LEVEL_H264_32 (11)
+#define MVE_OPT_LEVEL_H264_4 (12)
+#define MVE_OPT_LEVEL_H264_41 (13)
+#define MVE_OPT_LEVEL_H264_42 (14)
+#define MVE_OPT_LEVEL_H264_5 (15)
+#define MVE_OPT_LEVEL_H264_51 (16)
+#define MVE_OPT_LEVEL_H264_52 (17)
+#define MVE_OPT_LEVEL_H264_6 (18)
+#define MVE_OPT_LEVEL_H264_61 (19)
+#define MVE_OPT_LEVEL_H264_62 (20)
+#define MVE_OPT_LEVEL_H264_USER_SUPPLIED_BASE (32)
+/* The value (MVE_OPT_LEVEL_H264_USER_SUPPLIED_BASE + level_idc) encodes a user
+ * supplied level_idc value in the range 0 to 255 inclusive. If the host supplies a level_idc
+ * value by this method then the encoder will encode this level_idc value in the bitstream
+ * without checking the validity of the level_idc value
+ */
+#define MVE_OPT_LEVEL_H264_USER_SUPPLIED_MAX (MVE_OPT_LEVEL_H264_USER_SUPPLIED_BASE + 255)
+/* HEVC/H.265 levels */
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_1 (1)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_1 (2)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_2 (3)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_2 (4)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_21 (5)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_21 (6)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_3 (7)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_3 (8)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_31 (9)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_31 (10)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_4 (11)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_4 (12)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_41 (13)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_41 (14)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_5 (15)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_5 (16)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_51 (17)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_51 (18)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_52 (19)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_52 (20)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_6 (21)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_6 (22)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_61 (23)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_61 (24)
+#define MVE_OPT_LEVEL_H265_MAIN_TIER_62 (25)
+#define MVE_OPT_LEVEL_H265_HIGH_TIER_62 (26)
+		} profile_level;
+		struct {
+			int32_t mv_search_range_x;
+			int32_t mv_search_range_y;
+		} motion_vector_search_range;
+		struct {
+			uint32_t type;
+#define MVE_OPT_HUFFMAN_TABLE_DC_LUMA (1)
+#define MVE_OPT_HUFFMAN_TABLE_AC_LUMA (2)
+#define MVE_OPT_HUFFMAN_TABLE_DC_CHROMA (3)
+#define MVE_OPT_HUFFMAN_TABLE_AC_CHROMA (4)
+			uint8_t number_of_huffman_of_code_length[16];
+			uint8_t table[162]; /* 12 are used for DC, 162 for AC */
+		} huffman_table;
+		struct {
+			uint32_t type;
+#define MVE_OPT_QUANT_TABLE_LUMA (1)
+#define MVE_OPT_QUANT_TABLE_CHROMA (2)
+			uint8_t matrix[64];
+		} quant_table;
+		struct {
+			/* For HEVC, tile_cols must be zero. For VP9, tile_rows
+			 * and tile_cols must be powers of 2. */
+			uint16_t tile_rows;
+			uint16_t tile_cols;
+		} tiles;
+		struct {
+			uint16_t luma_bitdepth;
+			uint16_t chroma_bitdepth;
+		} bitdepth;
+		struct {
+			/* Scale factors, and their square roots, for the lambda
+			 * coefficients used by the encoder, in unsigned Q8 fixed-point
+			 * format. Default (no scaling) is 1.0 (so 0x0100 in hex).
+			 */
+			uint16_t lambda_scale_i_q8;
+			uint16_t lambda_scale_sqrt_i_q8;
+			uint16_t lambda_scale_p_q8;
+			uint16_t lambda_scale_sqrt_p_q8;
+			uint16_t lambda_scale_b_ref_q8;
+			uint16_t lambda_scale_sqrt_b_ref_q8;
+			uint16_t lambda_scale_b_nonref_q8;
+			uint16_t lambda_scale_sqrt_b_nonref_q8;
+		} lambda_scale;
+		/* ARBITRARY_DOWNSCALE */
+		struct {
+			uint16_t width;
+			uint16_t height;
+		} downscaled_frame;
+		struct {
+			uint32_t mode;
+		} dsl_pos;
+	} data;
+};
+
+struct mve_request_release_ref_frame {
+	/* Decode only: For a frame buffer that MVE has returned
+	 * marked as _REF_FRAME, the host can send this message
+	 * to ask the MVE to release the buffer as soon as it is
+	 * no longer used as reference anymore. (Otherwise, in
+	 * normal operation, the host would re-enqueue the buffer
+	 * to the MVE when it has been displayed and can be over-
+	 * written with a new frame.)
+	 *
+	 * Note: When a frame is no longer used as reference depends
+	 * on the stream being decoded, and there is no way to
+	 * guarantee a short response time, the response may not
+	 * come until the end of the stream.
+	 */
+	uint32_t buffer_address;
+};
+
+/*********************************************************************
+ *
+ *   RESPONSEs are messages from the host to the firmware
+ *
+ *   Some of the MVE_RESPONSE_CODE_ codes are followed by one of the
+ *   structs below.
+ *
+ *********************************************************************/
+
+/* Sent when firmware has booted.
+ */
+struct mve_response_switched_in {
+	uint32_t core;
+};
+
+/* Sent when last core in a session has switched out.
+ */
+struct mve_response_switched_out {
+	uint32_t core;
+	uint32_t reason;
+	uint32_t sub_reason;
+};
+
+/* Response confirming state transition after either GO or STOP
+ * command from host.
+ */
+struct mve_response_state_change {
+	uint32_t new_state;
+#define MVE_STATE_STOPPED (0)
+#define MVE_STATE_RUNNING (2)
+};
+
+/* Message sent when the all cores in the session have dequeued a
+ * job from the firmware job queue.
+ */
+struct mve_response_job_dequeued {
+	uint32_t valid_job;
+};
+
+/* Fatal error message from firmware, if sent then no further
+ * operation is possible.
+ */
+struct mve_response_error {
+	uint32_t error_code;
+#define MVE_ERROR_ABORT (1)
+#define MVE_ERROR_OUT_OF_MEMORY (2)
+#define MVE_ERROR_ASSERT (3)
+#define MVE_ERROR_UNSUPPORTED (4)
+#define MVE_ERROR_INVALID_BUFFER (6)
+#define MVE_ERROR_INVALID_STATE (8)
+#define MVE_ERROR_WATCHDOG (9)
+
+#define MVE_MAX_ERROR_MESSAGE_SIZE (128)
+	char message[MVE_MAX_ERROR_MESSAGE_SIZE];
+};
+
+/* When a set-option succeeds, a confirmation message is
+ * sent, including the index-code for that particular option.
+ */
+struct mve_response_set_option_confirm {
+	uint32_t index; /* Same as 'index' in struct mve_request_set_option */
+};
+
+/* If a set-option request fails, this message is returned.
+ * This is not a fatal error. The set-option had no effect,
+ * and the session is still alive.
+ * For example, trying to set an option with a too large
+ * or small parameter would result in this message.
+ * The included text string is meant for development and
+ * debugging purposes only.
+ * (When a set-option succeeds the set-option-confirm
+ * message code is sent instead.)
+ */
+struct mve_response_set_option_fail {
+	uint32_t index; /* Same as 'index' in struct mve_request_set_option */
+	char message[MVE_MAX_ERROR_MESSAGE_SIZE];
+};
+
+/* Decode only: This message is sent from MVE to the host so that it can
+ * allocate large enough output buffers. Output buffers that are to small
+ * will be returned to the host marked as 'rejected'.
+ */
+struct mve_response_frame_alloc_parameters {
+	/* Please note that the below information is a hint
+	 * for what buffers to allocate, it does not say
+	 * what actual resolution an output picture has.
+	 */
+
+	/* To use if allocating PLANAR YUV output buffers: */
+	uint16_t planar_alloc_frame_width;
+	uint16_t planar_alloc_frame_height;
+
+	/* To use if allocating AFBC output buffers
+	 * (if interlace, each field needs this size):
+	 */
+	uint32_t afbc_alloc_bytes;
+
+	/* For situations where downscaled AFBC is supported,
+	 * this number of bytes is needed for the downscaled frame.
+	 */
+	uint32_t afbc_alloc_bytes_downscaled;
+
+	/* When the host allocates an AFBC frame buffer, it should normally set
+	 * the the afbc_width_in_superblocks to be at least this recommended value.
+	 * Buffers with smaller values are likely to be returned rejected by the MVE.
+	 * See also comments above for afbc_alloc_bytes and
+	 * afbc_alloc_bytes_downscaled, they describe the situations where the
+	 * different values are used.
+	 */
+	uint16_t afbc_width_in_superblocks;
+	uint16_t afbc_width_in_superblocks_downscaled;
+
+	/* For PLANAR YUV output, every plane's address need to be adjusted to get
+	 * optimal AXI bursts when the pixel data is written, the values below may
+	 * be used to calculate address offsets.
+	 */
+	uint16_t cropx;
+	uint16_t cropy;
+
+	uint32_t mbinfo_alloc_bytes; /* Only for debugging */
+
+	/* downscaled frame width/height for decode */
+	/* ARBITRARY_DOWNSCALE */
+	uint16_t dsl_frame_width;
+	uint16_t dsl_frame_height;
+	uint16_t dsl_pos_mode;
+	uint8_t ctu_size; /* EXPORT_SEQ_INFO */
+};
+
+/* Decode only: This message is sent from MVE to the host so that it can
+ * allocate suitable output buffers. The needed size of the buffer is sent
+ * in a separate message (above).
+ * When MVE sends the message below, it enters a waiting-state and will not
+ * make any progress until the host sends an output-flush command, upon
+ * which MVE will return all output buffers, followed by a message saying
+ * that the output has been flushed. Only then should the host start
+ * enqueueing new output buffers.
+ */
+struct mve_response_sequence_parameters {
+	/* Other stream parameters affecting buffer allocation,
+	 * any change in these values will trigger a flush.
+	 */
+	uint8_t interlace; /* 0 or 1 */
+	uint8_t chroma_format;
+#define MVE_CHROMA_FORMAT_MONO (0x0)
+#define MVE_CHROMA_FORMAT_420 (0x1)
+#define MVE_CHROMA_FORMAT_422 (0x2)
+#define MVE_CHROMA_FORMAT_440 (0x3)
+#define MVE_CHROMA_FORMAT_ARGB (0x4)
+	uint8_t bitdepth_luma;	       /* 8, 9 or 10 */
+	uint8_t bitdepth_chroma;       /* 8, 9 or 10 */
+	uint8_t num_buffers_planar;    /* number of planar buffers needed */
+	uint8_t num_buffers_afbc;      /* number of AFBC buffers needed, for
+					* AFBC output more buffers are needed
+					* (for planar output, the firmware
+					* will allocate extra memory via RPC)
+					*/
+	uint8_t range_mapping_enabled; /* VC-1 AP specific feature, if enabled
+					* then AFBC buffers may need special
+					* filtering before they can be
+					* displayed correctly. If the host is
+					* not able to do that, then planar output
+					* should be used, for which MVE
+					* automatically performs the filtering.
+					*/
+	uint8_t reserved0;
+};
+
+struct mve_response_ref_frame_unused {
+	/* Decode only: If requested by the host with the message
+	 * MVE_REQUEST_CODE_RELEASE_REF_FRAME, the MVE will respond
+	 * with this message when (if ever) the buffer is no longer
+	 * used.
+	 */
+	uint32_t unused_buffer_address;
+};
+
+/* This message is only for debugging and performance profiling.
+ * Is sent by the firmware if the corresponding options is enabled.
+ */
+struct mve_event_processed {
+	uint8_t pic_format;
+	uint8_t qp;
+	uint8_t pad0;
+	uint8_t pad1;
+	uint32_t parse_start_time; /* Timestamp, absolute time */
+	uint32_t parse_end_time;   /* Timestamp, absolute time */
+	uint32_t parse_idle_time;  /* Definition of idle here is waiting for in/out buffers or available RAM */
+
+	uint32_t pipe_start_time; /* Timestamp */
+	uint32_t pipe_end_time;	  /* Timestamp, end-start = process time. Idle time while in a frame is
+				   * not measured. */
+	uint32_t pipe_idle_time;  /* Always 0 in decode, */
+
+	uint32_t parser_coreid; /* Core used to parse this frame */
+	uint32_t pipe_coreid;	/* Core used to pipe this frame */
+
+	uint32_t bitstream_bits; /* Number of bitstream bits used for this frame. */
+
+	uint32_t intermediate_buffer_size; /* Size of intermediate (mbinfo/residuals) buffer after this frame was
+					    * parsed. */
+	uint32_t total_memory_allocated;   /* after the frame was parsed. Including reference frames. */
+
+	uint32_t bus_read_bytes;  /* bus read bytes */
+	uint32_t bus_write_bytes; /* bus written bytes */
+
+	uint32_t afbc_bytes; /* afbc data transferred */
+
+	uint32_t slice0_end_time;   /* Timestamp, absolute time */
+	uint32_t stream_start_time; /* Timestamp, absolute stream start time */
+	uint32_t stream_open_time;  /* Timestamp, absolute stream open time */
+};
+
+/* This message is only for debugging, is sent by the
+ * firmware if the corresponding option is enabled.
+ */
+struct mve_event_ref_frame {
+	uint32_t ref_addr;	 /* MVE virtual address of AFBC reference frame */
+	uint32_t ref_width;	 /* Width of display area in luma pixels        */
+	uint32_t ref_height;	 /* Height of display area in luma pixels       */
+	uint32_t ref_mb_width;	 /* Width in macroblocks                        */
+	uint32_t ref_mb_height;	 /* Height in macroblocks                       */
+	uint32_t ref_left_crop;	 /* Left crop in luma pixels                    */
+	uint32_t ref_top_crop;	 /* Top crop in luma pixels                     */
+	uint32_t ref_frame_size; /* Total AFBC frame size in bytes              */
+	uint32_t ref_display_order;
+	uint16_t bit_width;	/* bit width of the YUV either 8 or 10 */
+	uint16_t tiled_headers; /* AFBC format is tiled */
+};
+
+/* This message is only for debugging, is sent by the firmware if event tracing
+ * is enabled.
+ */
+struct mve_event_trace_buffers {
+	uint16_t reserved;
+	uint8_t num_cores;
+	uint8_t rasc_mask;
+#define MVE_MAX_TRACE_BUFFERS 40
+	/* this array will contain one buffer per rasc in rasc_mask per num_core */
+	struct {
+		uint32_t rasc_addr; /* rasc address of the buffer */
+		uint32_t size;	    /* size of the buffer in bytes */
+	} buffers[MVE_MAX_TRACE_BUFFERS];
+};
+
+/* 'Events' are informative messages, the host is not required to react in
+ * any particular way.
+ */
+struct mve_response_event {
+	uint32_t event_code;
+#define MVE_EVENT_ERROR_STREAM_CORRUPT (1)	 /* message, text string */
+#define MVE_EVENT_ERROR_STREAM_NOT_SUPPORTED (2) /* message, text string */
+#define MVE_EVENT_PROCESSED (3)			 /* struct mve_event_processed */
+#define MVE_EVENT_REF_FRAME (4)			 /* struct mve_event_ref_frame */
+#define MVE_EVENT_TRACE_BUFFERS (5)		 /* struct mve_event_trace_buffers */
+	union {
+		struct mve_event_processed event_processed;
+		struct mve_event_ref_frame event_ref_frame;
+		struct mve_event_trace_buffers event_trace_buffers;
+		char message[MVE_MAX_ERROR_MESSAGE_SIZE];
+	} event_data;
+} __attribute__((packed));
+
+/*********************************************************************
+ *
+ *   BUFFERs are sent both ways, from host to firmware and back again
+ *
+ *   Each MVE_BUFFER_CODE_ code is followed by one of the structs
+ *   below.
+ *
+ *********************************************************************/
+
+/* Flags in mve_buffer_frame::frame_flags:
+ *                                     Set by whom?    Meaning:
+ *                                     DECODE: ENCODE:
+ * MVE_BUFFER_FRAME_FLAG_INTERLACE      host    -      Buffer is interlaced (both top and
+ *                                                     bottom fields are allocated)
+ * MVE_BUFFER_FRAME_FLAG_BOT_FIRST      fw      -      Bottom field should be displayed
+ *                                                     first (only if interlaced)
+ * MVE_BUFFER_FRAME_FLAG_TOP_PRESENT    fw      host   Top field present (or full frame if
+ *                                                     not interlaced)
+ * MVE_BUFFER_FRAME_FLAG_BOT_PRESENT    fw      -      Bottom present (only if interlaced)
+ *
+ * MVE_BUFFER_FRAME_FLAG_ROTATION_*     host    host   Decode: MVE will rotate the output frame
+ *                                                             according to this setting.
+ *                                                     Encode: MVE will rotate the input frame
+ *                                                             according to this setting before
+ *                                                             encoding them.
+ * MVE_BUFFER_FRAME_FLAG_SCALING_MASK   host    -      Output pictures should be downscaled
+ *
+ * MVE_BUFFER_FRAME_FLAG_MIRROR_*       -       host   Input frame should be mirrored before encoding
+ *
+ * MVE_BUFFER_FRAME_FLAG_REJECTED       fw      -      Buffer was too small, host should re-allocate
+ *
+ * MVE_BUFFER_FRAME_FLAG_CORRUPT        fw      -      Frame contains visual corruption
+ *
+ * MVE_BUFFER_FRAME_FLAG_DECODE_ONLY    fw      -      Frame should not be displayed
+ *
+ * MVE_BUFFER_FRAME_FLAG_REF_FRAME      fw      -      Frame is used by MVE as reference, host must
+ *                                                     not change, just re-enqueue when displayed
+ * MVE_BUFFER_FRAME_FLAG_EOS            fw      host   This is the last frame in the stream.
+ */
+
+/* mve_buffer_frame_planar stores uncompressed YUV pictures.
+ *       ________________________________________
+ *      |  ^                          |          |  ^
+ *      |<-:--visible_frame_width---->|          |  :
+ *      |  :                          |          |  :
+ *      |  :                          |          |  :
+ *      | visible_frame_height        |          | max_frame_height
+ *      |  :                          |          |  :
+ *      |  :                          |          |  :
+ *      |__v__________________________|          |  :
+ *      |                                        |  :
+ *      |<-------------max_frame_width---------->|  :
+ *      |________________________________________|  v
+ *
+ */
+struct mve_buffer_frame_planar {
+	/* Y,Cb,Cr top field */
+	uint32_t plane_top[3];
+
+	/* Y,Cb,Cr bottom field (interlace only) */
+	uint32_t plane_bot[3];
+
+	/* Stride between rows, in bytes */
+	int32_t stride[3];
+
+	/* Size of largest frame allowed to put in this buffer */
+	uint16_t max_frame_width;
+	uint16_t max_frame_height;
+};
+
+/* mve_buffer_frame_afbc stores AFBC compressed content that is also used
+ * as the reference frame. Out of loop processing (crop, rotation,
+ * range reduction) must be supported by the user of this buffer and
+ * the parameters are signaled within the buffer descriptor below.
+ *       ________________________________________
+ *      |         ^                              |
+ *      |       cropy                            |
+ *      |         v_____________________________ |
+ *      |<-cropx->|  ^                          ||
+ *      |         |<-:--visible_frame_width---->||
+ *      |         |  :                          ||
+ *      |         |  :                          ||
+ *      |         | visible_frame_height        ||
+ *      |         |  :                          ||
+ *      |         |  :                          ||
+ *      |         |__v__________________________||
+ *      |________________________________________|
+ *
+ *       <----- superblock_width --------------->
+ *              * afbc_width_in_superblocks
+ *
+ * Note that the sizes and cropping values need not be multiples of 16.
+ *
+ * For interlaced streams, the values refer to a full frame,
+ * while the output is actually separated into fields. Thus for fields,
+ * cropy and visible_frame_height should be divided by two.
+ *
+ * For dual-downscaled AFBC output (not supported for interlace),
+ * then the cropx, cropy, visible_frame_width and visible_frame_height
+ * should be divided by two for the downscaled plane.
+ */
+struct mve_buffer_frame_afbc {
+	uint32_t plane[2];				/* Addresses for up to two AFBC planes:
+							 * Top and bottom fields for interlace,
+							 * or standard and optional downscaled output. */
+	uint32_t alloc_bytes[2];			/* Size of allocation for each plane */
+	uint16_t cropx;					/* Luma x crop */
+	uint16_t cropy;					/* Luma y crop */
+	uint16_t afbc_width_in_superblocks[2];		/* Width of AFBC frame buffer, in units
+							 * of superblock width (32 or 16).
+							 * If dual-downscaled output is chosen,
+							 * this width can be different for the
+							 * two planes.
+							 * For first plane:
+							 * (cropx + frame_width)
+							 *      <= superblock_width  * afbc_width...
+							 */
+	uint32_t afbc_params;				/* AFBC parameters */
+#define MVE_BUFFER_FRAME_AFBC_TILED_BODY (0x00000001)	/* Output body blocks should be tiled */
+#define MVE_BUFFER_FRAME_AFBC_TILED_HEADER (0x00000002) /* Output headers should be tiled */
+#define MVE_BUFFER_FRAME_AFBC_32X8_SUPERBLOCK                                                                          \
+	(0x00000004) /* Super block is 32x8, default is 16x16,                                                         \
+		      * (only supported as input for encode) */
+#define MVE_BUFFER_FRAME_AFBC_DN_FORCE_8BIT                                                                            \
+	(0x00000008) /* For downscaled AFBC plane: It shall                                                            \
+		      * be 8-bit, even if full-scale is 10-bit */
+#define MVE_BUFFER_FRAME_AFBC_DN_FORCE_420                                                                             \
+	(0x00000010) /* For downscaled AFBC plane: It shall                                                            \
+		      * be 4:2:0, even if full-scale is 4:2:2 */
+#define MVE_BUFFER_FRAME_AFBC_STRIDE_SET_BY_MVE                                                                        \
+	(0x00000020) /* Decode only: By default, the host should                                                       \
+			set the afbc_width_in_superblocks. If the                                                      \
+			value is zero, or if this bit is set, then                                                     \
+			the MVE sets an appropriate value. */
+#define MVE_BUFFER_FRAME_AFBC_BLOCK_SPLIT                                                                              \
+	(0x00000040) /* For Superblock layout, block_split mode                                                        \
+			should be enabled*/
+};
+
+/*
+ * The FRAME buffer stores the common information for PLANAR and AFBC buffers,
+ * and a union of PLANAR and AFBC specific information.
+ */
+struct mve_buffer_frame {
+	/* For identification of the buffer, this is not changed by
+	 * the firmware. */
+	uint64_t host_handle;
+
+	/* For matching input buffer with output buffers, the firmware
+	 * copies these values between frame buffers and bitstream buffers. */
+	uint64_t user_data_tag;
+
+	/* Frame buffer flags, see commentary above */
+	uint32_t frame_flags;
+#define MVE_BUFFER_FRAME_FLAG_INTERLACE (0x00000001)
+#define MVE_BUFFER_FRAME_FLAG_BOT_FIRST (0x00000002)
+#define MVE_BUFFER_FRAME_FLAG_TOP_PRESENT (0x00000004)
+#define MVE_BUFFER_FRAME_FLAG_BOT_PRESENT (0x00000008)
+#define MVE_BUFFER_FRAME_FLAG_ROTATION_90 (0x00000010)
+#define MVE_BUFFER_FRAME_FLAG_ROTATION_180 (0x00000020)
+#define MVE_BUFFER_FRAME_FLAG_ROTATION_270 (0x00000030)
+#define MVE_BUFFER_FRAME_FLAG_SCALING_MASK (0x000000C0)
+#define MVE_BUFFER_FRAME_FLAG_MIRROR_HORI (0x00000100)
+#define MVE_BUFFER_FRAME_FLAG_MIRROR_VERT (0x00000200)
+#define MVE_BUFFER_FRAME_FLAG_REJECTED (0x00001000)
+#define MVE_BUFFER_FRAME_FLAG_CORRUPT (0x00002000)
+#define MVE_BUFFER_FRAME_FLAG_DECODE_ONLY (0x00004000)
+#define MVE_BUFFER_FRAME_FLAG_REF_FRAME (0x00008000)
+#define MVE_BUFFER_FRAME_FLAG_EOS (0x00010000)
+/*ARBITRARY_DOWNSCALE*/
+#define MVE_BUFFER_FRAME_FLAG_SCALING_MASKX (0xFF000000) // 8bit
+#define MVE_BUFFER_FRAME_FLAG_SCALING_MASKY (0x00FE0000) // 7bit
+
+	/* Height (in luma samples) of visible part of frame,
+	 * may be smaller than allocated frame size. */
+	uint16_t visible_frame_height;
+
+	/* Width (in luma samples) of visible part of frame,
+	 * may be smaller than allocated frame size. */
+	uint16_t visible_frame_width;
+
+	/* Color format of buffer */
+	uint16_t format;
+/* format bitfield: */
+#define MVE_FORMAT_BF_C (0)  /* 3 bits, chroma subsampling   */
+#define MVE_FORMAT_BF_B (4)  /* 4 bits, max bitdepth minus 8 */
+#define MVE_FORMAT_BF_N (8)  /* 2 bits, number of planes     */
+#define MVE_FORMAT_BF_V (12) /* 2 bits, format variant       */
+#define MVE_FORMAT_BF_A (15) /* 1 bit,  AFBC bit             */
+/* formats: */
+#define MVE_FORMAT_YUV420_AFBC_8                                                                                       \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV420_AFBC_10                                                                                      \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV422_AFBC_8                                                                                       \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV422_AFBC_10                                                                                      \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_A))
+
+#define MVE_FORMAT_YUV420_I420                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (3 << MVE_FORMAT_BF_N) |          \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_I420_10                                                                                      \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (3 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_NV12                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (2 << MVE_FORMAT_BF_N) |          \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_NV21                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (2 << MVE_FORMAT_BF_N) |          \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_P010                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((16 - 8) << MVE_FORMAT_BF_B) | (2 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_Y0L2                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV420_AQB1                                                                                         \
+	((MVE_CHROMA_FORMAT_420 << MVE_FORMAT_BF_C) | ((10 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV422_YUY2                                                                                         \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |          \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV422_UYVY                                                                                         \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |          \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_YUV422_Y210                                                                                         \
+	((MVE_CHROMA_FORMAT_422 << MVE_FORMAT_BF_C) | ((16 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_RGBA_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (0 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_BGRA_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (1 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_ARGB_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (2 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_ABGR_8888                                                                                           \
+	((MVE_CHROMA_FORMAT_ARGB << MVE_FORMAT_BF_C) | ((8 - 8) << MVE_FORMAT_BF_B) | (1 << MVE_FORMAT_BF_N) |         \
+	 (3 << MVE_FORMAT_BF_V))
+
+#define MVE_FORMAT_MBINFO (0x0001) /* only used for debugging */
+
+#define MVE_FORMAT_UNUSED (0x0000)
+
+	uint16_t reserved0; /* force 'data' to be 4-byte aligned */
+
+	union {
+		struct mve_buffer_frame_planar planar;
+		struct mve_buffer_frame_afbc afbc;
+	} data;
+
+	uint32_t reserved1; /* force size to be multiple of 8 bytes */
+};
+
+/* The bitstream buffer stores a number of bitstream bytes */
+struct mve_buffer_bitstream {
+	/* For identification of the buffer, this is not changed by
+	 * the firmware. */
+	uint64_t host_handle;
+
+	/* For matching input buffer with output buffers, the firmware
+	 * copies these values between frame buffers and bitstream buffers. */
+	uint64_t user_data_tag;
+
+	/* BufferFlags */
+	uint32_t bitstream_flags;
+#define MVE_BUFFER_BITSTREAM_FLAG_EOS (0x00000001)
+#define MVE_BUFFER_BITSTREAM_FLAG_ENDOFFRAME (0x00000010)
+#define MVE_BUFFER_BITSTREAM_FLAG_SYNCFRAME (0x00000020)
+#define MVE_BUFFER_BITSTREAM_FLAG_CODECCONFIG (0x00000080)
+#define MVE_BUFFER_BITSTREAM_FLAG_ENDOFSUBFRAME (0x00000400)
+
+	/* Length of allocated buffer */
+	uint32_t bitstream_alloc_bytes;
+
+	/* Byte offset from start to first byte */
+	uint32_t bitstream_offset;
+
+	/* Number of bytes in the buffer */
+	uint32_t bitstream_filled_len;
+
+	/* Pointer to buffer start */
+	uint32_t bitstream_buf_addr;
+
+	/* frame_type. 0:I, 1:p, 2:B, 3:b */
+	uint8_t frame_type;
+
+	/* Pad to force 8-byte alignment */
+	// uint32_t reserved;
+	uint8_t reserved[3];
+};
+
+/*
+ * Define a region in 16x16 units
+ *
+ * The region is macroblock positions (x,y) in the range
+ * mbx_left <= x < mbx_right
+ * mby_top  <= y < mby_bottom
+ */
+struct mve_buffer_param_region {
+	uint16_t mbx_left;   /* macroblock x left edge   (inclusive) */
+	uint16_t mbx_right;  /* macroblock x right edge  (exclusive) */
+	uint16_t mby_top;    /* macroblock y top edge    (inclusive) */
+	uint16_t mby_bottom; /* macroblock y bottom edge (exclusive) */
+	int16_t qp_delta;    /* QP delta value for this region, this
+			      * delta applies to QP values in the ranges:
+			      *   H264: 0-51
+			      *   HEVC: 0-51
+			      *   VP9:  0-255 */
+	uint16_t reserved;
+};
+
+/* input for encoder,
+ * the mve_buffer_param_regions buffer stores the information for FRAME buffers,
+ * and the information for regions of interest.
+ */
+struct mve_buffer_param_regions {
+	uint8_t n_regions; /* Number of regions */
+	uint8_t reserved[3];
+#define MVE_MAX_FRAME_REGIONS 16
+	struct mve_buffer_param_region region[MVE_MAX_FRAME_REGIONS];
+};
+
+/* the block parameter record specifies the various properties of a quad */
+struct mve_block_param_record {
+	uint16_t qp_delta; /* Bitset of four 4-bit QP delta values for a quad */
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_LEFT_16X16 (0)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_LEFT_16X16_SZ (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_RIGHT_16X16 (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_TOP_RIGHT_16X16_SZ (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_LEFT_16X16 (8)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_LEFT_16X16_SZ (4)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_RIGHT_16X16 (12)
+#define MVE_BLOCK_PARAM_RECORD_QP_DELTA_BOT_RIGHT_16X16_SZ (4)
+
+	uint8_t force;
+#define MVE_BLOCK_PARAM_RECORD_FORCE_NONE (0x00)
+#define MVE_BLOCK_PARAM_RECORD_FORCE_QP (0x01)
+
+	uint8_t reserved;
+};
+
+/* block configuration uncompressed rows header. this configures the size of the
+ * uncompressed body. */
+struct mve_buffer_general_rows_uncomp_hdr {
+	uint8_t n_cols_minus1; /* number of quad cols in picture minus 1 */
+	uint8_t n_rows_minus1; /* number of quad rows in picture minus 1 */
+	uint8_t reserved[2];
+};
+
+/* block configuration uncompressed rows body. this structure contains an array
+ * of block parameter records whose length is (n_cols_minus1 + 1) * (n_rows_minus1 + 1)
+ * elements. therefore the allocation of this structure needs to be dynamic and
+ * a pointer to the allocated memory should then be assigned to the general
+ * purpose buffer data pointer
+ */
+struct mve_buffer_general_rows_uncomp_body {
+	/* the size of this array is variable and not necessarily equal to 1.
+	 * therefore the sizeof operator should not be used
+	 */
+	struct mve_block_param_record bpr[1];
+};
+
+/* input for encoder, block level configurations.
+ * the row based block configurations can be defined in different formats. they
+ * are stored in the blk_cfgs union and identified by the blk_cfg_type member.
+ * these definitions consist of a header and body pair. the header part contains
+ * configuration information for the body. the body part describes the actual
+ * layout of the data buffer pointed to by the mve_buffer_general_hdr buffer_ptr.
+ */
+struct mve_buffer_general_block_configs {
+	uint8_t blk_cfg_type;
+#define MVE_BLOCK_CONFIGS_TYPE_NONE (0x00)
+#define MVE_BLOCK_CONFIGS_TYPE_ROW_UNCOMP (0xff)
+	uint8_t reserved[3];
+	union {
+		struct mve_buffer_general_rows_uncomp_hdr rows_uncomp;
+	} blk_cfgs;
+};
+
+/* input for encoder */
+struct mve_buffer_param_qp {
+	/* QP (quantization parameter) for encode.
+	 *
+	 * When used to set fixed QP for encode, with rate control
+	 * disabled, then the valid ranges are:
+	 *   H264: 0-51
+	 *   HEVC: 0-51
+	 *   VP8:  0-63
+	 *   VP9:  0-63
+	 * Note: The QP must be set separately for I, P and B frames.
+	 *
+	 * But when this message is used with the regions-feature,
+	 * then the valid ranges are the internal bitstream ranges:
+	 *   H264: 0-51
+	 *   HEVC: 0-51
+	 *   VP8:  0-127
+	 *   VP9:  0-255
+	 */
+	int32_t qp;
+};
+
+/* output from decoder */
+struct mve_buffer_param_display_size {
+	uint16_t display_width;
+	uint16_t display_height;
+};
+
+/* output from decoder, colour information needed for hdr */
+struct mve_buffer_param_colour_description {
+	uint32_t flags;
+#define MVE_BUFFER_PARAM_COLOUR_FLAG_MASTERING_DISPLAY_DATA_VALID (1)
+#define MVE_BUFFER_PARAM_COLOUR_FLAG_CONTENT_LIGHT_DATA_VALID (2)
+
+	uint8_t range; /* Unspecified=0, Limited=1, Full=2 */
+#define MVE_BUFFER_PARAM_COLOUR_RANGE_UNSPECIFIED (0)
+#define MVE_BUFFER_PARAM_COLOUR_RANGE_LIMITED (1)
+#define MVE_BUFFER_PARAM_COLOUR_RANGE_FULL (2)
+
+	uint8_t colour_primaries;	  /* see hevc spec. E.3.1 */
+	uint8_t transfer_characteristics; /* see hevc spec. E.3.1 */
+	uint8_t matrix_coeff;		  /* see hevc spec. E.3.1 */
+
+	uint16_t mastering_display_primaries_x[3]; /* see hevc spec. D.3.27 */
+	uint16_t mastering_display_primaries_y[3]; /* see hevc spec. D.3.27 */
+	uint16_t mastering_white_point_x;	   /* see hevc spec. D.3.27 */
+	uint16_t mastering_white_point_y;	   /* see hevc spec. D.3.27 */
+	uint32_t max_display_mastering_luminance;  /* see hevc spec. D.3.27 */
+	uint32_t min_display_mastering_luminance;  /* see hevc spec. D.3.27 */
+
+	uint32_t max_content_light_level; /* see hevc spec. D.3.35 */
+	uint32_t avg_content_light_level; /* see hevc spec. D.3.35 */
+
+	uint8_t video_format_present_flag;
+	uint8_t video_format;
+	uint8_t aspect_ratio_info_present_flag;
+	uint8_t aspect_ratio_idc;
+	uint8_t timing_flag_info_present_flag;
+	uint16_t sar_width;
+	uint16_t sar_height;
+	uint32_t num_units_in_tick;
+	uint32_t time_scale;
+};
+
+struct mve_buffer_param_sei_user_data_unregistered {
+	uint8_t flags;
+#define MVE_BUFFER_PARAM_USER_DATA_UNREGISTERED_VALID (1)
+	uint8_t uuid[16];
+	char user_data[256 - 35];
+	uint8_t user_data_len;
+
+	uint8_t reserved[5];
+};
+
+/* output from decoder see hevc spec. D.3.3 */
+struct mve_buffer_param_frame_field_info {
+	uint8_t pic_struct;
+	uint8_t source_scan_type;
+	uint8_t duplicate_flag;
+	uint8_t reserved;
+};
+
+/* output from decoder, VC-1 specific feature only relevant
+ * if using AFBC output
+ */
+struct mve_buffer_param_range_map {
+	uint8_t luma_map_enabled;
+	uint8_t luma_map_value;
+	uint8_t chroma_map_enabled;
+	uint8_t chroma_map_value;
+};
+
+/* input for encoder */
+struct mve_buffer_param_rate_control {
+	uint32_t rate_control_mode;
+#define MVE_OPT_RATE_CONTROL_MODE_OFF (0)
+#define MVE_OPT_RATE_CONTROL_MODE_STANDARD (1)
+#define MVE_OPT_RATE_CONTROL_MODE_VARIABLE (2)
+#define MVE_OPT_RATE_CONTROL_MODE_CONSTANT (3)
+#define MVE_OPT_RATE_CONTROL_MODE_C_VARIABLE (4)
+	uint32_t target_bitrate;  /* in bits per second */
+	uint32_t maximum_bitrate; /* in bits per second */
+};
+
+/* input for encoder */
+struct mve_buffer_param_rate_control_qp_range {
+	int32_t qp_min;
+	int32_t qp_max;
+};
+
+/* input for encoder, see hevc spec. D.3.16 */
+struct mve_buffer_param_frame_packing {
+	uint32_t flags;
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_QUINCUNX_SAMPLING (1)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_SPATIAL_FLIPPING (2)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_FRAME0_FLIPPED (4)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_FIELD_VIEWS (8)
+#define MVE_BUFFER_PARAM_FRAME_PACKING_FLAG_CURRENT_FRAME_IS_FRAME0 (16)
+
+	uint8_t frame_packing_arrangement_type;
+	uint8_t content_interpretation_type;
+
+	uint8_t frame0_grid_position_x;
+	uint8_t frame0_grid_position_y;
+	uint8_t frame1_grid_position_x;
+	uint8_t frame1_grid_position_y;
+
+	uint8_t reserved[2];
+};
+
+struct mve_buffer_param_rectangle {
+	uint16_t x_left;   /* pixel x left edge   (inclusive) */
+	uint16_t x_right;  /* pixel x right edge  (exclusive) */
+	uint16_t y_top;	   /* pixel y top edge    (inclusive) */
+	uint16_t y_bottom; /* pixel y bottom edge (exclusive) */
+};
+
+/* input for encoder,
+ * indicate which parts of the source picture has changed.
+ * The encoder can (optionally) use this information to
+ * reduce memory bandwidth.
+ *
+ * n_rectangles=0 indicates the source picture is unchanged.
+ *
+ * This parameter only applies to the picture that immediately
+ * follows (and not to subsequent ones).
+ */
+struct mve_buffer_param_change_rectangles {
+	uint8_t n_rectangles; /* Number of rectangles */
+	uint8_t reserved[3];
+#define MVE_MAX_FRAME_CHANGE_RECTANGLES 2
+	struct mve_buffer_param_rectangle rectangles[MVE_MAX_FRAME_CHANGE_RECTANGLES];
+};
+
+/* Parameters that are sent in the same communication channels
+ * as the buffers. A parameter applies to all subsequent buffers.
+ * Some types are only valid for decode, and some only for encode.
+ */
+struct mve_buffer_param {
+	uint32_t type;			       /* Extra data: */
+#define MVE_BUFFER_PARAM_TYPE_QP (2)	       /* qp */
+#define MVE_BUFFER_PARAM_TYPE_REGIONS (3)      /* regions */
+#define MVE_BUFFER_PARAM_TYPE_DISPLAY_SIZE (5) /* display_size */
+#define MVE_BUFFER_PARAM_TYPE_RANGE_MAP (6)    /* range_map */
+#define MVE_BUFFER_PARAM_TYPE_FRAME_RATE                                                                               \
+	(9)					      /* arg, in frames per second, as a                               \
+						       * fixed point Q16 value, for example                            \
+						       * 0x001e0000 == 30.0 fps */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL (10)	      /* rate_control */
+#define MVE_BUFFER_PARAM_TYPE_QP_I (12)		      /* qp for I frames, when no rate control */
+#define MVE_BUFFER_PARAM_TYPE_QP_P (13)		      /* qp for P frames, when no rate control */
+#define MVE_BUFFER_PARAM_TYPE_QP_B (14)		      /* qp for B frames, when no rate control */
+#define MVE_BUFFER_PARAM_TYPE_COLOUR_DESCRIPTION (15) /* colour_description */
+#define MVE_BUFFER_PARAM_TYPE_FRAME_PACKING (16)      /* frame_packing */
+#define MVE_BUFFER_PARAM_TYPE_FRAME_FIELD_INFO (17)   /* frame_field_info */
+#define MVE_BUFFER_PARAM_TYPE_GOP_RESET (18)	      /* no extra data */
+#define MVE_BUFFER_PARAM_TYPE_DPB_HELD_FRAMES                                                                          \
+	(19)						     /* arg, number of output buffers that are                 \
+							      * complete and held by firmware in the                   \
+							      * DPB for reordering purposes.                           \
+							      * Valid after the next frame is output */
+#define MVE_BUFFER_PARAM_TYPE_CHANGE_RECTANGLES (20)	     /* change rectangles */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_QP_RANGE (21)     /* rate_control_qp_range */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_HRD_BUF_SIZE (23) /* arg */
+#define MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_QP_RANGE_I                                                                  \
+	(25)						      /* special range for I frames,                           \
+							       * rate_control_qp_range */
+#define MVE_BUFFER_PARAM_TYPE_SEI_USER_DATA_UNREGISTERED (26) /* sei user_data_unregistered */
+
+	union {
+		uint32_t arg; /* some parameters only need a uint32_t as argument */
+		struct mve_buffer_param_qp qp;
+		struct mve_buffer_param_regions regions;
+		struct mve_buffer_param_display_size display_size;
+		struct mve_buffer_param_range_map range_map;
+		struct mve_buffer_param_rate_control rate_control;
+		struct mve_buffer_param_rate_control_qp_range rate_control_qp_range;
+		struct mve_buffer_param_colour_description colour_description;
+		struct mve_buffer_param_frame_packing frame_packing;
+		struct mve_buffer_param_frame_field_info frame_field_info;
+		struct mve_buffer_param_change_rectangles change_rectangles;
+		struct mve_buffer_param_sei_user_data_unregistered user_data_unregistered;
+	} data;
+};
+
+/* output from decoder, assertive display statistics.
+ * buffer_ptr points to a buffer of luma quad average values for the picture
+ * that can be used as a thumbnail. the type of content used to generate the
+ * assertive display statistics is indicated by MVE_AD_STATS_PIC_FMT_INTERLACED.
+ * for progressive content; the arrangement is in raster format with dimensions
+ * thumbnail_width by thumbnail_height. the overall frame average luma and
+ * chroma values are returned in frame_average.
+ * for interlaced content; the arrangement is in raster format, top field
+ * followed by bottom field with each field having dimensions thumbnail_width by
+ * thumbnail_height. the field averages for luma and chroma values are combined
+ * and returned in an overall value for the frame (frame_average).
+ */
+struct mve_buffer_general_ad_stats {
+	uint32_t frame_averages;
+// bitfields
+#define MVE_AD_STATS_PIC_AVGS_Y (0)
+#define MVE_AD_STATS_PIC_AVGS_Y_SZ (12)
+#define MVE_AD_STATS_PIC_AVGS_CB (12)
+#define MVE_AD_STATS_PIC_AVGS_CB_SZ (10)
+#define MVE_AD_STATS_PIC_AVGS_CR (22)
+#define MVE_AD_STATS_PIC_AVGS_CR_SZ (10)
+	uint16_t thumbnail_width;
+	uint16_t thumbnail_height;
+	uint8_t ad_stats_flags;
+#define MVE_AD_STATS_PIC_FMT_PROGRESSIVE (0)
+#define MVE_AD_STATS_PIC_FMT_INTERLACED (1)
+	uint8_t reserved[3];
+};
+
+/* The general purpose buffer header stores the common fields of an
+ * mve_buffer_general. it contains the pointer to the data buffer that contains
+ * the general purpose data
+ */
+struct mve_buffer_general_hdr {
+	/* For identification of the buffer, this is not changed by the firmware. */
+	uint64_t host_handle;
+
+	/* this depends upon the type of the general purpose buffer */
+	uint64_t user_data_tag;
+
+	/* pointer to the buffer containing the general purpose data. the format
+	 * of this data is defined by the configuration in the mve_buffer_general */
+	uint32_t buffer_ptr;
+
+	/* size of the buffer pointed to by buffer_ptr */
+	uint32_t buffer_size;
+
+	/* selects the type of semantics to use for the general purpose buffer. it
+	 * tags (or discriminates) the union config member in mve_buffer_general
+	 */
+	uint16_t type;				  /* Extra data: */
+#define MVE_BUFFER_GENERAL_TYPE_INVALID (0)	  /* invalid */
+#define MVE_BUFFER_GENERAL_TYPE_BLOCK_CONFIGS (1) /* block_configs */
+#define MVE_BUFFER_GENERAL_TYPE_AD_STATS (2)	  /* assertive display statistics */
+
+	/* size of the mve_buffer_general config member */
+	uint16_t config_size;
+
+	/* pad to force 8-byte alignment */
+	uint32_t reserved;
+};
+
+/* The general purpose buffer consists of a header and a configuration. The
+ * header contains a pointer to a buffer whose format is described by the
+ * configuration. The type of configuration is indicated by the type value in
+ * the header. N.B. In use, the size of the config part of this structure is
+ * defined in the header and is not necessarily equal to that returned by the
+ * sizeof() operator. This allows a more size efficient communication between
+ * the host and firmware.
+ */
+struct mve_buffer_general {
+	struct mve_buffer_general_hdr header;
+
+	/* used to describe the configuration of the general purpose buffer data
+	 * pointed to be buffer_ptr
+	 */
+	union {
+		struct mve_buffer_general_block_configs block_configs;
+		struct mve_buffer_general_ad_stats ad_stats;
+	} config;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __FW_INCLUDE__MVE_PROTOCOL_DEF_H__ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_bitops.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_bitops.h
new file mode 100644
index 000000000000..324122b2e8ce
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_bitops.h
@@ -0,0 +1,88 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_BITOPS_H_
+#define _MVX_BITOPS_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+/**
+ * mvx_set_bit() - Set a bit in the bitmask.
+ * @bit:		Bit to be set.
+ * @addr:		Pointer to bitmask.
+ *
+ * Works similar to set_bit but uses no locks, is not atomic and protects
+ * agains overflow.
+ */
+static inline void mvx_set_bit(unsigned int bit, uint64_t *addr)
+{
+	BUG_ON(bit >= (sizeof(*addr) * 8));
+	*addr |= 1ull << bit;
+}
+
+/**
+ * mvx_clear_bit() - Clear a bit in the bitmask.
+ * @bit:		Bit to be cleared.
+ * @addr:		Pointer to bitmask.
+ *
+ * Works similar to clear_bit but uses no locks, is not atomic and protects
+ * agains overflow.
+ */
+static inline void mvx_clear_bit(unsigned int bit, uint64_t *addr)
+{
+	BUG_ON(bit >= (sizeof(*addr) * 8));
+	*addr &= ~(1ull << bit);
+}
+
+/**
+ * mvx_test_bit() - Test a bit in the bitmask.
+ * @bit:		Bit to be tested.
+ * @addr:		Pointer to bitmask.
+ *
+ * Works similar to test_bit but uses no locks, is not atomic and protects
+ * agains overflow.
+ */
+static inline bool mvx_test_bit(unsigned int bit, uint64_t *addr)
+{
+	BUG_ON(bit >= (sizeof(*addr) * 8));
+	return 0 != (*addr & (1ull << bit));
+}
+
+#endif /* _MVX_BITOPS_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_buffer.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_buffer.c
new file mode 100644
index 000000000000..1d8ddb45b633
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_buffer.c
@@ -0,0 +1,478 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/kernel.h>
+#include "mvx_buffer.h"
+#include "mvx_seq.h"
+#include "mvx_log_group.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+/**
+ * Each 2x2 pixel square is subsampled. How many samples that are taken depends
+ * on the color format, but typically the luma channel (Y) gets 4 samples and
+ * the luma channels (UV) get 2 or 4 samples.
+ */
+#define SUBSAMPLE_PIXELS 2
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+/**
+ * get_stride() - Get 3 plane stride for 2x2 pixels square.
+ * @format:	MVX frame format.
+ * @stride:	[plane 0, plane 1, plane 2][x, y] stride.
+ *
+ * Calculate the stride in bytes for each plane for a subsampled (2x2) pixels
+ * square.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int get_stride(enum mvx_format format, uint8_t *nplanes, unsigned int stride[MVX_BUFFER_NPLANES][2])
+{
+	switch (format) {
+	case MVX_FORMAT_YUV420_I420:
+		*nplanes = 3;
+		stride[0][0] = 2;
+		stride[0][1] = 2;
+		stride[1][0] = 1;
+		stride[1][1] = 1;
+		stride[2][0] = 1;
+		stride[2][1] = 1;
+		break;
+	case MVX_FORMAT_YUV420_NV12:
+	case MVX_FORMAT_YUV420_NV21:
+		*nplanes = 2;
+		stride[0][0] = 2;
+		stride[0][1] = 2;
+		stride[1][0] = 2;
+		stride[1][1] = 1;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV420_P010:
+		*nplanes = 2;
+		stride[0][0] = 4;
+		stride[0][1] = 2;
+		stride[1][0] = 4;
+		stride[1][1] = 1;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV420_Y0L2:
+	case MVX_FORMAT_YUV420_AQB1:
+		*nplanes = 1;
+		stride[0][0] = 8;
+		stride[0][1] = 1;
+		stride[1][0] = 0;
+		stride[1][1] = 0;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV422_YUY2:
+	case MVX_FORMAT_YUV422_UYVY:
+		*nplanes = 1;
+		stride[0][0] = 4;
+		stride[0][1] = 2;
+		stride[1][0] = 0;
+		stride[1][1] = 0;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV422_Y210:
+	case MVX_FORMAT_RGBA_8888:
+	case MVX_FORMAT_BGRA_8888:
+	case MVX_FORMAT_ARGB_8888:
+	case MVX_FORMAT_ABGR_8888:
+		*nplanes = 1;
+		stride[0][0] = 8;
+		stride[0][1] = 2;
+		stride[1][0] = 0;
+		stride[1][1] = 0;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int map_plane(struct mvx_buffer *buf, mvx_mmu_va begin, mvx_mmu_va end, unsigned int plane)
+{
+	while (begin < end) {
+		struct mvx_buffer_plane *p = &buf->planes[plane];
+		int ret;
+
+		ret = mvx_mmu_map_pages(buf->mmu, begin, p->pages, MVX_ATTR_SHARED_RW, MVX_ACCESS_READ_WRITE);
+		if (ret == 0) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO,
+				      "Memory map buffer. buf=%p, plane=%u, va=0x%x, size=%zu.", buf, plane,
+				      p->pages->va, mvx_buffer_size(buf, plane));
+			return 0;
+		}
+
+		if (ret != -EAGAIN)
+			return ret;
+
+		begin += 1 * 1024 * 1024; /* 1 MB. */
+	}
+
+	return -ENOMEM;
+}
+
+/****************************************************************************
+ * External functions
+ ****************************************************************************/
+
+void mvx_buffer_show(struct mvx_buffer *buf, struct seq_file *s)
+{
+	int i;
+	int ind = 0;
+
+	mvx_seq_printf(s, "mvx_buffer", ind, "%p\n", buf);
+
+	ind++;
+	mvx_seq_printf(s, "format", ind, "0x%x\n", buf->format);
+	mvx_seq_printf(s, "dir", ind, "%u\n", buf->dir);
+	mvx_seq_printf(s, "flags", ind, "0x%0x\n", buf->flags);
+	mvx_seq_printf(s, "width", ind, "%u\n", buf->width);
+	mvx_seq_printf(s, "height", ind, "%u\n", buf->height);
+	mvx_seq_printf(s, "nplanes", ind, "%u\n", buf->nplanes);
+	mvx_seq_printf(s, "planes", ind, "\n");
+	ind++;
+	for (i = 0; i < buf->nplanes; ++i) {
+		char tag[10];
+		struct mvx_buffer_plane *plane = &buf->planes[i];
+
+		scnprintf(tag, sizeof(tag), "#%d", i);
+		mvx_seq_printf(s, tag, ind, "va: 0x%08x, size: %10zu, stride: %5u, filled: %10u\n",
+			       mvx_buffer_va(buf, i), mvx_buffer_size(buf, i), plane->stride, plane->filled);
+	}
+
+	ind--;
+}
+
+int mvx_buffer_construct(struct mvx_buffer *buf, struct device *dev, struct mvx_mmu *mmu, enum mvx_direction dir,
+			 unsigned int nplanes, struct sg_table **sgt)
+{
+	int i;
+
+	if (nplanes > MVX_BUFFER_NPLANES) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to construct buffer. Too many planes. nplanes=%u.",
+			      nplanes);
+		return -EINVAL;
+	}
+
+	memset(buf, 0, sizeof(*buf));
+
+	buf->dev = dev;
+	buf->mmu = mmu;
+	buf->dir = dir;
+	buf->nplanes = nplanes;
+
+	for (i = 0; i < buf->nplanes; ++i) {
+		struct mvx_buffer_plane *plane = &buf->planes[i];
+
+		if (sgt[i] == NULL)
+			break;
+
+		plane->pages = mvx_mmu_alloc_pages_sg(dev, sgt[i], 0);
+		if (IS_ERR(plane->pages))
+			goto free_pages;
+	}
+
+	return 0;
+
+free_pages:
+	while (i--)
+		mvx_mmu_free_pages(buf->planes[i].pages);
+
+	return -ENOMEM;
+}
+
+void mvx_buffer_destruct(struct mvx_buffer *buf)
+{
+	int i;
+
+	mvx_buffer_unmap(buf);
+
+	for (i = 0; i < buf->nplanes; i++)
+		if (buf->planes[i].pages != NULL)
+			mvx_mmu_free_pages(buf->planes[i].pages);
+}
+
+int mvx_buffer_map(struct mvx_buffer *buf, mvx_mmu_va begin, mvx_mmu_va end)
+{
+	int i;
+	int ret = 0;
+
+	for (i = 0; i < buf->nplanes; i++) {
+		struct mvx_buffer_plane *plane = &buf->planes[i];
+
+		if (plane->pages != NULL) {
+			ret = map_plane(buf, begin, end, i);
+			if (ret != 0) {
+				mvx_buffer_unmap(buf);
+				break;
+			}
+		}
+	}
+
+	return ret;
+}
+
+void mvx_buffer_unmap(struct mvx_buffer *buf)
+{
+	int i;
+
+	for (i = 0; i < buf->nplanes; i++) {
+		struct mvx_buffer_plane *plane = &buf->planes[i];
+
+		if ((plane->pages != NULL) && (plane->pages->va != 0))
+			mvx_mmu_unmap_pages(plane->pages);
+	}
+}
+
+bool mvx_buffer_is_mapped(struct mvx_buffer *buf)
+{
+	return (buf->planes[0].pages != NULL) && (buf->planes[0].pages->va != 0);
+}
+
+int mvx_buffer_synch(struct mvx_buffer *buf, enum dma_data_direction dir)
+{
+	int i;
+	int ret;
+	int page_count = 0;
+	for (i = 0; i < buf->nplanes; i++) {
+		struct mvx_buffer_plane *plane = &buf->planes[i];
+		/*calculate page offset of plane, follow as 'mvx_buffer_va' */
+		int page_off = (plane->offset + plane->pages->offset) >> PAGE_SHIFT;
+		/*calculate page end of plane if filled is valid */
+		int page_off_end = plane->pages->count;
+		if (plane->filled) {
+			page_off_end =
+				(plane->offset + plane->pages->offset + plane->filled + PAGE_SIZE - 1) >> PAGE_SHIFT;
+		}
+		page_count = page_off_end - page_off;
+		if (page_count + page_off > plane->pages->count) {
+			page_count = plane->pages->count - page_off;
+		}
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO,
+			      "plane [%d] sync pages: %p, dir: %d, page offset: %d, pages %d", i,
+			      plane->pages->pages[0], buf->dir, page_off, page_count);
+		if (plane->pages != NULL) {
+			ret = mvx_mmu_synch_pages(plane->pages, dir, page_off, page_count);
+			if (ret != 0)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+void mvx_buffer_clear(struct mvx_buffer *buf)
+{
+	unsigned int i;
+
+	buf->flags = 0;
+
+	for (i = 0; i < buf->nplanes; i++)
+		buf->planes[i].filled = 0;
+}
+
+int mvx_buffer_filled_set(struct mvx_buffer *buf, unsigned int plane, unsigned int filled, unsigned int offset)
+{
+	struct mvx_buffer_plane *p = &buf->planes[plane];
+	size_t size = mvx_buffer_size(buf, plane);
+
+	if (plane > buf->nplanes)
+		return -EINVAL;
+
+	if (size < (filled + offset)) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "Buffer plane too small. plane=%d, size=%zu, filled=%u, offset=%u.", plane, size, filled,
+			      offset);
+		buf->flags |= MVX_BUFFER_FRAME_NEED_REALLOC;
+		return -ENOMEM;
+	}
+
+	p->filled = filled;
+	p->offset = offset;
+
+	return 0;
+}
+
+size_t mvx_buffer_size(struct mvx_buffer *buf, unsigned int plane)
+{
+	struct mvx_buffer_plane *p = &buf->planes[plane];
+
+	if (plane >= buf->nplanes || p->pages == NULL)
+		return 0;
+
+	return mvx_mmu_size_pages(p->pages);
+}
+
+mvx_mmu_va mvx_buffer_va(struct mvx_buffer *buf, unsigned int plane)
+{
+	struct mvx_buffer_plane *p = &buf->planes[plane];
+
+	if (plane >= buf->nplanes || p->pages == NULL)
+		return 0;
+
+	return p->pages->va + p->pages->offset + p->offset;
+}
+
+int mvx_buffer_frame_dim(enum mvx_format format, unsigned int width, unsigned int height, uint8_t *nplanes,
+			 unsigned int *stride, unsigned int *size)
+{
+	unsigned int s[MVX_BUFFER_NPLANES][2];
+	unsigned int __nplanes = *nplanes;
+	int i;
+	int ret;
+
+	ret = get_stride(format, nplanes, s);
+	if (ret != 0)
+		return ret;
+
+	for (i = 0; i < *nplanes; i++) {
+		const unsigned int stride_align = 1;
+		unsigned int tmp = DIV_ROUND_UP(width * s[i][0], SUBSAMPLE_PIXELS);
+		/* Use optimal stride if no special stride was requested. */
+		if (i >= __nplanes || stride[i] == 0)
+			stride[i] = round_up(tmp, stride_align);
+		/* Else make sure to round up to minimum stride. */
+		else
+			stride[i] = max(stride[i], tmp);
+
+		size[i] = DIV_ROUND_UP(height * s[i][1], SUBSAMPLE_PIXELS) * stride[i];
+	}
+	/* a workaround patch for nv12/nv21/p010 odd height/width output*/
+	if (*nplanes == 2 && (width % 2 != 0 || height % 2 != 0)) {
+		unsigned int tmp = DIV_ROUND_UP(width, SUBSAMPLE_PIXELS) * s[1][0];
+		stride[1] = max(stride[1], tmp);
+		size[1] = DIV_ROUND_UP(height * s[1][1], SUBSAMPLE_PIXELS) * stride[1];
+	}
+
+	for (i = *nplanes; i < MVX_BUFFER_NPLANES; i++) {
+		size[i] = 0;
+	}
+	return 0;
+}
+
+int mvx_buffer_frame_set(struct mvx_buffer *buf, enum mvx_format format, unsigned int width, unsigned int height,
+			 unsigned int *stride, unsigned int *size, bool interlaced)
+{
+	int i;
+
+	buf->format = format;
+	buf->width = width;
+	buf->height = height;
+
+	for (i = 0; i < buf->nplanes; i++) {
+		struct mvx_buffer_plane *plane = &buf->planes[i];
+
+		plane->stride = stride[i];
+
+		if (buf->dir == MVX_DIR_OUTPUT) {
+			int ret;
+
+			ret = mvx_buffer_filled_set(buf, i, size[i], plane->offset);
+			if (ret != 0)
+				return ret;
+		}
+
+		/* Verify that plane has correct length. */
+		if (plane->filled > 0 && plane->filled != size[i]) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO,
+				      "Buffer filled length does not match plane size. plane=%i, filled=%zu, size=%u.",
+				      i, plane->filled, size[i]);
+			// return -ENOMEM;
+		}
+
+		/* Verify that there is no buffer overflow. */
+		if ((plane->filled + plane->offset) > mvx_buffer_size(buf, i)) {
+			MVX_LOG_PRINT(
+				&mvx_log_if, MVX_LOG_WARNING,
+				"Buffer plane size is too small. plane=%i, size=%zu, size=%u, filled=%u, offset=%u", i,
+				size[i], mvx_buffer_size(buf, i), plane->filled, plane->offset);
+			return -ENOMEM;
+		}
+	}
+
+	if (interlaced != false) {
+		buf->flags |= MVX_BUFFER_INTERLACE;
+	} else {
+		buf->flags &= ~MVX_BUFFER_INTERLACE;
+	}
+	return 0;
+}
+
+int mvx_buffer_afbc_set(struct mvx_buffer *buf, enum mvx_format format, unsigned int width, unsigned int height,
+			unsigned int afbc_width, unsigned int size, bool interlaced)
+{
+	int ret;
+
+	buf->format = format;
+	buf->width = width;
+	buf->height = height;
+	buf->planes[0].afbc_width = afbc_width;
+
+	if (buf->dir == MVX_DIR_INPUT) {
+		buf->crop_left = 0;
+		buf->crop_top = 0;
+	}
+
+	if (buf->dir == MVX_DIR_OUTPUT) {
+		ret = mvx_buffer_filled_set(buf, 0, size, buf->planes[0].offset);
+		if (ret != 0)
+			return ret;
+	}
+
+	if (size > mvx_buffer_size(buf, 0)) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "AFBC buffer too small. buf_size=%zu, size=%u.", size,
+			      mvx_buffer_size(buf, 0));
+		return -ENOMEM;
+	}
+
+	if (interlaced != false)
+		buf->flags |= MVX_BUFFER_INTERLACE;
+
+	return 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_buffer.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_buffer.h
new file mode 100644
index 000000000000..fb1ca2237127
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_buffer.h
@@ -0,0 +1,377 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_BUFFER_H_
+#define _MVX_BUFFER_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/seq_file.h>
+#include <linux/types.h>
+#include "mvx_if.h"
+#include "mvx_mmu.h"
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define MVX_BUFFER_NPLANES 3
+#define MVX_ROI_QP_NUMS 10
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+
+/**
+ * struct mvx_buffer_plane - Plane information.
+ * @pages:	MMU pages object.
+ * @stride:	Horizontal stride in bytes.
+ * @filled:	Number of bytes written to this plane. For a frame buffer this
+ *              value should always match the size of the plane.
+ * @offset:	Offset in bytes from begin of buffer to first bitstream data.
+ * @afbc_width: AFBC width in superblocks.
+ */
+struct mvx_buffer_plane {
+	struct mvx_mmu_pages *pages;
+	unsigned int stride;
+	unsigned int filled;
+	unsigned int offset;
+	unsigned int afbc_width;
+};
+
+struct mvx_buffer_general_encoder_stats {
+	uint8_t encoder_stats_type;
+#define MVX_ENCODER_STATS_TYPE_FULL (0x01)
+	uint8_t frame_type; // See MVE_FRAME_TYPE_*
+#define MVX_FRAME_TYPE_I 0
+#define MVX_FRAME_TYPE_P 1
+#define MVX_FRAME_TYPE_B 2
+	uint8_t used_as_reference; // 0=No, 1=Yes
+	uint8_t qp;		   // base quantizer used for the frame
+				   // HEVC, H.264: 0-51. VP9: 0-63
+	uint32_t picture_count;	   // display order picture count
+	uint16_t num_cols;	   // number of columns (each 32 pixels wide)
+	uint16_t num_rows;	   // number of rows    (each 32 pixels high)
+	uint32_t ref_pic_count[2]; // display order picture count of references
+				   // unused values are set to zero
+};
+
+struct mvx_buffer_general_rows_uncomp_hdr {
+	uint8_t n_cols_minus1; /* number of quad cols in picture minus 1 */
+	uint8_t n_rows_minus1; /* number of quad rows in picture minus 1 */
+	uint8_t reserved[2];
+};
+
+struct mvx_buffer_general_block_configs {
+	uint8_t blk_cfg_type;
+#define MVX_BLOCK_CONFIGS_TYPE_NONE (0x00)
+#define MVX_BLOCK_CONFIGS_TYPE_ROW_UNCOMP (0xff)
+	uint8_t reserved[3];
+	union {
+		struct mvx_buffer_general_rows_uncomp_hdr rows_uncomp;
+	} blk_cfgs;
+};
+
+struct mvx_buffer_general_hdr {
+	/* For identification of the buffer, this is not changed by the firmware. */
+	uint64_t host_handle;
+
+	/* this depends upon the type of the general purpose buffer */
+	uint64_t user_data_tag;
+
+	/* pointer to the buffer containing the general purpose data. the format
+	 * of this data is defined by the configuration in the mve_buffer_general */
+	uint32_t buffer_ptr;
+
+	/* size of the buffer pointed to by buffer_ptr */
+	uint32_t buffer_size;
+
+	/* selects the type of semantics to use for the general purpose buffer. it
+	 * tags (or discriminates) the union config member in mve_buffer_general
+	 */
+	uint16_t type;				  /* Extra data: */
+#define MVX_BUFFER_GENERAL_TYPE_BLOCK_CONFIGS (1) /* block_configs */
+#define MVX_BUFFER_GENERAL_TYPE_ENCODER_STATS (4) /* encoder_stats */
+
+	/* size of the mve_buffer_general config member */
+	uint16_t config_size;
+
+	/* pad to force 8-byte alignment */
+	uint32_t reserved;
+};
+
+struct mvx_buffer_general {
+	struct mvx_buffer_general_hdr header;
+
+	/* used to describe the configuration of the general purpose buffer data
+	 * pointed to be buffer_ptr
+	 */
+	union {
+		struct mvx_buffer_general_block_configs block_configs;
+		struct mvx_buffer_general_encoder_stats encoder_stats;
+	} config;
+};
+
+/**
+ * struct mvx_buffer - Buffer descriptor.
+ * @dev:	Pointer to device.
+ * @mmu:	Pointer to MMU.
+ * @head:	List head used to add buffer to various queues.
+ * @format:	Bitstream or frame format.
+ * @dir:	Direction the buffer was allocated for.
+ * @user_data:	User data copied from input- to output buffer.
+ * @flags:	Buffer flags.
+ * @width:	Frame width in pixels.
+ * @height:	Frame height in pixels.
+ * @crop_left:	Left crop in pixels.
+ * @crop_top:	Top crop in pixels.
+ * @nplanes:	Number of planes.
+ * @planes:	Array or planes.
+ */
+struct mvx_buffer {
+	struct device *dev;
+	struct mvx_mmu *mmu;
+	struct list_head head;
+	enum mvx_format format;
+	enum mvx_direction dir;
+	uint64_t user_data;
+	unsigned int flags;
+	unsigned int width;
+	unsigned int height;
+	unsigned int crop_left;
+	unsigned int crop_top;
+	unsigned int nplanes;
+	struct mvx_buffer_plane planes[MVX_BUFFER_NPLANES];
+	struct mvx_buffer_general general;
+};
+
+/**
+ * struct mvx_buffer - Buffer descriptor.
+ * @head:	List head used to add buffer to corrupt buffer queues.
+ * @user_data:	User data copied from input- to output buffer.
+ */
+struct mvx_corrupt_buffer {
+	struct list_head head;
+	uint64_t user_data;
+};
+
+#define MVX_BUFFER_EOS 0x00000001
+#define MVX_BUFFER_EOF 0x00000002
+#define MVX_BUFFER_CORRUPT 0x00000004
+#define MVX_BUFFER_REJECTED 0x00000008
+#define MVX_BUFFER_DECODE_ONLY 0x00000010
+#define MVX_BUFFER_CODEC_CONFIG 0x00000020
+#define MVX_BUFFER_AFBC_TILED_HEADERS 0x00000040
+#define MVX_BUFFER_AFBC_TILED_BODY 0x00000080
+#define MVX_BUFFER_AFBC_32X8_SUPERBLOCK 0x00000100
+#define MVX_BUFFER_INTERLACE 0x00000200
+#define MVX_BUFFER_END_OF_SUB_FRAME 0x00000400
+#define MVX_BUFFER_FRAME_PRESENT 0x00000800
+
+#define MVX_BUFFER_FRAME_FLAG_ROTATION_90 0x00001000  /* Frame is rotated 90 degrees */
+#define MVX_BUFFER_FRAME_FLAG_ROTATION_180 0x00002000 /* Frame is rotated 180 degrees */
+#define MVX_BUFFER_FRAME_FLAG_ROTATION_270 0x00003000 /* Frame is rotated 270 degrees */
+#define MVX_BUFFER_FRAME_FLAG_ROTATION_MASK 0x00003000
+
+#define MVX_BUFFER_FRAME_FLAG_MIRROR_HORI 0x00010000
+#define MVX_BUFFER_FRAME_FLAG_MIRROR_VERT 0x00020000
+#define MVX_BUFFER_FRAME_FLAG_MIRROR_MASK 0x00030000
+
+#define MVX_BUFFER_FRAME_FLAG_SCALING_2 0x00004000 /* Frame is scaled by half */
+#define MVX_BUFFER_FRAME_FLAG_SCALING_4 0x00008000 /* Frame is scaled by quarter */
+#define MVX_BUFFER_FRAME_FLAG_SCALING_MASK 0x0000C000
+
+#define MVX_BUFFER_FRAME_FLAG_GENERAL 0x00040000 /* Frame is a general buffer */
+#define MVX_BUFFER_FRAME_FLAG_ROI 0x00080000	 /* This buffer has a roi region */
+
+#define MVX_BUFFER_FRAME_NEED_REALLOC 0x00100000 /* This buffer needs realloc */
+
+#define MVX_BUFFER_FRAME_FLAG_IFRAME 0x20000000
+#define MVX_BUFFER_FRAME_FLAG_PFRAME 0x40000000
+#define MVX_BUFFER_FRAME_FLAG_BFRAME 0x80000000
+
+#define MVX_BUFFER_AFBC_BLOCK_SPLIT 0x10000000
+
+#define MVX_BUFFER_FLAG_DISABLE_CACHE_MAINTENANCE 0x01000000 /*disable cache maintenance for buffer */
+/****************************************************************************
+ * External functions
+ ****************************************************************************/
+
+/**
+ * mvx_buffer_construct() - Construct the buffer object.
+ * @buf:	Pointer to buffer.
+ * @dev:	Pointer to device.
+ * @mmu:	Pointer to MMU.
+ * @dir:	Which direction the buffer was allocated for.
+ * @nplanes:	Number of planes.
+ * @sgt:	Array with SG tables. Each table contains a list of memory
+ *              pages for corresponding plane.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_buffer_construct(struct mvx_buffer *buf, struct device *dev, struct mvx_mmu *mmu, enum mvx_direction dir,
+			 unsigned int nplanes, struct sg_table **sgt);
+
+/**
+ * mvx_buffer_construct() - Destruct the buffer object.
+ * @buf:	Pointer to buffer.
+ */
+void mvx_buffer_destruct(struct mvx_buffer *buf);
+
+/**
+ * mvx_buffer_map() - Map the buffer to the MVE virtual address space.
+ * @buf:	Pointer to buffer.
+ * @begin:	MVE virtual begin address.
+ * @end:	MVE virtual end address.
+ *
+ * Try to MMU map the buffer anywhere between the begin and end addresses.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_buffer_map(struct mvx_buffer *buf, mvx_mmu_va begin, mvx_mmu_va end);
+
+/**
+ * mvx_buffer_unmap() - Unmap the buffer from the MVE virtual address space.
+ * @buf:	Pointer to buffer.
+ */
+void mvx_buffer_unmap(struct mvx_buffer *buf);
+
+/**
+ * mvx_buffer_is_mapped() - Return if buffer has been mapped.
+ * @buf:	Pointer to buffer.
+ *
+ * Return: True if mapped, else false.
+ */
+bool mvx_buffer_is_mapped(struct mvx_buffer *buf);
+
+/**
+ * mvx_buffer_synch() - Synch the data caches.
+ * @buf:	Pointer to buffer.
+ * @dir:	Data direction.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_buffer_synch(struct mvx_buffer *buf, enum dma_data_direction dir);
+
+/**
+ * mvx_buffer_clear() - Clear and empty the buffer.
+ * @buf:	Pointer to buffer.
+ */
+void mvx_buffer_clear(struct mvx_buffer *buf);
+
+/**
+ * mvx_buffer_filled_set() - Set filled bytes for each plane.
+ * @buf:	Pointer to buffer.
+ * @plane:	Plane index.
+ * @filled:	Number of bytes filled.
+ * @offset:	Number of bytes offset.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_buffer_filled_set(struct mvx_buffer *buf, unsigned int plane, unsigned int filled, unsigned int offset);
+
+/**
+ * mvx_buffer_size() - Get size in bytes for a plane.
+ * @buf:	Pointer to buffer.
+ * @plane:	Which plane to get size for.
+ *
+ * Return: Size of plane.
+ */
+size_t mvx_buffer_size(struct mvx_buffer *buf, unsigned int plane);
+
+/**
+ * mvx_buffer_va() - Get VA for a plane.
+ * @buf:	Pointer to buffer.
+ * @plane:	Plane index.
+ *
+ * Return: VA address of plane, 0 if unmapped.
+ */
+mvx_mmu_va mvx_buffer_va(struct mvx_buffer *buf, unsigned int plane);
+
+/**
+ * mvx_buffer_frame_dim() - Get frame buffer dimensions.
+ * @format:	Bitstream or frame format.
+ * @width:	Width in pixels.
+ * @height:	Height in pixels.
+ * @nplanes:	Number of planes for this format.
+ * @stride:	Horizontal stride in bytes.
+ * @size:	Size in bytes for each plane.
+ *
+ * If *nplanes is larger than 0 then the stride is used as input to tell this
+ * function which stride that is desired, but it might be modified if the
+ * stride is too short or not optimal for the MVE hardware.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_buffer_frame_dim(enum mvx_format format, unsigned int width, unsigned int height, uint8_t *nplanes,
+			 unsigned int *stride, unsigned int *size);
+
+/**
+ * mvx_buffer_frame_set() - Set frame dimensions.
+ * @buf:	Pointer to buffer.
+ * @format:	Bitstream or frame format.
+ * @width:	Width in pixels.
+ * @height:	Height in pixels.
+ * @stride:	Horizontal stride in bytes.
+ * @size:	Size in bytes for each plane.
+ * @interlaced:	Defines if the buffer is interlaced.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_buffer_frame_set(struct mvx_buffer *buf, enum mvx_format format, unsigned int width, unsigned int height,
+			 unsigned int *stride, unsigned int *size, bool interlaced);
+
+/**
+ * mvx_buffer_afbc_set() - Set AFBC dimensions.
+ * @buf:	Pointer to buffer.
+ * @format:	Bitstream or frame format.
+ * @width:	Width in pixels.
+ * @height:	Height in pixels.
+ * @afbc_width:	AFBC width in superblocks.
+ * @size:	Size in bytes for AFBC plane.
+ * @interlaced:	Defines if the buffer is interlaced.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_buffer_afbc_set(struct mvx_buffer *buf, enum mvx_format format, unsigned int width, unsigned int height,
+			unsigned int afbc_width, unsigned int size, bool interlaced);
+
+/**
+ * mvx_buffer_show() - Print debug information into seq-file.
+ * @buf:	Pointer to buffer.
+ * @s:		Seq-file to print to.
+ */
+void mvx_buffer_show(struct mvx_buffer *buf, struct seq_file *s);
+
+#endif /* _MVX_BUFFER_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware.c
new file mode 100644
index 000000000000..80446dd85697
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware.c
@@ -0,0 +1,536 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/atomic.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/errno.h>
+#include <linux/gfp.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/firmware.h>
+#include <linux/kobject.h>
+#include <linux/kthread.h>
+#include <linux/timer.h>
+#include <linux/version.h>
+#include "mvx_if.h"
+#include "mvx_log_group.h"
+#include "mvx_firmware_cache.h"
+#include "mvx_firmware_priv.h"
+#include "mvx_mmu.h"
+#include "mvx_secure.h"
+#include "mvx_seq.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define FW_TEXT_BASE_ADDR 0x1000u
+
+/****************************************************************************
+ * Private functions
+ ****************************************************************************/
+
+/**
+ * test_bit_32() - 32 bit version Linux test_bit.
+ *
+ * Test if bit is set in bitmap array.
+ */
+static bool test_bit_32(int bit, const uint32_t *addr) { return 0 != (addr[bit >> 5] & (1 << (bit & 0x1f))); }
+
+/**
+ * get_major_version() - Get firmware major version.
+ *
+ * Return: Major version.
+ */
+static unsigned int get_major_version(const struct mvx_fw_bin *fw_bin)
+{
+	if (fw_bin->securevideo != false)
+		return fw_bin->secure.securefw->protocol.major;
+	else
+		return fw_bin->nonsecure.header->protocol_major;
+}
+
+/**
+ * get_minor_version() - Get firmware minor version.
+ *
+ * Return: Minor version.
+ */
+static unsigned int get_minor_version(const struct mvx_fw_bin *fw_bin)
+{
+	if (fw_bin->securevideo != false)
+		return fw_bin->secure.securefw->protocol.minor;
+	else
+		return fw_bin->nonsecure.header->protocol_minor;
+}
+
+/**
+ * fw_unmap() - Remove MMU mappings and release allocated memory.
+ */
+static void fw_unmap(struct mvx_fw *fw)
+{
+	unsigned int i;
+	uint32_t begin;
+	uint32_t end;
+	int ret;
+
+	if (fw->fw_bin->securevideo == false) {
+		/* Unmap a region of 4 MB for each core. */
+		for (i = 0; i < fw->ncores; i++) {
+			ret = fw->ops.get_region(MVX_FW_REGION_CORE_0 + i, &begin, &end);
+			if (ret == 0)
+				mvx_mmu_unmap_va(fw->mmu, begin, 4 * 1024 * 1024);
+		}
+
+		if (!IS_ERR_OR_NULL(fw->text))
+			mvx_mmu_free_pages(fw->text);
+
+		if (!IS_ERR_OR_NULL(fw->bss))
+			mvx_mmu_free_pages(fw->bss);
+
+		if (!IS_ERR_OR_NULL(fw->bss_shared))
+			mvx_mmu_free_pages(fw->bss_shared);
+	}
+
+	fw->ops.unmap_protocol(fw);
+}
+
+/**
+ * fw_map_core() - Map pages for the text and BSS segments for one core.
+ *
+ * This function assumes that the fw instance has been correctly allocated
+ * and instansiated and will therefor not make any NULL pointer checks. It
+ * assumes that all pointers - for example to the mmu or firmware binary - have
+ * been correctly set up.
+ */
+static int fw_map_core(struct mvx_fw *fw, unsigned int core)
+{
+	int ret;
+	const struct mvx_fw_header *header = fw->fw_bin->nonsecure.header;
+	mvx_mmu_va fw_base;
+	mvx_mmu_va end;
+	mvx_mmu_va va;
+	unsigned int i;
+	unsigned int bss_cnt = core * fw->fw_bin->nonsecure.bss_cnt;
+	unsigned int bss_scnt = 0;
+
+	/*
+	 * Get the base address where the pages for this cores should be
+	 * mapped.
+	 */
+	ret = fw->ops.get_region(MVX_FW_REGION_CORE_0 + core, &fw_base, &end);
+	if (ret != 0)
+		return ret;
+
+	/* Map text segment. */
+	ret = mvx_mmu_map_pages(fw->mmu, fw_base + FW_TEXT_BASE_ADDR, fw->text, MVX_ATTR_PRIVATE,
+				MVX_ACCESS_EXECUTABLE);
+	if (ret != 0)
+		return ret;
+
+	/* Map bss shared and private pages. */
+	va = header->bss_start_address;
+	for (i = 0; i < header->bss_bitmap_size; i++) {
+		if (va >= header->master_rw_start_address &&
+		    va < (header->master_rw_start_address + header->master_rw_size))
+			ret = mvx_mmu_map_pa(fw->mmu, fw_base + va, fw->bss_shared->pages[bss_scnt++], MVE_PAGE_SIZE,
+					     MVX_ATTR_PRIVATE, MVX_ACCESS_READ_WRITE);
+		else if (test_bit_32(i, header->bss_bitmap))
+			ret = mvx_mmu_map_pa(fw->mmu, fw_base + va, fw->bss->pages[bss_cnt++], MVE_PAGE_SIZE,
+					     MVX_ATTR_PRIVATE, MVX_ACCESS_READ_WRITE);
+
+		if (ret != 0)
+			return ret;
+
+		va += MVE_PAGE_SIZE;
+	}
+
+	return 0;
+}
+
+/**
+ * fw_map() - Map up MMU tables.
+ */
+static int fw_map(struct mvx_fw *fw)
+{
+	int ret;
+	unsigned int i;
+
+	if (fw->fw_bin->securevideo != false) {
+		/* Map MMU tables for each core. */
+		for (i = 0; i < fw->ncores; i++) {
+			mvx_mmu_va fw_base;
+			mvx_mmu_va end;
+			phys_addr_t l2 = fw->fw_bin->secure.securefw->l2pages + i * MVE_PAGE_SIZE;
+
+			ret = fw->ops.get_region(MVX_FW_REGION_CORE_0 + i, &fw_base, &end);
+			if (ret != 0)
+				return ret;
+
+			ret = mvx_mmu_map_l2(fw->mmu, fw_base, l2);
+			if (ret != 0)
+				goto unmap_fw;
+		}
+	} else {
+		const struct mvx_fw_bin *fw_bin = fw->fw_bin;
+		const struct mvx_fw_header *header = fw_bin->nonsecure.header;
+
+		/* Allocate memory for text segment. */
+		fw->text = mvx_mmu_alloc_pages(fw->dev, fw_bin->nonsecure.text_cnt, 0);
+		if (IS_ERR(fw->text))
+			return PTR_ERR(fw->text);
+
+		/* Allocate memory for BSS segment. */
+		fw->bss = mvx_mmu_alloc_pages(fw->dev, fw_bin->nonsecure.bss_cnt * fw->ncores, 0);
+		if (IS_ERR(fw->bss)) {
+			ret = PTR_ERR(fw->bss);
+			goto unmap_fw;
+		}
+
+		/* Allocate memory for BSS shared segment. */
+		fw->bss_shared = mvx_mmu_alloc_pages(fw->dev, fw_bin->nonsecure.sbss_cnt, 0);
+		if (IS_ERR(fw->bss_shared)) {
+			ret = PTR_ERR(fw->bss_shared);
+			goto unmap_fw;
+		}
+
+		/* Map MMU tables for each core. */
+		for (i = 0; i < fw->ncores; i++) {
+			ret = fw_map_core(fw, i);
+			if (ret != 0)
+				goto unmap_fw;
+		}
+
+		/* Copy firmware binary. */
+		ret = mvx_mmu_write(fw->mmu, FW_TEXT_BASE_ADDR, fw_bin->nonsecure.fw->data, header->text_length);
+		if (ret != 0)
+			goto unmap_fw;
+	}
+
+	/* Map MMU tables for the message queues. */
+	ret = fw->ops.map_protocol(fw);
+	if (ret != 0)
+		goto unmap_fw;
+
+	return 0;
+
+unmap_fw:
+	fw_unmap(fw);
+
+	return ret;
+}
+
+/**
+ * Callbacks and handlers for FW stats.
+ */
+static int fw_stat_show(struct seq_file *s, void *v)
+{
+	struct mvx_fw *fw = (struct mvx_fw *)s->private;
+	const struct mvx_fw_bin *fw_bin = fw->fw_bin;
+
+	mvx_seq_printf(s, "mvx_fw", 0, "%p\n", fw);
+	seq_puts(s, "\n");
+
+	mvx_seq_printf(s, "mmu", 0, "%p\n", fw->mmu);
+
+	if (fw_bin->securevideo == false) {
+		mvx_seq_printf(s, "text", 0, "%p\n", fw->text);
+		mvx_seq_printf(s, "bss", 0, "%p\n", fw->bss);
+		mvx_seq_printf(s, "bss_shared", 0, "%p\n", fw->bss_shared);
+	}
+
+	seq_puts(s, "\n");
+
+	mvx_seq_printf(s, "msg_host", 0, "%p\n", fw->msg_host);
+	mvx_seq_printf(s, "msg_mve", 0, "%p\n", fw->msg_mve);
+	mvx_seq_printf(s, "buf_in_host", 0, "%p\n", fw->buf_in_host);
+	mvx_seq_printf(s, "buf_in_mve", 0, "%p\n", fw->buf_in_mve);
+	mvx_seq_printf(s, "buf_out_host", 0, "%p\n", fw->buf_out_host);
+	mvx_seq_printf(s, "buf_out_mve", 0, "%p\n", fw->buf_out_mve);
+	seq_puts(s, "\n");
+
+	fw->ops.print_stat(fw, 0, s);
+	seq_puts(s, "\n");
+
+	mvx_seq_printf(s, "rpc", 0, "%p\n", fw->rpc);
+	mvx_seq_printf(s, "ncores", 0, "%u\n", fw->ncores);
+	mvx_seq_printf(s, "msg_pending", 0, "%u\n", fw->msg_pending);
+	seq_puts(s, "\n");
+
+	mvx_seq_printf(s, "ops.map_protocol", 0, "%ps\n", fw->ops.map_protocol);
+	mvx_seq_printf(s, "ops.unmap_protocol", 0, "%ps\n", fw->ops.unmap_protocol);
+	mvx_seq_printf(s, "ops.get_region", 0, "%ps\n", fw->ops.get_region);
+	mvx_seq_printf(s, "ops.get_message", 0, "%ps\n", fw->ops.get_message);
+	mvx_seq_printf(s, "ops.put_message", 0, "%ps\n", fw->ops.put_message);
+	mvx_seq_printf(s, "ops.handle_rpc", 0, "%ps\n", fw->ops.handle_rpc);
+	seq_puts(s, "\n");
+
+	mvx_seq_printf(s, "fw_bin", 0, "%p\n", fw_bin);
+	mvx_seq_printf(s, "fw_bin.cache", 0, "%p\n", fw_bin->cache);
+	mvx_seq_printf(s, "fw_bin.filename", 0, "%s\n", fw_bin->filename);
+	mvx_seq_printf(s, "fw_bin.format", 0, "%u\n", fw_bin->format);
+	mvx_seq_printf(s, "fw_bin.dir", 0, "%s\n",
+		       (fw_bin->dir == MVX_DIR_INPUT)	 ? "in"
+		       : (fw_bin->dir == MVX_DIR_OUTPUT) ? "out"
+							 : "invalid");
+
+	if (fw_bin->securevideo == false) {
+		mvx_seq_printf(s, "fw_bin.text_cnt", 0, "%u\n", fw_bin->nonsecure.text_cnt);
+		mvx_seq_printf(s, "fw_bin.bss_cnt", 0, "%u\n", fw_bin->nonsecure.bss_cnt);
+		mvx_seq_printf(s, "fw_bin.sbss_cnt", 0, "%u\n", fw_bin->nonsecure.sbss_cnt);
+	}
+
+	return 0;
+}
+
+static int fw_stat_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, fw_stat_show, inode->i_private);
+}
+
+static const struct file_operations fw_stat_fops = {
+	.open = fw_stat_open, .read = seq_read, .llseek = seq_lseek, .release = single_release};
+
+static void *rpcmem_seq_start(struct seq_file *s, loff_t *pos)
+{
+	struct mvx_fw *fw = s->private;
+	int ret;
+
+	ret = mutex_lock_interruptible(&fw->rpcmem_mutex);
+	if (ret != 0)
+		return ERR_PTR(-EINVAL);
+
+	return mvx_seq_hash_start(fw->dev, fw->rpc_mem, HASH_SIZE(fw->rpc_mem), *pos);
+}
+
+static void *rpcmem_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	struct mvx_fw *fw = s->private;
+
+	return mvx_seq_hash_next(v, fw->rpc_mem, HASH_SIZE(fw->rpc_mem), pos);
+}
+
+static void rpcmem_seq_stop(struct seq_file *s, void *v)
+{
+	struct mvx_fw *fw = s->private;
+
+	mutex_unlock(&fw->rpcmem_mutex);
+	mvx_seq_hash_stop(v);
+}
+
+static int rpcmem_seq_show(struct seq_file *s, void *v)
+{
+	struct mvx_seq_hash_it *it = v;
+	struct mvx_mmu_pages *pages = hlist_entry(it->node, struct mvx_mmu_pages, node);
+
+	if (pages == NULL)
+		return 0;
+
+	seq_printf(s, "va = %08x, cap = %08zu, count = %08zu\n", pages->va, pages->capacity, pages->count);
+
+	return 0;
+}
+
+static const struct seq_operations rpcmem_seq_ops = {
+	.start = rpcmem_seq_start,
+	.next  = rpcmem_seq_next,
+	.stop  = rpcmem_seq_stop,
+	.show  = rpcmem_seq_show
+};
+
+static int rpcmem_open(struct inode *inode, struct file *file)
+{
+	int ret;
+	struct seq_file *s;
+	struct mvx_fw *fw = inode->i_private;
+
+	ret = seq_open(file, &rpcmem_seq_ops);
+	if (ret != 0)
+		return ret;
+
+	s = file->private_data;
+	s->private = fw;
+
+	return 0;
+}
+
+static const struct file_operations rpcmem_fops = {
+	.open    = rpcmem_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = seq_release
+};
+
+/**
+ * fw_debugfs_init() - Create debugfs entries for mvx_fw.
+ */
+static int fw_debugfs_init(struct mvx_fw *fw, struct dentry *parent)
+{
+	int ret;
+	struct dentry *dentry;
+
+	fw->dentry = debugfs_create_dir("fw", parent);
+	if (IS_ERR_OR_NULL(fw->dentry))
+		return -ENOMEM;
+
+	dentry = debugfs_create_file("stat", 0400, fw->dentry, fw, &fw_stat_fops);
+	if (IS_ERR_OR_NULL(dentry)) {
+		ret = -ENOMEM;
+		goto remove_dentry;
+	}
+
+	if (fw->fw_bin->securevideo == false) {
+		ret = mvx_mmu_pages_debugfs_init(fw->text, "text", fw->dentry);
+		if (ret != 0)
+			goto remove_dentry;
+
+		ret = mvx_mmu_pages_debugfs_init(fw->bss, "bss", fw->dentry);
+		if (ret != 0)
+			goto remove_dentry;
+
+		ret = mvx_mmu_pages_debugfs_init(fw->bss_shared, "bss_shared", fw->dentry);
+		if (ret != 0)
+			goto remove_dentry;
+
+		dentry = debugfs_create_file("rpc_mem", 0400, fw->dentry, fw, &rpcmem_fops);
+		if (IS_ERR_OR_NULL(dentry)) {
+			ret = -ENOMEM;
+			goto remove_dentry;
+		}
+	}
+
+	return 0;
+
+remove_dentry:
+	debugfs_remove_recursive(fw->dentry);
+	return ret;
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_fw_factory(struct mvx_fw *fw,
+		   struct mvx_fw_bin *fw_bin,
+		   struct mvx_mmu *mmu,
+		   struct mvx_session *session,
+		   struct mvx_client_ops *client_ops,
+		   struct mvx_client_session *csession,
+		   unsigned int ncores,
+		   struct dentry *parent)
+{
+	unsigned int major;
+	unsigned int minor;
+	int ret;
+
+	/* Verifty that firmware loading was successful. */
+	if ((fw_bin->securevideo == false && IS_ERR_OR_NULL(fw_bin->nonsecure.fw)) ||
+	    (fw_bin->securevideo != false && IS_ERR_OR_NULL(fw_bin->secure.securefw))) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Firmware binary was loaded with error.");
+		return -EINVAL;
+	}
+
+	if (fw_bin->securevideo != false && ncores > fw_bin->secure.securefw->ncores) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Too few secure cores setup. max_ncores=%u, ncores=%u.",
+			      fw_bin->secure.securefw->ncores, ncores);
+		return -EINVAL;
+	}
+
+	major = get_major_version(fw_bin);
+	minor = get_minor_version(fw_bin);
+
+	/* Call constructor for derived class based on protocol version. */
+	switch (major) {
+	case 2:
+		ret = mvx_fw_construct_v2(fw, fw_bin, mmu, session, client_ops, csession, ncores, major, minor);
+		if (ret != 0)
+			return ret;
+
+		break;
+	case 3:
+		ret = mvx_fw_construct_v3(fw, fw_bin, mmu, session, client_ops, csession, ncores, major, minor);
+		if (ret != 0)
+			return ret;
+
+		break;
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "Unsupported firmware interface revision. major=%u, minor=%u.", major, minor);
+		return -EINVAL;
+	}
+
+	/* Map up the MMU tables. */
+	ret = fw_map(fw);
+	if (ret != 0)
+		return ret;
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		ret = fw_debugfs_init(fw, parent);
+
+	return ret;
+}
+
+int mvx_fw_construct(struct mvx_fw *fw,
+		     struct mvx_fw_bin *fw_bin,
+		     struct mvx_mmu *mmu,
+		     struct mvx_session *session,
+		     struct mvx_client_ops *client_ops,
+		     struct mvx_client_session *csession,
+		     unsigned int ncores)
+{
+	memset(fw, 0, sizeof(*fw));
+
+	fw->dev = fw_bin->dev;
+	fw->mmu = mmu;
+	fw->session = session;
+	fw->client_ops = client_ops;
+	fw->csession = csession;
+	fw->ncores = ncores;
+	fw->fw_bin = fw_bin;
+	mutex_init(&fw->rpcmem_mutex);
+
+	return 0;
+}
+
+void mvx_fw_destruct(struct mvx_fw *fw)
+{
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(fw->dentry);
+
+	/* Release and unmap allocates pages. */
+	fw_unmap(fw);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware.h
new file mode 100644
index 000000000000..4a011e42e66d
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware.h
@@ -0,0 +1,905 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_FIRMWARE_H_
+#define _MVX_FIRMWARE_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/hashtable.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/seq_file.h>
+#include "mvx_if.h"
+#include "mvx_buffer.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define MVX_FW_HTABLE_BITS 3
+#define MVX_FW_QUANT_LEN 64
+
+/****************************************************************************
+ * Firmware communication types
+ ****************************************************************************/
+
+/**
+ * enum mvx_fw_buffer_attr
+ */
+enum mvx_fw_buffer_attr {
+	MVX_FW_BUF_CACHEABLE,
+	MVX_FW_BUF_UNCACHEABLE,
+	MVX_FW_BUF_ATTR_NUM,
+};
+
+/**
+ * enum mvx_fw_state - Firmware state.
+ */
+enum mvx_fw_state {
+	MVX_FW_STATE_STOPPED,
+	MVX_FW_STATE_RUNNING
+};
+
+/**
+ * struct mvx_fw_job - Job request.
+ * @cores:	Number for cores to use.
+ * @frames:	Number of frames to process before job is switched out.
+ */
+struct mvx_fw_job {
+	unsigned int cores;
+	unsigned int frames;
+};
+
+/**
+ * struct mvx_fw_qp_range - QP range.
+ */
+struct mvx_fw_qp_range {
+	int min;
+	int max;
+};
+
+/**
+ * struct mvx_fw_profile_level - Profile and level.
+ */
+struct mvx_fw_profile_level {
+	unsigned int profile;
+	unsigned int level;
+};
+
+/**
+ * struct mvx_fw_tile - Tile size.
+ */
+struct mvx_fw_tile {
+	unsigned int rows;
+	unsigned int cols;
+};
+
+/**
+ * struct mvx_fw_mv - Motion vector search range.
+ */
+struct mvx_fw_mv {
+	unsigned int x;
+	unsigned int y;
+};
+
+/**
+ * struct mvx_fw_bitdepth - Bit depth.
+ */
+struct mvx_fw_bitdepth {
+	unsigned int chroma;
+	unsigned int luma;
+};
+
+struct mvx_buffer_param_region {
+	uint16_t mbx_left;   /**< X coordinate of the left most macroblock */
+	uint16_t mbx_right;  /**< X coordinate of the right most macroblock */
+	uint16_t mby_top;    /**< Y coordinate of the top most macroblock */
+	uint16_t mby_bottom; /**< Y coordinate of the bottom most macroblock */
+	int16_t qp_delta;    /**< QP delta value. This region will be encoded
+			      *   with qp = qp_default + qp_delta. */
+};
+
+struct mvx_roi_config {
+	unsigned int pic_index;
+	unsigned char qp_present;
+	unsigned char qp;
+	unsigned char roi_present;
+	unsigned char num_roi;
+#define MVX_MAX_FRAME_REGIONS 16
+	struct mvx_buffer_param_region roi[MVX_MAX_FRAME_REGIONS];
+};
+
+struct mvx_buffer_param_rate_control {
+	uint32_t rate_control_mode;
+#define MVX_OPT_RATE_CONTROL_MODE_OFF (0)
+#define MVX_OPT_RATE_CONTROL_MODE_STANDARD (1)
+#define MVX_OPT_RATE_CONTROL_MODE_VARIABLE (2)
+#define MVX_OPT_RATE_CONTROL_MODE_CONSTANT (3)
+#define MVX_OPT_RATE_CONTROL_MODE_C_VARIABLE (4)
+	uint32_t target_bitrate;  /* in bits per second */
+	uint32_t maximum_bitrate; /* in bits per second */
+};
+
+struct mvx_dsl_frame {
+	uint32_t width;
+	uint32_t height;
+};
+
+struct mvx_dsl_ratio {
+	uint32_t hor;
+	uint32_t ver;
+};
+
+struct mvx_long_term_ref {
+	uint32_t mode;
+	uint32_t period;
+};
+/**
+ * struct mvx_fw_error - Firmware error message.
+ * @error_code:	What kind of error that was reported.
+ * @message:	Error message string.
+ */
+struct mvx_fw_error {
+	enum {
+		MVX_FW_ERROR_ABORT,
+		MVX_FW_ERROR_OUT_OF_MEMORY,
+		MVX_FW_ERROR_ASSERT,
+		MVX_FW_ERROR_UNSUPPORTED,
+		MVX_FW_ERROR_INVALID_BUFFER,
+		MVX_FW_ERROR_INVALID_STATE,
+		MVX_FW_ERROR_WATCHDOG
+	} error_code;
+	char message[128];
+};
+
+/**
+ * struct mvx_fw_flush - Flush firmware buffers.
+ * @dir:	Which port to flush.
+ */
+struct mvx_fw_flush {
+	enum mvx_direction dir;
+};
+
+/**
+ * struct mvx_fw_alloc_param - Allocation parameters.
+ * @width:		Width in pixels.
+ * @height:		Height in pixels.
+ * @afbc_alloc_bytes:	AFBC buffer size.
+ * @afbc_width:		AFBC width in superblocks.
+ *
+ * Dimensions of a decoded frame buffer.
+ */
+struct mvx_fw_alloc_param {
+	unsigned int width;
+	unsigned int height;
+	unsigned int afbc_alloc_bytes;
+	unsigned int afbc_width;
+};
+
+/**
+ * struct mvx_fw_seq_param - Sequence parameters.
+ * @planar.buffers_min:	Minimum number of planar buffers required.
+ * @afbc.buffers_min:	Minimum number of AFBC buffers required.
+ */
+struct mvx_fw_seq_param {
+	struct {
+		unsigned int buffers_min;
+	} planar;
+	struct {
+		unsigned int buffers_min;
+	} afbc;
+};
+
+enum mvx_fw_range {
+	MVX_FW_RANGE_UNSPECIFIED,
+	MVX_FW_RANGE_FULL,
+	MVX_FW_RANGE_LIMITED
+};
+
+enum mvx_fw_primaries {
+	MVX_FW_PRIMARIES_UNSPECIFIED,
+	MVX_FW_PRIMARIES_BT709,	       /* Rec.ITU-R BT.709 */
+	MVX_FW_PRIMARIES_BT470M,       /* Rec.ITU-R BT.470 System M */
+	MVX_FW_PRIMARIES_BT601_625,    /* Rec.ITU-R BT.601 625 */
+	MVX_FW_PRIMARIES_BT601_525,    /* Rec.ITU-R BT.601 525 */
+	MVX_FW_PRIMARIES_GENERIC_FILM, /* Generic Film */
+	MVX_FW_PRIMARIES_BT2020	       /* Rec.ITU-R BT.2020 */
+};
+
+enum mvx_fw_transfer {
+	MVX_FW_TRANSFER_UNSPECIFIED,
+	MVX_FW_TRANSFER_LINEAR,	   /* Linear transfer characteristics */
+	MVX_FW_TRANSFER_SRGB,	   /* sRGB or equivalent */
+	MVX_FW_TRANSFER_SMPTE170M, /* SMPTE 170M  */
+	MVX_FW_TRANSFER_GAMMA22,   /* Assumed display gamma 2.2 */
+	MVX_FW_TRANSFER_GAMMA28,   /* Assumed display gamma 2.8 */
+	MVX_FW_TRANSFER_ST2084,	   /* SMPTE ST 2084 */
+	MVX_FW_TRANSFER_HLG,	   /* ARIB STD-B67 hybrid-log-gamma */
+	MVX_FW_TRANSFER_SMPTE240M, /* SMPTE 240M */
+	MVX_FW_TRANSFER_XVYCC,	   /* IEC 61966-2-4 */
+	MVX_FW_TRANSFER_BT1361,	   /* Rec.ITU-R BT.1361 extended gamut */
+	MVX_FW_TRANSFER_ST428	   /* SMPTE ST 428-1 */
+};
+
+enum mvx_fw_matrix {
+	MVX_FW_MATRIX_UNSPECIFIED,
+	MVX_FW_MATRIX_BT709,	     /* Rec.ITU-R BT.709 */
+	MVX_FW_MATRIX_BT470M,	     /* KR=0.30, KB=0.11 */
+	MVX_FW_MATRIX_BT601,	     /* Rec.ITU-R BT.601 625 */
+	MVX_FW_MATRIX_SMPTE240M,     /* SMPTE 240M or equivalent */
+	MVX_FW_MATRIX_BT2020,	     /* Rec.ITU-R BT.2020 non-const lum */
+	MVX_FW_MATRIX_BT2020Constant /* Rec.ITU-R BT.2020 const lum */
+};
+
+struct mvx_fw_primary {
+	unsigned int x;
+	unsigned int y;
+};
+
+/**
+ * struct mvx_fw_color_desc - HDR color description.
+ */
+struct mvx_fw_color_desc {
+	unsigned int flags;
+	enum mvx_fw_range range;
+	enum mvx_fw_primaries primaries;
+	enum mvx_fw_transfer transfer;
+	enum mvx_fw_matrix matrix;
+	struct {
+		struct mvx_fw_primary r;
+		struct mvx_fw_primary g;
+		struct mvx_fw_primary b;
+		struct mvx_fw_primary w;
+		unsigned int luminance_min;
+		unsigned int luminance_max;
+	} display;
+	struct {
+		unsigned int luminance_max;
+		unsigned int luminance_average;
+	} content;
+
+	uint8_t video_format;
+	uint8_t aspect_ratio_idc;
+	uint16_t sar_width;
+	uint16_t sar_height;
+	uint32_t num_units_in_tick;
+	uint32_t time_scale;
+};
+
+struct mvx_sei_userdata {
+	uint8_t flags;
+#define MVX_BUFFER_PARAM_USER_DATA_UNREGISTERED_VALID (1)
+	uint8_t uuid[16];
+	char user_data[256 - 35];
+	uint8_t user_data_len;
+};
+/**
+ * struct mvx_fw_set_option - Set firmware options.
+ */
+struct mvx_fw_set_option {
+	enum {
+		/**
+		 * Frame rate.
+		 * Extra data: frame_date.
+		 */
+		MVX_FW_SET_FRAME_RATE,
+
+		/**
+		 * Bitrate.
+		 * Extra data: target_bitrate.
+		 *
+		 * When target_bitrate is other than zero, rate control
+		 * in HW is enabled, otherwise rate control is disabled.
+		 */
+		MVX_FW_SET_TARGET_BITRATE,
+
+		/**
+		 * QP range.
+		 * Extra data: qp_range.
+		 *
+		 * QP range when rate controller is enabled.
+		 */
+		MVX_FW_SET_QP_RANGE,
+
+		/**
+		 * NALU format.
+		 * Extra data: nalu_format.
+		 */
+		MVX_FW_SET_NALU_FORMAT,
+
+		/**
+		 * Defines if stream escaping is enabled.
+		 * Extra data: stream_escaping.
+		 */
+		MVX_FW_SET_STREAM_ESCAPING,
+
+		/**
+		 * Defines profile and level for encoder.
+		 * Extra data: profile_level.
+		 */
+		MVX_FW_SET_PROFILE_LEVEL,
+
+		/**
+		 * Ignore stream headers.
+		 * Extra data: ignore_stream_headers.
+		 */
+		MVX_FW_SET_IGNORE_STREAM_HEADERS,
+
+		/**
+		 * Enable frame reordering for decoder.
+		 * Extra data: frame_reordering.
+		 */
+		MVX_FW_SET_FRAME_REORDERING,
+
+		/**
+		 * Suggested internal buffer size.
+		 * Extra data: intbuf_size.
+		 */
+		MVX_FW_SET_INTBUF_SIZE,
+
+		/**
+		 * Number of P frames for encoder.
+		 * Extra data: pb_frames.
+		 */
+		MVX_FW_SET_P_FRAMES,
+
+		/**
+		 * Number of B frames for encoder.
+		 * Extra data: pb_frames.
+		 */
+		MVX_FW_SET_B_FRAMES,
+
+		/**
+		 * GOP type for encoder.
+		 * Extra data: gop_type.
+		 */
+		MVX_FW_SET_GOP_TYPE,
+
+		/**
+		 * Intra MB refresh.
+		 * Extra data: intra_mb_refresh.
+		 */
+		MVX_FW_SET_INTRA_MB_REFRESH,
+
+		/**
+		 * Constrained intra prediction.
+		 * Extra data: constr_ipred.
+		 */
+		MVX_FW_SET_CONSTR_IPRED,
+
+		/**
+		 * Enable entropy synchronization.
+		 * Extra data: entropy_sync.
+		 */
+		MVX_FW_SET_ENTROPY_SYNC,
+
+		/**
+		 * Enable temporal motion vector prediction.
+		 * Extra data: temporal_mvp.
+		 */
+		MVX_FW_SET_TEMPORAL_MVP,
+
+		/**
+		 * Tiles size.
+		 * Extra data: tile.
+		 */
+		MVX_FW_SET_TILES,
+
+		/**
+		 * Minimum luma coding block size.
+		 * Extra data: min_luma_cb_size.
+		 */
+		MVX_FW_SET_MIN_LUMA_CB_SIZE,
+
+		/**
+		 * Entropy mode.
+		 * Extra data: entropy_mode.
+		 */
+		MVX_FW_SET_ENTROPY_MODE,
+
+		/**
+		 * Suggested number of CTUs in a slice.
+		 * Extra data: slice_spacing_mb.
+		 */
+		MVX_FW_SET_SLICE_SPACING_MB,
+
+		/**
+		 * Probability update method.
+		 * Extra data: vp9_prob_update.
+		 */
+		MVX_FW_SET_VP9_PROB_UPDATE,
+
+		/**
+		 * Search range for motion vectors.
+		 * Extra data: mv.
+		 */
+		MVX_FW_SET_MV_SEARCH_RANGE,
+
+		/**
+		 * Bitdepth.
+		 * Extra data: bitdepth.
+		 */
+		MVX_FW_SET_BITDEPTH,
+
+		/**
+		 * Chroma format.
+		 * Extra data: chroma_format.
+		 */
+		MVX_FW_SET_CHROMA_FORMAT,
+
+		/**
+		 * RGB to YUV conversion mode.
+		 * Extra data: rgb_to_yuv_mode.
+		 */
+		MVX_FW_SET_RGB_TO_YUV_MODE,
+
+		/**
+		 * Maximum bandwidth limit.
+		 * Extra data: band_limit.
+		 */
+		MVX_FW_SET_BAND_LIMIT,
+
+		/**
+		 * CABAC initialization table.
+		 * Extra data: cabac_init_idc.
+		 */
+		MVX_FW_SET_CABAC_INIT_IDC,
+
+		/**
+		 * QP for I frames when rate control is disabled.
+		 * Extra data: qp
+		 */
+		MVX_FW_SET_QP_I,
+
+		/**
+		 * QP for P frames when rate control is disabled.
+		 * Extra data: qp
+		 */
+		MVX_FW_SET_QP_P,
+
+		/**
+		 * QP for B frames when rate control is disabled.
+		 * Extra data: qp
+		 */
+		MVX_FW_SET_QP_B,
+
+		/**
+		 * JPEG resync interval.
+		 * Extra data: resync_interval
+		 */
+		MVX_FW_SET_RESYNC_INTERVAL,
+
+		/**
+		 * JPEG quantization table.
+		 * Extra data: quant_tbl.
+		 */
+		MVX_FW_SET_QUANT_TABLE,
+
+		/**
+		 * Set watchdog timeout. 0 to disable.
+		 */
+		MVX_FW_SET_WATCHDOG_TIMEOUT,
+
+		/**
+		 * QP for encode frame.
+		 * Extra data: qp
+		 */
+		MVX_FW_SET_QP_REGION,
+
+		/**
+		 * ROI for encode frame.
+		 * Extra data: ROI
+		 */
+		MVX_FW_SET_ROI_REGIONS,
+
+		/**
+		 * Rate Control for encode frame.
+		 * Extra data: rate control
+		 */
+		MVX_FW_SET_RATE_CONTROL,
+		/**
+		 * Crop left for encode frame.
+		 * Extra data: crop left
+		 */
+		MVX_FW_SET_CROP_LEFT,
+		/**
+		 * Crop right for encode frame.
+		 * Extra data: crop right
+		 */
+		MVX_FW_SET_CROP_RIGHT,
+		/**
+		 * Crop top for encode frame.
+		 * Extra data: crop top
+		 */
+		MVX_FW_SET_CROP_TOP,
+		/**
+		 * Crop bottom for encode frame.
+		 * Extra data: crop bottom
+		 */
+		MVX_FW_SET_CROP_BOTTOM,
+
+		MVX_FW_SET_COLOUR_DESC,
+
+		MVX_FW_SET_SEI_USERDATA,
+
+		MVX_FW_SET_HRD_BUF_SIZE,
+
+		MVX_FW_SET_DSL_FRAME,
+
+		MVX_FW_SET_LONG_TERM_REF,
+
+		MVX_FW_SET_DSL_MODE,
+
+		MVX_FW_SET_GOP_RESET,
+
+		MVX_FW_SET_INDEX_PROFILING
+	} code;
+
+	/**
+	 * Extra data for an option.
+	 */
+	union {
+		unsigned int frame_rate;
+		unsigned int target_bitrate;
+		struct mvx_fw_qp_range qp_range;
+		enum mvx_nalu_format nalu_format;
+		bool stream_escaping;
+		struct mvx_fw_profile_level profile_level;
+		bool ignore_stream_headers;
+		bool frame_reordering;
+		unsigned int intbuf_size;
+		unsigned int pb_frames;
+		enum mvx_gop_type gop_type;
+		unsigned int intra_mb_refresh;
+		bool constr_ipred;
+		bool entropy_sync;
+		bool temporal_mvp;
+		struct mvx_fw_tile tile;
+		unsigned int min_luma_cb_size;
+		enum mvx_entropy_mode entropy_mode;
+		unsigned int slice_spacing_mb;
+		enum mvx_vp9_prob_update vp9_prob_update;
+		struct mvx_fw_mv mv;
+		struct mvx_fw_bitdepth bitdepth;
+		unsigned int chroma_format;
+		enum mvx_rgb_to_yuv_mode rgb_to_yuv_mode;
+		unsigned int band_limit;
+		unsigned int cabac_init_idc;
+		int qp;
+		int resync_interval;
+		struct {
+			uint8_t *chroma;
+			uint8_t *luma;
+		} quant_tbl;
+		int watchdog_timeout;
+		struct mvx_roi_config roi_config;
+		struct mvx_buffer_param_rate_control rate_control;
+		unsigned int crop_left;
+		unsigned int crop_right;
+		unsigned int crop_top;
+		unsigned int crop_bottom;
+		struct mvx_fw_color_desc colour_desc;
+		struct mvx_sei_userdata userdata;
+		unsigned int nHRDBufsize;
+		struct mvx_dsl_frame dsl_frame;
+		struct mvx_long_term_ref ltr;
+		int dsl_pos_mode;
+		int index_profiling;
+	};
+};
+#define MVX_FW_COLOR_DESC_DISPLAY_VALID 0x1
+#define MVX_FW_COLOR_DESC_CONTENT_VALID 0x2
+
+/**
+ * enum mvx_fw_code - Codes for messages sent between driver and firmware.
+ */
+enum mvx_fw_code {
+	MVX_FW_CODE_ALLOC_PARAM,     /* Driver <- Firmware. */
+	MVX_FW_CODE_BUFFER,	     /* Driver <-> Firmware. */
+	MVX_FW_CODE_ERROR,	     /* Driver <- Firmware. */
+	MVX_FW_CODE_IDLE,	     /* Driver <- Firmware. */
+	MVX_FW_CODE_FLUSH,	     /* Driver <-> Firmware. */
+	MVX_FW_CODE_JOB,	     /* Driver -> Firmware. */
+	MVX_FW_CODE_PING,	     /* Driver -> Firmware. */
+	MVX_FW_CODE_PONG,	     /* Driver <- Firmware. */
+	MVX_FW_CODE_SEQ_PARAM,	     /* Driver <- Firmware. */
+	MVX_FW_CODE_SET_OPTION,	     /* Driver <-> Firmware. */
+	MVX_FW_CODE_STATE_CHANGE,    /* Driver <-> Firmware. */
+	MVX_FW_CODE_SWITCH_IN,	     /* Driver <- Firmware. */
+	MVX_FW_CODE_SWITCH_OUT,	     /* Driver <-> Firmware. */
+	MVX_FW_CODE_IDLE_ACK,	     /* Driver -> Firmware. */
+	MVX_FW_CODE_EOS,	     /* Driver <-> Firmware. */
+	MVX_FW_CODE_COLOR_DESC,	     /* Driver <- Firmware. */
+	MVX_FW_CODE_DUMP,	     /* Driver <-> Firmware. */
+	MVX_FW_CODE_DEBUG,	     /* Driver <-> Firmware. */
+	MVX_FW_CODE_BUFFER_GENERAL,  /* Driver <-> Firmware. */
+	MVX_FW_CODE_UNKNOWN,	     /* Driver <- Firmware. */
+	MVX_FW_CODE_DPB_HELD_FRAMES, /* Driver <- Firmware. */
+	MVX_FW_CODE_MAX
+};
+
+/**
+ * struct mvx_fw_msg - Union of all message types.
+ */
+struct mvx_fw_msg {
+	enum mvx_fw_code code;
+	union {
+		enum mvx_fw_state state;
+		struct mvx_fw_job job;
+		struct mvx_fw_error error;
+		struct mvx_fw_set_option set_option;
+		struct mvx_fw_flush flush;
+		struct mvx_fw_alloc_param alloc_param;
+		struct mvx_fw_seq_param seq_param;
+		struct mvx_fw_color_desc color_desc;
+		struct mvx_buffer *buf;
+		uint32_t arg;
+		bool eos_is_frame;
+	};
+};
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct mvx_fw_bin;
+struct mvx_mmu;
+struct mvx_mmu_pages;
+struct mvx_session;
+
+/**
+ * enum mvx_fw_region - Firmware memory regions.
+ */
+enum mvx_fw_region {
+	MVX_FW_REGION_CORE_0,
+	MVX_FW_REGION_CORE_1,
+	MVX_FW_REGION_CORE_2,
+	MVX_FW_REGION_CORE_3,
+	MVX_FW_REGION_CORE_4,
+	MVX_FW_REGION_CORE_5,
+	MVX_FW_REGION_CORE_6,
+	MVX_FW_REGION_CORE_7,
+	MVX_FW_REGION_PROTECTED,
+	MVX_FW_REGION_FRAMEBUF,
+	MVX_FW_REGION_MSG_HOST,
+	MVX_FW_REGION_MSG_MVE,
+	MVX_FW_REGION_BUF_IN_HOST,
+	MVX_FW_REGION_BUF_IN_MVE,
+	MVX_FW_REGION_BUF_OUT_HOST,
+	MVX_FW_REGION_BUF_OUT_MVE,
+	MVX_FW_REGION_RPC,
+	MVX_FW_REGION_PRINT_RAM
+};
+
+/**
+ * struct mvx_fw - Firmware class.
+ * @dev:		Pointer to device.
+ * @fw_bin:		Pointer to firmware binary.
+ * @mmu:		Pointer to MMU object.
+ * @session:		Pointer to session.
+ * @client_ops:		Client operations.
+ * @csession:		Client session this firmware instance is connected to.
+ * @text:		Pages allocated for the text segment.
+ * @bss:		Pages allocated for the bss segment.
+ * @bss_shared:		Pages allocated for the shared bss segment.
+ * @dentry:		Debugfs entry for the "fw" directory.
+ * @msg_host:		Host message queue.
+ * @msg_mve:		MVE message queue.
+ * @buf_in_host:	Input buffer queue. Host enqueue filled buffers.
+ * @buf_in_mve:		Input buffer queue. MVE return empty buffers.
+ * @buf_out_host:	Output buffer queue. Host enqueue empty buffers.
+ * @buf_out_mve:	Out buffer queue. MVE return filled buffers.
+ * @rpc:		RPC communication area.
+ * @ncores:		Number of cores the firmware has been mapped for.
+ * @rpc_mem:		Keeps track of RPC allocated memory. Maps MVE virtual
+ *                      address to 'struct mvx_mmu_pages' object.
+ * @msg_pending:	A subset of the messages that we are waiting for a
+ *                      response to.
+ * @ops:		Public firmware interface.
+ * @ops_priv:		Private firmware interface.
+ *
+ * There is one firmware instance per active session. The function pointers
+ * below are not reentrant and should be protected by the session mutex.
+ */
+struct mvx_fw {
+	struct device *dev;
+	const struct mvx_fw_bin *fw_bin;
+	struct mvx_mmu *mmu;
+	struct mvx_session *session;
+	struct mvx_client_ops *client_ops;
+	struct mvx_client_session *csession;
+	struct mvx_mmu_pages *text;
+	struct mvx_mmu_pages *bss;
+	struct mvx_mmu_pages *bss_shared;
+	struct dentry *dentry;
+	void *msg_host;
+	void *msg_mve;
+	void *buf_in_host;
+	void *buf_in_mve;
+	void *buf_out_host;
+	void *buf_out_mve;
+	void *rpc;
+	void *fw_print_ram;
+	unsigned int ncores;
+	DECLARE_HASHTABLE(rpc_mem, MVX_FW_HTABLE_BITS);
+	struct mutex rpcmem_mutex;
+	unsigned int msg_pending;
+	uint32_t latest_used_region_protected_pages;
+	uint32_t latest_used_region_outbuf_pages;
+	phys_addr_t buf_pa_addr[MVX_FW_REGION_PRINT_RAM + 1];
+	enum mvx_fw_buffer_attr buf_attr[MVX_FW_REGION_PRINT_RAM + 1];
+
+	int (*map_op[MVX_FW_BUF_ATTR_NUM])(struct mvx_fw *fw, void **data, enum mvx_fw_region region);
+	void (*unmap_op[MVX_FW_BUF_ATTR_NUM])(struct mvx_fw *fw, void **data, enum mvx_fw_region region);
+
+	struct {
+		/**
+		 * map_protocol() - MMU map firmware.
+		 * @fw:		Pointer to firmware object.
+		 */
+		int (*map_protocol)(struct mvx_fw *fw);
+
+		/**
+		 * unmap_protocol() - MMU unmap firmware.
+		 * @fw:		Pointer to firmware object.
+		 */
+		void (*unmap_protocol)(struct mvx_fw *fw);
+
+		/**
+		 * get_region() - Get begin and end address for memory region.
+		 * @region:	Which memory region to get addresses for.
+		 * @begin:	MVE virtual begin address.
+		 * @end:	MVE virtual end address.
+		 *
+		 * Return: 0 on success, else error code.
+		 */
+		int (*get_region)(enum mvx_fw_region region, uint32_t *begin, uint32_t *end);
+
+		/**
+		 * get_message() - Read message from firmware message queue.
+		 * @fw:		Pointer to firmware object.
+		 * @msg:	Firmware message.
+		 *
+		 * Return: 1 if message was received, 0 if no message was
+		 *         received, else error code.
+		 */
+		int (*get_message)(struct mvx_fw *fw, struct mvx_fw_msg *msg);
+
+		/**
+		 * put_message() - Write message to firmware message queue.
+		 * @fw:		Pointer to firmware object.
+		 * @msg:	Firmware message.
+		 *
+		 * Return: 0 on success, else error code.
+		 */
+		int (*put_message)(struct mvx_fw *fw, struct mvx_fw_msg *msg);
+
+		/**
+		 * handle_rpc() - Handle RPC message.
+		 * @fw:		Pointer to firmware object.
+		 *
+		 * Return: 1 RPC message handled, 0 no RPC message,
+		 *         else error code.
+		 */
+		int (*handle_rpc)(struct mvx_fw *fw);
+
+		/**
+		 * handle_fw_ram_print() - Print firmware log from share ram.
+		 * @fw:		Pointer to firmware object.
+		 *
+		 * Return: 1 FW ram log printed, 0 no FW ram log printed,
+		 *         else error code.
+		 */
+		int (*handle_fw_ram_print)(struct mvx_fw *fw);
+
+		/**
+		 * print_stat() - Print debug stats to seq-file.
+		 * @fw:		Pointer to firmware object.
+		 * @ind:	Indentation level.
+		 * s:		Pointer to seq-file.
+		 *
+		 * Return: 0 on success, else error code.
+		 */
+		int (*print_stat)(struct mvx_fw *fw, int ind, struct seq_file *s);
+
+		/**
+		 * print_debug() - Print debug information.
+		 * @fw:		Pointer to firmware object.
+		 */
+		void (*print_debug)(struct mvx_fw *fw);
+	} ops;
+
+	struct {
+		/**
+		 * send_idle_ack() - Send IDLE ACK message.
+		 * @fw:		Pointer to firmware object.
+		 *
+		 * IDLE ACK message will be sent to the firmware if it is
+		 * supported by a host protocol, otherwise the call will be
+		 * ignored.
+		 *
+		 * Return: 0 on success, else error code.
+		 */
+		int (*send_idle_ack)(struct mvx_fw *fw);
+
+		/**
+		 * to_mve_profile() - Convert MVX profile to MVE value.
+		 * @mvx_profile:	MVX profile.
+		 * @mvx_profile:	MVE profile.
+		 *
+		 * Return: 0 on success, else error code.
+		 */
+		int (*to_mve_profile)(unsigned int mvx_profile, uint16_t *mve_profile);
+
+		/**
+		 * to_mve_level() - Convert MVX level to MVE value.
+		 * @mvx_level:	MVX level.
+		 * @mvx_level:	MVE level.
+		 *
+		 * Return: 0 on success, else error code.
+		 */
+		int (*to_mve_level)(unsigned int mvx_level, uint16_t *mve_level);
+	} ops_priv;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_fw_factory() - Construct a firmware object.
+ * @fw:		Pointer to fw.
+ * @fw:_bin	Pointer for firmware binary.
+ * @mmu:	Pointer to MMU instance.
+ * @session:	Pointer to session.
+ * @client_ops:	Pointer to client operations.
+ * @csession:	Client session this firmware instance is registered to.
+ * @ncores:	Number of cores to configure.
+ * @parent:	Debugfs entry for parent debugfs directory entry.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_factory(struct mvx_fw *fw, struct mvx_fw_bin *fw_bin, struct mvx_mmu *mmu, struct mvx_session *session,
+		   struct mvx_client_ops *client_ops, struct mvx_client_session *csession, unsigned int ncores,
+		   struct dentry *parent);
+
+/**
+ * mvx_fw_destruct() - Destroy firmware interface instance.
+ * @fw:		Pointer to fw.
+ */
+void mvx_fw_destruct(struct mvx_fw *fw);
+
+#endif /* _MVX_FIRMWARE_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_cache.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_cache.c
new file mode 100644
index 000000000000..7914cc777804
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_cache.c
@@ -0,0 +1,699 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/errno.h>
+#include <linux/gfp.h>
+#include <linux/mm.h>
+#include <linux/firmware.h>
+#include <linux/kthread.h>
+#include <linux/version.h>
+#include "mvx_log_group.h"
+#include "mvx_firmware_cache.h"
+#include "mvx_log_ram.h"
+#include "mvx_mmu.h"
+#include "mvx_secure.h"
+#include "mvx_seq.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define CACHE_CLEANUP_INTERVAL_MS 5000
+
+#define MVX_SECURE_NUMCORES 8
+
+/****************************************************************************
+ * Private functions
+ ****************************************************************************/
+
+/*
+ * Backwards compliance with older kernels.
+ */
+#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE)
+static unsigned int kref_read(const struct kref *kref) { return atomic_read(&kref->refcount); }
+
+#endif
+
+/**
+ * test_bit_32() - 32 bit version Linux test_bit.
+ *
+ * Test if bit is set in bitmap array.
+ */
+static bool test_bit_32(int bit, uint32_t *addr) { return 0 != (addr[bit >> 5] & (1 << (bit & 0x1f))); }
+
+/**
+ * hw_id_to_name() - Convert HW id to string
+ */
+static const char *hw_id_to_string(enum mvx_hw_id id)
+{
+	switch (id) {
+	case MVE_v500:
+		return "v500";
+	case MVE_v550:
+		return "v550";
+	case MVE_v61:
+		return "v61";
+	case MVE_v52_v76:
+		return "v52_v76";
+	default:
+		return "unknown";
+	}
+}
+
+/**
+ * get_fw_name() - Return the file name for the requested format and direction.
+ *
+ * This function will neither check if there is hardware support nor if the
+ * firmware binary is available on the file system.
+ */
+static int get_fw_name(char *filename, size_t size, enum mvx_format format, enum mvx_direction dir,
+		       struct mvx_hw_ver *hw_ver)
+{
+	const char *codec = NULL;
+	const char *enc_dec = (dir == MVX_DIR_INPUT) ? "dec" : "enc";
+	size_t n;
+
+	switch (format) {
+	case MVX_FORMAT_H263:
+		codec = "mpeg4";
+		break;
+	case MVX_FORMAT_H264:
+		codec = "h264";
+		break;
+	case MVX_FORMAT_HEVC:
+		codec = "hevc";
+		break;
+	case MVX_FORMAT_JPEG:
+		codec = "jpeg";
+		break;
+	case MVX_FORMAT_MPEG2:
+		codec = "mpeg2";
+		break;
+	case MVX_FORMAT_MPEG4:
+		codec = "mpeg4";
+		break;
+	case MVX_FORMAT_RV:
+		codec = "rv";
+		break;
+	case MVX_FORMAT_VC1:
+		codec = "vc1";
+		break;
+	case MVX_FORMAT_VP8:
+		codec = "vp8";
+		break;
+	case MVX_FORMAT_VP9:
+		codec = "vp9";
+		break;
+	case MVX_FORMAT_AVS2:
+		codec = "avs2";
+		break;
+	case MVX_FORMAT_AVS:
+		codec = "avs";
+		break;
+	default:
+		return -ENOENT;
+	}
+
+	n = snprintf(filename, size, "linlon-%s-%u-%u/%s%s.fwb", hw_id_to_string(hw_ver->id), hw_ver->revision,
+		     hw_ver->patch, codec, enc_dec);
+	if (n >= size)
+		return -ENOENT;
+
+	return 0;
+}
+
+static struct mvx_fw_bin *kobj_to_fw_bin(struct kobject *kobj) { return container_of(kobj, struct mvx_fw_bin, kobj); }
+
+/**
+ * fw_bin_destroy() - Destroy instance of firmware binary.
+ */
+static void fw_bin_destroy(struct kobject *kobj)
+{
+	struct mvx_fw_bin *fw_bin = kobj_to_fw_bin(kobj);
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "Releasing firmware binary. bin=0x%p.", fw_bin);
+
+	if (fw_bin->securevideo == false && IS_ERR_OR_NULL(fw_bin->nonsecure.fw) == false)
+		release_firmware(fw_bin->nonsecure.fw);
+
+	if (fw_bin->securevideo != false && IS_ERR_OR_NULL(fw_bin->secure.securefw) == false)
+		mvx_secure_release_firmware(fw_bin->secure.securefw);
+
+	list_del(&fw_bin->cache_head);
+	devm_kfree(fw_bin->dev, fw_bin);
+}
+
+/**
+ * fw_bin_validate() - Verify that the loaded firmware is a valid binary.
+ */
+static int fw_bin_validate(const struct firmware *fw, struct device *dev)
+{
+	struct mvx_fw_header *header = (struct mvx_fw_header *)fw->data;
+
+	if (fw->size < sizeof(*header)) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "Firmware binary size smaller than firmware header. size=%zu.", fw->size);
+		return -EFAULT;
+	}
+
+	if (header->text_length > fw->size) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "Firmware text length larger than firmware binary size. text_length=%u, size=%zu.",
+			      header->text_length, fw->size);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * fw_bin_callback() - Call firmware ready callback.
+ */
+static void fw_bin_callback(struct mvx_fw_bin *fw_bin)
+{
+	struct mvx_fw_event *event;
+	struct mvx_fw_event *tmp;
+	int ret;
+
+	/*
+	 * Continue even if lock fails, or else any waiting session will
+	 * be blocked forever.
+	 */
+	ret = mutex_lock_interruptible(&fw_bin->mutex);
+
+	/*
+	 * Inform all clients that the firmware has been loaded. This must be
+	 * done even if the firmware load fails, or else the clients will hung
+	 * waiting for a firmware load the will never happen.
+	 */
+	list_for_each_entry_safe(event, tmp, &fw_bin->event_list, head) {
+		list_del(&event->head);
+		event->fw_bin_ready(fw_bin, event->arg, false);
+	}
+
+	if (ret == 0)
+		mutex_unlock(&fw_bin->mutex);
+}
+
+/**
+ * secure_request_firmware_done() - Firmware load callback routine.
+ */
+static void secure_request_firmware_done(struct mvx_secure_firmware *securefw, void *arg)
+{
+	struct mvx_fw_bin *fw_bin = arg;
+
+	if (securefw == NULL) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to load secure firmware binary. filename=%s.",
+			      fw_bin->filename);
+		securefw = ERR_PTR(-EINVAL);
+		goto fw_bin_callback;
+	}
+
+fw_bin_callback:
+	fw_bin->secure.securefw = securefw;
+
+	fw_bin_callback(fw_bin);
+}
+
+/**
+ * request_firmware_done() - Callback routine after firmware has been loaded.
+ */
+static void request_firmware_done(const struct firmware *fw, void *arg)
+{
+	struct mvx_fw_bin *fw_bin = arg;
+	struct mvx_fw_header *header;
+	mvx_mmu_va va;
+	int ret;
+	uint32_t i;
+
+	BUG_ON(!arg);
+
+	if (fw == NULL) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to load firmware binary. filename=%s.",
+			      fw_bin->filename);
+		fw = ERR_PTR(-EINVAL);
+		goto fw_ready_callback;
+	}
+
+	ret = fw_bin_validate(fw, fw_bin->dev);
+	if (ret != 0) {
+		release_firmware(fw);
+		fw = ERR_PTR(ret);
+		goto fw_ready_callback;
+	}
+
+	header = (struct mvx_fw_header *)fw->data;
+	fw_bin->nonsecure.header = header;
+
+	/* Calculate number of pages needed for the text segment. */
+	fw_bin->nonsecure.text_cnt = (header->text_length + MVE_PAGE_SIZE - 1) >> MVE_PAGE_SHIFT;
+
+	/* Calculate number of pages needed for the BSS segments. */
+	va = header->bss_start_address;
+	for (i = 0; i < header->bss_bitmap_size; i++) {
+		if (va >= header->master_rw_start_address &&
+		    va < (header->master_rw_start_address + header->master_rw_size))
+			fw_bin->nonsecure.sbss_cnt++;
+		else if (test_bit_32(i, header->bss_bitmap))
+			fw_bin->nonsecure.bss_cnt++;
+
+		va += MVE_PAGE_SIZE;
+	}
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO,
+		      "Loaded firmware binary. bin=0x%p, major=%u, minor=%u, info=\"%s\", jump=0x%x, pages={text=%u, "
+		      "bss=%u, shared=%u}, text_length=%u, bss=0x%x.",
+		      fw_bin, header->protocol_major, header->protocol_minor, header->info_string, header->rasc_jmp,
+		      fw_bin->nonsecure.text_cnt, fw_bin->nonsecure.bss_cnt, fw_bin->nonsecure.sbss_cnt,
+		      header->text_length, header->bss_start_address);
+
+fw_ready_callback:
+	fw_bin->nonsecure.fw = fw;
+
+	fw_bin_callback(fw_bin);
+}
+
+/**
+ * hwvercmp() - Compare two hardware versions.
+ *
+ * Semantic of this function equivalent to strcmp().
+ */
+static int hwvercmp(struct mvx_hw_ver *v1, struct mvx_hw_ver *v2)
+{
+	if (v1->id != v2->id)
+		return v1->id - v2->id;
+
+	if (v1->revision != v2->revision)
+		return v1->revision - v2->revision;
+
+	if (v1->patch != v2->patch)
+		return v1->patch - v2->patch;
+
+	return 0;
+}
+
+static ssize_t path_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct mvx_fw_bin *fw_bin = kobj_to_fw_bin(kobj);
+
+	return scnprintf(buf, PAGE_SIZE, "%s\n", fw_bin->filename);
+}
+
+static ssize_t hw_ver_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct mvx_fw_bin *fw_bin = kobj_to_fw_bin(kobj);
+	struct mvx_hw_ver *hw_ver = &fw_bin->hw_ver;
+
+	return scnprintf(buf, PAGE_SIZE, "%s-%u-%u\n", hw_id_to_string(hw_ver->id), hw_ver->revision, hw_ver->patch);
+}
+
+static ssize_t count_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n", kref_read(&kobj->kref) - 1);
+}
+
+static ssize_t dirty_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct mvx_fw_bin *fw_bin = kobj_to_fw_bin(kobj);
+	int dirty = 0;
+
+	if (atomic_read(&fw_bin->flush_cnt) != atomic_read(&fw_bin->cache->flush_cnt))
+		dirty = 1;
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n", dirty);
+}
+
+static struct kobj_attribute path_attr = __ATTR_RO(path);
+static struct kobj_attribute count_attr = __ATTR_RO(count);
+static struct kobj_attribute hw_ver_attr = __ATTR_RO(hw_ver);
+static struct kobj_attribute dirty_attr = __ATTR_RO(dirty);
+
+static struct attribute *fw_bin_attrs[] = {&path_attr.attr, &count_attr.attr, &hw_ver_attr.attr, &dirty_attr.attr,
+					   NULL};
+
+static const struct attribute_group fw_bin_group = {.name = "", .attrs = fw_bin_attrs};
+
+static const struct attribute_group *fw_bin_groups[] = {&fw_bin_group, NULL};
+
+static struct kobj_type fw_bin_ktype = {
+	.release = fw_bin_destroy, .sysfs_ops = &kobj_sysfs_ops, .default_groups = fw_bin_groups};
+
+/**
+ * fw_bin_create() - Create a new firmware binary instance.
+ */
+static struct mvx_fw_bin *fw_bin_create(struct mvx_fw_cache *cache, enum mvx_format format, enum mvx_direction dir,
+					struct mvx_hw_ver *hw_ver, bool securevideo)
+{
+	struct mvx_fw_bin *fw_bin;
+	int ret;
+
+	/* Allocate object and initialize members. */
+	fw_bin = devm_kzalloc(cache->dev, sizeof(*fw_bin), GFP_KERNEL);
+	if (fw_bin == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	fw_bin->dev = cache->dev;
+	fw_bin->cache = cache;
+	fw_bin->format = format;
+	fw_bin->dir = dir;
+	fw_bin->hw_ver = *hw_ver;
+	atomic_set(&fw_bin->flush_cnt, atomic_read(&cache->flush_cnt));
+	mutex_init(&fw_bin->mutex);
+	INIT_LIST_HEAD(&fw_bin->cache_head);
+	INIT_LIST_HEAD(&fw_bin->event_list);
+
+	fw_bin->securevideo = securevideo;
+	if (securevideo != false)
+		fw_bin->secure.secure = cache->secure;
+
+	ret = kobject_init_and_add(&fw_bin->kobj, &fw_bin_ktype, &cache->kobj, "%lx", (unsigned long)fw_bin);
+	if (ret != 0)
+		goto free_fw_bin;
+
+	ret = get_fw_name(fw_bin->filename, sizeof(fw_bin->filename), format, dir, &fw_bin->hw_ver);
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "No firmware available. format=%d, direction=%d.", format,
+			      dir);
+		goto free_fw_bin;
+	}
+
+	kobject_get(&fw_bin->kobj);
+
+	if (securevideo != false)
+		ret = mvx_secure_request_firmware_nowait(cache->secure, fw_bin->filename, MVX_SECURE_NUMCORES, fw_bin,
+							 secure_request_firmware_done);
+	else
+		ret = request_firmware_nowait(THIS_MODULE, true, fw_bin->filename, fw_bin->dev, GFP_KERNEL, fw_bin,
+					      request_firmware_done);
+
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "Failed to request firmware. filename=%s, securevideo=%d.",
+			      fw_bin->filename, securevideo);
+		kobject_put(&fw_bin->kobj);
+		goto free_fw_bin;
+	}
+
+	return fw_bin;
+
+free_fw_bin:
+	kobject_put(&fw_bin->kobj);
+
+	return ERR_PTR(ret);
+}
+
+/**
+ * fw_bin_get() - Get reference to firmware binary.
+ *
+ * If firmware binary has already been loaded the reference count is increased,
+ * else the function tries to create a new descriptor and load the firmware
+ * into memory.
+ */
+static struct mvx_fw_bin *fw_bin_get(struct mvx_fw_cache *cache, enum mvx_format format, enum mvx_direction dir,
+				     struct mvx_hw_ver *hw_ver, bool securevideo)
+{
+	struct mvx_fw_bin *fw_bin = NULL;
+	struct mvx_fw_bin *tmp;
+	int ret;
+
+	ret = mutex_lock_interruptible(&cache->mutex);
+	if (ret != 0)
+		return ERR_PTR(ret);
+
+	/* Search if firmware binary has already been loaded. */
+	list_for_each_entry(tmp, &cache->fw_bin_list, cache_head)
+	{
+		if (tmp->format == format && tmp->dir == dir && hwvercmp(&tmp->hw_ver, hw_ver) == 0 &&
+		    tmp->securevideo == securevideo && atomic_read(&tmp->flush_cnt) == atomic_read(&cache->flush_cnt)) {
+			fw_bin = tmp;
+			break;
+		}
+	}
+
+	/* If firmware was not found, then try to request firmware. */
+	if (fw_bin == NULL) {
+		fw_bin = fw_bin_create(cache, format, dir, hw_ver, securevideo);
+		if (!IS_ERR(fw_bin))
+			list_add(&fw_bin->cache_head, &cache->fw_bin_list);
+	} else {
+		kobject_get(&fw_bin->kobj);
+	}
+
+	mutex_unlock(&cache->mutex);
+
+	return fw_bin;
+}
+
+/****************************************************************************
+ * Private functions
+ ****************************************************************************/
+
+static struct mvx_fw_cache *kobj_to_fw_cache(struct kobject *kobj)
+{
+	return container_of(kobj, struct mvx_fw_cache, kobj);
+}
+
+/**
+ * cache_flush_show() - FW cache flush status is always 0.
+ */
+static ssize_t cache_flush_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "0\n");
+}
+
+/**
+ * cache_flush_store() - Trigger FW cache flush.
+ */
+static ssize_t cache_flush_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t size)
+{
+	struct mvx_fw_cache *cache = kobj_to_fw_cache(kobj);
+
+	atomic_inc(&cache->flush_cnt);
+	return size;
+}
+
+/**
+ * Sysfs attribute which triggers FW cache flush.
+ */
+static struct kobj_attribute cache_flush = __ATTR(flush, 0600, cache_flush_show, cache_flush_store);
+
+static struct attribute *cache_attrs[] = {
+	&cache_flush.attr,
+	NULL
+};
+
+static const struct attribute_group cache_group = {
+    .name = "",
+    .attrs = cache_attrs
+};
+
+static const struct attribute_group *cache_groups[] = {
+    &cache_group,
+    NULL
+};
+
+static void cache_release(struct kobject *kobj)
+{
+	struct mvx_fw_cache *cache = kobj_to_fw_cache(kobj);
+
+	kthread_stop(cache->cache_thread);
+	kobject_put(cache->kobj_parent);
+}
+
+static struct kobj_type cache_ktype = {
+	.release       = cache_release,
+	.sysfs_ops     = &kobj_sysfs_ops,
+	.default_groups = cache_groups
+};
+
+static void cache_update(struct mvx_fw_cache *cache)
+{
+	struct mvx_fw_bin *fw_bin;
+	struct mvx_fw_bin *tmp;
+	int ret;
+
+	ret = mutex_lock_interruptible(&cache->mutex);
+	if (ret != 0)
+		return;
+
+	list_for_each_entry_safe(fw_bin, tmp, &cache->fw_bin_list, cache_head) {
+		int ref;
+
+		ref = kref_read(&fw_bin->kobj.kref);
+		if (ref == 1)
+			kobject_put(&fw_bin->kobj);
+	}
+
+	mutex_unlock(&cache->mutex);
+}
+
+static int cache_thread(void *v)
+{
+	struct mvx_fw_cache *cache = (struct mvx_fw_cache *)v;
+
+	while (!kthread_should_stop()) {
+		cache_update(cache);
+		msleep_interruptible(CACHE_CLEANUP_INTERVAL_MS);
+	}
+
+	return 0;
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_fw_cache_construct(struct mvx_fw_cache *cache, struct device *dev, struct mvx_secure *secure,
+			   struct kobject *kobj_parent)
+{
+	int ret;
+
+	cache->dev = dev;
+	cache->secure = secure;
+	cache->kobj_parent = kobject_get(kobj_parent);
+	atomic_set(&cache->flush_cnt, 0);
+	mutex_init(&cache->mutex);
+	INIT_LIST_HEAD(&cache->fw_bin_list);
+
+	ret = kobject_init_and_add(&cache->kobj, &cache_ktype, kobj_parent, "fw_cache");
+	if (ret != 0)
+		goto kobj_put;
+
+	cache->cache_thread = kthread_run(cache_thread, cache, "fw_cache");
+	if (IS_ERR(cache->cache_thread))
+		goto kobj_put;
+
+	return 0;
+
+kobj_put:
+	kobject_put(&cache->kobj);
+	kobject_put(cache->kobj_parent);
+	return -EFAULT;
+}
+
+void mvx_fw_cache_destruct(struct mvx_fw_cache *cache) { kobject_put(&cache->kobj); }
+
+int mvx_fw_cache_get(struct mvx_fw_cache *cache, enum mvx_format format, enum mvx_direction dir,
+		     struct mvx_fw_event *event, struct mvx_hw_ver *hw_ver, bool securevideo)
+{
+	int ret;
+	struct mvx_fw_bin *fw_bin;
+
+	/* Allocate a new firmware binary or get handle to existing object. */
+	fw_bin = fw_bin_get(cache, format, dir, hw_ver, securevideo);
+	if (IS_ERR(fw_bin))
+		return PTR_ERR(fw_bin);
+
+	ret = mutex_lock_interruptible(&fw_bin->mutex);
+	if (ret != 0) {
+		mvx_fw_cache_put(cache, fw_bin);
+		return ret;
+	}
+
+	/*
+	 * If the firmware binary has already been loaded, then the callback
+	 * routine can be called right away.
+	 * Else the callback and argument is enqueued to the firmware
+	 * notification list.
+	 */
+	if ((fw_bin->securevideo != false && IS_ERR_OR_NULL(fw_bin->secure.securefw) == false)) {
+		mutex_unlock(&fw_bin->mutex);
+		event->fw_bin_ready(fw_bin, event->arg, true);
+	} else if (fw_bin->securevideo == false && IS_ERR_OR_NULL(fw_bin->nonsecure.fw) == false) {
+		mutex_unlock(&fw_bin->mutex);
+		event->fw_bin_ready(fw_bin, event->arg, true);
+	} else {
+		list_add(&event->head, &fw_bin->event_list);
+		mutex_unlock(&fw_bin->mutex);
+	}
+
+	return 0;
+}
+
+void mvx_fw_cache_put(struct mvx_fw_cache *cache, struct mvx_fw_bin *fw_bin)
+{
+	int ret;
+
+	ret = mutex_lock_interruptible(&cache->mutex);
+
+	kobject_put(&fw_bin->kobj);
+
+	if (ret == 0)
+		mutex_unlock(&cache->mutex);
+}
+
+void mvx_fw_cache_log(struct mvx_fw_bin *fw_bin, struct mvx_client_session *csession)
+{
+	struct mvx_log_header header;
+	struct mvx_log_fw_binary fw_binary;
+	struct timespec64 timespec;
+	struct iovec vec[3];
+
+	if (fw_bin->securevideo != false)
+		return;
+
+	ktime_get_real_ts64(&timespec);
+
+	header.magic = MVX_LOG_MAGIC;
+	header.length = sizeof(fw_binary) + sizeof(*fw_bin->nonsecure.header);
+	header.type = MVX_LOG_TYPE_FW_BINARY;
+	header.severity = MVX_LOG_INFO;
+	header.timestamp.sec = timespec.tv_sec;
+	header.timestamp.nsec = timespec.tv_nsec;
+
+	fw_binary.session = (uintptr_t)csession;
+
+	vec[0].iov_base = &header;
+	vec[0].iov_len = sizeof(header);
+
+	vec[1].iov_base = &fw_binary;
+	vec[1].iov_len = sizeof(fw_binary);
+
+	vec[2].iov_base = (void *)fw_bin->nonsecure.header;
+	vec[2].iov_len = sizeof(*fw_bin->nonsecure.header);
+
+	MVX_LOG_DATA(&mvx_log_fwif_if, MVX_LOG_INFO, vec, 3);
+}
+
+void mvx_fw_cache_get_formats(struct mvx_fw_cache *cache, enum mvx_direction direction, uint64_t *formats)
+{
+	/* Support all formats by default. */
+	*formats = (1ull << MVX_FORMAT_MAX) - 1ull;
+
+	/* TODO remove formats we can't find any firmware for. */
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_cache.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_cache.h
new file mode 100644
index 000000000000..0f851d6e4eb3
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_cache.h
@@ -0,0 +1,234 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_FIRMWARE_CACHE_H_
+#define _MVX_FIRMWARE_CACHE_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/atomic.h>
+#include <linux/kobject.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include "mvx_if.h"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct firmware;
+struct mvx_client_session;
+struct mvx_secure;
+struct mvx_secure_firmware;
+
+/**
+ * struct mvx_fw_cache - Firmware cache.
+ *
+ * There is exactly one firmware context per device. It keeps track of the
+ * firmware binaries.
+ */
+struct mvx_fw_cache {
+	struct device *dev;
+	struct mvx_secure *secure;
+	struct mutex mutex;
+	struct list_head fw_bin_list;
+	struct kobject kobj;
+	struct kobject *kobj_parent;
+	atomic_t flush_cnt;
+	struct task_struct *cache_thread;
+};
+
+/**
+ * struct mvx_fw_header - Firmware binary header.
+ * @rasc_jmp:		Start address.
+ * @protocol_minor:	Host internface protocol minor version.
+ * @protocol_major:	Host internface protocol major version.
+ * @reserved:		Reserved for future use. Always 0.
+ * @info_string:	Human readable codec information.
+ * @part_number:	Part number.
+ * @svn_revision:	SVN revision.
+ * @version_string:	Firmware version.
+ * @text_length:	Length in bytes of the read-only part of the firmware.
+ * @bss_start_address:	Start address for BSS segment. This is always
+ *                      page-aligned.
+ * @bss_bitmap_size:	The number of bits used in 'bss_bitmap'.
+ * @bss_bitmap:		Bitmap which pages that shall be allocated and MMU
+ *                      mapped. If bit N is set, then a page shall be allocated
+ *                      and MMU mapped to VA address
+ *                      FW_BASE + bss_start_address + N * MVE_PAGE_SIZE.
+ * @master_rw_start_address: Defines a region of shared pages.
+ * @master_rw_size:	Defines a region of shared pages.
+ */
+struct mvx_fw_header {
+	uint32_t rasc_jmp;
+	uint8_t protocol_minor;
+	uint8_t protocol_major;
+	uint8_t reserved[2];
+	uint8_t info_string[56];
+	uint8_t part_number[8];
+	uint8_t svn_revision[8];
+	uint8_t version_string[16];
+	uint32_t text_length;
+	uint32_t bss_start_address;
+	uint32_t bss_bitmap_size;
+	uint32_t bss_bitmap[16];
+	uint32_t master_rw_start_address;
+	uint32_t master_rw_size;
+};
+
+/**
+ * struct mvx_fw_bin - Structure describing a loaded firmware binary.
+ *
+ * Multiple sessions may share the same firmware binary.
+ */
+struct mvx_fw_bin {
+	struct device *dev;
+	struct mvx_fw_cache *cache;
+	struct mutex mutex;
+	struct kobject kobj;
+	struct list_head cache_head;
+	struct list_head event_list;
+	char filename[128];
+	enum mvx_format format;
+	enum mvx_direction dir;
+	struct mvx_hw_ver hw_ver;
+	atomic_t flush_cnt;
+	bool securevideo;
+	struct {
+		const struct firmware *fw;
+		const struct mvx_fw_header *header;
+		unsigned int text_cnt;
+		unsigned int bss_cnt;
+		unsigned int sbss_cnt;
+	} nonsecure;
+	struct {
+		struct mvx_secure *secure;
+		struct mvx_secure_firmware *securefw;
+	} secure;
+};
+
+/**
+ * struct mvx_fw_event - Firmware load event notification.
+ * @head:		Used by the firmware loader. Should not be used
+ *                      by the client.
+ * @fw_bin_ready:	Callback routine invoked after the firmware binary has
+ *                      finished loading. Will be called both on success and
+ *                      failure.
+ * @arg:		Argument passed to fw_bin_ready. Client may set this
+ *                      pointer to any value.
+ *
+ * Structure used to keep track of clients that have subscribed to event
+ * notification after the firmware binary has been loaded.
+ */
+struct mvx_fw_event {
+	struct list_head head;
+	void (*fw_bin_ready)(struct mvx_fw_bin *fw_bin, void *arg, bool same_thread);
+	void *arg;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_fw_cache_construct() - Construct the firmware object.
+ * @cache:	Pointer to firmware cache.
+ * @dev:	Pointer to device.
+ * @secure:	Pointer to secure video.
+ * @kobj:	Pointer to parent kobj.
+ *
+ * When FW cache is constructed, corresponding sysfs entry will be created
+ * and attached as a child to kobj.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_cache_construct(struct mvx_fw_cache *cache, struct device *dev, struct mvx_secure *secure,
+			   struct kobject *kobj);
+
+/**
+ * mvx_fw_cache_destruct() - Destroy the firmware object.
+ * @cache:	Pointer to firmware cache.
+ */
+void mvx_fw_cache_destruct(struct mvx_fw_cache *cache);
+
+/**
+ * mvx_fw_cache_get() - Get a reference to a firmware binary.
+ * @cache:	Pointer for firmware cache.
+ * @format:	Format used on the bitstream port.
+ * @dir:	Which port that is configured as bitstream port.
+ * @event:	Callback routine and argument that will be invoded after
+ *              the firmware binary has been loaded.
+ * @hw_ver:	MVE hardware version.
+ * @securevideo:Secure video enabled.
+ *
+ * Loading a firmware binary is an asynchronous operation. The client will be
+ * informed through a callback routine when the binary is ready.
+ *
+ * If the firmware binary is already in the cache, then the callback routine
+ * will be called directly from mvx_fw_cache_get(). The client must take care
+ * not to reaquire any mutexes already held.
+ *
+ * If the firmware binary was not found in the cache, then the callback routine
+ * will be called from a separete thread context. The client must make sure
+ * its data is protected by a mutex.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_cache_get(struct mvx_fw_cache *cache, enum mvx_format format, enum mvx_direction dir,
+		     struct mvx_fw_event *event, struct mvx_hw_ver *hw_ver, bool securevideo);
+
+/**
+ * mvx_fw_cache_put() - Return firmware binary to cache and decrement the
+ *                      reference count.
+ * @cache:	Pointer to firmware cache.
+ * @fw:_bin	Pointer to firmware binary.
+ */
+void mvx_fw_cache_put(struct mvx_fw_cache *cache, struct mvx_fw_bin *fw_bin);
+
+/**
+ * mvx_fw_cache_log() - Log firmware binary to ram log.
+ * @fw_bin:	Pointer to firmware binary.
+ * @csession:	Pointer to client session.
+ */
+void mvx_fw_cache_log(struct mvx_fw_bin *fw_bin, struct mvx_client_session *csession);
+
+/**
+ * mvx_fw_cache_get_formats() - Get supported formats.
+ * @cache:	Pointer to firmware cache.
+ * @direction:	Input or output port.
+ * @formats:	Pointer to bitmask listing supported formats.
+ */
+void mvx_fw_cache_get_formats(struct mvx_fw_cache *cache, enum mvx_direction direction, uint64_t *formats);
+
+#endif /* _MVX_FIRMWARE_CACHE_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_priv.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_priv.h
new file mode 100644
index 000000000000..0c974dd3dabe
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_priv.h
@@ -0,0 +1,144 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_FIRMWARE_PRIV_H_
+#define _MVX_FIRMWARE_PRIV_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_firmware.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#if !defined(MVE_REQUEST_CODE_IDLE_ACK)
+#define MVE_REQUEST_CODE_IDLE_ACK (1012)
+#endif
+
+/****************************************************************************
+ * Firmware
+ ****************************************************************************/
+
+struct mvx_client_ops;
+struct mvx_client_session;
+struct mvx_fw_bin;
+struct mvx_mmu;
+struct mvx_session;
+
+/**
+ * mvx_firmware_construct() - Firmware constructor.
+ * @fw:		Pointer to firmware object.
+ * @fw_bin:	Pointer to firmware binary.
+ * @mmu:	Pointer to MMU.
+ * @session:	Pointer to session.
+ * @client_ops:	Pointer to client operations.
+ * @csession:	Pointer to client session.
+ * ncores:	Number of cores.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_construct(struct mvx_fw *fw, struct mvx_fw_bin *fw_bin, struct mvx_mmu *mmu, struct mvx_session *session,
+		     struct mvx_client_ops *client_ops, struct mvx_client_session *csession, unsigned int ncores);
+
+/****************************************************************************
+ * Firmware v2
+ ****************************************************************************/
+
+/**
+ * mvx_fw_construct_v2() - Construct the object for the firmware v2 interface.
+ * @fw:		Pointer to firmware object.
+ * @fw_bin:	Pointer to firmware binary.
+ * @mmu:	Pointer to MMU.
+ * @session:	Pointer to session.
+ * @client_ops:	Pointer to client operations.
+ * @csession:	Pointer to client session.
+ * ncores:	Number of cores.
+ * @major:	Major firmware version.
+ * @minor:	Minor firmware version.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_construct_v2(struct mvx_fw *fw, struct mvx_fw_bin *fw_bin, struct mvx_mmu *mmu, struct mvx_session *session,
+			struct mvx_client_ops *client_ops, struct mvx_client_session *csession, unsigned int ncores,
+			unsigned char major, unsigned char minor);
+
+/**
+ * mvx_fw_send_idle_ack_v2() - Send idle ack.
+ * @fw:		Pointer to firmware object.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_send_idle_ack_v2(struct mvx_fw *fw);
+
+/**
+ * mvx_fw_to_mve_profile_v2() - Convert MVX to MVE profile.
+ * @mvx_profile:	Input profile.
+ * @mve_profile:	Output profile.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_to_mve_profile_v2(unsigned int mvx_profile, uint16_t *mve_profile);
+
+/**
+ * mvx_fw_to_mve_level_v2() - Convert MVX to MVE level.
+ * @mvx_level:		Input level.
+ * @mve_level:		Output level.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_fw_to_mve_level_v2(unsigned int mvx_level, uint16_t *mve_level);
+
+/****************************************************************************
+ * Firmware v3
+ ****************************************************************************/
+
+/**
+ * mvx_fw_construct_v3() - Construct the object for the firmware v3 interface.
+ * @fw:		Pointer to firmware object.
+ * @fw_bin:	Pointer to firmware binary.
+ * @mmu:	Pointer to MMU.
+ * @session:	Pointer to session.
+ * @client_ops:	Pointer to client operations.
+ * @csession:	Pointer to client session.
+ * ncores:	Number of cores.
+ * @major:	Major firmware version.
+ * @minor:	Minor firmware version.
+ *
+ * Return: 0 on sucess, else error code.
+ */
+int mvx_fw_construct_v3(struct mvx_fw *fw, struct mvx_fw_bin *fw_bin, struct mvx_mmu *mmu, struct mvx_session *session,
+			struct mvx_client_ops *client_ops, struct mvx_client_session *csession, unsigned int ncores,
+			unsigned char major, unsigned char minor);
+
+#endif
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_v2.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_v2.c
new file mode 100644
index 000000000000..6e4eb993ac56
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_v2.c
@@ -0,0 +1,3172 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreemen
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/string.h>
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+#include "fw_v2/mve_protocol_def.h"
+#include "mvx_firmware_cache.h"
+#include "mvx_firmware_priv.h"
+#include "mvx_log_group.h"
+#include "mvx_log_ram.h"
+#include "mvx_mmu.h"
+#include "mvx_secure.h"
+#include "mvx_seq.h"
+#include "mvx_session.h"
+#include "mvx_v4l2_session.h"
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+/**
+ * is_afbc() - Detect if format is AFBC.
+ * @format:		Color format.
+ *
+ * Return: True if AFBC, else false.
+ */
+static bool is_afbc(unsigned int format) { return (format & (1 << MVE_FORMAT_BF_A)) != 0; }
+
+/**
+ * log_message() - Log a message.
+ * @session:		Pointer to session.
+ * @channel:		The type of the firmware interface message;
+ *			message, input buffer, output buffer or RPC
+ * @direction:		The type of the firmware interface message;
+ *			host->firmware or firware->host.
+ * @msg_header:		The header of the message.
+ * @data:		Pointer to the message data.
+ */
+static void log_message(struct mvx_session *session, enum mvx_log_fwif_channel channel,
+			enum mvx_log_fwif_direction direction, struct mve_msg_header *msg_header, void *data)
+{
+	struct mvx_log_header header;
+	struct mvx_log_fwif fwif;
+	struct iovec vec[4];
+	struct timespec64 timespec;
+
+	ktime_get_real_ts64(&timespec);
+
+	header.magic = MVX_LOG_MAGIC;
+	header.length = sizeof(fwif) + sizeof(*msg_header) + msg_header->size;
+	header.type = MVX_LOG_TYPE_FWIF;
+	header.severity = MVX_LOG_INFO;
+	header.timestamp.sec = timespec.tv_sec;
+	header.timestamp.nsec = timespec.tv_nsec;
+
+	fwif.version_major = 2;
+	fwif.version_minor = 0;
+	fwif.channel = channel;
+	fwif.direction = direction;
+	fwif.session = (uintptr_t)session;
+
+	vec[0].iov_base = &header;
+	vec[0].iov_len = sizeof(header);
+
+	vec[1].iov_base = &fwif;
+	vec[1].iov_len = sizeof(fwif);
+
+	vec[2].iov_base = msg_header;
+	vec[2].iov_len = sizeof(*msg_header);
+
+	vec[3].iov_base = data;
+	vec[3].iov_len = msg_header->size;
+
+	MVX_LOG_DATA(&mvx_log_fwif_if, MVX_LOG_INFO, vec, 4);
+}
+
+/**
+ * log_rpc() - Log a RPC message.
+ * @session:		Pointer to session.
+ * @direction:		The type of the firmware interface message;
+ *			host->firmware or firware->host.
+ * @rpc:		RPC message.
+ */
+static void log_rpc(struct mvx_session *session, enum mvx_log_fwif_direction direction,
+		    struct mve_rpc_communication_area *rpc)
+{
+	struct mvx_log_header header;
+	struct mvx_log_fwif fwif;
+	size_t rpc_size;
+	struct iovec vec[3];
+	struct timespec64 timespec;
+
+	rpc_size = offsetof(typeof(*rpc), params) + rpc->size;
+
+	if (rpc_size > sizeof(*rpc))
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "RPC message size is too large. size=%u.", rpc->size);
+
+	ktime_get_real_ts64(&timespec);
+
+	header.magic = MVX_LOG_MAGIC;
+	header.length = sizeof(fwif) + rpc_size;
+	header.type = MVX_LOG_TYPE_FWIF;
+	header.severity = MVX_LOG_INFO;
+	header.timestamp.sec = timespec.tv_sec;
+	header.timestamp.nsec = timespec.tv_nsec;
+
+	fwif.version_major = 2;
+	fwif.version_minor = 0;
+	fwif.channel = MVX_LOG_FWIF_CHANNEL_RPC;
+	fwif.direction = direction;
+	fwif.session = (uintptr_t)session;
+
+	vec[0].iov_base = &header;
+	vec[0].iov_len = sizeof(header);
+
+	vec[1].iov_base = &fwif;
+	vec[1].iov_len = sizeof(fwif);
+
+	vec[2].iov_base = rpc;
+	vec[2].iov_len = rpc_size;
+
+	MVX_LOG_DATA(&mvx_log_fwif_if, MVX_LOG_INFO, vec, 3);
+}
+
+static int get_stride90(enum mvx_format format, uint8_t *nplanes, unsigned int stride[MVX_BUFFER_NPLANES][2])
+{
+	switch (format) {
+	case MVX_FORMAT_YUV420_I420:
+		*nplanes = 3;
+		stride[0][0] = 2;
+		stride[0][1] = 2;
+		stride[1][0] = 1;
+		stride[1][1] = 1;
+		stride[2][0] = 1;
+		stride[2][1] = 1;
+		break;
+	case MVX_FORMAT_YUV420_NV12:
+	case MVX_FORMAT_YUV420_NV21:
+		*nplanes = 2;
+		stride[0][0] = 2;
+		stride[0][1] = 2;
+		stride[1][0] = 2;
+		stride[1][1] = 1;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV420_P010:
+		*nplanes = 2;
+		stride[0][0] = 4;
+		stride[0][1] = 2;
+		stride[1][0] = 4;
+		stride[1][1] = 1;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV420_Y0L2:
+	case MVX_FORMAT_YUV420_AQB1:
+		*nplanes = 1;
+		stride[0][0] = 8;
+		stride[0][1] = 1;
+		stride[1][0] = 0;
+		stride[1][1] = 0;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV422_YUY2:
+	case MVX_FORMAT_YUV422_UYVY:
+		*nplanes = 1;
+		stride[0][0] = 4;
+		stride[0][1] = 2;
+		stride[1][0] = 0;
+		stride[1][1] = 0;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	case MVX_FORMAT_YUV422_Y210:
+	case MVX_FORMAT_RGBA_8888:
+	case MVX_FORMAT_BGRA_8888:
+	case MVX_FORMAT_ARGB_8888:
+	case MVX_FORMAT_ABGR_8888:
+		*nplanes = 1;
+		stride[0][0] = 8;
+		stride[0][1] = 2;
+		stride[1][0] = 0;
+		stride[1][1] = 0;
+		stride[2][0] = 0;
+		stride[2][1] = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * read32n() - Read a number of bytes from 'src' to 'dst'.
+ * @src:	Pointer to circular buffer of source data.
+ * @offset:	Current offset in the circular buffer.
+ * @dst:	Pointer to destination buffer.
+ * @size:	Size in bytes.
+ *
+ * Return: New offset in the circular buffer.
+ */
+static unsigned int read32n(volatile uint32_t *src, unsigned int offset, uint32_t *dst, size_t size)
+{
+	for (; size >= sizeof(uint32_t); size -= sizeof(uint32_t)) {
+		*dst++ = src[offset];
+		offset = (offset + 1) % MVE_COMM_QUEUE_SIZE_IN_WORDS;
+	}
+
+	if (size != 0) {
+		memcpy(dst, (void *)&src[offset], size);
+		offset = (offset + 1) % MVE_COMM_QUEUE_SIZE_IN_WORDS;
+	}
+
+	return offset;
+}
+
+/**
+ * read_message() - Read message from firmware message queue.
+ * @fw:		Pointer to firmware object.
+ * @host:	Host communication area.
+ * @mve:	MVE communication area.
+ * @code:	Pointer to where the message code shall be placed.
+ * @data:	Pointer to where message data shall be placed.
+ * @size:	Input: the size of the data. Output: The size of the message.
+ * @channel:	Firmware interface message type to log.
+ *
+ * Return: 1 if a message was read, 0 if no message was read, else error code.
+ */
+static int read_message(struct mvx_fw *fw, struct mve_comm_area_host *host, struct mve_comm_area_mve *mve,
+			unsigned int *code, void *data, size_t *size, enum mvx_log_fwif_channel channel,
+			enum mvx_fw_buffer_attr mve_buf_attr, enum mvx_fw_buffer_attr host_buf_attr)
+{
+	struct mve_msg_header header;
+	unsigned int rpos;
+	ssize_t capacity;
+
+	if (mve_buf_attr == MVX_FW_BUF_CACHEABLE) {
+		dma_sync_single_for_cpu(fw->dev, virt_to_phys(mve), MVE_PAGE_SIZE, DMA_FROM_DEVICE);
+	}
+
+	rpos = host->out_rpos;
+
+	/* Calculate how much data that is available in the buffer. */
+	capacity = mve->out_wpos - rpos;
+	if (capacity < 0)
+		capacity += MVE_COMM_QUEUE_SIZE_IN_WORDS;
+
+	if (capacity <= 0)
+		return 0;
+	/*
+	 * It's possible below sequence occurs due to process out-of-order execution:
+	 * 1. read msg_header first and host_area->out_rpos++;
+	 * 2. firmware writes msg_header and updates mve_area->out_wpos;
+	 * 3. compare host_area->out_rpos and mve_area->out_wpos, since the latter
+	 *    is updated in step 2, the available is not 0;
+	 * 4. parse the msg_header got in step 1 and it's invalid because it should
+	 *    be read after update in step 2.
+	 * Add barrier here to ensure msg_header is always read after firmware  updates.
+	 */
+	smp_rmb();
+
+	/* Read the header. */
+	rpos = read32n(mve->out_data, rpos, (uint32_t *)&header, sizeof(header));
+
+	if (!((header.code >= 1001 && header.code <= 1013) || (header.code >= 2001 && header.code <= 2019) ||
+	      (header.code >= 3001 && header.code <= 3004))) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "read_message header. header.size=%d, rpos=%d, code=%d, capacity=%zu", header.size,
+			      host->out_rpos, header.code, capacity);
+		return 0;
+	}
+
+	/* Make sure there is enough space for both header and message. */
+	capacity -= DIV_ROUND_UP(sizeof(header) + header.size, sizeof(uint32_t));
+	if (capacity < 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "Firmware v2 msg larger than capacity. code=%u, size=%u, wpos=%u, rpos=%u.", header.code,
+			      header.size, mve->out_wpos, host->out_rpos);
+		/* Don't escalate to EFAULT; just retry later. */
+		return 0;
+	}
+
+	if (header.size > *size) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "Firmware v2 message does not fit in buffer. code=%u, msg_size=%u, size=%zu.",
+			      header.code, header.size, *size);
+		return -ENOMEM;
+	}
+
+	/* Read message body. */
+	rpos = read32n(mve->out_data, rpos, data, header.size);
+	host->out_rpos = rpos;
+
+	/*
+	 * Make sure the read pointer has been written before the cache is
+	 * flushed.
+	 */
+	if (host_buf_attr == MVX_FW_BUF_CACHEABLE) {
+		wmb();
+		dma_sync_single_for_device(fw->dev, virt_to_phys(&host->out_rpos), sizeof(host->out_rpos),
+					   DMA_TO_DEVICE);
+	}
+
+	*code = header.code;
+	*size = header.size;
+
+	/* Log firmware message. */
+	MVX_LOG_EXECUTE(&mvx_log_fwif_if, MVX_LOG_INFO,
+			log_message(fw->session, channel, MVX_LOG_FWIF_DIRECTION_FIRMWARE_TO_HOST, &header, data));
+
+	return 1;
+}
+
+/**
+ * write32n() - Write a number of bytes to 'dst' from 'src'.
+ * @dst:	Pointer to circular buffer of destination data.
+ * @offset:	Current offset in the circular buffer.
+ * @src:	Pointer to source buffer.
+ * @size:	Size in bytes.
+ *
+ * Return: New offset in the circular buffer.
+ */
+static unsigned int write32n(volatile uint32_t *dst, unsigned int offset, uint32_t *src, size_t size)
+{
+	for (; size >= sizeof(uint32_t); size -= sizeof(uint32_t)) {
+		dst[offset] = *src++;
+		offset = (offset + 1) % MVE_COMM_QUEUE_SIZE_IN_WORDS;
+	}
+
+	if (size != 0) {
+		memcpy((void *)&dst[offset], src, size);
+		offset = (offset + 1) % MVE_COMM_QUEUE_SIZE_IN_WORDS;
+	}
+
+	return offset;
+}
+
+/**
+ * write_message() - Write message to firmware message queue.
+ * @fw:		Pointer to firmware object.
+ * @host:	Host communication area.
+ * @mve:	MVE communication area.
+ * @code:	Message code.
+ * @data:	Pointer to message data. May be NULL if size if 0.
+ * @size:	Size in bytes of data.
+ * @channel:	Firmware interface message type to log.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int write_message(struct mvx_fw *fw, struct mve_comm_area_host *host, struct mve_comm_area_mve *mve,
+			 unsigned int code, void *data, size_t size, enum mvx_log_fwif_channel channel,
+			 enum mvx_fw_buffer_attr mve_buf_attr, enum mvx_fw_buffer_attr host_buf_attr)
+{
+	struct mve_msg_header header = {.code = code, .size = size};
+	ssize_t capacity;
+	unsigned int wpos;
+
+	if (mve_buf_attr == MVX_FW_BUF_CACHEABLE) {
+		dma_sync_single_for_cpu(fw->dev, virt_to_phys(&mve->in_rpos), sizeof(mve->in_rpos), DMA_FROM_DEVICE);
+	}
+
+	wpos = host->in_wpos;
+
+	/* Calculate how much space that is available in the buffer. */
+	capacity = mve->in_rpos - wpos;
+	if (capacity <= 0)
+		capacity += MVE_COMM_QUEUE_SIZE_IN_WORDS;
+
+	/* Make sure there is enough space for both header and message. */
+	capacity -= DIV_ROUND_UP(sizeof(header) + size, sizeof(uint32_t));
+	if (capacity <= 0)
+		return -ENOMEM;
+
+	/* Write header. */
+	wpos = write32n(host->in_data, wpos, (uint32_t *)&header, sizeof(header));
+
+	/* Write message. */
+	wpos = write32n(host->in_data, wpos, data, size);
+
+	/*
+	 * Make sure all message data has been written before the cache is
+	 * flushed.
+	 */
+	if (host_buf_attr == MVX_FW_BUF_CACHEABLE) {
+		wmb();
+		dma_sync_single_for_device(fw->dev, virt_to_phys(host), MVE_PAGE_SIZE, DMA_TO_DEVICE);
+	}
+
+	host->in_wpos = wpos;
+
+	/*
+	 * Make sure the write pointer has been written before the cache is
+	 * flushed.
+	 */
+	if (host_buf_attr == MVX_FW_BUF_CACHEABLE) {
+		wmb();
+		dma_sync_single_for_device(fw->dev, virt_to_phys(&host->in_wpos), sizeof(host->in_wpos), DMA_TO_DEVICE);
+	}
+
+	/* Log firmware message. */
+	MVX_LOG_EXECUTE(&mvx_log_fwif_if, MVX_LOG_INFO,
+			log_message(fw->session, channel, MVX_LOG_FWIF_DIRECTION_HOST_TO_FIRMWARE, &header, data));
+
+	return 0;
+}
+
+static int get_region_v2(enum mvx_fw_region region, uint32_t *begin, uint32_t *end)
+{
+	switch (region) {
+	case MVX_FW_REGION_CORE_0:
+		*begin = MVE_MEM_REGION_FW_INSTANCE0_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE0_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_1:
+		*begin = MVE_MEM_REGION_FW_INSTANCE1_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE1_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_2:
+		*begin = MVE_MEM_REGION_FW_INSTANCE2_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE2_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_3:
+		*begin = MVE_MEM_REGION_FW_INSTANCE3_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE3_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_4:
+		*begin = MVE_MEM_REGION_FW_INSTANCE4_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE4_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_5:
+		*begin = MVE_MEM_REGION_FW_INSTANCE5_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE5_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_6:
+		*begin = MVE_MEM_REGION_FW_INSTANCE6_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE6_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_7:
+		*begin = MVE_MEM_REGION_FW_INSTANCE7_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE7_ADDR_END;
+		break;
+	case MVX_FW_REGION_PROTECTED:
+		*begin = MVE_MEM_REGION_PROTECTED_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_PROTECTED_ADDR_END;
+		break;
+	case MVX_FW_REGION_FRAMEBUF:
+		*begin = MVE_MEM_REGION_FRAMEBUF_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FRAMEBUF_ADDR_END;
+		break;
+	case MVX_FW_REGION_MSG_HOST:
+		*begin = MVE_COMM_MSG_INQ_ADDR;
+		*end = MVE_COMM_MSG_INQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_MSG_MVE:
+		*begin = MVE_COMM_MSG_OUTQ_ADDR;
+		*end = MVE_COMM_MSG_OUTQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_IN_HOST:
+		*begin = MVE_COMM_BUF_INQ_ADDR;
+		*end = MVE_COMM_BUF_INQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_IN_MVE:
+		*begin = MVE_COMM_BUF_INRQ_ADDR;
+		*end = MVE_COMM_BUF_INRQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_OUT_HOST:
+		*begin = MVE_COMM_BUF_OUTQ_ADDR;
+		*end = MVE_COMM_BUF_OUTQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_OUT_MVE:
+		*begin = MVE_COMM_BUF_OUTRQ_ADDR;
+		*end = MVE_COMM_BUF_OUTRQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_RPC:
+		*begin = MVE_COMM_RPC_ADDR;
+		*end = MVE_COMM_RPC_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_PRINT_RAM:
+		*begin = MVE_FW_PRINT_RAM_ADDR;
+		*end = MVE_FW_PRINT_RAM_ADDR + MVE_FW_PRINT_RAM_SIZE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void convert_buffer_general(struct mvx_fw *fw, enum mvx_direction dir, struct mvx_fw_msg *msg,
+				   struct mve_buffer_general *g)
+{
+	struct mvx_buffer *buf = (struct mvx_buffer *)g->header.host_handle;
+
+	if (g->header.host_handle == MVX_FW_CODE_EOS)
+		return;
+
+	WARN_ON(buf->dir != dir);
+
+	msg->code = MVX_FW_CODE_BUFFER_GENERAL;
+	msg->buf = buf;
+}
+
+static void convert_buffer_frame(struct mvx_fw *fw, enum mvx_direction dir, struct mvx_fw_msg *msg,
+				 struct mve_buffer_frame *f)
+{
+	struct mvx_buffer *buf = (struct mvx_buffer *)f->host_handle;
+
+	if (f->host_handle == MVX_FW_CODE_EOS)
+		return;
+
+	WARN_ON(buf->dir != dir);
+
+	msg->code = MVX_FW_CODE_BUFFER;
+	msg->buf = buf;
+
+	if (dir == MVX_DIR_OUTPUT) {
+		unsigned int i;
+
+		buf->width = f->visible_frame_width;
+		buf->height = f->visible_frame_height;
+		if (buf->width == 0 || buf->height == 0 ||
+		    (f->frame_flags & (MVE_BUFFER_FRAME_FLAG_TOP_PRESENT | MVE_BUFFER_FRAME_FLAG_BOT_PRESENT)) == 0)
+			for (i = 0; i < buf->nplanes; i++)
+				(void)mvx_buffer_filled_set(buf, i, 0, 0);
+
+		if (is_afbc(f->format) != false) {
+			struct mve_buffer_frame_afbc *afbc = &f->data.afbc;
+
+			buf->crop_left = afbc->cropx;
+			buf->crop_top = afbc->cropy;
+		}
+	}
+
+	buf->user_data = f->user_data_tag;
+	buf->flags = 0;
+	if (f->frame_flags & MVE_BUFFER_FRAME_FLAG_EOS)
+		buf->flags |= MVX_BUFFER_EOS;
+
+	if (f->frame_flags & MVE_BUFFER_FRAME_FLAG_REJECTED)
+		buf->flags |= MVX_BUFFER_REJECTED;
+
+	if (f->frame_flags & MVE_BUFFER_FRAME_FLAG_CORRUPT)
+		buf->flags |= MVX_BUFFER_CORRUPT;
+
+	if (f->frame_flags & MVE_BUFFER_FRAME_FLAG_DECODE_ONLY)
+		buf->flags |= MVX_BUFFER_DECODE_ONLY;
+
+	if (f->frame_flags & (MVE_BUFFER_FRAME_FLAG_TOP_PRESENT | MVE_BUFFER_FRAME_FLAG_BOT_PRESENT)) {
+		buf->flags |= MVX_BUFFER_FRAME_PRESENT;
+	}
+
+	if (is_afbc(f->format) != false) {
+		struct mve_buffer_frame_afbc *afbc = &f->data.afbc;
+
+		if (afbc->afbc_params & MVE_BUFFER_FRAME_AFBC_TILED_HEADER)
+			buf->flags |= MVX_BUFFER_AFBC_TILED_HEADERS;
+
+		if (afbc->afbc_params & MVE_BUFFER_FRAME_AFBC_TILED_BODY)
+			buf->flags |= MVX_BUFFER_AFBC_TILED_BODY;
+
+		if (afbc->afbc_params & MVE_BUFFER_FRAME_AFBC_32X8_SUPERBLOCK)
+			buf->flags |= MVX_BUFFER_AFBC_32X8_SUPERBLOCK;
+	}
+}
+
+static void convert_buffer_bitstream(struct mvx_fw *fw, enum mvx_direction dir, struct mvx_fw_msg *msg,
+				     struct mve_buffer_bitstream *b)
+{
+	struct mvx_buffer *buf = (struct mvx_buffer *)b->host_handle;
+
+	if (b->host_handle == MVX_FW_CODE_EOS)
+		return;
+
+	WARN_ON(buf->dir != dir);
+
+	msg->code = MVX_FW_CODE_BUFFER;
+	msg->buf = buf;
+
+	if (dir == MVX_DIR_OUTPUT)
+		mvx_buffer_filled_set(buf, 0, b->bitstream_filled_len, b->bitstream_offset);
+
+	buf->user_data = b->user_data_tag;
+	buf->flags = 0;
+	if (dir == MVX_DIR_INPUT) {
+		struct mvx_corrupt_buffer *corrupt_buf;
+		struct mvx_corrupt_buffer *tmp;
+		list_for_each_entry_safe(corrupt_buf, tmp, &fw->session->buffer_corrupt_queue, head)
+		{
+			if (corrupt_buf->user_data == buf->user_data) {
+				list_del(&corrupt_buf->head);
+				buf->flags |= MVX_BUFFER_CORRUPT;
+				vfree(corrupt_buf);
+			}
+		}
+	}
+	if (b->bitstream_flags & MVE_BUFFER_BITSTREAM_FLAG_EOS)
+		buf->flags |= MVX_BUFFER_EOS;
+
+	if (b->bitstream_flags & MVE_BUFFER_BITSTREAM_FLAG_ENDOFFRAME)
+		buf->flags |= MVX_BUFFER_EOF;
+
+	if (b->bitstream_flags & MVE_BUFFER_BITSTREAM_FLAG_CODECCONFIG)
+		buf->flags |= MVX_BUFFER_CODEC_CONFIG;
+
+	if (b->frame_type == 0) {
+		buf->flags |= MVX_BUFFER_FRAME_FLAG_IFRAME;
+	} else if (b->frame_type == 1) {
+		buf->flags |= MVX_BUFFER_FRAME_FLAG_PFRAME;
+	} else if (b->frame_type == 2) {
+		buf->flags |= MVX_BUFFER_FRAME_FLAG_BFRAME;
+	}
+}
+
+static int convert_buffer_param(struct mvx_fw *fw, struct mvx_fw_msg *msg, struct mve_buffer_param *p)
+{
+	switch (p->type) {
+	case MVE_BUFFER_PARAM_TYPE_COLOUR_DESCRIPTION: {
+		struct mve_buffer_param_colour_description *c = &p->data.colour_description;
+		struct mvx_fw_color_desc *d = &msg->color_desc;
+
+		d->flags = 0;
+
+		switch (c->range) {
+		case MVE_BUFFER_PARAM_COLOUR_RANGE_UNSPECIFIED:
+			d->range = MVX_FW_RANGE_UNSPECIFIED;
+			break;
+		case MVE_BUFFER_PARAM_COLOUR_RANGE_LIMITED:
+			d->range = MVX_FW_RANGE_LIMITED;
+			break;
+		case MVE_BUFFER_PARAM_COLOUR_RANGE_FULL:
+			d->range = MVX_FW_RANGE_FULL;
+			break;
+		default:
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+				      "Unknown fw buffer param color desc range. range=%u.", c->range);
+			return -EINVAL;
+		}
+
+		/* Color primaries according to HEVC E.3.1. */
+		switch (c->colour_primaries) {
+		case 1:
+			d->primaries = MVX_FW_PRIMARIES_BT709;
+			break;
+		case 4:
+			d->primaries = MVX_FW_PRIMARIES_BT470M;
+			break;
+		case 5:
+			d->primaries = MVX_FW_PRIMARIES_BT601_625;
+			break;
+		case 6:
+		case 7:
+			d->primaries = MVX_FW_PRIMARIES_BT601_525;
+			break;
+		case 8:
+			d->primaries = MVX_FW_PRIMARIES_GENERIC_FILM;
+			break;
+		case 9:
+			d->primaries = MVX_FW_PRIMARIES_BT2020;
+			break;
+		default:
+			d->primaries = MVX_FW_PRIMARIES_UNSPECIFIED;
+			break;
+		}
+
+		/* Transfer characteristics according to HEVC E.3.1. */
+		switch (c->transfer_characteristics) {
+		case 12:
+			d->transfer = MVX_FW_TRANSFER_BT1361;
+			break;
+		case 1:
+			d->transfer = MVX_FW_TRANSFER_SMPTE170M;
+			break;
+		case 4:
+			d->transfer = MVX_FW_TRANSFER_GAMMA22;
+			break;
+		case 5:
+			d->transfer = MVX_FW_TRANSFER_GAMMA28;
+			break;
+		case 6:
+		case 14:
+		case 15:
+			d->transfer = MVX_FW_TRANSFER_SMPTE170M;
+			break;
+		case 7:
+			d->transfer = MVX_FW_TRANSFER_SMPTE240M;
+			break;
+		case 8:
+			d->transfer = MVX_FW_TRANSFER_LINEAR;
+			break;
+		case 9:
+		case 10:
+			d->transfer = MVX_FW_TRANSFER_UNSPECIFIED;
+			break;
+		case 18:
+			d->transfer = MVX_FW_TRANSFER_HLG;
+			break;
+		case 11:
+			d->transfer = MVX_FW_TRANSFER_XVYCC;
+			break;
+		case 13:
+			d->transfer = MVX_FW_TRANSFER_SRGB;
+			break;
+		case 16:
+			d->transfer = MVX_FW_TRANSFER_ST2084;
+			break;
+		case 17:
+			d->transfer = MVX_FW_TRANSFER_ST428;
+			break;
+		default:
+			d->transfer = MVX_FW_TRANSFER_UNSPECIFIED;
+			break;
+		}
+
+		/* Matrix coefficient according to HEVC E.3.1. */
+		switch (c->matrix_coeff) {
+		case 1:
+			d->matrix = MVX_FW_MATRIX_BT709;
+			break;
+		case 4:
+			d->matrix = MVX_FW_MATRIX_BT470M;
+			break;
+		case 5:
+		case 6:
+			d->matrix = MVX_FW_MATRIX_BT601;
+			break;
+		case 7:
+			d->matrix = MVX_FW_MATRIX_SMPTE240M;
+			break;
+		case 9:
+			d->matrix = MVX_FW_MATRIX_BT2020;
+			break;
+		case 10:
+			d->matrix = MVX_FW_MATRIX_BT2020Constant;
+			break;
+		default:
+			d->matrix = MVX_FW_MATRIX_UNSPECIFIED;
+			break;
+		}
+
+		if (c->flags & MVE_BUFFER_PARAM_COLOUR_FLAG_MASTERING_DISPLAY_DATA_VALID) {
+			d->flags |= MVX_FW_COLOR_DESC_DISPLAY_VALID;
+
+			d->display.r.x = c->mastering_display_primaries_x[0];
+			d->display.r.y = c->mastering_display_primaries_y[0];
+			d->display.g.x = c->mastering_display_primaries_x[1];
+			d->display.g.y = c->mastering_display_primaries_y[1];
+			d->display.b.x = c->mastering_display_primaries_x[2];
+			d->display.b.y = c->mastering_display_primaries_y[2];
+			d->display.w.x = c->mastering_white_point_x;
+			d->display.w.y = c->mastering_white_point_y;
+
+			d->display.luminance_min = c->min_display_mastering_luminance;
+			d->display.luminance_max = c->max_display_mastering_luminance;
+		}
+
+		if (c->flags & MVE_BUFFER_PARAM_COLOUR_FLAG_CONTENT_LIGHT_DATA_VALID) {
+			d->flags |= MVX_FW_COLOR_DESC_CONTENT_VALID;
+
+			d->content.luminance_max = c->max_content_light_level;
+			d->content.luminance_average = c->avg_content_light_level;
+		}
+
+		msg->code = MVX_FW_CODE_COLOR_DESC;
+		break;
+	}
+	case MVE_BUFFER_PARAM_TYPE_DPB_HELD_FRAMES: {
+		msg->arg = p->data.arg;
+		msg->code = MVX_FW_CODE_DPB_HELD_FRAMES;
+		break;
+	}
+
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "Default buffer param. type=%d", p->type);
+		break;
+	}
+
+	return 1;
+}
+
+static int get_buffer(struct mvx_fw *fw, struct mve_comm_area_host *host, struct mve_comm_area_mve *mve,
+		      enum mvx_direction dir, struct mvx_fw_msg *msg, enum mvx_log_fwif_channel channel,
+		      enum mvx_fw_buffer_attr mve_buf_attr, enum mvx_fw_buffer_attr host_buf_attr)
+{
+	unsigned int code;
+	union {
+		struct mve_buffer_frame frame;
+		struct mve_buffer_bitstream bitstream;
+		struct mve_buffer_param param;
+		struct mve_buffer_general general;
+	} fw_msg;
+	size_t size = sizeof(fw_msg);
+	int ret;
+	struct mvx_session *session = fw->session;
+	struct mvx_v4l2_session *vsession = container_of(session, struct mvx_v4l2_session, session);
+
+	ret = read_message(fw, host, mve, &code, &fw_msg, &size, channel, mve_buf_attr, host_buf_attr);
+	if (ret <= 0)
+		return ret;
+
+	if (vsession->port[dir].q_set == false) {
+		MVX_SESSION_WARN(session, "vb2 queue is released. dir=%d, code=%d.", dir, code);
+		return 0;
+	}
+
+	switch (code) {
+	case MVE_BUFFER_CODE_FRAME:
+		convert_buffer_frame(fw, dir, msg, &fw_msg.frame);
+		break;
+	case MVE_BUFFER_CODE_BITSTREAM:
+		convert_buffer_bitstream(fw, dir, msg, &fw_msg.bitstream);
+		break;
+	case MVE_BUFFER_CODE_PARAM:
+		convert_buffer_param(fw, msg, &fw_msg.param);
+		break;
+	case MVE_BUFFER_CODE_GENERAL:
+		convert_buffer_general(fw, dir, msg, &fw_msg.general);
+		break;
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unknown fw buffer code. code=%u.", code);
+		break;
+	}
+
+	return 1;
+}
+
+static int get_message_v2(struct mvx_fw *fw, struct mvx_fw_msg *msg)
+{
+	unsigned int code;
+	union {
+		struct mve_request_job job;
+		struct mve_response_state_change state_change;
+		struct mve_response_error error;
+		struct mve_response_frame_alloc_parameters alloc_param;
+		struct mve_response_sequence_parameters seq_param;
+		struct mve_response_set_option_fail set_option_fail;
+		struct mve_buffer_param buffer_param;
+		struct mve_response_event event;
+	} fw_msg;
+	size_t size = sizeof(fw_msg);
+	int ret;
+	struct mvx_session *session = fw->session;
+
+	ret = read_message(fw, fw->msg_host, fw->msg_mve, &code, &fw_msg, &size, MVX_LOG_FWIF_CHANNEL_MESSAGE,
+			   fw->buf_attr[MVX_FW_REGION_MSG_MVE], fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+	if (ret <= 0)
+		return ret;
+
+	msg->code = MVX_FW_CODE_MAX;
+
+	switch (code) {
+	case MVE_RESPONSE_CODE_SWITCHED_IN:
+		msg->code = MVX_FW_CODE_SWITCH_IN;
+		break;
+	case MVE_RESPONSE_CODE_SWITCHED_OUT:
+		msg->code = MVX_FW_CODE_SWITCH_OUT;
+		break;
+	case MVE_RESPONSE_CODE_SET_OPTION_CONFIRM:
+		msg->code = MVX_FW_CODE_SET_OPTION;
+		fw->msg_pending--;
+		break;
+	case MVE_RESPONSE_CODE_SET_OPTION_FAIL: {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Firmware set option failed. index=%u, msg=%s.",
+			      fw_msg.set_option_fail.index, fw_msg.set_option_fail.message);
+		msg->code = MVX_FW_CODE_SET_OPTION;
+		fw->msg_pending--;
+		break;
+	}
+	case MVE_RESPONSE_CODE_JOB_DEQUEUED:
+		msg->code = MVX_FW_CODE_JOB;
+		break;
+	case MVE_RESPONSE_CODE_INPUT:
+		ret = get_buffer(fw, fw->buf_in_host, fw->buf_in_mve, MVX_DIR_INPUT, msg,
+				 MVX_LOG_FWIF_CHANNEL_INPUT_BUFFER, fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE],
+				 fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST]);
+		break;
+	case MVE_RESPONSE_CODE_OUTPUT:
+		ret = get_buffer(fw, fw->buf_out_host, fw->buf_out_mve, MVX_DIR_OUTPUT, msg,
+				 MVX_LOG_FWIF_CHANNEL_OUTPUT_BUFFER, fw->buf_attr[MVX_FW_REGION_BUF_OUT_MVE],
+				 fw->buf_attr[MVX_FW_REGION_BUF_OUT_HOST]);
+		break;
+	case MVE_BUFFER_CODE_PARAM:
+		ret = convert_buffer_param(fw, msg, &fw_msg.buffer_param);
+		break;
+	case MVE_RESPONSE_CODE_INPUT_FLUSHED:
+		msg->code = MVX_FW_CODE_FLUSH;
+		msg->flush.dir = MVX_DIR_INPUT;
+		fw->msg_pending--;
+		break;
+	case MVE_RESPONSE_CODE_OUTPUT_FLUSHED:
+		msg->code = MVX_FW_CODE_FLUSH;
+		msg->flush.dir = MVX_DIR_OUTPUT;
+		fw->msg_pending--;
+		break;
+	case MVE_RESPONSE_CODE_PONG:
+		msg->code = MVX_FW_CODE_PONG;
+		break;
+	case MVE_RESPONSE_CODE_ERROR: {
+		msg->code = MVX_FW_CODE_ERROR;
+
+		switch (fw_msg.error.error_code) {
+		case MVE_ERROR_ABORT:
+			msg->error.error_code = MVX_FW_ERROR_ABORT;
+			break;
+		case MVE_ERROR_OUT_OF_MEMORY:
+			msg->error.error_code = MVX_FW_ERROR_OUT_OF_MEMORY;
+			break;
+		case MVE_ERROR_ASSERT:
+			msg->error.error_code = MVX_FW_ERROR_ASSERT;
+			break;
+		case MVE_ERROR_UNSUPPORTED:
+			msg->error.error_code = MVX_FW_ERROR_UNSUPPORTED;
+			break;
+		case MVE_ERROR_INVALID_BUFFER:
+			msg->error.error_code = MVX_FW_ERROR_INVALID_BUFFER;
+			break;
+		case MVE_ERROR_INVALID_STATE:
+			msg->error.error_code = MVX_FW_ERROR_INVALID_STATE;
+			break;
+		case MVE_ERROR_WATCHDOG:
+			msg->error.error_code = MVX_FW_ERROR_WATCHDOG;
+			break;
+		default:
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unsupported fw error code. code=%u.",
+				      fw_msg.error.error_code);
+			break;
+		}
+#if (KERNEL_VERSION(6, 7, 0) <= LINUX_VERSION_CODE)
+		strncpy(msg->error.message, fw_msg.error.message,
+			min(sizeof(msg->error.message), sizeof(fw_msg.error.message)));
+#else
+		strlcpy(msg->error.message, fw_msg.error.message,
+			min(sizeof(msg->error.message), sizeof(fw_msg.error.message)));
+#endif
+
+		break;
+	}
+	case MVE_RESPONSE_CODE_STATE_CHANGE: {
+		msg->code = MVX_FW_CODE_STATE_CHANGE;
+
+		if (fw_msg.state_change.new_state == MVE_STATE_STOPPED)
+			msg->state = MVX_FW_STATE_STOPPED;
+		else
+			msg->state = MVX_FW_STATE_RUNNING;
+
+		fw->msg_pending--;
+		break;
+	}
+	case MVE_RESPONSE_CODE_DUMP:
+		msg->code = MVX_FW_CODE_DUMP;
+		fw->msg_pending--;
+		break;
+	case MVE_RESPONSE_CODE_DEBUG:
+		msg->code = MVX_FW_CODE_DEBUG;
+		fw->msg_pending--;
+		break;
+	case MVE_RESPONSE_CODE_IDLE:
+		msg->code = MVX_FW_CODE_IDLE;
+		break;
+	case MVE_RESPONSE_CODE_FRAME_ALLOC_PARAM:
+		msg->code = MVX_FW_CODE_ALLOC_PARAM;
+		msg->alloc_param.width = fw_msg.alloc_param.planar_alloc_frame_width;
+		msg->alloc_param.height = fw_msg.alloc_param.planar_alloc_frame_height;
+		msg->alloc_param.afbc_alloc_bytes = fw_msg.alloc_param.afbc_alloc_bytes;
+		msg->alloc_param.afbc_width = fw_msg.alloc_param.afbc_width_in_superblocks;
+
+		break;
+	case MVE_RESPONSE_CODE_SEQUENCE_PARAMETERS:
+		msg->code = MVX_FW_CODE_SEQ_PARAM;
+		msg->seq_param.planar.buffers_min = fw_msg.seq_param.num_buffers_planar;
+		msg->seq_param.afbc.buffers_min = fw_msg.seq_param.num_buffers_afbc;
+		session->port[MVX_DIR_OUTPUT].interlaced = fw_msg.seq_param.interlace;
+		break;
+	case MVE_RESPONSE_CODE_EVENT:
+		if (MVE_EVENT_ERROR_STREAM_CORRUPT == fw_msg.event.event_code) {
+			int ret = 0;
+			bool is_find = false;
+			struct mvx_corrupt_buffer *buf_corrupt;
+			uint64_t user_data = 0;
+			char *tmp;
+			char *err_msg = fw_msg.event.event_data.message;
+			char *err_msg_sep = vmalloc(sizeof(char) * MVE_MAX_ERROR_MESSAGE_SIZE);
+			memcpy(err_msg_sep, err_msg, MVE_MAX_ERROR_MESSAGE_SIZE);
+			tmp = strsep(&err_msg_sep, ",");
+			while (tmp != NULL) {
+				ret = sscanf(tmp, "UDT[%llx]", &user_data);
+				if (ret == 1) {
+					is_find = true;
+					break;
+				}
+				tmp = strsep(&err_msg_sep, ",");
+			}
+			vfree(err_msg_sep);
+			if (!is_find) {
+				MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "EVENT code=%d %s.but not found UDT ret:%d",
+					      fw_msg.event.event_code, fw_msg.event.event_data.message, ret);
+			} else {
+				buf_corrupt = vmalloc(sizeof(struct mvx_corrupt_buffer));
+				buf_corrupt->user_data = user_data;
+				list_add(&buf_corrupt->head, &fw->session->buffer_corrupt_queue);
+			}
+		}
+		if (MVE_EVENT_ERROR_STREAM_NOT_SUPPORTED == fw_msg.event.event_code) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "event STREAM_NOT_SUPPORTED. code=%d. %s.",
+				      fw_msg.event.event_code, fw_msg.event.event_data.message);
+			wake_up(&session->waitq);
+			session->event(session, MVX_SESSION_EVENT_ERROR, (void *)(-EINVAL));
+		}
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "EVENT code=%d. %s", fw_msg.event.event_code,
+			      (MVE_EVENT_ERROR_STREAM_CORRUPT == fw_msg.event.event_code ||
+			       MVE_EVENT_ERROR_STREAM_NOT_SUPPORTED == fw_msg.event.event_code)
+				      ? fw_msg.event.event_data.message
+				      : "");
+		if (fw_msg.event.event_code == MVE_EVENT_PROCESSED) {
+			session->bus_read_bytes_total += fw_msg.event.event_data.event_processed.bus_read_bytes;
+			session->bus_write_bytes_total += fw_msg.event.event_data.event_processed.bus_write_bytes;
+			if (fw_msg.event.event_data.event_processed.pic_format == 0) {
+				session->frame_id++;
+			}
+		}
+		break;
+	case MVE_RESPONSE_CODE_REF_FRAME_UNUSED:
+		break;
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unknown fw message code. code=%u, size=%u.", code, size);
+		msg->code = MVX_FW_CODE_UNKNOWN;
+		break;
+	}
+
+	return ret;
+}
+
+static int put_buffer_general(struct mvx_fw *fw, struct mve_comm_area_host *host, struct mve_comm_area_mve *mve,
+			      struct mvx_fw_msg *msg, enum mvx_log_fwif_channel channel,
+			      enum mvx_fw_buffer_attr mve_buf_attr, enum mvx_fw_buffer_attr host_buf_attr)
+{
+	int ret;
+	struct mve_buffer_general g = {0};
+	struct mvx_buffer *buf = msg->buf;
+	g.header.host_handle = (ptrdiff_t)buf;
+	g.header.user_data_tag = buf->user_data;
+	g.header.buffer_ptr = mvx_buffer_va(buf, 0);
+	g.header.buffer_size = buf->general.header.buffer_size;
+	g.header.config_size = buf->general.header.config_size;
+	g.header.type = buf->general.header.type;
+
+	memcpy(&g.config, &buf->general.config, sizeof(buf->general.config));
+	ret = write_message(fw, host, mve, MVE_BUFFER_CODE_GENERAL, &g, sizeof(g), channel, mve_buf_attr,
+			    host_buf_attr);
+
+	return ret;
+}
+
+static int put_buffer_frame(struct mvx_fw *fw, struct mve_comm_area_host *host, struct mve_comm_area_mve *mve,
+			    struct mvx_fw_msg *msg, enum mvx_log_fwif_channel channel,
+			    enum mvx_fw_buffer_attr mve_buf_attr, enum mvx_fw_buffer_attr host_buf_attr)
+{
+	struct mve_buffer_frame f = {0};
+	struct mvx_buffer *buf = msg->buf;
+	struct mvx_session *session = fw->session;
+	int ret;
+	int stride_shift = 0, stride = 0;
+	unsigned int strideRot[MVX_BUFFER_NPLANES];
+	int max_height;
+	uint32_t scaling_shift = 0;
+	uint32_t rotation = (buf->flags & MVX_BUFFER_FRAME_FLAG_ROTATION_MASK) >> 12;
+	scaling_shift = (buf->flags & MVX_BUFFER_FRAME_FLAG_SCALING_MASK) >> 14;
+	f.host_handle = (ptrdiff_t)buf;
+	f.user_data_tag = buf->user_data;
+
+	if (buf->dir == MVX_DIR_INPUT) {
+		f.visible_frame_width = buf->width;
+		f.visible_frame_height = buf->height;
+
+		if (buf->flags & MVX_BUFFER_EOS)
+			f.frame_flags |= MVE_BUFFER_FRAME_FLAG_EOS;
+
+		if (buf->planes[0].filled != 0)
+			f.frame_flags |= MVE_BUFFER_FRAME_FLAG_TOP_PRESENT;
+	}
+
+	if (buf->dir == MVX_DIR_OUTPUT && (session->dsl_ratio.hor != 1 || session->dsl_ratio.ver != 1)) {
+		f.frame_flags |= ((session->dsl_ratio.hor - 1) << 24 | (session->dsl_ratio.ver - 1) << 17);
+	}
+	if (buf->flags & MVX_BUFFER_INTERLACE)
+		f.frame_flags |= MVE_BUFFER_FRAME_FLAG_INTERLACE;
+
+	f.frame_flags |= (buf->flags & MVX_BUFFER_FRAME_FLAG_ROTATION_MASK) >> 8;
+	f.frame_flags |= (buf->flags & MVX_BUFFER_FRAME_FLAG_MIRROR_MASK) >> 8;
+	f.frame_flags |= (buf->flags & MVX_BUFFER_FRAME_FLAG_SCALING_MASK) >> 8;
+	if (buf->dir == MVX_DIR_OUTPUT && (rotation == 1 || rotation == 3)) {
+		uint8_t nplanes = 0;
+		unsigned int stride90[MVX_BUFFER_NPLANES][2];
+		int i;
+		get_stride90(buf->format, &nplanes, stride90);
+		for (i = 0; i < buf->nplanes; i++) {
+			const unsigned int stride_align = 1;
+			unsigned int tmp = DIV_ROUND_UP(buf->height * stride90[i][0], 2);
+			strideRot[i] = round_up(tmp, stride_align);
+		}
+	}
+
+	switch (buf->format) {
+	case MVX_FORMAT_YUV420_AFBC_8:
+		f.format = MVE_FORMAT_YUV420_AFBC_8;
+		break;
+	case MVX_FORMAT_YUV420_AFBC_10:
+		f.format = MVE_FORMAT_YUV420_AFBC_10;
+		break;
+	case MVX_FORMAT_YUV422_AFBC_8:
+		f.format = MVE_FORMAT_YUV422_AFBC_8;
+		break;
+	case MVX_FORMAT_YUV422_AFBC_10:
+		f.format = MVE_FORMAT_YUV422_AFBC_10;
+		break;
+	case MVX_FORMAT_YUV420_I420:
+		f.format = MVE_FORMAT_YUV420_I420;
+		break;
+	case MVX_FORMAT_YUV420_NV12:
+		f.format = MVE_FORMAT_YUV420_NV12;
+		break;
+	case MVX_FORMAT_YUV420_NV21:
+		f.format = MVE_FORMAT_YUV420_NV21;
+		break;
+	case MVX_FORMAT_YUV420_P010:
+		f.format = MVE_FORMAT_YUV420_P010;
+		break;
+	case MVX_FORMAT_YUV420_Y0L2:
+		f.format = MVE_FORMAT_YUV420_Y0L2;
+		break;
+	case MVX_FORMAT_YUV420_AQB1:
+		f.format = MVE_FORMAT_YUV420_AQB1;
+		break;
+	case MVX_FORMAT_YUV422_YUY2:
+		f.format = MVE_FORMAT_YUV422_YUY2;
+		break;
+	case MVX_FORMAT_YUV422_UYVY:
+		f.format = MVE_FORMAT_YUV422_UYVY;
+		break;
+	case MVX_FORMAT_YUV422_Y210:
+		f.format = MVE_FORMAT_YUV422_Y210;
+		break;
+	case MVX_FORMAT_RGBA_8888:
+		f.format = MVE_FORMAT_RGBA_8888;
+		break;
+	case MVX_FORMAT_BGRA_8888:
+		f.format = MVE_FORMAT_BGRA_8888;
+		break;
+	case MVX_FORMAT_ARGB_8888:
+		f.format = MVE_FORMAT_ARGB_8888;
+		break;
+	case MVX_FORMAT_ABGR_8888:
+		f.format = MVE_FORMAT_ABGR_8888;
+		break;
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unsupported frame format. format=%u.", buf->format);
+		return -EINVAL;
+	}
+
+	if (is_afbc(f.format) == false) {
+		struct mve_buffer_frame_planar *planar = &f.data.planar;
+		int i;
+
+		if (f.frame_flags & MVE_BUFFER_FRAME_FLAG_INTERLACE) {
+			max_height = buf->width;
+			stride_shift = 1;
+			max_height >>= 1;
+		}
+#if 0
+		for (i = 0; i < buf->nplanes; i++) {
+			struct mvx_buffer_plane *plane = &buf->planes[i];
+
+			if (plane->stride > 0)
+				planar->plane_top[i] = mvx_buffer_va(buf, i);
+
+			planar->stride[i] = plane->stride;
+			planar->plane_bot[i] = 0;
+
+			if (f.frame_flags & MVE_BUFFER_FRAME_FLAG_INTERLACE)
+
+				planar->plane_bot[i] = planar->plane_top[i] +
+						       DIV_ROUND_UP(
+					plane->filled, 2);
+		}
+#else
+		for (i = 0; i < buf->nplanes; i++) {
+			struct mvx_buffer_plane *plane = &buf->planes[i];
+
+			if (plane->stride > 0) {
+				planar->plane_top[i] = mvx_buffer_va(buf, i);
+			}
+
+			if (f.frame_flags & MVE_BUFFER_FRAME_FLAG_INTERLACE) {
+				// interlace mode
+				stride = plane->stride;
+				// stride_shift = 1;
+				if (stride_shift) {
+					stride = round_up(stride, 2) << stride_shift;
+				}
+				planar->stride[i] = stride;
+				planar->plane_bot[i] = planar->plane_top[i] + (round_up(stride, 2) >> stride_shift);
+				if (buf->dir == MVX_DIR_OUTPUT && (rotation == 1 || rotation == 3)) {
+					planar->stride[i] = strideRot[i];
+				}
+			} else {
+				// frame mode
+				if (buf->dir == MVX_DIR_OUTPUT && (rotation == 1 || rotation == 3)) {
+					planar->stride[i] = strideRot[i];
+				} else {
+					planar->stride[i] = plane->stride;
+				}
+				planar->plane_bot[i] = 0;
+			}
+		}
+
+#endif
+		if (buf->dir == MVX_DIR_OUTPUT && (rotation == 1 || rotation == 3)) {
+			planar->max_frame_width = buf->height;
+			planar->max_frame_height = buf->width;
+		} else {
+			planar->max_frame_width = buf->width;
+			planar->max_frame_height = buf->height;
+		}
+	} else {
+		struct mve_buffer_frame_afbc *afbc = &f.data.afbc;
+
+		afbc->afbc_width_in_superblocks[0] = buf->planes[0].afbc_width;
+		afbc->plane[0] = mvx_buffer_va(buf, 0);
+
+		if (f.frame_flags & MVE_BUFFER_FRAME_FLAG_INTERLACE) {
+			afbc->alloc_bytes[0] = ALIGN((buf->planes[0].filled / 2), 32);
+			afbc->alloc_bytes[1] = buf->planes[0].filled - afbc->alloc_bytes[0];
+			afbc->plane[1] = afbc->plane[0] + afbc->alloc_bytes[0];
+			afbc->afbc_width_in_superblocks[1] = afbc->afbc_width_in_superblocks[0];
+		} else {
+			afbc->alloc_bytes[0] = buf->planes[0].filled;
+		}
+
+		afbc->afbc_params = 0;
+		if (buf->flags & MVX_BUFFER_AFBC_TILED_HEADERS)
+			afbc->afbc_params |= MVE_BUFFER_FRAME_AFBC_TILED_HEADER;
+
+		if (buf->flags & MVX_BUFFER_AFBC_TILED_BODY)
+			afbc->afbc_params |= MVE_BUFFER_FRAME_AFBC_TILED_BODY;
+
+		if (buf->flags & MVX_BUFFER_AFBC_32X8_SUPERBLOCK)
+			afbc->afbc_params |= MVE_BUFFER_FRAME_AFBC_32X8_SUPERBLOCK;
+		if (buf->flags & MVX_BUFFER_AFBC_BLOCK_SPLIT)
+			afbc->afbc_params |= MVE_BUFFER_FRAME_AFBC_BLOCK_SPLIT;
+	}
+
+	ret = write_message(fw, host, mve, MVE_BUFFER_CODE_FRAME, &f, sizeof(f), channel, mve_buf_attr, host_buf_attr);
+
+	return ret;
+}
+
+static int put_buffer_bitstream(struct mvx_fw *fw, struct mve_comm_area_host *host, struct mve_comm_area_mve *mve,
+				struct mvx_fw_msg *msg, enum mvx_log_fwif_channel channel,
+				enum mvx_fw_buffer_attr mve_buf_attr, enum mvx_fw_buffer_attr host_buf_attr)
+{
+	struct mve_buffer_bitstream b = {0};
+	struct mvx_buffer *buf = msg->buf;
+	int ret;
+
+	if (buf->dir == MVX_DIR_INPUT)
+		b.bitstream_filled_len = buf->planes[0].filled;
+
+	b.host_handle = (ptrdiff_t)buf;
+	b.user_data_tag = buf->user_data;
+	b.bitstream_alloc_bytes = mvx_buffer_size(buf, 0);
+	b.bitstream_buf_addr = mvx_buffer_va(buf, 0);
+
+	if (buf->flags & MVX_BUFFER_EOS)
+		b.bitstream_flags |= MVE_BUFFER_BITSTREAM_FLAG_EOS;
+
+	if (buf->flags & MVX_BUFFER_EOF)
+		b.bitstream_flags |= MVE_BUFFER_BITSTREAM_FLAG_ENDOFFRAME;
+
+	if (buf->flags & MVX_BUFFER_END_OF_SUB_FRAME) {
+		b.bitstream_flags |= MVE_BUFFER_BITSTREAM_FLAG_ENDOFSUBFRAME;
+	}
+	if (buf->flags & MVX_BUFFER_CODEC_CONFIG) {
+		b.bitstream_flags |= MVE_BUFFER_BITSTREAM_FLAG_CODECCONFIG;
+		b.bitstream_flags |= MVE_BUFFER_BITSTREAM_FLAG_ENDOFSUBFRAME;
+	}
+
+	ret = write_message(fw, host, mve, MVE_BUFFER_CODE_BITSTREAM, &b, sizeof(b), channel, mve_buf_attr,
+			    host_buf_attr);
+
+	return ret;
+}
+
+static int to_mve_nalu_format(enum mvx_nalu_format fmt, int *mve_val)
+{
+	switch (fmt) {
+	case MVX_NALU_FORMAT_START_CODES:
+		*mve_val = MVE_OPT_NALU_FORMAT_START_CODES;
+		break;
+	case MVX_NALU_FORMAT_ONE_NALU_PER_BUFFER:
+		*mve_val = MVE_OPT_NALU_FORMAT_ONE_NALU_PER_BUFFER;
+		break;
+	case MVX_NALU_FORMAT_ONE_BYTE_LENGTH_FIELD:
+		*mve_val = MVE_OPT_NALU_FORMAT_ONE_BYTE_LENGTH_FIELD;
+		break;
+	case MVX_NALU_FORMAT_TWO_BYTE_LENGTH_FIELD:
+		*mve_val = MVE_OPT_NALU_FORMAT_TWO_BYTE_LENGTH_FIELD;
+		break;
+	case MVX_NALU_FORMAT_FOUR_BYTE_LENGTH_FIELD:
+		*mve_val = MVE_OPT_NALU_FORMAT_FOUR_BYTE_LENGTH_FIELD;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int mvx_fw_to_mve_profile_v2(unsigned int mvx_profile, uint16_t *mve_profile)
+{
+	switch (mvx_profile) {
+	case MVX_PROFILE_H264_BASELINE:
+		*mve_profile = MVE_OPT_PROFILE_H264_BASELINE;
+		break;
+	case MVX_PROFILE_H264_MAIN:
+		*mve_profile = MVE_OPT_PROFILE_H264_MAIN;
+		break;
+	case MVX_PROFILE_H264_HIGH:
+		*mve_profile = MVE_OPT_PROFILE_H264_HIGH;
+		break;
+	case MVX_PROFILE_H265_MAIN:
+		*mve_profile = MVE_OPT_PROFILE_H265_MAIN;
+		break;
+	case MVX_PROFILE_H265_MAIN_STILL:
+		*mve_profile = MVE_OPT_PROFILE_H265_MAIN_STILL;
+		break;
+	case MVX_PROFILE_H265_MAIN_INTRA:
+		*mve_profile = MVE_OPT_PROFILE_H265_MAIN_INTRA;
+		break;
+	case MVX_PROFILE_VC1_SIMPLE:
+		*mve_profile = MVE_OPT_PROFILE_VC1_SIMPLE;
+		break;
+	case MVX_PROFILE_VC1_MAIN:
+		*mve_profile = MVE_OPT_PROFILE_VC1_MAIN;
+		break;
+	case MVX_PROFILE_VC1_ADVANCED:
+		*mve_profile = MVE_OPT_PROFILE_VC1_ADVANCED;
+		break;
+	case MVX_PROFILE_VP8_MAIN:
+		*mve_profile = MVE_OPT_PROFILE_VP8_MAIN;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int mvx_fw_to_mve_level_v2(unsigned int mvx_level, uint16_t *mve_level)
+{
+	switch (mvx_level) {
+	case MVX_LEVEL_NONE:
+		*mve_level = 0;
+		break;
+	case MVX_LEVEL_H264_1:
+		*mve_level = MVE_OPT_LEVEL_H264_1;
+		break;
+	case MVX_LEVEL_H264_1b:
+		*mve_level = MVE_OPT_LEVEL_H264_1b;
+		break;
+	case MVX_LEVEL_H264_11:
+		*mve_level = MVE_OPT_LEVEL_H264_11;
+		break;
+	case MVX_LEVEL_H264_12:
+		*mve_level = MVE_OPT_LEVEL_H264_12;
+		break;
+	case MVX_LEVEL_H264_13:
+		*mve_level = MVE_OPT_LEVEL_H264_13;
+		break;
+	case MVX_LEVEL_H264_2:
+		*mve_level = MVE_OPT_LEVEL_H264_2;
+		break;
+	case MVX_LEVEL_H264_21:
+		*mve_level = MVE_OPT_LEVEL_H264_21;
+		break;
+	case MVX_LEVEL_H264_22:
+		*mve_level = MVE_OPT_LEVEL_H264_22;
+		break;
+	case MVX_LEVEL_H264_3:
+		*mve_level = MVE_OPT_LEVEL_H264_3;
+		break;
+	case MVX_LEVEL_H264_31:
+		*mve_level = MVE_OPT_LEVEL_H264_31;
+		break;
+	case MVX_LEVEL_H264_32:
+		*mve_level = MVE_OPT_LEVEL_H264_32;
+		break;
+	case MVX_LEVEL_H264_4:
+		*mve_level = MVE_OPT_LEVEL_H264_4;
+		break;
+	case MVX_LEVEL_H264_41:
+		*mve_level = MVE_OPT_LEVEL_H264_41;
+		break;
+	case MVX_LEVEL_H264_42:
+		*mve_level = MVE_OPT_LEVEL_H264_42;
+		break;
+	case MVX_LEVEL_H264_5:
+		*mve_level = MVE_OPT_LEVEL_H264_5;
+		break;
+	case MVX_LEVEL_H264_51:
+		*mve_level = MVE_OPT_LEVEL_H264_51;
+		break;
+
+	/**
+	 * Levels supported by the HW but not by V4L2 controls API.
+	 *
+	 * case MVX_LEVEL_H264_52:
+	 * mve_level = MVE_OPT_LEVEL_H264_52;
+	 *      break;
+	 * case MVX_LEVEL_H264_6:
+	 * mve_level = MVE_OPT_LEVEL_H264_6;
+	 *      break;
+	 * case MVX_LEVEL_H264_61:
+	 * mve_level = MVE_OPT_LEVEL_H264_61;
+	 *      break;
+	 * case MVX_LEVEL_H264_62:
+	 * mve_level = MVE_OPT_LEVEL_H264_62;
+	 *      break;
+	 */
+	case MVX_LEVEL_H265_MAIN_1:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_1;
+		break;
+	case MVX_LEVEL_H265_HIGH_1:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_1;
+		break;
+	case MVX_LEVEL_H265_MAIN_2:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_2;
+		break;
+	case MVX_LEVEL_H265_HIGH_2:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_2;
+		break;
+	case MVX_LEVEL_H265_MAIN_21:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_21;
+		break;
+	case MVX_LEVEL_H265_HIGH_21:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_21;
+		break;
+	case MVX_LEVEL_H265_MAIN_3:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_3;
+		break;
+	case MVX_LEVEL_H265_HIGH_3:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_3;
+		break;
+	case MVX_LEVEL_H265_MAIN_31:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_31;
+		break;
+	case MVX_LEVEL_H265_HIGH_31:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_31;
+		break;
+	case MVX_LEVEL_H265_MAIN_4:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_4;
+		break;
+	case MVX_LEVEL_H265_HIGH_4:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_4;
+		break;
+	case MVX_LEVEL_H265_MAIN_41:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_41;
+		break;
+	case MVX_LEVEL_H265_HIGH_41:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_41;
+		break;
+	case MVX_LEVEL_H265_MAIN_5:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_5;
+		break;
+	case MVX_LEVEL_H265_HIGH_5:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_5;
+		break;
+	case MVX_LEVEL_H265_MAIN_51:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_51;
+		break;
+	case MVX_LEVEL_H265_HIGH_51:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_51;
+		break;
+	case MVX_LEVEL_H265_MAIN_52:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_52;
+		break;
+	case MVX_LEVEL_H265_HIGH_52:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_52;
+		break;
+	case MVX_LEVEL_H265_MAIN_6:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_6;
+		break;
+	case MVX_LEVEL_H265_HIGH_6:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_6;
+		break;
+	case MVX_LEVEL_H265_MAIN_61:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_61;
+		break;
+	case MVX_LEVEL_H265_HIGH_61:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_61;
+		break;
+	case MVX_LEVEL_H265_MAIN_62:
+		*mve_level = MVE_OPT_LEVEL_H265_MAIN_TIER_62;
+		break;
+	case MVX_LEVEL_H265_HIGH_62:
+		*mve_level = MVE_OPT_LEVEL_H265_HIGH_TIER_62;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int to_mve_gop_type(enum mvx_gop_type gop, unsigned int *mve_arg)
+{
+	switch (gop) {
+	case MVX_GOP_TYPE_BIDIRECTIONAL:
+		*mve_arg = MVE_OPT_GOP_TYPE_BIDIRECTIONAL;
+		break;
+	case MVX_GOP_TYPE_LOW_DELAY:
+		*mve_arg = MVE_OPT_GOP_TYPE_LOW_DELAY;
+		break;
+	case MVX_GOP_TYPE_PYRAMID:
+		*mve_arg = MVE_OPT_GOP_TYPE_PYRAMID;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int to_mve_h264_cabac(enum mvx_entropy_mode entropy_mode, unsigned int *mve_arg)
+{
+	switch (entropy_mode) {
+	case MVX_ENTROPY_MODE_CABAC:
+		*mve_arg = 1;
+		break;
+	case MVX_ENTROPY_MODE_CAVLC:
+		*mve_arg = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int to_mve_vp9_prob_update(enum mvx_vp9_prob_update prob_update, unsigned int *mve_arg)
+{
+	switch (prob_update) {
+	case MVX_VP9_PROB_UPDATE_DISABLED:
+		*mve_arg = MVE_OPT_VP9_PROB_UPDATE_DISABLED;
+		break;
+	case MVX_VP9_PROB_UPDATE_IMPLICIT:
+		*mve_arg = MVE_OPT_VP9_PROB_UPDATE_IMPLICIT;
+		break;
+	case MVX_VP9_PROB_UPDATE_EXPLICIT:
+		*mve_arg = MVE_OPT_VP9_PROB_UPDATE_EXPLICIT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int to_mve_rgb_to_yuv(enum mvx_rgb_to_yuv_mode mode, unsigned int *mve_arg)
+{
+	switch (mode) {
+	case MVX_RGB_TO_YUV_MODE_BT601_STUDIO:
+		*mve_arg = MVE_OPT_RGB_TO_YUV_BT601_STUDIO;
+		break;
+	case MVX_RGB_TO_YUV_MODE_BT601_FULL:
+		*mve_arg = MVE_OPT_RGB_TO_YUV_BT601_FULL;
+		break;
+	case MVX_RGB_TO_YUV_MODE_BT709_STUDIO:
+		*mve_arg = MVE_OPT_RGB_TO_YUV_BT709_STUDIO;
+		break;
+	case MVX_RGB_TO_YUV_MODE_BT709_FULL:
+		*mve_arg = MVE_OPT_RGB_TO_YUV_BT709_FULL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int put_fw_opt(struct mvx_fw *fw, struct mve_request_set_option *opt, size_t size)
+{
+	int ret;
+
+	ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_SET_OPTION, opt,
+			    offsetof(typeof(*opt), data) + size, MVX_LOG_FWIF_CHANNEL_MESSAGE,
+			    fw->buf_attr[MVX_FW_REGION_MSG_MVE], fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+
+	if (ret == 0)
+		fw->msg_pending++;
+
+	return ret;
+}
+
+static int put_fw_buf_param(struct mvx_fw *fw, struct mve_buffer_param *param, size_t size)
+{
+	return write_message(fw, fw->buf_in_host, fw->buf_in_mve, MVE_BUFFER_CODE_PARAM, param,
+			     offsetof(typeof(*param), data) + size, MVX_LOG_FWIF_CHANNEL_MESSAGE,
+			     fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE], fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST]);
+}
+
+static int put_message_v2(struct mvx_fw *fw, struct mvx_fw_msg *msg)
+{
+	int ret = 0;
+
+	switch (msg->code) {
+	case MVX_FW_CODE_STATE_CHANGE: {
+		unsigned int code = msg->state == MVX_FW_STATE_STOPPED ? MVE_REQUEST_CODE_STOP : MVE_REQUEST_CODE_GO;
+
+		ret = write_message(fw, fw->msg_host, fw->msg_mve, code, NULL, 0, MVX_LOG_FWIF_CHANNEL_MESSAGE,
+				    fw->buf_attr[MVX_FW_REGION_MSG_MVE], fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+		if (ret == 0)
+			fw->msg_pending++;
+
+		break;
+	}
+	case MVX_FW_CODE_JOB: {
+		struct mve_request_job job;
+
+		job.cores = msg->job.cores;
+		job.frames = msg->job.frames;
+		job.flags = 0;
+
+		ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_JOB, &job, sizeof(job),
+				    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+				    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+		break;
+	}
+	case MVX_FW_CODE_SWITCH_OUT: {
+		ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_SWITCH, NULL, 0,
+				    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+				    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+		break;
+	}
+	case MVX_FW_CODE_PING: {
+		ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_PING, NULL, 0,
+				    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+				    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+		break;
+	}
+	case MVX_FW_CODE_SET_OPTION: {
+		switch (msg->set_option.code) {
+		case MVX_FW_SET_FRAME_RATE: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_FRAME_RATE;
+			param.data.arg = msg->set_option.frame_rate;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.arg));
+			break;
+		}
+		case MVX_FW_SET_TARGET_BITRATE: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_RATE_CONTROL;
+			if (msg->set_option.target_bitrate == 0) {
+				param.data.rate_control.rate_control_mode = MVE_OPT_RATE_CONTROL_MODE_OFF;
+				param.data.rate_control.target_bitrate = 0;
+			} else {
+				param.data.rate_control.rate_control_mode = MVE_OPT_RATE_CONTROL_MODE_STANDARD;
+				param.data.rate_control.target_bitrate = msg->set_option.target_bitrate;
+			}
+
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.rate_control));
+			break;
+		}
+		case MVX_FW_SET_RATE_CONTROL: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_RATE_CONTROL;
+			if (msg->set_option.rate_control.target_bitrate == 0) {
+				param.data.rate_control.rate_control_mode = MVE_OPT_RATE_CONTROL_MODE_OFF;
+				param.data.rate_control.target_bitrate = 0;
+			} else {
+				param.data.rate_control.rate_control_mode =
+					msg->set_option.rate_control.rate_control_mode;
+				param.data.rate_control.target_bitrate = msg->set_option.rate_control.target_bitrate;
+				if (msg->set_option.rate_control.rate_control_mode ==
+				    MVX_OPT_RATE_CONTROL_MODE_C_VARIABLE) {
+					param.data.rate_control.maximum_bitrate =
+						msg->set_option.rate_control.maximum_bitrate;
+				}
+			}
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.rate_control));
+			break;
+		}
+		case MVX_FW_SET_CROP_LEFT: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_CROP_RARAM_LEFT;
+			opt.data.arg = msg->set_option.crop_left;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_CROP_RIGHT: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_CROP_RARAM_RIGHT;
+			opt.data.arg = msg->set_option.crop_right;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_CROP_TOP: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_CROP_RARAM_TOP;
+			opt.data.arg = msg->set_option.crop_top;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_CROP_BOTTOM: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_CROP_RARAM_BOTTOM;
+			opt.data.arg = msg->set_option.crop_bottom;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_INDEX_PROFILING: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_PROFILING;
+			opt.data.arg = msg->set_option.index_profiling;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_HRD_BUF_SIZE: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_HRD_BUF_SIZE;
+			param.data.arg = msg->set_option.nHRDBufsize;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.arg));
+			break;
+		}
+		case MVX_FW_SET_COLOUR_DESC: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_COLOUR_DESCRIPTION;
+			param.data.colour_description.flags = msg->set_option.colour_desc.flags;
+			switch (msg->set_option.colour_desc.range) {
+			case MVX_FW_RANGE_UNSPECIFIED:
+				param.data.colour_description.range = MVE_BUFFER_PARAM_COLOUR_RANGE_UNSPECIFIED;
+				break;
+			case MVX_FW_RANGE_LIMITED:
+				param.data.colour_description.range = MVE_BUFFER_PARAM_COLOUR_RANGE_LIMITED;
+				break;
+			case MVX_FW_RANGE_FULL:
+				param.data.colour_description.range = MVE_BUFFER_PARAM_COLOUR_RANGE_FULL;
+				break;
+			default:
+				MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+					      "Unknown fw buffer param color desc range. range=%u.",
+					      msg->set_option.colour_desc.range);
+				return -EINVAL;
+			}
+			/* Color primaries according to HEVC E.3.1. */
+			switch (msg->set_option.colour_desc.primaries) {
+			case MVX_FW_PRIMARIES_BT709:
+				param.data.colour_description.colour_primaries = 1;
+				break;
+			case MVX_FW_PRIMARIES_BT470M:
+				param.data.colour_description.colour_primaries = 4;
+				break;
+			case MVX_FW_PRIMARIES_BT601_625:
+				param.data.colour_description.colour_primaries = 5;
+				break;
+			case MVX_FW_PRIMARIES_BT601_525:
+				param.data.colour_description.colour_primaries = 6;
+				break;
+			case MVX_FW_PRIMARIES_GENERIC_FILM:
+				param.data.colour_description.colour_primaries = 8;
+				break;
+			case MVX_FW_PRIMARIES_BT2020:
+				param.data.colour_description.colour_primaries = 9;
+				break;
+			default:
+				param.data.colour_description.colour_primaries = 2;
+				break;
+			}
+			/* Transfer characteristics according to HEVC E.3.1. */
+			switch (msg->set_option.colour_desc.transfer) {
+			case MVX_FW_TRANSFER_BT1361:
+				param.data.colour_description.transfer_characteristics = 12;
+				break;
+			case MVX_FW_TRANSFER_GAMMA22:
+				param.data.colour_description.transfer_characteristics = 4;
+				break;
+			case MVX_FW_TRANSFER_GAMMA28:
+				param.data.colour_description.transfer_characteristics = 5;
+				break;
+			case MVX_FW_TRANSFER_SMPTE170M:
+				param.data.colour_description.transfer_characteristics = 6;
+				break;
+			case MVX_FW_TRANSFER_SMPTE240M:
+				param.data.colour_description.transfer_characteristics = 7;
+				break;
+			case MVX_FW_TRANSFER_LINEAR:
+				param.data.colour_description.transfer_characteristics = 8;
+				break;
+			case MVX_FW_TRANSFER_HLG:
+				param.data.colour_description.transfer_characteristics = 18;
+				break;
+			case MVX_FW_TRANSFER_XVYCC:
+				param.data.colour_description.transfer_characteristics = 11;
+				break;
+			case MVX_FW_TRANSFER_SRGB:
+				param.data.colour_description.transfer_characteristics = 13;
+				break;
+			case MVX_FW_TRANSFER_ST2084:
+				param.data.colour_description.transfer_characteristics = 16;
+				break;
+			case MVX_FW_TRANSFER_ST428:
+				param.data.colour_description.transfer_characteristics = 17;
+				break;
+			default:
+				param.data.colour_description.transfer_characteristics = 2;
+				break;
+			}
+			/* Matrix coefficient according to HEVC E.3.1. */
+			switch (msg->set_option.colour_desc.matrix) {
+			case MVX_FW_MATRIX_BT709:
+				param.data.colour_description.matrix_coeff = 1;
+				break;
+			case MVX_FW_MATRIX_BT470M:
+				param.data.colour_description.matrix_coeff = 4;
+				break;
+			case MVX_FW_MATRIX_BT601:
+				param.data.colour_description.matrix_coeff = 6;
+				break;
+			case MVX_FW_MATRIX_SMPTE240M:
+				param.data.colour_description.matrix_coeff = 7;
+				break;
+			case MVX_FW_MATRIX_BT2020:
+				param.data.colour_description.matrix_coeff = 9;
+				break;
+			case MVX_FW_MATRIX_BT2020Constant:
+				param.data.colour_description.matrix_coeff = 10;
+				break;
+			default:
+				param.data.colour_description.matrix_coeff = 2;
+				break;
+			}
+			param.data.colour_description.sar_height = msg->set_option.colour_desc.sar_height;
+			param.data.colour_description.sar_width = msg->set_option.colour_desc.sar_width;
+			if (msg->set_option.colour_desc.aspect_ratio_idc != 0) {
+				param.data.colour_description.aspect_ratio_idc =
+					msg->set_option.colour_desc.aspect_ratio_idc;
+				param.data.colour_description.aspect_ratio_info_present_flag = 1;
+			}
+			if (msg->set_option.colour_desc.video_format != 0) {
+				param.data.colour_description.video_format = msg->set_option.colour_desc.video_format;
+				param.data.colour_description.video_format_present_flag = 1;
+			}
+			if (msg->set_option.colour_desc.time_scale != 0 ||
+			    msg->set_option.colour_desc.num_units_in_tick != 0) {
+				param.data.colour_description.time_scale = msg->set_option.colour_desc.time_scale;
+				param.data.colour_description.num_units_in_tick =
+					msg->set_option.colour_desc.num_units_in_tick;
+				param.data.colour_description.timing_flag_info_present_flag = 1;
+			}
+			if (msg->set_option.colour_desc.flags & MVX_FW_COLOR_DESC_CONTENT_VALID) {
+				param.data.colour_description.avg_content_light_level =
+					msg->set_option.colour_desc.content.luminance_average;
+				param.data.colour_description.max_content_light_level =
+					msg->set_option.colour_desc.content.luminance_max;
+			}
+			if (msg->set_option.colour_desc.flags & MVX_FW_COLOR_DESC_DISPLAY_VALID) {
+				param.data.colour_description.mastering_display_primaries_x[0] =
+					msg->set_option.colour_desc.display.r.x;
+				param.data.colour_description.mastering_display_primaries_x[1] =
+					msg->set_option.colour_desc.display.g.x;
+				param.data.colour_description.mastering_display_primaries_x[2] =
+					msg->set_option.colour_desc.display.b.x;
+				param.data.colour_description.mastering_display_primaries_y[0] =
+					msg->set_option.colour_desc.display.r.y;
+				param.data.colour_description.mastering_display_primaries_y[1] =
+					msg->set_option.colour_desc.display.g.y;
+				param.data.colour_description.mastering_display_primaries_y[2] =
+					msg->set_option.colour_desc.display.b.y;
+				param.data.colour_description.mastering_white_point_x =
+					msg->set_option.colour_desc.display.w.x;
+				param.data.colour_description.mastering_white_point_y =
+					msg->set_option.colour_desc.display.w.y;
+				param.data.colour_description.max_display_mastering_luminance =
+					msg->set_option.colour_desc.display.luminance_min;
+				param.data.colour_description.min_display_mastering_luminance =
+					msg->set_option.colour_desc.display.luminance_max;
+			}
+
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.colour_description));
+			break;
+		}
+		case MVX_FW_SET_SEI_USERDATA: {
+			struct mve_buffer_param param;
+			param.type = MVE_BUFFER_PARAM_TYPE_SEI_USER_DATA_UNREGISTERED;
+			param.data.user_data_unregistered.user_data_len = msg->set_option.userdata.user_data_len;
+			param.data.user_data_unregistered.flags = msg->set_option.userdata.flags;
+			memcpy(&param.data.user_data_unregistered.uuid, &msg->set_option.userdata.uuid,
+			       sizeof(param.data.user_data_unregistered.uuid));
+			memcpy(&param.data.user_data_unregistered.user_data, &msg->set_option.userdata.user_data,
+			       sizeof(param.data.user_data_unregistered.user_data));
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.user_data_unregistered));
+			break;
+		}
+		case MVX_FW_SET_NALU_FORMAT: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_NALU_FORMAT;
+			ret = to_mve_nalu_format(msg->set_option.nalu_format, &opt.data.arg);
+
+			if (ret == 0)
+				ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_STREAM_ESCAPING: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_STREAM_ESCAPING;
+			opt.data.arg = msg->set_option.stream_escaping ? 1 : 0;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_PROFILE_LEVEL: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_PROFILE_LEVEL;
+			ret = fw->ops_priv.to_mve_profile(msg->set_option.profile_level.profile,
+							  &opt.data.profile_level.profile);
+			if (ret != 0)
+				return ret;
+
+			ret = fw->ops_priv.to_mve_level(msg->set_option.profile_level.level,
+							&opt.data.profile_level.level);
+			if (ret != 0)
+				return ret;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.profile_level));
+
+			break;
+		}
+		case MVX_FW_SET_IGNORE_STREAM_HEADERS: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_IGNORE_STREAM_HEADERS;
+			opt.data.arg = msg->set_option.ignore_stream_headers ? 1 : 0;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_FRAME_REORDERING: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_FRAME_REORDERING;
+			opt.data.arg = msg->set_option.frame_reordering ? 1 : 0;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_INTBUF_SIZE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_INTBUF_SIZE;
+			opt.data.arg = msg->set_option.intbuf_size;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_P_FRAMES: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_P_FRAMES;
+			opt.data.arg = msg->set_option.pb_frames;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_B_FRAMES: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_B_FRAMES;
+			opt.data.arg = msg->set_option.pb_frames;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_GOP_TYPE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_GOP_TYPE;
+			ret = to_mve_gop_type(msg->set_option.gop_type, &opt.data.arg);
+			if (ret == 0)
+				ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_INTRA_MB_REFRESH: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_INTRA_MB_REFRESH;
+			opt.data.arg = msg->set_option.intra_mb_refresh;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_CONSTR_IPRED: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_CONSTR_IPRED;
+			opt.data.arg = msg->set_option.constr_ipred ? 1 : 0;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_ENTROPY_SYNC: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_ENTROPY_SYNC;
+			opt.data.arg = msg->set_option.entropy_sync ? 1 : 0;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_TEMPORAL_MVP: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_TEMPORAL_MVP;
+			opt.data.arg = msg->set_option.temporal_mvp ? 1 : 0;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_TILES: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_TILES;
+			opt.data.tiles.tile_rows = msg->set_option.tile.rows;
+			opt.data.tiles.tile_cols = msg->set_option.tile.cols;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.tiles));
+			break;
+		}
+		case MVX_FW_SET_MIN_LUMA_CB_SIZE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_MIN_LUMA_CB_SIZE;
+			opt.data.arg = msg->set_option.min_luma_cb_size;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_QP_RANGE: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_RATE_CONTROL_QP_RANGE;
+			param.data.rate_control_qp_range.qp_min = msg->set_option.qp_range.min;
+			param.data.rate_control_qp_range.qp_max = msg->set_option.qp_range.max;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.rate_control_qp_range));
+			break;
+		}
+		case MVX_FW_SET_ENTROPY_MODE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_H264_CABAC;
+			ret = to_mve_h264_cabac(msg->set_option.entropy_mode, &opt.data.arg);
+			if (ret == 0)
+				ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_SLICE_SPACING_MB: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_SLICE_SPACING;
+			opt.data.arg = msg->set_option.slice_spacing_mb;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_VP9_PROB_UPDATE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_VP9_PROB_UPDATE;
+			ret = to_mve_vp9_prob_update(msg->set_option.vp9_prob_update, &opt.data.arg);
+			if (ret == 0)
+				ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_MV_SEARCH_RANGE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_MV_SEARCH_RANGE;
+			opt.data.motion_vector_search_range.mv_search_range_x = msg->set_option.mv.x;
+			opt.data.motion_vector_search_range.mv_search_range_y = msg->set_option.mv.y;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.motion_vector_search_range));
+			break;
+		}
+		case MVX_FW_SET_BITDEPTH: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_STREAM_BITDEPTH;
+			opt.data.bitdepth.luma_bitdepth = msg->set_option.bitdepth.luma;
+			opt.data.bitdepth.chroma_bitdepth = msg->set_option.bitdepth.chroma;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.bitdepth));
+			break;
+		}
+		case MVX_FW_SET_CHROMA_FORMAT: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_STREAM_CHROMA_FORMAT;
+			opt.data.arg = msg->set_option.chroma_format;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_RGB_TO_YUV_MODE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_RGB_TO_YUV_MODE;
+			ret = to_mve_rgb_to_yuv(msg->set_option.rgb_to_yuv_mode, &opt.data.arg);
+			if (ret == 0)
+				ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+
+			break;
+		}
+		case MVX_FW_SET_BAND_LIMIT: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_BANDWIDTH_LIMIT;
+			opt.data.arg = msg->set_option.band_limit;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_CABAC_INIT_IDC: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_ENC_CABAC_INIT_IDC;
+			opt.data.arg = msg->set_option.cabac_init_idc;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_QP_I: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_QP_I;
+			param.data.qp.qp = msg->set_option.qp;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.qp));
+			break;
+		}
+		case MVX_FW_SET_QP_P: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_QP_P;
+			param.data.qp.qp = msg->set_option.qp;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.qp));
+			break;
+		}
+		case MVX_FW_SET_QP_B: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_QP_B;
+			param.data.qp.qp = msg->set_option.qp;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.qp));
+			break;
+		}
+		case MVX_FW_SET_GOP_RESET: {
+			struct mve_buffer_param param;
+			param.type = MVE_BUFFER_PARAM_TYPE_GOP_RESET;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.arg));
+			break;
+		}
+		case MVX_FW_SET_RESYNC_INTERVAL: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_RESYNC_INTERVAL;
+			opt.data.arg = msg->set_option.resync_interval;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_QUANT_TABLE: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_QUANT_TABLE;
+
+			opt.data.quant_table.type = MVE_OPT_QUANT_TABLE_LUMA;
+			memcpy(opt.data.quant_table.matrix, msg->set_option.quant_tbl.luma,
+			       sizeof(opt.data.quant_table.matrix));
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.quant_table));
+			if (ret != 0)
+				break;
+
+			opt.data.quant_table.type = MVE_OPT_QUANT_TABLE_CHROMA;
+			memcpy(opt.data.quant_table.matrix, msg->set_option.quant_tbl.chroma,
+			       sizeof(opt.data.quant_table.matrix));
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.quant_table));
+			break;
+		}
+		case MVX_FW_SET_WATCHDOG_TIMEOUT: {
+			struct mve_request_set_option opt;
+
+			opt.index = MVE_SET_OPT_INDEX_WATCHDOG_TIMEOUT;
+			opt.data.arg = msg->set_option.watchdog_timeout;
+
+			ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			break;
+		}
+		case MVX_FW_SET_ROI_REGIONS: {
+			struct mve_buffer_param param;
+			int i = 0;
+			param.type = MVE_BUFFER_PARAM_TYPE_REGIONS;
+			param.data.regions.n_regions = msg->set_option.roi_config.num_roi;
+			for (; i < msg->set_option.roi_config.num_roi; i++) {
+				param.data.regions.region[i].mbx_left = msg->set_option.roi_config.roi[i].mbx_left;
+				param.data.regions.region[i].mbx_right = msg->set_option.roi_config.roi[i].mbx_right;
+				param.data.regions.region[i].mby_top = msg->set_option.roi_config.roi[i].mby_top;
+				param.data.regions.region[i].mby_bottom = msg->set_option.roi_config.roi[i].mby_bottom;
+				param.data.regions.region[i].qp_delta = msg->set_option.roi_config.roi[i].qp_delta;
+			}
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.regions));
+			break;
+		}
+		case MVX_FW_SET_QP_REGION: {
+			struct mve_buffer_param param;
+
+			param.type = MVE_BUFFER_PARAM_TYPE_QP;
+			param.data.qp.qp = msg->set_option.qp;
+			ret = put_fw_buf_param(fw, &param, sizeof(param.data.qp));
+			break;
+		}
+		case MVX_FW_SET_DSL_FRAME: {
+			struct mve_request_set_option opt;
+			opt.index = MVE_SET_OPT_INDEX_DEC_DOWNSCALE;
+			opt.data.downscaled_frame.width = msg->set_option.dsl_frame.width;
+			opt.data.downscaled_frame.height = msg->set_option.dsl_frame.height;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.index) + sizeof(opt.data.downscaled_frame));
+			break;
+		}
+		case MVX_FW_SET_LONG_TERM_REF: {
+			struct mve_request_set_option opt;
+			if (msg->set_option.ltr.mode >= 1 && msg->set_option.ltr.mode <= 8) {
+				opt.index = MVE_SET_OPT_INDEX_ENC_LTR_MODE;
+				opt.data.arg = msg->set_option.ltr.mode;
+				ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			}
+			if (msg->set_option.ltr.period >= 2 && msg->set_option.ltr.period <= 254) {
+				opt.index = MVE_SET_OPT_INDEX_ENC_LTR_PERIOD;
+				opt.data.arg = msg->set_option.ltr.period;
+				ret = put_fw_opt(fw, &opt, sizeof(opt.data.arg));
+			}
+			break;
+		}
+		case MVX_FW_SET_DSL_MODE: {
+			struct mve_request_set_option opt;
+			opt.index = MVE_SET_OPT_INDEX_DEC_DOWNSCALE_POS_MODE;
+			opt.data.dsl_pos.mode = msg->set_option.dsl_pos_mode;
+			ret = put_fw_opt(fw, &opt, sizeof(opt.index) + sizeof(opt.data.dsl_pos));
+			break;
+		}
+		default:
+			ret = -EINVAL;
+		}
+
+		break;
+	}
+	case MVX_FW_CODE_FLUSH: {
+		switch (msg->flush.dir) {
+		case MVX_DIR_INPUT:
+			ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_INPUT_FLUSH, NULL, 0,
+					    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+					    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+			break;
+		case MVX_DIR_OUTPUT:
+			ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_OUTPUT_FLUSH, NULL, 0,
+					    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+					    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+			break;
+		default:
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Invalid flush direction. dir=%d.", msg->flush.dir);
+			return -EINVAL;
+		}
+
+		if (ret == 0)
+			fw->msg_pending++;
+
+		break;
+	}
+	case MVX_FW_CODE_BUFFER: {
+		struct mve_comm_area_host *host;
+		struct mve_comm_area_mve *mve;
+		enum mvx_log_fwif_channel channel;
+		enum mvx_fw_buffer_attr mve_buf_attr;
+		enum mvx_fw_buffer_attr host_buf_attr;
+
+		if (msg->buf->dir == MVX_DIR_INPUT) {
+			host = fw->buf_in_host;
+			mve = fw->buf_in_mve;
+			mve_buf_attr = fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE];
+			host_buf_attr = fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST];
+			channel = MVX_LOG_FWIF_CHANNEL_INPUT_BUFFER;
+		} else {
+			host = fw->buf_out_host;
+			mve = fw->buf_out_mve;
+			mve_buf_attr = fw->buf_attr[MVX_FW_REGION_BUF_OUT_MVE];
+			host_buf_attr = fw->buf_attr[MVX_FW_REGION_BUF_OUT_HOST];
+			channel = MVX_LOG_FWIF_CHANNEL_OUTPUT_BUFFER;
+		}
+
+		if (mvx_is_frame(msg->buf->format))
+			if ((msg->buf->flags & MVX_BUFFER_FRAME_FLAG_GENERAL) == MVX_BUFFER_FRAME_FLAG_GENERAL) {
+				ret = put_buffer_general(fw, host, mve, msg, channel, mve_buf_attr, host_buf_attr);
+			} else {
+				ret = put_buffer_frame(fw, host, mve, msg, channel, mve_buf_attr, host_buf_attr);
+			}
+		else
+			ret = put_buffer_bitstream(fw, host, mve, msg, channel, mve_buf_attr, host_buf_attr);
+
+		break;
+	}
+	case MVX_FW_CODE_IDLE_ACK: {
+		if (fw->ops_priv.send_idle_ack != NULL)
+			ret = fw->ops_priv.send_idle_ack(fw);
+
+		break;
+	}
+	case MVX_FW_CODE_EOS: {
+		struct mve_comm_area_host *host;
+		struct mve_comm_area_mve *mve;
+		enum mvx_log_fwif_channel channel;
+
+		/* The message is on the MVX_DIR_INPUT side. */
+		host = fw->buf_in_host;
+		mve = fw->buf_in_mve;
+		channel = MVX_LOG_FWIF_CHANNEL_INPUT_BUFFER;
+
+		if (msg->eos_is_frame != false) {
+			struct mve_buffer_frame f = {
+				.host_handle = MVX_FW_CODE_EOS,
+				.frame_flags = MVE_BUFFER_FRAME_FLAG_EOS,
+				.format      = MVE_FORMAT_YUV420_NV12
+			};
+			ret = write_message(fw, host, mve, MVE_BUFFER_CODE_FRAME, &f, sizeof(f), channel,
+					    fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE],
+					    fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST]);
+		} else {
+			struct mve_buffer_bitstream b = {
+				.host_handle        = MVX_FW_CODE_EOS,
+				.bitstream_buf_addr = MVE_MEM_REGION_PROTECTED_ADDR_BEGIN,
+				.bitstream_flags    = MVE_BUFFER_BITSTREAM_FLAG_EOS
+			};
+			ret = write_message(fw, host, mve, MVE_BUFFER_CODE_BITSTREAM, &b, sizeof(b), channel,
+					    fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE],
+					    fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST]);
+		}
+
+		break;
+	}
+	case MVX_FW_CODE_DUMP: {
+		ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_DUMP, NULL, 0,
+				    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+				    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+		fw->msg_pending++;
+		break;
+	}
+	case MVX_FW_CODE_DEBUG: {
+		ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_DEBUG, &msg->arg, sizeof(msg->arg),
+				    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+				    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+		fw->msg_pending++;
+		break;
+	}
+	default: {
+		ret = -EINVAL;
+		break;
+	}
+	}
+
+	if (ret != 0)
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Firmware put message failed. ret=%d.", ret);
+
+	return ret;
+}
+
+/**
+ * find_pages() - Find a page allocate in the map.
+ * @fw:		Pointer to firmware object.
+ * @va:		MVE virtual address.
+ *
+ * Return: Pointer to pages, NULL if not found.
+ */
+static struct mvx_mmu_pages *find_pages(struct mvx_fw *fw, mvx_mmu_va va)
+{
+	struct mvx_mmu_pages *pages;
+
+	hash_for_each_possible(fw->rpc_mem, pages, node, va) {
+		if (pages->va == va)
+			return pages;
+	}
+
+	return NULL;
+}
+
+static void rpc_mem_alloc(struct mvx_fw *fw, struct mve_rpc_communication_area *rpc_area)
+{
+	union mve_rpc_params *p = &rpc_area->params;
+	enum mvx_fw_region region;
+	struct mvx_mmu_pages *pages;
+	size_t npages;
+	size_t max_pages;
+	mvx_mmu_va va = 0;
+	mvx_mmu_va va0, va_next;
+	mvx_mmu_va end;
+	int ret;
+	uint8_t log2_alignment;
+	uint32_t alignment_pages;
+	uint32_t alignment_bytes;
+	uint32_t total_used_pages = 0;
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ret = mutex_lock_interruptible(&fw->rpcmem_mutex);
+		if (ret != 0) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "Cannot protect RPC alloc list.");
+			goto out;
+		}
+	}
+
+	switch (p->mem_alloc.region) {
+	case MVE_MEM_REGION_PROTECTED:
+		region = MVX_FW_REGION_PROTECTED;
+		total_used_pages = fw->latest_used_region_protected_pages;
+		break;
+	case MVE_MEM_REGION_OUTBUF:
+		region = MVX_FW_REGION_FRAMEBUF;
+		total_used_pages = fw->latest_used_region_outbuf_pages;
+		break;
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unsupported RPC mem alloc region. region=%u.",
+			      p->mem_alloc.region);
+		goto unlock_mutex;
+	}
+
+	ret = fw->ops.get_region(region, &va, &end);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	va0 = va;
+
+	npages = DIV_ROUND_UP(p->mem_alloc.size, MVE_PAGE_SIZE);
+	max_pages = DIV_ROUND_UP(p->mem_alloc.max_size, MVE_PAGE_SIZE);
+
+	if (fw->fw_bin->securevideo != false) {
+		struct dma_buf *dmabuf;
+
+		dmabuf = mvx_secure_mem_alloc(fw->fw_bin->secure.secure, p->mem_alloc.size);
+		if (IS_ERR(dmabuf))
+			goto unlock_mutex;
+
+		pages = mvx_mmu_alloc_pages_dma_buf(fw->dev, dmabuf, max_pages);
+		if (IS_ERR(pages)) {
+			dma_buf_put(dmabuf);
+			goto unlock_mutex;
+		}
+	} else {
+		pages = mvx_mmu_alloc_pages(fw->dev, npages, max_pages);
+		if (IS_ERR(pages))
+			goto unlock_mutex;
+	}
+
+	va += (total_used_pages << MVE_PAGE_SHIFT);
+	log2_alignment = p->mem_alloc.log2_alignment <= MVE_PAGE_SHIFT ? MVE_PAGE_SHIFT : p->mem_alloc.log2_alignment;
+	alignment_bytes = 1 << log2_alignment;
+	alignment_pages = alignment_bytes >> MVE_PAGE_SHIFT;
+	ret = -EINVAL;
+	while (va < end) {
+		va = (va + alignment_bytes - 1) & ~(alignment_bytes - 1);
+		ret = mvx_mmu_map_pages(fw->mmu, va, pages, MVX_ATTR_SHARED_RW, MVX_ACCESS_READ_WRITE);
+		if (ret == 0) {
+			va_next = va + MVE_PAGE_SIZE * pages->capacity;
+			total_used_pages = (va_next - va0) >> MVE_PAGE_SHIFT;
+			break;
+		}
+
+		// va += 16 * 1024 * 1024; /* 16MB */
+		va += alignment_bytes;
+	}
+
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to find memory region for RPC alloc.");
+		mvx_mmu_free_pages(pages);
+		va = 0;
+		goto unlock_mutex;
+	}
+
+	switch (p->mem_alloc.region) {
+	case MVE_MEM_REGION_PROTECTED:
+		fw->latest_used_region_protected_pages = total_used_pages;
+		break;
+	case MVE_MEM_REGION_OUTBUF:
+		fw->latest_used_region_outbuf_pages = total_used_pages;
+		break;
+	default:
+		break;
+	}
+
+	hash_add(fw->rpc_mem, &pages->node, pages->va);
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO,
+		      "RPC alloc memory. size=%u, max_size=%u, region=%u, npages=%zu, va=0x%x.", p->mem_alloc.size,
+		      p->mem_alloc.max_size, p->mem_alloc.region, npages, va);
+
+unlock_mutex:
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		mutex_unlock(&fw->rpcmem_mutex);
+
+out:
+	rpc_area->size = sizeof(uint32_t);
+	p->data[0] = va;
+}
+
+static void rpc_mem_resize(struct mvx_fw *fw, struct mve_rpc_communication_area *rpc_area)
+{
+	union mve_rpc_params *p = &rpc_area->params;
+	struct mvx_mmu_pages *pages;
+	mvx_mmu_va va = 0;
+	int ret;
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ret = mutex_lock_interruptible(&fw->rpcmem_mutex);
+		if (ret != 0) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "Cannot protect RPC alloc list.");
+			goto out;
+		}
+	}
+
+	pages = find_pages(fw, p->mem_resize.ve_pointer);
+	if (pages != 0) {
+		size_t size;
+		size_t npages;
+		int ret;
+
+		if (fw->fw_bin->securevideo != false) {
+			size = mvx_mmu_size_pages(pages);
+
+			/* The size of RPC memory is only increased. */
+			if (size < p->mem_resize.new_size) {
+				struct dma_buf *dmabuf;
+
+				size = p->mem_resize.new_size - size;
+
+				/* Allocate a new secure DMA buffer. */
+				dmabuf = mvx_secure_mem_alloc(fw->fw_bin->secure.secure, size);
+				if (IS_ERR(dmabuf))
+					goto unlock_mutex;
+
+				ret = mvx_mmu_pages_append_dma_buf(pages, dmabuf);
+				if (ret != 0) {
+					dma_buf_put(dmabuf);
+					goto unlock_mutex;
+				}
+			}
+		} else {
+			/* Resize the allocated pages. */
+			npages = DIV_ROUND_UP(p->mem_resize.new_size, MVE_PAGE_SIZE);
+			ret = mvx_mmu_resize_pages(pages, npages);
+			if (ret != 0) {
+				MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+					      "Failed to resize RPC mapped pages. ret=%d.", ret);
+				goto unlock_mutex;
+			}
+		}
+
+		va = pages->va;
+	} else {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Could not find pages for RPC resize. va=0x%x.",
+			      p->mem_resize.ve_pointer);
+	}
+
+	fw->client_ops->flush_mmu(fw->csession);
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "RPC resize memory. va=0x%x, new_size=%u.", p->mem_resize.ve_pointer,
+		      p->mem_resize.new_size);
+
+unlock_mutex:
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		mutex_unlock(&fw->rpcmem_mutex);
+
+out:
+	rpc_area->size = sizeof(uint32_t);
+	p->data[0] = va;
+}
+
+static void rpc_mem_free(struct mvx_fw *fw, struct mve_rpc_communication_area *rpc_area)
+{
+	union mve_rpc_params *p = &rpc_area->params;
+	struct mvx_mmu_pages *pages;
+	int ret;
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ret = mutex_lock_interruptible(&fw->rpcmem_mutex);
+		if (ret != 0) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "Cannot protect RPC alloc list.");
+			return;
+		}
+	}
+
+	pages = find_pages(fw, p->mem_free.ve_pointer);
+	if (pages != NULL) {
+		hash_del(&pages->node);
+		if (MVE_MEM_REGION_PROTECTED_ADDR_BEGIN <= p->mem_free.ve_pointer &&
+		    p->mem_free.ve_pointer < MVE_MEM_REGION_PROTECTED_ADDR_END) {
+			fw->latest_used_region_protected_pages =
+				fw->latest_used_region_protected_pages > pages->capacity
+					? fw->latest_used_region_protected_pages - pages->capacity
+					: 0;
+		} else if (MVE_MEM_REGION_FRAMEBUF_ADDR_BEGIN <= p->mem_free.ve_pointer &&
+			   p->mem_free.ve_pointer < MVE_MEM_REGION_FRAMEBUF_ADDR_END) {
+			fw->latest_used_region_outbuf_pages =
+				fw->latest_used_region_outbuf_pages > pages->capacity
+					? fw->latest_used_region_outbuf_pages - pages->capacity
+					: 0;
+		}
+		mvx_mmu_free_pages(pages);
+	} else {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Could not find pages for RPC free. va=0x%x.",
+			      p->mem_free.ve_pointer);
+	}
+
+	fw->client_ops->flush_mmu(fw->csession);
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "RPC free memory. va=0x%x.", p->mem_free.ve_pointer);
+
+	rpc_area->size = 0;
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		mutex_unlock(&fw->rpcmem_mutex);
+}
+
+/**
+ * rstrip() - Remove trailing chars from string.
+ * @s:		String to be stripped.
+ * @t:		String containing chars to be stripped.
+ *
+ * Return: Pointer to stripped string.
+ */
+static char *rstrip(char *str, char *trim)
+{
+	size_t l = strlen(str);
+
+	while (l-- > 0) {
+		char *t;
+
+		for (t = trim; *t != '\0'; t++)
+			if (str[l] == *t) {
+				str[l] = '\0';
+				break;
+			}
+
+		if (*t == '\0')
+			break;
+	}
+
+	return str;
+}
+
+static int handle_rpc_v2(struct mvx_fw *fw)
+{
+	struct mve_rpc_communication_area *rpc_area = fw->rpc;
+	int ret = 0;
+
+	if (fw->buf_attr[MVX_FW_REGION_RPC] == MVX_FW_BUF_CACHEABLE) {
+		dma_sync_single_for_cpu(fw->dev, virt_to_phys(rpc_area), sizeof(*rpc_area), DMA_FROM_DEVICE);
+	}
+
+	if (rpc_area->state == MVE_RPC_STATE_PARAM) {
+		ret = 1;
+
+		/* Log RPC request. */
+		MVX_LOG_EXECUTE(&mvx_log_fwif_if, MVX_LOG_INFO,
+				log_rpc(fw->session, MVX_LOG_FWIF_DIRECTION_FIRMWARE_TO_HOST, rpc_area));
+
+		switch (rpc_area->call_id) {
+		case MVE_RPC_FUNCTION_DEBUG_PRINTF: {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "RPC_PRINT=%s",
+				      rstrip(rpc_area->params.debug_print.string, "\n\r"));
+			break;
+		}
+		case MVE_RPC_FUNCTION_MEM_ALLOC: {
+			rpc_mem_alloc(fw, rpc_area);
+			break;
+		}
+		case MVE_RPC_FUNCTION_MEM_RESIZE: {
+			rpc_mem_resize(fw, rpc_area);
+			break;
+		}
+		case MVE_RPC_FUNCTION_MEM_FREE: {
+			rpc_mem_free(fw, rpc_area);
+			break;
+		}
+		default:
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "Unsupported RPC request. call_id=%u.",
+				      rpc_area->call_id);
+			ret = -EINVAL;
+			break;
+		}
+
+		/*
+		 * Make sure the whole RPC message body has been written before
+		 * the RPC message area is returned to the firmware.
+		 */
+		wmb();
+		rpc_area->state = MVE_RPC_STATE_RETURN;
+
+		/* Make sure state is written before memory is flushed. */
+		if (fw->buf_attr[MVX_FW_REGION_RPC] == MVX_FW_BUF_CACHEABLE) {
+			wmb();
+			dma_sync_single_for_device(fw->dev, virt_to_phys(rpc_area), sizeof(*rpc_area), DMA_TO_DEVICE);
+		}
+
+		/* Log RPC response. */
+		MVX_LOG_EXECUTE(&mvx_log_fwif_if, MVX_LOG_INFO,
+				log_rpc(fw->session, MVX_LOG_FWIF_DIRECTION_HOST_TO_FIRMWARE, rpc_area));
+
+		fw->client_ops->send_irq(fw->csession);
+	}
+
+	return ret;
+}
+
+#ifdef MVX_FW_DEBUG_ENABLE
+#define RAM_PRINTBUF_SIZE MVE_FW_PRINT_RAM_SIZE
+#define RAM_PRINT_MAX_LEN (128)
+#define RAM_PRINT_BUF_CNT ((RAM_PRINTBUF_SIZE / RAM_PRINT_MAX_LEN) - 1)
+#define RAM_PRINT_FLAG (0x11223356)
+static int handle_fw_ram_print_v2(struct mvx_fw *fw)
+{
+	struct mve_fw_ram_print_head_aera *rpt_area = fw->fw_print_ram;
+	int ret = 0;
+	uint32_t wr_cnt;
+	uint32_t rd_cnt = 0;
+	uint32_t cnt;
+	uint32_t rd_idx;
+	char *print_buf = NULL;
+
+	dma_sync_single_for_cpu(fw->dev, virt_to_phys(rpt_area), sizeof(*rpt_area), DMA_FROM_DEVICE);
+
+	wr_cnt = rpt_area->wr_cnt;
+	rd_cnt = rpt_area->rd_cnt;
+	cnt = (rd_cnt <= wr_cnt) ? wr_cnt - rd_cnt : wr_cnt - rd_cnt + (uint32_t)~0u;
+
+	if (RAM_PRINT_FLAG == rpt_area->flag && RAM_PRINT_BUF_CNT > rpt_area->index && cnt) {
+		// printk("RPT:flag=%x, idx=%u, wr_cnt=%u, rd_cnt=%u.\n", rpt_area->flag, rpt_area->index, wr_cnt,
+		// rd_cnt);
+
+		while (cnt--) {
+			rd_idx = rd_cnt % RAM_PRINT_BUF_CNT;
+			print_buf = (fw->fw_print_ram + RAM_PRINT_MAX_LEN) + rd_idx * RAM_PRINT_MAX_LEN;
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "FW-%u: %s\n", rd_cnt, print_buf);
+			rd_cnt++;
+		}
+
+		rpt_area->rd_cnt = rd_cnt;
+		/* Make sure rpt_area->rd_cnt is written before memory is flushed. */
+		wmb();
+		dma_sync_single_for_device(fw->dev, virt_to_phys(&rpt_area->rd_cnt), sizeof(rpt_area->rd_cnt),
+					   DMA_TO_DEVICE);
+
+		ret = 1;
+	}
+
+	return ret;
+}
+#endif
+
+static void unmap_msq(struct mvx_fw *fw, void **data, enum mvx_fw_region region)
+{
+	int ret;
+	mvx_mmu_va begin;
+	mvx_mmu_va end;
+
+	if (*data == NULL)
+		return;
+
+	ret = fw->ops.get_region(region, &begin, &end);
+	if (ret == 0)
+		mvx_mmu_unmap_va(fw->mmu, begin, MVE_PAGE_SIZE);
+
+	mvx_mmu_free_page(fw->dev, virt_to_phys(*data));
+
+	*data = NULL;
+}
+
+static int map_msq(struct mvx_fw *fw, void **data, enum mvx_fw_region region)
+{
+	phys_addr_t page;
+	mvx_mmu_va begin;
+	mvx_mmu_va end;
+	int ret;
+
+	/* Get virtual address where the message queue is to be mapped. */
+	ret = fw->ops.get_region(region, &begin, &end);
+	if (ret != 0)
+		return ret;
+
+	/* Allocate page and store Linux logical address in 'data'. */
+	page = mvx_mmu_alloc_page(fw->dev);
+	if (page == 0)
+		return -ENOMEM;
+
+	/* Memory map region. */
+	ret = mvx_mmu_map_pa(fw->mmu, begin, page, MVE_PAGE_SIZE, MVX_ATTR_SHARED_RW, MVX_ACCESS_READ_WRITE);
+	if (ret != 0) {
+		mvx_mmu_free_page(fw->dev, page);
+		return ret;
+	}
+
+	*data = phys_to_virt(page);
+
+	return 0;
+}
+
+static int map_msq_uncache(struct mvx_fw *fw, void **data, enum mvx_fw_region region)
+{
+	phys_addr_t page;
+	mvx_mmu_va begin;
+	mvx_mmu_va end;
+	void *vir_addr = NULL;
+	int ret;
+
+	/* Get virtual address where the message queue is to be mapped. */
+	ret = fw->ops.get_region(region, &begin, &end);
+	if (ret != 0)
+		return ret;
+
+	/* Allocate page and store Linux logical address in 'data'. */
+	page = mvx_mmu_dma_alloc_coherent(fw->dev, &vir_addr);
+	if (page == 0) {
+		return -ENOMEM;
+	}
+
+	/* Memory map region. */
+	ret = mvx_mmu_map_pa(fw->mmu, begin, page, MVE_PAGE_SIZE, MVX_ATTR_SHARED_RW, MVX_ACCESS_READ_WRITE);
+	if (ret != 0) {
+		mvx_mmu_dma_free_coherent(fw->dev, page, vir_addr);
+		return ret;
+	}
+
+	*data = vir_addr;
+	fw->buf_pa_addr[region] = page;
+
+	return 0;
+}
+
+static void unmap_msq_uncache(struct mvx_fw *fw, void **data, enum mvx_fw_region region)
+{
+	int ret;
+	mvx_mmu_va begin;
+	mvx_mmu_va end;
+
+	if (*data == NULL)
+		return;
+
+	ret = fw->ops.get_region(region, &begin, &end);
+	if (ret == 0) {
+		mvx_mmu_unmap_va(fw->mmu, begin, MVE_PAGE_SIZE);
+	}
+
+	mvx_mmu_dma_free_coherent(fw->dev, fw->buf_pa_addr[region], *data);
+
+	*data = NULL;
+	fw->buf_pa_addr[region] = 0;
+}
+
+#ifdef MVX_FW_DEBUG_ENABLE
+static void unmap_fw_print_ram(struct mvx_fw *fw, void **data, enum mvx_fw_region region)
+{
+	int ret;
+	mvx_mmu_va begin;
+	mvx_mmu_va end;
+
+	if (*data == NULL)
+		return;
+
+	ret = fw->ops.get_region(region, &begin, &end);
+	if (ret == 0)
+		mvx_mmu_unmap_va(fw->mmu, begin, MVE_FW_PRINT_RAM_SIZE);
+
+	mvx_mmu_free_contiguous_pages(fw->dev, virt_to_phys(*data), MVE_FW_PRINT_RAM_SIZE >> PAGE_SHIFT);
+
+	*data = NULL;
+}
+
+static int map_fw_print_ram(struct mvx_fw *fw, void **data, enum mvx_fw_region region)
+{
+	phys_addr_t page;
+	mvx_mmu_va begin;
+	mvx_mmu_va end;
+	int ret;
+
+	/* Get virtual address where the message queue is to be mapped. */
+	ret = fw->ops.get_region(region, &begin, &end);
+	if (ret != 0)
+		return ret;
+
+	/* Allocate pages and store Linux logical address in 'data'. */
+	page = mvx_mmu_alloc_contiguous_pages(fw->dev, MVE_FW_PRINT_RAM_SIZE >> PAGE_SHIFT);
+	if (page == 0)
+		return -ENOMEM;
+
+	/* Memory map region. */
+	ret = mvx_mmu_map_pa(fw->mmu, begin, page, MVE_FW_PRINT_RAM_SIZE, MVX_ATTR_SHARED_RW, MVX_ACCESS_READ_WRITE);
+	if (ret != 0) {
+		mvx_mmu_free_contiguous_pages(fw->dev, page, MVE_FW_PRINT_RAM_SIZE >> PAGE_SHIFT);
+		return ret;
+	}
+
+	*data = phys_to_virt(page);
+
+	return 0;
+}
+#endif
+
+static void unmap_protocol_v2(struct mvx_fw *fw)
+{
+	struct mvx_mmu_pages *pages;
+	struct hlist_node *tmp;
+	int bkt;
+
+	fw->unmap_op[fw->buf_attr[MVX_FW_REGION_MSG_HOST]](fw, &fw->msg_host, MVX_FW_REGION_MSG_HOST);
+	fw->unmap_op[fw->buf_attr[MVX_FW_REGION_MSG_MVE]](fw, &fw->msg_mve, MVX_FW_REGION_MSG_MVE);
+	fw->unmap_op[fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST]](fw, &fw->buf_in_host, MVX_FW_REGION_BUF_IN_HOST);
+	fw->unmap_op[fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE]](fw, &fw->buf_in_mve, MVX_FW_REGION_BUF_IN_MVE);
+	fw->unmap_op[fw->buf_attr[MVX_FW_REGION_BUF_OUT_HOST]](fw, &fw->buf_out_host, MVX_FW_REGION_BUF_OUT_HOST);
+	fw->unmap_op[fw->buf_attr[MVX_FW_REGION_BUF_OUT_MVE]](fw, &fw->buf_out_mve, MVX_FW_REGION_BUF_OUT_MVE);
+	fw->unmap_op[fw->buf_attr[MVX_FW_REGION_RPC]](fw, &fw->rpc, MVX_FW_REGION_RPC);
+
+#ifdef MVX_FW_DEBUG_ENABLE
+	unmap_fw_print_ram(fw, &fw->fw_print_ram, MVX_FW_REGION_PRINT_RAM);
+#endif
+	fw->latest_used_region_protected_pages = 0;
+	fw->latest_used_region_outbuf_pages = 0;
+
+	hash_for_each_safe(fw->rpc_mem, bkt, tmp, pages, node)
+	{
+		hash_del(&pages->node);
+		mvx_mmu_free_pages(pages);
+	}
+}
+
+static int map_protocol_v2(struct mvx_fw *fw)
+{
+	int ret;
+
+	ret = fw->map_op[fw->buf_attr[MVX_FW_REGION_MSG_HOST]](fw, &fw->msg_host, MVX_FW_REGION_MSG_HOST);
+	if (ret != 0)
+		goto unmap_fw;
+
+	ret = fw->map_op[fw->buf_attr[MVX_FW_REGION_MSG_MVE]](fw, &fw->msg_mve, MVX_FW_REGION_MSG_MVE);
+	if (ret != 0)
+		goto unmap_fw;
+
+	ret = fw->map_op[fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST]](fw, &fw->buf_in_host, MVX_FW_REGION_BUF_IN_HOST);
+	if (ret != 0)
+		goto unmap_fw;
+
+	ret = fw->map_op[fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE]](fw, &fw->buf_in_mve, MVX_FW_REGION_BUF_IN_MVE);
+	if (ret != 0)
+		goto unmap_fw;
+
+	ret = fw->map_op[fw->buf_attr[MVX_FW_REGION_BUF_OUT_HOST]](fw, &fw->buf_out_host, MVX_FW_REGION_BUF_OUT_HOST);
+	if (ret != 0)
+		goto unmap_fw;
+
+	ret = fw->map_op[fw->buf_attr[MVX_FW_REGION_BUF_OUT_MVE]](fw, &fw->buf_out_mve, MVX_FW_REGION_BUF_OUT_MVE);
+	if (ret != 0)
+		goto unmap_fw;
+
+	ret = fw->map_op[fw->buf_attr[MVX_FW_REGION_RPC]](fw, &fw->rpc, MVX_FW_REGION_RPC);
+	if (ret != 0)
+		goto unmap_fw;
+
+#ifdef MVX_FW_DEBUG_ENABLE
+	ret = map_fw_print_ram(fw, &fw->fw_print_ram, MVX_FW_REGION_PRINT_RAM);
+	if (ret != 0)
+		goto unmap_fw;
+#endif
+
+	return 0;
+
+unmap_fw:
+	unmap_protocol_v2(fw);
+
+	return ret;
+}
+
+static void print_pair(char *name_in, char *name_out, struct device *device, struct mve_comm_area_host *host,
+		       struct mve_comm_area_mve *mve, int ind, struct seq_file *s, enum mvx_fw_buffer_attr mve_buf_attr)
+{
+	if (mve_buf_attr == MVX_FW_BUF_CACHEABLE) {
+		dma_sync_single_for_cpu(device, virt_to_phys(mve), MVE_PAGE_SIZE, DMA_FROM_DEVICE);
+	}
+	mvx_seq_printf(s, name_in, ind, "wr=%10d, rd=%10d, avail=%10d\n", host->in_wpos, mve->in_rpos,
+		       (uint16_t)(host->in_wpos - mve->in_rpos));
+	mvx_seq_printf(s, name_out, ind, "wr=%10d, rd=%10d, avail=%10d\n", mve->out_wpos, host->out_rpos,
+		       (uint16_t)(mve->out_wpos - host->out_rpos));
+}
+
+static int print_stat_v2(struct mvx_fw *fw, int ind, struct seq_file *s)
+{
+	print_pair("Msg host->mve", "Msg host<-mve", fw->dev, fw->msg_host, fw->msg_mve, ind, s,
+		   fw->buf_attr[MVX_FW_REGION_MSG_MVE]);
+	print_pair("Inbuf host->mve", "Inbuf host<-mve", fw->dev, fw->buf_in_host, fw->buf_in_mve, ind, s,
+		   fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE]);
+	print_pair("Outbuf host->mve", "Outbuf host<-mve", fw->dev, fw->buf_out_host, fw->buf_out_mve, ind, s,
+		   fw->buf_attr[MVX_FW_REGION_BUF_OUT_MVE]);
+
+	return 0;
+}
+
+static ssize_t get_capacity(int rpos, int wpos)
+{
+	ssize_t capacity;
+
+	capacity = wpos - rpos;
+	if (capacity < 0)
+		capacity += MVE_COMM_QUEUE_SIZE_IN_WORDS;
+
+	return capacity * sizeof(uint32_t);
+}
+
+static void print_debug_v2(struct mvx_fw *fw)
+{
+	struct mve_comm_area_host *msg_host = fw->msg_host;
+	struct mve_comm_area_mve *msg_mve = fw->msg_mve;
+	unsigned int rpos, wpos;
+	ssize_t capacity;
+	struct mve_msg_header header;
+	struct mve_rpc_communication_area *rpc_area = fw->rpc;
+	union mve_rpc_params *p = &rpc_area->params;
+
+	if (fw->buf_attr[MVX_FW_REGION_MSG_MVE] == MVX_FW_BUF_CACHEABLE) {
+		dma_sync_single_for_cpu(fw->dev, virt_to_phys(msg_mve), MVE_PAGE_SIZE, DMA_FROM_DEVICE);
+	}
+
+	MVX_LOG_PRINT_SESSION(&mvx_log_session_if, MVX_LOG_WARNING, fw->session,
+			      "Dump message queue. msg={host={out_rpos=%u, in_wpos=%u}, mve={out_wpos=%u, in_rpos=%u}}",
+			      msg_host->out_rpos, msg_host->in_wpos, msg_mve->out_wpos, msg_mve->in_rpos);
+
+	MVX_LOG_PRINT_SESSION(&mvx_log_session_if, MVX_LOG_WARNING, fw->session,
+			      "Dump fw rpc. state=%u, call_id=%u, size=%u, data=%u, debug_print=%s", rpc_area->state,
+			      rpc_area->call_id, rpc_area->size, p->data, p->debug_print.string);
+
+	rpos = msg_host->out_rpos;
+	wpos = msg_mve->out_wpos;
+
+	while ((capacity = get_capacity(rpos, wpos)) >= sizeof(header)) {
+		unsigned int pos;
+
+		pos = read32n(msg_mve->out_data, rpos, (uint32_t *)&header, sizeof(header));
+
+		MVX_LOG_PRINT_SESSION(&mvx_log_session_if, MVX_LOG_WARNING, fw->session,
+				      "queue={rpos=%u, wpos=%u, capacity=%u}, msg={code=%u, size=%u}", rpos, wpos,
+				      capacity, header.code, header.size);
+
+		capacity = get_capacity(pos, wpos);
+		if (header.size > capacity) {
+			MVX_LOG_PRINT_SESSION(&mvx_log_session_if, MVX_LOG_WARNING, fw->session,
+					      "Size is larger than capacity. capacity=%zd, size=%u.", capacity,
+					      header.size);
+			return;
+		}
+
+		rpos = (pos + DIV_ROUND_UP(header.size, sizeof(uint32_t))) % MVE_COMM_QUEUE_SIZE_IN_WORDS;
+	}
+}
+
+int mvx_fw_send_idle_ack_v2(struct mvx_fw *fw)
+{
+	int ret = 0;
+
+	ret = write_message(fw, fw->msg_host, fw->msg_mve, MVE_REQUEST_CODE_IDLE_ACK, NULL, 0,
+			    MVX_LOG_FWIF_CHANNEL_MESSAGE, fw->buf_attr[MVX_FW_REGION_MSG_MVE],
+			    fw->buf_attr[MVX_FW_REGION_MSG_HOST]);
+
+	return ret;
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_fw_construct_v2(struct mvx_fw *fw, struct mvx_fw_bin *fw_bin, struct mvx_mmu *mmu, struct mvx_session *session,
+			struct mvx_client_ops *client_ops, struct mvx_client_session *csession, unsigned int ncores,
+			unsigned char major, unsigned char minor)
+{
+	int ret;
+
+	ret = mvx_fw_construct(fw, fw_bin, mmu, session, client_ops, csession, ncores);
+	if (ret != 0)
+		return ret;
+
+	fw->ops.map_protocol = map_protocol_v2;
+	fw->ops.unmap_protocol = unmap_protocol_v2;
+	fw->ops.get_region = get_region_v2;
+	fw->ops.get_message = get_message_v2;
+	fw->ops.put_message = put_message_v2;
+	fw->ops.handle_rpc = handle_rpc_v2;
+#ifdef MVX_FW_DEBUG_ENABLE
+	fw->ops.handle_fw_ram_print = handle_fw_ram_print_v2;
+#endif
+	fw->ops.print_stat = print_stat_v2;
+	fw->ops.print_debug = print_debug_v2;
+	fw->ops_priv.send_idle_ack = NULL;
+	fw->ops_priv.to_mve_profile = mvx_fw_to_mve_profile_v2;
+	fw->ops_priv.to_mve_level = mvx_fw_to_mve_level_v2;
+
+	fw->buf_attr[MVX_FW_REGION_MSG_HOST] = MVX_FW_BUF_CACHEABLE;
+	fw->buf_attr[MVX_FW_REGION_MSG_MVE] = MVX_FW_BUF_UNCACHEABLE;
+	fw->buf_attr[MVX_FW_REGION_BUF_IN_HOST] = MVX_FW_BUF_CACHEABLE;
+	fw->buf_attr[MVX_FW_REGION_BUF_IN_MVE] = MVX_FW_BUF_CACHEABLE;
+	fw->buf_attr[MVX_FW_REGION_BUF_OUT_HOST] = MVX_FW_BUF_CACHEABLE;
+	fw->buf_attr[MVX_FW_REGION_BUF_OUT_MVE] = MVX_FW_BUF_CACHEABLE;
+	fw->buf_attr[MVX_FW_REGION_RPC] = MVX_FW_BUF_CACHEABLE;
+
+	fw->map_op[MVX_FW_BUF_CACHEABLE] = map_msq;
+	fw->map_op[MVX_FW_BUF_UNCACHEABLE] = map_msq_uncache;
+	fw->unmap_op[MVX_FW_BUF_CACHEABLE] = unmap_msq;
+	fw->unmap_op[MVX_FW_BUF_UNCACHEABLE] = unmap_msq_uncache;
+
+	if (major == 2 && minor >= 4)
+		fw->ops_priv.send_idle_ack = mvx_fw_send_idle_ack_v2;
+
+	return 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_v3.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_v3.c
new file mode 100644
index 000000000000..abafe341075d
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_firmware_v3.c
@@ -0,0 +1,161 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_firmware_priv.h"
+#include "fw_v3/mve_protocol_def.h"
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+static int get_region_v3(enum mvx_fw_region region, uint32_t *begin, uint32_t *end)
+{
+	switch (region) {
+	case MVX_FW_REGION_CORE_0:
+		*begin = MVE_MEM_REGION_FW_INSTANCE0_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE0_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_1:
+		*begin = MVE_MEM_REGION_FW_INSTANCE1_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE1_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_2:
+		*begin = MVE_MEM_REGION_FW_INSTANCE2_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE2_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_3:
+		*begin = MVE_MEM_REGION_FW_INSTANCE3_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE3_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_4:
+		*begin = MVE_MEM_REGION_FW_INSTANCE4_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE4_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_5:
+		*begin = MVE_MEM_REGION_FW_INSTANCE5_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE5_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_6:
+		*begin = MVE_MEM_REGION_FW_INSTANCE6_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE6_ADDR_END;
+		break;
+	case MVX_FW_REGION_CORE_7:
+		*begin = MVE_MEM_REGION_FW_INSTANCE7_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FW_INSTANCE7_ADDR_END;
+		break;
+	case MVX_FW_REGION_PROTECTED:
+		*begin = MVE_MEM_REGION_PROTECTED_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_PROTECTED_ADDR_END;
+		break;
+	case MVX_FW_REGION_FRAMEBUF:
+		*begin = MVE_MEM_REGION_FRAMEBUF_ADDR_BEGIN;
+		*end = MVE_MEM_REGION_FRAMEBUF_ADDR_END;
+		break;
+	case MVX_FW_REGION_MSG_HOST:
+		*begin = MVE_COMM_MSG_INQ_ADDR;
+		*end = MVE_COMM_MSG_INQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_MSG_MVE:
+		*begin = MVE_COMM_MSG_OUTQ_ADDR;
+		*end = MVE_COMM_MSG_OUTQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_IN_HOST:
+		*begin = MVE_COMM_BUF_INQ_ADDR;
+		*end = MVE_COMM_BUF_INQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_IN_MVE:
+		*begin = MVE_COMM_BUF_INRQ_ADDR;
+		*end = MVE_COMM_BUF_INRQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_OUT_HOST:
+		*begin = MVE_COMM_BUF_OUTQ_ADDR;
+		*end = MVE_COMM_BUF_OUTQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_BUF_OUT_MVE:
+		*begin = MVE_COMM_BUF_OUTRQ_ADDR;
+		*end = MVE_COMM_BUF_OUTRQ_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_RPC:
+		*begin = MVE_COMM_RPC_ADDR;
+		*end = MVE_COMM_RPC_ADDR + MVE_PAGE_SIZE;
+		break;
+	case MVX_FW_REGION_PRINT_RAM:
+		*begin = MVE_FW_PRINT_RAM_ADDR;
+		*end = MVE_FW_PRINT_RAM_ADDR + MVE_FW_PRINT_RAM_SIZE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int to_mve_profile_v3(unsigned int mvx_profile, uint16_t *mve_profile)
+{
+	int ret = 0;
+
+	switch (mvx_profile) {
+	case MVX_PROFILE_H264_HIGH_10:
+		*mve_profile = MVE_OPT_PROFILE_H264_HIGH_10;
+		break;
+	default:
+		ret = mvx_fw_to_mve_profile_v2(mvx_profile, mve_profile);
+	}
+
+	return ret;
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_fw_construct_v3(struct mvx_fw *fw, struct mvx_fw_bin *fw_bin, struct mvx_mmu *mmu, struct mvx_session *session,
+			struct mvx_client_ops *client_ops, struct mvx_client_session *csession, unsigned int ncores,
+			unsigned char major, unsigned char minor)
+{
+	int ret;
+
+	ret = mvx_fw_construct_v2(fw, fw_bin, mmu, session, client_ops, csession, ncores, major, minor);
+	if (ret != 0)
+		return ret;
+
+	fw->ops.get_region = get_region_v3;
+	fw->ops_priv.to_mve_profile = to_mve_profile_v3;
+
+	if (major == 3 && minor >= 1)
+		fw->ops_priv.send_idle_ack = mvx_fw_send_idle_ack_v2;
+
+	return 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_if.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_if.c
new file mode 100644
index 000000000000..7cfcf4d7c92f
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_if.c
@@ -0,0 +1,231 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/version.h>
+#include <linux/dma-mapping.h>
+#include <linux/export.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include <linux/of_device.h>
+#include "mvx_ext_if.h"
+#include "mvx_if.h"
+#include "mvx_log_group.h"
+#include "mvx_firmware.h"
+#include "mvx_firmware_cache.h"
+#include "mvx_secure.h"
+#include "mvx_session.h"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+/**
+ * struct mvx_if_ctx - Device context.
+ *
+ * There is one instance of this structure for each device.
+ */
+struct mvx_if_ctx {
+	struct device *dev;
+	struct mvx_ext_if ext;
+	struct mvx_fw_cache firmware;
+	struct mvx_client_ops *client_ops;
+	struct mvx_if_ops if_ops;
+	struct mvx_secure secure;
+	struct kobject kobj;
+	struct completion kobj_unregister;
+	struct dentry *dentry;
+};
+
+/****************************************************************************
+ * Static variables and functions
+ ****************************************************************************/
+
+/* Physical hardware can handle 40 physical bits. */
+static uint64_t mvx_if_dma_mask = DMA_BIT_MASK(40);
+
+static struct mvx_if_ctx *if_ops_to_if_ctx(struct mvx_if_ops *ops)
+{
+	return container_of(ops, struct mvx_if_ctx, if_ops);
+}
+
+static void if_release(struct kobject *kobj)
+{
+	struct mvx_if_ctx *ctx = container_of(kobj, struct mvx_if_ctx, kobj);
+
+	complete(&ctx->kobj_unregister);
+}
+
+static struct kobj_type if_ktype = {.release = if_release, .sysfs_ops = &kobj_sysfs_ops};
+
+/****************************************************************************
+ * Exported variables and functions
+ ****************************************************************************/
+
+struct mvx_if_ops *mvx_if_create(struct device *dev, struct mvx_client_ops *client_ops, void *priv)
+{
+	struct mvx_if_ctx *ctx;
+	int ret;
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "probe");
+
+	dev->dma_mask = &mvx_if_dma_mask;
+	dev->coherent_dma_mask = mvx_if_dma_mask;
+
+	/*
+	 * This parameter is indirectly used by DMA-API to limit a lookup
+	 * through a hash table with allocated DMA regions. If the value is
+	 * not high enough, a lookup will be terminated too early and a false
+	 * negative warning will be generated for every DMA operation.
+	 *
+	 * To prevent this behavior vb2-dma-contig allocator keeps this value
+	 * set to the maximum requested buffer size. Unfortunately this is not
+	 * done for vb2-dma-sg which we are using, so we have to implement the
+	 * same logic.
+	 *
+	 * In this change I set a value permanently to 2Gb, but in the next
+	 * commit a functionality similar to vb2-dma-contig will be added.
+	 *
+	 * Mentioned structure also has one more member: segment_boundary_mask.
+	 * It has to be investigated if any value should be assigned to it.
+	 *
+	 * See the following kernel commit for the reference:
+	 * 3f03396918962b2f8b888d02b23cd1e0c88bf5e5
+	 */
+	dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL);
+	if (dev->dma_parms == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	dma_set_max_seg_size(dev, SZ_2G);
+
+#if (KERNEL_VERSION(4, 1, 0) <= LINUX_VERSION_CODE) && IS_ENABLED(CONFIG_OF)
+	of_dma_configure(dev, dev->of_node, true);
+#endif
+
+	/* Create device context and store pointer in device private data. */
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (ctx == NULL) {
+		ret = -ENOMEM;
+		goto free_dma_parms;
+	}
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		char name[20];
+
+		scnprintf(name, sizeof(name), "%s%u", MVX_IF_NAME, dev->id);
+		ctx->dentry = debugfs_create_dir(name, NULL);
+		if (IS_ERR_OR_NULL(ctx->dentry)) {
+			ret = -EINVAL;
+			goto free_ctx;
+		}
+	}
+
+	/* Store context in device private data. */
+	ctx->dev = dev;
+	ctx->client_ops = client_ops;
+
+	/* Initialize if ops. */
+	ctx->if_ops.irq = mvx_session_irq;
+
+	init_completion(&ctx->kobj_unregister);
+
+	/* Create sysfs entry for the device */
+	ret = kobject_init_and_add(&ctx->kobj, &if_ktype, kernel_kobj, "amvx%u", dev->id);
+	if (ret != 0) {
+		kobject_put(&ctx->kobj);
+		goto remove_debugfs;
+	}
+
+	/* Initialize secure video. */
+	ret = mvx_secure_construct(&ctx->secure, dev);
+	if (ret != 0)
+		goto delete_kobject;
+
+	/* Initialize firmware cache. */
+	ret = mvx_fw_cache_construct(&ctx->firmware, dev, &ctx->secure, &ctx->kobj);
+	if (ret != 0)
+		goto destroy_secure;
+
+	/* Create the external device interface. */
+	ret = mvx_ext_if_construct(&ctx->ext, dev, &ctx->firmware, ctx->client_ops, ctx->dentry);
+	if (ret != 0)
+		goto destroy_fw_cache;
+
+	return &ctx->if_ops;
+
+destroy_fw_cache:
+	mvx_fw_cache_destruct(&ctx->firmware);
+
+destroy_secure:
+	mvx_secure_destruct(&ctx->secure);
+
+delete_kobject:
+	kobject_put(&ctx->kobj);
+
+remove_debugfs:
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(ctx->dentry);
+
+free_ctx:
+	devm_kfree(dev, ctx);
+
+free_dma_parms:
+	devm_kfree(dev, dev->dma_parms);
+
+	return ERR_PTR(ret);
+}
+
+void mvx_if_destroy(struct mvx_if_ops *if_ops)
+{
+	struct mvx_if_ctx *ctx = if_ops_to_if_ctx(if_ops);
+	struct device *dev = ctx->dev;
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "remove");
+
+	mvx_ext_if_destruct(&ctx->ext);
+	mvx_fw_cache_destruct(&ctx->firmware);
+	mvx_secure_destruct(&ctx->secure);
+	kobject_put(&ctx->kobj);
+	wait_for_completion(&ctx->kobj_unregister);
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(ctx->dentry);
+
+	devm_kfree(dev, dev->dma_parms);
+	devm_kfree(dev, ctx);
+
+	dev->dma_mask = NULL;
+	dev->coherent_dma_mask = 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_if.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_if.h
new file mode 100644
index 000000000000..880b5d04e181
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_if.h
@@ -0,0 +1,460 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_IF_H_
+#define _MVX_IF_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/kref.h>
+#include <linux/list.h>
+#include "mvx_mmu.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+/**
+ * The name of the device driver.
+ */
+#define MVX_IF_NAME "amvx_if"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct mvx_client_ops;
+struct mvx_client_session;
+struct mvx_if_ctx;
+struct platform_device;
+
+/**
+ * enum mvx_direction - Direction from the point of view of the hardware block.
+ */
+enum mvx_direction {
+	MVX_DIR_INPUT,
+	MVX_DIR_OUTPUT,
+	MVX_DIR_MAX
+};
+
+/**
+ * enum mvx_tristate - Tristate boolean variable.
+ */
+enum mvx_tristate {
+	MVX_TRI_UNSET = -1,
+	MVX_TRI_TRUE  = 0,
+	MVX_TRI_FALSE = 1
+};
+
+/**
+ * enum mvx_format - List of compressed formats and frame formats.
+ *
+ * Enumeration of formats that are supported by all know hardware revisions.
+ *
+ * The enumeration should start at 0 and should not contain any gaps.
+ */
+enum mvx_format {
+	/* Compressed formats. */
+	MVX_FORMAT_BITSTREAM_FIRST,
+	MVX_FORMAT_AVS = MVX_FORMAT_BITSTREAM_FIRST,
+	MVX_FORMAT_AVS2,
+	MVX_FORMAT_H263,
+	MVX_FORMAT_H264,
+	MVX_FORMAT_HEVC,
+	MVX_FORMAT_JPEG,
+	MVX_FORMAT_MPEG2,
+	MVX_FORMAT_MPEG4,
+	MVX_FORMAT_RV,
+	MVX_FORMAT_VC1,
+	MVX_FORMAT_VP8,
+	MVX_FORMAT_VP9,
+	MVX_FORMAT_BITSTREAM_LAST = MVX_FORMAT_VP9,
+
+	/* Uncompressed formats. */
+	MVX_FORMAT_FRAME_FIRST,
+	MVX_FORMAT_YUV420_AFBC_8 = MVX_FORMAT_FRAME_FIRST,
+	MVX_FORMAT_YUV420_AFBC_10,
+	MVX_FORMAT_YUV422_AFBC_8,
+	MVX_FORMAT_YUV422_AFBC_10,
+	MVX_FORMAT_YUV420_I420,
+	MVX_FORMAT_YUV420_NV12,
+	MVX_FORMAT_YUV420_NV21,
+	MVX_FORMAT_YUV420_P010,
+	MVX_FORMAT_YUV420_Y0L2,
+	MVX_FORMAT_YUV420_AQB1,
+	MVX_FORMAT_YUV422_YUY2,
+	MVX_FORMAT_YUV422_UYVY,
+	MVX_FORMAT_YUV422_Y210,
+	MVX_FORMAT_RGBA_8888,
+	MVX_FORMAT_BGRA_8888,
+	MVX_FORMAT_ARGB_8888,
+	MVX_FORMAT_ABGR_8888,
+	MVX_FORMAT_FRAME_LAST = MVX_FORMAT_ABGR_8888,
+
+	MVX_FORMAT_MAX
+};
+
+/**
+ * enum mvx_hw_id - Enumeration of known hardware revisions.
+ */
+enum mvx_hw_id {
+	MVE_Unknown = 0x0,
+	MVE_v500    = 0x500,
+	MVE_v550    = 0x550,
+	MVE_v61     = 0x61,
+	MVE_v52_v76 = 0x5276
+};
+
+/**
+ * struct mvx_hw_ver - Hardware version.
+ */
+struct mvx_hw_ver {
+	enum mvx_hw_id id;
+	uint32_t revision;
+	uint32_t patch;
+};
+
+/**
+ * enum mvx_nalu_format - NALU format.
+ */
+enum mvx_nalu_format {
+	MVX_NALU_FORMAT_UNDEFINED,
+	MVX_NALU_FORMAT_START_CODES,
+	MVX_NALU_FORMAT_ONE_NALU_PER_BUFFER,
+	MVX_NALU_FORMAT_ONE_BYTE_LENGTH_FIELD,
+	MVX_NALU_FORMAT_TWO_BYTE_LENGTH_FIELD,
+	MVX_NALU_FORMAT_FOUR_BYTE_LENGTH_FIELD
+};
+
+/**
+ * enum mvx_profile - Profile for encoder.
+ */
+enum mvx_profile {
+	MVX_PROFILE_NONE,
+
+	MVX_PROFILE_H264_BASELINE,
+	MVX_PROFILE_H264_MAIN,
+	MVX_PROFILE_H264_HIGH,
+	MVX_PROFILE_H264_HIGH_10,
+
+	MVX_PROFILE_H265_MAIN,
+	MVX_PROFILE_H265_MAIN_STILL,
+	MVX_PROFILE_H265_MAIN_INTRA,
+
+	MVX_PROFILE_VC1_SIMPLE,
+	MVX_PROFILE_VC1_MAIN,
+	MVX_PROFILE_VC1_ADVANCED,
+
+	MVX_PROFILE_VP8_MAIN
+};
+
+/**
+ * enum mvx_level - Level for encoder.
+ */
+enum mvx_level {
+	MVX_LEVEL_NONE,
+
+	MVX_LEVEL_H264_1,
+	MVX_LEVEL_H264_1b,
+	MVX_LEVEL_H264_11,
+	MVX_LEVEL_H264_12,
+	MVX_LEVEL_H264_13,
+	MVX_LEVEL_H264_2,
+	MVX_LEVEL_H264_21,
+	MVX_LEVEL_H264_22,
+	MVX_LEVEL_H264_3,
+	MVX_LEVEL_H264_31,
+	MVX_LEVEL_H264_32,
+	MVX_LEVEL_H264_4,
+	MVX_LEVEL_H264_41,
+	MVX_LEVEL_H264_42,
+	MVX_LEVEL_H264_5,
+	MVX_LEVEL_H264_51,
+	MVX_LEVEL_H264_52,
+	MVX_LEVEL_H264_6,
+	MVX_LEVEL_H264_61,
+	MVX_LEVEL_H264_62,
+
+	MVX_LEVEL_H265_MAIN_1,
+	MVX_LEVEL_H265_HIGH_1,
+	MVX_LEVEL_H265_MAIN_2,
+	MVX_LEVEL_H265_HIGH_2,
+	MVX_LEVEL_H265_MAIN_21,
+	MVX_LEVEL_H265_HIGH_21,
+	MVX_LEVEL_H265_MAIN_3,
+	MVX_LEVEL_H265_HIGH_3,
+	MVX_LEVEL_H265_MAIN_31,
+	MVX_LEVEL_H265_HIGH_31,
+	MVX_LEVEL_H265_MAIN_4,
+	MVX_LEVEL_H265_HIGH_4,
+	MVX_LEVEL_H265_MAIN_41,
+	MVX_LEVEL_H265_HIGH_41,
+	MVX_LEVEL_H265_MAIN_5,
+	MVX_LEVEL_H265_HIGH_5,
+	MVX_LEVEL_H265_MAIN_51,
+	MVX_LEVEL_H265_HIGH_51,
+	MVX_LEVEL_H265_MAIN_52,
+	MVX_LEVEL_H265_HIGH_52,
+	MVX_LEVEL_H265_MAIN_6,
+	MVX_LEVEL_H265_HIGH_6,
+	MVX_LEVEL_H265_MAIN_61,
+	MVX_LEVEL_H265_HIGH_61,
+	MVX_LEVEL_H265_MAIN_62,
+	MVX_LEVEL_H265_HIGH_62
+};
+
+/**
+ * enum mvx_gop_type - GOP type for encoder.
+ */
+enum mvx_gop_type {
+	MVX_GOP_TYPE_NONE,
+	MVX_GOP_TYPE_BIDIRECTIONAL,
+	MVX_GOP_TYPE_LOW_DELAY,
+	MVX_GOP_TYPE_PYRAMID
+};
+
+/**
+ * enum mvx_entropy_mode - Entropy mode for encoder.
+ */
+enum mvx_entropy_mode {
+	MVX_ENTROPY_MODE_NONE,
+	MVX_ENTROPY_MODE_CAVLC,
+	MVX_ENTROPY_MODE_CABAC
+};
+
+/**
+ * enum mvx_multi_slice_mode - Multi slice mode.
+ */
+enum mvx_multi_slice_mode {
+	MVX_MULTI_SLICE_MODE_SINGLE,
+	MVX_MULTI_SLICE_MODE_MAX_MB
+};
+
+/**
+ * enum mvx_vp9_prob_update - Probability update method.
+ */
+enum mvx_vp9_prob_update {
+	MVX_VP9_PROB_UPDATE_DISABLED,
+	MVX_VP9_PROB_UPDATE_IMPLICIT,
+	MVX_VP9_PROB_UPDATE_EXPLICIT
+};
+
+/**
+ * enum mvx_rgb_to_yuv_mode - RGB to YUV conversion mode.
+ */
+enum mvx_rgb_to_yuv_mode {
+	MVX_RGB_TO_YUV_MODE_BT601_STUDIO,
+	MVX_RGB_TO_YUV_MODE_BT601_FULL,
+	MVX_RGB_TO_YUV_MODE_BT709_STUDIO,
+	MVX_RGB_TO_YUV_MODE_BT709_FULL
+};
+
+/**
+ * struct mvx_if_session - Structure holding members needed to map a session to
+ *                         a hardare device.
+ * @kref:	Reference counter for the session object.
+ * @release:	Function pointer that shall be passed to kref_put. If the
+ *              reference count reaches 0 this function will be called to
+ *              destruct and deallocate the object.
+ * @ncores:	Number of cores this session has been mapped to.
+ * @l0_pte:	Level 0 page table entry. This value is written to the hardware
+ *              MMU CTRL register to point out the location of the L1 page table
+ *              and to set access permissions and bus attributes.
+ * @securevideo:Secure video enabled.
+ */
+struct mvx_if_session {
+	struct kref kref;
+	struct mutex *mutex;
+	void (*release)(struct kref *kref);
+	unsigned int ncores;
+	mvx_mmu_pte l0_pte;
+	bool securevideo;
+};
+
+/**
+ * struct mvx_if_ops - Functions pointers the registered device may use to call
+ *                     the if device.
+ */
+struct mvx_if_ops {
+	/**
+	 * irq() - Handle IRQ sent from firmware to driver.
+	 */
+	void (*irq)(struct mvx_if_session *session);
+};
+
+/**
+ * struct mvx_client_ops - Functions pointers the if device may use to call
+ *                         the registered device.
+ */
+struct mvx_client_ops {
+	struct list_head list;
+
+	/**
+	 * get_hw_ver() - Get MVE hardware version
+	 */
+	void (*get_hw_ver)(struct mvx_client_ops *client, struct mvx_hw_ver *version);
+
+	/**
+	 * get_formats() - Get list of supported formats.
+	 *
+	 * Return: 0 on success, else error code.
+	 */
+	void (*get_formats)(struct mvx_client_ops *client, enum mvx_direction direction, uint64_t *formats);
+
+	/**
+	 * get_ncores() - Get number of cores.
+	 *
+	 * Return: Number of cores on success, else error code.
+	 */
+	unsigned int (*get_ncores)(struct mvx_client_ops *client);
+
+	/*
+	 * SESSION.
+	 */
+
+	/**
+	 * register_session() - Register if session with client.
+	 *
+	 * Return: Client session handle on success, else ERR_PTR.
+	 */
+	struct mvx_client_session *(*register_session)(struct mvx_client_ops *client, struct mvx_if_session *session);
+
+	/**
+	 * unregister_session() - Unregister session.
+	 *
+	 * Return: 0 on success, else error code.
+	 */
+	void (*unregister_session)(struct mvx_client_session *session);
+
+	/**
+	 * switch_in() - Switch in session.
+	 *
+	 * After a session has been switched in it must wait for a 'switched
+	 * out' event before it is allowed to requested to be switched in again.
+	 * Switching in a already switched in session is regarded as an error.
+	 *
+	 * Return: 0 on success, else error code.
+	 */
+	int (*switch_in)(struct mvx_client_session *session);
+
+	/**
+	 * send_irq() - Send IRQ from driver to firmware.
+	 *
+	 * Return: 0 on success, else error code.
+	 */
+	int (*send_irq)(struct mvx_client_session *session);
+
+	/**
+	 * flush_mmu() - Flush MMU tables.
+	 *
+	 * Flushing MMU tables is only required if pages have been removed
+	 * from the page tables.
+	 *
+	 * Return: 0 on success, else error code.
+	 */
+	int (*flush_mmu)(struct mvx_client_session *session);
+
+	/**
+	 * print_debug() - Print debug information.
+	 *
+	 * Return: 0 on success, else error code.
+	 */
+	void (*print_debug)(struct mvx_client_session *session);
+
+	void (*wait_session_idle)(struct mvx_client_session *session);
+};
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+/**
+ * mvx_is_bitstream(): Detect if format is of type bitstream.
+ * @format:	Format.
+ *
+ * Return: True if format is bitstream, else false.
+ */
+static inline bool mvx_is_bitstream(enum mvx_format format)
+{
+	return (format >= MVX_FORMAT_BITSTREAM_FIRST) && (format <= MVX_FORMAT_BITSTREAM_LAST);
+}
+
+/**
+ * mvx_is_frame(): Detect if format is of type frame.
+ * @format:	Format.
+ *
+ * Return: True if format is frame, else false.
+ */
+static inline bool mvx_is_frame(enum mvx_format format)
+{
+	return (format >= MVX_FORMAT_FRAME_FIRST) && (format <= MVX_FORMAT_FRAME_LAST);
+}
+
+/**
+ * mvx_is_rgb(): Detect if format is of type RGB.
+ * @format:	Format.
+ *
+ * Return: True if format is RGB, else false.
+ */
+static inline bool mvx_is_rgb(enum mvx_format format)
+{
+	return (format >= MVX_FORMAT_RGBA_8888) && (format <= MVX_FORMAT_ABGR_8888);
+}
+
+/**
+ * mvx_is_afbc(): Detect if format is of type AFBC.
+ * @format:	Format.
+ *
+ * Return: True if format is AFBC, else false.
+ */
+static inline bool mvx_is_afbc(enum mvx_format format)
+{
+	return (format >= MVX_FORMAT_YUV420_AFBC_8) && (format <= MVX_FORMAT_YUV422_AFBC_10);
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_if_create() - Create IF device.
+ */
+struct mvx_if_ops *mvx_if_create(struct device *dev, struct mvx_client_ops *client_ops, void *priv);
+
+/**
+ * mvx_if_destroy() - Destroy IF device.
+ */
+void mvx_if_destroy(struct mvx_if_ops *if_ops);
+
+#endif /* _MVX_IF_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_mmu.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_mmu.c
new file mode 100644
index 000000000000..29021c6627ad
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_mmu.c
@@ -0,0 +1,1221 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <asm-generic/memory_model.h>
+#include <linux/bitmap.h>
+#include <linux/debugfs.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/gfp.h>
+#include <linux/list.h>
+#include <linux/sched.h>
+#include <linux/vmalloc.h>
+#include "mvx_mmu.h"
+#include "mvx_log_group.h"
+
+MODULE_IMPORT_NS("DMA_BUF");
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+/* Number of bits for the physical address space. */
+#define MVE_PA_BITS 40
+#define MVE_PA_MASK GENMASK_ULL(MVE_PA_BITS - 1, 0)
+
+/* Number of bits for the virtual address space. */
+#define MVE_VA_BITS 32
+#define MVE_VA_MASK GENMASK(MVE_VA_BITS - 1, 0)
+
+/* Number of bits from the VA used to index a PTE in a page. */
+#define MVE_INDEX_SHIFT 10
+#define MVE_INDEX_SIZE (1 << MVE_INDEX_SHIFT)
+#define MVE_INDEX_MASK GENMASK(MVE_INDEX_SHIFT - 1, 0)
+
+/* Access permission defines. */
+#define MVE_PTE_AP_SHIFT 0
+#define MVE_PTE_AP_BITS 2
+#define MVE_PTE_AP_MASK ((1 << MVE_PTE_AP_BITS) - 1)
+
+/* Physical address defines. */
+#define MVE_PTE_PHYSADDR_SHIFT 2
+#define MVE_PTE_PHYSADDR_BITS 28
+#define MVE_PTE_PHYSADDR_MASK ((1 << MVE_PTE_PHYSADDR_BITS) - 1)
+
+/* Attributes defines. */
+#define MVE_PTE_ATTR_SHIFT 30
+#define MVE_PTE_ATTR_BITS 2
+#define MVE_PTE_ATTR_MASK ((1 << MVE_PTE_ATTR_BITS) - 1)
+
+/* Number of levels for Page Table Walk. */
+#define MVE_PTW_LEVELS 2
+
+/*
+ * A Linux physical page can be equal in size or larger than the MVE page size.
+ * This define calculates how many MVE pages that fit in one Linux page.
+ */
+#define MVX_PAGES_PER_PAGE (PAGE_SIZE / MVE_PAGE_SIZE)
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+/**
+ * struct mvx_mmu_dma_buf - MVX DMA buffer.
+ *
+ * Adds a list head to keep track of DMA buffers.
+ */
+struct mvx_mmu_dma_buf {
+	struct list_head head;
+	struct dma_buf *dmabuf;
+};
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+/**
+ * get_index() - Return the PTE index for a given level.
+ * @va:		Virtual address.
+ * @level:	Level (L1=0, L2=1).
+ *
+ *                   22                  12                       0
+ * +-------------------+-------------------+-----------------------+
+ * |      Level 1      |      Level 2      |      Page offset      |
+ * +-------------------+-------------------+-----------------------+
+ */
+static unsigned int get_index(const mvx_mmu_va va, const unsigned int level)
+{
+	return (va >> (MVE_PAGE_SHIFT + (MVE_PTW_LEVELS - level - 1) * MVE_INDEX_SHIFT)) & MVE_INDEX_MASK;
+}
+
+/**
+ * get_offset() - Return the page offset.
+ * @va:		Virtual address.
+ *
+ *                   22                  12                       0
+ * +-------------------+-------------------+-----------------------+
+ * |      Level 1      |      Level 2      |      Page offset      |
+ * +-------------------+-------------------+-----------------------+
+ */
+static unsigned int get_offset(const mvx_mmu_va va) { return va & MVE_PAGE_MASK; }
+
+/**
+ * get_pa() - Return physical address stored in PTE.
+ */
+static phys_addr_t get_pa(const mvx_mmu_pte pte)
+{
+	return (((phys_addr_t)pte >> MVE_PTE_PHYSADDR_SHIFT) & MVE_PTE_PHYSADDR_MASK) << MVE_PAGE_SHIFT;
+}
+
+/* LCOV_EXCL_START */
+
+/**
+ * get_attr() - Return attributes stored in PTE.
+ */
+static enum mvx_mmu_attr get_attr(const mvx_mmu_pte pte) { return (pte >> MVE_PTE_ATTR_SHIFT) & MVE_PTE_ATTR_MASK; }
+
+/**
+ * get_ap() - Return access permissions stored in PTE.
+ */
+static enum mvx_mmu_access get_ap(const mvx_mmu_pte pte) { return (pte >> MVE_PTE_AP_SHIFT) & MVE_PTE_AP_MASK; }
+
+/* LCOV_EXCL_STOP */
+
+/**
+ * ptw() - Perform Page Table Walk and return pointer to L2 PTE.
+ * @mmu:	Pointer to MMU context.
+ * @va:		Virtual address.
+ * @alloc:	True if missing L2 page should be allocated.
+ *
+ * Return: Pointer to PTE, ERR_PTR on error.
+ */
+static mvx_mmu_pte *ptw(struct mvx_mmu *mmu, mvx_mmu_va va, bool alloc)
+{
+	phys_addr_t l2;
+	mvx_mmu_pte *pte = mmu->page_table;
+	unsigned int index;
+
+	/* Level 1. */
+	index = get_index(va, 0);
+	l2 = get_pa(pte[index]);
+
+	/* We should never perform a page table walk for a protected page. */
+	if (test_bit(index, mmu->l2_page_is_external) != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "PTW virtual address to secure L2 page. va=0x%x.", va);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* Map in L2 page if it is missing. */
+	if (l2 == 0) {
+		if (alloc == false) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Missing L2 page in PTW. va=0x%x.", va);
+			return ERR_PTR(-EFAULT);
+		}
+
+		l2 = mvx_mmu_alloc_page(mmu->dev);
+		if (l2 == 0) {
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to allocate L2 page. va=0x%x.", va);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		pte[index] = mvx_mmu_set_pte(MVX_ATTR_PRIVATE, l2, MVX_ACCESS_READ_ONLY);
+		dma_sync_single_for_device(mmu->dev, virt_to_phys(&pte[index]), sizeof(pte[index]), DMA_TO_DEVICE);
+	}
+
+	/* Level 2. */
+	index = get_index(va, 1);
+	pte = phys_to_virt(l2);
+
+	return &pte[index];
+}
+
+/**
+ * map_page() - Map physical- to virtual address.
+ * @mmu:	Pointer to MMU context.
+ * @va:		MVE virtual address to map.
+ * @pa:		Linux kernel physical address to map.
+ * @attr:	MMU attributes.
+ * @access:	MMU access permissions.
+ *
+ * Create new L1 and L2 entries if necessary. If mapping already exist, then
+ * error is returned.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int map_page(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t pa, enum mvx_mmu_attr attr,
+		    enum mvx_mmu_access access)
+{
+	mvx_mmu_pte *pte;
+	phys_addr_t page;
+
+	/* Check that both VA and PA are page aligned. */
+	if ((va | pa) & MVE_PAGE_MASK) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "VA and PA must be page aligned. va=0x%x, pa=0x%llx.", va,
+			      pa);
+		return -EFAULT;
+	}
+
+	/* Check that VA is within valid address range. */
+	if (va & ~MVE_VA_MASK) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "VA out of valid range. va=0x%x.", va);
+		return -EFAULT;
+	}
+
+	/* Check that PA is within valid address range. */
+	if (pa & ~MVE_PA_MASK) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "PA out of valid range. pa=0x%llx.", pa);
+		return -EFAULT;
+	}
+
+	pte = ptw(mmu, va, true);
+	if (IS_ERR(pte))
+		return PTR_ERR(pte);
+
+	/* Return error if page already exists. */
+	page = get_pa(*pte);
+	if (page != 0)
+		return -EAGAIN;
+
+	/* Map in physical address and flush data. */
+	*pte = mvx_mmu_set_pte(attr, pa, access);
+	dma_sync_single_for_device(mmu->dev, virt_to_phys(pte), sizeof(*pte), DMA_TO_DEVICE);
+
+	return 0;
+}
+
+/**
+ * unmap_page() - Unmap a page from the virtual address space.
+ * @mmu:	Pointer to MMU context.
+ * @va:		Virtual address.
+ */
+static void unmap_page(struct mvx_mmu *mmu, mvx_mmu_va va)
+{
+	mvx_mmu_pte *pte;
+
+	pte = ptw(mmu, va, false);
+	if (IS_ERR(pte))
+		return;
+
+	/* Unmap virtual address and flush data. */
+	*pte = 0;
+	dma_sync_single_for_device(mmu->dev, virt_to_phys(pte), sizeof(*pte), DMA_TO_DEVICE);
+}
+
+/**
+ * remap_page() - Remap virtual address.
+ * @mmu:	Pointer to MMU context.
+ * @va:		MVE virtual address to map.
+ * @pa:		Linux kernel physical address to map.
+ * @attr:	MMU attributes.
+ * @access:	MMU access permissions.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int remap_page(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t pa, enum mvx_mmu_attr attr,
+		      enum mvx_mmu_access access)
+{
+	unmap_page(mmu, va);
+	return map_page(mmu, va, pa, attr, access);
+}
+
+/**
+ * remap_pages() - Remap virtual address range.
+ * @pages:	Pointer to pages object.
+ * @oldcount:	Count before object was resized.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int remap_pages(struct mvx_mmu_pages *pages, size_t oldcount)
+{
+	int ret;
+
+	if (pages->mmu == NULL)
+		return 0;
+
+	/* Remap pages to no access if new count is smaller than old count. */
+	while (pages->count < oldcount) {
+		oldcount--;
+
+		ret = remap_page(pages->mmu, pages->va + oldcount * MVE_PAGE_SIZE, MVE_PAGE_SIZE, MVX_ATTR_PRIVATE,
+				 MVX_ACCESS_NO);
+		if (ret != 0)
+			return ret;
+	}
+
+	/* Map up pages if new count is larger than old count. */
+	while (pages->count > oldcount) {
+		ret = remap_page(pages->mmu, pages->va + oldcount * MVE_PAGE_SIZE, pages->pages[oldcount], pages->attr,
+				 pages->access);
+		if (ret != 0)
+			return ret;
+
+		oldcount++;
+	}
+
+	return 0;
+}
+
+/**
+ * mapped_count() - Check if level 2 table entries point to mmu mapped pages.
+ * @pa:		Physical address of the table entry to be checked.
+ *
+ * Return: the number of mapped pages found.
+ */
+static int mapped_count(phys_addr_t pa)
+{
+	int count = 0;
+
+	if (pa != 0) {
+		int j;
+		phys_addr_t pa2;
+		mvx_mmu_pte *l2 = phys_to_virt(pa);
+
+		for (j = 0; j < MVE_INDEX_SIZE; j++) {
+			pa2 = get_pa(l2[j]);
+			if (pa2 != 0 && pa2 != MVE_PAGE_SIZE)
+				count++;
+		}
+	}
+
+	return count;
+}
+
+/**
+ * get_sg_table_npages() - Count number of pages in SG table.
+ * @sgt:	Pointer to scatter gather table.
+ *
+ * Return: Number of pages.
+ */
+static size_t get_sg_table_npages(struct sg_table *sgt)
+{
+	struct sg_page_iter piter;
+	size_t count = 0;
+
+	for_each_sg_page(sgt->sgl, &piter, sgt->nents, 0) { count++; }
+
+	return count;
+}
+
+/**
+ * append_sg_table() - Append SG table to pages object.
+ * @pages:	Pointer to pages object.
+ * @sgt:	Pointer to scatter gather table.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int append_sg_table(struct mvx_mmu_pages *pages, struct sg_table *sgt)
+{
+	size_t count;
+	struct sg_dma_page_iter piter;
+
+	count = get_sg_table_npages(sgt) * MVX_PAGES_PER_PAGE;
+
+	if ((pages->count + count) > pages->capacity) {
+		MVX_LOG_PRINT(
+			&mvx_log_if, MVX_LOG_WARNING,
+			"Failed to append SG table. Pages capacity too small. count=%zu, capacity=%zu, append=%zu.",
+			pages->count, pages->capacity, count);
+		return -ENOMEM;
+	}
+
+	for_each_sg_dma_page(sgt->sgl, &piter, sgt->nents, 0)
+	{
+		int j;
+		phys_addr_t base;
+
+		base = (phys_addr_t)sg_page_iter_dma_address(&piter) & PAGE_MASK;
+
+		for (j = 0; j < MVX_PAGES_PER_PAGE; ++j)
+			pages->pages[pages->count++] = base + j * MVE_PAGE_SIZE;
+	}
+
+	return 0;
+}
+
+/**
+ * stat_show() - Print debugfs info into seq-file.
+ *
+ * This is a callback used by debugfs subsystem.
+ *
+ * @s:		Seq-file
+ * @v:		Unused
+ * return: 0 on success, else error code.
+ */
+static int stat_show(struct seq_file *s, void *v)
+{
+	struct mvx_mmu_pages *pages = s->private;
+
+	seq_printf(s, "va: %08x\n", pages->va);
+	seq_printf(s, "capacity: %zu\n", pages->capacity);
+	seq_printf(s, "count: %zu\n", pages->count);
+
+	if (pages->mmu != NULL) {
+		seq_printf(s, "attr: %d\n", pages->attr);
+		seq_printf(s, "access: %d\n", pages->access);
+	}
+
+	return 0;
+}
+
+/**
+ * stat_open() - Open debugfs file.
+ *
+ * This is a callback used by debugfs subsystem.
+ *
+ * @inode:	Inode
+ * @file:	File
+ * return: 0 on success, else error code.
+ */
+static int stat_open(struct inode *inode, struct file *file) { return single_open(file, stat_show, inode->i_private); }
+
+/**
+ * File operations for debugfs entry.
+ */
+static const struct file_operations stat_fops = {
+	.open    = stat_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = single_release
+};
+
+/**
+ * pages_seq_start() - Iterator over pages list.
+ */
+static void *pages_seq_start(struct seq_file *s, loff_t *pos)
+{
+	struct mvx_mmu_pages *pages = s->private;
+
+	if (*pos >= pages->count)
+		return NULL;
+
+	seq_puts(s, "#Page: [  va_start -     va_end] -> [          pa_start -             pa_end]\n");
+	return pos;
+}
+
+/**
+ * pages_seq_start() - Iterator over pages list.
+ */
+static void *pages_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	struct mvx_mmu_pages *pages = s->private;
+
+	++*pos;
+	if (*pos >= pages->count)
+		return NULL;
+
+	return pos;
+}
+
+/**
+ * pages_seq_start() - Iterator over pages list.
+ */
+static void pages_seq_stop(struct seq_file *s, void *v) {}
+
+/**
+ * pages_seq_start() - Iterator over pages list.
+ */
+static int pages_seq_show(struct seq_file *s, void *v)
+{
+	struct mvx_mmu_pages *pages = s->private;
+	loff_t pos = *(loff_t *)v;
+
+	mvx_mmu_va va_start = pages->va + pos * MVE_PAGE_SIZE;
+	mvx_mmu_va va_end = va_start + MVE_PAGE_SIZE - 1;
+	phys_addr_t pa_start = pages->pages[pos];
+	phys_addr_t pa_end = pa_start + MVE_PAGE_SIZE - 1;
+
+	seq_printf(s, "%5llu: [0x%08x - 0x%08x] -> [%pap - %pap]\n", pos, va_start, va_end, &pa_start, &pa_end);
+	return 0;
+}
+
+/**
+ * mpages_seq_ops - Callbacks used by an iterator over pages list.
+ */
+static const struct seq_operations pages_seq_ops = {
+	.start = pages_seq_start,
+	.next  = pages_seq_next,
+	.stop  = pages_seq_stop,
+	.show  = pages_seq_show
+};
+
+/**
+ * list_open() - Callback for debugfs entry.
+ */
+static int list_open(struct inode *inode, struct file *file)
+{
+	int ret;
+	struct seq_file *s;
+
+	ret = seq_open(file, &pages_seq_ops);
+	if (ret != 0)
+		return ret;
+
+	s = (struct seq_file *)file->private_data;
+	s->private = inode->i_private;
+
+	return 0;
+}
+
+/**
+ * File operations for a debugfs entry.
+ */
+static const struct file_operations list_fops = {
+	.open    = list_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = seq_release
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_mmu_construct(struct mvx_mmu *mmu, struct device *dev)
+{
+	phys_addr_t page_table;
+
+	mmu->dev = dev;
+
+	/* Allocate Page Table Base (the L1 table). */
+	page_table = mvx_mmu_alloc_page(dev);
+	if (page_table == 0)
+		return -ENOMEM;
+
+	mmu->page_table = phys_to_virt(page_table);
+
+	return 0;
+}
+
+void mvx_mmu_destruct(struct mvx_mmu *mmu)
+{
+	mvx_mmu_pte *pte = mmu->page_table;
+	phys_addr_t pa;
+	int i;
+	int count = 0;
+
+	for (i = 0; i < MVE_INDEX_SIZE; i++) {
+		pa = get_pa(pte[i]);
+
+		/* Only free pages we have allocated ourselves. */
+		if (test_bit(i, mmu->l2_page_is_external) == 0) {
+			count += mapped_count(pa);
+			mvx_mmu_free_page(mmu->dev, pa);
+		}
+	}
+
+	pa = virt_to_phys(mmu->page_table);
+	mvx_mmu_free_page(mmu->dev, pa);
+
+	WARN_ON(count > 0);
+}
+
+phys_addr_t mvx_mmu_alloc_page(struct device *dev)
+{
+	struct page *page;
+	phys_addr_t pa;
+	dma_addr_t dma_handle;
+	int retry = 20;
+
+	do {
+		page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL);
+	} while (retry-- && page == NULL);
+
+	if (page == NULL)
+		return 0;
+
+	dma_handle = dma_map_page(dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dev, dma_handle) != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Cannot map page to DMA address space. page=%p.", page);
+		goto free_page;
+	}
+
+	pa = (phys_addr_t)dma_handle;
+
+	dma_sync_single_for_device(dev, pa, PAGE_SIZE, DMA_TO_DEVICE);
+
+	return pa;
+
+free_page:
+	__free_page(page);
+	return 0;
+}
+
+phys_addr_t mvx_mmu_dma_alloc_coherent(struct device *dev, void **data)
+{
+	phys_addr_t pa;
+	dma_addr_t dma_handle;
+	int retry = 20;
+	void *virt_addr;
+
+	do {
+		virt_addr =
+			dma_alloc_coherent(dev, PAGE_SIZE, &dma_handle, GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL);
+	} while (retry-- && virt_addr == NULL);
+
+	if (virt_addr == NULL) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "dma alloc coherent buffer faild. retry=%d", retry);
+		return 0;
+	}
+
+	pa = (phys_addr_t)dma_handle;
+
+	*data = virt_addr;
+
+	return pa;
+}
+
+void mvx_mmu_dma_free_coherent(struct device *dev, phys_addr_t pa, void *data)
+{
+	if (pa == 0)
+		return;
+
+	dma_free_coherent(dev, PAGE_SIZE, data, (dma_addr_t)pa);
+}
+
+void mvx_mmu_free_contiguous_pages(struct device *dev, phys_addr_t pa, size_t npages)
+{
+	struct page *page;
+
+	if (pa == 0)
+		return;
+
+	page = phys_to_page(pa);
+
+	dma_unmap_page(dev, pa, npages << PAGE_SHIFT, DMA_BIDIRECTIONAL);
+	__free_pages(page, get_order(npages << PAGE_SHIFT));
+}
+
+phys_addr_t mvx_mmu_alloc_contiguous_pages(struct device *dev, size_t npages)
+{
+	struct page *page;
+	phys_addr_t pa;
+	dma_addr_t dma_handle;
+	size_t size = (npages << PAGE_SHIFT);
+	int retry = 20;
+
+	do {
+		page = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL, get_order(size));
+	} while (retry-- && page == NULL);
+
+	if (page == NULL)
+		return 0;
+
+	dma_handle = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dev, dma_handle) != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Cannot map page to DMA address space. page=%p.", page);
+		goto free_pages;
+	}
+
+	pa = (phys_addr_t)dma_handle;
+
+	dma_sync_single_for_device(dev, pa, size, DMA_TO_DEVICE);
+
+	return pa;
+
+free_pages:
+	__free_pages(page, get_order(size));
+	return 0;
+}
+
+void mvx_mmu_free_page(struct device *dev, phys_addr_t pa)
+{
+	struct page *page;
+
+	if (pa == 0)
+		return;
+
+	page = phys_to_page(pa);
+
+	dma_unmap_page(dev, pa, PAGE_SIZE, DMA_BIDIRECTIONAL);
+	__free_page(page);
+}
+
+struct mvx_mmu_pages *mvx_mmu_alloc_pages(struct device *dev, size_t count, size_t capacity)
+{
+	struct mvx_mmu_pages *pages;
+	int ret;
+
+	count = roundup(count, MVX_PAGES_PER_PAGE);
+	capacity = roundup(capacity, MVX_PAGES_PER_PAGE);
+	capacity = max(count, capacity);
+
+	pages = vmalloc(sizeof(*pages) + sizeof(phys_addr_t) * capacity);
+	if (pages == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	memset(pages, 0, sizeof(*pages) + sizeof(phys_addr_t) * capacity);
+	pages->dev = dev;
+	pages->capacity = capacity;
+	INIT_LIST_HEAD(&pages->dmabuf);
+
+	for (pages->count = 0; pages->count < count;) {
+		phys_addr_t page;
+		unsigned int i;
+
+		/*
+		 * Allocate a Linux page. It will typically be of the same size
+		 * as the MVE page, but could also be larger.
+		 */
+		page = mvx_mmu_alloc_page(dev);
+		if (page == 0) {
+			ret = -ENOMEM;
+			goto release_pages;
+		}
+
+		/*
+		 * If the Linux page is larger than the MVE page, then
+		 * we iterate and add physical addresses with an offset from
+		 * the Linux page.
+		 */
+		for (i = 0; i < MVX_PAGES_PER_PAGE; i++)
+			pages->pages[pages->count++] = page + i * MVE_PAGE_SIZE;
+	}
+
+	return pages;
+
+release_pages:
+	mvx_mmu_free_pages(pages);
+
+	return ERR_PTR(ret);
+}
+
+struct mvx_mmu_pages *mvx_mmu_alloc_pages_sg(struct device *dev, struct sg_table *sgt, size_t capacity)
+{
+	struct mvx_mmu_pages *pages;
+	size_t count;
+	int ret;
+
+	count = get_sg_table_npages(sgt) * MVX_PAGES_PER_PAGE;
+	capacity = roundup(capacity, MVX_PAGES_PER_PAGE);
+	capacity = max(count, capacity);
+
+	pages = vmalloc(sizeof(*pages) + sizeof(phys_addr_t) * capacity);
+	if (pages == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	memset(pages, 0, sizeof(*pages) + sizeof(phys_addr_t) * capacity);
+	pages->dev = dev;
+	pages->capacity = capacity;
+	pages->is_external = true;
+	pages->offset = sgt->sgl != NULL ? sgt->sgl->offset : 0;
+	INIT_LIST_HEAD(&pages->dmabuf);
+
+	ret = append_sg_table(pages, sgt);
+	if (ret != 0) {
+		vfree(pages);
+		return ERR_PTR(ret);
+	}
+
+	return pages;
+}
+
+struct mvx_mmu_pages *mvx_mmu_alloc_pages_dma_buf(struct device *dev, struct dma_buf *dmabuf, size_t capacity)
+{
+	struct mvx_mmu_pages *pages;
+	struct dma_buf_attachment *attach;
+	struct sg_table *sgt;
+	struct mvx_mmu_dma_buf *mbuf;
+
+	attach = dma_buf_attach(dmabuf, dev);
+	if (IS_ERR(attach)) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to attach DMA buffer.");
+		return (struct mvx_mmu_pages *)attach;
+	}
+
+	sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sgt)) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to get SG table from DMA buffer.");
+		pages = (struct mvx_mmu_pages *)sgt;
+		goto detach;
+	}
+
+	pages = mvx_mmu_alloc_pages_sg(dev, sgt, capacity);
+	if (IS_ERR(pages))
+		goto unmap;
+
+	mbuf = vmalloc(sizeof(*mbuf));
+	if (mbuf == NULL) {
+		mvx_mmu_free_pages(pages);
+		pages = ERR_PTR(-ENOMEM);
+		goto unmap;
+	}
+
+	memset(mbuf, 0, sizeof(*mbuf));
+	mbuf->dmabuf = dmabuf;
+	list_add_tail(&mbuf->head, &pages->dmabuf);
+
+unmap:
+	dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
+
+detach:
+	dma_buf_detach(dmabuf, attach);
+
+	return pages;
+}
+
+int mvx_mmu_pages_append_dma_buf(struct mvx_mmu_pages *pages, struct dma_buf *dmabuf)
+{
+	struct dma_buf_attachment *attach;
+	struct sg_table *sgt;
+	struct mvx_mmu_dma_buf *mbuf;
+	size_t oldcount = pages->count;
+	int ret;
+
+	if (pages->is_external == false) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Can't append DMA buffer to internal pages object.");
+		return -EINVAL;
+	}
+
+	attach = dma_buf_attach(dmabuf, pages->dev);
+	if (IS_ERR(attach)) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to attach DMA buffer.");
+		return PTR_ERR(attach);
+	}
+
+	sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sgt)) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to get SG table from DMA buffer.");
+		ret = PTR_ERR(sgt);
+		goto detach;
+	}
+
+	ret = append_sg_table(pages, sgt);
+	if (ret != 0)
+		goto unmap;
+
+	ret = remap_pages(pages, oldcount);
+	if (ret != 0)
+		goto unmap;
+
+	mbuf = vmalloc(sizeof(*mbuf));
+	if (mbuf == NULL) {
+		ret = -ENOMEM;
+		goto unmap;
+	}
+
+	memset(mbuf, 0, sizeof(*mbuf));
+	mbuf->dmabuf = dmabuf;
+	list_add_tail(&mbuf->head, &pages->dmabuf);
+
+unmap:
+	dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
+
+detach:
+	dma_buf_detach(dmabuf, attach);
+
+	return ret;
+}
+
+int mvx_mmu_resize_pages(struct mvx_mmu_pages *pages, size_t npages)
+{
+	size_t oldcount = pages->count;
+
+	if (pages->is_external != false) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "MMU with externally managed pages cannot be resized.");
+		return -EINVAL;
+	}
+
+	npages = roundup(npages, MVX_PAGES_PER_PAGE);
+
+	if (npages > pages->capacity) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "New MMU pages size is larger than capacity. npages=%zu, capacity=%zu.", npages,
+			      pages->capacity);
+		return -ENOMEM;
+	}
+
+	/* Free pages if npage is smaller than allocated pages. */
+	while (pages->count > npages) {
+		pages->count--;
+
+		if ((pages->count % MVX_PAGES_PER_PAGE) == 0)
+			mvx_mmu_free_page(pages->dev, pages->pages[pages->count]);
+
+		pages->pages[pages->count] = 0;
+	}
+
+	/* Allocate pages if npage is larger than allocated pages. */
+	while (pages->count < npages) {
+		phys_addr_t page;
+		unsigned int i;
+
+		page = mvx_mmu_alloc_page(pages->dev);
+		if (page == 0)
+			return -ENOMEM;
+
+		for (i = 0; i < MVX_PAGES_PER_PAGE; i++)
+			pages->pages[pages->count++] = page + i * MVE_PAGE_SIZE;
+	}
+
+	return remap_pages(pages, oldcount);
+}
+
+void mvx_mmu_free_pages(struct mvx_mmu_pages *pages)
+{
+	struct mvx_mmu_dma_buf *mbuf;
+	struct mvx_mmu_dma_buf *tmp;
+	unsigned int i;
+
+	mvx_mmu_unmap_pages(pages);
+
+	if (pages->is_external == false)
+		for (i = 0; i < pages->count; i += MVX_PAGES_PER_PAGE)
+			mvx_mmu_free_page(pages->dev, pages->pages[i]);
+
+	list_for_each_entry_safe(mbuf, tmp, &pages->dmabuf, head)
+	{
+		dma_buf_put(mbuf->dmabuf);
+		vfree(mbuf);
+	}
+
+	vfree(pages);
+}
+
+size_t mvx_mmu_size_pages(struct mvx_mmu_pages *pages) { return pages->count * MVE_PAGE_SIZE; }
+
+int mvx_mmu_synch_pages(struct mvx_mmu_pages *pages, enum dma_data_direction dir, int page_off, int page_count)
+{
+	size_t i;
+	if (page_off + page_count > pages->count) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Illegal mmu sync offset/size (%d/%d).", page_off,
+			      page_count);
+		page_off = 0;
+		page_count = pages->count;
+	}
+	if (dir == DMA_FROM_DEVICE) {
+		for (i = 0; i < page_count; i += MVX_PAGES_PER_PAGE)
+			dma_sync_single_for_cpu(pages->dev, pages->pages[page_off + i], PAGE_SIZE, DMA_FROM_DEVICE);
+	} else if (dir == DMA_TO_DEVICE) {
+		for (i = 0; i < page_count; i += MVX_PAGES_PER_PAGE)
+			dma_sync_single_for_device(pages->dev, pages->pages[page_off + i], PAGE_SIZE, DMA_TO_DEVICE);
+	} else {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unsupported MMU flush direction. dir=%u.", dir);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int mvx_mmu_map_pages(struct mvx_mmu *mmu, mvx_mmu_va va, struct mvx_mmu_pages *pages, enum mvx_mmu_attr attr,
+		      enum mvx_mmu_access access)
+{
+	size_t i;
+	int ret;
+
+	/* Map the allocated pages. */
+	for (i = 0; i < pages->count; i++) {
+		ret = map_page(mmu, va + i * MVE_PAGE_SIZE, pages->pages[i], attr, access);
+		if (ret != 0)
+			goto unmap_pages;
+	}
+
+	/*
+	 * Reserve the rest of the address range. Adding a dummy page with
+	 * physical address 'PAGE_SIZE' should not lead to memory corruption,
+	 * because the page is marked as 'no access'.
+	 */
+	for (; i < pages->capacity; i++) {
+		ret = map_page(mmu, va + i * MVE_PAGE_SIZE, MVE_PAGE_SIZE, MVX_ATTR_PRIVATE, MVX_ACCESS_NO);
+		if (ret != 0)
+			goto unmap_pages;
+	}
+
+	pages->mmu = mmu;
+	pages->va = va;
+	pages->attr = attr;
+	pages->access = access;
+
+	return 0;
+
+unmap_pages:
+	while (i-- > 0)
+		unmap_page(mmu, va + i * MVE_PAGE_SIZE);
+
+	return ret;
+}
+
+void mvx_mmu_unmap_pages(struct mvx_mmu_pages *pages)
+{
+	size_t i;
+
+	if (pages->mmu == NULL)
+		return;
+
+	for (i = 0; i < pages->capacity; i++)
+		unmap_page(pages->mmu, pages->va + i * MVE_PAGE_SIZE);
+
+	pages->mmu = NULL;
+	pages->va = 0;
+}
+
+int mvx_mmu_map_pa(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t pa, size_t size, enum mvx_mmu_attr attr,
+		   enum mvx_mmu_access access)
+{
+	int ret;
+	size_t offset;
+
+	for (offset = 0; offset < size; offset += MVE_PAGE_SIZE) {
+		ret = map_page(mmu, va + offset, pa + offset, attr, access);
+		if (ret != 0)
+			goto unmap_pages;
+	}
+
+	return 0;
+
+unmap_pages:
+	/* Unroll mapped pages. */
+	while (offset > 0) {
+		offset -= MVE_PAGE_SIZE;
+		unmap_page(mmu, va + offset);
+	}
+
+	return ret;
+}
+
+int mvx_mmu_map_l2(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t pa)
+{
+	phys_addr_t l2;
+	mvx_mmu_pte *pte = mmu->page_table;
+	unsigned int index;
+
+	/* Level 1. */
+	index = get_index(va, 0);
+	l2 = get_pa(pte[index]);
+
+	if (l2 != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "Failed to map L2 page. Page already exists.");
+		return -EINVAL;
+	}
+
+	set_bit(index, mmu->l2_page_is_external);
+
+	pte[index] = mvx_mmu_set_pte(MVX_ATTR_PRIVATE, pa, MVX_ACCESS_READ_ONLY);
+	dma_sync_single_for_device(mmu->dev, virt_to_phys(&pte[index]), sizeof(pte[index]), DMA_TO_DEVICE);
+
+	return 0;
+}
+
+void mvx_mmu_unmap_va(struct mvx_mmu *mmu, mvx_mmu_va va, size_t size)
+{
+	size_t offset;
+
+	for (offset = 0; offset < size; offset += MVE_PAGE_SIZE)
+		unmap_page(mmu, va + offset);
+}
+
+int mvx_mmu_va_to_pa(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t *pa)
+{
+	mvx_mmu_pte *pte;
+	phys_addr_t page;
+
+	pte = ptw(mmu, va, false);
+	if (IS_ERR(pte))
+		return PTR_ERR(pte);
+
+	page = get_pa(*pte);
+	if (page == 0)
+		return -EFAULT;
+
+	*pa = page | get_offset(va);
+
+	return 0;
+}
+
+/* LCOV_EXCL_START */
+int mvx_mmu_read(struct mvx_mmu *mmu, mvx_mmu_va va, void *data, size_t size)
+{
+	mvx_mmu_va end = va + size;
+
+	while (va < end) {
+		int ret;
+		size_t n;
+		phys_addr_t pa = 0;
+		void *src;
+
+		/* Calculate number of bytes to be copied. */
+		n = min(end - va, MVE_PAGE_SIZE - (va & MVE_PAGE_MASK));
+
+		/* Translate virtual- to physical address. */
+		ret = mvx_mmu_va_to_pa(mmu, va, &pa);
+		if (ret != 0)
+			return ret;
+
+		/* Invalidate the data range. */
+		dma_sync_single_for_cpu(mmu->dev, pa, n, DMA_FROM_DEVICE);
+
+		/* Convert from physical to Linux logical address. */
+		src = phys_to_virt(pa);
+		memcpy(data, src, n);
+
+		va += n;
+		data += n;
+	}
+
+	return 0;
+}
+
+/* LCOV_EXCL_STOP */
+
+int mvx_mmu_write(struct mvx_mmu *mmu, mvx_mmu_va va, const void *data, size_t size)
+{
+	mvx_mmu_va end = va + size;
+
+	while (va < end) {
+		int ret;
+		size_t n;
+		phys_addr_t pa = 0;
+		void *dst;
+
+		/* Calculate number of bytes to be copied. */
+		n = min(end - va, MVE_PAGE_SIZE - (va & MVE_PAGE_MASK));
+
+		/* Translate virtual- to physical address. */
+		ret = mvx_mmu_va_to_pa(mmu, va, &pa);
+		if (ret != 0)
+			return ret;
+
+		/* Convert from physical to Linux logical address. */
+		dst = phys_to_virt(pa);
+		memcpy(dst, data, n);
+
+		/* Flush the data to memory. */
+		dma_sync_single_for_device(mmu->dev, pa, n, DMA_TO_DEVICE);
+
+		va += n;
+		data += n;
+	}
+
+	return 0;
+}
+
+mvx_mmu_pte mvx_mmu_set_pte(enum mvx_mmu_attr attr, phys_addr_t pa, enum mvx_mmu_access access)
+{
+	return (attr << MVE_PTE_ATTR_SHIFT) | ((pa >> MVE_PAGE_SHIFT) << MVE_PTE_PHYSADDR_SHIFT) |
+	       (access << MVE_PTE_AP_SHIFT);
+}
+
+/* LCOV_EXCL_START */
+void mvx_mmu_print(struct mvx_mmu *mmu)
+{
+	unsigned int i;
+	mvx_mmu_pte *l1 = mmu->page_table;
+
+	for (i = 0; i < MVE_INDEX_SIZE; i++) {
+		phys_addr_t pa = get_pa(l1[i]);
+		unsigned int j;
+
+		if (pa != 0) {
+			mvx_mmu_pte *l2 = phys_to_virt(pa);
+
+			MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "%-4u: PA=0x%llx, ATTR=%u, ACC=%u", i, pa,
+				      get_attr(l1[i]), get_ap(l1[i]));
+
+			for (j = 0; j < MVE_INDEX_SIZE; j++) {
+				pa = get_pa(l2[j]);
+				if (pa != 0) {
+					mvx_mmu_va va;
+
+					va = (i << (MVE_INDEX_SHIFT + MVE_PAGE_SHIFT)) | (j << MVE_PAGE_SHIFT);
+					MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO,
+						      "|------ %-4u: VA=0x%08x, PA=0x%llx, ATTR=%u, ACC=%u", j, va, pa,
+						      get_attr(l2[j]), get_ap(l2[j]));
+				}
+			}
+		}
+	}
+}
+
+/* LCOV_EXCL_STOP */
+
+int mvx_mmu_pages_debugfs_init(struct mvx_mmu_pages *pages, char *name, struct dentry *parent)
+{
+	struct dentry *dpages;
+	struct dentry *dentry;
+
+	dpages = debugfs_create_dir(name, parent);
+	if (IS_ERR_OR_NULL(dpages))
+		return -ENOMEM;
+
+	dentry = debugfs_create_file("stat", 0400, dpages, pages, &stat_fops);
+	if (IS_ERR_OR_NULL(dentry))
+		return -ENOMEM;
+
+	dentry = debugfs_create_file("list", 0400, dpages, pages, &list_fops);
+	if (IS_ERR_OR_NULL(dentry))
+		return -ENOMEM;
+
+	return 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_mmu.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_mmu.h
new file mode 100644
index 000000000000..079ddd231a9a
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_mmu.h
@@ -0,0 +1,435 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_MMU_H_
+#define _MVX_MMU_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/dma-mapping.h>
+#include <linux/hashtable.h>
+#include <linux/types.h>
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+/* Page size in bits. 2^12 = 4kB. */
+#define MVE_PAGE_SHIFT 12
+#define MVE_PAGE_SIZE (1 << MVE_PAGE_SHIFT)
+#define MVE_PAGE_MASK (MVE_PAGE_SIZE - 1)
+
+/* Number of page table entries per page. */
+#define MVE_PAGE_PTE_PER_PAGE (MVE_PAGE_SIZE / sizeof(mvx_mmu_pte))
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct dma_buf;
+struct mvx_mmu;
+struct page;
+
+/**
+ * typedef mvx_mmu_va - 32 bit virtual address.
+ *
+ * This is the address the firmware/MVE will use.
+ */
+typedef uint32_t mvx_mmu_va;
+
+/**
+ * typedef mvx_mmu_pte - Page table entry.
+ *
+ * A PTE pointer should always point at a Linux kernel virtual address.
+ *
+ * AT - Attribute.
+ * PA - Physical address.
+ * AP - Access permission.
+ *
+ *   30                                                       2   0
+ * +---+-------------------------------------------------------+---+
+ * | AT|                        PA 39:12                       | AP|
+ * +---+-------------------------------------------------------+---+
+ */
+typedef uint32_t mvx_mmu_pte;
+
+enum mvx_mmu_attr {
+	MVX_ATTR_PRIVATE   = 0,
+	MVX_ATTR_REFFRAME  = 1,
+	MVX_ATTR_SHARED_RO = 2,
+	MVX_ATTR_SHARED_RW = 3
+};
+
+enum mvx_mmu_access {
+	MVX_ACCESS_NO = 0,
+	MVX_ACCESS_READ_ONLY = 1,
+	MVX_ACCESS_EXECUTABLE = 2,
+	MVX_ACCESS_READ_WRITE = 3
+};
+
+/**
+ * struct mvx_mmu_pages - Structure used to allocate an array of pages.
+ * @dev:	Pointer to device.
+ * @node:	Hash table node. Used to keep track of allocated pages objects.
+ * @mmu:	Pointer to MMU instance.
+ * @va:		MVE virtual address. Set to 0 if objects is unmapped.
+ * @offset:	Offset from mapped VA to where the data begins.
+ * @attr:	Page table attributes.
+ * @access:	Page table access.
+ * @capacity:	Maximum number of MVE pages this object can hold.
+ * @count:	Current number of allocated pages.
+ * @is_external:If the physical pages have been externally allocated.
+ * @dmabuf:	List of DMA buffers.
+ * @pages:	Array of pages.
+ */
+struct mvx_mmu_pages {
+	struct device *dev;
+	struct hlist_node node;
+	struct mvx_mmu *mmu;
+	mvx_mmu_va va;
+	size_t offset;
+	enum mvx_mmu_attr attr;
+	enum mvx_mmu_access access;
+	size_t capacity;
+	size_t count;
+	bool is_external;
+	struct list_head dmabuf;
+	phys_addr_t pages[];
+};
+
+/**
+ * struct mvx_mmu - MMU context.
+ * @dev:	Pointer to device.
+ * @page_table:	Virtual address to L1 page.
+ * @l2_page_is_external: Bitmap of which L2 pages that have been mapped
+ *                       externally.
+ */
+struct mvx_mmu {
+	struct device *dev;
+	mvx_mmu_pte *page_table;
+	DECLARE_BITMAP(l2_page_is_external, MVE_PAGE_PTE_PER_PAGE);
+};
+
+/****************************************************************************
+ * Static functions
+ ****************************************************************************/
+
+#ifndef phys_to_page
+
+/**
+ * phys_to_page() - Convert a physical address to a pointer to a page.
+ * @pa:		Physical address.
+ *
+ * Return: Pointer to page struct.
+ */
+static inline struct page *phys_to_page(unsigned long pa)
+{
+	return pfn_to_page(__phys_to_pfn(pa));
+}
+
+#endif
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_mmu_construct() - Construct the MMU object.
+ * @mmu:	Pointer to MMU object.
+ * @dev:	Pointer to device.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_construct(struct mvx_mmu *mmu, struct device *dev);
+
+/**
+ * mvx_mmu_destruct() - Destroy the MMU object.
+ * @mmu:	Pointer to MMU object.
+ */
+void mvx_mmu_destruct(struct mvx_mmu *mmu);
+
+/**
+ * mvx_mmu_alloc_contiguous_pages() - Allocate contiguous pages.
+ * dev:		Pointer to device.
+ * npages:	Number of pages to allocate.
+ * Return: Physical page address on success, else 0.
+ */
+phys_addr_t mvx_mmu_alloc_contiguous_pages(struct device *dev, size_t npages);
+
+/*
+ * mvx_mmu_free_contiguous_pages() - Free contiguous pages.
+ *
+ * dev:		Pointer to device.
+ * pa:		Physical page address or 0.
+ * npages:	Number of pages to free.
+ */
+void mvx_mmu_free_contiguous_pages(struct device *dev, phys_addr_t pa, size_t npages);
+
+/**
+ * mvx_mmu_alloc_page() - Allocate one page.
+ * dev:		Pointer to device.
+ *
+ * Return: Physical page address on success, else 0.
+ */
+phys_addr_t mvx_mmu_alloc_page(struct device *dev);
+
+/*
+ * mvx_mmu_free_page() - Free one page.
+ *
+ * dev:		Pointer to device.
+ * pa:		Physical page address or 0.
+ */
+void mvx_mmu_free_page(struct device *dev, phys_addr_t pa);
+
+/**
+ * mvx_mmu_dma_alloc_coherent() - Allocate one page for non-cacheable attribute.
+ *
+ * dev:	   Pointer to device.
+ * data:   Virtual address for cpu access.
+ * Return: Physical page address or 0.
+ */
+phys_addr_t mvx_mmu_dma_alloc_coherent(struct device *dev, void **data);
+
+/**
+ * mvx_mmu_dma_free_coherent() - Free one page for non-cacheable attribute.
+ *
+ * dev:	   Pointer to device.
+ * pa:     Physical page address or 0.
+ * data:   Virtual address for cpu access.
+ */
+void mvx_mmu_dma_free_coherent(struct device *dev, phys_addr_t pa, void *data);
+
+/**
+ * mvx_mmu_alloc_pages() - Allocate array of pages.
+ * @dev:	Pointer to device.
+ * @npages	Number of pages to allocate.
+ * @capacity:	Maximum number of pages this allocation can be resized
+ *              to. If this value is 0 or smaller than npages, then it will be
+ *              set to npages.
+ *
+ * Pages are not guaranteed to be physically continuous.
+ *
+ * Return: Valid pointer on success, else ERR_PTR.
+ */
+struct mvx_mmu_pages *mvx_mmu_alloc_pages(struct device *dev, size_t npages, size_t capacity);
+
+/**
+ * mvx_mmu_alloc_pages_sg() - Allocate array of pages from SG table.
+ * @dev:	Pointer to device.
+ * @sgt:	Scatter-gatter table with pre-allocated memory pages.
+ * @capacity:	Maximum number of pages this allocation can be resized
+ *              to. If this value is 0 or smaller than number of pages
+ *              in scatter gather table, then it will be rounded up to
+ *              to SG table size.
+ *
+ * Pages are not guaranteed to be physically continuous.
+ *
+ * Return: Valid pointer on success, else ERR_PTR.
+ */
+struct mvx_mmu_pages *mvx_mmu_alloc_pages_sg(struct device *dev, struct sg_table *sgt, size_t capacity);
+
+/**
+ * mvx_mmu_alloc_pages_dma_buf() - Allocate pages object from DMA buffer.
+ * @dev:	Pointer to device.
+ * @dma_buf:	Pointer to DMA buffer.
+ * @capacity:	Maximum number of pages this allocation can be resized
+ *              to. If this value is 0 or smaller than number of pages
+ *              in DMA buffer, then it will be rounded up to DMA buffer
+ *              size.
+ *
+ * The pages object will take ownership of the DMA buffer and call
+ * dma_put_buf() when the pages object is destructed.
+ *
+ * Return: Valid pointer on success, else ERR_PTR.
+ */
+struct mvx_mmu_pages *mvx_mmu_alloc_pages_dma_buf(struct device *dev, struct dma_buf *dmabuf, size_t capacity);
+
+/**
+ * mvx_mmu_pages_append_dma_buf() - Append DMA buffer to pages object.
+ * @pages:	Pointer to pages object.
+ * @dma_buf:	Pointer to DMA buffer.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_pages_append_dma_buf(struct mvx_mmu_pages *pages, struct dma_buf *dmabuf);
+
+/**
+ * mvx_mmu_resize_pages() - Resize the page allocation.
+ * @pages:	Pointer to pages object.
+ * @npages:	Number of pages to allocate.
+ *
+ * If the number of pages is smaller, then pages will be freed.
+ *
+ * If the number of pages is larger, then additional memory will be allocated.
+ * The already allocates pages will keep their physical addresses.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_resize_pages(struct mvx_mmu_pages *pages, size_t npages);
+
+/**
+ * mvx_mmu_free_pages() - Free pages.
+ * @pages:	Pointer to pages object.
+ */
+void mvx_mmu_free_pages(struct mvx_mmu_pages *pages);
+
+/**
+ * mvx_mmu_size_pages() - Get number of allocated bytes.
+ * @pages:	Pointer to pages object.
+ *
+ * Return: Size in bytes of pages.
+ */
+size_t mvx_mmu_size_pages(struct mvx_mmu_pages *pages);
+
+/**
+ * mvx_buffer_synch() - Synch data caches.
+ * @pages:	Pointer to pages object.
+ * @dir:	Which direction to synch.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_synch_pages(struct mvx_mmu_pages *pages, enum dma_data_direction dir, int page_off, int page_count);
+
+/**
+ * mvx_mmu_map_pages() - Map an array of pages to a virtual address.
+ * @mmu:	Pointer to MMU object.
+ * @va:		Virtual address.
+ * @pages:	Pointer to pages object.
+ * @attr:	Bus attributes.
+ * @access:	Access permission.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_map_pages(struct mvx_mmu *mmu, mvx_mmu_va va, struct mvx_mmu_pages *pages, enum mvx_mmu_attr attr,
+		      enum mvx_mmu_access access);
+
+/**
+ * mvx_mmu_unmap_pages() - Unmap pages object.
+ * @pages:	Pointer to pages object.
+ */
+void mvx_mmu_unmap_pages(struct mvx_mmu_pages *pages);
+
+/**
+ * mvx_mmu_map_pa() - Map a physical- to a virtual address.
+ * @mmu:	Pointer to MMU object.
+ * @va:		Virtual address.
+ * @pa:		Physical address.
+ * @size:	Size of area to map.
+ * @attr:	Bus attributes.
+ * @access:	Access permission.
+ *
+ * Both the VA and PA must be page aligned.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_map_pa(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t pa, size_t size, enum mvx_mmu_attr attr,
+		   enum mvx_mmu_access access);
+
+/**
+ * mvx_mmu_map_l2() - Map a L2 page.
+ * @mmu:	Pointer to MMU object.
+ * @va:		Virtual address.
+ * @pa:		Physical address.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_map_l2(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t pa);
+
+/**
+ * mvx_mmu_unmap_va() - Unmap a virtual address range.
+ * @mmu:	Pointer to MMU object.
+ * @va:		Virtual address.
+ * @size:	Size of area to unmap.
+ */
+void mvx_mmu_unmap_va(struct mvx_mmu *mmu, mvx_mmu_va va, size_t size);
+
+/**
+ * mvx_mmu_va_to_pa() - Map a virtual- to a physical address.
+ * @mmu:	Pointer to MMU object.
+ * @va:		Virtual address.
+ * @pa:		Pointer to physical address.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_va_to_pa(struct mvx_mmu *mmu, mvx_mmu_va va, phys_addr_t *pa);
+
+/**
+ * mvx_mmu_read() - Read size bytes from virtual address.
+ * @mmu:	Pointer to MMU object.
+ * @va:		Source virtual address.
+ * @data:	Pointer to destination data.
+ * @size:	Number of bytes to copy.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_read(struct mvx_mmu *mmu, mvx_mmu_va va, void *data, size_t size);
+
+/**
+ * mvx_mmu_write() - Write size bytes to virtual address.
+ * @mmu:	Pointer to MMU object.
+ * @va:		Destination virtual address.
+ * @data:	Pointer to source data.
+ * @size:	Number of bytes to copy.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_write(struct mvx_mmu *mmu, mvx_mmu_va va, const void *data, size_t size);
+
+/**
+ * mvx_mmu_set_pte() - Construct PTE and return PTE value.
+ * @attr:	Bus attributes.
+ * @pa:		Physical address.
+ * @access:	Access permission.
+ *
+ * Return: Page table entry.
+ */
+mvx_mmu_pte mvx_mmu_set_pte(enum mvx_mmu_attr attr, phys_addr_t pa, enum mvx_mmu_access access);
+
+/**
+ * mvx_mmu_print() - Print the MMU table.
+ * @mmu:	Pointer to MMU object.
+ */
+void mvx_mmu_print(struct mvx_mmu *mmu);
+
+/**
+ * mvx_mmu_pages_debugfs_init() - Init debugfs entry.
+ * @pages:	Pointer to MMU pages.
+ * @name:	Name of debugfs entry.
+ * @parent:	Parent debugfs entry.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_mmu_pages_debugfs_init(struct mvx_mmu_pages *pages, char *name, struct dentry *parent);
+
+#endif /* _MVX_MMU_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_secure.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_secure.c
new file mode 100644
index 000000000000..ef995da8de36
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_secure.c
@@ -0,0 +1,382 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/kobject.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include "mvx_log_group.h"
+#include "mvx_secure.h"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+#pragma pack(push, 1)
+struct secure_firmware_desc {
+	int32_t fd;
+	uint32_t l2pages;
+	struct {
+		uint32_t major;
+		uint32_t minor;
+	} protocol;
+};
+#pragma pack(pop)
+
+struct mvx_secure_firmware_priv {
+	struct device *dev;
+	struct kobject kobj;
+	struct work_struct work;
+	wait_queue_head_t wait_queue;
+	struct mvx_secure_firmware fw;
+	mvx_secure_firmware_done done;
+	void *done_arg;
+};
+
+struct mvx_secure_mem {
+	struct device *dev;
+	struct kobject kobj;
+	wait_queue_head_t wait_queue;
+	struct dma_buf *dmabuf;
+};
+
+/****************************************************************************
+ * Secure
+ ****************************************************************************/
+
+int mvx_secure_construct(struct mvx_secure *secure, struct device *dev)
+{
+	secure->dev = dev;
+#ifndef MODULE
+	secure->kset = kset_create_and_add("securevideo", NULL, &dev->kobj);
+	if (secure->kset == NULL) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to create securevideo kset.");
+		return -EINVAL;
+	}
+#endif
+	secure->workqueue = alloc_workqueue("mvx_securevideo", WQ_UNBOUND | WQ_MEM_RECLAIM, 1);
+	if (secure->workqueue == NULL) {
+#ifndef MODULE
+		kset_unregister(secure->kset);
+#endif
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void mvx_secure_destruct(struct mvx_secure *secure)
+{
+	destroy_workqueue(secure->workqueue);
+#ifndef MODULE
+	kset_unregister(secure->kset);
+#endif
+}
+
+/****************************************************************************
+ * Secure firmware
+ ****************************************************************************/
+
+/**
+ * firmware_store() - Firmware sysfs store function.
+ *
+ * Store values from firmware descriptor, get the DMA handle and wake up any
+ * waiting process.
+ */
+static ssize_t firmware_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t size)
+{
+	struct mvx_secure_firmware_priv *securefw = container_of(kobj, struct mvx_secure_firmware_priv, kobj);
+	const struct secure_firmware_desc *desc = (const void *)buf;
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "Loaded secure firmware. fd=%d, l2=0x%llx, major=%u, minor=%u.",
+		      desc->fd, desc->l2pages, desc->protocol.major, desc->protocol.minor);
+
+	securefw->fw.l2pages = desc->l2pages;
+	securefw->fw.protocol.major = desc->protocol.major;
+	securefw->fw.protocol.minor = desc->protocol.minor;
+	securefw->fw.dmabuf = dma_buf_get(desc->fd);
+	if (IS_ERR_OR_NULL(securefw->fw.dmabuf))
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to get DMA buffer from fd. fd=%d.", desc->fd);
+
+	wake_up_interruptible(&securefw->wait_queue);
+
+	return size;
+}
+
+/**
+ * secure_firmware_release() - Release secure firmware.
+ * kobj:	Pointer to kobject.
+ */
+static void secure_firmware_release(struct kobject *kobj)
+{
+	struct mvx_secure_firmware_priv *securefw = container_of(kobj, struct mvx_secure_firmware_priv, kobj);
+
+	if (IS_ERR_OR_NULL(securefw->fw.dmabuf) == false)
+		dma_buf_put(securefw->fw.dmabuf);
+
+	devm_kfree(securefw->dev, securefw);
+}
+
+/**
+ * secure_firmware_wait() - Wait for firmware load.
+ * @work:	Pointer to work member in mvx_secure_firmware_priv.
+ *
+ * Worker thread used to wait for a secure firmware load to complete.
+ */
+static void secure_firmware_wait(struct work_struct *work)
+{
+	struct mvx_secure_firmware_priv *securefw = container_of(work, struct mvx_secure_firmware_priv, work);
+	int ret;
+
+	ret = wait_event_interruptible_timeout(securefw->wait_queue, securefw->fw.dmabuf != NULL,
+					       msecs_to_jiffies(10000));
+	if (ret == 0)
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Firmware load timed out.");
+
+	kobject_del(&securefw->kobj);
+
+	if (securefw->done != NULL)
+		securefw->done(&securefw->fw, securefw->done_arg);
+}
+
+/**
+ * secure_firmware_create() - Create a secure firmware object.
+ * @secure:	Pointer to secure context.
+ * @name:	Name for secure firmware binary.
+ * @ncores:	Number of cores to setup.
+ * @arg:	User argument to callback routine.
+ * @done:	Firware load callback routine.
+ *
+ * Return: Valid pointer on success, else ERR_PTR.
+ */
+static struct mvx_secure_firmware_priv *secure_firmware_create(struct mvx_secure *secure, const char *name,
+							       unsigned int ncores, void *arg,
+							       mvx_secure_firmware_done done)
+{
+	static struct kobj_attribute attr = __ATTR_WO(firmware);
+	static struct attribute *attrs[] = {
+		&attr.attr,
+		NULL
+	};
+	static const struct attribute_group secure_group = {
+        .name = "",
+        .attrs = attrs
+    };
+
+    static const struct attribute_group *secure_groups[] = {
+        &secure_group,
+        NULL
+    };
+	static struct kobj_type secure_ktype = {
+		.sysfs_ops     = &kobj_sysfs_ops,
+		.release       = secure_firmware_release,
+		.default_groups = secure_groups
+	};
+	struct mvx_secure_firmware_priv *securefw;
+	char numcores_env[32];
+	char fw_env[140];
+	char *env[] = {"TYPE=firmware", numcores_env, fw_env, NULL};
+	size_t n;
+	int ret;
+
+	n = snprintf(fw_env, sizeof(fw_env), "FIRMWARE=%s.enc", name);
+	if (n >= sizeof(fw_env))
+		return ERR_PTR(-EINVAL);
+
+	n = snprintf(numcores_env, sizeof(numcores_env), "NUMCORES=%u", ncores);
+	if (n >= sizeof(numcores_env))
+		return ERR_PTR(-EINVAL);
+
+	/* Allocate and initialize the secure firmware object. */
+	securefw = devm_kzalloc(secure->dev, sizeof(*securefw), GFP_KERNEL);
+	if (securefw == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	securefw->dev = secure->dev;
+	securefw->kobj.kset = secure->kset;
+	securefw->fw.ncores = ncores;
+	securefw->done = done;
+	securefw->done_arg = arg;
+	init_waitqueue_head(&securefw->wait_queue);
+
+	/* Create kobject that the user space helper can interact with. */
+	ret = kobject_init_and_add(&securefw->kobj, &secure_ktype, NULL, "%p", securefw);
+	if (ret != 0)
+		goto put_kobject;
+
+	/* Notify user space helper about the secure firmware load. */
+	ret = kobject_uevent_env(&securefw->kobj, KOBJ_ADD, env);
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to send secure firmware uevent. ret=%d.", ret);
+		goto put_kobject;
+	}
+
+	return securefw;
+
+put_kobject:
+	kobject_put(&securefw->kobj);
+	devm_kfree(secure->dev, securefw);
+
+	return ERR_PTR(ret);
+}
+
+int mvx_secure_request_firmware_nowait(struct mvx_secure *secure, const char *name, unsigned int ncores, void *arg,
+				       mvx_secure_firmware_done done)
+{
+	struct mvx_secure_firmware_priv *securefw;
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "Request secure firmware load nowait. firmware=%s.enc.", name);
+
+	securefw = secure_firmware_create(secure, name, ncores, arg, done);
+	if (IS_ERR(securefw))
+		return PTR_ERR(securefw);
+
+	INIT_WORK(&securefw->work, secure_firmware_wait);
+	queue_work(secure->workqueue, &securefw->work);
+
+	return 0;
+}
+
+void mvx_secure_release_firmware(struct mvx_secure_firmware *securefw)
+{
+	struct mvx_secure_firmware_priv *sfw = container_of(securefw, struct mvx_secure_firmware_priv, fw);
+
+	kobject_put(&sfw->kobj);
+}
+
+/****************************************************************************
+ * Secure memory
+ ****************************************************************************/
+
+/**
+ * secure_mem_release() - Release the secure memory object.
+ */
+static void secure_mem_release(struct kobject *kobj)
+{
+	struct mvx_secure_mem *smem = container_of(kobj, struct mvx_secure_mem, kobj);
+
+	devm_kfree(smem->dev, smem);
+}
+
+/**
+ * memory_store() - Memory sysfs store function.
+ *
+ * Store values from memory descriptor, get the DMA handle and wake up any
+ * waiting process.
+ */
+static ssize_t memory_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t size)
+{
+	struct mvx_secure_mem *smem = container_of(kobj, struct mvx_secure_mem, kobj);
+	const int32_t *fd = (const int32_t *)buf;
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "Secure memory allocated. fd=%d.", *fd);
+
+	smem->dmabuf = dma_buf_get(*fd);
+	if (IS_ERR_OR_NULL(smem->dmabuf))
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to get DMA buffer.");
+
+	wake_up_interruptible(&smem->wait_queue);
+
+	return size;
+}
+
+struct dma_buf *mvx_secure_mem_alloc(struct mvx_secure *secure, size_t size)
+{
+	static struct kobj_attribute attr = __ATTR_WO(memory);
+	static struct attribute *attrs[] = {
+		&attr.attr,
+		NULL
+	};
+	static const struct attribute_group secure_mem_group = {
+        .name = "",
+        .attrs = attrs
+    };
+
+    static const struct attribute_group *secure_mem_groups[] = {
+        &secure_mem_group,
+        NULL
+    };
+	static struct kobj_type secure_mem_ktype = {
+		.release       = secure_mem_release,
+		.sysfs_ops     = &kobj_sysfs_ops,
+		.default_groups = secure_mem_groups
+	};
+	struct mvx_secure_mem *smem;
+	char size_env[32];
+	char *env[] = {"TYPE=memory", size_env, NULL};
+	struct dma_buf *dmabuf = ERR_PTR(-EINVAL);
+	size_t n;
+	int ret;
+
+	MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "Request secure memory. size=%zu.", size);
+
+	n = snprintf(size_env, sizeof(size_env), "SIZE=%zu", size);
+	if (n >= sizeof(size_env))
+		return ERR_PTR(-EINVAL);
+
+	smem = devm_kzalloc(secure->dev, sizeof(*smem), GFP_KERNEL);
+	if (smem == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	smem->dev = secure->dev;
+	smem->kobj.kset = secure->kset;
+	init_waitqueue_head(&smem->wait_queue);
+
+	/* Create kobject that the user space helper can interact with. */
+	ret = kobject_init_and_add(&smem->kobj, &secure_mem_ktype, NULL, "%p", &smem->kobj);
+	if (ret != 0)
+		goto put_kobject;
+
+	/* Notify user space helper about the secure firmware load. */
+	ret = kobject_uevent_env(&smem->kobj, KOBJ_ADD, env);
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to send secure memory uevent. ret=%d.", ret);
+		goto put_kobject;
+	}
+
+	ret = wait_event_interruptible_timeout(smem->wait_queue, smem->dmabuf != NULL, msecs_to_jiffies(1000));
+	if (ret == 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Secure memory allocation timed out.");
+		goto put_kobject;
+	}
+
+	dmabuf = smem->dmabuf;
+
+put_kobject:
+	kobject_put(&smem->kobj);
+
+	return dmabuf;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_secure.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_secure.h
new file mode 100644
index 000000000000..8a4965152727
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_secure.h
@@ -0,0 +1,133 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_SECURE_H_
+#define _MVX_SECURE_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/types.h>
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct dma_buf;
+struct kset;
+struct mvx_secure_firmware;
+struct workqueue_struct;
+
+/**
+ * struct mvx_secure - Secure video.
+ * @dev:	Pointer to device.
+ * @kset:	Kset that allows uevents to be sent.
+ * @workqueue:	Work queue used to wait for firmware load.
+ */
+struct mvx_secure {
+	struct device *dev;
+	struct kset *kset;
+	struct workqueue_struct *workqueue;
+};
+
+/**
+ * typedef firmware_done - Firmware load callback.
+ */
+typedef void (*mvx_secure_firmware_done)(struct mvx_secure_firmware *, void *arg);
+
+/**
+ * struct mvx_secure_firmware - Secure firmware.
+ * @dmabuf:	Pointer to DMA buffer.
+ * @l2pages:	Array of L2 pages. One per core.
+ * @ncores:	Maximum number of cores.
+ * @major:	Firmware protocol major version.
+ * @minor:	Firmware protocol minor version.
+ */
+struct mvx_secure_firmware {
+	struct dma_buf *dmabuf;
+	phys_addr_t l2pages;
+	unsigned int ncores;
+	struct {
+		unsigned int major;
+		unsigned int minor;
+	} protocol;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_secure_construct() - Construct the secure object.
+ * @secure:	Pointer to secure object.
+ * @dev:	Pointer to device.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_secure_construct(struct mvx_secure *secure, struct device *dev);
+
+/**
+ * mvx_secure_destruct() - Destruct the secure object.
+ * @secure:	Pointer to secure object.
+ */
+void mvx_secure_destruct(struct mvx_secure *secure);
+
+/**
+ * mvx_secure_request_firmware_nowait() - Request secure firmware.
+ * @secure:     Pointer to secure object.
+ * @name:       Name of firmware binary.
+ * @ncores:	Number of cores to setup.
+ * @arg:        Callback argument.
+ * @done:       Done callback.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_secure_request_firmware_nowait(struct mvx_secure *secure, const char *name, unsigned int ncores, void *arg,
+				       mvx_secure_firmware_done done);
+
+/**
+ * mvx_secure_release_firmware() - Release secure firmware.
+ * @securefw:     Pointer to secure firmware.
+ */
+void mvx_secure_release_firmware(struct mvx_secure_firmware *securefw);
+
+/**
+ * mvx_secure_mem_alloc() - Secure memory allocation.
+ * @secure:	Pointer to secure object.
+ * @size:	Size in bytes to allocate.
+ *
+ * Return: Valid pointer on success, else ERR_PTR.
+ */
+struct dma_buf *mvx_secure_mem_alloc(struct mvx_secure *secure, size_t size);
+
+#endif /* _MVX_SECURE_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_session.c b/drivers/media/platform/spacemit/vpu_k3/if/mvx_session.c
new file mode 100644
index 000000000000..72b85a4a4b64
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_session.c
@@ -0,0 +1,3365 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/sched.h>
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+#include <linux/timer.h>
+#include "mvx-v4l2-controls.h"
+#include "mvx_bitops.h"
+#include "mvx_firmware.h"
+#include "mvx_firmware_cache.h"
+#include "mvx_session.h"
+#include "mvx_seq.h"
+#include "mvx_dvfs.h"
+#include "mvx_v4l2_session.h"
+
+/****************************************************************************
+ * Private variables
+ ****************************************************************************/
+
+int session_wait_pending_timeout = 10000; // 10s
+int session_wait_flush_timeout = 2000;	  // 2s
+
+int session_watchdog_timeout = 30000; // 30s
+module_param(session_watchdog_timeout, int, 0660);
+
+static int fw_watchdog_timeout;
+module_param(fw_watchdog_timeout, int, 0660);
+
+/****************************************************************************
+ * Private functions
+ ****************************************************************************/
+
+static void watchdog_start(struct mvx_session *session, unsigned int timeout_ms)
+{
+	int ret;
+
+	if (session->error != 0)
+		return;
+
+	MVX_SESSION_DEBUG(session, "Watchdog start. timeout_ms=%u.", timeout_ms);
+
+	ret = mod_timer(&session->watchdog_timer, jiffies + msecs_to_jiffies(timeout_ms));
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to start watchdog. ret=%d", ret);
+		return;
+	}
+
+	kref_get(&session->isession.kref);
+}
+
+static void watchdog_stop(struct mvx_session *session)
+{
+	int ret;
+
+#if (KERNEL_VERSION(6, 15, 0) <= LINUX_VERSION_CODE)
+	ret = timer_delete_sync(&session->watchdog_timer);
+#else
+	ret = del_timer_sync(&session->watchdog_timer);
+#endif
+
+	/* ret: 0=watchdog expired, 1=watchdog still running */
+	MVX_SESSION_DEBUG(session, "Watchdog stop. ret=%d", ret);
+	session->watchdog_count = 0;
+
+	/* Decrement the kref if the watchdog was still running. */
+	if (ret != 0)
+		kref_put(&session->isession.kref, session->isession.release);
+}
+
+static void watchdog_update(struct mvx_session *session, unsigned int timeout_ms)
+{
+	int ret;
+
+#ifndef MODULE
+	ret = mod_timer_pending(&session->watchdog_timer, jiffies + msecs_to_jiffies(timeout_ms));
+#else
+	if (timer_pending(&session->watchdog_timer))
+		ret = mod_timer(&session->watchdog_timer, jiffies + msecs_to_jiffies(timeout_ms));
+#endif
+
+	/* ret: 0=no restart, 1=restarted */
+	MVX_SESSION_DEBUG(session, "Watchdog update. ret=%d, timeout_ms=%u.", ret, timeout_ms);
+}
+
+static bool is_fw_loaded(struct mvx_session *session) { return (IS_ERR_OR_NULL(session->fw_bin) == false); }
+
+static void print_debug(struct mvx_session *session)
+{
+	MVX_SESSION_INFO(session, "Print debug.");
+
+	if (session->csession != NULL)
+		session->client_ops->print_debug(session->csession);
+
+	if (is_fw_loaded(session))
+		session->fw.ops.print_debug(&session->fw);
+}
+
+static void send_event_error(struct mvx_session *session, long error)
+{
+	session->error = error;
+	wake_up(&session->waitq);
+	MVX_SESSION_WARN(session, "send event error. error=%ld", error);
+	session->event(session, MVX_SESSION_EVENT_ERROR, (void *)session->error);
+}
+
+static void session_unregister(struct mvx_session *session)
+{
+	if (!IS_ERR_OR_NULL(session->csession)) {
+		mvx_dvfs_unregister_session(session);
+		session->client_ops->unregister_session(session->csession);
+		session->csession = NULL;
+	}
+}
+
+static void release_fw_bin(struct mvx_session *session)
+{
+	if (is_fw_loaded(session) != false) {
+		MVX_SESSION_INFO(session, "Release firmware binary.");
+
+		if (session->switched_in == true) {
+			if (IS_ERR_OR_NULL(session->csession)) {
+				MVX_SESSION_WARN(session, "release_fw_bin. csession is null. mvx_session:%p", session);
+			} else {
+				session->client_ops->wait_session_idle(session->csession);
+			}
+		}
+
+		mvx_fw_destruct(&session->fw);
+		mvx_fw_cache_put(session->cache, session->fw_bin);
+		session->fw_bin = NULL;
+	}
+
+	watchdog_stop(session);
+	session_unregister(session);
+}
+
+static struct mvx_session *kref_to_session(struct kref *kref)
+{
+	return container_of(kref, struct mvx_session, isession.kref);
+}
+
+static void session_destructor(struct kref *kref)
+{
+	struct mvx_session *session = kref_to_session(kref);
+
+	session->destructor(session);
+}
+
+static const char *state_to_string(enum mvx_fw_state state)
+{
+	switch (state) {
+	case MVX_FW_STATE_STOPPED:
+		return "Stopped";
+	case MVX_FW_STATE_RUNNING:
+		return "Running";
+	default:
+		return "Unknown";
+	}
+}
+
+static enum mvx_direction get_bitstream_port(struct mvx_session *session)
+{
+	if (mvx_is_bitstream(session->port[MVX_DIR_INPUT].format) && mvx_is_frame(session->port[MVX_DIR_OUTPUT].format))
+		return MVX_DIR_INPUT;
+	else if (mvx_is_frame(session->port[MVX_DIR_INPUT].format) &&
+		 mvx_is_bitstream(session->port[MVX_DIR_OUTPUT].format))
+		return MVX_DIR_OUTPUT;
+
+	return MVX_DIR_MAX;
+}
+
+static bool is_stream_on(struct mvx_session *session)
+{
+	return session->port[MVX_DIR_INPUT].stream_on && session->port[MVX_DIR_OUTPUT].stream_on;
+}
+
+/**
+ * wait_pending() - Wait for procedure to finish.
+ *
+ * Wait for the number of pending firmware messages to reach 0, or for an error
+ * to happen.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int wait_pending(struct mvx_session *session)
+{
+	int ret = 0;
+	int count = 0;
+	struct timespec64 curtime;
+	uint64_t timestart;
+	uint64_t timeend;
+
+	while (is_fw_loaded(session) != false && session->fw.msg_pending > 0 && session->error == 0) {
+		mutex_unlock(session->isession.mutex);
+
+		count++;
+		ktime_get_boottime_ts64(&curtime);
+		timestart = curtime.tv_sec * 1000 * 1000l + curtime.tv_nsec / 1000l;
+
+		ret = wait_event_timeout(session->waitq,
+					 is_fw_loaded(session) == false || session->fw.msg_pending == 0 ||
+						 session->error != 0,
+					 msecs_to_jiffies(session->watchdog_timeout * 2));
+
+		ktime_get_boottime_ts64(&curtime);
+		timeend = curtime.tv_sec * 1000 * 1000l + curtime.tv_nsec / 1000l;
+		if (timeend - timestart >= 10000000l) {
+			MVX_SESSION_WARN(session,
+					 "wait_pending cost time is greater than 10 secs. cost_time=%llu us, ret=%d, "
+					 "error=%d, msg_pending=%d, in_fmt=%d, out_fmt=%d",
+					 timeend - timestart, ret, session->error, session->fw.msg_pending,
+					 session->port[MVX_DIR_INPUT].format, session->port[MVX_DIR_OUTPUT].format);
+		}
+
+		if (ret < 0)
+			goto lock_mutex;
+
+		if (ret == 0) {
+			send_event_error(session, -ETIME);
+			ret = -ETIME;
+			goto lock_mutex;
+		}
+
+		mutex_lock(session->isession.mutex);
+	}
+
+	return session->error;
+
+lock_mutex:
+	mutex_lock(session->isession.mutex);
+
+	if (ret < 0) {
+		MVX_SESSION_WARN(session, "Wait pending returned error. ret=%d, error=%d, msg_pending=%d.", ret,
+				 session->error, session->fw.msg_pending);
+
+		print_debug(session);
+	}
+	return ret;
+}
+
+static int wait_flush_done(struct mvx_session *session, enum mvx_direction dir)
+{
+	int ret = 0;
+	struct mvx_session_port *port = &session->port[dir];
+
+	while (is_fw_loaded(session) != false && port->buffer_count > 0 && session->error == 0) {
+		mutex_unlock(session->isession.mutex);
+
+		ret = wait_event_timeout(session->waitq,
+					 is_fw_loaded(session) == false || port->buffer_count == 0 ||
+						 session->error != 0,
+					 msecs_to_jiffies(session_wait_flush_timeout));
+
+		if (ret < 0)
+			goto lock_mutex;
+
+		if (ret == 0) {
+			send_event_error(session, -ETIME);
+			ret = -ETIME;
+			goto lock_mutex;
+		}
+
+		mutex_lock(session->isession.mutex);
+	}
+
+	return session->error;
+
+lock_mutex:
+	mutex_lock(session->isession.mutex);
+
+	if (ret < 0) {
+		MVX_SESSION_WARN(session,
+				 "wait_flush_done returned error. ret=%d, error=%d, msg_pending=%d, buffer_count=%d",
+				 ret, session->error, session->fw.msg_pending, port->buffer_count);
+
+		print_debug(session);
+	}
+	return ret;
+}
+
+static int wait_switch_out(struct mvx_session *session)
+{
+	int ret = 0;
+
+	while (is_fw_loaded(session) != false && session->switched_in == true && session->error == 0) {
+		mutex_unlock(session->isession.mutex);
+
+		ret = wait_event_timeout(session->waitq,
+					 is_fw_loaded(session) == false || session->switched_in == false ||
+						 session->error != 0,
+					 msecs_to_jiffies(session_wait_pending_timeout));
+
+		if (ret < 0)
+			goto lock_mutex;
+
+		if (ret == 0) {
+			send_event_error(session, -ETIME);
+			ret = -ETIME;
+			goto lock_mutex;
+		}
+
+		mutex_lock(session->isession.mutex);
+	}
+
+	return session->error;
+
+lock_mutex:
+	mutex_lock(session->isession.mutex);
+
+	if (ret < 0) {
+		MVX_SESSION_WARN(session,
+				 "wait_switch_out returned error. ret=%d, error=%d, msg_pending=%d, switch_in=%d", ret,
+				 session->error, session->fw.msg_pending, session->switched_in);
+
+		print_debug(session);
+	}
+	return ret;
+}
+
+static int send_irq(struct mvx_session *session)
+{
+	if (IS_ERR_OR_NULL(session->csession))
+		return -EINVAL;
+
+	return session->client_ops->send_irq(session->csession);
+}
+
+/**
+ * switch_in() - Request the client device to switch in the session.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int switch_in(struct mvx_session *session)
+{
+	int ret;
+
+	session->idle_count = 0;
+
+	if (session->switched_in != false)
+		return 0;
+
+	if (IS_ERR_OR_NULL(session->csession))
+		return -EINVAL;
+
+	MVX_SESSION_INFO(session, "Switch in.");
+
+	ret = session->client_ops->switch_in(session->csession);
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to switch in session.");
+		send_event_error(session, ret);
+		return ret;
+	}
+
+	session->switched_in = true;
+
+	return 0;
+}
+
+/**
+ * fw_send_msg() - Send firmware message and signal IRQ.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int fw_send_msg(struct mvx_session *session, struct mvx_fw_msg *msg)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	ret = session->fw.ops.put_message(&session->fw, msg);
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to queue firmware message.");
+		goto send_error;
+	}
+
+	ret = send_irq(session);
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to send irq.");
+		goto send_error;
+	}
+
+	return switch_in(session);
+
+send_error:
+	send_event_error(session, ret);
+	return ret;
+}
+
+static int fw_send_msg_simple(struct mvx_session *session, enum mvx_fw_code code, const char *str)
+{
+	struct mvx_fw_msg msg = {.code = code};
+
+	MVX_SESSION_INFO(session, "Firmware req: %s.", str);
+
+	return fw_send_msg(session, &msg);
+}
+
+static int fw_flush(struct mvx_session *session, enum mvx_direction dir)
+{
+	struct mvx_fw_msg msg = {.code = MVX_FW_CODE_FLUSH, .flush.dir = dir};
+	int ret;
+
+	MVX_SESSION_INFO(session, "Firmware req: Flush. dir=%d.", dir);
+
+	ret = fw_send_msg(session, &msg);
+	if (ret != 0)
+		return ret;
+
+	session->port[dir].is_flushing = true;
+
+	return 0;
+}
+
+static int fw_state_change(struct mvx_session *session, enum mvx_fw_state state)
+{
+	struct mvx_fw_msg msg = {
+		.code  = MVX_FW_CODE_STATE_CHANGE,
+		.state = state
+	};
+	int ret = 0;
+
+	if (state != session->fw_state) {
+		MVX_SESSION_INFO(session, "Firmware req: State change. current=%d, new=%d.", session->fw_state, state);
+		ret = fw_send_msg(session, &msg);
+	}
+
+	return ret;
+}
+
+static int fw_job(struct mvx_session *session, unsigned int frames)
+{
+	struct mvx_fw_msg msg = {
+		.code       = MVX_FW_CODE_JOB,
+		.job.cores  = session->isession.ncores,
+		.job.frames = frames
+	};
+
+	MVX_SESSION_INFO(session, "Firmware req: Job. frames=%u.", frames);
+
+	return fw_send_msg(session, &msg);
+}
+
+static int fw_switch_out(struct mvx_session *session)
+{
+	unsigned int idle_count = session->idle_count;
+	int ret;
+
+	ret = fw_send_msg_simple(session, MVX_FW_CODE_SWITCH_OUT, "Switch out");
+
+	/*
+	 * Restore idle count. Switch out is the only message where we do not
+	 * want to reset the idle counter.
+	 */
+	session->idle_count = idle_count;
+
+	return ret;
+}
+
+static int fw_ping(struct mvx_session *session) { return fw_send_msg_simple(session, MVX_FW_CODE_PING, "Ping"); }
+
+static int fw_dump(struct mvx_session *session) { return fw_send_msg_simple(session, MVX_FW_CODE_DUMP, "Dump"); }
+
+#ifdef MVX_FW_DEBUG_ENABLE
+static int fw_set_debug(struct mvx_session *session, uint32_t debug_level)
+{
+	struct mvx_fw_msg msg = {
+		.code       = MVX_FW_CODE_DEBUG,
+		.arg = debug_level
+	};
+
+	MVX_SESSION_INFO(session, "Firmware req: Set debug. debug_level=%d.", debug_level);
+
+	return fw_send_msg(session, &msg);
+}
+#endif
+
+static int fw_set_option(struct mvx_session *session, struct mvx_fw_set_option *option)
+{
+	struct mvx_fw_msg msg = {
+		.code       = MVX_FW_CODE_SET_OPTION,
+		.set_option = *option
+	};
+
+	MVX_SESSION_INFO(session, "Firmware req: Set option. code=%d.", option->code);
+
+	return fw_send_msg(session, &msg);
+}
+
+static bool is_encoder(struct mvx_session *session) { return get_bitstream_port(session) == MVX_DIR_OUTPUT; }
+
+static int fw_eos(struct mvx_session *session)
+{
+	struct mvx_fw_msg msg = {
+		.code         = MVX_FW_CODE_EOS,
+		.eos_is_frame = is_encoder(session) ? true : false
+	};
+	int ret;
+
+	MVX_SESSION_INFO(session, "Firmware req: Buffer EOS.");
+
+	ret = fw_send_msg(session, &msg);
+	if (ret != 0)
+		return ret;
+
+	session->port[MVX_DIR_INPUT].flushed = false;
+
+	return 0;
+}
+
+static int fw_set_qp(struct mvx_session *session, int code, int qp)
+{
+	struct mvx_fw_set_option option;
+	int ret;
+
+	if (qp < 0)
+		return -EINVAL;
+
+	if (qp == 0)
+		return 0;
+
+	option.code = code;
+	option.qp = qp;
+	ret = fw_set_option(session, &option);
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to set QP. code=%d, ret=%d.", code, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int fw_set_roi_regions(struct mvx_session *session, int code, struct mvx_roi_config *roi)
+{
+	struct mvx_fw_set_option option;
+	int ret;
+
+	if (roi->num_roi < 0)
+		return -EINVAL;
+
+	if (roi->num_roi == 0)
+		return 0;
+
+	option.code = code;
+	option.roi_config = *roi;
+	ret = fw_set_option(session, &option);
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to set ROI. code=%d, ret=%d.", code, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int fw_common_setup(struct mvx_session *session)
+{
+	int ret = 0;
+	struct mvx_fw_set_option option;
+	enum mvx_direction dir = get_bitstream_port(session);
+
+	if ((session->port[dir].format == MVX_FORMAT_VP8 || session->port[dir].format == MVX_FORMAT_VP9) &&
+	    session->nalu_format != MVX_NALU_FORMAT_ONE_NALU_PER_BUFFER) {
+		session->nalu_format = MVX_NALU_FORMAT_UNDEFINED;
+	}
+
+	if (session->nalu_format != MVX_NALU_FORMAT_UNDEFINED) {
+		option.code = MVX_FW_SET_NALU_FORMAT;
+		option.nalu_format = session->nalu_format;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set NALU format.");
+			return ret;
+		}
+	}
+
+	if (session->stream_escaping != MVX_TRI_UNSET) {
+		option.code = MVX_FW_SET_STREAM_ESCAPING;
+		option.stream_escaping = session->stream_escaping;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set stream escaping.");
+			return ret;
+		}
+	}
+
+	if (session->enable_profiling) {
+		MVX_SESSION_WARN(session, "[Debug]set MVX_FW_SET_INDEX_PROFILING to read profiling data.");
+		session->bus_read_bytes_total = 0;
+		session->bus_write_bytes_total = 0;
+		option.code = MVX_FW_SET_INDEX_PROFILING;
+		option.index_profiling = 1;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set set index profiling");
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+/* JPEG standard, Annex K */
+static const uint8_t qtbl_chroma_ref[MVX_FW_QUANT_LEN] = {
+	17, 18, 24, 47, 99, 99, 99, 99,
+	18, 21, 26, 66, 99, 99, 99, 99,
+	24, 26, 56, 99, 99, 99, 99, 99,
+	47, 66, 99, 99, 99, 99, 99, 99,
+	99, 99, 99, 99, 99, 99, 99, 99,
+	99, 99, 99, 99, 99, 99, 99, 99,
+	99, 99, 99, 99, 99, 99, 99, 99,
+	99, 99, 99, 99, 99, 99, 99, 99
+};
+
+static const uint8_t qtbl_luma_ref[MVX_FW_QUANT_LEN] = {
+	16, 11, 10, 16, 24,  40,  51,  61,
+	12, 12, 14, 19, 26,  58,  60,  55,
+	14, 13, 16, 24, 40,  57,  69,  56,
+	14, 17, 22, 29, 51,  87,  80,  62,
+	18, 22, 37, 56, 68,  109, 103, 77,
+	24, 35, 55, 64, 81,  104, 113, 92,
+	49, 64, 78, 87, 103, 121, 120, 101,
+	72, 92, 95, 98, 112, 100, 103, 99
+};
+
+void generate_quant_tbl(int quality,
+			const uint8_t qtbl_ref[MVX_FW_QUANT_LEN],
+			uint8_t qtbl[MVX_FW_QUANT_LEN])
+{
+	int i;
+	int q;
+
+	q = (quality < 50) ? (5000 / quality) : (200 - 2 * quality);
+
+	for (i = 0; i < MVX_FW_QUANT_LEN; ++i) {
+		qtbl[i] = ((qtbl_ref[i] * q) + 50) / 100;
+		qtbl[i] = min_t(int, qtbl[i], 255);
+		qtbl[i] = max_t(int, qtbl[i], 1);
+	}
+}
+
+static int fw_encoder_setup(struct mvx_session *session)
+{
+	int ret;
+	enum mvx_format codec;
+	struct mvx_fw_set_option option;
+	enum mvx_direction dir;
+
+	dir = get_bitstream_port(session);
+	codec = session->port[dir].format;
+
+	if (session->profile[codec] != MVX_PROFILE_NONE) {
+		option.code = MVX_FW_SET_PROFILE_LEVEL;
+		option.profile_level.profile = session->profile[codec];
+		option.profile_level.level = session->level[codec];
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set profile/level.");
+			return ret;
+		}
+	}
+
+	if (codec != MVX_FORMAT_JPEG) {
+		option.code = MVX_FW_SET_FRAME_RATE;
+		option.frame_rate = session->frame_rate;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to put frame rate.");
+			return ret;
+		}
+
+		if (session->rc_type) {
+			option.code = MVX_FW_SET_RATE_CONTROL;
+			option.rate_control.target_bitrate = session->rc_type ? session->target_bitrate : 0;
+			option.rate_control.rate_control_mode = session->rc_type;
+			if (session->rc_type == MVX_OPT_RATE_CONTROL_MODE_C_VARIABLE) {
+				option.rate_control.maximum_bitrate = session->maximum_bitrate;
+			}
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to put target bitrate.");
+				return ret;
+			}
+		}
+
+		if (session->rc_enabled != false) {
+			if (session->qp[codec].min < session->qp[codec].max) {
+				option.code = MVX_FW_SET_QP_RANGE;
+				option.qp_range.min = session->qp[codec].min;
+				option.qp_range.max = session->qp[codec].max;
+				ret = fw_set_option(session, &option);
+				if (ret != 0) {
+					MVX_SESSION_WARN(session, "Failed to set qp range.");
+					return ret;
+				}
+			}
+		} else {
+			ret = fw_set_qp(session, MVX_FW_SET_QP_I, session->qp[codec].i_frame);
+			if (ret != 0)
+				return ret;
+
+			ret = fw_set_qp(session, MVX_FW_SET_QP_P, session->qp[codec].p_frame);
+			if (ret != 0)
+				return ret;
+
+			ret = fw_set_qp(session, MVX_FW_SET_QP_B, session->qp[codec].b_frame);
+			if (ret != 0)
+				return ret;
+		}
+
+		if (session->p_frames >= 0) {
+			option.code = MVX_FW_SET_P_FRAMES;
+			option.pb_frames = session->p_frames;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set P frames.");
+				return ret;
+			}
+		}
+
+		if (session->b_frames != 0) {
+			option.code = MVX_FW_SET_B_FRAMES;
+			option.pb_frames = session->b_frames;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set B frames.");
+				return ret;
+			}
+		}
+
+		if (session->gop_type != MVX_GOP_TYPE_NONE) {
+			option.code = MVX_FW_SET_GOP_TYPE;
+			option.gop_type = session->gop_type;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set GOP type.");
+				return ret;
+			}
+		}
+
+		if (session->cyclic_intra_refresh_mb != 0) {
+			option.code = MVX_FW_SET_INTRA_MB_REFRESH;
+			option.intra_mb_refresh = session->cyclic_intra_refresh_mb;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set cyclic intra refresh Mb.");
+				return ret;
+			}
+		}
+
+		if (session->constr_ipred != MVX_TRI_UNSET && (codec == MVX_FORMAT_H264 || codec == MVX_FORMAT_HEVC)) {
+			option.code = MVX_FW_SET_CONSTR_IPRED;
+			option.constr_ipred = session->constr_ipred;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set constr ipred.");
+				return ret;
+			}
+		}
+	}
+
+	if (codec == MVX_FORMAT_HEVC) {
+		if (session->entropy_sync != MVX_TRI_UNSET) {
+			option.code = MVX_FW_SET_ENTROPY_SYNC;
+			option.entropy_sync = session->entropy_sync;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set entropy sync.");
+				return ret;
+			}
+		}
+
+		if (session->temporal_mvp != MVX_TRI_UNSET) {
+			option.code = MVX_FW_SET_TEMPORAL_MVP;
+			option.temporal_mvp = session->temporal_mvp;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set temporal mvp.");
+				return ret;
+			}
+		}
+
+		option.code = MVX_FW_SET_MIN_LUMA_CB_SIZE;
+		option.min_luma_cb_size = session->min_luma_cb_size;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set min luma cb size.");
+			return ret;
+		}
+	}
+
+	if ((codec == MVX_FORMAT_HEVC || codec == MVX_FORMAT_VP9) &&
+	    (session->tile_rows != 0 || session->tile_cols != 0)) {
+		option.code = MVX_FW_SET_TILES;
+		option.tile.rows = session->tile_rows;
+		option.tile.cols = session->tile_cols;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set tile dims.");
+			return ret;
+		}
+	}
+
+	if (session->entropy_mode != MVX_ENTROPY_MODE_NONE && codec == MVX_FORMAT_H264) {
+		option.code = MVX_FW_SET_ENTROPY_MODE;
+		option.entropy_mode = session->entropy_mode;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set entropy mode.");
+			return ret;
+		}
+	}
+
+	if (codec == MVX_FORMAT_H264 || codec == MVX_FORMAT_HEVC) {
+		option.code = MVX_FW_SET_SLICE_SPACING_MB;
+		if (session->multi_slice_mode == MVX_MULTI_SLICE_MODE_SINGLE)
+			option.slice_spacing_mb = 0;
+		else
+			option.slice_spacing_mb = session->multi_slice_max_mb;
+
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set slice spacing.");
+			return ret;
+		}
+
+		option.code = MVX_FW_SET_CABAC_INIT_IDC;
+		option.cabac_init_idc = session->cabac_init_idc;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set CABAC init IDC.");
+			return ret;
+		}
+		if (session->crop_left != 0) {
+			option.code = MVX_FW_SET_CROP_LEFT;
+			option.crop_left = session->crop_left;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set crop left");
+				return ret;
+			}
+		}
+		if (session->crop_right != 0) {
+			option.code = MVX_FW_SET_CROP_RIGHT;
+			option.crop_right = session->crop_right;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set crop right");
+				return ret;
+			}
+		}
+		if (session->crop_top != 0) {
+			option.code = MVX_FW_SET_CROP_TOP;
+			option.crop_top = session->crop_top;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set crop top");
+				return ret;
+			}
+		}
+		if (session->crop_bottom != 0) {
+			option.code = MVX_FW_SET_CROP_BOTTOM;
+			option.crop_bottom = session->crop_bottom;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set crop bottom");
+				return ret;
+			}
+		}
+		if (session->nHRDBufsize != 0) {
+			option.code = MVX_FW_SET_HRD_BUF_SIZE;
+			option.nHRDBufsize = session->nHRDBufsize;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set HRD Buffer Size");
+				return ret;
+			}
+		}
+		if (session->color_desc.range != 0 || session->color_desc.matrix != 0 ||
+		    session->color_desc.primaries != 0 || session->color_desc.transfer != 0 ||
+		    session->color_desc.sar_height != 0 || session->color_desc.sar_width != 0 ||
+		    session->color_desc.aspect_ratio_idc != 0) {
+			struct mvx_fw_set_option option;
+
+			option.code = MVX_FW_SET_COLOUR_DESC;
+			option.colour_desc = session->color_desc;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set vui colour description");
+				return ret;
+			}
+		}
+
+		if (session->sei_userdata.flags) {
+			option.code = MVX_FW_SET_SEI_USERDATA;
+			option.userdata = session->sei_userdata;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set sei userdata");
+				return ret;
+			}
+		}
+
+		if (session->mvx_ltr.mode != 0 || session->mvx_ltr.period != 0) {
+			option.code = MVX_FW_SET_LONG_TERM_REF;
+			option.ltr.mode = session->mvx_ltr.mode;
+			option.ltr.period = session->mvx_ltr.period;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set ltr mode/period");
+				return ret;
+			}
+		}
+	}
+
+	if (codec == MVX_FORMAT_VP9) {
+		MVX_SESSION_INFO(session, "VP9 option!");
+		option.code = MVX_FW_SET_VP9_PROB_UPDATE;
+		option.vp9_prob_update = session->vp9_prob_update;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set VP9 prob update mode.");
+			return ret;
+		}
+	}
+
+	if (session->mv_h_search_range != 0 && session->mv_v_search_range != 0) {
+		option.code = MVX_FW_SET_MV_SEARCH_RANGE;
+		option.mv.x = session->mv_h_search_range;
+		option.mv.y = session->mv_v_search_range;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set motion vector search range.");
+			return ret;
+		}
+	}
+
+	if (session->bitdepth_chroma != 0 && session->bitdepth_luma != 0) {
+		option.code = MVX_FW_SET_BITDEPTH;
+		option.bitdepth.chroma = session->bitdepth_chroma;
+		option.bitdepth.luma = session->bitdepth_luma;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set bitdepth.");
+			return ret;
+		}
+	}
+
+	if (session->force_chroma_format != 0) {
+		option.code = MVX_FW_SET_CHROMA_FORMAT;
+		option.chroma_format = session->force_chroma_format;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set chroma format.");
+			return ret;
+		}
+	}
+
+	if (mvx_is_rgb(session->port[MVX_DIR_INPUT].format) != false) {
+		option.code = MVX_FW_SET_RGB_TO_YUV_MODE;
+		if (session->color_desc.primaries == MVX_FW_PRIMARIES_BT709) {
+			session->rgb_to_yuv = (session->color_desc.range == MVX_FW_RANGE_FULL)
+						      ? MVX_RGB_TO_YUV_MODE_BT709_FULL
+						      : MVX_RGB_TO_YUV_MODE_BT709_STUDIO;
+		} else if ((session->color_desc.primaries == MVX_FW_PRIMARIES_BT601_625) ||
+			   (session->color_desc.primaries == MVX_FW_PRIMARIES_BT601_525)) {
+			session->rgb_to_yuv = (session->color_desc.range == MVX_FW_RANGE_FULL)
+						      ? MVX_RGB_TO_YUV_MODE_BT601_FULL
+						      : MVX_RGB_TO_YUV_MODE_BT601_STUDIO;
+		}
+		option.rgb_to_yuv_mode = session->rgb_to_yuv;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set RGB to YUV mode.");
+			return ret;
+		}
+	}
+
+	if (session->band_limit != 0) {
+		option.code = MVX_FW_SET_BAND_LIMIT;
+		option.band_limit = session->band_limit;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set bandwidth limit.");
+			return ret;
+		}
+	}
+
+	if (codec == MVX_FORMAT_JPEG) {
+		if (session->resync_interval >= 0) {
+			option.code = MVX_FW_SET_RESYNC_INTERVAL;
+			option.resync_interval = session->resync_interval;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set resync interval.");
+				return ret;
+			}
+		}
+
+		if (session->jpeg_quality != 0) {
+			uint8_t qtbl_chroma[MVX_FW_QUANT_LEN];
+			uint8_t qtbl_luma[MVX_FW_QUANT_LEN];
+
+			generate_quant_tbl(session->jpeg_quality, qtbl_chroma_ref, qtbl_chroma);
+			generate_quant_tbl(session->jpeg_quality, qtbl_luma_ref, qtbl_luma);
+			option.code = MVX_FW_SET_QUANT_TABLE;
+			option.quant_tbl.chroma = qtbl_chroma;
+			option.quant_tbl.luma = qtbl_luma;
+			ret = fw_set_option(session, &option);
+			if (ret != 0) {
+				MVX_SESSION_WARN(session, "Failed to set quantization table.");
+				return ret;
+			}
+		}
+	}
+
+	ret = fw_common_setup(session);
+
+	return ret;
+}
+
+static int fw_decoder_setup(struct mvx_session *session)
+{
+	int ret;
+	struct mvx_fw_set_option option;
+
+	enum mvx_format codec;
+	enum mvx_direction dir;
+
+	dir = get_bitstream_port(session);
+	codec = session->port[dir].format;
+
+	if (codec == MVX_FORMAT_VC1 && session->profile[codec] != MVX_PROFILE_NONE) {
+		option.code = MVX_FW_SET_PROFILE_LEVEL;
+		option.profile_level.profile = session->profile[codec];
+		option.profile_level.level = session->level[codec];
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set profile/level.");
+			return ret;
+		}
+	}
+
+	if (session->ignore_stream_headers != MVX_TRI_UNSET) {
+		option.code = MVX_FW_SET_IGNORE_STREAM_HEADERS;
+		option.ignore_stream_headers = session->ignore_stream_headers;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set ignore stream headers.");
+			return ret;
+		}
+	}
+
+	if (session->frame_reordering != MVX_TRI_UNSET) {
+		option.code = MVX_FW_SET_FRAME_REORDERING;
+		option.frame_reordering = session->frame_reordering;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set frame reordering.");
+			return ret;
+		}
+	}
+
+	if (session->intbuf_size != 0) {
+		option.code = MVX_FW_SET_INTBUF_SIZE;
+		option.intbuf_size = session->intbuf_size;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set internal buffer size.");
+			return ret;
+		}
+	}
+
+	if (session->dsl_frame.width != 0 && session->dsl_frame.height != 0) {
+		option.code = MVX_FW_SET_DSL_FRAME;
+		option.dsl_frame.width = session->dsl_frame.width;
+		option.dsl_frame.height = session->dsl_frame.height;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set DSL frame width/height.");
+			return ret;
+		}
+	}
+
+	if (session->dsl_pos_mode >= 0 && session->dsl_pos_mode <= 2) {
+		option.code = MVX_FW_SET_DSL_MODE;
+		option.dsl_pos_mode = session->dsl_pos_mode;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to set DSL mode.");
+			return ret;
+		}
+	}
+
+	ret = fw_common_setup(session);
+
+	return ret;
+}
+
+static int fw_initial_setup(struct mvx_session *session)
+{
+	int ret;
+	enum mvx_direction dir;
+	enum mvx_format codec;
+	struct mvx_fw_set_option option;
+
+	MVX_SESSION_INFO(session, "Firmware initial setup.");
+
+#ifdef MVX_FW_DEBUG_ENABLE
+	fw_set_debug(session, 5);
+#endif
+
+	option.code = MVX_FW_SET_WATCHDOG_TIMEOUT;
+	option.watchdog_timeout = fw_watchdog_timeout;
+	ret = fw_set_option(session, &option);
+	if (ret != 0)
+		return ret;
+
+	dir = get_bitstream_port(session);
+	codec = session->port[dir].format;
+
+	ret = fw_job(session, 1);
+	if (ret != 0)
+		return ret;
+
+	if (is_encoder(session))
+		ret = fw_encoder_setup(session);
+	else
+		ret = fw_decoder_setup(session);
+
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to perform initial setup.\n");
+		return ret;
+	}
+
+	ret = fw_state_change(session, MVX_FW_STATE_RUNNING);
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to queue state change.");
+		return ret;
+	}
+
+	ret = fw_ping(session);
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "Failed to put ping message.");
+		send_event_error(session, ret);
+		return ret;
+	}
+
+	return ret;
+}
+
+static void ddr_qos_request_update(struct mvx_session *session)
+{
+	struct estimate_ddr_input ddr_input;
+	struct estimate_ddr_output ddr_output;
+
+	ddr_input.isEnc = is_encoder(session);
+	ddr_input.fps = session->frame_rate >> 16;
+
+	if (ddr_input.isEnc) {
+		ddr_input.isAFBC = mvx_is_afbc(session->port[MVX_DIR_INPUT].format);
+		ddr_input.width = session->port[MVX_DIR_INPUT].width;
+		ddr_input.height = session->port[MVX_DIR_INPUT].height;
+	} else {
+		ddr_input.isAFBC = mvx_is_afbc(session->port[MVX_DIR_OUTPUT].format);
+		ddr_input.width = session->port[MVX_DIR_OUTPUT].width;
+		ddr_input.height = session->port[MVX_DIR_OUTPUT].height;
+	}
+
+	mvx_dvfs_estimate_ddr_bandwidth(&ddr_input, &ddr_output);
+
+	session->estimated_ddr_read_throughput = (uint32_t)ddr_output.estimated_read;
+	session->estimated_ddr_write_throughput = (uint32_t)ddr_output.estimated_write;
+
+	mvx_dvfs_session_update_ddr_qos(session, session->estimated_ddr_read_throughput,
+					session->estimated_ddr_write_throughput);
+}
+
+/**
+ * map_buffer() - Memory map buffer to MVE address space.
+ *
+ * Return 0 on success, else error code.
+ */
+static int map_buffer(struct mvx_session *session, enum mvx_direction dir, struct mvx_buffer *buf)
+{
+	mvx_mmu_va begin;
+	mvx_mmu_va end;
+	enum mvx_fw_region region;
+	int ret;
+
+	if (mvx_is_bitstream(session->port[dir].format))
+		region = MVX_FW_REGION_PROTECTED;
+	else if (mvx_is_frame(session->port[dir].format))
+		region = MVX_FW_REGION_FRAMEBUF;
+	else
+		return -EINVAL;
+
+	ret = session->fw.ops.get_region(region, &begin, &end);
+	if (ret != 0)
+		return ret;
+
+	ret = mvx_buffer_map(buf, begin, end);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+static int queue_roi_regions(struct mvx_session *session, struct mvx_roi_config *roi_cfg)
+{
+	int ret = 0;
+	if (roi_cfg->qp_present) {
+		ret = fw_set_qp(session, MVX_FW_SET_QP_REGION, roi_cfg->qp);
+	}
+	if (roi_cfg->roi_present) {
+		ret = fw_set_roi_regions(session, MVX_FW_SET_ROI_REGIONS, roi_cfg);
+	}
+	return ret;
+}
+
+static int queue_qp_epr(struct mvx_session *session, int *qp)
+{
+	int ret = 0;
+	ret = fw_set_qp(session, MVX_FW_SET_QP_REGION, *qp);
+
+	return ret;
+}
+
+/**
+ * queue_buffer() - Put buffer to firmware queue.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int queue_buffer(struct mvx_session *session, enum mvx_direction dir, struct mvx_buffer *buf)
+{
+	struct mvx_session_port *port = &session->port[dir];
+	struct mvx_fw_msg msg;
+	int i;
+
+	/*
+	 * Vb2 cannot allocate buffers with bidirectional mapping, therefore
+	 * proper direction should be set.
+	 */
+	enum dma_data_direction dma_dir = (dir == MVX_DIR_OUTPUT) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+
+	int ret;
+	bool mapped = mvx_buffer_is_mapped(buf);
+
+	if (dir == MVX_DIR_OUTPUT) {
+		port->scaling_shift = (buf->flags & MVX_BUFFER_FRAME_FLAG_SCALING_MASK) >> 14;
+	}
+	if (mvx_buffer_is_mapped(buf) == false) {
+		ret = map_buffer(session, dir, buf);
+		if (ret != 0)
+			return ret;
+	}
+	buf->flags &= ~MVX_BUFFER_FRAME_NEED_REALLOC;
+
+	if (dir == MVX_DIR_OUTPUT && port->isreallocting == true) {
+		buf->flags |= MVX_BUFFER_FRAME_NEED_REALLOC;
+		return -EAGAIN;
+	}
+
+	if (dir == MVX_DIR_OUTPUT && port->buffer_allocated < port->buffer_min) {
+		buf->flags |= MVX_BUFFER_FRAME_NEED_REALLOC;
+		return -EAGAIN;
+	}
+
+	if (dir == MVX_DIR_INPUT && (buf->flags & MVX_BUFFER_EOS) != 0) {
+		session->eos_queued = true;
+	}
+
+	if (dir == MVX_DIR_OUTPUT) {
+		session->keep_freq_high = false;
+	}
+
+	/*
+	 * Update frame dimensions. They might have changed due to a resolution
+	 * change.
+	 */
+	if (mvx_is_afbc(port->format) != false) {
+		port->afbc_width = DIV_ROUND_UP(port->width, 16 << (!!(buf->flags & MVX_BUFFER_AFBC_32X8_SUPERBLOCK)));
+		ret = mvx_buffer_afbc_set(buf, port->format, port->width, port->height, port->afbc_width, port->size[0],
+					  port->interlaced);
+		if (ret != 0)
+			return ret;
+	} else if (mvx_is_frame(port->format) != false) {
+		ret = mvx_buffer_frame_set(buf, port->format, port->width, port->height, port->stride, port->size,
+					   port->interlaced);
+		if (ret != 0)
+			return ret;
+	}
+
+	if (mapped && ((dir == MVX_DIR_OUTPUT) || (dir == MVX_DIR_INPUT && mvx_is_frame(port->format) &&
+						   (buf->flags & MVX_BUFFER_FLAG_DISABLE_CACHE_MAINTENANCE)))) {
+		/*
+		1. no need to do cache invalidate each time for output buffer,
+				only invalidate cache when buffer is mapped
+		2. no need to do cache clean for input buffer, if there is
+			on cpu write/read usage.
+		*/
+	} else {
+		ret = mvx_buffer_synch(buf, dma_dir);
+	}
+	if (ret != 0)
+		return ret;
+
+	msg.code = MVX_FW_CODE_BUFFER;
+	msg.buf = buf;
+
+	MVX_SESSION_INFO(session, "Firmware req: Buffer. dir=%u, len=[%u, %u, %u], flags=0x%08x, eos=%u, interlace=%u",
+			 buf->dir, buf->planes[0].filled, buf->planes[1].filled, buf->planes[2].filled, buf->flags,
+			 (buf->flags & MVX_BUFFER_EOS) != 0, (buf->flags & MVX_BUFFER_INTERLACE) != 0);
+
+	ret = session->fw.ops.put_message(&session->fw, &msg);
+	if (ret != 0)
+		goto send_error;
+
+	port->buffer_count++;
+
+	if (dir == MVX_DIR_OUTPUT && mvx_is_frame(buf->format)) {
+		for (i = 0; i < buf->nplanes; i++) {
+			if (port->buffer_size[i] == 0)
+				port->buffer_size[i] = mvx_buffer_size(buf, i);
+		}
+	}
+
+	port->flushed = false;
+	if (dir == MVX_DIR_OUTPUT && port->isreallocting == true) {
+		port->isreallocting = false;
+	}
+	ret = send_irq(session);
+	if (ret != 0)
+		goto send_error;
+
+	return 0;
+
+send_error:
+	send_event_error(session, ret);
+	return ret;
+}
+
+/**
+ * queue_pending_buffers() - Queue pending buffers.
+ *
+ * Buffer that are queued when the port is still stream off will be put in the
+ * pending queue. Once both input- and output ports are stream on the pending
+ * buffers will be forwarded to the firmware.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int queue_pending_buffers(struct mvx_session *session, enum mvx_direction dir)
+{
+	struct mvx_buffer *buf;
+	struct mvx_buffer *tmp;
+	int roi_config_num = 0;
+	int roi_config_index = 0;
+	int qp_num = 0;
+	int qp_index = 0;
+	struct mvx_roi_config roi_config;
+	int ret = 0;
+
+	if (dir == MVX_DIR_INPUT && session->port[dir].roi_config_num > 0) {
+		roi_config_num = session->port[dir].roi_config_num;
+	}
+	if (dir == MVX_DIR_INPUT && session->port[dir].qp_num > 0) {
+		qp_num = session->port[dir].qp_num;
+	}
+	list_for_each_entry_safe(buf, tmp, &session->port[dir].buffer_queue, head)
+	{
+		if ((buf->flags & MVX_BUFFER_FRAME_FLAG_ROI) == MVX_BUFFER_FRAME_FLAG_ROI &&
+		    roi_config_index < roi_config_num) {
+			roi_config = session->port[dir].roi_config_queue[roi_config_index];
+			ret = queue_roi_regions(session, &roi_config);
+			roi_config_index++;
+		}
+		if ((buf->flags & MVX_BUFFER_FRAME_FLAG_GENERAL) == MVX_BUFFER_FRAME_FLAG_GENERAL &&
+		    qp_index < qp_num) {
+			ret = queue_qp_epr(session, &session->port[dir].qp_queue[qp_index]);
+			qp_index++;
+		}
+		ret = queue_buffer(session, dir, buf);
+		if ((buf->flags & MVX_BUFFER_FRAME_NEED_REALLOC) == MVX_BUFFER_FRAME_NEED_REALLOC) {
+			session->event(session, MVX_SESSION_EVENT_BUFFER, buf);
+		} else if (ret != 0) {
+			break;
+		}
+		list_del(&buf->head);
+	}
+
+	session->port[dir].roi_config_num = 0;
+	session->port[dir].qp_num = 0;
+	return ret;
+}
+
+/**
+ * fw_bin_ready() - Complete firmware configuration.
+ *
+ * The firmware binary load has completed and the firmware configuration can
+ * begin.
+ *
+ * If the session is no longer 'stream on' (someone issued 'stream off' before
+ * the firmware load completed) the firmware binary is put back to the cache.
+ *
+ * Else the the client session is registered and the firmware instance is
+ * constructed.
+ */
+static void fw_bin_ready(struct mvx_fw_bin *bin, void *arg, bool same_thread)
+{
+	struct mvx_session *session = arg;
+	int lock_failed = 1;
+	int ret;
+
+	/*
+	 * Only lock the mutex if the firmware binary was loaded by a
+	 * background thread.
+	 */
+	if (same_thread == false) {
+		lock_failed = mutex_lock_interruptible(session->isession.mutex);
+		if (lock_failed != 0) {
+			send_event_error(session, lock_failed);
+			goto put_fw_bin;
+		}
+	}
+
+	/* Return firmware binary if session is no longer 'stream on'. */
+	if (!is_stream_on(session))
+		goto put_fw_bin;
+
+	/* Create client session. */
+	session->isession.ncores = session->client_ops->get_ncores(session->client_ops);
+	session->isession.l0_pte =
+		mvx_mmu_set_pte(MVX_ATTR_PRIVATE, virt_to_phys(session->mmu.page_table), MVX_ACCESS_READ_WRITE);
+
+	session->csession = session->client_ops->register_session(session->client_ops, &session->isession);
+
+	mvx_dvfs_register_session(session, is_encoder(session));
+
+	if (IS_ERR(session->csession)) {
+		ret = PTR_ERR(session->csession);
+		send_event_error(session, ret);
+		goto put_fw_bin;
+	}
+
+	/* Construct the firmware instance. */
+	ret = mvx_fw_factory(&session->fw, bin, &session->mmu, session, session->client_ops, session->csession,
+			     session->isession.ncores, session->dentry);
+	if (ret != 0) {
+		send_event_error(session, ret);
+		goto unregister_csession;
+	}
+
+	session->fw_bin = bin;
+
+	mvx_fw_cache_log(bin, session->csession);
+
+	ret = fw_initial_setup(session);
+	if (ret != 0)
+		goto unregister_csession;
+
+	ret = queue_pending_buffers(session, MVX_DIR_INPUT);
+	if (ret != 0)
+		goto unregister_csession;
+
+	ret = queue_pending_buffers(session, MVX_DIR_OUTPUT);
+	if (ret != 0)
+		goto unregister_csession;
+
+	ret = mvx_session_put(session);
+	if (ret == 0 && lock_failed == 0)
+		mutex_unlock(session->isession.mutex);
+
+	return;
+
+unregister_csession:
+	mvx_dvfs_unregister_session(session);
+	session->client_ops->unregister_session(session->csession);
+	session->csession = NULL;
+
+put_fw_bin:
+	mvx_fw_cache_put(session->cache, bin);
+	session->fw_bin = NULL;
+
+	ret = mvx_session_put(session);
+	if (ret == 0 && lock_failed == 0)
+		mutex_unlock(session->isession.mutex);
+}
+
+static int calc_afbc_size(struct mvx_session *session, enum mvx_format format, unsigned int width, unsigned int height,
+			  bool tiled_headers, bool tiled_body, bool superblock, bool interlaced)
+{
+	static const unsigned int mb_header_size = 16;
+	unsigned int payload_align = 128;
+	unsigned int mb_size;
+	int size;
+
+	/* Calculate width and height in super blocks. */
+	if (superblock != false) {
+		width = DIV_ROUND_UP(width, 32);
+		height = DIV_ROUND_UP(height, 8) + 1;
+	} else {
+		width = DIV_ROUND_UP(width, 16);
+		height = DIV_ROUND_UP(height, 16) + 1;
+	}
+
+	/* Round up size to 8x8 tiles. */
+	if (tiled_headers != false || tiled_body != false) {
+		width = roundup(width, 8);
+		height = roundup(height, 8);
+	}
+
+	switch (format) {
+	case MVX_FORMAT_YUV420_AFBC_8:
+		mb_size = 384;
+		break;
+	case MVX_FORMAT_YUV420_AFBC_10:
+		mb_size = 480;
+		break;
+	case MVX_FORMAT_YUV422_AFBC_8:
+		mb_size = 512;
+		break;
+	case MVX_FORMAT_YUV422_AFBC_10:
+		mb_size = 656;
+		break;
+	default:
+		MVX_SESSION_WARN(session, "Unsupported AFBC format. format=%u.", format);
+		return -EINVAL;
+	}
+
+	/* Round up tiled body to 128 byte boundary. */
+	if (tiled_body != false)
+		mb_size = roundup(mb_size, payload_align);
+
+	if (interlaced != false)
+		height = DIV_ROUND_UP(height, 2);
+
+	/* Calculate size of AFBC makroblock headers. */
+	size = roundup(width * height * mb_header_size, payload_align);
+	size += roundup(width * height * mb_size, payload_align);
+
+	if (interlaced != false)
+		size *= 2;
+
+	return size;
+}
+
+static int try_format(struct mvx_session *session, enum mvx_direction dir, enum mvx_format format, unsigned int *width,
+		      unsigned int *height, uint8_t *nplanes, unsigned int *stride, unsigned int *size,
+		      bool *interlaced)
+{
+	int ret = 0;
+
+	/* Limit width and height to 8k. */
+	if (*width == ((unsigned int)(-1)))
+		*width = 0;
+	if (*height == ((unsigned int)(-1)))
+		*height = 0;
+	*width = min_t(unsigned int, *width, 8192);
+	*height = min_t(unsigned int, *height, 8192);
+
+	/* Stream dimensions are dictated by the input port. */
+	if (dir == MVX_DIR_OUTPUT) {
+		*width = session->port[MVX_DIR_INPUT].width >> session->port[MVX_DIR_OUTPUT].scaling_shift;
+		*height = session->port[MVX_DIR_INPUT].height >> session->port[MVX_DIR_OUTPUT].scaling_shift;
+	}
+	if (session->dsl_frame.width != 0 && session->dsl_frame.height != 0) {
+		*width = session->dsl_frame.width;
+		*height = session->dsl_frame.height;
+	} else if (dir == MVX_DIR_OUTPUT && (session->dsl_ratio.hor != 1 || session->dsl_ratio.ver != 1)) {
+		*width = session->port[MVX_DIR_INPUT].width / session->dsl_ratio.hor;
+		*height = session->port[MVX_DIR_INPUT].height / session->dsl_ratio.ver;
+		*width &= ~(1);
+		*height &= ~(1);
+	}
+	/* Interlaced input is not supported by the firmware. */
+	if (dir == MVX_DIR_INPUT)
+		*interlaced = false;
+
+	if (mvx_is_afbc(format) != false) {
+		unsigned int afbc_alloc_bytes = session->port[dir].afbc_alloc_bytes;
+		if (*nplanes <= 0)
+			size[0] = 0;
+
+		if (dir == MVX_DIR_INPUT) {
+			/* it is basically a worst-case calcualtion based on a size rounded up to tile size*/
+			int s1 = calc_afbc_size(session, format, *width, *height, false, false,
+						false, //*height, false, false, false,
+						*interlaced);
+			int s2 = calc_afbc_size(session, format, *width, *height, false, false,
+						true, //*height, false, false, false,
+						*interlaced);
+			int s = max_t(unsigned int, s1, s2);
+			if (s < 0)
+				return s;
+
+			size[0] = max_t(unsigned int, size[0], s);
+		}
+
+		if (*interlaced != false)
+			afbc_alloc_bytes *= 2;
+
+		size[0] = max_t(unsigned int, size[0], afbc_alloc_bytes);
+		size[0] = roundup(size[0], PAGE_SIZE);
+
+		*nplanes = 1;
+	} else if (mvx_is_frame(format) != false) {
+		ret = mvx_buffer_frame_dim(format, *width, *height, nplanes, stride, size);
+	} else {
+		/*
+		 * For compressed formats the size should be the maximum number
+		 * of bytes an image is expected to become. This is calculated
+		 * as width * height * 2 B/px / 2. Size should be at least one
+		 * page.
+		 */
+
+		stride[0] = 0;
+
+		if (*nplanes <= 0)
+			size[0] = 0;
+
+		size[0] = max_t(unsigned int, size[0], PAGE_SIZE);
+		size[0] = max_t(unsigned int, size[0], (*width) * (*height));
+		size[0] = roundup(size[0], PAGE_SIZE);
+
+		*nplanes = 1;
+	}
+
+	return ret;
+}
+
+static void watchdog_work(struct work_struct *work)
+{
+	struct mvx_session *session = container_of(work, struct mvx_session, watchdog_work);
+	int ret;
+
+	mutex_lock(session->isession.mutex);
+	session->watchdog_count++;
+
+	MVX_SESSION_WARN(session, "Watchdog timeout. count=%u. is_encoder=%d. in_fmt=%d, out_fmt=%d",
+			 session->watchdog_count, is_encoder(session), session->port[MVX_DIR_INPUT].format,
+			 session->port[MVX_DIR_OUTPUT].format);
+
+	/* Print debug information. */
+	print_debug(session);
+
+	if (session->watchdog_count == 1) {
+		/* Request firmware to dump its state. */
+		fw_dump(session);
+
+		/* Restart watchdog. */
+		watchdog_start(session, 3000);
+	} else {
+		send_event_error(session, -ETIME);
+	}
+
+	ret = kref_put(&session->isession.kref, session->isession.release);
+	if (ret != 0)
+		return;
+
+	mutex_unlock(session->isession.mutex);
+}
+
+static void watchdog_timeout(struct timer_list *timer)
+{
+	struct mvx_session *session = container_of(timer, struct mvx_session, watchdog_timer);
+
+	queue_work(system_unbound_wq, &session->watchdog_work);
+}
+
+#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE
+static void watchdog_timeout_legacy(unsigned long data) { watchdog_timeout((struct timer_list *)data); }
+
+#endif
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_session_construct(struct mvx_session *session, struct device *dev, struct mvx_client_ops *client_ops,
+			  struct mvx_fw_cache *cache, struct mutex *mutex,
+			  void (*destructor)(struct mvx_session *session),
+			  void (*event)(struct mvx_session *session, enum mvx_session_event event, void *arg),
+			  struct dentry *dentry)
+{
+	int i;
+	int ret;
+
+	if (event == NULL || destructor == NULL)
+		return -EINVAL;
+
+	memset(session, 0, sizeof(*session));
+	session->dev = dev;
+	session->client_ops = client_ops;
+	session->cache = cache;
+	kref_init(&session->isession.kref);
+	session->isession.release = session_destructor;
+	session->isession.mutex = mutex;
+	session->destructor = destructor;
+	session->event = event;
+	session->fw_event.fw_bin_ready = fw_bin_ready;
+	session->fw_event.arg = session;
+	session->fw_state = MVX_FW_STATE_STOPPED;
+	init_waitqueue_head(&session->waitq);
+	session->dentry = dentry;
+	session->port[MVX_DIR_INPUT].buffer_min = 1;
+	session->port[MVX_DIR_OUTPUT].buffer_min = 1;
+	session->port[MVX_DIR_OUTPUT].seq_param.afbc.buffers_min = 1;
+	session->port[MVX_DIR_OUTPUT].seq_param.planar.buffers_min = 1;
+	session->port[MVX_DIR_INPUT].buffer_allocated = 1;
+	session->port[MVX_DIR_OUTPUT].buffer_allocated = 1;
+	session->port[MVX_DIR_INPUT].scaling_shift = 0;
+	session->port[MVX_DIR_OUTPUT].scaling_shift = 0;
+	session->stream_escaping = MVX_TRI_UNSET;
+	session->ignore_stream_headers = MVX_TRI_UNSET;
+	session->frame_reordering = MVX_TRI_UNSET;
+	session->constr_ipred = MVX_TRI_UNSET;
+	session->entropy_sync = MVX_TRI_UNSET;
+	session->temporal_mvp = MVX_TRI_UNSET;
+	session->resync_interval = -1;
+	session->port[MVX_DIR_OUTPUT].roi_config_num = 0;
+	session->port[MVX_DIR_INPUT].roi_config_num = 0;
+	session->port[MVX_DIR_OUTPUT].qp_num = 0;
+	session->port[MVX_DIR_INPUT].qp_num = 0;
+	session->crop_left = 0;
+	session->crop_right = 0;
+	session->crop_top = 0;
+	session->crop_bottom = 0;
+	session->dsl_ratio.hor = 1;
+	session->dsl_ratio.ver = 1;
+	session->dsl_pos_mode = -1; // disable by default
+	session->estimated_ddr_read_throughput = 0;
+	session->estimated_ddr_write_throughput = 0;
+	session->port[MVX_DIR_OUTPUT].buffer_on_hold_count = 0;
+	session->port[MVX_DIR_OUTPUT].pending_buffer_on_hold_count = 0;
+	session->port[MVX_DIR_OUTPUT].isallocparam = false;
+	session->eos_queued = false;
+	session->keep_freq_high = true;
+	session->is_suspend = false;
+	for (i = 0; i < MVX_BUFFER_NPLANES; i++) {
+		session->port[MVX_DIR_OUTPUT].buffer_size[i] = 0;
+	}
+	session->port[MVX_DIR_OUTPUT].buffer_count = 0;
+	session->port[MVX_DIR_INPUT].buffer_count = 0;
+	session->watchdog_count = 0;
+	session->watchdog_timeout = session_watchdog_timeout;
+	session->frame_id = 0;
+
+	INIT_LIST_HEAD(&session->buffer_corrupt_queue);
+	ret = mvx_mmu_construct(&session->mmu, session->dev);
+	if (ret != 0)
+		return ret;
+
+	for (i = 0; i < MVX_DIR_MAX; i++)
+		INIT_LIST_HEAD(&session->port[i].buffer_queue);
+
+#if KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
+	timer_setup(&session->watchdog_timer, watchdog_timeout, 0);
+#else
+	setup_timer(&session->watchdog_timer, watchdog_timeout_legacy, (uintptr_t)&session->watchdog_timer);
+#endif
+	INIT_WORK(&session->watchdog_work, watchdog_work);
+
+	return 0;
+}
+
+void mvx_session_destruct(struct mvx_session *session)
+{
+	/* Destruct the session object. */
+	struct mvx_corrupt_buffer *corrupt_buf;
+	struct mvx_corrupt_buffer *tmp;
+
+	if (session->enable_profiling && session->frame_id > 0) {
+		MVX_SESSION_WARN(session, "[Debug] Destroy session. is_encoder=%d, frame_count=%d", is_encoder(session),
+				 session->frame_id);
+		MVX_SESSION_WARN(session,
+				 "[Debug] bus_write_bytes_total=%lu. bus_read_bytes_total=%lu. avg_write_bw=%lu, "
+				 "avg_read_bw=%lu, 30fps_write_bw=%lu, 30fps_read_bw=%lu",
+				 session->bus_write_bytes_total, session->bus_read_bytes_total,
+				 session->bus_write_bytes_total / session->frame_id,
+				 session->bus_read_bytes_total / session->frame_id,
+				 session->bus_write_bytes_total / session->frame_id * 30,
+				 session->bus_read_bytes_total / session->frame_id * 30);
+	}
+
+	MVX_SESSION_INFO(session, "Destroy session.");
+
+	release_fw_bin(session);
+	mvx_mmu_destruct(&session->mmu);
+	list_for_each_entry_safe(corrupt_buf, tmp, &session->buffer_corrupt_queue, head)
+	{
+		list_del(&corrupt_buf->head);
+		vfree(corrupt_buf);
+	}
+}
+
+void mvx_session_get(struct mvx_session *session) { kref_get(&session->isession.kref); }
+
+int mvx_session_put(struct mvx_session *session)
+{
+	return kref_put(&session->isession.kref, session->isession.release);
+}
+
+void mvx_session_get_formats(struct mvx_session *session, enum mvx_direction dir, uint64_t *formats)
+{
+	uint64_t fw_formats;
+
+	session->client_ops->get_formats(session->client_ops, dir, formats);
+	mvx_fw_cache_get_formats(session->cache, dir, &fw_formats);
+
+	*formats &= fw_formats;
+}
+
+int mvx_session_try_format(struct mvx_session *session, enum mvx_direction dir, enum mvx_format format,
+			   unsigned int *width, unsigned int *height, uint8_t *nplanes, unsigned int *stride,
+			   unsigned int *size, bool *interlaced)
+{
+	return try_format(session, dir, format, width, height, nplanes, stride, size, interlaced);
+}
+
+int mvx_session_set_format(struct mvx_session *session, enum mvx_direction dir, enum mvx_format format,
+			   unsigned int *width, unsigned int *height, uint8_t *nplanes, unsigned int *stride,
+			   unsigned int *size, bool *interlaced)
+{
+	struct mvx_session_port *port = &session->port[dir];
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	if (is_stream_on(session) != false)
+		return -EBUSY;
+
+	ret = try_format(session, dir, format, width, height, nplanes, stride, size, interlaced);
+	if (ret != 0)
+		return ret;
+
+	/*
+	 * If the bitstream format changes, then the firmware binary must be
+	 * released.
+	 */
+	if (mvx_is_bitstream(port->format) != false && format != port->format) {
+		if (IS_ERR(session->fw_bin) != false) {
+			MVX_SESSION_WARN(session, "Can't set format when firmware binary is pending. dir=%d.", dir);
+			return -EINVAL;
+		}
+
+		release_fw_bin(session);
+	}
+
+	/* Update port settings. */
+	port->format = format;
+	port->width = *width;
+	port->height = *height;
+	port->nplanes = *nplanes;
+	port->interlaced = *interlaced;
+	memcpy(port->stride, stride, sizeof(*stride) * MVX_BUFFER_NPLANES);
+	memcpy(port->size, size, sizeof(*size) * MVX_BUFFER_NPLANES);
+
+	/* TODO AFBC width will have to be provided by user space. */
+	if (dir == MVX_DIR_INPUT)
+		port->afbc_width = DIV_ROUND_UP(*width, 16);
+
+	/* Input dimensions dictate output dimensions. */
+	if (dir == MVX_DIR_INPUT) {
+		struct mvx_session_port *p = &session->port[MVX_DIR_OUTPUT];
+		(void)try_format(session, MVX_DIR_OUTPUT, p->format, &p->width, &p->height, &p->nplanes, p->stride,
+				 p->size, &p->interlaced);
+	}
+
+	if (dir == MVX_DIR_OUTPUT) {
+		if (mvx_is_afbc(port->format)) {
+			port->buffer_min = port->seq_param.afbc.buffers_min;
+		} else {
+			port->buffer_min = port->seq_param.planar.buffers_min;
+		}
+	}
+
+	return 0;
+}
+
+int mvx_session_qbuf(struct mvx_session *session, enum mvx_direction dir, struct mvx_buffer *buf)
+{
+	int ret;
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) == false || session->port[dir].is_flushing != false) {
+		list_add_tail(&buf->head, &session->port[dir].buffer_queue);
+		return 0;
+	}
+
+	ret = queue_buffer(session, dir, buf);
+	if (ret != 0)
+		return ret;
+
+	ret = switch_in(session);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_session_send_eos(struct mvx_session *session)
+{
+	struct mvx_session_port *port = &session->port[MVX_DIR_OUTPUT];
+	struct mvx_buffer *buf;
+
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return fw_eos(session);
+
+	if (list_empty(&port->buffer_queue) != false) {
+		MVX_SESSION_WARN(session, "Unable to signal EOS. Output buffer queue empty.");
+		return 0;
+	}
+
+	buf = list_first_entry(&port->buffer_queue, struct mvx_buffer, head);
+	list_del(&buf->head);
+
+	mvx_buffer_clear(buf);
+	buf->flags |= MVX_BUFFER_EOS;
+
+	session->event(session, MVX_SESSION_EVENT_BUFFER, buf);
+
+	return 0;
+}
+
+int mvx_session_streamon(struct mvx_session *session, enum mvx_direction dir)
+{
+	enum mvx_direction bdir;
+	struct mvx_hw_ver hw_ver;
+	enum mvx_direction i;
+	int ret;
+
+	MVX_SESSION_INFO(session, "Stream on. dir=%u.", dir);
+
+	if (dir == MVX_DIR_OUTPUT && session->port[dir].isallocparam == true) {
+		session->port[dir].isallocparam = false;
+		session->port[dir].isreallocting = false;
+	}
+
+	/* Verify that we don't enable an already activated port. */
+	if (session->port[dir].stream_on != false)
+		return 0;
+
+	session->port[dir].stream_on = true;
+
+	/* Check that both ports are stream on. */
+	if (!is_stream_on(session))
+		return 0;
+
+	ddr_qos_request_update(session);
+
+	/* Verify that a firmware binary load is not in progress. */
+	if (IS_ERR(session->fw_bin)) {
+		ret = PTR_ERR(session->fw_bin);
+		goto disable_port;
+	}
+
+	/* If a firmware binary is already loaded, then we are done. */
+	if (session->fw_bin != NULL) {
+		ret = wait_pending(session);
+		if (ret != 0)
+			goto disable_port;
+
+		ret = fw_state_change(session, MVX_FW_STATE_RUNNING);
+		if (ret != 0)
+			goto disable_port;
+
+		return 0;
+	}
+
+	bdir = get_bitstream_port(session);
+	if (bdir >= MVX_DIR_MAX) {
+		MVX_SESSION_WARN(session,
+				 "Session only support decoding and encoding, but not transcoding. input_format=%u, "
+				 "output_format=%u.",
+				 session->port[MVX_DIR_INPUT].format, session->port[MVX_DIR_OUTPUT].format);
+		ret = -EINVAL;
+		goto disable_port;
+	}
+
+	/* Verify that client can handle input and output formats. */
+	for (i = MVX_DIR_INPUT; i < MVX_DIR_MAX; i++) {
+		uint64_t formats;
+
+		session->client_ops->get_formats(session->client_ops, MVX_DIR_INPUT, &formats);
+
+		if (!mvx_test_bit(session->port[i].format, &formats)) {
+			MVX_SESSION_WARN(session,
+					 "Client cannot support requested formats. input_format=%u, output_format=%u.",
+					 session->port[MVX_DIR_INPUT].format, session->port[MVX_DIR_OUTPUT].format);
+			ret = -ENODEV;
+			goto disable_port;
+		}
+	}
+
+	/* Increment session reference count and flag fw bin as pending. */
+	mvx_session_get(session);
+	session->fw_bin = ERR_PTR(-EINPROGRESS);
+	session->client_ops->get_hw_ver(session->client_ops, &hw_ver);
+
+	/* Requesting firmware binary to be loaded. */
+	ret = mvx_fw_cache_get(session->cache, session->port[bdir].format, bdir, &session->fw_event, &hw_ver,
+			       session->isession.securevideo);
+	if (ret != 0) {
+		session->port[dir].stream_on = false;
+		session->fw_bin = NULL;
+		mvx_session_put(session);
+		return ret;
+	}
+
+	return 0;
+
+disable_port:
+	session->port[dir].stream_on = false;
+
+	return ret;
+}
+
+int mvx_session_streamoff(struct mvx_session *session, enum mvx_direction dir)
+{
+	struct mvx_session_port *port = &session->port[dir];
+	struct mvx_session_port *port_input = &session->port[MVX_DIR_INPUT];
+	struct mvx_buffer *buf;
+	struct mvx_buffer *tmp;
+	bool force_stop = (dir == MVX_DIR_OUTPUT && port_input->stream_on == false) ? 1 : 0;
+	int ret = 0;
+	int i;
+
+	MVX_SESSION_INFO(
+		session,
+		"Stream off. dir=%u, flushed=%d, is_flushing=%d, isreallocting=%d, isallocparam=%d, force_stop=%d", dir,
+		port->flushed, port->is_flushing, port->isreallocting, port->isallocparam, force_stop);
+
+	port->stream_on = false;
+
+	if (is_fw_loaded(session) != false) {
+		/*
+		 * Flush the ports if at least one buffer has been queued
+		 * since last flush.
+		 */
+		/* workaround for fw issue: It won't flush output buffer when STOP output stream
+		 * if flushed flag is true. It will result in no frame buffer output for seek operations.
+		 * Now force to flush output buffer if input stream is off.
+		 */
+
+		if ((port->flushed == false && port->is_flushing == false) || force_stop) {
+			ret = wait_pending(session);
+			if (ret != 0)
+				goto dequeue_buffers;
+			if (!(dir == MVX_DIR_OUTPUT && port->isreallocting == true) || force_stop) {
+				ret = fw_state_change(session, MVX_FW_STATE_STOPPED);
+				if (ret != 0)
+					goto dequeue_buffers;
+
+				ret = fw_flush(session, dir);
+				if (ret != 0) {
+					goto dequeue_buffers;
+				}
+			}
+			ret = wait_pending(session);
+			if (ret != 0)
+				goto dequeue_buffers;
+
+			send_irq(session);
+		}
+		if (dir == MVX_DIR_OUTPUT && port->isallocparam == true) {
+			wait_flush_done(session, dir);
+			for (i = 0; i < MVX_BUFFER_NPLANES; i++)
+				session->port[MVX_DIR_OUTPUT].buffer_size[i] = 0;
+		}
+		if (session->fw_state == MVX_FW_STATE_STOPPED) {
+			fw_switch_out(session);
+			wait_switch_out(session);
+			if (session->switched_in) {
+				MVX_SESSION_WARN(session, "warn: switch_in is %d when stream off done.",
+						 session->switched_in);
+			}
+		}
+	}
+dequeue_buffers:
+	if (ret != 0) {
+		MVX_SESSION_WARN(session, "stream off error. ret=%d. mvx_session=%p", ret, session);
+		session_unregister(session);
+	}
+
+	/* Return buffers in pending queue. */
+	list_for_each_entry_safe(buf, tmp, &port->buffer_queue, head)
+	{
+		list_del(&buf->head);
+		session->event(session, MVX_SESSION_EVENT_BUFFER, buf);
+	}
+
+	return 0;
+}
+
+static void handle_fw_message(struct mvx_session *session, struct mvx_fw_msg *msg)
+{
+	switch (msg->code) {
+	case MVX_FW_CODE_ALLOC_PARAM: {
+		struct mvx_session_port *input = &session->port[MVX_DIR_INPUT];
+		struct mvx_session_port *output = &session->port[MVX_DIR_OUTPUT];
+		unsigned int old_stride[MVX_BUFFER_NPLANES];
+
+		/* Update input port. */
+		input->width = msg->alloc_param.width;
+		input->height = msg->alloc_param.height;
+
+		try_format(session, MVX_DIR_INPUT, input->format, &input->width, &input->height, &input->nplanes,
+			   input->stride, input->size, &input->interlaced);
+
+		/*
+		 * Update output port. Set number of valid planes to 0 to force
+		 * stride to be recalculated.
+		 */
+
+		output->nplanes = 0;
+		output->afbc_alloc_bytes = msg->alloc_param.afbc_alloc_bytes;
+		output->afbc_width = msg->alloc_param.afbc_width;
+		old_stride[0] = output->stride[0];
+		old_stride[1] = output->stride[1];
+		old_stride[2] = output->stride[2];
+
+		try_format(session, MVX_DIR_OUTPUT, output->format, &output->width, &output->height, &output->nplanes,
+			   output->stride, output->size, &output->interlaced);
+
+		MVX_SESSION_INFO(session,
+				 "Firmware rsp: Alloc param. width=%u, height=%u, nplanes=%u, size=[%u, %u, %u], "
+				 "stride=[%u, %u, %u], interlaced=%d.",
+				 msg->alloc_param.width, msg->alloc_param.height, output->nplanes, output->size[0],
+				 output->size[1], output->size[2], output->stride[0], output->stride[1],
+				 output->stride[2], output->interlaced);
+
+		// update ddr qos for decoder output size changed.
+		ddr_qos_request_update(session);
+
+		if (output->size[0] > output->buffer_size[0] || output->size[1] > output->buffer_size[1] ||
+		    output->size[2] > output->buffer_size[2]) {
+			output->isreallocting = true;
+		} else {
+			// don't update strides for some vp9 cases. gralloc buffer stride is fixed if no realloc.
+			output->stride[0] = old_stride[0];
+			output->stride[1] = old_stride[1];
+			output->stride[2] = old_stride[2];
+		}
+		break;
+	}
+	case MVX_FW_CODE_BUFFER_GENERAL: {
+		struct mvx_buffer *buf = msg->buf;
+		session->port[buf->dir].buffer_count--;
+		session->event(session, MVX_SESSION_EVENT_BUFFER, buf);
+		break;
+	}
+	case MVX_FW_CODE_BUFFER: {
+		struct mvx_buffer *buf = msg->buf;
+		struct mvx_session_port *output = &session->port[MVX_DIR_OUTPUT];
+		struct mvx_v4l2_session *vsession = container_of(session, struct mvx_v4l2_session, session);
+		MVX_SESSION_INFO(session, "Firmware rsp: Buffer. dir=%u, len=[%u, %u, %u], flags=0x%08x, eos=%u",
+				 buf->dir, buf->planes[0].filled, buf->planes[1].filled, buf->planes[2].filled,
+				 buf->flags, (buf->flags & MVX_BUFFER_EOS) != 0);
+
+		session->port[buf->dir].buffer_count--;
+
+		if (buf->dir == MVX_DIR_OUTPUT && (buf->flags & MVX_BUFFER_EOS) != 0) {
+			session->eos_queued = false;
+		}
+
+		/*
+		 * There is no point to flush or invalidate input buffer
+		 * after it was returned from the HW.
+		 */
+		if (buf->dir == MVX_DIR_OUTPUT && mvx_is_frame(buf->format)) {
+			if (!(buf->flags & MVX_BUFFER_FRAME_PRESENT)) {
+				if (output->size[0] > mvx_buffer_size(buf, 0) ||
+				    output->size[1] > mvx_buffer_size(buf, 1) ||
+				    output->size[2] > mvx_buffer_size(buf, 2) ||
+				    session->port[buf->dir].buffer_allocated < session->port[buf->dir].buffer_min) {
+					buf->flags |= MVX_BUFFER_FRAME_NEED_REALLOC;
+					// output->isreallocting = true;
+				}
+			}
+		}
+
+		if (buf->dir == MVX_DIR_OUTPUT && vsession->port[MVX_DIR_OUTPUT].q_set == true)
+			mvx_buffer_synch(buf, DMA_FROM_DEVICE);
+
+		if (buf->dir == MVX_DIR_OUTPUT && !is_encoder(session)) {
+			session->port[MVX_DIR_OUTPUT].buffer_on_hold_count =
+				session->port[MVX_DIR_OUTPUT].pending_buffer_on_hold_count;
+		}
+
+		session->event(session, MVX_SESSION_EVENT_BUFFER, buf);
+		break;
+	}
+	case MVX_FW_CODE_COLOR_DESC: {
+		MVX_SESSION_INFO(session, "Firmware rsp: Color desc.");
+		session->color_desc = msg->color_desc;
+		session->event(session, MVX_SESSION_EVENT_COLOR_DESC, NULL);
+		break;
+	}
+	case MVX_FW_CODE_ERROR: {
+		MVX_SESSION_WARN(session, "Firmware rsp: Error. code=%u, message=%s.", msg->error.error_code,
+				 msg->error.message);
+		print_debug(session);
+		fw_dump(session);
+
+		/*
+		 * Release the dev session. It will prevent a dead session from
+		 * blocking the scheduler.
+		 */
+		watchdog_stop(session);
+		session_unregister(session);
+		send_event_error(session, -EINVAL);
+		break;
+	}
+	case MVX_FW_CODE_FLUSH: {
+		MVX_SESSION_INFO(session, "Firmware rsp: Flushed. dir=%d.", msg->flush.dir);
+		session->port[msg->flush.dir].is_flushing = false;
+		session->port[msg->flush.dir].flushed = true;
+		(void)queue_pending_buffers(session, msg->flush.dir);
+		break;
+	}
+	case MVX_FW_CODE_IDLE: {
+		int ret;
+		struct mvx_fw_msg msg_ack;
+
+		MVX_SESSION_INFO(session, "Firmware rsp: Idle.");
+
+		session->idle_count++;
+
+		if (session->idle_count == 2)
+			fw_switch_out(session);
+
+		msg_ack.code = MVX_FW_CODE_IDLE_ACK;
+		ret = session->fw.ops.put_message(&session->fw, &msg_ack);
+		if (ret == 0)
+			ret = send_irq(session);
+
+		if (ret != 0)
+			send_event_error(session, ret);
+
+		break;
+	}
+	case MVX_FW_CODE_JOB: {
+		MVX_SESSION_INFO(session, "Firmware rsp: Job.");
+		(void)fw_job(session, 1);
+		break;
+	}
+	case MVX_FW_CODE_PONG:
+		MVX_SESSION_INFO(session, "Firmware rsp: Pong.");
+		break;
+	case MVX_FW_CODE_SEQ_PARAM: {
+		struct mvx_session_port *p = &session->port[MVX_DIR_OUTPUT];
+
+		MVX_SESSION_INFO(
+			session,
+			"Firmware rsp: Seq param. planar={buffers_min=%u}, afbc={buffers_min=%u}, interlaced=%d.",
+			msg->seq_param.planar.buffers_min, msg->seq_param.afbc.buffers_min, p->interlaced);
+
+		if (mvx_is_afbc(p->format) != false)
+			p->buffer_min = msg->seq_param.afbc.buffers_min;
+		else
+			p->buffer_min = msg->seq_param.planar.buffers_min;
+		p->seq_param = msg->seq_param;
+
+		(void)fw_flush(session, MVX_DIR_OUTPUT);
+
+		// force to realloc if received seq_param message for it will flush output buffer.
+		p->isreallocting = true;
+
+		break;
+	}
+	case MVX_FW_CODE_SET_OPTION: {
+		MVX_SESSION_INFO(session, "Firmware rsp: Set option.");
+		break;
+	}
+	case MVX_FW_CODE_STATE_CHANGE: {
+		MVX_SESSION_INFO(session, "Firmware rsp: State changed. old=%s, new=%s.",
+				 state_to_string(session->fw_state), state_to_string(msg->state));
+		session->fw_state = msg->state;
+		break;
+	}
+	case MVX_FW_CODE_SWITCH_IN: {
+		watchdog_start(session, session->watchdog_timeout);
+		break;
+	}
+	case MVX_FW_CODE_SWITCH_OUT: {
+		MVX_SESSION_INFO(session, "Firmware rsp: Switched out.");
+
+		watchdog_stop(session);
+		session->switched_in = false;
+
+		if (session->is_suspend == false &&
+		    ((session->fw_state == MVX_FW_STATE_RUNNING && session->idle_count < 2) ||
+		     session->fw.msg_pending > 0))
+			switch_in(session);
+
+		break;
+	}
+	case MVX_FW_CODE_DUMP:
+		break;
+	case MVX_FW_CODE_DEBUG:
+		break;
+	case MVX_FW_CODE_UNKNOWN: {
+		print_debug(session);
+		break;
+	}
+	case MVX_FW_CODE_DPB_HELD_FRAMES: {
+		session->port[MVX_DIR_OUTPUT].pending_buffer_on_hold_count = msg->arg;
+		break;
+	}
+	case MVX_FW_CODE_MAX:
+		break;
+	default:
+		MVX_SESSION_WARN(session, "Unknown fw msg code. code=%u.", msg->code);
+	}
+}
+
+void mvx_session_irq(struct mvx_if_session *isession)
+{
+	struct mvx_session *session = mvx_if_session_to_session(isession);
+	int ret;
+	struct mvx_session_port *output = &session->port[MVX_DIR_OUTPUT];
+
+	if (is_fw_loaded(session) == false)
+		return;
+
+	if (IS_ERR_OR_NULL(session->csession)) {
+		return;
+	}
+
+	ret = session->fw.ops.handle_rpc(&session->fw);
+	if (ret < 0) {
+		send_event_error(session, ret);
+		return;
+	}
+
+	do {
+		struct mvx_fw_msg msg;
+
+		watchdog_update(session, session->watchdog_timeout);
+
+		ret = session->fw.ops.get_message(&session->fw, &msg);
+		if (ret < 0) {
+			session_unregister(session);
+			send_event_error(session, ret);
+			return;
+		}
+
+		if (ret > 0)
+			handle_fw_message(session, &msg);
+	} while (ret > 0 && session->error == 0);
+
+	if (output->isallocparam == false && output->isreallocting == true) {
+		output->isallocparam = true;
+		session->event(session, MVX_SESSION_EVENT_PORT_CHANGED, (void *)MVX_DIR_OUTPUT);
+	}
+
+#ifdef MVX_FW_DEBUG_ENABLE
+	ret = session->fw.ops.handle_fw_ram_print(&session->fw);
+	if (ret < 0) {
+		send_event_error(session, ret);
+		return;
+	}
+#endif
+
+	wake_up(&session->waitq);
+}
+
+void mvx_session_port_show(struct mvx_session_port *port, struct seq_file *s)
+{
+	mvx_seq_printf(s, "mvx_session_port", 0, "%p\n", port);
+	mvx_seq_printf(s, "format", 1, "%08x\n", port->format);
+	mvx_seq_printf(s, "width", 1, "%u\n", port->width);
+	mvx_seq_printf(s, "height", 1, "%u\n", port->height);
+	mvx_seq_printf(s, "buffer_min", 1, "%u\n", port->buffer_min);
+	mvx_seq_printf(s, "buffer_count", 1, "%u\n", port->buffer_count);
+}
+
+int mvx_session_set_securevideo(struct mvx_session *session, bool securevideo)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->isession.securevideo = securevideo;
+
+	return 0;
+}
+
+int mvx_session_set_frame_rate(struct mvx_session *session, int64_t frame_rate)
+{
+	int ret;
+	if (session->error != 0)
+		return session->error;
+
+	/*Frame rate values are limited to between 1 and 256 frames per second*/
+	if (frame_rate < (1 << 16))
+		frame_rate = (1 << 16); /*1 fps*/
+	else if (frame_rate > (256 << 16))
+		frame_rate = (256 << 16); /*256 fps*/
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_FRAME_RATE;
+		option.frame_rate = frame_rate;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+
+	// update ddr qos for framerate changed.
+	if (frame_rate != session->frame_rate) {
+		session->frame_rate = frame_rate;
+		ddr_qos_request_update(session);
+	}
+
+	return 0;
+}
+
+int mvx_session_set_rate_control(struct mvx_session *session, bool enabled)
+{
+	int ret;
+	if (session->error != 0)
+		return session->error;
+
+	session->rc_enabled = enabled;
+	// set default rc type.
+	session->rc_type = enabled ? ((session->rc_type) ? session->rc_type : MVX_OPT_RATE_CONTROL_MODE_STANDARD) : 0;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_TARGET_BITRATE;
+		option.target_bitrate = (session->rc_enabled != false) ? session->target_bitrate : 0;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+int mvx_session_set_bitrate(struct mvx_session *session, int bitrate)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->target_bitrate = bitrate;
+
+	if (is_fw_loaded(session) != false && session->rc_enabled != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_RATE_CONTROL;
+		option.rate_control.target_bitrate = session->target_bitrate;
+		option.rate_control.rate_control_mode = session->rc_type;
+
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+int mvx_session_set_bitrate_control(struct mvx_session *session, struct mvx_buffer_param_rate_control *rc)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->rc_type = rc->rate_control_mode;
+	session->target_bitrate = rc->target_bitrate;
+	session->maximum_bitrate = rc->maximum_bitrate;
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_RATE_CONTROL;
+		option.rate_control.target_bitrate = rc->target_bitrate;
+		option.rate_control.rate_control_mode = rc->rate_control_mode;
+		if (rc->rate_control_mode == MVX_OPT_RATE_CONTROL_MODE_C_VARIABLE) {
+			option.rate_control.maximum_bitrate = rc->maximum_bitrate;
+		}
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_crop_left(struct mvx_session *session, int32_t left)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->crop_left = left;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_CROP_LEFT;
+		option.crop_left = left;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_crop_right(struct mvx_session *session, int32_t right)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->crop_right = right;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_CROP_RIGHT;
+		option.crop_right = right;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_crop_top(struct mvx_session *session, int32_t top)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->crop_top = top;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_CROP_TOP;
+		option.crop_top = top;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_crop_bottom(struct mvx_session *session, int32_t bottom)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->crop_bottom = bottom;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_CROP_BOTTOM;
+		option.crop_bottom = bottom;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_nalu_format(struct mvx_session *session, enum mvx_nalu_format fmt)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->nalu_format = fmt;
+
+	return 0;
+}
+
+int mvx_session_set_stream_escaping(struct mvx_session *session, enum mvx_tristate status)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->stream_escaping = status;
+
+	return 0;
+}
+
+int mvx_session_set_profile(struct mvx_session *session, enum mvx_format format, enum mvx_profile profile)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->profile[format] = profile;
+
+	return 0;
+}
+
+int mvx_session_set_level(struct mvx_session *session, enum mvx_format format, enum mvx_level level)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->level[format] = level;
+
+	return 0;
+}
+
+int mvx_session_set_ignore_stream_headers(struct mvx_session *session, enum mvx_tristate status)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->ignore_stream_headers = status;
+
+	return 0;
+}
+
+int mvx_session_set_frame_reordering(struct mvx_session *session, enum mvx_tristate status)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->frame_reordering = status;
+
+	return 0;
+}
+
+int mvx_session_set_intbuf_size(struct mvx_session *session, int size)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->intbuf_size = size;
+
+	return 0;
+}
+
+int mvx_session_set_p_frames(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->p_frames = val;
+
+	return 0;
+}
+
+int mvx_session_set_b_frames(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->b_frames = val;
+
+	return 0;
+}
+
+int mvx_session_set_gop_type(struct mvx_session *session, enum mvx_gop_type gop_type)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->gop_type = gop_type;
+
+	return 0;
+}
+
+int mvx_session_set_cyclic_intra_refresh_mb(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->cyclic_intra_refresh_mb = val;
+
+	return 0;
+}
+
+int mvx_session_set_constr_ipred(struct mvx_session *session, enum mvx_tristate status)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->constr_ipred = status;
+
+	return 0;
+}
+
+int mvx_session_set_entropy_sync(struct mvx_session *session, enum mvx_tristate status)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->entropy_sync = status;
+
+	return 0;
+}
+
+int mvx_session_set_temporal_mvp(struct mvx_session *session, enum mvx_tristate status)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->temporal_mvp = status;
+
+	return 0;
+}
+
+int mvx_session_set_tile_rows(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->tile_rows = val;
+
+	return 0;
+}
+
+int mvx_session_set_tile_cols(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->tile_cols = val;
+
+	return 0;
+}
+
+int mvx_session_set_min_luma_cb_size(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->min_luma_cb_size = val;
+
+	return 0;
+}
+
+int mvx_session_set_mb_mask(struct mvx_session *session, int val)
+{
+	/*
+	 * This controls is not implemented.
+	 */
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->mb_mask = val;
+
+	return 0;
+}
+
+int mvx_session_set_entropy_mode(struct mvx_session *session, enum mvx_entropy_mode mode)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->entropy_mode = mode;
+
+	return 0;
+}
+
+int mvx_session_set_multi_slice_mode(struct mvx_session *session, enum mvx_multi_slice_mode mode)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->multi_slice_mode = mode;
+
+	return 0;
+}
+
+int mvx_session_set_multi_slice_max_mb(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->multi_slice_max_mb = val;
+
+	return 0;
+}
+
+int mvx_session_set_vp9_prob_update(struct mvx_session *session, enum mvx_vp9_prob_update mode)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->vp9_prob_update = mode;
+
+	return 0;
+}
+
+int mvx_session_set_mv_h_search_range(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->mv_h_search_range = val;
+
+	return 0;
+}
+
+int mvx_session_set_mv_v_search_range(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->mv_v_search_range = val;
+
+	return 0;
+}
+
+int mvx_session_set_bitdepth_chroma(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->bitdepth_chroma = val;
+
+	return 0;
+}
+
+int mvx_session_set_bitdepth_luma(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->bitdepth_luma = val;
+
+	return 0;
+}
+
+int mvx_session_set_force_chroma_format(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->force_chroma_format = val;
+
+	return 0;
+}
+
+int mvx_session_set_rgb_to_yuv_mode(struct mvx_session *session, enum mvx_rgb_to_yuv_mode mode)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->rgb_to_yuv = mode;
+
+	return 0;
+}
+
+int mvx_session_set_band_limit(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->band_limit = val;
+
+	return 0;
+}
+
+int mvx_session_set_cabac_init_idc(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->cabac_init_idc = val;
+
+	return 0;
+}
+
+int mvx_session_set_i_frame_qp(struct mvx_session *session, enum mvx_format fmt, int qp)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false) {
+		enum mvx_direction dir = get_bitstream_port(session);
+
+		fmt = session->port[dir].format;
+		ret = fw_set_qp(session, MVX_FW_SET_QP_I, qp);
+		if (ret != 0)
+			return ret;
+	}
+
+	session->qp[fmt].i_frame = qp;
+
+	return 0;
+}
+
+int mvx_session_set_p_frame_qp(struct mvx_session *session, enum mvx_format fmt, int qp)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false) {
+		enum mvx_direction dir = get_bitstream_port(session);
+
+		fmt = session->port[dir].format;
+		ret = fw_set_qp(session, MVX_FW_SET_QP_P, qp);
+		if (ret != 0)
+			return ret;
+	}
+
+	session->qp[fmt].p_frame = qp;
+
+	return 0;
+}
+
+int mvx_session_set_b_frame_qp(struct mvx_session *session, enum mvx_format fmt, int qp)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false) {
+		enum mvx_direction dir = get_bitstream_port(session);
+
+		fmt = session->port[dir].format;
+		ret = fw_set_qp(session, MVX_FW_SET_QP_B, qp);
+		if (ret != 0)
+			return ret;
+	}
+
+	session->qp[fmt].b_frame = qp;
+
+	return 0;
+}
+
+int mvx_session_set_min_qp(struct mvx_session *session, enum mvx_format fmt, int qp)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+		enum mvx_direction dir = get_bitstream_port(session);
+		int codec = session->port[dir].format;
+
+		option.code = MVX_FW_SET_QP_RANGE;
+		option.qp_range.min = qp;
+		option.qp_range.max = session->qp[codec].max;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+
+	session->qp[fmt].min = qp;
+
+	return 0;
+}
+
+int mvx_session_set_max_qp(struct mvx_session *session, enum mvx_format fmt, int qp)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+		enum mvx_direction dir = get_bitstream_port(session);
+		int codec = session->port[dir].format;
+
+		option.code = MVX_FW_SET_QP_RANGE;
+		option.qp_range.min = session->qp[codec].min;
+		option.qp_range.max = qp;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+
+	session->qp[fmt].max = qp;
+
+	return 0;
+}
+
+int mvx_session_set_resync_interval(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->resync_interval = val;
+
+	return 0;
+}
+
+int mvx_session_set_jpeg_quality(struct mvx_session *session, int val)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->jpeg_quality = val;
+
+	return 0;
+}
+
+int mvx_session_get_color_desc(struct mvx_session *session, struct mvx_fw_color_desc *color_desc)
+{
+	*color_desc = session->color_desc;
+	return 0;
+}
+
+int mvx_session_set_color_desc(struct mvx_session *session, struct mvx_fw_color_desc *color_desc)
+{
+	int ret = 0;
+	if (session->error != 0)
+		return session->error;
+
+	session->color_desc = *color_desc;
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_COLOUR_DESC;
+		option.colour_desc = *color_desc;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_roi_regions(struct mvx_session *session, struct mvx_roi_config *roi)
+{
+	int ret = 0;
+	int roi_config_num = 0;
+	if (is_fw_loaded(session) == false || session->port[MVX_DIR_INPUT].is_flushing != false) {
+		roi_config_num = session->port[MVX_DIR_INPUT].roi_config_num;
+		if (roi_config_num < MVX_ROI_QP_NUMS) {
+			MVX_SESSION_INFO(session, "fw is not ready!!!, pending roi num:%d", roi_config_num);
+			session->port[MVX_DIR_INPUT].roi_config_queue[roi_config_num] = *roi;
+			session->port[MVX_DIR_INPUT].roi_config_num++;
+		} else {
+			MVX_SESSION_ERR(session, "fw is not ready for long time, too many roi pending:%d",
+					roi_config_num);
+		}
+		return 0;
+	}
+	ret = queue_roi_regions(session, roi);
+	return 0;
+}
+
+int mvx_session_set_qp_epr(struct mvx_session *session, int *qp)
+{
+	int ret = 0;
+	int qp_num = 0;
+	if (is_fw_loaded(session) == false || session->port[MVX_DIR_INPUT].is_flushing != false) {
+		qp_num = session->port[MVX_DIR_INPUT].qp_num;
+		if (qp_num < MVX_ROI_QP_NUMS) {
+			MVX_SESSION_WARN(session, "fw is not ready!!!, pending qp num:%d", qp_num);
+			session->port[MVX_DIR_INPUT].qp_queue[qp_num] = *qp;
+			session->port[MVX_DIR_INPUT].qp_num++;
+		} else {
+			MVX_SESSION_ERR(session, "fw is not ready for long time, too many qp pending:%d", qp_num);
+		}
+		return 0;
+	}
+	ret = queue_qp_epr(session, qp);
+	return 0;
+}
+
+int mvx_session_set_sei_userdata(struct mvx_session *session, struct mvx_sei_userdata *userdata)
+{
+	int ret = 0;
+	if (session->error != 0)
+		return session->error;
+
+	session->sei_userdata = *userdata;
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_SEI_USERDATA;
+		option.userdata = *userdata;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return ret;
+}
+
+int mvx_session_set_hrd_buffer_size(struct mvx_session *session, int size)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->nHRDBufsize = size;
+
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_HRD_BUF_SIZE;
+		option.nHRDBufsize = size;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_dsl_frame(struct mvx_session *session, struct mvx_dsl_frame *dsl)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->dsl_frame.width = dsl->width;
+	session->dsl_frame.height = dsl->height;
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_DSL_FRAME;
+		option.dsl_frame.width = dsl->width;
+		option.dsl_frame.height = dsl->height;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_dsl_ratio(struct mvx_session *session, struct mvx_dsl_ratio *dsl)
+{
+	if (session->error != 0)
+		return session->error;
+
+	session->dsl_ratio.hor = dsl->hor;
+	session->dsl_ratio.ver = dsl->ver;
+	return 0;
+}
+
+int mvx_session_set_long_term_ref(struct mvx_session *session, struct mvx_long_term_ref *ltr)
+{
+	int ret;
+
+	if (session->error != 0)
+		return session->error;
+
+	session->mvx_ltr.mode = ltr->mode;
+	session->mvx_ltr.period = ltr->period;
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+
+		option.code = MVX_FW_SET_LONG_TERM_REF;
+		option.ltr.mode = ltr->mode;
+		option.ltr.period = ltr->period;
+		ret = fw_set_option(session, &option);
+		if (ret != 0)
+			return ret;
+	}
+	return 0;
+}
+
+int mvx_session_set_dsl_mode(struct mvx_session *session, int *mode)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->dsl_pos_mode = *mode;
+
+	return 0;
+}
+int mvx_session_set_force_idr(struct mvx_session *session)
+{
+	int ret;
+	if (session->error != 0)
+		return session->error;
+	if (is_fw_loaded(session) != false) {
+		struct mvx_fw_set_option option;
+		/*reset GOP type to force idr frame.*/
+		option.code = MVX_FW_SET_GOP_RESET;
+		ret = fw_set_option(session, &option);
+		if (ret != 0) {
+			MVX_SESSION_WARN(session, "Failed to  GOP reset.");
+			return ret;
+		}
+	}
+	return 0;
+}
+
+int mvx_session_set_watchdog_timeout(struct mvx_session *session, int timeout)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->watchdog_timeout = timeout * 1000;
+
+	return 0;
+}
+
+int mvx_session_set_profiling(struct mvx_session *session, int enable)
+{
+	if (session->error != 0)
+		return session->error;
+
+	if (is_fw_loaded(session) != false)
+		return -EBUSY;
+
+	session->enable_profiling = enable;
+
+	return 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/mvx_session.h b/drivers/media/platform/spacemit/vpu_k3/if/mvx_session.h
new file mode 100644
index 000000000000..46b9db1cd17c
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/mvx_session.h
@@ -0,0 +1,1050 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_SESSION_H_
+#define _MVX_SESSION_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+#include "mvx_buffer.h"
+#include "mvx_firmware.h"
+#include "mvx_firmware_cache.h"
+#include "mvx_if.h"
+#include "mvx_log_group.h"
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define MVX_SESSION_LOG(severity, session, msg, ...)                                                                   \
+	MVX_LOG_PRINT_SESSION(&mvx_log_session_if, severity, session, msg, ##__VA_ARGS__)
+
+#define MVX_SESSION_VERBOSE(session, msg, ...) MVX_SESSION_LOG(MVX_LOG_VERBOSE, session, msg, ##__VA_ARGS__)
+
+#define MVX_SESSION_DEBUG(session, msg, ...) MVX_SESSION_LOG(MVX_LOG_DEBUG, session, msg, ##__VA_ARGS__)
+
+#define MVX_SESSION_INFO(session, msg, ...) MVX_SESSION_LOG(MVX_LOG_INFO, session, msg, ##__VA_ARGS__)
+
+#define MVX_SESSION_WARN(session, msg, ...) MVX_SESSION_LOG(MVX_LOG_WARNING, session, msg, ##__VA_ARGS__)
+
+#define MVX_SESSION_ERR(session, msg, ...) MVX_SESSION_LOG(MVX_LOG_ERROR, session, msg, ##__VA_ARGS__)
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct file;
+struct mvx_csched;
+struct mvx_fw_cache;
+struct poll_table_struct;
+
+/**
+ * enum mvx_session_event - Session events.
+ * @MVX_SESSION_EVENT_BUFFER:		struct mvx_buffer.
+ * @MVX_SESSION_EVENT_PORT_CHANGED:	enum mvx_direction.
+ * @MVX_SESSION_EVENT_COLOR_DESC:	struct mvx_fw_color_desc.
+ * @MVX_SESSION_EVENT_ERROR:		void
+ */
+enum mvx_session_event {
+	MVX_SESSION_EVENT_BUFFER,
+	MVX_SESSION_EVENT_PORT_CHANGED,
+	MVX_SESSION_EVENT_COLOR_DESC,
+	MVX_SESSION_EVENT_ERROR
+};
+
+/**
+ * struct mvx_session_port - Session input and output port settings.
+ * @format:		Port format.
+ * @width:		Width in pixels.
+ * @height:		Height in pixels.
+ * @nplanes:		Number for planes for current format.
+ * @stride:		Stride per line in bytes for each plane.
+ * @size:		Size in bytes for each plane.
+ * @afbc_alloc_bytes:	Minimum number of bytes required for AFBC.
+ * @afbc_width:		AFBC width in superblocks.
+ * @stream_on:		Boolean if the port has been enabled.
+ * @buffer_min:		Minimum number of buffers required.
+ * @buffer_count:	Number of buffers currently queued to firmware.
+ * @buffer_queue:	Buffers waiting to be queued to the firmware.
+ * @is_flushing:	Set true when port is waiting for a fw flush confirm.
+ * @flushed:		Port has been flushed an no buffers have been queued.
+ * @interlaced:		True if frames are interlaced.
+ */
+struct mvx_session_port {
+	enum mvx_format format;
+	unsigned int width;
+	unsigned int height;
+	uint8_t nplanes;
+	unsigned int stride[MVX_BUFFER_NPLANES];
+	unsigned int size[MVX_BUFFER_NPLANES];
+	unsigned int afbc_alloc_bytes;
+	unsigned int afbc_width;
+	bool stream_on;
+	unsigned int buffer_min;
+	unsigned int buffer_count;
+	unsigned int buffer_allocated;
+	struct list_head buffer_queue;
+	bool is_flushing;
+	bool flushed;
+	bool interlaced;
+	unsigned int scaling_shift;
+	struct mvx_roi_config roi_config_queue[MVX_ROI_QP_NUMS];
+	int qp_queue[MVX_ROI_QP_NUMS];
+	unsigned int roi_config_num;
+	unsigned int qp_num;
+	bool isreallocting;
+	bool isallocparam;
+	struct mvx_fw_seq_param seq_param;
+	uint32_t buffer_on_hold_count;	       /**< Number of buffers that the firmware has completed process of,
+						*   but is holding on to for frame reordering. */
+	uint32_t pending_buffer_on_hold_count; /**< Some FW versions signal the DPB in the form of a message
+						*   that will be valid once the next frame has been returned
+						*   by the FW. In these cases, the new DPB is stored in this
+						*   variable and copied to buffer_on_hold_count once the next
+						*   buffer arrives. */
+	unsigned int buffer_size[MVX_BUFFER_NPLANES];
+};
+
+/**
+ * struct mvx_session_qp - QP settings.
+ * @i_frame:		QP for I frame.
+ * @p_frame:            QP for P frame.
+ * @b_frame:            QP for B frame.
+ * @min:		Minimum QP value.
+ * @max:		Maximum QP value.
+ */
+struct mvx_session_qp {
+	int i_frame;
+	int p_frame;
+	int b_frame;
+	int min;
+	int max;
+};
+
+/**
+ * struct mvx_session - Session instance.
+ * @dev:		Pointer to device.
+ * @cache:		Pointer to firmware cache.
+ * @isession:		This instance is used to register the session to the
+ *                      client.
+ * @client_ops:		Client operations.
+ * @csession:		Client session.
+ * @destructor:		When the isession.kref reaches zero and after the
+ * session
+ *                      object has been destructed, this callback routine is
+ * invoked
+ *                      to allow the owner of the session object to clean up any
+ *                      allocated resources.
+ * @event:		Event callback routine.
+ * @mutex:		Mutex protecting the session objects.
+ * @port:		Input and output port settings.
+ * @mmu:		MMU instance.
+ * @fw:			Firmware instance.
+ * @fw_bin:		Pointer to firmware binary.
+ * @fw_event:		Event handler for loading a firmware binary.
+ * @fw_state:		Current firmware state.
+ * @waitq:		Wait queue to signal changes to the session.
+ * @dentry:		Debugfs directory entry for the session.
+ * @frame_rate:		Frame rate in Q16 format.
+ * @target_bitrate:	Bitrate.
+ * @rc_enabled:		Defines if rate control is enabled for the session.
+ * @profile:		Profile for encoder.
+ * @level:		Level for encoder.
+ * @nalu_format:	NALU format.
+ * @stream_escaping:	Defines if stream escaping is enabled.
+ * @ignore_stream_headers:Defines if decoder should ignore stream headers.
+ * @frame_reordering:	Defines if decoder should reorder frames.
+ * @intbuf_size:	Suggested internal buffer size.
+ * @p_frames:		Number of P-frames for encoder.
+ * @b_frames:		Number of B-frames for encoder.
+ * @gop_type:		GOP type.
+ * @cyclic_intra_refresh_mb:Intra MB refresh.
+ * @constr_ipred:	Constrained intra prediction.
+ * @entropy_sync:	Enabled entropy synchronization.
+ * @temporal_mvp:	Enable temporal motion vector prediction.
+ * @tile_rows:		Tile size.
+ * @tile_cols:		Tile size.
+ * @min_luma_cb_size:	Minimum luma coding block size.
+ * @mb_mask:		MB mask.
+ * @entropy_mode:	Entropy mode.
+ * @multi_slice_mode:	Multi slice mode.
+ * @multi_slice_max_mb:	Maximum number of macroblocks in a slice.
+ * @vp9_prob_update:	Probability update method.
+ * @mv_h_search_range:	Horizontal search range.
+ * @mv_v_search_range:	Vertical search range.
+ * @bitdepth_chroma:	Bitdepth for chroma.
+ * @bitdepth_luma:	Bitdepth for luma.
+ * @force_chroma_format:Chroma format.
+ * @rgb_to_yuv:		RGB to YUV conversion mode.
+ * @band_limit:		Maximum bandwidth limit.
+ * @cabac_init_idc:	CABAC initialization table.
+ * @qp:			QP settings per codec.
+ * @resync_interval:	JPEG resync interval.
+ * @jpeg_quality:	JPEG quality level.
+ * @color_desc:		HDR color description.
+ *
+ * There is one session for each file handle that has been opened from the
+ * video device.
+ *
+ * There is a separate set of QP controls for every codec. There is no
+ * information on which codec will be used when controls are initialized with
+ * their default values. That's why a set of QP-settings is maintained for
+ * every codec.
+ */
+struct mvx_session {
+	struct device *dev;
+	struct mvx_fw_cache *cache;
+	struct mvx_if_session isession;
+	struct mvx_client_ops *client_ops;
+	struct mvx_client_session *csession;
+	void (*destructor)(struct mvx_session *session);
+	void (*event)(struct mvx_session *session, enum mvx_session_event event, void *arg);
+	struct mvx_session_port port[MVX_DIR_MAX];
+	struct mvx_mmu mmu;
+	struct mvx_fw fw;
+	struct mvx_fw_bin *fw_bin;
+	struct mvx_fw_event fw_event;
+	enum mvx_fw_state fw_state;
+	wait_queue_head_t waitq;
+	struct timer_list watchdog_timer;
+	struct work_struct watchdog_work;
+	unsigned int watchdog_count;
+	bool switched_in;
+	unsigned int idle_count;
+	long error;
+	struct dentry *dentry;
+
+	int64_t frame_rate;
+	unsigned int target_bitrate;
+	unsigned int maximum_bitrate;
+	bool rc_enabled;
+	int rc_type;
+	enum mvx_profile profile[MVX_FORMAT_BITSTREAM_LAST + 1];
+	enum mvx_level level[MVX_FORMAT_BITSTREAM_LAST + 1];
+	enum mvx_nalu_format nalu_format;
+	enum mvx_tristate stream_escaping;
+	enum mvx_tristate ignore_stream_headers;
+	enum mvx_tristate frame_reordering;
+	int64_t intbuf_size;
+	int p_frames;
+	int b_frames;
+	enum mvx_gop_type gop_type;
+	int cyclic_intra_refresh_mb;
+	enum mvx_tristate constr_ipred;
+	enum mvx_tristate entropy_sync;
+	enum mvx_tristate temporal_mvp;
+	int tile_rows;
+	int tile_cols;
+	int min_luma_cb_size;
+	int mb_mask;
+	enum mvx_entropy_mode entropy_mode;
+	enum mvx_multi_slice_mode multi_slice_mode;
+	int multi_slice_max_mb;
+	enum mvx_vp9_prob_update vp9_prob_update;
+	int mv_h_search_range;
+	int mv_v_search_range;
+	int bitdepth_chroma;
+	int bitdepth_luma;
+	int force_chroma_format;
+	enum mvx_rgb_to_yuv_mode rgb_to_yuv;
+	int band_limit;
+	int cabac_init_idc;
+	struct mvx_session_qp qp[MVX_FORMAT_BITSTREAM_LAST + 1];
+	int resync_interval;
+	int jpeg_quality;
+	struct mvx_fw_color_desc color_desc;
+	unsigned int crop_left;
+	unsigned int crop_right;
+	unsigned int crop_top;
+	unsigned int crop_bottom;
+	struct mvx_sei_userdata sei_userdata;
+	unsigned int nHRDBufsize;
+	struct mvx_dsl_frame dsl_frame;
+	struct mvx_dsl_ratio dsl_ratio;
+	struct mvx_long_term_ref mvx_ltr;
+	int dsl_pos_mode;
+	/*variable for DDR qos setting. */
+	uint32_t estimated_ddr_read_throughput;	 /**<estimated ddr throughput requirement for this session */
+	uint32_t estimated_ddr_write_throughput; /**<estimated ddr throughput requirement for this session */
+	bool eos_queued;			 /**< Indicates last incoming buffer has EOS flag */
+	bool keep_freq_high;			 /**< Control DVFS by avoiding low frequency at start of usecase */
+	bool is_suspend;
+	struct list_head buffer_corrupt_queue;
+	unsigned int watchdog_timeout; // timeout value[ms] for watchdog
+	/*for debug. */
+	uint64_t bus_write_bytes_total;
+	uint64_t bus_read_bytes_total;
+	uint64_t frame_id;
+	int enable_profiling;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_session_construct - Construct the session object.
+ * @session:	Pointer to session.
+ * @dev:	Pointer to device.
+ * @client_ops:	Pointer to client ops.
+ * @cache:	Pointer to firmware cache.
+ * @mutex:	Pointer to mutex protecting the session object.
+ * @destructor:	Destructor that will be invoked after the session referece count
+ *              has reached zero. The destructor may be NULL if the owner of the
+ *              session object does not need to be notified.
+ * @event:	Event notification from the session to the client. This function
+ *              must not call session API which could take mvx_session mutex.
+ * @dsession:	Debugfs directory entry for the session.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_construct(struct mvx_session *session, struct device *dev, struct mvx_client_ops *client_ops,
+			  struct mvx_fw_cache *cache, struct mutex *mutex,
+			  void (*destructor)(struct mvx_session *session),
+			  void (*event)(struct mvx_session *session, enum mvx_session_event event, void *arg),
+			  struct dentry *dsession);
+
+/**
+ * mvx_session_construct - Destruct the session object.
+ * @session:	Pointer to session.
+ */
+void mvx_session_destruct(struct mvx_session *session);
+
+/**
+ * mvx_session_get - Increment the session reference count.
+ * @session:	Pointer to session.
+ */
+void mvx_session_get(struct mvx_session *session);
+
+/**
+ * mvx_session_put - Decrement the session reference count.
+ * @session:	Pointer to session.
+ *
+ * If the reference count reaches 0 the session object will be destructed.
+ *
+ * Return: 1 if session was removed, else 0.
+ */
+int mvx_session_put(struct mvx_session *session);
+
+/**
+ * mvx_session_get_formats() - Get bitmask of supported formats.
+ * @session:	Pointer to session.
+ * @dir:	Which direction to get formats for.
+ * @formats:	Pointer to bitmask listing supported formats.
+ */
+void mvx_session_get_formats(struct mvx_session *session, enum mvx_direction dir, uint64_t *formats);
+
+/**
+ * mvx_session_try_format() - Validate port format.
+ * @session:	Pointer to session.
+ * @dir:	Which direction to get formats for.
+ * @format:	MVX format.
+ * @width:	Width. Only valid for frame formats.
+ * @height:	Height. Only valid for frame formats.
+ * @nplanes:	Number for planes.
+ * @stride:	Horizonal stride in bytes for each plane.
+ * @size:	Size in bytes for each plane.
+ * @interlace:	True if frames are interlaced.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_try_format(struct mvx_session *session, enum mvx_direction dir, enum mvx_format format,
+			   unsigned int *width, unsigned int *height, uint8_t *nplanes, unsigned int *stride,
+			   unsigned int *size, bool *interlaced);
+
+/**
+ * mvx_session_set_format() - Validate and set port format.
+ * @session:	Pointer to session.
+ * @dir:	Which direction to get formats for.
+ * @format:	MVX format.
+ * @width:	Width. Only valid for frame formats.
+ * @height:	Height. Only valid for frame formats.
+ * @nplanes:	Number for planes.
+ * @stride:	Horizonal stride in bytes for each plane.
+ * @size:	Size in bytes for each plane.
+ * @interlaced:	True if frames are interlaced.
+ *
+ * If *nplanes is 0, then the values of stride and size should be ignored, else
+ * size and stride should be used when setting the format.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_set_format(struct mvx_session *session, enum mvx_direction dir, enum mvx_format format,
+			   unsigned int *width, unsigned int *height, uint8_t *nplanes, unsigned int *stride,
+			   unsigned int *size, bool *interlaced);
+
+/**
+ * mvx_session_qbuf() - Queue a buffer.
+ * @session:	Pointer to session.
+ * @buf:	Pointer to buffer.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_qbuf(struct mvx_session *session, enum mvx_direction dir, struct mvx_buffer *buf);
+
+/**
+ * mvx_session_send_eos() - Queue an empty buffer with EOS flag.
+ * @session:	Pointer to session.
+ *
+ * If firmware is loaded an empty input buffer will be queued with the EOS flag
+ * set. EOS will be propagated by the firmware to the output queue.
+ *
+ * If the firmware is not loaded a buffer will be dequeued from the output
+ * queue, cleared and returned with the EOS flag set.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_send_eos(struct mvx_session *session);
+
+/**
+ * mvx_session_streamon() - Enable stream on input or output port.
+ * @session:	Pointer to session.
+ * @dir:	Port direction.
+ *
+ * Both input and output ports must be enabled for streaming to begin.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_streamon(struct mvx_session *session, enum mvx_direction dir);
+
+/**
+ * mvx_session_streamoff() - Disable stream on input or output port.
+ * @session:	Pointer to session.
+ * @dir:	Port direction.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_streamoff(struct mvx_session *session, enum mvx_direction dir);
+
+/**
+ * mvx_session_irq() - Handle IRQ event from the client.
+ * @isession:	Pointer to if-session.
+ */
+void mvx_session_irq(struct mvx_if_session *isession);
+
+/**
+ * mvx_if_session_to_session() - Convert mvx_is_session to mvx_session.
+ * @session:	Pointer to mvx_if_session object.
+ *
+ * Return: Pointer to mvx_session object.
+ */
+static inline struct mvx_session *mvx_if_session_to_session(struct mvx_if_session *session)
+{
+	return container_of(session, struct mvx_session, isession);
+}
+
+/**
+ * mvx_session_port_show() - Print debug information into seq-file.
+ * @port:	Pointer to port.
+ * @s:		Seq-file to print to.
+ */
+void mvx_session_port_show(struct mvx_session_port *port, struct seq_file *s);
+
+/*
+ * Functions bellow implement different settings for a session.
+ *
+ * Most of options could be set only when the FW is in STOPPED state or not
+ * loaded. In this case the value will be stored in mvx_session structure
+ * and applied lated in fw_initial_setup().
+ *
+ * Some options support runtime modification. For them we issue a command
+ * to mvx_fw module if the FW is loaded. For others we return -EBUSY if the
+ * FW is loaded.
+ *
+ * ATTENTION. Currently there is no way to querry from mvx_fw API of from
+ * mvx_session API if the option supports runtime configuration.
+ */
+
+/**
+ * mvx_session_set_securevideo() - Enabled or disable secure video.
+ * @session:	Session.
+ * @securevideo:Enable or disable secure video.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_set_securevideo(struct mvx_session *session, bool securevideo);
+
+/**
+ * mvx_session_set_frame_rate() - Set frame rate.
+ * @session:	Session.
+ * @frame_rate:	Frame rate in Q16 format.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_frame_rate(struct mvx_session *session, int64_t frame_rate);
+
+/**
+ * mvx_session_set_rate_control() - Enable/disable rate controller.
+ * @session:	Session.
+ * @enabled:	Rate controller status.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_rate_control(struct mvx_session *session, bool enabled);
+
+/**
+ * mvx_session_set_bitrate() - Set bitrate rate.
+ * @session:	Session.
+ * @bitrate:	Bitrate in bits per second.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_bitrate(struct mvx_session *session, int bitrate);
+
+/**
+ * mvx_session_set_bitrate() - Set bitrate rate control.
+ * @session:	Session.
+ * @rc_type:	bitrate rate control type.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+
+int mvx_session_set_bitrate_control(struct mvx_session *session, struct mvx_buffer_param_rate_control *rc_type);
+/**
+ * mvx_session_set_crop_left() - Set crop left.
+ * @session:	Session.
+ * @left:	    encoder SPS crop param, left offset.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+
+int mvx_session_set_crop_left(struct mvx_session *session, int32_t left);
+
+/**
+ * mvx_session_set_crop_right() - Set crop right.
+ * @session:	Session.
+ * @right:	    encoder SPS crop param, right offset.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+
+int mvx_session_set_crop_right(struct mvx_session *session, int32_t right);
+
+/**
+ * mvx_session_set_crop_top() - Set crop top.
+ * @session:	Session.
+ * @top:	    encoder SPS crop param, top offset.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+
+int mvx_session_set_crop_top(struct mvx_session *session, int32_t top);
+
+/**
+ * mvx_session_set_crop_bottom() - Set crop bottom.
+ * @session:	Session.
+ * @top:	    encoder SPS crop param, bottom offset.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+
+int mvx_session_set_crop_bottom(struct mvx_session *session, int32_t bottom);
+/**
+ * mvx_session_set_nalu_format() - Set NALU format.
+ * @session:	Session.
+ * @fmt:	NALU format.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_nalu_format(struct mvx_session *session, enum mvx_nalu_format fmt);
+
+/**
+ * mvx_session_set_stream_escaping() - Enable/disable stream escaping
+ * @session:	Session.
+ * @status:	Status
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_stream_escaping(struct mvx_session *session, enum mvx_tristate status);
+
+/**
+ * mvx_session_set_profile() - Set profile for encoder.
+ * @session:	Session.
+ * @format:	Format.
+ * @profile:	Encoder profile.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_profile(struct mvx_session *session, enum mvx_format format, enum mvx_profile profile);
+
+/**
+ * mvx_session_set_level() - Set level for encoder.
+ *
+ * @session:	Session.
+ * @format:	Format.
+ * @level:	Encoder level.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_level(struct mvx_session *session, enum mvx_format format, enum mvx_level level);
+
+/**
+ * mvx_session_set_ignore_stream_headers() - Enable/disable stream headers
+ * ignore.
+ * @session:	Session.
+ * @status:	Status.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_ignore_stream_headers(struct mvx_session *session, enum mvx_tristate status);
+
+/**
+ * mvx_session_set_frame_reordering() - Enable/disable frames reordering.
+ * @session:	Session.
+ * @status:	Status.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_frame_reordering(struct mvx_session *session, enum mvx_tristate status);
+
+/**
+ * mvx_session_set_intbuf_size() - Set internal buffer size.
+ * @session:	Session.
+ * @size:	Size.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_intbuf_size(struct mvx_session *session, int size);
+
+/**
+ * mvx_session_set_p_frame() - Set number of P-frames.
+ * @session:	Session.
+ * @val:	Number of P-frames.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_p_frames(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_b_frame() - Set number of B-frames.
+ * @session:	Session.
+ * @val:	Number of B-frames.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_b_frames(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_gop_type() - Set GOP type.
+ * @session:	Session.
+ * @gop_type:	GOP type.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_gop_type(struct mvx_session *session, enum mvx_gop_type gop_type);
+
+/**
+ * mvx_session_set_cyclic_intra_refresh_mb() - Set intra MB refresh.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_cyclic_intra_refresh_mb(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_constr_ipred() - Enabled/disable constrained intra
+ * prediction.
+ * @session:	Session.
+ * @status:	Status.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_constr_ipred(struct mvx_session *session, enum mvx_tristate status);
+
+/**
+ * mvx_session_set_entropy_sync() - Enable/disable entropy synchronization.
+ * @session:	Session.
+ * @status:	Status.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_entropy_sync(struct mvx_session *session, enum mvx_tristate status);
+
+/**
+ * mvx_session_set_temporal_mvp() - Enable/disable temporal MVP.
+ * @session:	Session.
+ * @status:	Status.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_temporal_mvp(struct mvx_session *session, enum mvx_tristate status);
+
+/**
+ * mvx_session_set_tile_rows() - Set tile size.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_tile_rows(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_tile_cols() - Set tile size.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_tile_cols(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_min_luma_cb_size() - Set minimum luma coding block size.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_min_luma_cb_size(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_mb_mask() - Set MB mask.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_mb_mask(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_entropy_mode() - Set entropy mode.
+ * @session:	Session.
+ * @mode:	Entropy mode.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_entropy_mode(struct mvx_session *session, enum mvx_entropy_mode mode);
+
+/**
+ * mvx_session_set_multi_slice_mode() - Set multi slice mode.
+ * @session:	Session.
+ * @mode:	Mode.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_multi_slice_mode(struct mvx_session *session, enum mvx_multi_slice_mode mode);
+
+/**
+ * mvx_session_set_multi_slice_max_mb() - Set suggested number of CTUs in a
+ * slice.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_multi_slice_max_mb(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_vp9_prob_update() - Set probability update mode.
+ * @session:	Session.
+ * @mode:	Mode.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_vp9_prob_update(struct mvx_session *session, enum mvx_vp9_prob_update mode);
+
+/**
+ * mvx_session_set_mv_h_search_range() - Set horizontal search range for motion
+ * vectors.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_mv_h_search_range(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_mv_v_search_range() - Set vertical search range for motion
+ * vectors.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_mv_v_search_range(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_bitdepth_chroma() - Set bitdepth.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_bitdepth_chroma(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_bitdepth_luma() - Set bitdepth.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_bitdepth_luma(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_force_chroma_format() - Set chroma format.
+ * @session:	Session.
+ * @fmt:	chroma format.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_force_chroma_format(struct mvx_session *session, int fmt);
+
+/**
+ * mvx_session_set_rgb_to_yuv_mode() - Set RGB to YUV conversion mode.
+ * @session:	Session.
+ * @mode:	Mode.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_rgb_to_yuv_mode(struct mvx_session *session, enum mvx_rgb_to_yuv_mode mode);
+
+/**
+ * mvx_session_set_band_limit() - Set maximum bandwidth limit.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_band_limit(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_cabac_init_idc() - Set CABAC initialization table.
+ * @session:	Session.
+ * @val:	Value.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_cabac_init_idc(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_i_frame_qp() - Set QP for I frames.
+ * @session:	Session.
+ * @format:	Format.
+ * @qp:		Quantization parameter.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_i_frame_qp(struct mvx_session *session, enum mvx_format format, int qp);
+
+/**
+ * mvx_session_set_p_frame_qp() - Set QP for P frames.
+ * @session:	Session.
+ * @format:	Format.
+ * @qp:		Quantization parameter.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_p_frame_qp(struct mvx_session *session, enum mvx_format format, int qp);
+
+/**
+ * mvx_session_set_b_frame_qp() - Set QP for B frames.
+ * @session:	Session.
+ * @format:	Format.
+ * @qp:		Quantization parameter.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_b_frame_qp(struct mvx_session *session, enum mvx_format format, int qp);
+
+/**
+ * mvx_session_set_min_qp() - Set minimum value of QP range.
+ * @session:	Session.
+ * @format:	Format.
+ * @qp:		Quantization parameter.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_min_qp(struct mvx_session *session, enum mvx_format format, int qp);
+
+/**
+ * mvx_session_set_max_qp() - Set maximum value of QP range.
+ * @session:	Session.
+ * @format:	Format.
+ * @qp:		Quantization parameter.
+ *
+ * This option could be set in runtime.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_max_qp(struct mvx_session *session, enum mvx_format format, int qp);
+
+/**
+ * mvx_session_set_resync_interval() - Set resync interval for JPEG encoder.
+ * @session:	Session.
+ * @val:	Resync interval.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+int mvx_session_set_resync_interval(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_set_jpeg_quality() - Set JPEG quality.
+ * @session:	Session.
+ * @val:	Quality level (1-100).
+ *
+ * Return: 0 in case of success, error otherwise.
+ */
+int mvx_session_set_jpeg_quality(struct mvx_session *session, int val);
+
+/**
+ * mvx_session_get_color_desc() - Get color description.
+ * @session:	Pointer to session.
+ * @color_desc:	Color description.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_get_color_desc(struct mvx_session *session, struct mvx_fw_color_desc *color_desc);
+
+/**
+ * mvx_session_set_color_desc() - Set color description.
+ * @session:	Pointer to session.
+ * @color_desc:	Color description.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_set_color_desc(struct mvx_session *session, struct mvx_fw_color_desc *color_desc);
+
+/**
+ * mvx_session_set_roi_regions() - Set ROI regions.
+ * @vsession:	Pointer to v4l2 session.
+ * @roi:	ROI regions.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_session_set_roi_regions(struct mvx_session *session, struct mvx_roi_config *roi);
+
+/**
+ * mvx_session_set_qp_epr() - Set qp for epr config.
+ * @vsession:	Pointer to v4l2 session.
+ * @qp:	qp.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_session_set_qp_epr(struct mvx_session *session, int *qp);
+
+/**
+ * mvx_session_set_sei_userdata() - Set SEI userdata.
+ * @vsession:	Pointer to v4l2 session.
+ * @userdata:	SEI userdata.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_session_set_sei_userdata(struct mvx_session *session, struct mvx_sei_userdata *userdata);
+
+/**
+ * mvx_session_set_hrd_buffer_size() - Set hrd buffer size.
+ * @vsession:	Pointer to v4l2 session.
+ * @size:	hrd buffer size.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_session_set_hrd_buffer_size(struct mvx_session *session, int size);
+
+/**
+ * mvx_session_set_dsl_frame() - Set DownScale dst frame.
+ * @vsession:	Pointer to v4l2 session.
+ * @dsl:	DownScale dst frame.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_session_set_dsl_frame(struct mvx_session *session, struct mvx_dsl_frame *dsl);
+
+/**
+ * mvx_session_set_dsl_ratio() - Set DownScale ratio.
+ * @vsession:	Pointer to v4l2 session.
+ * @dsl:	DownScale ratio.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_session_set_dsl_ratio(struct mvx_session *session, struct mvx_dsl_ratio *dsl);
+
+/**
+ * mvx_session_set_long_term_ref() - Set long term ref.
+ * @vsession:	Pointer to v4l2 session.
+ * @ltr:	long term ref.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_session_set_long_term_ref(struct mvx_session *session, struct mvx_long_term_ref *ltr);
+
+/**
+ * mvx_session_set_dsl_mode() - Set DownScale mode.
+ * @vsession:	Pointer to v4l2 session.
+ * @mode:	DownScale mode, oly enable on high precision mode.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_session_set_dsl_mode(struct mvx_session *session, int *mode);
+
+/*force insert IDR frame.*/
+int mvx_session_set_force_idr(struct mvx_session *session);
+
+/*set timeout value for watchdog*/
+int mvx_session_set_watchdog_timeout(struct mvx_session *session, int timeout);
+
+/*enable/disable profiling for vpu*/
+int mvx_session_set_profiling(struct mvx_session *session, int enable);
+
+void generate_quant_tbl(int quality, const uint8_t qtbl_ref[MVX_FW_QUANT_LEN], uint8_t qtbl[MVX_FW_QUANT_LEN]);
+
+#endif /* _MVX_SESSION_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx-v4l2-controls.h b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx-v4l2-controls.h
new file mode 100644
index 000000000000..1ae0fa1169f5
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx-v4l2-controls.h
@@ -0,0 +1,454 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_V4L2_CONTROLS_H_
+#define _MVX_V4L2_CONTROLS_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/videodev2.h>
+#include <linux/v4l2-controls.h>
+#include <drm/drm_fourcc.h>
+
+/****************************************************************************
+ * Pixel formats
+ ****************************************************************************/
+
+#define V4L2_PIX_FMT_YUV420_AFBC_8 v4l2_fourcc('Y', '0', 'A', '8')
+#define V4L2_PIX_FMT_YUV420_AFBC_10 v4l2_fourcc('Y', '0', 'A', 'A')
+#define V4L2_PIX_FMT_YUV422_AFBC_8 v4l2_fourcc('Y', '2', 'A', '8')
+#define V4L2_PIX_FMT_YUV422_AFBC_10 v4l2_fourcc('Y', '2', 'A', 'A')
+#define V4L2_PIX_FMT_Y210 v4l2_fourcc('Y', '2', '1', '0')
+// #define V4L2_PIX_FMT_P010            v4l2_fourcc('Y', '0', 'P', '1')
+#define V4L2_PIX_FMT_Y0L2 v4l2_fourcc('Y', '0', 'Y', 'L')
+#define V4L2_PIX_FMT_RV v4l2_fourcc('R', 'V', '0', '0')
+
+#ifndef V4L2_PIX_FMT_HEVC
+#define V4L2_PIX_FMT_HEVC v4l2_fourcc('H', 'E', 'V', 'C')
+#endif
+
+#ifndef V4L2_PIX_FMT_VP9
+#define V4L2_PIX_FMT_VP9 v4l2_fourcc('V', 'P', '9', '0')
+#endif
+
+#define V4L2_PIX_FMT_AVS v4l2_fourcc('A', 'V', 'S', '1')
+#define V4L2_PIX_FMT_AVS2 v4l2_fourcc('A', 'V', 'S', '2')
+
+/****************************************************************************
+ * Buffers
+ * @see v4l2_buffer
+ ****************************************************************************/
+
+/*
+ * Extended buffer flags.
+ */
+/*
+#define V4L2_BUF_FLAG_MVX_DECODE_ONLY           0x01000000
+#define V4L2_BUF_FLAG_MVX_CODEC_CONFIG          0x02000000
+#define V4L2_BUF_FLAG_MVX_AFBC_TILED_HEADERS    0x10000000
+#define V4L2_BUF_FLAG_MVX_AFBC_TILED_BODY       0x20000000
+#define V4L2_BUF_FLAG_MVX_AFBC_32X8_SUPERBLOCK  0x40000000
+#define V4L2_BUF_FLAG_MVX_MASK                  0xff000000
+#define V4L2_BUF_FLAG_END_OF_SUB_FRAME          0x04000000
+#define V4L2_BUF_FLAG_MVX_BUFFER_FRAME_PRESENT  0x08000000
+#define V4L2_BUF_FLAG_MVX_BUFFER_NEED_REALLOC   0x07000000
+
+
+#define V4L2_BUF_FRAME_FLAG_ROTATION_90   0x81000000
+#define V4L2_BUF_FRAME_FLAG_ROTATION_180  0x82000000
+#define V4L2_BUF_FRAME_FLAG_ROTATION_270  0x83000000
+#define V4L2_BUF_FRAME_FLAG_ROTATION_MASK 0x83000000
+#define V4L2_BUF_FRAME_FLAG_MIRROR_HORI   0x90000000
+#define V4L2_BUF_FRAME_FLAG_MIRROR_VERT   0xA0000000
+#define V4L2_BUF_FRAME_FLAG_MIRROR_MASK   0xB0000000
+#define V4L2_BUF_FRAME_FLAG_SCALING_2     0x84000000
+#define V4L2_BUF_FRAME_FLAG_SCALING_4     0x88000000
+#define V4L2_BUF_FRAME_FLAG_SCALING_MASK  0x8C000000
+
+#define V4L2_BUF_FLAG_MVX_BUFFER_EPR      0xC0000000
+#define V4L2_BUF_FLAG_MVX_BUFFER_ROI      0x70000000
+*/
+// redefine these flags
+/*use encode/decode frame/bitstream to update these flags*/
+
+#define V4L2_BUF_FLAG_MVX_MASK 0xff000000
+
+// for decode frame flag
+#define V4L2_BUF_FRAME_FLAG_ROTATION_90 0x01000000  /* Frame is rotated 90 degrees */
+#define V4L2_BUF_FRAME_FLAG_ROTATION_180 0x02000000 /* Frame is rotated 180 degrees */
+#define V4L2_BUF_FRAME_FLAG_ROTATION_270 0x03000000 /* Frame is rotated 270 degrees */
+#define V4L2_BUF_FRAME_FLAG_ROTATION_MASK 0x03000000
+#define V4L2_BUF_FRAME_FLAG_SCALING_2 0x04000000 /* Frame is scaled by half */
+#define V4L2_BUF_FRAME_FLAG_SCALING_4 0x08000000 /* Frame is scaled by quarter */
+#define V4L2_BUF_FRAME_FLAG_SCALING_MASK 0x0C000000
+#define V4L2_BUF_FLAG_MVX_BUFFER_FRAME_PRESENT 0x10000000
+#define V4L2_BUF_FLAG_MVX_BUFFER_NEED_REALLOC 0x20000000
+
+// for decode bitstream flag
+#define V4L2_BUF_FLAG_MVX_CODEC_CONFIG 0xC1000000
+#define V4L2_BUF_FLAG_END_OF_SUB_FRAME 0xC2000000
+#define V4L2_BUF_FLAG_MVX_DECODE_ONLY 0xC4000000
+
+// for encode frame flag
+#define V4L2_BUF_FRAME_FLAG_MIRROR_HORI 0x81000000
+#define V4L2_BUF_FRAME_FLAG_MIRROR_VERT 0x82000000
+#define V4L2_BUF_FRAME_FLAG_MIRROR_MASK 0x83000000
+#define V4L2_BUF_FLAG_MVX_BUFFER_ROI 0x84000000 /* this buffer has a roi region */
+#define V4L2_BUF_FLAG_MVX_BUFFER_EPR 0x88000000 /* EPR buffer flag */
+
+// afbc flag
+#define V4L2_BUF_FLAG_MVX_AFBC_TILED_HEADERS 0x01000000
+#define V4L2_BUF_FLAG_MVX_AFBC_TILED_BODY 0x02000000
+#define V4L2_BUF_FLAG_MVX_AFBC_32X8_SUPERBLOCK 0x04000000
+#define V4L2_BUF_FLAG_MVX_AFBC_BLOCK_SPLIT 0x08000000
+
+#define V4L2_BUF_FLAG_MVX_DISABLE_CACHE_MAINTENANCE 0x50000000 /*disable cache maintenance for buffer.*/
+
+/****************************************************************************
+ * HDR color description.
+ ****************************************************************************/
+
+#define V4L2_EVENT_MVX_COLOR_DESC V4L2_EVENT_PRIVATE_START
+#define V4L2_MVX_MAX_FRAME_REGIONS 16
+
+enum v4l2_mvx_range {
+	V4L2_MVX_RANGE_UNSPECIFIED,
+	V4L2_MVX_RANGE_FULL,
+	V4L2_MVX_RANGE_LIMITED
+};
+
+enum v4l2_mvx_primaries {
+	V4L2_MVX_PRIMARIES_UNSPECIFIED,
+	V4L2_MVX_PRIMARIES_BT709,	 /* Rec.ITU-R BT.709 */
+	V4L2_MVX_PRIMARIES_BT470M,	 /* Rec.ITU-R BT.470 System M */
+	V4L2_MVX_PRIMARIES_BT601_625,	 /* Rec.ITU-R BT.601 625 */
+	V4L2_MVX_PRIMARIES_BT601_525,	 /* Rec.ITU-R BT.601 525 */
+	V4L2_MVX_PRIMARIES_GENERIC_FILM, /* Generic Film */
+	V4L2_MVX_PRIMARIES_BT2020	 /* Rec.ITU-R BT.2020 */
+};
+
+enum v4l2_mvx_transfer {
+	V4L2_MVX_TRANSFER_UNSPECIFIED,
+	V4L2_MVX_TRANSFER_LINEAR,    /* Linear transfer characteristics */
+	V4L2_MVX_TRANSFER_SRGB,	     /* sRGB */
+	V4L2_MVX_TRANSFER_SMPTE170M, /* SMPTE 170M */
+	V4L2_MVX_TRANSFER_GAMMA22,   /* Assumed display gamma 2.2 */
+	V4L2_MVX_TRANSFER_GAMMA28,   /* Assumed display gamma 2.8 */
+	V4L2_MVX_TRANSFER_ST2084,    /* SMPTE ST 2084 */
+	V4L2_MVX_TRANSFER_HLG,	     /* ARIB STD-B67 hybrid-log-gamma */
+	V4L2_MVX_TRANSFER_SMPTE240M, /* SMPTE 240M */
+	V4L2_MVX_TRANSFER_XVYCC,     /* IEC 61966-2-4 */
+	V4L2_MVX_TRANSFER_BT1361,    /* Rec.ITU-R BT.1361 extended gamut */
+	V4L2_MVX_TRANSFER_ST428	     /* SMPTE ST 428-1 */
+};
+
+enum v4l2_mvx_matrix {
+	V4L2_MVX_MATRIX_UNSPECIFIED,
+	V4L2_MVX_MATRIX_BT709,	       /* Rec.ITU-R BT.709 */
+	V4L2_MVX_MATRIX_BT470M,	       /* KR=0.30, KB=0.11 */
+	V4L2_MVX_MATRIX_BT601,	       /* Rec.ITU-R BT.601 625 */
+	V4L2_MVX_MATRIX_SMPTE240M,     /* SMPTE 240M or equivalent */
+	V4L2_MVX_MATRIX_BT2020,	       /* Rec.ITU-R BT.2020 non-const lum */
+	V4L2_MVX_MATRIX_BT2020Constant /* Rec.ITU-R BT.2020 constant lum */
+};
+
+enum v4l2_nalu_format {
+	V4L2_OPT_NALU_FORMAT_START_CODES,
+	V4L2_OPT_NALU_FORMAT_ONE_NALU_PER_BUFFER,
+	V4L2_OPT_NALU_FORMAT_ONE_BYTE_LENGTH_FIELD,
+	V4L2_OPT_NALU_FORMAT_TWO_BYTE_LENGTH_FIELD,
+	V4L2_OPT_NALU_FORMAT_FOUR_BYTE_LENGTH_FIELD,
+	V4L2_OPT_NALU_FORMAT_ONE_FRAME_PER_BUFFER
+};
+
+struct v4l2_mvx_primary {
+	uint16_t x;
+	uint16_t y;
+};
+
+/**
+ * struct v4l2_mvx_color_desc - HDR color description.
+ * @flags:			Flags which fields that are valid.
+ * @range:			enum v4l2_mvx_range.
+ * @primaries:			enum v4l2_mvx_primaries.
+ * @transfer:			enum v4l2_mvx_transfer.
+ * @matrix:			enum v4l2_mvx_matrix.
+ * @display.r:			Red point.
+ * @display.g:			Green point.
+ * @display.b:			Blue point.
+ * @display.w:			White point.
+ * @display.luminance_min:	Minimum display luminance.
+ * @display.luminance_max:	Maximum display luminance.
+ * @content.luminance_max:	Maximum content luminance.
+ * @content.luminance_average:	Average content luminance.
+ *
+ * Color- and white point primaries are given in increments of 0.00002
+ * and in the range of 0 to 50'000.
+ *
+ * Luminance is given in increments of 0.0001 candelas per m3.
+ */
+struct v4l2_mvx_color_desc {
+	uint32_t flags;
+#define V4L2_BUFFER_PARAM_COLOUR_FLAG_MASTERING_DISPLAY_DATA_VALID (1)
+#define V4L2_BUFFER_PARAM_COLOUR_FLAG_CONTENT_LIGHT_DATA_VALID (2)
+	uint8_t range;
+	uint8_t primaries;
+	uint8_t transfer;
+	uint8_t matrix;
+	struct {
+		struct v4l2_mvx_primary r;
+		struct v4l2_mvx_primary g;
+		struct v4l2_mvx_primary b;
+		struct v4l2_mvx_primary w;
+		uint16_t luminance_min;
+		uint16_t luminance_max;
+	} display;
+	struct {
+		uint16_t luminance_max;
+		uint16_t luminance_average;
+	} content;
+
+	uint8_t video_format;
+	uint8_t aspect_ratio_idc;
+	uint16_t sar_width;
+	uint16_t sar_height;
+	uint32_t num_units_in_tick;
+	uint32_t time_scale;
+} __attribute__((packed));
+
+struct v4l2_buffer_param_region {
+	uint16_t mbx_left;   /**< X coordinate of the left most macroblock */
+	uint16_t mbx_right;  /**< X coordinate of the right most macroblock */
+	uint16_t mby_top;    /**< Y coordinate of the top most macroblock */
+	uint16_t mby_bottom; /**< Y coordinate of the bottom most macroblock */
+	int16_t qp_delta;    /**< QP delta value. This region will be encoded
+			      *   with qp = qp_default + qp_delta. */
+};
+
+struct v4l2_mvx_roi_regions {
+	unsigned int pic_index;
+	unsigned char qp_present;
+	unsigned char qp;
+	unsigned char roi_present;
+	unsigned char num_roi;
+	struct v4l2_buffer_param_region roi[V4L2_MVX_MAX_FRAME_REGIONS];
+};
+
+struct v4l2_sei_user_data {
+	uint8_t flags;
+#define V4L2_BUFFER_PARAM_USER_DATA_UNREGISTERED_VALID (1)
+	uint8_t uuid[16];
+	char user_data[256 - 35];
+	uint8_t user_data_len;
+};
+
+struct v4l2_rate_control {
+	uint32_t rc_type;
+#define V4L2_OPT_RATE_CONTROL_MODE_OFF (0)
+#define V4L2_OPT_RATE_CONTROL_MODE_STANDARD (1)
+#define V4L2_OPT_RATE_CONTROL_MODE_VARIABLE (2)
+#define V4L2_OPT_RATE_CONTROL_MODE_CONSTANT (3)
+#define V4L2_OPT_RATE_CONTROL_MODE_C_VARIABLE (4)
+	uint32_t target_bitrate;
+	uint32_t maximum_bitrate;
+};
+
+struct v4l2_mvx_dsl_frame {
+	uint32_t width;
+	uint32_t height;
+};
+
+struct v4l2_mvx_dsl_ratio {
+	uint32_t hor;
+	uint32_t ver;
+};
+
+struct v4l2_mvx_long_term_ref {
+	uint32_t mode;
+	uint32_t period;
+};
+
+#define V4L2_MVX_COLOR_DESC_DISPLAY_VALID 0x1
+#define V4L2_MVX_COLOR_DESC_CONTENT_VALID 0x2
+
+/****************************************************************************
+ * Custom IOCTL
+ ****************************************************************************/
+
+#define VIDIOC_G_MVX_COLORDESC _IOWR('V', BASE_VIDIOC_PRIVATE, struct v4l2_mvx_color_desc)
+#define VIDIOC_S_MVX_ROI_REGIONS _IOWR('V', BASE_VIDIOC_PRIVATE + 1, struct v4l2_mvx_roi_regions)
+#define VIDIOC_S_MVX_QP_EPR _IOWR('V', BASE_VIDIOC_PRIVATE + 2, int)
+#define VIDIOC_S_MVX_COLORDESC _IOWR('V', BASE_VIDIOC_PRIVATE + 3, struct v4l2_mvx_color_desc)
+#define VIDIOC_S_MVX_SEI_USERDATA _IOWR('V', BASE_VIDIOC_PRIVATE + 4, struct v4l2_sei_user_data)
+#define VIDIOC_S_MVX_RATE_CONTROL _IOWR('V', BASE_VIDIOC_PRIVATE + 5, struct v4l2_rate_control)
+#define VIDIOC_S_MVX_DSL_FRAME _IOWR('V', BASE_VIDIOC_PRIVATE + 6, struct v4l2_mvx_dsl_frame)
+#define VIDIOC_S_MVX_DSL_RATIO _IOWR('V', BASE_VIDIOC_PRIVATE + 7, struct v4l2_mvx_dsl_ratio)
+#define VIDIOC_S_MVX_LONG_TERM_REF _IOWR('V', BASE_VIDIOC_PRIVATE + 8, struct v4l2_mvx_long_term_ref)
+#define VIDIOC_S_MVX_DSL_MODE _IOWR('V', BASE_VIDIOC_PRIVATE + 9, int)
+/****************************************************************************
+ * Custom controls
+ ****************************************************************************/
+
+/*
+ * Video for Linux 2 custom controls.
+ */
+enum v4l2_cid_mve_video {
+	V4L2_CID_MVE_VIDEO_FRAME_RATE = V4L2_CTRL_CLASS_CODEC + 0x2000,
+	V4L2_CID_MVE_VIDEO_NALU_FORMAT,
+	V4L2_CID_MVE_VIDEO_STREAM_ESCAPING,
+	V4L2_CID_MVE_VIDEO_H265_PROFILE,
+	V4L2_CID_MVE_VIDEO_VC1_PROFILE,
+	V4L2_CID_MVE_VIDEO_H265_LEVEL,
+	V4L2_CID_MVE_VIDEO_IGNORE_STREAM_HEADERS,
+	V4L2_CID_MVE_VIDEO_FRAME_REORDERING,
+	V4L2_CID_MVE_VIDEO_INTBUF_SIZE,
+	V4L2_CID_MVE_VIDEO_P_FRAMES,
+	V4L2_CID_MVE_VIDEO_GOP_TYPE,
+	V4L2_CID_MVE_VIDEO_CONSTR_IPRED,
+	V4L2_CID_MVE_VIDEO_ENTROPY_SYNC,
+	V4L2_CID_MVE_VIDEO_TEMPORAL_MVP,
+	V4L2_CID_MVE_VIDEO_TILE_ROWS,
+	V4L2_CID_MVE_VIDEO_TILE_COLS,
+	V4L2_CID_MVE_VIDEO_MIN_LUMA_CB_SIZE,
+	V4L2_CID_MVE_VIDEO_MB_MASK,
+	V4L2_CID_MVE_VIDEO_VP9_PROB_UPDATE,
+	V4L2_CID_MVE_VIDEO_BITDEPTH_CHROMA,
+	V4L2_CID_MVE_VIDEO_BITDEPTH_LUMA,
+	V4L2_CID_MVE_VIDEO_FORCE_CHROMA_FORMAT,
+	V4L2_CID_MVE_VIDEO_RGB_TO_YUV_MODE,
+	V4L2_CID_MVE_VIDEO_BANDWIDTH_LIMIT,
+	V4L2_CID_MVE_VIDEO_CABAC_INIT_IDC,
+	V4L2_CID_MVE_VIDEO_VPX_B_FRAME_QP,
+	V4L2_CID_MVE_VIDEO_SECURE_VIDEO,
+	V4L2_CID_MVE_VIDEO_CROP_LEFT,
+	V4L2_CID_MVE_VIDEO_CROP_RIGHT,
+	V4L2_CID_MVE_VIDEO_CROP_TOP,
+	V4L2_CID_MVE_VIDEO_CROP_BOTTOM,
+	V4L2_CID_MVE_VIDEO_HRD_BUFFER_SIZE,
+	V4L2_CID_MVE_VIDEO_WATCHDOG_TIMEOUT,
+	V4L2_CID_MVE_VIDEO_PROFILING
+};
+
+/* block configuration uncompressed rows header. this configures the size of the
+ * uncompressed body. */
+struct v4l2_buffer_general_rows_uncomp_hdr {
+	uint8_t n_cols_minus1; /* number of quad cols in picture minus 1 */
+	uint8_t n_rows_minus1; /* number of quad rows in picture minus 1 */
+	uint8_t reserved[2];
+};
+
+struct v4l2_buffer_general_block_configs {
+	uint8_t blk_cfg_type;
+#define V4L2_BLOCK_CONFIGS_TYPE_NONE (0x00)
+#define V4L2_BLOCK_CONFIGS_TYPE_ROW_UNCOMP (0xff)
+	uint8_t reserved[3];
+	union {
+		struct v4l2_buffer_general_rows_uncomp_hdr rows_uncomp;
+	} blk_cfgs;
+};
+
+/* input for encoder */
+struct v4l2_buffer_param_qp {
+	/* QP (quantization parameter) for encode.
+	 *
+	 * When used to set fixed QP for encode, with rate control
+	 * disabled, then the valid ranges are:
+	 *   H264: 0-51
+	 *   HEVC: 0-51
+	 *   VP8:  0-63
+	 *   VP9:  0-63
+	 * Note: The QP must be set separately for I, P and B frames.
+	 *
+	 * But when this message is used with the regions-feature,
+	 * then the valid ranges are the internal bitstream ranges:
+	 *   H264: 0-51
+	 *   HEVC: 0-51
+	 *   VP8:  0-127
+	 *   VP9:  0-255
+	 */
+	int32_t qp;
+};
+
+/* the block parameter record specifies the various properties of a quad */
+struct v4l2_block_param_record {
+	uint16_t qp_delta;
+/* Bitset of four 4-bit QP delta values for a quad.
+ * For H.264 and HEVC these are qp delta values in the range -8 to +7.
+ * For Vp9 these are segment map values in the range 0 to 7.
+ */
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_TOP_LEFT_16X16 (0)
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_TOP_LEFT_16X16_SZ (4)
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_TOP_RIGHT_16X16 (4)
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_TOP_RIGHT_16X16_SZ (4)
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_BOT_LEFT_16X16 (8)
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_BOT_LEFT_16X16_SZ (4)
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_BOT_RIGHT_16X16 (12)
+#define V4L2_BLOCK_PARAM_RECORD_QP_DELTA_BOT_RIGHT_16X16_SZ (4)
+
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_LEFT_16X16 (0)
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_LEFT_16X16_SZ (3)
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_RIGHT_16X16 (4)
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_TOP_RIGHT_16X16_SZ (3)
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_LEFT_16X16 (8)
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_LEFT_16X16_SZ (3)
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_RIGHT_16X16 (12)
+#define V4L2_BLOCK_PARAM_RECORD_VP9_SEGID_BOT_RIGHT_16X16_SZ (3)
+
+	uint8_t force;
+#define V4L2_BLOCK_PARAM_RECORD_FORCE_NONE (0x00)
+#define V4L2_BLOCK_PARAM_RECORD_FORCE_QP (0x01)
+#define V4L2_BLOCK_PARAM_RECORD_FORCE_32X32 (0x02)
+#define V4L2_BLOCK_PARAM_RECORD_FORCE_RB (0x04)
+
+	uint8_t reserved;
+};
+
+struct v4l2_buffer_general_rows_uncomp_body {
+	/* the size of this array is variable and not necessarily equal to 1.
+	 * therefore the sizeof operator should not be used
+	 */
+	struct v4l2_block_param_record bpr[1];
+};
+
+struct v4l2_core_buffer_header_general {
+	// uint64_t user_data_tag;   // User supplied tracking identifier
+	// uint64_t app_handle;    // Host buffer handle number
+	uint16_t type;				   // type of config, value is one of V4L2_BUFFER_GENERAL_TYPE_X
+#define V4L2_BUFFER_GENERAL_TYPE_BLOCK_CONFIGS (1) /* block_configs */
+#define V4L2_BUFFER_GENERAL_TYPE_ENCODER_STATS (4) /* encoder_stats */
+	uint16_t config_size;			   // size of the configuration
+	uint32_t buffer_size;
+	struct v4l2_buffer_general_block_configs config;
+};
+
+#endif /* _MVX_V4L2_CONTROLS_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_ext_if.h b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_ext_if.h
new file mode 100644
index 000000000000..b9eadff79097
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_ext_if.h
@@ -0,0 +1,84 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_EXT_H_
+#define _MVX_EXT_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/mutex.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-device.h>
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct mvx_csched;
+struct mvx_fw_cache;
+
+struct mvx_ext_if {
+	struct mutex lock;
+	struct device *dev;
+	struct mvx_fw_cache *cache;
+	struct mvx_client_ops *client_ops;
+	struct video_device vdev;
+	struct v4l2_device v4l2_dev;
+	struct dentry *dsessions;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_ext_if_construct() - Construct the external interface object.
+ * @ext:	Pointer to interface object.
+ * @dev:	Pointer to device struct.
+ * @cache:	Pointer to firmware cache.
+ * @client_ops:	Pointer to client client_ops.
+ * @parent:	Parent debugfs directory entry.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_ext_if_construct(struct mvx_ext_if *ext, struct device *dev, struct mvx_fw_cache *cache,
+			 struct mvx_client_ops *client_ops, struct dentry *parent);
+
+/**
+ * mvx_ext_if_destruct() - Destroy external interface instance.
+ * @ext:   Pointer to interface object.
+ */
+void mvx_ext_if_destruct(struct mvx_ext_if *ext);
+
+#endif /* _MVX_EXT_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_ext_v4l2.c b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_ext_v4l2.c
new file mode 100644
index 000000000000..14057d2868b6
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_ext_v4l2.c
@@ -0,0 +1,173 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/videodev2.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-ioctl.h>
+#include "mvx_bitops.h"
+#include "mvx_buffer.h"
+#include "mvx_ext_if.h"
+#include "mvx_firmware.h"
+#include "mvx_if.h"
+#include "mvx_mmu.h"
+#include "mvx_session.h"
+#include "mvx_v4l2_buffer.h"
+#include "mvx_v4l2_session.h"
+#include "mvx_v4l2_vidioc.h"
+#include "mvx_v4l2_fops.h"
+#include "mvx_log_group.h"
+
+static const struct v4l2_file_operations mvx_v4l2_fops = {
+	.owner          = THIS_MODULE,
+	.open           = mvx_v4l2_open,
+	.release        = mvx_v4l2_release,
+	.poll           = mvx_v4l2_poll,
+	.unlocked_ioctl = video_ioctl2,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl32 = video_ioctl2, /* for mvx_v4l2_vidioc_default() */
+#endif
+	.mmap           = mvx_v4l2_mmap
+};
+
+static const struct v4l2_ioctl_ops mvx_v4l2_ioctl_ops = {
+	.vidioc_querycap = mvx_v4l2_vidioc_querycap,
+	.vidioc_enum_fmt_vid_cap = mvx_v4l2_vidioc_enum_fmt_vid_cap,
+	.vidioc_enum_fmt_vid_out = mvx_v4l2_vidioc_enum_fmt_vid_out,
+	//.vidioc_enum_fmt_vid_cap_mplane = mvx_v4l2_vidioc_enum_fmt_vid_cap,
+	//.vidioc_enum_fmt_vid_out_mplane = mvx_v4l2_vidioc_enum_fmt_vid_out,
+	.vidioc_enum_framesizes = mvx_v4l2_vidioc_enum_framesizes,
+	.vidioc_g_fmt_vid_cap = mvx_v4l2_vidioc_g_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap_mplane = mvx_v4l2_vidioc_g_fmt_vid_cap,
+	.vidioc_g_fmt_vid_out = mvx_v4l2_vidioc_g_fmt_vid_out,
+	.vidioc_g_fmt_vid_out_mplane = mvx_v4l2_vidioc_g_fmt_vid_out,
+	.vidioc_s_fmt_vid_cap = mvx_v4l2_vidioc_s_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap_mplane = mvx_v4l2_vidioc_s_fmt_vid_cap,
+	.vidioc_s_fmt_vid_out = mvx_v4l2_vidioc_s_fmt_vid_out,
+	.vidioc_s_fmt_vid_out_mplane = mvx_v4l2_vidioc_s_fmt_vid_out,
+	.vidioc_try_fmt_vid_cap = mvx_v4l2_vidioc_try_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap_mplane = mvx_v4l2_vidioc_try_fmt_vid_cap,
+	.vidioc_try_fmt_vid_out = mvx_v4l2_vidioc_try_fmt_vid_out,
+	.vidioc_try_fmt_vid_out_mplane = mvx_v4l2_vidioc_try_fmt_vid_out,
+	.vidioc_g_selection = mvx_v4l2_vidioc_g_selection,
+	.vidioc_s_selection = mvx_v4l2_vidioc_s_selection,
+	.vidioc_streamon = mvx_v4l2_vidioc_streamon,
+	.vidioc_streamoff = mvx_v4l2_vidioc_streamoff,
+	.vidioc_encoder_cmd = mvx_v4l2_vidioc_encoder_cmd,
+	.vidioc_try_encoder_cmd = mvx_v4l2_vidioc_try_encoder_cmd,
+	.vidioc_decoder_cmd = mvx_v4l2_vidioc_decoder_cmd,
+	.vidioc_try_decoder_cmd = mvx_v4l2_vidioc_try_decoder_cmd,
+	.vidioc_reqbufs = mvx_v4l2_vidioc_reqbufs,
+	.vidioc_create_bufs = mvx_v4l2_vidioc_create_bufs,
+	.vidioc_querybuf = mvx_v4l2_vidioc_querybuf,
+	.vidioc_qbuf = mvx_v4l2_vidioc_qbuf,
+	.vidioc_dqbuf = mvx_v4l2_vidioc_dqbuf,
+	.vidioc_subscribe_event = mvx_v4l2_vidioc_subscribe_event,
+	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+	.vidioc_default = mvx_v4l2_vidioc_default,
+	.vidioc_g_parm = mvx_v4l2_vidioc_g_parm,
+	.vidioc_s_parm = mvx_v4l2_vidioc_s_parm,
+};
+
+/****************************************************************************
+ * Exported functions and variables
+ ****************************************************************************/
+
+int mvx_ext_if_construct(struct mvx_ext_if *ext, struct device *dev, struct mvx_fw_cache *cache,
+			 struct mvx_client_ops *client_ops, struct dentry *parent)
+{
+	int ret;
+	const char name[] = "mvx";
+
+	ext->dev = dev;
+	ext->cache = cache;
+	ext->client_ops = client_ops;
+	mutex_init(&ext->lock);
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ext->dsessions = debugfs_create_dir("session", parent);
+		if (IS_ERR_OR_NULL(ext->dsessions))
+			return -ENOMEM;
+	}
+
+	ret = v4l2_device_register(dev, &ext->v4l2_dev);
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "Failed to register V4L2 device. ret=%d.", ret);
+		goto delete_dentry;
+	}
+
+	/* Video device. */
+	ext->vdev.fops = &mvx_v4l2_fops;
+	ext->vdev.ioctl_ops = &mvx_v4l2_ioctl_ops;
+	ext->vdev.release = video_device_release_empty;
+	ext->vdev.vfl_dir = VFL_DIR_M2M;
+	ext->vdev.v4l2_dev = &ext->v4l2_dev;
+	ext->vdev.device_caps =
+		V4L2_CAP_VIDEO_M2M | V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_EXT_PIX_FORMAT | V4L2_CAP_STREAMING;
+	strncpy(ext->vdev.name, name, sizeof(ext->vdev.name));
+
+	video_set_drvdata(&ext->vdev, ext);
+
+	ret = video_register_device(&ext->vdev, VFL_TYPE_VIDEO, -1);
+	if (ret != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_ERROR, "Failed to register video device. ret=%d.", ret);
+		goto unregister_device;
+	}
+
+	return 0;
+
+unregister_device:
+	v4l2_device_unregister(&ext->v4l2_dev);
+
+delete_dentry:
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(ext->dsessions);
+
+	return ret;
+}
+
+void mvx_ext_if_destruct(struct mvx_ext_if *ext)
+{
+	video_unregister_device(&ext->vdev);
+	v4l2_device_unregister(&ext->v4l2_dev);
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(ext->dsessions);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_buffer.c b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_buffer.c
new file mode 100644
index 000000000000..01ab65992eba
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_buffer.c
@@ -0,0 +1,490 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <linux/debugfs.h>
+#include <linux/time.h>
+#include <linux/version.h>
+#include "mvx_ext_if.h"
+#include "mvx_log_group.h"
+#include "mvx_seq.h"
+#include "mvx_v4l2_buffer.h"
+#include "mvx-v4l2-controls.h"
+
+/****************************************************************************
+ * Static functions and variables
+ ****************************************************************************/
+
+static void v4l2_buffer_show(struct mvx_v4l2_buffer *buffer, struct seq_file *s)
+{
+	struct vb2_v4l2_buffer *v4l2 = &buffer->vb2_v4l2_buffer;
+	struct vb2_buffer *vb2 = &v4l2->vb2_buf;
+	int is_multi = V4L2_TYPE_IS_MULTIPLANAR(vb2->type);
+	int i;
+	int ind = 0;
+
+	mvx_seq_printf(s, "mvx_v4l2_buffer", ind, "%p\n", buffer);
+
+	ind++;
+	mvx_seq_printf(s, "vb2", ind, "%p\n", vb2);
+
+	ind++;
+	mvx_seq_printf(s, "index", ind, "%u\n", vb2->index);
+	mvx_seq_printf(s, "type", ind, "%u (multi: %s)\n", vb2->type, is_multi ? "yes" : "no");
+	mvx_seq_printf(s, "flags", ind, "0x%08x\n", v4l2->flags);
+	mvx_seq_printf(s, "field", ind, "%u\n", v4l2->field);
+
+#if KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE
+	mvx_seq_printf(s, "timestamp", ind, "%llu\n", vb2->timestamp);
+#else
+	mvx_seq_printf(s, "timestamp", ind, "\n");
+	ind++;
+	mvx_seq_printf(s, "tv_sec", ind, "%lu\n", v4l2->vb2_buf.timestamp.tv_sec);
+	mvx_seq_printf(s, "tv_usec", ind, "%lu\n", v4l2->vb2_buf.timestamp.tv_usec);
+	ind--;
+#endif
+	mvx_seq_printf(s, "timecode", ind, "\n");
+	ind++;
+	mvx_seq_printf(s, "type", ind, "%u\n", v4l2->timecode.type);
+	mvx_seq_printf(s, "flags", ind, "%u\n", v4l2->timecode.flags);
+	mvx_seq_printf(s, "frames", ind, "%u\n", v4l2->timecode.frames);
+	mvx_seq_printf(s, "seconds", ind, "%u\n", v4l2->timecode.seconds);
+	mvx_seq_printf(s, "minutes", ind, "%u\n", v4l2->timecode.minutes);
+	mvx_seq_printf(s, "hours", ind, "%u\n", v4l2->timecode.hours);
+	ind--;
+
+	mvx_seq_printf(s, "sequence", ind, "%u\n", v4l2->sequence);
+	mvx_seq_printf(s, "memory", ind, "%u\n", vb2->memory);
+
+	mvx_seq_printf(s, "num_planes", ind, "%u\n", vb2->num_planes);
+
+	mvx_seq_printf(s, "planes", ind, "\n");
+	ind++;
+	for (i = 0; i < vb2->num_planes; ++i) {
+		char tag[10];
+		struct vb2_plane *plane = &vb2->planes[i];
+
+		scnprintf(tag, sizeof(tag), "#%d", i);
+		mvx_seq_printf(s, tag, ind,
+			       "bytesused: %10u, length: %10u, m.offset: %10u, m.userptr: %10lu, m.fd: %10d, "
+			       "data_offset: %10u\n",
+			       plane->bytesused, plane->length, plane->m.offset, plane->m.userptr, plane->m.fd,
+			       plane->data_offset);
+	}
+
+	ind--;
+}
+
+static int buffer_stat_show(struct seq_file *s, void *v)
+{
+	struct mvx_v4l2_buffer *vbuf = s->private;
+
+	v4l2_buffer_show(vbuf, s);
+	seq_puts(s, "\n");
+	mvx_buffer_show(&vbuf->buf, s);
+
+	return 0;
+}
+
+static int buffer_stat_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, buffer_stat_show, inode->i_private);
+}
+
+static const struct file_operations buffer_stat_fops = {
+	.open    = buffer_stat_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = seq_release
+};
+
+static int buffer_debugfs_init(struct dentry *parent, struct mvx_v4l2_buffer *vbuf)
+{
+	char name[20];
+	struct dentry *dentry;
+
+	scnprintf(name, sizeof(name), "buffer%u", to_vb2_buf(vbuf)->index);
+	vbuf->dentry = debugfs_create_dir(name, parent);
+	if (IS_ERR_OR_NULL(vbuf->dentry))
+		return -ENOMEM;
+
+	dentry = debugfs_create_file("stat", 0400, vbuf->dentry, vbuf, &buffer_stat_fops);
+	if (IS_ERR_OR_NULL(dentry))
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * get_bytesused() - Get total number of bytes used for Vb2 buffer.
+ */
+static size_t get_bytesused(struct vb2_buffer *b)
+{
+	size_t size;
+	uint32_t i;
+
+	for (i = 0, size = 0; i < b->num_planes; i++)
+		size += b->planes[i].bytesused;
+
+	return size;
+}
+
+static int clear_bytesused(struct vb2_buffer *b)
+{
+	uint32_t i;
+
+	for (i = 0; i < b->num_planes; i++)
+		b->planes[i].bytesused = 0;
+
+	return 0;
+}
+
+/* Update mvx_buffer flags from vb2_buffer flags */
+static int update_mvx_flags(struct mvx_buffer *buf, struct vb2_buffer *b)
+{
+	struct vb2_v4l2_buffer *vb2_v4l2 = to_vb2_v4l2_buffer(b);
+	__u32 flags = vb2_v4l2->flags;
+	__u32 rtt_flags = (buf->flags & MVX_BUFFER_FRAME_FLAG_ROTATION_MASK);
+	buf->flags = 0;
+
+	if (V4L2_TYPE_IS_OUTPUT(b->type) != false && get_bytesused(b) == 0)
+		flags |= V4L2_BUF_FLAG_LAST;
+
+	if (flags & V4L2_BUF_FLAG_LAST)
+		buf->flags |= MVX_BUFFER_EOS;
+
+	if ((flags & V4L2_BUF_FLAG_MVX_DISABLE_CACHE_MAINTENANCE) == V4L2_BUF_FLAG_MVX_DISABLE_CACHE_MAINTENANCE) {
+		buf->flags |= MVX_BUFFER_FLAG_DISABLE_CACHE_MAINTENANCE;
+	}
+	if (mvx_is_afbc(buf->format)) {
+		if ((flags & V4L2_BUF_FLAG_MVX_AFBC_TILED_HEADERS) == V4L2_BUF_FLAG_MVX_AFBC_TILED_HEADERS)
+			buf->flags |= MVX_BUFFER_AFBC_TILED_HEADERS;
+
+		if ((flags & V4L2_BUF_FLAG_MVX_AFBC_TILED_BODY) == V4L2_BUF_FLAG_MVX_AFBC_TILED_BODY)
+			buf->flags |= MVX_BUFFER_AFBC_TILED_BODY;
+
+		if ((flags & V4L2_BUF_FLAG_MVX_AFBC_32X8_SUPERBLOCK) == V4L2_BUF_FLAG_MVX_AFBC_32X8_SUPERBLOCK)
+			buf->flags |= MVX_BUFFER_AFBC_32X8_SUPERBLOCK;
+
+		if ((flags & V4L2_BUF_FLAG_MVX_AFBC_BLOCK_SPLIT) == V4L2_BUF_FLAG_MVX_AFBC_BLOCK_SPLIT)
+			buf->flags |= MVX_BUFFER_AFBC_BLOCK_SPLIT;
+	} else if (mvx_is_bitstream(buf->format)) {
+		if (buf->dir == MVX_DIR_INPUT) {
+			// decode bitstream port
+			if ((flags & V4L2_BUF_FLAG_END_OF_SUB_FRAME) == V4L2_BUF_FLAG_END_OF_SUB_FRAME) {
+				buf->flags |= MVX_BUFFER_END_OF_SUB_FRAME;
+			}
+			if (flags & V4L2_BUF_FLAG_KEYFRAME)
+				buf->flags |= MVX_BUFFER_EOF;
+			if ((flags & V4L2_BUF_FLAG_MVX_CODEC_CONFIG) == V4L2_BUF_FLAG_MVX_CODEC_CONFIG)
+				buf->flags |= MVX_BUFFER_CODEC_CONFIG;
+		}
+	} else if (mvx_is_frame(buf->format)) {
+		if (buf->dir == MVX_DIR_OUTPUT) {
+			// decode frame port
+			if (flags & V4L2_BUF_FRAME_FLAG_ROTATION_MASK) {
+				if ((flags & V4L2_BUF_FRAME_FLAG_ROTATION_MASK) == V4L2_BUF_FRAME_FLAG_ROTATION_90) {
+					buf->flags |= MVX_BUFFER_FRAME_FLAG_ROTATION_90;
+				} else if ((flags & V4L2_BUF_FRAME_FLAG_ROTATION_MASK) ==
+					   V4L2_BUF_FRAME_FLAG_ROTATION_180) {
+					buf->flags |= MVX_BUFFER_FRAME_FLAG_ROTATION_180;
+				} else if ((flags & V4L2_BUF_FRAME_FLAG_ROTATION_MASK) ==
+					   V4L2_BUF_FRAME_FLAG_ROTATION_270) {
+					buf->flags |= MVX_BUFFER_FRAME_FLAG_ROTATION_270;
+				}
+			}
+			if (flags & V4L2_BUF_FRAME_FLAG_SCALING_MASK) {
+				if ((flags & V4L2_BUF_FRAME_FLAG_SCALING_MASK) == V4L2_BUF_FRAME_FLAG_SCALING_2) {
+					buf->flags |= MVX_BUFFER_FRAME_FLAG_SCALING_2;
+				} else if ((flags & V4L2_BUF_FRAME_FLAG_SCALING_MASK) ==
+					   V4L2_BUF_FRAME_FLAG_SCALING_4) {
+					buf->flags |= MVX_BUFFER_FRAME_FLAG_SCALING_4;
+				}
+			}
+		} else if (buf->dir == MVX_DIR_INPUT) {
+			// encode frame port
+			if (flags & V4L2_BUF_FRAME_FLAG_MIRROR_MASK) {
+				if ((flags & V4L2_BUF_FRAME_FLAG_MIRROR_MASK) == V4L2_BUF_FRAME_FLAG_MIRROR_HORI) {
+					buf->flags |= MVX_BUFFER_FRAME_FLAG_MIRROR_HORI;
+				} else if ((flags & V4L2_BUF_FRAME_FLAG_MIRROR_MASK) ==
+					   V4L2_BUF_FRAME_FLAG_MIRROR_VERT) {
+					buf->flags |= MVX_BUFFER_FRAME_FLAG_MIRROR_VERT;
+				}
+			}
+			buf->flags |= rtt_flags;
+			if ((flags & V4L2_BUF_FLAG_MVX_BUFFER_EPR) == V4L2_BUF_FLAG_MVX_BUFFER_EPR) {
+				buf->flags |= MVX_BUFFER_FRAME_FLAG_GENERAL;
+			}
+			if ((flags & V4L2_BUF_FLAG_MVX_BUFFER_ROI) == V4L2_BUF_FLAG_MVX_BUFFER_ROI) {
+				buf->flags |= MVX_BUFFER_FRAME_FLAG_ROI;
+			}
+		}
+
+	} else {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "unrecognized buffer format!.");
+	}
+
+	return 0;
+}
+
+/* Update mvx_buffer from mvx_v4l2_buffer */
+static int update_mvx_buffer(struct mvx_v4l2_buffer *vbuf)
+{
+	struct vb2_buffer *vb2 = to_vb2_buf(vbuf);
+	struct mvx_buffer *mvx_buf = &vbuf->buf;
+	int i;
+	int ret;
+
+	if (vb2->num_planes != mvx_buf->nplanes) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "VB2 and MVX buffers have different number of planes. vb2_planes=%u, mvx_planes=%u.",
+			      vb2->num_planes, mvx_buf->nplanes);
+		return -EINVAL;
+	}
+
+#if KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE
+	mvx_buf->user_data = vb2->timestamp;
+#else
+	{
+		struct timeval *ts = &vbuf->vb2_v4l2_buffer.vb2_buf.timestamp;
+
+		mvx_buf->user_data = ((uint64_t)ts->tv_sec << 32) | (ts->tv_usec & 0xffffffff);
+	}
+#endif
+
+	for (i = 0; i < vb2->num_planes; i++) {
+		unsigned int offset = vb2->planes[i].data_offset;
+		unsigned int filled = vb2->planes[i].bytesused;
+		if (V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE == vb2->type && vb2->memory == V4L2_MEMORY_DMABUF && offset > 0) {
+			// TODO(crbug.com/901264): The way to pass an offset within a DMA-buf is not defined
+			// in V4L2 specification, so we abuse data_offset for now. Fix it when we have the
+			// right interface, including any necessary validation and potential alignment.
+			// workaround: for MPLANAR, if using vb2->planes[i].data_offset, bytesused should be
+			// larger than data_offset, otherwise __verify_length() will fail in vb2_qbuf()
+			// so 'bytesused - data_offset' is extact filled data size.
+			filled = (vb2->planes[i].bytesused - offset);
+		} else if (V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE == vb2->type && vb2->memory == V4L2_MEMORY_DMABUF &&
+			   i > 0) {
+			//'data_offset' is not passed in v4l2-core for buffer type OUTPUT in "vb2_fill_vb2_v4l2_buffer"
+			//'data_offset' is always 0, so reset plane offset in hard code.
+			int k;
+			offset = 0;
+			for (k = i; k > 0; k--)
+				offset += vb2->planes[i - k].length;
+		}
+		/*
+		 * For single planar mmap buffers the offset is carried by
+		 * the lower part of the offset.
+		 */
+		if (vb2->memory == V4L2_MEMORY_MMAP)
+			offset += vb2->planes[i].m.offset & ~PAGE_MASK;
+
+		ret = mvx_buffer_filled_set(mvx_buf, i, filled, offset);
+		if (ret != 0)
+			return ret;
+	}
+
+	ret = update_mvx_flags(mvx_buf, to_vb2_buf(vbuf));
+	if (ret != 0)
+		return 0;
+
+	return 0;
+}
+
+static int update_v4l2_bytesused(struct mvx_v4l2_buffer *vbuf)
+{
+	struct vb2_buffer *b = to_vb2_buf(vbuf);
+	struct mvx_buffer *buf = &vbuf->buf;
+	int i;
+
+	if (b->num_planes != buf->nplanes) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING,
+			      "VB2 and MVX buffers have different number of planes. vb2_planes=%u, mvx_planes=%u.",
+			      b->num_planes, buf->nplanes);
+		return -EINVAL;
+	}
+
+	/*
+	 * MVX filled is the number of bytes excluding the offset. The total
+	 * length is calculated as 'filled + offset' and should be <= length.
+	 *
+	 * V4L2 bytesused is the total length including the offset.
+	 * bytesused should be <= length and bytesused >= offset.
+	 */
+	for (i = 0; i < b->num_planes; i++) {
+		b->planes[i].bytesused = buf->planes[i].filled + buf->planes[i].offset;
+		b->planes[i].data_offset = buf->planes[i].offset;
+	}
+
+	return 0;
+}
+
+static int update_vb2_flags(struct mvx_v4l2_buffer *vbuf)
+{
+	struct vb2_v4l2_buffer *b = &vbuf->vb2_v4l2_buffer;
+	struct mvx_buffer *buf = &vbuf->buf;
+
+	b->flags &= ~(V4L2_BUF_FLAG_ERROR | V4L2_BUF_FLAG_KEYFRAME | V4L2_BUF_FLAG_LAST);
+
+	if (buf->flags & MVX_BUFFER_EOS)
+		b->flags |= V4L2_BUF_FLAG_LAST;
+
+	if ((buf->flags & MVX_BUFFER_EOF) && (buf->flags & MVX_BUFFER_FRAME_FLAG_IFRAME))
+		b->flags |= V4L2_BUF_FLAG_KEYFRAME;
+
+	if (buf->flags & MVX_BUFFER_CORRUPT)
+		b->flags |= V4L2_BUF_FLAG_ERROR;
+
+	if (buf->flags & MVX_BUFFER_REJECTED)
+		clear_bytesused(&b->vb2_buf);
+
+	if (buf->flags & MVX_BUFFER_DECODE_ONLY)
+		b->flags |= V4L2_BUF_FLAG_MVX_DECODE_ONLY;
+
+	if (buf->flags & MVX_BUFFER_CODEC_CONFIG)
+		b->flags |= V4L2_BUF_FLAG_MVX_CODEC_CONFIG;
+
+	if (buf->flags & MVX_BUFFER_AFBC_TILED_HEADERS)
+		b->flags |= V4L2_BUF_FLAG_MVX_AFBC_TILED_HEADERS;
+
+	if (buf->flags & MVX_BUFFER_AFBC_TILED_BODY)
+		b->flags |= V4L2_BUF_FLAG_MVX_AFBC_TILED_BODY;
+
+	if (buf->flags & MVX_BUFFER_AFBC_32X8_SUPERBLOCK)
+		b->flags |= V4L2_BUF_FLAG_MVX_AFBC_32X8_SUPERBLOCK;
+
+	if (buf->flags & MVX_BUFFER_FRAME_PRESENT)
+		b->flags |= V4L2_BUF_FLAG_MVX_BUFFER_FRAME_PRESENT;
+
+	if (buf->flags & MVX_BUFFER_FRAME_NEED_REALLOC)
+		b->flags |= V4L2_BUF_FLAG_MVX_BUFFER_NEED_REALLOC;
+
+	if ((buf->flags & MVX_BUFFER_EOF) && (buf->flags & MVX_BUFFER_FRAME_FLAG_PFRAME))
+		b->flags |= V4L2_BUF_FLAG_PFRAME;
+
+	if ((buf->flags & MVX_BUFFER_EOF) && (buf->flags & MVX_BUFFER_FRAME_FLAG_BFRAME))
+		b->flags |= V4L2_BUF_FLAG_BFRAME;
+
+	return 0;
+}
+
+/****************************************************************************
+ * Exported functions and variables
+ ****************************************************************************/
+
+int mvx_v4l2_buffer_construct(struct mvx_v4l2_buffer *vbuf, struct mvx_v4l2_session *vsession, enum mvx_direction dir,
+			      unsigned int nplanes, struct sg_table **sgt)
+{
+	int ret;
+
+	ret = mvx_buffer_construct(&vbuf->buf, vsession->ext->dev, &vsession->session.mmu, dir, nplanes, sgt);
+	if (ret != 0)
+		return ret;
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		struct mvx_v4l2_port *vport = &vsession->port[dir];
+
+		ret = buffer_debugfs_init(vport->dentry, vbuf);
+		if (ret != 0) {
+			MVX_SESSION_WARN(&vsession->session, "Failed to create buffer debugfs entry.");
+			goto destruct_buffer;
+		}
+	}
+
+	return 0;
+
+destruct_buffer:
+	mvx_buffer_destruct(&vbuf->buf);
+
+	return ret;
+}
+
+void mvx_v4l2_buffer_destruct(struct mvx_v4l2_buffer *vbuf)
+{
+	mvx_buffer_destruct(&vbuf->buf);
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(vbuf->dentry);
+}
+
+struct mvx_v4l2_buffer *mvx_buffer_to_v4l2_buffer(struct mvx_buffer *buffer)
+{
+	return container_of(buffer, struct mvx_v4l2_buffer, buf);
+}
+
+/* Update mvx_v4l2_buffer from vb2_buffer */
+int mvx_v4l2_buffer_set(struct mvx_v4l2_buffer *vbuf, struct vb2_buffer *b)
+{
+	int ret;
+
+	ret = update_mvx_buffer(vbuf);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+enum vb2_buffer_state mvx_v4l2_buffer_update(struct mvx_v4l2_buffer *vbuf)
+{
+	struct vb2_buffer *vb2 = to_vb2_buf(vbuf);
+	struct mvx_buffer *mvx_buf = &vbuf->buf;
+	int ret;
+
+	if (V4L2_TYPE_IS_OUTPUT(vb2->type) != false)
+		ret = clear_bytesused(vb2);
+	else
+		ret = update_v4l2_bytesused(vbuf);
+
+	if (ret != 0)
+		goto error;
+
+	ret = update_vb2_flags(vbuf);
+
+#if KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE
+	vb2->timestamp = mvx_buf->user_data;
+#else
+	{
+		struct timeval *ts = &vbuf->vb2_v4l2_buffer.timestamp;
+
+		ts->tv_sec = mvx_buf->user_data >> 32;
+		ts->tv_usec = mvx_buf->user_data & 0xffffffff;
+	}
+#endif
+	if (ret != 0 || (vbuf->vb2_v4l2_buffer.flags & V4L2_BUF_FLAG_ERROR) != 0) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_INFO, "buf flag is V4L2_BUF_FLAG_ERROR user_data:%llx dir:%d",
+			      mvx_buf->user_data, mvx_buf->dir);
+		goto error;
+	}
+
+	return VB2_BUF_STATE_DONE;
+
+error:
+	return VB2_BUF_STATE_ERROR;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_buffer.h b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_buffer.h
new file mode 100644
index 000000000000..9361a6f3cae9
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_buffer.h
@@ -0,0 +1,159 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_V4L2_BUFFER_H_
+#define _MVX_V4L2_BUFFER_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/types.h>
+#include <media/videobuf2-v4l2.h>
+#include "mvx_buffer.h"
+#include "mvx_if.h"
+#include "mvx_v4l2_session.h"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+#define vb2_v4l2_to_mvx_v4l2_buffer(v4l2) container_of(v4l2, struct mvx_v4l2_buffer, vb2_v4l2_buffer)
+
+#define vb2_to_mvx_v4l2_buffer(vb2) vb2_v4l2_to_mvx_v4l2_buffer(to_vb2_v4l2_buffer(vb2))
+
+#define to_vb2_buf(vbuf) (&((vbuf)->vb2_v4l2_buffer.vb2_buf))
+
+/**
+ * struct mvx_v4l2_buffer - MVX V4L2 buffer.
+ * @vb2_v4l2_buffer:	VB2 V4L2 buffer.
+ * @buf:	MVX buffer.
+ * @dentry:	Debug file system entry.
+ */
+struct mvx_v4l2_buffer {
+	struct vb2_v4l2_buffer vb2_v4l2_buffer;
+	struct mvx_buffer buf;
+	struct dentry *dentry;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_v4l2_buffer_construct() - Construct MVX V4L2 buffer object.
+ * @vbuf:	Pointer to MVX V4L2 buffer.
+ * @vsession:	Pointer to V4L2 session.
+ * @dir:	Direction of the buffer.
+ * @nplanes:	Number of planes.
+ * @sgt:	Array of pointers to scatter-gatter lists. Each SG list
+ *              contains memory pages for a corresponding plane.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_v4l2_buffer_construct(struct mvx_v4l2_buffer *vbuf, struct mvx_v4l2_session *vsession, enum mvx_direction dir,
+			      unsigned int nplanes, struct sg_table **sgt);
+
+/**
+ * mvx_v4l2_buffer_destruct() - Destruct v4l2 buffer object.
+ * @vbuf:	Pointer to MVX V4L2 buffer.
+ */
+void mvx_v4l2_buffer_destruct(struct mvx_v4l2_buffer *vbuf);
+
+/**
+ * mvx_buffer_to_v4l2_buffer() - Cast mvx_buffer to mvx_v4l2_buffer.
+ * @buf:	Pointer MVX buffer.
+ *
+ * This function casts a pointer to struct mvx_buffer to a pointer to
+ * a corresponding struct mvx_v4l2_buffer.
+ *
+ * Return: Pointer to corresponding mvx_v4l2_buffer object.
+ */
+struct mvx_v4l2_buffer *mvx_buffer_to_v4l2_buffer(struct mvx_buffer *buf);
+
+/**
+ * mvx_v4l2_buffer_set_status() - Set status for a buffer.
+ * @vbuf:	Pointer to MVX V4L2 buffer.
+ * @status:	Status to set.
+ *
+ * Status is a combination of the following flags:
+ * V4L2_BUF_FLAG_QUEUED,
+ * V4L2_BUF_FLAG_DONE,
+ * V4L2_BUF_FLAG_PREPARED,
+ * V4L2_BUF_FLAG_ERROR
+ */
+void mvx_v4l2_buffer_set_status(struct mvx_v4l2_buffer *vbuf, uint32_t status);
+
+/**
+ * mvx_v4l2_buffer_get_status() - Get the buffer status.
+ * @vbuf:	Pointer to MVX V4L2 buffer.
+ *
+ * Return: Buffer status.
+ */
+uint32_t mvx_v4l2_buffer_get_status(struct mvx_v4l2_buffer *vbuf);
+
+/**
+ * mvx_v4l2_buffer_set() - Copy Vb2 buffer to VBUF.
+ * @vbuf:	Destination MVX V4L2 buffer.
+ * @b:		Source Vb2 buffer.
+ *
+ * Copies and validates paramters from 'b' to 'vbuf'.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_v4l2_buffer_set(struct mvx_v4l2_buffer *vbuf, struct vb2_buffer *b);
+
+/**
+ * mvx_v4l2_buffer_get() - Copy VBUF to V4L2 buffer.
+ * @vbuf:	Source MVX V4L2 buffer.
+ * @b:		Destination V4L2 buffer.
+ *
+ * Copies parameters from 'vbuf' to 'b'.
+ */
+void mvx_v4l2_buffer_get(struct mvx_v4l2_buffer *vbuf, struct v4l2_buffer *b);
+
+/**
+ * mvx_v4l2_buffer_update() - Update the V4L2 buffer.
+ * @vbuf:	Pointer to MVX V4L2 buffer.
+ *
+ * This function copies parameters from the MVX buffer to the V4L2 buffer.
+ * It also sets the time stamp and validates that the buffer length is correct.
+ * If an error is detectd the buffer length is cleared and the error flag
+ * is set.
+ *
+ * This function should be called after the MVX buffer has changed, for example
+ * after it has been returned by the firmware or flushed.
+ *
+ * Return: VB2_BUF_STATE_DONE on success, else VB2_BUF_STATE_ERROR.
+ */
+enum vb2_buffer_state mvx_v4l2_buffer_update(struct mvx_v4l2_buffer *vbuf);
+
+#endif /* _MVX_V4L2_BUFFER_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_ctrls.c b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_ctrls.c
new file mode 100644
index 000000000000..d134c81210ff
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_ctrls.c
@@ -0,0 +1,1258 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_bitops.h"
+#include "mvx_v4l2_ctrls.h"
+#include "mvx_v4l2_session.h"
+#include "mvx-v4l2-controls.h"
+
+/****************************************************************************
+ * Static functions and variables
+ ****************************************************************************/
+
+/*
+ * V4L2_CID_MVE_VIDEO_NALU_FORMAT control defines.
+ */
+static const char *const nalu_format_str[] = {
+	"Start codes",
+	"One nalu per buffer",
+	"One byte length field",
+	"Two byte length field",
+	"Four byte length field"
+};
+
+static const enum mvx_nalu_format mvx_nalu_format_list[] = {
+	MVX_NALU_FORMAT_START_CODES,
+	MVX_NALU_FORMAT_ONE_NALU_PER_BUFFER,
+	MVX_NALU_FORMAT_ONE_BYTE_LENGTH_FIELD,
+	MVX_NALU_FORMAT_TWO_BYTE_LENGTH_FIELD,
+	MVX_NALU_FORMAT_FOUR_BYTE_LENGTH_FIELD
+};
+
+/*
+ * V4L2_CID_MVE_VIDEO_H265_PROFILE control defines.
+ */
+static const char *const h265_profile_str[] = {
+	"Main",
+	"Main still",
+	"Main intra"
+};
+
+static const int mvx_h265_profile_list[] = {
+	MVX_PROFILE_H265_MAIN,
+	MVX_PROFILE_H265_MAIN_STILL,
+	MVX_PROFILE_H265_MAIN_INTRA
+};
+
+/*
+ * V4L2_CID_MVE_VIDEO_VC1_PROFILE control defines.
+ */
+static const char *const vc1_profile_str[] = {
+	"Simple",
+	"Main",
+	"Advanced"
+};
+
+static const int mvx_vc1_profile_list[] = {
+	MVX_PROFILE_VC1_SIMPLE,
+	MVX_PROFILE_VC1_MAIN,
+	MVX_PROFILE_VC1_ADVANCED
+};
+
+/*
+ * V4L2_CID_MPEG_VIDEO_H264_PROFILE control defines.
+ */
+static const uint8_t h264_profile_list[] = {
+	V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
+	V4L2_MPEG_VIDEO_H264_PROFILE_MAIN,
+	V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
+	V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_10
+};
+static const uint8_t hevc_profile_list[] = {
+	V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+	// V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE,
+	// V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10
+};
+
+static const enum mvx_profile mvx_h264_profile_list[] = {
+	MVX_PROFILE_H264_BASELINE,
+	MVX_PROFILE_H264_MAIN,
+	MVX_PROFILE_H264_HIGH,
+	MVX_PROFILE_H264_HIGH_10
+};
+
+/*
+ * V4L2_CID_MPEG_VIDEO_H264_LEVEL control defines.
+ */
+static uint8_t h264_level_list[] = {
+	V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
+	V4L2_MPEG_VIDEO_H264_LEVEL_1B,
+	V4L2_MPEG_VIDEO_H264_LEVEL_1_1,
+	V4L2_MPEG_VIDEO_H264_LEVEL_1_2,
+	V4L2_MPEG_VIDEO_H264_LEVEL_1_3,
+	V4L2_MPEG_VIDEO_H264_LEVEL_2_0,
+	V4L2_MPEG_VIDEO_H264_LEVEL_2_1,
+	V4L2_MPEG_VIDEO_H264_LEVEL_2_2,
+	V4L2_MPEG_VIDEO_H264_LEVEL_3_0,
+	V4L2_MPEG_VIDEO_H264_LEVEL_3_1,
+	V4L2_MPEG_VIDEO_H264_LEVEL_3_2,
+	V4L2_MPEG_VIDEO_H264_LEVEL_4_0,
+	V4L2_MPEG_VIDEO_H264_LEVEL_4_1,
+	V4L2_MPEG_VIDEO_H264_LEVEL_4_2,
+	V4L2_MPEG_VIDEO_H264_LEVEL_5_0,
+	V4L2_MPEG_VIDEO_H264_LEVEL_5_1
+};
+static uint8_t hevc_level_list[] = {
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_2,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_3,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_4,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_5,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_6,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1,
+	V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
+};
+
+static const int mvx_h264_level_list[] = {
+	MVX_LEVEL_H264_1,
+	MVX_LEVEL_H264_1b,
+	MVX_LEVEL_H264_11,
+	MVX_LEVEL_H264_12,
+	MVX_LEVEL_H264_13,
+	MVX_LEVEL_H264_2,
+	MVX_LEVEL_H264_21,
+	MVX_LEVEL_H264_22,
+	MVX_LEVEL_H264_3,
+	MVX_LEVEL_H264_31,
+	MVX_LEVEL_H264_32,
+	MVX_LEVEL_H264_4,
+	MVX_LEVEL_H264_41,
+	MVX_LEVEL_H264_42,
+	MVX_LEVEL_H264_5,
+	MVX_LEVEL_H264_51
+};
+
+/*
+ * V4L2_CID_MVE_VIDEO_H265_LEVEL control defines.
+ */
+static const char *const h265_level_str[] = {
+	"Main 1",
+	"High 1",
+
+	"Main 2",
+	"High 2",
+	"Main 2.1",
+	"High 2.1",
+
+	"Main 3",
+	"High 3",
+	"Main 3.1",
+	"High 3.1",
+
+	"Main 4",
+	"High 4",
+	"Main 4.1",
+	"High 4.1",
+
+	"Main 5",
+	"High 5",
+	"Main 5.1",
+	"High 5.1",
+	"Main 5.2",
+	"High 5.2",
+
+	"Main 6",
+	"High 6",
+	"Main 6.1",
+	"High 6.1",
+	"Main 6.2",
+	"High 6.2"
+};
+
+static const int mvx_h265_level_list[] = {
+	MVX_LEVEL_H265_MAIN_1,
+	MVX_LEVEL_H265_HIGH_1,
+
+	MVX_LEVEL_H265_MAIN_2,
+	MVX_LEVEL_H265_HIGH_2,
+	MVX_LEVEL_H265_MAIN_21,
+	MVX_LEVEL_H265_HIGH_21,
+
+	MVX_LEVEL_H265_MAIN_3,
+	MVX_LEVEL_H265_HIGH_3,
+	MVX_LEVEL_H265_MAIN_31,
+	MVX_LEVEL_H265_HIGH_31,
+
+	MVX_LEVEL_H265_MAIN_4,
+	MVX_LEVEL_H265_HIGH_4,
+	MVX_LEVEL_H265_MAIN_41,
+	MVX_LEVEL_H265_HIGH_41,
+
+	MVX_LEVEL_H265_MAIN_5,
+	MVX_LEVEL_H265_HIGH_5,
+	MVX_LEVEL_H265_MAIN_51,
+	MVX_LEVEL_H265_HIGH_51,
+	MVX_LEVEL_H265_MAIN_52,
+	MVX_LEVEL_H265_HIGH_52,
+
+	MVX_LEVEL_H265_MAIN_6,
+	MVX_LEVEL_H265_HIGH_6,
+	MVX_LEVEL_H265_MAIN_61,
+	MVX_LEVEL_H265_HIGH_61,
+	MVX_LEVEL_H265_MAIN_62,
+	MVX_LEVEL_H265_HIGH_62
+};
+
+/*
+ * V4L2_CID_MVE_VIDEO_GOP_TYPE control defines.
+ */
+static const char *const gop_type_str[] = {
+	"None",
+	"Bidirectional",
+	"Low delay",
+	"Pyramid"
+};
+
+static const enum mvx_gop_type mvx_gop_type_list[] = {
+	MVX_GOP_TYPE_NONE,
+	MVX_GOP_TYPE_BIDIRECTIONAL,
+	MVX_GOP_TYPE_LOW_DELAY,
+	MVX_GOP_TYPE_PYRAMID
+};
+
+/*
+ * V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE control defines.
+ */
+#define V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_NONE 2
+
+static const uint8_t h264_entropy_mode_list[] = {
+	V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC,
+	V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
+	V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_NONE
+};
+
+static const enum mvx_entropy_mode mvx_h264_entropy_mode_list[] = {
+	MVX_ENTROPY_MODE_CAVLC,
+	MVX_ENTROPY_MODE_CABAC,
+	MVX_ENTROPY_MODE_NONE
+};
+
+static const char *const h264_entropy_mode_str[] = {
+	"CAVLC",
+	"CABAC",
+	"None"
+};
+
+/*
+ * V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE controls list.
+ */
+static uint8_t multi_slice_mode_list[] = {
+	V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_SINGLE,
+
+	/* Misspelling in the header file */
+	V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_MAX_MB
+};
+
+static const enum mvx_multi_slice_mode mvx_multi_slice_mode_list[] = {
+	MVX_MULTI_SLICE_MODE_SINGLE,
+	MVX_MULTI_SLICE_MODE_MAX_MB
+};
+
+/*
+ * V4L2_CID_MVE_VIDEO_VP9_PROB_UPDATE control defines.
+ */
+static const char *const vp9_prob_update_str[] = {
+	"Disabled",
+	"Implicit",
+	"Explicit"
+};
+
+static const enum mvx_vp9_prob_update mvx_vp9_prob_update_list[] = {
+	MVX_VP9_PROB_UPDATE_DISABLED,
+	MVX_VP9_PROB_UPDATE_IMPLICIT,
+	MVX_VP9_PROB_UPDATE_EXPLICIT
+};
+
+/*
+ * V4L2_CID_MVE_VIDEO_RGB_TO_YUV_MODE control defines.
+ */
+static const char *const rgb_to_yuv_mode_str[] = {
+	"BT601 studio",
+	"BT601 full",
+	"BT709 studio",
+	"BT709 full"
+};
+
+static const enum mvx_rgb_to_yuv_mode mvx_rgb_to_yuv_mode_list[] = {
+	MVX_RGB_TO_YUV_MODE_BT601_STUDIO,
+	MVX_RGB_TO_YUV_MODE_BT601_FULL,
+	MVX_RGB_TO_YUV_MODE_BT709_STUDIO,
+	MVX_RGB_TO_YUV_MODE_BT709_FULL
+};
+
+/**
+ * find_idx() - Find index of a value in an array.
+ * @list:	Pointer to an array.
+ * @size:	Size of an array.
+ * @val:	Value to look for.
+ *
+ * Return: Index of the first occurrence of 'val' in 'list',
+ *         or -EINVAL when not found.
+ */
+static int find_idx(const uint8_t *list, size_t size, uint8_t val)
+{
+	while (size--)
+		if (list[size] == val)
+			return size;
+
+	return -EINVAL;
+}
+
+/**
+ * set_ctrl() - Callback used by V4L2 framework to set a control.
+ * @ctrl:	V4L2 control.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+static int set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	int ret = 0;
+	struct mvx_v4l2_session *vsession = container_of(ctrl->handler, struct mvx_v4l2_session, v4l2_ctrl);
+	struct mvx_session *session = &vsession->session;
+	enum mvx_nalu_format nalu_fmt;
+	enum mvx_profile mvx_profile;
+	enum mvx_level mvx_level;
+	enum mvx_gop_type gop_type;
+	enum mvx_entropy_mode entropy_mode;
+	enum mvx_multi_slice_mode multi_slice_mode;
+	enum mvx_vp9_prob_update vp9_prob_update;
+	enum mvx_rgb_to_yuv_mode rgb_to_yuv_mode;
+	int32_t i32_val;
+	int64_t i64_val;
+	bool bool_val;
+	enum mvx_tristate tri_val;
+	struct mvx_buffer_param_rate_control rate_control_param;
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	switch (ctrl->id) {
+	case V4L2_CID_MVE_VIDEO_SECURE_VIDEO:
+		bool_val = *ctrl->p_new.p_s32 != 0;
+		ret = mvx_session_set_securevideo(session, bool_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_FRAME_RATE:
+		i64_val = (ctrl->type == V4L2_CTRL_TYPE_INTEGER) ? *ctrl->p_new.p_s32 : *ctrl->p_new.p_s64;
+		ret = mvx_session_set_frame_rate(session, i64_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE:
+		bool_val = *ctrl->p_new.p_s32 != 0;
+		ret = mvx_session_set_rate_control(session, bool_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_BITRATE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_bitrate(session, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: {
+		i32_val = *ctrl->p_new.p_s32;
+		rate_control_param.rate_control_mode = session->rc_type;
+		rate_control_param.target_bitrate = session->target_bitrate;
+		rate_control_param.maximum_bitrate = session->maximum_bitrate;
+		if (i32_val == V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) {
+			rate_control_param.rate_control_mode = MVX_OPT_RATE_CONTROL_MODE_VARIABLE;
+		} else if (i32_val == V4L2_MPEG_VIDEO_BITRATE_MODE_CBR) {
+			rate_control_param.rate_control_mode = MVX_OPT_RATE_CONTROL_MODE_CONSTANT;
+		} else if (i32_val == V4L2_MPEG_VIDEO_BITRATE_MODE_CQ) {
+			rate_control_param.rate_control_mode = MVX_OPT_RATE_CONTROL_MODE_OFF;
+		}
+		ret = mvx_session_set_bitrate_control(session, &rate_control_param);
+		break;
+	}
+	case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: {
+		rate_control_param.rate_control_mode = session->rc_type;
+		rate_control_param.target_bitrate = session->target_bitrate;
+		rate_control_param.maximum_bitrate = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_bitrate_control(session, &rate_control_param);
+		break;
+	}
+	case V4L2_CID_MVE_VIDEO_CROP_LEFT:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_crop_left(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_CROP_RIGHT:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_crop_right(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_CROP_TOP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_crop_top(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_CROP_BOTTOM:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_crop_bottom(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_HRD_BUFFER_SIZE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_hrd_buffer_size(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_NALU_FORMAT:
+		i32_val = *ctrl->p_new.p_s32;
+		nalu_fmt = mvx_nalu_format_list[i32_val];
+		ret = mvx_session_set_nalu_format(session, nalu_fmt);
+		break;
+	case V4L2_CID_MVE_VIDEO_STREAM_ESCAPING:
+		tri_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_stream_escaping(session, tri_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = find_idx(h264_profile_list, ARRAY_SIZE(h264_profile_list), i32_val);
+		if (ret == -EINVAL)
+			goto unlock_mutex;
+
+		mvx_profile = mvx_h264_profile_list[ret];
+		ret = mvx_session_set_profile(session, MVX_FORMAT_H264, mvx_profile);
+		break;
+	case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = find_idx(hevc_profile_list, ARRAY_SIZE(hevc_profile_list), i32_val);
+		if (ret == -EINVAL)
+			goto unlock_mutex;
+
+		mvx_profile = mvx_h265_profile_list[ret];
+		ret = mvx_session_set_profile(session, MVX_FORMAT_HEVC, mvx_profile);
+		break;
+	case V4L2_CID_MVE_VIDEO_H265_PROFILE:
+		i32_val = *ctrl->p_new.p_s32;
+		mvx_profile = mvx_h265_profile_list[i32_val];
+		ret = mvx_session_set_profile(session, MVX_FORMAT_HEVC, mvx_profile);
+		break;
+	case V4L2_CID_MVE_VIDEO_VC1_PROFILE:
+		i32_val = *ctrl->p_new.p_s32;
+		mvx_profile = mvx_vc1_profile_list[i32_val];
+		ret = mvx_session_set_profile(session, MVX_FORMAT_VC1, mvx_profile);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = find_idx(h264_level_list, ARRAY_SIZE(h264_level_list), i32_val);
+		if (ret == -EINVAL)
+			goto unlock_mutex;
+
+		mvx_level = mvx_h264_level_list[ret];
+		ret = mvx_session_set_level(session, MVX_FORMAT_H264, mvx_level);
+		break;
+	case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = find_idx(hevc_level_list, ARRAY_SIZE(hevc_level_list), i32_val);
+		if (ret == -EINVAL)
+			goto unlock_mutex;
+
+		mvx_level = mvx_h265_level_list[ret * 2];
+		ret = mvx_session_set_level(session, MVX_FORMAT_HEVC, mvx_level);
+		break;
+	case V4L2_CID_MVE_VIDEO_H265_LEVEL:
+		i32_val = *ctrl->p_new.p_s32;
+		mvx_level = mvx_h265_level_list[i32_val];
+		ret = mvx_session_set_level(session, MVX_FORMAT_HEVC, mvx_level);
+		break;
+	case V4L2_CID_MVE_VIDEO_IGNORE_STREAM_HEADERS:
+		tri_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_ignore_stream_headers(session, tri_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_FRAME_REORDERING:
+		tri_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_frame_reordering(session, tri_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_INTBUF_SIZE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_intbuf_size(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_P_FRAMES:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_p_frames(session, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_B_FRAMES:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_b_frames(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_GOP_TYPE:
+		i32_val = *ctrl->p_new.p_s32;
+		gop_type = mvx_gop_type_list[i32_val];
+		ret = mvx_session_set_gop_type(session, gop_type);
+		break;
+	case V4L2_CID_MPEG_VIDEO_CYCLIC_INTRA_REFRESH_MB:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_cyclic_intra_refresh_mb(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_CONSTR_IPRED:
+		tri_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_constr_ipred(session, tri_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_ENTROPY_SYNC:
+		tri_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_entropy_sync(session, tri_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_TEMPORAL_MVP:
+		tri_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_temporal_mvp(session, tri_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_TILE_ROWS:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_tile_rows(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_TILE_COLS:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_tile_cols(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_MIN_LUMA_CB_SIZE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_min_luma_cb_size(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_MB_MASK:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_mb_mask(session, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = find_idx(h264_entropy_mode_list, ARRAY_SIZE(h264_entropy_mode_list), i32_val);
+		if (ret == -EINVAL)
+			goto unlock_mutex;
+
+		entropy_mode = mvx_h264_entropy_mode_list[ret];
+		ret = mvx_session_set_entropy_mode(session, entropy_mode);
+		break;
+	case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME:
+		mvx_session_set_force_idr(session);
+		break;
+	case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = find_idx(multi_slice_mode_list, ARRAY_SIZE(multi_slice_mode_list), i32_val);
+		if (ret == -EINVAL)
+			goto unlock_mutex;
+
+		multi_slice_mode = mvx_multi_slice_mode_list[ret];
+		ret = mvx_session_set_multi_slice_mode(session, multi_slice_mode);
+		break;
+	case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MAX_MB:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_multi_slice_max_mb(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_VP9_PROB_UPDATE:
+		i32_val = *ctrl->p_new.p_s32;
+		vp9_prob_update = mvx_vp9_prob_update_list[i32_val];
+		ret = mvx_session_set_vp9_prob_update(session, vp9_prob_update);
+		break;
+	case V4L2_CID_MPEG_VIDEO_MV_H_SEARCH_RANGE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_mv_h_search_range(session, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_mv_v_search_range(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_BITDEPTH_CHROMA:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_bitdepth_chroma(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_BITDEPTH_LUMA:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_bitdepth_luma(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_FORCE_CHROMA_FORMAT:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_force_chroma_format(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_RGB_TO_YUV_MODE:
+		i32_val = *ctrl->p_new.p_s32;
+		rgb_to_yuv_mode = mvx_rgb_to_yuv_mode_list[i32_val];
+		ret = mvx_session_set_rgb_to_yuv_mode(session, rgb_to_yuv_mode);
+		break;
+	case V4L2_CID_MVE_VIDEO_BANDWIDTH_LIMIT:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_band_limit(session, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_CABAC_INIT_IDC:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_cabac_init_idc(session, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_i_frame_qp(session, MVX_FORMAT_H263, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_p_frame_qp(session, MVX_FORMAT_H263, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_b_frame_qp(session, MVX_FORMAT_H263, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H263_MIN_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_min_qp(session, MVX_FORMAT_H263, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H263_MAX_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_max_qp(session, MVX_FORMAT_H263, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_i_frame_qp(session, MVX_FORMAT_H264, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_p_frame_qp(session, MVX_FORMAT_H264, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_b_frame_qp(session, MVX_FORMAT_H264, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_MIN_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_min_qp(session, MVX_FORMAT_H264, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_MAX_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_max_qp(session, MVX_FORMAT_H264, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_i_frame_qp(session, MVX_FORMAT_HEVC, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_p_frame_qp(session, MVX_FORMAT_HEVC, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_b_frame_qp(session, MVX_FORMAT_HEVC, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_min_qp(session, MVX_FORMAT_HEVC, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_max_qp(session, MVX_FORMAT_HEVC, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_VPX_I_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_i_frame_qp(session, MVX_FORMAT_VP9, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_VPX_P_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_p_frame_qp(session, MVX_FORMAT_VP9, i32_val);
+		break;
+	case V4L2_CID_MVE_VIDEO_VPX_B_FRAME_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_b_frame_qp(session, MVX_FORMAT_VP9, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_VPX_MIN_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_min_qp(session, MVX_FORMAT_VP9, i32_val);
+		break;
+	case V4L2_CID_MPEG_VIDEO_VPX_MAX_QP:
+		i32_val = *ctrl->p_new.p_s32;
+		ret = mvx_session_set_max_qp(session, MVX_FORMAT_VP9, i32_val);
+		break;
+	case V4L2_CID_JPEG_RESTART_INTERVAL:
+		i32_val = *ctrl->p_new.p_s32;
+		if (i32_val != -1)
+			ret = mvx_session_set_resync_interval(session, i32_val);
+
+		break;
+	case V4L2_CID_JPEG_COMPRESSION_QUALITY:
+		i32_val = *ctrl->p_new.p_s32;
+		if (i32_val != 0)
+			ret = mvx_session_set_jpeg_quality(session, i32_val);
+
+		break;
+	case V4L2_CID_MVE_VIDEO_WATCHDOG_TIMEOUT:
+		i32_val = *ctrl->p_new.p_s32;
+		if (i32_val != 0)
+			ret = mvx_session_set_watchdog_timeout(session, i32_val);
+
+		break;
+	case V4L2_CID_MVE_VIDEO_PROFILING:
+		i32_val = *ctrl->p_new.p_s32;
+		if (i32_val != 0)
+			ret = mvx_session_set_profiling(session, i32_val);
+
+		break;
+	}
+
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+/**
+ * get_volatile_ctrl() - Get control value.
+ * @ctrl:	V4L2 control.
+ *
+ * Return: 0 on success, else error code.
+ */
+static int get_volatile_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct mvx_v4l2_session *vsession = container_of(ctrl->handler, struct mvx_v4l2_session, v4l2_ctrl);
+
+	switch (ctrl->id) {
+	case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT:
+		ctrl->val = vsession->session.port[MVX_DIR_INPUT].buffer_min;
+		break;
+	case V4L2_CID_MIN_BUFFERS_FOR_CAPTURE:
+		ctrl->val = vsession->session.port[MVX_DIR_OUTPUT].buffer_min;
+		break;
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unsupported get control. id=%u.", ctrl->id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/*
+ * Callbacks required by V4L2 framework to implement controls support.
+ */
+static const struct v4l2_ctrl_ops ctrl_ops = {.g_volatile_ctrl = get_volatile_ctrl, .s_ctrl = set_ctrl};
+
+/**
+ * get_skip_mask() - Calculate V4L2 menu skip mask.
+ * @list:	Array of menu items.
+ * @cnt:	Number of menu items.
+ *
+ * Return: V4L2 menu skip mask.
+ */
+static uint64_t get_skip_mask(const uint8_t *list, size_t cnt)
+{
+	uint64_t mask = 0;
+	int i;
+
+	for (i = 0; i < cnt; ++i)
+		mvx_set_bit(list[i], &mask);
+
+	return ~mask;
+}
+
+/**
+ * mvx_v4l2_ctrl_new_custom_int() - Create custom V4L2 integer control.
+ * @hnd:	V4L2 handler.
+ * @id:		Id of a control.
+ * @name:	Name of a control.
+ * @min:	Minimum allowed value.
+ * @max:	Maximum allowed value.
+ * @def:	Default value.
+ * @step:	Step.
+ *
+ * Return: Pointer to v4l2_ctrl structure in case of success,
+ *         or NULL in case of failure.
+ */
+static struct v4l2_ctrl *mvx_v4l2_ctrl_new_custom_int(struct v4l2_ctrl_handler *hnd, int id, const char *name,
+						      int64_t min, int64_t max, int64_t def, int32_t step)
+{
+	struct v4l2_ctrl_config cfg;
+
+	memset(&cfg, 0, sizeof(cfg));
+
+	cfg.id = id;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_INTEGER;
+	cfg.name = name;
+	cfg.min = min;
+	cfg.max = max;
+	cfg.def = def;
+	cfg.step = step;
+
+	return v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+}
+
+/**
+ * mvx_v4l2_ctrl_new_custom_tristate() - Create custom V4L2 tristate control.
+ * @hnd:	V4L2 handler.
+ * @id:		Id of a control.
+ * @name:	Name of a control.
+ * @def:	Default value.
+ *
+ * Return: Pointer to v4l2_ctrl structure in case of success,
+ *         or NULL in case of failure.
+ */
+static struct v4l2_ctrl *mvx_v4l2_ctrl_new_custom_tristate(struct v4l2_ctrl_handler *hnd, int id, const char *name,
+							   enum mvx_tristate def)
+{
+	struct v4l2_ctrl_config cfg;
+
+	memset(&cfg, 0, sizeof(cfg));
+
+	cfg.id = id;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_INTEGER;
+	cfg.name = name;
+	cfg.min = -1;
+	cfg.max = 1;
+	cfg.def = def;
+	cfg.step = 1;
+
+	return v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+}
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+int mvx_v4l2_ctrls_init(struct v4l2_ctrl_handler *hnd)
+{
+	int ret;
+	struct v4l2_ctrl_config cfg;
+	struct v4l2_ctrl *ctrl;
+
+	ret = v4l2_ctrl_handler_init(hnd, 128);
+	if (ret != 0)
+		return ret;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_SECURE_VIDEO, "secure video", 0, 1, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_FRAME_RATE, "frame rate", 0, 0x10000000, 30 << 16,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_CROP_LEFT, "bitrate control mode", 0, 10000000, 0,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_CROP_RIGHT, "bitrate control mode", 0, 10000000, 0,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_CROP_TOP, "bitrate control mode", 0, 10000000, 0,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_CROP_BOTTOM, "bitrate control mode", 0, 10000000, 0,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_HRD_BUFFER_SIZE, "HRD buffer size", 0, 1073741823,
+					    0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE, 0, 1, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std_menu(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_BITRATE_MODE, V4L2_MPEG_VIDEO_BITRATE_MODE_CQ,
+				      ~7, V4L2_MPEG_VIDEO_BITRATE_MODE_VBR);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_BITRATE_PEAK, 1000, 1000000000, 1, 64000);
+	if (ctrl == NULL)
+		goto handler_free;
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_BITRATE, 1000, 1000000000, 1, 64000);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_NALU_FORMAT;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "nalu format";
+	cfg.max = ARRAY_SIZE(nalu_format_str) - 1;
+	cfg.def = 0;
+	cfg.qmenu = nalu_format_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_tristate(hnd, V4L2_CID_MVE_VIDEO_STREAM_ESCAPING, "stream escaping",
+						 MVX_TRI_UNSET);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std_menu(
+		hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_PROFILE, V4L2_MPEG_VIDEO_H264_PROFILE_HIGH_10,
+		get_skip_mask(h264_profile_list, ARRAY_SIZE(h264_profile_list)), V4L2_MPEG_VIDEO_H264_PROFILE_MAIN);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std_menu(
+		hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_HEVC_PROFILE, V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+		get_skip_mask(hevc_profile_list, ARRAY_SIZE(hevc_profile_list)), V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_H265_PROFILE;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "h265 profile";
+	cfg.max = ARRAY_SIZE(h265_profile_str) - 1;
+	cfg.def = 0;
+	cfg.qmenu = h265_profile_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_VC1_PROFILE;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "vc1 profile";
+	cfg.max = ARRAY_SIZE(vc1_profile_str) - 1;
+	cfg.def = 0;
+	cfg.qmenu = vc1_profile_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std_menu(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, V4L2_MPEG_VIDEO_H264_LEVEL_5_1,
+				      get_skip_mask(h264_level_list, ARRAY_SIZE(h264_level_list)),
+				      V4L2_MPEG_VIDEO_H264_LEVEL_1_0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std_menu(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_HEVC_LEVEL, V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
+				      get_skip_mask(hevc_level_list, ARRAY_SIZE(hevc_level_list)),
+				      V4L2_MPEG_VIDEO_HEVC_LEVEL_1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_H265_LEVEL;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "h265 level";
+	cfg.max = ARRAY_SIZE(h265_level_str) - 1;
+	cfg.def = cfg.max;
+	cfg.qmenu = h265_level_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_tristate(hnd, V4L2_CID_MVE_VIDEO_IGNORE_STREAM_HEADERS, "ignore stream headers",
+						 MVX_TRI_UNSET);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_tristate(hnd, V4L2_CID_MVE_VIDEO_FRAME_REORDERING, "frame reordering",
+						 MVX_TRI_UNSET);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_INTBUF_SIZE, "internal buffer size", 0, INT_MAX, 0,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_P_FRAMES, "video P frames", 0, INT_MAX, 30, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, 0, INT_MAX, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_GOP_TYPE;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "GOP type";
+	cfg.max = ARRAY_SIZE(gop_type_str) - 1;
+	cfg.def = 0;
+	cfg.qmenu = gop_type_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_CYCLIC_INTRA_REFRESH_MB, 0, INT_MAX, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_tristate(hnd, V4L2_CID_MVE_VIDEO_CONSTR_IPRED, "constrained intra prediction",
+						 MVX_TRI_UNSET);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_tristate(hnd, V4L2_CID_MVE_VIDEO_ENTROPY_SYNC, "entropy sync", MVX_TRI_UNSET);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_tristate(hnd, V4L2_CID_MVE_VIDEO_TEMPORAL_MVP, "temporal mvp", MVX_TRI_UNSET);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_TILE_ROWS, "tile rows", 0, 65536, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_TILE_COLS, "tile columns", 0, 65536, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_MIN_LUMA_CB_SIZE, "min luma cb size", 8, 16, 8, 8);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_MB_MASK;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_BITMASK;
+	cfg.name = "macroblocks mask";
+	cfg.def = 0x7fff;
+	cfg.min = 0;
+	cfg.max = 0x7fff;
+	cfg.step = 0;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "H264 Entropy Mode";
+	cfg.max = ARRAY_SIZE(h264_entropy_mode_str) - 1;
+	cfg.def = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_NONE;
+	cfg.qmenu = h264_entropy_mode_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std_menu(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE,
+				      V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_MAX_MB,
+				      get_skip_mask(multi_slice_mode_list, ARRAY_SIZE(multi_slice_mode_list)),
+				      V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_SINGLE);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MAX_MB, 0, INT_MAX, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_VP9_PROB_UPDATE;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "VP9 prob update";
+	cfg.max = ARRAY_SIZE(vp9_prob_update_str) - 1;
+	cfg.def = cfg.max;
+	cfg.qmenu = vp9_prob_update_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_MV_H_SEARCH_RANGE, 0, INT_MAX, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE, 0, INT_MAX, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_BITDEPTH_CHROMA, "bitdepth chroma", 0, 0xff, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_BITDEPTH_LUMA, "bitdepth luma", 0, 0xff, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_FORCE_CHROMA_FORMAT, "force chroma format", -1,
+					    INT_MAX, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.id = V4L2_CID_MVE_VIDEO_RGB_TO_YUV_MODE;
+	cfg.ops = &ctrl_ops;
+	cfg.type = V4L2_CTRL_TYPE_MENU;
+	cfg.name = "RGB to YUV conversion mode";
+	cfg.max = ARRAY_SIZE(rgb_to_yuv_mode_str) - 1;
+	cfg.def = cfg.max;
+	cfg.qmenu = rgb_to_yuv_mode_str;
+	ctrl = v4l2_ctrl_new_custom(hnd, &cfg, NULL);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_BANDWIDTH_LIMIT, "bandwidth limit", 0, INT_MAX, 0,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_CABAC_INIT_IDC, "CABAC init IDC", 0, 4, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_WATCHDOG_TIMEOUT, "watchdog timeout", 5, 60, 30, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_PROFILING, "enable profiling", 0, 1, 0, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP, 0, 31, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP, 0, 31, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP, 0, 31, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H263_MIN_QP, 1, 31, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H263_MAX_QP, 1, 31, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_I_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_P_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_MIN_QP, 1, 51, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_MAX_QP, 1, 51, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP, 1, 51, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP, 1, 51, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_VPX_I_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_VPX_P_FRAME_QP, 0, 51, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = mvx_v4l2_ctrl_new_custom_int(hnd, V4L2_CID_MVE_VIDEO_VPX_B_FRAME_QP, "VPx B frame QP value", 0, 51, 0,
+					    1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_VPX_MIN_QP, 1, 51, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_VPX_MAX_QP, 1, 51, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_VOLATILE;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MIN_BUFFERS_FOR_CAPTURE, 1, 32, 1, 1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_VOLATILE;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_JPEG_RESTART_INTERVAL, -1, 0xffff, 1, -1);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_JPEG_COMPRESSION_QUALITY, 0, 100, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ctrl = v4l2_ctrl_new_std(hnd, &ctrl_ops, V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME, 0, 1, 1, 0);
+	if (ctrl == NULL)
+		goto handler_free;
+
+	ret = v4l2_ctrl_handler_setup(hnd);
+	if (ret != 0)
+		goto handler_free;
+
+	return 0;
+
+handler_free:
+	v4l2_ctrl_handler_free(hnd);
+	return -EINVAL;
+}
+
+void mvx_v4l2_ctrls_done(struct v4l2_ctrl_handler *hnd)
+{
+	v4l2_ctrl_handler_free(hnd);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_ctrls.h b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_ctrls.h
new file mode 100644
index 000000000000..63f52695000f
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_ctrls.h
@@ -0,0 +1,64 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_V4L2_CTRLS_H_
+#define _MVX_V4L2_CTRLS_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <media/v4l2-ctrls.h>
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_v4l2_ctrls_init() - Initialize V4L2 control handler.
+ * @hnd:	V4L2 control handler.
+ *
+ * This function initializes V4L2 controls for handler @hnd.
+ * Controls set to their default values.
+ *
+ * Return: 0 on success, error code otherwise.
+ */
+int mvx_v4l2_ctrls_init(struct v4l2_ctrl_handler *hnd);
+
+/**
+ * mvx_v4l2_ctrls_done() - Destroy V4L2 control handler.
+ * @hnd:	V4L2 control handler.
+ *
+ * This function destroys V4L2 control handler.
+ */
+void mvx_v4l2_ctrls_done(struct v4l2_ctrl_handler *hnd);
+
+#endif /* _MVX_V4L2_CTRLS_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_fops.c b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_fops.c
new file mode 100644
index 000000000000..c3df0e948f02
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_fops.c
@@ -0,0 +1,205 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <linux/fs.h>
+#include <linux/version.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-event.h>
+#include "mvx_ext_if.h"
+#include "mvx_v4l2_buffer.h"
+#include "mvx_v4l2_ctrls.h"
+#include "mvx_v4l2_fops.h"
+#include "mvx_v4l2_session.h"
+#include "mvx_v4l2_vidioc.h"
+#include "mvx_log_group.h"
+
+/****************************************************************************
+ * Exported functions and variables
+ ****************************************************************************/
+
+int mvx_v4l2_open(struct file *file)
+{
+	struct mvx_ext_if *ctx = video_drvdata(file);
+	struct mvx_v4l2_session *session;
+	struct v4l2_format fmt = {0};
+	int ret;
+
+	session = devm_kzalloc(ctx->dev, sizeof(*session), GFP_KERNEL);
+	if (session == NULL) {
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Failed to allocate V4L2 session.");
+		return -ENOMEM;
+	}
+
+	MVX_SESSION_INFO(&session->session, "v4l2: Open device. id=%u.", ctx->dev->id);
+
+	ret = mvx_v4l2_session_construct(session, ctx);
+	if (ret != 0)
+		goto free_session;
+
+	file->private_data = &session->fh;
+	v4l2_fh_init(&session->fh, &ctx->vdev);
+#if (KERNEL_VERSION(6, 18, 0) <= LINUX_VERSION_CODE)
+	v4l2_fh_add(&session->fh, file);
+#else
+	v4l2_fh_add(&session->fh);
+#endif
+
+	/* Set default port formats. */
+	fmt.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+	fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUV420M;
+	fmt.fmt.pix.width = 2;
+	fmt.fmt.pix.height = 2;
+	(void)mvx_v4l2_vidioc_s_fmt_vid_out(file, NULL, &fmt);
+
+	fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	(void)mvx_v4l2_vidioc_s_fmt_vid_cap(file, NULL, &fmt);
+
+	ret = mvx_v4l2_ctrls_init(&session->v4l2_ctrl);
+	if (ret != 0) {
+		MVX_SESSION_WARN(&session->session, "Failed to register V4L2 controls handler. ret=%x", ret);
+		goto put_session;
+	}
+
+	session->fh.ctrl_handler = &session->v4l2_ctrl;
+
+	return 0;
+
+put_session:
+
+	/*
+	 * Session was completely constructed, so we have to destroy it
+	 * gracefully using reference counting.
+	 */
+	mvx_session_put(&session->session);
+	return ret;
+
+free_session:
+	devm_kfree(ctx->dev, session);
+
+	return ret;
+}
+
+int mvx_v4l2_release(struct file *file)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	int i;
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Release.");
+
+	mvx_v4l2_ctrls_done(vsession->fh.ctrl_handler);
+
+	mutex_lock(&vsession->mutex);
+
+	for (i = 0; i < MVX_DIR_MAX; i++)
+		if (vsession->port[i].q_set != false) {
+			vb2_queue_release(&vsession->port[i].vb2_queue);
+			vsession->port[i].q_set = false;
+		}
+
+#if (KERNEL_VERSION(6, 18, 0) <= LINUX_VERSION_CODE)
+	v4l2_fh_del(&vsession->fh, file);
+#else
+	v4l2_fh_del(&vsession->fh);
+#endif
+	ret = mvx_session_put(&vsession->session);
+	if (ret == 0)
+		mutex_unlock(&vsession->mutex);
+
+	file->private_data = NULL;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Release exit.");
+
+	return 0;
+}
+
+unsigned int mvx_v4l2_poll(struct file *file, struct poll_table_struct *wait)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	unsigned long events = poll_requested_events(wait);
+	unsigned int revents = 0;
+
+	mutex_lock(&vsession->mutex);
+
+	if (vsession->session.error != 0) {
+		revents = POLLERR;
+		goto unlock_mutex;
+	}
+
+	/* POLLPRI events are handled by Vb2 */
+	if (vsession->port[MVX_DIR_INPUT].q_set)
+		revents |= vb2_poll(&vsession->port[MVX_DIR_INPUT].vb2_queue, file, wait);
+	if (vsession->port[MVX_DIR_OUTPUT].q_set)
+		revents |= vb2_poll(&vsession->port[MVX_DIR_OUTPUT].vb2_queue, file, wait);
+#ifndef MODULE
+	MVX_SESSION_VERBOSE(&vsession->session, "v4l2: Poll. events=0x%lx, revents=0x%x, nevents=%d.", events, revents,
+			    v4l2_event_pending(&vsession->fh));
+#else
+	MVX_SESSION_VERBOSE(&vsession->session, "v4l2: Poll. events=0x%lx, revents=0x%x", events, revents);
+#endif
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+	return revents;
+}
+
+int mvx_v4l2_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct mvx_v4l2_session *session = file_to_session(file);
+	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+	enum mvx_direction dir;
+	struct mvx_v4l2_port *vport;
+	struct vb2_queue *q;
+	int ret;
+
+	MVX_SESSION_INFO(&session->session,
+			 "v4l2: Memory map. start=0x%08lx, end=0x%08lx, pgoff=0x%08lx, flags=0x%08lx.", vma->vm_start,
+			 vma->vm_end, vma->vm_pgoff, vma->vm_flags);
+
+	if (offset >= DST_QUEUE_OFF_BASE) {
+		dir = MVX_DIR_OUTPUT;
+		vma->vm_pgoff -= (DST_QUEUE_OFF_BASE >> PAGE_SHIFT);
+	} else {
+		dir = MVX_DIR_INPUT;
+	}
+
+	vport = &session->port[dir];
+	q = &vport->vb2_queue;
+
+	ret = vb2_mmap(q, vma);
+	if (ret != 0) {
+		MVX_SESSION_WARN(&session->session, "Failed to memory map buffer. q=%p, pgoff=0x%08lx, dir=%d, ret=%d",
+				 q, vma->vm_pgoff, dir, ret);
+		return ret;
+	}
+
+	return 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_fops.h b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_fops.h
new file mode 100644
index 000000000000..32cecfc4fae6
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_fops.h
@@ -0,0 +1,62 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_V4L2_FOPS_H_
+#define _MVX_V4L2_FOPS_H_
+
+/*
+ * Callbacks for struct v4l2_file_operations.
+ *
+ * Prototypes declared bellow represent callbacks required by v4l2 framework.
+ * They are needed to implement certain syscalls.
+ */
+
+/**
+ * mvx_v4l2_open() - Callback needed to implement the open() syscall.
+ */
+int mvx_v4l2_open(struct file *file);
+
+/**
+ * mvx_v4l2_release() - Callback needed to implement the release() syscall.
+ */
+int mvx_v4l2_release(struct file *file);
+
+/**
+ * mvx_v4l2_poll() - Callback needed to implement the poll() syscall.
+ */
+unsigned int mvx_v4l2_poll(struct file *file, struct poll_table_struct *wait);
+
+/**
+ * mvx_v4l2_mmap() - Callback needed to implement the mmap() syscall.
+ */
+int mvx_v4l2_mmap(struct file *file, struct vm_area_struct *vma);
+
+#endif /* _MVX_V4L2_FOPS_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_session.c b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_session.c
new file mode 100644
index 000000000000..e7bf07bf9fe6
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_session.c
@@ -0,0 +1,570 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/debugfs.h>
+#include <linux/sched.h>
+#include <linux/version.h>
+#include <media/v4l2-event.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-dma-sg.h>
+#include "mvx_ext_if.h"
+#include "mvx_seq.h"
+#include "mvx_v4l2_buffer.h"
+#include "mvx_v4l2_session.h"
+#include "mvx_log_group.h"
+#include "mvx-v4l2-controls.h"
+
+/****************************************************************************
+ * Exported and static functions
+ ****************************************************************************/
+
+static void set_format(struct v4l2_pix_format_mplane *pix_mp, unsigned int width, unsigned int height,
+		       unsigned int num_planes, unsigned int *sizeimage, unsigned int *bytesperline)
+{
+	int i;
+
+	pix_mp->width = width;
+	pix_mp->height = height;
+	pix_mp->num_planes = num_planes;
+
+	for (i = 0; i < num_planes; ++i) {
+		pix_mp->plane_fmt[i].sizeimage = sizeimage[i];
+		pix_mp->plane_fmt[i].bytesperline = bytesperline[i];
+	}
+}
+
+static void v4l2_port_show(struct mvx_v4l2_port *port, struct seq_file *s)
+{
+	mvx_seq_printf(s, "mvx_v4l2_port", 0, "%p\n", port);
+	mvx_seq_printf(s, "pixelformat", 1, "0x%x\n", port->pix_mp.pixelformat);
+	mvx_seq_printf(s, "vb2_queue", 1, "\n");
+	mvx_seq_printf(s, "memory", 2, "%u\n", port->vb2_queue.memory);
+#if (KERNEL_VERSION(6, 7, 0) <= LINUX_VERSION_CODE)
+	mvx_seq_printf(s, "min_buffers_needed", 2, "%u\n", port->vb2_queue.min_queued_buffers);
+#else
+	mvx_seq_printf(s, "min_buffers_needed", 2, "%u\n", port->vb2_queue.min_buffers_needed);
+#endif
+	mvx_seq_printf(s, "num_buffers", 2, "%u\n", vb2_get_num_buffers(&port->vb2_queue));
+
+	mvx_seq_printf(s, "queued_count", 2, "%u\n", port->vb2_queue.queued_count);
+	mvx_seq_printf(s, "streaming", 2, "%u\n", port->vb2_queue.streaming);
+	mvx_seq_printf(s, "error", 2, "%u\n", port->vb2_queue.error);
+	mvx_seq_printf(s, "last_buffer_dequeued", 2, "%u\n", port->vb2_queue.last_buffer_dequeued);
+}
+
+static int port_stat_show(struct seq_file *s, void *v)
+{
+	struct mvx_v4l2_port *vport = s->private;
+	struct mvx_session_port *sport = vport->port;
+
+	mvx_session_port_show(sport, s);
+	seq_puts(s, "\n");
+	v4l2_port_show(vport, s);
+
+	return 0;
+}
+
+static int port_stat_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, port_stat_show, inode->i_private);
+}
+
+static const struct file_operations port_stat_fops = {
+	.open = port_stat_open, .read = seq_read, .llseek = seq_lseek, .release = single_release};
+
+static int port_debugfs_init(struct device *dev, unsigned int i, struct mvx_v4l2_port *vport,
+			     struct mvx_session_port *sport, struct dentry *parent)
+{
+	char name[20];
+	struct dentry *dentry;
+
+	scnprintf(name, sizeof(name), "port%u", i);
+	vport->dentry = debugfs_create_dir(name, parent);
+	if (IS_ERR_OR_NULL(vport->dentry))
+		return -ENOMEM;
+
+	dentry = debugfs_create_file("stat", 0400, vport->dentry, vport, &port_stat_fops);
+	if (IS_ERR_OR_NULL(dentry))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int session_debugfs_init(struct mvx_v4l2_session *session, struct dentry *parent)
+{
+	int ret;
+	char name[20];
+	int i;
+
+	scnprintf(name, sizeof(name), "%lx", (unsigned long)(&session->session));
+	session->dentry = debugfs_create_dir(name, parent);
+	if (IS_ERR_OR_NULL(session->dentry))
+		return -ENOMEM;
+
+	for (i = 0; i < MVX_DIR_MAX; i++) {
+		struct mvx_v4l2_port *vport = &session->port[i];
+		struct mvx_session_port *mport = &session->session.port[i];
+
+		ret = port_debugfs_init(session->ext->dev, i, vport, mport, session->dentry);
+		if (ret != 0)
+			goto remove_dentry;
+	}
+
+	return 0;
+
+remove_dentry:
+	debugfs_remove_recursive(session->dentry);
+	return ret;
+}
+
+static struct mvx_v4l2_session *mvx_session_to_v4l2_session(struct mvx_session *session)
+{
+	return container_of(session, struct mvx_v4l2_session, session);
+}
+
+static void free_session(struct mvx_session *session)
+{
+	struct mvx_v4l2_session *s = mvx_session_to_v4l2_session(session);
+
+	MVX_SESSION_INFO(session, "v4l2: Destroy session.");
+
+	mvx_session_destruct(session);
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(s->dentry);
+
+	v4l2_fh_exit(&s->fh);
+	if (mutex_is_locked(&s->mutex)) {
+		mutex_unlock(&s->mutex);
+	}
+	devm_kfree(s->ext->dev, s);
+}
+#ifdef MODULE
+static unsigned sev_pos(const struct v4l2_subscribed_event *sev, unsigned idx)
+{
+	idx += sev->first;
+	return idx >= sev->elems ? idx - sev->elems : idx;
+}
+
+/* Caller must hold fh->vdev->fh_lock! */
+static struct v4l2_subscribed_event *v4l2_event_subscribed(struct v4l2_fh *fh, u32 type, u32 id)
+{
+	struct v4l2_subscribed_event *sev;
+
+	assert_spin_locked(&fh->vdev->fh_lock);
+
+	list_for_each_entry(sev, &fh->subscribed, list) if (sev->type == type && sev->id == id) return sev;
+
+	return NULL;
+}
+
+static void __v4l2_event_queue_fh(struct v4l2_fh *fh, const struct v4l2_event *ev, u64 ts)
+{
+	struct v4l2_subscribed_event *sev;
+	struct v4l2_kevent *kev;
+	bool copy_payload = true;
+
+	/* Are we subscribed? */
+	sev = v4l2_event_subscribed(fh, ev->type, ev->id);
+	if (sev == NULL)
+		return;
+
+	/* Increase event sequence number on fh. */
+	fh->sequence++;
+
+	/* Do we have any free events? */
+	if (sev->in_use == sev->elems) {
+		/* no, remove the oldest one */
+		kev = sev->events + sev_pos(sev, 0);
+		list_del(&kev->list);
+		sev->in_use--;
+		sev->first = sev_pos(sev, 1);
+		fh->navailable--;
+		if (sev->elems == 1) {
+			if (sev->ops && sev->ops->replace) {
+				sev->ops->replace(&kev->event, ev);
+				copy_payload = false;
+			}
+		} else if (sev->ops && sev->ops->merge) {
+			struct v4l2_kevent *second_oldest = sev->events + sev_pos(sev, 0);
+			sev->ops->merge(&kev->event, &second_oldest->event);
+		}
+	}
+
+	/* Take one and fill it. */
+	kev = sev->events + sev_pos(sev, sev->in_use);
+	kev->event.type = ev->type;
+	if (copy_payload)
+		kev->event.u = ev->u;
+	kev->event.id = ev->id;
+	kev->ts = ts;
+	kev->event.sequence = fh->sequence;
+	sev->in_use++;
+	list_add_tail(&kev->list, &fh->available);
+
+	fh->navailable++;
+
+	wake_up_all(&fh->wait);
+}
+
+void mvx_v4l2_event_queue_fh(struct v4l2_fh *fh, const struct v4l2_event *ev)
+{
+	unsigned long flags;
+	u64 ts = ktime_get_ns();
+
+	spin_lock_irqsave(&fh->vdev->fh_lock, flags);
+	__v4l2_event_queue_fh(fh, ev, ts);
+	spin_unlock_irqrestore(&fh->vdev->fh_lock, flags);
+}
+void mvx_vb2_queue_error(struct vb2_queue *q)
+{
+	q->error = 1;
+
+	wake_up_all(&q->done_wq);
+}
+
+#endif
+static void handle_event(struct mvx_session *session, enum mvx_session_event event, void *arg)
+{
+	struct mvx_v4l2_session *vsession = mvx_session_to_v4l2_session(session);
+
+	MVX_SESSION_INFO(&vsession->session, "Event. event=%d, arg=%p.", event, arg);
+
+	switch (event) {
+	case MVX_SESSION_EVENT_BUFFER: {
+		struct mvx_v4l2_buffer *vbuf = mvx_buffer_to_v4l2_buffer(arg);
+		struct vb2_buffer *vb = &vbuf->vb2_v4l2_buffer.vb2_buf;
+
+		/*
+		 * When streaming is stopped we don't always receive all
+		 * buffers from FW back. So we just return them all to Vb2.
+		 * If the FW later returns a buffer to us, we could silently
+		 * skip it.
+		 */
+		if (vb->state != VB2_BUF_STATE_DEQUEUED && vb->state != VB2_BUF_STATE_DONE) {
+			enum vb2_buffer_state state = mvx_v4l2_buffer_update(vbuf);
+
+			vb2_buffer_done(vb, state);
+		}
+
+		if (vbuf->buf.dir == MVX_DIR_OUTPUT && vb->state != VB2_BUF_STATE_QUEUED &&
+		    !waitqueue_active(&vb->vb2_queue->done_wq) && vsession->port[MVX_DIR_INPUT].q_set &&
+		    waitqueue_active(&vsession->port[MVX_DIR_INPUT].vb2_queue.done_wq)) {
+			/*wake up waiters of input queue, because output queue may reallocated and not registered in
+			 * waiters list */
+			wake_up(&vsession->port[MVX_DIR_INPUT].vb2_queue.done_wq);
+		}
+		break;
+	}
+	case MVX_SESSION_EVENT_PORT_CHANGED: {
+		enum mvx_direction dir = (enum mvx_direction)arg;
+		struct mvx_v4l2_port *vport = &vsession->port[dir];
+		struct mvx_session_port *port = &session->port[dir];
+		const struct v4l2_event event = {
+			.type                 = V4L2_EVENT_SOURCE_CHANGE,
+			.u.src_change.changes = V4L2_EVENT_SRC_CH_RESOLUTION
+		};
+		struct v4l2_pix_format_mplane *p = &vport->pix_mp;
+		p->field = port->interlaced ? V4L2_FIELD_SEQ_TB : V4L2_FIELD_NONE;
+		set_format(&vport->pix_mp, port->width, port->height, port->nplanes, port->size, port->stride);
+#ifndef MODULE
+		v4l2_event_queue_fh(&vsession->fh, &event);
+#else
+		mvx_v4l2_event_queue_fh(&vsession->fh, &event);
+#endif
+		break;
+	}
+	case MVX_SESSION_EVENT_COLOR_DESC: {
+		const struct v4l2_event event = {
+			.type = V4L2_EVENT_MVX_COLOR_DESC,
+		};
+#ifndef MODULE
+		v4l2_event_queue_fh(&vsession->fh, &event);
+#else
+		mvx_v4l2_event_queue_fh(&vsession->fh, &event);
+#endif
+		break;
+	}
+	case MVX_SESSION_EVENT_ERROR: {
+		int i;
+
+		for (i = 0; i < MVX_DIR_MAX; ++i) {
+			struct vb2_queue *q = &vsession->port[i].vb2_queue;
+#ifndef MODULE
+			vb2_queue_error(q);
+#else
+			mvx_vb2_queue_error(q);
+#endif
+		}
+
+		break;
+	}
+	default:
+		MVX_SESSION_WARN(&vsession->session, "Unsupported session event. event=%d", event);
+	}
+}
+
+int mvx_v4l2_session_construct(struct mvx_v4l2_session *vsession, struct mvx_ext_if *ctx)
+{
+	int i;
+	int ret;
+
+	vsession->ext = ctx;
+	mutex_init(&vsession->mutex);
+
+	for (i = 0; i < MVX_DIR_MAX; i++) {
+		struct mvx_v4l2_port *vport = &vsession->port[i];
+
+		vport->port = &vsession->session.port[i];
+		vport->vsession = vsession;
+		vport->dir = i;
+		vport->q_set = false;
+	}
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ret = session_debugfs_init(vsession, ctx->dsessions);
+		if (ret != 0)
+			return ret;
+	}
+
+	ret = mvx_session_construct(&vsession->session, ctx->dev, ctx->client_ops, ctx->cache, &vsession->mutex,
+				    free_session, handle_event, vsession->dentry);
+	if (ret != 0)
+		goto remove_dentry;
+
+	return 0;
+
+remove_dentry:
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(vsession->dentry);
+
+	return ret;
+}
+
+struct mvx_v4l2_session *v4l2_fh_to_session(struct v4l2_fh *fh)
+{
+	return container_of(fh, struct mvx_v4l2_session, fh);
+}
+
+struct mvx_v4l2_session *file_to_session(struct file *file) { return v4l2_fh_to_session(file->private_data); }
+
+int mvx_v4l2_session_get_color_desc(struct mvx_v4l2_session *vsession, struct v4l2_mvx_color_desc *c)
+{
+	int ret;
+	struct mvx_fw_color_desc cd;
+
+	ret = mvx_session_get_color_desc(&vsession->session, &cd);
+	if (ret != 0)
+		return ret;
+
+	c->flags = 0;
+
+	/* Convert between generic fw_color_desc and V4L2 color desc. */
+	c->range = cd.range;
+	c->primaries = cd.primaries;
+	c->transfer = cd.transfer;
+	c->matrix = cd.matrix;
+
+	if (cd.flags & MVX_FW_COLOR_DESC_DISPLAY_VALID) {
+		c->flags |= V4L2_MVX_COLOR_DESC_DISPLAY_VALID;
+		c->display.r.x = cd.display.r.x;
+		c->display.r.y = cd.display.r.y;
+		c->display.g.x = cd.display.g.x;
+		c->display.g.y = cd.display.g.y;
+		c->display.b.x = cd.display.b.x;
+		c->display.b.y = cd.display.b.y;
+		c->display.w.x = cd.display.w.x;
+		c->display.w.y = cd.display.w.y;
+		c->display.luminance_min = cd.display.luminance_min;
+		c->display.luminance_max = cd.display.luminance_max;
+	}
+
+	if (cd.flags & MVX_FW_COLOR_DESC_CONTENT_VALID) {
+		c->flags |= V4L2_MVX_COLOR_DESC_CONTENT_VALID;
+		c->content.luminance_max = cd.content.luminance_max;
+		c->content.luminance_average = cd.content.luminance_average;
+	}
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_color_desc(struct mvx_v4l2_session *vsession, struct v4l2_mvx_color_desc *c)
+{
+	int ret;
+	struct mvx_fw_color_desc cd;
+	memset(&cd, 0, sizeof(cd));
+	cd.flags = c->flags;
+	cd.range = c->range;
+	cd.matrix = c->matrix;
+	cd.primaries = c->primaries;
+	cd.transfer = c->transfer;
+	cd.aspect_ratio_idc = c->aspect_ratio_idc;
+	cd.num_units_in_tick = c->num_units_in_tick;
+	cd.sar_height = c->sar_height;
+	cd.sar_width = c->sar_width;
+	cd.video_format = c->video_format;
+	cd.time_scale = c->time_scale;
+	cd.content.luminance_average = c->content.luminance_average;
+	cd.content.luminance_max = c->content.luminance_max;
+	// memcpy(&cd.display, &c->display, sizeof(c->display));
+	cd.display.r.x = c->display.r.x;
+	cd.display.r.y = c->display.r.y;
+	cd.display.g.x = c->display.g.x;
+	cd.display.g.y = c->display.g.y;
+	cd.display.b.x = c->display.b.x;
+	cd.display.b.y = c->display.b.y;
+	cd.display.w.x = c->display.w.x;
+	cd.display.w.y = c->display.w.y;
+	cd.display.luminance_min = c->display.luminance_min;
+	cd.display.luminance_max = c->display.luminance_max;
+	ret = mvx_session_set_color_desc(&vsession->session, &cd);
+	return ret;
+}
+
+int mvx_v4l2_session_set_roi_regions(struct mvx_v4l2_session *vsession, struct v4l2_mvx_roi_regions *roi)
+{
+	int ret;
+	struct mvx_roi_config roi_regions;
+	roi_regions.pic_index = roi->pic_index;
+	roi_regions.num_roi = roi->num_roi;
+	roi_regions.qp_present = roi->qp_present;
+	roi_regions.roi_present = roi->roi_present;
+	roi_regions.qp = roi->qp;
+
+	if (roi_regions.roi_present && roi_regions.num_roi > 0) {
+		int i = 0;
+		for (; i < roi_regions.num_roi; i++) {
+			roi_regions.roi[i].mbx_left = roi->roi[i].mbx_left;
+			roi_regions.roi[i].mbx_right = roi->roi[i].mbx_right;
+			roi_regions.roi[i].mby_top = roi->roi[i].mby_top;
+			roi_regions.roi[i].mby_bottom = roi->roi[i].mby_bottom;
+			roi_regions.roi[i].qp_delta = roi->roi[i].qp_delta;
+		}
+	}
+	ret = mvx_session_set_roi_regions(&vsession->session, &roi_regions);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_qp_epr(struct mvx_v4l2_session *vsession, int *qp)
+{
+	int ret;
+	ret = mvx_session_set_qp_epr(&vsession->session, qp);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_sei_userdata(struct mvx_v4l2_session *vsession, struct v4l2_sei_user_data *sei_userdata)
+{
+	int ret;
+	struct mvx_sei_userdata userdata;
+	userdata.flags = sei_userdata->flags;
+	userdata.user_data_len = sei_userdata->user_data_len;
+	memcpy(&userdata.user_data, &sei_userdata->user_data, sizeof(userdata.user_data));
+	memcpy(&userdata.uuid, &sei_userdata->uuid, sizeof(userdata.uuid));
+	ret = mvx_session_set_sei_userdata(&vsession->session, &userdata);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_rate_control(struct mvx_v4l2_session *vsession, struct v4l2_rate_control *rc)
+{
+	int ret;
+	struct mvx_buffer_param_rate_control mvx_rc;
+	mvx_rc.rate_control_mode = rc->rc_type;
+	mvx_rc.target_bitrate = rc->target_bitrate;
+	mvx_rc.maximum_bitrate = rc->maximum_bitrate;
+	ret = mvx_session_set_bitrate_control(&vsession->session, &mvx_rc);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_dsl_frame(struct mvx_v4l2_session *vsession, struct v4l2_mvx_dsl_frame *dsl)
+{
+	int ret;
+	struct mvx_dsl_frame dsl_frame;
+	dsl_frame.width = dsl->width;
+	dsl_frame.height = dsl->height;
+	ret = mvx_session_set_dsl_frame(&vsession->session, &dsl_frame);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_dsl_ratio(struct mvx_v4l2_session *vsession, struct v4l2_mvx_dsl_ratio *dsl)
+{
+	int ret;
+	struct mvx_dsl_ratio dsl_ratio;
+	dsl_ratio.hor = dsl->hor;
+	dsl_ratio.ver = dsl->ver;
+
+	ret = mvx_session_set_dsl_ratio(&vsession->session, &dsl_ratio);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_long_term_ref(struct mvx_v4l2_session *vsession, struct v4l2_mvx_long_term_ref *ltr)
+{
+	int ret;
+	struct mvx_long_term_ref mvx_ltr;
+	mvx_ltr.mode = ltr->mode;
+	mvx_ltr.period = ltr->period;
+	ret = mvx_session_set_long_term_ref(&vsession->session, &mvx_ltr);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
+
+int mvx_v4l2_session_set_dsl_mode(struct mvx_v4l2_session *vsession, int *mode)
+{
+	int ret;
+	ret = mvx_session_set_dsl_mode(&vsession->session, mode);
+	if (ret != 0)
+		return ret;
+
+	return 0;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_session.h b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_session.h
new file mode 100644
index 000000000000..ae6c03133d44
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_session.h
@@ -0,0 +1,232 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_V4L2_SESSION_H_
+#define _MVX_V4L2_SESSION_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/videodev2.h>
+#include <linux/mutex.h>
+#include <linux/wait.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-fh.h>
+#include <media/videobuf2-v4l2.h>
+
+#include "mvx_session.h"
+#include "mvx-v4l2-controls.h"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+/**
+ * Offset used to distinguish between input and output port.
+ */
+#define DST_QUEUE_OFF_BASE (1 << 30)
+
+/**
+ * struct mvx_v4l2_port - V4L2 port type.
+ *
+ * Most of this structure will become redundant when buffer management
+ * is transferred to Vb2 framework.
+ *
+ * @vsession:		Pointer to corresponding session.
+ * @port:		Pointer to corresponding mvx port.
+ * @dir:		Direction of a port.
+ * @type:		V4L2 port type.
+ * @pix_mp:		V4L2 multi planar pixel format.
+ * @crop:		V4L2 cropping information.
+ * @dentry:		Debugfs directory entry for the port.
+ * @q_set:		Indicates of Vb2 queue was setup.
+ * @vb2_queue:		Vb2 queue.
+ */
+struct mvx_v4l2_port {
+	struct mvx_v4l2_session *vsession;
+	struct mvx_session_port *port;
+	enum mvx_direction dir;
+	enum v4l2_buf_type type;
+	struct v4l2_pix_format_mplane pix_mp;
+	struct v4l2_rect crop;
+	struct dentry *dentry;
+	bool q_set;
+	struct vb2_queue vb2_queue;
+};
+
+/**
+ * struct mvx_v4l2_session - V4L2 session type.
+ * @ext:		Pointer to external interface object.
+ * @fh:			V4L2 file handler.
+ * @mutex:		Mutex protecting the session object.
+ * @session:		Session object.
+ * @port:		Array of v4l2 ports.
+ * @dentry:		Debugfs directory entry representing a session.
+ * @v4l2_ctrl:		v4l2 controls handler.
+ */
+struct mvx_v4l2_session {
+	struct mvx_ext_if *ext;
+	struct v4l2_fh fh;
+	struct mutex mutex;
+	struct mvx_session session;
+	struct mvx_v4l2_port port[MVX_DIR_MAX];
+	struct dentry *dentry;
+	struct v4l2_ctrl_handler v4l2_ctrl;
+};
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_v4l2_session_construct() - Construct v4l2 session object.
+ * @vsession:	Pointer to a session object.
+ * @ctx:	Pointer to an external interface object.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_v4l2_session_construct(struct mvx_v4l2_session *vsession, struct mvx_ext_if *ctx);
+
+/**
+ * v4l2_fh_to_session() - Cast v4l2 file handler to mvx_v4l2_session.
+ * @fh:		v4l2 file handler.
+ *
+ * Return: Pointer to a corresponding mvx_v4l2_session object.
+ */
+struct mvx_v4l2_session *v4l2_fh_to_session(struct v4l2_fh *fh);
+
+/**
+ * file_to_session() - Cast file object to mvx_v4l2_session.
+ * @file:	Pointer to a file object.
+ *
+ * Return: Pointer to a corresponding mvx_v4l2_session object.
+ */
+struct mvx_v4l2_session *file_to_session(struct file *file);
+
+/**
+ * mvx_v4l2_session_get_color_desc() - Get color description.
+ * @vsession:	Pointer to v4l2 session.
+ * @color:	Color description.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_v4l2_session_get_color_desc(struct mvx_v4l2_session *vsession, struct v4l2_mvx_color_desc *color_desc);
+
+/**
+ * mvx_v4l2_session_set_color_desc() - Set color description.
+ * @vsession:	Pointer to v4l2 session.
+ * @color:	Color description.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_color_desc(struct mvx_v4l2_session *vsession, struct v4l2_mvx_color_desc *color_desc);
+
+/**
+ * mvx_v4l2_session_set_sei_userdata() - Set SEI userdata.
+ * @vsession:	Pointer to v4l2 session.
+ * @sei_userdata:	SEI userdata.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_sei_userdata(struct mvx_v4l2_session *vsession, struct v4l2_sei_user_data *sei_userdata);
+
+/**
+ * mvx_v4l2_session_set_roi_regions() - Set Roi Regions.
+ * @vsession:	Pointer to v4l2 session.
+ * @roi:	ROI regions.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_v4l2_session_set_roi_regions(struct mvx_v4l2_session *vsession, struct v4l2_mvx_roi_regions *roi);
+
+/**
+ * mvx_v4l2_session_set_qp_epr() - Set qp.
+ * @vsession:	Pointer to v4l2 session.
+ * @qp:	qp value.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_qp_epr(struct mvx_v4l2_session *vsession, int *qp);
+
+/**
+ * mvx_v4l2_session_set_rate_control() - Set rate Control.
+ * @vsession:	Pointer to v4l2 session.
+ * @rc:	rate control type.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_rate_control(struct mvx_v4l2_session *vsession, struct v4l2_rate_control *rc);
+
+/**
+ * mvx_v4l2_session_set_dsl_frame() - Set DownScale dst frame.
+ * @vsession:	Pointer to v4l2 session.
+ * @dsl:	DownScale dst frame.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_dsl_frame(struct mvx_v4l2_session *vsession, struct v4l2_mvx_dsl_frame *dsl);
+
+/**
+ * mvx_v4l2_session_set_dsl_ratio() - Set DownScale ratio.
+ * @vsession:	Pointer to v4l2 session.
+ * @dsl:	DownScale ratio.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_dsl_ratio(struct mvx_v4l2_session *vsession, struct v4l2_mvx_dsl_ratio *dsl);
+
+/**
+ * mvx_v4l2_session_set_long_term_ref() - Set long term ref.
+ * @vsession:	Pointer to v4l2 session.
+ * @ltr:	long term ref.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_long_term_ref(struct mvx_v4l2_session *vsession, struct v4l2_mvx_long_term_ref *ltr);
+
+/**
+ * mvx_v4l2_session_set_dsl_mode() - Set DownScale mode.
+ * @vsession:	Pointer to v4l2 session.
+ * @mode:	DownScale mode, oly enable on high precision mode.
+ *
+ * Return: 0 on success, else error code.
+ */
+
+int mvx_v4l2_session_set_dsl_mode(struct mvx_v4l2_session *vsession, int *mode);
+#endif /* _MVX_V4L2_SESSION_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_vidioc.c b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_vidioc.c
new file mode 100644
index 000000000000..911d649c91ec
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_vidioc.c
@@ -0,0 +1,1420 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/device.h>
+#include <linux/sched.h>
+#include <linux/version.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-event.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-dma-sg.h>
+#include "mvx_bitops.h"
+#include "mvx_ext_if.h"
+#include "mvx_if.h"
+#include "mvx_v4l2_buffer.h"
+#include "mvx_v4l2_session.h"
+#include "mvx_v4l2_vidioc.h"
+#include "mvx-v4l2-controls.h"
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct mvx_format_map {
+	enum mvx_format format;
+	uint32_t flags;
+	uint32_t pixelformat;
+	const char *description;
+};
+
+/****************************************************************************
+ * Static functions and variables
+ ****************************************************************************/
+
+struct mvx_format_map mvx_fmts[] = {
+	{MVX_FORMAT_AVS, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_AVS, "AVS"},
+	{MVX_FORMAT_AVS2, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_AVS2, "AVS2"},
+	{MVX_FORMAT_H263, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_H263, "H.263"},
+	{MVX_FORMAT_H264, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_H264, "H.264"},
+	{MVX_FORMAT_H264, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_H264_MVC, "H.264 MVC"},
+	{MVX_FORMAT_H264, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_H264_NO_SC, "H.264 (No Start Codes)"},
+	{MVX_FORMAT_HEVC, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_HEVC, "HEVC"},
+	{MVX_FORMAT_MPEG2, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_MPEG2, "MPEG-2 ES"},
+	{MVX_FORMAT_MPEG4, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_MPEG4, "MPEG-4 part 2 ES"},
+	{MVX_FORMAT_RV, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_RV, "Real Video"},
+	{MVX_FORMAT_VC1, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_VC1_ANNEX_G, "VC-1 (SMPTE 412M Annex G)"},
+	{MVX_FORMAT_VC1, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_VC1_ANNEX_L, "VC-1 (SMPTE 412M Annex L)"},
+	{MVX_FORMAT_VP8, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_VP8, "VP8"},
+	{MVX_FORMAT_VP9, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_VP9, "VP9"},
+	{MVX_FORMAT_JPEG, V4L2_FMT_FLAG_COMPRESSED, V4L2_PIX_FMT_JPEG, "JPEG"},
+	{MVX_FORMAT_YUV420_AFBC_8, 0, V4L2_PIX_FMT_YUV420_AFBC_8, "YUV420 AFBC 8 bit"},
+	{MVX_FORMAT_YUV420_AFBC_10, 0, V4L2_PIX_FMT_YUV420_AFBC_10, "YUV420 AFBC 10 bit"},
+	{MVX_FORMAT_YUV422_AFBC_8, 0, V4L2_PIX_FMT_YUV422_AFBC_8, "YUV422 AFBC 8 bit"},
+	{MVX_FORMAT_YUV422_AFBC_10, 0, V4L2_PIX_FMT_YUV422_AFBC_10, "YUV422 AFBC 10 bit"},
+	{MVX_FORMAT_YUV420_I420, 0, V4L2_PIX_FMT_YUV420M, "Planar YUV 4:2:0 (N-C)"},
+	{MVX_FORMAT_YUV420_NV12, 0, V4L2_PIX_FMT_NV12, "Y/CbCr 4:2:0"},
+	{MVX_FORMAT_YUV420_NV21, 0, V4L2_PIX_FMT_NV21, "Y/CrCb 4:2:0 (N-C)"},
+	{MVX_FORMAT_YUV420_NV12, 0, V4L2_PIX_FMT_NV12M, "Y/CbCr 4:2:0"},
+	{MVX_FORMAT_YUV420_P010, 0, V4L2_PIX_FMT_P010, "YUV 4:2:0 P010 (Microsoft format)"},
+	{MVX_FORMAT_YUV420_Y0L2, 0, V4L2_PIX_FMT_Y0L2, "YUV 4:2:0 Y0L2 (ARM format)"},
+	{MVX_FORMAT_YUV420_AQB1, 0, v4l2_fourcc('Y', '0', 'A', 'B'), "YUV 4:2:0 AQB1 (ARM format)"},
+	{MVX_FORMAT_YUV422_YUY2, 0, V4L2_PIX_FMT_YUYV, "YYUV 4:2:2"},
+	{MVX_FORMAT_YUV422_UYVY, 0, V4L2_PIX_FMT_UYVY, "UYVY 4:2:2"},
+	{MVX_FORMAT_YUV422_Y210, 0, V4L2_PIX_FMT_Y210, "YUV 4:2:2 Y210 (Microsoft format)"},
+
+	/* ARGB */
+	{MVX_FORMAT_ARGB_8888, 0, DRM_FORMAT_BGRA8888, /* Equal to V4L2_PIX_FMT_ARGB32. */
+	 "32-bit ARGB 8-8-8-8"},
+	{MVX_FORMAT_ARGB_8888, 0, V4L2_PIX_FMT_RGB32, "32-bit ARGB 8-8-8-8"},
+
+	/* ABGR */
+	{MVX_FORMAT_ABGR_8888, 0, DRM_FORMAT_RGBA8888, "32-bit ABGR-8-8-8-8"},
+
+	/* RGBA */
+	{MVX_FORMAT_RGBA_8888, 0, DRM_FORMAT_ABGR8888, "32-bit RGBA 8-8-8-8"},
+
+	/* BGRA (new and legacy format) */
+	{MVX_FORMAT_BGRA_8888, 0, DRM_FORMAT_ARGB8888, /* Equal to V4L2_PIX_FMT_ABGR32. */
+	 "32-bit BGRA 8-8-8-8"},
+	{MVX_FORMAT_BGRA_8888, 0, V4L2_PIX_FMT_BGR32, "32-bit BGRA 8-8-8-8"}};
+
+/*
+ * Search for format map that matches given pixel format.
+ */
+static struct mvx_format_map *mvx_find_format(uint32_t pixelformat)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mvx_fmts); i++)
+		if (mvx_fmts[i].pixelformat == pixelformat)
+			return &mvx_fmts[i];
+
+	return ERR_PTR(-EINVAL);
+}
+
+static int to_v4l2_format(struct v4l2_format *f, enum v4l2_buf_type type, struct v4l2_pix_format_mplane *pix,
+			  unsigned int *stride, unsigned int *size, bool interlaced)
+{
+	f->type = type;
+
+	switch (f->type) {
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE: {
+		struct v4l2_pix_format *p = &f->fmt.pix;
+		uint32_t i;
+
+		p->width = pix->width;
+		p->height = pix->height;
+		p->pixelformat = pix->pixelformat;
+		p->field = interlaced ? V4L2_FIELD_SEQ_TB : V4L2_FIELD_NONE;
+		p->colorspace = pix->colorspace;
+		p->flags = pix->flags;
+		p->ycbcr_enc = pix->ycbcr_enc;
+		p->quantization = pix->quantization;
+		p->xfer_func = pix->xfer_func;
+
+		p->sizeimage = 0;
+		p->bytesperline = stride[0];
+		for (i = 0; i < pix->num_planes; ++i)
+			p->sizeimage += size[i];
+
+		break;
+	}
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: {
+		struct v4l2_pix_format_mplane *p = &f->fmt.pix_mp;
+		int i;
+
+		memcpy(p, pix, sizeof(*p));
+		memset(p->reserved, 0, sizeof(p->reserved));
+		p->field = interlaced ? V4L2_FIELD_SEQ_TB : V4L2_FIELD_NONE;
+
+		for (i = 0; i < pix->num_planes; i++) {
+			p->plane_fmt[i].bytesperline = stride[i];
+			p->plane_fmt[i].sizeimage = size[i];
+			memset(p->plane_fmt[i].reserved, 0, sizeof(p->plane_fmt[i].reserved));
+		}
+
+		break;
+	}
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int from_v4l2_format(struct mvx_v4l2_session *vsession, struct v4l2_format *f,
+			    struct v4l2_pix_format_mplane *pix, enum mvx_format *format, unsigned int *stride,
+			    unsigned int *size, bool *interlaced)
+{
+	struct mvx_format_map *map;
+
+	switch (f->type) {
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE: {
+		struct v4l2_pix_format *p = &f->fmt.pix;
+
+		memset(pix, 0, sizeof(*pix));
+
+		pix->width = p->width;
+		pix->height = p->height;
+		pix->pixelformat = p->pixelformat;
+		pix->field = p->field;
+		pix->colorspace = p->colorspace;
+		pix->flags = p->flags;
+
+		if (p->priv != V4L2_PIX_FMT_PRIV_MAGIC) {
+			pix->ycbcr_enc = V4L2_COLORSPACE_DEFAULT;
+			pix->quantization = V4L2_QUANTIZATION_DEFAULT;
+			pix->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+		}
+
+		pix->num_planes = 1;
+		pix->plane_fmt[0].sizeimage = p->sizeimage;
+		pix->plane_fmt[0].bytesperline = p->bytesperline;
+
+		size[0] = p->sizeimage;
+		stride[0] = p->bytesperline;
+
+		break;
+	}
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: {
+		struct v4l2_pix_format_mplane *p = &f->fmt.pix_mp;
+		unsigned int i;
+
+		if (p->num_planes > MVX_BUFFER_NPLANES)
+			MVX_SESSION_WARN(&vsession->session,
+					 "Too many planes for format. format=0x%08x, num_planes=%u.", pix->pixelformat,
+					 p->num_planes);
+
+		memcpy(pix, p, sizeof(*pix));
+
+		for (i = 0; i < min_t(unsigned int, MVX_BUFFER_NPLANES, p->num_planes); i++) {
+			size[i] = p->plane_fmt[i].sizeimage;
+			stride[i] = p->plane_fmt[i].bytesperline;
+		}
+
+		break;
+	}
+	default:
+		return -EINVAL;
+	}
+
+	/* Adjust default field and color spaces. */
+
+	if (pix->field == V4L2_FIELD_SEQ_TB) {
+		*interlaced = true;
+	} else {
+		pix->field = V4L2_FIELD_NONE;
+		*interlaced = false;
+	}
+
+	if (pix->colorspace == V4L2_COLORSPACE_DEFAULT)
+		pix->colorspace = V4L2_COLORSPACE_REC709;
+
+	if (pix->ycbcr_enc == V4L2_YCBCR_ENC_DEFAULT)
+		pix->ycbcr_enc = V4L2_YCBCR_ENC_709;
+
+	if (pix->quantization == V4L2_QUANTIZATION_DEFAULT)
+		pix->quantization = V4L2_QUANTIZATION_FULL_RANGE;
+
+	if (pix->xfer_func == V4L2_XFER_FUNC_DEFAULT)
+		pix->xfer_func = V4L2_XFER_FUNC_709;
+
+	/* Find mapping between pixel format and mvx format. */
+	map = mvx_find_format(pix->pixelformat);
+	if (IS_ERR(map)) {
+		MVX_SESSION_WARN(&vsession->session, "Unsupported V4L2 pixel format. format=0x%08x.", pix->pixelformat);
+		return PTR_ERR(map);
+	}
+
+	*format = map->format;
+
+	return 0;
+}
+
+/**
+ * print_format() - Print V4L2 format.
+ * @session:	Pointer to MVX session.
+ * @f:		V4L2 format.
+ * @prefix:	Prefix string.
+ */
+static void print_format(struct mvx_session *session, struct v4l2_format *f, const char *prefix)
+{
+	if (V4L2_TYPE_IS_MULTIPLANAR(f->type) != false) {
+		struct v4l2_pix_format_mplane *p = &f->fmt.pix_mp;
+
+		MVX_SESSION_INFO(session, "v4l2: %s. type=%u, pixelformat=0x%08x, width=%u, height=%u, num_planes=%u.",
+				 prefix, f->type, p->pixelformat, p->width, p->height, p->num_planes);
+	} else {
+		struct v4l2_pix_format *p = &f->fmt.pix;
+
+		MVX_SESSION_INFO(session, "v4l2: %s. type=%u, pixelformat=0x%08x, width=%u, height=%u.", prefix,
+				 f->type, p->pixelformat, p->width, p->height);
+	}
+}
+
+/**
+ * queue_setup() - Initialize or verify queue parameters.
+ * @q:		Videobuf2 queue.
+ * @buf_cnt:	Requested/requered buffers count.
+ * @plane_cnt:	Required number of planes.
+ * @plane_size:	Required size of each plane.
+ * @alloc_devs:	Device to allocate memory from.
+ *
+ * This callback is used to query parameters of a queue from the driver.
+ * Vb2 sets buf_cnt to requested amount of buffers, but a driver is free to
+ * choose another value and return it. Vb2 will then call queue_setup() again
+ * to verify that the new value is accepted by a driver.
+ *
+ * Vb2 also uses plane_cnt parameter to signal if queue_setup() was called
+ * from create_bufs() of reqbufs().
+ *
+ * No locking is required in this function. The reason is that will be called
+ * from within vb2_reqbufs() or vb2_create_bufs() which are executed from our
+ * code with session mutex already taken.
+ *
+ * Return: 0 on success, else error code.
+ */
+#if KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE
+static int queue_setup(struct vb2_queue *q, unsigned int *buf_cnt, unsigned int *plane_cnt, unsigned int plane_size[],
+		       struct device *alloc_devs[])
+#else
+static int queue_setup(struct vb2_queue *q, const void *unused, unsigned int *buf_cnt, unsigned int *plane_cnt,
+		       unsigned int plane_size[], void *alloc_devs[])
+#endif
+{
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_session_port *port = vport->port;
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+	unsigned int i;
+
+	/*
+	 * If the output frame resolution is not known, then there is no need
+	 * to allocate buffers yet. But 1 buffer will be needed to carry
+	 * information about 'resolution change' and 'end of stream'.
+	 */
+	if (vport->dir == MVX_DIR_OUTPUT && mvx_is_frame(port->format) != false &&
+	    (port->width == 0 || port->height == 0))
+		*buf_cnt = 1;
+
+	memset(plane_size, 0, sizeof(plane_size[0]) * VB2_MAX_PLANES);
+	*plane_cnt = port->nplanes;
+	for (i = 0; i < port->nplanes; ++i) {
+		/*  Vb2 allocator does not handle well buffers of zero size. */
+		plane_size[i] = max_t(unsigned int, port->size[i], 1);
+		alloc_devs[i] = session->dev;
+	}
+
+	MVX_SESSION_VERBOSE(session,
+			    "queue_setup. vsession=%p, vport=%p, vb2_queue=%p, dir=%d, format=0x%x, width=%u, "
+			    "height=%u, nplanes=%u, plane_size=[%u, %u, %u]",
+			    vsession, vport, q, vport->dir, port->format, port->width, port->height, port->nplanes,
+			    plane_size[0], plane_size[1], plane_size[2]);
+
+	return 0;
+}
+
+/**
+ * buf_init() - Perform initilization for Vb2 buffer.
+ * @b:		Pointer to Vb2 buffer.
+ *
+ * Vb2 framework calls this function once for every allocated buffer.
+ * A driver fetches a list of memory pages and constructs MVX V4L2 buffers.
+ *
+ * No locking is required in this function. The reason is that will be called
+ * from within vb2_reqbufs() or vb2_create_bufs() which are executed from our
+ * code with session mutex already taken.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+static int buf_init(struct vb2_buffer *b)
+{
+	struct mvx_v4l2_buffer *vbuf = vb2_to_mvx_v4l2_buffer(b);
+
+	int ret;
+	unsigned int i;
+	struct sg_table *sgt[MVX_BUFFER_NPLANES] = {0};
+	struct vb2_queue *q = b->vb2_queue;
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+	unsigned int flags = vbuf->buf.flags;
+
+	MVX_SESSION_VERBOSE(session, "v4l2: Initialize buffer. vb=%p, type=%u, index=%u, num_planes=%u.", b, b->type,
+			    b->index, b->num_planes);
+
+	if (b->num_planes > MVX_BUFFER_NPLANES) {
+		MVX_SESSION_WARN(session, "Failed to initialize buffer. Too many planes. vb=%p, num_planes=%u.", b,
+				 b->num_planes);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < b->num_planes; ++i) {
+		sgt[i] = vb2_dma_sg_plane_desc(b, i);
+		if (sgt[i] == NULL) {
+			MVX_SESSION_WARN(session, "Cannot fetch SG descriptor. vb=%p, plane=%u.", b, i);
+			return -ENOMEM;
+		}
+	}
+
+	ret = mvx_v4l2_buffer_construct(vbuf, vsession, vport->dir, b->num_planes, sgt);
+
+	vbuf->buf.flags = flags;
+
+	return ret;
+}
+
+/**
+ * buf_cleanup() - Destroy data associated to Vb2 buffer.
+ * @b:		Pointer to Vb2 buffer.
+ *
+ * Vb2 framework calls this function while destroying a buffer.
+ */
+static void buf_cleanup(struct vb2_buffer *b)
+{
+	struct vb2_queue *q = b->vb2_queue;
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+	struct mvx_v4l2_buffer *vbuf = vb2_to_mvx_v4l2_buffer(b);
+
+	if (session->port[vport->dir].stream_on) {
+		MVX_SESSION_VERBOSE(session,
+				    "v4l2: Cleanup buffer in the coding process. Will remap mve va and pa address. "
+				    "dir=%d, type=%u, index=%u, vb=%p, vbuf=%p.",
+				    vport->dir, b->type, b->index, b, vbuf);
+	} else {
+		MVX_SESSION_VERBOSE(session, "v4l2: Cleanup buffer. type=%u, index=%u, vb=%p, vbuf=%p.", b->type,
+				    b->index, b, vbuf);
+	}
+
+	mvx_v4l2_buffer_destruct(vbuf);
+}
+
+/**
+ * start_streaming() - Start streaming for queue.
+ * @q:		Pointer to a queue.
+ * @cnt:	Amount of buffers already owned by a driver.
+ *
+ * Vb2 calls this function when it is ready to start streaming for a queue.
+ * Vb2 ensures that minimum required amount of buffers were enqueued to the
+ * driver before calling this function.
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+static int start_streaming(struct vb2_queue *q, unsigned int cnt)
+{
+	/*
+	 * Parameter cnt is not used so far.
+	 */
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+	int ret;
+
+	MVX_SESSION_VERBOSE(session, "v4l2: Start streaming. queue=%p, type=%u, cnt=%u.", q, q->type, cnt);
+
+	ret = mvx_session_streamon(&vsession->session, vport->dir);
+
+	/*
+	 * If attempt was not successful, we should return all owned buffers
+	 * to Vb2 with vb2_buffer_done() with state VB2_BUF_STATE_QUEUED.
+	 */
+	if (ret != 0 && atomic_read(&q->owned_by_drv_count) > 0) {
+		int i;
+
+		for (i = 0; i < vb2_get_num_buffers(q); ++i)
+			if (q->bufs[i]->state == VB2_BUF_STATE_ACTIVE)
+				vb2_buffer_done(q->bufs[i], VB2_BUF_STATE_QUEUED);
+
+		WARN_ON(atomic_read(&q->owned_by_drv_count));
+	}
+
+	return ret;
+}
+
+/**
+ * stop_streaming() - Stop streaming for a queue.
+ * @q:		Pointer to a queue.
+ *
+ * Vb2 calls this function when streaming should be terminated.
+ * The driver must ensure that no DMA transfers are ongoing and
+ * return all buffers to Vb2 with vb2_buffer_done().
+ */
+static void stop_streaming(struct vb2_queue *q)
+{
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+
+	MVX_SESSION_VERBOSE(session, "v4l2: Stop streaming. queue=%p, type=%u.", q, q->type);
+
+	mvx_session_streamoff(&vsession->session, vport->dir);
+
+	/*
+	 * We have to return all owned buffers to Vb2 before exiting from
+	 * this callback.
+	 *
+	 * Note: there must be no access to buffers after they are returned.
+	 */
+	if (atomic_read(&q->owned_by_drv_count) > 0) {
+		int i;
+
+		for (i = 0; i < vb2_get_num_buffers(q); ++i)
+			if (q->bufs[i]->state == VB2_BUF_STATE_ACTIVE)
+				vb2_buffer_done(q->bufs[i], VB2_BUF_STATE_ERROR);
+
+		WARN_ON(atomic_read(&q->owned_by_drv_count));
+	}
+}
+
+/**
+ * buf_queue() - Enqueue buffer to a driver.
+ * @b:		Pointer to Vb2 buffer structure.
+ *
+ * Vb2 calls this function to enqueue a buffer to a driver.
+ * A driver should later return a buffer to Vb2 with vb2_buffer_done().
+ *
+ * Return: 0 in case of success, error code otherwise.
+ */
+static void buf_queue(struct vb2_buffer *b)
+{
+	struct vb2_queue *q = b->vb2_queue;
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_session_port *port = vport->port;
+	enum mvx_direction dir = vport->dir;
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+	struct mvx_v4l2_buffer *vbuf = vb2_to_mvx_v4l2_buffer(b);
+
+	int ret;
+
+	MVX_SESSION_VERBOSE(session, "v4l2: Queue buffer. b=%p, type=%u, index=%u.", b, b->type, b->index);
+	vbuf->buf.format = vport->port->format;
+	ret = mvx_v4l2_buffer_set(vbuf, b);
+	if (ret != 0) {
+		goto failed;
+	}
+	ret = mvx_session_qbuf(&vsession->session, dir, &vbuf->buf);
+	if (ret != 0) {
+		goto failed;
+	}
+	return;
+
+failed:
+	if (vbuf->buf.flags & MVX_BUFFER_FRAME_NEED_REALLOC) {
+		vbuf->vb2_v4l2_buffer.flags |= V4L2_BUF_FLAG_MVX_BUFFER_NEED_REALLOC;
+		port->isreallocting = true;
+		vb2_buffer_done(b, VB2_BUF_STATE_DONE);
+		return;
+	}
+	vb2_buffer_done(b, VB2_BUF_STATE_ERROR);
+}
+
+/**
+ * buf_finish() - Finish buffer before it is returned to user space.
+ * @vb:		Pointer to Vb2 buffer structure.
+ */
+static void buf_finish(struct vb2_buffer *vb)
+{
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(vb->vb2_queue);
+	struct mvx_v4l2_buffer *vbuf = vb2_to_mvx_v4l2_buffer(vb);
+
+	vport->crop.left = vbuf->buf.crop_left;
+	vport->crop.top = vbuf->buf.crop_top;
+}
+
+/**
+ * wait_prepare() - Prepare driver for waiting
+ * @q:		Pointer to Vb2 queue.
+ *
+ * Vb2 calls this function when it is about to wait for more buffers to
+ * be received. A driver should release any locks taken while calling Vb2
+ * functions.
+ * This is required to avoid a deadlock.
+ *
+ * This is unused for now and will be called from Vb2.
+ */
+static void wait_prepare(struct vb2_queue *q)
+{
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+
+	MVX_SESSION_VERBOSE(session, "v4l2: Wait prepare. queue=%p.", q);
+
+	mutex_unlock(&vsession->mutex);
+}
+
+/**
+ * wait_finish() - Wake up after sleep.
+ * @q:		Pointer to Vb2 queue.
+ *
+ * Require mutexes release before.
+ *
+ * This is unused for now and will be called from Vb2.
+ */
+static void wait_finish(struct vb2_queue *q)
+{
+	struct mvx_v4l2_port *vport = vb2_get_drv_priv(q);
+	struct mvx_v4l2_session *vsession = vport->vsession;
+	struct mvx_session *session = &vsession->session;
+	int ignore;
+
+	MVX_SESSION_VERBOSE(session, "v4l2: Wait finish. queue=%p.", q);
+
+	/*
+	 * mutex_lock_interruptible is declared with attribute
+	 * warn_unused_result, but we have no way to return a status
+	 * from wait_finish().
+	 */
+	ignore = mutex_lock_interruptible(&vsession->mutex);
+}
+
+/**
+ * mvx_vb2_ops - Callbacks for Vb2 framework
+ * Not all possible callbacks are implemented as some of them are optional.
+ */
+const struct vb2_ops mvx_vb2_ops = {
+	.queue_setup     = queue_setup,
+	.buf_init        = buf_init,
+	.buf_finish      = buf_finish,
+	.buf_cleanup     = buf_cleanup,
+	.start_streaming = start_streaming,
+	.stop_streaming  = stop_streaming,
+	.buf_queue       = buf_queue,
+	.wait_prepare    = wait_prepare,
+	.wait_finish     = wait_finish
+};
+
+/**
+ * setup_vb2_queue() - Initialize vb2_queue before it can be used by Vb2.
+ */
+static int setup_vb2_queue(struct mvx_v4l2_port *vport)
+{
+	struct vb2_queue *q = &vport->vb2_queue;
+#if KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE
+	struct device *dev = vport->vsession->ext->dev;
+#endif
+	int ret;
+
+	q->drv_priv = vport;
+	q->type = vport->type;
+	q->io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
+#if KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE
+	q->dev = dev;
+#endif
+	q->ops = &mvx_vb2_ops;
+	q->mem_ops = &vb2_dma_sg_memops;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+	q->allow_zero_bytesused = true;
+
+	/* Let Vb2 handle mvx_v4l2_buffer allocations. */
+	q->buf_struct_size = sizeof(struct mvx_v4l2_buffer);
+
+	ret = vb2_queue_init(q);
+
+	return ret;
+}
+
+/****************************************************************************
+ * Exported functions and variables
+ ****************************************************************************/
+
+int mvx_v4l2_vidioc_querycap(struct file *file, void *fh, struct v4l2_capability *cap)
+{
+	struct mvx_v4l2_session *session = file_to_session(file);
+
+	MVX_SESSION_INFO(&session->session, "v4l2: Query capabilities.");
+
+#if (KERNEL_VERSION(6, 7, 0) <= LINUX_VERSION_CODE)
+	strncpy(cap->driver, "mvx", sizeof(cap->driver));
+	strncpy(cap->card, "Linlon Video device", sizeof(cap->card));
+	strncpy(cap->bus_info, "platform:mvx", sizeof(cap->bus_info));
+#else
+	strlcpy(cap->driver, "mvx", sizeof(cap->driver));
+	strlcpy(cap->card, "Linlon Video device", sizeof(cap->card));
+	strlcpy(cap->bus_info, "platform:mvx", sizeof(cap->bus_info));
+#endif
+
+	cap->capabilities = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_VIDEO_M2M | V4L2_CAP_VIDEO_M2M_MPLANE |
+			    V4L2_CAP_EXT_PIX_FORMAT | V4L2_CAP_STREAMING;
+	cap->device_caps = cap->capabilities & ~V4L2_CAP_DEVICE_CAPS;
+
+	return 0;
+}
+
+/*
+ * Loop over the mvx_fmts searching for pixelformat at offset f->index.
+ *
+ * Formats that are not present in the 'formats' bitmask will be skipped.
+ * Which pixelformat that is mapped to which index will consequently depend
+ * on which mvx_formats that are enabled.
+ */
+static int mvx_v4l2_vidioc_enum_fmt_vid(struct mvx_v4l2_session *session, struct v4l2_fmtdesc *f,
+					enum mvx_direction dir)
+{
+	uint64_t formats;
+	int index;
+	int i;
+
+	mvx_session_get_formats(&session->session, dir, &formats);
+
+	for (i = 0, index = 0; i < ARRAY_SIZE(mvx_fmts); i++)
+		if (mvx_test_bit(mvx_fmts[i].format, &formats)) {
+			if (f->index == index) {
+				f->flags = mvx_fmts[i].flags;
+				f->pixelformat = mvx_fmts[i].pixelformat;
+#if (KERNEL_VERSION(6, 7, 0) <= LINUX_VERSION_CODE)
+				strncpy(f->description, mvx_fmts[i].description, sizeof(f->description));
+#else
+				strlcpy(f->description, mvx_fmts[i].description, sizeof(f->description));
+#endif
+				break;
+			}
+
+			index++;
+		}
+
+	if (i >= ARRAY_SIZE(mvx_fmts))
+		return -EINVAL;
+
+	return 0;
+}
+
+int mvx_v4l2_vidioc_enum_fmt_vid_cap(struct file *file, void *fh, struct v4l2_fmtdesc *f)
+{
+	struct mvx_v4l2_session *session = file_to_session(file);
+	int ret;
+
+	ret = mvx_v4l2_vidioc_enum_fmt_vid(session, f, MVX_DIR_OUTPUT);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_enum_fmt_vid_out(struct file *file, void *fh, struct v4l2_fmtdesc *f)
+{
+	struct mvx_v4l2_session *session = file_to_session(file);
+	int ret;
+
+	ret = mvx_v4l2_vidioc_enum_fmt_vid(session, f, MVX_DIR_INPUT);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_enum_framesizes(struct file *file, void *fh, struct v4l2_frmsizeenum *fsize)
+{
+	struct mvx_format_map *format;
+
+	/* Verify that format is supported. */
+	format = mvx_find_format(fsize->pixel_format);
+	if (IS_ERR(format))
+		return PTR_ERR(format);
+
+	/* For stepwise/continuous frame size the index must be 0. */
+	if (fsize->index != 0)
+		return -EINVAL;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
+	fsize->stepwise.min_width = 2;
+	fsize->stepwise.max_width = 8192;
+	fsize->stepwise.step_width = 2;
+	fsize->stepwise.min_height = 2;
+	fsize->stepwise.max_height = 8192;
+	fsize->stepwise.step_height = 2;
+
+	return 0;
+}
+
+static int mvx_v4l2_vidioc_g_fmt_vid(struct file *file, struct v4l2_format *f, enum mvx_direction dir)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	struct mvx_session_port *port = &vsession->session.port[dir];
+	int ret;
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	to_v4l2_format(f, f->type, &vport->pix_mp, port->stride, port->size, port->interlaced);
+
+	mutex_unlock(&vsession->mutex);
+
+	print_format(&vsession->session, f, "Get format");
+
+	return 0;
+}
+
+int mvx_v4l2_vidioc_g_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f)
+{
+	return mvx_v4l2_vidioc_g_fmt_vid(file, f, MVX_DIR_OUTPUT);
+}
+
+int mvx_v4l2_vidioc_g_fmt_vid_out(struct file *file, void *fh, struct v4l2_format *f)
+{
+	return mvx_v4l2_vidioc_g_fmt_vid(file, f, MVX_DIR_INPUT);
+}
+
+static int mvx_v4l2_vidioc_s_fmt_vid(struct file *file, struct v4l2_format *f, enum mvx_direction dir)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	struct v4l2_pix_format_mplane pix_mp;
+	enum mvx_format format;
+	unsigned int stride[MVX_BUFFER_NPLANES];
+	unsigned int size[MVX_BUFFER_NPLANES];
+	bool interlaced = false;
+	int ret;
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	if (vport->q_set != false && vb2_is_busy(&vport->vb2_queue) != false) {
+		MVX_SESSION_WARN(&vsession->session,
+				 "Can't set format when there there buffers allocated to the port.");
+		ret = -EBUSY;
+		goto unlock_mutex;
+	}
+
+	/* Convert V4L2 format to V4L2 multi planar pixel format. */
+	ret = from_v4l2_format(vsession, f, &pix_mp, &format, stride, size, &interlaced);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	/* Validate and adjust settings. */
+	ret = mvx_session_set_format(&vsession->session, dir, format, &pix_mp.width, &pix_mp.height, &pix_mp.num_planes,
+				     stride, size, &interlaced);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	/* Convert V4L2 multi planar pixel format to format. */
+	ret = to_v4l2_format(f, f->type, &pix_mp, stride, size, interlaced);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	vport->type = f->type;
+	vport->pix_mp = pix_mp;
+
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+	print_format(&vsession->session, f, "Set format");
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_s_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f)
+{
+	return mvx_v4l2_vidioc_s_fmt_vid(file, f, MVX_DIR_OUTPUT);
+}
+
+int mvx_v4l2_vidioc_s_fmt_vid_out(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	struct v4l2_pix_format_mplane *in = &vsession->port[MVX_DIR_INPUT].pix_mp;
+	struct v4l2_pix_format_mplane *out = &vsession->port[MVX_DIR_OUTPUT].pix_mp;
+	int ret;
+
+	ret = mvx_v4l2_vidioc_s_fmt_vid(file, f, MVX_DIR_INPUT);
+	if (ret != 0)
+		return ret;
+
+	/* Copy input formats to output port. */
+	out->colorspace = in->colorspace;
+	out->ycbcr_enc = in->ycbcr_enc;
+	out->quantization = in->quantization;
+	out->xfer_func = in->xfer_func;
+
+	return 0;
+}
+
+static int mvx_v4l2_vidioc_try_fmt_vid(struct file *file, struct v4l2_format *f, enum mvx_direction dir)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	struct v4l2_pix_format_mplane pix;
+	enum mvx_format format;
+	unsigned int stride[MVX_BUFFER_NPLANES];
+	unsigned int size[MVX_BUFFER_NPLANES];
+	bool interlaced = false;
+	int ret;
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	ret = from_v4l2_format(vsession, f, &pix, &format, stride, size, &interlaced);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	ret = mvx_session_try_format(&vsession->session, dir, format, &pix.width, &pix.height, &pix.num_planes, stride,
+				     size, &interlaced);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	ret = to_v4l2_format(f, f->type, &pix, stride, size, interlaced);
+	if (ret != 0)
+		goto unlock_mutex;
+
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+	print_format(&vsession->session, f, "Try format");
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f)
+{
+	return mvx_v4l2_vidioc_try_fmt_vid(file, f, MVX_DIR_OUTPUT);
+}
+
+int mvx_v4l2_vidioc_try_fmt_vid_out(struct file *file, void *fh, struct v4l2_format *f)
+{
+	return mvx_v4l2_vidioc_try_fmt_vid(file, f, MVX_DIR_INPUT);
+}
+int mvx_v4l2_vidioc_g_crop(struct file *file, void *fh, struct v4l2_crop *a)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(a->type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	struct mvx_session_port *port = &vsession->session.port[dir];
+
+	mutex_lock(&vsession->mutex);
+
+	a->c.left = vport->crop.left;
+	a->c.top = vport->crop.top;
+	a->c.width = port->width - vport->crop.left;
+	a->c.height = port->height - vport->crop.top;
+
+	mutex_unlock(&vsession->mutex);
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Get crop. dir=%u, crop={left=%u, top=%u, width=%u, height=%u.", dir,
+			 a->c.left, a->c.top, a->c.width, a->c.height);
+
+	return 0;
+}
+int mvx_v4l2_vidioc_g_selection(struct file *file, void *fh, struct v4l2_selection *s)
+{
+	struct v4l2_crop crop = {.type = s->type};
+	int ret;
+	ret = mvx_v4l2_vidioc_g_crop(file, fh, &crop);
+	if (ret == 0) {
+		s->r = crop.c;
+	}
+	return ret;
+}
+int mvx_v4l2_vidioc_s_selection(struct file *file, void *fh, struct v4l2_selection *s)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	int ret = 0;
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+	ret = mvx_session_set_crop_left(&vsession->session, s->r.left);
+	ret = mvx_session_set_crop_top(&vsession->session, s->r.top);
+	mutex_unlock(&vsession->mutex);
+	return ret;
+}
+
+int mvx_v4l2_vidioc_s_parm(struct file *file, void *fh, struct v4l2_streamparm *a)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	int ret = 0;
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+	if (V4L2_TYPE_IS_OUTPUT(a->type)) {
+		int64_t framerate = ((int64_t)a->parm.output.timeperframe.denominator << 16) /
+				    a->parm.output.timeperframe.numerator;
+		ret = mvx_session_set_frame_rate(&vsession->session, framerate);
+	}
+	mutex_unlock(&vsession->mutex);
+	return ret;
+}
+int mvx_v4l2_vidioc_g_parm(struct file *file, void *fh, struct v4l2_streamparm *a)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	int ret = 0;
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+	if (V4L2_TYPE_IS_OUTPUT(a->type)) {
+		a->parm.output.timeperframe.numerator = 1 << 16;
+		a->parm.output.timeperframe.denominator = (int32_t)vsession->session.frame_rate;
+	}
+	mutex_unlock(&vsession->mutex);
+	return ret;
+}
+
+int mvx_v4l2_vidioc_streamon(struct file *file, void *priv, enum v4l2_buf_type type)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Stream on. dir=%u.", dir);
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	ret = vb2_streamon(&vsession->port[dir].vb2_queue, type);
+	if (ret != 0)
+		MVX_SESSION_WARN(&vsession->session, "v4l2: Failed to stream on. dir=%u.", dir);
+
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_streamoff(struct file *file, void *priv, enum v4l2_buf_type type)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Stream off. dir=%u.", dir);
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	ret = vb2_streamoff(&vsession->port[dir].vb2_queue, type);
+	if (ret != 0)
+		MVX_SESSION_WARN(&vsession->session, "v4l2: Failed to stream off. dir=%u.", dir);
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Stream off exit. dir=%u, ret=%d.", dir, ret);
+
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_encoder_cmd(struct file *file, void *priv, struct v4l2_encoder_cmd *cmd)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: encoder cmd: %u.", cmd->cmd);
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	switch (cmd->cmd) {
+	case V4L2_ENC_CMD_STOP:
+		ret = mvx_session_send_eos(&vsession->session);
+		break;
+	case V4L2_ENC_CMD_START:
+		/*reset flag for v4l2 core, so that buffers can be queued normal.*/
+		vsession->port[1].vb2_queue.last_buffer_dequeued = false;
+		ret = 0;
+		break;
+	default:
+		MVX_SESSION_WARN(&vsession->session, "Unsupported command. cmd: %u.", cmd->cmd);
+		ret = -EINVAL;
+	}
+
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_try_encoder_cmd(struct file *file, void *priv, struct v4l2_encoder_cmd *cmd)
+{
+	switch (cmd->cmd) {
+	case V4L2_ENC_CMD_STOP:
+	case V4L2_ENC_CMD_START:
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+int mvx_v4l2_vidioc_decoder_cmd(struct file *file, void *priv, struct v4l2_decoder_cmd *cmd)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: decoder cmd: %u.", cmd->cmd);
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	switch (cmd->cmd) {
+	case V4L2_DEC_CMD_STOP:
+		ret = mvx_session_send_eos(&vsession->session);
+		break;
+	case V4L2_DEC_CMD_START:
+		/*reset flag for v4l2 core, so that buffers can be queued normal.*/
+		vsession->port[1].vb2_queue.last_buffer_dequeued = false;
+		ret = 0;
+		break;
+	default:
+		MVX_SESSION_WARN(&vsession->session, "Unsupported command. cmd: %u.", cmd->cmd);
+		ret = -EINVAL;
+	}
+
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_try_decoder_cmd(struct file *file, void *priv, struct v4l2_decoder_cmd *cmd)
+{
+	switch (cmd->cmd) {
+	case V4L2_DEC_CMD_STOP:
+	case V4L2_DEC_CMD_START:
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+int mvx_v4l2_vidioc_reqbufs(struct file *file, void *fh, struct v4l2_requestbuffers *b)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(b->type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Request buffers. dir=%d, type=%u, memory=%u, count=%u.", dir,
+			 b->type, b->memory, b->count);
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	if (b->count == 0) {
+		if (vport->q_set != false) {
+			vb2_queue_release(&vport->vb2_queue);
+			vport->q_set = false;
+		}
+	} else {
+		if (vport->q_set == false) {
+			ret = setup_vb2_queue(vport);
+			if (ret != 0)
+				goto unlock_mutex;
+
+			vport->q_set = true;
+		}
+
+		ret = vb2_reqbufs(&vport->vb2_queue, b);
+	}
+	vport->port->buffer_allocated = b->count;
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_create_bufs(struct file *file, void *fh, struct v4l2_create_buffers *b)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(b->format.type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Create buffers. dir=%d, type=%u, memory=%u, count=%u.", dir,
+			 b->format.type, b->memory, b->count);
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	if (vport->q_set == false)
+		ret = setup_vb2_queue(vport);
+
+	if (ret != 0)
+		goto unlock_mutex;
+
+	vport->q_set = true;
+#ifndef MODULE
+	ret = vb2_create_bufs(&vport->vb2_queue, b);
+#endif
+	vport->port->buffer_allocated = b->count;
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *b)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(b->type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Query buffer. dir=%d, type=%u, memory=%u, index=%u.", dir, b->type,
+			 b->memory, b->index);
+
+	ret = mutex_lock_interruptible(&vsession->mutex);
+	if (ret != 0)
+		return ret;
+
+	ret = vb2_querybuf(&vport->vb2_queue, b);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	/*
+	 * When user space wants to mmap() a buffer, we have to be able to
+	 * determine a direction of coresponding port. To make it easier we
+	 * adjust mem_offset on output port by DST_QUEUE_OFF_BASE for all
+	 * buffers.
+	 */
+	if (dir == MVX_DIR_OUTPUT) {
+		if (V4L2_TYPE_IS_MULTIPLANAR(b->type)) {
+			int i;
+
+			for (i = 0; i < b->length; ++i)
+				b->m.planes[i].m.mem_offset += DST_QUEUE_OFF_BASE;
+		} else {
+			b->m.offset += DST_QUEUE_OFF_BASE;
+		}
+	}
+
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_qbuf(struct file *file, void *fh, struct v4l2_buffer *b)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(b->type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	struct mvx_v4l2_buffer *vbuf;
+	struct mvx_buffer *buf;
+	struct vb2_buffer *vb;
+	struct v4l2_core_buffer_header_general *v4l2_general;
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Queue buffer. dir=%d, type=%u, index=%u, flags=0x%x.", dir, b->type,
+			 b->index, b->flags);
+
+	mutex_lock(&vsession->mutex);
+
+	if ((b->flags & V4L2_BUF_FLAG_MVX_BUFFER_EPR) == V4L2_BUF_FLAG_MVX_BUFFER_EPR) {
+		vb = vport->vb2_queue.bufs[b->index];
+		vbuf = vb2_to_mvx_v4l2_buffer(vb);
+		buf = &vbuf->buf;
+		v4l2_general = (struct v4l2_core_buffer_header_general *)&b->m.planes[0].reserved[0];
+		buf->general.header.buffer_size = v4l2_general->buffer_size;
+		buf->general.header.config_size = v4l2_general->config_size;
+		buf->general.header.type = v4l2_general->type;
+
+		memcpy(&buf->general.config.block_configs, &v4l2_general->config, sizeof(v4l2_general->config));
+		MVX_SESSION_INFO(&vsession->session,
+				 "v4l2: Queue buffer. type:%d, config size:%d, buffer size:%d, cfg_type:0x%x, cols and "
+				 "rows:%d, %d",
+				 v4l2_general->type, v4l2_general->config_size, v4l2_general->buffer_size,
+				 v4l2_general->config.blk_cfg_type,
+				 v4l2_general->config.blk_cfgs.rows_uncomp.n_cols_minus1,
+				 v4l2_general->config.blk_cfgs.rows_uncomp.n_rows_minus1);
+	}
+
+	if (dir == MVX_DIR_INPUT && V4L2_TYPE_IS_MULTIPLANAR(b->type)) {
+		vb = vport->vb2_queue.bufs[b->index];
+		vbuf = vb2_to_mvx_v4l2_buffer(vb);
+		buf = &vbuf->buf;
+
+		buf->flags = 0;
+		if ((b->reserved2 & V4L2_BUF_FRAME_FLAG_ROTATION_90) == V4L2_BUF_FRAME_FLAG_ROTATION_90) {
+			buf->flags |= MVX_BUFFER_FRAME_FLAG_ROTATION_90;
+		}
+		if ((b->reserved2 & V4L2_BUF_FRAME_FLAG_ROTATION_180) == V4L2_BUF_FRAME_FLAG_ROTATION_180) {
+			buf->flags |= MVX_BUFFER_FRAME_FLAG_ROTATION_180;
+		}
+		if ((b->reserved2 & V4L2_BUF_FRAME_FLAG_ROTATION_270) == V4L2_BUF_FRAME_FLAG_ROTATION_270) {
+			buf->flags |= MVX_BUFFER_FRAME_FLAG_ROTATION_270;
+		}
+	}
+	ret = vb2_qbuf(&vport->vb2_queue, NULL, b);
+	mutex_unlock(&vsession->mutex);
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *b)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	struct mvx_ext_if *ctx = vsession->ext;
+	enum mvx_direction dir = V4L2_TYPE_IS_OUTPUT(b->type) ? MVX_DIR_INPUT : MVX_DIR_OUTPUT;
+	struct mvx_v4l2_port *vport = &vsession->port[dir];
+	struct vb2_buffer *vb;
+	struct mvx_v4l2_buffer *vbuf;
+	struct mvx_buffer *buf;
+	int ret;
+
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Dequeue buffer. dir=%d, type=%u.", dir, b->type);
+
+	mutex_lock(&vsession->mutex);
+
+	ret = vb2_dqbuf(&vport->vb2_queue, b, file->f_flags & O_NONBLOCK);
+	if (ret != 0)
+		goto unlock_mutex;
+
+	if ((dir == MVX_DIR_OUTPUT) && (b->flags & V4L2_BUF_FLAG_LAST)) {
+		const struct v4l2_event event = {.type = V4L2_EVENT_EOS};
+		v4l2_event_queue(&ctx->vdev, &event);
+	}
+
+	/*
+	 * For single planar buffers there is no data offset. Instead the
+	 * offset is added to the memory pointer and subtraced from the
+	 * bytesused.
+	 */
+	vb = vport->vb2_queue.bufs[b->index];
+	if (V4L2_TYPE_IS_MULTIPLANAR(vb->type) == false) {
+		b->bytesused -= vb->planes[0].data_offset;
+
+		switch (vb->type) {
+		case V4L2_MEMORY_MMAP:
+			b->m.offset += vb->planes[0].data_offset;
+			break;
+		case V4L2_MEMORY_USERPTR:
+			b->m.userptr += vb->planes[0].data_offset;
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (vsession->port[MVX_DIR_INPUT].port->format <= MVX_FORMAT_BITSTREAM_LAST && dir == MVX_DIR_OUTPUT &&
+	    V4L2_TYPE_IS_MULTIPLANAR(b->type)) {
+		vbuf = vb2_to_mvx_v4l2_buffer(vb);
+		buf = &vbuf->buf;
+		b->reserved2 = (buf->width << 16) | (buf->height);
+	}
+
+unlock_mutex:
+	mutex_unlock(&vsession->mutex);
+
+#ifndef MODULE
+	MVX_SESSION_INFO(&vsession->session,
+			 "v4l2: Dequeued buffer. dir=%d, type=%u, index=%u, flags=0x%x, nevents=%u, fh=%p.", dir,
+			 b->type, b->index, b->flags, v4l2_event_pending(&vsession->fh), fh);
+#else
+	MVX_SESSION_INFO(&vsession->session, "v4l2: Dequeued buffer. dir=%d, type=%u, index=%u, flags=0x%x, fh=%p.",
+			 dir, b->type, b->index, b->flags, fh);
+#endif
+
+	return ret;
+}
+
+int mvx_v4l2_vidioc_subscribe_event(struct v4l2_fh *fh, const struct v4l2_event_subscription *sub)
+{
+	struct mvx_v4l2_session *session = v4l2_fh_to_session(fh);
+
+	MVX_SESSION_INFO(&session->session, "v4l2: Subscribe event. fh=%p, type=%u.", fh, sub->type);
+
+	switch (sub->type) {
+	case V4L2_EVENT_CTRL:
+		return v4l2_ctrl_subscribe_event(fh, sub);
+	case V4L2_EVENT_EOS:
+	case V4L2_EVENT_SOURCE_CHANGE:
+	case V4L2_EVENT_MVX_COLOR_DESC:
+		return v4l2_event_subscribe(fh, sub, 2, NULL);
+	default:
+		MVX_SESSION_WARN(&session->session, "Can't register for unsupported event. type=%u.", sub->type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+long mvx_v4l2_vidioc_default(struct file *file, void *fh, bool valid_prio, unsigned int cmd, void *arg)
+{
+	struct mvx_v4l2_session *vsession = file_to_session(file);
+	int ret;
+	MVX_SESSION_INFO(&vsession->session, "Custom ioctl. cmd=0x%x, arg=0x%p.", cmd, arg);
+
+	switch (cmd) {
+	case VIDIOC_G_MVX_COLORDESC: {
+		ret = mvx_v4l2_session_get_color_desc(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_ROI_REGIONS: {
+		ret = mvx_v4l2_session_set_roi_regions(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_QP_EPR: {
+		ret = mvx_v4l2_session_set_qp_epr(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_COLORDESC: {
+		ret = mvx_v4l2_session_set_color_desc(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_SEI_USERDATA: {
+		ret = mvx_v4l2_session_set_sei_userdata(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_RATE_CONTROL: {
+		ret = mvx_v4l2_session_set_rate_control(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_DSL_FRAME: {
+		ret = mvx_v4l2_session_set_dsl_frame(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_DSL_RATIO: {
+		ret = mvx_v4l2_session_set_dsl_ratio(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_LONG_TERM_REF: {
+		ret = mvx_v4l2_session_set_long_term_ref(vsession, arg);
+		break;
+	}
+	case VIDIOC_S_MVX_DSL_MODE: {
+		ret = mvx_v4l2_session_set_dsl_mode(vsession, arg);
+		break;
+	}
+	default:
+		MVX_LOG_PRINT(&mvx_log_if, MVX_LOG_WARNING, "Unsupported IOCTL. cmd=0x%x", cmd);
+		return -ENOTTY;
+	}
+
+	return ret;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_vidioc.h b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_vidioc.h
new file mode 100644
index 000000000000..357acb838042
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/if/v4l2/mvx_v4l2_vidioc.h
@@ -0,0 +1,94 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_V4L2_VIDIOC_H_
+#define _MVX_V4L2_VIDIOC_H_
+
+/****************************************************************************
+ * Exported functions
+ *
+ * Callbacks for struct v4l2_ioctl_ops.
+ *
+ * Prototypes declared bellow implement certain v4l2 ioctls and used to
+ * initialize members of v4l2_ioctl_ops structure.
+ ****************************************************************************/
+
+int mvx_v4l2_vidioc_querycap(struct file *file, void *fh, struct v4l2_capability *cap);
+
+int mvx_v4l2_vidioc_enum_fmt_vid_cap(struct file *file, void *fh, struct v4l2_fmtdesc *f);
+
+int mvx_v4l2_vidioc_enum_fmt_vid_out(struct file *file, void *fh, struct v4l2_fmtdesc *f);
+
+int mvx_v4l2_vidioc_enum_framesizes(struct file *file, void *fh, struct v4l2_frmsizeenum *fsize);
+
+int mvx_v4l2_vidioc_g_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f);
+
+int mvx_v4l2_vidioc_g_fmt_vid_out(struct file *file, void *fh, struct v4l2_format *f);
+
+int mvx_v4l2_vidioc_s_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f);
+
+int mvx_v4l2_vidioc_s_fmt_vid_out(struct file *file, void *fh, struct v4l2_format *f);
+
+int mvx_v4l2_vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f);
+
+int mvx_v4l2_vidioc_try_fmt_vid_out(struct file *file, void *fh, struct v4l2_format *f);
+
+int mvx_v4l2_vidioc_g_crop(struct file *file, void *fh, struct v4l2_crop *a);
+int mvx_v4l2_vidioc_g_selection(struct file *file, void *fh, struct v4l2_selection *s);
+int mvx_v4l2_vidioc_streamon(struct file *file, void *priv, enum v4l2_buf_type type);
+
+int mvx_v4l2_vidioc_streamoff(struct file *file, void *priv, enum v4l2_buf_type type);
+
+int mvx_v4l2_vidioc_encoder_cmd(struct file *file, void *priv, struct v4l2_encoder_cmd *cmd);
+
+int mvx_v4l2_vidioc_try_encoder_cmd(struct file *file, void *priv, struct v4l2_encoder_cmd *cmd);
+
+int mvx_v4l2_vidioc_decoder_cmd(struct file *file, void *priv, struct v4l2_decoder_cmd *cmd);
+
+int mvx_v4l2_vidioc_try_decoder_cmd(struct file *file, void *priv, struct v4l2_decoder_cmd *cmd);
+
+int mvx_v4l2_vidioc_reqbufs(struct file *file, void *fh, struct v4l2_requestbuffers *b);
+
+int mvx_v4l2_vidioc_create_bufs(struct file *file, void *fh, struct v4l2_create_buffers *b);
+
+int mvx_v4l2_vidioc_querybuf(struct file *file, void *fh, struct v4l2_buffer *b);
+
+int mvx_v4l2_vidioc_qbuf(struct file *file, void *fh, struct v4l2_buffer *b);
+
+int mvx_v4l2_vidioc_dqbuf(struct file *file, void *fh, struct v4l2_buffer *b);
+
+int mvx_v4l2_vidioc_subscribe_event(struct v4l2_fh *fh, const struct v4l2_event_subscription *sub);
+
+long mvx_v4l2_vidioc_default(struct file *file, void *fh, bool valid_prio, unsigned int cmd, void *arg);
+int mvx_v4l2_vidioc_s_selection(struct file *file, void *fh, struct v4l2_selection *s);
+int mvx_v4l2_vidioc_g_parm(struct file *file, void *fh, struct v4l2_streamparm *a);
+int mvx_v4l2_vidioc_s_parm(struct file *file, void *fh, struct v4l2_streamparm *a);
+#endif /* _MVX_V4L2_VIDIOC_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_driver.c b/drivers/media/platform/spacemit/vpu_k3/mvx_driver.c
new file mode 100644
index 000000000000..7c7ed8c6183c
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_driver.c
@@ -0,0 +1,70 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include "mvx_if.h"
+#include "mvx_dev.h"
+#include "mvx_log_group.h"
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("ARMChina");
+MODULE_DESCRIPTION("Tiube VPU Driver.");
+
+static int __init mvx_init(void)
+{
+	int ret;
+
+	ret = mvx_log_group_init("amvx");
+	if (ret != 0) {
+		pr_err("Failed to create MVx driver logging.\n");
+		return ret;
+	}
+
+	ret = mvx_dev_init();
+	if (ret != 0) {
+		pr_err("Failed to register MVx dev driver.\n");
+		mvx_log_group_deinit();
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit mvx_exit(void)
+{
+	mvx_dev_exit();
+	mvx_log_group_deinit();
+}
+
+module_init(mvx_init);
+module_exit(mvx_exit);
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_dvfs.c b/drivers/media/platform/spacemit/vpu_k3/mvx_dvfs.c
new file mode 100644
index 000000000000..8179d56e070c
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_dvfs.c
@@ -0,0 +1,1243 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+#include <linux/list.h>
+#include <linux/device.h>
+#include <linux/pm_qos.h>
+#include <linux/kthread.h>
+#include <linux/delay.h>
+#include <linux/stat.h>
+#include <linux/sysfs.h>
+#include <linux/thermal.h>
+#include <linux/clk.h>
+
+#include "mvx_dvfs.h"
+#include "mvx_log_group.h"
+#include "mvx_session.h"
+
+#define DVFS_INTERNAL_DEBUG
+/** Default value for an interval between frequency updates in milliseconds.
+ * It could be overwritten by user in debug build when sysfs is enabled.
+ */
+#define POLL_INTERVAL_MS 100
+
+/* Adjustment step in percents of maximum supported frequency */
+#define UP_STEP_PERCENT 25
+#define DOWN_STEP_PERCENT 13
+#define DVFS_FREQ_MAX 819200000
+
+#if defined(CONFIG_SYSFS) && defined(DVFS_INTERNAL_DEBUG)
+#define DVFS_DEBUG_MODE 1
+#else
+#define DVFS_DEBUG_MODE 0
+#endif
+
+#define VPU_DDR_QOS_ENABLE 0
+#define VPU_DDR_QOS_MIN 50000	/* 50MB/s */
+#define VPU_DDR_QOS_MAX 4096000 /* 4GB/s */
+#define VPU_DDR_QOS_PREDEFINED_FPS 30
+
+#define NELEMS(a) (sizeof(a) / sizeof((a)[0]))
+
+extern int session_wait_pending_timeout;
+extern int session_watchdog_timeout;
+
+/**
+ * Structure used by DVFS module to keep track of session usage and to
+ * take decisions about power management.
+ *
+ * Currently the only parameter taken into consideration is an amount of
+ * output buffers enqueued in FW for each session. DVFS tries to keep this
+ * parameter equal to 1 for all sessions. If some session has more than one
+ * enqueued buffer, it means that a client is waiting for more than one
+ * frame and the clock frequency should be increased. If some session has
+ * no buffers enqueued, it means that the client is not waiting for
+ * anything and the clock frequency could be decreased. Priority is given
+ * to frequency increasing (when more than one session is registered).
+ */
+struct session {
+	mvx_session_id session_id;
+	struct list_head list;
+	bool is_encoder;
+	uint32_t ddr_qos_read;
+	uint32_t ddr_qos_write;
+	int restrict_buffer_count;
+};
+
+/* Avaible VPU frequency. */
+enum {
+	VPU_VMIN_LEVEL_0 = 0,
+	VPU_VMIN_LEVEL_1,
+	VPU_VMIN_LEVEL_2,
+	VPU_VMIN_LEVEL_3,
+	VPU_VMIN_LEVEL_4,
+	VPU_VMIN_LEVEL_5,
+	VPU_VMIN_LEVEL_6,
+	VPU_VMIN_LEVEL_7,
+};
+struct vpu_freq_vmin_info {
+	uint32_t freq;
+	uint32_t vmin_level;
+};
+
+struct mvx_dvfs_ctx_t {
+	uint32_t ddr_qos_rsum;
+	uint32_t ddr_qos_wsum;
+
+	/* Frequency limits */
+	struct clk *clock;
+	uint32_t max_freq;
+	uint32_t min_freq;
+	uint32_t up_step_freq;
+	uint32_t down_step_freq;
+	/**
+	 * DVFS polling interval - an interval between frequency updates in milliseconds.
+	 * It is a constant value for non-debug and non-sysfs builds.
+	 */
+	uint32_t poll_interval_ms;
+
+#ifdef CONFIG_THERMAL
+	/* thermal restriction */
+	unsigned long max_state;
+	unsigned long cur_state;
+	struct thermal_cooling_device *cdev;
+#endif
+
+	bool sched_suspend;
+};
+
+/* A list containing all registered sessions */
+static LIST_HEAD(sessions);
+
+struct device *mvx_device;
+
+/* Flag used to prevent usage of DVFS module when it was not initialized */
+static bool initialized = false;
+
+/* Flag used to indicate that DVFS module is going to shut itself down */
+static bool shutdown = false;
+
+/* Semaphore used to prevent concurrent access to DVFS internal structures */
+static struct semaphore dvfs_sem;
+
+/* DVFS polling task */
+static struct task_struct *dvfs_task = NULL;
+static wait_queue_head_t dvfs_wq;
+
+#if (1 == DVFS_DEBUG_MODE)
+/**
+ * Counters used for debugging/verification purposes.
+ */
+
+/* Flag used to enable/disable DVFS in debug builds */
+static atomic_t dvfs_enabled = ATOMIC_INIT(1);
+
+#if (1 == VPU_DDR_QOS_ENABLE)
+static atomic_t ddr_qos_enabled = ATOMIC_INIT(1);
+#endif
+
+/* Amount of times clock frequency was changed by DVFS */
+static atomic_long_t changes_cnt = ATOMIC_LONG_INIT(0);
+
+/* Amount of times burst mode was used by DVFS */
+static atomic_long_t burst_cnt = ATOMIC_LONG_INIT(0);
+#endif
+
+static const struct vpu_freq_vmin_info vpufclk_freqtable[] =
+{
+	{307200000, VPU_VMIN_LEVEL_0},
+	{409600000, VPU_VMIN_LEVEL_1},
+	{491520000, VPU_VMIN_LEVEL_2},
+	{600000000, VPU_VMIN_LEVEL_3},
+	{614400000, VPU_VMIN_LEVEL_4},
+	{750000000, VPU_VMIN_LEVEL_5},
+	{819200000, VPU_VMIN_LEVEL_6},
+	{1000000000, VPU_VMIN_LEVEL_7}
+};
+
+#define FREQ_TABLE_SIZE (sizeof(vpufclk_freqtable) / sizeof(struct vpu_freq_vmin_info))
+
+static struct mvx_dvfs_ctx_t mvx_dvfs_ctx;
+
+static void set_clock_rate(uint32_t clk_rate)
+{
+	//clk_set_rate(mvx_dvfs_ctx.clock, clk_rate);
+}
+
+static uint32_t get_clock_rate(void)
+{
+	//return clk_get_rate(mvx_dvfs_ctx.clock);
+	return 0;
+}
+
+static uint32_t get_max_clock_rate(void) { return DVFS_FREQ_MAX; }
+
+/**
+ * Allocate and register a session in DVFS module.
+ *
+ * This function allocates needed resources for the session and registers
+ * it in the module.
+ *
+ * This function must be called when dvfs_sem semaphore IS locked.
+ *
+ * @param session_is Session id
+ * @return True when registration was successful,
+ *         False otherwise.
+ */
+static bool allocate_session(const mvx_session_id session_id, bool is_encoder)
+{
+	struct session *session;
+
+	session = devm_kzalloc(mvx_device, sizeof(*session), GFP_KERNEL);
+	if (NULL == session) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+			      "DVFS is unable to allocate memory for a new session. session=%p", session_id);
+		return false;
+	}
+
+	session->session_id = session_id;
+	session->ddr_qos_read = session->ddr_qos_write = 0;
+	session->is_encoder = is_encoder;
+
+	INIT_LIST_HEAD(&session->list);
+	list_add(&session->list, &sessions);
+
+	return true;
+}
+
+/**
+ * Unregister a session from DVFS module.
+ *
+ * When session is not NULL, the function releases all previously allocated
+ * resources for the session and unregisters it from DVFS.
+ *
+ * This function must be called when dvfs_sem semaphore IS locked.
+ *
+ * @param session Session or NULL
+ */
+static void free_session(struct session *session)
+{
+	if (NULL == session) {
+		return;
+	}
+
+	list_del(&session->list);
+	devm_kfree(mvx_device, session);
+}
+
+/**
+ * Find a session with provided session_id.
+ *
+ * This function tries to find previously registered session with provided
+ * session_id.
+ *
+ * This function must be called when dvfs_sem semaphore IS locked.
+ *
+ * @param session_id Session id
+ * @return pointer to session structure when a session was found,
+ *         NULL when a session was not found.
+ */
+static struct session *get_session(const mvx_session_id session_id)
+{
+	struct list_head *entry;
+	struct session *session;
+	list_for_each(entry, &sessions)
+	{
+		session = list_entry(entry, struct session, list);
+		if (session->session_id == session_id) {
+			return session;
+		}
+	}
+	return NULL;
+}
+
+/**
+ * Warm up VPU.
+ *
+ * This function increases VPU clock frequency for requested amount
+ * of steps when possible.
+ *
+ * @param steps Requested amount of steps.
+ */
+static void warm_up(const int steps)
+{
+	uint32_t old_freq = get_clock_rate();
+	uint32_t new_freq;
+#if (1 == DVFS_DEBUG_MODE)
+	bool do_burst = false;
+#endif
+
+	/**
+	 * If 3 or more steps are requested, we are far behind required
+	 * performance level.
+	 */
+	if (steps > 2) {
+		new_freq = mvx_dvfs_ctx.max_freq;
+#if (1 == DVFS_DEBUG_MODE)
+		do_burst = true;
+#endif
+	} else {
+		new_freq = min(old_freq + steps * mvx_dvfs_ctx.up_step_freq, mvx_dvfs_ctx.max_freq);
+	}
+
+	if (old_freq != new_freq) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "warm_up. buffer count: %d, old_freq: %d, new_freq: %d",
+			      steps, old_freq, new_freq);
+		set_clock_rate(new_freq);
+#if (1 == DVFS_DEBUG_MODE)
+		atomic_long_inc(&changes_cnt);
+		if (do_burst) {
+			atomic_long_inc(&burst_cnt);
+		}
+#endif
+	}
+}
+
+/**
+ * Cool down VPU.
+ *
+ * This function increases VPU clock frequency if possible.
+ */
+static void cool_down(void)
+{
+	uint32_t old_freq = get_clock_rate();
+	uint32_t new_freq;
+	if (old_freq == mvx_dvfs_ctx.min_freq) {
+		return;
+	}
+	new_freq = max(mvx_dvfs_ctx.min_freq, max(mvx_dvfs_ctx.down_step_freq, old_freq - mvx_dvfs_ctx.down_step_freq));
+	if (old_freq != new_freq) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "cool_down. old_freq: %d, new_freq: %d", old_freq, new_freq);
+		set_clock_rate(new_freq);
+#if (1 == DVFS_DEBUG_MODE)
+		atomic_long_inc(&changes_cnt);
+#endif
+	}
+}
+
+static int get_restrict_buffer_count(mvx_session_id session_id)
+{
+	struct mvx_session *session = (struct mvx_session *)session_id;
+	int buffers_cnt;
+	struct session *dvfs_session = get_session(session_id);
+	if (dvfs_session == NULL) {
+		return -1;
+	}
+
+	/* Don't have to lock the session since we just want to get the number of buffers */
+	if (!dvfs_session->is_encoder) {
+		buffers_cnt = session->port[MVX_DIR_OUTPUT].buffer_count;
+		buffers_cnt -= session->port[MVX_DIR_OUTPUT].buffer_on_hold_count;
+
+		/* There is no enough inputs, no need to boost vpu. */
+		if (session->port[MVX_DIR_INPUT].buffer_count <= 1) {
+			buffers_cnt = session->port[MVX_DIR_INPUT].buffer_count;
+		}
+		// MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "get_restrict_buffer_count. out buffer_count: %d,
+		// buffer_on_hold_count: %d, in buffer_count: %d", session->port[MVX_DIR_OUTPUT].buffer_count,
+		// session->port[MVX_DIR_OUTPUT].buffer_on_hold_count, session->port[MVX_DIR_INPUT].buffer_count);
+	} else {
+		buffers_cnt = session->port[MVX_DIR_INPUT].buffer_count;
+		if (false != session->eos_queued && buffers_cnt < 2) {
+			buffers_cnt = 2;
+		}
+		// MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "get_restrict_buffer_count. buffer_count: %d, eos_queued:
+		// %d", session->port[MVX_DIR_INPUT].buffer_count, session->eos_queued);
+	}
+
+	if (session->keep_freq_high) {
+		buffers_cnt += 2;
+	}
+
+	return buffers_cnt;
+}
+
+/**
+ * Update sessions list and VPU clock frequency.
+ *
+ * This function queries the state of all registered sessions and adjusts
+ * VPU clock frequency to meet their needs when dvfs_control is enabled.
+ * When SYSFS is enabled, the function also stores the status of all sessions
+ * so it could be retrieved by the user.
+ *
+ * This function must be called when dvfs_sem semaphore IS NOT locked.
+ */
+static void update_sessions(void)
+{
+	struct list_head *entry;
+	struct list_head *safe;
+	struct session *session;
+	int restrict_buffer_count;
+	unsigned int buf_max = 0;
+	unsigned int buf_min = UINT_MAX;
+	int sem_failed;
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	if (sem_failed) {
+		return;
+	}
+
+	if (mvx_dvfs_ctx.sched_suspend == true) {
+		up(&dvfs_sem);
+		return;
+	}
+
+	list_for_each_safe(entry, safe, &sessions)
+	{
+		session = list_entry(entry, struct session, list);
+
+		/**
+		 * To avoid potential dead lock we release dvfs_sem before a call to
+		 * get_session_status() callback. After a return from the callback
+		 * we have to take dvfs_sem again and to verify that current session
+		 * was not unregistered by the scheduler while we were sleeping.
+		 */
+		restrict_buffer_count = get_restrict_buffer_count(session->session_id);
+		session->restrict_buffer_count = restrict_buffer_count;
+
+		if (shutdown) {
+			up(&dvfs_sem);
+			return;
+		}
+
+		if (restrict_buffer_count < 0) {
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+				      "DVFS failed to retrieve status for the session. Session was removed? "
+				      "session=%p, restrict_buffer_count=%d",
+				      session->session_id, restrict_buffer_count);
+			continue;
+		}
+
+		if (restrict_buffer_count > buf_max) {
+			buf_max = restrict_buffer_count;
+		}
+		if (restrict_buffer_count < buf_min) {
+			buf_min = restrict_buffer_count;
+		}
+	}
+
+#if (1 == DVFS_DEBUG_MODE)
+	if (0 == atomic_read(&dvfs_enabled)) {
+		up(&dvfs_sem);
+		return;
+	}
+#endif
+
+	if (buf_max > 1) {
+		warm_up(buf_max);
+	} else if (buf_min < 1) {
+		cool_down();
+	}
+	up(&dvfs_sem);
+}
+
+/**
+ * DVFS polling thread.
+ *
+ * This function is executed in a separate kernel thread. It updates clock
+ * frequency every poll_interval_ms milliseconds.
+ */
+static int dvfs_thread(void *v)
+{
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "DVFS polling thread started");
+	while (!kthread_should_stop()) {
+		wait_event_interruptible(dvfs_wq, list_empty(&sessions) == 0 || shutdown);
+		update_sessions();
+		msleep_interruptible(mvx_dvfs_ctx.poll_interval_ms);
+	}
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "DVFS polling thread finished");
+	return 0;
+}
+
+/**
+ *  Return percent percents from a value val.
+ */
+static uint32_t ratio(const uint32_t val, const uint32_t percent)
+{
+	return (uint32_t)(((uint64_t)val * percent) / 100);
+}
+
+#if (1 == DVFS_DEBUG_MODE)
+/**
+ * Print DVFS statistics to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_stats(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	struct list_head *entry;
+	struct session *session;
+	uint32_t freq = get_clock_rate();
+
+	num += scnprintf(buf + num, PAGE_SIZE - num, "freq: %4u, max_freq: %4u, up_step_freq: %3u, down_step_freq: %3u",
+			 freq, mvx_dvfs_ctx.max_freq, mvx_dvfs_ctx.up_step_freq, mvx_dvfs_ctx.down_step_freq);
+#if (1 == DVFS_DEBUG_MODE)
+	num += scnprintf(buf + num, PAGE_SIZE - num,
+			 ", enabled: %1u, poll_interval_ms: %3u, changes_cnt: %10lu, burst_cnt: %10lu",
+			 atomic_read(&dvfs_enabled), mvx_dvfs_ctx.poll_interval_ms, atomic_long_read(&changes_cnt),
+			 atomic_long_read(&burst_cnt));
+#endif
+	num += scnprintf(buf + num, PAGE_SIZE - num, "\n");
+	list_for_each(entry, &sessions)
+	{
+		session = list_entry(entry, struct session, list);
+		num += scnprintf(buf + num, PAGE_SIZE - num, "%p: out_buf: %02u\n", session->session_id,
+				 session->restrict_buffer_count);
+	}
+
+	return num;
+}
+
+/**
+ * Print DVFS enabling status to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_enabled(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	num += scnprintf(buf, PAGE_SIZE, "%u\n", atomic_read(&dvfs_enabled) ? 1 : 0);
+	return num;
+}
+
+/**
+ * Set DVFS enabling status from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_enabled(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	int enabled;
+	failed = kstrtouint(buf, 10, &enabled);
+	if (!failed) {
+		atomic_set(&dvfs_enabled, enabled);
+	}
+	return (failed) ? failed : count;
+}
+
+/**
+ * Print current clock frequency to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_freq(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	uint32_t freq = get_clock_rate();
+	num += scnprintf(buf, PAGE_SIZE, "%u\n", freq);
+	return num;
+}
+
+/**
+ * Set current clock frequency from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_freq(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	unsigned int freq;
+	failed = kstrtouint(buf, 10, &freq);
+	if (!failed) {
+		set_clock_rate((uint32_t)freq);
+	}
+	return (failed) ? failed : count;
+}
+
+/**
+ * Print min clock frequency to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_min_freq(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	uint32_t freq = mvx_dvfs_ctx.min_freq;
+	num += scnprintf(buf, PAGE_SIZE, "%u\n", freq);
+	return num;
+}
+
+uint32_t clip_min_max_rate(uint32_t freq, bool is_min_freq)
+{
+	int i = 0;
+	bool clip = false;
+	uint32_t clip_freq;
+	if (is_min_freq) {
+		for (i = 0; i < FREQ_TABLE_SIZE; i++) {
+			if (freq <= vpufclk_freqtable[i].freq) {
+				clip = true;
+				clip_freq = vpufclk_freqtable[i].freq;
+				break;
+			}
+		}
+		if (!clip)
+			clip_freq = vpufclk_freqtable[FREQ_TABLE_SIZE - 1].freq;
+	} else {
+		for (i = FREQ_TABLE_SIZE - 1; i >= 0; i--) {
+			if (freq >= vpufclk_freqtable[i].freq) {
+				clip = true;
+				clip_freq = vpufclk_freqtable[i].freq;
+				break;
+			}
+		}
+		if (!clip)
+			clip_freq = vpufclk_freqtable[0].freq;
+	}
+
+	return clip_freq;
+}
+
+/**
+ * Set min clock frequency from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_min_freq(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	unsigned int freq;
+	failed = kstrtouint(buf, 10, &freq);
+	freq = clip_min_max_rate(freq, true);
+	if (!failed) {
+		mvx_dvfs_ctx.min_freq = (uint32_t)freq;
+	}
+	return (failed) ? failed : count;
+}
+
+/**
+ * Print max clock frequency to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_max_frep(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	uint32_t freq = mvx_dvfs_ctx.max_freq;
+	num += scnprintf(buf, PAGE_SIZE, "%u\n", freq);
+	return num;
+}
+
+/**
+ * Set max clock frequency from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_max_freq(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	unsigned int freq;
+	failed = kstrtouint(buf, 10, &freq);
+	freq = clip_min_max_rate(freq, false);
+	if (!failed) {
+		mvx_dvfs_ctx.max_freq = (uint32_t)freq;
+		mvx_dvfs_ctx.up_step_freq = ratio(mvx_dvfs_ctx.max_freq, UP_STEP_PERCENT);
+		mvx_dvfs_ctx.down_step_freq = ratio(mvx_dvfs_ctx.max_freq, DOWN_STEP_PERCENT);
+	}
+	return (failed) ? failed : count;
+}
+
+/**
+ * Set polling interval from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_poll_interval_ms(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	failed = kstrtouint(buf, 10, &mvx_dvfs_ctx.poll_interval_ms);
+	return (failed) ? failed : count;
+}
+
+/**
+ * Set up_step value from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_up_step_percent(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	unsigned int up_step_percent;
+	failed = kstrtouint(buf, 10, &up_step_percent);
+	if (!failed) {
+		mvx_dvfs_ctx.up_step_freq = ratio(mvx_dvfs_ctx.max_freq, up_step_percent);
+	}
+	return (failed) ? failed : count;
+}
+
+/**
+ * Set down_step value from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_down_step_percent(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	unsigned int down_step_percent;
+	failed = kstrtouint(buf, 10, &down_step_percent);
+	if (!failed) {
+		mvx_dvfs_ctx.down_step_freq = ratio(mvx_dvfs_ctx.max_freq, down_step_percent);
+	}
+	return (failed) ? failed : count;
+}
+/**
+ * Print available clock frequency to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_available_freq(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	int32_t i;
+	for (i = 0; i < FREQ_TABLE_SIZE; i++) {
+		num += scnprintf(buf + num, PAGE_SIZE - num, "%u	", vpufclk_freqtable[i].freq);
+	}
+	num += scnprintf(buf + num, PAGE_SIZE - num, "\n");
+	return num;
+}
+
+#if (1 == VPU_DDR_QOS_ENABLE)
+static ssize_t sysfs_print_ddr_qos_enable(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	num += snprintf(buf, PAGE_SIZE, "%u\n", atomic_read(&ddr_qos_enabled) ? 1 : 0);
+	return num;
+}
+
+static ssize_t sysfs_set_ddr_qos_enable(struct device *dev, struct device_attribute *attr, const char *buf,
+					size_t count)
+{
+	int failed;
+	int enable;
+	failed = kstrtouint(buf, 10, &enable);
+	if (!failed) {
+		atomic_set(&ddr_qos_enabled, enable);
+	}
+	return (failed) ? failed : count;
+}
+#endif
+
+/**
+ * Print watchdog timeout value to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_watchdog_timeout(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	uint32_t watchdog_timeout = session_watchdog_timeout;
+	num += scnprintf(buf, PAGE_SIZE, "%u\n", watchdog_timeout);
+	return num;
+}
+
+/**
+ * Set watchdog timeout value from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_watchdog_timeout(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	unsigned int watchdog_timeout;
+	failed = kstrtouint(buf, 10, &watchdog_timeout);
+	if (!failed) {
+		session_watchdog_timeout = watchdog_timeout;
+	}
+	return (failed) ? failed : count;
+}
+
+/**
+ * Print wait pending timeout value to sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+static ssize_t sysfs_print_wait_pending_timeout(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	ssize_t num = 0;
+	uint32_t wait_pending_timeout = session_wait_pending_timeout;
+	num += scnprintf(buf, PAGE_SIZE, "%u\n", wait_pending_timeout);
+	return num;
+}
+
+/**
+ * Set wait pending timeout value from sysfs attribute.
+ *
+ * Used for debugging/verification purposes.
+ */
+ssize_t sysfs_set_wait_pending_timeout(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	int failed;
+	unsigned int wait_pending_timeout;
+	failed = kstrtouint(buf, 10, &wait_pending_timeout);
+	if (!failed) {
+		session_wait_pending_timeout = wait_pending_timeout;
+	}
+	return (failed) ? failed : count;
+}
+
+/* Sysfs attributes used to debug/verify DVFS module */
+static struct device_attribute sysfs_files[] = {
+	__ATTR(dvfs_stats, S_IRUGO, sysfs_print_stats, NULL),
+	__ATTR(dvfs_enable, (S_IRUGO | S_IWUSR), sysfs_print_enabled, sysfs_set_enabled),
+	__ATTR(dvfs_freq, (S_IRUGO | S_IWUSR), sysfs_print_freq, sysfs_set_freq),
+	__ATTR(dvfs_poll_interval_ms, S_IWUSR, NULL, sysfs_set_poll_interval_ms),
+	__ATTR(dvfs_up_step_percent, S_IWUSR, NULL, sysfs_set_up_step_percent),
+	__ATTR(dvfs_down_step_percent, S_IWUSR, NULL, sysfs_set_down_step_percent),
+	__ATTR(dvfs_available_freqency, S_IRUGO, sysfs_print_available_freq, NULL),
+	__ATTR(dvfs_min_freq, (S_IRUGO | S_IWUSR), sysfs_print_min_freq, sysfs_set_min_freq),
+	__ATTR(dvfs_max_freq, (S_IRUGO | S_IWUSR), sysfs_print_max_frep, sysfs_set_max_freq),
+#if (1 == VPU_DDR_QOS_ENABLE)
+	__ATTR(ddr_qos_enable, (S_IRUGO | S_IWUSR), sysfs_print_ddr_qos_enable, sysfs_set_ddr_qos_enable),
+#endif
+	__ATTR(watchdog_timeout, (S_IRUGO | S_IWUSR), sysfs_print_watchdog_timeout, sysfs_set_watchdog_timeout),
+	__ATTR(wait_pending_timeout, (S_IRUGO | S_IWUSR), sysfs_print_wait_pending_timeout,
+	       sysfs_set_wait_pending_timeout),
+};
+
+/**
+ * Register all DVFS attributes in sysfs subsystem
+ */
+static void sysfs_register_devices(struct device *dev)
+{
+	int err;
+	int i = NELEMS(sysfs_files);
+
+	while (i--) {
+		err = device_create_file(dev, &sysfs_files[i]);
+		if (err < 0) {
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "DVFS is unable to create sysfs file. name=%s",
+				      sysfs_files[i].attr.name);
+		}
+	}
+}
+
+/**
+ * Remove DVFS attributes from sysfs subsystem
+ */
+static void sysfs_unregister_devices(struct device *dev)
+{
+	int i = NELEMS(sysfs_files);
+
+	while (i--) {
+		device_remove_file(dev, &sysfs_files[i]);
+	}
+}
+#endif /* DVFS_DEBUG_MODE */
+
+#ifdef CONFIG_THERMAL
+static int vpu_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state)
+{
+	struct mvx_dvfs_ctx_t *ctx = cdev->devdata;
+
+	*state = ctx->max_state;
+	return 0;
+}
+
+static int vpu_get_cur_state(struct thermal_cooling_device *cdev, unsigned long *state)
+{
+	struct mvx_dvfs_ctx_t *ctx = cdev->devdata;
+
+	*state = ctx->cur_state;
+	return 0;
+}
+
+static int vpu_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state)
+{
+	struct mvx_dvfs_ctx_t *ctx = cdev->devdata;
+
+	if (state > ctx->max_state)
+		return -EINVAL;
+
+	if (ctx->cur_state == state)
+		return 0;
+
+	ctx->max_freq = vpufclk_freqtable[FREQ_TABLE_SIZE - state - 1].freq;
+	ctx->cur_state = state;
+	ctx->up_step_freq = ratio(mvx_dvfs_ctx.max_freq, UP_STEP_PERCENT);
+	ctx->down_step_freq = ratio(mvx_dvfs_ctx.max_freq, DOWN_STEP_PERCENT);
+	return 0;
+}
+
+static struct thermal_cooling_device_ops vpu_cooling_ops = {
+	.get_max_state = vpu_get_max_state,
+	.get_cur_state = vpu_get_cur_state,
+	.set_cur_state = vpu_set_cur_state,
+};
+#endif
+
+/**
+ * Initialize the DVFS module.
+ *
+ * Must be called before any other function in this module.
+ *
+ * @param dev Device
+ */
+void mvx_dvfs_init(struct device *dev)
+{
+	if (!initialized) {
+		int i;
+		int min_vmin_level;
+		sema_init(&dvfs_sem, 1);
+
+		mvx_dvfs_ctx.max_freq = get_max_clock_rate();
+		mvx_dvfs_ctx.up_step_freq = ratio(mvx_dvfs_ctx.max_freq, UP_STEP_PERCENT);
+		mvx_dvfs_ctx.down_step_freq = ratio(mvx_dvfs_ctx.max_freq, DOWN_STEP_PERCENT);
+		mvx_dvfs_ctx.min_freq = vpufclk_freqtable[0].freq;
+		min_vmin_level = vpufclk_freqtable[0].vmin_level;
+		mvx_dvfs_ctx.sched_suspend = false;
+
+		/*Use the max clk freq with min vmin as bottom freq of dvfs */
+		for (i = 1; i < FREQ_TABLE_SIZE; i++) {
+			if (min_vmin_level < vpufclk_freqtable[i].vmin_level) {
+				break;
+			}
+			mvx_dvfs_ctx.min_freq = vpufclk_freqtable[i].freq;
+		}
+
+		mvx_dvfs_ctx.clock = devm_clk_get(dev, NULL);
+		mvx_dvfs_ctx.poll_interval_ms = POLL_INTERVAL_MS;
+
+		init_waitqueue_head(&dvfs_wq);
+		dvfs_task = kthread_run(dvfs_thread, NULL, "dvfs");
+
+#if (1 == DVFS_DEBUG_MODE)
+		if (NULL != dev && IS_ENABLED(CONFIG_DEBUG_FS)) {
+			sysfs_register_devices(dev);
+		}
+#endif
+
+		initialized = true;
+		shutdown = false;
+		mvx_device = dev;
+
+#ifdef CONFIG_THERMAL
+		if (of_find_property(dev->of_node, "#cooling-cells", NULL)) {
+			mvx_dvfs_ctx.max_state = FREQ_TABLE_SIZE - 1;
+			mvx_dvfs_ctx.cur_state = 0; // VPU_VMIN_LEVEL_6 for cur_state 0
+			mvx_dvfs_ctx.cdev =
+				thermal_cooling_device_register("mvx-vpu-clk", &mvx_dvfs_ctx, &vpu_cooling_ops);
+			if (IS_ERR(mvx_dvfs_ctx.cdev)) {
+				pr_err("%s: Failed to register vpu cooling device (%d)\n", __func__,
+				       (int)PTR_ERR(mvx_dvfs_ctx.cdev));
+				mvx_dvfs_ctx.cdev = NULL;
+			}
+		}
+#endif
+	} else {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "Attempt to initialize DVFS twice");
+	}
+}
+
+/**
+ * Deinitialize the DVFS module.
+ *
+ * All remaining sessions will be unregistered.
+ *
+ * @param dev Device
+ */
+void mvx_dvfs_deinit(struct device *dev)
+{
+	int sem_failed;
+	struct list_head *entry;
+	struct list_head *safe;
+	struct session *session;
+
+	if (!initialized) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING,
+			      "Attempt to deinitialize DVFS when it was not initialized");
+		return;
+	}
+
+#ifdef CONFIG_THERMAL
+	if (mvx_dvfs_ctx.cdev) {
+		thermal_cooling_device_unregister(mvx_dvfs_ctx.cdev);
+		mvx_dvfs_ctx.cdev = NULL;
+	}
+#endif
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	shutdown = true;
+	if (!sem_failed) {
+		up(&dvfs_sem);
+	}
+
+	wake_up_interruptible(&dvfs_wq);
+	if (!IS_ERR_OR_NULL(dvfs_task)) {
+		kthread_stop(dvfs_task);
+	}
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	list_for_each_safe(entry, safe, &sessions)
+	{
+		session = list_entry(entry, struct session, list);
+		free_session(session);
+	}
+
+#if (1 == DVFS_DEBUG_MODE)
+	if (NULL != dev && IS_ENABLED(CONFIG_DEBUG_FS)) {
+		sysfs_unregister_devices(dev);
+	}
+#endif
+
+	devm_clk_put(mvx_device, mvx_dvfs_ctx.clock);
+	initialized = false;
+	mvx_device = NULL;
+	if (!sem_failed) {
+		up(&dvfs_sem);
+	}
+}
+
+/**
+ * Register session in the DFVS module.
+ *
+ * @param session_id Session id
+ * @return True when registration was successful,
+ *         False, otherwise
+ */
+bool mvx_dvfs_register_session(const mvx_session_id session_id, bool is_encoder)
+{
+	bool success = false;
+	int sem_failed;
+
+	if (!initialized) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "DVFS module was not initialized");
+		return false;
+	}
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	if (sem_failed) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "DVFS semaphore was not obtained, sem_failed=%d",
+			      sem_failed);
+		return false;
+	}
+
+	if (shutdown) {
+		up(&dvfs_sem);
+		return false;
+	}
+
+	if (get_session(session_id) != NULL) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "this session is already registered. session=%p", session_id);
+		up(&dvfs_sem);
+		return true;
+	}
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_dvfs_register_session. session=%p", session_id);
+
+	success = allocate_session(session_id, is_encoder);
+	up(&dvfs_sem);
+
+	if (success) {
+		bool adjust = true;
+#if (1 == DVFS_DEBUG_MODE)
+		/* Has DVFS been disabled through the sysfs interface? */
+		adjust = atomic_read(&dvfs_enabled);
+#endif
+		if (adjust) {
+			set_clock_rate(mvx_dvfs_ctx.max_freq);
+		}
+	}
+	wake_up_interruptible(&dvfs_wq);
+
+	return success;
+}
+
+/**
+ * Unregister session from the DFVS module.
+ *
+ * Usage of corresponding session is not permitted after this call.
+ * @param session_id Session id
+ */
+void mvx_dvfs_unregister_session(const mvx_session_id session_id)
+{
+	struct session *session;
+	int sem_failed;
+
+	if (!initialized) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "DVFS module was not initialized");
+		return;
+	}
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	if (sem_failed) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "DVFS semaphore was not obtained, %d", sem_failed);
+		return;
+	}
+
+	session = get_session(session_id);
+	if (NULL != session) {
+#if (1 == VPU_DDR_QOS_ENABLE)
+		if ((session->ddr_qos_read + session->ddr_qos_write) &&
+		    mvx_dvfs_ctx.ddr_qos_rsum >= session->ddr_qos_read &&
+		    mvx_dvfs_ctx.ddr_qos_wsum >= session->ddr_qos_write) {
+			mvx_dvfs_ctx.ddr_qos_rsum -= session->ddr_qos_read;
+			mvx_dvfs_ctx.ddr_qos_wsum -= session->ddr_qos_write;
+			MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "DVFS remove session[%p] ddr qos: [%d, %d]/[%d, %d]",
+				      session_id, session->ddr_qos_read, session->ddr_qos_write,
+				      mvx_dvfs_ctx.ddr_qos_rsum, mvx_dvfs_ctx.ddr_qos_wsum);
+			session->ddr_qos_read = session->ddr_qos_write = 0;
+		}
+#endif
+
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "mvx_dvfs_unregister_session. session=%p", session_id);
+		free_session(session);
+
+	} else {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "session[%p] is already removed.", session_id);
+	}
+
+	up(&dvfs_sem);
+}
+
+void mvx_dvfs_estimate_ddr_bandwidth(struct estimate_ddr_input *input, struct estimate_ddr_output *output)
+{
+	/* predefined DDR throughput requirement for 1080p@30fps */
+	const int defined_width = 1920;
+	const int defined_height = 1080;
+	const uint32_t defined_bandwidth_tbl[2][2][2] = {{
+								 /*decoder*/
+								 {45000, 105000}, /*non-afbc[r, w]*/
+								 {45000, 20000}	  /*afbc*/
+							 },
+							 {
+								 /*encoder*/
+								 {162000, 54000}, /*non-afbc[r, w]*/
+								 {162000, 54000}  /*afbc*/
+							 }};
+	uint64_t estimated_read;
+	uint64_t estimated_write;
+
+	if (input->width == 0 || input->height == 0) {
+		input->width = defined_width;
+		input->height = defined_height;
+	}
+	if (input->fps <= 0) {
+		input->fps = VPU_DDR_QOS_PREDEFINED_FPS;
+	}
+
+	MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO,
+		      "estimate_ddr_bandwidth. isEnc=%d, isAFBC=%d, size=(%d, %d), framerate=%d.", input->isEnc,
+		      input->isAFBC, input->width, input->height, input->fps);
+
+	estimated_read =
+		((uint64_t)defined_bandwidth_tbl[input->isEnc][input->isAFBC][0] * (input->width * input->height) *
+		 input->fps / (defined_width * defined_height * VPU_DDR_QOS_PREDEFINED_FPS));
+	estimated_write =
+		((uint64_t)defined_bandwidth_tbl[input->isEnc][input->isAFBC][1] * (input->width * input->height) *
+		 input->fps / (defined_width * defined_height * VPU_DDR_QOS_PREDEFINED_FPS));
+	if ((estimated_read + estimated_write) < VPU_DDR_QOS_MIN) {
+		estimated_read = VPU_DDR_QOS_MIN / 2;
+		estimated_write = VPU_DDR_QOS_MIN / 2;
+	}
+
+	if ((estimated_read + estimated_write) > VPU_DDR_QOS_MAX) {
+		estimated_read = VPU_DDR_QOS_MAX / 2;
+		estimated_write = VPU_DDR_QOS_MAX / 2;
+	}
+
+	output->estimated_read = estimated_read;
+	output->estimated_write = estimated_write;
+}
+
+void mvx_dvfs_session_update_ddr_qos(const mvx_session_id session_id, uint32_t read_value, uint32_t write_value)
+{
+#if (1 == VPU_DDR_QOS_ENABLE)
+	struct session *session;
+	int sem_failed;
+
+#if (1 == DVFS_DEBUG_MODE)
+	if (0 == atomic_read(&ddr_qos_enabled)) {
+		return;
+	}
+#endif
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	if (sem_failed) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_ERROR, "DVFS semaphore was not obtained, %d", sem_failed);
+		return;
+	}
+
+	session = get_session(session_id);
+	if (NULL != session && (session->ddr_qos_read != read_value || session->ddr_qos_write != write_value) &&
+	    mvx_dvfs_ctx.ddr_qos_rsum >= session->ddr_qos_read && mvx_dvfs_ctx.ddr_qos_wsum >= session->ddr_qos_write) {
+		mvx_dvfs_ctx.ddr_qos_rsum -= session->ddr_qos_read;
+		mvx_dvfs_ctx.ddr_qos_rsum += read_value;
+		session->ddr_qos_read = read_value;
+		mvx_dvfs_ctx.ddr_qos_wsum -= session->ddr_qos_write;
+		mvx_dvfs_ctx.ddr_qos_wsum += write_value;
+		session->ddr_qos_write = write_value;
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_INFO, "DVFS update session[%p] ddr qos: [%d, %d]/[%d, %d]",
+			      session_id, read_value, write_value, mvx_dvfs_ctx.ddr_qos_rsum,
+			      mvx_dvfs_ctx.ddr_qos_wsum);
+	}
+
+	up(&dvfs_sem);
+#endif
+}
+
+void mvx_dvfs_suspend_session(void)
+{
+	int sem_failed;
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	if (sem_failed) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "DVFS semaphore was not obtained, sem_failed=%d",
+			      sem_failed);
+	}
+
+	mvx_dvfs_ctx.sched_suspend = true;
+
+	if (!sem_failed) {
+		up(&dvfs_sem);
+	}
+}
+
+void mvx_dvfs_resume_session(void)
+{
+	int sem_failed;
+
+	sem_failed = down_interruptible(&dvfs_sem);
+	if (sem_failed) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "DVFS semaphore was not obtained, sem_failed=%d",
+			      sem_failed);
+	}
+
+	mvx_dvfs_ctx.sched_suspend = false;
+
+	if (!sem_failed) {
+		up(&dvfs_sem);
+	}
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_dvfs.h b/drivers/media/platform/spacemit/vpu_k3/mvx_dvfs.h
new file mode 100644
index 000000000000..c0d90e3adaa5
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_dvfs.h
@@ -0,0 +1,120 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_DVFS_H_
+#define _MVX_DVFS_H_
+
+typedef void *mvx_session_id;
+
+struct estimate_ddr_input {
+	int width;
+	int height;
+	int isAFBC;
+	int fps;
+	int isEnc;
+};
+
+struct estimate_ddr_output {
+	uint64_t estimated_read;
+	uint64_t estimated_write;
+};
+
+/**
+ * Initialize the DVFS module.
+ *
+ * Must be called before any other function in this module.
+ *
+ * @param dev Device
+ */
+void mvx_dvfs_init(struct device *dev);
+
+/**
+ * Deinitialize the DVFS module.
+ *
+ * All remaining sessions will be unregistered.
+ *
+ * @param dev Device
+ */
+void mvx_dvfs_deinit(struct device *dev);
+
+/**
+ * Register session in the DFVS module.
+ *
+ * @param session_id Session id
+ * @return True when registration was successful,
+ *         False, otherwise
+ */
+bool mvx_dvfs_register_session(const mvx_session_id session_id, bool is_encoder);
+
+/**
+ * Unregister session from the DFVS module.
+ *
+ * Usage of corresponding session is not permitted after this call.
+ * @param session_id Session id
+ */
+void mvx_dvfs_unregister_session(const mvx_session_id session_id);
+
+void mvx_dvfs_estimate_ddr_bandwidth(struct estimate_ddr_input *input, struct estimate_ddr_output *output);
+
+void mvx_dvfs_session_update_ddr_qos(const mvx_session_id session_id, uint32_t read_value, uint32_t write_value);
+
+/**
+ * Suspend dvfs thread to adjust vpu clk when device enters suspend state.
+ */
+void mvx_dvfs_suspend_session(void);
+
+/**
+ * Resume dvfs thread to adjust vpu clk when device resumes from suspend state.
+ */
+void mvx_dvfs_resume_session(void);
+
+ssize_t sysfs_set_enabled(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+ssize_t sysfs_set_freq(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+uint32_t clip_min_max_rate(uint32_t freq, bool is_min_freq);
+
+ssize_t sysfs_set_min_freq(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+ssize_t sysfs_set_max_freq(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+ssize_t sysfs_set_poll_interval_ms(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+ssize_t sysfs_set_up_step_percent(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+ssize_t sysfs_set_watchdog_timeout(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+ssize_t sysfs_set_wait_pending_timeout(struct device *dev, struct device_attribute *attr, const char *buf,
+				       size_t count);
+
+ssize_t sysfs_set_down_step_percent(struct device *dev, struct device_attribute *attr, const char *buf, size_t count);
+
+#endif /* MVX_DVFS_H */
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_log.c b/drivers/media/platform/spacemit/vpu_k3/mvx_log.c
new file mode 100644
index 000000000000..649c057c2b0c
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_log.c
@@ -0,0 +1,840 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/******************************************************************************
+ * Includes
+ ******************************************************************************/
+
+#include "mvx_log.h"
+#include "mvx_log_ram.h"
+
+#include <linux/uaccess.h>
+#include <linux/aio.h>
+#include <linux/debugfs.h>
+#include <linux/dcache.h>
+#include <linux/export.h>
+#include <linux/fs.h>
+#include <linux/poll.h>
+#include <linux/namei.h>
+#include <linux/sched.h>
+#include <linux/time.h>
+#include <linux/un.h>
+#include <linux/version.h>
+#include <linux/vmalloc.h>
+
+/******************************************************************************
+ * Defines
+ ******************************************************************************/
+
+#ifndef UNUSED
+#define UNUSED(x) (void)(x)
+#endif /* UNUSED */
+
+/******************************************************************************
+ * Types
+ ******************************************************************************/
+
+/******************************************************************************
+ * Variables
+ ******************************************************************************/
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+
+/**
+ * Map severity to string.
+ */
+static const char *const severity_to_name[] = {
+	"Panic",
+	"Error",
+	"Warning",
+	"Info",
+	"Debug",
+	"Verbose"
+};
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+/**
+ * Map severity to kernel log level.
+ */
+static const char *const severity_to_kern_level[] = {
+	KERN_EMERG,
+	KERN_ERR,
+	KERN_WARNING,
+	KERN_NOTICE,
+	KERN_INFO,
+	KERN_DEBUG
+};
+
+/******************************************************************************
+ * Static functions
+ ******************************************************************************/
+
+/******************************************************************************
+ * Log
+ *
+ * Directory                    i_node->i_private
+ * --------------------------------------------------------
+ * mvx                          struct mvx_log *
+ * +-- group
+ * |   +-- <group>              struct mvx_log_group *
+ * |       +-- severity
+ * |       +-- drain
+ * +-- drain
+ *     +-- <drain>              struct mvx_log_drain *
+ *
+ ******************************************************************************/
+
+/**
+ * trim() - Trim of trailing new line.
+ * @str:	Pointer to string.
+ */
+static void trim(char *str)
+{
+	size_t len = strlen(str);
+
+	while (len-- > 0) {
+		if (str[len] != '\n')
+			break;
+
+		str[len] = '\0';
+	}
+}
+
+/**
+ * lookup() - Search for child dentry with matching name.
+ * @parent:	Pointer to parent dentry.
+ * @name:	Name of dentry to look for.
+ *
+ * Return: Pointer to dentry, NULL if not found.
+ */
+static struct dentry *lookup(struct dentry *parent, const char *name)
+{
+	struct dentry *child;
+
+	/* Loop over directory entries in mvx/drain/. */
+#if (KERNEL_VERSION(6, 7, 0) <= LINUX_VERSION_CODE)
+	hlist_for_each_entry(child, &parent->d_children, d_sib)
+#elif (KERNEL_VERSION(3, 18, 0) <= LINUX_VERSION_CODE)
+	list_for_each_entry(child, &parent->d_subdirs, d_child)
+#else
+	list_for_each_entry(child, &parent->d_subdirs, d_u.d_child)
+#endif
+	{
+		if (strcmp(name, child->d_name.name) == 0)
+			return child;
+	}
+
+	return NULL;
+}
+
+/**
+ * get_inode_private() - Get inode private member of parent directory.
+ * @file:		File pointer.
+ * @parent:		Number of parent directories.
+ *
+ * Return: Inode private member, or NULL on error.
+ */
+static void *get_inode_private(struct file *file, int parent)
+{
+	struct dentry *d = file->f_path.dentry;
+
+	while (d != NULL && parent-- > 0)
+		d = d->d_parent;
+
+	if (d == NULL || d->d_inode == NULL)
+		return NULL;
+
+	return d->d_inode->i_private;
+}
+
+/**
+ * readme_read() - Read handle function for mvx/group/<group>/drain. The
+ *		   function returns the usage instruction message.
+ * @file:		File pointer.
+ * @user_buffer:	The user space buffer that is read to.
+ * @count:		The maximum number of bytes to read.
+ * @position:		The current position in the buffer.
+ */
+static ssize_t readme_read(struct file *file, char __user *user_buffer, size_t count, loff_t *position)
+{
+	static const char msg[] = "LOG GROUPS\n"
+				  "\n"
+				  "The avaible log groups can be found under 'group'.\n"
+				  "$ ls group\n"
+				  "\n"
+				  "SEVERITY LEVELS\n"
+				  "    0 - Panic\n"
+				  "    1 - Error\n"
+				  "    2 - Warning\n"
+				  "    3 - Info\n"
+				  "    4 - Debug\n"
+				  "    5 - Verbose\n"
+				  "\n"
+				  "The severity level for a log group can be read and set at runtime.\n"
+				  "$ cat group/general/severity\n"
+				  "$ echo 3 > group/general/severity\n";
+
+	return simple_read_from_buffer(user_buffer, count, position, msg, sizeof(msg));
+}
+
+/**
+ * group_drain_read() - Read handle function for mvx/group/<group>/drain. The
+ *			function returns the name of the currently configured
+ *			drain.
+ * @file:		File pointer.
+ * @user_buffer:	The user space buffer that is read to.
+ * @count:		The maximum number of bytes to read.
+ * @position:		The current position in the buffer.
+ */
+static ssize_t group_drain_read(struct file *file, char __user *user_buffer, size_t count, loff_t *position)
+{
+	/* File path mvx/group/<group>/drain. */
+	struct mvx_log_group *group = get_inode_private(file, 1);
+	struct mvx_log_drain *drain = group->drain;
+	char name[100];
+	size_t len;
+
+	if (drain == NULL || drain->dentry == NULL) {
+		pr_err("MVX: No drain assigned to log group.\n");
+		return -EINVAL;
+	}
+
+	len = scnprintf(name, sizeof(name), "%s\n", drain->dentry->d_name.name);
+
+	return simple_read_from_buffer(user_buffer, count, position, name, len);
+}
+
+/**
+ * group_drain_write() - Write handle function for mvx/group/<group>/drain. The
+ *			 function sets the drain for the group. If the drain
+ *			 does not match any registered drain, then error is
+ *			 returned to user space.
+ * @file:		File pointer.
+ * @user_buffer:	The user space buffer that is written to.
+ * @count:		The maximum number of bytes to write.
+ * @position:		The current position in the buffer.
+ */
+static ssize_t group_drain_write(struct file *file, const char __user *user_buffer, size_t count, loff_t *position)
+{
+	/* File path mvx/group/<group>/drain. */
+	struct mvx_log_group *group = get_inode_private(file, 1);
+	struct mvx_log *log = get_inode_private(file, 3);
+	struct dentry *dentry;
+	char drain_str[100];
+	ssize_t size;
+
+	/* Check that input is not larger that path buffer. */
+	if (count > (sizeof(drain_str) - 1)) {
+		pr_err("MVX: Input overflow.\n");
+
+		return -EINVAL;
+	}
+
+	/* Append input to path. */
+	size = simple_write_to_buffer(drain_str, sizeof(drain_str) - 1, position, user_buffer, count);
+	drain_str[count] = '\0';
+	trim(drain_str);
+
+	dentry = lookup(log->drain_dir, drain_str);
+
+	if (IS_ERR_OR_NULL(dentry)) {
+		pr_warn("MVX: No drain matching '%s'.\n", drain_str);
+		return -EINVAL;
+	}
+
+	/* Assign drain to log group. */
+	group->drain = dentry->d_inode->i_private;
+
+	return size;
+}
+
+/**
+ * drain_ram_read() - Read the RAM buffer.
+ * @drain:		The RAM buffer drain.
+ * @user_buffer:	The user space buffer that is read to.
+ * @count:		The maximum number of bytes to read.
+ * @position:		The current position in the buffer.
+ * @pos:		The last used position of the drain buffer
+ */
+static ssize_t drain_ram_read(struct mvx_log_drain_ram *drain, char __user *user_buffer, size_t count, loff_t *position,
+			      size_t pos)
+{
+	ssize_t n = 0;
+
+	/* Make sure position is not beyond end of file. */
+	if (*position > pos)
+		return -EINVAL;
+
+	/* If position is more than BUFFER_SIZE bytes behind, then fast forward
+	 * to current position minus BUFFER_SIZE.
+	 */
+	if ((pos - *position) > drain->buffer_size)
+		*position = pos - drain->buffer_size;
+
+	/* Copy data to user space. */
+	while ((n < count) && (*position < pos)) {
+		size_t offset;
+		size_t length;
+
+		/* Offset in circular buffer. */
+		offset = *position & (drain->buffer_size - 1);
+
+		/* Available number of bytes. */
+		length = min((size_t)(pos - *position), count - n);
+
+		/* Make sure length does not go beyond end of circular buffer.
+		 */
+		length = min(length, drain->buffer_size - offset);
+
+		/* Copy data from kernel- to user space. */
+		length -= copy_to_user(&user_buffer[n], &drain->buf[offset], length);
+
+		/* No bytes were copied. Return error. */
+		if (length == 0)
+			return -EINVAL;
+
+		*position += length;
+		n += length;
+	}
+
+	return n;
+}
+
+/**
+ * drain_ram_read_msg() - Read of the RAM file.
+ * @file:		File pointer.
+ * @user_buffer:	The user space buffer that is read to.
+ * @count:		The maximum number of bytes to read.
+ * @position:		The current position in the buffer.
+ */
+static ssize_t drain_ram_read_msg(struct file *file, char __user *user_buffer, size_t count, loff_t *position)
+{
+	struct mvx_log_drain_ram *drain = get_inode_private(file, 1);
+
+	while (*position == drain->write_pos) {
+		int ret;
+
+		if (file->f_flags & O_NONBLOCK)
+			return -EAGAIN;
+
+		/* Block until there is data available. */
+		ret = wait_event_interruptible(drain->queue, *position < drain->write_pos);
+		if (ret != 0)
+			return -EINTR;
+	}
+
+	return drain_ram_read(drain, user_buffer, count, position, drain->write_pos);
+}
+
+/**
+ * drain_ram_msg_poll() - Handle poll.
+ * @file:		File pointer.
+ * @wait:		The poll table to which the wait queue is added.
+ */
+static unsigned int drain_ram_msg_poll(struct file *file, poll_table *wait)
+{
+	unsigned int mask = 0;
+	struct mvx_log_drain_ram *drain = get_inode_private(file, 1);
+
+	poll_wait(file, &drain->queue, wait);
+
+	if (file->f_pos < drain->write_pos)
+		mask |= POLLIN | POLLRDNORM;
+	else if (file->f_pos > drain->write_pos)
+		mask |= POLLERR;
+
+	return mask;
+}
+
+/**
+ * drain_ram_ioctl() - Handle IOCTL.
+ * @file:		File pointer.
+ * @cmd:		The value of the command to be handled.
+ * @arg:		Extra argument.
+ */
+static long drain_ram_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct mvx_log_drain_ram *drain_ram = get_inode_private(file, 1);
+
+	switch (cmd) {
+	case MVX_LOG_IOCTL_CLEAR:
+		drain_ram->read_pos = drain_ram->write_pos;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * drain_ram_open() - Open file handle function.
+ * @inode:		The inode associated with the file.
+ * @file:		Pointer to the opened file.
+ *
+ * Return: 0 Always succeeds.
+ */
+static int drain_ram_open(struct inode *inode, struct file *file)
+{
+	struct mvx_log_drain_ram *drain_ram = get_inode_private(file, 1);
+
+	file->f_pos = drain_ram->read_pos;
+
+	return 0;
+}
+
+/******************************************************************************
+ * External interface
+ ******************************************************************************/
+
+int mvx_log_construct(struct mvx_log *log, const char *entry_name)
+{
+	int ret;
+	static const struct file_operations readme_fops = {.read = readme_read};
+	struct dentry *dentry;
+
+	if (!IS_ENABLED(CONFIG_DEBUG_FS)) {
+		pr_info("MVX: Debugfs is not enabled. '%s' dir is not created.\n", entry_name);
+		return 0;
+	}
+
+	log->mvx_dir = debugfs_create_dir(entry_name, NULL);
+	if (IS_ERR_OR_NULL(log->mvx_dir)) {
+		pr_err("MVX: Failed to create '%s' dir.\n", entry_name);
+		return -ENOMEM;
+	}
+
+	log->log_dir = debugfs_create_dir("log", log->mvx_dir);
+	if (IS_ERR_OR_NULL(log->log_dir)) {
+		pr_err("MVX: Failed to create 'log' dir.\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	log->log_dir->d_inode->i_private = log;
+
+	log->drain_dir = debugfs_create_dir("drain", log->log_dir);
+	if (IS_ERR_OR_NULL(log->drain_dir)) {
+		pr_err("MVX: Failed to create 'drain' dir.\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	log->group_dir = debugfs_create_dir("group", log->log_dir);
+	if (IS_ERR_OR_NULL(log->group_dir)) {
+		pr_err("MVX: Failed to create 'group' dir.\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	/* Create <group>/drain. */
+	dentry = debugfs_create_file("README", 0400, log->log_dir, NULL, &readme_fops);
+	if (IS_ERR_OR_NULL(dentry)) {
+		pr_err("MVX: Failed to create 'README'.\n");
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	return 0;
+
+error:
+	debugfs_remove_recursive(log->mvx_dir);
+	return ret;
+}
+
+void mvx_log_destruct(struct mvx_log *log)
+{
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		debugfs_remove_recursive(log->mvx_dir);
+}
+
+/******************************************************************************
+ * Log Drain
+ ******************************************************************************/
+
+static int drain_construct(struct mvx_log_drain *drain, mvx_print_fptr print, mvx_data_fptr data)
+{
+	drain->print = print;
+	drain->data = data;
+
+	return 0;
+}
+
+static void drain_destruct(struct mvx_log_drain *drain) { UNUSED(drain); }
+
+static void drain_dmesg_print(struct mvx_log_drain *drain, enum mvx_log_severity severity, const char *tag,
+			      const char *msg, const unsigned int n_args, ...)
+{
+	va_list args;
+	char fmt[500];
+
+	severity = min_t(int, severity, MVX_LOG_VERBOSE);
+
+	snprintf(fmt, sizeof(fmt), "%s%s: %s\n", severity_to_kern_level[severity], tag, msg);
+	fmt[sizeof(fmt) - 1] = '\0';
+
+	va_start(args, n_args);
+	vprintk(fmt, args);
+	va_end(args);
+}
+
+static void drain_dmesg_data(struct mvx_log_drain *drain, enum mvx_log_severity severity, struct iovec *vec,
+			     size_t count)
+{
+	size_t i;
+
+	pr_info("count=%zu\n", count);
+
+	for (i = 0; i < count; ++i) {
+		const char *p = vec[i].iov_base;
+		size_t length = vec[i].iov_len;
+
+		pr_info("  length=%zu\n", length);
+
+		while (length > 0) {
+			size_t j = min_t(size_t, length, 32);
+			size_t buf_size = 3 + j * 3 + 1;
+			char buf[3 + 32 * 3 + 1];
+			size_t n = 0;
+
+			length -= j;
+
+			n += scnprintf(&buf[n], (buf_size)-n, "   ");
+
+			while (j-- > 0)
+				n += scnprintf(&buf[n], (buf_size)-n, " %02x", *p++);
+
+			pr_info("%s\n", buf);
+		}
+	}
+}
+
+int mvx_log_drain_dmesg_construct(struct mvx_log_drain *drain)
+{
+	return drain_construct(drain, drain_dmesg_print, drain_dmesg_data);
+}
+
+void mvx_log_drain_dmesg_destruct(struct mvx_log_drain *drain) { drain_destruct(drain); }
+
+int mvx_log_drain_add(struct mvx_log *log, const char *name, struct mvx_log_drain *drain)
+{
+	if (!IS_ENABLED(CONFIG_DEBUG_FS)) {
+		pr_info("MVX: Debugfs is not enabled. '%s' dir is not created.\n", name);
+		return 0;
+	}
+
+	/* Create <drain> directory. */
+	drain->dentry = debugfs_create_dir(name, log->drain_dir);
+	if (IS_ERR_OR_NULL(drain->dentry)) {
+		pr_err("MVX: Failed to create '%s' dir.\n", name);
+		return -ENOMEM;
+	}
+
+	/* Store pointer to drain object in inode private data. */
+	drain->dentry->d_inode->i_private = drain;
+
+	return 0;
+}
+
+static void drain_ram_data(struct mvx_log_drain *drain, enum mvx_log_severity severity, struct iovec *vec, size_t count)
+{
+	struct mvx_log_drain_ram *drain_ram = (struct mvx_log_drain_ram *)drain;
+	size_t i;
+	size_t length;
+	size_t pos;
+	int sem_taken;
+
+	if (!IS_ENABLED(CONFIG_DEBUG_FS))
+		return;
+
+	/* Calculate the total length of the output. */
+	for (i = 0, length = 0; i < count; ++i)
+		length += vec[i].iov_len;
+
+	/* Round up to next 32-bit boundary. */
+	length = (length + 3) & ~3;
+
+	if (length > drain_ram->buffer_size) {
+		pr_err("MVX: Logged data larger than output buffer. length=%zu, buffer_length=%zu.\n", length,
+		       (size_t)drain_ram->buffer_size);
+		return;
+	}
+
+	sem_taken = down_interruptible(&drain_ram->sem);
+
+	pos = drain_ram->write_pos & (drain_ram->buffer_size - 1);
+
+	/* Loop over scatter input. */
+	for (i = 0; i < count; ++i) {
+		const char *buf = vec[i].iov_base;
+		size_t len = vec[i].iov_len;
+
+		/* Copy log message to output buffer. */
+		while (len > 0) {
+			size_t n = min(len, drain_ram->buffer_size - pos);
+
+			memcpy(&drain_ram->buf[pos], buf, n);
+
+			len -= n;
+			buf += n;
+			pos = (pos + n) & (drain_ram->buffer_size - 1);
+		}
+	}
+
+	/* Update write_pos. Length has already been 4 byte aligned */
+	drain_ram->write_pos += length;
+
+	if (sem_taken == 0)
+		up(&drain_ram->sem);
+
+	wake_up_interruptible(&drain_ram->queue);
+}
+
+static void drain_ram_print(struct mvx_log_drain *drain, enum mvx_log_severity severity, const char *tag,
+			    const char *msg, const unsigned int n_args, ...)
+{
+	char buf[500];
+	va_list args;
+	size_t n = 0;
+	struct mvx_log_header header;
+	struct iovec vec[2];
+	struct timespec64 timespec;
+
+	if (!IS_ENABLED(CONFIG_DEBUG_FS))
+		return;
+
+	/* Write the log message. */
+	va_start(args, n_args);
+	n += vscnprintf(buf, sizeof(buf), msg, args);
+	va_end(args);
+
+	ktime_get_real_ts64(&timespec);
+
+	header.magic = MVX_LOG_MAGIC;
+	header.length = n;
+	header.type = MVX_LOG_TYPE_TEXT;
+	header.severity = severity;
+	header.timestamp.sec = timespec.tv_sec;
+	header.timestamp.nsec = timespec.tv_nsec;
+
+	vec[0].iov_base = &header;
+	vec[0].iov_len = sizeof(header);
+
+	vec[1].iov_base = buf;
+	vec[1].iov_len = n;
+
+	drain_ram_data(drain, severity, vec, 2);
+}
+
+int mvx_log_drain_ram_construct(struct mvx_log_drain_ram *drain, size_t buffer_size)
+{
+	int ret;
+
+	ret = drain_construct(&drain->base, drain_ram_print, drain_ram_data);
+	if (ret != 0)
+		return ret;
+
+	if (!IS_ENABLED(CONFIG_DEBUG_FS)) {
+		pr_info("MVX: No Debugfs no RAM drain.\n");
+		return 0;
+	}
+
+	drain->buf = vmalloc(buffer_size);
+	if (drain->buf == NULL)
+		return -ENOMEM;
+
+	*(size_t *)&drain->buffer_size = buffer_size;
+	drain->read_pos = 0;
+	drain->write_pos = 0;
+	init_waitqueue_head(&drain->queue);
+	sema_init(&drain->sem, 1);
+
+	return 0;
+}
+
+void mvx_log_drain_ram_destruct(struct mvx_log_drain_ram *drain)
+{
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		vfree(drain->buf);
+
+	drain_destruct(&drain->base);
+}
+
+int mvx_log_drain_ram_add(struct mvx_log *log, const char *name, struct mvx_log_drain_ram *drain)
+{
+	static const struct file_operations drain_ram_msg = {.read = drain_ram_read_msg,
+							     .poll = drain_ram_msg_poll,
+							     .open = drain_ram_open,
+							     .unlocked_ioctl = drain_ram_ioctl};
+	struct dentry *dentry;
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_DEBUG_FS)) {
+		pr_info("MVX: Debugfs is not enabled. RAM drain dirs are not created.\n");
+		return 0;
+	}
+
+	ret = mvx_log_drain_add(log, name, &drain->base);
+	if (ret != 0)
+		return ret;
+
+	/* Create dentry. */
+	dentry = debugfs_create_file("msg", 0600, drain->base.dentry, NULL, &drain_ram_msg);
+	if (IS_ERR_OR_NULL(dentry)) {
+		pr_err("MVX: Failed to create '%s/msg.\n", name);
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	return 0;
+
+error:
+	debugfs_remove_recursive(drain->base.dentry);
+
+	return ret;
+}
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+static void drain_ftrace_print(struct mvx_log_drain *drain, enum mvx_log_severity severity, const char *tag,
+			       const char *msg, const unsigned int n_args, ...)
+{
+	va_list args;
+	char fmt[500];
+
+	severity = min_t(int, severity, MVX_LOG_VERBOSE);
+
+	snprintf(fmt, sizeof(fmt), "%s %s: %s\n", severity_to_name[severity], tag, msg);
+	fmt[sizeof(fmt) - 1] = '\0';
+
+	va_start(args, n_args);
+	ftrace_vprintk(fmt, args);
+	va_end(args);
+}
+
+static void drain_ftrace_data(struct mvx_log_drain *drain, enum mvx_log_severity severity, struct iovec *vec,
+			      size_t count)
+{
+	size_t i;
+
+	trace_printk("count=%zu\n", count);
+
+	for (i = 0; i < count; ++i) {
+		const char *p = vec[i].iov_base;
+		size_t length = vec[i].iov_len;
+
+		trace_printk("  length=%zu\n", length);
+
+		while (length > 0) {
+			size_t j = min_t(size_t, length, 32);
+			char buf[3 + j * 3 + 1];
+			size_t n = 0;
+
+			length -= j;
+
+			n += scnprintf(&buf[n], sizeof(buf) - n, "   ");
+
+			while (j-- > 0)
+				n += scnprintf(&buf[n], sizeof(buf) - n, " %02x", *p++);
+
+			trace_printk("%s\n", buf);
+		}
+	}
+}
+
+int mvx_log_drain_ftrace_construct(struct mvx_log_drain *drain)
+{
+	return drain_construct(drain, drain_ftrace_print, drain_ftrace_data);
+}
+
+void mvx_log_drain_ftrace_destruct(struct mvx_log_drain *drain) { drain_destruct(drain); }
+
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+/******************************************************************************
+ * Log Group
+ ******************************************************************************/
+
+void mvx_log_group_construct(struct mvx_log_group *group, const char *tag, const enum mvx_log_severity severity,
+			     struct mvx_log_drain *drain)
+{
+	group->tag = tag;
+	group->severity = severity;
+	group->drain = drain;
+}
+
+int mvx_log_group_add(struct mvx_log *log, const char *name, struct mvx_log_group *group)
+{
+	static const struct file_operations group_drain_fops = {.read = group_drain_read, .write = group_drain_write};
+	struct dentry *dentry;
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_DEBUG_FS)) {
+		pr_info("MVX: Debugfs is not enabled. '%s' dir is not created.\n", name);
+		return 0;
+	}
+
+	/* Create <group> directory. */
+	group->dentry = debugfs_create_dir(name, log->group_dir);
+	if (IS_ERR_OR_NULL(group->dentry)) {
+		pr_err("MVX: Failed to create '%s' dir.\n", name);
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	/* Store reference to group object in inode private data. */
+	group->dentry->d_inode->i_private = group;
+
+	/* Create <group>/severity. */
+	debugfs_create_u32("severity", 0600, group->dentry, &group->severity);
+
+	/* Create <group>/drain. */
+	dentry = debugfs_create_file("drain", 0600, group->dentry, NULL, &group_drain_fops);
+	if (IS_ERR_OR_NULL(dentry)) {
+		pr_err("MVX: Failed to create '%s/severity' value.\n", name);
+		ret = -ENOMEM;
+		goto error;
+	}
+
+	return 0;
+
+error:
+	mvx_log_group_destruct(group);
+	return ret;
+}
+
+void mvx_log_group_destruct(struct mvx_log_group *group) { UNUSED(group); }
+
+const char *mvx_log_strrchr(const char *s)
+{
+	const char *p = strrchr(s, '/');
+
+	return (p == NULL) ? s : p + 1;
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_log.h b/drivers/media/platform/spacemit/vpu_k3/mvx_log.h
new file mode 100644
index 000000000000..23dfcf125bea
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_log.h
@@ -0,0 +1,361 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef MVX_LOG_H
+#define MVX_LOG_H
+
+/******************************************************************************
+ * Includes
+ ******************************************************************************/
+
+#include <linux/net.h>
+#include <linux/semaphore.h>
+#include <linux/types.h>
+#include <linux/uio.h>
+#include <linux/wait.h>
+
+/******************************************************************************
+ * Defines
+ ******************************************************************************/
+
+/**
+ * Print a log message.
+ *
+ * @_lg:		Pointer to log group.
+ * @_severity:		Severity.
+ * @_fmt:		Format string.
+ */
+#define MVX_LOG_PRINT(_lg, _severity, _fmt, ...)                                                                       \
+	do {                                                                                                           \
+		if ((_severity) <= (_lg)->severity) {                                                                  \
+			__MVX_LOG_PRINT(_lg, _severity, _fmt, ##__VA_ARGS__);                                          \
+		}                                                                                                      \
+	} while (0)
+
+/**
+ * Print a log message for a session.
+ *
+ * @_lg:		Pointer to log group.
+ * @_severity:		Severity.
+ * @_session:		Pointer to session.
+ * @_fmt:		Format string.
+ */
+#define MVX_LOG_PRINT_SESSION(_lg, _severity, _session, _fmt, ...)                                                     \
+	do {                                                                                                           \
+		if ((_severity) <= (_lg)->severity) {                                                                  \
+			__MVX_LOG_PRINT(_lg, _severity, "%px " _fmt, _session, ##__VA_ARGS__);                         \
+		}                                                                                                      \
+	} while (0)
+
+/**
+ * Print binary data.
+ *
+ * @_lg:		Pointer to log group.
+ * @_severity:		Severity.
+ * @_vec:		Scatter input vector data.
+ * @_count:		_vec array size.
+ */
+#define MVX_LOG_DATA(_lg, _severity, _vec, _count)                                                                     \
+	do {                                                                                                           \
+		if ((_severity) <= (_lg)->severity) {                                                                  \
+			(_lg)->drain->data((_lg)->drain, _severity, _vec, _count);                                     \
+		}                                                                                                      \
+	} while (0)
+
+/**
+ * Check if severity level for log group is enabled.
+ *
+ * @_lg:		Pointer to log group.
+ * @_severity:		Severity.
+ */
+#define MVX_LOG_ENABLED(_lg, _severity) ((_severity) <= (_lg)->severity)
+
+/**
+ * Execute function if log group is enabled.
+ *
+ * @_lg:		Pointer to log group.
+ * @_severity:		Severity.
+ * @_exec:		The function to be executed.
+ */
+#define MVX_LOG_EXECUTE(_lg, _severity, _exec)                                                                         \
+	do {                                                                                                           \
+		if (MVX_LOG_ENABLED(_lg, _severity)) {                                                                 \
+			_exec;                                                                                         \
+		}                                                                                                      \
+	} while (0)
+
+#ifdef MVX_LOG_PRINT_FILE_ENABLE
+#define __MVX_LOG_PRINT(_lg, _severity, _fmt, ...)                                                                     \
+	((_lg)->drain->print((_lg)->drain, _severity, (_lg)->tag, _fmt " (%s:%d)", __MVX_LOG_N_ARGS(__VA_ARGS__),      \
+			     ##__VA_ARGS__, mvx_log_strrchr(__FILE__), __LINE__))
+#else
+#define __MVX_LOG_PRINT(_lg, _severity, _fmt, ...)                                                                     \
+	((_lg)->drain->print((_lg)->drain, _severity, (_lg)->tag, _fmt, __MVX_LOG_N_ARGS(__VA_ARGS__), ##__VA_ARGS__))
+#endif /* MVX_LOG_PRINT_FILE_ENABLE */
+
+#define __MVX_LOG_N_ARGS(...) __MVX_LOG_COUNT(dummy, ##__VA_ARGS__, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+
+#define __MVX_LOG_COUNT(_0, _1, _2, _3, _4, _5, _6, _7, _8, N, ...) N
+
+/******************************************************************************
+ * Types
+ ******************************************************************************/
+
+/**
+ * enum mvx_log_severity - Severity levels.
+ */
+enum mvx_log_severity {
+	MVX_LOG_PANIC,
+	MVX_LOG_ERROR,
+	MVX_LOG_WARNING,
+	MVX_LOG_INFO,
+	MVX_LOG_DEBUG,
+	MVX_LOG_VERBOSE,
+	MVX_LOG_MAX
+};
+
+struct mvx_log_drain;
+
+/**
+ * mvx_print_fptr() - Function pointer to output text messages.
+ *
+ * @drain:		Pointer to drain.
+ * @severity:		Severity level.
+ * @tag:		Log group tag.
+ * @fmt:		Format string.
+ * @n_args:		Number of arguments to format string.
+ */
+typedef void (*mvx_print_fptr)(struct mvx_log_drain *drain, enum mvx_log_severity severity, const char *tag,
+			       const char *fmt, const unsigned int n_args, ...);
+
+/**
+ * mvx_data_fptr() - Function pointer to output binary data.
+ *
+ * @drain:		Pointer to drain.
+ * @severity:		Severity level.
+ * @vec:		Pointer to the buffers that are copied.
+ * @count:		The number of vec buffers.
+ */
+typedef void (*mvx_data_fptr)(struct mvx_log_drain *drain, enum mvx_log_severity severity, struct iovec *vec,
+			      size_t count);
+
+/**
+ * struct mvx_log_drain - Structure with information about the drain. The drain
+ *			  handles the formatting and redirection of the log
+ *			  messages.
+ * @print:		Print function pointer.
+ * @data:		Data function pointer.
+ * @dentry:		Debugfs dentry.
+ */
+struct mvx_log_drain {
+	mvx_print_fptr print;
+	mvx_data_fptr data;
+	struct dentry *dentry;
+};
+
+/**
+ * struct mvx_log_drain_ram - Structure describing a specialized RAM drain.
+ * @base:		Base class.
+ * @buf:		Pointer to output buffer.
+ * @buffer_size:	Size of the buffer. Must be power of 2.
+ * @read_pos:		Read position when a new file handle is opened. Is
+ *			updated when the buffer is cleared.
+ * @write_pos:		Current write position in RAM buffer.
+ * @queue:		Wait queue for blocking IO.
+ * @sem:		Semaphore to prevent concurrent writes.
+ */
+struct mvx_log_drain_ram {
+	struct mvx_log_drain base;
+	char *buf;
+	const size_t buffer_size;
+	size_t read_pos;
+	size_t write_pos;
+	wait_queue_head_t queue;
+	struct semaphore sem;
+};
+
+/**
+ * struct mvx_log_group - Structure describing log group. The log group filters
+ *			  which log messages that shall be forwarded to the
+ *			  drain.
+ * @tag:		Name of log group.
+ * @severity:		Severity level.
+ * @drain:		Drain.
+ * @dentry:		Debugfs dentry.
+ */
+struct mvx_log_group {
+	const char *tag;
+	enum mvx_log_severity severity;
+	struct mvx_log_drain *drain;
+	struct dentry *dentry;
+};
+
+/**
+ * struct mvx_log - Log class that keeps track of registered groups and drains.
+ */
+struct mvx_log {
+	struct dentry *mvx_dir;
+	struct dentry *log_dir;
+	struct dentry *drain_dir;
+	struct dentry *group_dir;
+};
+
+/****************************************************************************
+ * Log
+ ****************************************************************************/
+
+/**
+ * mvx_log_construct() - Log constructor.
+ * @log:		Pointer to log.
+ * @entry_name:		The name of the directory
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_construct(struct mvx_log *log, const char *entry_name);
+
+/**
+ * mvx_log_destruct() - Log destructor.
+ * @log:		Pointer to log.
+ */
+void mvx_log_destruct(struct mvx_log *log);
+
+/****************************************************************************
+ * Drain
+ ****************************************************************************/
+
+/**
+ * mvx_log_drain_dmesg_construct() - Dmesg drain constructor.
+ * @drain:		Pointer to drain.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_drain_dmesg_construct(struct mvx_log_drain *drain);
+
+/**
+ * mvx_log_drain_dmesg_destruct() - Dmesg drain destructor.
+ * @drain:		Pointer to drain.
+ */
+void mvx_log_drain_dmesg_destruct(struct mvx_log_drain *drain);
+
+/**
+ * mvx_log_drain_add() - Add drain to log.
+ * @log:		Pointer to log.
+ * @name:		Name of drain.
+ * @drain:		Pointer to drain.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_drain_add(struct mvx_log *log, const char *name, struct mvx_log_drain *drain);
+
+/**
+ * mvx_log_drain_ram_construct() - RAM drain constructor.
+ * @drain:		Pointer to drain.
+ * @print:		Print function pointer.
+ * @data:		Data function pointer.
+ * @buffer_size:	The size of the RAM drain buffer.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_drain_ram_construct(struct mvx_log_drain_ram *drain, size_t buffer_size);
+
+/**
+ * mvx_log_drain_ram_destruct() - RAM drain destructor.
+ * @drain:		Pointer to drain.
+ */
+void mvx_log_drain_ram_destruct(struct mvx_log_drain_ram *drain);
+
+/**
+ * mvx_log_drain_ram_add() - Derived function to add RAM drain to log.
+ * @log:		Pointer to log.
+ * @name:		Name of drain.
+ * @drain:		Pointer to drain.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_drain_ram_add(struct mvx_log *log, const char *name, struct mvx_log_drain_ram *drain);
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+
+/**
+ * mvx_log_drain_ftrace_construct() - Ftrace drain constructor.
+ * @drain:		Pointer to drain.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_drain_ftrace_construct(struct mvx_log_drain *drain);
+
+/**
+ * mvx_log_drain_ftrace_destruct() - Ftrace drain destructor.
+ * @drain:		Pointer to drain.
+ */
+void mvx_log_drain_ftrace_destruct(struct mvx_log_drain *drain);
+
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+/****************************************************************************
+ * Group
+ ****************************************************************************/
+
+/**
+ * mvx_log_group_construct() - Group constructor.
+ * @group:		Pointer to group.
+ * @tag:		Name of the group, to be used in log messages.
+ * @severity:		Minimum severity to output log message.
+ * @drain:		Pointer to drain.
+ */
+void mvx_log_group_construct(struct mvx_log_group *group, const char *tag, const enum mvx_log_severity severity,
+			     struct mvx_log_drain *drain);
+
+/**
+ * mvx_log_group_add() - Add a group with given name to log.
+ * @log:		Pointer to log.
+ * @name:		Name of group.
+ * @group:		Pointer to group.
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_group_add(struct mvx_log *log, const char *name, struct mvx_log_group *group);
+
+/**
+ * mvx_log_group_destruct() - Group destructor.
+ * @group:		Pointer to group.
+ */
+void mvx_log_group_destruct(struct mvx_log_group *group);
+
+/**
+ * mvx_log_strrchr() - Find last occurrence of '/' in string.
+ * @s:		Pointer to string.
+ *
+ * Return: Pointer to '/'+1, or pointer to begin of string.
+ */
+const char *mvx_log_strrchr(const char *s);
+
+#endif /* MVX_LOG_H */
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_log_group.c b/drivers/media/platform/spacemit/vpu_k3/mvx_log_group.c
new file mode 100644
index 000000000000..de1dfe275e5b
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_log_group.c
@@ -0,0 +1,162 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+/******************************************************************************
+ * Includes
+ ******************************************************************************/
+
+#include "mvx_log.h"
+#include "mvx_log_group.h"
+
+/******************************************************************************
+ * Private variables
+ ******************************************************************************/
+
+static struct mvx_log log;
+
+static struct mvx_log_drain drain_dmesg_if;
+static struct mvx_log_drain_ram drain_ram0_if;
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+static struct mvx_log_drain drain_ftrace_if;
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+struct mvx_log_group mvx_log_if;
+struct mvx_log_group mvx_log_fwif_if;
+struct mvx_log_group mvx_log_session_if;
+struct mvx_log_group mvx_log_dev;
+
+/******************************************************************************
+ * External interface
+ ******************************************************************************/
+
+int mvx_log_group_init(const char *entry_name)
+{
+	int ret;
+	struct mvx_log_drain *drain_default = &drain_dmesg_if;
+	struct mvx_log_drain *drain_ram = &drain_ram0_if.base;
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+	drain_default = &drain_ftrace_if;
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+	/* Construct log object. */
+	ret = mvx_log_construct(&log, entry_name);
+	if (ret != 0)
+		return ret;
+
+	/* Construct drain objects and add them to log. */
+	mvx_log_drain_dmesg_construct(&drain_dmesg_if);
+	ret = mvx_log_drain_add(&log, "dmesg", &drain_dmesg_if);
+	if (ret != 0)
+		goto delete_log_entry;
+
+	mvx_log_drain_ram_construct(&drain_ram0_if, 256 * 1024);
+	ret = mvx_log_drain_ram_add(&log, "ram0", &drain_ram0_if);
+	if (ret != 0)
+		goto delete_dmesg_drain;
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+	mvx_log_drain_ftrace_construct(&drain_ftrace_if);
+	mvx_log_drain_add(&log, "ftrace", &drain_ftrace_if);
+	if (ret != 0)
+		goto delete_ram_drain;
+
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+	/* Construct group objects. */
+	mvx_log_group_construct(&mvx_log_if, "MVX if", MVX_LOG_WARNING, drain_default);
+	ret = mvx_log_group_add(&log, "generic", &mvx_log_if);
+	if (ret != 0)
+		goto delete_ftrace_drain;
+
+	mvx_log_group_construct(&mvx_log_fwif_if, "MVX fwif", MVX_LOG_INFO, drain_ram);
+	ret = mvx_log_group_add(&log, "firmware_interface", &mvx_log_fwif_if);
+	if (ret != 0)
+		goto delete_generic_group;
+
+	mvx_log_group_construct(&mvx_log_session_if, "MVX session", MVX_LOG_WARNING, drain_default);
+	ret = mvx_log_group_add(&log, "session", &mvx_log_session_if);
+	if (ret != 0)
+		goto delete_fwif_group;
+
+	mvx_log_group_construct(&mvx_log_dev, "MVX dev", MVX_LOG_WARNING, drain_default);
+	ret = mvx_log_group_add(&log, "dev", &mvx_log_dev);
+	if (ret != 0)
+		goto delete_session_group;
+
+	return 0;
+
+delete_session_group:
+	mvx_log_group_destruct(&mvx_log_session_if);
+
+delete_fwif_group:
+	mvx_log_group_destruct(&mvx_log_fwif_if);
+
+delete_generic_group:
+	mvx_log_group_destruct(&mvx_log_if);
+
+delete_ftrace_drain:
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+	mvx_log_drain_ftrace_destruct(&drain_ftrace_if);
+
+delete_ram_drain:
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+	mvx_log_drain_ram_destruct(&drain_ram0_if);
+
+delete_dmesg_drain:
+	mvx_log_drain_dmesg_destruct(&drain_dmesg_if);
+
+delete_log_entry:
+	mvx_log_destruct(&log);
+
+	return ret;
+}
+
+void mvx_log_group_deinit(void)
+{
+	/* Destroy objects in reverse order. */
+	mvx_log_group_destruct(&mvx_log_dev);
+	mvx_log_group_destruct(&mvx_log_session_if);
+	mvx_log_group_destruct(&mvx_log_fwif_if);
+	mvx_log_group_destruct(&mvx_log_if);
+
+#ifdef MVX_LOG_FTRACE_ENABLE
+	mvx_log_drain_ftrace_destruct(&drain_ftrace_if);
+#endif /* MVX_LOG_FTRACE_ENABLE */
+
+	mvx_log_drain_ram_destruct(&drain_ram0_if);
+	mvx_log_drain_dmesg_destruct(&drain_dmesg_if);
+
+	mvx_log_destruct(&log);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_log_group.h b/drivers/media/platform/spacemit/vpu_k3/mvx_log_group.h
new file mode 100644
index 000000000000..db1d9ad9160f
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_log_group.h
@@ -0,0 +1,68 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_LOG_GROUP_H_
+#define _MVX_LOG_GROUP_H_
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "mvx_log.h"
+
+/******************************************************************************
+ * Prototypes
+ ******************************************************************************/
+
+extern struct mvx_log_group mvx_log_if;
+extern struct mvx_log_group mvx_log_fwif_if;
+extern struct mvx_log_group mvx_log_session_if;
+extern struct mvx_log_group mvx_log_dev;
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_log_group_init() - Initialize log module. This function must be called
+ *		       before any of the log groups is used.
+ * @entry_name:		The name of the directory
+ *
+ * Return: 0 on success, else error code.
+ */
+int mvx_log_group_init(const char *entry_name);
+
+/**
+ * mvx_log_group_deinit() - Destroy log module.
+ */
+void mvx_log_group_deinit(void);
+
+#endif /* _MVX_LOG_GROUP_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_log_ram.h b/drivers/media/platform/spacemit/vpu_k3/mvx_log_ram.h
new file mode 100644
index 000000000000..3935926b0006
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_log_ram.h
@@ -0,0 +1,207 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef MVX_LOG_RAM_H
+#define MVX_LOG_RAM_H
+
+/******************************************************************************
+ * Includes
+ ******************************************************************************/
+
+#ifndef __KERNEL__
+#include <stdint.h>
+#include <time.h>
+#include <sys/time.h>
+#else
+#include <linux/types.h>
+#include <linux/time.h>
+#endif
+
+/******************************************************************************
+ * Defines
+ ******************************************************************************/
+
+/**
+ * Magic word "MVXL" that prefix all messages.
+ *
+ * Messages are stored in native byte order. The magic word can be used to
+ * detect if the log has been stored in the same byte order as the application
+ * unpacking the log is using.
+ */
+#define MVX_LOG_MAGIC 0x4d56584c
+
+/**
+ * The maximum message length.
+ */
+#define MVX_LOG_MESSAGE_LENGTH_MAX 4096
+
+/******************************************************************************
+ * Types
+ ******************************************************************************/
+
+/**
+ * enum mvx_log_ioctl - IOCTL commands.
+ * @MVX_LOG_IOCTL_CLEAR:	Clear the log.
+ */
+enum mvx_log_ioctl {
+	MVX_LOG_IOCTL_CLEAR
+};
+
+/**
+ * enum mvx_log_type - Message type. The definitions are assigned values that
+ *		       are not allowed to change.
+ */
+enum mvx_log_type {
+	MVX_LOG_TYPE_TEXT      = 0,
+	MVX_LOG_TYPE_FWIF      = 1,
+	MVX_LOG_TYPE_FW_BINARY = 2,
+	MVX_LOG_TYPE_MAX
+};
+
+/**
+ * struct mvx_log_timeval - Portable time value format.
+ * @sec:		Seconds since 1970-01-01, Unix time epoch.
+ * @nsec:		Nano seconds.
+ */
+struct mvx_log_timeval {
+	uint64_t sec;
+	uint64_t nsec;
+} __attribute__((packed));
+
+/**
+ * struct mvx_log_header - Common header for all messages stored in RAM buffer.
+ * @magic:		Magic word.
+ * @length:		Length of message, excluding this header.
+ * @type:		Message type.
+ * @severity:		Message severity.
+ * @timestamp:		Time stamp.
+ */
+struct mvx_log_header {
+	uint32_t magic;
+	uint16_t length;
+	uint8_t type;
+	uint8_t severity;
+	struct mvx_log_timeval timestamp;
+} __attribute__((packed));
+
+/******************************************************************************
+ * Text message
+ ******************************************************************************/
+
+/**
+ * struct mvx_log_text - ASCII text message.
+ * @message[0]:		ASCII text message.
+ *
+ * The message shall be header.length long and should end with a standard ASCII
+ * character. The parser of the log will add new line and null terminate
+ * the string.
+ */
+struct mvx_log_text {
+	char message[0];
+} __attribute__((packed));
+
+/******************************************************************************
+ * Firmware interface
+ ******************************************************************************/
+
+/**
+ * enum mvx_log_fwif_channel - Firmware interface message types.
+ */
+enum mvx_log_fwif_channel {
+	MVX_LOG_FWIF_CHANNEL_MESSAGE,
+	MVX_LOG_FWIF_CHANNEL_INPUT_BUFFER,
+	MVX_LOG_FWIF_CHANNEL_OUTPUT_BUFFER,
+	MVX_LOG_FWIF_CHANNEL_RPC
+};
+
+/**
+ * enum mvx_log_fwif_direction - Firmware interface message types.
+ */
+enum mvx_log_fwif_direction {
+	MVX_LOG_FWIF_DIRECTION_HOST_TO_FIRMWARE,
+	MVX_LOG_FWIF_DIRECTION_FIRMWARE_TO_HOST
+};
+
+/**
+ * enum mvx_log_fwif_code - Special message codes for message types not defined
+ *			    by the firmware interface.
+ */
+enum mvx_log_fwif_code {
+	MVX_LOG_FWIF_CODE_STAT = 16000
+};
+
+/**
+ * struct mvx_log_fwif - Firmware interface header type.
+ * @version_minor:		Protocol version.
+ * @version_major:		Protocol version.
+ * @channel:			@see enum mvx_log_fwif_channel.
+ * @direction:			@see enum mvx_log_fwif_direction.
+ * @session:			Session id.
+ * @data[0]:			Data following the firmware interface message
+ *				header.
+ */
+struct mvx_log_fwif {
+	uint8_t version_minor;
+	uint8_t version_major;
+	uint8_t channel;
+	uint8_t direction;
+	uint64_t session;
+	uint8_t data[0];
+} __attribute__((packed));
+
+/**
+ * struct mvx_log_fwif_stat - Firmware interface statistics.
+ * @handle:		Buffer handle.
+ * @queued:		Number of buffers currently queued to the firmware.
+ */
+struct mvx_log_fwif_stat {
+	uint64_t handle;
+	uint32_t queued;
+} __attribute__((packed));
+
+/******************************************************************************
+ * Firmware binary header
+ ******************************************************************************/
+
+/**
+ * struct mvx_log_fw_binary - Firmware binary header.
+ * @session:		Session id.
+ * @data[0]:		Firmware binary, byte 0..length.
+ *
+ * The first ~100 bytes of the firmware binary contain information describing
+ * the codec.
+ */
+struct mvx_log_fw_binary {
+	uint64_t session;
+	uint8_t data[0];
+};
+
+#endif /* MVX_LOG_RAM_H */
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_pm_runtime.c b/drivers/media/platform/spacemit/vpu_k3/mvx_pm_runtime.c
new file mode 100644
index 000000000000..6a49a727ae5d
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_pm_runtime.c
@@ -0,0 +1,87 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <linux/pm_runtime.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include "mvx_log_group.h"
+#include "mvx_dev.h"
+#include "mvx_pm_runtime.h"
+
+int mvx_pm_runtime_get_sync(struct device *dev)
+{
+#ifdef CONFIG_PM
+	int ret;
+	struct mvx_dev_ctx *ctx;
+	ctx = dev_get_drvdata(dev);
+
+	mutex_lock(&ctx->pm_mutex);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0)
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "PM runtime get sync failed! ret=%d", ret);
+
+	mutex_unlock(&ctx->pm_mutex);
+	return ret;
+#else  /* !CONFIG_PM  */
+	return 1;
+#endif /* CONFIG_PM */
+}
+
+int mvx_pm_runtime_put_sync(struct device *dev)
+{
+#ifdef CONFIG_PM
+	int ret;
+	int retry_count = 10;
+	struct mvx_dev_ctx *ctx;
+	ctx = dev_get_drvdata(dev);
+
+	mutex_lock(&ctx->pm_mutex);
+	ret = pm_runtime_put_sync(dev);
+	if (ret < 0) {
+		MVX_LOG_PRINT(&mvx_log_dev, MVX_LOG_WARNING, "PM runtime put sync failed! ret=%d", ret);
+		while (ret == -EAGAIN && retry_count > 0) {
+			msleep(20);
+			pm_runtime_get_noresume(dev);
+			ret = pm_runtime_put_sync(dev);
+			MVX_LOG_PRINT(
+				&mvx_log_dev, MVX_LOG_WARNING,
+				"PM runtime put sync return EAGAIN. try to put sync again. ret=%d, retry_count=%d", ret,
+				retry_count);
+			retry_count--;
+		}
+	}
+
+	mutex_unlock(&ctx->pm_mutex);
+	return ret;
+#else  /* !CONFIG_PM  */
+	return 0;
+#endif /* CONFIG_PM */
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_pm_runtime.h b/drivers/media/platform/spacemit/vpu_k3/mvx_pm_runtime.h
new file mode 100644
index 000000000000..10640043b72a
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_pm_runtime.h
@@ -0,0 +1,67 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_PM_RUNTIME_H_
+#define _MVX_PM_RUNTIME_H_
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+
+/****************************************************************************
+ * Exported functions
+ ****************************************************************************/
+
+/**
+ * mvx_pm_runtime_get_sync() - The same function as pm_runtime_get_sync(), but
+ *			       with the addon that it prints a log line when
+ *			       error happens.
+ * @dev:	Pointer to device.
+ *
+ * Return: 0 on success, 1 if already 'active', else error code.
+ */
+int mvx_pm_runtime_get_sync(struct device *dev);
+
+/**
+ * mvx_pm_runtime_put_sync() - The same function as pm_runtime_put_sync(), but
+ *			       with the addon that it prints a log line when
+ *			       error happens.
+ *			       It will not return error if CONFIG_PM is
+ *			       undefined.
+ * @dev:	Pointer to device.
+ *
+ * Return: 0 on success, 1 if already 'suspended', else error code.
+ */
+int mvx_pm_runtime_put_sync(struct device *dev);
+
+#endif /* _MVX_PM_RUNTIME_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_seq.c b/drivers/media/platform/spacemit/vpu_k3/mvx_seq.c
new file mode 100644
index 000000000000..27f49fd5ae90
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_seq.c
@@ -0,0 +1,90 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/seq_file.h>
+#include "mvx_seq.h"
+
+struct mvx_seq_hash_it *mvx_seq_hash_start(struct device *dev, struct hlist_head *head, size_t size, loff_t pos)
+{
+	struct mvx_seq_hash_it *it;
+	size_t i;
+
+	it = devm_kzalloc(dev, sizeof(*it), GFP_KERNEL);
+	if (it == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	it->dev = dev;
+	for (i = 0; i < size; ++i) {
+		it->i = i;
+		hlist_for_each(it->node, &head[i])
+		{
+			if (pos-- == 0)
+				return it;
+		}
+	}
+
+	devm_kfree(dev, it);
+	return NULL;
+}
+
+struct mvx_seq_hash_it *mvx_seq_hash_next(void *v, struct hlist_head *head, size_t size, loff_t *pos)
+{
+	struct mvx_seq_hash_it *it = v;
+
+	++*pos;
+	it->node = it->node->next;
+
+	if (it->node != NULL)
+		return it;
+
+	do {
+		++it->i;
+	} while ((it->i < size) && hlist_empty(&head[it->i]));
+
+	if (it->i == size) {
+		devm_kfree(it->dev, it);
+		return NULL;
+	}
+
+	it->node = head[it->i].first;
+	return it;
+}
+
+void mvx_seq_hash_stop(void *v)
+{
+	struct mvx_seq_hash_it *it = v;
+
+	if (it == NULL)
+		return;
+
+	devm_kfree(it->dev, it);
+}
diff --git a/drivers/media/platform/spacemit/vpu_k3/mvx_seq.h b/drivers/media/platform/spacemit/vpu_k3/mvx_seq.h
new file mode 100644
index 000000000000..ca74ec87cc46
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/mvx_seq.h
@@ -0,0 +1,87 @@
+/*
+ * The confidential and proprietary information contained in this file may
+ * only be used by a person authorised under and to the extent permitted
+ * by a subsisting licensing agreement from Arm Technology (China) Co., Ltd.
+ *
+ *            (C) COPYRIGHT 2021-2021 Arm Technology (China) Co., Ltd.
+ *                ALL RIGHTS RESERVED
+ *
+ * This entire notice must be reproduced on all copies of this file
+ * and copies of this file may only be made by a person if such person is
+ * permitted to do so under the terms of a subsisting license agreement
+ * from Arm Technology (China) Co., Ltd.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#ifndef _MVX_SEQ_H_
+#define _MVX_SEQ_H_
+
+/****************************************************************************
+ * Defines
+ ****************************************************************************/
+
+#define mvx_seq_printf(s, tag, ind, fmt, ...)                                                                          \
+	seq_printf(s, "%-*s%-*s: " fmt, (3 * (ind)), "", 30 - (3 * (ind)), tag, ##__VA_ARGS__)
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+/**
+ * struct mvx_seq_hash_it - Iterator over hash table.
+ */
+struct mvx_seq_hash_it {
+	struct hlist_node *node;
+	size_t i;
+	struct device *dev;
+};
+
+/**
+ * mvx_seq_hash_start() - Initialize iterator.
+ * @dev:	Pointer to device.
+ * @head:	Pointer to a head of a hash table.
+ * @size:	Size of a hash table.
+ * @pos:	Position to start.
+ *
+ * Iterator created by this function should be provided to
+ * mvx_seq_hash_start and mvx_seq_hash_stop as the first parameter.
+ *
+ * Return: Pointer to an iterator on success or ERR_PTR().
+ */
+struct mvx_seq_hash_it *mvx_seq_hash_start(struct device *dev, struct hlist_head *head, size_t size, loff_t pos);
+
+/**
+ * mvx_seq_hash_next() - Move iterator to the next element.
+ * @v:		Pointer to an iterator.
+ * @head:	Pointer to a head of a hash table.
+ * @size:	Size of a hash table.
+ * @pos:	Position.
+ *
+ * Return: Iterator which points to a new element or NULL when the table
+ * is over.
+ */
+struct mvx_seq_hash_it *mvx_seq_hash_next(void *v, struct hlist_head *head, size_t size, loff_t *pos);
+
+/**
+ * mvx_seq_hash_stop() - Close an iterator.
+ * @v:		Pointer to an iterator.
+ */
+void mvx_seq_hash_stop(void *v);
+
+#endif /* _MVX_SEQ_H_ */
diff --git a/drivers/media/platform/spacemit/vpu_k3/sconscript b/drivers/media/platform/spacemit/vpu_k3/sconscript
new file mode 100644
index 000000000000..3a1f19ccc1a9
--- /dev/null
+++ b/drivers/media/platform/spacemit/vpu_k3/sconscript
@@ -0,0 +1,36 @@
+import os
+
+Import('env')
+
+# Get source path to current directory.
+path = env.Dir('.').srcnode().path
+
+prints = ""
+if env['log_ftrace'] == '1':
+    prints += " CONFIG_VIDEO_LINLON_FTRACE=y"
+if env['log_print_file'] == '1':
+    prints += " CONFIG_VIDEO_LINLON_PRINT_FILE=y"
+
+if env['mode'] == 'mono':
+    targets = [os.path.join('#', path, 'amvx.ko')]
+else:
+    targets = [os.path.join('#', path, 'amvx_if.ko'), os.path.join('#', path, 'amvx_dev.ko')]
+
+extra_ccflags=""
+if 'EXTRA_CCFLAGS' in os.environ:
+    extra_ccflags = os.environ['EXTRA_CCFLAGS']
+if env['coverage']:
+    extra_ccflags += ' -fprofile-arcs -ftest-coverage'
+
+amvx = env.Command(targets, [],
+                   'make -C %s %s_%s KDIR=%s EXTRA_CCFLAGS="%s" %s' %
+                   (path, env['mode'], env['interface'], env['KDIR'], extra_ccflags, prints))
+
+# Flag to always build.
+env.AlwaysBuild(amvx)
+
+# Install kernel module in bin directory.
+env.Install(env['BIN_DIR'], [amvx])
+
+# Install user space header files
+env.Install(env['INCLUDE_DIR'], ['if/v4l2/mvx-v4l2-controls.h'])
diff --git a/drivers/media/rc/Kconfig b/drivers/media/rc/Kconfig
index 0a8aeafdb7e0..afa962dc51a4 100644
--- a/drivers/media/rc/Kconfig
+++ b/drivers/media/rc/Kconfig
@@ -472,6 +472,15 @@ config RC_XBOX_DVD
 	   To compile this driver as a module, choose M here: the module will be
 	   called xbox_remote.
 
+config IR_SPACEMIT_K1
+	tristate "SPACEMIT IR remote Recriver control"
+	depends on SOC_SPACEMIT
+	help
+	   Say Y if you want to use spacemit IR Controller
+
+	   To compile this driver as a module, choose M here: the module will be
+	   called spacemit-ir.
+
 source "drivers/media/rc/img-ir/Kconfig"
 
 endif #RC_DEVICES
diff --git a/drivers/media/rc/Makefile b/drivers/media/rc/Makefile
index 2bca6f7f07bc..72c249f2f6d4 100644
--- a/drivers/media/rc/Makefile
+++ b/drivers/media/rc/Makefile
@@ -54,3 +54,4 @@ obj-$(CONFIG_RC_ATI_REMOTE) += ati_remote.o
 obj-$(CONFIG_RC_LOOPBACK) += rc-loopback.o
 obj-$(CONFIG_RC_ST) += st_rc.o
 obj-$(CONFIG_RC_XBOX_DVD) += xbox_remote.o
+obj-$(CONFIG_IR_SPACEMIT_K1) += ir-spacemit-k1.o
diff --git a/drivers/media/rc/ir-spacemit-k1.c b/drivers/media/rc/ir-spacemit-k1.c
new file mode 100644
index 000000000000..3a0a76b6bbd2
--- /dev/null
+++ b/drivers/media/rc/ir-spacemit-k1.c
@@ -0,0 +1,303 @@
+//*SPDX-License-Identifier: GPL-2.0
+/*
+ * spacaemit ir-rx controller driver
+ *
+ * Copyright (C) 2023 SPACEMIT Limited
+ */
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <media/rc-core.h>
+
+#define SPACEMIT_IR_DEV "spacemit-ir"
+
+/*Registers*/
+#define SPACEMIT_IRC_EN 0x0
+#define SPACEMIT_IR_CLKDIV 0x4
+#define SPACEMIT_IR_NOTHR 0x8
+#define SPACEMIT_IR_IDSTATE 0xc
+#define SPACEMIT_IR_FIFO 0x10
+#define SPACEMIT_IR_STS 0x14
+#define SPACEMIT_IR_CMP 0x18
+#define SPACEMIT_IR_INTEN 0x1c
+#define SPACEMIT_IR_INTSTA 0x20
+
+#define SPACEMIT_IR_ENABLE BIT(0)
+#define STATE_IDLE BIT(0)
+#define FIFO_EMPTY BIT(30)
+#define FIFO_FULL BIT(31)
+#define IR_IRQ_EN_PULLDOWN BIT(0)
+#define IR_IRQ_EN_PULLUP BIT(1)
+#define IR_IRQ_EN_COUNT BIT(2)
+#define IR_IRQ_EN_MATCH BIT(3)
+#define IR_FIFO_DATA(val) (val&GENMASK(5,0))
+#define SPACEMIT_IR_NOISE 1
+
+/*ir interrupt status*/
+#define IR_IRQ_PULLUP IR_IRQ_EN_PULLUP
+#define IR_IRQ_PULLDOWN IR_IRQ_EN_PULLDOWN
+#define IR_IRQ_COUNT IR_IRQ_EN_COUNT
+#define IR_IRQ_MATCH IR_IRQ_EN_MATCH
+#define IRQ_CLEAR_ALL      \
+		(IR_IRQ_PULLUP   | \
+		 IR_IRQ_PULLDOWN | \
+		 IR_IRQ_COUNT    | \
+		 IR_IRQ_MATCH)
+#define IR_IRQ_ENALL IRQ_CLEAR_ALL
+
+#define IR_FREQ 100000
+#define CYCLE_TIME 1000000/IR_FREQ
+#define SPACEMIT_IR_TIMEOUT 10000
+#define IRFIFO_DEF_CMP 1
+
+struct spacemit_ir {
+	struct rc_dev *rc;
+	void __iomem *base;
+	int irq;
+	int fifo_size;
+	int clkdiv;
+	int freq;
+	const char      *map_name;
+	struct clk *clk;
+	struct reset_control *rst;
+};
+
+struct spacemit_ir_data {
+	int fifo_size;
+};
+
+static irqreturn_t spacemit_ir_irq(int irqno,void *dev_id)
+{
+	u32 irq_flag,i,rc,fifo_status;
+	u8 info;
+	struct spacemit_ir *ir = dev_id;
+	struct ir_raw_event rawir = {};
+
+	irq_flag = readl(ir->base + SPACEMIT_IR_INTSTA);
+
+	/*clear all interrupt*/
+	writel(IRQ_CLEAR_ALL,ir->base + SPACEMIT_IR_INTSTA);
+
+	/*store data from fifo*/
+	fifo_status = readl(ir->base + SPACEMIT_IR_STS);
+
+	if(!(fifo_status & FIFO_EMPTY))
+	{
+		rc = IR_FIFO_DATA(fifo_status);
+		for(i = 0 ; i < rc ; i++)
+		{
+			info = readb(ir->base + SPACEMIT_IR_FIFO);
+			rawir.pulse = !(info & 0x80);
+			rawir.duration = (info & 0x7f)*ir->rc->rx_resolution;
+			ir_raw_event_store_with_filter(ir->rc, &rawir);
+		}
+	}
+
+	/*wake up ir-event thread*/
+	ir_raw_event_handle(ir->rc);
+
+	return IRQ_HANDLED;
+}
+
+static int spacemit_ir_config(struct rc_dev *rc_dev)
+{
+	struct spacemit_ir *ir = rc_dev->priv;
+
+	/*set clkdiv*/
+	ir->clkdiv = DIV_ROUND_UP(ir->freq, IR_FREQ) - 1;
+	writel(ir->clkdiv, ir->base + SPACEMIT_IR_CLKDIV);
+
+	writel(IRQ_CLEAR_ALL,ir->base + SPACEMIT_IR_INTSTA);
+
+	/*set noise threshold*/
+	writeb(SPACEMIT_IR_NOISE, ir->base + SPACEMIT_IR_NOTHR);
+
+	writel(IRFIFO_DEF_CMP, ir->base + SPACEMIT_IR_CMP);
+
+	rc_dev->s_idle(rc_dev, true);
+
+	return 0;
+}
+
+static void spacemit_ir_set_idle(struct rc_dev *rc_dev, bool idle)
+{
+	struct spacemit_ir *ir = rc_dev->priv;
+	writel(idle, ir->base + SPACEMIT_IR_IDSTATE);
+}
+
+static int spacemit_ir_hw_init(struct device *dev)
+{
+	struct spacemit_ir *ir = dev_get_drvdata(dev);
+	int ret;
+
+	ret = reset_control_deassert(ir->rst);
+	if(ret)
+		return ret;
+
+	ret = clk_prepare_enable(ir->clk);
+	if(ret){
+		dev_err(dev,"failed to enable ir clk\n");
+		goto exit;
+	}
+
+	spacemit_ir_config(ir->rc);
+	writel(IR_IRQ_ENALL, ir->base + SPACEMIT_IR_INTEN);
+	writel(SPACEMIT_IR_ENABLE, ir->base + SPACEMIT_IRC_EN);
+	return 0;
+
+exit:
+	reset_control_assert(ir->rst);
+
+	return ret;
+}
+
+static void spacemit_ir_hw_exit(struct device *dev)
+{
+	struct spacemit_ir *ir = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(ir->clk);
+	reset_control_assert(ir->rst);
+}
+
+static int spacemit_ir_probe(struct platform_device *pdev)
+{
+	int ret = 0;
+	struct device *dev = &pdev->dev;
+	struct device_node *dn = dev->of_node;
+	struct spacemit_ir *ir;
+
+	ir = devm_kzalloc(dev, sizeof(struct spacemit_ir), GFP_KERNEL);
+	if (!ir)
+		return -ENOMEM;
+
+	ir->freq = 0;
+	ir->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(ir->clk)) {
+		dev_err(dev, "failed to get a ir clock.\n");
+		return PTR_ERR(ir->clk);
+	}
+	ir->rst = devm_reset_control_get_exclusive(dev, NULL);
+	if (IS_ERR(ir->rst))
+		return PTR_ERR(ir->rst);
+
+	/* Base clock frequency (optional) */
+	of_property_read_u32(dn, "clock-frequency", &ir->freq);
+	if(!ir->freq) {
+		ir->freq = clk_get_rate(ir->clk);
+		if(!ir->freq) {
+			dev_err(dev, "failed to get ir function clk-freq\n");
+			return -ENODEV;
+		}
+	}
+
+	dev_dbg(dev, "ir base clock frequency : %d Hz.\n", ir->freq);
+
+	ir->base = devm_platform_ioremap_resource(pdev,0);
+	if(IS_ERR(ir->base)){
+		return PTR_ERR(ir->base);
+	}
+
+	ir->rc = rc_allocate_device(RC_DRIVER_IR_RAW);
+	if (!ir->rc) {
+		dev_err(dev, "failed to allocate device\n");
+		return -ENOMEM;
+	}
+
+	ir->rc->priv = ir;
+	ir->rc->device_name = SPACEMIT_IR_DEV;
+	ir->rc->input_phys = "spacemit-ir/input0";
+	ir->rc->input_id.bustype = BUS_HOST;
+	ir->rc->input_id.vendor = 0x0001;
+	ir->rc->input_id.product = 0x0001;
+	ir->rc->input_id.version = 0x0100;
+	ir->map_name = of_get_property(dn, "linux,rc-map-name", NULL);
+	ir->rc->map_name = ir->map_name ?: RC_MAP_EMPTY;
+	ir->rc->dev.parent = dev;
+	ir->rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
+	ir->rc->rx_resolution = CYCLE_TIME;
+	ir->rc->timeout = SPACEMIT_IR_TIMEOUT;
+	ir->rc->driver_name = SPACEMIT_IR_DEV;
+	ir->rc->s_idle = spacemit_ir_set_idle;
+
+	ret = rc_register_device(ir->rc);
+	if (ret) {
+		dev_err(dev, "failed to register rc device\n");
+		goto free_dev;
+	}
+
+	platform_set_drvdata(pdev, ir);
+
+	ir->irq = platform_get_irq(pdev, 0);
+	ret = devm_request_irq(dev, ir->irq, spacemit_ir_irq, 0, SPACEMIT_IR_DEV, ir);
+	if (ret) {
+		dev_err(dev, "failed request irq\n");
+		goto free_dev;
+	}
+
+	ret = spacemit_ir_hw_init(dev);
+
+free_dev:
+	rc_free_device(ir->rc);
+
+	return ret;
+}
+
+static void spacemit_ir_remove(struct platform_device *pdev)
+{
+	struct spacemit_ir *ir = platform_get_drvdata(pdev);
+
+	rc_unregister_device(ir->rc);
+	spacemit_ir_hw_exit(&pdev->dev);
+
+	return;
+}
+
+static void spacemit_ir_shutdown(struct platform_device *pdev)
+{
+	spacemit_ir_hw_exit(&pdev->dev);
+}
+
+static int __maybe_unused spacemit_ir_suspend(struct device *dev)
+{
+	spacemit_ir_hw_exit(dev);
+	return 0;
+}
+
+static int __maybe_unused spacemit_ir_resume(struct device *dev)
+{
+	return spacemit_ir_hw_init(dev);
+}
+
+static SIMPLE_DEV_PM_OPS(spacemit_ir_pm_ops, spacemit_ir_suspend, spacemit_ir_resume);
+
+static const struct spacemit_ir_data spacemit_k1_ir_data = {
+	.fifo_size = 32,
+};
+
+static const struct of_device_id spacemit_ir_match[] = {
+	{
+		.compatible = "spacemit-k1,irc",
+		.data = &spacemit_k1_ir_data,
+	},
+	{ /* end of list */ }
+};
+
+static struct platform_driver spacemit_ir_driver = {
+	.probe = spacemit_ir_probe,
+	.remove = spacemit_ir_remove,
+	.shutdown = spacemit_ir_shutdown,
+	.driver = {
+		.name = SPACEMIT_IR_DEV,
+		.of_match_table = spacemit_ir_match,
+		.pm = &spacemit_ir_pm_ops,
+	},
+};
+
+module_platform_driver(spacemit_ir_driver);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Spacemit");
diff --git a/drivers/mfd/simple-mfd-i2c.c b/drivers/mfd/simple-mfd-i2c.c
index 0a607a1e3ca1..3c7153155a5e 100644
--- a/drivers/mfd/simple-mfd-i2c.c
+++ b/drivers/mfd/simple-mfd-i2c.c
@@ -96,6 +96,7 @@ static const struct simple_mfd_data maxim_mon_max77705 = {
 static const struct regmap_config spacemit_p1_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
+	.max_register = 0xb0,
 };
 
 static const struct mfd_cell spacemit_p1_cells[] = {
@@ -109,6 +110,23 @@ static const struct simple_mfd_data spacemit_p1 = {
 	.mfd_cell_size = ARRAY_SIZE(spacemit_p1_cells),
 };
 
+static const struct regmap_config spacemit_mpq8655_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+	.max_register = 0x21,
+};
+
+static const struct mfd_cell spacemit_mpq8655_cells[] = {
+	{ .name = "spacemit,regulator,mpq8655", },
+};
+
+static const struct simple_mfd_data spacemit_mpq8655 = {
+	.regmap_config = &spacemit_mpq8655_regmap_config,
+	.mfd_cell = spacemit_mpq8655_cells,
+	.mfd_cell_size = ARRAY_SIZE(spacemit_mpq8655_cells),
+};
+
 static const struct of_device_id simple_mfd_i2c_of_match[] = {
 	{ .compatible = "fsl,ls1028aqds-fpga" },
 	{ .compatible = "fsl,lx2160aqds-fpga" },
@@ -119,6 +137,7 @@ static const struct of_device_id simple_mfd_i2c_of_match[] = {
 	{ .compatible = "maxim,max77705-battery", .data = &maxim_mon_max77705},
 	{ .compatible = "silergy,sy7636a", .data = &silergy_sy7636a},
 	{ .compatible = "spacemit,p1", .data = &spacemit_p1, },
+	{ .compatible = "spacemit,mpq8655", .data = &spacemit_mpq8655, },
 	{}
 };
 MODULE_DEVICE_TABLE(of, simple_mfd_i2c_of_match);
diff --git a/drivers/mfd/syscon.c b/drivers/mfd/syscon.c
index ae71a2710bed..e5d5def594f6 100644
--- a/drivers/mfd/syscon.c
+++ b/drivers/mfd/syscon.c
@@ -183,7 +183,7 @@ static struct regmap *device_node_get_regmap(struct device_node *np,
 		if (create_regmap)
 			syscon = of_syscon_register(np, check_res);
 		else
-			syscon = ERR_PTR(-EINVAL);
+			syscon = ERR_PTR(-EPROBE_DEFER);
 	}
 	mutex_unlock(&syscon_list_lock);
 
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index b9c11f67315f..e0f8f3772db5 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -165,6 +165,14 @@ config ICS932S401
 	  This driver can also be built as a module. If so, the module
 	  will be called ics932s401.
 
+config RT7451_RETIMER
+	tristate "Enable RT7451 Retimer driver"
+	depends on I2C
+	help
+	  Enable support for RT7451 retimer device. The driver performs
+	  initialization sequence during probe to configure the retimer
+	  for proper operation.
+
 config ATMEL_SSC
 	tristate "Device driver for Atmel SSC peripheral"
 	depends on HAS_IOMEM && (ARCH_AT91 || COMPILE_TEST)
@@ -606,6 +614,13 @@ config NSM
 	  To compile this driver as a module, choose M here.
 	  The module will be called nsm.
 
+config SPACEMIT_TCM
+	bool "Generic tcm alloc management driver"
+	depends on SOC_SPACEMIT && SPACEMIT_HMP
+	default y
+	help
+	  This driver allows you to alloc tcm for userspace.
+
 config MARVELL_CN10K_DPI
 	tristate "Octeon CN10K DPI driver"
 	depends on PCI && PCI_IOV
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index b32a2597d246..cf10617e7eeb 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_AD525X_DPOT_SPI)	+= ad525x_dpot-spi.o
 obj-$(CONFIG_ATMEL_SSC)		+= atmel-ssc.o
 obj-$(CONFIG_DUMMY_IRQ)		+= dummy-irq.o
 obj-$(CONFIG_ICS932S401)	+= ics932s401.o
+obj-$(CONFIG_RT7451_RETIMER)	+= rt7451.o
 obj-$(CONFIG_LKDTM)		+= lkdtm/
 obj-$(CONFIG_TI_FPC202)		+= ti_fpc202.o
 obj-$(CONFIG_TIFM_CORE)       	+= tifm_core.o
@@ -68,6 +69,7 @@ obj-$(CONFIG_TMR_INJECT)	+= xilinx_tmr_inject.o
 obj-$(CONFIG_TPS6594_ESM)	+= tps6594-esm.o
 obj-$(CONFIG_TPS6594_PFSM)	+= tps6594-pfsm.o
 obj-$(CONFIG_NSM)		+= nsm.o
+obj-$(CONFIG_SPACEMIT_TCM)	+= tcm.o
 obj-$(CONFIG_MARVELL_CN10K_DPI)	+= mrvl_cn10k_dpi.o
 lan966x-pci-objs		:= lan966x_pci.o
 lan966x-pci-objs		+= lan966x_pci.dtbo.o
diff --git a/drivers/misc/rt7451.c b/drivers/misc/rt7451.c
new file mode 100644
index 000000000000..bca6da17a89e
--- /dev/null
+++ b/drivers/misc/rt7451.c
@@ -0,0 +1,216 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RT7451 retimer driver
+ *
+ * Copyright (C) 2025 SPACEMIT Micro Limited
+ */
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/property.h>
+
+#define RT7451_PRIMARY_ADDR_DEFAULT	0x13
+#define RT7451_SECONDARY_ADDR_DEFAULT	0x29
+
+#define RT7451_REG_CONFIG1		0xf8
+#define RT7451_REG_CONFIG2		0xa4
+
+#define RT7451_CONFIG1_INIT_VAL		0x16
+#define RT7451_CONFIG2_INIT_VAL		0x28
+
+struct rt7451_data {
+	u8 primary_addr;
+	u8 secondary_addr;
+};
+
+static int rt7451_write_reg(struct i2c_adapter *adap, u8 chip_addr, u8 reg, u8 val)
+{
+	struct i2c_msg msg;
+	u8 buf[2] = { reg, val };
+	int ret;
+
+	msg.addr = chip_addr;
+	msg.flags = 0;
+	msg.len = sizeof(buf);
+	msg.buf = buf;
+
+	ret = i2c_transfer(adap, &msg, 1);
+	if (ret < 0)
+		return ret;
+	if (ret != 1)
+		return -EIO;
+
+	return 0;
+}
+
+static int rt7451_read_reg(struct i2c_adapter *adap, u8 chip_addr, u8 reg, u8 *val)
+{
+	struct i2c_msg msgs[2];
+	u8 reg_buf = reg;
+	int ret;
+
+	msgs[0].addr = chip_addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 1;
+	msgs[0].buf = &reg_buf;
+
+	msgs[1].addr = chip_addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = 1;
+	msgs[1].buf = val;
+
+	ret = i2c_transfer(adap, msgs, ARRAY_SIZE(msgs));
+	if (ret < 0)
+		return ret;
+	if (ret != ARRAY_SIZE(msgs))
+		return -EIO;
+
+	return 0;
+}
+
+static int rt7451_write_and_verify(struct device *dev, struct i2c_adapter *adap,
+				   u8 chip_addr, u8 reg, u8 val)
+{
+	u8 readback;
+	int ret;
+
+	ret = rt7451_write_reg(adap, chip_addr, reg, val);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to write chip 0x%02x reg 0x%02x\n",
+				     chip_addr, reg);
+
+	usleep_range(100, 200);
+
+	ret = rt7451_read_reg(adap, chip_addr, reg, &readback);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to verify chip 0x%02x reg 0x%02x\n",
+				     chip_addr, reg);
+
+	if (readback != val) {
+		dev_err(dev,
+			"verify failed for chip 0x%02x reg 0x%02x: wrote 0x%02x, read 0x%02x\n",
+			chip_addr, reg, val, readback);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int rt7451_reg_matches(struct device *dev, struct i2c_adapter *adap,
+			      u8 chip_addr, u8 reg, u8 expected, bool *matches)
+{
+	u8 val;
+	int ret;
+
+	ret = rt7451_read_reg(adap, chip_addr, reg, &val);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to read chip 0x%02x reg 0x%02x\n",
+				     chip_addr, reg);
+
+	*matches = val == expected;
+	if (!*matches)
+		dev_dbg(dev,
+			"chip 0x%02x reg 0x%02x is 0x%02x, expected 0x%02x\n",
+			chip_addr, reg, val, expected);
+
+	return 0;
+}
+
+static int rt7451_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct i2c_adapter *adap = client->adapter;
+	struct rt7451_data *data;
+	u32 prop;
+	bool primary_ok;
+	bool secondary_ok;
+	bool wrote_primary = false;
+	int ret;
+
+	if (!i2c_check_functionality(adap, I2C_FUNC_I2C))
+		return dev_err_probe(dev, -EOPNOTSUPP, "I2C transfers are not supported\n");
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->primary_addr = client->addr;
+	if (!device_property_read_u32(dev, "primary-addr", &prop))
+		data->primary_addr = prop;
+
+	data->secondary_addr = RT7451_SECONDARY_ADDR_DEFAULT;
+	if (!device_property_read_u32(dev, "secondary-addr", &prop))
+		data->secondary_addr = prop;
+
+	i2c_set_clientdata(client, data);
+
+	ret = rt7451_reg_matches(dev, adap, data->primary_addr, RT7451_REG_CONFIG1,
+				 RT7451_CONFIG1_INIT_VAL, &primary_ok);
+	if (ret)
+		return ret;
+
+	if (!primary_ok) {
+		ret = rt7451_write_and_verify(dev, adap, data->primary_addr,
+					      RT7451_REG_CONFIG1,
+					      RT7451_CONFIG1_INIT_VAL);
+		if (ret)
+			return ret;
+		wrote_primary = true;
+	}
+
+	ret = rt7451_reg_matches(dev, adap, data->secondary_addr, RT7451_REG_CONFIG2,
+				 RT7451_CONFIG2_INIT_VAL, &secondary_ok);
+	if (ret)
+		return ret;
+
+	if (primary_ok && secondary_ok) {
+		dev_info(dev, "already initialized by firmware\n");
+		return 0;
+	}
+
+	if (!secondary_ok) {
+		if (wrote_primary)
+			usleep_range(1000, 2000);
+
+		ret = rt7451_write_and_verify(dev, adap, data->secondary_addr,
+					      RT7451_REG_CONFIG2,
+					      RT7451_CONFIG2_INIT_VAL);
+		if (ret)
+			return ret;
+	}
+
+	dev_info(dev, "initialized\n");
+
+	return 0;
+}
+
+static const struct of_device_id rt7451_of_match[] = {
+	{ .compatible = "retimer,rt7451" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, rt7451_of_match);
+
+static const struct i2c_device_id rt7451_i2c_id[] = {
+	{ "rt7451" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, rt7451_i2c_id);
+
+static struct i2c_driver rt7451_driver = {
+	.probe = rt7451_probe,
+	.driver = {
+		.name = "rt7451",
+		.of_match_table = rt7451_of_match,
+	},
+	.id_table = rt7451_i2c_id,
+};
+
+module_i2c_driver(rt7451_driver);
+
+MODULE_AUTHOR("SPACEMIT");
+MODULE_DESCRIPTION("RT7451 retimer driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/misc/tcm.c b/drivers/misc/tcm.c
new file mode 100644
index 000000000000..757864c0fe05
--- /dev/null
+++ b/drivers/misc/tcm.c
@@ -0,0 +1,327 @@
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/miscdevice.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/version.h>
+
+#define TCM_NAME		"tcm"
+
+#define IOC_MAGIC		'c'
+#define TCM_INFO_GET		_IOR(IOC_MAGIC, 7, int)
+#define TCM_BLOCK_INFO_GET	_IOWR(IOC_MAGIC, 9, int)
+
+typedef struct {
+	void *base;
+	size_t block_size;
+	size_t block_num;
+} tcm_info_t;
+
+typedef struct {
+	u32 block_id;
+	u32 reserved;
+	u64 phys;
+	u64 size;
+	u64 cpu_affinity_mask;
+} tcm_block_info_t;
+
+struct tcm_block {
+	phys_addr_t phys;
+	size_t size;
+	u64 cpu_affinity_mask;
+};
+
+struct tcm_direct_dev {
+	struct miscdevice miscdev;
+	struct device *dev;
+	phys_addr_t phys;
+	size_t size;
+	size_t block_size;
+	size_t block_num;
+	struct tcm_block *blocks;
+};
+
+static int tcm_parse_cpu_mask(struct device_node *tcm_node, u64 *mask)
+{
+	struct device_node *cpu_node;
+	u64 cpu_mask = 0;
+	int i;
+
+	if (!mask)
+		return -EINVAL;
+
+	for (i = 0;; i++) {
+		u32 cpu_reg;
+
+		cpu_node = of_parse_phandle(tcm_node, "cpus", i);
+		if (!cpu_node)
+			break;
+
+		if (!of_property_read_u32(cpu_node, "reg", &cpu_reg) &&
+		    cpu_reg < 64)
+			cpu_mask |= BIT_ULL(cpu_reg);
+
+		of_node_put(cpu_node);
+	}
+
+	*mask = cpu_mask;
+	return i == 0 ? -ENOENT : 0;
+}
+
+static int tcm_parse_layout(struct platform_device *pdev,
+			    struct tcm_direct_dev *tcm)
+{
+	struct resource *parent_res;
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *child;
+	size_t child_count;
+	size_t block_idx = 0;
+	int ret;
+
+	parent_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!parent_res)
+		return -EINVAL;
+
+	tcm->phys = parent_res->start;
+	tcm->size = resource_size(parent_res);
+	tcm->block_size = tcm->size;
+	tcm->block_num = 1;
+	tcm->blocks = NULL;
+
+	child_count = np ? of_get_available_child_count(np) : 0;
+	if (!child_count) {
+		tcm->blocks = devm_kcalloc(&pdev->dev, 1,
+					   sizeof(*tcm->blocks),
+					   GFP_KERNEL);
+		if (!tcm->blocks)
+			return -ENOMEM;
+
+		tcm->blocks[0].phys = tcm->phys;
+		tcm->blocks[0].size = tcm->size;
+		return 0;
+	}
+
+	tcm->blocks = devm_kcalloc(&pdev->dev, child_count,
+				   sizeof(*tcm->blocks), GFP_KERNEL);
+	if (!tcm->blocks)
+		return -ENOMEM;
+
+	for_each_available_child_of_node(np, child) {
+		struct resource child_res;
+		size_t child_size;
+		u64 cpu_affinity_mask = 0;
+
+		ret = of_address_to_resource(child, 0, &child_res);
+		if (ret < 0)
+			return ret;
+
+		if (child_res.start < parent_res->start ||
+		    child_res.end > parent_res->end)
+			return -EINVAL;
+
+		child_size = resource_size(&child_res);
+		if (!block_idx) {
+			tcm->phys = child_res.start;
+			tcm->block_size = child_size;
+		} else if (child_size != tcm->block_size) {
+			dev_err(&pdev->dev,
+				"direct mmap layout requires equal-sized blocks, block%zu size=0x%zx expected=0x%zx\n",
+				block_idx, child_size, tcm->block_size);
+			return -EINVAL;
+		}
+
+		tcm->blocks[block_idx].phys = child_res.start;
+		tcm->blocks[block_idx].size = child_size;
+		if (tcm_parse_cpu_mask(child, &cpu_affinity_mask) == 0)
+			tcm->blocks[block_idx].cpu_affinity_mask =
+				cpu_affinity_mask;
+		block_idx++;
+	}
+
+	tcm->block_num = block_idx;
+	tcm->size = tcm->block_size * tcm->block_num;
+	return 0;
+}
+
+static int tcm_open(struct inode *inode, struct file *file)
+{
+	struct miscdevice *miscdev = file->private_data;
+	struct tcm_direct_dev *tcm;
+
+	tcm = container_of(miscdev, struct tcm_direct_dev, miscdev);
+	file->private_data = tcm;
+	return 0;
+}
+
+static int tcm_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct tcm_direct_dev *tcm = file->private_data;
+	unsigned long vma_size = vma->vm_end - vma->vm_start;
+	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long mapped = 0;
+
+	if (!tcm)
+		return -ENODEV;
+
+	if (offset >= tcm->size)
+		return -EINVAL;
+	if (vma_size > tcm->size - offset)
+		return -EINVAL;
+
+	if (!tcm->block_size || !tcm->blocks)
+		return -ENODEV;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)
+	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
+#else
+	vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
+#endif
+
+	while (mapped < vma_size) {
+		unsigned long cur = offset + mapped;
+		size_t block_idx = cur / tcm->block_size;
+		unsigned long block_off = cur % tcm->block_size;
+		unsigned long chunk;
+		unsigned long long phys;
+
+		if (block_idx >= tcm->block_num)
+			return -EINVAL;
+		if (block_off >= tcm->blocks[block_idx].size)
+			return -EINVAL;
+
+		chunk = tcm->blocks[block_idx].size - block_off;
+		if (chunk > vma_size - mapped)
+			chunk = vma_size - mapped;
+
+		phys = tcm->blocks[block_idx].phys + block_off;
+		if (remap_pfn_range(vma, vma->vm_start + mapped,
+				    phys >> PAGE_SHIFT, chunk,
+				    vma->vm_page_prot))
+			return -EAGAIN;
+
+		mapped += chunk;
+	}
+
+	return 0;
+}
+
+static long tcm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct tcm_direct_dev *tcm = file->private_data;
+
+	if (!tcm)
+		return -ENODEV;
+
+	if (cmd == TCM_INFO_GET) {
+		tcm_info_t info;
+
+		info.base = (void *)(uintptr_t)tcm->phys;
+		info.block_size = tcm->block_size;
+		info.block_num = tcm->block_num;
+
+		if (copy_to_user((void __user *)arg, &info, sizeof(info)))
+			return -EFAULT;
+
+		return 0;
+	}
+
+	if (cmd == TCM_BLOCK_INFO_GET) {
+		tcm_block_info_t block_info;
+
+		if (copy_from_user(&block_info, (void __user *)arg,
+				   sizeof(block_info)))
+			return -EFAULT;
+
+		if (block_info.block_id >= tcm->block_num)
+			return -EINVAL;
+
+		block_info.phys = tcm->blocks[block_info.block_id].phys;
+		block_info.size = tcm->blocks[block_info.block_id].size;
+		block_info.cpu_affinity_mask =
+			tcm->blocks[block_info.block_id].cpu_affinity_mask;
+
+		if (copy_to_user((void __user *)arg, &block_info,
+				 sizeof(block_info)))
+			return -EFAULT;
+
+		return 0;
+	}
+
+	return -ENOTTY;
+}
+
+static const struct file_operations tcm_fops = {
+	.owner = THIS_MODULE,
+	.open = tcm_open,
+	.mmap = tcm_mmap,
+	.unlocked_ioctl = tcm_ioctl,
+};
+
+static const struct of_device_id tcm_dt_ids[] = {
+	{ .compatible = "spacemit,k1-pro-tcm" },
+	{ .compatible = "spacemit,k1-x-tcm" },
+	{ .compatible = "spacemit,k1-tcm" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, tcm_dt_ids);
+
+static int tcm_probe(struct platform_device *pdev)
+{
+	struct tcm_direct_dev *tcm;
+	int ret;
+
+	tcm = devm_kzalloc(&pdev->dev, sizeof(*tcm), GFP_KERNEL);
+	if (!tcm)
+		return -ENOMEM;
+
+	tcm->dev = &pdev->dev;
+	ret = tcm_parse_layout(pdev, tcm);
+	if (ret < 0)
+		return ret;
+
+	tcm->miscdev.minor = MISC_DYNAMIC_MINOR;
+	tcm->miscdev.name = TCM_NAME;
+	tcm->miscdev.fops = &tcm_fops;
+	tcm->miscdev.mode = 0666;
+
+	ret = misc_register(&tcm->miscdev);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, tcm);
+
+	dev_info(&pdev->dev,
+		 "direct mmap phys 0x%llx size 0x%zx block_size 0x%zx block_num %zu via /dev/%s\n",
+		 (unsigned long long)tcm->phys, tcm->size,
+		 tcm->block_size, tcm->block_num, TCM_NAME);
+
+	return 0;
+}
+
+static void tcm_remove(struct platform_device *pdev)
+{
+	struct tcm_direct_dev *tcm = platform_get_drvdata(pdev);
+
+	if (!tcm)
+		return;
+
+	misc_deregister(&tcm->miscdev);
+}
+
+static struct platform_driver tcm_driver = {
+	.driver = {
+		.name = TCM_NAME,
+		.of_match_table = tcm_dt_ids,
+	},
+	.probe = tcm_probe,
+	.remove = tcm_remove,
+};
+
+module_platform_driver(tcm_driver);
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index c94ae4794545..b56cda7a07f1 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -251,14 +251,14 @@ config MMC_SDHCI_OF_DWCMSHC
 	  If unsure, say N.
 
 config MMC_SDHCI_OF_K1
-	tristate "SDHCI OF support for the SpacemiT K1 SoC"
+	tristate "SDHCI OF support for the SpacemiT SDHCI controller"
 	depends on ARCH_SPACEMIT || COMPILE_TEST
 	depends on MMC_SDHCI_PLTFM
 	depends on OF
 	depends on COMMON_CLK
 	help
 	  This selects the Secure Digital Host Controller Interface (SDHCI)
-	  found in the SpacemiT K1 SoC.
+	  found in the SpacemiT K1/K3 SoC.
 
 	  If you have a controller with this interface, say Y or M here.
 
diff --git a/drivers/mmc/host/sdhci-of-k1.c b/drivers/mmc/host/sdhci-of-k1.c
index 0cc97e23a2f9..9eb81f7054ac 100644
--- a/drivers/mmc/host/sdhci-of-k1.c
+++ b/drivers/mmc/host/sdhci-of-k1.c
@@ -12,14 +12,23 @@
 #include <linux/mmc/card.h>
 #include <linux/mmc/host.h>
 #include <linux/mmc/mmc.h>
+#include <linux/mmc/sd.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm.h>
 
 #include "sdhci.h"
 #include "sdhci-pltfm.h"
 
+#define SPACEMIT_SDHC_OP_EXT_REG	0x108
+#define  SDHC_OVRRD_CLK_OEN		BIT(11)
+#define  SDHC_FORCE_CLK_ON		BIT(12)
+
 #define SPACEMIT_SDHC_MMC_CTRL_REG	0x114
 #define  SDHC_MISC_INT_EN		BIT(1)
 #define  SDHC_MISC_INT			BIT(2)
@@ -28,10 +37,23 @@
 #define  SDHC_MMC_HS200			BIT(10)
 #define  SDHC_MMC_CARD_MODE		BIT(12)
 
+#define SPACEMIT_SDHC_RX_CFG_REG	0x118
+#define  SDHC_RX_SDCLK_SEL0		GENMASK(1, 0)
+#define  SDHC_RX_SDCLK_SEL1		GENMASK(3, 2)
+
 #define SPACEMIT_SDHC_TX_CFG_REG	0x11C
 #define  SDHC_TX_INT_CLK_SEL		BIT(30)
 #define  SDHC_TX_MUX_SEL		BIT(31)
 
+#define SPACEMIT_SDHC_DLINE_CTRL_REG	0x130
+#define  SDHC_DLINE_PU			BIT(0)
+#define  SDHC_RX_DLINE_CODE		GENMASK(23, 16)
+#define  SDHC_TX_DLINE_CODE		GENMASK(31, 24)
+
+#define SPACEMIT_SDHC_DLINE_CFG_REG	0x134
+#define  SDHC_RX_DLINE_REG		GENMASK(7, 0)
+#define  SDHC_TX_DLINE_REG		GENMASK(23, 16)
+
 #define SPACEMIT_SDHC_PHY_CTRL_REG	0x160
 #define  SDHC_PHY_FUNC_EN		BIT(0)
 #define  SDHC_PHY_PLL_LOCK		BIT(1)
@@ -60,11 +82,58 @@
 #define  SDHC_PHY_DRIVE_SEL		GENMASK(2, 0)
 #define  SDHC_RX_BIAS_CTRL		BIT(5)
 
+#define SDHC_RX_TUNE_DELAY_MIN		0x0
+#define SDHC_RX_TUNE_DELAY_MAX		0xFF
+#define SDHC_RX_TUNE_DELAY_STEP		0x1
+
+#define PHY_DRIVE_SEL_DEFAULT		0x4
+#define RX_TUNING_WINDOW_THRESHOLD	50
+#define RX_TUNING_DLINE_REG		0x00
+#define TX_TUNING_DLINE_REG		0x00
+#define TX_TUNING_DELAYCODE		0x7F
+
+#define SPACEMIT_RPM_DELAY_MS		50
+
+enum window_type {
+	LEFT_WINDOW = 0,
+	MIDDLE_WINDOW = 1,
+	RIGHT_WINDOW = 2,
+};
+
+struct tuning_window {
+	u8 min_delay;
+	u8 max_delay;
+};
+
+struct rx_tuning {
+	u8 tx_delaycode;
+	u8 tx_dline_reg;
+	u8 rx_dline_reg;
+	struct tuning_window windows;
+	u8 select_delay;
+
+	u8 window_limit;
+	u8 window_type;
+};
+
 struct spacemit_sdhci_host {
 	struct clk *clk_core;
 	struct clk *clk_io;
+	struct reset_control *reset;
+
+	struct rx_tuning rxtuning;
+	u8 phy_driver_sel;
+
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *pins_default;
+	struct pinctrl_state *pins_uhs;
+	struct pinctrl_state *pins_debug;
 };
 
+static struct sdhci_host *sdio_host;
+
+void spacemit_sdio_detect_change(int enable_scan);
+
 /* All helper functions will update clr/set while preserve rest bits */
 static inline void spacemit_sdhci_setbits(struct sdhci_host *host, u32 val, int reg)
 {
@@ -84,41 +153,124 @@ static inline void spacemit_sdhci_clrsetbits(struct sdhci_host *host, u32 clr, u
 	sdhci_writel(host, val, reg);
 }
 
+void spacemit_sdio_detect_change(int enable_scan)
+{
+	if (!sdio_host)
+		return;
+
+	sdio_host->mmc->rescan_entered = 0;
+	mmc_detect_change(sdio_host->mmc, 0);
+}
+EXPORT_SYMBOL(spacemit_sdio_detect_change);
+
 static void spacemit_sdhci_reset(struct sdhci_host *host, u8 mask)
 {
+#if !defined(CONFIG_SOC_SPACEMIT_K1_FPGA) && !defined(CONFIG_SOC_SPACEMIT_K3_FPGA)
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+#endif
 	sdhci_reset(host, mask);
 
 	if (mask != SDHCI_RESET_ALL)
 		return;
 
-	spacemit_sdhci_setbits(host, SDHC_PHY_FUNC_EN | SDHC_PHY_PLL_LOCK,
-			       SPACEMIT_SDHC_PHY_CTRL_REG);
-
-	spacemit_sdhci_clrsetbits(host, SDHC_PHY_DRIVE_SEL,
-				  SDHC_RX_BIAS_CTRL | FIELD_PREP(SDHC_PHY_DRIVE_SEL, 4),
-				  SPACEMIT_SDHC_PHY_PADCFG_REG);
-
-	if (!(host->mmc->caps2 & MMC_CAP2_NO_MMC))
+	if (host->mmc->caps2 & MMC_CAP2_NO_MMC) {
+		/* sd/sdio has no phy */
+		spacemit_sdhci_setbits(host, SDHC_TX_INT_CLK_SEL, SPACEMIT_SDHC_TX_CFG_REG);
+	} else {
+#if !defined(CONFIG_SOC_SPACEMIT_K1_FPGA) && !defined(CONFIG_SOC_SPACEMIT_K3_FPGA)
+		/* use phy func mode */
+		spacemit_sdhci_setbits(host, SDHC_PHY_FUNC_EN | SDHC_PHY_PLL_LOCK,
+				       SPACEMIT_SDHC_PHY_CTRL_REG);
+		spacemit_sdhci_clrsetbits(host, SDHC_PHY_DRIVE_SEL,
+					  SDHC_RX_BIAS_CTRL |
+					  FIELD_PREP(SDHC_PHY_DRIVE_SEL, sdhst->phy_driver_sel),
+					  SPACEMIT_SDHC_PHY_PADCFG_REG);
+#else
+		/* use phy bypass */
+		spacemit_sdhci_setbits(host, SDHC_TX_INT_CLK_SEL, SPACEMIT_SDHC_TX_CFG_REG);
+		spacemit_sdhci_setbits(host, SDHC_HOST_LEGACY_MODE, SPACEMIT_SDHC_PHY_CTRL_REG);
+		spacemit_sdhci_setbits(host, SDHC_PHY_TEST_EN, SPACEMIT_SDHC_PHY_FUNC_REG);
+#endif
+		/* mmc card mode */
 		spacemit_sdhci_setbits(host, SDHC_MMC_CARD_MODE, SPACEMIT_SDHC_MMC_CTRL_REG);
+	}
 }
 
 static void spacemit_sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned int timing)
 {
 	if (timing == MMC_TIMING_MMC_HS200)
 		spacemit_sdhci_setbits(host, SDHC_MMC_HS200, SPACEMIT_SDHC_MMC_CTRL_REG);
-
-	if (timing == MMC_TIMING_MMC_HS400)
+	else if (timing == MMC_TIMING_MMC_HS400)
 		spacemit_sdhci_setbits(host, SDHC_MMC_HS400, SPACEMIT_SDHC_MMC_CTRL_REG);
 
 	sdhci_set_uhs_signaling(host, timing);
+}
 
-	if (!(host->mmc->caps2 & MMC_CAP2_NO_SDIO))
-		spacemit_sdhci_setbits(host, SDHCI_CTRL_VDD_180, SDHCI_HOST_CONTROL2);
+static void spacemit_sdhci_set_clk_gate(struct sdhci_host *host, unsigned int auto_gate)
+{
+	if (auto_gate)
+		spacemit_sdhci_clrbits(host, SDHC_OVRRD_CLK_OEN | SDHC_FORCE_CLK_ON,
+				       SPACEMIT_SDHC_OP_EXT_REG);
+	else
+		spacemit_sdhci_setbits(host, SDHC_OVRRD_CLK_OEN | SDHC_FORCE_CLK_ON,
+				       SPACEMIT_SDHC_OP_EXT_REG);
+}
+
+static int spacemit_sdhci_card_busy(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	u32 present_state;
+
+	/* Check whether DAT[0] is 0 */
+	present_state = sdhci_readl(host, SDHCI_PRESENT_STATE);
+
+	if (host->mmc->caps2 & MMC_CAP2_NO_MMC) {
+		if ((SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)) == SD_SWITCH_VOLTAGE) &&
+		    host->mmc->ios.signal_voltage == MMC_SIGNAL_VOLTAGE_180)
+			/* recover the auto clock */
+			spacemit_sdhci_set_clk_gate(host, 1);
+	}
+
+	return !(present_state & SDHCI_DATA_0_LVL_MASK);
+}
+
+/*
+ * Select appropriate pinctrl state based on voltage and clock
+ * Returns: 0 on success, negative error code on failure
+ */
+static int spacemit_sdhci_select_pinctrl(struct spacemit_sdhci_host *sdhst,
+					 unsigned int clock,
+					 unsigned int signal_voltage)
+{
+	struct pinctrl_state *state = NULL;
+
+	if (!sdhst->pinctrl)
+		return 0;
+
+	if (signal_voltage == MMC_SIGNAL_VOLTAGE_180) {
+		/* 1.8V: use UHS mode */
+		state = sdhst->pins_uhs;
+	} else if (signal_voltage == MMC_SIGNAL_VOLTAGE_330) {
+		/* Only use debug mode when clock is off (host idle).
+		 * DAT3 is muxed with UART TX in debug pinctrl, so switching
+		 * during card initialization would corrupt the SD bus.
+		 */
+		if (!clock && sdhst->pins_debug)
+			state = sdhst->pins_debug;
+		else
+			state = sdhst->pins_default;
+	}
+
+	if (state)
+		return pinctrl_select_state(sdhst->pinctrl, state);
+
+	return 0;
 }
 
 static void spacemit_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
 {
 	struct mmc_host *mmc = host->mmc;
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
 
 	if (mmc->ios.timing <= MMC_TIMING_UHS_SDR50)
 		spacemit_sdhci_setbits(host, SDHC_TX_INT_CLK_SEL, SPACEMIT_SDHC_TX_CFG_REG);
@@ -126,7 +278,33 @@ static void spacemit_sdhci_set_clock(struct sdhci_host *host, unsigned int clock
 		spacemit_sdhci_clrbits(host, SDHC_TX_INT_CLK_SEL, SPACEMIT_SDHC_TX_CFG_REG);
 
 	sdhci_set_clock(host, clock);
-};
+
+	/* Switch pinctrl mode based on voltage */
+	spacemit_sdhci_select_pinctrl(sdhst, clock, mmc->ios.signal_voltage);
+
+	if (host->mmc->caps2 & MMC_CAP2_NO_MMC) {
+		/*
+		 * according to the SD spec, during a signal voltage level switch,
+		 * the clock must be closed for 5 ms.
+		 * then, the host starts providing clk at 1.8 and the host checks whether
+		 * DAT[3:0] is high after 1ms clk.
+		 *
+		 * for the above goal, temporarily disable the auto clk and keep clk
+		 * always on for 1ms.
+		 */
+		if ((SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)) == SD_SWITCH_VOLTAGE) &&
+		    host->mmc->ios.signal_voltage == MMC_SIGNAL_VOLTAGE_180) {
+			/* disable auto clock */
+			if (clock)
+				/*
+				 * some sdio device's signal level is already 1.8V before voltage
+				 * switch, so we should avoid generating clock multiple times
+				 * during switch sequence.
+				 */
+				spacemit_sdhci_set_clk_gate(host, 0);
+		}
+	}
+}
 
 static void spacemit_sdhci_phy_dll_init(struct sdhci_host *host)
 {
@@ -170,7 +348,10 @@ static unsigned int spacemit_sdhci_clk_get_max_clock(struct sdhci_host *host)
 {
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 
-	return clk_get_rate(pltfm_host->clk);
+	if (pltfm_host->clk)
+		return clk_get_rate(pltfm_host->clk);
+	else
+		return pltfm_host->clock;
 }
 
 static int spacemit_sdhci_pre_select_hs400(struct mmc_host *mmc)
@@ -208,17 +389,163 @@ static void spacemit_sdhci_pre_hs400_to_hs200(struct mmc_host *mmc)
 static inline int spacemit_sdhci_get_clocks(struct device *dev,
 					    struct sdhci_pltfm_host *pltfm_host)
 {
+#if !defined(CONFIG_SOC_SPACEMIT_K1_FPGA) && !defined(CONFIG_SOC_SPACEMIT_K3_FPGA)
 	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(pltfm_host);
+	int ret;
 
 	sdhst->clk_core = devm_clk_get_enabled(dev, "core");
 	if (IS_ERR(sdhst->clk_core))
-		return -EINVAL;
+		return PTR_ERR(sdhst->clk_core);
 
 	sdhst->clk_io = devm_clk_get_enabled(dev, "io");
 	if (IS_ERR(sdhst->clk_io))
-		return -EINVAL;
+		return PTR_ERR(sdhst->clk_io);
 
 	pltfm_host->clk = sdhst->clk_io;
+	if (pltfm_host->clock) {
+		ret = clk_set_rate(sdhst->clk_io, pltfm_host->clock);
+		if (ret) {
+			dev_err(dev, "failed to set io clock rate\n");
+			return ret;
+		}
+	}
+#endif
+	return 0;
+}
+
+static void spacemit_sw_rx_tuning_prepare(struct sdhci_host *host, u8 dline_reg)
+{
+	spacemit_sdhci_clrsetbits(host, SDHC_RX_DLINE_REG,
+				  FIELD_PREP(SDHC_RX_DLINE_REG, dline_reg),
+				  SPACEMIT_SDHC_DLINE_CFG_REG);
+
+	spacemit_sdhci_setbits(host, SDHC_DLINE_PU, SPACEMIT_SDHC_DLINE_CTRL_REG);
+	udelay(5);
+	spacemit_sdhci_clrsetbits(host, SDHC_RX_SDCLK_SEL1,
+				  FIELD_PREP(SDHC_RX_SDCLK_SEL1, 1),
+				  SPACEMIT_SDHC_RX_CFG_REG);
+
+	if (host->mmc->ios.timing == MMC_TIMING_MMC_HS200)
+		spacemit_sdhci_setbits(host, SDHC_HS200_USE_RFIFO, SPACEMIT_SDHC_PHY_FUNC_REG);
+}
+
+static void spacemit_sw_rx_set_delaycode(struct sdhci_host *host, u32 delay)
+{
+	spacemit_sdhci_clrsetbits(host, SDHC_RX_DLINE_CODE,
+				  FIELD_PREP(SDHC_RX_DLINE_CODE, delay),
+				  SPACEMIT_SDHC_DLINE_CTRL_REG);
+}
+
+static void spacemit_sw_tx_tuning_prepare(struct sdhci_host *host)
+{
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	struct rx_tuning *rxtuning = &sdhst->rxtuning;
+
+	/* set TX_DLINE_REG */
+	spacemit_sdhci_clrsetbits(host, SDHC_TX_DLINE_REG,
+				  FIELD_PREP(SDHC_TX_DLINE_REG, rxtuning->tx_dline_reg),
+				  SPACEMIT_SDHC_DLINE_CFG_REG);
+	/* set TX_DLINE_CODE */
+	spacemit_sdhci_clrsetbits(host, SDHC_TX_DLINE_CODE,
+				  FIELD_PREP(SDHC_TX_DLINE_CODE, rxtuning->tx_delaycode),
+				  SPACEMIT_SDHC_DLINE_CTRL_REG);
+	/* set TX_INT_CLK_SEL */
+	spacemit_sdhci_setbits(host, SDHC_TX_INT_CLK_SEL, SPACEMIT_SDHC_TX_CFG_REG);
+	spacemit_sdhci_setbits(host, SDHC_DLINE_PU, SPACEMIT_SDHC_DLINE_CTRL_REG);
+}
+
+static int spacemit_sw_rx_select_window(struct sdhci_host *host, u32 opcode)
+{
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	struct mmc_host *mmc = host->mmc;
+	struct rx_tuning *rxtuning = &sdhst->rxtuning;
+	struct tuning_window *window = &rxtuning->windows;
+	int min = 0, max, start, ret;
+	int cur_windows = 0;
+	int max_windows = 0;
+
+	start = SDHC_RX_TUNE_DELAY_MIN;
+	while (start <= SDHC_RX_TUNE_DELAY_MAX) {
+		if (!mmc->ops->get_cd(mmc))
+			return -ENODEV;
+
+		spacemit_sw_rx_set_delaycode(host, start);
+		ret = mmc_send_tuning(mmc, opcode, NULL);
+		if (ret) {
+			if (cur_windows)
+				dev_info(mmc_dev(mmc), "%s: pass window [%d %d)\n",
+					 mmc_hostname(host->mmc), min, start);
+			cur_windows = 0;
+		} else {
+			if (!cur_windows)
+				min = start;
+
+			cur_windows++;
+			if (cur_windows > max_windows) {
+				max_windows = cur_windows;
+				max = start;
+			}
+			if (start == SDHC_RX_TUNE_DELAY_MAX)
+				dev_info(mmc_dev(mmc), "%s: pass window [%d %d]\n",
+					 mmc_hostname(host->mmc), min, start);
+		}
+		start += SDHC_RX_TUNE_DELAY_STEP;
+	}
+
+	if (max_windows < rxtuning->window_limit) {
+		dev_warn(mmc_dev(mmc), "%s: fail to find valid tuning window, max_windows:%d\n",
+			 mmc_hostname(mmc), max_windows);
+		return -EIO;
+	}
+
+	window->min_delay = max - max_windows + 1;
+	window->max_delay = max;
+
+	if (rxtuning->window_type == LEFT_WINDOW)
+		rxtuning->select_delay = window->min_delay + DIV_ROUND_CLOSEST(max_windows, 3);
+	else if (rxtuning->window_type == RIGHT_WINDOW)
+		rxtuning->select_delay = window->min_delay + DIV_ROUND_CLOSEST(max_windows * 2, 3);
+	else
+		rxtuning->select_delay = window->min_delay + DIV_ROUND_CLOSEST(max_windows, 2);
+
+	return 0;
+}
+
+static int spacemit_sdhci_execute_sw_tuning(struct sdhci_host *host, u32 opcode)
+{
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	struct mmc_host *mmc = host->mmc;
+	struct rx_tuning *rxtuning = &sdhst->rxtuning;
+	int ret;
+	/*
+	 * Tuning is required for SDR50/SDR104, HS200/HS400 cards and
+	 * if clock frequency is greater than 100MHz in these modes.
+	 */
+	if (host->clock < 100 * 1000 * 1000 ||
+	    !(mmc->ios.timing == MMC_TIMING_MMC_HS200 ||
+	    mmc->ios.timing == MMC_TIMING_UHS_SDR50 ||
+	    mmc->ios.timing == MMC_TIMING_UHS_SDR104))
+		return 0;
+
+	/* step 1: set tx tuning config */
+	if (host->mmc->caps2 & MMC_CAP2_NO_MMC) {
+		dev_info(mmc_dev(mmc), "%s: set tx_delaycode: %d\n",
+			 mmc_hostname(mmc), rxtuning->tx_delaycode);
+		spacemit_sw_tx_tuning_prepare(host);
+	}
+
+	/* step 2: get pass window and calculate the select_delay */
+	spacemit_sw_rx_tuning_prepare(host, rxtuning->rx_dline_reg);
+	ret = spacemit_sw_rx_select_window(host, opcode);
+	if (ret) {
+		dev_warn(mmc_dev(mmc), "%s: tuning fail, err:%d\n", mmc_hostname(mmc), ret);
+		return ret;
+	}
+
+	/* step 3: set rx delay code */
+	spacemit_sw_rx_set_delaycode(host, rxtuning->select_delay);
+	dev_info(mmc_dev(mmc), "%s: tuning done, use delay_code:%d\n",
+		 mmc_hostname(mmc), rxtuning->select_delay);
 
 	return 0;
 }
@@ -229,6 +556,8 @@ static const struct sdhci_ops spacemit_sdhci_ops = {
 	.set_bus_width		= sdhci_set_bus_width,
 	.set_clock		= spacemit_sdhci_set_clock,
 	.set_uhs_signaling	= spacemit_sdhci_set_uhs_signaling,
+	.set_power		= sdhci_set_power_and_bus_voltage,
+	.platform_execute_tuning = spacemit_sdhci_execute_sw_tuning,
 };
 
 static const struct sdhci_pltfm_data spacemit_sdhci_k1_pdata = {
@@ -243,64 +572,299 @@ static const struct sdhci_pltfm_data spacemit_sdhci_k1_pdata = {
 		   SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
 };
 
+static const struct sdhci_pltfm_data spacemit_sdhci_k3_pdata = {
+	.ops = &spacemit_sdhci_ops,
+	.quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
+		  SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC |
+		  SDHCI_QUIRK_32BIT_ADMA_SIZE |
+		  SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
+		  SDHCI_QUIRK_BROKEN_CARD_DETECTION |
+		  SDHCI_QUIRK_BROKEN_TIMEOUT_VAL,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+};
+
 static const struct of_device_id spacemit_sdhci_of_match[] = {
-	{ .compatible = "spacemit,k1-sdhci" },
+	{
+		.compatible = "spacemit,k1-sdhci",
+		.data = &spacemit_sdhci_k1_pdata,
+	},
+	{
+		.compatible = "spacemit,k3-sdhci",
+		.data = &spacemit_sdhci_k3_pdata,
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, spacemit_sdhci_of_match);
 
+static void spacemit_sdhci_get_of_property(struct platform_device *pdev, struct sdhci_host *host)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	u32 property;
+
+	sdhst->rxtuning.rx_dline_reg = RX_TUNING_DLINE_REG;
+	sdhst->rxtuning.tx_dline_reg = TX_TUNING_DLINE_REG;
+
+	/* read rx tuning window limit */
+	if (!of_property_read_u32(node, "spacemit,rx_tuning_limit", &property))
+		sdhst->rxtuning.window_limit = (u8)property;
+	else
+		sdhst->rxtuning.window_limit = RX_TUNING_WINDOW_THRESHOLD;
+
+	/* read rx tuning window type */
+	if (!of_property_read_u32(node, "spacemit,rx_tuning_type", &property))
+		sdhst->rxtuning.window_type = (u8)property;
+	else
+		sdhst->rxtuning.window_type = MIDDLE_WINDOW;
+
+	/* read tx delaycode */
+	if (!of_property_read_u32(node, "spacemit,tx_delaycode", &property))
+		sdhst->rxtuning.tx_delaycode = (u8)property;
+	else
+		sdhst->rxtuning.tx_delaycode = TX_TUNING_DELAYCODE;
+
+	/* phy driver select */
+	if (!of_property_read_u32(node, "spacemit,phy_driver_sel", &property))
+		sdhst->phy_driver_sel = (u8)property;
+	else
+		sdhst->phy_driver_sel = PHY_DRIVE_SEL_DEFAULT;
+
+	sdhst->pinctrl = devm_pinctrl_get(&pdev->dev);
+	if (!IS_ERR(sdhst->pinctrl)) {
+		sdhst->pins_default = pinctrl_lookup_state(sdhst->pinctrl,
+							   "default");
+		if (IS_ERR(sdhst->pins_default))
+			sdhst->pins_default = NULL;
+
+		sdhst->pins_uhs = pinctrl_lookup_state(sdhst->pinctrl,
+						       "uhs");
+		if (IS_ERR(sdhst->pins_uhs))
+			sdhst->pins_uhs = NULL;
+
+		sdhst->pins_debug = pinctrl_lookup_state(sdhst->pinctrl,
+							 "debug");
+		if (IS_ERR(sdhst->pins_debug))
+			sdhst->pins_debug = NULL;
+
+		if (sdhst->pins_debug)
+			pinctrl_select_state(sdhst->pinctrl, sdhst->pins_debug);
+		else if (sdhst->pins_default)
+			pinctrl_select_state(sdhst->pinctrl, sdhst->pins_default);
+	} else {
+		sdhst->pinctrl = NULL;
+	}
+}
+
+static ssize_t spacemit_tx_delaycode_show(struct device *dev,
+					  struct device_attribute *attr,
+					  char *buf)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+
+	return sysfs_emit(buf, "0x%02x\n", sdhst->rxtuning.tx_delaycode);
+}
+
+static ssize_t spacemit_tx_delaycode_set(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	u8 delaycode;
+	int ret;
+
+	ret = kstrtou8(buf, 0, &delaycode);
+	if (ret)
+		return ret;
+
+	sdhst->rxtuning.tx_delaycode = delaycode;
+	return count;
+}
+
+static struct device_attribute spacemit_sysfs_files[] = {
+	__ATTR(tx_delaycode, 0644, spacemit_tx_delaycode_show, spacemit_tx_delaycode_set),
+};
+
 static int spacemit_sdhci_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct spacemit_sdhci_host *sdhst;
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_host *host;
+	const struct sdhci_pltfm_data *data;
 	struct mmc_host_ops *mops;
-	int ret;
+	int i, ret;
 
-	host = sdhci_pltfm_init(pdev, &spacemit_sdhci_k1_pdata, sizeof(*sdhst));
+	data = of_device_get_match_data(&pdev->dev);
+	if (!data) {
+		dev_err(dev, "no sdhci spacemit data\n");
+		return -EINVAL;
+	}
+
+	host = sdhci_pltfm_init(pdev, data, sizeof(*sdhst));
 	if (IS_ERR(host))
 		return PTR_ERR(host);
 
-	pltfm_host = sdhci_priv(host);
-
 	ret = mmc_of_parse(host->mmc);
 	if (ret)
-		goto err_pltfm;
+		return ret;
 
 	sdhci_get_of_property(pdev);
+	spacemit_sdhci_get_of_property(pdev, host);
 
+	mops = &host->mmc_host_ops;
 	if (!(host->mmc->caps2 & MMC_CAP2_NO_MMC)) {
-		mops = &host->mmc_host_ops;
 		mops->hs400_prepare_ddr	= spacemit_sdhci_pre_select_hs400;
 		mops->hs400_complete	= spacemit_sdhci_post_select_hs400;
 		mops->hs400_downgrade	= spacemit_sdhci_pre_hs400_to_hs200;
 		mops->hs400_enhanced_strobe = spacemit_sdhci_hs400_enhanced_strobe;
 	}
+	mops->card_busy = spacemit_sdhci_card_busy;
+
+	if (!(host->mmc->caps2 & MMC_CAP2_NO_SDIO))
+		/* skip auto rescan */
+		host->mmc->rescan_entered = 1;
 
 	host->mmc->caps |= MMC_CAP_NEED_RSP_BUSY;
 
+	pltfm_host = sdhci_priv(host);
 	ret = spacemit_sdhci_get_clocks(dev, pltfm_host);
 	if (ret)
-		goto err_pltfm;
+		return ret;
+
+	sdhst = sdhci_pltfm_priv(pltfm_host);
+	sdhst->reset = devm_reset_control_array_get_optional_shared(dev);
+	if (IS_ERR(sdhst->reset)) {
+		dev_err(dev, "failed to get reset control\n");
+		return PTR_ERR(sdhst->reset);
+	}
+
+	ret = reset_control_deassert(sdhst->reset);
+	if (ret)
+		return ret;
+
+	pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, SPACEMIT_RPM_DELAY_MS);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+	pm_suspend_ignore_children(&pdev->dev, 1);
 
 	ret = sdhci_add_host(host);
 	if (ret)
-		goto err_pltfm;
+		goto err_add_host;
+
+	if (!(host->mmc->caps2 & MMC_CAP2_NO_SDIO)) {
+		dev_notice(dev, "sdio host: 0x%p\n", host);
+		sdio_host = host;
+	}
+
+	if (host->mmc->caps2 & MMC_CAP2_NO_MMC) {
+		for (i = 0; i < ARRAY_SIZE(spacemit_sysfs_files); i++) {
+			ret = device_create_file(dev, &spacemit_sysfs_files[i]);
+			if (ret)
+				dev_warn(dev, "failed to create sysfs '%s': %d\n",
+					 spacemit_sysfs_files[i].attr.name, ret);
+		}
+	}
+
+	if (host->mmc->pm_caps & MMC_PM_WAKE_SDIO_IRQ)
+		device_init_wakeup(&pdev->dev, 1);
+
+	pm_runtime_put_autosuspend(&pdev->dev);
 
 	return 0;
 
-err_pltfm:
+err_add_host:
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+	reset_control_assert(sdhst->reset);
 	return ret;
 }
 
+static void spacemit_sdhci_remove(struct platform_device *pdev)
+{
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	int i;
+
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+
+	if (host->mmc->caps2 & MMC_CAP2_NO_MMC) {
+		for (i = 0; i < ARRAY_SIZE(spacemit_sysfs_files); i++)
+			device_remove_file(&pdev->dev, &spacemit_sysfs_files[i]);
+	}
+
+	sdhci_remove_host(host, 1);
+	reset_control_assert(sdhst->reset);
+}
+
+static int spacemit_sdhci_runtime_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	unsigned long flags;
+
+	sdhci_runtime_suspend_host(host);
+
+	spin_lock_irqsave(&host->lock, flags);
+	if (!(host->mmc->caps2 & MMC_CAP2_NO_MMC))
+		spacemit_sdhci_clrbits(host, SDHC_PHY_FUNC_EN, SPACEMIT_SDHC_PHY_CTRL_REG);
+	spin_unlock_irqrestore(&host->lock, flags);
+
+	clk_disable_unprepare(sdhst->clk_io);
+	clk_disable_unprepare(sdhst->clk_core);
+
+	return 0;
+}
+
+static int spacemit_sdhci_runtime_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(sdhci_priv(host));
+	unsigned long flags;
+	int ret;
+
+	ret = clk_prepare_enable(sdhst->clk_io);
+	if (ret) {
+		dev_err(dev, "failed to enable clk_io: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(sdhst->clk_core);
+	if (ret) {
+		dev_err(dev, "failed to enable clk_core: %d\n", ret);
+		clk_disable_unprepare(sdhst->clk_io);
+		return ret;
+	}
+
+	spin_lock_irqsave(&host->lock, flags);
+	if (!(host->mmc->caps2 & MMC_CAP2_NO_MMC))
+		spacemit_sdhci_setbits(host, SDHC_PHY_FUNC_EN, SPACEMIT_SDHC_PHY_CTRL_REG);
+	spin_unlock_irqrestore(&host->lock, flags);
+
+	sdhci_runtime_resume_host(host, 1);
+
+	return 0;
+}
+
+static const struct dev_pm_ops sdhci_spacemit_dev_pm_ops = {
+	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	RUNTIME_PM_OPS(spacemit_sdhci_runtime_suspend, spacemit_sdhci_runtime_resume, NULL)
+};
+
 static struct platform_driver spacemit_sdhci_driver = {
 	.driver		= {
 		.name	= "sdhci-spacemit",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 		.of_match_table = spacemit_sdhci_of_match,
+		.pm	= pm_ptr(&sdhci_spacemit_dev_pm_ops),
 	},
 	.probe		= spacemit_sdhci_probe,
-	.remove		= sdhci_pltfm_remove,
+	.remove		= spacemit_sdhci_remove,
 };
 module_platform_driver(spacemit_sdhci_driver);
 
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index ac12eaf11755..6e7cb154396d 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -514,6 +514,8 @@ source "drivers/net/dsa/Kconfig"
 
 source "drivers/net/ethernet/Kconfig"
 
+source "drivers/net/ethercat/Kconfig"
+
 source "drivers/net/fddi/Kconfig"
 
 source "drivers/net/hippi/Kconfig"
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 73bc63ecd65f..3c65df370bee 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -54,6 +54,7 @@ ifdef CONFIG_NET_DSA
 obj-y += dsa/
 endif
 obj-$(CONFIG_ETHERNET) += ethernet/
+obj-$(CONFIG_ETHERCAT) += ethercat/
 obj-$(CONFIG_FDDI) += fddi/
 obj-$(CONFIG_HIPPI) += hippi/
 obj-$(CONFIG_HAMRADIO) += hamradio/
diff --git a/drivers/net/can/flexcan/flexcan-core.c b/drivers/net/can/flexcan/flexcan-core.c
index 06d5d35fc1b5..4230ce558703 100644
--- a/drivers/net/can/flexcan/flexcan-core.c
+++ b/drivers/net/can/flexcan/flexcan-core.c
@@ -19,6 +19,7 @@
 #include <linux/firmware/imx/sci.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/reset.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/netdevice.h>
@@ -397,6 +398,24 @@ static const struct flexcan_devtype_data nxp_s32g2_devtype_data = {
 		FLEXCAN_QUIRK_SECONDARY_MB_IRQ,
 };
 
+#ifdef CONFIG_SOC_SPACEMIT
+static const struct flexcan_devtype_data spacemit_k1_devtype_data_can20 = {
+	.quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+		FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_FIFO |
+		FLEXCAN_QUIRK_SUPPORT_ECC,
+};
+static const struct flexcan_devtype_data spacemit_k1_devtype_data_fd = {
+	.quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+		FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+		FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_SUPPORT_FD |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR |
+		FLEXCAN_QUIRK_SUPPORT_ECC,
+};
+#endif
+
 static const struct can_bittiming_const flexcan_bittiming_const = {
 	.name = DRV_NAME,
 	.tseg1_min = 4,
@@ -634,13 +653,24 @@ static int flexcan_clks_enable(const struct flexcan_priv *priv)
 			clk_disable_unprepare(priv->clk_ipg);
 	}
 
+	if (priv->reset) {
+		err = reset_control_deassert(priv->reset);
+		if (err && priv->clk_ipg && priv->clk_per) {
+			clk_disable_unprepare(priv->clk_per);
+			clk_disable_unprepare(priv->clk_ipg);
+		}
+	}
 	return err;
 }
 
 static void flexcan_clks_disable(const struct flexcan_priv *priv)
 {
-	clk_disable_unprepare(priv->clk_per);
-	clk_disable_unprepare(priv->clk_ipg);
+	if (priv->reset)
+		reset_control_assert(priv->reset);
+	if (priv->clk_per)
+		clk_disable_unprepare(priv->clk_per);
+	if (priv->clk_ipg)
+		clk_disable_unprepare(priv->clk_ipg);
 }
 
 static inline int flexcan_transceiver_enable(const struct flexcan_priv *priv)
@@ -1354,6 +1384,49 @@ static void flexcan_set_bittiming(struct net_device *dev)
 		return flexcan_set_bittiming_ctrl(dev);
 }
 
+#ifdef CONFIG_SOC_SPACEMIT
+static void flexcan_ram_init(struct net_device *dev)
+{
+	struct flexcan_priv *priv = netdev_priv(dev);
+	struct flexcan_regs __iomem *regs = priv->regs;
+	u32 reg_ctrl2;
+	void __iomem *mb_addr;
+	u32 count;
+	u32 i;
+
+	/* 11.8.3.13 Detection and correction of memory errors:
+	 * CTRL2[WRMFRZ] grants write access to all memory positions
+	 * that require initialization, ranging from 0x080 to 0xADF
+	 * and from 0xF28 to 0xFFF when the CAN FD feature is enabled.
+	 * The RXMGMASK, RX14MASK, RX15MASK, and RXFGMASK registers
+	 * need to be initialized as well. MCR[RFEN] must not be set
+	 * during memory initialization.
+	 */
+	reg_ctrl2 = priv->read(&regs->ctrl2);
+	reg_ctrl2 |= FLEXCAN_CTRL2_WRMFRZ;
+	priv->write(reg_ctrl2, &regs->ctrl2);
+
+	/* use iowrite32 instead of memset_io on riscv */
+	mb_addr = &regs->mb[0][0];
+	count = offsetof(struct flexcan_regs, rx_smb1[3]) -
+		offsetof(struct flexcan_regs, mb[0][0]) + 0x4;
+	for (i = 0; i< count / 4; i++) {
+		priv->write(0, mb_addr + i * 4);
+	}
+
+	if (priv->can.ctrlmode & CAN_CTRLMODE_FD) {
+		mb_addr = &regs->tx_smb_fd[0];
+		count = offsetof(struct flexcan_regs, rx_smb1_fd[17]) -
+			offsetof(struct flexcan_regs, tx_smb_fd[0]) + 0x4;
+		for (i = 0; i< count / 4; i++) {
+			priv->write(0, mb_addr + i * 4);
+		}
+	}
+
+	reg_ctrl2 &= ~FLEXCAN_CTRL2_WRMFRZ;
+	priv->write(reg_ctrl2, &regs->ctrl2);
+}
+#else
 static void flexcan_ram_init(struct net_device *dev)
 {
 	struct flexcan_priv *priv = netdev_priv(dev);
@@ -1380,6 +1453,7 @@ static void flexcan_ram_init(struct net_device *dev)
 	reg_ctrl2 &= ~FLEXCAN_CTRL2_WRMFRZ;
 	priv->write(reg_ctrl2, &regs->ctrl2);
 }
+#endif
 
 static int flexcan_rx_offload_setup(struct net_device *dev)
 {
@@ -1494,8 +1568,15 @@ static int flexcan_chip_start(struct net_device *dev)
 	 */
 	reg_mcr = priv->read(&regs->mcr);
 	reg_mcr &= ~FLEXCAN_MCR_MAXMB(0xff);
+#ifndef CONFIG_SOC_SPACEMIT
 	reg_mcr |= FLEXCAN_MCR_SUPV | FLEXCAN_MCR_WRN_EN | FLEXCAN_MCR_IRMQ |
 		FLEXCAN_MCR_IDAM_C | FLEXCAN_MCR_MAXMB(priv->tx_mb_idx);
+#else
+	reg_mcr |= FLEXCAN_MCR_SUPV | FLEXCAN_MCR_WRN_EN |
+		FLEXCAN_MCR_IDAM_C | FLEXCAN_MCR_MAXMB(priv->tx_mb_idx);
+	if (priv->fix_id == false)
+		reg_mcr |= FLEXCAN_MCR_IRMQ;
+#endif
 
 	/* MCR
 	 *
@@ -1604,6 +1685,14 @@ static int flexcan_chip_start(struct net_device *dev)
 			mb = flexcan_get_mb(priv, i);
 			priv->write(FLEXCAN_MB_CODE_RX_EMPTY,
 				    &mb->can_ctrl);
+#ifdef CONFIG_SOC_SPACEMIT
+			if (priv->fix_id == true) {
+				if (priv->mb_bits[i - 1] == 11)
+					priv->write(priv->mb_ids[i - 1] << 18, &mb->can_id);
+				else
+					priv->write(priv->mb_ids[i - 1], &mb->can_id);
+			}
+#endif
 		}
 	} else {
 		/* clear and invalidate unused mailboxes first */
@@ -1623,7 +1712,14 @@ static int flexcan_chip_start(struct net_device *dev)
 		    &priv->tx_mb->can_ctrl);
 
 	/* acceptance mask/acceptance code (accept everything) */
+#ifndef CONFIG_SOC_SPACEMIT
 	priv->write(0x0, &regs->rxgmask);
+#else
+	if (priv->fix_id == false)
+		priv->write(0x0, &regs->rxgmask);
+	else
+		priv->write(0x1fffffff, &regs->rxgmask);
+#endif
 	priv->write(0x0, &regs->rx14mask);
 	priv->write(0x0, &regs->rx15mask);
 
@@ -2071,6 +2167,10 @@ static const struct of_device_id flexcan_of_match[] = {
 	{ .compatible = "fsl,ls1021ar2-flexcan", .data = &fsl_ls1021a_r2_devtype_data, },
 	{ .compatible = "fsl,lx2160ar1-flexcan", .data = &fsl_lx2160a_r1_devtype_data, },
 	{ .compatible = "nxp,s32g2-flexcan", .data = &nxp_s32g2_devtype_data, },
+#ifdef CONFIG_SOC_SPACEMIT
+	{ .compatible = "spacemit,k1-flexcan", .data = &spacemit_k1_devtype_data_fd, },
+	{ .compatible = "spacemit,k1-flexcan-can2.0", .data = &spacemit_k1_devtype_data_can20, },
+#endif
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, flexcan_of_match);
@@ -2093,11 +2193,15 @@ static int flexcan_probe(struct platform_device *pdev)
 	struct regulator *reg_xceiver;
 	struct phy *transceiver;
 	struct clk *clk_ipg = NULL, *clk_per = NULL;
+	struct reset_control *reset;
 	struct flexcan_regs __iomem *regs;
 	struct flexcan_platform_data *pdata;
 	int err, irq;
 	u8 clk_src = 1;
 	u32 clock_freq = 0;
+#ifdef CONFIG_SOC_SPACEMIT
+	int ret;
+#endif
 
 	reg_xceiver = devm_regulator_get_optional(&pdev->dev, "xceiver");
 	if (PTR_ERR(reg_xceiver) == -EPROBE_DEFER)
@@ -2138,6 +2242,27 @@ static int flexcan_probe(struct platform_device *pdev)
 			return PTR_ERR(clk_per);
 		}
 		clock_freq = clk_get_rate(clk_per);
+	} else {
+		clk_per = devm_clk_get(&pdev->dev, "per");
+		if (IS_ERR(clk_per)) {
+			dev_err(&pdev->dev, "no per clock defined\n");
+			return PTR_ERR(clk_per);
+		}
+		clk_set_rate(clk_per, clock_freq);
+	}
+
+	if (!clk_ipg) {
+		clk_ipg = devm_clk_get(&pdev->dev, "ipg");
+		if (IS_ERR(clk_ipg)) {
+			dev_err(&pdev->dev, "no ipg clock defined\n");
+			return PTR_ERR(clk_ipg);
+		}
+	}
+
+	reset = devm_reset_control_get_optional(&pdev->dev,NULL);
+	if(IS_ERR(reset)) {
+		dev_err(&pdev->dev, "flexcan get reset failed\n");
+		return PTR_ERR(reset);
 	}
 
 	irq = platform_get_irq(pdev, 0);
@@ -2208,9 +2333,30 @@ static int flexcan_probe(struct platform_device *pdev)
 	priv->clk_ipg = clk_ipg;
 	priv->clk_per = clk_per;
 	priv->clk_src = clk_src;
+	priv->reset = reset;
 	priv->reg_xceiver = reg_xceiver;
 	priv->transceiver = transceiver;
 
+#ifdef CONFIG_SOC_SPACEMIT
+	priv->rxmb_len = device_property_count_u32(&pdev->dev, "flexcan-mailbox-id");
+	if (priv->rxmb_len < 0) {
+		priv->fix_id = false;
+	} else if (priv->rxmb_len > MAX_RX_MAILBOX) {
+		dev_err(&pdev->dev, "flexcan rx mailbox id num is more than the max num\n");
+		return -EINVAL;
+	} else {
+		ret = device_property_read_u32_array(&pdev->dev, "flexcan-mailbox-id", priv->mb_ids, priv->rxmb_len);
+		if (ret)
+			return -EINVAL;
+		priv->fix_id = true;
+		ret = device_property_read_u8_array(&pdev->dev, "flexcan-mailbox-id-bits", priv->mb_bits, priv->rxmb_len);
+		if (ret) {
+			dev_err(&pdev->dev, "please add flexcan fix id length:11/29\n");
+			return -EINVAL;
+		}
+	}
+#endif
+
 	if (transceiver)
 		priv->can.bitrate_max = transceiver->attrs.max_link_rate;
 
diff --git a/drivers/net/can/flexcan/flexcan.h b/drivers/net/can/flexcan/flexcan.h
index 16692a2502eb..8ce1d2957d04 100644
--- a/drivers/net/can/flexcan/flexcan.h
+++ b/drivers/net/can/flexcan/flexcan.h
@@ -75,6 +75,10 @@
  */
 #define FLEXCAN_QUIRK_SECONDARY_MB_IRQ	BIT(18)
 
+#ifdef CONFIG_SOC_SPACEMIT
+#define MAX_RX_MAILBOX	12
+#endif
+
 struct flexcan_devtype_data {
 	u32 quirks;		/* quirks needed for different IP cores */
 };
@@ -105,6 +109,7 @@ struct flexcan_priv {
 
 	struct clk *clk_ipg;
 	struct clk *clk_per;
+	struct reset_control *reset;
 	struct flexcan_devtype_data devtype_data;
 	struct regulator *reg_xceiver;
 	struct phy *transceiver;
@@ -120,6 +125,12 @@ struct flexcan_priv {
 	/* Read and Write APIs */
 	u32 (*read)(void __iomem *addr);
 	void (*write)(u32 val, void __iomem *addr);
+#ifdef CONFIG_SOC_SPACEMIT
+	int rxmb_len;
+	u32 mb_ids[MAX_RX_MAILBOX];
+	u8 mb_bits[MAX_RX_MAILBOX];
+	bool fix_id;
+#endif
 };
 
 extern const struct ethtool_ops flexcan_ethtool_ops;
diff --git a/drivers/net/ethercat/Kconfig b/drivers/net/ethercat/Kconfig
new file mode 100644
index 000000000000..485dfa1f01a3
--- /dev/null
+++ b/drivers/net/ethercat/Kconfig
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# EtherCAT support
+#
+
+menuconfig ETHERCAT
+	bool "EtherCAT support"
+	depends on NET && NETDEVICES && ETHERNET
+	help
+	  EtherCAT is an industrial real-time fieldbus protocol that runs over
+	  Ethernet frames (Layer 2). Selecting this option enables the EtherCAT
+	  master subsystem and the corresponding network device interfaces.
+
+if ETHERCAT
+
+source "drivers/net/ethercat/master/Kconfig"
+source "drivers/net/ethercat/device/Kconfig"
+
+endif # ETHERCAT
+
diff --git a/drivers/net/ethercat/Makefile b/drivers/net/ethercat/Makefile
new file mode 100644
index 000000000000..4b0844580eeb
--- /dev/null
+++ b/drivers/net/ethercat/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# EtherCAT core and device interfaces
+#
+
+obj-$(CONFIG_ETHERCAT) += master/
+obj-$(CONFIG_ETHERCAT) += device/
+
diff --git a/drivers/net/ethercat/device/K1/Makefile b/drivers/net/ethercat/device/K1/Makefile
new file mode 100644
index 000000000000..579beeeb116e
--- /dev/null
+++ b/drivers/net/ethercat/device/K1/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for K1X EMAC EtherCAT device driver
+#
+
+obj-$(CONFIG_EC_K1_GMAC) += ec_k1x_emac.o
diff --git a/drivers/net/ethercat/device/K1/ec_k1x_emac.c b/drivers/net/ethercat/device/K1/ec_k1x_emac.c
new file mode 100644
index 000000000000..6a9601150d42
--- /dev/null
+++ b/drivers/net/ethercat/device/K1/ec_k1x_emac.c
@@ -0,0 +1,1689 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * spacemit k1x emac driver
+ *
+ * Copyright (c) 2023, spacemit Corporation.
+ *
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/crc32.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/etherdevice.h>
+#include <linux/dma-direct.h>
+#include <linux/in.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/ip.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/of_net.h>
+#include <linux/of_mdio.h>
+#include <linux/of_irq.h>
+#include <linux/phy.h>
+#include <linux/phylink.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm.h>
+#include <linux/tcp.h>
+#include <linux/timer.h>
+#include <linux/types.h>
+#include <linux/udp.h>
+#include <linux/workqueue.h>
+#include <linux/reset.h>
+
+#include "ec_k1x_emac.h"
+
+#define DRIVER_NAME				"k1x_ec_emac"
+
+/* k1x PMUap base */
+#define PMUA_BASE_REG		0xd4282800
+
+#define TUNING_CMD_LEN				50
+#define CLK_PHASE_CNT				256
+#define CLK_PHASE_REVERT			180
+
+#define TXCLK_PHASE_DEFAULT			0
+#define RXCLK_PHASE_DEFAULT			0
+
+#define TX_PHASE				1
+#define RX_PHASE				0
+
+#define DEFAULT_TX_THRESHOLD			(192)
+#define DEFAULT_RX_THRESHOLD			(12)
+#define DEFAULT_TX_RING_NUM			(128)
+#define DEFAULT_RX_RING_NUM			(128)
+#define DEFAULT_DMA_BURST_LEN			(1)
+#define HASH_TABLE_SIZE				(64)
+
+#define EMAC_DMA_REG_CNT			16
+#define EMAC_MAC_REG_CNT			124
+#define EMAC_REG_SPACE_SIZE			((EMAC_DMA_REG_CNT + \
+						 EMAC_MAC_REG_CNT) * 4)
+
+enum clk_tuning_way {
+	/* fpga clk tuning register */
+	CLK_TUNING_BY_REG,
+	/* zebu/evb rgmii delayline register */
+	CLK_TUNING_BY_DLINE,
+	/* evb rmii only revert tx/rx clock for clk tuning */
+	CLK_TUNING_BY_CLK_REVERT,
+	CLK_TUNING_MAX,
+};
+
+static int emac_open(struct net_device *ndev);
+static int emac_close(struct net_device *ndev);
+static int emac_start_xmit(struct sk_buff *skb, struct net_device *ndev);
+static void emac_clean_tx_desc_ring(struct emac_priv *priv);
+static void emac_clean_rx_desc_ring(struct emac_priv *priv);
+static void emac_configure_tx(struct emac_priv *priv);
+static void emac_configure_rx(struct emac_priv *priv);
+static int emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb, u32 max_tx_len,	u32 frag_num);
+static int emac_tx_clean_desc(struct emac_priv *priv);
+static int emac_rx_clean_desc(struct emac_priv *priv, int budget);
+static int emac_alloc_rx_desc_buffers(struct emac_priv *priv);
+static int emac_phy_connect(struct net_device *dev);
+static int emac_sw_init(struct emac_priv *priv);
+
+static bool emac_is_rmii(struct emac_priv *priv)
+{
+	return priv->phy_interface == PHY_INTERFACE_MODE_RMII;
+}
+
+static void emac_enable_axi_single_id_mode(struct emac_priv *priv, int en)
+{
+	u32 val;
+
+	val = readl(priv->ctrl_reg);
+	if (en)
+		val |= AXI_SINGLE_ID;
+	else
+		val &= ~AXI_SINGLE_ID;
+	writel(val, priv->ctrl_reg);
+}
+
+static void emac_phy_interface_config(struct emac_priv *priv)
+{
+	u32 val;
+
+	val = readl(priv->ctrl_reg);
+	if (emac_is_rmii(priv)) {
+		val &= ~PHY_INTF_RGMII;
+		if (priv->ref_clk_frm_soc)
+			val |= REF_CLK_SEL;
+		else
+			val &= ~REF_CLK_SEL;
+	} else {
+		val |= PHY_INTF_RGMII;
+		if (priv->ref_clk_frm_soc)
+			val |= RGMII_TX_CLK_SEL;
+	}
+	writel(val, priv->ctrl_reg);
+}
+
+/* Name		emac_reset_hw
+ * Arguments	priv : pointer to hardware data structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	TBDL
+ */
+static int emac_reset_hw(struct emac_priv *priv)
+{
+	/* disable all the interrupts */
+	emac_wr(priv, MAC_INTERRUPT_ENABLE, 0x0000);
+	emac_wr(priv, DMA_INTERRUPT_ENABLE, 0x0000);
+
+	/* disable transmit and receive units */
+	emac_wr(priv, MAC_RECEIVE_CONTROL, 0x0000);
+	emac_wr(priv, MAC_TRANSMIT_CONTROL, 0x0000);
+
+	/* stop the DMA */
+	emac_wr(priv, DMA_CONTROL, 0x0000);
+
+	/* reset mac, statistic counters */
+	emac_wr(priv, MAC_GLOBAL_CONTROL, 0x0018);
+
+	emac_wr(priv, MAC_GLOBAL_CONTROL, 0x0000);
+	return 0;
+}
+
+/* Name		emac_init_hw
+ * Arguments	pstHWData	: pointer to hardware data structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	TBDL
+ * Assumes that the controller has previously been reset
+ * and is in apost-reset uninitialized state.
+ * Initializes the receive address registers,
+ * multicast table, and VLAN filter table.
+ * Calls routines to setup link
+ * configuration and flow control settings.
+ * Clears all on-chip counters. Leaves
+ * the transmit and receive units disabled and uninitialized.
+ */
+static int emac_init_hw(struct emac_priv *priv)
+{
+	u32 val = 0;
+
+	emac_enable_axi_single_id_mode(priv, 1);
+
+	/* MAC Init
+	 * disable transmit and receive units
+	 */
+	emac_wr(priv, MAC_RECEIVE_CONTROL, 0x0000);
+	emac_wr(priv, MAC_TRANSMIT_CONTROL, 0x0000);
+
+	/* enable mac address 1 filtering */
+	emac_wr(priv, MAC_ADDRESS_CONTROL, MREGBIT_MAC_ADDRESS1_ENABLE);
+
+	/* zero initialize the multicast hash table */
+	emac_wr(priv, MAC_MULTICAST_HASH_TABLE1, 0x0);
+	emac_wr(priv, MAC_MULTICAST_HASH_TABLE2, 0x0);
+	emac_wr(priv, MAC_MULTICAST_HASH_TABLE3, 0x0);
+	emac_wr(priv, MAC_MULTICAST_HASH_TABLE4, 0x0);
+
+	emac_wr(priv, MAC_TRANSMIT_FIFO_ALMOST_FULL, 0x1f8);
+
+	emac_wr(priv, MAC_TRANSMIT_PACKET_START_THRESHOLD, priv->tx_threshold);
+
+	emac_wr(priv, MAC_RECEIVE_PACKET_START_THRESHOLD, priv->rx_threshold);
+
+	/* set emac rx mitigation frame count */
+	val = EMAC_RX_FRAMES & MREGBIT_RECEIVE_IRQ_FRAME_COUNTER_MSK;
+
+	/* set emac rx mitigation timeout */
+	val |= (EMAC_RX_COAL_TIMEOUT << MREGBIT_RECEIVE_IRQ_TIMEOUT_COUNTER_OFST) &
+		MREGBIT_RECEIVE_IRQ_TIMEOUT_COUNTER_MSK;
+
+	/* disable emac rx irq mitigation */
+	val &= ~MRGEBIT_RECEIVE_IRQ_MITIGATION_ENABLE;
+
+	emac_wr(priv, DMA_RECEIVE_IRQ_MITIGATION_CTRL, val);
+
+	/* reset dma */
+	emac_wr(priv, DMA_CONTROL, 0x0000);
+
+	emac_wr(priv, DMA_CONFIGURATION, 0x01);
+	usleep_range(9000, 10000);
+	emac_wr(priv, DMA_CONFIGURATION, 0x00);
+	usleep_range(9000, 10000);
+
+	val = 0;
+	val |= MREGBIT_STRICT_BURST;
+	val |= MREGBIT_DMA_64BIT_MODE;
+
+	if (priv->dma_burst_len)
+		val |= 1 << priv->dma_burst_len;
+	else
+		val |= MREGBIT_BURST_1WORD;
+
+	emac_wr(priv, DMA_CONFIGURATION, val);
+
+	return 0;
+}
+
+static int emac_set_mac_addr(struct emac_priv *priv, const unsigned char *addr)
+{
+	emac_wr(priv, MAC_ADDRESS1_HIGH, ((addr[1] << 8) | addr[0]));
+	emac_wr(priv, MAC_ADDRESS1_MED, ((addr[3] << 8) | addr[2]));
+	emac_wr(priv, MAC_ADDRESS1_LOW, ((addr[5] << 8) | addr[4]));
+
+	return 0;
+}
+
+static void emac_dma_start_transmit(struct emac_priv *priv)
+{
+	emac_wr(priv, DMA_TRANSMIT_POLL_DEMAND, 0xFF);
+}
+
+static inline u32 emac_tx_avail(struct emac_priv *priv)
+{
+	struct emac_desc_ring *tx_ring = &priv->tx_ring;
+	u32 avail;
+
+	if (tx_ring->tail > tx_ring->head)
+		avail = tx_ring->tail - tx_ring->head - 1;
+	else
+		avail = tx_ring->total_cnt - tx_ring->head + tx_ring->tail - 1;
+
+	return avail;
+}
+
+/* Name		emac_sw_init
+ * Arguments	priv	: pointer to driver private data structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	Reads PCI space configuration information and
+ *		initializes the variables with
+ *		their default values
+ */
+static int emac_sw_init(struct emac_priv *priv)
+{
+	priv->dma_buf_sz = EMAC_DEFAULT_BUFSIZE;
+
+	priv->tx_ring.total_cnt = priv->tx_ring_num;
+	priv->rx_ring.total_cnt = priv->rx_ring_num;
+	return 0;
+}
+
+/* Name		emac_configure_tx
+ * Arguments	priv : pointer to driver private data structure
+ * Return	none
+ * Description	Configures the transmit unit of the device
+ */
+static void emac_configure_tx(struct emac_priv *priv)
+{
+	u32 val;
+
+	/* set the transmit base address */
+	val = (u32)(priv->tx_ring.desc_dma_addr);
+
+	emac_wr(priv, DMA_TRANSMIT_BASE_ADDRESS, val);
+
+	/* Tx Inter Packet Gap value and enable the transmit */
+	val = emac_rd(priv, MAC_TRANSMIT_CONTROL);
+	val &= (~MREGBIT_IFG_LEN);
+	val |= MREGBIT_TRANSMIT_ENABLE;
+	val |= MREGBIT_TRANSMIT_AUTO_RETRY;
+	emac_wr(priv, MAC_TRANSMIT_CONTROL, val);
+
+	emac_wr(priv, DMA_TRANSMIT_AUTO_POLL_COUNTER, 0x00);
+
+	/* start tx dma */
+	val = emac_rd(priv, DMA_CONTROL);
+	val |= MREGBIT_START_STOP_TRANSMIT_DMA;
+	emac_wr(priv, DMA_CONTROL, val);
+}
+
+/* Name		emac_configure_rx
+ * Arguments	priv : pointer to driver private data structure
+ * Return	none
+ * Description	Configures the receive unit of the device
+ */
+static void emac_configure_rx(struct emac_priv *priv)
+{
+	u32 val;
+
+	/* set the receive base address */
+	val = (u32)(priv->rx_ring.desc_dma_addr);
+	emac_wr(priv, DMA_RECEIVE_BASE_ADDRESS, val);
+
+	/* enable the receive */
+	val = emac_rd(priv, MAC_RECEIVE_CONTROL);
+	val |= MREGBIT_RECEIVE_ENABLE;
+	val |= MREGBIT_STORE_FORWARD;
+	emac_wr(priv, MAC_RECEIVE_CONTROL, val);
+
+	/* start rx dma */
+	val = emac_rd(priv, DMA_CONTROL);
+	val |= MREGBIT_START_STOP_RECEIVE_DMA;
+	emac_wr(priv, DMA_CONTROL, val);
+}
+
+/* Name		emac_free_tx_buf
+ * Arguments	priv : pointer to driver private data structure
+ * 		i: ring idx
+ * Return	0 - Success;
+ * Description	Freeing the TX buffer data.
+ */
+static int emac_free_tx_buf(struct emac_priv *priv, int i)
+{
+	struct emac_desc_ring *tx_ring;
+	struct emac_tx_desc_buffer *tx_buf;
+	struct desc_buf *buf;
+	int j;
+
+	tx_ring = &priv->tx_ring;
+	tx_buf = &tx_ring->tx_desc_buf[i];
+
+	for (j = 0; j < 2; j++) {
+		buf = &tx_buf->buf[j];
+		if (buf->dma_addr) {
+			if (buf->map_as_page)
+				dma_unmap_page(&priv->pdev->dev, buf->dma_addr,
+					       buf->dma_len, DMA_TO_DEVICE);
+			else
+				dma_unmap_single(&priv->pdev->dev, buf->dma_addr,
+						 buf->dma_len, DMA_TO_DEVICE);
+
+			buf->dma_addr = 0;
+			buf->map_as_page = false;
+			buf->buff_addr = NULL;
+		}
+	}
+	return 0;
+}
+
+/* Name		emac_clean_tx_desc_ring
+ * Arguments	priv : pointer to driver private data structure
+ * Return	none
+ * Description	Freeing the TX resources allocated earlier.
+ */
+static void emac_clean_tx_desc_ring(struct emac_priv *priv)
+{
+	struct emac_desc_ring *tx_ring = &priv->tx_ring;
+	u32 i;
+
+	/* Free all the Tx ring sk_buffs */
+	for (i = 0; i < tx_ring->total_cnt; i++)
+		emac_free_tx_buf(priv, i);
+
+	tx_ring->head = 0;
+	tx_ring->tail = 0;
+}
+
+/* Name		emac_clean_rx_desc_ring
+ * Arguments	priv : pointer to driver private data structure
+ * Return	none
+ * Description	Freeing the RX resources allocated earlier.
+ */
+static void emac_clean_rx_desc_ring(struct emac_priv *priv)
+{
+	struct emac_desc_ring *rx_ring;
+	struct emac_desc_buffer *rx_buf;
+	u32 i;
+
+	rx_ring = &priv->rx_ring;
+
+	/* Free all the Rx ring sk_buffs */
+	for (i = 0; i < rx_ring->total_cnt; i++) {
+		rx_buf = &rx_ring->desc_buf[i];
+		if (rx_buf->skb) {
+			dma_unmap_single(&priv->pdev->dev, rx_buf->dma_addr,
+					 rx_buf->dma_len, DMA_FROM_DEVICE);
+
+			dev_kfree_skb(rx_buf->skb);
+			rx_buf->skb = NULL;
+		}
+	}
+
+	rx_ring->tail = 0;
+	rx_ring->head = 0;
+}
+
+/* Name		emac_up
+ * Arguments	priv : pointer to driver private data structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	This function is called from emac_open and
+ *		performs the things when net interface is about to up.
+ *		It configues the Tx and Rx unit of the device and
+ *		registers interrupt handler.
+ *		It also starts one watchdog timer to monitor
+ *		the net interface link status.
+ */
+static int emac_up(struct emac_priv *priv)
+{
+	struct net_device *ndev = priv->ndev;
+	int ret;
+
+	ret = emac_phy_connect(ndev);
+	if (ret) {
+		pr_err("%s  phy_connet failed\n", __func__);
+		goto err;
+	}
+	/* init hardware */
+	emac_init_hw(priv);
+
+	emac_set_mac_addr(priv, ndev->dev_addr);
+	/* configure transmit unit */
+	emac_configure_tx(priv);
+	/* configure rx unit */
+	emac_configure_rx(priv);
+
+	/* allocate buffers for receive descriptors */
+	ret = emac_alloc_rx_desc_buffers(priv);
+	if (ret) {
+		pr_err("%s  alloc rx desc buffers failed\n", __func__);
+		goto err;
+	}
+
+	if (ndev->phydev)
+		phy_start(ndev->phydev);
+
+	emac_wr(priv, MAC_INTERRUPT_ENABLE, 0x0000);
+
+	return 0;
+err:
+	return ret;
+}
+
+/* Name		emac_down
+ * Arguments	priv : pointer to driver private data structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	This function is called from emac_close and
+ *		performs the things when net interface is about to down.
+ *		It frees the irq, removes the various timers.
+ *		It sets the net interface off and
+ *		resets the hardware. Cleans the Tx and Rx
+ *		ring descriptor.
+ */
+static int emac_down(struct emac_priv *priv)
+{
+	struct net_device *ndev = priv->ndev;
+
+	/* Stop and disconnect the PHY */
+	if (ndev->phydev) {
+		phy_stop(ndev->phydev);
+		phy_disconnect(ndev->phydev);
+	}
+
+	ecdev_set_link(priv->ecdev, 0);
+	priv->link = false;
+	priv->duplex = DUPLEX_UNKNOWN;
+	priv->speed = SPEED_UNKNOWN;
+
+	emac_reset_hw(priv);
+
+	return 0;
+}
+
+/* Name		emac_alloc_tx_resources
+ * Arguments	priv : pointer to driver private data structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	Allocates TX resources and getting virtual & physical address.
+ */
+static int emac_alloc_tx_resources(struct emac_priv *priv)
+{
+	struct emac_desc_ring *tx_ring = &priv->tx_ring;
+	struct platform_device *pdev  = priv->pdev;
+	u32 size;
+
+	size = sizeof(struct emac_tx_desc_buffer) * tx_ring->total_cnt;
+
+	/* allocate memory */
+	tx_ring->tx_desc_buf = kzalloc(size, GFP_KERNEL);
+	if (!tx_ring->tx_desc_buf) {
+		pr_err("Memory allocation failed for the Transmit descriptor buffer\n");
+		return -ENOMEM;
+	}
+
+	memset(tx_ring->tx_desc_buf, 0, size);
+
+	tx_ring->total_size = tx_ring->total_cnt * sizeof(struct emac_tx_desc);
+
+	EMAC_ROUNDUP(tx_ring->total_size, 1024);
+
+	tx_ring->desc_addr = dma_alloc_coherent(&pdev->dev,
+							tx_ring->total_size,
+							&tx_ring->desc_dma_addr,
+							GFP_KERNEL);
+	if (!tx_ring->desc_addr) {
+		pr_err("Memory allocation failed for the Transmit descriptor ring\n");
+		kfree(tx_ring->tx_desc_buf);
+		return -ENOMEM;
+	}
+
+	memset(tx_ring->desc_addr, 0, tx_ring->total_size);
+
+	tx_ring->head = 0;
+	tx_ring->tail = 0;
+
+	return 0;
+}
+
+/* Name		emac_alloc_rx_resources
+ * Arguments	priv	: pointer to driver private data structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	Allocates RX resources and getting virtual & physical address.
+ */
+static int emac_alloc_rx_resources(struct emac_priv *priv)
+{
+	struct emac_desc_ring *rx_ring = &priv->rx_ring;
+	struct platform_device *pdev  = priv->pdev;
+	u32 buf_len;
+
+	buf_len = sizeof(struct emac_desc_buffer) * rx_ring->total_cnt;
+
+	rx_ring->desc_buf = kzalloc(buf_len, GFP_KERNEL);
+	if (!rx_ring->desc_buf) {
+		pr_err("Memory allocation failed for the Receive descriptor buffer\n");
+		return -ENOMEM;
+	}
+
+	memset(rx_ring->desc_buf, 0, buf_len);
+
+	/* round up to nearest 4K */
+	rx_ring->total_size = rx_ring->total_cnt * sizeof(struct emac_rx_desc);
+
+	EMAC_ROUNDUP(rx_ring->total_size, 1024);
+
+	rx_ring->desc_addr = dma_alloc_coherent(&pdev->dev,
+							rx_ring->total_size,
+							&rx_ring->desc_dma_addr,
+							GFP_KERNEL);
+	if (!rx_ring->desc_addr) {
+		pr_err("Memory allocation failed for the Receive descriptor ring\n");
+		kfree(rx_ring->desc_buf);
+		return -ENOMEM;
+	}
+
+	memset(rx_ring->desc_addr, 0, rx_ring->total_size);
+
+	rx_ring->head = 0;
+	rx_ring->tail = 0;
+
+	return 0;
+}
+
+/* Name		emac_free_tx_resources
+ * Arguments	priv : pointer to driver private data structure
+ * Return	none
+ * Description	Frees the Tx resources allocated
+ */
+static void emac_free_tx_resources(struct emac_priv *priv)
+{
+	emac_clean_tx_desc_ring(priv);
+	kfree(priv->tx_ring.tx_desc_buf);
+	priv->tx_ring.tx_desc_buf = NULL;
+	dma_free_coherent(&priv->pdev->dev, priv->tx_ring.total_size,
+				priv->tx_ring.desc_addr,
+				priv->tx_ring.desc_dma_addr);
+	priv->tx_ring.desc_addr = NULL;
+}
+
+/* Name		emac_free_rx_resources
+ * Arguments	priv : pointer to driver private data structure
+ * Return	none
+ * Description	Frees the Rx resources allocated
+ */
+static void emac_free_rx_resources(struct emac_priv *priv)
+{
+	emac_clean_rx_desc_ring(priv);
+	kfree(priv->rx_ring.desc_buf);
+	priv->rx_ring.desc_buf = NULL;
+	dma_free_coherent(&priv->pdev->dev, priv->rx_ring.total_size,
+				priv->rx_ring.desc_addr,
+				priv->rx_ring.desc_dma_addr);
+	priv->rx_ring.desc_addr = NULL;
+}
+
+/* Name		emac_open
+ * Arguments	pstNetdev : pointer to net_device structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	This function is called when net interface is made up.
+ *		Setting up Tx and Rx
+ *		resources and making the interface up.
+ */
+static int emac_open(struct net_device *ndev)
+{
+	struct emac_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	ret = emac_alloc_tx_resources(priv);
+	if (ret) {
+		pr_err("Error in setting up the Tx resources\n");
+		goto emac_alloc_tx_resource_fail;
+	}
+
+	ret = emac_alloc_rx_resources(priv);
+	if (ret) {
+		pr_err("Error in setting up the Rx resources\n");
+		goto emac_alloc_rx_resource_fail;
+	}
+
+	ret = emac_up(priv);
+	if (ret) {
+		pr_err("Error in making the net intrface up\n");
+		goto emac_up_fail;
+	}
+	return 0;
+
+emac_up_fail:
+	emac_free_rx_resources(priv);
+emac_alloc_rx_resource_fail:
+	emac_free_tx_resources(priv);
+emac_alloc_tx_resource_fail:
+	return ret;
+}
+
+/* Name		emac_close
+ * Arguments	pstNetdev : pointer to net_device structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	This function is called when net interface is made down.
+ *		It calls the appropriate functions to
+ *		free Tx and Rx resources.
+ */
+static int emac_close(struct net_device *ndev)
+{
+	struct emac_priv *priv = netdev_priv(ndev);
+
+	emac_down(priv);
+	emac_free_tx_resources(priv);
+	emac_free_rx_resources(priv);
+
+	return 0;
+}
+
+/* Name		emac_tx_clean_desc
+ * Arguments	priv : pointer to driver private data structure
+ * Return	1: Cleaned; 0:Failed
+ * Description
+ */
+static int emac_tx_clean_desc(struct emac_priv *priv)
+{
+	struct emac_desc_ring *tx_ring;
+	struct emac_tx_desc_buffer *tx_buf;
+	struct emac_tx_desc *tx_desc;
+
+	u32 i;
+
+	tx_ring = &priv->tx_ring;
+	i = tx_ring->tail;
+
+	while (i != tx_ring->head) {
+		tx_desc = &((struct emac_tx_desc *)tx_ring->desc_addr)[i];
+
+		/* if desc still own by dma, so we quit it */
+		if (tx_desc->OWN)
+			break;
+
+		tx_buf = &tx_ring->tx_desc_buf[i];
+
+		emac_free_tx_buf(priv, i);
+		memset(tx_desc, 0, sizeof(struct emac_tx_desc));
+
+		if (++i == tx_ring->total_cnt)
+			i = 0;
+	}
+
+	tx_ring->tail = i;
+
+	return 0;
+}
+
+static int emac_rx_frame_status(struct emac_priv *priv, struct emac_rx_desc *dsc)
+{
+	/* if last descritpor isn't set, so we drop it*/
+	if (!dsc->LastDescriptor) {
+		netdev_dbg(priv->ndev, "rx LD bit isn't set, drop it.\n");
+		return frame_discard;
+	}
+
+	/*
+	 * A Frame that is less than 64-bytes (from DA thru the FCS field)
+	 * is considered as Runt Frame.
+	 * Most of the Runt Frames happen because of collisions.
+	 */
+	if (dsc->ApplicationStatus & EMAC_RX_FRAME_RUNT) {
+		netdev_dbg(priv->ndev, "rx frame less than 64.\n");
+		return frame_discard;
+	}
+
+	/*
+	 * When the frame fails the CRC check,
+	 * the frame is assumed to have the CRC error
+	 */
+	if (dsc->ApplicationStatus & EMAC_RX_FRAME_CRC_ERR) {
+		netdev_dbg(priv->ndev, "rx frame crc error\n");
+		return frame_discard;
+	}
+
+	/*
+	 * When the length of the frame exceeds
+	 * the Programmed Max Frame Length
+	 */
+	if (dsc->ApplicationStatus & EMAC_RX_FRAME_MAX_LEN_ERR) {
+		netdev_dbg(priv->ndev, "rx frame too long\n");
+		return frame_discard;
+	}
+
+	/*
+	 * frame reception is truncated at that point and
+	 * frame is considered to have Jabber Error
+	 */
+	if (dsc->ApplicationStatus & EMAC_RX_FRAME_JABBER_ERR) {
+		netdev_dbg(priv->ndev, "rx frame has been truncated\n");
+		return frame_discard;
+	}
+
+	/* this bit is only for 802.3 Type Frames */
+	if (dsc->ApplicationStatus & EMAC_RX_FRAME_LENGTH_ERR) {
+		netdev_dbg(priv->ndev, "rx frame length err for 802.3\n");
+		return frame_discard;
+	}
+
+	if (dsc->FramePacketLength <= ETHERNET_FCS_SIZE ||
+	    dsc->FramePacketLength > priv->dma_buf_sz) {
+		netdev_dbg(priv->ndev, "rx frame len too small or too long\n");
+		return frame_discard;
+	}
+	return frame_ok;
+}
+
+/* Name		emac_rx_clean_desc
+ * Arguments	priv : pointer to driver private data structure
+ * Return	1: Cleaned; 0:Failed
+ * Description
+ */
+static int emac_rx_clean_desc(struct emac_priv *priv, int budget)
+{
+	struct emac_desc_ring *rx_ring;
+	struct emac_desc_buffer *rx_buf;
+	struct emac_rx_desc *rx_desc;
+	struct sk_buff *skb = NULL;
+	int status;
+	u32 receive_packet = 0;
+	u32 i;
+	u32 skb_len;
+
+	rx_ring = &priv->rx_ring;
+
+	i = rx_ring->tail;
+
+	while (budget--) {
+		/* get rx desc */
+		rx_desc = &((struct emac_rx_desc *)rx_ring->desc_addr)[i];
+
+		/* if rx_desc still owned by DMA, so we need to wait */
+		if (rx_desc->OWN)
+			break;
+
+		rx_buf = &rx_ring->desc_buf[i];
+		if (!rx_buf->skb)
+			break;
+
+		receive_packet++;
+
+		dma_unmap_single(&priv->pdev->dev, rx_buf->dma_addr,
+				 rx_buf->dma_len, DMA_FROM_DEVICE);
+
+		status = emac_rx_frame_status(priv, rx_desc);
+		if (likely(status != frame_discard)) {
+			skb = rx_buf->skb;
+			skb_len = rx_desc->FramePacketLength - ETHERNET_FCS_SIZE;
+			ecdev_receive(priv->ecdev, skb->data, skb_len);
+		}
+
+		rx_buf->dma_addr = dma_map_single(&priv->pdev->dev,
+						  skb->data,
+						  priv->dma_buf_sz,
+						  DMA_FROM_DEVICE);
+
+		memset(rx_desc, 0, sizeof(struct emac_rx_desc));
+
+		rx_desc->BufferAddr1 = rx_buf->dma_addr;
+		rx_desc->BufferSize1 = rx_buf->dma_len;
+
+		if (++i == rx_ring->total_cnt) {
+			rx_desc->EndRing = 1;
+			i = 0;
+		}
+		dma_wmb();
+		rx_desc->OWN = 1;
+	}
+
+	rx_ring->tail = i;
+	return receive_packet;
+}
+
+/* Name		emac_alloc_rx_desc_buffers
+ * Arguments	priv : pointer to driver private data structure
+ * Return	-1: fail; 0:success
+ * Description
+ */
+static int emac_alloc_rx_desc_buffers(struct emac_priv *priv)
+{
+	struct net_device *ndev = priv->ndev;
+	struct emac_desc_ring *rx_ring = &priv->rx_ring;
+	struct emac_desc_buffer *rx_buf;
+	struct sk_buff *skb = NULL;
+	struct emac_rx_desc *rx_desc;
+
+	for (int i = 0; i < rx_ring->total_cnt; ++i) {
+		rx_buf = &rx_ring->desc_buf[i];
+
+		skb = netdev_alloc_skb_ip_align(ndev, priv->dma_buf_sz);
+		if (!skb) {
+			pr_err("sk_buff allocation failed\n");
+			goto err_out;
+		}
+
+		skb->dev = ndev;
+
+		rx_buf->skb = skb;
+		rx_buf->dma_len = priv->dma_buf_sz;
+		rx_buf->dma_addr = dma_map_single(&priv->pdev->dev,
+						  skb->data,
+						  priv->dma_buf_sz,
+						  DMA_FROM_DEVICE);
+		if (dma_mapping_error(&priv->pdev->dev, rx_buf->dma_addr)) {
+			netdev_err(ndev, "dma mapping_error\n");
+			dev_kfree_skb_any(skb);
+			rx_buf->skb = NULL;
+			goto err_out;
+		}
+
+		rx_desc = &((struct emac_rx_desc *)rx_ring->desc_addr)[i];
+
+		memset(rx_desc, 0, sizeof(struct emac_rx_desc));
+
+		rx_desc->BufferAddr1 = rx_buf->dma_addr;
+		rx_desc->BufferSize1 = rx_buf->dma_len;
+
+		rx_desc->FirstDescriptor = 0;
+		rx_desc->LastDescriptor = 0;
+		if (i == rx_ring->total_cnt - 1)
+			rx_desc->EndRing = 1;
+		dma_wmb();
+		rx_desc->OWN = 1;
+	}
+
+	return 0;
+
+err_out:
+	emac_clean_rx_desc_ring(priv);
+	return -1;
+}
+
+/* Name		emac_tx_mem_map
+ * Arguments	priv : pointer to driver private data structure
+ *		pstSkb : pointer to sk_buff structure passed by upper layer
+ *		max_tx_len : max data len per descriptor
+ *		frag_num : number of fragments in the packet
+ * Return	number of descriptors needed for transmitting packet
+ * Description
+ */
+static int emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb,
+			   u32 max_tx_len, u32 frag_num)
+{
+	struct emac_desc_ring *tx_ring;
+	struct emac_tx_desc_buffer *tx_buf;
+	struct emac_tx_desc *tx_desc;
+	u32 skb_linear_len = skb_headlen(skb);
+	u32 len, i, f, first, buf_idx = 0;
+	phys_addr_t addr;
+
+	tx_ring = &priv->tx_ring;
+
+	i = tx_ring->head;
+	first = i;
+
+	if (++i == tx_ring->total_cnt)
+		i = 0;
+
+	/* if the data is fragmented */
+	for (f = 0; f < frag_num; f++) {
+		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
+
+		len = skb_frag_size(frag);
+
+		buf_idx = (f + 1) % 2;
+
+		/* first frag fill into second buffer of first descriptor */
+		if (f == 0) {
+			tx_buf = &tx_ring->tx_desc_buf[first];
+			tx_desc = &((struct emac_tx_desc *)tx_ring->desc_addr)[first];
+		} else {
+			/* from second frags to more frags,
+			 * we only get new descriptor when it frag num is odd.
+			 */
+			if (!buf_idx) {
+				tx_buf = &tx_ring->tx_desc_buf[i];
+				tx_desc = &((struct emac_tx_desc *)tx_ring->desc_addr)[i];
+			}
+		}
+		tx_buf->buf[buf_idx].dma_len = len;
+
+		addr = skb_frag_dma_map(&priv->pdev->dev, frag, 0,
+				       skb_frag_size(frag),
+				       DMA_TO_DEVICE);
+
+		if (dma_mapping_error(&priv->pdev->dev, addr)) {
+			netdev_err(priv->ndev, "%s dma map page:%d error \n",
+					   __func__, f);
+			goto dma_map_err;
+		}
+		tx_buf->buf[buf_idx].dma_addr = addr;
+
+		tx_buf->buf[buf_idx].map_as_page = true;
+
+		/*
+		 * every desc has two buffer for packet
+		 */
+
+		if (buf_idx) {
+			tx_desc->BufferAddr2 = addr;
+			tx_desc->BufferSize2 = len;
+		} else {
+			tx_desc->BufferAddr1 = addr;
+			tx_desc->BufferSize1 = len;
+
+			if (++i == tx_ring->total_cnt) {
+				tx_desc->EndRing = 1;
+				i = 0;
+			}
+		}
+		/*
+		 * if frag num equal 1, we don't set tx_desc except buffer addr & size
+		 */
+		if (f > 0) {
+			if (f == (frag_num - 1)) {
+				tx_desc->LastSegment = 1;
+				tx_buf->skb = skb;
+			}
+
+			tx_desc->OWN = 1;
+		}
+	}
+
+	/* fill out first descriptor for skb linear data */
+	tx_buf = &tx_ring->tx_desc_buf[first];
+
+	tx_buf->buf[0].dma_len = skb_linear_len;
+
+	addr = dma_map_single(&priv->pdev->dev, skb->data,
+			      skb_linear_len, DMA_TO_DEVICE);
+	if (dma_mapping_error(&priv->pdev->dev, addr)) {
+		netdev_err(priv->ndev, "%s dma mapping_error\n", __func__);
+		goto dma_map_err;
+	}
+
+	tx_buf->buf[0].dma_addr = addr;
+
+	tx_buf->buf[0].buff_addr = skb->data;
+	tx_buf->buf[0].map_as_page = false;
+
+	/* fill tx descriptor */
+	tx_desc = &((struct emac_tx_desc *)tx_ring->desc_addr)[first];
+	tx_desc->BufferAddr1 = addr;
+	tx_desc->BufferSize1 = skb_linear_len;
+	tx_desc->FirstSegment = 1;
+
+	/* if last desc for ring, need to end ring flag */
+	if (first == (tx_ring->total_cnt - 1)) {
+		tx_desc->EndRing = 1;
+	}
+	/*
+	 * if frag num more than 1, that means data need another desc
+	 * so current descriptor isn't last piece of packet data.
+	 */
+	tx_desc->LastSegment = frag_num > 1 ? 0 : 1;
+
+	/* only last descriptor had skb pointer */
+	if (tx_desc->LastSegment)
+		tx_buf->skb = skb;
+
+	tx_desc->OWN = 1;
+
+	dma_wmb();
+
+	emac_dma_start_transmit(priv);
+
+	/* update tx ring head */
+	tx_ring->head = i;
+
+	return 0;
+dma_map_err:
+	return 0;
+}
+
+/* Name		emac_start_xmit
+ * Arguments	pstSkb : pointer to sk_buff structure passed by upper layer
+ *		pstNetdev : pointer to net_device structure
+ * Return	Status: 0 - Success;  non-zero - Fail
+ * Description	This function is called by upper layer to
+ *		handover the Tx packet to the driver
+ *		for sending it to the device.
+ *		Currently this is doing nothing but
+ *		simply to simulate the tx packet handling.
+ */
+static int emac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+{
+	struct emac_priv *priv = netdev_priv(ndev);
+	int nfrags = skb_shinfo(skb)->nr_frags;
+
+	if (unlikely(emac_tx_avail(priv) < nfrags + 1))
+		return NETDEV_TX_BUSY;
+
+	emac_tx_mem_map(priv, skb, MAX_DATA_LEN_TX_DES, nfrags);
+
+	return NETDEV_TX_OK;
+}
+
+static int clk_phase_rgmii_set(struct emac_priv *priv, bool is_tx)
+{
+	u32 val;
+
+	switch (priv->clk_tuning_way) {
+	case CLK_TUNING_BY_REG:
+		val = readl(priv->ctrl_reg);
+		if (is_tx) {
+			val &= ~RGMII_TX_PHASE_MASK;
+			val |= (priv->tx_clk_phase & 0x7) << RGMII_TX_PHASE_OFFSET;
+		} else {
+			val &= ~RGMII_RX_PHASE_MASK;
+			val |= (priv->rx_clk_phase & 0x7) << RGMII_RX_PHASE_OFFSET;
+		}
+		writel(val, priv->ctrl_reg);
+		break;
+	case CLK_TUNING_BY_DLINE:
+		val = readl(priv->dline_reg);
+		if (is_tx) {
+			val &= ~EMAC_TX_DLINE_CODE_MASK;
+			val |= priv->tx_clk_phase << EMAC_TX_DLINE_CODE_OFFSET;
+			val |= EMAC_TX_DLINE_EN;
+		} else {
+			val &= ~EMAC_RX_DLINE_CODE_MASK;
+			val |= priv->rx_clk_phase << EMAC_RX_DLINE_CODE_OFFSET;
+			val |= EMAC_RX_DLINE_EN;
+		}
+		writel(val, priv->dline_reg);
+		break;
+	default:
+		pr_err("wrong clk tuning way:%d !!\n", priv->clk_tuning_way);
+		return -1;
+	}
+	pr_debug("%s tx phase:%d rx phase:%d\n",
+		__func__, priv->tx_clk_phase, priv->rx_clk_phase);
+	return 0;
+}
+
+static int clk_phase_rmii_set(struct emac_priv *priv, bool is_tx)
+{
+	u32 val;
+
+	switch (priv->clk_tuning_way) {
+	case CLK_TUNING_BY_REG:
+		val = readl(priv->ctrl_reg);
+		if (is_tx) {
+			val &= ~RMII_TX_PHASE_MASK;
+			val |= (priv->tx_clk_phase & 0x7) << RMII_TX_PHASE_OFFSET;
+		} else {
+			val &= ~RMII_RX_PHASE_MASK;
+			val |= (priv->rx_clk_phase & 0x7) << RMII_RX_PHASE_OFFSET;
+		}
+		writel(val, priv->ctrl_reg);
+		break;
+	case CLK_TUNING_BY_CLK_REVERT:
+		val = readl(priv->ctrl_reg);
+		if (is_tx) {
+			if (priv->tx_clk_phase == CLK_PHASE_REVERT)
+				val |= RMII_TX_CLK_SEL;
+			else
+				val &= ~RMII_TX_CLK_SEL;
+		} else {
+			if (priv->rx_clk_phase == CLK_PHASE_REVERT)
+				val |= RMII_RX_CLK_SEL;
+			else
+				val &= ~RMII_RX_CLK_SEL;
+		}
+		writel(val, priv->ctrl_reg);
+		break;
+	default:
+		pr_err("wrong clk tuning way:%d !!\n", priv->clk_tuning_way);
+		return -1;
+	}
+	pr_debug("%s tx phase:%d rx phase:%d\n",
+		__func__, priv->tx_clk_phase, priv->rx_clk_phase);
+	return 0;
+}
+
+static int clk_phase_set(struct emac_priv *priv, bool is_tx)
+{
+	if (priv->clk_tuning_enable) {
+		if (emac_is_rmii(priv)) {
+			clk_phase_rmii_set(priv, is_tx);
+		} else {
+			clk_phase_rgmii_set(priv, is_tx);
+		}
+	}
+
+	return 0;
+}
+
+static int emac_mii_reset(struct mii_bus *bus)
+{
+	struct emac_priv *priv = bus->priv;
+	struct device *dev = &priv->pdev->dev;
+	int rst_gpio, ldo_gpio;
+	int active_state;
+	u32 delays[3] = {0};
+
+	if (dev->of_node) {
+		struct device_node *np = dev->of_node;
+
+		if (!np)
+			return 0;
+
+		ldo_gpio = of_get_named_gpio(np, "emac,ldo-gpio", 0);
+		if (ldo_gpio >= 0) {
+			if (gpio_request(ldo_gpio, "mdio-ldo"))
+				return 0;
+
+			gpio_direction_output(ldo_gpio, 1);
+		}
+
+		rst_gpio = of_get_named_gpio(np, "emac,reset-gpio", 0);
+		if (rst_gpio < 0)
+			return 0;
+
+		active_state = of_property_read_bool(np,
+						     "emac,reset-active-low");
+		of_property_read_u32_array(np,
+					   "emac,reset-delays-us", delays, 3);
+
+		if (gpio_request(rst_gpio, "mdio-reset"))
+			return 0;
+
+		gpio_direction_output(rst_gpio,
+		                      active_state ? 1 : 0);
+		if (delays[0])
+		        msleep(DIV_ROUND_UP(delays[0], 1000));
+
+		gpio_set_value(rst_gpio, active_state ? 0 : 1);
+		if (delays[1])
+		        msleep(DIV_ROUND_UP(delays[1], 1000));
+
+		gpio_set_value(rst_gpio, active_state ? 1 : 0);
+		if (delays[2])
+		        msleep(DIV_ROUND_UP(delays[2], 1000));
+        }
+	return 0;
+}
+
+static int emac_mii_read(struct mii_bus *bus, int phy_addr, int regnum)
+{
+	struct emac_priv *priv = bus->priv;
+	u32 cmd = 0;
+	u32 val;
+
+	cmd |= phy_addr & 0x1F;
+	cmd |= (regnum & 0x1F) << 5;
+	cmd |= MREGBIT_START_MDIO_TRANS | MREGBIT_MDIO_READ_WRITE;
+
+	emac_wr(priv, MAC_MDIO_DATA, 0x0);
+	emac_wr(priv, MAC_MDIO_CONTROL, cmd);
+
+	if (readl_poll_timeout(priv->iobase + MAC_MDIO_CONTROL,
+			       val, !((val >> 15) & 0x1), 100, 10000))
+		return -EBUSY;
+
+	val = emac_rd(priv, MAC_MDIO_DATA);
+	return val;
+}
+
+static int emac_mii_write(struct mii_bus *bus, int phy_addr, int regnum,
+			    u16 value)
+{
+	struct emac_priv *priv = bus->priv;
+	u32 cmd = 0;
+	u32 val;
+
+	emac_wr(priv, MAC_MDIO_DATA, value);
+
+	cmd |= phy_addr & 0x1F;
+	cmd |= (regnum & 0x1F) << 5;
+	cmd |= MREGBIT_START_MDIO_TRANS;
+
+	emac_wr(priv, MAC_MDIO_CONTROL, cmd);
+
+	if (readl_poll_timeout(priv->iobase + MAC_MDIO_CONTROL,
+			       val, !((val >> 15) & 0x1), 100, 10000))
+		return -EBUSY;
+
+	return 0;
+}
+
+static void emac_adjust_link(struct net_device *dev)
+{
+	struct phy_device *phydev = dev->phydev;
+	struct emac_priv *priv = netdev_priv(dev);
+	bool link_changed = false;
+	u32 ctrl;
+
+	if (!phydev)
+		return;
+
+	if (phydev->link) {
+		ctrl = emac_rd(priv, MAC_GLOBAL_CONTROL);
+
+		/* Now we make sure that we can be in full duplex mode
+		 * If not, we operate in half-duplex mode.
+		 */
+		if (phydev->duplex != priv->duplex) {
+			link_changed = true;
+
+			if (!phydev->duplex)
+				ctrl &= ~MREGBIT_FULL_DUPLEX_MODE;
+			else
+				ctrl |= MREGBIT_FULL_DUPLEX_MODE;
+			priv->duplex = phydev->duplex;
+		}
+
+		if (phydev->speed != priv->speed) {
+			link_changed = true;
+
+			ctrl &= ~MREGBIT_SPEED;
+
+			switch (phydev->speed) {
+			case SPEED_1000:
+				ctrl |= MREGBIT_SPEED_1000M;
+				break;
+			case SPEED_100:
+				ctrl |= MREGBIT_SPEED_100M;
+				break;
+			case SPEED_10:
+				ctrl |= MREGBIT_SPEED_10M;
+				break;
+			default:
+				pr_err("broken speed: %d\n", phydev->speed);
+				phydev->speed = SPEED_UNKNOWN;
+				break;
+			}
+			if (phydev->speed != SPEED_UNKNOWN) {
+				priv->speed = phydev->speed;
+			}
+		}
+
+		emac_wr(priv, MAC_GLOBAL_CONTROL, ctrl);
+
+		if (!priv->link) {
+			priv->link = true;
+			link_changed = true;
+		}
+	} else if (priv->link) {
+		priv->link = false;
+		link_changed = true;
+		priv->duplex = DUPLEX_UNKNOWN;
+		priv->speed = SPEED_UNKNOWN;
+	}
+
+	if (link_changed) {
+		phy_print_status(phydev);
+		if (priv->ecdev) {
+			ecdev_set_link(priv->ecdev, priv->link ? 1 : 0);
+		}
+	}
+}
+
+static int emac_phy_connect(struct net_device *dev)
+{
+	struct phy_device *phydev;
+	struct device_node *np;
+	struct emac_priv *priv = netdev_priv(dev);
+
+	np = of_parse_phandle(priv->pdev->dev.of_node, "phy-handle", 0);
+	if (!np && of_phy_is_fixed_link(priv->pdev->dev.of_node))
+		np = of_node_get(priv->pdev->dev.of_node);
+	if (!np)
+		return -ENODEV;
+
+	of_get_phy_mode(np, &priv->phy_interface);
+	pr_info("priv phy_interface = %d\n", priv->phy_interface);
+
+	emac_phy_interface_config(priv);
+
+	phydev = of_phy_connect(dev, np,
+				&emac_adjust_link, 0, priv->phy_interface);
+	if (IS_ERR_OR_NULL(phydev)) {
+		pr_err("Could not attach to PHY\n");
+		if (!phydev)
+			return -ENODEV;
+		return PTR_ERR(phydev);
+	}
+
+	pr_info("%s:  %s: attached to PHY (UID 0x%x)"
+			" Link = %d\n", __func__,
+			dev->name, phydev->phy_id, phydev->link);
+
+	/* Indicate that the MAC is responsible for PHY PM */
+	phydev->mac_managed_pm = true;
+	dev->phydev = phydev;
+
+	clk_phase_set(priv, TX_PHASE);
+	clk_phase_set(priv, RX_PHASE);
+	return 0;
+}
+
+static int emac_mdio_init(struct emac_priv *priv)
+{
+	struct device_node *mii_np;
+	struct device *dev = &priv->pdev->dev;
+	int ret;
+
+	mii_np = of_get_child_by_name(dev->of_node, "mdio-bus");
+	if (!mii_np) {
+		if (of_phy_is_fixed_link(dev->of_node)) {
+			if ((of_phy_register_fixed_link(dev->of_node) < 0)) {
+				return -ENODEV;
+			}
+			dev_dbg(dev, "find fixed link\n");
+			return 0;
+		}
+
+		dev_err(dev, "no %s child node found", "mdio-bus");
+		return -ENODEV;
+	}
+
+	if (!of_device_is_available(mii_np)) {
+		ret = -ENODEV;
+		goto err_put_node;
+	}
+
+	priv->mii = devm_mdiobus_alloc(dev);
+	if (!priv->mii) {
+		ret = -ENOMEM;
+		goto err_put_node;
+	}
+	priv->mii->priv = priv;
+	priv->mii->name = "emac mii";
+	priv->mii->reset = emac_mii_reset;
+	priv->mii->read = emac_mii_read;
+	priv->mii->write = emac_mii_write;
+	snprintf(priv->mii->id, MII_BUS_ID_SIZE, "%s",
+			priv->pdev->name);
+	priv->mii->parent = dev;
+	priv->mii->phy_mask = 0xffffffff;
+	ret = of_mdiobus_register(priv->mii, mii_np);
+	if (ret) {
+		dev_err(dev, "Failed to register mdio bus.\n");
+		goto err_put_node;
+	}
+
+	priv->phy = phy_find_first(priv->mii);
+	if (!priv->phy) {
+		dev_err(dev, "no PHY found\n");
+		return -ENODEV;
+	}
+
+err_put_node:
+	of_node_put(mii_np);
+	return ret;
+}
+
+static int emac_mdio_deinit(struct emac_priv *priv)
+{
+	if (!priv->mii)
+		return 0;
+
+	mdiobus_unregister(priv->mii);
+	return 0;
+}
+
+static const struct net_device_ops emac_netdev_ops = {
+	.ndo_open               = emac_open,
+	.ndo_stop               = emac_close,
+	.ndo_start_xmit         = emac_start_xmit,
+};
+
+static int emac_config_dt(struct platform_device *pdev, struct emac_priv *priv)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct resource *res;
+	u8 mac_addr[ETH_ALEN] = {0};
+	u32 tx_phase, rx_phase;
+	u32 ctrl_reg;
+	int ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->iobase = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(priv->iobase)) {
+		dev_err(&pdev->dev, "failed to io remap res reg 0\n");
+		return -ENOMEM;
+	}
+
+	if (of_property_read_u32(np, "k1x,apmu-base-reg", &priv->apmu_base)) {
+		priv->apmu_base = PMUA_BASE_REG;
+	}
+
+	if (of_property_read_u32(np, "ctrl-reg", &ctrl_reg)) {
+		dev_err(&pdev->dev, "cannot find ctrl register in device tree\n");
+		return -EINVAL;
+	}
+
+	priv->ctrl_reg = ioremap(priv->apmu_base + ctrl_reg, 4);
+
+	if (of_property_read_u32(np, "tx-threshold",
+				 &priv->tx_threshold)) {
+		priv->tx_threshold = DEFAULT_TX_THRESHOLD;
+		dev_dbg(&pdev->dev, "%s tx_threshold using default value:%d \n",
+			__func__, priv->tx_threshold);
+	}
+
+	if (of_property_read_u32(np, "rx-threshold",
+				 &priv->rx_threshold)) {
+		priv->rx_threshold = DEFAULT_RX_THRESHOLD;
+		dev_dbg(&pdev->dev, "%s rx_threshold using default value:%d \n",
+			__func__, priv->rx_threshold);
+	}
+
+	if (of_property_read_u32(np, "tx-ring-num",
+				 &priv->tx_ring_num)) {
+		priv->tx_ring_num = DEFAULT_TX_RING_NUM;
+		dev_dbg(&pdev->dev, "%s tx_ring_num using default value:%d \n",
+			__func__, priv->tx_ring_num);
+	}
+
+	if (of_property_read_u32(np, "rx-ring-num",
+				 &priv->rx_ring_num)) {
+		priv->rx_ring_num = DEFAULT_RX_RING_NUM;
+		dev_dbg(&pdev->dev, "%s rx_ring_num using default value:%d \n",
+			__func__, priv->rx_ring_num);
+	}
+
+	if (of_property_read_u32(np, "dma-burst-len",
+				 &priv->dma_burst_len)) {
+		priv->dma_burst_len = DEFAULT_DMA_BURST_LEN;
+		dev_dbg(&pdev->dev, "%s dma_burst_len using default value:%d \n",
+			__func__, priv->dma_burst_len);
+	} else {
+		if (priv->dma_burst_len <= 0 || priv->dma_burst_len > 7) {
+			dev_err(&pdev->dev, "%s burst len illegal, use default vallue:%d\n",
+				__func__, DEFAULT_DMA_BURST_LEN);
+			priv->dma_burst_len = DEFAULT_DMA_BURST_LEN;
+		}
+	}
+
+	if (of_property_read_bool(np, "ref-clock-from-phy")) {
+		priv->ref_clk_frm_soc = 0;
+		dev_dbg(&pdev->dev, "%s ref clock from external phy \n", __func__);
+	} else
+		priv->ref_clk_frm_soc = 1;
+
+
+	ret = of_get_mac_address(np, mac_addr);
+	if (ret) {
+		if (ret == -EPROBE_DEFER)
+			return ret;
+
+		dev_info(&pdev->dev, "Using random mac address\n");
+		eth_hw_addr_random(priv->ndev);
+	} else {
+		eth_hw_addr_set(priv->ndev, mac_addr);
+	}
+
+	dev_dbg(&pdev->dev, "%s tx-threshold:%d rx_therhold:%d tx_ring_num:%d rx_ring_num:%d dma-bur_len:%d\n",
+		__func__, priv->tx_threshold, priv->rx_threshold, priv->tx_ring_num,
+		priv->rx_ring_num, priv->dma_burst_len);
+
+	priv->clk_tuning_enable = of_property_read_bool(np, "clk-tuning-enable");
+	if (priv->clk_tuning_enable) {
+		if (of_property_read_bool(np, "clk-tuning-by-reg"))
+			priv->clk_tuning_way = CLK_TUNING_BY_REG;
+		else if (of_property_read_bool(np, "clk-tuning-by-clk-revert"))
+			priv->clk_tuning_way = CLK_TUNING_BY_CLK_REVERT;
+		else if (of_property_read_bool(np, "clk-tuning-by-delayline")) {
+			priv->clk_tuning_way = CLK_TUNING_BY_DLINE;
+			if (of_property_read_u32(np, "dline-reg", &ctrl_reg)) {
+				dev_err(&pdev->dev, "cannot find delayline register in device tree\n");
+				return -EINVAL;
+			}
+			priv->dline_reg = ioremap(priv->apmu_base + ctrl_reg, 4);
+		} else
+			priv->clk_tuning_way = CLK_TUNING_BY_REG;
+
+		if (of_property_read_u32(np, "tx-phase", &tx_phase))
+			priv->tx_clk_phase = TXCLK_PHASE_DEFAULT;
+		else
+			priv->tx_clk_phase = tx_phase;
+
+		if (of_property_read_u32(np, "rx-phase", &rx_phase))
+			priv->rx_clk_phase = RXCLK_PHASE_DEFAULT;
+		else
+			priv->rx_clk_phase = rx_phase;
+	}
+
+	return 0;
+}
+
+void emac_ec_poll(struct net_device *ndev)
+{
+	struct emac_priv *priv = netdev_priv(ndev);
+
+	emac_rx_clean_desc(priv, 100);
+	emac_tx_clean_desc(priv);
+}
+
+static int emac_probe(struct platform_device *pdev)
+{
+	struct emac_priv *priv;
+	struct net_device *ndev = NULL;
+	int ret;
+
+	ndev = alloc_etherdev(sizeof(struct emac_priv));
+	if (!ndev)
+		return -ENOMEM;
+
+	ndev->hw_features = NETIF_F_SG;
+	ndev->features |= ndev->hw_features;
+	priv = netdev_priv(ndev);
+	priv->ndev = ndev;
+	priv->pdev = pdev;
+	platform_set_drvdata(pdev, priv);
+
+	ret = emac_config_dt(pdev, priv);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to config dt\n");
+		goto err_netdev;
+	}
+
+	ndev->watchdog_timeo = 5 * HZ;
+	ndev->base_addr = (unsigned long)priv->iobase;
+	ndev->netdev_ops = &emac_netdev_ops;
+
+	priv->mac_clk = devm_clk_get(&pdev->dev, "emac-clk");
+	if (IS_ERR(priv->mac_clk)) {
+		dev_err(&pdev->dev, "emac clock not found.\n");
+		ret = PTR_ERR(priv->mac_clk);
+		goto err_netdev;
+	}
+
+	ret = clk_prepare_enable(priv->mac_clk);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to enable emac clock: %d\n",
+			ret);
+		goto err_netdev;
+	}
+
+	if (priv->ref_clk_frm_soc) {
+		priv->phy_clk = devm_clk_get(&pdev->dev, "phy-clk");
+		if (IS_ERR(priv->phy_clk)) {
+			dev_err(&pdev->dev, "phy clock not found.\n");
+			ret = PTR_ERR(priv->phy_clk);
+			goto mac_clk_disable;
+		}
+
+		ret = clk_prepare_enable(priv->phy_clk);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "failed to enable phy clock: %d\n",
+				ret);
+			goto mac_clk_disable;
+		}
+	}
+
+	priv->reset = devm_reset_control_get_optional(&pdev->dev, NULL);
+	if (IS_ERR(priv->reset)) {
+		dev_err(&pdev->dev, "Failed to get emac's resets\n");
+		goto phy_clk_disable;
+	}
+
+	reset_control_deassert(priv->reset);
+
+	emac_sw_init(priv);
+
+	ret = emac_mdio_init(priv);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to init mdio.\n");
+		goto reset_assert;
+	}
+
+	SET_NETDEV_DEV(ndev, &pdev->dev);
+
+	priv->ecdev = ecdev_offer(priv->ndev, emac_ec_poll, THIS_MODULE);
+	if(!priv->ecdev) {
+		dev_err(&pdev->dev, "failed to offer EtherCAT device\n");
+		goto err_mdio_deinit;
+	} else {
+		pr_info("success to offer EtherCAT device\n");
+	}
+
+	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+
+	ret = ecdev_open(priv->ecdev);
+	if (ret) {
+		ecdev_withdraw(priv->ecdev);
+		goto err_mdio_deinit;
+	}
+	return 0;
+
+err_mdio_deinit:
+	emac_mdio_deinit(priv);
+reset_assert:
+	reset_control_assert(priv->reset);
+phy_clk_disable:
+	if (priv->ref_clk_frm_soc)
+		clk_disable_unprepare(priv->phy_clk);
+mac_clk_disable:
+	clk_disable_unprepare(priv->mac_clk);
+err_netdev:
+	free_netdev(ndev);
+    dev_info(&pdev->dev, "emac_probe failed ret = %d.\n", ret);
+	return ret;
+}
+
+static int emac_remove(struct platform_device *pdev)
+{
+	struct emac_priv *priv = platform_get_drvdata(pdev);
+	ecdev_close(priv->ecdev);
+	ecdev_withdraw(priv->ecdev);
+
+	emac_reset_hw(priv);
+	free_netdev(priv->ndev);
+	emac_mdio_deinit(priv);
+	reset_control_assert(priv->reset);
+	clk_disable_unprepare(priv->mac_clk);
+	if (priv->ref_clk_frm_soc)
+		clk_disable_unprepare(priv->phy_clk);
+	return 0;
+}
+
+static void emac_shutdown(struct platform_device *pdev)
+{
+}
+
+static const struct of_device_id emac_of_match[] = {
+	{ .compatible = "spacemit,k1x-ec-emac" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, emac_of_match);
+
+static struct platform_driver ec_emac_driver = {
+	.probe = emac_probe,
+	.remove = emac_remove,
+	.shutdown = emac_shutdown,
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = of_match_ptr(emac_of_match),
+#ifdef CONFIG_SPACEMIT_PARALLEL_BOOTING
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+#endif
+	},
+};
+
+module_platform_driver(ec_emac_driver); 
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Ethernet driver with EtherCAT support for Spacemit k1x Emac");
+MODULE_ALIAS("platform:spacemit_eth");
diff --git a/drivers/net/ethercat/device/K1/ec_k1x_emac.h b/drivers/net/ethercat/device/K1/ec_k1x_emac.h
new file mode 100644
index 000000000000..fba5b3bbaa42
--- /dev/null
+++ b/drivers/net/ethercat/device/K1/ec_k1x_emac.h
@@ -0,0 +1,654 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _K1X_EMAC_H_
+#define _K1X_EMAC_H_
+#include <linux/bitops.h>
+#include <linux/ptp_clock_kernel.h>
+
+/* EtherCAT includes */
+#include "../include/globals.h"
+#include "ecdev.h"
+
+#define PHY_INTF_RGMII					BIT(2)
+
+/*
+ * only valid for rmii mode
+ * 0: ref clock from external phy
+ * 1: ref clock from soc
+ */
+#define REF_CLK_SEL					BIT(3)
+
+/*
+ * emac function clock select
+ * 0: 208M
+ * 1: 312M
+ */
+#define FUNC_CLK_SEL					BIT(4)
+
+/* only valid for rmii, invert tx clk */
+#define RMII_TX_CLK_SEL					BIT(6)
+
+/* only valid for rmii, invert rx clk */
+#define RMII_RX_CLK_SEL					BIT(7)
+
+/*
+ * only valid for rgmiii
+ * 0: tx clk from rx clk
+ * 1: tx clk from soc
+ */
+#define RGMII_TX_CLK_SEL				BIT(8)
+
+#define PHY_IRQ_EN					BIT(12)
+#define AXI_SINGLE_ID					BIT(13)
+
+#define RMII_TX_PHASE_OFFSET				(16)
+#define RMII_TX_PHASE_MASK				GENMASK(18, 16)
+#define RMII_RX_PHASE_OFFSET				(20)
+#define RMII_RX_PHASE_MASK				GENMASK(22, 20)
+
+#define RGMII_TX_PHASE_OFFSET				(24)
+#define RGMII_TX_PHASE_MASK				GENMASK(26, 24)
+#define RGMII_RX_PHASE_OFFSET				(28)
+#define RGMII_RX_PHASE_MASK				GENMASK(30, 28)
+
+#define EMAC_RX_DLINE_EN				BIT(0)
+#define EMAC_RX_DLINE_STEP_OFFSET			(4)
+#define EMAC_RX_DLINE_STEP_MASK				GENMASK(5, 4)
+#define EMAC_RX_DLINE_CODE_OFFSET			(8)
+#define EMAC_RX_DLINE_CODE_MASK				GENMASK(15, 8)
+
+#define EMAC_TX_DLINE_EN				BIT(16)
+#define EMAC_TX_DLINE_STEP_OFFSET			(20)
+#define EMAC_TX_DLINE_STEP_MASK				GENMASK(21, 20)
+#define EMAC_TX_DLINE_CODE_OFFSET			(24)
+#define EMAC_TX_DLINE_CODE_MASK				GENMASK(31, 24)
+
+/* DMA register set */
+#define DMA_CONFIGURATION				0x0000
+#define DMA_CONTROL					0x0004
+#define DMA_STATUS_IRQ					0x0008
+#define DMA_INTERRUPT_ENABLE				0x000C
+
+#define DMA_TRANSMIT_AUTO_POLL_COUNTER			0x0010
+#define DMA_TRANSMIT_POLL_DEMAND			0x0014
+#define DMA_RECEIVE_POLL_DEMAND				0x0018
+
+#define DMA_TRANSMIT_BASE_ADDRESS			0x001C
+#define DMA_RECEIVE_BASE_ADDRESS			0x0020
+#define DMA_MISSED_FRAME_COUNTER			0x0024
+#define DMA_STOP_FLUSH_COUNTER				0x0028
+
+#define DMA_RECEIVE_IRQ_MITIGATION_CTRL			0x002C
+
+#define DMA_CURRENT_TRANSMIT_DESCRIPTOR_POINTER		0x0030
+#define DMA_CURRENT_TRANSMIT_BUFFER_POINTER		0x0034
+#define DMA_CURRENT_RECEIVE_DESCRIPTOR_POINTER		0x0038
+#define DMA_CURRENT_RECEIVE_BUFFER_POINTER		0x003C
+
+/* MAC Register set */
+#define MAC_GLOBAL_CONTROL				0x0100
+#define MAC_TRANSMIT_CONTROL				0x0104
+#define MAC_RECEIVE_CONTROL				0x0108
+#define MAC_MAXIMUM_FRAME_SIZE				0x010C
+#define MAC_TRANSMIT_JABBER_SIZE			0x0110
+#define MAC_RECEIVE_JABBER_SIZE				0x0114
+#define MAC_ADDRESS_CONTROL				0x0118
+#define MAC_MDIO_CLK_DIV				0x011C
+#define MAC_ADDRESS1_HIGH				0x0120
+#define MAC_ADDRESS1_MED				0x0124
+#define MAC_ADDRESS1_LOW				0x0128
+#define MAC_ADDRESS2_HIGH				0x012C
+#define MAC_ADDRESS2_MED				0x0130
+#define MAC_ADDRESS2_LOW				0x0134
+#define MAC_ADDRESS3_HIGH				0x0138
+#define MAC_ADDRESS3_MED				0x013C
+#define MAC_ADDRESS3_LOW				0x0140
+#define MAC_ADDRESS4_HIGH				0x0144
+#define MAC_ADDRESS4_MED				0x0148
+#define MAC_ADDRESS4_LOW				0x014C
+#define MAC_MULTICAST_HASH_TABLE1			0x0150
+#define MAC_MULTICAST_HASH_TABLE2			0x0154
+#define MAC_MULTICAST_HASH_TABLE3			0x0158
+#define MAC_MULTICAST_HASH_TABLE4			0x015C
+#define MAC_FC_CONTROL					0x0160
+#define MAC_FC_PAUSE_FRAME_GENERATE			0x0164
+#define MAC_FC_SOURCE_ADDRESS_HIGH			0x0168
+#define MAC_FC_SOURCE_ADDRESS_MED			0x016C
+#define MAC_FC_SOURCE_ADDRESS_LOW			0x0170
+#define MAC_FC_DESTINATION_ADDRESS_HIGH			0x0174
+#define MAC_FC_DESTINATION_ADDRESS_MED			0x0178
+#define MAC_FC_DESTINATION_ADDRESS_LOW			0x017C
+#define MAC_FC_PAUSE_TIME_VALUE				0x0180
+#define MAC_MDIO_CONTROL				0x01A0
+#define MAC_MDIO_DATA					0x01A4
+#define MAC_RX_STATCTR_CONTROL				0x01A8
+#define MAC_RX_STATCTR_DATA_HIGH			0x01AC
+#define MAC_RX_STATCTR_DATA_LOW				0x01B0
+#define MAC_TX_STATCTR_CONTROL				0x01B4
+#define MAC_TX_STATCTR_DATA_HIGH			0x01B8
+#define MAC_TX_STATCTR_DATA_LOW				0x01BC
+#define MAC_TRANSMIT_FIFO_ALMOST_FULL			0x01C0
+#define MAC_TRANSMIT_PACKET_START_THRESHOLD		0x01C4
+#define MAC_RECEIVE_PACKET_START_THRESHOLD		0x01C8
+#define MAC_STATUS_IRQ					0x01E0
+#define MAC_INTERRUPT_ENABLE				0x01E4
+
+/* DMA_CONFIGURATION (0x0000) register bit info
+ * 0-DMA controller in normal operation mode,
+ * 1-DMA controller reset to default state,
+ * clearing all internal state information
+ */
+#define MREGBIT_SOFTWARE_RESET				BIT(0)
+#define MREGBIT_BURST_1WORD				BIT(1)
+#define MREGBIT_BURST_2WORD				BIT(2)
+#define MREGBIT_BURST_4WORD				BIT(3)
+#define MREGBIT_BURST_8WORD				BIT(4)
+#define MREGBIT_BURST_16WORD				BIT(5)
+#define MREGBIT_BURST_32WORD				BIT(6)
+#define MREGBIT_BURST_64WORD				BIT(7)
+#define MREGBIT_BURST_LENGTH				GENMASK(7, 1)
+#define MREGBIT_DESCRIPTOR_SKIP_LENGTH			GENMASK(12, 8)
+/* For Receive and Transmit DMA operate in Big-Endian mode for Descriptors. */
+#define MREGBIT_DESCRIPTOR_BYTE_ORDERING		BIT(13)
+#define MREGBIT_BIG_LITLE_ENDIAN			BIT(14)
+#define MREGBIT_TX_RX_ARBITRATION			BIT(15)
+#define MREGBIT_WAIT_FOR_DONE				BIT(16)
+#define MREGBIT_STRICT_BURST				BIT(17)
+#define MREGBIT_DMA_64BIT_MODE				BIT(18)
+
+/* DMA_CONTROL (0x0004) register bit info */
+#define MREGBIT_START_STOP_TRANSMIT_DMA			BIT(0)
+#define MREGBIT_START_STOP_RECEIVE_DMA			BIT(1)
+
+/* DMA_STATUS_IRQ (0x0008) register bit info */
+#define MREGBIT_TRANSMIT_TRANSFER_DONE_IRQ		BIT(0)
+#define MREGBIT_TRANSMIT_DES_UNAVAILABLE_IRQ		BIT(1)
+#define MREGBIT_TRANSMIT_DMA_STOPPED_IRQ		BIT(2)
+#define MREGBIT_RECEIVE_TRANSFER_DONE_IRQ		BIT(4)
+#define MREGBIT_RECEIVE_DES_UNAVAILABLE_IRQ		BIT(5)
+#define MREGBIT_RECEIVE_DMA_STOPPED_IRQ			BIT(6)
+#define MREGBIT_RECEIVE_MISSED_FRAME_IRQ		BIT(7)
+#define MREGBIT_MAC_IRQ					BIT(8)
+#define MREGBIT_TRANSMIT_DMA_STATE			GENMASK(18, 16)
+#define MREGBIT_RECEIVE_DMA_STATE			GENMASK(23, 20)
+
+/* DMA_INTERRUPT_ENABLE ( 0x000C) register bit info */
+#define MREGBIT_TRANSMIT_TRANSFER_DONE_INTR_ENABLE	BIT(0)
+#define MREGBIT_TRANSMIT_DES_UNAVAILABLE_INTR_ENABLE	BIT(1)
+#define MREGBIT_TRANSMIT_DMA_STOPPED_INTR_ENABLE	BIT(2)
+#define MREGBIT_RECEIVE_TRANSFER_DONE_INTR_ENABLE	BIT(4)
+#define MREGBIT_RECEIVE_DES_UNAVAILABLE_INTR_ENABLE	BIT(5)
+#define MREGBIT_RECEIVE_DMA_STOPPED_INTR_ENABLE		BIT(6)
+#define MREGBIT_RECEIVE_MISSED_FRAME_INTR_ENABLE	BIT(7)
+#define MREGBIT_MAC_INTR_ENABLE				BIT(8)
+
+/* DMA RECEIVE IRQ MITIGATION CONTROL */
+#define MREGBIT_RECEIVE_IRQ_FRAME_COUNTER_MSK		GENMASK(7, 0)
+#define MREGBIT_RECEIVE_IRQ_TIMEOUT_COUNTER_OFST	(8)
+#define MREGBIT_RECEIVE_IRQ_TIMEOUT_COUNTER_MSK		GENMASK(27, 8)
+#define MRGEBIT_RECEIVE_IRQ_FRAME_COUNTER_MODE		BIT(30)
+#define MRGEBIT_RECEIVE_IRQ_MITIGATION_ENABLE		BIT(31)
+
+/* MAC_GLOBAL_CONTROL (0x0100) register bit info */
+#define MREGBIT_SPEED					GENMASK(1, 0)
+#define MREGBIT_SPEED_10M				0x0
+#define MREGBIT_SPEED_100M				BIT(0)
+#define MREGBIT_SPEED_1000M				BIT(1)
+#define MREGBIT_FULL_DUPLEX_MODE			BIT(2)
+#define MREGBIT_RESET_RX_STAT_COUNTERS			BIT(3)
+#define MREGBIT_RESET_TX_STAT_COUNTERS			BIT(4)
+#define MREGBIT_UNICAST_WAKEUP_MODE			BIT(8)
+#define MREGBIT_MAGIC_PACKET_WAKEUP_MODE		BIT(9)
+
+/* MAC_TRANSMIT_CONTROL (0x0104) register bit info */
+#define MREGBIT_TRANSMIT_ENABLE				BIT(0)
+#define MREGBIT_INVERT_FCS				BIT(1)
+#define MREGBIT_DISABLE_FCS_INSERT			BIT(2)
+#define MREGBIT_TRANSMIT_AUTO_RETRY			BIT(3)
+#define MREGBIT_IFG_LEN					GENMASK(6, 4)
+#define MREGBIT_PREAMBLE_LENGTH				GENMASK(9, 7)
+
+/* MAC_RECEIVE_CONTROL (0x0108) register bit info */
+#define MREGBIT_RECEIVE_ENABLE				BIT(0)
+#define MREGBIT_DISABLE_FCS_CHECK			BIT(1)
+#define MREGBIT_STRIP_FCS				BIT(2)
+#define MREGBIT_STORE_FORWARD				BIT(3)
+#define MREGBIT_STATUS_FIRST				BIT(4)
+#define MREGBIT_PASS_BAD_FRAMES				BIT(5)
+#define MREGBIT_ACOOUNT_VLAN				BIT(6)
+
+/* MAC_MAXIMUM_FRAME_SIZE (0x010C) register bit info */
+#define MREGBIT_MAX_FRAME_SIZE				GENMASK(13, 0)
+
+/* MAC_TRANSMIT_JABBER_SIZE (0x0110) register bit info */
+#define MREGBIT_TRANSMIT_JABBER_SIZE			GENMASK(15, 0)
+
+/* MAC_RECEIVE_JABBER_SIZE (0x0114) register bit info */
+#define MREGBIT_RECEIVE_JABBER_SIZE			GENMASK(15, 0)
+
+/* MAC_ADDRESS_CONTROL	 (0x0118) register bit info */
+#define MREGBIT_MAC_ADDRESS1_ENABLE			BIT(0)
+#define MREGBIT_MAC_ADDRESS2_ENABLE			BIT(1)
+#define MREGBIT_MAC_ADDRESS3_ENABLE			BIT(2)
+#define MREGBIT_MAC_ADDRESS4_ENABLE			BIT(3)
+#define MREGBIT_INVERSE_MAC_ADDRESS1_ENABLE		BIT(4)
+#define MREGBIT_INVERSE_MAC_ADDRESS2_ENABLE		BIT(5)
+#define MREGBIT_INVERSE_MAC_ADDRESS3_ENABLE		BIT(6)
+#define MREGBIT_INVERSE_MAC_ADDRESS4_ENABLE		BIT(7)
+#define MREGBIT_PROMISCUOUS_MODE			BIT(8)
+
+/* MAC_ADDRESSx_HIGH (0x0120) register bit info */
+#define MREGBIT_MAC_ADDRESS1_01_BYTE			GENMASK(7, 0)
+#define MREGBIT_MAC_ADDRESS1_02_BYTE			GENMASK(15, 8)
+/* MAC_ADDRESSx_MED (0x0124) register bit info */
+#define MREGBIT_MAC_ADDRESS1_03_BYTE			GENMASK(7, 0)
+#define MREGBIT_MAC_ADDRESS1_04_BYTE			GENMASK(15, 8)
+/* MAC_ADDRESSx_LOW (0x0128) register bit info */
+#define MREGBIT_MAC_ADDRESS1_05_BYTE			GENMASK(7, 0)
+#define MREGBIT_MAC_ADDRESS1_06_BYTE			GENMASK(15, 8)
+
+/* MAC_FC_CONTROL (0x0160) register bit info */
+#define MREGBIT_FC_DECODE_ENABLE			BIT(0)
+#define MREGBIT_FC_GENERATION_ENABLE			BIT(1)
+#define MREGBIT_AUTO_FC_GENERATION_ENABLE		BIT(2)
+#define MREGBIT_MULTICAST_MODE				BIT(3)
+#define MREGBIT_BLOCK_PAUSE_FRAMES			BIT(4)
+
+/* MAC_FC_PAUSE_FRAME_GENERATE (0x0164) register bit info */
+#define MREGBIT_GENERATE_PAUSE_FRAME			BIT(0)
+
+/* MAC_FC_SRC/DST_ADDRESS_HIGH (0x0168) register bit info */
+#define MREGBIT_MAC_ADDRESS_01_BYTE			GENMASK(7, 0)
+#define MREGBIT_MAC_ADDRESS_02_BYTE			GENMASK(15, 8)
+/* MAC_FC_SRC/DST_ADDRESS_MED (0x016C) register bit info */
+#define MREGBIT_MAC_ADDRESS_03_BYTE			GENMASK(7, 0)
+#define MREGBIT_MAC_ADDRESS_04_BYTE			GENMASK(15, 8)
+/* MAC_FC_SRC/DSTD_ADDRESS_LOW (0x0170) register bit info */
+#define MREGBIT_MAC_ADDRESS_05_BYTE			GENMASK(7, 0)
+#define MREGBIT_MAC_ADDRESS_06_BYTE			GENMASK(15, 8)
+
+/* MAC_FC_PAUSE_TIME_VALUE (0x0180) register bit info */
+#define MREGBIT_MAC_FC_PAUSE_TIME			GENMASK(15, 0)
+
+/* MAC_MDIO_CONTROL (0x01A0) register bit info */
+#define MREGBIT_PHY_ADDRESS				GENMASK(4, 0)
+#define MREGBIT_REGISTER_ADDRESS			GENMASK(9, 5)
+#define MREGBIT_MDIO_READ_WRITE				BIT(10)
+#define MREGBIT_START_MDIO_TRANS			BIT(15)
+
+/* MAC_MDIO_DATA (0x01A4) register bit info */
+#define MREGBIT_MDIO_DATA				GENMASK(15, 0)
+
+/* MAC_RX_STATCTR_CONTROL (0x01A8) register bit info */
+#define MREGBIT_RX_COUNTER_NUMBER			GENMASK(4, 0)
+#define MREGBIT_START_RX_COUNTER_READ			BIT(15)
+
+/* MAC_RX_STATCTR_DATA_HIGH (0x01AC) register bit info */
+#define MREGBIT_RX_STATCTR_DATA_HIGH			GENMASK(15, 0)
+/* MAC_RX_STATCTR_DATA_LOW (0x01B0) register bit info */
+#define MREGBIT_RX_STATCTR_DATA_LOW			GENMASK(15, 0)
+
+/* MAC_TX_STATCTR_CONTROL (0x01B4) register bit info */
+#define MREGBIT_TX_COUNTER_NUMBER			GENMASK(4, 0)
+#define MREGBIT_START_TX_COUNTER_READ			BIT(15)
+
+/* MAC_TX_STATCTR_DATA_HIGH (0x01B8) register bit info */
+#define MREGBIT_TX_STATCTR_DATA_HIGH			GENMASK(15, 0)
+/* MAC_TX_STATCTR_DATA_LOW (0x01BC) register bit info */
+#define MREGBIT_TX_STATCTR_DATA_LOW			GENMASK(15, 0)
+
+/* MAC_TRANSMIT_FIFO_ALMOST_FULL (0x01C0) register bit info */
+#define MREGBIT_TX_FIFO_AF				GENMASK(13, 0)
+
+/* MAC_TRANSMIT_PACKET_START_THRESHOLD (0x01C4) register bit info */
+#define MREGBIT_TX_PACKET_START_THRESHOLD		GENMASK(13, 0)
+
+/* MAC_RECEIVE_PACKET_START_THRESHOLD (0x01C8) register bit info */
+#define MREGBIT_RX_PACKET_START_THRESHOLD		GENMASK(13, 0)
+
+/* MAC_STATUS_IRQ  (0x01E0) register bit info */
+#define MREGBIT_MAC_UNDERRUN_IRQ			BIT(0)
+#define MREGBIT_MAC_JABBER_IRQ				BIT(1)
+
+/* MAC_INTERRUPT_ENABLE (0x01E4) register bit info */
+#define MREGBIT_MAC_UNDERRUN_INTERRUPT_ENABLE		BIT(0)
+#define MREGBIT_JABBER_INTERRUPT_ENABLE			BIT(1)
+
+/* Receive Descriptors */
+/* MAC_RECEIVE_DESCRIPTOR0 () register bit info */
+#define MREGBIT_FRAME_LENGTH				GENMASK(13, 0)
+#define MREGBIT_APPLICATION_STATUS			GENMASK(28, 14)
+#define MREGBIT_LAST_DESCRIPTOR				BIT(29)
+#define MREGBIT_FIRST_DESCRIPTOR			BIT(30)
+#define MREGBIT_OWN_BIT					BIT(31)
+
+/* MAC_RECEIVE_DESCRIPTOR1 () register bit info */
+#define MREGBIT_BUFFER1_SIZE				GENMASK(11, 0)
+#define MREGBIT_BUFFER2_SIZE				GENMASK(23, 12)
+#define MREGBIT_SECOND_ADDRESS_CHAINED			BIT(25)
+#define MREGBIT_END_OF_RING				BIT(26)
+
+/* MAC_RECEIVE_DESCRIPTOR2 () register bit info */
+#define MREGBIT_BUFFER_ADDRESS1				GENMASK(31, 0)
+
+/* MAC_RECEIVE_DESCRIPTOR3 () register bit info */
+#define MREGBIT_BUFFER_ADDRESS1				GENMASK(31, 0)
+
+/* Transmit Descriptors */
+/* TD_TRANSMIT_DESCRIPTOR0 () register bit info */
+#define MREGBIT_TX_PACKET_STATUS			GENMASK(29, 0)
+#define MREGBIT_OWN_BIT					BIT(31)
+
+/* TD_TRANSMIT_DESCRIPTOR1 () register bit info */
+#define MREGBIT_BUFFER1_SIZE				GENMASK(11, 0)
+#define MREGBIT_BUFFER2_SIZE				GENMASK(23, 12)
+#define MREGBIT_FORCE_EOP_ERROR				BIT(24)
+#define MREGBIT_SECOND_ADDRESS_CHAINED			BIT(25)
+#define MREGBIT_END_OF_RING				BIT(26)
+#define MREGBIT_DISABLE_PADDING				BIT(27)
+#define MREGBIT_ADD_CRC_DISABLE				BIT(28)
+#define MREGBIT_FIRST_SEGMENT				BIT(29)
+#define MREGBIT_LAST_SEGMENT				BIT(30)
+#define MREGBIT_INTERRUPT_ON_COMPLETION			BIT(31)
+
+/* TD_TRANSMIT_DESCRIPTOR2 () register bit info */
+#define MREGBIT_BUFFER_ADDRESS1				GENMASK(31, 0)
+
+/* TD_TRANSMIT_DESCRIPTOR3 () register bit info */
+#define MREGBIT_BUFFER_ADDRESS1				GENMASK(31, 0)
+
+/* RX frame status */
+#define EMAC_RX_FRAME_ALIGN_ERR				BIT(0)
+#define EMAC_RX_FRAME_RUNT				BIT(1)
+#define EMAC_RX_FRAME_ETHERNET_TYPE			BIT(2)
+#define EMAC_RX_FRAME_VLAN				BIT(3)
+#define EMAC_RX_FRAME_MULTICAST				BIT(4)
+#define EMAC_RX_FRAME_BROADCAST				BIT(5)
+#define EMAC_RX_FRAME_CRC_ERR				BIT(6)
+#define EMAC_RX_FRAME_MAX_LEN_ERR			BIT(7)
+#define EMAC_RX_FRAME_JABBER_ERR			BIT(8)
+#define EMAC_RX_FRAME_LENGTH_ERR			BIT(9)
+#define EMAC_RX_FRAME_MAC_ADDR1_MATCH			BIT(10)
+#define EMAC_RX_FRAME_MAC_ADDR2_MATCH			BIT(11)
+#define EMAC_RX_FRAME_MAC_ADDR3_MATCH			BIT(12)
+#define EMAC_RX_FRAME_MAC_ADDR4_MATCH			BIT(13)
+#define EMAC_RX_FRAME_PAUSE_CTRL			BIT(14)
+
+/* emac ptp 1588 register */
+#define PTP_1588_CTRL					(0x300)
+#define TX_TIMESTAMP_EN					BIT(1)
+#define RX_TIMESTAMP_EN					BIT(2)
+#define RX_PTP_PKT_TYPE_OFST				3
+#define RX_PTP_PKT_TYPE_MSK				GENMASK(5, 3)
+
+#define PTP_INRC_ATTR					(0x304)
+#define INRC_VAL_MSK					GENMASK(23, 0)
+#define INCR_PERIOD_OFST				24
+#define INCR_PERIOD_MSK					GENMASK(31, 24)
+
+#define PTP_ETH_TYPE					(0x308)
+#define PTP_ETH_TYPE_MSK				GENMASK(15, 0)
+
+#define PTP_MSG_ID					(0x30c)
+
+#define PTP_UDP_PORT					(0x310)
+#define PTP_UDP_PORT_MSK				GENMASK(15, 0)
+
+/* read current system time from controller */
+#define SYS_TIME_GET_LOW				(0x320)
+#define SYS_TIME_GET_HI					(0x324)
+
+#define SYS_TIME_ADJ_LOW				(0x328)
+#define SYS_TIME_LOW_MSK				GENMASK(31, 0)
+#define SYS_TIME_ADJ_HI					(0x32c)
+#define SYS_TIME_IS_NEG					BIT(31)
+
+#define TX_TIMESTAMP_LOW				(0x330)
+#define TX_TIMESTAMP_HI					(0x334)
+
+#define RX_TIMESTAMP_LOW				(0x340)
+#define RX_TIMESTAMP_HI					(0x344)
+
+#define RX_PTP_PKT_ATTR_LOW				(0x348)
+#define PTP_SEQ_ID_MSK					GENMASK(15, 0)
+#define PTP_SRC_ID_LOW_OFST				16
+#define PTP_SRC_ID_LOW_MSK				GENMASK(31, 16)
+
+#define RX_PTP_PKT_ATTR_MID				(0x34c)
+#define PTP_SRC_ID_MID_MSK				GENMASK(31, 0)
+
+#define RX_PTP_PKT_ATTR_HI				(0x350)
+#define PTP_SRC_ID_HI_MSK				GENMASK(31, 0)
+
+#define PTP_1588_IRQ_STS				(0x360)
+#define PTP_1588_IRQ_EN					(0x364)
+#define PTP_TX_TIMESTAMP				BIT(0)
+#define PTP_RX_TIMESTAMP				BIT(1)
+
+/* emac ptp register */
+
+#define EMAC_DEFAULT_BUFSIZE				1536
+#define EMAC_RX_BUF_2K					2048
+#define EMAC_RX_BUF_4K					4096
+
+#define MAX_DATA_PWR_TX_DES				11
+#define MAX_DATA_LEN_TX_DES				2048 //2048=1<<11
+
+#define MAX_TX_STATS_NUM				12
+#define MAX_RX_STATS_NUM				25
+
+/* The sizes (in bytes) of a ethernet packet */
+#define ETHERNET_HEADER_SIZE				14
+#define MAXIMUM_ETHERNET_FRAME_SIZE			1518  //With FCS
+#define MINIMUM_ETHERNET_FRAME_SIZE			64  //With FCS
+#define ETHERNET_FCS_SIZE				4
+#define MAXIMUM_ETHERNET_PACKET_SIZE \
+		(MAXIMUM_ETHERNET_FRAME_SIZE - ETHERNET_FCS_SIZE)
+
+#define MINIMUM_ETHERNET_PACKET_SIZE \
+		(MINIMUM_ETHERNET_FRAME_SIZE - ETHERNET_FCS_SIZE)
+
+#define CRC_LENGTH					ETHERNET_FCS_SIZE
+#define MAX_JUMBO_FRAME_SIZE				0x3F00
+
+#define TX_STORE_FORWARD_MODE				0x5EE
+
+#define EMAC_TX_FRAMES					64
+/* 40ms */
+#define EMAC_TX_COAL_TIMEOUT				40000
+
+#define EMAC_RX_FRAMES					64
+
+/* axi clk 312M, 1us = 312 cycle,
+ * every packet almost take 120us when operate at 100Mbps
+ * so we set 5 packet delay time which 600us as rx coal timeout
+ */
+#define EMAC_RX_COAL_TIMEOUT				(600 * 312)
+
+/* only works for sizes that are powers of 2 */
+#define EMAC_ROUNDUP(i, size) ((i) = (((i) + (size) - 1) & ~((size) - 1)))
+
+/* number of descriptors are required for len */
+#define EMAC_TXD_COUNT(S, X) (((S) >> (X)) + 1)
+
+/* calculate the number of descriptors unused */
+#define EMAC_DESC_UNUSED(R) \
+	((((R)->nxt_clean > (R)->nxt_use) ? 0 : (R)->total_cnt) + \
+	(R)->nxt_clean - (R)->nxt_use - 1)
+
+typedef struct ifreq  st_ifreq, *pst_ifreq;
+
+enum rx_frame_status {
+	frame_ok = 0,
+	frame_discard,
+	frame_max,
+};
+
+enum rx_ptp_type {
+	PTP_V2_L2_ONLY = 0x0,
+	PTP_V1_L4_ONLY = 0x1,
+	PTP_V2_L2_L4  = 0x2,
+};
+
+enum ptp_event_msg_id {
+	MSG_SYNC = 0x00,
+	MSG_DELAY_REQ = 0x01,
+	MSG_PDELAY_REQ = 0x02,
+	MSG_PDELAY_RESP = 0x03,
+	ALL_EVENTS = 0x03020100,
+};
+
+enum emac_state {
+	EMAC_DOWN,
+	EMAC_RESET_REQUESTED,
+	EMAC_RESETING,
+	EMAC_TASK_SCHED,
+	EMAC_STATE_MAX,
+};
+
+/* Receive Descriptor structure */
+struct emac_rx_desc {
+	u32 FramePacketLength:14;
+	u32 ApplicationStatus:15;
+	u32 LastDescriptor:1;
+	u32 FirstDescriptor:1;
+	u32 OWN:1;
+
+	u32 BufferSize1:12;
+	u32 BufferSize2:12;
+	u32 Reserved1:1;
+	u32 SecondAddressChained:1;
+	u32 EndRing:1;
+	u32 Reserved2:3;
+	u32 rx_timestamp:1;
+	u32 ptp_pkt:1;
+
+	u32 BufferAddr1;
+	u32 BufferAddr2;
+};
+
+/* Transmit Descriptor */
+struct emac_tx_desc {
+	u32 FramePacketStatus:30;
+	u32 tx_timestamp:1;
+	u32 OWN:1;
+
+	u32 BufferSize1:12;
+	u32 BufferSize2:12;
+	u32 ForceEOPError:1;
+	u32 SecondAddressChained:1;
+	u32 EndRing:1;
+	u32 DisablePadding:1;
+	u32 AddCRCDisable:1;
+	u32 FirstSegment:1;
+	u32 LastSegment:1;
+	u32 InterruptOnCompletion:1;
+
+	u32 BufferAddr1;
+	u32 BufferAddr2;
+};
+
+struct desc_buf {
+	u64 dma_addr;
+	void *buff_addr;
+	u16 dma_len;
+	u8 map_as_page;
+};
+
+/* Descriptor buffer structure */
+struct emac_tx_desc_buffer {
+	struct sk_buff *skb;
+	struct desc_buf buf[2];
+	u8 timestamped;
+};
+
+/* Descriptor buffer structure */
+struct emac_desc_buffer {
+	struct sk_buff *skb;
+	u64 dma_addr;
+	void *buff_addr;
+	unsigned long ulTimeStamp;
+	u16 dma_len;
+	u8 map_as_page;
+	u8 timestamped;
+};
+
+/* Descriptor ring structure */
+struct emac_desc_ring {
+	/* virtual memory address to the descriptor ring memory */
+	void *desc_addr;
+	/* physical address of the descriptor ring */
+	dma_addr_t desc_dma_addr;
+	/* length of descriptor ring in bytes */
+	u32 total_size;
+	/* number of descriptors in the ring */
+	u32 total_cnt;
+	/* next descriptor to associate a buffer with */
+	u32 head;
+	/* next descriptor to check for DD status bit */
+	u32 tail;
+	/* array of buffer information structs */
+	union {
+		struct emac_desc_buffer *desc_buf;
+		struct emac_tx_desc_buffer *tx_desc_buf;
+	};
+};
+
+struct emac_priv;
+
+struct emac_priv {
+	u32 dma_buf_sz;
+	u32 wol;
+	struct emac_desc_ring tx_ring;
+	struct emac_desc_ring rx_ring;
+	struct net_device *ndev;
+	struct platform_device *pdev;
+	struct clk *mac_clk;
+	struct clk *phy_clk;
+	struct reset_control *reset;
+	void __iomem *iobase;
+	u32 apmu_base;
+	int link;
+	int duplex;
+	int speed;
+	phy_interface_t phy_interface;
+	struct mii_bus *mii;
+	struct phy_device *phy;
+	u8 tx_clk_phase;
+	u8 rx_clk_phase;
+	u8 clk_tuning_way;
+	bool clk_tuning_enable;
+	unsigned long state;
+	u32 tx_threshold;
+	u32 rx_threshold;
+	u32 tx_ring_num;
+	u32 rx_ring_num;
+	u32 dma_burst_len;
+	u32 ref_clk_frm_soc;
+	void __iomem *ctrl_reg;
+	void __iomem *dline_reg;
+	s32 lpm_qos;
+	ec_device_t *ecdev;
+};
+
+
+static inline void emac_wr(struct emac_priv *priv, u32 reg, u32 val)
+{
+	writel(val, (priv->iobase + reg));
+}
+
+static inline int emac_rd(struct emac_priv *priv, u32 reg)
+{
+	return readl(priv->iobase + reg);
+}
+
+static int emac_init_hw(struct emac_priv *priv);
+static int emac_reset_hw(struct emac_priv *priv);
+static int emac_set_mac_addr(struct emac_priv *priv, const unsigned char *addr);
+static int emac_down(struct emac_priv *priv);
+
+static int emac_alloc_tx_resources(struct emac_priv *priv);
+static int emac_alloc_rx_resources(struct emac_priv *priv);
+static void emac_free_tx_resources(struct emac_priv *priv);
+static void emac_free_rx_resources(struct emac_priv *priv);
+#endif /* _K1X_EMAC_H_ */
diff --git a/drivers/net/ethercat/device/K3/Makefile b/drivers/net/ethercat/device/K3/Makefile
new file mode 100644
index 000000000000..10c0b0ad6d83
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/Makefile
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_EC_K3_GMAC) += ec_stmmac.o
+
+ec_stmmac-objs := stmmac_main-ethercat.o \
+                  stmmac_ethtool-ethercat.o \
+                  stmmac_mdio-ethercat.o \
+                  ring_mode-ethercat.o \
+                  chain_mode-ethercat.o \
+                  dwmac_lib-ethercat.o \
+                  dwmac1000_core-ethercat.o \
+                  dwmac1000_dma-ethercat.o \
+                  dwmac100_core-ethercat.o \
+                  dwmac100_dma-ethercat.o \
+                  enh_desc-ethercat.o \
+                  norm_desc-ethercat.o \
+                  mmc_core-ethercat.o \
+                  stmmac_hwtstamp-ethercat.o \
+                  stmmac_ptp-ethercat.o \
+                  dwmac4_descs-ethercat.o \
+                  dwmac4_dma-ethercat.o \
+                  dwmac4_lib-ethercat.o \
+                  dwmac4_core-ethercat.o \
+                  dwmac5-ethercat.o \
+                  hwif-ethercat.o \
+                  stmmac_tc-ethercat.o \
+                  dwxgmac2_core-ethercat.o \
+                  dwxgmac2_dma-ethercat.o \
+                  dwxgmac2_descs-ethercat.o \
+                  stmmac_xdp-ethercat.o \
+                  stmmac_est-ethercat.o \
+                  stmmac_fpe-ethercat.o \
+                  stmmac_vlan-ethercat.o \
+                  stmmac_platform-ethercat.o \
+                  dwmac-spacemit-ethqos-ethercat.o
diff --git a/drivers/net/ethercat/device/K3/chain_mode-ethercat.c b/drivers/net/ethercat/device/K3/chain_mode-ethercat.c
new file mode 100644
index 000000000000..da654d066283
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/chain_mode-ethercat.c
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  Specialised functions for managing Chained mode
+
+  Copyright(C) 2011  STMicroelectronics Ltd
+
+  It defines all the functions used to handle the normal/enhanced
+  descriptors in case of the DMA is configured to work in chained or
+  in ring mode.
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include "stmmac-ethercat.h"
+
+static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
+		     int csum)
+{
+	unsigned int nopaged_len = skb_headlen(skb);
+	struct stmmac_priv *priv = tx_q->priv_data;
+	unsigned int entry = tx_q->cur_tx;
+	unsigned int bmax, des2;
+	unsigned int i = 1, len;
+	struct dma_desc *desc;
+
+	desc = tx_q->dma_tx + entry;
+
+	if (priv->plat->enh_desc)
+		bmax = BUF_SIZE_8KiB;
+	else
+		bmax = BUF_SIZE_2KiB;
+
+	len = nopaged_len - bmax;
+
+	des2 = dma_map_single(priv->device, skb->data,
+			      bmax, DMA_TO_DEVICE);
+	desc->des2 = cpu_to_le32(des2);
+	if (dma_mapping_error(priv->device, des2))
+		return -1;
+	tx_q->tx_skbuff_dma[entry].buf = des2;
+	tx_q->tx_skbuff_dma[entry].len = bmax;
+	/* do not close the descriptor and do not set own bit */
+	stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum, STMMAC_CHAIN_MODE,
+			0, false, skb->len);
+
+	while (len != 0) {
+		tx_q->tx_skbuff[entry] = NULL;
+		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+		desc = tx_q->dma_tx + entry;
+
+		if (len > bmax) {
+			des2 = dma_map_single(priv->device,
+					      (skb->data + bmax * i),
+					      bmax, DMA_TO_DEVICE);
+			desc->des2 = cpu_to_le32(des2);
+			if (dma_mapping_error(priv->device, des2))
+				return -1;
+			tx_q->tx_skbuff_dma[entry].buf = des2;
+			tx_q->tx_skbuff_dma[entry].len = bmax;
+			stmmac_prepare_tx_desc(priv, desc, 0, bmax, csum,
+					STMMAC_CHAIN_MODE, 1, false, skb->len);
+			len -= bmax;
+			i++;
+		} else {
+			des2 = dma_map_single(priv->device,
+					      (skb->data + bmax * i), len,
+					      DMA_TO_DEVICE);
+			desc->des2 = cpu_to_le32(des2);
+			if (dma_mapping_error(priv->device, des2))
+				return -1;
+			tx_q->tx_skbuff_dma[entry].buf = des2;
+			tx_q->tx_skbuff_dma[entry].len = len;
+			/* last descriptor can be set now */
+			stmmac_prepare_tx_desc(priv, desc, 0, len, csum,
+					STMMAC_CHAIN_MODE, 1, true, skb->len);
+			len = 0;
+		}
+	}
+
+	tx_q->cur_tx = entry;
+
+	return entry;
+}
+
+static unsigned int is_jumbo_frm(int len, int enh_desc)
+{
+	unsigned int ret = 0;
+
+	if ((enh_desc && (len > BUF_SIZE_8KiB)) ||
+	    (!enh_desc && (len > BUF_SIZE_2KiB))) {
+		ret = 1;
+	}
+
+	return ret;
+}
+
+static void init_dma_chain(void *des, dma_addr_t phy_addr,
+				  unsigned int size, unsigned int extend_desc)
+{
+	/*
+	 * In chained mode the des3 points to the next element in the ring.
+	 * The latest element has to point to the head.
+	 */
+	int i;
+	dma_addr_t dma_phy = phy_addr;
+
+	if (extend_desc) {
+		struct dma_extended_desc *p = (struct dma_extended_desc *)des;
+		for (i = 0; i < (size - 1); i++) {
+			dma_phy += sizeof(struct dma_extended_desc);
+			p->basic.des3 = cpu_to_le32((unsigned int)dma_phy);
+			p++;
+		}
+		p->basic.des3 = cpu_to_le32((unsigned int)phy_addr);
+
+	} else {
+		struct dma_desc *p = (struct dma_desc *)des;
+		for (i = 0; i < (size - 1); i++) {
+			dma_phy += sizeof(struct dma_desc);
+			p->des3 = cpu_to_le32((unsigned int)dma_phy);
+			p++;
+		}
+		p->des3 = cpu_to_le32((unsigned int)phy_addr);
+	}
+}
+
+static void refill_desc3(struct stmmac_rx_queue *rx_q, struct dma_desc *p)
+{
+	struct stmmac_priv *priv = rx_q->priv_data;
+
+	if (priv->hwts_rx_en && !priv->extend_desc)
+		/* NOTE: Device will overwrite des3 with timestamp value if
+		 * 1588-2002 time stamping is enabled, hence reinitialize it
+		 * to keep explicit chaining in the descriptor.
+		 */
+		p->des3 = cpu_to_le32((unsigned int)(rx_q->dma_rx_phy +
+				      (((rx_q->dirty_rx) + 1) %
+				       priv->dma_conf.dma_rx_size) *
+				      sizeof(struct dma_desc)));
+}
+
+static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p)
+{
+	struct stmmac_priv *priv = tx_q->priv_data;
+	unsigned int entry = tx_q->dirty_tx;
+
+	if (tx_q->tx_skbuff_dma[entry].last_segment && !priv->extend_desc &&
+	    priv->hwts_tx_en)
+		/* NOTE: Device will overwrite des3 with timestamp value if
+		 * 1588-2002 time stamping is enabled, hence reinitialize it
+		 * to keep explicit chaining in the descriptor.
+		 */
+		p->des3 = cpu_to_le32((unsigned int)((tx_q->dma_tx_phy +
+				      ((tx_q->dirty_tx + 1) %
+				       priv->dma_conf.dma_tx_size))
+				      * sizeof(struct dma_desc)));
+}
+
+const struct stmmac_mode_ops ec_chain_mode_ops = {
+	.init = init_dma_chain,
+	.is_jumbo_frm = is_jumbo_frm,
+	.jumbo_frm = jumbo_frm,
+	.refill_desc3 = refill_desc3,
+	.clean_desc3 = clean_desc3,
+};
diff --git a/drivers/net/ethercat/device/K3/common-ethercat.h b/drivers/net/ethercat/device/K3/common-ethercat.h
new file mode 100644
index 000000000000..d6cab22aa274
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/common-ethercat.h
@@ -0,0 +1,640 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  STMMAC Common Header File
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __COMMON_H__
+#define __COMMON_H__
+
+#include <linux/etherdevice.h>
+#include <linux/netdevice.h>
+#include <linux/stmmac.h>
+#include <linux/phy.h>
+#include <linux/pcs/pcs-xpcs.h>
+#include <linux/module.h>
+#if IS_ENABLED(CONFIG_VLAN_8021Q)
+#define STMMAC_VLAN_TAG_USED
+#include <linux/if_vlan.h>
+#endif
+
+#include "descs-ethercat.h"
+#include "hwif-ethercat.h"
+#include "mmc-ethercat.h"
+
+/* Synopsys Core versions */
+#define	DWMAC_CORE_3_40		0x34
+#define	DWMAC_CORE_3_50		0x35
+#define	DWMAC_CORE_3_70		0x37
+#define	DWMAC_CORE_4_00		0x40
+#define DWMAC_CORE_4_10		0x41
+#define DWMAC_CORE_5_00		0x50
+#define DWMAC_CORE_5_10		0x51
+#define DWMAC_CORE_5_20		0x52
+#define DWXGMAC_CORE_2_10	0x21
+#define DWXGMAC_CORE_2_20	0x22
+#define DWXLGMAC_CORE_2_00	0x20
+
+/* Device ID */
+#define DWXGMAC_ID		0x76
+#define DWXLGMAC_ID		0x27
+
+#define STMMAC_CHAN0	0	/* Always supported and default for all chips */
+
+/* TX and RX Descriptor Length, these need to be power of two.
+ * TX descriptor length less than 64 may cause transmit queue timed out error.
+ * RX descriptor length less than 64 may cause inconsistent Rx chain error.
+ */
+#define DMA_MIN_TX_SIZE		64
+#define DMA_MAX_TX_SIZE		1024
+#define DMA_DEFAULT_TX_SIZE	512
+#define DMA_MIN_RX_SIZE		64
+#define DMA_MAX_RX_SIZE		1024
+#define DMA_DEFAULT_RX_SIZE	512
+#define STMMAC_GET_ENTRY(x, size)	((x + 1) & (size - 1))
+
+#undef FRAME_FILTER_DEBUG
+/* #define FRAME_FILTER_DEBUG */
+
+struct stmmac_q_tx_stats {
+	u64_stats_t tx_bytes;
+	u64_stats_t tx_set_ic_bit;
+	u64_stats_t tx_tso_frames;
+	u64_stats_t tx_tso_nfrags;
+};
+
+struct stmmac_napi_tx_stats {
+	u64_stats_t tx_packets;
+	u64_stats_t tx_pkt_n;
+	u64_stats_t poll;
+	u64_stats_t tx_clean;
+	u64_stats_t tx_set_ic_bit;
+};
+
+struct stmmac_txq_stats {
+	/* Updates protected by tx queue lock. */
+	struct u64_stats_sync q_syncp;
+	struct stmmac_q_tx_stats q;
+
+	/* Updates protected by NAPI poll logic. */
+	struct u64_stats_sync napi_syncp;
+	struct stmmac_napi_tx_stats napi;
+} ____cacheline_aligned_in_smp;
+
+struct stmmac_napi_rx_stats {
+	u64_stats_t rx_bytes;
+	u64_stats_t rx_packets;
+	u64_stats_t rx_pkt_n;
+	u64_stats_t poll;
+};
+
+struct stmmac_rxq_stats {
+	/* Updates protected by NAPI poll logic. */
+	struct u64_stats_sync napi_syncp;
+	struct stmmac_napi_rx_stats napi;
+} ____cacheline_aligned_in_smp;
+
+/* Updates on each CPU protected by not allowing nested irqs. */
+struct stmmac_pcpu_stats {
+	struct u64_stats_sync syncp;
+	u64_stats_t rx_normal_irq_n[MTL_MAX_RX_QUEUES];
+	u64_stats_t tx_normal_irq_n[MTL_MAX_TX_QUEUES];
+};
+
+/* Extra statistic and debug information exposed by ethtool */
+struct stmmac_extra_stats {
+	/* Transmit errors */
+	unsigned long tx_underflow ____cacheline_aligned;
+	unsigned long tx_carrier;
+	unsigned long tx_losscarrier;
+	unsigned long vlan_tag;
+	unsigned long tx_deferred;
+	unsigned long tx_vlan;
+	unsigned long tx_jabber;
+	unsigned long tx_frame_flushed;
+	unsigned long tx_payload_error;
+	unsigned long tx_ip_header_error;
+	unsigned long tx_collision;
+	/* Receive errors */
+	unsigned long rx_desc;
+	unsigned long sa_filter_fail;
+	unsigned long overflow_error;
+	unsigned long ipc_csum_error;
+	unsigned long rx_collision;
+	unsigned long rx_crc_errors;
+	unsigned long dribbling_bit;
+	unsigned long rx_length;
+	unsigned long rx_mii;
+	unsigned long rx_multicast;
+	unsigned long rx_gmac_overflow;
+	unsigned long rx_watchdog;
+	unsigned long da_rx_filter_fail;
+	unsigned long sa_rx_filter_fail;
+	unsigned long rx_missed_cntr;
+	unsigned long rx_overflow_cntr;
+	unsigned long rx_vlan;
+	unsigned long rx_split_hdr_pkt_n;
+	/* Tx/Rx IRQ error info */
+	unsigned long tx_undeflow_irq;
+	unsigned long tx_process_stopped_irq;
+	unsigned long tx_jabber_irq;
+	unsigned long rx_overflow_irq;
+	unsigned long rx_buf_unav_irq;
+	unsigned long rx_process_stopped_irq;
+	unsigned long rx_watchdog_irq;
+	unsigned long tx_early_irq;
+	unsigned long fatal_bus_error_irq;
+	/* Tx/Rx IRQ Events */
+	unsigned long rx_early_irq;
+	unsigned long threshold;
+	unsigned long irq_receive_pmt_irq_n;
+	/* MMC info */
+	unsigned long mmc_tx_irq_n;
+	unsigned long mmc_rx_irq_n;
+	unsigned long mmc_rx_csum_offload_irq_n;
+	/* EEE */
+	unsigned long irq_tx_path_in_lpi_mode_n;
+	unsigned long irq_tx_path_exit_lpi_mode_n;
+	unsigned long irq_rx_path_in_lpi_mode_n;
+	unsigned long irq_rx_path_exit_lpi_mode_n;
+	unsigned long phy_eee_wakeup_error_n;
+	/* Extended RDES status */
+	unsigned long ip_hdr_err;
+	unsigned long ip_payload_err;
+	unsigned long ip_csum_bypassed;
+	unsigned long ipv4_pkt_rcvd;
+	unsigned long ipv6_pkt_rcvd;
+	unsigned long no_ptp_rx_msg_type_ext;
+	unsigned long ptp_rx_msg_type_sync;
+	unsigned long ptp_rx_msg_type_follow_up;
+	unsigned long ptp_rx_msg_type_delay_req;
+	unsigned long ptp_rx_msg_type_delay_resp;
+	unsigned long ptp_rx_msg_type_pdelay_req;
+	unsigned long ptp_rx_msg_type_pdelay_resp;
+	unsigned long ptp_rx_msg_type_pdelay_follow_up;
+	unsigned long ptp_rx_msg_type_announce;
+	unsigned long ptp_rx_msg_type_management;
+	unsigned long ptp_rx_msg_pkt_reserved_type;
+	unsigned long ptp_frame_type;
+	unsigned long ptp_ver;
+	unsigned long timestamp_dropped;
+	unsigned long av_pkt_rcvd;
+	unsigned long av_tagged_pkt_rcvd;
+	unsigned long vlan_tag_priority_val;
+	unsigned long l3_filter_match;
+	unsigned long l4_filter_match;
+	unsigned long l3_l4_filter_no_match;
+	/* PCS */
+	unsigned long irq_pcs_ane_n;
+	unsigned long irq_pcs_link_n;
+	unsigned long irq_rgmii_n;
+	unsigned long pcs_link;
+	unsigned long pcs_duplex;
+	unsigned long pcs_speed;
+	/* debug register */
+	unsigned long mtl_tx_status_fifo_full;
+	unsigned long mtl_tx_fifo_not_empty;
+	unsigned long mmtl_fifo_ctrl;
+	unsigned long mtl_tx_fifo_read_ctrl_write;
+	unsigned long mtl_tx_fifo_read_ctrl_wait;
+	unsigned long mtl_tx_fifo_read_ctrl_read;
+	unsigned long mtl_tx_fifo_read_ctrl_idle;
+	unsigned long mac_tx_in_pause;
+	unsigned long mac_tx_frame_ctrl_xfer;
+	unsigned long mac_tx_frame_ctrl_idle;
+	unsigned long mac_tx_frame_ctrl_wait;
+	unsigned long mac_tx_frame_ctrl_pause;
+	unsigned long mac_gmii_tx_proto_engine;
+	unsigned long mtl_rx_fifo_fill_level_full;
+	unsigned long mtl_rx_fifo_fill_above_thresh;
+	unsigned long mtl_rx_fifo_fill_below_thresh;
+	unsigned long mtl_rx_fifo_fill_level_empty;
+	unsigned long mtl_rx_fifo_read_ctrl_flush;
+	unsigned long mtl_rx_fifo_read_ctrl_read_data;
+	unsigned long mtl_rx_fifo_read_ctrl_status;
+	unsigned long mtl_rx_fifo_read_ctrl_idle;
+	unsigned long mtl_rx_fifo_ctrl_active;
+	unsigned long mac_rx_frame_ctrl_fifo;
+	unsigned long mac_gmii_rx_proto_engine;
+	/* EST */
+	unsigned long mtl_est_cgce;
+	unsigned long mtl_est_hlbs;
+	unsigned long mtl_est_hlbf;
+	unsigned long mtl_est_btre;
+	unsigned long mtl_est_btrlm;
+	unsigned long max_sdu_txq_drop[MTL_MAX_TX_QUEUES];
+	unsigned long mtl_est_txq_hlbf[MTL_MAX_TX_QUEUES];
+	unsigned long mtl_est_txq_hlbs[MTL_MAX_TX_QUEUES];
+	/* per queue statistics */
+	struct stmmac_txq_stats txq_stats[MTL_MAX_TX_QUEUES];
+	struct stmmac_rxq_stats rxq_stats[MTL_MAX_RX_QUEUES];
+	struct stmmac_pcpu_stats __percpu *pcpu_stats;
+	unsigned long rx_dropped;
+	unsigned long rx_errors;
+	unsigned long tx_dropped;
+	unsigned long tx_errors;
+};
+
+/* Safety Feature statistics exposed by ethtool */
+struct stmmac_safety_stats {
+	unsigned long mac_errors[32];
+	unsigned long mtl_errors[32];
+	unsigned long dma_errors[32];
+	unsigned long dma_dpp_errors[32];
+};
+
+/* Number of fields in Safety Stats */
+#define STMMAC_SAFETY_FEAT_SIZE	\
+	(sizeof(struct stmmac_safety_stats) / sizeof(unsigned long))
+
+/* CSR Frequency Access Defines*/
+#define CSR_F_35M	35000000
+#define CSR_F_60M	60000000
+#define CSR_F_100M	100000000
+#define CSR_F_150M	150000000
+#define CSR_F_250M	250000000
+#define CSR_F_300M	300000000
+#define CSR_F_500M	500000000
+#define CSR_F_800M	800000000
+
+#define	MAC_CSR_H_FRQ_MASK	0x20
+
+#define HASH_TABLE_SIZE 64
+#define PAUSE_TIME 0xffff
+
+/* Flow Control defines */
+#define FLOW_OFF	0
+#define FLOW_RX		1
+#define FLOW_TX		2
+#define FLOW_AUTO	(FLOW_TX | FLOW_RX)
+
+/* PCS defines */
+#define STMMAC_PCS_RGMII	(1 << 0)
+#define STMMAC_PCS_SGMII	(1 << 1)
+
+#define SF_DMA_MODE 1		/* DMA STORE-AND-FORWARD Operation Mode */
+
+/* DMA HW feature register fields */
+#define DMA_HW_FEAT_MIISEL	0x00000001	/* 10/100 Mbps Support */
+#define DMA_HW_FEAT_GMIISEL	0x00000002	/* 1000 Mbps Support */
+#define DMA_HW_FEAT_HDSEL	0x00000004	/* Half-Duplex Support */
+#define DMA_HW_FEAT_EXTHASHEN	0x00000008	/* Expanded DA Hash Filter */
+#define DMA_HW_FEAT_HASHSEL	0x00000010	/* HASH Filter */
+#define DMA_HW_FEAT_ADDMAC	0x00000020	/* Multiple MAC Addr Reg */
+#define DMA_HW_FEAT_PCSSEL	0x00000040	/* PCS registers */
+#define DMA_HW_FEAT_L3L4FLTREN	0x00000080	/* Layer 3 & Layer 4 Feature */
+#define DMA_HW_FEAT_SMASEL	0x00000100	/* SMA(MDIO) Interface */
+#define DMA_HW_FEAT_RWKSEL	0x00000200	/* PMT Remote Wakeup */
+#define DMA_HW_FEAT_MGKSEL	0x00000400	/* PMT Magic Packet */
+#define DMA_HW_FEAT_MMCSEL	0x00000800	/* RMON Module */
+#define DMA_HW_FEAT_TSVER1SEL	0x00001000	/* Only IEEE 1588-2002 */
+#define DMA_HW_FEAT_TSVER2SEL	0x00002000	/* IEEE 1588-2008 PTPv2 */
+#define DMA_HW_FEAT_EEESEL	0x00004000	/* Energy Efficient Ethernet */
+#define DMA_HW_FEAT_AVSEL	0x00008000	/* AV Feature */
+#define DMA_HW_FEAT_TXCOESEL	0x00010000	/* Checksum Offload in Tx */
+#define DMA_HW_FEAT_RXTYP1COE	0x00020000	/* IP COE (Type 1) in Rx */
+#define DMA_HW_FEAT_RXTYP2COE	0x00040000	/* IP COE (Type 2) in Rx */
+#define DMA_HW_FEAT_RXFIFOSIZE	0x00080000	/* Rx FIFO > 2048 Bytes */
+#define DMA_HW_FEAT_RXCHCNT	0x00300000	/* No. additional Rx Channels */
+#define DMA_HW_FEAT_TXCHCNT	0x00c00000	/* No. additional Tx Channels */
+#define DMA_HW_FEAT_ENHDESSEL	0x01000000	/* Alternate Descriptor */
+/* Timestamping with Internal System Time */
+#define DMA_HW_FEAT_INTTSEN	0x02000000
+#define DMA_HW_FEAT_FLEXIPPSEN	0x04000000	/* Flexible PPS Output */
+#define DMA_HW_FEAT_SAVLANINS	0x08000000	/* Source Addr or VLAN */
+#define DMA_HW_FEAT_ACTPHYIF	0x70000000	/* Active/selected PHY iface */
+#define DEFAULT_DMA_PBL		8
+
+/* MSI defines */
+#define STMMAC_MSI_VEC_MAX	32
+
+/* PCS status and mask defines */
+#define	PCS_ANE_IRQ		BIT(2)	/* PCS Auto-Negotiation */
+#define	PCS_LINK_IRQ		BIT(1)	/* PCS Link */
+#define	PCS_RGSMIIIS_IRQ	BIT(0)	/* RGMII or SMII Interrupt */
+
+/* Max/Min RI Watchdog Timer count value */
+#define MAX_DMA_RIWT		0xff
+#define MIN_DMA_RIWT		0x10
+#define DEF_DMA_RIWT		0xa0
+/* Tx coalesce parameters */
+#define STMMAC_COAL_TX_TIMER	5000
+#define STMMAC_MAX_COAL_TX_TICK	100000
+#define STMMAC_TX_MAX_FRAMES	256
+#define STMMAC_TX_FRAMES	25
+#define STMMAC_RX_FRAMES	0
+
+/* Packets types */
+enum packets_types {
+	PACKET_AVCPQ = 0x1, /* AV Untagged Control packets */
+	PACKET_PTPQ = 0x2, /* PTP Packets */
+	PACKET_DCBCPQ = 0x3, /* DCB Control Packets */
+	PACKET_UPQ = 0x4, /* Untagged Packets */
+	PACKET_MCBCQ = 0x5, /* Multicast & Broadcast Packets */
+};
+
+/* Rx IPC status */
+enum rx_frame_status {
+	good_frame = 0x0,
+	discard_frame = 0x1,
+	csum_none = 0x2,
+	llc_snap = 0x4,
+	dma_own = 0x8,
+	rx_not_ls = 0x10,
+};
+
+/* Tx status */
+enum tx_frame_status {
+	tx_done = 0x0,
+	tx_not_ls = 0x1,
+	tx_err = 0x2,
+	tx_dma_own = 0x4,
+	tx_err_bump_tc = 0x8,
+};
+
+enum dma_irq_status {
+	tx_hard_error = 0x1,
+	tx_hard_error_bump_tc = 0x2,
+	handle_rx = 0x4,
+	handle_tx = 0x8,
+};
+
+enum dma_irq_dir {
+	DMA_DIR_RX = 0x1,
+	DMA_DIR_TX = 0x2,
+	DMA_DIR_RXTX = 0x3,
+};
+
+enum request_irq_err {
+	REQ_IRQ_ERR_ALL,
+	REQ_IRQ_ERR_TX,
+	REQ_IRQ_ERR_RX,
+	REQ_IRQ_ERR_SFTY,
+	REQ_IRQ_ERR_SFTY_UE,
+	REQ_IRQ_ERR_SFTY_CE,
+	REQ_IRQ_ERR_LPI,
+	REQ_IRQ_ERR_WOL,
+	REQ_IRQ_ERR_MAC,
+	REQ_IRQ_ERR_NO,
+};
+
+/* EEE and LPI defines */
+#define	CORE_IRQ_TX_PATH_IN_LPI_MODE	(1 << 0)
+#define	CORE_IRQ_TX_PATH_EXIT_LPI_MODE	(1 << 1)
+#define	CORE_IRQ_RX_PATH_IN_LPI_MODE	(1 << 2)
+#define	CORE_IRQ_RX_PATH_EXIT_LPI_MODE	(1 << 3)
+
+/* FPE defines */
+#define FPE_EVENT_UNKNOWN		0
+#define FPE_EVENT_TRSP			BIT(0)
+#define FPE_EVENT_TVER			BIT(1)
+#define FPE_EVENT_RRSP			BIT(2)
+#define FPE_EVENT_RVER			BIT(3)
+
+#define CORE_IRQ_MTL_RX_OVERFLOW	BIT(8)
+
+/* DMA HW capabilities */
+struct dma_features {
+	unsigned int mbps_10_100;
+	unsigned int mbps_1000;
+	unsigned int half_duplex;
+	unsigned int hash_filter;
+	unsigned int multi_addr;
+	unsigned int pcs;
+	unsigned int sma_mdio;
+	unsigned int pmt_remote_wake_up;
+	unsigned int pmt_magic_frame;
+	unsigned int rmon;
+	/* IEEE 1588-2002 */
+	unsigned int time_stamp;
+	/* IEEE 1588-2008 */
+	unsigned int atime_stamp;
+	/* 802.3az - Energy-Efficient Ethernet (EEE) */
+	unsigned int eee;
+	unsigned int av;
+	unsigned int hash_tb_sz;
+	unsigned int tsoen;
+	/* TX and RX csum */
+	unsigned int tx_coe;
+	unsigned int rx_coe;
+	unsigned int rx_coe_type1;
+	unsigned int rx_coe_type2;
+	unsigned int rxfifo_over_2048;
+	/* TX and RX number of channels */
+	unsigned int number_rx_channel;
+	unsigned int number_tx_channel;
+	/* TX and RX number of queues */
+	unsigned int number_rx_queues;
+	unsigned int number_tx_queues;
+	/* PPS output */
+	unsigned int pps_out_num;
+	/* Number of Traffic Classes */
+	unsigned int numtc;
+	/* DCB Feature Enable */
+	unsigned int dcben;
+	/* IEEE 1588 High Word Register Enable */
+	unsigned int advthword;
+	/* PTP Offload Enable */
+	unsigned int ptoen;
+	/* One-Step Timestamping Enable */
+	unsigned int osten;
+	/* Priority-Based Flow Control Enable */
+	unsigned int pfcen;
+	/* Alternate (enhanced) DESC mode */
+	unsigned int enh_desc;
+	/* TX and RX FIFO sizes */
+	unsigned int tx_fifo_size;
+	unsigned int rx_fifo_size;
+	/* Automotive Safety Package */
+	unsigned int asp;
+	/* RX Parser */
+	unsigned int frpsel;
+	unsigned int frpbs;
+	unsigned int frpes;
+	unsigned int addr64;
+	unsigned int host_dma_width;
+	unsigned int rssen;
+	unsigned int vlhash;
+	unsigned int sphen;
+	unsigned int vlins;
+	unsigned int dvlan;
+	unsigned int l3l4fnum;
+	unsigned int arpoffsel;
+	/* One Step for PTP over UDP/IP Feature Enable */
+	unsigned int pou_ost_en;
+	/* Tx Timestamp FIFO Depth */
+	unsigned int ttsfd;
+	/* Queue/Channel-Based VLAN tag insertion on Tx */
+	unsigned int cbtisel;
+	/* Supported Parallel Instruction Processor Engines */
+	unsigned int frppipe_num;
+	/* Number of Extended VLAN Tag Filters */
+	unsigned int nrvf_num;
+	/* TSN Features */
+	unsigned int estwid;
+	unsigned int estdep;
+	unsigned int estsel;
+	unsigned int fpesel;
+	unsigned int tbssel;
+	/* Number of DMA channels enabled for TBS */
+	unsigned int tbs_ch_num;
+	/* Per-Stream Filtering Enable */
+	unsigned int sgfsel;
+	/* Numbers of Auxiliary Snapshot Inputs */
+	unsigned int aux_snapshot_n;
+	/* Timestamp System Time Source */
+	unsigned int tssrc;
+	/* Enhanced DMA Enable */
+	unsigned int edma;
+	/* Different Descriptor Cache Enable */
+	unsigned int ediffc;
+	/* VxLAN/NVGRE Enable */
+	unsigned int vxn;
+	/* Debug Memory Interface Enable */
+	unsigned int dbgmem;
+	/* Number of Policing Counters */
+	unsigned int pcsel;
+};
+
+/* RX Buffer size must be multiple of 4/8/16 bytes */
+#define BUF_SIZE_16KiB 16368
+#define BUF_SIZE_8KiB 8188
+#define BUF_SIZE_4KiB 4096
+#define BUF_SIZE_2KiB 2048
+
+/* Power Down and WOL */
+#define PMT_NOT_SUPPORTED 0
+#define PMT_SUPPORTED 1
+
+/* Common MAC defines */
+#define MAC_CTRL_REG		0x00000000	/* MAC Control */
+#define MAC_ENABLE_TX		0x00000008	/* Transmitter Enable */
+#define MAC_ENABLE_RX		0x00000004	/* Receiver Enable */
+
+/* Default LPI timers */
+#define STMMAC_DEFAULT_LIT_LS	0x3E8
+#define STMMAC_DEFAULT_TWT_LS	0x1E
+#define STMMAC_ET_MAX		0xFFFFF
+
+/* Common LPI register bits */
+#define LPI_CTRL_STATUS_LPITCSE	BIT(21)	/* LPI Tx Clock Stop Enable, gmac4, xgmac2 only */
+#define LPI_CTRL_STATUS_LPIATE	BIT(20)	/* LPI Timer Enable, gmac4 only */
+#define LPI_CTRL_STATUS_LPITXA	BIT(19)	/* Enable LPI TX Automate */
+#define LPI_CTRL_STATUS_PLSEN	BIT(18)	/* Enable PHY Link Status */
+#define LPI_CTRL_STATUS_PLS	BIT(17)	/* PHY Link Status */
+#define LPI_CTRL_STATUS_LPIEN	BIT(16)	/* LPI Enable */
+#define LPI_CTRL_STATUS_RLPIST	BIT(9)	/* Receive LPI state, gmac1000 only? */
+#define LPI_CTRL_STATUS_TLPIST	BIT(8)	/* Transmit LPI state, gmac1000 only? */
+#define LPI_CTRL_STATUS_RLPIEX	BIT(3)	/* Receive LPI Exit */
+#define LPI_CTRL_STATUS_RLPIEN	BIT(2)	/* Receive LPI Entry */
+#define LPI_CTRL_STATUS_TLPIEX	BIT(1)	/* Transmit LPI Exit */
+#define LPI_CTRL_STATUS_TLPIEN	BIT(0)	/* Transmit LPI Entry */
+
+#define STMMAC_CHAIN_MODE	0x1
+#define STMMAC_RING_MODE	0x2
+
+#define JUMBO_LEN		9000
+
+/* Receive Side Scaling */
+#define STMMAC_RSS_HASH_KEY_SIZE	40
+#define STMMAC_RSS_MAX_TABLE_SIZE	256
+
+/* VLAN */
+#define STMMAC_VLAN_NONE	0x0
+#define STMMAC_VLAN_REMOVE	0x1
+#define STMMAC_VLAN_INSERT	0x2
+#define STMMAC_VLAN_REPLACE	0x3
+
+struct mac_device_info;
+
+struct mac_link {
+	u32 caps;
+	u32 speed_mask;
+	u32 speed10;
+	u32 speed100;
+	u32 speed1000;
+	u32 speed2500;
+	u32 duplex;
+	struct {
+		u32 speed2500;
+		u32 speed5000;
+		u32 speed10000;
+	} xgmii;
+	struct {
+		u32 speed25000;
+		u32 speed40000;
+		u32 speed50000;
+		u32 speed100000;
+	} xlgmii;
+};
+
+struct mii_regs {
+	unsigned int addr;	/* MII Address */
+	unsigned int data;	/* MII Data */
+	unsigned int addr_shift;	/* MII address shift */
+	unsigned int reg_shift;		/* MII reg shift */
+	unsigned int addr_mask;		/* MII address mask */
+	unsigned int reg_mask;		/* MII reg mask */
+	unsigned int clk_csr_shift;
+	unsigned int clk_csr_mask;
+};
+
+struct mac_device_info {
+	const struct stmmac_ops *mac;
+	const struct stmmac_desc_ops *desc;
+	const struct stmmac_dma_ops *dma;
+	const struct stmmac_mode_ops *mode;
+	const struct stmmac_hwtimestamp *ptp;
+	const struct stmmac_tc_ops *tc;
+	const struct stmmac_mmc_ops *mmc;
+	const struct stmmac_est_ops *est;
+	const struct stmmac_vlan_ops *vlan;
+	struct dw_xpcs *xpcs;
+	struct phylink_pcs *phylink_pcs;
+	struct mii_regs mii;	/* MII register Addresses */
+	struct mac_link link;
+	void __iomem *pcsr;     /* vpointer to device CSRs */
+	unsigned int multicast_filter_bins;
+	unsigned int unicast_filter_entries;
+	unsigned int mcast_bits_log2;
+	unsigned int rx_csum;
+	unsigned int pcs;
+	unsigned int ps;
+	unsigned int xlgmac;
+	unsigned int num_vlan;
+	u32 vlan_filter[32];
+	bool vlan_fail_q_en;
+	u8 vlan_fail_q;
+	bool hw_vlan_en;
+};
+
+struct stmmac_rx_routing {
+	u32 reg_mask;
+	u32 reg_shift;
+};
+
+int ec_dwmac100_setup(struct stmmac_priv *priv);
+int ec_dwmac1000_setup(struct stmmac_priv *priv);
+int ec_dwmac4_setup(struct stmmac_priv *priv);
+int ec_dwxgmac2_setup(struct stmmac_priv *priv);
+int ec_dwxlgmac2_setup(struct stmmac_priv *priv);
+
+void ec_stmmac_set_mac_addr(void __iomem *ioaddr, const u8 addr[6],
+			 unsigned int high, unsigned int low);
+void ec_stmmac_get_mac_addr(void __iomem *ioaddr, unsigned char *addr,
+			 unsigned int high, unsigned int low);
+void ec_stmmac_set_mac(void __iomem *ioaddr, bool enable);
+
+void ec_stmmac_dwmac4_set_mac_addr(void __iomem *ioaddr, const u8 addr[6],
+				unsigned int high, unsigned int low);
+void ec_stmmac_dwmac4_get_mac_addr(void __iomem *ioaddr, unsigned char *addr,
+				unsigned int high, unsigned int low);
+void ec_stmmac_dwmac4_set_mac(void __iomem *ioaddr, bool enable);
+
+void ec_dwmac_dma_flush_tx_fifo(void __iomem *ioaddr);
+
+#endif /* __COMMON_H__ */
diff --git a/drivers/net/ethercat/device/K3/descs-ethercat.h b/drivers/net/ethercat/device/K3/descs-ethercat.h
new file mode 100644
index 000000000000..49d6a866244f
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/descs-ethercat.h
@@ -0,0 +1,186 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  Header File to describe the DMA descriptors and related definitions.
+  This is for DWMAC100 and 1000 cores.
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __DESCS_H__
+#define __DESCS_H__
+
+#include <linux/bitops.h>
+
+/* Normal receive descriptor defines */
+
+/* RDES0 */
+#define	RDES0_PAYLOAD_CSUM_ERR	BIT(0)
+#define	RDES0_CRC_ERROR		BIT(1)
+#define	RDES0_DRIBBLING		BIT(2)
+#define	RDES0_MII_ERROR		BIT(3)
+#define	RDES0_RECEIVE_WATCHDOG	BIT(4)
+#define	RDES0_FRAME_TYPE	BIT(5)
+#define	RDES0_COLLISION		BIT(6)
+#define	RDES0_IPC_CSUM_ERROR	BIT(7)
+#define	RDES0_LAST_DESCRIPTOR	BIT(8)
+#define	RDES0_FIRST_DESCRIPTOR	BIT(9)
+#define	RDES0_VLAN_TAG		BIT(10)
+#define	RDES0_OVERFLOW_ERROR	BIT(11)
+#define	RDES0_LENGTH_ERROR	BIT(12)
+#define	RDES0_SA_FILTER_FAIL	BIT(13)
+#define	RDES0_DESCRIPTOR_ERROR	BIT(14)
+#define	RDES0_ERROR_SUMMARY	BIT(15)
+#define	RDES0_FRAME_LEN_MASK	GENMASK(29, 16)
+#define RDES0_FRAME_LEN_SHIFT	16
+#define	RDES0_DA_FILTER_FAIL	BIT(30)
+#define	RDES0_OWN		BIT(31)
+			/* RDES1 */
+#define	RDES1_BUFFER1_SIZE_MASK		GENMASK(10, 0)
+#define	RDES1_BUFFER2_SIZE_MASK		GENMASK(21, 11)
+#define	RDES1_BUFFER2_SIZE_SHIFT	11
+#define	RDES1_SECOND_ADDRESS_CHAINED	BIT(24)
+#define	RDES1_END_RING			BIT(25)
+#define	RDES1_DISABLE_IC		BIT(31)
+
+/* Enhanced receive descriptor defines */
+
+/* RDES0 (similar to normal RDES) */
+#define	 ERDES0_RX_MAC_ADDR	BIT(0)
+
+/* RDES1: completely differ from normal desc definitions */
+#define	ERDES1_BUFFER1_SIZE_MASK	GENMASK(12, 0)
+#define	ERDES1_SECOND_ADDRESS_CHAINED	BIT(14)
+#define	ERDES1_END_RING			BIT(15)
+#define	ERDES1_BUFFER2_SIZE_MASK	GENMASK(28, 16)
+#define ERDES1_BUFFER2_SIZE_SHIFT	16
+#define	ERDES1_DISABLE_IC		BIT(31)
+
+/* Normal transmit descriptor defines */
+/* TDES0 */
+#define	TDES0_DEFERRED			BIT(0)
+#define	TDES0_UNDERFLOW_ERROR		BIT(1)
+#define	TDES0_EXCESSIVE_DEFERRAL	BIT(2)
+#define	TDES0_COLLISION_COUNT_MASK	GENMASK(6, 3)
+#define	TDES0_VLAN_FRAME		BIT(7)
+#define	TDES0_EXCESSIVE_COLLISIONS	BIT(8)
+#define	TDES0_LATE_COLLISION		BIT(9)
+#define	TDES0_NO_CARRIER		BIT(10)
+#define	TDES0_LOSS_CARRIER		BIT(11)
+#define	TDES0_PAYLOAD_ERROR		BIT(12)
+#define	TDES0_FRAME_FLUSHED		BIT(13)
+#define	TDES0_JABBER_TIMEOUT		BIT(14)
+#define	TDES0_ERROR_SUMMARY		BIT(15)
+#define	TDES0_IP_HEADER_ERROR		BIT(16)
+#define	TDES0_TIME_STAMP_STATUS		BIT(17)
+#define	TDES0_OWN			((u32)BIT(31))	/* silence sparse */
+/* TDES1 */
+#define	TDES1_BUFFER1_SIZE_MASK		GENMASK(10, 0)
+#define	TDES1_BUFFER2_SIZE_MASK		GENMASK(21, 11)
+#define	TDES1_BUFFER2_SIZE_SHIFT	11
+#define	TDES1_TIME_STAMP_ENABLE		BIT(22)
+#define	TDES1_DISABLE_PADDING		BIT(23)
+#define	TDES1_SECOND_ADDRESS_CHAINED	BIT(24)
+#define	TDES1_END_RING			BIT(25)
+#define	TDES1_CRC_DISABLE		BIT(26)
+#define	TDES1_CHECKSUM_INSERTION_MASK	GENMASK(28, 27)
+#define	TDES1_CHECKSUM_INSERTION_SHIFT	27
+#define	TDES1_FIRST_SEGMENT		BIT(29)
+#define	TDES1_LAST_SEGMENT		BIT(30)
+#define	TDES1_INTERRUPT			BIT(31)
+
+/* Enhanced transmit descriptor defines */
+/* TDES0 */
+#define	ETDES0_DEFERRED			BIT(0)
+#define	ETDES0_UNDERFLOW_ERROR		BIT(1)
+#define	ETDES0_EXCESSIVE_DEFERRAL	BIT(2)
+#define	ETDES0_COLLISION_COUNT_MASK	GENMASK(6, 3)
+#define	ETDES0_VLAN_FRAME		BIT(7)
+#define	ETDES0_EXCESSIVE_COLLISIONS	BIT(8)
+#define	ETDES0_LATE_COLLISION		BIT(9)
+#define	ETDES0_NO_CARRIER		BIT(10)
+#define	ETDES0_LOSS_CARRIER		BIT(11)
+#define	ETDES0_PAYLOAD_ERROR		BIT(12)
+#define	ETDES0_FRAME_FLUSHED		BIT(13)
+#define	ETDES0_JABBER_TIMEOUT		BIT(14)
+#define	ETDES0_ERROR_SUMMARY		BIT(15)
+#define	ETDES0_IP_HEADER_ERROR		BIT(16)
+#define	ETDES0_TIME_STAMP_STATUS	BIT(17)
+#define	ETDES0_SECOND_ADDRESS_CHAINED	BIT(20)
+#define	ETDES0_END_RING			BIT(21)
+#define	ETDES0_CHECKSUM_INSERTION_MASK	GENMASK(23, 22)
+#define	ETDES0_CHECKSUM_INSERTION_SHIFT	22
+#define	ETDES0_TIME_STAMP_ENABLE	BIT(25)
+#define	ETDES0_DISABLE_PADDING		BIT(26)
+#define	ETDES0_CRC_DISABLE		BIT(27)
+#define	ETDES0_FIRST_SEGMENT		BIT(28)
+#define	ETDES0_LAST_SEGMENT		BIT(29)
+#define	ETDES0_INTERRUPT		BIT(30)
+#define	ETDES0_OWN			((u32)BIT(31))	/* silence sparse */
+/* TDES1 */
+#define	ETDES1_BUFFER1_SIZE_MASK	GENMASK(12, 0)
+#define	ETDES1_BUFFER2_SIZE_MASK	GENMASK(28, 16)
+#define	ETDES1_BUFFER2_SIZE_SHIFT	16
+
+/* Extended Receive descriptor definitions */
+#define	ERDES4_IP_PAYLOAD_TYPE_MASK	GENMASK(6, 2)
+#define	ERDES4_IP_HDR_ERR		BIT(3)
+#define	ERDES4_IP_PAYLOAD_ERR		BIT(4)
+#define	ERDES4_IP_CSUM_BYPASSED		BIT(5)
+#define	ERDES4_IPV4_PKT_RCVD		BIT(6)
+#define	ERDES4_IPV6_PKT_RCVD		BIT(7)
+#define	ERDES4_MSG_TYPE_MASK		GENMASK(11, 8)
+#define	ERDES4_PTP_FRAME_TYPE		BIT(12)
+#define	ERDES4_PTP_VER			BIT(13)
+#define	ERDES4_TIMESTAMP_DROPPED	BIT(14)
+#define	ERDES4_AV_PKT_RCVD		BIT(16)
+#define	ERDES4_AV_TAGGED_PKT_RCVD	BIT(17)
+#define	ERDES4_VLAN_TAG_PRI_VAL_MASK	GENMASK(20, 18)
+#define	ERDES4_L3_FILTER_MATCH		BIT(24)
+#define	ERDES4_L4_FILTER_MATCH		BIT(25)
+#define	ERDES4_L3_L4_FILT_NO_MATCH_MASK	GENMASK(27, 26)
+
+/* Extended RDES4 message type definitions */
+#define RDES_EXT_NO_PTP			0x0
+#define RDES_EXT_SYNC			0x1
+#define RDES_EXT_FOLLOW_UP		0x2
+#define RDES_EXT_DELAY_REQ		0x3
+#define RDES_EXT_DELAY_RESP		0x4
+#define RDES_EXT_PDELAY_REQ		0x5
+#define RDES_EXT_PDELAY_RESP		0x6
+#define RDES_EXT_PDELAY_FOLLOW_UP	0x7
+#define RDES_PTP_ANNOUNCE		0x8
+#define RDES_PTP_MANAGEMENT		0x9
+#define RDES_PTP_SIGNALING		0xa
+#define RDES_PTP_PKT_RESERVED_TYPE	0xf
+
+/* Basic descriptor structure for normal and alternate descriptors */
+struct dma_desc {
+	__le32 des0;
+	__le32 des1;
+	__le32 des2;
+	__le32 des3;
+};
+
+/* Extended descriptor structure (e.g. >= databook 3.50a) */
+struct dma_extended_desc {
+	struct dma_desc basic;	/* Basic descriptors */
+	__le32 des4;	/* Extended Status */
+	__le32 des5;	/* Reserved */
+	__le32 des6;	/* Tx/Rx Timestamp Low */
+	__le32 des7;	/* Tx/Rx Timestamp High */
+};
+
+/* Enhanced descriptor for TBS */
+struct dma_edesc {
+	__le32 des4;
+	__le32 des5;
+	__le32 des6;
+	__le32 des7;
+	struct dma_desc basic;
+};
+
+/* Transmit checksum insertion control */
+#define	TX_CIC_FULL	3	/* Include IP header and pseudoheader */
+
+#endif /* __DESCS_H__ */
diff --git a/drivers/net/ethercat/device/K3/descs_com-ethercat.h b/drivers/net/ethercat/device/K3/descs_com-ethercat.h
new file mode 100644
index 000000000000..40f7f2da9c5e
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/descs_com-ethercat.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  Header File to describe Normal/enhanced descriptor functions used for RING
+  and CHAINED modes.
+
+  Copyright(C) 2011  STMicroelectronics Ltd
+
+  It defines all the functions used to handle the normal/enhanced
+  descriptors in case of the DMA is configured to work in chained or
+  in ring mode.
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __DESC_COM_H__
+#define __DESC_COM_H__
+
+/* Specific functions used for Ring mode */
+
+/* Enhanced descriptors */
+static inline void ehn_desc_rx_set_on_ring(struct dma_desc *p, int end,
+					   int bfsize)
+{
+	if (bfsize == BUF_SIZE_16KiB)
+		p->des1 |= cpu_to_le32((BUF_SIZE_8KiB
+				<< ERDES1_BUFFER2_SIZE_SHIFT)
+			   & ERDES1_BUFFER2_SIZE_MASK);
+
+	if (end)
+		p->des1 |= cpu_to_le32(ERDES1_END_RING);
+}
+
+static inline void enh_desc_end_tx_desc_on_ring(struct dma_desc *p, int end)
+{
+	if (end)
+		p->des0 |= cpu_to_le32(ETDES0_END_RING);
+	else
+		p->des0 &= cpu_to_le32(~ETDES0_END_RING);
+}
+
+static inline void enh_set_tx_desc_len_on_ring(struct dma_desc *p, int len)
+{
+	if (unlikely(len > BUF_SIZE_4KiB)) {
+		p->des1 |= cpu_to_le32((((len - BUF_SIZE_4KiB)
+					<< ETDES1_BUFFER2_SIZE_SHIFT)
+			    & ETDES1_BUFFER2_SIZE_MASK) | (BUF_SIZE_4KiB
+			    & ETDES1_BUFFER1_SIZE_MASK));
+	} else
+		p->des1 |= cpu_to_le32((len & ETDES1_BUFFER1_SIZE_MASK));
+}
+
+/* Normal descriptors */
+static inline void ndesc_rx_set_on_ring(struct dma_desc *p, int end, int bfsize)
+{
+	if (bfsize >= BUF_SIZE_2KiB) {
+		int bfsize2;
+
+		bfsize2 = min(bfsize - BUF_SIZE_2KiB + 1, BUF_SIZE_2KiB - 1);
+		p->des1 |= cpu_to_le32((bfsize2 << RDES1_BUFFER2_SIZE_SHIFT)
+			    & RDES1_BUFFER2_SIZE_MASK);
+	}
+
+	if (end)
+		p->des1 |= cpu_to_le32(RDES1_END_RING);
+}
+
+static inline void ndesc_end_tx_desc_on_ring(struct dma_desc *p, int end)
+{
+	if (end)
+		p->des1 |= cpu_to_le32(TDES1_END_RING);
+	else
+		p->des1 &= cpu_to_le32(~TDES1_END_RING);
+}
+
+static inline void norm_set_tx_desc_len_on_ring(struct dma_desc *p, int len)
+{
+	if (unlikely(len > BUF_SIZE_2KiB)) {
+		unsigned int buffer1 = (BUF_SIZE_2KiB - 1)
+					& TDES1_BUFFER1_SIZE_MASK;
+		p->des1 |= cpu_to_le32((((len - buffer1)
+					<< TDES1_BUFFER2_SIZE_SHIFT)
+				& TDES1_BUFFER2_SIZE_MASK) | buffer1);
+	} else
+		p->des1 |= cpu_to_le32((len & TDES1_BUFFER1_SIZE_MASK));
+}
+
+/* Specific functions used for Chain mode */
+
+/* Enhanced descriptors */
+static inline void ehn_desc_rx_set_on_chain(struct dma_desc *p)
+{
+	p->des1 |= cpu_to_le32(ERDES1_SECOND_ADDRESS_CHAINED);
+}
+
+static inline void enh_desc_end_tx_desc_on_chain(struct dma_desc *p)
+{
+	p->des0 |= cpu_to_le32(ETDES0_SECOND_ADDRESS_CHAINED);
+}
+
+static inline void enh_set_tx_desc_len_on_chain(struct dma_desc *p, int len)
+{
+	p->des1 |= cpu_to_le32(len & ETDES1_BUFFER1_SIZE_MASK);
+}
+
+/* Normal descriptors */
+static inline void ndesc_rx_set_on_chain(struct dma_desc *p, int end)
+{
+	p->des1 |= cpu_to_le32(RDES1_SECOND_ADDRESS_CHAINED);
+}
+
+static inline void ndesc_tx_set_on_chain(struct dma_desc *p)
+{
+	p->des1 |= cpu_to_le32(TDES1_SECOND_ADDRESS_CHAINED);
+}
+
+static inline void norm_set_tx_desc_len_on_chain(struct dma_desc *p, int len)
+{
+	p->des1 |= cpu_to_le32(len & TDES1_BUFFER1_SIZE_MASK);
+}
+#endif /* __DESC_COM_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac-spacemit-ethqos-ethercat.c b/drivers/net/ethercat/device/K3/dwmac-spacemit-ethqos-ethercat.c
new file mode 100644
index 000000000000..0b8b81676345
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac-spacemit-ethqos-ethercat.c
@@ -0,0 +1,685 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Spacemit Ethernet QoS glue driver
+ *
+ * Copyright (c) 2025, Spacemit Corporation.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/platform_device.h>
+#include <linux/phy.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/regmap.h>
+#include <linux/bitfield.h>
+#include <linux/mfd/syscon.h>
+#ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#endif
+#include "stmmac-ethercat.h"
+#include "stmmac_platform-ethercat.h"
+
+#define DRIVER_NAME			"dwmac-spacemit-ethqos-ethercat"
+
+#define TUNING_CMD_LEN			50
+#define CLK_PHASE_CNT			256
+#define CLK_PHASE_REVERT		180
+
+#define TXCLK_PHASE_DEFAULT		0
+#define RXCLK_PHASE_DEFAULT		0
+
+struct spacemit_ethqos {
+	struct platform_device *pdev;
+	struct plat_stmmacenet_data *plat;
+	struct regmap *apmu;
+	u32 ctrl_off;
+	u32 dline_off;
+	phy_interface_t phy_iface;
+	u8 tx_clk_phase;
+	u8 rx_clk_phase;
+	u8 clk_tuning_way;
+	struct clk *tx_clk;
+	struct clk *phy_clk;
+	int speed;
+	bool clk_tuning_enable;
+	bool tx_clk_from_soc;
+	bool phy_clk_from_soc;
+	bool wol_irq_enable;
+#ifdef CONFIG_DEBUG_FS
+	struct dentry *dbg_dir;
+	struct dentry *dbg_clk_tuning;
+#endif
+};
+
+/**
+ * struct spacemit_ethqos_ops - Spacemit platform glue ops
+ * @devm_glue_parse_dt:        Parse platform data from DT (devm-managed)
+ * @devm_glue_bind_ops:        Install platform callbacks (devm-rollback)
+ * @devm_glue_config_plat:     Program platform registers (devm-managed)
+ */
+struct spacemit_ethqos_ops {
+	int (*devm_glue_parse_dt)(struct platform_device *pdev,
+				  struct spacemit_ethqos *eqos);
+	int (*devm_glue_bind_ops)(struct spacemit_ethqos *eqos);
+	int (*devm_glue_config_plat)(struct spacemit_ethqos *eqos);
+};
+
+static int devm_spacemit_glue_init(struct platform_device *pdev,
+				   struct plat_stmmacenet_data *plat_dat,
+				   const struct spacemit_ethqos_ops *ops)
+{
+	struct device *dev = &pdev->dev;
+	struct spacemit_ethqos *eqos;
+	int ret;
+
+	eqos = devm_kzalloc(dev, sizeof(*eqos), GFP_KERNEL);
+	if (!eqos)
+		return -ENOMEM;
+
+	eqos->pdev = pdev;
+	eqos->plat = plat_dat;
+	plat_dat->bsp_priv = eqos;
+
+	if (ops->devm_glue_parse_dt) {
+		ret = ops->devm_glue_parse_dt(pdev, eqos);
+		if (ret)
+			return dev_err_probe(dev, ret, "glue layer: dt parse failed\n");
+	}
+
+	if (ops->devm_glue_bind_ops) {
+		ret = ops->devm_glue_bind_ops(eqos);
+		if (ret)
+			return dev_err_probe(dev, ret, "glue layer: bind ops failed\n");
+	}
+
+	if (ops->devm_glue_config_plat) {
+		ret = ops->devm_glue_config_plat(eqos);
+		if (ret)
+			return dev_err_probe(dev, ret, "glue layer: config failed\n");
+	}
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * K3 SoC-specific macros
+ * -----------------------------------------------------------------------------
+ */
+#define TX_PHASE			1
+#define RX_PHASE			0
+
+#define PHY_INTF_MODE_MASK		GENMASK(4, 3)
+
+#define PHY_INTF_RMII			FIELD_PREP(PHY_INTF_MODE_MASK, 0x0)
+#define PHY_INTF_RGMII			FIELD_PREP(PHY_INTF_MODE_MASK, 0x1)
+#define PHY_INTF_MII			FIELD_PREP(PHY_INTF_MODE_MASK, 0x3)
+
+/* only valid for rmii, invert tx clk */
+#define RMII_TX_CLK_SEL			BIT(6)
+/* only valid for rmii, invert rx clk */
+#define RMII_RX_CLK_SEL			BIT(7)
+
+#define AXI_SINGLE_ID			BIT(13)
+
+/* dline register bits */
+#define EMAC_RX_DLINE_EN		BIT(0)
+#define EMAC_TX_DLINE_EN		BIT(16)
+
+#define RMII_TX_PHASE_MASK		GENMASK(18, 16)
+#define RMII_RX_PHASE_MASK		GENMASK(22, 20)
+
+#define RGMII_RX_PHASE_MASK		GENMASK(22, 20)
+#define RGMII_TX_PHASE_MASK		GENMASK(26, 24)
+
+#define EMAC_RX_DLINE_CODE_MASK		GENMASK(15, 8)
+#define EMAC_TX_DLINE_CODE_MASK		GENMASK(31, 24)
+
+enum clk_tuning_way {
+	/* fpga clk tuning register */
+	CLK_TUNING_BY_REG,
+	/* zebu/evb rgmii delayline register */
+	CLK_TUNING_BY_DLINE,
+	/* evb rmii only revert tx/rx clock for clk tuning */
+	CLK_TUNING_BY_CLK_REVERT,
+	CLK_TUNING_MAX,
+};
+
+static int clk_phase_rmii_set(struct spacemit_ethqos *eqos, bool is_tx)
+{
+	struct device *dev = &eqos->pdev->dev;
+	u32 mask, val, phase;
+	int ret;
+
+	switch (eqos->clk_tuning_way) {
+	case CLK_TUNING_BY_REG:
+		if (is_tx) {
+			mask = RMII_TX_PHASE_MASK;
+			val  = FIELD_PREP(RMII_TX_PHASE_MASK, eqos->tx_clk_phase);
+		} else {
+			mask = RMII_RX_PHASE_MASK;
+			val  = FIELD_PREP(RMII_RX_PHASE_MASK, eqos->rx_clk_phase);
+		}
+		ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off, mask, val);
+		break;
+
+	case CLK_TUNING_BY_CLK_REVERT: {
+		mask = is_tx ? RMII_TX_CLK_SEL : RMII_RX_CLK_SEL;
+		phase = is_tx ? eqos->tx_clk_phase : eqos->rx_clk_phase;
+		val = (phase == CLK_PHASE_REVERT) ? mask : 0;
+		ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off, mask, val);
+		break;
+	}
+
+	default:
+		dev_err(dev, "invalid clk tuning way: %d\n", eqos->clk_tuning_way);
+		return -EINVAL;
+	}
+
+	if (ret < 0)
+		dev_err(dev, "failed to update RMII %s phase (ret=%d)\n",
+			is_tx ? "tx" : "rx", ret);
+
+	return ret;
+}
+
+static int clk_phase_rgmii_set(struct spacemit_ethqos *eqos, bool is_tx)
+{
+	struct device *dev = &eqos->pdev->dev;
+	u32 mask, val;
+	int ret;
+
+	switch (eqos->clk_tuning_way) {
+	case CLK_TUNING_BY_REG:
+		if (is_tx) {
+			mask = RGMII_TX_PHASE_MASK;
+			val  = FIELD_PREP(RGMII_TX_PHASE_MASK, eqos->tx_clk_phase);
+		} else {
+			mask = RGMII_RX_PHASE_MASK;
+			val  = FIELD_PREP(RGMII_RX_PHASE_MASK, eqos->rx_clk_phase);
+		}
+		ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off, mask, val);
+		break;
+
+	case CLK_TUNING_BY_DLINE:
+		if (is_tx) {
+			mask = EMAC_TX_DLINE_CODE_MASK;
+			val  = FIELD_PREP(EMAC_TX_DLINE_CODE_MASK, eqos->tx_clk_phase);
+		} else {
+			mask = EMAC_RX_DLINE_CODE_MASK;
+			val  = FIELD_PREP(EMAC_RX_DLINE_CODE_MASK, eqos->rx_clk_phase);
+		}
+		ret = regmap_update_bits(eqos->apmu, eqos->dline_off, mask, val);
+		break;
+
+	default:
+		dev_err(dev, "invalid clk tuning way: %d\n", eqos->clk_tuning_way);
+		return -EINVAL;
+	}
+
+	if (ret < 0)
+		dev_err(dev, "failed to update RGMII %s phase (ret=%d)\n",
+			is_tx ? "tx" : "rx", ret);
+
+	return ret;
+}
+
+static int clk_phase_set(struct spacemit_ethqos *eqos, bool is_tx)
+{
+	if (!eqos->clk_tuning_enable)
+		return 0;
+
+	if (eqos->phy_iface == PHY_INTERFACE_MODE_MII)
+		return 0;
+
+	if (phy_interface_mode_is_rgmii(eqos->phy_iface))
+		return clk_phase_rgmii_set(eqos, is_tx);
+	else
+		return clk_phase_rmii_set(eqos, is_tx);
+}
+
+static int k3_delayline_init(struct spacemit_ethqos *eqos)
+{
+	u32 mask, val;
+	int ret;
+
+	/*
+	 * On K3, TX/RX delayline must be enabled for reliable DMA init.
+	 * This is required for all phy-modes (rgmii/rmii/mii).
+	 * Also clear delay codes to remove any prior configuration residue.
+	 */
+	mask = EMAC_TX_DLINE_EN | EMAC_RX_DLINE_EN |
+	       EMAC_TX_DLINE_CODE_MASK | EMAC_RX_DLINE_CODE_MASK;
+
+	val = EMAC_TX_DLINE_EN | EMAC_RX_DLINE_EN;
+
+	ret = regmap_update_bits(eqos->apmu, eqos->dline_off,
+				 mask, val);
+	if (ret)
+		dev_err(&eqos->pdev->dev, "failed to init delayline\n");
+
+	return ret;
+}
+
+static int k3_eqos_iface_config(struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &eqos->pdev->dev;
+	phy_interface_t iface = eqos->phy_iface;
+	u32 val;
+	int ret;
+
+	switch (iface) {
+	case PHY_INTERFACE_MODE_MII:
+		val = PHY_INTF_MII;
+		break;
+
+	case PHY_INTERFACE_MODE_RMII:
+		val = PHY_INTF_RMII;
+		break;
+
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		val = PHY_INTF_RGMII;
+		break;
+
+	default:
+		dev_warn(dev, "unsupported phy-mode: %s\n", phy_modes(iface));
+		return -EINVAL; /* don't write unexpected bits */
+	}
+	ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off,
+				 PHY_INTF_MODE_MASK, val);
+	if (!ret)
+		dev_info(dev, "phy-mode=%s val=0x%08x\n", phy_modes(iface), val);
+
+	return ret;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int clk_phase_show(struct seq_file *s, void *data)
+{
+	struct spacemit_ethqos *eqos = s->private;
+
+	seq_printf(s, "phy-mode : %s\n", phy_modes(eqos->phy_iface));
+	seq_printf(s, "rx phase : %d\n", eqos->rx_clk_phase);
+	seq_printf(s, "tx phase : %d\n", eqos->tx_clk_phase);
+
+	return 0;
+}
+
+static ssize_t clk_tuning_write(struct file *file,
+				const char __user *user_buf,
+				size_t count, loff_t *ppos)
+{
+	struct spacemit_ethqos *eqos =
+				((struct seq_file *)(file->private_data))->private;
+	char buff[TUNING_CMD_LEN];
+	char mode_str[20];
+	size_t len = min_t(size_t, count, TUNING_CMD_LEN - 1);
+	int err, clk_phase;
+
+	if (copy_from_user(buff, user_buf, len))
+		return -EFAULT;
+	buff[len] = '\0';
+
+	err = sscanf(buff, "%19s %d", mode_str, &clk_phase);
+	if (err != 2)
+		return -EINVAL;
+	if (clk_phase < 0 || clk_phase >= CLK_PHASE_CNT)
+		return -EINVAL;
+
+	if (!strcmp(mode_str, "tx")) {
+		eqos->tx_clk_phase = clk_phase;
+		clk_phase_set(eqos, TX_PHASE);
+	} else if (!strcmp(mode_str, "rx")) {
+		eqos->rx_clk_phase = clk_phase;
+		clk_phase_set(eqos, RX_PHASE);
+	} else {
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static int clk_tuning_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, clk_phase_show, inode->i_private);
+}
+
+static const struct file_operations clk_tuning_fops = {
+	.open		= clk_tuning_open,
+	.write		= clk_tuning_write,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+#endif
+
+static int k3_parse_dt(struct platform_device *pdev, struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	u32 tx_phase, rx_phase;
+	int ret;
+
+	eqos->phy_iface = eqos->plat->phy_interface;
+
+	eqos->tx_clk = NULL;
+	if (phy_interface_mode_is_rgmii(eqos->phy_iface)) {
+		eqos->tx_clk = devm_clk_get_optional(dev, "tx_clk");
+		if (IS_ERR(eqos->tx_clk))
+			return dev_err_probe(dev, PTR_ERR(eqos->tx_clk), "tx clock");
+		if (!eqos->tx_clk)
+			dev_info(dev, "rgmii tx clk is derived from the rx clk\n");
+	}
+
+	eqos->tx_clk_from_soc = !!eqos->tx_clk;
+
+	eqos->phy_clk = devm_clk_get_optional(dev, "phy_clk");
+	if (IS_ERR(eqos->phy_clk))
+		return dev_err_probe(dev, PTR_ERR(eqos->phy_clk), "phy clock");
+
+	eqos->phy_clk_from_soc = !!eqos->phy_clk;
+	if (!eqos->phy_clk_from_soc)
+		dev_info(dev, "phy clk is provided by a external crystal oscillator\n");
+
+	eqos->apmu = syscon_regmap_lookup_by_phandle(np, "spacemit,apmu");
+	if (IS_ERR(eqos->apmu))
+		return dev_err_probe(dev, PTR_ERR(eqos->apmu), "spacemit,apmu lookup failed");
+
+	ret = of_property_read_u32(np, "spacemit,ctrl-offset", &eqos->ctrl_off);
+	if (ret)
+		return dev_err_probe(dev, ret, "missing spacemit,ctrl-offset");
+
+	ret = of_property_read_u32(np, "spacemit,dline-offset", &eqos->dline_off);
+	if (ret)
+		return dev_err_probe(dev, ret, "missing spacemit,dline-offset");
+
+	eqos->wol_irq_enable = of_property_read_bool(np, "spacemit,wake-irq-enable");
+
+	eqos->clk_tuning_enable = of_property_read_bool(np, "spacemit,clk-tuning-enable");
+	if (!eqos->clk_tuning_enable)
+		return 0;
+
+	if (of_property_read_bool(np, "spacemit,clk-tuning-by-reg")) {
+		eqos->clk_tuning_way = CLK_TUNING_BY_REG;
+	} else if (of_property_read_bool(np, "spacemit,clk-tuning-by-clk-revert")) {
+		eqos->clk_tuning_way = CLK_TUNING_BY_CLK_REVERT;
+	} else if (of_property_read_bool(np, "spacemit,clk-tuning-by-delayline")) {
+		eqos->clk_tuning_way = CLK_TUNING_BY_DLINE;
+	} else {
+		eqos->clk_tuning_way = CLK_TUNING_BY_REG;
+	}
+
+	if (of_property_read_u32(np, "spacemit,tx-phase", &tx_phase))
+		eqos->tx_clk_phase = TXCLK_PHASE_DEFAULT;
+	else
+		eqos->tx_clk_phase = tx_phase;
+
+	if (of_property_read_u32(np, "spacemit,rx-phase", &rx_phase))
+		eqos->rx_clk_phase = RXCLK_PHASE_DEFAULT;
+	else
+		eqos->rx_clk_phase = rx_phase;
+
+#ifdef CONFIG_DEBUG_FS
+	if (!eqos->dbg_dir) {
+		eqos->dbg_dir = debugfs_create_dir(dev_name(dev), NULL);
+
+		if (IS_ERR_OR_NULL(eqos->dbg_dir)) {
+			dev_err(dev, "debugfs: failed to create dir\n");
+		} else {
+			eqos->dbg_clk_tuning = debugfs_create_file("clk_tuning", 0644,
+								   eqos->dbg_dir, eqos,
+								   &clk_tuning_fops);
+			if (IS_ERR_OR_NULL(eqos->dbg_clk_tuning))
+				dev_err(dev, "debugfs: failed to create file\n");
+		}
+	}
+#endif
+
+	return 0;
+}
+
+static void devm_k3_release_dt(void *arg)
+{
+	struct spacemit_ethqos *eqos = arg;
+
+#ifdef CONFIG_DEBUG_FS
+	debugfs_remove_recursive(eqos->dbg_dir);
+	eqos->dbg_dir = NULL;
+	eqos->dbg_clk_tuning = NULL;
+#endif
+}
+
+static int devm_k3_parse_dt(struct platform_device *pdev,
+			    struct spacemit_ethqos *eqos)
+{
+	int ret;
+
+	ret = k3_parse_dt(pdev, eqos);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(&pdev->dev,
+					devm_k3_release_dt, eqos);
+}
+
+static void k3_fix_mac_speed(void *bsp_priv, int speed, unsigned int mode)
+{
+	struct spacemit_ethqos *eqos = bsp_priv;
+	struct device *dev = &eqos->pdev->dev;
+	phy_interface_t iface = eqos->phy_iface;
+
+	eqos->speed = speed;
+
+	if (!eqos->clk_tuning_enable)
+		return;
+
+	switch (iface) {
+	case PHY_INTERFACE_MODE_RGMII_ID:
+		/* PHY already provides TX+RX delay */
+		return;
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		/* PHY provides TX delay; only adjust RX */
+		clk_phase_set(eqos, RX_PHASE);
+		return;
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+		/* PHY provides RX delay; only adjust TX */
+		clk_phase_set(eqos, TX_PHASE);
+		return;
+	case PHY_INTERFACE_MODE_RMII:
+	case PHY_INTERFACE_MODE_RGMII:
+		/* rgmii/rmii: adjust both TX and RX phases */
+		clk_phase_set(eqos, TX_PHASE);
+		clk_phase_set(eqos, RX_PHASE);
+		return;
+	default:
+		dev_warn(dev, "clk tuning skipped for phy-mode: %s\n", phy_modes(iface));
+		return;
+	}
+}
+
+static int k3_clks_config(void *bsp_priv, bool enabled)
+{
+	struct spacemit_ethqos *eqos = bsp_priv;
+	int ret = 0;
+
+	if (enabled) {
+		if (eqos->tx_clk_from_soc) {
+			ret = clk_prepare_enable(eqos->tx_clk);
+			if (ret)
+				return ret;
+		}
+
+		if (eqos->phy_clk_from_soc) {
+			ret = clk_prepare_enable(eqos->phy_clk);
+			if (ret) {
+				if (eqos->tx_clk_from_soc)
+					clk_disable_unprepare(eqos->tx_clk);
+				return ret;
+			}
+		}
+	} else {
+		if (eqos->phy_clk_from_soc)
+			clk_disable_unprepare(eqos->phy_clk);
+		if (eqos->tx_clk_from_soc)
+			clk_disable_unprepare(eqos->tx_clk);
+	}
+
+	return ret;
+}
+
+static int devm_k3_bind_plat_ops(struct spacemit_ethqos *eqos)
+{
+	struct plat_stmmacenet_data *plat_dat = eqos->plat;
+
+	plat_dat->fix_mac_speed = k3_fix_mac_speed;
+	plat_dat->clks_config = k3_clks_config;
+
+	return 0;
+}
+
+static void k3_eqos_config_caps(struct spacemit_ethqos *eqos)
+{
+	struct plat_stmmacenet_data *plat_dat = eqos->plat;
+	/*
+	 * If GMAC TX clock is derived from PHY RXC, stopping RXC in EEE/LPI
+	 * will remove the TX clock and cause TX timeouts. Require RXC to run
+	 * in LPI.
+	 */
+	if (phy_interface_mode_is_rgmii(eqos->phy_iface) && !eqos->tx_clk_from_soc)
+		plat_dat->flags |= STMMAC_FLAG_RX_CLK_RUNS_IN_LPI;
+
+	/* When used as an EtherCAT device, only one TX and one RX queue are used. */
+	plat_dat->tx_queues_to_use = 1;
+	plat_dat->rx_queues_to_use = 1;
+}
+
+static int k3_setup_plat(struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &eqos->pdev->dev;
+	int ret;
+
+	if (eqos->phy_clk_from_soc) {
+		ret = clk_prepare_enable(eqos->phy_clk);
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to enable phy_clk\n");
+	}
+
+	if (eqos->tx_clk_from_soc) {
+		ret = clk_prepare_enable(eqos->tx_clk);
+		if (ret) {
+			ret = dev_err_probe(dev, ret, "failed to enable tx_clk\n");
+			goto err_disable_phy_clk;
+		}
+	}
+
+	ret = k3_eqos_iface_config(eqos);
+	if (ret)
+		goto err_disable_tx_clk;
+
+	ret = k3_delayline_init(eqos);
+	if (ret)
+		goto err_disable_tx_clk;
+
+	k3_eqos_config_caps(eqos);
+
+	return 0;
+
+err_disable_tx_clk:
+	if (eqos->tx_clk_from_soc)
+		clk_disable_unprepare(eqos->tx_clk);
+
+err_disable_phy_clk:
+	if (eqos->phy_clk_from_soc)
+		clk_disable_unprepare(eqos->phy_clk);
+
+	return ret;
+}
+
+static void devm_k3_cleanup_plat(void *arg)
+{
+	struct spacemit_ethqos *eqos = arg;
+
+	if (eqos->tx_clk_from_soc)
+		clk_disable_unprepare(eqos->tx_clk);
+
+	if (eqos->phy_clk_from_soc)
+		clk_disable_unprepare(eqos->phy_clk);
+}
+
+static int devm_k3_setup_plat(struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &eqos->pdev->dev;
+	int ret;
+
+	ret = k3_setup_plat(eqos);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, devm_k3_cleanup_plat, eqos);
+}
+
+static const struct spacemit_ethqos_ops k3_gmac_ops = {
+	.devm_glue_parse_dt = devm_k3_parse_dt,
+	.devm_glue_bind_ops = devm_k3_bind_plat_ops,
+	.devm_glue_config_plat = devm_k3_setup_plat,
+};
+
+/* TODO: add K4/K5/K6 SoC-specific GMAC macros/ops here in future */
+
+static int spacemit_ethqos_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct stmmac_resources stmmac_res;
+	struct plat_stmmacenet_data *plat_dat;
+	const struct spacemit_ethqos_ops *ops;
+	int ret;
+
+	ops = of_device_get_match_data(dev);
+	if (!ops)
+		return dev_err_probe(dev, -EINVAL, "no of_match data");
+
+	ret = ec_stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (ret)
+		return ret;
+
+	plat_dat = devm_ec_stmmac_probe_config_dt(pdev, stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return PTR_ERR(plat_dat);
+
+	ret = devm_spacemit_glue_init(pdev, plat_dat, ops);
+	if (ret)
+		return ret;
+
+	ret = devm_ec_stmmac_pltfr_probe(pdev, plat_dat, &stmmac_res);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id spacemit_ethqos_match[] = {
+	{ .compatible = "spacemit,k3-ec-gmac", .data = &k3_gmac_ops },
+	{}
+};
+MODULE_DEVICE_TABLE(of, spacemit_ethqos_match);
+
+static struct platform_driver spacemit_ethqos_driver = {
+	.probe  = spacemit_ethqos_probe,
+	.driver = {
+		.name           = DRIVER_NAME,
+		.pm             = &ec_stmmac_pltfr_pm_ops,
+		.of_match_table = spacemit_ethqos_match,
+	},
+};
+module_platform_driver(spacemit_ethqos_driver);
+
+MODULE_DESCRIPTION("Spacemit dwmac ethqos specific glue layer");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethercat/device/K3/dwmac100-ethercat.h b/drivers/net/ethercat/device/K3/dwmac100-ethercat.h
new file mode 100644
index 000000000000..a6c6da3eab38
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac100-ethercat.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  MAC 10/100 Header File
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __DWMAC100_H__
+#define __DWMAC100_H__
+
+#include <linux/phy.h>
+#include "common-ethercat.h"
+
+/*----------------------------------------------------------------------------
+ *	 			MAC BLOCK defines
+ *---------------------------------------------------------------------------*/
+/* MAC CSR offset */
+#define MAC_CONTROL	0x00000000	/* MAC Control */
+#define MAC_ADDR_HIGH	0x00000004	/* MAC Address High */
+#define MAC_ADDR_LOW	0x00000008	/* MAC Address Low */
+#define MAC_HASH_HIGH	0x0000000c	/* Multicast Hash Table High */
+#define MAC_HASH_LOW	0x00000010	/* Multicast Hash Table Low */
+#define MAC_MII_ADDR	0x00000014	/* MII Address */
+#define MAC_MII_DATA	0x00000018	/* MII Data */
+#define MAC_FLOW_CTRL	0x0000001c	/* Flow Control */
+#define MAC_VLAN1	0x00000020	/* VLAN1 Tag */
+#define MAC_VLAN2	0x00000024	/* VLAN2 Tag */
+
+/* MAC CTRL defines */
+#define MAC_CONTROL_RA	0x80000000	/* Receive All Mode */
+#define MAC_CONTROL_BLE	0x40000000	/* Endian Mode */
+#define MAC_CONTROL_HBD	0x10000000	/* Heartbeat Disable */
+#define MAC_CONTROL_PS	0x08000000	/* Port Select */
+#define MAC_CONTROL_DRO	0x00800000	/* Disable Receive Own */
+#define MAC_CONTROL_EXT_LOOPBACK 0x00400000	/* Reserved (ext loopback?) */
+#define MAC_CONTROL_OM	0x00200000	/* Loopback Operating Mode */
+#define MAC_CONTROL_F	0x00100000	/* Full Duplex Mode */
+#define MAC_CONTROL_PM	0x00080000	/* Pass All Multicast */
+#define MAC_CONTROL_PR	0x00040000	/* Promiscuous Mode */
+#define MAC_CONTROL_IF	0x00020000	/* Inverse Filtering */
+#define MAC_CONTROL_PB	0x00010000	/* Pass Bad Frames */
+#define MAC_CONTROL_HO	0x00008000	/* Hash Only Filtering Mode */
+#define MAC_CONTROL_HP	0x00002000	/* Hash/Perfect Filtering Mode */
+#define MAC_CONTROL_LCC	0x00001000	/* Late Collision Control */
+#define MAC_CONTROL_DBF	0x00000800	/* Disable Broadcast Frames */
+#define MAC_CONTROL_DRTY	0x00000400	/* Disable Retry */
+#define MAC_CONTROL_ASTP	0x00000100	/* Automatic Pad Stripping */
+#define MAC_CONTROL_BOLMT_10	0x00000000	/* Back Off Limit 10 */
+#define MAC_CONTROL_BOLMT_8	0x00000040	/* Back Off Limit 8 */
+#define MAC_CONTROL_BOLMT_4	0x00000080	/* Back Off Limit 4 */
+#define MAC_CONTROL_BOLMT_1	0x000000c0	/* Back Off Limit 1 */
+#define MAC_CONTROL_DC		0x00000020	/* Deferral Check */
+#define MAC_CONTROL_TE		0x00000008	/* Transmitter Enable */
+#define MAC_CONTROL_RE		0x00000004	/* Receiver Enable */
+
+#define MAC_CORE_INIT (MAC_CONTROL_HBD)
+
+/* MAC FLOW CTRL defines */
+#define MAC_FLOW_CTRL_PT_MASK	0xffff0000	/* Pause Time Mask */
+#define MAC_FLOW_CTRL_PT_SHIFT	16
+#define MAC_FLOW_CTRL_PASS	0x00000004	/* Pass Control Frames */
+#define MAC_FLOW_CTRL_ENABLE	0x00000002	/* Flow Control Enable */
+#define MAC_FLOW_CTRL_PAUSE	0x00000001	/* Flow Control Busy ... */
+
+/* MII ADDR  defines */
+#define MAC_MII_ADDR_WRITE	0x00000002	/* MII Write */
+#define MAC_MII_ADDR_BUSY	0x00000001	/* MII Busy */
+
+/*----------------------------------------------------------------------------
+ * 				DMA BLOCK defines
+ *---------------------------------------------------------------------------*/
+
+/* DMA Bus Mode register defines */
+#define DMA_BUS_MODE_DBO	0x00100000	/* Descriptor Byte Ordering */
+#define DMA_BUS_MODE_BLE	0x00000080	/* Big Endian/Little Endian */
+#define DMA_BUS_MODE_PBL_MASK	0x00003f00	/* Programmable Burst Len */
+#define DMA_BUS_MODE_PBL_SHIFT	8
+#define DMA_BUS_MODE_DSL_MASK	0x0000007c	/* Descriptor Skip Length */
+#define DMA_BUS_MODE_DSL_SHIFT	2	/*   (in DWORDS)      */
+#define DMA_BUS_MODE_BAR_BUS	0x00000002	/* Bar-Bus Arbitration */
+#define DMA_BUS_MODE_DEFAULT	0x00000000
+
+/* DMA Control register defines */
+#define DMA_CONTROL_SF		0x00200000	/* Store And Forward */
+
+/* Transmit Threshold Control */
+enum ttc_control {
+	DMA_CONTROL_TTC_DEFAULT = 0x00000000,	/* Threshold is 32 DWORDS */
+	DMA_CONTROL_TTC_64 = 0x00004000,	/* Threshold is 64 DWORDS */
+	DMA_CONTROL_TTC_128 = 0x00008000,	/* Threshold is 128 DWORDS */
+	DMA_CONTROL_TTC_256 = 0x0000c000,	/* Threshold is 256 DWORDS */
+	DMA_CONTROL_TTC_18 = 0x00400000,	/* Threshold is 18 DWORDS */
+	DMA_CONTROL_TTC_24 = 0x00404000,	/* Threshold is 24 DWORDS */
+	DMA_CONTROL_TTC_32 = 0x00408000,	/* Threshold is 32 DWORDS */
+	DMA_CONTROL_TTC_40 = 0x0040c000,	/* Threshold is 40 DWORDS */
+	DMA_CONTROL_SE = 0x00000008,	/* Stop On Empty */
+	DMA_CONTROL_OSF = 0x00000004,	/* Operate On 2nd Frame */
+};
+
+/* STMAC110 DMA Missed Frame Counter register defines */
+#define DMA_MISSED_FRAME_OVE	0x10000000	/* FIFO Overflow Overflow */
+#define DMA_MISSED_FRAME_OVE_CNTR 0x0ffe0000	/* Overflow Frame Counter */
+#define DMA_MISSED_FRAME_OVE_M	0x00010000	/* Missed Frame Overflow */
+#define DMA_MISSED_FRAME_M_CNTR	0x0000ffff	/* Missed Frame Couinter */
+
+extern const struct stmmac_dma_ops ec_dwmac100_dma_ops;
+
+#endif /* __DWMAC100_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac1000-ethercat.h b/drivers/net/ethercat/device/K3/dwmac1000-ethercat.h
new file mode 100644
index 000000000000..4f3a9b17a8da
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac1000-ethercat.h
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+#ifndef __DWMAC1000_H__
+#define __DWMAC1000_H__
+
+#include <linux/phy.h>
+#include "common-ethercat.h"
+
+#define GMAC_CONTROL		0x00000000	/* Configuration */
+#define GMAC_FRAME_FILTER	0x00000004	/* Frame Filter */
+#define GMAC_HASH_HIGH		0x00000008	/* Multicast Hash Table High */
+#define GMAC_HASH_LOW		0x0000000c	/* Multicast Hash Table Low */
+#define GMAC_MII_ADDR		0x00000010	/* MII Address */
+#define GMAC_MII_DATA		0x00000014	/* MII Data */
+#define GMAC_FLOW_CTRL		0x00000018	/* Flow Control */
+#define GMAC_VLAN_TAG		0x0000001c	/* VLAN Tag */
+#define GMAC_DEBUG		0x00000024	/* GMAC debug register */
+#define GMAC_WAKEUP_FILTER	0x00000028	/* Wake-up Frame Filter */
+
+#define GMAC_INT_STATUS		0x00000038	/* interrupt status register */
+#define GMAC_INT_STATUS_PMT	BIT(3)
+#define GMAC_INT_STATUS_MMCIS	BIT(4)
+#define GMAC_INT_STATUS_MMCRIS	BIT(5)
+#define GMAC_INT_STATUS_MMCTIS	BIT(6)
+#define GMAC_INT_STATUS_MMCCSUM	BIT(7)
+#define GMAC_INT_STATUS_TSTAMP	BIT(9)
+#define GMAC_INT_STATUS_LPIIS	BIT(10)
+
+/* interrupt mask register */
+#define	GMAC_INT_MASK		0x0000003c
+#define	GMAC_INT_DISABLE_RGMII		BIT(0)
+#define	GMAC_INT_DISABLE_PCSLINK	BIT(1)
+#define	GMAC_INT_DISABLE_PCSAN		BIT(2)
+#define	GMAC_INT_DISABLE_PMT		BIT(3)
+#define	GMAC_INT_DISABLE_TIMESTAMP	BIT(9)
+#define	GMAC_INT_DISABLE_PCS	(GMAC_INT_DISABLE_RGMII | \
+				 GMAC_INT_DISABLE_PCSLINK | \
+				 GMAC_INT_DISABLE_PCSAN)
+#define	GMAC_INT_DEFAULT_MASK	(GMAC_INT_DISABLE_TIMESTAMP | \
+				 GMAC_INT_DISABLE_PCS)
+
+/* PMT Control and Status */
+#define GMAC_PMT		0x0000002c
+enum power_event {
+	pointer_reset = 0x80000000,
+	global_unicast = 0x00000200,
+	wake_up_rx_frame = 0x00000040,
+	magic_frame = 0x00000020,
+	wake_up_frame_en = 0x00000004,
+	magic_pkt_en = 0x00000002,
+	power_down = 0x00000001,
+};
+
+/* Energy Efficient Ethernet (EEE)
+ *
+ * LPI status, timer and control register offset
+ * For LPI control and status bit definitions, see common.h.
+ */
+#define LPI_CTRL_STATUS	0x0030
+#define LPI_TIMER_CTRL	0x0034
+
+/* GMAC HW ADDR regs */
+#define GMAC_ADDR_HIGH(reg)	((reg > 15) ? 0x00000800 + (reg - 16) * 8 : \
+				 0x00000040 + (reg * 8))
+#define GMAC_ADDR_LOW(reg)	((reg > 15) ? 0x00000804 + (reg - 16) * 8 : \
+				 0x00000044 + (reg * 8))
+#define GMAC_MAX_PERFECT_ADDRESSES	1
+
+#define GMAC_PCS_BASE		0x000000c0	/* PCS register base */
+#define GMAC_RGSMIIIS		0x000000d8	/* RGMII/SMII status */
+
+/* SGMII/RGMII status register */
+#define GMAC_RGSMIIIS_LNKMODE		BIT(0)
+#define GMAC_RGSMIIIS_SPEED		GENMASK(2, 1)
+#define GMAC_RGSMIIIS_SPEED_SHIFT	1
+#define GMAC_RGSMIIIS_LNKSTS		BIT(3)
+#define GMAC_RGSMIIIS_JABTO		BIT(4)
+#define GMAC_RGSMIIIS_FALSECARDET	BIT(5)
+#define GMAC_RGSMIIIS_SMIDRXS		BIT(16)
+/* LNKMOD */
+#define GMAC_RGSMIIIS_LNKMOD_MASK	0x1
+/* LNKSPEED */
+#define GMAC_RGSMIIIS_SPEED_125		0x2
+#define GMAC_RGSMIIIS_SPEED_25		0x1
+#define GMAC_RGSMIIIS_SPEED_2_5		0x0
+
+/* GMAC Configuration defines */
+#define GMAC_CONTROL_2K 0x08000000	/* IEEE 802.3as 2K packets */
+#define GMAC_CONTROL_TC	0x01000000	/* Transmit Conf. in RGMII/SGMII */
+#define GMAC_CONTROL_WD	0x00800000	/* Disable Watchdog on receive */
+#define GMAC_CONTROL_JD	0x00400000	/* Jabber disable */
+#define GMAC_CONTROL_BE	0x00200000	/* Frame Burst Enable */
+#define GMAC_CONTROL_JE	0x00100000	/* Jumbo frame */
+enum inter_frame_gap {
+	GMAC_CONTROL_IFG_88 = 0x00040000,
+	GMAC_CONTROL_IFG_80 = 0x00020000,
+	GMAC_CONTROL_IFG_40 = 0x000e0000,
+};
+#define GMAC_CONTROL_DCRS	0x00010000	/* Disable carrier sense */
+#define GMAC_CONTROL_PS		0x00008000	/* Port Select 0:GMI 1:MII */
+#define GMAC_CONTROL_FES	0x00004000	/* Speed 0:10 1:100 */
+#define GMAC_CONTROL_DO		0x00002000	/* Disable Rx Own */
+#define GMAC_CONTROL_LM		0x00001000	/* Loop-back mode */
+#define GMAC_CONTROL_DM		0x00000800	/* Duplex Mode */
+#define GMAC_CONTROL_IPC	0x00000400	/* Checksum Offload */
+#define GMAC_CONTROL_DR		0x00000200	/* Disable Retry */
+#define GMAC_CONTROL_LUD	0x00000100	/* Link up/down */
+#define GMAC_CONTROL_ACS	0x00000080	/* Auto Pad/FCS Stripping */
+#define GMAC_CONTROL_DC		0x00000010	/* Deferral Check */
+#define GMAC_CONTROL_TE		0x00000008	/* Transmitter Enable */
+#define GMAC_CONTROL_RE		0x00000004	/* Receiver Enable */
+
+#define GMAC_CORE_INIT (GMAC_CONTROL_JD | GMAC_CONTROL_PS | \
+			GMAC_CONTROL_BE | GMAC_CONTROL_DCRS)
+
+/* GMAC Frame Filter defines */
+#define GMAC_FRAME_FILTER_PR	0x00000001	/* Promiscuous Mode */
+#define GMAC_FRAME_FILTER_HUC	0x00000002	/* Hash Unicast */
+#define GMAC_FRAME_FILTER_HMC	0x00000004	/* Hash Multicast */
+#define GMAC_FRAME_FILTER_DAIF	0x00000008	/* DA Inverse Filtering */
+#define GMAC_FRAME_FILTER_PM	0x00000010	/* Pass all multicast */
+#define GMAC_FRAME_FILTER_DBF	0x00000020	/* Disable Broadcast frames */
+#define GMAC_FRAME_FILTER_PCF	0x00000080	/* Pass Control frames */
+#define GMAC_FRAME_FILTER_SAIF	0x00000100	/* Inverse Filtering */
+#define GMAC_FRAME_FILTER_SAF	0x00000200	/* Source Address Filter */
+#define GMAC_FRAME_FILTER_HPF	0x00000400	/* Hash or perfect Filter */
+#define GMAC_FRAME_FILTER_RA	0x80000000	/* Receive all mode */
+/* GMII ADDR  defines */
+#define GMAC_MII_ADDR_WRITE	0x00000002	/* MII Write */
+#define GMAC_MII_ADDR_BUSY	0x00000001	/* MII Busy */
+/* GMAC FLOW CTRL defines */
+#define GMAC_FLOW_CTRL_PT_MASK	0xffff0000	/* Pause Time Mask */
+#define GMAC_FLOW_CTRL_PT_SHIFT	16
+#define GMAC_FLOW_CTRL_UP	0x00000008	/* Unicast pause frame enable */
+#define GMAC_FLOW_CTRL_RFE	0x00000004	/* Rx Flow Control Enable */
+#define GMAC_FLOW_CTRL_TFE	0x00000002	/* Tx Flow Control Enable */
+#define GMAC_FLOW_CTRL_FCB_BPA	0x00000001	/* Flow Control Busy ... */
+
+/* DEBUG Register defines */
+/* MTL TxStatus FIFO */
+#define GMAC_DEBUG_TXSTSFSTS	BIT(25)	/* MTL TxStatus FIFO Full Status */
+#define GMAC_DEBUG_TXFSTS	BIT(24) /* MTL Tx FIFO Not Empty Status */
+#define GMAC_DEBUG_TWCSTS	BIT(22) /* MTL Tx FIFO Write Controller */
+/* MTL Tx FIFO Read Controller Status */
+#define GMAC_DEBUG_TRCSTS_MASK	GENMASK(21, 20)
+#define GMAC_DEBUG_TRCSTS_SHIFT	20
+#define GMAC_DEBUG_TRCSTS_IDLE	0
+#define GMAC_DEBUG_TRCSTS_READ	1
+#define GMAC_DEBUG_TRCSTS_TXW	2
+#define GMAC_DEBUG_TRCSTS_WRITE	3
+#define GMAC_DEBUG_TXPAUSED	BIT(19) /* MAC Transmitter in PAUSE */
+/* MAC Transmit Frame Controller Status */
+#define GMAC_DEBUG_TFCSTS_MASK	GENMASK(18, 17)
+#define GMAC_DEBUG_TFCSTS_SHIFT	17
+#define GMAC_DEBUG_TFCSTS_IDLE	0
+#define GMAC_DEBUG_TFCSTS_WAIT	1
+#define GMAC_DEBUG_TFCSTS_GEN_PAUSE	2
+#define GMAC_DEBUG_TFCSTS_XFER	3
+/* MAC GMII or MII Transmit Protocol Engine Status */
+#define GMAC_DEBUG_TPESTS	BIT(16)
+#define GMAC_DEBUG_RXFSTS_MASK	GENMASK(9, 8) /* MTL Rx FIFO Fill-level */
+#define GMAC_DEBUG_RXFSTS_SHIFT	8
+#define GMAC_DEBUG_RXFSTS_EMPTY	0
+#define GMAC_DEBUG_RXFSTS_BT	1
+#define GMAC_DEBUG_RXFSTS_AT	2
+#define GMAC_DEBUG_RXFSTS_FULL	3
+#define GMAC_DEBUG_RRCSTS_MASK	GENMASK(6, 5) /* MTL Rx FIFO Read Controller */
+#define GMAC_DEBUG_RRCSTS_SHIFT	5
+#define GMAC_DEBUG_RRCSTS_IDLE	0
+#define GMAC_DEBUG_RRCSTS_RDATA	1
+#define GMAC_DEBUG_RRCSTS_RSTAT	2
+#define GMAC_DEBUG_RRCSTS_FLUSH	3
+#define GMAC_DEBUG_RWCSTS	BIT(4) /* MTL Rx FIFO Write Controller Active */
+/* MAC Receive Frame Controller FIFO Status */
+#define GMAC_DEBUG_RFCFCSTS_MASK	GENMASK(2, 1)
+#define GMAC_DEBUG_RFCFCSTS_SHIFT	1
+/* MAC GMII or MII Receive Protocol Engine Status */
+#define GMAC_DEBUG_RPESTS	BIT(0)
+
+/*--- DMA BLOCK defines ---*/
+/* DMA Bus Mode register defines */
+#define DMA_BUS_MODE_DA		0x00000002	/* Arbitration scheme */
+#define DMA_BUS_MODE_DSL_MASK	0x0000007c	/* Descriptor Skip Length */
+#define DMA_BUS_MODE_DSL_SHIFT	2		/*   (in DWORDS)      */
+/* Programmable burst length (passed thorugh platform)*/
+#define DMA_BUS_MODE_PBL_MASK	0x00003f00	/* Programmable Burst Len */
+#define DMA_BUS_MODE_PBL_SHIFT	8
+#define DMA_BUS_MODE_ATDS	0x00000080	/* Alternate Descriptor Size */
+
+enum rx_tx_priority_ratio {
+	double_ratio = 0x00004000,	/* 2:1 */
+	triple_ratio = 0x00008000,	/* 3:1 */
+	quadruple_ratio = 0x0000c000,	/* 4:1 */
+};
+
+#define DMA_BUS_MODE_FB		0x00010000	/* Fixed burst */
+#define DMA_BUS_MODE_MB		0x04000000	/* Mixed burst */
+#define DMA_BUS_MODE_RPBL_MASK	0x007e0000	/* Rx-Programmable Burst Len */
+#define DMA_BUS_MODE_RPBL_SHIFT	17
+#define DMA_BUS_MODE_USP	0x00800000
+#define DMA_BUS_MODE_MAXPBL	0x01000000
+#define DMA_BUS_MODE_AAL	0x02000000
+
+/* DMA CRS Control and Status Register Mapping */
+#define DMA_HOST_TX_DESC	  0x00001048	/* Current Host Tx descriptor */
+#define DMA_HOST_RX_DESC	  0x0000104c	/* Current Host Rx descriptor */
+/*  DMA Bus Mode register defines */
+#define DMA_BUS_PR_RATIO_MASK	  0x0000c000	/* Rx/Tx priority ratio */
+#define DMA_BUS_PR_RATIO_SHIFT	  14
+#define DMA_BUS_FB	  	  0x00010000	/* Fixed Burst */
+
+/* DMA operation mode defines (start/stop tx/rx are placed in common header)*/
+/* Disable Drop TCP/IP csum error */
+#define DMA_CONTROL_DT		0x04000000
+#define DMA_CONTROL_RSF		0x02000000	/* Receive Store and Forward */
+#define DMA_CONTROL_DFF		0x01000000	/* Disaable flushing */
+/* Threshold for Activating the FC */
+enum rfa {
+	act_full_minus_1 = 0x00800000,
+	act_full_minus_2 = 0x00800200,
+	act_full_minus_3 = 0x00800400,
+	act_full_minus_4 = 0x00800600,
+};
+/* Threshold for Deactivating the FC */
+enum rfd {
+	deac_full_minus_1 = 0x00400000,
+	deac_full_minus_2 = 0x00400800,
+	deac_full_minus_3 = 0x00401000,
+	deac_full_minus_4 = 0x00401800,
+};
+#define DMA_CONTROL_TSF	0x00200000	/* Transmit  Store and Forward */
+
+enum ttc_control {
+	DMA_CONTROL_TTC_64 = 0x00000000,
+	DMA_CONTROL_TTC_128 = 0x00004000,
+	DMA_CONTROL_TTC_192 = 0x00008000,
+	DMA_CONTROL_TTC_256 = 0x0000c000,
+	DMA_CONTROL_TTC_40 = 0x00010000,
+	DMA_CONTROL_TTC_32 = 0x00014000,
+	DMA_CONTROL_TTC_24 = 0x00018000,
+	DMA_CONTROL_TTC_16 = 0x0001c000,
+};
+#define DMA_CONTROL_TC_TX_MASK	0xfffe3fff
+
+#define DMA_CONTROL_EFC		0x00000100
+#define DMA_CONTROL_FEF		0x00000080
+#define DMA_CONTROL_FUF		0x00000040
+
+/* Receive flow control activation field
+ * RFA field in DMA control register, bits 23,10:9
+ */
+#define DMA_CONTROL_RFA_MASK	0x00800600
+
+/* Receive flow control deactivation field
+ * RFD field in DMA control register, bits 22,12:11
+ */
+#define DMA_CONTROL_RFD_MASK	0x00401800
+
+/* RFD and RFA fields are encoded as follows
+ *
+ *   Bit Field
+ *   0,00 - Full minus 1KB (only valid when rxfifo >= 4KB and EFC enabled)
+ *   0,01 - Full minus 2KB (only valid when rxfifo >= 4KB and EFC enabled)
+ *   0,10 - Full minus 3KB (only valid when rxfifo >= 4KB and EFC enabled)
+ *   0,11 - Full minus 4KB (only valid when rxfifo > 4KB and EFC enabled)
+ *   1,00 - Full minus 5KB (only valid when rxfifo > 8KB and EFC enabled)
+ *   1,01 - Full minus 6KB (only valid when rxfifo > 8KB and EFC enabled)
+ *   1,10 - Full minus 7KB (only valid when rxfifo > 8KB and EFC enabled)
+ *   1,11 - Reserved
+ *
+ * RFD should always be > RFA for a given FIFO size. RFD == RFA may work,
+ * but packet throughput performance may not be as expected.
+ *
+ * Be sure that bit 3 in GMAC Register 6 is set for Unicast Pause frame
+ * detection (IEEE Specification Requirement, Annex 31B, 31B.1, Pause
+ * Description).
+ *
+ * Be sure that DZPA (bit 7 in Flow Control Register, GMAC Register 6),
+ * is set to 0. This allows pause frames with a quanta of 0 to be sent
+ * as an XOFF message to the link peer.
+ */
+
+#define RFA_FULL_MINUS_1K	0x00000000
+#define RFA_FULL_MINUS_2K	0x00000200
+#define RFA_FULL_MINUS_3K	0x00000400
+#define RFA_FULL_MINUS_4K	0x00000600
+#define RFA_FULL_MINUS_5K	0x00800000
+#define RFA_FULL_MINUS_6K	0x00800200
+#define RFA_FULL_MINUS_7K	0x00800400
+
+#define RFD_FULL_MINUS_1K	0x00000000
+#define RFD_FULL_MINUS_2K	0x00000800
+#define RFD_FULL_MINUS_3K	0x00001000
+#define RFD_FULL_MINUS_4K	0x00001800
+#define RFD_FULL_MINUS_5K	0x00400000
+#define RFD_FULL_MINUS_6K	0x00400800
+#define RFD_FULL_MINUS_7K	0x00401000
+
+enum rtc_control {
+	DMA_CONTROL_RTC_64 = 0x00000000,
+	DMA_CONTROL_RTC_32 = 0x00000008,
+	DMA_CONTROL_RTC_96 = 0x00000010,
+	DMA_CONTROL_RTC_128 = 0x00000018,
+};
+#define DMA_CONTROL_TC_RX_MASK	0xffffffe7
+
+#define DMA_CONTROL_OSF	0x00000004	/* Operate on second frame */
+
+/* MMC registers offset */
+#define GMAC_MMC_CTRL      0x100
+#define GMAC_MMC_RX_INTR   0x104
+#define GMAC_MMC_TX_INTR   0x108
+#define GMAC_MMC_RX_CSUM_OFFLOAD   0x208
+#define GMAC_EXTHASH_BASE  0x500
+
+/* PTP and timestamping registers */
+
+#define GMAC3_X_ATSNS       GENMASK(29, 25)
+#define GMAC3_X_ATSNS_SHIFT 25
+
+#define GMAC_PTP_TCR_ATSFC	BIT(24)
+#define GMAC_PTP_TCR_ATSEN0	BIT(25)
+
+#define GMAC3_X_TIMESTAMP_STATUS	0x28
+#define GMAC_PTP_ATNR	0x30
+#define GMAC_PTP_ATSR	0x34
+
+extern const struct stmmac_dma_ops ec_dwmac1000_dma_ops;
+#endif /* __DWMAC1000_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac1000_core-ethercat.c b/drivers/net/ethercat/device/K3/dwmac1000_core-ethercat.c
new file mode 100644
index 000000000000..4a96c9915366
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac1000_core-ethercat.c
@@ -0,0 +1,641 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This is the driver for the GMAC on-chip Ethernet controller for ST SoCs.
+  DWC Ether MAC 10/100/1000 Universal version 3.41a  has been used for
+  developing this code.
+
+  This only implements the mac core functions for this chip.
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/crc32.h>
+#include <linux/slab.h>
+#include <linux/ethtool.h>
+#include <linux/io.h>
+#include <linux/string_choices.h>
+#include "stmmac-ethercat.h"
+#include "stmmac_pcs-ethercat.h"
+#include "stmmac_ptp-ethercat.h"
+#include "dwmac1000-ethercat.h"
+
+static void dwmac1000_core_init(struct mac_device_info *hw,
+				struct net_device *dev)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + GMAC_CONTROL);
+	int mtu = dev->mtu;
+
+	/* Configure GMAC core */
+	value |= GMAC_CORE_INIT;
+
+	if (mtu > 1500)
+		value |= GMAC_CONTROL_2K;
+	if (mtu > 2000)
+		value |= GMAC_CONTROL_JE;
+
+	if (hw->ps) {
+		value |= GMAC_CONTROL_TE;
+
+		value &= ~hw->link.speed_mask;
+		switch (hw->ps) {
+		case SPEED_1000:
+			value |= hw->link.speed1000;
+			break;
+		case SPEED_100:
+			value |= hw->link.speed100;
+			break;
+		case SPEED_10:
+			value |= hw->link.speed10;
+			break;
+		}
+	}
+
+	writel(value, ioaddr + GMAC_CONTROL);
+
+	/* Mask GMAC interrupts */
+	value = GMAC_INT_DEFAULT_MASK;
+
+	if (hw->pcs)
+		value &= ~GMAC_INT_DISABLE_PCS;
+
+	writel(value, ioaddr + GMAC_INT_MASK);
+
+#ifdef STMMAC_VLAN_TAG_USED
+	/* Tag detection without filtering */
+	writel(0x0, ioaddr + GMAC_VLAN_TAG);
+#endif
+}
+
+static int dwmac1000_rx_ipc_enable(struct mac_device_info *hw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + GMAC_CONTROL);
+
+	if (hw->rx_csum)
+		value |= GMAC_CONTROL_IPC;
+	else
+		value &= ~GMAC_CONTROL_IPC;
+
+	writel(value, ioaddr + GMAC_CONTROL);
+
+	value = readl(ioaddr + GMAC_CONTROL);
+
+	return !!(value & GMAC_CONTROL_IPC);
+}
+
+static void dwmac1000_dump_regs(struct mac_device_info *hw, u32 *reg_space)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	int i;
+
+	for (i = 0; i < 55; i++)
+		reg_space[i] = readl(ioaddr + i * 4);
+}
+
+static void dwmac1000_set_umac_addr(struct mac_device_info *hw,
+				    const unsigned char *addr,
+				    unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	ec_stmmac_set_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
+			    GMAC_ADDR_LOW(reg_n));
+}
+
+static void dwmac1000_get_umac_addr(struct mac_device_info *hw,
+				    unsigned char *addr,
+				    unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	ec_stmmac_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
+			    GMAC_ADDR_LOW(reg_n));
+}
+
+static void dwmac1000_set_mchash(void __iomem *ioaddr, u32 *mcfilterbits,
+				 int mcbitslog2)
+{
+	int numhashregs, regs;
+
+	switch (mcbitslog2) {
+	case 6:
+		writel(mcfilterbits[0], ioaddr + GMAC_HASH_LOW);
+		writel(mcfilterbits[1], ioaddr + GMAC_HASH_HIGH);
+		return;
+	case 7:
+		numhashregs = 4;
+		break;
+	case 8:
+		numhashregs = 8;
+		break;
+	default:
+		pr_debug("STMMAC: err in setting multicast filter\n");
+		return;
+	}
+	for (regs = 0; regs < numhashregs; regs++)
+		writel(mcfilterbits[regs],
+		       ioaddr + GMAC_EXTHASH_BASE + regs * 4);
+}
+
+static void dwmac1000_set_filter(struct mac_device_info *hw,
+				 struct net_device *dev)
+{
+	void __iomem *ioaddr = (void __iomem *)dev->base_addr;
+	unsigned int value = 0;
+	unsigned int perfect_addr_number = hw->unicast_filter_entries;
+	u32 mc_filter[8];
+	int mcbitslog2 = hw->mcast_bits_log2;
+
+	pr_debug("%s: # mcasts %d, # unicast %d\n", __func__,
+		 netdev_mc_count(dev), netdev_uc_count(dev));
+
+	memset(mc_filter, 0, sizeof(mc_filter));
+
+	if (dev->flags & IFF_PROMISC) {
+		value = GMAC_FRAME_FILTER_PR | GMAC_FRAME_FILTER_PCF;
+	} else if (dev->flags & IFF_ALLMULTI) {
+		value = GMAC_FRAME_FILTER_PM;	/* pass all multi */
+	} else if (!netdev_mc_empty(dev) && (mcbitslog2 == 0)) {
+		/* Fall back to all multicast if we've no filter */
+		value = GMAC_FRAME_FILTER_PM;
+	} else if (!netdev_mc_empty(dev)) {
+		struct netdev_hw_addr *ha;
+
+		/* Hash filter for multicast */
+		value = GMAC_FRAME_FILTER_HMC;
+
+		netdev_for_each_mc_addr(ha, dev) {
+			/* The upper n bits of the calculated CRC are used to
+			 * index the contents of the hash table. The number of
+			 * bits used depends on the hardware configuration
+			 * selected at core configuration time.
+			 */
+			int bit_nr = bitrev32(~crc32_le(~0, ha->addr,
+					      ETH_ALEN)) >>
+					      (32 - mcbitslog2);
+			/* The most significant bit determines the register to
+			 * use (H/L) while the other 5 bits determine the bit
+			 * within the register.
+			 */
+			mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
+		}
+	}
+
+	value |= GMAC_FRAME_FILTER_HPF;
+	dwmac1000_set_mchash(ioaddr, mc_filter, mcbitslog2);
+
+	/* Handle multiple unicast addresses (perfect filtering) */
+	if (netdev_uc_count(dev) > perfect_addr_number)
+		/* Switch to promiscuous mode if more than unicast
+		 * addresses are requested than supported by hardware.
+		 */
+		value |= GMAC_FRAME_FILTER_PR;
+	else {
+		int reg = 1;
+		struct netdev_hw_addr *ha;
+
+		netdev_for_each_uc_addr(ha, dev) {
+			ec_stmmac_set_mac_addr(ioaddr, ha->addr,
+					    GMAC_ADDR_HIGH(reg),
+					    GMAC_ADDR_LOW(reg));
+			reg++;
+		}
+
+		while (reg < perfect_addr_number) {
+			writel(0, ioaddr + GMAC_ADDR_HIGH(reg));
+			writel(0, ioaddr + GMAC_ADDR_LOW(reg));
+			reg++;
+		}
+	}
+
+#ifdef FRAME_FILTER_DEBUG
+	/* Enable Receive all mode (to debug filtering_fail errors) */
+	value |= GMAC_FRAME_FILTER_RA;
+#endif
+	writel(value, ioaddr + GMAC_FRAME_FILTER);
+}
+
+
+static void dwmac1000_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
+				unsigned int fc, unsigned int pause_time,
+				u32 tx_cnt)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	/* Set flow such that DZPQ in Mac Register 6 is 0,
+	 * and unicast pause detect is enabled.
+	 */
+	unsigned int flow = GMAC_FLOW_CTRL_UP;
+
+	pr_debug("GMAC Flow-Control:\n");
+	if (fc & FLOW_RX) {
+		pr_debug("\tReceive Flow-Control ON\n");
+		flow |= GMAC_FLOW_CTRL_RFE;
+	}
+	if (fc & FLOW_TX) {
+		pr_debug("\tTransmit Flow-Control ON\n");
+		flow |= GMAC_FLOW_CTRL_TFE;
+	}
+
+	if (duplex) {
+		pr_debug("\tduplex mode: PAUSE %d\n", pause_time);
+		flow |= (pause_time << GMAC_FLOW_CTRL_PT_SHIFT);
+	}
+
+	writel(flow, ioaddr + GMAC_FLOW_CTRL);
+}
+
+static void dwmac1000_pmt(struct mac_device_info *hw, unsigned long mode)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	unsigned int pmt = 0;
+
+	if (mode & WAKE_MAGIC) {
+		pr_debug("GMAC: WOL Magic frame\n");
+		pmt |= power_down | magic_pkt_en;
+	}
+	if (mode & WAKE_UCAST) {
+		pr_debug("GMAC: WOL on global unicast\n");
+		pmt |= power_down | global_unicast | wake_up_frame_en;
+	}
+
+	writel(pmt, ioaddr + GMAC_PMT);
+}
+
+/* RGMII or SMII interface */
+static void dwmac1000_rgsmii(void __iomem *ioaddr, struct stmmac_extra_stats *x)
+{
+	u32 status;
+
+	status = readl(ioaddr + GMAC_RGSMIIIS);
+	x->irq_rgmii_n++;
+
+	/* Check the link status */
+	if (status & GMAC_RGSMIIIS_LNKSTS) {
+		int speed_value;
+
+		x->pcs_link = 1;
+
+		speed_value = ((status & GMAC_RGSMIIIS_SPEED) >>
+			       GMAC_RGSMIIIS_SPEED_SHIFT);
+		if (speed_value == GMAC_RGSMIIIS_SPEED_125)
+			x->pcs_speed = SPEED_1000;
+		else if (speed_value == GMAC_RGSMIIIS_SPEED_25)
+			x->pcs_speed = SPEED_100;
+		else
+			x->pcs_speed = SPEED_10;
+
+		x->pcs_duplex = (status & GMAC_RGSMIIIS_LNKMOD_MASK);
+
+		pr_info("Link is Up - %d/%s\n", (int)x->pcs_speed,
+			x->pcs_duplex ? "Full" : "Half");
+	} else {
+		x->pcs_link = 0;
+		pr_info("Link is Down\n");
+	}
+}
+
+static int dwmac1000_irq_status(struct mac_device_info *hw,
+				struct stmmac_extra_stats *x)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 intr_status = readl(ioaddr + GMAC_INT_STATUS);
+	u32 intr_mask = readl(ioaddr + GMAC_INT_MASK);
+	int ret = 0;
+
+	/* Discard masked bits */
+	intr_status &= ~intr_mask;
+
+	/* Not used events (e.g. MMC interrupts) are not handled. */
+	if ((intr_status & GMAC_INT_STATUS_MMCTIS))
+		x->mmc_tx_irq_n++;
+	if (unlikely(intr_status & GMAC_INT_STATUS_MMCRIS))
+		x->mmc_rx_irq_n++;
+	if (unlikely(intr_status & GMAC_INT_STATUS_MMCCSUM))
+		x->mmc_rx_csum_offload_irq_n++;
+	if (unlikely(intr_status & GMAC_INT_DISABLE_PMT)) {
+		/* clear the PMT bits 5 and 6 by reading the PMT status reg */
+		readl(ioaddr + GMAC_PMT);
+		x->irq_receive_pmt_irq_n++;
+	}
+
+	/* MAC tx/rx EEE LPI entry/exit interrupts */
+	if (intr_status & GMAC_INT_STATUS_LPIIS) {
+		/* Clean LPI interrupt by reading the Reg 12 */
+		ret = readl(ioaddr + LPI_CTRL_STATUS);
+
+		if (ret & LPI_CTRL_STATUS_TLPIEN)
+			x->irq_tx_path_in_lpi_mode_n++;
+		if (ret & LPI_CTRL_STATUS_TLPIEX)
+			x->irq_tx_path_exit_lpi_mode_n++;
+		if (ret & LPI_CTRL_STATUS_RLPIEN)
+			x->irq_rx_path_in_lpi_mode_n++;
+		if (ret & LPI_CTRL_STATUS_RLPIEX)
+			x->irq_rx_path_exit_lpi_mode_n++;
+	}
+
+	dwmac_pcs_isr(ioaddr, GMAC_PCS_BASE, intr_status, x);
+
+	if (intr_status & PCS_RGSMIIIS_IRQ)
+		dwmac1000_rgsmii(ioaddr, x);
+
+	return ret;
+}
+
+static int dwmac1000_set_lpi_mode(struct mac_device_info *hw,
+				  enum stmmac_lpi_mode mode,
+				  bool en_tx_lpi_clockgating, u32 et)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	if (mode == STMMAC_LPI_TIMER)
+		return -EOPNOTSUPP;
+
+	value = readl(ioaddr + LPI_CTRL_STATUS);
+	if (mode == STMMAC_LPI_FORCED)
+		value |= LPI_CTRL_STATUS_LPIEN | LPI_CTRL_STATUS_LPITXA;
+	else
+		value &= ~(LPI_CTRL_STATUS_LPIEN | LPI_CTRL_STATUS_LPITXA);
+	writel(value, ioaddr + LPI_CTRL_STATUS);
+
+	return 0;
+}
+
+static void dwmac1000_set_eee_pls(struct mac_device_info *hw, int link)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + LPI_CTRL_STATUS);
+
+	if (link)
+		value |= LPI_CTRL_STATUS_PLS;
+	else
+		value &= ~LPI_CTRL_STATUS_PLS;
+
+	writel(value, ioaddr + LPI_CTRL_STATUS);
+}
+
+static void dwmac1000_set_eee_timer(struct mac_device_info *hw, int ls, int tw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	int value = ((tw & 0xffff)) | ((ls & 0x7ff) << 16);
+
+	/* Program the timers in the LPI timer control register:
+	 * LS: minimum time (ms) for which the link
+	 *  status from PHY should be ok before transmitting
+	 *  the LPI pattern.
+	 * TW: minimum time (us) for which the core waits
+	 *  after it has stopped transmitting the LPI pattern.
+	 */
+	writel(value, ioaddr + LPI_TIMER_CTRL);
+}
+
+static void dwmac1000_ctrl_ane(struct stmmac_priv *priv, bool ane,
+			       bool srgmi_ral, bool loopback)
+{
+	dwmac_ctrl_ane(priv->ioaddr, GMAC_PCS_BASE, ane, srgmi_ral, loopback);
+}
+
+static void dwmac1000_debug(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    struct stmmac_extra_stats *x,
+			    u32 rx_queues, u32 tx_queues)
+{
+	u32 value = readl(ioaddr + GMAC_DEBUG);
+
+	if (value & GMAC_DEBUG_TXSTSFSTS)
+		x->mtl_tx_status_fifo_full++;
+	if (value & GMAC_DEBUG_TXFSTS)
+		x->mtl_tx_fifo_not_empty++;
+	if (value & GMAC_DEBUG_TWCSTS)
+		x->mmtl_fifo_ctrl++;
+	if (value & GMAC_DEBUG_TRCSTS_MASK) {
+		u32 trcsts = (value & GMAC_DEBUG_TRCSTS_MASK)
+			     >> GMAC_DEBUG_TRCSTS_SHIFT;
+		if (trcsts == GMAC_DEBUG_TRCSTS_WRITE)
+			x->mtl_tx_fifo_read_ctrl_write++;
+		else if (trcsts == GMAC_DEBUG_TRCSTS_TXW)
+			x->mtl_tx_fifo_read_ctrl_wait++;
+		else if (trcsts == GMAC_DEBUG_TRCSTS_READ)
+			x->mtl_tx_fifo_read_ctrl_read++;
+		else
+			x->mtl_tx_fifo_read_ctrl_idle++;
+	}
+	if (value & GMAC_DEBUG_TXPAUSED)
+		x->mac_tx_in_pause++;
+	if (value & GMAC_DEBUG_TFCSTS_MASK) {
+		u32 tfcsts = (value & GMAC_DEBUG_TFCSTS_MASK)
+			      >> GMAC_DEBUG_TFCSTS_SHIFT;
+
+		if (tfcsts == GMAC_DEBUG_TFCSTS_XFER)
+			x->mac_tx_frame_ctrl_xfer++;
+		else if (tfcsts == GMAC_DEBUG_TFCSTS_GEN_PAUSE)
+			x->mac_tx_frame_ctrl_pause++;
+		else if (tfcsts == GMAC_DEBUG_TFCSTS_WAIT)
+			x->mac_tx_frame_ctrl_wait++;
+		else
+			x->mac_tx_frame_ctrl_idle++;
+	}
+	if (value & GMAC_DEBUG_TPESTS)
+		x->mac_gmii_tx_proto_engine++;
+	if (value & GMAC_DEBUG_RXFSTS_MASK) {
+		u32 rxfsts = (value & GMAC_DEBUG_RXFSTS_MASK)
+			     >> GMAC_DEBUG_RRCSTS_SHIFT;
+
+		if (rxfsts == GMAC_DEBUG_RXFSTS_FULL)
+			x->mtl_rx_fifo_fill_level_full++;
+		else if (rxfsts == GMAC_DEBUG_RXFSTS_AT)
+			x->mtl_rx_fifo_fill_above_thresh++;
+		else if (rxfsts == GMAC_DEBUG_RXFSTS_BT)
+			x->mtl_rx_fifo_fill_below_thresh++;
+		else
+			x->mtl_rx_fifo_fill_level_empty++;
+	}
+	if (value & GMAC_DEBUG_RRCSTS_MASK) {
+		u32 rrcsts = (value & GMAC_DEBUG_RRCSTS_MASK) >>
+			     GMAC_DEBUG_RRCSTS_SHIFT;
+
+		if (rrcsts == GMAC_DEBUG_RRCSTS_FLUSH)
+			x->mtl_rx_fifo_read_ctrl_flush++;
+		else if (rrcsts == GMAC_DEBUG_RRCSTS_RSTAT)
+			x->mtl_rx_fifo_read_ctrl_read_data++;
+		else if (rrcsts == GMAC_DEBUG_RRCSTS_RDATA)
+			x->mtl_rx_fifo_read_ctrl_status++;
+		else
+			x->mtl_rx_fifo_read_ctrl_idle++;
+	}
+	if (value & GMAC_DEBUG_RWCSTS)
+		x->mtl_rx_fifo_ctrl_active++;
+	if (value & GMAC_DEBUG_RFCFCSTS_MASK)
+		x->mac_rx_frame_ctrl_fifo = (value & GMAC_DEBUG_RFCFCSTS_MASK)
+					    >> GMAC_DEBUG_RFCFCSTS_SHIFT;
+	if (value & GMAC_DEBUG_RPESTS)
+		x->mac_gmii_rx_proto_engine++;
+}
+
+static void dwmac1000_set_mac_loopback(void __iomem *ioaddr, bool enable)
+{
+	u32 value = readl(ioaddr + GMAC_CONTROL);
+
+	if (enable)
+		value |= GMAC_CONTROL_LM;
+	else
+		value &= ~GMAC_CONTROL_LM;
+
+	writel(value, ioaddr + GMAC_CONTROL);
+}
+
+const struct stmmac_ops ec_dwmac1000_ops = {
+	.core_init = dwmac1000_core_init,
+	.set_mac = ec_stmmac_set_mac,
+	.rx_ipc = dwmac1000_rx_ipc_enable,
+	.dump_regs = dwmac1000_dump_regs,
+	.host_irq_status = dwmac1000_irq_status,
+	.set_filter = dwmac1000_set_filter,
+	.flow_ctrl = dwmac1000_flow_ctrl,
+	.pmt = dwmac1000_pmt,
+	.set_umac_addr = dwmac1000_set_umac_addr,
+	.get_umac_addr = dwmac1000_get_umac_addr,
+	.set_lpi_mode = dwmac1000_set_lpi_mode,
+	.set_eee_timer = dwmac1000_set_eee_timer,
+	.set_eee_pls = dwmac1000_set_eee_pls,
+	.debug = dwmac1000_debug,
+	.pcs_ctrl_ane = dwmac1000_ctrl_ane,
+	.set_mac_loopback = dwmac1000_set_mac_loopback,
+};
+
+int ec_dwmac1000_setup(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	dev_info(priv->device, "\tDWMAC1000\n");
+
+	priv->dev->priv_flags |= IFF_UNICAST_FLT;
+	mac->pcsr = priv->ioaddr;
+	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
+	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
+	mac->mcast_bits_log2 = 0;
+
+	if (mac->multicast_filter_bins)
+		mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins);
+
+	mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+			 MAC_10 | MAC_100 | MAC_1000;
+	mac->link.duplex = GMAC_CONTROL_DM;
+	mac->link.speed10 = GMAC_CONTROL_PS;
+	mac->link.speed100 = GMAC_CONTROL_PS | GMAC_CONTROL_FES;
+	mac->link.speed1000 = 0;
+	mac->link.speed_mask = GMAC_CONTROL_PS | GMAC_CONTROL_FES;
+	mac->mii.addr = GMAC_MII_ADDR;
+	mac->mii.data = GMAC_MII_DATA;
+	mac->mii.addr_shift = 11;
+	mac->mii.addr_mask = 0x0000F800;
+	mac->mii.reg_shift = 6;
+	mac->mii.reg_mask = 0x000007C0;
+	mac->mii.clk_csr_shift = 2;
+	mac->mii.clk_csr_mask = GENMASK(5, 2);
+
+	return 0;
+}
+
+/* DWMAC 1000 HW Timestaming ops */
+
+void ec_dwmac1000_get_ptptime(void __iomem *ptpaddr, u64 *ptp_time)
+{
+	u64 ns;
+
+	ns = readl(ptpaddr + GMAC_PTP_ATNR);
+	ns += (u64)readl(ptpaddr + GMAC_PTP_ATSR) * NSEC_PER_SEC;
+
+	*ptp_time = ns;
+}
+
+void ec_dwmac1000_timestamp_interrupt(struct stmmac_priv *priv)
+{
+	struct ptp_clock_event event;
+	u32 ts_status, num_snapshot;
+	unsigned long flags;
+	u64 ptp_time;
+	int i;
+
+	/* Clears the timestamp interrupt */
+	ts_status = readl(priv->ptpaddr + GMAC3_X_TIMESTAMP_STATUS);
+
+	if (!(priv->plat->flags & STMMAC_FLAG_EXT_SNAPSHOT_EN))
+		return;
+
+	num_snapshot = (ts_status & GMAC3_X_ATSNS) >> GMAC3_X_ATSNS_SHIFT;
+
+	for (i = 0; i < num_snapshot; i++) {
+		read_lock_irqsave(&priv->ptp_lock, flags);
+		stmmac_get_ptptime(priv, priv->ptpaddr, &ptp_time);
+		read_unlock_irqrestore(&priv->ptp_lock, flags);
+
+		event.type = PTP_CLOCK_EXTTS;
+		event.index = 0;
+		event.timestamp = ptp_time;
+		ptp_clock_event(priv->ptp_clock, &event);
+	}
+}
+
+/* DWMAC 1000 ptp_clock_info ops */
+
+static void dwmac1000_timestamp_interrupt_cfg(struct stmmac_priv *priv, bool en)
+{
+	void __iomem *ioaddr = priv->ioaddr;
+
+	u32 intr_mask = readl(ioaddr + GMAC_INT_MASK);
+
+	if (en)
+		intr_mask &= ~GMAC_INT_DISABLE_TIMESTAMP;
+	else
+		intr_mask |= GMAC_INT_DISABLE_TIMESTAMP;
+
+	writel(intr_mask, ioaddr + GMAC_INT_MASK);
+}
+
+int ec_dwmac1000_ptp_enable(struct ptp_clock_info *ptp,
+			 struct ptp_clock_request *rq, int on)
+{
+	struct stmmac_priv *priv =
+	    container_of(ptp, struct stmmac_priv, ptp_clock_ops);
+	void __iomem *ptpaddr = priv->ptpaddr;
+	int ret = -EOPNOTSUPP;
+	u32 tcr_val;
+
+	switch (rq->type) {
+	case PTP_CLK_REQ_EXTTS:
+		mutex_lock(&priv->aux_ts_lock);
+		tcr_val = readl(ptpaddr + PTP_TCR);
+
+		if (on) {
+			tcr_val |= GMAC_PTP_TCR_ATSEN0;
+			tcr_val |= GMAC_PTP_TCR_ATSFC;
+			priv->plat->flags |= STMMAC_FLAG_EXT_SNAPSHOT_EN;
+		} else {
+			tcr_val &= ~GMAC_PTP_TCR_ATSEN0;
+			priv->plat->flags &= ~STMMAC_FLAG_EXT_SNAPSHOT_EN;
+		}
+
+		netdev_dbg(priv->dev, "Auxiliary Snapshot %s.\n",
+			   str_enabled_disabled(on));
+		writel(tcr_val, ptpaddr + PTP_TCR);
+
+		/* wait for auxts fifo clear to finish */
+		ret = readl_poll_timeout(ptpaddr + PTP_TCR, tcr_val,
+					 !(tcr_val & GMAC_PTP_TCR_ATSFC),
+					 10, 10000);
+
+		mutex_unlock(&priv->aux_ts_lock);
+
+		dwmac1000_timestamp_interrupt_cfg(priv, on);
+		break;
+
+	default:
+		break;
+	}
+
+	return ret;
+}
diff --git a/drivers/net/ethercat/device/K3/dwmac1000_dma-ethercat.c b/drivers/net/ethercat/device/K3/dwmac1000_dma-ethercat.c
new file mode 100644
index 000000000000..9d359f3e4e60
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac1000_dma-ethercat.c
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This is the driver for the GMAC on-chip Ethernet controller for ST SoCs.
+  DWC Ether MAC 10/100/1000 Universal version 3.41a  has been used for
+  developing this code.
+
+  This contains the functions to handle the dma.
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/io.h>
+#include "dwmac1000-ethercat.h"
+#include "dwmac_dma-ethercat.h"
+
+static void dwmac1000_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi)
+{
+	u32 value = readl(ioaddr + DMA_AXI_BUS_MODE);
+	int i;
+
+	pr_info("dwmac1000: Master AXI performs %s burst length\n",
+		!(value & DMA_AXI_UNDEF) ? "fixed" : "any");
+
+	if (axi->axi_lpi_en)
+		value |= DMA_AXI_EN_LPI;
+	if (axi->axi_xit_frm)
+		value |= DMA_AXI_LPI_XIT_FRM;
+
+	value &= ~DMA_AXI_WR_OSR_LMT;
+	value |= (axi->axi_wr_osr_lmt & DMA_AXI_WR_OSR_LMT_MASK) <<
+		 DMA_AXI_WR_OSR_LMT_SHIFT;
+
+	value &= ~DMA_AXI_RD_OSR_LMT;
+	value |= (axi->axi_rd_osr_lmt & DMA_AXI_RD_OSR_LMT_MASK) <<
+		 DMA_AXI_RD_OSR_LMT_SHIFT;
+
+	/* Depending on the UNDEF bit the Master AXI will perform any burst
+	 * length according to the BLEN programmed (by default all BLEN are
+	 * set).
+	 */
+	for (i = 0; i < AXI_BLEN; i++) {
+		switch (axi->axi_blen[i]) {
+		case 256:
+			value |= DMA_AXI_BLEN256;
+			break;
+		case 128:
+			value |= DMA_AXI_BLEN128;
+			break;
+		case 64:
+			value |= DMA_AXI_BLEN64;
+			break;
+		case 32:
+			value |= DMA_AXI_BLEN32;
+			break;
+		case 16:
+			value |= DMA_AXI_BLEN16;
+			break;
+		case 8:
+			value |= DMA_AXI_BLEN8;
+			break;
+		case 4:
+			value |= DMA_AXI_BLEN4;
+			break;
+		}
+	}
+
+	writel(value, ioaddr + DMA_AXI_BUS_MODE);
+}
+
+static void dwmac1000_dma_init_channel(struct stmmac_priv *priv,
+				       void __iomem *ioaddr,
+				       struct stmmac_dma_cfg *dma_cfg, u32 chan)
+{
+	int txpbl = dma_cfg->txpbl ?: dma_cfg->pbl;
+	int rxpbl = dma_cfg->rxpbl ?: dma_cfg->pbl;
+	u32 value;
+
+	value = readl(ioaddr + DMA_CHAN_BUS_MODE(chan));
+
+	/* Set the DMA PBL (Programmable Burst Length) mode.
+	 *
+	 * Note: before stmmac core 3.50 this mode bit was 4xPBL, and
+	 * post 3.5 mode bit acts as 8*PBL.
+	 */
+	if (dma_cfg->pblx8)
+		value |= DMA_BUS_MODE_MAXPBL;
+	value |= DMA_BUS_MODE_USP;
+	value &= ~(DMA_BUS_MODE_PBL_MASK | DMA_BUS_MODE_RPBL_MASK);
+	value |= (txpbl << DMA_BUS_MODE_PBL_SHIFT);
+	value |= (rxpbl << DMA_BUS_MODE_RPBL_SHIFT);
+
+	/* Set the Fixed burst mode */
+	if (dma_cfg->fixed_burst)
+		value |= DMA_BUS_MODE_FB;
+
+	/* Mixed Burst has no effect when fb is set */
+	if (dma_cfg->mixed_burst)
+		value |= DMA_BUS_MODE_MB;
+
+	if (dma_cfg->atds)
+		value |= DMA_BUS_MODE_ATDS;
+
+	if (dma_cfg->aal)
+		value |= DMA_BUS_MODE_AAL;
+
+	writel(value, ioaddr + DMA_CHAN_BUS_MODE(chan));
+
+	/* Mask interrupts by writing to CSR7 */
+	writel(DMA_INTR_DEFAULT_MASK, ioaddr + DMA_CHAN_INTR_ENA(chan));
+}
+
+static void dwmac1000_dma_init_rx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr,
+				  struct stmmac_dma_cfg *dma_cfg,
+				  dma_addr_t dma_rx_phy, u32 chan)
+{
+	/* RX descriptor base address list must be written into DMA CSR3 */
+	writel(lower_32_bits(dma_rx_phy), ioaddr + DMA_CHAN_RCV_BASE_ADDR(chan));
+}
+
+static void dwmac1000_dma_init_tx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr,
+				  struct stmmac_dma_cfg *dma_cfg,
+				  dma_addr_t dma_tx_phy, u32 chan)
+{
+	/* TX descriptor base address list must be written into DMA CSR4 */
+	writel(lower_32_bits(dma_tx_phy), ioaddr + DMA_CHAN_TX_BASE_ADDR(chan));
+}
+
+static u32 dwmac1000_configure_fc(u32 csr6, int rxfifosz)
+{
+	csr6 &= ~DMA_CONTROL_RFA_MASK;
+	csr6 &= ~DMA_CONTROL_RFD_MASK;
+
+	/* Leave flow control disabled if receive fifo size is less than
+	 * 4K or 0. Otherwise, send XOFF when fifo is 1K less than full,
+	 * and send XON when 2K less than full.
+	 */
+	if (rxfifosz < 4096) {
+		csr6 &= ~DMA_CONTROL_EFC;
+		pr_debug("GMAC: disabling flow control, rxfifo too small(%d)\n",
+			 rxfifosz);
+	} else {
+		csr6 |= DMA_CONTROL_EFC;
+		csr6 |= RFA_FULL_MINUS_1K;
+		csr6 |= RFD_FULL_MINUS_2K;
+	}
+	return csr6;
+}
+
+static void dwmac1000_dma_operation_mode_rx(struct stmmac_priv *priv,
+					    void __iomem *ioaddr, int mode,
+					    u32 channel, int fifosz, u8 qmode)
+{
+	u32 csr6 = readl(ioaddr + DMA_CHAN_CONTROL(channel));
+
+	if (mode == SF_DMA_MODE) {
+		pr_debug("GMAC: enable RX store and forward mode\n");
+		csr6 |= DMA_CONTROL_RSF;
+	} else {
+		pr_debug("GMAC: disable RX SF mode (threshold %d)\n", mode);
+		csr6 &= ~DMA_CONTROL_RSF;
+		csr6 &= DMA_CONTROL_TC_RX_MASK;
+		if (mode <= 32)
+			csr6 |= DMA_CONTROL_RTC_32;
+		else if (mode <= 64)
+			csr6 |= DMA_CONTROL_RTC_64;
+		else if (mode <= 96)
+			csr6 |= DMA_CONTROL_RTC_96;
+		else
+			csr6 |= DMA_CONTROL_RTC_128;
+	}
+
+	/* Configure flow control based on rx fifo size */
+	csr6 = dwmac1000_configure_fc(csr6, fifosz);
+
+	writel(csr6, ioaddr + DMA_CHAN_CONTROL(channel));
+}
+
+static void dwmac1000_dma_operation_mode_tx(struct stmmac_priv *priv,
+					    void __iomem *ioaddr, int mode,
+					    u32 channel, int fifosz, u8 qmode)
+{
+	u32 csr6 = readl(ioaddr + DMA_CHAN_CONTROL(channel));
+
+	if (mode == SF_DMA_MODE) {
+		pr_debug("GMAC: enable TX store and forward mode\n");
+		/* Transmit COE type 2 cannot be done in cut-through mode. */
+		csr6 |= DMA_CONTROL_TSF;
+		/* Operating on second frame increase the performance
+		 * especially when transmit store-and-forward is used.
+		 */
+		csr6 |= DMA_CONTROL_OSF;
+	} else {
+		pr_debug("GMAC: disabling TX SF (threshold %d)\n", mode);
+		csr6 &= ~DMA_CONTROL_TSF;
+		csr6 &= DMA_CONTROL_TC_TX_MASK;
+		/* Set the transmit threshold */
+		if (mode <= 32)
+			csr6 |= DMA_CONTROL_TTC_32;
+		else if (mode <= 64)
+			csr6 |= DMA_CONTROL_TTC_64;
+		else if (mode <= 128)
+			csr6 |= DMA_CONTROL_TTC_128;
+		else if (mode <= 192)
+			csr6 |= DMA_CONTROL_TTC_192;
+		else
+			csr6 |= DMA_CONTROL_TTC_256;
+	}
+
+	writel(csr6, ioaddr + DMA_CHAN_CONTROL(channel));
+}
+
+static void dwmac1000_dump_dma_regs(struct stmmac_priv *priv,
+				    void __iomem *ioaddr, u32 *reg_space)
+{
+	int i;
+
+	for (i = 0; i < NUM_DWMAC1000_DMA_REGS; i++)
+		if ((i < 12) || (i > 17))
+			reg_space[DMA_BUS_MODE / 4 + i] =
+				readl(ioaddr + DMA_BUS_MODE + i * 4);
+}
+
+static int dwmac1000_get_hw_feature(void __iomem *ioaddr,
+				    struct dma_features *dma_cap)
+{
+	u32 hw_cap = readl(ioaddr + DMA_HW_FEATURE);
+
+	if (!hw_cap) {
+		/* 0x00000000 is the value read on old hardware that does not
+		 * implement this register
+		 */
+		return -EOPNOTSUPP;
+	}
+
+	dma_cap->mbps_10_100 = (hw_cap & DMA_HW_FEAT_MIISEL);
+	dma_cap->mbps_1000 = (hw_cap & DMA_HW_FEAT_GMIISEL) >> 1;
+	dma_cap->half_duplex = (hw_cap & DMA_HW_FEAT_HDSEL) >> 2;
+	dma_cap->hash_filter = (hw_cap & DMA_HW_FEAT_HASHSEL) >> 4;
+	dma_cap->multi_addr = (hw_cap & DMA_HW_FEAT_ADDMAC) >> 5;
+	dma_cap->pcs = (hw_cap & DMA_HW_FEAT_PCSSEL) >> 6;
+	dma_cap->sma_mdio = (hw_cap & DMA_HW_FEAT_SMASEL) >> 8;
+	dma_cap->pmt_remote_wake_up = (hw_cap & DMA_HW_FEAT_RWKSEL) >> 9;
+	dma_cap->pmt_magic_frame = (hw_cap & DMA_HW_FEAT_MGKSEL) >> 10;
+	/* MMC */
+	dma_cap->rmon = (hw_cap & DMA_HW_FEAT_MMCSEL) >> 11;
+	/* IEEE 1588-2002 */
+	dma_cap->time_stamp =
+	    (hw_cap & DMA_HW_FEAT_TSVER1SEL) >> 12;
+	/* IEEE 1588-2008 */
+	dma_cap->atime_stamp = (hw_cap & DMA_HW_FEAT_TSVER2SEL) >> 13;
+	/* 802.3az - Energy-Efficient Ethernet (EEE) */
+	dma_cap->eee = (hw_cap & DMA_HW_FEAT_EEESEL) >> 14;
+	dma_cap->av = (hw_cap & DMA_HW_FEAT_AVSEL) >> 15;
+	/* TX and RX csum */
+	dma_cap->tx_coe = (hw_cap & DMA_HW_FEAT_TXCOESEL) >> 16;
+	dma_cap->rx_coe_type1 = (hw_cap & DMA_HW_FEAT_RXTYP1COE) >> 17;
+	dma_cap->rx_coe_type2 = (hw_cap & DMA_HW_FEAT_RXTYP2COE) >> 18;
+	dma_cap->rxfifo_over_2048 = (hw_cap & DMA_HW_FEAT_RXFIFOSIZE) >> 19;
+	/* TX and RX number of channels */
+	dma_cap->number_rx_channel = (hw_cap & DMA_HW_FEAT_RXCHCNT) >> 20;
+	dma_cap->number_tx_channel = (hw_cap & DMA_HW_FEAT_TXCHCNT) >> 22;
+	/* Alternate (enhanced) DESC mode */
+	dma_cap->enh_desc = (hw_cap & DMA_HW_FEAT_ENHDESSEL) >> 24;
+
+	return 0;
+}
+
+static void dwmac1000_rx_watchdog(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 riwt, u32 queue)
+{
+	writel(riwt, ioaddr + DMA_CHAN_RX_WATCHDOG(queue));
+}
+
+const struct stmmac_dma_ops ec_dwmac1000_dma_ops = {
+	.reset = ec_dwmac_dma_reset,
+	.init_chan = dwmac1000_dma_init_channel,
+	.init_rx_chan = dwmac1000_dma_init_rx,
+	.init_tx_chan = dwmac1000_dma_init_tx,
+	.axi = dwmac1000_dma_axi,
+	.dump_regs = dwmac1000_dump_dma_regs,
+	.dma_rx_mode = dwmac1000_dma_operation_mode_rx,
+	.dma_tx_mode = dwmac1000_dma_operation_mode_tx,
+	.enable_dma_transmission = ec_dwmac_enable_dma_transmission,
+	.enable_dma_irq = ec_dwmac_enable_dma_irq,
+	.disable_dma_irq = ec_dwmac_disable_dma_irq,
+	.start_tx = ec_dwmac_dma_start_tx,
+	.stop_tx = ec_dwmac_dma_stop_tx,
+	.start_rx = ec_dwmac_dma_start_rx,
+	.stop_rx = ec_dwmac_dma_stop_rx,
+	.dma_interrupt = ec_dwmac_dma_interrupt,
+	.get_hw_feature = dwmac1000_get_hw_feature,
+	.rx_watchdog = dwmac1000_rx_watchdog,
+};
+EXPORT_SYMBOL_GPL(ec_dwmac1000_dma_ops);
diff --git a/drivers/net/ethercat/device/K3/dwmac100_core-ethercat.c b/drivers/net/ethercat/device/K3/dwmac100_core-ethercat.c
new file mode 100644
index 000000000000..154db49b49fc
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac100_core-ethercat.c
@@ -0,0 +1,195 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This is the driver for the MAC 10/100 on-chip Ethernet controller
+  currently tested on all the ST boards based on STb7109 and stx7200 SoCs.
+
+  DWC Ether MAC 10/100 Universal version 4.0 has been used for developing
+  this code.
+
+  This only implements the mac core functions for this chip.
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/crc32.h>
+#include <linux/io.h>
+#include "stmmac-ethercat.h"
+#include "dwmac100-ethercat.h"
+
+static void dwmac100_core_init(struct mac_device_info *hw,
+			       struct net_device *dev)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + MAC_CONTROL);
+
+	value |= MAC_CORE_INIT;
+
+	writel(value, ioaddr + MAC_CONTROL);
+
+#ifdef STMMAC_VLAN_TAG_USED
+	writel(ETH_P_8021Q, ioaddr + MAC_VLAN1);
+#endif
+}
+
+static void dwmac100_dump_mac_regs(struct mac_device_info *hw, u32 *reg_space)
+{
+	void __iomem *ioaddr = hw->pcsr;
+
+	reg_space[MAC_CONTROL / 4] = readl(ioaddr + MAC_CONTROL);
+	reg_space[MAC_ADDR_HIGH / 4] = readl(ioaddr + MAC_ADDR_HIGH);
+	reg_space[MAC_ADDR_LOW / 4] = readl(ioaddr + MAC_ADDR_LOW);
+	reg_space[MAC_HASH_HIGH / 4] = readl(ioaddr + MAC_HASH_HIGH);
+	reg_space[MAC_HASH_LOW / 4] = readl(ioaddr + MAC_HASH_LOW);
+	reg_space[MAC_FLOW_CTRL / 4] = readl(ioaddr + MAC_FLOW_CTRL);
+	reg_space[MAC_VLAN1 / 4] = readl(ioaddr + MAC_VLAN1);
+	reg_space[MAC_VLAN2 / 4] = readl(ioaddr + MAC_VLAN2);
+}
+
+static int dwmac100_rx_ipc_enable(struct mac_device_info *hw)
+{
+	return 0;
+}
+
+static int dwmac100_irq_status(struct mac_device_info *hw,
+			       struct stmmac_extra_stats *x)
+{
+	return 0;
+}
+
+static void dwmac100_set_umac_addr(struct mac_device_info *hw,
+				   const unsigned char *addr,
+				   unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	ec_stmmac_set_mac_addr(ioaddr, addr, MAC_ADDR_HIGH, MAC_ADDR_LOW);
+}
+
+static void dwmac100_get_umac_addr(struct mac_device_info *hw,
+				   unsigned char *addr,
+				   unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	ec_stmmac_get_mac_addr(ioaddr, addr, MAC_ADDR_HIGH, MAC_ADDR_LOW);
+}
+
+static void dwmac100_set_filter(struct mac_device_info *hw,
+				struct net_device *dev)
+{
+	void __iomem *ioaddr = (void __iomem *)dev->base_addr;
+	u32 value = readl(ioaddr + MAC_CONTROL);
+
+	if (dev->flags & IFF_PROMISC) {
+		value |= MAC_CONTROL_PR;
+		value &= ~(MAC_CONTROL_PM | MAC_CONTROL_IF | MAC_CONTROL_HO |
+			   MAC_CONTROL_HP);
+	} else if ((netdev_mc_count(dev) > HASH_TABLE_SIZE)
+		   || (dev->flags & IFF_ALLMULTI)) {
+		value |= MAC_CONTROL_PM;
+		value &= ~(MAC_CONTROL_PR | MAC_CONTROL_IF | MAC_CONTROL_HO);
+		writel(0xffffffff, ioaddr + MAC_HASH_HIGH);
+		writel(0xffffffff, ioaddr + MAC_HASH_LOW);
+	} else if (netdev_mc_empty(dev)) {	/* no multicast */
+		value &= ~(MAC_CONTROL_PM | MAC_CONTROL_PR | MAC_CONTROL_IF |
+			   MAC_CONTROL_HO | MAC_CONTROL_HP);
+	} else {
+		u32 mc_filter[2];
+		struct netdev_hw_addr *ha;
+
+		/* Perfect filter mode for physical address and Hash
+		 * filter for multicast
+		 */
+		value |= MAC_CONTROL_HP;
+		value &= ~(MAC_CONTROL_PM | MAC_CONTROL_PR |
+			   MAC_CONTROL_IF | MAC_CONTROL_HO);
+
+		memset(mc_filter, 0, sizeof(mc_filter));
+		netdev_for_each_mc_addr(ha, dev) {
+			/* The upper 6 bits of the calculated CRC are used to
+			 * index the contens of the hash table
+			 */
+			int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26;
+			/* The most significant bit determines the register to
+			 * use (H/L) while the other 5 bits determine the bit
+			 * within the register.
+			 */
+			mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
+		}
+		writel(mc_filter[0], ioaddr + MAC_HASH_LOW);
+		writel(mc_filter[1], ioaddr + MAC_HASH_HIGH);
+	}
+
+	writel(value, ioaddr + MAC_CONTROL);
+}
+
+static void dwmac100_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
+			       unsigned int fc, unsigned int pause_time,
+			       u32 tx_cnt)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	unsigned int flow = MAC_FLOW_CTRL_ENABLE;
+
+	if (duplex)
+		flow |= (pause_time << MAC_FLOW_CTRL_PT_SHIFT);
+	writel(flow, ioaddr + MAC_FLOW_CTRL);
+}
+
+/* No PMT module supported on ST boards with this Eth chip. */
+static void dwmac100_pmt(struct mac_device_info *hw, unsigned long mode)
+{
+	return;
+}
+
+static void dwmac100_set_mac_loopback(void __iomem *ioaddr, bool enable)
+{
+	u32 value = readl(ioaddr + MAC_CONTROL);
+
+	if (enable)
+		value |= MAC_CONTROL_OM;
+	else
+		value &= ~MAC_CONTROL_OM;
+
+	writel(value, ioaddr + MAC_CONTROL);
+}
+
+const struct stmmac_ops ec_dwmac100_ops = {
+	.core_init = dwmac100_core_init,
+	.set_mac = ec_stmmac_set_mac,
+	.rx_ipc = dwmac100_rx_ipc_enable,
+	.dump_regs = dwmac100_dump_mac_regs,
+	.host_irq_status = dwmac100_irq_status,
+	.set_filter = dwmac100_set_filter,
+	.flow_ctrl = dwmac100_flow_ctrl,
+	.pmt = dwmac100_pmt,
+	.set_umac_addr = dwmac100_set_umac_addr,
+	.get_umac_addr = dwmac100_get_umac_addr,
+	.set_mac_loopback = dwmac100_set_mac_loopback,
+};
+
+int ec_dwmac100_setup(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	dev_info(priv->device, "\tDWMAC100\n");
+
+	mac->pcsr = priv->ioaddr;
+	mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+			 MAC_10 | MAC_100;
+	mac->link.duplex = MAC_CONTROL_F;
+	mac->link.speed10 = 0;
+	mac->link.speed100 = 0;
+	mac->link.speed1000 = 0;
+	mac->link.speed_mask = MAC_CONTROL_PS;
+	mac->mii.addr = MAC_MII_ADDR;
+	mac->mii.data = MAC_MII_DATA;
+	mac->mii.addr_shift = 11;
+	mac->mii.addr_mask = 0x0000F800;
+	mac->mii.reg_shift = 6;
+	mac->mii.reg_mask = 0x000007C0;
+	mac->mii.clk_csr_shift = 2;
+	mac->mii.clk_csr_mask = GENMASK(5, 2);
+
+	return 0;
+}
diff --git a/drivers/net/ethercat/device/K3/dwmac100_dma-ethercat.c b/drivers/net/ethercat/device/K3/dwmac100_dma-ethercat.c
new file mode 100644
index 000000000000..0112ed60142a
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac100_dma-ethercat.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This is the driver for the MAC 10/100 on-chip Ethernet controller
+  currently tested on all the ST boards based on STb7109 and stx7200 SoCs.
+
+  DWC Ether MAC 10/100 Universal version 4.0 has been used for developing
+  this code.
+
+  This contains the functions to handle the dma.
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/io.h>
+#include "dwmac100-ethercat.h"
+#include "dwmac_dma-ethercat.h"
+
+static void dwmac100_dma_init(void __iomem *ioaddr,
+			      struct stmmac_dma_cfg *dma_cfg)
+{
+	/* Enable Application Access by writing to DMA CSR0 */
+	writel(DMA_BUS_MODE_DEFAULT | (dma_cfg->pbl << DMA_BUS_MODE_PBL_SHIFT),
+	       ioaddr + DMA_BUS_MODE);
+
+	/* Mask interrupts by writing to CSR7 */
+	writel(DMA_INTR_DEFAULT_MASK, ioaddr + DMA_INTR_ENA);
+}
+
+static void dwmac100_dma_init_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 struct stmmac_dma_cfg *dma_cfg,
+				 dma_addr_t dma_rx_phy, u32 chan)
+{
+	/* RX descriptor base addr lists must be written into DMA CSR3 */
+	writel(lower_32_bits(dma_rx_phy), ioaddr + DMA_RCV_BASE_ADDR);
+}
+
+static void dwmac100_dma_init_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 struct stmmac_dma_cfg *dma_cfg,
+				 dma_addr_t dma_tx_phy, u32 chan)
+{
+	/* TX descriptor base addr lists must be written into DMA CSR4 */
+	writel(lower_32_bits(dma_tx_phy), ioaddr + DMA_TX_BASE_ADDR);
+}
+
+/* Store and Forward capability is not used at all.
+ *
+ * The transmit threshold can be programmed by setting the TTC bits in the DMA
+ * control register.
+ */
+static void dwmac100_dma_operation_mode_tx(struct stmmac_priv *priv,
+					   void __iomem *ioaddr, int mode,
+					   u32 channel, int fifosz, u8 qmode)
+{
+	u32 csr6 = readl(ioaddr + DMA_CONTROL);
+
+	if (mode <= 32)
+		csr6 |= DMA_CONTROL_TTC_32;
+	else if (mode <= 64)
+		csr6 |= DMA_CONTROL_TTC_64;
+	else
+		csr6 |= DMA_CONTROL_TTC_128;
+
+	writel(csr6, ioaddr + DMA_CONTROL);
+}
+
+static void dwmac100_dump_dma_regs(struct stmmac_priv *priv,
+				   void __iomem *ioaddr, u32 *reg_space)
+{
+	int i;
+
+	for (i = 0; i < NUM_DWMAC100_DMA_REGS; i++)
+		reg_space[DMA_BUS_MODE / 4 + i] =
+			readl(ioaddr + DMA_BUS_MODE + i * 4);
+
+	reg_space[DMA_CUR_TX_BUF_ADDR / 4] =
+		readl(ioaddr + DMA_CUR_TX_BUF_ADDR);
+	reg_space[DMA_CUR_RX_BUF_ADDR / 4] =
+		readl(ioaddr + DMA_CUR_RX_BUF_ADDR);
+}
+
+/* DMA controller has two counters to track the number of the missed frames. */
+static void dwmac100_dma_diagnostic_fr(struct stmmac_extra_stats *x,
+				       void __iomem *ioaddr)
+{
+	u32 csr8 = readl(ioaddr + DMA_MISSED_FRAME_CTR);
+
+	if (unlikely(csr8)) {
+		if (csr8 & DMA_MISSED_FRAME_OVE) {
+			x->rx_overflow_cntr += 0x800;
+		} else {
+			unsigned int ove_cntr;
+			ove_cntr = ((csr8 & DMA_MISSED_FRAME_OVE_CNTR) >> 17);
+			x->rx_overflow_cntr += ove_cntr;
+		}
+
+		if (csr8 & DMA_MISSED_FRAME_OVE_M) {
+			x->rx_missed_cntr += 0xffff;
+		} else {
+			unsigned int miss_f = (csr8 & DMA_MISSED_FRAME_M_CNTR);
+			x->rx_missed_cntr += miss_f;
+		}
+	}
+}
+
+const struct stmmac_dma_ops ec_dwmac100_dma_ops = {
+	.reset = ec_dwmac_dma_reset,
+	.init = dwmac100_dma_init,
+	.init_rx_chan = dwmac100_dma_init_rx,
+	.init_tx_chan = dwmac100_dma_init_tx,
+	.dump_regs = dwmac100_dump_dma_regs,
+	.dma_tx_mode = dwmac100_dma_operation_mode_tx,
+	.dma_diagnostic_fr = dwmac100_dma_diagnostic_fr,
+	.enable_dma_transmission = ec_dwmac_enable_dma_transmission,
+	.enable_dma_irq = ec_dwmac_enable_dma_irq,
+	.disable_dma_irq = ec_dwmac_disable_dma_irq,
+	.start_tx = ec_dwmac_dma_start_tx,
+	.stop_tx = ec_dwmac_dma_stop_tx,
+	.start_rx = ec_dwmac_dma_start_rx,
+	.stop_rx = ec_dwmac_dma_stop_rx,
+	.dma_interrupt = ec_dwmac_dma_interrupt,
+};
diff --git a/drivers/net/ethercat/device/K3/dwmac4-ethercat.h b/drivers/net/ethercat/device/K3/dwmac4-ethercat.h
new file mode 100644
index 000000000000..c0e3d7b90640
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac4-ethercat.h
@@ -0,0 +1,537 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * DWMAC4 Header file.
+ *
+ * Copyright (C) 2015  STMicroelectronics Ltd
+ *
+ * Author: Alexandre Torgue <alexandre.torgue@st.com>
+ */
+
+#ifndef __DWMAC4_H__
+#define __DWMAC4_H__
+
+#include "common-ethercat.h"
+
+/*  MAC registers */
+#define GMAC_CONFIG			0x00000000
+#define GMAC_EXT_CONFIG			0x00000004
+#define GMAC_PACKET_FILTER		0x00000008
+#define GMAC_HASH_TAB(x)		(0x10 + (x) * 4)
+#define GMAC_RX_FLOW_CTRL		0x00000090
+#define GMAC_QX_TX_FLOW_CTRL(x)		(0x70 + x * 4)
+#define GMAC_TXQ_PRTY_MAP0		0x98
+#define GMAC_TXQ_PRTY_MAP1		0x9C
+#define GMAC_RXQ_CTRL0			0x000000a0
+#define GMAC_RXQ_CTRL1			0x000000a4
+#define GMAC_RXQ_CTRL2			0x000000a8
+#define GMAC_RXQ_CTRL3			0x000000ac
+#define GMAC_INT_STATUS			0x000000b0
+#define GMAC_INT_EN			0x000000b4
+#define GMAC_PCS_BASE			0x000000e0
+#define GMAC_PHYIF_CONTROL_STATUS	0x000000f8
+#define GMAC_PMT			0x000000c0
+#define GMAC_DEBUG			0x00000114
+#define GMAC_HW_FEATURE0		0x0000011c
+#define GMAC_HW_FEATURE1		0x00000120
+#define GMAC_HW_FEATURE2		0x00000124
+#define GMAC_HW_FEATURE3		0x00000128
+#define GMAC_MDIO_ADDR			0x00000200
+#define GMAC_MDIO_DATA			0x00000204
+#define GMAC_GPIO_STATUS		0x0000020C
+#define GMAC_ARP_ADDR			0x00000210
+#define GMAC_EXT_CFG1			0x00000238
+#define GMAC_ADDR_HIGH(reg)		(0x300 + reg * 8)
+#define GMAC_ADDR_LOW(reg)		(0x304 + reg * 8)
+#define GMAC_L3L4_CTRL(reg)		(0x900 + (reg) * 0x30)
+#define GMAC_L4_ADDR(reg)		(0x904 + (reg) * 0x30)
+#define GMAC_L3_ADDR0(reg)		(0x910 + (reg) * 0x30)
+#define GMAC_L3_ADDR1(reg)		(0x914 + (reg) * 0x30)
+#define GMAC_TIMESTAMP_STATUS		0x00000b20
+
+/* RX Queues Routing */
+#define GMAC_RXQCTRL_AVCPQ_MASK		GENMASK(2, 0)
+#define GMAC_RXQCTRL_AVCPQ_SHIFT	0
+#define GMAC_RXQCTRL_PTPQ_MASK		GENMASK(6, 4)
+#define GMAC_RXQCTRL_PTPQ_SHIFT		4
+#define GMAC_RXQCTRL_DCBCPQ_MASK	GENMASK(10, 8)
+#define GMAC_RXQCTRL_DCBCPQ_SHIFT	8
+#define GMAC_RXQCTRL_UPQ_MASK		GENMASK(14, 12)
+#define GMAC_RXQCTRL_UPQ_SHIFT		12
+#define GMAC_RXQCTRL_MCBCQ_MASK		GENMASK(18, 16)
+#define GMAC_RXQCTRL_MCBCQ_SHIFT	16
+#define GMAC_RXQCTRL_MCBCQEN		BIT(20)
+#define GMAC_RXQCTRL_MCBCQEN_SHIFT	20
+#define GMAC_RXQCTRL_TACPQE		BIT(21)
+#define GMAC_RXQCTRL_TACPQE_SHIFT	21
+#define GMAC_RXQCTRL_FPRQ		GENMASK(26, 24)
+
+/* MAC Packet Filtering */
+#define GMAC_PACKET_FILTER_PR		BIT(0)
+#define GMAC_PACKET_FILTER_HMC		BIT(2)
+#define GMAC_PACKET_FILTER_PM		BIT(4)
+#define GMAC_PACKET_FILTER_PCF		BIT(7)
+#define GMAC_PACKET_FILTER_HPF		BIT(10)
+#define GMAC_PACKET_FILTER_VTFE		BIT(16)
+#define GMAC_PACKET_FILTER_IPFE		BIT(20)
+#define GMAC_PACKET_FILTER_RA		BIT(31)
+
+#define GMAC_MAX_PERFECT_ADDRESSES	128
+
+/* MAC RX Queue Enable */
+#define GMAC_RX_QUEUE_CLEAR(queue)	~(GENMASK(1, 0) << ((queue) * 2))
+#define GMAC_RX_AV_QUEUE_ENABLE(queue)	BIT((queue) * 2)
+#define GMAC_RX_DCB_QUEUE_ENABLE(queue)	BIT(((queue) * 2) + 1)
+
+/* MAC Flow Control RX */
+#define GMAC_RX_FLOW_CTRL_RFE		BIT(0)
+
+/* RX Queues Priorities */
+#define GMAC_RXQCTRL_PSRQX_MASK(x)	GENMASK(7 + ((x) * 8), 0 + ((x) * 8))
+#define GMAC_RXQCTRL_PSRQX_SHIFT(x)	((x) * 8)
+
+/* TX Queues Priorities */
+#define GMAC_TXQCTRL_PSTQX_MASK(x)	GENMASK(7 + ((x) * 8), 0 + ((x) * 8))
+#define GMAC_TXQCTRL_PSTQX_SHIFT(x)	((x) * 8)
+
+/* MAC Flow Control TX */
+#define GMAC_TX_FLOW_CTRL_TFE		BIT(1)
+#define GMAC_TX_FLOW_CTRL_PT_SHIFT	16
+
+/*  MAC Interrupt bitmap*/
+#define GMAC_INT_RGSMIIS		BIT(0)
+#define GMAC_INT_PCS_LINK		BIT(1)
+#define GMAC_INT_PCS_ANE		BIT(2)
+#define GMAC_INT_PCS_PHYIS		BIT(3)
+#define GMAC_INT_PMT_EN			BIT(4)
+#define GMAC_INT_LPI_EN			BIT(5)
+#define GMAC_INT_TSIE			BIT(12)
+
+#define	GMAC_PCS_IRQ_DEFAULT	(GMAC_INT_RGSMIIS | GMAC_INT_PCS_LINK |	\
+				 GMAC_INT_PCS_ANE)
+
+#define	GMAC_INT_DEFAULT_ENABLE	(GMAC_INT_PMT_EN | GMAC_INT_LPI_EN | \
+				 GMAC_INT_TSIE)
+
+enum dwmac4_irq_status {
+	time_stamp_irq = 0x00001000,
+	mmc_rx_csum_offload_irq = 0x00000800,
+	mmc_tx_irq = 0x00000400,
+	mmc_rx_irq = 0x00000200,
+	mmc_irq = 0x00000100,
+	lpi_irq = 0x00000020,
+	pmt_irq = 0x00000010,
+};
+
+/* MAC PMT bitmap */
+enum power_event {
+	pointer_reset =	0x80000000,
+	global_unicast = 0x00000200,
+	wake_up_rx_frame = 0x00000040,
+	magic_frame = 0x00000020,
+	wake_up_frame_en = 0x00000004,
+	magic_pkt_en = 0x00000002,
+	power_down = 0x00000001,
+};
+
+/* Energy Efficient Ethernet (EEE) for GMAC4
+ *
+ * LPI status, timer and control register offset
+ * For LPI control and status bit definitions, see common.h.
+ */
+#define GMAC4_LPI_CTRL_STATUS	0xd0
+#define GMAC4_LPI_TIMER_CTRL	0xd4
+#define GMAC4_LPI_ENTRY_TIMER	0xd8
+#define GMAC4_MAC_ONEUS_TIC_COUNTER	0xdc
+
+/* MAC Debug bitmap */
+#define GMAC_DEBUG_TFCSTS_MASK		GENMASK(18, 17)
+#define GMAC_DEBUG_TFCSTS_SHIFT		17
+#define GMAC_DEBUG_TFCSTS_IDLE		0
+#define GMAC_DEBUG_TFCSTS_WAIT		1
+#define GMAC_DEBUG_TFCSTS_GEN_PAUSE	2
+#define GMAC_DEBUG_TFCSTS_XFER		3
+#define GMAC_DEBUG_TPESTS		BIT(16)
+#define GMAC_DEBUG_RFCFCSTS_MASK	GENMASK(2, 1)
+#define GMAC_DEBUG_RFCFCSTS_SHIFT	1
+#define GMAC_DEBUG_RPESTS		BIT(0)
+
+/* MAC config */
+#define GMAC_CONFIG_ARPEN		BIT(31)
+#define GMAC_CONFIG_SARC		GENMASK(30, 28)
+#define GMAC_CONFIG_SARC_SHIFT		28
+#define GMAC_CONFIG_IPC			BIT(27)
+#define GMAC_CONFIG_IPG			GENMASK(26, 24)
+#define GMAC_CONFIG_IPG_SHIFT		24
+#define GMAC_CONFIG_2K			BIT(22)
+#define GMAC_CONFIG_ACS			BIT(20)
+#define GMAC_CONFIG_BE			BIT(18)
+#define GMAC_CONFIG_JD			BIT(17)
+#define GMAC_CONFIG_JE			BIT(16)
+#define GMAC_CONFIG_PS			BIT(15)
+#define GMAC_CONFIG_FES			BIT(14)
+#define GMAC_CONFIG_FES_SHIFT		14
+#define GMAC_CONFIG_DM			BIT(13)
+#define GMAC_CONFIG_LM			BIT(12)
+#define GMAC_CONFIG_DCRS		BIT(9)
+#define GMAC_CONFIG_TE			BIT(1)
+#define GMAC_CONFIG_RE			BIT(0)
+
+/* MAC extended config */
+#define GMAC_CONFIG_EIPG		GENMASK(29, 25)
+#define GMAC_CONFIG_EIPG_SHIFT		25
+#define GMAC_CONFIG_EIPG_EN		BIT(24)
+#define GMAC_CONFIG_HDSMS		GENMASK(22, 20)
+#define GMAC_CONFIG_HDSMS_SHIFT		20
+#define GMAC_CONFIG_HDSMS_256		(0x2 << GMAC_CONFIG_HDSMS_SHIFT)
+
+/* MAC HW features0 bitmap */
+#define GMAC_HW_FEAT_SAVLANINS		BIT(27)
+#define GMAC_HW_FEAT_ADDMAC		BIT(18)
+#define GMAC_HW_FEAT_RXCOESEL		BIT(16)
+#define GMAC_HW_FEAT_TXCOSEL		BIT(14)
+#define GMAC_HW_FEAT_EEESEL		BIT(13)
+#define GMAC_HW_FEAT_TSSEL		BIT(12)
+#define GMAC_HW_FEAT_ARPOFFSEL		BIT(9)
+#define GMAC_HW_FEAT_MMCSEL		BIT(8)
+#define GMAC_HW_FEAT_MGKSEL		BIT(7)
+#define GMAC_HW_FEAT_RWKSEL		BIT(6)
+#define GMAC_HW_FEAT_SMASEL		BIT(5)
+#define GMAC_HW_FEAT_VLHASH		BIT(4)
+#define GMAC_HW_FEAT_PCSSEL		BIT(3)
+#define GMAC_HW_FEAT_HDSEL		BIT(2)
+#define GMAC_HW_FEAT_GMIISEL		BIT(1)
+#define GMAC_HW_FEAT_MIISEL		BIT(0)
+
+/* MAC HW features1 bitmap */
+#define GMAC_HW_FEAT_L3L4FNUM		GENMASK(30, 27)
+#define GMAC_HW_HASH_TB_SZ		GENMASK(25, 24)
+#define GMAC_HW_FEAT_AVSEL		BIT(20)
+#define GMAC_HW_TSOEN			BIT(18)
+#define GMAC_HW_FEAT_SPHEN		BIT(17)
+#define GMAC_HW_ADDR64			GENMASK(15, 14)
+#define GMAC_HW_TXFIFOSIZE		GENMASK(10, 6)
+#define GMAC_HW_RXFIFOSIZE		GENMASK(4, 0)
+
+/* MAC HW features2 bitmap */
+#define GMAC_HW_FEAT_AUXSNAPNUM		GENMASK(30, 28)
+#define GMAC_HW_FEAT_PPSOUTNUM		GENMASK(26, 24)
+#define GMAC_HW_FEAT_TXCHCNT		GENMASK(21, 18)
+#define GMAC_HW_FEAT_RXCHCNT		GENMASK(15, 12)
+#define GMAC_HW_FEAT_TXQCNT		GENMASK(9, 6)
+#define GMAC_HW_FEAT_RXQCNT		GENMASK(3, 0)
+
+/* MAC HW features3 bitmap */
+#define GMAC_HW_FEAT_ASP		GENMASK(29, 28)
+#define GMAC_HW_FEAT_TBSSEL		BIT(27)
+#define GMAC_HW_FEAT_FPESEL		BIT(26)
+#define GMAC_HW_FEAT_ESTWID		GENMASK(21, 20)
+#define GMAC_HW_FEAT_ESTDEP		GENMASK(19, 17)
+#define GMAC_HW_FEAT_ESTSEL		BIT(16)
+#define GMAC_HW_FEAT_FRPES		GENMASK(14, 13)
+#define GMAC_HW_FEAT_FRPBS		GENMASK(12, 11)
+#define GMAC_HW_FEAT_FRPSEL		BIT(10)
+#define GMAC_HW_FEAT_DVLAN		BIT(5)
+#define GMAC_HW_FEAT_NRVF		GENMASK(2, 0)
+
+/* MAC extended config 1 */
+#define GMAC_CONFIG1_SAVE_EN		BIT(24)
+#define GMAC_CONFIG1_SPLM(v)		FIELD_PREP(GENMASK(9, 8), v)
+
+/* GMAC GPIO Status reg */
+#define GMAC_GPO0			BIT(16)
+#define GMAC_GPO1			BIT(17)
+#define GMAC_GPO2			BIT(18)
+#define GMAC_GPO3			BIT(19)
+
+/* MAC HW ADDR regs */
+#define GMAC_HI_DCS			GENMASK(18, 16)
+#define GMAC_HI_DCS_SHIFT		16
+#define GMAC_HI_REG_AE			BIT(31)
+
+/* L3/L4 Filters regs */
+#define GMAC_L4DPIM0			BIT(21)
+#define GMAC_L4DPM0			BIT(20)
+#define GMAC_L4SPIM0			BIT(19)
+#define GMAC_L4SPM0			BIT(18)
+#define GMAC_L4PEN0			BIT(16)
+#define GMAC_L3DAIM0			BIT(5)
+#define GMAC_L3DAM0			BIT(4)
+#define GMAC_L3SAIM0			BIT(3)
+#define GMAC_L3SAM0			BIT(2)
+#define GMAC_L3PEN0			BIT(0)
+#define GMAC_L4DP0			GENMASK(31, 16)
+#define GMAC_L4DP0_SHIFT		16
+#define GMAC_L4SP0			GENMASK(15, 0)
+
+/* MAC Timestamp Status */
+#define GMAC_TIMESTAMP_AUXTSTRIG	BIT(2)
+#define GMAC_TIMESTAMP_ATSNS_MASK	GENMASK(29, 25)
+#define GMAC_TIMESTAMP_ATSNS_SHIFT	25
+
+/*  MTL registers */
+#define MTL_OPERATION_MODE		0x00000c00
+#define MTL_FRPE			BIT(15)
+#define MTL_OPERATION_SCHALG_MASK	GENMASK(6, 5)
+#define MTL_OPERATION_SCHALG_WRR	(0x0 << 5)
+#define MTL_OPERATION_SCHALG_WFQ	(0x1 << 5)
+#define MTL_OPERATION_SCHALG_DWRR	(0x2 << 5)
+#define MTL_OPERATION_SCHALG_SP		(0x3 << 5)
+#define MTL_OPERATION_RAA		BIT(2)
+#define MTL_OPERATION_RAA_SP		(0x0 << 2)
+#define MTL_OPERATION_RAA_WSP		(0x1 << 2)
+
+#define MTL_INT_STATUS			0x00000c20
+#define MTL_INT_QX(x)			BIT(x)
+
+#define MTL_RXQ_DMA_MAP0		0x00000c30 /* queue 0 to 3 */
+#define MTL_RXQ_DMA_MAP1		0x00000c34 /* queue 4 to 7 */
+#define MTL_RXQ_DMA_QXMDMACH_MASK(x)	(0xf << 8 * (x))
+#define MTL_RXQ_DMA_QXMDMACH(chan, q)	((chan) << (8 * (q)))
+
+#define MTL_CHAN_BASE_ADDR		0x00000d00
+#define MTL_CHAN_BASE_OFFSET		0x40
+
+static inline u32 mtl_chanx_base_addr(const struct dwmac4_addrs *addrs,
+				      const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_chan + (x * addrs->mtl_chan_offset);
+	else
+		addr = MTL_CHAN_BASE_ADDR + (x * MTL_CHAN_BASE_OFFSET);
+
+	return addr;
+}
+
+#define MTL_CHAN_TX_OP_MODE(addrs, x)	mtl_chanx_base_addr(addrs, x)
+#define MTL_CHAN_TX_DEBUG(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x8)
+#define MTL_CHAN_INT_CTRL(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x2c)
+#define MTL_CHAN_RX_OP_MODE(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x30)
+#define MTL_CHAN_RX_DEBUG(addrs, x)	(mtl_chanx_base_addr(addrs, x) + 0x38)
+
+#define MTL_OP_MODE_RSF			BIT(5)
+#define MTL_OP_MODE_TXQEN_MASK		GENMASK(3, 2)
+#define MTL_OP_MODE_TXQEN_AV		BIT(2)
+#define MTL_OP_MODE_TXQEN		BIT(3)
+#define MTL_OP_MODE_TSF			BIT(1)
+
+#define MTL_OP_MODE_TQS_MASK		GENMASK(24, 16)
+#define MTL_OP_MODE_TQS_SHIFT		16
+
+#define MTL_OP_MODE_TTC_MASK		0x70
+#define MTL_OP_MODE_TTC_SHIFT		4
+
+#define MTL_OP_MODE_TTC_32		0
+#define MTL_OP_MODE_TTC_64		(1 << MTL_OP_MODE_TTC_SHIFT)
+#define MTL_OP_MODE_TTC_96		(2 << MTL_OP_MODE_TTC_SHIFT)
+#define MTL_OP_MODE_TTC_128		(3 << MTL_OP_MODE_TTC_SHIFT)
+#define MTL_OP_MODE_TTC_192		(4 << MTL_OP_MODE_TTC_SHIFT)
+#define MTL_OP_MODE_TTC_256		(5 << MTL_OP_MODE_TTC_SHIFT)
+#define MTL_OP_MODE_TTC_384		(6 << MTL_OP_MODE_TTC_SHIFT)
+#define MTL_OP_MODE_TTC_512		(7 << MTL_OP_MODE_TTC_SHIFT)
+
+#define MTL_OP_MODE_RQS_MASK		GENMASK(29, 20)
+#define MTL_OP_MODE_RQS_SHIFT		20
+
+#define MTL_OP_MODE_RFD_MASK		GENMASK(19, 14)
+#define MTL_OP_MODE_RFD_SHIFT		14
+
+#define MTL_OP_MODE_RFA_MASK		GENMASK(13, 8)
+#define MTL_OP_MODE_RFA_SHIFT		8
+
+#define MTL_OP_MODE_EHFC		BIT(7)
+#define MTL_OP_MODE_DIS_TCP_EF		BIT(6)
+
+#define MTL_OP_MODE_RTC_MASK		GENMASK(1, 0)
+#define MTL_OP_MODE_RTC_SHIFT		0
+
+#define MTL_OP_MODE_RTC_32		(1 << MTL_OP_MODE_RTC_SHIFT)
+#define MTL_OP_MODE_RTC_64		0
+#define MTL_OP_MODE_RTC_96		(2 << MTL_OP_MODE_RTC_SHIFT)
+#define MTL_OP_MODE_RTC_128		(3 << MTL_OP_MODE_RTC_SHIFT)
+
+/* MTL ETS Control register */
+#define MTL_ETS_CTRL_BASE_ADDR		0x00000d10
+#define MTL_ETS_CTRL_BASE_OFFSET	0x40
+
+static inline u32 mtl_etsx_ctrl_base_addr(const struct dwmac4_addrs *addrs,
+					  const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_ets_ctrl + (x * addrs->mtl_ets_ctrl_offset);
+	else
+		addr = MTL_ETS_CTRL_BASE_ADDR + (x * MTL_ETS_CTRL_BASE_OFFSET);
+
+	return addr;
+}
+
+#define MTL_ETS_CTRL_CC			BIT(3)
+#define MTL_ETS_CTRL_AVALG		BIT(2)
+
+/* MTL Queue Quantum Weight */
+#define MTL_TXQ_WEIGHT_BASE_ADDR	0x00000d18
+#define MTL_TXQ_WEIGHT_BASE_OFFSET	0x40
+
+static inline u32 mtl_txqx_weight_base_addr(const struct dwmac4_addrs *addrs,
+					    const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_txq_weight + (x * addrs->mtl_txq_weight_offset);
+	else
+		addr = MTL_TXQ_WEIGHT_BASE_ADDR + (x * MTL_TXQ_WEIGHT_BASE_OFFSET);
+
+	return addr;
+}
+
+#define MTL_TXQ_WEIGHT_ISCQW_MASK	GENMASK(20, 0)
+
+/* MTL sendSlopeCredit register */
+#define MTL_SEND_SLP_CRED_BASE_ADDR	0x00000d1c
+#define MTL_SEND_SLP_CRED_OFFSET	0x40
+
+static inline u32 mtl_send_slp_credx_base_addr(const struct dwmac4_addrs *addrs,
+					       const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_send_slp_cred + (x * addrs->mtl_send_slp_cred_offset);
+	else
+		addr = MTL_SEND_SLP_CRED_BASE_ADDR + (x * MTL_SEND_SLP_CRED_OFFSET);
+
+	return addr;
+}
+
+#define MTL_SEND_SLP_CRED_SSC_MASK	GENMASK(13, 0)
+
+/* MTL hiCredit register */
+#define MTL_HIGH_CRED_BASE_ADDR		0x00000d20
+#define MTL_HIGH_CRED_OFFSET		0x40
+
+static inline u32 mtl_high_credx_base_addr(const struct dwmac4_addrs *addrs,
+					   const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_high_cred + (x * addrs->mtl_high_cred_offset);
+	else
+		addr = MTL_HIGH_CRED_BASE_ADDR + (x * MTL_HIGH_CRED_OFFSET);
+
+	return addr;
+}
+
+#define MTL_HIGH_CRED_HC_MASK		GENMASK(28, 0)
+
+/* MTL loCredit register */
+#define MTL_LOW_CRED_BASE_ADDR		0x00000d24
+#define MTL_LOW_CRED_OFFSET		0x40
+
+static inline u32 mtl_low_credx_base_addr(const struct dwmac4_addrs *addrs,
+					  const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->mtl_low_cred + (x * addrs->mtl_low_cred_offset);
+	else
+		addr = MTL_LOW_CRED_BASE_ADDR + (x * MTL_LOW_CRED_OFFSET);
+
+	return addr;
+}
+
+#define MTL_HIGH_CRED_LC_MASK		GENMASK(28, 0)
+
+/*  MTL debug */
+#define MTL_DEBUG_TXSTSFSTS		BIT(5)
+#define MTL_DEBUG_TXFSTS		BIT(4)
+#define MTL_DEBUG_TWCSTS		BIT(3)
+
+/* MTL debug: Tx FIFO Read Controller Status */
+#define MTL_DEBUG_TRCSTS_MASK		GENMASK(2, 1)
+#define MTL_DEBUG_TRCSTS_SHIFT		1
+#define MTL_DEBUG_TRCSTS_IDLE		0
+#define MTL_DEBUG_TRCSTS_READ		1
+#define MTL_DEBUG_TRCSTS_TXW		2
+#define MTL_DEBUG_TRCSTS_WRITE		3
+#define MTL_DEBUG_TXPAUSED		BIT(0)
+
+/* MAC debug: GMII or MII Transmit Protocol Engine Status */
+#define MTL_DEBUG_RXFSTS_MASK		GENMASK(5, 4)
+#define MTL_DEBUG_RXFSTS_SHIFT		4
+#define MTL_DEBUG_RXFSTS_EMPTY		0
+#define MTL_DEBUG_RXFSTS_BT		1
+#define MTL_DEBUG_RXFSTS_AT		2
+#define MTL_DEBUG_RXFSTS_FULL		3
+#define MTL_DEBUG_RRCSTS_MASK		GENMASK(2, 1)
+#define MTL_DEBUG_RRCSTS_SHIFT		1
+#define MTL_DEBUG_RRCSTS_IDLE		0
+#define MTL_DEBUG_RRCSTS_RDATA		1
+#define MTL_DEBUG_RRCSTS_RSTAT		2
+#define MTL_DEBUG_RRCSTS_FLUSH		3
+#define MTL_DEBUG_RWCSTS		BIT(0)
+
+/*  MTL interrupt */
+#define MTL_RX_OVERFLOW_INT_EN		BIT(24)
+#define MTL_RX_OVERFLOW_INT		BIT(16)
+
+/* Default operating mode of the MAC */
+#define GMAC_CORE_INIT (GMAC_CONFIG_JD | GMAC_CONFIG_PS | \
+			GMAC_CONFIG_BE | GMAC_CONFIG_DCRS | \
+			GMAC_CONFIG_JE)
+
+/* To dump the core regs excluding  the Address Registers */
+#define	GMAC_REG_NUM	132
+
+/*  MTL debug */
+#define MTL_DEBUG_TXSTSFSTS		BIT(5)
+#define MTL_DEBUG_TXFSTS		BIT(4)
+#define MTL_DEBUG_TWCSTS		BIT(3)
+
+/* MTL debug: Tx FIFO Read Controller Status */
+#define MTL_DEBUG_TRCSTS_MASK		GENMASK(2, 1)
+#define MTL_DEBUG_TRCSTS_SHIFT		1
+#define MTL_DEBUG_TRCSTS_IDLE		0
+#define MTL_DEBUG_TRCSTS_READ		1
+#define MTL_DEBUG_TRCSTS_TXW		2
+#define MTL_DEBUG_TRCSTS_WRITE		3
+#define MTL_DEBUG_TXPAUSED		BIT(0)
+
+/* MAC debug: GMII or MII Transmit Protocol Engine Status */
+#define MTL_DEBUG_RXFSTS_MASK		GENMASK(5, 4)
+#define MTL_DEBUG_RXFSTS_SHIFT		4
+#define MTL_DEBUG_RXFSTS_EMPTY		0
+#define MTL_DEBUG_RXFSTS_BT		1
+#define MTL_DEBUG_RXFSTS_AT		2
+#define MTL_DEBUG_RXFSTS_FULL		3
+#define MTL_DEBUG_RRCSTS_MASK		GENMASK(2, 1)
+#define MTL_DEBUG_RRCSTS_SHIFT		1
+#define MTL_DEBUG_RRCSTS_IDLE		0
+#define MTL_DEBUG_RRCSTS_RDATA		1
+#define MTL_DEBUG_RRCSTS_RSTAT		2
+#define MTL_DEBUG_RRCSTS_FLUSH		3
+#define MTL_DEBUG_RWCSTS		BIT(0)
+
+/* SGMII/RGMII status register */
+#define GMAC_PHYIF_CTRLSTATUS_TC		BIT(0)
+#define GMAC_PHYIF_CTRLSTATUS_LUD		BIT(1)
+#define GMAC_PHYIF_CTRLSTATUS_SMIDRXS		BIT(4)
+#define GMAC_PHYIF_CTRLSTATUS_LNKMOD		BIT(16)
+#define GMAC_PHYIF_CTRLSTATUS_SPEED		GENMASK(18, 17)
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_SHIFT	17
+#define GMAC_PHYIF_CTRLSTATUS_LNKSTS		BIT(19)
+#define GMAC_PHYIF_CTRLSTATUS_JABTO		BIT(20)
+#define GMAC_PHYIF_CTRLSTATUS_FALSECARDET	BIT(21)
+/* LNKSPEED */
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_125		0x2
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_25		0x1
+#define GMAC_PHYIF_CTRLSTATUS_SPEED_2_5		0x0
+
+extern const struct stmmac_dma_ops ec_dwmac4_dma_ops;
+extern const struct stmmac_dma_ops ec_dwmac410_dma_ops;
+#endif /* __DWMAC4_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac4_core-ethercat.c b/drivers/net/ethercat/device/K3/dwmac4_core-ethercat.c
new file mode 100644
index 000000000000..85547d1f3f8f
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac4_core-ethercat.c
@@ -0,0 +1,1075 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This is the driver for the GMAC on-chip Ethernet controller for ST SoCs.
+ * DWC Ether MAC version 4.00  has been used for developing this code.
+ *
+ * This only implements the mac core functions for this chip.
+ *
+ * Copyright (C) 2015  STMicroelectronics Ltd
+ *
+ * Author: Alexandre Torgue <alexandre.torgue@st.com>
+ */
+
+#include <linux/crc32.h>
+#include <linux/slab.h>
+#include <linux/ethtool.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include "stmmac-ethercat.h"
+#include "stmmac_fpe-ethercat.h"
+#include "stmmac_pcs-ethercat.h"
+#include "stmmac_vlan-ethercat.h"
+#include "dwmac4-ethercat.h"
+#include "dwmac5-ethercat.h"
+
+static void dwmac4_core_init(struct mac_device_info *hw,
+			     struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + GMAC_CONFIG);
+	unsigned long clk_rate;
+
+	value |= GMAC_CORE_INIT;
+
+	if (hw->ps) {
+		value |= GMAC_CONFIG_TE;
+
+		value &= hw->link.speed_mask;
+		switch (hw->ps) {
+		case SPEED_1000:
+			value |= hw->link.speed1000;
+			break;
+		case SPEED_100:
+			value |= hw->link.speed100;
+			break;
+		case SPEED_10:
+			value |= hw->link.speed10;
+			break;
+		}
+	}
+
+	writel(value, ioaddr + GMAC_CONFIG);
+
+	/* Configure LPI 1us counter to number of CSR clock ticks in 1us - 1 */
+	clk_rate = clk_get_rate(priv->plat->stmmac_clk);
+	writel((clk_rate / 1000000) - 1, ioaddr + GMAC4_MAC_ONEUS_TIC_COUNTER);
+
+	/* Enable GMAC interrupts */
+	value = GMAC_INT_DEFAULT_ENABLE;
+
+	if (hw->pcs)
+		value |= GMAC_PCS_IRQ_DEFAULT;
+
+	writel(value, ioaddr + GMAC_INT_EN);
+
+	if (GMAC_INT_DEFAULT_ENABLE & GMAC_INT_TSIE)
+		init_waitqueue_head(&priv->tstamp_busy_wait);
+}
+
+static void dwmac4_update_caps(struct stmmac_priv *priv)
+{
+	if (priv->plat->tx_queues_to_use > 1)
+		priv->hw->link.caps &= ~(MAC_10HD | MAC_100HD | MAC_1000HD);
+	else
+		priv->hw->link.caps |= (MAC_10HD | MAC_100HD | MAC_1000HD);
+}
+
+static void dwmac4_rx_queue_enable(struct mac_device_info *hw,
+				   u8 mode, u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + GMAC_RXQ_CTRL0);
+
+	value &= GMAC_RX_QUEUE_CLEAR(queue);
+	if (mode == MTL_QUEUE_AVB)
+		value |= GMAC_RX_AV_QUEUE_ENABLE(queue);
+	else if (mode == MTL_QUEUE_DCB)
+		value |= GMAC_RX_DCB_QUEUE_ENABLE(queue);
+
+	writel(value, ioaddr + GMAC_RXQ_CTRL0);
+}
+
+static void dwmac4_rx_queue_priority(struct mac_device_info *hw,
+				     u32 prio, u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 clear_mask = 0;
+	u32 ctrl2, ctrl3;
+	int i;
+
+	ctrl2 = readl(ioaddr + GMAC_RXQ_CTRL2);
+	ctrl3 = readl(ioaddr + GMAC_RXQ_CTRL3);
+
+	/* The software must ensure that the same priority
+	 * is not mapped to multiple Rx queues
+	 */
+	for (i = 0; i < 4; i++)
+		clear_mask |= ((prio << GMAC_RXQCTRL_PSRQX_SHIFT(i)) &
+						GMAC_RXQCTRL_PSRQX_MASK(i));
+
+	ctrl2 &= ~clear_mask;
+	ctrl3 &= ~clear_mask;
+
+	/* First assign new priorities to a queue, then
+	 * clear them from others queues
+	 */
+	if (queue < 4) {
+		ctrl2 |= (prio << GMAC_RXQCTRL_PSRQX_SHIFT(queue)) &
+						GMAC_RXQCTRL_PSRQX_MASK(queue);
+
+		writel(ctrl2, ioaddr + GMAC_RXQ_CTRL2);
+		writel(ctrl3, ioaddr + GMAC_RXQ_CTRL3);
+	} else {
+		queue -= 4;
+
+		ctrl3 |= (prio << GMAC_RXQCTRL_PSRQX_SHIFT(queue)) &
+						GMAC_RXQCTRL_PSRQX_MASK(queue);
+
+		writel(ctrl3, ioaddr + GMAC_RXQ_CTRL3);
+		writel(ctrl2, ioaddr + GMAC_RXQ_CTRL2);
+	}
+}
+
+static void dwmac4_tx_queue_priority(struct mac_device_info *hw,
+				     u32 prio, u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 base_register;
+	u32 value;
+
+	base_register = (queue < 4) ? GMAC_TXQ_PRTY_MAP0 : GMAC_TXQ_PRTY_MAP1;
+	if (queue >= 4)
+		queue -= 4;
+
+	value = readl(ioaddr + base_register);
+
+	value &= ~GMAC_TXQCTRL_PSTQX_MASK(queue);
+	value |= (prio << GMAC_TXQCTRL_PSTQX_SHIFT(queue)) &
+						GMAC_TXQCTRL_PSTQX_MASK(queue);
+
+	writel(value, ioaddr + base_register);
+}
+
+static void dwmac4_rx_queue_routing(struct mac_device_info *hw,
+				    u8 packet, u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	static const struct stmmac_rx_routing route_possibilities[] = {
+		{ GMAC_RXQCTRL_AVCPQ_MASK, GMAC_RXQCTRL_AVCPQ_SHIFT },
+		{ GMAC_RXQCTRL_PTPQ_MASK, GMAC_RXQCTRL_PTPQ_SHIFT },
+		{ GMAC_RXQCTRL_DCBCPQ_MASK, GMAC_RXQCTRL_DCBCPQ_SHIFT },
+		{ GMAC_RXQCTRL_UPQ_MASK, GMAC_RXQCTRL_UPQ_SHIFT },
+		{ GMAC_RXQCTRL_MCBCQ_MASK, GMAC_RXQCTRL_MCBCQ_SHIFT },
+	};
+
+	value = readl(ioaddr + GMAC_RXQ_CTRL1);
+
+	/* routing configuration */
+	value &= ~route_possibilities[packet - 1].reg_mask;
+	value |= (queue << route_possibilities[packet-1].reg_shift) &
+		 route_possibilities[packet - 1].reg_mask;
+
+	/* some packets require extra ops */
+	if (packet == PACKET_AVCPQ) {
+		value &= ~GMAC_RXQCTRL_TACPQE;
+		value |= 0x1 << GMAC_RXQCTRL_TACPQE_SHIFT;
+	} else if (packet == PACKET_MCBCQ) {
+		value &= ~GMAC_RXQCTRL_MCBCQEN;
+		value |= 0x1 << GMAC_RXQCTRL_MCBCQEN_SHIFT;
+	}
+
+	writel(value, ioaddr + GMAC_RXQ_CTRL1);
+}
+
+static void dwmac4_prog_mtl_rx_algorithms(struct mac_device_info *hw,
+					  u32 rx_alg)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + MTL_OPERATION_MODE);
+
+	value &= ~MTL_OPERATION_RAA;
+	switch (rx_alg) {
+	case MTL_RX_ALGORITHM_SP:
+		value |= MTL_OPERATION_RAA_SP;
+		break;
+	case MTL_RX_ALGORITHM_WSP:
+		value |= MTL_OPERATION_RAA_WSP;
+		break;
+	default:
+		break;
+	}
+
+	writel(value, ioaddr + MTL_OPERATION_MODE);
+}
+
+static void dwmac4_prog_mtl_tx_algorithms(struct mac_device_info *hw,
+					  u32 tx_alg)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + MTL_OPERATION_MODE);
+
+	value &= ~MTL_OPERATION_SCHALG_MASK;
+	switch (tx_alg) {
+	case MTL_TX_ALGORITHM_WRR:
+		value |= MTL_OPERATION_SCHALG_WRR;
+		break;
+	case MTL_TX_ALGORITHM_WFQ:
+		value |= MTL_OPERATION_SCHALG_WFQ;
+		break;
+	case MTL_TX_ALGORITHM_DWRR:
+		value |= MTL_OPERATION_SCHALG_DWRR;
+		break;
+	case MTL_TX_ALGORITHM_SP:
+		value |= MTL_OPERATION_SCHALG_SP;
+		break;
+	default:
+		break;
+	}
+
+	writel(value, ioaddr + MTL_OPERATION_MODE);
+}
+
+static void dwmac4_set_mtl_tx_queue_weight(struct stmmac_priv *priv,
+					   struct mac_device_info *hw,
+					   u32 weight, u32 queue)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + mtl_txqx_weight_base_addr(dwmac4_addrs,
+							     queue));
+
+	value &= ~MTL_TXQ_WEIGHT_ISCQW_MASK;
+	value |= weight & MTL_TXQ_WEIGHT_ISCQW_MASK;
+	writel(value, ioaddr + mtl_txqx_weight_base_addr(dwmac4_addrs, queue));
+}
+
+static void dwmac4_map_mtl_dma(struct mac_device_info *hw, u32 queue, u32 chan)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	if (queue < 4) {
+		value = readl(ioaddr + MTL_RXQ_DMA_MAP0);
+		value &= ~MTL_RXQ_DMA_QXMDMACH_MASK(queue);
+		value |= MTL_RXQ_DMA_QXMDMACH(chan, queue);
+		writel(value, ioaddr + MTL_RXQ_DMA_MAP0);
+	} else {
+		value = readl(ioaddr + MTL_RXQ_DMA_MAP1);
+		value &= ~MTL_RXQ_DMA_QXMDMACH_MASK(queue - 4);
+		value |= MTL_RXQ_DMA_QXMDMACH(chan, queue - 4);
+		writel(value, ioaddr + MTL_RXQ_DMA_MAP1);
+	}
+}
+
+static void dwmac4_config_cbs(struct stmmac_priv *priv,
+			      struct mac_device_info *hw,
+			      u32 send_slope, u32 idle_slope,
+			      u32 high_credit, u32 low_credit, u32 queue)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	pr_debug("Queue %d configured as AVB. Parameters:\n", queue);
+	pr_debug("\tsend_slope: 0x%08x\n", send_slope);
+	pr_debug("\tidle_slope: 0x%08x\n", idle_slope);
+	pr_debug("\thigh_credit: 0x%08x\n", high_credit);
+	pr_debug("\tlow_credit: 0x%08x\n", low_credit);
+
+	/* enable AV algorithm */
+	value = readl(ioaddr + mtl_etsx_ctrl_base_addr(dwmac4_addrs, queue));
+	value |= MTL_ETS_CTRL_AVALG;
+	value |= MTL_ETS_CTRL_CC;
+	writel(value, ioaddr + mtl_etsx_ctrl_base_addr(dwmac4_addrs, queue));
+
+	/* configure send slope */
+	value = readl(ioaddr + mtl_send_slp_credx_base_addr(dwmac4_addrs,
+							    queue));
+	value &= ~MTL_SEND_SLP_CRED_SSC_MASK;
+	value |= send_slope & MTL_SEND_SLP_CRED_SSC_MASK;
+	writel(value, ioaddr + mtl_send_slp_credx_base_addr(dwmac4_addrs,
+							    queue));
+
+	/* configure idle slope (same register as tx weight) */
+	dwmac4_set_mtl_tx_queue_weight(priv, hw, idle_slope, queue);
+
+	/* configure high credit */
+	value = readl(ioaddr + mtl_high_credx_base_addr(dwmac4_addrs, queue));
+	value &= ~MTL_HIGH_CRED_HC_MASK;
+	value |= high_credit & MTL_HIGH_CRED_HC_MASK;
+	writel(value, ioaddr + mtl_high_credx_base_addr(dwmac4_addrs, queue));
+
+	/* configure high credit */
+	value = readl(ioaddr + mtl_low_credx_base_addr(dwmac4_addrs, queue));
+	value &= ~MTL_HIGH_CRED_LC_MASK;
+	value |= low_credit & MTL_HIGH_CRED_LC_MASK;
+	writel(value, ioaddr + mtl_low_credx_base_addr(dwmac4_addrs, queue));
+}
+
+static void dwmac4_dump_regs(struct mac_device_info *hw, u32 *reg_space)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	int i;
+
+	for (i = 0; i < GMAC_REG_NUM; i++)
+		reg_space[i] = readl(ioaddr + i * 4);
+}
+
+static int dwmac4_rx_ipc_enable(struct mac_device_info *hw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + GMAC_CONFIG);
+
+	if (hw->rx_csum)
+		value |= GMAC_CONFIG_IPC;
+	else
+		value &= ~GMAC_CONFIG_IPC;
+
+	writel(value, ioaddr + GMAC_CONFIG);
+
+	value = readl(ioaddr + GMAC_CONFIG);
+
+	return !!(value & GMAC_CONFIG_IPC);
+}
+
+static void dwmac4_pmt(struct mac_device_info *hw, unsigned long mode)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	unsigned int pmt = 0;
+	u32 config;
+
+	if (mode & WAKE_MAGIC) {
+		pr_debug("GMAC: WOL Magic frame\n");
+		pmt |= power_down | magic_pkt_en;
+	}
+	if (mode & WAKE_UCAST) {
+		pr_debug("GMAC: WOL on global unicast\n");
+		pmt |= power_down | global_unicast | wake_up_frame_en;
+	}
+
+	if (pmt) {
+		/* The receiver must be enabled for WOL before powering down */
+		config = readl(ioaddr + GMAC_CONFIG);
+		config |= GMAC_CONFIG_RE;
+		writel(config, ioaddr + GMAC_CONFIG);
+	}
+	writel(pmt, ioaddr + GMAC_PMT);
+}
+
+static void dwmac4_set_umac_addr(struct mac_device_info *hw,
+				 const unsigned char *addr, unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+
+	ec_stmmac_dwmac4_set_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
+				   GMAC_ADDR_LOW(reg_n));
+}
+
+static void dwmac4_get_umac_addr(struct mac_device_info *hw,
+				 unsigned char *addr, unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+
+	ec_stmmac_dwmac4_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
+				   GMAC_ADDR_LOW(reg_n));
+}
+
+static int dwmac4_set_lpi_mode(struct mac_device_info *hw,
+			       enum stmmac_lpi_mode mode,
+			       bool en_tx_lpi_clockgating, u32 et)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value, mask;
+
+	if (mode == STMMAC_LPI_DISABLE) {
+		value = 0;
+	} else {
+		value = LPI_CTRL_STATUS_LPIEN | LPI_CTRL_STATUS_LPITXA;
+
+		if (mode == STMMAC_LPI_TIMER) {
+			/* Return ERANGE if the timer is larger than the
+			 * register field.
+			 */
+			if (et > STMMAC_ET_MAX)
+				return -ERANGE;
+
+			/* Set the hardware LPI entry timer */
+			writel(et, ioaddr + GMAC4_LPI_ENTRY_TIMER);
+
+			/* Interpret a zero LPI entry timer to mean
+			 * immediate entry into LPI mode.
+			 */
+			if (et)
+				value |= LPI_CTRL_STATUS_LPIATE;
+		}
+
+		if (en_tx_lpi_clockgating)
+			value |= LPI_CTRL_STATUS_LPITCSE;
+	}
+
+	mask = LPI_CTRL_STATUS_LPIATE | LPI_CTRL_STATUS_LPIEN |
+	       LPI_CTRL_STATUS_LPITXA | LPI_CTRL_STATUS_LPITCSE;
+
+	value |= readl(ioaddr + GMAC4_LPI_CTRL_STATUS) & ~mask;
+	writel(value, ioaddr + GMAC4_LPI_CTRL_STATUS);
+
+	return 0;
+}
+
+static void dwmac4_set_eee_pls(struct mac_device_info *hw, int link)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + GMAC4_LPI_CTRL_STATUS);
+
+	if (link)
+		value |= LPI_CTRL_STATUS_PLS;
+	else
+		value &= ~LPI_CTRL_STATUS_PLS;
+
+	writel(value, ioaddr + GMAC4_LPI_CTRL_STATUS);
+}
+
+static void dwmac4_set_eee_timer(struct mac_device_info *hw, int ls, int tw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	int value = ((tw & 0xffff)) | ((ls & 0x3ff) << 16);
+
+	/* Program the timers in the LPI timer control register:
+	 * LS: minimum time (ms) for which the link
+	 *  status from PHY should be ok before transmitting
+	 *  the LPI pattern.
+	 * TW: minimum time (us) for which the core waits
+	 *  after it has stopped transmitting the LPI pattern.
+	 */
+	writel(value, ioaddr + GMAC4_LPI_TIMER_CTRL);
+}
+
+static void dwmac4_set_filter(struct mac_device_info *hw,
+			      struct net_device *dev)
+{
+	void __iomem *ioaddr = (void __iomem *)dev->base_addr;
+	int numhashregs = (hw->multicast_filter_bins >> 5);
+	int mcbitslog2 = hw->mcast_bits_log2;
+	unsigned int value;
+	u32 mc_filter[8];
+	int i;
+
+	memset(mc_filter, 0, sizeof(mc_filter));
+
+	value = readl(ioaddr + GMAC_PACKET_FILTER);
+	value &= ~GMAC_PACKET_FILTER_HMC;
+	value &= ~GMAC_PACKET_FILTER_HPF;
+	value &= ~GMAC_PACKET_FILTER_PCF;
+	value &= ~GMAC_PACKET_FILTER_PM;
+	value &= ~GMAC_PACKET_FILTER_PR;
+	value &= ~GMAC_PACKET_FILTER_RA;
+	if (dev->flags & IFF_PROMISC) {
+		/* VLAN Tag Filter Fail Packets Queuing */
+		if (hw->vlan_fail_q_en) {
+			value = readl(ioaddr + GMAC_RXQ_CTRL4);
+			value &= ~GMAC_RXQCTRL_VFFQ_MASK;
+			value |= GMAC_RXQCTRL_VFFQE |
+				 (hw->vlan_fail_q << GMAC_RXQCTRL_VFFQ_SHIFT);
+			writel(value, ioaddr + GMAC_RXQ_CTRL4);
+			value = GMAC_PACKET_FILTER_PR | GMAC_PACKET_FILTER_RA;
+		} else {
+			value = GMAC_PACKET_FILTER_PR | GMAC_PACKET_FILTER_PCF;
+		}
+
+	} else if ((dev->flags & IFF_ALLMULTI) ||
+		   (netdev_mc_count(dev) > hw->multicast_filter_bins)) {
+		/* Pass all multi */
+		value |= GMAC_PACKET_FILTER_PM;
+		/* Set all the bits of the HASH tab */
+		memset(mc_filter, 0xff, sizeof(mc_filter));
+	} else if (!netdev_mc_empty(dev) && (dev->flags & IFF_MULTICAST)) {
+		struct netdev_hw_addr *ha;
+
+		/* Hash filter for multicast */
+		value |= GMAC_PACKET_FILTER_HMC;
+
+		netdev_for_each_mc_addr(ha, dev) {
+			/* The upper n bits of the calculated CRC are used to
+			 * index the contents of the hash table. The number of
+			 * bits used depends on the hardware configuration
+			 * selected at core configuration time.
+			 */
+			u32 bit_nr = bitrev32(~crc32_le(~0, ha->addr,
+					ETH_ALEN)) >> (32 - mcbitslog2);
+			/* The most significant bit determines the register to
+			 * use (H/L) while the other 5 bits determine the bit
+			 * within the register.
+			 */
+			mc_filter[bit_nr >> 5] |= (1 << (bit_nr & 0x1f));
+		}
+	}
+
+	for (i = 0; i < numhashregs; i++)
+		writel(mc_filter[i], ioaddr + GMAC_HASH_TAB(i));
+
+	value |= GMAC_PACKET_FILTER_HPF;
+
+	/* Handle multiple unicast addresses */
+	if (netdev_uc_count(dev) > hw->unicast_filter_entries) {
+		/* Switch to promiscuous mode if more than 128 addrs
+		 * are required
+		 */
+		value |= GMAC_PACKET_FILTER_PR;
+	} else {
+		struct netdev_hw_addr *ha;
+		int reg = 1;
+
+		netdev_for_each_uc_addr(ha, dev) {
+			dwmac4_set_umac_addr(hw, ha->addr, reg);
+			reg++;
+		}
+
+		while (reg < GMAC_MAX_PERFECT_ADDRESSES) {
+			writel(0, ioaddr + GMAC_ADDR_HIGH(reg));
+			writel(0, ioaddr + GMAC_ADDR_LOW(reg));
+			reg++;
+		}
+	}
+
+	/* VLAN filtering */
+	if (dev->flags & IFF_PROMISC && !hw->vlan_fail_q_en)
+		value &= ~GMAC_PACKET_FILTER_VTFE;
+	else if (dev->features & NETIF_F_HW_VLAN_CTAG_FILTER)
+		value |= GMAC_PACKET_FILTER_VTFE;
+
+	writel(value, ioaddr + GMAC_PACKET_FILTER);
+}
+
+static void dwmac4_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
+			     unsigned int fc, unsigned int pause_time,
+			     u32 tx_cnt)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	unsigned int flow = 0;
+	u32 queue = 0;
+
+	pr_debug("GMAC Flow-Control:\n");
+	if (fc & FLOW_RX) {
+		pr_debug("\tReceive Flow-Control ON\n");
+		flow |= GMAC_RX_FLOW_CTRL_RFE;
+	} else {
+		pr_debug("\tReceive Flow-Control OFF\n");
+	}
+	writel(flow, ioaddr + GMAC_RX_FLOW_CTRL);
+
+	if (fc & FLOW_TX) {
+		pr_debug("\tTransmit Flow-Control ON\n");
+
+		if (duplex)
+			pr_debug("\tduplex mode: PAUSE %d\n", pause_time);
+
+		for (queue = 0; queue < tx_cnt; queue++) {
+			flow = GMAC_TX_FLOW_CTRL_TFE;
+
+			if (duplex)
+				flow |=
+				(pause_time << GMAC_TX_FLOW_CTRL_PT_SHIFT);
+
+			writel(flow, ioaddr + GMAC_QX_TX_FLOW_CTRL(queue));
+		}
+	} else {
+		for (queue = 0; queue < tx_cnt; queue++)
+			writel(0, ioaddr + GMAC_QX_TX_FLOW_CTRL(queue));
+	}
+}
+
+static void dwmac4_ctrl_ane(struct stmmac_priv *priv, bool ane, bool srgmi_ral,
+			    bool loopback)
+{
+	dwmac_ctrl_ane(priv->ioaddr, GMAC_PCS_BASE, ane, srgmi_ral, loopback);
+}
+
+/* RGMII or SMII interface */
+static void dwmac4_phystatus(void __iomem *ioaddr, struct stmmac_extra_stats *x)
+{
+	u32 status;
+
+	status = readl(ioaddr + GMAC_PHYIF_CONTROL_STATUS);
+	x->irq_rgmii_n++;
+
+	/* Check the link status */
+	if (status & GMAC_PHYIF_CTRLSTATUS_LNKSTS) {
+		int speed_value;
+
+		x->pcs_link = 1;
+
+		speed_value = ((status & GMAC_PHYIF_CTRLSTATUS_SPEED) >>
+			       GMAC_PHYIF_CTRLSTATUS_SPEED_SHIFT);
+		if (speed_value == GMAC_PHYIF_CTRLSTATUS_SPEED_125)
+			x->pcs_speed = SPEED_1000;
+		else if (speed_value == GMAC_PHYIF_CTRLSTATUS_SPEED_25)
+			x->pcs_speed = SPEED_100;
+		else
+			x->pcs_speed = SPEED_10;
+
+		x->pcs_duplex = (status & GMAC_PHYIF_CTRLSTATUS_LNKMOD);
+
+		pr_info("Link is Up - %d/%s\n", (int)x->pcs_speed,
+			x->pcs_duplex ? "Full" : "Half");
+	} else {
+		x->pcs_link = 0;
+		pr_info("Link is Down\n");
+	}
+}
+
+static int dwmac4_irq_mtl_status(struct stmmac_priv *priv,
+				 struct mac_device_info *hw, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	void __iomem *ioaddr = hw->pcsr;
+	u32 mtl_int_qx_status;
+	int ret = 0;
+
+	mtl_int_qx_status = readl(ioaddr + MTL_INT_STATUS);
+
+	/* Check MTL Interrupt */
+	if (mtl_int_qx_status & MTL_INT_QX(chan)) {
+		/* read Queue x Interrupt status */
+		u32 status = readl(ioaddr + MTL_CHAN_INT_CTRL(dwmac4_addrs,
+							      chan));
+
+		if (status & MTL_RX_OVERFLOW_INT) {
+			/*  clear Interrupt */
+			writel(status | MTL_RX_OVERFLOW_INT,
+			       ioaddr + MTL_CHAN_INT_CTRL(dwmac4_addrs, chan));
+			ret = CORE_IRQ_MTL_RX_OVERFLOW;
+		}
+	}
+
+	return ret;
+}
+
+static int dwmac4_irq_status(struct mac_device_info *hw,
+			     struct stmmac_extra_stats *x)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 intr_status = readl(ioaddr + GMAC_INT_STATUS);
+	u32 intr_enable = readl(ioaddr + GMAC_INT_EN);
+	int ret = 0;
+
+	/* Discard disabled bits */
+	intr_status &= intr_enable;
+
+	/* Not used events (e.g. MMC interrupts) are not handled. */
+	if ((intr_status & mmc_tx_irq))
+		x->mmc_tx_irq_n++;
+	if (unlikely(intr_status & mmc_rx_irq))
+		x->mmc_rx_irq_n++;
+	if (unlikely(intr_status & mmc_rx_csum_offload_irq))
+		x->mmc_rx_csum_offload_irq_n++;
+	/* Clear the PMT bits 5 and 6 by reading the PMT status reg */
+	if (unlikely(intr_status & pmt_irq)) {
+		readl(ioaddr + GMAC_PMT);
+		x->irq_receive_pmt_irq_n++;
+	}
+
+	/* MAC tx/rx EEE LPI entry/exit interrupts */
+	if (intr_status & lpi_irq) {
+		/* Clear LPI interrupt by reading MAC_LPI_Control_Status */
+		u32 status = readl(ioaddr + GMAC4_LPI_CTRL_STATUS);
+
+		if (status & LPI_CTRL_STATUS_TLPIEN) {
+			ret |= CORE_IRQ_TX_PATH_IN_LPI_MODE;
+			x->irq_tx_path_in_lpi_mode_n++;
+		}
+		if (status & LPI_CTRL_STATUS_TLPIEX) {
+			ret |= CORE_IRQ_TX_PATH_EXIT_LPI_MODE;
+			x->irq_tx_path_exit_lpi_mode_n++;
+		}
+		if (status & LPI_CTRL_STATUS_RLPIEN)
+			x->irq_rx_path_in_lpi_mode_n++;
+		if (status & LPI_CTRL_STATUS_RLPIEX)
+			x->irq_rx_path_exit_lpi_mode_n++;
+	}
+
+	dwmac_pcs_isr(ioaddr, GMAC_PCS_BASE, intr_status, x);
+	if (intr_status & PCS_RGSMIIIS_IRQ)
+		dwmac4_phystatus(ioaddr, x);
+
+	return ret;
+}
+
+static void dwmac4_debug(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 struct stmmac_extra_stats *x,
+			 u32 rx_queues, u32 tx_queues)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value;
+	u32 queue;
+
+	for (queue = 0; queue < tx_queues; queue++) {
+		value = readl(ioaddr + MTL_CHAN_TX_DEBUG(dwmac4_addrs, queue));
+
+		if (value & MTL_DEBUG_TXSTSFSTS)
+			x->mtl_tx_status_fifo_full++;
+		if (value & MTL_DEBUG_TXFSTS)
+			x->mtl_tx_fifo_not_empty++;
+		if (value & MTL_DEBUG_TWCSTS)
+			x->mmtl_fifo_ctrl++;
+		if (value & MTL_DEBUG_TRCSTS_MASK) {
+			u32 trcsts = (value & MTL_DEBUG_TRCSTS_MASK)
+				     >> MTL_DEBUG_TRCSTS_SHIFT;
+			if (trcsts == MTL_DEBUG_TRCSTS_WRITE)
+				x->mtl_tx_fifo_read_ctrl_write++;
+			else if (trcsts == MTL_DEBUG_TRCSTS_TXW)
+				x->mtl_tx_fifo_read_ctrl_wait++;
+			else if (trcsts == MTL_DEBUG_TRCSTS_READ)
+				x->mtl_tx_fifo_read_ctrl_read++;
+			else
+				x->mtl_tx_fifo_read_ctrl_idle++;
+		}
+		if (value & MTL_DEBUG_TXPAUSED)
+			x->mac_tx_in_pause++;
+	}
+
+	for (queue = 0; queue < rx_queues; queue++) {
+		value = readl(ioaddr + MTL_CHAN_RX_DEBUG(dwmac4_addrs, queue));
+
+		if (value & MTL_DEBUG_RXFSTS_MASK) {
+			u32 rxfsts = (value & MTL_DEBUG_RXFSTS_MASK)
+				     >> MTL_DEBUG_RRCSTS_SHIFT;
+
+			if (rxfsts == MTL_DEBUG_RXFSTS_FULL)
+				x->mtl_rx_fifo_fill_level_full++;
+			else if (rxfsts == MTL_DEBUG_RXFSTS_AT)
+				x->mtl_rx_fifo_fill_above_thresh++;
+			else if (rxfsts == MTL_DEBUG_RXFSTS_BT)
+				x->mtl_rx_fifo_fill_below_thresh++;
+			else
+				x->mtl_rx_fifo_fill_level_empty++;
+		}
+		if (value & MTL_DEBUG_RRCSTS_MASK) {
+			u32 rrcsts = (value & MTL_DEBUG_RRCSTS_MASK) >>
+				     MTL_DEBUG_RRCSTS_SHIFT;
+
+			if (rrcsts == MTL_DEBUG_RRCSTS_FLUSH)
+				x->mtl_rx_fifo_read_ctrl_flush++;
+			else if (rrcsts == MTL_DEBUG_RRCSTS_RSTAT)
+				x->mtl_rx_fifo_read_ctrl_read_data++;
+			else if (rrcsts == MTL_DEBUG_RRCSTS_RDATA)
+				x->mtl_rx_fifo_read_ctrl_status++;
+			else
+				x->mtl_rx_fifo_read_ctrl_idle++;
+		}
+		if (value & MTL_DEBUG_RWCSTS)
+			x->mtl_rx_fifo_ctrl_active++;
+	}
+
+	/* GMAC debug */
+	value = readl(ioaddr + GMAC_DEBUG);
+
+	if (value & GMAC_DEBUG_TFCSTS_MASK) {
+		u32 tfcsts = (value & GMAC_DEBUG_TFCSTS_MASK)
+			      >> GMAC_DEBUG_TFCSTS_SHIFT;
+
+		if (tfcsts == GMAC_DEBUG_TFCSTS_XFER)
+			x->mac_tx_frame_ctrl_xfer++;
+		else if (tfcsts == GMAC_DEBUG_TFCSTS_GEN_PAUSE)
+			x->mac_tx_frame_ctrl_pause++;
+		else if (tfcsts == GMAC_DEBUG_TFCSTS_WAIT)
+			x->mac_tx_frame_ctrl_wait++;
+		else
+			x->mac_tx_frame_ctrl_idle++;
+	}
+	if (value & GMAC_DEBUG_TPESTS)
+		x->mac_gmii_tx_proto_engine++;
+	if (value & GMAC_DEBUG_RFCFCSTS_MASK)
+		x->mac_rx_frame_ctrl_fifo = (value & GMAC_DEBUG_RFCFCSTS_MASK)
+					    >> GMAC_DEBUG_RFCFCSTS_SHIFT;
+	if (value & GMAC_DEBUG_RPESTS)
+		x->mac_gmii_rx_proto_engine++;
+}
+
+static void dwmac4_set_mac_loopback(void __iomem *ioaddr, bool enable)
+{
+	u32 value = readl(ioaddr + GMAC_CONFIG);
+
+	if (enable)
+		value |= GMAC_CONFIG_LM;
+	else
+		value &= ~GMAC_CONFIG_LM;
+
+	writel(value, ioaddr + GMAC_CONFIG);
+}
+
+static void dwmac4_sarc_configure(void __iomem *ioaddr, int val)
+{
+	u32 value = readl(ioaddr + GMAC_CONFIG);
+
+	value &= ~GMAC_CONFIG_SARC;
+	value |= val << GMAC_CONFIG_SARC_SHIFT;
+
+	writel(value, ioaddr + GMAC_CONFIG);
+}
+
+static void dwmac4_set_arp_offload(struct mac_device_info *hw, bool en,
+				   u32 addr)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	writel(addr, ioaddr + GMAC_ARP_ADDR);
+
+	value = readl(ioaddr + GMAC_CONFIG);
+	if (en)
+		value |= GMAC_CONFIG_ARPEN;
+	else
+		value &= ~GMAC_CONFIG_ARPEN;
+	writel(value, ioaddr + GMAC_CONFIG);
+}
+
+static int dwmac4_config_l3_filter(struct mac_device_info *hw, u32 filter_no,
+				   bool en, bool ipv6, bool sa, bool inv,
+				   u32 match)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + GMAC_PACKET_FILTER);
+	value |= GMAC_PACKET_FILTER_IPFE;
+	writel(value, ioaddr + GMAC_PACKET_FILTER);
+
+	value = readl(ioaddr + GMAC_L3L4_CTRL(filter_no));
+
+	/* For IPv6 not both SA/DA filters can be active */
+	if (ipv6) {
+		value |= GMAC_L3PEN0;
+		value &= ~(GMAC_L3SAM0 | GMAC_L3SAIM0);
+		value &= ~(GMAC_L3DAM0 | GMAC_L3DAIM0);
+		if (sa) {
+			value |= GMAC_L3SAM0;
+			if (inv)
+				value |= GMAC_L3SAIM0;
+		} else {
+			value |= GMAC_L3DAM0;
+			if (inv)
+				value |= GMAC_L3DAIM0;
+		}
+	} else {
+		value &= ~GMAC_L3PEN0;
+		if (sa) {
+			value |= GMAC_L3SAM0;
+			if (inv)
+				value |= GMAC_L3SAIM0;
+		} else {
+			value |= GMAC_L3DAM0;
+			if (inv)
+				value |= GMAC_L3DAIM0;
+		}
+	}
+
+	writel(value, ioaddr + GMAC_L3L4_CTRL(filter_no));
+
+	if (sa) {
+		writel(match, ioaddr + GMAC_L3_ADDR0(filter_no));
+	} else {
+		writel(match, ioaddr + GMAC_L3_ADDR1(filter_no));
+	}
+
+	if (!en)
+		writel(0, ioaddr + GMAC_L3L4_CTRL(filter_no));
+
+	return 0;
+}
+
+static int dwmac4_config_l4_filter(struct mac_device_info *hw, u32 filter_no,
+				   bool en, bool udp, bool sa, bool inv,
+				   u32 match)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + GMAC_PACKET_FILTER);
+	value |= GMAC_PACKET_FILTER_IPFE;
+	writel(value, ioaddr + GMAC_PACKET_FILTER);
+
+	value = readl(ioaddr + GMAC_L3L4_CTRL(filter_no));
+	if (udp) {
+		value |= GMAC_L4PEN0;
+	} else {
+		value &= ~GMAC_L4PEN0;
+	}
+
+	value &= ~(GMAC_L4SPM0 | GMAC_L4SPIM0);
+	value &= ~(GMAC_L4DPM0 | GMAC_L4DPIM0);
+	if (sa) {
+		value |= GMAC_L4SPM0;
+		if (inv)
+			value |= GMAC_L4SPIM0;
+	} else {
+		value |= GMAC_L4DPM0;
+		if (inv)
+			value |= GMAC_L4DPIM0;
+	}
+
+	writel(value, ioaddr + GMAC_L3L4_CTRL(filter_no));
+
+	if (sa) {
+		value = match & GMAC_L4SP0;
+	} else {
+		value = (match << GMAC_L4DP0_SHIFT) & GMAC_L4DP0;
+	}
+
+	writel(value, ioaddr + GMAC_L4_ADDR(filter_no));
+
+	if (!en)
+		writel(0, ioaddr + GMAC_L3L4_CTRL(filter_no));
+
+	return 0;
+}
+
+const struct stmmac_ops ec_dwmac4_ops = {
+	.core_init = dwmac4_core_init,
+	.update_caps = dwmac4_update_caps,
+	.set_mac = ec_stmmac_set_mac,
+	.rx_ipc = dwmac4_rx_ipc_enable,
+	.rx_queue_enable = dwmac4_rx_queue_enable,
+	.rx_queue_prio = dwmac4_rx_queue_priority,
+	.tx_queue_prio = dwmac4_tx_queue_priority,
+	.rx_queue_routing = dwmac4_rx_queue_routing,
+	.prog_mtl_rx_algorithms = dwmac4_prog_mtl_rx_algorithms,
+	.prog_mtl_tx_algorithms = dwmac4_prog_mtl_tx_algorithms,
+	.set_mtl_tx_queue_weight = dwmac4_set_mtl_tx_queue_weight,
+	.map_mtl_to_dma = dwmac4_map_mtl_dma,
+	.config_cbs = dwmac4_config_cbs,
+	.dump_regs = dwmac4_dump_regs,
+	.host_irq_status = dwmac4_irq_status,
+	.host_mtl_irq_status = dwmac4_irq_mtl_status,
+	.flow_ctrl = dwmac4_flow_ctrl,
+	.pmt = dwmac4_pmt,
+	.set_umac_addr = dwmac4_set_umac_addr,
+	.get_umac_addr = dwmac4_get_umac_addr,
+	.set_lpi_mode = dwmac4_set_lpi_mode,
+	.set_eee_timer = dwmac4_set_eee_timer,
+	.set_eee_pls = dwmac4_set_eee_pls,
+	.pcs_ctrl_ane = dwmac4_ctrl_ane,
+	.debug = dwmac4_debug,
+	.set_filter = dwmac4_set_filter,
+	.set_mac_loopback = dwmac4_set_mac_loopback,
+	.sarc_configure = dwmac4_sarc_configure,
+	.set_arp_offload = dwmac4_set_arp_offload,
+	.config_l3_filter = dwmac4_config_l3_filter,
+	.config_l4_filter = dwmac4_config_l4_filter,
+};
+
+const struct stmmac_ops ec_dwmac410_ops = {
+	.core_init = dwmac4_core_init,
+	.update_caps = dwmac4_update_caps,
+	.set_mac = ec_stmmac_dwmac4_set_mac,
+	.rx_ipc = dwmac4_rx_ipc_enable,
+	.rx_queue_enable = dwmac4_rx_queue_enable,
+	.rx_queue_prio = dwmac4_rx_queue_priority,
+	.tx_queue_prio = dwmac4_tx_queue_priority,
+	.rx_queue_routing = dwmac4_rx_queue_routing,
+	.prog_mtl_rx_algorithms = dwmac4_prog_mtl_rx_algorithms,
+	.prog_mtl_tx_algorithms = dwmac4_prog_mtl_tx_algorithms,
+	.set_mtl_tx_queue_weight = dwmac4_set_mtl_tx_queue_weight,
+	.map_mtl_to_dma = dwmac4_map_mtl_dma,
+	.config_cbs = dwmac4_config_cbs,
+	.dump_regs = dwmac4_dump_regs,
+	.host_irq_status = dwmac4_irq_status,
+	.host_mtl_irq_status = dwmac4_irq_mtl_status,
+	.flow_ctrl = dwmac4_flow_ctrl,
+	.pmt = dwmac4_pmt,
+	.set_umac_addr = dwmac4_set_umac_addr,
+	.get_umac_addr = dwmac4_get_umac_addr,
+	.set_lpi_mode = dwmac4_set_lpi_mode,
+	.set_eee_timer = dwmac4_set_eee_timer,
+	.set_eee_pls = dwmac4_set_eee_pls,
+	.pcs_ctrl_ane = dwmac4_ctrl_ane,
+	.debug = dwmac4_debug,
+	.set_filter = dwmac4_set_filter,
+	.flex_pps_config = ec_dwmac5_flex_pps_config,
+	.set_mac_loopback = dwmac4_set_mac_loopback,
+	.sarc_configure = dwmac4_sarc_configure,
+	.set_arp_offload = dwmac4_set_arp_offload,
+	.config_l3_filter = dwmac4_config_l3_filter,
+	.config_l4_filter = dwmac4_config_l4_filter,
+	.fpe_map_preemption_class = ec_dwmac5_fpe_map_preemption_class,
+};
+
+const struct stmmac_ops ec_dwmac510_ops = {
+	.core_init = dwmac4_core_init,
+	.update_caps = dwmac4_update_caps,
+	.set_mac = ec_stmmac_dwmac4_set_mac,
+	.rx_ipc = dwmac4_rx_ipc_enable,
+	.rx_queue_enable = dwmac4_rx_queue_enable,
+	.rx_queue_prio = dwmac4_rx_queue_priority,
+	.tx_queue_prio = dwmac4_tx_queue_priority,
+	.rx_queue_routing = dwmac4_rx_queue_routing,
+	.prog_mtl_rx_algorithms = dwmac4_prog_mtl_rx_algorithms,
+	.prog_mtl_tx_algorithms = dwmac4_prog_mtl_tx_algorithms,
+	.set_mtl_tx_queue_weight = dwmac4_set_mtl_tx_queue_weight,
+	.map_mtl_to_dma = dwmac4_map_mtl_dma,
+	.config_cbs = dwmac4_config_cbs,
+	.dump_regs = dwmac4_dump_regs,
+	.host_irq_status = dwmac4_irq_status,
+	.host_mtl_irq_status = dwmac4_irq_mtl_status,
+	.flow_ctrl = dwmac4_flow_ctrl,
+	.pmt = dwmac4_pmt,
+	.set_umac_addr = dwmac4_set_umac_addr,
+	.get_umac_addr = dwmac4_get_umac_addr,
+	.set_lpi_mode = dwmac4_set_lpi_mode,
+	.set_eee_timer = dwmac4_set_eee_timer,
+	.set_eee_pls = dwmac4_set_eee_pls,
+	.pcs_ctrl_ane = dwmac4_ctrl_ane,
+	.debug = dwmac4_debug,
+	.set_filter = dwmac4_set_filter,
+	.safety_feat_config = ec_dwmac5_safety_feat_config,
+	.safety_feat_irq_status = ec_dwmac5_safety_feat_irq_status,
+	.safety_feat_dump = ec_dwmac5_safety_feat_dump,
+	.rxp_config = ec_dwmac5_rxp_config,
+	.flex_pps_config = ec_dwmac5_flex_pps_config,
+	.set_mac_loopback = dwmac4_set_mac_loopback,
+	.sarc_configure = dwmac4_sarc_configure,
+	.set_arp_offload = dwmac4_set_arp_offload,
+	.config_l3_filter = dwmac4_config_l3_filter,
+	.config_l4_filter = dwmac4_config_l4_filter,
+	.fpe_map_preemption_class = ec_dwmac5_fpe_map_preemption_class,
+};
+
+int ec_dwmac4_setup(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	dev_info(priv->device, "\tDWMAC4/5\n");
+
+	priv->dev->priv_flags |= IFF_UNICAST_FLT;
+	mac->pcsr = priv->ioaddr;
+	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
+	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
+	mac->mcast_bits_log2 = 0;
+
+	if (mac->multicast_filter_bins)
+		mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins);
+
+	mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+			 MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD;
+	mac->link.duplex = GMAC_CONFIG_DM;
+	mac->link.speed10 = GMAC_CONFIG_PS;
+	mac->link.speed100 = GMAC_CONFIG_FES | GMAC_CONFIG_PS;
+	mac->link.speed1000 = 0;
+	mac->link.speed2500 = GMAC_CONFIG_FES;
+	mac->link.speed_mask = GMAC_CONFIG_FES | GMAC_CONFIG_PS;
+	mac->mii.addr = GMAC_MDIO_ADDR;
+	mac->mii.data = GMAC_MDIO_DATA;
+	mac->mii.addr_shift = 21;
+	mac->mii.addr_mask = GENMASK(25, 21);
+	mac->mii.reg_shift = 16;
+	mac->mii.reg_mask = GENMASK(20, 16);
+	mac->mii.clk_csr_shift = 8;
+	mac->mii.clk_csr_mask = GENMASK(11, 8);
+	mac->num_vlan = ec_stmmac_get_num_vlan(priv->ioaddr);
+
+	return 0;
+}
diff --git a/drivers/net/ethercat/device/K3/dwmac4_descs-ethercat.c b/drivers/net/ethercat/device/K3/dwmac4_descs-ethercat.c
new file mode 100644
index 000000000000..4d6e95b488f6
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac4_descs-ethercat.c
@@ -0,0 +1,600 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This contains the functions to handle the descriptors for DesignWare databook
+ * 4.xx.
+ *
+ * Copyright (C) 2015  STMicroelectronics Ltd
+ *
+ * Author: Alexandre Torgue <alexandre.torgue@st.com>
+ */
+
+#include <linux/stmmac.h>
+#include "common-ethercat.h"
+#include "dwmac4-ethercat.h"
+#include "dwmac4_descs-ethercat.h"
+
+static int dwmac4_wrback_get_tx_status(struct stmmac_extra_stats *x,
+				       struct dma_desc *p,
+				       void __iomem *ioaddr)
+{
+	unsigned int tdes3;
+	int ret = tx_done;
+
+	tdes3 = le32_to_cpu(p->des3);
+
+	/* Get tx owner first */
+	if (unlikely(tdes3 & TDES3_OWN))
+		return tx_dma_own;
+
+	/* Verify tx error by looking at the last segment. */
+	if (likely(!(tdes3 & TDES3_LAST_DESCRIPTOR)))
+		return tx_not_ls;
+
+	if (unlikely(tdes3 & TDES3_ERROR_SUMMARY)) {
+		ret = tx_err;
+
+		if (unlikely(tdes3 & TDES3_JABBER_TIMEOUT))
+			x->tx_jabber++;
+		if (unlikely(tdes3 & TDES3_PACKET_FLUSHED))
+			x->tx_frame_flushed++;
+		if (unlikely(tdes3 & TDES3_LOSS_CARRIER)) {
+			x->tx_losscarrier++;
+		}
+		if (unlikely(tdes3 & TDES3_NO_CARRIER)) {
+			x->tx_carrier++;
+		}
+		if (unlikely((tdes3 & TDES3_LATE_COLLISION) ||
+			     (tdes3 & TDES3_EXCESSIVE_COLLISION)))
+			x->tx_collision +=
+			    (tdes3 & TDES3_COLLISION_COUNT_MASK)
+			    >> TDES3_COLLISION_COUNT_SHIFT;
+
+		if (unlikely(tdes3 & TDES3_EXCESSIVE_DEFERRAL))
+			x->tx_deferred++;
+
+		if (unlikely(tdes3 & TDES3_UNDERFLOW_ERROR)) {
+			x->tx_underflow++;
+			ret |= tx_err_bump_tc;
+		}
+
+		if (unlikely(tdes3 & TDES3_IP_HDR_ERROR))
+			x->tx_ip_header_error++;
+
+		if (unlikely(tdes3 & TDES3_PAYLOAD_ERROR))
+			x->tx_payload_error++;
+	}
+
+	if (unlikely(tdes3 & TDES3_DEFERRED))
+		x->tx_deferred++;
+
+	return ret;
+}
+
+static int dwmac4_wrback_get_rx_status(struct stmmac_extra_stats *x,
+				       struct dma_desc *p)
+{
+	unsigned int rdes1 = le32_to_cpu(p->des1);
+	unsigned int rdes2 = le32_to_cpu(p->des2);
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+	int message_type;
+	int ret = good_frame;
+
+	if (unlikely(rdes3 & RDES3_OWN))
+		return dma_own;
+
+	if (unlikely(rdes3 & RDES3_CONTEXT_DESCRIPTOR))
+		return discard_frame;
+	if (likely(!(rdes3 & RDES3_LAST_DESCRIPTOR)))
+		return rx_not_ls;
+
+	if (unlikely(rdes3 & RDES3_ERROR_SUMMARY)) {
+		if (unlikely(rdes3 & RDES3_GIANT_PACKET))
+			x->rx_length++;
+		if (unlikely(rdes3 & RDES3_OVERFLOW_ERROR))
+			x->rx_gmac_overflow++;
+
+		if (unlikely(rdes3 & RDES3_RECEIVE_WATCHDOG))
+			x->rx_watchdog++;
+
+		if (unlikely(rdes3 & RDES3_RECEIVE_ERROR))
+			x->rx_mii++;
+
+		if (unlikely(rdes3 & RDES3_CRC_ERROR))
+			x->rx_crc_errors++;
+
+		if (unlikely(rdes3 & RDES3_DRIBBLE_ERROR))
+			x->dribbling_bit++;
+
+		ret = discard_frame;
+	}
+
+	message_type = (rdes1 & ERDES4_MSG_TYPE_MASK) >> 8;
+
+	if (rdes1 & RDES1_IP_HDR_ERROR) {
+		x->ip_hdr_err++;
+		ret |= csum_none;
+	}
+	if (rdes1 & RDES1_IP_CSUM_BYPASSED)
+		x->ip_csum_bypassed++;
+	if (rdes1 & RDES1_IPV4_HEADER)
+		x->ipv4_pkt_rcvd++;
+	if (rdes1 & RDES1_IPV6_HEADER)
+		x->ipv6_pkt_rcvd++;
+	if (rdes1 & RDES1_IP_PAYLOAD_ERROR) {
+		x->ip_payload_err++;
+		ret |= csum_none;
+	}
+
+	if (message_type == RDES_EXT_NO_PTP)
+		x->no_ptp_rx_msg_type_ext++;
+	else if (message_type == RDES_EXT_SYNC)
+		x->ptp_rx_msg_type_sync++;
+	else if (message_type == RDES_EXT_FOLLOW_UP)
+		x->ptp_rx_msg_type_follow_up++;
+	else if (message_type == RDES_EXT_DELAY_REQ)
+		x->ptp_rx_msg_type_delay_req++;
+	else if (message_type == RDES_EXT_DELAY_RESP)
+		x->ptp_rx_msg_type_delay_resp++;
+	else if (message_type == RDES_EXT_PDELAY_REQ)
+		x->ptp_rx_msg_type_pdelay_req++;
+	else if (message_type == RDES_EXT_PDELAY_RESP)
+		x->ptp_rx_msg_type_pdelay_resp++;
+	else if (message_type == RDES_EXT_PDELAY_FOLLOW_UP)
+		x->ptp_rx_msg_type_pdelay_follow_up++;
+	else if (message_type == RDES_PTP_ANNOUNCE)
+		x->ptp_rx_msg_type_announce++;
+	else if (message_type == RDES_PTP_MANAGEMENT)
+		x->ptp_rx_msg_type_management++;
+	else if (message_type == RDES_PTP_PKT_RESERVED_TYPE)
+		x->ptp_rx_msg_pkt_reserved_type++;
+
+	if (rdes1 & RDES1_PTP_PACKET_TYPE)
+		x->ptp_frame_type++;
+	if (rdes1 & RDES1_PTP_VER)
+		x->ptp_ver++;
+	if (rdes1 & RDES1_TIMESTAMP_DROPPED)
+		x->timestamp_dropped++;
+
+	if (unlikely(rdes2 & RDES2_SA_FILTER_FAIL)) {
+		x->sa_rx_filter_fail++;
+		ret = discard_frame;
+	}
+	if (unlikely(rdes2 & RDES2_DA_FILTER_FAIL)) {
+		x->da_rx_filter_fail++;
+		ret = discard_frame;
+	}
+
+	if (rdes2 & RDES2_L3_FILTER_MATCH)
+		x->l3_filter_match++;
+	if (rdes2 & RDES2_L4_FILTER_MATCH)
+		x->l4_filter_match++;
+	if ((rdes2 & RDES2_L3_L4_FILT_NB_MATCH_MASK)
+	    >> RDES2_L3_L4_FILT_NB_MATCH_SHIFT)
+		x->l3_l4_filter_no_match++;
+
+	return ret;
+}
+
+static int dwmac4_rd_get_tx_len(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des2) & TDES2_BUFFER1_SIZE_MASK);
+}
+
+static int dwmac4_get_tx_owner(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des3) & TDES3_OWN) >> TDES3_OWN_SHIFT;
+}
+
+static void dwmac4_set_tx_owner(struct dma_desc *p)
+{
+	p->des3 |= cpu_to_le32(TDES3_OWN);
+}
+
+static void dwmac4_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
+{
+	u32 flags = (RDES3_OWN | RDES3_BUFFER1_VALID_ADDR);
+
+	if (!disable_rx_ic)
+		flags |= RDES3_INT_ON_COMPLETION_EN;
+
+	p->des3 |= cpu_to_le32(flags);
+}
+
+static int dwmac4_get_tx_ls(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des3) & TDES3_LAST_DESCRIPTOR)
+		>> TDES3_LAST_DESCRIPTOR_SHIFT;
+}
+
+static u16 dwmac4_wrback_get_rx_vlan_tci(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des0) & RDES0_VLAN_TAG_MASK);
+}
+
+static bool dwmac4_wrback_get_rx_vlan_valid(struct dma_desc *p)
+{
+	return ((le32_to_cpu(p->des3) & RDES3_LAST_DESCRIPTOR) &&
+		(le32_to_cpu(p->des3) & RDES3_RDES0_VALID));
+}
+
+static int dwmac4_wrback_get_rx_frame_len(struct dma_desc *p, int rx_coe)
+{
+	return (le32_to_cpu(p->des3) & RDES3_PACKET_SIZE_MASK);
+}
+
+static void dwmac4_rd_enable_tx_timestamp(struct dma_desc *p)
+{
+	p->des2 |= cpu_to_le32(TDES2_TIMESTAMP_ENABLE);
+}
+
+static int dwmac4_wrback_get_tx_timestamp_status(struct dma_desc *p)
+{
+	/* Context type from W/B descriptor must be zero */
+	if (le32_to_cpu(p->des3) & TDES3_CONTEXT_TYPE)
+		return 0;
+
+	/* Tx Timestamp Status is 1 so des0 and des1'll have valid values */
+	if (le32_to_cpu(p->des3) & TDES3_TIMESTAMP_STATUS)
+		return 1;
+
+	return 0;
+}
+
+static inline void dwmac4_get_timestamp(void *desc, u32 ats, u64 *ts)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+	u64 ns;
+
+	ns = le32_to_cpu(p->des0);
+	/* convert high/sec time stamp value to nanosecond */
+	ns += le32_to_cpu(p->des1) * 1000000000ULL;
+
+	*ts = ns;
+}
+
+static int dwmac4_rx_check_timestamp(void *desc)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+	unsigned int rdes0 = le32_to_cpu(p->des0);
+	unsigned int rdes1 = le32_to_cpu(p->des1);
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+	u32 own, ctxt;
+	int ret = 1;
+
+	own = rdes3 & RDES3_OWN;
+	ctxt = ((rdes3 & RDES3_CONTEXT_DESCRIPTOR)
+		>> RDES3_CONTEXT_DESCRIPTOR_SHIFT);
+
+	if (likely(!own && ctxt)) {
+		if ((rdes0 == 0xffffffff) && (rdes1 == 0xffffffff))
+			/* Corrupted value */
+			ret = -EINVAL;
+		else
+			/* A valid Timestamp is ready to be read */
+			ret = 0;
+	}
+
+	/* Timestamp not ready */
+	return ret;
+}
+
+static int dwmac4_wrback_get_rx_timestamp_status(void *desc, void *next_desc,
+						 u32 ats)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+	int ret = -EINVAL;
+
+	/* Get the status from normal w/b descriptor */
+	if (likely(le32_to_cpu(p->des3) & RDES3_RDES1_VALID)) {
+		if (likely(le32_to_cpu(p->des1) & RDES1_TIMESTAMP_AVAILABLE)) {
+			int i = 0;
+
+			/* Check if timestamp is OK from context descriptor */
+			do {
+				ret = dwmac4_rx_check_timestamp(next_desc);
+				if (ret < 0)
+					goto exit;
+				i++;
+
+			} while ((ret == 1) && (i < 10));
+
+			if (i == 10)
+				ret = -EBUSY;
+		}
+	}
+exit:
+	if (likely(ret == 0))
+		return 1;
+
+	return 0;
+}
+
+static void dwmac4_rd_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
+				   int mode, int end, int bfsize)
+{
+	dwmac4_set_rx_owner(p, disable_rx_ic);
+}
+
+static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+}
+
+static void dwmac4_rd_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
+				      bool csum_flag, int mode, bool tx_own,
+				      bool ls, unsigned int tot_pkt_len)
+{
+	unsigned int tdes3 = le32_to_cpu(p->des3);
+
+	p->des2 |= cpu_to_le32(len & TDES2_BUFFER1_SIZE_MASK);
+
+	tdes3 |= tot_pkt_len & TDES3_PACKET_SIZE_MASK;
+	if (is_fs)
+		tdes3 |= TDES3_FIRST_DESCRIPTOR;
+	else
+		tdes3 &= ~TDES3_FIRST_DESCRIPTOR;
+
+	if (likely(csum_flag))
+		tdes3 |= (TX_CIC_FULL << TDES3_CHECKSUM_INSERTION_SHIFT);
+	else
+		tdes3 &= ~(TX_CIC_FULL << TDES3_CHECKSUM_INSERTION_SHIFT);
+
+	if (ls)
+		tdes3 |= TDES3_LAST_DESCRIPTOR;
+	else
+		tdes3 &= ~TDES3_LAST_DESCRIPTOR;
+
+	/* Finally set the OWN bit. Later the DMA will start! */
+	if (tx_own)
+		tdes3 |= TDES3_OWN;
+
+	if (is_fs && tx_own)
+		/* When the own bit, for the first frame, has to be set, all
+		 * descriptors for the same frame has to be set before, to
+		 * avoid race condition.
+		 */
+		dma_wmb();
+
+	p->des3 = cpu_to_le32(tdes3);
+}
+
+static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
+					  int len1, int len2, bool tx_own,
+					  bool ls, unsigned int tcphdrlen,
+					  unsigned int tcppayloadlen)
+{
+	unsigned int tdes3 = le32_to_cpu(p->des3);
+
+	if (len1)
+		p->des2 |= cpu_to_le32((len1 & TDES2_BUFFER1_SIZE_MASK));
+
+	if (len2)
+		p->des2 |= cpu_to_le32((len2 << TDES2_BUFFER2_SIZE_MASK_SHIFT)
+			    & TDES2_BUFFER2_SIZE_MASK);
+
+	if (is_fs) {
+		tdes3 |= TDES3_FIRST_DESCRIPTOR |
+			 TDES3_TCP_SEGMENTATION_ENABLE |
+			 ((tcphdrlen << TDES3_HDR_LEN_SHIFT) &
+			  TDES3_SLOT_NUMBER_MASK) |
+			 ((tcppayloadlen & TDES3_TCP_PKT_PAYLOAD_MASK));
+	} else {
+		tdes3 &= ~TDES3_FIRST_DESCRIPTOR;
+	}
+
+	if (ls)
+		tdes3 |= TDES3_LAST_DESCRIPTOR;
+	else
+		tdes3 &= ~TDES3_LAST_DESCRIPTOR;
+
+	/* Finally set the OWN bit. Later the DMA will start! */
+	if (tx_own)
+		tdes3 |= TDES3_OWN;
+
+	if (is_fs && tx_own)
+		/* When the own bit, for the first frame, has to be set, all
+		 * descriptors for the same frame has to be set before, to
+		 * avoid race condition.
+		 */
+		dma_wmb();
+
+	p->des3 = cpu_to_le32(tdes3);
+}
+
+static void dwmac4_release_tx_desc(struct dma_desc *p, int mode)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+}
+
+static void dwmac4_rd_set_tx_ic(struct dma_desc *p)
+{
+	p->des2 |= cpu_to_le32(TDES2_INTERRUPT_ON_COMPLETION);
+}
+
+static void dwmac4_display_ring(void *head, unsigned int size, bool rx,
+				dma_addr_t dma_rx_phy, unsigned int desc_size)
+{
+	dma_addr_t dma_addr;
+	int i;
+
+	pr_info("%s descriptor ring:\n", rx ? "RX" : "TX");
+
+	if (desc_size == sizeof(struct dma_desc)) {
+		struct dma_desc *p = (struct dma_desc *)head;
+
+		for (i = 0; i < size; i++) {
+			dma_addr = dma_rx_phy + i * sizeof(*p);
+			pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
+				i, &dma_addr,
+				le32_to_cpu(p->des0), le32_to_cpu(p->des1),
+				le32_to_cpu(p->des2), le32_to_cpu(p->des3));
+			p++;
+		}
+	} else if (desc_size == sizeof(struct dma_extended_desc)) {
+		struct dma_extended_desc *extp = (struct dma_extended_desc *)head;
+
+		for (i = 0; i < size; i++) {
+			dma_addr = dma_rx_phy + i * sizeof(*extp);
+			pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+				i, &dma_addr,
+				le32_to_cpu(extp->basic.des0), le32_to_cpu(extp->basic.des1),
+				le32_to_cpu(extp->basic.des2), le32_to_cpu(extp->basic.des3),
+				le32_to_cpu(extp->des4), le32_to_cpu(extp->des5),
+				le32_to_cpu(extp->des6), le32_to_cpu(extp->des7));
+			extp++;
+		}
+	} else if (desc_size == sizeof(struct dma_edesc)) {
+		struct dma_edesc *ep = (struct dma_edesc *)head;
+
+		for (i = 0; i < size; i++) {
+			dma_addr = dma_rx_phy + i * sizeof(*ep);
+			pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+				i, &dma_addr,
+				le32_to_cpu(ep->des4), le32_to_cpu(ep->des5),
+				le32_to_cpu(ep->des6), le32_to_cpu(ep->des7),
+				le32_to_cpu(ep->basic.des0), le32_to_cpu(ep->basic.des1),
+				le32_to_cpu(ep->basic.des2), le32_to_cpu(ep->basic.des3));
+			ep++;
+		}
+	} else {
+		pr_err("unsupported descriptor!");
+	}
+}
+
+static void dwmac4_set_mss_ctxt(struct dma_desc *p, unsigned int mss)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = cpu_to_le32(mss);
+	p->des3 = cpu_to_le32(TDES3_CONTEXT_TYPE | TDES3_CTXT_TCMSSV);
+}
+
+static void dwmac4_set_addr(struct dma_desc *p, dma_addr_t addr)
+{
+	p->des0 = cpu_to_le32(lower_32_bits(addr));
+	p->des1 = cpu_to_le32(upper_32_bits(addr));
+}
+
+static void dwmac4_clear(struct dma_desc *p)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+}
+
+static void dwmac4_set_sarc(struct dma_desc *p, u32 sarc_type)
+{
+	sarc_type <<= TDES3_SA_INSERT_CTRL_SHIFT;
+
+	p->des3 |= cpu_to_le32(sarc_type & TDES3_SA_INSERT_CTRL_MASK);
+}
+
+static int set_16kib_bfsize(int mtu)
+{
+	int ret = 0;
+
+	if (unlikely(mtu >= BUF_SIZE_8KiB))
+		ret = BUF_SIZE_16KiB;
+	return ret;
+}
+
+static void dwmac4_set_vlan_tag(struct dma_desc *p, u16 tag, u16 inner_tag,
+				u32 inner_type)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+
+	/* Inner VLAN */
+	if (inner_type) {
+		u32 des = inner_tag << TDES2_IVT_SHIFT;
+
+		des &= TDES2_IVT_MASK;
+		p->des2 = cpu_to_le32(des);
+
+		des = inner_type << TDES3_IVTIR_SHIFT;
+		des &= TDES3_IVTIR_MASK;
+		p->des3 = cpu_to_le32(des | TDES3_IVLTV);
+	}
+
+	/* Outer VLAN */
+	p->des3 |= cpu_to_le32(tag & TDES3_VLAN_TAG);
+	p->des3 |= cpu_to_le32(TDES3_VLTV);
+
+	p->des3 |= cpu_to_le32(TDES3_CONTEXT_TYPE);
+}
+
+static void dwmac4_set_vlan(struct dma_desc *p, u32 type)
+{
+	type <<= TDES2_VLAN_TAG_SHIFT;
+	p->des2 |= cpu_to_le32(type & TDES2_VLAN_TAG_MASK);
+}
+
+static void dwmac4_get_rx_header_len(struct dma_desc *p, unsigned int *len)
+{
+	*len = le32_to_cpu(p->des2) & RDES2_HL;
+}
+
+static void dwmac4_set_sec_addr(struct dma_desc *p, dma_addr_t addr, bool buf2_valid)
+{
+	p->des2 = cpu_to_le32(lower_32_bits(addr));
+	p->des3 = cpu_to_le32(upper_32_bits(addr));
+
+	if (buf2_valid)
+		p->des3 |= cpu_to_le32(RDES3_BUFFER2_VALID_ADDR);
+	else
+		p->des3 &= cpu_to_le32(~RDES3_BUFFER2_VALID_ADDR);
+}
+
+static void dwmac4_set_tbs(struct dma_edesc *p, u32 sec, u32 nsec)
+{
+	p->des4 = cpu_to_le32((sec & TDES4_LT) | TDES4_LTV);
+	p->des5 = cpu_to_le32(nsec & TDES5_LT);
+	p->des6 = 0;
+	p->des7 = 0;
+}
+
+const struct stmmac_desc_ops ec_dwmac4_desc_ops = {
+	.tx_status = dwmac4_wrback_get_tx_status,
+	.rx_status = dwmac4_wrback_get_rx_status,
+	.get_tx_len = dwmac4_rd_get_tx_len,
+	.get_tx_owner = dwmac4_get_tx_owner,
+	.set_tx_owner = dwmac4_set_tx_owner,
+	.set_rx_owner = dwmac4_set_rx_owner,
+	.get_tx_ls = dwmac4_get_tx_ls,
+	.get_rx_vlan_tci = dwmac4_wrback_get_rx_vlan_tci,
+	.get_rx_vlan_valid = dwmac4_wrback_get_rx_vlan_valid,
+	.get_rx_frame_len = dwmac4_wrback_get_rx_frame_len,
+	.enable_tx_timestamp = dwmac4_rd_enable_tx_timestamp,
+	.get_tx_timestamp_status = dwmac4_wrback_get_tx_timestamp_status,
+	.get_rx_timestamp_status = dwmac4_wrback_get_rx_timestamp_status,
+	.get_timestamp = dwmac4_get_timestamp,
+	.set_tx_ic = dwmac4_rd_set_tx_ic,
+	.prepare_tx_desc = dwmac4_rd_prepare_tx_desc,
+	.prepare_tso_tx_desc = dwmac4_rd_prepare_tso_tx_desc,
+	.release_tx_desc = dwmac4_release_tx_desc,
+	.init_rx_desc = dwmac4_rd_init_rx_desc,
+	.init_tx_desc = dwmac4_rd_init_tx_desc,
+	.display_ring = dwmac4_display_ring,
+	.set_mss = dwmac4_set_mss_ctxt,
+	.set_addr = dwmac4_set_addr,
+	.clear = dwmac4_clear,
+	.set_sarc = dwmac4_set_sarc,
+	.set_vlan_tag = dwmac4_set_vlan_tag,
+	.set_vlan = dwmac4_set_vlan,
+	.get_rx_header_len = dwmac4_get_rx_header_len,
+	.set_sec_addr = dwmac4_set_sec_addr,
+	.set_tbs = dwmac4_set_tbs,
+};
+
+const struct stmmac_mode_ops ec_dwmac4_ring_mode_ops = {
+	.set_16kib_bfsize = set_16kib_bfsize,
+};
diff --git a/drivers/net/ethercat/device/K3/dwmac4_descs-ethercat.h b/drivers/net/ethercat/device/K3/dwmac4_descs-ethercat.h
new file mode 100644
index 000000000000..de2a3afd0845
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac4_descs-ethercat.h
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Header File to describe the DMA descriptors and related definitions specific
+ * for DesignWare databook 4.xx.
+ *
+ * Copyright (C) 2015  STMicroelectronics Ltd
+ *
+ * Author: Alexandre Torgue <alexandre.torgue@st.com>
+ */
+
+#ifndef __DWMAC4_DESCS_H__
+#define __DWMAC4_DESCS_H__
+
+#include <linux/bitops.h>
+
+/* Normal transmit descriptor defines (without split feature) */
+
+/* TDES2 (read format) */
+#define TDES2_BUFFER1_SIZE_MASK		GENMASK(13, 0)
+#define TDES2_VLAN_TAG_MASK		GENMASK(15, 14)
+#define TDES2_VLAN_TAG_SHIFT		14
+#define TDES2_BUFFER2_SIZE_MASK		GENMASK(29, 16)
+#define TDES2_BUFFER2_SIZE_MASK_SHIFT	16
+#define TDES3_IVTIR_MASK		GENMASK(19, 18)
+#define TDES3_IVTIR_SHIFT		18
+#define TDES3_IVLTV			BIT(17)
+#define TDES2_TIMESTAMP_ENABLE		BIT(30)
+#define TDES2_IVT_MASK			GENMASK(31, 16)
+#define TDES2_IVT_SHIFT			16
+#define TDES2_INTERRUPT_ON_COMPLETION	BIT(31)
+
+/* TDES3 (read format) */
+#define TDES3_PACKET_SIZE_MASK		GENMASK(14, 0)
+#define TDES3_VLAN_TAG			GENMASK(15, 0)
+#define TDES3_VLTV			BIT(16)
+#define TDES3_CHECKSUM_INSERTION_MASK	GENMASK(17, 16)
+#define TDES3_CHECKSUM_INSERTION_SHIFT	16
+#define TDES3_TCP_PKT_PAYLOAD_MASK	GENMASK(17, 0)
+#define TDES3_TCP_SEGMENTATION_ENABLE	BIT(18)
+#define TDES3_HDR_LEN_SHIFT		19
+#define TDES3_SLOT_NUMBER_MASK		GENMASK(22, 19)
+#define TDES3_SA_INSERT_CTRL_MASK	GENMASK(25, 23)
+#define TDES3_SA_INSERT_CTRL_SHIFT	23
+#define TDES3_CRC_PAD_CTRL_MASK		GENMASK(27, 26)
+
+/* TDES3 (write back format) */
+#define TDES3_IP_HDR_ERROR		BIT(0)
+#define TDES3_DEFERRED			BIT(1)
+#define TDES3_UNDERFLOW_ERROR		BIT(2)
+#define TDES3_EXCESSIVE_DEFERRAL	BIT(3)
+#define TDES3_COLLISION_COUNT_MASK	GENMASK(7, 4)
+#define TDES3_COLLISION_COUNT_SHIFT	4
+#define TDES3_EXCESSIVE_COLLISION	BIT(8)
+#define TDES3_LATE_COLLISION		BIT(9)
+#define TDES3_NO_CARRIER		BIT(10)
+#define TDES3_LOSS_CARRIER		BIT(11)
+#define TDES3_PAYLOAD_ERROR		BIT(12)
+#define TDES3_PACKET_FLUSHED		BIT(13)
+#define TDES3_JABBER_TIMEOUT		BIT(14)
+#define TDES3_ERROR_SUMMARY		BIT(15)
+#define TDES3_TIMESTAMP_STATUS		BIT(17)
+#define TDES3_TIMESTAMP_STATUS_SHIFT	17
+
+/* TDES3 context */
+#define TDES3_CTXT_TCMSSV		BIT(26)
+
+/* TDES3 Common */
+#define	TDES3_RS1V			BIT(26)
+#define	TDES3_RS1V_SHIFT		26
+#define TDES3_LAST_DESCRIPTOR		BIT(28)
+#define TDES3_LAST_DESCRIPTOR_SHIFT	28
+#define TDES3_FIRST_DESCRIPTOR		BIT(29)
+#define TDES3_CONTEXT_TYPE		BIT(30)
+#define	TDES3_CONTEXT_TYPE_SHIFT	30
+
+/* TDES4 */
+#define TDES4_LTV			BIT(31)
+#define TDES4_LT			GENMASK(7, 0)
+
+/* TDES5 */
+#define TDES5_LT			GENMASK(31, 8)
+
+/* TDS3 use for both format (read and write back) */
+#define TDES3_OWN			BIT(31)
+#define TDES3_OWN_SHIFT			31
+
+/* Normal receive descriptor defines (without split feature) */
+
+/* RDES0 (write back format) */
+#define RDES0_VLAN_TAG_MASK		GENMASK(15, 0)
+
+/* RDES1 (write back format) */
+#define RDES1_IP_PAYLOAD_TYPE_MASK	GENMASK(2, 0)
+#define RDES1_IP_HDR_ERROR		BIT(3)
+#define RDES1_IPV4_HEADER		BIT(4)
+#define RDES1_IPV6_HEADER		BIT(5)
+#define RDES1_IP_CSUM_BYPASSED		BIT(6)
+#define RDES1_IP_PAYLOAD_ERROR		BIT(7)
+#define RDES1_PTP_MSG_TYPE_MASK		GENMASK(11, 8)
+#define RDES1_PTP_PACKET_TYPE		BIT(12)
+#define RDES1_PTP_VER			BIT(13)
+#define RDES1_TIMESTAMP_AVAILABLE	BIT(14)
+#define RDES1_TIMESTAMP_AVAILABLE_SHIFT	14
+#define RDES1_TIMESTAMP_DROPPED		BIT(15)
+#define RDES1_IP_TYPE1_CSUM_MASK	GENMASK(31, 16)
+
+/* RDES2 (write back format) */
+#define RDES2_L3_L4_HEADER_SIZE_MASK	GENMASK(9, 0)
+#define RDES2_VLAN_FILTER_STATUS	BIT(15)
+#define RDES2_SA_FILTER_FAIL		BIT(16)
+#define RDES2_DA_FILTER_FAIL		BIT(17)
+#define RDES2_HASH_FILTER_STATUS	BIT(18)
+#define RDES2_MAC_ADDR_MATCH_MASK	GENMASK(26, 19)
+#define RDES2_HASH_VALUE_MATCH_MASK	GENMASK(26, 19)
+#define RDES2_L3_FILTER_MATCH		BIT(27)
+#define RDES2_L4_FILTER_MATCH		BIT(28)
+#define RDES2_L3_L4_FILT_NB_MATCH_MASK	GENMASK(27, 26)
+#define RDES2_L3_L4_FILT_NB_MATCH_SHIFT	26
+#define RDES2_HL			GENMASK(9, 0)
+
+/* RDES3 (write back format) */
+#define RDES3_PACKET_SIZE_MASK		GENMASK(14, 0)
+#define RDES3_ERROR_SUMMARY		BIT(15)
+#define RDES3_PACKET_LEN_TYPE_MASK	GENMASK(18, 16)
+#define RDES3_DRIBBLE_ERROR		BIT(19)
+#define RDES3_RECEIVE_ERROR		BIT(20)
+#define RDES3_OVERFLOW_ERROR		BIT(21)
+#define RDES3_RECEIVE_WATCHDOG		BIT(22)
+#define RDES3_GIANT_PACKET		BIT(23)
+#define RDES3_CRC_ERROR			BIT(24)
+#define RDES3_RDES0_VALID		BIT(25)
+#define RDES3_RDES1_VALID		BIT(26)
+#define RDES3_RDES2_VALID		BIT(27)
+#define RDES3_LAST_DESCRIPTOR		BIT(28)
+#define RDES3_FIRST_DESCRIPTOR		BIT(29)
+#define RDES3_CONTEXT_DESCRIPTOR	BIT(30)
+#define RDES3_CONTEXT_DESCRIPTOR_SHIFT	30
+
+/* RDES3 (read format) */
+#define RDES3_BUFFER1_VALID_ADDR	BIT(24)
+#define RDES3_BUFFER2_VALID_ADDR	BIT(25)
+#define RDES3_INT_ON_COMPLETION_EN	BIT(30)
+
+/* TDS3 use for both format (read and write back) */
+#define RDES3_OWN			BIT(31)
+
+extern const struct stmmac_mode_ops ec_dwmac4_ring_mode_ops;
+extern const struct stmmac_desc_ops ec_dwmac4_desc_ops;
+
+#endif /* __DWMAC4_DESCS_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac4_dma-ethercat.c b/drivers/net/ethercat/device/K3/dwmac4_dma-ethercat.c
new file mode 100644
index 000000000000..de938eacafc5
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac4_dma-ethercat.c
@@ -0,0 +1,631 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This is the driver for the GMAC on-chip Ethernet controller for ST SoCs.
+ * DWC Ether MAC version 4.xx  has been used for  developing this code.
+ *
+ * This contains the functions to handle the dma.
+ *
+ * Copyright (C) 2015  STMicroelectronics Ltd
+ *
+ * Author: Alexandre Torgue <alexandre.torgue@st.com>
+ */
+
+#include <linux/io.h>
+#include "dwmac4-ethercat.h"
+#include "dwmac4_dma-ethercat.h"
+#include "stmmac-ethercat.h"
+
+static void dwmac4_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi)
+{
+	u32 value = readl(ioaddr + DMA_SYS_BUS_MODE);
+	int i;
+
+	pr_info("dwmac4: Master AXI performs %s burst length\n",
+		(value & DMA_SYS_BUS_FB) ? "fixed" : "any");
+
+	if (axi->axi_lpi_en)
+		value |= DMA_AXI_EN_LPI;
+	if (axi->axi_xit_frm)
+		value |= DMA_AXI_LPI_XIT_FRM;
+
+	value &= ~DMA_AXI_WR_OSR_LMT;
+	value |= (axi->axi_wr_osr_lmt & DMA_AXI_OSR_MAX) <<
+		 DMA_AXI_WR_OSR_LMT_SHIFT;
+
+	value &= ~DMA_AXI_RD_OSR_LMT;
+	value |= (axi->axi_rd_osr_lmt & DMA_AXI_OSR_MAX) <<
+		 DMA_AXI_RD_OSR_LMT_SHIFT;
+
+	/* Depending on the UNDEF bit the Master AXI will perform any burst
+	 * length according to the BLEN programmed (by default all BLEN are
+	 * set).
+	 */
+	for (i = 0; i < AXI_BLEN; i++) {
+		switch (axi->axi_blen[i]) {
+		case 256:
+			value |= DMA_AXI_BLEN256;
+			break;
+		case 128:
+			value |= DMA_AXI_BLEN128;
+			break;
+		case 64:
+			value |= DMA_AXI_BLEN64;
+			break;
+		case 32:
+			value |= DMA_AXI_BLEN32;
+			break;
+		case 16:
+			value |= DMA_AXI_BLEN16;
+			break;
+		case 8:
+			value |= DMA_AXI_BLEN8;
+			break;
+		case 4:
+			value |= DMA_AXI_BLEN4;
+			break;
+		}
+	}
+
+	writel(value, ioaddr + DMA_SYS_BUS_MODE);
+}
+
+static void dwmac4_dma_init_rx_chan(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
+				    struct stmmac_dma_cfg *dma_cfg,
+				    dma_addr_t dma_rx_phy, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value;
+	u32 rxpbl = dma_cfg->rxpbl ?: dma_cfg->pbl;
+
+	value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+	value = value | (rxpbl << DMA_BUS_MODE_RPBL_SHIFT);
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+
+	if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && likely(dma_cfg->eame))
+		writel(upper_32_bits(dma_rx_phy),
+		       ioaddr + DMA_CHAN_RX_BASE_ADDR_HI(dwmac4_addrs, chan));
+
+	writel(lower_32_bits(dma_rx_phy),
+	       ioaddr + DMA_CHAN_RX_BASE_ADDR(dwmac4_addrs, chan));
+}
+
+static void dwmac4_dma_init_tx_chan(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
+				    struct stmmac_dma_cfg *dma_cfg,
+				    dma_addr_t dma_tx_phy, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value;
+	u32 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl;
+
+	value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+	value = value | (txpbl << DMA_BUS_MODE_PBL_SHIFT);
+
+	/* Enable OSP to get best performance */
+	value |= DMA_CONTROL_OSP;
+
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+
+	if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && likely(dma_cfg->eame))
+		writel(upper_32_bits(dma_tx_phy),
+		       ioaddr + DMA_CHAN_TX_BASE_ADDR_HI(dwmac4_addrs, chan));
+
+	writel(lower_32_bits(dma_tx_phy),
+	       ioaddr + DMA_CHAN_TX_BASE_ADDR(dwmac4_addrs, chan));
+}
+
+static void dwmac4_dma_init_channel(struct stmmac_priv *priv,
+				    void __iomem *ioaddr,
+				    struct stmmac_dma_cfg *dma_cfg, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value;
+
+	/* common channel control register config */
+	value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
+	if (dma_cfg->pblx8)
+		value = value | DMA_BUS_MODE_PBL;
+	writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
+
+	/* Mask interrupts by writing to CSR7 */
+	writel(DMA_CHAN_INTR_DEFAULT_MASK,
+	       ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+}
+
+static void dwmac410_dma_init_channel(struct stmmac_priv *priv,
+				      void __iomem *ioaddr,
+				      struct stmmac_dma_cfg *dma_cfg, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value;
+
+	/* common channel control register config */
+	value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
+	if (dma_cfg->pblx8)
+		value = value | DMA_BUS_MODE_PBL;
+
+	writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
+
+	/* Mask interrupts by writing to CSR7 */
+	writel(DMA_CHAN_INTR_DEFAULT_MASK_4_10,
+	       ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+}
+
+static void dwmac4_dma_init(void __iomem *ioaddr,
+			    struct stmmac_dma_cfg *dma_cfg)
+{
+	u32 value = readl(ioaddr + DMA_SYS_BUS_MODE);
+
+	/* Set the Fixed burst mode */
+	if (dma_cfg->fixed_burst)
+		value |= DMA_SYS_BUS_FB;
+
+	/* Mixed Burst has no effect when fb is set */
+	if (dma_cfg->mixed_burst)
+		value |= DMA_SYS_BUS_MB;
+
+	if (dma_cfg->aal)
+		value |= DMA_SYS_BUS_AAL;
+
+	if (dma_cfg->eame)
+		value |= DMA_SYS_BUS_EAME;
+
+	writel(value, ioaddr + DMA_SYS_BUS_MODE);
+
+	value = readl(ioaddr + DMA_BUS_MODE);
+
+	if (dma_cfg->multi_msi_en) {
+		value &= ~DMA_BUS_MODE_INTM_MASK;
+		value |= (DMA_BUS_MODE_INTM_MODE1 << DMA_BUS_MODE_INTM_SHIFT);
+	}
+
+	if (dma_cfg->dche)
+		value |= DMA_BUS_MODE_DCHE;
+
+	writel(value, ioaddr + DMA_BUS_MODE);
+
+}
+
+static void _dwmac4_dump_dma_regs(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 channel,
+				  u32 *reg_space)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	const struct dwmac4_addrs *default_addrs = NULL;
+
+	/* Purposely save the registers in the "normal" layout, regardless of
+	 * platform modifications, to keep reg_space size constant
+	 */
+	reg_space[DMA_CHAN_CONTROL(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_CONTROL(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_CONTROL(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_BASE_ADDR_HI(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_BASE_ADDR_HI(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_BASE_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_BASE_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_BASE_ADDR_HI(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_BASE_ADDR_HI(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_BASE_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_BASE_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_END_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_END_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_END_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_END_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_TX_RING_LEN(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_TX_RING_LEN(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_RING_LEN(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_RING_LEN(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_INTR_ENA(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_RX_WATCHDOG(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_RX_WATCHDOG(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_SLOT_CTRL_STATUS(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_SLOT_CTRL_STATUS(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_TX_DESC(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_TX_DESC(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_RX_DESC(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_RX_DESC(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_TX_BUF_ADDR_HI(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_TX_BUF_ADDR_HI(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_TX_BUF_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_TX_BUF_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_RX_BUF_ADDR_HI(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_RX_BUF_ADDR_HI(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_CUR_RX_BUF_ADDR(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_CUR_RX_BUF_ADDR(dwmac4_addrs, channel));
+	reg_space[DMA_CHAN_STATUS(default_addrs, channel) / 4] =
+		readl(ioaddr + DMA_CHAN_STATUS(dwmac4_addrs, channel));
+}
+
+static void dwmac4_dump_dma_regs(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 *reg_space)
+{
+	int i;
+
+	for (i = 0; i < DMA_CHANNEL_NB_MAX; i++)
+		_dwmac4_dump_dma_regs(priv, ioaddr, i, reg_space);
+}
+
+static void dwmac4_rx_watchdog(struct stmmac_priv *priv, void __iomem *ioaddr,
+			       u32 riwt, u32 queue)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(riwt, ioaddr + DMA_CHAN_RX_WATCHDOG(dwmac4_addrs, queue));
+}
+
+static void dwmac4_dma_rx_chan_op_mode(struct stmmac_priv *priv,
+				       void __iomem *ioaddr, int mode,
+				       u32 channel, int fifosz, u8 qmode)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	unsigned int rqs = fifosz / 256 - 1;
+	u32 mtl_rx_op;
+
+	mtl_rx_op = readl(ioaddr + MTL_CHAN_RX_OP_MODE(dwmac4_addrs, channel));
+
+	mtl_rx_op |= MTL_OP_MODE_DIS_TCP_EF;
+
+	if (mode == SF_DMA_MODE) {
+		pr_debug("GMAC: enable RX store and forward mode\n");
+		mtl_rx_op |= MTL_OP_MODE_RSF;
+	} else {
+		pr_debug("GMAC: disable RX SF mode (threshold %d)\n", mode);
+		mtl_rx_op &= ~MTL_OP_MODE_RSF;
+		mtl_rx_op &= ~MTL_OP_MODE_RTC_MASK;
+		if (mode <= 32)
+			mtl_rx_op |= MTL_OP_MODE_RTC_32;
+		else if (mode <= 64)
+			mtl_rx_op |= MTL_OP_MODE_RTC_64;
+		else if (mode <= 96)
+			mtl_rx_op |= MTL_OP_MODE_RTC_96;
+		else
+			mtl_rx_op |= MTL_OP_MODE_RTC_128;
+	}
+
+	mtl_rx_op &= ~MTL_OP_MODE_RQS_MASK;
+	mtl_rx_op |= rqs << MTL_OP_MODE_RQS_SHIFT;
+
+	/* Enable flow control only if each channel gets 4 KiB or more FIFO and
+	 * only if channel is not an AVB channel.
+	 */
+	if ((fifosz >= 4096) && (qmode != MTL_QUEUE_AVB)) {
+		unsigned int rfd, rfa;
+
+		mtl_rx_op |= MTL_OP_MODE_EHFC;
+
+		/* Set Threshold for Activating Flow Control to min 2 frames,
+		 * i.e. 1500 * 2 = 3000 bytes.
+		 *
+		 * Set Threshold for Deactivating Flow Control to min 1 frame,
+		 * i.e. 1500 bytes.
+		 */
+		switch (fifosz) {
+		case 4096:
+			/* This violates the above formula because of FIFO size
+			 * limit therefore overflow may occur in spite of this.
+			 */
+			rfd = 0x03; /* Full-2.5K */
+			rfa = 0x01; /* Full-1.5K */
+			break;
+
+		default:
+			rfd = 0x07; /* Full-4.5K */
+			rfa = 0x04; /* Full-3K */
+			break;
+		}
+
+		mtl_rx_op &= ~MTL_OP_MODE_RFD_MASK;
+		mtl_rx_op |= rfd << MTL_OP_MODE_RFD_SHIFT;
+
+		mtl_rx_op &= ~MTL_OP_MODE_RFA_MASK;
+		mtl_rx_op |= rfa << MTL_OP_MODE_RFA_SHIFT;
+	}
+
+	writel(mtl_rx_op, ioaddr + MTL_CHAN_RX_OP_MODE(dwmac4_addrs, channel));
+}
+
+static void dwmac4_dma_tx_chan_op_mode(struct stmmac_priv *priv,
+				       void __iomem *ioaddr, int mode,
+				       u32 channel, int fifosz, u8 qmode)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 mtl_tx_op = readl(ioaddr + MTL_CHAN_TX_OP_MODE(dwmac4_addrs,
+							   channel));
+	unsigned int tqs = fifosz / 256 - 1;
+
+	if (mode == SF_DMA_MODE) {
+		pr_debug("GMAC: enable TX store and forward mode\n");
+		/* Transmit COE type 2 cannot be done in cut-through mode. */
+		mtl_tx_op |= MTL_OP_MODE_TSF;
+	} else {
+		pr_debug("GMAC: disabling TX SF (threshold %d)\n", mode);
+		mtl_tx_op &= ~MTL_OP_MODE_TSF;
+		mtl_tx_op &= ~MTL_OP_MODE_TTC_MASK;
+		/* Set the transmit threshold */
+		if (mode <= 32)
+			mtl_tx_op |= MTL_OP_MODE_TTC_32;
+		else if (mode <= 64)
+			mtl_tx_op |= MTL_OP_MODE_TTC_64;
+		else if (mode <= 96)
+			mtl_tx_op |= MTL_OP_MODE_TTC_96;
+		else if (mode <= 128)
+			mtl_tx_op |= MTL_OP_MODE_TTC_128;
+		else if (mode <= 192)
+			mtl_tx_op |= MTL_OP_MODE_TTC_192;
+		else if (mode <= 256)
+			mtl_tx_op |= MTL_OP_MODE_TTC_256;
+		else if (mode <= 384)
+			mtl_tx_op |= MTL_OP_MODE_TTC_384;
+		else
+			mtl_tx_op |= MTL_OP_MODE_TTC_512;
+	}
+	/* For an IP with DWC_EQOS_NUM_TXQ == 1, the fields TXQEN and TQS are RO
+	 * with reset values: TXQEN on, TQS == DWC_EQOS_TXFIFO_SIZE.
+	 * For an IP with DWC_EQOS_NUM_TXQ > 1, the fields TXQEN and TQS are R/W
+	 * with reset values: TXQEN off, TQS 256 bytes.
+	 *
+	 * TXQEN must be written for multi-channel operation and TQS must
+	 * reflect the available fifo size per queue (total fifo size / number
+	 * of enabled queues).
+	 */
+	mtl_tx_op &= ~MTL_OP_MODE_TXQEN_MASK;
+	if (qmode != MTL_QUEUE_AVB)
+		mtl_tx_op |= MTL_OP_MODE_TXQEN;
+	else
+		mtl_tx_op |= MTL_OP_MODE_TXQEN_AV;
+	mtl_tx_op &= ~MTL_OP_MODE_TQS_MASK;
+	mtl_tx_op |= tqs << MTL_OP_MODE_TQS_SHIFT;
+
+	writel(mtl_tx_op, ioaddr +  MTL_CHAN_TX_OP_MODE(dwmac4_addrs, channel));
+}
+
+static int dwmac4_get_hw_feature(void __iomem *ioaddr,
+				 struct dma_features *dma_cap)
+{
+	u32 hw_cap = readl(ioaddr + GMAC_HW_FEATURE0);
+
+	/*  MAC HW feature0 */
+	dma_cap->mbps_10_100 = (hw_cap & GMAC_HW_FEAT_MIISEL);
+	dma_cap->mbps_1000 = (hw_cap & GMAC_HW_FEAT_GMIISEL) >> 1;
+	dma_cap->half_duplex = (hw_cap & GMAC_HW_FEAT_HDSEL) >> 2;
+	dma_cap->vlhash = (hw_cap & GMAC_HW_FEAT_VLHASH) >> 4;
+	dma_cap->multi_addr = (hw_cap & GMAC_HW_FEAT_ADDMAC) >> 18;
+	dma_cap->pcs = (hw_cap & GMAC_HW_FEAT_PCSSEL) >> 3;
+	dma_cap->sma_mdio = (hw_cap & GMAC_HW_FEAT_SMASEL) >> 5;
+	dma_cap->pmt_remote_wake_up = (hw_cap & GMAC_HW_FEAT_RWKSEL) >> 6;
+	dma_cap->pmt_magic_frame = (hw_cap & GMAC_HW_FEAT_MGKSEL) >> 7;
+	/* MMC */
+	dma_cap->rmon = (hw_cap & GMAC_HW_FEAT_MMCSEL) >> 8;
+	/* IEEE 1588-2008 */
+	dma_cap->atime_stamp = (hw_cap & GMAC_HW_FEAT_TSSEL) >> 12;
+	/* 802.3az - Energy-Efficient Ethernet (EEE) */
+	dma_cap->eee = (hw_cap & GMAC_HW_FEAT_EEESEL) >> 13;
+	/* TX and RX csum */
+	dma_cap->tx_coe = (hw_cap & GMAC_HW_FEAT_TXCOSEL) >> 14;
+	dma_cap->rx_coe =  (hw_cap & GMAC_HW_FEAT_RXCOESEL) >> 16;
+	dma_cap->vlins = (hw_cap & GMAC_HW_FEAT_SAVLANINS) >> 27;
+	dma_cap->arpoffsel = (hw_cap & GMAC_HW_FEAT_ARPOFFSEL) >> 9;
+
+	/* MAC HW feature1 */
+	hw_cap = readl(ioaddr + GMAC_HW_FEATURE1);
+	dma_cap->l3l4fnum = (hw_cap & GMAC_HW_FEAT_L3L4FNUM) >> 27;
+	dma_cap->hash_tb_sz = (hw_cap & GMAC_HW_HASH_TB_SZ) >> 24;
+	dma_cap->av = (hw_cap & GMAC_HW_FEAT_AVSEL) >> 20;
+	dma_cap->tsoen = (hw_cap & GMAC_HW_TSOEN) >> 18;
+	dma_cap->sphen = (hw_cap & GMAC_HW_FEAT_SPHEN) >> 17;
+
+	dma_cap->addr64 = (hw_cap & GMAC_HW_ADDR64) >> 14;
+	switch (dma_cap->addr64) {
+	case 0:
+		dma_cap->addr64 = 32;
+		break;
+	case 1:
+		dma_cap->addr64 = 40;
+		break;
+	case 2:
+		dma_cap->addr64 = 48;
+		break;
+	default:
+		dma_cap->addr64 = 32;
+		break;
+	}
+
+	/* RX and TX FIFO sizes are encoded as log2(n / 128). Undo that by
+	 * shifting and store the sizes in bytes.
+	 */
+	dma_cap->tx_fifo_size = 128 << ((hw_cap & GMAC_HW_TXFIFOSIZE) >> 6);
+	dma_cap->rx_fifo_size = 128 << ((hw_cap & GMAC_HW_RXFIFOSIZE) >> 0);
+	/* MAC HW feature2 */
+	hw_cap = readl(ioaddr + GMAC_HW_FEATURE2);
+	/* TX and RX number of channels */
+	dma_cap->number_rx_channel =
+		((hw_cap & GMAC_HW_FEAT_RXCHCNT) >> 12) + 1;
+	dma_cap->number_tx_channel =
+		((hw_cap & GMAC_HW_FEAT_TXCHCNT) >> 18) + 1;
+	/* TX and RX number of queues */
+	dma_cap->number_rx_queues =
+		((hw_cap & GMAC_HW_FEAT_RXQCNT) >> 0) + 1;
+	dma_cap->number_tx_queues =
+		((hw_cap & GMAC_HW_FEAT_TXQCNT) >> 6) + 1;
+	/* PPS output */
+	dma_cap->pps_out_num = (hw_cap & GMAC_HW_FEAT_PPSOUTNUM) >> 24;
+
+	/* IEEE 1588-2002 */
+	dma_cap->time_stamp = 0;
+	/* Number of Auxiliary Snapshot Inputs */
+	dma_cap->aux_snapshot_n = (hw_cap & GMAC_HW_FEAT_AUXSNAPNUM) >> 28;
+
+	/* MAC HW feature3 */
+	hw_cap = readl(ioaddr + GMAC_HW_FEATURE3);
+
+	/* 5.10 Features */
+	dma_cap->asp = (hw_cap & GMAC_HW_FEAT_ASP) >> 28;
+	dma_cap->tbssel = (hw_cap & GMAC_HW_FEAT_TBSSEL) >> 27;
+	dma_cap->fpesel = (hw_cap & GMAC_HW_FEAT_FPESEL) >> 26;
+	dma_cap->estwid = (hw_cap & GMAC_HW_FEAT_ESTWID) >> 20;
+	dma_cap->estdep = (hw_cap & GMAC_HW_FEAT_ESTDEP) >> 17;
+	dma_cap->estsel = (hw_cap & GMAC_HW_FEAT_ESTSEL) >> 16;
+	dma_cap->frpes = (hw_cap & GMAC_HW_FEAT_FRPES) >> 13;
+	dma_cap->frpbs = (hw_cap & GMAC_HW_FEAT_FRPBS) >> 11;
+	dma_cap->frpsel = (hw_cap & GMAC_HW_FEAT_FRPSEL) >> 10;
+	dma_cap->dvlan = (hw_cap & GMAC_HW_FEAT_DVLAN) >> 5;
+
+	return 0;
+}
+
+/* Enable/disable TSO feature and set MSS */
+static void dwmac4_enable_tso(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      bool en, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value;
+
+	if (en) {
+		/* enable TSO */
+		value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+		writel(value | DMA_CONTROL_TSE,
+		       ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+	} else {
+		/* enable TSO */
+		value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+		writel(value & ~DMA_CONTROL_TSE,
+		       ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+	}
+}
+
+static void dwmac4_qmode(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 channel, u8 qmode)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 mtl_tx_op = readl(ioaddr + MTL_CHAN_TX_OP_MODE(dwmac4_addrs,
+							   channel));
+
+	mtl_tx_op &= ~MTL_OP_MODE_TXQEN_MASK;
+	if (qmode != MTL_QUEUE_AVB)
+		mtl_tx_op |= MTL_OP_MODE_TXQEN;
+	else
+		mtl_tx_op |= MTL_OP_MODE_TXQEN_AV;
+
+	writel(mtl_tx_op, ioaddr +  MTL_CHAN_TX_OP_MODE(dwmac4_addrs, channel));
+}
+
+static void dwmac4_set_bfsize(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      int bfsize, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+
+	value &= ~DMA_RBSZ_MASK;
+	value |= (bfsize << DMA_RBSZ_SHIFT) & DMA_RBSZ_MASK;
+
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+}
+
+static void dwmac4_enable_sph(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      bool en, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + GMAC_EXT_CONFIG);
+
+	value &= ~GMAC_CONFIG_HDSMS;
+	value |= GMAC_CONFIG_HDSMS_256; /* Segment max 256 bytes */
+	writel(value, ioaddr + GMAC_EXT_CONFIG);
+
+	value = readl(ioaddr + GMAC_EXT_CFG1);
+	value |= GMAC_CONFIG1_SPLM(1); /* Split mode set to L2OFST */
+	value |= GMAC_CONFIG1_SAVE_EN; /* Enable Split AV mode */
+	writel(value, ioaddr + GMAC_EXT_CFG1);
+
+	value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
+	if (en)
+		value |= DMA_CONTROL_SPH;
+	else
+		value &= ~DMA_CONTROL_SPH;
+	writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan));
+}
+
+static int dwmac4_enable_tbs(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     bool en, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+
+	if (en)
+		value |= DMA_CONTROL_EDSE;
+	else
+		value &= ~DMA_CONTROL_EDSE;
+
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+
+	value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs,
+						   chan)) & DMA_CONTROL_EDSE;
+	if (en && !value)
+		return -EIO;
+
+	writel(DMA_TBS_DEF_FTOS, ioaddr + DMA_TBS_CTRL);
+	return 0;
+}
+
+const struct stmmac_dma_ops ec_dwmac4_dma_ops = {
+	.reset = ec_dwmac4_dma_reset,
+	.init = dwmac4_dma_init,
+	.init_chan = dwmac4_dma_init_channel,
+	.init_rx_chan = dwmac4_dma_init_rx_chan,
+	.init_tx_chan = dwmac4_dma_init_tx_chan,
+	.axi = dwmac4_dma_axi,
+	.dump_regs = dwmac4_dump_dma_regs,
+	.dma_rx_mode = dwmac4_dma_rx_chan_op_mode,
+	.dma_tx_mode = dwmac4_dma_tx_chan_op_mode,
+	.enable_dma_irq = ec_dwmac4_enable_dma_irq,
+	.disable_dma_irq = ec_dwmac4_disable_dma_irq,
+	.start_tx = ec_dwmac4_dma_start_tx,
+	.stop_tx = ec_dwmac4_dma_stop_tx,
+	.start_rx = ec_dwmac4_dma_start_rx,
+	.stop_rx = ec_dwmac4_dma_stop_rx,
+	.dma_interrupt = ec_dwmac4_dma_interrupt,
+	.get_hw_feature = dwmac4_get_hw_feature,
+	.rx_watchdog = dwmac4_rx_watchdog,
+	.set_rx_ring_len = ec_dwmac4_set_rx_ring_len,
+	.set_tx_ring_len = ec_dwmac4_set_tx_ring_len,
+	.set_rx_tail_ptr = ec_dwmac4_set_rx_tail_ptr,
+	.set_tx_tail_ptr = ec_dwmac4_set_tx_tail_ptr,
+	.enable_tso = dwmac4_enable_tso,
+	.qmode = dwmac4_qmode,
+	.set_bfsize = dwmac4_set_bfsize,
+	.enable_sph = dwmac4_enable_sph,
+};
+
+const struct stmmac_dma_ops ec_dwmac410_dma_ops = {
+	.reset = ec_dwmac4_dma_reset,
+	.init = dwmac4_dma_init,
+	.init_chan = dwmac410_dma_init_channel,
+	.init_rx_chan = dwmac4_dma_init_rx_chan,
+	.init_tx_chan = dwmac4_dma_init_tx_chan,
+	.axi = dwmac4_dma_axi,
+	.dump_regs = dwmac4_dump_dma_regs,
+	.dma_rx_mode = dwmac4_dma_rx_chan_op_mode,
+	.dma_tx_mode = dwmac4_dma_tx_chan_op_mode,
+	.enable_dma_irq = ec_dwmac410_enable_dma_irq,
+	.disable_dma_irq = ec_dwmac4_disable_dma_irq,
+	.start_tx = ec_dwmac4_dma_start_tx,
+	.stop_tx = ec_dwmac4_dma_stop_tx,
+	.start_rx = ec_dwmac4_dma_start_rx,
+	.stop_rx = ec_dwmac4_dma_stop_rx,
+	.dma_interrupt = ec_dwmac4_dma_interrupt,
+	.get_hw_feature = dwmac4_get_hw_feature,
+	.rx_watchdog = dwmac4_rx_watchdog,
+	.set_rx_ring_len = ec_dwmac4_set_rx_ring_len,
+	.set_tx_ring_len = ec_dwmac4_set_tx_ring_len,
+	.set_rx_tail_ptr = ec_dwmac4_set_rx_tail_ptr,
+	.set_tx_tail_ptr = ec_dwmac4_set_tx_tail_ptr,
+	.enable_tso = dwmac4_enable_tso,
+	.qmode = dwmac4_qmode,
+	.set_bfsize = dwmac4_set_bfsize,
+	.enable_sph = dwmac4_enable_sph,
+	.enable_tbs = dwmac4_enable_tbs,
+};
diff --git a/drivers/net/ethercat/device/K3/dwmac4_dma-ethercat.h b/drivers/net/ethercat/device/K3/dwmac4_dma-ethercat.h
new file mode 100644
index 000000000000..f73aff9e70e1
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac4_dma-ethercat.h
@@ -0,0 +1,264 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * DWMAC4 DMA Header file.
+ *
+ * Copyright (C) 2007-2015  STMicroelectronics Ltd
+ *
+ * Author: Alexandre Torgue <alexandre.torgue@st.com>
+ */
+
+#ifndef __DWMAC4_DMA_H__
+#define __DWMAC4_DMA_H__
+
+/* Define the max channel number used for tx (also rx).
+ * dwmac4 accepts up to 8 channels for TX (and also 8 channels for RX
+ */
+#define DMA_CHANNEL_NB_MAX		1
+
+#define DMA_BUS_MODE			0x00001000
+#define DMA_SYS_BUS_MODE		0x00001004
+#define DMA_STATUS			0x00001008
+#define DMA_DEBUG_STATUS_0		0x0000100c
+#define DMA_DEBUG_STATUS_1		0x00001010
+#define DMA_DEBUG_STATUS_2		0x00001014
+#define DMA_AXI_BUS_MODE		0x00001028
+#define DMA_TBS_CTRL			0x00001050
+
+/* DMA Bus Mode bitmap */
+#define DMA_BUS_MODE_DCHE		BIT(19)
+#define DMA_BUS_MODE_INTM_MASK		GENMASK(17, 16)
+#define DMA_BUS_MODE_INTM_SHIFT		16
+#define DMA_BUS_MODE_INTM_MODE1		0x1
+#define DMA_BUS_MODE_SFT_RESET		BIT(0)
+
+/* DMA SYS Bus Mode bitmap */
+#define DMA_BUS_MODE_SPH		BIT(24)
+#define DMA_BUS_MODE_PBL		BIT(16)
+#define DMA_BUS_MODE_PBL_SHIFT		16
+#define DMA_BUS_MODE_RPBL_SHIFT		16
+#define DMA_BUS_MODE_MB			BIT(14)
+#define DMA_BUS_MODE_FB			BIT(0)
+
+/* DMA Interrupt top status */
+#define DMA_STATUS_MAC			BIT(17)
+#define DMA_STATUS_MTL			BIT(16)
+#define DMA_STATUS_CHAN7		BIT(7)
+#define DMA_STATUS_CHAN6		BIT(6)
+#define DMA_STATUS_CHAN5		BIT(5)
+#define DMA_STATUS_CHAN4		BIT(4)
+#define DMA_STATUS_CHAN3		BIT(3)
+#define DMA_STATUS_CHAN2		BIT(2)
+#define DMA_STATUS_CHAN1		BIT(1)
+#define DMA_STATUS_CHAN0		BIT(0)
+
+/* DMA debug status bitmap */
+#define DMA_DEBUG_STATUS_TS_MASK	0xf
+#define DMA_DEBUG_STATUS_RS_MASK	0xf
+
+/* DMA AXI bitmap */
+#define DMA_AXI_EN_LPI			BIT(31)
+#define DMA_AXI_LPI_XIT_FRM		BIT(30)
+#define DMA_AXI_WR_OSR_LMT		GENMASK(27, 24)
+#define DMA_AXI_WR_OSR_LMT_SHIFT	24
+#define DMA_AXI_RD_OSR_LMT		GENMASK(19, 16)
+#define DMA_AXI_RD_OSR_LMT_SHIFT	16
+
+#define DMA_AXI_OSR_MAX			0xf
+#define DMA_AXI_MAX_OSR_LIMIT ((DMA_AXI_OSR_MAX << DMA_AXI_WR_OSR_LMT_SHIFT) | \
+				(DMA_AXI_OSR_MAX << DMA_AXI_RD_OSR_LMT_SHIFT))
+
+#define DMA_SYS_BUS_MB			BIT(14)
+#define DMA_AXI_1KBBE			BIT(13)
+#define DMA_SYS_BUS_AAL			BIT(12)
+#define DMA_SYS_BUS_EAME		BIT(11)
+#define DMA_AXI_BLEN256			BIT(7)
+#define DMA_AXI_BLEN128			BIT(6)
+#define DMA_AXI_BLEN64			BIT(5)
+#define DMA_AXI_BLEN32			BIT(4)
+#define DMA_AXI_BLEN16			BIT(3)
+#define DMA_AXI_BLEN8			BIT(2)
+#define DMA_AXI_BLEN4			BIT(1)
+#define DMA_SYS_BUS_FB			BIT(0)
+
+#define DMA_BURST_LEN_DEFAULT		(DMA_AXI_BLEN256 | DMA_AXI_BLEN128 | \
+					DMA_AXI_BLEN64 | DMA_AXI_BLEN32 | \
+					DMA_AXI_BLEN16 | DMA_AXI_BLEN8 | \
+					DMA_AXI_BLEN4)
+
+#define DMA_AXI_BURST_LEN_MASK		0x000000FE
+
+/* DMA TBS Control */
+#define DMA_TBS_FTOS			GENMASK(31, 8)
+#define DMA_TBS_FTOV			BIT(0)
+#define DMA_TBS_DEF_FTOS		(DMA_TBS_FTOS | DMA_TBS_FTOV)
+
+/* Following DMA defines are channel-oriented */
+#define DMA_CHAN_BASE_ADDR		0x00001100
+#define DMA_CHAN_BASE_OFFSET		0x80
+
+static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs,
+				      const u32 x)
+{
+	u32 addr;
+
+	if (addrs)
+		addr = addrs->dma_chan + (x * addrs->dma_chan_offset);
+	else
+		addr = DMA_CHAN_BASE_ADDR + (x * DMA_CHAN_BASE_OFFSET);
+
+	return addr;
+}
+
+#define DMA_CHAN_REG_NUMBER		17
+
+#define DMA_CHAN_CONTROL(addrs, x)	dma_chanx_base_addr(addrs, x)
+#define DMA_CHAN_TX_CONTROL(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x4)
+#define DMA_CHAN_RX_CONTROL(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x8)
+#define DMA_CHAN_TX_BASE_ADDR_HI(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x10)
+#define DMA_CHAN_TX_BASE_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x14)
+#define DMA_CHAN_RX_BASE_ADDR_HI(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x18)
+#define DMA_CHAN_RX_BASE_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x1c)
+#define DMA_CHAN_TX_END_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x20)
+#define DMA_CHAN_RX_END_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x28)
+#define DMA_CHAN_TX_RING_LEN(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x2c)
+#define DMA_CHAN_RX_RING_LEN(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x30)
+#define DMA_CHAN_INTR_ENA(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x34)
+#define DMA_CHAN_RX_WATCHDOG(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x38)
+#define DMA_CHAN_SLOT_CTRL_STATUS(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x3c)
+#define DMA_CHAN_CUR_TX_DESC(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x44)
+#define DMA_CHAN_CUR_RX_DESC(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x4c)
+#define DMA_CHAN_CUR_TX_BUF_ADDR_HI(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x50)
+#define DMA_CHAN_CUR_TX_BUF_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x54)
+#define DMA_CHAN_CUR_RX_BUF_ADDR_HI(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x58)
+#define DMA_CHAN_CUR_RX_BUF_ADDR(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x5c)
+#define DMA_CHAN_STATUS(addrs, x)	(dma_chanx_base_addr(addrs, x) + 0x60)
+
+/* DMA Control X */
+#define DMA_CONTROL_SPH			BIT(24)
+#define DMA_CONTROL_MSS_MASK		GENMASK(13, 0)
+
+/* DMA Tx Channel X Control register defines */
+#define DMA_CONTROL_EDSE		BIT(28)
+#define DMA_CONTROL_TSE			BIT(12)
+#define DMA_CONTROL_OSP			BIT(4)
+#define DMA_CONTROL_ST			BIT(0)
+
+/* DMA Rx Channel X Control register defines */
+#define DMA_CONTROL_SR			BIT(0)
+#define DMA_RBSZ_MASK			GENMASK(14, 1)
+#define DMA_RBSZ_SHIFT			1
+
+/* Interrupt status per channel */
+#define DMA_CHAN_STATUS_REB		GENMASK(21, 19)
+#define DMA_CHAN_STATUS_REB_SHIFT	19
+#define DMA_CHAN_STATUS_TEB		GENMASK(18, 16)
+#define DMA_CHAN_STATUS_TEB_SHIFT	16
+#define DMA_CHAN_STATUS_NIS		BIT(15)
+#define DMA_CHAN_STATUS_AIS		BIT(14)
+#define DMA_CHAN_STATUS_CDE		BIT(13)
+#define DMA_CHAN_STATUS_FBE		BIT(12)
+#define DMA_CHAN_STATUS_ERI		BIT(11)
+#define DMA_CHAN_STATUS_ETI		BIT(10)
+#define DMA_CHAN_STATUS_RWT		BIT(9)
+#define DMA_CHAN_STATUS_RPS		BIT(8)
+#define DMA_CHAN_STATUS_RBU		BIT(7)
+#define DMA_CHAN_STATUS_RI		BIT(6)
+#define DMA_CHAN_STATUS_TBU		BIT(2)
+#define DMA_CHAN_STATUS_TPS		BIT(1)
+#define DMA_CHAN_STATUS_TI		BIT(0)
+
+#define DMA_CHAN_STATUS_MSK_COMMON	(DMA_CHAN_STATUS_NIS | \
+					 DMA_CHAN_STATUS_AIS | \
+					 DMA_CHAN_STATUS_CDE | \
+					 DMA_CHAN_STATUS_FBE)
+
+#define DMA_CHAN_STATUS_MSK_RX		(DMA_CHAN_STATUS_REB | \
+					 DMA_CHAN_STATUS_ERI | \
+					 DMA_CHAN_STATUS_RWT | \
+					 DMA_CHAN_STATUS_RPS | \
+					 DMA_CHAN_STATUS_RBU | \
+					 DMA_CHAN_STATUS_RI | \
+					 DMA_CHAN_STATUS_MSK_COMMON)
+
+#define DMA_CHAN_STATUS_MSK_TX		(DMA_CHAN_STATUS_ETI | \
+					 DMA_CHAN_STATUS_TBU | \
+					 DMA_CHAN_STATUS_TPS | \
+					 DMA_CHAN_STATUS_TI | \
+					 DMA_CHAN_STATUS_MSK_COMMON)
+
+/* Interrupt enable bits per channel */
+#define DMA_CHAN_INTR_ENA_NIE		BIT(16)
+#define DMA_CHAN_INTR_ENA_AIE		BIT(15)
+#define DMA_CHAN_INTR_ENA_NIE_4_10	BIT(15)
+#define DMA_CHAN_INTR_ENA_AIE_4_10	BIT(14)
+#define DMA_CHAN_INTR_ENA_CDE		BIT(13)
+#define DMA_CHAN_INTR_ENA_FBE		BIT(12)
+#define DMA_CHAN_INTR_ENA_ERE		BIT(11)
+#define DMA_CHAN_INTR_ENA_ETE		BIT(10)
+#define DMA_CHAN_INTR_ENA_RWE		BIT(9)
+#define DMA_CHAN_INTR_ENA_RSE		BIT(8)
+#define DMA_CHAN_INTR_ENA_RBUE		BIT(7)
+#define DMA_CHAN_INTR_ENA_RIE		BIT(6)
+#define DMA_CHAN_INTR_ENA_TBUE		BIT(2)
+#define DMA_CHAN_INTR_ENA_TSE		BIT(1)
+#define DMA_CHAN_INTR_ENA_TIE		BIT(0)
+
+#define DMA_CHAN_INTR_NORMAL		(DMA_CHAN_INTR_ENA_NIE | \
+					 DMA_CHAN_INTR_ENA_RIE | \
+					 DMA_CHAN_INTR_ENA_TIE)
+
+#define DMA_CHAN_INTR_ABNORMAL		(DMA_CHAN_INTR_ENA_AIE | \
+					 DMA_CHAN_INTR_ENA_FBE)
+/* DMA default interrupt mask for 4.00 */
+#define DMA_CHAN_INTR_DEFAULT_MASK	(DMA_CHAN_INTR_NORMAL | \
+					 DMA_CHAN_INTR_ABNORMAL)
+#define DMA_CHAN_INTR_DEFAULT_RX	(DMA_CHAN_INTR_ENA_RIE)
+#define DMA_CHAN_INTR_DEFAULT_TX	(DMA_CHAN_INTR_ENA_TIE)
+
+#define DMA_CHAN_INTR_NORMAL_4_10	(DMA_CHAN_INTR_ENA_NIE_4_10 | \
+					 DMA_CHAN_INTR_ENA_RIE | \
+					 DMA_CHAN_INTR_ENA_TIE)
+
+#define DMA_CHAN_INTR_ABNORMAL_4_10	(DMA_CHAN_INTR_ENA_AIE_4_10 | \
+					 DMA_CHAN_INTR_ENA_FBE)
+/* DMA default interrupt mask for 4.10a */
+#define DMA_CHAN_INTR_DEFAULT_MASK_4_10	(DMA_CHAN_INTR_NORMAL_4_10 | \
+					 DMA_CHAN_INTR_ABNORMAL_4_10)
+#define DMA_CHAN_INTR_DEFAULT_RX_4_10	(DMA_CHAN_INTR_ENA_RIE)
+#define DMA_CHAN_INTR_DEFAULT_TX_4_10	(DMA_CHAN_INTR_ENA_TIE)
+
+/* channel 0 specific fields */
+#define DMA_CHAN0_DBG_STAT_TPS		GENMASK(15, 12)
+#define DMA_CHAN0_DBG_STAT_TPS_SHIFT	12
+#define DMA_CHAN0_DBG_STAT_RPS		GENMASK(11, 8)
+#define DMA_CHAN0_DBG_STAT_RPS_SHIFT	8
+
+int ec_dwmac4_dma_reset(void __iomem *ioaddr);
+void ec_dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx);
+void ec_dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     u32 chan, bool rx, bool tx);
+void ec_dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 chan, bool rx, bool tx);
+void ec_dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      u32 chan, bool rx, bool tx);
+void ec_dwmac4_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+void ec_dwmac4_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+void ec_dwmac4_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+void ec_dwmac4_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+int ec_dwmac4_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 struct stmmac_extra_stats *x, u32 chan, u32 dir);
+void ec_dwmac4_set_rx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan);
+void ec_dwmac4_set_tx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan);
+void ec_dwmac4_set_rx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan);
+void ec_dwmac4_set_tx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan);
+
+#endif /* __DWMAC4_DMA_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac4_lib-ethercat.c b/drivers/net/ethercat/device/K3/dwmac4_lib-ethercat.c
new file mode 100644
index 000000000000..0f22a8d2a33d
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac4_lib-ethercat.c
@@ -0,0 +1,274 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2007-2015  STMicroelectronics Ltd
+ *
+ * Author: Alexandre Torgue <alexandre.torgue@st.com>
+ */
+
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/delay.h>
+#include "common-ethercat.h"
+#include "dwmac4_dma-ethercat.h"
+#include "dwmac4-ethercat.h"
+#include "stmmac-ethercat.h"
+
+int ec_dwmac4_dma_reset(void __iomem *ioaddr)
+{
+	u32 value = readl(ioaddr + DMA_BUS_MODE);
+
+	/* DMA SW reset */
+	value |= DMA_BUS_MODE_SFT_RESET;
+	writel(value, ioaddr + DMA_BUS_MODE);
+
+	return readl_poll_timeout(ioaddr + DMA_BUS_MODE, value,
+				 !(value & DMA_BUS_MODE_SFT_RESET),
+				 10000, 1000000);
+}
+
+void ec_dwmac4_set_rx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(tail_ptr, ioaddr + DMA_CHAN_RX_END_ADDR(dwmac4_addrs, chan));
+}
+
+void ec_dwmac4_set_tx_tail_ptr(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 tail_ptr, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(tail_ptr, ioaddr + DMA_CHAN_TX_END_ADDR(dwmac4_addrs, chan));
+}
+
+void ec_dwmac4_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+
+	value |= DMA_CONTROL_ST;
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+
+	value = readl(ioaddr + GMAC_CONFIG);
+	value |= GMAC_CONFIG_TE;
+	writel(value, ioaddr + GMAC_CONFIG);
+}
+
+void ec_dwmac4_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	u32 value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+
+	value &= ~DMA_CONTROL_ST;
+	writel(value, ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan));
+}
+
+void ec_dwmac4_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+
+	value |= DMA_CONTROL_SR;
+
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+
+	value = readl(ioaddr + GMAC_CONFIG);
+	value |= GMAC_CONFIG_RE;
+	writel(value, ioaddr + GMAC_CONFIG);
+}
+
+void ec_dwmac4_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+
+	value &= ~DMA_CONTROL_SR;
+	writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan));
+}
+
+void ec_dwmac4_set_tx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(len, ioaddr + DMA_CHAN_TX_RING_LEN(dwmac4_addrs, chan));
+}
+
+void ec_dwmac4_set_rx_ring_len(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 len, u32 chan)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+
+	writel(len, ioaddr + DMA_CHAN_RX_RING_LEN(dwmac4_addrs, chan));
+}
+
+void ec_dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+
+	if (rx)
+		value |= DMA_CHAN_INTR_DEFAULT_RX;
+	if (tx)
+		value |= DMA_CHAN_INTR_DEFAULT_TX;
+
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+}
+
+void ec_dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     u32 chan, bool rx, bool tx)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+
+	if (rx)
+		value |= DMA_CHAN_INTR_DEFAULT_RX_4_10;
+	if (tx)
+		value |= DMA_CHAN_INTR_DEFAULT_TX_4_10;
+
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+}
+
+void ec_dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 chan, bool rx, bool tx)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+
+	if (rx)
+		value &= ~DMA_CHAN_INTR_DEFAULT_RX;
+	if (tx)
+		value &= ~DMA_CHAN_INTR_DEFAULT_TX;
+
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+}
+
+void ec_dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			      u32 chan, bool rx, bool tx)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+
+	if (rx)
+		value &= ~DMA_CHAN_INTR_DEFAULT_RX_4_10;
+	if (tx)
+		value &= ~DMA_CHAN_INTR_DEFAULT_TX_4_10;
+
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+}
+
+int ec_dwmac4_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 struct stmmac_extra_stats *x, u32 chan, u32 dir)
+{
+	const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs;
+	u32 intr_status = readl(ioaddr + DMA_CHAN_STATUS(dwmac4_addrs, chan));
+	u32 intr_en = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan));
+	struct stmmac_pcpu_stats *stats = this_cpu_ptr(priv->xstats.pcpu_stats);
+	int ret = 0;
+
+	if (dir == DMA_DIR_RX)
+		intr_status &= DMA_CHAN_STATUS_MSK_RX;
+	else if (dir == DMA_DIR_TX)
+		intr_status &= DMA_CHAN_STATUS_MSK_TX;
+
+	/* ABNORMAL interrupts */
+	if (unlikely(intr_status & DMA_CHAN_STATUS_AIS)) {
+		if (unlikely(intr_status & DMA_CHAN_STATUS_RBU))
+			x->rx_buf_unav_irq++;
+		if (unlikely(intr_status & DMA_CHAN_STATUS_RPS))
+			x->rx_process_stopped_irq++;
+		if (unlikely(intr_status & DMA_CHAN_STATUS_ETI))
+			x->tx_early_irq++;
+		if (unlikely(intr_status & DMA_CHAN_STATUS_TPS)) {
+			x->tx_process_stopped_irq++;
+			ret = tx_hard_error;
+		}
+		if (unlikely(intr_status & DMA_CHAN_STATUS_FBE)) {
+			x->fatal_bus_error_irq++;
+			ret = tx_hard_error;
+		}
+	}
+
+	if (unlikely(intr_status & DMA_CHAN_STATUS_RWT))
+		x->rx_watchdog_irq++;
+
+	/* TX/RX NORMAL interrupts */
+	if (likely(intr_status & DMA_CHAN_STATUS_RI)) {
+		u64_stats_update_begin(&stats->syncp);
+		u64_stats_inc(&stats->rx_normal_irq_n[chan]);
+		u64_stats_update_end(&stats->syncp);
+		ret |= handle_rx;
+	}
+	if (likely(intr_status & DMA_CHAN_STATUS_TI)) {
+		u64_stats_update_begin(&stats->syncp);
+		u64_stats_inc(&stats->tx_normal_irq_n[chan]);
+		u64_stats_update_end(&stats->syncp);
+		ret |= handle_tx;
+	}
+
+	if (unlikely(intr_status & DMA_CHAN_STATUS_TBU))
+		ret |= handle_tx;
+	if (unlikely(intr_status & DMA_CHAN_STATUS_ERI))
+		x->rx_early_irq++;
+
+	writel(intr_status & intr_en,
+	       ioaddr + DMA_CHAN_STATUS(dwmac4_addrs, chan));
+	return ret;
+}
+
+void ec_stmmac_dwmac4_set_mac_addr(void __iomem *ioaddr, const u8 addr[6],
+				unsigned int high, unsigned int low)
+{
+	unsigned long data;
+
+	data = (addr[5] << 8) | addr[4];
+	/* For MAC Addr registers se have to set the Address Enable (AE)
+	 * bit that has no effect on the High Reg 0 where the bit 31 (MO)
+	 * is RO.
+	 */
+	data |= (STMMAC_CHAN0 << GMAC_HI_DCS_SHIFT);
+	writel(data | GMAC_HI_REG_AE, ioaddr + high);
+	data = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0];
+	writel(data, ioaddr + low);
+}
+
+/* Enable disable MAC RX/TX */
+void ec_stmmac_dwmac4_set_mac(void __iomem *ioaddr, bool enable)
+{
+	u32 value = readl(ioaddr + GMAC_CONFIG);
+	u32 old_val = value;
+
+	if (enable)
+		value |= GMAC_CONFIG_RE | GMAC_CONFIG_TE;
+	else
+		value &= ~(GMAC_CONFIG_TE | GMAC_CONFIG_RE);
+
+	if (value != old_val)
+		writel(value, ioaddr + GMAC_CONFIG);
+}
+
+void ec_stmmac_dwmac4_get_mac_addr(void __iomem *ioaddr, unsigned char *addr,
+				unsigned int high, unsigned int low)
+{
+	unsigned int hi_addr, lo_addr;
+
+	/* Read the MAC address from the hardware */
+	hi_addr = readl(ioaddr + high);
+	lo_addr = readl(ioaddr + low);
+
+	/* Extract the MAC address from the high and low words */
+	addr[0] = lo_addr & 0xff;
+	addr[1] = (lo_addr >> 8) & 0xff;
+	addr[2] = (lo_addr >> 16) & 0xff;
+	addr[3] = (lo_addr >> 24) & 0xff;
+	addr[4] = hi_addr & 0xff;
+	addr[5] = (hi_addr >> 8) & 0xff;
+}
diff --git a/drivers/net/ethercat/device/K3/dwmac5-ethercat.c b/drivers/net/ethercat/device/K3/dwmac5-ethercat.c
new file mode 100644
index 000000000000..68e089f1f2a9
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac5-ethercat.c
@@ -0,0 +1,574 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+// Copyright (c) 2017 Synopsys, Inc. and/or its affiliates.
+// stmmac Support for 5.xx Ethernet QoS cores
+
+#include <linux/bitops.h>
+#include <linux/iopoll.h>
+#include "common-ethercat.h"
+#include "dwmac4-ethercat.h"
+#include "dwmac5-ethercat.h"
+#include "stmmac-ethercat.h"
+#include "stmmac_ptp-ethercat.h"
+
+struct dwmac5_error_desc {
+	bool valid;
+	const char *desc;
+	const char *detailed_desc;
+};
+
+#define STAT_OFF(field)		offsetof(struct stmmac_safety_stats, field)
+
+static void dwmac5_log_error(struct net_device *ndev, u32 value, bool corr,
+		const char *module_name, const struct dwmac5_error_desc *desc,
+		unsigned long field_offset, struct stmmac_safety_stats *stats)
+{
+	unsigned long loc, mask;
+	u8 *bptr = (u8 *)stats;
+	unsigned long *ptr;
+
+	ptr = (unsigned long *)(bptr + field_offset);
+
+	mask = value;
+	for_each_set_bit(loc, &mask, 32) {
+		netdev_err(ndev, "Found %s error in %s: '%s: %s'\n", corr ?
+				"correctable" : "uncorrectable", module_name,
+				desc[loc].desc, desc[loc].detailed_desc);
+
+		/* Update counters */
+		ptr[loc]++;
+	}
+}
+
+static const struct dwmac5_error_desc dwmac5_mac_errors[32]= {
+	{ true, "ATPES", "Application Transmit Interface Parity Check Error" },
+	{ true, "TPES", "TSO Data Path Parity Check Error" },
+	{ true, "RDPES", "Read Descriptor Parity Check Error" },
+	{ true, "MPES", "MTL Data Path Parity Check Error" },
+	{ true, "MTSPES", "MTL TX Status Data Path Parity Check Error" },
+	{ true, "ARPES", "Application Receive Interface Data Path Parity Check Error" },
+	{ true, "CWPES", "CSR Write Data Path Parity Check Error" },
+	{ true, "ASRPES", "AXI Slave Read Data Path Parity Check Error" },
+	{ true, "TTES", "TX FSM Timeout Error" },
+	{ true, "RTES", "RX FSM Timeout Error" },
+	{ true, "CTES", "CSR FSM Timeout Error" },
+	{ true, "ATES", "APP FSM Timeout Error" },
+	{ true, "PTES", "PTP FSM Timeout Error" },
+	{ true, "T125ES", "TX125 FSM Timeout Error" },
+	{ true, "R125ES", "RX125 FSM Timeout Error" },
+	{ true, "RVCTES", "REV MDC FSM Timeout Error" },
+	{ true, "MSTTES", "Master Read/Write Timeout Error" },
+	{ true, "SLVTES", "Slave Read/Write Timeout Error" },
+	{ true, "ATITES", "Application Timeout on ATI Interface Error" },
+	{ true, "ARITES", "Application Timeout on ARI Interface Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 21 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 22 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 23 */
+	{ true, "FSMPES", "FSM State Parity Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 25 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 31 */
+};
+
+static void dwmac5_handle_mac_err(struct net_device *ndev,
+		void __iomem *ioaddr, bool correctable,
+		struct stmmac_safety_stats *stats)
+{
+	u32 value;
+
+	value = readl(ioaddr + MAC_DPP_FSM_INT_STATUS);
+	writel(value, ioaddr + MAC_DPP_FSM_INT_STATUS);
+
+	dwmac5_log_error(ndev, value, correctable, "MAC", dwmac5_mac_errors,
+			STAT_OFF(mac_errors), stats);
+}
+
+static const struct dwmac5_error_desc dwmac5_mtl_errors[32]= {
+	{ true, "TXCES", "MTL TX Memory Error" },
+	{ true, "TXAMS", "MTL TX Memory Address Mismatch Error" },
+	{ true, "TXUES", "MTL TX Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 3 */
+	{ true, "RXCES", "MTL RX Memory Error" },
+	{ true, "RXAMS", "MTL RX Memory Address Mismatch Error" },
+	{ true, "RXUES", "MTL RX Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 7 */
+	{ true, "ECES", "MTL EST Memory Error" },
+	{ true, "EAMS", "MTL EST Memory Address Mismatch Error" },
+	{ true, "EUES", "MTL EST Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 11 */
+	{ true, "RPCES", "MTL RX Parser Memory Error" },
+	{ true, "RPAMS", "MTL RX Parser Memory Address Mismatch Error" },
+	{ true, "RPUES", "MTL RX Parser Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 15 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 16 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 17 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 21 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 22 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 23 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 24 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 25 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 31 */
+};
+
+static void dwmac5_handle_mtl_err(struct net_device *ndev,
+		void __iomem *ioaddr, bool correctable,
+		struct stmmac_safety_stats *stats)
+{
+	u32 value;
+
+	value = readl(ioaddr + MTL_ECC_INT_STATUS);
+	writel(value, ioaddr + MTL_ECC_INT_STATUS);
+
+	dwmac5_log_error(ndev, value, correctable, "MTL", dwmac5_mtl_errors,
+			STAT_OFF(mtl_errors), stats);
+}
+
+static const struct dwmac5_error_desc dwmac5_dma_errors[32]= {
+	{ true, "TCES", "DMA TSO Memory Error" },
+	{ true, "TAMS", "DMA TSO Memory Address Mismatch Error" },
+	{ true, "TUES", "DMA TSO Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 3 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 4 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 5 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 6 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 7 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 8 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 9 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 10 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 11 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 12 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 13 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 14 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 15 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 16 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 17 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 21 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 22 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 23 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 24 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 25 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 31 */
+};
+
+static void dwmac5_handle_dma_err(struct net_device *ndev,
+		void __iomem *ioaddr, bool correctable,
+		struct stmmac_safety_stats *stats)
+{
+	u32 value;
+
+	value = readl(ioaddr + DMA_ECC_INT_STATUS);
+	writel(value, ioaddr + DMA_ECC_INT_STATUS);
+
+	dwmac5_log_error(ndev, value, correctable, "DMA", dwmac5_dma_errors,
+			STAT_OFF(dma_errors), stats);
+}
+
+int ec_dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp,
+			      struct stmmac_safety_feature_cfg *safety_feat_cfg)
+{
+	struct stmmac_safety_feature_cfg all_safety_feats = {
+		.tsoee = 1,
+		.mrxpee = 1,
+		.mestee = 1,
+		.mrxee = 1,
+		.mtxee = 1,
+		.epsi = 1,
+		.edpp = 1,
+		.prtyen = 1,
+		.tmouten = 1,
+	};
+	u32 value;
+
+	if (!asp)
+		return -EINVAL;
+
+	if (!safety_feat_cfg)
+		safety_feat_cfg = &all_safety_feats;
+
+	/* 1. Enable Safety Features */
+	value = readl(ioaddr + MTL_ECC_CONTROL);
+	value |= MEEAO; /* MTL ECC Error Addr Status Override */
+	if (safety_feat_cfg->tsoee)
+		value |= TSOEE; /* TSO ECC */
+	if (safety_feat_cfg->mrxpee)
+		value |= MRXPEE; /* MTL RX Parser ECC */
+	if (safety_feat_cfg->mestee)
+		value |= MESTEE; /* MTL EST ECC */
+	if (safety_feat_cfg->mrxee)
+		value |= MRXEE; /* MTL RX FIFO ECC */
+	if (safety_feat_cfg->mtxee)
+		value |= MTXEE; /* MTL TX FIFO ECC */
+	writel(value, ioaddr + MTL_ECC_CONTROL);
+
+	/* 2. Enable MTL Safety Interrupts */
+	value = readl(ioaddr + MTL_ECC_INT_ENABLE);
+	value |= RPCEIE; /* RX Parser Memory Correctable Error */
+	value |= ECEIE; /* EST Memory Correctable Error */
+	value |= RXCEIE; /* RX Memory Correctable Error */
+	value |= TXCEIE; /* TX Memory Correctable Error */
+	writel(value, ioaddr + MTL_ECC_INT_ENABLE);
+
+	/* 3. Enable DMA Safety Interrupts */
+	value = readl(ioaddr + DMA_ECC_INT_ENABLE);
+	value |= TCEIE; /* TSO Memory Correctable Error */
+	writel(value, ioaddr + DMA_ECC_INT_ENABLE);
+
+	/* Only ECC Protection for External Memory feature is selected */
+	if (asp <= 0x1)
+		return 0;
+
+	/* 5. Enable Parity and Timeout for FSM */
+	value = readl(ioaddr + MAC_FSM_CONTROL);
+	if (safety_feat_cfg->prtyen)
+		value |= PRTYEN; /* FSM Parity Feature */
+	if (safety_feat_cfg->tmouten)
+		value |= TMOUTEN; /* FSM Timeout Feature */
+	writel(value, ioaddr + MAC_FSM_CONTROL);
+
+	/* 4. Enable Data Parity Protection */
+	value = readl(ioaddr + MTL_DPP_CONTROL);
+	if (safety_feat_cfg->edpp)
+		value |= EDPP;
+	writel(value, ioaddr + MTL_DPP_CONTROL);
+
+	/*
+	 * All the Automotive Safety features are selected without the "Parity
+	 * Port Enable for external interface" feature.
+	 */
+	if (asp <= 0x2)
+		return 0;
+
+	if (safety_feat_cfg->epsi)
+		value |= EPSI;
+	writel(value, ioaddr + MTL_DPP_CONTROL);
+	return 0;
+}
+
+int ec_dwmac5_safety_feat_irq_status(struct net_device *ndev,
+		void __iomem *ioaddr, unsigned int asp,
+		struct stmmac_safety_stats *stats)
+{
+	bool err, corr;
+	u32 mtl, dma;
+	int ret = 0;
+
+	if (!asp)
+		return -EINVAL;
+
+	mtl = readl(ioaddr + MTL_SAFETY_INT_STATUS);
+	dma = readl(ioaddr + DMA_SAFETY_INT_STATUS);
+
+	err = (mtl & MCSIS) || (dma & MCSIS);
+	corr = false;
+	if (err) {
+		dwmac5_handle_mac_err(ndev, ioaddr, corr, stats);
+		ret |= !corr;
+	}
+
+	err = (mtl & (MEUIS | MECIS)) || (dma & (MSUIS | MSCIS));
+	corr = (mtl & MECIS) || (dma & MSCIS);
+	if (err) {
+		dwmac5_handle_mtl_err(ndev, ioaddr, corr, stats);
+		ret |= !corr;
+	}
+
+	err = dma & (DEUIS | DECIS);
+	corr = dma & DECIS;
+	if (err) {
+		dwmac5_handle_dma_err(ndev, ioaddr, corr, stats);
+		ret |= !corr;
+	}
+
+	return ret;
+}
+
+static const struct dwmac5_error {
+	const struct dwmac5_error_desc *desc;
+} dwmac5_all_errors[] = {
+	{ dwmac5_mac_errors },
+	{ dwmac5_mtl_errors },
+	{ dwmac5_dma_errors },
+};
+
+int ec_dwmac5_safety_feat_dump(struct stmmac_safety_stats *stats,
+			int index, unsigned long *count, const char **desc)
+{
+	int module = index / 32, offset = index % 32;
+	unsigned long *ptr = (unsigned long *)stats;
+
+	if (module >= ARRAY_SIZE(dwmac5_all_errors))
+		return -EINVAL;
+	if (!dwmac5_all_errors[module].desc[offset].valid)
+		return -EINVAL;
+	if (count)
+		*count = *(ptr + index);
+	if (desc)
+		*desc = dwmac5_all_errors[module].desc[offset].desc;
+	return 0;
+}
+
+static int dwmac5_rxp_disable(void __iomem *ioaddr)
+{
+	u32 val;
+
+	val = readl(ioaddr + MTL_OPERATION_MODE);
+	val &= ~MTL_FRPE;
+	writel(val, ioaddr + MTL_OPERATION_MODE);
+
+	return readl_poll_timeout(ioaddr + MTL_RXP_CONTROL_STATUS, val,
+			val & RXPI, 1, 10000);
+}
+
+static void dwmac5_rxp_enable(void __iomem *ioaddr)
+{
+	u32 val;
+
+	val = readl(ioaddr + MTL_OPERATION_MODE);
+	val |= MTL_FRPE;
+	writel(val, ioaddr + MTL_OPERATION_MODE);
+}
+
+static int dwmac5_rxp_update_single_entry(void __iomem *ioaddr,
+					  struct stmmac_tc_entry *entry,
+					  int pos)
+{
+	int ret, i;
+
+	for (i = 0; i < (sizeof(entry->val) / sizeof(u32)); i++) {
+		int real_pos = pos * (sizeof(entry->val) / sizeof(u32)) + i;
+		u32 val;
+
+		/* Wait for ready */
+		ret = readl_poll_timeout(ioaddr + MTL_RXP_IACC_CTRL_STATUS,
+				val, !(val & STARTBUSY), 1, 10000);
+		if (ret)
+			return ret;
+
+		/* Write data */
+		val = *((u32 *)&entry->val + i);
+		writel(val, ioaddr + MTL_RXP_IACC_DATA);
+
+		/* Write pos */
+		val = real_pos & ADDR;
+		writel(val, ioaddr + MTL_RXP_IACC_CTRL_STATUS);
+
+		/* Write OP */
+		val |= WRRDN;
+		writel(val, ioaddr + MTL_RXP_IACC_CTRL_STATUS);
+
+		/* Start Write */
+		val |= STARTBUSY;
+		writel(val, ioaddr + MTL_RXP_IACC_CTRL_STATUS);
+
+		/* Wait for done */
+		ret = readl_poll_timeout(ioaddr + MTL_RXP_IACC_CTRL_STATUS,
+				val, !(val & STARTBUSY), 1, 10000);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static struct stmmac_tc_entry *
+dwmac5_rxp_get_next_entry(struct stmmac_tc_entry *entries, unsigned int count,
+			  u32 curr_prio)
+{
+	struct stmmac_tc_entry *entry;
+	u32 min_prio = ~0x0;
+	int i, min_prio_idx;
+	bool found = false;
+
+	for (i = count - 1; i >= 0; i--) {
+		entry = &entries[i];
+
+		/* Do not update unused entries */
+		if (!entry->in_use)
+			continue;
+		/* Do not update already updated entries (i.e. fragments) */
+		if (entry->in_hw)
+			continue;
+		/* Let last entry be updated last */
+		if (entry->is_last)
+			continue;
+		/* Do not return fragments */
+		if (entry->is_frag)
+			continue;
+		/* Check if we already checked this prio */
+		if (entry->prio < curr_prio)
+			continue;
+		/* Check if this is the minimum prio */
+		if (entry->prio < min_prio) {
+			min_prio = entry->prio;
+			min_prio_idx = i;
+			found = true;
+		}
+	}
+
+	if (found)
+		return &entries[min_prio_idx];
+	return NULL;
+}
+
+int ec_dwmac5_rxp_config(void __iomem *ioaddr, struct stmmac_tc_entry *entries,
+		      unsigned int count)
+{
+	struct stmmac_tc_entry *entry, *frag;
+	int i, ret, nve = 0;
+	u32 curr_prio = 0;
+	u32 old_val, val;
+
+	/* Force disable RX */
+	old_val = readl(ioaddr + GMAC_CONFIG);
+	val = old_val & ~GMAC_CONFIG_RE;
+	writel(val, ioaddr + GMAC_CONFIG);
+
+	/* Disable RX Parser */
+	ret = dwmac5_rxp_disable(ioaddr);
+	if (ret)
+		goto re_enable;
+
+	/* Set all entries as NOT in HW */
+	for (i = 0; i < count; i++) {
+		entry = &entries[i];
+		entry->in_hw = false;
+	}
+
+	/* Update entries by reverse order */
+	while (1) {
+		entry = dwmac5_rxp_get_next_entry(entries, count, curr_prio);
+		if (!entry)
+			break;
+
+		curr_prio = entry->prio;
+		frag = entry->frag_ptr;
+
+		/* Set special fragment requirements */
+		if (frag) {
+			entry->val.af = 0;
+			entry->val.rf = 0;
+			entry->val.nc = 1;
+			entry->val.ok_index = nve + 2;
+		}
+
+		ret = dwmac5_rxp_update_single_entry(ioaddr, entry, nve);
+		if (ret)
+			goto re_enable;
+
+		entry->table_pos = nve++;
+		entry->in_hw = true;
+
+		if (frag && !frag->in_hw) {
+			ret = dwmac5_rxp_update_single_entry(ioaddr, frag, nve);
+			if (ret)
+				goto re_enable;
+			frag->table_pos = nve++;
+			frag->in_hw = true;
+		}
+	}
+
+	if (!nve)
+		goto re_enable;
+
+	/* Update all pass entry */
+	for (i = 0; i < count; i++) {
+		entry = &entries[i];
+		if (!entry->is_last)
+			continue;
+
+		ret = dwmac5_rxp_update_single_entry(ioaddr, entry, nve);
+		if (ret)
+			goto re_enable;
+
+		entry->table_pos = nve++;
+	}
+
+	/* Assume n. of parsable entries == n. of valid entries */
+	val = (nve << 16) & NPE;
+	val |= nve & NVE;
+	writel(val, ioaddr + MTL_RXP_CONTROL_STATUS);
+
+	/* Enable RX Parser */
+	dwmac5_rxp_enable(ioaddr);
+
+re_enable:
+	/* Re-enable RX */
+	writel(old_val, ioaddr + GMAC_CONFIG);
+	return ret;
+}
+
+int ec_dwmac5_flex_pps_config(void __iomem *ioaddr, int index,
+			   struct stmmac_pps_cfg *cfg, bool enable,
+			   u32 sub_second_inc, u32 systime_flags)
+{
+	u32 tnsec = readl(ioaddr + MAC_PPSx_TARGET_TIME_NSEC(index));
+	u32 val = readl(ioaddr + MAC_PPS_CONTROL);
+	u64 period;
+
+	if (!cfg->available)
+		return -EINVAL;
+	if (tnsec & TRGTBUSY0)
+		return -EBUSY;
+	if (!sub_second_inc || !systime_flags)
+		return -EINVAL;
+
+	val &= ~PPSx_MASK(index);
+
+	if (!enable) {
+		val |= PPSCMDx(index, 0x5);
+		val |= PPSEN0;
+		writel(val, ioaddr + MAC_PPS_CONTROL);
+		return 0;
+	}
+
+	val |= TRGTMODSELx(index, 0x2);
+	val |= PPSEN0;
+	writel(val, ioaddr + MAC_PPS_CONTROL);
+
+	writel(cfg->start.tv_sec, ioaddr + MAC_PPSx_TARGET_TIME_SEC(index));
+
+	if (!(systime_flags & PTP_TCR_TSCTRLSSR))
+		cfg->start.tv_nsec = (cfg->start.tv_nsec * 1000) / 465;
+	writel(cfg->start.tv_nsec, ioaddr + MAC_PPSx_TARGET_TIME_NSEC(index));
+
+	period = cfg->period.tv_sec * 1000000000;
+	period += cfg->period.tv_nsec;
+
+	do_div(period, sub_second_inc);
+
+	if (period <= 1)
+		return -EINVAL;
+
+	writel(period - 1, ioaddr + MAC_PPSx_INTERVAL(index));
+
+	period >>= 1;
+	if (period <= 1)
+		return -EINVAL;
+
+	writel(period - 1, ioaddr + MAC_PPSx_WIDTH(index));
+
+	/* Finally, activate it */
+	val |= PPSCMDx(index, 0x2);
+	writel(val, ioaddr + MAC_PPS_CONTROL);
+	return 0;
+}
diff --git a/drivers/net/ethercat/device/K3/dwmac5-ethercat.h b/drivers/net/ethercat/device/K3/dwmac5-ethercat.h
new file mode 100644
index 000000000000..c6e2d537c9b8
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac5-ethercat.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+// Copyright (c) 2017 Synopsys, Inc. and/or its affiliates.
+// stmmac Support for 5.xx Ethernet QoS cores
+
+#ifndef __DWMAC5_H__
+#define __DWMAC5_H__
+
+#define MAC_DPP_FSM_INT_STATUS		0x00000140
+#define MAC_AXI_SLV_DPE_ADDR_STATUS	0x00000144
+#define MAC_FSM_CONTROL			0x00000148
+#define PRTYEN				BIT(1)
+#define TMOUTEN				BIT(0)
+
+#define MAC_PPS_CONTROL			0x00000b70
+#define PPS_MAXIDX(x)			((((x) + 1) * 8) - 1)
+#define PPS_MINIDX(x)			((x) * 8)
+#define PPSx_MASK(x)			GENMASK(PPS_MAXIDX(x), PPS_MINIDX(x))
+#define MCGRENx(x)			BIT(PPS_MAXIDX(x))
+#define TRGTMODSELx(x, val)		\
+	GENMASK(PPS_MAXIDX(x) - 1, PPS_MAXIDX(x) - 2) & \
+	((val) << (PPS_MAXIDX(x) - 2))
+#define PPSCMDx(x, val)			\
+	GENMASK(PPS_MINIDX(x) + 3, PPS_MINIDX(x)) & \
+	((val) << PPS_MINIDX(x))
+#define PPSEN0				BIT(4)
+#define MAC_PPSx_TARGET_TIME_SEC(x)	(0x00000b80 + ((x) * 0x10))
+#define MAC_PPSx_TARGET_TIME_NSEC(x)	(0x00000b84 + ((x) * 0x10))
+#define TRGTBUSY0			BIT(31)
+#define TTSL0				GENMASK(30, 0)
+#define MAC_PPSx_INTERVAL(x)		(0x00000b88 + ((x) * 0x10))
+#define MAC_PPSx_WIDTH(x)		(0x00000b8c + ((x) * 0x10))
+
+#define MTL_RXP_CONTROL_STATUS		0x00000ca0
+#define RXPI				BIT(31)
+#define NPE				GENMASK(23, 16)
+#define NVE				GENMASK(7, 0)
+#define MTL_RXP_IACC_CTRL_STATUS	0x00000cb0
+#define STARTBUSY			BIT(31)
+#define RXPEIEC				GENMASK(22, 21)
+#define RXPEIEE				BIT(20)
+#define WRRDN				BIT(16)
+#define ADDR				GENMASK(15, 0)
+#define MTL_RXP_IACC_DATA		0x00000cb4
+#define MTL_ECC_CONTROL			0x00000cc0
+#define MEEAO				BIT(8)
+#define TSOEE				BIT(4)
+#define MRXPEE				BIT(3)
+#define MESTEE				BIT(2)
+#define MRXEE				BIT(1)
+#define MTXEE				BIT(0)
+
+#define MTL_SAFETY_INT_STATUS		0x00000cc4
+#define MCSIS				BIT(31)
+#define MEUIS				BIT(1)
+#define MECIS				BIT(0)
+#define MTL_ECC_INT_ENABLE		0x00000cc8
+#define RPCEIE				BIT(12)
+#define ECEIE				BIT(8)
+#define RXCEIE				BIT(4)
+#define TXCEIE				BIT(0)
+#define MTL_ECC_INT_STATUS		0x00000ccc
+#define MTL_DPP_CONTROL			0x00000ce0
+#define EPSI				BIT(2)
+#define OPE				BIT(1)
+#define EDPP				BIT(0)
+
+#define DMA_SAFETY_INT_STATUS		0x00001080
+#define MSUIS				BIT(29)
+#define MSCIS				BIT(28)
+#define DEUIS				BIT(1)
+#define DECIS				BIT(0)
+#define DMA_ECC_INT_ENABLE		0x00001084
+#define TCEIE				BIT(0)
+#define DMA_ECC_INT_STATUS		0x00001088
+
+/* EQoS version 5.xx VLAN Tag Filter Fail Packets Queuing */
+#define GMAC_RXQ_CTRL4			0x00000094
+#define GMAC_RXQCTRL_VFFQ_MASK		GENMASK(19, 17)
+#define GMAC_RXQCTRL_VFFQ_SHIFT		17
+#define GMAC_RXQCTRL_VFFQE		BIT(16)
+
+#define GMAC_INT_FPE_EN			BIT(17)
+
+int ec_dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp,
+			      struct stmmac_safety_feature_cfg *safety_cfg);
+int ec_dwmac5_safety_feat_irq_status(struct net_device *ndev,
+		void __iomem *ioaddr, unsigned int asp,
+		struct stmmac_safety_stats *stats);
+int ec_dwmac5_safety_feat_dump(struct stmmac_safety_stats *stats,
+			int index, unsigned long *count, const char **desc);
+int ec_dwmac5_rxp_config(void __iomem *ioaddr, struct stmmac_tc_entry *entries,
+		      unsigned int count);
+int ec_dwmac5_flex_pps_config(void __iomem *ioaddr, int index,
+			   struct stmmac_pps_cfg *cfg, bool enable,
+			   u32 sub_second_inc, u32 systime_flags);
+
+#endif /* __DWMAC5_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac_dma-ethercat.h b/drivers/net/ethercat/device/K3/dwmac_dma-ethercat.h
new file mode 100644
index 000000000000..83308fd7574c
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac_dma-ethercat.h
@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  DWMAC DMA Header file.
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __DWMAC_DMA_H__
+#define __DWMAC_DMA_H__
+
+/* DMA CRS Control and Status Register Mapping */
+#define DMA_BUS_MODE		0x00001000	/* Bus Mode */
+#define DMA_XMT_POLL_DEMAND	0x00001004	/* Transmit Poll Demand */
+#define DMA_RCV_POLL_DEMAND	0x00001008	/* Received Poll Demand */
+#define DMA_RCV_BASE_ADDR	0x0000100c	/* Receive List Base */
+#define DMA_TX_BASE_ADDR	0x00001010	/* Transmit List Base */
+#define DMA_STATUS		0x00001014	/* Status Register */
+#define DMA_CONTROL		0x00001018	/* Ctrl (Operational Mode) */
+#define DMA_INTR_ENA		0x0000101c	/* Interrupt Enable */
+#define DMA_MISSED_FRAME_CTR	0x00001020	/* Missed Frame Counter */
+
+/* Following DMA defines are channels oriented */
+#define DMA_CHAN_BASE_OFFSET			0x100
+
+static inline u32 dma_chan_base_addr(u32 base, u32 chan)
+{
+	return base + chan * DMA_CHAN_BASE_OFFSET;
+}
+
+#define DMA_CHAN_BUS_MODE(chan)	dma_chan_base_addr(DMA_BUS_MODE, chan)
+#define DMA_CHAN_XMT_POLL_DEMAND(chan)	\
+				dma_chan_base_addr(DMA_XMT_POLL_DEMAND, chan)
+#define DMA_CHAN_RCV_POLL_DEMAND(chan)	\
+				dma_chan_base_addr(DMA_RCV_POLL_DEMAND, chan)
+#define DMA_CHAN_RCV_BASE_ADDR(chan)	\
+				dma_chan_base_addr(DMA_RCV_BASE_ADDR, chan)
+#define DMA_CHAN_TX_BASE_ADDR(chan)	\
+				dma_chan_base_addr(DMA_TX_BASE_ADDR, chan)
+#define DMA_CHAN_STATUS(chan)	dma_chan_base_addr(DMA_STATUS, chan)
+#define DMA_CHAN_CONTROL(chan)	dma_chan_base_addr(DMA_CONTROL, chan)
+#define DMA_CHAN_INTR_ENA(chan)	dma_chan_base_addr(DMA_INTR_ENA, chan)
+#define DMA_CHAN_MISSED_FRAME_CTR(chan)	\
+				dma_chan_base_addr(DMA_MISSED_FRAME_CTR, chan)
+#define DMA_CHAN_RX_WATCHDOG(chan)	\
+				dma_chan_base_addr(DMA_RX_WATCHDOG, chan)
+
+/* SW Reset */
+#define DMA_BUS_MODE_SFT_RESET	0x00000001	/* Software Reset */
+
+/* Rx watchdog register */
+#define DMA_RX_WATCHDOG		0x00001024
+
+/* AXI Master Bus Mode */
+#define DMA_AXI_BUS_MODE	0x00001028
+
+#define DMA_AXI_EN_LPI		BIT(31)
+#define DMA_AXI_LPI_XIT_FRM	BIT(30)
+#define DMA_AXI_WR_OSR_LMT	GENMASK(23, 20)
+#define DMA_AXI_WR_OSR_LMT_SHIFT	20
+#define DMA_AXI_WR_OSR_LMT_MASK	0xf
+#define DMA_AXI_RD_OSR_LMT	GENMASK(19, 16)
+#define DMA_AXI_RD_OSR_LMT_SHIFT	16
+#define DMA_AXI_RD_OSR_LMT_MASK	0xf
+
+#define DMA_AXI_OSR_MAX		0xf
+#define DMA_AXI_MAX_OSR_LIMIT ((DMA_AXI_OSR_MAX << DMA_AXI_WR_OSR_LMT_SHIFT) | \
+			       (DMA_AXI_OSR_MAX << DMA_AXI_RD_OSR_LMT_SHIFT))
+#define	DMA_AXI_1KBBE		BIT(13)
+#define DMA_AXI_AAL		BIT(12)
+#define DMA_AXI_BLEN256		BIT(7)
+#define DMA_AXI_BLEN128		BIT(6)
+#define DMA_AXI_BLEN64		BIT(5)
+#define DMA_AXI_BLEN32		BIT(4)
+#define DMA_AXI_BLEN16		BIT(3)
+#define DMA_AXI_BLEN8		BIT(2)
+#define DMA_AXI_BLEN4		BIT(1)
+#define DMA_BURST_LEN_DEFAULT	(DMA_AXI_BLEN256 | DMA_AXI_BLEN128 | \
+				 DMA_AXI_BLEN64 | DMA_AXI_BLEN32 | \
+				 DMA_AXI_BLEN16 | DMA_AXI_BLEN8 | \
+				 DMA_AXI_BLEN4)
+
+#define DMA_AXI_UNDEF		BIT(0)
+
+#define DMA_AXI_BURST_LEN_MASK	0x000000FE
+
+#define DMA_CUR_TX_BUF_ADDR	0x00001050	/* Current Host Tx Buffer */
+#define DMA_CUR_RX_BUF_ADDR	0x00001054	/* Current Host Rx Buffer */
+#define DMA_HW_FEATURE		0x00001058	/* HW Feature Register */
+
+/* DMA Control register defines */
+#define DMA_CONTROL_ST		0x00002000	/* Start/Stop Transmission */
+#define DMA_CONTROL_SR		0x00000002	/* Start/Stop Receive */
+
+/* DMA Normal interrupt */
+#define DMA_INTR_ENA_NIE 0x00010000	/* Normal Summary */
+#define DMA_INTR_ENA_TIE 0x00000001	/* Transmit Interrupt */
+#define DMA_INTR_ENA_TUE 0x00000004	/* Transmit Buffer Unavailable */
+#define DMA_INTR_ENA_RIE 0x00000040	/* Receive Interrupt */
+#define DMA_INTR_ENA_ERE 0x00004000	/* Early Receive */
+
+#define DMA_INTR_NORMAL	(DMA_INTR_ENA_NIE | DMA_INTR_ENA_RIE | \
+			DMA_INTR_ENA_TIE)
+
+/* DMA Abnormal interrupt */
+#define DMA_INTR_ENA_AIE 0x00008000	/* Abnormal Summary */
+#define DMA_INTR_ENA_FBE 0x00002000	/* Fatal Bus Error */
+#define DMA_INTR_ENA_ETE 0x00000400	/* Early Transmit */
+#define DMA_INTR_ENA_RWE 0x00000200	/* Receive Watchdog */
+#define DMA_INTR_ENA_RSE 0x00000100	/* Receive Stopped */
+#define DMA_INTR_ENA_RUE 0x00000080	/* Receive Buffer Unavailable */
+#define DMA_INTR_ENA_UNE 0x00000020	/* Tx Underflow */
+#define DMA_INTR_ENA_OVE 0x00000010	/* Receive Overflow */
+#define DMA_INTR_ENA_TJE 0x00000008	/* Transmit Jabber */
+#define DMA_INTR_ENA_TSE 0x00000002	/* Transmit Stopped */
+
+#define DMA_INTR_ABNORMAL	(DMA_INTR_ENA_AIE | DMA_INTR_ENA_FBE | \
+				DMA_INTR_ENA_UNE)
+
+/* DMA default interrupt mask */
+#define DMA_INTR_DEFAULT_MASK	(DMA_INTR_NORMAL | DMA_INTR_ABNORMAL)
+#define DMA_INTR_DEFAULT_RX	(DMA_INTR_ENA_RIE)
+#define DMA_INTR_DEFAULT_TX	(DMA_INTR_ENA_TIE)
+
+/* DMA Status register defines */
+#define DMA_STATUS_GLPII	0x40000000	/* GMAC LPI interrupt */
+#define DMA_STATUS_GPI		0x10000000	/* PMT interrupt */
+#define DMA_STATUS_GMI		0x08000000	/* MMC interrupt */
+#define DMA_STATUS_GLI		0x04000000	/* GMAC Line interface int */
+#define DMA_STATUS_EB_MASK	0x00380000	/* Error Bits Mask */
+#define DMA_STATUS_EB_TX_ABORT	0x00080000	/* Error Bits - TX Abort */
+#define DMA_STATUS_EB_RX_ABORT	0x00100000	/* Error Bits - RX Abort */
+#define DMA_STATUS_TS_MASK	0x00700000	/* Transmit Process State */
+#define DMA_STATUS_TS_SHIFT	20
+#define DMA_STATUS_RS_MASK	0x000e0000	/* Receive Process State */
+#define DMA_STATUS_RS_SHIFT	17
+#define DMA_STATUS_NIS	0x00010000	/* Normal Interrupt Summary */
+#define DMA_STATUS_AIS	0x00008000	/* Abnormal Interrupt Summary */
+#define DMA_STATUS_ERI	0x00004000	/* Early Receive Interrupt */
+#define DMA_STATUS_FBI	0x00002000	/* Fatal Bus Error Interrupt */
+#define DMA_STATUS_ETI	0x00000400	/* Early Transmit Interrupt */
+#define DMA_STATUS_RWT	0x00000200	/* Receive Watchdog Timeout */
+#define DMA_STATUS_RPS	0x00000100	/* Receive Process Stopped */
+#define DMA_STATUS_RU	0x00000080	/* Receive Buffer Unavailable */
+#define DMA_STATUS_RI	0x00000040	/* Receive Interrupt */
+#define DMA_STATUS_UNF	0x00000020	/* Transmit Underflow */
+#define DMA_STATUS_OVF	0x00000010	/* Receive Overflow */
+#define DMA_STATUS_TJT	0x00000008	/* Transmit Jabber Timeout */
+#define DMA_STATUS_TU	0x00000004	/* Transmit Buffer Unavailable */
+#define DMA_STATUS_TPS	0x00000002	/* Transmit Process Stopped */
+#define DMA_STATUS_TI	0x00000001	/* Transmit Interrupt */
+#define DMA_CONTROL_FTF		0x00100000	/* Flush transmit FIFO */
+
+#define DMA_STATUS_MSK_COMMON		(DMA_STATUS_NIS | \
+					 DMA_STATUS_AIS | \
+					 DMA_STATUS_FBI)
+
+#define DMA_STATUS_MSK_RX		(DMA_STATUS_ERI | \
+					 DMA_STATUS_RWT | \
+					 DMA_STATUS_RPS | \
+					 DMA_STATUS_RU | \
+					 DMA_STATUS_RI | \
+					 DMA_STATUS_OVF | \
+					 DMA_STATUS_MSK_COMMON)
+
+#define DMA_STATUS_MSK_TX		(DMA_STATUS_ETI | \
+					 DMA_STATUS_UNF | \
+					 DMA_STATUS_TJT | \
+					 DMA_STATUS_TU | \
+					 DMA_STATUS_TPS | \
+					 DMA_STATUS_TI | \
+					 DMA_STATUS_MSK_COMMON)
+
+#define NUM_DWMAC100_DMA_REGS	9
+#define NUM_DWMAC1000_DMA_REGS	23
+#define NUM_DWMAC4_DMA_REGS	27
+
+void ec_dwmac_enable_dma_transmission(void __iomem *ioaddr, u32 chan);
+void ec_dwmac_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  u32 chan, bool rx, bool tx);
+void ec_dwmac_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx);
+void ec_dwmac_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+void ec_dwmac_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+		       u32 chan);
+void ec_dwmac_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+void ec_dwmac_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+		       u32 chan);
+int ec_dwmac_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
+			struct stmmac_extra_stats *x, u32 chan, u32 dir);
+int ec_dwmac_dma_reset(void __iomem *ioaddr);
+
+#endif /* __DWMAC_DMA_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwmac_lib-ethercat.c b/drivers/net/ethercat/device/K3/dwmac_lib-ethercat.c
new file mode 100644
index 000000000000..3adf2af8148e
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwmac_lib-ethercat.c
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include "common-ethercat.h"
+#include "dwmac_dma-ethercat.h"
+#include "stmmac-ethercat.h"
+
+#define GMAC_HI_REG_AE		0x80000000
+
+int ec_dwmac_dma_reset(void __iomem *ioaddr)
+{
+	u32 value = readl(ioaddr + DMA_BUS_MODE);
+
+	/* DMA SW reset */
+	value |= DMA_BUS_MODE_SFT_RESET;
+	writel(value, ioaddr + DMA_BUS_MODE);
+
+	return readl_poll_timeout(ioaddr + DMA_BUS_MODE, value,
+				 !(value & DMA_BUS_MODE_SFT_RESET),
+				 10000, 200000);
+}
+
+/* CSR1 enables the transmit DMA to check for new descriptor */
+void ec_dwmac_enable_dma_transmission(void __iomem *ioaddr, u32 chan)
+{
+	writel(1, ioaddr + DMA_CHAN_XMT_POLL_DEMAND(chan));
+}
+
+void ec_dwmac_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  u32 chan, bool rx, bool tx)
+{
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(chan));
+
+	if (rx)
+		value |= DMA_INTR_DEFAULT_RX;
+	if (tx)
+		value |= DMA_INTR_DEFAULT_TX;
+
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(chan));
+}
+
+void ec_dwmac_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 chan, bool rx, bool tx)
+{
+	u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(chan));
+
+	if (rx)
+		value &= ~DMA_INTR_DEFAULT_RX;
+	if (tx)
+		value &= ~DMA_INTR_DEFAULT_TX;
+
+	writel(value, ioaddr + DMA_CHAN_INTR_ENA(chan));
+}
+
+void ec_dwmac_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
+{
+	u32 value = readl(ioaddr + DMA_CHAN_CONTROL(chan));
+	value |= DMA_CONTROL_ST;
+	writel(value, ioaddr + DMA_CHAN_CONTROL(chan));
+}
+
+void ec_dwmac_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan)
+{
+	u32 value = readl(ioaddr + DMA_CHAN_CONTROL(chan));
+	value &= ~DMA_CONTROL_ST;
+	writel(value, ioaddr + DMA_CHAN_CONTROL(chan));
+}
+
+void ec_dwmac_dma_start_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan)
+{
+	u32 value = readl(ioaddr + DMA_CHAN_CONTROL(chan));
+	value |= DMA_CONTROL_SR;
+	writel(value, ioaddr + DMA_CHAN_CONTROL(chan));
+}
+
+void ec_dwmac_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan)
+{
+	u32 value = readl(ioaddr + DMA_CHAN_CONTROL(chan));
+	value &= ~DMA_CONTROL_SR;
+	writel(value, ioaddr + DMA_CHAN_CONTROL(chan));
+}
+
+#ifdef DWMAC_DMA_DEBUG
+static void show_tx_process_state(unsigned int status)
+{
+	unsigned int state;
+	state = (status & DMA_STATUS_TS_MASK) >> DMA_STATUS_TS_SHIFT;
+
+	switch (state) {
+	case 0:
+		pr_debug("- TX (Stopped): Reset or Stop command\n");
+		break;
+	case 1:
+		pr_debug("- TX (Running): Fetching the Tx desc\n");
+		break;
+	case 2:
+		pr_debug("- TX (Running): Waiting for end of tx\n");
+		break;
+	case 3:
+		pr_debug("- TX (Running): Reading the data "
+		       "and queuing the data into the Tx buf\n");
+		break;
+	case 6:
+		pr_debug("- TX (Suspended): Tx Buff Underflow "
+		       "or an unavailable Transmit descriptor\n");
+		break;
+	case 7:
+		pr_debug("- TX (Running): Closing Tx descriptor\n");
+		break;
+	default:
+		break;
+	}
+}
+
+static void show_rx_process_state(unsigned int status)
+{
+	unsigned int state;
+	state = (status & DMA_STATUS_RS_MASK) >> DMA_STATUS_RS_SHIFT;
+
+	switch (state) {
+	case 0:
+		pr_debug("- RX (Stopped): Reset or Stop command\n");
+		break;
+	case 1:
+		pr_debug("- RX (Running): Fetching the Rx desc\n");
+		break;
+	case 2:
+		pr_debug("- RX (Running): Checking for end of pkt\n");
+		break;
+	case 3:
+		pr_debug("- RX (Running): Waiting for Rx pkt\n");
+		break;
+	case 4:
+		pr_debug("- RX (Suspended): Unavailable Rx buf\n");
+		break;
+	case 5:
+		pr_debug("- RX (Running): Closing Rx descriptor\n");
+		break;
+	case 6:
+		pr_debug("- RX(Running): Flushing the current frame"
+		       " from the Rx buf\n");
+		break;
+	case 7:
+		pr_debug("- RX (Running): Queuing the Rx frame"
+		       " from the Rx buf into memory\n");
+		break;
+	default:
+		break;
+	}
+}
+#endif
+
+int ec_dwmac_dma_interrupt(struct stmmac_priv *priv, void __iomem *ioaddr,
+			struct stmmac_extra_stats *x, u32 chan, u32 dir)
+{
+	struct stmmac_pcpu_stats *stats = this_cpu_ptr(priv->xstats.pcpu_stats);
+	int ret = 0;
+	/* read the status register (CSR5) */
+	u32 intr_status = readl(ioaddr + DMA_CHAN_STATUS(chan));
+
+#ifdef DWMAC_DMA_DEBUG
+	/* Enable it to monitor DMA rx/tx status in case of critical problems */
+	pr_debug("%s: [CSR5: 0x%08x]\n", __func__, intr_status);
+	show_tx_process_state(intr_status);
+	show_rx_process_state(intr_status);
+#endif
+
+	if (dir == DMA_DIR_RX)
+		intr_status &= DMA_STATUS_MSK_RX;
+	else if (dir == DMA_DIR_TX)
+		intr_status &= DMA_STATUS_MSK_TX;
+
+	/* ABNORMAL interrupts */
+	if (unlikely(intr_status & DMA_STATUS_AIS)) {
+		if (unlikely(intr_status & DMA_STATUS_UNF)) {
+			ret = tx_hard_error_bump_tc;
+			x->tx_undeflow_irq++;
+		}
+		if (unlikely(intr_status & DMA_STATUS_TJT))
+			x->tx_jabber_irq++;
+
+		if (unlikely(intr_status & DMA_STATUS_OVF))
+			x->rx_overflow_irq++;
+
+		if (unlikely(intr_status & DMA_STATUS_RU))
+			x->rx_buf_unav_irq++;
+		if (unlikely(intr_status & DMA_STATUS_RPS))
+			x->rx_process_stopped_irq++;
+		if (unlikely(intr_status & DMA_STATUS_RWT))
+			x->rx_watchdog_irq++;
+		if (unlikely(intr_status & DMA_STATUS_ETI))
+			x->tx_early_irq++;
+		if (unlikely(intr_status & DMA_STATUS_TPS)) {
+			x->tx_process_stopped_irq++;
+			ret = tx_hard_error;
+		}
+		if (unlikely(intr_status & DMA_STATUS_FBI)) {
+			x->fatal_bus_error_irq++;
+			ret = tx_hard_error;
+		}
+	}
+	/* TX/RX NORMAL interrupts */
+	if (likely(intr_status & DMA_STATUS_NIS)) {
+		if (likely(intr_status & DMA_STATUS_RI)) {
+			u32 value = readl(ioaddr + DMA_INTR_ENA);
+			/* to schedule NAPI on real RIE event. */
+			if (likely(value & DMA_INTR_ENA_RIE)) {
+				u64_stats_update_begin(&stats->syncp);
+				u64_stats_inc(&stats->rx_normal_irq_n[chan]);
+				u64_stats_update_end(&stats->syncp);
+				ret |= handle_rx;
+			}
+		}
+		if (likely(intr_status & DMA_STATUS_TI)) {
+			u64_stats_update_begin(&stats->syncp);
+			u64_stats_inc(&stats->tx_normal_irq_n[chan]);
+			u64_stats_update_end(&stats->syncp);
+			ret |= handle_tx;
+		}
+		if (unlikely(intr_status & DMA_STATUS_ERI))
+			x->rx_early_irq++;
+	}
+	/* Optional hardware blocks, interrupts should be disabled */
+	if (unlikely(intr_status &
+		     (DMA_STATUS_GPI | DMA_STATUS_GMI | DMA_STATUS_GLI)))
+		pr_warn("%s: unexpected status %08x\n", __func__, intr_status);
+
+	/* Clear the interrupt by writing a logic 1 to the CSR5[15-0] */
+	writel((intr_status & 0x1ffff), ioaddr + DMA_STATUS);
+
+	return ret;
+}
+
+void ec_dwmac_dma_flush_tx_fifo(void __iomem *ioaddr)
+{
+	u32 csr6 = readl(ioaddr + DMA_CONTROL);
+	writel((csr6 | DMA_CONTROL_FTF), ioaddr + DMA_CONTROL);
+
+	do {} while ((readl(ioaddr + DMA_CONTROL) & DMA_CONTROL_FTF));
+}
+
+void ec_stmmac_set_mac_addr(void __iomem *ioaddr, const u8 addr[6],
+			 unsigned int high, unsigned int low)
+{
+	u32 data;
+
+	data = (addr[5] << 8) | addr[4];
+	/* For MAC Addr registers we have to set the Address Enable (AE)
+	 * bit that has no effect on the High Reg 0 where the bit 31 (MO)
+	 * is RO.
+	 */
+	writel(data | GMAC_HI_REG_AE, ioaddr + high);
+	data = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0];
+	writel(data, ioaddr + low);
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_set_mac_addr);
+
+/* Enable disable MAC RX/TX */
+void ec_stmmac_set_mac(void __iomem *ioaddr, bool enable)
+{
+	u32 old_val, value;
+
+	old_val = readl(ioaddr + MAC_CTRL_REG);
+	value = old_val;
+
+	if (enable)
+		value |= MAC_ENABLE_RX | MAC_ENABLE_TX;
+	else
+		value &= ~(MAC_ENABLE_TX | MAC_ENABLE_RX);
+
+	if (value != old_val)
+		writel(value, ioaddr + MAC_CTRL_REG);
+}
+
+void ec_stmmac_get_mac_addr(void __iomem *ioaddr, unsigned char *addr,
+			 unsigned int high, unsigned int low)
+{
+	unsigned int hi_addr, lo_addr;
+
+	/* Read the MAC address from the hardware */
+	hi_addr = readl(ioaddr + high);
+	lo_addr = readl(ioaddr + low);
+
+	/* Extract the MAC address from the high and low words */
+	addr[0] = lo_addr & 0xff;
+	addr[1] = (lo_addr >> 8) & 0xff;
+	addr[2] = (lo_addr >> 16) & 0xff;
+	addr[3] = (lo_addr >> 24) & 0xff;
+	addr[4] = hi_addr & 0xff;
+	addr[5] = (hi_addr >> 8) & 0xff;
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_get_mac_addr);
diff --git a/drivers/net/ethercat/device/K3/dwxgmac2-ethercat.h b/drivers/net/ethercat/device/K3/dwxgmac2-ethercat.h
new file mode 100644
index 000000000000..b825c78fc452
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwxgmac2-ethercat.h
@@ -0,0 +1,493 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac XGMAC definitions.
+ */
+
+#ifndef __STMMAC_DWXGMAC2_H__
+#define __STMMAC_DWXGMAC2_H__
+
+#include "common-ethercat.h"
+
+/* Misc */
+#define XGMAC_JUMBO_LEN			16368
+
+/* MAC Registers */
+#define XGMAC_TX_CONFIG			0x00000000
+#define XGMAC_CONFIG_SS_OFF		29
+#define XGMAC_CONFIG_SS_MASK		GENMASK(31, 29)
+#define XGMAC_CONFIG_SS_10000		(0x0 << XGMAC_CONFIG_SS_OFF)
+#define XGMAC_CONFIG_SS_2500_GMII	(0x2 << XGMAC_CONFIG_SS_OFF)
+#define XGMAC_CONFIG_SS_1000_GMII	(0x3 << XGMAC_CONFIG_SS_OFF)
+#define XGMAC_CONFIG_SS_100_MII		(0x4 << XGMAC_CONFIG_SS_OFF)
+#define XGMAC_CONFIG_SS_5000		(0x5 << XGMAC_CONFIG_SS_OFF)
+#define XGMAC_CONFIG_SS_2500		(0x6 << XGMAC_CONFIG_SS_OFF)
+#define XGMAC_CONFIG_SS_10_MII		(0x7 << XGMAC_CONFIG_SS_OFF)
+#define XGMAC_CONFIG_SARC		GENMASK(22, 20)
+#define XGMAC_CONFIG_SARC_SHIFT		20
+#define XGMAC_CONFIG_JD			BIT(16)
+#define XGMAC_CONFIG_TE			BIT(0)
+#define XGMAC_CORE_INIT_TX		(XGMAC_CONFIG_JD)
+#define XGMAC_RX_CONFIG			0x00000004
+#define XGMAC_CONFIG_ARPEN		BIT(31)
+#define XGMAC_CONFIG_GPSL		GENMASK(29, 16)
+#define XGMAC_CONFIG_GPSL_SHIFT		16
+#define XGMAC_CONFIG_HDSMS		GENMASK(14, 12)
+#define XGMAC_CONFIG_HDSMS_SHIFT	12
+#define XGMAC_CONFIG_HDSMS_256		(0x2 << XGMAC_CONFIG_HDSMS_SHIFT)
+#define XGMAC_CONFIG_S2KP		BIT(11)
+#define XGMAC_CONFIG_LM			BIT(10)
+#define XGMAC_CONFIG_IPC		BIT(9)
+#define XGMAC_CONFIG_JE			BIT(8)
+#define XGMAC_CONFIG_WD			BIT(7)
+#define XGMAC_CONFIG_GPSLCE		BIT(6)
+#define XGMAC_CONFIG_CST		BIT(2)
+#define XGMAC_CONFIG_ACS		BIT(1)
+#define XGMAC_CONFIG_RE			BIT(0)
+#define XGMAC_CORE_INIT_RX		(XGMAC_CONFIG_GPSLCE | XGMAC_CONFIG_WD | \
+					 (XGMAC_JUMBO_LEN << XGMAC_CONFIG_GPSL_SHIFT))
+#define XGMAC_PACKET_FILTER		0x00000008
+#define XGMAC_FILTER_RA			BIT(31)
+#define XGMAC_FILTER_IPFE		BIT(20)
+#define XGMAC_FILTER_VTFE		BIT(16)
+#define XGMAC_FILTER_HPF		BIT(10)
+#define XGMAC_FILTER_PCF		BIT(7)
+#define XGMAC_FILTER_PM			BIT(4)
+#define XGMAC_FILTER_HMC		BIT(2)
+#define XGMAC_FILTER_PR			BIT(0)
+#define XGMAC_HASH_TABLE(x)		(0x00000010 + (x) * 4)
+#define XGMAC_MAX_HASH_TABLE		8
+#define XGMAC_RXQ_CTRL0			0x000000a0
+#define XGMAC_RXQEN(x)			GENMASK((x) * 2 + 1, (x) * 2)
+#define XGMAC_RXQEN_SHIFT(x)		((x) * 2)
+#define XGMAC_RXQ_CTRL1			0x000000a4
+#define XGMAC_AVCPQ			GENMASK(31, 28)
+#define XGMAC_AVCPQ_SHIFT		28
+#define XGMAC_PTPQ			GENMASK(27, 24)
+#define XGMAC_PTPQ_SHIFT		24
+#define XGMAC_TACPQE			BIT(23)
+#define XGMAC_DCBCPQ			GENMASK(19, 16)
+#define XGMAC_DCBCPQ_SHIFT		16
+#define XGMAC_MCBCQEN			BIT(15)
+#define XGMAC_MCBCQ			GENMASK(11, 8)
+#define XGMAC_MCBCQ_SHIFT		8
+#define XGMAC_FPRQ			GENMASK(7, 4)
+#define XGMAC_UPQ			GENMASK(3, 0)
+#define XGMAC_UPQ_SHIFT			0
+#define XGMAC_RXQ_CTRL2			0x000000a8
+#define XGMAC_RXQ_CTRL3			0x000000ac
+#define XGMAC_PSRQ(x)			GENMASK((x) * 8 + 7, (x) * 8)
+#define XGMAC_PSRQ_SHIFT(x)		((x) * 8)
+#define XGMAC_INT_STATUS		0x000000b0
+#define XGMAC_LPIIS			BIT(5)
+#define XGMAC_PMTIS			BIT(4)
+#define XGMAC_INT_EN			0x000000b4
+#define XGMAC_FPEIE			BIT(15)
+#define XGMAC_TSIE			BIT(12)
+#define XGMAC_LPIIE			BIT(5)
+#define XGMAC_PMTIE			BIT(4)
+#define XGMAC_INT_DEFAULT_EN		(XGMAC_LPIIE | XGMAC_PMTIE)
+#define XGMAC_Qx_TX_FLOW_CTRL(x)	(0x00000070 + (x) * 4)
+#define XGMAC_PT			GENMASK(31, 16)
+#define XGMAC_PT_SHIFT			16
+#define XGMAC_TFE			BIT(1)
+#define XGMAC_RX_FLOW_CTRL		0x00000090
+#define XGMAC_RFE			BIT(0)
+#define XGMAC_PMT			0x000000c0
+#define XGMAC_GLBLUCAST			BIT(9)
+#define XGMAC_RWKPKTEN			BIT(2)
+#define XGMAC_MGKPKTEN			BIT(1)
+#define XGMAC_PWRDWN			BIT(0)
+#define XGMAC_LPI_CTRL			0x000000d0
+/* For definitions, see LPI_CTRL_STATUS_xxx in common.h */
+#define XGMAC_LPI_TIMER_CTRL		0x000000d4
+#define XGMAC_HW_FEATURE0		0x0000011c
+#define XGMAC_HWFEAT_EDMA		BIT(31)
+#define XGMAC_HWFEAT_EDIFFC		BIT(30)
+#define XGMAC_HWFEAT_VXN		BIT(29)
+#define XGMAC_HWFEAT_SAVLANINS		BIT(27)
+#define XGMAC_HWFEAT_TSSTSSEL		GENMASK(26, 25)
+#define XGMAC_HWFEAT_ADDMACADRSEL	GENMASK(22, 18)
+#define XGMAC_HWFEAT_RXCOESEL		BIT(16)
+#define XGMAC_HWFEAT_TXCOESEL		BIT(14)
+#define XGMAC_HWFEAT_EEESEL		BIT(13)
+#define XGMAC_HWFEAT_TSSEL		BIT(12)
+#define XGMAC_HWFEAT_AVSEL		BIT(11)
+#define XGMAC_HWFEAT_RAVSEL		BIT(10)
+#define XGMAC_HWFEAT_ARPOFFSEL		BIT(9)
+#define XGMAC_HWFEAT_MMCSEL		BIT(8)
+#define XGMAC_HWFEAT_MGKSEL		BIT(7)
+#define XGMAC_HWFEAT_RWKSEL		BIT(6)
+#define XGMAC_HWFEAT_SMASEL		BIT(5)
+#define XGMAC_HWFEAT_VLHASH		BIT(4)
+#define XGMAC_HWFEAT_HDSEL		BIT(3)
+#define XGMAC_HWFEAT_GMIISEL		BIT(1)
+#define XGMAC_HW_FEATURE1		0x00000120
+#define XGMAC_HWFEAT_L3L4FNUM		GENMASK(30, 27)
+#define XGMAC_HWFEAT_HASHTBLSZ		GENMASK(25, 24)
+#define XGMAC_HWFEAT_NUMTC		GENMASK(23, 21)
+#define XGMAC_HWFEAT_RSSEN		BIT(20)
+#define XGMAC_HWFEAT_DBGMEMA		BIT(19)
+#define XGMAC_HWFEAT_TSOEN		BIT(18)
+#define XGMAC_HWFEAT_SPHEN		BIT(17)
+#define XGMAC_HWFEAT_DCBEN		BIT(16)
+#define XGMAC_HWFEAT_ADDR64		GENMASK(15, 14)
+#define XGMAC_HWFEAT_ADVTHWORD		BIT(13)
+#define XGMAC_HWFEAT_PTOEN		BIT(12)
+#define XGMAC_HWFEAT_OSTEN		BIT(11)
+#define XGMAC_HWFEAT_TXFIFOSIZE		GENMASK(10, 6)
+#define XGMAC_HWFEAT_PFCEN		BIT(5)
+#define XGMAC_HWFEAT_RXFIFOSIZE		GENMASK(4, 0)
+#define XGMAC_HW_FEATURE2		0x00000124
+#define XGMAC_HWFEAT_AUXSNAPNUM		GENMASK(30, 28)
+#define XGMAC_HWFEAT_PPSOUTNUM		GENMASK(26, 24)
+#define XGMAC_HWFEAT_TXCHCNT		GENMASK(21, 18)
+#define XGMAC_HWFEAT_RXCHCNT		GENMASK(15, 12)
+#define XGMAC_HWFEAT_TXQCNT		GENMASK(9, 6)
+#define XGMAC_HWFEAT_RXQCNT		GENMASK(3, 0)
+#define XGMAC_HW_FEATURE3		0x00000128
+#define XGMAC_HWFEAT_TBSCH		GENMASK(31, 28)
+#define XGMAC_HWFEAT_TBSSEL		BIT(27)
+#define XGMAC_HWFEAT_FPESEL		BIT(26)
+#define XGMAC_HWFEAT_SGFSEL		BIT(25)
+#define XGMAC_HWFEAT_ESTWID		GENMASK(24, 23)
+#define XGMAC_HWFEAT_ESTDEP		GENMASK(22, 20)
+#define XGMAC_HWFEAT_ESTSEL		BIT(19)
+#define XGMAC_HWFEAT_TTSFD		GENMASK(18, 16)
+#define XGMAC_HWFEAT_ASP		GENMASK(15, 14)
+#define XGMAC_HWFEAT_DVLAN		BIT(13)
+#define XGMAC_HWFEAT_FRPES		GENMASK(12, 11)
+#define XGMAC_HWFEAT_FRPPB		GENMASK(10, 9)
+#define XGMAC_HWFEAT_POUOST		BIT(8)
+#define XGMAC_HWFEAT_FRPPIPE		GENMASK(7, 5)
+#define XGMAC_HWFEAT_CBTISEL		BIT(4)
+#define XGMAC_HWFEAT_FRPSEL		BIT(3)
+#define XGMAC_HWFEAT_NRVF		GENMASK(2, 0)
+#define XGMAC_HW_FEATURE4		0x0000012c
+#define XGMAC_HWFEAT_EASP		BIT(4)
+#define XGMAC_HWFEAT_PCSEL		GENMASK(1, 0)
+#define XGMAC_MAC_DPP_FSM_INT_STATUS	0x00000150
+#define XGMAC_MAC_FSM_CONTROL		0x00000158
+#define XGMAC_PRTYEN			BIT(1)
+#define XGMAC_TMOUTEN			BIT(0)
+#define XGMAC_MDIO_ADDR			0x00000200
+#define XGMAC_MDIO_DATA			0x00000204
+#define XGMAC_MDIO_C22P			0x00000220
+#define XGMAC_ADDRx_HIGH(x)		(0x00000300 + (x) * 0x8)
+#define XGMAC_ADDR_MAX			32
+#define XGMAC_AE			BIT(31)
+#define XGMAC_DCS			GENMASK(19, 16)
+#define XGMAC_DCS_SHIFT			16
+#define XGMAC_ADDRx_LOW(x)		(0x00000304 + (x) * 0x8)
+#define XGMAC_L3L4_ADDR_CTRL		0x00000c00
+#define XGMAC_IDDR			GENMASK(16, 8)
+#define XGMAC_IDDR_SHIFT		8
+#define XGMAC_IDDR_FNUM			4
+#define XGMAC_TT			BIT(1)
+#define XGMAC_XB			BIT(0)
+#define XGMAC_L3L4_DATA			0x00000c04
+#define XGMAC_L3L4_CTRL			0x0
+#define XGMAC_L4DPIM0			BIT(21)
+#define XGMAC_L4DPM0			BIT(20)
+#define XGMAC_L4SPIM0			BIT(19)
+#define XGMAC_L4SPM0			BIT(18)
+#define XGMAC_L4PEN0			BIT(16)
+#define XGMAC_L3HDBM0			GENMASK(15, 11)
+#define XGMAC_L3HSBM0			GENMASK(10, 6)
+#define XGMAC_L3DAIM0			BIT(5)
+#define XGMAC_L3DAM0			BIT(4)
+#define XGMAC_L3SAIM0			BIT(3)
+#define XGMAC_L3SAM0			BIT(2)
+#define XGMAC_L3PEN0			BIT(0)
+#define XGMAC_L4_ADDR			0x1
+#define XGMAC_L4DP0			GENMASK(31, 16)
+#define XGMAC_L4DP0_SHIFT		16
+#define XGMAC_L4SP0			GENMASK(15, 0)
+#define XGMAC_L3_ADDR0			0x4
+#define XGMAC_L3_ADDR1			0x5
+#define XGMAC_L3_ADDR2			0x6
+#define XMGAC_L3_ADDR3			0x7
+#define XGMAC_ARP_ADDR			0x00000c10
+#define XGMAC_RSS_CTRL			0x00000c80
+#define XGMAC_UDP4TE			BIT(3)
+#define XGMAC_TCP4TE			BIT(2)
+#define XGMAC_IP2TE			BIT(1)
+#define XGMAC_RSSE			BIT(0)
+#define XGMAC_RSS_ADDR			0x00000c88
+#define XGMAC_RSSIA_SHIFT		8
+#define XGMAC_ADDRT			BIT(2)
+#define XGMAC_CT			BIT(1)
+#define XGMAC_OB			BIT(0)
+#define XGMAC_RSS_DATA			0x00000c8c
+#define XGMAC_TIMESTAMP_STATUS		0x00000d20
+#define XGMAC_TXTSC			BIT(15)
+#define XGMAC_TXTIMESTAMP_NSEC		0x00000d30
+#define XGMAC_TXTSSTSLO			GENMASK(30, 0)
+#define XGMAC_TXTIMESTAMP_SEC		0x00000d34
+#define XGMAC_PPS_CONTROL		0x00000d70
+#define XGMAC_PPS_MAXIDX(x)		((((x) + 1) * 8) - 1)
+#define XGMAC_PPS_MINIDX(x)		((x) * 8)
+#define XGMAC_PPSx_MASK(x)		\
+	GENMASK(XGMAC_PPS_MAXIDX(x), XGMAC_PPS_MINIDX(x))
+#define XGMAC_TRGTMODSELx(x, val)	\
+	GENMASK(XGMAC_PPS_MAXIDX(x) - 1, XGMAC_PPS_MAXIDX(x) - 2) & \
+	((val) << (XGMAC_PPS_MAXIDX(x) - 2))
+#define XGMAC_PPSCMDx(x, val)		\
+	GENMASK(XGMAC_PPS_MINIDX(x) + 3, XGMAC_PPS_MINIDX(x)) & \
+	((val) << XGMAC_PPS_MINIDX(x))
+#define XGMAC_PPSCMD_START		0x2
+#define XGMAC_PPSCMD_STOP		0x5
+#define XGMAC_PPSENx(x)			BIT(4 + (x) * 8)
+#define XGMAC_PPSx_TARGET_TIME_SEC(x)	(0x00000d80 + (x) * 0x10)
+#define XGMAC_PPSx_TARGET_TIME_NSEC(x)	(0x00000d84 + (x) * 0x10)
+#define XGMAC_TRGTBUSY0			BIT(31)
+#define XGMAC_PPSx_INTERVAL(x)		(0x00000d88 + (x) * 0x10)
+#define XGMAC_PPSx_WIDTH(x)		(0x00000d8c + (x) * 0x10)
+
+/* MTL Registers */
+#define XGMAC_MTL_OPMODE		0x00001000
+#define XGMAC_FRPE			BIT(15)
+#define XGMAC_ETSALG			GENMASK(6, 5)
+#define XGMAC_WRR			(0x0 << 5)
+#define XGMAC_WFQ			(0x1 << 5)
+#define XGMAC_DWRR			(0x2 << 5)
+#define XGMAC_RAA			BIT(2)
+#define XGMAC_MTL_INT_STATUS		0x00001020
+#define XGMAC_MTL_RXQ_DMA_MAP0		0x00001030
+#define XGMAC_MTL_RXQ_DMA_MAP1		0x00001034
+#define XGMAC_QxMDMACH(x)		GENMASK((x) * 8 + 7, (x) * 8)
+#define XGMAC_QxMDMACH_SHIFT(x)		((x) * 8)
+#define XGMAC_QDDMACH			BIT(7)
+#define XGMAC_TC_PRTY_MAP0		0x00001040
+#define XGMAC_TC_PRTY_MAP1		0x00001044
+#define XGMAC_PSTC(x)			GENMASK((x) * 8 + 7, (x) * 8)
+#define XGMAC_PSTC_SHIFT(x)		((x) * 8)
+#define XGMAC_MTL_RXP_CONTROL_STATUS	0x000010a0
+#define XGMAC_RXPI			BIT(31)
+#define XGMAC_NPE			GENMASK(23, 16)
+#define XGMAC_NVE			GENMASK(7, 0)
+#define XGMAC_MTL_RXP_IACC_CTRL_ST	0x000010b0
+#define XGMAC_STARTBUSY			BIT(31)
+#define XGMAC_WRRDN			BIT(16)
+#define XGMAC_ADDR			GENMASK(9, 0)
+#define XGMAC_MTL_RXP_IACC_DATA		0x000010b4
+#define XGMAC_MTL_ECC_CONTROL		0x000010c0
+#define XGMAC_MTL_SAFETY_INT_STATUS	0x000010c4
+#define XGMAC_MEUIS			BIT(1)
+#define XGMAC_MECIS			BIT(0)
+#define XGMAC_MTL_ECC_INT_ENABLE	0x000010c8
+#define XGMAC_RPCEIE			BIT(12)
+#define XGMAC_ECEIE			BIT(8)
+#define XGMAC_RXCEIE			BIT(4)
+#define XGMAC_TXCEIE			BIT(0)
+#define XGMAC_MTL_ECC_INT_STATUS	0x000010cc
+#define XGMAC_MTL_DPP_CONTROL		0x000010e0
+#define XGMAC_DPP_DISABLE		BIT(0)
+#define XGMAC_MTL_TXQ_OPMODE(x)		(0x00001100 + (0x80 * (x)))
+#define XGMAC_TQS			GENMASK(25, 16)
+#define XGMAC_TQS_SHIFT			16
+#define XGMAC_Q2TCMAP			GENMASK(10, 8)
+#define XGMAC_Q2TCMAP_SHIFT		8
+#define XGMAC_TTC			GENMASK(6, 4)
+#define XGMAC_TTC_SHIFT			4
+#define XGMAC_TXQEN			GENMASK(3, 2)
+#define XGMAC_TXQEN_SHIFT		2
+#define XGMAC_TSF			BIT(1)
+#define XGMAC_MTL_TCx_ETS_CONTROL(x)	(0x00001110 + (0x80 * (x)))
+#define XGMAC_MTL_TCx_QUANTUM_WEIGHT(x)	(0x00001118 + (0x80 * (x)))
+#define XGMAC_MTL_TCx_SENDSLOPE(x)	(0x0000111c + (0x80 * (x)))
+#define XGMAC_MTL_TCx_HICREDIT(x)	(0x00001120 + (0x80 * (x)))
+#define XGMAC_MTL_TCx_LOCREDIT(x)	(0x00001124 + (0x80 * (x)))
+#define XGMAC_CC			BIT(3)
+#define XGMAC_TSA			GENMASK(1, 0)
+#define XGMAC_SP			(0x0 << 0)
+#define XGMAC_CBS			(0x1 << 0)
+#define XGMAC_ETS			(0x2 << 0)
+#define XGMAC_MTL_RXQ_OPMODE(x)		(0x00001140 + (0x80 * (x)))
+#define XGMAC_RQS			GENMASK(25, 16)
+#define XGMAC_RQS_SHIFT			16
+#define XGMAC_EHFC			BIT(7)
+#define XGMAC_RSF			BIT(5)
+#define XGMAC_RTC			GENMASK(1, 0)
+#define XGMAC_RTC_SHIFT			0
+#define XGMAC_MTL_RXQ_FLOW_CONTROL(x)	(0x00001150 + (0x80 * (x)))
+#define XGMAC_RFD			GENMASK(31, 17)
+#define XGMAC_RFD_SHIFT			17
+#define XGMAC_RFA			GENMASK(15, 1)
+#define XGMAC_RFA_SHIFT			1
+#define XGMAC_MTL_QINTEN(x)		(0x00001170 + (0x80 * (x)))
+#define XGMAC_RXOIE			BIT(16)
+#define XGMAC_MTL_QINT_STATUS(x)	(0x00001174 + (0x80 * (x)))
+#define XGMAC_RXOVFIS			BIT(16)
+#define XGMAC_ABPSIS			BIT(1)
+#define XGMAC_TXUNFIS			BIT(0)
+#define XGMAC_MAC_REGSIZE		(XGMAC_MTL_QINT_STATUS(15) / 4)
+
+/* DMA Registers */
+#define XGMAC_DMA_MODE			0x00003000
+#define XGMAC_SWR			BIT(0)
+#define XGMAC_DMA_SYSBUS_MODE		0x00003004
+#define XGMAC_WR_OSR_LMT		GENMASK(29, 24)
+#define XGMAC_WR_OSR_LMT_SHIFT		24
+#define XGMAC_RD_OSR_LMT		GENMASK(21, 16)
+#define XGMAC_RD_OSR_LMT_SHIFT		16
+#define XGMAC_EN_LPI			BIT(15)
+#define XGMAC_LPI_XIT_PKT		BIT(14)
+#define XGMAC_AAL			BIT(12)
+#define XGMAC_EAME			BIT(11)
+#define XGMAC_BLEN			GENMASK(7, 1)
+#define XGMAC_BLEN256			BIT(7)
+#define XGMAC_BLEN128			BIT(6)
+#define XGMAC_BLEN64			BIT(5)
+#define XGMAC_BLEN32			BIT(4)
+#define XGMAC_BLEN16			BIT(3)
+#define XGMAC_BLEN8			BIT(2)
+#define XGMAC_BLEN4			BIT(1)
+#define XGMAC_UNDEF			BIT(0)
+#define XGMAC_TX_EDMA_CTRL		0x00003040
+#define XGMAC_TDPS			GENMASK(29, 0)
+#define XGMAC_RX_EDMA_CTRL		0x00003044
+#define XGMAC_RDPS			GENMASK(29, 0)
+#define XGMAC_DMA_TBS_CTRL0		0x00003054
+#define XGMAC_DMA_TBS_CTRL1		0x00003058
+#define XGMAC_DMA_TBS_CTRL2		0x0000305c
+#define XGMAC_DMA_TBS_CTRL3		0x00003060
+#define XGMAC_FTOS			GENMASK(31, 8)
+#define XGMAC_FTOV			BIT(0)
+#define XGMAC_DEF_FTOS			(XGMAC_FTOS | XGMAC_FTOV)
+#define XGMAC_DMA_SAFETY_INT_STATUS	0x00003064
+#define XGMAC_MCSIS			BIT(31)
+#define XGMAC_MSUIS			BIT(29)
+#define XGMAC_MSCIS			BIT(28)
+#define XGMAC_DEUIS			BIT(1)
+#define XGMAC_DECIS			BIT(0)
+#define XGMAC_DMA_ECC_INT_ENABLE	0x00003068
+#define XGMAC_DCEIE			BIT(1)
+#define XGMAC_TCEIE			BIT(0)
+#define XGMAC_DMA_ECC_INT_STATUS	0x0000306c
+#define XGMAC_DMA_DPP_INT_STATUS	0x00003074
+#define XGMAC_DMA_CH_CONTROL(x)		(0x00003100 + (0x80 * (x)))
+#define XGMAC_SPH			BIT(24)
+#define XGMAC_PBLx8			BIT(16)
+#define XGMAC_DMA_CH_TX_CONTROL(x)	(0x00003104 + (0x80 * (x)))
+#define XGMAC_EDSE			BIT(28)
+#define XGMAC_TxPBL			GENMASK(21, 16)
+#define XGMAC_TxPBL_SHIFT		16
+#define XGMAC_TSE			BIT(12)
+#define XGMAC_OSP			BIT(4)
+#define XGMAC_TXST			BIT(0)
+#define XGMAC_DMA_CH_RX_CONTROL(x)	(0x00003108 + (0x80 * (x)))
+#define XGMAC_RxPBL			GENMASK(21, 16)
+#define XGMAC_RxPBL_SHIFT		16
+#define XGMAC_RBSZ			GENMASK(14, 1)
+#define XGMAC_RBSZ_SHIFT		1
+#define XGMAC_RXST			BIT(0)
+#define XGMAC_DMA_CH_TxDESC_HADDR(x)	(0x00003110 + (0x80 * (x)))
+#define XGMAC_DMA_CH_TxDESC_LADDR(x)	(0x00003114 + (0x80 * (x)))
+#define XGMAC_DMA_CH_RxDESC_HADDR(x)	(0x00003118 + (0x80 * (x)))
+#define XGMAC_DMA_CH_RxDESC_LADDR(x)	(0x0000311c + (0x80 * (x)))
+#define XGMAC_DMA_CH_TxDESC_TAIL_LPTR(x)	(0x00003124 + (0x80 * (x)))
+#define XGMAC_DMA_CH_RxDESC_TAIL_LPTR(x)	(0x0000312c + (0x80 * (x)))
+#define XGMAC_DMA_CH_TxDESC_RING_LEN(x)		(0x00003130 + (0x80 * (x)))
+#define XGMAC_DMA_CH_RxDESC_RING_LEN(x)		(0x00003134 + (0x80 * (x)))
+#define XGMAC_DMA_CH_INT_EN(x)		(0x00003138 + (0x80 * (x)))
+#define XGMAC_NIE			BIT(15)
+#define XGMAC_AIE			BIT(14)
+#define XGMAC_RBUE			BIT(7)
+#define XGMAC_RIE			BIT(6)
+#define XGMAC_TBUE			BIT(2)
+#define XGMAC_TIE			BIT(0)
+#define XGMAC_DMA_INT_DEFAULT_EN	(XGMAC_NIE | XGMAC_AIE | XGMAC_RBUE | \
+					XGMAC_RIE | XGMAC_TIE)
+#define XGMAC_DMA_INT_DEFAULT_RX	(XGMAC_RBUE | XGMAC_RIE)
+#define XGMAC_DMA_INT_DEFAULT_TX	(XGMAC_TIE)
+#define XGMAC_DMA_CH_Rx_WATCHDOG(x)	(0x0000313c + (0x80 * (x)))
+#define XGMAC_RWT			GENMASK(7, 0)
+#define XGMAC_DMA_CH_STATUS(x)		(0x00003160 + (0x80 * (x)))
+#define XGMAC_NIS			BIT(15)
+#define XGMAC_AIS			BIT(14)
+#define XGMAC_FBE			BIT(12)
+#define XGMAC_RBU			BIT(7)
+#define XGMAC_RI			BIT(6)
+#define XGMAC_TBU			BIT(2)
+#define XGMAC_TPS			BIT(1)
+#define XGMAC_TI			BIT(0)
+#define XGMAC_REGSIZE			((0x0000317c + (0x80 * 15)) / 4)
+
+#define XGMAC_DMA_STATUS_MSK_COMMON	(XGMAC_NIS | XGMAC_AIS | XGMAC_FBE)
+#define XGMAC_DMA_STATUS_MSK_RX		(XGMAC_RBU | XGMAC_RI | \
+					 XGMAC_DMA_STATUS_MSK_COMMON)
+#define XGMAC_DMA_STATUS_MSK_TX		(XGMAC_TBU | XGMAC_TPS | XGMAC_TI | \
+					 XGMAC_DMA_STATUS_MSK_COMMON)
+
+/* Descriptors */
+#define XGMAC_TDES0_LTV			BIT(31)
+#define XGMAC_TDES0_LT			GENMASK(7, 0)
+#define XGMAC_TDES1_LT			GENMASK(31, 8)
+#define XGMAC_TDES2_IVT			GENMASK(31, 16)
+#define XGMAC_TDES2_IVT_SHIFT		16
+#define XGMAC_TDES2_IOC			BIT(31)
+#define XGMAC_TDES2_TTSE		BIT(30)
+#define XGMAC_TDES2_B2L			GENMASK(29, 16)
+#define XGMAC_TDES2_B2L_SHIFT		16
+#define XGMAC_TDES2_VTIR		GENMASK(15, 14)
+#define XGMAC_TDES2_VTIR_SHIFT		14
+#define XGMAC_TDES2_B1L			GENMASK(13, 0)
+#define XGMAC_TDES3_OWN			BIT(31)
+#define XGMAC_TDES3_CTXT		BIT(30)
+#define XGMAC_TDES3_FD			BIT(29)
+#define XGMAC_TDES3_LD			BIT(28)
+#define XGMAC_TDES3_CPC			GENMASK(27, 26)
+#define XGMAC_TDES3_CPC_SHIFT		26
+#define XGMAC_TDES3_TCMSSV		BIT(26)
+#define XGMAC_TDES3_SAIC		GENMASK(25, 23)
+#define XGMAC_TDES3_SAIC_SHIFT		23
+#define XGMAC_TDES3_TBSV		BIT(24)
+#define XGMAC_TDES3_THL			GENMASK(22, 19)
+#define XGMAC_TDES3_THL_SHIFT		19
+#define XGMAC_TDES3_IVTIR		GENMASK(19, 18)
+#define XGMAC_TDES3_IVTIR_SHIFT		18
+#define XGMAC_TDES3_TSE			BIT(18)
+#define XGMAC_TDES3_IVLTV		BIT(17)
+#define XGMAC_TDES3_CIC			GENMASK(17, 16)
+#define XGMAC_TDES3_CIC_SHIFT		16
+#define XGMAC_TDES3_TPL			GENMASK(17, 0)
+#define XGMAC_TDES3_VLTV		BIT(16)
+#define XGMAC_TDES3_VT			GENMASK(15, 0)
+#define XGMAC_TDES3_FL			GENMASK(14, 0)
+#define XGMAC_RDES2_HL			GENMASK(9, 0)
+#define XGMAC_RDES3_OWN			BIT(31)
+#define XGMAC_RDES3_CTXT		BIT(30)
+#define XGMAC_RDES3_IOC			BIT(30)
+#define XGMAC_RDES3_LD			BIT(28)
+#define XGMAC_RDES3_CDA			BIT(27)
+#define XGMAC_RDES3_RSV			BIT(26)
+#define XGMAC_RDES3_L34T		GENMASK(23, 20)
+#define XGMAC_RDES3_L34T_SHIFT		20
+#define XGMAC_RDES3_ET_LT		GENMASK(19, 16)
+#define XGMAC_L34T_IP4TCP		0x1
+#define XGMAC_L34T_IP4UDP		0x2
+#define XGMAC_L34T_IP6TCP		0x9
+#define XGMAC_L34T_IP6UDP		0xA
+#define XGMAC_RDES3_ES			BIT(15)
+#define XGMAC_RDES3_PL			GENMASK(13, 0)
+#define XGMAC_RDES3_TSD			BIT(6)
+#define XGMAC_RDES3_TSA			BIT(4)
+
+/* RDES0 (write back format) */
+#define XGMAC_RDES0_VLAN_TAG_MASK	GENMASK(15, 0)
+
+/* Error Type or L2 Type(ET/LT) Field Number */
+#define XGMAC_ET_LT_VLAN_STAG		8
+#define XGMAC_ET_LT_VLAN_CTAG		9
+#define XGMAC_ET_LT_DVLAN_CTAG_CTAG	10
+#define XGMAC_ET_LT_DVLAN_STAG_STAG	11
+#define XGMAC_ET_LT_DVLAN_CTAG_STAG	12
+#define XGMAC_ET_LT_DVLAN_STAG_CTAG	13
+
+extern const struct stmmac_ops ec_dwxgmac210_ops;
+extern const struct stmmac_ops ec_dwxlgmac2_ops;
+extern const struct stmmac_dma_ops ec_dwxgmac210_dma_ops;
+extern const struct stmmac_desc_ops ec_dwxgmac210_desc_ops;
+
+#endif /* __STMMAC_DWXGMAC2_H__ */
diff --git a/drivers/net/ethercat/device/K3/dwxgmac2_core-ethercat.c b/drivers/net/ethercat/device/K3/dwxgmac2_core-ethercat.c
new file mode 100644
index 000000000000..40e5ec6ac43a
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwxgmac2_core-ethercat.c
@@ -0,0 +1,1609 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac XGMAC support.
+ */
+
+#include <linux/bitrev.h>
+#include <linux/crc32.h>
+#include <linux/iopoll.h>
+#include "stmmac-ethercat.h"
+#include "stmmac_fpe-ethercat.h"
+#include "stmmac_ptp-ethercat.h"
+#include "stmmac_vlan-ethercat.h"
+#include "dwxlgmac2-ethercat.h"
+#include "dwxgmac2-ethercat.h"
+
+static void dwxgmac2_core_init(struct mac_device_info *hw,
+			       struct net_device *dev)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 tx, rx;
+
+	tx = readl(ioaddr + XGMAC_TX_CONFIG);
+	rx = readl(ioaddr + XGMAC_RX_CONFIG);
+
+	tx |= XGMAC_CORE_INIT_TX;
+	rx |= XGMAC_CORE_INIT_RX;
+
+	if (hw->ps) {
+		tx |= XGMAC_CONFIG_TE;
+		tx &= ~hw->link.speed_mask;
+
+		switch (hw->ps) {
+		case SPEED_10000:
+			tx |= hw->link.xgmii.speed10000;
+			break;
+		case SPEED_2500:
+			tx |= hw->link.speed2500;
+			break;
+		case SPEED_1000:
+		default:
+			tx |= hw->link.speed1000;
+			break;
+		}
+	}
+
+	writel(tx, ioaddr + XGMAC_TX_CONFIG);
+	writel(rx, ioaddr + XGMAC_RX_CONFIG);
+	writel(XGMAC_INT_DEFAULT_EN, ioaddr + XGMAC_INT_EN);
+}
+
+static void dwxgmac2_update_caps(struct stmmac_priv *priv)
+{
+	if (!priv->dma_cap.mbps_10_100)
+		priv->hw->link.caps &= ~(MAC_10 | MAC_100);
+	else if (!priv->dma_cap.half_duplex)
+		priv->hw->link.caps &= ~(MAC_10HD | MAC_100HD);
+}
+
+static void dwxgmac2_set_mac(void __iomem *ioaddr, bool enable)
+{
+	u32 tx = readl(ioaddr + XGMAC_TX_CONFIG);
+	u32 rx = readl(ioaddr + XGMAC_RX_CONFIG);
+
+	if (enable) {
+		tx |= XGMAC_CONFIG_TE;
+		rx |= XGMAC_CONFIG_RE;
+	} else {
+		tx &= ~XGMAC_CONFIG_TE;
+		rx &= ~XGMAC_CONFIG_RE;
+	}
+
+	writel(tx, ioaddr + XGMAC_TX_CONFIG);
+	writel(rx, ioaddr + XGMAC_RX_CONFIG);
+}
+
+static int dwxgmac2_rx_ipc(struct mac_device_info *hw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_RX_CONFIG);
+	if (hw->rx_csum)
+		value |= XGMAC_CONFIG_IPC;
+	else
+		value &= ~XGMAC_CONFIG_IPC;
+	writel(value, ioaddr + XGMAC_RX_CONFIG);
+
+	return !!(readl(ioaddr + XGMAC_RX_CONFIG) & XGMAC_CONFIG_IPC);
+}
+
+static void dwxgmac2_rx_queue_enable(struct mac_device_info *hw, u8 mode,
+				     u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_RXQ_CTRL0) & ~XGMAC_RXQEN(queue);
+	if (mode == MTL_QUEUE_AVB)
+		value |= 0x1 << XGMAC_RXQEN_SHIFT(queue);
+	else if (mode == MTL_QUEUE_DCB)
+		value |= 0x2 << XGMAC_RXQEN_SHIFT(queue);
+	writel(value, ioaddr + XGMAC_RXQ_CTRL0);
+}
+
+static void dwxgmac2_rx_queue_prio(struct mac_device_info *hw, u32 prio,
+				   u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 clear_mask = 0;
+	u32 ctrl2, ctrl3;
+	int i;
+
+	ctrl2 = readl(ioaddr + XGMAC_RXQ_CTRL2);
+	ctrl3 = readl(ioaddr + XGMAC_RXQ_CTRL3);
+
+	/* The software must ensure that the same priority
+	 * is not mapped to multiple Rx queues
+	 */
+	for (i = 0; i < 4; i++)
+		clear_mask |= ((prio << XGMAC_PSRQ_SHIFT(i)) &
+						XGMAC_PSRQ(i));
+
+	ctrl2 &= ~clear_mask;
+	ctrl3 &= ~clear_mask;
+
+	/* First assign new priorities to a queue, then
+	 * clear them from others queues
+	 */
+	if (queue < 4) {
+		ctrl2 |= (prio << XGMAC_PSRQ_SHIFT(queue)) &
+						XGMAC_PSRQ(queue);
+
+		writel(ctrl2, ioaddr + XGMAC_RXQ_CTRL2);
+		writel(ctrl3, ioaddr + XGMAC_RXQ_CTRL3);
+	} else {
+		queue -= 4;
+
+		ctrl3 |= (prio << XGMAC_PSRQ_SHIFT(queue)) &
+						XGMAC_PSRQ(queue);
+
+		writel(ctrl3, ioaddr + XGMAC_RXQ_CTRL3);
+		writel(ctrl2, ioaddr + XGMAC_RXQ_CTRL2);
+	}
+}
+
+static void dwxgmac2_tx_queue_prio(struct mac_device_info *hw, u32 prio,
+				   u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value, reg;
+
+	reg = (queue < 4) ? XGMAC_TC_PRTY_MAP0 : XGMAC_TC_PRTY_MAP1;
+	if (queue >= 4)
+		queue -= 4;
+
+	value = readl(ioaddr + reg);
+	value &= ~XGMAC_PSTC(queue);
+	value |= (prio << XGMAC_PSTC_SHIFT(queue)) & XGMAC_PSTC(queue);
+
+	writel(value, ioaddr + reg);
+}
+
+static void dwxgmac2_rx_queue_routing(struct mac_device_info *hw,
+				      u8 packet, u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	static const struct stmmac_rx_routing dwxgmac2_route_possibilities[] = {
+		{ XGMAC_AVCPQ, XGMAC_AVCPQ_SHIFT },
+		{ XGMAC_PTPQ, XGMAC_PTPQ_SHIFT },
+		{ XGMAC_DCBCPQ, XGMAC_DCBCPQ_SHIFT },
+		{ XGMAC_UPQ, XGMAC_UPQ_SHIFT },
+		{ XGMAC_MCBCQ, XGMAC_MCBCQ_SHIFT },
+	};
+
+	value = readl(ioaddr + XGMAC_RXQ_CTRL1);
+
+	/* routing configuration */
+	value &= ~dwxgmac2_route_possibilities[packet - 1].reg_mask;
+	value |= (queue << dwxgmac2_route_possibilities[packet - 1].reg_shift) &
+		 dwxgmac2_route_possibilities[packet - 1].reg_mask;
+
+	/* some packets require extra ops */
+	if (packet == PACKET_AVCPQ)
+		value |= FIELD_PREP(XGMAC_TACPQE, 1);
+	else if (packet == PACKET_MCBCQ)
+		value |= FIELD_PREP(XGMAC_MCBCQEN, 1);
+
+	writel(value, ioaddr + XGMAC_RXQ_CTRL1);
+}
+
+static void dwxgmac2_prog_mtl_rx_algorithms(struct mac_device_info *hw,
+					    u32 rx_alg)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_MTL_OPMODE);
+	value &= ~XGMAC_RAA;
+
+	switch (rx_alg) {
+	case MTL_RX_ALGORITHM_SP:
+		break;
+	case MTL_RX_ALGORITHM_WSP:
+		value |= XGMAC_RAA;
+		break;
+	default:
+		break;
+	}
+
+	writel(value, ioaddr + XGMAC_MTL_OPMODE);
+}
+
+static void dwxgmac2_prog_mtl_tx_algorithms(struct mac_device_info *hw,
+					    u32 tx_alg)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	bool ets = true;
+	u32 value;
+	int i;
+
+	value = readl(ioaddr + XGMAC_MTL_OPMODE);
+	value &= ~XGMAC_ETSALG;
+
+	switch (tx_alg) {
+	case MTL_TX_ALGORITHM_WRR:
+		value |= XGMAC_WRR;
+		break;
+	case MTL_TX_ALGORITHM_WFQ:
+		value |= XGMAC_WFQ;
+		break;
+	case MTL_TX_ALGORITHM_DWRR:
+		value |= XGMAC_DWRR;
+		break;
+	default:
+		ets = false;
+		break;
+	}
+
+	writel(value, ioaddr + XGMAC_MTL_OPMODE);
+
+	/* Set ETS if desired */
+	for (i = 0; i < MTL_MAX_TX_QUEUES; i++) {
+		value = readl(ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(i));
+		value &= ~XGMAC_TSA;
+		if (ets)
+			value |= XGMAC_ETS;
+		writel(value, ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(i));
+	}
+}
+
+static void dwxgmac2_set_mtl_tx_queue_weight(struct stmmac_priv *priv,
+					     struct mac_device_info *hw,
+					     u32 weight, u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+
+	writel(weight, ioaddr + XGMAC_MTL_TCx_QUANTUM_WEIGHT(queue));
+}
+
+static void dwxgmac2_map_mtl_to_dma(struct mac_device_info *hw, u32 queue,
+				    u32 chan)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value, reg;
+
+	reg = (queue < 4) ? XGMAC_MTL_RXQ_DMA_MAP0 : XGMAC_MTL_RXQ_DMA_MAP1;
+	if (queue >= 4)
+		queue -= 4;
+
+	value = readl(ioaddr + reg);
+	value &= ~XGMAC_QxMDMACH(queue);
+	value |= (chan << XGMAC_QxMDMACH_SHIFT(queue)) & XGMAC_QxMDMACH(queue);
+
+	writel(value, ioaddr + reg);
+}
+
+static void dwxgmac2_config_cbs(struct stmmac_priv *priv,
+				struct mac_device_info *hw,
+				u32 send_slope, u32 idle_slope,
+				u32 high_credit, u32 low_credit, u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	writel(send_slope, ioaddr + XGMAC_MTL_TCx_SENDSLOPE(queue));
+	writel(idle_slope, ioaddr + XGMAC_MTL_TCx_QUANTUM_WEIGHT(queue));
+	writel(high_credit, ioaddr + XGMAC_MTL_TCx_HICREDIT(queue));
+	writel(low_credit, ioaddr + XGMAC_MTL_TCx_LOCREDIT(queue));
+
+	value = readl(ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(queue));
+	value &= ~XGMAC_TSA;
+	value |= XGMAC_CC | XGMAC_CBS;
+	writel(value, ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(queue));
+}
+
+static void dwxgmac2_dump_regs(struct mac_device_info *hw, u32 *reg_space)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	int i;
+
+	for (i = 0; i < XGMAC_MAC_REGSIZE; i++)
+		reg_space[i] = readl(ioaddr + i * 4);
+}
+
+static int dwxgmac2_host_irq_status(struct mac_device_info *hw,
+				    struct stmmac_extra_stats *x)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 stat, en;
+	int ret = 0;
+
+	en = readl(ioaddr + XGMAC_INT_EN);
+	stat = readl(ioaddr + XGMAC_INT_STATUS);
+
+	stat &= en;
+
+	if (stat & XGMAC_PMTIS) {
+		x->irq_receive_pmt_irq_n++;
+		readl(ioaddr + XGMAC_PMT);
+	}
+
+	if (stat & XGMAC_LPIIS) {
+		u32 lpi = readl(ioaddr + XGMAC_LPI_CTRL);
+
+		if (lpi & LPI_CTRL_STATUS_TLPIEN) {
+			ret |= CORE_IRQ_TX_PATH_IN_LPI_MODE;
+			x->irq_tx_path_in_lpi_mode_n++;
+		}
+		if (lpi & LPI_CTRL_STATUS_TLPIEX) {
+			ret |= CORE_IRQ_TX_PATH_EXIT_LPI_MODE;
+			x->irq_tx_path_exit_lpi_mode_n++;
+		}
+		if (lpi & LPI_CTRL_STATUS_RLPIEN)
+			x->irq_rx_path_in_lpi_mode_n++;
+		if (lpi & LPI_CTRL_STATUS_RLPIEX)
+			x->irq_rx_path_exit_lpi_mode_n++;
+	}
+
+	return ret;
+}
+
+static int dwxgmac2_host_mtl_irq_status(struct stmmac_priv *priv,
+					struct mac_device_info *hw, u32 chan)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	int ret = 0;
+	u32 status;
+
+	status = readl(ioaddr + XGMAC_MTL_INT_STATUS);
+	if (status & BIT(chan)) {
+		u32 chan_status = readl(ioaddr + XGMAC_MTL_QINT_STATUS(chan));
+
+		if (chan_status & XGMAC_RXOVFIS)
+			ret |= CORE_IRQ_MTL_RX_OVERFLOW;
+
+		writel(~0x0, ioaddr + XGMAC_MTL_QINT_STATUS(chan));
+	}
+
+	return ret;
+}
+
+static void dwxgmac2_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
+			       unsigned int fc, unsigned int pause_time,
+			       u32 tx_cnt)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 i;
+
+	if (fc & FLOW_RX)
+		writel(XGMAC_RFE, ioaddr + XGMAC_RX_FLOW_CTRL);
+	if (fc & FLOW_TX) {
+		for (i = 0; i < tx_cnt; i++) {
+			u32 value = XGMAC_TFE;
+
+			if (duplex)
+				value |= pause_time << XGMAC_PT_SHIFT;
+
+			writel(value, ioaddr + XGMAC_Qx_TX_FLOW_CTRL(i));
+		}
+	}
+}
+
+static void dwxgmac2_pmt(struct mac_device_info *hw, unsigned long mode)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 val = 0x0;
+
+	if (mode & WAKE_MAGIC)
+		val |= XGMAC_PWRDWN | XGMAC_MGKPKTEN;
+	if (mode & WAKE_UCAST)
+		val |= XGMAC_PWRDWN | XGMAC_GLBLUCAST | XGMAC_RWKPKTEN;
+	if (val) {
+		u32 cfg = readl(ioaddr + XGMAC_RX_CONFIG);
+		cfg |= XGMAC_CONFIG_RE;
+		writel(cfg, ioaddr + XGMAC_RX_CONFIG);
+	}
+
+	writel(val, ioaddr + XGMAC_PMT);
+}
+
+static void dwxgmac2_set_umac_addr(struct mac_device_info *hw,
+				   const unsigned char *addr,
+				   unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = (addr[5] << 8) | addr[4];
+	writel(value | XGMAC_AE, ioaddr + XGMAC_ADDRx_HIGH(reg_n));
+
+	value = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0];
+	writel(value, ioaddr + XGMAC_ADDRx_LOW(reg_n));
+}
+
+static void dwxgmac2_get_umac_addr(struct mac_device_info *hw,
+				   unsigned char *addr, unsigned int reg_n)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 hi_addr, lo_addr;
+
+	/* Read the MAC address from the hardware */
+	hi_addr = readl(ioaddr + XGMAC_ADDRx_HIGH(reg_n));
+	lo_addr = readl(ioaddr + XGMAC_ADDRx_LOW(reg_n));
+
+	/* Extract the MAC address from the high and low words */
+	addr[0] = lo_addr & 0xff;
+	addr[1] = (lo_addr >> 8) & 0xff;
+	addr[2] = (lo_addr >> 16) & 0xff;
+	addr[3] = (lo_addr >> 24) & 0xff;
+	addr[4] = hi_addr & 0xff;
+	addr[5] = (hi_addr >> 8) & 0xff;
+}
+
+static int dwxgmac2_set_lpi_mode(struct mac_device_info *hw,
+				 enum stmmac_lpi_mode mode,
+				 bool en_tx_lpi_clockgating, u32 et)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	if (mode == STMMAC_LPI_TIMER)
+		return -EOPNOTSUPP;
+
+	value = readl(ioaddr + XGMAC_LPI_CTRL);
+	if (mode == STMMAC_LPI_FORCED) {
+		value |= LPI_CTRL_STATUS_LPIEN | LPI_CTRL_STATUS_LPITXA;
+		if (en_tx_lpi_clockgating)
+			value |= LPI_CTRL_STATUS_LPITCSE;
+	} else {
+		value &= ~(LPI_CTRL_STATUS_LPIEN | LPI_CTRL_STATUS_LPITXA |
+			   LPI_CTRL_STATUS_LPITCSE);
+	}
+	writel(value, ioaddr + XGMAC_LPI_CTRL);
+
+	return 0;
+}
+
+static void dwxgmac2_set_eee_pls(struct mac_device_info *hw, int link)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_LPI_CTRL);
+	if (link)
+		value |= LPI_CTRL_STATUS_PLS;
+	else
+		value &= ~LPI_CTRL_STATUS_PLS;
+	writel(value, ioaddr + XGMAC_LPI_CTRL);
+}
+
+static void dwxgmac2_set_eee_timer(struct mac_device_info *hw, int ls, int tw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = (tw & 0xffff) | ((ls & 0x3ff) << 16);
+	writel(value, ioaddr + XGMAC_LPI_TIMER_CTRL);
+}
+
+static void dwxgmac2_set_mchash(void __iomem *ioaddr, u32 *mcfilterbits,
+				int mcbitslog2)
+{
+	int numhashregs, regs;
+
+	switch (mcbitslog2) {
+	case 6:
+		numhashregs = 2;
+		break;
+	case 7:
+		numhashregs = 4;
+		break;
+	case 8:
+		numhashregs = 8;
+		break;
+	default:
+		return;
+	}
+
+	for (regs = 0; regs < numhashregs; regs++)
+		writel(mcfilterbits[regs], ioaddr + XGMAC_HASH_TABLE(regs));
+}
+
+static void dwxgmac2_set_filter(struct mac_device_info *hw,
+				struct net_device *dev)
+{
+	void __iomem *ioaddr = (void __iomem *)dev->base_addr;
+	u32 value = readl(ioaddr + XGMAC_PACKET_FILTER);
+	int mcbitslog2 = hw->mcast_bits_log2;
+	u32 mc_filter[8];
+	int i;
+
+	value &= ~(XGMAC_FILTER_PR | XGMAC_FILTER_HMC | XGMAC_FILTER_PM);
+	value |= XGMAC_FILTER_HPF;
+
+	memset(mc_filter, 0, sizeof(mc_filter));
+
+	if (dev->flags & IFF_PROMISC) {
+		value |= XGMAC_FILTER_PR;
+		value |= XGMAC_FILTER_PCF;
+	} else if ((dev->flags & IFF_ALLMULTI) ||
+		   (netdev_mc_count(dev) > hw->multicast_filter_bins)) {
+		value |= XGMAC_FILTER_PM;
+
+		for (i = 0; i < XGMAC_MAX_HASH_TABLE; i++)
+			writel(~0x0, ioaddr + XGMAC_HASH_TABLE(i));
+	} else if (!netdev_mc_empty(dev) && (dev->flags & IFF_MULTICAST)) {
+		struct netdev_hw_addr *ha;
+
+		value |= XGMAC_FILTER_HMC;
+
+		netdev_for_each_mc_addr(ha, dev) {
+			u32 nr = (bitrev32(~crc32_le(~0, ha->addr, 6)) >>
+					(32 - mcbitslog2));
+			mc_filter[nr >> 5] |= (1 << (nr & 0x1F));
+		}
+	}
+
+	dwxgmac2_set_mchash(ioaddr, mc_filter, mcbitslog2);
+
+	/* Handle multiple unicast addresses */
+	if (netdev_uc_count(dev) > hw->unicast_filter_entries) {
+		value |= XGMAC_FILTER_PR;
+	} else {
+		struct netdev_hw_addr *ha;
+		int reg = 1;
+
+		netdev_for_each_uc_addr(ha, dev) {
+			dwxgmac2_set_umac_addr(hw, ha->addr, reg);
+			reg++;
+		}
+
+		for ( ; reg < XGMAC_ADDR_MAX; reg++) {
+			writel(0, ioaddr + XGMAC_ADDRx_HIGH(reg));
+			writel(0, ioaddr + XGMAC_ADDRx_LOW(reg));
+		}
+	}
+
+	writel(value, ioaddr + XGMAC_PACKET_FILTER);
+}
+
+static void dwxgmac2_set_mac_loopback(void __iomem *ioaddr, bool enable)
+{
+	u32 value = readl(ioaddr + XGMAC_RX_CONFIG);
+
+	if (enable)
+		value |= XGMAC_CONFIG_LM;
+	else
+		value &= ~XGMAC_CONFIG_LM;
+
+	writel(value, ioaddr + XGMAC_RX_CONFIG);
+}
+
+static int dwxgmac2_rss_write_reg(void __iomem *ioaddr, bool is_key, int idx,
+				  u32 val)
+{
+	u32 ctrl = 0;
+
+	writel(val, ioaddr + XGMAC_RSS_DATA);
+	ctrl |= idx << XGMAC_RSSIA_SHIFT;
+	ctrl |= is_key ? XGMAC_ADDRT : 0x0;
+	ctrl |= XGMAC_OB;
+	writel(ctrl, ioaddr + XGMAC_RSS_ADDR);
+
+	return readl_poll_timeout(ioaddr + XGMAC_RSS_ADDR, ctrl,
+				  !(ctrl & XGMAC_OB), 100, 10000);
+}
+
+static int dwxgmac2_rss_configure(struct mac_device_info *hw,
+				  struct stmmac_rss *cfg, u32 num_rxq)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value, *key;
+	int i, ret;
+
+	value = readl(ioaddr + XGMAC_RSS_CTRL);
+	if (!cfg || !cfg->enable) {
+		value &= ~XGMAC_RSSE;
+		writel(value, ioaddr + XGMAC_RSS_CTRL);
+		return 0;
+	}
+
+	key = (u32 *)cfg->key;
+	for (i = 0; i < (ARRAY_SIZE(cfg->key) / sizeof(u32)); i++) {
+		ret = dwxgmac2_rss_write_reg(ioaddr, true, i, key[i]);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(cfg->table); i++) {
+		ret = dwxgmac2_rss_write_reg(ioaddr, false, i, cfg->table[i]);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < num_rxq; i++)
+		dwxgmac2_map_mtl_to_dma(hw, i, XGMAC_QDDMACH);
+
+	value |= XGMAC_UDP4TE | XGMAC_TCP4TE | XGMAC_IP2TE | XGMAC_RSSE;
+	writel(value, ioaddr + XGMAC_RSS_CTRL);
+	return 0;
+}
+
+struct dwxgmac3_error_desc {
+	bool valid;
+	const char *desc;
+	const char *detailed_desc;
+};
+
+#define STAT_OFF(field)		offsetof(struct stmmac_safety_stats, field)
+
+static void dwxgmac3_log_error(struct net_device *ndev, u32 value, bool corr,
+			       const char *module_name,
+			       const struct dwxgmac3_error_desc *desc,
+			       unsigned long field_offset,
+			       struct stmmac_safety_stats *stats)
+{
+	unsigned long loc, mask;
+	u8 *bptr = (u8 *)stats;
+	unsigned long *ptr;
+
+	ptr = (unsigned long *)(bptr + field_offset);
+
+	mask = value;
+	for_each_set_bit(loc, &mask, 32) {
+		netdev_err(ndev, "Found %s error in %s: '%s: %s'\n", corr ?
+				"correctable" : "uncorrectable", module_name,
+				desc[loc].desc, desc[loc].detailed_desc);
+
+		/* Update counters */
+		ptr[loc]++;
+	}
+}
+
+static const struct dwxgmac3_error_desc dwxgmac3_mac_errors[32]= {
+	{ true, "ATPES", "Application Transmit Interface Parity Check Error" },
+	{ true, "DPES", "Descriptor Cache Data Path Parity Check Error" },
+	{ true, "TPES", "TSO Data Path Parity Check Error" },
+	{ true, "TSOPES", "TSO Header Data Path Parity Check Error" },
+	{ true, "MTPES", "MTL Data Path Parity Check Error" },
+	{ true, "MTSPES", "MTL TX Status Data Path Parity Check Error" },
+	{ true, "MTBUPES", "MAC TBU Data Path Parity Check Error" },
+	{ true, "MTFCPES", "MAC TFC Data Path Parity Check Error" },
+	{ true, "ARPES", "Application Receive Interface Data Path Parity Check Error" },
+	{ true, "MRWCPES", "MTL RWC Data Path Parity Check Error" },
+	{ true, "MRRCPES", "MTL RCC Data Path Parity Check Error" },
+	{ true, "CWPES", "CSR Write Data Path Parity Check Error" },
+	{ true, "ASRPES", "AXI Slave Read Data Path Parity Check Error" },
+	{ true, "TTES", "TX FSM Timeout Error" },
+	{ true, "RTES", "RX FSM Timeout Error" },
+	{ true, "CTES", "CSR FSM Timeout Error" },
+	{ true, "ATES", "APP FSM Timeout Error" },
+	{ true, "PTES", "PTP FSM Timeout Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
+	{ true, "MSTTES", "Master Read/Write Timeout Error" },
+	{ true, "SLVTES", "Slave Read/Write Timeout Error" },
+	{ true, "ATITES", "Application Timeout on ATI Interface Error" },
+	{ true, "ARITES", "Application Timeout on ARI Interface Error" },
+	{ true, "FSMPES", "FSM State Parity Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
+	{ true, "CPI", "Control Register Parity Check Error" },
+};
+
+static void dwxgmac3_handle_mac_err(struct net_device *ndev,
+				    void __iomem *ioaddr, bool correctable,
+				    struct stmmac_safety_stats *stats)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_MAC_DPP_FSM_INT_STATUS);
+	writel(value, ioaddr + XGMAC_MAC_DPP_FSM_INT_STATUS);
+
+	dwxgmac3_log_error(ndev, value, correctable, "MAC",
+			   dwxgmac3_mac_errors, STAT_OFF(mac_errors), stats);
+}
+
+static const struct dwxgmac3_error_desc dwxgmac3_mtl_errors[32]= {
+	{ true, "TXCES", "MTL TX Memory Error" },
+	{ true, "TXAMS", "MTL TX Memory Address Mismatch Error" },
+	{ true, "TXUES", "MTL TX Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 3 */
+	{ true, "RXCES", "MTL RX Memory Error" },
+	{ true, "RXAMS", "MTL RX Memory Address Mismatch Error" },
+	{ true, "RXUES", "MTL RX Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 7 */
+	{ true, "ECES", "MTL EST Memory Error" },
+	{ true, "EAMS", "MTL EST Memory Address Mismatch Error" },
+	{ true, "EUES", "MTL EST Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 11 */
+	{ true, "RPCES", "MTL RX Parser Memory Error" },
+	{ true, "RPAMS", "MTL RX Parser Memory Address Mismatch Error" },
+	{ true, "RPUES", "MTL RX Parser Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 15 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 16 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 17 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 21 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 22 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 23 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 24 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 25 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 31 */
+};
+
+static void dwxgmac3_handle_mtl_err(struct net_device *ndev,
+				    void __iomem *ioaddr, bool correctable,
+				    struct stmmac_safety_stats *stats)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_MTL_ECC_INT_STATUS);
+	writel(value, ioaddr + XGMAC_MTL_ECC_INT_STATUS);
+
+	dwxgmac3_log_error(ndev, value, correctable, "MTL",
+			   dwxgmac3_mtl_errors, STAT_OFF(mtl_errors), stats);
+}
+
+static const struct dwxgmac3_error_desc dwxgmac3_dma_errors[32]= {
+	{ true, "TCES", "DMA TSO Memory Error" },
+	{ true, "TAMS", "DMA TSO Memory Address Mismatch Error" },
+	{ true, "TUES", "DMA TSO Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 3 */
+	{ true, "DCES", "DMA DCACHE Memory Error" },
+	{ true, "DAMS", "DMA DCACHE Address Mismatch Error" },
+	{ true, "DUES", "DMA DCACHE Memory Error" },
+	{ false, "UNKNOWN", "Unknown Error" }, /* 7 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 8 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 9 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 10 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 11 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 12 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 13 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 14 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 15 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 16 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 17 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 21 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 22 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 23 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 24 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 25 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
+	{ false, "UNKNOWN", "Unknown Error" }, /* 31 */
+};
+
+static const char dpp_rx_err[] = "Read Rx Descriptor Parity checker Error";
+static const char dpp_tx_err[] = "Read Tx Descriptor Parity checker Error";
+static const struct dwxgmac3_error_desc dwxgmac3_dma_dpp_errors[32] = {
+	{ true, "TDPES0", dpp_tx_err },
+	{ true, "TDPES1", dpp_tx_err },
+	{ true, "TDPES2", dpp_tx_err },
+	{ true, "TDPES3", dpp_tx_err },
+	{ true, "TDPES4", dpp_tx_err },
+	{ true, "TDPES5", dpp_tx_err },
+	{ true, "TDPES6", dpp_tx_err },
+	{ true, "TDPES7", dpp_tx_err },
+	{ true, "TDPES8", dpp_tx_err },
+	{ true, "TDPES9", dpp_tx_err },
+	{ true, "TDPES10", dpp_tx_err },
+	{ true, "TDPES11", dpp_tx_err },
+	{ true, "TDPES12", dpp_tx_err },
+	{ true, "TDPES13", dpp_tx_err },
+	{ true, "TDPES14", dpp_tx_err },
+	{ true, "TDPES15", dpp_tx_err },
+	{ true, "RDPES0", dpp_rx_err },
+	{ true, "RDPES1", dpp_rx_err },
+	{ true, "RDPES2", dpp_rx_err },
+	{ true, "RDPES3", dpp_rx_err },
+	{ true, "RDPES4", dpp_rx_err },
+	{ true, "RDPES5", dpp_rx_err },
+	{ true, "RDPES6", dpp_rx_err },
+	{ true, "RDPES7", dpp_rx_err },
+	{ true, "RDPES8", dpp_rx_err },
+	{ true, "RDPES9", dpp_rx_err },
+	{ true, "RDPES10", dpp_rx_err },
+	{ true, "RDPES11", dpp_rx_err },
+	{ true, "RDPES12", dpp_rx_err },
+	{ true, "RDPES13", dpp_rx_err },
+	{ true, "RDPES14", dpp_rx_err },
+	{ true, "RDPES15", dpp_rx_err },
+};
+
+static void dwxgmac3_handle_dma_err(struct net_device *ndev,
+				    void __iomem *ioaddr, bool correctable,
+				    struct stmmac_safety_stats *stats)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_ECC_INT_STATUS);
+	writel(value, ioaddr + XGMAC_DMA_ECC_INT_STATUS);
+
+	dwxgmac3_log_error(ndev, value, correctable, "DMA",
+			   dwxgmac3_dma_errors, STAT_OFF(dma_errors), stats);
+
+	value = readl(ioaddr + XGMAC_DMA_DPP_INT_STATUS);
+	writel(value, ioaddr + XGMAC_DMA_DPP_INT_STATUS);
+
+	dwxgmac3_log_error(ndev, value, false, "DMA_DPP",
+			   dwxgmac3_dma_dpp_errors,
+			   STAT_OFF(dma_dpp_errors), stats);
+}
+
+static int
+dwxgmac3_safety_feat_config(void __iomem *ioaddr, unsigned int asp,
+			    struct stmmac_safety_feature_cfg *safety_cfg)
+{
+	u32 value;
+
+	if (!asp)
+		return -EINVAL;
+
+	/* 1. Enable Safety Features */
+	writel(0x0, ioaddr + XGMAC_MTL_ECC_CONTROL);
+
+	/* 2. Enable MTL Safety Interrupts */
+	value = readl(ioaddr + XGMAC_MTL_ECC_INT_ENABLE);
+	value |= XGMAC_RPCEIE; /* RX Parser Memory Correctable Error */
+	value |= XGMAC_ECEIE; /* EST Memory Correctable Error */
+	value |= XGMAC_RXCEIE; /* RX Memory Correctable Error */
+	value |= XGMAC_TXCEIE; /* TX Memory Correctable Error */
+	writel(value, ioaddr + XGMAC_MTL_ECC_INT_ENABLE);
+
+	/* 3. Enable DMA Safety Interrupts */
+	value = readl(ioaddr + XGMAC_DMA_ECC_INT_ENABLE);
+	value |= XGMAC_DCEIE; /* Descriptor Cache Memory Correctable Error */
+	value |= XGMAC_TCEIE; /* TSO Memory Correctable Error */
+	writel(value, ioaddr + XGMAC_DMA_ECC_INT_ENABLE);
+
+	/* 0x2: Without ECC or Parity Ports on External Application Interface
+	 * 0x4: Only ECC Protection for External Memory feature is selected
+	 */
+	if (asp == 0x2 || asp == 0x4)
+		return 0;
+
+	/* 4. Enable Parity and Timeout for FSM */
+	value = readl(ioaddr + XGMAC_MAC_FSM_CONTROL);
+	value |= XGMAC_PRTYEN; /* FSM Parity Feature */
+	value |= XGMAC_TMOUTEN; /* FSM Timeout Feature */
+	writel(value, ioaddr + XGMAC_MAC_FSM_CONTROL);
+
+	/* 5. Enable Data Path Parity Protection */
+	value = readl(ioaddr + XGMAC_MTL_DPP_CONTROL);
+	/* already enabled by default, explicit enable it again */
+	value &= ~XGMAC_DPP_DISABLE;
+	writel(value, ioaddr + XGMAC_MTL_DPP_CONTROL);
+
+	return 0;
+}
+
+static int dwxgmac3_safety_feat_irq_status(struct net_device *ndev,
+					   void __iomem *ioaddr,
+					   unsigned int asp,
+					   struct stmmac_safety_stats *stats)
+{
+	bool err, corr;
+	u32 mtl, dma;
+	int ret = 0;
+
+	if (!asp)
+		return -EINVAL;
+
+	mtl = readl(ioaddr + XGMAC_MTL_SAFETY_INT_STATUS);
+	dma = readl(ioaddr + XGMAC_DMA_SAFETY_INT_STATUS);
+
+	err = (mtl & XGMAC_MCSIS) || (dma & XGMAC_MCSIS);
+	corr = false;
+	if (err) {
+		dwxgmac3_handle_mac_err(ndev, ioaddr, corr, stats);
+		ret |= !corr;
+	}
+
+	err = (mtl & (XGMAC_MEUIS | XGMAC_MECIS)) ||
+	      (dma & (XGMAC_MSUIS | XGMAC_MSCIS));
+	corr = (mtl & XGMAC_MECIS) || (dma & XGMAC_MSCIS);
+	if (err) {
+		dwxgmac3_handle_mtl_err(ndev, ioaddr, corr, stats);
+		ret |= !corr;
+	}
+
+	/* DMA_DPP_Interrupt_Status is indicated by MCSIS bit in
+	 * DMA_Safety_Interrupt_Status, so we handle DMA Data Path
+	 * Parity Errors here
+	 */
+	err = dma & (XGMAC_DEUIS | XGMAC_DECIS | XGMAC_MCSIS);
+	corr = dma & XGMAC_DECIS;
+	if (err) {
+		dwxgmac3_handle_dma_err(ndev, ioaddr, corr, stats);
+		ret |= !corr;
+	}
+
+	return ret;
+}
+
+static const struct dwxgmac3_error {
+	const struct dwxgmac3_error_desc *desc;
+} dwxgmac3_all_errors[] = {
+	{ dwxgmac3_mac_errors },
+	{ dwxgmac3_mtl_errors },
+	{ dwxgmac3_dma_errors },
+	{ dwxgmac3_dma_dpp_errors },
+};
+
+static int dwxgmac3_safety_feat_dump(struct stmmac_safety_stats *stats,
+				     int index, unsigned long *count,
+				     const char **desc)
+{
+	int module = index / 32, offset = index % 32;
+	unsigned long *ptr = (unsigned long *)stats;
+
+	if (module >= ARRAY_SIZE(dwxgmac3_all_errors))
+		return -EINVAL;
+	if (!dwxgmac3_all_errors[module].desc[offset].valid)
+		return -EINVAL;
+	if (count)
+		*count = *(ptr + index);
+	if (desc)
+		*desc = dwxgmac3_all_errors[module].desc[offset].desc;
+	return 0;
+}
+
+static int dwxgmac3_rxp_disable(void __iomem *ioaddr)
+{
+	u32 val = readl(ioaddr + XGMAC_MTL_OPMODE);
+
+	val &= ~XGMAC_FRPE;
+	writel(val, ioaddr + XGMAC_MTL_OPMODE);
+
+	return 0;
+}
+
+static void dwxgmac3_rxp_enable(void __iomem *ioaddr)
+{
+	u32 val;
+
+	val = readl(ioaddr + XGMAC_MTL_OPMODE);
+	val |= XGMAC_FRPE;
+	writel(val, ioaddr + XGMAC_MTL_OPMODE);
+}
+
+static int dwxgmac3_rxp_update_single_entry(void __iomem *ioaddr,
+					    struct stmmac_tc_entry *entry,
+					    int pos)
+{
+	int ret, i;
+
+	for (i = 0; i < (sizeof(entry->val) / sizeof(u32)); i++) {
+		int real_pos = pos * (sizeof(entry->val) / sizeof(u32)) + i;
+		u32 val;
+
+		/* Wait for ready */
+		ret = readl_poll_timeout(ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST,
+					 val, !(val & XGMAC_STARTBUSY), 1, 10000);
+		if (ret)
+			return ret;
+
+		/* Write data */
+		val = *((u32 *)&entry->val + i);
+		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_DATA);
+
+		/* Write pos */
+		val = real_pos & XGMAC_ADDR;
+		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST);
+
+		/* Write OP */
+		val |= XGMAC_WRRDN;
+		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST);
+
+		/* Start Write */
+		val |= XGMAC_STARTBUSY;
+		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST);
+
+		/* Wait for done */
+		ret = readl_poll_timeout(ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST,
+					 val, !(val & XGMAC_STARTBUSY), 1, 10000);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static struct stmmac_tc_entry *
+dwxgmac3_rxp_get_next_entry(struct stmmac_tc_entry *entries,
+			    unsigned int count, u32 curr_prio)
+{
+	struct stmmac_tc_entry *entry;
+	u32 min_prio = ~0x0;
+	int i, min_prio_idx;
+	bool found = false;
+
+	for (i = count - 1; i >= 0; i--) {
+		entry = &entries[i];
+
+		/* Do not update unused entries */
+		if (!entry->in_use)
+			continue;
+		/* Do not update already updated entries (i.e. fragments) */
+		if (entry->in_hw)
+			continue;
+		/* Let last entry be updated last */
+		if (entry->is_last)
+			continue;
+		/* Do not return fragments */
+		if (entry->is_frag)
+			continue;
+		/* Check if we already checked this prio */
+		if (entry->prio < curr_prio)
+			continue;
+		/* Check if this is the minimum prio */
+		if (entry->prio < min_prio) {
+			min_prio = entry->prio;
+			min_prio_idx = i;
+			found = true;
+		}
+	}
+
+	if (found)
+		return &entries[min_prio_idx];
+	return NULL;
+}
+
+static int dwxgmac3_rxp_config(void __iomem *ioaddr,
+			       struct stmmac_tc_entry *entries,
+			       unsigned int count)
+{
+	struct stmmac_tc_entry *entry, *frag;
+	int i, ret, nve = 0;
+	u32 curr_prio = 0;
+	u32 old_val, val;
+
+	/* Force disable RX */
+	old_val = readl(ioaddr + XGMAC_RX_CONFIG);
+	val = old_val & ~XGMAC_CONFIG_RE;
+	writel(val, ioaddr + XGMAC_RX_CONFIG);
+
+	/* Disable RX Parser */
+	ret = dwxgmac3_rxp_disable(ioaddr);
+	if (ret)
+		goto re_enable;
+
+	/* Set all entries as NOT in HW */
+	for (i = 0; i < count; i++) {
+		entry = &entries[i];
+		entry->in_hw = false;
+	}
+
+	/* Update entries by reverse order */
+	while (1) {
+		entry = dwxgmac3_rxp_get_next_entry(entries, count, curr_prio);
+		if (!entry)
+			break;
+
+		curr_prio = entry->prio;
+		frag = entry->frag_ptr;
+
+		/* Set special fragment requirements */
+		if (frag) {
+			entry->val.af = 0;
+			entry->val.rf = 0;
+			entry->val.nc = 1;
+			entry->val.ok_index = nve + 2;
+		}
+
+		ret = dwxgmac3_rxp_update_single_entry(ioaddr, entry, nve);
+		if (ret)
+			goto re_enable;
+
+		entry->table_pos = nve++;
+		entry->in_hw = true;
+
+		if (frag && !frag->in_hw) {
+			ret = dwxgmac3_rxp_update_single_entry(ioaddr, frag, nve);
+			if (ret)
+				goto re_enable;
+			frag->table_pos = nve++;
+			frag->in_hw = true;
+		}
+	}
+
+	if (!nve)
+		goto re_enable;
+
+	/* Update all pass entry */
+	for (i = 0; i < count; i++) {
+		entry = &entries[i];
+		if (!entry->is_last)
+			continue;
+
+		ret = dwxgmac3_rxp_update_single_entry(ioaddr, entry, nve);
+		if (ret)
+			goto re_enable;
+
+		entry->table_pos = nve++;
+	}
+
+	/* Assume n. of parsable entries == n. of valid entries */
+	val = (nve << 16) & XGMAC_NPE;
+	val |= nve & XGMAC_NVE;
+	writel(val, ioaddr + XGMAC_MTL_RXP_CONTROL_STATUS);
+
+	/* Enable RX Parser */
+	dwxgmac3_rxp_enable(ioaddr);
+
+re_enable:
+	/* Re-enable RX */
+	writel(old_val, ioaddr + XGMAC_RX_CONFIG);
+	return ret;
+}
+
+static int dwxgmac2_get_mac_tx_timestamp(struct mac_device_info *hw, u64 *ts)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	if (readl_poll_timeout_atomic(ioaddr + XGMAC_TIMESTAMP_STATUS,
+				      value, value & XGMAC_TXTSC, 100, 10000))
+		return -EBUSY;
+
+	*ts = readl(ioaddr + XGMAC_TXTIMESTAMP_NSEC) & XGMAC_TXTSSTSLO;
+	*ts += readl(ioaddr + XGMAC_TXTIMESTAMP_SEC) * 1000000000ULL;
+	return 0;
+}
+
+static int dwxgmac2_flex_pps_config(void __iomem *ioaddr, int index,
+				    struct stmmac_pps_cfg *cfg, bool enable,
+				    u32 sub_second_inc, u32 systime_flags)
+{
+	u32 tnsec = readl(ioaddr + XGMAC_PPSx_TARGET_TIME_NSEC(index));
+	u32 val = readl(ioaddr + XGMAC_PPS_CONTROL);
+	u64 period;
+
+	if (!cfg->available)
+		return -EINVAL;
+	if (tnsec & XGMAC_TRGTBUSY0)
+		return -EBUSY;
+	if (!sub_second_inc || !systime_flags)
+		return -EINVAL;
+
+	val &= ~XGMAC_PPSx_MASK(index);
+
+	if (!enable) {
+		val |= XGMAC_PPSCMDx(index, XGMAC_PPSCMD_STOP);
+		writel(val, ioaddr + XGMAC_PPS_CONTROL);
+		return 0;
+	}
+
+	val |= XGMAC_PPSCMDx(index, XGMAC_PPSCMD_START);
+	val |= XGMAC_TRGTMODSELx(index, XGMAC_PPSCMD_START);
+
+	/* XGMAC Core has 4 PPS outputs at most.
+	 *
+	 * Prior XGMAC Core 3.20, Fixed mode or Flexible mode are selectable for
+	 * PPS0 only via PPSEN0. PPS{1,2,3} are in Flexible mode by default,
+	 * and can not be switched to Fixed mode, since PPSEN{1,2,3} are
+	 * read-only reserved to 0.
+	 * But we always set PPSEN{1,2,3} do not make things worse ;-)
+	 *
+	 * From XGMAC Core 3.20 and later, PPSEN{0,1,2,3} are writable and must
+	 * be set, or the PPS outputs stay in Fixed PPS mode by default.
+	 */
+	val |= XGMAC_PPSENx(index);
+
+	writel(cfg->start.tv_sec, ioaddr + XGMAC_PPSx_TARGET_TIME_SEC(index));
+
+	if (!(systime_flags & PTP_TCR_TSCTRLSSR))
+		cfg->start.tv_nsec = (cfg->start.tv_nsec * 1000) / 465;
+	writel(cfg->start.tv_nsec, ioaddr + XGMAC_PPSx_TARGET_TIME_NSEC(index));
+
+	period = cfg->period.tv_sec * 1000000000;
+	period += cfg->period.tv_nsec;
+
+	do_div(period, sub_second_inc);
+
+	if (period <= 1)
+		return -EINVAL;
+
+	writel(period - 1, ioaddr + XGMAC_PPSx_INTERVAL(index));
+
+	period >>= 1;
+	if (period <= 1)
+		return -EINVAL;
+
+	writel(period - 1, ioaddr + XGMAC_PPSx_WIDTH(index));
+
+	/* Finally, activate it */
+	writel(val, ioaddr + XGMAC_PPS_CONTROL);
+	return 0;
+}
+
+static void dwxgmac2_sarc_configure(void __iomem *ioaddr, int val)
+{
+	u32 value = readl(ioaddr + XGMAC_TX_CONFIG);
+
+	value &= ~XGMAC_CONFIG_SARC;
+	value |= val << XGMAC_CONFIG_SARC_SHIFT;
+
+	writel(value, ioaddr + XGMAC_TX_CONFIG);
+}
+
+static int dwxgmac2_filter_wait(struct mac_device_info *hw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	if (readl_poll_timeout(ioaddr + XGMAC_L3L4_ADDR_CTRL, value,
+			       !(value & XGMAC_XB), 100, 10000))
+		return -EBUSY;
+	return 0;
+}
+
+static int dwxgmac2_filter_read(struct mac_device_info *hw, u32 filter_no,
+				u8 reg, u32 *data)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+	int ret;
+
+	ret = dwxgmac2_filter_wait(hw);
+	if (ret)
+		return ret;
+
+	value = ((filter_no << XGMAC_IDDR_FNUM) | reg) << XGMAC_IDDR_SHIFT;
+	value |= XGMAC_TT | XGMAC_XB;
+	writel(value, ioaddr + XGMAC_L3L4_ADDR_CTRL);
+
+	ret = dwxgmac2_filter_wait(hw);
+	if (ret)
+		return ret;
+
+	*data = readl(ioaddr + XGMAC_L3L4_DATA);
+	return 0;
+}
+
+static int dwxgmac2_filter_write(struct mac_device_info *hw, u32 filter_no,
+				 u8 reg, u32 data)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+	int ret;
+
+	ret = dwxgmac2_filter_wait(hw);
+	if (ret)
+		return ret;
+
+	writel(data, ioaddr + XGMAC_L3L4_DATA);
+
+	value = ((filter_no << XGMAC_IDDR_FNUM) | reg) << XGMAC_IDDR_SHIFT;
+	value |= XGMAC_XB;
+	writel(value, ioaddr + XGMAC_L3L4_ADDR_CTRL);
+
+	return dwxgmac2_filter_wait(hw);
+}
+
+static int dwxgmac2_config_l3_filter(struct mac_device_info *hw, u32 filter_no,
+				     bool en, bool ipv6, bool sa, bool inv,
+				     u32 match)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+	int ret;
+
+	value = readl(ioaddr + XGMAC_PACKET_FILTER);
+	value |= XGMAC_FILTER_IPFE;
+	writel(value, ioaddr + XGMAC_PACKET_FILTER);
+
+	ret = dwxgmac2_filter_read(hw, filter_no, XGMAC_L3L4_CTRL, &value);
+	if (ret)
+		return ret;
+
+	/* For IPv6 not both SA/DA filters can be active */
+	if (ipv6) {
+		value |= XGMAC_L3PEN0;
+		value &= ~(XGMAC_L3SAM0 | XGMAC_L3SAIM0);
+		value &= ~(XGMAC_L3DAM0 | XGMAC_L3DAIM0);
+		if (sa) {
+			value |= XGMAC_L3SAM0;
+			if (inv)
+				value |= XGMAC_L3SAIM0;
+		} else {
+			value |= XGMAC_L3DAM0;
+			if (inv)
+				value |= XGMAC_L3DAIM0;
+		}
+	} else {
+		value &= ~XGMAC_L3PEN0;
+		if (sa) {
+			value |= XGMAC_L3SAM0;
+			if (inv)
+				value |= XGMAC_L3SAIM0;
+		} else {
+			value |= XGMAC_L3DAM0;
+			if (inv)
+				value |= XGMAC_L3DAIM0;
+		}
+	}
+
+	ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, value);
+	if (ret)
+		return ret;
+
+	if (sa) {
+		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3_ADDR0, match);
+		if (ret)
+			return ret;
+	} else {
+		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3_ADDR1, match);
+		if (ret)
+			return ret;
+	}
+
+	if (!en)
+		return dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, 0);
+
+	return 0;
+}
+
+static int dwxgmac2_config_l4_filter(struct mac_device_info *hw, u32 filter_no,
+				     bool en, bool udp, bool sa, bool inv,
+				     u32 match)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+	int ret;
+
+	value = readl(ioaddr + XGMAC_PACKET_FILTER);
+	value |= XGMAC_FILTER_IPFE;
+	writel(value, ioaddr + XGMAC_PACKET_FILTER);
+
+	ret = dwxgmac2_filter_read(hw, filter_no, XGMAC_L3L4_CTRL, &value);
+	if (ret)
+		return ret;
+
+	if (udp) {
+		value |= XGMAC_L4PEN0;
+	} else {
+		value &= ~XGMAC_L4PEN0;
+	}
+
+	value &= ~(XGMAC_L4SPM0 | XGMAC_L4SPIM0);
+	value &= ~(XGMAC_L4DPM0 | XGMAC_L4DPIM0);
+	if (sa) {
+		value |= XGMAC_L4SPM0;
+		if (inv)
+			value |= XGMAC_L4SPIM0;
+	} else {
+		value |= XGMAC_L4DPM0;
+		if (inv)
+			value |= XGMAC_L4DPIM0;
+	}
+
+	ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, value);
+	if (ret)
+		return ret;
+
+	if (sa) {
+		value = match & XGMAC_L4SP0;
+
+		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L4_ADDR, value);
+		if (ret)
+			return ret;
+	} else {
+		value = (match << XGMAC_L4DP0_SHIFT) & XGMAC_L4DP0;
+
+		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L4_ADDR, value);
+		if (ret)
+			return ret;
+	}
+
+	if (!en)
+		return dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, 0);
+
+	return 0;
+}
+
+static void dwxgmac2_set_arp_offload(struct mac_device_info *hw, bool en,
+				     u32 addr)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	writel(addr, ioaddr + XGMAC_ARP_ADDR);
+
+	value = readl(ioaddr + XGMAC_RX_CONFIG);
+	if (en)
+		value |= XGMAC_CONFIG_ARPEN;
+	else
+		value &= ~XGMAC_CONFIG_ARPEN;
+	writel(value, ioaddr + XGMAC_RX_CONFIG);
+}
+
+const struct stmmac_ops ec_dwxgmac210_ops = {
+	.core_init = dwxgmac2_core_init,
+	.update_caps = dwxgmac2_update_caps,
+	.set_mac = dwxgmac2_set_mac,
+	.rx_ipc = dwxgmac2_rx_ipc,
+	.rx_queue_enable = dwxgmac2_rx_queue_enable,
+	.rx_queue_prio = dwxgmac2_rx_queue_prio,
+	.tx_queue_prio = dwxgmac2_tx_queue_prio,
+	.rx_queue_routing = dwxgmac2_rx_queue_routing,
+	.prog_mtl_rx_algorithms = dwxgmac2_prog_mtl_rx_algorithms,
+	.prog_mtl_tx_algorithms = dwxgmac2_prog_mtl_tx_algorithms,
+	.set_mtl_tx_queue_weight = dwxgmac2_set_mtl_tx_queue_weight,
+	.map_mtl_to_dma = dwxgmac2_map_mtl_to_dma,
+	.config_cbs = dwxgmac2_config_cbs,
+	.dump_regs = dwxgmac2_dump_regs,
+	.host_irq_status = dwxgmac2_host_irq_status,
+	.host_mtl_irq_status = dwxgmac2_host_mtl_irq_status,
+	.flow_ctrl = dwxgmac2_flow_ctrl,
+	.pmt = dwxgmac2_pmt,
+	.set_umac_addr = dwxgmac2_set_umac_addr,
+	.get_umac_addr = dwxgmac2_get_umac_addr,
+	.set_lpi_mode = dwxgmac2_set_lpi_mode,
+	.set_eee_timer = dwxgmac2_set_eee_timer,
+	.set_eee_pls = dwxgmac2_set_eee_pls,
+	.debug = NULL,
+	.set_filter = dwxgmac2_set_filter,
+	.safety_feat_config = dwxgmac3_safety_feat_config,
+	.safety_feat_irq_status = dwxgmac3_safety_feat_irq_status,
+	.safety_feat_dump = dwxgmac3_safety_feat_dump,
+	.set_mac_loopback = dwxgmac2_set_mac_loopback,
+	.rss_configure = dwxgmac2_rss_configure,
+	.rxp_config = dwxgmac3_rxp_config,
+	.get_mac_tx_timestamp = dwxgmac2_get_mac_tx_timestamp,
+	.flex_pps_config = dwxgmac2_flex_pps_config,
+	.sarc_configure = dwxgmac2_sarc_configure,
+	.config_l3_filter = dwxgmac2_config_l3_filter,
+	.config_l4_filter = dwxgmac2_config_l4_filter,
+	.set_arp_offload = dwxgmac2_set_arp_offload,
+	.fpe_map_preemption_class = ec_dwxgmac3_fpe_map_preemption_class,
+};
+
+static void dwxlgmac2_rx_queue_enable(struct mac_device_info *hw, u8 mode,
+				      u32 queue)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + XLGMAC_RXQ_ENABLE_CTRL0) & ~XGMAC_RXQEN(queue);
+	if (mode == MTL_QUEUE_AVB)
+		value |= 0x1 << XGMAC_RXQEN_SHIFT(queue);
+	else if (mode == MTL_QUEUE_DCB)
+		value |= 0x2 << XGMAC_RXQEN_SHIFT(queue);
+	writel(value, ioaddr + XLGMAC_RXQ_ENABLE_CTRL0);
+}
+
+const struct stmmac_ops ec_dwxlgmac2_ops = {
+	.core_init = dwxgmac2_core_init,
+	.set_mac = dwxgmac2_set_mac,
+	.rx_ipc = dwxgmac2_rx_ipc,
+	.rx_queue_enable = dwxlgmac2_rx_queue_enable,
+	.rx_queue_prio = dwxgmac2_rx_queue_prio,
+	.tx_queue_prio = dwxgmac2_tx_queue_prio,
+	.rx_queue_routing = dwxgmac2_rx_queue_routing,
+	.prog_mtl_rx_algorithms = dwxgmac2_prog_mtl_rx_algorithms,
+	.prog_mtl_tx_algorithms = dwxgmac2_prog_mtl_tx_algorithms,
+	.set_mtl_tx_queue_weight = dwxgmac2_set_mtl_tx_queue_weight,
+	.map_mtl_to_dma = dwxgmac2_map_mtl_to_dma,
+	.config_cbs = dwxgmac2_config_cbs,
+	.dump_regs = dwxgmac2_dump_regs,
+	.host_irq_status = dwxgmac2_host_irq_status,
+	.host_mtl_irq_status = dwxgmac2_host_mtl_irq_status,
+	.flow_ctrl = dwxgmac2_flow_ctrl,
+	.pmt = dwxgmac2_pmt,
+	.set_umac_addr = dwxgmac2_set_umac_addr,
+	.get_umac_addr = dwxgmac2_get_umac_addr,
+	.set_lpi_mode = dwxgmac2_set_lpi_mode,
+	.set_eee_timer = dwxgmac2_set_eee_timer,
+	.set_eee_pls = dwxgmac2_set_eee_pls,
+	.debug = NULL,
+	.set_filter = dwxgmac2_set_filter,
+	.safety_feat_config = dwxgmac3_safety_feat_config,
+	.safety_feat_irq_status = dwxgmac3_safety_feat_irq_status,
+	.safety_feat_dump = dwxgmac3_safety_feat_dump,
+	.set_mac_loopback = dwxgmac2_set_mac_loopback,
+	.rss_configure = dwxgmac2_rss_configure,
+	.rxp_config = dwxgmac3_rxp_config,
+	.get_mac_tx_timestamp = dwxgmac2_get_mac_tx_timestamp,
+	.flex_pps_config = dwxgmac2_flex_pps_config,
+	.sarc_configure = dwxgmac2_sarc_configure,
+	.config_l3_filter = dwxgmac2_config_l3_filter,
+	.config_l4_filter = dwxgmac2_config_l4_filter,
+	.set_arp_offload = dwxgmac2_set_arp_offload,
+	.fpe_map_preemption_class = ec_dwxgmac3_fpe_map_preemption_class,
+};
+
+int ec_dwxgmac2_setup(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	dev_info(priv->device, "\tXGMAC2\n");
+
+	priv->dev->priv_flags |= IFF_UNICAST_FLT;
+	mac->pcsr = priv->ioaddr;
+	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
+	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
+	mac->mcast_bits_log2 = 0;
+
+	if (mac->multicast_filter_bins)
+		mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins);
+
+	mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+			 MAC_10 | MAC_100 | MAC_1000FD |
+			 MAC_2500FD | MAC_5000FD | MAC_10000FD;
+	mac->link.duplex = 0;
+	mac->link.speed10 = XGMAC_CONFIG_SS_10_MII;
+	mac->link.speed100 = XGMAC_CONFIG_SS_100_MII;
+	mac->link.speed1000 = XGMAC_CONFIG_SS_1000_GMII;
+	mac->link.speed2500 = XGMAC_CONFIG_SS_2500_GMII;
+	mac->link.xgmii.speed2500 = XGMAC_CONFIG_SS_2500;
+	mac->link.xgmii.speed5000 = XGMAC_CONFIG_SS_5000;
+	mac->link.xgmii.speed10000 = XGMAC_CONFIG_SS_10000;
+	mac->link.speed_mask = XGMAC_CONFIG_SS_MASK;
+
+	mac->mii.addr = XGMAC_MDIO_ADDR;
+	mac->mii.data = XGMAC_MDIO_DATA;
+	mac->mii.addr_shift = 16;
+	mac->mii.addr_mask = GENMASK(20, 16);
+	mac->mii.reg_shift = 0;
+	mac->mii.reg_mask = GENMASK(15, 0);
+	mac->mii.clk_csr_shift = 19;
+	mac->mii.clk_csr_mask = GENMASK(21, 19);
+	mac->num_vlan = ec_stmmac_get_num_vlan(priv->ioaddr);
+
+	return 0;
+}
+
+int ec_dwxlgmac2_setup(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	dev_info(priv->device, "\tXLGMAC\n");
+
+	priv->dev->priv_flags |= IFF_UNICAST_FLT;
+	mac->pcsr = priv->ioaddr;
+	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
+	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
+	mac->mcast_bits_log2 = 0;
+
+	if (mac->multicast_filter_bins)
+		mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins);
+
+	mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+			 MAC_1000FD | MAC_2500FD | MAC_5000FD |
+			 MAC_10000FD | MAC_25000FD |
+			 MAC_40000FD | MAC_50000FD |
+			 MAC_100000FD;
+	mac->link.duplex = 0;
+	mac->link.speed1000 = XLGMAC_CONFIG_SS_1000;
+	mac->link.speed2500 = XLGMAC_CONFIG_SS_2500;
+	mac->link.xgmii.speed10000 = XLGMAC_CONFIG_SS_10G;
+	mac->link.xlgmii.speed25000 = XLGMAC_CONFIG_SS_25G;
+	mac->link.xlgmii.speed40000 = XLGMAC_CONFIG_SS_40G;
+	mac->link.xlgmii.speed50000 = XLGMAC_CONFIG_SS_50G;
+	mac->link.xlgmii.speed100000 = XLGMAC_CONFIG_SS_100G;
+	mac->link.speed_mask = XLGMAC_CONFIG_SS;
+
+	mac->mii.addr = XGMAC_MDIO_ADDR;
+	mac->mii.data = XGMAC_MDIO_DATA;
+	mac->mii.addr_shift = 16;
+	mac->mii.addr_mask = GENMASK(20, 16);
+	mac->mii.reg_shift = 0;
+	mac->mii.reg_mask = GENMASK(15, 0);
+	mac->mii.clk_csr_shift = 19;
+	mac->mii.clk_csr_mask = GENMASK(21, 19);
+
+	return 0;
+}
diff --git a/drivers/net/ethercat/device/K3/dwxgmac2_descs-ethercat.c b/drivers/net/ethercat/device/K3/dwxgmac2_descs-ethercat.c
new file mode 100644
index 000000000000..9047fc5c5323
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwxgmac2_descs-ethercat.c
@@ -0,0 +1,393 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac XGMAC support.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/stmmac.h>
+#include "common-ethercat.h"
+#include "dwxgmac2-ethercat.h"
+
+static int dwxgmac2_get_tx_status(struct stmmac_extra_stats *x,
+				  struct dma_desc *p, void __iomem *ioaddr)
+{
+	unsigned int tdes3 = le32_to_cpu(p->des3);
+	int ret = tx_done;
+
+	if (unlikely(tdes3 & XGMAC_TDES3_OWN))
+		return tx_dma_own;
+	if (likely(!(tdes3 & XGMAC_TDES3_LD)))
+		return tx_not_ls;
+
+	return ret;
+}
+
+static int dwxgmac2_get_rx_status(struct stmmac_extra_stats *x,
+				  struct dma_desc *p)
+{
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+
+	if (unlikely(rdes3 & XGMAC_RDES3_OWN))
+		return dma_own;
+	if (unlikely(rdes3 & XGMAC_RDES3_CTXT))
+		return discard_frame;
+	if (likely(!(rdes3 & XGMAC_RDES3_LD)))
+		return rx_not_ls;
+	if (unlikely((rdes3 & XGMAC_RDES3_ES) && (rdes3 & XGMAC_RDES3_LD)))
+		return discard_frame;
+
+	return good_frame;
+}
+
+static int dwxgmac2_get_tx_len(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des2) & XGMAC_TDES2_B1L);
+}
+
+static int dwxgmac2_get_tx_owner(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des3) & XGMAC_TDES3_OWN) > 0;
+}
+
+static void dwxgmac2_set_tx_owner(struct dma_desc *p)
+{
+	p->des3 |= cpu_to_le32(XGMAC_TDES3_OWN);
+}
+
+static void dwxgmac2_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
+{
+	u32 flags = XGMAC_RDES3_OWN;
+
+	if (!disable_rx_ic)
+		flags |= XGMAC_RDES3_IOC;
+
+	p->des3 |= cpu_to_le32(flags);
+}
+
+static int dwxgmac2_get_tx_ls(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des3) & XGMAC_RDES3_LD) > 0;
+}
+
+static u16 dwxgmac2_wrback_get_rx_vlan_tci(struct dma_desc *p)
+{
+	return le32_to_cpu(p->des0) & XGMAC_RDES0_VLAN_TAG_MASK;
+}
+
+static bool dwxgmac2_wrback_get_rx_vlan_valid(struct dma_desc *p)
+{
+	u32 et_lt;
+
+	et_lt = FIELD_GET(XGMAC_RDES3_ET_LT, le32_to_cpu(p->des3));
+
+	return et_lt >= XGMAC_ET_LT_VLAN_STAG &&
+	       et_lt <= XGMAC_ET_LT_DVLAN_STAG_CTAG;
+}
+
+static int dwxgmac2_get_rx_frame_len(struct dma_desc *p, int rx_coe)
+{
+	return (le32_to_cpu(p->des3) & XGMAC_RDES3_PL);
+}
+
+static void dwxgmac2_enable_tx_timestamp(struct dma_desc *p)
+{
+	p->des2 |= cpu_to_le32(XGMAC_TDES2_TTSE);
+}
+
+static int dwxgmac2_get_tx_timestamp_status(struct dma_desc *p)
+{
+	return 0; /* Not supported */
+}
+
+static inline void dwxgmac2_get_timestamp(void *desc, u32 ats, u64 *ts)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+	u64 ns = 0;
+
+	ns += le32_to_cpu(p->des1) * 1000000000ULL;
+	ns += le32_to_cpu(p->des0);
+
+	*ts = ns;
+}
+
+static int dwxgmac2_rx_check_timestamp(void *desc)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+	bool desc_valid, ts_valid;
+
+	dma_rmb();
+
+	desc_valid = !(rdes3 & XGMAC_RDES3_OWN) && (rdes3 & XGMAC_RDES3_CTXT);
+	ts_valid = !(rdes3 & XGMAC_RDES3_TSD) && (rdes3 & XGMAC_RDES3_TSA);
+
+	if (likely(desc_valid && ts_valid)) {
+		if ((p->des0 == 0xffffffff) && (p->des1 == 0xffffffff))
+			return -EINVAL;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int dwxgmac2_get_rx_timestamp_status(void *desc, void *next_desc,
+					    u32 ats)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+	int ret = -EBUSY;
+
+	if (likely(rdes3 & XGMAC_RDES3_CDA))
+		ret = dwxgmac2_rx_check_timestamp(next_desc);
+
+	return !ret;
+}
+
+static void dwxgmac2_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
+				  int mode, int end, int bfsize)
+{
+	dwxgmac2_set_rx_owner(p, disable_rx_ic);
+}
+
+static void dwxgmac2_init_tx_desc(struct dma_desc *p, int mode, int end)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+}
+
+static void dwxgmac2_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
+				     bool csum_flag, int mode, bool tx_own,
+				     bool ls, unsigned int tot_pkt_len)
+{
+	unsigned int tdes3 = le32_to_cpu(p->des3);
+
+	p->des2 |= cpu_to_le32(len & XGMAC_TDES2_B1L);
+
+	tdes3 |= tot_pkt_len & XGMAC_TDES3_FL;
+	if (is_fs)
+		tdes3 |= XGMAC_TDES3_FD;
+	else
+		tdes3 &= ~XGMAC_TDES3_FD;
+
+	if (csum_flag)
+		tdes3 |= 0x3 << XGMAC_TDES3_CIC_SHIFT;
+	else
+		tdes3 &= ~XGMAC_TDES3_CIC;
+
+	if (ls)
+		tdes3 |= XGMAC_TDES3_LD;
+	else
+		tdes3 &= ~XGMAC_TDES3_LD;
+
+	/* Finally set the OWN bit. Later the DMA will start! */
+	if (tx_own)
+		tdes3 |= XGMAC_TDES3_OWN;
+
+	if (is_fs && tx_own)
+		/* When the own bit, for the first frame, has to be set, all
+		 * descriptors for the same frame has to be set before, to
+		 * avoid race condition.
+		 */
+		dma_wmb();
+
+	p->des3 = cpu_to_le32(tdes3);
+}
+
+static void dwxgmac2_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
+					 int len1, int len2, bool tx_own,
+					 bool ls, unsigned int tcphdrlen,
+					 unsigned int tcppayloadlen)
+{
+	unsigned int tdes3 = le32_to_cpu(p->des3);
+
+	if (len1)
+		p->des2 |= cpu_to_le32(len1 & XGMAC_TDES2_B1L);
+	if (len2)
+		p->des2 |= cpu_to_le32((len2 << XGMAC_TDES2_B2L_SHIFT) &
+				XGMAC_TDES2_B2L);
+	if (is_fs) {
+		tdes3 |= XGMAC_TDES3_FD | XGMAC_TDES3_TSE;
+		tdes3 |= (tcphdrlen << XGMAC_TDES3_THL_SHIFT) &
+			XGMAC_TDES3_THL;
+		tdes3 |= tcppayloadlen & XGMAC_TDES3_TPL;
+	} else {
+		tdes3 &= ~XGMAC_TDES3_FD;
+	}
+
+	if (ls)
+		tdes3 |= XGMAC_TDES3_LD;
+	else
+		tdes3 &= ~XGMAC_TDES3_LD;
+
+	/* Finally set the OWN bit. Later the DMA will start! */
+	if (tx_own)
+		tdes3 |= XGMAC_TDES3_OWN;
+
+	if (is_fs && tx_own)
+		/* When the own bit, for the first frame, has to be set, all
+		 * descriptors for the same frame has to be set before, to
+		 * avoid race condition.
+		 */
+		dma_wmb();
+
+	p->des3 = cpu_to_le32(tdes3);
+}
+
+static void dwxgmac2_release_tx_desc(struct dma_desc *p, int mode)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+}
+
+static void dwxgmac2_set_tx_ic(struct dma_desc *p)
+{
+	p->des2 |= cpu_to_le32(XGMAC_TDES2_IOC);
+}
+
+static void dwxgmac2_set_mss(struct dma_desc *p, unsigned int mss)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = cpu_to_le32(mss);
+	p->des3 = cpu_to_le32(XGMAC_TDES3_CTXT | XGMAC_TDES3_TCMSSV);
+}
+
+static void dwxgmac2_set_addr(struct dma_desc *p, dma_addr_t addr)
+{
+	p->des0 = cpu_to_le32(lower_32_bits(addr));
+	p->des1 = cpu_to_le32(upper_32_bits(addr));
+}
+
+static void dwxgmac2_clear(struct dma_desc *p)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+}
+
+static int dwxgmac2_get_rx_hash(struct dma_desc *p, u32 *hash,
+				enum pkt_hash_types *type)
+{
+	unsigned int rdes3 = le32_to_cpu(p->des3);
+	u32 ptype;
+
+	if (rdes3 & XGMAC_RDES3_RSV) {
+		ptype = (rdes3 & XGMAC_RDES3_L34T) >> XGMAC_RDES3_L34T_SHIFT;
+
+		switch (ptype) {
+		case XGMAC_L34T_IP4TCP:
+		case XGMAC_L34T_IP4UDP:
+		case XGMAC_L34T_IP6TCP:
+		case XGMAC_L34T_IP6UDP:
+			*type = PKT_HASH_TYPE_L4;
+			break;
+		default:
+			*type = PKT_HASH_TYPE_L3;
+			break;
+		}
+
+		*hash = le32_to_cpu(p->des1);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static void dwxgmac2_get_rx_header_len(struct dma_desc *p, unsigned int *len)
+{
+	if (le32_to_cpu(p->des3) & XGMAC_RDES3_L34T)
+		*len = le32_to_cpu(p->des2) & XGMAC_RDES2_HL;
+}
+
+static void dwxgmac2_set_sec_addr(struct dma_desc *p, dma_addr_t addr, bool is_valid)
+{
+	p->des2 = cpu_to_le32(lower_32_bits(addr));
+	p->des3 = cpu_to_le32(upper_32_bits(addr));
+}
+
+static void dwxgmac2_set_sarc(struct dma_desc *p, u32 sarc_type)
+{
+	sarc_type <<= XGMAC_TDES3_SAIC_SHIFT;
+
+	p->des3 |= cpu_to_le32(sarc_type & XGMAC_TDES3_SAIC);
+}
+
+static void dwxgmac2_set_vlan_tag(struct dma_desc *p, u16 tag, u16 inner_tag,
+				  u32 inner_type)
+{
+	p->des0 = 0;
+	p->des1 = 0;
+	p->des2 = 0;
+	p->des3 = 0;
+
+	/* Inner VLAN */
+	if (inner_type) {
+		u32 des = inner_tag << XGMAC_TDES2_IVT_SHIFT;
+
+		des &= XGMAC_TDES2_IVT;
+		p->des2 = cpu_to_le32(des);
+
+		des = inner_type << XGMAC_TDES3_IVTIR_SHIFT;
+		des &= XGMAC_TDES3_IVTIR;
+		p->des3 = cpu_to_le32(des | XGMAC_TDES3_IVLTV);
+	}
+
+	/* Outer VLAN */
+	p->des3 |= cpu_to_le32(tag & XGMAC_TDES3_VT);
+	p->des3 |= cpu_to_le32(XGMAC_TDES3_VLTV);
+
+	p->des3 |= cpu_to_le32(XGMAC_TDES3_CTXT);
+}
+
+static void dwxgmac2_set_vlan(struct dma_desc *p, u32 type)
+{
+	type <<= XGMAC_TDES2_VTIR_SHIFT;
+	p->des2 |= cpu_to_le32(type & XGMAC_TDES2_VTIR);
+}
+
+static void dwxgmac2_set_tbs(struct dma_edesc *p, u32 sec, u32 nsec)
+{
+	p->des4 = cpu_to_le32((sec & XGMAC_TDES0_LT) | XGMAC_TDES0_LTV);
+	p->des5 = cpu_to_le32(nsec & XGMAC_TDES1_LT);
+	p->des6 = 0;
+	p->des7 = 0;
+}
+
+const struct stmmac_desc_ops ec_dwxgmac210_desc_ops = {
+	.tx_status = dwxgmac2_get_tx_status,
+	.rx_status = dwxgmac2_get_rx_status,
+	.get_tx_len = dwxgmac2_get_tx_len,
+	.get_tx_owner = dwxgmac2_get_tx_owner,
+	.set_tx_owner = dwxgmac2_set_tx_owner,
+	.set_rx_owner = dwxgmac2_set_rx_owner,
+	.get_tx_ls = dwxgmac2_get_tx_ls,
+	.get_rx_vlan_tci = dwxgmac2_wrback_get_rx_vlan_tci,
+	.get_rx_vlan_valid = dwxgmac2_wrback_get_rx_vlan_valid,
+	.get_rx_frame_len = dwxgmac2_get_rx_frame_len,
+	.enable_tx_timestamp = dwxgmac2_enable_tx_timestamp,
+	.get_tx_timestamp_status = dwxgmac2_get_tx_timestamp_status,
+	.get_rx_timestamp_status = dwxgmac2_get_rx_timestamp_status,
+	.get_timestamp = dwxgmac2_get_timestamp,
+	.set_tx_ic = dwxgmac2_set_tx_ic,
+	.prepare_tx_desc = dwxgmac2_prepare_tx_desc,
+	.prepare_tso_tx_desc = dwxgmac2_prepare_tso_tx_desc,
+	.release_tx_desc = dwxgmac2_release_tx_desc,
+	.init_rx_desc = dwxgmac2_init_rx_desc,
+	.init_tx_desc = dwxgmac2_init_tx_desc,
+	.set_mss = dwxgmac2_set_mss,
+	.set_addr = dwxgmac2_set_addr,
+	.clear = dwxgmac2_clear,
+	.get_rx_hash = dwxgmac2_get_rx_hash,
+	.get_rx_header_len = dwxgmac2_get_rx_header_len,
+	.set_sec_addr = dwxgmac2_set_sec_addr,
+	.set_sarc = dwxgmac2_set_sarc,
+	.set_vlan_tag = dwxgmac2_set_vlan_tag,
+	.set_vlan = dwxgmac2_set_vlan,
+	.set_tbs = dwxgmac2_set_tbs,
+};
diff --git a/drivers/net/ethercat/device/K3/dwxgmac2_dma-ethercat.c b/drivers/net/ethercat/device/K3/dwxgmac2_dma-ethercat.c
new file mode 100644
index 000000000000..45040af5e004
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwxgmac2_dma-ethercat.c
@@ -0,0 +1,642 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac XGMAC support.
+ */
+
+#include <linux/iopoll.h>
+#include "stmmac-ethercat.h"
+#include "dwxgmac2-ethercat.h"
+
+static int dwxgmac2_dma_reset(void __iomem *ioaddr)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_MODE);
+
+	/* DMA SW reset */
+	writel(value | XGMAC_SWR, ioaddr + XGMAC_DMA_MODE);
+
+	return readl_poll_timeout(ioaddr + XGMAC_DMA_MODE, value,
+				  !(value & XGMAC_SWR), 0, 100000);
+}
+
+static void dwxgmac2_dma_init(void __iomem *ioaddr,
+			      struct stmmac_dma_cfg *dma_cfg)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_SYSBUS_MODE);
+
+	if (dma_cfg->aal)
+		value |= XGMAC_AAL;
+
+	if (dma_cfg->eame)
+		value |= XGMAC_EAME;
+
+	writel(value, ioaddr + XGMAC_DMA_SYSBUS_MODE);
+}
+
+static void dwxgmac2_dma_init_chan(struct stmmac_priv *priv,
+				   void __iomem *ioaddr,
+				   struct stmmac_dma_cfg *dma_cfg, u32 chan)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_CH_CONTROL(chan));
+
+	if (dma_cfg->pblx8)
+		value |= XGMAC_PBLx8;
+
+	writel(value, ioaddr + XGMAC_DMA_CH_CONTROL(chan));
+	writel(XGMAC_DMA_INT_DEFAULT_EN, ioaddr + XGMAC_DMA_CH_INT_EN(chan));
+}
+
+static void dwxgmac2_dma_init_rx_chan(struct stmmac_priv *priv,
+				      void __iomem *ioaddr,
+				      struct stmmac_dma_cfg *dma_cfg,
+				      dma_addr_t phy, u32 chan)
+{
+	u32 rxpbl = dma_cfg->rxpbl ?: dma_cfg->pbl;
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+	value &= ~XGMAC_RxPBL;
+	value |= (rxpbl << XGMAC_RxPBL_SHIFT) & XGMAC_RxPBL;
+	writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+
+	writel(upper_32_bits(phy), ioaddr + XGMAC_DMA_CH_RxDESC_HADDR(chan));
+	writel(lower_32_bits(phy), ioaddr + XGMAC_DMA_CH_RxDESC_LADDR(chan));
+}
+
+static void dwxgmac2_dma_init_tx_chan(struct stmmac_priv *priv,
+				      void __iomem *ioaddr,
+				      struct stmmac_dma_cfg *dma_cfg,
+				      dma_addr_t phy, u32 chan)
+{
+	u32 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl;
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+	value &= ~XGMAC_TxPBL;
+	value |= (txpbl << XGMAC_TxPBL_SHIFT) & XGMAC_TxPBL;
+	value |= XGMAC_OSP;
+	writel(value, ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+
+	writel(upper_32_bits(phy), ioaddr + XGMAC_DMA_CH_TxDESC_HADDR(chan));
+	writel(lower_32_bits(phy), ioaddr + XGMAC_DMA_CH_TxDESC_LADDR(chan));
+}
+
+static void dwxgmac2_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_SYSBUS_MODE);
+	int i;
+
+	if (axi->axi_lpi_en)
+		value |= XGMAC_EN_LPI;
+	if (axi->axi_xit_frm)
+		value |= XGMAC_LPI_XIT_PKT;
+
+	value &= ~XGMAC_WR_OSR_LMT;
+	value |= (axi->axi_wr_osr_lmt << XGMAC_WR_OSR_LMT_SHIFT) &
+		XGMAC_WR_OSR_LMT;
+
+	value &= ~XGMAC_RD_OSR_LMT;
+	value |= (axi->axi_rd_osr_lmt << XGMAC_RD_OSR_LMT_SHIFT) &
+		XGMAC_RD_OSR_LMT;
+
+	if (!axi->axi_fb)
+		value |= XGMAC_UNDEF;
+
+	value &= ~XGMAC_BLEN;
+	for (i = 0; i < AXI_BLEN; i++) {
+		switch (axi->axi_blen[i]) {
+		case 256:
+			value |= XGMAC_BLEN256;
+			break;
+		case 128:
+			value |= XGMAC_BLEN128;
+			break;
+		case 64:
+			value |= XGMAC_BLEN64;
+			break;
+		case 32:
+			value |= XGMAC_BLEN32;
+			break;
+		case 16:
+			value |= XGMAC_BLEN16;
+			break;
+		case 8:
+			value |= XGMAC_BLEN8;
+			break;
+		case 4:
+			value |= XGMAC_BLEN4;
+			break;
+		}
+	}
+
+	writel(value, ioaddr + XGMAC_DMA_SYSBUS_MODE);
+	writel(XGMAC_TDPS, ioaddr + XGMAC_TX_EDMA_CTRL);
+	writel(XGMAC_RDPS, ioaddr + XGMAC_RX_EDMA_CTRL);
+}
+
+static void dwxgmac2_dma_dump_regs(struct stmmac_priv *priv,
+				   void __iomem *ioaddr, u32 *reg_space)
+{
+	int i;
+
+	for (i = (XGMAC_DMA_MODE / 4); i < XGMAC_REGSIZE; i++)
+		reg_space[i] = readl(ioaddr + i * 4);
+}
+
+static void dwxgmac2_dma_rx_mode(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 int mode, u32 channel, int fifosz, u8 qmode)
+{
+	u32 value = readl(ioaddr + XGMAC_MTL_RXQ_OPMODE(channel));
+	unsigned int rqs = fifosz / 256 - 1;
+
+	if (mode == SF_DMA_MODE) {
+		value |= XGMAC_RSF;
+	} else {
+		value &= ~XGMAC_RSF;
+		value &= ~XGMAC_RTC;
+
+		if (mode <= 64)
+			value |= 0x0 << XGMAC_RTC_SHIFT;
+		else if (mode <= 96)
+			value |= 0x2 << XGMAC_RTC_SHIFT;
+		else
+			value |= 0x3 << XGMAC_RTC_SHIFT;
+	}
+
+	value &= ~XGMAC_RQS;
+	value |= (rqs << XGMAC_RQS_SHIFT) & XGMAC_RQS;
+
+	if ((fifosz >= 4096) && (qmode != MTL_QUEUE_AVB)) {
+		u32 flow = readl(ioaddr + XGMAC_MTL_RXQ_FLOW_CONTROL(channel));
+		unsigned int rfd, rfa;
+
+		value |= XGMAC_EHFC;
+
+		/* Set Threshold for Activating Flow Control to min 2 frames,
+		 * i.e. 1500 * 2 = 3000 bytes.
+		 *
+		 * Set Threshold for Deactivating Flow Control to min 1 frame,
+		 * i.e. 1500 bytes.
+		 */
+		switch (fifosz) {
+		case 4096:
+			/* This violates the above formula because of FIFO size
+			 * limit therefore overflow may occur in spite of this.
+			 */
+			rfd = 0x03; /* Full-2.5K */
+			rfa = 0x01; /* Full-1.5K */
+			break;
+
+		default:
+			rfd = 0x07; /* Full-4.5K */
+			rfa = 0x04; /* Full-3K */
+			break;
+		}
+
+		flow &= ~XGMAC_RFD;
+		flow |= rfd << XGMAC_RFD_SHIFT;
+
+		flow &= ~XGMAC_RFA;
+		flow |= rfa << XGMAC_RFA_SHIFT;
+
+		writel(flow, ioaddr + XGMAC_MTL_RXQ_FLOW_CONTROL(channel));
+	}
+
+	writel(value, ioaddr + XGMAC_MTL_RXQ_OPMODE(channel));
+}
+
+static void dwxgmac2_dma_tx_mode(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 int mode, u32 channel, int fifosz, u8 qmode)
+{
+	u32 value = readl(ioaddr + XGMAC_MTL_TXQ_OPMODE(channel));
+	unsigned int tqs = fifosz / 256 - 1;
+
+	if (mode == SF_DMA_MODE) {
+		value |= XGMAC_TSF;
+	} else {
+		value &= ~XGMAC_TSF;
+		value &= ~XGMAC_TTC;
+
+		if (mode <= 64)
+			value |= 0x0 << XGMAC_TTC_SHIFT;
+		else if (mode <= 96)
+			value |= 0x2 << XGMAC_TTC_SHIFT;
+		else if (mode <= 128)
+			value |= 0x3 << XGMAC_TTC_SHIFT;
+		else if (mode <= 192)
+			value |= 0x4 << XGMAC_TTC_SHIFT;
+		else if (mode <= 256)
+			value |= 0x5 << XGMAC_TTC_SHIFT;
+		else if (mode <= 384)
+			value |= 0x6 << XGMAC_TTC_SHIFT;
+		else
+			value |= 0x7 << XGMAC_TTC_SHIFT;
+	}
+
+	/* Use static TC to Queue mapping */
+	value |= (channel << XGMAC_Q2TCMAP_SHIFT) & XGMAC_Q2TCMAP;
+
+	value &= ~XGMAC_TXQEN;
+	if (qmode != MTL_QUEUE_AVB)
+		value |= 0x2 << XGMAC_TXQEN_SHIFT;
+	else
+		value |= 0x1 << XGMAC_TXQEN_SHIFT;
+
+	value &= ~XGMAC_TQS;
+	value |= (tqs << XGMAC_TQS_SHIFT) & XGMAC_TQS;
+
+	writel(value, ioaddr +  XGMAC_MTL_TXQ_OPMODE(channel));
+}
+
+static void dwxgmac2_enable_dma_irq(struct stmmac_priv *priv,
+				    void __iomem *ioaddr, u32 chan,
+				    bool rx, bool tx)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_CH_INT_EN(chan));
+
+	if (rx)
+		value |= XGMAC_DMA_INT_DEFAULT_RX;
+	if (tx)
+		value |= XGMAC_DMA_INT_DEFAULT_TX;
+
+	writel(value, ioaddr + XGMAC_DMA_CH_INT_EN(chan));
+}
+
+static void dwxgmac2_disable_dma_irq(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 chan,
+				     bool rx, bool tx)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_CH_INT_EN(chan));
+
+	if (rx)
+		value &= ~XGMAC_DMA_INT_DEFAULT_RX;
+	if (tx)
+		value &= ~XGMAC_DMA_INT_DEFAULT_TX;
+
+	writel(value, ioaddr + XGMAC_DMA_CH_INT_EN(chan));
+}
+
+static void dwxgmac2_dma_start_tx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 chan)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+	value |= XGMAC_TXST;
+	writel(value, ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+
+	value = readl(ioaddr + XGMAC_TX_CONFIG);
+	value |= XGMAC_CONFIG_TE;
+	writel(value, ioaddr + XGMAC_TX_CONFIG);
+}
+
+static void dwxgmac2_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 chan)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+	value &= ~XGMAC_TXST;
+	writel(value, ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+
+	value = readl(ioaddr + XGMAC_TX_CONFIG);
+	value &= ~XGMAC_CONFIG_TE;
+	writel(value, ioaddr + XGMAC_TX_CONFIG);
+}
+
+static void dwxgmac2_dma_start_rx(struct stmmac_priv *priv,
+				  void __iomem *ioaddr, u32 chan)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+	value |= XGMAC_RXST;
+	writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+
+	value = readl(ioaddr + XGMAC_RX_CONFIG);
+	value |= XGMAC_CONFIG_RE;
+	writel(value, ioaddr + XGMAC_RX_CONFIG);
+}
+
+static void dwxgmac2_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 chan)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+	value &= ~XGMAC_RXST;
+	writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+}
+
+static int dwxgmac2_dma_interrupt(struct stmmac_priv *priv,
+				  void __iomem *ioaddr,
+				  struct stmmac_extra_stats *x, u32 chan,
+				  u32 dir)
+{
+	struct stmmac_pcpu_stats *stats = this_cpu_ptr(priv->xstats.pcpu_stats);
+	u32 intr_status = readl(ioaddr + XGMAC_DMA_CH_STATUS(chan));
+	u32 intr_en = readl(ioaddr + XGMAC_DMA_CH_INT_EN(chan));
+	int ret = 0;
+
+	if (dir == DMA_DIR_RX)
+		intr_status &= XGMAC_DMA_STATUS_MSK_RX;
+	else if (dir == DMA_DIR_TX)
+		intr_status &= XGMAC_DMA_STATUS_MSK_TX;
+
+	/* ABNORMAL interrupts */
+	if (unlikely(intr_status & XGMAC_AIS)) {
+		if (unlikely(intr_status & XGMAC_RBU)) {
+			x->rx_buf_unav_irq++;
+			ret |= handle_rx;
+		}
+		if (unlikely(intr_status & XGMAC_TPS)) {
+			x->tx_process_stopped_irq++;
+			ret |= tx_hard_error;
+		}
+		if (unlikely(intr_status & XGMAC_FBE)) {
+			x->fatal_bus_error_irq++;
+			ret |= tx_hard_error;
+		}
+	}
+
+	/* TX/RX NORMAL interrupts */
+	if (likely(intr_status & XGMAC_RI)) {
+		u64_stats_update_begin(&stats->syncp);
+		u64_stats_inc(&stats->rx_normal_irq_n[chan]);
+		u64_stats_update_end(&stats->syncp);
+		ret |= handle_rx;
+	}
+	if (likely(intr_status & (XGMAC_TI | XGMAC_TBU))) {
+		u64_stats_update_begin(&stats->syncp);
+		u64_stats_inc(&stats->tx_normal_irq_n[chan]);
+		u64_stats_update_end(&stats->syncp);
+		ret |= handle_tx;
+	}
+
+	/* Clear interrupts */
+	writel(intr_en & intr_status, ioaddr + XGMAC_DMA_CH_STATUS(chan));
+
+	return ret;
+}
+
+static int dwxgmac2_get_hw_feature(void __iomem *ioaddr,
+				   struct dma_features *dma_cap)
+{
+	struct stmmac_priv *priv;
+	u32 hw_cap;
+
+	priv = container_of(dma_cap, struct stmmac_priv, dma_cap);
+
+	/* MAC HW feature 0 */
+	hw_cap = readl(ioaddr + XGMAC_HW_FEATURE0);
+	dma_cap->edma = (hw_cap & XGMAC_HWFEAT_EDMA) >> 31;
+	dma_cap->ediffc = (hw_cap & XGMAC_HWFEAT_EDIFFC) >> 30;
+	dma_cap->vxn = (hw_cap & XGMAC_HWFEAT_VXN) >> 29;
+	dma_cap->vlins = (hw_cap & XGMAC_HWFEAT_SAVLANINS) >> 27;
+	dma_cap->tssrc = (hw_cap & XGMAC_HWFEAT_TSSTSSEL) >> 25;
+	dma_cap->multi_addr = (hw_cap & XGMAC_HWFEAT_ADDMACADRSEL) >> 18;
+	dma_cap->rx_coe = (hw_cap & XGMAC_HWFEAT_RXCOESEL) >> 16;
+	dma_cap->tx_coe = (hw_cap & XGMAC_HWFEAT_TXCOESEL) >> 14;
+	dma_cap->eee = (hw_cap & XGMAC_HWFEAT_EEESEL) >> 13;
+	dma_cap->atime_stamp = (hw_cap & XGMAC_HWFEAT_TSSEL) >> 12;
+	dma_cap->av = (hw_cap & XGMAC_HWFEAT_AVSEL) >> 11;
+	dma_cap->av &= !((hw_cap & XGMAC_HWFEAT_RAVSEL) >> 10);
+	dma_cap->arpoffsel = (hw_cap & XGMAC_HWFEAT_ARPOFFSEL) >> 9;
+	dma_cap->rmon = (hw_cap & XGMAC_HWFEAT_MMCSEL) >> 8;
+	dma_cap->pmt_magic_frame = (hw_cap & XGMAC_HWFEAT_MGKSEL) >> 7;
+	dma_cap->pmt_remote_wake_up = (hw_cap & XGMAC_HWFEAT_RWKSEL) >> 6;
+	dma_cap->sma_mdio = (hw_cap & XGMAC_HWFEAT_SMASEL) >> 5;
+	dma_cap->vlhash = (hw_cap & XGMAC_HWFEAT_VLHASH) >> 4;
+	dma_cap->half_duplex = (hw_cap & XGMAC_HWFEAT_HDSEL) >> 3;
+	dma_cap->mbps_1000 = (hw_cap & XGMAC_HWFEAT_GMIISEL) >> 1;
+	if (dma_cap->mbps_1000 && priv->synopsys_id >= DWXGMAC_CORE_2_20)
+		dma_cap->mbps_10_100 = 1;
+
+	/* MAC HW feature 1 */
+	hw_cap = readl(ioaddr + XGMAC_HW_FEATURE1);
+	dma_cap->l3l4fnum = (hw_cap & XGMAC_HWFEAT_L3L4FNUM) >> 27;
+	/* If L3L4FNUM < 8, then the number of L3L4 filters supported by
+	 * XGMAC is equal to L3L4FNUM. From L3L4FNUM >= 8 the number of
+	 * L3L4 filters goes on like 8, 16, 32, ... Current maximum of
+	 * L3L4FNUM = 10.
+	 */
+	if (dma_cap->l3l4fnum >= 8 && dma_cap->l3l4fnum <= 10)
+		dma_cap->l3l4fnum = 8 << (dma_cap->l3l4fnum - 8);
+	else if (dma_cap->l3l4fnum > 10)
+		dma_cap->l3l4fnum = 32;
+
+	dma_cap->hash_tb_sz = (hw_cap & XGMAC_HWFEAT_HASHTBLSZ) >> 24;
+	dma_cap->numtc = ((hw_cap & XGMAC_HWFEAT_NUMTC) >> 21) + 1;
+	dma_cap->rssen = (hw_cap & XGMAC_HWFEAT_RSSEN) >> 20;
+	dma_cap->dbgmem = (hw_cap & XGMAC_HWFEAT_DBGMEMA) >> 19;
+	dma_cap->tsoen = (hw_cap & XGMAC_HWFEAT_TSOEN) >> 18;
+	dma_cap->sphen = (hw_cap & XGMAC_HWFEAT_SPHEN) >> 17;
+	dma_cap->dcben = (hw_cap & XGMAC_HWFEAT_DCBEN) >> 16;
+
+	dma_cap->addr64 = (hw_cap & XGMAC_HWFEAT_ADDR64) >> 14;
+	switch (dma_cap->addr64) {
+	case 0:
+		dma_cap->addr64 = 32;
+		break;
+	case 1:
+		dma_cap->addr64 = 40;
+		break;
+	case 2:
+		dma_cap->addr64 = 48;
+		break;
+	default:
+		dma_cap->addr64 = 32;
+		break;
+	}
+
+	dma_cap->advthword = (hw_cap & XGMAC_HWFEAT_ADVTHWORD) >> 13;
+	dma_cap->ptoen = (hw_cap & XGMAC_HWFEAT_PTOEN) >> 12;
+	dma_cap->osten = (hw_cap & XGMAC_HWFEAT_OSTEN) >> 11;
+	dma_cap->tx_fifo_size =
+		128 << ((hw_cap & XGMAC_HWFEAT_TXFIFOSIZE) >> 6);
+	dma_cap->pfcen = (hw_cap & XGMAC_HWFEAT_PFCEN) >> 5;
+	dma_cap->rx_fifo_size =
+		128 << ((hw_cap & XGMAC_HWFEAT_RXFIFOSIZE) >> 0);
+
+	/* MAC HW feature 2 */
+	hw_cap = readl(ioaddr + XGMAC_HW_FEATURE2);
+	dma_cap->aux_snapshot_n = (hw_cap & XGMAC_HWFEAT_AUXSNAPNUM) >> 28;
+	dma_cap->pps_out_num = (hw_cap & XGMAC_HWFEAT_PPSOUTNUM) >> 24;
+	dma_cap->number_tx_channel =
+		((hw_cap & XGMAC_HWFEAT_TXCHCNT) >> 18) + 1;
+	dma_cap->number_rx_channel =
+		((hw_cap & XGMAC_HWFEAT_RXCHCNT) >> 12) + 1;
+	dma_cap->number_tx_queues =
+		((hw_cap & XGMAC_HWFEAT_TXQCNT) >> 6) + 1;
+	dma_cap->number_rx_queues =
+		((hw_cap & XGMAC_HWFEAT_RXQCNT) >> 0) + 1;
+
+	/* MAC HW feature 3 */
+	hw_cap = readl(ioaddr + XGMAC_HW_FEATURE3);
+	dma_cap->tbs_ch_num = ((hw_cap & XGMAC_HWFEAT_TBSCH) >> 28) + 1;
+	dma_cap->tbssel = (hw_cap & XGMAC_HWFEAT_TBSSEL) >> 27;
+	dma_cap->fpesel = (hw_cap & XGMAC_HWFEAT_FPESEL) >> 26;
+	dma_cap->sgfsel = (hw_cap & XGMAC_HWFEAT_SGFSEL) >> 25;
+	dma_cap->estwid = (hw_cap & XGMAC_HWFEAT_ESTWID) >> 23;
+	dma_cap->estdep = (hw_cap & XGMAC_HWFEAT_ESTDEP) >> 20;
+	dma_cap->estsel = (hw_cap & XGMAC_HWFEAT_ESTSEL) >> 19;
+	dma_cap->ttsfd = (hw_cap & XGMAC_HWFEAT_TTSFD) >> 16;
+	dma_cap->asp = (hw_cap & XGMAC_HWFEAT_ASP) >> 14;
+	dma_cap->dvlan = (hw_cap & XGMAC_HWFEAT_DVLAN) >> 13;
+	dma_cap->frpes = (hw_cap & XGMAC_HWFEAT_FRPES) >> 11;
+	dma_cap->frpbs = (hw_cap & XGMAC_HWFEAT_FRPPB) >> 9;
+	dma_cap->pou_ost_en = (hw_cap & XGMAC_HWFEAT_POUOST) >> 8;
+	dma_cap->frppipe_num = ((hw_cap & XGMAC_HWFEAT_FRPPIPE) >> 5) + 1;
+	dma_cap->cbtisel = (hw_cap & XGMAC_HWFEAT_CBTISEL) >> 4;
+	dma_cap->frpsel = (hw_cap & XGMAC_HWFEAT_FRPSEL) >> 3;
+	dma_cap->nrvf_num = (hw_cap & XGMAC_HWFEAT_NRVF) >> 0;
+
+	/* MAC HW feature 4 */
+	hw_cap = readl(ioaddr + XGMAC_HW_FEATURE4);
+	dma_cap->asp |= (hw_cap & XGMAC_HWFEAT_EASP) >> 2;
+	dma_cap->pcsel = (hw_cap & XGMAC_HWFEAT_PCSEL) >> 0;
+
+	return 0;
+}
+
+static void dwxgmac2_rx_watchdog(struct stmmac_priv *priv, void __iomem *ioaddr,
+				 u32 riwt, u32 queue)
+{
+	writel(riwt & XGMAC_RWT, ioaddr + XGMAC_DMA_CH_Rx_WATCHDOG(queue));
+}
+
+static void dwxgmac2_set_rx_ring_len(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 len, u32 chan)
+{
+	writel(len, ioaddr + XGMAC_DMA_CH_RxDESC_RING_LEN(chan));
+}
+
+static void dwxgmac2_set_tx_ring_len(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 len, u32 chan)
+{
+	writel(len, ioaddr + XGMAC_DMA_CH_TxDESC_RING_LEN(chan));
+}
+
+static void dwxgmac2_set_rx_tail_ptr(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 ptr, u32 chan)
+{
+	writel(ptr, ioaddr + XGMAC_DMA_CH_RxDESC_TAIL_LPTR(chan));
+}
+
+static void dwxgmac2_set_tx_tail_ptr(struct stmmac_priv *priv,
+				     void __iomem *ioaddr, u32 ptr, u32 chan)
+{
+	writel(ptr, ioaddr + XGMAC_DMA_CH_TxDESC_TAIL_LPTR(chan));
+}
+
+static void dwxgmac2_enable_tso(struct stmmac_priv *priv, void __iomem *ioaddr,
+				bool en, u32 chan)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+
+	if (en)
+		value |= XGMAC_TSE;
+	else
+		value &= ~XGMAC_TSE;
+
+	writel(value, ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+}
+
+static void dwxgmac2_qmode(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   u32 channel, u8 qmode)
+{
+	u32 value = readl(ioaddr + XGMAC_MTL_TXQ_OPMODE(channel));
+	u32 flow = readl(ioaddr + XGMAC_RX_FLOW_CTRL);
+
+	value &= ~XGMAC_TXQEN;
+	if (qmode != MTL_QUEUE_AVB) {
+		value |= 0x2 << XGMAC_TXQEN_SHIFT;
+		writel(0, ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(channel));
+	} else {
+		value |= 0x1 << XGMAC_TXQEN_SHIFT;
+		writel(flow & (~XGMAC_RFE), ioaddr + XGMAC_RX_FLOW_CTRL);
+	}
+
+	writel(value, ioaddr +  XGMAC_MTL_TXQ_OPMODE(channel));
+}
+
+static void dwxgmac2_set_bfsize(struct stmmac_priv *priv, void __iomem *ioaddr,
+				int bfsize, u32 chan)
+{
+	u32 value;
+
+	value = readl(ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+	value &= ~XGMAC_RBSZ;
+	value |= bfsize << XGMAC_RBSZ_SHIFT;
+	writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan));
+}
+
+static void dwxgmac2_enable_sph(struct stmmac_priv *priv, void __iomem *ioaddr,
+				bool en, u32 chan)
+{
+	u32 value = readl(ioaddr + XGMAC_RX_CONFIG);
+
+	value &= ~XGMAC_CONFIG_HDSMS;
+	value |= XGMAC_CONFIG_HDSMS_256; /* Segment max 256 bytes */
+	writel(value, ioaddr + XGMAC_RX_CONFIG);
+
+	value = readl(ioaddr + XGMAC_DMA_CH_CONTROL(chan));
+	if (en)
+		value |= XGMAC_SPH;
+	else
+		value &= ~XGMAC_SPH;
+	writel(value, ioaddr + XGMAC_DMA_CH_CONTROL(chan));
+}
+
+static int dwxgmac2_enable_tbs(struct stmmac_priv *priv, void __iomem *ioaddr,
+			       bool en, u32 chan)
+{
+	u32 value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+
+	if (en)
+		value |= XGMAC_EDSE;
+	else
+		value &= ~XGMAC_EDSE;
+
+	writel(value, ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan));
+
+	value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan)) & XGMAC_EDSE;
+	if (en && !value)
+		return -EIO;
+
+	writel(XGMAC_DEF_FTOS, ioaddr + XGMAC_DMA_TBS_CTRL0);
+	writel(XGMAC_DEF_FTOS, ioaddr + XGMAC_DMA_TBS_CTRL1);
+	writel(XGMAC_DEF_FTOS, ioaddr + XGMAC_DMA_TBS_CTRL2);
+	writel(XGMAC_DEF_FTOS, ioaddr + XGMAC_DMA_TBS_CTRL3);
+	return 0;
+}
+
+const struct stmmac_dma_ops ec_dwxgmac210_dma_ops = {
+	.reset = dwxgmac2_dma_reset,
+	.init = dwxgmac2_dma_init,
+	.init_chan = dwxgmac2_dma_init_chan,
+	.init_rx_chan = dwxgmac2_dma_init_rx_chan,
+	.init_tx_chan = dwxgmac2_dma_init_tx_chan,
+	.axi = dwxgmac2_dma_axi,
+	.dump_regs = dwxgmac2_dma_dump_regs,
+	.dma_rx_mode = dwxgmac2_dma_rx_mode,
+	.dma_tx_mode = dwxgmac2_dma_tx_mode,
+	.enable_dma_irq = dwxgmac2_enable_dma_irq,
+	.disable_dma_irq = dwxgmac2_disable_dma_irq,
+	.start_tx = dwxgmac2_dma_start_tx,
+	.stop_tx = dwxgmac2_dma_stop_tx,
+	.start_rx = dwxgmac2_dma_start_rx,
+	.stop_rx = dwxgmac2_dma_stop_rx,
+	.dma_interrupt = dwxgmac2_dma_interrupt,
+	.get_hw_feature = dwxgmac2_get_hw_feature,
+	.rx_watchdog = dwxgmac2_rx_watchdog,
+	.set_rx_ring_len = dwxgmac2_set_rx_ring_len,
+	.set_tx_ring_len = dwxgmac2_set_tx_ring_len,
+	.set_rx_tail_ptr = dwxgmac2_set_rx_tail_ptr,
+	.set_tx_tail_ptr = dwxgmac2_set_tx_tail_ptr,
+	.enable_tso = dwxgmac2_enable_tso,
+	.qmode = dwxgmac2_qmode,
+	.set_bfsize = dwxgmac2_set_bfsize,
+	.enable_sph = dwxgmac2_enable_sph,
+	.enable_tbs = dwxgmac2_enable_tbs,
+};
diff --git a/drivers/net/ethercat/device/K3/dwxlgmac2-ethercat.h b/drivers/net/ethercat/device/K3/dwxlgmac2-ethercat.h
new file mode 100644
index 000000000000..726090d49221
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/dwxlgmac2-ethercat.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2020 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare XLGMAC definitions.
+ */
+
+#ifndef __STMMAC_DWXLGMAC2_H__
+#define __STMMAC_DWXLGMAC2_H__
+
+/* MAC Registers */
+#define XLGMAC_CONFIG_SS		GENMASK(30, 28)
+#define XLGMAC_CONFIG_SS_SHIFT		28
+#define XLGMAC_CONFIG_SS_40G		(0x0 << XLGMAC_CONFIG_SS_SHIFT)
+#define XLGMAC_CONFIG_SS_25G		(0x1 << XLGMAC_CONFIG_SS_SHIFT)
+#define XLGMAC_CONFIG_SS_50G		(0x2 << XLGMAC_CONFIG_SS_SHIFT)
+#define XLGMAC_CONFIG_SS_100G		(0x3 << XLGMAC_CONFIG_SS_SHIFT)
+#define XLGMAC_CONFIG_SS_10G		(0x4 << XLGMAC_CONFIG_SS_SHIFT)
+#define XLGMAC_CONFIG_SS_2500		(0x6 << XLGMAC_CONFIG_SS_SHIFT)
+#define XLGMAC_CONFIG_SS_1000		(0x7 << XLGMAC_CONFIG_SS_SHIFT)
+#define XLGMAC_RXQ_ENABLE_CTRL0		0x00000140
+
+#endif /* __STMMAC_DWXLGMAC2_H__ */
diff --git a/drivers/net/ethercat/device/K3/enh_desc-ethercat.c b/drivers/net/ethercat/device/K3/enh_desc-ethercat.c
new file mode 100644
index 000000000000..203843fc0cb5
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/enh_desc-ethercat.c
@@ -0,0 +1,470 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This contains the functions to handle the enhanced descriptors.
+
+  Copyright (C) 2007-2014  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/stmmac.h>
+#include "common-ethercat.h"
+#include "descs_com-ethercat.h"
+
+static int enh_desc_get_tx_status(struct stmmac_extra_stats *x,
+				  struct dma_desc *p, void __iomem *ioaddr)
+{
+	unsigned int tdes0 = le32_to_cpu(p->des0);
+	int ret = tx_done;
+
+	/* Get tx owner first */
+	if (unlikely(tdes0 & ETDES0_OWN))
+		return tx_dma_own;
+
+	/* Verify tx error by looking at the last segment. */
+	if (likely(!(tdes0 & ETDES0_LAST_SEGMENT)))
+		return tx_not_ls;
+
+	if (unlikely(tdes0 & ETDES0_ERROR_SUMMARY)) {
+		if (unlikely(tdes0 & ETDES0_JABBER_TIMEOUT))
+			x->tx_jabber++;
+
+		if (unlikely(tdes0 & ETDES0_FRAME_FLUSHED)) {
+			x->tx_frame_flushed++;
+			ec_dwmac_dma_flush_tx_fifo(ioaddr);
+		}
+
+		if (unlikely(tdes0 & ETDES0_LOSS_CARRIER)) {
+			x->tx_losscarrier++;
+		}
+		if (unlikely(tdes0 & ETDES0_NO_CARRIER)) {
+			x->tx_carrier++;
+		}
+		if (unlikely((tdes0 & ETDES0_LATE_COLLISION) ||
+			     (tdes0 & ETDES0_EXCESSIVE_COLLISIONS)))
+			x->tx_collision +=
+				(tdes0 & ETDES0_COLLISION_COUNT_MASK) >> 3;
+
+		if (unlikely(tdes0 & ETDES0_EXCESSIVE_DEFERRAL))
+			x->tx_deferred++;
+
+		if (unlikely(tdes0 & ETDES0_UNDERFLOW_ERROR)) {
+			ec_dwmac_dma_flush_tx_fifo(ioaddr);
+			x->tx_underflow++;
+		}
+
+		if (unlikely(tdes0 & ETDES0_IP_HEADER_ERROR))
+			x->tx_ip_header_error++;
+
+		if (unlikely(tdes0 & ETDES0_PAYLOAD_ERROR)) {
+			x->tx_payload_error++;
+			ec_dwmac_dma_flush_tx_fifo(ioaddr);
+		}
+
+		ret = tx_err;
+	}
+
+	if (unlikely(tdes0 & ETDES0_DEFERRED))
+		x->tx_deferred++;
+
+#ifdef STMMAC_VLAN_TAG_USED
+	if (tdes0 & ETDES0_VLAN_FRAME)
+		x->tx_vlan++;
+#endif
+
+	return ret;
+}
+
+static int enh_desc_get_tx_len(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des1) & ETDES1_BUFFER1_SIZE_MASK);
+}
+
+static int enh_desc_coe_rdes0(int ipc_err, int type, int payload_err)
+{
+	int ret = good_frame;
+	u32 status = (type << 2 | ipc_err << 1 | payload_err) & 0x7;
+
+	/* bits 5 7 0 | Frame status
+	 * ----------------------------------------------------------
+	 *      0 0 0 | IEEE 802.3 Type frame (length < 1536 octects)
+	 *      1 0 0 | IPv4/6 No CSUM errorS.
+	 *      1 0 1 | IPv4/6 CSUM PAYLOAD error
+	 *      1 1 0 | IPv4/6 CSUM IP HR error
+	 *      1 1 1 | IPv4/6 IP PAYLOAD AND HEADER errorS
+	 *      0 0 1 | IPv4/6 unsupported IP PAYLOAD
+	 *      0 1 1 | COE bypassed.. no IPv4/6 frame
+	 *      0 1 0 | Reserved.
+	 */
+	if (status == 0x0)
+		ret = llc_snap;
+	else if (status == 0x4)
+		ret = good_frame;
+	else if (status == 0x5)
+		ret = csum_none;
+	else if (status == 0x6)
+		ret = csum_none;
+	else if (status == 0x7)
+		ret = csum_none;
+	else if (status == 0x1)
+		ret = discard_frame;
+	else if (status == 0x3)
+		ret = discard_frame;
+	return ret;
+}
+
+static void enh_desc_get_ext_status(struct stmmac_extra_stats *x,
+				    struct dma_extended_desc *p)
+{
+	unsigned int rdes0 = le32_to_cpu(p->basic.des0);
+	unsigned int rdes4 = le32_to_cpu(p->des4);
+
+	if (unlikely(rdes0 & ERDES0_RX_MAC_ADDR)) {
+		int message_type = (rdes4 & ERDES4_MSG_TYPE_MASK) >> 8;
+
+		if (rdes4 & ERDES4_IP_HDR_ERR)
+			x->ip_hdr_err++;
+		if (rdes4 & ERDES4_IP_PAYLOAD_ERR)
+			x->ip_payload_err++;
+		if (rdes4 & ERDES4_IP_CSUM_BYPASSED)
+			x->ip_csum_bypassed++;
+		if (rdes4 & ERDES4_IPV4_PKT_RCVD)
+			x->ipv4_pkt_rcvd++;
+		if (rdes4 & ERDES4_IPV6_PKT_RCVD)
+			x->ipv6_pkt_rcvd++;
+
+		if (message_type == RDES_EXT_NO_PTP)
+			x->no_ptp_rx_msg_type_ext++;
+		else if (message_type == RDES_EXT_SYNC)
+			x->ptp_rx_msg_type_sync++;
+		else if (message_type == RDES_EXT_FOLLOW_UP)
+			x->ptp_rx_msg_type_follow_up++;
+		else if (message_type == RDES_EXT_DELAY_REQ)
+			x->ptp_rx_msg_type_delay_req++;
+		else if (message_type == RDES_EXT_DELAY_RESP)
+			x->ptp_rx_msg_type_delay_resp++;
+		else if (message_type == RDES_EXT_PDELAY_REQ)
+			x->ptp_rx_msg_type_pdelay_req++;
+		else if (message_type == RDES_EXT_PDELAY_RESP)
+			x->ptp_rx_msg_type_pdelay_resp++;
+		else if (message_type == RDES_EXT_PDELAY_FOLLOW_UP)
+			x->ptp_rx_msg_type_pdelay_follow_up++;
+		else if (message_type == RDES_PTP_ANNOUNCE)
+			x->ptp_rx_msg_type_announce++;
+		else if (message_type == RDES_PTP_MANAGEMENT)
+			x->ptp_rx_msg_type_management++;
+		else if (message_type == RDES_PTP_PKT_RESERVED_TYPE)
+			x->ptp_rx_msg_pkt_reserved_type++;
+
+		if (rdes4 & ERDES4_PTP_FRAME_TYPE)
+			x->ptp_frame_type++;
+		if (rdes4 & ERDES4_PTP_VER)
+			x->ptp_ver++;
+		if (rdes4 & ERDES4_TIMESTAMP_DROPPED)
+			x->timestamp_dropped++;
+		if (rdes4 & ERDES4_AV_PKT_RCVD)
+			x->av_pkt_rcvd++;
+		if (rdes4 & ERDES4_AV_TAGGED_PKT_RCVD)
+			x->av_tagged_pkt_rcvd++;
+		if ((rdes4 & ERDES4_VLAN_TAG_PRI_VAL_MASK) >> 18)
+			x->vlan_tag_priority_val++;
+		if (rdes4 & ERDES4_L3_FILTER_MATCH)
+			x->l3_filter_match++;
+		if (rdes4 & ERDES4_L4_FILTER_MATCH)
+			x->l4_filter_match++;
+		if ((rdes4 & ERDES4_L3_L4_FILT_NO_MATCH_MASK) >> 26)
+			x->l3_l4_filter_no_match++;
+	}
+}
+
+static int enh_desc_get_rx_status(struct stmmac_extra_stats *x,
+				  struct dma_desc *p)
+{
+	unsigned int rdes0 = le32_to_cpu(p->des0);
+	int ret = good_frame;
+
+	if (unlikely(rdes0 & RDES0_OWN))
+		return dma_own;
+
+	if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) {
+		x->rx_length++;
+		return discard_frame;
+	}
+
+	if (unlikely(rdes0 & RDES0_ERROR_SUMMARY)) {
+		if (unlikely(rdes0 & RDES0_DESCRIPTOR_ERROR)) {
+			x->rx_desc++;
+			x->rx_length++;
+		}
+		if (unlikely(rdes0 & RDES0_OVERFLOW_ERROR))
+			x->rx_gmac_overflow++;
+
+		if (unlikely(rdes0 & RDES0_IPC_CSUM_ERROR))
+			pr_err("\tIPC Csum Error/Giant frame\n");
+
+		if (unlikely(rdes0 & RDES0_COLLISION))
+			x->rx_collision++;
+		if (unlikely(rdes0 & RDES0_RECEIVE_WATCHDOG))
+			x->rx_watchdog++;
+
+		if (unlikely(rdes0 & RDES0_MII_ERROR))	/* GMII */
+			x->rx_mii++;
+
+		if (unlikely(rdes0 & RDES0_CRC_ERROR)) {
+			x->rx_crc_errors++;
+		}
+		ret = discard_frame;
+	}
+
+	/* After a payload csum error, the ES bit is set.
+	 * It doesn't match with the information reported into the databook.
+	 * At any rate, we need to understand if the CSUM hw computation is ok
+	 * and report this info to the upper layers. */
+	if (likely(ret == good_frame))
+		ret = enh_desc_coe_rdes0(!!(rdes0 & RDES0_IPC_CSUM_ERROR),
+					 !!(rdes0 & RDES0_FRAME_TYPE),
+					 !!(rdes0 & ERDES0_RX_MAC_ADDR));
+
+	if (unlikely(rdes0 & RDES0_DRIBBLING))
+		x->dribbling_bit++;
+
+	if (unlikely(rdes0 & RDES0_SA_FILTER_FAIL)) {
+		x->sa_rx_filter_fail++;
+		ret = discard_frame;
+	}
+	if (unlikely(rdes0 & RDES0_DA_FILTER_FAIL)) {
+		x->da_rx_filter_fail++;
+		ret = discard_frame;
+	}
+	if (unlikely(rdes0 & RDES0_LENGTH_ERROR)) {
+		x->rx_length++;
+		ret = discard_frame;
+	}
+#ifdef STMMAC_VLAN_TAG_USED
+	if (rdes0 & RDES0_VLAN_TAG)
+		x->rx_vlan++;
+#endif
+
+	return ret;
+}
+
+static void enh_desc_init_rx_desc(struct dma_desc *p, int disable_rx_ic,
+				  int mode, int end, int bfsize)
+{
+	int bfsize1;
+
+	p->des0 |= cpu_to_le32(RDES0_OWN);
+
+	bfsize1 = min(bfsize, BUF_SIZE_8KiB);
+	p->des1 |= cpu_to_le32(bfsize1 & ERDES1_BUFFER1_SIZE_MASK);
+
+	if (mode == STMMAC_CHAIN_MODE)
+		ehn_desc_rx_set_on_chain(p);
+	else
+		ehn_desc_rx_set_on_ring(p, end, bfsize);
+
+	if (disable_rx_ic)
+		p->des1 |= cpu_to_le32(ERDES1_DISABLE_IC);
+}
+
+static void enh_desc_init_tx_desc(struct dma_desc *p, int mode, int end)
+{
+	p->des0 &= cpu_to_le32(~ETDES0_OWN);
+	if (mode == STMMAC_CHAIN_MODE)
+		enh_desc_end_tx_desc_on_chain(p);
+	else
+		enh_desc_end_tx_desc_on_ring(p, end);
+}
+
+static int enh_desc_get_tx_owner(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des0) & ETDES0_OWN) >> 31;
+}
+
+static void enh_desc_set_tx_owner(struct dma_desc *p)
+{
+	p->des0 |= cpu_to_le32(ETDES0_OWN);
+}
+
+static void enh_desc_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
+{
+	p->des0 |= cpu_to_le32(RDES0_OWN);
+}
+
+static int enh_desc_get_tx_ls(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des0) & ETDES0_LAST_SEGMENT) >> 29;
+}
+
+static void enh_desc_release_tx_desc(struct dma_desc *p, int mode)
+{
+	int ter = (le32_to_cpu(p->des0) & ETDES0_END_RING) >> 21;
+
+	memset(p, 0, offsetof(struct dma_desc, des2));
+	if (mode == STMMAC_CHAIN_MODE)
+		enh_desc_end_tx_desc_on_chain(p);
+	else
+		enh_desc_end_tx_desc_on_ring(p, ter);
+}
+
+static void enh_desc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
+				     bool csum_flag, int mode, bool tx_own,
+				     bool ls, unsigned int tot_pkt_len)
+{
+	unsigned int tdes0 = le32_to_cpu(p->des0);
+
+	if (mode == STMMAC_CHAIN_MODE)
+		enh_set_tx_desc_len_on_chain(p, len);
+	else
+		enh_set_tx_desc_len_on_ring(p, len);
+
+	if (is_fs)
+		tdes0 |= ETDES0_FIRST_SEGMENT;
+	else
+		tdes0 &= ~ETDES0_FIRST_SEGMENT;
+
+	if (likely(csum_flag))
+		tdes0 |= (TX_CIC_FULL << ETDES0_CHECKSUM_INSERTION_SHIFT);
+	else
+		tdes0 &= ~(TX_CIC_FULL << ETDES0_CHECKSUM_INSERTION_SHIFT);
+
+	if (ls)
+		tdes0 |= ETDES0_LAST_SEGMENT;
+
+	/* Finally set the OWN bit. Later the DMA will start! */
+	if (tx_own)
+		tdes0 |= ETDES0_OWN;
+
+	if (is_fs && tx_own)
+		/* When the own bit, for the first frame, has to be set, all
+		 * descriptors for the same frame has to be set before, to
+		 * avoid race condition.
+		 */
+		dma_wmb();
+
+	p->des0 = cpu_to_le32(tdes0);
+}
+
+static void enh_desc_set_tx_ic(struct dma_desc *p)
+{
+	p->des0 |= cpu_to_le32(ETDES0_INTERRUPT);
+}
+
+static int enh_desc_get_rx_frame_len(struct dma_desc *p, int rx_coe_type)
+{
+	unsigned int csum = 0;
+	/* The type-1 checksum offload engines append the checksum at
+	 * the end of frame and the two bytes of checksum are added in
+	 * the length.
+	 * Adjust for that in the framelen for type-1 checksum offload
+	 * engines.
+	 */
+	if (rx_coe_type == STMMAC_RX_COE_TYPE1)
+		csum = 2;
+
+	return (((le32_to_cpu(p->des0) & RDES0_FRAME_LEN_MASK)
+				>> RDES0_FRAME_LEN_SHIFT) - csum);
+}
+
+static void enh_desc_enable_tx_timestamp(struct dma_desc *p)
+{
+	p->des0 |= cpu_to_le32(ETDES0_TIME_STAMP_ENABLE);
+}
+
+static int enh_desc_get_tx_timestamp_status(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des0) & ETDES0_TIME_STAMP_STATUS) >> 17;
+}
+
+static void enh_desc_get_timestamp(void *desc, u32 ats, u64 *ts)
+{
+	u64 ns;
+
+	if (ats) {
+		struct dma_extended_desc *p = (struct dma_extended_desc *)desc;
+		ns = le32_to_cpu(p->des6);
+		/* convert high/sec time stamp value to nanosecond */
+		ns += le32_to_cpu(p->des7) * 1000000000ULL;
+	} else {
+		struct dma_desc *p = (struct dma_desc *)desc;
+		ns = le32_to_cpu(p->des2);
+		ns += le32_to_cpu(p->des3) * 1000000000ULL;
+	}
+
+	*ts = ns;
+}
+
+static int enh_desc_get_rx_timestamp_status(void *desc, void *next_desc,
+					    u32 ats)
+{
+	if (ats) {
+		struct dma_extended_desc *p = (struct dma_extended_desc *)desc;
+		return (le32_to_cpu(p->basic.des0) & RDES0_IPC_CSUM_ERROR) >> 7;
+	} else {
+		struct dma_desc *p = (struct dma_desc *)desc;
+		if ((le32_to_cpu(p->des2) == 0xffffffff) &&
+		    (le32_to_cpu(p->des3) == 0xffffffff))
+			/* timestamp is corrupted, hence don't store it */
+			return 0;
+		else
+			return 1;
+	}
+}
+
+static void enh_desc_display_ring(void *head, unsigned int size, bool rx,
+				  dma_addr_t dma_rx_phy, unsigned int desc_size)
+{
+	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
+	dma_addr_t dma_addr;
+	int i;
+
+	pr_info("Extended %s descriptor ring:\n", rx ? "RX" : "TX");
+
+	for (i = 0; i < size; i++) {
+		u64 x;
+		dma_addr = dma_rx_phy + i * sizeof(*ep);
+
+		x = *(u64 *)ep;
+		pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
+			i, &dma_addr,
+			(unsigned int)x, (unsigned int)(x >> 32),
+			ep->basic.des2, ep->basic.des3);
+		ep++;
+	}
+	pr_info("\n");
+}
+
+static void enh_desc_set_addr(struct dma_desc *p, dma_addr_t addr)
+{
+	p->des2 = cpu_to_le32(addr);
+}
+
+static void enh_desc_clear(struct dma_desc *p)
+{
+	p->des2 = 0;
+}
+
+const struct stmmac_desc_ops ec_enh_desc_ops = {
+	.tx_status = enh_desc_get_tx_status,
+	.rx_status = enh_desc_get_rx_status,
+	.get_tx_len = enh_desc_get_tx_len,
+	.init_rx_desc = enh_desc_init_rx_desc,
+	.init_tx_desc = enh_desc_init_tx_desc,
+	.get_tx_owner = enh_desc_get_tx_owner,
+	.release_tx_desc = enh_desc_release_tx_desc,
+	.prepare_tx_desc = enh_desc_prepare_tx_desc,
+	.set_tx_ic = enh_desc_set_tx_ic,
+	.get_tx_ls = enh_desc_get_tx_ls,
+	.set_tx_owner = enh_desc_set_tx_owner,
+	.set_rx_owner = enh_desc_set_rx_owner,
+	.get_rx_frame_len = enh_desc_get_rx_frame_len,
+	.rx_extended_status = enh_desc_get_ext_status,
+	.enable_tx_timestamp = enh_desc_enable_tx_timestamp,
+	.get_tx_timestamp_status = enh_desc_get_tx_timestamp_status,
+	.get_timestamp = enh_desc_get_timestamp,
+	.get_rx_timestamp_status = enh_desc_get_rx_timestamp_status,
+	.display_ring = enh_desc_display_ring,
+	.set_addr = enh_desc_set_addr,
+	.clear = enh_desc_clear,
+};
diff --git a/drivers/net/ethercat/device/K3/hwif-ethercat.c b/drivers/net/ethercat/device/K3/hwif-ethercat.c
new file mode 100644
index 000000000000..58fde269f8b2
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/hwif-ethercat.c
@@ -0,0 +1,405 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac HW Interface Handling
+ */
+
+#include "common-ethercat.h"
+#include "stmmac-ethercat.h"
+#include "stmmac_fpe-ethercat.h"
+#include "stmmac_ptp-ethercat.h"
+#include "stmmac_est-ethercat.h"
+#include "stmmac_vlan-ethercat.h"
+#include "dwmac4_descs-ethercat.h"
+#include "dwxgmac2-ethercat.h"
+
+static u32 stmmac_get_id(struct stmmac_priv *priv, u32 id_reg)
+{
+	u32 reg = readl(priv->ioaddr + id_reg);
+
+	if (!reg) {
+		dev_info(priv->device, "Version ID not available\n");
+		return 0x0;
+	}
+
+	dev_info(priv->device, "User ID: 0x%x, Synopsys ID: 0x%x\n",
+			(unsigned int)(reg & GENMASK(15, 8)) >> 8,
+			(unsigned int)(reg & GENMASK(7, 0)));
+	return reg & GENMASK(7, 0);
+}
+
+static u32 stmmac_get_dev_id(struct stmmac_priv *priv, u32 id_reg)
+{
+	u32 reg = readl(priv->ioaddr + id_reg);
+
+	if (!reg) {
+		dev_info(priv->device, "Version ID not available\n");
+		return 0x0;
+	}
+
+	return (reg & GENMASK(15, 8)) >> 8;
+}
+
+static void stmmac_dwmac_mode_quirk(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	if (priv->chain_mode) {
+		dev_info(priv->device, "Chain mode enabled\n");
+		priv->mode = STMMAC_CHAIN_MODE;
+		mac->mode = &ec_chain_mode_ops;
+	} else {
+		dev_info(priv->device, "Ring mode enabled\n");
+		priv->mode = STMMAC_RING_MODE;
+		mac->mode = &ec_ring_mode_ops;
+	}
+}
+
+static int stmmac_dwmac1_quirks(struct stmmac_priv *priv)
+{
+	struct mac_device_info *mac = priv->hw;
+
+	if (priv->plat->enh_desc) {
+		dev_info(priv->device, "Enhanced/Alternate descriptors\n");
+
+		/* GMAC older than 3.50 has no extended descriptors */
+		if (priv->synopsys_id >= DWMAC_CORE_3_50) {
+			dev_info(priv->device, "Enabled extended descriptors\n");
+			priv->extend_desc = 1;
+		} else {
+			dev_warn(priv->device, "Extended descriptors not supported\n");
+		}
+
+		mac->desc = &ec_enh_desc_ops;
+	} else {
+		dev_info(priv->device, "Normal descriptors\n");
+		mac->desc = &ec_ndesc_ops;
+	}
+
+	stmmac_dwmac_mode_quirk(priv);
+	return 0;
+}
+
+static int stmmac_dwmac4_quirks(struct stmmac_priv *priv)
+{
+	stmmac_dwmac_mode_quirk(priv);
+	return 0;
+}
+
+static int stmmac_dwxlgmac_quirks(struct stmmac_priv *priv)
+{
+	priv->hw->xlgmac = true;
+	return 0;
+}
+
+int ec_stmmac_reset(struct stmmac_priv *priv, void __iomem *ioaddr)
+{
+	struct plat_stmmacenet_data *plat = priv ? priv->plat : NULL;
+
+	if (!priv)
+		return -EINVAL;
+
+	if (plat && plat->fix_soc_reset)
+		return plat->fix_soc_reset(priv, ioaddr);
+
+	return stmmac_do_callback(priv, dma, reset, ioaddr);
+}
+
+static const struct stmmac_hwif_entry {
+	bool gmac;
+	bool gmac4;
+	bool xgmac;
+	u32 min_id;
+	u32 dev_id;
+	const struct stmmac_regs_off regs;
+	const void *desc;
+	const void *dma;
+	const void *mac;
+	const void *hwtimestamp;
+	const void *ptp;
+	const void *mode;
+	const void *tc;
+	const void *mmc;
+	const void *est;
+	const void *vlan;
+	int (*setup)(struct stmmac_priv *priv);
+	int (*quirks)(struct stmmac_priv *priv);
+} stmmac_hw[] = {
+	/* NOTE: New HW versions shall go to the end of this table */
+	{
+		.gmac = false,
+		.gmac4 = false,
+		.xgmac = false,
+		.min_id = 0,
+		.regs = {
+			.ptp_off = PTP_GMAC3_X_OFFSET,
+			.mmc_off = MMC_GMAC3_X_OFFSET,
+		},
+		.desc = NULL,
+		.dma = &ec_dwmac100_dma_ops,
+		.mac = &ec_dwmac100_ops,
+		.hwtimestamp = &ec_dwmac1000_ptp,
+		.ptp = &ec_dwmac1000_ptp_clock_ops,
+		.mode = NULL,
+		.tc = NULL,
+		.mmc = &ec_dwmac_mmc_ops,
+		.setup = ec_dwmac100_setup,
+		.quirks = stmmac_dwmac1_quirks,
+	}, {
+		.gmac = true,
+		.gmac4 = false,
+		.xgmac = false,
+		.min_id = 0,
+		.regs = {
+			.ptp_off = PTP_GMAC3_X_OFFSET,
+			.mmc_off = MMC_GMAC3_X_OFFSET,
+		},
+		.desc = NULL,
+		.dma = &ec_dwmac1000_dma_ops,
+		.mac = &ec_dwmac1000_ops,
+		.hwtimestamp = &ec_dwmac1000_ptp,
+		.ptp = &ec_dwmac1000_ptp_clock_ops,
+		.mode = NULL,
+		.tc = NULL,
+		.mmc = &ec_dwmac_mmc_ops,
+		.setup = ec_dwmac1000_setup,
+		.quirks = stmmac_dwmac1_quirks,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.xgmac = false,
+		.min_id = 0,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+			.est_off = EST_GMAC4_OFFSET,
+		},
+		.desc = &ec_dwmac4_desc_ops,
+		.dma = &ec_dwmac4_dma_ops,
+		.mac = &ec_dwmac4_ops,
+		.vlan = &ec_dwmac_vlan_ops,
+		.hwtimestamp = &ec_stmmac_ptp,
+		.ptp = &ec_stmmac_ptp_clock_ops,
+		.mode = NULL,
+		.tc = &ec_dwmac4_tc_ops,
+		.mmc = &ec_dwmac_mmc_ops,
+		.est = &ec_dwmac510_est_ops,
+		.setup = ec_dwmac4_setup,
+		.quirks = stmmac_dwmac4_quirks,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.xgmac = false,
+		.min_id = DWMAC_CORE_4_00,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+			.est_off = EST_GMAC4_OFFSET,
+			.fpe_reg = &ec_dwmac5_fpe_reg,
+		},
+		.desc = &ec_dwmac4_desc_ops,
+		.dma = &ec_dwmac4_dma_ops,
+		.mac = &ec_dwmac410_ops,
+		.vlan = &ec_dwmac_vlan_ops,
+		.hwtimestamp = &ec_stmmac_ptp,
+		.ptp = &ec_stmmac_ptp_clock_ops,
+		.mode = &ec_dwmac4_ring_mode_ops,
+		.tc = &ec_dwmac510_tc_ops,
+		.mmc = &ec_dwmac_mmc_ops,
+		.est = &ec_dwmac510_est_ops,
+		.setup = ec_dwmac4_setup,
+		.quirks = NULL,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.xgmac = false,
+		.min_id = DWMAC_CORE_4_10,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+			.est_off = EST_GMAC4_OFFSET,
+			.fpe_reg = &ec_dwmac5_fpe_reg,
+		},
+		.desc = &ec_dwmac4_desc_ops,
+		.dma = &ec_dwmac410_dma_ops,
+		.mac = &ec_dwmac410_ops,
+		.vlan = &ec_dwmac_vlan_ops,
+		.hwtimestamp = &ec_stmmac_ptp,
+		.ptp = &ec_stmmac_ptp_clock_ops,
+		.mode = &ec_dwmac4_ring_mode_ops,
+		.tc = &ec_dwmac510_tc_ops,
+		.mmc = &ec_dwmac_mmc_ops,
+		.est = &ec_dwmac510_est_ops,
+		.setup = ec_dwmac4_setup,
+		.quirks = NULL,
+	}, {
+		.gmac = false,
+		.gmac4 = true,
+		.xgmac = false,
+		.min_id = DWMAC_CORE_5_10,
+		.regs = {
+			.ptp_off = PTP_GMAC4_OFFSET,
+			.mmc_off = MMC_GMAC4_OFFSET,
+			.est_off = EST_GMAC4_OFFSET,
+			.fpe_reg = &ec_dwmac5_fpe_reg,
+		},
+		.desc = &ec_dwmac4_desc_ops,
+		.dma = &ec_dwmac410_dma_ops,
+		.mac = &ec_dwmac510_ops,
+		.vlan = &ec_dwmac_vlan_ops,
+		.hwtimestamp = &ec_stmmac_ptp,
+		.ptp = &ec_stmmac_ptp_clock_ops,
+		.mode = &ec_dwmac4_ring_mode_ops,
+		.tc = &ec_dwmac510_tc_ops,
+		.mmc = &ec_dwmac_mmc_ops,
+		.est = &ec_dwmac510_est_ops,
+		.setup = ec_dwmac4_setup,
+		.quirks = NULL,
+	}, {
+		.gmac = false,
+		.gmac4 = false,
+		.xgmac = true,
+		.min_id = DWXGMAC_CORE_2_10,
+		.dev_id = DWXGMAC_ID,
+		.regs = {
+			.ptp_off = PTP_XGMAC_OFFSET,
+			.mmc_off = MMC_XGMAC_OFFSET,
+			.est_off = EST_XGMAC_OFFSET,
+			.fpe_reg = &ec_dwxgmac3_fpe_reg,
+		},
+		.desc = &ec_dwxgmac210_desc_ops,
+		.dma = &ec_dwxgmac210_dma_ops,
+		.mac = &ec_dwxgmac210_ops,
+		.vlan = &ec_dwxgmac210_vlan_ops,
+		.hwtimestamp = &ec_stmmac_ptp,
+		.ptp = &ec_stmmac_ptp_clock_ops,
+		.mode = NULL,
+		.tc = &ec_dwmac510_tc_ops,
+		.mmc = &ec_dwxgmac_mmc_ops,
+		.est = &ec_dwmac510_est_ops,
+		.setup = ec_dwxgmac2_setup,
+		.quirks = NULL,
+	}, {
+		.gmac = false,
+		.gmac4 = false,
+		.xgmac = true,
+		.min_id = DWXLGMAC_CORE_2_00,
+		.dev_id = DWXLGMAC_ID,
+		.regs = {
+			.ptp_off = PTP_XGMAC_OFFSET,
+			.mmc_off = MMC_XGMAC_OFFSET,
+			.est_off = EST_XGMAC_OFFSET,
+			.fpe_reg = &ec_dwxgmac3_fpe_reg,
+		},
+		.desc = &ec_dwxgmac210_desc_ops,
+		.dma = &ec_dwxgmac210_dma_ops,
+		.mac = &ec_dwxlgmac2_ops,
+		.vlan = &ec_dwxlgmac2_vlan_ops,
+		.hwtimestamp = &ec_stmmac_ptp,
+		.ptp = &ec_stmmac_ptp_clock_ops,
+		.mode = NULL,
+		.tc = &ec_dwmac510_tc_ops,
+		.mmc = &ec_dwxgmac_mmc_ops,
+		.est = &ec_dwmac510_est_ops,
+		.setup = ec_dwxlgmac2_setup,
+		.quirks = stmmac_dwxlgmac_quirks,
+	},
+};
+
+int ec_stmmac_hwif_init(struct stmmac_priv *priv)
+{
+	bool needs_xgmac = priv->plat->has_xgmac;
+	bool needs_gmac4 = priv->plat->has_gmac4;
+	bool needs_gmac = priv->plat->has_gmac;
+	const struct stmmac_hwif_entry *entry;
+	struct mac_device_info *mac;
+	bool needs_setup = true;
+	u32 id, dev_id = 0;
+	int i, ret;
+
+	if (needs_gmac) {
+		id = stmmac_get_id(priv, GMAC_VERSION);
+	} else if (needs_gmac4 || needs_xgmac) {
+		id = stmmac_get_id(priv, GMAC4_VERSION);
+		if (needs_xgmac)
+			dev_id = stmmac_get_dev_id(priv, GMAC4_VERSION);
+	} else {
+		id = 0;
+	}
+
+	/* Save ID for later use */
+	priv->synopsys_id = id;
+
+	/* Lets assume some safe values first */
+	priv->ptpaddr = priv->ioaddr +
+		(needs_gmac4 ? PTP_GMAC4_OFFSET : PTP_GMAC3_X_OFFSET);
+	priv->mmcaddr = priv->ioaddr +
+		(needs_gmac4 ? MMC_GMAC4_OFFSET : MMC_GMAC3_X_OFFSET);
+	if (needs_gmac4)
+		priv->estaddr = priv->ioaddr + EST_GMAC4_OFFSET;
+	else if (needs_xgmac)
+		priv->estaddr = priv->ioaddr + EST_XGMAC_OFFSET;
+
+	/* Check for HW specific setup first */
+	if (priv->plat->setup) {
+		mac = priv->plat->setup(priv);
+		needs_setup = false;
+	} else {
+		mac = devm_kzalloc(priv->device, sizeof(*mac), GFP_KERNEL);
+	}
+
+	if (!mac)
+		return -ENOMEM;
+
+	/* Fallback to generic HW */
+	for (i = ARRAY_SIZE(stmmac_hw) - 1; i >= 0; i--) {
+		entry = &stmmac_hw[i];
+
+		if (needs_gmac ^ entry->gmac)
+			continue;
+		if (needs_gmac4 ^ entry->gmac4)
+			continue;
+		if (needs_xgmac ^ entry->xgmac)
+			continue;
+		/* Use synopsys_id var because some setups can override this */
+		if (priv->synopsys_id < entry->min_id)
+			continue;
+		if (needs_xgmac && (dev_id ^ entry->dev_id))
+			continue;
+
+		/* Only use generic HW helpers if needed */
+		mac->desc = mac->desc ? : entry->desc;
+		mac->dma = mac->dma ? : entry->dma;
+		mac->mac = mac->mac ? : entry->mac;
+		mac->ptp = mac->ptp ? : entry->hwtimestamp;
+		mac->mode = mac->mode ? : entry->mode;
+		mac->tc = mac->tc ? : entry->tc;
+		mac->mmc = mac->mmc ? : entry->mmc;
+		mac->est = mac->est ? : entry->est;
+		mac->vlan = mac->vlan ? : entry->vlan;
+
+		priv->hw = mac;
+		priv->fpe_cfg.reg = entry->regs.fpe_reg;
+		priv->ptpaddr = priv->ioaddr + entry->regs.ptp_off;
+		priv->mmcaddr = priv->ioaddr + entry->regs.mmc_off;
+		memcpy(&priv->ptp_clock_ops, entry->ptp,
+		       sizeof(struct ptp_clock_info));
+		if (entry->est)
+			priv->estaddr = priv->ioaddr + entry->regs.est_off;
+
+		/* Entry found */
+		if (needs_setup) {
+			ret = entry->setup(priv);
+			if (ret)
+				return ret;
+		}
+
+		/* Save quirks, if needed for posterior use */
+		priv->hwif_quirks = entry->quirks;
+		return 0;
+	}
+
+	dev_err(priv->device, "Failed to find HW IF (id=0x%x, gmac=%d/%d)\n",
+			id, needs_gmac, needs_gmac4);
+	return -EINVAL;
+}
diff --git a/drivers/net/ethercat/device/K3/hwif-ethercat.h b/drivers/net/ethercat/device/K3/hwif-ethercat.h
new file mode 100644
index 000000000000..0a956e9f7d46
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/hwif-ethercat.h
@@ -0,0 +1,696 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+// Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+// stmmac HW Interface Callbacks
+
+#ifndef __STMMAC_HWIF_H__
+#define __STMMAC_HWIF_H__
+
+#include <linux/netdevice.h>
+#include <linux/stmmac.h>
+#include <net/pkt_cls.h>
+
+#define stmmac_do_void_callback(__priv, __module, __cname,  __arg0, __args...) \
+({ \
+	int __result = -EINVAL; \
+	if ((__priv)->hw->__module && (__priv)->hw->__module->__cname) { \
+		(__priv)->hw->__module->__cname((__arg0), ##__args); \
+		__result = 0; \
+	} \
+	__result; \
+})
+#define stmmac_do_callback(__priv, __module, __cname,  __arg0, __args...) \
+({ \
+	int __result = -EINVAL; \
+	if ((__priv)->hw->__module && (__priv)->hw->__module->__cname) \
+		__result = (__priv)->hw->__module->__cname((__arg0), ##__args); \
+	__result; \
+})
+
+struct stmmac_extra_stats;
+struct stmmac_priv;
+struct stmmac_safety_stats;
+struct stmmac_fpe_cfg;
+enum stmmac_mpacket_type;
+struct dma_desc;
+struct dma_extended_desc;
+struct dma_edesc;
+
+/* Descriptors helpers */
+struct stmmac_desc_ops {
+	/* DMA RX descriptor ring initialization */
+	void (*init_rx_desc)(struct dma_desc *p, int disable_rx_ic, int mode,
+			int end, int bfsize);
+	/* DMA TX descriptor ring initialization */
+	void (*init_tx_desc)(struct dma_desc *p, int mode, int end);
+	/* Invoked by the xmit function to prepare the tx descriptor */
+	void (*prepare_tx_desc)(struct dma_desc *p, int is_fs, int len,
+			bool csum_flag, int mode, bool tx_own, bool ls,
+			unsigned int tot_pkt_len);
+	void (*prepare_tso_tx_desc)(struct dma_desc *p, int is_fs, int len1,
+			int len2, bool tx_own, bool ls, unsigned int tcphdrlen,
+			unsigned int tcppayloadlen);
+	/* Set/get the owner of the descriptor */
+	void (*set_tx_owner)(struct dma_desc *p);
+	int (*get_tx_owner)(struct dma_desc *p);
+	/* Clean the tx descriptor as soon as the tx irq is received */
+	void (*release_tx_desc)(struct dma_desc *p, int mode);
+	/* Clear interrupt on tx frame completion. When this bit is
+	 * set an interrupt happens as soon as the frame is transmitted */
+	void (*set_tx_ic)(struct dma_desc *p);
+	/* Last tx segment reports the transmit status */
+	int (*get_tx_ls)(struct dma_desc *p);
+	/* Get the tag of the descriptor */
+	u16 (*get_rx_vlan_tci)(struct dma_desc *p);
+	/* Get the valid status of descriptor */
+	bool (*get_rx_vlan_valid)(struct dma_desc *p);
+	/* Return the transmit status looking at the TDES1 */
+	int (*tx_status)(struct stmmac_extra_stats *x,
+			 struct dma_desc *p, void __iomem *ioaddr);
+	/* Get the buffer size from the descriptor */
+	int (*get_tx_len)(struct dma_desc *p);
+	/* Handle extra events on specific interrupts hw dependent */
+	void (*set_rx_owner)(struct dma_desc *p, int disable_rx_ic);
+	/* Get the receive frame size */
+	int (*get_rx_frame_len)(struct dma_desc *p, int rx_coe_type);
+	/* Return the reception status looking at the RDES1 */
+	int (*rx_status)(struct stmmac_extra_stats *x,
+			 struct dma_desc *p);
+	void (*rx_extended_status)(struct stmmac_extra_stats *x,
+				   struct dma_extended_desc *p);
+	/* Set tx timestamp enable bit */
+	void (*enable_tx_timestamp) (struct dma_desc *p);
+	/* get tx timestamp status */
+	int (*get_tx_timestamp_status) (struct dma_desc *p);
+	/* get timestamp value */
+	void (*get_timestamp)(void *desc, u32 ats, u64 *ts);
+	/* get rx timestamp status */
+	int (*get_rx_timestamp_status)(void *desc, void *next_desc, u32 ats);
+	/* Display ring */
+	void (*display_ring)(void *head, unsigned int size, bool rx,
+			     dma_addr_t dma_rx_phy, unsigned int desc_size);
+	/* set MSS via context descriptor */
+	void (*set_mss)(struct dma_desc *p, unsigned int mss);
+	/* set descriptor skbuff address */
+	void (*set_addr)(struct dma_desc *p, dma_addr_t addr);
+	/* clear descriptor */
+	void (*clear)(struct dma_desc *p);
+	/* RSS */
+	int (*get_rx_hash)(struct dma_desc *p, u32 *hash,
+			   enum pkt_hash_types *type);
+	void (*get_rx_header_len)(struct dma_desc *p, unsigned int *len);
+	void (*set_sec_addr)(struct dma_desc *p, dma_addr_t addr, bool buf2_valid);
+	void (*set_sarc)(struct dma_desc *p, u32 sarc_type);
+	void (*set_vlan_tag)(struct dma_desc *p, u16 tag, u16 inner_tag,
+			     u32 inner_type);
+	void (*set_vlan)(struct dma_desc *p, u32 type);
+	void (*set_tbs)(struct dma_edesc *p, u32 sec, u32 nsec);
+};
+
+#define stmmac_init_rx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, init_rx_desc, __args)
+#define stmmac_init_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, init_tx_desc, __args)
+#define stmmac_prepare_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, prepare_tx_desc, __args)
+#define stmmac_prepare_tso_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, prepare_tso_tx_desc, __args)
+#define stmmac_set_tx_owner(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_tx_owner, __args)
+#define stmmac_get_tx_owner(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_owner, __args)
+#define stmmac_release_tx_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, release_tx_desc, __args)
+#define stmmac_set_tx_ic(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_tx_ic, __args)
+#define stmmac_get_tx_ls(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_ls, __args)
+#define stmmac_get_rx_vlan_tci(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_rx_vlan_tci, __args)
+#define stmmac_get_rx_vlan_valid(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_rx_vlan_valid, __args)
+#define stmmac_tx_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, tx_status, __args)
+#define stmmac_get_tx_len(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_len, __args)
+#define stmmac_set_rx_owner(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_rx_owner, __args)
+#define stmmac_get_rx_frame_len(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_rx_frame_len, __args)
+#define stmmac_rx_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, rx_status, __args)
+#define stmmac_rx_extended_status(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, rx_extended_status, __args)
+#define stmmac_enable_tx_timestamp(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, enable_tx_timestamp, __args)
+#define stmmac_get_tx_timestamp_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_tx_timestamp_status, __args)
+#define stmmac_get_timestamp(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, get_timestamp, __args)
+#define stmmac_get_rx_timestamp_status(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_rx_timestamp_status, __args)
+#define stmmac_display_ring(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, display_ring, __args)
+#define stmmac_set_mss(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_mss, __args)
+#define stmmac_set_desc_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_addr, __args)
+#define stmmac_clear_desc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, clear, __args)
+#define stmmac_get_rx_hash(__priv, __args...) \
+	stmmac_do_callback(__priv, desc, get_rx_hash, __args)
+#define stmmac_get_rx_header_len(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, get_rx_header_len, __args)
+#define stmmac_set_desc_sec_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_sec_addr, __args)
+#define stmmac_set_desc_sarc(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_sarc, __args)
+#define stmmac_set_desc_vlan_tag(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_vlan_tag, __args)
+#define stmmac_set_desc_vlan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_vlan, __args)
+#define stmmac_set_desc_tbs(__priv, __args...) \
+	stmmac_do_void_callback(__priv, desc, set_tbs, __args)
+
+struct stmmac_dma_cfg;
+struct dma_features;
+
+/* Specific DMA helpers */
+struct stmmac_dma_ops {
+	/* DMA core initialization */
+	int (*reset)(void __iomem *ioaddr);
+	void (*init)(void __iomem *ioaddr, struct stmmac_dma_cfg *dma_cfg);
+	void (*init_chan)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  struct stmmac_dma_cfg *dma_cfg, u32 chan);
+	void (*init_rx_chan)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     struct stmmac_dma_cfg *dma_cfg,
+			     dma_addr_t phy, u32 chan);
+	void (*init_tx_chan)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     struct stmmac_dma_cfg *dma_cfg,
+			     dma_addr_t phy, u32 chan);
+	/* Configure the AXI Bus Mode Register */
+	void (*axi)(void __iomem *ioaddr, struct stmmac_axi *axi);
+	/* Dump DMA registers */
+	void (*dump_regs)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  u32 *reg_space);
+	void (*dma_rx_mode)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    int mode, u32 channel,
+			    int fifosz, u8 qmode);
+	void (*dma_tx_mode)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    int mode, u32 channel, int fifosz, u8 qmode);
+	/* To track extra statistic (if supported) */
+	void (*dma_diagnostic_fr)(struct stmmac_extra_stats *x,
+				  void __iomem *ioaddr);
+	void (*enable_dma_transmission)(void __iomem *ioaddr, u32 chan);
+	void (*enable_dma_irq)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			       u32 chan, bool rx, bool tx);
+	void (*disable_dma_irq)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 chan, bool rx, bool tx);
+	void (*start_tx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+	void (*stop_tx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+	void (*start_rx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			 u32 chan);
+	void (*stop_rx)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			u32 chan);
+	int (*dma_interrupt)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			     struct stmmac_extra_stats *x, u32 chan, u32 dir);
+	/* If supported then get the optional core features */
+	int (*get_hw_feature)(void __iomem *ioaddr,
+			      struct dma_features *dma_cap);
+	/* Program the HW RX Watchdog */
+	void (*rx_watchdog)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			    u32 riwt, u32 queue);
+	void (*set_tx_ring_len)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 len, u32 chan);
+	void (*set_rx_ring_len)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 len, u32 chan);
+	void (*set_rx_tail_ptr)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 tail_ptr, u32 chan);
+	void (*set_tx_tail_ptr)(struct stmmac_priv *priv, void __iomem *ioaddr,
+				u32 tail_ptr, u32 chan);
+	void (*enable_tso)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   bool en, u32 chan);
+	void (*qmode)(struct stmmac_priv *priv, void __iomem *ioaddr,
+		      u32 channel, u8 qmode);
+	void (*set_bfsize)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   int bfsize, u32 chan);
+	void (*enable_sph)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			   bool en, u32 chan);
+	int (*enable_tbs)(struct stmmac_priv *priv, void __iomem *ioaddr,
+			  bool en, u32 chan);
+};
+
+#define stmmac_dma_init(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init, __args)
+#define stmmac_init_chan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init_chan, __priv, __args)
+#define stmmac_init_rx_chan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init_rx_chan, __priv, __args)
+#define stmmac_init_tx_chan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, init_tx_chan, __priv, __args)
+#define stmmac_axi(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, axi, __args)
+#define stmmac_dump_dma_regs(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dump_regs, __priv, __args)
+#define stmmac_dma_rx_mode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dma_rx_mode, __priv, __args)
+#define stmmac_dma_tx_mode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dma_tx_mode, __priv, __args)
+#define stmmac_dma_diagnostic_fr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, dma_diagnostic_fr, __args)
+#define stmmac_enable_dma_transmission(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, enable_dma_transmission, __args)
+#define stmmac_enable_dma_irq(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, enable_dma_irq, __priv, __args)
+#define stmmac_disable_dma_irq(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, disable_dma_irq, __priv, __args)
+#define stmmac_start_tx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, start_tx, __priv, __args)
+#define stmmac_stop_tx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, stop_tx, __priv, __args)
+#define stmmac_start_rx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, start_rx, __priv, __args)
+#define stmmac_stop_rx(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, stop_rx, __priv, __args)
+#define stmmac_dma_interrupt_status(__priv, __args...) \
+	stmmac_do_callback(__priv, dma, dma_interrupt, __priv, __args)
+#define stmmac_get_hw_feature(__priv, __args...) \
+	stmmac_do_callback(__priv, dma, get_hw_feature, __args)
+#define stmmac_rx_watchdog(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, rx_watchdog, __priv, __args)
+#define stmmac_set_tx_ring_len(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_tx_ring_len, __priv, __args)
+#define stmmac_set_rx_ring_len(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_rx_ring_len, __priv, __args)
+#define stmmac_set_rx_tail_ptr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_rx_tail_ptr, __priv, __args)
+#define stmmac_set_tx_tail_ptr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_tx_tail_ptr, __priv, __args)
+#define stmmac_enable_tso(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, enable_tso, __priv, __args)
+#define stmmac_dma_qmode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, qmode, __priv, __args)
+#define stmmac_set_dma_bfsize(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, set_bfsize, __priv, __args)
+#define stmmac_enable_sph(__priv, __args...) \
+	stmmac_do_void_callback(__priv, dma, enable_sph, __priv, __args)
+#define stmmac_enable_tbs(__priv, __args...) \
+	stmmac_do_callback(__priv, dma, enable_tbs, __priv, __args)
+
+struct mac_device_info;
+struct net_device;
+struct stmmac_tc_entry;
+struct stmmac_pps_cfg;
+struct stmmac_rss;
+struct stmmac_est;
+
+enum stmmac_lpi_mode {
+	STMMAC_LPI_DISABLE,
+	STMMAC_LPI_FORCED,
+	STMMAC_LPI_TIMER,
+};
+
+/* Helpers to program the MAC core */
+struct stmmac_ops {
+	/* MAC core initialization */
+	void (*core_init)(struct mac_device_info *hw, struct net_device *dev);
+	/* Update MAC capabilities */
+	void (*update_caps)(struct stmmac_priv *priv);
+	/* Enable the MAC RX/TX */
+	void (*set_mac)(void __iomem *ioaddr, bool enable);
+	/* Enable and verify that the IPC module is supported */
+	int (*rx_ipc)(struct mac_device_info *hw);
+	/* Enable RX Queues */
+	void (*rx_queue_enable)(struct mac_device_info *hw, u8 mode, u32 queue);
+	/* RX Queues Priority */
+	void (*rx_queue_prio)(struct mac_device_info *hw, u32 prio, u32 queue);
+	/* TX Queues Priority */
+	void (*tx_queue_prio)(struct mac_device_info *hw, u32 prio, u32 queue);
+	/* RX Queues Routing */
+	void (*rx_queue_routing)(struct mac_device_info *hw, u8 packet,
+				 u32 queue);
+	/* Program RX Algorithms */
+	void (*prog_mtl_rx_algorithms)(struct mac_device_info *hw, u32 rx_alg);
+	/* Program TX Algorithms */
+	void (*prog_mtl_tx_algorithms)(struct mac_device_info *hw, u32 tx_alg);
+	/* Set MTL TX queues weight */
+	void (*set_mtl_tx_queue_weight)(struct stmmac_priv *priv,
+					struct mac_device_info *hw,
+					u32 weight, u32 queue);
+	/* RX MTL queue to RX dma mapping */
+	void (*map_mtl_to_dma)(struct mac_device_info *hw, u32 queue, u32 chan);
+	/* Configure AV Algorithm */
+	void (*config_cbs)(struct stmmac_priv *priv, struct mac_device_info *hw,
+			   u32 send_slope, u32 idle_slope, u32 high_credit,
+			   u32 low_credit, u32 queue);
+	/* Dump MAC registers */
+	void (*dump_regs)(struct mac_device_info *hw, u32 *reg_space);
+	/* Handle extra events on specific interrupts hw dependent */
+	int (*host_irq_status)(struct mac_device_info *hw,
+			       struct stmmac_extra_stats *x);
+	/* Handle MTL interrupts */
+	int (*host_mtl_irq_status)(struct stmmac_priv *priv,
+				   struct mac_device_info *hw, u32 chan);
+	/* Multicast filter setting */
+	void (*set_filter)(struct mac_device_info *hw, struct net_device *dev);
+	/* Flow control setting */
+	void (*flow_ctrl)(struct mac_device_info *hw, unsigned int duplex,
+			  unsigned int fc, unsigned int pause_time, u32 tx_cnt);
+	/* Set power management mode (e.g. magic frame) */
+	void (*pmt)(struct mac_device_info *hw, unsigned long mode);
+	/* Set/Get Unicast MAC addresses */
+	void (*set_umac_addr)(struct mac_device_info *hw,
+			      const unsigned char *addr,
+			      unsigned int reg_n);
+	void (*get_umac_addr)(struct mac_device_info *hw, unsigned char *addr,
+			      unsigned int reg_n);
+	int (*set_lpi_mode)(struct mac_device_info *hw,
+			    enum stmmac_lpi_mode mode,
+			    bool en_tx_lpi_clockgating, u32 et);
+	void (*set_eee_timer)(struct mac_device_info *hw, int ls, int tw);
+	void (*set_eee_pls)(struct mac_device_info *hw, int link);
+	void (*debug)(struct stmmac_priv *priv, void __iomem *ioaddr,
+		      struct stmmac_extra_stats *x, u32 rx_queues,
+		      u32 tx_queues);
+	/* PCS calls */
+	void (*pcs_ctrl_ane)(struct stmmac_priv *priv, bool ane, bool srgmi_ral,
+			     bool loopback);
+	/* Safety Features */
+	int (*safety_feat_config)(void __iomem *ioaddr, unsigned int asp,
+				  struct stmmac_safety_feature_cfg *safety_cfg);
+	int (*safety_feat_irq_status)(struct net_device *ndev,
+			void __iomem *ioaddr, unsigned int asp,
+			struct stmmac_safety_stats *stats);
+	int (*safety_feat_dump)(struct stmmac_safety_stats *stats,
+			int index, unsigned long *count, const char **desc);
+	/* Flexible RX Parser */
+	int (*rxp_config)(void __iomem *ioaddr, struct stmmac_tc_entry *entries,
+			  unsigned int count);
+	/* Flexible PPS */
+	int (*flex_pps_config)(void __iomem *ioaddr, int index,
+			       struct stmmac_pps_cfg *cfg, bool enable,
+			       u32 sub_second_inc, u32 systime_flags);
+	/* Loopback for selftests */
+	void (*set_mac_loopback)(void __iomem *ioaddr, bool enable);
+	/* RSS */
+	int (*rss_configure)(struct mac_device_info *hw,
+			     struct stmmac_rss *cfg, u32 num_rxq);
+	/* TX Timestamp */
+	int (*get_mac_tx_timestamp)(struct mac_device_info *hw, u64 *ts);
+	/* Source Address Insertion / Replacement */
+	void (*sarc_configure)(void __iomem *ioaddr, int val);
+	/* Filtering */
+	int (*config_l3_filter)(struct mac_device_info *hw, u32 filter_no,
+				bool en, bool ipv6, bool sa, bool inv,
+				u32 match);
+	int (*config_l4_filter)(struct mac_device_info *hw, u32 filter_no,
+				bool en, bool udp, bool sa, bool inv,
+				u32 match);
+	void (*set_arp_offload)(struct mac_device_info *hw, bool en, u32 addr);
+	int (*fpe_map_preemption_class)(struct net_device *ndev,
+					struct netlink_ext_ack *extack,
+					u32 pclass);
+};
+
+#define stmmac_core_init(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, core_init, __args)
+#define stmmac_mac_update_caps(__priv) \
+	stmmac_do_void_callback(__priv, mac, update_caps, __priv)
+#define stmmac_mac_set(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_mac, __args)
+#define stmmac_rx_ipc(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, rx_ipc, __args)
+#define stmmac_rx_queue_enable(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, rx_queue_enable, __args)
+#define stmmac_rx_queue_prio(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, rx_queue_prio, __args)
+#define stmmac_tx_queue_prio(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, tx_queue_prio, __args)
+#define stmmac_rx_queue_routing(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, rx_queue_routing, __args)
+#define stmmac_prog_mtl_rx_algorithms(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, prog_mtl_rx_algorithms, __args)
+#define stmmac_prog_mtl_tx_algorithms(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, prog_mtl_tx_algorithms, __args)
+#define stmmac_set_mtl_tx_queue_weight(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_mtl_tx_queue_weight, __priv, __args)
+#define stmmac_map_mtl_to_dma(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, map_mtl_to_dma, __args)
+#define stmmac_config_cbs(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, config_cbs, __priv, __args)
+#define stmmac_dump_mac_regs(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, dump_regs, __args)
+#define stmmac_host_irq_status(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, host_irq_status, __args)
+#define stmmac_host_mtl_irq_status(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, host_mtl_irq_status, __priv, __args)
+#define stmmac_set_filter(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_filter, __args)
+#define stmmac_flow_ctrl(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, flow_ctrl, __args)
+#define stmmac_pmt(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, pmt, __args)
+#define stmmac_set_umac_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_umac_addr, __args)
+#define stmmac_get_umac_addr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, get_umac_addr, __args)
+#define stmmac_set_lpi_mode(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, set_lpi_mode, __args)
+#define stmmac_set_eee_timer(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_eee_timer, __args)
+#define stmmac_set_eee_pls(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_eee_pls, __args)
+#define stmmac_mac_debug(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, debug, __priv, __args)
+#define stmmac_pcs_ctrl_ane(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, pcs_ctrl_ane, __priv, __args)
+#define stmmac_safety_feat_config(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, safety_feat_config, __args)
+#define stmmac_safety_feat_irq_status(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, safety_feat_irq_status, __args)
+#define stmmac_safety_feat_dump(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, safety_feat_dump, __args)
+#define stmmac_rxp_config(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, rxp_config, __args)
+#define stmmac_flex_pps_config(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, flex_pps_config, __args)
+#define stmmac_set_mac_loopback(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_mac_loopback, __args)
+#define stmmac_rss_configure(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, rss_configure, __args)
+#define stmmac_get_mac_tx_timestamp(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, get_mac_tx_timestamp, __args)
+#define stmmac_sarc_configure(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, sarc_configure, __args)
+#define stmmac_config_l3_filter(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, config_l3_filter, __args)
+#define stmmac_config_l4_filter(__priv, __args...) \
+	stmmac_do_callback(__priv, mac, config_l4_filter, __args)
+#define stmmac_set_arp_offload(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, set_arp_offload, __args)
+#define stmmac_fpe_map_preemption_class(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mac, fpe_map_preemption_class, __args)
+
+/* PTP and HW Timer helpers */
+struct stmmac_hwtimestamp {
+	void (*config_hw_tstamping) (void __iomem *ioaddr, u32 data);
+	void (*config_sub_second_increment)(void __iomem *ioaddr, u32 ptp_clock,
+					   int gmac4, u32 *ssinc);
+	int (*init_systime) (void __iomem *ioaddr, u32 sec, u32 nsec);
+	int (*config_addend) (void __iomem *ioaddr, u32 addend);
+	int (*adjust_systime) (void __iomem *ioaddr, u32 sec, u32 nsec,
+			       int add_sub, int gmac4);
+	void (*get_systime) (void __iomem *ioaddr, u64 *systime);
+	void (*get_ptptime)(void __iomem *ioaddr, u64 *ptp_time);
+	void (*timestamp_interrupt)(struct stmmac_priv *priv);
+	void (*hwtstamp_correct_latency)(struct stmmac_priv *priv);
+};
+
+#define stmmac_config_hw_tstamping(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, config_hw_tstamping, __args)
+#define stmmac_config_sub_second_increment(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, config_sub_second_increment, __args)
+#define stmmac_init_systime(__priv, __args...) \
+	stmmac_do_callback(__priv, ptp, init_systime, __args)
+#define stmmac_config_addend(__priv, __args...) \
+	stmmac_do_callback(__priv, ptp, config_addend, __args)
+#define stmmac_adjust_systime(__priv, __args...) \
+	stmmac_do_callback(__priv, ptp, adjust_systime, __args)
+#define stmmac_get_systime(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, get_systime, __args)
+#define stmmac_get_ptptime(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, get_ptptime, __args)
+#define stmmac_timestamp_interrupt(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, timestamp_interrupt, __args)
+#define stmmac_hwtstamp_correct_latency(__priv, __args...) \
+	stmmac_do_void_callback(__priv, ptp, hwtstamp_correct_latency, __args)
+
+struct stmmac_tx_queue;
+struct stmmac_rx_queue;
+
+/* Helpers to manage the descriptors for chain and ring modes */
+struct stmmac_mode_ops {
+	void (*init) (void *des, dma_addr_t phy_addr, unsigned int size,
+		      unsigned int extend_desc);
+	unsigned int (*is_jumbo_frm) (int len, int ehn_desc);
+	int (*jumbo_frm)(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
+			 int csum);
+	int (*set_16kib_bfsize)(int mtu);
+	void (*init_desc3)(struct dma_desc *p);
+	void (*refill_desc3)(struct stmmac_rx_queue *rx_q, struct dma_desc *p);
+	void (*clean_desc3)(struct stmmac_tx_queue *tx_q, struct dma_desc *p);
+};
+
+#define stmmac_mode_init(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, init, __args)
+#define stmmac_is_jumbo_frm(__priv, __args...) \
+	stmmac_do_callback(__priv, mode, is_jumbo_frm, __args)
+#define stmmac_jumbo_frm(__priv, __args...) \
+	stmmac_do_callback(__priv, mode, jumbo_frm, __args)
+#define stmmac_set_16kib_bfsize(__priv, __args...) \
+	stmmac_do_callback(__priv, mode, set_16kib_bfsize, __args)
+#define stmmac_init_desc3(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, init_desc3, __args)
+#define stmmac_refill_desc3(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, refill_desc3, __args)
+#define stmmac_clean_desc3(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mode, clean_desc3, __args)
+
+struct tc_cls_u32_offload;
+struct tc_cbs_qopt_offload;
+struct flow_cls_offload;
+struct tc_taprio_qopt_offload;
+struct tc_etf_qopt_offload;
+struct tc_query_caps_base;
+
+struct stmmac_tc_ops {
+	int (*init)(struct stmmac_priv *priv);
+	int (*setup_cls_u32)(struct stmmac_priv *priv,
+			     struct tc_cls_u32_offload *cls);
+	int (*setup_cbs)(struct stmmac_priv *priv,
+			 struct tc_cbs_qopt_offload *qopt);
+	int (*setup_cls)(struct stmmac_priv *priv,
+			 struct flow_cls_offload *cls);
+	int (*setup_taprio)(struct stmmac_priv *priv,
+			    struct tc_taprio_qopt_offload *qopt);
+	int (*setup_etf)(struct stmmac_priv *priv,
+			 struct tc_etf_qopt_offload *qopt);
+	int (*query_caps)(struct stmmac_priv *priv,
+			  struct tc_query_caps_base *base);
+	int (*setup_mqprio)(struct stmmac_priv *priv,
+			    struct tc_mqprio_qopt_offload *qopt);
+};
+
+#define stmmac_tc_init(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, init, __args)
+#define stmmac_tc_setup_cls_u32(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, setup_cls_u32, __args)
+#define stmmac_tc_setup_cbs(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, setup_cbs, __args)
+#define stmmac_tc_setup_cls(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, setup_cls, __args)
+#define stmmac_tc_setup_taprio(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, setup_taprio, __args)
+#define stmmac_tc_setup_etf(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, setup_etf, __args)
+#define stmmac_tc_query_caps(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, query_caps, __args)
+#define stmmac_tc_setup_mqprio(__priv, __args...) \
+	stmmac_do_callback(__priv, tc, setup_mqprio, __args)
+
+struct stmmac_counters;
+
+struct stmmac_mmc_ops {
+	void (*ctrl)(void __iomem *ioaddr, unsigned int mode);
+	void (*intr_all_mask)(void __iomem *ioaddr);
+	void (*read)(void __iomem *ioaddr, struct stmmac_counters *mmc);
+};
+
+#define stmmac_mmc_ctrl(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mmc, ctrl, __args)
+#define stmmac_mmc_intr_all_mask(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mmc, intr_all_mask, __args)
+#define stmmac_mmc_read(__priv, __args...) \
+	stmmac_do_void_callback(__priv, mmc, read, __args)
+
+struct stmmac_est_ops {
+	int (*configure)(struct stmmac_priv *priv, struct stmmac_est *cfg,
+			 unsigned int ptp_rate);
+	void (*irq_status)(struct stmmac_priv *priv, struct net_device *dev,
+			   struct stmmac_extra_stats *x, u32 txqcnt);
+};
+
+#define stmmac_est_configure(__priv, __args...) \
+	stmmac_do_callback(__priv, est, configure, __args)
+#define stmmac_est_irq_status(__priv, __args...) \
+	stmmac_do_void_callback(__priv, est, irq_status, __args)
+
+struct stmmac_vlan_ops {
+	/* VLAN */
+	void (*update_vlan_hash)(struct mac_device_info *hw, u32 hash,
+				 u16 perfect_match, bool is_double);
+	void (*enable_vlan)(struct mac_device_info *hw, u32 type);
+	void (*rx_hw_vlan)(struct mac_device_info *hw, struct dma_desc *rx_desc,
+			   struct sk_buff *skb);
+	void (*set_hw_vlan_mode)(struct mac_device_info *hw);
+	int (*add_hw_vlan_rx_fltr)(struct net_device *dev,
+				   struct mac_device_info *hw,
+				   __be16 proto, u16 vid);
+	int (*del_hw_vlan_rx_fltr)(struct net_device *dev,
+				   struct mac_device_info *hw,
+				   __be16 proto, u16 vid);
+	void (*restore_hw_vlan_rx_fltr)(struct net_device *dev,
+					struct mac_device_info *hw);
+};
+
+#define stmmac_update_vlan_hash(__priv, __args...) \
+	stmmac_do_void_callback(__priv, vlan, update_vlan_hash, __args)
+#define stmmac_enable_vlan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, vlan, enable_vlan, __args)
+#define stmmac_rx_hw_vlan(__priv, __args...) \
+	stmmac_do_void_callback(__priv, vlan, rx_hw_vlan, __args)
+#define stmmac_set_hw_vlan_mode(__priv, __args...) \
+	stmmac_do_void_callback(__priv, vlan, set_hw_vlan_mode, __args)
+#define stmmac_add_hw_vlan_rx_fltr(__priv, __args...) \
+	stmmac_do_callback(__priv, vlan, add_hw_vlan_rx_fltr, __args)
+#define stmmac_del_hw_vlan_rx_fltr(__priv, __args...) \
+	stmmac_do_callback(__priv, vlan, del_hw_vlan_rx_fltr, __args)
+#define stmmac_restore_hw_vlan_rx_fltr(__priv, __args...) \
+	stmmac_do_void_callback(__priv, vlan, restore_hw_vlan_rx_fltr, __args)
+
+struct stmmac_regs_off {
+	const struct stmmac_fpe_reg *fpe_reg;
+	u32 ptp_off;
+	u32 mmc_off;
+	u32 est_off;
+};
+
+extern const struct stmmac_desc_ops ec_enh_desc_ops;
+extern const struct stmmac_desc_ops ec_ndesc_ops;
+
+extern const struct stmmac_hwtimestamp ec_stmmac_ptp;
+extern const struct stmmac_hwtimestamp ec_dwmac1000_ptp;
+
+extern const struct stmmac_mode_ops ec_ring_mode_ops;
+extern const struct stmmac_mode_ops ec_chain_mode_ops;
+
+extern const struct stmmac_ops ec_dwmac100_ops;
+extern const struct stmmac_dma_ops ec_dwmac100_dma_ops;
+extern const struct stmmac_ops ec_dwmac1000_ops;
+extern const struct stmmac_dma_ops ec_dwmac1000_dma_ops;
+extern const struct stmmac_ops ec_dwmac4_ops;
+extern const struct stmmac_dma_ops ec_dwmac4_dma_ops;
+extern const struct stmmac_ops ec_dwmac410_ops;
+extern const struct stmmac_dma_ops ec_dwmac410_dma_ops;
+extern const struct stmmac_ops ec_dwmac510_ops;
+extern const struct stmmac_tc_ops ec_dwmac4_tc_ops;
+extern const struct stmmac_tc_ops ec_dwmac510_tc_ops;
+
+#define GMAC_VERSION		0x00000020	/* GMAC CORE Version */
+#define GMAC4_VERSION		0x00000110	/* GMAC4+ CORE Version */
+
+int ec_stmmac_reset(struct stmmac_priv *priv, void __iomem *ioaddr);
+int ec_stmmac_hwif_init(struct stmmac_priv *priv);
+
+#endif /* __STMMAC_HWIF_H__ */
diff --git a/drivers/net/ethercat/device/K3/mmc-ethercat.h b/drivers/net/ethercat/device/K3/mmc-ethercat.h
new file mode 100644
index 000000000000..88abeadd236c
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/mmc-ethercat.h
@@ -0,0 +1,145 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  MMC Header file
+
+  Copyright (C) 2011  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __MMC_H__
+#define __MMC_H__
+
+/* MMC control register */
+/* When set, all counter are reset */
+#define MMC_CNTRL_COUNTER_RESET		0x1
+/* When set, do not roll over zero after reaching the max value*/
+#define MMC_CNTRL_COUNTER_STOP_ROLLOVER	0x2
+#define MMC_CNTRL_RESET_ON_READ		0x4	/* Reset after reading */
+#define MMC_CNTRL_COUNTER_FREEZER	0x8	/* Freeze counter values to the
+						 * current value.*/
+#define MMC_CNTRL_PRESET		0x10
+#define MMC_CNTRL_FULL_HALF_PRESET	0x20
+
+#define MMC_GMAC4_OFFSET		0x700
+#define MMC_GMAC3_X_OFFSET		0x100
+#define MMC_XGMAC_OFFSET		0x800
+
+struct stmmac_counters {
+	unsigned int mmc_tx_octetcount_gb;
+	unsigned int mmc_tx_framecount_gb;
+	unsigned int mmc_tx_broadcastframe_g;
+	unsigned int mmc_tx_multicastframe_g;
+	unsigned int mmc_tx_64_octets_gb;
+	unsigned int mmc_tx_65_to_127_octets_gb;
+	unsigned int mmc_tx_128_to_255_octets_gb;
+	unsigned int mmc_tx_256_to_511_octets_gb;
+	unsigned int mmc_tx_512_to_1023_octets_gb;
+	unsigned int mmc_tx_1024_to_max_octets_gb;
+	unsigned int mmc_tx_unicast_gb;
+	unsigned int mmc_tx_multicast_gb;
+	unsigned int mmc_tx_broadcast_gb;
+	unsigned int mmc_tx_underflow_error;
+	unsigned int mmc_tx_singlecol_g;
+	unsigned int mmc_tx_multicol_g;
+	unsigned int mmc_tx_deferred;
+	unsigned int mmc_tx_latecol;
+	unsigned int mmc_tx_exesscol;
+	unsigned int mmc_tx_carrier_error;
+	unsigned int mmc_tx_octetcount_g;
+	unsigned int mmc_tx_framecount_g;
+	unsigned int mmc_tx_excessdef;
+	unsigned int mmc_tx_pause_frame;
+	unsigned int mmc_tx_vlan_frame_g;
+	unsigned int mmc_tx_oversize_g;
+	unsigned int mmc_tx_lpi_usec;
+	unsigned int mmc_tx_lpi_tran;
+
+	/* MMC RX counter registers */
+	unsigned int mmc_rx_framecount_gb;
+	unsigned int mmc_rx_octetcount_gb;
+	unsigned int mmc_rx_octetcount_g;
+	unsigned int mmc_rx_broadcastframe_g;
+	unsigned int mmc_rx_multicastframe_g;
+	unsigned int mmc_rx_crc_error;
+	unsigned int mmc_rx_align_error;
+	unsigned int mmc_rx_run_error;
+	unsigned int mmc_rx_jabber_error;
+	unsigned int mmc_rx_undersize_g;
+	unsigned int mmc_rx_oversize_g;
+	unsigned int mmc_rx_64_octets_gb;
+	unsigned int mmc_rx_65_to_127_octets_gb;
+	unsigned int mmc_rx_128_to_255_octets_gb;
+	unsigned int mmc_rx_256_to_511_octets_gb;
+	unsigned int mmc_rx_512_to_1023_octets_gb;
+	unsigned int mmc_rx_1024_to_max_octets_gb;
+	unsigned int mmc_rx_unicast_g;
+	unsigned int mmc_rx_length_error;
+	unsigned int mmc_rx_autofrangetype;
+	unsigned int mmc_rx_pause_frames;
+	unsigned int mmc_rx_fifo_overflow;
+	unsigned int mmc_rx_vlan_frames_gb;
+	unsigned int mmc_rx_watchdog_error;
+	unsigned int mmc_rx_error;
+	unsigned int mmc_rx_lpi_usec;
+	unsigned int mmc_rx_lpi_tran;
+	unsigned int mmc_rx_discard_frames_gb;
+	unsigned int mmc_rx_discard_octets_gb;
+	unsigned int mmc_rx_align_err_frames;
+
+	/* IPv4 */
+	unsigned int mmc_rx_ipv4_gd;
+	unsigned int mmc_rx_ipv4_hderr;
+	unsigned int mmc_rx_ipv4_nopay;
+	unsigned int mmc_rx_ipv4_frag;
+	unsigned int mmc_rx_ipv4_udsbl;
+
+	unsigned int mmc_rx_ipv4_gd_octets;
+	unsigned int mmc_rx_ipv4_hderr_octets;
+	unsigned int mmc_rx_ipv4_nopay_octets;
+	unsigned int mmc_rx_ipv4_frag_octets;
+	unsigned int mmc_rx_ipv4_udsbl_octets;
+
+	/* IPV6 */
+	unsigned int mmc_rx_ipv6_gd_octets;
+	unsigned int mmc_rx_ipv6_hderr_octets;
+	unsigned int mmc_rx_ipv6_nopay_octets;
+
+	unsigned int mmc_rx_ipv6_gd;
+	unsigned int mmc_rx_ipv6_hderr;
+	unsigned int mmc_rx_ipv6_nopay;
+
+	/* Protocols */
+	unsigned int mmc_rx_udp_gd;
+	unsigned int mmc_rx_udp_err;
+	unsigned int mmc_rx_tcp_gd;
+	unsigned int mmc_rx_tcp_err;
+	unsigned int mmc_rx_icmp_gd;
+	unsigned int mmc_rx_icmp_err;
+
+	unsigned int mmc_rx_udp_gd_octets;
+	unsigned int mmc_rx_udp_err_octets;
+	unsigned int mmc_rx_tcp_gd_octets;
+	unsigned int mmc_rx_tcp_err_octets;
+	unsigned int mmc_rx_icmp_gd_octets;
+	unsigned int mmc_rx_icmp_err_octets;
+
+	/* Stream-Gate Filter */
+	unsigned int mmc_sgf_pass_fragment_cntr;
+	unsigned int mmc_sgf_fail_fragment_cntr;
+
+	/* FPE */
+	unsigned int mmc_tx_fpe_fragment_cntr;
+	unsigned int mmc_tx_hold_req_cntr;
+	unsigned int mmc_tx_gate_overrun_cntr;
+	unsigned int mmc_rx_packet_assembly_err_cntr;
+	unsigned int mmc_rx_packet_smd_err_cntr;
+	unsigned int mmc_rx_packet_assembly_ok_cntr;
+	unsigned int mmc_rx_fpe_fragment_cntr;
+};
+
+extern const struct stmmac_mmc_ops ec_dwmac_mmc_ops;
+extern const struct stmmac_mmc_ops ec_dwxgmac_mmc_ops;
+
+#endif /* __MMC_H__ */
diff --git a/drivers/net/ethercat/device/K3/mmc_core-ethercat.c b/drivers/net/ethercat/device/K3/mmc_core-ethercat.c
new file mode 100644
index 000000000000..58105f9fec2a
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/mmc_core-ethercat.c
@@ -0,0 +1,606 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  DWMAC Management Counters
+
+  Copyright (C) 2011  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include "hwif-ethercat.h"
+#include "mmc-ethercat.h"
+
+/* MAC Management Counters register offset */
+
+#define MMC_CNTRL		0x00	/* MMC Control */
+#define MMC_RX_INTR		0x04	/* MMC RX Interrupt */
+#define MMC_TX_INTR		0x08	/* MMC TX Interrupt */
+#define MMC_RX_INTR_MASK	0x0c	/* MMC Interrupt Mask */
+#define MMC_TX_INTR_MASK	0x10	/* MMC Interrupt Mask */
+#define MMC_DEFAULT_MASK	0xffffffff
+
+/* MMC TX counter registers */
+
+/* Note:
+ * _GB register stands for good and bad frames
+ * _G is for good only.
+ */
+#define MMC_TX_OCTETCOUNT_GB		0x14
+#define MMC_TX_FRAMECOUNT_GB		0x18
+#define MMC_TX_BROADCASTFRAME_G		0x1c
+#define MMC_TX_MULTICASTFRAME_G		0x20
+#define MMC_TX_64_OCTETS_GB		0x24
+#define MMC_TX_65_TO_127_OCTETS_GB	0x28
+#define MMC_TX_128_TO_255_OCTETS_GB	0x2c
+#define MMC_TX_256_TO_511_OCTETS_GB	0x30
+#define MMC_TX_512_TO_1023_OCTETS_GB	0x34
+#define MMC_TX_1024_TO_MAX_OCTETS_GB	0x38
+#define MMC_TX_UNICAST_GB		0x3c
+#define MMC_TX_MULTICAST_GB		0x40
+#define MMC_TX_BROADCAST_GB		0x44
+#define MMC_TX_UNDERFLOW_ERROR		0x48
+#define MMC_TX_SINGLECOL_G		0x4c
+#define MMC_TX_MULTICOL_G		0x50
+#define MMC_TX_DEFERRED			0x54
+#define MMC_TX_LATECOL			0x58
+#define MMC_TX_EXESSCOL			0x5c
+#define MMC_TX_CARRIER_ERROR		0x60
+#define MMC_TX_OCTETCOUNT_G		0x64
+#define MMC_TX_FRAMECOUNT_G		0x68
+#define MMC_TX_EXCESSDEF		0x6c
+#define MMC_TX_PAUSE_FRAME		0x70
+#define MMC_TX_VLAN_FRAME_G		0x74
+#define MMC_TX_OVERSIZE_G		0x78
+
+/* MMC RX counter registers */
+#define MMC_RX_FRAMECOUNT_GB		0x80
+#define MMC_RX_OCTETCOUNT_GB		0x84
+#define MMC_RX_OCTETCOUNT_G		0x88
+#define MMC_RX_BROADCASTFRAME_G		0x8c
+#define MMC_RX_MULTICASTFRAME_G		0x90
+#define MMC_RX_CRC_ERROR		0x94
+#define MMC_RX_ALIGN_ERROR		0x98
+#define MMC_RX_RUN_ERROR		0x9C
+#define MMC_RX_JABBER_ERROR		0xA0
+#define MMC_RX_UNDERSIZE_G		0xA4
+#define MMC_RX_OVERSIZE_G		0xA8
+#define MMC_RX_64_OCTETS_GB		0xAC
+#define MMC_RX_65_TO_127_OCTETS_GB	0xb0
+#define MMC_RX_128_TO_255_OCTETS_GB	0xb4
+#define MMC_RX_256_TO_511_OCTETS_GB	0xb8
+#define MMC_RX_512_TO_1023_OCTETS_GB	0xbc
+#define MMC_RX_1024_TO_MAX_OCTETS_GB	0xc0
+#define MMC_RX_UNICAST_G		0xc4
+#define MMC_RX_LENGTH_ERROR		0xc8
+#define MMC_RX_AUTOFRANGETYPE		0xcc
+#define MMC_RX_PAUSE_FRAMES		0xd0
+#define MMC_RX_FIFO_OVERFLOW		0xd4
+#define MMC_RX_VLAN_FRAMES_GB		0xd8
+#define MMC_RX_WATCHDOG_ERROR		0xdc
+#define MMC_RX_ERROR			0xe0
+
+#define MMC_TX_LPI_USEC			0xec
+#define MMC_TX_LPI_TRAN			0xf0
+#define MMC_RX_LPI_USEC			0xf4
+#define MMC_RX_LPI_TRAN			0xf8
+
+/* IPC*/
+#define MMC_RX_IPC_INTR_MASK		0x100
+#define MMC_RX_IPC_INTR			0x108
+/* IPv4*/
+#define MMC_RX_IPV4_GD			0x110
+#define MMC_RX_IPV4_HDERR		0x114
+#define MMC_RX_IPV4_NOPAY		0x118
+#define MMC_RX_IPV4_FRAG		0x11C
+#define MMC_RX_IPV4_UDSBL		0x120
+
+#define MMC_RX_IPV4_GD_OCTETS		0x150
+#define MMC_RX_IPV4_HDERR_OCTETS	0x154
+#define MMC_RX_IPV4_NOPAY_OCTETS	0x158
+#define MMC_RX_IPV4_FRAG_OCTETS		0x15c
+#define MMC_RX_IPV4_UDSBL_OCTETS	0x160
+
+/* IPV6*/
+#define MMC_RX_IPV6_GD_OCTETS		0x164
+#define MMC_RX_IPV6_HDERR_OCTETS	0x168
+#define MMC_RX_IPV6_NOPAY_OCTETS	0x16c
+
+#define MMC_RX_IPV6_GD			0x124
+#define MMC_RX_IPV6_HDERR		0x128
+#define MMC_RX_IPV6_NOPAY		0x12c
+
+/* Protocols*/
+#define MMC_RX_UDP_GD			0x130
+#define MMC_RX_UDP_ERR			0x134
+#define MMC_RX_TCP_GD			0x138
+#define MMC_RX_TCP_ERR			0x13c
+#define MMC_RX_ICMP_GD			0x140
+#define MMC_RX_ICMP_ERR			0x144
+
+#define MMC_RX_UDP_GD_OCTETS		0x170
+#define MMC_RX_UDP_ERR_OCTETS		0x174
+#define MMC_RX_TCP_GD_OCTETS		0x178
+#define MMC_RX_TCP_ERR_OCTETS		0x17c
+#define MMC_RX_ICMP_GD_OCTETS		0x180
+#define MMC_RX_ICMP_ERR_OCTETS		0x184
+
+#define MMC_TX_FPE_FRAG			0x1a8
+#define MMC_TX_HOLD_REQ			0x1ac
+#define MMC_RX_PKT_ASSEMBLY_ERR		0x1c8
+#define MMC_RX_PKT_SMD_ERR		0x1cc
+#define MMC_RX_PKT_ASSEMBLY_OK		0x1d0
+#define MMC_RX_FPE_FRAG			0x1d4
+
+/* XGMAC MMC Registers */
+#define MMC_XGMAC_TX_OCTET_GB		0x14
+#define MMC_XGMAC_TX_PKT_GB		0x1c
+#define MMC_XGMAC_TX_BROAD_PKT_G	0x24
+#define MMC_XGMAC_TX_MULTI_PKT_G	0x2c
+#define MMC_XGMAC_TX_64OCT_GB		0x34
+#define MMC_XGMAC_TX_65OCT_GB		0x3c
+#define MMC_XGMAC_TX_128OCT_GB		0x44
+#define MMC_XGMAC_TX_256OCT_GB		0x4c
+#define MMC_XGMAC_TX_512OCT_GB		0x54
+#define MMC_XGMAC_TX_1024OCT_GB		0x5c
+#define MMC_XGMAC_TX_UNI_PKT_GB		0x64
+#define MMC_XGMAC_TX_MULTI_PKT_GB	0x6c
+#define MMC_XGMAC_TX_BROAD_PKT_GB	0x74
+#define MMC_XGMAC_TX_UNDER		0x7c
+#define MMC_XGMAC_TX_OCTET_G		0x84
+#define MMC_XGMAC_TX_PKT_G		0x8c
+#define MMC_XGMAC_TX_PAUSE		0x94
+#define MMC_XGMAC_TX_VLAN_PKT_G		0x9c
+#define MMC_XGMAC_TX_LPI_USEC		0xa4
+#define MMC_XGMAC_TX_LPI_TRAN		0xa8
+
+#define MMC_XGMAC_RX_PKT_GB		0x100
+#define MMC_XGMAC_RX_OCTET_GB		0x108
+#define MMC_XGMAC_RX_OCTET_G		0x110
+#define MMC_XGMAC_RX_BROAD_PKT_G	0x118
+#define MMC_XGMAC_RX_MULTI_PKT_G	0x120
+#define MMC_XGMAC_RX_CRC_ERR		0x128
+#define MMC_XGMAC_RX_RUNT_ERR		0x130
+#define MMC_XGMAC_RX_JABBER_ERR		0x134
+#define MMC_XGMAC_RX_UNDER		0x138
+#define MMC_XGMAC_RX_OVER		0x13c
+#define MMC_XGMAC_RX_64OCT_GB		0x140
+#define MMC_XGMAC_RX_65OCT_GB		0x148
+#define MMC_XGMAC_RX_128OCT_GB		0x150
+#define MMC_XGMAC_RX_256OCT_GB		0x158
+#define MMC_XGMAC_RX_512OCT_GB		0x160
+#define MMC_XGMAC_RX_1024OCT_GB		0x168
+#define MMC_XGMAC_RX_UNI_PKT_G		0x170
+#define MMC_XGMAC_RX_LENGTH_ERR		0x178
+#define MMC_XGMAC_RX_RANGE		0x180
+#define MMC_XGMAC_RX_PAUSE		0x188
+#define MMC_XGMAC_RX_FIFOOVER_PKT	0x190
+#define MMC_XGMAC_RX_VLAN_PKT_GB	0x198
+#define MMC_XGMAC_RX_WATCHDOG_ERR	0x1a0
+#define MMC_XGMAC_RX_LPI_USEC		0x1a4
+#define MMC_XGMAC_RX_LPI_TRAN		0x1a8
+#define MMC_XGMAC_RX_DISCARD_PKT_GB	0x1ac
+#define MMC_XGMAC_RX_DISCARD_OCT_GB	0x1b4
+#define MMC_XGMAC_RX_ALIGN_ERR_PKT	0x1bc
+
+#define MMC_XGMAC_SGF_PASS_PKT		0x1f0
+#define MMC_XGMAC_SGF_FAIL_PKT		0x1f4
+#define MMC_XGMAC_TX_FPE_INTR_MASK	0x204
+#define MMC_XGMAC_TX_FPE_FRAG		0x208
+#define MMC_XGMAC_TX_HOLD_REQ		0x20c
+#define MMC_XGMAC_TX_GATE_OVERRUN	0x210
+#define MMC_XGMAC_RX_FPE_INTR_MASK	0x224
+#define MMC_XGMAC_RX_PKT_ASSEMBLY_ERR	0x228
+#define MMC_XGMAC_RX_PKT_SMD_ERR	0x22c
+#define MMC_XGMAC_RX_PKT_ASSEMBLY_OK	0x230
+#define MMC_XGMAC_RX_FPE_FRAG		0x234
+#define MMC_XGMAC_RX_IPC_INTR_MASK	0x25c
+
+#define MMC_XGMAC_RX_IPV4_GD		0x264
+#define MMC_XGMAC_RX_IPV4_HDERR		0x26c
+#define MMC_XGMAC_RX_IPV4_NOPAY		0x274
+#define MMC_XGMAC_RX_IPV4_FRAG		0x27c
+#define MMC_XGMAC_RX_IPV4_UDSBL		0x284
+
+#define MMC_XGMAC_RX_IPV6_GD		0x28c
+#define MMC_XGMAC_RX_IPV6_HDERR		0x294
+#define MMC_XGMAC_RX_IPV6_NOPAY		0x29c
+
+#define MMC_XGMAC_RX_UDP_GD		0x2a4
+#define MMC_XGMAC_RX_UDP_ERR		0x2ac
+#define MMC_XGMAC_RX_TCP_GD		0x2b4
+#define MMC_XGMAC_RX_TCP_ERR		0x2bc
+#define MMC_XGMAC_RX_ICMP_GD		0x2c4
+#define MMC_XGMAC_RX_ICMP_ERR		0x2cc
+
+#define MMC_XGMAC_RX_IPV4_GD_OCTETS	0x2d4
+#define MMC_XGMAC_RX_IPV4_HDERR_OCTETS	0x2dc
+#define MMC_XGMAC_RX_IPV4_NOPAY_OCTETS	0x2e4
+#define MMC_XGMAC_RX_IPV4_FRAG_OCTETS	0x2ec
+#define MMC_XGMAC_RX_IPV4_UDSBL_OCTETS	0x2f4
+
+#define MMC_XGMAC_RX_IPV6_GD_OCTETS	0x2fc
+#define MMC_XGMAC_RX_IPV6_HDERR_OCTETS	0x304
+#define MMC_XGMAC_RX_IPV6_NOPAY_OCTETS	0x30c
+
+#define MMC_XGMAC_RX_UDP_GD_OCTETS	0x314
+#define MMC_XGMAC_RX_UDP_ERR_OCTETS	0x31c
+#define MMC_XGMAC_RX_TCP_GD_OCTETS	0x324
+#define MMC_XGMAC_RX_TCP_ERR_OCTETS	0x32c
+#define MMC_XGMAC_RX_ICMP_GD_OCTETS	0x334
+#define MMC_XGMAC_RX_ICMP_ERR_OCTETS	0x33c
+
+static void dwmac_mmc_ctrl(void __iomem *mmcaddr, unsigned int mode)
+{
+	u32 value = readl(mmcaddr + MMC_CNTRL);
+
+	value |= (mode & 0x3F);
+
+	writel(value, mmcaddr + MMC_CNTRL);
+
+	pr_debug("stmmac: MMC ctrl register (offset 0x%x): 0x%08x\n",
+		 MMC_CNTRL, value);
+}
+
+/* To mask all interrupts.*/
+static void dwmac_mmc_intr_all_mask(void __iomem *mmcaddr)
+{
+	writel(MMC_DEFAULT_MASK, mmcaddr + MMC_RX_INTR_MASK);
+	writel(MMC_DEFAULT_MASK, mmcaddr + MMC_TX_INTR_MASK);
+	writel(MMC_DEFAULT_MASK, mmcaddr + MMC_RX_IPC_INTR_MASK);
+}
+
+/* This reads the MAC core counters (if actaully supported).
+ * by default the MMC core is programmed to reset each
+ * counter after a read. So all the field of the mmc struct
+ * have to be incremented.
+ */
+static void dwmac_mmc_read(void __iomem *mmcaddr, struct stmmac_counters *mmc)
+{
+	mmc->mmc_tx_octetcount_gb += readl(mmcaddr + MMC_TX_OCTETCOUNT_GB);
+	mmc->mmc_tx_framecount_gb += readl(mmcaddr + MMC_TX_FRAMECOUNT_GB);
+	mmc->mmc_tx_broadcastframe_g += readl(mmcaddr +
+					      MMC_TX_BROADCASTFRAME_G);
+	mmc->mmc_tx_multicastframe_g += readl(mmcaddr +
+					      MMC_TX_MULTICASTFRAME_G);
+	mmc->mmc_tx_64_octets_gb += readl(mmcaddr + MMC_TX_64_OCTETS_GB);
+	mmc->mmc_tx_65_to_127_octets_gb +=
+	    readl(mmcaddr + MMC_TX_65_TO_127_OCTETS_GB);
+	mmc->mmc_tx_128_to_255_octets_gb +=
+	    readl(mmcaddr + MMC_TX_128_TO_255_OCTETS_GB);
+	mmc->mmc_tx_256_to_511_octets_gb +=
+	    readl(mmcaddr + MMC_TX_256_TO_511_OCTETS_GB);
+	mmc->mmc_tx_512_to_1023_octets_gb +=
+	    readl(mmcaddr + MMC_TX_512_TO_1023_OCTETS_GB);
+	mmc->mmc_tx_1024_to_max_octets_gb +=
+	    readl(mmcaddr + MMC_TX_1024_TO_MAX_OCTETS_GB);
+	mmc->mmc_tx_unicast_gb += readl(mmcaddr + MMC_TX_UNICAST_GB);
+	mmc->mmc_tx_multicast_gb += readl(mmcaddr + MMC_TX_MULTICAST_GB);
+	mmc->mmc_tx_broadcast_gb += readl(mmcaddr + MMC_TX_BROADCAST_GB);
+	mmc->mmc_tx_underflow_error += readl(mmcaddr + MMC_TX_UNDERFLOW_ERROR);
+	mmc->mmc_tx_singlecol_g += readl(mmcaddr + MMC_TX_SINGLECOL_G);
+	mmc->mmc_tx_multicol_g += readl(mmcaddr + MMC_TX_MULTICOL_G);
+	mmc->mmc_tx_deferred += readl(mmcaddr + MMC_TX_DEFERRED);
+	mmc->mmc_tx_latecol += readl(mmcaddr + MMC_TX_LATECOL);
+	mmc->mmc_tx_exesscol += readl(mmcaddr + MMC_TX_EXESSCOL);
+	mmc->mmc_tx_carrier_error += readl(mmcaddr + MMC_TX_CARRIER_ERROR);
+	mmc->mmc_tx_octetcount_g += readl(mmcaddr + MMC_TX_OCTETCOUNT_G);
+	mmc->mmc_tx_framecount_g += readl(mmcaddr + MMC_TX_FRAMECOUNT_G);
+	mmc->mmc_tx_excessdef += readl(mmcaddr + MMC_TX_EXCESSDEF);
+	mmc->mmc_tx_pause_frame += readl(mmcaddr + MMC_TX_PAUSE_FRAME);
+	mmc->mmc_tx_vlan_frame_g += readl(mmcaddr + MMC_TX_VLAN_FRAME_G);
+	mmc->mmc_tx_oversize_g	 += readl(mmcaddr + MMC_TX_OVERSIZE_G);
+	mmc->mmc_tx_lpi_usec += readl(mmcaddr + MMC_TX_LPI_USEC);
+	mmc->mmc_tx_lpi_tran += readl(mmcaddr + MMC_TX_LPI_TRAN);
+
+	/* MMC RX counter registers */
+	mmc->mmc_rx_framecount_gb += readl(mmcaddr + MMC_RX_FRAMECOUNT_GB);
+	mmc->mmc_rx_octetcount_gb += readl(mmcaddr + MMC_RX_OCTETCOUNT_GB);
+	mmc->mmc_rx_octetcount_g += readl(mmcaddr + MMC_RX_OCTETCOUNT_G);
+	mmc->mmc_rx_broadcastframe_g += readl(mmcaddr +
+					      MMC_RX_BROADCASTFRAME_G);
+	mmc->mmc_rx_multicastframe_g += readl(mmcaddr +
+					      MMC_RX_MULTICASTFRAME_G);
+	mmc->mmc_rx_crc_error += readl(mmcaddr + MMC_RX_CRC_ERROR);
+	mmc->mmc_rx_align_error += readl(mmcaddr + MMC_RX_ALIGN_ERROR);
+	mmc->mmc_rx_run_error += readl(mmcaddr + MMC_RX_RUN_ERROR);
+	mmc->mmc_rx_jabber_error += readl(mmcaddr + MMC_RX_JABBER_ERROR);
+	mmc->mmc_rx_undersize_g += readl(mmcaddr + MMC_RX_UNDERSIZE_G);
+	mmc->mmc_rx_oversize_g += readl(mmcaddr + MMC_RX_OVERSIZE_G);
+	mmc->mmc_rx_64_octets_gb += readl(mmcaddr + MMC_RX_64_OCTETS_GB);
+	mmc->mmc_rx_65_to_127_octets_gb +=
+	    readl(mmcaddr + MMC_RX_65_TO_127_OCTETS_GB);
+	mmc->mmc_rx_128_to_255_octets_gb +=
+	    readl(mmcaddr + MMC_RX_128_TO_255_OCTETS_GB);
+	mmc->mmc_rx_256_to_511_octets_gb +=
+	    readl(mmcaddr + MMC_RX_256_TO_511_OCTETS_GB);
+	mmc->mmc_rx_512_to_1023_octets_gb +=
+	    readl(mmcaddr + MMC_RX_512_TO_1023_OCTETS_GB);
+	mmc->mmc_rx_1024_to_max_octets_gb +=
+	    readl(mmcaddr + MMC_RX_1024_TO_MAX_OCTETS_GB);
+	mmc->mmc_rx_unicast_g += readl(mmcaddr + MMC_RX_UNICAST_G);
+	mmc->mmc_rx_length_error += readl(mmcaddr + MMC_RX_LENGTH_ERROR);
+	mmc->mmc_rx_autofrangetype += readl(mmcaddr + MMC_RX_AUTOFRANGETYPE);
+	mmc->mmc_rx_pause_frames += readl(mmcaddr + MMC_RX_PAUSE_FRAMES);
+	mmc->mmc_rx_fifo_overflow += readl(mmcaddr + MMC_RX_FIFO_OVERFLOW);
+	mmc->mmc_rx_vlan_frames_gb += readl(mmcaddr + MMC_RX_VLAN_FRAMES_GB);
+	mmc->mmc_rx_watchdog_error += readl(mmcaddr + MMC_RX_WATCHDOG_ERROR);
+	mmc->mmc_rx_error += readl(mmcaddr + MMC_RX_ERROR);
+	mmc->mmc_rx_lpi_usec += readl(mmcaddr + MMC_RX_LPI_USEC);
+	mmc->mmc_rx_lpi_tran += readl(mmcaddr + MMC_RX_LPI_TRAN);
+
+	/* IPv4 */
+	mmc->mmc_rx_ipv4_gd += readl(mmcaddr + MMC_RX_IPV4_GD);
+	mmc->mmc_rx_ipv4_hderr += readl(mmcaddr + MMC_RX_IPV4_HDERR);
+	mmc->mmc_rx_ipv4_nopay += readl(mmcaddr + MMC_RX_IPV4_NOPAY);
+	mmc->mmc_rx_ipv4_frag += readl(mmcaddr + MMC_RX_IPV4_FRAG);
+	mmc->mmc_rx_ipv4_udsbl += readl(mmcaddr + MMC_RX_IPV4_UDSBL);
+
+	mmc->mmc_rx_ipv4_gd_octets += readl(mmcaddr + MMC_RX_IPV4_GD_OCTETS);
+	mmc->mmc_rx_ipv4_hderr_octets +=
+	    readl(mmcaddr + MMC_RX_IPV4_HDERR_OCTETS);
+	mmc->mmc_rx_ipv4_nopay_octets +=
+	    readl(mmcaddr + MMC_RX_IPV4_NOPAY_OCTETS);
+	mmc->mmc_rx_ipv4_frag_octets += readl(mmcaddr +
+					      MMC_RX_IPV4_FRAG_OCTETS);
+	mmc->mmc_rx_ipv4_udsbl_octets +=
+	    readl(mmcaddr + MMC_RX_IPV4_UDSBL_OCTETS);
+
+	/* IPV6 */
+	mmc->mmc_rx_ipv6_gd_octets += readl(mmcaddr + MMC_RX_IPV6_GD_OCTETS);
+	mmc->mmc_rx_ipv6_hderr_octets +=
+	    readl(mmcaddr + MMC_RX_IPV6_HDERR_OCTETS);
+	mmc->mmc_rx_ipv6_nopay_octets +=
+	    readl(mmcaddr + MMC_RX_IPV6_NOPAY_OCTETS);
+
+	mmc->mmc_rx_ipv6_gd += readl(mmcaddr + MMC_RX_IPV6_GD);
+	mmc->mmc_rx_ipv6_hderr += readl(mmcaddr + MMC_RX_IPV6_HDERR);
+	mmc->mmc_rx_ipv6_nopay += readl(mmcaddr + MMC_RX_IPV6_NOPAY);
+
+	/* Protocols */
+	mmc->mmc_rx_udp_gd += readl(mmcaddr + MMC_RX_UDP_GD);
+	mmc->mmc_rx_udp_err += readl(mmcaddr + MMC_RX_UDP_ERR);
+	mmc->mmc_rx_tcp_gd += readl(mmcaddr + MMC_RX_TCP_GD);
+	mmc->mmc_rx_tcp_err += readl(mmcaddr + MMC_RX_TCP_ERR);
+	mmc->mmc_rx_icmp_gd += readl(mmcaddr + MMC_RX_ICMP_GD);
+	mmc->mmc_rx_icmp_err += readl(mmcaddr + MMC_RX_ICMP_ERR);
+
+	mmc->mmc_rx_udp_gd_octets += readl(mmcaddr + MMC_RX_UDP_GD_OCTETS);
+	mmc->mmc_rx_udp_err_octets += readl(mmcaddr + MMC_RX_UDP_ERR_OCTETS);
+	mmc->mmc_rx_tcp_gd_octets += readl(mmcaddr + MMC_RX_TCP_GD_OCTETS);
+	mmc->mmc_rx_tcp_err_octets += readl(mmcaddr + MMC_RX_TCP_ERR_OCTETS);
+	mmc->mmc_rx_icmp_gd_octets += readl(mmcaddr + MMC_RX_ICMP_GD_OCTETS);
+	mmc->mmc_rx_icmp_err_octets += readl(mmcaddr + MMC_RX_ICMP_ERR_OCTETS);
+
+	mmc->mmc_tx_fpe_fragment_cntr += readl(mmcaddr + MMC_TX_FPE_FRAG);
+	mmc->mmc_tx_hold_req_cntr += readl(mmcaddr + MMC_TX_HOLD_REQ);
+	mmc->mmc_rx_packet_assembly_err_cntr +=
+		readl(mmcaddr + MMC_RX_PKT_ASSEMBLY_ERR);
+	mmc->mmc_rx_packet_smd_err_cntr += readl(mmcaddr + MMC_RX_PKT_SMD_ERR);
+	mmc->mmc_rx_packet_assembly_ok_cntr +=
+		readl(mmcaddr + MMC_RX_PKT_ASSEMBLY_OK);
+	mmc->mmc_rx_fpe_fragment_cntr += readl(mmcaddr + MMC_RX_FPE_FRAG);
+}
+
+const struct stmmac_mmc_ops ec_dwmac_mmc_ops = {
+	.ctrl = dwmac_mmc_ctrl,
+	.intr_all_mask = dwmac_mmc_intr_all_mask,
+	.read = dwmac_mmc_read,
+};
+
+static void dwxgmac_mmc_ctrl(void __iomem *mmcaddr, unsigned int mode)
+{
+	u32 value = readl(mmcaddr + MMC_CNTRL);
+
+	value |= (mode & 0x3F);
+
+	writel(value, mmcaddr + MMC_CNTRL);
+}
+
+static void dwxgmac_mmc_intr_all_mask(void __iomem *mmcaddr)
+{
+	writel(0x0, mmcaddr + MMC_RX_INTR_MASK);
+	writel(0x0, mmcaddr + MMC_TX_INTR_MASK);
+	writel(MMC_DEFAULT_MASK, mmcaddr + MMC_XGMAC_TX_FPE_INTR_MASK);
+	writel(MMC_DEFAULT_MASK, mmcaddr + MMC_XGMAC_RX_FPE_INTR_MASK);
+	writel(MMC_DEFAULT_MASK, mmcaddr + MMC_XGMAC_RX_IPC_INTR_MASK);
+}
+
+static void dwxgmac_read_mmc_reg(void __iomem *addr, u32 reg, u32 *dest)
+{
+	u64 tmp = 0;
+
+	tmp += readl(addr + reg);
+	tmp += ((u64 )readl(addr + reg + 0x4)) << 32;
+	if (tmp > GENMASK(31, 0))
+		*dest = ~0x0;
+	else
+		*dest = *dest + tmp;
+}
+
+/* This reads the MAC core counters (if actaully supported).
+ * by default the MMC core is programmed to reset each
+ * counter after a read. So all the field of the mmc struct
+ * have to be incremented.
+ */
+static void dwxgmac_mmc_read(void __iomem *mmcaddr, struct stmmac_counters *mmc)
+{
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_OCTET_GB,
+			     &mmc->mmc_tx_octetcount_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_PKT_GB,
+			     &mmc->mmc_tx_framecount_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_BROAD_PKT_G,
+			     &mmc->mmc_tx_broadcastframe_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_MULTI_PKT_G,
+			     &mmc->mmc_tx_multicastframe_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_64OCT_GB,
+			     &mmc->mmc_tx_64_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_65OCT_GB,
+			     &mmc->mmc_tx_65_to_127_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_128OCT_GB,
+			     &mmc->mmc_tx_128_to_255_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_256OCT_GB,
+			     &mmc->mmc_tx_256_to_511_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_512OCT_GB,
+			     &mmc->mmc_tx_512_to_1023_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_1024OCT_GB,
+			     &mmc->mmc_tx_1024_to_max_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_UNI_PKT_GB,
+			     &mmc->mmc_tx_unicast_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_MULTI_PKT_GB,
+			     &mmc->mmc_tx_multicast_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_BROAD_PKT_GB,
+			     &mmc->mmc_tx_broadcast_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_UNDER,
+			     &mmc->mmc_tx_underflow_error);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_OCTET_G,
+			     &mmc->mmc_tx_octetcount_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_PKT_G,
+			     &mmc->mmc_tx_framecount_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_PAUSE,
+			     &mmc->mmc_tx_pause_frame);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_VLAN_PKT_G,
+			     &mmc->mmc_tx_vlan_frame_g);
+	mmc->mmc_tx_lpi_usec += readl(mmcaddr + MMC_XGMAC_TX_LPI_USEC);
+	mmc->mmc_tx_lpi_tran += readl(mmcaddr + MMC_XGMAC_TX_LPI_TRAN);
+
+	/* MMC RX counter registers */
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_PKT_GB,
+			     &mmc->mmc_rx_framecount_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_OCTET_GB,
+			     &mmc->mmc_rx_octetcount_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_OCTET_G,
+			     &mmc->mmc_rx_octetcount_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_BROAD_PKT_G,
+			     &mmc->mmc_rx_broadcastframe_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_MULTI_PKT_G,
+			     &mmc->mmc_rx_multicastframe_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_CRC_ERR,
+			     &mmc->mmc_rx_crc_error);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_CRC_ERR,
+			     &mmc->mmc_rx_crc_error);
+	mmc->mmc_rx_run_error += readl(mmcaddr + MMC_XGMAC_RX_RUNT_ERR);
+	mmc->mmc_rx_jabber_error += readl(mmcaddr + MMC_XGMAC_RX_JABBER_ERR);
+	mmc->mmc_rx_undersize_g += readl(mmcaddr + MMC_XGMAC_RX_UNDER);
+	mmc->mmc_rx_oversize_g += readl(mmcaddr + MMC_XGMAC_RX_OVER);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_64OCT_GB,
+			     &mmc->mmc_rx_64_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_65OCT_GB,
+			     &mmc->mmc_rx_65_to_127_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_128OCT_GB,
+			     &mmc->mmc_rx_128_to_255_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_256OCT_GB,
+			     &mmc->mmc_rx_256_to_511_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_512OCT_GB,
+			     &mmc->mmc_rx_512_to_1023_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_1024OCT_GB,
+			     &mmc->mmc_rx_1024_to_max_octets_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_UNI_PKT_G,
+			     &mmc->mmc_rx_unicast_g);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_LENGTH_ERR,
+			     &mmc->mmc_rx_length_error);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_RANGE,
+			     &mmc->mmc_rx_autofrangetype);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_PAUSE,
+			     &mmc->mmc_rx_pause_frames);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_FIFOOVER_PKT,
+			     &mmc->mmc_rx_fifo_overflow);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_VLAN_PKT_GB,
+			     &mmc->mmc_rx_vlan_frames_gb);
+	mmc->mmc_rx_watchdog_error += readl(mmcaddr + MMC_XGMAC_RX_WATCHDOG_ERR);
+	mmc->mmc_rx_lpi_usec += readl(mmcaddr + MMC_XGMAC_RX_LPI_USEC);
+	mmc->mmc_rx_lpi_tran += readl(mmcaddr + MMC_XGMAC_RX_LPI_TRAN);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_DISCARD_PKT_GB,
+			     &mmc->mmc_rx_discard_frames_gb);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_DISCARD_OCT_GB,
+			     &mmc->mmc_rx_discard_octets_gb);
+	mmc->mmc_rx_align_err_frames +=
+		readl(mmcaddr + MMC_XGMAC_RX_ALIGN_ERR_PKT);
+
+	mmc->mmc_sgf_pass_fragment_cntr +=
+		readl(mmcaddr + MMC_XGMAC_SGF_PASS_PKT);
+	mmc->mmc_sgf_fail_fragment_cntr +=
+		readl(mmcaddr + MMC_XGMAC_SGF_FAIL_PKT);
+	mmc->mmc_tx_fpe_fragment_cntr += readl(mmcaddr + MMC_XGMAC_TX_FPE_FRAG);
+	mmc->mmc_tx_hold_req_cntr += readl(mmcaddr + MMC_XGMAC_TX_HOLD_REQ);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_TX_GATE_OVERRUN,
+			     &mmc->mmc_tx_gate_overrun_cntr);
+	mmc->mmc_rx_packet_assembly_err_cntr +=
+		readl(mmcaddr + MMC_XGMAC_RX_PKT_ASSEMBLY_ERR);
+	mmc->mmc_rx_packet_smd_err_cntr +=
+		readl(mmcaddr + MMC_XGMAC_RX_PKT_SMD_ERR);
+	mmc->mmc_rx_packet_assembly_ok_cntr +=
+		readl(mmcaddr + MMC_XGMAC_RX_PKT_ASSEMBLY_OK);
+	mmc->mmc_rx_fpe_fragment_cntr +=
+		readl(mmcaddr + MMC_XGMAC_RX_FPE_FRAG);
+
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_GD,
+			     &mmc->mmc_rx_ipv4_gd);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_HDERR,
+			     &mmc->mmc_rx_ipv4_hderr);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_NOPAY,
+			     &mmc->mmc_rx_ipv4_nopay);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_FRAG,
+			     &mmc->mmc_rx_ipv4_frag);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_UDSBL,
+			     &mmc->mmc_rx_ipv4_udsbl);
+
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV6_GD,
+			     &mmc->mmc_rx_ipv6_gd);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV6_HDERR,
+			     &mmc->mmc_rx_ipv6_hderr);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV6_NOPAY,
+			     &mmc->mmc_rx_ipv6_nopay);
+
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_UDP_GD,
+			     &mmc->mmc_rx_udp_gd);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_UDP_ERR,
+			     &mmc->mmc_rx_udp_err);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_TCP_GD,
+			     &mmc->mmc_rx_tcp_gd);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_TCP_ERR,
+			     &mmc->mmc_rx_tcp_err);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_ICMP_GD,
+			     &mmc->mmc_rx_icmp_gd);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_ICMP_ERR,
+			     &mmc->mmc_rx_icmp_err);
+
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_GD_OCTETS,
+			     &mmc->mmc_rx_ipv4_gd_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_HDERR_OCTETS,
+			     &mmc->mmc_rx_ipv4_hderr_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_NOPAY_OCTETS,
+			     &mmc->mmc_rx_ipv4_nopay_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_FRAG_OCTETS,
+			     &mmc->mmc_rx_ipv4_frag_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV4_UDSBL_OCTETS,
+			     &mmc->mmc_rx_ipv4_udsbl_octets);
+
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV6_GD_OCTETS,
+			     &mmc->mmc_rx_ipv6_gd_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV6_HDERR_OCTETS,
+			     &mmc->mmc_rx_ipv6_hderr_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_IPV6_NOPAY_OCTETS,
+			     &mmc->mmc_rx_ipv6_nopay_octets);
+
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_UDP_GD_OCTETS,
+			     &mmc->mmc_rx_udp_gd_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_UDP_ERR_OCTETS,
+			     &mmc->mmc_rx_udp_err_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_TCP_GD_OCTETS,
+			     &mmc->mmc_rx_tcp_gd_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_TCP_ERR_OCTETS,
+			     &mmc->mmc_rx_tcp_err_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_ICMP_GD_OCTETS,
+			     &mmc->mmc_rx_icmp_gd_octets);
+	dwxgmac_read_mmc_reg(mmcaddr, MMC_XGMAC_RX_ICMP_ERR_OCTETS,
+			     &mmc->mmc_rx_icmp_err_octets);
+}
+
+const struct stmmac_mmc_ops ec_dwxgmac_mmc_ops = {
+	.ctrl = dwxgmac_mmc_ctrl,
+	.intr_all_mask = dwxgmac_mmc_intr_all_mask,
+	.read = dwxgmac_mmc_read,
+};
diff --git a/drivers/net/ethercat/device/K3/norm_desc-ethercat.c b/drivers/net/ethercat/device/K3/norm_desc-ethercat.c
new file mode 100644
index 000000000000..cc795b1e7c1b
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/norm_desc-ethercat.c
@@ -0,0 +1,319 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This contains the functions to handle the normal descriptors.
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/stmmac.h>
+#include "common-ethercat.h"
+#include "descs_com-ethercat.h"
+
+static int ndesc_get_tx_status(struct stmmac_extra_stats *x,
+			       struct dma_desc *p, void __iomem *ioaddr)
+{
+	unsigned int tdes0 = le32_to_cpu(p->des0);
+	unsigned int tdes1 = le32_to_cpu(p->des1);
+	int ret = tx_done;
+
+	/* Get tx owner first */
+	if (unlikely(tdes0 & TDES0_OWN))
+		return tx_dma_own;
+
+	/* Verify tx error by looking at the last segment. */
+	if (likely(!(tdes1 & TDES1_LAST_SEGMENT)))
+		return tx_not_ls;
+
+	if (unlikely(tdes0 & TDES0_ERROR_SUMMARY)) {
+		if (unlikely(tdes0 & TDES0_UNDERFLOW_ERROR)) {
+			x->tx_underflow++;
+		}
+		if (unlikely(tdes0 & TDES0_NO_CARRIER)) {
+			x->tx_carrier++;
+		}
+		if (unlikely(tdes0 & TDES0_LOSS_CARRIER)) {
+			x->tx_losscarrier++;
+		}
+		if (unlikely((tdes0 & TDES0_EXCESSIVE_DEFERRAL) ||
+			     (tdes0 & TDES0_EXCESSIVE_COLLISIONS) ||
+			     (tdes0 & TDES0_LATE_COLLISION))) {
+			unsigned int collisions;
+
+			collisions = (tdes0 & TDES0_COLLISION_COUNT_MASK) >> 3;
+			x->tx_collision += collisions;
+		}
+		ret = tx_err;
+	}
+
+	if (tdes0 & TDES0_VLAN_FRAME)
+		x->tx_vlan++;
+
+	if (unlikely(tdes0 & TDES0_DEFERRED))
+		x->tx_deferred++;
+
+	return ret;
+}
+
+static int ndesc_get_tx_len(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des1) & RDES1_BUFFER1_SIZE_MASK);
+}
+
+/* This function verifies if each incoming frame has some errors
+ * and, if required, updates the multicast statistics.
+ * In case of success, it returns good_frame because the GMAC device
+ * is supposed to be able to compute the csum in HW. */
+static int ndesc_get_rx_status(struct stmmac_extra_stats *x,
+			       struct dma_desc *p)
+{
+	int ret = good_frame;
+	unsigned int rdes0 = le32_to_cpu(p->des0);
+
+	if (unlikely(rdes0 & RDES0_OWN))
+		return dma_own;
+
+	if (unlikely(!(rdes0 & RDES0_LAST_DESCRIPTOR))) {
+		x->rx_length++;
+		return discard_frame;
+	}
+
+	if (unlikely(rdes0 & RDES0_ERROR_SUMMARY)) {
+		if (unlikely(rdes0 & RDES0_DESCRIPTOR_ERROR))
+			x->rx_desc++;
+		if (unlikely(rdes0 & RDES0_SA_FILTER_FAIL))
+			x->sa_filter_fail++;
+		if (unlikely(rdes0 & RDES0_OVERFLOW_ERROR))
+			x->overflow_error++;
+		if (unlikely(rdes0 & RDES0_IPC_CSUM_ERROR))
+			x->ipc_csum_error++;
+		if (unlikely(rdes0 & RDES0_COLLISION)) {
+			x->rx_collision++;
+		}
+		if (unlikely(rdes0 & RDES0_CRC_ERROR)) {
+			x->rx_crc_errors++;
+		}
+		ret = discard_frame;
+	}
+	if (unlikely(rdes0 & RDES0_DRIBBLING))
+		x->dribbling_bit++;
+
+	if (unlikely(rdes0 & RDES0_LENGTH_ERROR)) {
+		x->rx_length++;
+		ret = discard_frame;
+	}
+	if (unlikely(rdes0 & RDES0_MII_ERROR)) {
+		x->rx_mii++;
+		ret = discard_frame;
+	}
+#ifdef STMMAC_VLAN_TAG_USED
+	if (rdes0 & RDES0_VLAN_TAG)
+		x->vlan_tag++;
+#endif
+	return ret;
+}
+
+static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, int mode,
+			       int end, int bfsize)
+{
+	int bfsize1;
+
+	p->des0 |= cpu_to_le32(RDES0_OWN);
+
+	bfsize1 = min(bfsize, BUF_SIZE_2KiB - 1);
+	p->des1 |= cpu_to_le32(bfsize1 & RDES1_BUFFER1_SIZE_MASK);
+
+	if (mode == STMMAC_CHAIN_MODE)
+		ndesc_rx_set_on_chain(p, end);
+	else
+		ndesc_rx_set_on_ring(p, end, bfsize);
+
+	if (disable_rx_ic)
+		p->des1 |= cpu_to_le32(RDES1_DISABLE_IC);
+}
+
+static void ndesc_init_tx_desc(struct dma_desc *p, int mode, int end)
+{
+	p->des0 &= cpu_to_le32(~TDES0_OWN);
+	if (mode == STMMAC_CHAIN_MODE)
+		ndesc_tx_set_on_chain(p);
+	else
+		ndesc_end_tx_desc_on_ring(p, end);
+}
+
+static int ndesc_get_tx_owner(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des0) & TDES0_OWN) >> 31;
+}
+
+static void ndesc_set_tx_owner(struct dma_desc *p)
+{
+	p->des0 |= cpu_to_le32(TDES0_OWN);
+}
+
+static void ndesc_set_rx_owner(struct dma_desc *p, int disable_rx_ic)
+{
+	p->des0 |= cpu_to_le32(RDES0_OWN);
+}
+
+static int ndesc_get_tx_ls(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des1) & TDES1_LAST_SEGMENT) >> 30;
+}
+
+static void ndesc_release_tx_desc(struct dma_desc *p, int mode)
+{
+	int ter = (le32_to_cpu(p->des1) & TDES1_END_RING) >> 25;
+
+	memset(p, 0, offsetof(struct dma_desc, des2));
+	if (mode == STMMAC_CHAIN_MODE)
+		ndesc_tx_set_on_chain(p);
+	else
+		ndesc_end_tx_desc_on_ring(p, ter);
+}
+
+static void ndesc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
+				  bool csum_flag, int mode, bool tx_own,
+				  bool ls, unsigned int tot_pkt_len)
+{
+	unsigned int tdes1 = le32_to_cpu(p->des1);
+
+	if (is_fs)
+		tdes1 |= TDES1_FIRST_SEGMENT;
+	else
+		tdes1 &= ~TDES1_FIRST_SEGMENT;
+
+	if (likely(csum_flag))
+		tdes1 |= (TX_CIC_FULL) << TDES1_CHECKSUM_INSERTION_SHIFT;
+	else
+		tdes1 &= ~(TX_CIC_FULL << TDES1_CHECKSUM_INSERTION_SHIFT);
+
+	if (ls)
+		tdes1 |= TDES1_LAST_SEGMENT;
+
+	p->des1 = cpu_to_le32(tdes1);
+
+	if (mode == STMMAC_CHAIN_MODE)
+		norm_set_tx_desc_len_on_chain(p, len);
+	else
+		norm_set_tx_desc_len_on_ring(p, len);
+
+	if (tx_own)
+		p->des0 |= cpu_to_le32(TDES0_OWN);
+}
+
+static void ndesc_set_tx_ic(struct dma_desc *p)
+{
+	p->des1 |= cpu_to_le32(TDES1_INTERRUPT);
+}
+
+static int ndesc_get_rx_frame_len(struct dma_desc *p, int rx_coe_type)
+{
+	unsigned int csum = 0;
+
+	/* The type-1 checksum offload engines append the checksum at
+	 * the end of frame and the two bytes of checksum are added in
+	 * the length.
+	 * Adjust for that in the framelen for type-1 checksum offload
+	 * engines
+	 */
+	if (rx_coe_type == STMMAC_RX_COE_TYPE1)
+		csum = 2;
+
+	return (((le32_to_cpu(p->des0) & RDES0_FRAME_LEN_MASK)
+				>> RDES0_FRAME_LEN_SHIFT) -
+		csum);
+
+}
+
+static void ndesc_enable_tx_timestamp(struct dma_desc *p)
+{
+	p->des1 |= cpu_to_le32(TDES1_TIME_STAMP_ENABLE);
+}
+
+static int ndesc_get_tx_timestamp_status(struct dma_desc *p)
+{
+	return (le32_to_cpu(p->des0) & TDES0_TIME_STAMP_STATUS) >> 17;
+}
+
+static void ndesc_get_timestamp(void *desc, u32 ats, u64 *ts)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+	u64 ns;
+
+	ns = le32_to_cpu(p->des2);
+	/* convert high/sec time stamp value to nanosecond */
+	ns += le32_to_cpu(p->des3) * 1000000000ULL;
+
+	*ts = ns;
+}
+
+static int ndesc_get_rx_timestamp_status(void *desc, void *next_desc, u32 ats)
+{
+	struct dma_desc *p = (struct dma_desc *)desc;
+
+	if ((le32_to_cpu(p->des2) == 0xffffffff) &&
+	    (le32_to_cpu(p->des3) == 0xffffffff))
+		/* timestamp is corrupted, hence don't store it */
+		return 0;
+	else
+		return 1;
+}
+
+static void ndesc_display_ring(void *head, unsigned int size, bool rx,
+			       dma_addr_t dma_rx_phy, unsigned int desc_size)
+{
+	struct dma_desc *p = (struct dma_desc *)head;
+	dma_addr_t dma_addr;
+	int i;
+
+	pr_info("%s descriptor ring:\n", rx ? "RX" : "TX");
+
+	for (i = 0; i < size; i++) {
+		u64 x;
+		dma_addr = dma_rx_phy + i * sizeof(*p);
+
+		x = *(u64 *)p;
+		pr_info("%03d [%pad]: 0x%x 0x%x 0x%x 0x%x",
+			i, &dma_addr,
+			(unsigned int)x, (unsigned int)(x >> 32),
+			p->des2, p->des3);
+		p++;
+	}
+	pr_info("\n");
+}
+
+static void ndesc_set_addr(struct dma_desc *p, dma_addr_t addr)
+{
+	p->des2 = cpu_to_le32(addr);
+}
+
+static void ndesc_clear(struct dma_desc *p)
+{
+	p->des2 = 0;
+}
+
+const struct stmmac_desc_ops ec_ndesc_ops = {
+	.tx_status = ndesc_get_tx_status,
+	.rx_status = ndesc_get_rx_status,
+	.get_tx_len = ndesc_get_tx_len,
+	.init_rx_desc = ndesc_init_rx_desc,
+	.init_tx_desc = ndesc_init_tx_desc,
+	.get_tx_owner = ndesc_get_tx_owner,
+	.release_tx_desc = ndesc_release_tx_desc,
+	.prepare_tx_desc = ndesc_prepare_tx_desc,
+	.set_tx_ic = ndesc_set_tx_ic,
+	.get_tx_ls = ndesc_get_tx_ls,
+	.set_tx_owner = ndesc_set_tx_owner,
+	.set_rx_owner = ndesc_set_rx_owner,
+	.get_rx_frame_len = ndesc_get_rx_frame_len,
+	.enable_tx_timestamp = ndesc_enable_tx_timestamp,
+	.get_tx_timestamp_status = ndesc_get_tx_timestamp_status,
+	.get_timestamp = ndesc_get_timestamp,
+	.get_rx_timestamp_status = ndesc_get_rx_timestamp_status,
+	.display_ring = ndesc_display_ring,
+	.set_addr = ndesc_set_addr,
+	.clear = ndesc_clear,
+};
diff --git a/drivers/net/ethercat/device/K3/ring_mode-ethercat.c b/drivers/net/ethercat/device/K3/ring_mode-ethercat.c
new file mode 100644
index 000000000000..03466d5e9246
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/ring_mode-ethercat.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  Specialised functions for managing Ring mode
+
+  Copyright(C) 2011  STMicroelectronics Ltd
+
+  It defines all the functions used to handle the normal/enhanced
+  descriptors in case of the DMA is configured to work in chained or
+  in ring mode.
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include "stmmac-ethercat.h"
+
+static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
+		     int csum)
+{
+	unsigned int nopaged_len = skb_headlen(skb);
+	struct stmmac_priv *priv = tx_q->priv_data;
+	unsigned int entry = tx_q->cur_tx;
+	unsigned int bmax, len, des2;
+	struct dma_desc *desc;
+
+	if (priv->extend_desc)
+		desc = (struct dma_desc *)(tx_q->dma_etx + entry);
+	else
+		desc = tx_q->dma_tx + entry;
+
+	if (priv->plat->enh_desc)
+		bmax = BUF_SIZE_8KiB;
+	else
+		bmax = BUF_SIZE_2KiB;
+
+	len = nopaged_len - bmax;
+
+	if (nopaged_len > BUF_SIZE_8KiB) {
+
+		des2 = dma_map_single(priv->device, skb->data, bmax,
+				      DMA_TO_DEVICE);
+		desc->des2 = cpu_to_le32(des2);
+		if (dma_mapping_error(priv->device, des2))
+			return -1;
+
+		tx_q->tx_skbuff_dma[entry].buf = des2;
+		tx_q->tx_skbuff_dma[entry].len = bmax;
+		tx_q->tx_skbuff_dma[entry].is_jumbo = true;
+
+		desc->des3 = cpu_to_le32(des2 + BUF_SIZE_4KiB);
+		stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum,
+				STMMAC_RING_MODE, 0, false, skb->len);
+		tx_q->tx_skbuff[entry] = NULL;
+		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+
+		if (priv->extend_desc)
+			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
+		else
+			desc = tx_q->dma_tx + entry;
+
+		des2 = dma_map_single(priv->device, skb->data + bmax, len,
+				      DMA_TO_DEVICE);
+		desc->des2 = cpu_to_le32(des2);
+		if (dma_mapping_error(priv->device, des2))
+			return -1;
+		tx_q->tx_skbuff_dma[entry].buf = des2;
+		tx_q->tx_skbuff_dma[entry].len = len;
+		tx_q->tx_skbuff_dma[entry].is_jumbo = true;
+
+		desc->des3 = cpu_to_le32(des2 + BUF_SIZE_4KiB);
+		stmmac_prepare_tx_desc(priv, desc, 0, len, csum,
+				STMMAC_RING_MODE, 1, !skb_is_nonlinear(skb),
+				skb->len);
+	} else {
+		des2 = dma_map_single(priv->device, skb->data,
+				      nopaged_len, DMA_TO_DEVICE);
+		desc->des2 = cpu_to_le32(des2);
+		if (dma_mapping_error(priv->device, des2))
+			return -1;
+		tx_q->tx_skbuff_dma[entry].buf = des2;
+		tx_q->tx_skbuff_dma[entry].len = nopaged_len;
+		tx_q->tx_skbuff_dma[entry].is_jumbo = true;
+		desc->des3 = cpu_to_le32(des2 + BUF_SIZE_4KiB);
+		stmmac_prepare_tx_desc(priv, desc, 1, nopaged_len, csum,
+				STMMAC_RING_MODE, 0, !skb_is_nonlinear(skb),
+				skb->len);
+	}
+
+	tx_q->cur_tx = entry;
+
+	return entry;
+}
+
+static unsigned int is_jumbo_frm(int len, int enh_desc)
+{
+	unsigned int ret = 0;
+
+	if (len >= BUF_SIZE_4KiB)
+		ret = 1;
+
+	return ret;
+}
+
+static void refill_desc3(struct stmmac_rx_queue *rx_q, struct dma_desc *p)
+{
+	struct stmmac_priv *priv = rx_q->priv_data;
+
+	/* Fill DES3 in case of RING mode */
+	if (priv->dma_conf.dma_buf_sz == BUF_SIZE_16KiB)
+		p->des3 = cpu_to_le32(le32_to_cpu(p->des2) + BUF_SIZE_8KiB);
+}
+
+/* In ring mode we need to fill the desc3 because it is used as buffer */
+static void init_desc3(struct dma_desc *p)
+{
+	p->des3 = cpu_to_le32(le32_to_cpu(p->des2) + BUF_SIZE_8KiB);
+}
+
+static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p)
+{
+	struct stmmac_priv *priv = tx_q->priv_data;
+	unsigned int entry = tx_q->dirty_tx;
+
+	/* des3 is only used for jumbo frames tx or time stamping */
+	if (unlikely(tx_q->tx_skbuff_dma[entry].is_jumbo ||
+		     (tx_q->tx_skbuff_dma[entry].last_segment &&
+		      !priv->extend_desc && priv->hwts_tx_en)))
+		p->des3 = 0;
+}
+
+static int set_16kib_bfsize(int mtu)
+{
+	int ret = 0;
+	if (unlikely(mtu > BUF_SIZE_8KiB))
+		ret = BUF_SIZE_16KiB;
+	return ret;
+}
+
+const struct stmmac_mode_ops ec_ring_mode_ops = {
+	.is_jumbo_frm = is_jumbo_frm,
+	.jumbo_frm = jumbo_frm,
+	.refill_desc3 = refill_desc3,
+	.init_desc3 = init_desc3,
+	.clean_desc3 = clean_desc3,
+	.set_16kib_bfsize = set_16kib_bfsize,
+};
diff --git a/drivers/net/ethercat/device/K3/stmmac-ethercat.h b/drivers/net/ethercat/device/K3/stmmac-ethercat.h
new file mode 100644
index 000000000000..5a74703051a2
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac-ethercat.h
@@ -0,0 +1,467 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __STMMAC_H__
+#define __STMMAC_H__
+
+#define STMMAC_RESOURCE_NAME   "stmmaceth"
+
+#include <linux/clk.h>
+#include <linux/hrtimer.h>
+#include <linux/if_vlan.h>
+#include <linux/stmmac.h>
+#include <linux/phylink.h>
+#include <linux/pci.h>
+#include "common-ethercat.h"
+#include <linux/ptp_clock_kernel.h>
+#include <linux/net_tstamp.h>
+#include <linux/reset.h>
+#include <net/page_pool/types.h>
+#include <net/xdp.h>
+#include <uapi/linux/bpf.h>
+#include <linux/irq_work.h>
+
+/* EtherCAT header file */
+#include "../ecdev.h"
+
+int __cold stmmac_init(void);
+void __cold stmmac_exit(void);
+
+struct stmmac_resources {
+	void __iomem *addr;
+	u8 mac[ETH_ALEN];
+	int wol_irq;
+	int lpi_irq;
+	int irq;
+	int sfty_irq;
+	int sfty_ce_irq;
+	int sfty_ue_irq;
+	int rx_irq[MTL_MAX_RX_QUEUES];
+	int tx_irq[MTL_MAX_TX_QUEUES];
+};
+
+enum stmmac_txbuf_type {
+	STMMAC_TXBUF_T_SKB,
+	STMMAC_TXBUF_T_XDP_TX,
+	STMMAC_TXBUF_T_XDP_NDO,
+	STMMAC_TXBUF_T_XSK_TX,
+};
+
+struct stmmac_tx_info {
+	dma_addr_t buf;
+	bool map_as_page;
+	unsigned len;
+	bool last_segment;
+	bool is_jumbo;
+	enum stmmac_txbuf_type buf_type;
+	struct xsk_tx_metadata_compl xsk_meta;
+};
+
+#define STMMAC_TBS_AVAIL	BIT(0)
+#define STMMAC_TBS_EN		BIT(1)
+
+/* Frequently used values are kept adjacent for cache effect */
+struct stmmac_tx_queue {
+	u32 tx_count_frames;
+	int tbs;
+	struct hrtimer txtimer;
+	u32 queue_index;
+	struct stmmac_priv *priv_data;
+	struct dma_extended_desc *dma_etx ____cacheline_aligned_in_smp;
+	struct dma_edesc *dma_entx;
+	struct dma_desc *dma_tx;
+	union {
+		struct sk_buff **tx_skbuff;
+		struct xdp_frame **xdpf;
+	};
+	struct stmmac_tx_info *tx_skbuff_dma;
+	struct xsk_buff_pool *xsk_pool;
+	u32 xsk_frames_done;
+	unsigned int cur_tx;
+	unsigned int dirty_tx;
+	dma_addr_t dma_tx_phy;
+	dma_addr_t tx_tail_addr;
+	u32 mss;
+};
+
+struct stmmac_rx_buffer {
+	union {
+		struct {
+			struct page *page;
+			dma_addr_t addr;
+			__u32 page_offset;
+		};
+		struct xdp_buff *xdp;
+	};
+	struct page *sec_page;
+	dma_addr_t sec_addr;
+};
+
+struct stmmac_xdp_buff {
+	struct xdp_buff xdp;
+	struct stmmac_priv *priv;
+	struct dma_desc *desc;
+	struct dma_desc *ndesc;
+};
+
+struct stmmac_metadata_request {
+	struct stmmac_priv *priv;
+	struct dma_desc *tx_desc;
+	bool *set_ic;
+	struct dma_edesc *edesc;
+	int tbs;
+};
+
+struct stmmac_xsk_tx_complete {
+	struct stmmac_priv *priv;
+	struct dma_desc *desc;
+};
+
+struct stmmac_rx_queue {
+	u32 rx_count_frames;
+	u32 queue_index;
+	struct xdp_rxq_info xdp_rxq;
+	struct xsk_buff_pool *xsk_pool;
+	struct page_pool *page_pool;
+	struct stmmac_rx_buffer *buf_pool;
+	struct stmmac_priv *priv_data;
+	struct dma_extended_desc *dma_erx;
+	struct dma_desc *dma_rx ____cacheline_aligned_in_smp;
+	unsigned int cur_rx;
+	unsigned int dirty_rx;
+	unsigned int buf_alloc_num;
+	unsigned int napi_skb_frag_size;
+	dma_addr_t dma_rx_phy;
+	u32 rx_tail_addr;
+	unsigned int state_saved;
+	struct {
+		struct sk_buff *skb;
+		unsigned int len;
+		unsigned int error;
+	} state;
+};
+
+struct stmmac_channel {
+	struct napi_struct rx_napi ____cacheline_aligned_in_smp;
+	struct napi_struct tx_napi ____cacheline_aligned_in_smp;
+	struct napi_struct rxtx_napi ____cacheline_aligned_in_smp;
+	struct stmmac_priv *priv_data;
+	spinlock_t lock;
+	u32 index;
+};
+
+struct stmmac_fpe_cfg {
+	struct ethtool_mmsv mmsv;
+	const struct stmmac_fpe_reg *reg;
+	u32 fpe_csr;	/* MAC_FPE_CTRL_STS reg cache */
+};
+
+struct stmmac_tc_entry {
+	bool in_use;
+	bool in_hw;
+	bool is_last;
+	bool is_frag;
+	void *frag_ptr;
+	unsigned int table_pos;
+	u32 handle;
+	u32 prio;
+	struct {
+		u32 match_data;
+		u32 match_en;
+		u8 af:1;
+		u8 rf:1;
+		u8 im:1;
+		u8 nc:1;
+		u8 res1:4;
+		u8 frame_offset;
+		u8 ok_index;
+		u8 dma_ch_no;
+		u32 res2;
+	} __packed val;
+};
+
+#define STMMAC_PPS_MAX		4
+struct stmmac_pps_cfg {
+	bool available;
+	struct timespec64 start;
+	struct timespec64 period;
+};
+
+struct stmmac_rss {
+	int enable;
+	u8 key[STMMAC_RSS_HASH_KEY_SIZE];
+	u32 table[STMMAC_RSS_MAX_TABLE_SIZE];
+};
+
+#define STMMAC_FLOW_ACTION_DROP		BIT(0)
+struct stmmac_flow_entry {
+	unsigned long cookie;
+	unsigned long action;
+	u8 ip_proto;
+	int in_use;
+	int idx;
+	int is_l4;
+};
+
+/* Rx Frame Steering */
+enum stmmac_rfs_type {
+	STMMAC_RFS_T_VLAN,
+	STMMAC_RFS_T_LLDP,
+	STMMAC_RFS_T_1588,
+	STMMAC_RFS_T_MAX,
+};
+
+struct stmmac_rfs_entry {
+	unsigned long cookie;
+	u16 etype;
+	int in_use;
+	int type;
+	int tc;
+};
+
+struct stmmac_dma_conf {
+	unsigned int dma_buf_sz;
+
+	/* RX Queue */
+	struct stmmac_rx_queue rx_queue[MTL_MAX_RX_QUEUES];
+	unsigned int dma_rx_size;
+
+	/* TX Queue */
+	struct stmmac_tx_queue tx_queue[MTL_MAX_TX_QUEUES];
+	unsigned int dma_tx_size;
+};
+
+#define EST_GCL         1024
+struct stmmac_est {
+	int enable;
+	u32 btr_reserve[2];
+	u32 btr_offset[2];
+	u32 btr[2];
+	u32 ctr[2];
+	u32 ter;
+	u32 gcl_unaligned[EST_GCL];
+	u32 gcl[EST_GCL];
+	u32 gcl_size;
+	u32 max_sdu[MTL_MAX_TX_QUEUES];
+};
+
+struct stmmac_priv {
+	/* Frequently used values are kept adjacent for cache effect */
+	u32 tx_coal_frames[MTL_MAX_TX_QUEUES];
+	u32 tx_coal_timer[MTL_MAX_TX_QUEUES];
+	u32 rx_coal_frames[MTL_MAX_RX_QUEUES];
+
+	int hwts_tx_en;
+	bool tx_path_in_lpi_mode;
+	bool tso;
+	int sph;
+	int sph_cap;
+	u32 sarc_type;
+	u32 rx_riwt[MTL_MAX_RX_QUEUES];
+	int hwts_rx_en;
+
+	void __iomem *ioaddr;
+	struct net_device *dev;
+	struct device *device;
+	struct mac_device_info *hw;
+	int (*hwif_quirks)(struct stmmac_priv *priv);
+	struct mutex lock;
+
+	struct stmmac_dma_conf dma_conf;
+
+	/* Generic channel for NAPI */
+	struct stmmac_channel channel[STMMAC_CH_MAX];
+
+	unsigned int pause_time;
+	struct mii_bus *mii;
+
+	struct phylink_config phylink_config;
+	struct phylink *phylink;
+
+	struct stmmac_extra_stats xstats ____cacheline_aligned_in_smp;
+	struct stmmac_safety_stats sstats;
+	struct plat_stmmacenet_data *plat;
+	/* Protect est parameters */
+	struct mutex est_lock;
+	struct stmmac_est *est;
+	struct dma_features dma_cap;
+	struct stmmac_counters mmc;
+	int hw_cap_support;
+	int synopsys_id;
+	u32 msg_enable;
+	int wolopts;
+	int wol_irq;
+	u32 gmii_address_bus_config;
+	struct timer_list eee_ctrl_timer;
+	int lpi_irq;
+	u32 tx_lpi_timer;
+	bool tx_lpi_clk_stop;
+	bool eee_enabled;
+	bool eee_active;
+	bool eee_sw_timer_en;
+	unsigned int mode;
+	unsigned int chain_mode;
+	int extend_desc;
+	struct kernel_hwtstamp_config tstamp_config;
+	struct ptp_clock *ptp_clock;
+	struct ptp_clock_info ptp_clock_ops;
+	unsigned int default_addend;
+	u32 sub_second_inc;
+	u32 systime_flags;
+	u32 adv_ts;
+	int use_riwt;
+	int irq_wake;
+	rwlock_t ptp_lock;
+	/* Protects auxiliary snapshot registers from concurrent access. */
+	struct mutex aux_ts_lock;
+	wait_queue_head_t tstamp_busy_wait;
+
+	void __iomem *mmcaddr;
+	void __iomem *ptpaddr;
+	void __iomem *estaddr;
+	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
+	int sfty_irq;
+	int sfty_ce_irq;
+	int sfty_ue_irq;
+	int rx_irq[MTL_MAX_RX_QUEUES];
+	int tx_irq[MTL_MAX_TX_QUEUES];
+	/*irq name */
+	char int_name_mac[IFNAMSIZ + 9];
+	char int_name_wol[IFNAMSIZ + 9];
+	char int_name_lpi[IFNAMSIZ + 9];
+	char int_name_sfty[IFNAMSIZ + 10];
+	char int_name_sfty_ce[IFNAMSIZ + 10];
+	char int_name_sfty_ue[IFNAMSIZ + 10];
+	char int_name_rx_irq[MTL_MAX_RX_QUEUES][IFNAMSIZ + 14];
+	char int_name_tx_irq[MTL_MAX_TX_QUEUES][IFNAMSIZ + 18];
+
+#ifdef CONFIG_DEBUG_FS
+	struct dentry *dbgfs_dir;
+#endif
+
+	unsigned long state;
+	struct workqueue_struct *wq;
+	struct work_struct service_task;
+
+	/* Frame Preemption feature (FPE) */
+	struct stmmac_fpe_cfg fpe_cfg;
+
+	/* TC Handling */
+	unsigned int tc_entries_max;
+	unsigned int tc_off_max;
+	struct stmmac_tc_entry *tc_entries;
+	unsigned int flow_entries_max;
+	struct stmmac_flow_entry *flow_entries;
+	unsigned int rfs_entries_max[STMMAC_RFS_T_MAX];
+	unsigned int rfs_entries_cnt[STMMAC_RFS_T_MAX];
+	unsigned int rfs_entries_total;
+	struct stmmac_rfs_entry *rfs_entries;
+
+	/* Pulse Per Second output */
+	struct stmmac_pps_cfg pps[STMMAC_PPS_MAX];
+
+	/* Receive Side Scaling */
+	struct stmmac_rss rss;
+
+	/* XDP BPF Program */
+	unsigned long *af_xdp_zc_qps;
+	struct bpf_prog *xdp_prog;
+
+	/* EtherCAT device variables */
+	ec_device_t *ecdev_;
+	unsigned long ec_watchdog_jiffies;
+	struct irq_work ec_watchdog_kicker;
+	bool ecdev_initialized;
+};
+
+static inline ec_device_t *get_ecdev(struct stmmac_priv *adapter)
+{
+#ifdef EC_ENABLE_DRIVER_RESOURCE_VERIFYING
+	WARN_ON(!adapter->ecdev_initialized);
+#endif
+	return adapter->ecdev_;
+}
+
+enum stmmac_state {
+	STMMAC_DOWN,
+	STMMAC_RESET_REQUESTED,
+	STMMAC_RESETING,
+	STMMAC_SERVICE_SCHED,
+};
+
+extern const struct dev_pm_ops ec_stmmac_simple_pm_ops;
+
+static inline bool stmmac_wol_enabled_mac(struct stmmac_priv *priv)
+{
+	return priv->plat->pmt && device_may_wakeup(priv->device);
+}
+
+static inline bool stmmac_wol_enabled_phy(struct stmmac_priv *priv)
+{
+	return !priv->plat->pmt && device_may_wakeup(priv->device);
+}
+
+int ec_stmmac_mdio_unregister(struct net_device *ndev);
+int ec_stmmac_mdio_register(struct net_device *ndev);
+int ec_stmmac_mdio_reset(struct mii_bus *mii);
+int ec_stmmac_pcs_setup(struct net_device *ndev);
+void ec_stmmac_pcs_clean(struct net_device *ndev);
+void ec_stmmac_set_ethtool_ops(struct net_device *netdev);
+
+void ec_stmmac_ptp_register(struct stmmac_priv *priv);
+void ec_stmmac_ptp_unregister(struct stmmac_priv *priv);
+int ec_stmmac_xdp_open(struct net_device *dev);
+void ec_stmmac_xdp_release(struct net_device *dev);
+int ec_stmmac_resume(struct device *dev);
+int ec_stmmac_suspend(struct device *dev);
+void ec_stmmac_dvr_remove(struct device *dev);
+int ec_stmmac_dvr_probe(struct device *device,
+		     struct plat_stmmacenet_data *plat_dat,
+		     struct stmmac_resources *res);
+int ec_stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt);
+int ec_stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size);
+int ec_stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i,
+			   phy_interface_t interface, int speed);
+
+static inline bool stmmac_xdp_is_enabled(struct stmmac_priv *priv)
+{
+	return !!priv->xdp_prog;
+}
+
+void ec_stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue);
+void ec_stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue);
+void ec_stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue);
+void ec_stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue);
+int ec_stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags);
+struct timespec64 ec_stmmac_calc_tas_basetime(ktime_t old_base_time,
+					   ktime_t current_time,
+					   u64 cycle_time);
+
+#if IS_ENABLED(CONFIG_STMMAC_SELFTESTS)
+void stmmac_selftest_run(struct net_device *dev,
+			 struct ethtool_test *etest, u64 *buf);
+void stmmac_selftest_get_strings(struct stmmac_priv *priv, u8 *data);
+int stmmac_selftest_get_count(struct stmmac_priv *priv);
+#else
+static inline void stmmac_selftest_run(struct net_device *dev,
+				       struct ethtool_test *etest, u64 *buf)
+{
+	/* Not enabled */
+}
+static inline void stmmac_selftest_get_strings(struct stmmac_priv *priv,
+					       u8 *data)
+{
+	/* Not enabled */
+}
+static inline int stmmac_selftest_get_count(struct stmmac_priv *priv)
+{
+	return -EOPNOTSUPP;
+}
+#endif /* CONFIG_STMMAC_SELFTESTS */
+
+#endif /* __STMMAC_H__ */
diff --git a/drivers/net/ethercat/device/K3/stmmac_est-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_est-ethercat.c
new file mode 100644
index 000000000000..f1d2fc66670f
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_est-ethercat.c
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023, Intel Corporation
+ * stmmac EST(802.3 Qbv) handling
+ */
+#include <linux/iopoll.h>
+#include <linux/types.h>
+#include "stmmac-ethercat.h"
+#include "stmmac_est-ethercat.h"
+
+static int est_write(void __iomem *est_addr, u32 reg, u32 val, bool gcl)
+{
+	u32 ctrl;
+
+	writel(val, est_addr + EST_GCL_DATA);
+
+	ctrl = (reg << EST_ADDR_SHIFT);
+	ctrl |= gcl ? 0 : EST_GCRR;
+	writel(ctrl, est_addr + EST_GCL_CONTROL);
+
+	ctrl |= EST_SRWO;
+	writel(ctrl, est_addr + EST_GCL_CONTROL);
+
+	return readl_poll_timeout(est_addr + EST_GCL_CONTROL, ctrl,
+				  !(ctrl & EST_SRWO), 100, 5000);
+}
+
+static int est_configure(struct stmmac_priv *priv, struct stmmac_est *cfg,
+			 unsigned int ptp_rate)
+{
+	void __iomem *est_addr = priv->estaddr;
+	int i, ret = 0;
+	u32 ctrl;
+
+	if (!ptp_rate) {
+		netdev_warn(priv->dev, "Invalid PTP rate");
+		return -EINVAL;
+	}
+
+	ret |= est_write(est_addr, EST_BTR_LOW, cfg->btr[0], false);
+	ret |= est_write(est_addr, EST_BTR_HIGH, cfg->btr[1], false);
+	ret |= est_write(est_addr, EST_TER, cfg->ter, false);
+	ret |= est_write(est_addr, EST_LLR, cfg->gcl_size, false);
+	ret |= est_write(est_addr, EST_CTR_LOW, cfg->ctr[0], false);
+	ret |= est_write(est_addr, EST_CTR_HIGH, cfg->ctr[1], false);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < cfg->gcl_size; i++) {
+		ret = est_write(est_addr, i, cfg->gcl[i], true);
+		if (ret)
+			return ret;
+	}
+
+	ctrl = readl(est_addr + EST_CONTROL);
+	if (priv->plat->has_xgmac) {
+		ctrl &= ~EST_XGMAC_PTOV;
+		ctrl |= ((NSEC_PER_SEC / ptp_rate) * EST_XGMAC_PTOV_MUL) <<
+			 EST_XGMAC_PTOV_SHIFT;
+	} else {
+		ctrl &= ~EST_GMAC5_PTOV;
+		ctrl |= ((NSEC_PER_SEC / ptp_rate) * EST_GMAC5_PTOV_MUL) <<
+			 EST_GMAC5_PTOV_SHIFT;
+	}
+	if (cfg->enable)
+		ctrl |= EST_EEST | EST_SSWL | EST_DFBS;
+	else
+		ctrl &= ~EST_EEST;
+
+	writel(ctrl, est_addr + EST_CONTROL);
+
+	/* Configure EST interrupt */
+	if (cfg->enable)
+		ctrl = EST_IECGCE | EST_IEHS | EST_IEHF | EST_IEBE | EST_IECC;
+	else
+		ctrl = 0;
+
+	writel(ctrl, est_addr + EST_INT_EN);
+
+	return 0;
+}
+
+static void est_irq_status(struct stmmac_priv *priv, struct net_device *dev,
+			   struct stmmac_extra_stats *x, u32 txqcnt)
+{
+	u32 status, value, feqn, hbfq, hbfs, btrl, btrl_max;
+	void __iomem *est_addr = priv->estaddr;
+	u32 txqcnt_mask = BIT(txqcnt) - 1;
+	int i;
+
+	status = readl(est_addr + EST_STATUS);
+
+	value = EST_CGCE | EST_HLBS | EST_HLBF | EST_BTRE | EST_SWLC;
+
+	/* Return if there is no error */
+	if (!(status & value))
+		return;
+
+	if (status & EST_CGCE) {
+		/* Clear Interrupt */
+		writel(EST_CGCE, est_addr + EST_STATUS);
+
+		x->mtl_est_cgce++;
+	}
+
+	if (status & EST_HLBS) {
+		value = readl(est_addr + EST_SCH_ERR);
+		value &= txqcnt_mask;
+
+		x->mtl_est_hlbs++;
+
+		for (i = 0; i < txqcnt; i++)
+			if (value & BIT(i))
+				x->mtl_est_txq_hlbs[i]++;
+
+		/* Clear Interrupt */
+		writel(value, est_addr + EST_SCH_ERR);
+
+		/* Collecting info to shows all the queues that has HLBS
+		 * issue. The only way to clear this is to clear the
+		 * statistic
+		 */
+		if (net_ratelimit())
+			netdev_err(dev, "EST: HLB(sched) Queue 0x%x\n", value);
+	}
+
+	if (status & EST_HLBF) {
+		value = readl(est_addr + EST_FRM_SZ_ERR);
+		feqn = value & txqcnt_mask;
+
+		value = readl(est_addr + EST_FRM_SZ_CAP);
+		hbfq = (value & EST_SZ_CAP_HBFQ_MASK(txqcnt)) >>
+			EST_SZ_CAP_HBFQ_SHIFT;
+		hbfs = value & EST_SZ_CAP_HBFS_MASK;
+
+		x->mtl_est_hlbf++;
+
+		for (i = 0; i < txqcnt; i++)
+			if (feqn & BIT(i))
+				x->mtl_est_txq_hlbf[i]++;
+
+		/* Clear Interrupt */
+		writel(feqn, est_addr + EST_FRM_SZ_ERR);
+
+		if (net_ratelimit())
+			netdev_err(dev, "EST: HLB(size) Queue %u Size %u\n",
+				   hbfq, hbfs);
+	}
+
+	if (status & EST_BTRE) {
+		if (priv->plat->has_xgmac) {
+			btrl = FIELD_GET(EST_XGMAC_BTRL, status);
+			btrl_max = FIELD_MAX(EST_XGMAC_BTRL);
+		} else {
+			btrl = FIELD_GET(EST_GMAC5_BTRL, status);
+			btrl_max = FIELD_MAX(EST_GMAC5_BTRL);
+		}
+		if (btrl == btrl_max)
+			x->mtl_est_btrlm++;
+		else
+			x->mtl_est_btre++;
+
+		if (net_ratelimit())
+			netdev_info(dev, "EST: BTR Error Loop Count %u\n",
+				    btrl);
+
+		writel(EST_BTRE, est_addr + EST_STATUS);
+	}
+
+	if (status & EST_SWLC) {
+		writel(EST_SWLC, est_addr + EST_STATUS);
+		netdev_info(dev, "EST: SWOL has been switched\n");
+	}
+}
+
+const struct stmmac_est_ops ec_dwmac510_est_ops = {
+	.configure = est_configure,
+	.irq_status = est_irq_status,
+};
diff --git a/drivers/net/ethercat/device/K3/stmmac_est-ethercat.h b/drivers/net/ethercat/device/K3/stmmac_est-ethercat.h
new file mode 100644
index 000000000000..fb1bdeaf2cb6
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_est-ethercat.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2023, Intel Corporation
+ * stmmac EST(802.3 Qbv) handling
+ */
+
+#define EST_GMAC4_OFFSET		0x00000c50
+#define EST_XGMAC_OFFSET		0x00001050
+
+#define EST_CONTROL			0x00000000
+#define EST_GMAC5_PTOV			GENMASK(31, 24)
+#define EST_GMAC5_PTOV_SHIFT		24
+#define EST_GMAC5_PTOV_MUL		6
+#define EST_XGMAC_PTOV			GENMASK(31, 23)
+#define EST_XGMAC_PTOV_SHIFT		23
+#define EST_XGMAC_PTOV_MUL		9
+#define EST_SSWL			BIT(1)
+#define EST_EEST			BIT(0)
+#define EST_DFBS			BIT(5)
+
+#define EST_STATUS			0x00000008
+#define EST_GMAC5_BTRL			GENMASK(11, 8)
+#define EST_XGMAC_BTRL			GENMASK(15, 8)
+#define EST_SWOL			BIT(7)
+#define EST_SWOL_SHIFT			7
+#define EST_CGCE			BIT(4)
+#define EST_HLBS			BIT(3)
+#define EST_HLBF			BIT(2)
+#define EST_BTRE			BIT(1)
+#define EST_SWLC			BIT(0)
+
+#define EST_SCH_ERR			0x00000010
+
+#define EST_FRM_SZ_ERR			0x00000014
+
+#define EST_FRM_SZ_CAP			0x00000018
+#define EST_SZ_CAP_HBFS_MASK		GENMASK(14, 0)
+#define EST_SZ_CAP_HBFQ_SHIFT		16
+#define EST_SZ_CAP_HBFQ_MASK(val)		\
+	({					\
+		typeof(val) _val = (val);	\
+		(_val > 4 ? GENMASK(18, 16) :	\
+		 _val > 2 ? GENMASK(17, 16) :	\
+		 BIT(16));			\
+	})
+
+#define EST_INT_EN			0x00000020
+#define EST_IECGCE			EST_CGCE
+#define EST_IEHS			EST_HLBS
+#define EST_IEHF			EST_HLBF
+#define EST_IEBE			EST_BTRE
+#define EST_IECC			EST_SWLC
+
+#define EST_GCL_CONTROL			0x00000030
+#define EST_BTR_LOW			0x0
+#define EST_BTR_HIGH			0x1
+#define EST_CTR_LOW			0x2
+#define EST_CTR_HIGH			0x3
+#define EST_TER				0x4
+#define EST_LLR				0x5
+#define EST_ADDR_SHIFT			8
+#define EST_GCRR			BIT(2)
+#define EST_SRWO			BIT(0)
+
+#define EST_GCL_DATA			0x00000034
+
+extern const struct stmmac_est_ops ec_dwmac510_est_ops;
diff --git a/drivers/net/ethercat/device/K3/stmmac_ethtool-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_ethtool-ethercat.c
new file mode 100644
index 000000000000..d43517291fe7
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_ethtool-ethercat.c
@@ -0,0 +1,1237 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  STMMAC Ethtool support
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mii.h>
+#include <linux/phylink.h>
+#include <linux/net_tstamp.h>
+
+#include "stmmac-ethercat.h"
+#include "stmmac_fpe-ethercat.h"
+#include "dwmac_dma-ethercat.h"
+#include "dwxgmac2-ethercat.h"
+
+#define REG_SPACE_SIZE	0x1060
+#define GMAC4_REG_SPACE_SIZE	0x116C
+#define MAC100_ETHTOOL_NAME	"st_mac100"
+#define GMAC_ETHTOOL_NAME	"st_gmac"
+#define XGMAC_ETHTOOL_NAME	"st_xgmac"
+
+/* Same as DMA_CHAN_BASE_ADDR defined in dwmac4_dma.h
+ *
+ * It is here because dwmac_dma.h and dwmac4_dam.h can not be included at the
+ * same time due to the conflicting macro names.
+ */
+#define GMAC4_DMA_CHAN_BASE_ADDR  0x00001100
+
+#define ETHTOOL_DMA_OFFSET	55
+
+struct stmmac_stats {
+	char stat_string[ETH_GSTRING_LEN] __nonstring;
+	int sizeof_stat;
+	int stat_offset;
+};
+
+#define STMMAC_STAT(m)	\
+	{ #m, sizeof_field(struct stmmac_extra_stats, m),	\
+	offsetof(struct stmmac_priv, xstats.m)}
+
+static const struct stmmac_stats stmmac_gstrings_stats[] = {
+	/* Transmit errors */
+	STMMAC_STAT(tx_underflow),
+	STMMAC_STAT(tx_carrier),
+	STMMAC_STAT(tx_losscarrier),
+	STMMAC_STAT(vlan_tag),
+	STMMAC_STAT(tx_deferred),
+	STMMAC_STAT(tx_vlan),
+	STMMAC_STAT(tx_jabber),
+	STMMAC_STAT(tx_frame_flushed),
+	STMMAC_STAT(tx_payload_error),
+	STMMAC_STAT(tx_ip_header_error),
+	/* Receive errors */
+	STMMAC_STAT(rx_desc),
+	STMMAC_STAT(sa_filter_fail),
+	STMMAC_STAT(overflow_error),
+	STMMAC_STAT(ipc_csum_error),
+	STMMAC_STAT(rx_collision),
+	STMMAC_STAT(rx_crc_errors),
+	STMMAC_STAT(dribbling_bit),
+	STMMAC_STAT(rx_length),
+	STMMAC_STAT(rx_mii),
+	STMMAC_STAT(rx_multicast),
+	STMMAC_STAT(rx_gmac_overflow),
+	STMMAC_STAT(rx_watchdog),
+	STMMAC_STAT(da_rx_filter_fail),
+	STMMAC_STAT(sa_rx_filter_fail),
+	STMMAC_STAT(rx_missed_cntr),
+	STMMAC_STAT(rx_overflow_cntr),
+	STMMAC_STAT(rx_vlan),
+	STMMAC_STAT(rx_split_hdr_pkt_n),
+	/* Tx/Rx IRQ error info */
+	STMMAC_STAT(tx_undeflow_irq),
+	STMMAC_STAT(tx_process_stopped_irq),
+	STMMAC_STAT(tx_jabber_irq),
+	STMMAC_STAT(rx_overflow_irq),
+	STMMAC_STAT(rx_buf_unav_irq),
+	STMMAC_STAT(rx_process_stopped_irq),
+	STMMAC_STAT(rx_watchdog_irq),
+	STMMAC_STAT(tx_early_irq),
+	STMMAC_STAT(fatal_bus_error_irq),
+	/* Tx/Rx IRQ Events */
+	STMMAC_STAT(rx_early_irq),
+	STMMAC_STAT(threshold),
+	STMMAC_STAT(irq_receive_pmt_irq_n),
+	/* MMC info */
+	STMMAC_STAT(mmc_tx_irq_n),
+	STMMAC_STAT(mmc_rx_irq_n),
+	STMMAC_STAT(mmc_rx_csum_offload_irq_n),
+	/* EEE */
+	STMMAC_STAT(irq_tx_path_in_lpi_mode_n),
+	STMMAC_STAT(irq_tx_path_exit_lpi_mode_n),
+	STMMAC_STAT(irq_rx_path_in_lpi_mode_n),
+	STMMAC_STAT(irq_rx_path_exit_lpi_mode_n),
+	STMMAC_STAT(phy_eee_wakeup_error_n),
+	/* Extended RDES status */
+	STMMAC_STAT(ip_hdr_err),
+	STMMAC_STAT(ip_payload_err),
+	STMMAC_STAT(ip_csum_bypassed),
+	STMMAC_STAT(ipv4_pkt_rcvd),
+	STMMAC_STAT(ipv6_pkt_rcvd),
+	STMMAC_STAT(no_ptp_rx_msg_type_ext),
+	STMMAC_STAT(ptp_rx_msg_type_sync),
+	STMMAC_STAT(ptp_rx_msg_type_follow_up),
+	STMMAC_STAT(ptp_rx_msg_type_delay_req),
+	STMMAC_STAT(ptp_rx_msg_type_delay_resp),
+	STMMAC_STAT(ptp_rx_msg_type_pdelay_req),
+	STMMAC_STAT(ptp_rx_msg_type_pdelay_resp),
+	STMMAC_STAT(ptp_rx_msg_type_pdelay_follow_up),
+	STMMAC_STAT(ptp_rx_msg_type_announce),
+	STMMAC_STAT(ptp_rx_msg_type_management),
+	STMMAC_STAT(ptp_rx_msg_pkt_reserved_type),
+	STMMAC_STAT(ptp_frame_type),
+	STMMAC_STAT(ptp_ver),
+	STMMAC_STAT(timestamp_dropped),
+	STMMAC_STAT(av_pkt_rcvd),
+	STMMAC_STAT(av_tagged_pkt_rcvd),
+	STMMAC_STAT(vlan_tag_priority_val),
+	STMMAC_STAT(l3_filter_match),
+	STMMAC_STAT(l4_filter_match),
+	STMMAC_STAT(l3_l4_filter_no_match),
+	/* PCS */
+	STMMAC_STAT(irq_pcs_ane_n),
+	STMMAC_STAT(irq_pcs_link_n),
+	STMMAC_STAT(irq_rgmii_n),
+	/* DEBUG */
+	STMMAC_STAT(mtl_tx_status_fifo_full),
+	STMMAC_STAT(mtl_tx_fifo_not_empty),
+	STMMAC_STAT(mmtl_fifo_ctrl),
+	STMMAC_STAT(mtl_tx_fifo_read_ctrl_write),
+	STMMAC_STAT(mtl_tx_fifo_read_ctrl_wait),
+	STMMAC_STAT(mtl_tx_fifo_read_ctrl_read),
+	STMMAC_STAT(mtl_tx_fifo_read_ctrl_idle),
+	STMMAC_STAT(mac_tx_in_pause),
+	STMMAC_STAT(mac_tx_frame_ctrl_xfer),
+	STMMAC_STAT(mac_tx_frame_ctrl_idle),
+	STMMAC_STAT(mac_tx_frame_ctrl_wait),
+	STMMAC_STAT(mac_tx_frame_ctrl_pause),
+	STMMAC_STAT(mac_gmii_tx_proto_engine),
+	STMMAC_STAT(mtl_rx_fifo_fill_level_full),
+	STMMAC_STAT(mtl_rx_fifo_fill_above_thresh),
+	STMMAC_STAT(mtl_rx_fifo_fill_below_thresh),
+	STMMAC_STAT(mtl_rx_fifo_fill_level_empty),
+	STMMAC_STAT(mtl_rx_fifo_read_ctrl_flush),
+	STMMAC_STAT(mtl_rx_fifo_read_ctrl_read_data),
+	STMMAC_STAT(mtl_rx_fifo_read_ctrl_status),
+	STMMAC_STAT(mtl_rx_fifo_read_ctrl_idle),
+	STMMAC_STAT(mtl_rx_fifo_ctrl_active),
+	STMMAC_STAT(mac_rx_frame_ctrl_fifo),
+	STMMAC_STAT(mac_gmii_rx_proto_engine),
+	/* EST */
+	STMMAC_STAT(mtl_est_cgce),
+	STMMAC_STAT(mtl_est_hlbs),
+	STMMAC_STAT(mtl_est_hlbf),
+	STMMAC_STAT(mtl_est_btre),
+	STMMAC_STAT(mtl_est_btrlm),
+};
+#define STMMAC_STATS_LEN ARRAY_SIZE(stmmac_gstrings_stats)
+
+/* statistics collected in queue which will be summed up for all TX or RX
+ * queues, or summed up for both TX and RX queues(napi_poll, normal_irq_n).
+ */
+static const char stmmac_qstats_string[][ETH_GSTRING_LEN] = {
+	"rx_pkt_n",
+	"rx_normal_irq_n",
+	"tx_pkt_n",
+	"tx_normal_irq_n",
+	"tx_clean",
+	"tx_set_ic_bit",
+	"tx_tso_frames",
+	"tx_tso_nfrags",
+	"normal_irq_n",
+	"napi_poll",
+};
+#define STMMAC_QSTATS ARRAY_SIZE(stmmac_qstats_string)
+
+/* HW MAC Management counters (if supported) */
+#define STMMAC_MMC_STAT(m)	\
+	{ #m, sizeof_field(struct stmmac_counters, m),	\
+	offsetof(struct stmmac_priv, mmc.m)}
+
+static const struct stmmac_stats stmmac_mmc[] = {
+	STMMAC_MMC_STAT(mmc_tx_octetcount_gb),
+	STMMAC_MMC_STAT(mmc_tx_framecount_gb),
+	STMMAC_MMC_STAT(mmc_tx_broadcastframe_g),
+	STMMAC_MMC_STAT(mmc_tx_multicastframe_g),
+	STMMAC_MMC_STAT(mmc_tx_64_octets_gb),
+	STMMAC_MMC_STAT(mmc_tx_65_to_127_octets_gb),
+	STMMAC_MMC_STAT(mmc_tx_128_to_255_octets_gb),
+	STMMAC_MMC_STAT(mmc_tx_256_to_511_octets_gb),
+	STMMAC_MMC_STAT(mmc_tx_512_to_1023_octets_gb),
+	STMMAC_MMC_STAT(mmc_tx_1024_to_max_octets_gb),
+	STMMAC_MMC_STAT(mmc_tx_unicast_gb),
+	STMMAC_MMC_STAT(mmc_tx_multicast_gb),
+	STMMAC_MMC_STAT(mmc_tx_broadcast_gb),
+	STMMAC_MMC_STAT(mmc_tx_underflow_error),
+	STMMAC_MMC_STAT(mmc_tx_singlecol_g),
+	STMMAC_MMC_STAT(mmc_tx_multicol_g),
+	STMMAC_MMC_STAT(mmc_tx_deferred),
+	STMMAC_MMC_STAT(mmc_tx_latecol),
+	STMMAC_MMC_STAT(mmc_tx_exesscol),
+	STMMAC_MMC_STAT(mmc_tx_carrier_error),
+	STMMAC_MMC_STAT(mmc_tx_octetcount_g),
+	STMMAC_MMC_STAT(mmc_tx_framecount_g),
+	STMMAC_MMC_STAT(mmc_tx_excessdef),
+	STMMAC_MMC_STAT(mmc_tx_pause_frame),
+	STMMAC_MMC_STAT(mmc_tx_vlan_frame_g),
+	STMMAC_MMC_STAT(mmc_tx_oversize_g),
+	STMMAC_MMC_STAT(mmc_tx_lpi_usec),
+	STMMAC_MMC_STAT(mmc_tx_lpi_tran),
+	STMMAC_MMC_STAT(mmc_rx_framecount_gb),
+	STMMAC_MMC_STAT(mmc_rx_octetcount_gb),
+	STMMAC_MMC_STAT(mmc_rx_octetcount_g),
+	STMMAC_MMC_STAT(mmc_rx_broadcastframe_g),
+	STMMAC_MMC_STAT(mmc_rx_multicastframe_g),
+	STMMAC_MMC_STAT(mmc_rx_crc_error),
+	STMMAC_MMC_STAT(mmc_rx_align_error),
+	STMMAC_MMC_STAT(mmc_rx_run_error),
+	STMMAC_MMC_STAT(mmc_rx_jabber_error),
+	STMMAC_MMC_STAT(mmc_rx_undersize_g),
+	STMMAC_MMC_STAT(mmc_rx_oversize_g),
+	STMMAC_MMC_STAT(mmc_rx_64_octets_gb),
+	STMMAC_MMC_STAT(mmc_rx_65_to_127_octets_gb),
+	STMMAC_MMC_STAT(mmc_rx_128_to_255_octets_gb),
+	STMMAC_MMC_STAT(mmc_rx_256_to_511_octets_gb),
+	STMMAC_MMC_STAT(mmc_rx_512_to_1023_octets_gb),
+	STMMAC_MMC_STAT(mmc_rx_1024_to_max_octets_gb),
+	STMMAC_MMC_STAT(mmc_rx_unicast_g),
+	STMMAC_MMC_STAT(mmc_rx_length_error),
+	STMMAC_MMC_STAT(mmc_rx_autofrangetype),
+	STMMAC_MMC_STAT(mmc_rx_pause_frames),
+	STMMAC_MMC_STAT(mmc_rx_fifo_overflow),
+	STMMAC_MMC_STAT(mmc_rx_vlan_frames_gb),
+	STMMAC_MMC_STAT(mmc_rx_watchdog_error),
+	STMMAC_MMC_STAT(mmc_rx_error),
+	STMMAC_MMC_STAT(mmc_rx_lpi_usec),
+	STMMAC_MMC_STAT(mmc_rx_lpi_tran),
+	STMMAC_MMC_STAT(mmc_rx_discard_frames_gb),
+	STMMAC_MMC_STAT(mmc_rx_discard_octets_gb),
+	STMMAC_MMC_STAT(mmc_rx_align_err_frames),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_gd),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_hderr),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_nopay),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_frag),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_udsbl),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_gd_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_hderr_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_nopay_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_frag_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv4_udsbl_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv6_gd_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv6_hderr_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv6_nopay_octets),
+	STMMAC_MMC_STAT(mmc_rx_ipv6_gd),
+	STMMAC_MMC_STAT(mmc_rx_ipv6_hderr),
+	STMMAC_MMC_STAT(mmc_rx_ipv6_nopay),
+	STMMAC_MMC_STAT(mmc_rx_udp_gd),
+	STMMAC_MMC_STAT(mmc_rx_udp_err),
+	STMMAC_MMC_STAT(mmc_rx_tcp_gd),
+	STMMAC_MMC_STAT(mmc_rx_tcp_err),
+	STMMAC_MMC_STAT(mmc_rx_icmp_gd),
+	STMMAC_MMC_STAT(mmc_rx_icmp_err),
+	STMMAC_MMC_STAT(mmc_rx_udp_gd_octets),
+	STMMAC_MMC_STAT(mmc_rx_udp_err_octets),
+	STMMAC_MMC_STAT(mmc_rx_tcp_gd_octets),
+	STMMAC_MMC_STAT(mmc_rx_tcp_err_octets),
+	STMMAC_MMC_STAT(mmc_rx_icmp_gd_octets),
+	STMMAC_MMC_STAT(mmc_rx_icmp_err_octets),
+	STMMAC_MMC_STAT(mmc_sgf_pass_fragment_cntr),
+	STMMAC_MMC_STAT(mmc_sgf_fail_fragment_cntr),
+	STMMAC_MMC_STAT(mmc_tx_fpe_fragment_cntr),
+	STMMAC_MMC_STAT(mmc_tx_hold_req_cntr),
+	STMMAC_MMC_STAT(mmc_tx_gate_overrun_cntr),
+	STMMAC_MMC_STAT(mmc_rx_packet_assembly_err_cntr),
+	STMMAC_MMC_STAT(mmc_rx_packet_smd_err_cntr),
+	STMMAC_MMC_STAT(mmc_rx_packet_assembly_ok_cntr),
+	STMMAC_MMC_STAT(mmc_rx_fpe_fragment_cntr),
+};
+#define STMMAC_MMC_STATS_LEN ARRAY_SIZE(stmmac_mmc)
+
+static const char stmmac_qstats_tx_string[][ETH_GSTRING_LEN] = {
+	"tx_pkt_n",
+	"tx_irq_n",
+#define STMMAC_TXQ_STATS ARRAY_SIZE(stmmac_qstats_tx_string)
+};
+
+static const char stmmac_qstats_rx_string[][ETH_GSTRING_LEN] = {
+	"rx_pkt_n",
+	"rx_irq_n",
+#define STMMAC_RXQ_STATS ARRAY_SIZE(stmmac_qstats_rx_string)
+};
+
+static void stmmac_ethtool_getdrvinfo(struct net_device *dev,
+				      struct ethtool_drvinfo *info)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (priv->plat->has_gmac || priv->plat->has_gmac4)
+		strscpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver));
+	else if (priv->plat->has_xgmac)
+		strscpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver));
+	else
+		strscpy(info->driver, MAC100_ETHTOOL_NAME,
+			sizeof(info->driver));
+
+	if (priv->plat->pdev) {
+		strscpy(info->bus_info, pci_name(priv->plat->pdev),
+			sizeof(info->bus_info));
+	}
+}
+
+static int stmmac_ethtool_get_link_ksettings(struct net_device *dev,
+					     struct ethtool_link_ksettings *cmd)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (!(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS) &&
+	    (priv->hw->pcs & STMMAC_PCS_RGMII ||
+	     priv->hw->pcs & STMMAC_PCS_SGMII)) {
+		u32 supported, advertising, lp_advertising;
+
+		if (!priv->xstats.pcs_link) {
+			cmd->base.speed = SPEED_UNKNOWN;
+			cmd->base.duplex = DUPLEX_UNKNOWN;
+			return 0;
+		}
+		cmd->base.duplex = priv->xstats.pcs_duplex;
+
+		cmd->base.speed = priv->xstats.pcs_speed;
+
+		/* Encoding of PSE bits is defined in 802.3z, 37.2.1.4 */
+
+		ethtool_convert_link_mode_to_legacy_u32(
+			&supported, cmd->link_modes.supported);
+		ethtool_convert_link_mode_to_legacy_u32(
+			&advertising, cmd->link_modes.advertising);
+		ethtool_convert_link_mode_to_legacy_u32(
+			&lp_advertising, cmd->link_modes.lp_advertising);
+
+		/* Reg49[3] always set because ANE is always supported */
+		cmd->base.autoneg = ADVERTISED_Autoneg;
+		supported |= SUPPORTED_Autoneg;
+		advertising |= ADVERTISED_Autoneg;
+		lp_advertising |= ADVERTISED_Autoneg;
+
+		cmd->base.port = PORT_OTHER;
+
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.supported, supported);
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.advertising, advertising);
+		ethtool_convert_legacy_u32_to_link_mode(
+			cmd->link_modes.lp_advertising, lp_advertising);
+
+		return 0;
+	}
+
+	return phylink_ethtool_ksettings_get(priv->phylink, cmd);
+}
+
+static int
+stmmac_ethtool_set_link_ksettings(struct net_device *dev,
+				  const struct ethtool_link_ksettings *cmd)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (!(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS) &&
+	    (priv->hw->pcs & STMMAC_PCS_RGMII ||
+	     priv->hw->pcs & STMMAC_PCS_SGMII)) {
+		/* Only support ANE */
+		if (cmd->base.autoneg != AUTONEG_ENABLE)
+			return -EINVAL;
+
+		mutex_lock(&priv->lock);
+		stmmac_pcs_ctrl_ane(priv, 1, priv->hw->ps, 0);
+		mutex_unlock(&priv->lock);
+
+		return 0;
+	}
+
+	return phylink_ethtool_ksettings_set(priv->phylink, cmd);
+}
+
+static u32 stmmac_ethtool_getmsglevel(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	return priv->msg_enable;
+}
+
+static void stmmac_ethtool_setmsglevel(struct net_device *dev, u32 level)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	priv->msg_enable = level;
+
+}
+
+static int stmmac_ethtool_get_regs_len(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (priv->plat->has_xgmac)
+		return XGMAC_REGSIZE * 4;
+	else if (priv->plat->has_gmac4)
+		return GMAC4_REG_SPACE_SIZE;
+	return REG_SPACE_SIZE;
+}
+
+static void stmmac_ethtool_gregs(struct net_device *dev,
+			  struct ethtool_regs *regs, void *space)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 *reg_space = (u32 *) space;
+
+	stmmac_dump_mac_regs(priv, priv->hw, reg_space);
+	stmmac_dump_dma_regs(priv, priv->ioaddr, reg_space);
+
+	/* Copy DMA registers to where ethtool expects them */
+	if (priv->plat->has_gmac4) {
+		/* GMAC4 dumps its DMA registers at its DMA_CHAN_BASE_ADDR */
+		memcpy(&reg_space[ETHTOOL_DMA_OFFSET],
+		       &reg_space[GMAC4_DMA_CHAN_BASE_ADDR / 4],
+		       NUM_DWMAC4_DMA_REGS * 4);
+	} else if (!priv->plat->has_xgmac) {
+		memcpy(&reg_space[ETHTOOL_DMA_OFFSET],
+		       &reg_space[DMA_BUS_MODE / 4],
+		       NUM_DWMAC1000_DMA_REGS * 4);
+	}
+}
+
+static int stmmac_nway_reset(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	return phylink_ethtool_nway_reset(priv->phylink);
+}
+
+static void stmmac_get_ringparam(struct net_device *netdev,
+				 struct ethtool_ringparam *ring,
+				 struct kernel_ethtool_ringparam *kernel_ring,
+				 struct netlink_ext_ack *extack)
+{
+	struct stmmac_priv *priv = netdev_priv(netdev);
+
+	ring->rx_max_pending = DMA_MAX_RX_SIZE;
+	ring->tx_max_pending = DMA_MAX_TX_SIZE;
+	ring->rx_pending = priv->dma_conf.dma_rx_size;
+	ring->tx_pending = priv->dma_conf.dma_tx_size;
+}
+
+static int stmmac_set_ringparam(struct net_device *netdev,
+				struct ethtool_ringparam *ring,
+				struct kernel_ethtool_ringparam *kernel_ring,
+				struct netlink_ext_ack *extack)
+{
+	if (ring->rx_mini_pending || ring->rx_jumbo_pending ||
+	    ring->rx_pending < DMA_MIN_RX_SIZE ||
+	    ring->rx_pending > DMA_MAX_RX_SIZE ||
+	    !is_power_of_2(ring->rx_pending) ||
+	    ring->tx_pending < DMA_MIN_TX_SIZE ||
+	    ring->tx_pending > DMA_MAX_TX_SIZE ||
+	    !is_power_of_2(ring->tx_pending))
+		return -EINVAL;
+
+	return ec_stmmac_reinit_ringparam(netdev, ring->rx_pending,
+				       ring->tx_pending);
+}
+
+static void
+stmmac_get_pauseparam(struct net_device *netdev,
+		      struct ethtool_pauseparam *pause)
+{
+	struct stmmac_priv *priv = netdev_priv(netdev);
+
+	if (priv->hw->pcs) {
+		pause->autoneg = 1;
+	} else {
+		phylink_ethtool_get_pauseparam(priv->phylink, pause);
+	}
+}
+
+static int
+stmmac_set_pauseparam(struct net_device *netdev,
+		      struct ethtool_pauseparam *pause)
+{
+	struct stmmac_priv *priv = netdev_priv(netdev);
+
+	if (priv->hw->pcs) {
+		pause->autoneg = 1;
+		return 0;
+	} else {
+		return phylink_ethtool_set_pauseparam(priv->phylink, pause);
+	}
+}
+
+static u64 stmmac_get_rx_normal_irq_n(struct stmmac_priv *priv, int q)
+{
+	u64 total;
+	int cpu;
+
+	total = 0;
+	for_each_possible_cpu(cpu) {
+		struct stmmac_pcpu_stats *pcpu;
+		unsigned int start;
+		u64 irq_n;
+
+		pcpu = per_cpu_ptr(priv->xstats.pcpu_stats, cpu);
+		do {
+			start = u64_stats_fetch_begin(&pcpu->syncp);
+			irq_n = u64_stats_read(&pcpu->rx_normal_irq_n[q]);
+		} while (u64_stats_fetch_retry(&pcpu->syncp, start));
+		total += irq_n;
+	}
+	return total;
+}
+
+static u64 stmmac_get_tx_normal_irq_n(struct stmmac_priv *priv, int q)
+{
+	u64 total;
+	int cpu;
+
+	total = 0;
+	for_each_possible_cpu(cpu) {
+		struct stmmac_pcpu_stats *pcpu;
+		unsigned int start;
+		u64 irq_n;
+
+		pcpu = per_cpu_ptr(priv->xstats.pcpu_stats, cpu);
+		do {
+			start = u64_stats_fetch_begin(&pcpu->syncp);
+			irq_n = u64_stats_read(&pcpu->tx_normal_irq_n[q]);
+		} while (u64_stats_fetch_retry(&pcpu->syncp, start));
+		total += irq_n;
+	}
+	return total;
+}
+
+static void stmmac_get_per_qstats(struct stmmac_priv *priv, u64 *data)
+{
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	unsigned int start;
+	int q;
+
+	for (q = 0; q < tx_cnt; q++) {
+		struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q];
+		u64 pkt_n;
+
+		do {
+			start = u64_stats_fetch_begin(&txq_stats->napi_syncp);
+			pkt_n = u64_stats_read(&txq_stats->napi.tx_pkt_n);
+		} while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start));
+
+		*data++ = pkt_n;
+		*data++ = stmmac_get_tx_normal_irq_n(priv, q);
+	}
+
+	for (q = 0; q < rx_cnt; q++) {
+		struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q];
+		u64 pkt_n;
+
+		do {
+			start = u64_stats_fetch_begin(&rxq_stats->napi_syncp);
+			pkt_n = u64_stats_read(&rxq_stats->napi.rx_pkt_n);
+		} while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start));
+
+		*data++ = pkt_n;
+		*data++ = stmmac_get_rx_normal_irq_n(priv, q);
+	}
+}
+
+static void stmmac_get_ethtool_stats(struct net_device *dev,
+				 struct ethtool_stats *dummy, u64 *data)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 rx_queues_count = priv->plat->rx_queues_to_use;
+	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	u64 napi_poll = 0, normal_irq_n = 0;
+	int i, j = 0, pos, ret;
+	unsigned long count;
+	unsigned int start;
+
+	if (priv->dma_cap.asp) {
+		for (i = 0; i < STMMAC_SAFETY_FEAT_SIZE; i++) {
+			if (!stmmac_safety_feat_dump(priv, &priv->sstats, i,
+						&count, NULL))
+				data[j++] = count;
+		}
+	}
+
+	/* Update the DMA HW counters for dwmac10/100 */
+	ret = stmmac_dma_diagnostic_fr(priv, &priv->xstats, priv->ioaddr);
+	if (ret) {
+		/* If supported, for new GMAC chips expose the MMC counters */
+		if (priv->dma_cap.rmon) {
+			stmmac_mmc_read(priv, priv->mmcaddr, &priv->mmc);
+
+			for (i = 0; i < STMMAC_MMC_STATS_LEN; i++) {
+				char *p;
+				p = (char *)priv + stmmac_mmc[i].stat_offset;
+
+				data[j++] = (stmmac_mmc[i].sizeof_stat ==
+					     sizeof(u64)) ? (*(u64 *)p) :
+					     (*(u32 *)p);
+			}
+		}
+		if (priv->dma_cap.eee) {
+			int val = phylink_get_eee_err(priv->phylink);
+			if (val)
+				priv->xstats.phy_eee_wakeup_error_n = val;
+		}
+
+		if (priv->synopsys_id >= DWMAC_CORE_3_50)
+			stmmac_mac_debug(priv, priv->ioaddr,
+					(void *)&priv->xstats,
+					rx_queues_count, tx_queues_count);
+	}
+	for (i = 0; i < STMMAC_STATS_LEN; i++) {
+		char *p = (char *)priv + stmmac_gstrings_stats[i].stat_offset;
+		data[j++] = (stmmac_gstrings_stats[i].sizeof_stat ==
+			     sizeof(u64)) ? (*(u64 *)p) : (*(u32 *)p);
+	}
+
+	pos = j;
+	for (i = 0; i < rx_queues_count; i++) {
+		struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[i];
+		struct stmmac_napi_rx_stats snapshot;
+		u64 n_irq;
+
+		j = pos;
+		do {
+			start = u64_stats_fetch_begin(&rxq_stats->napi_syncp);
+			snapshot = rxq_stats->napi;
+		} while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start));
+
+		data[j++] += u64_stats_read(&snapshot.rx_pkt_n);
+		n_irq = stmmac_get_rx_normal_irq_n(priv, i);
+		data[j++] += n_irq;
+		normal_irq_n += n_irq;
+		napi_poll += u64_stats_read(&snapshot.poll);
+	}
+
+	pos = j;
+	for (i = 0; i < tx_queues_count; i++) {
+		struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[i];
+		struct stmmac_napi_tx_stats napi_snapshot;
+		struct stmmac_q_tx_stats q_snapshot;
+		u64 n_irq;
+
+		j = pos;
+		do {
+			start = u64_stats_fetch_begin(&txq_stats->q_syncp);
+			q_snapshot = txq_stats->q;
+		} while (u64_stats_fetch_retry(&txq_stats->q_syncp, start));
+		do {
+			start = u64_stats_fetch_begin(&txq_stats->napi_syncp);
+			napi_snapshot = txq_stats->napi;
+		} while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start));
+
+		data[j++] += u64_stats_read(&napi_snapshot.tx_pkt_n);
+		n_irq = stmmac_get_tx_normal_irq_n(priv, i);
+		data[j++] += n_irq;
+		normal_irq_n += n_irq;
+		data[j++] += u64_stats_read(&napi_snapshot.tx_clean);
+		data[j++] += u64_stats_read(&q_snapshot.tx_set_ic_bit) +
+			u64_stats_read(&napi_snapshot.tx_set_ic_bit);
+		data[j++] += u64_stats_read(&q_snapshot.tx_tso_frames);
+		data[j++] += u64_stats_read(&q_snapshot.tx_tso_nfrags);
+		napi_poll += u64_stats_read(&napi_snapshot.poll);
+	}
+	normal_irq_n += priv->xstats.rx_early_irq;
+	data[j++] = normal_irq_n;
+	data[j++] = napi_poll;
+
+	stmmac_get_per_qstats(priv, &data[j]);
+}
+
+static int stmmac_get_sset_count(struct net_device *netdev, int sset)
+{
+	struct stmmac_priv *priv = netdev_priv(netdev);
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	int i, len, safety_len = 0;
+
+	switch (sset) {
+	case ETH_SS_STATS:
+		len = STMMAC_STATS_LEN + STMMAC_QSTATS +
+		      STMMAC_TXQ_STATS * tx_cnt +
+		      STMMAC_RXQ_STATS * rx_cnt;
+
+		if (priv->dma_cap.rmon)
+			len += STMMAC_MMC_STATS_LEN;
+		if (priv->dma_cap.asp) {
+			for (i = 0; i < STMMAC_SAFETY_FEAT_SIZE; i++) {
+				if (!stmmac_safety_feat_dump(priv,
+							&priv->sstats, i,
+							NULL, NULL))
+					safety_len++;
+			}
+
+			len += safety_len;
+		}
+
+		return len;
+	case ETH_SS_TEST:
+		/* return stmmac_selftest_get_count(priv); */
+		return -EOPNOTSUPP;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void stmmac_get_qstats_string(struct stmmac_priv *priv, u8 *data)
+{
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	int q, stat;
+
+	for (q = 0; q < tx_cnt; q++) {
+		for (stat = 0; stat < STMMAC_TXQ_STATS; stat++) {
+			snprintf(data, ETH_GSTRING_LEN, "q%d_%s", q,
+				 stmmac_qstats_tx_string[stat]);
+			data += ETH_GSTRING_LEN;
+		}
+	}
+	for (q = 0; q < rx_cnt; q++) {
+		for (stat = 0; stat < STMMAC_RXQ_STATS; stat++) {
+			snprintf(data, ETH_GSTRING_LEN, "q%d_%s", q,
+				 stmmac_qstats_rx_string[stat]);
+			data += ETH_GSTRING_LEN;
+		}
+	}
+}
+
+static void stmmac_get_strings(struct net_device *dev, u32 stringset, u8 *data)
+{
+	int i;
+	u8 *p = data;
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		if (priv->dma_cap.asp) {
+			for (i = 0; i < STMMAC_SAFETY_FEAT_SIZE; i++) {
+				const char *desc;
+				if (!stmmac_safety_feat_dump(priv,
+							&priv->sstats, i,
+							NULL, &desc)) {
+					memcpy(p, desc, ETH_GSTRING_LEN);
+					p += ETH_GSTRING_LEN;
+				}
+			}
+		}
+		if (priv->dma_cap.rmon)
+			for (i = 0; i < STMMAC_MMC_STATS_LEN; i++) {
+				memcpy(p, stmmac_mmc[i].stat_string,
+				       ETH_GSTRING_LEN);
+				p += ETH_GSTRING_LEN;
+			}
+		for (i = 0; i < STMMAC_STATS_LEN; i++) {
+			memcpy(p, stmmac_gstrings_stats[i].stat_string, ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+		for (i = 0; i < STMMAC_QSTATS; i++) {
+			memcpy(p, stmmac_qstats_string[i], ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+		stmmac_get_qstats_string(priv, p);
+		break;
+	case ETH_SS_TEST:
+		/* stmmac_selftest_get_strings(priv, p); */
+		break;
+	default:
+		WARN_ON(1);
+		break;
+	}
+}
+
+/* Currently only support WOL through Magic packet. */
+static void stmmac_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (!priv->plat->pmt)
+		return phylink_ethtool_get_wol(priv->phylink, wol);
+
+	mutex_lock(&priv->lock);
+	if (device_can_wakeup(priv->device)) {
+		wol->supported = WAKE_MAGIC | WAKE_UCAST;
+		if (priv->hw_cap_support && !priv->dma_cap.pmt_magic_frame)
+			wol->supported &= ~WAKE_MAGIC;
+		wol->wolopts = priv->wolopts;
+	}
+	mutex_unlock(&priv->lock);
+}
+
+static int stmmac_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (!device_can_wakeup(priv->device))
+		return -EOPNOTSUPP;
+
+	if (!priv->plat->pmt) {
+		int ret = phylink_ethtool_set_wol(priv->phylink, wol);
+
+		if (!ret)
+			device_set_wakeup_enable(priv->device, !!wol->wolopts);
+		return ret;
+	}
+
+	device_set_wakeup_enable(priv->device, !!wol->wolopts);
+
+	mutex_lock(&priv->lock);
+	priv->wolopts = wol->wolopts;
+	mutex_unlock(&priv->lock);
+
+	return 0;
+}
+
+static int stmmac_ethtool_op_get_eee(struct net_device *dev,
+				     struct ethtool_keee *edata)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	return phylink_ethtool_get_eee(priv->phylink, edata);
+}
+
+static int stmmac_ethtool_op_set_eee(struct net_device *dev,
+				     struct ethtool_keee *edata)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	return phylink_ethtool_set_eee(priv->phylink, edata);
+}
+
+static u32 stmmac_usec2riwt(u32 usec, struct stmmac_priv *priv)
+{
+	unsigned long clk = clk_get_rate(priv->plat->stmmac_clk);
+
+	if (!clk) {
+		clk = priv->plat->clk_ref_rate;
+		if (!clk)
+			return 0;
+	}
+
+	return (usec * (clk / 1000000)) / 256;
+}
+
+static u32 stmmac_riwt2usec(u32 riwt, struct stmmac_priv *priv)
+{
+	unsigned long clk = clk_get_rate(priv->plat->stmmac_clk);
+
+	if (!clk) {
+		clk = priv->plat->clk_ref_rate;
+		if (!clk)
+			return 0;
+	}
+
+	return (riwt * 256) / (clk / 1000000);
+}
+
+static int __stmmac_get_coalesce(struct net_device *dev,
+				 struct ethtool_coalesce *ec,
+				 int queue)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 max_cnt;
+	u32 rx_cnt;
+	u32 tx_cnt;
+
+	rx_cnt = priv->plat->rx_queues_to_use;
+	tx_cnt = priv->plat->tx_queues_to_use;
+	max_cnt = max(rx_cnt, tx_cnt);
+
+	if (queue < 0)
+		queue = 0;
+	else if (queue >= max_cnt)
+		return -EINVAL;
+
+	if (queue < tx_cnt) {
+		ec->tx_coalesce_usecs = priv->tx_coal_timer[queue];
+		ec->tx_max_coalesced_frames = priv->tx_coal_frames[queue];
+	} else {
+		ec->tx_coalesce_usecs = 0;
+		ec->tx_max_coalesced_frames = 0;
+	}
+
+	if (priv->use_riwt && queue < rx_cnt) {
+		ec->rx_max_coalesced_frames = priv->rx_coal_frames[queue];
+		ec->rx_coalesce_usecs = stmmac_riwt2usec(priv->rx_riwt[queue],
+							 priv);
+	} else {
+		ec->rx_max_coalesced_frames = 0;
+		ec->rx_coalesce_usecs = 0;
+	}
+
+	return 0;
+}
+
+static int stmmac_get_coalesce(struct net_device *dev,
+			       struct ethtool_coalesce *ec,
+			       struct kernel_ethtool_coalesce *kernel_coal,
+			       struct netlink_ext_ack *extack)
+{
+	return __stmmac_get_coalesce(dev, ec, -1);
+}
+
+static int stmmac_get_per_queue_coalesce(struct net_device *dev, u32 queue,
+					 struct ethtool_coalesce *ec)
+{
+	return __stmmac_get_coalesce(dev, ec, queue);
+}
+
+static int __stmmac_set_coalesce(struct net_device *dev,
+				 struct ethtool_coalesce *ec,
+				 int queue)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	bool all_queues = false;
+	unsigned int rx_riwt;
+	u32 max_cnt;
+	u32 rx_cnt;
+	u32 tx_cnt;
+
+	rx_cnt = priv->plat->rx_queues_to_use;
+	tx_cnt = priv->plat->tx_queues_to_use;
+	max_cnt = max(rx_cnt, tx_cnt);
+
+	if (queue < 0)
+		all_queues = true;
+	else if (queue >= max_cnt)
+		return -EINVAL;
+
+	if (priv->use_riwt) {
+		rx_riwt = stmmac_usec2riwt(ec->rx_coalesce_usecs, priv);
+
+		if ((rx_riwt > MAX_DMA_RIWT) || (rx_riwt < MIN_DMA_RIWT))
+			return -EINVAL;
+
+		if (all_queues) {
+			int i;
+
+			for (i = 0; i < rx_cnt; i++) {
+				priv->rx_riwt[i] = rx_riwt;
+				stmmac_rx_watchdog(priv, priv->ioaddr,
+						   rx_riwt, i);
+				priv->rx_coal_frames[i] =
+					ec->rx_max_coalesced_frames;
+			}
+		} else if (queue < rx_cnt) {
+			priv->rx_riwt[queue] = rx_riwt;
+			stmmac_rx_watchdog(priv, priv->ioaddr,
+					   rx_riwt, queue);
+			priv->rx_coal_frames[queue] =
+				ec->rx_max_coalesced_frames;
+		}
+	}
+
+	if ((ec->tx_coalesce_usecs == 0) &&
+	    (ec->tx_max_coalesced_frames == 0))
+		return -EINVAL;
+
+	if ((ec->tx_coalesce_usecs > STMMAC_MAX_COAL_TX_TICK) ||
+	    (ec->tx_max_coalesced_frames > STMMAC_TX_MAX_FRAMES))
+		return -EINVAL;
+
+	if (all_queues) {
+		int i;
+
+		for (i = 0; i < tx_cnt; i++) {
+			priv->tx_coal_frames[i] =
+				ec->tx_max_coalesced_frames;
+			priv->tx_coal_timer[i] =
+				ec->tx_coalesce_usecs;
+		}
+	} else if (queue < tx_cnt) {
+		priv->tx_coal_frames[queue] =
+			ec->tx_max_coalesced_frames;
+		priv->tx_coal_timer[queue] =
+			ec->tx_coalesce_usecs;
+	}
+
+	return 0;
+}
+
+static int stmmac_set_coalesce(struct net_device *dev,
+			       struct ethtool_coalesce *ec,
+			       struct kernel_ethtool_coalesce *kernel_coal,
+			       struct netlink_ext_ack *extack)
+{
+	return __stmmac_set_coalesce(dev, ec, -1);
+}
+
+static int stmmac_set_per_queue_coalesce(struct net_device *dev, u32 queue,
+					 struct ethtool_coalesce *ec)
+{
+	return __stmmac_set_coalesce(dev, ec, queue);
+}
+
+static int stmmac_get_rxnfc(struct net_device *dev,
+			    struct ethtool_rxnfc *rxnfc, u32 *rule_locs)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	switch (rxnfc->cmd) {
+	case ETHTOOL_GRXRINGS:
+		rxnfc->data = priv->plat->rx_queues_to_use;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static u32 stmmac_get_rxfh_key_size(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	return sizeof(priv->rss.key);
+}
+
+static u32 stmmac_get_rxfh_indir_size(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	return ARRAY_SIZE(priv->rss.table);
+}
+
+static int stmmac_get_rxfh(struct net_device *dev,
+			   struct ethtool_rxfh_param *rxfh)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int i;
+
+	if (rxfh->indir) {
+		for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
+			rxfh->indir[i] = priv->rss.table[i];
+	}
+
+	if (rxfh->key)
+		memcpy(rxfh->key, priv->rss.key, sizeof(priv->rss.key));
+	rxfh->hfunc = ETH_RSS_HASH_TOP;
+
+	return 0;
+}
+
+static int stmmac_set_rxfh(struct net_device *dev,
+			   struct ethtool_rxfh_param *rxfh,
+			   struct netlink_ext_ack *extack)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int i;
+
+	if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
+	    rxfh->hfunc != ETH_RSS_HASH_TOP)
+		return -EOPNOTSUPP;
+
+	if (rxfh->indir) {
+		for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
+			priv->rss.table[i] = rxfh->indir[i];
+	}
+
+	if (rxfh->key)
+		memcpy(priv->rss.key, rxfh->key, sizeof(priv->rss.key));
+
+	return stmmac_rss_configure(priv, priv->hw, &priv->rss,
+				    priv->plat->rx_queues_to_use);
+}
+
+static void stmmac_get_channels(struct net_device *dev,
+				struct ethtool_channels *chan)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	chan->rx_count = priv->plat->rx_queues_to_use;
+	chan->tx_count = priv->plat->tx_queues_to_use;
+	chan->max_rx = priv->dma_cap.number_rx_queues;
+	chan->max_tx = priv->dma_cap.number_tx_queues;
+}
+
+static int stmmac_set_channels(struct net_device *dev,
+			       struct ethtool_channels *chan)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (chan->rx_count > priv->dma_cap.number_rx_queues ||
+	    chan->tx_count > priv->dma_cap.number_tx_queues ||
+	    !chan->rx_count || !chan->tx_count)
+		return -EINVAL;
+
+	return ec_stmmac_reinit_queues(dev, chan->rx_count, chan->tx_count);
+}
+
+static int stmmac_get_ts_info(struct net_device *dev,
+			      struct kernel_ethtool_ts_info *info)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if ((priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) {
+
+		info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
+					SOF_TIMESTAMPING_TX_HARDWARE |
+					SOF_TIMESTAMPING_RX_HARDWARE |
+					SOF_TIMESTAMPING_RAW_HARDWARE;
+
+		if (priv->ptp_clock)
+			info->phc_index = ptp_clock_index(priv->ptp_clock);
+		else
+			info->phc_index = 0;
+
+		info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
+
+		info->rx_filters = ((1 << HWTSTAMP_FILTER_NONE) |
+				    (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
+				    (1 << HWTSTAMP_FILTER_PTP_V1_L4_SYNC) |
+				    (1 << HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ) |
+				    (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT) |
+				    (1 << HWTSTAMP_FILTER_PTP_V2_L4_SYNC) |
+				    (1 << HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ) |
+				    (1 << HWTSTAMP_FILTER_PTP_V2_EVENT) |
+				    (1 << HWTSTAMP_FILTER_PTP_V2_SYNC) |
+				    (1 << HWTSTAMP_FILTER_PTP_V2_DELAY_REQ) |
+				    (1 << HWTSTAMP_FILTER_ALL));
+		return 0;
+	} else
+		return ethtool_op_get_ts_info(dev, info);
+}
+
+static int stmmac_get_mm(struct net_device *ndev,
+			 struct ethtool_mm_state *state)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	u32 frag_size;
+
+	if (!ec_stmmac_fpe_supported(priv))
+		return -EOPNOTSUPP;
+
+	state->rx_min_frag_size = ETH_ZLEN;
+	frag_size = ec_stmmac_fpe_get_add_frag_size(priv);
+	state->tx_min_frag_size = ethtool_mm_frag_size_add_to_min(frag_size);
+
+	ethtool_mmsv_get_mm(&priv->fpe_cfg.mmsv, state);
+
+	return 0;
+}
+
+static int stmmac_set_mm(struct net_device *ndev, struct ethtool_mm_cfg *cfg,
+			 struct netlink_ext_ack *extack)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	u32 frag_size;
+	int err;
+
+	err = ethtool_mm_frag_size_min_to_add(cfg->tx_min_frag_size,
+					      &frag_size, extack);
+	if (err)
+		return err;
+
+	ec_stmmac_fpe_set_add_frag_size(priv, frag_size);
+	ethtool_mmsv_set_mm(&priv->fpe_cfg.mmsv, cfg);
+
+	return 0;
+}
+
+static void stmmac_get_mm_stats(struct net_device *ndev,
+				struct ethtool_mm_stats *s)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	struct stmmac_counters *mmc = &priv->mmc;
+
+	if (!priv->dma_cap.rmon)
+		return;
+
+	stmmac_mmc_read(priv, priv->mmcaddr, mmc);
+
+	s->MACMergeFrameAssErrorCount = mmc->mmc_rx_packet_assembly_err_cntr;
+	s->MACMergeFrameAssOkCount = mmc->mmc_rx_packet_assembly_ok_cntr;
+	s->MACMergeFrameSmdErrorCount = mmc->mmc_rx_packet_smd_err_cntr;
+	s->MACMergeFragCountRx = mmc->mmc_rx_fpe_fragment_cntr;
+	s->MACMergeFragCountTx = mmc->mmc_tx_fpe_fragment_cntr;
+	s->MACMergeHoldCount = mmc->mmc_tx_hold_req_cntr;
+}
+
+static const struct ethtool_ops stmmac_ethtool_ops = {
+	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
+				     ETHTOOL_COALESCE_MAX_FRAMES,
+	.get_drvinfo = stmmac_ethtool_getdrvinfo,
+	.get_msglevel = stmmac_ethtool_getmsglevel,
+	.set_msglevel = stmmac_ethtool_setmsglevel,
+	.get_regs = stmmac_ethtool_gregs,
+	.get_regs_len = stmmac_ethtool_get_regs_len,
+	.get_link = ethtool_op_get_link,
+	.nway_reset = stmmac_nway_reset,
+	.get_ringparam = stmmac_get_ringparam,
+	.set_ringparam = stmmac_set_ringparam,
+	.get_pauseparam = stmmac_get_pauseparam,
+	.set_pauseparam = stmmac_set_pauseparam,
+	/* .self_test = stmmac_selftest_run, */
+	.get_ethtool_stats = stmmac_get_ethtool_stats,
+	.get_strings = stmmac_get_strings,
+	.get_wol = stmmac_get_wol,
+	.set_wol = stmmac_set_wol,
+	.get_eee = stmmac_ethtool_op_get_eee,
+	.set_eee = stmmac_ethtool_op_set_eee,
+	.get_sset_count	= stmmac_get_sset_count,
+	.get_rxnfc = stmmac_get_rxnfc,
+	.get_rxfh_key_size = stmmac_get_rxfh_key_size,
+	.get_rxfh_indir_size = stmmac_get_rxfh_indir_size,
+	.get_rxfh = stmmac_get_rxfh,
+	.set_rxfh = stmmac_set_rxfh,
+	.get_ts_info = stmmac_get_ts_info,
+	.get_coalesce = stmmac_get_coalesce,
+	.set_coalesce = stmmac_set_coalesce,
+	.get_per_queue_coalesce = stmmac_get_per_queue_coalesce,
+	.set_per_queue_coalesce = stmmac_set_per_queue_coalesce,
+	.get_channels = stmmac_get_channels,
+	.set_channels = stmmac_set_channels,
+	.get_link_ksettings = stmmac_ethtool_get_link_ksettings,
+	.set_link_ksettings = stmmac_ethtool_set_link_ksettings,
+	.get_mm = stmmac_get_mm,
+	.set_mm = stmmac_set_mm,
+	.get_mm_stats = stmmac_get_mm_stats,
+};
+
+void ec_stmmac_set_ethtool_ops(struct net_device *netdev)
+{
+	netdev->ethtool_ops = &stmmac_ethtool_ops;
+}
diff --git a/drivers/net/ethercat/device/K3/stmmac_fpe-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_fpe-ethercat.c
new file mode 100644
index 000000000000..49727e52e021
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_fpe-ethercat.c
@@ -0,0 +1,311 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 Furong Xu <0x1207@gmail.com>
+ * stmmac FPE(802.3 Qbu) handling
+ */
+#include "stmmac-ethercat.h"
+#include "stmmac_fpe-ethercat.h"
+#include "dwmac4-ethercat.h"
+#include "dwmac5-ethercat.h"
+#include "dwxgmac2-ethercat.h"
+
+#define GMAC5_MAC_FPE_CTRL_STS		0x00000234
+#define XGMAC_MAC_FPE_CTRL_STS		0x00000280
+
+#define GMAC5_MTL_FPE_CTRL_STS		0x00000c90
+#define XGMAC_MTL_FPE_CTRL_STS		0x00001090
+/* Preemption Classification */
+#define FPE_MTL_PREEMPTION_CLASS	GENMASK(15, 8)
+/* Additional Fragment Size of preempted frames */
+#define FPE_MTL_ADD_FRAG_SZ		GENMASK(1, 0)
+
+#define STMMAC_MAC_FPE_CTRL_STS_TRSP	BIT(19)
+#define STMMAC_MAC_FPE_CTRL_STS_TVER	BIT(18)
+#define STMMAC_MAC_FPE_CTRL_STS_RRSP	BIT(17)
+#define STMMAC_MAC_FPE_CTRL_STS_RVER	BIT(16)
+#define STMMAC_MAC_FPE_CTRL_STS_SRSP	BIT(2)
+#define STMMAC_MAC_FPE_CTRL_STS_SVER	BIT(1)
+#define STMMAC_MAC_FPE_CTRL_STS_EFPE	BIT(0)
+
+struct stmmac_fpe_reg {
+	const u32 mac_fpe_reg;		/* offset of MAC_FPE_CTRL_STS */
+	const u32 mtl_fpe_reg;		/* offset of MTL_FPE_CTRL_STS */
+	const u32 rxq_ctrl1_reg;	/* offset of MAC_RxQ_Ctrl1 */
+	const u32 fprq_mask;		/* Frame Preemption Residue Queue */
+	const u32 int_en_reg;		/* offset of MAC_Interrupt_Enable */
+	const u32 int_en_bit;		/* Frame Preemption Interrupt Enable */
+};
+
+bool ec_stmmac_fpe_supported(struct stmmac_priv *priv)
+{
+	return priv->dma_cap.fpesel && priv->fpe_cfg.reg &&
+		priv->hw->mac->fpe_map_preemption_class;
+}
+
+static void stmmac_fpe_configure_tx(struct ethtool_mmsv *mmsv, bool tx_enable)
+{
+	struct stmmac_fpe_cfg *cfg = container_of(mmsv, struct stmmac_fpe_cfg, mmsv);
+	struct stmmac_priv *priv = container_of(cfg, struct stmmac_priv, fpe_cfg);
+	const struct stmmac_fpe_reg *reg = cfg->reg;
+	u32 num_rxq = priv->plat->rx_queues_to_use;
+	void __iomem *ioaddr = priv->ioaddr;
+	u32 value;
+
+	if (tx_enable) {
+		cfg->fpe_csr = STMMAC_MAC_FPE_CTRL_STS_EFPE;
+		value = readl(ioaddr + reg->rxq_ctrl1_reg);
+		value &= ~reg->fprq_mask;
+		/* Keep this SHIFT, FIELD_PREP() expects a constant mask :-/ */
+		value |= (num_rxq - 1) << __ffs(reg->fprq_mask);
+		writel(value, ioaddr + reg->rxq_ctrl1_reg);
+	} else {
+		cfg->fpe_csr = 0;
+	}
+	writel(cfg->fpe_csr, ioaddr + reg->mac_fpe_reg);
+}
+
+static void stmmac_fpe_configure_pmac(struct ethtool_mmsv *mmsv, bool pmac_enable)
+{
+	struct stmmac_fpe_cfg *cfg = container_of(mmsv, struct stmmac_fpe_cfg, mmsv);
+	struct stmmac_priv *priv = container_of(cfg, struct stmmac_priv, fpe_cfg);
+	const struct stmmac_fpe_reg *reg = cfg->reg;
+	void __iomem *ioaddr = priv->ioaddr;
+	u32 value;
+
+	value = readl(ioaddr + reg->int_en_reg);
+
+	if (pmac_enable) {
+		if (!(value & reg->int_en_bit)) {
+			/* Dummy read to clear any pending masked interrupts */
+			readl(ioaddr + reg->mac_fpe_reg);
+
+			value |= reg->int_en_bit;
+		}
+	} else {
+		value &= ~reg->int_en_bit;
+	}
+
+	writel(value, ioaddr + reg->int_en_reg);
+}
+
+static void stmmac_fpe_send_mpacket(struct ethtool_mmsv *mmsv,
+				    enum ethtool_mpacket type)
+{
+	struct stmmac_fpe_cfg *cfg = container_of(mmsv, struct stmmac_fpe_cfg, mmsv);
+	struct stmmac_priv *priv = container_of(cfg, struct stmmac_priv, fpe_cfg);
+	const struct stmmac_fpe_reg *reg = cfg->reg;
+	void __iomem *ioaddr = priv->ioaddr;
+	u32 value = cfg->fpe_csr;
+
+	if (type == ETHTOOL_MPACKET_VERIFY)
+		value |= STMMAC_MAC_FPE_CTRL_STS_SVER;
+	else if (type == ETHTOOL_MPACKET_RESPONSE)
+		value |= STMMAC_MAC_FPE_CTRL_STS_SRSP;
+
+	writel(value, ioaddr + reg->mac_fpe_reg);
+}
+
+static const struct ethtool_mmsv_ops stmmac_mmsv_ops = {
+	.configure_tx = stmmac_fpe_configure_tx,
+	.configure_pmac = stmmac_fpe_configure_pmac,
+	.send_mpacket = stmmac_fpe_send_mpacket,
+};
+
+static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status)
+{
+	struct stmmac_fpe_cfg *fpe_cfg = &priv->fpe_cfg;
+	struct ethtool_mmsv *mmsv = &fpe_cfg->mmsv;
+
+	if (status == FPE_EVENT_UNKNOWN)
+		return;
+
+	if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER)
+		ethtool_mmsv_event_handle(mmsv, ETHTOOL_MMSV_LP_SENT_VERIFY_MPACKET);
+
+	if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER)
+		ethtool_mmsv_event_handle(mmsv, ETHTOOL_MMSV_LD_SENT_VERIFY_MPACKET);
+
+	if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP)
+		ethtool_mmsv_event_handle(mmsv, ETHTOOL_MMSV_LP_SENT_RESPONSE_MPACKET);
+}
+
+void ec_stmmac_fpe_irq_status(struct stmmac_priv *priv)
+{
+	const struct stmmac_fpe_reg *reg = priv->fpe_cfg.reg;
+	void __iomem *ioaddr = priv->ioaddr;
+	struct net_device *dev = priv->dev;
+	int status = FPE_EVENT_UNKNOWN;
+	u32 value;
+
+	/* Reads from the MAC_FPE_CTRL_STS register should only be performed
+	 * here, since the status flags of MAC_FPE_CTRL_STS are "clear on read"
+	 */
+	value = readl(ioaddr + reg->mac_fpe_reg);
+
+	if (value & STMMAC_MAC_FPE_CTRL_STS_TRSP) {
+		status |= FPE_EVENT_TRSP;
+		netdev_dbg(dev, "FPE: Respond mPacket is transmitted\n");
+	}
+
+	if (value & STMMAC_MAC_FPE_CTRL_STS_TVER) {
+		status |= FPE_EVENT_TVER;
+		netdev_dbg(dev, "FPE: Verify mPacket is transmitted\n");
+	}
+
+	if (value & STMMAC_MAC_FPE_CTRL_STS_RRSP) {
+		status |= FPE_EVENT_RRSP;
+		netdev_dbg(dev, "FPE: Respond mPacket is received\n");
+	}
+
+	if (value & STMMAC_MAC_FPE_CTRL_STS_RVER) {
+		status |= FPE_EVENT_RVER;
+		netdev_dbg(dev, "FPE: Verify mPacket is received\n");
+	}
+
+	stmmac_fpe_event_status(priv, status);
+}
+
+void ec_stmmac_fpe_init(struct stmmac_priv *priv)
+{
+	ethtool_mmsv_init(&priv->fpe_cfg.mmsv, priv->dev,
+			  &stmmac_mmsv_ops);
+
+	if ((!priv->fpe_cfg.reg || !priv->hw->mac->fpe_map_preemption_class) &&
+	    priv->dma_cap.fpesel)
+		dev_info(priv->device, "FPE is not supported by driver.\n");
+}
+
+int ec_stmmac_fpe_get_add_frag_size(struct stmmac_priv *priv)
+{
+	const struct stmmac_fpe_reg *reg = priv->fpe_cfg.reg;
+	void __iomem *ioaddr = priv->ioaddr;
+
+	return FIELD_GET(FPE_MTL_ADD_FRAG_SZ, readl(ioaddr + reg->mtl_fpe_reg));
+}
+
+void ec_stmmac_fpe_set_add_frag_size(struct stmmac_priv *priv, u32 add_frag_size)
+{
+	const struct stmmac_fpe_reg *reg = priv->fpe_cfg.reg;
+	void __iomem *ioaddr = priv->ioaddr;
+	u32 value;
+
+	value = readl(ioaddr + reg->mtl_fpe_reg);
+	writel(u32_replace_bits(value, add_frag_size, FPE_MTL_ADD_FRAG_SZ),
+	       ioaddr + reg->mtl_fpe_reg);
+}
+
+#define ALG_ERR_MSG "TX algorithm SP is not suitable for one-to-many mapping"
+#define WEIGHT_ERR_MSG "TXQ weight %u differs across other TXQs in TC: [%u]"
+
+int ec_dwmac5_fpe_map_preemption_class(struct net_device *ndev,
+				    struct netlink_ext_ack *extack, u32 pclass)
+{
+	u32 val, offset, count, queue_weight, preemptible_txqs = 0;
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int num_tc = netdev_get_num_tc(ndev);
+
+	if (!pclass)
+		goto update_mapping;
+
+	/* DWMAC CORE4+ can not program TC:TXQ mapping to hardware.
+	 *
+	 * Synopsys Databook:
+	 * "The number of Tx DMA channels is equal to the number of Tx queues,
+	 * and is direct one-to-one mapping."
+	 */
+	for (u32 tc = 0; tc < num_tc; tc++) {
+		count = ndev->tc_to_txq[tc].count;
+		offset = ndev->tc_to_txq[tc].offset;
+
+		if (pclass & BIT(tc))
+			preemptible_txqs |= GENMASK(offset + count - 1, offset);
+
+		/* This is 1:1 mapping, go to next TC */
+		if (count == 1)
+			continue;
+
+		if (priv->plat->tx_sched_algorithm == MTL_TX_ALGORITHM_SP) {
+			NL_SET_ERR_MSG_MOD(extack, ALG_ERR_MSG);
+			return -EINVAL;
+		}
+
+		queue_weight = priv->plat->tx_queues_cfg[offset].weight;
+
+		for (u32 i = 1; i < count; i++) {
+			if (priv->plat->tx_queues_cfg[offset + i].weight !=
+			    queue_weight) {
+				NL_SET_ERR_MSG_FMT_MOD(extack, WEIGHT_ERR_MSG,
+						       queue_weight, tc);
+				return -EINVAL;
+			}
+		}
+	}
+
+update_mapping:
+	val = readl(priv->ioaddr + GMAC5_MTL_FPE_CTRL_STS);
+	writel(u32_replace_bits(val, preemptible_txqs, FPE_MTL_PREEMPTION_CLASS),
+	       priv->ioaddr + GMAC5_MTL_FPE_CTRL_STS);
+
+	return 0;
+}
+
+int ec_dwxgmac3_fpe_map_preemption_class(struct net_device *ndev,
+				      struct netlink_ext_ack *extack, u32 pclass)
+{
+	u32 val, offset, count, preemptible_txqs = 0;
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int num_tc = netdev_get_num_tc(ndev);
+
+	if (!num_tc) {
+		/* Restore default TC:Queue mapping */
+		for (u32 i = 0; i < priv->plat->tx_queues_to_use; i++) {
+			val = readl(priv->ioaddr + XGMAC_MTL_TXQ_OPMODE(i));
+			writel(u32_replace_bits(val, i, XGMAC_Q2TCMAP),
+			       priv->ioaddr + XGMAC_MTL_TXQ_OPMODE(i));
+		}
+	}
+
+	/* Synopsys Databook:
+	 * "All Queues within a traffic class are selected in a round robin
+	 * fashion (when packets are available) when the traffic class is
+	 * selected by the scheduler for packet transmission. This is true for
+	 * any of the scheduling algorithms."
+	 */
+	for (u32 tc = 0; tc < num_tc; tc++) {
+		count = ndev->tc_to_txq[tc].count;
+		offset = ndev->tc_to_txq[tc].offset;
+
+		if (pclass & BIT(tc))
+			preemptible_txqs |= GENMASK(offset + count - 1, offset);
+
+		for (u32 i = 0; i < count; i++) {
+			val = readl(priv->ioaddr + XGMAC_MTL_TXQ_OPMODE(offset + i));
+			writel(u32_replace_bits(val, tc, XGMAC_Q2TCMAP),
+			       priv->ioaddr + XGMAC_MTL_TXQ_OPMODE(offset + i));
+		}
+	}
+
+	val = readl(priv->ioaddr + XGMAC_MTL_FPE_CTRL_STS);
+	writel(u32_replace_bits(val, preemptible_txqs, FPE_MTL_PREEMPTION_CLASS),
+	       priv->ioaddr + XGMAC_MTL_FPE_CTRL_STS);
+
+	return 0;
+}
+
+const struct stmmac_fpe_reg ec_dwmac5_fpe_reg = {
+	.mac_fpe_reg = GMAC5_MAC_FPE_CTRL_STS,
+	.mtl_fpe_reg = GMAC5_MTL_FPE_CTRL_STS,
+	.rxq_ctrl1_reg = GMAC_RXQ_CTRL1,
+	.fprq_mask = GMAC_RXQCTRL_FPRQ,
+	.int_en_reg = GMAC_INT_EN,
+	.int_en_bit = GMAC_INT_FPE_EN,
+};
+
+const struct stmmac_fpe_reg ec_dwxgmac3_fpe_reg = {
+	.mac_fpe_reg = XGMAC_MAC_FPE_CTRL_STS,
+	.mtl_fpe_reg = XGMAC_MTL_FPE_CTRL_STS,
+	.rxq_ctrl1_reg = XGMAC_RXQ_CTRL1,
+	.fprq_mask = XGMAC_FPRQ,
+	.int_en_reg = XGMAC_INT_EN,
+	.int_en_bit = XGMAC_FPEIE,
+};
diff --git a/drivers/net/ethercat/device/K3/stmmac_fpe-ethercat.h b/drivers/net/ethercat/device/K3/stmmac_fpe-ethercat.h
new file mode 100644
index 000000000000..de388893b387
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_fpe-ethercat.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 Furong Xu <0x1207@gmail.com>
+ * stmmac FPE(802.3 Qbu) handling
+ */
+#ifndef _STMMAC_FPE_H_
+#define _STMMAC_FPE_H_
+
+#include <linux/types.h>
+#include <linux/netdevice.h>
+
+struct stmmac_priv;
+
+bool ec_stmmac_fpe_supported(struct stmmac_priv *priv);
+void ec_stmmac_fpe_init(struct stmmac_priv *priv);
+void ec_stmmac_fpe_irq_status(struct stmmac_priv *priv);
+int ec_stmmac_fpe_get_add_frag_size(struct stmmac_priv *priv);
+void ec_stmmac_fpe_set_add_frag_size(struct stmmac_priv *priv, u32 add_frag_size);
+
+int ec_dwmac5_fpe_map_preemption_class(struct net_device *ndev,
+				    struct netlink_ext_ack *extack, u32 pclass);
+int ec_dwxgmac3_fpe_map_preemption_class(struct net_device *ndev,
+				      struct netlink_ext_ack *extack, u32 pclass);
+
+extern const struct stmmac_fpe_reg ec_dwmac5_fpe_reg;
+extern const struct stmmac_fpe_reg ec_dwxgmac3_fpe_reg;
+
+#endif
diff --git a/drivers/net/ethercat/device/K3/stmmac_hwtstamp-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_hwtstamp-ethercat.c
new file mode 100644
index 000000000000..42310ee9f50a
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_hwtstamp-ethercat.c
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  Copyright (C) 2013  Vayavya Labs Pvt Ltd
+
+  This implements all the API for managing HW timestamp & PTP.
+
+
+  Author: Rayagond Kokatanur <rayagond@vayavyalabs.com>
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/delay.h>
+#include <linux/ptp_clock_kernel.h>
+#include "common-ethercat.h"
+#include "stmmac_ptp-ethercat.h"
+#include "dwmac4-ethercat.h"
+#include "stmmac-ethercat.h"
+
+#define STMMAC_HWTS_CFG_MASK	(PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \
+				 PTP_TCR_TSINIT | PTP_TCR_TSUPDT | \
+				 PTP_TCR_TSCTRLSSR | PTP_TCR_SNAPTYPSEL_1 | \
+				 PTP_TCR_TSIPV4ENA | PTP_TCR_TSIPV6ENA | \
+				 PTP_TCR_TSEVNTENA | PTP_TCR_TSMSTRENA | \
+				 PTP_TCR_TSVER2ENA | PTP_TCR_TSIPENA | \
+				 PTP_TCR_TSTRIG | PTP_TCR_TSENALL)
+
+static void config_hw_tstamping(void __iomem *ioaddr, u32 data)
+{
+	u32 regval = readl(ioaddr + PTP_TCR);
+
+	regval &= ~STMMAC_HWTS_CFG_MASK;
+	regval |= data;
+
+	writel(regval, ioaddr + PTP_TCR);
+}
+
+static void config_sub_second_increment(void __iomem *ioaddr,
+		u32 ptp_clock, int gmac4, u32 *ssinc)
+{
+	u32 value = readl(ioaddr + PTP_TCR);
+	unsigned long data;
+	u32 reg_value;
+
+	/* For GMAC3.x, 4.x versions, in "fine adjustement mode" set sub-second
+	 * increment to twice the number of nanoseconds of a clock cycle.
+	 * The calculation of the default_addend value by the caller will set it
+	 * to mid-range = 2^31 when the remainder of this division is zero,
+	 * which will make the accumulator overflow once every 2 ptp_clock
+	 * cycles, adding twice the number of nanoseconds of a clock cycle :
+	 * 2000000000ULL / ptp_clock.
+	 */
+	if (value & PTP_TCR_TSCFUPDT)
+		data = (2000000000ULL / ptp_clock);
+	else
+		data = (1000000000ULL / ptp_clock);
+
+	/* 0.465ns accuracy */
+	if (!(value & PTP_TCR_TSCTRLSSR))
+		data = (data * 1000) / 465;
+
+	if (data > PTP_SSIR_SSINC_MAX)
+		data = PTP_SSIR_SSINC_MAX;
+
+	reg_value = data;
+	if (gmac4)
+		reg_value <<= GMAC4_PTP_SSIR_SSINC_SHIFT;
+
+	writel(reg_value, ioaddr + PTP_SSIR);
+
+	if (ssinc)
+		*ssinc = data;
+}
+
+static void hwtstamp_correct_latency(struct stmmac_priv *priv)
+{
+	void __iomem *ioaddr = priv->ptpaddr;
+	u32 reg_tsic, reg_tsicsns;
+	u32 reg_tsec, reg_tsecsns;
+	u64 scaled_ns;
+	u32 val;
+
+	/* MAC-internal ingress latency */
+	scaled_ns = readl(ioaddr + PTP_TS_INGR_LAT);
+
+	/* See section 11.7.2.5.3.1 "Ingress Correction" on page 4001 of
+	 * i.MX8MP Applications Processor Reference Manual Rev. 1, 06/2021
+	 */
+	val = readl(ioaddr + PTP_TCR);
+	if (val & PTP_TCR_TSCTRLSSR)
+		/* nanoseconds field is in decimal format with granularity of 1ns/bit */
+		scaled_ns = ((u64)NSEC_PER_SEC << 16) - scaled_ns;
+	else
+		/* nanoseconds field is in binary format with granularity of ~0.466ns/bit */
+		scaled_ns = ((1ULL << 31) << 16) -
+			DIV_U64_ROUND_CLOSEST(scaled_ns * PSEC_PER_NSEC, 466U);
+
+	reg_tsic = scaled_ns >> 16;
+	reg_tsicsns = scaled_ns & 0xff00;
+
+	/* set bit 31 for 2's compliment */
+	reg_tsic |= BIT(31);
+
+	writel(reg_tsic, ioaddr + PTP_TS_INGR_CORR_NS);
+	writel(reg_tsicsns, ioaddr + PTP_TS_INGR_CORR_SNS);
+
+	/* MAC-internal egress latency */
+	scaled_ns = readl(ioaddr + PTP_TS_EGR_LAT);
+
+	reg_tsec = scaled_ns >> 16;
+	reg_tsecsns = scaled_ns & 0xff00;
+
+	writel(reg_tsec, ioaddr + PTP_TS_EGR_CORR_NS);
+	writel(reg_tsecsns, ioaddr + PTP_TS_EGR_CORR_SNS);
+}
+
+static int init_systime(void __iomem *ioaddr, u32 sec, u32 nsec)
+{
+	u32 value;
+
+	writel(sec, ioaddr + PTP_STSUR);
+	writel(nsec, ioaddr + PTP_STNSUR);
+	/* issue command to initialize the system time value */
+	value = readl(ioaddr + PTP_TCR);
+	value |= PTP_TCR_TSINIT;
+	writel(value, ioaddr + PTP_TCR);
+
+	/* wait for present system time initialize to complete */
+	return readl_poll_timeout_atomic(ioaddr + PTP_TCR, value,
+				 !(value & PTP_TCR_TSINIT),
+				 10, 100000);
+}
+
+static int config_addend(void __iomem *ioaddr, u32 addend)
+{
+	u32 value;
+
+	writel(addend, ioaddr + PTP_TAR);
+	/* issue command to update the addend value */
+	value = readl(ioaddr + PTP_TCR);
+	value |= PTP_TCR_TSADDREG;
+	writel(value, ioaddr + PTP_TCR);
+
+	/* wait for present addend update to complete */
+	return readl_poll_timeout_atomic(ioaddr + PTP_TCR, value,
+					 !(value & PTP_TCR_TSADDREG),
+					 10, 100000);
+}
+
+static int adjust_systime(void __iomem *ioaddr, u32 sec, u32 nsec,
+		int add_sub, int gmac4)
+{
+	u32 value;
+
+	if (add_sub) {
+		/* If the new sec value needs to be subtracted with
+		 * the system time, then MAC_STSUR reg should be
+		 * programmed with (2^32 – <new_sec_value>)
+		 */
+		if (gmac4)
+			sec = -sec;
+
+		value = readl(ioaddr + PTP_TCR);
+		if (value & PTP_TCR_TSCTRLSSR)
+			nsec = (PTP_DIGITAL_ROLLOVER_MODE - nsec);
+		else
+			nsec = (PTP_BINARY_ROLLOVER_MODE - nsec);
+	}
+
+	writel(sec, ioaddr + PTP_STSUR);
+	value = (add_sub << PTP_STNSUR_ADDSUB_SHIFT) | nsec;
+	writel(value, ioaddr + PTP_STNSUR);
+
+	/* issue command to initialize the system time value */
+	value = readl(ioaddr + PTP_TCR);
+	value |= PTP_TCR_TSUPDT;
+	writel(value, ioaddr + PTP_TCR);
+
+	/* wait for present system time adjust/update to complete */
+	return readl_poll_timeout_atomic(ioaddr + PTP_TCR, value,
+					 !(value & PTP_TCR_TSUPDT),
+					 10, 100000);
+}
+
+static void get_systime(void __iomem *ioaddr, u64 *systime)
+{
+	u64 ns, sec0, sec1;
+
+	/* Get the TSS value */
+	sec1 = readl_relaxed(ioaddr + PTP_STSR);
+	do {
+		sec0 = sec1;
+		/* Get the TSSS value */
+		ns = readl_relaxed(ioaddr + PTP_STNSR);
+		/* Get the TSS value */
+		sec1 = readl_relaxed(ioaddr + PTP_STSR);
+	} while (sec0 != sec1);
+
+	if (systime)
+		*systime = ns + (sec1 * 1000000000ULL);
+}
+
+static void get_ptptime(void __iomem *ptpaddr, u64 *ptp_time)
+{
+	u64 ns;
+
+	ns = readl(ptpaddr + PTP_ATNR);
+	ns += (u64)readl(ptpaddr + PTP_ATSR) * NSEC_PER_SEC;
+
+	*ptp_time = ns;
+}
+
+static void timestamp_interrupt(struct stmmac_priv *priv)
+{
+	u32 num_snapshot, ts_status, tsync_int;
+	struct ptp_clock_event event;
+	u32 acr_value, channel;
+	unsigned long flags;
+	u64 ptp_time;
+	int i;
+
+	if (priv->plat->flags & STMMAC_FLAG_INT_SNAPSHOT_EN) {
+		wake_up(&priv->tstamp_busy_wait);
+		return;
+	}
+
+	tsync_int = readl(priv->ioaddr + GMAC_INT_STATUS) & GMAC_INT_TSIE;
+
+	if (!tsync_int)
+		return;
+
+	/* Read timestamp status to clear interrupt from either external
+	 * timestamp or start/end of PPS.
+	 */
+	ts_status = readl(priv->ioaddr + GMAC_TIMESTAMP_STATUS);
+
+	if (!(priv->plat->flags & STMMAC_FLAG_EXT_SNAPSHOT_EN))
+		return;
+
+	num_snapshot = (ts_status & GMAC_TIMESTAMP_ATSNS_MASK) >>
+		       GMAC_TIMESTAMP_ATSNS_SHIFT;
+
+	acr_value = readl(priv->ptpaddr + PTP_ACR);
+	channel = ilog2(FIELD_GET(PTP_ACR_MASK, acr_value));
+
+	for (i = 0; i < num_snapshot; i++) {
+		read_lock_irqsave(&priv->ptp_lock, flags);
+		get_ptptime(priv->ptpaddr, &ptp_time);
+		read_unlock_irqrestore(&priv->ptp_lock, flags);
+		event.type = PTP_CLOCK_EXTTS;
+		event.index = channel;
+		event.timestamp = ptp_time;
+		ptp_clock_event(priv->ptp_clock, &event);
+	}
+}
+
+const struct stmmac_hwtimestamp ec_stmmac_ptp = {
+	.config_hw_tstamping = config_hw_tstamping,
+	.init_systime = init_systime,
+	.config_sub_second_increment = config_sub_second_increment,
+	.config_addend = config_addend,
+	.adjust_systime = adjust_systime,
+	.get_systime = get_systime,
+	.get_ptptime = get_ptptime,
+	.timestamp_interrupt = timestamp_interrupt,
+	.hwtstamp_correct_latency = hwtstamp_correct_latency,
+};
+
+const struct stmmac_hwtimestamp ec_dwmac1000_ptp = {
+	.config_hw_tstamping = config_hw_tstamping,
+	.init_systime = init_systime,
+	.config_sub_second_increment = config_sub_second_increment,
+	.config_addend = config_addend,
+	.adjust_systime = adjust_systime,
+	.get_systime = get_systime,
+	.get_ptptime = ec_dwmac1000_get_ptptime,
+	.timestamp_interrupt = ec_dwmac1000_timestamp_interrupt,
+};
diff --git a/drivers/net/ethercat/device/K3/stmmac_main-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_main-ethercat.c
new file mode 100644
index 000000000000..5c9fea312f4f
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_main-ethercat.c
@@ -0,0 +1,8191 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
+  ST Ethernet IPs are built around a Synopsys IP Core.
+
+	Copyright(C) 2007-2011 STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+
+  Documentation available at:
+	http://www.stlinux.com
+  Support available at:
+	https://bugzilla.stlinux.com/
+*******************************************************************************/
+
+#include <linux/clk.h>
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/ip.h>
+#include <linux/tcp.h>
+#include <linux/skbuff.h>
+#include <linux/ethtool.h>
+#include <linux/if_ether.h>
+#include <linux/crc32.h>
+#include <linux/mii.h>
+#include <linux/if.h>
+#include <linux/if_vlan.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_wakeirq.h>
+#include <linux/prefetch.h>
+#include <linux/pinctrl/consumer.h>
+#ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#endif /* CONFIG_DEBUG_FS */
+#include <linux/net_tstamp.h>
+#include <linux/phylink.h>
+#include <linux/udp.h>
+#include <linux/bpf_trace.h>
+#include <net/page_pool/helpers.h>
+#include <net/pkt_cls.h>
+#include <net/xdp_sock_drv.h>
+#include "stmmac_ptp-ethercat.h"
+#include "stmmac_fpe-ethercat.h"
+#include "stmmac-ethercat.h"
+#include "stmmac_xdp-ethercat.h"
+#include <linux/reset.h>
+#include <linux/of_mdio.h>
+#include "dwmac1000-ethercat.h"
+#include "dwxgmac2-ethercat.h"
+#include "hwif-ethercat.h"
+
+static inline void ec_txq_trans_cond_update(struct netdev_queue *txq)
+{
+	unsigned long now = jiffies;
+
+	if (READ_ONCE(txq->trans_start) != now)
+		WRITE_ONCE(txq->trans_start, now);
+}
+
+/* As long as the interface is active, we keep the timestamping counter enabled
+ * with fine resolution and binary rollover. This avoid non-monotonic behavior
+ * (clock jumps) when changing timestamping settings at runtime.
+ */
+#define STMMAC_HWTS_ACTIVE	(PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \
+				 PTP_TCR_TSCTRLSSR)
+
+#define	STMMAC_ALIGN(x)		ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
+#define	TSO_MAX_BUFF_SIZE	(SZ_16K - 1)
+
+/* Module parameters */
+#define TX_TIMEO	5000
+static int watchdog = TX_TIMEO;
+module_param(watchdog, int, 0644);
+MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
+
+static int debug = -1;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
+
+static int phyaddr = -1;
+module_param(phyaddr, int, 0444);
+MODULE_PARM_DESC(phyaddr, "Physical device address");
+
+#define STMMAC_TX_THRESH(x)	((x)->dma_conf.dma_tx_size / 4)
+
+/* Limit to make sure XDP TX and slow path can coexist */
+#define STMMAC_XSK_TX_BUDGET_MAX	256
+#define STMMAC_TX_XSK_AVAIL		16
+#define STMMAC_RX_FILL_BATCH		16
+
+#define STMMAC_XDP_PASS		0
+#define STMMAC_XDP_CONSUMED	BIT(0)
+#define STMMAC_XDP_TX		BIT(1)
+#define STMMAC_XDP_REDIRECT	BIT(2)
+
+static int flow_ctrl = 0xdead;
+module_param(flow_ctrl, int, 0644);
+MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)");
+
+static int pause = PAUSE_TIME;
+module_param(pause, int, 0644);
+MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)");
+
+#define TC_DEFAULT 64
+static int tc = TC_DEFAULT;
+module_param(tc, int, 0644);
+MODULE_PARM_DESC(tc, "DMA threshold control value");
+
+/* This is unused */
+#define	DEFAULT_BUFSIZE	1536
+static int buf_sz = DEFAULT_BUFSIZE;
+module_param(buf_sz, int, 0644);
+MODULE_PARM_DESC(buf_sz, "DMA buffer size");
+
+static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
+				      NETIF_MSG_LINK | NETIF_MSG_IFUP |
+				      NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
+
+#define STMMAC_DEFAULT_LPI_TIMER	1000
+static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
+module_param(eee_timer, uint, 0644);
+MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
+#define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x))
+
+/* By default the driver will use the ring mode to manage tx and rx descriptors,
+ * but allow user to force to use the chain instead of the ring
+ */
+static unsigned int chain_mode;
+module_param(chain_mode, int, 0444);
+MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
+
+static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
+/* For MSI interrupts handling */
+static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id);
+static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id);
+static irqreturn_t stmmac_msi_intr_tx(int irq, void *data);
+static irqreturn_t stmmac_msi_intr_rx(int irq, void *data);
+static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue);
+static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue);
+static void stmmac_reset_queues_param(struct stmmac_priv *priv);
+static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
+static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
+static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
+					  u32 rxmode, u32 chan);
+
+#ifdef CONFIG_DEBUG_FS
+static const struct net_device_ops stmmac_netdev_ops;
+static void stmmac_init_fs(struct net_device *dev);
+static void stmmac_exit_fs(struct net_device *dev);
+#endif
+
+#define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC))
+
+/**
+ * ec_stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock
+ * @bsp_priv: BSP private data structure (unused)
+ * @clk_tx_i: the transmit clock
+ * @interface: the selected interface mode
+ * @speed: the speed that the MAC will be operating at
+ *
+ * Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps,
+ * 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least
+ * MII, GMII, RGMII and RMII interface modes. Platforms can hook this into
+ * the plat_data->set_clk_tx_rate method directly, call it via their own
+ * implementation, or implement their own method should they have more
+ * complex requirements. It is intended to only be used in this method.
+ *
+ * plat_data->clk_tx_i must be filled in.
+ */
+int ec_stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i,
+			   phy_interface_t interface, int speed)
+{
+	long rate = rgmii_clock(speed);
+
+	/* Silently ignore unsupported speeds as rgmii_clock() only
+	 * supports 10, 100 and 1000Mbps. We do not want to spit
+	 * errors for 2500 and higher speeds here.
+	 */
+	if (rate < 0)
+		return 0;
+
+	return clk_set_rate(clk_tx_i, rate);
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_set_clk_tx_rate);
+
+/**
+ * stmmac_verify_args - verify the driver parameters.
+ * Description: it checks the driver parameters and set a default in case of
+ * errors.
+ */
+static void stmmac_verify_args(void)
+{
+	if (unlikely(watchdog < 0))
+		watchdog = TX_TIMEO;
+	if (unlikely((pause < 0) || (pause > 0xffff)))
+		pause = PAUSE_TIME;
+
+	if (flow_ctrl != 0xdead)
+		pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n");
+}
+
+static void __stmmac_disable_all_queues(struct stmmac_priv *priv)
+{
+	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
+	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
+	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
+	u32 queue;
+
+	if (get_ecdev(priv))
+		return;
+
+	for (queue = 0; queue < maxq; queue++) {
+		struct stmmac_channel *ch = &priv->channel[queue];
+
+		if (stmmac_xdp_is_enabled(priv) &&
+		    test_bit(queue, priv->af_xdp_zc_qps)) {
+			napi_disable(&ch->rxtx_napi);
+			continue;
+		}
+
+		if (queue < rx_queues_cnt)
+			napi_disable(&ch->rx_napi);
+		if (queue < tx_queues_cnt)
+			napi_disable(&ch->tx_napi);
+	}
+}
+
+/**
+ * stmmac_disable_all_queues - Disable all queues
+ * @priv: driver private structure
+ */
+static void stmmac_disable_all_queues(struct stmmac_priv *priv)
+{
+	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
+	struct stmmac_rx_queue *rx_q;
+	u32 queue;
+
+	/* synchronize_rcu() needed for pending XDP buffers to drain */
+	for (queue = 0; queue < rx_queues_cnt; queue++) {
+		rx_q = &priv->dma_conf.rx_queue[queue];
+		if (rx_q->xsk_pool) {
+			synchronize_rcu();
+			break;
+		}
+	}
+	if (get_ecdev(priv))
+		return;
+
+	__stmmac_disable_all_queues(priv);
+}
+
+/**
+ * stmmac_enable_all_queues - Enable all queues
+ * @priv: driver private structure
+ */
+static void stmmac_enable_all_queues(struct stmmac_priv *priv)
+{
+	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
+	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
+	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
+	u32 queue;
+
+	if (get_ecdev(priv))
+		return;
+
+	for (queue = 0; queue < maxq; queue++) {
+		struct stmmac_channel *ch = &priv->channel[queue];
+
+		if (stmmac_xdp_is_enabled(priv) &&
+		    test_bit(queue, priv->af_xdp_zc_qps)) {
+			napi_enable(&ch->rxtx_napi);
+			continue;
+		}
+
+		if (queue < rx_queues_cnt)
+			napi_enable(&ch->rx_napi);
+		if (queue < tx_queues_cnt)
+			napi_enable(&ch->tx_napi);
+	}
+}
+
+static void stmmac_service_event_schedule(struct stmmac_priv *priv)
+{
+	if (!test_bit(STMMAC_DOWN, &priv->state) &&
+	    !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
+		queue_work(priv->wq, &priv->service_task);
+}
+
+static void stmmac_global_err(struct stmmac_priv *priv)
+{
+	if (get_ecdev(priv)) {
+		ecdev_set_link(get_ecdev(priv), 0);
+	} else {
+		netif_carrier_off(priv->dev);
+	}
+	set_bit(STMMAC_RESET_REQUESTED, &priv->state);
+	stmmac_service_event_schedule(priv);
+}
+
+static void print_pkt(unsigned char *buf, int len)
+{
+	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
+	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
+}
+
+static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	u32 avail;
+
+	if (tx_q->dirty_tx > tx_q->cur_tx)
+		avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
+	else
+		avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;
+
+	return avail;
+}
+
+/**
+ * stmmac_rx_dirty - Get RX queue dirty
+ * @priv: driver private structure
+ * @queue: RX queue index
+ */
+static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+	u32 dirty;
+
+	if (rx_q->dirty_rx <= rx_q->cur_rx)
+		dirty = rx_q->cur_rx - rx_q->dirty_rx;
+	else
+		dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;
+
+	return dirty;
+}
+
+static bool stmmac_eee_tx_busy(struct stmmac_priv *priv)
+{
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 queue;
+
+	/* check if all TX queues have the work finished */
+	for (queue = 0; queue < tx_cnt; queue++) {
+		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+
+		if (tx_q->dirty_tx != tx_q->cur_tx)
+			return true; /* still unfinished work */
+	}
+
+	return false;
+}
+
+static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv)
+{
+	mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
+}
+
+/**
+ * stmmac_try_to_start_sw_lpi - check and enter in LPI mode
+ * @priv: driver private structure
+ * Description: this function is to verify and enter in LPI mode in case of
+ * EEE.
+ */
+static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv)
+{
+	if (stmmac_eee_tx_busy(priv)) {
+		stmmac_restart_sw_lpi_timer(priv);
+		return;
+	}
+
+	/* Check and enter in LPI mode */
+	if (!priv->tx_path_in_lpi_mode)
+		stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED,
+				    priv->tx_lpi_clk_stop, 0);
+}
+
+/**
+ * stmmac_stop_sw_lpi - stop transmitting LPI
+ * @priv: driver private structure
+ * Description: When using software-controlled LPI, stop transmitting LPI state.
+ */
+static void stmmac_stop_sw_lpi(struct stmmac_priv *priv)
+{
+	timer_delete_sync(&priv->eee_ctrl_timer);
+	stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0);
+	priv->tx_path_in_lpi_mode = false;
+}
+
+/**
+ * stmmac_eee_ctrl_timer - EEE TX SW timer.
+ * @t:  timer_list struct containing private info
+ * Description:
+ *  if there is no data transfer and if we are not in LPI state,
+ *  then MAC Transmitter can be moved to LPI state.
+ */
+static void stmmac_eee_ctrl_timer(struct timer_list *t)
+{
+	struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer);
+
+	stmmac_try_to_start_sw_lpi(priv);
+}
+
+/* stmmac_get_tx_hwtstamp - get HW TX timestamps
+ * @priv: driver private structure
+ * @p : descriptor pointer
+ * @skb : the socket buffer
+ * Description :
+ * This function will read timestamp from the descriptor & pass it to stack.
+ * and also perform some sanity checks.
+ */
+static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
+				   struct dma_desc *p, struct sk_buff *skb)
+{
+	struct skb_shared_hwtstamps shhwtstamp;
+	bool found = false;
+	u64 ns = 0;
+
+	if (!priv->hwts_tx_en)
+		return;
+
+	/* exit if skb doesn't support hw tstamp */
+	if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
+		return;
+
+	/* check tx tstamp status */
+	if (stmmac_get_tx_timestamp_status(priv, p)) {
+		stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
+		found = true;
+	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
+		found = true;
+	}
+
+	if (found) {
+		ns -= priv->plat->cdc_error_adj;
+
+		memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
+		shhwtstamp.hwtstamp = ns_to_ktime(ns);
+
+		netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
+		/* pass tstamp to stack */
+		skb_tstamp_tx(skb, &shhwtstamp);
+	}
+}
+
+/* stmmac_get_rx_hwtstamp - get HW RX timestamps
+ * @priv: driver private structure
+ * @p : descriptor pointer
+ * @np : next descriptor pointer
+ * @skb : the socket buffer
+ * Description :
+ * This function will read received packet's timestamp from the descriptor
+ * and pass it to stack. It also perform some sanity checks.
+ */
+static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
+				   struct dma_desc *np, struct sk_buff *skb)
+{
+	struct skb_shared_hwtstamps *shhwtstamp = NULL;
+	struct dma_desc *desc = p;
+	u64 ns = 0;
+
+	if (!priv->hwts_rx_en)
+		return;
+	/* For GMAC4, the valid timestamp is from CTX next desc. */
+	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
+		desc = np;
+
+	/* Check if timestamp is available */
+	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
+		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
+
+		ns -= priv->plat->cdc_error_adj;
+
+		netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
+		shhwtstamp = skb_hwtstamps(skb);
+		memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
+		shhwtstamp->hwtstamp = ns_to_ktime(ns);
+	} else  {
+		netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
+	}
+}
+
+/**
+ *  stmmac_hwtstamp_set - control hardware timestamping.
+ *  @dev: device pointer.
+ *  @config: the timestamping configuration.
+ *  @extack: netlink extended ack structure for error reporting.
+ *  Description:
+ *  This function configures the MAC to enable/disable both outgoing(TX)
+ *  and incoming(RX) packets time stamping based on user input.
+ *  Return Value:
+ *  0 on success and an appropriate -ve integer on failure.
+ */
+static int stmmac_hwtstamp_set(struct net_device *dev,
+			       struct kernel_hwtstamp_config *config,
+			       struct netlink_ext_ack *extack)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 ptp_v2 = 0;
+	u32 tstamp_all = 0;
+	u32 ptp_over_ipv4_udp = 0;
+	u32 ptp_over_ipv6_udp = 0;
+	u32 ptp_over_ethernet = 0;
+	u32 snap_type_sel = 0;
+	u32 ts_master_en = 0;
+	u32 ts_event_en = 0;
+
+	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
+		NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping");
+		priv->hwts_tx_en = 0;
+		priv->hwts_rx_en = 0;
+
+		return -EOPNOTSUPP;
+	}
+
+	if (!netif_running(dev)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot change timestamping configuration while down");
+		return -ENODEV;
+	}
+
+	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
+		   __func__, config->flags, config->tx_type, config->rx_filter);
+
+	if (config->tx_type != HWTSTAMP_TX_OFF &&
+	    config->tx_type != HWTSTAMP_TX_ON)
+		return -ERANGE;
+
+	if (priv->adv_ts) {
+		switch (config->rx_filter) {
+		case HWTSTAMP_FILTER_NONE:
+			/* time stamp no incoming packet at all */
+			config->rx_filter = HWTSTAMP_FILTER_NONE;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
+			/* PTP v1, UDP, any kind of event packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
+			/* 'xmac' hardware can support Sync, Pdelay_Req and
+			 * Pdelay_resp by setting bit14 and bits17/16 to 01
+			 * This leaves Delay_Req timestamps out.
+			 * Enable all events *and* general purpose message
+			 * timestamping
+			 */
+			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+			/* PTP v1, UDP, Sync packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
+			/* take time stamp for SYNC messages only */
+			ts_event_en = PTP_TCR_TSEVNTENA;
+
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
+			/* PTP v1, UDP, Delay_req packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
+			/* take time stamp for Delay_Req messages only */
+			ts_master_en = PTP_TCR_TSMSTRENA;
+			ts_event_en = PTP_TCR_TSEVNTENA;
+
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+			/* PTP v2, UDP, any kind of event packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
+			ptp_v2 = PTP_TCR_TSVER2ENA;
+			/* take time stamp for all event messages */
+			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
+
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+			/* PTP v2, UDP, Sync packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
+			ptp_v2 = PTP_TCR_TSVER2ENA;
+			/* take time stamp for SYNC messages only */
+			ts_event_en = PTP_TCR_TSEVNTENA;
+
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+			/* PTP v2, UDP, Delay_req packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
+			ptp_v2 = PTP_TCR_TSVER2ENA;
+			/* take time stamp for Delay_Req messages only */
+			ts_master_en = PTP_TCR_TSMSTRENA;
+			ts_event_en = PTP_TCR_TSEVNTENA;
+
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V2_EVENT:
+			/* PTP v2/802.AS1 any layer, any kind of event packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
+			ptp_v2 = PTP_TCR_TSVER2ENA;
+			snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
+			if (priv->synopsys_id < DWMAC_CORE_4_10)
+				ts_event_en = PTP_TCR_TSEVNTENA;
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			ptp_over_ethernet = PTP_TCR_TSIPENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V2_SYNC:
+			/* PTP v2/802.AS1, any layer, Sync packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
+			ptp_v2 = PTP_TCR_TSVER2ENA;
+			/* take time stamp for SYNC messages only */
+			ts_event_en = PTP_TCR_TSEVNTENA;
+
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			ptp_over_ethernet = PTP_TCR_TSIPENA;
+			break;
+
+		case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
+			/* PTP v2/802.AS1, any layer, Delay_req packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
+			ptp_v2 = PTP_TCR_TSVER2ENA;
+			/* take time stamp for Delay_Req messages only */
+			ts_master_en = PTP_TCR_TSMSTRENA;
+			ts_event_en = PTP_TCR_TSEVNTENA;
+
+			ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
+			ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
+			ptp_over_ethernet = PTP_TCR_TSIPENA;
+			break;
+
+		case HWTSTAMP_FILTER_NTP_ALL:
+		case HWTSTAMP_FILTER_ALL:
+			/* time stamp any incoming packet */
+			config->rx_filter = HWTSTAMP_FILTER_ALL;
+			tstamp_all = PTP_TCR_TSENALL;
+			break;
+
+		default:
+			return -ERANGE;
+		}
+	} else {
+		switch (config->rx_filter) {
+		case HWTSTAMP_FILTER_NONE:
+			config->rx_filter = HWTSTAMP_FILTER_NONE;
+			break;
+		default:
+			/* PTP v1, UDP, any kind of event packet */
+			config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
+			break;
+		}
+	}
+	priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE;
+	priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON;
+
+	priv->systime_flags = STMMAC_HWTS_ACTIVE;
+
+	if (priv->hwts_tx_en || priv->hwts_rx_en) {
+		priv->systime_flags |= tstamp_all | ptp_v2 |
+				       ptp_over_ethernet | ptp_over_ipv6_udp |
+				       ptp_over_ipv4_udp | ts_event_en |
+				       ts_master_en | snap_type_sel;
+	}
+
+	stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);
+
+	priv->tstamp_config = *config;
+
+	return 0;
+}
+
+/**
+ *  stmmac_hwtstamp_get - read hardware timestamping.
+ *  @dev: device pointer.
+ *  @config: the timestamping configuration.
+ *  Description:
+ *  This function obtain the current hardware timestamping settings
+ *  as requested.
+ */
+static int stmmac_hwtstamp_get(struct net_device *dev,
+			       struct kernel_hwtstamp_config *config)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
+		return -EOPNOTSUPP;
+
+	*config = priv->tstamp_config;
+
+	return 0;
+}
+
+/**
+ * stmmac_init_tstamp_counter - init hardware timestamping counter
+ * @priv: driver private structure
+ * @systime_flags: timestamping flags
+ * Description:
+ * Initialize hardware counter for packet timestamping.
+ * This is valid as long as the interface is open and not suspended.
+ * Will be rerun after resuming from suspend, case in which the timestamping
+ * flags updated by stmmac_hwtstamp_set() also need to be restored.
+ */
+static int stmmac_init_tstamp_counter(struct stmmac_priv *priv,
+				      u32 systime_flags)
+{
+	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
+	struct timespec64 now;
+	u32 sec_inc = 0;
+	u64 temp = 0;
+
+	if (!priv->plat->clk_ptp_rate) {
+		netdev_err(priv->dev, "Invalid PTP clock rate");
+		return -EINVAL;
+	}
+
+	stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);
+	priv->systime_flags = systime_flags;
+
+	/* program Sub Second Increment reg */
+	stmmac_config_sub_second_increment(priv, priv->ptpaddr,
+					   priv->plat->clk_ptp_rate,
+					   xmac, &sec_inc);
+	temp = div_u64(1000000000ULL, sec_inc);
+
+	/* Store sub second increment for later use */
+	priv->sub_second_inc = sec_inc;
+
+	/* calculate default added value:
+	 * formula is :
+	 * addend = (2^32)/freq_div_ratio;
+	 * where, freq_div_ratio = 1e9ns/sec_inc
+	 */
+	temp = (u64)(temp << 32);
+	priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
+	stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
+
+	/* initialize system time */
+	ktime_get_real_ts64(&now);
+
+	/* lower 32 bits of tv_sec are safe until y2106 */
+	stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec);
+
+	return 0;
+}
+
+/**
+ * stmmac_init_timestamping - initialise timestamping
+ * @priv: driver private structure
+ * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
+ * This is done by looking at the HW cap. register.
+ * This function also registers the ptp driver.
+ */
+static int stmmac_init_timestamping(struct stmmac_priv *priv)
+{
+	bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
+	int ret;
+
+	if (priv->plat->ptp_clk_freq_config)
+		priv->plat->ptp_clk_freq_config(priv);
+
+	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) {
+		netdev_info(priv->dev, "PTP not supported by HW\n");
+		return -EOPNOTSUPP;
+	}
+
+	ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE);
+	if (ret) {
+		netdev_warn(priv->dev, "PTP init failed\n");
+		return ret;
+	}
+
+	priv->adv_ts = 0;
+	/* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
+	if (xmac && priv->dma_cap.atime_stamp)
+		priv->adv_ts = 1;
+	/* Dwmac 3.x core with extend_desc can support adv_ts */
+	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
+		priv->adv_ts = 1;
+
+	if (priv->dma_cap.time_stamp)
+		netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
+
+	if (priv->adv_ts)
+		netdev_info(priv->dev,
+			    "IEEE 1588-2008 Advanced Timestamp supported\n");
+
+	priv->hwts_tx_en = 0;
+	priv->hwts_rx_en = 0;
+
+	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)
+		stmmac_hwtstamp_correct_latency(priv, priv);
+
+	return 0;
+}
+
+static void stmmac_setup_ptp(struct stmmac_priv *priv)
+{
+	int ret;
+
+	ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
+	if (ret < 0)
+		netdev_warn(priv->dev,
+			    "failed to enable PTP reference clock: %pe\n",
+			    ERR_PTR(ret));
+
+	if (stmmac_init_timestamping(priv) == 0)
+		ec_stmmac_ptp_register(priv);
+}
+
+static void stmmac_release_ptp(struct stmmac_priv *priv)
+{
+	ec_stmmac_ptp_unregister(priv);
+	clk_disable_unprepare(priv->plat->clk_ptp_ref);
+}
+
+/**
+ *  stmmac_mac_flow_ctrl - Configure flow control in all queues
+ *  @priv: driver private structure
+ *  @duplex: duplex passed to the next function
+ *  @flow_ctrl: desired flow control modes
+ *  Description: It is used for configuring the flow control in all queues
+ */
+static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex,
+				 unsigned int flow_ctrl)
+{
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+
+	stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time,
+			 tx_cnt);
+}
+
+static unsigned long stmmac_mac_get_caps(struct phylink_config *config,
+					 phy_interface_t interface)
+{
+	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
+
+	/* Refresh the MAC-specific capabilities */
+	stmmac_mac_update_caps(priv);
+
+	config->mac_capabilities = priv->hw->link.caps;
+
+	if (priv->plat->max_speed)
+		phylink_limit_mac_speed(config, priv->plat->max_speed);
+
+	return config->mac_capabilities;
+}
+
+static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config,
+						 phy_interface_t interface)
+{
+	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
+	struct phylink_pcs *pcs;
+
+	if (priv->plat->select_pcs) {
+		pcs = priv->plat->select_pcs(priv, interface);
+		if (!IS_ERR(pcs))
+			return pcs;
+	}
+
+	return NULL;
+}
+
+static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
+			      const struct phylink_link_state *state)
+{
+	/* Nothing to do, xpcs_config() handles everything */
+}
+
+static void stmmac_mac_link_down(struct phylink_config *config,
+				 unsigned int mode, phy_interface_t interface)
+{
+	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
+
+	if (get_ecdev(priv)) {
+		ecdev_set_link(get_ecdev(priv), false);
+	}
+	stmmac_mac_set(priv, priv->ioaddr, false);
+	if (priv->dma_cap.eee)
+		stmmac_set_eee_pls(priv, priv->hw, false);
+
+	if (ec_stmmac_fpe_supported(priv))
+		ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, false);
+}
+
+static void stmmac_mac_link_up(struct phylink_config *config,
+			       struct phy_device *phy,
+			       unsigned int mode, phy_interface_t interface,
+			       int speed, int duplex,
+			       bool tx_pause, bool rx_pause)
+{
+	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
+	unsigned int flow_ctrl;
+	u32 old_ctrl, ctrl;
+	int ret;
+
+	if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&
+	    priv->plat->serdes_powerup)
+		priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv);
+
+	old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
+	ctrl = old_ctrl & ~priv->hw->link.speed_mask;
+
+	if (interface == PHY_INTERFACE_MODE_USXGMII) {
+		switch (speed) {
+		case SPEED_10000:
+			ctrl |= priv->hw->link.xgmii.speed10000;
+			break;
+		case SPEED_5000:
+			ctrl |= priv->hw->link.xgmii.speed5000;
+			break;
+		case SPEED_2500:
+			ctrl |= priv->hw->link.xgmii.speed2500;
+			break;
+		default:
+			return;
+		}
+	} else if (interface == PHY_INTERFACE_MODE_XLGMII) {
+		switch (speed) {
+		case SPEED_100000:
+			ctrl |= priv->hw->link.xlgmii.speed100000;
+			break;
+		case SPEED_50000:
+			ctrl |= priv->hw->link.xlgmii.speed50000;
+			break;
+		case SPEED_40000:
+			ctrl |= priv->hw->link.xlgmii.speed40000;
+			break;
+		case SPEED_25000:
+			ctrl |= priv->hw->link.xlgmii.speed25000;
+			break;
+		case SPEED_10000:
+			ctrl |= priv->hw->link.xgmii.speed10000;
+			break;
+		case SPEED_2500:
+			ctrl |= priv->hw->link.speed2500;
+			break;
+		case SPEED_1000:
+			ctrl |= priv->hw->link.speed1000;
+			break;
+		default:
+			return;
+		}
+	} else {
+		switch (speed) {
+		case SPEED_2500:
+			ctrl |= priv->hw->link.speed2500;
+			break;
+		case SPEED_1000:
+			ctrl |= priv->hw->link.speed1000;
+			break;
+		case SPEED_100:
+			ctrl |= priv->hw->link.speed100;
+			break;
+		case SPEED_10:
+			ctrl |= priv->hw->link.speed10;
+			break;
+		default:
+			return;
+		}
+	}
+
+	if (priv->plat->fix_mac_speed)
+		priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode);
+
+	if (!duplex)
+		ctrl &= ~priv->hw->link.duplex;
+	else
+		ctrl |= priv->hw->link.duplex;
+
+	/* Flow Control operation */
+	if (rx_pause && tx_pause)
+		flow_ctrl = FLOW_AUTO;
+	else if (rx_pause && !tx_pause)
+		flow_ctrl = FLOW_RX;
+	else if (!rx_pause && tx_pause)
+		flow_ctrl = FLOW_TX;
+	else
+		flow_ctrl = FLOW_OFF;
+
+	stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl);
+
+	if (ctrl != old_ctrl)
+		writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
+
+	if (priv->plat->set_clk_tx_rate) {
+		ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv,
+						priv->plat->clk_tx_i,
+						interface, speed);
+		if (ret < 0)
+			netdev_err(priv->dev,
+				   "failed to configure %s transmit clock for %dMbps: %pe\n",
+				   phy_modes(interface), speed, ERR_PTR(ret));
+	}
+
+	stmmac_mac_set(priv, priv->ioaddr, true);
+	if (priv->dma_cap.eee)
+		stmmac_set_eee_pls(priv, priv->hw, true);
+
+	if (ec_stmmac_fpe_supported(priv))
+		ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, true);
+
+	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)
+		stmmac_hwtstamp_correct_latency(priv, priv);
+
+	if (get_ecdev(priv)) {
+		ecdev_set_link(get_ecdev(priv), true);
+	}
+}
+
+static void stmmac_mac_disable_tx_lpi(struct phylink_config *config)
+{
+	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
+
+	priv->eee_active = false;
+
+	mutex_lock(&priv->lock);
+
+	priv->eee_enabled = false;
+
+	netdev_dbg(priv->dev, "disable EEE\n");
+	priv->eee_sw_timer_en = false;
+	timer_delete_sync(&priv->eee_ctrl_timer);
+	stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0);
+	priv->tx_path_in_lpi_mode = false;
+
+	stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS);
+	mutex_unlock(&priv->lock);
+}
+
+static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer,
+				    bool tx_clk_stop)
+{
+	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
+	int ret;
+
+	priv->tx_lpi_timer = timer;
+	priv->eee_active = true;
+
+	mutex_lock(&priv->lock);
+
+	priv->eee_enabled = true;
+
+	/* Update the transmit clock stop according to PHY capability if
+	 * the platform allows
+	 */
+	if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP)
+		priv->tx_lpi_clk_stop = tx_clk_stop;
+
+	stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
+			     STMMAC_DEFAULT_TWT_LS);
+
+	/* Try to cnfigure the hardware timer. */
+	ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER,
+				  priv->tx_lpi_clk_stop, priv->tx_lpi_timer);
+
+	if (ret) {
+		/* Hardware timer mode not supported, or value out of range.
+		 * Fall back to using software LPI mode
+		 */
+		priv->eee_sw_timer_en = true;
+		stmmac_restart_sw_lpi_timer(priv);
+	}
+
+	mutex_unlock(&priv->lock);
+	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
+
+	return 0;
+}
+
+static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode,
+			     phy_interface_t interface)
+{
+	struct net_device *ndev = to_net_dev(config->dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	if (priv->plat->mac_finish)
+		priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, interface);
+
+	return 0;
+}
+
+static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
+	.mac_get_caps = stmmac_mac_get_caps,
+	.mac_select_pcs = stmmac_mac_select_pcs,
+	.mac_config = stmmac_mac_config,
+	.mac_link_down = stmmac_mac_link_down,
+	.mac_link_up = stmmac_mac_link_up,
+	.mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi,
+	.mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi,
+	.mac_finish = stmmac_mac_finish,
+};
+
+/**
+ * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
+ * @priv: driver private structure
+ * Description: this is to verify if the HW supports the PCS.
+ * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
+ * configured for the TBI, RTBI, or SGMII PHY interface.
+ */
+static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
+{
+	int interface = priv->plat->phy_interface;
+
+	if (priv->dma_cap.pcs) {
+		if ((interface == PHY_INTERFACE_MODE_RGMII) ||
+		    (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
+		    (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
+		    (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
+			netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
+			priv->hw->pcs = STMMAC_PCS_RGMII;
+		} else if (interface == PHY_INTERFACE_MODE_SGMII) {
+			netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
+			priv->hw->pcs = STMMAC_PCS_SGMII;
+		}
+	}
+}
+
+/**
+ * stmmac_init_phy - PHY initialization
+ * @dev: net device structure
+ * Description: it initializes the driver's PHY state, and attaches the PHY
+ * to the mac driver.
+ *  Return value:
+ *  0 on success
+ */
+static int stmmac_init_phy(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int mode = priv->plat->phy_interface;
+	struct fwnode_handle *phy_fwnode;
+	struct fwnode_handle *fwnode;
+	struct ethtool_keee eee;
+	int ret;
+
+	if (!phylink_expects_phy(priv->phylink))
+		return 0;
+
+	if (priv->hw->xpcs &&
+	    xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73)
+		return 0;
+
+	fwnode = priv->plat->port_node;
+	if (!fwnode)
+		fwnode = dev_fwnode(priv->device);
+
+	if (fwnode)
+		phy_fwnode = fwnode_get_phy_node(fwnode);
+	else
+		phy_fwnode = NULL;
+
+	if (get_ecdev(priv)) {
+		rtnl_lock();
+	}
+
+	/* Some DT bindings do not set-up the PHY handle. Let's try to
+	 * manually parse it
+	 */
+	if (!phy_fwnode || IS_ERR(phy_fwnode)) {
+		int addr = priv->plat->phy_addr;
+		struct phy_device *phydev;
+
+		if (addr < 0) {
+			netdev_err(priv->dev, "no phy found\n");
+			return -ENODEV;
+		}
+
+		phydev = mdiobus_get_phy(priv->mii, addr);
+		if (!phydev) {
+			netdev_err(priv->dev, "no phy at addr %d\n", addr);
+			return -ENODEV;
+		}
+
+		ret = phylink_connect_phy(priv->phylink, phydev);
+	} else {
+		fwnode_handle_put(phy_fwnode);
+		ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0);
+	}
+
+	if (ret) {
+		netdev_err(priv->dev, "cannot attach to PHY (error: %pe)\n",
+			   ERR_PTR(ret));
+		return ret;
+	}
+
+	/* Configure phylib's copy of the LPI timer. Normally,
+	 * phylink_config.lpi_timer_default would do this, but there is a
+	 * chance that userspace could change the eee_timer setting via sysfs
+	 * before the first open. Thus, preserve existing behaviour.
+	 */
+	if (!phylink_ethtool_get_eee(priv->phylink, &eee)) {
+		eee.tx_lpi_timer = priv->tx_lpi_timer;
+		phylink_ethtool_set_eee(priv->phylink, &eee);
+	}
+
+	if (!get_ecdev(priv) && !priv->plat->pmt) {
+		struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
+
+		phylink_ethtool_get_wol(priv->phylink, &wol);
+		device_set_wakeup_capable(priv->device, !!wol.supported);
+		device_set_wakeup_enable(priv->device, !!wol.wolopts);
+	}
+
+	if (get_ecdev(priv)) {
+		rtnl_unlock();
+	}
+
+	return 0;
+}
+
+static int stmmac_phy_setup(struct stmmac_priv *priv)
+{
+	struct stmmac_mdio_bus_data *mdio_bus_data;
+	struct phylink_config *config;
+	struct fwnode_handle *fwnode;
+	struct phylink_pcs *pcs;
+	struct phylink *phylink;
+
+	config = &priv->phylink_config;
+
+	config->dev = &priv->dev->dev;
+	config->type = PHYLINK_NETDEV;
+	config->mac_managed_pm = true;
+
+	/* Stmmac always requires an RX clock for hardware initialization */
+	config->mac_requires_rxc = true;
+
+	if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI))
+		config->eee_rx_clk_stop_enable = true;
+
+	/* Set the default transmit clock stop bit based on the platform glue */
+	priv->tx_lpi_clk_stop = priv->plat->flags &
+				STMMAC_FLAG_EN_TX_LPI_CLOCKGATING;
+
+	mdio_bus_data = priv->plat->mdio_bus_data;
+	if (mdio_bus_data)
+		config->default_an_inband = mdio_bus_data->default_an_inband;
+
+	/* Get the PHY interface modes (at the PHY end of the link) that
+	 * are supported by the platform.
+	 */
+	if (priv->plat->get_interfaces)
+		priv->plat->get_interfaces(priv, priv->plat->bsp_priv,
+					   config->supported_interfaces);
+
+	/* Set the platform/firmware specified interface mode if the
+	 * supported interfaces have not already been provided using
+	 * phy_interface as a last resort.
+	 */
+	if (phy_interface_empty(config->supported_interfaces))
+		__set_bit(priv->plat->phy_interface,
+			  config->supported_interfaces);
+
+	/* If we have an xpcs, it defines which PHY interfaces are supported. */
+	if (priv->hw->xpcs)
+		pcs = xpcs_to_phylink_pcs(priv->hw->xpcs);
+	else
+		pcs = priv->hw->phylink_pcs;
+
+	if (pcs)
+		phy_interface_or(config->supported_interfaces,
+				 config->supported_interfaces,
+				 pcs->supported_interfaces);
+
+	if (priv->dma_cap.eee) {
+		/* Assume all supported interfaces also support LPI */
+		memcpy(config->lpi_interfaces, config->supported_interfaces,
+		       sizeof(config->lpi_interfaces));
+
+		/* All full duplex speeds above 100Mbps are supported */
+		config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD;
+		config->lpi_timer_default = eee_timer * 1000;
+		config->eee_enabled_default = true;
+	}
+
+	fwnode = priv->plat->port_node;
+	if (!fwnode)
+		fwnode = dev_fwnode(priv->device);
+
+	phylink = phylink_create(config, fwnode, priv->plat->phy_interface,
+				 &stmmac_phylink_mac_ops);
+	if (IS_ERR(phylink))
+		return PTR_ERR(phylink);
+
+	priv->phylink = phylink;
+	return 0;
+}
+
+static void stmmac_display_rx_rings(struct stmmac_priv *priv,
+				    struct stmmac_dma_conf *dma_conf)
+{
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	unsigned int desc_size;
+	void *head_rx;
+	u32 queue;
+
+	/* Display RX rings */
+	for (queue = 0; queue < rx_cnt; queue++) {
+		struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+
+		pr_info("\tRX Queue %u rings\n", queue);
+
+		if (priv->extend_desc) {
+			head_rx = (void *)rx_q->dma_erx;
+			desc_size = sizeof(struct dma_extended_desc);
+		} else {
+			head_rx = (void *)rx_q->dma_rx;
+			desc_size = sizeof(struct dma_desc);
+		}
+
+		/* Display RX ring */
+		stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true,
+				    rx_q->dma_rx_phy, desc_size);
+	}
+}
+
+static void stmmac_display_tx_rings(struct stmmac_priv *priv,
+				    struct stmmac_dma_conf *dma_conf)
+{
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	unsigned int desc_size;
+	void *head_tx;
+	u32 queue;
+
+	/* Display TX rings */
+	for (queue = 0; queue < tx_cnt; queue++) {
+		struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+
+		pr_info("\tTX Queue %d rings\n", queue);
+
+		if (priv->extend_desc) {
+			head_tx = (void *)tx_q->dma_etx;
+			desc_size = sizeof(struct dma_extended_desc);
+		} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
+			head_tx = (void *)tx_q->dma_entx;
+			desc_size = sizeof(struct dma_edesc);
+		} else {
+			head_tx = (void *)tx_q->dma_tx;
+			desc_size = sizeof(struct dma_desc);
+		}
+
+		stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false,
+				    tx_q->dma_tx_phy, desc_size);
+	}
+}
+
+static void stmmac_display_rings(struct stmmac_priv *priv,
+				 struct stmmac_dma_conf *dma_conf)
+{
+	/* Display RX ring */
+	stmmac_display_rx_rings(priv, dma_conf);
+
+	/* Display TX ring */
+	stmmac_display_tx_rings(priv, dma_conf);
+}
+
+static unsigned int stmmac_rx_offset(struct stmmac_priv *priv)
+{
+	if (stmmac_xdp_is_enabled(priv))
+		return XDP_PACKET_HEADROOM;
+
+	return NET_SKB_PAD;
+}
+
+static int stmmac_set_bfsize(int mtu, int bufsize)
+{
+	int ret = bufsize;
+
+	if (mtu >= BUF_SIZE_8KiB)
+		ret = BUF_SIZE_16KiB;
+	else if (mtu >= BUF_SIZE_4KiB)
+		ret = BUF_SIZE_8KiB;
+	else if (mtu >= BUF_SIZE_2KiB)
+		ret = BUF_SIZE_4KiB;
+	else if (mtu > DEFAULT_BUFSIZE)
+		ret = BUF_SIZE_2KiB;
+	else
+		ret = DEFAULT_BUFSIZE;
+
+	return ret;
+}
+
+/**
+ * stmmac_clear_rx_descriptors - clear RX descriptors
+ * @priv: driver private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: RX queue index
+ * Description: this function is called to clear the RX descriptors
+ * in case of both basic and extended descriptors are used.
+ */
+static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv,
+					struct stmmac_dma_conf *dma_conf,
+					u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	int i;
+
+	/* Clear the RX descriptors */
+	for (i = 0; i < dma_conf->dma_rx_size; i++)
+		if (priv->extend_desc)
+			stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
+					priv->use_riwt, priv->mode,
+					(i == dma_conf->dma_rx_size - 1),
+					dma_conf->dma_buf_sz);
+		else
+			stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
+					priv->use_riwt, priv->mode,
+					(i == dma_conf->dma_rx_size - 1),
+					dma_conf->dma_buf_sz);
+}
+
+/**
+ * stmmac_clear_tx_descriptors - clear tx descriptors
+ * @priv: driver private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: TX queue index.
+ * Description: this function is called to clear the TX descriptors
+ * in case of both basic and extended descriptors are used.
+ */
+static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv,
+					struct stmmac_dma_conf *dma_conf,
+					u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+	int i;
+
+	/* Clear the TX descriptors */
+	for (i = 0; i < dma_conf->dma_tx_size; i++) {
+		int last = (i == (dma_conf->dma_tx_size - 1));
+		struct dma_desc *p;
+
+		if (priv->extend_desc)
+			p = &tx_q->dma_etx[i].basic;
+		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			p = &tx_q->dma_entx[i].basic;
+		else
+			p = &tx_q->dma_tx[i];
+
+		stmmac_init_tx_desc(priv, p, priv->mode, last);
+	}
+}
+
+/**
+ * stmmac_clear_descriptors - clear descriptors
+ * @priv: driver private structure
+ * @dma_conf: structure to take the dma data
+ * Description: this function is called to clear the TX and RX descriptors
+ * in case of both basic and extended descriptors are used.
+ */
+static void stmmac_clear_descriptors(struct stmmac_priv *priv,
+				     struct stmmac_dma_conf *dma_conf)
+{
+	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
+	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
+	u32 queue;
+
+	/* Clear the RX descriptors */
+	for (queue = 0; queue < rx_queue_cnt; queue++)
+		stmmac_clear_rx_descriptors(priv, dma_conf, queue);
+
+	/* Clear the TX descriptors */
+	for (queue = 0; queue < tx_queue_cnt; queue++)
+		stmmac_clear_tx_descriptors(priv, dma_conf, queue);
+}
+
+/**
+ * stmmac_init_rx_buffers - init the RX descriptor buffer.
+ * @priv: driver private structure
+ * @dma_conf: structure to take the dma data
+ * @p: descriptor pointer
+ * @i: descriptor index
+ * @flags: gfp flag
+ * @queue: RX queue index
+ * Description: this function is called to allocate a receive buffer, perform
+ * the DMA mapping and init the descriptor.
+ */
+static int stmmac_init_rx_buffers(struct stmmac_priv *priv,
+				  struct stmmac_dma_conf *dma_conf,
+				  struct dma_desc *p,
+				  int i, gfp_t flags, u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
+	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
+
+	if (priv->dma_cap.host_dma_width <= 32)
+		gfp |= GFP_DMA32;
+
+	if (!buf->page) {
+		buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp);
+		if (!buf->page)
+			return -ENOMEM;
+		buf->page_offset = stmmac_rx_offset(priv);
+	}
+
+	if (priv->sph && !buf->sec_page) {
+		buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
+		if (!buf->sec_page)
+			return -ENOMEM;
+
+		buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
+		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
+	} else {
+		buf->sec_page = NULL;
+		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
+	}
+
+	buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
+
+	stmmac_set_desc_addr(priv, p, buf->addr);
+	if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB)
+		stmmac_init_desc3(priv, p);
+
+	return 0;
+}
+
+/**
+ * stmmac_free_rx_buffer - free RX dma buffers
+ * @priv: private structure
+ * @rx_q: RX queue
+ * @i: buffer index.
+ */
+static void stmmac_free_rx_buffer(struct stmmac_priv *priv,
+				  struct stmmac_rx_queue *rx_q,
+				  int i)
+{
+	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
+
+	if (buf->page)
+		page_pool_put_full_page(rx_q->page_pool, buf->page, false);
+	buf->page = NULL;
+
+	if (buf->sec_page)
+		page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false);
+	buf->sec_page = NULL;
+}
+
+/**
+ * stmmac_free_tx_buffer - free RX dma buffers
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: RX queue index
+ * @i: buffer index.
+ */
+static void stmmac_free_tx_buffer(struct stmmac_priv *priv,
+				  struct stmmac_dma_conf *dma_conf,
+				  u32 queue, int i)
+{
+	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+
+	if (tx_q->tx_skbuff_dma[i].buf &&
+	    tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) {
+		if (tx_q->tx_skbuff_dma[i].map_as_page)
+			dma_unmap_page(priv->device,
+				       tx_q->tx_skbuff_dma[i].buf,
+				       tx_q->tx_skbuff_dma[i].len,
+				       DMA_TO_DEVICE);
+		else
+			dma_unmap_single(priv->device,
+					 tx_q->tx_skbuff_dma[i].buf,
+					 tx_q->tx_skbuff_dma[i].len,
+					 DMA_TO_DEVICE);
+	}
+
+	if (tx_q->xdpf[i] &&
+	    (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX ||
+	     tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) {
+		xdp_return_frame(tx_q->xdpf[i]);
+		tx_q->xdpf[i] = NULL;
+	}
+
+	if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX)
+		tx_q->xsk_frames_done++;
+
+	if (!get_ecdev(priv) &&
+		tx_q->tx_skbuff[i] &&
+	    tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) {
+		dev_kfree_skb_any(tx_q->tx_skbuff[i]);
+		tx_q->tx_skbuff[i] = NULL;
+	}
+
+	tx_q->tx_skbuff_dma[i].buf = 0;
+	tx_q->tx_skbuff_dma[i].map_as_page = false;
+}
+
+/**
+ * dma_free_rx_skbufs - free RX dma buffers
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: RX queue index
+ */
+static void dma_free_rx_skbufs(struct stmmac_priv *priv,
+			       struct stmmac_dma_conf *dma_conf,
+			       u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	int i;
+
+	for (i = 0; i < dma_conf->dma_rx_size; i++)
+		stmmac_free_rx_buffer(priv, rx_q, i);
+}
+
+static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv,
+				   struct stmmac_dma_conf *dma_conf,
+				   u32 queue, gfp_t flags)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	int i;
+
+	for (i = 0; i < dma_conf->dma_rx_size; i++) {
+		struct dma_desc *p;
+		int ret;
+
+		if (priv->extend_desc)
+			p = &((rx_q->dma_erx + i)->basic);
+		else
+			p = rx_q->dma_rx + i;
+
+		ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags,
+					     queue);
+		if (ret)
+			return ret;
+
+		rx_q->buf_alloc_num++;
+	}
+
+	return 0;
+}
+
+/**
+ * dma_free_rx_xskbufs - free RX dma buffers from XSK pool
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: RX queue index
+ */
+static void dma_free_rx_xskbufs(struct stmmac_priv *priv,
+				struct stmmac_dma_conf *dma_conf,
+				u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	int i;
+
+	for (i = 0; i < dma_conf->dma_rx_size; i++) {
+		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
+
+		if (!buf->xdp)
+			continue;
+
+		xsk_buff_free(buf->xdp);
+		buf->xdp = NULL;
+	}
+}
+
+static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv,
+				      struct stmmac_dma_conf *dma_conf,
+				      u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	int i;
+
+	/* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes)
+	 * in struct xdp_buff_xsk to stash driver specific information. Thus,
+	 * use this macro to make sure no size violations.
+	 */
+	XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff);
+
+	for (i = 0; i < dma_conf->dma_rx_size; i++) {
+		struct stmmac_rx_buffer *buf;
+		dma_addr_t dma_addr;
+		struct dma_desc *p;
+
+		if (priv->extend_desc)
+			p = (struct dma_desc *)(rx_q->dma_erx + i);
+		else
+			p = rx_q->dma_rx + i;
+
+		buf = &rx_q->buf_pool[i];
+
+		buf->xdp = xsk_buff_alloc(rx_q->xsk_pool);
+		if (!buf->xdp)
+			return -ENOMEM;
+
+		dma_addr = xsk_buff_xdp_get_dma(buf->xdp);
+		stmmac_set_desc_addr(priv, p, dma_addr);
+		rx_q->buf_alloc_num++;
+	}
+
+	return 0;
+}
+
+static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue)
+{
+	if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps))
+		return NULL;
+
+	return xsk_get_pool_from_qid(priv->dev, queue);
+}
+
+/**
+ * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue)
+ * @priv: driver private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: RX queue index
+ * @flags: gfp flag.
+ * Description: this function initializes the DMA RX descriptors
+ * and allocates the socket buffers. It supports the chained and ring
+ * modes.
+ */
+static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,
+				    struct stmmac_dma_conf *dma_conf,
+				    u32 queue, gfp_t flags)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	int ret;
+
+	netif_dbg(priv, probe, priv->dev,
+		  "(%s) dma_rx_phy=0x%08x\n", __func__,
+		  (u32)rx_q->dma_rx_phy);
+
+	stmmac_clear_rx_descriptors(priv, dma_conf, queue);
+
+	xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq);
+
+	rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
+
+	if (rx_q->xsk_pool) {
+		WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
+						   MEM_TYPE_XSK_BUFF_POOL,
+						   NULL));
+		netdev_info(priv->dev,
+			    "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n",
+			    rx_q->queue_index);
+		xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq);
+	} else {
+		WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
+						   MEM_TYPE_PAGE_POOL,
+						   rx_q->page_pool));
+		netdev_info(priv->dev,
+			    "Register MEM_TYPE_PAGE_POOL RxQ-%d\n",
+			    rx_q->queue_index);
+	}
+
+	if (rx_q->xsk_pool) {
+		/* RX XDP ZC buffer pool may not be populated, e.g.
+		 * xdpsock TX-only.
+		 */
+		stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue);
+	} else {
+		ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags);
+		if (ret < 0)
+			return -ENOMEM;
+	}
+
+	/* Setup the chained descriptor addresses */
+	if (priv->mode == STMMAC_CHAIN_MODE) {
+		if (priv->extend_desc)
+			stmmac_mode_init(priv, rx_q->dma_erx,
+					 rx_q->dma_rx_phy,
+					 dma_conf->dma_rx_size, 1);
+		else
+			stmmac_mode_init(priv, rx_q->dma_rx,
+					 rx_q->dma_rx_phy,
+					 dma_conf->dma_rx_size, 0);
+	}
+
+	return 0;
+}
+
+static int init_dma_rx_desc_rings(struct net_device *dev,
+				  struct stmmac_dma_conf *dma_conf,
+				  gfp_t flags)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 rx_count = priv->plat->rx_queues_to_use;
+	int queue;
+	int ret;
+
+	/* RX INITIALIZATION */
+	netif_dbg(priv, probe, priv->dev,
+		  "SKB addresses:\nskb\t\tskb data\tdma data\n");
+
+	for (queue = 0; queue < rx_count; queue++) {
+		ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags);
+		if (ret)
+			goto err_init_rx_buffers;
+	}
+
+	return 0;
+
+err_init_rx_buffers:
+	while (queue >= 0) {
+		struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+
+		if (rx_q->xsk_pool)
+			dma_free_rx_xskbufs(priv, dma_conf, queue);
+		else
+			dma_free_rx_skbufs(priv, dma_conf, queue);
+
+		rx_q->buf_alloc_num = 0;
+		rx_q->xsk_pool = NULL;
+
+		queue--;
+	}
+
+	return ret;
+}
+
+/**
+ * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue)
+ * @priv: driver private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: TX queue index
+ * Description: this function initializes the DMA TX descriptors
+ * and allocates the socket buffers. It supports the chained and ring
+ * modes.
+ */
+static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,
+				    struct stmmac_dma_conf *dma_conf,
+				    u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+	int i;
+
+	netif_dbg(priv, probe, priv->dev,
+		  "(%s) dma_tx_phy=0x%08x\n", __func__,
+		  (u32)tx_q->dma_tx_phy);
+
+	/* Setup the chained descriptor addresses */
+	if (priv->mode == STMMAC_CHAIN_MODE) {
+		if (priv->extend_desc)
+			stmmac_mode_init(priv, tx_q->dma_etx,
+					 tx_q->dma_tx_phy,
+					 dma_conf->dma_tx_size, 1);
+		else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
+			stmmac_mode_init(priv, tx_q->dma_tx,
+					 tx_q->dma_tx_phy,
+					 dma_conf->dma_tx_size, 0);
+	}
+
+	tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
+
+	for (i = 0; i < dma_conf->dma_tx_size; i++) {
+		struct dma_desc *p;
+
+		if (priv->extend_desc)
+			p = &((tx_q->dma_etx + i)->basic);
+		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			p = &((tx_q->dma_entx + i)->basic);
+		else
+			p = tx_q->dma_tx + i;
+
+		stmmac_clear_desc(priv, p);
+
+		tx_q->tx_skbuff_dma[i].buf = 0;
+		tx_q->tx_skbuff_dma[i].map_as_page = false;
+		tx_q->tx_skbuff_dma[i].len = 0;
+		tx_q->tx_skbuff_dma[i].last_segment = false;
+		tx_q->tx_skbuff[i] = NULL;
+	}
+
+	return 0;
+}
+
+static int init_dma_tx_desc_rings(struct net_device *dev,
+				  struct stmmac_dma_conf *dma_conf)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 tx_queue_cnt;
+	u32 queue;
+
+	tx_queue_cnt = priv->plat->tx_queues_to_use;
+
+	for (queue = 0; queue < tx_queue_cnt; queue++)
+		__init_dma_tx_desc_rings(priv, dma_conf, queue);
+
+	return 0;
+}
+
+/**
+ * init_dma_desc_rings - init the RX/TX descriptor rings
+ * @dev: net device structure
+ * @dma_conf: structure to take the dma data
+ * @flags: gfp flag.
+ * Description: this function initializes the DMA RX/TX descriptors
+ * and allocates the socket buffers. It supports the chained and ring
+ * modes.
+ */
+static int init_dma_desc_rings(struct net_device *dev,
+			       struct stmmac_dma_conf *dma_conf,
+			       gfp_t flags)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int ret;
+
+	ret = init_dma_rx_desc_rings(dev, dma_conf, flags);
+	if (ret)
+		return ret;
+
+	ret = init_dma_tx_desc_rings(dev, dma_conf);
+
+	stmmac_clear_descriptors(priv, dma_conf);
+
+	if (netif_msg_hw(priv))
+		stmmac_display_rings(priv, dma_conf);
+
+	return ret;
+}
+
+/**
+ * dma_free_tx_skbufs - free TX dma buffers
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: TX queue index
+ */
+static void dma_free_tx_skbufs(struct stmmac_priv *priv,
+			       struct stmmac_dma_conf *dma_conf,
+			       u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+	int i;
+
+	tx_q->xsk_frames_done = 0;
+
+	for (i = 0; i < dma_conf->dma_tx_size; i++)
+		stmmac_free_tx_buffer(priv, dma_conf, queue, i);
+
+	if (tx_q->xsk_pool && tx_q->xsk_frames_done) {
+		xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done);
+		tx_q->xsk_frames_done = 0;
+		tx_q->xsk_pool = NULL;
+	}
+}
+
+/**
+ * stmmac_free_tx_skbufs - free TX skb buffers
+ * @priv: private structure
+ */
+static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)
+{
+	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
+	u32 queue;
+
+	for (queue = 0; queue < tx_queue_cnt; queue++)
+		dma_free_tx_skbufs(priv, &priv->dma_conf, queue);
+}
+
+/**
+ * __free_dma_rx_desc_resources - free RX dma desc resources (per queue)
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: RX queue index
+ */
+static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,
+					 struct stmmac_dma_conf *dma_conf,
+					 u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+
+	/* Release the DMA RX socket buffers */
+	if (rx_q->xsk_pool)
+		dma_free_rx_xskbufs(priv, dma_conf, queue);
+	else
+		dma_free_rx_skbufs(priv, dma_conf, queue);
+
+	rx_q->buf_alloc_num = 0;
+	rx_q->xsk_pool = NULL;
+
+	/* Free DMA regions of consistent memory previously allocated */
+	if (!priv->extend_desc)
+		dma_free_coherent(priv->device, dma_conf->dma_rx_size *
+				  sizeof(struct dma_desc),
+				  rx_q->dma_rx, rx_q->dma_rx_phy);
+	else
+		dma_free_coherent(priv->device, dma_conf->dma_rx_size *
+				  sizeof(struct dma_extended_desc),
+				  rx_q->dma_erx, rx_q->dma_rx_phy);
+
+	if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq))
+		xdp_rxq_info_unreg(&rx_q->xdp_rxq);
+
+	kfree(rx_q->buf_pool);
+	if (rx_q->page_pool)
+		page_pool_destroy(rx_q->page_pool);
+}
+
+static void free_dma_rx_desc_resources(struct stmmac_priv *priv,
+				       struct stmmac_dma_conf *dma_conf)
+{
+	u32 rx_count = priv->plat->rx_queues_to_use;
+	u32 queue;
+
+	/* Free RX queue resources */
+	for (queue = 0; queue < rx_count; queue++)
+		__free_dma_rx_desc_resources(priv, dma_conf, queue);
+}
+
+/**
+ * __free_dma_tx_desc_resources - free TX dma desc resources (per queue)
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: TX queue index
+ */
+static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,
+					 struct stmmac_dma_conf *dma_conf,
+					 u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+	size_t size;
+	void *addr;
+
+	/* Release the DMA TX socket buffers */
+	dma_free_tx_skbufs(priv, dma_conf, queue);
+
+	if (priv->extend_desc) {
+		size = sizeof(struct dma_extended_desc);
+		addr = tx_q->dma_etx;
+	} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
+		size = sizeof(struct dma_edesc);
+		addr = tx_q->dma_entx;
+	} else {
+		size = sizeof(struct dma_desc);
+		addr = tx_q->dma_tx;
+	}
+
+	size *= dma_conf->dma_tx_size;
+
+	dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
+
+	kfree(tx_q->tx_skbuff_dma);
+	kfree(tx_q->tx_skbuff);
+}
+
+static void free_dma_tx_desc_resources(struct stmmac_priv *priv,
+				       struct stmmac_dma_conf *dma_conf)
+{
+	u32 tx_count = priv->plat->tx_queues_to_use;
+	u32 queue;
+
+	/* Free TX queue resources */
+	for (queue = 0; queue < tx_count; queue++)
+		__free_dma_tx_desc_resources(priv, dma_conf, queue);
+}
+
+/**
+ * __alloc_dma_rx_desc_resources - alloc RX resources (per queue).
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: RX queue index
+ * Description: according to which descriptor can be used (extend or basic)
+ * this function allocates the resources for TX and RX paths. In case of
+ * reception, for example, it pre-allocated the RX socket buffer in order to
+ * allow zero-copy mechanism.
+ */
+static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
+					 struct stmmac_dma_conf *dma_conf,
+					 u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
+	struct stmmac_channel *ch = &priv->channel[queue];
+	bool xdp_prog = stmmac_xdp_is_enabled(priv);
+	struct page_pool_params pp_params = { 0 };
+	unsigned int dma_buf_sz_pad, num_pages;
+	unsigned int napi_id;
+	int ret;
+
+	dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz +
+			 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE);
+
+	rx_q->queue_index = queue;
+	rx_q->priv_data = priv;
+	rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE;
+
+	pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
+	pp_params.pool_size = dma_conf->dma_rx_size;
+	pp_params.order = order_base_2(num_pages);
+	pp_params.nid = dev_to_node(priv->device);
+	pp_params.dev = priv->device;
+	pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
+	pp_params.offset = stmmac_rx_offset(priv);
+	pp_params.max_len = dma_conf->dma_buf_sz;
+
+	if (priv->sph) {
+		pp_params.offset = 0;
+		pp_params.max_len += stmmac_rx_offset(priv);
+	}
+
+	rx_q->page_pool = page_pool_create(&pp_params);
+	if (IS_ERR(rx_q->page_pool)) {
+		ret = PTR_ERR(rx_q->page_pool);
+		rx_q->page_pool = NULL;
+		return ret;
+	}
+
+	rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size,
+				 sizeof(*rx_q->buf_pool),
+				 GFP_KERNEL);
+	if (!rx_q->buf_pool)
+		return -ENOMEM;
+
+	if (priv->extend_desc) {
+		rx_q->dma_erx = dma_alloc_coherent(priv->device,
+						   dma_conf->dma_rx_size *
+						   sizeof(struct dma_extended_desc),
+						   &rx_q->dma_rx_phy,
+						   GFP_KERNEL);
+		if (!rx_q->dma_erx)
+			return -ENOMEM;
+
+	} else {
+		rx_q->dma_rx = dma_alloc_coherent(priv->device,
+						  dma_conf->dma_rx_size *
+						  sizeof(struct dma_desc),
+						  &rx_q->dma_rx_phy,
+						  GFP_KERNEL);
+		if (!rx_q->dma_rx)
+			return -ENOMEM;
+	}
+
+	if (stmmac_xdp_is_enabled(priv) &&
+	    test_bit(queue, priv->af_xdp_zc_qps))
+		napi_id = ch->rxtx_napi.napi_id;
+	else
+		napi_id = ch->rx_napi.napi_id;
+
+	ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev,
+			       rx_q->queue_index,
+			       napi_id);
+	if (ret) {
+		netdev_err(priv->dev, "Failed to register xdp rxq info\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
+				       struct stmmac_dma_conf *dma_conf)
+{
+	u32 rx_count = priv->plat->rx_queues_to_use;
+	u32 queue;
+	int ret;
+
+	/* RX queues buffers and DMA */
+	for (queue = 0; queue < rx_count; queue++) {
+		ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue);
+		if (ret)
+			goto err_dma;
+	}
+
+	return 0;
+
+err_dma:
+	free_dma_rx_desc_resources(priv, dma_conf);
+
+	return ret;
+}
+
+/**
+ * __alloc_dma_tx_desc_resources - alloc TX resources (per queue).
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * @queue: TX queue index
+ * Description: according to which descriptor can be used (extend or basic)
+ * this function allocates the resources for TX and RX paths. In case of
+ * reception, for example, it pre-allocated the RX socket buffer in order to
+ * allow zero-copy mechanism.
+ */
+static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
+					 struct stmmac_dma_conf *dma_conf,
+					 u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
+	size_t size;
+	void *addr;
+
+	tx_q->queue_index = queue;
+	tx_q->priv_data = priv;
+
+	tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size,
+				      sizeof(*tx_q->tx_skbuff_dma),
+				      GFP_KERNEL);
+	if (!tx_q->tx_skbuff_dma)
+		return -ENOMEM;
+
+	tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size,
+				  sizeof(struct sk_buff *),
+				  GFP_KERNEL);
+	if (!tx_q->tx_skbuff)
+		return -ENOMEM;
+
+	if (priv->extend_desc)
+		size = sizeof(struct dma_extended_desc);
+	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		size = sizeof(struct dma_edesc);
+	else
+		size = sizeof(struct dma_desc);
+
+	size *= dma_conf->dma_tx_size;
+
+	addr = dma_alloc_coherent(priv->device, size,
+				  &tx_q->dma_tx_phy, GFP_KERNEL);
+	if (!addr)
+		return -ENOMEM;
+
+	if (priv->extend_desc)
+		tx_q->dma_etx = addr;
+	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		tx_q->dma_entx = addr;
+	else
+		tx_q->dma_tx = addr;
+
+	return 0;
+}
+
+static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
+				       struct stmmac_dma_conf *dma_conf)
+{
+	u32 tx_count = priv->plat->tx_queues_to_use;
+	u32 queue;
+	int ret;
+
+	/* TX queues buffers and DMA */
+	for (queue = 0; queue < tx_count; queue++) {
+		ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue);
+		if (ret)
+			goto err_dma;
+	}
+
+	return 0;
+
+err_dma:
+	free_dma_tx_desc_resources(priv, dma_conf);
+	return ret;
+}
+
+/**
+ * alloc_dma_desc_resources - alloc TX/RX resources.
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ * Description: according to which descriptor can be used (extend or basic)
+ * this function allocates the resources for TX and RX paths. In case of
+ * reception, for example, it pre-allocated the RX socket buffer in order to
+ * allow zero-copy mechanism.
+ */
+static int alloc_dma_desc_resources(struct stmmac_priv *priv,
+				    struct stmmac_dma_conf *dma_conf)
+{
+	/* RX Allocation */
+	int ret = alloc_dma_rx_desc_resources(priv, dma_conf);
+
+	if (ret)
+		return ret;
+
+	ret = alloc_dma_tx_desc_resources(priv, dma_conf);
+
+	return ret;
+}
+
+/**
+ * free_dma_desc_resources - free dma desc resources
+ * @priv: private structure
+ * @dma_conf: structure to take the dma data
+ */
+static void free_dma_desc_resources(struct stmmac_priv *priv,
+				    struct stmmac_dma_conf *dma_conf)
+{
+	/* Release the DMA TX socket buffers */
+	free_dma_tx_desc_resources(priv, dma_conf);
+
+	/* Release the DMA RX socket buffers later
+	 * to ensure all pending XDP_TX buffers are returned.
+	 */
+	free_dma_rx_desc_resources(priv, dma_conf);
+}
+
+/**
+ *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
+ *  @priv: driver private structure
+ *  Description: It is used for enabling the rx queues in the MAC
+ */
+static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
+{
+	u32 rx_queues_count = priv->plat->rx_queues_to_use;
+	int queue;
+	u8 mode;
+
+	for (queue = 0; queue < rx_queues_count; queue++) {
+		mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
+		stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
+	}
+}
+
+/**
+ * stmmac_start_rx_dma - start RX DMA channel
+ * @priv: driver private structure
+ * @chan: RX channel index
+ * Description:
+ * This starts a RX DMA channel
+ */
+static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
+{
+	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
+	stmmac_start_rx(priv, priv->ioaddr, chan);
+}
+
+/**
+ * stmmac_start_tx_dma - start TX DMA channel
+ * @priv: driver private structure
+ * @chan: TX channel index
+ * Description:
+ * This starts a TX DMA channel
+ */
+static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
+{
+	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
+	stmmac_start_tx(priv, priv->ioaddr, chan);
+}
+
+/**
+ * stmmac_stop_rx_dma - stop RX DMA channel
+ * @priv: driver private structure
+ * @chan: RX channel index
+ * Description:
+ * This stops a RX DMA channel
+ */
+static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
+{
+	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
+	stmmac_stop_rx(priv, priv->ioaddr, chan);
+}
+
+/**
+ * stmmac_stop_tx_dma - stop TX DMA channel
+ * @priv: driver private structure
+ * @chan: TX channel index
+ * Description:
+ * This stops a TX DMA channel
+ */
+static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
+{
+	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
+	stmmac_stop_tx(priv, priv->ioaddr, chan);
+}
+
+static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)
+{
+	u32 rx_channels_count = priv->plat->rx_queues_to_use;
+	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
+	u32 chan;
+
+	if (get_ecdev(priv))
+		return;
+
+	for (chan = 0; chan < dma_csr_ch; chan++) {
+		struct stmmac_channel *ch = &priv->channel[chan];
+		unsigned long flags;
+
+		spin_lock_irqsave(&ch->lock, flags);
+		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
+		spin_unlock_irqrestore(&ch->lock, flags);
+	}
+}
+
+/**
+ * stmmac_start_all_dma - start all RX and TX DMA channels
+ * @priv: driver private structure
+ * Description:
+ * This starts all the RX and TX DMA channels
+ */
+static void stmmac_start_all_dma(struct stmmac_priv *priv)
+{
+	u32 rx_channels_count = priv->plat->rx_queues_to_use;
+	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u32 chan = 0;
+
+	for (chan = 0; chan < rx_channels_count; chan++)
+		stmmac_start_rx_dma(priv, chan);
+
+	for (chan = 0; chan < tx_channels_count; chan++)
+		stmmac_start_tx_dma(priv, chan);
+}
+
+/**
+ * stmmac_stop_all_dma - stop all RX and TX DMA channels
+ * @priv: driver private structure
+ * Description:
+ * This stops the RX and TX DMA channels
+ */
+static void stmmac_stop_all_dma(struct stmmac_priv *priv)
+{
+	u32 rx_channels_count = priv->plat->rx_queues_to_use;
+	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u32 chan = 0;
+
+	for (chan = 0; chan < rx_channels_count; chan++)
+		stmmac_stop_rx_dma(priv, chan);
+
+	for (chan = 0; chan < tx_channels_count; chan++)
+		stmmac_stop_tx_dma(priv, chan);
+}
+
+/**
+ *  stmmac_dma_operation_mode - HW DMA operation mode
+ *  @priv: driver private structure
+ *  Description: it is used for configuring the DMA operation mode register in
+ *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
+ */
+static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
+{
+	u32 rx_channels_count = priv->plat->rx_queues_to_use;
+	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	int rxfifosz = priv->plat->rx_fifo_size;
+	int txfifosz = priv->plat->tx_fifo_size;
+	u32 txmode = 0;
+	u32 rxmode = 0;
+	u32 chan = 0;
+	u8 qmode = 0;
+
+	if (rxfifosz == 0)
+		rxfifosz = priv->dma_cap.rx_fifo_size;
+	if (txfifosz == 0)
+		txfifosz = priv->dma_cap.tx_fifo_size;
+
+	/* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */
+	if (priv->plat->has_gmac4 || priv->plat->has_xgmac) {
+		rxfifosz /= rx_channels_count;
+		txfifosz /= tx_channels_count;
+	}
+
+	if (priv->plat->force_thresh_dma_mode) {
+		txmode = tc;
+		rxmode = tc;
+	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
+		/*
+		 * In case of GMAC, SF mode can be enabled
+		 * to perform the TX COE in HW. This depends on:
+		 * 1) TX COE if actually supported
+		 * 2) There is no bugged Jumbo frame support
+		 *    that needs to not insert csum in the TDES.
+		 */
+		txmode = SF_DMA_MODE;
+		rxmode = SF_DMA_MODE;
+		priv->xstats.threshold = SF_DMA_MODE;
+	} else {
+		txmode = tc;
+		rxmode = SF_DMA_MODE;
+	}
+
+	/* configure all channels */
+	for (chan = 0; chan < rx_channels_count; chan++) {
+		struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];
+		u32 buf_size;
+
+		qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
+
+		stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
+				rxfifosz, qmode);
+
+		if (rx_q->xsk_pool) {
+			buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
+			stmmac_set_dma_bfsize(priv, priv->ioaddr,
+					      buf_size,
+					      chan);
+		} else {
+			stmmac_set_dma_bfsize(priv, priv->ioaddr,
+					      priv->dma_conf.dma_buf_sz,
+					      chan);
+		}
+	}
+
+	for (chan = 0; chan < tx_channels_count; chan++) {
+		qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
+
+		stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
+				txfifosz, qmode);
+	}
+}
+
+static void stmmac_xsk_request_timestamp(void *_priv)
+{
+	struct stmmac_metadata_request *meta_req = _priv;
+
+	stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc);
+	*meta_req->set_ic = true;
+}
+
+static u64 stmmac_xsk_fill_timestamp(void *_priv)
+{
+	struct stmmac_xsk_tx_complete *tx_compl = _priv;
+	struct stmmac_priv *priv = tx_compl->priv;
+	struct dma_desc *desc = tx_compl->desc;
+	bool found = false;
+	u64 ns = 0;
+
+	if (!priv->hwts_tx_en)
+		return 0;
+
+	/* check tx tstamp status */
+	if (stmmac_get_tx_timestamp_status(priv, desc)) {
+		stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
+		found = true;
+	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
+		found = true;
+	}
+
+	if (found) {
+		ns -= priv->plat->cdc_error_adj;
+		return ns_to_ktime(ns);
+	}
+
+	return 0;
+}
+
+static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv)
+{
+	struct timespec64 ts = ns_to_timespec64(launch_time);
+	struct stmmac_metadata_request *meta_req = _priv;
+
+	if (meta_req->tbs & STMMAC_TBS_EN)
+		stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec,
+				    ts.tv_nsec);
+}
+
+static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = {
+	.tmo_request_timestamp		= stmmac_xsk_request_timestamp,
+	.tmo_fill_timestamp		= stmmac_xsk_fill_timestamp,
+	.tmo_request_launch_time	= stmmac_xsk_request_launch_time,
+};
+
+static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
+{
+	struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue);
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];
+	bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported;
+	struct xsk_buff_pool *pool = tx_q->xsk_pool;
+	unsigned int entry = tx_q->cur_tx;
+	struct dma_desc *tx_desc = NULL;
+	struct xdp_desc xdp_desc;
+	bool work_done = true;
+	u32 tx_set_ic_bit = 0;
+
+	/* Avoids TX time-out as we are sharing with slow path */
+	ec_txq_trans_cond_update(nq);
+
+	budget = min(budget, stmmac_tx_avail(priv, queue));
+
+	for (; budget > 0; budget--) {
+		struct stmmac_metadata_request meta_req;
+		struct xsk_tx_metadata *meta = NULL;
+		dma_addr_t dma_addr;
+		bool set_ic;
+
+		/* We are sharing with slow path and stop XSK TX desc submission when
+		 * available TX ring is less than threshold.
+		 */
+		if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) ||
+		    !netif_carrier_ok(priv->dev)) {
+			work_done = false;
+			break;
+		}
+
+		if (!xsk_tx_peek_desc(pool, &xdp_desc))
+			break;
+
+		if (priv->est && priv->est->enable &&
+		    priv->est->max_sdu[queue] &&
+		    xdp_desc.len > priv->est->max_sdu[queue]) {
+			priv->xstats.max_sdu_txq_drop[queue]++;
+			continue;
+		}
+
+		if (likely(priv->extend_desc))
+			tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
+		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			tx_desc = &tx_q->dma_entx[entry].basic;
+		else
+			tx_desc = tx_q->dma_tx + entry;
+
+		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
+		meta = xsk_buff_get_metadata(pool, xdp_desc.addr);
+		xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len);
+
+		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX;
+
+		/* To return XDP buffer to XSK pool, we simple call
+		 * xsk_tx_completed(), so we don't need to fill up
+		 * 'buf' and 'xdpf'.
+		 */
+		tx_q->tx_skbuff_dma[entry].buf = 0;
+		tx_q->xdpf[entry] = NULL;
+
+		tx_q->tx_skbuff_dma[entry].map_as_page = false;
+		tx_q->tx_skbuff_dma[entry].len = xdp_desc.len;
+		tx_q->tx_skbuff_dma[entry].last_segment = true;
+		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
+
+		stmmac_set_desc_addr(priv, tx_desc, dma_addr);
+
+		tx_q->tx_count_frames++;
+
+		if (!priv->tx_coal_frames[queue])
+			set_ic = false;
+		else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
+			set_ic = true;
+		else
+			set_ic = false;
+
+		meta_req.priv = priv;
+		meta_req.tx_desc = tx_desc;
+		meta_req.set_ic = &set_ic;
+		meta_req.tbs = tx_q->tbs;
+		meta_req.edesc = &tx_q->dma_entx[entry];
+		xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops,
+					&meta_req);
+		if (set_ic) {
+			tx_q->tx_count_frames = 0;
+			stmmac_set_tx_ic(priv, tx_desc);
+			tx_set_ic_bit++;
+		}
+
+		stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len,
+				       csum, priv->mode, true, true,
+				       xdp_desc.len);
+
+		stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
+
+		xsk_tx_metadata_to_compl(meta,
+					 &tx_q->tx_skbuff_dma[entry].xsk_meta);
+
+		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
+		entry = tx_q->cur_tx;
+	}
+	u64_stats_update_begin(&txq_stats->napi_syncp);
+	u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit);
+	u64_stats_update_end(&txq_stats->napi_syncp);
+
+	if (tx_desc) {
+		stmmac_flush_tx_descriptors(priv, queue);
+		xsk_tx_release(pool);
+	}
+
+	/* Return true if all of the 3 conditions are met
+	 *  a) TX Budget is still available
+	 *  b) work_done = true when XSK TX desc peek is empty (no more
+	 *     pending XSK TX for transmission)
+	 */
+	return !!budget && work_done;
+}
+
+static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan)
+{
+	if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) {
+		tc += 64;
+
+		if (priv->plat->force_thresh_dma_mode)
+			stmmac_set_dma_operation_mode(priv, tc, tc, chan);
+		else
+			stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE,
+						      chan);
+
+		priv->xstats.threshold = tc;
+	}
+}
+
+/**
+ * stmmac_tx_clean - to manage the transmission completion
+ * @priv: driver private structure
+ * @budget: napi budget limiting this functions packet handling
+ * @queue: TX queue index
+ * @pending_packets: signal to arm the TX coal timer
+ * Description: it reclaims the transmit resources after transmission completes.
+ * If some packets still needs to be handled, due to TX coalesce, set
+ * pending_packets to true to make NAPI arm the TX coal timer.
+ */
+static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
+			   bool *pending_packets)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];
+	unsigned int bytes_compl = 0, pkts_compl = 0;
+	unsigned int entry, xmits = 0, count = 0;
+	u32 tx_packets = 0, tx_errors = 0;
+
+	if (!get_ecdev(priv))
+		__netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
+
+	tx_q->xsk_frames_done = 0;
+
+	entry = tx_q->dirty_tx;
+
+	/* Try to clean all TX complete frame in 1 shot */
+	while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) {
+		struct xdp_frame *xdpf;
+		struct sk_buff *skb;
+		struct dma_desc *p;
+		int status;
+
+		if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX ||
+		    tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
+			xdpf = tx_q->xdpf[entry];
+			skb = NULL;
+		} else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
+			xdpf = NULL;
+			skb = tx_q->tx_skbuff[entry];
+		} else {
+			xdpf = NULL;
+			skb = NULL;
+		}
+
+		if (priv->extend_desc)
+			p = (struct dma_desc *)(tx_q->dma_etx + entry);
+		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			p = &tx_q->dma_entx[entry].basic;
+		else
+			p = tx_q->dma_tx + entry;
+
+		status = stmmac_tx_status(priv,	&priv->xstats, p, priv->ioaddr);
+		/* Check if the descriptor is owned by the DMA */
+		if (unlikely(status & tx_dma_own))
+			break;
+
+		count++;
+
+		/* Make sure descriptor fields are read after reading
+		 * the own bit.
+		 */
+		dma_rmb();
+
+		/* Just consider the last segment and ...*/
+		if (likely(!(status & tx_not_ls))) {
+			/* ... verify the status error condition */
+			if (unlikely(status & tx_err)) {
+				tx_errors++;
+				if (unlikely(status & tx_err_bump_tc))
+					stmmac_bump_dma_threshold(priv, queue);
+			} else {
+				tx_packets++;
+			}
+			if (skb) {
+				stmmac_get_tx_hwtstamp(priv, p, skb);
+			} else if (tx_q->xsk_pool &&
+				   xp_tx_metadata_enabled(tx_q->xsk_pool)) {
+				struct stmmac_xsk_tx_complete tx_compl = {
+					.priv = priv,
+					.desc = p,
+				};
+
+				xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta,
+							 &stmmac_xsk_tx_metadata_ops,
+							 &tx_compl);
+			}
+		}
+
+		if (likely(tx_q->tx_skbuff_dma[entry].buf &&
+			   tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) {
+			if (tx_q->tx_skbuff_dma[entry].map_as_page)
+				dma_unmap_page(priv->device,
+					       tx_q->tx_skbuff_dma[entry].buf,
+					       tx_q->tx_skbuff_dma[entry].len,
+					       DMA_TO_DEVICE);
+			else
+				dma_unmap_single(priv->device,
+						 tx_q->tx_skbuff_dma[entry].buf,
+						 tx_q->tx_skbuff_dma[entry].len,
+						 DMA_TO_DEVICE);
+			tx_q->tx_skbuff_dma[entry].buf = 0;
+			tx_q->tx_skbuff_dma[entry].len = 0;
+			tx_q->tx_skbuff_dma[entry].map_as_page = false;
+		}
+
+		stmmac_clean_desc3(priv, tx_q, p);
+
+		tx_q->tx_skbuff_dma[entry].last_segment = false;
+		tx_q->tx_skbuff_dma[entry].is_jumbo = false;
+
+		if (xdpf &&
+		    tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) {
+			xdp_return_frame_rx_napi(xdpf);
+			tx_q->xdpf[entry] = NULL;
+		}
+
+		if (xdpf &&
+		    tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
+			xdp_return_frame(xdpf);
+			tx_q->xdpf[entry] = NULL;
+		}
+
+		if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX)
+			tx_q->xsk_frames_done++;
+
+		if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
+			if (likely(skb)) {
+				pkts_compl++;
+				bytes_compl += skb->len;
+				if (!get_ecdev(priv)) {
+					dev_consume_skb_any(skb);
+				}
+				tx_q->tx_skbuff[entry] = NULL;
+			}
+		}
+
+		stmmac_release_tx_desc(priv, p, priv->mode);
+
+		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+	}
+	tx_q->dirty_tx = entry;
+
+	if (!get_ecdev(priv)) {
+		netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
+				  pkts_compl, bytes_compl);
+	}
+
+	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
+								queue))) &&
+	    stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {
+
+		netif_dbg(priv, tx_done, priv->dev,
+			  "%s: restart transmit\n", __func__);
+		netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
+	}
+
+	if (tx_q->xsk_pool) {
+		bool work_done;
+
+		if (tx_q->xsk_frames_done)
+			xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done);
+
+		if (xsk_uses_need_wakeup(tx_q->xsk_pool))
+			xsk_set_tx_need_wakeup(tx_q->xsk_pool);
+
+		/* For XSK TX, we try to send as many as possible.
+		 * If XSK work done (XSK TX desc empty and budget still
+		 * available), return "budget - 1" to reenable TX IRQ.
+		 * Else, return "budget" to make NAPI continue polling.
+		 */
+		work_done = stmmac_xdp_xmit_zc(priv, queue,
+					       STMMAC_XSK_TX_BUDGET_MAX);
+		if (work_done)
+			xmits = budget - 1;
+		else
+			xmits = budget;
+	}
+
+	if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode)
+		stmmac_restart_sw_lpi_timer(priv);
+
+	/* We still have pending packets, let's call for a new scheduling */
+	if (!get_ecdev(priv) && tx_q->dirty_tx != tx_q->cur_tx)
+		*pending_packets = true;
+
+	u64_stats_update_begin(&txq_stats->napi_syncp);
+	u64_stats_add(&txq_stats->napi.tx_packets, tx_packets);
+	u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets);
+	u64_stats_inc(&txq_stats->napi.tx_clean);
+	u64_stats_update_end(&txq_stats->napi_syncp);
+
+	priv->xstats.tx_errors += tx_errors;
+
+	if (!get_ecdev(priv))
+		__netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
+
+	/* Combine decisions from TX clean and XSK TX */
+	return max(count, xmits);
+}
+
+/**
+ * stmmac_tx_err - to manage the tx error
+ * @priv: driver private structure
+ * @chan: channel index
+ * Description: it cleans the descriptors and restarts the transmission
+ * in case of transmission errors.
+ */
+static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
+
+	netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
+
+	stmmac_stop_tx_dma(priv, chan);
+	dma_free_tx_skbufs(priv, &priv->dma_conf, chan);
+	stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan);
+	stmmac_reset_tx_queue(priv, chan);
+	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+			    tx_q->dma_tx_phy, chan);
+	stmmac_start_tx_dma(priv, chan);
+
+	priv->xstats.tx_errors++;
+	netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
+}
+
+/**
+ *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
+ *  @priv: driver private structure
+ *  @txmode: TX operating mode
+ *  @rxmode: RX operating mode
+ *  @chan: channel index
+ *  Description: it is used for configuring of the DMA operation mode in
+ *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
+ *  mode.
+ */
+static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
+					  u32 rxmode, u32 chan)
+{
+	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
+	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
+	u32 rx_channels_count = priv->plat->rx_queues_to_use;
+	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	int rxfifosz = priv->plat->rx_fifo_size;
+	int txfifosz = priv->plat->tx_fifo_size;
+
+	if (rxfifosz == 0)
+		rxfifosz = priv->dma_cap.rx_fifo_size;
+	if (txfifosz == 0)
+		txfifosz = priv->dma_cap.tx_fifo_size;
+
+	/* Adjust for real per queue fifo size */
+	rxfifosz /= rx_channels_count;
+	txfifosz /= tx_channels_count;
+
+	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
+	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
+}
+
+static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
+{
+	int ret;
+
+	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
+			priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
+	if (ret && (ret != -EINVAL)) {
+		stmmac_global_err(priv);
+		return true;
+	}
+
+	return false;
+}
+
+static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)
+{
+	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
+						 &priv->xstats, chan, dir);
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
+	struct stmmac_channel *ch = &priv->channel[chan];
+	struct napi_struct *rx_napi;
+	struct napi_struct *tx_napi;
+	unsigned long flags;
+
+	rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi;
+	tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
+
+	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
+		if (napi_schedule_prep(rx_napi)) {
+			spin_lock_irqsave(&ch->lock, flags);
+			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
+			spin_unlock_irqrestore(&ch->lock, flags);
+			__napi_schedule(rx_napi);
+		}
+	}
+
+	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
+		if (napi_schedule_prep(tx_napi)) {
+			spin_lock_irqsave(&ch->lock, flags);
+			stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
+			spin_unlock_irqrestore(&ch->lock, flags);
+			__napi_schedule(tx_napi);
+		}
+	}
+
+	return status;
+}
+
+/**
+ * stmmac_dma_interrupt - DMA ISR
+ * @priv: driver private structure
+ * Description: this is the DMA ISR. It is called by the main ISR.
+ * It calls the dwmac dma routine and schedule poll method in case of some
+ * work can be done.
+ */
+static void stmmac_dma_interrupt(struct stmmac_priv *priv)
+{
+	u32 tx_channel_count = priv->plat->tx_queues_to_use;
+	u32 rx_channel_count = priv->plat->rx_queues_to_use;
+	u32 channels_to_check = tx_channel_count > rx_channel_count ?
+				tx_channel_count : rx_channel_count;
+	u32 chan;
+	int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
+
+	/* Make sure we never check beyond our status buffer. */
+	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
+		channels_to_check = ARRAY_SIZE(status);
+
+	for (chan = 0; chan < channels_to_check; chan++)
+		status[chan] = stmmac_napi_check(priv, chan,
+						 DMA_DIR_RXTX);
+
+	for (chan = 0; chan < tx_channel_count; chan++) {
+		if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
+			/* Try to bump up the dma threshold on this failure */
+			stmmac_bump_dma_threshold(priv, chan);
+		} else if (unlikely(status[chan] == tx_hard_error)) {
+			stmmac_tx_err(priv, chan);
+		}
+	}
+}
+
+/**
+ * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
+ * @priv: driver private structure
+ * Description: this masks the MMC irq, in fact, the counters are managed in SW.
+ */
+static void stmmac_mmc_setup(struct stmmac_priv *priv)
+{
+	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
+			    MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
+
+	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
+
+	if (priv->dma_cap.rmon) {
+		stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
+		memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
+	} else
+		netdev_info(priv->dev, "No MAC Management Counters available\n");
+}
+
+/**
+ * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
+ * @priv: driver private structure
+ * Description:
+ *  new GMAC chip generations have a new register to indicate the
+ *  presence of the optional feature/functions.
+ *  This can be also used to override the value passed through the
+ *  platform and necessary for old MAC10/100 and GMAC chips.
+ */
+static int stmmac_get_hw_features(struct stmmac_priv *priv)
+{
+	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
+}
+
+/**
+ * stmmac_check_ether_addr - check if the MAC addr is valid
+ * @priv: driver private structure
+ * Description:
+ * it is to verify if the MAC address is valid, in case of failures it
+ * generates a random MAC address
+ */
+static void stmmac_check_ether_addr(struct stmmac_priv *priv)
+{
+	u8 addr[ETH_ALEN];
+
+	if (!is_valid_ether_addr(priv->dev->dev_addr)) {
+		stmmac_get_umac_addr(priv, priv->hw, addr, 0);
+		if (is_valid_ether_addr(addr))
+			eth_hw_addr_set(priv->dev, addr);
+		else
+			eth_hw_addr_random(priv->dev);
+		dev_info(priv->device, "device MAC address %pM\n",
+			 priv->dev->dev_addr);
+	}
+}
+
+/**
+ * stmmac_init_dma_engine - DMA init.
+ * @priv: driver private structure
+ * Description:
+ * It inits the DMA invoking the specific MAC/GMAC callback.
+ * Some DMA parameters can be passed from the platform;
+ * in case of these are not passed a default is kept for the MAC or GMAC.
+ */
+static int stmmac_init_dma_engine(struct stmmac_priv *priv)
+{
+	u32 rx_channels_count = priv->plat->rx_queues_to_use;
+	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
+	struct stmmac_rx_queue *rx_q;
+	struct stmmac_tx_queue *tx_q;
+	u32 chan = 0;
+	int ret = 0;
+
+	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
+		netdev_err(priv->dev, "Invalid DMA configuration\n");
+		return -EINVAL;
+	}
+
+	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
+		priv->plat->dma_cfg->atds = 1;
+
+	ret = ec_stmmac_reset(priv, priv->ioaddr);
+	if (ret) {
+		netdev_err(priv->dev, "Failed to reset the dma\n");
+		return ret;
+	}
+
+	/* DMA Configuration */
+	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg);
+
+	if (priv->plat->axi)
+		stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
+
+	/* DMA CSR Channel configuration */
+	for (chan = 0; chan < dma_csr_ch; chan++) {
+		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
+		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
+	}
+
+	/* DMA RX Channel Configuration */
+	for (chan = 0; chan < rx_channels_count; chan++) {
+		rx_q = &priv->dma_conf.rx_queue[chan];
+
+		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+				    rx_q->dma_rx_phy, chan);
+
+		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
+				     (rx_q->buf_alloc_num *
+				      sizeof(struct dma_desc));
+		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
+				       rx_q->rx_tail_addr, chan);
+	}
+
+	/* DMA TX Channel Configuration */
+	for (chan = 0; chan < tx_channels_count; chan++) {
+		tx_q = &priv->dma_conf.tx_queue[chan];
+
+		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+				    tx_q->dma_tx_phy, chan);
+
+		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
+		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
+				       tx_q->tx_tail_addr, chan);
+	}
+
+	return ret;
+}
+
+static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	u32 tx_coal_timer = priv->tx_coal_timer[queue];
+	struct stmmac_channel *ch;
+	struct napi_struct *napi;
+
+	if (!tx_coal_timer || get_ecdev(priv))
+		return;
+
+	ch = &priv->channel[tx_q->queue_index];
+	napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
+
+	/* Arm timer only if napi is not already scheduled.
+	 * Try to cancel any timer if napi is scheduled, timer will be armed
+	 * again in the next scheduled napi.
+	 */
+	if (unlikely(!napi_is_scheduled(napi)))
+		hrtimer_start(&tx_q->txtimer,
+			      STMMAC_COAL_TIMER(tx_coal_timer),
+			      HRTIMER_MODE_REL);
+	else
+		hrtimer_try_to_cancel(&tx_q->txtimer);
+}
+
+/**
+ * stmmac_tx_timer - mitigation sw timer for tx.
+ * @t: data pointer
+ * Description:
+ * This is the timer handler to directly invoke the stmmac_tx_clean.
+ */
+static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t)
+{
+	struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer);
+	struct stmmac_priv *priv = tx_q->priv_data;
+	struct stmmac_channel *ch;
+	struct napi_struct *napi;
+
+	ch = &priv->channel[tx_q->queue_index];
+	napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
+
+	if (likely(napi_schedule_prep(napi))) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&ch->lock, flags);
+		stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
+		spin_unlock_irqrestore(&ch->lock, flags);
+		__napi_schedule(napi);
+	}
+
+	return HRTIMER_NORESTART;
+}
+
+/**
+ * stmmac_init_coalesce - init mitigation options.
+ * @priv: driver private structure
+ * Description:
+ * This inits the coalesce parameters: i.e. timer rate,
+ * timer handler and default threshold used for enabling the
+ * interrupt on completion bit.
+ */
+static void stmmac_init_coalesce(struct stmmac_priv *priv)
+{
+	u32 tx_channel_count = priv->plat->tx_queues_to_use;
+	u32 rx_channel_count = priv->plat->rx_queues_to_use;
+	u32 chan;
+
+	for (chan = 0; chan < tx_channel_count; chan++) {
+		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
+
+		priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES;
+		priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER;
+
+		hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	}
+
+	for (chan = 0; chan < rx_channel_count; chan++)
+		priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES;
+}
+
+static void stmmac_set_rings_length(struct stmmac_priv *priv)
+{
+	u32 rx_channels_count = priv->plat->rx_queues_to_use;
+	u32 tx_channels_count = priv->plat->tx_queues_to_use;
+	u32 chan;
+
+	/* set TX ring length */
+	for (chan = 0; chan < tx_channels_count; chan++)
+		stmmac_set_tx_ring_len(priv, priv->ioaddr,
+				       (priv->dma_conf.dma_tx_size - 1), chan);
+
+	/* set RX ring length */
+	for (chan = 0; chan < rx_channels_count; chan++)
+		stmmac_set_rx_ring_len(priv, priv->ioaddr,
+				       (priv->dma_conf.dma_rx_size - 1), chan);
+}
+
+/**
+ *  stmmac_set_tx_queue_weight - Set TX queue weight
+ *  @priv: driver private structure
+ *  Description: It is used for setting TX queues weight
+ */
+static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
+{
+	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	u32 weight;
+	u32 queue;
+
+	for (queue = 0; queue < tx_queues_count; queue++) {
+		weight = priv->plat->tx_queues_cfg[queue].weight;
+		stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
+	}
+}
+
+/**
+ *  stmmac_configure_cbs - Configure CBS in TX queue
+ *  @priv: driver private structure
+ *  Description: It is used for configuring CBS in AVB TX queues
+ */
+static void stmmac_configure_cbs(struct stmmac_priv *priv)
+{
+	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	u32 mode_to_use;
+	u32 queue;
+
+	/* queue 0 is reserved for legacy traffic */
+	for (queue = 1; queue < tx_queues_count; queue++) {
+		mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
+		if (mode_to_use == MTL_QUEUE_DCB)
+			continue;
+
+		stmmac_config_cbs(priv, priv->hw,
+				priv->plat->tx_queues_cfg[queue].send_slope,
+				priv->plat->tx_queues_cfg[queue].idle_slope,
+				priv->plat->tx_queues_cfg[queue].high_credit,
+				priv->plat->tx_queues_cfg[queue].low_credit,
+				queue);
+	}
+}
+
+/**
+ *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
+ *  @priv: driver private structure
+ *  Description: It is used for mapping RX queues to RX dma channels
+ */
+static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
+{
+	u32 rx_queues_count = priv->plat->rx_queues_to_use;
+	u32 queue;
+	u32 chan;
+
+	for (queue = 0; queue < rx_queues_count; queue++) {
+		chan = priv->plat->rx_queues_cfg[queue].chan;
+		stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
+	}
+}
+
+/**
+ *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
+ *  @priv: driver private structure
+ *  Description: It is used for configuring the RX Queue Priority
+ */
+static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
+{
+	u32 rx_queues_count = priv->plat->rx_queues_to_use;
+	u32 queue;
+	u32 prio;
+
+	for (queue = 0; queue < rx_queues_count; queue++) {
+		if (!priv->plat->rx_queues_cfg[queue].use_prio)
+			continue;
+
+		prio = priv->plat->rx_queues_cfg[queue].prio;
+		stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
+	}
+}
+
+/**
+ *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
+ *  @priv: driver private structure
+ *  Description: It is used for configuring the TX Queue Priority
+ */
+static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
+{
+	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	u32 queue;
+	u32 prio;
+
+	for (queue = 0; queue < tx_queues_count; queue++) {
+		if (!priv->plat->tx_queues_cfg[queue].use_prio)
+			continue;
+
+		prio = priv->plat->tx_queues_cfg[queue].prio;
+		stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
+	}
+}
+
+/**
+ *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
+ *  @priv: driver private structure
+ *  Description: It is used for configuring the RX queue routing
+ */
+static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
+{
+	u32 rx_queues_count = priv->plat->rx_queues_to_use;
+	u32 queue;
+	u8 packet;
+
+	for (queue = 0; queue < rx_queues_count; queue++) {
+		/* no specific packet type routing specified for the queue */
+		if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
+			continue;
+
+		packet = priv->plat->rx_queues_cfg[queue].pkt_route;
+		stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
+	}
+}
+
+static void stmmac_mac_config_rss(struct stmmac_priv *priv)
+{
+	if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
+		priv->rss.enable = false;
+		return;
+	}
+
+	if (priv->dev->features & NETIF_F_RXHASH)
+		priv->rss.enable = true;
+	else
+		priv->rss.enable = false;
+
+	stmmac_rss_configure(priv, priv->hw, &priv->rss,
+			     priv->plat->rx_queues_to_use);
+}
+
+/**
+ *  stmmac_mtl_configuration - Configure MTL
+ *  @priv: driver private structure
+ *  Description: It is used for configurring MTL
+ */
+static void stmmac_mtl_configuration(struct stmmac_priv *priv)
+{
+	u32 rx_queues_count = priv->plat->rx_queues_to_use;
+	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+
+	if (tx_queues_count > 1)
+		stmmac_set_tx_queue_weight(priv);
+
+	/* Configure MTL RX algorithms */
+	if (rx_queues_count > 1)
+		stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
+				priv->plat->rx_sched_algorithm);
+
+	/* Configure MTL TX algorithms */
+	if (tx_queues_count > 1)
+		stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
+				priv->plat->tx_sched_algorithm);
+
+	/* Configure CBS in AVB TX queues */
+	if (tx_queues_count > 1)
+		stmmac_configure_cbs(priv);
+
+	/* Map RX MTL to DMA channels */
+	stmmac_rx_queue_dma_chan_map(priv);
+
+	/* Enable MAC RX Queues */
+	stmmac_mac_enable_rx_queues(priv);
+
+	/* Set RX priorities */
+	if (rx_queues_count > 1)
+		stmmac_mac_config_rx_queues_prio(priv);
+
+	/* Set TX priorities */
+	if (tx_queues_count > 1)
+		stmmac_mac_config_tx_queues_prio(priv);
+
+	/* Set RX routing */
+	if (rx_queues_count > 1)
+		stmmac_mac_config_rx_queues_routing(priv);
+
+	/* Receive Side Scaling */
+	if (rx_queues_count > 1)
+		stmmac_mac_config_rss(priv);
+}
+
+static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
+{
+	if (priv->dma_cap.asp) {
+		netdev_info(priv->dev, "Enabling Safety Features\n");
+		stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp,
+					  priv->plat->safety_feat_cfg);
+	} else {
+		netdev_info(priv->dev, "No Safety Features support found\n");
+	}
+}
+
+/**
+ * stmmac_hw_setup - setup mac in a usable state.
+ *  @dev : pointer to the device structure.
+ *  Description:
+ *  this is the main function to setup the HW in a usable state because the
+ *  dma engine is reset, the core registers are configured (e.g. AXI,
+ *  Checksum features, timers). The DMA is ready to start receiving and
+ *  transmitting.
+ *  Return value:
+ *  0 on success and an appropriate (-)ve integer as defined in errno.h
+ *  file on failure.
+ */
+static int stmmac_hw_setup(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	bool sph_en;
+	u32 chan;
+	int ret;
+
+	/* Make sure RX clock is enabled */
+	if (priv->hw->phylink_pcs)
+		phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs);
+
+	/* Note that clk_rx_i must be running for reset to complete. This
+	 * clock may also be required when setting the MAC address.
+	 *
+	 * Block the receive clock stop for LPI mode at the PHY in case
+	 * the link is established with EEE mode active.
+	 */
+	phylink_rx_clk_stop_block(priv->phylink);
+
+	/* DMA initialization and SW reset */
+	ret = stmmac_init_dma_engine(priv);
+	if (ret < 0) {
+		phylink_rx_clk_stop_unblock(priv->phylink);
+		netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Copy the MAC addr into the HW  */
+	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
+	phylink_rx_clk_stop_unblock(priv->phylink);
+
+	/* PS and related bits will be programmed according to the speed */
+	if (priv->hw->pcs) {
+		int speed = priv->plat->mac_port_sel_speed;
+
+		if ((speed == SPEED_10) || (speed == SPEED_100) ||
+		    (speed == SPEED_1000)) {
+			priv->hw->ps = speed;
+		} else {
+			dev_warn(priv->device, "invalid port speed\n");
+			priv->hw->ps = 0;
+		}
+	}
+
+	/* Initialize the MAC Core */
+	stmmac_core_init(priv, priv->hw, dev);
+
+	/* Initialize MTL*/
+	stmmac_mtl_configuration(priv);
+
+	/* Initialize Safety Features */
+	stmmac_safety_feat_configuration(priv);
+
+	ret = stmmac_rx_ipc(priv, priv->hw);
+	if (!ret) {
+		netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
+		priv->plat->rx_coe = STMMAC_RX_COE_NONE;
+		priv->hw->rx_csum = 0;
+	}
+
+	/* Enable the MAC Rx/Tx */
+	stmmac_mac_set(priv, priv->ioaddr, true);
+
+	/* Set the HW DMA mode and the COE */
+	stmmac_dma_operation_mode(priv);
+
+	stmmac_mmc_setup(priv);
+
+	if (priv->use_riwt) {
+		u32 queue;
+
+		for (queue = 0; queue < rx_cnt; queue++) {
+			if (!priv->rx_riwt[queue])
+				priv->rx_riwt[queue] = DEF_DMA_RIWT;
+
+			stmmac_rx_watchdog(priv, priv->ioaddr,
+					   priv->rx_riwt[queue], queue);
+		}
+	}
+
+	if (priv->hw->pcs)
+		stmmac_pcs_ctrl_ane(priv, 1, priv->hw->ps, 0);
+
+	/* set TX and RX rings length */
+	stmmac_set_rings_length(priv);
+
+	/* Enable TSO */
+	if (priv->tso) {
+		for (chan = 0; chan < tx_cnt; chan++) {
+			struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
+
+			/* TSO and TBS cannot co-exist */
+			if (tx_q->tbs & STMMAC_TBS_AVAIL)
+				continue;
+
+			stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
+		}
+	}
+
+	/* Enable Split Header */
+	sph_en = (priv->hw->rx_csum > 0) && priv->sph;
+	for (chan = 0; chan < rx_cnt; chan++)
+		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
+
+
+	/* VLAN Tag Insertion */
+	if (priv->dma_cap.vlins)
+		stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
+
+	/* TBS */
+	for (chan = 0; chan < tx_cnt; chan++) {
+		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
+		int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
+
+		stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
+	}
+
+	/* Configure real RX and TX queues */
+	netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use);
+	netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use);
+
+	/* Start the ball rolling... */
+	stmmac_start_all_dma(priv);
+
+	phylink_rx_clk_stop_block(priv->phylink);
+	stmmac_set_hw_vlan_mode(priv, priv->hw);
+	phylink_rx_clk_stop_unblock(priv->phylink);
+
+	return 0;
+}
+
+static void stmmac_free_irq(struct net_device *dev,
+			    enum request_irq_err irq_err, int irq_idx)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int j;
+
+	switch (irq_err) {
+	case REQ_IRQ_ERR_ALL:
+		irq_idx = priv->plat->tx_queues_to_use;
+		fallthrough;
+	case REQ_IRQ_ERR_TX:
+		for (j = irq_idx - 1; j >= 0; j--) {
+			if (priv->tx_irq[j] > 0) {
+				irq_set_affinity_hint(priv->tx_irq[j], NULL);
+				free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]);
+			}
+		}
+		irq_idx = priv->plat->rx_queues_to_use;
+		fallthrough;
+	case REQ_IRQ_ERR_RX:
+		for (j = irq_idx - 1; j >= 0; j--) {
+			if (priv->rx_irq[j] > 0) {
+				irq_set_affinity_hint(priv->rx_irq[j], NULL);
+				free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]);
+			}
+		}
+
+		if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq)
+			free_irq(priv->sfty_ue_irq, dev);
+		fallthrough;
+	case REQ_IRQ_ERR_SFTY_UE:
+		if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq)
+			free_irq(priv->sfty_ce_irq, dev);
+		fallthrough;
+	case REQ_IRQ_ERR_SFTY_CE:
+		if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq)
+			free_irq(priv->lpi_irq, dev);
+		fallthrough;
+	case REQ_IRQ_ERR_LPI:
+		if (priv->wol_irq > 0 && priv->wol_irq != dev->irq)
+			free_irq(priv->wol_irq, dev);
+		fallthrough;
+	case REQ_IRQ_ERR_SFTY:
+		if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq)
+			free_irq(priv->sfty_irq, dev);
+		fallthrough;
+	case REQ_IRQ_ERR_WOL:
+		free_irq(dev->irq, dev);
+		fallthrough;
+	case REQ_IRQ_ERR_MAC:
+	case REQ_IRQ_ERR_NO:
+		/* If MAC IRQ request error, no more IRQ to free */
+		break;
+	}
+}
+
+static int stmmac_request_irq_multi_msi(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	enum request_irq_err irq_err;
+	int irq_idx = 0;
+	char *int_name;
+	int ret;
+	int i;
+
+	/* For common interrupt */
+	int_name = priv->int_name_mac;
+	sprintf(int_name, "%s:%s", dev->name, "mac");
+	ret = request_irq(dev->irq, stmmac_mac_interrupt,
+			  0, int_name, dev);
+	if (unlikely(ret < 0)) {
+		netdev_err(priv->dev,
+			   "%s: alloc mac MSI %d (error: %d)\n",
+			   __func__, dev->irq, ret);
+		irq_err = REQ_IRQ_ERR_MAC;
+		goto irq_error;
+	}
+
+	/* Request the Wake IRQ in case of another line
+	 * is used for WoL
+	 */
+	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
+		int_name = priv->int_name_wol;
+		sprintf(int_name, "%s:%s", dev->name, "wol");
+		ret = request_irq(priv->wol_irq,
+				  stmmac_mac_interrupt,
+				  0, int_name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: alloc wol MSI %d (error: %d)\n",
+				   __func__, priv->wol_irq, ret);
+			irq_err = REQ_IRQ_ERR_WOL;
+			goto irq_error;
+		}
+	}
+
+	/* Request the LPI IRQ in case of another line
+	 * is used for LPI
+	 */
+	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
+		int_name = priv->int_name_lpi;
+		sprintf(int_name, "%s:%s", dev->name, "lpi");
+		ret = request_irq(priv->lpi_irq,
+				  stmmac_mac_interrupt,
+				  0, int_name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: alloc lpi MSI %d (error: %d)\n",
+				   __func__, priv->lpi_irq, ret);
+			irq_err = REQ_IRQ_ERR_LPI;
+			goto irq_error;
+		}
+	}
+
+	/* Request the common Safety Feature Correctible/Uncorrectible
+	 * Error line in case of another line is used
+	 */
+	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {
+		int_name = priv->int_name_sfty;
+		sprintf(int_name, "%s:%s", dev->name, "safety");
+		ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt,
+				  0, int_name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: alloc sfty MSI %d (error: %d)\n",
+				   __func__, priv->sfty_irq, ret);
+			irq_err = REQ_IRQ_ERR_SFTY;
+			goto irq_error;
+		}
+	}
+
+	/* Request the Safety Feature Correctible Error line in
+	 * case of another line is used
+	 */
+	if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) {
+		int_name = priv->int_name_sfty_ce;
+		sprintf(int_name, "%s:%s", dev->name, "safety-ce");
+		ret = request_irq(priv->sfty_ce_irq,
+				  stmmac_safety_interrupt,
+				  0, int_name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: alloc sfty ce MSI %d (error: %d)\n",
+				   __func__, priv->sfty_ce_irq, ret);
+			irq_err = REQ_IRQ_ERR_SFTY_CE;
+			goto irq_error;
+		}
+	}
+
+	/* Request the Safety Feature Uncorrectible Error line in
+	 * case of another line is used
+	 */
+	if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) {
+		int_name = priv->int_name_sfty_ue;
+		sprintf(int_name, "%s:%s", dev->name, "safety-ue");
+		ret = request_irq(priv->sfty_ue_irq,
+				  stmmac_safety_interrupt,
+				  0, int_name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: alloc sfty ue MSI %d (error: %d)\n",
+				   __func__, priv->sfty_ue_irq, ret);
+			irq_err = REQ_IRQ_ERR_SFTY_UE;
+			goto irq_error;
+		}
+	}
+
+	/* Request Rx MSI irq */
+	for (i = 0; i < priv->plat->rx_queues_to_use; i++) {
+		if (i >= MTL_MAX_RX_QUEUES)
+			break;
+		if (priv->rx_irq[i] == 0)
+			continue;
+
+		int_name = priv->int_name_rx_irq[i];
+		sprintf(int_name, "%s:%s-%d", dev->name, "rx", i);
+		ret = request_irq(priv->rx_irq[i],
+				  stmmac_msi_intr_rx,
+				  0, int_name, &priv->dma_conf.rx_queue[i]);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: alloc rx-%d  MSI %d (error: %d)\n",
+				   __func__, i, priv->rx_irq[i], ret);
+			irq_err = REQ_IRQ_ERR_RX;
+			irq_idx = i;
+			goto irq_error;
+		}
+		irq_set_affinity_hint(priv->rx_irq[i],
+				      cpumask_of(i % num_online_cpus()));
+	}
+
+	/* Request Tx MSI irq */
+	for (i = 0; i < priv->plat->tx_queues_to_use; i++) {
+		if (i >= MTL_MAX_TX_QUEUES)
+			break;
+		if (priv->tx_irq[i] == 0)
+			continue;
+
+		int_name = priv->int_name_tx_irq[i];
+		sprintf(int_name, "%s:%s-%d", dev->name, "tx", i);
+		ret = request_irq(priv->tx_irq[i],
+				  stmmac_msi_intr_tx,
+				  0, int_name, &priv->dma_conf.tx_queue[i]);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: alloc tx-%d  MSI %d (error: %d)\n",
+				   __func__, i, priv->tx_irq[i], ret);
+			irq_err = REQ_IRQ_ERR_TX;
+			irq_idx = i;
+			goto irq_error;
+		}
+		irq_set_affinity_hint(priv->tx_irq[i],
+				      cpumask_of(i % num_online_cpus()));
+	}
+
+	return 0;
+
+irq_error:
+	stmmac_free_irq(dev, irq_err, irq_idx);
+	return ret;
+}
+
+static int stmmac_request_irq_single(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	enum request_irq_err irq_err;
+	int ret;
+
+	ret = request_irq(dev->irq, stmmac_interrupt,
+			  IRQF_SHARED, dev->name, dev);
+	if (unlikely(ret < 0)) {
+		netdev_err(priv->dev,
+			   "%s: ERROR: allocating the IRQ %d (error: %d)\n",
+			   __func__, dev->irq, ret);
+		irq_err = REQ_IRQ_ERR_MAC;
+		goto irq_error;
+	}
+
+	/* Request the Wake IRQ in case of another line
+	 * is used for WoL
+	 */
+	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
+		ret = request_irq(priv->wol_irq, stmmac_interrupt,
+				  IRQF_SHARED, dev->name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
+				   __func__, priv->wol_irq, ret);
+			irq_err = REQ_IRQ_ERR_WOL;
+			goto irq_error;
+		}
+	}
+
+	/* Request the IRQ lines */
+	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
+		ret = request_irq(priv->lpi_irq, stmmac_interrupt,
+				  IRQF_SHARED, dev->name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
+				   __func__, priv->lpi_irq, ret);
+			irq_err = REQ_IRQ_ERR_LPI;
+			goto irq_error;
+		}
+	}
+
+	/* Request the common Safety Feature Correctible/Uncorrectible
+	 * Error line in case of another line is used
+	 */
+	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {
+		ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt,
+				  IRQF_SHARED, dev->name, dev);
+		if (unlikely(ret < 0)) {
+			netdev_err(priv->dev,
+				   "%s: ERROR: allocating the sfty IRQ %d (%d)\n",
+				   __func__, priv->sfty_irq, ret);
+			irq_err = REQ_IRQ_ERR_SFTY;
+			goto irq_error;
+		}
+	}
+
+	return 0;
+
+irq_error:
+	stmmac_free_irq(dev, irq_err, 0);
+	return ret;
+}
+
+static int stmmac_request_irq(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int ret;
+
+	/* Request the IRQ lines */
+	if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN)
+		ret = stmmac_request_irq_multi_msi(dev);
+	else
+		ret = stmmac_request_irq_single(dev);
+
+	return ret;
+}
+
+/**
+ *  stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue
+ *  @priv: driver private structure
+ *  @mtu: MTU to setup the dma queue and buf with
+ *  Description: Allocate and generate a dma_conf based on the provided MTU.
+ *  Allocate the Tx/Rx DMA queue and init them.
+ *  Return value:
+ *  the dma_conf allocated struct on success and an appropriate ERR_PTR on failure.
+ */
+static struct stmmac_dma_conf *
+stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)
+{
+	struct stmmac_dma_conf *dma_conf;
+	int chan, bfsize, ret;
+
+	dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL);
+	if (!dma_conf) {
+		netdev_err(priv->dev, "%s: DMA conf allocation failed\n",
+			   __func__);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	bfsize = stmmac_set_16kib_bfsize(priv, mtu);
+	if (bfsize < 0)
+		bfsize = 0;
+
+	if (bfsize < BUF_SIZE_16KiB)
+		bfsize = stmmac_set_bfsize(mtu, 0);
+
+	dma_conf->dma_buf_sz = bfsize;
+	/* Chose the tx/rx size from the already defined one in the
+	 * priv struct. (if defined)
+	 */
+	dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size;
+	dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size;
+
+	if (!dma_conf->dma_tx_size)
+		dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE;
+	if (!dma_conf->dma_rx_size)
+		dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE;
+
+	/* Earlier check for TBS */
+	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
+		struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan];
+		int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
+
+		/* Setup per-TXQ tbs flag before TX descriptor alloc */
+		tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0;
+	}
+
+	ret = alloc_dma_desc_resources(priv, dma_conf);
+	if (ret < 0) {
+		netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
+			   __func__);
+		goto alloc_error;
+	}
+
+	ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL);
+	if (ret < 0) {
+		netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
+			   __func__);
+		goto init_error;
+	}
+
+	return dma_conf;
+
+init_error:
+	free_dma_desc_resources(priv, dma_conf);
+alloc_error:
+	kfree(dma_conf);
+	return ERR_PTR(ret);
+}
+
+/**
+ *  __stmmac_open - open entry point of the driver
+ *  @dev : pointer to the device structure.
+ *  @dma_conf :  structure to take the dma data
+ *  Description:
+ *  This function is the open entry point of the driver.
+ *  Return value:
+ *  0 on success and an appropriate (-)ve integer as defined in errno.h
+ *  file on failure.
+ */
+static int __stmmac_open(struct net_device *dev,
+			 struct stmmac_dma_conf *dma_conf)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 chan;
+	int ret;
+
+	for (int i = 0; i < MTL_MAX_TX_QUEUES; i++)
+		if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN)
+			dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs;
+	memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf));
+
+	stmmac_reset_queues_param(priv);
+
+	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&
+	    priv->plat->serdes_powerup) {
+		ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv);
+		if (ret < 0) {
+			netdev_err(priv->dev, "%s: Serdes powerup failed\n",
+				   __func__);
+			goto init_error;
+		}
+	}
+
+	if (get_ecdev(priv)) {
+		rtnl_lock();
+	}
+
+	ret = stmmac_hw_setup(dev);
+	if (ret < 0) {
+		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
+		goto init_error;
+	}
+
+	stmmac_setup_ptp(priv);
+
+	if (!get_ecdev(priv)) {
+		stmmac_init_coalesce(priv);
+	}
+
+	phylink_start(priv->phylink);
+	/* We may have called phylink_speed_down before */
+	phylink_speed_up(priv->phylink);
+	if (get_ecdev(priv)) {
+		rtnl_unlock();
+	}
+
+	ret = stmmac_request_irq(dev);
+	if (ret)
+		goto irq_error;
+
+	if (!get_ecdev(priv)) {
+		stmmac_enable_all_queues(priv);
+		netif_tx_start_all_queues(priv->dev);
+		stmmac_enable_all_dma_irq(priv);
+	}
+
+	return 0;
+
+irq_error:
+	phylink_stop(priv->phylink);
+
+	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
+		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);
+
+	stmmac_release_ptp(priv);
+init_error:
+	return ret;
+}
+
+static int stmmac_open(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	struct stmmac_dma_conf *dma_conf;
+	int ret;
+
+	/* Initialise the tx lpi timer, converting from msec to usec */
+	if (!priv->tx_lpi_timer)
+		priv->tx_lpi_timer = eee_timer * 1000;
+
+	dma_conf = stmmac_setup_dma_desc(priv, dev->mtu);
+	if (IS_ERR(dma_conf))
+		return PTR_ERR(dma_conf);
+
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		goto err_dma_resources;
+
+	ret = stmmac_init_phy(dev);
+	if (ret)
+		goto err_runtime_pm;
+
+	ret = __stmmac_open(dev, dma_conf);
+	if (ret)
+		goto err_disconnect_phy;
+
+	kfree(dma_conf);
+
+	return ret;
+
+err_disconnect_phy:
+	phylink_disconnect_phy(priv->phylink);
+err_runtime_pm:
+	pm_runtime_put(priv->device);
+err_dma_resources:
+	free_dma_desc_resources(priv, dma_conf);
+	kfree(dma_conf);
+	return ret;
+}
+
+static void __stmmac_release(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 chan;
+
+	if (get_ecdev(priv)) {
+		rtnl_lock();
+	}
+	/* If the PHY or MAC has WoL enabled, then the PHY will not be
+	 * suspended when phylink_stop() is called below. Set the PHY
+	 * to its slowest speed to save power.
+	 */
+	if (device_may_wakeup(priv->device))
+		phylink_speed_down(priv->phylink, false);
+
+	/* Stop and disconnect the PHY */
+	phylink_stop(priv->phylink);
+	if (get_ecdev(priv)) {
+		rtnl_unlock();
+	}
+
+	stmmac_disable_all_queues(priv);
+
+	if (!get_ecdev(priv)) {
+		for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
+			hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);
+	}
+
+	netif_tx_disable(dev);
+
+	/* Free the IRQ lines */
+	stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0);
+
+	/* Stop TX/RX DMA and clear the descriptors */
+	stmmac_stop_all_dma(priv);
+
+	/* Release and free the Rx/Tx resources */
+	free_dma_desc_resources(priv, &priv->dma_conf);
+
+	/* Powerdown Serdes if there is */
+	if (priv->plat->serdes_powerdown)
+		priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv);
+
+	stmmac_release_ptp(priv);
+
+	if (ec_stmmac_fpe_supported(priv))
+		ethtool_mmsv_stop(&priv->fpe_cfg.mmsv);
+}
+
+/**
+ *  stmmac_release - close entry point of the driver
+ *  @dev : device pointer.
+ *  Description:
+ *  This is the stop entry point of the driver.
+ */
+static int stmmac_release(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	__stmmac_release(dev);
+
+	if (get_ecdev(priv)) {
+		rtnl_lock();
+	}
+
+	phylink_disconnect_phy(priv->phylink);
+
+	if (get_ecdev(priv)) {
+		rtnl_unlock();
+	}
+
+	pm_runtime_put(priv->device);
+
+	return 0;
+}
+
+static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
+			       struct stmmac_tx_queue *tx_q)
+{
+	struct dma_desc *p;
+	u16 tag = 0x0;
+
+	if (!priv->dma_cap.vlins || !skb_vlan_tag_present(skb))
+		return false;
+
+	tag = skb_vlan_tag_get(skb);
+
+	if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		p = &tx_q->dma_entx[tx_q->cur_tx].basic;
+	else
+		p = &tx_q->dma_tx[tx_q->cur_tx];
+
+	if (stmmac_set_desc_vlan_tag(priv, p, tag, 0x0, 0x0))
+		return false;
+
+	stmmac_set_tx_owner(priv, p);
+	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
+	return true;
+}
+
+/**
+ *  stmmac_tso_allocator - close entry point of the driver
+ *  @priv: driver private structure
+ *  @des: buffer start address
+ *  @total_len: total length to fill in descriptors
+ *  @last_segment: condition for the last descriptor
+ *  @queue: TX queue index
+ *  Description:
+ *  This function fills descriptor and request new descriptors according to
+ *  buffer length to fill
+ */
+static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
+				 int total_len, bool last_segment, u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	struct dma_desc *desc;
+	u32 buff_size;
+	int tmp_len;
+
+	tmp_len = total_len;
+
+	while (tmp_len > 0) {
+		dma_addr_t curr_addr;
+
+		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
+						priv->dma_conf.dma_tx_size);
+		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
+
+		if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
+		else
+			desc = &tx_q->dma_tx[tx_q->cur_tx];
+
+		curr_addr = des + (total_len - tmp_len);
+		stmmac_set_desc_addr(priv, desc, curr_addr);
+		buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
+			    TSO_MAX_BUFF_SIZE : tmp_len;
+
+		stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
+				0, 1,
+				(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
+				0, 0);
+
+		tmp_len -= TSO_MAX_BUFF_SIZE;
+	}
+}
+
+static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	int desc_size;
+
+	if (likely(priv->extend_desc))
+		desc_size = sizeof(struct dma_extended_desc);
+	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		desc_size = sizeof(struct dma_edesc);
+	else
+		desc_size = sizeof(struct dma_desc);
+
+	/* The own bit must be the latest setting done when prepare the
+	 * descriptor and then barrier is needed to make sure that
+	 * all is coherent before granting the DMA engine.
+	 */
+	wmb();
+
+	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
+	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
+}
+
+/**
+ *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
+ *  @skb : the socket buffer
+ *  @dev : device pointer
+ *  Description: this is the transmit function that is called on TSO frames
+ *  (support available on GMAC4 and newer chips).
+ *  Diagram below show the ring programming in case of TSO frames:
+ *
+ *  First Descriptor
+ *   --------
+ *   | DES0 |---> buffer1 = L2/L3/L4 header
+ *   | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address
+ *   |      |     width is 32-bit, but we never use it.
+ *   |      |     Also can be used as the most-significant 8-bits or 16-bits of
+ *   |      |     buffer1 address pointer if the DMA AXI address width is 40-bit
+ *   |      |     or 48-bit, and we always use it.
+ *   | DES2 |---> buffer1 len
+ *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
+ *   --------
+ *   --------
+ *   | DES0 |---> buffer1 = TCP Payload (can continue on next descr...)
+ *   | DES1 |---> same as the First Descriptor
+ *   | DES2 |---> buffer1 len
+ *   | DES3 |
+ *   --------
+ *	|
+ *     ...
+ *	|
+ *   --------
+ *   | DES0 |---> buffer1 = Split TCP Payload
+ *   | DES1 |---> same as the First Descriptor
+ *   | DES2 |---> buffer1 len
+ *   | DES3 |
+ *   --------
+ *
+ * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
+ */
+static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct dma_desc *desc, *first, *mss_desc = NULL;
+	struct stmmac_priv *priv = netdev_priv(dev);
+	unsigned int first_entry, tx_packets;
+	struct stmmac_txq_stats *txq_stats;
+	struct stmmac_tx_queue *tx_q;
+	u32 pay_len, mss, queue;
+	int i, first_tx, nfrags;
+	u8 proto_hdr_len, hdr;
+	dma_addr_t des;
+	bool set_ic;
+
+	/* Always insert VLAN tag to SKB payload for TSO frames.
+	 *
+	 * Never insert VLAN tag by HW, since segments splited by
+	 * TSO engine will be un-tagged by mistake.
+	 */
+	if (skb_vlan_tag_present(skb)) {
+		skb = __vlan_hwaccel_push_inside(skb);
+		if (unlikely(!skb)) {
+			priv->xstats.tx_dropped++;
+			return NETDEV_TX_OK;
+		}
+	}
+
+	nfrags = skb_shinfo(skb)->nr_frags;
+	queue = skb_get_queue_mapping(skb);
+
+	tx_q = &priv->dma_conf.tx_queue[queue];
+	txq_stats = &priv->xstats.txq_stats[queue];
+	first_tx = tx_q->cur_tx;
+
+	/* Compute header lengths */
+	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
+		proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
+		hdr = sizeof(struct udphdr);
+	} else {
+		proto_hdr_len = skb_tcp_all_headers(skb);
+		hdr = tcp_hdrlen(skb);
+	}
+
+	/* Desc availability based on threshold should be enough safe */
+	if (unlikely(stmmac_tx_avail(priv, queue) <
+		(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
+		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
+			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
+								queue));
+			/* This is a hard error, log it. */
+			netdev_err(priv->dev,
+				   "%s: Tx Ring full when queue awake\n",
+				   __func__);
+		}
+		return NETDEV_TX_BUSY;
+	}
+
+	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
+
+	mss = skb_shinfo(skb)->gso_size;
+
+	/* set new MSS value if needed */
+	if (mss != tx_q->mss) {
+		if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
+		else
+			mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
+
+		stmmac_set_mss(priv, mss_desc, mss);
+		tx_q->mss = mss;
+		tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
+						priv->dma_conf.dma_tx_size);
+		WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
+	}
+
+	if (netif_msg_tx_queued(priv)) {
+		pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
+			__func__, hdr, proto_hdr_len, pay_len, mss);
+		pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
+			skb->data_len);
+	}
+
+	first_entry = tx_q->cur_tx;
+	WARN_ON(tx_q->tx_skbuff[first_entry]);
+
+	if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		desc = &tx_q->dma_entx[first_entry].basic;
+	else
+		desc = &tx_q->dma_tx[first_entry];
+	first = desc;
+
+	/* first descriptor: fill Headers on Buf1 */
+	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
+			     DMA_TO_DEVICE);
+	if (dma_mapping_error(priv->device, des))
+		goto dma_map_err;
+
+	stmmac_set_desc_addr(priv, first, des);
+	stmmac_tso_allocator(priv, des + proto_hdr_len, pay_len,
+			     (nfrags == 0), queue);
+
+	/* In case two or more DMA transmit descriptors are allocated for this
+	 * non-paged SKB data, the DMA buffer address should be saved to
+	 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor,
+	 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee
+	 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early
+	 * since the tail areas of the DMA buffer can be accessed by DMA engine
+	 * sooner or later.
+	 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf
+	 * corresponding to the last descriptor, stmmac_tx_clean() will unmap
+	 * this DMA buffer right after the DMA engine completely finishes the
+	 * full buffer transmission.
+	 */
+	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
+	tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb);
+	tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false;
+	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
+
+	/* Prepare fragments */
+	for (i = 0; i < nfrags; i++) {
+		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+		des = skb_frag_dma_map(priv->device, frag, 0,
+				       skb_frag_size(frag),
+				       DMA_TO_DEVICE);
+		if (dma_mapping_error(priv->device, des))
+			goto dma_map_err;
+
+		stmmac_tso_allocator(priv, des, skb_frag_size(frag),
+				     (i == nfrags - 1), queue);
+
+		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
+		tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
+		tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
+		tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
+	}
+
+	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
+
+	/* Only the last descriptor gets to point to the skb. */
+	tx_q->tx_skbuff[tx_q->cur_tx] = skb;
+	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
+
+	/* Manage tx mitigation */
+	tx_packets = (tx_q->cur_tx + 1) - first_tx;
+	tx_q->tx_count_frames += tx_packets;
+
+	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
+		set_ic = true;
+	else if (!priv->tx_coal_frames[queue])
+		set_ic = false;
+	else if (tx_packets > priv->tx_coal_frames[queue])
+		set_ic = true;
+	else if ((tx_q->tx_count_frames %
+		  priv->tx_coal_frames[queue]) < tx_packets)
+		set_ic = true;
+	else
+		set_ic = false;
+
+	if (set_ic) {
+		if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
+		else
+			desc = &tx_q->dma_tx[tx_q->cur_tx];
+
+		tx_q->tx_count_frames = 0;
+		stmmac_set_tx_ic(priv, desc);
+	}
+
+	/* We've used all descriptors we need for this skb, however,
+	 * advance cur_tx so that it references a fresh descriptor.
+	 * ndo_start_xmit will fill this descriptor the next time it's
+	 * called and stmmac_tx_clean may clean up to this descriptor.
+	 */
+	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
+
+	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
+		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
+			  __func__);
+		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
+	}
+
+	u64_stats_update_begin(&txq_stats->q_syncp);
+	u64_stats_add(&txq_stats->q.tx_bytes, skb->len);
+	u64_stats_inc(&txq_stats->q.tx_tso_frames);
+	u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags);
+	if (set_ic)
+		u64_stats_inc(&txq_stats->q.tx_set_ic_bit);
+	u64_stats_update_end(&txq_stats->q_syncp);
+
+	if (priv->sarc_type)
+		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
+
+	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
+		     priv->hwts_tx_en)) {
+		/* declare that device is doing timestamping */
+		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+		stmmac_enable_tx_timestamp(priv, first);
+	}
+
+	/* Complete the first descriptor before granting the DMA */
+	stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1,
+				   tx_q->tx_skbuff_dma[first_entry].last_segment,
+				   hdr / 4, (skb->len - proto_hdr_len));
+
+	/* If context desc is used to change MSS */
+	if (mss_desc) {
+		/* Make sure that first descriptor has been completely
+		 * written, including its own bit. This is because MSS is
+		 * actually before first descriptor, so we need to make
+		 * sure that MSS's own bit is the last thing written.
+		 */
+		dma_wmb();
+		stmmac_set_tx_owner(priv, mss_desc);
+	}
+
+	if (netif_msg_pktdata(priv)) {
+		pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
+			__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
+			tx_q->cur_tx, first, nfrags);
+		pr_info(">>> frame to be transmitted: ");
+		print_pkt(skb->data, skb_headlen(skb));
+	}
+
+	netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
+	skb_tx_timestamp(skb);
+
+	stmmac_flush_tx_descriptors(priv, queue);
+	stmmac_tx_timer_arm(priv, queue);
+
+	return NETDEV_TX_OK;
+
+dma_map_err:
+	dev_err(priv->device, "Tx dma map failed\n");
+	if (!get_ecdev(priv))
+		dev_kfree_skb(skb);
+	priv->xstats.tx_dropped++;
+	return NETDEV_TX_OK;
+}
+
+/**
+ * stmmac_has_ip_ethertype() - Check if packet has IP ethertype
+ * @skb: socket buffer to check
+ *
+ * Check if a packet has an ethertype that will trigger the IP header checks
+ * and IP/TCP checksum engine of the stmmac core.
+ *
+ * Return: true if the ethertype can trigger the checksum engine, false
+ * otherwise
+ */
+static bool stmmac_has_ip_ethertype(struct sk_buff *skb)
+{
+	int depth = 0;
+	__be16 proto;
+
+	proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb),
+				    &depth);
+
+	return (depth <= ETH_HLEN) &&
+		(proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6));
+}
+
+/**
+ *  stmmac_xmit - Tx entry point of the driver
+ *  @skb : the socket buffer
+ *  @dev : device pointer
+ *  Description : this is the tx entry point of the driver.
+ *  It programs the chain or the ring and supports oversized frames
+ *  and SG feature.
+ */
+static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	unsigned int first_entry, tx_packets, enh_desc;
+	struct stmmac_priv *priv = netdev_priv(dev);
+	unsigned int nopaged_len = skb_headlen(skb);
+	int i, csum_insertion = 0, is_jumbo = 0;
+	u32 queue = skb_get_queue_mapping(skb);
+	int nfrags = skb_shinfo(skb)->nr_frags;
+	int gso = skb_shinfo(skb)->gso_type;
+	struct stmmac_txq_stats *txq_stats;
+	struct dma_edesc *tbs_desc = NULL;
+	struct dma_desc *desc, *first;
+	struct stmmac_tx_queue *tx_q;
+	bool has_vlan, set_ic;
+	int entry, first_tx;
+	dma_addr_t des;
+	u32 sdu_len;
+
+	tx_q = &priv->dma_conf.tx_queue[queue];
+	txq_stats = &priv->xstats.txq_stats[queue];
+	first_tx = tx_q->cur_tx;
+
+	if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en)
+		stmmac_stop_sw_lpi(priv);
+
+	/* Manage oversized TCP frames for GMAC4 device */
+	if (skb_is_gso(skb) && priv->tso) {
+		if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
+			return stmmac_tso_xmit(skb, dev);
+		if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
+			return stmmac_tso_xmit(skb, dev);
+	}
+
+	if (priv->est && priv->est->enable &&
+	    priv->est->max_sdu[queue]) {
+		sdu_len = skb->len;
+		/* Add VLAN tag length if VLAN tag insertion offload is requested */
+		if (priv->dma_cap.vlins && skb_vlan_tag_present(skb))
+			sdu_len += VLAN_HLEN;
+		if (sdu_len > priv->est->max_sdu[queue]) {
+			priv->xstats.max_sdu_txq_drop[queue]++;
+			goto max_sdu_err;
+		}
+	}
+
+	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
+		if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
+			netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
+								queue));
+			/* This is a hard error, log it. */
+			netdev_err(priv->dev,
+				   "%s: Tx Ring full when queue awake\n",
+				   __func__);
+		}
+		return NETDEV_TX_BUSY;
+	}
+
+	/* Check if VLAN can be inserted by HW */
+	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
+
+	entry = tx_q->cur_tx;
+	first_entry = entry;
+	WARN_ON(tx_q->tx_skbuff[first_entry]);
+
+	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
+	/* DWMAC IPs can be synthesized to support tx coe only for a few tx
+	 * queues. In that case, checksum offloading for those queues that don't
+	 * support tx coe needs to fallback to software checksum calculation.
+	 *
+	 * Packets that won't trigger the COE e.g. most DSA-tagged packets will
+	 * also have to be checksummed in software.
+	 */
+	if (csum_insertion &&
+	    (priv->plat->tx_queues_cfg[queue].coe_unsupported ||
+	     !stmmac_has_ip_ethertype(skb))) {
+		if (unlikely(skb_checksum_help(skb)))
+			goto dma_map_err;
+		csum_insertion = !csum_insertion;
+	}
+
+	if (likely(priv->extend_desc))
+		desc = (struct dma_desc *)(tx_q->dma_etx + entry);
+	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		desc = &tx_q->dma_entx[entry].basic;
+	else
+		desc = tx_q->dma_tx + entry;
+
+	first = desc;
+
+	if (has_vlan)
+		stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
+
+	enh_desc = priv->plat->enh_desc;
+	/* To program the descriptors according to the size of the frame */
+	if (enh_desc)
+		is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
+
+	if (unlikely(is_jumbo)) {
+		entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
+		if (unlikely(entry < 0) && (entry != -EINVAL))
+			goto dma_map_err;
+	}
+
+	for (i = 0; i < nfrags; i++) {
+		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		int len = skb_frag_size(frag);
+		bool last_segment = (i == (nfrags - 1));
+
+		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+		WARN_ON(tx_q->tx_skbuff[entry]);
+
+		if (likely(priv->extend_desc))
+			desc = (struct dma_desc *)(tx_q->dma_etx + entry);
+		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			desc = &tx_q->dma_entx[entry].basic;
+		else
+			desc = tx_q->dma_tx + entry;
+
+		des = skb_frag_dma_map(priv->device, frag, 0, len,
+				       DMA_TO_DEVICE);
+		if (dma_mapping_error(priv->device, des))
+			goto dma_map_err; /* should reuse desc w/o issues */
+
+		tx_q->tx_skbuff_dma[entry].buf = des;
+
+		stmmac_set_desc_addr(priv, desc, des);
+
+		tx_q->tx_skbuff_dma[entry].map_as_page = true;
+		tx_q->tx_skbuff_dma[entry].len = len;
+		tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
+		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
+
+		/* Prepare the descriptor and set the own bit too */
+		stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
+				priv->mode, 1, last_segment, skb->len);
+	}
+
+	/* Only the last descriptor gets to point to the skb. */
+	tx_q->tx_skbuff[entry] = skb;
+	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
+
+	/* According to the coalesce parameter the IC bit for the latest
+	 * segment is reset and the timer re-started to clean the tx status.
+	 * This approach takes care about the fragments: desc is the first
+	 * element in case of no SG.
+	 */
+	tx_packets = (entry + 1) - first_tx;
+	tx_q->tx_count_frames += tx_packets;
+
+	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
+		set_ic = true;
+	else if (!priv->tx_coal_frames[queue])
+		set_ic = false;
+	else if (tx_packets > priv->tx_coal_frames[queue])
+		set_ic = true;
+	else if ((tx_q->tx_count_frames %
+		  priv->tx_coal_frames[queue]) < tx_packets)
+		set_ic = true;
+	else
+		set_ic = false;
+
+	if (set_ic) {
+		if (likely(priv->extend_desc))
+			desc = &tx_q->dma_etx[entry].basic;
+		else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+			desc = &tx_q->dma_entx[entry].basic;
+		else
+			desc = &tx_q->dma_tx[entry];
+
+		tx_q->tx_count_frames = 0;
+		stmmac_set_tx_ic(priv, desc);
+	}
+
+	/* We've used all descriptors we need for this skb, however,
+	 * advance cur_tx so that it references a fresh descriptor.
+	 * ndo_start_xmit will fill this descriptor the next time it's
+	 * called and stmmac_tx_clean may clean up to this descriptor.
+	 */
+	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+	tx_q->cur_tx = entry;
+
+	if (netif_msg_pktdata(priv)) {
+		netdev_dbg(priv->dev,
+			   "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
+			   __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
+			   entry, first, nfrags);
+
+		netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
+		print_pkt(skb->data, skb->len);
+	}
+
+	if ((!get_ecdev(priv)) && (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1)))) {
+		netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
+			  __func__);
+		netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
+	}
+
+	u64_stats_update_begin(&txq_stats->q_syncp);
+	u64_stats_add(&txq_stats->q.tx_bytes, skb->len);
+	if (set_ic)
+		u64_stats_inc(&txq_stats->q.tx_set_ic_bit);
+	u64_stats_update_end(&txq_stats->q_syncp);
+
+	if (priv->sarc_type)
+		stmmac_set_desc_sarc(priv, first, priv->sarc_type);
+
+	/* Ready to fill the first descriptor and set the OWN bit w/o any
+	 * problems because all the descriptors are actually ready to be
+	 * passed to the DMA engine.
+	 */
+	if (likely(!is_jumbo)) {
+		bool last_segment = (nfrags == 0);
+
+		des = dma_map_single(priv->device, skb->data,
+				     nopaged_len, DMA_TO_DEVICE);
+		if (dma_mapping_error(priv->device, des))
+			goto dma_map_err;
+
+		tx_q->tx_skbuff_dma[first_entry].buf = des;
+		tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB;
+		tx_q->tx_skbuff_dma[first_entry].map_as_page = false;
+
+		stmmac_set_desc_addr(priv, first, des);
+
+		tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
+		tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
+
+		if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
+			     priv->hwts_tx_en)) {
+			/* declare that device is doing timestamping */
+			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+			stmmac_enable_tx_timestamp(priv, first);
+		}
+
+		/* Prepare the first descriptor setting the OWN bit too */
+		stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
+				csum_insertion, priv->mode, 0, last_segment,
+				skb->len);
+	}
+
+	if (tx_q->tbs & STMMAC_TBS_EN) {
+		struct timespec64 ts = ns_to_timespec64(skb->tstamp);
+
+		tbs_desc = &tx_q->dma_entx[first_entry];
+		stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
+	}
+
+	stmmac_set_tx_owner(priv, first);
+
+	if (!get_ecdev(priv))
+		netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
+
+	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
+	skb_tx_timestamp(skb);
+	stmmac_flush_tx_descriptors(priv, queue);
+	stmmac_tx_timer_arm(priv, queue);
+
+	return NETDEV_TX_OK;
+
+dma_map_err:
+	netdev_err(priv->dev, "Tx DMA map failed\n");
+max_sdu_err:
+	if (!get_ecdev(priv))
+		dev_kfree_skb(skb);
+	priv->xstats.tx_dropped++;
+	return NETDEV_TX_OK;
+}
+
+static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
+{
+	struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb);
+	__be16 vlan_proto = veth->h_vlan_proto;
+	u16 vlanid;
+
+	if ((vlan_proto == htons(ETH_P_8021Q) &&
+	     dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
+	    (vlan_proto == htons(ETH_P_8021AD) &&
+	     dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
+		/* pop the vlan tag */
+		vlanid = ntohs(veth->h_vlan_TCI);
+		memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
+		skb_pull(skb, VLAN_HLEN);
+		__vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
+	}
+}
+
+/**
+ * stmmac_rx_refill - refill used skb preallocated buffers
+ * @priv: driver private structure
+ * @queue: RX queue index
+ * Description : this is to reallocate the skb for the reception process
+ * that is based on zero-copy.
+ */
+static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+	int dirty = stmmac_rx_dirty(priv, queue);
+	unsigned int entry = rx_q->dirty_rx;
+	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
+
+	if (priv->dma_cap.host_dma_width <= 32)
+		gfp |= GFP_DMA32;
+
+	while (dirty-- > 0) {
+		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
+		struct dma_desc *p;
+		bool use_rx_wd;
+
+		if (priv->extend_desc)
+			p = (struct dma_desc *)(rx_q->dma_erx + entry);
+		else
+			p = rx_q->dma_rx + entry;
+
+		if (!buf->page) {
+			buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp);
+			if (!buf->page)
+				break;
+		}
+
+		if (priv->sph && !buf->sec_page) {
+			buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
+			if (!buf->sec_page)
+				break;
+
+			buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
+		}
+
+		buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
+
+		stmmac_set_desc_addr(priv, p, buf->addr);
+		if (priv->sph)
+			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
+		else
+			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
+		stmmac_refill_desc3(priv, rx_q, p);
+
+		rx_q->rx_count_frames++;
+		rx_q->rx_count_frames += priv->rx_coal_frames[queue];
+		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
+			rx_q->rx_count_frames = 0;
+
+		use_rx_wd = !priv->rx_coal_frames[queue];
+		use_rx_wd |= rx_q->rx_count_frames > 0;
+		if (!priv->use_riwt)
+			use_rx_wd = false;
+
+		dma_wmb();
+		stmmac_set_rx_owner(priv, p, use_rx_wd);
+
+		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
+	}
+	rx_q->dirty_rx = entry;
+	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
+			    (rx_q->dirty_rx * sizeof(struct dma_desc));
+	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
+}
+
+static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
+				       struct dma_desc *p,
+				       int status, unsigned int len)
+{
+	unsigned int plen = 0, hlen = 0;
+	int coe = priv->hw->rx_csum;
+
+	/* Not first descriptor, buffer is always zero */
+	if (priv->sph && len)
+		return 0;
+
+	/* First descriptor, get split header length */
+	stmmac_get_rx_header_len(priv, p, &hlen);
+	if (priv->sph && hlen) {
+		priv->xstats.rx_split_hdr_pkt_n++;
+		return hlen;
+	}
+
+	/* First descriptor, not last descriptor and not split header */
+	if (status & rx_not_ls)
+		return priv->dma_conf.dma_buf_sz;
+
+	plen = stmmac_get_rx_frame_len(priv, p, coe);
+
+	/* First descriptor and last descriptor and not split header */
+	return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen);
+}
+
+static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
+				       struct dma_desc *p,
+				       int status, unsigned int len)
+{
+	int coe = priv->hw->rx_csum;
+	unsigned int plen = 0;
+
+	/* Not split header, buffer is not available */
+	if (!priv->sph)
+		return 0;
+
+	/* Not last descriptor */
+	if (status & rx_not_ls)
+		return priv->dma_conf.dma_buf_sz;
+
+	plen = stmmac_get_rx_frame_len(priv, p, coe);
+
+	/* Last descriptor */
+	return plen - len;
+}
+
+static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
+				struct xdp_frame *xdpf, bool dma_map)
+{
+	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported;
+	unsigned int entry = tx_q->cur_tx;
+	struct dma_desc *tx_desc;
+	dma_addr_t dma_addr;
+	bool set_ic;
+
+	if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv))
+		return STMMAC_XDP_CONSUMED;
+
+	if (priv->est && priv->est->enable &&
+	    priv->est->max_sdu[queue] &&
+	    xdpf->len > priv->est->max_sdu[queue]) {
+		priv->xstats.max_sdu_txq_drop[queue]++;
+		return STMMAC_XDP_CONSUMED;
+	}
+
+	if (likely(priv->extend_desc))
+		tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
+	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		tx_desc = &tx_q->dma_entx[entry].basic;
+	else
+		tx_desc = tx_q->dma_tx + entry;
+
+	if (dma_map) {
+		dma_addr = dma_map_single(priv->device, xdpf->data,
+					  xdpf->len, DMA_TO_DEVICE);
+		if (dma_mapping_error(priv->device, dma_addr))
+			return STMMAC_XDP_CONSUMED;
+
+		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO;
+	} else {
+		struct page *page = virt_to_page(xdpf->data);
+
+		dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) +
+			   xdpf->headroom;
+		dma_sync_single_for_device(priv->device, dma_addr,
+					   xdpf->len, DMA_BIDIRECTIONAL);
+
+		tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX;
+	}
+
+	tx_q->tx_skbuff_dma[entry].buf = dma_addr;
+	tx_q->tx_skbuff_dma[entry].map_as_page = false;
+	tx_q->tx_skbuff_dma[entry].len = xdpf->len;
+	tx_q->tx_skbuff_dma[entry].last_segment = true;
+	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
+
+	tx_q->xdpf[entry] = xdpf;
+
+	stmmac_set_desc_addr(priv, tx_desc, dma_addr);
+
+	stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len,
+			       csum, priv->mode, true, true,
+			       xdpf->len);
+
+	tx_q->tx_count_frames++;
+
+	if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
+		set_ic = true;
+	else
+		set_ic = false;
+
+	if (set_ic) {
+		tx_q->tx_count_frames = 0;
+		stmmac_set_tx_ic(priv, tx_desc);
+		u64_stats_update_begin(&txq_stats->q_syncp);
+		u64_stats_inc(&txq_stats->q.tx_set_ic_bit);
+		u64_stats_update_end(&txq_stats->q_syncp);
+	}
+
+	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
+
+	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
+	tx_q->cur_tx = entry;
+
+	return STMMAC_XDP_TX;
+}
+
+static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv,
+				   int cpu)
+{
+	int index = cpu;
+
+	if (unlikely(index < 0))
+		index = 0;
+
+	while (index >= priv->plat->tx_queues_to_use)
+		index -= priv->plat->tx_queues_to_use;
+
+	return index;
+}
+
+static int stmmac_xdp_xmit_back(struct stmmac_priv *priv,
+				struct xdp_buff *xdp)
+{
+	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
+	int cpu = smp_processor_id();
+	struct netdev_queue *nq;
+	int queue;
+	int res;
+
+	if (unlikely(!xdpf))
+		return STMMAC_XDP_CONSUMED;
+
+	queue = stmmac_xdp_get_tx_queue(priv, cpu);
+	nq = netdev_get_tx_queue(priv->dev, queue);
+
+	__netif_tx_lock(nq, cpu);
+	/* Avoids TX time-out as we are sharing with slow path */
+	ec_txq_trans_cond_update(nq);
+
+	res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false);
+	if (res == STMMAC_XDP_TX)
+		stmmac_flush_tx_descriptors(priv, queue);
+
+	__netif_tx_unlock(nq);
+
+	return res;
+}
+
+static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,
+				 struct bpf_prog *prog,
+				 struct xdp_buff *xdp)
+{
+	u32 act;
+	int res;
+
+	act = bpf_prog_run_xdp(prog, xdp);
+	switch (act) {
+	case XDP_PASS:
+		res = STMMAC_XDP_PASS;
+		break;
+	case XDP_TX:
+		res = stmmac_xdp_xmit_back(priv, xdp);
+		break;
+	case XDP_REDIRECT:
+		if (xdp_do_redirect(priv->dev, xdp, prog) < 0)
+			res = STMMAC_XDP_CONSUMED;
+		else
+			res = STMMAC_XDP_REDIRECT;
+		break;
+	default:
+		bpf_warn_invalid_xdp_action(priv->dev, prog, act);
+		fallthrough;
+	case XDP_ABORTED:
+		trace_xdp_exception(priv->dev, prog, act);
+		fallthrough;
+	case XDP_DROP:
+		res = STMMAC_XDP_CONSUMED;
+		break;
+	}
+
+	return res;
+}
+
+static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv,
+					   struct xdp_buff *xdp)
+{
+	struct bpf_prog *prog;
+	int res;
+
+	prog = READ_ONCE(priv->xdp_prog);
+	if (!prog) {
+		res = STMMAC_XDP_PASS;
+		goto out;
+	}
+
+	res = __stmmac_xdp_run_prog(priv, prog, xdp);
+out:
+	return ERR_PTR(-res);
+}
+
+static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv,
+				   int xdp_status)
+{
+	int cpu = smp_processor_id();
+	int queue;
+
+	if(get_ecdev(priv)) {
+		return;
+	}
+	cpu = smp_processor_id();
+
+	queue = stmmac_xdp_get_tx_queue(priv, cpu);
+
+	if (xdp_status & STMMAC_XDP_TX)
+		stmmac_tx_timer_arm(priv, queue);
+
+	if (xdp_status & STMMAC_XDP_REDIRECT)
+		xdp_do_flush();
+}
+
+static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch,
+					       struct xdp_buff *xdp)
+{
+	unsigned int metasize = xdp->data - xdp->data_meta;
+	unsigned int datasize = xdp->data_end - xdp->data;
+	struct sk_buff *skb;
+
+	skb = napi_alloc_skb(&ch->rxtx_napi,
+			     xdp->data_end - xdp->data_hard_start);
+	if (unlikely(!skb))
+		return NULL;
+
+	skb_reserve(skb, xdp->data - xdp->data_hard_start);
+	memcpy(__skb_put(skb, datasize), xdp->data, datasize);
+	if (metasize)
+		skb_metadata_set(skb, metasize);
+
+	return skb;
+}
+
+static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,
+				   struct dma_desc *p, struct dma_desc *np,
+				   struct xdp_buff *xdp)
+{
+	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];
+	struct stmmac_channel *ch = &priv->channel[queue];
+	unsigned int len = xdp->data_end - xdp->data;
+	enum pkt_hash_types hash_type;
+	int coe = priv->hw->rx_csum;
+	struct sk_buff *skb;
+	u32 hash;
+
+	skb = stmmac_construct_skb_zc(ch, xdp);
+	if (!skb) {
+		priv->xstats.rx_dropped++;
+		return;
+	}
+
+	stmmac_get_rx_hwtstamp(priv, p, np, skb);
+	if (priv->hw->hw_vlan_en)
+		/* MAC level stripping. */
+		stmmac_rx_hw_vlan(priv, priv->hw, p, skb);
+	else
+		/* Driver level stripping. */
+		stmmac_rx_vlan(priv->dev, skb);
+	skb->protocol = eth_type_trans(skb, priv->dev);
+
+	if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb))
+		skb_checksum_none_assert(skb);
+	else
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+	if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
+		skb_set_hash(skb, hash, hash_type);
+
+	skb_record_rx_queue(skb, queue);
+	napi_gro_receive(&ch->rxtx_napi, skb);
+
+	u64_stats_update_begin(&rxq_stats->napi_syncp);
+	u64_stats_inc(&rxq_stats->napi.rx_pkt_n);
+	u64_stats_add(&rxq_stats->napi.rx_bytes, len);
+	u64_stats_update_end(&rxq_stats->napi_syncp);
+}
+
+static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
+{
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+	unsigned int entry = rx_q->dirty_rx;
+	struct dma_desc *rx_desc = NULL;
+	bool ret = true;
+
+	budget = min(budget, stmmac_rx_dirty(priv, queue));
+
+	while (budget-- > 0 && entry != rx_q->cur_rx) {
+		struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
+		dma_addr_t dma_addr;
+		bool use_rx_wd;
+
+		if (!buf->xdp) {
+			buf->xdp = xsk_buff_alloc(rx_q->xsk_pool);
+			if (!buf->xdp) {
+				ret = false;
+				break;
+			}
+		}
+
+		if (priv->extend_desc)
+			rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry);
+		else
+			rx_desc = rx_q->dma_rx + entry;
+
+		dma_addr = xsk_buff_xdp_get_dma(buf->xdp);
+		stmmac_set_desc_addr(priv, rx_desc, dma_addr);
+		stmmac_set_desc_sec_addr(priv, rx_desc, 0, false);
+		stmmac_refill_desc3(priv, rx_q, rx_desc);
+
+		rx_q->rx_count_frames++;
+		rx_q->rx_count_frames += priv->rx_coal_frames[queue];
+		if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
+			rx_q->rx_count_frames = 0;
+
+		use_rx_wd = !priv->rx_coal_frames[queue];
+		use_rx_wd |= rx_q->rx_count_frames > 0;
+		if (!priv->use_riwt)
+			use_rx_wd = false;
+
+		dma_wmb();
+		stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);
+
+		entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
+	}
+
+	if (rx_desc) {
+		rx_q->dirty_rx = entry;
+		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
+				     (rx_q->dirty_rx * sizeof(struct dma_desc));
+		stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
+	}
+
+	return ret;
+}
+
+static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp)
+{
+	/* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used
+	 * to represent incoming packet, whereas cb field in the same structure
+	 * is used to store driver specific info. Thus, struct stmmac_xdp_buff
+	 * is laid on top of xdp and cb fields of struct xdp_buff_xsk.
+	 */
+	return (struct stmmac_xdp_buff *)xdp;
+}
+
+static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
+{
+	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+	unsigned int count = 0, error = 0, len = 0;
+	int dirty = stmmac_rx_dirty(priv, queue);
+	unsigned int next_entry = rx_q->cur_rx;
+	u32 rx_errors = 0, rx_dropped = 0;
+	unsigned int desc_size;
+	struct bpf_prog *prog;
+	bool failure = false;
+	int xdp_status = 0;
+	int status = 0;
+
+	if (netif_msg_rx_status(priv)) {
+		void *rx_head;
+
+		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
+		if (priv->extend_desc) {
+			rx_head = (void *)rx_q->dma_erx;
+			desc_size = sizeof(struct dma_extended_desc);
+		} else {
+			rx_head = (void *)rx_q->dma_rx;
+			desc_size = sizeof(struct dma_desc);
+		}
+
+		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
+				    rx_q->dma_rx_phy, desc_size);
+	}
+	while (count < limit) {
+		struct stmmac_rx_buffer *buf;
+		struct stmmac_xdp_buff *ctx;
+		unsigned int buf1_len = 0;
+		struct dma_desc *np, *p;
+		int entry;
+		int res;
+
+		if (!count && rx_q->state_saved) {
+			error = rx_q->state.error;
+			len = rx_q->state.len;
+		} else {
+			rx_q->state_saved = false;
+			error = 0;
+			len = 0;
+		}
+
+read_again:
+		if (count >= limit)
+			break;
+
+		buf1_len = 0;
+		entry = next_entry;
+		buf = &rx_q->buf_pool[entry];
+
+		if (dirty >= STMMAC_RX_FILL_BATCH) {
+			failure = failure ||
+				  !stmmac_rx_refill_zc(priv, queue, dirty);
+			dirty = 0;
+		}
+
+		if (priv->extend_desc)
+			p = (struct dma_desc *)(rx_q->dma_erx + entry);
+		else
+			p = rx_q->dma_rx + entry;
+
+		/* read the status of the incoming frame */
+		status = stmmac_rx_status(priv, &priv->xstats, p);
+		/* check if managed by the DMA otherwise go ahead */
+		if (unlikely(status & dma_own))
+			break;
+
+		/* Prefetch the next RX descriptor */
+		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
+						priv->dma_conf.dma_rx_size);
+		next_entry = rx_q->cur_rx;
+
+		if (priv->extend_desc)
+			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
+		else
+			np = rx_q->dma_rx + next_entry;
+
+		prefetch(np);
+
+		/* Ensure a valid XSK buffer before proceed */
+		if (!buf->xdp)
+			break;
+
+		if (priv->extend_desc)
+			stmmac_rx_extended_status(priv, &priv->xstats,
+						  rx_q->dma_erx + entry);
+		if (unlikely(status == discard_frame)) {
+			xsk_buff_free(buf->xdp);
+			buf->xdp = NULL;
+			dirty++;
+			error = 1;
+			if (!priv->hwts_rx_en)
+				rx_errors++;
+		}
+
+		if (unlikely(error && (status & rx_not_ls)))
+			goto read_again;
+		if (unlikely(error)) {
+			count++;
+			continue;
+		}
+
+		/* XSK pool expects RX frame 1:1 mapped to XSK buffer */
+		if (likely(status & rx_not_ls)) {
+			xsk_buff_free(buf->xdp);
+			buf->xdp = NULL;
+			dirty++;
+			count++;
+			goto read_again;
+		}
+
+		ctx = xsk_buff_to_stmmac_ctx(buf->xdp);
+		ctx->priv = priv;
+		ctx->desc = p;
+		ctx->ndesc = np;
+
+		/* XDP ZC Frame only support primary buffers for now */
+		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
+		len += buf1_len;
+
+		/* ACS is disabled; strip manually. */
+		if (likely(!(status & rx_not_ls))) {
+			buf1_len -= ETH_FCS_LEN;
+			len -= ETH_FCS_LEN;
+		}
+
+		/* RX buffer is good and fit into a XSK pool buffer */
+		buf->xdp->data_end = buf->xdp->data + buf1_len;
+		xsk_buff_dma_sync_for_cpu(buf->xdp);
+
+		prog = READ_ONCE(priv->xdp_prog);
+		res = __stmmac_xdp_run_prog(priv, prog, buf->xdp);
+
+		switch (res) {
+		case STMMAC_XDP_PASS:
+			stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp);
+			xsk_buff_free(buf->xdp);
+			break;
+		case STMMAC_XDP_CONSUMED:
+			xsk_buff_free(buf->xdp);
+			rx_dropped++;
+			break;
+		case STMMAC_XDP_TX:
+		case STMMAC_XDP_REDIRECT:
+			xdp_status |= res;
+			break;
+		}
+
+		buf->xdp = NULL;
+		dirty++;
+		count++;
+	}
+
+	if (status & rx_not_ls) {
+		rx_q->state_saved = true;
+		rx_q->state.error = error;
+		rx_q->state.len = len;
+	}
+
+	stmmac_finalize_xdp_rx(priv, xdp_status);
+
+	u64_stats_update_begin(&rxq_stats->napi_syncp);
+	u64_stats_add(&rxq_stats->napi.rx_pkt_n, count);
+	u64_stats_update_end(&rxq_stats->napi_syncp);
+
+	priv->xstats.rx_dropped += rx_dropped;
+	priv->xstats.rx_errors += rx_errors;
+
+	if (xsk_uses_need_wakeup(rx_q->xsk_pool)) {
+		if (failure || stmmac_rx_dirty(priv, queue) > 0)
+			xsk_set_rx_need_wakeup(rx_q->xsk_pool);
+		else
+			xsk_clear_rx_need_wakeup(rx_q->xsk_pool);
+
+		return (int)count;
+	}
+
+	return failure ? limit : (int)count;
+}
+
+/**
+ * stmmac_rx - manage the receive process
+ * @priv: driver private structure
+ * @limit: napi bugget
+ * @queue: RX queue index.
+ * Description :  this the function called by the napi poll method.
+ * It gets all the frames inside the ring.
+ */
+static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
+{
+	u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0;
+	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+	struct stmmac_channel *ch = &priv->channel[queue];
+	unsigned int count = 0, error = 0, len = 0;
+	int status = 0, coe = priv->hw->rx_csum;
+	unsigned int next_entry = rx_q->cur_rx;
+	enum dma_data_direction dma_dir;
+	unsigned int desc_size;
+	struct sk_buff *skb = NULL;
+	struct stmmac_xdp_buff ctx;
+	int xdp_status = 0;
+	int bufsz;
+
+	dma_dir = page_pool_get_dma_dir(rx_q->page_pool);
+	bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
+	limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit);
+
+	if (netif_msg_rx_status(priv)) {
+		void *rx_head;
+
+		netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
+		if (priv->extend_desc) {
+			rx_head = (void *)rx_q->dma_erx;
+			desc_size = sizeof(struct dma_extended_desc);
+		} else {
+			rx_head = (void *)rx_q->dma_rx;
+			desc_size = sizeof(struct dma_desc);
+		}
+
+		stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
+				    rx_q->dma_rx_phy, desc_size);
+	}
+	while (count < limit) {
+		unsigned int buf1_len = 0, buf2_len = 0;
+		enum pkt_hash_types hash_type;
+		struct stmmac_rx_buffer *buf;
+		struct dma_desc *np, *p;
+		int entry;
+		u32 hash;
+
+		if (!count && rx_q->state_saved) {
+			skb = rx_q->state.skb;
+			error = rx_q->state.error;
+			len = rx_q->state.len;
+		} else {
+			rx_q->state_saved = false;
+			skb = NULL;
+			error = 0;
+			len = 0;
+		}
+
+read_again:
+		if (count >= limit)
+			break;
+
+		buf1_len = 0;
+		buf2_len = 0;
+		entry = next_entry;
+		buf = &rx_q->buf_pool[entry];
+
+		if (priv->extend_desc)
+			p = (struct dma_desc *)(rx_q->dma_erx + entry);
+		else
+			p = rx_q->dma_rx + entry;
+
+		/* read the status of the incoming frame */
+		status = stmmac_rx_status(priv, &priv->xstats, p);
+		/* check if managed by the DMA otherwise go ahead */
+		if (unlikely(status & dma_own))
+			break;
+
+		rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
+						priv->dma_conf.dma_rx_size);
+		next_entry = rx_q->cur_rx;
+
+		if (priv->extend_desc)
+			np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
+		else
+			np = rx_q->dma_rx + next_entry;
+
+		prefetch(np);
+
+		if (priv->extend_desc)
+			stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry);
+		if (unlikely(status == discard_frame)) {
+			if (!get_ecdev(priv)) {
+				page_pool_put_page(rx_q->page_pool, buf->page, 0, true);
+				buf->page = NULL;
+			}
+			error = 1;
+			if (!priv->hwts_rx_en)
+				rx_errors++;
+		}
+
+		if (unlikely(error && (status & rx_not_ls)))
+			goto read_again;
+		if (unlikely(error)) {
+			dev_kfree_skb(skb);
+			skb = NULL;
+			count++;
+			continue;
+		}
+
+		/* Buffer is good. Go on. */
+
+		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
+		len += buf1_len;
+		buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
+		len += buf2_len;
+
+		/* ACS is disabled; strip manually. */
+		if (likely(!(status & rx_not_ls))) {
+			if (buf2_len) {
+				buf2_len -= ETH_FCS_LEN;
+				len -= ETH_FCS_LEN;
+			} else if (buf1_len) {
+				buf1_len -= ETH_FCS_LEN;
+				len -= ETH_FCS_LEN;
+			}
+		}
+
+		if (get_ecdev(priv)) {
+			unsigned char *va;
+			dma_sync_single_for_cpu(priv->device, buf->addr,
+						buf1_len, dma_dir);
+			va = page_address(buf->page) + buf->page_offset;
+			ecdev_receive(get_ecdev(priv), va, len);
+			netdev_dbg(priv->dev, "ecdev_receive: %u", buf1_len);
+			priv->ec_watchdog_jiffies = jiffies;
+			/* keep the page and pass it back to the device manually */
+			dma_sync_single_for_device(priv->device, buf->addr,
+							buf1_len, dma_dir);
+			count++;
+			continue;
+		}
+
+		if (!skb) {
+			unsigned int pre_len, sync_len;
+
+			dma_sync_single_for_cpu(priv->device, buf->addr,
+						buf1_len, dma_dir);
+			net_prefetch(page_address(buf->page) +
+				     buf->page_offset);
+
+			xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq);
+			xdp_prepare_buff(&ctx.xdp, page_address(buf->page),
+					 buf->page_offset, buf1_len, true);
+
+			pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
+				  buf->page_offset;
+
+			ctx.priv = priv;
+			ctx.desc = p;
+			ctx.ndesc = np;
+
+			skb = stmmac_xdp_run_prog(priv, &ctx.xdp);
+			/* Due xdp_adjust_tail: DMA sync for_device
+			 * cover max len CPU touch
+			 */
+			sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
+				   buf->page_offset;
+			sync_len = max(sync_len, pre_len);
+
+			/* For Not XDP_PASS verdict */
+			if (IS_ERR(skb)) {
+				unsigned int xdp_res = -PTR_ERR(skb);
+
+				if (xdp_res & STMMAC_XDP_CONSUMED) {
+					page_pool_put_page(rx_q->page_pool,
+							   virt_to_head_page(ctx.xdp.data),
+							   sync_len, true);
+					buf->page = NULL;
+					rx_dropped++;
+
+					/* Clear skb as it was set as
+					 * status by XDP program.
+					 */
+					skb = NULL;
+
+					if (unlikely((status & rx_not_ls)))
+						goto read_again;
+
+					count++;
+					continue;
+				} else if (xdp_res & (STMMAC_XDP_TX |
+						      STMMAC_XDP_REDIRECT)) {
+					xdp_status |= xdp_res;
+					buf->page = NULL;
+					skb = NULL;
+					count++;
+					continue;
+				}
+			}
+		}
+
+		if (!skb) {
+			unsigned int head_pad_len;
+
+			/* XDP program may expand or reduce tail */
+			buf1_len = ctx.xdp.data_end - ctx.xdp.data;
+
+			skb = napi_build_skb(page_address(buf->page),
+					     rx_q->napi_skb_frag_size);
+			if (!skb) {
+				page_pool_recycle_direct(rx_q->page_pool,
+							 buf->page);
+				rx_dropped++;
+				count++;
+				goto drain_data;
+			}
+
+			/* XDP program may adjust header */
+			head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start;
+			skb_reserve(skb, head_pad_len);
+			skb_put(skb, buf1_len);
+			skb_mark_for_recycle(skb);
+			buf->page = NULL;
+		} else if (buf1_len) {
+			dma_sync_single_for_cpu(priv->device, buf->addr,
+						buf1_len, dma_dir);
+			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+					buf->page, buf->page_offset, buf1_len,
+					priv->dma_conf.dma_buf_sz);
+			buf->page = NULL;
+		}
+
+		if (buf2_len) {
+			dma_sync_single_for_cpu(priv->device, buf->sec_addr,
+						buf2_len, dma_dir);
+			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+					buf->sec_page, 0, buf2_len,
+					priv->dma_conf.dma_buf_sz);
+			buf->sec_page = NULL;
+		}
+
+drain_data:
+		if (likely(status & rx_not_ls))
+			goto read_again;
+		if (!skb)
+			continue;
+
+		/* Got entire packet into SKB. Finish it. */
+
+		stmmac_get_rx_hwtstamp(priv, p, np, skb);
+
+		if (priv->hw->hw_vlan_en)
+			/* MAC level stripping. */
+			stmmac_rx_hw_vlan(priv, priv->hw, p, skb);
+		else
+			/* Driver level stripping. */
+			stmmac_rx_vlan(priv->dev, skb);
+
+		skb->protocol = eth_type_trans(skb, priv->dev);
+
+		if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) ||
+		    (status & csum_none))
+			skb_checksum_none_assert(skb);
+		else
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+		if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
+			skb_set_hash(skb, hash, hash_type);
+
+		skb_record_rx_queue(skb, queue);
+		napi_gro_receive(&ch->rx_napi, skb);
+		skb = NULL;
+
+		rx_packets++;
+		rx_bytes += len;
+		count++;
+	}
+
+	if (status & rx_not_ls || skb) {
+		rx_q->state_saved = true;
+		rx_q->state.skb = skb;
+		rx_q->state.error = error;
+		rx_q->state.len = len;
+	}
+
+	stmmac_finalize_xdp_rx(priv, xdp_status);
+
+	stmmac_rx_refill(priv, queue);
+
+	u64_stats_update_begin(&rxq_stats->napi_syncp);
+	u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets);
+	u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes);
+	u64_stats_add(&rxq_stats->napi.rx_pkt_n, count);
+	u64_stats_update_end(&rxq_stats->napi_syncp);
+
+	priv->xstats.rx_dropped += rx_dropped;
+	priv->xstats.rx_errors += rx_errors;
+
+	return count;
+}
+
+static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
+{
+	struct stmmac_channel *ch =
+		container_of(napi, struct stmmac_channel, rx_napi);
+	struct stmmac_priv *priv = ch->priv_data;
+	struct stmmac_rxq_stats *rxq_stats;
+	u32 chan = ch->index;
+	int work_done;
+
+	BUG_ON(get_ecdev(priv));
+
+	rxq_stats = &priv->xstats.rxq_stats[chan];
+	u64_stats_update_begin(&rxq_stats->napi_syncp);
+	u64_stats_inc(&rxq_stats->napi.poll);
+	u64_stats_update_end(&rxq_stats->napi_syncp);
+
+	work_done = stmmac_rx(priv, budget, chan);
+	if (work_done < budget && napi_complete_done(napi, work_done)) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&ch->lock, flags);
+		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
+		spin_unlock_irqrestore(&ch->lock, flags);
+	}
+
+	return work_done;
+}
+
+static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
+{
+	struct stmmac_channel *ch =
+		container_of(napi, struct stmmac_channel, tx_napi);
+	struct stmmac_priv *priv = ch->priv_data;
+	struct stmmac_txq_stats *txq_stats;
+	bool pending_packets = false;
+	u32 chan = ch->index;
+	int work_done;
+
+	BUG_ON(get_ecdev(priv));
+
+	txq_stats = &priv->xstats.txq_stats[chan];
+	u64_stats_update_begin(&txq_stats->napi_syncp);
+	u64_stats_inc(&txq_stats->napi.poll);
+	u64_stats_update_end(&txq_stats->napi_syncp);
+
+	work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets);
+	work_done = min(work_done, budget);
+
+	if (work_done < budget && napi_complete_done(napi, work_done)) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&ch->lock, flags);
+		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
+		spin_unlock_irqrestore(&ch->lock, flags);
+	}
+
+	/* TX still have packet to handle, check if we need to arm tx timer */
+	if (pending_packets)
+		stmmac_tx_timer_arm(priv, chan);
+
+	return work_done;
+}
+
+static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget)
+{
+	struct stmmac_channel *ch =
+		container_of(napi, struct stmmac_channel, rxtx_napi);
+	struct stmmac_priv *priv = ch->priv_data;
+	bool tx_pending_packets = false;
+	int rx_done, tx_done, rxtx_done;
+	struct stmmac_rxq_stats *rxq_stats;
+	struct stmmac_txq_stats *txq_stats;
+	u32 chan = ch->index;
+
+	BUG_ON(get_ecdev(priv));
+
+	rxq_stats = &priv->xstats.rxq_stats[chan];
+	u64_stats_update_begin(&rxq_stats->napi_syncp);
+	u64_stats_inc(&rxq_stats->napi.poll);
+	u64_stats_update_end(&rxq_stats->napi_syncp);
+
+	txq_stats = &priv->xstats.txq_stats[chan];
+	u64_stats_update_begin(&txq_stats->napi_syncp);
+	u64_stats_inc(&txq_stats->napi.poll);
+	u64_stats_update_end(&txq_stats->napi_syncp);
+
+	tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets);
+	tx_done = min(tx_done, budget);
+
+	rx_done = stmmac_rx_zc(priv, budget, chan);
+
+	rxtx_done = max(tx_done, rx_done);
+
+	/* If either TX or RX work is not complete, return budget
+	 * and keep pooling
+	 */
+	if (rxtx_done >= budget)
+		return budget;
+
+	/* all work done, exit the polling mode */
+	if (napi_complete_done(napi, rxtx_done)) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&ch->lock, flags);
+		/* Both RX and TX work done are compelte,
+		 * so enable both RX & TX IRQs.
+		 */
+		stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
+		spin_unlock_irqrestore(&ch->lock, flags);
+	}
+
+	/* TX still have packet to handle, check if we need to arm tx timer */
+	if (tx_pending_packets)
+		stmmac_tx_timer_arm(priv, chan);
+
+	return min(rxtx_done, budget - 1);
+}
+
+/**
+ *  stmmac_tx_timeout
+ *  @dev : Pointer to net device structure
+ *  @txqueue: the index of the hanging transmit queue
+ *  Description: this function is called when a packet transmission fails to
+ *   complete within a reasonable time. The driver will mark the error in the
+ *   netdev structure and arrange for the device to be reset to a sane state
+ *   in order to transmit a new packet.
+ */
+static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	stmmac_global_err(priv);
+}
+
+/**
+ *  stmmac_set_rx_mode - entry point for multicast addressing
+ *  @dev : pointer to the device structure
+ *  Description:
+ *  This function is a driver entry point which gets called by the kernel
+ *  whenever multicast addresses must be enabled/disabled.
+ *  Return value:
+ *  void.
+ *
+ *  FIXME: This may need RXC to be running, but it may be called with BH
+ *  disabled, which means we can't call phylink_rx_clk_stop*().
+ */
+static void stmmac_set_rx_mode(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	stmmac_set_filter(priv, priv->hw, dev);
+}
+
+/**
+ *  stmmac_change_mtu - entry point to change MTU size for the device.
+ *  @dev : device pointer.
+ *  @new_mtu : the new MTU size for the device.
+ *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
+ *  to drive packet transmission. Ethernet has an MTU of 1500 octets
+ *  (ETH_DATA_LEN). This value can be changed with ifconfig.
+ *  Return value:
+ *  0 on success and an appropriate (-)ve integer as defined in errno.h
+ *  file on failure.
+ */
+static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int txfifosz = priv->plat->tx_fifo_size;
+	struct stmmac_dma_conf *dma_conf;
+	const int mtu = new_mtu;
+	int ret;
+
+	if (txfifosz == 0)
+		txfifosz = priv->dma_cap.tx_fifo_size;
+
+	txfifosz /= priv->plat->tx_queues_to_use;
+
+	if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) {
+		netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n");
+		return -EINVAL;
+	}
+
+	new_mtu = STMMAC_ALIGN(new_mtu);
+
+	/* If condition true, FIFO is too small or MTU too large */
+	if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
+		return -EINVAL;
+
+	if (netif_running(dev)) {
+		netdev_dbg(priv->dev, "restarting interface to change its MTU\n");
+		/* Try to allocate the new DMA conf with the new mtu */
+		dma_conf = stmmac_setup_dma_desc(priv, mtu);
+		if (IS_ERR(dma_conf)) {
+			netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n",
+				   mtu);
+			return PTR_ERR(dma_conf);
+		}
+
+		__stmmac_release(dev);
+
+		ret = __stmmac_open(dev, dma_conf);
+		if (ret) {
+			free_dma_desc_resources(priv, dma_conf);
+			kfree(dma_conf);
+			netdev_err(priv->dev, "failed reopening the interface after MTU change\n");
+			return ret;
+		}
+
+		kfree(dma_conf);
+
+		stmmac_set_rx_mode(dev);
+	}
+
+	WRITE_ONCE(dev->mtu, mtu);
+	netdev_update_features(dev);
+
+	return 0;
+}
+
+static netdev_features_t stmmac_fix_features(struct net_device *dev,
+					     netdev_features_t features)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
+		features &= ~NETIF_F_RXCSUM;
+
+	if (!priv->plat->tx_coe)
+		features &= ~NETIF_F_CSUM_MASK;
+
+	/* Some GMAC devices have a bugged Jumbo frame support that
+	 * needs to have the Tx COE disabled for oversized frames
+	 * (due to limited buffer sizes). In this case we disable
+	 * the TX csum insertion in the TDES and not use SF.
+	 */
+	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
+		features &= ~NETIF_F_CSUM_MASK;
+
+	/* Disable tso if asked by ethtool */
+	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {
+		if (features & NETIF_F_TSO)
+			priv->tso = true;
+		else
+			priv->tso = false;
+	}
+
+	return features;
+}
+
+static int stmmac_set_features(struct net_device *netdev,
+			       netdev_features_t features)
+{
+	struct stmmac_priv *priv = netdev_priv(netdev);
+
+	/* Keep the COE Type in case of csum is supporting */
+	if (features & NETIF_F_RXCSUM)
+		priv->hw->rx_csum = priv->plat->rx_coe;
+	else
+		priv->hw->rx_csum = 0;
+	/* No check needed because rx_coe has been set before and it will be
+	 * fixed in case of issue.
+	 */
+	stmmac_rx_ipc(priv, priv->hw);
+
+	if (priv->sph_cap) {
+		bool sph_en = (priv->hw->rx_csum > 0) && priv->sph;
+		u32 chan;
+
+		for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
+			stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
+	}
+
+	if (features & NETIF_F_HW_VLAN_CTAG_RX)
+		priv->hw->hw_vlan_en = true;
+	else
+		priv->hw->hw_vlan_en = false;
+
+	phylink_rx_clk_stop_block(priv->phylink);
+	stmmac_set_hw_vlan_mode(priv, priv->hw);
+	phylink_rx_clk_stop_unblock(priv->phylink);
+
+	return 0;
+}
+
+static void stmmac_common_interrupt(struct stmmac_priv *priv)
+{
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 queues_count;
+	u32 queue;
+	bool xmac;
+
+	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
+	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
+
+	if (priv->irq_wake)
+		pm_wakeup_event(priv->device, 0);
+
+	if (priv->dma_cap.estsel)
+		stmmac_est_irq_status(priv, priv, priv->dev,
+				      &priv->xstats, tx_cnt);
+
+	if (ec_stmmac_fpe_supported(priv))
+		ec_stmmac_fpe_irq_status(priv);
+
+	/* To handle GMAC own interrupts */
+	if ((priv->plat->has_gmac) || xmac) {
+		int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
+
+		if (unlikely(status)) {
+			/* For LPI we need to save the tx status */
+			if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
+				priv->tx_path_in_lpi_mode = true;
+			if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
+				priv->tx_path_in_lpi_mode = false;
+		}
+
+		for (queue = 0; queue < queues_count; queue++)
+			stmmac_host_mtl_irq_status(priv, priv->hw, queue);
+
+		/* PCS link status */
+		if (priv->hw->pcs &&
+		    !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) {
+			if (priv->xstats.pcs_link)
+				netif_carrier_on(priv->dev);
+			else
+				netif_carrier_off(priv->dev);
+		}
+
+		stmmac_timestamp_interrupt(priv, priv);
+	}
+}
+
+/**
+ *  stmmac_interrupt - main ISR
+ *  @irq: interrupt number.
+ *  @dev_id: to pass the net device pointer.
+ *  Description: this is the main driver interrupt service routine.
+ *  It can call:
+ *  o DMA service routine (to manage incoming frame reception and transmission
+ *    status)
+ *  o Core interrupts to manage: remote wake-up, management counter, LPI
+ *    interrupts.
+ */
+static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
+{
+	struct net_device *dev = (struct net_device *)dev_id;
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	/* Check if adapter is up */
+	if (test_bit(STMMAC_DOWN, &priv->state))
+		return IRQ_HANDLED;
+
+	/* Check ASP error if it isn't delivered via an individual IRQ */
+	if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv))
+		return IRQ_HANDLED;
+
+	/* To handle Common interrupts */
+	stmmac_common_interrupt(priv);
+
+	/* To handle DMA interrupts */
+	stmmac_dma_interrupt(priv);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id)
+{
+	struct net_device *dev = (struct net_device *)dev_id;
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	/* Check if adapter is up */
+	if (test_bit(STMMAC_DOWN, &priv->state))
+		return IRQ_HANDLED;
+
+	/* To handle Common interrupts */
+	stmmac_common_interrupt(priv);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id)
+{
+	struct net_device *dev = (struct net_device *)dev_id;
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	/* Check if adapter is up */
+	if (test_bit(STMMAC_DOWN, &priv->state))
+		return IRQ_HANDLED;
+
+	/* Check if a fatal error happened */
+	stmmac_safety_feat_interrupt(priv);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stmmac_msi_intr_tx(int irq, void *data)
+{
+	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data;
+	struct stmmac_dma_conf *dma_conf;
+	int chan = tx_q->queue_index;
+	struct stmmac_priv *priv;
+	int status;
+
+	dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]);
+	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);
+
+	/* Check if adapter is up */
+	if (test_bit(STMMAC_DOWN, &priv->state))
+		return IRQ_HANDLED;
+
+	status = stmmac_napi_check(priv, chan, DMA_DIR_TX);
+
+	if (unlikely(status & tx_hard_error_bump_tc)) {
+		/* Try to bump up the dma threshold on this failure */
+		stmmac_bump_dma_threshold(priv, chan);
+	} else if (unlikely(status == tx_hard_error)) {
+		stmmac_tx_err(priv, chan);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stmmac_msi_intr_rx(int irq, void *data)
+{
+	struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data;
+	struct stmmac_dma_conf *dma_conf;
+	int chan = rx_q->queue_index;
+	struct stmmac_priv *priv;
+
+	dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]);
+	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);
+
+	/* Check if adapter is up */
+	if (test_bit(STMMAC_DOWN, &priv->state))
+		return IRQ_HANDLED;
+
+	stmmac_napi_check(priv, chan, DMA_DIR_RX);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ *  stmmac_ioctl - Entry point for the Ioctl
+ *  @dev: Device pointer.
+ *  @rq: An IOCTL specefic structure, that can contain a pointer to
+ *  a proprietary structure used to pass information to the driver.
+ *  @cmd: IOCTL command
+ *  Description:
+ *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
+ */
+static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
+{
+	struct stmmac_priv *priv = netdev_priv (dev);
+	int ret = -EOPNOTSUPP;
+
+	if (!netif_running(dev))
+		return -EINVAL;
+
+	switch (cmd) {
+	case SIOCGMIIPHY:
+	case SIOCGMIIREG:
+	case SIOCSMIIREG:
+		ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
+				    void *cb_priv)
+{
+	struct stmmac_priv *priv = cb_priv;
+	int ret = -EOPNOTSUPP;
+
+	if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
+		return ret;
+
+	__stmmac_disable_all_queues(priv);
+
+	if (get_ecdev(priv))
+		return -EBUSY;
+
+	switch (type) {
+	case TC_SETUP_CLSU32:
+		ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
+		break;
+	case TC_SETUP_CLSFLOWER:
+		ret = stmmac_tc_setup_cls(priv, priv, type_data);
+		break;
+	default:
+		break;
+	}
+
+	stmmac_enable_all_queues(priv);
+	return ret;
+}
+
+static LIST_HEAD(stmmac_block_cb_list);
+
+static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
+			   void *type_data)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	switch (type) {
+	case TC_QUERY_CAPS:
+		return stmmac_tc_query_caps(priv, priv, type_data);
+	case TC_SETUP_QDISC_MQPRIO:
+		return stmmac_tc_setup_mqprio(priv, priv, type_data);
+	case TC_SETUP_BLOCK:
+		return flow_block_cb_setup_simple(type_data,
+						  &stmmac_block_cb_list,
+						  stmmac_setup_tc_block_cb,
+						  priv, priv, true);
+	case TC_SETUP_QDISC_CBS:
+		return stmmac_tc_setup_cbs(priv, priv, type_data);
+	case TC_SETUP_QDISC_TAPRIO:
+		return stmmac_tc_setup_taprio(priv, priv, type_data);
+	case TC_SETUP_QDISC_ETF:
+		return stmmac_tc_setup_etf(priv, priv, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
+			       struct net_device *sb_dev)
+{
+	int gso = skb_shinfo(skb)->gso_type;
+
+	if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
+		/*
+		 * There is no way to determine the number of TSO/USO
+		 * capable Queues. Let's use always the Queue 0
+		 * because if TSO/USO is supported then at least this
+		 * one will be capable.
+		 */
+		return 0;
+	}
+
+	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
+}
+
+static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int ret = 0;
+
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		return ret;
+
+	ret = eth_mac_addr(ndev, addr);
+	if (ret)
+		goto set_mac_error;
+
+	phylink_rx_clk_stop_block(priv->phylink);
+	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
+	phylink_rx_clk_stop_unblock(priv->phylink);
+
+set_mac_error:
+	pm_runtime_put(priv->device);
+
+	return ret;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static struct dentry *stmmac_fs_dir;
+
+static void sysfs_display_ring(void *head, int size, int extend_desc,
+			       struct seq_file *seq, dma_addr_t dma_phy_addr)
+{
+	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
+	struct dma_desc *p = (struct dma_desc *)head;
+	unsigned int desc_size;
+	dma_addr_t dma_addr;
+	int i;
+
+	desc_size = extend_desc ? sizeof(*ep) : sizeof(*p);
+	for (i = 0; i < size; i++) {
+		dma_addr = dma_phy_addr + i * desc_size;
+		seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
+				i, &dma_addr,
+				le32_to_cpu(p->des0), le32_to_cpu(p->des1),
+				le32_to_cpu(p->des2), le32_to_cpu(p->des3));
+		if (extend_desc)
+			p = &(++ep)->basic;
+		else
+			p++;
+	}
+}
+
+static int stmmac_rings_status_show(struct seq_file *seq, void *v)
+{
+	struct net_device *dev = seq->private;
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 rx_count = priv->plat->rx_queues_to_use;
+	u32 tx_count = priv->plat->tx_queues_to_use;
+	u32 queue;
+
+	if ((dev->flags & IFF_UP) == 0)
+		return 0;
+
+	for (queue = 0; queue < rx_count; queue++) {
+		struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+
+		seq_printf(seq, "RX Queue %d:\n", queue);
+
+		if (priv->extend_desc) {
+			seq_printf(seq, "Extended descriptor ring:\n");
+			sysfs_display_ring((void *)rx_q->dma_erx,
+					   priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy);
+		} else {
+			seq_printf(seq, "Descriptor ring:\n");
+			sysfs_display_ring((void *)rx_q->dma_rx,
+					   priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy);
+		}
+	}
+
+	for (queue = 0; queue < tx_count; queue++) {
+		struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+
+		seq_printf(seq, "TX Queue %d:\n", queue);
+
+		if (priv->extend_desc) {
+			seq_printf(seq, "Extended descriptor ring:\n");
+			sysfs_display_ring((void *)tx_q->dma_etx,
+					   priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy);
+		} else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
+			seq_printf(seq, "Descriptor ring:\n");
+			sysfs_display_ring((void *)tx_q->dma_tx,
+					   priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy);
+		}
+	}
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
+
+static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
+{
+	static const char * const dwxgmac_timestamp_source[] = {
+		"None",
+		"Internal",
+		"External",
+		"Both",
+	};
+	static const char * const dwxgmac_safety_feature_desc[] = {
+		"No",
+		"All Safety Features with ECC and Parity",
+		"All Safety Features without ECC or Parity",
+		"All Safety Features with Parity Only",
+		"ECC Only",
+		"UNDEFINED",
+		"UNDEFINED",
+		"UNDEFINED",
+	};
+	struct net_device *dev = seq->private;
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (!priv->hw_cap_support) {
+		seq_printf(seq, "DMA HW features not supported\n");
+		return 0;
+	}
+
+	seq_printf(seq, "==============================\n");
+	seq_printf(seq, "\tDMA HW features\n");
+	seq_printf(seq, "==============================\n");
+
+	seq_printf(seq, "\t10/100 Mbps: %s\n",
+		   (priv->dma_cap.mbps_10_100) ? "Y" : "N");
+	seq_printf(seq, "\t1000 Mbps: %s\n",
+		   (priv->dma_cap.mbps_1000) ? "Y" : "N");
+	seq_printf(seq, "\tHalf duplex: %s\n",
+		   (priv->dma_cap.half_duplex) ? "Y" : "N");
+	if (priv->plat->has_xgmac) {
+		seq_printf(seq,
+			   "\tNumber of Additional MAC address registers: %d\n",
+			   priv->dma_cap.multi_addr);
+	} else {
+		seq_printf(seq, "\tHash Filter: %s\n",
+			   (priv->dma_cap.hash_filter) ? "Y" : "N");
+		seq_printf(seq, "\tMultiple MAC address registers: %s\n",
+			   (priv->dma_cap.multi_addr) ? "Y" : "N");
+	}
+	seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
+		   (priv->dma_cap.pcs) ? "Y" : "N");
+	seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
+		   (priv->dma_cap.sma_mdio) ? "Y" : "N");
+	seq_printf(seq, "\tPMT Remote wake up: %s\n",
+		   (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
+	seq_printf(seq, "\tPMT Magic Frame: %s\n",
+		   (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
+	seq_printf(seq, "\tRMON module: %s\n",
+		   (priv->dma_cap.rmon) ? "Y" : "N");
+	seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
+		   (priv->dma_cap.time_stamp) ? "Y" : "N");
+	seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
+		   (priv->dma_cap.atime_stamp) ? "Y" : "N");
+	if (priv->plat->has_xgmac)
+		seq_printf(seq, "\tTimestamp System Time Source: %s\n",
+			   dwxgmac_timestamp_source[priv->dma_cap.tssrc]);
+	seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
+		   (priv->dma_cap.eee) ? "Y" : "N");
+	seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
+	seq_printf(seq, "\tChecksum Offload in TX: %s\n",
+		   (priv->dma_cap.tx_coe) ? "Y" : "N");
+	if (priv->synopsys_id >= DWMAC_CORE_4_00 ||
+	    priv->plat->has_xgmac) {
+		seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
+			   (priv->dma_cap.rx_coe) ? "Y" : "N");
+	} else {
+		seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
+			   (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
+		seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
+			   (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
+		seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
+			   (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
+	}
+	seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
+		   priv->dma_cap.number_rx_channel);
+	seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
+		   priv->dma_cap.number_tx_channel);
+	seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
+		   priv->dma_cap.number_rx_queues);
+	seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
+		   priv->dma_cap.number_tx_queues);
+	seq_printf(seq, "\tEnhanced descriptors: %s\n",
+		   (priv->dma_cap.enh_desc) ? "Y" : "N");
+	seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
+	seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
+	seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ?
+		   (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0);
+	seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
+	seq_printf(seq, "\tNumber of PPS Outputs: %d\n",
+		   priv->dma_cap.pps_out_num);
+	seq_printf(seq, "\tSafety Features: %s\n",
+		   dwxgmac_safety_feature_desc[priv->dma_cap.asp]);
+	seq_printf(seq, "\tFlexible RX Parser: %s\n",
+		   priv->dma_cap.frpsel ? "Y" : "N");
+	seq_printf(seq, "\tEnhanced Addressing: %d\n",
+		   priv->dma_cap.host_dma_width);
+	seq_printf(seq, "\tReceive Side Scaling: %s\n",
+		   priv->dma_cap.rssen ? "Y" : "N");
+	seq_printf(seq, "\tVLAN Hash Filtering: %s\n",
+		   priv->dma_cap.vlhash ? "Y" : "N");
+	seq_printf(seq, "\tSplit Header: %s\n",
+		   priv->dma_cap.sphen ? "Y" : "N");
+	seq_printf(seq, "\tVLAN TX Insertion: %s\n",
+		   priv->dma_cap.vlins ? "Y" : "N");
+	seq_printf(seq, "\tDouble VLAN: %s\n",
+		   priv->dma_cap.dvlan ? "Y" : "N");
+	seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n",
+		   priv->dma_cap.l3l4fnum);
+	seq_printf(seq, "\tARP Offloading: %s\n",
+		   priv->dma_cap.arpoffsel ? "Y" : "N");
+	seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n",
+		   priv->dma_cap.estsel ? "Y" : "N");
+	seq_printf(seq, "\tFrame Preemption (FPE): %s\n",
+		   priv->dma_cap.fpesel ? "Y" : "N");
+	seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n",
+		   priv->dma_cap.tbssel ? "Y" : "N");
+	seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n",
+		   priv->dma_cap.tbs_ch_num);
+	seq_printf(seq, "\tPer-Stream Filtering: %s\n",
+		   priv->dma_cap.sgfsel ? "Y" : "N");
+	seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n",
+		   BIT(priv->dma_cap.ttsfd) >> 1);
+	seq_printf(seq, "\tNumber of Traffic Classes: %d\n",
+		   priv->dma_cap.numtc);
+	seq_printf(seq, "\tDCB Feature: %s\n",
+		   priv->dma_cap.dcben ? "Y" : "N");
+	seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n",
+		   priv->dma_cap.advthword ? "Y" : "N");
+	seq_printf(seq, "\tPTP Offload: %s\n",
+		   priv->dma_cap.ptoen ? "Y" : "N");
+	seq_printf(seq, "\tOne-Step Timestamping: %s\n",
+		   priv->dma_cap.osten ? "Y" : "N");
+	seq_printf(seq, "\tPriority-Based Flow Control: %s\n",
+		   priv->dma_cap.pfcen ? "Y" : "N");
+	seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n",
+		   BIT(priv->dma_cap.frpes) << 6);
+	seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n",
+		   BIT(priv->dma_cap.frpbs) << 6);
+	seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n",
+		   priv->dma_cap.frppipe_num);
+	seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n",
+		   priv->dma_cap.nrvf_num ?
+		   (BIT(priv->dma_cap.nrvf_num) << 1) : 0);
+	seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n",
+		   priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0);
+	seq_printf(seq, "\tDepth of GCL: %lu\n",
+		   priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0);
+	seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n",
+		   priv->dma_cap.cbtisel ? "Y" : "N");
+	seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n",
+		   priv->dma_cap.aux_snapshot_n);
+	seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n",
+		   priv->dma_cap.pou_ost_en ? "Y" : "N");
+	seq_printf(seq, "\tEnhanced DMA: %s\n",
+		   priv->dma_cap.edma ? "Y" : "N");
+	seq_printf(seq, "\tDifferent Descriptor Cache: %s\n",
+		   priv->dma_cap.ediffc ? "Y" : "N");
+	seq_printf(seq, "\tVxLAN/NVGRE: %s\n",
+		   priv->dma_cap.vxn ? "Y" : "N");
+	seq_printf(seq, "\tDebug Memory Interface: %s\n",
+		   priv->dma_cap.dbgmem ? "Y" : "N");
+	seq_printf(seq, "\tNumber of Policing Counters: %lu\n",
+		   priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0);
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
+
+/* Use network device events to rename debugfs file entries.
+ */
+static int stmmac_device_event(struct notifier_block *unused,
+			       unsigned long event, void *ptr)
+{
+	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (dev->netdev_ops != &stmmac_netdev_ops)
+		goto done;
+
+	switch (event) {
+	case NETDEV_CHANGENAME:
+		debugfs_change_name(priv->dbgfs_dir, "%s", dev->name);
+		break;
+	}
+done:
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block stmmac_notifier = {
+	.notifier_call = stmmac_device_event,
+};
+
+static void stmmac_init_fs(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	rtnl_lock();
+
+	/* Create per netdev entries */
+	priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
+
+	/* Entry to report DMA RX/TX rings */
+	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
+			    &stmmac_rings_status_fops);
+
+	/* Entry to report the DMA HW features */
+	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
+			    &stmmac_dma_cap_fops);
+
+	rtnl_unlock();
+}
+
+static void stmmac_exit_fs(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	debugfs_remove_recursive(priv->dbgfs_dir);
+}
+#endif /* CONFIG_DEBUG_FS */
+
+static u32 stmmac_vid_crc32_le(__le16 vid_le)
+{
+	unsigned char *data = (unsigned char *)&vid_le;
+	unsigned char data_byte = 0;
+	u32 crc = ~0x0;
+	u32 temp = 0;
+	int i, bits;
+
+	bits = get_bitmask_order(VLAN_VID_MASK);
+	for (i = 0; i < bits; i++) {
+		if ((i % 8) == 0)
+			data_byte = data[i / 8];
+
+		temp = ((crc & 1) ^ data_byte) & 1;
+		crc >>= 1;
+		data_byte >>= 1;
+
+		if (temp)
+			crc ^= 0xedb88320;
+	}
+
+	return crc;
+}
+
+static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
+{
+	u32 crc, hash = 0;
+	u16 pmatch = 0;
+	int count = 0;
+	u16 vid = 0;
+
+	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
+		__le16 vid_le = cpu_to_le16(vid);
+		crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
+		hash |= (1 << crc);
+		count++;
+	}
+
+	if (!priv->dma_cap.vlhash) {
+		if (count > 2) /* VID = 0 always passes filter */
+			return -EOPNOTSUPP;
+
+		pmatch = vid;
+		hash = 0;
+	}
+
+	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
+}
+
+/* FIXME: This may need RXC to be running, but it may be called with BH
+ * disabled, which means we can't call phylink_rx_clk_stop*().
+ */
+static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	bool is_double = false;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		return ret;
+
+	if (be16_to_cpu(proto) == ETH_P_8021AD)
+		is_double = true;
+
+	set_bit(vid, priv->active_vlans);
+	ret = stmmac_vlan_update(priv, is_double);
+	if (ret) {
+		clear_bit(vid, priv->active_vlans);
+		goto err_pm_put;
+	}
+
+	if (priv->hw->num_vlan) {
+		ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
+		if (ret)
+			goto err_pm_put;
+	}
+err_pm_put:
+	pm_runtime_put(priv->device);
+
+	return ret;
+}
+
+/* FIXME: This may need RXC to be running, but it may be called with BH
+ * disabled, which means we can't call phylink_rx_clk_stop*().
+ */
+static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	bool is_double = false;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		return ret;
+
+	if (be16_to_cpu(proto) == ETH_P_8021AD)
+		is_double = true;
+
+	clear_bit(vid, priv->active_vlans);
+
+	if (priv->hw->num_vlan) {
+		ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
+		if (ret)
+			goto del_vlan_error;
+	}
+
+	ret = stmmac_vlan_update(priv, is_double);
+
+del_vlan_error:
+	pm_runtime_put(priv->device);
+
+	return ret;
+}
+
+static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+
+	if (get_ecdev(priv)) {
+		return -EBUSY;
+	}
+
+	switch (bpf->command) {
+	case XDP_SETUP_PROG:
+		return ec_stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack);
+	case XDP_SETUP_XSK_POOL:
+		return ec_stmmac_xdp_setup_pool(priv, bpf->xsk.pool,
+					     bpf->xsk.queue_id);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int stmmac_xdp_xmit(struct net_device *dev, int num_frames,
+			   struct xdp_frame **frames, u32 flags)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int cpu = smp_processor_id();
+	struct netdev_queue *nq;
+	int i, nxmit = 0;
+	int queue;
+
+	if (unlikely(test_bit(STMMAC_DOWN, &priv->state)))
+		return -ENETDOWN;
+
+	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+		return -EINVAL;
+
+	queue = stmmac_xdp_get_tx_queue(priv, cpu);
+	nq = netdev_get_tx_queue(priv->dev, queue);
+
+	__netif_tx_lock(nq, cpu);
+	/* Avoids TX time-out as we are sharing with slow path */
+	ec_txq_trans_cond_update(nq);
+
+	for (i = 0; i < num_frames; i++) {
+		int res;
+
+		res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true);
+		if (res == STMMAC_XDP_CONSUMED)
+			break;
+
+		nxmit++;
+	}
+
+	if (flags & XDP_XMIT_FLUSH) {
+		stmmac_flush_tx_descriptors(priv, queue);
+		stmmac_tx_timer_arm(priv, queue);
+	}
+
+	__netif_tx_unlock(nq);
+
+	return nxmit;
+}
+
+void ec_stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_channel *ch = &priv->channel[queue];
+	unsigned long flags;
+
+	spin_lock_irqsave(&ch->lock, flags);
+	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
+	spin_unlock_irqrestore(&ch->lock, flags);
+
+	stmmac_stop_rx_dma(priv, queue);
+	__free_dma_rx_desc_resources(priv, &priv->dma_conf, queue);
+}
+
+void ec_stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+	struct stmmac_channel *ch = &priv->channel[queue];
+	unsigned long flags;
+	u32 buf_size;
+	int ret;
+
+	ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue);
+	if (ret) {
+		netdev_err(priv->dev, "Failed to alloc RX desc.\n");
+		return;
+	}
+
+	ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL);
+	if (ret) {
+		__free_dma_rx_desc_resources(priv, &priv->dma_conf, queue);
+		netdev_err(priv->dev, "Failed to init RX desc.\n");
+		return;
+	}
+
+	stmmac_reset_rx_queue(priv, queue);
+	stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue);
+
+	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+			    rx_q->dma_rx_phy, rx_q->queue_index);
+
+	rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num *
+			     sizeof(struct dma_desc));
+	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
+			       rx_q->rx_tail_addr, rx_q->queue_index);
+
+	if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
+		buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
+		stmmac_set_dma_bfsize(priv, priv->ioaddr,
+				      buf_size,
+				      rx_q->queue_index);
+	} else {
+		stmmac_set_dma_bfsize(priv, priv->ioaddr,
+				      priv->dma_conf.dma_buf_sz,
+				      rx_q->queue_index);
+	}
+
+	stmmac_start_rx_dma(priv, queue);
+
+	spin_lock_irqsave(&ch->lock, flags);
+	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
+	spin_unlock_irqrestore(&ch->lock, flags);
+}
+
+void ec_stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_channel *ch = &priv->channel[queue];
+	unsigned long flags;
+
+	spin_lock_irqsave(&ch->lock, flags);
+	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
+	spin_unlock_irqrestore(&ch->lock, flags);
+
+	stmmac_stop_tx_dma(priv, queue);
+	__free_dma_tx_desc_resources(priv, &priv->dma_conf, queue);
+}
+
+void ec_stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+	struct stmmac_channel *ch = &priv->channel[queue];
+	unsigned long flags;
+	int ret;
+
+	ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue);
+	if (ret) {
+		netdev_err(priv->dev, "Failed to alloc TX desc.\n");
+		return;
+	}
+
+	ret = __init_dma_tx_desc_rings(priv,  &priv->dma_conf, queue);
+	if (ret) {
+		__free_dma_tx_desc_resources(priv, &priv->dma_conf, queue);
+		netdev_err(priv->dev, "Failed to init TX desc.\n");
+		return;
+	}
+
+	stmmac_reset_tx_queue(priv, queue);
+	stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue);
+
+	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+			    tx_q->dma_tx_phy, tx_q->queue_index);
+
+	if (tx_q->tbs & STMMAC_TBS_AVAIL)
+		stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index);
+
+	tx_q->tx_tail_addr = tx_q->dma_tx_phy;
+	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
+			       tx_q->tx_tail_addr, tx_q->queue_index);
+
+	stmmac_start_tx_dma(priv, queue);
+
+	spin_lock_irqsave(&ch->lock, flags);
+	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
+	spin_unlock_irqrestore(&ch->lock, flags);
+}
+
+void ec_stmmac_xdp_release(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 chan;
+
+	/* Ensure tx function is not running */
+	netif_tx_disable(dev);
+
+	BUG_ON(get_ecdev(priv));
+
+	/* Disable NAPI process */
+	stmmac_disable_all_queues(priv);
+
+	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
+		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);
+
+	/* Free the IRQ lines */
+	stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0);
+
+	/* Stop TX/RX DMA channels */
+	stmmac_stop_all_dma(priv);
+
+	/* Release and free the Rx/Tx resources */
+	free_dma_desc_resources(priv, &priv->dma_conf);
+
+	/* Disable the MAC Rx/Tx */
+	stmmac_mac_set(priv, priv->ioaddr, false);
+
+	/* set trans_start so we don't get spurious
+	 * watchdogs during reset
+	 */
+	netif_trans_update(dev);
+	netif_carrier_off(dev);
+}
+
+int ec_stmmac_xdp_open(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 dma_csr_ch = max(rx_cnt, tx_cnt);
+	struct stmmac_rx_queue *rx_q;
+	struct stmmac_tx_queue *tx_q;
+	u32 buf_size;
+	bool sph_en;
+	u32 chan;
+	int ret;
+
+	BUG_ON(get_ecdev(priv));
+
+	ret = alloc_dma_desc_resources(priv, &priv->dma_conf);
+	if (ret < 0) {
+		netdev_err(dev, "%s: DMA descriptors allocation failed\n",
+			   __func__);
+		goto dma_desc_error;
+	}
+
+	ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL);
+	if (ret < 0) {
+		netdev_err(dev, "%s: DMA descriptors initialization failed\n",
+			   __func__);
+		goto init_error;
+	}
+
+	stmmac_reset_queues_param(priv);
+
+	/* DMA CSR Channel configuration */
+	for (chan = 0; chan < dma_csr_ch; chan++) {
+		stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
+		stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
+	}
+
+	/* Adjust Split header */
+	sph_en = (priv->hw->rx_csum > 0) && priv->sph;
+
+	/* DMA RX Channel Configuration */
+	for (chan = 0; chan < rx_cnt; chan++) {
+		rx_q = &priv->dma_conf.rx_queue[chan];
+
+		stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+				    rx_q->dma_rx_phy, chan);
+
+		rx_q->rx_tail_addr = rx_q->dma_rx_phy +
+				     (rx_q->buf_alloc_num *
+				      sizeof(struct dma_desc));
+		stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
+				       rx_q->rx_tail_addr, chan);
+
+		if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
+			buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
+			stmmac_set_dma_bfsize(priv, priv->ioaddr,
+					      buf_size,
+					      rx_q->queue_index);
+		} else {
+			stmmac_set_dma_bfsize(priv, priv->ioaddr,
+					      priv->dma_conf.dma_buf_sz,
+					      rx_q->queue_index);
+		}
+
+		stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
+	}
+
+	/* DMA TX Channel Configuration */
+	for (chan = 0; chan < tx_cnt; chan++) {
+		tx_q = &priv->dma_conf.tx_queue[chan];
+
+		stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
+				    tx_q->dma_tx_phy, chan);
+
+		tx_q->tx_tail_addr = tx_q->dma_tx_phy;
+		stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
+				       tx_q->tx_tail_addr, chan);
+
+		hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	}
+
+	/* Enable the MAC Rx/Tx */
+	stmmac_mac_set(priv, priv->ioaddr, true);
+
+	/* Start Rx & Tx DMA Channels */
+	stmmac_start_all_dma(priv);
+
+	ret = stmmac_request_irq(dev);
+	if (ret)
+		goto irq_error;
+
+	/* Enable NAPI process*/
+	stmmac_enable_all_queues(priv);
+	netif_carrier_on(dev);
+	netif_tx_start_all_queues(dev);
+	stmmac_enable_all_dma_irq(priv);
+
+	return 0;
+
+irq_error:
+	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
+		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);
+
+init_error:
+	free_dma_desc_resources(priv, &priv->dma_conf);
+dma_desc_error:
+	return ret;
+}
+
+int ec_stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	struct stmmac_rx_queue *rx_q;
+	struct stmmac_tx_queue *tx_q;
+	struct stmmac_channel *ch;
+
+	if (get_ecdev(priv)) {
+		return -EBUSY;
+	}
+
+	if (test_bit(STMMAC_DOWN, &priv->state) ||
+	    !netif_carrier_ok(priv->dev))
+		return -ENETDOWN;
+
+	if (!stmmac_xdp_is_enabled(priv))
+		return -EINVAL;
+
+	if (queue >= priv->plat->rx_queues_to_use ||
+	    queue >= priv->plat->tx_queues_to_use)
+		return -EINVAL;
+
+	rx_q = &priv->dma_conf.rx_queue[queue];
+	tx_q = &priv->dma_conf.tx_queue[queue];
+	ch = &priv->channel[queue];
+
+	if (!rx_q->xsk_pool && !tx_q->xsk_pool)
+		return -EINVAL;
+
+	if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) {
+		/* EQoS does not have per-DMA channel SW interrupt,
+		 * so we schedule RX Napi straight-away.
+		 */
+		if (likely(napi_schedule_prep(&ch->rxtx_napi)))
+			__napi_schedule(&ch->rxtx_napi);
+	}
+
+	return 0;
+}
+
+static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	unsigned int start;
+	int q;
+
+	for (q = 0; q < tx_cnt; q++) {
+		struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q];
+		u64 tx_packets;
+		u64 tx_bytes;
+
+		do {
+			start = u64_stats_fetch_begin(&txq_stats->q_syncp);
+			tx_bytes   = u64_stats_read(&txq_stats->q.tx_bytes);
+		} while (u64_stats_fetch_retry(&txq_stats->q_syncp, start));
+		do {
+			start = u64_stats_fetch_begin(&txq_stats->napi_syncp);
+			tx_packets = u64_stats_read(&txq_stats->napi.tx_packets);
+		} while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start));
+
+		stats->tx_packets += tx_packets;
+		stats->tx_bytes += tx_bytes;
+	}
+
+	for (q = 0; q < rx_cnt; q++) {
+		struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q];
+		u64 rx_packets;
+		u64 rx_bytes;
+
+		do {
+			start = u64_stats_fetch_begin(&rxq_stats->napi_syncp);
+			rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets);
+			rx_bytes   = u64_stats_read(&rxq_stats->napi.rx_bytes);
+		} while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start));
+
+		stats->rx_packets += rx_packets;
+		stats->rx_bytes += rx_bytes;
+	}
+
+	stats->rx_dropped = priv->xstats.rx_dropped;
+	stats->rx_errors = priv->xstats.rx_errors;
+	stats->tx_dropped = priv->xstats.tx_dropped;
+	stats->tx_errors = priv->xstats.tx_errors;
+	stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier;
+	stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision;
+	stats->rx_length_errors = priv->xstats.rx_length;
+	stats->rx_crc_errors = priv->xstats.rx_crc_errors;
+	stats->rx_over_errors = priv->xstats.rx_overflow_cntr;
+	stats->rx_missed_errors = priv->xstats.rx_missed_cntr;
+}
+
+static const struct net_device_ops stmmac_netdev_ops = {
+	.ndo_open = stmmac_open,
+	.ndo_start_xmit = stmmac_xmit,
+	.ndo_stop = stmmac_release,
+	.ndo_change_mtu = stmmac_change_mtu,
+	.ndo_fix_features = stmmac_fix_features,
+	.ndo_set_features = stmmac_set_features,
+	.ndo_set_rx_mode = stmmac_set_rx_mode,
+	.ndo_tx_timeout = stmmac_tx_timeout,
+	.ndo_eth_ioctl = stmmac_ioctl,
+	.ndo_get_stats64 = stmmac_get_stats64,
+	.ndo_setup_tc = stmmac_setup_tc,
+	.ndo_select_queue = stmmac_select_queue,
+	.ndo_set_mac_address = stmmac_set_mac_address,
+	.ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
+	.ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
+	.ndo_bpf = stmmac_bpf,
+	.ndo_xdp_xmit = stmmac_xdp_xmit,
+	.ndo_xsk_wakeup = ec_stmmac_xsk_wakeup,
+	.ndo_hwtstamp_get = stmmac_hwtstamp_get,
+	.ndo_hwtstamp_set = stmmac_hwtstamp_set,
+};
+
+static void ec_kick_watchdog(struct irq_work *work)
+{
+	struct stmmac_priv *priv =
+		container_of(work, struct stmmac_priv, ec_watchdog_kicker);
+	stmmac_common_interrupt(priv);
+}
+
+/**
+ * ec_poll - EtherCAT poll routine
+ * @netdev: net device structure
+ *
+ * This function can never fail.
+ *
+ **/
+static void ec_poll(struct net_device *netdev)
+{
+	struct stmmac_priv *priv = netdev_priv(netdev);
+	int i;
+	int budget = 128;
+	u32 maxq;
+
+	if (!get_ecdev(priv))
+		return;
+	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
+
+	if (jiffies - priv->ec_watchdog_jiffies >= 2 * HZ) {
+		priv->ec_watchdog_jiffies = jiffies;
+		irq_work_queue(&priv->ec_watchdog_kicker);
+	}
+
+	for (i = 0; i < maxq; i++) {
+		struct stmmac_channel *ch = &priv->channel[i];
+		struct stmmac_priv *priv = ch->priv_data;
+		u32 chan = ch->index;
+		int work_done;
+		bool pending_packes = false;
+
+		work_done = stmmac_rx(priv, budget, chan);
+		work_done = stmmac_tx_clean(priv, budget, chan, &pending_packes);
+	}
+}
+
+static void stmmac_reset_subtask(struct stmmac_priv *priv)
+{
+	if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
+		return;
+	if (test_bit(STMMAC_DOWN, &priv->state))
+		return;
+
+	netdev_err(priv->dev, "Reset adapter.\n");
+
+	rtnl_lock();
+	netif_trans_update(priv->dev);
+	while (test_and_set_bit(STMMAC_RESETING, &priv->state))
+		usleep_range(1000, 2000);
+
+	set_bit(STMMAC_DOWN, &priv->state);
+	dev_close(priv->dev);
+	dev_open(priv->dev, NULL);
+	clear_bit(STMMAC_DOWN, &priv->state);
+	clear_bit(STMMAC_RESETING, &priv->state);
+	rtnl_unlock();
+}
+
+static void stmmac_service_task(struct work_struct *work)
+{
+	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
+			service_task);
+
+	stmmac_reset_subtask(priv);
+	clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
+}
+
+/**
+ *  stmmac_hw_init - Init the MAC device
+ *  @priv: driver private structure
+ *  Description: this function is to configure the MAC device according to
+ *  some platform parameters or the HW capability register. It prepares the
+ *  driver to use either ring or chain modes and to setup either enhanced or
+ *  normal descriptors.
+ */
+static int stmmac_hw_init(struct stmmac_priv *priv)
+{
+	int ret;
+
+	/* dwmac-sun8i only work in chain mode */
+	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I)
+		chain_mode = 1;
+	priv->chain_mode = chain_mode;
+
+	/* Initialize HW Interface */
+	ret = ec_stmmac_hwif_init(priv);
+	if (ret)
+		return ret;
+
+	/* Get the HW capability (new GMAC newer than 3.50a) */
+	priv->hw_cap_support = stmmac_get_hw_features(priv);
+	if (priv->hw_cap_support) {
+		dev_info(priv->device, "DMA HW capability register supported\n");
+
+		/* We can override some gmac/dma configuration fields: e.g.
+		 * enh_desc, tx_coe (e.g. that are passed through the
+		 * platform) with the values from the HW capability
+		 * register (if supported).
+		 */
+		priv->plat->enh_desc = priv->dma_cap.enh_desc;
+		priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up &&
+				!(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL);
+		if (priv->dma_cap.hash_tb_sz) {
+			priv->hw->multicast_filter_bins =
+					(BIT(priv->dma_cap.hash_tb_sz) << 5);
+			priv->hw->mcast_bits_log2 =
+					ilog2(priv->hw->multicast_filter_bins);
+		}
+
+		/* TXCOE doesn't work in thresh DMA mode */
+		if (priv->plat->force_thresh_dma_mode)
+			priv->plat->tx_coe = 0;
+		else
+			priv->plat->tx_coe = priv->dma_cap.tx_coe;
+
+		/* In case of GMAC4 rx_coe is from HW cap register. */
+		priv->plat->rx_coe = priv->dma_cap.rx_coe;
+
+		if (priv->dma_cap.rx_coe_type2)
+			priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
+		else if (priv->dma_cap.rx_coe_type1)
+			priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
+
+	} else {
+		dev_info(priv->device, "No HW DMA feature register supported\n");
+	}
+
+	if (priv->plat->rx_coe) {
+		priv->hw->rx_csum = priv->plat->rx_coe;
+		dev_info(priv->device, "RX Checksum Offload Engine supported\n");
+		if (priv->synopsys_id < DWMAC_CORE_4_00)
+			dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
+	}
+	if (priv->plat->tx_coe)
+		dev_info(priv->device, "TX Checksum insertion supported\n");
+
+	if (priv->plat->pmt) {
+		dev_info(priv->device, "Wake-Up On Lan supported\n");
+		device_set_wakeup_capable(priv->device, 1);
+		devm_pm_set_wake_irq(priv->device, priv->wol_irq);
+	}
+
+	if (priv->dma_cap.tsoen)
+		dev_info(priv->device, "TSO supported\n");
+
+	if (priv->dma_cap.number_rx_queues &&
+	    priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) {
+		dev_warn(priv->device,
+			 "Number of Rx queues (%u) exceeds dma capability\n",
+			 priv->plat->rx_queues_to_use);
+		priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues;
+	}
+	if (priv->dma_cap.number_tx_queues &&
+	    priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) {
+		dev_warn(priv->device,
+			 "Number of Tx queues (%u) exceeds dma capability\n",
+			 priv->plat->tx_queues_to_use);
+		priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues;
+	}
+
+	if (priv->dma_cap.rx_fifo_size &&
+	    priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) {
+		dev_warn(priv->device,
+			 "Rx FIFO size (%u) exceeds dma capability\n",
+			 priv->plat->rx_fifo_size);
+		priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size;
+	}
+	if (priv->dma_cap.tx_fifo_size &&
+	    priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) {
+		dev_warn(priv->device,
+			 "Tx FIFO size (%u) exceeds dma capability\n",
+			 priv->plat->tx_fifo_size);
+		priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size;
+	}
+
+	priv->hw->vlan_fail_q_en =
+		(priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN);
+	priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;
+
+	/* Run HW quirks, if any */
+	if (priv->hwif_quirks) {
+		ret = priv->hwif_quirks(priv);
+		if (ret)
+			return ret;
+	}
+
+	/* Rx Watchdog is available in the COREs newer than the 3.40.
+	 * In some case, for example on bugged HW this feature
+	 * has to be disable and this can be done by passing the
+	 * riwt_off field from the platform.
+	 */
+	if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
+	    (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
+		priv->use_riwt = 1;
+		dev_info(priv->device,
+			 "Enable RX Mitigation via HW Watchdog Timer\n");
+	}
+
+	return 0;
+}
+
+static void stmmac_napi_add(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 queue, maxq;
+
+	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
+
+	for (queue = 0; queue < maxq; queue++) {
+		struct stmmac_channel *ch = &priv->channel[queue];
+
+		ch->priv_data = priv;
+		ch->index = queue;
+		spin_lock_init(&ch->lock);
+
+		if (queue < priv->plat->rx_queues_to_use) {
+			netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx);
+		}
+		if (queue < priv->plat->tx_queues_to_use) {
+			netif_napi_add_tx(dev, &ch->tx_napi,
+					  stmmac_napi_poll_tx);
+		}
+		if (queue < priv->plat->rx_queues_to_use &&
+		    queue < priv->plat->tx_queues_to_use) {
+			netif_napi_add(dev, &ch->rxtx_napi,
+				       stmmac_napi_poll_rxtx);
+		}
+	}
+}
+
+static void stmmac_napi_del(struct net_device *dev)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	u32 queue, maxq;
+
+	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
+
+	for (queue = 0; queue < maxq; queue++) {
+		struct stmmac_channel *ch = &priv->channel[queue];
+
+		if (queue < priv->plat->rx_queues_to_use)
+			netif_napi_del(&ch->rx_napi);
+		if (queue < priv->plat->tx_queues_to_use)
+			netif_napi_del(&ch->tx_napi);
+		if (queue < priv->plat->rx_queues_to_use &&
+		    queue < priv->plat->tx_queues_to_use) {
+			netif_napi_del(&ch->rxtx_napi);
+		}
+	}
+}
+
+int ec_stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int ret = 0, i;
+
+	if (netif_running(dev))
+		stmmac_release(dev);
+
+	stmmac_napi_del(dev);
+
+	priv->plat->rx_queues_to_use = rx_cnt;
+	priv->plat->tx_queues_to_use = tx_cnt;
+	if (!netif_is_rxfh_configured(dev))
+		for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
+			priv->rss.table[i] = ethtool_rxfh_indir_default(i,
+									rx_cnt);
+
+	stmmac_napi_add(dev);
+
+	if (netif_running(dev))
+		ret = stmmac_open(dev);
+
+	return ret;
+}
+
+int ec_stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size)
+{
+	struct stmmac_priv *priv = netdev_priv(dev);
+	int ret = 0;
+
+	if (netif_running(dev))
+		stmmac_release(dev);
+
+	priv->dma_conf.dma_rx_size = rx_size;
+	priv->dma_conf.dma_tx_size = tx_size;
+
+	if (netif_running(dev))
+		ret = stmmac_open(dev);
+
+	return ret;
+}
+
+static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp)
+{
+	const struct stmmac_xdp_buff *ctx = (void *)_ctx;
+	struct dma_desc *desc_contains_ts = ctx->desc;
+	struct stmmac_priv *priv = ctx->priv;
+	struct dma_desc *ndesc = ctx->ndesc;
+	struct dma_desc *desc = ctx->desc;
+	u64 ns = 0;
+
+	if (!priv->hwts_rx_en)
+		return -ENODATA;
+
+	/* For GMAC4, the valid timestamp is from CTX next desc. */
+	if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
+		desc_contains_ts = ndesc;
+
+	/* Check if timestamp is available */
+	if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) {
+		stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns);
+		ns -= priv->plat->cdc_error_adj;
+		*timestamp = ns_to_ktime(ns);
+		return 0;
+	}
+
+	return -ENODATA;
+}
+
+static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = {
+	.xmo_rx_timestamp		= stmmac_xdp_rx_timestamp,
+};
+
+/**
+ * ec_stmmac_dvr_probe
+ * @device: device pointer
+ * @plat_dat: platform data pointer
+ * @res: stmmac resource pointer
+ * Description: this is the main probe function used to
+ * call the alloc_etherdev, allocate the priv structure.
+ * Return:
+ * returns 0 on success, otherwise errno.
+ */
+int ec_stmmac_dvr_probe(struct device *device,
+		     struct plat_stmmacenet_data *plat_dat,
+		     struct stmmac_resources *res)
+{
+	struct net_device *ndev = NULL;
+	struct stmmac_priv *priv;
+	u32 rxq;
+	int i, ret = 0;
+
+	ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
+				       MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
+	if (!ndev)
+		return -ENOMEM;
+
+	SET_NETDEV_DEV(ndev, device);
+
+	priv = netdev_priv(ndev);
+	priv->device = device;
+	priv->dev = ndev;
+	priv->ecdev_ = NULL;
+	priv->ecdev_initialized = false;
+
+	for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
+		u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp);
+	for (i = 0; i < MTL_MAX_TX_QUEUES; i++) {
+		u64_stats_init(&priv->xstats.txq_stats[i].q_syncp);
+		u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp);
+	}
+
+	priv->xstats.pcpu_stats =
+		devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats);
+	if (!priv->xstats.pcpu_stats)
+		return -ENOMEM;
+
+	ec_stmmac_set_ethtool_ops(ndev);
+	priv->pause_time = pause;
+	priv->plat = plat_dat;
+	priv->ioaddr = res->addr;
+	priv->dev->base_addr = (unsigned long)res->addr;
+	priv->plat->dma_cfg->multi_msi_en =
+		(priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN);
+
+	priv->dev->irq = res->irq;
+	priv->wol_irq = res->wol_irq;
+	priv->lpi_irq = res->lpi_irq;
+	priv->sfty_irq = res->sfty_irq;
+	priv->sfty_ce_irq = res->sfty_ce_irq;
+	priv->sfty_ue_irq = res->sfty_ue_irq;
+	for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
+		priv->rx_irq[i] = res->rx_irq[i];
+	for (i = 0; i < MTL_MAX_TX_QUEUES; i++)
+		priv->tx_irq[i] = res->tx_irq[i];
+
+	if (!is_zero_ether_addr(res->mac))
+		eth_hw_addr_set(priv->dev, res->mac);
+
+	dev_set_drvdata(device, priv->dev);
+
+	/* Verify driver arguments */
+	stmmac_verify_args();
+
+	priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL);
+	if (!priv->af_xdp_zc_qps)
+		return -ENOMEM;
+
+	/* Allocate workqueue */
+	priv->wq = create_singlethread_workqueue("stmmac_wq");
+	if (!priv->wq) {
+		dev_err(priv->device, "failed to create workqueue\n");
+		ret = -ENOMEM;
+		goto error_wq_init;
+	}
+
+	INIT_WORK(&priv->service_task, stmmac_service_task);
+
+	timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
+
+	/* Override with kernel parameters if supplied XXX CRS XXX
+	 * this needs to have multiple instances
+	 */
+	if ((phyaddr >= 0) && (phyaddr <= 31))
+		priv->plat->phy_addr = phyaddr;
+
+	if (priv->plat->stmmac_rst) {
+		ret = reset_control_assert(priv->plat->stmmac_rst);
+		reset_control_deassert(priv->plat->stmmac_rst);
+		/* Some reset controllers have only reset callback instead of
+		 * assert + deassert callbacks pair.
+		 */
+		if (ret == -ENOTSUPP)
+			reset_control_reset(priv->plat->stmmac_rst);
+	}
+
+	ret = reset_control_deassert(priv->plat->stmmac_ahb_rst);
+	if (ret == -ENOTSUPP)
+		dev_err(priv->device, "unable to bring out of ahb reset: %pe\n",
+			ERR_PTR(ret));
+
+	/* Wait a bit for the reset to take effect */
+	udelay(10);
+
+	/* Init MAC and get the capabilities */
+	ret = stmmac_hw_init(priv);
+	if (ret)
+		goto error_hw_init;
+
+	/* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch.
+	 */
+	if (priv->synopsys_id < DWMAC_CORE_5_20)
+		priv->plat->dma_cfg->dche = false;
+
+	stmmac_check_ether_addr(priv);
+
+	ndev->netdev_ops = &stmmac_netdev_ops;
+
+	ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops;
+	ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops;
+
+	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
+			    NETIF_F_RXCSUM;
+	ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
+			     NETDEV_XDP_ACT_XSK_ZEROCOPY;
+
+	ret = stmmac_tc_init(priv, priv);
+	if (!ret) {
+		ndev->hw_features |= NETIF_F_HW_TC;
+	}
+
+	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {
+		ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
+		if (priv->plat->has_gmac4)
+			ndev->hw_features |= NETIF_F_GSO_UDP_L4;
+		priv->tso = true;
+		dev_info(priv->device, "TSO feature enabled\n");
+	}
+
+	if (priv->dma_cap.sphen &&
+	    !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) {
+		ndev->hw_features |= NETIF_F_GRO;
+		priv->sph_cap = true;
+		priv->sph = priv->sph_cap;
+		dev_info(priv->device, "SPH feature enabled\n");
+	}
+
+	/* Ideally our host DMA address width is the same as for the
+	 * device. However, it may differ and then we have to use our
+	 * host DMA width for allocation and the device DMA width for
+	 * register handling.
+	 */
+	if (priv->plat->host_dma_width)
+		priv->dma_cap.host_dma_width = priv->plat->host_dma_width;
+	else
+		priv->dma_cap.host_dma_width = priv->dma_cap.addr64;
+
+	if (priv->dma_cap.host_dma_width) {
+		ret = dma_set_mask_and_coherent(device,
+				DMA_BIT_MASK(priv->dma_cap.host_dma_width));
+		if (!ret) {
+			dev_info(priv->device, "Using %d/%d bits DMA host/device width\n",
+				 priv->dma_cap.host_dma_width, priv->dma_cap.addr64);
+
+			/*
+			 * If more than 32 bits can be addressed, make sure to
+			 * enable enhanced addressing mode.
+			 */
+			if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
+				priv->plat->dma_cfg->eame = true;
+		} else {
+			ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
+			if (ret) {
+				dev_err(priv->device, "Failed to set DMA Mask\n");
+				goto error_hw_init;
+			}
+
+			priv->dma_cap.host_dma_width = 32;
+		}
+	}
+
+	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
+	ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
+#ifdef STMMAC_VLAN_TAG_USED
+	/* Both mac100 and gmac support receive VLAN tag detection */
+	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
+	if (priv->plat->has_gmac4 || priv->plat->has_xgmac) {
+		ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
+		priv->hw->hw_vlan_en = true;
+	}
+	if (priv->dma_cap.vlhash) {
+		ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+		ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
+	}
+	if (priv->dma_cap.vlins)
+		ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
+#endif
+	priv->msg_enable = netif_msg_init(debug, default_msg_level);
+
+	priv->xstats.threshold = tc;
+
+	/* Initialize RSS */
+	rxq = priv->plat->rx_queues_to_use;
+	netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
+	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
+		priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
+
+	if (priv->dma_cap.rssen && priv->plat->rss_en)
+		ndev->features |= NETIF_F_RXHASH;
+
+	ndev->vlan_features |= ndev->features;
+
+	/* MTU range: 46 - hw-specific max */
+	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
+	if (priv->plat->has_xgmac)
+		ndev->max_mtu = XGMAC_JUMBO_LEN;
+	else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
+		ndev->max_mtu = JUMBO_LEN;
+	else
+		ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
+	/* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
+	 * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
+	 */
+	if ((priv->plat->maxmtu < ndev->max_mtu) &&
+	    (priv->plat->maxmtu >= ndev->min_mtu))
+		ndev->max_mtu = priv->plat->maxmtu;
+	else if (priv->plat->maxmtu < ndev->min_mtu)
+		dev_warn(priv->device,
+			 "%s: warning: maxmtu having invalid value (%d)\n",
+			 __func__, priv->plat->maxmtu);
+
+	ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
+
+	/* Setup channels NAPI */
+	stmmac_napi_add(ndev);
+
+	mutex_init(&priv->lock);
+
+	ec_stmmac_fpe_init(priv);
+
+	stmmac_check_pcs_mode(priv);
+
+	pm_runtime_get_noresume(device);
+	pm_runtime_set_active(device);
+	if (!pm_runtime_enabled(device))
+		pm_runtime_enable(device);
+
+	ret = ec_stmmac_mdio_register(ndev);
+	if (ret < 0) {
+		dev_err_probe(priv->device, ret,
+			      "MDIO bus (id: %d) registration failed\n",
+			      priv->plat->bus_id);
+		goto error_mdio_register;
+	}
+
+	ret = ec_stmmac_pcs_setup(ndev);
+	if (ret)
+		goto error_pcs_setup;
+
+	ret = stmmac_phy_setup(priv);
+	if (ret) {
+		netdev_err(ndev, "failed to setup phy (%d)\n", ret);
+		goto error_phy_setup;
+	}
+
+	priv->ecdev_ = ecdev_offer(ndev, ec_poll, THIS_MODULE);
+	priv->ecdev_initialized = true;
+	/*
+	 * Disable SPH when using EtherCAT: with SPH enabled, the software has to
+	 * actively merge packets before forwarding them to the EtherCAT master,
+	 * which adds extra complexity.
+	 */
+	int sph_orig = priv->sph;
+	priv->sph = false;
+
+	if (!get_ecdev(priv)) {
+		priv->sph = sph_orig;
+		ret = register_netdev(ndev);
+		if (ret) {
+			dev_err(priv->device, "%s: ERROR %i registering the device\n",
+					__func__, ret);
+			goto error_netdev_register;
+		}
+	}
+
+#ifdef CONFIG_DEBUG_FS
+	stmmac_init_fs(ndev);
+#endif
+
+	if (priv->plat->dump_debug_regs)
+		priv->plat->dump_debug_regs(priv->plat->bsp_priv);
+
+	/* Let pm_runtime_put() disable the clocks.
+	 * If CONFIG_PM is not enabled, the clocks will stay powered.
+	 */
+	pm_runtime_put(device);
+	if (get_ecdev(priv)) {
+		init_irq_work(&priv->ec_watchdog_kicker, ec_kick_watchdog);
+		ret = ecdev_open(get_ecdev(priv));
+		if (ret) {
+			ecdev_withdraw(get_ecdev(priv));
+			goto error_netdev_register;
+		}
+	}
+
+	return ret;
+
+error_netdev_register:
+	phylink_destroy(priv->phylink);
+error_phy_setup:
+	ec_stmmac_pcs_clean(ndev);
+error_pcs_setup:
+	ec_stmmac_mdio_unregister(ndev);
+error_mdio_register:
+	stmmac_napi_del(ndev);
+error_hw_init:
+	destroy_workqueue(priv->wq);
+error_wq_init:
+	bitmap_free(priv->af_xdp_zc_qps);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_dvr_probe);
+
+/**
+ * ec_stmmac_dvr_remove
+ * @dev: device pointer
+ * Description: this function resets the TX/RX processes, disables the MAC RX/TX
+ * changes the link status, releases the DMA descriptor rings.
+ */
+void ec_stmmac_dvr_remove(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	netdev_info(priv->dev, "%s: removing driver", __func__);
+
+	pm_runtime_get_sync(dev);
+
+	if (get_ecdev(priv)) {
+		ecdev_close(get_ecdev(priv));
+		irq_work_sync(&priv->ec_watchdog_kicker);
+		ecdev_withdraw(get_ecdev(priv));
+		priv->ecdev_ = NULL;
+		priv->ecdev_initialized = false;
+	} else {
+		unregister_netdev(ndev);
+	}
+
+#ifdef CONFIG_DEBUG_FS
+	stmmac_exit_fs(ndev);
+#endif
+	phylink_destroy(priv->phylink);
+	if (priv->plat->stmmac_rst)
+		reset_control_assert(priv->plat->stmmac_rst);
+	reset_control_assert(priv->plat->stmmac_ahb_rst);
+
+	ec_stmmac_pcs_clean(ndev);
+	ec_stmmac_mdio_unregister(ndev);
+
+	destroy_workqueue(priv->wq);
+	mutex_destroy(&priv->lock);
+	bitmap_free(priv->af_xdp_zc_qps);
+
+	pm_runtime_disable(dev);
+	pm_runtime_put_noidle(dev);
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_dvr_remove);
+
+/**
+ * ec_stmmac_suspend - suspend callback
+ * @dev: device pointer
+ * Description: this is the function to suspend the device and it is called
+ * by the platform driver to stop the network queue, release the resources,
+ * program the PMT register (for WoL), clean and release driver resources.
+ */
+int ec_stmmac_suspend(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	u32 chan;
+
+	if (get_ecdev(priv)) {
+		return -EBUSY;
+	}
+
+	if (!ndev || !netif_running(ndev))
+		return 0;
+
+	mutex_lock(&priv->lock);
+
+	netif_device_detach(ndev);
+
+	stmmac_disable_all_queues(priv);
+
+	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
+		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);
+
+	if (priv->eee_sw_timer_en) {
+		priv->tx_path_in_lpi_mode = false;
+		timer_delete_sync(&priv->eee_ctrl_timer);
+	}
+
+	/* Stop TX/RX DMA */
+	stmmac_stop_all_dma(priv);
+
+	if (priv->plat->serdes_powerdown)
+		priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
+
+	/* Enable Power down mode by programming the PMT regs */
+	if (stmmac_wol_enabled_mac(priv)) {
+		stmmac_pmt(priv, priv->hw, priv->wolopts);
+		priv->irq_wake = 1;
+	} else {
+		stmmac_mac_set(priv, priv->ioaddr, false);
+		pinctrl_pm_select_sleep_state(priv->device);
+	}
+
+	mutex_unlock(&priv->lock);
+
+	rtnl_lock();
+	if (stmmac_wol_enabled_phy(priv))
+		phylink_speed_down(priv->phylink, false);
+
+	phylink_suspend(priv->phylink, stmmac_wol_enabled_mac(priv));
+	rtnl_unlock();
+
+	if (ec_stmmac_fpe_supported(priv))
+		ethtool_mmsv_stop(&priv->fpe_cfg.mmsv);
+
+	if (priv->plat->suspend)
+		return priv->plat->suspend(dev, priv->plat->bsp_priv);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_suspend);
+
+static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
+
+	rx_q->cur_rx = 0;
+	rx_q->dirty_rx = 0;
+}
+
+static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue)
+{
+	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
+
+	tx_q->cur_tx = 0;
+	tx_q->dirty_tx = 0;
+	tx_q->mss = 0;
+
+	netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
+}
+
+/**
+ * stmmac_reset_queues_param - reset queue parameters
+ * @priv: device pointer
+ */
+static void stmmac_reset_queues_param(struct stmmac_priv *priv)
+{
+	u32 rx_cnt = priv->plat->rx_queues_to_use;
+	u32 tx_cnt = priv->plat->tx_queues_to_use;
+	u32 queue;
+
+	for (queue = 0; queue < rx_cnt; queue++)
+		stmmac_reset_rx_queue(priv, queue);
+
+	for (queue = 0; queue < tx_cnt; queue++)
+		stmmac_reset_tx_queue(priv, queue);
+}
+
+/**
+ * ec_stmmac_resume - resume callback
+ * @dev: device pointer
+ * Description: when resume this function is invoked to setup the DMA and CORE
+ * in a usable state.
+ */
+int ec_stmmac_resume(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	if (get_ecdev(priv)) {
+		return -EBUSY;
+	}
+
+	if (priv->plat->resume) {
+		ret = priv->plat->resume(dev, priv->plat->bsp_priv);
+		if (ret)
+			return ret;
+	}
+
+	if (!netif_running(ndev))
+		return 0;
+
+	/* Power Down bit, into the PM register, is cleared
+	 * automatically as soon as a magic packet or a Wake-up frame
+	 * is received. Anyway, it's better to manually clear
+	 * this bit because it can generate problems while resuming
+	 * from another devices (e.g. serial console).
+	 */
+	if (stmmac_wol_enabled_mac(priv)) {
+		mutex_lock(&priv->lock);
+		stmmac_pmt(priv, priv->hw, 0);
+		mutex_unlock(&priv->lock);
+		priv->irq_wake = 0;
+	} else {
+		pinctrl_pm_select_default_state(priv->device);
+		/* reset the phy so that it's ready */
+		if (priv->mii)
+			ec_stmmac_mdio_reset(priv->mii);
+	}
+
+	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&
+	    priv->plat->serdes_powerup) {
+		ret = priv->plat->serdes_powerup(ndev,
+						 priv->plat->bsp_priv);
+
+		if (ret < 0)
+			return ret;
+	}
+
+	rtnl_lock();
+
+	/* Prepare the PHY to resume, ensuring that its clocks which are
+	 * necessary for the MAC DMA reset to complete are running
+	 */
+	phylink_prepare_resume(priv->phylink);
+
+	mutex_lock(&priv->lock);
+
+	stmmac_reset_queues_param(priv);
+
+	stmmac_free_tx_skbufs(priv);
+	stmmac_clear_descriptors(priv, &priv->dma_conf);
+
+	ret = stmmac_hw_setup(ndev);
+	if (ret < 0) {
+		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
+		mutex_unlock(&priv->lock);
+		rtnl_unlock();
+		return ret;
+	}
+
+	stmmac_init_timestamping(priv);
+
+	stmmac_init_coalesce(priv);
+	phylink_rx_clk_stop_block(priv->phylink);
+	stmmac_set_rx_mode(ndev);
+
+	stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
+	phylink_rx_clk_stop_unblock(priv->phylink);
+
+	stmmac_enable_all_queues(priv);
+	stmmac_enable_all_dma_irq(priv);
+
+	mutex_unlock(&priv->lock);
+
+	/* phylink_resume() must be called after the hardware has been
+	 * initialised because it may bring the link up immediately in a
+	 * workqueue thread, which will race with initialisation.
+	 */
+	phylink_resume(priv->phylink);
+	if (stmmac_wol_enabled_phy(priv))
+		phylink_speed_up(priv->phylink);
+
+	rtnl_unlock();
+
+	netif_device_attach(ndev);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_resume);
+
+/* This is not the same as EXPORT_GPL_SIMPLE_DEV_PM_OPS() when CONFIG_PM=n */
+DEFINE_SIMPLE_DEV_PM_OPS(ec_stmmac_simple_pm_ops, ec_stmmac_suspend, ec_stmmac_resume);
+EXPORT_SYMBOL_GPL(ec_stmmac_simple_pm_ops);
+
+#ifndef MODULE
+static int __init stmmac_cmdline_opt(char *str)
+{
+	char *opt;
+
+	if (!str || !*str)
+		return 1;
+	while ((opt = strsep(&str, ",")) != NULL) {
+		if (!strncmp(opt, "debug:", 6)) {
+			if (kstrtoint(opt + 6, 0, &debug))
+				goto err;
+		} else if (!strncmp(opt, "phyaddr:", 8)) {
+			if (kstrtoint(opt + 8, 0, &phyaddr))
+				goto err;
+		} else if (!strncmp(opt, "tc:", 3)) {
+			if (kstrtoint(opt + 3, 0, &tc))
+				goto err;
+		} else if (!strncmp(opt, "watchdog:", 9)) {
+			if (kstrtoint(opt + 9, 0, &watchdog))
+				goto err;
+		} else if (!strncmp(opt, "flow_ctrl:", 10)) {
+			if (kstrtoint(opt + 10, 0, &flow_ctrl))
+				goto err;
+		} else if (!strncmp(opt, "pause:", 6)) {
+			if (kstrtoint(opt + 6, 0, &pause))
+				goto err;
+		} else if (!strncmp(opt, "eee_timer:", 10)) {
+			if (kstrtoint(opt + 10, 0, &eee_timer))
+				goto err;
+		} else if (!strncmp(opt, "chain_mode:", 11)) {
+			if (kstrtoint(opt + 11, 0, &chain_mode))
+				goto err;
+		}
+	}
+	return 1;
+
+err:
+	pr_err("%s: ERROR broken module parameter conversion", __func__);
+	return 1;
+}
+
+__setup("stmmaceth=", stmmac_cmdline_opt);
+#endif /* MODULE */
+
+int __cold stmmac_init(void)
+{
+#ifdef CONFIG_DEBUG_FS
+	/* Create debugfs main directory if it doesn't exist yet */
+	if (!stmmac_fs_dir)
+		stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
+	register_netdevice_notifier(&stmmac_notifier);
+#endif
+
+	return 0;
+}
+
+void __cold stmmac_exit(void)
+{
+#ifdef CONFIG_DEBUG_FS
+	unregister_netdevice_notifier(&stmmac_notifier);
+	debugfs_remove_recursive(stmmac_fs_dir);
+#endif
+}
+
+MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
+MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethercat/device/K3/stmmac_mdio-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_mdio-ethercat.c
new file mode 100644
index 000000000000..22109b915ca1
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_mdio-ethercat.c
@@ -0,0 +1,736 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  STMMAC Ethernet Driver -- MDIO bus implementation
+  Provides Bus interface for MII registers
+
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Carl Shaw <carl.shaw@st.com>
+  Maintainer: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/gpio/consumer.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/mii.h>
+#include <linux/of_mdio.h>
+#include <linux/pm_runtime.h>
+#include <linux/phy.h>
+#include <linux/property.h>
+#include <linux/slab.h>
+
+#include "dwxgmac2-ethercat.h"
+#include "stmmac-ethercat.h"
+
+#define MII_ADDR_GBUSY			BIT(0)
+#define MII_ADDR_GWRITE			BIT(1)
+#define MII_DATA_GD_MASK		GENMASK(15, 0)
+
+/* GMAC4 defines */
+#define MII_GMAC4_GOC_SHIFT		2
+#define MII_GMAC4_REG_ADDR_SHIFT	16
+#define MII_GMAC4_WRITE			(1 << MII_GMAC4_GOC_SHIFT)
+#define MII_GMAC4_READ			(3 << MII_GMAC4_GOC_SHIFT)
+#define MII_GMAC4_C45E			BIT(1)
+
+/* XGMAC defines */
+#define MII_XGMAC_SADDR			BIT(18)
+#define MII_XGMAC_CMD_SHIFT		16
+#define MII_XGMAC_WRITE			(1 << MII_XGMAC_CMD_SHIFT)
+#define MII_XGMAC_READ			(3 << MII_XGMAC_CMD_SHIFT)
+#define MII_XGMAC_BUSY			BIT(22)
+#define MII_XGMAC_MAX_C22ADDR		3
+#define MII_XGMAC_C22P_MASK		GENMASK(MII_XGMAC_MAX_C22ADDR, 0)
+#define MII_XGMAC_PA_SHIFT		16
+#define MII_XGMAC_DA_SHIFT		21
+
+static int stmmac_mdio_wait(void __iomem *reg, u32 mask)
+{
+	u32 v;
+
+	if (readl_poll_timeout(reg, v, !(v & mask), 100, 10000))
+		return -EBUSY;
+
+	return 0;
+}
+
+static void stmmac_xgmac2_c45_format(struct stmmac_priv *priv, int phyaddr,
+				     int devad, int phyreg, u32 *hw_addr)
+{
+	u32 tmp;
+
+	/* Set port as Clause 45 */
+	tmp = readl(priv->ioaddr + XGMAC_MDIO_C22P);
+	tmp &= ~BIT(phyaddr);
+	writel(tmp, priv->ioaddr + XGMAC_MDIO_C22P);
+
+	*hw_addr = (phyaddr << MII_XGMAC_PA_SHIFT) | (phyreg & 0xffff);
+	*hw_addr |= devad << MII_XGMAC_DA_SHIFT;
+}
+
+static void stmmac_xgmac2_c22_format(struct stmmac_priv *priv, int phyaddr,
+				     int phyreg, u32 *hw_addr)
+{
+	u32 tmp = 0;
+
+	if (priv->synopsys_id < DWXGMAC_CORE_2_20) {
+		/* Until ver 2.20 XGMAC does not support C22 addr >= 4. Those
+		 * bits above bit 3 of XGMAC_MDIO_C22P register are reserved.
+		 */
+		tmp = readl(priv->ioaddr + XGMAC_MDIO_C22P);
+		tmp &= ~MII_XGMAC_C22P_MASK;
+	}
+	/* Set port as Clause 22 */
+	tmp |= BIT(phyaddr);
+	writel(tmp, priv->ioaddr + XGMAC_MDIO_C22P);
+
+	*hw_addr = (phyaddr << MII_XGMAC_PA_SHIFT) | (phyreg & 0x1f);
+}
+
+static int stmmac_xgmac2_mdio_read(struct stmmac_priv *priv, u32 addr,
+				   u32 value)
+{
+	unsigned int mii_address = priv->hw->mii.addr;
+	unsigned int mii_data = priv->hw->mii.data;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		return ret;
+
+	/* Wait until any existing MII operation is complete */
+	ret = stmmac_mdio_wait(priv->ioaddr + mii_data, MII_XGMAC_BUSY);
+	if (ret)
+		goto err_disable_clks;
+
+	value |= priv->gmii_address_bus_config | MII_XGMAC_READ;
+
+	/* Wait until any existing MII operation is complete */
+	ret = stmmac_mdio_wait(priv->ioaddr + mii_data, MII_XGMAC_BUSY);
+	if (ret)
+		goto err_disable_clks;
+
+	/* Set the MII address register to read */
+	writel(addr, priv->ioaddr + mii_address);
+	writel(value, priv->ioaddr + mii_data);
+
+	/* Wait until any existing MII operation is complete */
+	ret = stmmac_mdio_wait(priv->ioaddr + mii_data, MII_XGMAC_BUSY);
+	if (ret)
+		goto err_disable_clks;
+
+	/* Read the data from the MII data register */
+	ret = (int)readl(priv->ioaddr + mii_data) & GENMASK(15, 0);
+
+err_disable_clks:
+	pm_runtime_put(priv->device);
+
+	return ret;
+}
+
+static int stmmac_xgmac2_mdio_read_c22(struct mii_bus *bus, int phyaddr,
+				       int phyreg)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	u32 addr;
+
+	/* Until ver 2.20 XGMAC does not support C22 addr >= 4 */
+	if (priv->synopsys_id < DWXGMAC_CORE_2_20 &&
+	    phyaddr > MII_XGMAC_MAX_C22ADDR)
+		return -ENODEV;
+
+	stmmac_xgmac2_c22_format(priv, phyaddr, phyreg, &addr);
+
+	return stmmac_xgmac2_mdio_read(priv, addr, MII_XGMAC_BUSY);
+}
+
+static int stmmac_xgmac2_mdio_read_c45(struct mii_bus *bus, int phyaddr,
+				       int devad, int phyreg)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	u32 addr;
+
+	stmmac_xgmac2_c45_format(priv, phyaddr, devad, phyreg, &addr);
+
+	return stmmac_xgmac2_mdio_read(priv, addr, MII_XGMAC_BUSY);
+}
+
+static int stmmac_xgmac2_mdio_write(struct stmmac_priv *priv, u32 addr,
+				    u32 value, u16 phydata)
+{
+	unsigned int mii_address = priv->hw->mii.addr;
+	unsigned int mii_data = priv->hw->mii.data;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		return ret;
+
+	/* Wait until any existing MII operation is complete */
+	ret = stmmac_mdio_wait(priv->ioaddr + mii_data, MII_XGMAC_BUSY);
+	if (ret)
+		goto err_disable_clks;
+
+	value |= priv->gmii_address_bus_config | phydata | MII_XGMAC_WRITE;
+
+	/* Wait until any existing MII operation is complete */
+	ret = stmmac_mdio_wait(priv->ioaddr + mii_data, MII_XGMAC_BUSY);
+	if (ret)
+		goto err_disable_clks;
+
+	/* Set the MII address register to write */
+	writel(addr, priv->ioaddr + mii_address);
+	writel(value, priv->ioaddr + mii_data);
+
+	/* Wait until any existing MII operation is complete */
+	ret = stmmac_mdio_wait(priv->ioaddr + mii_data, MII_XGMAC_BUSY);
+
+err_disable_clks:
+	pm_runtime_put(priv->device);
+
+	return ret;
+}
+
+static int stmmac_xgmac2_mdio_write_c22(struct mii_bus *bus, int phyaddr,
+					int phyreg, u16 phydata)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	u32 addr;
+
+	/* Until ver 2.20 XGMAC does not support C22 addr >= 4 */
+	if (priv->synopsys_id < DWXGMAC_CORE_2_20 &&
+	    phyaddr > MII_XGMAC_MAX_C22ADDR)
+		return -ENODEV;
+
+	stmmac_xgmac2_c22_format(priv, phyaddr, phyreg, &addr);
+
+	return stmmac_xgmac2_mdio_write(priv, addr,
+					MII_XGMAC_BUSY | MII_XGMAC_SADDR, phydata);
+}
+
+static int stmmac_xgmac2_mdio_write_c45(struct mii_bus *bus, int phyaddr,
+					int devad, int phyreg, u16 phydata)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	u32 addr;
+
+	stmmac_xgmac2_c45_format(priv, phyaddr, devad, phyreg, &addr);
+
+	return stmmac_xgmac2_mdio_write(priv, addr, MII_XGMAC_BUSY,
+					phydata);
+}
+
+/**
+ * stmmac_mdio_format_addr() - format the address register
+ * @priv: struct stmmac_priv pointer
+ * @pa: 5-bit MDIO package address
+ * @gr: 5-bit MDIO register address (C22) or MDIO device address (C45)
+ *
+ * Return: formatted address register
+ */
+static u32 stmmac_mdio_format_addr(struct stmmac_priv *priv,
+				   unsigned int pa, unsigned int gr)
+{
+	const struct mii_regs *mii_regs = &priv->hw->mii;
+
+	return ((pa << mii_regs->addr_shift) & mii_regs->addr_mask) |
+	       ((gr << mii_regs->reg_shift) & mii_regs->reg_mask) |
+	       priv->gmii_address_bus_config |
+	       MII_ADDR_GBUSY;
+}
+
+static int stmmac_mdio_access(struct stmmac_priv *priv, unsigned int pa,
+			      unsigned int gr, u32 cmd, u32 data, bool read)
+{
+	void __iomem *mii_address = priv->ioaddr + priv->hw->mii.addr;
+	void __iomem *mii_data = priv->ioaddr + priv->hw->mii.data;
+	u32 addr;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(priv->device);
+	if (ret < 0)
+		return ret;
+
+	ret = stmmac_mdio_wait(mii_address, MII_ADDR_GBUSY);
+	if (ret)
+		goto out;
+
+	addr = stmmac_mdio_format_addr(priv, pa, gr) | cmd;
+
+	writel(data, mii_data);
+	writel(addr, mii_address);
+
+	ret = stmmac_mdio_wait(mii_address, MII_ADDR_GBUSY);
+	if (ret)
+		goto out;
+
+	/* Read the data from the MII data register if in read mode */
+	ret = read ? readl(mii_data) & MII_DATA_GD_MASK : 0;
+
+out:
+	pm_runtime_put(priv->device);
+
+	return ret;
+}
+
+static int stmmac_mdio_read(struct stmmac_priv *priv, unsigned int pa,
+			    unsigned int gr, u32 cmd, int data)
+{
+	return stmmac_mdio_access(priv, pa, gr, cmd, data, true);
+}
+
+static int stmmac_mdio_write(struct stmmac_priv *priv, unsigned int pa,
+			     unsigned int gr, u32 cmd, int data)
+{
+	return stmmac_mdio_access(priv, pa, gr, cmd, data, false);
+}
+
+/**
+ * stmmac_mdio_read_c22
+ * @bus: points to the mii_bus structure
+ * @phyaddr: MII addr
+ * @phyreg: MII reg
+ * Description: it reads data from the MII register from within the phy device.
+ * For the 7111 GMAC, we must set the bit 0 in the MII address register while
+ * accessing the PHY registers.
+ * Fortunately, it seems this has no drawback for the 7109 MAC.
+ */
+static int stmmac_mdio_read_c22(struct mii_bus *bus, int phyaddr, int phyreg)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	u32 cmd;
+
+	if (priv->plat->has_gmac4)
+		cmd = MII_GMAC4_READ;
+	else
+		cmd = 0;
+
+	return stmmac_mdio_read(priv, phyaddr, phyreg, cmd, 0);
+}
+
+/**
+ * stmmac_mdio_read_c45
+ * @bus: points to the mii_bus structure
+ * @phyaddr: MII addr
+ * @devad: device address to read
+ * @phyreg: MII reg
+ * Description: it reads data from the MII register from within the phy device.
+ * For the 7111 GMAC, we must set the bit 0 in the MII address register while
+ * accessing the PHY registers.
+ * Fortunately, it seems this has no drawback for the 7109 MAC.
+ */
+static int stmmac_mdio_read_c45(struct mii_bus *bus, int phyaddr, int devad,
+				int phyreg)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	int data = phyreg << MII_GMAC4_REG_ADDR_SHIFT;
+	u32 cmd = MII_GMAC4_READ | MII_GMAC4_C45E;
+
+	return stmmac_mdio_read(priv, phyaddr, devad, cmd, data);
+}
+
+/**
+ * stmmac_mdio_write_c22
+ * @bus: points to the mii_bus structure
+ * @phyaddr: MII addr
+ * @phyreg: MII reg
+ * @phydata: phy data
+ * Description: it writes the data into the MII register from within the device.
+ */
+static int stmmac_mdio_write_c22(struct mii_bus *bus, int phyaddr, int phyreg,
+				 u16 phydata)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	u32 cmd;
+
+	if (priv->plat->has_gmac4)
+		cmd = MII_GMAC4_WRITE;
+	else
+		cmd = MII_ADDR_GWRITE;
+
+	return stmmac_mdio_write(priv, phyaddr, phyreg, cmd, phydata);
+}
+
+/**
+ * stmmac_mdio_write_c45
+ * @bus: points to the mii_bus structure
+ * @phyaddr: MII addr
+ * @phyreg: MII reg
+ * @devad: device address to read
+ * @phydata: phy data
+ * Description: it writes the data into the MII register from within the device.
+ */
+static int stmmac_mdio_write_c45(struct mii_bus *bus, int phyaddr,
+				 int devad, int phyreg, u16 phydata)
+{
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	u32 cmd = MII_GMAC4_WRITE | MII_GMAC4_C45E;
+	int data = phydata;
+
+	data |= phyreg << MII_GMAC4_REG_ADDR_SHIFT;
+
+	return stmmac_mdio_write(priv, phyaddr, devad, cmd, data);
+}
+
+/**
+ * ec_stmmac_mdio_reset
+ * @bus: points to the mii_bus structure
+ * Description: reset the MII bus
+ */
+int ec_stmmac_mdio_reset(struct mii_bus *bus)
+{
+#if IS_ENABLED(CONFIG_STMMAC_PLATFORM)
+	struct stmmac_priv *priv = netdev_priv(bus->priv);
+	unsigned int mii_address = priv->hw->mii.addr;
+
+#ifdef CONFIG_OF
+	if (priv->device->of_node) {
+		struct gpio_desc *reset_gpio;
+		u32 delays[3] = { 0, 0, 0 };
+
+		reset_gpio = devm_gpiod_get_optional(priv->device,
+						     "snps,reset",
+						     GPIOD_OUT_LOW);
+		if (IS_ERR(reset_gpio))
+			return PTR_ERR(reset_gpio);
+
+		device_property_read_u32_array(priv->device,
+					       "snps,reset-delays-us",
+					       delays, ARRAY_SIZE(delays));
+
+		if (delays[0])
+			msleep(DIV_ROUND_UP(delays[0], 1000));
+
+		gpiod_set_value_cansleep(reset_gpio, 1);
+		if (delays[1])
+			msleep(DIV_ROUND_UP(delays[1], 1000));
+
+		gpiod_set_value_cansleep(reset_gpio, 0);
+		if (delays[2])
+			msleep(DIV_ROUND_UP(delays[2], 1000));
+	}
+#endif
+
+	/* This is a workaround for problems with the STE101P PHY.
+	 * It doesn't complete its reset until at least one clock cycle
+	 * on MDC, so perform a dummy mdio read. To be updated for GMAC4
+	 * if needed.
+	 */
+	if (!priv->plat->has_gmac4)
+		writel(0, priv->ioaddr + mii_address);
+#endif
+	return 0;
+}
+
+int ec_stmmac_pcs_setup(struct net_device *ndev)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	struct fwnode_handle *devnode, *pcsnode;
+	struct dw_xpcs *xpcs = NULL;
+	int addr, ret;
+
+	devnode = priv->plat->port_node;
+
+	if (priv->plat->pcs_init) {
+		ret = priv->plat->pcs_init(priv);
+	} else if (fwnode_property_present(devnode, "pcs-handle")) {
+		pcsnode = fwnode_find_reference(devnode, "pcs-handle", 0);
+		xpcs = xpcs_create_fwnode(pcsnode);
+		fwnode_handle_put(pcsnode);
+		ret = PTR_ERR_OR_ZERO(xpcs);
+	} else if (priv->plat->mdio_bus_data &&
+		   priv->plat->mdio_bus_data->pcs_mask) {
+		addr = ffs(priv->plat->mdio_bus_data->pcs_mask) - 1;
+		xpcs = xpcs_create_mdiodev(priv->mii, addr);
+		ret = PTR_ERR_OR_ZERO(xpcs);
+	} else {
+		return 0;
+	}
+
+	if (ret)
+		return dev_err_probe(priv->device, ret, "No xPCS found\n");
+
+	if (xpcs)
+		xpcs_config_eee_mult_fact(xpcs, priv->plat->mult_fact_100ns);
+
+	priv->hw->xpcs = xpcs;
+
+	return 0;
+}
+
+void ec_stmmac_pcs_clean(struct net_device *ndev)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	if (priv->plat->pcs_exit)
+		priv->plat->pcs_exit(priv);
+
+	if (!priv->hw->xpcs)
+		return;
+
+	xpcs_destroy(priv->hw->xpcs);
+	priv->hw->xpcs = NULL;
+}
+
+/**
+ * stmmac_clk_csr_set - dynamically set the MDC clock
+ * @priv: driver private structure
+ * Description: this is to dynamically set the MDC clock according to the csr
+ * clock input.
+ * Return: MII register CR field value
+ * Note:
+ *	If a specific clk_csr value is passed from the platform
+ *	this means that the CSR Clock Range selection cannot be
+ *	changed at run-time and it is fixed (as reported in the driver
+ *	documentation). Viceversa the driver will try to set the MDC
+ *	clock dynamically according to the actual clock input.
+ */
+static u32 stmmac_clk_csr_set(struct stmmac_priv *priv)
+{
+	unsigned long clk_rate;
+	u32 value = ~0;
+
+	clk_rate = clk_get_rate(priv->plat->stmmac_clk);
+
+	/* Platform provided default clk_csr would be assumed valid
+	 * for all other cases except for the below mentioned ones.
+	 * For values higher than the IEEE 802.3 specified frequency
+	 * we can not estimate the proper divider as it is not known
+	 * the frequency of clk_csr_i. So we do not change the default
+	 * divider.
+	 */
+	if (clk_rate < CSR_F_35M)
+		value = STMMAC_CSR_20_35M;
+	else if (clk_rate < CSR_F_60M)
+		value = STMMAC_CSR_35_60M;
+	else if (clk_rate < CSR_F_100M)
+		value = STMMAC_CSR_60_100M;
+	else if (clk_rate < CSR_F_150M)
+		value = STMMAC_CSR_100_150M;
+	else if (clk_rate < CSR_F_250M)
+		value = STMMAC_CSR_150_250M;
+	else if (clk_rate <= CSR_F_300M)
+		value = STMMAC_CSR_250_300M;
+	else if (clk_rate < CSR_F_500M)
+		value = STMMAC_CSR_300_500M;
+	else if (clk_rate < CSR_F_800M)
+		value = STMMAC_CSR_500_800M;
+
+	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) {
+		if (clk_rate > 160000000)
+			value = 0x03;
+		else if (clk_rate > 80000000)
+			value = 0x02;
+		else if (clk_rate > 40000000)
+			value = 0x01;
+		else
+			value = 0;
+	}
+
+	if (priv->plat->has_xgmac) {
+		if (clk_rate > 400000000)
+			value = 0x5;
+		else if (clk_rate > 350000000)
+			value = 0x4;
+		else if (clk_rate > 300000000)
+			value = 0x3;
+		else if (clk_rate > 250000000)
+			value = 0x2;
+		else if (clk_rate > 150000000)
+			value = 0x1;
+		else
+			value = 0x0;
+	}
+
+	return value;
+}
+
+static void stmmac_mdio_bus_config(struct stmmac_priv *priv)
+{
+	u32 value;
+
+	/* If a specific clk_csr value is passed from the platform, this means
+	 * that the CSR Clock Range value should not be computed from the CSR
+	 * clock.
+	 */
+	if (priv->plat->clk_csr >= 0)
+		value = priv->plat->clk_csr;
+	else
+		value = stmmac_clk_csr_set(priv);
+
+	value <<= priv->hw->mii.clk_csr_shift;
+
+	if (value & ~priv->hw->mii.clk_csr_mask)
+		dev_warn(priv->device,
+			 "clk_csr value out of range (0x%08x exceeds mask 0x%08x), truncating\n",
+			 value, priv->hw->mii.clk_csr_mask);
+
+	priv->gmii_address_bus_config = value & priv->hw->mii.clk_csr_mask;
+}
+
+/**
+ * ec_stmmac_mdio_register
+ * @ndev: net device structure
+ * Description: it registers the MII bus
+ */
+int ec_stmmac_mdio_register(struct net_device *ndev)
+{
+	int err = 0;
+	struct mii_bus *new_bus;
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data;
+	struct device_node *mdio_node = priv->plat->mdio_node;
+	struct device *dev = ndev->dev.parent;
+	struct fwnode_handle *fixed_node;
+	struct fwnode_handle *fwnode;
+	int addr, found, max_addr;
+
+	if (!mdio_bus_data)
+		return 0;
+
+	stmmac_mdio_bus_config(priv);
+
+	new_bus = mdiobus_alloc();
+	if (!new_bus)
+		return -ENOMEM;
+
+	if (mdio_bus_data->irqs)
+		memcpy(new_bus->irq, mdio_bus_data->irqs, sizeof(new_bus->irq));
+
+	new_bus->name = "stmmac";
+
+	if (priv->plat->has_xgmac) {
+		new_bus->read = &stmmac_xgmac2_mdio_read_c22;
+		new_bus->write = &stmmac_xgmac2_mdio_write_c22;
+		new_bus->read_c45 = &stmmac_xgmac2_mdio_read_c45;
+		new_bus->write_c45 = &stmmac_xgmac2_mdio_write_c45;
+
+		if (priv->synopsys_id < DWXGMAC_CORE_2_20) {
+			/* Right now only C22 phys are supported */
+			max_addr = MII_XGMAC_MAX_C22ADDR + 1;
+
+			/* Check if DT specified an unsupported phy addr */
+			if (priv->plat->phy_addr > MII_XGMAC_MAX_C22ADDR)
+				dev_err(dev, "Unsupported phy_addr (max=%d)\n",
+					MII_XGMAC_MAX_C22ADDR);
+		} else {
+			/* XGMAC version 2.20 onwards support 32 phy addr */
+			max_addr = PHY_MAX_ADDR;
+		}
+	} else {
+		new_bus->read = &stmmac_mdio_read_c22;
+		new_bus->write = &stmmac_mdio_write_c22;
+		if (priv->plat->has_gmac4) {
+			new_bus->read_c45 = &stmmac_mdio_read_c45;
+			new_bus->write_c45 = &stmmac_mdio_write_c45;
+		}
+
+		max_addr = PHY_MAX_ADDR;
+	}
+
+	if (mdio_bus_data->needs_reset)
+		new_bus->reset = &ec_stmmac_mdio_reset;
+
+	snprintf(new_bus->id, MII_BUS_ID_SIZE, "%s-%x",
+		 new_bus->name, priv->plat->bus_id);
+	new_bus->priv = ndev;
+	new_bus->phy_mask = mdio_bus_data->phy_mask | mdio_bus_data->pcs_mask;
+	new_bus->parent = priv->device;
+
+	err = of_mdiobus_register(new_bus, mdio_node);
+	if (err == -ENODEV) {
+		err = 0;
+		dev_info(dev, "MDIO bus is disabled\n");
+		goto bus_register_fail;
+	} else if (err) {
+		dev_err_probe(dev, err, "Cannot register the MDIO bus\n");
+		goto bus_register_fail;
+	}
+
+	/* Looks like we need a dummy read for XGMAC only and C45 PHYs */
+	if (priv->plat->has_xgmac)
+		stmmac_xgmac2_mdio_read_c45(new_bus, 0, 0, 0);
+
+	/* If fixed-link is set, skip PHY scanning */
+	fwnode = priv->plat->port_node;
+	if (!fwnode)
+		fwnode = dev_fwnode(priv->device);
+
+	if (fwnode) {
+		fixed_node = fwnode_get_named_child_node(fwnode, "fixed-link");
+		if (fixed_node) {
+			fwnode_handle_put(fixed_node);
+			goto bus_register_done;
+		}
+	}
+
+	if (priv->plat->phy_node || mdio_node)
+		goto bus_register_done;
+
+	found = 0;
+	for (addr = 0; addr < max_addr; addr++) {
+		struct phy_device *phydev = mdiobus_get_phy(new_bus, addr);
+
+		if (!phydev)
+			continue;
+
+		/*
+		 * If an IRQ was provided to be assigned after
+		 * the bus probe, do it here.
+		 */
+		if (!mdio_bus_data->irqs &&
+		    (mdio_bus_data->probed_phy_irq > 0)) {
+			new_bus->irq[addr] = mdio_bus_data->probed_phy_irq;
+			phydev->irq = mdio_bus_data->probed_phy_irq;
+		}
+
+		/*
+		 * If we're going to bind the MAC to this PHY bus,
+		 * and no PHY number was provided to the MAC,
+		 * use the one probed here.
+		 */
+		if (priv->plat->phy_addr == -1)
+			priv->plat->phy_addr = addr;
+
+		phy_attached_info(phydev);
+		found = 1;
+	}
+
+	if (!found && !mdio_node) {
+		dev_warn(dev, "No PHY found\n");
+		err = -ENODEV;
+		goto no_phy_found;
+	}
+
+bus_register_done:
+	priv->mii = new_bus;
+
+	return 0;
+
+no_phy_found:
+	mdiobus_unregister(new_bus);
+bus_register_fail:
+	mdiobus_free(new_bus);
+	return err;
+}
+
+/**
+ * ec_stmmac_mdio_unregister
+ * @ndev: net device structure
+ * Description: it unregisters the MII bus
+ */
+int ec_stmmac_mdio_unregister(struct net_device *ndev)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	if (!priv->mii)
+		return 0;
+
+	mdiobus_unregister(priv->mii);
+	priv->mii->priv = NULL;
+	mdiobus_free(priv->mii);
+	priv->mii = NULL;
+
+	return 0;
+}
diff --git a/drivers/net/ethercat/device/K3/stmmac_pci-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_pci-ethercat.c
new file mode 100644
index 000000000000..c30b613cdc26
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_pci-ethercat.c
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This contains the functions to handle the pci driver.
+
+  Copyright (C) 2011-2012  Vayavya Labs Pvt Ltd
+
+
+  Author: Rayagond Kokatanur <rayagond@vayavyalabs.com>
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/clk-provider.h>
+#include <linux/pci.h>
+#include <linux/dmi.h>
+
+#include "stmmac-ethercat.h"
+
+struct stmmac_pci_info {
+	int (*setup)(struct pci_dev *pdev, struct plat_stmmacenet_data *plat);
+};
+
+static void common_default_data(struct plat_stmmacenet_data *plat)
+{
+	/* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */
+	plat->clk_csr = STMMAC_CSR_20_35M;
+	plat->has_gmac = 1;
+	plat->force_sf_dma_mode = 1;
+
+	plat->mdio_bus_data->needs_reset = true;
+
+	/* Set default value for multicast hash bins */
+	plat->multicast_filter_bins = HASH_TABLE_SIZE;
+
+	/* Set default value for unicast filter entries */
+	plat->unicast_filter_entries = 1;
+
+	/* Set the maxmtu to a default of JUMBO_LEN */
+	plat->maxmtu = JUMBO_LEN;
+
+	/* Set default number of RX and TX queues to use */
+	plat->tx_queues_to_use = 1;
+	plat->rx_queues_to_use = 1;
+
+	/* Disable Priority config by default */
+	plat->tx_queues_cfg[0].use_prio = false;
+	plat->rx_queues_cfg[0].use_prio = false;
+
+	/* Disable RX queues routing by default */
+	plat->rx_queues_cfg[0].pkt_route = 0x0;
+}
+
+static int stmmac_default_data(struct pci_dev *pdev,
+			       struct plat_stmmacenet_data *plat)
+{
+	/* Set common default data first */
+	common_default_data(plat);
+
+	plat->bus_id = 1;
+	plat->phy_addr = 0;
+	plat->phy_interface = PHY_INTERFACE_MODE_GMII;
+
+	plat->dma_cfg->pbl = 32;
+	plat->dma_cfg->pblx8 = true;
+	/* TODO: AXI */
+
+	return 0;
+}
+
+static const struct stmmac_pci_info stmmac_pci_info = {
+	.setup = stmmac_default_data,
+};
+
+static int snps_gmac5_default_data(struct pci_dev *pdev,
+				   struct plat_stmmacenet_data *plat)
+{
+	int i;
+
+	plat->clk_csr = STMMAC_CSR_250_300M;
+	plat->has_gmac4 = 1;
+	plat->force_sf_dma_mode = 1;
+	plat->flags |= STMMAC_FLAG_TSO_EN;
+	plat->pmt = 1;
+
+	/* Set default value for multicast hash bins */
+	plat->multicast_filter_bins = HASH_TABLE_SIZE;
+
+	/* Set default value for unicast filter entries */
+	plat->unicast_filter_entries = 1;
+
+	/* Set the maxmtu to a default of JUMBO_LEN */
+	plat->maxmtu = JUMBO_LEN;
+
+	/* Set default number of RX and TX queues to use */
+	plat->tx_queues_to_use = 4;
+	plat->rx_queues_to_use = 4;
+
+	plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WRR;
+	for (i = 0; i < plat->tx_queues_to_use; i++) {
+		plat->tx_queues_cfg[i].use_prio = false;
+		plat->tx_queues_cfg[i].mode_to_use = MTL_QUEUE_DCB;
+		plat->tx_queues_cfg[i].weight = 25;
+		if (i > 0)
+			plat->tx_queues_cfg[i].tbs_en = 1;
+	}
+
+	plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP;
+	for (i = 0; i < plat->rx_queues_to_use; i++) {
+		plat->rx_queues_cfg[i].use_prio = false;
+		plat->rx_queues_cfg[i].mode_to_use = MTL_QUEUE_DCB;
+		plat->rx_queues_cfg[i].pkt_route = 0x0;
+		plat->rx_queues_cfg[i].chan = i;
+	}
+
+	plat->bus_id = 1;
+	plat->phy_addr = -1;
+	plat->phy_interface = PHY_INTERFACE_MODE_GMII;
+
+	plat->dma_cfg->pbl = 32;
+	plat->dma_cfg->pblx8 = true;
+
+	/* Axi Configuration */
+	plat->axi = devm_kzalloc(&pdev->dev, sizeof(*plat->axi), GFP_KERNEL);
+	if (!plat->axi)
+		return -ENOMEM;
+
+	plat->axi->axi_wr_osr_lmt = 31;
+	plat->axi->axi_rd_osr_lmt = 31;
+
+	plat->axi->axi_fb = false;
+	plat->axi->axi_blen[0] = 4;
+	plat->axi->axi_blen[1] = 8;
+	plat->axi->axi_blen[2] = 16;
+	plat->axi->axi_blen[3] = 32;
+
+	return 0;
+}
+
+static const struct stmmac_pci_info snps_gmac5_pci_info = {
+	.setup = snps_gmac5_default_data,
+};
+
+static int stmmac_pci_suspend(struct device *dev, void *bsp_priv)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	int ret;
+
+	ret = pci_save_state(pdev);
+	if (ret)
+		return ret;
+
+	pci_disable_device(pdev);
+	pci_wake_from_d3(pdev, true);
+	return 0;
+}
+
+static int stmmac_pci_resume(struct device *dev, void *bsp_priv)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	int ret;
+
+	pci_restore_state(pdev);
+	pci_set_power_state(pdev, PCI_D0);
+
+	ret = pci_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	pci_set_master(pdev);
+
+	return 0;
+}
+
+/**
+ * stmmac_pci_probe
+ *
+ * @pdev: pci device pointer
+ * @id: pointer to table of device id/id's.
+ *
+ * Description: This probing function gets called for all PCI devices which
+ * match the ID table and are not "owned" by other driver yet. This function
+ * gets passed a "struct pci_dev *" for each device whose entry in the ID table
+ * matches the device. The probe functions returns zero when the driver choose
+ * to take "ownership" of the device or an error code(-ve no) otherwise.
+ */
+static int stmmac_pci_probe(struct pci_dev *pdev,
+			    const struct pci_device_id *id)
+{
+	struct stmmac_pci_info *info = (struct stmmac_pci_info *)id->driver_data;
+	struct plat_stmmacenet_data *plat;
+	struct stmmac_resources res = {};
+	int ret;
+	int i;
+
+	plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL);
+	if (!plat)
+		return -ENOMEM;
+
+	plat->mdio_bus_data = devm_kzalloc(&pdev->dev,
+					   sizeof(*plat->mdio_bus_data),
+					   GFP_KERNEL);
+	if (!plat->mdio_bus_data)
+		return -ENOMEM;
+
+	plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg),
+				     GFP_KERNEL);
+	if (!plat->dma_cfg)
+		return -ENOMEM;
+
+	plat->safety_feat_cfg = devm_kzalloc(&pdev->dev,
+					     sizeof(*plat->safety_feat_cfg),
+					     GFP_KERNEL);
+	if (!plat->safety_feat_cfg)
+		return -ENOMEM;
+
+	/* Enable pci device */
+	ret = pcim_enable_device(pdev);
+	if (ret) {
+		dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n",
+			__func__);
+		return ret;
+	}
+
+	/* Get the base address of device */
+	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
+		if (pci_resource_len(pdev, i) == 0)
+			continue;
+		res.addr = pcim_iomap_region(pdev, i, STMMAC_RESOURCE_NAME);
+		if (IS_ERR(res.addr))
+			return PTR_ERR(res.addr);
+		break;
+	}
+
+	pci_set_master(pdev);
+
+	ret = info->setup(pdev, plat);
+	if (ret)
+		return ret;
+
+	res.wol_irq = pdev->irq;
+	res.irq = pdev->irq;
+
+	plat->safety_feat_cfg->tsoee = 1;
+	plat->safety_feat_cfg->mrxpee = 1;
+	plat->safety_feat_cfg->mestee = 1;
+	plat->safety_feat_cfg->mrxee = 1;
+	plat->safety_feat_cfg->mtxee = 1;
+	plat->safety_feat_cfg->epsi = 1;
+	plat->safety_feat_cfg->edpp = 1;
+	plat->safety_feat_cfg->prtyen = 1;
+	plat->safety_feat_cfg->tmouten = 1;
+
+	plat->suspend = stmmac_pci_suspend;
+	plat->resume = stmmac_pci_resume;
+
+	return ec_stmmac_dvr_probe(&pdev->dev, plat, &res);
+}
+
+/**
+ * stmmac_pci_remove
+ *
+ * @pdev: platform device pointer
+ * Description: this function calls the main to free the net resources.
+ */
+static void stmmac_pci_remove(struct pci_dev *pdev)
+{
+	ec_stmmac_dvr_remove(&pdev->dev);
+}
+
+/* synthetic ID, no official vendor */
+#define PCI_VENDOR_ID_STMMAC		0x0700
+
+#define PCI_DEVICE_ID_STMMAC_STMMAC		0x1108
+#define PCI_DEVICE_ID_SYNOPSYS_GMAC5_ID		0x7102
+
+static const struct pci_device_id stmmac_id_table[] = {
+	{ PCI_DEVICE_DATA(STMMAC, STMMAC, &stmmac_pci_info) },
+	{ PCI_DEVICE_DATA(STMICRO, MAC, &stmmac_pci_info) },
+	{ PCI_DEVICE_DATA(SYNOPSYS, GMAC5_ID, &snps_gmac5_pci_info) },
+	{}
+};
+
+MODULE_DEVICE_TABLE(pci, stmmac_id_table);
+
+static struct pci_driver stmmac_pci_driver = {
+	.name = STMMAC_RESOURCE_NAME,
+	.id_table = stmmac_id_table,
+	.probe = stmmac_pci_probe,
+	.remove = stmmac_pci_remove,
+	.driver         = {
+		.pm     = &ec_stmmac_simple_pm_ops,
+	},
+};
+
+module_pci_driver(stmmac_pci_driver);
+
+MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet PCI driver");
+MODULE_AUTHOR("Rayagond Kokatanur <rayagond.kokatanur@vayavyalabs.com>");
+MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethercat/device/K3/stmmac_pcs-ethercat.h b/drivers/net/ethercat/device/K3/stmmac_pcs-ethercat.h
new file mode 100644
index 000000000000..227ac3822326
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_pcs-ethercat.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * stmmac_pcs.h: Physical Coding Sublayer Header File
+ *
+ * Copyright (C) 2016 STMicroelectronics (R&D) Limited
+ * Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+ */
+
+#ifndef __STMMAC_PCS_H__
+#define __STMMAC_PCS_H__
+
+#include <linux/slab.h>
+#include <linux/io.h>
+#include "common-ethercat.h"
+
+/* PCS registers (AN/TBI/SGMII/RGMII) offsets */
+#define GMAC_AN_CTRL(x)		(x)		/* AN control */
+#define GMAC_AN_STATUS(x)	(x + 0x4)	/* AN status */
+
+/* ADV, LPA and EXP are only available for the TBI and RTBI interfaces */
+#define GMAC_ANE_ADV(x)		(x + 0x8)	/* ANE Advertisement */
+#define GMAC_ANE_LPA(x)		(x + 0xc)	/* ANE link partener ability */
+#define GMAC_ANE_EXP(x)		(x + 0x10)	/* ANE expansion */
+#define GMAC_TBI(x)		(x + 0x14)	/* TBI extend status */
+
+/* AN Configuration defines */
+#define GMAC_AN_CTRL_RAN	BIT(9)	/* Restart Auto-Negotiation */
+#define GMAC_AN_CTRL_ANE	BIT(12)	/* Auto-Negotiation Enable */
+#define GMAC_AN_CTRL_ELE	BIT(14)	/* External Loopback Enable */
+#define GMAC_AN_CTRL_ECD	BIT(16)	/* Enable Comma Detect */
+#define GMAC_AN_CTRL_LR		BIT(17)	/* Lock to Reference */
+#define GMAC_AN_CTRL_SGMRAL	BIT(18)	/* SGMII RAL Control */
+
+/* AN Status defines */
+#define GMAC_AN_STATUS_LS	BIT(2)	/* Link Status 0:down 1:up */
+#define GMAC_AN_STATUS_ANA	BIT(3)	/* Auto-Negotiation Ability */
+#define GMAC_AN_STATUS_ANC	BIT(5)	/* Auto-Negotiation Complete */
+#define GMAC_AN_STATUS_ES	BIT(8)	/* Extended Status */
+
+/* ADV and LPA defines */
+#define GMAC_ANE_FD		BIT(5)
+#define GMAC_ANE_HD		BIT(6)
+#define GMAC_ANE_PSE		GENMASK(8, 7)
+#define GMAC_ANE_PSE_SHIFT	7
+#define GMAC_ANE_RFE		GENMASK(13, 12)
+#define GMAC_ANE_RFE_SHIFT	12
+#define GMAC_ANE_ACK		BIT(14)
+
+/**
+ * dwmac_pcs_isr - TBI, RTBI, or SGMII PHY ISR
+ * @ioaddr: IO registers pointer
+ * @reg: Base address of the AN Control Register.
+ * @intr_status: GMAC core interrupt status
+ * @x: pointer to log these events as stats
+ * Description: it is the ISR for PCS events: Auto-Negotiation Completed and
+ * Link status.
+ */
+static inline void dwmac_pcs_isr(void __iomem *ioaddr, u32 reg,
+				 unsigned int intr_status,
+				 struct stmmac_extra_stats *x)
+{
+	u32 val = readl(ioaddr + GMAC_AN_STATUS(reg));
+
+	if (intr_status & PCS_ANE_IRQ) {
+		x->irq_pcs_ane_n++;
+		if (val & GMAC_AN_STATUS_ANC)
+			pr_info("stmmac_pcs: ANE process completed\n");
+	}
+
+	if (intr_status & PCS_LINK_IRQ) {
+		x->irq_pcs_link_n++;
+		if (val & GMAC_AN_STATUS_LS)
+			pr_info("stmmac_pcs: Link Up\n");
+		else
+			pr_info("stmmac_pcs: Link Down\n");
+	}
+}
+
+/**
+ * dwmac_ctrl_ane - To program the AN Control Register.
+ * @ioaddr: IO registers pointer
+ * @reg: Base address of the AN Control Register.
+ * @ane: to enable the auto-negotiation
+ * @srgmi_ral: to manage MAC-2-MAC SGMII connections.
+ * @loopback: to cause the PHY to loopback tx data into rx path.
+ * Description: this is the main function to configure the AN control register
+ * and init the ANE, select loopback (usually for debugging purpose) and
+ * configure SGMII RAL.
+ */
+static inline void dwmac_ctrl_ane(void __iomem *ioaddr, u32 reg, bool ane,
+				  bool srgmi_ral, bool loopback)
+{
+	u32 value = readl(ioaddr + GMAC_AN_CTRL(reg));
+
+	/* Enable and restart the Auto-Negotiation */
+	if (ane)
+		value |= GMAC_AN_CTRL_ANE | GMAC_AN_CTRL_RAN;
+	else
+		value &= ~GMAC_AN_CTRL_ANE;
+
+	/* In case of MAC-2-MAC connection, block is configured to operate
+	 * according to MAC conf register.
+	 */
+	if (srgmi_ral)
+		value |= GMAC_AN_CTRL_SGMRAL;
+
+	if (loopback)
+		value |= GMAC_AN_CTRL_ELE;
+
+	writel(value, ioaddr + GMAC_AN_CTRL(reg));
+}
+#endif /* __STMMAC_PCS_H__ */
diff --git a/drivers/net/ethercat/device/K3/stmmac_platform-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_platform-ethercat.c
new file mode 100644
index 000000000000..cece3def2e6c
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_platform-ethercat.c
@@ -0,0 +1,1021 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  This contains the functions to handle the platform driver.
+
+  Copyright (C) 2007-2011  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#include <linux/device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/of_mdio.h>
+
+#include "stmmac-ethercat.h"
+#include "stmmac_platform-ethercat.h"
+
+#ifdef CONFIG_OF
+
+/**
+ * dwmac1000_validate_mcast_bins - validates the number of Multicast filter bins
+ * @dev: struct device of the platform device
+ * @mcast_bins: Multicast filtering bins
+ * Description:
+ * this function validates the number of Multicast filtering bins specified
+ * by the configuration through the device tree. The Synopsys GMAC supports
+ * 64 bins, 128 bins, or 256 bins. "bins" refer to the division of CRC
+ * number space. 64 bins correspond to 6 bits of the CRC, 128 corresponds
+ * to 7 bits, and 256 refers to 8 bits of the CRC. Any other setting is
+ * invalid and will cause the filtering algorithm to use Multicast
+ * promiscuous mode.
+ */
+static int dwmac1000_validate_mcast_bins(struct device *dev, int mcast_bins)
+{
+	int x = mcast_bins;
+
+	switch (x) {
+	case HASH_TABLE_SIZE:
+	case 128:
+	case 256:
+		break;
+	default:
+		x = 0;
+		dev_info(dev, "Hash table entries set to unexpected value %d\n",
+			 mcast_bins);
+		break;
+	}
+	return x;
+}
+
+/**
+ * dwmac1000_validate_ucast_entries - validate the Unicast address entries
+ * @dev: struct device of the platform device
+ * @ucast_entries: number of Unicast address entries
+ * Description:
+ * This function validates the number of Unicast address entries supported
+ * by a particular Synopsys 10/100/1000 controller. The Synopsys controller
+ * supports 1..32, 64, or 128 Unicast filter entries for its Unicast filter
+ * logic. This function validates a valid, supported configuration is
+ * selected, and defaults to 1 Unicast address if an unsupported
+ * configuration is selected.
+ */
+static int dwmac1000_validate_ucast_entries(struct device *dev,
+					    int ucast_entries)
+{
+	int x = ucast_entries;
+
+	switch (x) {
+	case 1 ... 32:
+	case 64:
+	case 128:
+		break;
+	default:
+		x = 1;
+		dev_info(dev, "Unicast table entries set to unexpected value %d\n",
+			 ucast_entries);
+		break;
+	}
+	return x;
+}
+
+/**
+ * stmmac_axi_setup - parse DT parameters for programming the AXI register
+ * @pdev: platform device
+ * Description:
+ * if required, from device-tree the AXI internal register can be tuned
+ * by using platform parameters.
+ */
+static struct stmmac_axi *stmmac_axi_setup(struct platform_device *pdev)
+{
+	struct device_node *np;
+	struct stmmac_axi *axi;
+
+	np = of_parse_phandle(pdev->dev.of_node, "snps,axi-config", 0);
+	if (!np)
+		return NULL;
+
+	axi = devm_kzalloc(&pdev->dev, sizeof(*axi), GFP_KERNEL);
+	if (!axi) {
+		of_node_put(np);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	axi->axi_lpi_en = of_property_read_bool(np, "snps,lpi_en");
+	axi->axi_xit_frm = of_property_read_bool(np, "snps,xit_frm");
+	axi->axi_kbbe = of_property_read_bool(np, "snps,kbbe");
+	axi->axi_fb = of_property_read_bool(np, "snps,fb");
+	axi->axi_mb = of_property_read_bool(np, "snps,mb");
+	axi->axi_rb =  of_property_read_bool(np, "snps,rb");
+
+	if (of_property_read_u32(np, "snps,wr_osr_lmt", &axi->axi_wr_osr_lmt))
+		axi->axi_wr_osr_lmt = 1;
+	if (of_property_read_u32(np, "snps,rd_osr_lmt", &axi->axi_rd_osr_lmt))
+		axi->axi_rd_osr_lmt = 1;
+	of_property_read_u32_array(np, "snps,blen", axi->axi_blen, AXI_BLEN);
+	of_node_put(np);
+
+	return axi;
+}
+
+/**
+ * stmmac_mtl_setup - parse DT parameters for multiple queues configuration
+ * @pdev: platform device
+ * @plat: enet data
+ */
+static int stmmac_mtl_setup(struct platform_device *pdev,
+			    struct plat_stmmacenet_data *plat)
+{
+	struct device_node *q_node;
+	struct device_node *rx_node;
+	struct device_node *tx_node;
+	u8 queue = 0;
+	int ret = 0;
+
+	/* For backwards-compatibility with device trees that don't have any
+	 * snps,mtl-rx-config or snps,mtl-tx-config properties, we fall back
+	 * to one RX and TX queues each.
+	 */
+	plat->rx_queues_to_use = 1;
+	plat->tx_queues_to_use = 1;
+
+	/* First Queue must always be in DCB mode. As MTL_QUEUE_DCB = 1 we need
+	 * to always set this, otherwise Queue will be classified as AVB
+	 * (because MTL_QUEUE_AVB = 0).
+	 */
+	plat->rx_queues_cfg[0].mode_to_use = MTL_QUEUE_DCB;
+	plat->tx_queues_cfg[0].mode_to_use = MTL_QUEUE_DCB;
+
+	rx_node = of_parse_phandle(pdev->dev.of_node, "snps,mtl-rx-config", 0);
+	if (!rx_node)
+		return ret;
+
+	tx_node = of_parse_phandle(pdev->dev.of_node, "snps,mtl-tx-config", 0);
+	if (!tx_node) {
+		of_node_put(rx_node);
+		return ret;
+	}
+
+	/* Processing RX queues common config */
+	if (of_property_read_u32(rx_node, "snps,rx-queues-to-use",
+				 &plat->rx_queues_to_use))
+		plat->rx_queues_to_use = 1;
+
+	if (of_property_read_bool(rx_node, "snps,rx-sched-sp"))
+		plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP;
+	else if (of_property_read_bool(rx_node, "snps,rx-sched-wsp"))
+		plat->rx_sched_algorithm = MTL_RX_ALGORITHM_WSP;
+	else
+		plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP;
+
+	/* Processing individual RX queue config */
+	for_each_child_of_node(rx_node, q_node) {
+		if (queue >= plat->rx_queues_to_use)
+			break;
+
+		if (of_property_read_bool(q_node, "snps,dcb-algorithm"))
+			plat->rx_queues_cfg[queue].mode_to_use = MTL_QUEUE_DCB;
+		else if (of_property_read_bool(q_node, "snps,avb-algorithm"))
+			plat->rx_queues_cfg[queue].mode_to_use = MTL_QUEUE_AVB;
+		else
+			plat->rx_queues_cfg[queue].mode_to_use = MTL_QUEUE_DCB;
+
+		if (of_property_read_u32(q_node, "snps,map-to-dma-channel",
+					 &plat->rx_queues_cfg[queue].chan))
+			plat->rx_queues_cfg[queue].chan = queue;
+		/* TODO: Dynamic mapping to be included in the future */
+
+		if (of_property_read_u32(q_node, "snps,priority",
+					&plat->rx_queues_cfg[queue].prio)) {
+			plat->rx_queues_cfg[queue].prio = 0;
+			plat->rx_queues_cfg[queue].use_prio = false;
+		} else {
+			plat->rx_queues_cfg[queue].use_prio = true;
+		}
+
+		/* RX queue specific packet type routing */
+		if (of_property_read_bool(q_node, "snps,route-avcp"))
+			plat->rx_queues_cfg[queue].pkt_route = PACKET_AVCPQ;
+		else if (of_property_read_bool(q_node, "snps,route-ptp"))
+			plat->rx_queues_cfg[queue].pkt_route = PACKET_PTPQ;
+		else if (of_property_read_bool(q_node, "snps,route-dcbcp"))
+			plat->rx_queues_cfg[queue].pkt_route = PACKET_DCBCPQ;
+		else if (of_property_read_bool(q_node, "snps,route-up"))
+			plat->rx_queues_cfg[queue].pkt_route = PACKET_UPQ;
+		else if (of_property_read_bool(q_node, "snps,route-multi-broad"))
+			plat->rx_queues_cfg[queue].pkt_route = PACKET_MCBCQ;
+		else
+			plat->rx_queues_cfg[queue].pkt_route = 0x0;
+
+		queue++;
+	}
+	if (queue != plat->rx_queues_to_use) {
+		ret = -EINVAL;
+		dev_err(&pdev->dev, "Not all RX queues were configured\n");
+		goto out;
+	}
+
+	/* Processing TX queues common config */
+	if (of_property_read_u32(tx_node, "snps,tx-queues-to-use",
+				 &plat->tx_queues_to_use))
+		plat->tx_queues_to_use = 1;
+
+	if (of_property_read_bool(tx_node, "snps,tx-sched-wrr"))
+		plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WRR;
+	else if (of_property_read_bool(tx_node, "snps,tx-sched-wfq"))
+		plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WFQ;
+	else if (of_property_read_bool(tx_node, "snps,tx-sched-dwrr"))
+		plat->tx_sched_algorithm = MTL_TX_ALGORITHM_DWRR;
+	else
+		plat->tx_sched_algorithm = MTL_TX_ALGORITHM_SP;
+
+	queue = 0;
+
+	/* Processing individual TX queue config */
+	for_each_child_of_node(tx_node, q_node) {
+		if (queue >= plat->tx_queues_to_use)
+			break;
+
+		if (of_property_read_u32(q_node, "snps,weight",
+					 &plat->tx_queues_cfg[queue].weight))
+			plat->tx_queues_cfg[queue].weight = 0x10 + queue;
+
+		if (of_property_read_bool(q_node, "snps,dcb-algorithm")) {
+			plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_DCB;
+		} else if (of_property_read_bool(q_node,
+						 "snps,avb-algorithm")) {
+			plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_AVB;
+
+			/* Credit Base Shaper parameters used by AVB */
+			if (of_property_read_u32(q_node, "snps,send_slope",
+				&plat->tx_queues_cfg[queue].send_slope))
+				plat->tx_queues_cfg[queue].send_slope = 0x0;
+			if (of_property_read_u32(q_node, "snps,idle_slope",
+				&plat->tx_queues_cfg[queue].idle_slope))
+				plat->tx_queues_cfg[queue].idle_slope = 0x0;
+			if (of_property_read_u32(q_node, "snps,high_credit",
+				&plat->tx_queues_cfg[queue].high_credit))
+				plat->tx_queues_cfg[queue].high_credit = 0x0;
+			if (of_property_read_u32(q_node, "snps,low_credit",
+				&plat->tx_queues_cfg[queue].low_credit))
+				plat->tx_queues_cfg[queue].low_credit = 0x0;
+		} else {
+			plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_DCB;
+		}
+
+		if (of_property_read_u32(q_node, "snps,priority",
+					&plat->tx_queues_cfg[queue].prio)) {
+			plat->tx_queues_cfg[queue].prio = 0;
+			plat->tx_queues_cfg[queue].use_prio = false;
+		} else {
+			plat->tx_queues_cfg[queue].use_prio = true;
+		}
+
+		plat->tx_queues_cfg[queue].coe_unsupported =
+			of_property_read_bool(q_node, "snps,coe-unsupported");
+
+		queue++;
+	}
+	if (queue != plat->tx_queues_to_use) {
+		ret = -EINVAL;
+		dev_err(&pdev->dev, "Not all TX queues were configured\n");
+		goto out;
+	}
+
+out:
+	of_node_put(rx_node);
+	of_node_put(tx_node);
+	of_node_put(q_node);
+
+	return ret;
+}
+
+/**
+ * stmmac_of_get_mdio() - Gets the MDIO bus from the devicetree.
+ * @np: devicetree node
+ *
+ * The MDIO bus will be searched for in the following ways:
+ * 1. The compatible is "snps,dwc-qos-ethernet-4.10" && a "mdio" named
+ *    child node exists
+ * 2. A child node with the "snps,dwmac-mdio" compatible is present
+ *
+ * Return: The MDIO node if present otherwise NULL
+ */
+static struct device_node *stmmac_of_get_mdio(struct device_node *np)
+{
+	static const struct of_device_id need_mdio_ids[] = {
+		{ .compatible = "snps,dwc-qos-ethernet-4.10" },
+		{},
+	};
+	struct device_node *mdio_node = NULL;
+
+	if (of_match_node(need_mdio_ids, np)) {
+		mdio_node = of_get_child_by_name(np, "mdio");
+	} else {
+		/**
+		 * If snps,dwmac-mdio is passed from DT, always register
+		 * the MDIO
+		 */
+		for_each_child_of_node(np, mdio_node) {
+			if (of_device_is_compatible(mdio_node,
+						    "snps,dwmac-mdio"))
+				break;
+		}
+	}
+
+	return mdio_node;
+}
+
+/**
+ * stmmac_mdio_setup() - Populate platform related MDIO structures.
+ * @plat: driver data platform structure
+ * @np: devicetree node
+ * @dev: device pointer
+ *
+ * This searches for MDIO information from the devicetree.
+ * If an MDIO node is found, it's assigned to plat->mdio_node and
+ * plat->mdio_bus_data is allocated.
+ * If no connection can be determined, just plat->mdio_bus_data is allocated
+ * to indicate a bus should be created and scanned for a phy.
+ * If it's determined there's no MDIO bus needed, both are left NULL.
+ *
+ * This expects that plat->phy_node has already been searched for.
+ *
+ * Return: 0 on success, errno otherwise.
+ */
+static int stmmac_mdio_setup(struct plat_stmmacenet_data *plat,
+			     struct device_node *np, struct device *dev)
+{
+	bool legacy_mdio;
+
+	plat->mdio_node = stmmac_of_get_mdio(np);
+	if (plat->mdio_node)
+		dev_dbg(dev, "Found MDIO subnode\n");
+
+	/* Legacy devicetrees allowed for no MDIO bus description and expect
+	 * the bus to be scanned for devices. If there's no phy or fixed-link
+	 * described assume this is the case since there must be something
+	 * connected to the MAC.
+	 */
+	legacy_mdio = !of_phy_is_fixed_link(np) && !plat->phy_node;
+	if (legacy_mdio)
+		dev_info(dev, "Deprecated MDIO bus assumption used\n");
+
+	if (plat->mdio_node || legacy_mdio) {
+		plat->mdio_bus_data = devm_kzalloc(dev,
+						   sizeof(*plat->mdio_bus_data),
+						   GFP_KERNEL);
+		if (!plat->mdio_bus_data)
+			return -ENOMEM;
+
+		plat->mdio_bus_data->needs_reset = true;
+	}
+
+	return 0;
+}
+
+/**
+ * stmmac_of_get_mac_mode - retrieves the interface of the MAC
+ * @np: - device-tree node
+ * Description:
+ * Similar to `of_get_phy_mode()`, this function will retrieve (from
+ * the device-tree) the interface mode on the MAC side. This assumes
+ * that there is mode converter in-between the MAC & PHY
+ * (e.g. GMII-to-RGMII).
+ */
+static int stmmac_of_get_mac_mode(struct device_node *np)
+{
+	const char *pm;
+	int err, i;
+
+	err = of_property_read_string(np, "mac-mode", &pm);
+	if (err < 0)
+		return err;
+
+	for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++) {
+		if (!strcasecmp(pm, phy_modes(i)))
+			return i;
+	}
+
+	return -ENODEV;
+}
+
+/* Compatible string array for all gmac4 devices */
+static const char * const stmmac_gmac4_compats[] = {
+	"snps,dwmac-4.00",
+	"snps,dwmac-4.10a",
+	"snps,dwmac-4.20a",
+	"snps,dwmac-5.00a",
+	"snps,dwmac-5.10a",
+	"snps,dwmac-5.20",
+	"snps,dwmac-5.30a",
+	NULL
+};
+
+/**
+ * stmmac_probe_config_dt - parse device-tree driver parameters
+ * @pdev: platform_device structure
+ * @mac: MAC address to use
+ * Description:
+ * this function is to read the driver parameters from device-tree and
+ * set some private fields that will be used by the main at runtime.
+ */
+static struct plat_stmmacenet_data *
+stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct plat_stmmacenet_data *plat;
+	struct stmmac_dma_cfg *dma_cfg;
+	static int bus_id = -ENODEV;
+	int phy_mode;
+	void *ret;
+	int rc;
+
+	plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL);
+	if (!plat)
+		return ERR_PTR(-ENOMEM);
+
+	rc = of_get_mac_address(np, mac);
+	if (rc) {
+		if (rc == -EPROBE_DEFER)
+			return ERR_PTR(rc);
+
+		eth_zero_addr(mac);
+	}
+
+	phy_mode = device_get_phy_mode(&pdev->dev);
+	if (phy_mode < 0)
+		return ERR_PTR(phy_mode);
+
+	plat->phy_interface = phy_mode;
+
+	rc = stmmac_of_get_mac_mode(np);
+	if (rc >= 0 && rc != phy_mode)
+		dev_warn(&pdev->dev,
+			 "\"mac-mode\" property used for %s but differs to \"phy-mode\" of %s, and will be ignored. Please report.\n",
+			 phy_modes(rc), phy_modes(phy_mode));
+
+	/* Some wrapper drivers still rely on phy_node. Let's save it while
+	 * they are not converted to phylink. */
+	plat->phy_node = of_parse_phandle(np, "phy-handle", 0);
+
+	/* PHYLINK automatically parses the phy-handle property */
+	plat->port_node = of_fwnode_handle(np);
+
+	/* Get max speed of operation from device tree */
+	of_property_read_u32(np, "max-speed", &plat->max_speed);
+
+	plat->bus_id = of_alias_get_id(np, "ethernet");
+	if (plat->bus_id < 0) {
+		if (bus_id < 0)
+			bus_id = of_alias_get_highest_id("ethernet");
+		/* No ethernet alias found, init at -1 so first bus_id is 0 */
+		if (bus_id < 0)
+			bus_id = -1;
+		plat->bus_id = ++bus_id;
+	}
+
+	/* Default to phy auto-detection */
+	plat->phy_addr = -1;
+
+	/* Default to get clk_csr from stmmac_clk_csr_set(),
+	 * or get clk_csr from device tree.
+	 */
+	plat->clk_csr = -1;
+	if (of_property_read_u32(np, "snps,clk-csr", &plat->clk_csr))
+		of_property_read_u32(np, "clk_csr", &plat->clk_csr);
+
+	/* "snps,phy-addr" is not a standard property. Mark it as deprecated
+	 * and warn of its use. Remove this when phy node support is added.
+	 */
+	if (of_property_read_u32(np, "snps,phy-addr", &plat->phy_addr) == 0)
+		dev_warn(&pdev->dev, "snps,phy-addr property is deprecated\n");
+
+	rc = stmmac_mdio_setup(plat, np, &pdev->dev);
+	if (rc) {
+		ret = ERR_PTR(rc);
+		goto error_put_phy;
+	}
+
+	of_property_read_u32(np, "tx-fifo-depth", &plat->tx_fifo_size);
+
+	of_property_read_u32(np, "rx-fifo-depth", &plat->rx_fifo_size);
+
+	plat->force_sf_dma_mode =
+		of_property_read_bool(np, "snps,force_sf_dma_mode");
+
+	if (of_property_read_bool(np, "snps,en-tx-lpi-clockgating")) {
+		dev_warn(&pdev->dev,
+			 "OF property snps,en-tx-lpi-clockgating is deprecated, please convert driver to use STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP\n");
+		plat->flags |= STMMAC_FLAG_EN_TX_LPI_CLOCKGATING;
+	}
+
+	/* Set the maxmtu to a default of JUMBO_LEN in case the
+	 * parameter is not present in the device tree.
+	 */
+	plat->maxmtu = JUMBO_LEN;
+
+	/* Set default value for multicast hash bins */
+	plat->multicast_filter_bins = HASH_TABLE_SIZE;
+
+	/* Set default value for unicast filter entries */
+	plat->unicast_filter_entries = 1;
+
+	/*
+	 * Currently only the properties needed on SPEAr600
+	 * are provided. All other properties should be added
+	 * once needed on other platforms.
+	 */
+	if (of_device_is_compatible(np, "st,spear600-gmac") ||
+		of_device_is_compatible(np, "snps,dwmac-3.50a") ||
+		of_device_is_compatible(np, "snps,dwmac-3.70a") ||
+		of_device_is_compatible(np, "snps,dwmac-3.72a") ||
+		of_device_is_compatible(np, "snps,dwmac")) {
+		/* Note that the max-frame-size parameter as defined in the
+		 * ePAPR v1.1 spec is defined as max-frame-size, it's
+		 * actually used as the IEEE definition of MAC Client
+		 * data, or MTU. The ePAPR specification is confusing as
+		 * the definition is max-frame-size, but usage examples
+		 * are clearly MTUs
+		 */
+		of_property_read_u32(np, "max-frame-size", &plat->maxmtu);
+		of_property_read_u32(np, "snps,multicast-filter-bins",
+				     &plat->multicast_filter_bins);
+		of_property_read_u32(np, "snps,perfect-filter-entries",
+				     &plat->unicast_filter_entries);
+		plat->unicast_filter_entries = dwmac1000_validate_ucast_entries(
+				&pdev->dev, plat->unicast_filter_entries);
+		plat->multicast_filter_bins = dwmac1000_validate_mcast_bins(
+				&pdev->dev, plat->multicast_filter_bins);
+		plat->has_gmac = 1;
+		plat->pmt = 1;
+	}
+
+	if (of_device_is_compatible(np, "snps,dwmac-3.40a")) {
+		plat->has_gmac = 1;
+		plat->enh_desc = 1;
+		plat->tx_coe = 1;
+		plat->bugged_jumbo = 1;
+		plat->pmt = 1;
+	}
+
+	if (of_device_compatible_match(np, stmmac_gmac4_compats)) {
+		plat->has_gmac4 = 1;
+		plat->has_gmac = 0;
+		plat->pmt = 1;
+		if (of_property_read_bool(np, "snps,tso"))
+			plat->flags |= STMMAC_FLAG_TSO_EN;
+	}
+
+	if (of_device_is_compatible(np, "snps,dwmac-3.610") ||
+		of_device_is_compatible(np, "snps,dwmac-3.710")) {
+		plat->enh_desc = 1;
+		plat->bugged_jumbo = 1;
+		plat->force_sf_dma_mode = 1;
+	}
+
+	if (of_device_is_compatible(np, "snps,dwxgmac")) {
+		plat->has_xgmac = 1;
+		plat->pmt = 1;
+		if (of_property_read_bool(np, "snps,tso"))
+			plat->flags |= STMMAC_FLAG_TSO_EN;
+		of_property_read_u32(np, "snps,multicast-filter-bins",
+				     &plat->multicast_filter_bins);
+	}
+
+	dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*dma_cfg),
+			       GFP_KERNEL);
+	if (!dma_cfg) {
+		ret = ERR_PTR(-ENOMEM);
+		goto error_put_mdio;
+	}
+	plat->dma_cfg = dma_cfg;
+
+	of_property_read_u32(np, "snps,pbl", &dma_cfg->pbl);
+	if (!dma_cfg->pbl)
+		dma_cfg->pbl = DEFAULT_DMA_PBL;
+	of_property_read_u32(np, "snps,txpbl", &dma_cfg->txpbl);
+	of_property_read_u32(np, "snps,rxpbl", &dma_cfg->rxpbl);
+	dma_cfg->pblx8 = !of_property_read_bool(np, "snps,no-pbl-x8");
+
+	dma_cfg->aal = of_property_read_bool(np, "snps,aal");
+	dma_cfg->fixed_burst = of_property_read_bool(np, "snps,fixed-burst");
+	dma_cfg->mixed_burst = of_property_read_bool(np, "snps,mixed-burst");
+
+	plat->force_thresh_dma_mode = of_property_read_bool(np, "snps,force_thresh_dma_mode");
+	if (plat->force_thresh_dma_mode && plat->force_sf_dma_mode) {
+		plat->force_sf_dma_mode = 0;
+		dev_warn(&pdev->dev,
+			 "force_sf_dma_mode is ignored if force_thresh_dma_mode is set.\n");
+	}
+
+	of_property_read_u32(np, "snps,ps-speed", &plat->mac_port_sel_speed);
+
+	plat->axi = stmmac_axi_setup(pdev);
+
+	rc = stmmac_mtl_setup(pdev, plat);
+	if (rc) {
+		ret = ERR_PTR(rc);
+		goto error_put_mdio;
+	}
+
+	/* clock setup */
+	if (!of_device_is_compatible(np, "snps,dwc-qos-ethernet-4.10")) {
+		plat->stmmac_clk = devm_clk_get(&pdev->dev,
+						STMMAC_RESOURCE_NAME);
+		if (IS_ERR(plat->stmmac_clk)) {
+			dev_warn(&pdev->dev, "Cannot get CSR clock\n");
+			plat->stmmac_clk = NULL;
+		}
+		clk_prepare_enable(plat->stmmac_clk);
+	}
+
+	plat->pclk = devm_clk_get_optional(&pdev->dev, "pclk");
+	if (IS_ERR(plat->pclk)) {
+		ret = plat->pclk;
+		goto error_pclk_get;
+	}
+	clk_prepare_enable(plat->pclk);
+
+	/* Fall-back to main clock in case of no PTP ref is passed */
+	plat->clk_ptp_ref = devm_clk_get(&pdev->dev, "ptp_ref");
+	if (IS_ERR(plat->clk_ptp_ref)) {
+		plat->clk_ptp_rate = clk_get_rate(plat->stmmac_clk);
+		plat->clk_ptp_ref = NULL;
+		dev_info(&pdev->dev, "PTP uses main clock\n");
+	} else {
+		plat->clk_ptp_rate = clk_get_rate(plat->clk_ptp_ref);
+		dev_dbg(&pdev->dev, "PTP rate %lu\n", plat->clk_ptp_rate);
+	}
+
+	plat->stmmac_rst = devm_reset_control_get_optional(&pdev->dev,
+							   STMMAC_RESOURCE_NAME);
+	if (IS_ERR(plat->stmmac_rst)) {
+		ret = plat->stmmac_rst;
+		goto error_hw_init;
+	}
+
+	plat->stmmac_ahb_rst = devm_reset_control_get_optional_shared(
+							&pdev->dev, "ahb");
+	if (IS_ERR(plat->stmmac_ahb_rst)) {
+		ret = plat->stmmac_ahb_rst;
+		goto error_hw_init;
+	}
+
+	return plat;
+
+error_hw_init:
+	clk_disable_unprepare(plat->pclk);
+error_pclk_get:
+	clk_disable_unprepare(plat->stmmac_clk);
+error_put_mdio:
+	of_node_put(plat->mdio_node);
+error_put_phy:
+	of_node_put(plat->phy_node);
+
+	return ret;
+}
+
+static void devm_stmmac_remove_config_dt(void *data)
+{
+	struct plat_stmmacenet_data *plat = data;
+
+	clk_disable_unprepare(plat->stmmac_clk);
+	clk_disable_unprepare(plat->pclk);
+	of_node_put(plat->mdio_node);
+	of_node_put(plat->phy_node);
+}
+
+/**
+ * devm_ec_stmmac_probe_config_dt
+ * @pdev: platform_device structure
+ * @mac: MAC address to use
+ * Description: Devres variant of stmmac_probe_config_dt().
+ */
+struct plat_stmmacenet_data *
+devm_ec_stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
+{
+	struct plat_stmmacenet_data *plat;
+	int ret;
+
+	plat = stmmac_probe_config_dt(pdev, mac);
+	if (IS_ERR(plat))
+		return plat;
+
+	ret = devm_add_action_or_reset(&pdev->dev,
+				       devm_stmmac_remove_config_dt, plat);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return plat;
+}
+#else
+struct plat_stmmacenet_data *
+devm_ec_stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
+{
+	return ERR_PTR(-EINVAL);
+}
+#endif /* CONFIG_OF */
+EXPORT_SYMBOL_GPL(devm_ec_stmmac_probe_config_dt);
+
+struct clk *ec_stmmac_pltfr_find_clk(struct plat_stmmacenet_data *plat_dat,
+				  const char *name)
+{
+	for (int i = 0; i < plat_dat->num_clks; i++)
+		if (strcmp(plat_dat->clks[i].id, name) == 0)
+			return plat_dat->clks[i].clk;
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_pltfr_find_clk);
+
+int ec_stmmac_get_platform_resources(struct platform_device *pdev,
+				  struct stmmac_resources *stmmac_res)
+{
+	memset(stmmac_res, 0, sizeof(*stmmac_res));
+
+	/* Get IRQ information early to have an ability to ask for deferred
+	 * probe if needed before we went too far with resource allocation.
+	 */
+	stmmac_res->irq = platform_get_irq_byname(pdev, "macirq");
+	if (stmmac_res->irq < 0)
+		return stmmac_res->irq;
+
+	/* On some platforms e.g. SPEAr the wake up irq differs from the mac irq
+	 * The external wake up irq can be passed through the platform code
+	 * named as "eth_wake_irq"
+	 *
+	 * In case the wake up interrupt is not passed from the platform
+	 * so the driver will continue to use the mac irq (ndev->irq)
+	 */
+	stmmac_res->wol_irq =
+		platform_get_irq_byname_optional(pdev, "eth_wake_irq");
+	if (stmmac_res->wol_irq < 0) {
+		if (stmmac_res->wol_irq == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(&pdev->dev, "IRQ eth_wake_irq not found\n");
+		stmmac_res->wol_irq = stmmac_res->irq;
+	}
+
+	stmmac_res->lpi_irq =
+		platform_get_irq_byname_optional(pdev, "eth_lpi");
+	if (stmmac_res->lpi_irq < 0) {
+		if (stmmac_res->lpi_irq == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(&pdev->dev, "IRQ eth_lpi not found\n");
+	}
+
+	stmmac_res->sfty_irq =
+		platform_get_irq_byname_optional(pdev, "sfty");
+	if (stmmac_res->sfty_irq < 0) {
+		if (stmmac_res->sfty_irq == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(&pdev->dev, "IRQ sfty not found\n");
+	}
+
+	stmmac_res->addr = devm_platform_ioremap_resource(pdev, 0);
+
+	return PTR_ERR_OR_ZERO(stmmac_res->addr);
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_get_platform_resources);
+
+/**
+ * stmmac_pltfr_init
+ * @pdev: pointer to the platform device
+ * @plat: driver data platform structure
+ * Description: Call the platform's init callback (if any) and propagate
+ * the return value.
+ */
+static int stmmac_pltfr_init(struct platform_device *pdev,
+			     struct plat_stmmacenet_data *plat)
+{
+	int ret = 0;
+
+	if (plat->init)
+		ret = plat->init(pdev, plat->bsp_priv);
+
+	return ret;
+}
+
+/**
+ * stmmac_pltfr_exit
+ * @pdev: pointer to the platform device
+ * @plat: driver data platform structure
+ * Description: Call the platform's exit callback (if any).
+ */
+static void stmmac_pltfr_exit(struct platform_device *pdev,
+			      struct plat_stmmacenet_data *plat)
+{
+	if (plat->exit)
+		plat->exit(pdev, plat->bsp_priv);
+}
+
+static int stmmac_plat_suspend(struct device *dev, void *bsp_priv)
+{
+	struct stmmac_priv *priv = netdev_priv(dev_get_drvdata(dev));
+
+	stmmac_pltfr_exit(to_platform_device(dev), priv->plat);
+
+	return 0;
+}
+
+static int stmmac_plat_resume(struct device *dev, void *bsp_priv)
+{
+	struct stmmac_priv *priv = netdev_priv(dev_get_drvdata(dev));
+
+	return stmmac_pltfr_init(to_platform_device(dev), priv->plat);
+}
+
+/**
+ * ec_stmmac_pltfr_probe
+ * @pdev: platform device pointer
+ * @plat: driver data platform structure
+ * @res: stmmac resources structure
+ * Description: This calls the platform's init() callback and probes the
+ * stmmac driver.
+ */
+int ec_stmmac_pltfr_probe(struct platform_device *pdev,
+		       struct plat_stmmacenet_data *plat,
+		       struct stmmac_resources *res)
+{
+	int ret;
+
+	if (!plat->suspend && plat->exit)
+		plat->suspend = stmmac_plat_suspend;
+	if (!plat->resume && plat->init)
+		plat->resume = stmmac_plat_resume;
+
+	ret = stmmac_pltfr_init(pdev, plat);
+	if (ret)
+		return ret;
+
+	ret = ec_stmmac_dvr_probe(&pdev->dev, plat, res);
+	if (ret) {
+		stmmac_pltfr_exit(pdev, plat);
+		return ret;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_pltfr_probe);
+
+static void devm_stmmac_pltfr_remove(void *data)
+{
+	struct platform_device *pdev = data;
+
+	ec_stmmac_pltfr_remove(pdev);
+}
+
+/**
+ * devm_ec_stmmac_pltfr_probe
+ * @pdev: pointer to the platform device
+ * @plat: driver data platform structure
+ * @res: stmmac resources
+ * Description: Devres variant of ec_stmmac_pltfr_probe(). Allows users to skip
+ * calling ec_stmmac_pltfr_remove() on driver detach.
+ */
+int devm_ec_stmmac_pltfr_probe(struct platform_device *pdev,
+			    struct plat_stmmacenet_data *plat,
+			    struct stmmac_resources *res)
+{
+	int ret;
+
+	ret = ec_stmmac_pltfr_probe(pdev, plat, res);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(&pdev->dev, devm_stmmac_pltfr_remove,
+					pdev);
+}
+EXPORT_SYMBOL_GPL(devm_ec_stmmac_pltfr_probe);
+
+/**
+ * ec_stmmac_pltfr_remove
+ * @pdev: pointer to the platform device
+ * Description: This undoes the effects of ec_stmmac_pltfr_probe() by removing the
+ * driver and calling the platform's exit() callback.
+ */
+void ec_stmmac_pltfr_remove(struct platform_device *pdev)
+{
+	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	struct plat_stmmacenet_data *plat = priv->plat;
+
+	ec_stmmac_dvr_remove(&pdev->dev);
+	stmmac_pltfr_exit(pdev, plat);
+}
+EXPORT_SYMBOL_GPL(ec_stmmac_pltfr_remove);
+
+static int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled)
+{
+	struct plat_stmmacenet_data *plat_dat = priv->plat;
+	int ret;
+
+	if (enabled) {
+		ret = clk_prepare_enable(plat_dat->stmmac_clk);
+		if (ret)
+			return ret;
+		ret = clk_prepare_enable(plat_dat->pclk);
+		if (ret) {
+			clk_disable_unprepare(plat_dat->stmmac_clk);
+			return ret;
+		}
+		if (plat_dat->clks_config) {
+			ret = plat_dat->clks_config(plat_dat->bsp_priv, enabled);
+			if (ret) {
+				clk_disable_unprepare(plat_dat->stmmac_clk);
+				clk_disable_unprepare(plat_dat->pclk);
+				return ret;
+			}
+		}
+	} else {
+		clk_disable_unprepare(plat_dat->stmmac_clk);
+		clk_disable_unprepare(plat_dat->pclk);
+		if (plat_dat->clks_config)
+			plat_dat->clks_config(plat_dat->bsp_priv, enabled);
+	}
+
+	return 0;
+}
+
+static int __maybe_unused stmmac_runtime_suspend(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	stmmac_bus_clks_config(priv, false);
+
+	return 0;
+}
+
+static int __maybe_unused stmmac_runtime_resume(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	return stmmac_bus_clks_config(priv, true);
+}
+
+static int __maybe_unused stmmac_pltfr_noirq_suspend(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	if (!netif_running(ndev))
+		return 0;
+
+	if (!stmmac_wol_enabled_mac(priv)) {
+		/* Disable clock in case of PWM is off */
+		clk_disable_unprepare(priv->plat->clk_ptp_ref);
+
+		ret = pm_runtime_force_suspend(dev);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int __maybe_unused stmmac_pltfr_noirq_resume(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	if (!netif_running(ndev))
+		return 0;
+
+	if (!stmmac_wol_enabled_mac(priv)) {
+		/* enable the clk previously disabled */
+		ret = pm_runtime_force_resume(dev);
+		if (ret)
+			return ret;
+
+		ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
+		if (ret < 0) {
+			netdev_warn(priv->dev,
+				    "failed to enable PTP reference clock: %pe\n",
+				    ERR_PTR(ret));
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+const struct dev_pm_ops ec_stmmac_pltfr_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(ec_stmmac_suspend, ec_stmmac_resume)
+	SET_RUNTIME_PM_OPS(stmmac_runtime_suspend, stmmac_runtime_resume, NULL)
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(stmmac_pltfr_noirq_suspend, stmmac_pltfr_noirq_resume)
+};
+EXPORT_SYMBOL_GPL(ec_stmmac_pltfr_pm_ops);
+
+MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet platform support");
+MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethercat/device/K3/stmmac_platform-ethercat.h b/drivers/net/ethercat/device/K3/stmmac_platform-ethercat.h
new file mode 100644
index 000000000000..2f45cac3b645
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_platform-ethercat.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*******************************************************************************
+  Copyright (C) 2007-2009  STMicroelectronics Ltd
+
+
+  Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
+*******************************************************************************/
+
+#ifndef __STMMAC_PLATFORM_H__
+#define __STMMAC_PLATFORM_H__
+
+#include "stmmac-ethercat.h"
+
+struct plat_stmmacenet_data *
+devm_ec_stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac);
+
+struct clk *ec_stmmac_pltfr_find_clk(struct plat_stmmacenet_data *plat_dat,
+				  const char *name);
+
+int ec_stmmac_get_platform_resources(struct platform_device *pdev,
+				  struct stmmac_resources *stmmac_res);
+
+int ec_stmmac_pltfr_probe(struct platform_device *pdev,
+		       struct plat_stmmacenet_data *plat,
+		       struct stmmac_resources *res);
+int devm_ec_stmmac_pltfr_probe(struct platform_device *pdev,
+			    struct plat_stmmacenet_data *plat,
+			    struct stmmac_resources *res);
+void ec_stmmac_pltfr_remove(struct platform_device *pdev);
+extern const struct dev_pm_ops ec_stmmac_pltfr_pm_ops;
+
+static inline void *get_stmmac_bsp_priv(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	return priv->plat->bsp_priv;
+}
+
+#endif /* __STMMAC_PLATFORM_H__ */
diff --git a/drivers/net/ethercat/device/K3/stmmac_ptp-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_ptp-ethercat.c
new file mode 100644
index 000000000000..3c8b6cb4702c
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_ptp-ethercat.c
@@ -0,0 +1,397 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*******************************************************************************
+  PTP 1588 clock using the STMMAC.
+
+  Copyright (C) 2013  Vayavya Labs Pvt Ltd
+
+
+  Author: Rayagond Kokatanur <rayagond@vayavyalabs.com>
+*******************************************************************************/
+#include "stmmac-ethercat.h"
+#include "stmmac_ptp-ethercat.h"
+
+#define PTP_SAFE_TIME_OFFSET_NS	500000
+
+/**
+ * stmmac_adjust_freq
+ *
+ * @ptp: pointer to ptp_clock_info structure
+ * @scaled_ppm: desired period change in scaled parts per million
+ *
+ * Description: this function will adjust the frequency of hardware clock.
+ *
+ * Scaled parts per million is ppm with a 16-bit binary fractional field.
+ */
+static int stmmac_adjust_freq(struct ptp_clock_info *ptp, long scaled_ppm)
+{
+	struct stmmac_priv *priv =
+	    container_of(ptp, struct stmmac_priv, ptp_clock_ops);
+	unsigned long flags;
+	u32 addend;
+
+	addend = adjust_by_scaled_ppm(priv->default_addend, scaled_ppm);
+
+	write_lock_irqsave(&priv->ptp_lock, flags);
+	stmmac_config_addend(priv, priv->ptpaddr, addend);
+	write_unlock_irqrestore(&priv->ptp_lock, flags);
+
+	return 0;
+}
+
+/**
+ * stmmac_adjust_time
+ *
+ * @ptp: pointer to ptp_clock_info structure
+ * @delta: desired change in nanoseconds
+ *
+ * Description: this function will shift/adjust the hardware clock time.
+ */
+static int stmmac_adjust_time(struct ptp_clock_info *ptp, s64 delta)
+{
+	struct stmmac_priv *priv =
+	    container_of(ptp, struct stmmac_priv, ptp_clock_ops);
+	unsigned long flags;
+	u32 sec, nsec;
+	u32 quotient, reminder;
+	int neg_adj = 0;
+	bool xmac, est_rst = false;
+	int ret;
+
+	xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
+
+	if (delta < 0) {
+		neg_adj = 1;
+		delta = -delta;
+	}
+
+	quotient = div_u64_rem(delta, 1000000000ULL, &reminder);
+	sec = quotient;
+	nsec = reminder;
+
+	/* If EST is enabled, disabled it before adjust ptp time. */
+	if (priv->est && priv->est->enable) {
+		est_rst = true;
+		mutex_lock(&priv->est_lock);
+		priv->est->enable = false;
+		stmmac_est_configure(priv, priv, priv->est,
+				     priv->plat->clk_ptp_rate);
+		mutex_unlock(&priv->est_lock);
+	}
+
+	write_lock_irqsave(&priv->ptp_lock, flags);
+	stmmac_adjust_systime(priv, priv->ptpaddr, sec, nsec, neg_adj, xmac);
+	write_unlock_irqrestore(&priv->ptp_lock, flags);
+
+	/* Calculate new basetime and re-configured EST after PTP time adjust. */
+	if (est_rst) {
+		struct timespec64 current_time, time;
+		ktime_t current_time_ns, basetime;
+		u64 cycle_time;
+
+		mutex_lock(&priv->est_lock);
+		priv->ptp_clock_ops.gettime64(&priv->ptp_clock_ops, &current_time);
+		current_time_ns = timespec64_to_ktime(current_time);
+		time.tv_nsec = priv->est->btr_reserve[0];
+		time.tv_sec = priv->est->btr_reserve[1];
+		basetime = timespec64_to_ktime(time);
+		cycle_time = (u64)priv->est->ctr[1] * NSEC_PER_SEC +
+			     priv->est->ctr[0];
+		time = ec_stmmac_calc_tas_basetime(basetime,
+						current_time_ns,
+						cycle_time);
+
+		priv->est->btr[0] = (u32)time.tv_nsec;
+		priv->est->btr[1] = (u32)time.tv_sec;
+		priv->est->enable = true;
+		ret = stmmac_est_configure(priv, priv, priv->est,
+					   priv->plat->clk_ptp_rate);
+		mutex_unlock(&priv->est_lock);
+		if (ret)
+			netdev_err(priv->dev, "failed to configure EST\n");
+	}
+
+	return 0;
+}
+
+/**
+ * stmmac_get_time
+ *
+ * @ptp: pointer to ptp_clock_info structure
+ * @ts: pointer to hold time/result
+ *
+ * Description: this function will read the current time from the
+ * hardware clock and store it in @ts.
+ */
+static int stmmac_get_time(struct ptp_clock_info *ptp, struct timespec64 *ts)
+{
+	struct stmmac_priv *priv =
+	    container_of(ptp, struct stmmac_priv, ptp_clock_ops);
+	unsigned long flags;
+	u64 ns = 0;
+
+	read_lock_irqsave(&priv->ptp_lock, flags);
+	stmmac_get_systime(priv, priv->ptpaddr, &ns);
+	read_unlock_irqrestore(&priv->ptp_lock, flags);
+
+	*ts = ns_to_timespec64(ns);
+
+	return 0;
+}
+
+/**
+ * stmmac_set_time
+ *
+ * @ptp: pointer to ptp_clock_info structure
+ * @ts: time value to set
+ *
+ * Description: this function will set the current time on the
+ * hardware clock.
+ */
+static int stmmac_set_time(struct ptp_clock_info *ptp,
+			   const struct timespec64 *ts)
+{
+	struct stmmac_priv *priv =
+	    container_of(ptp, struct stmmac_priv, ptp_clock_ops);
+	unsigned long flags;
+
+	write_lock_irqsave(&priv->ptp_lock, flags);
+	stmmac_init_systime(priv, priv->ptpaddr, ts->tv_sec, ts->tv_nsec);
+	write_unlock_irqrestore(&priv->ptp_lock, flags);
+
+	return 0;
+}
+
+static int stmmac_enable(struct ptp_clock_info *ptp,
+			 struct ptp_clock_request *rq, int on)
+{
+	struct stmmac_priv *priv =
+	    container_of(ptp, struct stmmac_priv, ptp_clock_ops);
+	void __iomem *ptpaddr = priv->ptpaddr;
+	struct stmmac_pps_cfg *cfg;
+	int ret = -EOPNOTSUPP;
+	unsigned long flags;
+	u32 acr_value;
+
+	switch (rq->type) {
+	case PTP_CLK_REQ_PEROUT: {
+		struct timespec64 curr_time;
+		u64 target_ns = 0;
+		u64 ns = 0;
+
+		/* Reject requests with unsupported flags */
+		if (rq->perout.flags)
+			return -EOPNOTSUPP;
+
+		cfg = &priv->pps[rq->perout.index];
+
+		cfg->start.tv_sec = rq->perout.start.sec;
+		cfg->start.tv_nsec = rq->perout.start.nsec;
+
+		/* A time set in the past won't trigger the start of the flexible PPS generation for
+		 * the GMAC5. For some reason it does for the GMAC4 but setting a time in the past
+		 * should be addressed anyway. Therefore, any value set it the past is considered as
+		 * an offset compared to the current MAC system time.
+		 * Be aware that an offset too low may not trigger flexible PPS generation
+		 * if time spent in this configuration makes the targeted time already outdated.
+		 * To address this, add a safe time offset.
+		 */
+		if (!cfg->start.tv_sec && cfg->start.tv_nsec < PTP_SAFE_TIME_OFFSET_NS)
+			cfg->start.tv_nsec += PTP_SAFE_TIME_OFFSET_NS;
+
+		target_ns = cfg->start.tv_nsec + ((u64)cfg->start.tv_sec * NSEC_PER_SEC);
+
+		stmmac_get_systime(priv, priv->ptpaddr, &ns);
+		if (ns > TIME64_MAX - PTP_SAFE_TIME_OFFSET_NS)
+			return -EINVAL;
+
+		curr_time = ns_to_timespec64(ns);
+		if (target_ns < ns + PTP_SAFE_TIME_OFFSET_NS) {
+			cfg->start = timespec64_add_safe(cfg->start, curr_time);
+			if (cfg->start.tv_sec == TIME64_MAX)
+				return -EINVAL;
+		}
+
+		cfg->period.tv_sec = rq->perout.period.sec;
+		cfg->period.tv_nsec = rq->perout.period.nsec;
+
+		write_lock_irqsave(&priv->ptp_lock, flags);
+		ret = stmmac_flex_pps_config(priv, priv->ioaddr,
+					     rq->perout.index, cfg, on,
+					     priv->sub_second_inc,
+					     priv->systime_flags);
+		write_unlock_irqrestore(&priv->ptp_lock, flags);
+		break;
+	}
+	case PTP_CLK_REQ_EXTTS: {
+		u8 channel;
+
+		mutex_lock(&priv->aux_ts_lock);
+		acr_value = readl(ptpaddr + PTP_ACR);
+		channel = ilog2(FIELD_GET(PTP_ACR_MASK, acr_value));
+		acr_value &= ~PTP_ACR_MASK;
+
+		if (on) {
+			if (FIELD_GET(PTP_ACR_MASK, acr_value)) {
+				netdev_err(priv->dev,
+					   "Cannot enable auxiliary snapshot %d as auxiliary snapshot %d is already enabled",
+					rq->extts.index, channel);
+				mutex_unlock(&priv->aux_ts_lock);
+				return -EBUSY;
+			}
+
+			priv->plat->flags |= STMMAC_FLAG_EXT_SNAPSHOT_EN;
+
+			/* Enable External snapshot trigger */
+			acr_value |= PTP_ACR_ATSEN(rq->extts.index);
+			acr_value |= PTP_ACR_ATSFC;
+		} else {
+			priv->plat->flags &= ~STMMAC_FLAG_EXT_SNAPSHOT_EN;
+		}
+		netdev_dbg(priv->dev, "Auxiliary Snapshot %d %s.\n",
+			   rq->extts.index, on ? "enabled" : "disabled");
+		writel(acr_value, ptpaddr + PTP_ACR);
+		mutex_unlock(&priv->aux_ts_lock);
+		/* wait for auxts fifo clear to finish */
+		ret = readl_poll_timeout(ptpaddr + PTP_ACR, acr_value,
+					 !(acr_value & PTP_ACR_ATSFC),
+					 10, 10000);
+		break;
+	}
+
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+/**
+ * stmmac_get_syncdevicetime
+ * @device: current device time
+ * @system: system counter value read synchronously with device time
+ * @ctx: context provided by timekeeping code
+ * Description: Read device and system clock simultaneously and return the
+ * corrected clock values in ns.
+ **/
+static int stmmac_get_syncdevicetime(ktime_t *device,
+				     struct system_counterval_t *system,
+				     void *ctx)
+{
+	struct stmmac_priv *priv = (struct stmmac_priv *)ctx;
+
+	return priv->plat->crosststamp(device, system, ctx);
+}
+
+static int stmmac_getcrosststamp(struct ptp_clock_info *ptp,
+				 struct system_device_crosststamp *xtstamp)
+{
+	struct stmmac_priv *priv =
+		container_of(ptp, struct stmmac_priv, ptp_clock_ops);
+
+	return get_device_system_crosststamp(stmmac_get_syncdevicetime,
+					     priv, NULL, xtstamp);
+}
+
+/* structure describing a PTP hardware clock */
+const struct ptp_clock_info ec_stmmac_ptp_clock_ops = {
+	.owner = THIS_MODULE,
+	.name = "stmmac ptp",
+	.max_adj = 62500000,
+	.n_alarm = 0,
+	.n_ext_ts = 0, /* will be overwritten in ec_stmmac_ptp_register */
+	.n_per_out = 0, /* will be overwritten in ec_stmmac_ptp_register */
+	.n_pins = 0,
+	.pps = 0,
+	.adjfine = stmmac_adjust_freq,
+	.adjtime = stmmac_adjust_time,
+	.gettime64 = stmmac_get_time,
+	.settime64 = stmmac_set_time,
+	.enable = stmmac_enable,
+};
+
+/* structure describing a PTP hardware clock */
+const struct ptp_clock_info ec_dwmac1000_ptp_clock_ops = {
+	.owner = THIS_MODULE,
+	.name = "stmmac ptp",
+	.max_adj = 62500000,
+	.n_alarm = 0,
+	.n_ext_ts = 1,
+	.n_per_out = 0,
+	.n_pins = 0,
+	.pps = 0,
+	.adjfine = stmmac_adjust_freq,
+	.adjtime = stmmac_adjust_time,
+	.gettime64 = stmmac_get_time,
+	.settime64 = stmmac_set_time,
+	.enable = ec_dwmac1000_ptp_enable,
+};
+
+/**
+ * ec_stmmac_ptp_register
+ * @priv: driver private structure
+ * Description: this function will register the ptp clock driver
+ * to kernel. It also does some house keeping work.
+ */
+void ec_stmmac_ptp_register(struct stmmac_priv *priv)
+{
+	int i;
+
+	for (i = 0; i < priv->dma_cap.pps_out_num; i++) {
+		if (i >= STMMAC_PPS_MAX)
+			break;
+		priv->pps[i].available = true;
+	}
+
+	/* Calculate the clock domain crossing (CDC) error if necessary */
+	priv->plat->cdc_error_adj = 0;
+	if (priv->plat->has_gmac4)
+		priv->plat->cdc_error_adj = (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate;
+
+	/* Update the ptp clock parameters based on feature discovery, when
+	 * available
+	 */
+	if (priv->dma_cap.pps_out_num)
+		priv->ptp_clock_ops.n_per_out = priv->dma_cap.pps_out_num;
+
+	if (priv->dma_cap.aux_snapshot_n)
+		priv->ptp_clock_ops.n_ext_ts = priv->dma_cap.aux_snapshot_n;
+
+	if (priv->plat->ptp_max_adj)
+		priv->ptp_clock_ops.max_adj = priv->plat->ptp_max_adj;
+
+	if (priv->plat->crosststamp)
+		priv->ptp_clock_ops.getcrosststamp = stmmac_getcrosststamp;
+
+	rwlock_init(&priv->ptp_lock);
+	mutex_init(&priv->aux_ts_lock);
+
+	priv->ptp_clock = ptp_clock_register(&priv->ptp_clock_ops,
+					     priv->device);
+	if (IS_ERR(priv->ptp_clock)) {
+		netdev_err(priv->dev, "ptp_clock_register failed\n");
+		priv->ptp_clock = NULL;
+	}
+
+	if (priv->ptp_clock)
+		netdev_info(priv->dev, "registered PTP clock\n");
+	else
+		mutex_destroy(&priv->aux_ts_lock);
+}
+
+/**
+ * ec_stmmac_ptp_unregister
+ * @priv: driver private structure
+ * Description: this function will remove/unregister the ptp clock driver
+ * from the kernel.
+ */
+void ec_stmmac_ptp_unregister(struct stmmac_priv *priv)
+{
+	if (priv->ptp_clock) {
+		ptp_clock_unregister(priv->ptp_clock);
+		priv->ptp_clock = NULL;
+		pr_debug("Removed PTP HW clock successfully on %s\n",
+			 priv->dev->name);
+
+		mutex_destroy(&priv->aux_ts_lock);
+	}
+}
diff --git a/drivers/net/ethercat/device/K3/stmmac_ptp-ethercat.h b/drivers/net/ethercat/device/K3/stmmac_ptp-ethercat.h
new file mode 100644
index 000000000000..9170e2f28075
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_ptp-ethercat.h
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/******************************************************************************
+  PTP Header file
+
+  Copyright (C) 2013  Vayavya Labs Pvt Ltd
+
+
+  Author: Rayagond Kokatanur <rayagond@vayavyalabs.com>
+******************************************************************************/
+
+#ifndef	__STMMAC_PTP_H__
+#define	__STMMAC_PTP_H__
+
+#define PTP_XGMAC_OFFSET	0xd00
+#define	PTP_GMAC4_OFFSET	0xb00
+#define	PTP_GMAC3_X_OFFSET	0x700
+
+/* IEEE 1588 PTP register offsets */
+#define	PTP_TCR		0x00	/* Timestamp Control Reg */
+#define	PTP_SSIR	0x04	/* Sub-Second Increment Reg */
+#define	PTP_STSR	0x08	/* System Time – Seconds Regr */
+#define	PTP_STNSR	0x0c	/* System Time – Nanoseconds Reg */
+#define	PTP_STSUR	0x10	/* System Time – Seconds Update Reg */
+#define	PTP_STNSUR	0x14	/* System Time – Nanoseconds Update Reg */
+#define	PTP_TAR		0x18	/* Timestamp Addend Reg */
+#define	PTP_ACR		0x40	/* Auxiliary Control Reg */
+#define	PTP_ATNR	0x48	/* Auxiliary Timestamp - Nanoseconds Reg */
+#define	PTP_ATSR	0x4c	/* Auxiliary Timestamp - Seconds Reg */
+#define	PTP_TS_INGR_CORR_NS	0x58	/* Ingress timestamp correction nanoseconds */
+#define	PTP_TS_EGR_CORR_NS	0x5C	/* Egress timestamp correction nanoseconds*/
+#define	PTP_TS_INGR_CORR_SNS	0x60	/* Ingress timestamp correction subnanoseconds */
+#define	PTP_TS_EGR_CORR_SNS	0x64	/* Egress timestamp correction subnanoseconds */
+#define	PTP_TS_INGR_LAT	0x68	/* MAC internal Ingress Latency */
+#define	PTP_TS_EGR_LAT	0x6c	/* MAC internal Egress Latency */
+
+#define	PTP_STNSUR_ADDSUB_SHIFT	31
+#define	PTP_DIGITAL_ROLLOVER_MODE	0x3B9ACA00	/* 10e9-1 ns */
+#define	PTP_BINARY_ROLLOVER_MODE	0x80000000	/* ~0.466 ns */
+
+/* PTP Timestamp control register defines */
+#define	PTP_TCR_TSENA		BIT(0)	/* Timestamp Enable */
+#define	PTP_TCR_TSCFUPDT	BIT(1)	/* Timestamp Fine/Coarse Update */
+#define	PTP_TCR_TSINIT		BIT(2)	/* Timestamp Initialize */
+#define	PTP_TCR_TSUPDT		BIT(3)	/* Timestamp Update */
+#define	PTP_TCR_TSTRIG		BIT(4)	/* Timestamp Interrupt Trigger Enable */
+#define	PTP_TCR_TSADDREG	BIT(5)	/* Addend Reg Update */
+#define	PTP_TCR_TSENALL		BIT(8)	/* Enable Timestamp for All Frames */
+#define	PTP_TCR_TSCTRLSSR	BIT(9)	/* Digital or Binary Rollover Control */
+/* Enable PTP packet Processing for Version 2 Format */
+#define	PTP_TCR_TSVER2ENA	BIT(10)
+/* Enable Processing of PTP over Ethernet Frames */
+#define	PTP_TCR_TSIPENA		BIT(11)
+/* Enable Processing of PTP Frames Sent over IPv6-UDP */
+#define	PTP_TCR_TSIPV6ENA	BIT(12)
+/* Enable Processing of PTP Frames Sent over IPv4-UDP */
+#define	PTP_TCR_TSIPV4ENA	BIT(13)
+/* Enable Timestamp Snapshot for Event Messages */
+#define	PTP_TCR_TSEVNTENA	BIT(14)
+/* Enable Snapshot for Messages Relevant to Master */
+#define	PTP_TCR_TSMSTRENA	BIT(15)
+/* Select PTP packets for Taking Snapshots
+ * On gmac4 specifically:
+ * Enable SYNC, Pdelay_Req, Pdelay_Resp when TSEVNTENA is enabled.
+ * or
+ * Enable  SYNC, Follow_Up, Delay_Req, Delay_Resp, Pdelay_Req, Pdelay_Resp,
+ * Pdelay_Resp_Follow_Up if TSEVNTENA is disabled
+ */
+#define	PTP_TCR_SNAPTYPSEL_1	BIT(16)
+/* Enable MAC address for PTP Frame Filtering */
+#define	PTP_TCR_TSENMACADDR	BIT(18)
+
+/* SSIR defines */
+#define	PTP_SSIR_SSINC_MAX		0xff
+#define	GMAC4_PTP_SSIR_SSINC_SHIFT	16
+
+/* Auxiliary Control defines */
+#define	PTP_ACR_ATSFC		BIT(0)	/* Auxiliary Snapshot FIFO Clear */
+#define	PTP_ACR_ATSEN0		BIT(4)	/* Auxiliary Snapshot 0 Enable */
+#define	PTP_ACR_ATSEN1		BIT(5)	/* Auxiliary Snapshot 1 Enable */
+#define	PTP_ACR_ATSEN2		BIT(6)	/* Auxiliary Snapshot 2 Enable */
+#define	PTP_ACR_ATSEN3		BIT(7)	/* Auxiliary Snapshot 3 Enable */
+#define	PTP_ACR_ATSEN(index)	(PTP_ACR_ATSEN0 << (index))
+#define	PTP_ACR_MASK		GENMASK(7, 4)	/* Aux Snapshot Mask */
+#define	PMC_ART_VALUE0		0x01	/* PMC_ART[15:0] timer value */
+#define	PMC_ART_VALUE1		0x02	/* PMC_ART[31:16] timer value */
+#define	PMC_ART_VALUE2		0x03	/* PMC_ART[47:32] timer value */
+#define	PMC_ART_VALUE3		0x04	/* PMC_ART[63:48] timer value */
+#define	GMAC4_ART_TIME_SHIFT	16	/* ART TIME 16-bits shift */
+
+enum aux_snapshot {
+	AUX_SNAPSHOT0 = 0x10,
+	AUX_SNAPSHOT1 = 0x20,
+	AUX_SNAPSHOT2 = 0x40,
+	AUX_SNAPSHOT3 = 0x80,
+};
+
+struct ptp_clock_info;
+struct ptp_clock_request;
+struct stmmac_priv;
+
+int ec_dwmac1000_ptp_enable(struct ptp_clock_info *ptp,
+			 struct ptp_clock_request *rq, int on);
+
+void ec_dwmac1000_get_ptptime(void __iomem *ptpaddr, u64 *ptp_time);
+void ec_dwmac1000_timestamp_interrupt(struct stmmac_priv *priv);
+
+extern const struct ptp_clock_info ec_stmmac_ptp_clock_ops;
+extern const struct ptp_clock_info ec_dwmac1000_ptp_clock_ops;
+
+#endif	/* __STMMAC_PTP_H__ */
diff --git a/drivers/net/ethercat/device/K3/stmmac_tc-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_tc-ethercat.c
new file mode 100644
index 000000000000..850593996920
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_tc-ethercat.c
@@ -0,0 +1,1289 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
+ * stmmac TC Handling (HW only)
+ */
+
+#include <net/pkt_cls.h>
+#include <net/tc_act/tc_gact.h>
+#include "common-ethercat.h"
+#include "dwmac4-ethercat.h"
+#include "dwmac5-ethercat.h"
+#include "stmmac-ethercat.h"
+
+static void tc_fill_all_pass_entry(struct stmmac_tc_entry *entry)
+{
+	memset(entry, 0, sizeof(*entry));
+	entry->in_use = true;
+	entry->is_last = true;
+	entry->is_frag = false;
+	entry->prio = ~0x0;
+	entry->handle = 0;
+	entry->val.match_data = 0x0;
+	entry->val.match_en = 0x0;
+	entry->val.af = 1;
+	entry->val.dma_ch_no = 0x0;
+}
+
+static struct stmmac_tc_entry *tc_find_entry(struct stmmac_priv *priv,
+					     struct tc_cls_u32_offload *cls,
+					     bool free)
+{
+	struct stmmac_tc_entry *entry, *first = NULL, *dup = NULL;
+	u32 loc = cls->knode.handle;
+	int i;
+
+	for (i = 0; i < priv->tc_entries_max; i++) {
+		entry = &priv->tc_entries[i];
+		if (!entry->in_use && !first && free)
+			first = entry;
+		if ((entry->handle == loc) && !free && !entry->is_frag)
+			dup = entry;
+	}
+
+	if (dup)
+		return dup;
+	if (first) {
+		first->handle = loc;
+		first->in_use = true;
+
+		/* Reset HW values */
+		memset(&first->val, 0, sizeof(first->val));
+	}
+
+	return first;
+}
+
+static int tc_fill_actions(struct stmmac_tc_entry *entry,
+			   struct stmmac_tc_entry *frag,
+			   struct tc_cls_u32_offload *cls)
+{
+	struct stmmac_tc_entry *action_entry = entry;
+	const struct tc_action *act;
+	struct tcf_exts *exts;
+	int i;
+
+	exts = cls->knode.exts;
+	if (!tcf_exts_has_actions(exts))
+		return -EINVAL;
+	if (frag)
+		action_entry = frag;
+
+	tcf_exts_for_each_action(i, act, exts) {
+		/* Accept */
+		if (is_tcf_gact_ok(act)) {
+			action_entry->val.af = 1;
+			break;
+		}
+		/* Drop */
+		if (is_tcf_gact_shot(act)) {
+			action_entry->val.rf = 1;
+			break;
+		}
+
+		/* Unsupported */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int tc_fill_entry(struct stmmac_priv *priv,
+			 struct tc_cls_u32_offload *cls)
+{
+	struct stmmac_tc_entry *entry, *frag = NULL;
+	struct tc_u32_sel *sel = cls->knode.sel;
+	u32 off, data, mask, real_off, rem;
+	u32 prio = cls->common.prio << 16;
+	int ret;
+
+	/* Only 1 match per entry */
+	if (sel->nkeys <= 0 || sel->nkeys > 1)
+		return -EINVAL;
+
+	off = sel->keys[0].off << sel->offshift;
+	data = sel->keys[0].val;
+	mask = sel->keys[0].mask;
+
+	switch (ntohs(cls->common.protocol)) {
+	case ETH_P_ALL:
+		break;
+	case ETH_P_IP:
+		off += ETH_HLEN;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (off > priv->tc_off_max)
+		return -EINVAL;
+
+	real_off = off / 4;
+	rem = off % 4;
+
+	entry = tc_find_entry(priv, cls, true);
+	if (!entry)
+		return -EINVAL;
+
+	if (rem) {
+		frag = tc_find_entry(priv, cls, true);
+		if (!frag) {
+			ret = -EINVAL;
+			goto err_unuse;
+		}
+
+		entry->frag_ptr = frag;
+		entry->val.match_en = (mask << (rem * 8)) &
+			GENMASK(31, rem * 8);
+		entry->val.match_data = (data << (rem * 8)) &
+			GENMASK(31, rem * 8);
+		entry->val.frame_offset = real_off;
+		entry->prio = prio;
+
+		frag->val.match_en = (mask >> (rem * 8)) &
+			GENMASK(rem * 8 - 1, 0);
+		frag->val.match_data = (data >> (rem * 8)) &
+			GENMASK(rem * 8 - 1, 0);
+		frag->val.frame_offset = real_off + 1;
+		frag->prio = prio;
+		frag->is_frag = true;
+	} else {
+		entry->frag_ptr = NULL;
+		entry->val.match_en = mask;
+		entry->val.match_data = data;
+		entry->val.frame_offset = real_off;
+		entry->prio = prio;
+	}
+
+	ret = tc_fill_actions(entry, frag, cls);
+	if (ret)
+		goto err_unuse;
+
+	return 0;
+
+err_unuse:
+	if (frag)
+		frag->in_use = false;
+	entry->in_use = false;
+	return ret;
+}
+
+static void tc_unfill_entry(struct stmmac_priv *priv,
+			    struct tc_cls_u32_offload *cls)
+{
+	struct stmmac_tc_entry *entry;
+
+	entry = tc_find_entry(priv, cls, false);
+	if (!entry)
+		return;
+
+	entry->in_use = false;
+	if (entry->frag_ptr) {
+		entry = entry->frag_ptr;
+		entry->is_frag = false;
+		entry->in_use = false;
+	}
+}
+
+static int tc_config_knode(struct stmmac_priv *priv,
+			   struct tc_cls_u32_offload *cls)
+{
+	int ret;
+
+	ret = tc_fill_entry(priv, cls);
+	if (ret)
+		return ret;
+
+	ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries,
+			priv->tc_entries_max);
+	if (ret)
+		goto err_unfill;
+
+	return 0;
+
+err_unfill:
+	tc_unfill_entry(priv, cls);
+	return ret;
+}
+
+static int tc_delete_knode(struct stmmac_priv *priv,
+			   struct tc_cls_u32_offload *cls)
+{
+	/* Set entry and fragments as not used */
+	tc_unfill_entry(priv, cls);
+
+	return stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries,
+				 priv->tc_entries_max);
+}
+
+static int tc_setup_cls_u32(struct stmmac_priv *priv,
+			    struct tc_cls_u32_offload *cls)
+{
+	switch (cls->command) {
+	case TC_CLSU32_REPLACE_KNODE:
+		tc_unfill_entry(priv, cls);
+		fallthrough;
+	case TC_CLSU32_NEW_KNODE:
+		return tc_config_knode(priv, cls);
+	case TC_CLSU32_DELETE_KNODE:
+		return tc_delete_knode(priv, cls);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int tc_rfs_init(struct stmmac_priv *priv)
+{
+	int i;
+
+	priv->rfs_entries_max[STMMAC_RFS_T_VLAN] = 8;
+	priv->rfs_entries_max[STMMAC_RFS_T_LLDP] = 1;
+	priv->rfs_entries_max[STMMAC_RFS_T_1588] = 1;
+
+	for (i = 0; i < STMMAC_RFS_T_MAX; i++)
+		priv->rfs_entries_total += priv->rfs_entries_max[i];
+
+	priv->rfs_entries = devm_kcalloc(priv->device,
+					 priv->rfs_entries_total,
+					 sizeof(*priv->rfs_entries),
+					 GFP_KERNEL);
+	if (!priv->rfs_entries)
+		return -ENOMEM;
+
+	dev_info(priv->device, "Enabled RFS Flow TC (entries=%d)\n",
+		 priv->rfs_entries_total);
+
+	return 0;
+}
+
+static int tc_init(struct stmmac_priv *priv)
+{
+	struct dma_features *dma_cap = &priv->dma_cap;
+	unsigned int count;
+	int ret, i;
+
+	if (dma_cap->l3l4fnum) {
+		priv->flow_entries_max = dma_cap->l3l4fnum;
+		priv->flow_entries = devm_kcalloc(priv->device,
+						  dma_cap->l3l4fnum,
+						  sizeof(*priv->flow_entries),
+						  GFP_KERNEL);
+		if (!priv->flow_entries)
+			return -ENOMEM;
+
+		for (i = 0; i < priv->flow_entries_max; i++)
+			priv->flow_entries[i].idx = i;
+
+		dev_info(priv->device, "Enabled L3L4 Flow TC (entries=%d)\n",
+			 priv->flow_entries_max);
+	}
+
+	ret = tc_rfs_init(priv);
+	if (ret)
+		return -ENOMEM;
+
+	/* Fail silently as we can still use remaining features, e.g. CBS */
+	if (!dma_cap->frpsel)
+		return 0;
+
+	switch (dma_cap->frpbs) {
+	case 0x0:
+		priv->tc_off_max = 64;
+		break;
+	case 0x1:
+		priv->tc_off_max = 128;
+		break;
+	case 0x2:
+		priv->tc_off_max = 256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (dma_cap->frpes) {
+	case 0x0:
+		count = 64;
+		break;
+	case 0x1:
+		count = 128;
+		break;
+	case 0x2:
+		count = 256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Reserve one last filter which lets all pass */
+	priv->tc_entries_max = count;
+	priv->tc_entries = devm_kcalloc(priv->device,
+			count, sizeof(*priv->tc_entries), GFP_KERNEL);
+	if (!priv->tc_entries)
+		return -ENOMEM;
+
+	tc_fill_all_pass_entry(&priv->tc_entries[count - 1]);
+
+	dev_info(priv->device, "Enabling HW TC (entries=%d, max_off=%d)\n",
+			priv->tc_entries_max, priv->tc_off_max);
+
+	return 0;
+}
+
+static int tc_setup_cbs(struct stmmac_priv *priv,
+			struct tc_cbs_qopt_offload *qopt)
+{
+	u32 tx_queues_count = priv->plat->tx_queues_to_use;
+	s64 port_transmit_rate_kbps;
+	u32 queue = qopt->queue;
+	u32 mode_to_use;
+	u64 value;
+	u32 ptr;
+	int ret;
+
+	/* Queue 0 is not AVB capable */
+	if (queue <= 0 || queue >= tx_queues_count)
+		return -EINVAL;
+	if (!priv->dma_cap.av)
+		return -EOPNOTSUPP;
+
+	port_transmit_rate_kbps = qopt->idleslope - qopt->sendslope;
+
+	if (qopt->enable) {
+		/* Port Transmit Rate and Speed Divider */
+		switch (div_s64(port_transmit_rate_kbps, 1000)) {
+		case SPEED_10000:
+		case SPEED_5000:
+			ptr = 32;
+			break;
+		case SPEED_2500:
+		case SPEED_1000:
+			ptr = 8;
+			break;
+		case SPEED_100:
+			ptr = 4;
+			break;
+		default:
+			netdev_err(priv->dev,
+				   "Invalid portTransmitRate %lld (idleSlope - sendSlope)\n",
+				   port_transmit_rate_kbps);
+			return -EINVAL;
+		}
+	} else {
+		ptr = 0;
+	}
+
+	mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
+	if (mode_to_use == MTL_QUEUE_DCB && qopt->enable) {
+		ret = stmmac_dma_qmode(priv, priv->ioaddr, queue, MTL_QUEUE_AVB);
+		if (ret)
+			return ret;
+
+		priv->plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_AVB;
+	} else if (!qopt->enable) {
+		ret = stmmac_dma_qmode(priv, priv->ioaddr, queue,
+				       MTL_QUEUE_DCB);
+		if (ret)
+			return ret;
+
+		priv->plat->tx_queues_cfg[queue].mode_to_use = MTL_QUEUE_DCB;
+		return 0;
+	}
+
+	/* Final adjustments for HW */
+	value = div_s64(qopt->idleslope * 1024ll * ptr, port_transmit_rate_kbps);
+	priv->plat->tx_queues_cfg[queue].idle_slope = value & GENMASK(31, 0);
+
+	value = div_s64(-qopt->sendslope * 1024ll * ptr, port_transmit_rate_kbps);
+	priv->plat->tx_queues_cfg[queue].send_slope = value & GENMASK(31, 0);
+
+	value = qopt->hicredit * 1024ll * 8;
+	priv->plat->tx_queues_cfg[queue].high_credit = value & GENMASK(31, 0);
+
+	value = qopt->locredit * 1024ll * 8;
+	priv->plat->tx_queues_cfg[queue].low_credit = value & GENMASK(31, 0);
+
+	ret = stmmac_config_cbs(priv, priv->hw,
+				priv->plat->tx_queues_cfg[queue].send_slope,
+				priv->plat->tx_queues_cfg[queue].idle_slope,
+				priv->plat->tx_queues_cfg[queue].high_credit,
+				priv->plat->tx_queues_cfg[queue].low_credit,
+				queue);
+	if (ret)
+		return ret;
+
+	dev_info(priv->device, "CBS queue %d: send %d, idle %d, hi %d, lo %d\n",
+			queue, qopt->sendslope, qopt->idleslope,
+			qopt->hicredit, qopt->locredit);
+	return 0;
+}
+
+static int tc_parse_flow_actions(struct stmmac_priv *priv,
+				 struct flow_action *action,
+				 struct stmmac_flow_entry *entry,
+				 struct netlink_ext_ack *extack)
+{
+	struct flow_action_entry *act;
+	int i;
+
+	if (!flow_action_has_entries(action))
+		return -EINVAL;
+
+	if (!flow_action_basic_hw_stats_check(action, extack))
+		return -EOPNOTSUPP;
+
+	flow_action_for_each(i, act, action) {
+		switch (act->id) {
+		case FLOW_ACTION_DROP:
+			entry->action |= STMMAC_FLOW_ACTION_DROP;
+			return 0;
+		default:
+			break;
+		}
+	}
+
+	/* Nothing to do, maybe inverse filter ? */
+	return 0;
+}
+
+#define ETHER_TYPE_FULL_MASK	cpu_to_be16(~0)
+
+static int tc_add_basic_flow(struct stmmac_priv *priv,
+			     struct flow_cls_offload *cls,
+			     struct stmmac_flow_entry *entry)
+{
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct flow_dissector *dissector = rule->match.dissector;
+	struct flow_match_basic match;
+
+	/* Nothing to do here */
+	if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC))
+		return -EINVAL;
+
+	flow_rule_match_basic(rule, &match);
+
+	entry->ip_proto = match.key->ip_proto;
+	return 0;
+}
+
+static int tc_add_ip4_flow(struct stmmac_priv *priv,
+			   struct flow_cls_offload *cls,
+			   struct stmmac_flow_entry *entry)
+{
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct flow_dissector *dissector = rule->match.dissector;
+	bool inv = entry->action & STMMAC_FLOW_ACTION_DROP;
+	struct flow_match_ipv4_addrs match;
+	u32 hw_match;
+	int ret;
+
+	/* Nothing to do here */
+	if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS))
+		return -EINVAL;
+
+	flow_rule_match_ipv4_addrs(rule, &match);
+	hw_match = ntohl(match.key->src) & ntohl(match.mask->src);
+	if (hw_match) {
+		ret = stmmac_config_l3_filter(priv, priv->hw, entry->idx, true,
+					      false, true, inv, hw_match);
+		if (ret)
+			return ret;
+	}
+
+	hw_match = ntohl(match.key->dst) & ntohl(match.mask->dst);
+	if (hw_match) {
+		ret = stmmac_config_l3_filter(priv, priv->hw, entry->idx, true,
+					      false, false, inv, hw_match);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int tc_add_ports_flow(struct stmmac_priv *priv,
+			     struct flow_cls_offload *cls,
+			     struct stmmac_flow_entry *entry)
+{
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct flow_dissector *dissector = rule->match.dissector;
+	bool inv = entry->action & STMMAC_FLOW_ACTION_DROP;
+	struct flow_match_ports match;
+	u32 hw_match;
+	bool is_udp;
+	int ret;
+
+	/* Nothing to do here */
+	if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS))
+		return -EINVAL;
+
+	switch (entry->ip_proto) {
+	case IPPROTO_TCP:
+		is_udp = false;
+		break;
+	case IPPROTO_UDP:
+		is_udp = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	flow_rule_match_ports(rule, &match);
+
+	hw_match = ntohs(match.key->src) & ntohs(match.mask->src);
+	if (hw_match) {
+		ret = stmmac_config_l4_filter(priv, priv->hw, entry->idx, true,
+					      is_udp, true, inv, hw_match);
+		if (ret)
+			return ret;
+	}
+
+	hw_match = ntohs(match.key->dst) & ntohs(match.mask->dst);
+	if (hw_match) {
+		ret = stmmac_config_l4_filter(priv, priv->hw, entry->idx, true,
+					      is_udp, false, inv, hw_match);
+		if (ret)
+			return ret;
+	}
+
+	entry->is_l4 = true;
+	return 0;
+}
+
+static struct stmmac_flow_entry *tc_find_flow(struct stmmac_priv *priv,
+					      struct flow_cls_offload *cls,
+					      bool get_free)
+{
+	int i;
+
+	for (i = 0; i < priv->flow_entries_max; i++) {
+		struct stmmac_flow_entry *entry = &priv->flow_entries[i];
+
+		if (entry->cookie == cls->cookie)
+			return entry;
+		if (get_free && (entry->in_use == false))
+			return entry;
+	}
+
+	return NULL;
+}
+
+static struct {
+	int (*fn)(struct stmmac_priv *priv, struct flow_cls_offload *cls,
+		  struct stmmac_flow_entry *entry);
+} tc_flow_parsers[] = {
+	{ .fn = tc_add_basic_flow },
+	{ .fn = tc_add_ip4_flow },
+	{ .fn = tc_add_ports_flow },
+};
+
+static int tc_add_flow(struct stmmac_priv *priv,
+		       struct flow_cls_offload *cls)
+{
+	struct stmmac_flow_entry *entry = tc_find_flow(priv, cls, false);
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	int i, ret;
+
+	if (!entry) {
+		entry = tc_find_flow(priv, cls, true);
+		if (!entry)
+			return -ENOENT;
+	}
+
+	ret = tc_parse_flow_actions(priv, &rule->action, entry,
+				    cls->common.extack);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < ARRAY_SIZE(tc_flow_parsers); i++) {
+		ret = tc_flow_parsers[i].fn(priv, cls, entry);
+		if (!ret)
+			entry->in_use = true;
+	}
+
+	if (!entry->in_use)
+		return -EINVAL;
+
+	entry->cookie = cls->cookie;
+	return 0;
+}
+
+static int tc_del_flow(struct stmmac_priv *priv,
+		       struct flow_cls_offload *cls)
+{
+	struct stmmac_flow_entry *entry = tc_find_flow(priv, cls, false);
+	int ret;
+
+	if (!entry || !entry->in_use)
+		return -ENOENT;
+
+	if (entry->is_l4) {
+		ret = stmmac_config_l4_filter(priv, priv->hw, entry->idx, false,
+					      false, false, false, 0);
+	} else {
+		ret = stmmac_config_l3_filter(priv, priv->hw, entry->idx, false,
+					      false, false, false, 0);
+	}
+
+	entry->in_use = false;
+	entry->cookie = 0;
+	entry->is_l4 = false;
+	return ret;
+}
+
+static struct stmmac_rfs_entry *tc_find_rfs(struct stmmac_priv *priv,
+					    struct flow_cls_offload *cls,
+					    bool get_free)
+{
+	int i;
+
+	for (i = 0; i < priv->rfs_entries_total; i++) {
+		struct stmmac_rfs_entry *entry = &priv->rfs_entries[i];
+
+		if (entry->cookie == cls->cookie)
+			return entry;
+		if (get_free && entry->in_use == false)
+			return entry;
+	}
+
+	return NULL;
+}
+
+#define VLAN_PRIO_FULL_MASK (0x07)
+
+static int tc_add_vlan_flow(struct stmmac_priv *priv,
+			    struct flow_cls_offload *cls)
+{
+	struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false);
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct flow_dissector *dissector = rule->match.dissector;
+	int tc = tc_classid_to_hwtc(priv->dev, cls->classid);
+	struct flow_match_vlan match;
+
+	if (!entry) {
+		entry = tc_find_rfs(priv, cls, true);
+		if (!entry)
+			return -ENOENT;
+	}
+
+	if (priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN] >=
+	    priv->rfs_entries_max[STMMAC_RFS_T_VLAN])
+		return -ENOENT;
+
+	/* Nothing to do here */
+	if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN))
+		return -EINVAL;
+
+	if (tc < 0) {
+		netdev_err(priv->dev, "Invalid traffic class\n");
+		return -EINVAL;
+	}
+
+	flow_rule_match_vlan(rule, &match);
+
+	if (match.mask->vlan_priority) {
+		u32 prio;
+
+		if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
+			netdev_err(priv->dev, "Only full mask is supported for VLAN priority");
+			return -EINVAL;
+		}
+
+		prio = BIT(match.key->vlan_priority);
+		stmmac_rx_queue_prio(priv, priv->hw, prio, tc);
+
+		entry->in_use = true;
+		entry->cookie = cls->cookie;
+		entry->tc = tc;
+		entry->type = STMMAC_RFS_T_VLAN;
+		priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN]++;
+	}
+
+	return 0;
+}
+
+static int tc_del_vlan_flow(struct stmmac_priv *priv,
+			    struct flow_cls_offload *cls)
+{
+	struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false);
+
+	if (!entry || !entry->in_use || entry->type != STMMAC_RFS_T_VLAN)
+		return -ENOENT;
+
+	stmmac_rx_queue_prio(priv, priv->hw, 0, entry->tc);
+
+	entry->in_use = false;
+	entry->cookie = 0;
+	entry->tc = 0;
+	entry->type = 0;
+
+	priv->rfs_entries_cnt[STMMAC_RFS_T_VLAN]--;
+
+	return 0;
+}
+
+static int tc_add_ethtype_flow(struct stmmac_priv *priv,
+			       struct flow_cls_offload *cls)
+{
+	struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false);
+	struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct flow_dissector *dissector = rule->match.dissector;
+	int tc = tc_classid_to_hwtc(priv->dev, cls->classid);
+	struct flow_match_basic match;
+
+	if (!entry) {
+		entry = tc_find_rfs(priv, cls, true);
+		if (!entry)
+			return -ENOENT;
+	}
+
+	/* Nothing to do here */
+	if (!dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC))
+		return -EINVAL;
+
+	if (tc < 0) {
+		netdev_err(priv->dev, "Invalid traffic class\n");
+		return -EINVAL;
+	}
+
+	flow_rule_match_basic(rule, &match);
+
+	if (match.mask->n_proto) {
+		u16 etype = ntohs(match.key->n_proto);
+
+		if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) {
+			netdev_err(priv->dev, "Only full mask is supported for EthType filter");
+			return -EINVAL;
+		}
+		switch (etype) {
+		case ETH_P_LLDP:
+			if (priv->rfs_entries_cnt[STMMAC_RFS_T_LLDP] >=
+			    priv->rfs_entries_max[STMMAC_RFS_T_LLDP])
+				return -ENOENT;
+
+			entry->type = STMMAC_RFS_T_LLDP;
+			priv->rfs_entries_cnt[STMMAC_RFS_T_LLDP]++;
+
+			stmmac_rx_queue_routing(priv, priv->hw,
+						PACKET_DCBCPQ, tc);
+			break;
+		case ETH_P_1588:
+			if (priv->rfs_entries_cnt[STMMAC_RFS_T_1588] >=
+			    priv->rfs_entries_max[STMMAC_RFS_T_1588])
+				return -ENOENT;
+
+			entry->type = STMMAC_RFS_T_1588;
+			priv->rfs_entries_cnt[STMMAC_RFS_T_1588]++;
+
+			stmmac_rx_queue_routing(priv, priv->hw,
+						PACKET_PTPQ, tc);
+			break;
+		default:
+			netdev_err(priv->dev, "EthType(0x%x) is not supported", etype);
+			return -EINVAL;
+		}
+
+		entry->in_use = true;
+		entry->cookie = cls->cookie;
+		entry->tc = tc;
+		entry->etype = etype;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int tc_del_ethtype_flow(struct stmmac_priv *priv,
+			       struct flow_cls_offload *cls)
+{
+	struct stmmac_rfs_entry *entry = tc_find_rfs(priv, cls, false);
+
+	if (!entry || !entry->in_use ||
+	    entry->type < STMMAC_RFS_T_LLDP ||
+	    entry->type > STMMAC_RFS_T_1588)
+		return -ENOENT;
+
+	switch (entry->etype) {
+	case ETH_P_LLDP:
+		stmmac_rx_queue_routing(priv, priv->hw,
+					PACKET_DCBCPQ, 0);
+		priv->rfs_entries_cnt[STMMAC_RFS_T_LLDP]--;
+		break;
+	case ETH_P_1588:
+		stmmac_rx_queue_routing(priv, priv->hw,
+					PACKET_PTPQ, 0);
+		priv->rfs_entries_cnt[STMMAC_RFS_T_1588]--;
+		break;
+	default:
+		netdev_err(priv->dev, "EthType(0x%x) is not supported",
+			   entry->etype);
+		return -EINVAL;
+	}
+
+	entry->in_use = false;
+	entry->cookie = 0;
+	entry->tc = 0;
+	entry->etype = 0;
+	entry->type = 0;
+
+	return 0;
+}
+
+static int tc_add_flow_cls(struct stmmac_priv *priv,
+			   struct flow_cls_offload *cls)
+{
+	int ret;
+
+	ret = tc_add_flow(priv, cls);
+	if (!ret)
+		return ret;
+
+	ret = tc_add_ethtype_flow(priv, cls);
+	if (!ret)
+		return ret;
+
+	return tc_add_vlan_flow(priv, cls);
+}
+
+static int tc_del_flow_cls(struct stmmac_priv *priv,
+			   struct flow_cls_offload *cls)
+{
+	int ret;
+
+	ret = tc_del_flow(priv, cls);
+	if (!ret)
+		return ret;
+
+	ret = tc_del_ethtype_flow(priv, cls);
+	if (!ret)
+		return ret;
+
+	return tc_del_vlan_flow(priv, cls);
+}
+
+static int tc_setup_cls(struct stmmac_priv *priv,
+			struct flow_cls_offload *cls)
+{
+	int ret = 0;
+
+	/* When RSS is enabled, the filtering will be bypassed */
+	if (priv->rss.enable)
+		return -EBUSY;
+
+	switch (cls->command) {
+	case FLOW_CLS_REPLACE:
+		ret = tc_add_flow_cls(priv, cls);
+		break;
+	case FLOW_CLS_DESTROY:
+		ret = tc_del_flow_cls(priv, cls);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
+struct timespec64 ec_stmmac_calc_tas_basetime(ktime_t old_base_time,
+					   ktime_t current_time,
+					   u64 cycle_time)
+{
+	struct timespec64 time;
+
+	if (ktime_after(old_base_time, current_time)) {
+		time = ktime_to_timespec64(old_base_time);
+	} else {
+		s64 n;
+		ktime_t base_time;
+
+		n = div64_s64(ktime_sub_ns(current_time, old_base_time),
+			      cycle_time);
+		base_time = ktime_add_ns(old_base_time,
+					 (n + 1) * cycle_time);
+
+		time = ktime_to_timespec64(base_time);
+	}
+
+	return time;
+}
+
+static void tc_taprio_map_maxsdu_txq(struct stmmac_priv *priv,
+				     struct tc_taprio_qopt_offload *qopt)
+{
+	u32 num_tc = qopt->mqprio.qopt.num_tc;
+	u32 offset, count, i, j;
+
+	/* QueueMaxSDU received from the driver corresponds to the Linux traffic
+	 * class. Map queueMaxSDU per Linux traffic class to DWMAC Tx queues.
+	 */
+	for (i = 0; i < num_tc; i++) {
+		if (!qopt->max_sdu[i])
+			continue;
+
+		offset = qopt->mqprio.qopt.offset[i];
+		count = qopt->mqprio.qopt.count[i];
+
+		for (j = offset; j < offset + count; j++)
+			priv->est->max_sdu[j] = qopt->max_sdu[i] + ETH_HLEN - ETH_TLEN;
+	}
+}
+
+static int tc_taprio_configure(struct stmmac_priv *priv,
+			       struct tc_taprio_qopt_offload *qopt)
+{
+	u32 size, wid = priv->dma_cap.estwid, dep = priv->dma_cap.estdep;
+	struct netlink_ext_ack *extack = qopt->mqprio.extack;
+	struct timespec64 time, current_time, qopt_time;
+	ktime_t current_time_ns;
+	int i, ret = 0;
+	u64 ctr;
+
+	if (qopt->base_time < 0)
+		return -ERANGE;
+
+	if (!priv->dma_cap.estsel)
+		return -EOPNOTSUPP;
+
+	switch (wid) {
+	case 0x1:
+		wid = 16;
+		break;
+	case 0x2:
+		wid = 20;
+		break;
+	case 0x3:
+		wid = 24;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	switch (dep) {
+	case 0x1:
+		dep = 64;
+		break;
+	case 0x2:
+		dep = 128;
+		break;
+	case 0x3:
+		dep = 256;
+		break;
+	case 0x4:
+		dep = 512;
+		break;
+	case 0x5:
+		dep = 1024;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	if (qopt->cmd == TAPRIO_CMD_DESTROY)
+		goto disable;
+
+	if (qopt->num_entries > dep)
+		return -EINVAL;
+	if (!qopt->cycle_time)
+		return -ERANGE;
+	if (qopt->cycle_time_extension >= BIT(wid + 7))
+		return -ERANGE;
+
+	if (!priv->est) {
+		priv->est = devm_kzalloc(priv->device, sizeof(*priv->est),
+					 GFP_KERNEL);
+		if (!priv->est)
+			return -ENOMEM;
+
+		mutex_init(&priv->est_lock);
+	} else {
+		mutex_lock(&priv->est_lock);
+		memset(priv->est, 0, sizeof(*priv->est));
+		mutex_unlock(&priv->est_lock);
+	}
+
+	size = qopt->num_entries;
+
+	mutex_lock(&priv->est_lock);
+	priv->est->gcl_size = size;
+	priv->est->enable = qopt->cmd == TAPRIO_CMD_REPLACE;
+	mutex_unlock(&priv->est_lock);
+
+	for (i = 0; i < size; i++) {
+		s64 delta_ns = qopt->entries[i].interval;
+		u32 gates = qopt->entries[i].gate_mask;
+
+		if (delta_ns > GENMASK(wid - 1, 0))
+			return -ERANGE;
+		if (gates > GENMASK(31 - wid, 0))
+			return -ERANGE;
+
+		switch (qopt->entries[i].command) {
+		case TC_TAPRIO_CMD_SET_GATES:
+			break;
+		case TC_TAPRIO_CMD_SET_AND_HOLD:
+			gates |= BIT(0);
+			break;
+		case TC_TAPRIO_CMD_SET_AND_RELEASE:
+			gates &= ~BIT(0);
+			break;
+		default:
+			return -EOPNOTSUPP;
+		}
+
+		priv->est->gcl[i] = delta_ns | (gates << wid);
+	}
+
+	mutex_lock(&priv->est_lock);
+	/* Adjust for real system time */
+	priv->ptp_clock_ops.gettime64(&priv->ptp_clock_ops, &current_time);
+	current_time_ns = timespec64_to_ktime(current_time);
+	time = ec_stmmac_calc_tas_basetime(qopt->base_time, current_time_ns,
+					qopt->cycle_time);
+
+	priv->est->btr[0] = (u32)time.tv_nsec;
+	priv->est->btr[1] = (u32)time.tv_sec;
+
+	qopt_time = ktime_to_timespec64(qopt->base_time);
+	priv->est->btr_reserve[0] = (u32)qopt_time.tv_nsec;
+	priv->est->btr_reserve[1] = (u32)qopt_time.tv_sec;
+
+	ctr = qopt->cycle_time;
+	priv->est->ctr[0] = do_div(ctr, NSEC_PER_SEC);
+	priv->est->ctr[1] = (u32)ctr;
+
+	priv->est->ter = qopt->cycle_time_extension;
+
+	tc_taprio_map_maxsdu_txq(priv, qopt);
+
+	ret = stmmac_est_configure(priv, priv, priv->est,
+				   priv->plat->clk_ptp_rate);
+	mutex_unlock(&priv->est_lock);
+	if (ret) {
+		netdev_err(priv->dev, "failed to configure EST\n");
+		goto disable;
+	}
+
+	ret = stmmac_fpe_map_preemption_class(priv, priv->dev, extack,
+					      qopt->mqprio.preemptible_tcs);
+	if (ret)
+		goto disable;
+
+	return 0;
+
+disable:
+	if (priv->est) {
+		mutex_lock(&priv->est_lock);
+		priv->est->enable = false;
+		stmmac_est_configure(priv, priv, priv->est,
+				     priv->plat->clk_ptp_rate);
+		/* Reset taprio status */
+		for (i = 0; i < priv->plat->tx_queues_to_use; i++) {
+			priv->xstats.max_sdu_txq_drop[i] = 0;
+			priv->xstats.mtl_est_txq_hlbf[i] = 0;
+			priv->xstats.mtl_est_txq_hlbs[i] = 0;
+		}
+		mutex_unlock(&priv->est_lock);
+	}
+
+	stmmac_fpe_map_preemption_class(priv, priv->dev, extack, 0);
+
+	return ret;
+}
+
+static void tc_taprio_stats(struct stmmac_priv *priv,
+			    struct tc_taprio_qopt_offload *qopt)
+{
+	u64 window_drops = 0;
+	int i = 0;
+
+	for (i = 0; i < priv->plat->tx_queues_to_use; i++)
+		window_drops += priv->xstats.max_sdu_txq_drop[i] +
+				priv->xstats.mtl_est_txq_hlbf[i] +
+				priv->xstats.mtl_est_txq_hlbs[i];
+	qopt->stats.window_drops = window_drops;
+
+	/* Transmission overrun doesn't happen for stmmac, hence always 0 */
+	qopt->stats.tx_overruns = 0;
+}
+
+static void tc_taprio_queue_stats(struct stmmac_priv *priv,
+				  struct tc_taprio_qopt_offload *qopt)
+{
+	struct tc_taprio_qopt_queue_stats *q_stats = &qopt->queue_stats;
+	int queue = qopt->queue_stats.queue;
+
+	q_stats->stats.window_drops = priv->xstats.max_sdu_txq_drop[queue] +
+				      priv->xstats.mtl_est_txq_hlbf[queue] +
+				      priv->xstats.mtl_est_txq_hlbs[queue];
+
+	/* Transmission overrun doesn't happen for stmmac, hence always 0 */
+	q_stats->stats.tx_overruns = 0;
+}
+
+static int tc_setup_taprio(struct stmmac_priv *priv,
+			   struct tc_taprio_qopt_offload *qopt)
+{
+	int err = 0;
+
+	switch (qopt->cmd) {
+	case TAPRIO_CMD_REPLACE:
+	case TAPRIO_CMD_DESTROY:
+		err = tc_taprio_configure(priv, qopt);
+		break;
+	case TAPRIO_CMD_STATS:
+		tc_taprio_stats(priv, qopt);
+		break;
+	case TAPRIO_CMD_QUEUE_STATS:
+		tc_taprio_queue_stats(priv, qopt);
+		break;
+	default:
+		err = -EOPNOTSUPP;
+	}
+
+	return err;
+}
+
+static int tc_setup_taprio_without_fpe(struct stmmac_priv *priv,
+				       struct tc_taprio_qopt_offload *qopt)
+{
+	if (!qopt->mqprio.preemptible_tcs)
+		return tc_setup_taprio(priv, qopt);
+
+	NL_SET_ERR_MSG_MOD(qopt->mqprio.extack,
+			   "taprio with FPE is not implemented for this MAC");
+
+	return -EOPNOTSUPP;
+}
+
+static int tc_setup_etf(struct stmmac_priv *priv,
+			struct tc_etf_qopt_offload *qopt)
+{
+	if (!priv->dma_cap.tbssel)
+		return -EOPNOTSUPP;
+	if (qopt->queue >= priv->plat->tx_queues_to_use)
+		return -EINVAL;
+	if (!(priv->dma_conf.tx_queue[qopt->queue].tbs & STMMAC_TBS_AVAIL))
+		return -EINVAL;
+
+	if (qopt->enable)
+		priv->dma_conf.tx_queue[qopt->queue].tbs |= STMMAC_TBS_EN;
+	else
+		priv->dma_conf.tx_queue[qopt->queue].tbs &= ~STMMAC_TBS_EN;
+
+	netdev_info(priv->dev, "%s ETF for Queue %d\n",
+		    qopt->enable ? "enabled" : "disabled", qopt->queue);
+	return 0;
+}
+
+static int tc_query_caps(struct stmmac_priv *priv,
+			 struct tc_query_caps_base *base)
+{
+	switch (base->type) {
+	case TC_SETUP_QDISC_MQPRIO: {
+		struct tc_mqprio_caps *caps = base->caps;
+
+		caps->validate_queue_counts = true;
+
+		return 0;
+	}
+	case TC_SETUP_QDISC_TAPRIO: {
+		struct tc_taprio_caps *caps = base->caps;
+
+		if (!priv->dma_cap.estsel)
+			return -EOPNOTSUPP;
+
+		caps->gate_mask_per_txq = true;
+		caps->supports_queue_max_sdu = true;
+
+		return 0;
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void stmmac_reset_tc_mqprio(struct net_device *ndev,
+				   struct netlink_ext_ack *extack)
+{
+	struct stmmac_priv *priv = netdev_priv(ndev);
+
+	netdev_reset_tc(ndev);
+	netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
+	stmmac_fpe_map_preemption_class(priv, ndev, extack, 0);
+}
+
+static int tc_setup_dwmac510_mqprio(struct stmmac_priv *priv,
+				    struct tc_mqprio_qopt_offload *mqprio)
+{
+	struct netlink_ext_ack *extack = mqprio->extack;
+	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
+	u32 offset, count, num_stack_tx_queues = 0;
+	struct net_device *ndev = priv->dev;
+	u32 num_tc = qopt->num_tc;
+	int err;
+
+	if (!num_tc) {
+		stmmac_reset_tc_mqprio(ndev, extack);
+		return 0;
+	}
+
+	err = netdev_set_num_tc(ndev, num_tc);
+	if (err)
+		return err;
+
+	for (u32 tc = 0; tc < num_tc; tc++) {
+		offset = qopt->offset[tc];
+		count = qopt->count[tc];
+		num_stack_tx_queues += count;
+
+		err = netdev_set_tc_queue(ndev, tc, count, offset);
+		if (err)
+			goto err_reset_tc;
+	}
+
+	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
+	if (err)
+		goto err_reset_tc;
+
+	err = stmmac_fpe_map_preemption_class(priv, ndev, extack,
+					      mqprio->preemptible_tcs);
+	if (err)
+		goto err_reset_tc;
+
+	return 0;
+
+err_reset_tc:
+	stmmac_reset_tc_mqprio(ndev, extack);
+
+	return err;
+}
+
+static int tc_setup_mqprio_unimplemented(struct stmmac_priv *priv,
+					 struct tc_mqprio_qopt_offload *mqprio)
+{
+	NL_SET_ERR_MSG_MOD(mqprio->extack,
+			   "mqprio HW offload is not implemented for this MAC");
+	return -EOPNOTSUPP;
+}
+
+const struct stmmac_tc_ops ec_dwmac4_tc_ops = {
+	.init = tc_init,
+	.setup_cls_u32 = tc_setup_cls_u32,
+	.setup_cbs = tc_setup_cbs,
+	.setup_cls = tc_setup_cls,
+	.setup_taprio = tc_setup_taprio_without_fpe,
+	.setup_etf = tc_setup_etf,
+	.query_caps = tc_query_caps,
+	.setup_mqprio = tc_setup_mqprio_unimplemented,
+};
+
+const struct stmmac_tc_ops ec_dwmac510_tc_ops = {
+	.init = tc_init,
+	.setup_cls_u32 = tc_setup_cls_u32,
+	.setup_cbs = tc_setup_cbs,
+	.setup_cls = tc_setup_cls,
+	.setup_taprio = tc_setup_taprio,
+	.setup_etf = tc_setup_etf,
+	.query_caps = tc_query_caps,
+	.setup_mqprio = tc_setup_dwmac510_mqprio,
+};
diff --git a/drivers/net/ethercat/device/K3/stmmac_vlan-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_vlan-ethercat.c
new file mode 100644
index 000000000000..6853c1afe14f
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_vlan-ethercat.c
@@ -0,0 +1,375 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025, Altera Corporation
+ * stmmac VLAN (802.1Q) handling
+ */
+
+#include "stmmac-ethercat.h"
+#include "stmmac_vlan-ethercat.h"
+
+static void vlan_write_single(struct net_device *dev, u16 vid)
+{
+	void __iomem *ioaddr = (void __iomem *)dev->base_addr;
+	u32 val;
+
+	val = readl(ioaddr + VLAN_TAG);
+	val &= ~VLAN_TAG_VID;
+	val |= VLAN_TAG_ETV | vid;
+
+	writel(val, ioaddr + VLAN_TAG);
+}
+
+static int vlan_write_filter(struct net_device *dev,
+			     struct mac_device_info *hw,
+			     u8 index, u32 data)
+{
+	void __iomem *ioaddr = (void __iomem *)dev->base_addr;
+	int ret;
+	u32 val;
+
+	if (index >= hw->num_vlan)
+		return -EINVAL;
+
+	writel(data, ioaddr + VLAN_TAG_DATA);
+
+	val = readl(ioaddr + VLAN_TAG);
+	val &= ~(VLAN_TAG_CTRL_OFS_MASK |
+		VLAN_TAG_CTRL_CT |
+		VLAN_TAG_CTRL_OB);
+	val |= (index << VLAN_TAG_CTRL_OFS_SHIFT) | VLAN_TAG_CTRL_OB;
+
+	writel(val, ioaddr + VLAN_TAG);
+
+	ret = readl_poll_timeout(ioaddr + VLAN_TAG, val,
+				 !(val & VLAN_TAG_CTRL_OB),
+				 1000, 500000);
+	if (ret) {
+		netdev_err(dev, "Timeout accessing MAC_VLAN_Tag_Filter\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int vlan_add_hw_rx_fltr(struct net_device *dev,
+			       struct mac_device_info *hw,
+			       __be16 proto, u16 vid)
+{
+	int index = -1;
+	u32 val = 0;
+	int i, ret;
+
+	if (vid > 4095)
+		return -EINVAL;
+
+	/* Single Rx VLAN Filter */
+	if (hw->num_vlan == 1) {
+		/* For single VLAN filter, VID 0 means VLAN promiscuous */
+		if (vid == 0) {
+			netdev_warn(dev, "Adding VLAN ID 0 is not supported\n");
+			return -EPERM;
+		}
+
+		if (hw->vlan_filter[0] & VLAN_TAG_VID) {
+			netdev_err(dev, "Only single VLAN ID supported\n");
+			return -EPERM;
+		}
+
+		hw->vlan_filter[0] = vid;
+		vlan_write_single(dev, vid);
+
+		return 0;
+	}
+
+	/* Extended Rx VLAN Filter Enable */
+	val |= VLAN_TAG_DATA_ETV | VLAN_TAG_DATA_VEN | vid;
+
+	for (i = 0; i < hw->num_vlan; i++) {
+		if (hw->vlan_filter[i] == val)
+			return 0;
+		else if (!(hw->vlan_filter[i] & VLAN_TAG_DATA_VEN))
+			index = i;
+	}
+
+	if (index == -1) {
+		netdev_err(dev, "MAC_VLAN_Tag_Filter full (size: %0u)\n",
+			   hw->num_vlan);
+		return -EPERM;
+	}
+
+	ret = vlan_write_filter(dev, hw, index, val);
+
+	if (!ret)
+		hw->vlan_filter[index] = val;
+
+	return ret;
+}
+
+static int vlan_del_hw_rx_fltr(struct net_device *dev,
+			       struct mac_device_info *hw,
+			       __be16 proto, u16 vid)
+{
+	int i, ret = 0;
+
+	/* Single Rx VLAN Filter */
+	if (hw->num_vlan == 1) {
+		if ((hw->vlan_filter[0] & VLAN_TAG_VID) == vid) {
+			hw->vlan_filter[0] = 0;
+			vlan_write_single(dev, 0);
+		}
+		return 0;
+	}
+
+	/* Extended Rx VLAN Filter Enable */
+	for (i = 0; i < hw->num_vlan; i++) {
+		if ((hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) &&
+		    ((hw->vlan_filter[i] & VLAN_TAG_DATA_VID) == vid)) {
+			ret = vlan_write_filter(dev, hw, i, 0);
+
+			if (!ret)
+				hw->vlan_filter[i] = 0;
+			else
+				return ret;
+		}
+	}
+
+	return ret;
+}
+
+static void vlan_restore_hw_rx_fltr(struct net_device *dev,
+				    struct mac_device_info *hw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+	u32 hash;
+	u32 val;
+	int i;
+
+	/* Single Rx VLAN Filter */
+	if (hw->num_vlan == 1) {
+		vlan_write_single(dev, hw->vlan_filter[0]);
+		return;
+	}
+
+	/* Extended Rx VLAN Filter Enable */
+	for (i = 0; i < hw->num_vlan; i++) {
+		if (hw->vlan_filter[i] & VLAN_TAG_DATA_VEN) {
+			val = hw->vlan_filter[i];
+			vlan_write_filter(dev, hw, i, val);
+		}
+	}
+
+	hash = readl(ioaddr + VLAN_HASH_TABLE);
+	if (hash & VLAN_VLHT) {
+		value = readl(ioaddr + VLAN_TAG);
+		value |= VLAN_VTHM;
+		writel(value, ioaddr + VLAN_TAG);
+	}
+}
+
+static void vlan_update_hash(struct mac_device_info *hw, u32 hash,
+			     u16 perfect_match, bool is_double)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	writel(hash, ioaddr + VLAN_HASH_TABLE);
+
+	value = readl(ioaddr + VLAN_TAG);
+
+	if (hash) {
+		value |= VLAN_VTHM | VLAN_ETV;
+		if (is_double) {
+			value |= VLAN_EDVLP;
+			value |= VLAN_ESVL;
+			value |= VLAN_DOVLTC;
+		}
+
+		writel(value, ioaddr + VLAN_TAG);
+	} else if (perfect_match) {
+		u32 value = VLAN_ETV;
+
+		if (is_double) {
+			value |= VLAN_EDVLP;
+			value |= VLAN_ESVL;
+			value |= VLAN_DOVLTC;
+		}
+
+		writel(value | perfect_match, ioaddr + VLAN_TAG);
+	} else {
+		value &= ~(VLAN_VTHM | VLAN_ETV);
+		value &= ~(VLAN_EDVLP | VLAN_ESVL);
+		value &= ~VLAN_DOVLTC;
+		value &= ~VLAN_VID;
+
+		writel(value, ioaddr + VLAN_TAG);
+	}
+}
+
+static void vlan_enable(struct mac_device_info *hw, u32 type)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value;
+
+	value = readl(ioaddr + VLAN_INCL);
+	value |= VLAN_VLTI;
+	value &= ~VLAN_CSVL; /* Only use CVLAN */
+	value &= ~VLAN_VLC;
+	value |= (type << VLAN_VLC_SHIFT) & VLAN_VLC;
+	writel(value, ioaddr + VLAN_INCL);
+}
+
+static void vlan_rx_hw(struct mac_device_info *hw,
+		       struct dma_desc *rx_desc, struct sk_buff *skb)
+{
+	if (hw->desc->get_rx_vlan_valid(rx_desc)) {
+		u16 vid = hw->desc->get_rx_vlan_tci(rx_desc);
+
+		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
+	}
+}
+
+static void vlan_set_hw_mode(struct mac_device_info *hw)
+{
+	void __iomem *ioaddr = hw->pcsr;
+	u32 value = readl(ioaddr + VLAN_TAG);
+
+	value &= ~VLAN_TAG_CTRL_EVLS_MASK;
+
+	if (hw->hw_vlan_en)
+		/* Always strip VLAN on Receive */
+		value |= VLAN_TAG_STRIP_ALL;
+	else
+		/* Do not strip VLAN on Receive */
+		value |= VLAN_TAG_STRIP_NONE;
+
+	/* Enable outer VLAN Tag in Rx DMA descriptor */
+	value |= VLAN_TAG_CTRL_EVLRXS;
+	writel(value, ioaddr + VLAN_TAG);
+}
+
+static void dwxgmac2_update_vlan_hash(struct mac_device_info *hw, u32 hash,
+				      u16 perfect_match, bool is_double)
+{
+	void __iomem *ioaddr = hw->pcsr;
+
+	writel(hash, ioaddr + VLAN_HASH_TABLE);
+
+	if (hash) {
+		u32 value = readl(ioaddr + XGMAC_PACKET_FILTER);
+
+		value |= XGMAC_FILTER_VTFE;
+
+		writel(value, ioaddr + XGMAC_PACKET_FILTER);
+
+		value = readl(ioaddr + VLAN_TAG);
+
+		value |= VLAN_VTHM | VLAN_ETV;
+		if (is_double) {
+			value |= VLAN_EDVLP;
+			value |= VLAN_ESVL;
+			value |= VLAN_DOVLTC;
+		} else {
+			value &= ~VLAN_EDVLP;
+			value &= ~VLAN_ESVL;
+			value &= ~VLAN_DOVLTC;
+		}
+
+		value &= ~VLAN_VID;
+		writel(value, ioaddr + VLAN_TAG);
+	} else if (perfect_match) {
+		u32 value = readl(ioaddr + XGMAC_PACKET_FILTER);
+
+		value |= XGMAC_FILTER_VTFE;
+
+		writel(value, ioaddr + XGMAC_PACKET_FILTER);
+
+		value = readl(ioaddr + VLAN_TAG);
+
+		value &= ~VLAN_VTHM;
+		value |= VLAN_ETV;
+		if (is_double) {
+			value |= VLAN_EDVLP;
+			value |= VLAN_ESVL;
+			value |= VLAN_DOVLTC;
+		} else {
+			value &= ~VLAN_EDVLP;
+			value &= ~VLAN_ESVL;
+			value &= ~VLAN_DOVLTC;
+		}
+
+		value &= ~VLAN_VID;
+		writel(value | perfect_match, ioaddr + VLAN_TAG);
+	} else {
+		u32 value = readl(ioaddr + XGMAC_PACKET_FILTER);
+
+		value &= ~XGMAC_FILTER_VTFE;
+
+		writel(value, ioaddr + XGMAC_PACKET_FILTER);
+
+		value = readl(ioaddr + VLAN_TAG);
+
+		value &= ~(VLAN_VTHM | VLAN_ETV);
+		value &= ~(VLAN_EDVLP | VLAN_ESVL);
+		value &= ~VLAN_DOVLTC;
+		value &= ~VLAN_VID;
+
+		writel(value, ioaddr + VLAN_TAG);
+	}
+}
+
+const struct stmmac_vlan_ops ec_dwmac_vlan_ops = {
+	.update_vlan_hash = vlan_update_hash,
+	.enable_vlan = vlan_enable,
+	.add_hw_vlan_rx_fltr = vlan_add_hw_rx_fltr,
+	.del_hw_vlan_rx_fltr = vlan_del_hw_rx_fltr,
+	.restore_hw_vlan_rx_fltr = vlan_restore_hw_rx_fltr,
+	.rx_hw_vlan = vlan_rx_hw,
+	.set_hw_vlan_mode = vlan_set_hw_mode,
+};
+
+const struct stmmac_vlan_ops ec_dwxlgmac2_vlan_ops = {
+	.update_vlan_hash = dwxgmac2_update_vlan_hash,
+	.enable_vlan = vlan_enable,
+};
+
+const struct stmmac_vlan_ops ec_dwxgmac210_vlan_ops = {
+	.update_vlan_hash = dwxgmac2_update_vlan_hash,
+	.enable_vlan = vlan_enable,
+	.add_hw_vlan_rx_fltr = vlan_add_hw_rx_fltr,
+	.del_hw_vlan_rx_fltr = vlan_del_hw_rx_fltr,
+	.restore_hw_vlan_rx_fltr = vlan_restore_hw_rx_fltr,
+	.rx_hw_vlan = vlan_rx_hw,
+	.set_hw_vlan_mode = vlan_set_hw_mode,
+};
+
+u32 ec_stmmac_get_num_vlan(void __iomem *ioaddr)
+{
+	u32 val, num_vlan;
+
+	val = readl(ioaddr + HW_FEATURE3);
+	switch (val & VLAN_HW_FEAT_NRVF) {
+	case 0:
+		num_vlan = 1;
+		break;
+	case 1:
+		num_vlan = 4;
+		break;
+	case 2:
+		num_vlan = 8;
+		break;
+	case 3:
+		num_vlan = 16;
+		break;
+	case 4:
+		num_vlan = 24;
+		break;
+	case 5:
+		num_vlan = 32;
+		break;
+	default:
+		num_vlan = 1;
+	}
+
+	return num_vlan;
+}
diff --git a/drivers/net/ethercat/device/K3/stmmac_vlan-ethercat.h b/drivers/net/ethercat/device/K3/stmmac_vlan-ethercat.h
new file mode 100644
index 000000000000..5b91c50911f5
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_vlan-ethercat.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025, Altera Corporation
+ * stmmac VLAN(802.1Q) handling
+ */
+
+#ifndef __STMMAC_VLAN_H__
+#define __STMMAC_VLAN_H__
+
+#include <linux/bitfield.h>
+#include "dwxgmac2-ethercat.h"
+
+#define VLAN_TAG			0x00000050
+#define VLAN_TAG_DATA			0x00000054
+#define VLAN_HASH_TABLE			0x00000058
+#define VLAN_INCL			0x00000060
+
+/* MAC VLAN */
+#define VLAN_EDVLP			BIT(26)
+#define VLAN_VTHM			BIT(25)
+#define VLAN_DOVLTC			BIT(20)
+#define VLAN_ESVL			BIT(18)
+#define VLAN_ETV			BIT(16)
+#define VLAN_VID			GENMASK(15, 0)
+#define VLAN_VLTI			BIT(20)
+#define VLAN_CSVL			BIT(19)
+#define VLAN_VLC			GENMASK(17, 16)
+#define VLAN_VLC_SHIFT			16
+#define VLAN_VLHT			GENMASK(15, 0)
+
+/* MAC VLAN Tag */
+#define VLAN_TAG_VID			GENMASK(15, 0)
+#define VLAN_TAG_ETV			BIT(16)
+
+/* MAC VLAN Tag Control */
+#define VLAN_TAG_CTRL_OB		BIT(0)
+#define VLAN_TAG_CTRL_CT		BIT(1)
+#define VLAN_TAG_CTRL_OFS_MASK		GENMASK(6, 2)
+#define VLAN_TAG_CTRL_OFS_SHIFT		2
+#define VLAN_TAG_CTRL_EVLS_MASK		GENMASK(22, 21)
+#define VLAN_TAG_CTRL_EVLS_SHIFT	21
+#define VLAN_TAG_CTRL_EVLRXS		BIT(24)
+
+#define VLAN_TAG_STRIP_NONE		FIELD_PREP(VLAN_TAG_CTRL_EVLS_MASK, 0x0)
+#define VLAN_TAG_STRIP_PASS		FIELD_PREP(VLAN_TAG_CTRL_EVLS_MASK, 0x1)
+#define VLAN_TAG_STRIP_FAIL		FIELD_PREP(VLAN_TAG_CTRL_EVLS_MASK, 0x2)
+#define VLAN_TAG_STRIP_ALL		FIELD_PREP(VLAN_TAG_CTRL_EVLS_MASK, 0x3)
+
+/* MAC VLAN Tag Data/Filter */
+#define VLAN_TAG_DATA_VID		GENMASK(15, 0)
+#define VLAN_TAG_DATA_VEN		BIT(16)
+#define VLAN_TAG_DATA_ETV		BIT(17)
+
+/* MAC VLAN HW FEAT */
+#define HW_FEATURE3			0x00000128
+#define VLAN_HW_FEAT_NRVF		GENMASK(2, 0)
+
+extern const struct stmmac_vlan_ops ec_dwmac_vlan_ops;
+extern const struct stmmac_vlan_ops ec_dwxgmac210_vlan_ops;
+extern const struct stmmac_vlan_ops ec_dwxlgmac2_vlan_ops;
+
+u32 ec_stmmac_get_num_vlan(void __iomem *ioaddr);
+
+#endif /* __STMMAC_VLAN_H__ */
diff --git a/drivers/net/ethercat/device/K3/stmmac_xdp-ethercat.c b/drivers/net/ethercat/device/K3/stmmac_xdp-ethercat.c
new file mode 100644
index 000000000000..de98074da1ac
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_xdp-ethercat.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2021, Intel Corporation. */
+
+#include <net/xdp_sock_drv.h>
+
+#include "stmmac-ethercat.h"
+#include "stmmac_xdp-ethercat.h"
+
+static int stmmac_xdp_enable_pool(struct stmmac_priv *priv,
+				  struct xsk_buff_pool *pool, u16 queue)
+{
+	struct stmmac_channel *ch = &priv->channel[queue];
+	bool need_update;
+	u32 frame_size;
+	int err;
+
+	if (queue >= priv->plat->rx_queues_to_use ||
+	    queue >= priv->plat->tx_queues_to_use)
+		return -EINVAL;
+
+	frame_size = xsk_pool_get_rx_frame_size(pool);
+	/* XDP ZC does not span multiple frame, make sure XSK pool buffer
+	 * size can at least store Q-in-Q frame.
+	 */
+	if (frame_size < ETH_FRAME_LEN + VLAN_HLEN * 2)
+		return -EOPNOTSUPP;
+
+	err = xsk_pool_dma_map(pool, priv->device, STMMAC_RX_DMA_ATTR);
+	if (err) {
+		netdev_err(priv->dev, "Failed to map xsk pool\n");
+		return err;
+	}
+
+	need_update = netif_running(priv->dev) && stmmac_xdp_is_enabled(priv);
+
+	if (need_update) {
+		napi_disable(&ch->rx_napi);
+		napi_disable(&ch->tx_napi);
+		ec_stmmac_disable_rx_queue(priv, queue);
+		ec_stmmac_disable_tx_queue(priv, queue);
+	}
+
+	set_bit(queue, priv->af_xdp_zc_qps);
+
+	if (need_update) {
+		ec_stmmac_enable_rx_queue(priv, queue);
+		ec_stmmac_enable_tx_queue(priv, queue);
+		napi_enable(&ch->rxtx_napi);
+
+		err = ec_stmmac_xsk_wakeup(priv->dev, queue, XDP_WAKEUP_RX);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int stmmac_xdp_disable_pool(struct stmmac_priv *priv, u16 queue)
+{
+	struct stmmac_channel *ch = &priv->channel[queue];
+	struct xsk_buff_pool *pool;
+	bool need_update;
+
+	if (queue >= priv->plat->rx_queues_to_use ||
+	    queue >= priv->plat->tx_queues_to_use)
+		return -EINVAL;
+
+	pool = xsk_get_pool_from_qid(priv->dev, queue);
+	if (!pool)
+		return -EINVAL;
+
+	need_update = netif_running(priv->dev) && stmmac_xdp_is_enabled(priv);
+
+	if (need_update) {
+		napi_disable(&ch->rxtx_napi);
+		ec_stmmac_disable_rx_queue(priv, queue);
+		ec_stmmac_disable_tx_queue(priv, queue);
+		synchronize_rcu();
+	}
+
+	xsk_pool_dma_unmap(pool, STMMAC_RX_DMA_ATTR);
+
+	clear_bit(queue, priv->af_xdp_zc_qps);
+
+	if (need_update) {
+		ec_stmmac_enable_rx_queue(priv, queue);
+		ec_stmmac_enable_tx_queue(priv, queue);
+		napi_enable(&ch->rx_napi);
+		napi_enable(&ch->tx_napi);
+	}
+
+	return 0;
+}
+
+int ec_stmmac_xdp_setup_pool(struct stmmac_priv *priv, struct xsk_buff_pool *pool,
+			  u16 queue)
+{
+	return pool ? stmmac_xdp_enable_pool(priv, pool, queue) :
+		      stmmac_xdp_disable_pool(priv, queue);
+}
+
+int ec_stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog,
+			struct netlink_ext_ack *extack)
+{
+	struct net_device *dev = priv->dev;
+	struct bpf_prog *old_prog;
+	bool need_update;
+	bool if_running;
+
+	if_running = netif_running(dev);
+
+	if (prog && dev->mtu > ETH_DATA_LEN) {
+		/* For now, the driver doesn't support XDP functionality with
+		 * jumbo frames so we return error.
+		 */
+		NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported");
+		return -EOPNOTSUPP;
+	}
+
+	if (!prog)
+		xdp_features_clear_redirect_target(dev);
+
+	need_update = !!priv->xdp_prog != !!prog;
+	if (if_running && need_update)
+		ec_stmmac_xdp_release(dev);
+
+	old_prog = xchg(&priv->xdp_prog, prog);
+	if (old_prog)
+		bpf_prog_put(old_prog);
+
+	/* Disable RX SPH for XDP operation */
+	priv->sph = priv->sph_cap && !stmmac_xdp_is_enabled(priv);
+
+	if (if_running && need_update)
+		ec_stmmac_xdp_open(dev);
+
+	if (prog)
+		xdp_features_set_redirect_target(dev, false);
+
+	return 0;
+}
diff --git a/drivers/net/ethercat/device/K3/stmmac_xdp-ethercat.h b/drivers/net/ethercat/device/K3/stmmac_xdp-ethercat.h
new file mode 100644
index 000000000000..89916af46617
--- /dev/null
+++ b/drivers/net/ethercat/device/K3/stmmac_xdp-ethercat.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021, Intel Corporation. */
+
+#ifndef _STMMAC_XDP_H_
+#define _STMMAC_XDP_H_
+
+#define STMMAC_RX_DMA_ATTR	(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
+
+int ec_stmmac_xdp_setup_pool(struct stmmac_priv *priv, struct xsk_buff_pool *pool,
+			  u16 queue);
+int ec_stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog,
+			struct netlink_ext_ack *extack);
+
+#endif /* _STMMAC_XDP_H_ */
diff --git a/drivers/net/ethercat/device/Kconfig b/drivers/net/ethercat/device/Kconfig
new file mode 100644
index 000000000000..da49fff17f11
--- /dev/null
+++ b/drivers/net/ethercat/device/Kconfig
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# EtherCAT device support
+#
+
+menuconfig EC_DEVICE
+	tristate "EtherCAT device"
+	depends on ETHERCAT && EC_MASTER
+	help
+	  Enable EtherCAT device support.
+
+if EC_DEVICE
+
+config EC_GENERIC
+	tristate "Generic EtherCAT device support"
+	depends on EC_MASTER
+	help
+	  Generic EtherCAT device support using the standard Linux networking
+	  stack. This is portable but may have higher latency/jitter.
+
+config EC_K1X_GMAC
+	tristate "Spacemit K1 GMAC EtherCAT device support"
+	depends on EC_MASTER && (SOC_SPACEMIT_K1 || COMPILE_TEST)
+	help
+	  EtherCAT device support for the Spacemit K1X GMAC controller.
+
+config EC_K3_GMAC
+	tristate "Spacemit K3 GMAC EtherCAT device support"
+	depends on EC_MASTER && SOC_SPACEMIT_K3
+	select PAGE_POOL
+	help
+	  EtherCAT device support for Spacemit K3 GMAC controller.
+
+endif # EC_DEVICE
diff --git a/drivers/net/ethercat/device/Makefile b/drivers/net/ethercat/device/Makefile
new file mode 100644
index 000000000000..12b5c16b4374
--- /dev/null
+++ b/drivers/net/ethercat/device/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for K1X EMAC EtherCAT device driver
+#
+
+obj-$(CONFIG_EC_GENERIC) += generic/
+obj-$(CONFIG_EC_K1X_GMAC) += K1/
+obj-$(CONFIG_EC_K3_GMAC) += K3/
diff --git a/drivers/net/ethercat/device/ecdev.h b/drivers/net/ethercat/device/ecdev.h
new file mode 100644
index 000000000000..2f2e51d50ef9
--- /dev/null
+++ b/drivers/net/ethercat/device/ecdev.h
@@ -0,0 +1,71 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ *
+ * EtherCAT interface for EtherCAT device drivers.
+ *
+ * \defgroup DeviceInterface EtherCAT Device Interface
+ *
+ * Master interface for EtherCAT-capable network device drivers. Through the
+ * EtherCAT device interface, EtherCAT-capable network device drivers are able
+ * to connect their device(s) to the master, pass received frames and notify
+ * the master about status changes. The master on his part, can send his
+ * frames through connected devices.
+ */
+
+/****************************************************************************/
+
+#ifndef __ECDEV_H__
+#define __ECDEV_H__
+
+#include <linux/netdevice.h>
+
+/****************************************************************************/
+
+struct ec_device;
+typedef struct ec_device ec_device_t; /**< \see ec_device */
+
+/** Device poll function type.
+ */
+typedef void (*ec_pollfunc_t)(struct net_device *);
+
+/*****************************************************************************
+ * Offering/withdrawal functions
+ ****************************************************************************/
+
+ec_device_t *ecdev_offer(struct net_device *net_dev, ec_pollfunc_t poll,
+        struct module *module);
+void ecdev_withdraw(ec_device_t *device);
+
+/*****************************************************************************
+ * Device methods
+ ****************************************************************************/
+
+int ecdev_open(ec_device_t *device);
+void ecdev_close(ec_device_t *device);
+void ecdev_receive(ec_device_t *device, const void *data, size_t size);
+void ecdev_set_link(ec_device_t *device, uint8_t state);
+uint8_t ecdev_get_link(const ec_device_t *device);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/device/generic/Makefile b/drivers/net/ethercat/device/generic/Makefile
new file mode 100644
index 000000000000..53e14e0d9895
--- /dev/null
+++ b/drivers/net/ethercat/device/generic/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for generic device driver
+#
+
+obj-$(CONFIG_EC_GENERIC) += ec_generic.o
diff --git a/drivers/net/ethercat/device/generic/ec_generic.c b/drivers/net/ethercat/device/generic/ec_generic.c
new file mode 100644
index 000000000000..a07e26e92fb6
--- /dev/null
+++ b/drivers/net/ethercat/device/generic/ec_generic.c
@@ -0,0 +1,483 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT generic Ethernet device module.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/version.h>
+#include <linux/if_arp.h> /* ARPHRD_ETHER */
+#include <linux/etherdevice.h>
+
+#include "../include/globals.h"
+#include "ecdev.h"
+
+#define PFX "ec_generic: "
+
+#define ETH_P_ETHERCAT 0x88A4
+
+#define EC_GEN_RX_BUF_SIZE 1600
+
+#if defined(CONFIG_SUSE_KERNEL) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
+#include <linux/suse_version.h>
+#else
+#  ifndef SUSE_VERSION
+#    define SUSE_VERSION 0
+#  endif
+#  ifndef SUSE_PATCHLEVEL
+#    define SUSE_PATCHLEVEL 0
+#  endif
+#endif
+
+/****************************************************************************/
+
+int __init ec_gen_init_module(void);
+void __exit ec_gen_cleanup_module(void);
+void ec_gen_poll(struct net_device *);
+
+/****************************************************************************/
+
+/** \cond */
+
+MODULE_AUTHOR("Florian Pose <fp@igh.de>");
+MODULE_DESCRIPTION("EtherCAT master generic Ethernet device module");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(EC_MASTER_VERSION);
+
+/** \endcond */
+
+struct list_head generic_devices;
+
+typedef struct {
+    struct list_head list;
+    struct net_device *netdev;
+    struct net_device *used_netdev;
+    struct socket *socket;
+    ec_device_t *ecdev;
+    uint8_t *rx_buf;
+} ec_gen_device_t;
+
+typedef struct {
+    struct list_head list;
+    struct net_device *netdev;
+    char name[IFNAMSIZ];
+    int ifindex;
+    uint8_t dev_addr[ETH_ALEN];
+} ec_gen_interface_desc_t;
+
+int ec_gen_device_init(ec_gen_device_t *);
+void ec_gen_device_clear(ec_gen_device_t *);
+int ec_gen_device_create_socket(ec_gen_device_t *, ec_gen_interface_desc_t *);
+int ec_gen_device_offer(ec_gen_device_t *, ec_gen_interface_desc_t *);
+int ec_gen_device_open(ec_gen_device_t *);
+int ec_gen_device_stop(ec_gen_device_t *);
+int ec_gen_device_start_xmit(ec_gen_device_t *, struct sk_buff *);
+void ec_gen_device_poll(ec_gen_device_t *);
+
+int offer_device(ec_gen_interface_desc_t *);
+void clear_devices(void);
+
+/****************************************************************************/
+
+static int ec_gen_netdev_open(struct net_device *dev)
+{
+    ec_gen_device_t *gendev = *((ec_gen_device_t **) netdev_priv(dev));
+    return ec_gen_device_open(gendev);
+}
+
+/****************************************************************************/
+
+static int ec_gen_netdev_stop(struct net_device *dev)
+{
+    ec_gen_device_t *gendev = *((ec_gen_device_t **) netdev_priv(dev));
+    return ec_gen_device_stop(gendev);
+}
+
+/****************************************************************************/
+
+static int ec_gen_netdev_start_xmit(
+        struct sk_buff *skb,
+        struct net_device *dev
+        )
+{
+    ec_gen_device_t *gendev = *((ec_gen_device_t **) netdev_priv(dev));
+    return ec_gen_device_start_xmit(gendev, skb);
+}
+
+/****************************************************************************/
+
+void ec_gen_poll(struct net_device *dev)
+{
+    ec_gen_device_t *gendev = *((ec_gen_device_t **) netdev_priv(dev));
+    ec_gen_device_poll(gendev);
+}
+
+/****************************************************************************/
+
+static const struct net_device_ops ec_gen_netdev_ops = {
+    .ndo_open       = ec_gen_netdev_open,
+    .ndo_stop       = ec_gen_netdev_stop,
+    .ndo_start_xmit = ec_gen_netdev_start_xmit,
+};
+
+/****************************************************************************/
+
+/** Init generic device.
+ */
+int ec_gen_device_init(
+        ec_gen_device_t *dev
+        )
+{
+    ec_gen_device_t **priv;
+    char null = 0x00;
+
+    dev->ecdev = NULL;
+    dev->socket = NULL;
+    dev->rx_buf = NULL;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+    dev->netdev = alloc_netdev(sizeof(ec_gen_device_t *), &null,
+            NET_NAME_UNKNOWN, ether_setup);
+#else
+    dev->netdev = alloc_netdev(sizeof(ec_gen_device_t *), &null, ether_setup);
+#endif
+    if (!dev->netdev) {
+        return -ENOMEM;
+    }
+
+    dev->netdev->netdev_ops = &ec_gen_netdev_ops;
+
+    priv = netdev_priv(dev->netdev);
+    *priv = dev;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Clear generic device.
+ */
+void ec_gen_device_clear(
+        ec_gen_device_t *dev
+        )
+{
+    if (dev->ecdev) {
+        ecdev_close(dev->ecdev);
+        ecdev_withdraw(dev->ecdev);
+    }
+    if (dev->socket) {
+        sock_release(dev->socket);
+    }
+    free_netdev(dev->netdev);
+
+    if (dev->rx_buf) {
+        kfree(dev->rx_buf);
+    }
+}
+
+/****************************************************************************/
+
+/** Creates a network socket.
+ */
+int ec_gen_device_create_socket(
+        ec_gen_device_t *dev,
+        ec_gen_interface_desc_t *desc
+        )
+{
+    int ret;
+    struct sockaddr_ll sa;
+
+    dev->rx_buf = kmalloc(EC_GEN_RX_BUF_SIZE, GFP_KERNEL);
+    if (!dev->rx_buf) {
+        return -ENOMEM;
+    }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)
+    ret = sock_create_kern(&init_net, PF_PACKET, SOCK_RAW,
+            htons(ETH_P_ETHERCAT), &dev->socket);
+#else
+    ret = sock_create_kern(PF_PACKET, SOCK_RAW, htons(ETH_P_ETHERCAT),
+            &dev->socket);
+#endif
+    if (ret) {
+        printk(KERN_ERR PFX "Failed to create socket (ret = %i).\n", ret);
+        return ret;
+    }
+
+    printk(KERN_ERR PFX "Binding socket to interface %i (%s).\n",
+            desc->ifindex, desc->name);
+
+    memset(&sa, 0x00, sizeof(sa));
+    sa.sll_family = AF_PACKET;
+    sa.sll_protocol = htons(ETH_P_ETHERCAT);
+    sa.sll_ifindex = desc->ifindex;
+    ret = kernel_bind(dev->socket, (struct sockaddr *) &sa, sizeof(sa));
+    if (ret) {
+        printk(KERN_ERR PFX "Failed to bind() socket to interface"
+                " (ret = %i).\n", ret);
+        sock_release(dev->socket);
+        dev->socket = NULL;
+        return ret;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Offer generic device to master.
+ */
+int ec_gen_device_offer(
+        ec_gen_device_t *dev,
+        ec_gen_interface_desc_t *desc
+        )
+{
+    int ret = 0;
+
+    dev->used_netdev = desc->netdev;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) || (SUSE_VERSION == 15 && SUSE_PATCHLEVEL >= 5)
+    eth_hw_addr_set(dev->netdev, desc->dev_addr);
+#else
+    memcpy(dev->netdev->dev_addr, desc->dev_addr, ETH_ALEN);
+#endif
+
+    dev->ecdev = ecdev_offer(dev->netdev, ec_gen_poll, THIS_MODULE);
+    if (dev->ecdev) {
+        if (ec_gen_device_create_socket(dev, desc)) {
+            ecdev_withdraw(dev->ecdev);
+            dev->ecdev = NULL;
+        } else if (ecdev_open(dev->ecdev)) {
+            ecdev_withdraw(dev->ecdev);
+            dev->ecdev = NULL;
+        } else {
+            ecdev_set_link(dev->ecdev, netif_carrier_ok(dev->used_netdev)); // FIXME
+            ret = 1;
+        }
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Open the device.
+ */
+int ec_gen_device_open(
+        ec_gen_device_t *dev
+        )
+{
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Stop the device.
+ */
+int ec_gen_device_stop(
+        ec_gen_device_t *dev
+        )
+{
+    return 0;
+}
+
+/****************************************************************************/
+
+int ec_gen_device_start_xmit(
+        ec_gen_device_t *dev,
+        struct sk_buff *skb
+        )
+{
+    struct msghdr msg;
+    struct kvec iov;
+    size_t len = skb->len;
+    int ret;
+
+    ecdev_set_link(dev->ecdev, netif_carrier_ok(dev->used_netdev));
+
+    iov.iov_base = skb->data;
+    iov.iov_len = len;
+    memset(&msg, 0, sizeof(msg));
+
+    ret = kernel_sendmsg(dev->socket, &msg, &iov, 1, len);
+
+    return ret == len ? NETDEV_TX_OK : NETDEV_TX_BUSY;
+}
+
+/****************************************************************************/
+
+/** Polls the device.
+ */
+void ec_gen_device_poll(
+        ec_gen_device_t *dev
+        )
+{
+    struct msghdr msg;
+    struct kvec iov;
+    int ret, budget = 10; // FIXME
+
+    ecdev_set_link(dev->ecdev, netif_carrier_ok(dev->used_netdev));
+
+    do {
+        iov.iov_base = dev->rx_buf;
+        iov.iov_len = EC_GEN_RX_BUF_SIZE;
+        memset(&msg, 0, sizeof(msg));
+
+        ret = kernel_recvmsg(dev->socket, &msg, &iov, 1, iov.iov_len,
+                MSG_DONTWAIT);
+        if (ret > 0) {
+            ecdev_receive(dev->ecdev, dev->rx_buf, ret);
+        } else if (ret < 0) {
+            break;
+        }
+        budget--;
+    } while (budget);
+}
+
+/****************************************************************************/
+
+/** Offer device.
+ */
+int offer_device(
+        ec_gen_interface_desc_t *desc
+        )
+{
+    ec_gen_device_t *gendev;
+    int ret = 0;
+
+    gendev = kmalloc(sizeof(ec_gen_device_t), GFP_KERNEL);
+    if (!gendev) {
+        return -ENOMEM;
+    }
+
+    ret = ec_gen_device_init(gendev);
+    if (ret) {
+        kfree(gendev);
+        return ret;
+    }
+
+    if (ec_gen_device_offer(gendev, desc)) {
+        list_add_tail(&gendev->list, &generic_devices);
+    } else {
+        ec_gen_device_clear(gendev);
+        kfree(gendev);
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Clear devices.
+ */
+void clear_devices(void)
+{
+    ec_gen_device_t *gendev, *next;
+
+    list_for_each_entry_safe(gendev, next, &generic_devices, list) {
+        list_del(&gendev->list);
+        ec_gen_device_clear(gendev);
+        kfree(gendev);
+    }
+}
+
+/****************************************************************************/
+
+/** Module initialization.
+ *
+ * Initializes \a master_count masters.
+ * \return 0 on success, else < 0
+ */
+int __init ec_gen_init_module(void)
+{
+    int ret = 0;
+    struct list_head descs;
+    struct net_device *netdev;
+    ec_gen_interface_desc_t *desc, *next;
+
+    printk(KERN_INFO PFX "EtherCAT master generic Ethernet device module %s\n",
+            EC_MASTER_VERSION);
+
+    INIT_LIST_HEAD(&generic_devices);
+    INIT_LIST_HEAD(&descs);
+
+    rcu_read_lock();
+    for_each_netdev_rcu(&init_net, netdev) {
+        if (netdev->type != ARPHRD_ETHER)
+            continue;
+        desc = kmalloc(sizeof(ec_gen_interface_desc_t), GFP_ATOMIC);
+        if (!desc) {
+            ret = -ENOMEM;
+            rcu_read_unlock();
+            goto out_err;
+        }
+        strncpy(desc->name, netdev->name, IFNAMSIZ);
+        desc->netdev = netdev;
+        desc->ifindex = netdev->ifindex;
+        memcpy(desc->dev_addr, netdev->dev_addr, ETH_ALEN);
+        list_add_tail(&desc->list, &descs);
+    }
+    rcu_read_unlock();
+
+    list_for_each_entry_safe(desc, next, &descs, list) {
+        ret = offer_device(desc);
+        if (ret) {
+            goto out_err;
+        }
+        kfree(desc);
+    }
+    return ret;
+
+out_err:
+    list_for_each_entry_safe(desc, next, &descs, list) {
+        list_del(&desc->list);
+        kfree(desc);
+    }
+    clear_devices();
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Module cleanup.
+ *
+ * Clears all master instances.
+ */
+void __exit ec_gen_cleanup_module(void)
+{
+    clear_devices();
+    printk(KERN_INFO PFX "Unloading.\n");
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+module_init(ec_gen_init_module);
+module_exit(ec_gen_cleanup_module);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/include/config.h b/drivers/net/ethercat/include/config.h
new file mode 100644
index 000000000000..03600e5322a4
--- /dev/null
+++ b/drivers/net/ethercat/include/config.h
@@ -0,0 +1,98 @@
+/* config.h.  Generated from config.h.in by configure.  */
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Debug interfaces enabled */
+/* #undef EC_DEBUG_IF */
+
+/* Debug ring enabled */
+/* #undef EC_DEBUG_RING */
+
+/* EoE support enabled */
+#define EC_EOE 1
+
+/* Use CPU timestamp counter */
+/* #undef EC_HAVE_CYCLES */
+
+/* Use vendor id / product code wildcards */
+/* #undef EC_IDENT_WILDCARDS */
+
+/* Max. number of Ethernet devices per master */
+#define EC_MAX_NUM_DEVICES 1
+
+/* Read alias adresses from register */
+/* #undef EC_REGALIAS */
+
+/* RTDM interface enabled */
+/* #undef EC_RTDM */
+
+/* Use Xenomai3 RTDM flavour */
+/* #undef EC_RTDM_XENOMAI_V3 */
+
+/* Output to syslog in RT context */
+#define EC_RT_SYSLOG 1
+
+/* Assign SII to PDI */
+#define EC_SII_ASSIGN 1
+
+/* Use hrtimer for scheduling */
+/* #undef EC_USE_HRTIMER */
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to the sub-directory where libtool stores uninstalled libraries. */
+#define LT_OBJDIR ".libs/"
+
+/* Name of package */
+#define PACKAGE "ethercat"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT "fp@igh.de"
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "ethercat"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "ethercat 1.6.8"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "ethercat"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "1.6.8"
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Version number of package */
+#define VERSION "1.6.8"
diff --git a/drivers/net/ethercat/include/ecrt.h b/drivers/net/ethercat/include/ecrt.h
new file mode 100644
index 000000000000..3ab462fbc2b5
--- /dev/null
+++ b/drivers/net/ethercat/include/ecrt.h
@@ -0,0 +1,3153 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT master userspace library.
+ *
+ *  The IgH EtherCAT master userspace library is free software; you can
+ *  redistribute it and/or modify it under the terms of the GNU Lesser General
+ *  Public License as published by the Free Software Foundation; version 2.1
+ *  of the License.
+ *
+ *  The IgH EtherCAT master userspace library is distributed in the hope that
+ *  it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+ *  warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with the IgH EtherCAT master userspace library. If not, see
+ *  <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+/** \file
+ *
+ * EtherCAT master application interface.
+ *
+ * \defgroup ApplicationInterface EtherCAT Application Interface
+ *
+ * EtherCAT interface for realtime applications. This interface is designed
+ * for realtime modules that want to use EtherCAT. There are functions to
+ * request a master, to map process data, to communicate with slaves via CoE
+ * and to configure and activate the bus.
+ *
+ *
+ * Changes in version 1.6.0:
+ *
+ * - Added the ecrt_master_scan_progress() method, the
+ *   ec_master_scan_progress_t structure and the EC_HAVE_SCAN_PROGRESS
+ *   definition to check for its existence.
+ * - Added the EoE configuration methods ecrt_slave_config_eoe_mac_address(),
+ *   ecrt_slave_config_eoe_ip_address(), ecrt_slave_config_eoe_subnet_mask(),
+ *   ecrt_slave_config_eoe_default_gateway(),
+ *   ecrt_slave_config_eoe_dns_address(),
+ *   ecrt_slave_config_eoe_hostname() and the EC_HAVE_SET_IP
+ *   definition to check for its existence.
+ * - Added ecrt_slave_config_state_timeout() to set the application-layer
+ *   state change timeout and EC_HAVE_STATE_TIMEOUT to check for its
+ *   existence.
+ *
+ * Changes since version 1.5.2:
+ *
+ * - Added the ecrt_slave_config_flag() method and the EC_HAVE_FLAGS
+ *   definition to check for its existence.
+ * - Added SoE IDN requests, including the datatype ec_soe_request_t and the
+ *   methods ecrt_slave_config_create_soe_request(),
+ *   ecrt_soe_request_object(), ecrt_soe_request_timeout(),
+ *   ecrt_soe_request_data(), ecrt_soe_request_data_size(),
+ *   ecrt_soe_request_state(), ecrt_soe_request_write() and
+ *   ecrt_soe_request_read(). Use the EC_HAVE_SOE_REQUESTS to check, if the
+ *   functionality is available.
+ *
+ * Changes in version 1.5.2:
+ *
+ * - Added redundancy_active flag to ec_domain_state_t.
+ * - Added ecrt_master_link_state() method and ec_master_link_state_t to query
+ *   the state of a redundant link.
+ * - Added the EC_HAVE_REDUNDANCY define, to check, if the interface contains
+ *   redundancy features.
+ * - Added ecrt_sdo_request_index() to change SDO index and subindex after
+ *   request creation.
+ * - Added interface for retrieving CoE emergency messages, i. e.
+ *   ecrt_slave_config_emerg_size(), ecrt_slave_config_emerg_pop(),
+ *   ecrt_slave_config_emerg_clear(), ecrt_slave_config_emerg_overruns() and
+ *   the defines EC_HAVE_EMERGENCY and EC_COE_EMERGENCY_MSG_SIZE.
+ * - Added interface for direct EtherCAT register access: Added data type
+ *   ec_reg_request_t and methods ecrt_slave_config_create_reg_request(),
+ *   ecrt_reg_request_data(), ecrt_reg_request_state(),
+ *   ecrt_reg_request_write(), ecrt_reg_request_read() and the feature flag
+ *   EC_HAVE_REG_ACCESS.
+ * - Added method to select the reference clock,
+ *   ecrt_master_select_reference_clock() and the feature flag
+ *   EC_HAVE_SELECT_REF_CLOCK to check, if the method is available.
+ * - Added method to get the reference clock time,
+ *   ecrt_master_reference_clock_time() and the feature flag
+ *   EC_HAVE_REF_CLOCK_TIME to have the possibility to synchronize the master
+ *   clock to the reference clock.
+ * - Changed the data types of the shift times in ecrt_slave_config_dc() to
+ *   int32_t to correctly display negative shift times.
+ * - Added ecrt_slave_config_reg_pdo_entry_pos() and the feature flag
+ *   EC_HAVE_REG_BY_POS for registering PDO entries with non-unique indices
+ *   via their positions in the mapping.
+ *
+ * Changes in version 1.5:
+ *
+ * - Added the distributed clocks feature and the respective method
+ *   ecrt_slave_config_dc() to configure a slave for cyclic operation, and
+ *   ecrt_master_application_time(), ecrt_master_sync_reference_clock() and
+ *   ecrt_master_sync_slave_clocks() for offset and drift compensation. The
+ *   EC_TIMEVAL2NANO() macro can be used for epoch time conversion, while the
+ *   ecrt_master_sync_monitor_queue() and ecrt_master_sync_monitor_process()
+ *   methods can be used to monitor the synchrony.
+ * - Improved the callback mechanism. ecrt_master_callbacks() now takes two
+ *   callback functions for sending and receiving datagrams.
+ *   ecrt_master_send_ext() is used to execute the sending of non-application
+ *   datagrams.
+ * - Added watchdog configuration (method ecrt_slave_config_watchdog(),
+ *   #ec_watchdog_mode_t, \a watchdog_mode parameter in ec_sync_info_t and
+ *   ecrt_slave_config_sync_manager()).
+ * - Added ecrt_slave_config_complete_sdo() method to download an SDO during
+ *   configuration via CompleteAccess.
+ * - Added ecrt_master_deactivate() to remove the master configuration.
+ * - Added ecrt_open_master() and ecrt_master_reserve() separation for
+ *   userspace.
+ * - Added master information interface (methods ecrt_master(),
+ *   ecrt_master_get_slave(), ecrt_master_get_sync_manager(),
+ *   ecrt_master_get_pdo() and ecrt_master_get_pdo_entry()) to get information
+ *   about the currently connected slaves and the PDO entries provided.
+ * - Added ecrt_master_sdo_download(), ecrt_master_sdo_download_complete() and
+ *   ecrt_master_sdo_upload() methods to let an application transfer SDOs
+ *   before activating the master.
+ * - Changed the meaning of the negative return values of
+ *   ecrt_slave_config_reg_pdo_entry() and ecrt_slave_config_sdo*().
+ * - Implemented the Vendor-specific over EtherCAT mailbox protocol. See
+ *   ecrt_slave_config_create_voe_handler().
+ * - Renamed ec_sdo_request_state_t to #ec_request_state_t, because it is also
+ *   used by VoE handlers.
+ * - Removed 'const' from argument of ecrt_sdo_request_state(), because the
+ *   userspace library has to modify object internals.
+ * - Added 64-bit data access macros.
+ * - Added ecrt_slave_config_idn() method for storing SoE IDN configurations,
+ *   and ecrt_master_read_idn() and ecrt_master_write_idn() to read/write IDNs
+ *   ad-hoc via the user-space library.
+ * - Added ecrt_master_reset() to initiate retrying to configure slaves.
+ *
+ * @{
+ */
+
+/****************************************************************************/
+
+#ifndef __ECRT_H__
+#define __ECRT_H__
+
+#ifdef __KERNEL__
+#include <asm/byteorder.h>
+#include <linux/types.h>
+#include <linux/time.h>
+#include <linux/in.h> // struct in_addr
+#else
+#include <stdlib.h> // for size_t
+#include <stdint.h>
+#include <sys/time.h> // for struct timeval
+#include <netinet/in.h> // struct in_addr
+#endif
+
+/*****************************************************************************
+ * Global definitions
+ ****************************************************************************/
+
+/** EtherCAT realtime interface major version number.
+ */
+#define ECRT_VER_MAJOR 1
+
+/** EtherCAT realtime interface minor version number.
+ */
+#define ECRT_VER_MINOR 6
+
+/** EtherCAT realtime interface version word generator.
+ */
+#define ECRT_VERSION(a, b) (((a) << 8) + (b))
+
+/** EtherCAT realtime interface version word.
+ */
+#define ECRT_VERSION_MAGIC ECRT_VERSION(ECRT_VER_MAJOR, ECRT_VER_MINOR)
+
+/*****************************************************************************
+ * Feature flags
+ ****************************************************************************/
+
+/** Defined, if the redundancy features are available.
+ *
+ * I. e. if the \a redundancy_active flag in ec_domain_state_t and the
+ * ecrt_master_link_state() method are available.
+ */
+#define EC_HAVE_REDUNDANCY
+
+/** Defined, if the CoE emergency ring feature is available.
+ *
+ * I. e. if the ecrt_slave_config_emerg_*() methods are available.
+ */
+#define EC_HAVE_EMERGENCY
+
+/** Defined, if the register access interface is available.
+ *
+ * I. e. if the methods ecrt_slave_config_create_reg_request(),
+ * ecrt_reg_request_data(), ecrt_reg_request_state(), ecrt_reg_request_write()
+ * and ecrt_reg_request_read() are available.
+ */
+#define EC_HAVE_REG_ACCESS
+
+/** Defined if the method ecrt_master_select_reference_clock() is available.
+ */
+#define EC_HAVE_SELECT_REF_CLOCK
+
+/** Defined if the method ecrt_master_reference_clock_time() is available.
+ */
+#define EC_HAVE_REF_CLOCK_TIME
+
+/** Defined if the method ecrt_slave_config_reg_pdo_entry_pos() is available.
+ */
+#define EC_HAVE_REG_BY_POS
+
+/** Defined if the method ecrt_master_sync_reference_clock_to() is available.
+ */
+#define EC_HAVE_SYNC_TO
+
+/** Defined if the method ecrt_slave_config_flag() is available.
+ */
+#define EC_HAVE_FLAGS
+
+/** Defined if the methods ecrt_slave_config_create_soe_request(),
+ * ecrt_soe_request_object(), ecrt_soe_request_timeout(),
+ * ecrt_soe_request_data(), ecrt_soe_request_data_size(),
+ * ecrt_soe_request_state(), ecrt_soe_request_write() and
+ * ecrt_soe_request_read() and the datatype ec_soe_request_t are available.
+ */
+#define EC_HAVE_SOE_REQUESTS
+
+/** Defined, if the method ecrt_master_scan_progress() and the
+ * ec_master_scan_progress_t structure are available.
+ */
+#define EC_HAVE_SCAN_PROGRESS
+
+/** Defined, if the methods ecrt_slave_config_eoe_mac_address(),
+ * ecrt_slave_config_eoe_ip_address(), ecrt_slave_config_eoe_subnet_mask(),
+ * ecrt_slave_config_eoe_default_gateway(),
+ * ecrt_slave_config_eoe_dns_address(), ecrt_slave_config_eoe_hostname() are
+ * available.
+ */
+#define EC_HAVE_SET_IP
+
+/** Defined, if the method ecrt_slave_config_state_timeout() is available.
+ */
+#define EC_HAVE_STATE_TIMEOUT
+
+/****************************************************************************/
+
+/** Symbol visibility control macro.
+ */
+#ifndef EC_PUBLIC_API
+# if defined(ethercat_EXPORTS) && !defined(__KERNEL__)
+#  define EC_PUBLIC_API __attribute__ ((visibility ("default")))
+# else
+#  define EC_PUBLIC_API
+# endif
+#endif
+
+/****************************************************************************/
+
+/** End of list marker.
+ *
+ * This can be used with ecrt_slave_config_pdos().
+ */
+#define EC_END ~0U
+
+/** Maximum number of sync managers per slave.
+ */
+#define EC_MAX_SYNC_MANAGERS 16
+
+/** Maximum string length.
+ *
+ * Used in ec_slave_info_t.
+ */
+#define EC_MAX_STRING_LENGTH 64
+
+/** Maximum number of slave ports. */
+#define EC_MAX_PORTS 4
+
+/** Timeval to nanoseconds conversion.
+ *
+ * This macro converts a Unix epoch time to EtherCAT DC time.
+ *
+ * \see void ecrt_master_application_time()
+ *
+ * \param TV struct timeval containing epoch time.
+ */
+#define EC_TIMEVAL2NANO(TV) \
+    (((TV).tv_sec - 946684800ULL) * 1000000000ULL + (TV).tv_usec * 1000ULL)
+
+/** Size of a CoE emergency message in byte.
+ *
+ * \see ecrt_slave_config_emerg_pop().
+ */
+#define EC_COE_EMERGENCY_MSG_SIZE 8
+
+/*****************************************************************************
+ * Data types
+ ****************************************************************************/
+
+struct ec_master;
+typedef struct ec_master ec_master_t; /**< \see ec_master */
+
+struct ec_slave_config;
+typedef struct ec_slave_config ec_slave_config_t; /**< \see ec_slave_config */
+
+struct ec_domain;
+typedef struct ec_domain ec_domain_t; /**< \see ec_domain */
+
+struct ec_sdo_request;
+typedef struct ec_sdo_request ec_sdo_request_t; /**< \see ec_sdo_request. */
+
+struct ec_soe_request;
+typedef struct ec_soe_request ec_soe_request_t; /**< \see ec_soe_request. */
+
+struct ec_voe_handler;
+typedef struct ec_voe_handler ec_voe_handler_t; /**< \see ec_voe_handler. */
+
+struct ec_reg_request;
+typedef struct ec_reg_request ec_reg_request_t; /**< \see ec_reg_request. */
+
+/****************************************************************************/
+
+/** Master state.
+ *
+ * This is used for the output parameter of ecrt_master_state().
+ *
+ * \see ecrt_master_state().
+ */
+typedef struct {
+    unsigned int slaves_responding; /**< Sum of responding slaves on all
+                                      Ethernet devices. */
+    unsigned int al_states : 4; /**< Application-layer states of all slaves.
+                                  The states are coded in the lower 4 bits.
+                                  If a bit is set, it means that at least one
+                                  slave in the network is in the corresponding
+                                  state:
+                                  - Bit 0: \a INIT
+                                  - Bit 1: \a PREOP
+                                  - Bit 2: \a SAFEOP
+                                  - Bit 3: \a OP */
+    unsigned int link_up : 1; /**< \a true, if at least one Ethernet link is
+                                up. */
+} ec_master_state_t;
+
+/****************************************************************************/
+
+/** Redundant link state.
+ *
+ * This is used for the output parameter of ecrt_master_link_state().
+ *
+ * \see ecrt_master_link_state().
+ */
+typedef struct {
+    unsigned int slaves_responding; /**< Sum of responding slaves on the given
+                                      link. */
+    unsigned int al_states : 4; /**< Application-layer states of the slaves on
+                                  the given link. The states are coded in the
+                                  lower 4 bits. If a bit is set, it means
+                                  that at least one slave in the network is in
+                                  the corresponding state:
+                                  - Bit 0: \a INIT
+                                  - Bit 1: \a PREOP
+                                  - Bit 2: \a SAFEOP
+                                  - Bit 3: \a OP */
+    unsigned int link_up : 1; /**< \a true, if the given Ethernet link is up.
+                               */
+} ec_master_link_state_t;
+
+/****************************************************************************/
+
+/** Slave configuration state.
+ *
+ * This is used as an output parameter of ecrt_slave_config_state().
+ *
+ * \see ecrt_slave_config_state().
+ */
+typedef struct  {
+    unsigned int online : 1; /**< The slave is online. */
+    unsigned int operational : 1; /**< The slave was brought into \a OP state
+                                    using the specified configuration. */
+    unsigned int al_state : 4; /**< The application-layer state of the slave.
+                                 - 1: \a INIT
+                                 - 2: \a PREOP
+                                 - 4: \a SAFEOP
+                                 - 8: \a OP
+
+                                 Note that each state is coded in a different
+                                 bit! */
+} ec_slave_config_state_t;
+
+/****************************************************************************/
+
+/** Master information.
+ *
+ * This is used as an output parameter of ecrt_master().
+ *
+ * \see ecrt_master().
+ */
+typedef struct {
+    unsigned int slave_count; /**< Number of slaves in the network. */
+    unsigned int link_up : 1; /**< \a true, if the network link is up. */
+    uint8_t scan_busy; /**< \a true, while the master is scanning the network.
+                        */
+    uint64_t app_time; /**< Application time. */
+} ec_master_info_t;
+
+/****************************************************************************/
+
+/** Master scan progress information.
+ *
+ * This is used as an output parameter of ecrt_master_scan_progress().
+ *
+ * \see ecrt_master_scan_progress().
+ */
+typedef struct {
+    unsigned int slave_count; /**< Number of slaves detected. */
+    unsigned int scan_index; /**< Index of the slave that is currently
+                               scanned.  If it is less than the \a
+                               slave_count, the network scan is in progress.
+                              */
+} ec_master_scan_progress_t;
+
+/****************************************************************************/
+
+/** EtherCAT slave port descriptor.
+ */
+typedef enum {
+    EC_PORT_NOT_IMPLEMENTED, /**< Port is not implemented. */
+    EC_PORT_NOT_CONFIGURED, /**< Port is not configured. */
+    EC_PORT_EBUS, /**< Port is an E-Bus. */
+    EC_PORT_MII /**< Port is a MII. */
+} ec_slave_port_desc_t;
+
+/****************************************************************************/
+
+/** EtherCAT slave port information.
+ */
+typedef struct {
+    uint8_t link_up; /**< Link detected. */
+    uint8_t loop_closed; /**< Loop closed. */
+    uint8_t signal_detected; /**< Detected signal on RX port. */
+} ec_slave_port_link_t;
+
+/****************************************************************************/
+
+/** Slave information.
+ *
+ * This is used as an output parameter of ecrt_master_get_slave().
+ *
+ * \see ecrt_master_get_slave().
+ */
+typedef struct {
+    uint16_t position; /**< Offset of the slave in the ring. */
+    uint32_t vendor_id; /**< Vendor-ID stored on the slave. */
+    uint32_t product_code; /**< Product-Code stored on the slave. */
+    uint32_t revision_number; /**< Revision-Number stored on the slave. */
+    uint32_t serial_number; /**< Serial-Number stored on the slave. */
+    uint16_t alias; /**< The slaves alias if not equal to 0. */
+    int16_t current_on_ebus; /**< Used current in mA. */
+    struct {
+        ec_slave_port_desc_t desc; /**< Physical port type. */
+        ec_slave_port_link_t link; /**< Port link state. */
+        uint32_t receive_time; /**< Receive time on DC transmission delay
+                                 measurement. */
+        uint16_t next_slave; /**< Ring position of next DC slave on that
+                               port.  */
+        uint32_t delay_to_next_dc; /**< Delay [ns] to next DC slave. */
+    } ports[EC_MAX_PORTS]; /**< Port information. */
+    uint8_t al_state; /**< Current state of the slave. */
+    uint8_t error_flag; /**< Error flag for that slave. */
+    uint8_t sync_count; /**< Number of sync managers. */
+    uint16_t sdo_count; /**< Number of SDOs. */
+    char name[EC_MAX_STRING_LENGTH]; /**< Name of the slave. */
+} ec_slave_info_t;
+
+/****************************************************************************/
+
+/** Domain working counter interpretation.
+ *
+ * This is used in ec_domain_state_t.
+ */
+typedef enum {
+    EC_WC_ZERO = 0,   /**< No registered process data were exchanged. */
+    EC_WC_INCOMPLETE, /**< Some of the registered process data were
+                        exchanged. */
+    EC_WC_COMPLETE    /**< All registered process data were exchanged. */
+} ec_wc_state_t;
+
+/****************************************************************************/
+
+/** Domain state.
+ *
+ * This is used for the output parameter of ecrt_domain_state().
+ */
+typedef struct {
+    unsigned int working_counter; /**< Value of the last working counter. */
+    ec_wc_state_t wc_state; /**< Working counter interpretation. */
+    unsigned int redundancy_active; /**< Redundant link is in use. */
+} ec_domain_state_t;
+
+/****************************************************************************/
+
+/** Direction type for PDO assignment functions.
+ */
+typedef enum {
+    EC_DIR_INVALID, /**< Invalid direction. Do not use this value. */
+    EC_DIR_OUTPUT, /**< Values written by the master. */
+    EC_DIR_INPUT, /**< Values read by the master. */
+    EC_DIR_COUNT /**< Number of directions. For internal use only. */
+} ec_direction_t;
+
+/****************************************************************************/
+
+/** Watchdog mode for sync manager configuration.
+ *
+ * Used to specify, if a sync manager's watchdog is to be enabled.
+ */
+typedef enum {
+    EC_WD_DEFAULT, /**< Use the default setting of the sync manager. */
+    EC_WD_ENABLE, /**< Enable the watchdog. */
+    EC_WD_DISABLE, /**< Disable the watchdog. */
+} ec_watchdog_mode_t;
+
+/****************************************************************************/
+
+/** PDO entry configuration information.
+ *
+ * This is the data type of the \a entries field in ec_pdo_info_t.
+ *
+ * \see ecrt_slave_config_pdos().
+ */
+typedef struct {
+    uint16_t index; /**< PDO entry index. */
+    uint8_t subindex; /**< PDO entry subindex. */
+    uint8_t bit_length; /**< Size of the PDO entry in bit. */
+} ec_pdo_entry_info_t;
+
+/****************************************************************************/
+
+/** PDO configuration information.
+ *
+ * This is the data type of the \a pdos field in ec_sync_info_t.
+ *
+ * \see ecrt_slave_config_pdos().
+ */
+typedef struct {
+    uint16_t index; /**< PDO index. */
+    unsigned int n_entries; /**< Number of PDO entries in \a entries to map.
+                              Zero means, that the default mapping shall be
+                              used (this can only be done if the slave is
+                              present at configuration time). */
+    ec_pdo_entry_info_t const *entries; /**< Array of PDO entries to map. Can
+                                          either be \a NULL, or must contain
+                                          at least \a n_entries values. */
+} ec_pdo_info_t;
+
+/****************************************************************************/
+
+/** Sync manager configuration information.
+ *
+ * This can be use to configure multiple sync managers including the PDO
+ * assignment and PDO mapping. It is used as an input parameter type in
+ * ecrt_slave_config_pdos().
+ */
+typedef struct {
+    uint8_t index; /**< Sync manager index. Must be less
+                     than #EC_MAX_SYNC_MANAGERS for a valid sync manager,
+                     but can also be \a 0xff to mark the end of the list. */
+    ec_direction_t dir; /**< Sync manager direction. */
+    unsigned int n_pdos; /**< Number of PDOs in \a pdos. */
+    ec_pdo_info_t const *pdos; /**< Array with PDOs to assign. This must
+                                 contain at least \a n_pdos PDOs. */
+    ec_watchdog_mode_t watchdog_mode; /**< Watchdog mode. */
+} ec_sync_info_t;
+
+/****************************************************************************/
+
+/** List record type for PDO entry mass-registration.
+ *
+ * This type is used for the array parameter of the
+ * ecrt_domain_reg_pdo_entry_list()
+ */
+typedef struct {
+    uint16_t alias; /**< Slave alias address. */
+    uint16_t position; /**< Slave position. */
+    uint32_t vendor_id; /**< Slave vendor ID. */
+    uint32_t product_code; /**< Slave product code. */
+    uint16_t index; /**< PDO entry index. */
+    uint8_t subindex; /**< PDO entry subindex. */
+    unsigned int *offset; /**< Pointer to a variable to store the PDO entry's
+                       (byte-)offset in the process data. */
+    unsigned int *bit_position; /**< Pointer to a variable to store a bit
+                                  position (0-7) within the \a offset. Can be
+                                  NULL, in which case an error is raised if
+                                  the PDO entry does not byte-align. */
+} ec_pdo_entry_reg_t;
+
+/****************************************************************************/
+
+/** Request state.
+ *
+ * This is used as return type for ecrt_sdo_request_state() and
+ * ecrt_voe_handler_state().
+ */
+typedef enum {
+    EC_REQUEST_UNUSED, /**< Not requested. */
+    EC_REQUEST_BUSY, /**< Request is being processed. */
+    EC_REQUEST_SUCCESS, /**< Request was processed successfully. */
+    EC_REQUEST_ERROR, /**< Request processing failed. */
+} ec_request_state_t;
+
+/****************************************************************************/
+
+/** Application-layer state.
+ */
+typedef enum {
+    EC_AL_STATE_INIT = 1, /**< Init. */
+    EC_AL_STATE_PREOP = 2, /**< Pre-operational. */
+    EC_AL_STATE_SAFEOP = 4, /**< Safe-operational. */
+    EC_AL_STATE_OP = 8, /**< Operational. */
+} ec_al_state_t;
+
+/*****************************************************************************
+ * Global functions
+ ****************************************************************************/
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Returns the version magic of the realtime interface.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Value of ECRT_VERSION_MAGIC() at EtherCAT master compile time.
+ */
+EC_PUBLIC_API unsigned int ecrt_version_magic(void);
+
+/** Requests an EtherCAT master for realtime operation.
+ *
+ * Before an application can access an EtherCAT master, it has to reserve one
+ * for exclusive use.
+ *
+ * In userspace, this is a convenience function for ecrt_open_master() and
+ * ecrt_master_reserve().
+ *
+ * This function has to be the first function an application has to call to
+ * use EtherCAT. The function takes the index of the master as its argument.
+ * The first master has index 0, the n-th master has index n - 1. The number
+ * of masters has to be specified when loading the master module.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return Pointer to the reserved master, otherwise \a NULL.
+ */
+EC_PUBLIC_API ec_master_t *ecrt_request_master(
+        unsigned int master_index /**< Index of the master to request. */
+        );
+
+#ifndef __KERNEL__
+
+/** Opens an EtherCAT master for userspace access.
+ *
+ * This function has to be the first function an application has to call to
+ * use EtherCAT. The function takes the index of the master as its argument.
+ * The first master has index 0, the n-th master has index n - 1. The number
+ * of masters has to be specified when loading the master module.
+ *
+ * For convenience, the function ecrt_request_master() can be used.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return Pointer to the opened master, otherwise \a NULL.
+ */
+EC_PUBLIC_API ec_master_t *ecrt_open_master(
+        unsigned int master_index /**< Index of the master to request. */
+        );
+
+#endif // #ifndef __KERNEL__
+
+/** Releases a requested EtherCAT master.
+ *
+ * After use, a master it has to be released to make it available for other
+ * applications.
+ *
+ * This method frees all created data structures. It should not be called in
+ * realtime context.
+ *
+ * If the master was activated, ecrt_master_deactivate() is called internally.
+ *
+ * \apiusage{master_any,blocking}
+ */
+EC_PUBLIC_API void ecrt_release_master(
+        ec_master_t *master /**< EtherCAT master */
+        );
+
+/*****************************************************************************
+ * Master methods
+ ****************************************************************************/
+
+#ifndef __KERNEL__
+
+/** Reserves an EtherCAT master for realtime operation.
+ *
+ * Before an application can use PDO/domain registration functions or SDO
+ * request functions on the master, it has to reserve one for exclusive use.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return 0 in case of success, else < 0
+ */
+EC_PUBLIC_API int ecrt_master_reserve(
+        ec_master_t *master /**< EtherCAT master */
+        );
+
+#endif // #ifndef __KERNEL__
+
+#ifdef __KERNEL__
+
+/** Sets the locking callbacks.
+ *
+ * For concurrent master access, i. e. if other instances than the application
+ * want to send and receive datagrams on the network, the application has to
+ * provide a callback mechanism. This method takes two function pointers as
+ * its parameters. Asynchronous master access (like EoE processing) is only
+ * possible if the callbacks have been set.
+ *
+ * The task of the send callback (\a send_cb) is to decide, if the network
+ * hardware is currently accessible and whether or not to call the
+ * ecrt_master_send_ext() method.
+ *
+ * The task of the receive callback (\a receive_cb) is to decide, if a call to
+ * ecrt_master_receive() is allowed and to execute it respectively.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \attention This method has to be called before ecrt_master_activate().
+ */
+void ecrt_master_callbacks(
+        ec_master_t *master, /**< EtherCAT master */
+        void (*send_cb)(void *), /**< Datagram sending callback. */
+        void (*receive_cb)(void *), /**< Receive callback. */
+        void *cb_data /**< Arbitrary pointer passed to the callback functions.
+                       */
+        );
+
+#endif /* __KERNEL__ */
+
+/** Creates a new process data domain.
+ *
+ * For process data exchange, at least one process data domain is needed.
+ * This method creates a new process data domain and returns a pointer to the
+ * new domain object. This object can be used for registering PDOs and
+ * exchanging them in cyclic operation.
+ *
+ * This method allocates memory and should be called in non-realtime context
+ * before ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return Pointer to the new domain on success, else NULL.
+ */
+EC_PUBLIC_API ec_domain_t *ecrt_master_create_domain(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Obtains a slave configuration.
+ *
+ * Creates a slave configuration object for the given \a alias and \a position
+ * tuple and returns it. If a configuration with the same \a alias and \a
+ * position already exists, it will be re-used. In the latter case, the given
+ * vendor ID and product code are compared to the stored ones. On mismatch, an
+ * error message is raised and the function returns \a NULL.
+ *
+ * Slaves are addressed with the \a alias and \a position parameters.
+ * - If \a alias is zero, \a position is interpreted as the desired slave's
+ *   ring position.
+ * - If \a alias is non-zero, it matches a slave with the given alias. In this
+ *   case, \a position is interpreted as ring offset, starting from the
+ *   aliased slave, so a position of zero means the aliased slave itself and a
+ *   positive value matches the n-th slave behind the aliased one.
+ *
+ * If the slave with the given address is found during the configuration,
+ * its vendor ID and product code are matched against the given value. On
+ * mismatch, the slave is not configured and an error message is raised.
+ *
+ * If different slave configurations are pointing to the same slave during
+ * configuration, a warning is raised and only the first configuration is
+ * applied.
+ *
+ * This method allocates memory and should be called in non-realtime context
+ * before ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval >0 Pointer to the slave configuration structure.
+ * \retval NULL in the error case.
+ */
+EC_PUBLIC_API ec_slave_config_t *ecrt_master_slave_config(
+        ec_master_t *master, /**< EtherCAT master */
+        uint16_t alias, /**< Slave alias. */
+        uint16_t position, /**< Slave position. */
+        uint32_t vendor_id, /**< Expected vendor ID. */
+        uint32_t product_code /**< Expected product code. */
+        );
+
+/** Selects the reference clock for distributed clocks.
+ *
+ * If this method is not called for a certain master, or if the slave
+ * configuration pointer is NULL, then the first slave with DC functionality
+ * will provide the reference clock.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_select_reference_clock(
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_slave_config_t *sc /**< Slave config of the slave to use as the
+                               * reference slave (or NULL). */
+        );
+
+/** Obtains master information.
+ *
+ * No memory is allocated on the heap in this function.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \attention The pointer to this structure must point to a valid variable.
+ *
+ * \return 0 in case of success, else < 0
+ */
+EC_PUBLIC_API int ecrt_master(
+        ec_master_t *master, /**< EtherCAT master */
+        ec_master_info_t *master_info /**< Structure that will output the
+                                        information */
+        );
+
+/** Obtains network scan progress information.
+ *
+ * No memory is allocated on the heap in this function.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \attention The pointer to this structure must point to a valid variable.
+ *
+ * \return 0 in case of success, else < 0
+ */
+EC_PUBLIC_API int ecrt_master_scan_progress(
+        ec_master_t *master, /**< EtherCAT master */
+        ec_master_scan_progress_t *progress /**< Structure that will output
+                                              the progress information. */
+        );
+
+/** Obtains slave information.
+ *
+ * Tries to find the slave with the given ring position. The obtained
+ * information is stored in a structure. No memory is allocated on the heap in
+ * this function.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \attention The pointer to this structure must point to a valid variable.
+ *
+ * \return 0 in case of success, else < 0
+ */
+EC_PUBLIC_API int ecrt_master_get_slave(
+        ec_master_t *master, /**< EtherCAT master */
+        uint16_t slave_position, /**< Slave position. */
+        ec_slave_info_t *slave_info /**< Structure that will output the
+                                      information */
+        );
+
+#ifndef __KERNEL__
+
+/** Returns the proposed configuration of a slave's sync manager.
+ *
+ * Fills a given ec_sync_info_t structure with the attributes of a sync
+ * manager. The \a pdos field of the return value is left empty. Use
+ * ecrt_master_get_pdo() to get the PDO information.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \return zero on success, else non-zero
+ */
+EC_PUBLIC_API int ecrt_master_get_sync_manager(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint8_t sync_index, /**< Sync manager index. Must be less
+                                than #EC_MAX_SYNC_MANAGERS. */
+        ec_sync_info_t *sync /**< Pointer to output structure. */
+        );
+
+/** Returns information about a currently assigned PDO.
+ *
+ * Fills a given ec_pdo_info_t structure with the attributes of a currently
+ * assigned PDO of the given sync manager. The \a entries field of the return
+ * value is left empty. Use ecrt_master_get_pdo_entry() to get the PDO
+ * entry information.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \retval zero on success, else non-zero
+ */
+EC_PUBLIC_API int ecrt_master_get_pdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint8_t sync_index, /**< Sync manager index. Must be less
+                                 than #EC_MAX_SYNC_MANAGERS. */
+        uint16_t pos, /**< Zero-based PDO position. */
+        ec_pdo_info_t *pdo /**< Pointer to output structure. */
+        );
+
+/** Returns information about a currently mapped PDO entry.
+ *
+ * Fills a given ec_pdo_entry_info_t structure with the attributes of a
+ * currently mapped PDO entry of the given PDO.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \retval zero on success, else non-zero
+ */
+EC_PUBLIC_API int ecrt_master_get_pdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint8_t sync_index, /**< Sync manager index. Must be less
+                                 than #EC_MAX_SYNC_MANAGERS. */
+        uint16_t pdo_pos, /**< Zero-based PDO position. */
+        uint16_t entry_pos, /**< Zero-based PDO entry position. */
+        ec_pdo_entry_info_t *entry /**< Pointer to output structure. */
+        );
+
+#endif /* #ifndef __KERNEL__ */
+
+/** Executes an SDO download request to write data to a slave.
+ *
+ * This request is processed by the master state machine. This method blocks,
+ * until the request has been processed and may not be called in realtime
+ * context.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_master_sdo_download(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint16_t index, /**< Index of the SDO. */
+        uint8_t subindex, /**< Subindex of the SDO. */
+        const uint8_t *data, /**< Data buffer to download. */
+        size_t data_size, /**< Size of the data buffer. */
+        uint32_t *abort_code /**< Abort code of the SDO download. */
+        );
+
+/** Executes an SDO download request to write data to a slave via complete
+ * access.
+ *
+ * This request is processed by the master state machine. This method blocks,
+ * until the request has been processed and may not be called in realtime
+ * context.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_master_sdo_download_complete(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint16_t index, /**< Index of the SDO. */
+        const uint8_t *data, /**< Data buffer to download. */
+        size_t data_size, /**< Size of the data buffer. */
+        uint32_t *abort_code /**< Abort code of the SDO download. */
+        );
+
+/** Executes an SDO upload request to read data from a slave.
+ *
+ * This request is processed by the master state machine. This method blocks,
+ * until the request has been processed and may not be called in realtime
+ * context.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_master_sdo_upload(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint16_t index, /**< Index of the SDO. */
+        uint8_t subindex, /**< Subindex of the SDO. */
+        uint8_t *target, /**< Target buffer for the upload. */
+        size_t target_size, /**< Size of the target buffer. */
+        size_t *result_size, /**< Uploaded data size. */
+        uint32_t *abort_code /**< Abort code of the SDO upload. */
+        );
+
+/** Executes an SoE write request.
+ *
+ * Starts writing an IDN and blocks until the request was processed, or an
+ * error occurred.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_master_write_idn(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint8_t drive_no, /**< Drive number. */
+        uint16_t idn, /**< SoE IDN (see ecrt_slave_config_idn()). */
+        const uint8_t *data, /**< Pointer to data to write. */
+        size_t data_size, /**< Size of data to write. */
+        uint16_t *error_code /**< Pointer to variable, where an SoE error code
+                               can be stored. */
+        );
+
+/** Executes an SoE read request.
+ *
+ * Starts reading an IDN and blocks until the request was processed, or an
+ * error occurred.
+ *
+ * \apiusage{master_any,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_master_read_idn(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t slave_position, /**< Slave position. */
+        uint8_t drive_no, /**< Drive number. */
+        uint16_t idn, /**< SoE IDN (see ecrt_slave_config_idn()). */
+        uint8_t *target, /**< Pointer to memory where the read data can be
+                           stored. */
+        size_t target_size, /**< Size of the memory \a target points to. */
+        size_t *result_size, /**< Actual size of the received data. */
+        uint16_t *error_code /**< Pointer to variable, where an SoE error code
+                               can be stored. */
+        );
+
+/** Finishes the configuration phase and prepares for cyclic operation.
+ *
+ * This function tells the master that the configuration phase is finished and
+ * the realtime operation will begin. The function allocates internal memory
+ * for the domains and calculates the logical FMMU addresses for domain
+ * members. It tells the master state machine that the configuration is
+ * now to be applied to the network.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \attention After this function has been called, the realtime application is
+ * in charge of cyclically calling ecrt_master_send() and
+ * ecrt_master_receive() to ensure network communication. Before calling this
+ * function, the master thread is responsible for that, so these functions may
+ * not be called! The method itself allocates memory and should not be called
+ * in realtime context.
+ *
+ * \return 0 in case of success, else < 0
+ */
+EC_PUBLIC_API int ecrt_master_activate(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Deactivates the master.
+ *
+ * Removes the master configuration. All objects created by
+ * ecrt_master_create_domain(), ecrt_master_slave_config(), ecrt_domain_data()
+ * ecrt_slave_config_create_sdo_request() and
+ * ecrt_slave_config_create_voe_handler() are freed, so pointers to them
+ * become invalid.
+ *
+ * \apiusage{master_op,blocking}
+ *
+ * This method should not be called in realtime context.
+ * \return 0 on success, otherwise negative error code.
+ * \retval 0 Success.
+ * \retval -EINVAL Master has not been activated before.
+ */
+EC_PUBLIC_API int ecrt_master_deactivate(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Set interval between calls to ecrt_master_send().
+ *
+ * This information helps the master to decide, how much data can be appended
+ * to a frame by the master state machine. When the master is configured with
+ * --enable-hrtimers, this is used to calculate the scheduling of the master
+ * thread.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval 0 on success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_master_set_send_interval(
+        ec_master_t *master, /**< EtherCAT master. */
+        size_t send_interval /**< Send interval in us */
+        );
+
+/** Sends all datagrams in the queue.
+ *
+ * This method takes all datagrams, that have been queued for transmission,
+ * puts them into frames, and passes them to the Ethernet device for sending.
+ *
+ * Has to be called cyclically by the application after ecrt_master_activate()
+ * has returned.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_send(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Fetches received frames from the hardware and processes the datagrams.
+ *
+ * Queries the network device for received frames by calling the interrupt
+ * service routine. Extracts received datagrams and dispatches the results to
+ * the datagram objects in the queue. Received datagrams, and the ones that
+ * timed out, will be marked, and dequeued.
+ *
+ * Has to be called cyclically by the realtime application after
+ * ecrt_master_activate() has returned.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_receive(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+#ifdef __KERNEL__
+/** Sends non-application datagrams.
+ *
+ * This method has to be called in the send callback function passed via
+ * ecrt_master_callbacks() to allow the sending of non-application datagrams.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ * \retval -EAGAIN Lock could not be acquired, try again later.
+ */
+int ecrt_master_send_ext(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+#endif
+
+/** Reads the current master state.
+ *
+ * Stores the master state information in the given \a state structure.
+ *
+ * This method returns a global state. For the link-specific states in a
+ * redundant network topology, use the ecrt_master_link_state() method.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_state(
+        const ec_master_t *master, /**< EtherCAT master. */
+        ec_master_state_t *state /**< Structure to store the information. */
+        );
+
+/** Reads the current state of a redundant link.
+ *
+ * Stores the link state information in the given \a state structure.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_link_state(
+        const ec_master_t *master, /**< EtherCAT master. */
+        unsigned int dev_idx, /**< Index of the device (0 = main device, 1 =
+                                first backup device, ...). */
+        ec_master_link_state_t *state /**< Structure to store the information.
+                                       */
+        );
+
+/** Sets the application time.
+ *
+ * The master has to know the application's time when operating slaves with
+ * distributed clocks. The time is not incremented by the master itself, so
+ * this method has to be called cyclically.
+ *
+ * \attention The time passed to this method is used to calculate the phase of
+ * the slaves' SYNC0/1 interrupts. It should be called constantly at the same
+ * point of the realtime cycle. So it is recommended to call it at the start
+ * of the calculations to avoid deviancies due to changing execution times.
+ * Avoid calling this method before the realtime cycle is established.
+ *
+ * The time is used when setting the slaves' <tt>System Time Offset</tt> and
+ * <tt>Cyclic Operation Start Time</tt> registers and when synchronizing the
+ * DC reference clock to the application time via
+ * ecrt_master_sync_reference_clock().
+ *
+ * The time is defined as nanoseconds from 2000-01-01 00:00. Converting an
+ * epoch time can be done with the EC_TIMEVAL2NANO() macro, but is not
+ * necessary, since the absolute value is not of any interest.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_application_time(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint64_t app_time /**< Application time. */
+        );
+
+/** Queues the DC reference clock drift compensation datagram for sending.
+ *
+ * The reference clock will by synchronized to the application time provided
+ * by the last call off ecrt_master_application_time().
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ * \retval 0 Success.
+ * \retval -ENXIO No reference clock found.
+ */
+EC_PUBLIC_API int ecrt_master_sync_reference_clock(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Queues the DC reference clock drift compensation datagram for sending.
+ *
+ * The reference clock will by synchronized to the time passed in the
+ * sync_time parameter.
+ *
+ * Has to be called by the application after ecrt_master_activate()
+ * has returned.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Zero on success, otherwise negative error code.
+ * \retval 0 Success.
+ * \retval -ENXIO No reference clock found.
+ */
+EC_PUBLIC_API int ecrt_master_sync_reference_clock_to(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint64_t sync_time /**< Sync reference clock to this time. */
+        );
+
+/** Queues the DC clock drift compensation datagram for sending.
+ *
+ * All slave clocks synchronized to the reference clock.
+ *
+ * Has to be called by the application after ecrt_master_activate()
+ * has returned.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ * \retval 0 Success.
+ * \retval -ENXIO No reference clock found.
+ */
+EC_PUBLIC_API int ecrt_master_sync_slave_clocks(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Get the lower 32 bit of the reference clock system time.
+ *
+ * This method can be used to synchronize the master to the reference clock.
+ *
+ * The reference clock system time is queried via the
+ * ecrt_master_sync_slave_clocks() method, that reads the system time of the
+ * reference clock and writes it to the slave clocks (so be sure to call it
+ * cyclically to get valid data).
+ *
+ * \attention The returned time is the system time of the reference clock
+ * minus the transmission delay of the reference clock.
+ *
+ * Calling this method makes only sense in realtime context (after master
+ * activation), when the ecrt_master_sync_slave_clocks() method is called
+ * cyclically.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \retval 0 success, system time was written into \a time.
+ * \retval -ENXIO No reference clock found.
+ * \retval -EIO Slave synchronization datagram was not received.
+ */
+EC_PUBLIC_API int ecrt_master_reference_clock_time(
+        const ec_master_t *master, /**< EtherCAT master. */
+        uint32_t *time /**< Pointer to store the queried system time. */
+        );
+
+/** Queues the DC synchrony monitoring datagram for sending.
+ *
+ * The datagram broadcast-reads all "System time difference" registers (\a
+ * 0x092c) to get an upper estimation of the DC synchrony. The result can be
+ * checked with the ecrt_master_sync_monitor_process() method.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_sync_monitor_queue(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Processes the DC synchrony monitoring datagram.
+ *
+ * If the sync monitoring datagram was sent before with
+ * ecrt_master_sync_monitor_queue(), the result can be queried with this
+ * method.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Upper estimation of the maximum time difference in ns, -1 on error.
+ * \retval (uint32_t)-1 Error.
+ */
+EC_PUBLIC_API uint32_t ecrt_master_sync_monitor_process(
+        const ec_master_t *master /**< EtherCAT master. */
+        );
+
+/** Retry configuring slaves.
+ *
+ * Via this method, the application can tell the master to bring all slaves to
+ * OP state. In general, this is not necessary, because it is automatically
+ * done by the master. But with special slaves, that can be reconfigured by
+ * the vendor during runtime, it can be useful.
+ *
+ * Calling this method only makes sense in realtime context (after
+ * activation), because slaves will not be configured before.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_master_reset(
+        ec_master_t *master /**< EtherCAT master. */
+        );
+
+/*****************************************************************************
+ * Slave configuration methods
+ ****************************************************************************/
+
+/** Configure a sync manager.
+ *
+ * Sets the direction of a sync manager. This overrides the direction bits
+ * from the default control register from SII.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return zero on success, else non-zero
+ */
+EC_PUBLIC_API int ecrt_slave_config_sync_manager(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint8_t sync_index, /**< Sync manager index. Must be less
+                              than #EC_MAX_SYNC_MANAGERS. */
+        ec_direction_t direction, /**< Input/Output. */
+        ec_watchdog_mode_t watchdog_mode /** Watchdog mode. */
+        );
+
+/** Configure a slave's watchdog times.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_watchdog(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t watchdog_divider, /**< Number of 40 ns intervals (register
+                                     0x0400). Used as a base unit for all
+                                     slave watchdogs^. If set to zero, the
+                                     value is not written, so the default is
+                                     used. */
+        uint16_t watchdog_intervals /**< Number of base intervals for sync
+                                      manager watchdog (register 0x0420). If
+                                      set to zero, the value is not written,
+                                      so the default is used. */
+        );
+
+/** Add a PDO to a sync manager's PDO assignment.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \see ecrt_slave_config_pdos()
+ * \return zero on success, else non-zero
+ */
+EC_PUBLIC_API int ecrt_slave_config_pdo_assign_add(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint8_t sync_index, /**< Sync manager index. Must be less
+                              than #EC_MAX_SYNC_MANAGERS. */
+        uint16_t index /**< Index of the PDO to assign. */
+        );
+
+/** Clear a sync manager's PDO assignment.
+ *
+ * This can be called before assigning PDOs via
+ * ecrt_slave_config_pdo_assign_add(), to clear the default assignment of a
+ * sync manager.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \see ecrt_slave_config_pdos()
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_pdo_assign_clear(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint8_t sync_index /**< Sync manager index. Must be less
+                              than #EC_MAX_SYNC_MANAGERS. */
+        );
+
+/** Add a PDO entry to the given PDO's mapping.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \see ecrt_slave_config_pdos()
+ * \return zero on success, else non-zero
+ */
+EC_PUBLIC_API int ecrt_slave_config_pdo_mapping_add(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t pdo_index, /**< Index of the PDO. */
+        uint16_t entry_index, /**< Index of the PDO entry to add to the PDO's
+                                mapping. */
+        uint8_t entry_subindex, /**< Subindex of the PDO entry to add to the
+                                  PDO's mapping. */
+        uint8_t entry_bit_length /**< Size of the PDO entry in bit. */
+        );
+
+/** Clear the mapping of a given PDO.
+ *
+ * This can be called before mapping PDO entries via
+ * ecrt_slave_config_pdo_mapping_add(), to clear the default mapping.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \see ecrt_slave_config_pdos()
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_pdo_mapping_clear(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t pdo_index /**< Index of the PDO. */
+        );
+
+/** Specify a complete PDO configuration.
+ *
+ * This function is a convenience wrapper for the functions
+ * ecrt_slave_config_sync_manager(), ecrt_slave_config_pdo_assign_clear(),
+ * ecrt_slave_config_pdo_assign_add(), ecrt_slave_config_pdo_mapping_clear()
+ * and ecrt_slave_config_pdo_mapping_add(), that are better suitable for
+ * automatic code generation.
+ *
+ * The following example shows, how to specify a complete configuration,
+ * including the PDO mappings. With this information, the master is able to
+ * reserve the complete process data, even if the slave is not present at
+ * configuration time:
+ *
+ * \code
+ * ec_pdo_entry_info_t el3162_channel1[] = {
+ *     {0x3101, 1,  8}, // status
+ *     {0x3101, 2, 16}  // value
+ * };
+ *
+ * ec_pdo_entry_info_t el3162_channel2[] = {
+ *     {0x3102, 1,  8}, // status
+ *     {0x3102, 2, 16}  // value
+ * };
+ *
+ * ec_pdo_info_t el3162_pdos[] = {
+ *     {0x1A00, 2, el3162_channel1},
+ *     {0x1A01, 2, el3162_channel2}
+ * };
+ *
+ * ec_sync_info_t el3162_syncs[] = {
+ *     {2, EC_DIR_OUTPUT},
+ *     {3, EC_DIR_INPUT, 2, el3162_pdos},
+ *     {0xff}
+ * };
+ *
+ * if (ecrt_slave_config_pdos(sc_ana_in, EC_END, el3162_syncs)) {
+ *     // handle error
+ * }
+ * \endcode
+ *
+ * The next example shows, how to configure the PDO assignment only. The
+ * entries for each assigned PDO are taken from the PDO's default mapping.
+ * Please note, that PDO entry registration will fail, if the PDO
+ * configuration is left empty and the slave is offline.
+ *
+ * \code
+ * ec_pdo_info_t pdos[] = {
+ *     {0x1600}, // Channel 1
+ *     {0x1601}  // Channel 2
+ * };
+ *
+ * ec_sync_info_t syncs[] = {
+ *     {3, EC_DIR_INPUT, 2, pdos},
+ * };
+ *
+ * if (ecrt_slave_config_pdos(slave_config_ana_in, 1, syncs)) {
+ *     // handle error
+ * }
+ * \endcode
+ *
+ * Processing of \a syncs will stop, if
+ * - the number of processed items reaches \a n_syncs, or
+ * - the \a index member of an ec_sync_info_t item is 0xff. In this case,
+ *   \a n_syncs should set to a number greater than the number of list items;
+ *   using EC_END is recommended.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return zero on success, else non-zero
+ */
+EC_PUBLIC_API int ecrt_slave_config_pdos(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int n_syncs, /**< Number of sync manager configurations in
+                                \a syncs. */
+        const ec_sync_info_t syncs[] /**< Array of sync manager
+                                       configurations. */
+        );
+
+/** Registers a PDO entry for process data exchange in a domain.
+ *
+ * Searches the assigned PDOs for the given PDO entry. An error is raised, if
+ * the given entry is not mapped. Otherwise, the corresponding sync manager
+ * and FMMU configurations are provided for slave configuration and the
+ * respective sync manager's assigned PDOs are appended to the given domain,
+ * if not already done. The offset of the requested PDO entry's data inside
+ * the domain's process data is returned. Optionally, the PDO entry bit
+ * position (0-7) can be retrieved via the \a bit_position output parameter.
+ * This pointer may be \a NULL, in this case an error is raised if the PDO
+ * entry does not byte-align.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval >=0 Success: Offset of the PDO entry's process data.
+ * \retval  <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_reg_pdo_entry(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t entry_index, /**< Index of the PDO entry to register. */
+        uint8_t entry_subindex, /**< Subindex of the PDO entry to register. */
+        ec_domain_t *domain, /**< Domain. */
+        unsigned int *bit_position /**< Optional address if bit addressing
+                                 is desired */
+        );
+
+/** Registers a PDO entry using its position.
+ *
+ * Similar to ecrt_slave_config_reg_pdo_entry(), but not using PDO indices but
+ * offsets in the PDO mapping, because PDO entry indices may not be unique
+ * inside a slave's PDO mapping. An error is raised, if
+ * one of the given positions is out of range.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval >=0 Success: Offset of the PDO entry's process data.
+ * \retval  <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_reg_pdo_entry_pos(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint8_t sync_index, /**< Sync manager index. */
+        unsigned int pdo_pos, /**< Position of the PDO inside the SM. */
+        unsigned int entry_pos, /**< Position of the entry inside the PDO. */
+        ec_domain_t *domain, /**< Domain. */
+        unsigned int *bit_position /**< Optional address if bit addressing
+                                 is desired */
+        );
+
+/** Configure distributed clocks.
+ *
+ * Sets the AssignActivate word and the cycle and shift times for the sync
+ * signals.
+ *
+ * The AssignActivate word is vendor-specific and can be taken from the XML
+ * device description file (Device -> Dc -> AssignActivate). Set this to zero,
+ * if the slave shall be operated without distributed clocks (default).
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \attention The \a sync1_shift time is ignored.
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_dc(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t assign_activate, /**< AssignActivate word. */
+        uint32_t sync0_cycle, /**< SYNC0 cycle time [ns]. */
+        int32_t sync0_shift, /**< SYNC0 shift time [ns]. */
+        uint32_t sync1_cycle, /**< SYNC1 cycle time [ns]. */
+        int32_t sync1_shift /**< SYNC1 shift time [ns]. */
+        );
+
+/** Add an SDO configuration.
+ *
+ * An SDO configuration is stored in the slave configuration object and is
+ * downloaded to the slave whenever the slave is being configured by the
+ * master. This usually happens once on master activation, but can be repeated
+ * subsequently, for example after the slave's power supply failed.
+ *
+ * \attention The SDOs for PDO assignment (\p 0x1C10 - \p 0x1C2F) and PDO
+ * mapping (\p 0x1600 - \p 0x17FF and \p 0x1A00 - \p 0x1BFF) should not be
+ * configured with this function, because they are part of the slave
+ * configuration done by the master. Please use ecrt_slave_config_pdos() and
+ * friends instead.
+ *
+ * This is the generic function for adding an SDO configuration. Please note
+ * that the this function does not do any endianness correction. If
+ * datatype-specific functions are needed (that automatically correct the
+ * endianness), have a look at ecrt_slave_config_sdo8(),
+ * ecrt_slave_config_sdo16() and ecrt_slave_config_sdo32().
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_sdo(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t index, /**< Index of the SDO to configure. */
+        uint8_t subindex, /**< Subindex of the SDO to configure. */
+        const uint8_t *data, /**< Pointer to the data. */
+        size_t size /**< Size of the \a data. */
+        );
+
+/** Add a configuration value for an 8-bit SDO.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \see ecrt_slave_config_sdo().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_sdo8(
+        ec_slave_config_t *sc, /**< Slave configuration */
+        uint16_t sdo_index, /**< Index of the SDO to configure. */
+        uint8_t sdo_subindex, /**< Subindex of the SDO to configure. */
+        uint8_t value /**< Value to set. */
+        );
+
+/** Add a configuration value for a 16-bit SDO.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \see ecrt_slave_config_sdo().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_sdo16(
+        ec_slave_config_t *sc, /**< Slave configuration */
+        uint16_t sdo_index, /**< Index of the SDO to configure. */
+        uint8_t sdo_subindex, /**< Subindex of the SDO to configure. */
+        uint16_t value /**< Value to set. */
+        );
+
+/** Add a configuration value for a 32-bit SDO.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \see ecrt_slave_config_sdo().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_sdo32(
+        ec_slave_config_t *sc, /**< Slave configuration */
+        uint16_t sdo_index, /**< Index of the SDO to configure. */
+        uint8_t sdo_subindex, /**< Subindex of the SDO to configure. */
+        uint32_t value /**< Value to set. */
+        );
+
+/** Add configuration data for a complete SDO.
+ *
+ * The SDO data are transferred via CompleteAccess. Data for the first
+ * subindex (0) have to be included.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \see ecrt_slave_config_sdo().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_complete_sdo(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t index, /**< Index of the SDO to configure. */
+        const uint8_t *data, /**< Pointer to the data. */
+        size_t size /**< Size of the \a data. */
+        );
+
+/** Set the size of the CoE emergency ring buffer.
+ *
+ * The initial size is zero, so all messages will be dropped. This method can
+ * be called even after master activation, but it will clear the ring buffer!
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return 0 on success, or negative error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_emerg_size(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        size_t elements /**< Number of records of the CoE emergency ring. */
+        );
+
+/** Read and remove one record from the CoE emergency ring buffer.
+ *
+ * A record consists of 8 bytes:
+ *
+ * Byte 0-1: Error code (little endian)
+ * Byte   2: Error register
+ * Byte 3-7: Data
+ *
+ * Calling this method makes only sense in realtime context (after master
+ * activation).
+ *
+ * \return 0 on success (record popped), or negative error code (i. e.
+ * -ENOENT, if ring is empty).
+ *
+ * \apiusage{master_op,any_context}
+ */
+EC_PUBLIC_API int ecrt_slave_config_emerg_pop(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint8_t *target /**< Pointer to target memory (at least
+                          EC_COE_EMERGENCY_MSG_SIZE bytes). */
+        );
+
+/** Clears CoE emergency ring buffer and the overrun counter.
+ *
+ * Calling this method makes only sense in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,any_context}
+ *
+ * \return 0 on success, or negative error code.
+ *
+ */
+EC_PUBLIC_API int ecrt_slave_config_emerg_clear(
+        ec_slave_config_t *sc /**< Slave configuration. */
+        );
+
+/** Read the number of CoE emergency overruns.
+ *
+ * The overrun counter will be incremented when a CoE emergency message could
+ * not be stored in the ring buffer and had to be dropped. Call
+ * ecrt_slave_config_emerg_clear() to reset the counter.
+ *
+ * Calling this method makes only sense in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,any_context}
+ *
+ * \return Number of overruns since last clear, or negative error code.
+ *
+ */
+EC_PUBLIC_API int ecrt_slave_config_emerg_overruns(
+        const ec_slave_config_t *sc /**< Slave configuration. */
+        );
+
+/** Create an SDO request to exchange SDOs during realtime operation.
+ *
+ * The created SDO request object is freed automatically when the master is
+ * released.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return New SDO request, or NULL on error.
+ */
+EC_PUBLIC_API ec_sdo_request_t *ecrt_slave_config_create_sdo_request(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint16_t index, /**< SDO index. */
+        uint8_t subindex, /**< SDO subindex. */
+        size_t size /**< Data size to reserve. */
+        );
+
+/** Create an SoE request to exchange SoE IDNs during realtime operation.
+ *
+ * The created SoE request object is freed automatically when the master is
+ * released.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return New SoE request, or NULL on error.
+ */
+EC_PUBLIC_API ec_soe_request_t *ecrt_slave_config_create_soe_request(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint8_t drive_no, /**< Drive number. */
+        uint16_t idn, /**< Sercos ID-Number. */
+        size_t size /**< Data size to reserve. */
+        );
+
+/** Create an VoE handler to exchange vendor-specific data during realtime
+ * operation.
+ *
+ * The number of VoE handlers per slave configuration is not limited, but
+ * usually it is enough to create one for sending and one for receiving, if
+ * both can be done simultaneously.
+ *
+ * The created VoE handler object is freed automatically when the master is
+ * released.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return New VoE handler, or NULL on error.
+ */
+EC_PUBLIC_API ec_voe_handler_t *ecrt_slave_config_create_voe_handler(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        size_t size /**< Data size to reserve. */
+        );
+
+/** Create a register request to exchange EtherCAT register contents during
+ * realtime operation.
+ *
+ * This interface should not be used to take over master functionality,
+ * instead it is intended for debugging and monitoring reasons.
+ *
+ * The created register request object is freed automatically when the master
+ * is released.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return New register request, or NULL on error.
+ */
+EC_PUBLIC_API ec_reg_request_t *ecrt_slave_config_create_reg_request(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        size_t size /**< Data size to reserve. */
+        );
+
+/** Outputs the state of the slave configuration.
+ *
+ * Stores the state information in the given \a state structure. The state
+ * information is updated by the master state machine, so it may take a few
+ * cycles, until it changes.
+ *
+ * \attention If the state of process data exchange shall be monitored in
+ * realtime, ecrt_domain_state() should be used.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_state(
+        const ec_slave_config_t *sc, /**< Slave configuration */
+        ec_slave_config_state_t *state /**< State object to write to. */
+        );
+
+/** Add an SoE IDN configuration.
+ *
+ * A configuration for a Sercos-over-EtherCAT IDN is stored in the slave
+ * configuration object and is written to the slave whenever the slave is
+ * being configured by the master. This usually happens once on master
+ * activation, but can be repeated subsequently, for example after the slave's
+ * power supply failed.
+ *
+ * The \a idn parameter can be separated into several sections:
+ *  - Bit 15: Standard data (0) or Product data (1)
+ *  - Bit 14 - 12: Parameter set (0 - 7)
+ *  - Bit 11 - 0: Data block number (0 - 4095)
+ *
+ * Please note that the this function does not do any endianness correction.
+ * Multi-byte data have to be passed in EtherCAT endianness (little-endian).
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_idn(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        uint8_t drive_no, /**< Drive number. */
+        uint16_t idn, /**< SoE IDN. */
+        ec_al_state_t state, /**< AL state in which to write the IDN (PREOP or
+                               SAFEOP). */
+        const uint8_t *data, /**< Pointer to the data. */
+        size_t size /**< Size of the \a data. */
+        );
+
+/** Adds a feature flag to a slave configuration.
+ *
+ * Feature flags are a generic way to configure slave-specific behavior.
+ *
+ * Multiple calls with the same slave configuration and key will overwrite the
+ * configuration.
+ *
+ * The following flags may be available:
+ * - AssignToPdi: Zero (default) keeps the slave information interface (SII)
+ *   assigned to EtherCAT (except during transition to PREOP). Non-zero
+ *   assigns the SII to the slave controller side before going to PREOP and
+ *   leaves it there until a write command happens.
+ * - WaitBeforeSAFEOPms: Number of milliseconds to wait before commanding the
+ *   transition from PREOP to SAFEOP. This can be used as a workaround for
+ *   slaves that need a little time to initialize.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_flag(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        const char *key, /**< Key as null-terminated ASCII string. */
+        int32_t value /**< Value to store. */
+        );
+
+/** Sets the link/MAC address for Ethernet-over-EtherCAT (EoE) operation.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * The MAC address is stored in the slave configuration object and will be
+ * written to the slave during the configuration process.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_eoe_mac_address(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        const unsigned char *mac_address /**< MAC address. */
+        );
+
+/** Sets the IP address for Ethernet-over-EtherCAT (EoE) operation.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * The IP address is stored in the slave configuration object and will be
+ * written to the slave during the configuration process.
+ *
+ * The IP address is passed by-value as a `struct in_addr`. This structure
+ * contains the 32-bit IPv4 address in network byte order (big endian).
+ *
+ * A string-represented IPv4 address can be converted to a `struct in_addr`
+ * for example via the POSIX function `inet_pton()` (see man 3 inet_pton):
+ *
+ * \code{.c}
+ *     #include <arpa/inet.h>
+ *     struct in_addr addr;
+ *     if (inet_aton("192.168.0.1", &addr) == 0) {
+ *         fprintf(stderr, "Failed to convert IP address.\n");
+ *         return -1;
+ *     }
+ *     if (ecrt_slave_config_eoe_ip_address(sc, addr)) {
+ *         fprintf(stderr, "Failed to set IP address.\n");
+ *         return -1;
+ *     }
+ * \endcode
+ *
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_eoe_ip_address(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        struct in_addr ip_address /**< IPv4 address. */
+        );
+
+/** Sets the subnet mask for Ethernet-over-EtherCAT (EoE) operation.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * The subnet mask is stored in the slave configuration object and will be
+ * written to the slave during the configuration process.
+ *
+ * The subnet mask is passed by-value as a `struct in_addr`. This structure
+ * contains the 32-bit mask in network byte order (big endian).
+ *
+ * See ecrt_slave_config_eoe_ip_address() on how to convert string-coded masks
+ * to `struct in_addr`.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_eoe_subnet_mask(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        struct in_addr subnet_mask /**< IPv4 subnet mask. */
+        );
+
+/** Sets the gateway address for Ethernet-over-EtherCAT (EoE) operation.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * The gateway address is stored in the slave configuration object and will be
+ * written to the slave during the configuration process.
+ *
+ * The address is passed by-value as a `struct in_addr`. This structure
+ * contains the 32-bit IPv4 address in network byte order (big endian).
+ *
+ * See ecrt_slave_config_eoe_ip_address() on how to convert string-coded IPv4
+ * addresses to `struct in_addr`.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_eoe_default_gateway(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        struct in_addr gateway_address /**< Gateway's IPv4 address. */
+        );
+
+/** Sets the IPv4 address of the DNS server for Ethernet-over-EtherCAT (EoE)
+ * operation.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * The DNS server address is stored in the slave configuration object and will
+ * be written to the slave during the configuration process.
+ *
+ * The address is passed by-value as a `struct in_addr`. This structure
+ * contains the 32-bit IPv4 address in network byte order (big endian).
+ *
+ * See ecrt_slave_config_eoe_ip_address() on how to convert string-coded IPv4
+ * addresses to `struct in_addr`.
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_eoe_dns_address(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        struct in_addr dns_address /**< IPv4 address of the DNS server. */
+        );
+
+/** Sets the host name for Ethernet-over-EtherCAT (EoE) operation.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * The host name is stored in the slave configuration object and will
+ * be written to the slave during the configuration process.
+ *
+ * The maximum size of the host name is 32 bytes (including the zero
+ * terminator).
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_eoe_hostname(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        const char *name /**< Zero-terminated host name. */
+        );
+
+/** Sets the application-layer state transition timeout in ms.
+ *
+ * Change the maximum allowed time for a slave to make an application-layer
+ * state transition for the given state transition (for example from PREOP to
+ * SAFEOP). The default values are defined in ETG.2000.
+ *
+ * A timeout value of zero ms will restore the default value.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+EC_PUBLIC_API int ecrt_slave_config_state_timeout(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        ec_al_state_t from_state, /**< Initial state. */
+        ec_al_state_t to_state, /**< Target state. */
+        unsigned int timeout_ms /**< Timeout in [ms]. */
+        );
+
+/*****************************************************************************
+ * Domain methods
+ ****************************************************************************/
+
+/** Registers a bunch of PDO entries for a domain.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \see ecrt_slave_config_reg_pdo_entry()
+ *
+ * \attention The registration array has to be terminated with an empty
+ *            structure, or one with the \a index field set to zero!
+ *
+ * \apiusage{master_idle,blocking}
+ *
+ * \return 0 on success, else non-zero.
+ */
+EC_PUBLIC_API int ecrt_domain_reg_pdo_entry_list(
+        ec_domain_t *domain, /**< Domain. */
+        const ec_pdo_entry_reg_t *pdo_entry_regs /**< Array of PDO
+                                                   registrations. */
+        );
+
+/** Returns the current size of the domain's process data.
+ *
+ * The domain size is calculated after master activation.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Size of the process data image, or a negative error code.
+ */
+EC_PUBLIC_API size_t ecrt_domain_size(
+        const ec_domain_t *domain /**< Domain. */
+        );
+
+#ifdef __KERNEL__
+
+/** Provide external memory to store the domain's process data.
+ *
+ * Call this after all PDO entries have been registered and before activating
+ * the master.
+ *
+ * The size of the allocated memory must be at least ecrt_domain_size(), after
+ * all PDO entries have been registered.
+ *
+ * This method has to be called in non-realtime context before
+ * ecrt_master_activate().
+ *
+ * \apiusage{master_idle,blocking}
+ */
+void ecrt_domain_external_memory(
+        ec_domain_t *domain, /**< Domain. */
+        uint8_t *memory /**< Address of the memory to store the process
+                          data in. */
+        );
+
+#endif /* __KERNEL__ */
+
+/** Returns the domain's process data.
+ *
+ * - In kernel context: If external memory was provided with
+ * ecrt_domain_external_memory(), the returned pointer will contain the
+ * address of that memory. Otherwise it will point to the internally allocated
+ * memory. In the latter case, this method may not be called before
+ * ecrt_master_activate().
+ *
+ * - In userspace context: This method has to be called after
+ * ecrt_master_activate() to get the mapped domain process data memory.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Pointer to the process data memory.
+ */
+EC_PUBLIC_API uint8_t *ecrt_domain_data(
+        const ec_domain_t *domain /**< Domain. */
+        );
+
+/** Determines the states of the domain's datagrams.
+ *
+ * Evaluates the working counters of the received datagrams and outputs
+ * statistics, if necessary. This must be called after ecrt_master_receive()
+ * is expected to receive the domain datagrams in order to make
+ * ecrt_domain_state() return the result of the last process data exchange.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_domain_process(
+        ec_domain_t *domain /**< Domain. */
+        );
+
+/** (Re-)queues all domain datagrams in the master's datagram queue.
+ *
+ * Call this function to mark the domain's datagrams for exchanging at the
+ * next call of ecrt_master_send().
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_domain_queue(
+        ec_domain_t *domain /**< Domain. */
+        );
+
+/** Reads the state of a domain.
+ *
+ * Stores the domain state in the given \a state structure.
+ *
+ * Using this method, the process data exchange can be monitored in realtime.
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_domain_state(
+        const ec_domain_t *domain, /**< Domain. */
+        ec_domain_state_t *state /**< Pointer to a state object to store the
+                                   information. */
+        );
+
+/*****************************************************************************
+ * SDO request methods.
+ ****************************************************************************/
+
+/** Set the SDO index and subindex.
+ *
+ * \attention If the SDO index and/or subindex is changed while
+ * ecrt_sdo_request_state() returns EC_REQUEST_BUSY, this may lead to
+ * unexpected results.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation). To initialize the SDO request, the index and subindex can be
+ * set via ecrt_slave_config_create_sdo_request().
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_sdo_request_index(
+        ec_sdo_request_t *req, /**< SDO request. */
+        uint16_t index, /**< SDO index. */
+        uint8_t subindex /**< SDO subindex. */
+        );
+
+/** Set the timeout for an SDO request.
+ *
+ * If the request cannot be processed in the specified time, if will be marked
+ * as failed.
+ *
+ * The timeout is permanently stored in the request object and is valid until
+ * the next call of this method.
+ *
+ * The timeout should be defined in non-realtime context, but can also be
+ * changed afterwards.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_sdo_request_timeout(
+        ec_sdo_request_t *req, /**< SDO request. */
+        uint32_t timeout /**< Timeout in milliseconds. Zero means no
+                           timeout. */
+        );
+
+/** Access to the SDO request's data.
+ *
+ * This function returns a pointer to the request's internal SDO data memory.
+ *
+ * - After a read operation was successful, integer data can be evaluated
+ *   using the EC_READ_*() macros as usual. Example:
+ *   \code
+ *   uint16_t value = EC_READ_U16(ecrt_sdo_request_data(sdo)));
+ *   \endcode
+ * - If a write operation shall be triggered, the data have to be written to
+ *   the internal memory. Use the EC_WRITE_*() macros, if you are writing
+ *   integer data. Be sure, that the data fit into the memory. The memory size
+ *   is a parameter of ecrt_slave_config_create_sdo_request().
+ *   \code
+ *   EC_WRITE_U16(ecrt_sdo_request_data(sdo), 0xFFFF);
+ *   \endcode
+ *
+ * \attention The return value can be invalid during a read operation, because
+ * the internal SDO data memory could be re-allocated if the read SDO data do
+ * not fit inside.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation), but can also be used to initialize data before.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Pointer to the internal SDO data memory.
+ *
+ */
+EC_PUBLIC_API uint8_t *ecrt_sdo_request_data(
+        const ec_sdo_request_t *req /**< SDO request. */
+        );
+
+/** Returns the current SDO data size.
+ *
+ * When the SDO request is created, the data size is set to the size of the
+ * reserved memory. After a read operation the size is set to the size of the
+ * read data. The size is not modified in any other situation.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return SDO data size in bytes.
+ *
+ */
+EC_PUBLIC_API size_t ecrt_sdo_request_data_size(
+        const ec_sdo_request_t *req /**< SDO request. */
+        );
+
+/** Get the current state of the SDO request.
+ *
+ * The user-space implementation fetches incoming data and stores the received
+ * data size in the request object, so the request is not const.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Request state.
+ *
+ */
+EC_PUBLIC_API ec_request_state_t ecrt_sdo_request_state(
+#ifdef __KERNEL__
+        const
+#endif
+        ec_sdo_request_t *req /**< SDO request. */
+        );
+
+/** Schedule an SDO write operation.
+ *
+ * \attention This method may not be called while ecrt_sdo_request_state()
+ * returns EC_REQUEST_BUSY.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ * \retval -EINVAL Invalid input data, e.g. data size == 0.
+ * \retval -ENOBUFS Reserved memory in ecrt_slave_config_create_sdo_request()
+ *              too small.
+ */
+EC_PUBLIC_API int ecrt_sdo_request_write(
+        ec_sdo_request_t *req /**< SDO request. */
+        );
+
+/** Schedule an SDO read operation.
+ *
+ * \attention This method may not be called while ecrt_sdo_request_state()
+ * returns EC_REQUEST_BUSY.
+ *
+ * \attention After calling this function, the return value of
+ * ecrt_sdo_request_data() must be considered as invalid while
+ * ecrt_sdo_request_state() returns EC_REQUEST_BUSY.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_sdo_request_read(
+        ec_sdo_request_t *req /**< SDO request. */
+        );
+
+/*****************************************************************************
+ * SoE request methods.
+ ****************************************************************************/
+
+/** Set the request's drive and Sercos ID numbers.
+ *
+ * \attention If the drive number and/or IDN is changed while
+ * ecrt_soe_request_state() returns EC_REQUEST_BUSY, this may lead to
+ * unexpected results.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation). To initialize the SoE request, the drive_no and IDN can be
+ * set via ecrt_slave_config_create_soe_request().
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_soe_request_idn(
+        ec_soe_request_t *req, /**< IDN request. */
+        uint8_t drive_no, /**< SDO index. */
+        uint16_t idn /**< SoE IDN. */
+        );
+
+/** Set the timeout for an SoE request.
+ *
+ * If the request cannot be processed in the specified time, if will be marked
+ * as failed.
+ *
+ * The timeout is permanently stored in the request object and is valid until
+ * the next call of this method.
+ *
+ * The timeout should be defined in non-realtime context, but can also be
+ * changed afterwards.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_soe_request_timeout(
+        ec_soe_request_t *req, /**< SoE request. */
+        uint32_t timeout /**< Timeout in milliseconds. Zero means no
+                           timeout. */
+        );
+
+/** Access to the SoE request's data.
+ *
+ * This function returns a pointer to the request's internal IDN data memory.
+ *
+ * - After a read operation was successful, integer data can be evaluated
+ *   using the EC_READ_*() macros as usual. Example:
+ *   \code
+ *   uint16_t value = EC_READ_U16(ecrt_soe_request_data(idn_req)));
+ *   \endcode
+ * - If a write operation shall be triggered, the data have to be written to
+ *   the internal memory. Use the EC_WRITE_*() macros, if you are writing
+ *   integer data. Be sure, that the data fit into the memory. The memory size
+ *   is a parameter of ecrt_slave_config_create_soe_request().
+ *   \code
+ *   EC_WRITE_U16(ecrt_soe_request_data(idn_req), 0xFFFF);
+ *   \endcode
+ *
+ * \attention The return value can be invalidated during a read operation,
+ * because the internal IDN data memory could be re-allocated if the read IDN
+ * data do not fit inside.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation), but can also be used to initialize data before.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Pointer to the internal IDN data memory.
+ *
+ */
+EC_PUBLIC_API uint8_t *ecrt_soe_request_data(
+        const ec_soe_request_t *req /**< SoE request. */
+        );
+
+/** Returns the current IDN data size.
+ *
+ * When the SoE request is created, the data size is set to the size of the
+ * reserved memory. After a read operation the size is set to the size of the
+ * read data. The size is not modified in any other situation.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return IDN data size in bytes.
+ */
+EC_PUBLIC_API size_t ecrt_soe_request_data_size(
+        const ec_soe_request_t *req /**< SoE request. */
+        );
+
+/** Get the current state of the SoE request.
+ *
+ * \return Request state.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * In the user-space implementation, the method fetches the size of the
+ * incoming data, so the request object is not const.
+ *
+ * \apiusage{master_op,rt_safe}
+ */
+EC_PUBLIC_API ec_request_state_t ecrt_soe_request_state(
+#ifdef __KERNEL__
+        const
+#endif
+        ec_soe_request_t *req /**< SoE request. */
+        );
+
+/** Schedule an SoE IDN write operation.
+ *
+ * \attention This method may not be called while ecrt_soe_request_state()
+ * returns EC_REQUEST_BUSY.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ * \retval -EINVAL Invalid input data, e.g. data size == 0.
+ * \retval -ENOBUFS Reserved memory in ecrt_slave_config_create_soe_request()
+ *              too small.
+ */
+EC_PUBLIC_API int ecrt_soe_request_write(
+        ec_soe_request_t *req /**< SoE request. */
+        );
+
+/** Schedule an SoE IDN read operation.
+ *
+ * \attention This method may not be called while ecrt_soe_request_state()
+ * returns EC_REQUEST_BUSY.
+ *
+ * \attention After calling this function, the return value of
+ * ecrt_soe_request_data() must be considered as invalid while
+ * ecrt_soe_request_state() returns EC_REQUEST_BUSY.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_soe_request_read(
+        ec_soe_request_t *req /**< SoE request. */
+        );
+
+/*****************************************************************************
+ * VoE handler methods.
+ ****************************************************************************/
+
+/** Sets the VoE header for future send operations.
+ *
+ * A VoE message shall contain a 4-byte vendor ID, followed by a 2-byte vendor
+ * type at as header. These numbers can be set with this function. The values
+ * are valid and will be used for future send operations until the next call
+ * of this method.
+ *
+ * This method is meant to be called in non-realtime context (before master
+ * activation) to initialize the header data, but it is also safe to
+ * change the header later on in realtime context.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_voe_handler_send_header(
+        ec_voe_handler_t *voe, /**< VoE handler. */
+        uint32_t vendor_id, /**< Vendor ID. */
+        uint16_t vendor_type /**< Vendor-specific type. */
+        );
+
+/** Reads the header data of a received VoE message.
+ *
+ * This method can be used to get the received VoE header information after a
+ * read operation has succeeded.
+ *
+ * The header information is stored at the memory given by the pointer
+ * parameters.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_voe_handler_received_header(
+        const ec_voe_handler_t *voe, /**< VoE handler. */
+        uint32_t *vendor_id, /**< Vendor ID. */
+        uint16_t *vendor_type /**< Vendor-specific type. */
+        );
+
+/** Access to the VoE handler's data.
+ *
+ * This function returns a pointer to the VoE handler's internal memory, that
+ * points to the actual VoE data right after the VoE header (see
+ * ecrt_voe_handler_send_header()).
+ *
+ * - After a read operation was successful, the memory contains the received
+ *   data. The size of the received data can be determined via
+ *   ecrt_voe_handler_data_size().
+ * - Before a write operation is triggered, the data have to be written to the
+ *   internal memory. Be sure, that the data fit into the memory. The reserved
+ *   memory size is a parameter of ecrt_slave_config_create_voe_handler().
+ *
+ * \attention The returned pointer is not necessarily persistent: After a read
+ * operation, the internal memory may have been reallocated. This can be
+ * avoided by reserving enough memory via the \a size parameter of
+ * ecrt_slave_config_create_voe_handler().
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Pointer to the internal memory.
+ */
+EC_PUBLIC_API uint8_t *ecrt_voe_handler_data(
+        const ec_voe_handler_t *voe /**< VoE handler. */
+        );
+
+/** Returns the current data size.
+ *
+ * The data size is the size of the VoE data without the header (see
+ * ecrt_voe_handler_send_header()).
+ *
+ * When the VoE handler is created, the data size is set to the size of the
+ * reserved memory. At a write operation, the data size is set to the number
+ * of bytes to write. After a read operation the size is set to the size of
+ * the read data. The size is not modified in any other situation.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Data size in bytes.
+ */
+EC_PUBLIC_API size_t ecrt_voe_handler_data_size(
+        const ec_voe_handler_t *voe /**< VoE handler. */
+        );
+
+/** Start a VoE write operation.
+ *
+ * After this function has been called, the ecrt_voe_handler_execute() method
+ * must be called in every realtime cycle as long as it returns
+ * EC_REQUEST_BUSY. No other operation may be started while the handler is
+ * busy.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ * \retval -ENOBUFS Reserved memory in ecrt_slave_config_create_voe_handler
+ *                  too small.
+ */
+EC_PUBLIC_API int ecrt_voe_handler_write(
+        ec_voe_handler_t *voe, /**< VoE handler. */
+        size_t size /**< Number of bytes to write (without the VoE header). */
+        );
+
+/** Start a VoE read operation.
+ *
+ * After this function has been called, the ecrt_voe_handler_execute() method
+ * must be called in every realtime cycle as long as it returns
+ * EC_REQUEST_BUSY. No other operation may be started while the handler is
+ * busy.
+ *
+ * The state machine queries the slave's send mailbox for new data to be send
+ * to the master. If no data appear within the EC_VOE_RESPONSE_TIMEOUT
+ * (defined in master/voe_handler.c), the operation fails.
+ *
+ * On success, the size of the read data can be determined via
+ * ecrt_voe_handler_data_size(), while the VoE header of the received data
+ * can be retrieved with ecrt_voe_handler_received_header().
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_voe_handler_read(
+        ec_voe_handler_t *voe /**< VoE handler. */
+        );
+
+/** Start a VoE read operation without querying the sync manager status.
+ *
+ * After this function has been called, the ecrt_voe_handler_execute() method
+ * must be called in every realtime cycle as long as it returns
+ * EC_REQUEST_BUSY. No other operation may be started while the handler is
+ * busy.
+ *
+ * The state machine queries the slave by sending an empty mailbox. The slave
+ * fills its data to the master in this mailbox. If no data appear within the
+ * EC_VOE_RESPONSE_TIMEOUT (defined in master/voe_handler.c), the operation
+ * fails.
+ *
+ * On success, the size of the read data can be determined via
+ * ecrt_voe_handler_data_size(), while the VoE header of the received data
+ * can be retrieved with ecrt_voe_handler_received_header().
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ */
+EC_PUBLIC_API int ecrt_voe_handler_read_nosync(
+        ec_voe_handler_t *voe /**< VoE handler. */
+        );
+
+/** Execute the handler.
+ *
+ * This method executes the VoE handler. It has to be called in every realtime
+ * cycle as long as it returns EC_REQUEST_BUSY.
+ *
+ * \return Handler state.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ */
+EC_PUBLIC_API ec_request_state_t ecrt_voe_handler_execute(
+        ec_voe_handler_t *voe /**< VoE handler. */
+        );
+
+/*****************************************************************************
+ * Register request methods.
+ ****************************************************************************/
+
+/** Access to the register request's data.
+ *
+ * This function returns a pointer to the request's internal memory.
+ *
+ * - After a read operation was successful, integer data can be evaluated
+ *   using the EC_READ_*() macros as usual. Example:
+ *   \code
+ *   uint16_t value = EC_READ_U16(ecrt_reg_request_data(reg_request)));
+ *   \endcode
+ * - If a write operation shall be triggered, the data have to be written to
+ *   the internal memory. Use the EC_WRITE_*() macros, if you are writing
+ *   integer data. Be sure, that the data fit into the memory. The memory size
+ *   is a parameter of ecrt_slave_config_create_reg_request().
+ *   \code
+ *   EC_WRITE_U16(ecrt_reg_request_data(reg_request), 0xFFFF);
+ *   \endcode
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation), but can also be used to initialize data before.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \return Pointer to the internal memory.
+ *
+ */
+EC_PUBLIC_API uint8_t *ecrt_reg_request_data(
+        const ec_reg_request_t *req /**< Register request. */
+        );
+
+/** Get the current state of the register request.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return Request state.
+ *
+ */
+EC_PUBLIC_API ec_request_state_t ecrt_reg_request_state(
+        const ec_reg_request_t *req /**< Register request. */
+    );
+
+/** Schedule an register write operation.
+ *
+ * \attention This method may not be called while ecrt_reg_request_state()
+ * returns EC_REQUEST_BUSY.
+ *
+ * \attention The \a size parameter is truncated to the size given at request
+ * creation.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ * \retval -ENOBUFS Reserved memory in ecrt_slave_config_create_reg_request
+ *              too small.
+ */
+EC_PUBLIC_API int ecrt_reg_request_write(
+        ec_reg_request_t *req, /**< Register request. */
+        uint16_t address, /**< Register address. */
+        size_t size /**< Size to write. */
+        );
+
+/** Schedule a register read operation.
+ *
+ * \attention This method may not be called while ecrt_reg_request_state()
+ * returns EC_REQUEST_BUSY.
+ *
+ * \attention The \a size parameter is truncated to the size given at request
+ * creation.
+ *
+ * This method is meant to be called in realtime context (after master
+ * activation).
+ *
+ * \apiusage{master_op,rt_safe}
+ *
+ * \return 0 on success, otherwise negative error code.
+ * \retval -ENOBUFS Reserved memory in ecrt_slave_config_create_reg_request
+ *              too small.
+ */
+EC_PUBLIC_API int ecrt_reg_request_read(
+        ec_reg_request_t *req, /**< Register request. */
+        uint16_t address, /**< Register address. */
+        size_t size /**< Size to write. */
+        );
+
+/*****************************************************************************
+ * Bitwise read/write macros
+ ****************************************************************************/
+
+/** Read a certain bit of an EtherCAT data byte.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param POS bit position
+ */
+#define EC_READ_BIT(DATA, POS) ((*((uint8_t *) (DATA)) >> (POS)) & 0x01)
+
+/** Write a certain bit of an EtherCAT data byte.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param POS bit position
+ * \param VAL new bit value
+ */
+#define EC_WRITE_BIT(DATA, POS, VAL) \
+    do { \
+        if (VAL) *((uint8_t *) (DATA)) |=  (1 << (POS)); \
+        else     *((uint8_t *) (DATA)) &= ~(1 << (POS)); \
+    } while (0)
+
+/*****************************************************************************
+ * Byte-swapping functions for user space
+ ****************************************************************************/
+
+#ifndef __KERNEL__
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+
+#define le16_to_cpu(x) x
+#define le32_to_cpu(x) x
+#define le64_to_cpu(x) x
+
+#define cpu_to_le16(x) x
+#define cpu_to_le32(x) x
+#define cpu_to_le64(x) x
+
+#elif __BYTE_ORDER == __BIG_ENDIAN
+
+#define swap16(x) \
+        ((uint16_t)( \
+        (((uint16_t)(x) & 0x00ffU) << 8) | \
+        (((uint16_t)(x) & 0xff00U) >> 8) ))
+#define swap32(x) \
+        ((uint32_t)( \
+        (((uint32_t)(x) & 0x000000ffUL) << 24) | \
+        (((uint32_t)(x) & 0x0000ff00UL) <<  8) | \
+        (((uint32_t)(x) & 0x00ff0000UL) >>  8) | \
+        (((uint32_t)(x) & 0xff000000UL) >> 24) ))
+#define swap64(x) \
+        ((uint64_t)( \
+        (((uint64_t)(x) & 0x00000000000000ffULL) << 56) | \
+        (((uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \
+        (((uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \
+        (((uint64_t)(x) & 0x00000000ff000000ULL) <<  8) | \
+        (((uint64_t)(x) & 0x000000ff00000000ULL) >>  8) | \
+        (((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \
+        (((uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \
+        (((uint64_t)(x) & 0xff00000000000000ULL) >> 56) ))
+
+#define le16_to_cpu(x) swap16(x)
+#define le32_to_cpu(x) swap32(x)
+#define le64_to_cpu(x) swap64(x)
+
+#define cpu_to_le16(x) swap16(x)
+#define cpu_to_le32(x) swap32(x)
+#define cpu_to_le64(x) swap64(x)
+
+#endif
+
+#define le16_to_cpup(x) le16_to_cpu(*((uint16_t *)(x)))
+#define le32_to_cpup(x) le32_to_cpu(*((uint32_t *)(x)))
+#define le64_to_cpup(x) le64_to_cpu(*((uint64_t *)(x)))
+
+#endif /* ifndef __KERNEL__ */
+
+/*****************************************************************************
+ * Read macros
+ ****************************************************************************/
+
+/** Read an 8-bit unsigned value from EtherCAT data.
+ *
+ * \return EtherCAT data value
+ */
+#define EC_READ_U8(DATA) \
+    ((uint8_t) *((uint8_t *) (DATA)))
+
+/** Read an 8-bit signed value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_S8(DATA) \
+     ((int8_t) *((uint8_t *) (DATA)))
+
+/** Read a 16-bit unsigned value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_U16(DATA) \
+     ((uint16_t) le16_to_cpup((void *) (DATA)))
+
+/** Read a 16-bit signed value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_S16(DATA) \
+     ((int16_t) le16_to_cpup((void *) (DATA)))
+
+/** Read a 32-bit unsigned value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_U32(DATA) \
+     ((uint32_t) le32_to_cpup((void *) (DATA)))
+
+/** Read a 32-bit signed value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_S32(DATA) \
+     ((int32_t) le32_to_cpup((void *) (DATA)))
+
+/** Read a 64-bit unsigned value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_U64(DATA) \
+     ((uint64_t) le64_to_cpup((void *) (DATA)))
+
+/** Read a 64-bit signed value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_S64(DATA) \
+     ((int64_t) le64_to_cpup((void *) (DATA)))
+
+/*****************************************************************************
+ * Floating-point read functions and macros (userspace only)
+ ****************************************************************************/
+
+#ifndef __KERNEL__
+
+/** Read a 32-bit floating-point value from EtherCAT data.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \param data EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+EC_PUBLIC_API float ecrt_read_real(const void *data);
+
+/** Read a 32-bit floating-point value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_REAL(DATA) ecrt_read_real(DATA)
+
+/** Read a 64-bit floating-point value from EtherCAT data.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \param data EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+EC_PUBLIC_API double ecrt_read_lreal(const void *data);
+
+/** Read a 64-bit floating-point value from EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \return EtherCAT data value
+ */
+#define EC_READ_LREAL(DATA) ecrt_read_lreal(DATA)
+
+#endif // ifndef __KERNEL__
+
+/*****************************************************************************
+ * Write macros
+ ****************************************************************************/
+
+/** Write an 8-bit unsigned value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_U8(DATA, VAL) \
+    do { \
+        *((uint8_t *)(DATA)) = ((uint8_t) (VAL)); \
+    } while (0)
+
+/** Write an 8-bit signed value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_S8(DATA, VAL) EC_WRITE_U8(DATA, VAL)
+
+/** Write a 16-bit unsigned value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_U16(DATA, VAL) \
+    do { \
+        *((uint16_t *) (DATA)) = cpu_to_le16((uint16_t) (VAL)); \
+    } while (0)
+
+/** Write a 16-bit signed value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_S16(DATA, VAL) EC_WRITE_U16(DATA, VAL)
+
+/** Write a 32-bit unsigned value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_U32(DATA, VAL) \
+    do { \
+        *((uint32_t *) (DATA)) = cpu_to_le32((uint32_t) (VAL)); \
+    } while (0)
+
+/** Write a 32-bit signed value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_S32(DATA, VAL) EC_WRITE_U32(DATA, VAL)
+
+/** Write a 64-bit unsigned value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_U64(DATA, VAL) \
+    do { \
+        *((uint64_t *) (DATA)) = cpu_to_le64((uint64_t) (VAL)); \
+    } while (0)
+
+/** Write a 64-bit signed value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_S64(DATA, VAL) EC_WRITE_U64(DATA, VAL)
+
+/*****************************************************************************
+ * Floating-point write functions and macros (userspace only)
+ ****************************************************************************/
+
+#ifndef __KERNEL__
+
+/** Write a 32-bit floating-point value to EtherCAT data.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \param data EtherCAT data pointer
+ * \param value new value
+ */
+EC_PUBLIC_API void ecrt_write_real(void *data, float value);
+
+/** Write a 32-bit floating-point value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_REAL(DATA, VAL) ecrt_write_real(DATA, VAL)
+
+/** Write a 64-bit floating-point value to EtherCAT data.
+ *
+ * \apiusage{master_any,rt_safe}
+ *
+ * \param data EtherCAT data pointer
+ * \param value new value
+ */
+EC_PUBLIC_API void ecrt_write_lreal(void *data, double value);
+
+/** Write a 64-bit floating-point value to EtherCAT data.
+ *
+ * \param DATA EtherCAT data pointer
+ * \param VAL new value
+ */
+#define EC_WRITE_LREAL(DATA, VAL) ecrt_write_lreal(DATA, VAL)
+
+#endif // ifndef __KERNEL__
+
+/****************************************************************************/
+
+#ifdef __cplusplus
+}
+#endif
+
+/****************************************************************************/
+
+/** @} */
+
+#endif
diff --git a/drivers/net/ethercat/include/ectty.h b/drivers/net/ethercat/include/ectty.h
new file mode 100644
index 000000000000..3178d3ce9ba9
--- /dev/null
+++ b/drivers/net/ethercat/include/ectty.h
@@ -0,0 +1,106 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT master userspace library.
+ *
+ *  The IgH EtherCAT master userspace library is free software; you can
+ *  redistribute it and/or modify it under the terms of the GNU Lesser General
+ *  Public License as published by the Free Software Foundation; version 2.1
+ *  of the License.
+ *
+ *  The IgH EtherCAT master userspace library is distributed in the hope that
+ *  it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+ *  warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with the IgH EtherCAT master userspace library. If not, see
+ *  <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+/** \file
+ *
+ * EtherCAT virtual TTY interface.
+ *
+ * \defgroup TTYInterface EtherCAT Virtual TTY Interface
+ *
+ * @{
+ */
+
+/****************************************************************************/
+
+#ifndef __ECTTY_H__
+#define __ECTTY_H__
+
+#include <linux/termios.h>
+
+/*****************************************************************************
+ * Data types
+ ****************************************************************************/
+
+struct ec_tty;
+typedef struct ec_tty ec_tty_t; /**< \see ec_tty */
+
+/** Operations on the virtual TTY interface.
+ */
+typedef struct {
+    int (*cflag_changed)(void *, tcflag_t); /**< Called when the serial
+                                              * settings shall be changed. The
+                                              * \a cflag argument contains the
+                                              * new settings. */
+} ec_tty_operations_t;
+
+/*****************************************************************************
+ * Global functions
+ ****************************************************************************/
+
+/** Create a virtual TTY interface.
+ *
+ * \param ops Set of callbacks.
+ * \param cb_data Arbitrary data, that is passed to any callback.
+ *
+ * \return Pointer to the interface object, otherwise an ERR_PTR value.
+ */
+ec_tty_t *ectty_create(
+        const ec_tty_operations_t *ops,
+        void *cb_data
+        );
+
+/*****************************************************************************
+ * TTY interface methods
+ ****************************************************************************/
+
+/** Releases a virtual TTY interface.
+ */
+void ectty_free(
+        ec_tty_t *tty /**< TTY interface. */
+        );
+
+/** Reads data to send from the TTY interface.
+ *
+ * If there are data to send, they are copied into the \a buffer. At maximum,
+ * \a size bytes are copied. The actual number of bytes copied is returned.
+ *
+ * \return Number of bytes copied.
+ */
+unsigned int ectty_tx_data(
+        ec_tty_t *tty, /**< TTY interface. */
+        uint8_t *buffer, /**< Buffer for data to transmit. */
+        size_t size /**< Available space in \a buffer. */
+        );
+
+/** Pushes received data to the TTY interface.
+ */
+void ectty_rx_data(
+        ec_tty_t *tty, /**< TTY interface. */
+        const uint8_t *buffer, /**< Buffer with received data. */
+        size_t size /**< Number of bytes in \a buffer. */
+        );
+
+/****************************************************************************/
+
+/** @} */
+
+#endif
diff --git a/drivers/net/ethercat/include/globals.h b/drivers/net/ethercat/include/globals.h
new file mode 100644
index 000000000000..d2a470e9ce8c
--- /dev/null
+++ b/drivers/net/ethercat/include/globals.h
@@ -0,0 +1,55 @@
+/******************************************************************************
+ *
+ *  Copyright (C) 2006-2021  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT master.
+ *
+ *  The file is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU Lesser General Public License as published by the
+ *  Free Software Foundation; version 2.1 of the License.
+ *
+ *  This file is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+ *  License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this file. If not, see <http://www.gnu.org/licenses/>.
+ *
+ *****************************************************************************/
+
+/**
+   \file
+   Global definitions and macros.
+*/
+
+/*****************************************************************************/
+
+#ifndef __EC_GLOBALS_H__
+#define __EC_GLOBALS_H__
+
+#include "config.h"
+
+/******************************************************************************
+ *  Overall macros
+ *****************************************************************************/
+
+/** Helper macro for EC_STR(), literates a macro argument.
+ *
+ * \param X argument to literate.
+ */
+#define EC_LIT(X) #X
+
+/** Converts a macro argument to a string.
+ *
+ * \param X argument to stringify.
+ */
+#define EC_STR(X) EC_LIT(X)
+
+/** Master version string
+ */
+#define EC_MASTER_VERSION VERSION " " EC_STR(REV)
+
+/*****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/Kconfig b/drivers/net/ethercat/master/Kconfig
new file mode 100644
index 000000000000..ed0f89b89b86
--- /dev/null
+++ b/drivers/net/ethercat/master/Kconfig
@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# EtherCAT master support
+#
+
+config EC_MASTER
+    tristate "EtherCAT master (IgH)"
+    depends on ETHERCAT
+    help
+      Say Y or M here to build the IgH EtherCAT master driver.
+      The master is controlled via a character device interface.
+      If unsure, say N.
+
+if EC_MASTER
+
+config EC_MASTER_OF
+    bool "Use Device Tree for EtherCAT master setup"
+    depends on OF
+    default y
+    help
+      Use Device Tree to provide EtherCAT master configuration,
+      including master instances and bound Ethernet devices.
+
+      This mode is intended for platform-integrated setups where
+      EtherCAT master parameters are fixed by the board description,
+      so runtime module parameters are not needed.
+
+      If disabled, the original upstream module-parameter based
+      configuration path is used instead.
+
+if EC_MASTER_OF
+
+config EC_MASTER_RUN_ON_CPU
+    int "Run EtherCAT master on CPU"
+    default 1
+    range 0 7
+    help
+      Set the CPU (0-7) on which to run the EtherCAT master.
+      If unsure, the default is 1.
+
+config EC_MASTER_DEBUG_LEVEL
+    int "Debug level for EtherCAT master"
+    default 0
+    range 0 2
+    help
+      Set the debug level for the EtherCAT master driver.
+      0: No debug information.
+      1: Error messages.
+      2: Full debug information.
+
+endif # EC_MASTER_OF
+
+endif # EC_MASTER
diff --git a/drivers/net/ethercat/master/Makefile b/drivers/net/ethercat/master/Makefile
new file mode 100644
index 000000000000..89268246779b
--- /dev/null
+++ b/drivers/net/ethercat/master/Makefile
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for the IgH EtherCAT master driver.
+#
+
+obj-$(CONFIG_EC_MASTER) += ec_master.o
+
+ec_master-y += \
+	cdev.o \
+	coe_emerg_ring.o \
+	datagram.o \
+	datagram_pair.o \
+	device.o \
+	domain.o \
+	flag.o \
+	fmmu_config.o \
+	foe_request.o \
+	fsm_change.o \
+	fsm_coe.o \
+	fsm_foe.o \
+	fsm_master.o \
+	fsm_pdo.o \
+	fsm_pdo_entry.o \
+	fsm_sii.o \
+	fsm_slave.o \
+	fsm_slave_config.o \
+	fsm_slave_scan.o \
+	fsm_soe.o \
+	ioctl.o \
+	mailbox.o \
+	master.o \
+	pdo.o \
+	pdo_entry.o \
+	pdo_list.o \
+	reg_request.o \
+	sdo.o \
+	sdo_entry.o \
+	sdo_request.o \
+	slave.o \
+	slave_config.o \
+	soe_errors.o \
+	soe_request.o \
+	sync.o \
+	sync_config.o \
+	voe_handler.o \
+	eoe_request.o \
+	ethernet.o \
+	fsm_eoe.o
+
+ifdef CONFIG_EC_MASTER_OF
+ec_master-y += module_of.o
+else
+ec_master-y += module.o
+endif
diff --git a/drivers/net/ethercat/master/cdev.c b/drivers/net/ethercat/master/cdev.c
new file mode 100644
index 000000000000..eb95bfe94372
--- /dev/null
+++ b/drivers/net/ethercat/master/cdev.c
@@ -0,0 +1,281 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2020  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT master character device.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+
+#include "cdev.h"
+#include "master.h"
+#include "slave_config.h"
+#include "voe_handler.h"
+#include "ethernet.h"
+#include "ioctl.h"
+
+/** Set to 1 to enable device operations debugging.
+ */
+#define DEBUG 0
+
+/****************************************************************************/
+
+static int eccdev_open(struct inode *, struct file *);
+static int eccdev_release(struct inode *, struct file *);
+static long eccdev_ioctl(struct file *, unsigned int, unsigned long);
+static int eccdev_mmap(struct file *, struct vm_area_struct *);
+
+/** This is the kernel version from which the .fault member of the
+ * vm_operations_struct is usable.
+ */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)
+# define FAULT_RETURN_TYPE int
+#else
+# define FAULT_RETURN_TYPE vm_fault_t
+#endif
+
+static FAULT_RETURN_TYPE eccdev_vma_fault(
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
+        struct vm_area_struct *,
+#endif
+        struct vm_fault *);
+
+/****************************************************************************/
+
+/** File operation callbacks for the EtherCAT character device.
+ */
+static struct file_operations eccdev_fops = {
+    .owner          = THIS_MODULE,
+    .open           = eccdev_open,
+    .release        = eccdev_release,
+    .unlocked_ioctl = eccdev_ioctl,
+    .mmap           = eccdev_mmap
+};
+
+/** Callbacks for a virtual memory area retrieved with ecdevc_mmap().
+ */
+struct vm_operations_struct eccdev_vm_ops = {
+    .fault = eccdev_vma_fault
+};
+
+/****************************************************************************/
+
+/** Private data structure for file handles.
+ */
+typedef struct {
+    ec_cdev_t *cdev; /**< Character device. */
+    ec_ioctl_context_t ctx; /**< Context. */
+} ec_cdev_priv_t;
+
+/****************************************************************************/
+
+/** Constructor.
+ *
+ * \return 0 in case of success, else < 0
+ */
+int ec_cdev_init(
+        ec_cdev_t *cdev, /**< EtherCAT master character device. */
+        ec_master_t *master, /**< Parent master. */
+        dev_t dev_num /**< Device number. */
+        )
+{
+    int ret;
+
+    cdev->master = master;
+
+    cdev_init(&cdev->cdev, &eccdev_fops);
+    cdev->cdev.owner = THIS_MODULE;
+
+    ret = cdev_add(&cdev->cdev,
+            MKDEV(MAJOR(dev_num), master->index), 1);
+    if (ret) {
+        EC_MASTER_ERR(master, "Failed to add character device!\n");
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_cdev_clear(ec_cdev_t *cdev /**< EtherCAT XML device */)
+{
+    cdev_del(&cdev->cdev);
+}
+
+/*****************************************************************************
+ * File operations
+ ****************************************************************************/
+
+/** Called when the cdev is opened.
+ */
+int eccdev_open(struct inode *inode, struct file *filp)
+{
+    ec_cdev_t *cdev = container_of(inode->i_cdev, ec_cdev_t, cdev);
+    ec_cdev_priv_t *priv;
+
+    priv = kmalloc(sizeof(ec_cdev_priv_t), GFP_KERNEL);
+    if (!priv) {
+        EC_MASTER_ERR(cdev->master,
+                "Failed to allocate memory for private data structure.\n");
+        return -ENOMEM;
+    }
+
+    priv->cdev = cdev;
+    priv->ctx.writable = (filp->f_mode & FMODE_WRITE) != 0;
+    priv->ctx.requested = 0;
+    priv->ctx.process_data = NULL;
+    priv->ctx.process_data_size = 0;
+
+    filp->private_data = priv;
+
+#if DEBUG
+    EC_MASTER_DBG(cdev->master, 0, "File opened.\n");
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Called when the cdev is closed.
+ */
+int eccdev_release(struct inode *inode, struct file *filp)
+{
+    ec_cdev_priv_t *priv = (ec_cdev_priv_t *) filp->private_data;
+    ec_master_t *master = priv->cdev->master;
+
+    if (priv->ctx.requested) {
+        ecrt_release_master(master);
+    }
+
+    if (priv->ctx.process_data) {
+        vfree(priv->ctx.process_data);
+    }
+
+#if DEBUG
+    EC_MASTER_DBG(master, 0, "File closed.\n");
+#endif
+
+    kfree(priv);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Called when an ioctl() command is issued.
+ */
+long eccdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+    ec_cdev_priv_t *priv = (ec_cdev_priv_t *) filp->private_data;
+
+#if DEBUG
+    EC_MASTER_DBG(priv->cdev->master, 0,
+            "ioctl(filp = 0x%p, cmd = 0x%08x (0x%02x), arg = 0x%lx)\n",
+            filp, cmd, _IOC_NR(cmd), arg);
+#endif
+
+    return ec_ioctl(priv->cdev->master, &priv->ctx, cmd, (void __user *) arg);
+}
+
+/****************************************************************************/
+
+#ifndef VM_DONTDUMP
+/** VM_RESERVED disappeared in 3.7.
+ */
+#define VM_DONTDUMP VM_RESERVED
+#endif
+
+/** Memory-map callback for the EtherCAT character device.
+ *
+ * The actual mapping will be done in the eccdev_vma_nopage() callback of the
+ * virtual memory area.
+ *
+ * \return Always zero (success).
+ */
+int eccdev_mmap(
+        struct file *filp,
+        struct vm_area_struct *vma
+        )
+{
+    ec_cdev_priv_t *priv = (ec_cdev_priv_t *) filp->private_data;
+
+    EC_MASTER_DBG(priv->cdev->master, 1, "mmap()\n");
+
+    vma->vm_ops = &eccdev_vm_ops;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)
+    vm_flags_set(vma, VM_DONTDUMP);
+#else
+    vma->vm_flags |= VM_DONTDUMP; /* Pages will not be swapped out */
+#endif
+    vma->vm_private_data = priv;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Page fault callback for a virtual memory area.
+ *
+ * Called at the first access on a virtual-memory area retrieved with
+ * ecdev_mmap().
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static FAULT_RETURN_TYPE eccdev_vma_fault(
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
+        struct vm_area_struct *vma, /**< Virtual memory area. */
+#endif
+        struct vm_fault *vmf /**< Fault data. */
+        )
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+    struct vm_area_struct *vma = vmf->vma;
+#endif
+    unsigned long offset = vmf->pgoff << PAGE_SHIFT;
+    ec_cdev_priv_t *priv = (ec_cdev_priv_t *) vma->vm_private_data;
+    struct page *page;
+
+    if (offset >= priv->ctx.process_data_size) {
+        return VM_FAULT_SIGBUS;
+    }
+
+    page = vmalloc_to_page(priv->ctx.process_data + offset);
+    if (!page) {
+        return VM_FAULT_SIGBUS;
+    }
+
+    get_page(page);
+    vmf->page = page;
+
+    EC_MASTER_DBG(priv->cdev->master, 1, "Vma fault,"
+            " offset = %lu, page = %p\n", offset, page);
+
+    return 0;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/cdev.h b/drivers/net/ethercat/master/cdev.h
new file mode 100644
index 000000000000..825d360ee0d5
--- /dev/null
+++ b/drivers/net/ethercat/master/cdev.h
@@ -0,0 +1,53 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT master character device.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_CDEV_H__
+#define __EC_CDEV_H__
+
+#include <linux/fs.h>
+#include <linux/cdev.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** EtherCAT master character device.
+*/
+typedef struct {
+    ec_master_t *master; /**< Master owning the device. */
+    struct cdev cdev; /**< Character device. */
+} ec_cdev_t;
+
+/****************************************************************************/
+
+int ec_cdev_init(ec_cdev_t *, ec_master_t *, dev_t);
+void ec_cdev_clear(ec_cdev_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/coe_emerg_ring.c b/drivers/net/ethercat/master/coe_emerg_ring.c
new file mode 100644
index 000000000000..b0026c3b8085
--- /dev/null
+++ b/drivers/net/ethercat/master/coe_emerg_ring.c
@@ -0,0 +1,169 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *  vim: expandtab
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT CoE emergency ring buffer methods.
+ */
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+
+#include "coe_emerg_ring.h"
+
+/****************************************************************************/
+
+/** Emergency ring buffer constructor.
+ */
+void ec_coe_emerg_ring_init(
+        ec_coe_emerg_ring_t *ring, /**< Emergency ring. */
+        ec_slave_config_t *sc /**< Slave configuration. */
+        )
+{
+    ring->sc = sc;
+    ring->msgs = NULL;
+    ring->size = 0;
+    ring->read_index = 0;
+    ring->write_index = 0;
+    ring->overruns = 0;
+}
+
+/****************************************************************************/
+
+/** Emergency ring buffer destructor.
+ */
+void ec_coe_emerg_ring_clear(
+        ec_coe_emerg_ring_t *ring /**< Emergency ring. */
+        )
+{
+    if (ring->msgs) {
+        kfree(ring->msgs);
+    }
+}
+
+/****************************************************************************/
+
+/** Set the ring size.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_coe_emerg_ring_size(
+        ec_coe_emerg_ring_t *ring, /**< Emergency ring. */
+        size_t size /**< Maximum number of messages in the ring. */
+        )
+{
+    ring->size = 0;
+
+    if (size < 0) {
+        size = 0;
+    }
+
+    ring->read_index = ring->write_index = 0;
+
+    if (ring->msgs) {
+        kfree(ring->msgs);
+    }
+    ring->msgs = NULL;
+
+    if (size == 0) {
+        return 0;
+    }
+
+    ring->msgs = kmalloc(sizeof(ec_coe_emerg_msg_t) * (size + 1), GFP_KERNEL);
+    if (!ring->msgs) {
+        return -ENOMEM;
+    }
+
+    ring->size = size;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Add a new emergency message.
+ */
+void ec_coe_emerg_ring_push(
+        ec_coe_emerg_ring_t *ring, /**< Emergency ring. */
+        const u8 *msg /**< Emergency message. */
+        )
+{
+    if (!ring->size ||
+            (ring->write_index + 1) % (ring->size + 1) == ring->read_index) {
+        ring->overruns++;
+        return;
+    }
+
+    memcpy(ring->msgs[ring->write_index].data, msg,
+            EC_COE_EMERGENCY_MSG_SIZE);
+    ring->write_index = (ring->write_index + 1) % (ring->size + 1);
+}
+
+/****************************************************************************/
+
+/** Remove an emergency message from the ring.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_coe_emerg_ring_pop(
+        ec_coe_emerg_ring_t *ring, /**< Emergency ring. */
+        u8 *msg /**< Memory to store the emergency message. */
+        )
+{
+    if (ring->read_index == ring->write_index) {
+        return -ENOENT;
+    }
+
+    memcpy(msg, ring->msgs[ring->read_index].data, EC_COE_EMERGENCY_MSG_SIZE);
+    ring->read_index = (ring->read_index + 1) % (ring->size + 1);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Clear the ring.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_coe_emerg_ring_clear_ring(
+        ec_coe_emerg_ring_t *ring /**< Emergency ring. */
+        )
+{
+    ring->read_index = ring->write_index;
+    ring->overruns = 0;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Read the number of overruns.
+ *
+ * \return Number of overruns.
+ */
+int ec_coe_emerg_ring_overruns(
+        const ec_coe_emerg_ring_t *ring /**< Emergency ring. */
+        )
+{
+    return ring->overruns;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/coe_emerg_ring.h b/drivers/net/ethercat/master/coe_emerg_ring.h
new file mode 100644
index 000000000000..36ab2c41f43c
--- /dev/null
+++ b/drivers/net/ethercat/master/coe_emerg_ring.h
@@ -0,0 +1,70 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT CoE emergency ring buffer structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_COE_EMERG_RING_H__
+#define __EC_COE_EMERG_RING_H__
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** EtherCAT CoE emergency message record.
+ */
+typedef struct {
+    u8 data[EC_COE_EMERGENCY_MSG_SIZE]; /**< Message data. */
+} ec_coe_emerg_msg_t;
+
+/****************************************************************************/
+
+/** EtherCAT CoE emergency ring buffer.
+ */
+typedef struct {
+    ec_slave_config_t *sc; /**< Slave configuration  owning the ring. */
+
+    ec_coe_emerg_msg_t *msgs; /**< Message ring. */
+    size_t size; /**< Ring size. */
+
+    unsigned int read_index; /**< Read index. */
+    unsigned int write_index; /**< Write index. */
+    unsigned int overruns; /**< Number of overruns since last reset. */
+} ec_coe_emerg_ring_t;
+
+/****************************************************************************/
+
+void ec_coe_emerg_ring_init(ec_coe_emerg_ring_t *, ec_slave_config_t *);
+void ec_coe_emerg_ring_clear(ec_coe_emerg_ring_t *);
+
+int ec_coe_emerg_ring_size(ec_coe_emerg_ring_t *, size_t);
+void ec_coe_emerg_ring_push(ec_coe_emerg_ring_t *, const u8 *);
+int ec_coe_emerg_ring_pop(ec_coe_emerg_ring_t *, u8 *);
+int ec_coe_emerg_ring_clear_ring(ec_coe_emerg_ring_t *);
+int ec_coe_emerg_ring_overruns(const ec_coe_emerg_ring_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/datagram.c b/drivers/net/ethercat/master/datagram.c
new file mode 100644
index 000000000000..c0f3360afd6e
--- /dev/null
+++ b/drivers/net/ethercat/master/datagram.c
@@ -0,0 +1,644 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Methods of an EtherCAT datagram.
+*/
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+
+#include "datagram.h"
+#include "master.h"
+
+/****************************************************************************/
+
+/** \cond */
+
+#define EC_FUNC_HEADER \
+    ret = ec_datagram_prealloc(datagram, data_size); \
+    if (unlikely(ret)) \
+        return ret; \
+    datagram->index = 0; \
+    datagram->working_counter = 0; \
+    datagram->state = EC_DATAGRAM_INIT;
+
+#define EC_FUNC_FOOTER \
+    datagram->data_size = data_size; \
+    return 0;
+
+/** \endcond */
+
+/****************************************************************************/
+
+/** Array of datagram type strings used in ec_datagram_type_string().
+ *
+ * \attention This is indexed by ec_datagram_type_t.
+ */
+static const char *type_strings[] = {
+    "?",
+    "APRD",
+    "APWR",
+    "APRW",
+    "FPRD",
+    "FPWR",
+    "FPRW",
+    "BRD",
+    "BWR",
+    "BRW",
+    "LRD",
+    "LWR",
+    "LRW",
+    "ARMW",
+    "FRMW"
+};
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_datagram_init(ec_datagram_t *datagram /**< EtherCAT datagram. */)
+{
+    INIT_LIST_HEAD(&datagram->queue); // mark as unqueued
+    datagram->device_index = EC_DEVICE_MAIN;
+    datagram->type = EC_DATAGRAM_NONE;
+    memset(datagram->address, 0x00, EC_ADDR_LEN);
+    datagram->data = NULL;
+    datagram->data_origin = EC_ORIG_INTERNAL;
+    datagram->mem_size = 0;
+    datagram->data_size = 0;
+    datagram->index = 0x00;
+    datagram->working_counter = 0x0000;
+    datagram->state = EC_DATAGRAM_INIT;
+#ifdef EC_HAVE_CYCLES
+    datagram->cycles_sent = 0;
+#endif
+    datagram->jiffies_sent = 0;
+#ifdef EC_HAVE_CYCLES
+    datagram->cycles_received = 0;
+#endif
+    datagram->jiffies_received = 0;
+    datagram->skip_count = 0;
+    datagram->stats_output_jiffies = 0;
+    memset(datagram->name, 0x00, EC_DATAGRAM_NAME_SIZE);
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_datagram_clear(ec_datagram_t *datagram /**< EtherCAT datagram. */)
+{
+    ec_datagram_unqueue(datagram);
+
+    if (datagram->data_origin == EC_ORIG_INTERNAL && datagram->data) {
+        kfree(datagram->data);
+        datagram->data = NULL;
+    }
+}
+
+/****************************************************************************/
+
+/** Unqueue datagram.
+ */
+void ec_datagram_unqueue(ec_datagram_t *datagram /**< EtherCAT datagram. */)
+{
+    if (!list_empty(&datagram->queue)) {
+        list_del_init(&datagram->queue);
+    }
+}
+
+/****************************************************************************/
+
+/** Allocates internal payload memory.
+ *
+ * If the allocated memory is already larger than requested, nothing ist done.
+ *
+ * \attention If external payload memory has been provided, no range checking
+ *            is done!
+ *
+ * \return 0 in case of success, otherwise \a -ENOMEM.
+ */
+int ec_datagram_prealloc(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        size_t size /**< New payload size in bytes. */
+        )
+{
+    if (datagram->data_origin == EC_ORIG_EXTERNAL
+            || size <= datagram->mem_size)
+        return 0;
+
+    if (datagram->data) {
+        kfree(datagram->data);
+        datagram->data = NULL;
+        datagram->mem_size = 0;
+    }
+
+    if (!(datagram->data = kmalloc(size, GFP_KERNEL))) {
+        EC_ERR("Failed to allocate %zu bytes of datagram memory!\n", size);
+        return -ENOMEM;
+    }
+
+    datagram->mem_size = size;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Fills the datagram payload memory with zeros.
+ */
+void ec_datagram_zero(ec_datagram_t *datagram /**< EtherCAT datagram. */)
+{
+    memset(datagram->data, 0x00, datagram->data_size);
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT APRD datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_aprd(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t ring_position, /**< Auto-increment address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to read. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_APRD;
+    EC_WRITE_S16(datagram->address, (int16_t) ring_position * (-1));
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT APWR datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_apwr(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t ring_position, /**< Auto-increment address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to write. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_APWR;
+    EC_WRITE_S16(datagram->address, (int16_t) ring_position * (-1));
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT APRW datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_aprw(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t ring_position, /**< Auto-increment address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to write. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_APRW;
+    EC_WRITE_S16(datagram->address, (int16_t) ring_position * (-1));
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT ARMW datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_armw(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t ring_position, /**< Auto-increment address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to read. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_ARMW;
+    EC_WRITE_S16(datagram->address, (int16_t) ring_position * (-1));
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT FPRD datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_fprd(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t configured_address, /**< Configured station address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to read. */
+        )
+{
+    int ret;
+
+    if (unlikely(configured_address == 0x0000))
+        EC_WARN("Using configured station address 0x0000!\n");
+
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_FPRD;
+    EC_WRITE_U16(datagram->address, configured_address);
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT FPWR datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_fpwr(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t configured_address, /**< Configured station address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to write. */
+        )
+{
+    int ret;
+
+    if (unlikely(configured_address == 0x0000))
+        EC_WARN("Using configured station address 0x0000!\n");
+
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_FPWR;
+    EC_WRITE_U16(datagram->address, configured_address);
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT FPRW datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_fprw(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t configured_address, /**< Configured station address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to write. */
+        )
+{
+    int ret;
+
+    if (unlikely(configured_address == 0x0000))
+        EC_WARN("Using configured station address 0x0000!\n");
+
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_FPRW;
+    EC_WRITE_U16(datagram->address, configured_address);
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT FRMW datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_frmw(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t configured_address, /**< Configured station address. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to write. */
+        )
+{
+    int ret;
+
+    if (unlikely(configured_address == 0x0000))
+        EC_WARN("Using configured station address 0x0000!\n");
+
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_FRMW;
+    EC_WRITE_U16(datagram->address, configured_address);
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT BRD datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_brd(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to read. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_BRD;
+    EC_WRITE_U16(datagram->address, 0x0000);
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT BWR datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_bwr(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to write. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_BWR;
+    EC_WRITE_U16(datagram->address, 0x0000);
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT BRW datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_brw(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint16_t mem_address, /**< Physical memory address. */
+        size_t data_size /**< Number of bytes to write. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_BRW;
+    EC_WRITE_U16(datagram->address, 0x0000);
+    EC_WRITE_U16(datagram->address + 2, mem_address);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT LRD datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_lrd(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint32_t offset, /**< Logical address. */
+        size_t data_size /**< Number of bytes to read/write. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_LRD;
+    EC_WRITE_U32(datagram->address, offset);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT LWR datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_lwr(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint32_t offset, /**< Logical address. */
+        size_t data_size /**< Number of bytes to read/write. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_LWR;
+    EC_WRITE_U32(datagram->address, offset);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT LRW datagram.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_lrw(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint32_t offset, /**< Logical address. */
+        size_t data_size /**< Number of bytes to read/write. */
+        )
+{
+    int ret;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_LRW;
+    EC_WRITE_U32(datagram->address, offset);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT LRD datagram with external memory.
+ *
+ * \attention It is assumed, that the external memory is at least \a data_size
+ *            bytes large.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_lrd_ext(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint32_t offset, /**< Logical address. */
+        size_t data_size, /**< Number of bytes to read/write. */
+        uint8_t *external_memory /**< Pointer to the memory to use. */
+        )
+{
+    int ret;
+    datagram->data = external_memory;
+    datagram->data_origin = EC_ORIG_EXTERNAL;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_LRD;
+    EC_WRITE_U32(datagram->address, offset);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT LWR datagram with external memory.
+ *
+ * \attention It is assumed, that the external memory is at least \a data_size
+ *            bytes large.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_lwr_ext(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint32_t offset, /**< Logical address. */
+        size_t data_size, /**< Number of bytes to read/write. */
+        uint8_t *external_memory /**< Pointer to the memory to use. */
+        )
+{
+    int ret;
+    datagram->data = external_memory;
+    datagram->data_origin = EC_ORIG_EXTERNAL;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_LWR;
+    EC_WRITE_U32(datagram->address, offset);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Initializes an EtherCAT LRW datagram with external memory.
+ *
+ * \attention It is assumed, that the external memory is at least \a data_size
+ *            bytes large.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_datagram_lrw_ext(
+        ec_datagram_t *datagram, /**< EtherCAT datagram. */
+        uint32_t offset, /**< Logical address. */
+        size_t data_size, /**< Number of bytes to read/write. */
+        uint8_t *external_memory /**< Pointer to the memory to use. */
+        )
+{
+    int ret;
+    datagram->data = external_memory;
+    datagram->data_origin = EC_ORIG_EXTERNAL;
+    EC_FUNC_HEADER;
+    datagram->type = EC_DATAGRAM_LRW;
+    EC_WRITE_U32(datagram->address, offset);
+    EC_FUNC_FOOTER;
+}
+
+/****************************************************************************/
+
+/** Prints the state of a datagram.
+ *
+ * Outputs a text message.
+ */
+void ec_datagram_print_state(
+        const ec_datagram_t *datagram /**< EtherCAT datagram */
+        )
+{
+    printk(KERN_CONT "Datagram ");
+    switch (datagram->state) {
+        case EC_DATAGRAM_INIT:
+            printk(KERN_CONT "initialized");
+            break;
+        case EC_DATAGRAM_QUEUED:
+            printk(KERN_CONT "queued");
+            break;
+        case EC_DATAGRAM_SENT:
+            printk(KERN_CONT "sent");
+            break;
+        case EC_DATAGRAM_RECEIVED:
+            printk(KERN_CONT "received");
+            break;
+        case EC_DATAGRAM_TIMED_OUT:
+            printk(KERN_CONT "timed out");
+            break;
+        case EC_DATAGRAM_ERROR:
+            printk(KERN_CONT "error");
+            break;
+        default:
+            printk(KERN_CONT "???");
+    }
+
+    printk(KERN_CONT ".\n");
+}
+
+/****************************************************************************/
+
+/** Evaluates the working counter of a single-cast datagram.
+ *
+ * Outputs an error message.
+ */
+void ec_datagram_print_wc_error(
+        const ec_datagram_t *datagram /**< EtherCAT datagram */
+        )
+{
+    if (datagram->working_counter == 0) {
+        printk(KERN_CONT "No response.");
+    }
+    else if (datagram->working_counter > 1) {
+        printk(KERN_CONT "%u slaves responded!", datagram->working_counter);
+    }
+    else {
+        printk(KERN_CONT "Success.");
+    }
+    printk(KERN_CONT "\n");
+}
+
+/****************************************************************************/
+
+/** Outputs datagram statistics at most every second.
+ */
+void ec_datagram_output_stats(
+        ec_datagram_t *datagram
+        )
+{
+    if (jiffies - datagram->stats_output_jiffies > HZ) {
+        datagram->stats_output_jiffies = jiffies;
+
+        if (unlikely(datagram->skip_count)) {
+            EC_WARN("Datagram %p (%s) was SKIPPED %u time%s.\n",
+                    datagram, datagram->name,
+                    datagram->skip_count,
+                    datagram->skip_count == 1 ? "" : "s");
+            datagram->skip_count = 0;
+        }
+    }
+}
+
+/****************************************************************************/
+
+/** Returns a string describing the datagram type.
+ *
+ * \return Pointer on a static memory containing the requested string.
+ */
+const char *ec_datagram_type_string(
+        const ec_datagram_t *datagram /**< EtherCAT datagram. */
+        )
+{
+    return type_strings[datagram->type];
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/datagram.h b/drivers/net/ethercat/master/datagram.h
new file mode 100644
index 000000000000..131849286300
--- /dev/null
+++ b/drivers/net/ethercat/master/datagram.h
@@ -0,0 +1,142 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT datagram structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_DATAGRAM_H__
+#define __EC_DATAGRAM_H__
+
+#include <linux/list.h>
+#include <linux/time.h>
+#include <linux/timex.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** EtherCAT datagram type.
+ */
+typedef enum {
+    EC_DATAGRAM_NONE = 0x00, /**< Dummy. */
+    EC_DATAGRAM_APRD = 0x01, /**< Auto Increment Physical Read. */
+    EC_DATAGRAM_APWR = 0x02, /**< Auto Increment Physical Write. */
+    EC_DATAGRAM_APRW = 0x03, /**< Auto Increment Physical ReadWrite. */
+    EC_DATAGRAM_FPRD = 0x04, /**< Configured Address Physical Read. */
+    EC_DATAGRAM_FPWR = 0x05, /**< Configured Address Physical Write. */
+    EC_DATAGRAM_FPRW = 0x06, /**< Configured Address Physical ReadWrite. */
+    EC_DATAGRAM_BRD  = 0x07, /**< Broadcast Read. */
+    EC_DATAGRAM_BWR  = 0x08, /**< Broadcast Write. */
+    EC_DATAGRAM_BRW  = 0x09, /**< Broadcast ReadWrite. */
+    EC_DATAGRAM_LRD  = 0x0A, /**< Logical Read. */
+    EC_DATAGRAM_LWR  = 0x0B, /**< Logical Write. */
+    EC_DATAGRAM_LRW  = 0x0C, /**< Logical ReadWrite. */
+    EC_DATAGRAM_ARMW = 0x0D, /**< Auto Increment Physical Read Multiple
+                               Write.  */
+    EC_DATAGRAM_FRMW = 0x0E, /**< Configured Address Physical Read Multiple
+                               Write. */
+} ec_datagram_type_t;
+
+/****************************************************************************/
+
+/** EtherCAT datagram state.
+ */
+typedef enum {
+    EC_DATAGRAM_INIT,      /**< Initial state of a new datagram. */
+    EC_DATAGRAM_QUEUED,    /**< Queued for sending. */
+    EC_DATAGRAM_SENT,      /**< Sent (still in the queue). */
+    EC_DATAGRAM_RECEIVED,  /**< Received (dequeued). */
+    EC_DATAGRAM_TIMED_OUT, /**< Timed out (dequeued). */
+    EC_DATAGRAM_ERROR      /**< Error while sending/receiving (dequeued). */
+} ec_datagram_state_t;
+
+/****************************************************************************/
+
+/** EtherCAT datagram.
+ */
+typedef struct {
+    struct list_head queue; /**< Master datagram queue item,
+        protected by user-supplied mutex. */
+    struct list_head ext_queue; /**< External datagram queue item, protected by ext_queue_sem. */
+    struct list_head sent; /**< Master list item for sent datagrams. */
+    ec_device_index_t device_index; /**< Device via which the datagram shall
+                                      be / was sent. */
+    ec_datagram_type_t type; /**< Datagram type (APRD, BWR, etc.). */
+    uint8_t address[EC_ADDR_LEN]; /**< Recipient address. */
+    uint8_t *data; /**< Datagram payload. */
+    ec_origin_t data_origin; /**< Origin of the \a data memory. */
+    size_t mem_size; /**< Datagram \a data memory size. */
+    size_t data_size; /**< Size of the data in \a data. */
+    uint8_t index; /**< Index (set by master). */
+    uint16_t working_counter; /**< Working counter. */
+    ec_datagram_state_t state; /**< State. */
+#ifdef EC_HAVE_CYCLES
+    cycles_t cycles_sent; /**< Time, when the datagram was sent. */
+#endif
+    unsigned long jiffies_sent; /**< Jiffies, when the datagram was sent. */
+#ifdef EC_HAVE_CYCLES
+    cycles_t cycles_received; /**< Time, when the datagram was received. */
+#endif
+    unsigned long jiffies_received; /**< Jiffies, when the datagram was
+                                      received. */
+    unsigned int skip_count; /**< Number of requeues when not yet received. */
+    unsigned long stats_output_jiffies; /**< Last statistics output. */
+    char name[EC_DATAGRAM_NAME_SIZE]; /**< Description of the datagram. */
+} ec_datagram_t;
+
+/****************************************************************************/
+
+void ec_datagram_init(ec_datagram_t *);
+void ec_datagram_clear(ec_datagram_t *);
+void ec_datagram_unqueue(ec_datagram_t *);
+int ec_datagram_prealloc(ec_datagram_t *, size_t);
+void ec_datagram_zero(ec_datagram_t *);
+
+int ec_datagram_aprd(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_apwr(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_aprw(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_armw(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_fprd(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_fpwr(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_fprw(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_frmw(ec_datagram_t *, uint16_t, uint16_t, size_t);
+int ec_datagram_brd(ec_datagram_t *, uint16_t, size_t);
+int ec_datagram_bwr(ec_datagram_t *, uint16_t, size_t);
+int ec_datagram_brw(ec_datagram_t *, uint16_t, size_t);
+int ec_datagram_lrd(ec_datagram_t *, uint32_t, size_t);
+int ec_datagram_lwr(ec_datagram_t *, uint32_t, size_t);
+int ec_datagram_lrw(ec_datagram_t *, uint32_t, size_t);
+int ec_datagram_lrd_ext(ec_datagram_t *, uint32_t, size_t, uint8_t *);
+int ec_datagram_lwr_ext(ec_datagram_t *, uint32_t, size_t, uint8_t *);
+int ec_datagram_lrw_ext(ec_datagram_t *, uint32_t, size_t, uint8_t *);
+
+void ec_datagram_print_state(const ec_datagram_t *);
+void ec_datagram_print_wc_error(const ec_datagram_t *);
+void ec_datagram_output_stats(ec_datagram_t *);
+const char *ec_datagram_type_string(const ec_datagram_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/datagram_pair.c b/drivers/net/ethercat/master/datagram_pair.c
new file mode 100644
index 000000000000..3e3d28388658
--- /dev/null
+++ b/drivers/net/ethercat/master/datagram_pair.c
@@ -0,0 +1,193 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT datagram pair methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+
+#include "master.h"
+#include "datagram_pair.h"
+
+/****************************************************************************/
+
+/** Datagram pair constructor.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_datagram_pair_init(
+        ec_datagram_pair_t *pair, /**< Datagram pair. */
+        ec_domain_t *domain, /**< Parent domain. */
+        uint32_t logical_offset, /**< Logical offset. */
+        uint8_t *data, /**< Data pointer. */
+        size_t data_size, /**< Data size. */
+        const unsigned int used[] /**< input/output use count. */
+        )
+{
+    ec_device_index_t dev_idx;
+    int ret;
+
+    INIT_LIST_HEAD(&pair->list);
+    pair->domain = domain;
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        ec_datagram_init(&pair->datagrams[dev_idx]);
+        snprintf(pair->datagrams[dev_idx].name,
+                EC_DATAGRAM_NAME_SIZE, "domain%u-%u-%s", domain->index,
+                logical_offset, ec_device_names[dev_idx != 0]);
+        pair->datagrams[dev_idx].device_index = dev_idx;
+    }
+
+    pair->expected_working_counter = 0U;
+
+    for (dev_idx = EC_DEVICE_BACKUP;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        /* backup datagrams have their own memory */
+        ret = ec_datagram_prealloc(&pair->datagrams[dev_idx], data_size);
+        if (ret) {
+            goto out_datagrams;
+        }
+    }
+
+#if EC_MAX_NUM_DEVICES > 1
+    if (!(pair->send_buffer = kmalloc(data_size, GFP_KERNEL))) {
+        EC_MASTER_ERR(domain->master,
+                "Failed to allocate domain send buffer!\n");
+        ret = -ENOMEM;
+        goto out_datagrams;
+    }
+#endif
+
+    /* The ec_datagram_lxx() calls below can not fail, because either the
+     * datagram has external memory or it is preallocated. */
+
+    if (used[EC_DIR_OUTPUT] && used[EC_DIR_INPUT]) { // inputs and outputs
+        ec_datagram_lrw_ext(&pair->datagrams[EC_DEVICE_MAIN],
+                logical_offset, data_size, data);
+
+        for (dev_idx = EC_DEVICE_BACKUP;
+                dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+            ec_datagram_lrw(&pair->datagrams[dev_idx],
+                    logical_offset, data_size);
+        }
+
+        // If LRW is used, output FMMUs increment the working counter by 2,
+        // while input FMMUs increment it by 1.
+        pair->expected_working_counter =
+            used[EC_DIR_OUTPUT] * 2 + used[EC_DIR_INPUT];
+    } else if (used[EC_DIR_OUTPUT]) { // outputs only
+        ec_datagram_lwr_ext(&pair->datagrams[EC_DEVICE_MAIN],
+                logical_offset, data_size, data);
+        for (dev_idx = EC_DEVICE_BACKUP;
+                dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+            ec_datagram_lwr(&pair->datagrams[dev_idx],
+                    logical_offset, data_size);
+        }
+
+        pair->expected_working_counter = used[EC_DIR_OUTPUT];
+    } else { // inputs only (or nothing)
+        ec_datagram_lrd_ext(&pair->datagrams[EC_DEVICE_MAIN],
+                logical_offset, data_size, data);
+        for (dev_idx = EC_DEVICE_BACKUP;
+                dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+            ec_datagram_lrd(&pair->datagrams[dev_idx], logical_offset,
+                    data_size);
+        }
+
+        pair->expected_working_counter = used[EC_DIR_INPUT];
+    }
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        ec_datagram_zero(&pair->datagrams[dev_idx]);
+    }
+
+    return 0;
+
+out_datagrams:
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        ec_datagram_clear(&pair->datagrams[dev_idx]);
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Datagram pair destructor.
+ */
+void ec_datagram_pair_clear(
+        ec_datagram_pair_t *pair /**< Datagram pair. */
+        )
+{
+    unsigned int dev_idx;
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(pair->domain->master);
+            dev_idx++) {
+        ec_datagram_clear(&pair->datagrams[dev_idx]);
+    }
+
+#if EC_MAX_NUM_DEVICES > 1
+    if (pair->send_buffer) {
+        kfree(pair->send_buffer);
+    }
+#endif
+}
+
+/****************************************************************************/
+
+/** Process received data.
+ *
+ * \return Working counter sum over all devices.
+ */
+uint16_t ec_datagram_pair_process(
+        ec_datagram_pair_t *pair, /**< Datagram pair. */
+        uint16_t wc_sum[] /**< Working counter sums. */
+        )
+{
+    unsigned int dev_idx;
+    uint16_t pair_wc = 0;
+
+    for (dev_idx = 0; dev_idx < ec_master_num_devices(pair->domain->master);
+            dev_idx++) {
+        ec_datagram_t *datagram = &pair->datagrams[dev_idx];
+
+#ifdef EC_RT_SYSLOG
+        ec_datagram_output_stats(datagram);
+#endif
+
+        if (datagram->state == EC_DATAGRAM_RECEIVED) {
+            pair_wc += datagram->working_counter;
+            wc_sum[dev_idx] += datagram->working_counter;
+        }
+    }
+
+    return pair_wc;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/datagram_pair.h b/drivers/net/ethercat/master/datagram_pair.h
new file mode 100644
index 000000000000..0294a2162019
--- /dev/null
+++ b/drivers/net/ethercat/master/datagram_pair.h
@@ -0,0 +1,61 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT datagram pair structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_DATAGRAM_PAIR_H__
+#define __EC_DATAGRAM_PAIR_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+#include "datagram.h"
+
+/****************************************************************************/
+
+/** Domain datagram pair.
+ */
+typedef struct {
+    struct list_head list; /**< List header. */
+    ec_domain_t *domain; /**< Parent domain. */
+    ec_datagram_t datagrams[EC_MAX_NUM_DEVICES]; /**< Datagrams.  */
+#if EC_MAX_NUM_DEVICES > 1
+    uint8_t *send_buffer;
+#endif
+    unsigned int expected_working_counter; /**< Expectord working conter. */
+} ec_datagram_pair_t;
+
+/****************************************************************************/
+
+int ec_datagram_pair_init(ec_datagram_pair_t *, ec_domain_t *, uint32_t,
+        uint8_t *, size_t, const unsigned int []);
+void ec_datagram_pair_clear(ec_datagram_pair_t *);
+
+uint16_t ec_datagram_pair_process(ec_datagram_pair_t *, uint16_t[]);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/debug.c b/drivers/net/ethercat/master/debug.c
new file mode 100644
index 000000000000..268a6bb7128a
--- /dev/null
+++ b/drivers/net/ethercat/master/debug.c
@@ -0,0 +1,258 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Ethernet interface for debugging purposes.
+*/
+
+/****************************************************************************/
+
+#include <linux/version.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+
+#include "globals.h"
+#include "master.h"
+#include "debug.h"
+
+/****************************************************************************/
+
+// net_device functions
+int ec_dbgdev_open(struct net_device *);
+int ec_dbgdev_stop(struct net_device *);
+int ec_dbgdev_tx(struct sk_buff *, struct net_device *);
+struct net_device_stats *ec_dbgdev_stats(struct net_device *);
+
+/** Device operations for debug interfaces.
+ */
+static const struct net_device_ops ec_dbg_netdev_ops =
+{
+    .ndo_open = ec_dbgdev_open,
+    .ndo_stop = ec_dbgdev_stop,
+    .ndo_start_xmit = ec_dbgdev_tx,
+    .ndo_get_stats = ec_dbgdev_stats,
+};
+
+/****************************************************************************/
+
+/** Debug interface constructor.
+ *
+ * Initializes the debug object, creates a net_device and registeres it.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_debug_init(
+        ec_debug_t *dbg, /**< Debug object. */
+        ec_device_t *device, /**< EtherCAT device. */
+        const char *name /**< Interface name. */
+        )
+{
+    dbg->device = device;
+    dbg->registered = 0;
+    dbg->opened = 0;
+
+    memset(&dbg->stats, 0, sizeof(struct net_device_stats));
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+    dbg->dev = alloc_netdev(sizeof(ec_debug_t *), name, NET_NAME_UNKNOWN, ether_setup);
+#else
+    dbg->dev = alloc_netdev(sizeof(ec_debug_t *), name, ether_setup);
+#endif
+    if (!(dbg->dev))
+    {
+        EC_MASTER_ERR(device->master, "Unable to allocate net_device"
+                " for debug object!\n");
+        return -ENODEV;
+    }
+
+    // initialize net_device
+    dbg->dev->netdev_ops = &ec_dbg_netdev_ops;
+
+    // initialize private data
+    *((ec_debug_t **) netdev_priv(dbg->dev)) = dbg;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Debug interface destructor.
+ *
+ * Unregisters the net_device and frees allocated memory.
+ */
+void ec_debug_clear(
+        ec_debug_t *dbg /**< debug object */
+        )
+{
+    ec_debug_unregister(dbg);
+    free_netdev(dbg->dev);
+}
+
+/****************************************************************************/
+
+/** Register debug interface.
+ */
+void ec_debug_register(
+        ec_debug_t *dbg, /**< debug object */
+        const struct net_device *net_dev /**< 'Real' Ethernet device. */
+        )
+{
+    int result;
+
+    ec_debug_unregister(dbg);
+
+    // use the Ethernet address of the physical device for the debug device
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)
+    eth_hw_addr_set(dbg->dev, net_dev->dev_addr);
+#else
+    memcpy(dbg->dev->dev_addr, net_dev->dev_addr, ETH_ALEN);
+#endif
+
+    // connect the net_device to the kernel
+    if ((result = register_netdev(dbg->dev))) {
+        EC_MASTER_WARN(dbg->device->master, "Unable to register net_device:"
+                " error %i\n", result);
+    } else {
+        dbg->registered = 1;
+    }
+}
+
+/****************************************************************************/
+
+/** Unregister debug interface.
+ */
+void ec_debug_unregister(
+        ec_debug_t *dbg /**< debug object */
+        )
+{
+    if (dbg->registered) {
+        dbg->opened = 0;
+        dbg->registered = 0;
+        unregister_netdev(dbg->dev);
+    }
+}
+
+/****************************************************************************/
+
+/** Sends frame data to the interface.
+ */
+void ec_debug_send(
+        ec_debug_t *dbg, /**< debug object */
+        const uint8_t *data, /**< frame data */
+        size_t size /**< size of the frame data */
+        )
+{
+    struct sk_buff *skb;
+
+    if (!dbg->opened)
+        return;
+
+    // allocate socket buffer
+    if (!(skb = dev_alloc_skb(size))) {
+        dbg->stats.rx_dropped++;
+        return;
+    }
+
+    // copy frame contents into socket buffer
+    memcpy(skb_put(skb, size), data, size);
+
+    // update device statistics
+    dbg->stats.rx_packets++;
+    dbg->stats.rx_bytes += size;
+
+    // pass socket buffer to network stack
+    skb->dev = dbg->dev;
+    skb->protocol = eth_type_trans(skb, dbg->dev);
+    skb->ip_summed = CHECKSUM_UNNECESSARY;
+    netif_rx(skb);
+}
+
+/*****************************************************************************
+ *  NET_DEVICE functions
+ ****************************************************************************/
+
+/** Opens the virtual network device.
+ *
+ * \return Always zero (success).
+ */
+int ec_dbgdev_open(
+        struct net_device *dev /**< debug net_device */
+        )
+{
+    ec_debug_t *dbg = *((ec_debug_t **) netdev_priv(dev));
+    dbg->opened = 1;
+    EC_MASTER_INFO(dbg->device->master, "Debug interface %s opened.\n",
+            dev->name);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Stops the virtual network device.
+ *
+ * \return Always zero (success).
+ */
+int ec_dbgdev_stop(
+        struct net_device *dev /**< debug net_device */
+        )
+{
+    ec_debug_t *dbg = *((ec_debug_t **) netdev_priv(dev));
+    dbg->opened = 0;
+    EC_MASTER_INFO(dbg->device->master, "Debug interface %s stopped.\n",
+            dev->name);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Transmits data via the virtual network device.
+ *
+ * \return Always zero (success).
+ */
+int ec_dbgdev_tx(
+        struct sk_buff *skb, /**< transmit socket buffer */
+        struct net_device *dev /**< EoE net_device */
+        )
+{
+    ec_debug_t *dbg = *((ec_debug_t **) netdev_priv(dev));
+
+    dev_kfree_skb(skb);
+    dbg->stats.tx_dropped++;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Gets statistics about the virtual network device.
+ *
+ * \return Statistics.
+ */
+struct net_device_stats *ec_dbgdev_stats(
+        struct net_device *dev /**< debug net_device */
+        )
+{
+    ec_debug_t *dbg = *((ec_debug_t **) netdev_priv(dev));
+    return &dbg->stats;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/debug.h b/drivers/net/ethercat/master/debug.h
new file mode 100644
index 000000000000..5d435d4c788f
--- /dev/null
+++ b/drivers/net/ethercat/master/debug.h
@@ -0,0 +1,58 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Network interface for debugging purposes.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_DEBUG_H__
+#define __EC_DEBUG_H__
+
+#include "../devices/ecdev.h"
+
+/****************************************************************************/
+
+/** Debugging network interface.
+ */
+typedef struct
+{
+    ec_device_t *device; /**< Parent device. */
+    struct net_device *dev; /**< net_device for virtual ethernet device */
+    struct net_device_stats stats; /**< device statistics */
+    uint8_t registered; /**< net_device is opened */
+    uint8_t opened; /**< net_device is opened */
+}
+ec_debug_t;
+
+/****************************************************************************/
+
+int ec_debug_init(ec_debug_t *, ec_device_t *, const char *);
+void ec_debug_clear(ec_debug_t *);
+void ec_debug_register(ec_debug_t *, const struct net_device *);
+void ec_debug_unregister(ec_debug_t *);
+void ec_debug_send(ec_debug_t *, const uint8_t *, size_t);
+
+#endif
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/device.c b/drivers/net/ethercat/master/device.c
new file mode 100644
index 000000000000..598d163a9722
--- /dev/null
+++ b/drivers/net/ethercat/master/device.c
@@ -0,0 +1,716 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT device methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/skbuff.h>
+#include <linux/if_ether.h>
+#include <linux/netdevice.h>
+
+#include "device.h"
+#include "master.h"
+
+#ifdef EC_DEBUG_RING
+#define timersub(a, b, result) \
+    do { \
+        (result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \
+        (result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \
+        if ((result)->tv_usec < 0) { \
+            --(result)->tv_sec; \
+            (result)->tv_usec += 1000000; \
+        } \
+    } while (0)
+#endif
+
+/****************************************************************************/
+
+enum {
+    /* genet driver needs extra headroom in skb for status block */
+    EXTRA_HEADROOM = 64,
+};
+
+/** Constructor.
+ *
+ * \return 0 in case of success, else < 0
+ */
+int ec_device_init(
+        ec_device_t *device, /**< EtherCAT device */
+        ec_master_t *master /**< master owning the device */
+        )
+{
+    int ret;
+    unsigned int i;
+    struct ethhdr *eth;
+#ifdef EC_DEBUG_IF
+    char ifname[10];
+    char mb = 'x';
+#endif
+
+    device->master = master;
+    device->dev = NULL;
+    device->poll = NULL;
+    device->module = NULL;
+    device->open = 0;
+    device->link_state = 0;
+    for (i = 0; i < EC_TX_RING_SIZE; i++) {
+        device->tx_skb[i] = NULL;
+    }
+    device->tx_ring_index = 0;
+#ifdef EC_HAVE_CYCLES
+    device->cycles_poll = 0;
+#endif
+#ifdef EC_DEBUG_RING
+    device->timeval_poll.tv_sec = 0;
+    device->timeval_poll.tv_usec = 0;
+#endif
+    device->jiffies_poll = 0;
+
+    ec_device_clear_stats(device);
+
+#ifdef EC_DEBUG_RING
+    for (i = 0; i < EC_DEBUG_RING_SIZE; i++) {
+        ec_debug_frame_t *df = &device->debug_frames[i];
+        df->dir = TX;
+        df->t.tv_sec = 0;
+        df->t.tv_usec = 0;
+        memset(df->data, 0, EC_MAX_DATA_SIZE);
+        df->data_size = 0;
+    }
+#endif
+#ifdef EC_DEBUG_RING
+    device->debug_frame_index = 0;
+    device->debug_frame_count = 0;
+#endif
+
+#ifdef EC_DEBUG_IF
+    if (device == &master->devices[EC_DEVICE_MAIN]) {
+        mb = 'm';
+    }
+    else {
+        mb = 'b';
+    }
+
+    sprintf(ifname, "ecdbg%c%u", mb, master->index);
+
+    ret = ec_debug_init(&device->dbg, device, ifname);
+    if (ret < 0) {
+        EC_MASTER_ERR(master, "Failed to init debug device!\n");
+        goto out_return;
+    }
+#endif
+
+    for (i = 0; i < EC_TX_RING_SIZE; i++) {
+        if (!(device->tx_skb[i] = dev_alloc_skb(ETH_FRAME_LEN + EXTRA_HEADROOM))) {
+            EC_MASTER_ERR(master, "Error allocating device socket buffer!\n");
+            ret = -ENOMEM;
+            goto out_tx_ring;
+        }
+
+        // add Ethernet-II-header
+        skb_reserve(device->tx_skb[i], ETH_HLEN + EXTRA_HEADROOM);
+        eth = (struct ethhdr *) skb_push(device->tx_skb[i], ETH_HLEN);
+        eth->h_proto = htons(0x88A4);
+        memset(eth->h_dest, 0xFF, ETH_ALEN);
+    }
+
+    return 0;
+
+out_tx_ring:
+    for (i = 0; i < EC_TX_RING_SIZE; i++) {
+        if (device->tx_skb[i]) {
+            dev_kfree_skb(device->tx_skb[i]);
+        }
+    }
+#ifdef EC_DEBUG_IF
+    ec_debug_clear(&device->dbg);
+out_return:
+#endif
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_device_clear(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+    unsigned int i;
+
+    if (device->open) {
+        ec_device_close(device);
+    }
+    for (i = 0; i < EC_TX_RING_SIZE; i++)
+        dev_kfree_skb(device->tx_skb[i]);
+#ifdef EC_DEBUG_IF
+    ec_debug_clear(&device->dbg);
+#endif
+}
+
+/****************************************************************************/
+
+/** Associate with net_device.
+ */
+void ec_device_attach(
+        ec_device_t *device, /**< EtherCAT device */
+        struct net_device *net_dev, /**< net_device structure */
+        ec_pollfunc_t poll, /**< pointer to device's poll function */
+        struct module *module /**< the device's module */
+        )
+{
+    unsigned int i;
+    struct ethhdr *eth;
+
+    ec_device_detach(device); // resets fields
+
+    device->dev = net_dev;
+    device->poll = poll;
+    device->module = module;
+
+    for (i = 0; i < EC_TX_RING_SIZE; i++) {
+        device->tx_skb[i]->dev = net_dev;
+        eth = (struct ethhdr *) (device->tx_skb[i]->data);
+        memcpy(eth->h_source, net_dev->dev_addr, ETH_ALEN);
+    }
+
+#ifdef EC_DEBUG_IF
+    ec_debug_register(&device->dbg, net_dev);
+#endif
+}
+
+/****************************************************************************/
+
+/** Disconnect from net_device.
+ */
+void ec_device_detach(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+    unsigned int i;
+
+#ifdef EC_DEBUG_IF
+    ec_debug_unregister(&device->dbg);
+#endif
+
+    device->dev = NULL;
+    device->poll = NULL;
+    device->module = NULL;
+    device->open = 0;
+    device->link_state = 0; // down
+
+    ec_device_clear_stats(device);
+
+    for (i = 0; i < EC_TX_RING_SIZE; i++) {
+        device->tx_skb[i]->dev = NULL;
+    }
+}
+
+/****************************************************************************/
+
+/** Opens the EtherCAT device.
+ *
+ * \return 0 in case of success, else < 0
+ */
+int ec_device_open(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+    int ret;
+
+    if (!device->dev) {
+        EC_MASTER_ERR(device->master, "No net_device to open!\n");
+        return -ENODEV;
+    }
+
+    if (device->open) {
+        EC_MASTER_WARN(device->master, "Device already opened!\n");
+        return 0;
+    }
+
+    device->link_state = 0;
+
+    ec_device_clear_stats(device);
+
+    ret = device->dev->netdev_ops->ndo_open(device->dev);
+    if (!ret)
+        device->open = 1;
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Stops the EtherCAT device.
+ *
+ * \return 0 in case of success, else < 0
+ */
+int ec_device_close(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+    int ret;
+
+    if (!device->dev) {
+        EC_MASTER_ERR(device->master, "No device to close!\n");
+        return -ENODEV;
+    }
+
+    if (!device->open) {
+        EC_MASTER_WARN(device->master, "Device already closed!\n");
+        return 0;
+    }
+
+    ret = device->dev->netdev_ops->ndo_stop(device->dev);
+    if (!ret)
+        device->open = 0;
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Returns a pointer to the device's transmit memory.
+ *
+ * \return pointer to the TX socket buffer
+ */
+uint8_t *ec_device_tx_data(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+    /* cycle through socket buffers, because otherwise there is a race
+     * condition, if multiple frames are sent and the DMA is not scheduled in
+     * between. */
+    device->tx_ring_index++;
+    device->tx_ring_index %= EC_TX_RING_SIZE;
+    return device->tx_skb[device->tx_ring_index]->data + ETH_HLEN;
+}
+
+/****************************************************************************/
+
+/** Sends the content of the transmit socket buffer.
+ *
+ * Cuts the socket buffer content to the (now known) size, and calls the
+ * start_xmit() function of the assigned net_device.
+ */
+void ec_device_send(
+        ec_device_t *device, /**< EtherCAT device */
+        size_t size /**< number of bytes to send */
+        )
+{
+    struct sk_buff *skb = device->tx_skb[device->tx_ring_index];
+
+    // set the right length for the data
+    skb->len = ETH_HLEN + size;
+
+    if (unlikely(device->master->debug_level > 1)) {
+        EC_MASTER_DBG(device->master, 2, "Sending frame:\n");
+        ec_print_data(skb->data, ETH_HLEN + size);
+    }
+
+    // start sending
+    if (device->dev->netdev_ops->ndo_start_xmit(skb, device->dev) ==
+            NETDEV_TX_OK)
+    {
+        device->tx_count++;
+        device->master->device_stats.tx_count++;
+        device->tx_bytes += ETH_HLEN + size;
+        device->master->device_stats.tx_bytes += ETH_HLEN + size;
+#ifdef EC_DEBUG_IF
+        ec_debug_send(&device->dbg, skb->data, ETH_HLEN + size);
+#endif
+#ifdef EC_DEBUG_RING
+        ec_device_debug_ring_append(
+                device, TX, skb->data + ETH_HLEN, size);
+#endif
+    } else {
+        device->tx_errors++;
+    }
+}
+
+/****************************************************************************/
+
+/** Clears the frame statistics.
+ */
+void ec_device_clear_stats(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+    unsigned int i;
+
+    // zero frame statistics
+    device->tx_count = 0;
+    device->last_tx_count = 0;
+    device->rx_count = 0;
+    device->last_rx_count = 0;
+    device->tx_bytes = 0;
+    device->last_tx_bytes = 0;
+    device->rx_bytes = 0;
+    device->last_rx_bytes = 0;
+    device->tx_errors = 0;
+
+    for (i = 0; i < EC_RATE_COUNT; i++) {
+        device->tx_frame_rates[i] = 0;
+        device->rx_frame_rates[i] = 0;
+        device->tx_byte_rates[i] = 0;
+        device->rx_byte_rates[i] = 0;
+    }
+}
+
+/****************************************************************************/
+
+#ifdef EC_DEBUG_RING
+/** Appends frame data to the debug ring.
+ */
+void ec_device_debug_ring_append(
+        ec_device_t *device, /**< EtherCAT device */
+        ec_debug_frame_dir_t dir, /**< direction */
+        const void *data, /**< frame data */
+        size_t size /**< data size */
+        )
+{
+    ec_debug_frame_t *df = &device->debug_frames[device->debug_frame_index];
+
+    df->dir = dir;
+    if (dir == TX) {
+        do_gettimeofday(&df->t);
+    }
+    else {
+        df->t = device->timeval_poll;
+    }
+    memcpy(df->data, data, size);
+    df->data_size = size;
+
+    device->debug_frame_index++;
+    device->debug_frame_index %= EC_DEBUG_RING_SIZE;
+    if (unlikely(device->debug_frame_count < EC_DEBUG_RING_SIZE))
+        device->debug_frame_count++;
+}
+
+/****************************************************************************/
+
+/** Outputs the debug ring.
+ */
+void ec_device_debug_ring_print(
+        const ec_device_t *device /**< EtherCAT device */
+        )
+{
+    int i;
+    unsigned int ring_index;
+    const ec_debug_frame_t *df;
+    struct timeval t0, diff;
+
+    // calculate index of the newest frame in the ring to get its time
+    ring_index = (device->debug_frame_index + EC_DEBUG_RING_SIZE - 1)
+        % EC_DEBUG_RING_SIZE;
+    t0 = device->debug_frames[ring_index].t;
+
+    EC_MASTER_DBG(device->master, 1, "Debug ring %u:\n", ring_index);
+
+    // calculate index of the oldest frame in the ring
+    ring_index = (device->debug_frame_index + EC_DEBUG_RING_SIZE
+            - device->debug_frame_count) % EC_DEBUG_RING_SIZE;
+
+    for (i = 0; i < device->debug_frame_count; i++) {
+        df = &device->debug_frames[ring_index];
+        timersub(&t0, &df->t, &diff);
+
+        EC_MASTER_DBG(device->master, 1, "Frame %u, dt=%u.%06u s, %s:\n",
+                i + 1 - device->debug_frame_count,
+                (unsigned int) diff.tv_sec,
+                (unsigned int) diff.tv_usec,
+                (df->dir == TX) ? "TX" : "RX");
+        ec_print_data(df->data, df->data_size);
+
+        ring_index++;
+        ring_index %= EC_DEBUG_RING_SIZE;
+    }
+}
+#endif
+
+/****************************************************************************/
+
+/** Calls the poll function of the assigned net_device.
+ *
+ * The master itself works without using interrupts. Therefore the processing
+ * of received data and status changes of the network device has to be
+ * done by the master calling the ISR "manually".
+ */
+void ec_device_poll(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+#ifdef EC_HAVE_CYCLES
+    device->cycles_poll = get_cycles();
+#endif
+    device->jiffies_poll = jiffies;
+#ifdef EC_DEBUG_RING
+    do_gettimeofday(&device->timeval_poll);
+#endif
+    device->poll(device->dev);
+}
+
+/****************************************************************************/
+
+/** Update device statistics.
+ */
+void ec_device_update_stats(
+        ec_device_t *device /**< EtherCAT device */
+        )
+{
+    unsigned int i;
+
+    s32 tx_frame_rate = (device->tx_count - device->last_tx_count) * 1000;
+    s32 rx_frame_rate = (device->rx_count - device->last_rx_count) * 1000;
+    s32 tx_byte_rate = (device->tx_bytes - device->last_tx_bytes);
+    s32 rx_byte_rate = (device->rx_bytes - device->last_rx_bytes);
+
+    /* Low-pass filter:
+     *      Y_n = y_(n - 1) + T / tau * (x - y_(n - 1))   | T = 1
+     *   -> Y_n += (x - y_(n - 1)) / tau
+     */
+    for (i = 0; i < EC_RATE_COUNT; i++) {
+        s32 n = rate_intervals[i];
+        device->tx_frame_rates[i] +=
+            (tx_frame_rate - device->tx_frame_rates[i]) / n;
+        device->rx_frame_rates[i] +=
+            (rx_frame_rate - device->rx_frame_rates[i]) / n;
+        device->tx_byte_rates[i] +=
+            (tx_byte_rate - device->tx_byte_rates[i]) / n;
+        device->rx_byte_rates[i] +=
+            (rx_byte_rate - device->rx_byte_rates[i]) / n;
+    }
+
+    device->last_tx_count = device->tx_count;
+    device->last_rx_count = device->rx_count;
+    device->last_tx_bytes = device->tx_bytes;
+    device->last_rx_bytes = device->rx_bytes;
+}
+
+/*****************************************************************************
+ *  Device interface
+ ****************************************************************************/
+
+/** Withdraws an EtherCAT device from the master.
+ *
+ * The device is disconnected from the master and all device ressources
+ * are freed.
+ *
+ * \attention Before calling this function, the ecdev_stop() function has
+ *            to be called, to be sure that the master does not use the device
+ *            any more.
+ * \ingroup DeviceInterface
+ */
+void ecdev_withdraw(ec_device_t *device /**< EtherCAT device */)
+{
+    ec_master_t *master = device->master;
+    char dev_str[20], mac_str[20];
+
+    ec_mac_print(device->dev->dev_addr, mac_str);
+
+    if (device == &master->devices[EC_DEVICE_MAIN]) {
+        sprintf(dev_str, "main");
+    } else if (device == &master->devices[EC_DEVICE_BACKUP]) {
+        sprintf(dev_str, "backup");
+    } else {
+        EC_MASTER_WARN(master, "%s() called with unknown device %s!\n",
+                __func__, mac_str);
+        sprintf(dev_str, "UNKNOWN");
+    }
+
+    EC_MASTER_INFO(master, "Releasing %s device %s.\n", dev_str, mac_str);
+
+    down(&master->device_sem);
+    ec_device_detach(device);
+    up(&master->device_sem);
+}
+
+/****************************************************************************/
+
+/** Opens the network device and makes the master enter IDLE phase.
+ *
+ * \return 0 on success, else < 0
+ * \ingroup DeviceInterface
+ */
+int ecdev_open(ec_device_t *device /**< EtherCAT device */)
+{
+    int ret;
+    ec_master_t *master = device->master;
+    unsigned int all_open = 1, dev_idx;
+
+    ret = ec_device_open(device);
+    if (ret) {
+        EC_MASTER_ERR(master, "Failed to open device: error %d!\n", ret);
+        return ret;
+    }
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(device->master); dev_idx++) {
+        if (!master->devices[dev_idx].open) {
+            all_open = 0;
+            break;
+        }
+    }
+
+    if (all_open) {
+        ret = ec_master_enter_idle_phase(device->master);
+        if (ret) {
+            EC_MASTER_ERR(device->master, "Failed to enter IDLE phase!\n");
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Makes the master leave IDLE phase and closes the network device.
+ *
+ * \return 0 on success, else < 0
+ * \ingroup DeviceInterface
+ */
+void ecdev_close(ec_device_t *device /**< EtherCAT device */)
+{
+    ec_master_t *master = device->master;
+
+    if (master->phase == EC_IDLE) {
+        ec_master_leave_idle_phase(master);
+    }
+
+    if (ec_device_close(device)) {
+        EC_MASTER_WARN(master, "Failed to close device!\n");
+    }
+}
+
+/****************************************************************************/
+
+/** Accepts a received frame.
+ *
+ * Forwards the received data to the master. The master will analyze the frame
+ * and dispatch the received commands to the sending instances.
+ *
+ * The data have to begin with the Ethernet header (target MAC address).
+ *
+ * \ingroup DeviceInterface
+ */
+void ecdev_receive(
+        ec_device_t *device, /**< EtherCAT device */
+        const void *data, /**< pointer to received data */
+        size_t size /**< number of bytes received */
+        )
+{
+    const void *ec_data = data + ETH_HLEN;
+    size_t ec_size = size - ETH_HLEN;
+
+    if (unlikely(!data)) {
+        EC_MASTER_WARN(device->master, "%s() called with NULL data.\n",
+                __func__);
+        return;
+    }
+
+    device->rx_count++;
+    device->master->device_stats.rx_count++;
+    device->rx_bytes += size;
+    device->master->device_stats.rx_bytes += size;
+
+    if (unlikely(device->master->debug_level > 1)) {
+        EC_MASTER_DBG(device->master, 2, "Received frame:\n");
+        ec_print_data(data, size);
+    }
+
+#ifdef EC_DEBUG_IF
+    ec_debug_send(&device->dbg, data, size);
+#endif
+#ifdef EC_DEBUG_RING
+    ec_device_debug_ring_append(device, RX, ec_data, ec_size);
+#endif
+
+    ec_master_receive_datagrams(device->master, device, ec_data, ec_size);
+}
+
+/****************************************************************************/
+
+/** Sets a new link state.
+ *
+ * If the device notifies the master about the link being down, the master
+ * will not try to send frames using this device.
+ *
+ * \ingroup DeviceInterface
+ */
+void ecdev_set_link(
+        ec_device_t *device, /**< EtherCAT device */
+        uint8_t state /**< new link state */
+        )
+{
+    if (unlikely(!device)) {
+        EC_WARN("ecdev_set_link() called with null device!\n");
+        return;
+    }
+
+    if (likely(state != device->link_state)) {
+        device->link_state = state;
+        EC_MASTER_INFO(device->master,
+                "Link state of %s changed to %s.\n",
+                device->dev->name, (state ? "UP" : "DOWN"));
+    }
+}
+
+/****************************************************************************/
+
+/** Reads the link state.
+ *
+ * \ingroup DeviceInterface
+ *
+ * \return Link state.
+ */
+uint8_t ecdev_get_link(
+        const ec_device_t *device /**< EtherCAT device */
+        )
+{
+    if (unlikely(!device)) {
+        EC_WARN("ecdev_get_link() called with null device!\n");
+        return 0;
+    }
+
+    return device->link_state;
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecdev_withdraw);
+EXPORT_SYMBOL(ecdev_open);
+EXPORT_SYMBOL(ecdev_close);
+EXPORT_SYMBOL(ecdev_receive);
+EXPORT_SYMBOL(ecdev_get_link);
+EXPORT_SYMBOL(ecdev_set_link);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/device.h b/drivers/net/ethercat/master/device.h
new file mode 100644
index 000000000000..fe7c0c725e65
--- /dev/null
+++ b/drivers/net/ethercat/master/device.h
@@ -0,0 +1,150 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT device structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_DEVICE_H__
+#define __EC_DEVICE_H__
+
+#include <linux/interrupt.h>
+
+#include "../device/ecdev.h"
+#include "globals.h"
+
+/**
+ * Size of the transmit ring.
+ * This memory ring is used to transmit frames. It is necessary to use
+ * different memory regions, because otherwise the network device DMA could
+ * send the same data twice, if it is called twice.
+ */
+#define EC_TX_RING_SIZE 2
+
+#ifdef EC_DEBUG_IF
+#include "debug.h"
+#endif
+
+#ifdef EC_DEBUG_RING
+#define EC_DEBUG_RING_SIZE 10
+
+typedef enum {
+    TX, RX
+} ec_debug_frame_dir_t;
+
+typedef struct {
+    ec_debug_frame_dir_t dir;
+    struct timeval t;
+    uint8_t data[EC_MAX_DATA_SIZE];
+    unsigned int data_size;
+} ec_debug_frame_t;
+
+#endif
+
+/****************************************************************************/
+
+/**
+   EtherCAT device.
+   An EtherCAT device is a network interface card, that is owned by an
+   EtherCAT master to send and receive EtherCAT frames with.
+*/
+
+struct ec_device
+{
+    ec_master_t *master; /**< EtherCAT master */
+    struct net_device *dev; /**< pointer to the assigned net_device */
+    ec_pollfunc_t poll; /**< pointer to the device's poll function */
+    struct module *module; /**< pointer to the device's owning module */
+    uint8_t open; /**< true, if the net_device has been opened */
+    uint8_t link_state; /**< device link state */
+    struct sk_buff *tx_skb[EC_TX_RING_SIZE]; /**< transmit skb ring */
+    unsigned int tx_ring_index; /**< last ring entry used to transmit */
+#ifdef EC_HAVE_CYCLES
+    cycles_t cycles_poll; /**< cycles of last poll */
+#endif
+#ifdef EC_DEBUG_RING
+    struct timeval timeval_poll;
+#endif
+    unsigned long jiffies_poll; /**< jiffies of last poll */
+
+    // Frame statistics
+    u64 tx_count; /**< Number of frames sent. */
+    u64 last_tx_count; /**< Number of frames sent of last statistics cycle. */
+    u64 rx_count; /**< Number of frames received. */
+    u64 last_rx_count; /**< Number of frames received of last statistics
+                         cycle. */
+    u64 tx_bytes; /**< Number of bytes sent. */
+    u64 last_tx_bytes; /**< Number of bytes sent of last statistics cycle. */
+    u64 rx_bytes; /**< Number of bytes received. */
+    u64 last_rx_bytes; /**< Number of bytes received of last statistics cycle.
+                        */
+    u64 tx_errors; /**< Number of transmit errors. */
+    s32 tx_frame_rates[EC_RATE_COUNT]; /**< Transmit rates in frames/s for
+                                         different statistics cycle periods.
+                                        */
+    s32 rx_frame_rates[EC_RATE_COUNT]; /**< Receive rates in frames/s for
+                                         different statistics cycle periods.
+                                        */
+    s32 tx_byte_rates[EC_RATE_COUNT]; /**< Transmit rates in byte/s for
+                                        different statistics cycle periods. */
+    s32 rx_byte_rates[EC_RATE_COUNT]; /**< Receive rates in byte/s for
+                                        different statistics cycle periods. */
+
+#ifdef EC_DEBUG_IF
+    ec_debug_t dbg; /**< debug device */
+#endif
+#ifdef EC_DEBUG_RING
+    ec_debug_frame_t debug_frames[EC_DEBUG_RING_SIZE];
+    unsigned int debug_frame_index;
+    unsigned int debug_frame_count;
+#endif
+};
+
+/****************************************************************************/
+
+int ec_device_init(ec_device_t *, ec_master_t *);
+void ec_device_clear(ec_device_t *);
+
+void ec_device_attach(ec_device_t *, struct net_device *, ec_pollfunc_t,
+        struct module *);
+void ec_device_detach(ec_device_t *);
+
+int ec_device_open(ec_device_t *);
+int ec_device_close(ec_device_t *);
+
+void ec_device_poll(ec_device_t *);
+uint8_t *ec_device_tx_data(ec_device_t *);
+void ec_device_send(ec_device_t *, size_t);
+void ec_device_clear_stats(ec_device_t *);
+void ec_device_update_stats(ec_device_t *);
+
+#ifdef EC_DEBUG_RING
+void ec_device_debug_ring_append(ec_device_t *, ec_debug_frame_dir_t,
+        const void *, size_t);
+void ec_device_debug_ring_print(const ec_device_t *);
+#endif
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/domain.c b/drivers/net/ethercat/master/domain.c
new file mode 100644
index 000000000000..d74fd715b0e1
--- /dev/null
+++ b/drivers/net/ethercat/master/domain.c
@@ -0,0 +1,719 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT domain methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+
+#include "globals.h"
+#include "master.h"
+#include "slave_config.h"
+
+#include "domain.h"
+#include "datagram_pair.h"
+
+/** Extra debug output for redundancy functions.
+ */
+#define DEBUG_REDUNDANCY 0
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_domain_clear_data(ec_domain_t *);
+int ec_domain_add_datagram_pair(ec_domain_t *, uint32_t, size_t, uint8_t *,
+        const unsigned int []);
+int shall_count(const ec_fmmu_config_t *, const ec_fmmu_config_t *);
+#if EC_MAX_NUM_DEVICES > 1
+int data_changed(uint8_t *, const ec_datagram_t *, size_t, size_t);
+#endif
+
+/****************************************************************************/
+
+/** Domain constructor.
+ */
+void ec_domain_init(
+        ec_domain_t *domain, /**< EtherCAT domain. */
+        ec_master_t *master, /**< Parent master. */
+        unsigned int index /**< Index. */
+        )
+{
+    unsigned int dev_idx;
+
+    domain->master = master;
+    domain->index = index;
+    INIT_LIST_HEAD(&domain->fmmu_configs);
+    domain->data_size = 0;
+    domain->data = NULL;
+    domain->data_origin = EC_ORIG_INTERNAL;
+    domain->logical_base_address = 0x00000000;
+    INIT_LIST_HEAD(&domain->datagram_pairs);
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        domain->working_counter[dev_idx] = 0x0000;
+    }
+    domain->expected_working_counter = 0x0000;
+    domain->working_counter_changes = 0;
+    domain->redundancy_active = 0;
+    domain->notify_jiffies = 0;
+}
+
+/****************************************************************************/
+
+/** Domain destructor.
+ */
+void ec_domain_clear(ec_domain_t *domain /**< EtherCAT domain */)
+{
+    ec_datagram_pair_t *datagram_pair, *next_pair;
+
+    // dequeue and free datagrams
+    list_for_each_entry_safe(datagram_pair, next_pair,
+            &domain->datagram_pairs, list) {
+        ec_datagram_pair_clear(datagram_pair);
+        kfree(datagram_pair);
+    }
+
+    ec_domain_clear_data(domain);
+}
+
+/****************************************************************************/
+
+/** Frees internally allocated memory.
+ */
+void ec_domain_clear_data(
+        ec_domain_t *domain /**< EtherCAT domain. */
+        )
+{
+    if (domain->data_origin == EC_ORIG_INTERNAL && domain->data) {
+        kfree(domain->data);
+    }
+
+    domain->data = NULL;
+    domain->data_origin = EC_ORIG_INTERNAL;
+}
+
+/****************************************************************************/
+
+/** Adds an FMMU configuration to the domain.
+ */
+void ec_domain_add_fmmu_config(
+        ec_domain_t *domain, /**< EtherCAT domain. */
+        ec_fmmu_config_t *fmmu /**< FMMU configuration. */
+        )
+{
+    fmmu->domain = domain;
+
+    domain->data_size += fmmu->data_size;
+    list_add_tail(&fmmu->list, &domain->fmmu_configs);
+
+    EC_MASTER_DBG(domain->master, 1, "Domain %u:"
+            " Added %u bytes, total %zu.\n",
+            domain->index, fmmu->data_size, domain->data_size);
+}
+
+/****************************************************************************/
+
+/** Allocates a domain datagram pair and appends it to the list.
+ *
+ * The datagrams' types and expected working counters are determined by the
+ * number of input and output fmmus that share the datagrams.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_domain_add_datagram_pair(
+        ec_domain_t *domain, /**< EtherCAT domain. */
+        uint32_t logical_offset, /**< Logical offset. */
+        size_t data_size, /**< Size of the data. */
+        uint8_t *data, /**< Process data. */
+        const unsigned int used[] /**< Slave config counter for in/out. */
+        )
+{
+    ec_datagram_pair_t *datagram_pair;
+    int ret;
+
+    if (!(datagram_pair = kmalloc(sizeof(ec_datagram_pair_t), GFP_KERNEL))) {
+        EC_MASTER_ERR(domain->master,
+                "Failed to allocate domain datagram pair!\n");
+        return -ENOMEM;
+    }
+
+    ret = ec_datagram_pair_init(datagram_pair, domain, logical_offset, data,
+            data_size, used);
+    if (ret) {
+        kfree(datagram_pair);
+        return ret;
+    }
+
+    domain->expected_working_counter +=
+        datagram_pair->expected_working_counter;
+
+    EC_MASTER_DBG(domain->master, 1,
+            "Adding datagram pair with expected WC %u.\n",
+            datagram_pair->expected_working_counter);
+
+
+    list_add_tail(&datagram_pair->list, &domain->datagram_pairs);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Domain finish helper function.
+ *
+ * Detects, if a slave configuration has already been taken into account for
+ * a datagram's expected working counter calculation.
+ *
+ * Walks through the list of all FMMU configurations for the current datagram
+ * and ends before the current datagram.
+ *
+ * \return Non-zero if slave connfig was already counted.
+ */
+int shall_count(
+        const ec_fmmu_config_t *cur_fmmu, /**< Current FMMU with direction to
+                                            search for. */
+        const ec_fmmu_config_t *first_fmmu /**< Datagram's first FMMU. */
+        )
+{
+    for (; first_fmmu != cur_fmmu;
+            first_fmmu = list_entry(first_fmmu->list.next,
+                ec_fmmu_config_t, list)) {
+
+        if (first_fmmu->sc == cur_fmmu->sc
+                && first_fmmu->dir == cur_fmmu->dir) {
+            return 0; // was already counted
+        }
+    }
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Finishes a domain.
+ *
+ * This allocates the necessary datagrams and writes the correct logical
+ * addresses to every configured FMMU.
+ *
+ * \todo Check for FMMUs that do not fit into any datagram.
+ *
+ * \retval  0 Success
+ * \retval <0 Error code.
+ */
+int ec_domain_finish(
+        ec_domain_t *domain, /**< EtherCAT domain. */
+        uint32_t base_address /**< Logical base address. */
+        )
+{
+    uint32_t datagram_offset;
+    size_t datagram_size;
+    unsigned int datagram_count;
+    unsigned int datagram_used[EC_DIR_COUNT];
+    ec_fmmu_config_t *fmmu;
+    const ec_fmmu_config_t *datagram_first_fmmu = NULL;
+    const ec_datagram_pair_t *datagram_pair;
+    int ret;
+
+    domain->logical_base_address = base_address;
+
+    if (domain->data_size && domain->data_origin == EC_ORIG_INTERNAL) {
+        if (!(domain->data =
+                    (uint8_t *) kmalloc(domain->data_size, GFP_KERNEL))) {
+            EC_MASTER_ERR(domain->master, "Failed to allocate %zu bytes"
+                    " internal memory for domain %u!\n",
+                    domain->data_size, domain->index);
+            return -ENOMEM;
+        }
+    }
+
+    // Cycle through all domain FMMUs and
+    // - correct the logical base addresses
+    // - set up the datagrams to carry the process data
+    // - calculate the datagrams' expected working counters
+    datagram_offset = 0;
+    datagram_size = 0;
+    datagram_count = 0;
+    datagram_used[EC_DIR_OUTPUT] = 0;
+    datagram_used[EC_DIR_INPUT] = 0;
+
+    if (!list_empty(&domain->fmmu_configs)) {
+        datagram_first_fmmu =
+            list_entry(domain->fmmu_configs.next, ec_fmmu_config_t, list);
+    }
+
+    list_for_each_entry(fmmu, &domain->fmmu_configs, list) {
+
+        // Correct logical FMMU address
+        fmmu->logical_start_address += base_address;
+
+        // If the current FMMU's data do not fit in the current datagram,
+        // allocate a new one.
+        if (datagram_size + fmmu->data_size > EC_MAX_DATA_SIZE) {
+            ret = ec_domain_add_datagram_pair(domain,
+                    domain->logical_base_address + datagram_offset,
+                    datagram_size, domain->data + datagram_offset,
+                    datagram_used);
+            if (ret < 0)
+                return ret;
+
+            datagram_offset += datagram_size;
+            datagram_size = 0;
+            datagram_count++;
+            datagram_used[EC_DIR_OUTPUT] = 0;
+            datagram_used[EC_DIR_INPUT] = 0;
+            datagram_first_fmmu = fmmu;
+        }
+
+        // Increment Input/Output counter to determine datagram types
+        // and calculate expected working counters
+        if (shall_count(fmmu, datagram_first_fmmu)) {
+            datagram_used[fmmu->dir]++;
+        }
+
+        datagram_size += fmmu->data_size;
+    }
+
+    /* Allocate last datagram pair, if data are left (this is also the case if
+     * the process data fit into a single datagram) */
+    if (datagram_size) {
+        ret = ec_domain_add_datagram_pair(domain,
+                domain->logical_base_address + datagram_offset,
+                datagram_size, domain->data + datagram_offset,
+                datagram_used);
+        if (ret < 0)
+            return ret;
+        datagram_count++;
+    }
+
+    EC_MASTER_INFO(domain->master, "Domain%u: Logical address 0x%08x,"
+            " %zu byte, expected working counter %u.\n", domain->index,
+            domain->logical_base_address, domain->data_size,
+            domain->expected_working_counter);
+
+    list_for_each_entry(datagram_pair, &domain->datagram_pairs, list) {
+        const ec_datagram_t *datagram =
+            &datagram_pair->datagrams[EC_DEVICE_MAIN];
+        EC_MASTER_INFO(domain->master, "  Datagram %s: Logical offset 0x%08x,"
+                " %zu byte, type %s.\n", datagram->name,
+                EC_READ_U32(datagram->address), datagram->data_size,
+                ec_datagram_type_string(datagram));
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get the number of FMMU configurations of the domain.
+ */
+unsigned int ec_domain_fmmu_count(const ec_domain_t *domain)
+{
+    const ec_fmmu_config_t *fmmu;
+    unsigned int num = 0;
+
+    list_for_each_entry(fmmu, &domain->fmmu_configs, list) {
+        num++;
+    }
+
+    return num;
+}
+
+/****************************************************************************/
+
+/** Get a certain FMMU configuration via its position in the list.
+ *
+ * \return FMMU at position \a pos, or NULL.
+ */
+const ec_fmmu_config_t *ec_domain_find_fmmu(
+        const ec_domain_t *domain, /**< EtherCAT domain. */
+        unsigned int pos /**< List position. */
+        )
+{
+    const ec_fmmu_config_t *fmmu;
+
+    list_for_each_entry(fmmu, &domain->fmmu_configs, list) {
+        if (pos--)
+            continue;
+        return fmmu;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+#if EC_MAX_NUM_DEVICES > 1
+
+/** Process received data.
+ */
+int data_changed(
+        uint8_t *send_buffer,
+        const ec_datagram_t *datagram,
+        size_t offset,
+        size_t size
+        )
+{
+    uint8_t *sent = send_buffer + offset;
+    uint8_t *recv = datagram->data + offset;
+    size_t i;
+
+    for (i = 0; i < size; i++) {
+        if (recv[i] != sent[i]) {
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+#endif
+
+/*****************************************************************************
+ *  Application interface
+ ****************************************************************************/
+
+int ecrt_domain_reg_pdo_entry_list(ec_domain_t *domain,
+        const ec_pdo_entry_reg_t *regs)
+{
+    const ec_pdo_entry_reg_t *reg;
+    ec_slave_config_t *sc;
+    int ret;
+
+    EC_MASTER_DBG(domain->master, 1, "ecrt_domain_reg_pdo_entry_list("
+            "domain = 0x%p, regs = 0x%p)\n", domain, regs);
+
+    for (reg = regs; reg->index; reg++) {
+        sc = ecrt_master_slave_config_err(domain->master, reg->alias,
+                reg->position, reg->vendor_id, reg->product_code);
+        if (IS_ERR(sc))
+            return PTR_ERR(sc);
+
+        ret = ecrt_slave_config_reg_pdo_entry(sc, reg->index,
+                        reg->subindex, domain, reg->bit_position);
+        if (ret < 0)
+            return ret;
+
+        *reg->offset = ret;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+size_t ecrt_domain_size(const ec_domain_t *domain)
+{
+    return domain->data_size;
+}
+
+/****************************************************************************/
+
+void ecrt_domain_external_memory(ec_domain_t *domain, uint8_t *mem)
+{
+    EC_MASTER_DBG(domain->master, 1, "ecrt_domain_external_memory("
+            "domain = 0x%p, mem = 0x%p)\n", domain, mem);
+
+    down(&domain->master->master_sem);
+
+    ec_domain_clear_data(domain);
+
+    domain->data = mem;
+    domain->data_origin = EC_ORIG_EXTERNAL;
+
+    up(&domain->master->master_sem);
+}
+
+/****************************************************************************/
+
+uint8_t *ecrt_domain_data(const ec_domain_t *domain)
+{
+    return domain->data;
+}
+
+/****************************************************************************/
+
+int ecrt_domain_process(ec_domain_t *domain)
+{
+    uint16_t wc_sum[EC_MAX_NUM_DEVICES] = {}, wc_total;
+    ec_datagram_pair_t *pair;
+#if EC_MAX_NUM_DEVICES > 1
+    uint16_t datagram_pair_wc, redundant_wc;
+    unsigned int datagram_offset;
+    ec_fmmu_config_t *fmmu = list_first_entry(&domain->fmmu_configs,
+            ec_fmmu_config_t, list);
+    unsigned int redundancy;
+#endif
+    unsigned int dev_idx;
+#ifdef EC_RT_SYSLOG
+    unsigned int wc_change;
+#endif
+
+#if DEBUG_REDUNDANCY
+    EC_MASTER_DBG(domain->master, 1, "domain %u process\n", domain->index);
+#endif
+
+    list_for_each_entry(pair, &domain->datagram_pairs, list) {
+#if EC_MAX_NUM_DEVICES > 1
+        datagram_pair_wc = ec_datagram_pair_process(pair, wc_sum);
+#else
+        ec_datagram_pair_process(pair, wc_sum);
+#endif
+
+#if EC_MAX_NUM_DEVICES > 1
+        if (ec_master_num_devices(domain->master) > 1) {
+            ec_datagram_t *main_datagram = &pair->datagrams[EC_DEVICE_MAIN];
+            uint32_t logical_datagram_address =
+                EC_READ_U32(main_datagram->address);
+            size_t datagram_size = main_datagram->data_size;
+
+#if DEBUG_REDUNDANCY
+            EC_MASTER_DBG(domain->master, 1, "dgram %s log=%u\n",
+                    main_datagram->name, logical_datagram_address);
+#endif
+
+            /* Redundancy: Go through FMMU configs to detect data changes. */
+            list_for_each_entry_from(fmmu, &domain->fmmu_configs, list) {
+                ec_datagram_t *backup_datagram =
+                    &pair->datagrams[EC_DEVICE_BACKUP];
+
+                if (fmmu->dir != EC_DIR_INPUT) {
+                    continue;
+                }
+
+                if (fmmu->logical_start_address >=
+                        logical_datagram_address + datagram_size) {
+                    // fmmu data contained in next datagram pair
+                    break;
+                }
+
+                datagram_offset =
+                    fmmu->logical_start_address - logical_datagram_address;
+
+#if DEBUG_REDUNDANCY
+                EC_MASTER_DBG(domain->master, 1,
+                        "input fmmu log=%u size=%u offset=%u\n",
+                        fmmu->logical_start_address, fmmu->data_size,
+                        datagram_offset);
+                if (domain->master->debug_level > 0) {
+                    ec_print_data(pair->send_buffer + datagram_offset,
+                            fmmu->data_size);
+                    ec_print_data(main_datagram->data + datagram_offset,
+                            fmmu->data_size);
+                    ec_print_data(backup_datagram->data + datagram_offset,
+                            fmmu->data_size);
+                }
+#endif
+
+                if (data_changed(pair->send_buffer, main_datagram,
+                            datagram_offset, fmmu->data_size)) {
+                    /* data changed on main link: no copying necessary. */
+#if DEBUG_REDUNDANCY
+                    EC_MASTER_DBG(domain->master, 1, "main changed\n");
+#endif
+                } else if (data_changed(pair->send_buffer, backup_datagram,
+                            datagram_offset, fmmu->data_size)) {
+                    /* data changed on backup link: copy to main memory. */
+#if DEBUG_REDUNDANCY
+                    EC_MASTER_DBG(domain->master, 1, "backup changed\n");
+#endif
+                    memcpy(main_datagram->data + datagram_offset,
+                            backup_datagram->data + datagram_offset,
+                            fmmu->data_size);
+                } else if (datagram_pair_wc ==
+                        pair->expected_working_counter) {
+                    /* no change, but WC complete: use main data. */
+#if DEBUG_REDUNDANCY
+                    EC_MASTER_DBG(domain->master, 1,
+                            "no change but complete\n");
+#endif
+                } else {
+                    /* no change and WC incomplete: mark WC as zero to avoid
+                     * data.dependent WC flickering. */
+                    datagram_pair_wc = 0;
+#if DEBUG_REDUNDANCY
+                    EC_MASTER_DBG(domain->master, 1,
+                            "no change and incomplete\n");
+#endif
+                }
+            }
+        }
+#endif // EC_MAX_NUM_DEVICES > 1
+    }
+
+#if EC_MAX_NUM_DEVICES > 1
+    redundant_wc = 0;
+    for (dev_idx = EC_DEVICE_BACKUP;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        redundant_wc += wc_sum[dev_idx];
+    }
+
+    redundancy = redundant_wc > 0;
+    if (redundancy != domain->redundancy_active) {
+#ifdef EC_RT_SYSLOG
+        if (redundancy) {
+            EC_MASTER_WARN(domain->master,
+                    "Domain %u: Redundant link in use!\n",
+                    domain->index);
+        } else {
+            EC_MASTER_INFO(domain->master,
+                    "Domain %u: Redundant link unused again.\n",
+                    domain->index);
+        }
+#endif
+        domain->redundancy_active = redundancy;
+    }
+#else
+    domain->redundancy_active = 0;
+#endif
+
+#ifdef EC_RT_SYSLOG
+    wc_change = 0;
+#endif
+    wc_total = 0;
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        if (wc_sum[dev_idx] != domain->working_counter[dev_idx]) {
+#ifdef EC_RT_SYSLOG
+            wc_change = 1;
+#endif
+            domain->working_counter[dev_idx] = wc_sum[dev_idx];
+        }
+        wc_total += wc_sum[dev_idx];
+    }
+
+#ifdef EC_RT_SYSLOG
+    if (wc_change) {
+        domain->working_counter_changes++;
+    }
+
+    if (domain->working_counter_changes &&
+        jiffies - domain->notify_jiffies > HZ) {
+        domain->notify_jiffies = jiffies;
+        if (domain->working_counter_changes == 1) {
+            EC_MASTER_INFO(domain->master, "Domain %u: Working counter"
+                    " changed to %u/%u", domain->index,
+                    wc_total, domain->expected_working_counter);
+        } else {
+            EC_MASTER_INFO(domain->master, "Domain %u: %u working counter"
+                    " changes - now %u/%u", domain->index,
+                    domain->working_counter_changes,
+                    wc_total, domain->expected_working_counter);
+        }
+#if EC_MAX_NUM_DEVICES > 1
+        if (ec_master_num_devices(domain->master) > 1) {
+            printk(KERN_CONT " (");
+            for (dev_idx = EC_DEVICE_MAIN;
+                    dev_idx < ec_master_num_devices(domain->master);
+                    dev_idx++) {
+                printk(KERN_CONT "%u", domain->working_counter[dev_idx]);
+                if (dev_idx + 1 < ec_master_num_devices(domain->master)) {
+                    printk(KERN_CONT "+");
+                }
+            }
+            printk(KERN_CONT ")");
+        }
+#endif
+        printk(KERN_CONT ".\n");
+
+        domain->working_counter_changes = 0;
+    }
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_domain_queue(ec_domain_t *domain)
+{
+    ec_datagram_pair_t *datagram_pair;
+    ec_device_index_t dev_idx;
+
+    list_for_each_entry(datagram_pair, &domain->datagram_pairs, list) {
+
+#if EC_MAX_NUM_DEVICES > 1
+        /* copy main data to send buffer */
+        memcpy(datagram_pair->send_buffer,
+                datagram_pair->datagrams[EC_DEVICE_MAIN].data,
+                datagram_pair->datagrams[EC_DEVICE_MAIN].data_size);
+#endif
+        ec_master_queue_datagram(domain->master,
+                &datagram_pair->datagrams[EC_DEVICE_MAIN]);
+
+        /* copy main data to backup datagram */
+        for (dev_idx = EC_DEVICE_BACKUP;
+                dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+            memcpy(datagram_pair->datagrams[dev_idx].data,
+                    datagram_pair->datagrams[EC_DEVICE_MAIN].data,
+                    datagram_pair->datagrams[EC_DEVICE_MAIN].data_size);
+            ec_master_queue_datagram(domain->master,
+                    &datagram_pair->datagrams[dev_idx]);
+        }
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_domain_state(const ec_domain_t *domain, ec_domain_state_t *state)
+{
+    unsigned int dev_idx;
+    uint16_t wc = 0;
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        wc += domain->working_counter[dev_idx];
+    }
+
+    state->working_counter = wc;
+
+    if (wc) {
+        if (wc == domain->expected_working_counter) {
+            state->wc_state = EC_WC_COMPLETE;
+        } else {
+            state->wc_state = EC_WC_INCOMPLETE;
+        }
+    } else {
+        state->wc_state = EC_WC_ZERO;
+    }
+
+    state->redundancy_active = domain->redundancy_active;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecrt_domain_reg_pdo_entry_list);
+EXPORT_SYMBOL(ecrt_domain_size);
+EXPORT_SYMBOL(ecrt_domain_external_memory);
+EXPORT_SYMBOL(ecrt_domain_data);
+EXPORT_SYMBOL(ecrt_domain_process);
+EXPORT_SYMBOL(ecrt_domain_queue);
+EXPORT_SYMBOL(ecrt_domain_state);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/domain.h b/drivers/net/ethercat/master/domain.h
new file mode 100644
index 000000000000..5408628b3f09
--- /dev/null
+++ b/drivers/net/ethercat/master/domain.h
@@ -0,0 +1,82 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT domain structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_DOMAIN_H__
+#define __EC_DOMAIN_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+#include "datagram.h"
+#include "master.h"
+#include "fmmu_config.h"
+
+/****************************************************************************/
+
+/** EtherCAT domain.
+ *
+ * Handles the process data and the therefore needed datagrams of a certain
+ * group of slaves.
+ */
+struct ec_domain
+{
+    struct list_head list; /**< List item. */
+    ec_master_t *master; /**< EtherCAT master owning the domain. */
+    unsigned int index; /**< Index (just a number). */
+
+    struct list_head fmmu_configs; /**< FMMU configurations contained. */
+    size_t data_size; /**< Size of the process data. */
+    uint8_t *data; /**< Memory for the process data. */
+    ec_origin_t data_origin; /**< Origin of the \a data memory. */
+    uint32_t logical_base_address; /**< Logical offset address of the
+                                     process data. */
+    struct list_head datagram_pairs; /**< Datagrams pairs (main/backup) for
+                                       process data exchange. */
+    uint16_t working_counter[EC_MAX_NUM_DEVICES]; /**< Last working counter
+                                                values. */
+    uint16_t expected_working_counter; /**< Expected working counter. */
+    unsigned int working_counter_changes; /**< Working counter changes
+                                             since last notification. */
+    unsigned int redundancy_active; /**< Non-zero, if redundancy is in use. */
+    unsigned long notify_jiffies; /**< Time of last notification. */
+};
+
+/****************************************************************************/
+
+void ec_domain_init(ec_domain_t *, ec_master_t *, unsigned int);
+void ec_domain_clear(ec_domain_t *);
+
+void ec_domain_add_fmmu_config(ec_domain_t *, ec_fmmu_config_t *);
+int ec_domain_finish(ec_domain_t *, uint32_t);
+
+unsigned int ec_domain_fmmu_count(const ec_domain_t *);
+const ec_fmmu_config_t *ec_domain_find_fmmu(const ec_domain_t *, unsigned int);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/doxygen.c b/drivers/net/ethercat/master/doxygen.c
new file mode 100644
index 000000000000..a036af2c8d54
--- /dev/null
+++ b/drivers/net/ethercat/master/doxygen.c
@@ -0,0 +1,82 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+// This file only contains the doxygen mainpage.
+
+/** \file
+ * Doxygen mainpage only.
+ */
+
+/****************************************************************************/
+
+/**
+   \mainpage
+
+   \section sec_general General information
+
+   This HTML contains the complete code documentation.
+
+   The API documentations are in the <a href="modules.html">modules
+   section</a>.
+
+   A list of all native network card drivers can be found
+   <a href="devicedrivers.html">here</a>.
+
+   A <a href="libfakeethercat.html">second userspace library</a> can be used for a dry-run mode
+   or simulating Process Data.
+
+   For information how to build and install, see the INSTALL file in the source
+   root.
+
+   \section sec_contact Contact
+
+   \verbatim
+   Florian Pose <fp@igh.de>
+   Ingenieurgemeinschaft IgH
+   Nordsternstraße 66
+   D-45329 Essen
+   http://igh.de
+   \endverbatim
+
+   \section sec_license License
+
+   \verbatim
+   Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+
+   This file is part of the IgH EtherCAT Master.
+
+   The IgH EtherCAT Master is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License version 2, as
+   published by the Free Software Foundation.
+
+   The IgH EtherCAT Master is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+   Public License for more details.
+
+   You should have received a copy of the GNU General Public License along
+   with the IgH EtherCAT Master; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+   \endverbatim
+*/
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/eoe_request.c b/drivers/net/ethercat/master/eoe_request.c
new file mode 100644
index 000000000000..f336cf340f41
--- /dev/null
+++ b/drivers/net/ethercat/master/eoe_request.c
@@ -0,0 +1,80 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * Ethernet-over-EtherCAT request functions.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/jiffies.h>
+#include <linux/slab.h>
+
+#include "eoe_request.h"
+
+/****************************************************************************/
+
+/** EoE request constructor.
+ */
+void ec_eoe_request_init(
+        ec_eoe_request_t *req /**< EoE request. */
+        )
+{
+    INIT_LIST_HEAD(&req->list);
+    req->state = EC_INT_REQUEST_INIT;
+    req->jiffies_sent = 0U;
+
+    req->mac_address_included = 0;
+    req->ip_address_included = 0;
+    req->subnet_mask_included = 0;
+    req->gateway_included = 0;
+    req->dns_included = 0;
+    req->name_included = 0;
+
+    memset(req->mac_address, 0x00, ETH_ALEN);
+    req->ip_address.s_addr = 0;
+    req->subnet_mask.s_addr = 0;
+    req->gateway.s_addr = 0;
+    req->dns.s_addr = 0;
+    req->name[0] = 0x00;
+
+    req->result = 0x0000;
+}
+
+/****************************************************************************/
+
+/** Checks if EoE request has something to set.
+ */
+int ec_eoe_request_valid(
+        const ec_eoe_request_t *req /**< EoE request. */
+        )
+{
+    return
+        req->mac_address_included ||
+        req->ip_address_included ||
+        req->subnet_mask_included ||
+        req->gateway_included ||
+        req->dns_included ||
+        req->name_included;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/eoe_request.h b/drivers/net/ethercat/master/eoe_request.h
new file mode 100644
index 000000000000..6751f1f99b09
--- /dev/null
+++ b/drivers/net/ethercat/master/eoe_request.h
@@ -0,0 +1,70 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2014  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT EoE request structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_EOE_REQUEST_H__
+#define __EC_EOE_REQUEST_H__
+
+#include <linux/list.h>
+#include <linux/etherdevice.h> // ETH_ALEN
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** Ethernet-over-EtherCAT set IP parameter request.
+ */
+typedef struct {
+    struct list_head list; /**< List item. */
+    ec_internal_request_state_t state; /**< Request state. */
+    unsigned long jiffies_sent; /**< Jiffies, when the request was sent. */
+
+    uint8_t mac_address_included;
+    uint8_t ip_address_included;
+    uint8_t subnet_mask_included;
+    uint8_t gateway_included;
+    uint8_t dns_included;
+    uint8_t name_included;
+
+    unsigned char mac_address[ETH_ALEN];
+    struct in_addr ip_address;
+    struct in_addr subnet_mask;
+    struct in_addr gateway;
+    struct in_addr dns;
+    char name[EC_MAX_HOSTNAME_SIZE];
+
+    uint16_t result;
+} ec_eoe_request_t;
+
+/****************************************************************************/
+
+void ec_eoe_request_init(ec_eoe_request_t *);
+int ec_eoe_request_valid(const ec_eoe_request_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/ethernet.c b/drivers/net/ethercat/master/ethernet.c
new file mode 100644
index 000000000000..55cff3119cd4
--- /dev/null
+++ b/drivers/net/ethercat/master/ethernet.c
@@ -0,0 +1,886 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+  \file
+  Ethernet over EtherCAT (EoE).
+*/
+
+/****************************************************************************/
+
+#include <linux/version.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/lockdep.h>
+#include <linux/skbuff.h>
+
+#include "globals.h"
+#include "master.h"
+#include "slave.h"
+#include "mailbox.h"
+#include "ethernet.h"
+
+#if defined(CONFIG_SUSE_KERNEL) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
+#include <linux/suse_version.h>
+#else
+#  ifndef SUSE_VERSION
+#    define SUSE_VERSION 0
+#  endif
+#  ifndef SUSE_PATCHLEVEL
+#    define SUSE_PATCHLEVEL 0
+#  endif
+#endif
+
+/****************************************************************************/
+
+/** Defines the debug level of EoE processing.
+ *
+ * 0 = No debug messages.
+ * 1 = Output warnings.
+ * 2 = Output actions.
+ * 3 = Output actions and frame data.
+ */
+#define EOE_DEBUG_LEVEL 1
+
+/** Size of the EoE tx queue.
+ */
+#define EC_EOE_TX_QUEUE_SIZE 100
+
+/** Number of tries.
+ */
+#define EC_EOE_TRIES 100
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_eoe_flush(ec_eoe_t *);
+int ec_eoe_send(ec_eoe_t *);
+
+/****************************************************************************/
+
+// state functions
+void ec_eoe_state_rx_start(ec_eoe_t *);
+void ec_eoe_state_rx_check(ec_eoe_t *);
+void ec_eoe_state_rx_fetch(ec_eoe_t *);
+void ec_eoe_state_tx_start(ec_eoe_t *);
+void ec_eoe_state_tx_sent(ec_eoe_t *);
+
+// net_device functions
+int ec_eoedev_open(struct net_device *);
+int ec_eoedev_stop(struct net_device *);
+int ec_eoedev_tx(struct sk_buff *, struct net_device *);
+struct net_device_stats *ec_eoedev_stats(struct net_device *);
+
+/****************************************************************************/
+
+/** Device operations for EoE interfaces.
+ */
+static const struct net_device_ops ec_eoedev_ops = {
+    .ndo_open = ec_eoedev_open,
+    .ndo_stop = ec_eoedev_stop,
+    .ndo_start_xmit = ec_eoedev_tx,
+    .ndo_get_stats = ec_eoedev_stats,
+};
+
+/****************************************************************************/
+
+/** EoE constructor.
+ *
+ * Initializes the EoE handler, creates a net_device and registers it.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_eoe_init(
+        ec_eoe_t *eoe, /**< EoE handler */
+        ec_slave_t *slave /**< EtherCAT slave */
+        )
+{
+    ec_eoe_t **priv;
+    int ret = 0;
+    char name[EC_DATAGRAM_NAME_SIZE];
+    u8 mac_addr[ETH_ALEN] = {0x00, 0x11, 0x22, 0x33, 0x44, 0x55};
+
+    eoe->slave = slave;
+
+    ec_datagram_init(&eoe->datagram);
+    eoe->queue_datagram = 0;
+    eoe->state = ec_eoe_state_rx_start;
+    eoe->opened = 0;
+    eoe->rx_skb = NULL;
+    eoe->rx_expected_fragment = 0;
+    INIT_LIST_HEAD(&eoe->tx_queue);
+    eoe->tx_frame = NULL;
+    eoe->tx_queue_active = 0;
+    eoe->tx_queue_size = EC_EOE_TX_QUEUE_SIZE;
+    eoe->tx_queued_frames = 0;
+
+    eoe->tx_frame_number = 0xFF;
+    memset(&eoe->stats, 0, sizeof(struct net_device_stats));
+
+    eoe->rx_counter = 0;
+    eoe->tx_counter = 0;
+    eoe->rx_rate = 0;
+    eoe->tx_rate = 0;
+    eoe->rate_jiffies = 0;
+    eoe->rx_idle = 1;
+    eoe->tx_idle = 1;
+
+    /* device name eoe<MASTER>[as]<SLAVE>, because networking scripts don't
+     * like hyphens etc. in interface names. */
+    if (slave->effective_alias) {
+        snprintf(name, EC_DATAGRAM_NAME_SIZE,
+                "eoe%ua%u", slave->master->index, slave->effective_alias);
+    } else {
+        snprintf(name, EC_DATAGRAM_NAME_SIZE,
+                "eoe%us%u", slave->master->index, slave->ring_position);
+    }
+
+    snprintf(eoe->datagram.name, EC_DATAGRAM_NAME_SIZE, name);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
+    eoe->dev = alloc_netdev(sizeof(ec_eoe_t *), name, NET_NAME_UNKNOWN,
+            ether_setup);
+#else
+    eoe->dev = alloc_netdev(sizeof(ec_eoe_t *), name, ether_setup);
+#endif
+    if (!eoe->dev) {
+        EC_SLAVE_ERR(slave, "Unable to allocate net_device %s"
+                " for EoE handler!\n", name);
+        ret = -ENODEV;
+        goto out_return;
+    }
+
+    // initialize net_device
+    eoe->dev->netdev_ops = &ec_eoedev_ops;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) || (SUSE_VERSION == 15 && SUSE_PATCHLEVEL >= 5)
+    eth_hw_addr_set(eoe->dev, mac_addr);
+#else
+    memcpy(eoe->dev->dev_addr, mac_addr, sizeof(mac_addr));
+#endif
+
+    // initialize private data
+    priv = netdev_priv(eoe->dev);
+    *priv = eoe;
+
+    // Usually setting the MTU appropriately makes the upper layers
+    // do the frame fragmenting. In some cases this doesn't work
+    // so the MTU is left on the Ethernet standard value and fragmenting
+    // is done "manually".
+#if 0
+    eoe->dev->mtu = slave->configured_rx_mailbox_size - ETH_HLEN - 10;
+#endif
+
+    // connect the net_device to the kernel
+    ret = register_netdev(eoe->dev);
+    if (ret) {
+        EC_SLAVE_ERR(slave, "Unable to register net_device:"
+                " error %i\n", ret);
+        goto out_free;
+    }
+
+    // make the last address octet unique
+    mac_addr[ETH_ALEN - 1] = (uint8_t) eoe->dev->ifindex;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) || (SUSE_VERSION == 15 && SUSE_PATCHLEVEL >= 5)
+    eth_hw_addr_set(eoe->dev, mac_addr);
+#else
+    memcpy(eoe->dev->dev_addr, mac_addr, sizeof(mac_addr));
+#endif
+
+    return 0;
+
+ out_free:
+    free_netdev(eoe->dev);
+    eoe->dev = NULL;
+ out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** EoE destructor.
+ *
+ * Unregisteres the net_device and frees allocated memory.
+ */
+void ec_eoe_clear(ec_eoe_t *eoe /**< EoE handler */)
+{
+    unregister_netdev(eoe->dev); // possibly calls close callback
+
+    // empty transmit queue
+    ec_eoe_flush(eoe);
+
+    if (eoe->tx_frame) {
+        dev_kfree_skb(eoe->tx_frame->skb);
+        kfree(eoe->tx_frame);
+    }
+
+    if (eoe->rx_skb)
+        dev_kfree_skb(eoe->rx_skb);
+
+    free_netdev(eoe->dev);
+
+    ec_datagram_clear(&eoe->datagram);
+}
+
+/****************************************************************************/
+
+/** Empties the transmit queue.
+ */
+void ec_eoe_flush(ec_eoe_t *eoe /**< EoE handler */)
+{
+    ec_eoe_frame_t *frame, *next;
+    struct list_head tx_queue;
+
+    netif_tx_lock_bh(eoe->dev);
+
+    list_replace_init(&eoe->tx_queue, &tx_queue);
+    eoe->tx_queued_frames = 0;
+
+    netif_tx_unlock_bh(eoe->dev);
+
+    list_for_each_entry_safe(frame, next, &tx_queue, queue) {
+        list_del(&frame->queue);
+        dev_kfree_skb(frame->skb);
+        kfree(frame);
+    }
+}
+
+/****************************************************************************/
+
+/** Sends a frame or the next fragment.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_eoe_send(ec_eoe_t *eoe /**< EoE handler */)
+{
+    size_t remaining_size, current_size, complete_offset;
+    unsigned int last_fragment;
+    uint8_t *data;
+#if EOE_DEBUG_LEVEL >= 3
+    unsigned int i;
+#endif
+
+    remaining_size = eoe->tx_frame->skb->len - eoe->tx_offset;
+
+    if (remaining_size <= eoe->slave->configured_tx_mailbox_size - 10) {
+        current_size = remaining_size;
+        last_fragment = 1;
+    } else {
+        current_size = ((eoe->slave->configured_tx_mailbox_size - 10) / 32) * 32;
+        last_fragment = 0;
+    }
+
+    if (eoe->tx_fragment_number) {
+        complete_offset = eoe->tx_offset / 32;
+    }
+    else {
+        // complete size in 32 bit blocks, rounded up.
+        complete_offset = remaining_size / 32 + 1;
+    }
+
+#if EOE_DEBUG_LEVEL >= 2
+    EC_SLAVE_DBG(eoe->slave, 0, "EoE %s TX sending fragment %u%s"
+            " with %zu octets (%zu). %u frames queued.\n",
+            eoe->dev->name, eoe->tx_fragment_number,
+            last_fragment ? "" : "+", current_size, complete_offset,
+            eoe->tx_queued_frames);
+#endif
+
+#if EOE_DEBUG_LEVEL >= 3
+    EC_SLAVE_DBG(eoe->slave, 0, "");
+    for (i = 0; i < current_size; i++) {
+        printk(KERN_CONT "%02X ",
+                eoe->tx_frame->skb->data[eoe->tx_offset + i]);
+        if ((i + 1) % 16 == 0) {
+            printk(KERN_CONT "\n");
+            EC_SLAVE_DBG(eoe->slave, 0, "");
+        }
+    }
+    printk(KERN_CONT "\n");
+#endif
+
+    data = ec_slave_mbox_prepare_send(eoe->slave, &eoe->datagram,
+            EC_MBOX_TYPE_EOE, current_size + 4);
+    if (IS_ERR(data))
+        return PTR_ERR(data);
+
+    EC_WRITE_U8 (data, EC_EOE_FRAMETYPE_INIT_REQ); // Initiate EoE Request
+    EC_WRITE_U8 (data + 1, last_fragment);
+    EC_WRITE_U16(data + 2, ((eoe->tx_fragment_number & 0x3F) |
+                            (complete_offset & 0x3F) << 6 |
+                            (eoe->tx_frame_number & 0x0F) << 12));
+
+    memcpy(data + 4, eoe->tx_frame->skb->data + eoe->tx_offset, current_size);
+    eoe->queue_datagram = 1;
+
+    eoe->tx_offset += current_size;
+    eoe->tx_fragment_number++;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Runs the EoE state machine.
+ */
+void ec_eoe_run(ec_eoe_t *eoe /**< EoE handler */)
+{
+    if (!eoe->opened)
+        return;
+
+    // if the datagram was not sent, or is not yet received, skip this cycle
+    if (eoe->queue_datagram || eoe->datagram.state == EC_DATAGRAM_SENT)
+        return;
+
+    // call state function
+    eoe->state(eoe);
+
+    // update statistics
+    if (jiffies - eoe->rate_jiffies > HZ) {
+        eoe->rx_rate = eoe->rx_counter;
+        eoe->tx_rate = eoe->tx_counter;
+        eoe->rx_counter = 0;
+        eoe->tx_counter = 0;
+        eoe->rate_jiffies = jiffies;
+    }
+
+    ec_datagram_output_stats(&eoe->datagram);
+}
+
+/****************************************************************************/
+
+/** Queues the datagram, if necessary.
+ */
+void ec_eoe_queue(ec_eoe_t *eoe /**< EoE handler */)
+{
+    if (eoe->queue_datagram) {
+        ec_master_queue_datagram_ext(eoe->slave->master, &eoe->datagram);
+        eoe->queue_datagram = 0;
+    }
+}
+
+/****************************************************************************/
+
+/** Returns the state of the device.
+ *
+ * \return 1 if the device is "up", 0 if it is "down"
+ */
+int ec_eoe_is_open(const ec_eoe_t *eoe /**< EoE handler */)
+{
+    return eoe->opened;
+}
+
+/****************************************************************************/
+
+/** Returns the idle state.
+ *
+ * \retval 1 The device is idle.
+ * \retval 0 The device is busy.
+ */
+int ec_eoe_is_idle(const ec_eoe_t *eoe /**< EoE handler */)
+{
+    return eoe->rx_idle && eoe->tx_idle;
+}
+
+/*****************************************************************************
+ *  STATE PROCESSING FUNCTIONS
+ ****************************************************************************/
+
+/** State: RX_START.
+ *
+ * Starts a new receiving sequence by queueing a datagram that checks the
+ * slave's mailbox for a new EoE datagram.
+ *
+ * \todo Use both devices.
+ */
+void ec_eoe_state_rx_start(ec_eoe_t *eoe /**< EoE handler */)
+{
+    if (eoe->slave->error_flag ||
+            !eoe->slave->master->devices[EC_DEVICE_MAIN].link_state) {
+        eoe->rx_idle = 1;
+        eoe->tx_idle = 1;
+        return;
+    }
+
+    ec_slave_mbox_prepare_check(eoe->slave, &eoe->datagram);
+    eoe->queue_datagram = 1;
+    eoe->state = ec_eoe_state_rx_check;
+}
+
+/****************************************************************************/
+
+/** State: RX_CHECK.
+ *
+ * Processes the checking datagram sent in RX_START and issues a receive
+ * datagram, if new data is available.
+ */
+void ec_eoe_state_rx_check(ec_eoe_t *eoe /**< EoE handler */)
+{
+    if (eoe->datagram.state != EC_DATAGRAM_RECEIVED) {
+        eoe->stats.rx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+        EC_SLAVE_WARN(eoe->slave, "Failed to receive mbox"
+                " check datagram for %s.\n", eoe->dev->name);
+#endif
+        eoe->state = ec_eoe_state_tx_start;
+        return;
+    }
+
+    if (!ec_slave_mbox_check(&eoe->datagram)) {
+        eoe->rx_idle = 1;
+        eoe->state = ec_eoe_state_tx_start;
+        return;
+    }
+
+    eoe->rx_idle = 0;
+    ec_slave_mbox_prepare_fetch(eoe->slave, &eoe->datagram);
+    eoe->queue_datagram = 1;
+    eoe->state = ec_eoe_state_rx_fetch;
+}
+
+/****************************************************************************/
+
+/** State: RX_FETCH.
+ *
+ * Checks if the requested data of RX_CHECK was received and processes the EoE
+ * datagram.
+ */
+void ec_eoe_state_rx_fetch(ec_eoe_t *eoe /**< EoE handler */)
+{
+    size_t rec_size, data_size;
+    uint8_t *data, frame_type, last_fragment, time_appended, mbox_prot;
+    uint8_t fragment_offset, fragment_number;
+#if EOE_DEBUG_LEVEL >= 2
+    uint8_t frame_number;
+#endif
+    off_t offset;
+#if EOE_DEBUG_LEVEL >= 3
+    unsigned int i;
+#endif
+
+    if (eoe->datagram.state != EC_DATAGRAM_RECEIVED) {
+        eoe->stats.rx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+        EC_SLAVE_WARN(eoe->slave, "Failed to receive mbox"
+                " fetch datagram for %s.\n", eoe->dev->name);
+#endif
+        eoe->state = ec_eoe_state_tx_start;
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(eoe->slave, &eoe->datagram,
+            &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        eoe->stats.rx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+        EC_SLAVE_WARN(eoe->slave, "Invalid mailbox response for %s.\n",
+                eoe->dev->name);
+#endif
+        eoe->state = ec_eoe_state_tx_start;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_EOE) { // FIXME mailbox handler necessary
+        eoe->stats.rx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+        EC_SLAVE_WARN(eoe->slave, "Other mailbox protocol response for %s.\n",
+                eoe->dev->name);
+#endif
+        eoe->state = ec_eoe_state_tx_start;
+        return;
+    }
+
+    frame_type = EC_READ_U16(data) & 0x000F;
+
+    if (frame_type != EC_EOE_FRAMETYPE_INIT_REQ) { // EoE Fragment Data
+#if EOE_DEBUG_LEVEL >= 1
+        EC_SLAVE_WARN(eoe->slave, "%s: Other frame received."
+                " Dropping.\n", eoe->dev->name);
+#endif
+        eoe->stats.rx_dropped++;
+        eoe->state = ec_eoe_state_tx_start;
+        return;
+    }
+
+    // EoE Fragment Request received
+
+    last_fragment = (EC_READ_U16(data) >> 8) & 0x0001;
+    time_appended = (EC_READ_U16(data) >> 9) & 0x0001;
+    fragment_number = EC_READ_U16(data + 2) & 0x003F;
+    fragment_offset = (EC_READ_U16(data + 2) >> 6) & 0x003F;
+#if EOE_DEBUG_LEVEL >= 2
+    frame_number = (EC_READ_U16(data + 2) >> 12) & 0x000F;
+#endif
+
+#if EOE_DEBUG_LEVEL >= 2
+    EC_SLAVE_DBG(eoe->slave, 0, "EoE %s RX fragment %u%s, offset %u,"
+            " frame %u%s, %zu octets\n", eoe->dev->name, fragment_number,
+           last_fragment ? "" : "+", fragment_offset, frame_number,
+           time_appended ? ", + timestamp" : "",
+           time_appended ? rec_size - 8 : rec_size - 4);
+#endif
+
+#if EOE_DEBUG_LEVEL >= 3
+    EC_SLAVE_DBG(eoe->slave, 0, "");
+    for (i = 0; i < rec_size - 4; i++) {
+        printk(KERN_CONT "%02X ", data[i + 4]);
+        if ((i + 1) % 16 == 0) {
+            printk(KERN_CONT "\n");
+            EC_SLAVE_DBG(eoe->slave, 0, "");
+        }
+    }
+    printk(KERN_CONT "\n");
+#endif
+
+    data_size = time_appended ? rec_size - 8 : rec_size - 4;
+
+    if (!fragment_number) {
+        if (eoe->rx_skb) {
+            EC_SLAVE_WARN(eoe->slave, "EoE RX freeing old socket buffer.\n");
+            dev_kfree_skb(eoe->rx_skb);
+        }
+
+        // new socket buffer
+        if (!(eoe->rx_skb = dev_alloc_skb(fragment_offset * 32))) {
+            if (printk_ratelimit())
+                EC_SLAVE_WARN(eoe->slave, "EoE RX low on mem,"
+                        " frame dropped.\n");
+            eoe->stats.rx_dropped++;
+            eoe->state = ec_eoe_state_tx_start;
+            return;
+        }
+
+        eoe->rx_skb_offset = 0;
+        eoe->rx_skb_size = fragment_offset * 32;
+        eoe->rx_expected_fragment = 0;
+    }
+    else {
+        if (!eoe->rx_skb) {
+            eoe->stats.rx_dropped++;
+            eoe->state = ec_eoe_state_tx_start;
+            return;
+        }
+
+        offset = fragment_offset * 32;
+        if (offset != eoe->rx_skb_offset ||
+            offset + data_size > eoe->rx_skb_size ||
+            fragment_number != eoe->rx_expected_fragment) {
+            dev_kfree_skb(eoe->rx_skb);
+            eoe->rx_skb = NULL;
+            eoe->stats.rx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+            EC_SLAVE_WARN(eoe->slave, "Fragmenting error at %s.\n",
+                    eoe->dev->name);
+#endif
+            eoe->state = ec_eoe_state_tx_start;
+            return;
+        }
+    }
+
+    // copy fragment into socket buffer
+    memcpy(skb_put(eoe->rx_skb, data_size), data + 4, data_size);
+    eoe->rx_skb_offset += data_size;
+
+    if (last_fragment) {
+        // update statistics
+        eoe->stats.rx_packets++;
+        eoe->stats.rx_bytes += eoe->rx_skb->len;
+        eoe->rx_counter += eoe->rx_skb->len;
+
+#if EOE_DEBUG_LEVEL >= 2
+        EC_SLAVE_DBG(eoe->slave, 0, "EoE %s RX frame completed"
+                " with %u octets.\n", eoe->dev->name, eoe->rx_skb->len);
+#endif
+
+        // pass socket buffer to network stack
+        eoe->rx_skb->dev = eoe->dev;
+        eoe->rx_skb->protocol = eth_type_trans(eoe->rx_skb, eoe->dev);
+        eoe->rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
+        if (netif_rx(eoe->rx_skb)) {
+            EC_SLAVE_WARN(eoe->slave, "EoE RX netif_rx failed.\n");
+        }
+        eoe->rx_skb = NULL;
+
+        eoe->state = ec_eoe_state_tx_start;
+    }
+    else {
+        eoe->rx_expected_fragment++;
+#if EOE_DEBUG_LEVEL >= 2
+        EC_SLAVE_DBG(eoe->slave, 0, "EoE %s RX expecting fragment %u\n",
+               eoe->dev->name, eoe->rx_expected_fragment);
+#endif
+        eoe->state = ec_eoe_state_rx_start;
+    }
+}
+
+/****************************************************************************/
+
+/** State: TX START.
+ *
+ * Starts a new transmit sequence. If no data is available, a new receive
+ * sequence is started instead.
+ *
+ * \todo Use both devices.
+ */
+void ec_eoe_state_tx_start(ec_eoe_t *eoe /**< EoE handler */)
+{
+#if EOE_DEBUG_LEVEL >= 2
+    unsigned int wakeup = 0;
+#endif
+
+    if (eoe->slave->error_flag ||
+            !eoe->slave->master->devices[EC_DEVICE_MAIN].link_state) {
+        eoe->rx_idle = 1;
+        eoe->tx_idle = 1;
+        return;
+    }
+
+    netif_tx_lock_bh(eoe->dev);
+
+    if (!eoe->tx_queued_frames || list_empty(&eoe->tx_queue)) {
+        netif_tx_unlock_bh(eoe->dev);
+        eoe->tx_idle = 1;
+        // no data available.
+        // start a new receive immediately.
+        ec_eoe_state_rx_start(eoe);
+        return;
+    }
+
+    // take the first frame out of the queue
+    eoe->tx_frame = list_entry(eoe->tx_queue.next, ec_eoe_frame_t, queue);
+    list_del(&eoe->tx_frame->queue);
+    if (!eoe->tx_queue_active &&
+        eoe->tx_queued_frames == eoe->tx_queue_size / 2) {
+        netif_wake_queue(eoe->dev);
+        eoe->tx_queue_active = 1;
+#if EOE_DEBUG_LEVEL >= 2
+        wakeup = 1;
+#endif
+    }
+
+    eoe->tx_queued_frames--;
+    netif_tx_unlock_bh(eoe->dev);
+
+    eoe->tx_idle = 0;
+
+    eoe->tx_frame_number++;
+    eoe->tx_frame_number %= 16;
+    eoe->tx_fragment_number = 0;
+    eoe->tx_offset = 0;
+
+    if (ec_eoe_send(eoe)) {
+        dev_kfree_skb(eoe->tx_frame->skb);
+        kfree(eoe->tx_frame);
+        eoe->tx_frame = NULL;
+        eoe->stats.tx_errors++;
+        eoe->state = ec_eoe_state_rx_start;
+#if EOE_DEBUG_LEVEL >= 1
+        EC_SLAVE_WARN(eoe->slave, "Send error at %s.\n", eoe->dev->name);
+#endif
+        return;
+    }
+
+#if EOE_DEBUG_LEVEL >= 2
+    if (wakeup)
+        EC_SLAVE_DBG(eoe->slave, 0, "EoE %s waking up TX queue...\n",
+                eoe->dev->name);
+#endif
+
+    eoe->tries = EC_EOE_TRIES;
+    eoe->state = ec_eoe_state_tx_sent;
+}
+
+/****************************************************************************/
+
+/** State: TX SENT.
+ *
+ * Checks is the previous transmit datagram succeded and sends the next
+ * fragment, if necessary.
+ */
+void ec_eoe_state_tx_sent(ec_eoe_t *eoe /**< EoE handler */)
+{
+    if (eoe->datagram.state != EC_DATAGRAM_RECEIVED) {
+        if (eoe->tries) {
+            eoe->tries--; // try again
+            eoe->queue_datagram = 1;
+        } else {
+            eoe->stats.tx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+            EC_SLAVE_WARN(eoe->slave, "Failed to receive send"
+                    " datagram for %s after %u tries.\n",
+                    eoe->dev->name, EC_EOE_TRIES);
+#endif
+            eoe->state = ec_eoe_state_rx_start;
+        }
+        return;
+    }
+
+    if (eoe->datagram.working_counter != 1) {
+        if (eoe->tries) {
+            eoe->tries--; // try again
+            eoe->queue_datagram = 1;
+        } else {
+            eoe->stats.tx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+            EC_SLAVE_WARN(eoe->slave, "No sending response"
+                    " for %s after %u tries.\n",
+                    eoe->dev->name, EC_EOE_TRIES);
+#endif
+            eoe->state = ec_eoe_state_rx_start;
+        }
+        return;
+    }
+
+    // frame completely sent
+    if (eoe->tx_offset >= eoe->tx_frame->skb->len) {
+        eoe->stats.tx_packets++;
+        eoe->stats.tx_bytes += eoe->tx_frame->skb->len;
+        eoe->tx_counter += eoe->tx_frame->skb->len;
+        dev_kfree_skb(eoe->tx_frame->skb);
+        kfree(eoe->tx_frame);
+        eoe->tx_frame = NULL;
+        eoe->state = ec_eoe_state_rx_start;
+    }
+    else { // send next fragment
+        if (ec_eoe_send(eoe)) {
+            dev_kfree_skb(eoe->tx_frame->skb);
+            kfree(eoe->tx_frame);
+            eoe->tx_frame = NULL;
+            eoe->stats.tx_errors++;
+#if EOE_DEBUG_LEVEL >= 1
+            EC_SLAVE_WARN(eoe->slave, "Send error at %s.\n", eoe->dev->name);
+#endif
+            eoe->state = ec_eoe_state_rx_start;
+        }
+    }
+}
+
+/*****************************************************************************
+ *  NET_DEVICE functions
+ ****************************************************************************/
+
+/** Opens the virtual network device.
+ *
+ * \return Always zero (success).
+ */
+int ec_eoedev_open(struct net_device *dev /**< EoE net_device */)
+{
+    ec_eoe_t *eoe = *((ec_eoe_t **) netdev_priv(dev));
+    ec_eoe_flush(eoe);
+    eoe->opened = 1;
+    eoe->rx_idle = 0;
+    eoe->tx_idle = 0;
+    netif_start_queue(dev);
+    eoe->tx_queue_active = 1;
+#if EOE_DEBUG_LEVEL >= 2
+    EC_SLAVE_DBG(eoe->slave, 0, "%s opened.\n", dev->name);
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Stops the virtual network device.
+ *
+ * \return Always zero (success).
+ */
+int ec_eoedev_stop(struct net_device *dev /**< EoE net_device */)
+{
+    ec_eoe_t *eoe = *((ec_eoe_t **) netdev_priv(dev));
+    netif_stop_queue(dev);
+    eoe->rx_idle = 1;
+    eoe->tx_idle = 1;
+    eoe->tx_queue_active = 0;
+    eoe->opened = 0;
+    ec_eoe_flush(eoe);
+#if EOE_DEBUG_LEVEL >= 2
+    EC_SLAVE_DBG(eoe->slave, 0, "%s stopped.\n", dev->name);
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Transmits data via the virtual network device.
+ *
+ * \return Zero on success, non-zero on failure.
+ */
+int ec_eoedev_tx(struct sk_buff *skb, /**< transmit socket buffer */
+                 struct net_device *dev /**< EoE net_device */
+                )
+{
+    ec_eoe_t *eoe = *((ec_eoe_t **) netdev_priv(dev));
+    ec_eoe_frame_t *frame;
+
+#if 0
+    if (skb->len > eoe->slave->configured_tx_mailbox_size - 10) {
+        EC_SLAVE_WARN(eoe->slave, "EoE TX frame (%u octets)"
+                " exceeds MTU. dropping.\n", skb->len);
+        dev_kfree_skb(skb);
+        eoe->stats.tx_dropped++;
+        return 0;
+    }
+#endif
+
+    WARN_ON_ONCE(skb_get_queue_mapping(skb) != 0);
+    lockdep_assert_held(&netdev_get_tx_queue(dev, 0)->_xmit_lock);
+
+    if (!(frame =
+          (ec_eoe_frame_t *) kmalloc(sizeof(ec_eoe_frame_t), GFP_ATOMIC))) {
+        if (printk_ratelimit())
+            EC_SLAVE_WARN(eoe->slave, "EoE TX: low on mem. frame dropped.\n");
+        return 1;
+    }
+
+    frame->skb = skb;
+
+    list_add_tail(&frame->queue, &eoe->tx_queue);
+    eoe->tx_queued_frames++;
+    if (eoe->tx_queued_frames == eoe->tx_queue_size) {
+        netif_stop_queue(dev);
+        eoe->tx_queue_active = 0;
+    }
+
+#if EOE_DEBUG_LEVEL >= 2
+    EC_SLAVE_DBG(eoe->slave, 0, "EoE %s TX queued frame"
+            " with %u octets (%u frames queued).\n",
+            eoe->dev->name, skb->len, eoe->tx_queued_frames);
+    if (!eoe->tx_queue_active)
+        EC_SLAVE_WARN(eoe->slave, "EoE TX queue is now full.\n");
+#endif
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Gets statistics about the virtual network device.
+ *
+ * \return Statistics.
+ */
+struct net_device_stats *ec_eoedev_stats(
+        struct net_device *dev /**< EoE net_device */
+        )
+{
+    ec_eoe_t *eoe = *((ec_eoe_t **) netdev_priv(dev));
+    return &eoe->stats;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/ethernet.h b/drivers/net/ethercat/master/ethernet.h
new file mode 100644
index 000000000000..ef0a8810621a
--- /dev/null
+++ b/drivers/net/ethercat/master/ethernet.h
@@ -0,0 +1,122 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Ethernet over EtherCAT (EoE)
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_ETHERNET_H__
+#define __EC_ETHERNET_H__
+
+#include <linux/list.h>
+#include <linux/netdevice.h>
+
+#include "globals.h"
+#include "slave.h"
+#include "datagram.h"
+
+/****************************************************************************/
+
+/** EoE frame types.
+ */
+enum {
+    EC_EOE_FRAMETYPE_INIT_REQ =   0x00, /** Initiate EoE Request. */
+    EC_EOE_FRAMETYPE_SET_IP_REQ = 0x02, /** Set IP Parameter Request. */
+    EC_EOE_FRAMETYPE_SET_IP_RES = 0x03, /** Set IP Parameter Response. */
+    EC_EOE_FRAMETYPE_FILT_REQ =   0x04, /** Set Address Filter Request. */
+    EC_EOE_FRAMETYPE_FILT_RES =   0x05, /** Set Address Filter Response. */
+};
+
+/*****************************************************************************/
+
+/**
+   Queued frame structure.
+*/
+
+typedef struct
+{
+    struct list_head queue; /**< list item */
+    struct sk_buff *skb; /**< socket buffer */
+}
+ec_eoe_frame_t;
+
+/****************************************************************************/
+
+typedef struct ec_eoe ec_eoe_t; /**< \see ec_eoe */
+
+/**
+   Ethernet over EtherCAT (EoE) handler.
+   The master creates one of these objects for each slave that supports the
+   EoE protocol.
+*/
+
+struct ec_eoe
+{
+    struct list_head list; /**< list item */
+    ec_slave_t *slave; /**< pointer to the corresponding slave */
+    ec_datagram_t datagram; /**< datagram */
+    unsigned int queue_datagram; /**< the datagram is ready for queuing */
+    void (*state)(ec_eoe_t *); /**< state function for the state machine */
+    struct net_device *dev; /**< net_device for virtual ethernet device */
+    struct net_device_stats stats; /**< device statistics */
+    unsigned int opened; /**< net_device is opened */
+    unsigned long rate_jiffies; /**< time of last rate output */
+
+    struct sk_buff *rx_skb; /**< current rx socket buffer */
+    off_t rx_skb_offset; /**< current write pointer in the socket buffer */
+    size_t rx_skb_size; /**< size of the allocated socket buffer memory */
+    uint8_t rx_expected_fragment; /**< next expected fragment number */
+    uint32_t rx_counter; /**< octets received during last second */
+    uint32_t rx_rate; /**< receive rate (bps) */
+    unsigned int rx_idle; /**< Idle flag. */
+
+    struct list_head tx_queue; /**< queue for frames to send */
+    unsigned int tx_queue_size; /**< Transmit queue size. */
+    unsigned int tx_queue_active; /**< kernel netif queue started */
+    unsigned int tx_queued_frames; /**< number of frames in the queue */
+    ec_eoe_frame_t *tx_frame; /**< current TX frame */
+    uint8_t tx_frame_number; /**< number of the transmitted frame */
+    uint8_t tx_fragment_number; /**< number of the fragment */
+    size_t tx_offset; /**< number of octets sent */
+    uint32_t tx_counter; /**< octets transmitted during last second */
+    uint32_t tx_rate; /**< transmit rate (bps) */
+    unsigned int tx_idle; /**< Idle flag. */
+
+    unsigned int tries; /**< Tries. */
+};
+
+/****************************************************************************/
+
+int ec_eoe_init(ec_eoe_t *, ec_slave_t *);
+void ec_eoe_clear(ec_eoe_t *);
+void ec_eoe_run(ec_eoe_t *);
+void ec_eoe_queue(ec_eoe_t *);
+int ec_eoe_is_open(const ec_eoe_t *);
+int ec_eoe_is_idle(const ec_eoe_t *);
+
+/****************************************************************************/
+
+#endif
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/flag.c b/drivers/net/ethercat/master/flag.c
new file mode 100644
index 000000000000..652d35b17292
--- /dev/null
+++ b/drivers/net/ethercat/master/flag.c
@@ -0,0 +1,69 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2021  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * Slave Configuration Feature Flag.
+ */
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+
+#include "flag.h"
+
+/****************************************************************************/
+
+/** SDO request constructor.
+ */
+int ec_flag_init(
+        ec_flag_t *flag, /**< Feature flag. */
+        const char *key, /**< Feature key. */
+        int32_t value /**< Feature value. */
+        )
+{
+    if (!key || strlen(key) == 0) {
+        return -EINVAL;
+    }
+
+    if (!(flag->key = (uint8_t *) kmalloc(strlen(key) + 1, GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    strcpy(flag->key, key); // no strncpy, buffer is alloc'ed with strlen
+    flag->value = value;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** SDO request destructor.
+ */
+void ec_flag_clear(
+        ec_flag_t *flag /**< Feature flag. */
+        )
+{
+    if (flag->key) {
+        kfree(flag->key);
+        flag->key = NULL;
+    }
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/flag.h b/drivers/net/ethercat/master/flag.h
new file mode 100644
index 000000000000..e7679c2f2bef
--- /dev/null
+++ b/drivers/net/ethercat/master/flag.h
@@ -0,0 +1,51 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2021  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT Slave Configuration Feature Flag.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FLAG_H__
+#define __EC_FLAG_H__
+
+#include <linux/list.h>
+
+/****************************************************************************/
+
+/** Slave configutation feature flag.
+ */
+typedef struct {
+    struct list_head list; /**< List item. */
+    char *key; /**< Flag key (null-terminated ASCII string. */
+    int32_t value; /**< Flag value (meaning depends on key). */
+} ec_flag_t;
+
+/****************************************************************************/
+
+int ec_flag_init(ec_flag_t *, const char *, int32_t);
+void ec_flag_clear(ec_flag_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fmmu_config.c b/drivers/net/ethercat/master/fmmu_config.c
new file mode 100644
index 000000000000..9e247a46ddbe
--- /dev/null
+++ b/drivers/net/ethercat/master/fmmu_config.c
@@ -0,0 +1,91 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT FMMU configuration methods.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "slave_config.h"
+#include "master.h"
+
+#include "fmmu_config.h"
+
+/****************************************************************************/
+
+/** FMMU configuration constructor.
+ *
+ * Inits an FMMU configuration, sets the logical start address and adds the
+ * process data size for the mapped PDOs of the given direction to the domain
+ * data size.
+ */
+void ec_fmmu_config_init(
+        ec_fmmu_config_t *fmmu, /**< EtherCAT FMMU configuration. */
+        ec_slave_config_t *sc, /**< EtherCAT slave configuration. */
+        ec_domain_t *domain, /**< EtherCAT domain. */
+        uint8_t sync_index, /**< Sync manager index to use. */
+        ec_direction_t dir /**< PDO direction. */
+        )
+{
+    INIT_LIST_HEAD(&fmmu->list);
+    fmmu->sc = sc;
+    fmmu->sync_index = sync_index;
+    fmmu->dir = dir;
+
+    fmmu->logical_start_address = domain->data_size;
+    fmmu->data_size = ec_pdo_list_total_size(
+            &sc->sync_configs[sync_index].pdos);
+
+    ec_domain_add_fmmu_config(domain, fmmu);
+}
+
+/****************************************************************************/
+
+/** Initializes an FMMU configuration page.
+ *
+ * The referenced memory (\a data) must be at least EC_FMMU_PAGE_SIZE bytes.
+ */
+void ec_fmmu_config_page(
+        const ec_fmmu_config_t *fmmu, /**< EtherCAT FMMU configuration. */
+        const ec_sync_t *sync, /**< Sync manager. */
+        uint8_t *data /**> Configuration page memory. */
+        )
+{
+    EC_CONFIG_DBG(fmmu->sc, 1, "FMMU: LogAddr 0x%08X, Size %3u,"
+            " PhysAddr 0x%04X, SM%u, Dir %s\n",
+            fmmu->logical_start_address, fmmu->data_size,
+            sync->physical_start_address, fmmu->sync_index,
+            fmmu->dir == EC_DIR_INPUT ? "in" : "out");
+
+    EC_WRITE_U32(data,      fmmu->logical_start_address);
+    EC_WRITE_U16(data + 4,  fmmu->data_size); // size of fmmu
+    EC_WRITE_U8 (data + 6,  0x00); // logical start bit
+    EC_WRITE_U8 (data + 7,  0x07); // logical end bit
+    EC_WRITE_U16(data + 8,  sync->physical_start_address);
+    EC_WRITE_U8 (data + 10, 0x00); // physical start bit
+    EC_WRITE_U8 (data + 11, fmmu->dir == EC_DIR_INPUT ? 0x01 : 0x02);
+    EC_WRITE_U16(data + 12, 0x0001); // enable
+    EC_WRITE_U16(data + 14, 0x0000); // reserved
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fmmu_config.h b/drivers/net/ethercat/master/fmmu_config.h
new file mode 100644
index 000000000000..a4c8dc8ab640
--- /dev/null
+++ b/drivers/net/ethercat/master/fmmu_config.h
@@ -0,0 +1,58 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT FMMU configuration structure.
+ */
+
+/****************************************************************************/
+
+#ifndef __EC_FMMU_CONFIG_H__
+#define __EC_FMMU_CONFIG_H__
+
+#include "globals.h"
+#include "sync.h"
+
+/****************************************************************************/
+
+/** FMMU configuration.
+ */
+typedef struct {
+    struct list_head list; /**< List node used by domain. */
+    const ec_slave_config_t *sc; /**< EtherCAT slave config. */
+    const ec_domain_t *domain; /**< Domain. */
+    uint8_t sync_index; /**< Index of sync manager to use. */
+    ec_direction_t dir; /**< FMMU direction. */
+    uint32_t logical_start_address; /**< Logical start address. */
+    unsigned int data_size; /**< Covered PDO size. */
+} ec_fmmu_config_t;
+
+/****************************************************************************/
+
+void ec_fmmu_config_init(ec_fmmu_config_t *, ec_slave_config_t *,
+        ec_domain_t *, uint8_t, ec_direction_t);
+
+void ec_fmmu_config_page(const ec_fmmu_config_t *, const ec_sync_t *,
+        uint8_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/foe.h b/drivers/net/ethercat/master/foe.h
new file mode 100644
index 000000000000..ae4fc4b1f44a
--- /dev/null
+++ b/drivers/net/ethercat/master/foe.h
@@ -0,0 +1,54 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * FoE defines.
+ */
+
+#ifndef __FOE_H__
+#define __FOE_H__
+
+/****************************************************************************/
+
+/** FoE error enumeration type.
+ */
+typedef enum {
+    FOE_BUSY               = 0, /**< Busy. */
+    FOE_READY              = 1, /**< Ready. */
+    FOE_IDLE               = 2, /**< Idle. */
+    FOE_WC_ERROR           = 3, /**< Working counter error. */
+    FOE_RECEIVE_ERROR      = 4, /**< Receive error. */
+    FOE_PROT_ERROR         = 5, /**< Protocol error. */
+    FOE_NODATA_ERROR       = 6, /**< No data error. */
+    FOE_PACKETNO_ERROR     = 7, /**< Packet number error. */
+    FOE_OPCODE_ERROR       = 8, /**< OpCode error. */
+    FOE_TIMEOUT_ERROR      = 9, /**< Timeout error. */
+    FOE_SEND_RX_DATA_ERROR = 10, /**< Error sending received data. */
+    FOE_RX_DATA_ACK_ERROR  = 11, /**< Error acknowledging received data. */
+    FOE_ACK_ERROR          = 12, /**< Acknowledge error. */
+    FOE_MBOX_FETCH_ERROR   = 13, /**< Error fetching data from mailbox. */
+    FOE_READ_NODATA_ERROR  = 14, /**< No data while reading. */
+    FOE_MBOX_PROT_ERROR    = 15, /**< Mailbox protocol error. */
+} ec_foe_error_t;
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/foe_request.c b/drivers/net/ethercat/master/foe_request.c
new file mode 100644
index 000000000000..3b68f734c579
--- /dev/null
+++ b/drivers/net/ethercat/master/foe_request.c
@@ -0,0 +1,195 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2008  Olav Zarges, imc Messsysteme GmbH
+ *  Copyright (C) 2020  Florian Pose, IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * File-over-EtherCAT request functions.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/jiffies.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include "foe_request.h"
+#include "foe.h"
+
+/****************************************************************************/
+
+/** Default timeout in ms to wait for FoE transfer responses.
+ */
+#define EC_FOE_REQUEST_RESPONSE_TIMEOUT 3000
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_foe_request_clear_data(ec_foe_request_t *);
+
+/****************************************************************************/
+
+/** FoE request constructor.
+ */
+void ec_foe_request_init(
+        ec_foe_request_t *req, /**< FoE request. */
+        uint8_t* file_name /** filename */)
+{
+    INIT_LIST_HEAD(&req->list);
+    req->buffer = NULL;
+    req->file_name = file_name;
+    req->buffer_size = 0;
+    req->data_size = 0;
+    req->dir = EC_DIR_INVALID;
+    req->issue_timeout = 0; // no timeout
+    req->response_timeout = EC_FOE_REQUEST_RESPONSE_TIMEOUT;
+    req->state = EC_INT_REQUEST_INIT;
+    req->result = FOE_BUSY;
+    req->error_code = 0x00000000;
+}
+
+/****************************************************************************/
+
+/** FoE request destructor.
+ */
+void ec_foe_request_clear(
+        ec_foe_request_t *req /**< FoE request. */
+        )
+{
+    ec_foe_request_clear_data(req);
+}
+
+/****************************************************************************/
+
+/** FoE request destructor.
+ */
+void ec_foe_request_clear_data(
+        ec_foe_request_t *req /**< FoE request. */
+        )
+{
+    if (req->buffer) {
+        vfree(req->buffer);
+        req->buffer = NULL;
+    }
+
+    req->buffer_size = 0;
+    req->data_size = 0;
+}
+
+/****************************************************************************/
+
+/** Pre-allocates the data memory.
+ *
+ * If the internal \a buffer_size is already bigger than \a size, nothing is
+ * done.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_foe_request_alloc(
+        ec_foe_request_t *req, /**< FoE request. */
+        size_t size /**< Data size to allocate. */
+        )
+{
+    if (size <= req->buffer_size) {
+        return 0;
+    }
+
+    ec_foe_request_clear_data(req);
+
+    if (!(req->buffer = (uint8_t *) vmalloc(size))) {
+        EC_ERR("Failed to allocate %zu bytes of FoE memory.\n", size);
+        return -ENOMEM;
+    }
+
+    req->buffer_size = size;
+    req->data_size = 0;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Copies FoE data from an external source.
+ *
+ * If the \a buffer_size is to small, new memory is allocated.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_foe_request_copy_data(
+        ec_foe_request_t *req, /**< FoE request. */
+        const uint8_t *source, /**< Source data. */
+        size_t size /**< Number of bytes in \a source. */
+        )
+{
+    int ret;
+
+    ret = ec_foe_request_alloc(req, size);
+    if (ret) {
+        return ret;
+    }
+
+    memcpy(req->buffer, source, size);
+    req->data_size = size;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Checks, if the timeout was exceeded.
+ *
+ * \return non-zero if the timeout was exceeded, else zero.
+ */
+int ec_foe_request_timed_out(
+        const ec_foe_request_t *req /**< FoE request. */
+        )
+{
+    return req->issue_timeout
+        && jiffies - req->jiffies_start > HZ * req->issue_timeout / 1000;
+}
+
+/****************************************************************************/
+
+/** Prepares a read request (slave to master).
+ */
+void ec_foe_request_read(
+        ec_foe_request_t *req /**< FoE request. */
+        )
+{
+    req->dir = EC_DIR_INPUT;
+    req->state = EC_INT_REQUEST_QUEUED;
+    req->result = FOE_BUSY;
+    req->jiffies_start = jiffies;
+}
+
+/****************************************************************************/
+
+/** Prepares a write request (master to slave).
+ */
+void ec_foe_request_write(
+        ec_foe_request_t *req /**< FoE request. */
+        )
+{
+    req->dir = EC_DIR_OUTPUT;
+    req->state = EC_INT_REQUEST_QUEUED;
+    req->result = FOE_BUSY;
+    req->jiffies_start = jiffies;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/foe_request.h b/drivers/net/ethercat/master/foe_request.h
new file mode 100644
index 000000000000..ba5ceb661e99
--- /dev/null
+++ b/drivers/net/ethercat/master/foe_request.h
@@ -0,0 +1,78 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2008  Olav Zarges, imc Messsysteme GmbH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT FoE request structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FOE_REQUEST_H__
+#define __EC_FOE_REQUEST_H__
+
+#include <linux/list.h>
+
+#include "../include/ecrt.h"
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** FoE request.
+ */
+typedef struct {
+    struct list_head list; /**< List item. */
+    uint8_t *buffer; /**< Pointer to FoE data. */
+    size_t buffer_size; /**< Size of FoE data memory. */
+    size_t data_size; /**< Size of FoE data. */
+
+    uint32_t issue_timeout; /**< Maximum time in ms, the processing of the
+                              request may take. */
+    uint32_t response_timeout; /**< Maximum time in ms, the transfer is
+                                 retried, if the slave does not respond. */
+    ec_direction_t dir; /**< Direction. EC_DIR_OUTPUT means downloading to
+                          the slave, EC_DIR_INPUT means uploading from the
+                          slave. */
+    ec_internal_request_state_t state; /**< FoE request state. */
+    unsigned long jiffies_start; /**< Jiffies, when the request was issued. */
+    unsigned long jiffies_sent; /**< Jiffies, when the upload/download
+                                     request was sent. */
+    uint8_t *file_name; /**< Pointer to the filename. */
+    uint32_t result; /**< FoE request abort code. Zero on success. */
+    uint32_t error_code; /**< Error code from an FoE Error Request. */
+} ec_foe_request_t;
+
+/****************************************************************************/
+
+void ec_foe_request_init(ec_foe_request_t *, uint8_t *file_name);
+void ec_foe_request_clear(ec_foe_request_t *);
+
+int ec_foe_request_alloc(ec_foe_request_t *, size_t);
+int ec_foe_request_copy_data(ec_foe_request_t *, const uint8_t *, size_t);
+int ec_foe_request_timed_out(const ec_foe_request_t *);
+
+void ec_foe_request_write(ec_foe_request_t *);
+void ec_foe_request_read(ec_foe_request_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_change.c b/drivers/net/ethercat/master/fsm_change.c
new file mode 100644
index 000000000000..98fae1847acf
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_change.c
@@ -0,0 +1,615 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT state change FSM.
+*/
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "fsm_change.h"
+#include "slave_config.h"
+
+/****************************************************************************/
+
+unsigned int ec_fsm_change_timeout_ms(const ec_fsm_change_t *);
+
+void ec_fsm_change_state_start(ec_fsm_change_t *);
+void ec_fsm_change_state_check(ec_fsm_change_t *);
+void ec_fsm_change_state_status(ec_fsm_change_t *);
+void ec_fsm_change_state_start_code(ec_fsm_change_t *);
+void ec_fsm_change_state_code(ec_fsm_change_t *);
+void ec_fsm_change_state_ack(ec_fsm_change_t *);
+void ec_fsm_change_state_check_ack(ec_fsm_change_t *);
+void ec_fsm_change_state_end(ec_fsm_change_t *);
+void ec_fsm_change_state_error(ec_fsm_change_t *);
+
+/****************************************************************************/
+
+/**
+   Constructor.
+*/
+
+void ec_fsm_change_init(ec_fsm_change_t *fsm, /**< finite state machine */
+                        ec_datagram_t *datagram /**< datagram */
+                        )
+{
+    fsm->state = NULL;
+    fsm->datagram = datagram;
+    fsm->spontaneous_change = 0;
+}
+
+/****************************************************************************/
+
+/**
+   Destructor.
+*/
+
+void ec_fsm_change_clear(ec_fsm_change_t *fsm /**< finite state machine */)
+{
+}
+
+/****************************************************************************/
+
+/** Get timeout in ms.
+ *
+ * For defaults see ETG2000_S_R_V1i0i15 section 5.3.6.2.
+ */
+unsigned int ec_fsm_change_timeout_ms(
+        const ec_fsm_change_t *fsm /**< finite state machine */
+        )
+{
+    ec_slave_state_t from = fsm->old_state;
+    ec_slave_state_t to = fsm->requested_state;
+
+    /* Search for specified timeout in slave configuration */
+    if (fsm->slave->config) {
+        unsigned int timeout_ms =
+            ec_slave_config_al_timeout(fsm->slave->config, from, to);
+        if (timeout_ms) {
+            return timeout_ms;
+        }
+    }
+
+    /* No specific timeout found. Use defaults from spec. */
+
+    if (from == EC_SLAVE_STATE_INIT &&
+            (to == EC_SLAVE_STATE_PREOP || to == EC_SLAVE_STATE_BOOT)) {
+        return 3000; // PreopTimeout
+    }
+    if ((from == EC_SLAVE_STATE_PREOP && to == EC_SLAVE_STATE_SAFEOP) ||
+            (from == EC_SLAVE_STATE_SAFEOP && to == EC_SLAVE_STATE_OP)) {
+        return 10000; // SafeopOpTimeout
+    }
+    if (to == EC_SLAVE_STATE_INIT ||
+            ((from == EC_SLAVE_STATE_OP || from == EC_SLAVE_STATE_SAFEOP)
+             && to == EC_SLAVE_STATE_PREOP)) {
+        return 5000; // BackToInitTimeout
+    }
+    if (from == EC_SLAVE_STATE_OP && to == EC_SLAVE_STATE_SAFEOP) {
+        return 200; // BackToSafeopTimeout
+    }
+
+    return 10000; // default [ms]
+}
+
+/****************************************************************************/
+
+/**
+   Starts the change state machine.
+*/
+
+void ec_fsm_change_start(ec_fsm_change_t *fsm, /**< finite state machine */
+                         ec_slave_t *slave, /**< EtherCAT slave */
+                         ec_slave_state_t state /**< requested state */
+                         )
+{
+    fsm->mode = EC_FSM_CHANGE_MODE_FULL;
+    fsm->slave = slave;
+    fsm->requested_state = state;
+    fsm->state = ec_fsm_change_state_start;
+}
+
+/****************************************************************************/
+
+/**
+   Starts the change state machine to only acknowlegde a slave's state.
+*/
+
+void ec_fsm_change_ack(ec_fsm_change_t *fsm, /**< finite state machine */
+                       ec_slave_t *slave /**< EtherCAT slave */
+                       )
+{
+    fsm->mode = EC_FSM_CHANGE_MODE_ACK_ONLY;
+    fsm->slave = slave;
+    fsm->requested_state = EC_SLAVE_STATE_UNKNOWN;
+    fsm->state = ec_fsm_change_state_start_code;
+}
+
+/****************************************************************************/
+
+/**
+   Executes the current state of the state machine.
+   \return false, if the state machine has terminated
+*/
+
+int ec_fsm_change_exec(ec_fsm_change_t *fsm /**< finite state machine */)
+{
+    fsm->state(fsm);
+
+    return fsm->state != ec_fsm_change_state_end
+        && fsm->state != ec_fsm_change_state_error;
+}
+
+/****************************************************************************/
+
+/**
+   Returns, if the state machine terminated with success.
+   \return non-zero if successful.
+*/
+
+int ec_fsm_change_success(ec_fsm_change_t *fsm /**< Finite state machine */)
+{
+    return fsm->state == ec_fsm_change_state_end;
+}
+
+/*****************************************************************************
+ *  state change state machine
+ ****************************************************************************/
+
+/**
+   Change state: START.
+*/
+
+void ec_fsm_change_state_start(ec_fsm_change_t *fsm
+                               /**< finite state machine */)
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    fsm->take_time = 1;
+    fsm->old_state = fsm->slave->current_state;
+
+    // write new state to slave
+    ec_datagram_fpwr(datagram, slave->station_address, 0x0120, 2);
+    EC_WRITE_U16(datagram->data, fsm->requested_state);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_change_state_check;
+}
+
+/****************************************************************************/
+
+/**
+   Change state: CHECK.
+*/
+
+void ec_fsm_change_state_check(ec_fsm_change_t *fsm
+                               /**< finite state machine */)
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive state datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (fsm->take_time) {
+        fsm->take_time = 0;
+        fsm->jiffies_start = datagram->jiffies_sent;
+    }
+
+    if (datagram->working_counter == 0) {
+        if (datagram->jiffies_received - fsm->jiffies_start >= 3 * HZ) {
+            char state_str[EC_STATE_STRING_SIZE];
+            ec_state_string(fsm->requested_state, state_str, 0);
+            fsm->state = ec_fsm_change_state_error;
+            EC_SLAVE_ERR(slave, "Failed to set state %s: ", state_str);
+            ec_datagram_print_wc_error(datagram);
+            return;
+        }
+
+        // repeat writing new state to slave
+        ec_datagram_fpwr(datagram, slave->station_address, 0x0120, 2);
+        EC_WRITE_U16(datagram->data, fsm->requested_state);
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    if (unlikely(datagram->working_counter > 1)) {
+        char state_str[EC_STATE_STRING_SIZE];
+        ec_state_string(fsm->requested_state, state_str, 0);
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Failed to set state %s: ", state_str);
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    fsm->take_time = 1;
+
+    // read AL status from slave
+    ec_datagram_fprd(datagram, slave->station_address, 0x0130, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->spontaneous_change = 0;
+    fsm->state = ec_fsm_change_state_status;
+}
+
+/****************************************************************************/
+
+/**
+   Change state: STATUS.
+*/
+
+void ec_fsm_change_state_status(ec_fsm_change_t *fsm
+                                /**< finite state machine */)
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    unsigned int timeout_ms;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive state checking datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        char req_state[EC_STATE_STRING_SIZE];
+        ec_state_string(fsm->requested_state, req_state, 0);
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Failed to check state %s: ", req_state);
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    if (fsm->take_time) {
+        fsm->take_time = 0;
+        fsm->jiffies_start = datagram->jiffies_sent;
+    }
+
+    slave->current_state = EC_READ_U8(datagram->data);
+
+    if (slave->current_state == fsm->requested_state) {
+        // state has been set successfully
+        fsm->state = ec_fsm_change_state_end;
+        return;
+    }
+
+    if (slave->current_state != fsm->old_state) { // state changed
+        char req_state[EC_STATE_STRING_SIZE], cur_state[EC_STATE_STRING_SIZE];
+
+        ec_state_string(slave->current_state, cur_state, 0);
+
+        if ((slave->current_state & 0x0F) != (fsm->old_state & 0x0F)) {
+            // Slave spontaneously changed its state just before the new state
+            // was written. Accept current state as old state and wait for
+            // state change
+            fsm->spontaneous_change = 1;
+            fsm->old_state = slave->current_state;
+            EC_SLAVE_WARN(slave, "Changed to %s in the meantime.\n",
+                    cur_state);
+            goto check_again;
+        }
+
+        // state change error
+
+        slave->error_flag = 1;
+        ec_state_string(fsm->requested_state, req_state, 0);
+
+        EC_SLAVE_ERR(slave, "Failed to set %s state, slave refused state"
+                " change (%s).\n", req_state, cur_state);
+
+        ec_fsm_change_state_start_code(fsm);
+        return;
+    }
+
+    // still old state
+
+    timeout_ms = ec_fsm_change_timeout_ms(fsm);
+    if (datagram->jiffies_received - fsm->jiffies_start >=
+            timeout_ms * HZ / 1000) {
+        // timeout while checking
+        char state_str[EC_STATE_STRING_SIZE];
+        ec_state_string(fsm->requested_state, state_str, 0);
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Timeout after %u ms while setting state %s.\n",
+                timeout_ms, state_str);
+        return;
+    }
+
+ check_again:
+    // no timeout yet. check again
+    ec_datagram_fprd(datagram, slave->station_address, 0x0130, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+}
+
+/****************************************************************************/
+
+/** Enter reading AL status code.
+ */
+void ec_fsm_change_state_start_code(
+        ec_fsm_change_t *fsm /**< finite state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+
+    // fetch AL status error code
+    ec_datagram_fprd(datagram, slave->station_address, 0x0134, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_change_state_code;
+}
+
+/****************************************************************************/
+
+/**
+   Application layer status messages.
+*/
+
+const ec_code_msg_t al_status_messages[] = {
+    {0x0000, "No error"},
+    {0x0001, "Unspecified error"},
+    {0x0002, "No Memory"},
+    {0x0011, "Invalid requested state change"},
+    {0x0012, "Unknown requested state"},
+    {0x0013, "Bootstrap not supported"},
+    {0x0014, "No valid firmware"},
+    {0x0015, "Invalid mailbox configuration"},
+    {0x0016, "Invalid mailbox configuration"},
+    {0x0017, "Invalid sync manager configuration"},
+    {0x0018, "No valid inputs available"},
+    {0x0019, "No valid outputs"},
+    {0x001A, "Synchronization error"},
+    {0x001B, "Sync manager watchdog"},
+    {0x001C, "Invalid sync manager types"},
+    {0x001D, "Invalid output configuration"},
+    {0x001E, "Invalid input configuration"},
+    {0x001F, "Invalid watchdog configuration"},
+    {0x0020, "Slave needs cold start"},
+    {0x0021, "Slave needs INIT"},
+    {0x0022, "Slave needs PREOP"},
+    {0x0023, "Slave needs SAFEOP"},
+    {0x0024, "Invalid Input Mapping"},
+    {0x0025, "Invalid Output Mapping"},
+    {0x0026, "Inconsistent Settings"},
+    {0x0027, "Freerun not supported"},
+    {0x0028, "Synchronization not supported"},
+    {0x0029, "Freerun needs 3 Buffer Mode"},
+    {0x002A, "Background Watchdog"},
+    {0x002B, "No Valid Inputs and Outputs"},
+    {0x002C, "Fatal Sync Error"},
+    {0x002D, "No Sync Error"},
+    {0x0030, "Invalid DC SYNCH configuration"},
+    {0x0031, "Invalid DC latch configuration"},
+    {0x0032, "PLL error"},
+    {0x0033, "DC Sync IO Error"},
+    {0x0034, "DC Sync Timeout Error"},
+    {0x0035, "DC Invalid Sync Cycle Time"},
+    {0x0036, "DC Sync0 Cycle Time"},
+    {0x0037, "DC Sync1 Cycle Time"},
+    {0x0041, "MBX_AOE"},
+    {0x0042, "MBX_EOE"},
+    {0x0043, "MBX_COE"},
+    {0x0044, "MBX_FOE"},
+    {0x0045, "MBX_SOE"},
+    {0x004F, "MBX_VOE"},
+    {0x0050, "EEPROM No Access"},
+    {0x0051, "EEPROM Error"},
+    {0x0060, "Slave Restarted Locally"},
+    {0xffff}
+};
+
+
+/****************************************************************************/
+
+/**
+   Change state: CODE.
+*/
+
+void ec_fsm_change_state_code(ec_fsm_change_t *fsm
+                              /**< finite state machine */)
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    uint32_t code;
+    const ec_code_msg_t *al_msg;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed to receive"
+                " AL status code datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_WARN(fsm->slave, "Reception of AL status code"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+    } else {
+        code = EC_READ_U16(datagram->data);
+        for (al_msg = al_status_messages; al_msg->code != 0xffff; al_msg++) {
+            if (al_msg->code != code) {
+                continue;
+            }
+
+            EC_SLAVE_ERR(fsm->slave, "AL status message 0x%04X: \"%s\".\n",
+                    al_msg->code, al_msg->message);
+            break;
+        }
+        if (al_msg->code == 0xffff) { /* not found in our list. */
+            EC_SLAVE_ERR(fsm->slave, "Unknown AL status code 0x%04X.\n",
+                    code);
+        }
+    }
+
+    // acknowledge "old" slave state
+    ec_datagram_fpwr(datagram, fsm->slave->station_address, 0x0120, 2);
+    EC_WRITE_U16(datagram->data, fsm->slave->current_state);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_change_state_ack;
+}
+
+/****************************************************************************/
+
+/**
+   Change state: ACK.
+*/
+
+void ec_fsm_change_state_ack(ec_fsm_change_t *fsm /**< finite state machine */)
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive state ack datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Reception of state ack datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    fsm->take_time = 1;
+
+    // read new AL status
+    ec_datagram_fprd(datagram, slave->station_address, 0x0130, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_change_state_check_ack;
+}
+
+/****************************************************************************/
+
+/**
+   Change state: CHECK ACK.
+*/
+
+void ec_fsm_change_state_check_ack(ec_fsm_change_t *fsm
+                                   /**< finite state machine */)
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    unsigned int timeout_ms;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive state ack check datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Reception of state ack check datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    if (fsm->take_time) {
+        fsm->take_time = 0;
+        fsm->jiffies_start = datagram->jiffies_sent;
+    }
+
+    slave->current_state = EC_READ_U8(datagram->data);
+
+    if (!(slave->current_state & EC_SLAVE_STATE_ACK_ERR)) {
+        char state_str[EC_STATE_STRING_SIZE];
+        ec_state_string(slave->current_state, state_str, 0);
+        if (fsm->mode == EC_FSM_CHANGE_MODE_FULL) {
+            fsm->state = ec_fsm_change_state_error;
+        }
+        else { // EC_FSM_CHANGE_MODE_ACK_ONLY
+            fsm->state = ec_fsm_change_state_end;
+        }
+        EC_SLAVE_INFO(slave, "Acknowledged state %s.\n", state_str);
+        return;
+    }
+
+    timeout_ms = ec_fsm_change_timeout_ms(fsm);
+    if (datagram->jiffies_received - fsm->jiffies_start >=
+            timeout_ms * HZ / 1000) {
+        // timeout while checking
+        char state_str[EC_STATE_STRING_SIZE];
+        ec_state_string(slave->current_state, state_str, 0);
+        fsm->state = ec_fsm_change_state_error;
+        EC_SLAVE_ERR(slave, "Timeout after %u ms while acknowledging"
+                " state %s.\n", timeout_ms, state_str);
+        return;
+    }
+
+    // reread new AL status
+    ec_datagram_fprd(datagram, slave->station_address, 0x0130, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+}
+
+/****************************************************************************/
+
+/**
+   State: ERROR.
+*/
+
+void ec_fsm_change_state_error(ec_fsm_change_t *fsm
+                               /**< finite state machine */)
+{
+}
+
+/****************************************************************************/
+
+/**
+   State: END.
+*/
+
+void ec_fsm_change_state_end(ec_fsm_change_t *fsm
+                             /**< finite state machine */)
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_change.h b/drivers/net/ethercat/master/fsm_change.h
new file mode 100644
index 000000000000..c30ac2578479
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_change.h
@@ -0,0 +1,84 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT state change FSM.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_CHANGE_H__
+#define __EC_FSM_CHANGE_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "slave.h"
+
+/****************************************************************************/
+
+/**
+   Mode of the change state machine.
+*/
+
+typedef enum {
+    EC_FSM_CHANGE_MODE_FULL, /**< full state change */
+    EC_FSM_CHANGE_MODE_ACK_ONLY /**< only state acknowledgement */
+}
+ec_fsm_change_mode_t;
+
+/****************************************************************************/
+
+typedef struct ec_fsm_change ec_fsm_change_t; /**< \see ec_fsm_change */
+
+/**
+   EtherCAT state change FSM.
+*/
+
+struct ec_fsm_change
+{
+    ec_slave_t *slave; /**< slave the FSM runs on */
+    ec_datagram_t *datagram; /**< datagram used in the state machine */
+    unsigned int retries; /**< retries upon datagram timeout */
+
+    void (*state)(ec_fsm_change_t *); /**< slave state change state function */
+    ec_fsm_change_mode_t mode; /**< full state change, or ack only. */
+    ec_slave_state_t requested_state; /**< input: state */
+    ec_slave_state_t old_state; /**< prior slave state */
+    unsigned long jiffies_start; /**< change timer */
+    uint8_t take_time; /**< take sending timestamp */
+    uint8_t spontaneous_change; /**< spontaneous state change detected */
+};
+
+/****************************************************************************/
+
+void ec_fsm_change_init(ec_fsm_change_t *, ec_datagram_t *);
+void ec_fsm_change_clear(ec_fsm_change_t *);
+
+void ec_fsm_change_start(ec_fsm_change_t *, ec_slave_t *, ec_slave_state_t);
+void ec_fsm_change_ack(ec_fsm_change_t *, ec_slave_t *);
+
+int ec_fsm_change_exec(ec_fsm_change_t *);
+int ec_fsm_change_success(ec_fsm_change_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_coe.c b/drivers/net/ethercat/master/fsm_coe.c
new file mode 100644
index 000000000000..69765cd006eb
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_coe.c
@@ -0,0 +1,2537 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT CoE state machines.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "fsm_coe.h"
+#include "slave_config.h"
+
+/****************************************************************************/
+
+/** Maximum time in ms to wait for responses when reading out the dictionary.
+ */
+#define EC_FSM_COE_DICT_TIMEOUT 1000
+
+/** CoE download request header size.
+ */
+#define EC_COE_DOWN_REQ_HEADER_SIZE 10
+
+/** CoE download segment request header size.
+ */
+#define EC_COE_DOWN_SEG_REQ_HEADER_SIZE 3
+
+/** Minimum size of download segment.
+ */
+#define EC_COE_DOWN_SEG_MIN_DATA_SIZE 7
+
+/** Enable debug output for CoE retries.
+ */
+#define DEBUG_RETRIES 0
+
+/** Enable warning output if transfers take too long.
+ */
+#define DEBUG_LONG 0
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_canopen_abort_msg(const ec_slave_t *, uint32_t);
+int ec_fsm_coe_check_emergency(const ec_fsm_coe_t *, const uint8_t *, size_t);
+int ec_fsm_coe_prepare_dict(ec_fsm_coe_t *, ec_datagram_t *);
+int ec_fsm_coe_dict_prepare_desc(ec_fsm_coe_t *, ec_datagram_t *);
+int ec_fsm_coe_dict_prepare_entry(ec_fsm_coe_t *, ec_datagram_t *);
+int ec_fsm_coe_prepare_down_start(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_down_prepare_segment_request(ec_fsm_coe_t *, ec_datagram_t *);
+int ec_fsm_coe_prepare_up(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_up_prepare_segment_request(ec_fsm_coe_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+void ec_fsm_coe_dict_start(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_request(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_check(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_response(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_desc_request(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_desc_check(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_desc_response(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_entry_request(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_entry_check(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_dict_entry_response(ec_fsm_coe_t *, ec_datagram_t *);
+
+void ec_fsm_coe_down_start(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_down_request(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_down_check(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_down_response(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_down_seg_check(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_down_seg_response(ec_fsm_coe_t *, ec_datagram_t *);
+
+void ec_fsm_coe_up_start(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_up_request(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_up_check(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_up_response(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_up_seg_request(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_up_seg_check(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_up_seg_response(ec_fsm_coe_t *, ec_datagram_t *);
+
+void ec_fsm_coe_end(ec_fsm_coe_t *, ec_datagram_t *);
+void ec_fsm_coe_error(ec_fsm_coe_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+/** SDO abort messages.
+ *
+ * The "abort SDO transfer request" supplies an abort code, which can be
+ * translated to clear text. This table does the mapping of the codes and
+ * messages.
+ */
+const ec_code_msg_t sdo_abort_messages[] = {
+    {0x05030000, "Toggle bit not changed"},
+    {0x05040000, "SDO protocol timeout"},
+    {0x05040001, "Client/Server command specifier not valid or unknown"},
+    {0x05040005, "Out of memory"},
+    {0x06010000, "Unsupported access to an object"},
+    {0x06010001, "Attempt to read a write-only object"},
+    {0x06010002, "Attempt to write a read-only object"},
+    {0x06020000, "This object does not exist in the object directory"},
+    {0x06040041, "The object cannot be mapped into the PDO"},
+    {0x06040042, "The number and length of the objects to be mapped would"
+     " exceed the PDO length"},
+    {0x06040043, "General parameter incompatibility reason"},
+    {0x06040047, "Gerneral internal incompatibility in device"},
+    {0x06060000, "Access failure due to a hardware error"},
+    {0x06070010, "Data type does not match, length of service parameter does"
+     " not match"},
+    {0x06070012, "Data type does not match, length of service parameter too"
+     " high"},
+    {0x06070013, "Data type does not match, length of service parameter too"
+     " low"},
+    {0x06090011, "Subindex does not exist"},
+    {0x06090030, "Value range of parameter exceeded"},
+    {0x06090031, "Value of parameter written too high"},
+    {0x06090032, "Value of parameter written too low"},
+    {0x06090036, "Maximum value is less than minimum value"},
+    {0x08000000, "General error"},
+    {0x08000020, "Data cannot be transferred or stored to the application"},
+    {0x08000021, "Data cannot be transferred or stored to the application"
+     " because of local control"},
+    {0x08000022, "Data cannot be transferred or stored to the application"
+     " because of the present device state"},
+    {0x08000023, "Object dictionary dynamic generation fails or no object"
+     " dictionary is present"},
+    {}
+};
+
+/****************************************************************************/
+
+/** Outputs an SDO abort message.
+ */
+void ec_canopen_abort_msg(
+        const ec_slave_t *slave, /**< Slave. */
+        uint32_t abort_code /**< Abort code to search for. */
+        )
+{
+    const ec_code_msg_t *abort_msg;
+
+    for (abort_msg = sdo_abort_messages; abort_msg->code; abort_msg++) {
+        if (abort_msg->code == abort_code) {
+            EC_SLAVE_ERR(slave, "SDO abort message 0x%08X: \"%s\".\n",
+                   abort_msg->code, abort_msg->message);
+            return;
+        }
+    }
+
+    EC_SLAVE_ERR(slave, "Unknown SDO abort code 0x%08X.\n", abort_code);
+}
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_coe_init(
+        ec_fsm_coe_t *fsm /**< Finite state machine */
+        )
+{
+    fsm->state = NULL;
+    fsm->datagram = NULL;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_coe_clear(
+        ec_fsm_coe_t *fsm /**< Finite state machine */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** Starts reading a slaves' SDO dictionary.
+ */
+void ec_fsm_coe_dictionary(
+        ec_fsm_coe_t *fsm, /**< Finite state machine */
+        ec_slave_t *slave /**< EtherCAT slave */
+        )
+{
+    fsm->slave = slave;
+    fsm->state = ec_fsm_coe_dict_start;
+}
+
+/****************************************************************************/
+
+/** Starts to transfer an SDO to/from a slave.
+ */
+void ec_fsm_coe_transfer(
+        ec_fsm_coe_t *fsm, /**< State machine. */
+        ec_slave_t *slave, /**< EtherCAT slave. */
+        ec_sdo_request_t *request /**< SDO request. */
+        )
+{
+    fsm->slave = slave;
+    fsm->request = request;
+
+    if (request->dir == EC_DIR_OUTPUT) {
+        fsm->state = ec_fsm_coe_down_start;
+    }
+    else {
+        fsm->state = ec_fsm_coe_up_start;
+    }
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * \return 1 if the datagram was used, else 0.
+ */
+int ec_fsm_coe_exec(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    int datagram_used = 0;
+
+    if (fsm->datagram &&
+            (fsm->datagram->state == EC_DATAGRAM_INIT ||
+             fsm->datagram->state == EC_DATAGRAM_QUEUED ||
+             fsm->datagram->state == EC_DATAGRAM_SENT)) {
+        // datagram not received yet
+        return datagram_used;
+    }
+
+    fsm->state(fsm, datagram);
+
+    datagram_used =
+        fsm->state != ec_fsm_coe_end && fsm->state != ec_fsm_coe_error;
+
+    if (datagram_used) {
+        fsm->datagram = datagram;
+    } else {
+        fsm->datagram = NULL;
+    }
+
+    return datagram_used;
+}
+
+/****************************************************************************/
+
+/** Returns, if the state machine terminated with success.
+ * \return non-zero if successful.
+ */
+int ec_fsm_coe_success(
+        const ec_fsm_coe_t *fsm /**< Finite state machine */
+        )
+{
+    return fsm->state == ec_fsm_coe_end;
+}
+
+/****************************************************************************/
+
+/** Check if the received data are a CoE emergency request.
+ *
+ * If the check is positive, the emergency request is output.
+ *
+ * \return The data were an emergency request.
+ */
+int ec_fsm_coe_check_emergency(
+        const ec_fsm_coe_t *fsm, /**< Finite state machine */
+        const uint8_t *data, /**< CoE mailbox data. */
+        size_t size /**< CoE mailbox data size. */
+        )
+{
+    if (size < 2 || ((EC_READ_U16(data) >> 12) & 0x0F) != 0x01)
+        return 0;
+
+    if (size < 10) {
+        EC_SLAVE_WARN(fsm->slave, "Received incomplete CoE Emergency"
+                " request:\n");
+        ec_print_data(data, size);
+        return 1;
+    }
+
+    {
+        ec_slave_config_t *sc = fsm->slave->config;
+        if (sc) {
+            ec_coe_emerg_ring_push(&sc->emerg_ring, data + 2);
+        }
+    }
+
+    EC_SLAVE_WARN(fsm->slave, "CoE Emergency Request received:\n"
+            "Error code 0x%04X, Error register 0x%02X, data:\n",
+            EC_READ_U16(data + 2), EC_READ_U8(data + 4));
+    ec_print_data(data + 5, 5);
+    return 1;
+}
+
+/*****************************************************************************
+ *  CoE dictionary state machine
+ ****************************************************************************/
+
+/** Prepare a dictionary request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_fsm_coe_prepare_dict(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    uint8_t *data = ec_slave_mbox_prepare_send(slave, datagram,
+			EC_MBOX_TYPE_COE, 8);
+    if (IS_ERR(data)) {
+        return PTR_ERR(data);
+    }
+
+    EC_WRITE_U16(data, 0x8 << 12); // SDO information
+    EC_WRITE_U8 (data + 2, 0x01); // Get OD List Request
+    EC_WRITE_U8 (data + 3, 0x00);
+    EC_WRITE_U16(data + 4, 0x0000);
+    EC_WRITE_U16(data + 6, 0x0001); // deliver all SDOs!
+
+    fsm->state = ec_fsm_coe_dict_request;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** CoE state: DICT START.
+ */
+void ec_fsm_coe_dict_start(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_COE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support CoE!\n");
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (slave->sii.has_general && !slave->sii.coe_details.enable_sdo_info) {
+        EC_SLAVE_ERR(slave, "Slave does not support"
+                " SDO information service!\n");
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    fsm->retries = EC_FSM_RETRIES;
+
+    if (ec_fsm_coe_prepare_dict(fsm, datagram)) {
+        fsm->state = ec_fsm_coe_error;
+    }
+}
+
+/****************************************************************************/
+
+/** CoE state: DICT REQUEST.
+ * \todo Timeout behavior
+ */
+void ec_fsm_coe_dict_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        if (ec_fsm_coe_prepare_dict(fsm, datagram)) {
+            fsm->state = ec_fsm_coe_error;
+        }
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE dictionary"
+                " request datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE dictionary request failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_dict_check;
+}
+
+/****************************************************************************/
+
+/** CoE state: DICT CHECK.
+ */
+void ec_fsm_coe_dict_check(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave,"Reception of CoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= EC_FSM_COE_DICT_TIMEOUT) {
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Timeout while waiting for"
+                    " SDO dictionary list response.\n");
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_dict_response;
+}
+
+/****************************************************************************/
+
+/** Prepare an object description request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_fsm_coe_dict_prepare_desc(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    u8 *data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_COE,
+			8);
+    if (IS_ERR(data)) {
+        return PTR_ERR(data);
+    }
+
+    EC_WRITE_U16(data, 0x8 << 12); // SDO information
+    EC_WRITE_U8 (data + 2, 0x03); // Get object description request
+    EC_WRITE_U8 (data + 3, 0x00);
+    EC_WRITE_U16(data + 4, 0x0000);
+    EC_WRITE_U16(data + 6, fsm->sdo->index); // SDO index
+
+    fsm->state = ec_fsm_coe_dict_desc_request;
+    return 0;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DICT RESPONSE.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_dict_response(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    uint8_t *data, mbox_prot;
+    size_t rec_size;
+    unsigned int sdo_count, i;
+    uint16_t sdo_index, fragments_left;
+    ec_sdo_t *sdo;
+    bool first_segment;
+    size_t index_list_offset;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE dictionary"
+                " response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE dictionary response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_COE) {
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (ec_fsm_coe_check_emergency(fsm, data, rec_size)) {
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_dict_check;
+        return;
+    }
+
+    if (rec_size < 3) {
+        EC_SLAVE_ERR(slave, "Received corrupted SDO dictionary response"
+                " (size %zu).\n", rec_size);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 == 0x8 && // SDO information
+        (EC_READ_U8(data + 2) & 0x7F) == 0x07) { // error response
+        EC_SLAVE_ERR(slave, "SDO information error response!\n");
+        if (rec_size < 10) {
+            EC_SLAVE_ERR(slave, "Incomplete SDO information"
+                    " error response:\n");
+            ec_print_data(data, rec_size);
+        } else {
+            ec_canopen_abort_msg(slave, EC_READ_U32(data + 6));
+        }
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 != 0x8 || // SDO information
+        (EC_READ_U8 (data + 2) & 0x7F) != 0x02) { // Get OD List response
+        if (fsm->slave->master->debug_level) {
+            EC_SLAVE_DBG(slave, 1, "Invalid SDO list response!"
+                    " Retrying...\n");
+            ec_print_data(data, rec_size);
+        }
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_dict_check;
+        return;
+    }
+
+    first_segment = list_empty(&slave->sdo_dictionary) ? true : false;
+    index_list_offset = first_segment ? 8 : 6;
+
+    if (rec_size < index_list_offset || rec_size % 2) {
+        EC_SLAVE_ERR(slave, "Invalid data size %zu!\n", rec_size);
+        ec_print_data(data, rec_size);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    sdo_count = (rec_size - index_list_offset) / 2;
+
+    for (i = 0; i < sdo_count; i++) {
+        sdo_index = EC_READ_U16(data + index_list_offset + i * 2);
+        if (!sdo_index) {
+            EC_SLAVE_DBG(slave, 1, "SDO dictionary contains index 0x0000.\n");
+            continue;
+        }
+
+        if (!(sdo = (ec_sdo_t *) kmalloc(sizeof(ec_sdo_t), GFP_KERNEL))) {
+            EC_SLAVE_ERR(slave, "Failed to allocate memory for SDO!\n");
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+
+        ec_sdo_init(sdo, slave, sdo_index);
+        list_add_tail(&sdo->list, &slave->sdo_dictionary);
+    }
+
+    fragments_left = EC_READ_U16(data + 4);
+    if (fragments_left) {
+        EC_SLAVE_DBG(slave, 1, "SDO list fragments left: %u\n",
+                fragments_left);
+    }
+
+    if (EC_READ_U8(data + 2) & 0x80 || fragments_left) {
+        // more messages waiting. check again.
+        fsm->jiffies_start = fsm->datagram->jiffies_sent;
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_dict_check;
+        return;
+    }
+
+    if (list_empty(&slave->sdo_dictionary)) {
+        // no SDOs in dictionary. finished.
+        fsm->state = ec_fsm_coe_end; // success
+        return;
+    }
+
+    // fetch SDO descriptions
+    fsm->sdo = list_entry(slave->sdo_dictionary.next, ec_sdo_t, list);
+
+    fsm->retries = EC_FSM_RETRIES;
+    if (ec_fsm_coe_dict_prepare_desc(fsm, datagram)) {
+        fsm->state = ec_fsm_coe_error;
+    }
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DICT DESC REQUEST.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_dict_desc_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        if (ec_fsm_coe_dict_prepare_desc(fsm, datagram)) {
+            fsm->state = ec_fsm_coe_error;
+        }
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE SDO"
+                " description request datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE SDO description"
+                " request failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_dict_desc_check;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DICT DESC CHECK.
+*/
+
+void ec_fsm_coe_dict_desc_check(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= EC_FSM_COE_DICT_TIMEOUT) {
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Timeout while waiting for"
+                    " SDO 0x%04x object description response.\n",
+                    fsm->sdo->index);
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_dict_desc_response;
+}
+
+/****************************************************************************/
+
+/** Prepare an entry description request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_fsm_coe_dict_prepare_entry(
+        ec_fsm_coe_t *fsm, /**< Finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    u8 *data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_COE,
+			10);
+    if (IS_ERR(data)) {
+        return PTR_ERR(data);
+    }
+
+    EC_WRITE_U16(data, 0x8 << 12); // SDO information
+    EC_WRITE_U8 (data + 2, 0x05); // Get entry description request
+    EC_WRITE_U8 (data + 3, 0x00);
+    EC_WRITE_U16(data + 4, 0x0000);
+    EC_WRITE_U16(data + 6, fsm->sdo->index); // SDO index
+    EC_WRITE_U8 (data + 8, fsm->subindex); // SDO subindex
+    EC_WRITE_U8 (data + 9, 0x01); // value info (access rights only)
+
+    fsm->state = ec_fsm_coe_dict_entry_request;
+    return 0;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DICT DESC RESPONSE.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_dict_desc_response(
+        ec_fsm_coe_t *fsm, /**< Finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_t *sdo = fsm->sdo;
+    uint8_t *data, mbox_prot;
+    size_t rec_size, name_size;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE SDO description"
+                " response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE SDO description"
+                " response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_COE) {
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (ec_fsm_coe_check_emergency(fsm, data, rec_size)) {
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_dict_desc_check;
+        return;
+    }
+
+    if (rec_size < 3) {
+        EC_SLAVE_ERR(slave, "Received corrupted SDO description response"
+                " (size %zu).\n", rec_size);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 == 0x8 && // SDO information
+        (EC_READ_U8 (data + 2) & 0x7F) == 0x07) { // error response
+        EC_SLAVE_ERR(slave, "SDO information error response while"
+                " fetching SDO 0x%04X!\n", sdo->index);
+        ec_canopen_abort_msg(slave, EC_READ_U32(data + 6));
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (rec_size < 8) {
+        EC_SLAVE_ERR(slave, "Received corrupted SDO"
+                " description response (size %zu).\n", rec_size);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 != 0x8 || // SDO information
+        (EC_READ_U8 (data + 2) & 0x7F) != 0x04 || // Object desc. response
+        EC_READ_U16(data + 6) != sdo->index) { // SDO index
+        if (fsm->slave->master->debug_level) {
+            EC_SLAVE_DBG(slave, 1, "Invalid object description response while"
+                    " fetching SDO 0x%04X!\n", sdo->index);
+            ec_print_data(data, rec_size);
+        }
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_dict_desc_check;
+        return;
+    }
+
+    if (rec_size < 12) {
+        EC_SLAVE_ERR(slave, "Invalid data size!\n");
+        ec_print_data(data, rec_size);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    sdo->max_subindex = EC_READ_U8(data + 10);
+    sdo->object_code = EC_READ_U8(data + 11);
+
+    name_size = rec_size - 12;
+    if (name_size) {
+        if (!(sdo->name = kmalloc(name_size + 1, GFP_KERNEL))) {
+            EC_SLAVE_ERR(slave, "Failed to allocate SDO name!\n");
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+
+        memcpy(sdo->name, data + 12, name_size);
+        sdo->name[name_size] = 0;
+    }
+
+    if (EC_READ_U8(data + 2) & 0x80) {
+        EC_SLAVE_ERR(slave, "Fragment follows (not implemented)!\n");
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    // start fetching entries
+
+    fsm->subindex = 0;
+    fsm->retries = EC_FSM_RETRIES;
+
+    if (ec_fsm_coe_dict_prepare_entry(fsm, datagram)) {
+        fsm->state = ec_fsm_coe_error;
+    }
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DICT ENTRY REQUEST.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_dict_entry_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        if (ec_fsm_coe_dict_prepare_entry(fsm, datagram)) {
+            fsm->state = ec_fsm_coe_error;
+        }
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE SDO entry"
+                " request datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE SDO entry request failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_dict_entry_check;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DICT ENTRY CHECK.
+*/
+
+void ec_fsm_coe_dict_entry_check(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= EC_FSM_COE_DICT_TIMEOUT) {
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Timeout while waiting for"
+                    " SDO entry 0x%04x:%x description response.\n",
+                    fsm->sdo->index, fsm->subindex);
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_dict_entry_response;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DICT ENTRY RESPONSE.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_dict_entry_response(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_t *sdo = fsm->sdo;
+    uint8_t *data, mbox_prot;
+    size_t rec_size, data_size;
+    ec_sdo_entry_t *entry;
+    u16 word;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE SDO"
+                " description response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE SDO description"
+                " response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_COE) {
+        EC_SLAVE_ERR(slave, "Received mailbox protocol"
+                " 0x%02X as response.\n", mbox_prot);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (ec_fsm_coe_check_emergency(fsm, data, rec_size)) {
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_dict_entry_check;
+        return;
+    }
+
+    if (rec_size < 3) {
+        EC_SLAVE_ERR(slave, "Received corrupted SDO entry"
+                " description response (size %zu).\n", rec_size);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 == 0x8 && // SDO information
+        (EC_READ_U8 (data + 2) & 0x7F) == 0x07) { // error response
+        EC_SLAVE_WARN(slave, "SDO information error response while"
+               " fetching SDO entry 0x%04X:%02X!\n",
+               sdo->index, fsm->subindex);
+        ec_canopen_abort_msg(slave, EC_READ_U32(data + 6));
+
+        /* There may be gaps in the subindices, so try to continue with next
+         * subindex. */
+
+    } else {
+
+        if (rec_size < 9) {
+            EC_SLAVE_ERR(slave, "Received corrupted SDO entry"
+                    " description response (size %zu).\n", rec_size);
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+
+        if (EC_READ_U16(data) >> 12 != 0x8 || // SDO information
+            (EC_READ_U8(data + 2) & 0x7F) != 0x06 || // Entry desc. response
+            EC_READ_U16(data + 6) != sdo->index || // SDO index
+            EC_READ_U8(data + 8) != fsm->subindex) { // SDO subindex
+            if (fsm->slave->master->debug_level) {
+                EC_SLAVE_DBG(slave, 1, "Invalid entry description response"
+                        " while fetching SDO entry 0x%04X:%02X!\n",
+                        sdo->index, fsm->subindex);
+                ec_print_data(data, rec_size);
+            }
+            // check for CoE response again
+            ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+            fsm->retries = EC_FSM_RETRIES;
+            fsm->state = ec_fsm_coe_dict_entry_check;
+            return;
+        }
+
+        if (rec_size < 16) {
+            EC_SLAVE_ERR(slave, "Invalid data size %zu!\n", rec_size);
+            ec_print_data(data, rec_size);
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+
+        data_size = rec_size - 16;
+
+        if (!(entry = (ec_sdo_entry_t *)
+              kmalloc(sizeof(ec_sdo_entry_t), GFP_KERNEL))) {
+            EC_SLAVE_ERR(slave, "Failed to allocate entry!\n");
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+
+        ec_sdo_entry_init(entry, sdo, fsm->subindex);
+        entry->data_type = EC_READ_U16(data + 10);
+        entry->bit_length = EC_READ_U16(data + 12);
+
+        // read access rights
+        word = EC_READ_U16(data + 14);
+        entry->read_access[EC_SDO_ENTRY_ACCESS_PREOP] = word & 0x0001;
+        entry->read_access[EC_SDO_ENTRY_ACCESS_SAFEOP] =
+            (word >> 1)  & 0x0001;
+        entry->read_access[EC_SDO_ENTRY_ACCESS_OP] = (word >> 2)  & 0x0001;
+        entry->write_access[EC_SDO_ENTRY_ACCESS_PREOP] = (word >> 3) & 0x0001;
+        entry->write_access[EC_SDO_ENTRY_ACCESS_SAFEOP] =
+            (word >> 4)  & 0x0001;
+        entry->write_access[EC_SDO_ENTRY_ACCESS_OP] = (word >> 5)  & 0x0001;
+
+        if (data_size) {
+            uint8_t *desc;
+            if (!(desc = kmalloc(data_size + 1, GFP_KERNEL))) {
+                EC_SLAVE_ERR(slave, "Failed to allocate SDO entry name!\n");
+                fsm->state = ec_fsm_coe_error;
+                return;
+            }
+            memcpy(desc, data + 16, data_size);
+            desc[data_size] = 0;
+            entry->description = desc;
+        }
+
+        list_add_tail(&entry->list, &sdo->entries);
+    }
+
+    if (fsm->subindex < sdo->max_subindex) {
+
+        fsm->subindex++;
+        fsm->retries = EC_FSM_RETRIES;
+
+        if (ec_fsm_coe_dict_prepare_entry(fsm, datagram)) {
+            fsm->state = ec_fsm_coe_error;
+        }
+
+        return;
+    }
+
+    // another SDO description to fetch?
+    if (fsm->sdo->list.next != &slave->sdo_dictionary) {
+
+        fsm->sdo = list_entry(fsm->sdo->list.next, ec_sdo_t, list);
+        fsm->retries = EC_FSM_RETRIES;
+
+        if (ec_fsm_coe_dict_prepare_desc(fsm, datagram)) {
+            fsm->state = ec_fsm_coe_error;
+        }
+
+        return;
+    }
+
+    fsm->state = ec_fsm_coe_end;
+}
+
+/*****************************************************************************
+ *  CoE state machine
+ ****************************************************************************/
+
+/** Prepare a donwnload request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_fsm_coe_prepare_down_start(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    u8 *data;
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_request_t *request = fsm->request;
+    uint8_t data_set_size;
+
+    if (request->data_size > 0 && request->data_size <= 4) {
+        // use expedited transfer mode for lengths between 1 and 4 bytes
+        data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_COE,
+                EC_COE_DOWN_REQ_HEADER_SIZE);
+        if (IS_ERR(data)) {
+            request->errno = PTR_ERR(data);
+            return PTR_ERR(data);
+        }
+
+        fsm->remaining = 0;
+
+        data_set_size = 4 - request->data_size;
+
+        EC_WRITE_U16(data, 0x2 << 12); // SDO request
+        EC_WRITE_U8 (data + 2, (0x3 // size specified, expedited
+                    | data_set_size << 2
+                    | ((request->complete_access ? 1 : 0) << 4)
+                    | 0x1 << 5)); // Download request
+        EC_WRITE_U16(data + 3, request->index);
+        EC_WRITE_U8 (data + 5,
+                request->complete_access ? 0x00 : request->subindex);
+        memcpy(data + 6, request->data, request->data_size);
+        memset(data + 6 + request->data_size, 0x00, 4 - request->data_size);
+
+        if (slave->master->debug_level) {
+            EC_SLAVE_DBG(slave, 1, "Expedited download request:\n");
+            ec_print_data(data, EC_COE_DOWN_REQ_HEADER_SIZE);
+        }
+    }
+    else { // data_size < 1 or data_size > 4, use normal transfer type
+        size_t data_size,
+               max_data_size =
+                   slave->configured_rx_mailbox_size - EC_MBOX_HEADER_SIZE,
+               required_data_size =
+                   EC_COE_DOWN_REQ_HEADER_SIZE + request->data_size;
+
+        if (max_data_size < required_data_size) {
+            // segmenting needed
+            data_size = max_data_size;
+        } else {
+            data_size = required_data_size;
+        }
+
+        data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_COE,
+                data_size);
+        if (IS_ERR(data)) {
+            request->errno = PTR_ERR(data);
+            return PTR_ERR(data);
+        }
+
+        fsm->offset = 0;
+        fsm->remaining = request->data_size;
+
+        EC_WRITE_U16(data, 0x2 << 12); // SDO request
+        EC_WRITE_U8(data + 2,
+                0x1 // size indicator, normal
+                | ((request->complete_access ? 1 : 0) << 4)
+                | 0x1 << 5); // Download request
+        EC_WRITE_U16(data + 3, request->index);
+        EC_WRITE_U8 (data + 5,
+                request->complete_access ? 0x00 : request->subindex);
+        EC_WRITE_U32(data + 6, request->data_size);
+
+        if (data_size > EC_COE_DOWN_REQ_HEADER_SIZE) {
+            size_t segment_size = data_size - EC_COE_DOWN_REQ_HEADER_SIZE;
+            memcpy(data + EC_COE_DOWN_REQ_HEADER_SIZE,
+                    request->data, segment_size);
+            fsm->offset += segment_size;
+            fsm->remaining -= segment_size;
+        }
+
+        if (slave->master->debug_level) {
+            EC_SLAVE_DBG(slave, 1, "Normal download request:\n");
+            ec_print_data(data, data_size);
+        }
+    }
+
+    fsm->state = ec_fsm_coe_down_request;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** CoE state: DOWN START.
+ */
+void ec_fsm_coe_down_start(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_request_t *request = fsm->request;
+
+    if (fsm->slave->master->debug_level) {
+        char subidxstr[10];
+        if (request->complete_access) {
+            subidxstr[0] = 0x00;
+        } else {
+            sprintf(subidxstr, ":%02X", request->subindex);
+        }
+        EC_SLAVE_DBG(slave, 1, "Downloading SDO 0x%04X%s.\n",
+                request->index, subidxstr);
+        ec_print_data(request->data, request->data_size);
+    }
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_COE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support CoE!\n");
+        request->errno = EPROTONOSUPPORT;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (slave->configured_rx_mailbox_size <
+            EC_MBOX_HEADER_SIZE + EC_COE_DOWN_REQ_HEADER_SIZE) {
+        EC_SLAVE_ERR(slave, "Mailbox too small!\n");
+        request->errno = ENOBUFS;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+
+    fsm->request->jiffies_sent = jiffies;
+    fsm->retries = EC_FSM_RETRIES;
+
+    if (ec_fsm_coe_prepare_down_start(fsm, datagram)) {
+        fsm->state = ec_fsm_coe_error;
+    }
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DOWN REQUEST.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_down_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    unsigned long diff_ms;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        if (ec_fsm_coe_prepare_down_start(fsm, datagram)) {
+            fsm->state = ec_fsm_coe_error;
+        }
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE download"
+                " request datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    diff_ms = (jiffies - fsm->request->jiffies_sent) * 1000 / HZ;
+
+    if (fsm->datagram->working_counter != 1) {
+        if (!fsm->datagram->working_counter) {
+            if (diff_ms < fsm->request->response_timeout) {
+#if DEBUG_RETRIES
+                EC_SLAVE_DBG(slave, 1, "Slave did not respond to SDO"
+                        " download request. Retrying after %lu ms...\n",
+                        diff_ms);
+#endif
+                // no response; send request datagram again
+                if (ec_fsm_coe_prepare_down_start(fsm, datagram)) {
+                    fsm->state = ec_fsm_coe_error;
+                }
+                return;
+            }
+        }
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE download request"
+                " for SDO 0x%04x:%x failed with timeout after %lu ms: ",
+                fsm->request->index, fsm->request->subindex, diff_ms);
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+#if DEBUG_LONG
+    if (diff_ms > 200) {
+        EC_SLAVE_WARN(slave, "SDO 0x%04x:%x download took %lu ms.\n",
+                fsm->request->index, fsm->request->subindex, diff_ms);
+    }
+#endif
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_down_check;
+}
+
+/****************************************************************************/
+
+/** CoE state: DOWN CHECK.
+ */
+void ec_fsm_coe_down_check(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE mailbox check"
+                " datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= fsm->request->response_timeout) {
+            fsm->request->errno = EIO;
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Timeout after %lu ms while waiting"
+                    " for SDO 0x%04x:%x download response.\n", diff_ms,
+                    fsm->request->index, fsm->request->subindex);
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_down_response;
+}
+
+/****************************************************************************/
+
+/** Prepare a download segment request.
+ */
+void ec_fsm_coe_down_prepare_segment_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_request_t *request = fsm->request;
+    size_t max_segment_size =
+        slave->configured_rx_mailbox_size
+        - EC_MBOX_HEADER_SIZE
+        - EC_COE_DOWN_SEG_REQ_HEADER_SIZE;
+    size_t data_size;
+    uint8_t last_segment, seg_data_size, *data;
+
+    if (fsm->remaining > max_segment_size) {
+        fsm->segment_size = max_segment_size;
+        last_segment = 0;
+    } else {
+        fsm->segment_size = fsm->remaining;
+        last_segment = 1;
+    }
+
+    if (fsm->segment_size > EC_COE_DOWN_SEG_MIN_DATA_SIZE) {
+        seg_data_size = 0x00;
+        data_size = EC_COE_DOWN_SEG_REQ_HEADER_SIZE + fsm->segment_size;
+    } else {
+        seg_data_size = EC_COE_DOWN_SEG_MIN_DATA_SIZE - fsm->segment_size;
+        data_size = EC_COE_DOWN_SEG_REQ_HEADER_SIZE
+            + EC_COE_DOWN_SEG_MIN_DATA_SIZE;
+    }
+
+    data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_COE,
+            data_size);
+    if (IS_ERR(data)) {
+        request->errno = PTR_ERR(data);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    EC_WRITE_U16(data, 0x2 << 12); // SDO request
+    EC_WRITE_U8(data + 2, (last_segment ? 1 : 0)
+            | (seg_data_size << 1)
+            | (fsm->toggle << 4)
+            | (0x00 << 5)); // Download segment request
+    memcpy(data + EC_COE_DOWN_SEG_REQ_HEADER_SIZE,
+            request->data + fsm->offset, fsm->segment_size);
+    if (fsm->segment_size < EC_COE_DOWN_SEG_MIN_DATA_SIZE) {
+        memset(data + EC_COE_DOWN_SEG_REQ_HEADER_SIZE + fsm->segment_size,
+                0x00, EC_COE_DOWN_SEG_MIN_DATA_SIZE - fsm->segment_size);
+    }
+
+    if (slave->master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Download segment request:\n");
+        ec_print_data(data, data_size);
+    }
+
+    fsm->state = ec_fsm_coe_down_seg_check;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DOWN RESPONSE.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_down_response(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    uint8_t *data, mbox_prot;
+    size_t rec_size;
+    ec_sdo_request_t *request = fsm->request;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE download"
+                " response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE download response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        request->errno = PTR_ERR(data);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_COE) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        return;
+    }
+
+    if (ec_fsm_coe_check_emergency(fsm, data, rec_size)) {
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_down_check;
+        return;
+    }
+
+    if (slave->master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Download response:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    if (rec_size < 6) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Received data are too small (%zu bytes):\n",
+                rec_size);
+        ec_print_data(data, rec_size);
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 == 0x2 && // SDO request
+        EC_READ_U8 (data + 2) >> 5 == 0x4) { // abort SDO transfer request
+        char subidxstr[10];
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        if (request->complete_access) {
+            subidxstr[0] = 0x00;
+        } else {
+            sprintf(subidxstr, ":%02X", request->subindex);
+        }
+        EC_SLAVE_ERR(slave, "SDO download 0x%04X%s (%zu bytes) aborted.\n",
+                request->index, subidxstr, request->data_size);
+        if (rec_size < 10) {
+            EC_SLAVE_ERR(slave, "Incomplete abort command:\n");
+            ec_print_data(data, rec_size);
+        } else {
+            fsm->request->abort_code = EC_READ_U32(data + 6);
+            ec_canopen_abort_msg(slave, fsm->request->abort_code);
+        }
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 != 0x3 || // SDO response
+        EC_READ_U8 (data + 2) >> 5 != 0x3 || // Download response
+        EC_READ_U16(data + 3) != request->index || // index
+        EC_READ_U8 (data + 5) != request->subindex) { // subindex
+        if (slave->master->debug_level) {
+            EC_SLAVE_DBG(slave, 1, "Invalid SDO download response!"
+                    " Retrying...\n");
+            ec_print_data(data, rec_size);
+        }
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_down_check;
+        return;
+    }
+
+    if (fsm->remaining) { // more segments to download
+        fsm->toggle = 0;
+        ec_fsm_coe_down_prepare_segment_request(fsm, datagram);
+    } else {
+        fsm->state = ec_fsm_coe_end; // success
+    }
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DOWN SEG CHECK.
+*/
+
+void ec_fsm_coe_down_seg_check(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE mailbox segment check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= fsm->request->response_timeout) {
+            fsm->request->errno = EIO;
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Timeout while waiting for SDO download"
+                    " segment response.\n");
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_down_seg_response;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: DOWN SEG RESPONSE.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_down_seg_response(
+        ec_fsm_coe_t *fsm, /**< Finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    uint8_t *data, mbox_prot;
+    size_t rec_size;
+    ec_sdo_request_t *request = fsm->request;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE download response"
+                " datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE download response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        request->errno = PTR_ERR(data);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_COE) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        return;
+    }
+
+    if (ec_fsm_coe_check_emergency(fsm, data, rec_size)) {
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_down_check;
+        return;
+    }
+
+    if (slave->master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Download response:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    if (rec_size < 6) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Received data are too small (%zu bytes):\n",
+                rec_size);
+        ec_print_data(data, rec_size);
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 == 0x2 && // SDO request
+        EC_READ_U8 (data + 2) >> 5 == 0x4) { // abort SDO transfer request
+        char subidxstr[10];
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        if (request->complete_access) {
+            subidxstr[0] = 0x00;
+        } else {
+            sprintf(subidxstr, ":%02X", request->subindex);
+        }
+        EC_SLAVE_ERR(slave, "SDO download 0x%04X%s (%zu bytes) aborted.\n",
+                request->index, subidxstr, request->data_size);
+        if (rec_size < 10) {
+            EC_SLAVE_ERR(slave, "Incomplete abort command:\n");
+            ec_print_data(data, rec_size);
+        } else {
+            fsm->request->abort_code = EC_READ_U32(data + 6);
+            ec_canopen_abort_msg(slave, fsm->request->abort_code);
+        }
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 != 0x3 ||
+            ((EC_READ_U8(data + 2) >> 5) != 0x01)) { // segment response
+        if (slave->master->debug_level) {
+            EC_SLAVE_DBG(slave, 1, "Invalid SDO download response!"
+                    " Retrying...\n");
+            ec_print_data(data, rec_size);
+        }
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_down_seg_check;
+        return;
+    }
+
+    if (((EC_READ_U8(data + 2) >> 4) & 0x01) != fsm->toggle) {
+        EC_SLAVE_ERR(slave, "Invalid toggle received during"
+                " segmented download:\n");
+        ec_print_data(data, rec_size);
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    fsm->offset += fsm->segment_size;
+    fsm->remaining -= fsm->segment_size;
+
+    if (fsm->remaining) { // more segments to download
+        fsm->toggle = !fsm->toggle;
+        ec_fsm_coe_down_prepare_segment_request(fsm, datagram);
+    } else {
+        fsm->state = ec_fsm_coe_end; // success
+    }
+}
+
+/****************************************************************************/
+
+/** Prepare an upload request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_fsm_coe_prepare_up(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_request_t *request = fsm->request;
+    ec_master_t *master = slave->master;
+
+    u8 *data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_COE,
+			10);
+    if (IS_ERR(data)) {
+        request->errno = PTR_ERR(data);
+        return PTR_ERR(data);
+    }
+
+    EC_WRITE_U16(data, 0x2 << 12); // SDO request
+    EC_WRITE_U8 (data + 2, 0x2 << 5); // initiate upload request
+    EC_WRITE_U16(data + 3, request->index);
+    EC_WRITE_U8 (data + 5, request->subindex);
+    memset(data + 6, 0x00, 4);
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Upload request:\n");
+        ec_print_data(data, 10);
+    }
+
+    fsm->state = ec_fsm_coe_up_request;
+    return 0;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: UP START.
+*/
+
+void ec_fsm_coe_up_start(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_request_t *request = fsm->request;
+
+    EC_SLAVE_DBG(slave, 1, "Uploading SDO 0x%04X:%02X.\n",
+            request->index, request->subindex);
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_COE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support CoE!\n");
+        request->errno = EPROTONOSUPPORT;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->request->jiffies_sent = jiffies;
+
+    if (ec_fsm_coe_prepare_up(fsm, datagram)) {
+        fsm->state = ec_fsm_coe_error;
+    }
+}
+
+/****************************************************************************/
+/**
+   CoE state: UP REQUEST.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_up_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    unsigned long diff_ms;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        if (ec_fsm_coe_prepare_up(fsm, datagram)) {
+            fsm->state = ec_fsm_coe_error;
+        }
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE upload request: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    diff_ms = (jiffies - fsm->request->jiffies_sent) * 1000 / HZ;
+
+    if (fsm->datagram->working_counter != 1) {
+        if (!fsm->datagram->working_counter) {
+            if (diff_ms < fsm->request->response_timeout) {
+#if DEBUG_RETRIES
+                EC_SLAVE_DBG(slave, 1, "Slave did not respond to"
+                        " SDO upload request. Retrying after %lu ms...\n",
+                        diff_ms);
+#endif
+                // no response; send request datagram again
+                if (ec_fsm_coe_prepare_up(fsm, datagram)) {
+                    fsm->state = ec_fsm_coe_error;
+                }
+                return;
+            }
+        }
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE upload request for"
+                " SDO 0x%04x:%x failed with timeout after %lu ms: ",
+                fsm->request->index, fsm->request->subindex, diff_ms);
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+#if DEBUG_LONG
+    if (diff_ms > 200) {
+        EC_SLAVE_WARN(slave, "SDO 0x%04x:%x upload took %lu ms.\n",
+                fsm->request->index, fsm->request->subindex, diff_ms);
+    }
+#endif
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_up_check;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: UP CHECK.
+*/
+
+void ec_fsm_coe_up_check(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= fsm->request->response_timeout) {
+            fsm->request->errno = EIO;
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Timeout after %lu ms while waiting for"
+                    " SDO 0x%04x:%x upload response.\n", diff_ms,
+                    fsm->request->index, fsm->request->subindex);
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_up_response;
+}
+
+/****************************************************************************/
+
+/** Prepare an SDO upload segment request.
+ */
+void ec_fsm_coe_up_prepare_segment_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    uint8_t *data =
+        ec_slave_mbox_prepare_send(fsm->slave, datagram, EC_MBOX_TYPE_COE,
+				10);
+    if (IS_ERR(data)) {
+        fsm->request->errno = PTR_ERR(data);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    EC_WRITE_U16(data, 0x2 << 12); // SDO request
+    EC_WRITE_U8 (data + 2, (fsm->toggle << 4 // toggle
+                | 0x3 << 5)); // upload segment request
+    memset(data + 3, 0x00, 7);
+
+    if (fsm->slave->master->debug_level) {
+        EC_SLAVE_DBG(fsm->slave, 1, "Upload segment request:\n");
+        ec_print_data(data, 10);
+    }
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: UP RESPONSE.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_up_response(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    uint16_t rec_index;
+    uint8_t *data, mbox_prot, rec_subindex;
+    size_t rec_size, data_size;
+    ec_sdo_request_t *request = fsm->request;
+    unsigned int expedited, size_specified;
+    int ret;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE upload response"
+                " datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE upload response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        request->errno = PTR_ERR(data);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Upload response:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_COE) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_WARN(slave, "Received mailbox protocol 0x%02X"
+                " as response.\n", mbox_prot);
+        return;
+    }
+
+    if (ec_fsm_coe_check_emergency(fsm, data, rec_size)) {
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_up_check;
+        return;
+    }
+
+    if (rec_size < 6) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Received currupted SDO upload response"
+                " (%zu bytes)!\n", rec_size);
+        ec_print_data(data, rec_size);
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 == 0x2 && // SDO request
+            EC_READ_U8(data + 2) >> 5 == 0x4) { // abort SDO transfer request
+        EC_SLAVE_ERR(slave, "SDO upload 0x%04X:%02X aborted.\n",
+               request->index, request->subindex);
+        if (rec_size >= 10) {
+            request->abort_code = EC_READ_U32(data + 6);
+            ec_canopen_abort_msg(slave, request->abort_code);
+        } else {
+            EC_SLAVE_ERR(slave, "No abort message.\n");
+        }
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 != 0x3 || // SDO response
+            EC_READ_U8(data + 2) >> 5 != 0x2) { // upload response
+        EC_SLAVE_ERR(slave, "Received unknown response while"
+                " uploading SDO 0x%04X:%02X.\n",
+                request->index, request->subindex);
+        ec_print_data(data, rec_size);
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    rec_index = EC_READ_U16(data + 3);
+    rec_subindex = EC_READ_U8(data + 5);
+
+    if (rec_index != request->index || rec_subindex != request->subindex) {
+        EC_SLAVE_ERR(slave, "Received upload response for wrong SDO"
+                " (0x%04X:%02X, requested: 0x%04X:%02X).\n",
+                rec_index, rec_subindex, request->index, request->subindex);
+        ec_print_data(data, rec_size);
+
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_up_check;
+        return;
+    }
+
+    // normal or expedited?
+    expedited = EC_READ_U8(data + 2) & 0x02;
+
+    if (expedited) {
+        size_specified = EC_READ_U8(data + 2) & 0x01;
+        if (size_specified) {
+            fsm->complete_size = 4 - ((EC_READ_U8(data + 2) & 0x0C) >> 2);
+        } else {
+            fsm->complete_size = 4;
+        }
+
+        if (rec_size < 6 + fsm->complete_size) {
+            request->errno = EIO;
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Received corrupted SDO expedited upload"
+                    " response (only %zu bytes)!\n", rec_size);
+            ec_print_data(data, rec_size);
+            return;
+        }
+
+        ret = ec_sdo_request_copy_data(request, data + 6, fsm->complete_size);
+        if (ret) {
+            request->errno = -ret;
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+    } else { // normal
+        if (rec_size < 10) {
+            request->errno = EIO;
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Received currupted SDO normal upload"
+                    " response (only %zu bytes)!\n", rec_size);
+            ec_print_data(data, rec_size);
+            return;
+        }
+
+        data_size = rec_size - 10;
+        fsm->complete_size = EC_READ_U32(data + 6);
+
+        ret = ec_sdo_request_alloc(request, fsm->complete_size);
+        if (ret) {
+            request->errno = -ret;
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+
+        ret = ec_sdo_request_copy_data(request, data + 10, data_size);
+        if (ret) {
+            request->errno = -ret;
+            fsm->state = ec_fsm_coe_error;
+            return;
+        }
+
+        fsm->toggle = 0;
+
+        if (data_size < fsm->complete_size) {
+            EC_SLAVE_DBG(slave, 1, "SDO data incomplete (%zu / %u)."
+                    " Segmenting...\n", data_size, fsm->complete_size);
+            ec_fsm_coe_up_prepare_segment_request(fsm, datagram);
+            fsm->retries = EC_FSM_RETRIES;
+            fsm->state = ec_fsm_coe_up_seg_request;
+            return;
+        }
+    }
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Uploaded data:\n");
+        ec_print_data(request->data, request->data_size);
+    }
+
+    fsm->state = ec_fsm_coe_end; // success
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: UP REQUEST.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_up_seg_request(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_fsm_coe_up_prepare_segment_request(fsm, datagram);
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE upload segment"
+                " request datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE upload segment"
+                " request failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_up_seg_check;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: UP CHECK.
+*/
+
+void ec_fsm_coe_up_seg_check(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE mailbox check"
+                " datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE mailbox check datagram"
+                " failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= fsm->request->response_timeout) {
+            fsm->request->errno = EIO;
+            fsm->state = ec_fsm_coe_error;
+            EC_SLAVE_ERR(slave, "Timeout while waiting for SDO upload"
+                    " segment response.\n");
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_coe_up_seg_response;
+}
+
+/****************************************************************************/
+
+/**
+   CoE state: UP RESPONSE.
+   \todo Timeout behavior
+*/
+
+void ec_fsm_coe_up_seg_response(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    uint8_t *data, mbox_prot;
+    size_t rec_size, data_size;
+    ec_sdo_request_t *request = fsm->request;
+    unsigned int last_segment;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive CoE upload segment"
+                " response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        EC_SLAVE_ERR(slave, "Reception of CoE upload segment"
+                " response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        request->errno = PTR_ERR(data);
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Upload segment response:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_COE) {
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (ec_fsm_coe_check_emergency(fsm, data, rec_size)) {
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_up_seg_check;
+        return;
+    }
+
+    if (rec_size < 10) {
+        EC_SLAVE_ERR(slave, "Received currupted SDO upload"
+                " segment response!\n");
+        ec_print_data(data, rec_size);
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 == 0x2 && // SDO request
+            EC_READ_U8 (data + 2) >> 5 == 0x4) { // abort SDO transfer request
+        EC_SLAVE_ERR(slave, "SDO upload 0x%04X:%02X aborted.\n",
+               request->index, request->subindex);
+        request->abort_code = EC_READ_U32(data + 6);
+        ec_canopen_abort_msg(slave, request->abort_code);
+        request->errno = EIO;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    if (EC_READ_U16(data) >> 12 != 0x3 || // SDO response
+        EC_READ_U8 (data + 2) >> 5 != 0x0) { // upload segment response
+        if (fsm->slave->master->debug_level) {
+            EC_SLAVE_DBG(slave, 1, "Invalid SDO upload segment response!\n");
+            ec_print_data(data, rec_size);
+        }
+        // check for CoE response again
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_up_seg_check;
+        return;
+    }
+
+    data_size = rec_size - 3; /* Header of segment upload is smaller than
+                                 normal upload */
+    if (rec_size == 10) {
+        uint8_t seg_size = (EC_READ_U8(data + 2) & 0xE) >> 1;
+        data_size -= seg_size;
+    }
+
+    if (request->data_size + data_size > fsm->complete_size) {
+        EC_SLAVE_ERR(slave, "SDO upload 0x%04X:%02X failed: Fragment"
+                " exceeding complete size!\n",
+                request->index, request->subindex);
+        request->errno = ENOBUFS;
+        fsm->state = ec_fsm_coe_error;
+        return;
+    }
+
+    memcpy(request->data + request->data_size, data + 3, data_size);
+    request->data_size += data_size;
+
+    last_segment = EC_READ_U8(data + 2) & 0x01;
+    if (!last_segment) {
+        fsm->toggle = !fsm->toggle;
+        ec_fsm_coe_up_prepare_segment_request(fsm, datagram);
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_coe_up_seg_request;
+        return;
+    }
+
+    if (request->data_size != fsm->complete_size) {
+        EC_SLAVE_WARN(slave, "SDO upload 0x%04X:%02X: Assembled data"
+                " size (%zu) does not match complete size (%u)!\n",
+                request->index, request->subindex,
+                request->data_size, fsm->complete_size);
+    }
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Uploaded data:\n");
+        ec_print_data(request->data, request->data_size);
+    }
+
+    fsm->state = ec_fsm_coe_end; // success
+}
+
+/****************************************************************************/
+
+/**
+   State: ERROR.
+*/
+
+void ec_fsm_coe_error(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
+
+/**
+   State: END.
+*/
+
+void ec_fsm_coe_end(
+        ec_fsm_coe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_coe.h b/drivers/net/ethercat/master/fsm_coe.h
new file mode 100644
index 000000000000..846e14473489
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_coe.h
@@ -0,0 +1,74 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT CoE state machines.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_COE_H__
+#define __EC_FSM_COE_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "slave.h"
+#include "sdo.h"
+#include "sdo_request.h"
+
+/****************************************************************************/
+
+typedef struct ec_fsm_coe ec_fsm_coe_t; /**< \see ec_fsm_coe */
+
+/** Finite state machines for the CANopen over EtherCAT protocol.
+ */
+struct ec_fsm_coe {
+    ec_slave_t *slave; /**< slave the FSM runs on */
+    unsigned int retries; /**< retries upon datagram timeout */
+
+    void (*state)(ec_fsm_coe_t *, ec_datagram_t *); /**< CoE state function */
+    ec_datagram_t *datagram; /**< Datagram used in last step. */
+    unsigned long jiffies_start; /**< CoE timestamp. */
+    ec_sdo_t *sdo; /**< current SDO */
+    uint8_t subindex; /**< current subindex */
+    ec_sdo_request_t *request; /**< SDO request */
+    uint32_t complete_size; /**< Used when segmenting. */
+    uint8_t toggle; /**< toggle bit for segment commands */
+    uint32_t offset; /**< Data offset during segmented download. */
+    uint32_t remaining; /**< Remaining bytes during segmented download. */
+    size_t segment_size; /**< Current segment size. */
+};
+
+/****************************************************************************/
+
+void ec_fsm_coe_init(ec_fsm_coe_t *);
+void ec_fsm_coe_clear(ec_fsm_coe_t *);
+
+void ec_fsm_coe_dictionary(ec_fsm_coe_t *, ec_slave_t *);
+void ec_fsm_coe_transfer(ec_fsm_coe_t *, ec_slave_t *, ec_sdo_request_t *);
+
+int ec_fsm_coe_exec(ec_fsm_coe_t *, ec_datagram_t *);
+int ec_fsm_coe_success(const ec_fsm_coe_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_eoe.c b/drivers/net/ethercat/master/fsm_eoe.c
new file mode 100644
index 000000000000..b3ef6898939f
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_eoe.c
@@ -0,0 +1,509 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2014  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT EoE state machines.
+*/
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "fsm_eoe.h"
+
+/****************************************************************************/
+
+/** Maximum time to wait for a set IP parameter response.
+ */
+#define EC_EOE_RESPONSE_TIMEOUT 3000 // [ms]
+
+/****************************************************************************/
+
+// prototypes for private functions
+void memcpy_swap32(void *, const void *);
+
+int ec_fsm_eoe_prepare_set(ec_fsm_eoe_t *, ec_datagram_t *);
+
+void ec_fsm_eoe_set_ip_start(ec_fsm_eoe_t *, ec_datagram_t *);
+void ec_fsm_eoe_set_ip_request(ec_fsm_eoe_t *, ec_datagram_t *);
+void ec_fsm_eoe_set_ip_check(ec_fsm_eoe_t *, ec_datagram_t *);
+void ec_fsm_eoe_set_ip_response(ec_fsm_eoe_t *, ec_datagram_t *);
+
+void ec_fsm_eoe_end(ec_fsm_eoe_t *, ec_datagram_t *);
+void ec_fsm_eoe_error(ec_fsm_eoe_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+/** Host-architecture-independent 32-bit swap function.
+ *
+ * The internal storage of struct in_addr is always big-endian.
+ * The mailbox protocol format to supply IPv4 adresses is little-endian
+ * (Yuck!). So we need a swap function, that is independent of the CPU
+ * architecture. ntohl()/htonl() can not be used, because they evaluate to
+ * NOPs if the host architecture matches the target architecture!
+ */
+void memcpy_swap32(void *dst, const void *src)
+{
+    int i;
+    for (i = 0; i < 4; i++) {
+        ((u8 *) dst)[i] = ((const u8 *) src)[3 - i];
+    }
+}
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_eoe_init(
+        ec_fsm_eoe_t *fsm /**< finite state machine */
+        )
+{
+    fsm->slave = NULL;
+    fsm->retries = 0;
+    fsm->state = NULL;
+    fsm->datagram = NULL;
+    fsm->jiffies_start = 0;
+    fsm->request = NULL;
+    fsm->frame_type_retries = 0;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_eoe_clear(
+        ec_fsm_eoe_t *fsm /**< finite state machine */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** Starts to set the EoE IP partameters of a slave.
+ */
+void ec_fsm_eoe_set_ip_param(
+        ec_fsm_eoe_t *fsm, /**< State machine. */
+        ec_slave_t *slave, /**< EtherCAT slave. */
+        ec_eoe_request_t *request /**< EoE request. */
+        )
+{
+    fsm->slave = slave;
+    fsm->request = request;
+    fsm->state = ec_fsm_eoe_set_ip_start;
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * \return 1 if the datagram was used, else 0.
+ */
+int ec_fsm_eoe_exec(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    int datagram_used = 0;
+
+    if (fsm->datagram &&
+            (fsm->datagram->state == EC_DATAGRAM_INIT ||
+             fsm->datagram->state == EC_DATAGRAM_QUEUED ||
+             fsm->datagram->state == EC_DATAGRAM_SENT)) {
+        // datagram not received yet
+        return datagram_used;
+    }
+
+    fsm->state(fsm, datagram);
+
+    datagram_used =
+        fsm->state != ec_fsm_eoe_end && fsm->state != ec_fsm_eoe_error;
+
+    if (datagram_used) {
+        fsm->datagram = datagram;
+    } else {
+        fsm->datagram = NULL;
+    }
+
+    return datagram_used;
+}
+
+/****************************************************************************/
+
+/** Returns, if the state machine terminated with success.
+ *
+ * \return non-zero if successful.
+ */
+int ec_fsm_eoe_success(const ec_fsm_eoe_t *fsm /**< Finite state machine */)
+{
+    return fsm->state == ec_fsm_eoe_end;
+}
+
+/*****************************************************************************
+ * EoE set IP parameter state machine
+ ****************************************************************************/
+
+/** Prepare a set IP parameters operation.
+ *
+ * \return 0 on success, otherwise a negative error code.
+ */
+int ec_fsm_eoe_prepare_set(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    uint8_t *data, *cur;
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    ec_eoe_request_t *req = fsm->request;
+
+    // Note: based on wireshark packet filter it suggests that the EOE_INIT
+    //   information is a fixed size with fixed information positions.
+    //   see: packet-ecatmb.h and packet-ecatmb.c
+    //   However, TwinCAT 2.1 testing also indicates that if a piece of
+    //   information is missing then all subsequent items are ignored
+    //   Also, if you want DHCP, then only set the mac address.
+    size_t size = 8 +                       // header + flags
+                  ETH_ALEN +                // mac address
+                  4 +                       // ip address
+                  4 +                       // subnet mask
+                  4 +                       // gateway
+                  4 +                       // dns server
+                  EC_MAX_HOSTNAME_SIZE;     // dns name
+
+    data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_EOE,
+            size);
+    if (IS_ERR(data)) {
+        return PTR_ERR(data);
+    }
+
+    // zero data
+    memset(data, 0, size);
+
+    // header
+    EC_WRITE_U8(data, EC_EOE_FRAMETYPE_SET_IP_REQ); // Set IP parameter req.
+    EC_WRITE_U8(data + 1, 0x00);                    // not used
+    EC_WRITE_U16(data + 2, 0x0000);                 // not used
+
+    EC_WRITE_U32(data + 4,
+            ((req->mac_address_included != 0) << 0) |
+            ((req->ip_address_included != 0) << 1) |
+            ((req->subnet_mask_included != 0) << 2) |
+            ((req->gateway_included != 0) << 3) |
+            ((req->dns_included != 0) << 4) |
+            ((req->name_included != 0) << 5)
+            );
+
+    cur = data + 8;
+
+    if (req->mac_address_included) {
+        memcpy(cur, req->mac_address, ETH_ALEN);
+    }
+    cur += ETH_ALEN;
+
+    if (req->ip_address_included) {
+        memcpy_swap32(cur, &req->ip_address);
+    }
+    cur += 4;
+
+    if (req->subnet_mask_included) {
+        memcpy_swap32(cur, &req->subnet_mask);
+    }
+    cur += 4;
+
+    if (req->gateway_included) {
+        memcpy_swap32(cur, &req->gateway);
+    }
+    cur += 4;
+
+    if (req->dns_included) {
+        memcpy_swap32(cur, &req->dns);
+    }
+    cur += 4;
+
+    if (req->name_included) {
+        memcpy(cur, req->name, EC_MAX_HOSTNAME_SIZE);
+    }
+    cur += EC_MAX_HOSTNAME_SIZE;
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 0, "Set IP parameter request:\n");
+        ec_print_data(data, cur - data);
+    }
+
+    fsm->request->jiffies_sent = jiffies;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** EoE state: SET IP START.
+ */
+void ec_fsm_eoe_set_ip_start(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    EC_SLAVE_DBG(slave, 1, "Setting IP parameters.\n");
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_EOE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support EoE!\n");
+        fsm->state = ec_fsm_eoe_error;
+        return;
+    }
+
+    if (ec_fsm_eoe_prepare_set(fsm, datagram)) {
+        fsm->state = ec_fsm_eoe_error;
+        return;
+    }
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_eoe_set_ip_request;
+}
+
+/****************************************************************************/
+
+/** EoE state: SET IP REQUEST.
+ */
+void ec_fsm_eoe_set_ip_request(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        if (ec_fsm_eoe_prepare_set(fsm, datagram)) {
+            fsm->state = ec_fsm_eoe_error;
+        }
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive EoE set IP parameter"
+                " request: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        unsigned long diff_ms =
+            (jiffies - fsm->request->jiffies_sent) * 1000 / HZ;
+
+        if (!fsm->datagram->working_counter) {
+            if (diff_ms < EC_EOE_RESPONSE_TIMEOUT) {
+                // no response; send request datagram again
+                if (ec_fsm_eoe_prepare_set(fsm, datagram)) {
+                    fsm->state = ec_fsm_eoe_error;
+                }
+                return;
+            }
+        }
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Reception of EoE set IP parameter request"
+                " failed after %lu ms: ", diff_ms);
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_eoe_set_ip_check;
+    fsm->frame_type_retries = 10;
+}
+
+/****************************************************************************/
+
+/** EoE state: SET IP CHECK.
+ */
+void ec_fsm_eoe_set_ip_check(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive EoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Reception of EoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= EC_EOE_RESPONSE_TIMEOUT) {
+            fsm->state = ec_fsm_eoe_error;
+            EC_SLAVE_ERR(slave, "Timeout after %lu ms while waiting for"
+                    " set IP parameter response.\n", diff_ms);
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_eoe_set_ip_response;
+}
+
+/****************************************************************************/
+
+/** EoE state: SET IP RESPONSE.
+ */
+void ec_fsm_eoe_set_ip_response(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    uint8_t *data, mbox_prot, frame_type;
+    size_t rec_size;
+    ec_eoe_request_t *req = fsm->request;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive EoE read response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Reception of EoE read response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        fsm->state = ec_fsm_eoe_error;
+        return;
+    }
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 0, "Set IP parameter response:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_EOE) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        return;
+    }
+
+    if (rec_size < 4) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_ERR(slave, "Received currupted EoE set IP parameter response"
+                " (%zu bytes)!\n", rec_size);
+        ec_print_data(data, rec_size);
+        return;
+    }
+
+    frame_type = EC_READ_U8(data) & 0x0f;
+
+    if (frame_type != EC_EOE_FRAMETYPE_SET_IP_RES) {
+        if (master->debug_level) {
+            EC_SLAVE_DBG(slave, 0, "Received no set IP parameter response"
+                    " (frame type %x).\n", frame_type);
+            ec_print_data(data, rec_size);
+        }
+        if (fsm->frame_type_retries--) {
+            // there may be an EoE segment left in the mailbox.
+            // discard it and receive again.
+            fsm->jiffies_start = fsm->datagram->jiffies_sent;
+            ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+            fsm->retries = EC_FSM_RETRIES;
+            fsm->state = ec_fsm_eoe_set_ip_check;
+            return;
+        }
+        else {
+            EC_SLAVE_ERR(slave, "Received no set IP parameter response.\n");
+            fsm->state = ec_fsm_eoe_error;
+            return;
+        }
+    }
+
+    req->result = EC_READ_U16(data + 2); // result code 0x0000 means success
+
+    if (req->result) {
+        fsm->state = ec_fsm_eoe_error;
+        EC_SLAVE_DBG(slave, 1, "EoE set IP parameters failed with result code"
+                " 0x%04X.\n", req->result);
+    } else {
+        fsm->state = ec_fsm_eoe_end; // success
+    }
+}
+
+/****************************************************************************/
+
+/** State: ERROR.
+ */
+void ec_fsm_eoe_error(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** State: END.
+ */
+void ec_fsm_eoe_end(
+        ec_fsm_eoe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_eoe.h b/drivers/net/ethercat/master/fsm_eoe.h
new file mode 100644
index 000000000000..7f901bdd5e3b
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_eoe.h
@@ -0,0 +1,67 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2014  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT EoE set IP parameter state machines.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_EOE_H__
+#define __EC_FSM_EOE_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "slave.h"
+#include "eoe_request.h"
+
+/****************************************************************************/
+
+typedef struct ec_fsm_eoe ec_fsm_eoe_t; /**< \see ec_fsm_eoe */
+
+/** Finite state machines for the Ethernet over EtherCAT protocol.
+ */
+struct ec_fsm_eoe {
+    ec_slave_t *slave; /**< slave the FSM runs on */
+    unsigned int retries; /**< retries upon datagram timeout */
+
+    void (*state)(ec_fsm_eoe_t *, ec_datagram_t *); /**< EoE state function */
+    ec_datagram_t *datagram; /**< Datagram used in the previous step. */
+    unsigned long jiffies_start; /**< Timestamp. */
+    ec_eoe_request_t *request; /**< EoE request */
+    unsigned int frame_type_retries; /**< retries upon wrong frame type. */
+};
+
+/****************************************************************************/
+
+void ec_fsm_eoe_init(ec_fsm_eoe_t *);
+void ec_fsm_eoe_clear(ec_fsm_eoe_t *);
+
+void ec_fsm_eoe_set_ip_param(ec_fsm_eoe_t *, ec_slave_t *,
+        ec_eoe_request_t *);
+
+int ec_fsm_eoe_exec(ec_fsm_eoe_t *, ec_datagram_t *);
+int ec_fsm_eoe_success(const ec_fsm_eoe_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_foe.c b/drivers/net/ethercat/master/fsm_foe.c
new file mode 100644
index 000000000000..378ae369401c
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_foe.c
@@ -0,0 +1,915 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2008  Olav Zarges, imc Messsysteme GmbH
+ *                2013  Florian Pose <fp@igh.de>
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT FoE state machines.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "fsm_foe.h"
+#include "foe.h"
+
+/****************************************************************************/
+
+/** Maximum time in ms to wait for responses when reading out the dictionary.
+ */
+#define EC_FSM_FOE_TIMEOUT 3000
+
+/** Size of the FoE header.
+ */
+#define EC_FOE_HEADER_SIZE 6
+// uint8_t  OpCode
+// uint8_t  reserved
+// uint32_t PacketNo, Password, ErrorCode
+
+//#define DEBUG_FOE
+
+/****************************************************************************/
+
+/** FoE OpCodes.
+ */
+enum {
+    EC_FOE_OPCODE_RRQ  = 1, /**< Read request. */
+    EC_FOE_OPCODE_WRQ  = 2, /**< Write request. */
+    EC_FOE_OPCODE_DATA = 3, /**< Data. */
+    EC_FOE_OPCODE_ACK  = 4, /**< Acknowledge. */
+    EC_FOE_OPCODE_ERR  = 5, /**< Error. */
+    EC_FOE_OPCODE_BUSY = 6  /**< Busy. */
+};
+
+/****************************************************************************/
+
+int ec_foe_prepare_data_send(ec_fsm_foe_t *, ec_datagram_t *);
+int ec_foe_prepare_wrq_send(ec_fsm_foe_t *, ec_datagram_t *);
+int ec_foe_prepare_rrq_send(ec_fsm_foe_t *, ec_datagram_t *);
+int ec_foe_prepare_send_ack(ec_fsm_foe_t *, ec_datagram_t *);
+
+void ec_foe_set_tx_error(ec_fsm_foe_t *, uint32_t);
+void ec_foe_set_rx_error(ec_fsm_foe_t *, uint32_t);
+
+void ec_fsm_foe_end(ec_fsm_foe_t *, ec_datagram_t *);
+void ec_fsm_foe_error(ec_fsm_foe_t *, ec_datagram_t *);
+
+void ec_fsm_foe_state_wrq_sent(ec_fsm_foe_t *, ec_datagram_t *);
+void ec_fsm_foe_state_rrq_sent(ec_fsm_foe_t *, ec_datagram_t *);
+
+void ec_fsm_foe_state_ack_check(ec_fsm_foe_t *, ec_datagram_t *);
+void ec_fsm_foe_state_ack_read(ec_fsm_foe_t *, ec_datagram_t *);
+
+void ec_fsm_foe_state_data_sent(ec_fsm_foe_t *, ec_datagram_t *);
+
+void ec_fsm_foe_state_data_check(ec_fsm_foe_t *, ec_datagram_t *);
+void ec_fsm_foe_state_data_read(ec_fsm_foe_t *, ec_datagram_t *);
+void ec_fsm_foe_state_sent_ack(ec_fsm_foe_t *, ec_datagram_t *);
+
+void ec_fsm_foe_write_start(ec_fsm_foe_t *, ec_datagram_t *);
+void ec_fsm_foe_read_start(ec_fsm_foe_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_foe_init(
+        ec_fsm_foe_t *fsm /**< finite state machine */
+        )
+{
+    fsm->state = NULL;
+    fsm->datagram = NULL;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_foe_clear(ec_fsm_foe_t *fsm /**< finite state machine */)
+{
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * \return 1, if the datagram was used, else 0.
+ */
+int ec_fsm_foe_exec(
+        ec_fsm_foe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    int datagram_used = 0;
+
+    if (fsm->datagram &&
+            (fsm->datagram->state == EC_DATAGRAM_INIT ||
+             fsm->datagram->state == EC_DATAGRAM_QUEUED ||
+             fsm->datagram->state == EC_DATAGRAM_SENT)) {
+        // datagram not received yet
+        return datagram_used;
+    }
+
+    fsm->state(fsm, datagram);
+
+    datagram_used =
+        fsm->state != ec_fsm_foe_end && fsm->state != ec_fsm_foe_error;
+
+    if (datagram_used) {
+        fsm->datagram = datagram;
+    } else {
+        fsm->datagram = NULL;
+    }
+
+    return datagram_used;
+}
+
+/****************************************************************************/
+
+/** Returns, if the state machine terminated with success.
+ * \return non-zero if successful.
+ */
+int ec_fsm_foe_success(const ec_fsm_foe_t *fsm /**< Finite state machine */)
+{
+    return fsm->state == ec_fsm_foe_end;
+}
+
+/****************************************************************************/
+
+/** Prepares an FoE transfer.
+ */
+void ec_fsm_foe_transfer(
+        ec_fsm_foe_t *fsm, /**< State machine. */
+        ec_slave_t *slave, /**< EtherCAT slave. */
+        ec_foe_request_t *request /**< Sdo request. */
+        )
+{
+    fsm->slave = slave;
+    fsm->request = request;
+
+    if (request->dir == EC_DIR_OUTPUT) {
+        fsm->tx_buffer = fsm->request->buffer;
+        fsm->tx_buffer_size = fsm->request->data_size;
+        fsm->tx_buffer_offset = 0;
+
+        fsm->tx_filename = fsm->request->file_name;
+        fsm->tx_filename_len = strlen(fsm->tx_filename);
+
+        fsm->state = ec_fsm_foe_write_start;
+    }
+    else {
+        fsm->rx_buffer = fsm->request->buffer;
+        fsm->rx_buffer_size = fsm->request->buffer_size;
+
+        fsm->rx_filename = fsm->request->file_name;
+        fsm->rx_filename_len = strlen(fsm->rx_filename);
+
+        fsm->state = ec_fsm_foe_read_start;
+    }
+}
+
+/****************************************************************************/
+
+/** State: ERROR.
+ */
+void ec_fsm_foe_error(
+        ec_fsm_foe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+}
+
+/****************************************************************************/
+
+/** State: END.
+ */
+void ec_fsm_foe_end(
+        ec_fsm_foe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+}
+
+/****************************************************************************/
+
+/** Sends a file or the next fragment.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_foe_prepare_data_send(
+        ec_fsm_foe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    size_t remaining_size, current_size;
+    uint8_t *data;
+
+    remaining_size = fsm->tx_buffer_size - fsm->tx_buffer_offset;
+
+    if (remaining_size < fsm->slave->configured_tx_mailbox_size
+            - EC_MBOX_HEADER_SIZE - EC_FOE_HEADER_SIZE) {
+        current_size = remaining_size;
+        fsm->tx_last_packet = 1;
+    } else {
+        current_size = fsm->slave->configured_tx_mailbox_size
+            - EC_MBOX_HEADER_SIZE - EC_FOE_HEADER_SIZE;
+    }
+
+    data = ec_slave_mbox_prepare_send(fsm->slave,
+            datagram, EC_MBOX_TYPE_FOE, current_size + EC_FOE_HEADER_SIZE);
+    if (IS_ERR(data)) {
+        return -1;
+    }
+
+    EC_WRITE_U16(data, EC_FOE_OPCODE_DATA);    // OpCode = DataBlock req.
+    EC_WRITE_U32(data + 2, fsm->tx_packet_no); // PacketNo, Password
+
+    memcpy(data + EC_FOE_HEADER_SIZE,
+            fsm->tx_buffer + fsm->tx_buffer_offset, current_size);
+    fsm->tx_current_size = current_size;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Prepare a write request (WRQ) with filename
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_foe_prepare_wrq_send(
+        ec_fsm_foe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    size_t current_size;
+    uint8_t *data;
+
+    fsm->tx_buffer_offset = 0;
+    fsm->tx_current_size = 0;
+    fsm->tx_packet_no = 0;
+    fsm->tx_last_packet = 0;
+
+    current_size = fsm->tx_filename_len;
+
+    data = ec_slave_mbox_prepare_send(fsm->slave, datagram,
+            EC_MBOX_TYPE_FOE, current_size + EC_FOE_HEADER_SIZE);
+    if (IS_ERR(data)) {
+        return -1;
+    }
+
+    EC_WRITE_U16( data, EC_FOE_OPCODE_WRQ); // fsm write request
+    EC_WRITE_U32( data + 2, fsm->tx_packet_no );
+
+    memcpy(data + EC_FOE_HEADER_SIZE, fsm->tx_filename, current_size);
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Initializes the FoE write state machine.
+ */
+void ec_fsm_foe_write_start(
+        ec_fsm_foe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    fsm->tx_buffer_offset = 0;
+    fsm->tx_current_size = 0;
+    fsm->tx_packet_no = 0;
+    fsm->tx_last_packet = 0;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_FOE)) {
+        ec_foe_set_tx_error(fsm, FOE_MBOX_PROT_ERROR);
+        EC_SLAVE_ERR(slave, "Slave does not support FoE!\n");
+        return;
+    }
+
+    if (ec_foe_prepare_wrq_send(fsm, datagram)) {
+        ec_foe_set_tx_error(fsm, FOE_PROT_ERROR);
+        return;
+    }
+
+    fsm->state = ec_fsm_foe_state_wrq_sent;
+}
+
+/****************************************************************************/
+
+/** Check for acknowledge.
+ */
+void ec_fsm_foe_state_ack_check(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_rx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to receive FoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        ec_foe_set_rx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE mailbox check datagram"
+                " failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        // slave did not put anything in the mailbox yet
+        unsigned long diff_ms = (fsm->datagram->jiffies_received -
+                fsm->jiffies_start) * 1000 / HZ;
+        if (diff_ms >= EC_FSM_FOE_TIMEOUT) {
+            ec_foe_set_tx_error(fsm, FOE_TIMEOUT_ERROR);
+            EC_SLAVE_ERR(slave, "Timeout while waiting for ack response.\n");
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_foe_state_ack_read;
+}
+
+/****************************************************************************/
+
+/** Acknowledge a read operation.
+ */
+void ec_fsm_foe_state_ack_read(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    uint8_t *data, mbox_prot;
+    uint8_t opCode;
+    size_t rec_size;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_rx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to receive FoE ack response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        ec_foe_set_rx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE ack response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        ec_foe_set_tx_error(fsm, FOE_PROT_ERROR);
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_FOE) {
+        ec_foe_set_tx_error(fsm, FOE_MBOX_PROT_ERROR);
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        return;
+    }
+
+    opCode = EC_READ_U8(data);
+
+    if (opCode == EC_FOE_OPCODE_BUSY) {
+        // slave not ready
+        if (ec_foe_prepare_data_send(fsm, datagram)) {
+            ec_foe_set_tx_error(fsm, FOE_PROT_ERROR);
+            EC_SLAVE_ERR(slave, "Slave is busy.\n");
+            return;
+        }
+        fsm->state = ec_fsm_foe_state_data_sent;
+        return;
+    }
+
+    if (opCode == EC_FOE_OPCODE_ACK) {
+        fsm->tx_packet_no++;
+        fsm->tx_buffer_offset += fsm->tx_current_size;
+
+        if (fsm->tx_last_packet) {
+            fsm->state = ec_fsm_foe_end;
+            return;
+        }
+
+        if (ec_foe_prepare_data_send(fsm, datagram)) {
+            ec_foe_set_tx_error(fsm, FOE_PROT_ERROR);
+            return;
+        }
+        fsm->state = ec_fsm_foe_state_data_sent;
+        return;
+    }
+    ec_foe_set_tx_error(fsm, FOE_ACK_ERROR);
+}
+
+/****************************************************************************/
+
+/** State: WRQ SENT.
+ *
+ * Checks is the previous transmit datagram succeded and sends the next
+ * fragment, if necessary.
+ */
+void ec_fsm_foe_state_wrq_sent(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_rx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to send FoE WRQ: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        // slave did not put anything in the mailbox yet
+        ec_foe_set_rx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE WRQ failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_foe_state_ack_check;
+}
+
+/****************************************************************************/
+
+/** State: WRQ SENT.
+ *
+ * Checks is the previous transmit datagram succeded and sends the next
+ * fragment, if necessary.
+ */
+void ec_fsm_foe_state_data_sent(
+        ec_fsm_foe_t *fsm, /**< Foe statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_tx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to receive FoE ack response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        ec_foe_set_tx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE data send failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    ec_slave_mbox_prepare_check(slave, datagram);
+    fsm->jiffies_start = jiffies;
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_foe_state_ack_check;
+}
+
+/****************************************************************************/
+
+/** Prepare a read request (RRQ) with filename
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_foe_prepare_rrq_send(
+        ec_fsm_foe_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    size_t current_size;
+    uint8_t *data;
+
+    current_size = fsm->rx_filename_len;
+
+    data = ec_slave_mbox_prepare_send(fsm->slave, datagram,
+            EC_MBOX_TYPE_FOE, current_size + EC_FOE_HEADER_SIZE);
+    if (IS_ERR(data)) {
+        return -1;
+    }
+
+    EC_WRITE_U16(data, EC_FOE_OPCODE_RRQ); // fsm read request
+    EC_WRITE_U32(data + 2, 0x00000000); // no passwd
+    memcpy(data + EC_FOE_HEADER_SIZE, fsm->rx_filename, current_size);
+
+    if (fsm->slave->master->debug_level) {
+        EC_SLAVE_DBG(fsm->slave, 1, "FoE Read Request:\n");
+        ec_print_data(data, current_size + EC_FOE_HEADER_SIZE);
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Prepare to send an acknowledge.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_foe_prepare_send_ack(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    uint8_t *data;
+
+    data = ec_slave_mbox_prepare_send(fsm->slave, datagram,
+            EC_MBOX_TYPE_FOE, EC_FOE_HEADER_SIZE);
+    if (IS_ERR(data)) {
+        return -1;
+    }
+
+    EC_WRITE_U16(data, EC_FOE_OPCODE_ACK);
+    EC_WRITE_U32(data + 2, fsm->rx_expected_packet_no);
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** State: RRQ SENT.
+ *
+ * Checks is the previous transmit datagram succeeded and sends the next
+ * fragment, if necessary.
+ */
+void ec_fsm_foe_state_rrq_sent(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_rx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to send FoE RRQ: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        // slave did not put anything in the mailbox yet
+        ec_foe_set_rx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE RRQ failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_foe_state_data_check;
+}
+
+/****************************************************************************/
+
+/** Starting state for read operations.
+ */
+void ec_fsm_foe_read_start(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    fsm->rx_buffer_offset = 0;
+    fsm->rx_expected_packet_no = 1;
+    fsm->rx_last_packet = 0;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_FOE)) {
+        ec_foe_set_tx_error(fsm, FOE_MBOX_PROT_ERROR);
+        EC_SLAVE_ERR(slave, "Slave does not support FoE!\n");
+        return;
+    }
+
+    if (ec_foe_prepare_rrq_send(fsm, datagram)) {
+        ec_foe_set_rx_error(fsm, FOE_PROT_ERROR);
+        return;
+    }
+
+    fsm->state = ec_fsm_foe_state_rrq_sent;
+}
+
+/****************************************************************************/
+
+/** Check for data.
+ */
+void ec_fsm_foe_state_data_check(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_rx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to send FoE DATA READ: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        ec_foe_set_rx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE DATA READ: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms = (fsm->datagram->jiffies_received -
+                fsm->jiffies_start) * 1000 / HZ;
+        if (diff_ms >= EC_FSM_FOE_TIMEOUT) {
+            ec_foe_set_tx_error(fsm, FOE_TIMEOUT_ERROR);
+            EC_SLAVE_ERR(slave, "Timeout while waiting for ack response.\n");
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_foe_state_data_read;
+}
+
+/****************************************************************************/
+
+/** Start reading data.
+ */
+void ec_fsm_foe_state_data_read(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    size_t rec_size;
+    uint8_t *data, opCode, packet_no, mbox_prot;
+
+    ec_slave_t *slave = fsm->slave;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_rx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to receive FoE DATA READ datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        ec_foe_set_rx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE DATA READ failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        ec_foe_set_rx_error(fsm, FOE_MBOX_FETCH_ERROR);
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_FOE) {
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        ec_foe_set_rx_error(fsm, FOE_PROT_ERROR);
+        return;
+    }
+
+    opCode = EC_READ_U8(data);
+
+    if (opCode == EC_FOE_OPCODE_BUSY) {
+        if (ec_foe_prepare_send_ack(fsm, datagram)) {
+            ec_foe_set_rx_error(fsm, FOE_PROT_ERROR);
+        }
+        return;
+    }
+
+    if (opCode == EC_FOE_OPCODE_ERR) {
+        fsm->request->error_code = EC_READ_U32(data + 2);
+        EC_SLAVE_ERR(slave, "Received FoE Error Request (code 0x%08x).\n",
+                fsm->request->error_code);
+        if (rec_size > 6) {
+            uint8_t text[256];
+            strncpy(text, data + 6, min(rec_size - 6, sizeof(text)));
+            EC_SLAVE_ERR(slave, "FoE Error Text: %s\n", text);
+        }
+        ec_foe_set_rx_error(fsm, FOE_OPCODE_ERROR);
+        return;
+    }
+
+    if (opCode != EC_FOE_OPCODE_DATA) {
+        EC_SLAVE_ERR(slave, "Received OPCODE %x, expected %x.\n",
+                opCode, EC_FOE_OPCODE_DATA);
+        fsm->request->error_code = 0x00000000;
+        ec_foe_set_rx_error(fsm, FOE_OPCODE_ERROR);
+        return;
+    }
+
+    packet_no = EC_READ_U16(data + 2);
+    if (packet_no != fsm->rx_expected_packet_no) {
+        EC_SLAVE_ERR(slave, "Received unexpected packet number.\n");
+        ec_foe_set_rx_error(fsm, FOE_PACKETNO_ERROR);
+        return;
+    }
+
+    rec_size -= EC_FOE_HEADER_SIZE;
+
+    if (fsm->rx_buffer_size >= fsm->rx_buffer_offset + rec_size) {
+        memcpy(fsm->rx_buffer + fsm->rx_buffer_offset,
+                data + EC_FOE_HEADER_SIZE, rec_size);
+        fsm->rx_buffer_offset += rec_size;
+    }
+
+    fsm->rx_last_packet =
+        (rec_size + EC_MBOX_HEADER_SIZE + EC_FOE_HEADER_SIZE
+         != slave->configured_rx_mailbox_size);
+
+    if (fsm->rx_last_packet ||
+            (slave->configured_rx_mailbox_size - EC_MBOX_HEADER_SIZE
+             - EC_FOE_HEADER_SIZE + fsm->rx_buffer_offset)
+            <= fsm->rx_buffer_size) {
+        // either it was the last packet or a new packet will fit into the
+        // delivered buffer
+#ifdef DEBUG_FOE
+        EC_SLAVE_DBG(fsm->slave, 0, "last_packet=true\n");
+#endif
+        if (ec_foe_prepare_send_ack(fsm, datagram)) {
+            ec_foe_set_rx_error(fsm, FOE_RX_DATA_ACK_ERROR);
+            return;
+        }
+
+        fsm->state = ec_fsm_foe_state_sent_ack;
+    }
+    else {
+        // no more data fits into the delivered buffer
+        // ... wait for new read request
+        EC_SLAVE_ERR(slave, "Data do not fit in receive buffer!\n");
+        printk(KERN_CONT "  rx_buffer_size = %d\n", fsm->rx_buffer_size);
+        printk(KERN_CONT "rx_buffer_offset = %d\n", fsm->rx_buffer_offset);
+        printk(KERN_CONT "        rec_size = %zd\n", rec_size);
+        printk(KERN_CONT " rx_mailbox_size = %d\n",
+                slave->configured_rx_mailbox_size);
+        printk(KERN_CONT "  rx_last_packet = %d\n", fsm->rx_last_packet);
+        fsm->request->result = FOE_READY;
+    }
+}
+
+/****************************************************************************/
+
+/** Sent an acknowledge.
+ */
+void ec_fsm_foe_state_sent_ack(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+#ifdef DEBUG_FOE
+    EC_SLAVE_DBG(fsm->slave, 0, "%s()\n", __func__);
+#endif
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        ec_foe_set_rx_error(fsm, FOE_RECEIVE_ERROR);
+        EC_SLAVE_ERR(slave, "Failed to send FoE ACK: ");
+        ec_datagram_print_state(fsm->datagram);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        // slave did not put anything into the mailbox yet
+        ec_foe_set_rx_error(fsm, FOE_WC_ERROR);
+        EC_SLAVE_ERR(slave, "Reception of FoE ACK failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+
+    if (fsm->rx_last_packet) {
+        fsm->rx_expected_packet_no = 0;
+        fsm->request->data_size = fsm->rx_buffer_offset;
+        fsm->state = ec_fsm_foe_end;
+    }
+    else {
+        fsm->rx_expected_packet_no++;
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_foe_state_data_check;
+    }
+}
+
+/****************************************************************************/
+
+/** Set an error code and go to the send error state.
+ */
+void ec_foe_set_tx_error(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        uint32_t errorcode /**< FoE error code. */
+        )
+{
+    fsm->request->result = errorcode;
+    fsm->state = ec_fsm_foe_error;
+}
+
+/****************************************************************************/
+
+/** Set an error code and go to the receive error state.
+ */
+void ec_foe_set_rx_error(
+        ec_fsm_foe_t *fsm, /**< FoE statemachine. */
+        uint32_t errorcode /**< FoE error code. */
+        )
+{
+    fsm->request->result = errorcode;
+    fsm->state = ec_fsm_foe_error;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_foe.h b/drivers/net/ethercat/master/fsm_foe.h
new file mode 100644
index 000000000000..55711dccee06
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_foe.h
@@ -0,0 +1,87 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2008       Olav Zarges, imc Messsysteme GmbH
+ *                2009-2012  Florian Pose <fp@igh.de>
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT FoE state machines.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_FOE_H__
+#define __EC_FSM_FOE_H__
+
+#include "globals.h"
+#include "../include/ecrt.h"
+#include "datagram.h"
+#include "slave.h"
+#include "foe_request.h"
+
+/****************************************************************************/
+
+typedef struct ec_fsm_foe ec_fsm_foe_t; /**< \see ec_fsm_foe */
+
+/** Finite state machines for the CANopen-over-EtherCAT protocol.
+ */
+struct ec_fsm_foe {
+    ec_slave_t *slave; /**< Slave the FSM runs on. */
+    unsigned int retries; /**< Retries upon datagram timeout */
+
+    void (*state)(ec_fsm_foe_t *, ec_datagram_t *); /**< FoE state function.
+                                                     */
+    ec_datagram_t *datagram; /**< Datagram used in previous step. */
+    unsigned long jiffies_start; /**< FoE timestamp. */
+    uint8_t subindex; /**< Current subindex. */
+    ec_foe_request_t *request; /**< FoE request. */
+    uint8_t toggle; /**< Toggle bit for segment commands. */
+
+    uint8_t *tx_buffer; /**< Buffer with data to transmit. */
+    uint32_t tx_buffer_size; /**< Size of data to transmit. */
+    uint32_t tx_buffer_offset; /**< Offset of data to tranmit next. */
+    uint32_t tx_last_packet; /**< Current packet is last one to send. */
+    uint32_t tx_packet_no; /**< FoE packet number. */
+    uint32_t tx_current_size; /**< Size of current packet to send. */
+    uint8_t *tx_filename; /**< Name of file to transmit. */
+    uint32_t tx_filename_len; /**< Lenth of transmit file name. */
+
+    uint8_t *rx_buffer; /**< Buffer for received data. */
+    uint32_t rx_buffer_size; /**< Size of receive buffer. */
+    uint32_t rx_buffer_offset; /**< Offset in receive buffer. */
+    uint32_t rx_expected_packet_no; /**< Expected receive packet number. */
+    uint32_t rx_last_packet; /**< Current packet is the last to receive. */
+    uint8_t *rx_filename; /**< Name of the file to receive. */
+    uint32_t rx_filename_len; /**< Length of the receive file name. */
+};
+
+/****************************************************************************/
+
+void ec_fsm_foe_init(ec_fsm_foe_t *);
+void ec_fsm_foe_clear(ec_fsm_foe_t *);
+
+int ec_fsm_foe_exec(ec_fsm_foe_t *, ec_datagram_t *);
+int ec_fsm_foe_success(const ec_fsm_foe_t *);
+
+void ec_fsm_foe_transfer(ec_fsm_foe_t *, ec_slave_t *, ec_foe_request_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_master.c b/drivers/net/ethercat/master/fsm_master.c
new file mode 100644
index 000000000000..909c0115f483
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_master.c
@@ -0,0 +1,1472 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT master state machine.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "slave_config.h"
+#ifdef EC_EOE
+#include "ethernet.h"
+#endif
+
+#include "fsm_master.h"
+#include "fsm_foe.h"
+
+/****************************************************************************/
+
+/** Time difference [ns] to tolerate without setting a new system time offset.
+ */
+#define EC_SYSTEM_TIME_TOLERANCE_NS 1000000
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_fsm_master_restart(ec_fsm_master_t *);
+int ec_fsm_master_action_process_sii(ec_fsm_master_t *);
+int ec_fsm_master_action_process_int_request(ec_fsm_master_t *);
+void ec_fsm_master_action_idle(ec_fsm_master_t *);
+void ec_fsm_master_action_next_slave_state(ec_fsm_master_t *);
+void ec_fsm_master_action_configure(ec_fsm_master_t *);
+u64 ec_fsm_master_dc_offset32(ec_fsm_master_t *, u64, u64, unsigned long);
+u64 ec_fsm_master_dc_offset64(ec_fsm_master_t *, u64, u64, unsigned long);
+
+/****************************************************************************/
+
+void ec_fsm_master_state_start(ec_fsm_master_t *);
+void ec_fsm_master_state_broadcast(ec_fsm_master_t *);
+void ec_fsm_master_state_read_state(ec_fsm_master_t *);
+void ec_fsm_master_state_acknowledge(ec_fsm_master_t *);
+void ec_fsm_master_state_configure_slave(ec_fsm_master_t *);
+void ec_fsm_master_state_clear_addresses(ec_fsm_master_t *);
+void ec_fsm_master_state_dc_measure_delays(ec_fsm_master_t *);
+void ec_fsm_master_state_scan_slave(ec_fsm_master_t *);
+void ec_fsm_master_state_dc_read_offset(ec_fsm_master_t *);
+void ec_fsm_master_state_dc_write_offset(ec_fsm_master_t *);
+void ec_fsm_master_state_assign_sii(ec_fsm_master_t *);
+void ec_fsm_master_state_write_sii(ec_fsm_master_t *);
+void ec_fsm_master_state_sdo_dictionary(ec_fsm_master_t *);
+void ec_fsm_master_state_sdo_request(ec_fsm_master_t *);
+void ec_fsm_master_state_soe_request(ec_fsm_master_t *);
+
+void ec_fsm_master_enter_clear_addresses(ec_fsm_master_t *);
+void ec_fsm_master_enter_write_system_times(ec_fsm_master_t *);
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_master_init(
+        ec_fsm_master_t *fsm, /**< Master state machine. */
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_datagram_t *datagram /**< Datagram object to use. */
+        )
+{
+    fsm->master = master;
+    fsm->datagram = datagram;
+
+    // inits the member variables state, idle, dev_idx, link_state,
+    // slaves_responding, slave_states and rescan_required
+    ec_fsm_master_reset(fsm);
+
+    fsm->retries = 0;
+    fsm->scan_jiffies = 0;
+    fsm->slave = NULL;
+    fsm->sii_request = NULL;
+    fsm->sii_index = 0;
+    fsm->sdo_request = NULL;
+    fsm->soe_request = NULL;
+
+    // init sub-state-machines
+    ec_fsm_coe_init(&fsm->fsm_coe);
+    ec_fsm_soe_init(&fsm->fsm_soe);
+    ec_fsm_pdo_init(&fsm->fsm_pdo, &fsm->fsm_coe);
+#ifdef EC_EOE
+    ec_fsm_eoe_init(&fsm->fsm_eoe);
+#endif
+    ec_fsm_change_init(&fsm->fsm_change, fsm->datagram);
+    ec_fsm_slave_config_init(&fsm->fsm_slave_config, fsm->datagram,
+            &fsm->fsm_change, &fsm->fsm_coe, &fsm->fsm_soe, &fsm->fsm_pdo,
+            &fsm->fsm_eoe);
+    ec_fsm_slave_scan_init(&fsm->fsm_slave_scan, fsm->datagram,
+            &fsm->fsm_slave_config, &fsm->fsm_pdo);
+    ec_fsm_sii_init(&fsm->fsm_sii, fsm->datagram);
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_master_clear(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    // clear sub-state machines
+    ec_fsm_coe_clear(&fsm->fsm_coe);
+    ec_fsm_soe_clear(&fsm->fsm_soe);
+    ec_fsm_pdo_clear(&fsm->fsm_pdo);
+#ifdef EC_EOE
+    ec_fsm_eoe_clear(&fsm->fsm_eoe);
+#endif
+    ec_fsm_change_clear(&fsm->fsm_change);
+    ec_fsm_slave_config_clear(&fsm->fsm_slave_config);
+    ec_fsm_slave_scan_clear(&fsm->fsm_slave_scan);
+    ec_fsm_sii_clear(&fsm->fsm_sii);
+}
+
+/****************************************************************************/
+
+/** Reset state machine.
+ */
+void ec_fsm_master_reset(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_device_index_t dev_idx;
+
+    fsm->state = ec_fsm_master_state_start;
+    fsm->idle = 0;
+    fsm->dev_idx = EC_DEVICE_MAIN;
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(fsm->master); dev_idx++) {
+        fsm->link_state[dev_idx] = 0;
+        fsm->slaves_responding[dev_idx] = 0;
+        fsm->slave_states[dev_idx] = EC_SLAVE_STATE_UNKNOWN;
+    }
+
+    fsm->rescan_required = 0;
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * If the state machine's datagram is not sent or received yet, the execution
+ * of the state machine is delayed to the next cycle.
+ *
+ * \return true, if the state machine was executed
+ */
+int ec_fsm_master_exec(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    if (fsm->datagram->state == EC_DATAGRAM_SENT
+        || fsm->datagram->state == EC_DATAGRAM_QUEUED) {
+        // datagram was not sent or received yet.
+        return 0;
+    }
+
+    fsm->state(fsm);
+    return 1;
+}
+
+/****************************************************************************/
+
+/**
+ * \return true, if the state machine is in an idle phase
+ */
+int ec_fsm_master_idle(
+        const ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    return fsm->idle;
+}
+
+/****************************************************************************/
+
+/** Restarts the master state machine.
+ */
+void ec_fsm_master_restart(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    fsm->dev_idx = EC_DEVICE_MAIN;
+    fsm->state = ec_fsm_master_state_start;
+    fsm->state(fsm); // execute immediately
+}
+
+/*****************************************************************************
+ * Master state machine
+ ****************************************************************************/
+
+/** Master state: START.
+ *
+ * Starts with getting slave count and slave states.
+ */
+void ec_fsm_master_state_start(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+
+    fsm->idle = 1;
+
+    // check for emergency requests
+    if (!list_empty(&master->emerg_reg_requests)) {
+        ec_reg_request_t *request;
+
+        // get first request
+        request = list_entry(master->emerg_reg_requests.next,
+                ec_reg_request_t, list);
+        list_del_init(&request->list); // dequeue
+        request->state = EC_INT_REQUEST_BUSY;
+
+        if (request->transfer_size > fsm->datagram->mem_size) {
+            EC_MASTER_ERR(master, "Emergency request data too large!\n");
+            request->state = EC_INT_REQUEST_FAILURE;
+            wake_up_all(&master->request_queue);
+            fsm->state(fsm); // continue
+            return;
+        }
+
+        if (request->dir != EC_DIR_OUTPUT) {
+            EC_MASTER_ERR(master, "Emergency requests must be"
+                    " write requests!\n");
+            request->state = EC_INT_REQUEST_FAILURE;
+            wake_up_all(&master->request_queue);
+            fsm->state(fsm); // continue
+            return;
+        }
+
+        EC_MASTER_DBG(master, 1, "Writing emergency register request...\n");
+        ec_datagram_apwr(fsm->datagram, request->ring_position,
+                request->address, request->transfer_size);
+        memcpy(fsm->datagram->data, request->data, request->transfer_size);
+        fsm->datagram->device_index = EC_DEVICE_MAIN;
+        request->state = EC_INT_REQUEST_SUCCESS;
+        wake_up_all(&master->request_queue);
+        return;
+    }
+
+    ec_datagram_brd(fsm->datagram, 0x0130, 2);
+    ec_datagram_zero(fsm->datagram);
+    fsm->datagram->device_index = fsm->dev_idx;
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_master_state_broadcast;
+}
+
+/****************************************************************************/
+
+/** Master state: BROADCAST.
+ *
+ * Processes the broadcast read slave count and slaves states.
+ */
+void ec_fsm_master_state_broadcast(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    unsigned int i, size;
+    ec_slave_t *slave;
+    ec_master_t *master = fsm->master;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        return;
+    }
+
+    // bus topology change?
+    if (datagram->working_counter != fsm->slaves_responding[fsm->dev_idx]) {
+        fsm->rescan_required = 1;
+        fsm->slaves_responding[fsm->dev_idx] = datagram->working_counter;
+        EC_MASTER_INFO(master, "%u slave(s) responding on %s device. "
+                "Re-scanning on next possibility.\n",
+                fsm->slaves_responding[fsm->dev_idx],
+                ec_device_names[fsm->dev_idx != 0]);
+    }
+
+    if (fsm->link_state[fsm->dev_idx] &&
+            !master->devices[fsm->dev_idx].link_state) {
+        ec_device_index_t dev_idx;
+
+        EC_MASTER_DBG(master, 1, "Master state machine detected "
+                "link down on %s device. Clearing slave list.\n",
+                ec_device_names[fsm->dev_idx != 0]);
+
+#ifdef EC_EOE
+        ec_master_eoe_stop(master);
+        ec_master_clear_eoe_handlers(master);
+#endif
+        ec_master_clear_slaves(master);
+
+        for (dev_idx = EC_DEVICE_MAIN;
+                dev_idx < ec_master_num_devices(master); dev_idx++) {
+            fsm->slave_states[dev_idx] = 0x00;
+            fsm->slaves_responding[dev_idx] = 0; /* Reset to trigger rescan on
+                                                    next link up. */
+        }
+    }
+    fsm->link_state[fsm->dev_idx] = master->devices[fsm->dev_idx].link_state;
+
+    if (datagram->state == EC_DATAGRAM_RECEIVED &&
+            fsm->slaves_responding[fsm->dev_idx]) {
+        uint8_t states = EC_READ_U8(datagram->data);
+        if (states != fsm->slave_states[fsm->dev_idx]) {
+            // slave states changed
+            char state_str[EC_STATE_STRING_SIZE];
+            fsm->slave_states[fsm->dev_idx] = states;
+            ec_state_string(states, state_str, 1);
+            EC_MASTER_INFO(master, "Slave states on %s device: %s.\n",
+                    ec_device_names[fsm->dev_idx != 0], state_str);
+        }
+    } else {
+        fsm->slave_states[fsm->dev_idx] = 0x00;
+    }
+
+    fsm->dev_idx++;
+    if (fsm->dev_idx < ec_master_num_devices(master)) {
+        // check number of responding slaves on next device
+        fsm->state = ec_fsm_master_state_start;
+        fsm->state(fsm); // execute immediately
+        return;
+    }
+
+    if (fsm->rescan_required) {
+        down(&master->scan_sem);
+        if (!master->allow_scan) {
+            up(&master->scan_sem);
+        } else {
+            unsigned int count = 0, next_dev_slave, ring_position;
+            ec_device_index_t dev_idx;
+
+            master->scan_busy = 1;
+            master->scan_index = 0;
+            up(&master->scan_sem);
+
+            EC_MASTER_INFO(master, "Re-scanning now.\n");
+
+            // clear all slaves and scan the bus
+            fsm->rescan_required = 0;
+            fsm->idle = 0;
+            fsm->scan_jiffies = jiffies;
+
+#ifdef EC_EOE
+            ec_master_eoe_stop(master);
+            ec_master_clear_eoe_handlers(master);
+#endif
+            ec_master_clear_slaves(master);
+
+            for (dev_idx = EC_DEVICE_MAIN;
+                    dev_idx < ec_master_num_devices(master); dev_idx++) {
+                count += fsm->slaves_responding[dev_idx];
+            }
+
+            if (!count) {
+                // no slaves present -> finish state machine.
+                master->scan_busy = 0;
+                wake_up_interruptible(&master->scan_queue);
+                ec_fsm_master_restart(fsm);
+                return;
+            }
+
+            size = sizeof(ec_slave_t) * count;
+            if (!(master->slaves =
+                        (ec_slave_t *) kmalloc(size, GFP_KERNEL))) {
+                EC_MASTER_ERR(master, "Failed to allocate %u bytes"
+                        " of slave memory!\n", size);
+                master->scan_busy = 0;
+                wake_up_interruptible(&master->scan_queue);
+                ec_fsm_master_restart(fsm);
+                return;
+            }
+
+            // init slaves
+            dev_idx = EC_DEVICE_MAIN;
+            next_dev_slave = fsm->slaves_responding[dev_idx];
+            ring_position = 0;
+            for (i = 0; i < count; i++, ring_position++) {
+                slave = master->slaves + i;
+                while (i >= next_dev_slave) {
+                    dev_idx++;
+                    next_dev_slave += fsm->slaves_responding[dev_idx];
+                    ring_position = 0;
+                }
+
+                ec_slave_init(slave, master, dev_idx, ring_position, i + 1);
+
+                // do not force reconfiguration in operation phase to avoid
+                // unnecesssary process data interruptions
+                if (master->phase != EC_OPERATION) {
+                    slave->force_config = 1;
+                }
+            }
+            master->slave_count = count;
+            master->fsm_slave = master->slaves;
+
+            /* start with first device with slaves responding; at least one
+             * has responding slaves, otherwise count would be zero. */
+            fsm->dev_idx = EC_DEVICE_MAIN;
+            while (!fsm->slaves_responding[fsm->dev_idx]) {
+                fsm->dev_idx++;
+            }
+
+            ec_fsm_master_enter_clear_addresses(fsm);
+            return;
+        }
+    }
+
+    if (master->slave_count) {
+
+        // application applied configurations
+        if (master->config_changed) {
+            master->config_changed = 0;
+
+            EC_MASTER_DBG(master, 1, "Configuration changed.\n");
+
+            fsm->slave = master->slaves; // begin with first slave
+            ec_fsm_master_enter_write_system_times(fsm);
+
+        } else {
+            // fetch state from first slave
+            fsm->slave = master->slaves;
+            ec_datagram_fprd(fsm->datagram, fsm->slave->station_address,
+                    0x0130, 2);
+            ec_datagram_zero(datagram);
+            fsm->datagram->device_index = fsm->slave->device_index;
+            fsm->retries = EC_FSM_RETRIES;
+            fsm->state = ec_fsm_master_state_read_state;
+        }
+    } else {
+        ec_fsm_master_restart(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** Check for pending SII write requests and process one.
+ *
+ * \return non-zero, if an SII write request is processed.
+ */
+int ec_fsm_master_action_process_sii(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+    ec_sii_write_request_t *request;
+    ec_slave_config_t *config;
+    ec_flag_t *flag;
+    int assign_to_pdi;
+
+    // search the first request to be processed
+    while (1) {
+        if (list_empty(&master->sii_requests))
+            break;
+
+        // get first request
+        request = list_entry(master->sii_requests.next,
+                ec_sii_write_request_t, list);
+        list_del_init(&request->list); // dequeue
+        request->state = EC_INT_REQUEST_BUSY;
+
+        assign_to_pdi = 0;
+        config = request->slave->config;
+        if (config) {
+            flag = ec_slave_config_find_flag(config, "AssignToPdi");
+            if (flag) {
+                assign_to_pdi = flag->value;
+            }
+        }
+
+        if (assign_to_pdi) {
+            fsm->sii_request = request;
+            EC_SLAVE_DBG(request->slave, 1,
+                    "Assigning SII back to EtherCAT.\n");
+            ec_datagram_fpwr(fsm->datagram, request->slave->station_address,
+                    0x0500, 0x01);
+            EC_WRITE_U8(fsm->datagram->data, 0x00); // EtherCAT
+            fsm->retries = EC_FSM_RETRIES;
+            fsm->state = ec_fsm_master_state_assign_sii;
+            return 1;
+        }
+
+        // found pending SII write operation. execute it!
+        EC_SLAVE_DBG(request->slave, 1, "Writing SII data...\n");
+        fsm->sii_request = request;
+        fsm->sii_index = 0;
+        ec_fsm_sii_write(&fsm->fsm_sii, request->slave, request->offset,
+                request->words, EC_FSM_SII_USE_CONFIGURED_ADDRESS);
+        fsm->state = ec_fsm_master_state_write_sii;
+        fsm->state(fsm); // execute immediately
+        return 1;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Check for pending internal SDO/SoE requests and process one.
+ *
+ * \return non-zero, if an SDO request is processed.
+ */
+int ec_fsm_master_action_process_int_request(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+    ec_slave_t *slave;
+    ec_sdo_request_t *sdo_req;
+    ec_soe_request_t *soe_req;
+
+    // search for internal requests to be processed
+    for (slave = master->slaves;
+            slave < master->slaves + master->slave_count;
+            slave++) {
+
+        if (!slave->config) {
+            continue;
+        }
+
+        list_for_each_entry(sdo_req, &slave->config->sdo_requests, list) {
+            if (sdo_req->state == EC_INT_REQUEST_QUEUED) {
+
+                if (ec_sdo_request_timed_out(sdo_req)) {
+                    sdo_req->state = EC_INT_REQUEST_FAILURE;
+                    EC_SLAVE_DBG(slave, 1, "Internal SDO request"
+                            " timed out.\n");
+                    continue;
+                }
+
+                if (slave->current_state == EC_SLAVE_STATE_INIT) {
+                    sdo_req->state = EC_INT_REQUEST_FAILURE;
+                    continue;
+                }
+
+                sdo_req->state = EC_INT_REQUEST_BUSY;
+                EC_SLAVE_DBG(slave, 1, "Processing internal"
+                        " SDO request...\n");
+                fsm->idle = 0;
+                fsm->sdo_request = sdo_req;
+                fsm->slave = slave;
+                fsm->state = ec_fsm_master_state_sdo_request;
+                ec_fsm_coe_transfer(&fsm->fsm_coe, slave, sdo_req);
+                ec_fsm_coe_exec(&fsm->fsm_coe, fsm->datagram);
+                return 1;
+            }
+        }
+
+        list_for_each_entry(soe_req, &slave->config->soe_requests, list) {
+            if (soe_req->state == EC_INT_REQUEST_QUEUED) {
+
+                if (ec_soe_request_timed_out(soe_req)) {
+                    soe_req->state = EC_INT_REQUEST_FAILURE;
+                    EC_SLAVE_DBG(slave, 1, "Internal SoE request"
+                            " timed out.\n");
+                    continue;
+                }
+
+                if (slave->current_state == EC_SLAVE_STATE_INIT) {
+                    soe_req->state = EC_INT_REQUEST_FAILURE;
+                    continue;
+                }
+
+                soe_req->state = EC_INT_REQUEST_BUSY;
+                EC_SLAVE_DBG(slave, 1, "Processing internal"
+                        " SoE request...\n");
+                fsm->idle = 0;
+                fsm->soe_request = soe_req;
+                fsm->slave = slave;
+                fsm->state = ec_fsm_master_state_soe_request;
+                ec_fsm_soe_transfer(&fsm->fsm_soe, slave, soe_req);
+                ec_fsm_soe_exec(&fsm->fsm_soe, fsm->datagram);
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Master action: IDLE.
+ *
+ * Does secondary work.
+ */
+void ec_fsm_master_action_idle(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+    ec_slave_t *slave;
+
+    // Check for pending internal SDO or SoE requests
+    if (ec_fsm_master_action_process_int_request(fsm)) {
+        return;
+    }
+
+    // enable processing of requests
+    for (slave = master->slaves;
+            slave < master->slaves + master->slave_count;
+            slave++) {
+        ec_fsm_slave_set_ready(&slave->fsm);
+    }
+
+    // check, if slaves have an SDO dictionary to read out.
+    for (slave = master->slaves;
+            slave < master->slaves + master->slave_count;
+            slave++) {
+        if (!(slave->sii.mailbox_protocols & EC_MBOX_COE)
+                || (slave->sii.has_general
+                    && !slave->sii.coe_details.enable_sdo_info)
+                || slave->sdo_dictionary_fetched
+                || slave->current_state == EC_SLAVE_STATE_INIT
+                || slave->current_state == EC_SLAVE_STATE_UNKNOWN
+                || jiffies - slave->jiffies_preop < EC_WAIT_SDO_DICT * HZ
+                ) continue;
+
+        EC_SLAVE_DBG(slave, 1, "Fetching SDO dictionary.\n");
+
+        slave->sdo_dictionary_fetched = 1;
+
+        // start fetching SDO dictionary
+        fsm->idle = 0;
+        fsm->slave = slave;
+        fsm->state = ec_fsm_master_state_sdo_dictionary;
+        ec_fsm_coe_dictionary(&fsm->fsm_coe, slave);
+        ec_fsm_coe_exec(&fsm->fsm_coe, fsm->datagram); // execute immediately
+        fsm->datagram->device_index = fsm->slave->device_index;
+        return;
+    }
+
+    // check for pending SII write operations.
+    if (ec_fsm_master_action_process_sii(fsm)) {
+        return; // SII write request found
+    }
+
+    ec_fsm_master_restart(fsm);
+}
+
+/****************************************************************************/
+
+/** Master action: Get state of next slave.
+ */
+void ec_fsm_master_action_next_slave_state(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+
+    // is there another slave to query?
+    fsm->slave++;
+    if (fsm->slave < master->slaves + master->slave_count) {
+        // fetch state from next slave
+        fsm->idle = 1;
+        ec_datagram_fprd(fsm->datagram,
+                fsm->slave->station_address, 0x0130, 2);
+        ec_datagram_zero(fsm->datagram);
+        fsm->datagram->device_index = fsm->slave->device_index;
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_master_state_read_state;
+        return;
+    }
+
+    // all slaves processed
+    ec_fsm_master_action_idle(fsm);
+}
+
+/****************************************************************************/
+
+/** Master action: Configure.
+ */
+void ec_fsm_master_action_configure(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+    ec_slave_t *slave = fsm->slave;
+
+    if (master->config_changed) {
+        master->config_changed = 0;
+
+        // abort iterating through slaves,
+        // first compensate DC system time offsets,
+        // then begin configuring at slave 0
+        EC_MASTER_DBG(master, 1, "Configuration changed"
+                " (aborting state check).\n");
+
+        fsm->slave = master->slaves; // begin with first slave
+        ec_fsm_master_enter_write_system_times(fsm);
+        return;
+    }
+
+    // Does the slave have to be configured?
+    if ((slave->current_state != slave->requested_state
+                || slave->force_config) && !slave->error_flag) {
+
+        // Start slave configuration
+        down(&master->config_sem);
+        master->config_busy = 1;
+        up(&master->config_sem);
+
+        if (master->debug_level) {
+            char old_state[EC_STATE_STRING_SIZE],
+                 new_state[EC_STATE_STRING_SIZE];
+            ec_state_string(slave->current_state, old_state, 0);
+            ec_state_string(slave->requested_state, new_state, 0);
+            EC_SLAVE_DBG(slave, 1, "Changing state from %s to %s%s.\n",
+                    old_state, new_state,
+                    slave->force_config ? " (forced)" : "");
+        }
+
+        fsm->idle = 0;
+        fsm->state = ec_fsm_master_state_configure_slave;
+        ec_fsm_slave_config_start(&fsm->fsm_slave_config, slave);
+        fsm->state(fsm); // execute immediately
+        fsm->datagram->device_index = fsm->slave->device_index;
+        return;
+    }
+
+    // process next slave
+    ec_fsm_master_action_next_slave_state(fsm);
+}
+
+/****************************************************************************/
+
+/** Master state: READ STATE.
+ *
+ * Fetches the AL state of a slave.
+ */
+void ec_fsm_master_state_read_state(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        return;
+    }
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_ERR(slave, "Failed to receive AL state datagram: ");
+        ec_datagram_print_state(datagram);
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    // did the slave not respond to its station address?
+    if (datagram->working_counter != 1) {
+        if (!slave->error_flag) {
+            slave->error_flag = 1;
+            EC_SLAVE_DBG(slave, 1, "Slave did not respond to state query.\n");
+        }
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    // A single slave responded
+    ec_slave_set_state(slave, EC_READ_U8(datagram->data));
+
+    if (!slave->error_flag) {
+        // Check, if new slave state has to be acknowledged
+        if (slave->current_state & EC_SLAVE_STATE_ACK_ERR) {
+            fsm->idle = 0;
+            fsm->state = ec_fsm_master_state_acknowledge;
+            ec_fsm_change_ack(&fsm->fsm_change, slave);
+            fsm->state(fsm); // execute immediately
+            return;
+        }
+
+        // No acknowlegde necessary; check for configuration
+        ec_fsm_master_action_configure(fsm);
+        return;
+    }
+
+    // slave has error flag set; process next one
+    ec_fsm_master_action_next_slave_state(fsm);
+}
+
+/****************************************************************************/
+
+/** Master state: ACKNOWLEDGE.
+ */
+void ec_fsm_master_state_acknowledge(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (ec_fsm_change_exec(&fsm->fsm_change)) {
+        return;
+    }
+
+    if (!ec_fsm_change_success(&fsm->fsm_change)) {
+        fsm->slave->error_flag = 1;
+        EC_SLAVE_ERR(slave, "Failed to acknowledge state change.\n");
+    }
+
+    ec_fsm_master_action_configure(fsm);
+}
+
+/****************************************************************************/
+
+/** Start clearing slave addresses.
+ */
+void ec_fsm_master_enter_clear_addresses(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    // broadcast clear all station addresses
+    ec_datagram_bwr(fsm->datagram, 0x0010, 2);
+    EC_WRITE_U16(fsm->datagram->data, 0x0000);
+    fsm->datagram->device_index = fsm->dev_idx;
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_master_state_clear_addresses;
+}
+
+/****************************************************************************/
+
+/** Master state: CLEAR ADDRESSES.
+ */
+void ec_fsm_master_state_clear_addresses(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        return;
+    }
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_MASTER_ERR(master, "Failed to receive address"
+                " clearing datagram on %s link: ",
+                ec_device_names[fsm->dev_idx != 0]);
+        ec_datagram_print_state(datagram);
+        master->scan_busy = 0;
+        master->scan_index = master->slave_count;
+        wake_up_interruptible(&master->scan_queue);
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    if (datagram->working_counter != fsm->slaves_responding[fsm->dev_idx]) {
+        EC_MASTER_WARN(master, "Failed to clear station addresses on %s link:"
+                " Cleared %u of %u",
+                ec_device_names[fsm->dev_idx != 0], datagram->working_counter,
+                fsm->slaves_responding[fsm->dev_idx]);
+    }
+
+    EC_MASTER_DBG(master, 1, "Sending broadcast-write"
+            " to measure transmission delays on %s link.\n",
+            ec_device_names[fsm->dev_idx != 0]);
+
+    ec_datagram_bwr(datagram, 0x0900, 1);
+    ec_datagram_zero(datagram);
+    fsm->datagram->device_index = fsm->dev_idx;
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_master_state_dc_measure_delays;
+}
+
+/****************************************************************************/
+
+/** Master state: DC MEASURE DELAYS.
+ */
+void ec_fsm_master_state_dc_measure_delays(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        return;
+    }
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_MASTER_ERR(master, "Failed to receive delay measuring datagram"
+                " on %s link: ", ec_device_names[fsm->dev_idx != 0]);
+        ec_datagram_print_state(datagram);
+        master->scan_busy = 0;
+        master->scan_index = master->slave_count;
+        wake_up_interruptible(&master->scan_queue);
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    EC_MASTER_DBG(master, 1, "%u slaves responded to delay measuring"
+            " on %s link.\n",
+            datagram->working_counter, ec_device_names[fsm->dev_idx != 0]);
+
+    do {
+        fsm->dev_idx++;
+    } while (fsm->dev_idx < ec_master_num_devices(master) &&
+            !fsm->slaves_responding[fsm->dev_idx]);
+    if (fsm->dev_idx < ec_master_num_devices(master)) {
+        ec_fsm_master_enter_clear_addresses(fsm);
+        return;
+    }
+
+    EC_MASTER_INFO(master, "Scanning bus.\n");
+
+    // begin scanning of slaves
+    fsm->slave = master->slaves;
+    master->scan_index = 0;
+    EC_MASTER_DBG(master, 1, "Scanning slave %u on %s link.\n",
+            fsm->slave->ring_position,
+            ec_device_names[fsm->slave->device_index != 0]);
+    fsm->state = ec_fsm_master_state_scan_slave;
+    ec_fsm_slave_scan_start(&fsm->fsm_slave_scan, fsm->slave);
+    ec_fsm_slave_scan_exec(&fsm->fsm_slave_scan); // execute immediately
+    fsm->datagram->device_index = fsm->slave->device_index;
+}
+
+/****************************************************************************/
+
+/** Master state: SCAN SLAVE.
+ *
+ * Executes the sub-statemachine for the scanning of a slave.
+ */
+void ec_fsm_master_state_scan_slave(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+#ifdef EC_EOE
+    ec_slave_t *slave = fsm->slave;
+#endif
+
+    if (ec_fsm_slave_scan_exec(&fsm->fsm_slave_scan)) {
+        return;
+    }
+
+#ifdef EC_EOE
+    if (slave->sii.mailbox_protocols & EC_MBOX_EOE) {
+        // create EoE handler for this slave
+        ec_eoe_t *eoe;
+        if (!(eoe = kmalloc(sizeof(ec_eoe_t), GFP_KERNEL))) {
+            EC_SLAVE_ERR(slave, "Failed to allocate EoE handler memory!\n");
+        } else if (ec_eoe_init(eoe, slave)) {
+            EC_SLAVE_ERR(slave, "Failed to init EoE handler!\n");
+            kfree(eoe);
+        } else {
+            list_add_tail(&eoe->list, &master->eoe_handlers);
+        }
+    }
+#endif
+
+    // another slave to fetch?
+    fsm->slave++;
+    master->scan_index++;
+    if (fsm->slave < master->slaves + master->slave_count) {
+        EC_MASTER_DBG(master, 1, "Scanning slave %u on %s link.\n",
+                fsm->slave->ring_position,
+                ec_device_names[fsm->slave->device_index != 0]);
+        ec_fsm_slave_scan_start(&fsm->fsm_slave_scan, fsm->slave);
+        ec_fsm_slave_scan_exec(&fsm->fsm_slave_scan); // execute immediately
+        fsm->datagram->device_index = fsm->slave->device_index;
+        return;
+    }
+
+    EC_MASTER_INFO(master, "Bus scanning completed in %lu ms.\n",
+            (jiffies - fsm->scan_jiffies) * 1000 / HZ);
+
+    master->scan_busy = 0;
+    master->scan_index = master->slave_count;
+    wake_up_interruptible(&master->scan_queue);
+
+    ec_master_calc_dc(master);
+
+    // Attach slave configurations
+    ec_master_attach_slave_configs(master);
+
+#ifdef EC_EOE
+    // check if EoE processing has to be started
+    ec_master_eoe_start(master);
+#endif
+
+    if (master->slave_count) {
+        master->config_changed = 0;
+
+        fsm->slave = master->slaves; // begin with first slave
+        ec_fsm_master_enter_write_system_times(fsm);
+    } else {
+        ec_fsm_master_restart(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** Master state: CONFIGURE SLAVE.
+ *
+ * Starts configuring a slave.
+ */
+void ec_fsm_master_state_configure_slave(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+
+    if (ec_fsm_slave_config_exec(&fsm->fsm_slave_config)) {
+        return;
+    }
+
+    fsm->slave->force_config = 0;
+
+    // configuration finished
+    master->config_busy = 0;
+    wake_up_interruptible(&master->config_queue);
+
+    if (!ec_fsm_slave_config_success(&fsm->fsm_slave_config)) {
+        // TODO: mark slave_config as failed.
+    }
+
+    fsm->idle = 1;
+    ec_fsm_master_action_next_slave_state(fsm);
+}
+
+/****************************************************************************/
+
+/** Start writing DC system times.
+ */
+void ec_fsm_master_enter_write_system_times(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+
+    if (master->dc_ref_time) {
+
+        while (fsm->slave < master->slaves + master->slave_count) {
+            if (!fsm->slave->base_dc_supported
+                    || !fsm->slave->has_dc_system_time) {
+                fsm->slave++;
+                continue;
+            }
+
+            EC_SLAVE_DBG(fsm->slave, 1, "Checking system time offset.\n");
+
+            // read DC system time (0x0910, 64 bit)
+            //                         gap (64 bit)
+            //     and time offset (0x0920, 64 bit)
+            ec_datagram_fprd(fsm->datagram, fsm->slave->station_address,
+                    0x0910, 24);
+            fsm->datagram->device_index = fsm->slave->device_index;
+            fsm->retries = EC_FSM_RETRIES;
+            fsm->state = ec_fsm_master_state_dc_read_offset;
+            return;
+        }
+
+    } else {
+        if (master->active) {
+            EC_MASTER_WARN(master, "No application time received up to now,"
+                    " but master already active.\n");
+        } else {
+            EC_MASTER_DBG(master, 1, "No app_time received up to now.\n");
+        }
+    }
+
+    // scanning and setting system times complete
+    ec_master_request_op(master);
+    EC_MASTER_DBG(master, 1, "After requesting OP, rescan_required is %u.\n",
+            fsm->rescan_required);
+    ec_fsm_master_restart(fsm);
+}
+
+/****************************************************************************/
+
+/** Configure 32 bit time offset.
+ *
+ * \return New offset.
+ */
+u64 ec_fsm_master_dc_offset32(
+        ec_fsm_master_t *fsm, /**< Master state machine. */
+        u64 system_time, /**< System time register. */
+        u64 old_offset, /**< Time offset register. */
+        unsigned long jiffies_since_read /**< Jiffies for correction. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    u32 correction, system_time32, old_offset32, new_offset;
+    s32 time_diff;
+
+    system_time32 = (u32) system_time;
+    old_offset32 = (u32) old_offset;
+
+    // correct read system time by elapsed time since read operation
+    correction = jiffies_since_read * 1000 / HZ * 1000000;
+    system_time32 += correction;
+    time_diff = (u32) slave->master->app_time - system_time32;
+
+    EC_SLAVE_DBG(slave, 1, "DC 32 bit system time offset calculation:"
+            " system_time=%u (corrected with %u),"
+            " app_time=%llu, diff=%i\n",
+            system_time32, correction,
+            slave->master->app_time, time_diff);
+
+    if (EC_ABS(time_diff) > EC_SYSTEM_TIME_TOLERANCE_NS) {
+        new_offset = time_diff + old_offset32;
+        EC_SLAVE_DBG(slave, 1, "Setting time offset to %u (was %u)\n",
+                new_offset, old_offset32);
+        return (u64) new_offset;
+    } else {
+        EC_SLAVE_DBG(slave, 1, "Not touching time offset.\n");
+        return old_offset;
+    }
+}
+
+/****************************************************************************/
+
+/** Configure 64 bit time offset.
+ *
+ * \return New offset.
+ */
+u64 ec_fsm_master_dc_offset64(
+        ec_fsm_master_t *fsm, /**< Master state machine. */
+        u64 system_time, /**< System time register. */
+        u64 old_offset, /**< Time offset register. */
+        unsigned long jiffies_since_read /**< Jiffies for correction. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    u64 new_offset, correction;
+    s64 time_diff;
+
+    // correct read system time by elapsed time since read operation
+    correction = (u64) (jiffies_since_read * 1000 / HZ) * 1000000;
+    system_time += correction;
+    time_diff = fsm->slave->master->app_time - system_time;
+
+    EC_SLAVE_DBG(slave, 1, "DC 64 bit system time offset calculation:"
+            " system_time=%llu (corrected with %llu),"
+            " app_time=%llu, diff=%lli\n",
+            system_time, correction,
+            slave->master->app_time, time_diff);
+
+    if (EC_ABS(time_diff) > EC_SYSTEM_TIME_TOLERANCE_NS) {
+        new_offset = time_diff + old_offset;
+        EC_SLAVE_DBG(slave, 1, "Setting time offset to %llu (was %llu)\n",
+                new_offset, old_offset);
+    } else {
+        new_offset = old_offset;
+        EC_SLAVE_DBG(slave, 1, "Not touching time offset.\n");
+    }
+
+    return new_offset;
+}
+
+/****************************************************************************/
+
+/** Master state: DC READ OFFSET.
+ */
+void ec_fsm_master_state_dc_read_offset(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    u64 system_time, old_offset, new_offset;
+    unsigned long jiffies_since_read;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_ERR(slave, "Failed to receive DC times datagram: ");
+        ec_datagram_print_state(datagram);
+        fsm->slave++;
+        ec_fsm_master_enter_write_system_times(fsm);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_WARN(slave, "Failed to get DC times: ");
+        ec_datagram_print_wc_error(datagram);
+        fsm->slave++;
+        ec_fsm_master_enter_write_system_times(fsm);
+        return;
+    }
+
+    system_time = EC_READ_U64(datagram->data);     // 0x0910
+    old_offset = EC_READ_U64(datagram->data + 16); // 0x0920
+    jiffies_since_read = jiffies - datagram->jiffies_sent;
+
+    if (slave->base_dc_range == EC_DC_32) {
+        new_offset = ec_fsm_master_dc_offset32(fsm,
+                system_time, old_offset, jiffies_since_read);
+    } else {
+        new_offset = ec_fsm_master_dc_offset64(fsm,
+                system_time, old_offset, jiffies_since_read);
+    }
+
+    // set DC system time offset and transmission delay
+    ec_datagram_fpwr(datagram, slave->station_address, 0x0920, 12);
+    EC_WRITE_U64(datagram->data, new_offset);
+    EC_WRITE_U32(datagram->data + 8, slave->transmission_delay);
+    fsm->datagram->device_index = slave->device_index;
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_master_state_dc_write_offset;
+}
+
+/****************************************************************************/
+
+/** Master state: DC WRITE OFFSET.
+ */
+void ec_fsm_master_state_dc_write_offset(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_ERR(slave,
+                "Failed to receive DC system time offset datagram: ");
+        ec_datagram_print_state(datagram);
+        fsm->slave++;
+        ec_fsm_master_enter_write_system_times(fsm);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_ERR(slave, "Failed to set DC system time offset: ");
+        ec_datagram_print_wc_error(datagram);
+        fsm->slave++;
+        ec_fsm_master_enter_write_system_times(fsm);
+        return;
+    }
+
+    fsm->slave++;
+    ec_fsm_master_enter_write_system_times(fsm);
+}
+
+/****************************************************************************/
+
+/** Master state: ASSIGN SII.
+ */
+void ec_fsm_master_state_assign_sii(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_sii_write_request_t *request = fsm->sii_request;
+    ec_slave_t *slave = request->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_ERR(slave, "Failed to receive SII assignment datagram: ");
+        ec_datagram_print_state(datagram);
+        goto cont;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_ERR(slave, "Failed to assign SII back to EtherCAT: ");
+        ec_datagram_print_wc_error(datagram);
+        goto cont;
+    }
+
+cont:
+    // found pending SII write operation. execute it!
+    EC_SLAVE_DBG(slave, 1, "Writing SII data (after assignment)...\n");
+    fsm->sii_index = 0;
+    ec_fsm_sii_write(&fsm->fsm_sii, slave, request->offset,
+            request->words, EC_FSM_SII_USE_CONFIGURED_ADDRESS);
+    fsm->state = ec_fsm_master_state_write_sii;
+    fsm->state(fsm); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Master state: WRITE SII.
+ */
+void ec_fsm_master_state_write_sii(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_master_t *master = fsm->master;
+    ec_sii_write_request_t *request = fsm->sii_request;
+    ec_slave_t *slave = request->slave;
+
+    if (ec_fsm_sii_exec(&fsm->fsm_sii)) return;
+
+    if (!ec_fsm_sii_success(&fsm->fsm_sii)) {
+        EC_SLAVE_ERR(slave, "Failed to write SII data.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&master->request_queue);
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    fsm->sii_index++;
+    if (fsm->sii_index < request->nwords) {
+        ec_fsm_sii_write(&fsm->fsm_sii, slave,
+                request->offset + fsm->sii_index,
+                request->words + fsm->sii_index,
+                EC_FSM_SII_USE_CONFIGURED_ADDRESS);
+        ec_fsm_sii_exec(&fsm->fsm_sii); // execute immediately
+        return;
+    }
+
+    // finished writing SII
+    EC_SLAVE_DBG(slave, 1, "Finished writing %zu words of SII data.\n",
+            request->nwords);
+
+    if (request->offset <= 4 && request->offset + request->nwords > 4) {
+        // alias was written
+        slave->sii.alias = EC_READ_U16(request->words + 4);
+        // TODO: read alias from register 0x0012
+        slave->effective_alias = slave->sii.alias;
+    }
+    // TODO: Evaluate other SII contents!
+
+    request->state = EC_INT_REQUEST_SUCCESS;
+    wake_up_all(&master->request_queue);
+
+    // check for another SII write request
+    if (ec_fsm_master_action_process_sii(fsm))
+        return; // processing another request
+
+    ec_fsm_master_restart(fsm);
+}
+
+/****************************************************************************/
+
+/** Master state: SDO DICTIONARY.
+ */
+void ec_fsm_master_state_sdo_dictionary(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = fsm->master;
+
+    if (ec_fsm_coe_exec(&fsm->fsm_coe, fsm->datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(&fsm->fsm_coe)) {
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    // SDO dictionary fetching finished
+
+    if (master->debug_level) {
+        unsigned int sdo_count, entry_count;
+        ec_slave_sdo_dict_info(slave, &sdo_count, &entry_count);
+        EC_SLAVE_DBG(slave, 1, "Fetched %u SDOs and %u entries.\n",
+               sdo_count, entry_count);
+    }
+
+    // attach pdo names from dictionary
+    ec_slave_attach_pdo_names(slave);
+
+    ec_fsm_master_restart(fsm);
+}
+
+/****************************************************************************/
+
+/** Master state: SDO REQUEST.
+ */
+void ec_fsm_master_state_sdo_request(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_sdo_request_t *request = fsm->sdo_request;
+
+    if (!request) {
+        // configuration was cleared in the meantime
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    if (ec_fsm_coe_exec(&fsm->fsm_coe, fsm->datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(&fsm->fsm_coe)) {
+        EC_SLAVE_DBG(fsm->slave, 1,
+                "Failed to process internal SDO request.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&fsm->master->request_queue);
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    // SDO request finished
+    request->state = EC_INT_REQUEST_SUCCESS;
+    wake_up_all(&fsm->master->request_queue);
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Finished internal SDO request.\n");
+
+    // check for another SDO/SoE request
+    if (ec_fsm_master_action_process_int_request(fsm)) {
+        return; // processing another request
+    }
+
+    ec_fsm_master_restart(fsm);
+}
+
+/****************************************************************************/
+
+/** Master state: SoE REQUEST.
+ */
+void ec_fsm_master_state_soe_request(
+        ec_fsm_master_t *fsm /**< Master state machine. */
+        )
+{
+    ec_soe_request_t *request = fsm->soe_request;
+
+    if (!request) {
+        // configuration was cleared in the meantime
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    if (ec_fsm_soe_exec(&fsm->fsm_soe, fsm->datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_soe_success(&fsm->fsm_soe)) {
+        EC_SLAVE_DBG(fsm->slave, 1,
+                "Failed to process internal SoE request.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&fsm->master->request_queue);
+        ec_fsm_master_restart(fsm);
+        return;
+    }
+
+    // SoE request finished
+    request->state = EC_INT_REQUEST_SUCCESS;
+    wake_up_all(&fsm->master->request_queue);
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Finished internal SoE request.\n");
+
+    // check for another CoE/SoE request
+    if (ec_fsm_master_action_process_int_request(fsm)) {
+        return; // processing another request
+    }
+
+    ec_fsm_master_restart(fsm);
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_master.h b/drivers/net/ethercat/master/fsm_master.h
new file mode 100644
index 000000000000..d645786651ee
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_master.h
@@ -0,0 +1,107 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT master state machine.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_MASTER_H__
+#define __EC_FSM_MASTER_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "foe_request.h"
+#include "sdo_request.h"
+#include "soe_request.h"
+#include "fsm_slave_config.h"
+#include "fsm_slave_scan.h"
+#include "fsm_pdo.h"
+
+/****************************************************************************/
+
+/** SII write request.
+ */
+typedef struct {
+    struct list_head list; /**< List head. */
+    ec_slave_t *slave; /**< EtherCAT slave. */
+    uint16_t offset; /**< SII word offset. */
+    size_t nwords; /**< Number of words. */
+    const uint16_t *words; /**< Pointer to the data words. */
+    ec_internal_request_state_t state; /**< State of the request. */
+} ec_sii_write_request_t;
+
+/****************************************************************************/
+
+typedef struct ec_fsm_master ec_fsm_master_t; /**< \see ec_fsm_master */
+
+/** Finite state machine of an EtherCAT master.
+ */
+struct ec_fsm_master {
+    ec_master_t *master; /**< master the FSM runs on */
+    ec_datagram_t *datagram; /**< datagram used in the state machine */
+    unsigned int retries; /**< retries on datagram timeout. */
+
+    void (*state)(ec_fsm_master_t *); /**< master state function */
+    ec_device_index_t dev_idx; /**< Current device index (for scanning etc.).
+                                */
+    int idle; /**< state machine is in idle phase */
+    unsigned long scan_jiffies; /**< beginning of slave scanning */
+    uint8_t link_state[EC_MAX_NUM_DEVICES]; /**< Last link state for every
+                                              device. */
+    unsigned int slaves_responding[EC_MAX_NUM_DEVICES]; /**< Number of
+                                                          responding slaves
+                                                          for every device. */
+    unsigned int rescan_required; /**< A bus rescan is required. */
+    ec_slave_state_t slave_states[EC_MAX_NUM_DEVICES]; /**< AL states of
+                                                         responding slaves for
+                                                         every device. */
+    ec_slave_t *slave; /**< current slave */
+    ec_sii_write_request_t *sii_request; /**< SII write request */
+    off_t sii_index; /**< index to SII write request data */
+    ec_sdo_request_t *sdo_request; /**< SDO request to process. */
+    ec_soe_request_t *soe_request; /**< SoE request to process. */
+
+    ec_fsm_coe_t fsm_coe; /**< CoE state machine */
+    ec_fsm_soe_t fsm_soe; /**< SoE state machine */
+    ec_fsm_pdo_t fsm_pdo; /**< PDO configuration state machine. */
+    ec_fsm_eoe_t fsm_eoe; /**< EoE state machine */
+    ec_fsm_change_t fsm_change; /**< State change state machine */
+    ec_fsm_slave_config_t fsm_slave_config; /**< slave state machine */
+    ec_fsm_slave_scan_t fsm_slave_scan; /**< slave state machine */
+    ec_fsm_sii_t fsm_sii; /**< SII state machine */
+};
+
+/****************************************************************************/
+
+void ec_fsm_master_init(ec_fsm_master_t *, ec_master_t *, ec_datagram_t *);
+void ec_fsm_master_clear(ec_fsm_master_t *);
+
+void ec_fsm_master_reset(ec_fsm_master_t *);
+
+int ec_fsm_master_exec(ec_fsm_master_t *);
+int ec_fsm_master_idle(const ec_fsm_master_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_pdo.c b/drivers/net/ethercat/master/fsm_pdo.c
new file mode 100644
index 000000000000..0d5f87e16e78
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_pdo.c
@@ -0,0 +1,806 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT PDO configuration state machine.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "slave_config.h"
+
+#include "fsm_pdo.h"
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_fsm_pdo_print(const ec_fsm_pdo_t *);
+int ec_fsm_pdo_running(const ec_fsm_pdo_t *);
+ec_pdo_t *ec_fsm_pdo_conf_action_next_pdo(const ec_fsm_pdo_t *,
+        const struct list_head *);
+
+/****************************************************************************/
+
+void ec_fsm_pdo_read_state_start(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_read_state_pdo_count(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_read_state_pdo(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_read_state_pdo_entries(ec_fsm_pdo_t *, ec_datagram_t *);
+
+void ec_fsm_pdo_read_action_next_sync(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_read_action_next_pdo(ec_fsm_pdo_t *, ec_datagram_t *);
+
+void ec_fsm_pdo_conf_state_start(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_state_read_mapping(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_state_mapping(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_state_zero_pdo_count(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_state_assign_pdo(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_state_set_pdo_count(ec_fsm_pdo_t *, ec_datagram_t *);
+
+void ec_fsm_pdo_conf_action_next_sync(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_action_pdo_mapping(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_action_check_mapping(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_action_next_pdo_mapping(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_action_check_assignment(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_conf_action_assign_pdo(ec_fsm_pdo_t *, ec_datagram_t *);
+
+void ec_fsm_pdo_state_end(ec_fsm_pdo_t *, ec_datagram_t *);
+void ec_fsm_pdo_state_error(ec_fsm_pdo_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_pdo_init(
+        ec_fsm_pdo_t *fsm, /**< PDO configuration state machine. */
+        ec_fsm_coe_t *fsm_coe /**< CoE state machine to use */
+        )
+{
+    fsm->fsm_coe = fsm_coe;
+    ec_fsm_pdo_entry_init(&fsm->fsm_pdo_entry, fsm_coe);
+    ec_pdo_list_init(&fsm->pdos);
+    ec_sdo_request_init(&fsm->request);
+    ec_pdo_init(&fsm->slave_pdo);
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_pdo_clear(
+        ec_fsm_pdo_t *fsm /**< PDO configuration state machine. */
+        )
+{
+    ec_fsm_pdo_entry_clear(&fsm->fsm_pdo_entry);
+    ec_pdo_list_clear(&fsm->pdos);
+    ec_sdo_request_clear(&fsm->request);
+    ec_pdo_clear(&fsm->slave_pdo);
+}
+
+/****************************************************************************/
+
+/** Print the current and desired PDO assignment.
+ */
+void ec_fsm_pdo_print(
+        const ec_fsm_pdo_t *fsm /**< PDO configuration state machine. */
+        )
+{
+    printk(KERN_CONT "Currently assigned PDOs: ");
+    ec_pdo_list_print(&fsm->sync->pdos);
+    printk(KERN_CONT ". PDOs to assign: ");
+    ec_pdo_list_print(&fsm->pdos);
+    printk(KERN_CONT "\n");
+}
+
+/****************************************************************************/
+
+/** Start reading the PDO configuration.
+ */
+void ec_fsm_pdo_start_reading(
+        ec_fsm_pdo_t *fsm, /**< PDO configuration state machine. */
+        ec_slave_t *slave /**< slave to configure */
+        )
+{
+    fsm->slave = slave;
+    fsm->state = ec_fsm_pdo_read_state_start;
+}
+
+/****************************************************************************/
+
+/** Start writing the PDO configuration.
+ */
+void ec_fsm_pdo_start_configuration(
+        ec_fsm_pdo_t *fsm, /**< PDO configuration state machine. */
+        ec_slave_t *slave /**< slave to configure */
+        )
+{
+    fsm->slave = slave;
+    fsm->state = ec_fsm_pdo_conf_state_start;
+}
+
+/****************************************************************************/
+
+/** Get running state.
+ *
+ * \return false, if state machine has terminated
+ */
+int ec_fsm_pdo_running(
+        const ec_fsm_pdo_t *fsm /**< PDO configuration state machine. */
+        )
+{
+    return fsm->state != ec_fsm_pdo_state_end
+        && fsm->state != ec_fsm_pdo_state_error;
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * If the state machine's datagram is not sent or received yet, the execution
+ * of the state machine is delayed to the next cycle.
+ *
+ * \return false, if state machine has terminated
+ */
+int ec_fsm_pdo_exec(
+        ec_fsm_pdo_t *fsm, /**< PDO configuration state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    fsm->state(fsm, datagram);
+
+    return ec_fsm_pdo_running(fsm);
+}
+
+/****************************************************************************/
+
+/** Get execution result.
+ *
+ * \return true, if the state machine terminated gracefully
+ */
+int ec_fsm_pdo_success(
+        const ec_fsm_pdo_t *fsm /**< PDO configuration state machine. */
+        )
+{
+    return fsm->state == ec_fsm_pdo_state_end;
+}
+
+/*****************************************************************************
+ * Reading state funtions.
+ ****************************************************************************/
+
+/** Start reading PDO assignment.
+ */
+void ec_fsm_pdo_read_state_start(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    // read PDO assignment for first sync manager not reserved for mailbox
+    fsm->sync_index = 1; // next is 2
+    ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Read PDO assignment of next sync manager.
+ */
+void ec_fsm_pdo_read_action_next_sync(
+        ec_fsm_pdo_t *fsm, /**< finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    fsm->sync_index++;
+
+    for (; fsm->sync_index < EC_MAX_SYNC_MANAGERS; fsm->sync_index++) {
+        if (!(fsm->sync = ec_slave_get_sync(slave, fsm->sync_index)))
+            continue;
+
+        EC_SLAVE_DBG(slave, 1, "Reading PDO assignment of SM%u.\n",
+                fsm->sync_index);
+
+        ec_pdo_list_clear_pdos(&fsm->pdos);
+
+        ecrt_sdo_request_index(&fsm->request, 0x1C10 + fsm->sync_index, 0);
+        ecrt_sdo_request_read(&fsm->request);
+        fsm->state = ec_fsm_pdo_read_state_pdo_count;
+        ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+        ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Reading of PDO configuration finished.\n");
+
+    ec_pdo_list_clear_pdos(&fsm->pdos);
+    fsm->state = ec_fsm_pdo_state_end;
+}
+
+/****************************************************************************/
+
+/** Count assigned PDOs.
+ */
+void ec_fsm_pdo_read_state_pdo_count(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_ERR(fsm->slave, "Failed to read number of assigned PDOs"
+                " for SM%u.\n", fsm->sync_index);
+        ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+        return;
+    }
+
+    if (fsm->request.data_size != sizeof(uint8_t)) {
+        EC_SLAVE_ERR(fsm->slave, "Invalid data size %zu returned"
+                " when uploading SDO 0x%04X:%02X.\n", fsm->request.data_size,
+                fsm->request.index, fsm->request.subindex);
+        ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+        return;
+    }
+    fsm->pdo_count = EC_READ_U8(fsm->request.data);
+
+    EC_SLAVE_DBG(fsm->slave, 1, "%u PDOs assigned.\n", fsm->pdo_count);
+
+    // read first PDO
+    fsm->pdo_pos = 1;
+    ec_fsm_pdo_read_action_next_pdo(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Read next PDO.
+ */
+void ec_fsm_pdo_read_action_next_pdo(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (fsm->pdo_pos <= fsm->pdo_count) {
+        ecrt_sdo_request_index(&fsm->request, 0x1C10 + fsm->sync_index,
+                fsm->pdo_pos);
+        ecrt_sdo_request_read(&fsm->request);
+        fsm->state = ec_fsm_pdo_read_state_pdo;
+        ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+        ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+        return;
+    }
+
+    // finished reading PDO configuration
+
+    ec_pdo_list_copy(&fsm->sync->pdos, &fsm->pdos);
+    ec_pdo_list_clear_pdos(&fsm->pdos);
+
+    // next sync manager
+    ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Fetch PDO information.
+ */
+void ec_fsm_pdo_read_state_pdo(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_ERR(fsm->slave, "Failed to read index of"
+                " assigned PDO %u from SM%u.\n",
+                fsm->pdo_pos, fsm->sync_index);
+        ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+        return;
+    }
+
+    if (fsm->request.data_size != sizeof(uint16_t)) {
+        EC_SLAVE_ERR(fsm->slave, "Invalid data size %zu returned"
+                " when uploading SDO 0x%04X:%02X.\n", fsm->request.data_size,
+                fsm->request.index, fsm->request.subindex);
+        ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+        return;
+    }
+
+    if (!(fsm->pdo = (ec_pdo_t *)
+                kmalloc(sizeof(ec_pdo_t), GFP_KERNEL))) {
+        EC_SLAVE_ERR(fsm->slave, "Failed to allocate PDO.\n");
+        ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+        return;
+    }
+
+    ec_pdo_init(fsm->pdo);
+    fsm->pdo->index = EC_READ_U16(fsm->request.data);
+    fsm->pdo->sync_index = fsm->sync_index;
+
+    EC_SLAVE_DBG(fsm->slave, 1, "PDO 0x%04X.\n", fsm->pdo->index);
+
+    list_add_tail(&fsm->pdo->list, &fsm->pdos.list);
+
+    fsm->state = ec_fsm_pdo_read_state_pdo_entries;
+    ec_fsm_pdo_entry_start_reading(&fsm->fsm_pdo_entry, fsm->slave, fsm->pdo);
+    fsm->state(fsm, datagram); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Fetch PDO information.
+ */
+void ec_fsm_pdo_read_state_pdo_entries(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_pdo_entry_exec(&fsm->fsm_pdo_entry, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_pdo_entry_success(&fsm->fsm_pdo_entry)) {
+        EC_SLAVE_ERR(fsm->slave, "Failed to read mapped PDO entries"
+                " for PDO 0x%04X.\n", fsm->pdo->index);
+        ec_fsm_pdo_read_action_next_sync(fsm, datagram);
+        return;
+    }
+
+    // next PDO
+    fsm->pdo_pos++;
+    ec_fsm_pdo_read_action_next_pdo(fsm, datagram);
+}
+
+/*****************************************************************************
+ * Writing state functions.
+ ****************************************************************************/
+
+/** Start PDO configuration.
+ */
+void ec_fsm_pdo_conf_state_start(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (!fsm->slave->config) {
+        fsm->state = ec_fsm_pdo_state_end;
+        return;
+    }
+
+    fsm->sync_index = 1; // next is 2
+    ec_fsm_pdo_conf_action_next_sync(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Assign next PDO.
+ *
+ * \return Next PDO, or NULL.
+ */
+ec_pdo_t *ec_fsm_pdo_conf_action_next_pdo(
+        const ec_fsm_pdo_t *fsm, /**< PDO configuration state machine. */
+        const struct list_head *list /**< current PDO list item */
+        )
+{
+    list = list->next;
+    if (list == &fsm->pdos.list)
+        return NULL; // no next PDO
+    return list_entry(list, ec_pdo_t, list);
+}
+
+/****************************************************************************/
+
+/** Get the next sync manager for a pdo configuration.
+ */
+void ec_fsm_pdo_conf_action_next_sync(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    fsm->sync_index++;
+
+    for (; fsm->sync_index < EC_MAX_SYNC_MANAGERS; fsm->sync_index++) {
+        if (!fsm->slave->config) {
+            // slave configuration removed in the meantime
+            fsm->state = ec_fsm_pdo_state_error;
+            return;
+        }
+
+        if (ec_pdo_list_copy(&fsm->pdos,
+                    &fsm->slave->config->sync_configs[fsm->sync_index].pdos))
+        {
+            fsm->state = ec_fsm_pdo_state_error;
+            return;
+        }
+
+        if (!(fsm->sync = ec_slave_get_sync(fsm->slave, fsm->sync_index))) {
+            if (!list_empty(&fsm->pdos.list))
+                EC_SLAVE_WARN(fsm->slave, "PDOs configured for SM%u,"
+                        " but slave does not provide the"
+                        " sync manager information!\n",
+                        fsm->sync_index);
+            continue;
+        }
+
+        // get first configured PDO
+        if (!(fsm->pdo =
+                    ec_fsm_pdo_conf_action_next_pdo(fsm, &fsm->pdos.list))) {
+            // no pdos configured
+            ec_fsm_pdo_conf_action_check_assignment(fsm, datagram);
+            return;
+        }
+
+        ec_fsm_pdo_conf_action_pdo_mapping(fsm, datagram);
+        return;
+    }
+
+    fsm->state = ec_fsm_pdo_state_end;
+}
+
+/****************************************************************************/
+
+/** Check if the mapping has to be read, otherwise start to configure it.
+ */
+void ec_fsm_pdo_conf_action_pdo_mapping(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    const ec_pdo_t *assigned_pdo;
+
+    fsm->slave_pdo.index = fsm->pdo->index;
+
+    if ((assigned_pdo = ec_slave_find_pdo(fsm->slave, fsm->pdo->index))) {
+        ec_pdo_copy_entries(&fsm->slave_pdo, assigned_pdo);
+    } else { // configured PDO is not assigned and thus unknown
+        ec_pdo_clear_entries(&fsm->slave_pdo);
+    }
+
+    if (list_empty(&fsm->slave_pdo.entries)) {
+        EC_SLAVE_DBG(fsm->slave, 1, "Reading mapping of PDO 0x%04X.\n",
+                fsm->pdo->index);
+
+        // pdo mapping is unknown; start loading it
+        ec_fsm_pdo_entry_start_reading(&fsm->fsm_pdo_entry, fsm->slave,
+                &fsm->slave_pdo);
+        fsm->state = ec_fsm_pdo_conf_state_read_mapping;
+        fsm->state(fsm, datagram); // execute immediately
+        return;
+    }
+
+    // pdo mapping is known, check if it most be re-configured
+    ec_fsm_pdo_conf_action_check_mapping(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Execute the PDO entry state machine to read the current PDO's mapping.
+ */
+void ec_fsm_pdo_conf_state_read_mapping(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_pdo_entry_exec(&fsm->fsm_pdo_entry, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_pdo_entry_success(&fsm->fsm_pdo_entry))
+        EC_SLAVE_WARN(fsm->slave,
+                "Failed to read PDO entries for PDO 0x%04X.\n",
+                fsm->pdo->index);
+
+    // check if the mapping must be re-configured
+    ec_fsm_pdo_conf_action_check_mapping(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Check if the mapping has to be re-configured.
+ *
+ * \todo Display mapping differences.
+ */
+void ec_fsm_pdo_conf_action_check_mapping(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    // check, if slave supports PDO configuration
+    if ((fsm->slave->sii.mailbox_protocols & EC_MBOX_COE)
+            && fsm->slave->sii.has_general
+            && fsm->slave->sii.coe_details.enable_pdo_configuration) {
+
+        // always write PDO mapping
+        ec_fsm_pdo_entry_start_configuration(&fsm->fsm_pdo_entry, fsm->slave,
+                fsm->pdo, &fsm->slave_pdo);
+        fsm->state = ec_fsm_pdo_conf_state_mapping;
+        fsm->state(fsm, datagram); // execure immediately
+        return;
+    }
+    else if (!ec_pdo_equal_entries(fsm->pdo, &fsm->slave_pdo)) {
+        EC_SLAVE_WARN(fsm->slave, "Slave does not support"
+                " changing the PDO mapping!\n");
+        EC_SLAVE_WARN(fsm->slave, "");
+        printk(KERN_CONT "Currently mapped PDO entries: ");
+        ec_pdo_print_entries(&fsm->slave_pdo);
+        printk(KERN_CONT ". Entries to map: ");
+        ec_pdo_print_entries(fsm->pdo);
+        printk(KERN_CONT "\n");
+    }
+
+    ec_fsm_pdo_conf_action_next_pdo_mapping(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Let the PDO entry state machine configure the current PDO's mapping.
+ */
+void ec_fsm_pdo_conf_state_mapping(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_pdo_entry_exec(&fsm->fsm_pdo_entry, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_pdo_entry_success(&fsm->fsm_pdo_entry))
+        EC_SLAVE_WARN(fsm->slave,
+                "Failed to configure mapping of PDO 0x%04X.\n",
+                fsm->pdo->index);
+
+    ec_fsm_pdo_conf_action_next_pdo_mapping(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Check mapping of next PDO, otherwise configure assignment.
+ */
+void ec_fsm_pdo_conf_action_next_pdo_mapping(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    // get next configured PDO
+    if (!(fsm->pdo = ec_fsm_pdo_conf_action_next_pdo(fsm, &fsm->pdo->list))) {
+        // no more configured pdos
+        ec_fsm_pdo_conf_action_check_assignment(fsm, datagram);
+        return;
+    }
+
+    ec_fsm_pdo_conf_action_pdo_mapping(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Check if the PDO assignment of the current SM has to be re-configured.
+ */
+void ec_fsm_pdo_conf_action_check_assignment(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if ((fsm->slave->sii.mailbox_protocols & EC_MBOX_COE)
+            && fsm->slave->sii.has_general
+            && fsm->slave->sii.coe_details.enable_pdo_assign) {
+
+        // always write PDO assignment
+        if (fsm->slave->master->debug_level) {
+            EC_SLAVE_DBG(fsm->slave, 1, "Setting PDO assignment of SM%u:\n",
+                    fsm->sync_index);
+            EC_SLAVE_DBG(fsm->slave, 1, ""); ec_fsm_pdo_print(fsm);
+        }
+
+        if (ec_sdo_request_alloc(&fsm->request, 2)) {
+            fsm->state = ec_fsm_pdo_state_error;
+            return;
+        }
+
+        // set mapped PDO count to zero
+        EC_WRITE_U8(fsm->request.data, 0); // zero PDOs mapped
+        fsm->request.data_size = 1;
+        ecrt_sdo_request_index(&fsm->request, 0x1C10 + fsm->sync_index, 0);
+        ecrt_sdo_request_write(&fsm->request);
+
+        EC_SLAVE_DBG(fsm->slave, 1, "Setting number of assigned"
+                " PDOs to zero.\n");
+
+        fsm->state = ec_fsm_pdo_conf_state_zero_pdo_count;
+        ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+        ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+        return;
+    }
+    else if (!ec_pdo_list_equal(&fsm->sync->pdos, &fsm->pdos)) {
+        EC_SLAVE_WARN(fsm->slave, "Slave does not support assigning PDOs!\n");
+        EC_SLAVE_WARN(fsm->slave, ""); ec_fsm_pdo_print(fsm);
+    }
+
+    ec_fsm_pdo_conf_action_next_sync(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Set the number of assigned PDOs to zero.
+ */
+void ec_fsm_pdo_conf_state_zero_pdo_count(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_WARN(fsm->slave, "Failed to clear PDO assignment of SM%u.\n",
+                fsm->sync_index);
+        EC_SLAVE_WARN(fsm->slave, "");
+        ec_fsm_pdo_print(fsm);
+        ec_fsm_pdo_conf_action_next_sync(fsm, datagram);
+        return;
+    }
+
+    // the sync manager's assigned PDOs have been cleared
+    ec_pdo_list_clear_pdos(&fsm->sync->pdos);
+
+    // assign all PDOs belonging to the current sync manager
+
+    // find first PDO
+    if (!(fsm->pdo = ec_fsm_pdo_conf_action_next_pdo(fsm, &fsm->pdos.list))) {
+        // check for mapping to be altered
+        ec_fsm_pdo_conf_action_next_sync(fsm, datagram);
+        return;
+    }
+
+    // assign first PDO
+    fsm->pdo_pos = 1;
+    ec_fsm_pdo_conf_action_assign_pdo(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Assign a PDO.
+ */
+void ec_fsm_pdo_conf_action_assign_pdo(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    EC_WRITE_U16(fsm->request.data, fsm->pdo->index);
+    fsm->request.data_size = 2;
+    ecrt_sdo_request_index(&fsm->request,
+            0x1C10 + fsm->sync_index, fsm->pdo_pos);
+    ecrt_sdo_request_write(&fsm->request);
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Assigning PDO 0x%04X at position %u.\n",
+            fsm->pdo->index, fsm->pdo_pos);
+
+    fsm->state = ec_fsm_pdo_conf_state_assign_pdo;
+    ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+    ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Add a PDO to the sync managers PDO assignment.
+ */
+void ec_fsm_pdo_conf_state_assign_pdo(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_WARN(fsm->slave, "Failed to assign PDO 0x%04X at position %u"
+                " of SM%u.\n",
+                fsm->pdo->index, fsm->pdo_pos, fsm->sync_index);
+        EC_SLAVE_WARN(fsm->slave, ""); ec_fsm_pdo_print(fsm);
+        fsm->state = ec_fsm_pdo_state_error;
+        return;
+    }
+
+    // find next PDO
+    if (!(fsm->pdo = ec_fsm_pdo_conf_action_next_pdo(fsm, &fsm->pdo->list))) {
+        // no more PDOs to assign, set PDO count
+        EC_WRITE_U8(fsm->request.data, fsm->pdo_pos);
+        fsm->request.data_size = 1;
+        ecrt_sdo_request_index(&fsm->request, 0x1C10 + fsm->sync_index, 0);
+        ecrt_sdo_request_write(&fsm->request);
+
+        EC_SLAVE_DBG(fsm->slave, 1,
+                "Setting number of assigned PDOs to %u.\n",
+                fsm->pdo_pos);
+
+        fsm->state = ec_fsm_pdo_conf_state_set_pdo_count;
+        ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+        ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+        return;
+    }
+
+    // add next PDO to assignment
+    fsm->pdo_pos++;
+    ec_fsm_pdo_conf_action_assign_pdo(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Set the number of assigned PDOs.
+ */
+void ec_fsm_pdo_conf_state_set_pdo_count(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_WARN(fsm->slave, "Failed to set number of"
+                " assigned PDOs of SM%u.\n", fsm->sync_index);
+        EC_SLAVE_WARN(fsm->slave, ""); ec_fsm_pdo_print(fsm);
+        fsm->state = ec_fsm_pdo_state_error;
+        return;
+    }
+
+    // PDOs have been configured
+    ec_pdo_list_copy(&fsm->sync->pdos, &fsm->pdos);
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Successfully configured"
+            " PDO assignment of SM%u.\n", fsm->sync_index);
+
+    // check if PDO mapping has to be altered
+    ec_fsm_pdo_conf_action_next_sync(fsm, datagram);
+}
+
+/*****************************************************************************
+ * Common state functions
+ ****************************************************************************/
+
+/** State: ERROR.
+ */
+void ec_fsm_pdo_state_error(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** State: END.
+ */
+void ec_fsm_pdo_state_end(
+        ec_fsm_pdo_t *fsm, /**< Finite state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_pdo.h b/drivers/net/ethercat/master/fsm_pdo.h
new file mode 100644
index 000000000000..41fc876a23ba
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_pdo.h
@@ -0,0 +1,76 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT PDO configuration state machine structures.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_PDO_H__
+#define __EC_FSM_PDO_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "fsm_coe.h"
+#include "fsm_pdo_entry.h"
+
+/****************************************************************************/
+
+/**
+ * \see ec_fsm_pdo
+ */
+typedef struct ec_fsm_pdo ec_fsm_pdo_t;
+
+/** PDO configuration state machine.
+ */
+struct ec_fsm_pdo
+{
+    void (*state)(ec_fsm_pdo_t *, ec_datagram_t *); /**< State function. */
+    ec_fsm_coe_t *fsm_coe; /**< CoE state machine to use. */
+    ec_fsm_pdo_entry_t fsm_pdo_entry; /**< PDO entry state machine. */
+    ec_pdo_list_t pdos; /**< PDO configuration. */
+    ec_sdo_request_t request; /**< SDO request. */
+    ec_pdo_t slave_pdo; /**< PDO actually appearing in a slave. */
+
+    ec_slave_t *slave; /**< Slave the FSM runs on. */
+    uint8_t sync_index; /**< Current sync manager index. */
+    ec_sync_t *sync; /**< Current sync manager. */
+    ec_pdo_t *pdo; /**< Current PDO. */
+    unsigned int pdo_pos; /**< Assignment position of current PDOs. */
+    unsigned int pdo_count; /**< Number of assigned PDOs. */
+};
+
+/****************************************************************************/
+
+void ec_fsm_pdo_init(ec_fsm_pdo_t *, ec_fsm_coe_t *);
+void ec_fsm_pdo_clear(ec_fsm_pdo_t *);
+
+void ec_fsm_pdo_start_reading(ec_fsm_pdo_t *, ec_slave_t *);
+void ec_fsm_pdo_start_configuration(ec_fsm_pdo_t *, ec_slave_t *);
+
+int ec_fsm_pdo_exec(ec_fsm_pdo_t *, ec_datagram_t *);
+int ec_fsm_pdo_success(const ec_fsm_pdo_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_pdo_entry.c b/drivers/net/ethercat/master/fsm_pdo_entry.c
new file mode 100644
index 000000000000..3c7be4289252
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_pdo_entry.c
@@ -0,0 +1,541 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT PDO mapping state machine.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "slave_config.h"
+
+#include "fsm_pdo_entry.h"
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_fsm_pdo_entry_print(const ec_fsm_pdo_entry_t *);
+int ec_fsm_pdo_entry_running(const ec_fsm_pdo_entry_t *);
+ec_pdo_entry_t *ec_fsm_pdo_entry_conf_next_entry(const ec_fsm_pdo_entry_t *,
+        const struct list_head *);
+
+/****************************************************************************/
+
+void ec_fsm_pdo_entry_read_state_start(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+void ec_fsm_pdo_entry_read_state_count(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+void ec_fsm_pdo_entry_read_state_entry(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+
+void ec_fsm_pdo_entry_read_action_next(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+
+void ec_fsm_pdo_entry_conf_state_start(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+void ec_fsm_pdo_entry_conf_state_zero_entry_count(ec_fsm_pdo_entry_t *,
+        ec_datagram_t *);
+void ec_fsm_pdo_entry_conf_state_map_entry(ec_fsm_pdo_entry_t *,
+        ec_datagram_t *);
+void ec_fsm_pdo_entry_conf_state_set_entry_count(ec_fsm_pdo_entry_t *,
+        ec_datagram_t *);
+
+void ec_fsm_pdo_entry_conf_action_map(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+
+void ec_fsm_pdo_entry_state_end(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+void ec_fsm_pdo_entry_state_error(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_pdo_entry_init(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_fsm_coe_t *fsm_coe /**< CoE state machine to use. */
+        )
+{
+    fsm->fsm_coe = fsm_coe;
+    ec_sdo_request_init(&fsm->request);
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_pdo_entry_clear(
+        ec_fsm_pdo_entry_t *fsm /**< PDO mapping state machine. */
+        )
+{
+    ec_sdo_request_clear(&fsm->request);
+}
+
+/****************************************************************************/
+
+/** Print the current and desired PDO mapping.
+ */
+void ec_fsm_pdo_entry_print(
+        const ec_fsm_pdo_entry_t *fsm /**< PDO mapping state machine. */
+        )
+{
+    printk(KERN_CONT "Currently mapped PDO entries: ");
+    ec_pdo_print_entries(fsm->cur_pdo);
+    printk(KERN_CONT ". Entries to map: ");
+    ec_pdo_print_entries(fsm->source_pdo);
+    printk(KERN_CONT "\n");
+}
+
+/****************************************************************************/
+
+/** Start reading a PDO's entries.
+ */
+void ec_fsm_pdo_entry_start_reading(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_slave_t *slave, /**< Slave to configure. */
+        ec_pdo_t *pdo /**< PDO to read entries for. */
+        )
+{
+    fsm->slave = slave;
+    fsm->target_pdo = pdo;
+
+    ec_pdo_clear_entries(fsm->target_pdo);
+
+    fsm->state = ec_fsm_pdo_entry_read_state_start;
+}
+
+/****************************************************************************/
+
+/** Start PDO mapping state machine.
+ */
+void ec_fsm_pdo_entry_start_configuration(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_slave_t *slave, /**< Slave to configure. */
+        const ec_pdo_t *pdo, /**< PDO with the desired entries. */
+        const ec_pdo_t *cur_pdo /**< Current PDO mapping. */
+        )
+{
+    fsm->slave = slave;
+    fsm->source_pdo = pdo;
+    fsm->cur_pdo = cur_pdo;
+
+    if (fsm->slave->master->debug_level) {
+        EC_SLAVE_DBG(slave, 1, "Changing mapping of PDO 0x%04X.\n",
+                pdo->index);
+        EC_SLAVE_DBG(slave, 1, ""); ec_fsm_pdo_entry_print(fsm);
+    }
+
+    fsm->state = ec_fsm_pdo_entry_conf_state_start;
+}
+
+/****************************************************************************/
+
+/** Get running state.
+ *
+ * \return false, if state machine has terminated
+ */
+int ec_fsm_pdo_entry_running(
+        const ec_fsm_pdo_entry_t *fsm /**< PDO mapping state machine. */
+        )
+{
+    return fsm->state != ec_fsm_pdo_entry_state_end
+        && fsm->state != ec_fsm_pdo_entry_state_error;
+}
+
+/****************************************************************************/
+
+/** Executes the current state.
+ *
+ * \return false, if state machine has terminated
+ */
+int ec_fsm_pdo_entry_exec(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    fsm->state(fsm, datagram);
+
+    return ec_fsm_pdo_entry_running(fsm);
+}
+
+/****************************************************************************/
+
+/** Get execution result.
+ *
+ * \return true, if the state machine terminated gracefully
+ */
+int ec_fsm_pdo_entry_success(
+        const ec_fsm_pdo_entry_t *fsm /**< PDO mapping state machine. */
+        )
+{
+    return fsm->state == ec_fsm_pdo_entry_state_end;
+}
+
+/*****************************************************************************
+ * Reading state functions.
+ ****************************************************************************/
+
+/** Request reading the number of mapped PDO entries.
+ */
+void ec_fsm_pdo_entry_read_state_start(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ecrt_sdo_request_index(&fsm->request, fsm->target_pdo->index, 0);
+    ecrt_sdo_request_read(&fsm->request);
+
+    fsm->state = ec_fsm_pdo_entry_read_state_count;
+    ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+    ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Read number of mapped PDO entries.
+ */
+void ec_fsm_pdo_entry_read_state_count(
+        ec_fsm_pdo_entry_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_ERR(fsm->slave,
+                "Failed to read number of mapped PDO entries.\n");
+        fsm->state = ec_fsm_pdo_entry_state_error;
+        return;
+    }
+
+    if (fsm->request.data_size != sizeof(uint8_t)) {
+        EC_SLAVE_ERR(fsm->slave, "Invalid data size %zu at uploading"
+                " SDO 0x%04X:%02X.\n",
+                fsm->request.data_size, fsm->request.index,
+                fsm->request.subindex);
+        fsm->state = ec_fsm_pdo_entry_state_error;
+        return;
+    }
+
+    fsm->entry_count = EC_READ_U8(fsm->request.data);
+
+    EC_SLAVE_DBG(fsm->slave, 1, "%u PDO entries mapped.\n", fsm->entry_count);
+
+    // read first PDO entry
+    fsm->entry_pos = 1;
+    ec_fsm_pdo_entry_read_action_next(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Read next PDO entry.
+ */
+void ec_fsm_pdo_entry_read_action_next(
+        ec_fsm_pdo_entry_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (fsm->entry_pos <= fsm->entry_count) {
+        ecrt_sdo_request_index(&fsm->request, fsm->target_pdo->index,
+                fsm->entry_pos);
+        ecrt_sdo_request_read(&fsm->request);
+        fsm->state = ec_fsm_pdo_entry_read_state_entry;
+        ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+        ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+        return;
+    }
+
+    // finished reading entries.
+    fsm->state = ec_fsm_pdo_entry_state_end;
+}
+
+/****************************************************************************/
+
+/** Read PDO entry information.
+ */
+void ec_fsm_pdo_entry_read_state_entry(
+        ec_fsm_pdo_entry_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_ERR(fsm->slave, "Failed to read mapped PDO entry.\n");
+        fsm->state = ec_fsm_pdo_entry_state_error;
+        return;
+    }
+
+    if (fsm->request.data_size != sizeof(uint32_t)) {
+        EC_SLAVE_ERR(fsm->slave, "Invalid data size %zu at"
+                " uploading SDO 0x%04X:%02X.\n",
+                fsm->request.data_size, fsm->request.index,
+                fsm->request.subindex);
+        fsm->state = ec_fsm_pdo_entry_state_error;
+    } else {
+        uint32_t pdo_entry_info;
+        ec_pdo_entry_t *pdo_entry;
+
+        pdo_entry_info = EC_READ_U32(fsm->request.data);
+
+        if (!(pdo_entry = (ec_pdo_entry_t *)
+                    kmalloc(sizeof(ec_pdo_entry_t), GFP_KERNEL))) {
+            EC_SLAVE_ERR(fsm->slave, "Failed to allocate PDO entry.\n");
+            fsm->state = ec_fsm_pdo_entry_state_error;
+            return;
+        }
+
+        ec_pdo_entry_init(pdo_entry);
+        pdo_entry->index = pdo_entry_info >> 16;
+        pdo_entry->subindex = (pdo_entry_info >> 8) & 0xFF;
+        pdo_entry->bit_length = pdo_entry_info & 0xFF;
+
+        if (!pdo_entry->index && !pdo_entry->subindex) {
+            if (ec_pdo_entry_set_name(pdo_entry, "Gap")) {
+                ec_pdo_entry_clear(pdo_entry);
+                kfree(pdo_entry);
+                fsm->state = ec_fsm_pdo_entry_state_error;
+                return;
+            }
+        }
+
+        EC_SLAVE_DBG(fsm->slave, 1,
+                "PDO entry 0x%04X:%02X, %u bit, \"%s\".\n",
+                pdo_entry->index, pdo_entry->subindex,
+                pdo_entry->bit_length,
+                pdo_entry->name ? pdo_entry->name : "???");
+
+        list_add_tail(&pdo_entry->list, &fsm->target_pdo->entries);
+
+        // next PDO entry
+        fsm->entry_pos++;
+        ec_fsm_pdo_entry_read_action_next(fsm, datagram);
+    }
+}
+
+/*****************************************************************************
+ * Configuration state functions.
+ ****************************************************************************/
+
+/** Start PDO mapping.
+ */
+void ec_fsm_pdo_entry_conf_state_start(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_sdo_request_alloc(&fsm->request, 4)) {
+        fsm->state = ec_fsm_pdo_entry_state_error;
+        return;
+    }
+
+    // set mapped PDO entry count to zero
+    EC_WRITE_U8(fsm->request.data, 0);
+    fsm->request.data_size = 1;
+    ecrt_sdo_request_index(&fsm->request, fsm->source_pdo->index, 0);
+    ecrt_sdo_request_write(&fsm->request);
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Setting entry count to zero.\n");
+
+    fsm->state = ec_fsm_pdo_entry_conf_state_zero_entry_count;
+    ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+    ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Process next PDO entry.
+ *
+ * \return Next PDO entry, or NULL.
+ */
+ec_pdo_entry_t *ec_fsm_pdo_entry_conf_next_entry(
+        const ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        const struct list_head *list /**< current entry list item */
+        )
+{
+    list = list->next;
+    if (list == &fsm->source_pdo->entries)
+        return NULL; // no next entry
+    return list_entry(list, ec_pdo_entry_t, list);
+}
+
+/****************************************************************************/
+
+/** Set the number of mapped entries to zero.
+ */
+void ec_fsm_pdo_entry_conf_state_zero_entry_count(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_WARN(fsm->slave, "Failed to clear PDO mapping.\n");
+        EC_SLAVE_WARN(fsm->slave, ""); ec_fsm_pdo_entry_print(fsm);
+        fsm->state = ec_fsm_pdo_entry_state_error;
+        return;
+    }
+
+    // find first entry
+    if (!(fsm->entry = ec_fsm_pdo_entry_conf_next_entry(
+                    fsm, &fsm->source_pdo->entries))) {
+
+        EC_SLAVE_DBG(fsm->slave, 1, "No entries to map.\n");
+
+        fsm->state = ec_fsm_pdo_entry_state_end; // finished
+        return;
+    }
+
+    // add first entry
+    fsm->entry_pos = 1;
+    ec_fsm_pdo_entry_conf_action_map(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Starts to add a PDO entry.
+ */
+void ec_fsm_pdo_entry_conf_action_map(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    uint32_t value;
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Mapping PDO entry 0x%04X:%02X (%u bit)"
+            " at position %u.\n",
+            fsm->entry->index, fsm->entry->subindex,
+            fsm->entry->bit_length, fsm->entry_pos);
+
+    value = fsm->entry->index << 16
+        | fsm->entry->subindex << 8 | fsm->entry->bit_length;
+    EC_WRITE_U32(fsm->request.data, value);
+    fsm->request.data_size = 4;
+    ecrt_sdo_request_index(&fsm->request, fsm->source_pdo->index,
+            fsm->entry_pos);
+    ecrt_sdo_request_write(&fsm->request);
+
+    fsm->state = ec_fsm_pdo_entry_conf_state_map_entry;
+    ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+    ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Add a PDO entry.
+ */
+void ec_fsm_pdo_entry_conf_state_map_entry(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_WARN(fsm->slave, "Failed to map PDO entry"
+                " 0x%04X:%02X (%u bit) to position %u.\n",
+                fsm->entry->index, fsm->entry->subindex,
+                fsm->entry->bit_length, fsm->entry_pos);
+        EC_SLAVE_WARN(fsm->slave, ""); ec_fsm_pdo_entry_print(fsm);
+        fsm->state = ec_fsm_pdo_entry_state_error;
+        return;
+    }
+
+    // find next entry
+    if (!(fsm->entry = ec_fsm_pdo_entry_conf_next_entry(
+                    fsm, &fsm->entry->list))) {
+
+        // No more entries to add. Write entry count.
+        EC_WRITE_U8(fsm->request.data, fsm->entry_pos);
+        fsm->request.data_size = 1;
+        ecrt_sdo_request_index(&fsm->request, fsm->source_pdo->index, 0);
+        ecrt_sdo_request_write(&fsm->request);
+
+        EC_SLAVE_DBG(fsm->slave, 1, "Setting number of PDO entries to %u.\n",
+                fsm->entry_pos);
+
+        fsm->state = ec_fsm_pdo_entry_conf_state_set_entry_count;
+        ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request);
+        ec_fsm_coe_exec(fsm->fsm_coe, datagram); // execute immediately
+        return;
+    }
+
+    // add next entry
+    fsm->entry_pos++;
+    ec_fsm_pdo_entry_conf_action_map(fsm, datagram);
+}
+
+/****************************************************************************/
+
+/** Set the number of entries.
+ */
+void ec_fsm_pdo_entry_conf_state_set_entry_count(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_WARN(fsm->slave, "Failed to set number of entries.\n");
+        EC_SLAVE_WARN(fsm->slave, ""); ec_fsm_pdo_entry_print(fsm);
+        fsm->state = ec_fsm_pdo_entry_state_error;
+        return;
+    }
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Successfully configured"
+            " mapping for PDO 0x%04X.\n", fsm->source_pdo->index);
+
+    fsm->state = ec_fsm_pdo_entry_state_end; // finished
+}
+
+/*****************************************************************************
+ * Common state functions
+ ****************************************************************************/
+
+/** State: ERROR.
+ */
+void ec_fsm_pdo_entry_state_error(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** State: END.
+ */
+void ec_fsm_pdo_entry_state_end(
+        ec_fsm_pdo_entry_t *fsm, /**< PDO mapping state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_pdo_entry.h b/drivers/net/ethercat/master/fsm_pdo_entry.h
new file mode 100644
index 000000000000..d6638af65c70
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_pdo_entry.h
@@ -0,0 +1,75 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT PDO entry configuration state machine structures.
+ */
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_PDO_ENTRY_H__
+#define __EC_FSM_PDO_ENTRY_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "fsm_coe.h"
+
+/****************************************************************************/
+
+/**
+ * \see ec_fsm_pdo_entry
+ */
+typedef struct ec_fsm_pdo_entry ec_fsm_pdo_entry_t;
+
+/** PDO configuration state machine.
+ */
+struct ec_fsm_pdo_entry
+{
+    void (*state)(ec_fsm_pdo_entry_t *, ec_datagram_t *); /**< state function
+                                                           */
+    ec_fsm_coe_t *fsm_coe; /**< CoE state machine to use */
+    ec_sdo_request_t request; /**< SDO request. */
+
+    ec_slave_t *slave; /**< Slave the FSM runs on. */
+    ec_pdo_t *target_pdo; /**< PDO to read the mapping for. */
+    const ec_pdo_t *source_pdo; /**< PDO with desired mapping. */
+    const ec_pdo_t *cur_pdo; /**< PDO with current mapping (display only). */
+    const ec_pdo_entry_t *entry; /**< Current entry. */
+    unsigned int entry_count; /**< Number of entries. */
+    unsigned int entry_pos; /**< Position in PDO mapping. */
+};
+
+/****************************************************************************/
+
+void ec_fsm_pdo_entry_init(ec_fsm_pdo_entry_t *, ec_fsm_coe_t *);
+void ec_fsm_pdo_entry_clear(ec_fsm_pdo_entry_t *);
+
+void ec_fsm_pdo_entry_start_reading(ec_fsm_pdo_entry_t *, ec_slave_t *,
+        ec_pdo_t *);
+void ec_fsm_pdo_entry_start_configuration(ec_fsm_pdo_entry_t *, ec_slave_t *,
+        const ec_pdo_t *, const ec_pdo_t *);
+
+int ec_fsm_pdo_entry_exec(ec_fsm_pdo_entry_t *, ec_datagram_t *);
+int ec_fsm_pdo_entry_success(const ec_fsm_pdo_entry_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_sii.c b/drivers/net/ethercat/master/fsm_sii.c
new file mode 100644
index 000000000000..7e5954915499
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_sii.c
@@ -0,0 +1,482 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave information interface FSM.
+*/
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "mailbox.h"
+#include "master.h"
+#include "fsm_sii.h"
+
+/** Read/write timeout [ms].
+ *
+ * Used to calculate timeouts bsed on the jiffies counter.
+ *
+ * \attention Must be more than 10 to avoid problems on kernels that run with
+ * a timer interupt frequency of 100 Hz.
+ */
+#define SII_TIMEOUT 20
+
+/** Time before evaluating answer at writing [ms].
+ */
+#define SII_INHIBIT 5
+
+//#define SII_DEBUG
+
+/****************************************************************************/
+
+void ec_fsm_sii_state_start_reading(ec_fsm_sii_t *);
+void ec_fsm_sii_state_read_check(ec_fsm_sii_t *);
+void ec_fsm_sii_state_read_fetch(ec_fsm_sii_t *);
+void ec_fsm_sii_state_start_writing(ec_fsm_sii_t *);
+void ec_fsm_sii_state_write_check(ec_fsm_sii_t *);
+void ec_fsm_sii_state_write_check2(ec_fsm_sii_t *);
+void ec_fsm_sii_state_end(ec_fsm_sii_t *);
+void ec_fsm_sii_state_error(ec_fsm_sii_t *);
+
+/****************************************************************************/
+
+/**
+   Constructor.
+*/
+
+void ec_fsm_sii_init(ec_fsm_sii_t *fsm, /**< finite state machine */
+                     ec_datagram_t *datagram /**< datagram structure to use */
+                     )
+{
+    fsm->state = NULL;
+    fsm->datagram = datagram;
+}
+
+/****************************************************************************/
+
+/**
+   Destructor.
+*/
+
+void ec_fsm_sii_clear(ec_fsm_sii_t *fsm /**< finite state machine */)
+{
+}
+
+/****************************************************************************/
+
+/**
+   Initializes the SII read state machine.
+*/
+
+void ec_fsm_sii_read(ec_fsm_sii_t *fsm, /**< finite state machine */
+                     ec_slave_t *slave, /**< slave to read from */
+                     uint16_t word_offset, /**< offset to read from */
+                     ec_fsm_sii_addressing_t mode /**< addressing scheme */
+                     )
+{
+    fsm->state = ec_fsm_sii_state_start_reading;
+    fsm->slave = slave;
+    fsm->word_offset = word_offset;
+    fsm->mode = mode;
+}
+
+/****************************************************************************/
+
+/**
+   Initializes the SII write state machine.
+*/
+
+void ec_fsm_sii_write(ec_fsm_sii_t *fsm, /**< finite state machine */
+                      ec_slave_t *slave, /**< slave to read from */
+                      uint16_t word_offset, /**< offset to read from */
+                      const uint16_t *value, /**< pointer to 2 bytes of data */
+                      ec_fsm_sii_addressing_t mode /**< addressing scheme */
+                      )
+{
+    fsm->state = ec_fsm_sii_state_start_writing;
+    fsm->slave = slave;
+    fsm->word_offset = word_offset;
+    fsm->mode = mode;
+    memcpy(fsm->value, value, 2);
+}
+
+/****************************************************************************/
+
+/**
+   Executes the SII state machine.
+   \return false, if the state machine has terminated
+*/
+
+int ec_fsm_sii_exec(ec_fsm_sii_t *fsm /**< finite state machine */)
+{
+    fsm->state(fsm);
+
+    return fsm->state != ec_fsm_sii_state_end
+        && fsm->state != ec_fsm_sii_state_error;
+}
+
+/****************************************************************************/
+
+/**
+   Returns, if the master startup state machine terminated with success.
+   \return non-zero if successful.
+*/
+
+int ec_fsm_sii_success(ec_fsm_sii_t *fsm /**< Finite state machine */)
+{
+    return fsm->state == ec_fsm_sii_state_end;
+}
+
+/*****************************************************************************
+ * state functions
+ ****************************************************************************/
+
+/**
+   SII state: START READING.
+   Starts reading the slave information interface.
+*/
+
+void ec_fsm_sii_state_start_reading(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    // initiate read operation
+    switch (fsm->mode) {
+        case EC_FSM_SII_USE_INCREMENT_ADDRESS:
+            ec_datagram_apwr(datagram, fsm->slave->ring_position, 0x502, 4);
+            break;
+        case EC_FSM_SII_USE_CONFIGURED_ADDRESS:
+            ec_datagram_fpwr(datagram, fsm->slave->station_address, 0x502, 4);
+            break;
+    }
+
+    EC_WRITE_U8 (datagram->data,     0x80); // two address octets
+    EC_WRITE_U8 (datagram->data + 1, 0x01); // request read operation
+    EC_WRITE_U16(datagram->data + 2, fsm->word_offset);
+
+#ifdef SII_DEBUG
+    EC_SLAVE_DBG(fsm->slave, 0, "reading SII data, word %u:\n",
+            fsm->word_offset);
+    ec_print_data(datagram->data, 4);
+#endif
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_sii_state_read_check;
+}
+
+/****************************************************************************/
+
+/**
+   SII state: READ CHECK.
+   Checks, if the SII-read-datagram has been sent and issues a fetch datagram.
+*/
+
+void ec_fsm_sii_state_read_check(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed to receive SII read datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Reception of SII read datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    fsm->jiffies_start = datagram->jiffies_sent;
+    fsm->check_once_more = 1;
+
+    // issue check/fetch datagram
+    switch (fsm->mode) {
+        case EC_FSM_SII_USE_INCREMENT_ADDRESS:
+            ec_datagram_aprd(datagram, fsm->slave->ring_position, 0x502, 10);
+            break;
+        case EC_FSM_SII_USE_CONFIGURED_ADDRESS:
+            ec_datagram_fprd(datagram, fsm->slave->station_address, 0x502, 10);
+            break;
+    }
+
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_sii_state_read_fetch;
+}
+
+/****************************************************************************/
+
+/**
+   SII state: READ FETCH.
+   Fetches the result of an SII-read datagram.
+*/
+void ec_fsm_sii_state_read_fetch(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave,
+                "Failed to receive SII check/fetch datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave,
+                "Reception of SII check/fetch datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+#ifdef SII_DEBUG
+    EC_SLAVE_DBG(fsm->slave, 0, "checking SII read state:\n");
+    ec_print_data(datagram->data, 10);
+#endif
+
+    if (EC_READ_U8(datagram->data + 1) & 0x20) {
+        EC_SLAVE_ERR(fsm->slave, "Error on last command while"
+                " reading from SII word 0x%04x.\n", fsm->word_offset);
+        fsm->state = ec_fsm_sii_state_error;
+        return;
+    }
+
+    // check "busy bit"
+    if (EC_READ_U8(datagram->data + 1) & 0x81) { /* busy bit or
+                                                    read operation busy */
+        // still busy... timeout?
+        unsigned long diff_ms =
+            (datagram->jiffies_received - fsm->jiffies_start) * 1000 / HZ;
+        if (diff_ms >= SII_TIMEOUT) {
+            if (fsm->check_once_more) {
+                fsm->check_once_more = 0;
+            } else {
+                EC_SLAVE_ERR(fsm->slave, "SII: Read timeout.\n");
+                fsm->state = ec_fsm_sii_state_error;
+                return;
+            }
+        }
+
+        // issue check/fetch datagram again
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // SII value received.
+    memcpy(fsm->value, datagram->data + 6, 4);
+    fsm->state = ec_fsm_sii_state_end;
+}
+
+/****************************************************************************/
+
+/**
+   SII state: START WRITING.
+   Starts writing a word through the slave information interface.
+*/
+
+void ec_fsm_sii_state_start_writing(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    // initiate write operation
+    ec_datagram_fpwr(datagram, fsm->slave->station_address, 0x502, 8);
+    EC_WRITE_U8 (datagram->data,     0x81); /* two address octets
+                                               + enable write access */
+    EC_WRITE_U8 (datagram->data + 1, 0x02); // request write operation
+    EC_WRITE_U16(datagram->data + 2, fsm->word_offset);
+    memset(datagram->data + 4, 0x00, 2);
+    memcpy(datagram->data + 6, fsm->value, 2);
+
+#ifdef SII_DEBUG
+    EC_SLAVE_DBG(fsm->slave, 0, "writing SII data:\n");
+    ec_print_data(datagram->data, 8);
+#endif
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_sii_state_write_check;
+}
+
+/****************************************************************************/
+
+/**
+   SII state: WRITE CHECK.
+*/
+
+void ec_fsm_sii_state_write_check(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed to receive SII write datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Reception of SII write datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    fsm->jiffies_start = datagram->jiffies_sent;
+    fsm->check_once_more = 1;
+
+    // issue check datagram
+    ec_datagram_fprd(datagram, fsm->slave->station_address, 0x502, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_sii_state_write_check2;
+}
+
+/****************************************************************************/
+
+/**
+   SII state: WRITE CHECK 2.
+*/
+
+void ec_fsm_sii_state_write_check2(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    unsigned long diff_ms;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave,
+                "Failed to receive SII write check datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->state = ec_fsm_sii_state_error;
+        EC_SLAVE_ERR(fsm->slave,
+                "Reception of SII write check datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+#ifdef SII_DEBUG
+    EC_SLAVE_DBG(fsm->slave, 0, "checking SII write state:\n");
+    ec_print_data(datagram->data, 2);
+#endif
+
+    if (EC_READ_U8(datagram->data + 1) & 0x20) {
+        EC_SLAVE_ERR(fsm->slave, "SII: Error on last SII command!\n");
+        fsm->state = ec_fsm_sii_state_error;
+        return;
+    }
+
+    /* FIXME: some slaves never answer with the busy flag set...
+     * wait a few ms for the write operation to complete. */
+    diff_ms = (datagram->jiffies_received - fsm->jiffies_start) * 1000 / HZ;
+    if (diff_ms < SII_INHIBIT) {
+#ifdef SII_DEBUG
+        EC_SLAVE_DBG(fsm->slave, 0, "too early.\n");
+#endif
+        // issue check datagram again
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    if (EC_READ_U8(datagram->data + 1) & 0x82) { /* busy bit or
+                                                    write operation busy bit */
+        // still busy... timeout?
+        if (diff_ms >= SII_TIMEOUT) {
+            if (fsm->check_once_more) {
+                fsm->check_once_more = 0;
+            } else {
+                EC_SLAVE_ERR(fsm->slave, "SII: Write timeout.\n");
+                fsm->state = ec_fsm_sii_state_error;
+                return;
+            }
+        }
+
+        // issue check datagram again
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    if (EC_READ_U8(datagram->data + 1) & 0x40) {
+        EC_SLAVE_ERR(fsm->slave, "SII: Write operation failed!\n");
+        fsm->state = ec_fsm_sii_state_error;
+        return;
+    }
+
+    // success
+    fsm->state = ec_fsm_sii_state_end;
+}
+
+/****************************************************************************/
+
+/**
+   State: ERROR.
+*/
+
+void ec_fsm_sii_state_error(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+}
+
+/****************************************************************************/
+
+/**
+   State: END.
+*/
+
+void ec_fsm_sii_state_end(
+        ec_fsm_sii_t *fsm /**< finite state machine */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_sii.h b/drivers/net/ethercat/master/fsm_sii.h
new file mode 100644
index 000000000000..405e28502492
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_sii.h
@@ -0,0 +1,82 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave information interface FSM structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_SII_H__
+#define __EC_FSM_SII_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "slave.h"
+
+/****************************************************************************/
+
+/** SII access addressing mode.
+ */
+typedef enum {
+    EC_FSM_SII_USE_INCREMENT_ADDRESS, /**< Use auto-increment addressing. */
+    EC_FSM_SII_USE_CONFIGURED_ADDRESS /**< Use configured addresses. */
+} ec_fsm_sii_addressing_t;
+
+/****************************************************************************/
+
+typedef struct ec_fsm_sii ec_fsm_sii_t; /**< \see ec_fsm_sii */
+
+/**
+   Slave information interface FSM.
+*/
+
+struct ec_fsm_sii
+{
+    ec_slave_t *slave; /**< slave the FSM runs on */
+    ec_datagram_t *datagram; /**< datagram used in the state machine */
+    unsigned int retries; /**< retries upon datagram timeout */
+
+    void (*state)(ec_fsm_sii_t *); /**< SII state function */
+    uint16_t word_offset; /**< input: word offset in SII */
+    ec_fsm_sii_addressing_t mode; /**< reading via APRD or NPRD */
+    uint8_t value[4]; /**< raw SII value (32bit) */
+    unsigned long jiffies_start; /**< Start timestamp. */
+    uint8_t check_once_more; /**< one more try after timeout */
+};
+
+/****************************************************************************/
+
+void ec_fsm_sii_init(ec_fsm_sii_t *, ec_datagram_t *);
+void ec_fsm_sii_clear(ec_fsm_sii_t *);
+
+void ec_fsm_sii_read(ec_fsm_sii_t *, ec_slave_t *,
+                     uint16_t, ec_fsm_sii_addressing_t);
+void ec_fsm_sii_write(ec_fsm_sii_t *, ec_slave_t *, uint16_t,
+        const uint16_t *, ec_fsm_sii_addressing_t);
+
+int ec_fsm_sii_exec(ec_fsm_sii_t *);
+int ec_fsm_sii_success(ec_fsm_sii_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_slave.c b/drivers/net/ethercat/master/fsm_slave.c
new file mode 100644
index 000000000000..6e1f9c369965
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_slave.c
@@ -0,0 +1,685 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT slave (SDO) state machine.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "slave_config.h"
+
+#include "fsm_slave.h"
+
+/****************************************************************************/
+
+void ec_fsm_slave_state_idle(ec_fsm_slave_t *, ec_datagram_t *);
+void ec_fsm_slave_state_ready(ec_fsm_slave_t *, ec_datagram_t *);
+int ec_fsm_slave_action_process_sdo(ec_fsm_slave_t *, ec_datagram_t *);
+void ec_fsm_slave_state_sdo_request(ec_fsm_slave_t *, ec_datagram_t *);
+int ec_fsm_slave_action_process_reg(ec_fsm_slave_t *, ec_datagram_t *);
+void ec_fsm_slave_state_reg_request(ec_fsm_slave_t *, ec_datagram_t *);
+int ec_fsm_slave_action_process_foe(ec_fsm_slave_t *, ec_datagram_t *);
+void ec_fsm_slave_state_foe_request(ec_fsm_slave_t *, ec_datagram_t *);
+int ec_fsm_slave_action_process_soe(ec_fsm_slave_t *, ec_datagram_t *);
+void ec_fsm_slave_state_soe_request(ec_fsm_slave_t *, ec_datagram_t *);
+#ifdef EC_EOE
+int ec_fsm_slave_action_process_eoe(ec_fsm_slave_t *, ec_datagram_t *);
+void ec_fsm_slave_state_eoe_request(ec_fsm_slave_t *, ec_datagram_t *);
+#endif
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_slave_init(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_slave_t *slave /**< EtherCAT slave. */
+        )
+{
+    fsm->slave = slave;
+    INIT_LIST_HEAD(&fsm->list); // mark as unlisted
+
+    fsm->state = ec_fsm_slave_state_idle;
+    fsm->datagram = NULL;
+    fsm->sdo_request = NULL;
+    fsm->reg_request = NULL;
+    fsm->foe_request = NULL;
+    fsm->soe_request = NULL;
+#ifdef EC_EOE
+    fsm->eoe_request = NULL;
+#endif
+
+    // Init sub-state-machines
+    ec_fsm_coe_init(&fsm->fsm_coe);
+    ec_fsm_foe_init(&fsm->fsm_foe);
+    ec_fsm_soe_init(&fsm->fsm_soe);
+#ifdef EC_EOE
+    ec_fsm_eoe_init(&fsm->fsm_eoe);
+#endif
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_slave_clear(
+        ec_fsm_slave_t *fsm /**< Master state machine. */
+        )
+{
+    // signal requests that are currently in operation
+
+    if (fsm->sdo_request) {
+        fsm->sdo_request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&fsm->slave->master->request_queue);
+    }
+
+    if (fsm->reg_request) {
+        fsm->reg_request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&fsm->slave->master->request_queue);
+    }
+
+    if (fsm->foe_request) {
+        fsm->foe_request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&fsm->slave->master->request_queue);
+    }
+
+    if (fsm->soe_request) {
+        fsm->soe_request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&fsm->slave->master->request_queue);
+    }
+
+#ifdef EC_EOE
+    if (fsm->eoe_request) {
+        fsm->soe_request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&fsm->slave->master->request_queue);
+    }
+#endif
+
+    // clear sub-state machines
+    ec_fsm_coe_clear(&fsm->fsm_coe);
+    ec_fsm_foe_clear(&fsm->fsm_foe);
+    ec_fsm_soe_clear(&fsm->fsm_soe);
+#ifdef EC_EOE
+    ec_fsm_eoe_clear(&fsm->fsm_eoe);
+#endif
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * \return 1 if \a datagram was used, else 0.
+ */
+int ec_fsm_slave_exec(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< New datagram to use. */
+        )
+{
+    int datagram_used;
+
+    fsm->state(fsm, datagram);
+
+    datagram_used = fsm->state != ec_fsm_slave_state_idle &&
+        fsm->state != ec_fsm_slave_state_ready;
+
+    if (datagram_used) {
+        fsm->datagram = datagram;
+    } else {
+        fsm->datagram = NULL;
+    }
+
+    return datagram_used;
+}
+
+/****************************************************************************/
+
+/** Sets the current state of the state machine to READY
+ */
+void ec_fsm_slave_set_ready(
+        ec_fsm_slave_t *fsm /**< Slave state machine. */
+        )
+{
+    if (fsm->state == ec_fsm_slave_state_idle) {
+        EC_SLAVE_DBG(fsm->slave, 1, "Ready for requests.\n");
+        fsm->state = ec_fsm_slave_state_ready;
+    }
+}
+
+/****************************************************************************/
+
+/** Returns, if the FSM is currently not busy and ready to execute.
+ *
+ * \return Non-zero if ready.
+ */
+int ec_fsm_slave_is_ready(
+        const ec_fsm_slave_t *fsm /**< Slave state machine. */
+        )
+{
+    return fsm->state == ec_fsm_slave_state_ready;
+}
+
+/*****************************************************************************
+ * Slave state machine
+ ****************************************************************************/
+
+/** Slave state: IDLE.
+ */
+void ec_fsm_slave_state_idle(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    // do nothing
+}
+
+/****************************************************************************/
+
+/** Slave state: READY.
+ */
+void ec_fsm_slave_state_ready(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    // Check for pending external SDO requests
+    if (ec_fsm_slave_action_process_sdo(fsm, datagram)) {
+        return;
+    }
+
+    // Check for pending external register requests
+    if (ec_fsm_slave_action_process_reg(fsm, datagram)) {
+        return;
+    }
+
+    // Check for pending FoE requests
+    if (ec_fsm_slave_action_process_foe(fsm, datagram)) {
+        return;
+    }
+
+    // Check for pending SoE requests
+    if (ec_fsm_slave_action_process_soe(fsm, datagram)) {
+        return;
+    }
+
+#ifdef EC_EOE
+    // Check for pending EoE IP parameter requests
+    if (ec_fsm_slave_action_process_eoe(fsm, datagram)) {
+        return;
+    }
+#endif
+}
+
+/****************************************************************************/
+
+/** Check for pending SDO requests and process one.
+ *
+ * \return non-zero, if an SDO request is processed.
+ */
+int ec_fsm_slave_action_process_sdo(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_request_t *request;
+
+    if (list_empty(&slave->sdo_requests)) {
+        return 0;
+    }
+
+    // take the first request to be processed
+    request = list_entry(slave->sdo_requests.next, ec_sdo_request_t, list);
+    list_del_init(&request->list); // dequeue
+
+    if (slave->current_state & EC_SLAVE_STATE_ACK_ERR) {
+        EC_SLAVE_WARN(slave, "Aborting SDO request,"
+                " slave has error flag set.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    if (slave->current_state == EC_SLAVE_STATE_INIT) {
+        EC_SLAVE_WARN(slave, "Aborting SDO request, slave is in INIT.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    fsm->sdo_request = request;
+    request->state = EC_INT_REQUEST_BUSY;
+
+    // Found pending SDO request. Execute it!
+    EC_SLAVE_DBG(slave, 1, "Processing SDO request...\n");
+
+    // Start SDO transfer
+    fsm->state = ec_fsm_slave_state_sdo_request;
+    ec_fsm_coe_transfer(&fsm->fsm_coe, slave, request);
+    ec_fsm_coe_exec(&fsm->fsm_coe, datagram); // execute immediately
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Slave state: SDO_REQUEST.
+ */
+void ec_fsm_slave_state_sdo_request(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_sdo_request_t *request = fsm->sdo_request;
+
+    if (ec_fsm_coe_exec(&fsm->fsm_coe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(&fsm->fsm_coe)) {
+        EC_SLAVE_ERR(slave, "Failed to process SDO request.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->sdo_request = NULL;
+        fsm->state = ec_fsm_slave_state_ready;
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Finished SDO request.\n");
+
+    // SDO request finished
+    request->state = EC_INT_REQUEST_SUCCESS;
+    wake_up_all(&slave->master->request_queue);
+    fsm->sdo_request = NULL;
+    fsm->state = ec_fsm_slave_state_ready;
+}
+
+/****************************************************************************/
+
+/** Check for pending register requests and process one.
+ *
+ * \return non-zero, if a register request is processed.
+ */
+int ec_fsm_slave_action_process_reg(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_reg_request_t *reg;
+
+    fsm->reg_request = NULL;
+
+    if (slave->config) {
+        // search the first internal register request to be processed
+        list_for_each_entry(reg, &slave->config->reg_requests, list) {
+            if (reg->state == EC_INT_REQUEST_QUEUED) {
+                fsm->reg_request = reg;
+                break;
+            }
+        }
+    }
+
+    if (!fsm->reg_request && !list_empty(&slave->reg_requests)) {
+        // take the first external request to be processed
+        fsm->reg_request =
+            list_entry(slave->reg_requests.next, ec_reg_request_t, list);
+        list_del_init(&fsm->reg_request->list); // dequeue
+    }
+
+    if (!fsm->reg_request) { // no register request to process
+        return 0;
+    }
+
+    if (slave->current_state & EC_SLAVE_STATE_ACK_ERR) {
+        EC_SLAVE_WARN(slave, "Aborting register request,"
+                " slave has error flag set.\n");
+        fsm->reg_request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->reg_request = NULL;
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    // Found pending register request. Execute it!
+    EC_SLAVE_DBG(slave, 1, "Processing register request...\n");
+
+    fsm->reg_request->state = EC_INT_REQUEST_BUSY;
+
+    // Start register access
+    if (fsm->reg_request->dir == EC_DIR_INPUT) {
+        ec_datagram_fprd(datagram, slave->station_address,
+                fsm->reg_request->address, fsm->reg_request->transfer_size);
+        ec_datagram_zero(datagram);
+    } else {
+        ec_datagram_fpwr(datagram, slave->station_address,
+                fsm->reg_request->address, fsm->reg_request->transfer_size);
+        memcpy(datagram->data, fsm->reg_request->data,
+                fsm->reg_request->transfer_size);
+    }
+    datagram->device_index = slave->device_index;
+    fsm->state = ec_fsm_slave_state_reg_request;
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Slave state: Register request.
+ */
+void ec_fsm_slave_state_reg_request(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_reg_request_t *reg = fsm->reg_request;
+
+    if (!reg) {
+        // configuration was cleared in the meantime
+        fsm->state = ec_fsm_slave_state_ready;
+        fsm->reg_request = NULL;
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_ERR(slave, "Failed to receive register"
+                " request datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        reg->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->reg_request = NULL;
+        fsm->state = ec_fsm_slave_state_ready;
+        return;
+    }
+
+    if (fsm->datagram->working_counter == 1) {
+        if (reg->dir == EC_DIR_INPUT) { // read request
+            memcpy(reg->data, fsm->datagram->data, reg->transfer_size);
+        }
+
+        reg->state = EC_INT_REQUEST_SUCCESS;
+        EC_SLAVE_DBG(slave, 1, "Register request successful.\n");
+    } else {
+        reg->state = EC_INT_REQUEST_FAILURE;
+        ec_datagram_print_state(fsm->datagram);
+        EC_SLAVE_ERR(slave, "Register request failed"
+                " (working counter is %u).\n",
+                fsm->datagram->working_counter);
+    }
+
+    wake_up_all(&slave->master->request_queue);
+    fsm->reg_request = NULL;
+    fsm->state = ec_fsm_slave_state_ready;
+}
+
+/****************************************************************************/
+
+/** Check for pending FoE requests and process one.
+ *
+ * \return non-zero, if an FoE request is processed.
+ */
+int ec_fsm_slave_action_process_foe(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_foe_request_t *request;
+
+    if (list_empty(&slave->foe_requests)) {
+        return 0;
+    }
+
+    // take the first request to be processed
+    request = list_entry(slave->foe_requests.next, ec_foe_request_t, list);
+    list_del_init(&request->list); // dequeue
+
+    if (slave->current_state & EC_SLAVE_STATE_ACK_ERR) {
+        EC_SLAVE_WARN(slave, "Aborting FoE request,"
+                " slave has error flag set.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    request->state = EC_INT_REQUEST_BUSY;
+    fsm->foe_request = request;
+
+    EC_SLAVE_DBG(slave, 1, "Processing FoE request.\n");
+
+    fsm->state = ec_fsm_slave_state_foe_request;
+    ec_fsm_foe_transfer(&fsm->fsm_foe, slave, request);
+    ec_fsm_foe_exec(&fsm->fsm_foe, datagram);
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Slave state: FOE REQUEST.
+ */
+void ec_fsm_slave_state_foe_request(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_foe_request_t *request = fsm->foe_request;
+
+    if (ec_fsm_foe_exec(&fsm->fsm_foe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_foe_success(&fsm->fsm_foe)) {
+        EC_SLAVE_ERR(slave, "Failed to handle FoE request.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->foe_request = NULL;
+        fsm->state = ec_fsm_slave_state_ready;
+        return;
+    }
+
+    // finished transferring FoE
+    EC_SLAVE_DBG(slave, 1, "Successfully transferred %zu bytes of FoE"
+            " data.\n", request->data_size);
+
+    request->state = EC_INT_REQUEST_SUCCESS;
+    wake_up_all(&slave->master->request_queue);
+    fsm->foe_request = NULL;
+    fsm->state = ec_fsm_slave_state_ready;
+}
+
+/****************************************************************************/
+
+/** Check for pending SoE requests and process one.
+ *
+ * \return non-zero, if a request is processed.
+ */
+int ec_fsm_slave_action_process_soe(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_soe_request_t *req;
+
+    if (list_empty(&slave->soe_requests)) {
+        return 0;
+    }
+
+    // take the first request to be processed
+    req = list_entry(slave->soe_requests.next, ec_soe_request_t, list);
+    list_del_init(&req->list); // dequeue
+
+    if (slave->current_state & EC_SLAVE_STATE_ACK_ERR) {
+        EC_SLAVE_WARN(slave, "Aborting SoE request,"
+                " slave has error flag set.\n");
+        req->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    if (slave->current_state == EC_SLAVE_STATE_INIT) {
+        EC_SLAVE_WARN(slave, "Aborting SoE request, slave is in INIT.\n");
+        req->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    fsm->soe_request = req;
+    req->state = EC_INT_REQUEST_BUSY;
+
+    // Found pending request. Execute it!
+    EC_SLAVE_DBG(slave, 1, "Processing SoE request...\n");
+
+    // Start SoE transfer
+    fsm->state = ec_fsm_slave_state_soe_request;
+    ec_fsm_soe_transfer(&fsm->fsm_soe, slave, req);
+    ec_fsm_soe_exec(&fsm->fsm_soe, datagram); // execute immediately
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Slave state: SOE_REQUEST.
+ */
+void ec_fsm_slave_state_soe_request(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_soe_request_t *request = fsm->soe_request;
+
+    if (ec_fsm_soe_exec(&fsm->fsm_soe, datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_soe_success(&fsm->fsm_soe)) {
+        EC_SLAVE_ERR(slave, "Failed to process SoE request.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->soe_request = NULL;
+        fsm->state = ec_fsm_slave_state_ready;
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Finished SoE request.\n");
+
+    // SoE request finished
+    request->state = EC_INT_REQUEST_SUCCESS;
+    wake_up_all(&slave->master->request_queue);
+    fsm->soe_request = NULL;
+    fsm->state = ec_fsm_slave_state_ready;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+/** Check for pending EoE IP parameter requests and process one.
+ *
+ * \return non-zero, if a request is processed.
+ */
+int ec_fsm_slave_action_process_eoe(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_eoe_request_t *request;
+
+    if (list_empty(&slave->eoe_requests)) {
+        return 0;
+    }
+
+    // take the first request to be processed
+    request = list_entry(slave->eoe_requests.next, ec_eoe_request_t, list);
+    list_del_init(&request->list); // dequeue
+
+    if (slave->current_state & EC_SLAVE_STATE_ACK_ERR) {
+        EC_SLAVE_WARN(slave, "Aborting EoE request,"
+                " slave has error flag set.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    if (slave->current_state == EC_SLAVE_STATE_INIT) {
+        EC_SLAVE_WARN(slave, "Aborting EoE request, slave is in INIT.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&slave->master->request_queue);
+        fsm->state = ec_fsm_slave_state_idle;
+        return 0;
+    }
+
+    fsm->eoe_request = request;
+    request->state = EC_INT_REQUEST_BUSY;
+
+    // Found pending request. Execute it!
+    EC_SLAVE_DBG(slave, 1, "Processing EoE request...\n");
+
+    // Start EoE command
+    fsm->state = ec_fsm_slave_state_eoe_request;
+    ec_fsm_eoe_set_ip_param(&fsm->fsm_eoe, slave, request);
+    ec_fsm_eoe_exec(&fsm->fsm_eoe, datagram); // execute immediately
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Slave state: EOE_REQUEST.
+ */
+void ec_fsm_slave_state_eoe_request(
+        ec_fsm_slave_t *fsm, /**< Slave state machine. */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_eoe_request_t *req = fsm->eoe_request;
+
+    if (ec_fsm_eoe_exec(&fsm->fsm_eoe, datagram)) {
+        return;
+    }
+
+    if (ec_fsm_eoe_success(&fsm->fsm_eoe)) {
+		req->state = EC_INT_REQUEST_SUCCESS;
+		EC_SLAVE_DBG(slave, 1, "Finished EoE request.\n");
+    }
+	else {
+        req->state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Failed to process EoE request.\n");
+	}
+
+    wake_up_all(&slave->master->request_queue);
+    fsm->eoe_request = NULL;
+    fsm->state = ec_fsm_slave_state_ready;
+}
+
+/****************************************************************************/
+#endif
diff --git a/drivers/net/ethercat/master/fsm_slave.h b/drivers/net/ethercat/master/fsm_slave.h
new file mode 100644
index 000000000000..079bd5f8b473
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_slave.h
@@ -0,0 +1,87 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave request state machine.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_SLAVE_H__
+#define __EC_FSM_SLAVE_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "sdo_request.h"
+#include "reg_request.h"
+#ifdef EC_EOE
+#include "eoe_request.h"
+#endif
+#include "fsm_coe.h"
+#include "fsm_foe.h"
+#include "fsm_soe.h"
+#ifdef EC_EOE
+#include "fsm_eoe.h"
+#endif
+
+/****************************************************************************/
+
+typedef struct ec_fsm_slave ec_fsm_slave_t; /**< \see ec_fsm_slave */
+
+/** Finite state machine of an EtherCAT slave.
+ */
+struct ec_fsm_slave {
+    ec_slave_t *slave; /**< slave the FSM runs on */
+    struct list_head list; /**< Used for execution list. */
+
+    void (*state)(ec_fsm_slave_t *, ec_datagram_t *); /**< State function. */
+    ec_datagram_t *datagram; /**< Previous state datagram. */
+    ec_sdo_request_t *sdo_request; /**< SDO request to process. */
+    ec_reg_request_t *reg_request; /**< Register request to process. */
+    ec_foe_request_t *foe_request; /**< FoE request to process. */
+    off_t foe_index; /**< Index to FoE write request data. */
+    ec_soe_request_t *soe_request; /**< SoE request to process. */
+#ifdef EC_EOE
+    ec_eoe_request_t *eoe_request; /**< SoE request to process. */
+#endif
+
+    ec_fsm_coe_t fsm_coe; /**< CoE state machine. */
+    ec_fsm_foe_t fsm_foe; /**< FoE state machine. */
+    ec_fsm_soe_t fsm_soe; /**< SoE state machine. */
+#ifdef EC_EOE
+    ec_fsm_eoe_t fsm_eoe; /**< EoE state machine. */
+#endif
+};
+
+/****************************************************************************/
+
+void ec_fsm_slave_init(ec_fsm_slave_t *, ec_slave_t *);
+void ec_fsm_slave_clear(ec_fsm_slave_t *);
+
+int ec_fsm_slave_exec(ec_fsm_slave_t *, ec_datagram_t *);
+void ec_fsm_slave_set_ready(ec_fsm_slave_t *);
+int ec_fsm_slave_is_ready(const ec_fsm_slave_t *);
+
+/****************************************************************************/
+
+
+#endif // __EC_FSM_SLAVE_H__
diff --git a/drivers/net/ethercat/master/fsm_slave_config.c b/drivers/net/ethercat/master/fsm_slave_config.c
new file mode 100644
index 000000000000..47e750f46b24
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_slave_config.c
@@ -0,0 +1,1864 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+/** \file
+ *
+ * EtherCAT slave configuration state machine.
+ */
+
+/****************************************************************************/
+
+#include <asm/div64.h>
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "slave_config.h"
+#include "fsm_slave_config.h"
+
+/****************************************************************************/
+
+/** Maximum clock difference (in ns) before going to SAFEOP.
+ *
+ * Wait for DC time difference to drop under this absolute value before
+ * requesting SAFEOP.
+ */
+#define EC_DC_MAX_SYNC_DIFF_NS 10000
+
+/** Maximum time (in ms) to wait for clock discipline.
+ */
+#define EC_DC_SYNC_WAIT_MS 5000
+
+/** Time offset (in ns), that is added to cyclic start time.
+ */
+#define EC_DC_START_OFFSET 100000000ULL
+
+/****************************************************************************/
+
+// prototypes for private methods
+int ec_fsm_slave_config_running(const ec_fsm_slave_config_t *);
+
+/****************************************************************************/
+
+void ec_fsm_slave_config_state_start(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_init(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_clear_fmmus(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_clear_sync(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_dc_clear_assign(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_mbox_sync(ec_fsm_slave_config_t *);
+#ifdef EC_SII_ASSIGN
+void ec_fsm_slave_config_state_assign_pdi(ec_fsm_slave_config_t *);
+#endif
+void ec_fsm_slave_config_state_boot_preop(ec_fsm_slave_config_t *);
+#ifdef EC_SII_ASSIGN
+void ec_fsm_slave_config_state_assign_ethercat(ec_fsm_slave_config_t *);
+#endif
+void ec_fsm_slave_config_state_sdo_conf(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_soe_conf_preop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_eoe_ip_param(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_watchdog_divider(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_watchdog(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_pdo_sync(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_pdo_conf(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_fmmu(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_dc_cycle(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_dc_sync_check(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_dc_start(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_dc_assign(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_wait_safeop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_safeop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_soe_conf_safeop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_op(ec_fsm_slave_config_t *);
+
+void ec_fsm_slave_config_enter_init(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_clear_sync(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_dc_clear_assign(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_mbox_sync(ec_fsm_slave_config_t *);
+#ifdef EC_SII_ASSIGN
+void ec_fsm_slave_config_enter_assign_pdi(ec_fsm_slave_config_t *);
+#endif
+void ec_fsm_slave_config_enter_boot_preop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_sdo_conf(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_soe_conf_preop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_eoe_ip_param(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_pdo_conf(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_watchdog_divider(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_watchdog(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_pdo_sync(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_fmmu(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_dc_cycle(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_wait_safeop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_safeop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_soe_conf_safeop(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_enter_op(ec_fsm_slave_config_t *);
+
+void ec_fsm_slave_config_state_end(ec_fsm_slave_config_t *);
+void ec_fsm_slave_config_state_error(ec_fsm_slave_config_t *);
+
+void ec_fsm_slave_config_reconfigure(ec_fsm_slave_config_t *);
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_slave_config_init(
+        ec_fsm_slave_config_t *fsm, /**< slave state machine */
+        ec_datagram_t *datagram, /**< datagram structure to use */
+        ec_fsm_change_t *fsm_change, /**< State change state machine to use. */
+        ec_fsm_coe_t *fsm_coe, /**< CoE state machine to use. */
+        ec_fsm_soe_t *fsm_soe, /**< SoE state machine to use. */
+        ec_fsm_pdo_t *fsm_pdo, /**< PDO configuration state machine to use. */
+        ec_fsm_eoe_t *fsm_eoe /**< EoE state machine to use. */
+        )
+{
+    ec_sdo_request_init(&fsm->request_copy);
+    ec_soe_request_init(&fsm->soe_request_copy);
+
+    fsm->datagram = datagram;
+    fsm->fsm_change = fsm_change;
+    fsm->fsm_coe = fsm_coe;
+    fsm->fsm_soe = fsm_soe;
+    fsm->fsm_pdo = fsm_pdo;
+    fsm->fsm_eoe = fsm_eoe;
+
+    fsm->wait_ms = 0;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_slave_config_clear(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_sdo_request_clear(&fsm->request_copy);
+    ec_soe_request_clear(&fsm->soe_request_copy);
+}
+
+/****************************************************************************/
+
+/** Start slave configuration state machine.
+ */
+void ec_fsm_slave_config_start(
+        ec_fsm_slave_config_t *fsm, /**< slave state machine */
+        ec_slave_t *slave /**< slave to configure */
+        )
+{
+    fsm->slave = slave;
+    fsm->state = ec_fsm_slave_config_state_start;
+}
+
+/****************************************************************************/
+
+/**
+ * \return false, if state machine has terminated
+ */
+int ec_fsm_slave_config_running(
+        const ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    return fsm->state != ec_fsm_slave_config_state_end
+        && fsm->state != ec_fsm_slave_config_state_error;
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * If the state machine's datagram is not sent or received yet, the execution
+ * of the state machine is delayed to the next cycle.
+ *
+ * \return false, if state machine has terminated
+ */
+int ec_fsm_slave_config_exec(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    if (fsm->datagram->state == EC_DATAGRAM_SENT
+        || fsm->datagram->state == EC_DATAGRAM_QUEUED) {
+        // datagram was not sent or received yet.
+        return ec_fsm_slave_config_running(fsm);
+    }
+
+    fsm->state(fsm);
+    return ec_fsm_slave_config_running(fsm);
+}
+
+/****************************************************************************/
+
+/**
+ * \return true, if the state machine terminated gracefully
+ */
+int ec_fsm_slave_config_success(
+        const ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    return fsm->state == ec_fsm_slave_config_state_end;
+}
+
+/*****************************************************************************
+ * Slave configuration state machine
+ ****************************************************************************/
+
+/** Slave configuration state: START.
+ */
+void ec_fsm_slave_config_state_start(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    EC_SLAVE_DBG(fsm->slave, 1, "Configuring...\n");
+    ec_fsm_slave_config_enter_init(fsm);
+}
+
+/****************************************************************************/
+
+/** Start state change to INIT.
+ */
+void ec_fsm_slave_config_enter_init(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_fsm_change_start(fsm->fsm_change, fsm->slave, EC_SLAVE_STATE_INIT);
+    ec_fsm_change_exec(fsm->fsm_change);
+    fsm->state = ec_fsm_slave_config_state_init;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: INIT.
+ */
+void ec_fsm_slave_config_state_init(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (ec_fsm_change_exec(fsm->fsm_change)) return;
+
+    if (!ec_fsm_change_success(fsm->fsm_change)) {
+        if (!fsm->fsm_change->spontaneous_change)
+            slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Now in INIT.\n");
+
+    if (!slave->base_fmmu_count) { // skip FMMU configuration
+        ec_fsm_slave_config_enter_clear_sync(fsm);
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Clearing FMMU configurations...\n");
+
+    // clear FMMU configurations
+    ec_datagram_fpwr(datagram, slave->station_address,
+            0x0600, EC_FMMU_PAGE_SIZE * slave->base_fmmu_count);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_clear_fmmus;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: CLEAR FMMU.
+ */
+void ec_fsm_slave_config_state_clear_fmmus(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed receive FMMU clearing datagram.\n");
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed to clear FMMUs: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    ec_fsm_slave_config_enter_clear_sync(fsm);
+}
+
+/****************************************************************************/
+
+/** Clear the sync manager configurations.
+ */
+void ec_fsm_slave_config_enter_clear_sync(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+    size_t sync_size;
+
+    if (!slave->base_sync_count) {
+        // no sync managers
+        ec_fsm_slave_config_enter_dc_clear_assign(fsm);
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Clearing sync manager configurations...\n");
+
+    sync_size = EC_SYNC_PAGE_SIZE * slave->base_sync_count;
+
+    // clear sync manager configurations
+    ec_datagram_fpwr(datagram, slave->station_address, 0x0800, sync_size);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_clear_sync;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: CLEAR SYNC.
+ */
+void ec_fsm_slave_config_state_clear_sync(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed receive sync manager"
+                " clearing datagram.\n");
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(fsm->slave,
+                "Failed to clear sync manager configurations: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    ec_fsm_slave_config_enter_dc_clear_assign(fsm);
+}
+
+/****************************************************************************/
+
+/** Clear the DC assignment.
+ */
+void ec_fsm_slave_config_enter_dc_clear_assign(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (!slave->base_dc_supported || !slave->has_dc_system_time) {
+        ec_fsm_slave_config_enter_mbox_sync(fsm);
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Clearing DC assignment...\n");
+
+    ec_datagram_fpwr(datagram, slave->station_address, 0x0980, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_dc_clear_assign;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: CLEAR DC ASSIGN.
+ */
+void ec_fsm_slave_config_state_dc_clear_assign(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed receive DC assignment"
+                " clearing datagram.\n");
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        // clearing the DC assignment does not succeed on simple slaves
+        EC_SLAVE_DBG(fsm->slave, 1, "Failed to clear DC assignment: ");
+        ec_datagram_print_wc_error(datagram);
+    }
+
+    ec_fsm_slave_config_enter_mbox_sync(fsm);
+}
+
+/****************************************************************************/
+
+/** Check for mailbox sync managers to be configured.
+ */
+void ec_fsm_slave_config_enter_mbox_sync(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+    unsigned int i;
+
+    // slave is now in INIT
+    if (slave->current_state == slave->requested_state) {
+        fsm->state = ec_fsm_slave_config_state_end; // successful
+        EC_SLAVE_DBG(slave, 1, "Finished configuration.\n");
+        return;
+    }
+
+    if (!slave->sii.mailbox_protocols) {
+        // no mailbox protocols supported
+        EC_SLAVE_DBG(slave, 1, "Slave does not support"
+                " mailbox communication.\n");
+#ifdef EC_SII_ASSIGN
+        ec_fsm_slave_config_enter_assign_pdi(fsm);
+#else
+        ec_fsm_slave_config_enter_boot_preop(fsm);
+#endif
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Configuring mailbox sync managers...\n");
+
+    if (slave->requested_state == EC_SLAVE_STATE_BOOT) {
+        ec_sync_t sync;
+
+        ec_datagram_fpwr(datagram, slave->station_address, 0x0800,
+                EC_SYNC_PAGE_SIZE * 2);
+        ec_datagram_zero(datagram);
+
+        ec_sync_init(&sync, slave);
+        sync.physical_start_address = slave->sii.boot_rx_mailbox_offset;
+        sync.control_register = 0x26;
+        sync.enable = 1;
+        ec_sync_page(&sync, 0, slave->sii.boot_rx_mailbox_size,
+                EC_DIR_INVALID, // use default direction
+                0, // no PDO xfer
+                datagram->data);
+        slave->configured_rx_mailbox_offset =
+            slave->sii.boot_rx_mailbox_offset;
+        slave->configured_rx_mailbox_size =
+            slave->sii.boot_rx_mailbox_size;
+
+        ec_sync_init(&sync, slave);
+        sync.physical_start_address = slave->sii.boot_tx_mailbox_offset;
+        sync.control_register = 0x22;
+        sync.enable = 1;
+        ec_sync_page(&sync, 1, slave->sii.boot_tx_mailbox_size,
+                EC_DIR_INVALID, // use default direction
+                0, // no PDO xfer
+                datagram->data + EC_SYNC_PAGE_SIZE);
+        slave->configured_tx_mailbox_offset =
+            slave->sii.boot_tx_mailbox_offset;
+        slave->configured_tx_mailbox_size =
+            slave->sii.boot_tx_mailbox_size;
+
+    } else if (slave->sii.sync_count >= 2) { // mailbox configuration provided
+        ec_datagram_fpwr(datagram, slave->station_address, 0x0800,
+                EC_SYNC_PAGE_SIZE * slave->sii.sync_count);
+        ec_datagram_zero(datagram);
+
+        for (i = 0; i < 2; i++) {
+            ec_sync_page(&slave->sii.syncs[i], i,
+                    slave->sii.syncs[i].default_length,
+                    NULL, // use default sync manager configuration
+                    0, // no PDO xfer
+                    datagram->data + EC_SYNC_PAGE_SIZE * i);
+        }
+
+        slave->configured_rx_mailbox_offset =
+            slave->sii.syncs[0].physical_start_address;
+        slave->configured_rx_mailbox_size =
+            slave->sii.syncs[0].default_length;
+        slave->configured_tx_mailbox_offset =
+            slave->sii.syncs[1].physical_start_address;
+        slave->configured_tx_mailbox_size =
+            slave->sii.syncs[1].default_length;
+    } else { // no mailbox sync manager configurations provided
+        ec_sync_t sync;
+
+        EC_SLAVE_DBG(slave, 1, "Slave does not provide"
+                " mailbox sync manager configurations.\n");
+
+        ec_datagram_fpwr(datagram, slave->station_address, 0x0800,
+                EC_SYNC_PAGE_SIZE * 2);
+        ec_datagram_zero(datagram);
+
+        ec_sync_init(&sync, slave);
+        sync.physical_start_address = slave->sii.std_rx_mailbox_offset;
+        sync.control_register = 0x26;
+        sync.enable = 1;
+        ec_sync_page(&sync, 0, slave->sii.std_rx_mailbox_size,
+                NULL, // use default sync manager configuration
+                0, // no PDO xfer
+                datagram->data);
+        slave->configured_rx_mailbox_offset =
+            slave->sii.std_rx_mailbox_offset;
+        slave->configured_rx_mailbox_size =
+            slave->sii.std_rx_mailbox_size;
+
+        ec_sync_init(&sync, slave);
+        sync.physical_start_address = slave->sii.std_tx_mailbox_offset;
+        sync.control_register = 0x22;
+        sync.enable = 1;
+        ec_sync_page(&sync, 1, slave->sii.std_tx_mailbox_size,
+                NULL, // use default sync manager configuration
+                0, // no PDO xfer
+                datagram->data + EC_SYNC_PAGE_SIZE);
+        slave->configured_tx_mailbox_offset =
+            slave->sii.std_tx_mailbox_offset;
+        slave->configured_tx_mailbox_size =
+            slave->sii.std_tx_mailbox_size;
+    }
+
+    fsm->take_time = 1;
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_mbox_sync;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: SYNC.
+ *
+ * \todo Timeout for response.
+ */
+void ec_fsm_slave_config_state_mbox_sync(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive sync manager"
+                " configuration datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (fsm->take_time) {
+        fsm->take_time = 0;
+        fsm->jiffies_start = datagram->jiffies_sent;
+    }
+
+    /* Because the sync manager configurations are cleared during the last
+     * cycle, some slaves do not immediately respond to the mailbox sync
+     * manager configuration datagram. Therefore, resend the datagram for
+     * a certain time, if the slave does not respond.
+     */
+    if (datagram->working_counter == 0) {
+        unsigned long diff = datagram->jiffies_received - fsm->jiffies_start;
+
+        if (diff >= HZ) {
+            slave->error_flag = 1;
+            fsm->state = ec_fsm_slave_config_state_error;
+            EC_SLAVE_ERR(slave, "Timeout while configuring"
+                    " mailbox sync managers.\n");
+            return;
+        } else {
+            EC_SLAVE_DBG(slave, 1, "Resending after %u ms...\n",
+                    (unsigned int) diff * 1000 / HZ);
+        }
+
+        // send configuration datagram again
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+    else if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to set sync managers: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+#ifdef EC_SII_ASSIGN
+    ec_fsm_slave_config_enter_assign_pdi(fsm);
+#else
+    ec_fsm_slave_config_enter_boot_preop(fsm);
+#endif
+}
+
+/****************************************************************************/
+
+#ifdef EC_SII_ASSIGN
+
+/** Assign SII to PDI.
+ */
+void ec_fsm_slave_config_enter_assign_pdi(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->slave->requested_state != EC_SLAVE_STATE_BOOT) {
+        EC_SLAVE_DBG(slave, 1, "Assigning SII access to PDI.\n");
+
+        ec_datagram_fpwr(datagram, slave->station_address, 0x0500, 0x01);
+        EC_WRITE_U8(datagram->data, 0x01); // PDI
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_slave_config_state_assign_pdi;
+    }
+    else {
+        ec_fsm_slave_config_enter_boot_preop(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: ASSIGN_PDI.
+ */
+void ec_fsm_slave_config_state_assign_pdi(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        return;
+    }
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_WARN(slave, "Failed receive SII assignment datagram: ");
+        ec_datagram_print_state(datagram);
+        goto cont_preop;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_WARN(slave, "Failed to assign SII to PDI: ");
+        ec_datagram_print_wc_error(datagram);
+    }
+
+cont_preop:
+    ec_fsm_slave_config_enter_boot_preop(fsm);
+}
+
+#endif
+
+/****************************************************************************/
+
+/** Request PREOP state.
+ */
+void ec_fsm_slave_config_enter_boot_preop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    fsm->state = ec_fsm_slave_config_state_boot_preop;
+
+    if (fsm->slave->requested_state != EC_SLAVE_STATE_BOOT) {
+        ec_fsm_change_start(fsm->fsm_change,
+                fsm->slave, EC_SLAVE_STATE_PREOP);
+    } else { // BOOT
+        ec_fsm_change_start(fsm->fsm_change,
+                fsm->slave, EC_SLAVE_STATE_BOOT);
+    }
+
+    ec_fsm_change_exec(fsm->fsm_change); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: BOOT/PREOP.
+ */
+void ec_fsm_slave_config_state_boot_preop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+#ifdef EC_SII_ASSIGN
+    int assign_to_pdi;
+    ec_slave_config_t *config;
+    ec_flag_t *flag;
+#endif
+
+    if (ec_fsm_change_exec(fsm->fsm_change)) {
+        return;
+    }
+
+    if (!ec_fsm_change_success(fsm->fsm_change)) {
+        if (!fsm->fsm_change->spontaneous_change)
+            slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        return;
+    }
+
+    // slave is now in BOOT or PREOP
+    slave->jiffies_preop = fsm->datagram->jiffies_received;
+
+    EC_SLAVE_DBG(slave, 1, "Now in %s.\n",
+            slave->requested_state != EC_SLAVE_STATE_BOOT ? "PREOP" : "BOOT");
+
+#ifdef EC_SII_ASSIGN
+    assign_to_pdi = 0;
+    config = fsm->slave->config;
+    if (config) {
+        flag = ec_slave_config_find_flag(config, "AssignToPdi");
+        if (flag) {
+            assign_to_pdi = flag->value;
+        }
+    }
+
+    if (assign_to_pdi) {
+        EC_SLAVE_DBG(slave, 1, "Skipping SII assignment back to EtherCAT.\n");
+        if (slave->current_state == slave->requested_state) {
+            fsm->state = ec_fsm_slave_config_state_end; // successful
+            EC_SLAVE_DBG(slave, 1, "Finished configuration.\n");
+            return;
+        }
+
+        ec_fsm_slave_config_enter_sdo_conf(fsm);
+    }
+    else {
+        EC_SLAVE_DBG(slave, 1, "Assigning SII access back to EtherCAT.\n");
+
+        ec_datagram_fpwr(fsm->datagram, slave->station_address, 0x0500, 0x01);
+        EC_WRITE_U8(fsm->datagram->data, 0x00); // EtherCAT
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_slave_config_state_assign_ethercat;
+    }
+#else
+    if (slave->current_state == slave->requested_state) {
+        fsm->state = ec_fsm_slave_config_state_end; // successful
+        EC_SLAVE_DBG(slave, 1, "Finished configuration.\n");
+        return;
+    }
+
+    ec_fsm_slave_config_enter_sdo_conf(fsm);
+#endif
+}
+
+/****************************************************************************/
+
+#ifdef EC_SII_ASSIGN
+
+/** Slave configuration state: ASSIGN_ETHERCAT.
+ */
+void ec_fsm_slave_config_state_assign_ethercat(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        return;
+    }
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_WARN(slave, "Failed receive SII assignment datagram: ");
+        ec_datagram_print_state(datagram);
+        goto cont_sdo_conf;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_WARN(slave, "Failed to assign SII back to EtherCAT: ");
+        ec_datagram_print_wc_error(datagram);
+    }
+
+cont_sdo_conf:
+    if (slave->current_state == slave->requested_state) {
+        fsm->state = ec_fsm_slave_config_state_end; // successful
+        EC_SLAVE_DBG(slave, 1, "Finished configuration.\n");
+        return;
+    }
+
+    ec_fsm_slave_config_enter_sdo_conf(fsm);
+}
+
+#endif
+
+/****************************************************************************/
+
+/** Check for SDO configurations to be applied.
+ */
+void ec_fsm_slave_config_enter_sdo_conf(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (!slave->config) {
+        ec_fsm_slave_config_enter_pdo_sync(fsm);
+        return;
+    }
+
+    // No CoE configuration to be applied?
+    if (list_empty(&slave->config->sdo_configs)) { // skip SDO configuration
+        ec_fsm_slave_config_enter_soe_conf_preop(fsm);
+        return;
+    }
+
+    // start SDO configuration
+    fsm->state = ec_fsm_slave_config_state_sdo_conf;
+    fsm->request = list_entry(fsm->slave->config->sdo_configs.next,
+            ec_sdo_request_t, list);
+    ec_sdo_request_copy(&fsm->request_copy, fsm->request);
+    ecrt_sdo_request_write(&fsm->request_copy);
+    ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request_copy);
+    ec_fsm_coe_exec(fsm->fsm_coe, fsm->datagram); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: SDO_CONF.
+ */
+void ec_fsm_slave_config_state_sdo_conf(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    if (ec_fsm_coe_exec(fsm->fsm_coe, fsm->datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_coe_success(fsm->fsm_coe)) {
+        EC_SLAVE_ERR(fsm->slave, "SDO configuration failed.\n");
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        return;
+    }
+
+    if (!fsm->slave->config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    // Another SDO to configure?
+    if (fsm->request->list.next != &fsm->slave->config->sdo_configs) {
+        fsm->request = list_entry(fsm->request->list.next,
+                ec_sdo_request_t, list);
+        ec_sdo_request_copy(&fsm->request_copy, fsm->request);
+        ecrt_sdo_request_write(&fsm->request_copy);
+        ec_fsm_coe_transfer(fsm->fsm_coe, fsm->slave, &fsm->request_copy);
+        ec_fsm_coe_exec(fsm->fsm_coe, fsm->datagram); // execute immediately
+        return;
+    }
+
+    // All SDOs are now configured.
+    ec_fsm_slave_config_enter_soe_conf_preop(fsm);
+}
+
+/****************************************************************************/
+
+/** Check for SoE configurations to be applied.
+ */
+void ec_fsm_slave_config_enter_soe_conf_preop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_soe_request_t *req;
+
+    if (!slave->config) {
+        ec_fsm_slave_config_enter_pdo_sync(fsm);
+        return;
+    }
+
+    list_for_each_entry(req, &slave->config->soe_configs, list) {
+        if (req->al_state == EC_AL_STATE_PREOP) {
+            // start SoE configuration
+            fsm->state = ec_fsm_slave_config_state_soe_conf_preop;
+            fsm->soe_request = req;
+            ec_soe_request_copy(&fsm->soe_request_copy, fsm->soe_request);
+            ec_soe_request_write(&fsm->soe_request_copy);
+            ec_fsm_soe_transfer(fsm->fsm_soe, fsm->slave,
+                    &fsm->soe_request_copy);
+            ec_fsm_soe_exec(fsm->fsm_soe, fsm->datagram);
+            return;
+        }
+    }
+
+    // No SoE configuration to be applied in PREOP
+    ec_fsm_slave_config_enter_eoe_ip_param(fsm);
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: SOE_CONF.
+ */
+void ec_fsm_slave_config_state_soe_conf_preop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (ec_fsm_soe_exec(fsm->fsm_soe, fsm->datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_soe_success(fsm->fsm_soe)) {
+        EC_SLAVE_ERR(slave, "SoE configuration failed.\n");
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        return;
+    }
+
+    if (!fsm->slave->config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    // Another IDN to configure in PREOP?
+    while (fsm->soe_request->list.next != &fsm->slave->config->soe_configs) {
+        fsm->soe_request = list_entry(fsm->soe_request->list.next,
+                ec_soe_request_t, list);
+        if (fsm->soe_request->al_state == EC_AL_STATE_PREOP) {
+            ec_soe_request_copy(&fsm->soe_request_copy, fsm->soe_request);
+            ec_soe_request_write(&fsm->soe_request_copy);
+            ec_fsm_soe_transfer(fsm->fsm_soe, fsm->slave,
+                    &fsm->soe_request_copy);
+            ec_fsm_soe_exec(fsm->fsm_soe, fsm->datagram);
+            return;
+        }
+    }
+
+    // All PREOP IDNs are now configured.
+    ec_fsm_slave_config_enter_eoe_ip_param(fsm);
+}
+
+/****************************************************************************/
+
+/** EOE_IP_PARAM entry function.
+ */
+void ec_fsm_slave_config_enter_eoe_ip_param(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+#ifdef EC_EOE
+    ec_slave_t *slave = fsm->slave;
+    ec_eoe_request_t *request = &slave->config->eoe_ip_param_request;
+
+    if (ec_eoe_request_valid(request)) {
+        EC_SLAVE_DBG(slave, 1, "Setting EoE IP parameters...\n");
+
+        // Start EoE command
+        fsm->state = ec_fsm_slave_config_state_eoe_ip_param;
+        ec_fsm_eoe_set_ip_param(fsm->fsm_eoe, slave, request);
+        ec_fsm_eoe_exec(fsm->fsm_eoe, fsm->datagram); // execute immediately
+        return;
+    }
+#endif
+
+    ec_fsm_slave_config_enter_pdo_conf(fsm);
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: EOE_IP_PARAM.
+ */
+void ec_fsm_slave_config_state_eoe_ip_param(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+#ifdef EC_EOE
+    ec_slave_t *slave = fsm->slave;
+
+    if (ec_fsm_eoe_exec(fsm->fsm_eoe, fsm->datagram)) {
+        return;
+    }
+
+    if (ec_fsm_eoe_success(fsm->fsm_eoe)) {
+        EC_SLAVE_DBG(slave, 1, "Finished setting EoE IP parameters.\n");
+    }
+    else {
+        EC_SLAVE_ERR(slave, "Failed to set EoE IP parameters.\n");
+    }
+#endif
+    ec_fsm_slave_config_enter_pdo_conf(fsm);
+}
+
+/****************************************************************************/
+
+/** PDO_CONF entry function.
+ */
+void ec_fsm_slave_config_enter_pdo_conf(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    // Start configuring PDOs
+    ec_fsm_pdo_start_configuration(fsm->fsm_pdo, fsm->slave);
+    fsm->state = ec_fsm_slave_config_state_pdo_conf;
+    fsm->state(fsm); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: PDO_CONF.
+ */
+void ec_fsm_slave_config_state_pdo_conf(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    // TODO check for config here
+
+    if (ec_fsm_pdo_exec(fsm->fsm_pdo, fsm->datagram)) {
+        return;
+    }
+
+    if (!fsm->slave->config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    if (!ec_fsm_pdo_success(fsm->fsm_pdo)) {
+        EC_SLAVE_WARN(fsm->slave, "PDO configuration failed.\n");
+    }
+
+    ec_fsm_slave_config_enter_watchdog_divider(fsm);
+}
+
+/****************************************************************************/
+
+/** WATCHDOG_DIVIDER entry function.
+ */
+void ec_fsm_slave_config_enter_watchdog_divider(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_config_t *config = slave->config;
+
+    if (config && config->watchdog_divider) {
+        EC_SLAVE_DBG(slave, 1, "Setting watchdog divider to %u.\n",
+                config->watchdog_divider);
+
+        ec_datagram_fpwr(datagram, slave->station_address, 0x0400, 2);
+        EC_WRITE_U16(datagram->data, config->watchdog_divider);
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_slave_config_state_watchdog_divider;
+    } else {
+        ec_fsm_slave_config_enter_watchdog(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: WATCHDOG_DIVIDER.
+ */
+void ec_fsm_slave_config_state_watchdog_divider(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive watchdog divider"
+                " configuration datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        EC_SLAVE_WARN(slave, "Failed to set watchdog divider: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    ec_fsm_slave_config_enter_watchdog(fsm);
+}
+
+/****************************************************************************/
+
+/** WATCHDOG entry function
+ */
+void ec_fsm_slave_config_enter_watchdog(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    ec_slave_config_t *config = slave->config;
+
+    if (config && config->watchdog_intervals) {
+        EC_SLAVE_DBG(slave, 1, "Setting process data"
+                " watchdog intervals to %u.\n", config->watchdog_intervals);
+
+        ec_datagram_fpwr(datagram, slave->station_address, 0x0420, 2);
+        EC_WRITE_U16(datagram->data, config->watchdog_intervals);
+
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_slave_config_state_watchdog;
+    } else {
+        ec_fsm_slave_config_enter_pdo_sync(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: WATCHDOG.
+ */
+
+void ec_fsm_slave_config_state_watchdog(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive sync manager"
+                " watchdog configuration datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_WARN(slave, "Failed to set process data"
+                " watchdog intervals: ");
+        ec_datagram_print_wc_error(datagram);
+    }
+
+    ec_fsm_slave_config_enter_pdo_sync(fsm);
+}
+
+/****************************************************************************/
+
+/** Check for PDO sync managers to be configured.
+ */
+void ec_fsm_slave_config_enter_pdo_sync(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+    unsigned int i, j, offset, num_pdo_syncs;
+    uint8_t sync_index;
+    const ec_sync_t *sync;
+    uint16_t size;
+
+    if (slave->sii.mailbox_protocols) {
+        offset = 2; // slave has mailboxes
+    } else {
+        offset = 0;
+    }
+
+    if (slave->sii.sync_count <= offset) {
+        // no PDO sync managers to configure
+        ec_fsm_slave_config_enter_fmmu(fsm);
+        return;
+    }
+
+    num_pdo_syncs = slave->sii.sync_count - offset;
+
+    // configure sync managers for process data
+    ec_datagram_fpwr(datagram, slave->station_address,
+            0x0800 + EC_SYNC_PAGE_SIZE * offset,
+            EC_SYNC_PAGE_SIZE * num_pdo_syncs);
+    ec_datagram_zero(datagram);
+
+    for (i = 0; i < num_pdo_syncs; i++) {
+        const ec_sync_config_t *sync_config;
+        uint8_t pdo_xfer = 0;
+        sync_index = i + offset;
+        sync = &slave->sii.syncs[sync_index];
+
+        if (slave->config) {
+            const ec_slave_config_t *sc = slave->config;
+            sync_config = &sc->sync_configs[sync_index];
+            size = ec_pdo_list_total_size(&sync_config->pdos);
+
+            // determine, if PDOs shall be transferred via this SM
+            // inthat case, enable sync manager in every case
+            for (j = 0; j < sc->used_fmmus; j++) {
+                if (sc->fmmu_configs[j].sync_index == sync_index) {
+                    pdo_xfer = 1;
+                    break;
+                }
+            }
+
+        } else {
+            sync_config = NULL;
+            size = sync->default_length;
+        }
+
+        ec_sync_page(sync, sync_index, size, sync_config, pdo_xfer,
+                datagram->data + EC_SYNC_PAGE_SIZE * i);
+    }
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_pdo_sync;
+}
+
+/****************************************************************************/
+
+/** Configure PDO sync managers.
+ */
+void ec_fsm_slave_config_state_pdo_sync(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive process data sync"
+                " manager configuration datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to set process data sync managers: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    ec_fsm_slave_config_enter_fmmu(fsm);
+}
+
+/****************************************************************************/
+
+/** Check for FMMUs to be configured.
+ */
+void ec_fsm_slave_config_enter_fmmu(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_datagram_t *datagram = fsm->datagram;
+    unsigned int i;
+    const ec_fmmu_config_t *fmmu;
+    const ec_sync_t *sync;
+
+    if (!slave->config) {
+        ec_fsm_slave_config_enter_wait_safeop(fsm);
+        return;
+    }
+
+    if (slave->base_fmmu_count < slave->config->used_fmmus) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Slave has less FMMUs (%u)"
+                " than requested (%u).\n", slave->base_fmmu_count,
+                slave->config->used_fmmus);
+        return;
+    }
+
+    if (!slave->base_fmmu_count) { // skip FMMU configuration
+        ec_fsm_slave_config_enter_dc_cycle(fsm);
+        return;
+    }
+
+    // configure FMMUs
+    ec_datagram_fpwr(datagram, slave->station_address,
+                     0x0600, EC_FMMU_PAGE_SIZE * slave->base_fmmu_count);
+    ec_datagram_zero(datagram);
+    for (i = 0; i < slave->config->used_fmmus; i++) {
+        fmmu = &slave->config->fmmu_configs[i];
+        if (!(sync = ec_slave_get_sync(slave, fmmu->sync_index))) {
+            slave->error_flag = 1;
+            fsm->state = ec_fsm_slave_config_state_error;
+            EC_SLAVE_ERR(slave, "Failed to determine PDO sync manager"
+                    " for FMMU!\n");
+            return;
+        }
+        ec_fmmu_config_page(fmmu, sync,
+                datagram->data + EC_FMMU_PAGE_SIZE * i);
+    }
+
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_fmmu;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: FMMU.
+ */
+void ec_fsm_slave_config_state_fmmu(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive FMMUs datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to set FMMUs: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    ec_fsm_slave_config_enter_dc_cycle(fsm);
+}
+
+/****************************************************************************/
+
+/** Check for DC to be configured.
+ */
+void ec_fsm_slave_config_enter_dc_cycle(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    ec_slave_config_t *config = slave->config;
+
+    if (!config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    if (config->dc_assign_activate) {
+        if (!slave->base_dc_supported || !slave->has_dc_system_time) {
+            EC_SLAVE_WARN(slave, "Slave seems not to support"
+                    " distributed clocks!\n");
+        }
+
+        EC_SLAVE_DBG(slave, 1, "Setting DC cycle times to %u / %u.\n",
+                config->dc_sync[0].cycle_time, config->dc_sync[1].cycle_time);
+
+        // set DC cycle times
+        ec_datagram_fpwr(datagram, slave->station_address, 0x09A0, 8);
+        EC_WRITE_U32(datagram->data, config->dc_sync[0].cycle_time);
+        EC_WRITE_U32(datagram->data + 4, config->dc_sync[1].cycle_time);
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_slave_config_state_dc_cycle;
+    } else {
+        // DC are unused
+        ec_fsm_slave_config_enter_wait_safeop(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: DC CYCLE.
+ */
+void ec_fsm_slave_config_state_dc_cycle(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    ec_slave_config_t *config = slave->config;
+
+    if (!config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive DC cycle times datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to set DC cycle times: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Checking for synchrony.\n");
+
+    fsm->jiffies_start = jiffies;
+    ec_datagram_fprd(datagram, slave->station_address, 0x092c, 4);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_dc_sync_check;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: DC SYNC CHECK.
+ */
+void ec_fsm_slave_config_state_dc_sync_check(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    ec_slave_config_t *config = slave->config;
+    uint32_t abs_sync_diff;
+    unsigned long diff_ms;
+    ec_sync_signal_t *sync0 = &config->dc_sync[0];
+    ec_sync_signal_t *sync1 = &config->dc_sync[1];
+    u64 start_time;
+
+    if (!config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive DC sync check datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to check DC synchrony: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    abs_sync_diff = EC_READ_U32(datagram->data) & 0x7fffffff;
+    diff_ms = (datagram->jiffies_received - fsm->jiffies_start) * 1000 / HZ;
+
+    if (abs_sync_diff > EC_DC_MAX_SYNC_DIFF_NS) {
+
+        if (diff_ms >= EC_DC_SYNC_WAIT_MS) {
+            EC_SLAVE_WARN(slave, "Slave did not sync after %lu ms.\n",
+                    diff_ms);
+        } else {
+            EC_SLAVE_DBG(slave, 1, "Sync after %4lu ms: %10u ns\n",
+                    diff_ms, abs_sync_diff);
+
+            // check synchrony again
+            ec_datagram_fprd(datagram, slave->station_address, 0x092c, 4);
+            fsm->retries = EC_FSM_RETRIES;
+            return;
+        }
+    } else {
+        EC_SLAVE_DBG(slave, 1, "%u ns difference after %lu ms.\n",
+                abs_sync_diff, diff_ms);
+    }
+
+    // set DC start time (roughly in the future, not in-phase)
+    start_time = master->app_time + EC_DC_START_OFFSET; // now + X ns
+
+    if (sync0->cycle_time) {
+        // find correct phase
+        if (master->dc_ref_time) {
+            u64 diff, start;
+            u32 remainder, cycle;
+
+            diff = start_time - master->dc_ref_time;
+            cycle = sync0->cycle_time + sync1->cycle_time;
+            remainder = do_div(diff, cycle);
+
+            start = start_time + cycle - remainder + sync0->shift_time;
+
+            EC_SLAVE_DBG(slave, 1, "   ref_time=%llu\n", master->dc_ref_time);
+            EC_SLAVE_DBG(slave, 1, "   app_time=%llu\n", master->app_time);
+            EC_SLAVE_DBG(slave, 1, " start_time=%llu\n", start_time);
+            EC_SLAVE_DBG(slave, 1, "      cycle=%u\n", cycle);
+            EC_SLAVE_DBG(slave, 1, " shift_time=%i\n", sync0->shift_time);
+            EC_SLAVE_DBG(slave, 1, "  remainder=%u\n", remainder);
+            EC_SLAVE_DBG(slave, 1, "       start=%llu\n", start);
+            start_time = start;
+        } else {
+            EC_SLAVE_WARN(slave, "No application time supplied."
+                    " Cyclic start time will not be in phase.\n");
+        }
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Setting DC cyclic operation"
+            " start time to %llu.\n", start_time);
+
+    ec_datagram_fpwr(datagram, slave->station_address, 0x0990, 8);
+    EC_WRITE_U64(datagram->data, start_time);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_dc_start;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: DC START.
+ */
+void ec_fsm_slave_config_state_dc_start(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    ec_slave_config_t *config = slave->config;
+
+    if (!config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive DC start time datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to set DC start time: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Setting DC AssignActivate to 0x%04x.\n",
+            config->dc_assign_activate);
+
+    // assign sync unit to EtherCAT or PDI
+    ec_datagram_fpwr(datagram, slave->station_address, 0x0980, 2);
+    EC_WRITE_U16(datagram->data, config->dc_assign_activate);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_config_state_dc_assign;
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: DC ASSIGN.
+ */
+void ec_fsm_slave_config_state_dc_assign(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive DC activation datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        EC_SLAVE_ERR(slave, "Failed to activate DC: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    ec_fsm_slave_config_enter_wait_safeop(fsm);
+}
+
+/****************************************************************************/
+
+/** Wait before SAFEOP transition.
+ *
+ * The feature flag WaitBeforeSAFEOPms can be used to add a wait time before
+ * going to SAFEOP. This can be used as a workaround for slaves that need some
+ * extra time for initialisation.
+ */
+void ec_fsm_slave_config_enter_wait_safeop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_config_t *config = fsm->slave->config;
+    fsm->wait_ms = 0UL;
+    if (config) {
+        ec_flag_t *flag = ec_slave_config_find_flag(config,
+                "WaitBeforeSAFEOPms");
+        if (flag && flag->value > 0) {
+            fsm->wait_ms = (unsigned long) flag->value;
+        }
+    }
+
+    if (fsm->wait_ms > 0) {
+        fsm->state = ec_fsm_slave_config_state_wait_safeop;
+
+        /* dummy read */
+        ec_datagram_fprd(fsm->datagram, fsm->slave->station_address,
+                0x0600, 1);
+
+        fsm->jiffies_start = jiffies;
+    }
+    else {
+        ec_fsm_slave_config_enter_safeop(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: WAIT SAFEOP.
+ */
+void ec_fsm_slave_config_state_wait_safeop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    unsigned long diff = jiffies - fsm->jiffies_start;
+
+    if (diff * 1000 / HZ < fsm->wait_ms) {
+        return;
+    }
+
+    ec_fsm_slave_config_enter_safeop(fsm);
+}
+
+/****************************************************************************/
+
+/** Request SAFEOP state.
+ */
+void ec_fsm_slave_config_enter_safeop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    fsm->state = ec_fsm_slave_config_state_safeop;
+    ec_fsm_change_start(fsm->fsm_change, fsm->slave, EC_SLAVE_STATE_SAFEOP);
+    ec_fsm_change_exec(fsm->fsm_change); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: SAFEOP.
+ */
+void ec_fsm_slave_config_state_safeop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (ec_fsm_change_exec(fsm->fsm_change)) return;
+
+    if (!ec_fsm_change_success(fsm->fsm_change)) {
+        if (!fsm->fsm_change->spontaneous_change)
+            fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        return;
+    }
+
+    // slave is now in SAFEOP
+
+    EC_SLAVE_DBG(slave, 1, "Now in SAFEOP.\n");
+
+    if (fsm->slave->current_state == fsm->slave->requested_state) {
+        fsm->state = ec_fsm_slave_config_state_end; // successful
+        EC_SLAVE_DBG(slave, 1, "Finished configuration.\n");
+        return;
+    }
+
+    ec_fsm_slave_config_enter_soe_conf_safeop(fsm);
+}
+
+/****************************************************************************/
+
+/** Check for SoE configurations to be applied in SAFEOP.
+ */
+void ec_fsm_slave_config_enter_soe_conf_safeop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_soe_request_t *req;
+
+    if (!slave->config) {
+        ec_fsm_slave_config_enter_op(fsm);
+        return;
+    }
+
+    list_for_each_entry(req, &slave->config->soe_configs, list) {
+        if (req->al_state == EC_AL_STATE_SAFEOP) {
+            // start SoE configuration
+            fsm->state = ec_fsm_slave_config_state_soe_conf_safeop;
+            fsm->soe_request = req;
+            ec_soe_request_copy(&fsm->soe_request_copy, fsm->soe_request);
+            ec_soe_request_write(&fsm->soe_request_copy);
+            ec_fsm_soe_transfer(fsm->fsm_soe, fsm->slave,
+                    &fsm->soe_request_copy);
+            ec_fsm_soe_exec(fsm->fsm_soe, fsm->datagram);
+            return;
+        }
+    }
+
+    // No SoE configuration to be applied in SAFEOP
+    ec_fsm_slave_config_enter_op(fsm);
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: SOE_CONF.
+ */
+void ec_fsm_slave_config_state_soe_conf_safeop(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (ec_fsm_soe_exec(fsm->fsm_soe, fsm->datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_soe_success(fsm->fsm_soe)) {
+        EC_SLAVE_ERR(slave, "SoE configuration failed.\n");
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        return;
+    }
+
+    if (!fsm->slave->config) { // config removed in the meantime
+        ec_fsm_slave_config_reconfigure(fsm);
+        return;
+    }
+
+    // Another IDN to configure in SAFEOP?
+    while (fsm->soe_request->list.next != &fsm->slave->config->soe_configs) {
+        fsm->soe_request = list_entry(fsm->soe_request->list.next,
+                ec_soe_request_t, list);
+        if (fsm->soe_request->al_state == EC_AL_STATE_SAFEOP) {
+            ec_soe_request_copy(&fsm->soe_request_copy, fsm->soe_request);
+            ec_soe_request_write(&fsm->soe_request_copy);
+            ec_fsm_soe_transfer(fsm->fsm_soe, fsm->slave,
+                    &fsm->soe_request_copy);
+            ec_fsm_soe_exec(fsm->fsm_soe, fsm->datagram);
+            return;
+        }
+    }
+
+    // All SAFEOP IDNs are now configured.
+    ec_fsm_slave_config_enter_op(fsm);
+}
+
+/****************************************************************************/
+
+/** Bring slave to OP.
+ */
+void ec_fsm_slave_config_enter_op(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    // set state to OP
+    fsm->state = ec_fsm_slave_config_state_op;
+    ec_fsm_change_start(fsm->fsm_change, fsm->slave, EC_SLAVE_STATE_OP);
+    ec_fsm_change_exec(fsm->fsm_change); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Slave configuration state: OP
+ */
+void ec_fsm_slave_config_state_op(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (ec_fsm_change_exec(fsm->fsm_change)) return;
+
+    if (!ec_fsm_change_success(fsm->fsm_change)) {
+        if (!fsm->fsm_change->spontaneous_change)
+            slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_config_state_error;
+        return;
+    }
+
+    // slave is now in OP
+
+    EC_SLAVE_DBG(slave, 1, "Now in OP. Finished configuration.\n");
+
+    fsm->state = ec_fsm_slave_config_state_end; // successful
+}
+
+/****************************************************************************/
+
+/** Reconfigure the slave starting at INIT.
+ */
+void ec_fsm_slave_config_reconfigure(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+    EC_SLAVE_DBG(fsm->slave, 1, "Slave configuration detached during "
+            "configuration. Reconfiguring.");
+
+    ec_fsm_slave_config_enter_init(fsm); // reconfigure
+}
+
+/*****************************************************************************
+ *  Common state functions
+ ****************************************************************************/
+
+/** State: ERROR.
+ */
+void ec_fsm_slave_config_state_error(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** State: END.
+ */
+void ec_fsm_slave_config_state_end(
+        ec_fsm_slave_config_t *fsm /**< slave state machine */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_slave_config.h b/drivers/net/ethercat/master/fsm_slave_config.h
new file mode 100644
index 000000000000..fa33c83b76de
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_slave_config.h
@@ -0,0 +1,81 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+/**
+   \file
+   EtherCAT slave configuration state machine.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_SLAVE_CONFIG_H__
+#define __EC_FSM_SLAVE_CONFIG_H__
+
+#include "globals.h"
+#include "slave.h"
+#include "datagram.h"
+#include "fsm_change.h"
+#include "fsm_coe.h"
+#include "fsm_pdo.h"
+#include "fsm_eoe.h"
+
+/****************************************************************************/
+
+/** \see ec_fsm_slave_config */
+typedef struct ec_fsm_slave_config ec_fsm_slave_config_t;
+
+/** Finite state machine to configure an EtherCAT slave.
+ */
+struct ec_fsm_slave_config
+{
+    ec_datagram_t *datagram; /**< Datagram used in the state machine. */
+    ec_fsm_change_t *fsm_change; /**< State change state machine. */
+    ec_fsm_coe_t *fsm_coe; /**< CoE state machine. */
+    ec_fsm_soe_t *fsm_soe; /**< SoE state machine. */
+    ec_fsm_pdo_t *fsm_pdo; /**< PDO configuration state machine. */
+    ec_fsm_eoe_t *fsm_eoe; /**< EoE state machine. */
+
+    ec_slave_t *slave; /**< Slave the FSM runs on. */
+    void (*state)(ec_fsm_slave_config_t *); /**< State function. */
+    unsigned int retries; /**< Retries on datagram timeout. */
+    ec_sdo_request_t *request; /**< SDO request for SDO configuration. */
+    ec_sdo_request_t request_copy; /**< Copied SDO request. */
+    ec_soe_request_t *soe_request; /**< SDO request for SDO configuration. */
+    ec_soe_request_t soe_request_copy; /**< Copied SDO request. */
+    unsigned long jiffies_start; /**< For timeout calculations. */
+    unsigned int take_time; /**< Store jiffies after datagram reception. */
+    unsigned long wait_ms; /**< Wait time (used to wait before SAFEOP). */
+};
+
+/****************************************************************************/
+
+void ec_fsm_slave_config_init(ec_fsm_slave_config_t *, ec_datagram_t *,
+        ec_fsm_change_t *, ec_fsm_coe_t *, ec_fsm_soe_t *, ec_fsm_pdo_t *,
+        ec_fsm_eoe_t *);
+void ec_fsm_slave_config_clear(ec_fsm_slave_config_t *);
+
+void ec_fsm_slave_config_start(ec_fsm_slave_config_t *, ec_slave_t *);
+
+int ec_fsm_slave_config_exec(ec_fsm_slave_config_t *);
+int ec_fsm_slave_config_success(const ec_fsm_slave_config_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_slave_scan.c b/drivers/net/ethercat/master/fsm_slave_scan.c
new file mode 100644
index 000000000000..e4b563583e0c
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_slave_scan.c
@@ -0,0 +1,1121 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave state machines.
+*/
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "slave_config.h"
+
+#include "fsm_slave_scan.h"
+
+/****************************************************************************/
+
+// prototypes for private methods
+int ec_fsm_slave_scan_running(const ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_enter_sii_size(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_enter_assign_sii(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_enter_datalink(ec_fsm_slave_scan_t *);
+#ifdef EC_REGALIAS
+void ec_fsm_slave_scan_enter_regalias(ec_fsm_slave_scan_t *);
+#endif
+void ec_fsm_slave_scan_enter_preop(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_enter_pdos(ec_fsm_slave_scan_t *);
+
+/****************************************************************************/
+
+void ec_fsm_slave_scan_state_start(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_address(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_state(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_base(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_dc_cap(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_dc_times(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_datalink(ec_fsm_slave_scan_t *);
+#ifdef EC_SII_ASSIGN
+void ec_fsm_slave_scan_state_assign_sii(ec_fsm_slave_scan_t *);
+#endif
+void ec_fsm_slave_scan_state_sii_size(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_sii_data(ec_fsm_slave_scan_t *);
+#ifdef EC_REGALIAS
+void ec_fsm_slave_scan_state_regalias(ec_fsm_slave_scan_t *);
+#endif
+void ec_fsm_slave_scan_state_preop(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_sync(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_pdos(ec_fsm_slave_scan_t *);
+
+void ec_fsm_slave_scan_state_end(ec_fsm_slave_scan_t *);
+void ec_fsm_slave_scan_state_error(ec_fsm_slave_scan_t *);
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_slave_scan_init(
+        ec_fsm_slave_scan_t *fsm, /**< Slave scanning state machine. */
+        ec_datagram_t *datagram, /**< Datagram to use. */
+        ec_fsm_slave_config_t *fsm_slave_config, /**< Slave configuration
+                                                  state machine to use. */
+        ec_fsm_pdo_t *fsm_pdo /**< PDO configuration machine to use. */
+        )
+{
+    fsm->datagram = datagram;
+    fsm->fsm_slave_config = fsm_slave_config;
+    fsm->fsm_pdo = fsm_pdo;
+
+    // init sub state machines
+    ec_fsm_sii_init(&fsm->fsm_sii, fsm->datagram);
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_slave_scan_clear(ec_fsm_slave_scan_t *fsm /**< slave state machine */)
+{
+    // clear sub state machines
+    ec_fsm_sii_clear(&fsm->fsm_sii);
+}
+
+/****************************************************************************/
+
+/**
+ * Start slave scan state machine.
+ */
+
+void ec_fsm_slave_scan_start(
+        ec_fsm_slave_scan_t *fsm, /**< slave state machine */
+        ec_slave_t *slave /**< slave to configure */
+        )
+{
+    fsm->slave = slave;
+    fsm->state = ec_fsm_slave_scan_state_start;
+}
+
+/****************************************************************************/
+
+/**
+   \return false, if state machine has terminated
+*/
+
+int ec_fsm_slave_scan_running(
+        const ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    return fsm->state != ec_fsm_slave_scan_state_end
+        && fsm->state != ec_fsm_slave_scan_state_error;
+}
+
+/****************************************************************************/
+
+/**
+   Executes the current state of the state machine.
+   If the state machine's datagram is not sent or received yet, the execution
+   of the state machine is delayed to the next cycle.
+   \return false, if state machine has terminated
+*/
+
+int ec_fsm_slave_scan_exec(ec_fsm_slave_scan_t *fsm /**< slave state machine */)
+{
+    if (fsm->datagram->state == EC_DATAGRAM_SENT
+        || fsm->datagram->state == EC_DATAGRAM_QUEUED) {
+        // datagram was not sent or received yet.
+        return ec_fsm_slave_scan_running(fsm);
+    }
+
+    fsm->state(fsm);
+    return ec_fsm_slave_scan_running(fsm);
+}
+
+/****************************************************************************/
+
+/**
+   \return true, if the state machine terminated gracefully
+*/
+
+int ec_fsm_slave_scan_success(const ec_fsm_slave_scan_t *fsm /**< slave state machine */)
+{
+    return fsm->state == ec_fsm_slave_scan_state_end;
+}
+
+/*****************************************************************************
+ *  slave scan state machine
+ ****************************************************************************/
+
+/**
+   Slave scan state: START.
+   First state of the slave state machine. Writes the station address to the
+   slave, according to its ring position.
+*/
+
+void ec_fsm_slave_scan_state_start(ec_fsm_slave_scan_t *fsm /**< slave state machine */)
+{
+    // write station address
+    ec_datagram_apwr(fsm->datagram, fsm->slave->ring_position, 0x0010, 2);
+    EC_WRITE_U16(fsm->datagram->data, fsm->slave->station_address);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_scan_state_address;
+}
+
+/****************************************************************************/
+
+/**
+   Slave scan state: ADDRESS.
+*/
+
+void ec_fsm_slave_scan_state_address(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(fsm->slave,
+                "Failed to receive station address datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(fsm->slave, "Failed to write station address: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    // Read AL state
+    ec_datagram_fprd(datagram, fsm->slave->station_address, 0x0130, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_scan_state_state;
+}
+
+/****************************************************************************/
+
+/**
+   Slave scan state: STATE.
+*/
+
+void ec_fsm_slave_scan_state_state(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive AL state datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to read AL state: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    slave->current_state = EC_READ_U8(datagram->data);
+    if (slave->current_state & EC_SLAVE_STATE_ACK_ERR) {
+        char state_str[EC_STATE_STRING_SIZE];
+        ec_state_string(slave->current_state, state_str, 0);
+        EC_SLAVE_WARN(slave, "Slave has state error bit set (%s)!\n",
+                state_str);
+    }
+
+    // read base data
+    ec_datagram_fprd(datagram, fsm->slave->station_address, 0x0000, 12);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_scan_state_base;
+}
+
+/****************************************************************************/
+
+/** Slave scan state: BASE.
+ */
+void ec_fsm_slave_scan_state_base(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    u8 octet;
+    int i;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive base data datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to read base data: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    slave->base_type       = EC_READ_U8 (datagram->data);
+    slave->base_revision   = EC_READ_U8 (datagram->data + 1);
+    slave->base_build      = EC_READ_U16(datagram->data + 2);
+
+    slave->base_fmmu_count = EC_READ_U8 (datagram->data + 4);
+    if (slave->base_fmmu_count > EC_MAX_FMMUS) {
+        EC_SLAVE_WARN(slave, "Slave has more FMMUs (%u) than the master can"
+                " handle (%u).\n", slave->base_fmmu_count, EC_MAX_FMMUS);
+        slave->base_fmmu_count = EC_MAX_FMMUS;
+    }
+
+    slave->base_sync_count = EC_READ_U8(datagram->data + 5);
+    if (slave->base_sync_count > EC_MAX_SYNC_MANAGERS) {
+        EC_SLAVE_WARN(slave, "Slave provides more sync managers (%u)"
+                " than the master can handle (%u).\n",
+                slave->base_sync_count, EC_MAX_SYNC_MANAGERS);
+        slave->base_sync_count = EC_MAX_SYNC_MANAGERS;
+    }
+
+    octet = EC_READ_U8(datagram->data + 7);
+    for (i = 0; i < EC_MAX_PORTS; i++) {
+        slave->ports[i].desc = (octet >> (2 * i)) & 0x03;
+    }
+
+    octet = EC_READ_U8(datagram->data + 8);
+    slave->base_fmmu_bit_operation = octet & 0x01;
+    slave->base_dc_supported = (octet >> 2) & 0x01;
+    slave->base_dc_range = ((octet >> 3) & 0x01) ? EC_DC_64 : EC_DC_32;
+
+    if (slave->base_dc_supported) {
+        // read DC capabilities
+        ec_datagram_fprd(datagram, slave->station_address, 0x0910,
+                slave->base_dc_range == EC_DC_64 ? 8 : 4);
+        ec_datagram_zero(datagram);
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_slave_scan_state_dc_cap;
+    } else {
+        ec_fsm_slave_scan_enter_datalink(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/**
+   Slave scan state: DC CAPABILITIES.
+*/
+
+void ec_fsm_slave_scan_state_dc_cap(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive system time datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter == 1) {
+        slave->has_dc_system_time = 1;
+        EC_SLAVE_DBG(slave, 1, "Slave has the System Time register.\n");
+    } else if (datagram->working_counter == 0) {
+        EC_SLAVE_DBG(slave, 1, "Slave has no System Time register; delay "
+                "measurement only.\n");
+    } else {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to determine, if system time register is "
+                "supported: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    // read DC port receive times
+    ec_datagram_fprd(datagram, slave->station_address, 0x0900, 16);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_scan_state_dc_times;
+}
+
+/****************************************************************************/
+
+/**
+   Slave scan state: DC TIMES.
+*/
+
+void ec_fsm_slave_scan_state_dc_times(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    int i;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive system time datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to get DC receive times: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    for (i = 0; i < EC_MAX_PORTS; i++) {
+        slave->ports[i].receive_time = EC_READ_U32(datagram->data + 4 * i);
+    }
+
+    ec_fsm_slave_scan_enter_datalink(fsm);
+}
+
+/****************************************************************************/
+
+/**
+   Slave scan entry function: DATALINK.
+*/
+
+void ec_fsm_slave_scan_enter_datalink(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    // read data link status
+    ec_datagram_fprd(datagram, slave->station_address, 0x0110, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_scan_state_datalink;
+}
+
+/****************************************************************************/
+
+/** Enter slave scan state SII_SIZE.
+ */
+void ec_fsm_slave_scan_enter_sii_size(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    // Start fetching SII size
+
+    EC_SLAVE_DBG(fsm->slave, 1, "Determining SII size.\n");
+
+    fsm->sii_offset = EC_FIRST_SII_CATEGORY_OFFSET; // first category header
+    ec_fsm_sii_read(&fsm->fsm_sii, fsm->slave, fsm->sii_offset,
+            EC_FSM_SII_USE_CONFIGURED_ADDRESS);
+    fsm->state = ec_fsm_slave_scan_state_sii_size;
+    fsm->state(fsm); // execute state immediately
+}
+
+/****************************************************************************/
+
+#ifdef EC_SII_ASSIGN
+
+/** Enter slave scan state ASSIGN_SII.
+ */
+void ec_fsm_slave_scan_enter_assign_sii(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    EC_SLAVE_DBG(slave, 1, "Assigning SII access to EtherCAT.\n");
+
+    // assign SII to ECAT
+    ec_datagram_fpwr(datagram, slave->station_address, 0x0500, 1);
+    EC_WRITE_U8(datagram->data, 0x00); // EtherCAT
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_scan_state_assign_sii;
+}
+
+#endif
+
+/****************************************************************************/
+
+/**
+   Slave scan state: DATALINK.
+*/
+
+void ec_fsm_slave_scan_state_datalink(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    uint16_t dl_status;
+    unsigned int i;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive DL status datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to read DL status: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    dl_status = EC_READ_U16(datagram->data);
+    for (i = 0; i < EC_MAX_PORTS; i++) {
+        slave->ports[i].link.link_up =
+            dl_status & (1 << (4 + i)) ? 1 : 0;
+        slave->ports[i].link.loop_closed =
+            dl_status & (1 << (8 + i * 2)) ? 1 : 0;
+        slave->ports[i].link.signal_detected =
+            dl_status & (1 << (9 + i * 2)) ? 1 : 0;
+    }
+
+#ifdef EC_SII_ASSIGN
+    ec_fsm_slave_scan_enter_assign_sii(fsm);
+#else
+    ec_fsm_slave_scan_enter_sii_size(fsm);
+#endif
+}
+
+/****************************************************************************/
+
+#ifdef EC_SII_ASSIGN
+
+/**
+   Slave scan state: ASSIGN_SII.
+*/
+
+void ec_fsm_slave_scan_state_assign_sii(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        return;
+    }
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        EC_SLAVE_WARN(slave, "Failed to receive SII assignment datagram: ");
+        ec_datagram_print_state(datagram);
+        // Try to go on, probably assignment is correct
+        goto continue_with_sii_size;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_WARN(slave, "Failed to assign SII to EtherCAT: ");
+        ec_datagram_print_wc_error(datagram);
+        // Try to go on, probably assignment is correct
+    }
+
+continue_with_sii_size:
+    ec_fsm_slave_scan_enter_sii_size(fsm);
+}
+
+#endif
+
+/****************************************************************************/
+
+/**
+   Slave scan state: SII SIZE.
+*/
+
+void ec_fsm_slave_scan_state_sii_size(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    uint16_t cat_type, cat_size;
+
+    if (ec_fsm_sii_exec(&fsm->fsm_sii))
+        return;
+
+    if (!ec_fsm_sii_success(&fsm->fsm_sii)) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to determine SII content size:"
+                " Reading word offset 0x%04x failed. Assuming %u words.\n",
+                fsm->sii_offset, EC_FIRST_SII_CATEGORY_OFFSET);
+        slave->sii_nwords = EC_FIRST_SII_CATEGORY_OFFSET;
+        goto alloc_sii;
+    }
+
+    cat_type = EC_READ_U16(fsm->fsm_sii.value);
+    cat_size = EC_READ_U16(fsm->fsm_sii.value + 2);
+
+    if (cat_type != 0xFFFF) { // not the last category
+        off_t next_offset = 2UL + fsm->sii_offset + cat_size;
+
+        EC_SLAVE_DBG(slave, 1, "Found category type %u with size %u."
+                " Proceeding to offset %zd.\n",
+                cat_type, cat_size, (ssize_t)next_offset);
+
+        if (next_offset >= EC_MAX_SII_SIZE) {
+            EC_SLAVE_WARN(slave, "SII size exceeds %u words"
+                    " (0xffff limiter missing?).\n", EC_MAX_SII_SIZE);
+            // cut off category data...
+            slave->sii_nwords = EC_FIRST_SII_CATEGORY_OFFSET;
+            goto alloc_sii;
+        }
+        fsm->sii_offset = next_offset;
+        ec_fsm_sii_read(&fsm->fsm_sii, slave, fsm->sii_offset,
+                        EC_FSM_SII_USE_CONFIGURED_ADDRESS);
+        ec_fsm_sii_exec(&fsm->fsm_sii); // execute state immediately
+        return;
+    }
+
+    slave->sii_nwords = fsm->sii_offset + 1;
+
+alloc_sii:
+    if (slave->sii_words) {
+        EC_SLAVE_WARN(slave, "Freeing old SII data...\n");
+        kfree(slave->sii_words);
+    }
+
+    if (!(slave->sii_words =
+                (uint16_t *) kmalloc(slave->sii_nwords * 2, GFP_KERNEL))) {
+        EC_SLAVE_ERR(slave, "Failed to allocate %zu words of SII data.\n",
+               slave->sii_nwords);
+        slave->sii_nwords = 0;
+        slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        return;
+    }
+
+    // Start fetching SII contents
+
+    fsm->state = ec_fsm_slave_scan_state_sii_data;
+    fsm->sii_offset = 0x0000;
+    ec_fsm_sii_read(&fsm->fsm_sii, slave, fsm->sii_offset,
+            EC_FSM_SII_USE_CONFIGURED_ADDRESS);
+    ec_fsm_sii_exec(&fsm->fsm_sii); // execute state immediately
+}
+
+/****************************************************************************/
+
+/**
+   Slave scan state: SII DATA.
+*/
+
+void ec_fsm_slave_scan_state_sii_data(ec_fsm_slave_scan_t *fsm
+        /**< slave state machine */)
+{
+    ec_slave_t *slave = fsm->slave;
+    uint16_t *cat_word, cat_type, cat_size;
+
+    if (ec_fsm_sii_exec(&fsm->fsm_sii)) return;
+
+    if (!ec_fsm_sii_success(&fsm->fsm_sii)) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to fetch SII contents.\n");
+        return;
+    }
+
+    // 2 words fetched
+
+    if (fsm->sii_offset + 2 <= slave->sii_nwords) { // 2 words fit
+        memcpy(slave->sii_words + fsm->sii_offset, fsm->fsm_sii.value, 4);
+    } else { // copy the last word
+        memcpy(slave->sii_words + fsm->sii_offset, fsm->fsm_sii.value, 2);
+    }
+
+    if (fsm->sii_offset + 2 < slave->sii_nwords) {
+        // fetch the next 2 words
+        fsm->sii_offset += 2;
+        ec_fsm_sii_read(&fsm->fsm_sii, slave, fsm->sii_offset,
+                        EC_FSM_SII_USE_CONFIGURED_ADDRESS);
+        ec_fsm_sii_exec(&fsm->fsm_sii); // execute state immediately
+        return;
+    }
+
+    // Evaluate SII contents
+
+    ec_slave_clear_sync_managers(slave);
+
+    slave->sii.alias =
+        EC_READ_U16(slave->sii_words + 0x0004);
+    slave->effective_alias = slave->sii.alias;
+    slave->sii.vendor_id =
+        EC_READ_U32(slave->sii_words + 0x0008);
+    slave->sii.product_code =
+        EC_READ_U32(slave->sii_words + 0x000A);
+    slave->sii.revision_number =
+        EC_READ_U32(slave->sii_words + 0x000C);
+    slave->sii.serial_number =
+        EC_READ_U32(slave->sii_words + 0x000E);
+    slave->sii.boot_rx_mailbox_offset =
+        EC_READ_U16(slave->sii_words + 0x0014);
+    slave->sii.boot_rx_mailbox_size =
+        EC_READ_U16(slave->sii_words + 0x0015);
+    slave->sii.boot_tx_mailbox_offset =
+        EC_READ_U16(slave->sii_words + 0x0016);
+    slave->sii.boot_tx_mailbox_size =
+        EC_READ_U16(slave->sii_words + 0x0017);
+    slave->sii.std_rx_mailbox_offset =
+        EC_READ_U16(slave->sii_words + 0x0018);
+    slave->sii.std_rx_mailbox_size =
+        EC_READ_U16(slave->sii_words + 0x0019);
+    slave->sii.std_tx_mailbox_offset =
+        EC_READ_U16(slave->sii_words + 0x001A);
+    slave->sii.std_tx_mailbox_size =
+        EC_READ_U16(slave->sii_words + 0x001B);
+    slave->sii.mailbox_protocols =
+        EC_READ_U16(slave->sii_words + 0x001C);
+    if (slave->sii.mailbox_protocols) {
+        int need_delim = 0;
+        uint16_t all = EC_MBOX_AOE | EC_MBOX_COE | EC_MBOX_FOE |
+            EC_MBOX_SOE | EC_MBOX_VOE;
+        if ((slave->sii.mailbox_protocols & all) &&
+                slave->master->debug_level >= 1) {
+            EC_SLAVE_DBG(slave, 1, "Slave announces to support ");
+            if (slave->sii.mailbox_protocols & EC_MBOX_AOE) {
+                printk(KERN_CONT "AoE");
+                need_delim = 1;
+            }
+            if (slave->sii.mailbox_protocols & EC_MBOX_COE) {
+                if (need_delim) {
+                    printk(KERN_CONT ", ");
+                }
+                printk(KERN_CONT "CoE");
+                need_delim = 1;
+            }
+            if (slave->sii.mailbox_protocols & EC_MBOX_FOE) {
+                if (need_delim) {
+                    printk(KERN_CONT ", ");
+                }
+                printk(KERN_CONT "FoE");
+                need_delim = 1;
+            }
+            if (slave->sii.mailbox_protocols & EC_MBOX_SOE) {
+                if (need_delim) {
+                    printk(KERN_CONT ", ");
+                }
+                printk(KERN_CONT "SoE");
+                need_delim = 1;
+            }
+            if (slave->sii.mailbox_protocols & EC_MBOX_VOE) {
+                if (need_delim) {
+                    printk(KERN_CONT ", ");
+                }
+                printk(KERN_CONT "VoE");
+                need_delim = 1;
+            }
+            printk(KERN_CONT ".\n");
+        }
+        if (slave->sii.mailbox_protocols & ~all) {
+            EC_SLAVE_DBG(slave, 1, "Slave announces to support unknown"
+                    " mailbox protocols 0x%04X.",
+                    slave->sii.mailbox_protocols & ~all);
+        }
+    }
+    else {
+        EC_SLAVE_DBG(slave, 1, "Slave announces to support no mailbox"
+                " protocols.");
+    }
+
+    if (slave->sii.boot_rx_mailbox_offset == 0xffff ||
+            slave->sii.boot_rx_mailbox_size == 0xffff ||
+            slave->sii.boot_tx_mailbox_offset == 0xffff ||
+            slave->sii.boot_tx_mailbox_size == 0xffff ||
+            slave->sii.std_rx_mailbox_offset == 0xffff ||
+            slave->sii.std_rx_mailbox_size == 0xffff ||
+            slave->sii.std_tx_mailbox_offset == 0xffff ||
+            slave->sii.std_tx_mailbox_size == 0xffff) {
+        slave->sii.mailbox_protocols = 0x0000;
+        EC_SLAVE_ERR(slave, "Invalid mailbox settings in SII."
+                " Disabling mailbox communication.");
+    }
+
+    if (slave->sii_nwords == EC_FIRST_SII_CATEGORY_OFFSET) {
+        // sii does not contain category data
+        fsm->state = ec_fsm_slave_scan_state_end;
+        return;
+    }
+
+    if (slave->sii_nwords < EC_FIRST_SII_CATEGORY_OFFSET + 1) {
+        EC_SLAVE_ERR(slave, "Unexpected end of SII data:"
+                " First category header missing.\n");
+        goto end;
+    }
+
+    // evaluate category data
+    cat_word = slave->sii_words + EC_FIRST_SII_CATEGORY_OFFSET;
+    while (EC_READ_U16(cat_word) != 0xFFFF) {
+
+        // type and size words must fit
+        if (cat_word + 2 - slave->sii_words > slave->sii_nwords) {
+            EC_SLAVE_ERR(slave, "Unexpected end of SII data:"
+                    " Category header incomplete.\n");
+            goto end;
+        }
+
+        cat_type = EC_READ_U16(cat_word) & 0x7FFF;
+        cat_size = EC_READ_U16(cat_word + 1);
+        cat_word += 2;
+
+        if (cat_word + cat_size - slave->sii_words > slave->sii_nwords) {
+            EC_SLAVE_WARN(slave, "Unexpected end of SII data:"
+                    " Category data incomplete.\n");
+            goto end;
+        }
+
+        switch (cat_type) {
+            case 0x000A:
+                if (ec_slave_fetch_sii_strings(slave, (uint8_t *) cat_word,
+                            cat_size * 2))
+                    goto end;
+                break;
+            case 0x001E:
+                if (ec_slave_fetch_sii_general(slave, (uint8_t *) cat_word,
+                            cat_size * 2))
+                    goto end;
+                break;
+            case 0x0028:
+                break;
+            case 0x0029:
+                if (ec_slave_fetch_sii_syncs(slave, (uint8_t *) cat_word,
+                            cat_size * 2))
+                    goto end;
+                break;
+            case 0x0032:
+                if (ec_slave_fetch_sii_pdos( slave, (uint8_t *) cat_word,
+                            cat_size * 2, EC_DIR_INPUT)) // TxPDO
+                    goto end;
+                break;
+            case 0x0033:
+                if (ec_slave_fetch_sii_pdos( slave, (uint8_t *) cat_word,
+                            cat_size * 2, EC_DIR_OUTPUT)) // RxPDO
+                    goto end;
+                break;
+            default:
+                EC_SLAVE_DBG(slave, 1, "Unknown category type 0x%04X.\n",
+                        cat_type);
+        }
+
+        cat_word += cat_size;
+        if (cat_word - slave->sii_words >= slave->sii_nwords) {
+            EC_SLAVE_WARN(slave, "Unexpected end of SII data:"
+                    " Next category header missing.\n");
+            goto end;
+        }
+    }
+
+#ifdef EC_REGALIAS
+    ec_fsm_slave_scan_enter_regalias(fsm);
+#else
+    if (slave->sii.mailbox_protocols & EC_MBOX_COE) {
+        ec_fsm_slave_scan_enter_preop(fsm);
+    } else {
+        fsm->state = ec_fsm_slave_scan_state_end;
+    }
+#endif
+    return;
+
+end:
+    EC_SLAVE_ERR(slave, "Failed to analyze category data.\n");
+    fsm->slave->error_flag = 1;
+    fsm->state = ec_fsm_slave_scan_state_error;
+}
+
+/****************************************************************************/
+
+#ifdef EC_REGALIAS
+
+/** Slave scan entry function: REGALIAS.
+ */
+void ec_fsm_slave_scan_enter_regalias(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    // read alias from register
+    EC_SLAVE_DBG(slave, 1, "Reading alias from register.\n");
+    ec_datagram_fprd(datagram, slave->station_address, 0x0012, 2);
+    ec_datagram_zero(datagram);
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_slave_scan_state_regalias;
+}
+
+/****************************************************************************/
+
+/** Slave scan state: REGALIAS.
+ */
+void ec_fsm_slave_scan_state_regalias(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive register alias datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        EC_SLAVE_DBG(slave, 1, "Failed to read register alias.\n");
+    } else {
+        slave->effective_alias = EC_READ_U16(datagram->data);
+        EC_SLAVE_DBG(slave, 1, "Read alias %u from register.\n",
+                slave->effective_alias);
+    }
+
+    if (slave->sii.mailbox_protocols & EC_MBOX_COE) {
+        ec_fsm_slave_scan_enter_preop(fsm);
+    } else {
+        fsm->state = ec_fsm_slave_scan_state_end;
+    }
+}
+
+#endif // defined EC_REGALIAS
+
+/****************************************************************************/
+
+/** Enter slave scan state PREOP.
+ */
+void ec_fsm_slave_scan_enter_preop(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    uint8_t current_state = slave->current_state & EC_SLAVE_STATE_MASK;
+
+    if (current_state != EC_SLAVE_STATE_PREOP
+            && current_state != EC_SLAVE_STATE_SAFEOP
+            && current_state != EC_SLAVE_STATE_OP) {
+        if (slave->master->debug_level) {
+            char str[EC_STATE_STRING_SIZE];
+            ec_state_string(current_state, str, 0);
+            EC_SLAVE_DBG(slave, 0, "Slave is not in the state"
+                    " to do mailbox com (%s), setting to PREOP.\n", str);
+        }
+
+        fsm->state = ec_fsm_slave_scan_state_preop;
+        ec_slave_request_state(slave, EC_SLAVE_STATE_PREOP);
+        ec_fsm_slave_config_start(fsm->fsm_slave_config, slave);
+        ec_fsm_slave_config_exec(fsm->fsm_slave_config);
+    } else {
+        EC_SLAVE_DBG(slave, 1, "Reading mailbox"
+                " sync manager configuration.\n");
+
+        /* Scan current sync manager configuration to get configured mailbox
+         * sizes. */
+        ec_datagram_fprd(fsm->datagram, slave->station_address, 0x0800,
+                EC_SYNC_PAGE_SIZE * 2);
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_slave_scan_state_sync;
+    }
+}
+
+/****************************************************************************/
+
+/** Slave scan state: PREOP.
+ */
+void ec_fsm_slave_scan_state_preop(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    if (ec_fsm_slave_config_exec(fsm->fsm_slave_config))
+        return;
+
+    if (!ec_fsm_slave_config_success(fsm->fsm_slave_config)) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        return;
+    }
+
+    ec_fsm_slave_scan_enter_pdos(fsm);
+}
+
+/****************************************************************************/
+
+/** Slave scan state: SYNC.
+ */
+void ec_fsm_slave_scan_state_sync(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_datagram_t *datagram = fsm->datagram;
+    ec_slave_t *slave = fsm->slave;
+    uint16_t tx_offset, tx_size, rx_offset, rx_size;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to receive sync manager"
+                " configuration datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        fsm->slave->error_flag = 1;
+        fsm->state = ec_fsm_slave_scan_state_error;
+        EC_SLAVE_ERR(slave, "Failed to read DL status: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    rx_offset = EC_READ_U16(datagram->data);
+    rx_size = EC_READ_U16(datagram->data + 2);
+    tx_offset = EC_READ_U16(datagram->data + 8);
+    tx_size = EC_READ_U16(datagram->data + 10);
+
+    if (rx_size == 0xffff) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        slave->sii.mailbox_protocols = 0x0000;
+        EC_SLAVE_ERR(slave, "Invalid RX mailbox size (%u) configured."
+                " Disabling mailbox communication.", rx_size);
+        return;
+    }
+
+    if (tx_size == 0xffff) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        slave->sii.mailbox_protocols = 0x0000;
+        EC_SLAVE_ERR(slave, "Invalid TX mailbox size (%u) configured."
+                " Disabling mailbox communication.", tx_size);
+        return;
+    }
+
+    slave->configured_rx_mailbox_offset = rx_offset;
+    slave->configured_rx_mailbox_size = rx_size;
+    slave->configured_tx_mailbox_offset = tx_offset;
+    slave->configured_tx_mailbox_size = tx_size;
+
+    EC_SLAVE_DBG(slave, 1, "Mailbox configuration:\n");
+    EC_SLAVE_DBG(slave, 1, " RX offset=0x%04x size=%u\n",
+            slave->configured_rx_mailbox_offset,
+            slave->configured_rx_mailbox_size);
+    EC_SLAVE_DBG(slave, 1, " TX offset=0x%04x size=%u\n",
+            slave->configured_tx_mailbox_offset,
+            slave->configured_tx_mailbox_size);
+
+    ec_fsm_slave_scan_enter_pdos(fsm);
+}
+
+/****************************************************************************/
+
+/** Enter slave scan state PDOS.
+ */
+void ec_fsm_slave_scan_enter_pdos(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    EC_SLAVE_DBG(slave, 1, "Scanning PDO assignment and mapping.\n");
+    fsm->state = ec_fsm_slave_scan_state_pdos;
+    ec_fsm_pdo_start_reading(fsm->fsm_pdo, slave);
+    ec_fsm_pdo_exec(fsm->fsm_pdo, fsm->datagram); // execute immediately
+}
+
+/****************************************************************************/
+
+/** Slave scan state: PDOS.
+ */
+void ec_fsm_slave_scan_state_pdos(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+    if (ec_fsm_pdo_exec(fsm->fsm_pdo, fsm->datagram)) {
+        return;
+    }
+
+    if (!ec_fsm_pdo_success(fsm->fsm_pdo)) {
+        fsm->state = ec_fsm_slave_scan_state_error;
+        return;
+    }
+
+    // reading PDO configuration finished
+    fsm->state = ec_fsm_slave_scan_state_end;
+}
+
+/*****************************************************************************
+ * Common state functions
+ ****************************************************************************/
+
+/** State: ERROR.
+ */
+void ec_fsm_slave_scan_state_error(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** State: END.
+ */
+void ec_fsm_slave_scan_state_end(
+        ec_fsm_slave_scan_t *fsm /**< slave state machine */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_slave_scan.h b/drivers/net/ethercat/master/fsm_slave_scan.h
new file mode 100644
index 000000000000..72c8a39ace14
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_slave_scan.h
@@ -0,0 +1,75 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave scanning state machine.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_SLAVE_SCAN_H__
+#define __EC_FSM_SLAVE_SCAN_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "slave.h"
+#include "fsm_sii.h"
+#include "fsm_change.h"
+#include "fsm_coe.h"
+#include "fsm_pdo.h"
+
+/****************************************************************************/
+
+/** \see ec_fsm_slave_scan */
+typedef struct ec_fsm_slave_scan ec_fsm_slave_scan_t;
+
+/** Finite state machine for scanning an EtherCAT slave.
+ */
+struct ec_fsm_slave_scan
+{
+    ec_slave_t *slave; /**< Slave the FSM runs on. */
+    ec_datagram_t *datagram; /**< Datagram used in the state machine. */
+    ec_fsm_slave_config_t *fsm_slave_config; /**< Slave configuration state
+                                               machine to use. */
+    ec_fsm_pdo_t *fsm_pdo; /**< PDO configuration state machine to use. */
+    unsigned int retries; /**< Retries on datagram timeout. */
+
+    void (*state)(ec_fsm_slave_scan_t *); /**< State function. */
+    uint16_t sii_offset; /**< SII offset in words. */
+
+    ec_fsm_sii_t fsm_sii; /**< SII state machine. */
+};
+
+/****************************************************************************/
+
+void ec_fsm_slave_scan_init(ec_fsm_slave_scan_t *, ec_datagram_t *,
+        ec_fsm_slave_config_t *, ec_fsm_pdo_t *);
+void ec_fsm_slave_scan_clear(ec_fsm_slave_scan_t *);
+
+void ec_fsm_slave_scan_start(ec_fsm_slave_scan_t *, ec_slave_t *);
+
+int ec_fsm_slave_scan_exec(ec_fsm_slave_scan_t *);
+int ec_fsm_slave_scan_success(const ec_fsm_slave_scan_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/fsm_soe.c b/drivers/net/ethercat/master/fsm_soe.c
new file mode 100644
index 000000000000..b24f03a9417b
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_soe.c
@@ -0,0 +1,844 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2020  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT SoE state machines.
+*/
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "master.h"
+#include "mailbox.h"
+#include "fsm_soe.h"
+
+/****************************************************************************/
+
+/** SoE operations
+ */
+enum {
+    OPCODE_READ_REQUEST   = 0x01, /**< Read request. */
+    OPCODE_READ_RESPONSE  = 0x02, /**< Read response. */
+    OPCODE_WRITE_REQUEST  = 0x03, /**< Write request. */
+    OPCODE_WRITE_RESPONSE = 0x04  /**< Write response. */
+};
+
+/** Size of all SoE headers.
+ */
+#define EC_SOE_SIZE 0x04
+
+/** SoE header size.
+ */
+#define EC_SOE_HEADER_SIZE (EC_MBOX_HEADER_SIZE + EC_SOE_SIZE)
+
+/** SoE response timeout [ms].
+ */
+#define EC_SOE_RESPONSE_TIMEOUT 1000
+
+/****************************************************************************/
+
+// prototypes for private methods
+void ec_print_soe_error(const ec_slave_t *, uint16_t);
+void ec_fsm_soe_print_error(ec_fsm_soe_t *);
+int ec_fsm_soe_prepare_read(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_write_next_fragment(ec_fsm_soe_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+void ec_fsm_soe_read_start(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_read_request(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_read_check(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_read_response(ec_fsm_soe_t *, ec_datagram_t *);
+
+void ec_fsm_soe_write_start(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_write_request(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_write_check(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_write_response(ec_fsm_soe_t *, ec_datagram_t *);
+
+void ec_fsm_soe_end(ec_fsm_soe_t *, ec_datagram_t *);
+void ec_fsm_soe_error(ec_fsm_soe_t *, ec_datagram_t *);
+
+/****************************************************************************/
+
+extern const ec_code_msg_t soe_error_codes[];
+
+/****************************************************************************/
+
+/** Outputs an SoE error code.
+*/
+void ec_print_soe_error(const ec_slave_t *slave, uint16_t error_code)
+{
+    const ec_code_msg_t *error_msg;
+
+    for (error_msg = soe_error_codes; error_msg->code; error_msg++) {
+        if (error_msg->code == error_code) {
+            EC_SLAVE_ERR(slave, "SoE error 0x%04X: \"%s\".\n",
+                   error_msg->code, error_msg->message);
+            return;
+        }
+    }
+
+    EC_SLAVE_ERR(slave, "Unknown SoE error 0x%04X.\n", error_code);
+}
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_fsm_soe_init(
+        ec_fsm_soe_t *fsm /**< finite state machine */
+        )
+{
+    fsm->state = NULL;
+    fsm->datagram = NULL;
+    fsm->fragment_size = 0;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_fsm_soe_clear(
+        ec_fsm_soe_t *fsm /**< finite state machine */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** Starts to transfer an IDN to/from a slave.
+ */
+void ec_fsm_soe_transfer(
+        ec_fsm_soe_t *fsm, /**< State machine. */
+        ec_slave_t *slave, /**< EtherCAT slave. */
+        ec_soe_request_t *request /**< SoE request. */
+        )
+{
+    fsm->slave = slave;
+    fsm->request = request;
+
+    if (request->dir == EC_DIR_OUTPUT) {
+        fsm->state = ec_fsm_soe_write_start;
+    } else {
+        fsm->state = ec_fsm_soe_read_start;
+    }
+}
+
+/****************************************************************************/
+
+/** Executes the current state of the state machine.
+ *
+ * \return 1 if the datagram was used, else 0.
+ */
+int ec_fsm_soe_exec(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    int datagram_used = 0;
+
+    if (fsm->datagram &&
+            (fsm->datagram->state == EC_DATAGRAM_INIT ||
+             fsm->datagram->state == EC_DATAGRAM_QUEUED ||
+             fsm->datagram->state == EC_DATAGRAM_SENT)) {
+        // datagram not received yet
+        return datagram_used;
+    }
+
+    fsm->state(fsm, datagram);
+
+    datagram_used =
+        fsm->state != ec_fsm_soe_end && fsm->state != ec_fsm_soe_error;
+
+    if (datagram_used) {
+        fsm->datagram = datagram;
+    } else {
+        fsm->datagram = NULL;
+    }
+
+    return datagram_used;
+}
+
+/****************************************************************************/
+
+/** Returns, if the state machine terminated with success.
+ *
+ * \return non-zero if successful.
+ */
+int ec_fsm_soe_success(const ec_fsm_soe_t *fsm /**< Finite state machine */)
+{
+    return fsm->state == ec_fsm_soe_end;
+}
+
+/****************************************************************************/
+
+/** Output information about a failed SoE transfer.
+ */
+void ec_fsm_soe_print_error(ec_fsm_soe_t *fsm /**< Finite state machine */)
+{
+    ec_soe_request_t *request = fsm->request;
+
+    EC_SLAVE_ERR(fsm->slave, "");
+
+    if (request->dir == EC_DIR_OUTPUT) {
+        printk(KERN_CONT "Writing");
+    } else {
+        printk(KERN_CONT "Reading");
+    }
+
+    printk(KERN_CONT " IDN 0x%04X failed.\n", request->idn);
+}
+
+/*****************************************************************************
+ * SoE read state machine
+ ****************************************************************************/
+
+/** Prepare a read operation.
+ *
+ * \return 0 on success, otherwise a negative error code.
+ */
+int ec_fsm_soe_prepare_read(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    uint8_t *data;
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    ec_soe_request_t *request = fsm->request;
+
+    data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_SOE,
+            EC_SOE_SIZE);
+    if (IS_ERR(data)) {
+        return PTR_ERR(data);
+    }
+
+    EC_WRITE_U8(data, OPCODE_READ_REQUEST | (request->drive_no & 0x07) << 5);
+    EC_WRITE_U8(data + 1, 1 << 6); // request value
+    EC_WRITE_U16(data + 2, request->idn);
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 0, "SSC read request:\n");
+        ec_print_data(data, EC_SOE_SIZE);
+    }
+
+    fsm->request->jiffies_sent = jiffies;
+    fsm->state = ec_fsm_soe_read_request;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** SoE state: READ START.
+ */
+void ec_fsm_soe_read_start(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_soe_request_t *request = fsm->request;
+
+    EC_SLAVE_DBG(slave, 1, "Reading IDN 0x%04X of drive %u.\n", request->idn,
+            request->drive_no);
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_SOE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support SoE!\n");
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    request->data_size = 0;
+    fsm->retries = EC_FSM_RETRIES;
+
+    if (ec_fsm_soe_prepare_read(fsm, datagram)) {
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+    }
+}
+
+/****************************************************************************/
+
+/** SoE state: READ REQUEST.
+ */
+void ec_fsm_soe_read_request(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    unsigned long diff_ms;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        if (ec_fsm_soe_prepare_read(fsm, datagram)) {
+            fsm->state = ec_fsm_soe_error;
+            ec_fsm_soe_print_error(fsm);
+        }
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive SoE read request: ");
+        ec_datagram_print_state(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    diff_ms = (jiffies - fsm->request->jiffies_sent) * 1000 / HZ;
+
+    if (fsm->datagram->working_counter != 1) {
+        if (!fsm->datagram->working_counter) {
+            if (diff_ms < EC_SOE_RESPONSE_TIMEOUT) {
+                // no response; send request datagram again
+                if (ec_fsm_soe_prepare_read(fsm, datagram)) {
+                    fsm->state = ec_fsm_soe_error;
+                    ec_fsm_soe_print_error(fsm);
+                }
+                return;
+            }
+        }
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Reception of SoE read request"
+                " failed after %lu ms: ", diff_ms);
+        ec_datagram_print_wc_error(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    fsm->jiffies_start = fsm->datagram->jiffies_sent;
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_soe_read_check;
+}
+
+/****************************************************************************/
+
+/** CoE state: READ CHECK.
+ */
+void ec_fsm_soe_read_check(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive SoE mailbox check datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Reception of SoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (fsm->datagram->jiffies_received - fsm->jiffies_start) *
+            1000 / HZ;
+        if (diff_ms >= EC_SOE_RESPONSE_TIMEOUT) {
+            fsm->state = ec_fsm_soe_error;
+            EC_SLAVE_ERR(slave, "Timeout after %lu ms while waiting for"
+                    " read response.\n", diff_ms);
+            ec_fsm_soe_print_error(fsm);
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_soe_read_response;
+}
+
+/****************************************************************************/
+
+/** SoE state: READ RESPONSE.
+ */
+void ec_fsm_soe_read_response(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    uint8_t *data, mbox_prot, header, opcode, incomplete, error_flag,
+            value_included;
+    size_t rec_size, data_size;
+    ec_soe_request_t *req = fsm->request;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive SoE read response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Reception of SoE read response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 0, "SSC read response:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_SOE) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (rec_size < EC_SOE_SIZE) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Received currupted SoE read response"
+                " (%zu bytes)!\n", rec_size);
+        ec_print_data(data, rec_size);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    header = EC_READ_U8(data);
+    opcode = header & 0x7;
+    incomplete = (header >> 3) & 1;
+    error_flag = (header >> 4) & 1;
+
+    if (opcode != OPCODE_READ_RESPONSE) {
+        EC_SLAVE_ERR(slave, "Received no read response (opcode %x).\n",
+                opcode);
+        ec_print_data(data, rec_size);
+        ec_fsm_soe_print_error(fsm);
+        fsm->state = ec_fsm_soe_error;
+        return;
+    }
+
+    if (error_flag) {
+        req->error_code = EC_READ_U16(data + rec_size - 2);
+        EC_SLAVE_ERR(slave, "Received error response:\n");
+        ec_print_soe_error(slave, req->error_code);
+        ec_fsm_soe_print_error(fsm);
+        fsm->state = ec_fsm_soe_error;
+        return;
+    } else {
+        req->error_code = 0x0000;
+    }
+
+    value_included = (EC_READ_U8(data + 1) >> 6) & 1;
+    if (!value_included) {
+        EC_SLAVE_ERR(slave, "No value included!\n");
+        ec_fsm_soe_print_error(fsm);
+        fsm->state = ec_fsm_soe_error;
+        return;
+    }
+
+    data_size = rec_size - EC_SOE_SIZE;
+    if (ec_soe_request_append_data(req,
+                data + EC_SOE_SIZE, data_size)) {
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (incomplete) {
+        EC_SLAVE_DBG(slave, 1, "SoE data incomplete. Waiting for fragment"
+                " at offset %zu.\n", req->data_size);
+        fsm->jiffies_start = fsm->datagram->jiffies_sent;
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_soe_read_check;
+    } else {
+        if (master->debug_level) {
+            EC_SLAVE_DBG(slave, 0, "IDN data:\n");
+            ec_print_data(req->data, req->data_size);
+        }
+
+        fsm->state = ec_fsm_soe_end; // success
+    }
+}
+
+/*****************************************************************************
+ * SoE write state machine
+ ****************************************************************************/
+
+/** Write next fragment.
+ */
+void ec_fsm_soe_write_next_fragment(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    ec_soe_request_t *req = fsm->request;
+    uint8_t incomplete, *data;
+    size_t max_fragment_size, remaining_size;
+    uint16_t fragments_left;
+
+    remaining_size = req->data_size - fsm->offset;
+    max_fragment_size = slave->configured_rx_mailbox_size - EC_SOE_HEADER_SIZE;
+    incomplete = remaining_size > max_fragment_size;
+    fsm->fragment_size = incomplete ? max_fragment_size : remaining_size;
+    fragments_left = remaining_size / fsm->fragment_size - 1;
+    if (remaining_size % fsm->fragment_size) {
+        fragments_left++;
+    }
+
+    data = ec_slave_mbox_prepare_send(slave, datagram, EC_MBOX_TYPE_SOE,
+            EC_SOE_SIZE + fsm->fragment_size);
+    if (IS_ERR(data)) {
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    EC_WRITE_U8(data, OPCODE_WRITE_REQUEST | incomplete << 3 |
+            (req->drive_no & 0x07) << 5);
+    EC_WRITE_U8(data + 1, 1 << 6); // only value included
+    EC_WRITE_U16(data + 2, incomplete ? fragments_left : req->idn);
+    memcpy(data + EC_SOE_SIZE, req->data + fsm->offset, fsm->fragment_size);
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 0, "SSC write request:\n");
+        ec_print_data(data, EC_SOE_SIZE + fsm->fragment_size);
+    }
+
+    fsm->state = ec_fsm_soe_write_request;
+}
+
+/****************************************************************************/
+
+/** SoE state: WRITE START.
+ */
+void ec_fsm_soe_write_start(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_soe_request_t *req = fsm->request;
+
+    EC_SLAVE_DBG(slave, 1, "Writing IDN 0x%04X of drive %u (%zu byte).\n",
+            req->idn, req->drive_no, req->data_size);
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_SOE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support SoE!\n");
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (slave->configured_rx_mailbox_size <= EC_SOE_HEADER_SIZE) {
+        EC_SLAVE_ERR(slave, "Mailbox size (%u) too small for SoE write.\n",
+                slave->configured_rx_mailbox_size);
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    fsm->offset = 0;
+    fsm->retries = EC_FSM_RETRIES;
+    ec_fsm_soe_write_next_fragment(fsm, datagram);
+    req->jiffies_sent = jiffies;
+}
+
+/****************************************************************************/
+
+/** SoE state: WRITE REQUEST.
+ */
+void ec_fsm_soe_write_request(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    unsigned long diff_ms;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_fsm_soe_write_next_fragment(fsm, datagram);
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive SoE write request: ");
+        ec_datagram_print_state(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    diff_ms = (jiffies - fsm->request->jiffies_sent) * 1000 / HZ;
+
+    if (fsm->datagram->working_counter != 1) {
+        if (!fsm->datagram->working_counter) {
+            if (diff_ms < EC_SOE_RESPONSE_TIMEOUT) {
+                // no response; send request datagram again
+                ec_fsm_soe_write_next_fragment(fsm, datagram);
+                return;
+            }
+        }
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Reception of SoE write request"
+                " failed after %lu ms: ", diff_ms);
+        ec_datagram_print_wc_error(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    // fragment successfully sent
+    fsm->offset += fsm->fragment_size;
+
+    if (fsm->offset < fsm->request->data_size) {
+        // next fragment
+        fsm->retries = EC_FSM_RETRIES;
+        ec_fsm_soe_write_next_fragment(fsm, datagram);
+        fsm->request->jiffies_sent = jiffies;
+    } else {
+        // all fragments sent; query response
+        fsm->jiffies_start = fsm->datagram->jiffies_sent;
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        fsm->state = ec_fsm_soe_write_check;
+    }
+}
+
+/****************************************************************************/
+
+/** CoE state: WRITE CHECK.
+ */
+void ec_fsm_soe_write_check(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        return;
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive SoE write request datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Reception of SoE write request datagram: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(fsm->datagram)) {
+        unsigned long diff_ms =
+            (datagram->jiffies_received - fsm->jiffies_start) * 1000 / HZ;
+        if (diff_ms >= EC_SOE_RESPONSE_TIMEOUT) {
+            fsm->state = ec_fsm_soe_error;
+            EC_SLAVE_ERR(slave, "Timeout after %lu ms while waiting"
+                    " for write response.\n", diff_ms);
+            ec_fsm_soe_print_error(fsm);
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        fsm->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    fsm->retries = EC_FSM_RETRIES;
+    fsm->state = ec_fsm_soe_write_response;
+}
+
+/****************************************************************************/
+
+/** SoE state: WRITE RESPONSE.
+ */
+void ec_fsm_soe_write_response(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+    ec_slave_t *slave = fsm->slave;
+    ec_master_t *master = slave->master;
+    ec_soe_request_t *req = fsm->request;
+    uint8_t *data, mbox_prot, opcode, error_flag;
+    uint16_t idn;
+    size_t rec_size;
+
+    if (fsm->datagram->state == EC_DATAGRAM_TIMED_OUT && fsm->retries--) {
+        ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+        return; // FIXME: request again?
+    }
+
+    if (fsm->datagram->state != EC_DATAGRAM_RECEIVED) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Failed to receive SoE write"
+                " response datagram: ");
+        ec_datagram_print_state(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (fsm->datagram->working_counter != 1) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Reception of SoE write response failed: ");
+        ec_datagram_print_wc_error(fsm->datagram);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, fsm->datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        fsm->state = ec_fsm_soe_error;
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (master->debug_level) {
+        EC_SLAVE_DBG(slave, 0, "SSC write response:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_SOE) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Received mailbox protocol 0x%02X as response.\n",
+                mbox_prot);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    if (rec_size < EC_SOE_SIZE) {
+        fsm->state = ec_fsm_soe_error;
+        EC_SLAVE_ERR(slave, "Received corrupted SoE write response"
+                " (%zu bytes)!\n", rec_size);
+        ec_print_data(data, rec_size);
+        ec_fsm_soe_print_error(fsm);
+        return;
+    }
+
+    opcode = EC_READ_U8(data) & 0x7;
+    if (opcode != OPCODE_WRITE_RESPONSE) {
+        EC_SLAVE_ERR(slave, "Received no write response"
+                " (opcode %x).\n", opcode);
+        ec_print_data(data, rec_size);
+        ec_fsm_soe_print_error(fsm);
+        fsm->state = ec_fsm_soe_error;
+        return;
+    }
+
+    idn = EC_READ_U16(data + 2);
+    if (idn != req->idn) {
+        EC_SLAVE_ERR(slave, "Received response for"
+                " wrong IDN 0x%04x.\n", idn);
+        ec_print_data(data, rec_size);
+        ec_fsm_soe_print_error(fsm);
+        fsm->state = ec_fsm_soe_error;
+        return;
+    }
+
+    error_flag = (EC_READ_U8(data) >> 4) & 1;
+    if (error_flag) {
+        if (rec_size < EC_SOE_SIZE + 2) {
+            EC_SLAVE_ERR(slave, "Received corrupted error response"
+                    " - error flag set, but received size is %zu.\n",
+                    rec_size);
+        } else {
+            req->error_code = EC_READ_U16(data + EC_SOE_SIZE);
+            EC_SLAVE_ERR(slave, "Received error response:\n");
+            ec_print_soe_error(slave, req->error_code);
+        }
+        ec_print_data(data, rec_size);
+        ec_fsm_soe_print_error(fsm);
+        fsm->state = ec_fsm_soe_error;
+    } else {
+        req->error_code = 0x0000;
+        fsm->state = ec_fsm_soe_end; // success
+    }
+}
+
+/****************************************************************************/
+
+/** State: ERROR.
+ */
+void ec_fsm_soe_error(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
+
+/** State: END.
+ */
+void ec_fsm_soe_end(
+        ec_fsm_soe_t *fsm, /**< finite state machine */
+        ec_datagram_t *datagram /**< Datagram to use. */
+        )
+{
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/fsm_soe.h b/drivers/net/ethercat/master/fsm_soe.h
new file mode 100644
index 000000000000..b99253d8d16c
--- /dev/null
+++ b/drivers/net/ethercat/master/fsm_soe.h
@@ -0,0 +1,67 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT CoE state machines.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_FSM_SOE_H__
+#define __EC_FSM_SOE_H__
+
+#include "globals.h"
+#include "datagram.h"
+#include "slave.h"
+#include "soe_request.h"
+
+/****************************************************************************/
+
+typedef struct ec_fsm_soe ec_fsm_soe_t; /**< \see ec_fsm_soe */
+
+/** Finite state machines for the Sercos over EtherCAT protocol.
+ */
+struct ec_fsm_soe {
+    ec_slave_t *slave; /**< slave the FSM runs on */
+    unsigned int retries; /**< retries upon datagram timeout */
+
+    void (*state)(ec_fsm_soe_t *, ec_datagram_t *); /**< CoE state function */
+    ec_datagram_t *datagram; /**< Datagram used in the previous step. */
+    unsigned long jiffies_start; /**< Timestamp. */
+    ec_soe_request_t *request; /**< SoE request */
+    off_t offset; /**< IDN data offset during fragmented write. */
+    size_t fragment_size; /**< Size of the current fragment. */
+};
+
+/****************************************************************************/
+
+void ec_fsm_soe_init(ec_fsm_soe_t *);
+void ec_fsm_soe_clear(ec_fsm_soe_t *);
+
+void ec_fsm_soe_transfer(ec_fsm_soe_t *, ec_slave_t *, ec_soe_request_t *);
+
+int ec_fsm_soe_exec(ec_fsm_soe_t *, ec_datagram_t *);
+int ec_fsm_soe_success(const ec_fsm_soe_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/globals.h b/drivers/net/ethercat/master/globals.h
new file mode 100644
index 000000000000..4945cbee3022
--- /dev/null
+++ b/drivers/net/ethercat/master/globals.h
@@ -0,0 +1,314 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2021  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT master.
+ *
+ *  The file is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU Lesser General Public License as published by the
+ *  Free Software Foundation; version 2.1 of the License.
+ *
+ *  This file is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+ *  License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this file. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+/** \file
+ * Global definitions and macros.
+ */
+
+/****************************************************************************/
+
+#ifndef __EC_MASTER_GLOBALS_H__
+#define __EC_MASTER_GLOBALS_H__
+
+#include "../include/globals.h"
+#include "../include/ecrt.h"
+
+/*****************************************************************************
+ * EtherCAT master
+ ****************************************************************************/
+
+/** Datagram timeout in microseconds. */
+#define EC_IO_TIMEOUT 500
+
+/** Time to send a byte in nanoseconds.
+ *
+ * t_ns = 1 / (100 MBit/s / 8 bit/byte) = 80 ns/byte
+ */
+#define EC_BYTE_TRANSMISSION_TIME_NS 80
+
+/** Number of state machine retries on datagram timeout. */
+#define EC_FSM_RETRIES 3
+
+/** Seconds to wait before fetching SDO dictionary
+    after slave entered PREOP state. */
+#define EC_WAIT_SDO_DICT 3
+
+/** Minimum size of a buffer used with ec_state_string(). */
+#define EC_STATE_STRING_SIZE 32
+
+/** Maximum SII size in words, to avoid infinite reading. */
+#define EC_MAX_SII_SIZE 4096
+
+/** Number of statistic rate intervals to maintain. */
+#define EC_RATE_COUNT 3
+
+/*****************************************************************************
+ * EtherCAT protocol
+ ****************************************************************************/
+
+/** Size of an EtherCAT frame header. */
+#define EC_FRAME_HEADER_SIZE 2
+
+/** Size of an EtherCAT datagram header. */
+#define EC_DATAGRAM_HEADER_SIZE 10
+
+/** Size of an EtherCAT datagram footer. */
+#define EC_DATAGRAM_FOOTER_SIZE 2
+
+/** Size of the EtherCAT address field. */
+#define EC_ADDR_LEN 4
+
+/** Resulting maximum data size of a single datagram in a frame. */
+#define EC_MAX_DATA_SIZE (ETH_DATA_LEN - EC_FRAME_HEADER_SIZE \
+                          - EC_DATAGRAM_HEADER_SIZE - EC_DATAGRAM_FOOTER_SIZE)
+
+/** Mailbox header size.  */
+#define EC_MBOX_HEADER_SIZE 6
+
+/** Word offset of first SII category. */
+#define EC_FIRST_SII_CATEGORY_OFFSET 0x40
+
+/** Size of a sync manager configuration page. */
+#define EC_SYNC_PAGE_SIZE 8
+
+/** Maximum number of FMMUs per slave. */
+#define EC_MAX_FMMUS 16
+
+/** Size of an FMMU configuration page. */
+#define EC_FMMU_PAGE_SIZE 16
+
+/** Number of DC sync signals. */
+#define EC_SYNC_SIGNAL_COUNT 2
+
+/** Size of the datagram description string.
+ *
+ * This is also used as the maximum lenth of EoE device names.
+ **/
+#define EC_DATAGRAM_NAME_SIZE 20
+
+/** Maximum hostname size.
+ *
+ * Used inside the EoE set IP parameter request.
+ */
+#define EC_MAX_HOSTNAME_SIZE 32
+
+/** Slave state mask.
+ *
+ * Apply this mask to a slave state byte to get the slave state without
+ * the error flag.
+ */
+#define EC_SLAVE_STATE_MASK 0x0F
+
+/** State of an EtherCAT slave.
+ */
+typedef enum {
+    EC_SLAVE_STATE_UNKNOWN = 0x00,
+    /**< unknown state */
+    EC_SLAVE_STATE_INIT = 0x01,
+    /**< INIT state (no mailbox communication, no IO) */
+    EC_SLAVE_STATE_PREOP = 0x02,
+    /**< PREOP state (mailbox communication, no IO) */
+    EC_SLAVE_STATE_BOOT = 0x03,
+    /**< Bootstrap state (mailbox communication, firmware update) */
+    EC_SLAVE_STATE_SAFEOP = 0x04,
+    /**< SAFEOP (mailbox communication and input update) */
+    EC_SLAVE_STATE_OP = 0x08,
+    /**< OP (mailbox communication and input/output update) */
+    EC_SLAVE_STATE_ACK_ERR = 0x10
+    /**< Acknowledge/Error bit (no actual state) */
+} ec_slave_state_t;
+
+/** Supported mailbox protocols.
+ *
+ * Not to mix up with the mailbox type field in the mailbox header defined in
+ * master/mailbox.h.
+ */
+enum {
+    EC_MBOX_AOE = 0x01, /**< ADS over EtherCAT */
+    EC_MBOX_EOE = 0x02, /**< Ethernet over EtherCAT */
+    EC_MBOX_COE = 0x04, /**< CANopen over EtherCAT */
+    EC_MBOX_FOE = 0x08, /**< File-Access over EtherCAT */
+    EC_MBOX_SOE = 0x10, /**< Servo-Profile over EtherCAT */
+    EC_MBOX_VOE = 0x20  /**< Vendor specific */
+};
+
+/** Slave information interface CANopen over EtherCAT details flags.
+ */
+typedef struct {
+    uint8_t enable_sdo : 1; /**< Enable SDO access. */
+    uint8_t enable_sdo_info : 1; /**< SDO information service available. */
+    uint8_t enable_pdo_assign : 1; /**< PDO mapping configurable. */
+    uint8_t enable_pdo_configuration : 1; /**< PDO configuration possible. */
+    uint8_t enable_upload_at_startup : 1; /**< ?. */
+    uint8_t enable_sdo_complete_access : 1; /**< Complete access possible. */
+} ec_sii_coe_details_t;
+
+/** Slave information interface general flags.
+ */
+typedef struct {
+    uint8_t enable_safeop : 1; /**< ?. */
+    uint8_t enable_not_lrw : 1; /**< Slave does not support LRW. */
+} ec_sii_general_flags_t;
+
+/** EtherCAT slave distributed clocks range.
+ */
+typedef enum {
+    EC_DC_32, /**< 32 bit. */
+    EC_DC_64 /*< 64 bit for system time, system time offset and
+               port 0 receive time. */
+} ec_slave_dc_range_t;
+
+/** EtherCAT slave sync signal configuration.
+ */
+typedef struct {
+    uint32_t cycle_time; /**< Cycle time [ns]. */
+    int32_t shift_time; /**< Shift time [ns]. */
+} ec_sync_signal_t;
+
+/** Access states for SDO entries.
+ *
+ * The access rights are managed per AL state.
+ */
+enum {
+    EC_SDO_ENTRY_ACCESS_PREOP, /**< Access rights in PREOP. */
+    EC_SDO_ENTRY_ACCESS_SAFEOP, /**< Access rights in SAFEOP. */
+    EC_SDO_ENTRY_ACCESS_OP, /**< Access rights in OP. */
+    EC_SDO_ENTRY_ACCESS_COUNT /**< Number of states. */
+};
+
+/** Master devices.
+ */
+typedef enum {
+    EC_DEVICE_MAIN, /**< Main device. */
+    EC_DEVICE_BACKUP /**< Backup device */
+} ec_device_index_t;
+
+extern const char *ec_device_names[2]; // only main and backup!
+
+/****************************************************************************/
+
+/** Convenience macro for printing EtherCAT-specific information to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT: ".
+ *
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_INFO(fmt, args...) \
+    printk(KERN_INFO "EtherCAT: " fmt, ##args)
+
+/** Convenience macro for printing EtherCAT-specific errors to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT ERROR: ".
+ *
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_ERR(fmt, args...) \
+    printk(KERN_ERR "EtherCAT ERROR: " fmt, ##args)
+
+/** Convenience macro for printing EtherCAT-specific warnings to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT WARNING: ".
+ *
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_WARN(fmt, args...) \
+    printk(KERN_WARNING "EtherCAT WARNING: " fmt, ##args)
+
+/** Convenience macro for printing EtherCAT debug messages to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT DEBUG: ".
+ *
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_DBG(fmt, args...) \
+    printk(KERN_DEBUG "EtherCAT DEBUG: " fmt, ##args)
+
+/****************************************************************************/
+
+/** Absolute value.
+ */
+#define EC_ABS(X) ((X) >= 0 ? (X) : -(X))
+
+/****************************************************************************/
+
+extern char *ec_master_version_str;
+
+/****************************************************************************/
+
+unsigned int ec_master_count(void);
+void ec_print_data(const uint8_t *, size_t);
+void ec_print_data_diff(const uint8_t *, const uint8_t *, size_t);
+size_t ec_state_string(uint8_t, char *, uint8_t);
+ssize_t ec_mac_print(const uint8_t *, char *);
+int ec_mac_is_zero(const uint8_t *);
+
+ec_master_t *ecrt_request_master_err(unsigned int);
+
+/****************************************************************************/
+
+/** Code/Message pair.
+ *
+ * Some EtherCAT datagrams support reading a status code to display a certain
+ * message. This type allows to map a code to a message string.
+ */
+typedef struct {
+    uint32_t code; /**< Code. */
+    const char *message; /**< Message belonging to \a code. */
+} ec_code_msg_t;
+
+/****************************************************************************/
+
+/** Generic request state.
+ *
+ * \attention If ever changing this, please be sure to adjust the \a
+ * state_table in master/sdo_request.c.
+ */
+typedef enum {
+    EC_INT_REQUEST_INIT,
+    EC_INT_REQUEST_QUEUED,
+    EC_INT_REQUEST_BUSY,
+    EC_INT_REQUEST_SUCCESS,
+    EC_INT_REQUEST_FAILURE
+} ec_internal_request_state_t;
+
+/****************************************************************************/
+
+extern const ec_request_state_t ec_request_state_translation_table[];
+
+/****************************************************************************/
+
+/** Origin type.
+ */
+typedef enum {
+    EC_ORIG_INTERNAL, /**< Internal. */
+    EC_ORIG_EXTERNAL /**< External. */
+} ec_origin_t;
+
+/****************************************************************************/
+
+typedef struct ec_slave ec_slave_t; /**< \see ec_slave. */
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/ioctl.c b/drivers/net/ethercat/master/ioctl.c
new file mode 100644
index 000000000000..c590d5d24d94
--- /dev/null
+++ b/drivers/net/ethercat/master/ioctl.c
@@ -0,0 +1,5591 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+/**
+   \file
+   EtherCAT master character device.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+
+#include "master.h"
+#include "slave_config.h"
+#include "voe_handler.h"
+#include "ethernet.h"
+#include "ioctl.h"
+
+/** Set to 1 to enable ioctl() latency tracing.
+ *
+ * Requires CPU timestamp counter!
+ */
+#define DEBUG_LATENCY 0
+
+/** Optional compiler attributes fo ioctl() functions.
+ */
+#if 0
+#define ATTRIBUTES __attribute__ ((__noinline__))
+#else
+#define ATTRIBUTES
+#endif
+
+#ifdef EC_IOCTL_RTDM
+# include "rtdm_details.h"
+/* RTDM does not support locking yet,
+ * therefore no send/receive callbacks are set too. */
+# define ec_ioctl_lock(lock) do {} while(0)
+# define ec_ioctl_unlock(lock) do {} while(0)
+# define ec_ioctl_lock_interruptible(lock) (0)
+# define ec_copy_to_user(to, from, n, ctx) \
+    rtdm_safe_copy_to_user(ec_ioctl_to_rtdm(ctx), to, from, n)
+# define ec_copy_from_user(to, from, n, ctx) \
+    rtdm_safe_copy_from_user(ec_ioctl_to_rtdm(ctx), to, from, n)
+#else
+# define ec_ioctl_lock(lock)   rt_mutex_lock(lock)
+# define ec_ioctl_unlock(lock) rt_mutex_unlock(lock)
+#  if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) || \
+      (defined(CONFIG_PREEMPT_RT_FULL) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+#   define ec_ioctl_lock_interruptible(lock) \
+           rt_mutex_lock_interruptible(lock)
+#  else
+#   define ec_ioctl_lock_interruptible(lock) \
+           rt_mutex_lock_interruptible(lock, 0)
+# endif
+# define ec_copy_to_user(to, from, n, ctx) copy_to_user(to, from, n)
+# define ec_copy_from_user(to, from, n, ctx) copy_from_user(to, from, n)
+#endif  // EC_IOCTL_RTDM
+
+/****************************************************************************/
+
+/** Copies a string to an ioctl structure.
+ */
+static void ec_ioctl_strcpy(
+        char *target, /**< Target. */
+        const char *source /**< Source. */
+        )
+{
+    if (source) {
+        strncpy(target, source, EC_IOCTL_STRING_SIZE);
+        target[EC_IOCTL_STRING_SIZE - 1] = 0;
+    } else {
+        target[0] = 0;
+    }
+}
+
+/****************************************************************************/
+
+/** Get module information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_module(
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_module_t data;
+
+    data.ioctl_version_magic = EC_IOCTL_VERSION_MAGIC;
+    data.master_count = ec_master_count();
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get master information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_master_t io;
+    unsigned int dev_idx, j;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    io.slave_count = master->slave_count;
+    io.scan_index = master->scan_index;
+    io.config_count = ec_master_config_count(master);
+    io.domain_count = ec_master_domain_count(master);
+#ifdef EC_EOE
+    io.eoe_handler_count = ec_master_eoe_handler_count(master);
+#else
+    io.eoe_handler_count = 0;
+#endif
+    io.phase = (uint8_t) master->phase;
+    io.active = (uint8_t) master->active;
+    io.scan_busy = master->scan_busy;
+
+    up(&master->master_sem);
+
+    if (down_interruptible(&master->device_sem)) {
+        return -EINTR;
+    }
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(master); dev_idx++) {
+        ec_device_t *device = &master->devices[dev_idx];
+
+        if (device->dev) {
+            memcpy(io.devices[dev_idx].address, device->dev->dev_addr,
+                    ETH_ALEN);
+        } else {
+            memcpy(io.devices[dev_idx].address, master->macs[dev_idx],
+                    ETH_ALEN);
+        }
+        io.devices[dev_idx].attached = device->dev ? 1 : 0;
+        io.devices[dev_idx].link_state = device->link_state ? 1 : 0;
+        io.devices[dev_idx].tx_count = device->tx_count;
+        io.devices[dev_idx].rx_count = device->rx_count;
+        io.devices[dev_idx].tx_bytes = device->tx_bytes;
+        io.devices[dev_idx].rx_bytes = device->rx_bytes;
+        io.devices[dev_idx].tx_errors = device->tx_errors;
+        for (j = 0; j < EC_RATE_COUNT; j++) {
+            io.devices[dev_idx].tx_frame_rates[j] =
+                device->tx_frame_rates[j];
+            io.devices[dev_idx].rx_frame_rates[j] =
+                device->rx_frame_rates[j];
+            io.devices[dev_idx].tx_byte_rates[j] =
+                device->tx_byte_rates[j];
+            io.devices[dev_idx].rx_byte_rates[j] =
+                device->rx_byte_rates[j];
+        }
+    }
+    io.num_devices = ec_master_num_devices(master);
+
+    io.tx_count = master->device_stats.tx_count;
+    io.rx_count = master->device_stats.rx_count;
+    io.tx_bytes = master->device_stats.tx_bytes;
+    io.rx_bytes = master->device_stats.rx_bytes;
+    for (j = 0; j < EC_RATE_COUNT; j++) {
+        io.tx_frame_rates[j] =
+            master->device_stats.tx_frame_rates[j];
+        io.rx_frame_rates[j] =
+            master->device_stats.rx_frame_rates[j];
+        io.tx_byte_rates[j] =
+            master->device_stats.tx_byte_rates[j];
+        io.rx_byte_rates[j] =
+            master->device_stats.rx_byte_rates[j];
+        io.loss_rates[j] =
+            master->device_stats.loss_rates[j];
+    }
+
+    up(&master->device_sem);
+
+    io.app_time = master->app_time;
+    io.dc_ref_time = master->dc_ref_time;
+    io.ref_clock =
+        master->dc_ref_clock ? master->dc_ref_clock->ring_position : 0xffff;
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_t data;
+    const ec_slave_t *slave;
+    int i;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n", data.position);
+        return -EINVAL;
+    }
+
+    data.device_index = slave->device_index;
+    data.vendor_id = slave->sii.vendor_id;
+    data.product_code = slave->sii.product_code;
+    data.revision_number = slave->sii.revision_number;
+    data.serial_number = slave->sii.serial_number;
+    data.alias = slave->effective_alias;
+    data.boot_rx_mailbox_offset = slave->sii.boot_rx_mailbox_offset;
+    data.boot_rx_mailbox_size = slave->sii.boot_rx_mailbox_size;
+    data.boot_tx_mailbox_offset = slave->sii.boot_tx_mailbox_offset;
+    data.boot_tx_mailbox_size = slave->sii.boot_tx_mailbox_size;
+    data.std_rx_mailbox_offset = slave->sii.std_rx_mailbox_offset;
+    data.std_rx_mailbox_size = slave->sii.std_rx_mailbox_size;
+    data.std_tx_mailbox_offset = slave->sii.std_tx_mailbox_offset;
+    data.std_tx_mailbox_size = slave->sii.std_tx_mailbox_size;
+    data.mailbox_protocols = slave->sii.mailbox_protocols;
+    data.has_general_category = slave->sii.has_general;
+    data.coe_details = slave->sii.coe_details;
+    data.general_flags = slave->sii.general_flags;
+    data.current_on_ebus = slave->sii.current_on_ebus;
+    for (i = 0; i < EC_MAX_PORTS; i++) {
+        data.ports[i].desc = slave->ports[i].desc;
+        data.ports[i].link.link_up = slave->ports[i].link.link_up;
+        data.ports[i].link.loop_closed = slave->ports[i].link.loop_closed;
+        data.ports[i].link.signal_detected =
+            slave->ports[i].link.signal_detected;
+        data.ports[i].receive_time = slave->ports[i].receive_time;
+        if (slave->ports[i].next_slave) {
+            data.ports[i].next_slave =
+                slave->ports[i].next_slave->ring_position;
+        } else {
+            data.ports[i].next_slave = 0xffff;
+        }
+        data.ports[i].delay_to_next_dc = slave->ports[i].delay_to_next_dc;
+    }
+    data.fmmu_bit = slave->base_fmmu_bit_operation;
+    data.dc_supported = slave->base_dc_supported;
+    data.dc_range = slave->base_dc_range;
+    data.has_dc_system_time = slave->has_dc_system_time;
+    data.transmission_delay = slave->transmission_delay;
+    data.al_state = slave->current_state;
+    data.error_flag = slave->error_flag;
+
+    data.sync_count = slave->sii.sync_count;
+    data.sdo_count = ec_slave_sdo_count(slave);
+    data.sii_nwords = slave->sii_nwords;
+    ec_ioctl_strcpy(data.group, slave->sii.group);
+    ec_ioctl_strcpy(data.image, slave->sii.image);
+    ec_ioctl_strcpy(data.order, slave->sii.order);
+    ec_ioctl_strcpy(data.name, slave->sii.name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave sync manager information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sync(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_sync_t data;
+    const ec_slave_t *slave;
+    const ec_sync_t *sync;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sync_index >= slave->sii.sync_count) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not exist!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    sync = &slave->sii.syncs[data.sync_index];
+
+    data.physical_start_address = sync->physical_start_address;
+    data.default_size = sync->default_length;
+    data.control_register = sync->control_register;
+    data.enable = sync->enable;
+    data.pdo_count = ec_pdo_list_count(&sync->pdos);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave sync manager PDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sync_pdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_sync_pdo_t data;
+    const ec_slave_t *slave;
+    const ec_sync_t *sync;
+    const ec_pdo_t *pdo;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sync_index >= slave->sii.sync_count) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not exist!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    sync = &slave->sii.syncs[data.sync_index];
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sync->pdos, data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not contain a PDO with "
+                "position %u!\n", data.sync_index, data.pdo_pos);
+        return -EINVAL;
+    }
+
+    data.index = pdo->index;
+    data.entry_count = ec_pdo_entry_count(pdo);
+    ec_ioctl_strcpy(data.name, pdo->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave sync manager PDO entry information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sync_pdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_sync_pdo_entry_t data;
+    const ec_slave_t *slave;
+    const ec_sync_t *sync;
+    const ec_pdo_t *pdo;
+    const ec_pdo_entry_t *entry;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sync_index >= slave->sii.sync_count) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not exist!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    sync = &slave->sii.syncs[data.sync_index];
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sync->pdos, data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not contain a PDO with "
+                "position %u!\n", data.sync_index, data.pdo_pos);
+        return -EINVAL;
+    }
+
+    if (!(entry = ec_pdo_find_entry_by_pos_const(
+                    pdo, data.entry_pos))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "PDO 0x%04X does not contain an entry with "
+                "position %u!\n", data.pdo_pos, data.entry_pos);
+        return -EINVAL;
+    }
+
+    data.index = entry->index;
+    data.subindex = entry->subindex;
+    data.bit_length = entry->bit_length;
+    ec_ioctl_strcpy(data.name, entry->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get domain information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_domain_t data;
+    const ec_domain_t *domain;
+    unsigned int dev_idx;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(domain = ec_master_find_domain_const(master, data.index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u does not exist!\n", data.index);
+        return -EINVAL;
+    }
+
+    data.data_size = domain->data_size;
+    data.logical_base_address = domain->logical_base_address;
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        data.working_counter[dev_idx] = domain->working_counter[dev_idx];
+    }
+    data.expected_working_counter = domain->expected_working_counter;
+    data.fmmu_count = ec_domain_fmmu_count(domain);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get domain FMMU information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_fmmu(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_domain_fmmu_t data;
+    const ec_domain_t *domain;
+    const ec_fmmu_config_t *fmmu;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(domain = ec_master_find_domain_const(master, data.domain_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u does not exist!\n",
+                data.domain_index);
+        return -EINVAL;
+    }
+
+    if (!(fmmu = ec_domain_find_fmmu(domain, data.fmmu_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u has less than %u"
+                " fmmu configurations.\n",
+                data.domain_index, data.fmmu_index + 1);
+        return -EINVAL;
+    }
+
+    data.slave_config_alias = fmmu->sc->alias;
+    data.slave_config_position = fmmu->sc->position;
+    data.sync_index = fmmu->sync_index;
+    data.dir = fmmu->dir;
+    data.logical_address = fmmu->logical_start_address;
+    data.data_size = fmmu->data_size;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get domain data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_domain_data_t data;
+    const ec_domain_t *domain;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(domain = ec_master_find_domain_const(master, data.domain_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u does not exist!\n",
+                data.domain_index);
+        return -EINVAL;
+    }
+
+    if (domain->data_size != data.data_size) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Data size mismatch %u/%zu!\n",
+                data.data_size, domain->data_size);
+        return -EFAULT;
+    }
+
+    if (copy_to_user((void __user *) data.target, domain->data,
+                domain->data_size)) {
+        up(&master->master_sem);
+        return -EFAULT;
+    }
+
+    up(&master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Set master debug level.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master_debug(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    return ec_master_debug_level(master, (unsigned long) arg);
+}
+
+/****************************************************************************/
+
+/** Issue a bus scan.
+ *
+ * \return Always zero (success).
+ */
+static ATTRIBUTES int ec_ioctl_master_rescan(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    EC_MASTER_DBG(master, 1, "Got rescan command via ioctl()."
+		    " Re-scanning on next possibility.\n");
+    master->fsm.rescan_required = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Set slave state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_state_t data;
+    ec_slave_t *slave;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    ec_slave_request_state(slave, data.al_state);
+
+    up(&master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave SDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_t data;
+    const ec_slave_t *slave;
+    const ec_sdo_t *sdo;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (!(sdo = ec_slave_get_sdo_by_pos_const(
+                    slave, data.sdo_position))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "SDO %u does not exist!\n", data.sdo_position);
+        return -EINVAL;
+    }
+
+    data.sdo_index = sdo->index;
+    data.max_subindex = sdo->max_subindex;
+    ec_ioctl_strcpy(data.name, sdo->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave SDO entry information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_entry_t data;
+    const ec_slave_t *slave;
+    const ec_sdo_t *sdo;
+    const ec_sdo_entry_t *entry;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sdo_spec <= 0) {
+        if (!(sdo = ec_slave_get_sdo_by_pos_const(
+                        slave, -data.sdo_spec))) {
+            up(&master->master_sem);
+            EC_SLAVE_ERR(slave, "SDO %u does not exist!\n", -data.sdo_spec);
+            return -EINVAL;
+        }
+    } else {
+        if (!(sdo = ec_slave_get_sdo_const(
+                        slave, data.sdo_spec))) {
+            up(&master->master_sem);
+            EC_SLAVE_ERR(slave, "SDO 0x%04X does not exist!\n",
+                    data.sdo_spec);
+            return -EINVAL;
+        }
+    }
+
+    if (!(entry = ec_sdo_get_entry_const(
+                    sdo, data.sdo_entry_subindex))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "SDO entry 0x%04X:%02X does not exist!\n",
+                sdo->index, data.sdo_entry_subindex);
+        return -EINVAL;
+    }
+
+    data.data_type = entry->data_type;
+    data.bit_length = entry->bit_length;
+    data.read_access[EC_SDO_ENTRY_ACCESS_PREOP] =
+        entry->read_access[EC_SDO_ENTRY_ACCESS_PREOP];
+    data.read_access[EC_SDO_ENTRY_ACCESS_SAFEOP] =
+        entry->read_access[EC_SDO_ENTRY_ACCESS_SAFEOP];
+    data.read_access[EC_SDO_ENTRY_ACCESS_OP] =
+        entry->read_access[EC_SDO_ENTRY_ACCESS_OP];
+    data.write_access[EC_SDO_ENTRY_ACCESS_PREOP] =
+        entry->write_access[EC_SDO_ENTRY_ACCESS_PREOP];
+    data.write_access[EC_SDO_ENTRY_ACCESS_SAFEOP] =
+        entry->write_access[EC_SDO_ENTRY_ACCESS_SAFEOP];
+    data.write_access[EC_SDO_ENTRY_ACCESS_OP] =
+        entry->write_access[EC_SDO_ENTRY_ACCESS_OP];
+    ec_ioctl_strcpy(data.description, entry->description);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Upload SDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo_upload(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_upload_t data;
+    uint8_t *target;
+    int ret;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (!(target = kmalloc(data.target_size, GFP_KERNEL))) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes"
+                " for SDO upload.\n", data.target_size);
+        return -ENOMEM;
+    }
+
+    ret = ecrt_master_sdo_upload(master, data.slave_position,
+            data.sdo_index, data.sdo_entry_subindex, target,
+            data.target_size, &data.data_size, &data.abort_code);
+
+    if (!ret) {
+        if (copy_to_user((void __user *) data.target,
+                    target, data.data_size)) {
+            kfree(target);
+            return -EFAULT;
+        }
+    }
+
+    kfree(target);
+
+    if (__copy_to_user((void __user *) arg, &data, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Download SDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo_download(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_download_t data;
+    uint8_t *sdo_data;
+    int retval;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (!(sdo_data = kmalloc(data.data_size, GFP_KERNEL))) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes"
+                " for SDO download.\n", data.data_size);
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(sdo_data, (void __user *) data.data, data.data_size)) {
+        kfree(sdo_data);
+        return -EFAULT;
+    }
+
+    if (data.complete_access) {
+        retval = ecrt_master_sdo_download_complete(master, data.slave_position,
+                data.sdo_index, sdo_data, data.data_size, &data.abort_code);
+    } else {
+        retval = ecrt_master_sdo_download(master, data.slave_position,
+                data.sdo_index, data.sdo_entry_subindex, sdo_data,
+                data.data_size, &data.abort_code);
+    }
+
+    kfree(sdo_data);
+
+    if (__copy_to_user((void __user *) arg, &data, sizeof(data))) {
+        retval = -EFAULT;
+    }
+
+    return retval;
+}
+
+/****************************************************************************/
+
+/** Read a slave's SII.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sii_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sii_t data;
+    const ec_slave_t *slave;
+    int retval;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (!data.nwords
+            || data.offset + data.nwords > slave->sii_nwords) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Invalid SII read offset/size %u/%u for slave SII"
+                " size %zu!\n", data.offset, data.nwords, slave->sii_nwords);
+        return -EINVAL;
+    }
+
+    if (copy_to_user((void __user *) data.words,
+                slave->sii_words + data.offset, data.nwords * 2))
+        retval = -EFAULT;
+    else
+        retval = 0;
+
+    up(&master->master_sem);
+    return retval;
+}
+
+/****************************************************************************/
+
+/** Write a slave's SII.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sii_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sii_t data;
+    ec_slave_t *slave;
+    unsigned int byte_size;
+    uint16_t *words;
+    ec_sii_write_request_t request;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (!data.nwords) {
+        return 0;
+    }
+
+    byte_size = sizeof(uint16_t) * data.nwords;
+    if (!(words = kmalloc(byte_size, GFP_KERNEL))) {
+        EC_MASTER_ERR(master, "Failed to allocate %u bytes"
+                " for SII contents.\n", byte_size);
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(words,
+                (void __user *) data.words, byte_size)) {
+        kfree(words);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(words);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        kfree(words);
+        return -EINVAL;
+    }
+
+    // init SII write request
+    INIT_LIST_HEAD(&request.list);
+    request.slave = slave;
+    request.words = words;
+    request.offset = data.offset;
+    request.nwords = data.nwords;
+    request.state = EC_INT_REQUEST_QUEUED;
+
+    // schedule SII write request.
+    list_add_tail(&request.list, &master->sii_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            kfree(words);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    kfree(words);
+
+    return request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+
+/****************************************************************************/
+
+/** Read a slave's registers.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_reg_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_reg_t io;
+    ec_slave_t *slave;
+    ec_reg_request_t request;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (!io.size) {
+        return 0;
+    }
+
+    // init register request
+    ret = ec_reg_request_init(&request, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    ret = ecrt_reg_request_read(&request, io.address, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_reg_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        ec_reg_request_clear(&request);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        return -EINVAL;
+    }
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->reg_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_reg_request_clear(&request);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    if (request.state == EC_INT_REQUEST_SUCCESS) {
+        if (copy_to_user((void __user *) io.data, request.data, io.size)) {
+            return -EFAULT;
+        }
+    }
+    ec_reg_request_clear(&request);
+
+    return request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+
+/****************************************************************************/
+
+/** Write a slave's registers.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_reg_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_reg_t io;
+    ec_slave_t *slave;
+    ec_reg_request_t request;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (!io.size) {
+        return 0;
+    }
+
+    // init register request
+    ret = ec_reg_request_init(&request, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    if (copy_from_user(request.data, (void __user *) io.data, io.size)) {
+        ec_reg_request_clear(&request);
+        return -EFAULT;
+    }
+
+    ret = ecrt_reg_request_write(&request, io.address, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_reg_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (io.emergency) {
+        request.ring_position = io.slave_position;
+        // schedule request.
+        list_add_tail(&request.list, &master->emerg_reg_requests);
+    }
+    else {
+        if (!(slave = ec_master_find_slave(master, 0, io.slave_position))) {
+            up(&master->master_sem);
+            ec_reg_request_clear(&request);
+            EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                    io.slave_position);
+            return -EINVAL;
+        }
+
+        // schedule request.
+        list_add_tail(&request.list, &slave->reg_requests);
+    }
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_reg_request_clear(&request);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    ec_reg_request_clear(&request);
+
+    return request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_t data;
+    const ec_slave_config_t *sc;
+    uint8_t i;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config_const(
+                    master, data.config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                data.config_index);
+        return -EINVAL;
+    }
+
+    data.alias = sc->alias;
+    data.position = sc->position;
+    data.vendor_id = sc->vendor_id;
+    data.product_code = sc->product_code;
+    for (i = 0; i < EC_MAX_SYNC_MANAGERS; i++) {
+        data.syncs[i].dir = sc->sync_configs[i].dir;
+        data.syncs[i].watchdog_mode = sc->sync_configs[i].watchdog_mode;
+        data.syncs[i].pdo_count =
+            ec_pdo_list_count(&sc->sync_configs[i].pdos);
+    }
+    data.watchdog_divider = sc->watchdog_divider;
+    data.watchdog_intervals = sc->watchdog_intervals;
+    data.sdo_count = ec_slave_config_sdo_count(sc);
+    data.idn_count = ec_slave_config_idn_count(sc);
+    data.flag_count = ec_slave_config_flag_count(sc);
+    data.slave_position = sc->slave ? sc->slave->ring_position : -1;
+    data.dc_assign_activate = sc->dc_assign_activate;
+    for (i = 0; i < EC_SYNC_SIGNAL_COUNT; i++) {
+        data.dc_sync[i] = sc->dc_sync[i];
+    }
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration PDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_pdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    const ec_slave_config_t *sc;
+    const ec_pdo_t *pdo;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (data.sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_MASTER_ERR(master, "Invalid sync manager index %u!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config_const(
+                    master, data.config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                data.config_index);
+        return -EINVAL;
+    }
+
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sc->sync_configs[data.sync_index].pdos,
+                    data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid PDO position!\n");
+        return -EINVAL;
+    }
+
+    data.index = pdo->index;
+    data.entry_count = ec_pdo_entry_count(pdo);
+    ec_ioctl_strcpy(data.name, pdo->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration PDO entry information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_pdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_pdo_entry_t data;
+    const ec_slave_config_t *sc;
+    const ec_pdo_t *pdo;
+    const ec_pdo_entry_t *entry;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (data.sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_MASTER_ERR(master, "Invalid sync manager index %u!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config_const(
+                    master, data.config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                data.config_index);
+        return -EINVAL;
+    }
+
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sc->sync_configs[data.sync_index].pdos,
+                    data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid PDO position!\n");
+        return -EINVAL;
+    }
+
+    if (!(entry = ec_pdo_find_entry_by_pos_const(
+                    pdo, data.entry_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Entry not found!\n");
+        return -EINVAL;
+    }
+
+    data.index = entry->index;
+    data.subindex = entry->subindex;
+    data.bit_length = entry->bit_length;
+    ec_ioctl_strcpy(data.name, entry->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration SDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_sdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_sdo_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_sdo_request_t *req;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(
+                    master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    if (!(req = ec_slave_config_get_sdo_by_pos_const(
+                    sc, ioctl->sdo_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid SDO position!\n");
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    ioctl->index = req->index;
+    ioctl->subindex = req->subindex;
+    ioctl->size = req->data_size;
+    memcpy(ioctl->data, req->data,
+            min((u32) ioctl->size, (u32) EC_MAX_SDO_DATA_SIZE));
+    ioctl->complete_access = req->complete_access;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration IDN information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_idn(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_idn_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_soe_request_t *req;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(
+                    master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    if (!(req = ec_slave_config_get_idn_by_pos_const(
+                    sc, ioctl->idn_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid IDN position!\n");
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    ioctl->drive_no = req->drive_no;
+    ioctl->idn = req->idn;
+    ioctl->state = req->al_state;
+    ioctl->size = req->data_size;
+    memcpy(ioctl->data, req->data,
+            min((u32) ioctl->size, (u32) EC_MAX_IDN_DATA_SIZE));
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration feature flag information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_flag(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_flag_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_flag_t *flag;
+    size_t size;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(
+                    master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    if (!(flag = ec_slave_config_get_flag_by_pos_const(
+                    sc, ioctl->flag_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid flag position!\n");
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    size = min((u32) strlen(flag->key), (u32) EC_MAX_FLAG_KEY_SIZE - 1);
+    memcpy(ioctl->key, flag->key, size);
+    ioctl->key[size] = 0x00;
+    ioctl->value = flag->value;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/** Get configured EoE IP parameters for a given slave configuration.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_ip(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_eoe_ip_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_eoe_request_t *req;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    req = &sc->eoe_ip_param_request;
+
+    ioctl->mac_address_included = req->mac_address_included;
+    ioctl->ip_address_included = req->ip_address_included;
+    ioctl->subnet_mask_included = req->subnet_mask_included;
+    ioctl->gateway_included = req->gateway_included;
+    ioctl->dns_included = req->dns_included;
+    ioctl->name_included = req->name_included;
+
+    memcpy(ioctl->mac_address, req->mac_address, EC_ETH_ALEN);
+    ioctl->ip_address = req->ip_address;
+    ioctl->subnet_mask = req->subnet_mask;
+    ioctl->gateway = req->gateway;
+    ioctl->dns = req->dns;
+    strncpy(ioctl->name, req->name, EC_MAX_HOSTNAME_SIZE);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/** Get EoE handler information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_eoe_handler(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_eoe_handler_t data;
+    const ec_eoe_t *eoe;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(eoe = ec_master_get_eoe_handler_const(master, data.eoe_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "EoE handler %u does not exist!\n",
+                data.eoe_index);
+        return -EINVAL;
+    }
+
+    if (eoe->slave) {
+        data.slave_position = eoe->slave->ring_position;
+    } else {
+        data.slave_position = 0xffff;
+    }
+    snprintf(data.name, EC_DATAGRAM_NAME_SIZE, eoe->dev->name);
+    data.open = eoe->opened;
+    data.rx_bytes = eoe->stats.tx_bytes;
+    data.rx_rate = eoe->tx_rate;
+    data.tx_bytes = eoe->stats.rx_bytes;
+    data.tx_rate = eoe->tx_rate;
+    data.tx_queued_frames = eoe->tx_queued_frames;
+    data.tx_queue_size = eoe->tx_queue_size;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+/** Request EoE IP parameter setting.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_eoe_ip_param(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_eoe_ip_t io;
+    ec_eoe_request_t req;
+    ec_slave_t *slave;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    // init EoE request
+    ec_eoe_request_init(&req);
+
+    req.mac_address_included = io.mac_address_included;
+    req.ip_address_included = io.ip_address_included;
+    req.subnet_mask_included = io.subnet_mask_included;
+    req.gateway_included = io.gateway_included;
+    req.dns_included = io.dns_included;
+    req.name_included = io.name_included;
+
+    memcpy(req.mac_address, io.mac_address, EC_ETH_ALEN);
+    req.ip_address = io.ip_address;
+    req.subnet_mask = io.subnet_mask;
+    req.gateway = io.gateway;
+    req.dns = io.dns;
+    memcpy(req.name, io.name, EC_MAX_HOSTNAME_SIZE);
+
+    req.state = EC_INT_REQUEST_QUEUED;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        return -EINVAL;
+    }
+
+    EC_MASTER_DBG(master, 1, "Scheduling EoE request.\n");
+
+    // schedule request.
+    list_add_tail(&req.list, &slave->eoe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                req.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (req.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&req.list);
+            up(&master->master_sem);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, req.state != EC_INT_REQUEST_BUSY);
+
+    io.result = req.result;
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    return req.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+#endif
+
+/*****************************************************************************/
+
+/** Request the master from userspace.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_master_t *m;
+    int ret = 0;
+
+    m = ecrt_request_master_err(master->index);
+    if (IS_ERR(m)) {
+        ret = PTR_ERR(m);
+    } else {
+        ctx->requested = 1;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Create a domain.
+ *
+ * \return Domain index on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_create_domain(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    domain = ecrt_master_create_domain_err(master);
+    if (IS_ERR(domain))
+        return PTR_ERR(domain);
+
+    return domain->index;
+}
+
+/****************************************************************************/
+
+/** Create a slave configuration.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_create_slave_config(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc, *entry;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    sc = ecrt_master_slave_config_err(master, data.alias, data.position,
+            data.vendor_id, data.product_code);
+    if (IS_ERR(sc))
+        return PTR_ERR(sc);
+
+    data.config_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    list_for_each_entry(entry, &master->configs, list) {
+        if (entry == sc)
+            break;
+        data.config_index++;
+    }
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Select the DC reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_select_ref_clock(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    unsigned long config_index = (unsigned long) arg;
+    ec_slave_config_t *sc = NULL;
+    int ret = 0;
+
+    if (unlikely(!ctx->requested)) {
+        ret = -EPERM;
+        goto out_return;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ret = -EINTR;
+        goto out_return;
+    }
+
+    if (config_index != 0xFFFFFFFF) {
+        if (!(sc = ec_master_get_config(master, config_index))) {
+            ret = -ENOENT;
+            goto out_up;
+        }
+    }
+
+    ecrt_master_select_reference_clock(master, sc);
+
+out_up:
+    up(&master->master_sem);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Activates the master.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_activate(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_master_activate_t io;
+    ec_domain_t *domain;
+    off_t offset;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    io.process_data = NULL;
+
+    /* Get the sum of the domains' process data sizes. */
+
+    ctx->process_data_size = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    list_for_each_entry(domain, &master->domains, list) {
+        ctx->process_data_size += ecrt_domain_size(domain);
+    }
+
+    up(&master->master_sem);
+
+    if (ctx->process_data_size) {
+        ctx->process_data = vmalloc(ctx->process_data_size);
+        if (!ctx->process_data) {
+            ctx->process_data_size = 0;
+            return -ENOMEM;
+        }
+
+        /* Set the memory as external process data memory for the
+         * domains.
+         */
+        offset = 0;
+        list_for_each_entry(domain, &master->domains, list) {
+            ecrt_domain_external_memory(domain,
+                    ctx->process_data + offset);
+            offset += ecrt_domain_size(domain);
+        }
+
+#if defined(EC_IOCTL_RTDM) && !defined(EC_RTDM_XENOMAI_V3)
+        /* RTDM uses a different approach for memory-mapping, which has to be
+         * initiated by the kernel.
+         */
+        ret = ec_rtdm_mmap(ctx, &io.process_data);
+        if (ret < 0) {
+            EC_MASTER_ERR(master, "Failed to map process data"
+                    " memory to user space (code %i).\n", ret);
+            return ret;
+        }
+#endif
+    }
+
+    io.process_data_size = ctx->process_data_size;
+
+#ifndef EC_IOCTL_RTDM
+    /* RTDM does not support locking yet. */
+    ecrt_master_callbacks(master, ec_master_internal_send_cb,
+            ec_master_internal_receive_cb, master);
+#endif
+
+    ret = ecrt_master_activate(master);
+    if (ret < 0)
+        return ret;
+
+    if (copy_to_user((void __user *) arg, &io,
+                sizeof(ec_ioctl_master_activate_t)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Deactivates the master.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_deactivate(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    return ecrt_master_deactivate(master);
+}
+
+/****************************************************************************/
+
+/** Set max. number of databytes in a cycle
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_set_send_interval(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    size_t send_interval;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&send_interval, (void __user *) arg,
+                sizeof(send_interval))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    ec_master_set_send_interval(master, send_interval);
+
+    up(&master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Send frames.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_send(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_send(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Receive frames.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_receive(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_receive(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get the master state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_master_state_t data;
+    int ret;
+
+    ret = ecrt_master_state(master, &data);
+    if (ret)
+        return ret;
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get the link state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master_link_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_link_state_t ioctl;
+    ec_master_link_state_t state;
+    int ret;
+
+    if (ec_copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl), ctx)) {
+        return -EFAULT;
+    }
+
+    ret = ecrt_master_link_state(master, ioctl.dev_idx, &state);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ec_copy_to_user((void __user *) ioctl.state,
+                        &state, sizeof(state), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Set the master DC application time.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_app_time(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    uint64_t time;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&time, (void __user *) arg, sizeof(time), ctx)) {
+        return -EFAULT;
+    }
+
+    return ecrt_master_application_time(master, time);
+}
+
+/****************************************************************************/
+
+/** Sync the reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_ref(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_reference_clock(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sync the reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_ref_to(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+    uint64_t time;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&time, (void __user *) arg, sizeof(time), ctx)) {
+        return -EFAULT;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_reference_clock_to(master, time);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sync the slave clocks.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_slaves(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_slave_clocks(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get the system time of the reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_ref_clock_time(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    uint32_t time;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    ret = ecrt_master_reference_clock_time(master, &time);
+    if (ret) {
+        return ret;
+    }
+
+    if (ec_copy_to_user((void __user *) arg, &time, sizeof(time), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Queue the sync monitoring datagram.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_mon_queue(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_monitor_queue(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Processes the sync monitoring datagram.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_mon_process(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    uint32_t time_diff;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    time_diff = ecrt_master_sync_monitor_process(master);
+
+    if (ec_copy_to_user((void __user *) arg, &time_diff,
+                        sizeof(time_diff), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Reset configuration.
+ *
+ * \return Always zero (success).
+ */
+static ATTRIBUTES int ec_ioctl_reset(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+#ifdef EC_IOCTL_RTDM
+    /* Xenomai/LXRT is like NMI context, so we do a two-stage schedule. */
+    irq_work_queue(&master->sc_reset_work_kicker);
+#else
+    schedule_work(&master->sc_reset_work);
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Configure a sync manager.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_sync(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc;
+    unsigned int i;
+    int ret = 0;
+
+    if (unlikely(!ctx->requested)) {
+        ret = -EPERM;
+        goto out_return;
+    }
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        ret = -EFAULT;
+        goto out_return;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ret = -EINTR;
+        goto out_return;
+    }
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        ret = -ENOENT;
+        goto out_up;
+    }
+
+    for (i = 0; i < EC_MAX_SYNC_MANAGERS; i++) {
+        if (data.syncs[i].config_this) {
+            ret = ecrt_slave_config_sync_manager(sc, i, data.syncs[i].dir,
+                        data.syncs[i].watchdog_mode);
+            if (ret) {
+                goto out_up;
+            }
+        }
+    }
+
+out_up:
+    up(&master->master_sem);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Configure a slave's watchdogs.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_watchdog(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc;
+    int ret = 0;
+
+    if (unlikely(!ctx->requested)) {
+        ret = -EPERM;
+        goto out_return;
+    }
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        ret = -EFAULT;
+        goto out_return;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ret = -EINTR;
+        goto out_return;
+    }
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        ret = -ENOENT;
+        goto out_up;
+    }
+
+    ret = ecrt_slave_config_watchdog(sc,
+            data.watchdog_divider, data.watchdog_intervals);
+
+out_up:
+    up(&master->master_sem);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Add a PDO to the assignment.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_add_pdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_assign_add(sc, data.sync_index, data.index);
+}
+
+/****************************************************************************/
+
+/** Clears the PDO assignment.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_clear_pdos(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_assign_clear(sc, data.sync_index);
+}
+
+/****************************************************************************/
+
+/** Add an entry to a PDO's mapping.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_add_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_add_pdo_entry_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_mapping_add(sc, data.pdo_index,
+            data.entry_index, data.entry_subindex, data.entry_bit_length);
+}
+
+/****************************************************************************/
+
+/** Clears the mapping of a PDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_clear_entries(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_mapping_clear(sc, data.index);
+}
+
+/****************************************************************************/
+
+/** Registers a PDO entry.
+ *
+ * \return Process data offset on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_reg_pdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_pdo_entry_t data;
+    ec_slave_config_t *sc;
+    ec_domain_t *domain;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    if (!(domain = ec_master_find_domain(master, data.domain_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc or domain could be invalidated */
+
+    ret = ecrt_slave_config_reg_pdo_entry(sc, data.entry_index,
+            data.entry_subindex, domain, &data.bit_position);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Registers a PDO entry by its position.
+ *
+ * \return Process data offset on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_reg_pdo_pos(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_pdo_pos_t io;
+    ec_slave_config_t *sc;
+    ec_domain_t *domain;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    if (!(domain = ec_master_find_domain(master, io.domain_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc or domain could be invalidated */
+
+    ret = ecrt_slave_config_reg_pdo_entry_pos(sc, io.sync_index,
+            io.pdo_pos, io.entry_pos, domain, &io.bit_position);
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io)))
+        return -EFAULT;
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sets the DC AssignActivate word and the sync signal times.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_dc(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_dc(sc, data.dc_assign_activate,
+            data.dc_sync[0].cycle_time,
+            data.dc_sync[0].shift_time,
+            data.dc_sync[1].cycle_time,
+            data.dc_sync[1].shift_time);
+
+    up(&master->master_sem);
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Configures an SDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_sdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_sdo_t data;
+    ec_slave_config_t *sc;
+    uint8_t *sdo_data = NULL;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (!data.size)
+        return -EINVAL;
+
+    if (!(sdo_data = kmalloc(data.size, GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(sdo_data, (void __user *) data.data, data.size)) {
+        kfree(sdo_data);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(sdo_data);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        kfree(sdo_data);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    if (data.complete_access) {
+        ret = ecrt_slave_config_complete_sdo(sc,
+                data.index, sdo_data, data.size);
+    } else {
+        ret = ecrt_slave_config_sdo(sc, data.index, data.subindex, sdo_data,
+                data.size);
+    }
+    kfree(sdo_data);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Set the emergency ring buffer size.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_size(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_emerg_size(sc, io.size);
+
+    up(&master->master_sem);
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get an emergency message from the ring.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_pop(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+    u8 msg[EC_COE_EMERGENCY_MSG_SIZE];
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because configuration will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_emerg_pop(sc, msg);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ec_copy_to_user((void __user *) io.target, msg, sizeof(msg), ctx)) {
+        return -EFAULT;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Clear the emergency ring.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_clear(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because configuration will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_slave_config_emerg_clear(sc);
+}
+
+/****************************************************************************/
+
+/** Get the number of emergency overruns.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_overruns(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because configuration will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_emerg_overruns(sc);
+    if (ret < 0) {
+        return ret;
+    }
+
+    io.overruns = ret;
+
+    if (ec_copy_to_user((void __user *) arg, &io, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create an SDO request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_sdo_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    data.request_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    sc = ec_master_get_config(master, data.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(req, &sc->sdo_requests, list) {
+        data.request_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    req = ecrt_slave_config_create_sdo_request_err(sc, data.sdo_index,
+            data.sdo_subindex, data.size);
+    if (IS_ERR(req))
+        return PTR_ERR(req);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create an SoE request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_soe_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    data.request_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    sc = ec_master_get_config(master, data.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(req, &sc->soe_requests, list) {
+        data.request_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    req = ecrt_slave_config_create_soe_request_err(sc, data.drive_no,
+            data.idn, data.size);
+    if (IS_ERR(req)) {
+        return PTR_ERR(req);
+    }
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create a register request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_reg_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    io.request_index = 0;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    sc = ec_master_get_config(master, io.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(reg, &sc->reg_requests, list) {
+        io.request_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    reg = ecrt_slave_config_create_reg_request_err(sc, io.mem_size);
+    if (IS_ERR(reg)) {
+        return PTR_ERR(reg);
+    }
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create a VoE handler.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_voe_handler(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    data.voe_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    sc = ec_master_get_config(master, data.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(voe, &sc->voe_handlers, list) {
+        data.voe_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    voe = ecrt_slave_config_create_voe_handler_err(sc, data.size);
+    if (IS_ERR(voe))
+        return PTR_ERR(voe);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get the slave configuration's state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_state_t data;
+    const ec_slave_config_t *sc;
+    ec_slave_config_state_t state;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because sc will not be deleted in the
+     * meantime. */
+
+    if (!(sc = ec_master_get_config_const(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_state(sc, &state);
+    if (ret)
+        return ret;
+
+    if (ec_copy_to_user((void __user *) data.state,
+                        &state, sizeof(state), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Configures an IDN.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_idn(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_idn_t ioctl;
+    ec_slave_config_t *sc;
+    uint8_t *data = NULL;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl)))
+        return -EFAULT;
+
+    if (!ioctl.size)
+        return -EINVAL;
+
+    if (!(data = kmalloc(ioctl.size, GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(data, (void __user *) ioctl.data, ioctl.size)) {
+        kfree(data);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(data);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, ioctl.config_index))) {
+        up(&master->master_sem);
+        kfree(data);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    ret = ecrt_slave_config_idn(
+            sc, ioctl.drive_no, ioctl.idn, ioctl.al_state, data, ioctl.size);
+    kfree(data);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Configures a feature flag.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_flag(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_flag_t ioctl;
+    ec_slave_config_t *sc;
+    uint8_t *key;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    if (!ioctl.key_size) {
+        return -EINVAL;
+    }
+
+    if (!(key = kmalloc(ioctl.key_size + 1, GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(key, (void __user *) ioctl.key, ioctl.key_size)) {
+        kfree(key);
+        return -EFAULT;
+    }
+    key[ioctl.key_size] = '\0';
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(key);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, ioctl.config_index))) {
+        up(&master->master_sem);
+        kfree(key);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    ret = ecrt_slave_config_flag(sc, key, ioctl.value);
+    kfree(key);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sets an AL state transition timeout.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_state_timeout(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_state_timeout_t ioctl;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, ioctl.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    ret = ecrt_slave_config_state_timeout(sc, ioctl.from_state,
+            ioctl.to_state, ioctl.timeout_ms);
+    return ret;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/** Configures EoE IP parameters.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_ip(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_eoe_ip_t io;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    /* the kernel versions of the EoE set IP methods never fail. */
+    if (io.mac_address_included) {
+        ecrt_slave_config_eoe_mac_address(sc, io.mac_address);
+    }
+    if (io.ip_address_included) {
+        ecrt_slave_config_eoe_ip_address(sc, io.ip_address);
+    }
+    if (io.subnet_mask_included) {
+        ecrt_slave_config_eoe_subnet_mask(sc, io.subnet_mask);
+    }
+    if (io.gateway_included) {
+        ecrt_slave_config_eoe_default_gateway(sc, io.gateway);
+    }
+    if (io.dns_included) {
+        ecrt_slave_config_eoe_dns_address(sc, io.dns);
+    }
+    if (io.name_included) {
+        ecrt_slave_config_eoe_hostname(sc, io.name);
+    }
+
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+/** Gets the domain's data size.
+ *
+ * \return Domain size, or a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_size(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    const ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    list_for_each_entry(domain, &master->domains, list) {
+        if (domain->index == (unsigned long) arg) {
+            size_t size = ecrt_domain_size(domain);
+            up(&master->master_sem);
+            return size;
+        }
+    }
+
+    up(&master->master_sem);
+    return -ENOENT;
+}
+
+/****************************************************************************/
+
+/** Gets the domain's offset in the total process data.
+ *
+ * \return Domain offset, or a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_offset(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int offset = 0;
+    const ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    list_for_each_entry(domain, &master->domains, list) {
+        if (domain->index == (unsigned long) arg) {
+            up(&master->master_sem);
+            return offset;
+        }
+        offset += ecrt_domain_size(domain);
+    }
+
+    up(&master->master_sem);
+    return -ENOENT;
+}
+
+/****************************************************************************/
+
+/** Process the domain.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_process(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    /* no locking of master_sem needed, because domain will not be deleted in
+     * the meantime. */
+
+    if (!(domain = ec_master_find_domain(master, (unsigned long) arg))) {
+        return -ENOENT;
+    }
+
+    return ecrt_domain_process(domain);
+}
+
+/****************************************************************************/
+
+/** Queue the domain.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_queue(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_domain_t *domain;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    /* no locking of master_sem needed, because domain will not be deleted in
+     * the meantime. */
+
+    if (!(domain = ec_master_find_domain(master, (unsigned long) arg))) {
+        return -ENOENT;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_domain_queue(domain);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get the domain state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_domain_state_t data;
+    const ec_domain_t *domain;
+    ec_domain_state_t state;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because domain will not be deleted in
+     * the meantime. */
+
+    if (!(domain = ec_master_find_domain_const(master, data.domain_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_domain_state(domain, &state);
+    if (ret)
+        return ret;
+
+    if (ec_copy_to_user((void __user *) data.state, &state, sizeof(state),
+                ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Sets an SDO request's SDO index and subindex.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_index(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_sdo_request_index(req, data.sdo_index, data.sdo_subindex);
+}
+
+/****************************************************************************/
+
+/** Sets an SDO request's timeout.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_timeout(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_sdo_request_timeout(req, data.timeout);
+}
+
+/****************************************************************************/
+
+/** Gets an SDO request's state.
+ *
+ * Also pre-fetches the size of incoming data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    data.state = ecrt_sdo_request_state(req);
+    if (data.state == EC_REQUEST_SUCCESS && req->dir == EC_DIR_INPUT)
+        data.size = ecrt_sdo_request_data_size(req);
+    else
+        data.size = 0;
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts an SDO read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_sdo_request_read(req);
+}
+
+/****************************************************************************/
+
+/** Starts an SDO write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    if (!data.size) {
+        EC_MASTER_ERR(master, "SDO download: Data size may not be zero!\n");
+        return -EINVAL;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (data.size > req->mem_size)
+        return -ENOBUFS;
+
+    if (ec_copy_from_user(req->data, (void __user *) data.data,
+                          data.size, ctx))
+        return -EFAULT;
+
+    req->data_size = data.size;
+    return ecrt_sdo_request_write(req);
+}
+
+/****************************************************************************/
+
+/** Read SDO data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) data.data, ecrt_sdo_request_data(req),
+                ecrt_sdo_request_data_size(req), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Sets an SoE request's drive number and IDN.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_index(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_soe_request_idn(req, data.drive_no, data.idn);
+}
+
+/****************************************************************************/
+
+/** Sets an CoE request's timeout.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_timeout(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_soe_request_timeout(req, data.timeout);
+}
+
+/****************************************************************************/
+
+/** Gets an SoE request's state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    data.state = ecrt_soe_request_state(req);
+    if (data.state == EC_REQUEST_SUCCESS && req->dir == EC_DIR_INPUT) {
+        data.size = ecrt_soe_request_data_size(req);
+    }
+    else {
+        data.size = 0;
+    }
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts an SoE IDN read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_soe_request_read(req);
+}
+
+/****************************************************************************/
+
+/** Starts an SoE IDN write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    if (!data.size) {
+        EC_MASTER_ERR(master, "IDN write: Data size may not be zero!\n");
+        return -EINVAL;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (data.size > req->mem_size)
+        return -ENOBUFS;
+
+    if (ec_copy_from_user(req->data, (void __user *) data.data,
+                          data.size, ctx))
+        return -EFAULT;
+
+    req->data_size = data.size;
+    return ecrt_soe_request_write(req);
+}
+
+/****************************************************************************/
+
+/** Read SoE IDN data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) data.data, ecrt_soe_request_data(req),
+                ecrt_soe_request_data_size(req), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Read register data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    if (io.mem_size <= 0) {
+        return 0;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) io.data, ecrt_reg_request_data(reg),
+                min(reg->mem_size, io.mem_size), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Gets an register request's state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    io.state = ecrt_reg_request_state(reg);
+    io.new_data = io.state == EC_REQUEST_SUCCESS && reg->dir == EC_DIR_INPUT;
+
+    if (ec_copy_to_user((void __user *) arg, &io, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts an register write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    if (io.transfer_size > reg->mem_size) {
+        return -ENOBUFS;
+    }
+
+    if (ec_copy_from_user(reg->data, (void __user *) io.data,
+                io.transfer_size, ctx)) {
+        return -EFAULT;
+    }
+
+    return ecrt_reg_request_write(reg, io.address, io.transfer_size);
+}
+
+/****************************************************************************/
+
+/** Starts an register read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    if (io.transfer_size > reg->mem_size) {
+        return -ENOBUFS;
+    }
+
+    return ecrt_reg_request_read(reg, io.address, io.transfer_size);
+}
+
+/****************************************************************************/
+
+/** Sets the VoE send header.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_send_header(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+    uint32_t vendor_id;
+    uint16_t vendor_type;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    if (ec_copy_from_user(&vendor_id, data.vendor_id, sizeof(vendor_id), ctx))
+        return -EFAULT;
+
+    if (ec_copy_from_user(&vendor_type, data.vendor_type,
+                          sizeof(vendor_type), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_voe_handler_send_header(voe, vendor_id, vendor_type);
+}
+
+/****************************************************************************/
+
+/** Gets the received VoE header.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_rec_header(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+    uint32_t vendor_id;
+    uint16_t vendor_type;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_voe_handler_received_header(voe, &vendor_id, &vendor_type);
+    if (ret)
+        return ret;
+
+    if (likely(data.vendor_id))
+        if (ec_copy_to_user(data.vendor_id, &vendor_id,
+                            sizeof(vendor_id), ctx))
+            return -EFAULT;
+
+    if (likely(data.vendor_type))
+        if (ec_copy_to_user(data.vendor_type, &vendor_type,
+            sizeof(vendor_type), ctx))
+            return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts a VoE read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_voe_handler_read(voe);
+}
+
+/****************************************************************************/
+
+/** Starts a VoE read operation without sending a sync message first.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_read_nosync(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_voe_handler_read_nosync(voe);
+}
+
+/****************************************************************************/
+
+/** Starts a VoE write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    if (data.size) {
+        if (data.size > ec_voe_handler_mem_size(voe))
+            return -ENOBUFS;
+
+        if (ec_copy_from_user(ecrt_voe_handler_data(voe),
+                    (void __user *) data.data, data.size, ctx))
+            return -EFAULT;
+    }
+
+    return ecrt_voe_handler_write(voe, data.size);
+}
+
+/****************************************************************************/
+
+/** Executes the VoE state machine.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_exec(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    data.state = ecrt_voe_handler_execute(voe);
+    ec_ioctl_unlock(&master->io_mutex);
+    if (data.state == EC_REQUEST_SUCCESS && voe->dir == EC_DIR_INPUT)
+        data.size = ecrt_voe_handler_data_size(voe);
+    else
+        data.size = 0;
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Reads the received VoE data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) data.data, ecrt_voe_handler_data(voe),
+                ecrt_voe_handler_data_size(voe), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Read a file from a slave via FoE.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_foe_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_foe_t io;
+    ec_foe_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    ec_foe_request_init(&request, io.file_name);
+    ret = ec_foe_request_alloc(&request, 10000); // FIXME
+    if (ret) {
+        ec_foe_request_clear(&request);
+        return ret;
+    }
+
+    ec_foe_request_read(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_foe_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        ec_foe_request_clear(&request);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling FoE read request.\n");
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->foe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_foe_request_clear(&request);
+            return -EINTR;
+        }
+        // request already processing: interrupt not possible.
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    io.result = request.result;
+    io.error_code = request.error_code;
+
+    if (request.state != EC_INT_REQUEST_SUCCESS) {
+        io.data_size = 0;
+        ret = -EIO;
+    } else {
+        if (request.data_size > io.buffer_size) {
+            EC_SLAVE_ERR(slave, "%s(): Buffer too small.\n", __func__);
+            ec_foe_request_clear(&request);
+            return -ENOBUFS;
+        }
+        io.data_size = request.data_size;
+        if (copy_to_user((void __user *) io.buffer,
+                    request.buffer, io.data_size)) {
+            ec_foe_request_clear(&request);
+            return -EFAULT;
+        }
+        ret = 0;
+    }
+
+    if (__copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        ret = -EFAULT;
+    }
+
+    ec_foe_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Write a file to a slave via FoE
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_foe_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_foe_t io;
+    ec_foe_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    ec_foe_request_init(&request, io.file_name);
+
+    ret = ec_foe_request_alloc(&request, io.buffer_size);
+    if (ret) {
+        ec_foe_request_clear(&request);
+        return ret;
+    }
+
+    if (copy_from_user(request.buffer,
+                (void __user *) io.buffer, io.buffer_size)) {
+        ec_foe_request_clear(&request);
+        return -EFAULT;
+    }
+
+    request.data_size = io.buffer_size;
+    ec_foe_request_write(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_foe_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        ec_foe_request_clear(&request);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling FoE write request.\n");
+
+    // schedule FoE write request.
+    list_add_tail(&request.list, &slave->foe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_foe_request_clear(&request);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    io.result = request.result;
+    io.error_code = request.error_code;
+
+    ret = request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+
+    if (__copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        ret = -EFAULT;
+    }
+
+    ec_foe_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Read an SoE IDN.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_soe_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_soe_read_t ioctl;
+    u8 *data;
+    int retval;
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    data = kmalloc(ioctl.mem_size, GFP_KERNEL);
+    if (!data) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes of IDN data.\n",
+                ioctl.mem_size);
+        return -ENOMEM;
+    }
+
+    retval = ecrt_master_read_idn(master, ioctl.slave_position,
+            ioctl.drive_no, ioctl.idn, data, ioctl.mem_size, &ioctl.data_size,
+            &ioctl.error_code);
+    if (retval) {
+        kfree(data);
+        return retval;
+    }
+
+    if (copy_to_user((void __user *) ioctl.data,
+                data, ioctl.data_size)) {
+        kfree(data);
+        return -EFAULT;
+    }
+    kfree(data);
+
+    if (__copy_to_user((void __user *) arg, &ioctl, sizeof(ioctl))) {
+        retval = -EFAULT;
+    }
+
+    EC_MASTER_DBG(master, 1, "Finished SoE read request.\n");
+    return retval;
+}
+
+/****************************************************************************/
+
+/** Write an IDN to a slave via SoE.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_soe_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_soe_write_t ioctl;
+    u8 *data;
+    int retval;
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    data = kmalloc(ioctl.data_size, GFP_KERNEL);
+    if (!data) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes of IDN data.\n",
+                ioctl.data_size);
+        return -ENOMEM;
+    }
+    if (copy_from_user(data, (void __user *) ioctl.data, ioctl.data_size)) {
+        kfree(data);
+        return -EFAULT;
+    }
+
+    retval = ecrt_master_write_idn(master, ioctl.slave_position,
+            ioctl.drive_no, ioctl.idn, data, ioctl.data_size,
+            &ioctl.error_code);
+    kfree(data);
+    if (retval) {
+        return retval;
+    }
+
+    if (__copy_to_user((void __user *) arg, &ioctl, sizeof(ioctl))) {
+        retval = -EFAULT;
+    }
+
+    EC_MASTER_DBG(master, 1, "Finished SoE write request.\n");
+    return retval;
+}
+
+/*****************************************************************************
+ * ioctl() file operation functions
+ ****************************************************************************/
+
+/** ioctl() function to use.
+ *
+ * For RTDM, there will be ec_ioctl_rtdm_rt and ec_ioctl_rtdm_nrt.
+ * For "normal" cdev, there will be ec_ioctl_rt only.
+ */
+#ifndef EC_IOCTL_RTDM
+static long ec_ioctl_nrt(
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */);
+#endif
+
+/****************************************************************************/
+
+/** Called when an ioctl() command is issued.
+ * Both RT and nRT context.
+ *
+ * \return ioctl() return code.
+ */
+static long ec_ioctl_both(
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    int ret;
+
+    switch (cmd) {
+        case EC_IOCTL_MODULE:
+            ret = ec_ioctl_module(arg, ctx);
+            break;
+        case EC_IOCTL_MASTER_RESCAN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_master_rescan(master, arg);
+            break;
+        case EC_IOCTL_MASTER_STATE:
+            ret = ec_ioctl_master_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_MASTER_LINK_STATE:
+            ret = ec_ioctl_master_link_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_TIMEOUT:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_timeout(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_DATA:
+            ret = ec_ioctl_sdo_request_data(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_TIMEOUT:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_timeout(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_DATA:
+            ret = ec_ioctl_soe_request_data(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_DATA:
+            ret = ec_ioctl_reg_request_data(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_SEND_HEADER:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_send_header(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_DATA:
+            ret = ec_ioctl_voe_data(master, arg, ctx);
+            break;
+        default:
+#ifdef EC_IOCTL_RTDM
+            ret = -ENOTTY;
+#else
+            /* chain non-rt commands for normal cdev */
+            ret = ec_ioctl_nrt(master, ctx, cmd, arg);
+#endif
+            break;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Called when an ioctl() command is issued.
+ * RTDM: RT only.
+ *
+ * \return ioctl() return code.
+ */
+#ifdef EC_IOCTL_RTDM
+long ec_ioctl_rtdm_rt
+#else
+long ec_ioctl
+#endif
+        (
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+#if DEBUG_LATENCY
+    cycles_t a = get_cycles(), b;
+    unsigned int t;
+#endif
+    long ret;
+
+    switch (cmd) {
+        case EC_IOCTL_SEND:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_send(master, arg, ctx);
+            break;
+        case EC_IOCTL_RECEIVE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_receive(master, arg, ctx);
+            break;
+        case EC_IOCTL_APP_TIME:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_app_time(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_REF:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_ref(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_REF_TO:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_ref_to(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_SLAVES:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_slaves(master, arg, ctx);
+            break;
+        case EC_IOCTL_REF_CLOCK_TIME:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_ref_clock_time(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_MON_QUEUE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_mon_queue(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_MON_PROCESS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_mon_process(master, arg, ctx);
+            break;
+        case EC_IOCTL_RESET:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_reset(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_POP:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_emerg_pop(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_CLEAR:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_emerg_clear(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_OVERRUNS:
+            ret = ec_ioctl_sc_emerg_overruns(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_STATE:
+            ret = ec_ioctl_sc_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_PROCESS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_domain_process(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_QUEUE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_domain_queue(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_STATE:
+            ret = ec_ioctl_domain_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_INDEX:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_index(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_STATE:
+            ret = ec_ioctl_sdo_request_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_IDN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_index(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_STATE:
+            ret = ec_ioctl_soe_request_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_STATE:
+            ret = ec_ioctl_reg_request_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_reg_request_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_reg_request_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_REC_HEADER:
+            ret = ec_ioctl_voe_rec_header(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_READ_NOSYNC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_read_nosync(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_EXEC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_exec(master, arg, ctx);
+            break;
+        default:
+            ret = ec_ioctl_both(master, ctx, cmd, arg);
+            break;
+    }
+
+#if DEBUG_LATENCY
+    b = get_cycles();
+    t = (unsigned int) ((b - a) * 1000LL) / cpu_khz;
+    if (t > 50) {
+        EC_MASTER_WARN(master, "ioctl(0x%02x) took %u us.\n",
+                _IOC_NR(cmd), t);
+    }
+#endif
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Called when an ioctl() command is issued.
+ * nRT context only.
+ *
+ * \return ioctl() return code.
+ */
+#ifdef EC_IOCTL_RTDM
+long ec_ioctl_rtdm_nrt
+#else
+static long ec_ioctl_nrt
+#endif
+        (
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+#if DEBUG_LATENCY && !defined(EC_IOCTL_RTDM)
+    cycles_t a = get_cycles(), b;
+    unsigned int t;
+#endif
+    int ret;
+
+    switch (cmd) {
+        case EC_IOCTL_MASTER:
+            ret = ec_ioctl_master(master, arg);
+            break;
+        case EC_IOCTL_SLAVE:
+            ret = ec_ioctl_slave(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SYNC:
+            ret = ec_ioctl_slave_sync(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SYNC_PDO:
+            ret = ec_ioctl_slave_sync_pdo(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SYNC_PDO_ENTRY:
+            ret = ec_ioctl_slave_sync_pdo_entry(master, arg);
+            break;
+        case EC_IOCTL_DOMAIN:
+            ret = ec_ioctl_domain(master, arg);
+            break;
+        case EC_IOCTL_DOMAIN_FMMU:
+            ret = ec_ioctl_domain_fmmu(master, arg);
+            break;
+        case EC_IOCTL_DOMAIN_DATA:
+            ret = ec_ioctl_domain_data(master, arg);
+            break;
+        case EC_IOCTL_MASTER_DEBUG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_master_debug(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_STATE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_state(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO:
+            ret = ec_ioctl_slave_sdo(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO_ENTRY:
+            ret = ec_ioctl_slave_sdo_entry(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO_UPLOAD:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_sdo_upload(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO_DOWNLOAD:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_sdo_download(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SII_READ:
+            ret = ec_ioctl_slave_sii_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SII_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_sii_write(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_REG_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_reg_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_REG_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_reg_write(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_FOE_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_foe_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_FOE_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_foe_write(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SOE_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_soe_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SOE_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_soe_write(master, arg);
+            break;
+#ifdef EC_EOE
+        case EC_IOCTL_SLAVE_EOE_IP_PARAM:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_eoe_ip_param(master, arg);
+            break;
+#endif
+        case EC_IOCTL_CONFIG:
+            ret = ec_ioctl_config(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_PDO:
+            ret = ec_ioctl_config_pdo(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_PDO_ENTRY:
+            ret = ec_ioctl_config_pdo_entry(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_SDO:
+            ret = ec_ioctl_config_sdo(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_IDN:
+            ret = ec_ioctl_config_idn(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_FLAG:
+            ret = ec_ioctl_config_flag(master, arg);
+            break;
+#ifdef EC_EOE
+        case EC_IOCTL_CONFIG_EOE_IP_PARAM:
+            ret = ec_ioctl_config_ip(master, arg);
+            break;
+        case EC_IOCTL_EOE_HANDLER:
+            ret = ec_ioctl_eoe_handler(master, arg);
+            break;
+#endif
+
+        /* Application interface */
+
+        case EC_IOCTL_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_CREATE_DOMAIN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_create_domain(master, arg, ctx);
+            break;
+        case EC_IOCTL_CREATE_SLAVE_CONFIG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_create_slave_config(master, arg, ctx);
+            break;
+        case EC_IOCTL_SELECT_REF_CLOCK:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_select_ref_clock(master, arg, ctx);
+            break;
+        case EC_IOCTL_ACTIVATE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_activate(master, arg, ctx);
+            break;
+        case EC_IOCTL_DEACTIVATE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_deactivate(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SYNC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_sync(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_WATCHDOG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_watchdog(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_ADD_PDO:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_add_pdo(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_CLEAR_PDOS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_clear_pdos(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_ADD_ENTRY:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_add_entry(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_CLEAR_ENTRIES:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_clear_entries(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_REG_PDO_ENTRY:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_reg_pdo_entry(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_REG_PDO_POS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_reg_pdo_pos(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_DC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_dc(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SDO:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_sdo(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_SIZE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_emerg_size(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SDO_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_sdo_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SOE_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_soe_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_REG_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_reg_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_VOE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_voe_handler(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_IDN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_idn(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_FLAG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_flag(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_STATE_TIMEOUT:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_state_timeout(master, arg, ctx);
+            break;
+#ifdef EC_EOE
+        case EC_IOCTL_SC_EOE_IP_PARAM:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_ip(master, arg, ctx);
+            break;
+#endif
+        case EC_IOCTL_DOMAIN_SIZE:
+            ret = ec_ioctl_domain_size(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_OFFSET:
+            ret = ec_ioctl_domain_offset(master, arg, ctx);
+            break;
+        case EC_IOCTL_SET_SEND_INTERVAL:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_set_send_interval(master, arg, ctx);
+            break;
+        default:
+#ifdef EC_IOCTL_RTDM
+            ret = ec_ioctl_both(master, ctx, cmd, arg);
+#else
+            ret = -ENOTTY;
+#endif
+            break;
+    }
+
+#if DEBUG_LATENCY && !defined(EC_IOCTL_RTDM)
+    b = get_cycles();
+    t = (unsigned int) ((b - a) * 1000LL) / cpu_khz;
+    if (t > 50) {
+        EC_MASTER_WARN(master, "ioctl(0x%02x) took %u us.\n",
+                _IOC_NR(cmd), t);
+    }
+#endif
+
+    return ret;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/ioctl.h b/drivers/net/ethercat/master/ioctl.h
new file mode 100644
index 000000000000..0c51e6380f43
--- /dev/null
+++ b/drivers/net/ethercat/master/ioctl.h
@@ -0,0 +1,879 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT master.
+ *
+ *  The file is free software; you can redistribute it and/or modify it under
+ *  the terms of the GNU Lesser General Public License as published by the
+ *  Free Software Foundation; version 2.1 of the License.
+ *
+ *  This file is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+ *  License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this file. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT master character device IOCTL commands.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_IOCTL_H__
+#define __EC_IOCTL_H__
+
+#include <linux/ioctl.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** \cond */
+
+#define EC_IOCTL_TYPE 0xa4
+
+#define EC_IO(nr)           _IO(EC_IOCTL_TYPE, nr)
+#define EC_IOR(nr, type)   _IOR(EC_IOCTL_TYPE, nr, type)
+#define EC_IOW(nr, type)   _IOW(EC_IOCTL_TYPE, nr, type)
+#define EC_IOWR(nr, type) _IOWR(EC_IOCTL_TYPE, nr, type)
+
+/** EtherCAT master ioctl() version magic.
+ *
+ * Increment this when changing the ioctl interface!
+ */
+#define EC_IOCTL_VERSION_MAGIC 37
+
+// Command-line tool
+#define EC_IOCTL_MODULE                EC_IOR(0x00, ec_ioctl_module_t)
+#define EC_IOCTL_MASTER                EC_IOR(0x01, ec_ioctl_master_t)
+#define EC_IOCTL_SLAVE                EC_IOWR(0x02, ec_ioctl_slave_t)
+#define EC_IOCTL_SLAVE_SYNC           EC_IOWR(0x03, ec_ioctl_slave_sync_t)
+#define EC_IOCTL_SLAVE_SYNC_PDO       EC_IOWR(0x04, ec_ioctl_slave_sync_pdo_t)
+#define EC_IOCTL_SLAVE_SYNC_PDO_ENTRY EC_IOWR(0x05, ec_ioctl_slave_sync_pdo_entry_t)
+#define EC_IOCTL_DOMAIN               EC_IOWR(0x06, ec_ioctl_domain_t)
+#define EC_IOCTL_DOMAIN_FMMU          EC_IOWR(0x07, ec_ioctl_domain_fmmu_t)
+#define EC_IOCTL_DOMAIN_DATA          EC_IOWR(0x08, ec_ioctl_domain_data_t)
+#define EC_IOCTL_MASTER_DEBUG           EC_IO(0x09)
+#define EC_IOCTL_MASTER_RESCAN          EC_IO(0x0a)
+#define EC_IOCTL_SLAVE_STATE           EC_IOW(0x0b, ec_ioctl_slave_state_t)
+#define EC_IOCTL_SLAVE_SDO            EC_IOWR(0x0c, ec_ioctl_slave_sdo_t)
+#define EC_IOCTL_SLAVE_SDO_ENTRY      EC_IOWR(0x0d, ec_ioctl_slave_sdo_entry_t)
+#define EC_IOCTL_SLAVE_SDO_UPLOAD     EC_IOWR(0x0e, ec_ioctl_slave_sdo_upload_t)
+#define EC_IOCTL_SLAVE_SDO_DOWNLOAD   EC_IOWR(0x0f, ec_ioctl_slave_sdo_download_t)
+#define EC_IOCTL_SLAVE_SII_READ       EC_IOWR(0x10, ec_ioctl_slave_sii_t)
+#define EC_IOCTL_SLAVE_SII_WRITE       EC_IOW(0x11, ec_ioctl_slave_sii_t)
+#define EC_IOCTL_SLAVE_REG_READ       EC_IOWR(0x12, ec_ioctl_slave_reg_t)
+#define EC_IOCTL_SLAVE_REG_WRITE       EC_IOW(0x13, ec_ioctl_slave_reg_t)
+#define EC_IOCTL_SLAVE_FOE_READ       EC_IOWR(0x14, ec_ioctl_slave_foe_t)
+#define EC_IOCTL_SLAVE_FOE_WRITE       EC_IOW(0x15, ec_ioctl_slave_foe_t)
+#define EC_IOCTL_SLAVE_SOE_READ       EC_IOWR(0x16, ec_ioctl_slave_soe_read_t)
+#define EC_IOCTL_SLAVE_SOE_WRITE      EC_IOWR(0x17, ec_ioctl_slave_soe_write_t)
+#ifdef EC_EOE
+#define EC_IOCTL_SLAVE_EOE_IP_PARAM    EC_IOW(0x18, ec_ioctl_eoe_ip_t)
+#endif
+#define EC_IOCTL_CONFIG               EC_IOWR(0x19, ec_ioctl_config_t)
+#define EC_IOCTL_CONFIG_PDO           EC_IOWR(0x1a, ec_ioctl_config_pdo_t)
+#define EC_IOCTL_CONFIG_PDO_ENTRY     EC_IOWR(0x1b, ec_ioctl_config_pdo_entry_t)
+#define EC_IOCTL_CONFIG_SDO           EC_IOWR(0x1c, ec_ioctl_config_sdo_t)
+#define EC_IOCTL_CONFIG_IDN           EC_IOWR(0x1d, ec_ioctl_config_idn_t)
+#define EC_IOCTL_CONFIG_FLAG          EC_IOWR(0x1e, ec_ioctl_config_flag_t)
+#ifdef EC_EOE
+#define EC_IOCTL_CONFIG_EOE_IP_PARAM  EC_IOWR(0x1f, ec_ioctl_eoe_ip_t)
+#define EC_IOCTL_EOE_HANDLER          EC_IOWR(0x20, ec_ioctl_eoe_handler_t)
+#endif
+
+// Application interface
+#define EC_IOCTL_REQUEST                EC_IO(0x21)
+#define EC_IOCTL_CREATE_DOMAIN          EC_IO(0x22)
+#define EC_IOCTL_CREATE_SLAVE_CONFIG  EC_IOWR(0x23, ec_ioctl_config_t)
+#define EC_IOCTL_SELECT_REF_CLOCK      EC_IOW(0x24, uint32_t)
+#define EC_IOCTL_ACTIVATE              EC_IOR(0x25, ec_ioctl_master_activate_t)
+#define EC_IOCTL_DEACTIVATE             EC_IO(0x26)
+#define EC_IOCTL_SEND                   EC_IO(0x27)
+#define EC_IOCTL_RECEIVE                EC_IO(0x28)
+#define EC_IOCTL_MASTER_STATE          EC_IOR(0x29, ec_master_state_t)
+#define EC_IOCTL_MASTER_LINK_STATE    EC_IOWR(0x2a, ec_ioctl_link_state_t)
+#define EC_IOCTL_APP_TIME              EC_IOW(0x2b, uint64_t)
+#define EC_IOCTL_SYNC_REF               EC_IO(0x2c)
+#define EC_IOCTL_SYNC_REF_TO           EC_IOW(0x2d, uint64_t)
+#define EC_IOCTL_SYNC_SLAVES            EC_IO(0x2e)
+#define EC_IOCTL_REF_CLOCK_TIME        EC_IOR(0x2f, uint32_t)
+#define EC_IOCTL_SYNC_MON_QUEUE         EC_IO(0x30)
+#define EC_IOCTL_SYNC_MON_PROCESS      EC_IOR(0x31, uint32_t)
+#define EC_IOCTL_RESET                  EC_IO(0x32)
+#define EC_IOCTL_SC_SYNC               EC_IOW(0x33, ec_ioctl_config_t)
+#define EC_IOCTL_SC_WATCHDOG           EC_IOW(0x34, ec_ioctl_config_t)
+#define EC_IOCTL_SC_ADD_PDO            EC_IOW(0x35, ec_ioctl_config_pdo_t)
+#define EC_IOCTL_SC_CLEAR_PDOS         EC_IOW(0x36, ec_ioctl_config_pdo_t)
+#define EC_IOCTL_SC_ADD_ENTRY          EC_IOW(0x37, ec_ioctl_add_pdo_entry_t)
+#define EC_IOCTL_SC_CLEAR_ENTRIES      EC_IOW(0x38, ec_ioctl_config_pdo_t)
+#define EC_IOCTL_SC_REG_PDO_ENTRY     EC_IOWR(0x39, ec_ioctl_reg_pdo_entry_t)
+#define EC_IOCTL_SC_REG_PDO_POS       EC_IOWR(0x3a, ec_ioctl_reg_pdo_pos_t)
+#define EC_IOCTL_SC_DC                 EC_IOW(0x3b, ec_ioctl_config_t)
+#define EC_IOCTL_SC_SDO                EC_IOW(0x3c, ec_ioctl_sc_sdo_t)
+#define EC_IOCTL_SC_EMERG_SIZE         EC_IOW(0x3d, ec_ioctl_sc_emerg_t)
+#define EC_IOCTL_SC_EMERG_POP         EC_IOWR(0x3e, ec_ioctl_sc_emerg_t)
+#define EC_IOCTL_SC_EMERG_CLEAR        EC_IOW(0x3f, ec_ioctl_sc_emerg_t)
+#define EC_IOCTL_SC_EMERG_OVERRUNS    EC_IOWR(0x40, ec_ioctl_sc_emerg_t)
+#define EC_IOCTL_SC_SDO_REQUEST       EC_IOWR(0x41, ec_ioctl_sdo_request_t)
+#define EC_IOCTL_SC_SOE_REQUEST       EC_IOWR(0x42, ec_ioctl_soe_request_t)
+#define EC_IOCTL_SC_REG_REQUEST       EC_IOWR(0x43, ec_ioctl_reg_request_t)
+#define EC_IOCTL_SC_VOE               EC_IOWR(0x44, ec_ioctl_voe_t)
+#define EC_IOCTL_SC_STATE             EC_IOWR(0x45, ec_ioctl_sc_state_t)
+#define EC_IOCTL_SC_IDN                EC_IOW(0x46, ec_ioctl_sc_idn_t)
+#define EC_IOCTL_SC_FLAG               EC_IOW(0x47, ec_ioctl_sc_flag_t)
+#ifdef EC_EOE
+#define EC_IOCTL_SC_EOE_IP_PARAM       EC_IOW(0x48, ec_ioctl_eoe_ip_t)
+#endif
+#define EC_IOCTL_SC_STATE_TIMEOUT      EC_IOW(0x49, ec_ioctl_sc_state_timeout_t)
+#define EC_IOCTL_DOMAIN_SIZE            EC_IO(0x4a)
+#define EC_IOCTL_DOMAIN_OFFSET          EC_IO(0x4b)
+#define EC_IOCTL_DOMAIN_PROCESS         EC_IO(0x4c)
+#define EC_IOCTL_DOMAIN_QUEUE           EC_IO(0x4d)
+#define EC_IOCTL_DOMAIN_STATE         EC_IOWR(0x4e, ec_ioctl_domain_state_t)
+#define EC_IOCTL_SDO_REQUEST_INDEX    EC_IOWR(0x4f, ec_ioctl_sdo_request_t)
+#define EC_IOCTL_SDO_REQUEST_TIMEOUT  EC_IOWR(0x50, ec_ioctl_sdo_request_t)
+#define EC_IOCTL_SDO_REQUEST_STATE    EC_IOWR(0x51, ec_ioctl_sdo_request_t)
+#define EC_IOCTL_SDO_REQUEST_READ     EC_IOWR(0x52, ec_ioctl_sdo_request_t)
+#define EC_IOCTL_SDO_REQUEST_WRITE    EC_IOWR(0x53, ec_ioctl_sdo_request_t)
+#define EC_IOCTL_SDO_REQUEST_DATA     EC_IOWR(0x54, ec_ioctl_sdo_request_t)
+#define EC_IOCTL_SOE_REQUEST_IDN      EC_IOWR(0x55, ec_ioctl_soe_request_t)
+#define EC_IOCTL_SOE_REQUEST_TIMEOUT  EC_IOWR(0x56, ec_ioctl_soe_request_t)
+#define EC_IOCTL_SOE_REQUEST_STATE    EC_IOWR(0x57, ec_ioctl_soe_request_t)
+#define EC_IOCTL_SOE_REQUEST_READ     EC_IOWR(0x58, ec_ioctl_soe_request_t)
+#define EC_IOCTL_SOE_REQUEST_WRITE    EC_IOWR(0x59, ec_ioctl_soe_request_t)
+#define EC_IOCTL_SOE_REQUEST_DATA     EC_IOWR(0x5a, ec_ioctl_soe_request_t)
+#define EC_IOCTL_REG_REQUEST_DATA     EC_IOWR(0x5b, ec_ioctl_reg_request_t)
+#define EC_IOCTL_REG_REQUEST_STATE    EC_IOWR(0x5c, ec_ioctl_reg_request_t)
+#define EC_IOCTL_REG_REQUEST_WRITE    EC_IOWR(0x5d, ec_ioctl_reg_request_t)
+#define EC_IOCTL_REG_REQUEST_READ     EC_IOWR(0x5e, ec_ioctl_reg_request_t)
+#define EC_IOCTL_VOE_SEND_HEADER       EC_IOW(0x5f, ec_ioctl_voe_t)
+#define EC_IOCTL_VOE_REC_HEADER       EC_IOWR(0x60, ec_ioctl_voe_t)
+#define EC_IOCTL_VOE_READ              EC_IOW(0x61, ec_ioctl_voe_t)
+#define EC_IOCTL_VOE_READ_NOSYNC       EC_IOW(0x62, ec_ioctl_voe_t)
+#define EC_IOCTL_VOE_WRITE            EC_IOWR(0x63, ec_ioctl_voe_t)
+#define EC_IOCTL_VOE_EXEC             EC_IOWR(0x64, ec_ioctl_voe_t)
+#define EC_IOCTL_VOE_DATA             EC_IOWR(0x65, ec_ioctl_voe_t)
+#define EC_IOCTL_SET_SEND_INTERVAL     EC_IOW(0x66, size_t)
+
+/****************************************************************************/
+
+#define EC_IOCTL_STRING_SIZE 64
+
+/****************************************************************************/
+
+typedef struct {
+    uint32_t ioctl_version_magic;
+    uint32_t master_count;
+} ec_ioctl_module_t;
+
+/****************************************************************************/
+
+typedef struct {
+    uint32_t slave_count;
+    uint32_t scan_index;
+    uint32_t config_count;
+    uint32_t domain_count;
+    uint32_t eoe_handler_count;
+    uint8_t phase;
+    uint8_t active;
+    uint8_t scan_busy;
+    struct ec_ioctl_device {
+        uint8_t address[6];
+        uint8_t attached;
+        uint8_t link_state;
+        uint64_t tx_count;
+        uint64_t rx_count;
+        uint64_t tx_bytes;
+        uint64_t rx_bytes;
+        uint64_t tx_errors;
+        int32_t tx_frame_rates[EC_RATE_COUNT];
+        int32_t rx_frame_rates[EC_RATE_COUNT];
+        int32_t tx_byte_rates[EC_RATE_COUNT];
+        int32_t rx_byte_rates[EC_RATE_COUNT];
+    } devices[EC_MAX_NUM_DEVICES];
+    uint32_t num_devices;
+    uint64_t tx_count;
+    uint64_t rx_count;
+    uint64_t tx_bytes;
+    uint64_t rx_bytes;
+    int32_t tx_frame_rates[EC_RATE_COUNT];
+    int32_t rx_frame_rates[EC_RATE_COUNT];
+    int32_t tx_byte_rates[EC_RATE_COUNT];
+    int32_t rx_byte_rates[EC_RATE_COUNT];
+    int32_t loss_rates[EC_RATE_COUNT];
+    uint64_t app_time;
+    uint64_t dc_ref_time;
+    uint16_t ref_clock;
+} ec_ioctl_master_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // input
+    uint16_t position;
+
+    // outputs
+    unsigned int device_index;
+    uint32_t vendor_id;
+    uint32_t product_code;
+    uint32_t revision_number;
+    uint32_t serial_number;
+    uint16_t alias;
+    uint16_t boot_rx_mailbox_offset;
+    uint16_t boot_rx_mailbox_size;
+    uint16_t boot_tx_mailbox_offset;
+    uint16_t boot_tx_mailbox_size;
+    uint16_t std_rx_mailbox_offset;
+    uint16_t std_rx_mailbox_size;
+    uint16_t std_tx_mailbox_offset;
+    uint16_t std_tx_mailbox_size;
+    uint16_t mailbox_protocols;
+    uint8_t has_general_category;
+    ec_sii_coe_details_t coe_details;
+    ec_sii_general_flags_t general_flags;
+    int16_t current_on_ebus;
+    struct {
+        ec_slave_port_desc_t desc;
+        ec_slave_port_link_t link;
+        uint32_t receive_time;
+        uint16_t next_slave;
+        uint32_t delay_to_next_dc;
+    } ports[EC_MAX_PORTS];
+    uint8_t fmmu_bit;
+    uint8_t dc_supported;
+    ec_slave_dc_range_t dc_range;
+    uint8_t has_dc_system_time;
+    uint32_t transmission_delay;
+    uint8_t al_state;
+    uint8_t error_flag;
+    uint8_t sync_count;
+    uint16_t sdo_count;
+    uint32_t sii_nwords;
+    char group[EC_IOCTL_STRING_SIZE];
+    char image[EC_IOCTL_STRING_SIZE];
+    char order[EC_IOCTL_STRING_SIZE];
+    char name[EC_IOCTL_STRING_SIZE];
+} ec_ioctl_slave_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint32_t sync_index;
+
+    // outputs
+    uint16_t physical_start_address;
+    uint16_t default_size;
+    uint8_t control_register;
+    uint8_t enable;
+    uint8_t pdo_count;
+} ec_ioctl_slave_sync_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint32_t sync_index;
+    uint32_t pdo_pos;
+
+    // outputs
+    uint16_t index;
+    uint8_t entry_count;
+    int8_t name[EC_IOCTL_STRING_SIZE];
+} ec_ioctl_slave_sync_pdo_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint32_t sync_index;
+    uint32_t pdo_pos;
+    uint32_t entry_pos;
+
+    // outputs
+    uint16_t index;
+    uint8_t subindex;
+    uint8_t bit_length;
+    int8_t name[EC_IOCTL_STRING_SIZE];
+} ec_ioctl_slave_sync_pdo_entry_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t index;
+
+    // outputs
+    uint32_t data_size;
+    uint32_t logical_base_address;
+    uint16_t working_counter[EC_MAX_NUM_DEVICES];
+    uint16_t expected_working_counter;
+    uint32_t fmmu_count;
+} ec_ioctl_domain_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t domain_index;
+    uint32_t fmmu_index;
+
+    // outputs
+    uint16_t slave_config_alias;
+    uint16_t slave_config_position;
+    uint8_t sync_index;
+    ec_direction_t dir;
+    uint32_t logical_address;
+    uint32_t data_size;
+} ec_ioctl_domain_fmmu_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t domain_index;
+    uint32_t data_size;
+    uint8_t *target;
+} ec_ioctl_domain_data_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint8_t al_state;
+} ec_ioctl_slave_state_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint16_t sdo_position;
+
+    // outputs
+    uint16_t sdo_index;
+    uint8_t max_subindex;
+    int8_t name[EC_IOCTL_STRING_SIZE];
+} ec_ioctl_slave_sdo_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    int sdo_spec; // positive: index, negative: list position
+    uint8_t sdo_entry_subindex;
+
+    // outputs
+    uint16_t data_type;
+    uint16_t bit_length;
+    uint8_t read_access[EC_SDO_ENTRY_ACCESS_COUNT];
+    uint8_t write_access[EC_SDO_ENTRY_ACCESS_COUNT];
+    int8_t description[EC_IOCTL_STRING_SIZE];
+} ec_ioctl_slave_sdo_entry_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint16_t sdo_index;
+    uint8_t sdo_entry_subindex;
+    size_t target_size;
+    uint8_t *target;
+
+    // outputs
+    size_t data_size;
+    uint32_t abort_code;
+} ec_ioctl_slave_sdo_upload_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint16_t sdo_index;
+    uint8_t sdo_entry_subindex;
+    uint8_t complete_access;
+    size_t data_size;
+    uint8_t *data;
+
+    // outputs
+    uint32_t abort_code;
+} ec_ioctl_slave_sdo_download_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint16_t offset;
+    uint32_t nwords;
+    uint16_t *words;
+} ec_ioctl_slave_sii_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint8_t emergency;
+    uint16_t address;
+    size_t size;
+    uint8_t *data;
+} ec_ioctl_slave_reg_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint16_t offset;
+    size_t buffer_size;
+    uint8_t *buffer;
+
+    // outputs
+    size_t data_size;
+    uint32_t result;
+    uint32_t error_code;
+    char file_name[32];
+} ec_ioctl_slave_foe_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint8_t drive_no;
+    uint16_t idn;
+    size_t mem_size;
+    uint8_t *data;
+
+    // outputs
+    size_t data_size;
+    uint16_t error_code;
+} ec_ioctl_slave_soe_read_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint16_t slave_position;
+    uint8_t drive_no;
+    uint16_t idn;
+    size_t data_size;
+    uint8_t *data;
+
+    // outputs
+    uint16_t error_code;
+} ec_ioctl_slave_soe_write_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+
+    // outputs
+    uint16_t alias;
+    uint16_t position;
+    uint32_t vendor_id;
+    uint32_t product_code;
+    struct {
+        ec_direction_t dir;
+        ec_watchdog_mode_t watchdog_mode;
+        uint32_t pdo_count;
+        uint8_t config_this;
+    } syncs[EC_MAX_SYNC_MANAGERS];
+    uint16_t watchdog_divider;
+    uint16_t watchdog_intervals;
+    uint32_t sdo_count;
+    uint32_t idn_count;
+    uint32_t flag_count;
+    int32_t slave_position;
+    uint16_t dc_assign_activate;
+    ec_sync_signal_t dc_sync[EC_SYNC_SIGNAL_COUNT];
+} ec_ioctl_config_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint8_t sync_index;
+    uint16_t pdo_pos;
+
+    // outputs
+    uint16_t index;
+    uint8_t entry_count;
+    int8_t name[EC_IOCTL_STRING_SIZE];
+} ec_ioctl_config_pdo_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint8_t sync_index;
+    uint16_t pdo_pos;
+    uint8_t entry_pos;
+
+    // outputs
+    uint16_t index;
+    uint8_t subindex;
+    uint8_t bit_length;
+    int8_t name[EC_IOCTL_STRING_SIZE];
+} ec_ioctl_config_pdo_entry_t;
+
+/****************************************************************************/
+
+/** Maximum size for displayed SDO data.
+ * \todo Make this dynamic.
+ */
+#define EC_MAX_SDO_DATA_SIZE 1024
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint32_t sdo_pos;
+
+    // outputs
+    uint16_t index;
+    uint8_t subindex;
+    size_t size;
+    uint8_t data[EC_MAX_SDO_DATA_SIZE];
+    uint8_t complete_access;
+} ec_ioctl_config_sdo_t;
+
+/****************************************************************************/
+
+/** Maximum size for displayed IDN data.
+ * \todo Make this dynamic.
+ */
+#define EC_MAX_IDN_DATA_SIZE 1024
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint32_t idn_pos;
+
+    // outputs
+    uint8_t drive_no;
+    uint16_t idn;
+    ec_al_state_t state;
+    size_t size;
+    uint8_t data[EC_MAX_IDN_DATA_SIZE];
+} ec_ioctl_config_idn_t;
+
+/****************************************************************************/
+
+/** Maximum size for key.
+ */
+#define EC_MAX_FLAG_KEY_SIZE 128
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint32_t flag_pos;
+
+    // outputs
+    char key[EC_MAX_FLAG_KEY_SIZE];
+    int32_t value;
+} ec_ioctl_config_flag_t;
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+typedef struct {
+    // input
+    uint16_t eoe_index;
+
+    // outputs
+    char name[EC_DATAGRAM_NAME_SIZE];
+    uint16_t slave_position;
+    uint8_t open;
+    uint32_t rx_bytes;
+    uint32_t rx_rate;
+    uint32_t tx_bytes;
+    uint32_t tx_rate;
+    uint32_t tx_queued_frames;
+    uint32_t tx_queue_size;
+} ec_ioctl_eoe_handler_t;
+
+#endif
+
+/****************************************************************************/
+
+#define EC_ETH_ALEN 6
+#ifdef ETH_ALEN
+#if ETH_ALEN != EC_ETH_ALEN
+#error Ethernet address length mismatch
+#endif
+#endif
+
+typedef struct {
+    // input
+    uint16_t slave_position;
+    uint16_t config_index; // alternatively
+
+    uint8_t mac_address_included;
+    uint8_t ip_address_included;
+    uint8_t subnet_mask_included;
+    uint8_t gateway_included;
+    uint8_t dns_included;
+    uint8_t name_included;
+
+    unsigned char mac_address[EC_ETH_ALEN];
+    struct in_addr ip_address;
+    struct in_addr subnet_mask;
+    struct in_addr gateway;
+    struct in_addr dns;
+    char name[EC_MAX_HOSTNAME_SIZE];
+
+	// output
+	uint16_t result;
+} ec_ioctl_eoe_ip_t;
+
+/*****************************************************************************/
+
+typedef struct {
+    // outputs
+    void *process_data;
+    size_t process_data_size;
+} ec_ioctl_master_activate_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint16_t pdo_index;
+    uint16_t entry_index;
+    uint8_t entry_subindex;
+    uint8_t entry_bit_length;
+} ec_ioctl_add_pdo_entry_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint16_t entry_index;
+    uint8_t entry_subindex;
+    uint32_t domain_index;
+
+    // outputs
+    unsigned int bit_position;
+} ec_ioctl_reg_pdo_entry_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint32_t sync_index;
+    uint32_t pdo_pos;
+    uint32_t entry_pos;
+    uint32_t domain_index;
+
+    // outputs
+    unsigned int bit_position;
+} ec_ioctl_reg_pdo_pos_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint16_t index;
+    uint8_t subindex;
+    const uint8_t *data;
+    size_t size;
+    uint8_t complete_access;
+} ec_ioctl_sc_sdo_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    size_t size;
+    uint8_t *target;
+
+    // outputs
+    int32_t overruns;
+} ec_ioctl_sc_emerg_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+
+    // outputs
+    ec_slave_config_state_t *state;
+} ec_ioctl_sc_state_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    uint8_t drive_no;
+    uint16_t idn;
+    ec_al_state_t al_state;
+    const uint8_t *data;
+    size_t size;
+} ec_ioctl_sc_idn_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    size_t key_size;
+    char *key;
+    int32_t value;
+} ec_ioctl_sc_flag_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    ec_al_state_t from_state;
+    ec_al_state_t to_state;
+    uint32_t timeout_ms;
+} ec_ioctl_sc_state_timeout_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t domain_index;
+
+    // outputs
+    ec_domain_state_t *state;
+} ec_ioctl_domain_state_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+
+    // inputs/outputs
+    uint32_t request_index;
+    uint16_t sdo_index;
+    uint8_t sdo_subindex;
+    size_t size;
+    uint8_t *data;
+    uint32_t timeout;
+    ec_request_state_t state;
+} ec_ioctl_sdo_request_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+
+    // inputs/outputs
+    uint32_t request_index;
+    uint8_t drive_no;
+    uint16_t idn;
+    size_t size;
+    uint8_t *data;
+    uint32_t timeout;
+    ec_request_state_t state;
+} ec_ioctl_soe_request_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+    size_t mem_size;
+
+    // inputs/outputs
+    uint32_t request_index;
+    uint8_t *data;
+    ec_request_state_t state;
+    uint8_t new_data;
+    uint16_t address;
+    size_t transfer_size;
+} ec_ioctl_reg_request_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t config_index;
+
+    // inputs/outputs
+    uint32_t voe_index;
+    uint32_t *vendor_id;
+    uint16_t *vendor_type;
+    size_t size;
+    uint8_t *data;
+    ec_request_state_t state;
+} ec_ioctl_voe_t;
+
+/****************************************************************************/
+
+typedef struct {
+    // inputs
+    uint32_t dev_idx;
+
+    // outputs
+    ec_master_link_state_t *state;
+} ec_ioctl_link_state_t;
+
+/****************************************************************************/
+
+#ifdef __KERNEL__
+
+/** Context data structure for file handles.
+ */
+typedef struct {
+    unsigned int writable; /**< Device was opened with write permission. */
+    unsigned int requested; /**< Master was requested via this file handle. */
+    uint8_t *process_data; /**< Total process data area. */
+    size_t process_data_size; /**< Size of the \a process_data. */
+} ec_ioctl_context_t;
+
+long ec_ioctl(ec_master_t *, ec_ioctl_context_t *, unsigned int,
+        void __user *);
+
+#ifdef EC_RTDM
+
+long ec_ioctl_rtdm_rt(ec_master_t *, ec_ioctl_context_t *, unsigned int,
+        void __user *);
+long ec_ioctl_rtdm_nrt(ec_master_t *, ec_ioctl_context_t *, unsigned int,
+        void __user *);
+
+#ifndef EC_RTDM_XENOMAI_V3
+int ec_rtdm_mmap(ec_ioctl_context_t *, void **);
+#endif
+
+#endif
+
+#endif
+
+/****************************************************************************/
+
+/** \endcond */
+
+#endif
diff --git a/drivers/net/ethercat/master/mailbox.c b/drivers/net/ethercat/master/mailbox.c
new file mode 100644
index 000000000000..05a10fd1c3b1
--- /dev/null
+++ b/drivers/net/ethercat/master/mailbox.c
@@ -0,0 +1,203 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Mailbox functionality.
+*/
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+#include <linux/delay.h>
+
+#include "mailbox.h"
+#include "datagram.h"
+#include "master.h"
+
+/****************************************************************************/
+
+/**
+   Prepares a mailbox-send datagram.
+   \return Pointer to mailbox datagram data, or ERR_PTR() code.
+*/
+
+uint8_t *ec_slave_mbox_prepare_send(const ec_slave_t *slave, /**< slave */
+                                    ec_datagram_t *datagram, /**< datagram */
+                                    uint8_t type, /**< mailbox protocol */
+                                    size_t size /**< size of the data */
+                                    )
+{
+    size_t total_size;
+    int ret;
+
+    if (unlikely(!slave->sii.mailbox_protocols)) {
+        EC_SLAVE_ERR(slave, "Slave does not support mailbox"
+                " communication!\n");
+        return ERR_PTR(-EPROTONOSUPPORT);
+    }
+
+    total_size = EC_MBOX_HEADER_SIZE + size;
+
+    if (unlikely(total_size > slave->configured_rx_mailbox_size)) {
+        EC_SLAVE_ERR(slave, "Data size (%zu) does not fit in mailbox (%u)!\n",
+                total_size, slave->configured_rx_mailbox_size);
+        return ERR_PTR(-ENOBUFS);
+    }
+
+    ret = ec_datagram_fpwr(datagram, slave->station_address,
+            slave->configured_rx_mailbox_offset,
+            slave->configured_rx_mailbox_size);
+    if (ret)
+        return ERR_PTR(ret);
+
+    EC_WRITE_U16(datagram->data,     size); // mailbox service data length
+    EC_WRITE_U16(datagram->data + 2, slave->station_address); // station addr.
+    EC_WRITE_U8 (datagram->data + 4, 0x00); // channel & priority
+    EC_WRITE_U8 (datagram->data + 5, type); // underlying protocol type
+
+    return datagram->data + EC_MBOX_HEADER_SIZE;
+}
+
+/****************************************************************************/
+
+/**
+   Prepares a datagram for checking the mailbox state.
+   \todo Determine sync manager used for receive mailbox
+   \return 0 in case of success, else < 0
+*/
+
+int ec_slave_mbox_prepare_check(const ec_slave_t *slave, /**< slave */
+                                ec_datagram_t *datagram /**< datagram */
+                                )
+{
+    int ret = ec_datagram_fprd(datagram, slave->station_address, 0x808, 8);
+    if (ret)
+        return ret;
+
+    ec_datagram_zero(datagram);
+    return 0;
+}
+
+/****************************************************************************/
+
+/**
+   Processes a mailbox state checking datagram.
+   \return 0 in case of success, else < 0
+*/
+
+int ec_slave_mbox_check(const ec_datagram_t *datagram /**< datagram */)
+{
+    return EC_READ_U8(datagram->data + 5) & 8 ? 1 : 0;
+}
+
+/****************************************************************************/
+
+/**
+   Prepares a datagram to fetch mailbox data.
+   \return 0 in case of success, else < 0
+*/
+
+int ec_slave_mbox_prepare_fetch(const ec_slave_t *slave, /**< slave */
+                                ec_datagram_t *datagram /**< datagram */
+                                )
+{
+    int ret = ec_datagram_fprd(datagram, slave->station_address,
+            slave->configured_tx_mailbox_offset,
+            slave->configured_tx_mailbox_size);
+    if (ret)
+        return ret;
+
+    ec_datagram_zero(datagram);
+    return 0;
+}
+
+/****************************************************************************/
+
+/**
+   Mailbox error codes.
+*/
+
+const ec_code_msg_t mbox_error_messages[] = {
+    {0x00000001, "MBXERR_SYNTAX"},
+    {0x00000002, "MBXERR_UNSUPPORTEDPROTOCOL"},
+    {0x00000003, "MBXERR_INVAILDCHANNEL"},
+    {0x00000004, "MBXERR_SERVICENOTSUPPORTED"},
+    {0x00000005, "MBXERR_INVALIDHEADER"},
+    {0x00000006, "MBXERR_SIZETOOSHORT"},
+    {0x00000007, "MBXERR_NOMOREMEMORY"},
+    {0x00000008, "MBXERR_INVALIDSIZE"},
+    {}
+};
+
+/****************************************************************************/
+
+/** Processes received mailbox data.
+ *
+ * \return Pointer to the received data, or ERR_PTR() code.
+ */
+uint8_t *ec_slave_mbox_fetch(const ec_slave_t *slave, /**< slave */
+                             const ec_datagram_t *datagram, /**< datagram */
+                             uint8_t *type, /**< expected mailbox protocol */
+                             size_t *size /**< size of the received data */
+                             )
+{
+    size_t data_size;
+
+    data_size = EC_READ_U16(datagram->data);
+
+    if (data_size + EC_MBOX_HEADER_SIZE > slave->configured_tx_mailbox_size) {
+        EC_SLAVE_ERR(slave, "Corrupt mailbox response received!\n");
+        ec_print_data(datagram->data, slave->configured_tx_mailbox_size);
+        return ERR_PTR(-EPROTO);
+    }
+
+    *type = EC_READ_U8(datagram->data + 5) & 0x0F;
+    *size = data_size;
+
+    if (*type == 0x00) {
+        const ec_code_msg_t *mbox_msg;
+        uint16_t code = EC_READ_U16(datagram->data + 8);
+
+        EC_SLAVE_ERR(slave, "Mailbox error response received - ");
+
+        for (mbox_msg = mbox_error_messages; mbox_msg->code; mbox_msg++) {
+            if (mbox_msg->code != code)
+                continue;
+            printk(KERN_CONT "Code 0x%04X: \"%s\".\n",
+                    mbox_msg->code, mbox_msg->message);
+            break;
+        }
+
+        if (!mbox_msg->code) {
+            printk(KERN_CONT "Unknown error reply code 0x%04X.\n", code);
+        }
+
+        if (slave->master->debug_level)
+            ec_print_data(datagram->data + EC_MBOX_HEADER_SIZE, data_size);
+
+        return ERR_PTR(-EPROTO);
+    }
+
+    return datagram->data + EC_MBOX_HEADER_SIZE;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/mailbox.h b/drivers/net/ethercat/master/mailbox.h
new file mode 100644
index 000000000000..e9de78cf0ab8
--- /dev/null
+++ b/drivers/net/ethercat/master/mailbox.h
@@ -0,0 +1,64 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Mailbox functionality.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_MAILBOX_H__
+#define __EC_MAILBOX_H__
+
+#include "slave.h"
+
+/****************************************************************************/
+
+/** Size of the mailbox header.
+ */
+#define EC_MBOX_HEADER_SIZE 6
+
+/** Mailbox types.
+ *
+ * These are used in the 'Type' field of the mailbox header.
+ */
+enum {
+    EC_MBOX_TYPE_EOE = 0x02,
+    EC_MBOX_TYPE_COE = 0x03,
+    EC_MBOX_TYPE_FOE = 0x04,
+    EC_MBOX_TYPE_SOE = 0x05,
+    EC_MBOX_TYPE_VOE = 0x0f,
+};
+
+/****************************************************************************/
+
+uint8_t *ec_slave_mbox_prepare_send(const ec_slave_t *, ec_datagram_t *,
+                                    uint8_t, size_t);
+int      ec_slave_mbox_prepare_check(const ec_slave_t *, ec_datagram_t *);
+int      ec_slave_mbox_check(const ec_datagram_t *);
+int      ec_slave_mbox_prepare_fetch(const ec_slave_t *, ec_datagram_t *);
+uint8_t *ec_slave_mbox_fetch(const ec_slave_t *, const ec_datagram_t *,
+                             uint8_t *, size_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/master.c b/drivers/net/ethercat/master/master.c
new file mode 100644
index 000000000000..cf97c10d033b
--- /dev/null
+++ b/drivers/net/ethercat/master/master.c
@@ -0,0 +1,3366 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2020  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *  vim: expandtab
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT master methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/version.h>
+#include <linux/hrtimer.h>
+#include <linux/kthread.h>
+
+#include "globals.h"
+#include "slave.h"
+#include "slave_config.h"
+#include "device.h"
+#include "datagram.h"
+
+#ifdef EC_EOE
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#include <uapi/linux/sched/types.h> // struct sched_param
+#include <linux/sched/types.h> // sched_setscheduler
+#endif
+#include "ethernet.h"
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) || \
+    (defined(CONFIG_PREEMPT_RT_FULL) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+#  define ec_rt_lock_interruptible(lock) \
+          rt_mutex_lock_interruptible(lock)
+#else
+#  define ec_rt_lock_interruptible(lock) \
+          rt_mutex_lock_interruptible(lock, 0)
+#endif
+
+#include "master.h"
+
+/****************************************************************************/
+
+/** Set to 1 to enable external datagram injection debugging.
+ */
+#define DEBUG_INJECT 0
+
+/** Always output corrupted frames.
+ */
+#define FORCE_OUTPUT_CORRUPTED 0
+
+/** SDO injection timeout in microseconds. */
+#define EC_SDO_INJECTION_TIMEOUT 10000
+
+#ifdef EC_HAVE_CYCLES
+
+/** Frame timeout in cycles.
+ */
+static cycles_t timeout_cycles;
+
+/** Timeout for external datagram injection [cycles].
+ */
+static cycles_t ext_injection_timeout_cycles;
+
+#else
+
+/** Frame timeout in jiffies.
+ */
+static unsigned long timeout_jiffies;
+
+/** Timeout for external datagram injection [jiffies].
+ */
+static unsigned long ext_injection_timeout_jiffies;
+
+#endif
+
+/** List of intervals for statistics [s].
+ */
+const unsigned int rate_intervals[] = {
+    1, 10, 60
+};
+
+/****************************************************************************/
+
+void ec_master_clear_config(ec_master_t *);
+void ec_master_clear_slave_configs(ec_master_t *);
+void ec_master_clear_domains(ec_master_t *);
+int ec_master_thread_start(ec_master_t *, int (*)(void *), const char *);
+void ec_master_thread_stop(ec_master_t *);
+void ec_master_inject_external_datagrams(ec_master_t *);
+ec_datagram_t *ec_master_get_external_datagram(ec_master_t *);
+void ec_master_exec_slave_fsms(ec_master_t *);
+void ec_master_send_datagrams(ec_master_t *, ec_device_index_t);
+int ec_master_calc_topology_rec(ec_master_t *, ec_slave_t *, unsigned int *);
+void ec_master_calc_topology(ec_master_t *);
+void ec_master_calc_transmission_delays(ec_master_t *);
+static int ec_master_idle_thread(void *);
+static int ec_master_operation_thread(void *);
+#ifdef EC_EOE
+static int ec_master_eoe_thread(void *);
+#endif
+void ec_master_find_dc_ref_clock(ec_master_t *);
+void ec_master_clear_device_stats(ec_master_t *);
+void ec_master_update_device_stats(ec_master_t *);
+void ec_master_nanosleep(const unsigned long);
+static void sc_reset_task_kicker(struct irq_work *work);
+static void sc_reset_task(struct work_struct *work);
+
+/****************************************************************************/
+
+/** Static variables initializer.
+*/
+void ec_master_init_static(void)
+{
+#ifdef EC_HAVE_CYCLES
+    timeout_cycles = (cycles_t) EC_IO_TIMEOUT /* us */ * (cpu_khz / 1000);
+    ext_injection_timeout_cycles =
+        (cycles_t) EC_SDO_INJECTION_TIMEOUT /* us */ * (cpu_khz / 1000);
+#else
+    // one jiffy may always elapse between time measurement
+    timeout_jiffies = max(EC_IO_TIMEOUT * HZ / 1000000, 1);
+    ext_injection_timeout_jiffies =
+        max(EC_SDO_INJECTION_TIMEOUT * HZ / 1000000, 1);
+#endif
+}
+
+/****************************************************************************/
+
+/**
+   Master constructor.
+   \return 0 in case of success, else < 0
+*/
+
+int ec_master_init(ec_master_t *master, /**< EtherCAT master */
+        unsigned int index, /**< master index */
+        const uint8_t *main_mac, /**< MAC address of main device */
+        const uint8_t *backup_mac, /**< MAC address of backup device */
+        dev_t device_number, /**< Character device number. */
+        struct class *class, /**< Device class. */
+        unsigned int debug_level, /**< Debug level (module parameter). */
+        unsigned int run_on_cpu /**< bind created kernel threads to a cpu */
+        )
+{
+    int ret;
+    unsigned int dev_idx, i;
+
+    master->index = index;
+    master->reserved = 0;
+
+    sema_init(&master->master_sem, 1);
+
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < EC_MAX_NUM_DEVICES; dev_idx++) {
+        master->macs[dev_idx] = NULL;
+    }
+
+    master->macs[EC_DEVICE_MAIN] = main_mac;
+
+#if EC_MAX_NUM_DEVICES > 1
+    master->macs[EC_DEVICE_BACKUP] = backup_mac;
+    master->num_devices = 1 + !ec_mac_is_zero(backup_mac);
+#else
+    if (!ec_mac_is_zero(backup_mac)) {
+        EC_MASTER_WARN(master, "Ignoring backup MAC address!");
+    }
+#endif
+
+    ec_master_clear_device_stats(master);
+
+    sema_init(&master->device_sem, 1);
+
+    master->phase = EC_ORPHANED;
+    master->active = 0;
+    master->config_changed = 0;
+    master->injection_seq_fsm = 0;
+    master->injection_seq_rt = 0;
+
+    master->slaves = NULL;
+    master->slave_count = 0;
+
+    INIT_LIST_HEAD(&master->configs);
+    INIT_LIST_HEAD(&master->domains);
+
+    master->app_time = 0ULL;
+    master->dc_ref_time = 0ULL;
+
+    master->scan_busy = 0;
+    master->scan_index = 0;
+    master->allow_scan = 1;
+    sema_init(&master->scan_sem, 1);
+    init_waitqueue_head(&master->scan_queue);
+
+    master->config_busy = 0;
+    sema_init(&master->config_sem, 1);
+    init_waitqueue_head(&master->config_queue);
+
+    INIT_LIST_HEAD(&master->datagram_queue);
+    master->datagram_index = 0;
+
+    INIT_LIST_HEAD(&master->ext_datagram_queue);
+    sema_init(&master->ext_queue_sem, 1);
+
+    master->ext_ring_idx_rt = 0;
+    master->ext_ring_idx_fsm = 0;
+
+    // init external datagram ring
+    for (i = 0; i < EC_EXT_RING_SIZE; i++) {
+        ec_datagram_t *datagram = &master->ext_datagram_ring[i];
+        ec_datagram_init(datagram);
+        snprintf(datagram->name, EC_DATAGRAM_NAME_SIZE, "ext-%u", i);
+    }
+
+    // send interval in IDLE phase
+    ec_master_set_send_interval(master, 1000000 / HZ);
+
+    master->fsm_slave = NULL;
+    INIT_LIST_HEAD(&master->fsm_exec_list);
+    master->fsm_exec_count = 0U;
+
+    master->debug_level = debug_level;
+    master->run_on_cpu = run_on_cpu;
+    master->stats.timeouts = 0;
+    master->stats.corrupted = 0;
+    master->stats.unmatched = 0;
+    master->stats.output_jiffies = 0;
+
+    master->thread = NULL;
+
+#ifdef EC_EOE
+    master->eoe_thread = NULL;
+    INIT_LIST_HEAD(&master->eoe_handlers);
+#endif
+
+    rt_mutex_init(&master->io_mutex);
+    master->send_cb = NULL;
+    master->receive_cb = NULL;
+    master->cb_data = NULL;
+    master->app_send_cb = NULL;
+    master->app_receive_cb = NULL;
+    master->app_cb_data = NULL;
+
+    INIT_LIST_HEAD(&master->sii_requests);
+    INIT_LIST_HEAD(&master->emerg_reg_requests);
+
+    init_waitqueue_head(&master->request_queue);
+
+    // init devices
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        ret = ec_device_init(&master->devices[dev_idx], master);
+        if (ret < 0) {
+            goto out_clear_devices;
+        }
+    }
+
+    // init state machine datagram
+    ec_datagram_init(&master->fsm_datagram);
+    snprintf(master->fsm_datagram.name, EC_DATAGRAM_NAME_SIZE, "master-fsm");
+    ret = ec_datagram_prealloc(&master->fsm_datagram, EC_MAX_DATA_SIZE);
+    if (ret < 0) {
+        ec_datagram_clear(&master->fsm_datagram);
+        EC_MASTER_ERR(master, "Failed to allocate FSM datagram.\n");
+        goto out_clear_devices;
+    }
+
+    // create state machine object
+    ec_fsm_master_init(&master->fsm, master, &master->fsm_datagram);
+
+    // alloc external datagram ring
+    for (i = 0; i < EC_EXT_RING_SIZE; i++) {
+        ec_datagram_t *datagram = &master->ext_datagram_ring[i];
+        ret = ec_datagram_prealloc(datagram, EC_MAX_DATA_SIZE);
+        if (ret) {
+            EC_MASTER_ERR(master, "Failed to allocate external"
+                    " datagram %u.\n", i);
+            goto out_clear_ext_datagrams;
+        }
+    }
+
+    // init reference sync datagram
+    ec_datagram_init(&master->ref_sync_datagram);
+    snprintf(master->ref_sync_datagram.name, EC_DATAGRAM_NAME_SIZE,
+            "refsync");
+    ret = ec_datagram_prealloc(&master->ref_sync_datagram, 4);
+    if (ret < 0) {
+        ec_datagram_clear(&master->ref_sync_datagram);
+        EC_MASTER_ERR(master, "Failed to allocate reference"
+                " synchronisation datagram.\n");
+        goto out_clear_ext_datagrams;
+    }
+
+    // init sync datagram
+    ec_datagram_init(&master->sync_datagram);
+    snprintf(master->sync_datagram.name, EC_DATAGRAM_NAME_SIZE, "sync");
+    ret = ec_datagram_prealloc(&master->sync_datagram, 4);
+    if (ret < 0) {
+        ec_datagram_clear(&master->sync_datagram);
+        EC_MASTER_ERR(master, "Failed to allocate"
+                " synchronisation datagram.\n");
+        goto out_clear_ref_sync;
+    }
+
+    // init sync monitor datagram
+    ec_datagram_init(&master->sync_mon_datagram);
+    snprintf(master->sync_mon_datagram.name, EC_DATAGRAM_NAME_SIZE,
+            "syncmon");
+    ret = ec_datagram_brd(&master->sync_mon_datagram, 0x092c, 4);
+    if (ret < 0) {
+        ec_datagram_clear(&master->sync_mon_datagram);
+        EC_MASTER_ERR(master, "Failed to allocate sync"
+                " monitoring datagram.\n");
+        goto out_clear_sync;
+    }
+
+    master->dc_ref_config = NULL;
+    master->dc_ref_clock = NULL;
+
+    INIT_WORK(&master->sc_reset_work, sc_reset_task);
+    init_irq_work(&master->sc_reset_work_kicker, sc_reset_task_kicker);
+
+    // init character device
+    ret = ec_cdev_init(&master->cdev, master, device_number);
+    if (ret)
+        goto out_clear_sync_mon;
+
+    master->class_device = device_create(class, NULL,
+            MKDEV(MAJOR(device_number), master->index), NULL,
+            "EtherCAT%u", master->index);
+    if (IS_ERR(master->class_device)) {
+        EC_MASTER_ERR(master, "Failed to create class device!\n");
+        ret = PTR_ERR(master->class_device);
+        goto out_clear_cdev;
+    }
+
+#ifdef EC_RTDM
+    // init RTDM device
+    ret = ec_rtdm_dev_init(&master->rtdm_dev, master);
+    if (ret) {
+        goto out_unregister_class_device;
+    }
+#endif
+
+    return 0;
+
+#ifdef EC_RTDM
+out_unregister_class_device:
+    device_unregister(master->class_device);
+#endif
+out_clear_cdev:
+    ec_cdev_clear(&master->cdev);
+out_clear_sync_mon:
+    ec_datagram_clear(&master->sync_mon_datagram);
+out_clear_sync:
+    ec_datagram_clear(&master->sync_datagram);
+out_clear_ref_sync:
+    ec_datagram_clear(&master->ref_sync_datagram);
+out_clear_ext_datagrams:
+    for (i = 0; i < EC_EXT_RING_SIZE; i++) {
+        ec_datagram_clear(&master->ext_datagram_ring[i]);
+    }
+    ec_fsm_master_clear(&master->fsm);
+    ec_datagram_clear(&master->fsm_datagram);
+out_clear_devices:
+    for (; dev_idx > 0; dev_idx--) {
+        ec_device_clear(&master->devices[dev_idx - 1]);
+    }
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+*/
+void ec_master_clear(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    unsigned int dev_idx, i;
+
+#ifdef EC_RTDM
+    ec_rtdm_dev_clear(&master->rtdm_dev);
+#endif
+
+    device_unregister(master->class_device);
+
+    ec_cdev_clear(&master->cdev);
+
+    irq_work_sync(&master->sc_reset_work_kicker);
+    cancel_work_sync(&master->sc_reset_work);
+
+#ifdef EC_EOE
+    ec_master_clear_eoe_handlers(master);
+#endif
+    ec_master_clear_domains(master);
+    ec_master_clear_slave_configs(master);
+    ec_master_clear_slaves(master);
+
+    ec_datagram_clear(&master->sync_mon_datagram);
+    ec_datagram_clear(&master->sync_datagram);
+    ec_datagram_clear(&master->ref_sync_datagram);
+
+    for (i = 0; i < EC_EXT_RING_SIZE; i++) {
+        ec_datagram_clear(&master->ext_datagram_ring[i]);
+    }
+
+    ec_fsm_master_clear(&master->fsm);
+    ec_datagram_clear(&master->fsm_datagram);
+
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        ec_device_clear(&master->devices[dev_idx]);
+    }
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+/** Clear and free all EoE handlers.
+ */
+void ec_master_clear_eoe_handlers(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    ec_eoe_t *eoe, *next;
+
+    list_for_each_entry_safe(eoe, next, &master->eoe_handlers, list) {
+        list_del(&eoe->list);
+        ec_eoe_clear(eoe);
+        kfree(eoe);
+    }
+}
+#endif
+
+/****************************************************************************/
+
+/** Clear all slave configurations.
+ */
+void ec_master_clear_slave_configs(ec_master_t *master)
+{
+    ec_slave_config_t *sc, *next;
+
+    master->dc_ref_config = NULL;
+    master->fsm.sdo_request = NULL; // mark sdo_request as invalid
+
+    list_for_each_entry_safe(sc, next, &master->configs, list) {
+        list_del(&sc->list);
+        ec_slave_config_clear(sc);
+        kfree(sc);
+    }
+}
+
+/****************************************************************************/
+
+/** Clear all slaves.
+ */
+void ec_master_clear_slaves(ec_master_t *master)
+{
+    ec_slave_t *slave;
+
+    master->dc_ref_clock = NULL;
+
+    // External requests are obsolete, so we wake pending waiters and remove
+    // them from the list.
+
+    while (!list_empty(&master->sii_requests)) {
+        ec_sii_write_request_t *request =
+            list_entry(master->sii_requests.next,
+                    ec_sii_write_request_t, list);
+        list_del_init(&request->list); // dequeue
+        EC_MASTER_WARN(master, "Discarding SII request, slave %u about"
+                " to be deleted.\n", request->slave->ring_position);
+        request->state = EC_INT_REQUEST_FAILURE;
+        wake_up_all(&master->request_queue);
+    }
+
+    master->fsm_slave = NULL;
+    INIT_LIST_HEAD(&master->fsm_exec_list);
+    master->fsm_exec_count = 0;
+
+    for (slave = master->slaves;
+            slave < master->slaves + master->slave_count;
+            slave++) {
+        ec_slave_clear(slave);
+    }
+
+    if (master->slaves) {
+        kfree(master->slaves);
+        master->slaves = NULL;
+    }
+
+    master->slave_count = 0;
+}
+
+/****************************************************************************/
+
+/** Clear all domains.
+ */
+void ec_master_clear_domains(ec_master_t *master)
+{
+    ec_domain_t *domain, *next;
+
+    list_for_each_entry_safe(domain, next, &master->domains, list) {
+        list_del(&domain->list);
+        ec_domain_clear(domain);
+        kfree(domain);
+    }
+}
+
+/****************************************************************************/
+
+/** Clear the configuration applied by the application.
+ */
+void ec_master_clear_config(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    down(&master->master_sem);
+    ec_master_clear_domains(master);
+    ec_master_clear_slave_configs(master);
+    up(&master->master_sem);
+}
+
+/****************************************************************************/
+
+/** Internal sending callback.
+ */
+void ec_master_internal_send_cb(
+        void *cb_data /**< Callback data. */
+        )
+{
+    ec_master_t *master = (ec_master_t *) cb_data;
+    if (ec_rt_lock_interruptible(&master->io_mutex))
+        return;
+    ecrt_master_send_ext(master);
+    rt_mutex_unlock(&master->io_mutex);
+}
+
+/****************************************************************************/
+
+/** Internal receiving callback.
+ */
+void ec_master_internal_receive_cb(
+        void *cb_data /**< Callback data. */
+        )
+{
+    ec_master_t *master = (ec_master_t *) cb_data;
+    if (ec_rt_lock_interruptible(&master->io_mutex))
+        return;
+    ecrt_master_receive(master);
+    rt_mutex_unlock(&master->io_mutex);
+}
+
+/****************************************************************************/
+
+/** Starts the master thread.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_master_thread_start(
+        ec_master_t *master, /**< EtherCAT master */
+        int (*thread_func)(void *), /**< thread function to start */
+        const char *name /**< Thread name. */
+        )
+{
+    EC_MASTER_INFO(master, "Starting %s thread.\n", name);
+    master->thread = kthread_create(thread_func, master, name);
+    if (IS_ERR(master->thread)) {
+        int err = (int) PTR_ERR(master->thread);
+        EC_MASTER_ERR(master, "Failed to start master thread (error %i)!\n",
+                err);
+        master->thread = NULL;
+        return err;
+    }
+    if (0xffffffff != master->run_on_cpu) {
+        EC_MASTER_INFO(master, " binding thread to cpu %u\n",master->run_on_cpu);
+        kthread_bind(master->thread,master->run_on_cpu);
+    }
+    /* Ignoring return value of wake_up_process */
+    (void) wake_up_process(master->thread);
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Stops the master thread.
+ */
+void ec_master_thread_stop(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    unsigned long sleep_jiffies;
+
+    if (!master->thread) {
+        EC_MASTER_WARN(master, "%s(): Already finished!\n", __func__);
+        return;
+    }
+
+    EC_MASTER_DBG(master, 1, "Stopping master thread.\n");
+
+    kthread_stop(master->thread);
+    master->thread = NULL;
+    EC_MASTER_INFO(master, "Master thread exited.\n");
+
+    if (master->fsm_datagram.state != EC_DATAGRAM_SENT) {
+        return;
+    }
+
+    // wait for FSM datagram
+    sleep_jiffies = max(HZ / 100, 1); // 10 ms, at least 1 jiffy
+    schedule_timeout(sleep_jiffies);
+}
+
+/****************************************************************************/
+
+/** Transition function from ORPHANED to IDLE phase.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_master_enter_idle_phase(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    int ret;
+    ec_device_index_t dev_idx;
+
+    EC_MASTER_DBG(master, 1, "ORPHANED -> IDLE.\n");
+
+    master->send_cb = ec_master_internal_send_cb;
+    master->receive_cb = ec_master_internal_receive_cb;
+    master->cb_data = master;
+
+    master->phase = EC_IDLE;
+
+    // reset number of responding slaves to trigger scanning
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        master->fsm.slaves_responding[dev_idx] = 0;
+    }
+
+    ret = ec_master_thread_start(master, ec_master_idle_thread,
+            "EtherCAT-IDLE");
+    if (ret)
+        master->phase = EC_ORPHANED;
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Transition function from IDLE to ORPHANED phase.
+ */
+void ec_master_leave_idle_phase(ec_master_t *master /**< EtherCAT master */)
+{
+    EC_MASTER_DBG(master, 1, "IDLE -> ORPHANED.\n");
+
+    master->phase = EC_ORPHANED;
+
+#ifdef EC_EOE
+    ec_master_eoe_stop(master);
+#endif
+    ec_master_thread_stop(master);
+
+    down(&master->master_sem);
+    ec_master_clear_slaves(master);
+    up(&master->master_sem);
+
+    ec_fsm_master_reset(&master->fsm);
+}
+
+/****************************************************************************/
+
+/** Transition function from IDLE to OPERATION phase.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_master_enter_operation_phase(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    int ret = 0;
+    ec_slave_t *slave;
+
+    EC_MASTER_DBG(master, 1, "IDLE -> OPERATION.\n");
+
+    down(&master->config_sem);
+    if (master->config_busy) {
+        up(&master->config_sem);
+
+        // wait for slave configuration to complete
+        ret = wait_event_interruptible(master->config_queue,
+                    !master->config_busy);
+        if (ret) {
+            EC_MASTER_INFO(master, "Finishing slave configuration"
+                    " interrupted by signal.\n");
+            goto out_return;
+        }
+
+        EC_MASTER_DBG(master, 1, "Waiting for pending slave"
+                " configuration returned.\n");
+    } else {
+        up(&master->config_sem);
+    }
+
+    down(&master->scan_sem);
+    master->allow_scan = 0; // 'lock' the slave list
+    if (!master->scan_busy) {
+        up(&master->scan_sem);
+    } else {
+        up(&master->scan_sem);
+
+        // wait for slave scan to complete
+        ret = wait_event_interruptible(master->scan_queue,
+                !master->scan_busy);
+        if (ret) {
+            EC_MASTER_INFO(master, "Waiting for slave scan"
+                    " interrupted by signal.\n");
+            goto out_allow;
+        }
+
+        EC_MASTER_DBG(master, 1, "Waiting for pending"
+                " slave scan returned.\n");
+    }
+
+    // set states for all slaves
+    for (slave = master->slaves;
+            slave < master->slaves + master->slave_count;
+            slave++) {
+        ec_slave_request_state(slave, EC_SLAVE_STATE_PREOP);
+    }
+
+    master->phase = EC_OPERATION;
+    master->app_send_cb = NULL;
+    master->app_receive_cb = NULL;
+    master->app_cb_data = NULL;
+    return ret;
+
+out_allow:
+    master->allow_scan = 1;
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Transition function from OPERATION to IDLE phase.
+ */
+void ec_master_leave_operation_phase(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    if (master->active) {
+        ecrt_master_deactivate(master); // also clears config
+    } else {
+        ec_master_clear_config(master);
+    }
+
+    /* Re-allow scanning for IDLE phase. */
+    master->allow_scan = 1;
+
+    EC_MASTER_DBG(master, 1, "OPERATION -> IDLE.\n");
+
+    master->phase = EC_IDLE;
+}
+
+/****************************************************************************/
+
+/** Injects external datagrams that fit into the datagram queue.
+ */
+void ec_master_inject_external_datagrams(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    ec_datagram_t *datagram;
+    size_t queue_size = 0, new_queue_size = 0;
+#if DEBUG_INJECT
+    unsigned int datagram_count = 0;
+#endif
+
+    if (master->ext_ring_idx_rt == master->ext_ring_idx_fsm) {
+        // nothing to inject
+        return;
+    }
+
+    list_for_each_entry(datagram, &master->datagram_queue, queue) {
+        if (datagram->state == EC_DATAGRAM_QUEUED) {
+            queue_size += datagram->data_size;
+        }
+    }
+
+#if DEBUG_INJECT
+    EC_MASTER_DBG(master, 1, "Injecting datagrams, queue_size=%zu\n",
+            queue_size);
+#endif
+
+    while (master->ext_ring_idx_rt != master->ext_ring_idx_fsm) {
+        datagram = &master->ext_datagram_ring[master->ext_ring_idx_rt];
+
+        if (datagram->state != EC_DATAGRAM_INIT) {
+            // skip datagram
+            master->ext_ring_idx_rt =
+                (master->ext_ring_idx_rt + 1) % EC_EXT_RING_SIZE;
+            continue;
+        }
+
+        new_queue_size = queue_size + datagram->data_size;
+        if (new_queue_size <= master->max_queue_size) {
+#if DEBUG_INJECT
+            EC_MASTER_DBG(master, 1, "Injecting datagram %s"
+                    " size=%zu, queue_size=%zu\n", datagram->name,
+                    datagram->data_size, new_queue_size);
+            datagram_count++;
+#endif
+#ifdef EC_HAVE_CYCLES
+            datagram->cycles_sent = 0;
+#endif
+            datagram->jiffies_sent = 0;
+            ec_master_queue_datagram(master, datagram);
+            queue_size = new_queue_size;
+        }
+        else if (datagram->data_size > master->max_queue_size) {
+            datagram->state = EC_DATAGRAM_ERROR;
+            EC_MASTER_ERR(master, "External datagram %s is too large,"
+                    " size=%zu, max_queue_size=%zu\n",
+                    datagram->name, datagram->data_size,
+                    master->max_queue_size);
+        }
+        else { // datagram does not fit in the current cycle
+#ifdef EC_HAVE_CYCLES
+            cycles_t cycles_now = get_cycles();
+
+            if (cycles_now - datagram->cycles_sent
+                    > ext_injection_timeout_cycles)
+#else
+            if (jiffies - datagram->jiffies_sent
+                    > ext_injection_timeout_jiffies)
+#endif
+            {
+#if defined EC_RT_SYSLOG || DEBUG_INJECT
+                unsigned int time_us;
+#endif
+
+                datagram->state = EC_DATAGRAM_ERROR;
+
+#if defined EC_RT_SYSLOG || DEBUG_INJECT
+#ifdef EC_HAVE_CYCLES
+                time_us = (unsigned int)
+                    ((cycles_now - datagram->cycles_sent) * 1000LL)
+                    / cpu_khz;
+#else
+                time_us = (unsigned int)
+                    ((jiffies - datagram->jiffies_sent) * 1000000 / HZ);
+#endif
+                EC_MASTER_ERR(master, "Timeout %u us: Injecting"
+                        " external datagram %s size=%zu,"
+                        " max_queue_size=%zu\n", time_us, datagram->name,
+                        datagram->data_size, master->max_queue_size);
+#endif
+            }
+            else {
+#if DEBUG_INJECT
+                EC_MASTER_DBG(master, 1, "Deferred injecting"
+                        " external datagram %s size=%u, queue_size=%u\n",
+                        datagram->name, datagram->data_size, queue_size);
+#endif
+                break;
+            }
+        }
+
+        master->ext_ring_idx_rt =
+            (master->ext_ring_idx_rt + 1) % EC_EXT_RING_SIZE;
+    }
+
+#if DEBUG_INJECT
+    EC_MASTER_DBG(master, 1, "Injected %u datagrams.\n", datagram_count);
+#endif
+}
+
+/****************************************************************************/
+
+/** Sets the expected interval between calls to ecrt_master_send
+ * and calculates the maximum amount of data to queue.
+ */
+void ec_master_set_send_interval(
+        ec_master_t *master, /**< EtherCAT master */
+        unsigned int send_interval /**< Send interval */
+        )
+{
+    master->send_interval = send_interval;
+    master->max_queue_size =
+        (send_interval * 1000) / EC_BYTE_TRANSMISSION_TIME_NS;
+    master->max_queue_size -= master->max_queue_size / 10;
+}
+
+/****************************************************************************/
+
+/** Searches for a free datagram in the external datagram ring.
+ *
+ * \return Next free datagram, or NULL.
+ */
+ec_datagram_t *ec_master_get_external_datagram(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    if ((master->ext_ring_idx_fsm + 1) % EC_EXT_RING_SIZE !=
+            master->ext_ring_idx_rt) {
+        ec_datagram_t *datagram =
+            &master->ext_datagram_ring[master->ext_ring_idx_fsm];
+        return datagram;
+    }
+    else {
+        return NULL;
+    }
+}
+
+/****************************************************************************/
+
+/** Places a datagram in the datagram queue.
+ */
+void ec_master_queue_datagram(
+        ec_master_t *master, /**< EtherCAT master */
+        ec_datagram_t *datagram /**< datagram */
+        )
+{
+    ec_datagram_t *queued_datagram;
+
+    /* It is possible, that a datagram in the queue is re-initialized with the
+     * ec_datagram_<type>() methods and then shall be queued with this method.
+     * In that case, the state is already reset to EC_DATAGRAM_INIT. Check if
+     * the datagram is queued to avoid duplicate queuing (which results in an
+     * infinite loop!). Set the state to EC_DATAGRAM_QUEUED again, probably
+     * causing an unmatched datagram. */
+    list_for_each_entry(queued_datagram, &master->datagram_queue, queue) {
+        if (queued_datagram == datagram) {
+            datagram->skip_count++;
+#ifdef EC_RT_SYSLOG
+            EC_MASTER_DBG(master, 1,
+                    "Datagram %p already queued (skipping).\n", datagram);
+#endif
+            datagram->state = EC_DATAGRAM_QUEUED;
+            return;
+        }
+    }
+
+    list_add_tail(&datagram->queue, &master->datagram_queue);
+    datagram->state = EC_DATAGRAM_QUEUED;
+}
+
+/****************************************************************************/
+
+/** Places a datagram in the non-application datagram queue.
+ */
+void ec_master_queue_datagram_ext(
+        ec_master_t *master, /**< EtherCAT master */
+        ec_datagram_t *datagram /**< datagram */
+        )
+{
+    down(&master->ext_queue_sem);
+    list_add_tail(&datagram->ext_queue, &master->ext_datagram_queue);
+    up(&master->ext_queue_sem);
+}
+
+/****************************************************************************/
+
+/** Sends the datagrams in the queue for a certain device.
+ *
+ */
+void ec_master_send_datagrams(
+        ec_master_t *master, /**< EtherCAT master */
+        ec_device_index_t device_index /**< Device index. */
+        )
+{
+    ec_datagram_t *datagram, *next;
+    size_t datagram_size;
+    uint8_t *frame_data, *cur_data = NULL;
+    void *follows_word;
+#ifdef EC_HAVE_CYCLES
+    cycles_t cycles_start, cycles_sent, cycles_end;
+#endif
+    unsigned long jiffies_sent;
+    unsigned int frame_count, more_datagrams_waiting;
+    struct list_head sent_datagrams;
+
+#ifdef EC_HAVE_CYCLES
+    cycles_start = get_cycles();
+#endif
+    frame_count = 0;
+    INIT_LIST_HEAD(&sent_datagrams);
+
+    EC_MASTER_DBG(master, 2, "%s(device_index = %u)\n",
+            __func__, device_index);
+
+    do {
+        frame_data = NULL;
+        follows_word = NULL;
+        more_datagrams_waiting = 0;
+
+        // fill current frame with datagrams
+        list_for_each_entry(datagram, &master->datagram_queue, queue) {
+            if (datagram->state != EC_DATAGRAM_QUEUED ||
+                    datagram->device_index != device_index) {
+                continue;
+            }
+
+            if (!frame_data) {
+                // fetch pointer to transmit socket buffer
+                frame_data =
+                    ec_device_tx_data(&master->devices[device_index]);
+                cur_data = frame_data + EC_FRAME_HEADER_SIZE;
+            }
+
+            // does the current datagram fit in the frame?
+            datagram_size = EC_DATAGRAM_HEADER_SIZE + datagram->data_size
+                + EC_DATAGRAM_FOOTER_SIZE;
+            if (cur_data - frame_data + datagram_size > ETH_DATA_LEN) {
+                more_datagrams_waiting = 1;
+                break;
+            }
+
+            list_add_tail(&datagram->sent, &sent_datagrams);
+            datagram->index = master->datagram_index++;
+
+            EC_MASTER_DBG(master, 2, "Adding datagram 0x%02X\n",
+                    datagram->index);
+
+            // set "datagram following" flag in previous datagram
+            if (follows_word) {
+                EC_WRITE_U16(follows_word,
+                        EC_READ_U16(follows_word) | 0x8000);
+            }
+
+            // EtherCAT datagram header
+            EC_WRITE_U8 (cur_data, datagram->type);
+            EC_WRITE_U8 (cur_data + 1, datagram->index);
+            memcpy(cur_data + 2, datagram->address, EC_ADDR_LEN);
+            EC_WRITE_U16(cur_data + 6, datagram->data_size & 0x7FF);
+            EC_WRITE_U16(cur_data + 8, 0x0000);
+            follows_word = cur_data + 6;
+            cur_data += EC_DATAGRAM_HEADER_SIZE;
+
+            // EtherCAT datagram data
+            memcpy(cur_data, datagram->data, datagram->data_size);
+            cur_data += datagram->data_size;
+
+            // EtherCAT datagram footer
+            EC_WRITE_U16(cur_data, 0x0000); // reset working counter
+            cur_data += EC_DATAGRAM_FOOTER_SIZE;
+        }
+
+        if (list_empty(&sent_datagrams)) {
+            EC_MASTER_DBG(master, 2, "nothing to send.\n");
+            break;
+        }
+
+        // EtherCAT frame header
+        EC_WRITE_U16(frame_data, ((cur_data - frame_data
+                        - EC_FRAME_HEADER_SIZE) & 0x7FF) | 0x1000);
+
+        // pad frame
+        while (cur_data - frame_data < ETH_ZLEN - ETH_HLEN)
+            EC_WRITE_U8(cur_data++, 0x00);
+
+        EC_MASTER_DBG(master, 2, "frame size: %zu\n", cur_data - frame_data);
+
+        // send frame
+        ec_device_send(&master->devices[device_index],
+                cur_data - frame_data);
+#ifdef EC_HAVE_CYCLES
+        cycles_sent = get_cycles();
+#endif
+        jiffies_sent = jiffies;
+
+        // set datagram states and sending timestamps
+        list_for_each_entry_safe(datagram, next, &sent_datagrams, sent) {
+            datagram->state = EC_DATAGRAM_SENT;
+#ifdef EC_HAVE_CYCLES
+            datagram->cycles_sent = cycles_sent;
+#endif
+            datagram->jiffies_sent = jiffies_sent;
+            list_del_init(&datagram->sent); // empty list of sent datagrams
+        }
+
+        frame_count++;
+    }
+    while (more_datagrams_waiting);
+
+#ifdef EC_HAVE_CYCLES
+    if (unlikely(master->debug_level > 1)) {
+        cycles_end = get_cycles();
+        EC_MASTER_DBG(master, 0, "%s()"
+                " sent %u frames in %uus.\n", __func__, frame_count,
+               (unsigned int) (cycles_end - cycles_start) * 1000 / cpu_khz);
+    }
+#endif
+}
+
+/****************************************************************************/
+
+/** Processes a received frame.
+ *
+ * This function is called by the network driver for every received frame.
+ *
+ * \return 0 in case of success, else < 0
+ */
+void ec_master_receive_datagrams(
+        ec_master_t *master, /**< EtherCAT master */
+        ec_device_t *device, /**< EtherCAT device */
+        const uint8_t *frame_data, /**< frame data */
+        size_t size /**< size of the received data */
+        )
+{
+    size_t frame_size, data_size;
+    uint8_t datagram_type, datagram_index;
+    unsigned int cmd_follows, matched;
+    const uint8_t *cur_data;
+    ec_datagram_t *datagram;
+
+    if (unlikely(size < EC_FRAME_HEADER_SIZE)) {
+        if (master->debug_level || FORCE_OUTPUT_CORRUPTED) {
+            EC_MASTER_DBG(master, 0, "Corrupted frame received"
+                    " on %s (size %zu < %u byte):\n",
+                    device->dev->name, size, EC_FRAME_HEADER_SIZE);
+            ec_print_data(frame_data, size);
+        }
+        master->stats.corrupted++;
+#ifdef EC_RT_SYSLOG
+        ec_master_output_stats(master);
+#endif
+        return;
+    }
+
+    cur_data = frame_data;
+
+    // check length of entire frame
+    frame_size = EC_READ_U16(cur_data) & 0x07FF;
+    cur_data += EC_FRAME_HEADER_SIZE;
+
+    if (unlikely(frame_size > size)) {
+        if (master->debug_level || FORCE_OUTPUT_CORRUPTED) {
+            EC_MASTER_DBG(master, 0, "Corrupted frame received"
+                    " on %s (invalid frame size %zu for "
+                    "received size %zu):\n", device->dev->name,
+                    frame_size, size);
+            ec_print_data(frame_data, size);
+        }
+        master->stats.corrupted++;
+#ifdef EC_RT_SYSLOG
+        ec_master_output_stats(master);
+#endif
+        return;
+    }
+
+    cmd_follows = 1;
+    while (cmd_follows) {
+        // process datagram header
+        datagram_type  = EC_READ_U8 (cur_data);
+        datagram_index = EC_READ_U8 (cur_data + 1);
+        data_size      = EC_READ_U16(cur_data + 6) & 0x07FF;
+        cmd_follows    = EC_READ_U16(cur_data + 6) & 0x8000;
+        cur_data += EC_DATAGRAM_HEADER_SIZE;
+
+        if (unlikely(cur_data - frame_data
+                     + data_size + EC_DATAGRAM_FOOTER_SIZE > size)) {
+            if (master->debug_level || FORCE_OUTPUT_CORRUPTED) {
+                EC_MASTER_DBG(master, 0, "Corrupted frame received"
+                        " on %s (invalid data size %zu):\n",
+                        device->dev->name, data_size);
+                ec_print_data(frame_data, size);
+            }
+            master->stats.corrupted++;
+#ifdef EC_RT_SYSLOG
+            ec_master_output_stats(master);
+#endif
+            return;
+        }
+
+        // search for matching datagram in the queue
+        matched = 0;
+        list_for_each_entry(datagram, &master->datagram_queue, queue) {
+            if (datagram->index == datagram_index
+                && datagram->state == EC_DATAGRAM_SENT
+                && datagram->type == datagram_type
+                && datagram->data_size == data_size) {
+                matched = 1;
+                break;
+            }
+        }
+
+        // no matching datagram was found
+        if (!matched) {
+            master->stats.unmatched++;
+#ifdef EC_RT_SYSLOG
+            ec_master_output_stats(master);
+#endif
+
+            if (unlikely(master->debug_level > 0)) {
+                EC_MASTER_DBG(master, 0, "UNMATCHED datagram:\n");
+                ec_print_data(cur_data - EC_DATAGRAM_HEADER_SIZE,
+                        EC_DATAGRAM_HEADER_SIZE + data_size
+                        + EC_DATAGRAM_FOOTER_SIZE);
+#ifdef EC_DEBUG_RING
+                ec_device_debug_ring_print(&master->devices[EC_DEVICE_MAIN]);
+#endif
+            }
+
+            cur_data += data_size + EC_DATAGRAM_FOOTER_SIZE;
+            continue;
+        }
+
+        if (datagram->type != EC_DATAGRAM_APWR &&
+                datagram->type != EC_DATAGRAM_FPWR &&
+                datagram->type != EC_DATAGRAM_BWR &&
+                datagram->type != EC_DATAGRAM_LWR) {
+            // copy received data into the datagram memory,
+            // if something has been read
+            memcpy(datagram->data, cur_data, data_size);
+        }
+        cur_data += data_size;
+
+        // set the datagram's working counter
+        datagram->working_counter = EC_READ_U16(cur_data);
+        cur_data += EC_DATAGRAM_FOOTER_SIZE;
+
+        /*
+         * During warm-up, ensure that once the OP thread observes
+         * datagram->state == EC_DATAGRAM_RECEIVED, all other fields
+         * of the datagram are already visible.
+         */
+        smp_wmb();
+
+        // dequeue the received datagram
+        datagram->state = EC_DATAGRAM_RECEIVED;
+#ifdef EC_HAVE_CYCLES
+        datagram->cycles_received =
+            master->devices[EC_DEVICE_MAIN].cycles_poll;
+#endif
+        datagram->jiffies_received =
+            master->devices[EC_DEVICE_MAIN].jiffies_poll;
+        list_del_init(&datagram->queue);
+    }
+}
+
+/****************************************************************************/
+
+/** Output master statistics.
+ *
+ * This function outputs statistical data on demand, but not more often than
+ * necessary. The output happens at most once a second.
+ */
+void ec_master_output_stats(ec_master_t *master /**< EtherCAT master */)
+{
+    if (unlikely(jiffies - master->stats.output_jiffies >= HZ)) {
+        master->stats.output_jiffies = jiffies;
+
+        if (master->stats.timeouts) {
+            EC_MASTER_WARN(master, "%u datagram%s TIMED OUT!\n",
+                    master->stats.timeouts,
+                    master->stats.timeouts == 1 ? "" : "s");
+            master->stats.timeouts = 0;
+        }
+        if (master->stats.corrupted) {
+            EC_MASTER_WARN(master, "%u frame%s CORRUPTED!\n",
+                    master->stats.corrupted,
+                    master->stats.corrupted == 1 ? "" : "s");
+            master->stats.corrupted = 0;
+        }
+        if (master->stats.unmatched) {
+            EC_MASTER_WARN(master, "%u datagram%s UNMATCHED!\n",
+                    master->stats.unmatched,
+                    master->stats.unmatched == 1 ? "" : "s");
+            master->stats.unmatched = 0;
+        }
+    }
+}
+
+/****************************************************************************/
+
+/** Clears the common device statistics.
+ */
+void ec_master_clear_device_stats(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    unsigned int i;
+
+    // zero frame statistics
+    master->device_stats.tx_count = 0;
+    master->device_stats.last_tx_count = 0;
+    master->device_stats.rx_count = 0;
+    master->device_stats.last_rx_count = 0;
+    master->device_stats.tx_bytes = 0;
+    master->device_stats.last_tx_bytes = 0;
+    master->device_stats.rx_bytes = 0;
+    master->device_stats.last_rx_bytes = 0;
+    master->device_stats.last_loss = 0;
+
+    for (i = 0; i < EC_RATE_COUNT; i++) {
+        master->device_stats.tx_frame_rates[i] = 0;
+        master->device_stats.rx_frame_rates[i] = 0;
+        master->device_stats.tx_byte_rates[i] = 0;
+        master->device_stats.rx_byte_rates[i] = 0;
+        master->device_stats.loss_rates[i] = 0;
+    }
+
+    master->device_stats.jiffies = 0;
+}
+
+/****************************************************************************/
+
+/** Updates the common device statistics.
+ */
+void ec_master_update_device_stats(
+        ec_master_t *master /**< EtherCAT master */
+        )
+{
+    ec_device_stats_t *s = &master->device_stats;
+    s32 tx_frame_rate, rx_frame_rate, tx_byte_rate, rx_byte_rate, loss_rate;
+    u64 loss;
+    unsigned int i, dev_idx;
+
+    // frame statistics
+    if (likely(jiffies - s->jiffies < HZ)) {
+        return;
+    }
+
+    tx_frame_rate = (s->tx_count - s->last_tx_count) * 1000;
+    rx_frame_rate = (s->rx_count - s->last_rx_count) * 1000;
+    tx_byte_rate = s->tx_bytes - s->last_tx_bytes;
+    rx_byte_rate = s->rx_bytes - s->last_rx_bytes;
+    loss = s->tx_count - s->rx_count;
+    loss_rate = (loss - s->last_loss) * 1000;
+
+    /* Low-pass filter:
+     *      Y_n = y_(n - 1) + T / tau * (x - y_(n - 1))   | T = 1
+     *   -> Y_n += (x - y_(n - 1)) / tau
+     */
+    for (i = 0; i < EC_RATE_COUNT; i++) {
+        s32 n = rate_intervals[i];
+        s->tx_frame_rates[i] += (tx_frame_rate - s->tx_frame_rates[i]) / n;
+        s->rx_frame_rates[i] += (rx_frame_rate - s->rx_frame_rates[i]) / n;
+        s->tx_byte_rates[i] += (tx_byte_rate - s->tx_byte_rates[i]) / n;
+        s->rx_byte_rates[i] += (rx_byte_rate - s->rx_byte_rates[i]) / n;
+        s->loss_rates[i] += (loss_rate - s->loss_rates[i]) / n;
+    }
+
+    s->last_tx_count = s->tx_count;
+    s->last_rx_count = s->rx_count;
+    s->last_tx_bytes = s->tx_bytes;
+    s->last_rx_bytes = s->rx_bytes;
+    s->last_loss = loss;
+
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        ec_device_update_stats(&master->devices[dev_idx]);
+    }
+
+    s->jiffies = jiffies;
+}
+
+/****************************************************************************/
+
+#ifdef EC_USE_HRTIMER
+
+/*
+ * Sleep related functions:
+ */
+static enum hrtimer_restart ec_master_nanosleep_wakeup(struct hrtimer *timer)
+{
+    struct hrtimer_sleeper *t =
+        container_of(timer, struct hrtimer_sleeper, timer);
+    struct task_struct *task = t->task;
+
+    t->task = NULL;
+    if (task)
+        wake_up_process(task);
+
+    return HRTIMER_NORESTART;
+}
+
+/****************************************************************************/
+
+void ec_master_nanosleep(const unsigned long nsecs)
+{
+    struct hrtimer_sleeper t;
+    enum hrtimer_mode mode = HRTIMER_MODE_REL;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
+    hrtimer_setup(&t.timer, ec_master_nanosleep_wakeup,
+            CLOCK_MONOTONIC, mode);
+#else
+    hrtimer_init(&t.timer, CLOCK_MONOTONIC, mode);
+    t.timer.function = ec_master_nanosleep_wakeup;
+#endif
+    t.task = current;
+    hrtimer_set_expires(&t.timer, ktime_set(0, nsecs));
+
+    do {
+        set_current_state(TASK_INTERRUPTIBLE);
+        hrtimer_start(&t.timer, hrtimer_get_expires(&t.timer), mode);
+
+        if (likely(t.task))
+            schedule();
+
+        hrtimer_cancel(&t.timer);
+        mode = HRTIMER_MODE_ABS;
+
+    } while (t.task && !signal_pending(current));
+}
+
+#endif // EC_USE_HRTIMER
+
+/****************************************************************************/
+
+/** Execute slave FSMs.
+ */
+void ec_master_exec_slave_fsms(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    ec_datagram_t *datagram;
+    ec_fsm_slave_t *fsm, *next;
+    unsigned int count = 0;
+
+    list_for_each_entry_safe(fsm, next, &master->fsm_exec_list, list) {
+        if (!fsm->datagram) {
+            EC_MASTER_WARN(master, "Slave %u FSM has zero datagram."
+                    "This is a bug!\n", fsm->slave->ring_position);
+            list_del_init(&fsm->list);
+            master->fsm_exec_count--;
+            return;
+        }
+
+        if (fsm->datagram->state == EC_DATAGRAM_INIT ||
+                fsm->datagram->state == EC_DATAGRAM_QUEUED ||
+                fsm->datagram->state == EC_DATAGRAM_SENT) {
+            // previous datagram was not sent or received yet.
+            // wait until next thread execution
+            return;
+        }
+
+        datagram = ec_master_get_external_datagram(master);
+        if (!datagram) {
+            // no free datagrams at the moment
+            EC_MASTER_WARN(master, "No free datagram during"
+                    " slave FSM execution. This is a bug!\n");
+            continue;
+        }
+
+#if DEBUG_INJECT
+        EC_MASTER_DBG(master, 1, "Executing slave %u FSM.\n",
+                fsm->slave->ring_position);
+#endif
+        if (ec_fsm_slave_exec(fsm, datagram)) {
+            // FSM consumed datagram
+#if DEBUG_INJECT
+            EC_MASTER_DBG(master, 1, "FSM consumed datagram %s\n",
+                    datagram->name);
+#endif
+            master->ext_ring_idx_fsm =
+                (master->ext_ring_idx_fsm + 1) % EC_EXT_RING_SIZE;
+        }
+        else {
+            // FSM finished
+            list_del_init(&fsm->list);
+            master->fsm_exec_count--;
+#if DEBUG_INJECT
+            EC_MASTER_DBG(master, 1, "FSM finished. %u remaining.\n",
+                    master->fsm_exec_count);
+#endif
+        }
+    }
+
+    while (master->fsm_exec_count < EC_EXT_RING_SIZE / 2
+            && count < master->slave_count) {
+
+        if (ec_fsm_slave_is_ready(&master->fsm_slave->fsm)) {
+            datagram = ec_master_get_external_datagram(master);
+
+            if (ec_fsm_slave_exec(&master->fsm_slave->fsm, datagram)) {
+                master->ext_ring_idx_fsm =
+                    (master->ext_ring_idx_fsm + 1) % EC_EXT_RING_SIZE;
+                list_add_tail(&master->fsm_slave->fsm.list,
+                        &master->fsm_exec_list);
+                master->fsm_exec_count++;
+#if DEBUG_INJECT
+                EC_MASTER_DBG(master, 1, "New slave %u FSM"
+                        " consumed datagram %s, now %u FSMs in list.\n",
+                        master->fsm_slave->ring_position, datagram->name,
+                        master->fsm_exec_count);
+#endif
+            }
+        }
+
+        master->fsm_slave++;
+        if (master->fsm_slave >= master->slaves + master->slave_count) {
+            master->fsm_slave = master->slaves;
+        }
+        count++;
+    }
+}
+
+/****************************************************************************/
+
+/** Master kernel thread function for IDLE phase.
+ */
+static int ec_master_idle_thread(void *priv_data)
+{
+    ec_master_t *master = (ec_master_t *) priv_data;
+    int fsm_exec;
+#ifdef EC_USE_HRTIMER
+    size_t sent_bytes;
+#endif
+
+    // send interval in IDLE phase
+    ec_master_set_send_interval(master, 1000000 / HZ);
+
+    EC_MASTER_DBG(master, 1, "Idle thread running with send interval = %u us,"
+            " max data size=%zu\n", master->send_interval,
+            master->max_queue_size);
+
+    while (!kthread_should_stop()) {
+        ec_datagram_output_stats(&master->fsm_datagram);
+
+        // receive
+        if (ec_rt_lock_interruptible(&master->io_mutex))
+            break;
+        ecrt_master_receive(master);
+        rt_mutex_unlock(&master->io_mutex);
+
+        // execute master & slave state machines
+        if (down_interruptible(&master->master_sem)) {
+            break;
+        }
+
+        fsm_exec = ec_fsm_master_exec(&master->fsm);
+
+        ec_master_exec_slave_fsms(master);
+
+        up(&master->master_sem);
+
+        // queue and send
+        if (ec_rt_lock_interruptible(&master->io_mutex))
+            break;
+        if (fsm_exec) {
+            ec_master_queue_datagram(master, &master->fsm_datagram);
+        }
+        ecrt_master_send(master);
+#ifdef EC_USE_HRTIMER
+        sent_bytes = master->devices[EC_DEVICE_MAIN].tx_skb[
+            master->devices[EC_DEVICE_MAIN].tx_ring_index]->len;
+#endif
+        rt_mutex_unlock(&master->io_mutex);
+
+        if (ec_fsm_master_idle(&master->fsm)) {
+#ifdef EC_USE_HRTIMER
+            ec_master_nanosleep(master->send_interval * 1000);
+#else
+            set_current_state(TASK_INTERRUPTIBLE);
+            schedule_timeout(1);
+#endif
+        } else {
+#ifdef EC_USE_HRTIMER
+            ec_master_nanosleep(sent_bytes * EC_BYTE_TRANSMISSION_TIME_NS);
+#else
+            schedule();
+#endif
+        }
+    }
+
+    EC_MASTER_DBG(master, 1, "Master IDLE thread exiting...\n");
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Master kernel thread function for OPERATION phase.
+ */
+static int ec_master_operation_thread(void *priv_data)
+{
+    ec_master_t *master = (ec_master_t *) priv_data;
+
+    EC_MASTER_DBG(master, 1, "Operation thread running"
+            " with fsm interval = %u us, max data size=%zu\n",
+            master->send_interval, master->max_queue_size);
+
+    while (!kthread_should_stop()) {
+        ec_datagram_output_stats(&master->fsm_datagram);
+
+        if (master->injection_seq_rt == master->injection_seq_fsm) {
+            // output statistics
+            ec_master_output_stats(master);
+
+            // execute master & slave state machines
+            if (down_interruptible(&master->master_sem)) {
+                break;
+            }
+
+            if (ec_fsm_master_exec(&master->fsm)) {
+                // Inject datagrams (let the RT thread queue them, see
+                // ecrt_master_send())
+                master->injection_seq_fsm++;
+            }
+
+            ec_master_exec_slave_fsms(master);
+
+            up(&master->master_sem);
+        }
+
+#ifdef EC_USE_HRTIMER
+        // the op thread should not work faster than the sending RT thread
+        ec_master_nanosleep(master->send_interval * 1000);
+#else
+        if (ec_fsm_master_idle(&master->fsm)) {
+            set_current_state(TASK_INTERRUPTIBLE);
+            schedule_timeout(1);
+        }
+        else {
+            schedule();
+        }
+#endif
+    }
+
+    EC_MASTER_DBG(master, 1, "Master OP thread exiting...\n");
+    return 0;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/* compatibility for priority changes */
+static inline void set_normal_priority(struct task_struct *p, int nice)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0)
+    sched_set_normal(p, nice);
+#else
+    struct sched_param param = { .sched_priority = 0 };
+    sched_setscheduler(p, SCHED_NORMAL, &param);
+    set_user_nice(p, nice);
+#endif
+}
+
+/****************************************************************************/
+
+/** Starts Ethernet over EtherCAT processing on demand.
+ */
+void ec_master_eoe_start(ec_master_t *master /**< EtherCAT master */)
+{
+    if (master->eoe_thread) {
+        EC_MASTER_WARN(master, "EoE already running!\n");
+        return;
+    }
+
+    if (list_empty(&master->eoe_handlers))
+        return;
+
+    if (!master->send_cb || !master->receive_cb) {
+        EC_MASTER_WARN(master, "No EoE processing"
+                " because of missing callbacks!\n");
+        return;
+    }
+
+    EC_MASTER_INFO(master, "Starting EoE thread.\n");
+    master->eoe_thread = kthread_run(ec_master_eoe_thread, master,
+            "EtherCAT-EoE");
+    if (IS_ERR(master->eoe_thread)) {
+        int err = (int) PTR_ERR(master->eoe_thread);
+        EC_MASTER_ERR(master, "Failed to start EoE thread (error %i)!\n",
+                err);
+        master->eoe_thread = NULL;
+        return;
+    }
+
+    set_normal_priority(master->eoe_thread, 0);
+}
+
+/****************************************************************************/
+
+/** Stops the Ethernet over EtherCAT processing.
+ */
+void ec_master_eoe_stop(ec_master_t *master /**< EtherCAT master */)
+{
+    if (master->eoe_thread) {
+        EC_MASTER_INFO(master, "Stopping EoE thread.\n");
+
+        kthread_stop(master->eoe_thread);
+        master->eoe_thread = NULL;
+        EC_MASTER_INFO(master, "EoE thread exited.\n");
+    }
+}
+
+/****************************************************************************/
+
+/** Does the Ethernet over EtherCAT processing.
+ */
+static int ec_master_eoe_thread(void *priv_data)
+{
+    ec_master_t *master = (ec_master_t *) priv_data;
+    ec_eoe_t *eoe;
+    unsigned int none_open, sth_to_send, all_idle;
+
+    EC_MASTER_DBG(master, 1, "EoE thread running.\n");
+
+    while (!kthread_should_stop()) {
+        none_open = 1;
+        all_idle = 1;
+
+        list_for_each_entry(eoe, &master->eoe_handlers, list) {
+            if (ec_eoe_is_open(eoe)) {
+                none_open = 0;
+                break;
+            }
+        }
+        if (none_open)
+            goto schedule;
+
+        // receive datagrams
+        master->receive_cb(master->cb_data);
+
+        // actual EoE processing
+        sth_to_send = 0;
+        list_for_each_entry(eoe, &master->eoe_handlers, list) {
+            ec_eoe_run(eoe);
+            if (eoe->queue_datagram) {
+                sth_to_send = 1;
+            }
+            if (!ec_eoe_is_idle(eoe)) {
+                all_idle = 0;
+            }
+        }
+
+        if (sth_to_send) {
+            list_for_each_entry(eoe, &master->eoe_handlers, list) {
+                ec_eoe_queue(eoe);
+            }
+            // (try to) send datagrams
+            master->send_cb(master->cb_data);
+        }
+
+schedule:
+        if (all_idle) {
+            set_current_state(TASK_INTERRUPTIBLE);
+            schedule_timeout(1);
+        } else {
+            schedule();
+        }
+    }
+
+    EC_MASTER_DBG(master, 1, "EoE thread exiting...\n");
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+/** Attaches the slave configurations to the slaves.
+ */
+void ec_master_attach_slave_configs(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    ec_slave_config_t *sc;
+
+    list_for_each_entry(sc, &master->configs, list) {
+        ec_slave_config_attach(sc);
+    }
+}
+
+/****************************************************************************/
+
+/** Common implementation for ec_master_find_slave()
+ * and ec_master_find_slave_const().
+ */
+#define EC_FIND_SLAVE \
+    do { \
+        if (alias) { \
+            for (; slave < master->slaves + master->slave_count; \
+                    slave++) { \
+                if (slave->effective_alias == alias) \
+                break; \
+            } \
+            if (slave == master->slaves + master->slave_count) \
+            return NULL; \
+        } \
+        \
+        slave += position; \
+        if (slave < master->slaves + master->slave_count) { \
+            return slave; \
+        } else { \
+            return NULL; \
+        } \
+    } while (0)
+
+/** Finds a slave in the bus, given the alias and position.
+ *
+ * \return Search result, or NULL.
+ */
+ec_slave_t *ec_master_find_slave(
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t alias, /**< Slave alias. */
+        uint16_t position /**< Slave position. */
+        )
+{
+    ec_slave_t *slave = master->slaves;
+    EC_FIND_SLAVE;
+}
+
+/** Finds a slave in the bus, given the alias and position.
+ *
+ * Const version.
+ *
+ * \return Search result, or NULL.
+ */
+const ec_slave_t *ec_master_find_slave_const(
+        const ec_master_t *master, /**< EtherCAT master. */
+        uint16_t alias, /**< Slave alias. */
+        uint16_t position /**< Slave position. */
+        )
+{
+    const ec_slave_t *slave = master->slaves;
+    EC_FIND_SLAVE;
+}
+
+/****************************************************************************/
+
+/** Get the number of slave configurations provided by the application.
+ *
+ * \return Number of configurations.
+ */
+unsigned int ec_master_config_count(
+        const ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    const ec_slave_config_t *sc;
+    unsigned int count = 0;
+
+    list_for_each_entry(sc, &master->configs, list) {
+        count++;
+    }
+
+    return count;
+}
+
+/****************************************************************************/
+
+/** Common implementation for ec_master_get_config()
+ * and ec_master_get_config_const().
+ */
+#define EC_FIND_CONFIG \
+    do { \
+        list_for_each_entry(sc, &master->configs, list) { \
+            if (pos--) \
+                continue; \
+            return sc; \
+        } \
+        return NULL; \
+    } while (0)
+
+/** Get a slave configuration via its position in the list.
+ *
+ * \return Slave configuration or \a NULL.
+ */
+ec_slave_config_t *ec_master_get_config(
+        const ec_master_t *master, /**< EtherCAT master. */
+        unsigned int pos /**< List position. */
+        )
+{
+    ec_slave_config_t *sc;
+    EC_FIND_CONFIG;
+}
+
+/** Get a slave configuration via its position in the list.
+ *
+ * Const version.
+ *
+ * \return Slave configuration or \a NULL.
+ */
+const ec_slave_config_t *ec_master_get_config_const(
+        const ec_master_t *master, /**< EtherCAT master. */
+        unsigned int pos /**< List position. */
+        )
+{
+    const ec_slave_config_t *sc;
+    EC_FIND_CONFIG;
+}
+
+/****************************************************************************/
+
+/** Get the number of domains.
+ *
+ * \return Number of domains.
+ */
+unsigned int ec_master_domain_count(
+        const ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    const ec_domain_t *domain;
+    unsigned int count = 0;
+
+    list_for_each_entry(domain, &master->domains, list) {
+        count++;
+    }
+
+    return count;
+}
+
+/****************************************************************************/
+
+/** Common implementation for ec_master_find_domain() and
+ * ec_master_find_domain_const().
+ */
+#define EC_FIND_DOMAIN \
+    do { \
+        list_for_each_entry(domain, &master->domains, list) { \
+            if (index--) \
+                continue; \
+            return domain; \
+        } \
+        \
+        return NULL; \
+    } while (0)
+
+/** Get a domain via its position in the list.
+ *
+ * \return Domain pointer, or \a NULL if not found.
+ */
+ec_domain_t *ec_master_find_domain(
+        ec_master_t *master, /**< EtherCAT master. */
+        unsigned int index /**< Domain index. */
+        )
+{
+    ec_domain_t *domain;
+    EC_FIND_DOMAIN;
+}
+
+/** Get a domain via its position in the list.
+ *
+ * Const version.
+ *
+ * \return Domain pointer, or \a NULL if not found.
+ */
+const ec_domain_t *ec_master_find_domain_const(
+        const ec_master_t *master, /**< EtherCAT master. */
+        unsigned int index /**< Domain index. */
+        )
+{
+    const ec_domain_t *domain;
+    EC_FIND_DOMAIN;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/** Get the number of EoE handlers.
+ *
+ * \return Number of EoE handlers.
+ */
+uint16_t ec_master_eoe_handler_count(
+        const ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    const ec_eoe_t *eoe;
+    unsigned int count = 0;
+
+    list_for_each_entry(eoe, &master->eoe_handlers, list) {
+        count++;
+    }
+
+    return count;
+}
+
+/****************************************************************************/
+
+/** Get an EoE handler via its position in the list.
+ *
+ * Const version.
+ *
+ * \return EoE handler pointer, or \a NULL if not found.
+ */
+const ec_eoe_t *ec_master_get_eoe_handler_const(
+        const ec_master_t *master, /**< EtherCAT master. */
+        uint16_t index /**< EoE handler index. */
+        )
+{
+    const ec_eoe_t *eoe;
+
+    list_for_each_entry(eoe, &master->eoe_handlers, list) {
+        if (index--)
+            continue;
+        return eoe;
+    }
+
+    return NULL;
+}
+
+#endif
+
+/****************************************************************************/
+
+/** Set the debug level.
+ *
+ * \retval       0 Success.
+ * \retval -EINVAL Invalid debug level.
+ */
+int ec_master_debug_level(
+        ec_master_t *master, /**< EtherCAT master. */
+        unsigned int level /**< Debug level. May be 0, 1 or 2. */
+        )
+{
+    if (level > 2) {
+        EC_MASTER_ERR(master, "Invalid debug level %u!\n", level);
+        return -EINVAL;
+    }
+
+    if (level != master->debug_level) {
+        master->debug_level = level;
+        EC_MASTER_INFO(master, "Master debug level set to %u.\n",
+                master->debug_level);
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Finds the DC reference clock.
+ */
+void ec_master_find_dc_ref_clock(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    ec_slave_t *slave, *ref = NULL;
+
+    if (master->dc_ref_config) {
+        // Use application-selected reference clock
+        slave = master->dc_ref_config->slave;
+
+        if (slave) {
+            if (slave->base_dc_supported && slave->has_dc_system_time) {
+                ref = slave;
+            }
+            else {
+                EC_MASTER_WARN(master, "Slave %u can not act as a"
+                        " DC reference clock!", slave->ring_position);
+            }
+        }
+        else {
+            EC_MASTER_WARN(master, "DC reference clock config (%u-%u)"
+                    " has no slave attached!\n", master->dc_ref_config->alias,
+                    master->dc_ref_config->position);
+        }
+    }
+    else {
+        // Use first slave with DC support as reference clock
+        for (slave = master->slaves;
+                slave < master->slaves + master->slave_count;
+                slave++) {
+            if (slave->base_dc_supported && slave->has_dc_system_time) {
+                ref = slave;
+                break;
+            }
+        }
+
+    }
+
+    master->dc_ref_clock = ref;
+
+    if (ref) {
+        EC_MASTER_INFO(master, "Using slave %u as DC reference clock.\n",
+                ref->ring_position);
+    }
+    else {
+        EC_MASTER_INFO(master, "No DC reference clock found.\n");
+    }
+
+    // These calls always succeed, because the
+    // datagrams have been pre-allocated.
+    ec_datagram_fpwr(&master->ref_sync_datagram,
+            ref ? ref->station_address : 0xffff, 0x0910, 4);
+    ec_datagram_frmw(&master->sync_datagram,
+            ref ? ref->station_address : 0xffff, 0x0910, 4);
+}
+
+/****************************************************************************/
+
+/** Calculates the bus topology; recursion function.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_master_calc_topology_rec(
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_slave_t *port0_slave, /**< Slave at port 0. */
+        unsigned int *slave_position /**< Slave position. */
+        )
+{
+    ec_slave_t *slave = master->slaves + *slave_position;
+    unsigned int port_index;
+    int ret;
+
+    static const unsigned int next_table[EC_MAX_PORTS] = {
+        3, 2, 0, 1
+    };
+
+    slave->ports[0].next_slave = port0_slave;
+
+    port_index = 3;
+    while (port_index != 0) {
+        if (!slave->ports[port_index].link.loop_closed) {
+            *slave_position = *slave_position + 1;
+            if (*slave_position < master->slave_count) {
+                slave->ports[port_index].next_slave =
+                    master->slaves + *slave_position;
+                ret = ec_master_calc_topology_rec(master,
+                        slave, slave_position);
+                if (ret) {
+                    return ret;
+                }
+            } else {
+                return -1;
+            }
+        }
+
+        port_index = next_table[port_index];
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Calculates the bus topology.
+ */
+void ec_master_calc_topology(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    unsigned int slave_position = 0;
+
+    if (master->slave_count == 0)
+        return;
+
+    if (ec_master_calc_topology_rec(master, NULL, &slave_position))
+        EC_MASTER_ERR(master, "Failed to calculate bus topology.\n");
+}
+
+/****************************************************************************/
+
+/** Calculates the bus transmission delays.
+ */
+void ec_master_calc_transmission_delays(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    ec_slave_t *slave;
+
+    for (slave = master->slaves;
+            slave < master->slaves + master->slave_count;
+            slave++) {
+        ec_slave_calc_port_delays(slave);
+    }
+
+    if (master->dc_ref_clock) {
+        uint32_t delay = 0;
+        ec_slave_calc_transmission_delays_rec(master->dc_ref_clock, &delay);
+    }
+}
+
+/****************************************************************************/
+
+/** Distributed-clocks calculations.
+ */
+void ec_master_calc_dc(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    // find DC reference clock
+    ec_master_find_dc_ref_clock(master);
+
+    // calculate bus topology
+    ec_master_calc_topology(master);
+
+    ec_master_calc_transmission_delays(master);
+}
+
+/****************************************************************************/
+
+/** Request OP state for configured slaves.
+ */
+void ec_master_request_op(
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    unsigned int i;
+    ec_slave_t *slave;
+
+    if (!master->active)
+        return;
+
+    EC_MASTER_DBG(master, 1, "Requesting OP...\n");
+
+    // request OP for all configured slaves
+    for (i = 0; i < master->slave_count; i++) {
+        slave = master->slaves + i;
+        if (slave->config) {
+            ec_slave_request_state(slave, EC_SLAVE_STATE_OP);
+        }
+    }
+
+    // always set DC reference clock to OP
+    if (master->dc_ref_clock) {
+        ec_slave_request_state(master->dc_ref_clock, EC_SLAVE_STATE_OP);
+    }
+}
+
+/*****************************************************************************
+ *  Application interface
+ ****************************************************************************/
+
+/** Same as ecrt_master_create_domain(), but with ERR_PTR() return value.
+ *
+ * \return New domain, or ERR_PTR() return value.
+ */
+ec_domain_t *ecrt_master_create_domain_err(
+        ec_master_t *master /**< master */
+        )
+{
+    ec_domain_t *domain, *last_domain;
+    unsigned int index;
+
+    EC_MASTER_DBG(master, 1, "ecrt_master_create_domain(master = 0x%p)\n",
+            master);
+
+    if (!(domain =
+                (ec_domain_t *) kmalloc(sizeof(ec_domain_t), GFP_KERNEL))) {
+        EC_MASTER_ERR(master, "Error allocating domain memory!\n");
+        return ERR_PTR(-ENOMEM);
+    }
+
+    down(&master->master_sem);
+
+    if (list_empty(&master->domains)) {
+        index = 0;
+    } else {
+        last_domain = list_entry(master->domains.prev, ec_domain_t, list);
+        index = last_domain->index + 1;
+    }
+
+    ec_domain_init(domain, master, index);
+    list_add_tail(&domain->list, &master->domains);
+
+    up(&master->master_sem);
+
+    EC_MASTER_DBG(master, 1, "Created domain %u.\n", domain->index);
+
+    return domain;
+}
+
+/****************************************************************************/
+
+ec_domain_t *ecrt_master_create_domain(
+        ec_master_t *master /**< master */
+        )
+{
+    ec_domain_t *d = ecrt_master_create_domain_err(master);
+    return IS_ERR(d) ? NULL : d;
+}
+
+/****************************************************************************/
+
+int ecrt_master_activate(ec_master_t *master)
+{
+    uint32_t domain_offset;
+    ec_domain_t *domain;
+    int ret;
+#ifdef EC_EOE
+    int eoe_was_running;
+#endif
+
+    EC_MASTER_DBG(master, 1, "ecrt_master_activate(master = 0x%p)\n", master);
+
+    if (master->active) {
+        EC_MASTER_WARN(master, "%s: Master already active!\n", __func__);
+        return 0;
+    }
+
+    down(&master->master_sem);
+
+    // finish all domains
+    domain_offset = 0;
+    list_for_each_entry(domain, &master->domains, list) {
+        ret = ec_domain_finish(domain, domain_offset);
+        if (ret < 0) {
+            up(&master->master_sem);
+            EC_MASTER_ERR(master, "Failed to finish domain 0x%p!\n", domain);
+            return ret;
+        }
+        domain_offset += domain->data_size;
+    }
+
+    up(&master->master_sem);
+
+    // restart EoE process and master thread with new locking
+
+    ec_master_thread_stop(master);
+#ifdef EC_EOE
+    eoe_was_running = master->eoe_thread != NULL;
+    ec_master_eoe_stop(master);
+#endif
+
+    EC_MASTER_DBG(master, 1, "FSM datagram is %p.\n", &master->fsm_datagram);
+
+    master->injection_seq_fsm = 0;
+    master->injection_seq_rt = 0;
+
+    master->send_cb = master->app_send_cb;
+    master->receive_cb = master->app_receive_cb;
+    master->cb_data = master->app_cb_data;
+
+#ifdef EC_EOE
+    if (eoe_was_running) {
+        ec_master_eoe_start(master);
+    }
+#endif
+    ret = ec_master_thread_start(master, ec_master_operation_thread,
+                "EtherCAT-OP");
+    if (ret < 0) {
+        EC_MASTER_ERR(master, "Failed to start master thread!\n");
+        return ret;
+    }
+
+    /* Allow scanning after a topology change. */
+    master->allow_scan = 1;
+
+    master->active = 1;
+
+    // notify state machine, that the configuration shall now be applied
+    master->config_changed = 1;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_deactivate(ec_master_t *master)
+{
+    ec_slave_t *slave;
+#ifdef EC_EOE
+    ec_eoe_t *eoe;
+    int eoe_was_running;
+#endif
+
+    EC_MASTER_DBG(master, 1, "%s(master = 0x%p)\n", __func__, master);
+
+    if (!master->active) {
+        EC_MASTER_WARN(master, "%s: Master not active.\n", __func__);
+        return -EINVAL;
+    }
+
+    ec_master_thread_stop(master);
+#ifdef EC_EOE
+    eoe_was_running = master->eoe_thread != NULL;
+    ec_master_eoe_stop(master);
+#endif
+
+    master->send_cb = ec_master_internal_send_cb;
+    master->receive_cb = ec_master_internal_receive_cb;
+    master->cb_data = master;
+
+    ec_master_clear_config(master);
+
+    for (slave = master->slaves;
+            slave < master->slaves + master->slave_count;
+            slave++) {
+
+        // set states for all slaves
+        ec_slave_request_state(slave, EC_SLAVE_STATE_PREOP);
+
+        // mark for reconfiguration, because the master could have no
+        // possibility for a reconfiguration between two sequential operation
+        // phases.
+        slave->force_config = 1;
+    }
+
+#ifdef EC_EOE
+    // ... but leave EoE slaves in OP
+    list_for_each_entry(eoe, &master->eoe_handlers, list) {
+        if (ec_eoe_is_open(eoe))
+            ec_slave_request_state(eoe->slave, EC_SLAVE_STATE_OP);
+    }
+#endif
+
+    master->app_time = 0ULL;
+    master->dc_ref_time = 0ULL;
+
+#ifdef EC_EOE
+    if (eoe_was_running) {
+        ec_master_eoe_start(master);
+    }
+#endif
+    if (ec_master_thread_start(master, ec_master_idle_thread,
+                "EtherCAT-IDLE")) {
+        EC_MASTER_WARN(master, "Failed to restart master thread!\n");
+    }
+
+    /* Disallow scanning to get into the same state like after a master
+     * request (after ec_master_enter_operation_phase() is called). */
+    master->allow_scan = 0;
+
+    master->active = 0;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_send(ec_master_t *master)
+{
+    ec_datagram_t *datagram, *n;
+    ec_device_index_t dev_idx;
+
+    if (master->injection_seq_rt != master->injection_seq_fsm) {
+        // inject datagram produced by master FSM
+        ec_master_queue_datagram(master, &master->fsm_datagram);
+
+        /*
+         * During warm-up, if the OP thread gets scheduled and runs quickly,
+         * it may observe injection_seq_rt updated before the datagram->state
+         * update from ec_master_queue_datagram(), breaking the FSM.
+         * smp_wmb() enforces the required ordering.
+         */
+        smp_wmb();
+
+        master->injection_seq_rt = master->injection_seq_fsm;
+    }
+
+    ec_master_inject_external_datagrams(master);
+
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        if (unlikely(!master->devices[dev_idx].link_state)) {
+            // link is down, no datagram can be sent
+            list_for_each_entry_safe(datagram, n,
+                    &master->datagram_queue, queue) {
+                if (datagram->device_index == dev_idx) {
+                    datagram->state = EC_DATAGRAM_ERROR;
+                    list_del_init(&datagram->queue);
+                }
+            }
+
+            if (!master->devices[dev_idx].dev) {
+                continue;
+            }
+
+            // query link state
+            ec_device_poll(&master->devices[dev_idx]);
+
+            // clear frame statistics
+            ec_device_clear_stats(&master->devices[dev_idx]);
+            continue;
+        }
+
+        // send frames
+        ec_master_send_datagrams(master, dev_idx);
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_receive(ec_master_t *master)
+{
+    unsigned int dev_idx;
+    ec_datagram_t *datagram, *next;
+
+    // receive datagrams
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        ec_device_poll(&master->devices[dev_idx]);
+    }
+    ec_master_update_device_stats(master);
+
+    // dequeue all datagrams that timed out
+    list_for_each_entry_safe(datagram, next, &master->datagram_queue, queue) {
+        if (datagram->state != EC_DATAGRAM_SENT) continue;
+
+#ifdef EC_HAVE_CYCLES
+        if (master->devices[EC_DEVICE_MAIN].cycles_poll -
+                datagram->cycles_sent > timeout_cycles) {
+#else
+        if (master->devices[EC_DEVICE_MAIN].jiffies_poll -
+                datagram->jiffies_sent > timeout_jiffies) {
+#endif
+            list_del_init(&datagram->queue);
+            datagram->state = EC_DATAGRAM_TIMED_OUT;
+            master->stats.timeouts++;
+
+#ifdef EC_RT_SYSLOG
+            ec_master_output_stats(master);
+
+            if (unlikely(master->debug_level > 0)) {
+                unsigned int time_us;
+#ifdef EC_HAVE_CYCLES
+                time_us = (unsigned int)
+                    (master->devices[EC_DEVICE_MAIN].cycles_poll -
+                        datagram->cycles_sent) * 1000 / cpu_khz;
+#else
+                time_us = (unsigned int)
+                    ((master->devices[EC_DEVICE_MAIN].jiffies_poll -
+                            datagram->jiffies_sent) * 1000000 / HZ);
+#endif
+                EC_MASTER_DBG(master, 0, "TIMED OUT datagram %p,"
+                        " index %02X waited %u us.\n",
+                        datagram, datagram->index, time_us);
+            }
+#endif /* RT_SYSLOG */
+        }
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_send_ext(ec_master_t *master)
+{
+    ec_datagram_t *datagram, *next;
+
+    if (down_trylock(&master->ext_queue_sem))
+        return -EAGAIN;
+
+    list_for_each_entry_safe(datagram, next, &master->ext_datagram_queue,
+            ext_queue) {
+        list_del_init(&datagram->ext_queue);
+        ec_master_queue_datagram(master, datagram);
+    }
+    up(&master->ext_queue_sem);
+
+    return ecrt_master_send(master);
+}
+
+/****************************************************************************/
+
+/** Same as ecrt_master_slave_config(), but with ERR_PTR() return value.
+ */
+ec_slave_config_t *ecrt_master_slave_config_err(ec_master_t *master,
+        uint16_t alias, uint16_t position, uint32_t vendor_id,
+        uint32_t product_code)
+{
+    ec_slave_config_t *sc;
+    unsigned int found = 0;
+
+
+    EC_MASTER_DBG(master, 1, "ecrt_master_slave_config(master = 0x%p,"
+            " alias = %u, position = %u, vendor_id = 0x%08x,"
+            " product_code = 0x%08x)\n",
+            master, alias, position, vendor_id, product_code);
+
+    list_for_each_entry(sc, &master->configs, list) {
+        if (sc->alias == alias && sc->position == position) {
+            found = 1;
+            break;
+        }
+    }
+
+    if (found) { // config with same alias/position already existing
+        if (sc->vendor_id != vendor_id || sc->product_code != product_code) {
+            EC_MASTER_ERR(master, "Slave type mismatch. Slave was"
+                    " configured as 0x%08X/0x%08X before. Now configuring"
+                    " with 0x%08X/0x%08X.\n", sc->vendor_id, sc->product_code,
+                    vendor_id, product_code);
+            return ERR_PTR(-ENOENT);
+        }
+    } else {
+        EC_MASTER_DBG(master, 1, "Creating slave configuration for %u:%u,"
+                " 0x%08X/0x%08X.\n",
+                alias, position, vendor_id, product_code);
+
+        if (!(sc = (ec_slave_config_t *) kmalloc(sizeof(ec_slave_config_t),
+                        GFP_KERNEL))) {
+            EC_MASTER_ERR(master, "Failed to allocate memory"
+                    " for slave configuration.\n");
+            return ERR_PTR(-ENOMEM);
+        }
+
+        ec_slave_config_init(sc, master,
+                alias, position, vendor_id, product_code);
+
+        down(&master->master_sem);
+
+        // try to find the addressed slave
+        ec_slave_config_attach(sc);
+        ec_slave_config_load_default_sync_config(sc);
+        list_add_tail(&sc->list, &master->configs);
+
+        up(&master->master_sem);
+    }
+
+    return sc;
+}
+
+/****************************************************************************/
+
+ec_slave_config_t *ecrt_master_slave_config(ec_master_t *master,
+        uint16_t alias, uint16_t position, uint32_t vendor_id,
+        uint32_t product_code)
+{
+    ec_slave_config_t *sc = ecrt_master_slave_config_err(master, alias,
+            position, vendor_id, product_code);
+    return IS_ERR(sc) ? NULL : sc;
+}
+
+/****************************************************************************/
+
+int ecrt_master_select_reference_clock(ec_master_t *master,
+        ec_slave_config_t *sc)
+{
+    if (sc) {
+        ec_slave_t *slave = sc->slave;
+
+        // output an early warning
+        if (slave &&
+                (!slave->base_dc_supported || !slave->has_dc_system_time)) {
+            EC_MASTER_WARN(master, "Slave %u can not act as"
+                    " a reference clock!", slave->ring_position);
+        }
+    }
+
+    master->dc_ref_config = sc;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master(ec_master_t *master, ec_master_info_t *master_info)
+{
+    EC_MASTER_DBG(master, 1, "ecrt_master(master = 0x%p,"
+            " master_info = 0x%p)\n", master, master_info);
+
+    master_info->slave_count = master->slave_count;
+    master_info->link_up = master->devices[EC_DEVICE_MAIN].link_state;
+    master_info->scan_busy = master->scan_busy;
+    master_info->app_time = master->app_time;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_scan_progress(ec_master_t *master,
+        ec_master_scan_progress_t *progress)
+{
+    EC_MASTER_DBG(master, 1, "ecrt_master_scan_progress(master = 0x%p,"
+            " progress = 0x%p)\n", master, progress);
+
+    progress->slave_count = master->slave_count;
+    progress->scan_index = master->scan_index;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_get_slave(ec_master_t *master, uint16_t slave_position,
+        ec_slave_info_t *slave_info)
+{
+    const ec_slave_t *slave;
+    unsigned int i;
+    int ret = 0;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    slave = ec_master_find_slave_const(master, 0, slave_position);
+
+    if (slave == NULL) {
+       ret = -ENOENT;
+       goto out_get_slave;
+    }
+
+    slave_info->position = slave->ring_position;
+    slave_info->vendor_id = slave->sii.vendor_id;
+    slave_info->product_code = slave->sii.product_code;
+    slave_info->revision_number = slave->sii.revision_number;
+    slave_info->serial_number = slave->sii.serial_number;
+    slave_info->alias = slave->effective_alias;
+    slave_info->current_on_ebus = slave->sii.current_on_ebus;
+
+    for (i = 0; i < EC_MAX_PORTS; i++) {
+        slave_info->ports[i].desc = slave->ports[i].desc;
+        slave_info->ports[i].link.link_up = slave->ports[i].link.link_up;
+        slave_info->ports[i].link.loop_closed =
+            slave->ports[i].link.loop_closed;
+        slave_info->ports[i].link.signal_detected =
+            slave->ports[i].link.signal_detected;
+        slave_info->ports[i].receive_time = slave->ports[i].receive_time;
+        if (slave->ports[i].next_slave) {
+            slave_info->ports[i].next_slave =
+                slave->ports[i].next_slave->ring_position;
+        } else {
+            slave_info->ports[i].next_slave = 0xffff;
+        }
+        slave_info->ports[i].delay_to_next_dc =
+            slave->ports[i].delay_to_next_dc;
+    }
+
+    slave_info->al_state = slave->current_state;
+    slave_info->error_flag = slave->error_flag;
+    slave_info->sync_count = slave->sii.sync_count;
+    slave_info->sdo_count = ec_slave_sdo_count(slave);
+    if (slave->sii.name) {
+        strncpy(slave_info->name, slave->sii.name, EC_MAX_STRING_LENGTH);
+    } else {
+        slave_info->name[0] = 0;
+    }
+
+out_get_slave:
+    up(&master->master_sem);
+
+    return ret;
+}
+
+/****************************************************************************/
+
+void ecrt_master_callbacks(ec_master_t *master,
+        void (*send_cb)(void *), void (*receive_cb)(void *), void *cb_data)
+{
+    EC_MASTER_DBG(master, 1, "ecrt_master_callbacks(master = 0x%p,"
+            " send_cb = 0x%p, receive_cb = 0x%p, cb_data = 0x%p)\n",
+            master, send_cb, receive_cb, cb_data);
+
+    master->app_send_cb = send_cb;
+    master->app_receive_cb = receive_cb;
+    master->app_cb_data = cb_data;
+}
+
+/****************************************************************************/
+
+int ecrt_master_state(const ec_master_t *master, ec_master_state_t *state)
+{
+    ec_device_index_t dev_idx;
+
+    state->slaves_responding = 0U;
+    state->al_states = 0;
+    state->link_up = 0U;
+
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        /* Announce sum of responding slaves on all links. */
+        state->slaves_responding += master->fsm.slaves_responding[dev_idx];
+
+        /* Binary-or slave states of all links. */
+        state->al_states |= master->fsm.slave_states[dev_idx];
+
+        /* Signal link up if at least one device has link. */
+        state->link_up |= master->devices[dev_idx].link_state;
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_link_state(const ec_master_t *master, unsigned int dev_idx,
+        ec_master_link_state_t *state)
+{
+    if (dev_idx >= ec_master_num_devices(master)) {
+        return -EINVAL;
+    }
+
+    state->slaves_responding = master->fsm.slaves_responding[dev_idx];
+    state->al_states = master->fsm.slave_states[dev_idx];
+    state->link_up = master->devices[dev_idx].link_state;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_application_time(ec_master_t *master, uint64_t app_time)
+{
+    master->app_time = app_time;
+
+    if (unlikely(!master->dc_ref_time)) {
+        master->dc_ref_time = app_time;
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_reference_clock_time(const ec_master_t *master,
+        uint32_t *time)
+{
+    if (!master->dc_ref_clock) {
+        return -ENXIO;
+    }
+
+    if (master->sync_datagram.state != EC_DATAGRAM_RECEIVED) {
+        return -EIO;
+    }
+
+    // Get returned datagram time, transmission delay removed.
+    *time = EC_READ_U32(master->sync_datagram.data) -
+        master->dc_ref_clock->transmission_delay;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_sync_reference_clock(ec_master_t *master)
+{
+    if (master->dc_ref_clock) {
+        EC_WRITE_U32(master->ref_sync_datagram.data, master->app_time);
+        ec_master_queue_datagram(master, &master->ref_sync_datagram);
+    } else {
+        return -ENXIO;
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_sync_reference_clock_to(
+        ec_master_t *master,
+        uint64_t sync_time
+        )
+{
+    if (master->dc_ref_clock) {
+        EC_WRITE_U32(master->ref_sync_datagram.data, sync_time);
+        ec_master_queue_datagram(master, &master->ref_sync_datagram);
+    } else {
+        return -ENXIO;
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_sync_slave_clocks(ec_master_t *master)
+{
+    if (master->dc_ref_clock) {
+        ec_datagram_zero(&master->sync_datagram);
+        ec_master_queue_datagram(master, &master->sync_datagram);
+    } else {
+        return -ENXIO;
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_master_sync_monitor_queue(ec_master_t *master)
+{
+    ec_datagram_zero(&master->sync_mon_datagram);
+    ec_master_queue_datagram(master, &master->sync_mon_datagram);
+    return 0;
+}
+
+/****************************************************************************/
+
+uint32_t ecrt_master_sync_monitor_process(const ec_master_t *master)
+{
+    if (master->sync_mon_datagram.state == EC_DATAGRAM_RECEIVED) {
+        return EC_READ_U32(master->sync_mon_datagram.data) & 0x7fffffff;
+    } else {
+        return 0xffffffff;
+    }
+}
+
+/****************************************************************************/
+
+int ecrt_master_sdo_download(ec_master_t *master, uint16_t slave_position,
+        uint16_t index, uint8_t subindex, const uint8_t *data,
+        size_t data_size, uint32_t *abort_code)
+{
+    ec_sdo_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    EC_MASTER_DBG(master, 1, "%s(master = 0x%p,"
+            " slave_position = %u, index = 0x%04X, subindex = 0x%02X,"
+            " data = 0x%p, data_size = %zu, abort_code = 0x%p)\n",
+            __func__, master, slave_position, index, subindex,
+            data, data_size, abort_code);
+
+    ec_sdo_request_init(&request);
+    ecrt_sdo_request_index(&request, index, subindex);
+    ret = ec_sdo_request_alloc(&request, data_size);
+    if (ret) {
+        ec_sdo_request_clear(&request);
+        return ret;
+    }
+
+    memcpy(request.data, data, data_size);
+    request.data_size = data_size;
+    ecrt_sdo_request_write(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_sdo_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n", slave_position);
+        ec_sdo_request_clear(&request);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling SDO download request.\n");
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->sdo_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_sdo_request_clear(&request);
+            return -EINTR;
+        }
+        // request already processing: interrupt not possible.
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    *abort_code = request.abort_code;
+
+    if (request.state == EC_INT_REQUEST_SUCCESS) {
+        ret = 0;
+    } else if (request.errno) {
+        ret = -request.errno;
+    } else {
+        ret = -EIO;
+    }
+
+    ec_sdo_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+int ecrt_master_sdo_download_complete(ec_master_t *master,
+        uint16_t slave_position, uint16_t index, const uint8_t *data,
+        size_t data_size, uint32_t *abort_code)
+{
+    ec_sdo_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    EC_MASTER_DBG(master, 1, "%s(master = 0x%p,"
+            " slave_position = %u, index = 0x%04X,"
+            " data = 0x%p, data_size = %zu, abort_code = 0x%p)\n",
+            __func__, master, slave_position, index, data, data_size,
+            abort_code);
+
+    ec_sdo_request_init(&request);
+    ecrt_sdo_request_index(&request, index, 0);
+    ret = ec_sdo_request_alloc(&request, data_size);
+    if (ret) {
+        ec_sdo_request_clear(&request);
+        return ret;
+    }
+
+    request.complete_access = 1;
+    memcpy(request.data, data, data_size);
+    request.data_size = data_size;
+    ecrt_sdo_request_write(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_sdo_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n", slave_position);
+        ec_sdo_request_clear(&request);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling SDO download request"
+            " (complete access).\n");
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->sdo_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_sdo_request_clear(&request);
+            return -EINTR;
+        }
+        // request already processing: interrupt not possible.
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    *abort_code = request.abort_code;
+
+    if (request.state == EC_INT_REQUEST_SUCCESS) {
+        ret = 0;
+    } else if (request.errno) {
+        ret = -request.errno;
+    } else {
+        ret = -EIO;
+    }
+
+    ec_sdo_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+int ecrt_master_sdo_upload(ec_master_t *master, uint16_t slave_position,
+        uint16_t index, uint8_t subindex, uint8_t *target,
+        size_t target_size, size_t *result_size, uint32_t *abort_code)
+{
+    ec_sdo_request_t request;
+    ec_slave_t *slave;
+    int ret = 0;
+
+    EC_MASTER_DBG(master, 1, "%s(master = 0x%p,"
+            " slave_position = %u, index = 0x%04X, subindex = 0x%02X,"
+            " target = 0x%p, target_size = %zu, result_size = 0x%p,"
+            " abort_code = 0x%p)\n",
+            __func__, master, slave_position, index, subindex,
+            target, target_size, result_size, abort_code);
+
+    ec_sdo_request_init(&request);
+    ecrt_sdo_request_index(&request, index, subindex);
+    ecrt_sdo_request_read(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_sdo_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, slave_position))) {
+        up(&master->master_sem);
+        ec_sdo_request_clear(&request);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n", slave_position);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling SDO upload request.\n");
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->sdo_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_sdo_request_clear(&request);
+            return -EINTR;
+        }
+        // request already processing: interrupt not possible.
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    *abort_code = request.abort_code;
+
+    if (request.state != EC_INT_REQUEST_SUCCESS) {
+        *result_size = 0;
+        if (request.errno) {
+            ret = -request.errno;
+        } else {
+            ret = -EIO;
+        }
+    } else {
+        if (request.data_size > target_size) {
+            EC_SLAVE_ERR(slave, "%s(): Buffer too small.\n", __func__);
+            ret = -ENOBUFS;
+        }
+        else {
+            memcpy(target, request.data, request.data_size);
+            *result_size = request.data_size;
+            ret = 0;
+        }
+    }
+
+    ec_sdo_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+int ecrt_master_write_idn(ec_master_t *master, uint16_t slave_position,
+        uint8_t drive_no, uint16_t idn, const uint8_t *data, size_t data_size,
+        uint16_t *error_code)
+{
+    ec_soe_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    if (drive_no > 7) {
+        EC_MASTER_ERR(master, "Invalid drive number!\n");
+        return -EINVAL;
+    }
+
+    ec_soe_request_init(&request);
+    ec_soe_request_set_drive_no(&request, drive_no);
+    ec_soe_request_set_idn(&request, idn);
+
+    ret = ec_soe_request_alloc(&request, data_size);
+    if (ret) {
+        ec_soe_request_clear(&request);
+        return ret;
+    }
+
+    memcpy(request.data, data, data_size);
+    request.data_size = data_size;
+    ec_soe_request_write(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_soe_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                slave_position);
+        ec_soe_request_clear(&request);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling SoE write request.\n");
+
+    // schedule SoE write request.
+    list_add_tail(&request.list, &slave->soe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_soe_request_clear(&request);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    if (error_code) {
+        *error_code = request.error_code;
+    }
+    ret = request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+    ec_soe_request_clear(&request);
+
+    return ret;
+}
+
+/****************************************************************************/
+
+int ecrt_master_read_idn(ec_master_t *master, uint16_t slave_position,
+        uint8_t drive_no, uint16_t idn, uint8_t *target, size_t target_size,
+        size_t *result_size, uint16_t *error_code)
+{
+    ec_soe_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    if (drive_no > 7) {
+        EC_MASTER_ERR(master, "Invalid drive number!\n");
+        return -EINVAL;
+    }
+
+    ec_soe_request_init(&request);
+    ec_soe_request_set_drive_no(&request, drive_no);
+    ec_soe_request_set_idn(&request, idn);
+    ec_soe_request_read(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_soe_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, slave_position))) {
+        up(&master->master_sem);
+        ec_soe_request_clear(&request);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n", slave_position);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling SoE read request.\n");
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->soe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_soe_request_clear(&request);
+            return -EINTR;
+        }
+        // request already processing: interrupt not possible.
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    if (error_code) {
+        *error_code = request.error_code;
+    }
+
+    if (request.state != EC_INT_REQUEST_SUCCESS) {
+        if (result_size) {
+            *result_size = 0;
+        }
+        ret = -EIO;
+    } else { // success
+        if (request.data_size > target_size) {
+            EC_SLAVE_ERR(slave, "%s(): Buffer too small.\n", __func__);
+            ret = -ENOBUFS;
+        }
+        else { // data fits in buffer
+            if (result_size) {
+                *result_size = request.data_size;
+            }
+            memcpy(target, request.data, request.data_size);
+            ret = 0;
+        }
+    }
+
+    ec_soe_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+int ecrt_master_reset(ec_master_t *master)
+{
+    ec_slave_config_t *sc;
+
+    list_for_each_entry(sc, &master->configs, list) {
+        if (sc->slave) {
+            ec_slave_request_state(sc->slave, EC_SLAVE_STATE_OP);
+        }
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+static void sc_reset_task_kicker(struct irq_work *work)
+{
+    struct ec_master *master =
+        container_of(work, struct ec_master, sc_reset_work_kicker);
+    schedule_work(&master->sc_reset_work);
+}
+
+/****************************************************************************/
+
+static void sc_reset_task(struct work_struct *work)
+{
+    struct ec_master *master =
+        container_of(work, struct ec_master, sc_reset_work);
+
+    down(&master->master_sem);
+    ecrt_master_reset(master);
+    up(&master->master_sem);
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecrt_master_create_domain);
+EXPORT_SYMBOL(ecrt_master_activate);
+EXPORT_SYMBOL(ecrt_master_deactivate);
+EXPORT_SYMBOL(ecrt_master_send);
+EXPORT_SYMBOL(ecrt_master_send_ext);
+EXPORT_SYMBOL(ecrt_master_receive);
+EXPORT_SYMBOL(ecrt_master_callbacks);
+EXPORT_SYMBOL(ecrt_master);
+EXPORT_SYMBOL(ecrt_master_scan_progress);
+EXPORT_SYMBOL(ecrt_master_get_slave);
+EXPORT_SYMBOL(ecrt_master_slave_config);
+EXPORT_SYMBOL(ecrt_master_select_reference_clock);
+EXPORT_SYMBOL(ecrt_master_state);
+EXPORT_SYMBOL(ecrt_master_link_state);
+EXPORT_SYMBOL(ecrt_master_application_time);
+EXPORT_SYMBOL(ecrt_master_sync_reference_clock);
+EXPORT_SYMBOL(ecrt_master_sync_reference_clock_to);
+EXPORT_SYMBOL(ecrt_master_sync_slave_clocks);
+EXPORT_SYMBOL(ecrt_master_reference_clock_time);
+EXPORT_SYMBOL(ecrt_master_sync_monitor_queue);
+EXPORT_SYMBOL(ecrt_master_sync_monitor_process);
+EXPORT_SYMBOL(ecrt_master_sdo_download);
+EXPORT_SYMBOL(ecrt_master_sdo_download_complete);
+EXPORT_SYMBOL(ecrt_master_sdo_upload);
+EXPORT_SYMBOL(ecrt_master_write_idn);
+EXPORT_SYMBOL(ecrt_master_read_idn);
+EXPORT_SYMBOL(ecrt_master_reset);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/master.h b/drivers/net/ethercat/master/master.h
new file mode 100644
index 000000000000..a1125e85a360
--- /dev/null
+++ b/drivers/net/ethercat/master/master.h
@@ -0,0 +1,386 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT master structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_MASTER_H__
+#define __EC_MASTER_H__
+
+#include <linux/version.h>
+#include <linux/irq_work.h>
+#include <linux/list.h>
+#include <linux/timer.h>
+#include <linux/wait.h>
+#include <linux/kthread.h>
+#include <linux/rtmutex.h>
+#include <linux/workqueue.h>
+
+#include "device.h"
+#include "domain.h"
+#include "ethernet.h"
+#include "fsm_master.h"
+#include "cdev.h"
+
+#ifdef EC_RTDM
+#include "rtdm.h"
+#endif
+
+/****************************************************************************/
+
+/** Convenience macro for printing master-specific information to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT <INDEX>: ",
+ * where INDEX is the master index.
+ *
+ * \param master EtherCAT master
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_MASTER_INFO(master, fmt, args...) \
+    printk(KERN_INFO "EtherCAT %u: " fmt, master->index, ##args)
+
+/** Convenience macro for printing master-specific errors to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT <INDEX>: ",
+ * where INDEX is the master index.
+ *
+ * \param master EtherCAT master
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_MASTER_ERR(master, fmt, args...) \
+    printk(KERN_ERR "EtherCAT ERROR %u: " fmt, master->index, ##args)
+
+/** Convenience macro for printing master-specific warnings to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT <INDEX>: ",
+ * where INDEX is the master index.
+ *
+ * \param master EtherCAT master
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_MASTER_WARN(master, fmt, args...) \
+    printk(KERN_WARNING "EtherCAT WARNING %u: " fmt, master->index, ##args)
+
+/** Convenience macro for printing master-specific debug messages to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed "EtherCAT <INDEX>: ",
+ * where INDEX is the master index.
+ *
+ * \param master EtherCAT master
+ * \param level Debug level. Master's debug level must be >= \a level for
+ * output.
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_MASTER_DBG(master, level, fmt, args...) \
+    do { \
+        if (master->debug_level >= level) { \
+            printk(KERN_DEBUG "EtherCAT DEBUG %u: " fmt, \
+                    master->index, ##args); \
+        } \
+    } while (0)
+
+
+/** Size of the external datagram ring.
+ *
+ * The external datagram ring is used for slave FSMs.
+ */
+#define EC_EXT_RING_SIZE 32
+
+/** Maximum number of masters.
+ */
+#define EC_MAX_MASTERS 32
+
+/****************************************************************************/
+
+/** EtherCAT master phase.
+ */
+typedef enum {
+    EC_ORPHANED, /**< Orphaned phase. The master has no Ethernet device
+                   attached. */
+    EC_IDLE, /**< Idle phase. An Ethernet device is attached, but the master
+               is not in use, yet. */
+    EC_OPERATION /**< Operation phase. The master was requested by a realtime
+                   application. */
+} ec_master_phase_t;
+
+/****************************************************************************/
+
+/** Cyclic statistics.
+ */
+typedef struct {
+    unsigned int timeouts; /**< datagram timeouts */
+    unsigned int corrupted; /**< corrupted frames */
+    unsigned int unmatched; /**< unmatched datagrams (received, but not
+                               queued any longer) */
+    unsigned long output_jiffies; /**< time of last output */
+} ec_stats_t;
+
+/****************************************************************************/
+
+/** Device statistics.
+ */
+typedef struct {
+    u64 tx_count; /**< Number of frames sent. */
+    u64 last_tx_count; /**< Number of frames sent of last statistics cycle. */
+    u64 rx_count; /**< Number of frames received. */
+    u64 last_rx_count; /**< Number of frames received of last statistics
+                         cycle. */
+    u64 tx_bytes; /**< Number of bytes sent. */
+    u64 last_tx_bytes; /**< Number of bytes sent of last statistics cycle. */
+    u64 rx_bytes; /**< Number of bytes received. */
+    u64 last_rx_bytes; /**< Number of bytes received of last statistics cycle.
+                        */
+    u64 last_loss; /**< Tx/Rx difference of last statistics cycle. */
+    s32 tx_frame_rates[EC_RATE_COUNT]; /**< Transmit rates in frames/s for
+                                         different statistics cycle periods.
+                                        */
+    s32 rx_frame_rates[EC_RATE_COUNT]; /**< Receive rates in frames/s for
+                                         different statistics cycle periods.
+                                        */
+    s32 tx_byte_rates[EC_RATE_COUNT]; /**< Transmit rates in byte/s for
+                                        different statistics cycle periods. */
+    s32 rx_byte_rates[EC_RATE_COUNT]; /**< Receive rates in byte/s for
+                                        different statistics cycle periods. */
+    s32 loss_rates[EC_RATE_COUNT]; /**< Frame loss rates for different
+                                     statistics cycle periods. */
+    unsigned long jiffies; /**< Jiffies of last statistic cycle. */
+} ec_device_stats_t;
+
+/****************************************************************************/
+
+#if EC_MAX_NUM_DEVICES < 1
+#error Invalid number of devices
+#endif
+
+/****************************************************************************/
+
+/** EtherCAT master.
+ *
+ * Manages slaves, domains and IO.
+ */
+struct ec_master {
+    unsigned int index; /**< Index. */
+    unsigned int reserved; /**< \a True, if the master is in use. */
+
+    ec_cdev_t cdev; /**< Master character device. */
+    struct device *class_device; /**< Master class device. */
+
+#ifdef EC_RTDM
+    ec_rtdm_dev_t rtdm_dev; /**< RTDM device. */
+#endif
+
+    struct semaphore master_sem; /**< Master semaphore. */
+
+    ec_device_t devices[EC_MAX_NUM_DEVICES]; /**< EtherCAT devices. */
+    const uint8_t *macs[EC_MAX_NUM_DEVICES]; /**< Device MAC addresses. */
+#if EC_MAX_NUM_DEVICES > 1
+    unsigned int num_devices; /**< Number of devices. Access this always via
+                                ec_master_num_devices(), because it may be
+                                optimized! */
+#endif
+    struct semaphore device_sem; /**< Device semaphore. */
+    ec_device_stats_t device_stats; /**< Device statistics. */
+
+    ec_fsm_master_t fsm; /**< Master state machine. */
+    ec_datagram_t fsm_datagram; /**< Datagram used for state machines. */
+    ec_master_phase_t phase; /**< Master phase. */
+    unsigned int active; /**< Master has been activated. */
+    unsigned int config_changed; /**< The configuration changed. */
+    unsigned int injection_seq_fsm; /**< Datagram injection sequence number
+                                      for the FSM side. */
+    unsigned int injection_seq_rt; /**< Datagram injection sequence number
+                                     for the realtime side. */
+
+    ec_slave_t *slaves; /**< Array of slaves on the bus. */
+    unsigned int slave_count; /**< Number of slaves on the bus. */
+
+    /* Configuration applied by the application. */
+    struct list_head configs; /**< List of slave configurations. */
+    struct list_head domains; /**< List of domains. */
+
+    u64 app_time; /**< Time of the last ecrt_master_sync() call. */
+    u64 dc_ref_time; /**< Common reference timestamp for DC start times. */
+    ec_datagram_t ref_sync_datagram; /**< Datagram used for synchronizing the
+                                       reference clock to the master clock. */
+    ec_datagram_t sync_datagram; /**< Datagram used for DC drift
+                                   compensation. */
+    ec_datagram_t sync_mon_datagram; /**< Datagram used for DC synchronisation
+                                       monitoring. */
+    ec_slave_config_t *dc_ref_config; /**< Application-selected DC reference
+                                        clock slave config. */
+    ec_slave_t *dc_ref_clock; /**< DC reference clock slave. */
+
+    unsigned int scan_busy; /**< Current scan state. */
+    unsigned int scan_index; /**< Index of slave currently scanned. */
+    unsigned int allow_scan; /**< \a True, if slave scanning is allowed. */
+    struct semaphore scan_sem; /**< Semaphore protecting the \a scan_busy
+                                 variable and the \a allow_scan flag. */
+    wait_queue_head_t scan_queue; /**< Queue for processes that wait for
+                                    slave scanning. */
+
+    unsigned int config_busy; /**< State of slave configuration. */
+    struct semaphore config_sem; /**< Semaphore protecting the \a config_busy
+                                   variable and the allow_config flag. */
+    wait_queue_head_t config_queue; /**< Queue for processes that wait for
+                                      slave configuration. */
+
+    struct list_head datagram_queue; /**< Datagram queue. */
+    uint8_t datagram_index; /**< Current datagram index. */
+
+    struct list_head ext_datagram_queue; /**< Queue for non-application
+                                           datagrams. */
+    struct semaphore ext_queue_sem; /**< Semaphore protecting the \a
+                                      ext_datagram_queue. */
+
+    ec_datagram_t ext_datagram_ring[EC_EXT_RING_SIZE]; /**< External datagram
+                                                         ring. */
+    unsigned int ext_ring_idx_rt; /**< Index in external datagram ring for RT
+                                    side. */
+    unsigned int ext_ring_idx_fsm; /**< Index in external datagram ring for
+                                     FSM side. */
+    unsigned int send_interval; /**< Interval between two calls to
+                                  ecrt_master_send(). */
+    size_t max_queue_size; /**< Maximum size of datagram queue */
+
+    ec_slave_t *fsm_slave; /**< Slave that is queried next for FSM exec. */
+    struct list_head fsm_exec_list; /**< Slave FSM execution list. */
+    unsigned int fsm_exec_count; /**< Number of entries in execution list. */
+
+    unsigned int debug_level; /**< Master debug level. */
+    unsigned int run_on_cpu;  /**< bind kernel threads to this cpu */
+    ec_stats_t stats; /**< Cyclic statistics. */
+
+    struct task_struct *thread; /**< Master thread. */
+
+#ifdef EC_EOE
+    struct task_struct *eoe_thread; /**< EoE thread. */
+    struct list_head eoe_handlers; /**< Ethernet over EtherCAT handlers. */
+#endif
+
+    struct rt_mutex io_mutex;  /**< Mutex used in \a IDLE and \a OP phase. */
+
+    void (*send_cb)(void *); /**< Current send datagrams callback. */
+    void (*receive_cb)(void *); /**< Current receive datagrams callback. */
+    void *cb_data; /**< Current callback data. */
+    void (*app_send_cb)(void *); /**< Application's send datagrams
+                                          callback. */
+    void (*app_receive_cb)(void *); /**< Application's receive datagrams
+                                      callback. */
+    void *app_cb_data; /**< Application callback data. */
+
+    struct list_head sii_requests; /**< SII write requests. */
+    struct list_head emerg_reg_requests; /**< Emergency register access
+                                           requests. */
+
+    wait_queue_head_t request_queue; /**< Wait queue for external requests
+                                       from user space. */
+    struct work_struct sc_reset_work; /**< Task to reset slave configuration. */
+    struct irq_work sc_reset_work_kicker; /**< NMI-Safe kicker to trigger
+                                            reset task above. */
+};
+
+/****************************************************************************/
+
+// static funtions
+void ec_master_init_static(void);
+
+// master creation/deletion
+int ec_master_init(ec_master_t *, unsigned int, const uint8_t *,
+        const uint8_t *, dev_t, struct class *, unsigned int, unsigned int);
+void ec_master_clear(ec_master_t *);
+
+/** Number of Ethernet devices.
+ */
+#if EC_MAX_NUM_DEVICES > 1
+#define ec_master_num_devices(MASTER) ((MASTER)->num_devices)
+#else
+#define ec_master_num_devices(MASTER) 1
+#endif
+
+// phase transitions
+int ec_master_enter_idle_phase(ec_master_t *);
+void ec_master_leave_idle_phase(ec_master_t *);
+int ec_master_enter_operation_phase(ec_master_t *);
+void ec_master_leave_operation_phase(ec_master_t *);
+
+#ifdef EC_EOE
+// EoE
+void ec_master_eoe_start(ec_master_t *);
+void ec_master_eoe_stop(ec_master_t *);
+#endif
+
+// datagram IO
+void ec_master_receive_datagrams(ec_master_t *, ec_device_t *,
+        const uint8_t *, size_t);
+void ec_master_queue_datagram(ec_master_t *, ec_datagram_t *);
+void ec_master_queue_datagram_ext(ec_master_t *, ec_datagram_t *);
+
+// misc.
+void ec_master_set_send_interval(ec_master_t *, unsigned int);
+void ec_master_attach_slave_configs(ec_master_t *);
+ec_slave_t *ec_master_find_slave(ec_master_t *, uint16_t, uint16_t);
+const ec_slave_t *ec_master_find_slave_const(const ec_master_t *, uint16_t,
+        uint16_t);
+void ec_master_output_stats(ec_master_t *);
+#ifdef EC_EOE
+void ec_master_clear_eoe_handlers(ec_master_t *);
+#endif
+void ec_master_clear_slaves(ec_master_t *);
+
+unsigned int ec_master_config_count(const ec_master_t *);
+ec_slave_config_t *ec_master_get_config(
+        const ec_master_t *, unsigned int);
+const ec_slave_config_t *ec_master_get_config_const(
+        const ec_master_t *, unsigned int);
+unsigned int ec_master_domain_count(const ec_master_t *);
+ec_domain_t *ec_master_find_domain(ec_master_t *, unsigned int);
+const ec_domain_t *ec_master_find_domain_const(const ec_master_t *,
+        unsigned int);
+#ifdef EC_EOE
+uint16_t ec_master_eoe_handler_count(const ec_master_t *);
+const ec_eoe_t *ec_master_get_eoe_handler_const(const ec_master_t *, uint16_t);
+#endif
+
+int ec_master_debug_level(ec_master_t *, unsigned int);
+
+ec_domain_t *ecrt_master_create_domain_err(ec_master_t *);
+ec_slave_config_t *ecrt_master_slave_config_err(ec_master_t *, uint16_t,
+        uint16_t, uint32_t, uint32_t);
+
+void ec_master_calc_dc(ec_master_t *);
+void ec_master_request_op(ec_master_t *);
+
+void ec_master_internal_send_cb(void *);
+void ec_master_internal_receive_cb(void *);
+
+extern const unsigned int rate_intervals[EC_RATE_COUNT]; // see master.c
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/module.c b/drivers/net/ethercat/master/module.c
new file mode 100644
index 000000000000..b8156a1d3d52
--- /dev/null
+++ b/drivers/net/ethercat/master/module.c
@@ -0,0 +1,681 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT master driver module.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/err.h>
+
+#include "globals.h"
+#include "master.h"
+#include "device.h"
+
+/****************************************************************************/
+
+/****************************************************************************/
+
+int __init ec_init_module(void);
+void __exit ec_cleanup_module(void);
+
+static int ec_mac_parse(uint8_t *, const char *, int);
+
+// prototypes for private functions
+int ec_mac_equal(const uint8_t *, const uint8_t *);
+int ec_mac_is_broadcast(const uint8_t *);
+
+/****************************************************************************/
+
+static char *main_devices[EC_MAX_MASTERS]; /**< Main devices parameter. */
+static unsigned int master_count; /**< Number of masters. */
+static char *backup_devices[EC_MAX_MASTERS]; /**< Backup devices parameter. */
+static unsigned int backup_count; /**< Number of backup devices. */
+static unsigned int debug_level;  /**< Debug level parameter. */
+static unsigned int run_on_cpu = 0xffffffff; /**< Bind created kernel threads
+                                               to a cpu. Default do not bind.
+                                              */
+
+static ec_master_t *masters; /**< Array of masters. */
+static struct semaphore master_sem; /**< Master semaphore. */
+
+dev_t device_number; /**< Device number for master cdevs. */
+struct class *class; /**< Device class. */
+
+static uint8_t macs[EC_MAX_MASTERS][2][ETH_ALEN]; /**< MAC addresses. */
+
+char *ec_master_version_str = EC_MASTER_VERSION; /**< Version string. */
+
+/****************************************************************************/
+
+/** \cond */
+
+MODULE_AUTHOR("Florian Pose <fp@igh.de>");
+MODULE_DESCRIPTION("EtherCAT master driver module");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(EC_MASTER_VERSION);
+
+module_param_array(main_devices, charp, &master_count, S_IRUGO);
+MODULE_PARM_DESC(main_devices, "MAC addresses of main devices");
+module_param_array(backup_devices, charp, &backup_count, S_IRUGO);
+MODULE_PARM_DESC(backup_devices, "MAC addresses of backup devices");
+module_param_named(debug_level, debug_level, uint, S_IRUGO);
+MODULE_PARM_DESC(debug_level, "Debug level");
+module_param_named(run_on_cpu, run_on_cpu, uint, S_IRUGO);
+MODULE_PARM_DESC(run_on_cpu, "Bind kthreads to a specific cpu");
+
+/** \endcond */
+
+/****************************************************************************/
+
+/** Module initialization.
+ *
+ * Initializes \a master_count masters.
+ * \return 0 on success, else < 0
+ */
+int __init ec_init_module(void)
+{
+    int i, ret = 0;
+
+    EC_INFO("Master driver %s\n", EC_MASTER_VERSION);
+
+    sema_init(&master_sem, 1);
+
+    if (master_count) {
+        if (alloc_chrdev_region(&device_number,
+                    0, master_count, "EtherCAT")) {
+            EC_ERR("Failed to obtain device number(s)!\n");
+            ret = -EBUSY;
+            goto out_return;
+        }
+    }
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)
+    class = class_create(THIS_MODULE, "EtherCAT");
+#else
+    class = class_create("EtherCAT");
+#endif
+    if (IS_ERR(class)) {
+        EC_ERR("Failed to create device class.\n");
+        ret = PTR_ERR(class);
+        goto out_cdev;
+    }
+
+    // zero MAC addresses
+    memset(macs, 0x00, sizeof(uint8_t) * EC_MAX_MASTERS * 2 * ETH_ALEN);
+
+    // process MAC parameters
+    for (i = 0; i < master_count; i++) {
+        ret = ec_mac_parse(macs[i][0], main_devices[i], 0);
+        if (ret)
+            goto out_class;
+
+        if (i < backup_count) {
+            ret = ec_mac_parse(macs[i][1], backup_devices[i], 1);
+            if (ret)
+                goto out_class;
+        }
+    }
+
+    // initialize static master variables
+    ec_master_init_static();
+
+    if (master_count) {
+        if (!(masters = kmalloc(sizeof(ec_master_t) * master_count,
+                        GFP_KERNEL))) {
+            EC_ERR("Failed to allocate memory"
+                    " for EtherCAT masters.\n");
+            ret = -ENOMEM;
+            goto out_class;
+        }
+    }
+
+    for (i = 0; i < master_count; i++) {
+        ret = ec_master_init(&masters[i], i, macs[i][0], macs[i][1],
+                    device_number, class, debug_level, run_on_cpu);
+        if (ret)
+            goto out_free_masters;
+    }
+
+    EC_INFO("%u master%s waiting for devices.\n",
+            master_count, (master_count == 1 ? "" : "s"));
+    return ret;
+
+out_free_masters:
+    for (i--; i >= 0; i--)
+        ec_master_clear(&masters[i]);
+    kfree(masters);
+out_class:
+    class_destroy(class);
+out_cdev:
+    if (master_count)
+        unregister_chrdev_region(device_number, master_count);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Module cleanup.
+ *
+ * Clears all master instances.
+ */
+void __exit ec_cleanup_module(void)
+{
+    unsigned int i;
+
+    for (i = 0; i < master_count; i++) {
+        ec_master_clear(&masters[i]);
+    }
+
+    if (master_count)
+        kfree(masters);
+
+    class_destroy(class);
+
+    if (master_count)
+        unregister_chrdev_region(device_number, master_count);
+
+    EC_INFO("Master module cleaned up.\n");
+}
+
+/****************************************************************************/
+
+/** Get the number of masters.
+ */
+unsigned int ec_master_count(void)
+{
+    return master_count;
+}
+
+/*****************************************************************************
+ * MAC address functions
+ ****************************************************************************/
+
+/**
+ * \return true, if two MAC addresses are equal.
+ */
+int ec_mac_equal(
+        const uint8_t *mac1, /**< First MAC address. */
+        const uint8_t *mac2 /**< Second MAC address. */
+        )
+{
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++)
+        if (mac1[i] != mac2[i])
+            return 0;
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Maximum MAC string size.
+ */
+#define EC_MAX_MAC_STRING_SIZE (3 * ETH_ALEN)
+
+/** Print a MAC address to a buffer.
+ *
+ * The buffer size must be at least EC_MAX_MAC_STRING_SIZE.
+ *
+ * \return number of bytes written.
+ */
+ssize_t ec_mac_print(
+        const uint8_t *mac, /**< MAC address */
+        char *buffer /**< Target buffer. */
+        )
+{
+    off_t off = 0;
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++) {
+        off += sprintf(buffer + off, "%02X", mac[i]);
+        if (i < ETH_ALEN - 1) off += sprintf(buffer + off, ":");
+    }
+
+    return off;
+}
+
+/****************************************************************************/
+
+/**
+ * \return true, if the MAC address is all-zero.
+ */
+int ec_mac_is_zero(
+        const uint8_t *mac /**< MAC address. */
+        )
+{
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++)
+        if (mac[i])
+            return 0;
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/**
+ * \return true, if the given MAC address is the broadcast address.
+ */
+int ec_mac_is_broadcast(
+        const uint8_t *mac /**< MAC address. */
+        )
+{
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++)
+        if (mac[i] != 0xff)
+            return 0;
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Parse a MAC address from a string.
+ *
+ * The MAC address must match the regular expression
+ * "([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}".
+ *
+ * \return 0 on success, else < 0
+ */
+static int ec_mac_parse(uint8_t *mac, const char *src, int allow_empty)
+{
+    unsigned int i, value;
+    const char *orig = src;
+    char *rem;
+
+    if (!strlen(src)) {
+        if (allow_empty){
+            return 0;
+        } else {
+            EC_ERR("MAC address may not be empty.\n");
+            return -EINVAL;
+        }
+    }
+
+    for (i = 0; i < ETH_ALEN; i++) {
+        value = simple_strtoul(src, &rem, 16);
+        if (rem != src + 2
+                || value > 0xFF
+                || (i < ETH_ALEN - 1 && *rem != ':')) {
+            EC_ERR("Invalid MAC address \"%s\".\n", orig);
+            return -EINVAL;
+        }
+        mac[i] = value;
+        if (i < ETH_ALEN - 1) {
+            src = rem + 1; // skip colon
+        }
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Outputs frame contents for debugging purposes.
+ * If the data block is larger than 256 bytes, only the first 128
+ * and the last 128 bytes will be shown
+ */
+void ec_print_data(const uint8_t *data, /**< pointer to data */
+                   size_t size /**< number of bytes to output */
+                   )
+{
+    unsigned int i;
+
+    EC_DBG("");
+    for (i = 0; i < size; i++) {
+        printk(KERN_CONT "%02X ", data[i]);
+
+        if ((i + 1) % 16 == 0 && i < size - 1) {
+            printk(KERN_CONT "\n");
+            EC_DBG("");
+        }
+
+        if (i + 1 == 128 && size > 256) {
+            printk(KERN_CONT "dropped %zu bytes\n", size - 128 - i);
+            i = size - 128;
+            EC_DBG("");
+        }
+    }
+    printk(KERN_CONT "\n");
+}
+
+/****************************************************************************/
+
+/** Outputs frame contents and differences for debugging purposes.
+ */
+void ec_print_data_diff(const uint8_t *d1, /**< first data */
+                        const uint8_t *d2, /**< second data */
+                        size_t size /** number of bytes to output */
+                        )
+{
+    unsigned int i;
+
+    EC_DBG("");
+    for (i = 0; i < size; i++) {
+        if (d1[i] == d2[i]) {
+            printk(KERN_CONT ".. ");
+        }
+        else {
+            printk(KERN_CONT "%02X ", d2[i]);
+        }
+        if ((i + 1) % 16 == 0) {
+            printk(KERN_CONT "\n");
+            EC_DBG("");
+        }
+    }
+    printk(KERN_CONT "\n");
+}
+
+/****************************************************************************/
+
+/** Prints slave states in clear text.
+ *
+ * \return Size of the created string.
+ */
+size_t ec_state_string(uint8_t states, /**< slave states */
+                       char *buffer, /**< target buffer
+                                       (min. EC_STATE_STRING_SIZE bytes) */
+                       uint8_t multi /**< Show multi-state mask. */
+                       )
+{
+    off_t off = 0;
+    unsigned int first = 1;
+
+    if (!states) {
+        off += sprintf(buffer + off, "(unknown)");
+        return off;
+    }
+
+    if (multi) { // multiple slaves
+        if (states & EC_SLAVE_STATE_INIT) {
+            off += sprintf(buffer + off, "INIT");
+            first = 0;
+        }
+        if (states & EC_SLAVE_STATE_PREOP) {
+            if (!first) off += sprintf(buffer + off, ", ");
+            off += sprintf(buffer + off, "PREOP");
+            first = 0;
+        }
+        if (states & EC_SLAVE_STATE_SAFEOP) {
+            if (!first) off += sprintf(buffer + off, ", ");
+            off += sprintf(buffer + off, "SAFEOP");
+            first = 0;
+        }
+        if (states & EC_SLAVE_STATE_OP) {
+            if (!first) off += sprintf(buffer + off, ", ");
+            off += sprintf(buffer + off, "OP");
+        }
+    } else { // single slave
+        if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_INIT) {
+            off += sprintf(buffer + off, "INIT");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_PREOP) {
+            off += sprintf(buffer + off, "PREOP");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_BOOT) {
+            off += sprintf(buffer + off, "BOOT");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_SAFEOP) {
+            off += sprintf(buffer + off, "SAFEOP");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_OP) {
+            off += sprintf(buffer + off, "OP");
+        } else {
+            off += sprintf(buffer + off, "(invalid)");
+        }
+        first = 0;
+    }
+
+    if (states & EC_SLAVE_STATE_ACK_ERR) {
+        if (!first) off += sprintf(buffer + off, " + ");
+        off += sprintf(buffer + off, "ERROR");
+    }
+
+    return off;
+}
+
+/*****************************************************************************
+ *  Device interface
+ ****************************************************************************/
+
+/** Device names.
+ */
+const char *ec_device_names[2] = {
+    "main",
+    "backup"
+};
+
+/** Offers an EtherCAT device to a certain master.
+ *
+ * The master decides, if it wants to use the device for EtherCAT operation,
+ * or not. It is important, that the offered net_device is not used by the
+ * kernel IP stack. If the master, accepted the offer, the address of the
+ * newly created EtherCAT device is returned, else \a NULL is returned.
+ *
+ * \return Pointer to device, if accepted, or NULL if declined.
+ * \ingroup DeviceInterface
+ */
+ec_device_t *ecdev_offer(
+        struct net_device *net_dev, /**< net_device to offer */
+        ec_pollfunc_t poll, /**< device poll function */
+        struct module *module /**< pointer to the module */
+        )
+{
+    ec_master_t *master;
+    char str[EC_MAX_MAC_STRING_SIZE];
+    unsigned int i, dev_idx;
+
+    for (i = 0; i < master_count; i++) {
+        master = &masters[i];
+        ec_mac_print(net_dev->dev_addr, str);
+
+        if (down_interruptible(&master->device_sem)) {
+            EC_MASTER_WARN(master, "%s() interrupted!\n", __func__);
+            return NULL;
+        }
+
+        for (dev_idx = EC_DEVICE_MAIN;
+                dev_idx < ec_master_num_devices(master); dev_idx++) {
+            if (!master->devices[dev_idx].dev
+                && (ec_mac_equal(master->macs[dev_idx], net_dev->dev_addr)
+                    || ec_mac_is_broadcast(master->macs[dev_idx]))) {
+
+                EC_INFO("Accepting %s as %s device for master %u.\n",
+                        str, ec_device_names[dev_idx != 0], master->index);
+
+                ec_device_attach(&master->devices[dev_idx],
+                        net_dev, poll, module);
+                up(&master->device_sem);
+
+                snprintf(net_dev->name, IFNAMSIZ, "ec%c%u",
+                        ec_device_names[dev_idx != 0][0], master->index);
+
+                return &master->devices[dev_idx]; // offer accepted
+            }
+        }
+
+        up(&master->device_sem);
+
+        EC_MASTER_DBG(master, 1, "Master declined device %s.\n", str);
+    }
+
+    return NULL; // offer declined
+}
+
+/*****************************************************************************
+ * Application interface
+ ****************************************************************************/
+
+/** Request a master.
+ *
+ * Same as ecrt_request_master(), but with ERR_PTR() return value.
+ *
+ * \return Requested master.
+ */
+ec_master_t *ecrt_request_master_err(
+        unsigned int master_index /**< Master index. */
+        )
+{
+    ec_master_t *master, *errptr = NULL;
+    unsigned int dev_idx = EC_DEVICE_MAIN;
+
+    EC_INFO("Requesting master %u...\n", master_index);
+
+    if (master_index >= master_count) {
+        EC_ERR("Invalid master index %u.\n", master_index);
+        errptr = ERR_PTR(-EINVAL);
+        goto out_return;
+    }
+    master = &masters[master_index];
+
+    if (down_interruptible(&master_sem)) {
+        errptr = ERR_PTR(-EINTR);
+        goto out_return;
+    }
+
+    if (master->reserved) {
+        up(&master_sem);
+        EC_MASTER_ERR(master, "Master already in use!\n");
+        errptr = ERR_PTR(-EBUSY);
+        goto out_return;
+    }
+    master->reserved = 1;
+    up(&master_sem);
+
+    if (down_interruptible(&master->device_sem)) {
+        errptr = ERR_PTR(-EINTR);
+        goto out_release;
+    }
+
+    if (master->phase != EC_IDLE) {
+        up(&master->device_sem);
+        EC_MASTER_ERR(master, "Master still waiting for devices!\n");
+        errptr = ERR_PTR(-ENODEV);
+        goto out_release;
+    }
+
+    for (; dev_idx < ec_master_num_devices(master); dev_idx++) {
+        ec_device_t *device = &master->devices[dev_idx];
+        if (!try_module_get(device->module)) {
+            up(&master->device_sem);
+            EC_MASTER_ERR(master, "Device module is unloading!\n");
+            errptr = ERR_PTR(-ENODEV);
+            goto out_module_put;
+        }
+    }
+
+    up(&master->device_sem);
+
+    if (ec_master_enter_operation_phase(master)) {
+        EC_MASTER_ERR(master, "Failed to enter OPERATION phase!\n");
+        errptr = ERR_PTR(-EIO);
+        goto out_module_put;
+    }
+
+    EC_INFO("Successfully requested master %u.\n", master_index);
+    return master;
+
+ out_module_put:
+    for (; dev_idx > 0; dev_idx--) {
+        ec_device_t *device = &master->devices[dev_idx - 1];
+        module_put(device->module);
+    }
+ out_release:
+    master->reserved = 0;
+ out_return:
+    return errptr;
+}
+
+/****************************************************************************/
+
+ec_master_t *ecrt_request_master(unsigned int master_index)
+{
+    ec_master_t *master = ecrt_request_master_err(master_index);
+    return IS_ERR(master) ? NULL : master;
+}
+
+/****************************************************************************/
+
+void ecrt_release_master(ec_master_t *master)
+{
+    unsigned int dev_idx;
+
+    EC_MASTER_INFO(master, "Releasing master...\n");
+
+    if (!master->reserved) {
+        EC_MASTER_WARN(master, "%s(): Master was was not requested!\n",
+                __func__);
+        return;
+    }
+
+    ec_master_leave_operation_phase(master);
+
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        module_put(master->devices[dev_idx].module);
+    }
+
+    master->reserved = 0;
+
+    EC_MASTER_INFO(master, "Released.\n");
+}
+
+/****************************************************************************/
+
+unsigned int ecrt_version_magic(void)
+{
+    return ECRT_VERSION_MAGIC;
+}
+
+/****************************************************************************/
+
+/** Global request state type translation table.
+ *
+ * Translates an internal request state to an external one.
+ */
+const ec_request_state_t ec_request_state_translation_table[] = {
+    EC_REQUEST_UNUSED,  // EC_INT_REQUEST_INIT,
+    EC_REQUEST_BUSY,    // EC_INT_REQUEST_QUEUED,
+    EC_REQUEST_BUSY,    // EC_INT_REQUEST_BUSY,
+    EC_REQUEST_SUCCESS, // EC_INT_REQUEST_SUCCESS,
+    EC_REQUEST_ERROR    // EC_INT_REQUEST_FAILURE
+};
+
+/****************************************************************************/
+
+/** \cond */
+
+module_init(ec_init_module);
+module_exit(ec_cleanup_module);
+
+EXPORT_SYMBOL(ecdev_offer);
+
+EXPORT_SYMBOL(ecrt_request_master);
+EXPORT_SYMBOL(ecrt_release_master);
+EXPORT_SYMBOL(ecrt_version_magic);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/module_of.c b/drivers/net/ethercat/master/module_of.c
new file mode 100644
index 000000000000..fab239ac759a
--- /dev/null
+++ b/drivers/net/ethercat/master/module_of.c
@@ -0,0 +1,725 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT master driver module.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/err.h>
+
+#include "globals.h"
+#include "master.h"
+#include "device.h"
+
+#include <linux/platform_device.h>
+#include <linux/of_net.h>
+#include <linux/of.h>
+#include <linux/io.h>
+#include <generated/autoconf.h>
+
+#define DRIVER_NAME "igh_ec_master"
+
+/****************************************************************************/
+
+int ec_mac_parse(uint8_t *, const char *, int);
+
+// prototypes for private functions
+int ec_mac_equal(const uint8_t *, const uint8_t *);
+int ec_mac_is_broadcast(const uint8_t *);
+
+/****************************************************************************/
+
+//static char *main_devices[EC_MAX_MASTERS]; /**< Main devices parameter. */
+static unsigned int master_count; /**< Number of masters. */
+static unsigned int debug_level = CONFIG_EC_MASTER_DEBUG_LEVEL;  /**< Debug level parameter. */
+static unsigned int run_on_cpu = CONFIG_EC_MASTER_RUN_ON_CPU; /**< Bind created kernel threads
+                                               		           to a cpu. Default do not bind.
+                                              		        */
+
+static ec_master_t *masters; /**< Array of masters. */
+static struct semaphore master_sem; /**< Master semaphore. */
+
+dev_t device_number; /**< Device number for master cdevs. */
+struct class *class; /**< Device class. */
+
+static uint8_t macs[EC_MAX_MASTERS][2][ETH_ALEN]; /**< MAC addresses. */
+
+char *ec_master_version_str = EC_MASTER_VERSION; /**< Version string. */
+
+/****************************************************************************/
+
+static int ec_parse_dt(struct platform_device *pdev)
+{
+    struct device_node *node = pdev->dev.of_node;
+    struct device_node *master_node, *eth_node;
+    int ret, i;
+
+    if (!node) {
+        EC_ERR("No device tree node found\n");
+        return -EINVAL;
+    }
+
+    if (of_property_read_u32(node, "master-count", &master_count)) {
+        EC_ERR("Failed to read master-count\n");
+        return -EINVAL;
+    }
+
+    if (master_count <= 0 || master_count >= EC_MAX_MASTERS) {
+        EC_ERR("Invalid master-count: %u\n", master_count);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < master_count; i++) {
+        char master_name[16];
+        snprintf(master_name, sizeof(master_name), "master%d", i);
+
+        master_node = of_get_child_by_name(node, master_name);
+        if (!master_node) {
+            EC_ERR("Missing master@%d subnode\n", i);
+            return -EINVAL;
+        }
+
+        eth_node = of_parse_phandle(master_node, "main-device", 0);
+        if (!eth_node) {
+            EC_ERR("Missing main-device in %s\n", master_name);
+            of_node_put(master_node);
+            return -EINVAL;
+        }
+
+        ret = of_get_mac_address(eth_node, macs[i][0]);
+        if (ret) {
+            EC_ERR("Failed to parse main-device for %s\n", master_name);
+            of_node_put(eth_node);
+            of_node_put(master_node);
+            return -EINVAL;
+        }
+
+        of_node_put(eth_node);
+
+        eth_node = of_parse_phandle(master_node, "backup-device", 0);
+        if (eth_node) {
+            ret = of_get_mac_address(eth_node, macs[i][1]);
+            if (ret) {
+                EC_ERR("Failed to parse backup-device for %s\n", master_name);
+                of_node_put(eth_node);
+                of_node_put(master_node);
+                return -EINVAL;
+            }
+            of_node_put(eth_node);
+        }
+
+        of_node_put(master_node);
+    }
+
+    return 0;
+}
+
+static int ec_probe(struct platform_device *pdev)
+{
+    int ret, i;
+
+    sema_init(&master_sem, 1);
+
+    ret = ec_parse_dt(pdev);
+    if (ret) {
+	EC_ERR(DRIVER_NAME ": Failed to parse dts, error %d\n", ret);
+	master_count = 0;
+    }
+
+    if (master_count) {
+        if (alloc_chrdev_region(&device_number,
+                    0, master_count, "EtherCAT")) {
+            EC_ERR("Failed to obtain device number(s)!\n");
+            ret = -EBUSY;
+            goto out_return;
+        }
+    }
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)
+    class = class_create(THIS_MODULE, "EtherCAT");
+#else
+    class = class_create("EtherCAT");
+#endif
+    if (IS_ERR(class)) {
+        EC_ERR("Failed to create device class.\n");
+        ret = PTR_ERR(class);
+        goto out_cdev;
+    }
+
+    // initialize static master variables
+    ec_master_init_static();
+
+    if (master_count) {
+        if (!(masters = kmalloc(sizeof(ec_master_t) * master_count,
+                        GFP_KERNEL))) {
+            EC_ERR("Failed to allocate memory"
+                    " for EtherCAT masters.\n");
+            ret = -ENOMEM;
+            goto out_class;
+        }
+    }
+
+    for (i = 0; i < master_count; i++) {
+        ret = ec_master_init(&masters[i], i, macs[i][0], macs[i][1],
+                    device_number, class, debug_level, run_on_cpu);
+        if (ret)
+            goto out_free_masters;
+    }
+
+    EC_INFO("%u master%s waiting for devices.\n",
+            master_count, (master_count == 1 ? "" : "s"));
+    return ret;
+
+out_free_masters:
+    for (i--; i >= 0; i--)
+        ec_master_clear(&masters[i]);
+    kfree(masters);
+out_class:
+    class_destroy(class);
+out_cdev:
+    if (master_count)
+        unregister_chrdev_region(device_number, master_count);
+out_return:
+    return ret;
+}
+
+static void ec_remove(struct platform_device *pdev)
+{
+    unsigned int i;
+
+    for (i = 0; i < master_count; i++) {
+        ec_master_clear(&masters[i]);
+    }
+
+    if (master_count)
+        kfree(masters);
+
+    class_destroy(class);
+
+    if (master_count)
+        unregister_chrdev_region(device_number, master_count);
+
+    EC_INFO("Master module cleaned up.\n");
+}
+
+/****************************************************************************/
+
+/** Get the number of masters.
+ */
+unsigned int ec_master_count(void)
+{
+    return master_count;
+}
+
+/*****************************************************************************
+ * MAC address functions
+ ****************************************************************************/
+
+/**
+ * \return true, if two MAC addresses are equal.
+ */
+int ec_mac_equal(
+        const uint8_t *mac1, /**< First MAC address. */
+        const uint8_t *mac2 /**< Second MAC address. */
+        )
+{
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++)
+        if (mac1[i] != mac2[i])
+            return 0;
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Maximum MAC string size.
+ */
+#define EC_MAX_MAC_STRING_SIZE (3 * ETH_ALEN)
+
+/** Print a MAC address to a buffer.
+ *
+ * The buffer size must be at least EC_MAX_MAC_STRING_SIZE.
+ *
+ * \return number of bytes written.
+ */
+ssize_t ec_mac_print(
+        const uint8_t *mac, /**< MAC address */
+        char *buffer /**< Target buffer. */
+        )
+{
+    off_t off = 0;
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++) {
+        off += sprintf(buffer + off, "%02X", mac[i]);
+        if (i < ETH_ALEN - 1) off += sprintf(buffer + off, ":");
+    }
+
+    return off;
+}
+
+/****************************************************************************/
+
+/**
+ * \return true, if the MAC address is all-zero.
+ */
+int ec_mac_is_zero(
+        const uint8_t *mac /**< MAC address. */
+        )
+{
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++)
+        if (mac[i])
+            return 0;
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/**
+ * \return true, if the given MAC address is the broadcast address.
+ */
+int ec_mac_is_broadcast(
+        const uint8_t *mac /**< MAC address. */
+        )
+{
+    unsigned int i;
+
+    for (i = 0; i < ETH_ALEN; i++)
+        if (mac[i] != 0xff)
+            return 0;
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Parse a MAC address from a string.
+ *
+ * The MAC address must match the regular expression
+ * "([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}".
+ *
+ * \return 0 on success, else < 0
+ */
+int ec_mac_parse(uint8_t *mac, const char *src, int allow_empty)
+{
+    unsigned int i, value;
+    const char *orig = src;
+    char *rem;
+
+    if (!strlen(src)) {
+        if (allow_empty){
+            return 0;
+        } else {
+            EC_ERR("MAC address may not be empty.\n");
+            return -EINVAL;
+        }
+    }
+
+    for (i = 0; i < ETH_ALEN; i++) {
+        value = simple_strtoul(src, &rem, 16);
+        if (rem != src + 2
+                || value > 0xFF
+                || (i < ETH_ALEN - 1 && *rem != ':')) {
+            EC_ERR("Invalid MAC address \"%s\".\n", orig);
+            return -EINVAL;
+        }
+        mac[i] = value;
+        if (i < ETH_ALEN - 1) {
+            src = rem + 1; // skip colon
+        }
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Outputs frame contents for debugging purposes.
+ * If the data block is larger than 256 bytes, only the first 128
+ * and the last 128 bytes will be shown
+ */
+void ec_print_data(const uint8_t *data, /**< pointer to data */
+                   size_t size /**< number of bytes to output */
+                   )
+{
+    unsigned int i;
+
+    EC_DBG("");
+    for (i = 0; i < size; i++) {
+        printk(KERN_CONT "%02X ", data[i]);
+
+        if ((i + 1) % 16 == 0 && i < size - 1) {
+            printk(KERN_CONT "\n");
+            EC_DBG("");
+        }
+
+        if (i + 1 == 128 && size > 256) {
+            printk(KERN_CONT "dropped %zu bytes\n", size - 128 - i);
+            i = size - 128;
+            EC_DBG("");
+        }
+    }
+    printk(KERN_CONT "\n");
+}
+
+/****************************************************************************/
+
+/** Outputs frame contents and differences for debugging purposes.
+ */
+void ec_print_data_diff(const uint8_t *d1, /**< first data */
+                        const uint8_t *d2, /**< second data */
+                        size_t size /** number of bytes to output */
+                        )
+{
+    unsigned int i;
+
+    EC_DBG("");
+    for (i = 0; i < size; i++) {
+        if (d1[i] == d2[i]) {
+            printk(KERN_CONT ".. ");
+        }
+        else {
+            printk(KERN_CONT "%02X ", d2[i]);
+        }
+        if ((i + 1) % 16 == 0) {
+            printk(KERN_CONT "\n");
+            EC_DBG("");
+        }
+    }
+    printk(KERN_CONT "\n");
+}
+
+/****************************************************************************/
+
+/** Prints slave states in clear text.
+ *
+ * \return Size of the created string.
+ */
+size_t ec_state_string(uint8_t states, /**< slave states */
+                       char *buffer, /**< target buffer
+                                       (min. EC_STATE_STRING_SIZE bytes) */
+                       uint8_t multi /**< Show multi-state mask. */
+                       )
+{
+    off_t off = 0;
+    unsigned int first = 1;
+
+    if (!states) {
+        off += sprintf(buffer + off, "(unknown)");
+        return off;
+    }
+
+    if (multi) { // multiple slaves
+        if (states & EC_SLAVE_STATE_INIT) {
+            off += sprintf(buffer + off, "INIT");
+            first = 0;
+        }
+        if (states & EC_SLAVE_STATE_PREOP) {
+            if (!first) off += sprintf(buffer + off, ", ");
+            off += sprintf(buffer + off, "PREOP");
+            first = 0;
+        }
+        if (states & EC_SLAVE_STATE_SAFEOP) {
+            if (!first) off += sprintf(buffer + off, ", ");
+            off += sprintf(buffer + off, "SAFEOP");
+            first = 0;
+        }
+        if (states & EC_SLAVE_STATE_OP) {
+            if (!first) off += sprintf(buffer + off, ", ");
+            off += sprintf(buffer + off, "OP");
+        }
+    } else { // single slave
+        if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_INIT) {
+            off += sprintf(buffer + off, "INIT");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_PREOP) {
+            off += sprintf(buffer + off, "PREOP");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_BOOT) {
+            off += sprintf(buffer + off, "BOOT");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_SAFEOP) {
+            off += sprintf(buffer + off, "SAFEOP");
+        } else if ((states & EC_SLAVE_STATE_MASK) == EC_SLAVE_STATE_OP) {
+            off += sprintf(buffer + off, "OP");
+        } else {
+            off += sprintf(buffer + off, "(invalid)");
+        }
+        first = 0;
+    }
+
+    if (states & EC_SLAVE_STATE_ACK_ERR) {
+        if (!first) off += sprintf(buffer + off, " + ");
+        off += sprintf(buffer + off, "ERROR");
+    }
+
+    return off;
+}
+
+/*****************************************************************************
+ *  Device interface
+ ****************************************************************************/
+
+/** Device names.
+ */
+const char *ec_device_names[2] = {
+    "main",
+    "backup"
+};
+
+/** Offers an EtherCAT device to a certain master.
+ *
+ * The master decides, if it wants to use the device for EtherCAT operation,
+ * or not. It is important, that the offered net_device is not used by the
+ * kernel IP stack. If the master, accepted the offer, the address of the
+ * newly created EtherCAT device is returned, else \a NULL is returned.
+ *
+ * \return Pointer to device, if accepted, or NULL if declined.
+ * \ingroup DeviceInterface
+ */
+
+ec_device_t *ecdev_offer(
+        struct net_device *net_dev, /**< net_device to offer */
+        ec_pollfunc_t poll, /**< device poll function */
+        struct module *module /**< pointer to the module */
+        )
+{
+    ec_master_t *master;
+    char str[EC_MAX_MAC_STRING_SIZE];
+    unsigned int i, dev_idx;
+
+    for (i = 0; i < master_count; i++) {
+        master = &masters[i];
+        ec_mac_print(net_dev->dev_addr, str);
+
+        if (down_interruptible(&master->device_sem)) {
+            EC_MASTER_WARN(master, "%s() interrupted!\n", __func__);
+            return NULL;
+        }
+
+        for (dev_idx = EC_DEVICE_MAIN;
+                dev_idx < ec_master_num_devices(master); dev_idx++) {
+            if (!master->devices[dev_idx].dev
+                && (ec_mac_equal(master->macs[dev_idx], net_dev->dev_addr)
+                    || ec_mac_is_broadcast(master->macs[dev_idx]))) {
+
+                EC_INFO("Accepting %s as %s device for master %u.\n",
+                        str, ec_device_names[dev_idx != 0], master->index);
+
+                ec_device_attach(&master->devices[dev_idx],
+                        net_dev, poll, module);
+                up(&master->device_sem);
+
+                snprintf(net_dev->name, IFNAMSIZ, "ec%c%u",
+                        ec_device_names[dev_idx != 0][0], master->index);
+
+                return &master->devices[dev_idx]; // offer accepted
+            }
+        }
+
+        up(&master->device_sem);
+
+        EC_MASTER_DBG(master, 1, "Master declined device %s.\n", str);
+    }
+
+    return NULL; // offer declined
+}
+
+/*****************************************************************************
+ * Application interface
+ ****************************************************************************/
+
+/** Request a master.
+ *
+ * Same as ecrt_request_master(), but with ERR_PTR() return value.
+ *
+ * \return Requested master.
+ */
+ec_master_t *ecrt_request_master_err(
+        unsigned int master_index /**< Master index. */
+        )
+{
+    ec_master_t *master, *errptr = NULL;
+    unsigned int dev_idx = EC_DEVICE_MAIN;
+
+    EC_INFO("Requesting master %u...\n", master_index);
+
+    if (master_index >= master_count) {
+        EC_ERR("Invalid master index %u.\n", master_index);
+        errptr = ERR_PTR(-EINVAL);
+        goto out_return;
+    }
+    master = &masters[master_index];
+
+    if (down_interruptible(&master_sem)) {
+        errptr = ERR_PTR(-EINTR);
+        goto out_return;
+    }
+
+    if (master->reserved) {
+        up(&master_sem);
+        EC_MASTER_ERR(master, "Master already in use!\n");
+        errptr = ERR_PTR(-EBUSY);
+        goto out_return;
+    }
+    master->reserved = 1;
+    up(&master_sem);
+
+    if (down_interruptible(&master->device_sem)) {
+        errptr = ERR_PTR(-EINTR);
+        goto out_release;
+    }
+
+    if (master->phase != EC_IDLE) {
+        up(&master->device_sem);
+        EC_MASTER_ERR(master, "Master still waiting for devices!\n");
+        errptr = ERR_PTR(-ENODEV);
+        goto out_release;
+    }
+
+    for (; dev_idx < ec_master_num_devices(master); dev_idx++) {
+        ec_device_t *device = &master->devices[dev_idx];
+        if (!try_module_get(device->module)) {
+            up(&master->device_sem);
+            EC_MASTER_ERR(master, "Device module is unloading!\n");
+            errptr = ERR_PTR(-ENODEV);
+            goto out_module_put;
+        }
+    }
+
+    up(&master->device_sem);
+
+    if (ec_master_enter_operation_phase(master)) {
+        EC_MASTER_ERR(master, "Failed to enter OPERATION phase!\n");
+        errptr = ERR_PTR(-EIO);
+        goto out_module_put;
+    }
+
+    EC_INFO("Successfully requested master %u.\n", master_index);
+    return master;
+
+ out_module_put:
+    for (; dev_idx > 0; dev_idx--) {
+        ec_device_t *device = &master->devices[dev_idx - 1];
+        module_put(device->module);
+    }
+ out_release:
+    master->reserved = 0;
+ out_return:
+    return errptr;
+}
+
+/****************************************************************************/
+
+ec_master_t *ecrt_request_master(unsigned int master_index)
+{
+    ec_master_t *master = ecrt_request_master_err(master_index);
+    return IS_ERR(master) ? NULL : master;
+}
+
+/****************************************************************************/
+
+void ecrt_release_master(ec_master_t *master)
+{
+    unsigned int dev_idx;
+
+    EC_MASTER_INFO(master, "Releasing master...\n");
+
+    if (!master->reserved) {
+        EC_MASTER_WARN(master, "%s(): Master was was not requested!\n",
+                __func__);
+        return;
+    }
+
+    ec_master_leave_operation_phase(master);
+
+    for (dev_idx = EC_DEVICE_MAIN; dev_idx < ec_master_num_devices(master);
+            dev_idx++) {
+        module_put(master->devices[dev_idx].module);
+    }
+
+    master->reserved = 0;
+
+    EC_MASTER_INFO(master, "Released.\n");
+}
+
+/****************************************************************************/
+
+unsigned int ecrt_version_magic(void)
+{
+    return ECRT_VERSION_MAGIC;
+}
+
+/****************************************************************************/
+
+/** Global request state type translation table.
+ *
+ * Translates an internal request state to an external one.
+ */
+const ec_request_state_t ec_request_state_translation_table[] = {
+    EC_REQUEST_UNUSED,  // EC_INT_REQUEST_INIT,
+    EC_REQUEST_BUSY,    // EC_INT_REQUEST_QUEUED,
+    EC_REQUEST_BUSY,    // EC_INT_REQUEST_BUSY,
+    EC_REQUEST_SUCCESS, // EC_INT_REQUEST_SUCCESS,
+    EC_REQUEST_ERROR    // EC_INT_REQUEST_FAILURE
+};
+
+/****************************************************************************/
+
+static const struct of_device_id ec_of_match[] = {
+    { .compatible = "spacemit,igh-ec-master" },
+    { },
+};
+MODULE_DEVICE_TABLE(of, ec_of_match);
+
+static struct platform_driver ec_master_driver = {
+    .probe = ec_probe,
+    .remove = ec_remove,
+    .driver = {
+        .name = DRIVER_NAME,
+        .of_match_table = of_match_ptr(ec_of_match),
+    },
+};
+
+module_platform_driver(ec_master_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Igh Ethercat master driver");
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecdev_offer);
+
+EXPORT_SYMBOL(ecrt_request_master);
+EXPORT_SYMBOL(ecrt_release_master);
+EXPORT_SYMBOL(ecrt_version_magic);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/pdo.c b/drivers/net/ethercat/master/pdo.c
new file mode 100644
index 000000000000..9c1459395745
--- /dev/null
+++ b/drivers/net/ethercat/master/pdo.c
@@ -0,0 +1,309 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT process data object methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+#include <linux/err.h>
+
+#include "pdo.h"
+
+/****************************************************************************/
+
+/** PDO constructor.
+ */
+void ec_pdo_init(
+        ec_pdo_t *pdo /**< EtherCAT PDO */
+        )
+{
+    pdo->sync_index = -1; // not assigned
+    pdo->name = NULL;
+    INIT_LIST_HEAD(&pdo->entries);
+}
+
+/****************************************************************************/
+
+/** PDO copy constructor.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_pdo_init_copy(
+        ec_pdo_t *pdo, /**< PDO to create. */
+        const ec_pdo_t *other_pdo /**< PDO to copy from. */
+        )
+{
+    int ret = 0;
+
+    pdo->index = other_pdo->index;
+    pdo->sync_index = other_pdo->sync_index;
+    pdo->name = NULL;
+    INIT_LIST_HEAD(&pdo->entries);
+
+    ret = ec_pdo_set_name(pdo, other_pdo->name);
+    if (ret < 0)
+        goto out_return;
+
+    ret = ec_pdo_copy_entries(pdo, other_pdo);
+    if (ret < 0)
+        goto out_clear;
+
+    return 0;
+
+out_clear:
+    ec_pdo_clear(pdo);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** PDO destructor.
+ */
+void ec_pdo_clear(ec_pdo_t *pdo /**< EtherCAT PDO. */)
+{
+    if (pdo->name)
+        kfree(pdo->name);
+
+    ec_pdo_clear_entries(pdo);
+}
+
+/****************************************************************************/
+
+/** Clear PDO entry list.
+ */
+void ec_pdo_clear_entries(ec_pdo_t *pdo /**< EtherCAT PDO. */)
+{
+    ec_pdo_entry_t *entry, *next;
+
+    // free all PDO entries
+    list_for_each_entry_safe(entry, next, &pdo->entries, list) {
+        list_del(&entry->list);
+        ec_pdo_entry_clear(entry);
+        kfree(entry);
+    }
+}
+
+/****************************************************************************/
+
+/** Set PDO name.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_pdo_set_name(
+        ec_pdo_t *pdo, /**< PDO. */
+        const char *name /**< New name. */
+        )
+{
+    unsigned int len;
+
+    if (pdo->name && name && !strcmp(pdo->name, name))
+        return 0;
+
+    if (pdo->name)
+        kfree(pdo->name);
+
+    if (name && (len = strlen(name))) {
+        if (!(pdo->name = (char *) kmalloc(len + 1, GFP_KERNEL))) {
+            EC_ERR("Failed to allocate PDO name.\n");
+            return -ENOMEM;
+        }
+        memcpy(pdo->name, name, len + 1);
+    } else {
+        pdo->name = NULL;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Add a new PDO entry to the configuration.
+ *
+ * \retval Pointer to the added entry, otherwise a ERR_PTR() code.
+ */
+ec_pdo_entry_t *ec_pdo_add_entry(
+        ec_pdo_t *pdo, /**< PDO. */
+        uint16_t index, /**< New entry's index. */
+        uint8_t subindex, /**< New entry's subindex. */
+        uint8_t bit_length /**< New entry's bit length. */
+        )
+{
+    ec_pdo_entry_t *entry;
+
+    if (!(entry = kmalloc(sizeof(ec_pdo_entry_t), GFP_KERNEL))) {
+        EC_ERR("Failed to allocate memory for PDO entry.\n");
+        return ERR_PTR(-ENOMEM);
+    }
+
+    ec_pdo_entry_init(entry);
+    entry->index = index;
+    entry->subindex = subindex;
+    entry->bit_length = bit_length;
+    list_add_tail(&entry->list, &pdo->entries);
+    return entry;
+}
+
+/****************************************************************************/
+
+/** Copy PDO entries from another PDO.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_pdo_copy_entries(
+        ec_pdo_t *pdo, /**< PDO whos entries shall be replaced. */
+        const ec_pdo_t *other /**< Pdo with entries to copy. */
+        )
+{
+    ec_pdo_entry_t *entry, *other_entry;
+    int ret;
+
+    ec_pdo_clear_entries(pdo);
+
+    list_for_each_entry(other_entry, &other->entries, list) {
+        if (!(entry = (ec_pdo_entry_t *)
+                    kmalloc(sizeof(ec_pdo_entry_t), GFP_KERNEL))) {
+            EC_ERR("Failed to allocate memory for PDO entry copy.\n");
+            return -ENOMEM;
+        }
+
+        ret = ec_pdo_entry_init_copy(entry, other_entry);
+        if (ret < 0) {
+            kfree(entry);
+            return ret;
+        }
+
+        list_add_tail(&entry->list, &pdo->entries);
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Compares the entries of two PDOs.
+ *
+ * \retval 1 The entries of the given PDOs are equal.
+ * \retval 0 The entries of the given PDOs differ.
+ */
+int ec_pdo_equal_entries(
+        const ec_pdo_t *pdo1, /**< First PDO. */
+        const ec_pdo_t *pdo2 /**< Second PDO. */
+        )
+{
+    const struct list_head *head1, *head2, *item1, *item2;
+    const ec_pdo_entry_t *entry1, *entry2;
+
+    head1 = item1 = &pdo1->entries;
+    head2 = item2 = &pdo2->entries;
+
+    while (1) {
+        item1 = item1->next;
+        item2 = item2->next;
+
+        if ((item1 == head1) ^ (item2 == head2)) // unequal lengths
+            return 0;
+        if (item1 == head1) // both finished
+            break;
+
+        entry1 = list_entry(item1, ec_pdo_entry_t, list);
+        entry2 = list_entry(item2, ec_pdo_entry_t, list);
+        if (!ec_pdo_entry_equal(entry1, entry2))
+            return 0;
+    }
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Get the number of PDO entries.
+ *
+ * \return Number of PDO entries.
+ */
+unsigned int ec_pdo_entry_count(
+        const ec_pdo_t *pdo /**< PDO. */
+        )
+{
+    const ec_pdo_entry_t *entry;
+    unsigned int num = 0;
+
+    list_for_each_entry(entry, &pdo->entries, list) {
+        num++;
+    }
+
+    return num;
+}
+
+/****************************************************************************/
+
+/** Finds a PDO entry via its position in the list.
+ *
+ * Const version.
+ *
+ * \return Search result, or NULL.
+ */
+const ec_pdo_entry_t *ec_pdo_find_entry_by_pos_const(
+        const ec_pdo_t *pdo, /**< PDO. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    const ec_pdo_entry_t *entry;
+
+    list_for_each_entry(entry, &pdo->entries, list) {
+        if (pos--)
+            continue;
+        return entry;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Outputs the PDOs in the list.
+ */
+void ec_pdo_print_entries(
+        const ec_pdo_t *pdo /**< PDO. */
+        )
+{
+    const ec_pdo_entry_t *entry;
+
+    if (list_empty(&pdo->entries)) {
+        printk(KERN_CONT "(none)");
+    } else {
+        list_for_each_entry(entry, &pdo->entries, list) {
+            printk(KERN_CONT "0x%04X:%02X/%u",
+                    entry->index, entry->subindex, entry->bit_length);
+            if (entry->list.next != &pdo->entries)
+                printk(KERN_CONT " ");
+        }
+    }
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/pdo.h b/drivers/net/ethercat/master/pdo.h
new file mode 100644
index 000000000000..1819c836578f
--- /dev/null
+++ b/drivers/net/ethercat/master/pdo.h
@@ -0,0 +1,67 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT Process data object structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_PDO_H__
+#define __EC_PDO_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+#include "pdo_entry.h"
+
+/****************************************************************************/
+
+/** PDO description.
+ */
+typedef struct {
+    struct list_head list; /**< List item. */
+    uint16_t index; /**< PDO index. */
+    int8_t sync_index; /**< Assigned sync manager. \todo remove? */
+    char *name; /**< PDO name. */
+    struct list_head entries; /**< List of PDO entries. */
+} ec_pdo_t;
+
+/****************************************************************************/
+
+void ec_pdo_init(ec_pdo_t *);
+int ec_pdo_init_copy(ec_pdo_t *, const ec_pdo_t *);
+void ec_pdo_clear(ec_pdo_t *);
+void ec_pdo_clear_entries(ec_pdo_t *);
+int ec_pdo_set_name(ec_pdo_t *, const char *);
+ec_pdo_entry_t *ec_pdo_add_entry(ec_pdo_t *, uint16_t, uint8_t, uint8_t);
+int ec_pdo_copy_entries(ec_pdo_t *, const ec_pdo_t *);
+int ec_pdo_equal_entries(const ec_pdo_t *, const ec_pdo_t *);
+unsigned int ec_pdo_entry_count(const ec_pdo_t *);
+const ec_pdo_entry_t *ec_pdo_find_entry_by_pos_const(
+        const ec_pdo_t *, unsigned int);
+
+void ec_pdo_print_entries(const ec_pdo_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/pdo_entry.c b/drivers/net/ethercat/master/pdo_entry.c
new file mode 100644
index 000000000000..836cb3df4597
--- /dev/null
+++ b/drivers/net/ethercat/master/pdo_entry.c
@@ -0,0 +1,124 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT process data object entry methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+
+#include "pdo_entry.h"
+
+/****************************************************************************/
+
+/** PDO entry constructor.
+ */
+void ec_pdo_entry_init(
+        ec_pdo_entry_t *entry /**< PDO entry. */
+        )
+{
+    entry->name = NULL;
+}
+
+/****************************************************************************/
+
+/** PDO entry copy constructor.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_pdo_entry_init_copy(
+        ec_pdo_entry_t *entry, /**< PDO entry. */
+        const ec_pdo_entry_t *other /**< PDO entry to copy from. */
+        )
+{
+    entry->index = other->index;
+    entry->subindex = other->subindex;
+    entry->name = NULL;
+    entry->bit_length = other->bit_length;
+
+    return ec_pdo_entry_set_name(entry, other->name);
+}
+
+/****************************************************************************/
+
+/** PDO entry destructor.
+ */
+void ec_pdo_entry_clear(ec_pdo_entry_t *entry /**< PDO entry. */)
+{
+    if (entry->name)
+        kfree(entry->name);
+}
+
+/****************************************************************************/
+
+/** Set PDO entry name.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_pdo_entry_set_name(
+        ec_pdo_entry_t *entry, /**< PDO entry. */
+        const char *name /**< New name. */
+        )
+{
+    unsigned int len;
+
+    if (entry->name && name && !strcmp(entry->name, name))
+        return 0;
+
+    if (entry->name)
+        kfree(entry->name);
+
+    if (name && (len = strlen(name))) {
+        if (!(entry->name = (char *) kmalloc(len + 1, GFP_KERNEL))) {
+            EC_ERR("Failed to allocate PDO entry name.\n");
+            return -ENOMEM;
+        }
+        memcpy(entry->name, name, len + 1);
+    } else {
+        entry->name = NULL;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Compares two PDO entries.
+ *
+ * \retval 1 The entries are equal.
+ * \retval 0 The entries differ.
+ */
+int ec_pdo_entry_equal(
+        const ec_pdo_entry_t *entry1, /**< First PDO entry. */
+        const ec_pdo_entry_t *entry2 /**< Second PDO entry. */
+        )
+{
+    return entry1->index == entry2->index
+        && entry1->subindex == entry2->subindex
+        && entry1->bit_length == entry2->bit_length;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/pdo_entry.h b/drivers/net/ethercat/master/pdo_entry.h
new file mode 100644
index 000000000000..96dc41ebfd57
--- /dev/null
+++ b/drivers/net/ethercat/master/pdo_entry.h
@@ -0,0 +1,58 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT Process data object structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_PDO_ENTRY_H__
+#define __EC_PDO_ENTRY_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** PDO entry description.
+ */
+typedef struct {
+    struct list_head list; /**< list item */
+    uint16_t index; /**< PDO entry index */
+    uint8_t subindex; /**< PDO entry subindex */
+    char *name; /**< entry name */
+    uint8_t bit_length; /**< entry length in bit */
+} ec_pdo_entry_t;
+
+/****************************************************************************/
+
+void ec_pdo_entry_init(ec_pdo_entry_t *);
+int ec_pdo_entry_init_copy(ec_pdo_entry_t *, const ec_pdo_entry_t *);
+void ec_pdo_entry_clear(ec_pdo_entry_t *);
+int ec_pdo_entry_set_name(ec_pdo_entry_t *, const char *);
+int ec_pdo_entry_equal(const ec_pdo_entry_t *, const ec_pdo_entry_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/pdo_list.c b/drivers/net/ethercat/master/pdo_list.c
new file mode 100644
index 000000000000..f26b1c880c28
--- /dev/null
+++ b/drivers/net/ethercat/master/pdo_list.c
@@ -0,0 +1,338 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT PDO list methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+
+#include "globals.h"
+#include "pdo.h"
+#include "slave_config.h"
+#include "master.h"
+
+#include "pdo_list.h"
+
+/****************************************************************************/
+
+/** PDO list constructor.
+ */
+void ec_pdo_list_init(
+        ec_pdo_list_t *pl /**< PDO list. */
+        )
+{
+    INIT_LIST_HEAD(&pl->list);
+}
+
+/****************************************************************************/
+
+/** PDO list destructor.
+ */
+void ec_pdo_list_clear(ec_pdo_list_t *pl /**< PDO list. */)
+{
+    ec_pdo_list_clear_pdos(pl);
+}
+
+/****************************************************************************/
+
+/** Clears the list of mapped PDOs.
+ */
+void ec_pdo_list_clear_pdos(ec_pdo_list_t *pl /**< PDO list. */)
+{
+    ec_pdo_t *pdo, *next;
+
+    list_for_each_entry_safe(pdo, next, &pl->list, list) {
+        list_del_init(&pdo->list);
+        ec_pdo_clear(pdo);
+        kfree(pdo);
+    }
+}
+
+/****************************************************************************/
+
+/** Calculates the total size of the mapped PDO entries.
+ *
+ * \retval Data size in byte.
+ */
+uint16_t ec_pdo_list_total_size(
+        const ec_pdo_list_t *pl /**< PDO list. */
+        )
+{
+    unsigned int bit_size;
+    const ec_pdo_t *pdo;
+    const ec_pdo_entry_t *pdo_entry;
+    uint16_t byte_size;
+
+    bit_size = 0;
+    list_for_each_entry(pdo, &pl->list, list) {
+        list_for_each_entry(pdo_entry, &pdo->entries, list) {
+            bit_size += pdo_entry->bit_length;
+        }
+    }
+
+    if (bit_size % 8) // round up to full bytes
+        byte_size = bit_size / 8 + 1;
+    else
+        byte_size = bit_size / 8;
+
+    return byte_size;
+}
+
+/****************************************************************************/
+
+/** Add a new PDO to the list.
+ *
+ * \return Pointer to new PDO, otherwise an ERR_PTR() code.
+ */
+ec_pdo_t *ec_pdo_list_add_pdo(
+        ec_pdo_list_t *pl, /**< PDO list. */
+        uint16_t index /**< PDO index. */
+        )
+{
+    ec_pdo_t *pdo;
+
+    if (!(pdo = (ec_pdo_t *) kmalloc(sizeof(ec_pdo_t), GFP_KERNEL))) {
+        EC_ERR("Failed to allocate memory for PDO.\n");
+        return ERR_PTR(-ENOMEM);
+    }
+
+    ec_pdo_init(pdo);
+    pdo->index = index;
+    list_add_tail(&pdo->list, &pl->list);
+    return pdo;
+}
+
+/****************************************************************************/
+
+/** Add the copy of an existing PDO to the list.
+ *
+ * \return 0 on success, else < 0
+ */
+int ec_pdo_list_add_pdo_copy(
+        ec_pdo_list_t *pl, /**< PDO list. */
+        const ec_pdo_t *pdo /**< PDO to add. */
+        )
+{
+    ec_pdo_t *mapped_pdo;
+    int ret;
+
+    // PDO already mapped?
+    list_for_each_entry(mapped_pdo, &pl->list, list) {
+        if (mapped_pdo->index != pdo->index) continue;
+        EC_ERR("PDO 0x%04X is already mapped!\n", pdo->index);
+        return -EEXIST;
+    }
+
+    if (!(mapped_pdo = kmalloc(sizeof(ec_pdo_t), GFP_KERNEL))) {
+        EC_ERR("Failed to allocate PDO memory.\n");
+        return -ENOMEM;
+    }
+
+    ret = ec_pdo_init_copy(mapped_pdo, pdo);
+    if (ret < 0) {
+        kfree(mapped_pdo);
+        return ret;
+    }
+
+    list_add_tail(&mapped_pdo->list, &pl->list);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Makes a deep copy of another PDO list.
+ *
+ * \return 0 on success, else < 0
+ */
+int ec_pdo_list_copy(
+        ec_pdo_list_t *pl, /**< PDO list. */
+        const ec_pdo_list_t *other /**< PDO list to copy from. */
+        )
+{
+    ec_pdo_t *other_pdo;
+    int ret;
+
+    ec_pdo_list_clear_pdos(pl);
+
+    // PDO already mapped?
+    list_for_each_entry(other_pdo, &other->list, list) {
+        ret = ec_pdo_list_add_pdo_copy(pl, other_pdo);
+        if (ret)
+            return ret;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Compares two PDO lists.
+ *
+ * Only the list is compared, not the PDO entries (i. e. the PDO
+ * mapping).
+ *
+ * \retval 1 The given PDO lists are equal.
+ * \retval 0 The given PDO lists differ.
+ */
+int ec_pdo_list_equal(
+        const ec_pdo_list_t *pl1, /**< First list. */
+        const ec_pdo_list_t *pl2 /**< Second list. */
+        )
+{
+    const struct list_head *h1, *h2, *l1, *l2;
+    const ec_pdo_t *p1, *p2;
+
+    h1 = l1 = &pl1->list;
+    h2 = l2 = &pl2->list;
+
+    while (1) {
+        l1 = l1->next;
+        l2 = l2->next;
+
+        if ((l1 == h1) ^ (l2 == h2)) // unequal lengths
+            return 0;
+        if (l1 == h1) // both finished
+            break;
+
+        p1 = list_entry(l1, ec_pdo_t, list);
+        p2 = list_entry(l2, ec_pdo_t, list);
+
+        if (p1->index != p2->index)
+            return 0;
+    }
+
+    return 1;
+}
+
+/****************************************************************************/
+
+/** Finds a PDO with the given index.
+ *
+ * \return Search result, or NULL.
+ */
+ec_pdo_t *ec_pdo_list_find_pdo(
+        const ec_pdo_list_t *pl, /**< PDO list. */
+        uint16_t index /**< PDO index. */
+        )
+{
+    ec_pdo_t *pdo;
+
+    list_for_each_entry(pdo, &pl->list, list) {
+        if (pdo->index != index)
+            continue;
+        return pdo;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Finds a PDO with the given index and returns a const pointer.
+ *
+ * \return Search result, or NULL.
+ */
+const ec_pdo_t *ec_pdo_list_find_pdo_const(
+        const ec_pdo_list_t *pl, /**< PDO list. */
+        uint16_t index /**< PDO index. */
+        )
+{
+    const ec_pdo_t *pdo;
+
+    list_for_each_entry(pdo, &pl->list, list) {
+        if (pdo->index != index)
+            continue;
+        return pdo;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Finds a PDO via its position in the list.
+ *
+ * Const version.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+const ec_pdo_t *ec_pdo_list_find_pdo_by_pos_const(
+        const ec_pdo_list_t *pl, /**< PDO list. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    const ec_pdo_t *pdo;
+
+    list_for_each_entry(pdo, &pl->list, list) {
+        if (pos--)
+            continue;
+        return pdo;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Get the number of PDOs in the list.
+ *
+ * \return Number of PDOs.
+ */
+unsigned int ec_pdo_list_count(
+        const ec_pdo_list_t *pl /**< PDO list. */
+        )
+{
+    const ec_pdo_t *pdo;
+    unsigned int num = 0;
+
+    list_for_each_entry(pdo, &pl->list, list) {
+        num++;
+    }
+
+    return num;
+}
+
+/****************************************************************************/
+
+/** Outputs the PDOs in the list.
+ */
+void ec_pdo_list_print(
+        const ec_pdo_list_t *pl /**< PDO list. */
+        )
+{
+    const ec_pdo_t *pdo;
+
+    if (list_empty(&pl->list)) {
+        printk(KERN_CONT "(none)");
+    } else {
+        list_for_each_entry(pdo, &pl->list, list) {
+            printk(KERN_CONT "0x%04X", pdo->index);
+            if (pdo->list.next != &pl->list)
+                printk(KERN_CONT " ");
+        }
+    }
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/pdo_list.h b/drivers/net/ethercat/master/pdo_list.h
new file mode 100644
index 000000000000..5bd9be7f9a90
--- /dev/null
+++ b/drivers/net/ethercat/master/pdo_list.h
@@ -0,0 +1,71 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT PDO list structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_PDO_LIST_H__
+#define __EC_PDO_LIST_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+#include "pdo.h"
+
+/****************************************************************************/
+
+/** EtherCAT PDO list.
+ */
+typedef struct {
+    struct list_head list; /**< List of PDOs. */
+} ec_pdo_list_t;
+
+/****************************************************************************/
+
+void ec_pdo_list_init(ec_pdo_list_t *);
+void ec_pdo_list_clear(ec_pdo_list_t *);
+
+void ec_pdo_list_clear_pdos(ec_pdo_list_t *);
+
+ec_pdo_t *ec_pdo_list_add_pdo(ec_pdo_list_t *, uint16_t);
+int ec_pdo_list_add_pdo_copy(ec_pdo_list_t *, const ec_pdo_t *);
+
+int ec_pdo_list_copy(ec_pdo_list_t *, const ec_pdo_list_t *);
+
+uint16_t ec_pdo_list_total_size(const ec_pdo_list_t *);
+int ec_pdo_list_equal(const ec_pdo_list_t *, const ec_pdo_list_t *);
+
+ec_pdo_t *ec_pdo_list_find_pdo(const ec_pdo_list_t *, uint16_t);
+const ec_pdo_t *ec_pdo_list_find_pdo_const(const ec_pdo_list_t *,
+        uint16_t);
+const ec_pdo_t *ec_pdo_list_find_pdo_by_pos_const(
+        const ec_pdo_list_t *, unsigned int);
+unsigned int ec_pdo_list_count(const ec_pdo_list_t *);
+
+void ec_pdo_list_print(const ec_pdo_list_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/reg_request.c b/drivers/net/ethercat/master/reg_request.c
new file mode 100644
index 000000000000..eab70b1d140b
--- /dev/null
+++ b/drivers/net/ethercat/master/reg_request.c
@@ -0,0 +1,125 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * Register request functions.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/jiffies.h>
+#include <linux/slab.h>
+
+#include "reg_request.h"
+
+/****************************************************************************/
+
+/** Register request constructor.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_reg_request_init(
+        ec_reg_request_t *reg, /**< Register request. */
+        size_t size /**< Memory size. */
+        )
+{
+    if (!(reg->data = (uint8_t *) kmalloc(size, GFP_KERNEL))) {
+        EC_ERR("Failed to allocate %zu bytes of register memory.\n", size);
+        return -ENOMEM;
+    }
+
+    INIT_LIST_HEAD(&reg->list);
+    reg->mem_size = size;
+    memset(reg->data, 0x00, size);
+    reg->dir = EC_DIR_INVALID;
+    reg->address = 0;
+    reg->transfer_size = 0;
+    reg->state = EC_INT_REQUEST_INIT;
+    reg->ring_position = 0;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Register request destructor.
+ */
+void ec_reg_request_clear(
+        ec_reg_request_t *reg /**< Register request. */
+        )
+{
+    if (reg->data) {
+        kfree(reg->data);
+    }
+}
+
+/*****************************************************************************
+ * Application interface.
+ ****************************************************************************/
+
+uint8_t *ecrt_reg_request_data(const ec_reg_request_t *reg)
+{
+    return reg->data;
+}
+
+/****************************************************************************/
+
+ec_request_state_t ecrt_reg_request_state(const ec_reg_request_t *reg)
+{
+   return ec_request_state_translation_table[reg->state];
+}
+
+/****************************************************************************/
+
+int ecrt_reg_request_write(ec_reg_request_t *reg, uint16_t address,
+        size_t size)
+{
+    reg->dir = EC_DIR_OUTPUT;
+    reg->address = address;
+    reg->transfer_size = min(size, reg->mem_size);
+    reg->state = EC_INT_REQUEST_QUEUED;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_reg_request_read(ec_reg_request_t *reg, uint16_t address,
+        size_t size)
+{
+    reg->dir = EC_DIR_INPUT;
+    reg->address = address;
+    reg->transfer_size = min(size, reg->mem_size);
+    reg->state = EC_INT_REQUEST_QUEUED;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecrt_reg_request_data);
+EXPORT_SYMBOL(ecrt_reg_request_state);
+EXPORT_SYMBOL(ecrt_reg_request_write);
+EXPORT_SYMBOL(ecrt_reg_request_read);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/reg_request.h b/drivers/net/ethercat/master/reg_request.h
new file mode 100644
index 000000000000..86876b647916
--- /dev/null
+++ b/drivers/net/ethercat/master/reg_request.h
@@ -0,0 +1,59 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT register request structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_REG_REQUEST_H__
+#define __EC_REG_REQUEST_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** Register request.
+ */
+struct ec_reg_request {
+    struct list_head list; /**< List item. */
+    size_t mem_size; /**< Size of data memory. */
+    uint8_t *data; /**< Pointer to data memory. */
+    ec_direction_t dir; /**< Direction. EC_DIR_OUTPUT means writing to the
+                          slave, EC_DIR_INPUT means reading from the slave. */
+    uint16_t address; /**< Register address. */
+    size_t transfer_size; /**< Size of the data to transfer. */
+    ec_internal_request_state_t state; /**< Request state. */
+    uint16_t ring_position; /**< Ring position for emergency requests. */
+};
+
+/****************************************************************************/
+
+int ec_reg_request_init(ec_reg_request_t *, size_t);
+void ec_reg_request_clear(ec_reg_request_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/rt_locks.h b/drivers/net/ethercat/master/rt_locks.h
new file mode 100644
index 000000000000..bfe592171f29
--- /dev/null
+++ b/drivers/net/ethercat/master/rt_locks.h
@@ -0,0 +1,75 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Abstract locks for realtime part of interface.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_LOCKS_H__
+#define __EC_LOCKS_H__
+
+#include "globals.h"
+#include <linux/version.h>
+
+#include <linux/semaphore.h>
+
+/****************************************************************************/
+
+#ifdef EC_USE_RTMUTEX
+
+#include <linux/rtmutex.h>
+
+typedef struct rt_mutex ec_lock_t;
+
+static inline void ec_lock_init(ec_lock_t *sem) { rt_mutex_init(sem); }
+static inline void ec_lock_down(ec_lock_t *sem) { rt_mutex_lock(sem); }
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 34)
+static inline int ec_lock_down_interruptible(ec_lock_t *sem) {
+    return rt_mutex_lock_interruptible(sem);
+}
+#else
+static inline int ec_lock_down_interruptible(ec_lock_t *sem) {
+    return rt_mutex_lock_interruptible(sem, 1);
+}
+#endif
+static inline void ec_lock_up(ec_lock_t *sem) { rt_mutex_unlock(sem); }
+
+#else
+
+typedef struct semaphore ec_lock_t;
+
+static inline void ec_lock_init(ec_lock_t *sem) { sema_init(sem, 1); }
+static inline void ec_lock_down(ec_lock_t *sem) { down(sem); }
+static inline int ec_lock_down_interruptible(ec_lock_t *sem) {
+    return down_interruptible(sem);
+}
+static inline void ec_lock_up(ec_lock_t *sem) { up(sem); }
+
+#endif
+
+/****************************************************************************/
+
+#endif
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/rtdm-ioctl.c b/drivers/net/ethercat/master/rtdm-ioctl.c
new file mode 100644
index 000000000000..c590d5d24d94
--- /dev/null
+++ b/drivers/net/ethercat/master/rtdm-ioctl.c
@@ -0,0 +1,5591 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+/**
+   \file
+   EtherCAT master character device.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+
+#include "master.h"
+#include "slave_config.h"
+#include "voe_handler.h"
+#include "ethernet.h"
+#include "ioctl.h"
+
+/** Set to 1 to enable ioctl() latency tracing.
+ *
+ * Requires CPU timestamp counter!
+ */
+#define DEBUG_LATENCY 0
+
+/** Optional compiler attributes fo ioctl() functions.
+ */
+#if 0
+#define ATTRIBUTES __attribute__ ((__noinline__))
+#else
+#define ATTRIBUTES
+#endif
+
+#ifdef EC_IOCTL_RTDM
+# include "rtdm_details.h"
+/* RTDM does not support locking yet,
+ * therefore no send/receive callbacks are set too. */
+# define ec_ioctl_lock(lock) do {} while(0)
+# define ec_ioctl_unlock(lock) do {} while(0)
+# define ec_ioctl_lock_interruptible(lock) (0)
+# define ec_copy_to_user(to, from, n, ctx) \
+    rtdm_safe_copy_to_user(ec_ioctl_to_rtdm(ctx), to, from, n)
+# define ec_copy_from_user(to, from, n, ctx) \
+    rtdm_safe_copy_from_user(ec_ioctl_to_rtdm(ctx), to, from, n)
+#else
+# define ec_ioctl_lock(lock)   rt_mutex_lock(lock)
+# define ec_ioctl_unlock(lock) rt_mutex_unlock(lock)
+#  if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) || \
+      (defined(CONFIG_PREEMPT_RT_FULL) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+#   define ec_ioctl_lock_interruptible(lock) \
+           rt_mutex_lock_interruptible(lock)
+#  else
+#   define ec_ioctl_lock_interruptible(lock) \
+           rt_mutex_lock_interruptible(lock, 0)
+# endif
+# define ec_copy_to_user(to, from, n, ctx) copy_to_user(to, from, n)
+# define ec_copy_from_user(to, from, n, ctx) copy_from_user(to, from, n)
+#endif  // EC_IOCTL_RTDM
+
+/****************************************************************************/
+
+/** Copies a string to an ioctl structure.
+ */
+static void ec_ioctl_strcpy(
+        char *target, /**< Target. */
+        const char *source /**< Source. */
+        )
+{
+    if (source) {
+        strncpy(target, source, EC_IOCTL_STRING_SIZE);
+        target[EC_IOCTL_STRING_SIZE - 1] = 0;
+    } else {
+        target[0] = 0;
+    }
+}
+
+/****************************************************************************/
+
+/** Get module information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_module(
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_module_t data;
+
+    data.ioctl_version_magic = EC_IOCTL_VERSION_MAGIC;
+    data.master_count = ec_master_count();
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get master information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_master_t io;
+    unsigned int dev_idx, j;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    io.slave_count = master->slave_count;
+    io.scan_index = master->scan_index;
+    io.config_count = ec_master_config_count(master);
+    io.domain_count = ec_master_domain_count(master);
+#ifdef EC_EOE
+    io.eoe_handler_count = ec_master_eoe_handler_count(master);
+#else
+    io.eoe_handler_count = 0;
+#endif
+    io.phase = (uint8_t) master->phase;
+    io.active = (uint8_t) master->active;
+    io.scan_busy = master->scan_busy;
+
+    up(&master->master_sem);
+
+    if (down_interruptible(&master->device_sem)) {
+        return -EINTR;
+    }
+
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(master); dev_idx++) {
+        ec_device_t *device = &master->devices[dev_idx];
+
+        if (device->dev) {
+            memcpy(io.devices[dev_idx].address, device->dev->dev_addr,
+                    ETH_ALEN);
+        } else {
+            memcpy(io.devices[dev_idx].address, master->macs[dev_idx],
+                    ETH_ALEN);
+        }
+        io.devices[dev_idx].attached = device->dev ? 1 : 0;
+        io.devices[dev_idx].link_state = device->link_state ? 1 : 0;
+        io.devices[dev_idx].tx_count = device->tx_count;
+        io.devices[dev_idx].rx_count = device->rx_count;
+        io.devices[dev_idx].tx_bytes = device->tx_bytes;
+        io.devices[dev_idx].rx_bytes = device->rx_bytes;
+        io.devices[dev_idx].tx_errors = device->tx_errors;
+        for (j = 0; j < EC_RATE_COUNT; j++) {
+            io.devices[dev_idx].tx_frame_rates[j] =
+                device->tx_frame_rates[j];
+            io.devices[dev_idx].rx_frame_rates[j] =
+                device->rx_frame_rates[j];
+            io.devices[dev_idx].tx_byte_rates[j] =
+                device->tx_byte_rates[j];
+            io.devices[dev_idx].rx_byte_rates[j] =
+                device->rx_byte_rates[j];
+        }
+    }
+    io.num_devices = ec_master_num_devices(master);
+
+    io.tx_count = master->device_stats.tx_count;
+    io.rx_count = master->device_stats.rx_count;
+    io.tx_bytes = master->device_stats.tx_bytes;
+    io.rx_bytes = master->device_stats.rx_bytes;
+    for (j = 0; j < EC_RATE_COUNT; j++) {
+        io.tx_frame_rates[j] =
+            master->device_stats.tx_frame_rates[j];
+        io.rx_frame_rates[j] =
+            master->device_stats.rx_frame_rates[j];
+        io.tx_byte_rates[j] =
+            master->device_stats.tx_byte_rates[j];
+        io.rx_byte_rates[j] =
+            master->device_stats.rx_byte_rates[j];
+        io.loss_rates[j] =
+            master->device_stats.loss_rates[j];
+    }
+
+    up(&master->device_sem);
+
+    io.app_time = master->app_time;
+    io.dc_ref_time = master->dc_ref_time;
+    io.ref_clock =
+        master->dc_ref_clock ? master->dc_ref_clock->ring_position : 0xffff;
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_t data;
+    const ec_slave_t *slave;
+    int i;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n", data.position);
+        return -EINVAL;
+    }
+
+    data.device_index = slave->device_index;
+    data.vendor_id = slave->sii.vendor_id;
+    data.product_code = slave->sii.product_code;
+    data.revision_number = slave->sii.revision_number;
+    data.serial_number = slave->sii.serial_number;
+    data.alias = slave->effective_alias;
+    data.boot_rx_mailbox_offset = slave->sii.boot_rx_mailbox_offset;
+    data.boot_rx_mailbox_size = slave->sii.boot_rx_mailbox_size;
+    data.boot_tx_mailbox_offset = slave->sii.boot_tx_mailbox_offset;
+    data.boot_tx_mailbox_size = slave->sii.boot_tx_mailbox_size;
+    data.std_rx_mailbox_offset = slave->sii.std_rx_mailbox_offset;
+    data.std_rx_mailbox_size = slave->sii.std_rx_mailbox_size;
+    data.std_tx_mailbox_offset = slave->sii.std_tx_mailbox_offset;
+    data.std_tx_mailbox_size = slave->sii.std_tx_mailbox_size;
+    data.mailbox_protocols = slave->sii.mailbox_protocols;
+    data.has_general_category = slave->sii.has_general;
+    data.coe_details = slave->sii.coe_details;
+    data.general_flags = slave->sii.general_flags;
+    data.current_on_ebus = slave->sii.current_on_ebus;
+    for (i = 0; i < EC_MAX_PORTS; i++) {
+        data.ports[i].desc = slave->ports[i].desc;
+        data.ports[i].link.link_up = slave->ports[i].link.link_up;
+        data.ports[i].link.loop_closed = slave->ports[i].link.loop_closed;
+        data.ports[i].link.signal_detected =
+            slave->ports[i].link.signal_detected;
+        data.ports[i].receive_time = slave->ports[i].receive_time;
+        if (slave->ports[i].next_slave) {
+            data.ports[i].next_slave =
+                slave->ports[i].next_slave->ring_position;
+        } else {
+            data.ports[i].next_slave = 0xffff;
+        }
+        data.ports[i].delay_to_next_dc = slave->ports[i].delay_to_next_dc;
+    }
+    data.fmmu_bit = slave->base_fmmu_bit_operation;
+    data.dc_supported = slave->base_dc_supported;
+    data.dc_range = slave->base_dc_range;
+    data.has_dc_system_time = slave->has_dc_system_time;
+    data.transmission_delay = slave->transmission_delay;
+    data.al_state = slave->current_state;
+    data.error_flag = slave->error_flag;
+
+    data.sync_count = slave->sii.sync_count;
+    data.sdo_count = ec_slave_sdo_count(slave);
+    data.sii_nwords = slave->sii_nwords;
+    ec_ioctl_strcpy(data.group, slave->sii.group);
+    ec_ioctl_strcpy(data.image, slave->sii.image);
+    ec_ioctl_strcpy(data.order, slave->sii.order);
+    ec_ioctl_strcpy(data.name, slave->sii.name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave sync manager information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sync(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_sync_t data;
+    const ec_slave_t *slave;
+    const ec_sync_t *sync;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sync_index >= slave->sii.sync_count) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not exist!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    sync = &slave->sii.syncs[data.sync_index];
+
+    data.physical_start_address = sync->physical_start_address;
+    data.default_size = sync->default_length;
+    data.control_register = sync->control_register;
+    data.enable = sync->enable;
+    data.pdo_count = ec_pdo_list_count(&sync->pdos);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave sync manager PDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sync_pdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_sync_pdo_t data;
+    const ec_slave_t *slave;
+    const ec_sync_t *sync;
+    const ec_pdo_t *pdo;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sync_index >= slave->sii.sync_count) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not exist!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    sync = &slave->sii.syncs[data.sync_index];
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sync->pdos, data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not contain a PDO with "
+                "position %u!\n", data.sync_index, data.pdo_pos);
+        return -EINVAL;
+    }
+
+    data.index = pdo->index;
+    data.entry_count = ec_pdo_entry_count(pdo);
+    ec_ioctl_strcpy(data.name, pdo->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave sync manager PDO entry information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sync_pdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_slave_sync_pdo_entry_t data;
+    const ec_slave_t *slave;
+    const ec_sync_t *sync;
+    const ec_pdo_t *pdo;
+    const ec_pdo_entry_t *entry;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sync_index >= slave->sii.sync_count) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not exist!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    sync = &slave->sii.syncs[data.sync_index];
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sync->pdos, data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Sync manager %u does not contain a PDO with "
+                "position %u!\n", data.sync_index, data.pdo_pos);
+        return -EINVAL;
+    }
+
+    if (!(entry = ec_pdo_find_entry_by_pos_const(
+                    pdo, data.entry_pos))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "PDO 0x%04X does not contain an entry with "
+                "position %u!\n", data.pdo_pos, data.entry_pos);
+        return -EINVAL;
+    }
+
+    data.index = entry->index;
+    data.subindex = entry->subindex;
+    data.bit_length = entry->bit_length;
+    ec_ioctl_strcpy(data.name, entry->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get domain information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_domain_t data;
+    const ec_domain_t *domain;
+    unsigned int dev_idx;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(domain = ec_master_find_domain_const(master, data.index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u does not exist!\n", data.index);
+        return -EINVAL;
+    }
+
+    data.data_size = domain->data_size;
+    data.logical_base_address = domain->logical_base_address;
+    for (dev_idx = EC_DEVICE_MAIN;
+            dev_idx < ec_master_num_devices(domain->master); dev_idx++) {
+        data.working_counter[dev_idx] = domain->working_counter[dev_idx];
+    }
+    data.expected_working_counter = domain->expected_working_counter;
+    data.fmmu_count = ec_domain_fmmu_count(domain);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get domain FMMU information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_fmmu(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_domain_fmmu_t data;
+    const ec_domain_t *domain;
+    const ec_fmmu_config_t *fmmu;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(domain = ec_master_find_domain_const(master, data.domain_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u does not exist!\n",
+                data.domain_index);
+        return -EINVAL;
+    }
+
+    if (!(fmmu = ec_domain_find_fmmu(domain, data.fmmu_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u has less than %u"
+                " fmmu configurations.\n",
+                data.domain_index, data.fmmu_index + 1);
+        return -EINVAL;
+    }
+
+    data.slave_config_alias = fmmu->sc->alias;
+    data.slave_config_position = fmmu->sc->position;
+    data.sync_index = fmmu->sync_index;
+    data.dir = fmmu->dir;
+    data.logical_address = fmmu->logical_start_address;
+    data.data_size = fmmu->data_size;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get domain data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< Userspace address to store the results. */
+        )
+{
+    ec_ioctl_domain_data_t data;
+    const ec_domain_t *domain;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(domain = ec_master_find_domain_const(master, data.domain_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Domain %u does not exist!\n",
+                data.domain_index);
+        return -EINVAL;
+    }
+
+    if (domain->data_size != data.data_size) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Data size mismatch %u/%zu!\n",
+                data.data_size, domain->data_size);
+        return -EFAULT;
+    }
+
+    if (copy_to_user((void __user *) data.target, domain->data,
+                domain->data_size)) {
+        up(&master->master_sem);
+        return -EFAULT;
+    }
+
+    up(&master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Set master debug level.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master_debug(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    return ec_master_debug_level(master, (unsigned long) arg);
+}
+
+/****************************************************************************/
+
+/** Issue a bus scan.
+ *
+ * \return Always zero (success).
+ */
+static ATTRIBUTES int ec_ioctl_master_rescan(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    EC_MASTER_DBG(master, 1, "Got rescan command via ioctl()."
+		    " Re-scanning on next possibility.\n");
+    master->fsm.rescan_required = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Set slave state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_state_t data;
+    ec_slave_t *slave;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    ec_slave_request_state(slave, data.al_state);
+
+    up(&master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave SDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_t data;
+    const ec_slave_t *slave;
+    const ec_sdo_t *sdo;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (!(sdo = ec_slave_get_sdo_by_pos_const(
+                    slave, data.sdo_position))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "SDO %u does not exist!\n", data.sdo_position);
+        return -EINVAL;
+    }
+
+    data.sdo_index = sdo->index;
+    data.max_subindex = sdo->max_subindex;
+    ec_ioctl_strcpy(data.name, sdo->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave SDO entry information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_entry_t data;
+    const ec_slave_t *slave;
+    const ec_sdo_t *sdo;
+    const ec_sdo_entry_t *entry;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (data.sdo_spec <= 0) {
+        if (!(sdo = ec_slave_get_sdo_by_pos_const(
+                        slave, -data.sdo_spec))) {
+            up(&master->master_sem);
+            EC_SLAVE_ERR(slave, "SDO %u does not exist!\n", -data.sdo_spec);
+            return -EINVAL;
+        }
+    } else {
+        if (!(sdo = ec_slave_get_sdo_const(
+                        slave, data.sdo_spec))) {
+            up(&master->master_sem);
+            EC_SLAVE_ERR(slave, "SDO 0x%04X does not exist!\n",
+                    data.sdo_spec);
+            return -EINVAL;
+        }
+    }
+
+    if (!(entry = ec_sdo_get_entry_const(
+                    sdo, data.sdo_entry_subindex))) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "SDO entry 0x%04X:%02X does not exist!\n",
+                sdo->index, data.sdo_entry_subindex);
+        return -EINVAL;
+    }
+
+    data.data_type = entry->data_type;
+    data.bit_length = entry->bit_length;
+    data.read_access[EC_SDO_ENTRY_ACCESS_PREOP] =
+        entry->read_access[EC_SDO_ENTRY_ACCESS_PREOP];
+    data.read_access[EC_SDO_ENTRY_ACCESS_SAFEOP] =
+        entry->read_access[EC_SDO_ENTRY_ACCESS_SAFEOP];
+    data.read_access[EC_SDO_ENTRY_ACCESS_OP] =
+        entry->read_access[EC_SDO_ENTRY_ACCESS_OP];
+    data.write_access[EC_SDO_ENTRY_ACCESS_PREOP] =
+        entry->write_access[EC_SDO_ENTRY_ACCESS_PREOP];
+    data.write_access[EC_SDO_ENTRY_ACCESS_SAFEOP] =
+        entry->write_access[EC_SDO_ENTRY_ACCESS_SAFEOP];
+    data.write_access[EC_SDO_ENTRY_ACCESS_OP] =
+        entry->write_access[EC_SDO_ENTRY_ACCESS_OP];
+    ec_ioctl_strcpy(data.description, entry->description);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Upload SDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo_upload(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_upload_t data;
+    uint8_t *target;
+    int ret;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (!(target = kmalloc(data.target_size, GFP_KERNEL))) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes"
+                " for SDO upload.\n", data.target_size);
+        return -ENOMEM;
+    }
+
+    ret = ecrt_master_sdo_upload(master, data.slave_position,
+            data.sdo_index, data.sdo_entry_subindex, target,
+            data.target_size, &data.data_size, &data.abort_code);
+
+    if (!ret) {
+        if (copy_to_user((void __user *) data.target,
+                    target, data.data_size)) {
+            kfree(target);
+            return -EFAULT;
+        }
+    }
+
+    kfree(target);
+
+    if (__copy_to_user((void __user *) arg, &data, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Download SDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sdo_download(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sdo_download_t data;
+    uint8_t *sdo_data;
+    int retval;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (!(sdo_data = kmalloc(data.data_size, GFP_KERNEL))) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes"
+                " for SDO download.\n", data.data_size);
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(sdo_data, (void __user *) data.data, data.data_size)) {
+        kfree(sdo_data);
+        return -EFAULT;
+    }
+
+    if (data.complete_access) {
+        retval = ecrt_master_sdo_download_complete(master, data.slave_position,
+                data.sdo_index, sdo_data, data.data_size, &data.abort_code);
+    } else {
+        retval = ecrt_master_sdo_download(master, data.slave_position,
+                data.sdo_index, data.sdo_entry_subindex, sdo_data,
+                data.data_size, &data.abort_code);
+    }
+
+    kfree(sdo_data);
+
+    if (__copy_to_user((void __user *) arg, &data, sizeof(data))) {
+        retval = -EFAULT;
+    }
+
+    return retval;
+}
+
+/****************************************************************************/
+
+/** Read a slave's SII.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sii_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sii_t data;
+    const ec_slave_t *slave;
+    int retval;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(slave = ec_master_find_slave_const(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        return -EINVAL;
+    }
+
+    if (!data.nwords
+            || data.offset + data.nwords > slave->sii_nwords) {
+        up(&master->master_sem);
+        EC_SLAVE_ERR(slave, "Invalid SII read offset/size %u/%u for slave SII"
+                " size %zu!\n", data.offset, data.nwords, slave->sii_nwords);
+        return -EINVAL;
+    }
+
+    if (copy_to_user((void __user *) data.words,
+                slave->sii_words + data.offset, data.nwords * 2))
+        retval = -EFAULT;
+    else
+        retval = 0;
+
+    up(&master->master_sem);
+    return retval;
+}
+
+/****************************************************************************/
+
+/** Write a slave's SII.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_sii_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_sii_t data;
+    ec_slave_t *slave;
+    unsigned int byte_size;
+    uint16_t *words;
+    ec_sii_write_request_t request;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (!data.nwords) {
+        return 0;
+    }
+
+    byte_size = sizeof(uint16_t) * data.nwords;
+    if (!(words = kmalloc(byte_size, GFP_KERNEL))) {
+        EC_MASTER_ERR(master, "Failed to allocate %u bytes"
+                " for SII contents.\n", byte_size);
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(words,
+                (void __user *) data.words, byte_size)) {
+        kfree(words);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(words);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, data.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                data.slave_position);
+        kfree(words);
+        return -EINVAL;
+    }
+
+    // init SII write request
+    INIT_LIST_HEAD(&request.list);
+    request.slave = slave;
+    request.words = words;
+    request.offset = data.offset;
+    request.nwords = data.nwords;
+    request.state = EC_INT_REQUEST_QUEUED;
+
+    // schedule SII write request.
+    list_add_tail(&request.list, &master->sii_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            kfree(words);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    kfree(words);
+
+    return request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+
+/****************************************************************************/
+
+/** Read a slave's registers.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_reg_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_reg_t io;
+    ec_slave_t *slave;
+    ec_reg_request_t request;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (!io.size) {
+        return 0;
+    }
+
+    // init register request
+    ret = ec_reg_request_init(&request, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    ret = ecrt_reg_request_read(&request, io.address, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_reg_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        ec_reg_request_clear(&request);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        return -EINVAL;
+    }
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->reg_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_reg_request_clear(&request);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    if (request.state == EC_INT_REQUEST_SUCCESS) {
+        if (copy_to_user((void __user *) io.data, request.data, io.size)) {
+            return -EFAULT;
+        }
+    }
+    ec_reg_request_clear(&request);
+
+    return request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+
+/****************************************************************************/
+
+/** Write a slave's registers.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_reg_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_reg_t io;
+    ec_slave_t *slave;
+    ec_reg_request_t request;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (!io.size) {
+        return 0;
+    }
+
+    // init register request
+    ret = ec_reg_request_init(&request, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    if (copy_from_user(request.data, (void __user *) io.data, io.size)) {
+        ec_reg_request_clear(&request);
+        return -EFAULT;
+    }
+
+    ret = ecrt_reg_request_write(&request, io.address, io.size);
+    if (ret) {
+        return ret;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_reg_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (io.emergency) {
+        request.ring_position = io.slave_position;
+        // schedule request.
+        list_add_tail(&request.list, &master->emerg_reg_requests);
+    }
+    else {
+        if (!(slave = ec_master_find_slave(master, 0, io.slave_position))) {
+            up(&master->master_sem);
+            ec_reg_request_clear(&request);
+            EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                    io.slave_position);
+            return -EINVAL;
+        }
+
+        // schedule request.
+        list_add_tail(&request.list, &slave->reg_requests);
+    }
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_reg_request_clear(&request);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    ec_reg_request_clear(&request);
+
+    return request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_t data;
+    const ec_slave_config_t *sc;
+    uint8_t i;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config_const(
+                    master, data.config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                data.config_index);
+        return -EINVAL;
+    }
+
+    data.alias = sc->alias;
+    data.position = sc->position;
+    data.vendor_id = sc->vendor_id;
+    data.product_code = sc->product_code;
+    for (i = 0; i < EC_MAX_SYNC_MANAGERS; i++) {
+        data.syncs[i].dir = sc->sync_configs[i].dir;
+        data.syncs[i].watchdog_mode = sc->sync_configs[i].watchdog_mode;
+        data.syncs[i].pdo_count =
+            ec_pdo_list_count(&sc->sync_configs[i].pdos);
+    }
+    data.watchdog_divider = sc->watchdog_divider;
+    data.watchdog_intervals = sc->watchdog_intervals;
+    data.sdo_count = ec_slave_config_sdo_count(sc);
+    data.idn_count = ec_slave_config_idn_count(sc);
+    data.flag_count = ec_slave_config_flag_count(sc);
+    data.slave_position = sc->slave ? sc->slave->ring_position : -1;
+    data.dc_assign_activate = sc->dc_assign_activate;
+    for (i = 0; i < EC_SYNC_SIGNAL_COUNT; i++) {
+        data.dc_sync[i] = sc->dc_sync[i];
+    }
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration PDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_pdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    const ec_slave_config_t *sc;
+    const ec_pdo_t *pdo;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (data.sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_MASTER_ERR(master, "Invalid sync manager index %u!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config_const(
+                    master, data.config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                data.config_index);
+        return -EINVAL;
+    }
+
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sc->sync_configs[data.sync_index].pdos,
+                    data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid PDO position!\n");
+        return -EINVAL;
+    }
+
+    data.index = pdo->index;
+    data.entry_count = ec_pdo_entry_count(pdo);
+    ec_ioctl_strcpy(data.name, pdo->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration PDO entry information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_pdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_pdo_entry_t data;
+    const ec_slave_config_t *sc;
+    const ec_pdo_t *pdo;
+    const ec_pdo_entry_t *entry;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (data.sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_MASTER_ERR(master, "Invalid sync manager index %u!\n",
+                data.sync_index);
+        return -EINVAL;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config_const(
+                    master, data.config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                data.config_index);
+        return -EINVAL;
+    }
+
+    if (!(pdo = ec_pdo_list_find_pdo_by_pos_const(
+                    &sc->sync_configs[data.sync_index].pdos,
+                    data.pdo_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid PDO position!\n");
+        return -EINVAL;
+    }
+
+    if (!(entry = ec_pdo_find_entry_by_pos_const(
+                    pdo, data.entry_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Entry not found!\n");
+        return -EINVAL;
+    }
+
+    data.index = entry->index;
+    data.subindex = entry->subindex;
+    data.bit_length = entry->bit_length;
+    ec_ioctl_strcpy(data.name, entry->name);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration SDO information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_sdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_sdo_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_sdo_request_t *req;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(
+                    master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    if (!(req = ec_slave_config_get_sdo_by_pos_const(
+                    sc, ioctl->sdo_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid SDO position!\n");
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    ioctl->index = req->index;
+    ioctl->subindex = req->subindex;
+    ioctl->size = req->data_size;
+    memcpy(ioctl->data, req->data,
+            min((u32) ioctl->size, (u32) EC_MAX_SDO_DATA_SIZE));
+    ioctl->complete_access = req->complete_access;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration IDN information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_idn(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_idn_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_soe_request_t *req;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(
+                    master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    if (!(req = ec_slave_config_get_idn_by_pos_const(
+                    sc, ioctl->idn_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid IDN position!\n");
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    ioctl->drive_no = req->drive_no;
+    ioctl->idn = req->idn;
+    ioctl->state = req->al_state;
+    ioctl->size = req->data_size;
+    memcpy(ioctl->data, req->data,
+            min((u32) ioctl->size, (u32) EC_MAX_IDN_DATA_SIZE));
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get slave configuration feature flag information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_flag(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_config_flag_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_flag_t *flag;
+    size_t size;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(
+                    master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    if (!(flag = ec_slave_config_get_flag_by_pos_const(
+                    sc, ioctl->flag_pos))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Invalid flag position!\n");
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    size = min((u32) strlen(flag->key), (u32) EC_MAX_FLAG_KEY_SIZE - 1);
+    memcpy(ioctl->key, flag->key, size);
+    ioctl->key[size] = 0x00;
+    ioctl->value = flag->value;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/** Get configured EoE IP parameters for a given slave configuration.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_config_ip(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_eoe_ip_t *ioctl;
+    const ec_slave_config_t *sc;
+    const ec_eoe_request_t *req;
+
+    if (!(ioctl = kmalloc(sizeof(*ioctl), GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(ioctl, (void __user *) arg, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(ioctl);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config_const(master, ioctl->config_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave config %u does not exist!\n",
+                ioctl->config_index);
+        kfree(ioctl);
+        return -EINVAL;
+    }
+
+    req = &sc->eoe_ip_param_request;
+
+    ioctl->mac_address_included = req->mac_address_included;
+    ioctl->ip_address_included = req->ip_address_included;
+    ioctl->subnet_mask_included = req->subnet_mask_included;
+    ioctl->gateway_included = req->gateway_included;
+    ioctl->dns_included = req->dns_included;
+    ioctl->name_included = req->name_included;
+
+    memcpy(ioctl->mac_address, req->mac_address, EC_ETH_ALEN);
+    ioctl->ip_address = req->ip_address;
+    ioctl->subnet_mask = req->subnet_mask;
+    ioctl->gateway = req->gateway;
+    ioctl->dns = req->dns;
+    strncpy(ioctl->name, req->name, EC_MAX_HOSTNAME_SIZE);
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, ioctl, sizeof(*ioctl))) {
+        kfree(ioctl);
+        return -EFAULT;
+    }
+
+    kfree(ioctl);
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/** Get EoE handler information.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_eoe_handler(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_eoe_handler_t data;
+    const ec_eoe_t *eoe;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(eoe = ec_master_get_eoe_handler_const(master, data.eoe_index))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "EoE handler %u does not exist!\n",
+                data.eoe_index);
+        return -EINVAL;
+    }
+
+    if (eoe->slave) {
+        data.slave_position = eoe->slave->ring_position;
+    } else {
+        data.slave_position = 0xffff;
+    }
+    snprintf(data.name, EC_DATAGRAM_NAME_SIZE, eoe->dev->name);
+    data.open = eoe->opened;
+    data.rx_bytes = eoe->stats.tx_bytes;
+    data.rx_rate = eoe->tx_rate;
+    data.tx_bytes = eoe->stats.rx_bytes;
+    data.tx_rate = eoe->tx_rate;
+    data.tx_queued_frames = eoe->tx_queued_frames;
+    data.tx_queue_size = eoe->tx_queue_size;
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+/** Request EoE IP parameter setting.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_eoe_ip_param(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_eoe_ip_t io;
+    ec_eoe_request_t req;
+    ec_slave_t *slave;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    // init EoE request
+    ec_eoe_request_init(&req);
+
+    req.mac_address_included = io.mac_address_included;
+    req.ip_address_included = io.ip_address_included;
+    req.subnet_mask_included = io.subnet_mask_included;
+    req.gateway_included = io.gateway_included;
+    req.dns_included = io.dns_included;
+    req.name_included = io.name_included;
+
+    memcpy(req.mac_address, io.mac_address, EC_ETH_ALEN);
+    req.ip_address = io.ip_address;
+    req.subnet_mask = io.subnet_mask;
+    req.gateway = io.gateway;
+    req.dns = io.dns;
+    memcpy(req.name, io.name, EC_MAX_HOSTNAME_SIZE);
+
+    req.state = EC_INT_REQUEST_QUEUED;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(
+                    master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        return -EINVAL;
+    }
+
+    EC_MASTER_DBG(master, 1, "Scheduling EoE request.\n");
+
+    // schedule request.
+    list_add_tail(&req.list, &slave->eoe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                req.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (req.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&req.list);
+            up(&master->master_sem);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, req.state != EC_INT_REQUEST_BUSY);
+
+    io.result = req.result;
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    return req.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+}
+#endif
+
+/*****************************************************************************/
+
+/** Request the master from userspace.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_master_t *m;
+    int ret = 0;
+
+    m = ecrt_request_master_err(master->index);
+    if (IS_ERR(m)) {
+        ret = PTR_ERR(m);
+    } else {
+        ctx->requested = 1;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Create a domain.
+ *
+ * \return Domain index on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_create_domain(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    domain = ecrt_master_create_domain_err(master);
+    if (IS_ERR(domain))
+        return PTR_ERR(domain);
+
+    return domain->index;
+}
+
+/****************************************************************************/
+
+/** Create a slave configuration.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_create_slave_config(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc, *entry;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    sc = ecrt_master_slave_config_err(master, data.alias, data.position,
+            data.vendor_id, data.product_code);
+    if (IS_ERR(sc))
+        return PTR_ERR(sc);
+
+    data.config_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    list_for_each_entry(entry, &master->configs, list) {
+        if (entry == sc)
+            break;
+        data.config_index++;
+    }
+
+    up(&master->master_sem);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Select the DC reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_select_ref_clock(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    unsigned long config_index = (unsigned long) arg;
+    ec_slave_config_t *sc = NULL;
+    int ret = 0;
+
+    if (unlikely(!ctx->requested)) {
+        ret = -EPERM;
+        goto out_return;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ret = -EINTR;
+        goto out_return;
+    }
+
+    if (config_index != 0xFFFFFFFF) {
+        if (!(sc = ec_master_get_config(master, config_index))) {
+            ret = -ENOENT;
+            goto out_up;
+        }
+    }
+
+    ecrt_master_select_reference_clock(master, sc);
+
+out_up:
+    up(&master->master_sem);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Activates the master.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_activate(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_master_activate_t io;
+    ec_domain_t *domain;
+    off_t offset;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    io.process_data = NULL;
+
+    /* Get the sum of the domains' process data sizes. */
+
+    ctx->process_data_size = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    list_for_each_entry(domain, &master->domains, list) {
+        ctx->process_data_size += ecrt_domain_size(domain);
+    }
+
+    up(&master->master_sem);
+
+    if (ctx->process_data_size) {
+        ctx->process_data = vmalloc(ctx->process_data_size);
+        if (!ctx->process_data) {
+            ctx->process_data_size = 0;
+            return -ENOMEM;
+        }
+
+        /* Set the memory as external process data memory for the
+         * domains.
+         */
+        offset = 0;
+        list_for_each_entry(domain, &master->domains, list) {
+            ecrt_domain_external_memory(domain,
+                    ctx->process_data + offset);
+            offset += ecrt_domain_size(domain);
+        }
+
+#if defined(EC_IOCTL_RTDM) && !defined(EC_RTDM_XENOMAI_V3)
+        /* RTDM uses a different approach for memory-mapping, which has to be
+         * initiated by the kernel.
+         */
+        ret = ec_rtdm_mmap(ctx, &io.process_data);
+        if (ret < 0) {
+            EC_MASTER_ERR(master, "Failed to map process data"
+                    " memory to user space (code %i).\n", ret);
+            return ret;
+        }
+#endif
+    }
+
+    io.process_data_size = ctx->process_data_size;
+
+#ifndef EC_IOCTL_RTDM
+    /* RTDM does not support locking yet. */
+    ecrt_master_callbacks(master, ec_master_internal_send_cb,
+            ec_master_internal_receive_cb, master);
+#endif
+
+    ret = ecrt_master_activate(master);
+    if (ret < 0)
+        return ret;
+
+    if (copy_to_user((void __user *) arg, &io,
+                sizeof(ec_ioctl_master_activate_t)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Deactivates the master.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_deactivate(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    return ecrt_master_deactivate(master);
+}
+
+/****************************************************************************/
+
+/** Set max. number of databytes in a cycle
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_set_send_interval(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    size_t send_interval;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&send_interval, (void __user *) arg,
+                sizeof(send_interval))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    ec_master_set_send_interval(master, send_interval);
+
+    up(&master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Send frames.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_send(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_send(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Receive frames.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_receive(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_receive(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get the master state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_master_state_t data;
+    int ret;
+
+    ret = ecrt_master_state(master, &data);
+    if (ret)
+        return ret;
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get the link state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_master_link_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_link_state_t ioctl;
+    ec_master_link_state_t state;
+    int ret;
+
+    if (ec_copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl), ctx)) {
+        return -EFAULT;
+    }
+
+    ret = ecrt_master_link_state(master, ioctl.dev_idx, &state);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ec_copy_to_user((void __user *) ioctl.state,
+                        &state, sizeof(state), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Set the master DC application time.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_app_time(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    uint64_t time;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&time, (void __user *) arg, sizeof(time), ctx)) {
+        return -EFAULT;
+    }
+
+    return ecrt_master_application_time(master, time);
+}
+
+/****************************************************************************/
+
+/** Sync the reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_ref(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_reference_clock(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sync the reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_ref_to(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+    uint64_t time;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&time, (void __user *) arg, sizeof(time), ctx)) {
+        return -EFAULT;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_reference_clock_to(master, time);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sync the slave clocks.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_slaves(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_slave_clocks(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get the system time of the reference clock.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_ref_clock_time(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    uint32_t time;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    ret = ecrt_master_reference_clock_time(master, &time);
+    if (ret) {
+        return ret;
+    }
+
+    if (ec_copy_to_user((void __user *) arg, &time, sizeof(time), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Queue the sync monitoring datagram.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_mon_queue(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_master_sync_monitor_queue(master);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Processes the sync monitoring datagram.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sync_mon_process(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    uint32_t time_diff;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    time_diff = ecrt_master_sync_monitor_process(master);
+
+    if (ec_copy_to_user((void __user *) arg, &time_diff,
+                        sizeof(time_diff), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Reset configuration.
+ *
+ * \return Always zero (success).
+ */
+static ATTRIBUTES int ec_ioctl_reset(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+#ifdef EC_IOCTL_RTDM
+    /* Xenomai/LXRT is like NMI context, so we do a two-stage schedule. */
+    irq_work_queue(&master->sc_reset_work_kicker);
+#else
+    schedule_work(&master->sc_reset_work);
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Configure a sync manager.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_sync(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc;
+    unsigned int i;
+    int ret = 0;
+
+    if (unlikely(!ctx->requested)) {
+        ret = -EPERM;
+        goto out_return;
+    }
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        ret = -EFAULT;
+        goto out_return;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ret = -EINTR;
+        goto out_return;
+    }
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        ret = -ENOENT;
+        goto out_up;
+    }
+
+    for (i = 0; i < EC_MAX_SYNC_MANAGERS; i++) {
+        if (data.syncs[i].config_this) {
+            ret = ecrt_slave_config_sync_manager(sc, i, data.syncs[i].dir,
+                        data.syncs[i].watchdog_mode);
+            if (ret) {
+                goto out_up;
+            }
+        }
+    }
+
+out_up:
+    up(&master->master_sem);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Configure a slave's watchdogs.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_watchdog(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc;
+    int ret = 0;
+
+    if (unlikely(!ctx->requested)) {
+        ret = -EPERM;
+        goto out_return;
+    }
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        ret = -EFAULT;
+        goto out_return;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        ret = -EINTR;
+        goto out_return;
+    }
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        ret = -ENOENT;
+        goto out_up;
+    }
+
+    ret = ecrt_slave_config_watchdog(sc,
+            data.watchdog_divider, data.watchdog_intervals);
+
+out_up:
+    up(&master->master_sem);
+out_return:
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Add a PDO to the assignment.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_add_pdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_assign_add(sc, data.sync_index, data.index);
+}
+
+/****************************************************************************/
+
+/** Clears the PDO assignment.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_clear_pdos(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_assign_clear(sc, data.sync_index);
+}
+
+/****************************************************************************/
+
+/** Add an entry to a PDO's mapping.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_add_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_add_pdo_entry_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_mapping_add(sc, data.pdo_index,
+            data.entry_index, data.entry_subindex, data.entry_bit_length);
+}
+
+/****************************************************************************/
+
+/** Clears the mapping of a PDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_clear_entries(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_pdo_t data;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    return ecrt_slave_config_pdo_mapping_clear(sc, data.index);
+}
+
+/****************************************************************************/
+
+/** Registers a PDO entry.
+ *
+ * \return Process data offset on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_reg_pdo_entry(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_pdo_entry_t data;
+    ec_slave_config_t *sc;
+    ec_domain_t *domain;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    if (!(domain = ec_master_find_domain(master, data.domain_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc or domain could be invalidated */
+
+    ret = ecrt_slave_config_reg_pdo_entry(sc, data.entry_index,
+            data.entry_subindex, domain, &data.bit_position);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Registers a PDO entry by its position.
+ *
+ * \return Process data offset on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_reg_pdo_pos(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_pdo_pos_t io;
+    ec_slave_config_t *sc;
+    ec_domain_t *domain;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    if (!(domain = ec_master_find_domain(master, io.domain_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc or domain could be invalidated */
+
+    ret = ecrt_slave_config_reg_pdo_entry_pos(sc, io.sync_index,
+            io.pdo_pos, io.entry_pos, domain, &io.bit_position);
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io)))
+        return -EFAULT;
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sets the DC AssignActivate word and the sync signal times.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_dc(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_config_t data;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_dc(sc, data.dc_assign_activate,
+            data.dc_sync[0].cycle_time,
+            data.dc_sync[0].shift_time,
+            data.dc_sync[1].cycle_time,
+            data.dc_sync[1].shift_time);
+
+    up(&master->master_sem);
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Configures an SDO.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_sdo(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_sdo_t data;
+    ec_slave_config_t *sc;
+    uint8_t *sdo_data = NULL;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data)))
+        return -EFAULT;
+
+    if (!data.size)
+        return -EINVAL;
+
+    if (!(sdo_data = kmalloc(data.size, GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(sdo_data, (void __user *) data.data, data.size)) {
+        kfree(sdo_data);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(sdo_data);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        up(&master->master_sem);
+        kfree(sdo_data);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    if (data.complete_access) {
+        ret = ecrt_slave_config_complete_sdo(sc,
+                data.index, sdo_data, data.size);
+    } else {
+        ret = ecrt_slave_config_sdo(sc, data.index, data.subindex, sdo_data,
+                data.size);
+    }
+    kfree(sdo_data);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Set the emergency ring buffer size.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_size(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io)))
+        return -EFAULT;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_emerg_size(sc, io.size);
+
+    up(&master->master_sem);
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get an emergency message from the ring.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_pop(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+    u8 msg[EC_COE_EMERGENCY_MSG_SIZE];
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because configuration will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_emerg_pop(sc, msg);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ec_copy_to_user((void __user *) io.target, msg, sizeof(msg), ctx)) {
+        return -EFAULT;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Clear the emergency ring.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_clear(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because configuration will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_slave_config_emerg_clear(sc);
+}
+
+/****************************************************************************/
+
+/** Get the number of emergency overruns.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_emerg_overruns(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_emerg_t io;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because configuration will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_emerg_overruns(sc);
+    if (ret < 0) {
+        return ret;
+    }
+
+    io.overruns = ret;
+
+    if (ec_copy_to_user((void __user *) arg, &io, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create an SDO request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_sdo_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    data.request_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    sc = ec_master_get_config(master, data.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(req, &sc->sdo_requests, list) {
+        data.request_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    req = ecrt_slave_config_create_sdo_request_err(sc, data.sdo_index,
+            data.sdo_subindex, data.size);
+    if (IS_ERR(req))
+        return PTR_ERR(req);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create an SoE request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_soe_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    data.request_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    sc = ec_master_get_config(master, data.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(req, &sc->soe_requests, list) {
+        data.request_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    req = ecrt_slave_config_create_soe_request_err(sc, data.drive_no,
+            data.idn, data.size);
+    if (IS_ERR(req)) {
+        return PTR_ERR(req);
+    }
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create a register request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_reg_request(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    io.request_index = 0;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    sc = ec_master_get_config(master, io.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(reg, &sc->reg_requests, list) {
+        io.request_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    reg = ecrt_slave_config_create_reg_request_err(sc, io.mem_size);
+    if (IS_ERR(reg)) {
+        return PTR_ERR(reg);
+    }
+
+    if (copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Create a VoE handler.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_create_voe_handler(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&data, (void __user *) arg, sizeof(data))) {
+        return -EFAULT;
+    }
+
+    data.voe_index = 0;
+
+    if (down_interruptible(&master->master_sem))
+        return -EINTR;
+
+    sc = ec_master_get_config(master, data.config_index);
+    if (!sc) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    list_for_each_entry(voe, &sc->voe_handlers, list) {
+        data.voe_index++;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    voe = ecrt_slave_config_create_voe_handler_err(sc, data.size);
+    if (IS_ERR(voe))
+        return PTR_ERR(voe);
+
+    if (copy_to_user((void __user *) arg, &data, sizeof(data)))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Get the slave configuration's state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_state_t data;
+    const ec_slave_config_t *sc;
+    ec_slave_config_state_t state;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because sc will not be deleted in the
+     * meantime. */
+
+    if (!(sc = ec_master_get_config_const(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_slave_config_state(sc, &state);
+    if (ret)
+        return ret;
+
+    if (ec_copy_to_user((void __user *) data.state,
+                        &state, sizeof(state), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Configures an IDN.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_idn(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_idn_t ioctl;
+    ec_slave_config_t *sc;
+    uint8_t *data = NULL;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl)))
+        return -EFAULT;
+
+    if (!ioctl.size)
+        return -EINVAL;
+
+    if (!(data = kmalloc(ioctl.size, GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(data, (void __user *) ioctl.data, ioctl.size)) {
+        kfree(data);
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(data);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, ioctl.config_index))) {
+        up(&master->master_sem);
+        kfree(data);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    ret = ecrt_slave_config_idn(
+            sc, ioctl.drive_no, ioctl.idn, ioctl.al_state, data, ioctl.size);
+    kfree(data);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Configures a feature flag.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_flag(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_flag_t ioctl;
+    ec_slave_config_t *sc;
+    uint8_t *key;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    if (!ioctl.key_size) {
+        return -EINVAL;
+    }
+
+    if (!(key = kmalloc(ioctl.key_size + 1, GFP_KERNEL))) {
+        return -ENOMEM;
+    }
+
+    if (copy_from_user(key, (void __user *) ioctl.key, ioctl.key_size)) {
+        kfree(key);
+        return -EFAULT;
+    }
+    key[ioctl.key_size] = '\0';
+
+    if (down_interruptible(&master->master_sem)) {
+        kfree(key);
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, ioctl.config_index))) {
+        up(&master->master_sem);
+        kfree(key);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    ret = ecrt_slave_config_flag(sc, key, ioctl.value);
+    kfree(key);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Sets an AL state transition timeout.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_state_timeout(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sc_state_timeout_t ioctl;
+    ec_slave_config_t *sc;
+    int ret;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, ioctl.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    ret = ecrt_slave_config_state_timeout(sc, ioctl.from_state,
+            ioctl.to_state, ioctl.timeout_ms);
+    return ret;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+/** Configures EoE IP parameters.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sc_ip(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_eoe_ip_t io;
+    ec_slave_config_t *sc;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        up(&master->master_sem);
+        return -ENOENT;
+    }
+
+    up(&master->master_sem); /** \todo sc could be invalidated */
+
+    /* the kernel versions of the EoE set IP methods never fail. */
+    if (io.mac_address_included) {
+        ecrt_slave_config_eoe_mac_address(sc, io.mac_address);
+    }
+    if (io.ip_address_included) {
+        ecrt_slave_config_eoe_ip_address(sc, io.ip_address);
+    }
+    if (io.subnet_mask_included) {
+        ecrt_slave_config_eoe_subnet_mask(sc, io.subnet_mask);
+    }
+    if (io.gateway_included) {
+        ecrt_slave_config_eoe_default_gateway(sc, io.gateway);
+    }
+    if (io.dns_included) {
+        ecrt_slave_config_eoe_dns_address(sc, io.dns);
+    }
+    if (io.name_included) {
+        ecrt_slave_config_eoe_hostname(sc, io.name);
+    }
+
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+/** Gets the domain's data size.
+ *
+ * \return Domain size, or a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_size(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    const ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    list_for_each_entry(domain, &master->domains, list) {
+        if (domain->index == (unsigned long) arg) {
+            size_t size = ecrt_domain_size(domain);
+            up(&master->master_sem);
+            return size;
+        }
+    }
+
+    up(&master->master_sem);
+    return -ENOENT;
+}
+
+/****************************************************************************/
+
+/** Gets the domain's offset in the total process data.
+ *
+ * \return Domain offset, or a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_offset(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    int offset = 0;
+    const ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (down_interruptible(&master->master_sem)) {
+        return -EINTR;
+    }
+
+    list_for_each_entry(domain, &master->domains, list) {
+        if (domain->index == (unsigned long) arg) {
+            up(&master->master_sem);
+            return offset;
+        }
+        offset += ecrt_domain_size(domain);
+    }
+
+    up(&master->master_sem);
+    return -ENOENT;
+}
+
+/****************************************************************************/
+
+/** Process the domain.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_process(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_domain_t *domain;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    /* no locking of master_sem needed, because domain will not be deleted in
+     * the meantime. */
+
+    if (!(domain = ec_master_find_domain(master, (unsigned long) arg))) {
+        return -ENOENT;
+    }
+
+    return ecrt_domain_process(domain);
+}
+
+/****************************************************************************/
+
+/** Queue the domain.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_queue(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_domain_t *domain;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    /* no locking of master_sem needed, because domain will not be deleted in
+     * the meantime. */
+
+    if (!(domain = ec_master_find_domain(master, (unsigned long) arg))) {
+        return -ENOENT;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    ret = ecrt_domain_queue(domain);
+    ec_ioctl_unlock(&master->io_mutex);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Get the domain state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_domain_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_domain_state_t data;
+    const ec_domain_t *domain;
+    ec_domain_state_t state;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because domain will not be deleted in
+     * the meantime. */
+
+    if (!(domain = ec_master_find_domain_const(master, data.domain_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_domain_state(domain, &state);
+    if (ret)
+        return ret;
+
+    if (ec_copy_to_user((void __user *) data.state, &state, sizeof(state),
+                ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Sets an SDO request's SDO index and subindex.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_index(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_sdo_request_index(req, data.sdo_index, data.sdo_subindex);
+}
+
+/****************************************************************************/
+
+/** Sets an SDO request's timeout.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_timeout(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_sdo_request_timeout(req, data.timeout);
+}
+
+/****************************************************************************/
+
+/** Gets an SDO request's state.
+ *
+ * Also pre-fetches the size of incoming data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    data.state = ecrt_sdo_request_state(req);
+    if (data.state == EC_REQUEST_SUCCESS && req->dir == EC_DIR_INPUT)
+        data.size = ecrt_sdo_request_data_size(req);
+    else
+        data.size = 0;
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts an SDO read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_sdo_request_read(req);
+}
+
+/****************************************************************************/
+
+/** Starts an SDO write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    if (!data.size) {
+        EC_MASTER_ERR(master, "SDO download: Data size may not be zero!\n");
+        return -EINVAL;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (data.size > req->mem_size)
+        return -ENOBUFS;
+
+    if (ec_copy_from_user(req->data, (void __user *) data.data,
+                          data.size, ctx))
+        return -EFAULT;
+
+    req->data_size = data.size;
+    return ecrt_sdo_request_write(req);
+}
+
+/****************************************************************************/
+
+/** Read SDO data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_sdo_request_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_sdo_request_t data;
+    ec_slave_config_t *sc;
+    ec_sdo_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_sdo_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) data.data, ecrt_sdo_request_data(req),
+                ecrt_sdo_request_data_size(req), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Sets an SoE request's drive number and IDN.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_index(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_soe_request_idn(req, data.drive_no, data.idn);
+}
+
+/****************************************************************************/
+
+/** Sets an CoE request's timeout.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_timeout(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_soe_request_timeout(req, data.timeout);
+}
+
+/****************************************************************************/
+
+/** Gets an SoE request's state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    data.state = ecrt_soe_request_state(req);
+    if (data.state == EC_REQUEST_SUCCESS && req->dir == EC_DIR_INPUT) {
+        data.size = ecrt_soe_request_data_size(req);
+    }
+    else {
+        data.size = 0;
+    }
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts an SoE IDN read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_soe_request_read(req);
+}
+
+/****************************************************************************/
+
+/** Starts an SoE IDN write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    if (!data.size) {
+        EC_MASTER_ERR(master, "IDN write: Data size may not be zero!\n");
+        return -EINVAL;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (data.size > req->mem_size)
+        return -ENOBUFS;
+
+    if (ec_copy_from_user(req->data, (void __user *) data.data,
+                          data.size, ctx))
+        return -EFAULT;
+
+    req->data_size = data.size;
+    return ecrt_soe_request_write(req);
+}
+
+/****************************************************************************/
+
+/** Read SoE IDN data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_soe_request_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_soe_request_t data;
+    ec_slave_config_t *sc;
+    ec_soe_request_t *req;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor req will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(req = ec_slave_config_find_soe_request(sc, data.request_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) data.data, ecrt_soe_request_data(req),
+                ecrt_soe_request_data_size(req), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Read register data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    if (io.mem_size <= 0) {
+        return 0;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) io.data, ecrt_reg_request_data(reg),
+                min(reg->mem_size, io.mem_size), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Gets an register request's state.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_state(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    io.state = ecrt_reg_request_state(reg);
+    io.new_data = io.state == EC_REQUEST_SUCCESS && reg->dir == EC_DIR_INPUT;
+
+    if (ec_copy_to_user((void __user *) arg, &io, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts an register write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    if (io.transfer_size > reg->mem_size) {
+        return -ENOBUFS;
+    }
+
+    if (ec_copy_from_user(reg->data, (void __user *) io.data,
+                io.transfer_size, ctx)) {
+        return -EFAULT;
+    }
+
+    return ecrt_reg_request_write(reg, io.address, io.transfer_size);
+}
+
+/****************************************************************************/
+
+/** Starts an register read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_reg_request_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_reg_request_t io;
+    ec_slave_config_t *sc;
+    ec_reg_request_t *reg;
+
+    if (unlikely(!ctx->requested)) {
+        return -EPERM;
+    }
+
+    if (ec_copy_from_user(&io, (void __user *) arg, sizeof(io), ctx)) {
+        return -EFAULT;
+    }
+
+    /* no locking of master_sem needed, because neither sc nor reg will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, io.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(reg = ec_slave_config_find_reg_request(sc, io.request_index))) {
+        return -ENOENT;
+    }
+
+    if (io.transfer_size > reg->mem_size) {
+        return -ENOBUFS;
+    }
+
+    return ecrt_reg_request_read(reg, io.address, io.transfer_size);
+}
+
+/****************************************************************************/
+
+/** Sets the VoE send header.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_send_header(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+    uint32_t vendor_id;
+    uint16_t vendor_type;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    if (ec_copy_from_user(&vendor_id, data.vendor_id, sizeof(vendor_id), ctx))
+        return -EFAULT;
+
+    if (ec_copy_from_user(&vendor_type, data.vendor_type,
+                          sizeof(vendor_type), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_voe_handler_send_header(voe, vendor_id, vendor_type);
+}
+
+/****************************************************************************/
+
+/** Gets the received VoE header.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_rec_header(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+    uint32_t vendor_id;
+    uint16_t vendor_type;
+    int ret;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    ret = ecrt_voe_handler_received_header(voe, &vendor_id, &vendor_type);
+    if (ret)
+        return ret;
+
+    if (likely(data.vendor_id))
+        if (ec_copy_to_user(data.vendor_id, &vendor_id,
+                            sizeof(vendor_id), ctx))
+            return -EFAULT;
+
+    if (likely(data.vendor_type))
+        if (ec_copy_to_user(data.vendor_type, &vendor_type,
+            sizeof(vendor_type), ctx))
+            return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Starts a VoE read operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_voe_handler_read(voe);
+}
+
+/****************************************************************************/
+
+/** Starts a VoE read operation without sending a sync message first.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_read_nosync(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    return ecrt_voe_handler_read_nosync(voe);
+}
+
+/****************************************************************************/
+
+/** Starts a VoE write operation.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    if (data.size) {
+        if (data.size > ec_voe_handler_mem_size(voe))
+            return -ENOBUFS;
+
+        if (ec_copy_from_user(ecrt_voe_handler_data(voe),
+                    (void __user *) data.data, data.size, ctx))
+            return -EFAULT;
+    }
+
+    return ecrt_voe_handler_write(voe, data.size);
+}
+
+/****************************************************************************/
+
+/** Executes the VoE state machine.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_exec(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_ioctl_lock_interruptible(&master->io_mutex))
+        return -EINTR;
+
+    data.state = ecrt_voe_handler_execute(voe);
+    ec_ioctl_unlock(&master->io_mutex);
+    if (data.state == EC_REQUEST_SUCCESS && voe->dir == EC_DIR_INPUT)
+        data.size = ecrt_voe_handler_data_size(voe);
+    else
+        data.size = 0;
+
+    if (ec_copy_to_user((void __user *) arg, &data, sizeof(data), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Reads the received VoE data.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_voe_data(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg, /**< ioctl() argument. */
+        ec_ioctl_context_t *ctx /**< Private data structure of file handle. */
+        )
+{
+    ec_ioctl_voe_t data;
+    ec_slave_config_t *sc;
+    ec_voe_handler_t *voe;
+
+    if (unlikely(!ctx->requested))
+        return -EPERM;
+
+    if (ec_copy_from_user(&data, (void __user *) arg, sizeof(data), ctx))
+        return -EFAULT;
+
+    /* no locking of master_sem needed, because neither sc nor voe will not be
+     * deleted in the meantime. */
+
+    if (!(sc = ec_master_get_config(master, data.config_index))) {
+        return -ENOENT;
+    }
+
+    if (!(voe = ec_slave_config_find_voe_handler(sc, data.voe_index))) {
+        return -ENOENT;
+    }
+
+    if (ec_copy_to_user((void __user *) data.data, ecrt_voe_handler_data(voe),
+                ecrt_voe_handler_data_size(voe), ctx))
+        return -EFAULT;
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Read a file from a slave via FoE.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_foe_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_foe_t io;
+    ec_foe_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    ec_foe_request_init(&request, io.file_name);
+    ret = ec_foe_request_alloc(&request, 10000); // FIXME
+    if (ret) {
+        ec_foe_request_clear(&request);
+        return ret;
+    }
+
+    ec_foe_request_read(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_foe_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        ec_foe_request_clear(&request);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling FoE read request.\n");
+
+    // schedule request.
+    list_add_tail(&request.list, &slave->foe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_foe_request_clear(&request);
+            return -EINTR;
+        }
+        // request already processing: interrupt not possible.
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    io.result = request.result;
+    io.error_code = request.error_code;
+
+    if (request.state != EC_INT_REQUEST_SUCCESS) {
+        io.data_size = 0;
+        ret = -EIO;
+    } else {
+        if (request.data_size > io.buffer_size) {
+            EC_SLAVE_ERR(slave, "%s(): Buffer too small.\n", __func__);
+            ec_foe_request_clear(&request);
+            return -ENOBUFS;
+        }
+        io.data_size = request.data_size;
+        if (copy_to_user((void __user *) io.buffer,
+                    request.buffer, io.data_size)) {
+            ec_foe_request_clear(&request);
+            return -EFAULT;
+        }
+        ret = 0;
+    }
+
+    if (__copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        ret = -EFAULT;
+    }
+
+    ec_foe_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Write a file to a slave via FoE
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_foe_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_foe_t io;
+    ec_foe_request_t request;
+    ec_slave_t *slave;
+    int ret;
+
+    if (copy_from_user(&io, (void __user *) arg, sizeof(io))) {
+        return -EFAULT;
+    }
+
+    ec_foe_request_init(&request, io.file_name);
+
+    ret = ec_foe_request_alloc(&request, io.buffer_size);
+    if (ret) {
+        ec_foe_request_clear(&request);
+        return ret;
+    }
+
+    if (copy_from_user(request.buffer,
+                (void __user *) io.buffer, io.buffer_size)) {
+        ec_foe_request_clear(&request);
+        return -EFAULT;
+    }
+
+    request.data_size = io.buffer_size;
+    ec_foe_request_write(&request);
+
+    if (down_interruptible(&master->master_sem)) {
+        ec_foe_request_clear(&request);
+        return -EINTR;
+    }
+
+    if (!(slave = ec_master_find_slave(master, 0, io.slave_position))) {
+        up(&master->master_sem);
+        EC_MASTER_ERR(master, "Slave %u does not exist!\n",
+                io.slave_position);
+        ec_foe_request_clear(&request);
+        return -EINVAL;
+    }
+
+    EC_SLAVE_DBG(slave, 1, "Scheduling FoE write request.\n");
+
+    // schedule FoE write request.
+    list_add_tail(&request.list, &slave->foe_requests);
+
+    up(&master->master_sem);
+
+    // wait for processing through FSM
+    if (wait_event_interruptible(master->request_queue,
+                request.state != EC_INT_REQUEST_QUEUED)) {
+        // interrupted by signal
+        down(&master->master_sem);
+        if (request.state == EC_INT_REQUEST_QUEUED) {
+            // abort request
+            list_del(&request.list);
+            up(&master->master_sem);
+            ec_foe_request_clear(&request);
+            return -EINTR;
+        }
+        up(&master->master_sem);
+    }
+
+    // wait until master FSM has finished processing
+    wait_event(master->request_queue, request.state != EC_INT_REQUEST_BUSY);
+
+    io.result = request.result;
+    io.error_code = request.error_code;
+
+    ret = request.state == EC_INT_REQUEST_SUCCESS ? 0 : -EIO;
+
+    if (__copy_to_user((void __user *) arg, &io, sizeof(io))) {
+        ret = -EFAULT;
+    }
+
+    ec_foe_request_clear(&request);
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Read an SoE IDN.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_soe_read(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_soe_read_t ioctl;
+    u8 *data;
+    int retval;
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    data = kmalloc(ioctl.mem_size, GFP_KERNEL);
+    if (!data) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes of IDN data.\n",
+                ioctl.mem_size);
+        return -ENOMEM;
+    }
+
+    retval = ecrt_master_read_idn(master, ioctl.slave_position,
+            ioctl.drive_no, ioctl.idn, data, ioctl.mem_size, &ioctl.data_size,
+            &ioctl.error_code);
+    if (retval) {
+        kfree(data);
+        return retval;
+    }
+
+    if (copy_to_user((void __user *) ioctl.data,
+                data, ioctl.data_size)) {
+        kfree(data);
+        return -EFAULT;
+    }
+    kfree(data);
+
+    if (__copy_to_user((void __user *) arg, &ioctl, sizeof(ioctl))) {
+        retval = -EFAULT;
+    }
+
+    EC_MASTER_DBG(master, 1, "Finished SoE read request.\n");
+    return retval;
+}
+
+/****************************************************************************/
+
+/** Write an IDN to a slave via SoE.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+static ATTRIBUTES int ec_ioctl_slave_soe_write(
+        ec_master_t *master, /**< EtherCAT master. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    ec_ioctl_slave_soe_write_t ioctl;
+    u8 *data;
+    int retval;
+
+    if (copy_from_user(&ioctl, (void __user *) arg, sizeof(ioctl))) {
+        return -EFAULT;
+    }
+
+    data = kmalloc(ioctl.data_size, GFP_KERNEL);
+    if (!data) {
+        EC_MASTER_ERR(master, "Failed to allocate %zu bytes of IDN data.\n",
+                ioctl.data_size);
+        return -ENOMEM;
+    }
+    if (copy_from_user(data, (void __user *) ioctl.data, ioctl.data_size)) {
+        kfree(data);
+        return -EFAULT;
+    }
+
+    retval = ecrt_master_write_idn(master, ioctl.slave_position,
+            ioctl.drive_no, ioctl.idn, data, ioctl.data_size,
+            &ioctl.error_code);
+    kfree(data);
+    if (retval) {
+        return retval;
+    }
+
+    if (__copy_to_user((void __user *) arg, &ioctl, sizeof(ioctl))) {
+        retval = -EFAULT;
+    }
+
+    EC_MASTER_DBG(master, 1, "Finished SoE write request.\n");
+    return retval;
+}
+
+/*****************************************************************************
+ * ioctl() file operation functions
+ ****************************************************************************/
+
+/** ioctl() function to use.
+ *
+ * For RTDM, there will be ec_ioctl_rtdm_rt and ec_ioctl_rtdm_nrt.
+ * For "normal" cdev, there will be ec_ioctl_rt only.
+ */
+#ifndef EC_IOCTL_RTDM
+static long ec_ioctl_nrt(
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */);
+#endif
+
+/****************************************************************************/
+
+/** Called when an ioctl() command is issued.
+ * Both RT and nRT context.
+ *
+ * \return ioctl() return code.
+ */
+static long ec_ioctl_both(
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+    int ret;
+
+    switch (cmd) {
+        case EC_IOCTL_MODULE:
+            ret = ec_ioctl_module(arg, ctx);
+            break;
+        case EC_IOCTL_MASTER_RESCAN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_master_rescan(master, arg);
+            break;
+        case EC_IOCTL_MASTER_STATE:
+            ret = ec_ioctl_master_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_MASTER_LINK_STATE:
+            ret = ec_ioctl_master_link_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_TIMEOUT:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_timeout(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_DATA:
+            ret = ec_ioctl_sdo_request_data(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_TIMEOUT:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_timeout(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_DATA:
+            ret = ec_ioctl_soe_request_data(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_DATA:
+            ret = ec_ioctl_reg_request_data(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_SEND_HEADER:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_send_header(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_DATA:
+            ret = ec_ioctl_voe_data(master, arg, ctx);
+            break;
+        default:
+#ifdef EC_IOCTL_RTDM
+            ret = -ENOTTY;
+#else
+            /* chain non-rt commands for normal cdev */
+            ret = ec_ioctl_nrt(master, ctx, cmd, arg);
+#endif
+            break;
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Called when an ioctl() command is issued.
+ * RTDM: RT only.
+ *
+ * \return ioctl() return code.
+ */
+#ifdef EC_IOCTL_RTDM
+long ec_ioctl_rtdm_rt
+#else
+long ec_ioctl
+#endif
+        (
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+#if DEBUG_LATENCY
+    cycles_t a = get_cycles(), b;
+    unsigned int t;
+#endif
+    long ret;
+
+    switch (cmd) {
+        case EC_IOCTL_SEND:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_send(master, arg, ctx);
+            break;
+        case EC_IOCTL_RECEIVE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_receive(master, arg, ctx);
+            break;
+        case EC_IOCTL_APP_TIME:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_app_time(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_REF:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_ref(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_REF_TO:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_ref_to(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_SLAVES:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_slaves(master, arg, ctx);
+            break;
+        case EC_IOCTL_REF_CLOCK_TIME:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_ref_clock_time(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_MON_QUEUE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_mon_queue(master, arg, ctx);
+            break;
+        case EC_IOCTL_SYNC_MON_PROCESS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sync_mon_process(master, arg, ctx);
+            break;
+        case EC_IOCTL_RESET:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_reset(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_POP:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_emerg_pop(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_CLEAR:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_emerg_clear(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_OVERRUNS:
+            ret = ec_ioctl_sc_emerg_overruns(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_STATE:
+            ret = ec_ioctl_sc_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_PROCESS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_domain_process(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_QUEUE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_domain_queue(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_STATE:
+            ret = ec_ioctl_domain_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_INDEX:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_index(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_STATE:
+            ret = ec_ioctl_sdo_request_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_SDO_REQUEST_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sdo_request_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_IDN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_index(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_STATE:
+            ret = ec_ioctl_soe_request_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_SOE_REQUEST_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_soe_request_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_STATE:
+            ret = ec_ioctl_reg_request_state(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_reg_request_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_REG_REQUEST_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_reg_request_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_REC_HEADER:
+            ret = ec_ioctl_voe_rec_header(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_read(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_READ_NOSYNC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_read_nosync(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_write(master, arg, ctx);
+            break;
+        case EC_IOCTL_VOE_EXEC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_voe_exec(master, arg, ctx);
+            break;
+        default:
+            ret = ec_ioctl_both(master, ctx, cmd, arg);
+            break;
+    }
+
+#if DEBUG_LATENCY
+    b = get_cycles();
+    t = (unsigned int) ((b - a) * 1000LL) / cpu_khz;
+    if (t > 50) {
+        EC_MASTER_WARN(master, "ioctl(0x%02x) took %u us.\n",
+                _IOC_NR(cmd), t);
+    }
+#endif
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Called when an ioctl() command is issued.
+ * nRT context only.
+ *
+ * \return ioctl() return code.
+ */
+#ifdef EC_IOCTL_RTDM
+long ec_ioctl_rtdm_nrt
+#else
+static long ec_ioctl_nrt
+#endif
+        (
+        ec_master_t *master, /**< EtherCAT master. */
+        ec_ioctl_context_t *ctx, /**< Device context. */
+        unsigned int cmd, /**< ioctl() command identifier. */
+        void *arg /**< ioctl() argument. */
+        )
+{
+#if DEBUG_LATENCY && !defined(EC_IOCTL_RTDM)
+    cycles_t a = get_cycles(), b;
+    unsigned int t;
+#endif
+    int ret;
+
+    switch (cmd) {
+        case EC_IOCTL_MASTER:
+            ret = ec_ioctl_master(master, arg);
+            break;
+        case EC_IOCTL_SLAVE:
+            ret = ec_ioctl_slave(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SYNC:
+            ret = ec_ioctl_slave_sync(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SYNC_PDO:
+            ret = ec_ioctl_slave_sync_pdo(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SYNC_PDO_ENTRY:
+            ret = ec_ioctl_slave_sync_pdo_entry(master, arg);
+            break;
+        case EC_IOCTL_DOMAIN:
+            ret = ec_ioctl_domain(master, arg);
+            break;
+        case EC_IOCTL_DOMAIN_FMMU:
+            ret = ec_ioctl_domain_fmmu(master, arg);
+            break;
+        case EC_IOCTL_DOMAIN_DATA:
+            ret = ec_ioctl_domain_data(master, arg);
+            break;
+        case EC_IOCTL_MASTER_DEBUG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_master_debug(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_STATE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_state(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO:
+            ret = ec_ioctl_slave_sdo(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO_ENTRY:
+            ret = ec_ioctl_slave_sdo_entry(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO_UPLOAD:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_sdo_upload(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SDO_DOWNLOAD:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_sdo_download(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SII_READ:
+            ret = ec_ioctl_slave_sii_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SII_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_sii_write(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_REG_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_reg_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_REG_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_reg_write(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_FOE_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_foe_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_FOE_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_foe_write(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SOE_READ:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_soe_read(master, arg);
+            break;
+        case EC_IOCTL_SLAVE_SOE_WRITE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_soe_write(master, arg);
+            break;
+#ifdef EC_EOE
+        case EC_IOCTL_SLAVE_EOE_IP_PARAM:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_slave_eoe_ip_param(master, arg);
+            break;
+#endif
+        case EC_IOCTL_CONFIG:
+            ret = ec_ioctl_config(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_PDO:
+            ret = ec_ioctl_config_pdo(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_PDO_ENTRY:
+            ret = ec_ioctl_config_pdo_entry(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_SDO:
+            ret = ec_ioctl_config_sdo(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_IDN:
+            ret = ec_ioctl_config_idn(master, arg);
+            break;
+        case EC_IOCTL_CONFIG_FLAG:
+            ret = ec_ioctl_config_flag(master, arg);
+            break;
+#ifdef EC_EOE
+        case EC_IOCTL_CONFIG_EOE_IP_PARAM:
+            ret = ec_ioctl_config_ip(master, arg);
+            break;
+        case EC_IOCTL_EOE_HANDLER:
+            ret = ec_ioctl_eoe_handler(master, arg);
+            break;
+#endif
+
+        /* Application interface */
+
+        case EC_IOCTL_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_CREATE_DOMAIN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_create_domain(master, arg, ctx);
+            break;
+        case EC_IOCTL_CREATE_SLAVE_CONFIG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_create_slave_config(master, arg, ctx);
+            break;
+        case EC_IOCTL_SELECT_REF_CLOCK:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_select_ref_clock(master, arg, ctx);
+            break;
+        case EC_IOCTL_ACTIVATE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_activate(master, arg, ctx);
+            break;
+        case EC_IOCTL_DEACTIVATE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_deactivate(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SYNC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_sync(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_WATCHDOG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_watchdog(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_ADD_PDO:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_add_pdo(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_CLEAR_PDOS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_clear_pdos(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_ADD_ENTRY:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_add_entry(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_CLEAR_ENTRIES:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_clear_entries(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_REG_PDO_ENTRY:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_reg_pdo_entry(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_REG_PDO_POS:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_reg_pdo_pos(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_DC:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_dc(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SDO:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_sdo(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_EMERG_SIZE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_emerg_size(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SDO_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_sdo_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_SOE_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_soe_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_REG_REQUEST:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_reg_request(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_VOE:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_create_voe_handler(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_IDN:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_idn(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_FLAG:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_flag(master, arg, ctx);
+            break;
+        case EC_IOCTL_SC_STATE_TIMEOUT:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_state_timeout(master, arg, ctx);
+            break;
+#ifdef EC_EOE
+        case EC_IOCTL_SC_EOE_IP_PARAM:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_sc_ip(master, arg, ctx);
+            break;
+#endif
+        case EC_IOCTL_DOMAIN_SIZE:
+            ret = ec_ioctl_domain_size(master, arg, ctx);
+            break;
+        case EC_IOCTL_DOMAIN_OFFSET:
+            ret = ec_ioctl_domain_offset(master, arg, ctx);
+            break;
+        case EC_IOCTL_SET_SEND_INTERVAL:
+            if (!ctx->writable) {
+                ret = -EPERM;
+                break;
+            }
+            ret = ec_ioctl_set_send_interval(master, arg, ctx);
+            break;
+        default:
+#ifdef EC_IOCTL_RTDM
+            ret = ec_ioctl_both(master, ctx, cmd, arg);
+#else
+            ret = -ENOTTY;
+#endif
+            break;
+    }
+
+#if DEBUG_LATENCY && !defined(EC_IOCTL_RTDM)
+    b = get_cycles();
+    t = (unsigned int) ((b - a) * 1000LL) / cpu_khz;
+    if (t > 50) {
+        EC_MASTER_WARN(master, "ioctl(0x%02x) took %u us.\n",
+                _IOC_NR(cmd), t);
+    }
+#endif
+
+    return ret;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/rtdm.c b/drivers/net/ethercat/master/rtdm.c
new file mode 100644
index 000000000000..0112e1a7ba16
--- /dev/null
+++ b/drivers/net/ethercat/master/rtdm.c
@@ -0,0 +1,259 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2009-2010  Moehwald GmbH B. Benner
+ *                     2011  IgH Andreas Stewering-Bone
+ *                     2012  Florian Pose <fp@igh.de>
+ *
+ *  This file is part of the IgH EtherCAT master.
+ *
+ *  The IgH EtherCAT master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License as published
+ *  by the Free Software Foundation; version 2 of the License.
+ *
+ *  The IgH EtherCAT master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT master. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+/** \file
+ * RTDM interface.
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/mman.h>
+
+
+#include "master.h"
+#include "ioctl.h"
+#include "rtdm.h"
+#include "rtdm_details.h"
+
+/* include last because it does some redefinitions */
+#include <rtdm/rtdm_driver.h>
+
+/** Set to 1 to enable device operations debugging.
+ */
+#define DEBUG 0
+
+/****************************************************************************/
+
+static int ec_rtdm_open(struct rtdm_dev_context *, rtdm_user_info_t *, int);
+static int ec_rtdm_close(struct rtdm_dev_context *, rtdm_user_info_t *);
+static int ec_rtdm_ioctl_nrt_handler(struct rtdm_dev_context *,
+        rtdm_user_info_t *, unsigned int, void __user *);
+static int ec_rtdm_ioctl_rt_handler(struct rtdm_dev_context *,
+        rtdm_user_info_t *, unsigned int, void __user *);
+
+/****************************************************************************/
+
+/** Initialize an RTDM device.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_rtdm_dev_init(
+        ec_rtdm_dev_t *rtdm_dev, /**< EtherCAT RTDM device. */
+        ec_master_t *master /**< EtherCAT master. */
+        )
+{
+    int ret;
+
+    rtdm_dev->master = master;
+
+    rtdm_dev->dev = kzalloc(sizeof(struct rtdm_device), GFP_KERNEL);
+    if (!rtdm_dev->dev) {
+        EC_MASTER_ERR(master, "Failed to reserve memory for RTDM device.\n");
+        return -ENOMEM;
+    }
+
+    rtdm_dev->dev->struct_version = RTDM_DEVICE_STRUCT_VER;
+    rtdm_dev->dev->device_flags = RTDM_NAMED_DEVICE;
+    rtdm_dev->dev->context_size = sizeof(ec_rtdm_context_t);
+    snprintf(rtdm_dev->dev->device_name, RTDM_MAX_DEVNAME_LEN,
+            "EtherCAT%u", master->index);
+    rtdm_dev->dev->open_nrt = ec_rtdm_open;
+    rtdm_dev->dev->ops.close_nrt = ec_rtdm_close;
+    rtdm_dev->dev->ops.ioctl_rt = ec_rtdm_ioctl_rt_handler;
+    rtdm_dev->dev->ops.ioctl_nrt = ec_rtdm_ioctl_nrt_handler;
+    rtdm_dev->dev->device_class = RTDM_CLASS_EXPERIMENTAL;
+    rtdm_dev->dev->device_sub_class = 222;
+    rtdm_dev->dev->driver_name = "EtherCAT";
+    rtdm_dev->dev->driver_version = RTDM_DRIVER_VER(1, 0, 2);
+    rtdm_dev->dev->peripheral_name = rtdm_dev->dev->device_name;
+    rtdm_dev->dev->provider_name = "EtherLab Community";
+    rtdm_dev->dev->proc_name = rtdm_dev->dev->device_name;
+    rtdm_dev->dev->device_data = rtdm_dev; /* pointer to parent */
+
+    EC_MASTER_INFO(master, "Registering RTDM device %s.\n",
+            rtdm_dev->dev->driver_name);
+    ret = rtdm_dev_register(rtdm_dev->dev);
+    if (ret) {
+        EC_MASTER_ERR(master, "Initialization of RTDM interface failed"
+                " (return value %i).\n", ret);
+        kfree(rtdm_dev->dev);
+    }
+
+    return ret;
+}
+
+/****************************************************************************/
+
+/** Clear an RTDM device.
+ */
+void ec_rtdm_dev_clear(
+        ec_rtdm_dev_t *rtdm_dev /**< EtherCAT RTDM device. */
+        )
+{
+    int ret;
+
+    EC_MASTER_INFO(rtdm_dev->master, "Unregistering RTDM device %s.\n",
+            rtdm_dev->dev->driver_name);
+    ret = rtdm_dev_unregister(rtdm_dev->dev, 1000 /* poll delay [ms] */);
+    if (ret < 0) {
+        EC_MASTER_WARN(rtdm_dev->master,
+                "Failed to unregister RTDM device (code %i).\n", ret);
+    }
+
+    kfree(rtdm_dev->dev);
+}
+
+/****************************************************************************/
+
+/** Driver open.
+ *
+ * \return Always zero (success).
+ */
+static int ec_rtdm_open(
+        struct rtdm_dev_context *context, /**< Context. */
+        rtdm_user_info_t *user_info, /**< User data. */
+        int oflags /**< Open flags. */
+        )
+{
+    ec_rtdm_context_t *ctx = (ec_rtdm_context_t *) context->dev_private;
+#if DEBUG
+    ec_rtdm_dev_t *rtdm_dev = (ec_rtdm_dev_t *) context->device->device_data;
+#endif
+
+    ctx->user_fd = user_info;
+    ctx->ioctl_ctx.writable = oflags & O_WRONLY || oflags & O_RDWR;
+    ctx->ioctl_ctx.requested = 0;
+    ctx->ioctl_ctx.process_data = NULL;
+    ctx->ioctl_ctx.process_data_size = 0;
+
+#if DEBUG
+    EC_MASTER_INFO(rtdm_dev->master, "RTDM device %s opened.\n",
+            context->device->device_name);
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Driver close.
+ *
+ * \return Always zero (success).
+ */
+static int ec_rtdm_close(
+        struct rtdm_dev_context *context, /**< Context. */
+        rtdm_user_info_t *user_info /**< User data. */
+        )
+{
+    ec_rtdm_context_t *ctx = (ec_rtdm_context_t *) context->dev_private;
+    ec_rtdm_dev_t *rtdm_dev = (ec_rtdm_dev_t *) context->device->device_data;
+
+    if (ctx->ioctl_ctx.requested) {
+        ecrt_release_master(rtdm_dev->master);
+	}
+
+#if DEBUG
+    EC_MASTER_INFO(rtdm_dev->master, "RTDM device %s closed.\n",
+            context->device->device_name);
+#endif
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Driver ioctl.
+ *
+ * \return ioctl() return code.
+ */
+static int ec_rtdm_ioctl_nrt_handler(
+        struct rtdm_dev_context *context, /**< Context. */
+        rtdm_user_info_t *user_info, /**< User data. */
+        unsigned int request, /**< Request. */
+        void __user *arg /**< Argument. */
+        )
+{
+    ec_rtdm_context_t *ctx = (ec_rtdm_context_t *) context->dev_private;
+    ec_rtdm_dev_t *rtdm_dev = (ec_rtdm_dev_t *) context->device->device_data;
+
+#if DEBUG
+    EC_MASTER_INFO(rtdm_dev->master, "ioctl(request = %u, ctl = %02x)"
+            " on RTDM device %s.\n", request, _IOC_NR(request),
+            context->device->device_name);
+#endif
+    return ec_ioctl_rtdm_nrt(rtdm_dev->master, &ctx->ioctl_ctx, request, arg);
+}
+
+/****************************************************************************/
+
+static int ec_rtdm_ioctl_rt_handler(
+        struct rtdm_dev_context *context, /**< Context. */
+        rtdm_user_info_t *user_info, /**< User data. */
+        unsigned int request, /**< Request. */
+        void __user *arg /**< Argument. */
+        )
+{
+    int result;
+    ec_rtdm_context_t *ctx = (ec_rtdm_context_t *) context->dev_private;
+    ec_rtdm_dev_t *rtdm_dev = (ec_rtdm_dev_t *) context->device->device_data;
+
+#if DEBUG
+    EC_MASTER_INFO(rtdm_dev->master, "ioctl(request = %u, ctl = %02x)"
+            " on RTDM device %s.\n", request, _IOC_NR(request),
+            context->device->device_name);
+#endif
+    result =
+        ec_ioctl_rtdm_rt(rtdm_dev->master, &ctx->ioctl_ctx, request, arg);
+
+    if (result == -ENOTTY) {
+		/* Try again with nrt ioctl handler above in secondary mode. */
+        return -ENOSYS;
+    }
+    return result;
+}
+
+/****************************************************************************/
+
+/** Memory-map process data to user space.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_rtdm_mmap(
+        ec_ioctl_context_t *ioctl_ctx, /**< Context. */
+        void **user_address /**< Userspace address. */
+        )
+{
+    ec_rtdm_context_t *ctx =
+        container_of(ioctl_ctx, ec_rtdm_context_t, ioctl_ctx);
+    int ret;
+
+    ret = rtdm_mmap_to_user(ctx->user_fd,
+            ioctl_ctx->process_data, ioctl_ctx->process_data_size,
+            PROT_READ | PROT_WRITE,
+            user_address,
+            NULL, NULL);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/rtdm.h b/drivers/net/ethercat/master/rtdm.h
new file mode 100644
index 000000000000..3c2fa1380a0b
--- /dev/null
+++ b/drivers/net/ethercat/master/rtdm.h
@@ -0,0 +1,50 @@
+/*****************************************************************************
+ *
+ *  Copyright (C)      2012  Florian Pose <fp@igh.de>
+ *
+ *  This file is part of the IgH EtherCAT master.
+ *
+ *  The IgH EtherCAT master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License as published
+ *  by the Free Software Foundation; version 2 of the License.
+ *
+ *  The IgH EtherCAT master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT master. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+/** \file
+ * RTDM interface.
+ */
+
+#ifndef __EC_RTDM_H__
+#define __EC_RTDM_H__
+
+#include "../include/ecrt.h" /* ec_master_t */
+
+/****************************************************************************/
+
+struct rtdm_device;
+
+/****************************************************************************/
+
+/** EtherCAT RTDM device.
+ */
+typedef struct ec_rtdm_dev {
+    ec_master_t *master; /**< Master pointer. */
+    struct rtdm_device *dev; /**< RTDM device. */
+} ec_rtdm_dev_t;
+
+/****************************************************************************/
+
+int ec_rtdm_dev_init(ec_rtdm_dev_t *, ec_master_t *);
+void ec_rtdm_dev_clear(ec_rtdm_dev_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/rtdm_details.h b/drivers/net/ethercat/master/rtdm_details.h
new file mode 100644
index 000000000000..d57ce01e5feb
--- /dev/null
+++ b/drivers/net/ethercat/master/rtdm_details.h
@@ -0,0 +1,60 @@
+/*****************************************************************************
+ *
+ *  Copyright (C)      2024  Bjarne von Horn <vh@igh.de>
+ *
+ *  This file is part of the IgH EtherCAT master.
+ *
+ *  The IgH EtherCAT master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License as published
+ *  by the Free Software Foundation; version 2 of the License.
+ *
+ *  The IgH EtherCAT master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT master. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+#ifndef __EC_RTDM_DETAILS_H__
+#define __EC_RTDM_DETAILS_H__
+
+#include "../config.h"
+#include "ioctl.h"
+
+#include <linux/kernel.h>
+
+#ifdef EC_RTDM_XENOMAI_V3
+
+#include <rtdm/driver.h>
+#define EC_RTDM_USERFD_T struct rtdm_fd
+
+#else // EC_RTDM_XENOMAI_V3
+
+#include <rtdm/rtdm_driver.h>
+
+#define EC_RTDM_USERFD_T rtdm_user_info_t
+
+#endif // EC_RTDM_XENOMAI_V3
+
+/****************************************************************************/
+
+/** Context structure for an open RTDM file handle.
+ */
+typedef struct ec_rtdm_context {
+    EC_RTDM_USERFD_T *user_fd; /**< RTDM user data. */
+    ec_ioctl_context_t ioctl_ctx; /**< Context structure. */
+} ec_rtdm_context_t;
+
+/****************************************************************************/
+
+static inline EC_RTDM_USERFD_T *ec_ioctl_to_rtdm(ec_ioctl_context_t *ctx)
+{
+    return container_of(ctx, ec_rtdm_context_t, ioctl_ctx)->user_fd;
+}
+
+/****************************************************************************/
+
+#endif  // __EC_RTDM_DETAILS_H__
diff --git a/drivers/net/ethercat/master/rtdm_xenomai_v3.c b/drivers/net/ethercat/master/rtdm_xenomai_v3.c
new file mode 100644
index 000000000000..230e4172f9c7
--- /dev/null
+++ b/drivers/net/ethercat/master/rtdm_xenomai_v3.c
@@ -0,0 +1,193 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2009-2010  Moehwald GmbH B. Benner
+ *                     2011  IgH Andreas Stewering-Bone
+ *                     2012  Florian Pose <fp@igh.de>
+ *
+ *  This file is part of the IgH EtherCAT master.
+ *
+ *  The IgH EtherCAT master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License as published
+ *  by the Free Software Foundation; version 2 of the License.
+ *
+ *  The IgH EtherCAT master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT master. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+/** \file
+ * RTDM interface.
+ */
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <rtdm/driver.h>
+
+#include "master.h"
+#include "ioctl.h"
+#include "rtdm.h"
+#include "rtdm_details.h"
+
+/** Set to 1 to enable device operations debugging.
+ */
+#define DEBUG_RTDM 0
+
+/****************************************************************************/
+
+static int ec_rtdm_open(struct rtdm_fd *fd, int oflags)
+{
+	struct ec_rtdm_context *ctx = rtdm_fd_to_private(fd);
+#if DEBUG_RTDM
+	struct rtdm_device *dev = rtdm_fd_device(fd);
+	ec_rtdm_dev_t *rtdm_dev = dev->device_data;
+#endif
+
+	ctx->user_fd = fd;
+
+	ctx->ioctl_ctx.writable = oflags & O_WRONLY || oflags & O_RDWR;
+	ctx->ioctl_ctx.requested = 0;
+	ctx->ioctl_ctx.process_data = NULL;
+	ctx->ioctl_ctx.process_data_size = 0;
+
+#if DEBUG_RTDM
+	EC_MASTER_INFO(rtdm_dev->master, "RTDM device %s opened.\n",
+			dev->name);
+#endif
+
+	return 0;
+}
+
+/****************************************************************************/
+
+static void ec_rtdm_close(struct rtdm_fd *fd)
+{
+	struct ec_rtdm_context *ctx = rtdm_fd_to_private(fd);
+	struct rtdm_device *dev = rtdm_fd_device(fd);
+	ec_rtdm_dev_t *rtdm_dev = dev->device_data;
+
+	if (ctx->ioctl_ctx.requested)
+		ecrt_release_master(rtdm_dev->master);
+
+	if (ctx->ioctl_ctx.process_data)
+		vfree(ctx->ioctl_ctx.process_data);
+
+#if DEBUG_RTDM
+	EC_MASTER_INFO(rtdm_dev->master, "RTDM device %s closed.\n",
+			dev->name);
+#endif
+}
+
+/****************************************************************************/
+
+static int ec_rtdm_ioctl_rt_handler(struct rtdm_fd *fd, unsigned int request,
+			 void __user *arg)
+{
+	int result;
+	struct ec_rtdm_context *ctx = rtdm_fd_to_private(fd);
+	struct rtdm_device *dev = rtdm_fd_device(fd);
+	ec_rtdm_dev_t *rtdm_dev = dev->device_data;
+
+	result =
+        ec_ioctl_rtdm_rt(rtdm_dev->master, &ctx->ioctl_ctx, request, arg);
+
+	if (result == -ENOTTY)
+		/* Try again with nrt ioctl handler below in secondary mode. */
+		return -ENOSYS;
+
+	return result;
+}
+
+/****************************************************************************/
+
+static int ec_rtdm_ioctl_nrt_handler(struct rtdm_fd *fd, unsigned int request,
+			 void __user *arg)
+{
+	struct ec_rtdm_context *ctx = rtdm_fd_to_private(fd);
+	struct rtdm_device *dev = rtdm_fd_device(fd);
+	ec_rtdm_dev_t *rtdm_dev = dev->device_data;
+
+	return ec_ioctl_rtdm_nrt(rtdm_dev->master, &ctx->ioctl_ctx, request, arg);
+}
+
+/****************************************************************************/
+
+static int ec_rtdm_mmap(struct rtdm_fd *fd, struct vm_area_struct *vma)
+{
+	struct ec_rtdm_context *ctx =
+        (struct ec_rtdm_context *) rtdm_fd_to_private(fd);
+	return rtdm_mmap_kmem(vma, (void *)ctx->ioctl_ctx.process_data);
+}
+
+/****************************************************************************/
+
+static struct rtdm_driver ec_rtdm_driver = {
+	.profile_info		= RTDM_PROFILE_INFO(ec_rtdm,
+						    RTDM_CLASS_EXPERIMENTAL,
+						    222,
+						    0),
+	.device_flags		= RTDM_NAMED_DEVICE,
+	.device_count		= EC_MAX_MASTERS,
+	.context_size		= sizeof(struct ec_rtdm_context),
+	.ops = {
+		.open		= ec_rtdm_open,
+		.close		= ec_rtdm_close,
+		.ioctl_rt	= ec_rtdm_ioctl_rt_handler,
+		.ioctl_nrt	= ec_rtdm_ioctl_nrt_handler,
+		.mmap		= ec_rtdm_mmap,
+	},
+};
+
+/****************************************************************************/
+
+int ec_rtdm_dev_init(ec_rtdm_dev_t *rtdm_dev, ec_master_t *master)
+{
+	struct rtdm_device *dev;
+	int ret;
+
+	rtdm_dev->master = master;
+
+	rtdm_dev->dev = kzalloc(sizeof(struct rtdm_device), GFP_KERNEL);
+	if (!rtdm_dev->dev) {
+		EC_MASTER_ERR(master,
+				"Failed to reserve memory for RTDM device.\n");
+		return -ENOMEM;
+	}
+
+	dev = rtdm_dev->dev;
+
+	dev->driver = &ec_rtdm_driver;
+	dev->device_data = rtdm_dev;
+	dev->label = "EtherCAT%u";
+	dev->minor = master->index;
+
+	ret = rtdm_dev_register(dev);
+	if (ret) {
+		EC_MASTER_ERR(master, "Initialization of RTDM interface failed"
+				" (return value %i).\n", ret);
+		kfree(dev);
+		return ret;
+	}
+
+	EC_MASTER_INFO(master, "Registered RTDM device %s.\n", dev->name);
+
+	return 0;
+}
+
+/****************************************************************************/
+
+void ec_rtdm_dev_clear(ec_rtdm_dev_t *rtdm_dev)
+{
+	rtdm_dev_unregister(rtdm_dev->dev);
+
+	EC_MASTER_INFO(rtdm_dev->master, "Unregistered RTDM device %s.\n",
+			rtdm_dev->dev->name);
+
+	kfree(rtdm_dev->dev);
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/sdo.c b/drivers/net/ethercat/master/sdo.c
new file mode 100644
index 000000000000..3b8d3ac6ce7a
--- /dev/null
+++ b/drivers/net/ethercat/master/sdo.c
@@ -0,0 +1,124 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   CANopen SDO functions.
+*/
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+
+#include "master.h"
+
+#include "sdo.h"
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_sdo_init(
+        ec_sdo_t *sdo, /**< SDO. */
+        ec_slave_t *slave, /**< Parent slave. */
+        uint16_t index /**< SDO index. */
+        )
+{
+    sdo->slave = slave;
+    sdo->index = index;
+    sdo->object_code = 0x00;
+    sdo->name = NULL;
+    sdo->max_subindex = 0;
+    INIT_LIST_HEAD(&sdo->entries);
+}
+
+/****************************************************************************/
+
+/** SDO destructor.
+ *
+ * Clears and frees an SDO object.
+ */
+void ec_sdo_clear(
+        ec_sdo_t *sdo /**< SDO. */
+        )
+{
+    ec_sdo_entry_t *entry, *next;
+
+    // free all entries
+    list_for_each_entry_safe(entry, next, &sdo->entries, list) {
+        list_del(&entry->list);
+        ec_sdo_entry_clear(entry);
+        kfree(entry);
+    }
+
+    if (sdo->name)
+        kfree(sdo->name);
+}
+
+/****************************************************************************/
+
+/** Get an SDO entry from an SDO via its subindex.
+ *
+ * \retval >0 Pointer to the requested SDO entry.
+ * \retval NULL SDO entry not found.
+ */
+ec_sdo_entry_t *ec_sdo_get_entry(
+        ec_sdo_t *sdo, /**< SDO. */
+        uint8_t subindex /**< Entry subindex. */
+        )
+{
+    ec_sdo_entry_t *entry;
+
+    list_for_each_entry(entry, &sdo->entries, list) {
+        if (entry->subindex != subindex)
+            continue;
+        return entry;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Get an SDO entry from an SDO via its subindex.
+ *
+ * const version.
+ *
+ * \retval >0 Pointer to the requested SDO entry.
+ * \retval NULL SDO entry not found.
+ */
+const ec_sdo_entry_t *ec_sdo_get_entry_const(
+        const ec_sdo_t *sdo, /**< SDO. */
+        uint8_t subindex /**< Entry subindex. */
+        )
+{
+    const ec_sdo_entry_t *entry;
+
+    list_for_each_entry(entry, &sdo->entries, list) {
+        if (entry->subindex != subindex)
+            continue;
+        return entry;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/sdo.h b/drivers/net/ethercat/master/sdo.h
new file mode 100644
index 000000000000..5426bcaa2ec7
--- /dev/null
+++ b/drivers/net/ethercat/master/sdo.h
@@ -0,0 +1,61 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT CANopen SDO structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_SDO_H__
+#define __EC_SDO_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+#include "sdo_entry.h"
+
+/****************************************************************************/
+
+/** CANopen SDO.
+ */
+struct ec_sdo {
+    struct list_head list; /**< List item. */
+    ec_slave_t *slave; /**< Parent slave. */
+    uint16_t index; /**< SDO index. */
+    uint8_t object_code; /**< Object code. */
+    char *name; /**< SDO name. */
+    uint8_t max_subindex; /**< Maximum subindex. */
+    struct list_head entries; /**< List of entries. */
+};
+
+/****************************************************************************/
+
+void ec_sdo_init(ec_sdo_t *, ec_slave_t *, uint16_t);
+void ec_sdo_clear(ec_sdo_t *);
+
+ec_sdo_entry_t *ec_sdo_get_entry(ec_sdo_t *, uint8_t);
+const ec_sdo_entry_t *ec_sdo_get_entry_const(const ec_sdo_t *, uint8_t);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/sdo_entry.c b/drivers/net/ethercat/master/sdo_entry.c
new file mode 100644
index 000000000000..ae4b36619828
--- /dev/null
+++ b/drivers/net/ethercat/master/sdo_entry.c
@@ -0,0 +1,69 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   CANopen over EtherCAT SDO entry functions.
+*/
+
+/****************************************************************************/
+
+#include <linux/slab.h>
+
+#include "sdo_entry.h"
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_sdo_entry_init(
+        ec_sdo_entry_t *entry, /**< SDO entry. */
+        ec_sdo_t *sdo, /**< Parent SDO. */
+        uint8_t subindex /**< Subindex. */
+        )
+{
+    entry->sdo = sdo;
+    entry->subindex = subindex;
+    entry->data_type = 0x0000;
+    entry->bit_length = 0;
+    entry->read_access[EC_SDO_ENTRY_ACCESS_PREOP] = 0;
+    entry->read_access[EC_SDO_ENTRY_ACCESS_SAFEOP] = 0;
+    entry->read_access[EC_SDO_ENTRY_ACCESS_OP] = 0;
+    entry->write_access[EC_SDO_ENTRY_ACCESS_PREOP] = 0;
+    entry->write_access[EC_SDO_ENTRY_ACCESS_SAFEOP] = 0;
+    entry->write_access[EC_SDO_ENTRY_ACCESS_OP] = 0;
+    entry->description = NULL;
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_sdo_entry_clear(
+        ec_sdo_entry_t *entry /**< SDO entry. */
+        )
+{
+
+    if (entry->description)
+        kfree(entry->description);
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/sdo_entry.h b/drivers/net/ethercat/master/sdo_entry.h
new file mode 100644
index 000000000000..87c92bbd803c
--- /dev/null
+++ b/drivers/net/ethercat/master/sdo_entry.h
@@ -0,0 +1,64 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT CANopen SDO entry structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_SDO_ENTRY_H__
+#define __EC_SDO_ENTRY_H__
+
+#include <linux/list.h>
+#include <linux/kobject.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+struct ec_sdo;
+typedef struct ec_sdo ec_sdo_t; /**< \see ec_sdo. */
+
+/****************************************************************************/
+
+/** CANopen SDO entry.
+ */
+typedef struct {
+    struct list_head list; /**< List item. */
+    ec_sdo_t *sdo; /**< Parent SDO. */
+    uint8_t subindex; /**< Subindex. */
+    uint16_t data_type; /**< Data type. */
+    uint16_t bit_length; /**< Data size in bit. */
+    uint8_t read_access[EC_SDO_ENTRY_ACCESS_COUNT]; /**< Read access. */
+    uint8_t write_access[EC_SDO_ENTRY_ACCESS_COUNT]; /**< Write access. */
+    char *description; /**< Description. */
+} ec_sdo_entry_t;
+
+/****************************************************************************/
+
+void ec_sdo_entry_init(ec_sdo_entry_t *, ec_sdo_t *, uint8_t);
+void ec_sdo_entry_clear(ec_sdo_entry_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/sdo_request.c b/drivers/net/ethercat/master/sdo_request.c
new file mode 100644
index 000000000000..040c6b5d5a8a
--- /dev/null
+++ b/drivers/net/ethercat/master/sdo_request.c
@@ -0,0 +1,256 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * Canopen over EtherCAT SDO request functions.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/jiffies.h>
+#include <linux/slab.h>
+
+#include "sdo_request.h"
+
+/****************************************************************************/
+
+/** Default timeout in ms to wait for SDO transfer responses.
+ */
+#define EC_SDO_REQUEST_RESPONSE_TIMEOUT 1000
+
+/****************************************************************************/
+
+void ec_sdo_request_clear_data(ec_sdo_request_t *);
+
+/****************************************************************************/
+
+/** SDO request constructor.
+ */
+void ec_sdo_request_init(
+        ec_sdo_request_t *req /**< SDO request. */
+        )
+{
+    req->complete_access = 0;
+    req->data = NULL;
+    req->mem_size = 0;
+    req->data_size = 0;
+    req->issue_timeout = 0; // no timeout
+    req->response_timeout = EC_SDO_REQUEST_RESPONSE_TIMEOUT;
+    req->dir = EC_DIR_INVALID;
+    req->state = EC_INT_REQUEST_INIT;
+    req->jiffies_start = 0U;
+    req->jiffies_sent = 0U;
+    req->errno = 0;
+    req->abort_code = 0x00000000;
+}
+
+/****************************************************************************/
+
+/** SDO request destructor.
+ */
+void ec_sdo_request_clear(
+        ec_sdo_request_t *req /**< SDO request. */
+        )
+{
+    ec_sdo_request_clear_data(req);
+}
+
+/****************************************************************************/
+
+/** Copy another SDO request.
+ *
+ * \attention Only the index subindex and data are copied.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_sdo_request_copy(
+        ec_sdo_request_t *req, /**< SDO request. */
+        const ec_sdo_request_t *other /**< Other SDO request to copy from. */
+        )
+{
+    req->complete_access = other->complete_access;
+    req->index = other->index;
+    req->subindex = other->subindex;
+    return ec_sdo_request_copy_data(req, other->data, other->data_size);
+}
+
+/****************************************************************************/
+
+/** SDO request destructor.
+ */
+void ec_sdo_request_clear_data(
+        ec_sdo_request_t *req /**< SDO request. */
+        )
+{
+    if (req->data) {
+        kfree(req->data);
+        req->data = NULL;
+    }
+
+    req->mem_size = 0;
+    req->data_size = 0;
+}
+
+/****************************************************************************/
+
+/** Pre-allocates the data memory.
+ *
+ * If the \a mem_size is already bigger than \a size, nothing is done.
+ *
+ * \return 0 on success, otherwise -ENOMEM.
+ */
+int ec_sdo_request_alloc(
+        ec_sdo_request_t *req, /**< SDO request. */
+        size_t size /**< Data size to allocate. */
+        )
+{
+    if (size <= req->mem_size)
+        return 0;
+
+    ec_sdo_request_clear_data(req);
+
+    if (!(req->data = (uint8_t *) kmalloc(size, GFP_KERNEL))) {
+        EC_ERR("Failed to allocate %zu bytes of SDO memory.\n", size);
+        return -ENOMEM;
+    }
+
+    req->mem_size = size;
+    req->data_size = 0;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Copies SDO data from an external source.
+ *
+ * If the \a mem_size is to small, new memory is allocated.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_sdo_request_copy_data(
+        ec_sdo_request_t *req, /**< SDO request. */
+        const uint8_t *source, /**< Source data. */
+        size_t size /**< Number of bytes in \a source. */
+        )
+{
+    int ret = ec_sdo_request_alloc(req, size);
+    if (ret < 0)
+        return ret;
+
+    memcpy(req->data, source, size);
+    req->data_size = size;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Checks, if the timeout was exceeded.
+ *
+ * \return non-zero if the timeout was exceeded, else zero.
+ */
+int ec_sdo_request_timed_out(const ec_sdo_request_t *req /**< SDO request. */)
+{
+    return req->issue_timeout
+        && jiffies - req->jiffies_start > HZ * req->issue_timeout / 1000;
+}
+
+/*****************************************************************************
+ * Application interface.
+ ****************************************************************************/
+
+int ecrt_sdo_request_index(ec_sdo_request_t *req, uint16_t index,
+        uint8_t subindex)
+{
+    req->index = index;
+    req->subindex = subindex;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_sdo_request_timeout(ec_sdo_request_t *req, uint32_t timeout)
+{
+    req->issue_timeout = timeout;
+    return 0;
+}
+
+/****************************************************************************/
+
+uint8_t *ecrt_sdo_request_data(const ec_sdo_request_t *req)
+{
+    return req->data;
+}
+
+/****************************************************************************/
+
+size_t ecrt_sdo_request_data_size(const ec_sdo_request_t *req)
+{
+    return req->data_size;
+}
+
+/****************************************************************************/
+
+ec_request_state_t ecrt_sdo_request_state(const ec_sdo_request_t *req)
+{
+   return ec_request_state_translation_table[req->state];
+}
+
+/****************************************************************************/
+
+int ecrt_sdo_request_read(ec_sdo_request_t *req)
+{
+    req->dir = EC_DIR_INPUT;
+    req->state = EC_INT_REQUEST_QUEUED;
+    req->errno = 0;
+    req->abort_code = 0x00000000;
+    req->jiffies_start = jiffies;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_sdo_request_write(ec_sdo_request_t *req)
+{
+    req->dir = EC_DIR_OUTPUT;
+    req->state = EC_INT_REQUEST_QUEUED;
+    req->errno = 0;
+    req->abort_code = 0x00000000;
+    req->jiffies_start = jiffies;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecrt_sdo_request_index);
+EXPORT_SYMBOL(ecrt_sdo_request_timeout);
+EXPORT_SYMBOL(ecrt_sdo_request_data);
+EXPORT_SYMBOL(ecrt_sdo_request_data_size);
+EXPORT_SYMBOL(ecrt_sdo_request_state);
+EXPORT_SYMBOL(ecrt_sdo_request_read);
+EXPORT_SYMBOL(ecrt_sdo_request_write);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/sdo_request.h b/drivers/net/ethercat/master/sdo_request.h
new file mode 100644
index 000000000000..aa2d1849cec3
--- /dev/null
+++ b/drivers/net/ethercat/master/sdo_request.h
@@ -0,0 +1,75 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT CANopen SDO request structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_SDO_REQUEST_H__
+#define __EC_SDO_REQUEST_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** CANopen SDO request.
+ */
+struct ec_sdo_request {
+    struct list_head list; /**< List item. */
+    uint16_t index; /**< SDO index. */
+    uint8_t subindex; /**< SDO subindex. */
+    uint8_t *data; /**< Pointer to SDO data. */
+    size_t mem_size; /**< Size of SDO data memory. */
+    size_t data_size; /**< Size of SDO data. */
+    uint8_t complete_access; /**< SDO shall be transferred completely. */
+    uint32_t issue_timeout; /**< Maximum time in ms, the processing of the
+                              request may take. */
+    uint32_t response_timeout; /**< Maximum time in ms, the transfer is
+                                 retried, if the slave does not respond. */
+    ec_direction_t dir; /**< Direction. EC_DIR_OUTPUT means downloading to
+                          the slave, EC_DIR_INPUT means uploading from the
+                          slave. */
+    ec_internal_request_state_t state; /**< SDO request state. */
+    unsigned long jiffies_start; /**< Jiffies, when the request was issued. */
+    unsigned long jiffies_sent; /**< Jiffies, when the upload/download
+                                     request was sent. */
+    int errno; /**< Error number. */
+    uint32_t abort_code; /**< SDO request abort code. Zero on success. */
+};
+
+/****************************************************************************/
+
+void ec_sdo_request_init(ec_sdo_request_t *);
+void ec_sdo_request_clear(ec_sdo_request_t *);
+
+int ec_sdo_request_copy(ec_sdo_request_t *, const ec_sdo_request_t *);
+int ec_sdo_request_alloc(ec_sdo_request_t *, size_t);
+int ec_sdo_request_copy_data(ec_sdo_request_t *, const uint8_t *, size_t);
+int ec_sdo_request_timed_out(const ec_sdo_request_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/slave.c b/drivers/net/ethercat/master/slave.c
new file mode 100644
index 000000000000..04cf2151daff
--- /dev/null
+++ b/drivers/net/ethercat/master/slave.c
@@ -0,0 +1,1010 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave methods.
+*/
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/delay.h>
+
+#include "globals.h"
+#include "datagram.h"
+#include "master.h"
+#include "slave_config.h"
+
+#include "slave.h"
+
+/****************************************************************************/
+
+extern const ec_code_msg_t al_status_messages[];
+
+/****************************************************************************/
+
+// prototypes for private methods
+char *ec_slave_sii_string(const ec_slave_t *, unsigned int);
+void ec_slave_find_names_for_pdo(ec_slave_t *, ec_pdo_t *);
+unsigned int ec_slave_get_previous_port(const ec_slave_t *, unsigned int);
+unsigned int ec_slave_get_next_port(const ec_slave_t *, unsigned int);
+uint32_t ec_slave_calc_rtt_sum(const ec_slave_t *);
+ec_slave_t *ec_slave_find_next_dc_slave(ec_slave_t *);
+
+/****************************************************************************/
+
+/**
+   Slave constructor.
+   \return 0 in case of success, else < 0
+*/
+
+void ec_slave_init(
+        ec_slave_t *slave, /**< EtherCAT slave */
+        ec_master_t *master, /**< EtherCAT master */
+        ec_device_index_t dev_idx, /**< Device index. */
+        uint16_t ring_position, /**< ring position */
+        uint16_t station_address /**< station address to configure */
+        )
+{
+    unsigned int i;
+
+    slave->master = master;
+    slave->device_index = dev_idx;
+    slave->ring_position = ring_position;
+    slave->station_address = station_address;
+    slave->effective_alias = 0x0000;
+
+    slave->config = NULL;
+    slave->requested_state = EC_SLAVE_STATE_PREOP;
+    slave->current_state = EC_SLAVE_STATE_UNKNOWN;
+    slave->error_flag = 0;
+    slave->force_config = 0;
+    slave->configured_rx_mailbox_offset = 0x0000;
+    slave->configured_rx_mailbox_size = 0x0000;
+    slave->configured_tx_mailbox_offset = 0x0000;
+    slave->configured_tx_mailbox_size = 0x0000;
+
+    slave->base_type = 0;
+    slave->base_revision = 0;
+    slave->base_build = 0;
+    slave->base_fmmu_count = 0;
+    slave->base_sync_count = 0;
+
+    for (i = 0; i < EC_MAX_PORTS; i++) {
+        slave->ports[i].desc = EC_PORT_NOT_IMPLEMENTED;
+
+        slave->ports[i].link.link_up = 0;
+        slave->ports[i].link.loop_closed = 0;
+        slave->ports[i].link.signal_detected = 0;
+        slave->sii.physical_layer[i] = 0xFF;
+
+        slave->ports[i].receive_time = 0U;
+
+        slave->ports[i].next_slave = NULL;
+        slave->ports[i].delay_to_next_dc = 0U;
+    }
+
+    slave->base_fmmu_bit_operation = 0;
+    slave->base_dc_supported = 0;
+    slave->base_dc_range = EC_DC_32;
+    slave->has_dc_system_time = 0;
+    slave->transmission_delay = 0U;
+
+    slave->sii_words = NULL;
+    slave->sii_nwords = 0;
+
+    slave->sii.alias = 0x0000;
+    slave->sii.vendor_id = 0x00000000;
+    slave->sii.product_code = 0x00000000;
+    slave->sii.revision_number = 0x00000000;
+    slave->sii.serial_number = 0x00000000;
+    slave->sii.boot_rx_mailbox_offset = 0x0000;
+    slave->sii.boot_rx_mailbox_size = 0x0000;
+    slave->sii.boot_tx_mailbox_offset = 0x0000;
+    slave->sii.boot_tx_mailbox_size = 0x0000;
+    slave->sii.std_rx_mailbox_offset = 0x0000;
+    slave->sii.std_rx_mailbox_size = 0x0000;
+    slave->sii.std_tx_mailbox_offset = 0x0000;
+    slave->sii.std_tx_mailbox_size = 0x0000;
+    slave->sii.mailbox_protocols = 0;
+
+    slave->sii.strings = NULL;
+    slave->sii.string_count = 0;
+
+    slave->sii.has_general = 0;
+    slave->sii.group = NULL;
+    slave->sii.image = NULL;
+    slave->sii.order = NULL;
+    slave->sii.name = NULL;
+    memset(&slave->sii.coe_details, 0x00, sizeof(ec_sii_coe_details_t));
+    memset(&slave->sii.general_flags, 0x00, sizeof(ec_sii_general_flags_t));
+    slave->sii.current_on_ebus = 0;
+
+    slave->sii.syncs = NULL;
+    slave->sii.sync_count = 0;
+
+    INIT_LIST_HEAD(&slave->sii.pdos);
+
+    INIT_LIST_HEAD(&slave->sdo_dictionary);
+
+    slave->sdo_dictionary_fetched = 0;
+    slave->jiffies_preop = 0;
+
+    INIT_LIST_HEAD(&slave->sdo_requests);
+    INIT_LIST_HEAD(&slave->reg_requests);
+    INIT_LIST_HEAD(&slave->foe_requests);
+    INIT_LIST_HEAD(&slave->soe_requests);
+    INIT_LIST_HEAD(&slave->eoe_requests);
+
+    // create state machine object
+    ec_fsm_slave_init(&slave->fsm, slave);
+}
+
+/****************************************************************************/
+
+/**
+   Slave destructor.
+   Clears and frees a slave object.
+*/
+
+void ec_slave_clear(ec_slave_t *slave /**< EtherCAT slave */)
+{
+    ec_sdo_t *sdo, *next_sdo;
+    unsigned int i;
+    ec_pdo_t *pdo, *next_pdo;
+
+    // abort all pending requests
+
+    while (!list_empty(&slave->sdo_requests)) {
+        ec_sdo_request_t *request =
+            list_entry(slave->sdo_requests.next, ec_sdo_request_t, list);
+        list_del_init(&request->list); // dequeue
+        EC_SLAVE_WARN(slave, "Discarding SDO request,"
+                " slave about to be deleted.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+    }
+
+    while (!list_empty(&slave->reg_requests)) {
+        ec_reg_request_t *reg =
+            list_entry(slave->reg_requests.next, ec_reg_request_t, list);
+        list_del_init(&reg->list); // dequeue
+        EC_SLAVE_WARN(slave, "Discarding register request,"
+                " slave about to be deleted.\n");
+        reg->state = EC_INT_REQUEST_FAILURE;
+    }
+
+    while (!list_empty(&slave->foe_requests)) {
+        ec_foe_request_t *request =
+            list_entry(slave->foe_requests.next, ec_foe_request_t, list);
+        list_del_init(&request->list); // dequeue
+        EC_SLAVE_WARN(slave, "Discarding FoE request,"
+                " slave about to be deleted.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+    }
+
+    while (!list_empty(&slave->soe_requests)) {
+        ec_soe_request_t *request =
+            list_entry(slave->soe_requests.next, ec_soe_request_t, list);
+        list_del_init(&request->list); // dequeue
+        EC_SLAVE_WARN(slave, "Discarding SoE request,"
+                " slave about to be deleted.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+    }
+
+#ifdef EC_EOE
+    while (!list_empty(&slave->eoe_requests)) {
+        ec_eoe_request_t *request =
+            list_entry(slave->eoe_requests.next, ec_eoe_request_t, list);
+        list_del_init(&request->list); // dequeue
+        EC_SLAVE_WARN(slave, "Discarding EoE request,"
+                " slave about to be deleted.\n");
+        request->state = EC_INT_REQUEST_FAILURE;
+    }
+#endif
+
+    wake_up_all(&slave->master->request_queue);
+
+    if (slave->config) {
+        ec_slave_config_detach(slave->config);
+    }
+
+    // free all SDOs
+    list_for_each_entry_safe(sdo, next_sdo, &slave->sdo_dictionary, list) {
+        list_del(&sdo->list);
+        ec_sdo_clear(sdo);
+        kfree(sdo);
+    }
+
+    // free all strings
+    if (slave->sii.strings) {
+        for (i = 0; i < slave->sii.string_count; i++)
+            kfree(slave->sii.strings[i]);
+        kfree(slave->sii.strings);
+    }
+
+    // free all sync managers
+    ec_slave_clear_sync_managers(slave);
+
+    // free all SII PDOs
+    list_for_each_entry_safe(pdo, next_pdo, &slave->sii.pdos, list) {
+        list_del(&pdo->list);
+        ec_pdo_clear(pdo);
+        kfree(pdo);
+    }
+
+    if (slave->sii_words) {
+        kfree(slave->sii_words);
+    }
+
+    ec_fsm_slave_clear(&slave->fsm);
+}
+
+/****************************************************************************/
+
+/** Clear the sync manager array.
+ */
+void ec_slave_clear_sync_managers(ec_slave_t *slave /**< EtherCAT slave. */)
+{
+    unsigned int i;
+
+    if (slave->sii.syncs) {
+        for (i = 0; i < slave->sii.sync_count; i++) {
+            ec_sync_clear(&slave->sii.syncs[i]);
+        }
+        kfree(slave->sii.syncs);
+        slave->sii.syncs = NULL;
+    }
+}
+
+/****************************************************************************/
+
+/**
+ * Sets the application state of a slave.
+ */
+
+void ec_slave_set_state(ec_slave_t *slave, /**< EtherCAT slave */
+        ec_slave_state_t new_state /**< new application state */
+        )
+{
+    if (new_state != slave->current_state) {
+        if (slave->master->debug_level) {
+            char old_state[EC_STATE_STRING_SIZE],
+                cur_state[EC_STATE_STRING_SIZE];
+            ec_state_string(slave->current_state, old_state, 0);
+            ec_state_string(new_state, cur_state, 0);
+            EC_SLAVE_DBG(slave, 0, "%s -> %s.\n", old_state, cur_state);
+        }
+        slave->current_state = new_state;
+    }
+}
+
+/****************************************************************************/
+
+/**
+ * Request a slave state and resets the error flag.
+ */
+
+void ec_slave_request_state(ec_slave_t *slave, /**< EtherCAT slave */
+                            ec_slave_state_t state /**< new state */
+                            )
+{
+    slave->requested_state = state;
+    slave->error_flag = 0;
+}
+
+/****************************************************************************/
+
+/**
+   Fetches data from a STRING category.
+   \todo range checking
+   \return 0 in case of success, else < 0
+*/
+
+int ec_slave_fetch_sii_strings(
+        ec_slave_t *slave, /**< EtherCAT slave */
+        const uint8_t *data, /**< category data */
+        size_t data_size /**< number of bytes */
+        )
+{
+    int i, err;
+    size_t size;
+    off_t offset;
+
+    slave->sii.string_count = data[0];
+
+    if (slave->sii.string_count) {
+        if (!(slave->sii.strings =
+                    kmalloc(sizeof(char *) * slave->sii.string_count,
+                        GFP_KERNEL))) {
+            EC_SLAVE_ERR(slave, "Failed to allocate string array memory.\n");
+            err = -ENOMEM;
+            goto out_zero;
+        }
+
+        offset = 1;
+        for (i = 0; i < slave->sii.string_count; i++) {
+            size = data[offset];
+            // allocate memory for string structure and data at a single blow
+            if (!(slave->sii.strings[i] =
+                        kmalloc(sizeof(char) * size + 1, GFP_KERNEL))) {
+                EC_SLAVE_ERR(slave, "Failed to allocate string memory.\n");
+                err = -ENOMEM;
+                goto out_free;
+            }
+            memcpy(slave->sii.strings[i], data + offset + 1, size);
+            slave->sii.strings[i][size] = 0x00; // append binary zero
+            offset += 1 + size;
+        }
+    }
+
+    return 0;
+
+out_free:
+    for (i--; i >= 0; i--)
+        kfree(slave->sii.strings[i]);
+    kfree(slave->sii.strings);
+    slave->sii.strings = NULL;
+out_zero:
+    slave->sii.string_count = 0;
+    return err;
+}
+
+/****************************************************************************/
+
+/**
+   Fetches data from a GENERAL category.
+   \return 0 in case of success, else < 0
+*/
+
+int ec_slave_fetch_sii_general(
+        ec_slave_t *slave, /**< EtherCAT slave */
+        const uint8_t *data, /**< category data */
+        size_t data_size /**< size in bytes */
+        )
+{
+    unsigned int i;
+    uint8_t flags;
+
+    if (data_size != 32) {
+        EC_SLAVE_ERR(slave, "Wrong size of general category (%zu/32).\n",
+                data_size);
+        return -EINVAL;
+    }
+
+    slave->sii.group = ec_slave_sii_string(slave, data[0]);
+    slave->sii.image = ec_slave_sii_string(slave, data[1]);
+    slave->sii.order = ec_slave_sii_string(slave, data[2]);
+    slave->sii.name = ec_slave_sii_string(slave, data[3]);
+
+    for (i = 0; i < 4; i++)
+        slave->sii.physical_layer[i] =
+            (data[4] & (0x03 << (i * 2))) >> (i * 2);
+
+    // read CoE details
+    flags = EC_READ_U8(data + 5);
+    slave->sii.coe_details.enable_sdo =                 (flags >> 0) & 0x01;
+    slave->sii.coe_details.enable_sdo_info =            (flags >> 1) & 0x01;
+    slave->sii.coe_details.enable_pdo_assign =          (flags >> 2) & 0x01;
+    slave->sii.coe_details.enable_pdo_configuration =   (flags >> 3) & 0x01;
+    slave->sii.coe_details.enable_upload_at_startup =   (flags >> 4) & 0x01;
+    slave->sii.coe_details.enable_sdo_complete_access = (flags >> 5) & 0x01;
+
+    // read general flags
+    flags = EC_READ_U8(data + 0x000B);
+    slave->sii.general_flags.enable_safeop =  (flags >> 0) & 0x01;
+    slave->sii.general_flags.enable_not_lrw = (flags >> 1) & 0x01;
+
+    slave->sii.current_on_ebus = EC_READ_S16(data + 0x0C);
+    slave->sii.has_general = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Fetches data from a SYNC MANAGER category.
+ *
+ * Appends the sync managers described in the category to the existing ones.
+ *
+ * \return 0 in case of success, else < 0
+ */
+int ec_slave_fetch_sii_syncs(
+        ec_slave_t *slave, /**< EtherCAT slave. */
+        const uint8_t *data, /**< Category data. */
+        size_t data_size /**< Number of bytes. */
+        )
+{
+    unsigned int i, count, total_count;
+    ec_sync_t *sync;
+    size_t memsize;
+    ec_sync_t *syncs;
+    uint8_t index;
+
+    // one sync manager struct is 4 words long
+    if (data_size % 8) {
+        EC_SLAVE_ERR(slave, "Invalid SII sync manager category size %zu.\n",
+                data_size);
+        return -EINVAL;
+    }
+
+    count = data_size / 8;
+
+    if (count) {
+        total_count = count + slave->sii.sync_count;
+        if (total_count > EC_MAX_SYNC_MANAGERS) {
+            EC_SLAVE_ERR(slave, "Exceeded maximum number of"
+                    " sync managers!\n");
+            return -EOVERFLOW;
+        }
+        memsize = sizeof(ec_sync_t) * total_count;
+        if (!(syncs = kmalloc(memsize, GFP_KERNEL))) {
+            EC_SLAVE_ERR(slave, "Failed to allocate %zu bytes"
+                    " for sync managers.\n", memsize);
+            return -ENOMEM;
+        }
+
+        for (i = 0; i < slave->sii.sync_count; i++)
+            ec_sync_init_copy(syncs + i, slave->sii.syncs + i);
+
+        // initialize new sync managers
+        for (i = 0; i < count; i++, data += 8) {
+            index = i + slave->sii.sync_count;
+            sync = &syncs[index];
+
+            ec_sync_init(sync, slave);
+            sync->physical_start_address = EC_READ_U16(data);
+            sync->default_length = EC_READ_U16(data + 2);
+            sync->control_register = EC_READ_U8(data + 4);
+            sync->enable = EC_READ_U8(data + 6);
+        }
+
+        if (slave->sii.syncs)
+            kfree(slave->sii.syncs);
+        slave->sii.syncs = syncs;
+        slave->sii.sync_count = total_count;
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/**
+   Fetches data from a [RT]xPDO category.
+   \return 0 in case of success, else < 0
+*/
+
+int ec_slave_fetch_sii_pdos(
+        ec_slave_t *slave, /**< EtherCAT slave */
+        const uint8_t *data, /**< category data */
+        size_t data_size, /**< number of bytes */
+        ec_direction_t dir /**< PDO direction. */
+        )
+{
+    int ret;
+    ec_pdo_t *pdo;
+    ec_pdo_entry_t *entry;
+    unsigned int entry_count, i;
+
+    while (data_size >= 8) {
+        if (!(pdo = kmalloc(sizeof(ec_pdo_t), GFP_KERNEL))) {
+            EC_SLAVE_ERR(slave, "Failed to allocate PDO memory.\n");
+            return -ENOMEM;
+        }
+
+        ec_pdo_init(pdo);
+        pdo->index = EC_READ_U16(data);
+        entry_count = EC_READ_U8(data + 2);
+        pdo->sync_index = EC_READ_U8(data + 3);
+        ret = ec_pdo_set_name(pdo,
+                ec_slave_sii_string(slave, EC_READ_U8(data + 5)));
+        if (ret) {
+            ec_pdo_clear(pdo);
+            kfree(pdo);
+            return ret;
+        }
+        list_add_tail(&pdo->list, &slave->sii.pdos);
+
+        data_size -= 8;
+        data += 8;
+
+        for (i = 0; i < entry_count; i++) {
+            if (!(entry = kmalloc(sizeof(ec_pdo_entry_t), GFP_KERNEL))) {
+                EC_SLAVE_ERR(slave, "Failed to allocate PDO entry memory.\n");
+                return -ENOMEM;
+            }
+
+            ec_pdo_entry_init(entry);
+            entry->index = EC_READ_U16(data);
+            entry->subindex = EC_READ_U8(data + 2);
+            ret = ec_pdo_entry_set_name(entry,
+                    ec_slave_sii_string(slave, EC_READ_U8(data + 3)));
+            if (ret) {
+                ec_pdo_entry_clear(entry);
+                kfree(entry);
+                return ret;
+            }
+            entry->bit_length = EC_READ_U8(data + 5);
+            list_add_tail(&entry->list, &pdo->entries);
+
+            data_size -= 8;
+            data += 8;
+        }
+
+        // if sync manager index is positive, the PDO is mapped by default
+        if (pdo->sync_index >= 0) {
+            ec_sync_t *sync;
+
+            if (!(sync = ec_slave_get_sync(slave, pdo->sync_index))) {
+                EC_SLAVE_ERR(slave, "Invalid SM index %i for PDO 0x%04X.",
+                        pdo->sync_index, pdo->index);
+                return -ENOENT;
+            }
+
+            ret = ec_pdo_list_add_pdo_copy(&sync->pdos, pdo);
+            if (ret)
+                return ret;
+        }
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/**
+   Searches the string list for an index.
+   \return 0 in case of success, else < 0
+*/
+
+char *ec_slave_sii_string(
+        const ec_slave_t *slave, /**< EtherCAT slave */
+        unsigned int index /**< string index */
+        )
+{
+    if (!index--)
+        return NULL;
+
+    if (index >= slave->sii.string_count) {
+        EC_SLAVE_DBG(slave, 1, "String %u not found.\n", index);
+        return NULL;
+    }
+
+    return slave->sii.strings[index];
+}
+
+/****************************************************************************/
+
+/** Get the sync manager given an index.
+ *
+ * \return pointer to sync manager, or NULL.
+ */
+ec_sync_t *ec_slave_get_sync(
+        ec_slave_t *slave, /**< EtherCAT slave. */
+        uint8_t sync_index /**< Sync manager index. */
+        )
+{
+    if (sync_index < slave->sii.sync_count) {
+        return &slave->sii.syncs[sync_index];
+    } else {
+        return NULL;
+    }
+}
+
+/****************************************************************************/
+
+/**
+   Counts the total number of SDOs and entries in the dictionary.
+*/
+
+void ec_slave_sdo_dict_info(const ec_slave_t *slave, /**< EtherCAT slave */
+                            unsigned int *sdo_count, /**< number of SDOs */
+                            unsigned int *entry_count /**< total number of
+                                                         entries */
+                            )
+{
+    unsigned int sdos = 0, entries = 0;
+    ec_sdo_t *sdo;
+    ec_sdo_entry_t *entry;
+
+    list_for_each_entry(sdo, &slave->sdo_dictionary, list) {
+        sdos++;
+        list_for_each_entry(entry, &sdo->entries, list) {
+            entries++;
+        }
+    }
+
+    *sdo_count = sdos;
+    *entry_count = entries;
+}
+
+/****************************************************************************/
+
+/**
+ * Get an SDO from the dictionary.
+ * \returns The desired SDO, or NULL.
+ */
+
+ec_sdo_t *ec_slave_get_sdo(
+        ec_slave_t *slave, /**< EtherCAT slave */
+        uint16_t index /**< SDO index */
+        )
+{
+    ec_sdo_t *sdo;
+
+    list_for_each_entry(sdo, &slave->sdo_dictionary, list) {
+        if (sdo->index != index)
+            continue;
+        return sdo;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/**
+ * Get an SDO from the dictionary.
+ *
+ * const version.
+ *
+ * \returns The desired SDO, or NULL.
+ */
+
+const ec_sdo_t *ec_slave_get_sdo_const(
+        const ec_slave_t *slave, /**< EtherCAT slave */
+        uint16_t index /**< SDO index */
+        )
+{
+    const ec_sdo_t *sdo;
+
+    list_for_each_entry(sdo, &slave->sdo_dictionary, list) {
+        if (sdo->index != index)
+            continue;
+        return sdo;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Get an SDO from the dictionary, given its position in the list.
+ * \returns The desired SDO, or NULL.
+ */
+
+const ec_sdo_t *ec_slave_get_sdo_by_pos_const(
+        const ec_slave_t *slave, /**< EtherCAT slave. */
+        uint16_t sdo_position /**< SDO list position. */
+        )
+{
+    const ec_sdo_t *sdo;
+
+    list_for_each_entry(sdo, &slave->sdo_dictionary, list) {
+        if (sdo_position--)
+            continue;
+        return sdo;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Get the number of SDOs in the dictionary.
+ * \returns SDO count.
+ */
+
+uint16_t ec_slave_sdo_count(
+        const ec_slave_t *slave /**< EtherCAT slave. */
+        )
+{
+    const ec_sdo_t *sdo;
+    uint16_t count = 0;
+
+    list_for_each_entry(sdo, &slave->sdo_dictionary, list) {
+        count++;
+    }
+
+    return count;
+}
+
+/****************************************************************************/
+
+/** Finds a mapped PDO.
+ * \returns The desired PDO object, or NULL.
+ */
+const ec_pdo_t *ec_slave_find_pdo(
+        const ec_slave_t *slave, /**< Slave. */
+        uint16_t index /**< PDO index to find. */
+        )
+{
+    unsigned int i;
+    const ec_sync_t *sync;
+    const ec_pdo_t *pdo;
+
+    for (i = 0; i < slave->sii.sync_count; i++) {
+        sync = &slave->sii.syncs[i];
+
+        if (!(pdo = ec_pdo_list_find_pdo_const(&sync->pdos, index)))
+            continue;
+
+        return pdo;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Find name for a PDO and its entries.
+ */
+void ec_slave_find_names_for_pdo(
+        ec_slave_t *slave,
+        ec_pdo_t *pdo
+        )
+{
+    const ec_sdo_t *sdo;
+    ec_pdo_entry_t *pdo_entry;
+    const ec_sdo_entry_t *sdo_entry;
+
+    list_for_each_entry(sdo, &slave->sdo_dictionary, list) {
+        if (sdo->index == pdo->index) {
+            ec_pdo_set_name(pdo, sdo->name);
+        } else {
+            list_for_each_entry(pdo_entry, &pdo->entries, list) {
+                if (sdo->index == pdo_entry->index) {
+                    sdo_entry = ec_sdo_get_entry_const(
+                            sdo, pdo_entry->subindex);
+                    if (sdo_entry) {
+                        ec_pdo_entry_set_name(pdo_entry,
+                                sdo_entry->description);
+                    }
+                }
+            }
+        }
+    }
+}
+
+/****************************************************************************/
+
+/** Attach PDO names.
+ */
+void ec_slave_attach_pdo_names(
+        ec_slave_t *slave
+        )
+{
+    unsigned int i;
+    ec_sync_t *sync;
+    ec_pdo_t *pdo;
+
+    for (i = 0; i < slave->sii.sync_count; i++) {
+        sync = slave->sii.syncs + i;
+        list_for_each_entry(pdo, &sync->pdos.list, list) {
+            ec_slave_find_names_for_pdo(slave, pdo);
+        }
+    }
+}
+
+/****************************************************************************/
+
+/** Returns the previous connected port of a given port.
+ *
+ * \return Port index.
+ */
+unsigned int ec_slave_get_previous_port(
+        const ec_slave_t *slave, /**< EtherCAT slave. */
+        unsigned int port_index /**< Port index. */
+        )
+{
+    static const unsigned int prev_table[EC_MAX_PORTS] = {
+        2, 3, 1, 0
+    };
+
+    if (port_index >= EC_MAX_PORTS) {
+        EC_SLAVE_WARN(slave, "%s(port_index=%u): Invalid port index!\n",
+                __func__, port_index);
+    }
+
+    do {
+        port_index = prev_table[port_index];
+        if (slave->ports[port_index].next_slave) {
+            return port_index;
+        }
+    } while (port_index);
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Returns the next connected port of a given port.
+ *
+ * \return Port index.
+ */
+unsigned int ec_slave_get_next_port(
+        const ec_slave_t *slave, /**< EtherCAT slave. */
+        unsigned int port_index /**< Port index. */
+        )
+{
+    static const unsigned int next_table[EC_MAX_PORTS] = {
+        3, 2, 0, 1
+    };
+
+    if (port_index >= EC_MAX_PORTS) {
+        EC_SLAVE_WARN(slave, "%s(port_index=%u): Invalid port index!\n",
+                __func__, port_index);
+    }
+
+    do {
+        port_index = next_table[port_index];
+        if (slave->ports[port_index].next_slave) {
+            return port_index;
+        }
+    } while (port_index);
+
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Calculates the sum of round-trip-times of connected ports 1-3.
+ *
+ * \return Round-trip-time in ns.
+ */
+uint32_t ec_slave_calc_rtt_sum(
+        const ec_slave_t *slave /**< EtherCAT slave. */
+        )
+{
+    uint32_t rtt_sum = 0, rtt;
+    unsigned int port_index = ec_slave_get_next_port(slave, 0);
+
+    while (port_index != 0) {
+        unsigned int prev_index =
+            ec_slave_get_previous_port(slave, port_index);
+
+        rtt = slave->ports[port_index].receive_time -
+            slave->ports[prev_index].receive_time;
+        rtt_sum += rtt;
+        port_index = ec_slave_get_next_port(slave, port_index);
+    }
+
+    return rtt_sum;
+}
+
+/****************************************************************************/
+
+/** Finds the next slave supporting DC delay measurement.
+ *
+ * \return Next DC slave, or NULL.
+ */
+ec_slave_t *ec_slave_find_next_dc_slave(
+        ec_slave_t *slave /**< EtherCAT slave. */
+        )
+{
+    unsigned int port_index;
+    ec_slave_t *dc_slave = NULL;
+
+    if (slave->base_dc_supported) {
+        dc_slave = slave;
+    } else {
+        port_index = ec_slave_get_next_port(slave, 0);
+
+        while (port_index != 0) {
+            ec_slave_t *next = slave->ports[port_index].next_slave;
+
+            if (next) {
+                dc_slave = ec_slave_find_next_dc_slave(next);
+
+                if (dc_slave) {
+                    break;
+                }
+            }
+            port_index = ec_slave_get_next_port(slave, port_index);
+        }
+    }
+
+    return dc_slave;
+}
+
+/****************************************************************************/
+
+/** Calculates the port transmission delays.
+ */
+void ec_slave_calc_port_delays(
+        ec_slave_t *slave /**< EtherCAT slave. */
+        )
+{
+    unsigned int port_index;
+    ec_slave_t *next_slave, *next_dc;
+    uint32_t rtt, next_rtt_sum;
+
+    if (!slave->base_dc_supported)
+        return;
+
+    port_index = ec_slave_get_next_port(slave, 0);
+
+    while (port_index != 0) {
+        next_slave = slave->ports[port_index].next_slave;
+        next_dc = ec_slave_find_next_dc_slave(next_slave);
+
+        if (next_dc) {
+            unsigned int prev_port =
+                ec_slave_get_previous_port(slave, port_index);
+
+            rtt = slave->ports[port_index].receive_time -
+                slave->ports[prev_port].receive_time;
+            next_rtt_sum = ec_slave_calc_rtt_sum(next_dc);
+
+            slave->ports[port_index].delay_to_next_dc =
+                (rtt - next_rtt_sum) / 2; // FIXME
+            next_dc->ports[0].delay_to_next_dc =
+                (rtt - next_rtt_sum) / 2;
+
+#if 0
+            EC_SLAVE_DBG(slave, 1, "delay %u:%u rtt=%u"
+                    " next_rtt_sum=%u delay=%u\n",
+                    slave->ring_position, port_index, rtt, next_rtt_sum,
+                    slave->ports[port_index].delay_to_next_dc);
+#endif
+        }
+
+        port_index = ec_slave_get_next_port(slave, port_index);
+    }
+}
+
+/****************************************************************************/
+
+/** Recursively calculates transmission delays.
+ */
+void ec_slave_calc_transmission_delays_rec(
+        ec_slave_t *slave, /**< Current slave. */
+        uint32_t *delay /**< Sum of delays. */
+        )
+{
+    unsigned int i;
+    ec_slave_t *next_dc;
+
+    EC_SLAVE_DBG(slave, 1, "%s(delay = %u ns)\n", __func__, *delay);
+
+    slave->transmission_delay = *delay;
+
+    i = ec_slave_get_next_port(slave, 0);
+
+    while (i != 0) {
+        ec_slave_port_t *port = &slave->ports[i];
+        next_dc = ec_slave_find_next_dc_slave(port->next_slave);
+        if (next_dc) {
+            *delay = *delay + port->delay_to_next_dc;
+#if 0
+            EC_SLAVE_DBG(slave, 1, "%u:%u %u\n",
+                    slave->ring_position, i, *delay);
+#endif
+            ec_slave_calc_transmission_delays_rec(next_dc, delay);
+        }
+
+        i = ec_slave_get_next_port(slave, i);
+    }
+
+    *delay = *delay + slave->ports[0].delay_to_next_dc;
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/slave.h b/drivers/net/ethercat/master/slave.h
new file mode 100644
index 000000000000..4e113e288e1a
--- /dev/null
+++ b/drivers/net/ethercat/master/slave.h
@@ -0,0 +1,266 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2012  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_SLAVE_H__
+#define __EC_SLAVE_H__
+
+#include <linux/list.h>
+#include <linux/kobject.h>
+
+#include "globals.h"
+#include "datagram.h"
+#include "pdo.h"
+#include "sync.h"
+#include "sdo.h"
+#include "fsm_slave.h"
+
+/****************************************************************************/
+
+/** Convenience macro for printing slave-specific information to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX>-<POSITION>: ", where INDEX is the master index and
+ * POSITION is the slave's ring position.
+ *
+ * \param slave EtherCAT slave
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_SLAVE_INFO(slave, fmt, args...) \
+    printk(KERN_INFO "EtherCAT %u-%u: " fmt, slave->master->index, \
+            slave->ring_position, ##args)
+
+/** Convenience macro for printing slave-specific errors to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX>-<POSITION>: ", where INDEX is the master index and
+ * POSITION is the slave's ring position.
+ *
+ * \param slave EtherCAT slave
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_SLAVE_ERR(slave, fmt, args...) \
+    printk(KERN_ERR "EtherCAT ERROR %u-%u: " fmt, slave->master->index, \
+            slave->ring_position, ##args)
+
+/** Convenience macro for printing slave-specific warnings to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX>-<POSITION>: ", where INDEX is the master index and
+ * POSITION is the slave's ring position.
+ *
+ * \param slave EtherCAT slave
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_SLAVE_WARN(slave, fmt, args...) \
+    printk(KERN_WARNING "EtherCAT WARNING %u-%u: " fmt, \
+            slave->master->index, slave->ring_position, ##args)
+
+/** Convenience macro for printing slave-specific debug messages to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX>-<POSITION>: ", where INDEX is the master index and
+ * POSITION is the slave's ring position.
+ *
+ * \param slave EtherCAT slave
+ * \param level Debug level. Master's debug level must be >= \a level for
+ * output.
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_SLAVE_DBG(slave, level, fmt, args...) \
+    do { \
+        if (slave->master->debug_level >= level) { \
+            printk(KERN_DEBUG "EtherCAT DEBUG %u-%u: " fmt, \
+                    slave->master->index, slave->ring_position, ##args); \
+        } \
+    } while (0)
+
+/****************************************************************************/
+
+/** Slave port.
+ */
+typedef struct {
+    ec_slave_port_desc_t desc; /**< Port descriptors. */
+    ec_slave_port_link_t link; /**< Port link status. */
+    ec_slave_t *next_slave; /**< Connected slaves. */
+    uint32_t receive_time; /**< Port receive times for delay
+                                            measurement. */
+    uint32_t delay_to_next_dc; /**< Delay to next slave with DC support behind
+                                 this port [ns]. */
+} ec_slave_port_t;
+
+/****************************************************************************/
+
+/** Slave information interface data.
+ */
+typedef struct {
+    // Non-category data
+    uint16_t alias; /**< Configured station alias. */
+    uint32_t vendor_id; /**< Vendor ID. */
+    uint32_t product_code; /**< Vendor-specific product code. */
+    uint32_t revision_number; /**< Revision number. */
+    uint32_t serial_number; /**< Serial number. */
+    uint16_t boot_rx_mailbox_offset; /**< Bootstrap receive mailbox address. */
+    uint16_t boot_rx_mailbox_size; /**< Bootstrap receive mailbox size. */
+    uint16_t boot_tx_mailbox_offset; /**< Bootstrap transmit mailbox address. */
+    uint16_t boot_tx_mailbox_size; /**< Bootstrap transmit mailbox size. */
+    uint16_t std_rx_mailbox_offset; /**< Standard receive mailbox address. */
+    uint16_t std_rx_mailbox_size; /**< Standard receive mailbox size. */
+    uint16_t std_tx_mailbox_offset; /**< Standard transmit mailbox address. */
+    uint16_t std_tx_mailbox_size; /**< Standard transmit mailbox size. */
+    uint16_t mailbox_protocols; /**< Supported mailbox protocols. */
+
+    // Strings
+    char **strings; /**< Strings in SII categories. */
+    unsigned int string_count; /**< Number of SII strings. */
+
+    // General
+    unsigned int has_general; /**< General category present. */
+    char *group; /**< Group name. */
+    char *image; /**< Image name. */
+    char *order; /**< Order number. */
+    char *name; /**< Slave name. */
+    uint8_t physical_layer[EC_MAX_PORTS]; /**< Port media. */
+    ec_sii_coe_details_t coe_details; /**< CoE detail flags. */
+    ec_sii_general_flags_t general_flags; /**< General flags. */
+    int16_t current_on_ebus; /**< Power consumption in mA. */
+
+    // SyncM
+    ec_sync_t *syncs; /**< SYNC MANAGER categories. */
+    unsigned int sync_count; /**< Number of sync managers. */
+
+    // [RT]XPDO
+    struct list_head pdos; /**< SII [RT]XPDO categories. */
+} ec_sii_t;
+
+/****************************************************************************/
+
+/** EtherCAT slave.
+ */
+struct ec_slave
+{
+    ec_master_t *master; /**< Master owning the slave. */
+    ec_device_index_t device_index; /**< Index of device the slave responds
+                                      on. */
+
+    // addresses
+    uint16_t ring_position; /**< Ring position. */
+    uint16_t station_address; /**< Configured station address. */
+    uint16_t effective_alias; /**< Effective alias address. */
+
+    ec_slave_port_t ports[EC_MAX_PORTS]; /**< Ports. */
+
+    // configuration
+    ec_slave_config_t *config; /**< Current configuration. */
+    ec_slave_state_t requested_state; /**< Requested application state. */
+    ec_slave_state_t current_state; /**< Current application state. */
+    unsigned int error_flag; /**< Stop processing after an error. */
+    unsigned int force_config; /**< Force (re-)configuration. */
+    uint16_t configured_rx_mailbox_offset; /**< Configured receive mailbox
+                                             offset. */
+    uint16_t configured_rx_mailbox_size; /**< Configured receive mailbox size.
+                                          */
+    uint16_t configured_tx_mailbox_offset; /**< Configured send mailbox
+                                             offset. */
+    uint16_t configured_tx_mailbox_size; /**< Configured send mailbox size. */
+
+    // base data
+    uint8_t base_type; /**< Slave type. */
+    uint8_t base_revision; /**< Revision. */
+    uint16_t base_build; /**< Build number. */
+    uint8_t base_fmmu_count; /**< Number of supported FMMUs. */
+    uint8_t base_sync_count; /**< Number of supported sync managers. */
+    uint8_t base_fmmu_bit_operation; /**< FMMU bit operation is supported. */
+    uint8_t base_dc_supported; /**< Distributed clocks are supported. */
+    ec_slave_dc_range_t base_dc_range; /**< DC range. */
+    uint8_t has_dc_system_time; /**< The slave supports the DC system time
+                                  register. Otherwise it can only be used for
+                                  delay measurement. */
+    uint32_t transmission_delay; /**< DC system time transmission delay
+                                   (offset from reference clock). */
+
+    // SII
+    uint16_t *sii_words; /**< Complete SII image. */
+    size_t sii_nwords; /**< Size of the SII contents in words. */
+
+    // Slave information interface
+    ec_sii_t sii; /**< Extracted SII data. */
+
+    struct list_head sdo_dictionary; /**< SDO dictionary list */
+    uint8_t sdo_dictionary_fetched; /**< Dictionary has been fetched. */
+    unsigned long jiffies_preop; /**< Time, the slave went to PREOP. */
+
+    struct list_head sdo_requests; /**< SDO access requests. */
+    struct list_head reg_requests; /**< Register access requests. */
+    struct list_head foe_requests; /**< FoE requests. */
+    struct list_head soe_requests; /**< SoE requests. */
+    struct list_head eoe_requests; /**< EoE set IP parameter requests. */
+
+    ec_fsm_slave_t fsm; /**< Slave state machine. */
+};
+
+/****************************************************************************/
+
+// slave construction/destruction
+void ec_slave_init(ec_slave_t *, ec_master_t *, ec_device_index_t,
+        uint16_t, uint16_t);
+void ec_slave_clear(ec_slave_t *);
+
+void ec_slave_clear_sync_managers(ec_slave_t *);
+
+void ec_slave_request_state(ec_slave_t *, ec_slave_state_t);
+void ec_slave_set_state(ec_slave_t *, ec_slave_state_t);
+
+// SII categories
+int ec_slave_fetch_sii_strings(ec_slave_t *, const uint8_t *, size_t);
+int ec_slave_fetch_sii_general(ec_slave_t *, const uint8_t *, size_t);
+int ec_slave_fetch_sii_syncs(ec_slave_t *, const uint8_t *, size_t);
+int ec_slave_fetch_sii_pdos(ec_slave_t *, const uint8_t *, size_t,
+        ec_direction_t);
+
+// misc.
+ec_sync_t *ec_slave_get_sync(ec_slave_t *, uint8_t);
+
+void ec_slave_sdo_dict_info(const ec_slave_t *,
+        unsigned int *, unsigned int *);
+ec_sdo_t *ec_slave_get_sdo(ec_slave_t *, uint16_t);
+const ec_sdo_t *ec_slave_get_sdo_const(const ec_slave_t *, uint16_t);
+const ec_sdo_t *ec_slave_get_sdo_by_pos_const(const ec_slave_t *, uint16_t);
+uint16_t ec_slave_sdo_count(const ec_slave_t *);
+const ec_pdo_t *ec_slave_find_pdo(const ec_slave_t *, uint16_t);
+void ec_slave_attach_pdo_names(ec_slave_t *);
+
+void ec_slave_calc_port_delays(ec_slave_t *);
+void ec_slave_calc_transmission_delays_rec(ec_slave_t *, uint32_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/slave_config.c b/drivers/net/ethercat/master/slave_config.c
new file mode 100644
index 000000000000..87e81a6c8895
--- /dev/null
+++ b/drivers/net/ethercat/master/slave_config.c
@@ -0,0 +1,1652 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *  vim: expandtab
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave configuration methods.
+*/
+
+/****************************************************************************/
+
+#include "slave_config.h"
+
+#include "globals.h"
+#include "master.h"
+#include "voe_handler.h"
+#include "flag.h"
+#include "ioctl.h"
+
+#ifdef EC_EOE
+#include "eoe_request.h"
+#endif
+
+#include <linux/module.h>
+#include <linux/slab.h>
+
+/****************************************************************************/
+
+// prototypes for private methods
+int ec_slave_config_prepare_fmmu(ec_slave_config_t *, ec_domain_t *, uint8_t,
+        ec_direction_t);
+void ec_slave_config_load_default_mapping(const ec_slave_config_t *,
+        ec_pdo_t *);
+
+/****************************************************************************/
+
+/** EtherCAT application-layer transition timeout.
+ */
+typedef struct {
+    struct list_head list;
+    ec_slave_state_t from;
+    ec_slave_state_t to;
+    unsigned int timeout_ms;
+} ec_al_timeout_t;
+
+/****************************************************************************/
+
+/** Slave configuration constructor.
+ *
+ * See ecrt_master_slave_config() for the usage of the \a alias and \a
+ * position parameters.
+ */
+void ec_slave_config_init(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        ec_master_t *master, /**< EtherCAT master. */
+        uint16_t alias, /**< Slave alias. */
+        uint16_t position, /**< Slave position. */
+        uint32_t vendor_id, /**< Expected vendor ID. */
+        uint32_t product_code /**< Expected product code. */
+        )
+{
+    unsigned int i;
+
+    sc->master = master;
+
+    sc->alias = alias;
+    sc->position = position;
+    sc->vendor_id = vendor_id;
+    sc->product_code = product_code;
+    sc->watchdog_divider = 0; // use default
+    sc->watchdog_intervals = 0; // use default
+
+    sc->slave = NULL;
+
+    for (i = 0; i < EC_MAX_SYNC_MANAGERS; i++)
+        ec_sync_config_init(&sc->sync_configs[i]);
+
+    sc->used_fmmus = 0;
+    sc->dc_assign_activate = 0x0000;
+    sc->dc_sync[0].cycle_time = 0U;
+    sc->dc_sync[1].cycle_time = 0;
+    sc->dc_sync[0].shift_time = 0U;
+    sc->dc_sync[1].shift_time = 0;
+
+    INIT_LIST_HEAD(&sc->sdo_configs);
+    INIT_LIST_HEAD(&sc->sdo_requests);
+    INIT_LIST_HEAD(&sc->soe_requests);
+    INIT_LIST_HEAD(&sc->reg_requests);
+    INIT_LIST_HEAD(&sc->voe_handlers);
+    INIT_LIST_HEAD(&sc->soe_configs);
+    INIT_LIST_HEAD(&sc->flags);
+    INIT_LIST_HEAD(&sc->al_timeouts);
+
+#ifdef EC_EOE
+    ec_eoe_request_init(&sc->eoe_ip_param_request);
+#endif
+
+    ec_coe_emerg_ring_init(&sc->emerg_ring, sc);
+}
+
+/****************************************************************************/
+
+/** Slave configuration destructor.
+ *
+ * Clears and frees a slave configuration object.
+ */
+void ec_slave_config_clear(
+        ec_slave_config_t *sc /**< Slave configuration. */
+        )
+{
+    unsigned int i;
+    ec_sdo_request_t *req, *next_req;
+    ec_voe_handler_t *voe, *next_voe;
+    ec_reg_request_t *reg, *next_reg;
+    ec_soe_request_t *soe, *next_soe;
+    ec_flag_t *flag, *next_flag;
+    ec_al_timeout_t *timeout, *next_timeout;
+
+    ec_slave_config_detach(sc);
+
+    // Free sync managers
+    for (i = 0; i < EC_MAX_SYNC_MANAGERS; i++)
+        ec_sync_config_clear(&sc->sync_configs[i]);
+
+    // free all SDO configurations
+    list_for_each_entry_safe(req, next_req, &sc->sdo_configs, list) {
+        list_del(&req->list);
+        ec_sdo_request_clear(req);
+        kfree(req);
+    }
+
+    // free all SDO requests
+    list_for_each_entry_safe(req, next_req, &sc->sdo_requests, list) {
+        list_del(&req->list);
+        ec_sdo_request_clear(req);
+        kfree(req);
+    }
+
+    // free all SoE requests
+    list_for_each_entry_safe(soe, next_soe, &sc->soe_requests, list) {
+        list_del(&soe->list);
+        ec_soe_request_clear(soe);
+        kfree(soe);
+    }
+
+    // free all register requests
+    list_for_each_entry_safe(reg, next_reg, &sc->reg_requests, list) {
+        list_del(&reg->list);
+        ec_reg_request_clear(reg);
+        kfree(reg);
+    }
+
+    // free all VoE handlers
+    list_for_each_entry_safe(voe, next_voe, &sc->voe_handlers, list) {
+        list_del(&voe->list);
+        ec_voe_handler_clear(voe);
+        kfree(voe);
+    }
+
+    // free all SoE configurations
+    list_for_each_entry_safe(soe, next_soe, &sc->soe_configs, list) {
+        list_del(&soe->list);
+        ec_soe_request_clear(soe);
+        kfree(soe);
+    }
+
+    // free all flags
+    list_for_each_entry_safe(flag, next_flag, &sc->flags, list) {
+        list_del(&flag->list);
+        ec_flag_clear(flag);
+        kfree(flag);
+    }
+
+    // free all AL timeouts
+    list_for_each_entry_safe(timeout, next_timeout, &sc->al_timeouts, list) {
+        list_del(&timeout->list);
+        kfree(timeout);
+    }
+
+    ec_coe_emerg_ring_clear(&sc->emerg_ring);
+}
+
+/****************************************************************************/
+
+/** Prepares an FMMU configuration.
+ *
+ * Configuration data for the FMMU is saved in the slave config structure and
+ * is written to the slave during the configuration. The FMMU configuration
+ * is done in a way, that the complete data range of the corresponding sync
+ * manager is covered. Seperate FMMUs are configured for each domain. If the
+ * FMMU configuration is already prepared, the function does nothing and
+ * returns with success.
+ *
+ * \retval >=0 Success, logical offset byte address.
+ * \retval  <0 Error code.
+ */
+int ec_slave_config_prepare_fmmu(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        ec_domain_t *domain, /**< Domain. */
+        uint8_t sync_index, /**< Sync manager index. */
+        ec_direction_t dir /**< PDO direction. */
+        )
+{
+    unsigned int i;
+    ec_fmmu_config_t *fmmu;
+
+    // FMMU configuration already prepared?
+    for (i = 0; i < sc->used_fmmus; i++) {
+        fmmu = &sc->fmmu_configs[i];
+        if (fmmu->domain == domain && fmmu->sync_index == sync_index)
+            return fmmu->logical_start_address;
+    }
+
+    if (sc->used_fmmus == EC_MAX_FMMUS) {
+        EC_CONFIG_ERR(sc, "FMMU limit reached!\n");
+        return -EOVERFLOW;
+    }
+
+    fmmu = &sc->fmmu_configs[sc->used_fmmus++];
+
+    down(&sc->master->master_sem);
+    ec_fmmu_config_init(fmmu, sc, domain, sync_index, dir);
+    up(&sc->master->master_sem);
+
+    return fmmu->logical_start_address;
+}
+
+/****************************************************************************/
+
+/** Attaches the configuration to the addressed slave object.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_slave_config_attach(
+        ec_slave_config_t *sc /**< Slave configuration. */
+        )
+{
+    ec_slave_t *slave;
+
+    if (sc->slave)
+        return 0; // already attached
+
+    if (!(slave = ec_master_find_slave(
+                    sc->master, sc->alias, sc->position))) {
+        EC_CONFIG_DBG(sc, 1, "Failed to find slave for configuration.\n");
+        return -ENOENT;
+    }
+
+    if (slave->config) {
+        EC_CONFIG_DBG(sc, 1, "Failed to attach configuration. Slave %u"
+                " already has a configuration!\n", slave->ring_position);
+        return -EEXIST;
+    }
+
+    if (
+#ifdef EC_IDENT_WILDCARDS
+            sc->vendor_id != 0xffffffff &&
+#endif
+            slave->sii.vendor_id != sc->vendor_id
+       ) {
+        EC_CONFIG_DBG(sc, 1, "Slave %u has no matching vendor ID (0x%08X)"
+                " for configuration (0x%08X).\n",
+                slave->ring_position, slave->sii.vendor_id, sc->vendor_id);
+        return -EINVAL;
+    }
+
+    if (
+#ifdef EC_IDENT_WILDCARDS
+            sc->product_code != 0xffffffff &&
+#endif
+            slave->sii.product_code != sc->product_code
+       ) {
+        EC_CONFIG_DBG(sc, 1, "Slave %u has no matching product code (0x%08X)"
+                " for configuration (0x%08X).\n",
+                slave->ring_position, slave->sii.product_code,
+                sc->product_code);
+        return -EINVAL;
+    }
+
+    // attach slave
+    slave->config = sc;
+    sc->slave = slave;
+
+    EC_CONFIG_DBG(sc, 1, "Attached slave %u.\n", slave->ring_position);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Detaches the configuration from a slave object.
+ */
+void ec_slave_config_detach(
+        ec_slave_config_t *sc /**< Slave configuration. */
+        )
+{
+    if (sc->slave) {
+        ec_reg_request_t *reg;
+
+        sc->slave->config = NULL;
+
+        // invalidate processing register request
+        list_for_each_entry(reg, &sc->reg_requests, list) {
+            if (sc->slave->fsm.reg_request == reg) {
+                sc->slave->fsm.reg_request = NULL;
+                break;
+            }
+        }
+
+        sc->slave = NULL;
+    }
+}
+
+/****************************************************************************/
+
+/** Loads the default PDO assignment from the slave object.
+ */
+void ec_slave_config_load_default_sync_config(ec_slave_config_t *sc)
+{
+    uint8_t sync_index;
+    ec_sync_config_t *sync_config;
+    const ec_sync_t *sync;
+
+    if (!sc->slave)
+        return;
+
+    for (sync_index = 0; sync_index < EC_MAX_SYNC_MANAGERS; sync_index++) {
+        sync_config = &sc->sync_configs[sync_index];
+        if ((sync = ec_slave_get_sync(sc->slave, sync_index))) {
+            sync_config->dir = ec_sync_default_direction(sync);
+            if (sync_config->dir == EC_DIR_INVALID)
+                EC_SLAVE_WARN(sc->slave,
+                        "SM%u has an invalid direction field!\n", sync_index);
+            ec_pdo_list_copy(&sync_config->pdos, &sync->pdos);
+        }
+    }
+}
+
+/****************************************************************************/
+
+/** Loads the default mapping for a PDO from the slave object.
+ */
+void ec_slave_config_load_default_mapping(
+        const ec_slave_config_t *sc,
+        ec_pdo_t *pdo
+        )
+{
+    unsigned int i;
+    const ec_sync_t *sync;
+    const ec_pdo_t *default_pdo;
+
+    if (!sc->slave)
+        return;
+
+    EC_CONFIG_DBG(sc, 1, "Loading default mapping for PDO 0x%04X.\n",
+            pdo->index);
+
+    // find PDO in any sync manager (it could be reassigned later)
+    for (i = 0; i < sc->slave->sii.sync_count; i++) {
+        sync = &sc->slave->sii.syncs[i];
+
+        list_for_each_entry(default_pdo, &sync->pdos.list, list) {
+            if (default_pdo->index != pdo->index)
+                continue;
+
+            if (default_pdo->name) {
+                EC_CONFIG_DBG(sc, 1, "Found PDO name \"%s\".\n",
+                        default_pdo->name);
+
+                // take PDO name from assigned one
+                ec_pdo_set_name(pdo, default_pdo->name);
+            }
+
+            // copy entries (= default PDO mapping)
+            if (ec_pdo_copy_entries(pdo, default_pdo))
+                return;
+
+            if (sc->master->debug_level) {
+                const ec_pdo_entry_t *entry;
+                list_for_each_entry(entry, &pdo->entries, list) {
+                    EC_CONFIG_DBG(sc, 1, "Entry 0x%04X:%02X.\n",
+                            entry->index, entry->subindex);
+                }
+            }
+
+            return;
+        }
+    }
+
+    EC_CONFIG_DBG(sc, 1, "No default mapping found.\n");
+}
+
+/****************************************************************************/
+
+/** Get the number of SDO configurations.
+ *
+ * \return Number of SDO configurations.
+ */
+unsigned int ec_slave_config_sdo_count(
+        const ec_slave_config_t *sc /**< Slave configuration. */
+        )
+{
+    const ec_sdo_request_t *req;
+    unsigned int count = 0;
+
+    list_for_each_entry(req, &sc->sdo_configs, list) {
+        count++;
+    }
+
+    return count;
+}
+
+/****************************************************************************/
+
+/** Finds an SDO configuration via its position in the list.
+ *
+ * Const version.
+ *
+ * \return Search result, or NULL.
+ */
+const ec_sdo_request_t *ec_slave_config_get_sdo_by_pos_const(
+        const ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    const ec_sdo_request_t *req;
+
+    list_for_each_entry(req, &sc->sdo_configs, list) {
+        if (pos--)
+            continue;
+        return req;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Get the number of IDN configurations.
+ *
+ * \return Number of SDO configurations.
+ */
+unsigned int ec_slave_config_idn_count(
+        const ec_slave_config_t *sc /**< Slave configuration. */
+        )
+{
+    const ec_soe_request_t *req;
+    unsigned int count = 0;
+
+    list_for_each_entry(req, &sc->soe_configs, list) {
+        count++;
+    }
+
+    return count;
+}
+
+/****************************************************************************/
+
+/** Finds an IDN configuration via its position in the list.
+ *
+ * Const version.
+ *
+ * \return Search result, or NULL.
+ */
+const ec_soe_request_t *ec_slave_config_get_idn_by_pos_const(
+        const ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    const ec_soe_request_t *req;
+
+    list_for_each_entry(req, &sc->soe_configs, list) {
+        if (pos--)
+            continue;
+        return req;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Get the number of feature flags.
+ *
+ * \return Number of feature flags.
+ */
+unsigned int ec_slave_config_flag_count(
+        const ec_slave_config_t *sc /**< Slave configuration. */
+        )
+{
+    const ec_flag_t *flag;
+    unsigned int count = 0;
+
+    list_for_each_entry(flag, &sc->flags, list) {
+        count++;
+    }
+
+    return count;
+}
+
+/****************************************************************************/
+
+/** Finds a flag via its position in the list.
+ *
+ * Const version.
+ *
+ * \return Search result, or NULL.
+ */
+const ec_flag_t *ec_slave_config_get_flag_by_pos_const(
+        const ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    const ec_flag_t *flag;
+
+    list_for_each_entry(flag, &sc->flags, list) {
+        if (pos--)
+            continue;
+        return flag;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Finds a CoE SDO request via its position in the list.
+ *
+ * \return Search result, or NULL.
+ */
+ec_sdo_request_t *ec_slave_config_find_sdo_request(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    ec_sdo_request_t *req;
+
+    list_for_each_entry(req, &sc->sdo_requests, list) {
+        if (pos--)
+            continue;
+        return req;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Finds a SoE request via its position in the list.
+ *
+ * \return Search result, or NULL.
+ */
+ec_soe_request_t *ec_slave_config_find_soe_request(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    ec_soe_request_t *req;
+
+    list_for_each_entry(req, &sc->soe_requests, list) {
+        if (pos--)
+            continue;
+        return req;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Finds a register handler via its position in the list.
+ *
+ * \return Search result, or NULL.
+ */
+ec_reg_request_t *ec_slave_config_find_reg_request(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    ec_reg_request_t *reg;
+
+    list_for_each_entry(reg, &sc->reg_requests, list) {
+        if (pos--)
+            continue;
+        return reg;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Finds a VoE handler via its position in the list.
+ *
+ * \return Search result, or NULL.
+ */
+ec_voe_handler_t *ec_slave_config_find_voe_handler(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        unsigned int pos /**< Position in the list. */
+        )
+{
+    ec_voe_handler_t *voe;
+
+    list_for_each_entry(voe, &sc->voe_handlers, list) {
+        if (pos--)
+            continue;
+        return voe;
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Finds a flag.
+ *
+ * \return Search result, or NULL.
+ */
+ec_flag_t *ec_slave_config_find_flag(
+        ec_slave_config_t *sc, /**< Slave configuration. */
+        const char *key /**< Flag key. */
+        )
+{
+    if (sc) {
+        ec_flag_t *flag;
+        list_for_each_entry(flag, &sc->flags, list) {
+            if (!strcmp(flag->key, key)) {
+                return flag;
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/****************************************************************************/
+
+/** Return an AL state timeout.
+ *
+ * \return Search result, or 0.
+ */
+unsigned int ec_slave_config_al_timeout(const ec_slave_config_t *sc,
+        ec_slave_state_t from, ec_slave_state_t to)
+{
+    ec_al_timeout_t *timeout;
+
+    list_for_each_entry(timeout, &sc->al_timeouts, list) {
+        if (timeout->from == from && timeout->to == to) {
+            return timeout->timeout_ms;
+        }
+    }
+
+    return 0;
+}
+
+/*****************************************************************************
+ *  Application interface
+ ****************************************************************************/
+
+int ecrt_slave_config_sync_manager(ec_slave_config_t *sc, uint8_t sync_index,
+        ec_direction_t dir, ec_watchdog_mode_t watchdog_mode)
+{
+    ec_sync_config_t *sync_config;
+
+    EC_CONFIG_DBG(sc, 1, "ecrt_slave_config_sync_manager(sc = 0x%p,"
+            " sync_index = %u, dir = %i, watchdog_mode = %i)\n",
+            sc, sync_index, dir, watchdog_mode);
+
+    if (sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_CONFIG_ERR(sc, "Invalid sync manager index %u!\n", sync_index);
+        return -ENOENT;
+    }
+
+    if (dir != EC_DIR_OUTPUT && dir != EC_DIR_INPUT) {
+        EC_CONFIG_ERR(sc, "Invalid direction %u!\n", (unsigned int) dir);
+        return -EINVAL;
+    }
+
+    sync_config = &sc->sync_configs[sync_index];
+    sync_config->dir = dir;
+    sync_config->watchdog_mode = watchdog_mode;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_watchdog(ec_slave_config_t *sc,
+        uint16_t divider, uint16_t intervals)
+{
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, divider = %u, intervals = %u)\n",
+            __func__, sc, divider, intervals);
+
+    sc->watchdog_divider = divider;
+    sc->watchdog_intervals = intervals;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_pdo_assign_add(ec_slave_config_t *sc,
+        uint8_t sync_index, uint16_t pdo_index)
+{
+    ec_pdo_t *pdo;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, sync_index = %u, "
+            "pdo_index = 0x%04X)\n", __func__, sc, sync_index, pdo_index);
+
+    if (sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_CONFIG_ERR(sc, "Invalid sync manager index %u!\n", sync_index);
+        return -EINVAL;
+    }
+
+    down(&sc->master->master_sem);
+
+    pdo = ec_pdo_list_add_pdo(&sc->sync_configs[sync_index].pdos, pdo_index);
+    if (IS_ERR(pdo)) {
+        up(&sc->master->master_sem);
+        return PTR_ERR(pdo);
+    }
+    pdo->sync_index = sync_index;
+
+    ec_slave_config_load_default_mapping(sc, pdo);
+
+    up(&sc->master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_pdo_assign_clear(ec_slave_config_t *sc,
+        uint8_t sync_index)
+{
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, sync_index = %u)\n",
+            __func__, sc, sync_index);
+
+    if (sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_CONFIG_ERR(sc, "Invalid sync manager index %u!\n", sync_index);
+        return -EINVAL;
+    }
+
+    down(&sc->master->master_sem);
+    ec_pdo_list_clear_pdos(&sc->sync_configs[sync_index].pdos);
+    up(&sc->master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_pdo_mapping_add(ec_slave_config_t *sc,
+        uint16_t pdo_index, uint16_t entry_index, uint8_t entry_subindex,
+        uint8_t entry_bit_length)
+{
+    uint8_t sync_index;
+    ec_pdo_t *pdo = NULL;
+    ec_pdo_entry_t *entry;
+    int retval = 0;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, "
+            "pdo_index = 0x%04X, entry_index = 0x%04X, "
+            "entry_subindex = 0x%02X, entry_bit_length = %u)\n",
+            __func__, sc, pdo_index, entry_index, entry_subindex,
+            entry_bit_length);
+
+    for (sync_index = 0; sync_index < EC_MAX_SYNC_MANAGERS; sync_index++)
+        if ((pdo = ec_pdo_list_find_pdo(
+                        &sc->sync_configs[sync_index].pdos, pdo_index)))
+            break;
+
+    if (pdo) {
+        down(&sc->master->master_sem);
+        entry = ec_pdo_add_entry(pdo, entry_index, entry_subindex,
+                entry_bit_length);
+        up(&sc->master->master_sem);
+        if (IS_ERR(entry))
+            retval = PTR_ERR(entry);
+    } else {
+        EC_CONFIG_ERR(sc, "PDO 0x%04X is not assigned.\n", pdo_index);
+        retval = -ENOENT;
+    }
+
+    return retval;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_pdo_mapping_clear(ec_slave_config_t *sc,
+        uint16_t pdo_index)
+{
+    uint8_t sync_index;
+    ec_pdo_t *pdo = NULL;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, pdo_index = 0x%04X)\n",
+            __func__, sc, pdo_index);
+
+    for (sync_index = 0; sync_index < EC_MAX_SYNC_MANAGERS; sync_index++)
+        if ((pdo = ec_pdo_list_find_pdo(
+                        &sc->sync_configs[sync_index].pdos, pdo_index)))
+            break;
+
+    if (pdo) {
+        down(&sc->master->master_sem);
+        ec_pdo_clear_entries(pdo);
+        up(&sc->master->master_sem);
+    } else {
+        EC_CONFIG_WARN(sc, "PDO 0x%04X is not assigned.\n", pdo_index);
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_pdos(ec_slave_config_t *sc,
+        unsigned int n_syncs, const ec_sync_info_t syncs[])
+{
+    int ret;
+    unsigned int i, j, k;
+    const ec_sync_info_t *sync_info;
+    const ec_pdo_info_t *pdo_info;
+    const ec_pdo_entry_info_t *entry_info;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, n_syncs = %u, syncs = 0x%p)\n",
+            __func__, sc, n_syncs, syncs);
+
+    if (!syncs)
+        return 0;
+
+    for (i = 0; i < n_syncs; i++) {
+        sync_info = &syncs[i];
+
+        if (sync_info->index == (uint8_t) EC_END)
+            break;
+
+        if (sync_info->index >= EC_MAX_SYNC_MANAGERS) {
+            EC_CONFIG_ERR(sc, "Invalid sync manager index %u!\n",
+                    sync_info->index);
+            return -ENOENT;
+        }
+
+        ret = ecrt_slave_config_sync_manager(sc, sync_info->index,
+                sync_info->dir, sync_info->watchdog_mode);
+        if (ret)
+            return ret;
+
+        ecrt_slave_config_pdo_assign_clear(sc, sync_info->index);
+
+        if (sync_info->n_pdos && sync_info->pdos) {
+
+            for (j = 0; j < sync_info->n_pdos; j++) {
+                pdo_info = &sync_info->pdos[j];
+
+                ret = ecrt_slave_config_pdo_assign_add(
+                        sc, sync_info->index, pdo_info->index);
+                if (ret)
+                    return ret;
+
+                ecrt_slave_config_pdo_mapping_clear(sc, pdo_info->index);
+
+                if (pdo_info->n_entries && pdo_info->entries) {
+                    for (k = 0; k < pdo_info->n_entries; k++) {
+                        entry_info = &pdo_info->entries[k];
+
+                        ret = ecrt_slave_config_pdo_mapping_add(sc,
+                                pdo_info->index, entry_info->index,
+                                entry_info->subindex,
+                                entry_info->bit_length);
+                        if (ret)
+                            return ret;
+                    }
+                }
+            }
+        }
+    }
+
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_reg_pdo_entry(
+        ec_slave_config_t *sc,
+        uint16_t index,
+        uint8_t subindex,
+        ec_domain_t *domain,
+        unsigned int *bit_position
+        )
+{
+    uint8_t sync_index;
+    const ec_sync_config_t *sync_config;
+    unsigned int bit_offset, bit_pos;
+    ec_pdo_t *pdo;
+    ec_pdo_entry_t *entry;
+    int sync_offset;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, index = 0x%04X, "
+            "subindex = 0x%02X, domain = 0x%p, bit_position = 0x%p)\n",
+            __func__, sc, index, subindex, domain, bit_position);
+
+    for (sync_index = 0; sync_index < EC_MAX_SYNC_MANAGERS; sync_index++) {
+        sync_config = &sc->sync_configs[sync_index];
+        bit_offset = 0;
+
+        list_for_each_entry(pdo, &sync_config->pdos.list, list) {
+            list_for_each_entry(entry, &pdo->entries, list) {
+                if (entry->index != index || entry->subindex != subindex) {
+                    bit_offset += entry->bit_length;
+                } else {
+                    bit_pos = bit_offset % 8;
+                    if (bit_position) {
+                        *bit_position = bit_pos;
+                    } else if (bit_pos) {
+                        EC_CONFIG_ERR(sc, "PDO entry 0x%04X:%02X does"
+                                " not byte-align.\n", index, subindex);
+                        return -EFAULT;
+                    }
+
+                    sync_offset = ec_slave_config_prepare_fmmu(
+                            sc, domain, sync_index, sync_config->dir);
+                    if (sync_offset < 0)
+                        return sync_offset;
+
+                    return sync_offset + bit_offset / 8;
+                }
+            }
+        }
+    }
+
+    EC_CONFIG_ERR(sc, "PDO entry 0x%04X:%02X is not mapped.\n",
+           index, subindex);
+    return -ENOENT;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_reg_pdo_entry_pos(
+        ec_slave_config_t *sc,
+        uint8_t sync_index,
+        unsigned int pdo_pos,
+        unsigned int entry_pos,
+        ec_domain_t *domain,
+        unsigned int *bit_position
+        )
+{
+    const ec_sync_config_t *sync_config;
+    unsigned int bit_offset, pp, ep;
+    ec_pdo_t *pdo;
+    ec_pdo_entry_t *entry;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, sync_index = %u, pdo_pos = %u,"
+            " entry_pos = %u, domain = 0x%p, bit_position = 0x%p)\n",
+            __func__, sc, sync_index, pdo_pos, entry_pos,
+            domain, bit_position);
+
+    if (sync_index >= EC_MAX_SYNC_MANAGERS) {
+        EC_CONFIG_ERR(sc, "Invalid syncmanager position %u.\n", sync_index);
+        return -EINVAL;
+    }
+
+    sync_config = &sc->sync_configs[sync_index];
+    bit_offset = 0;
+    pp = 0;
+
+    list_for_each_entry(pdo, &sync_config->pdos.list, list) {
+        ep = 0;
+        list_for_each_entry(entry, &pdo->entries, list) {
+            if (pp != pdo_pos || ep != entry_pos) {
+                bit_offset += entry->bit_length;
+            } else {
+                unsigned int bit_pos = bit_offset % 8;
+                int sync_offset;
+
+                if (bit_position) {
+                    *bit_position = bit_pos;
+                } else if (bit_pos) {
+                    EC_CONFIG_ERR(sc, "PDO entry 0x%04X:%02X does"
+                            " not byte-align.\n",
+                            pdo->index, entry->subindex);
+                    return -EFAULT;
+                }
+
+                sync_offset = ec_slave_config_prepare_fmmu(
+                        sc, domain, sync_index, sync_config->dir);
+                if (sync_offset < 0)
+                    return sync_offset;
+
+                return sync_offset + bit_offset / 8;
+            }
+            ep++;
+        }
+        pp++;
+    }
+
+    EC_CONFIG_ERR(sc, "PDO entry specification %u/%u/%u out of range.\n",
+           sync_index, pdo_pos, entry_pos);
+    return -ENOENT;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_dc(ec_slave_config_t *sc, uint16_t assign_activate,
+        uint32_t sync0_cycle_time, int32_t sync0_shift_time,
+        uint32_t sync1_cycle_time, int32_t sync1_shift_time)
+{
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, assign_activate = 0x%04X,"
+            " sync0_cycle = %u, sync0_shift = %i,"
+            " sync1_cycle = %u, sync1_shift = %i\n",
+            __func__, sc, assign_activate, sync0_cycle_time, sync0_shift_time,
+            sync1_cycle_time, sync1_shift_time);
+
+    sc->dc_assign_activate = assign_activate;
+    sc->dc_sync[0].cycle_time = sync0_cycle_time;
+    sc->dc_sync[0].shift_time = sync0_shift_time;
+    sc->dc_sync[1].cycle_time = sync1_cycle_time;
+    sc->dc_sync[1].shift_time = sync1_shift_time;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_sdo(ec_slave_config_t *sc, uint16_t index,
+        uint8_t subindex, const uint8_t *data, size_t size)
+{
+    ec_slave_t *slave = sc->slave;
+    ec_sdo_request_t *req;
+    int ret;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, index = 0x%04X, "
+            "subindex = 0x%02X, data = 0x%p, size = %zu)\n",
+            __func__, sc, index, subindex, data, size);
+
+    if (slave && !(slave->sii.mailbox_protocols & EC_MBOX_COE)) {
+        EC_CONFIG_WARN(sc, "Attached slave does not support CoE!\n");
+    }
+
+    if (!(req = (ec_sdo_request_t *)
+          kmalloc(sizeof(ec_sdo_request_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate memory for"
+                " SDO configuration!\n");
+        return -ENOMEM;
+    }
+
+    ec_sdo_request_init(req);
+    ecrt_sdo_request_index(req, index, subindex);
+
+    ret = ec_sdo_request_copy_data(req, data, size);
+    if (ret < 0) {
+        ec_sdo_request_clear(req);
+        kfree(req);
+        return ret;
+    }
+
+    down(&sc->master->master_sem);
+    list_add_tail(&req->list, &sc->sdo_configs);
+    up(&sc->master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_sdo8(ec_slave_config_t *sc, uint16_t index,
+        uint8_t subindex, uint8_t value)
+{
+    uint8_t data[1];
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, index = 0x%04X, "
+            "subindex = 0x%02X, value = %u)\n",
+            __func__, sc, index, subindex, (unsigned int) value);
+
+    EC_WRITE_U8(data, value);
+    return ecrt_slave_config_sdo(sc, index, subindex, data, 1);
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_sdo16(ec_slave_config_t *sc, uint16_t index,
+        uint8_t subindex, uint16_t value)
+{
+    uint8_t data[2];
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, index = 0x%04X, "
+            "subindex = 0x%02X, value = %u)\n",
+            __func__, sc, index, subindex, value);
+
+    EC_WRITE_U16(data, value);
+    return ecrt_slave_config_sdo(sc, index, subindex, data, 2);
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_sdo32(ec_slave_config_t *sc, uint16_t index,
+        uint8_t subindex, uint32_t value)
+{
+    uint8_t data[4];
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, index = 0x%04X, "
+            "subindex = 0x%02X, value = %u)\n",
+            __func__, sc, index, subindex, value);
+
+    EC_WRITE_U32(data, value);
+    return ecrt_slave_config_sdo(sc, index, subindex, data, 4);
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_complete_sdo(ec_slave_config_t *sc, uint16_t index,
+        const uint8_t *data, size_t size)
+{
+    ec_slave_t *slave = sc->slave;
+    ec_sdo_request_t *req;
+    int ret;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, index = 0x%04X, "
+            "data = 0x%p, size = %zu)\n", __func__, sc, index, data, size);
+
+    if (slave && !(slave->sii.mailbox_protocols & EC_MBOX_COE)) {
+        EC_CONFIG_WARN(sc, "Attached slave does not support CoE!\n");
+    }
+
+    if (!(req = (ec_sdo_request_t *)
+          kmalloc(sizeof(ec_sdo_request_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate memory for"
+                " SDO configuration!\n");
+        return -ENOMEM;
+    }
+
+    ec_sdo_request_init(req);
+    ecrt_sdo_request_index(req, index, 0);
+    req->complete_access = 1;
+
+    ret = ec_sdo_request_copy_data(req, data, size);
+    if (ret < 0) {
+        ec_sdo_request_clear(req);
+        kfree(req);
+        return ret;
+    }
+
+    down(&sc->master->master_sem);
+    list_add_tail(&req->list, &sc->sdo_configs);
+    up(&sc->master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_emerg_size(ec_slave_config_t *sc, size_t elements)
+{
+    return ec_coe_emerg_ring_size(&sc->emerg_ring, elements);
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_emerg_pop(ec_slave_config_t *sc, uint8_t *target)
+{
+    return ec_coe_emerg_ring_pop(&sc->emerg_ring, target);
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_emerg_clear(ec_slave_config_t *sc)
+{
+    return ec_coe_emerg_ring_clear_ring(&sc->emerg_ring);
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_emerg_overruns(const ec_slave_config_t *sc)
+{
+    return ec_coe_emerg_ring_overruns(&sc->emerg_ring);
+}
+
+/****************************************************************************/
+
+/** Same as ecrt_slave_config_create_sdo_request(), but with ERR_PTR() return
+ * value.
+ */
+ec_sdo_request_t *ecrt_slave_config_create_sdo_request_err(
+        ec_slave_config_t *sc, uint16_t index, uint8_t subindex, size_t size)
+{
+    ec_sdo_request_t *req;
+    int ret;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, "
+            "index = 0x%04X, subindex = 0x%02X, size = %zu)\n",
+            __func__, sc, index, subindex, size);
+
+    if (!(req = (ec_sdo_request_t *)
+                kmalloc(sizeof(ec_sdo_request_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate SDO request memory!\n");
+        return ERR_PTR(-ENOMEM);
+    }
+
+    ec_sdo_request_init(req);
+    ecrt_sdo_request_index(req, index, subindex);
+
+    ret = ec_sdo_request_alloc(req, size);
+    if (ret < 0) {
+        ec_sdo_request_clear(req);
+        kfree(req);
+        return ERR_PTR(ret);
+    }
+
+    // prepare data for optional writing
+    memset(req->data, 0x00, size);
+    req->data_size = size;
+
+    down(&sc->master->master_sem);
+    list_add_tail(&req->list, &sc->sdo_requests);
+    up(&sc->master->master_sem);
+
+    return req;
+}
+
+/****************************************************************************/
+
+ec_sdo_request_t *ecrt_slave_config_create_sdo_request(
+        ec_slave_config_t *sc, uint16_t index, uint8_t subindex, size_t size)
+{
+    ec_sdo_request_t *s = ecrt_slave_config_create_sdo_request_err(sc, index,
+            subindex, size);
+    return IS_ERR(s) ? NULL : s;
+}
+
+/****************************************************************************/
+
+/** Same as ecrt_slave_config_create_soe_request(), but with ERR_PTR() return
+ * value.
+ */
+ec_soe_request_t *ecrt_slave_config_create_soe_request_err(
+        ec_slave_config_t *sc, uint8_t drive_no, uint16_t idn, size_t size)
+{
+    ec_soe_request_t *req;
+    int ret;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, "
+            "drive_no = 0x%02X, idn = 0x%04X, size = %zu)\n",
+            __func__, sc, drive_no, idn, size);
+
+    if (!(req = (ec_soe_request_t *)
+                kmalloc(sizeof(ec_soe_request_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate IDN request memory!\n");
+        return ERR_PTR(-ENOMEM);
+    }
+
+    ec_soe_request_init(req);
+    ecrt_soe_request_idn(req, drive_no, idn);
+
+    ret = ec_soe_request_alloc(req, size);
+    if (ret < 0) {
+        ec_soe_request_clear(req);
+        kfree(req);
+        return ERR_PTR(ret);
+    }
+
+    // prepare data for optional writing
+    memset(req->data, 0x00, size);
+    req->data_size = size;
+
+    down(&sc->master->master_sem);
+    list_add_tail(&req->list, &sc->soe_requests);
+    up(&sc->master->master_sem);
+
+    return req;
+}
+
+/****************************************************************************/
+
+ec_soe_request_t *ecrt_slave_config_create_soe_request(
+        ec_slave_config_t *sc, uint8_t drive_no, uint16_t idn, size_t size)
+{
+    ec_soe_request_t *req = ecrt_slave_config_create_soe_request_err(sc,
+            drive_no, idn, size);
+    return IS_ERR(req) ? NULL : req;
+}
+
+/****************************************************************************/
+
+/** Same as ecrt_slave_config_create_reg_request(), but with ERR_PTR() return
+ * value.
+ */
+ec_reg_request_t *ecrt_slave_config_create_reg_request_err(
+        ec_slave_config_t *sc, size_t size)
+{
+    ec_reg_request_t *reg;
+    int ret;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, size = %zu)\n",
+            __func__, sc, size);
+
+    if (!(reg = (ec_reg_request_t *)
+                kmalloc(sizeof(ec_reg_request_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate register request memory!\n");
+        return ERR_PTR(-ENOMEM);
+    }
+
+    ret = ec_reg_request_init(reg, size);
+    if (ret) {
+        kfree(reg);
+        return ERR_PTR(ret);
+    }
+
+    down(&sc->master->master_sem);
+    list_add_tail(&reg->list, &sc->reg_requests);
+    up(&sc->master->master_sem);
+
+    return reg;
+}
+
+/****************************************************************************/
+
+ec_reg_request_t *ecrt_slave_config_create_reg_request(
+        ec_slave_config_t *sc, size_t size)
+{
+    ec_reg_request_t *reg =
+        ecrt_slave_config_create_reg_request_err(sc, size);
+    return IS_ERR(reg) ? NULL : reg;
+}
+
+/****************************************************************************/
+
+/** Same as ecrt_slave_config_create_voe_handler(), but with ERR_PTR() return
+ * value.
+ */
+ec_voe_handler_t *ecrt_slave_config_create_voe_handler_err(
+        ec_slave_config_t *sc, size_t size)
+{
+    ec_voe_handler_t *voe;
+    int ret;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, size = %zu)\n", __func__, sc, size);
+
+    if (!(voe = (ec_voe_handler_t *)
+                kmalloc(sizeof(ec_voe_handler_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate VoE request memory!\n");
+        return ERR_PTR(-ENOMEM);
+    }
+
+    ret = ec_voe_handler_init(voe, sc, size);
+    if (ret < 0) {
+        kfree(voe);
+        return ERR_PTR(ret);
+    }
+
+    down(&sc->master->master_sem);
+    list_add_tail(&voe->list, &sc->voe_handlers);
+    up(&sc->master->master_sem);
+
+    return voe;
+}
+
+/****************************************************************************/
+
+ec_voe_handler_t *ecrt_slave_config_create_voe_handler(
+        ec_slave_config_t *sc, size_t size)
+{
+    ec_voe_handler_t *voe = ecrt_slave_config_create_voe_handler_err(sc,
+            size);
+    return IS_ERR(voe) ? NULL : voe;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_state(const ec_slave_config_t *sc,
+        ec_slave_config_state_t *state)
+{
+    const ec_slave_t *slave = sc->slave;
+
+    state->online = slave ? 1 : 0;
+    if (state->online) {
+        state->operational =
+            slave->current_state == EC_SLAVE_STATE_OP && !slave->force_config;
+        state->al_state = slave->current_state;
+    } else {
+        state->operational = 0;
+        state->al_state = EC_SLAVE_STATE_UNKNOWN;
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_idn(ec_slave_config_t *sc, uint8_t drive_no,
+        uint16_t idn, ec_al_state_t state, const uint8_t *data,
+        size_t size)
+{
+    ec_slave_t *slave = sc->slave;
+    ec_soe_request_t *req;
+    int ret;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, drive_no = %u, idn = 0x%04X, "
+            "state = %u, data = 0x%p, size = %zu)\n",
+            __func__, sc, drive_no, idn, state, data, size);
+
+    if (drive_no > 7) {
+        EC_CONFIG_ERR(sc, "Invalid drive number %u!\n",
+                (unsigned int) drive_no);
+        return -EINVAL;
+    }
+
+    if (state != EC_AL_STATE_PREOP && state != EC_AL_STATE_SAFEOP) {
+        EC_CONFIG_ERR(sc, "AL state for IDN config"
+                " must be PREOP or SAFEOP!\n");
+        return -EINVAL;
+    }
+
+    if (slave && !(slave->sii.mailbox_protocols & EC_MBOX_SOE)) {
+        EC_CONFIG_WARN(sc, "Attached slave does not support SoE!\n");
+    }
+
+    if (!(req = (ec_soe_request_t *)
+          kmalloc(sizeof(ec_soe_request_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate memory for"
+                " IDN configuration!\n");
+        return -ENOMEM;
+    }
+
+    ec_soe_request_init(req);
+    ec_soe_request_set_drive_no(req, drive_no);
+    ec_soe_request_set_idn(req, idn);
+    req->al_state = state;
+
+    ret = ec_soe_request_copy_data(req, data, size);
+    if (ret < 0) {
+        ec_soe_request_clear(req);
+        kfree(req);
+        return ret;
+    }
+
+    down(&sc->master->master_sem);
+    list_add_tail(&req->list, &sc->soe_configs);
+    up(&sc->master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_flag(ec_slave_config_t *sc, const char *key,
+        int32_t value)
+{
+    ec_flag_t *flag;
+
+    EC_CONFIG_DBG(sc, 1, "%s(sc = 0x%p, key = %s, value = %i)\n",
+            __func__, sc, key, value);
+
+
+    flag = ec_slave_config_find_flag(sc, key);
+    if (flag) {
+        flag->value = value; // overwrite value
+    }
+    else { // new flag
+        int ret;
+
+        if (!(flag = (ec_flag_t *) kmalloc(sizeof(ec_flag_t), GFP_KERNEL))) {
+            EC_CONFIG_ERR(sc, "Failed to allocate memory for flag!\n");
+            return -ENOMEM;
+        }
+
+        ret = ec_flag_init(flag, key, value);
+        if (ret) {
+            kfree(flag);
+            return ret;
+        }
+
+        down(&sc->master->master_sem);
+        list_add_tail(&flag->list, &sc->flags);
+        up(&sc->master->master_sem);
+    }
+    return 0;
+}
+
+/****************************************************************************/
+
+#ifdef EC_EOE
+
+int ecrt_slave_config_eoe_mac_address(ec_slave_config_t *sc,
+        const unsigned char *mac_address)
+{
+    memcpy(sc->eoe_ip_param_request.mac_address, mac_address, EC_ETH_ALEN);
+    sc->eoe_ip_param_request.mac_address_included = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_eoe_ip_address(ec_slave_config_t *sc,
+        struct in_addr ip_address)
+{
+    sc->eoe_ip_param_request.ip_address = ip_address;
+    sc->eoe_ip_param_request.ip_address_included = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_eoe_subnet_mask(ec_slave_config_t *sc,
+        struct in_addr subnet_mask)
+{
+    sc->eoe_ip_param_request.subnet_mask = subnet_mask;
+    sc->eoe_ip_param_request.subnet_mask_included = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_eoe_default_gateway(ec_slave_config_t *sc,
+        struct in_addr gateway_address)
+{
+    sc->eoe_ip_param_request.gateway = gateway_address;
+    sc->eoe_ip_param_request.gateway_included = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_eoe_dns_address(ec_slave_config_t *sc,
+        struct in_addr dns_address)
+{
+    sc->eoe_ip_param_request.dns = dns_address;
+    sc->eoe_ip_param_request.dns_included = 1;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_slave_config_eoe_hostname(ec_slave_config_t *sc,
+        const char *name)
+{
+    strncpy(sc->eoe_ip_param_request.name, name, EC_MAX_HOSTNAME_SIZE - 1);
+    sc->eoe_ip_param_request.name_included = 1;
+    return 0;
+}
+
+#endif
+
+/****************************************************************************/
+
+int ecrt_slave_config_state_timeout(ec_slave_config_t *sc,
+        ec_al_state_t from, ec_al_state_t to, unsigned int timeout_ms)
+{
+    ec_al_timeout_t *timeout;
+    ec_slave_state_t from_state, to_state;
+
+    if (from != EC_AL_STATE_INIT && from != EC_AL_STATE_PREOP &&
+            from != EC_AL_STATE_SAFEOP && from != EC_AL_STATE_OP) {
+        EC_CONFIG_ERR(sc, "Invalid from state %i.\n", from);
+        return -EINVAL;
+    }
+    if (to != EC_AL_STATE_INIT && to != EC_AL_STATE_PREOP &&
+            to != EC_AL_STATE_SAFEOP && to != EC_AL_STATE_OP) {
+        EC_CONFIG_ERR(sc, "Invalid to state %i.\n", to);
+        return -EINVAL;
+    }
+
+    from_state = (ec_slave_state_t) from;
+    to_state = (ec_slave_state_t) to;
+
+    /* try to find an already configured timeout. */
+    list_for_each_entry(timeout, &sc->al_timeouts, list) {
+        if (timeout->from == from_state && timeout->to == to_state) {
+            if (timeout_ms == 0) {
+                // delete configured value
+                list_del(&timeout->list);
+                kfree(timeout);
+                return 0;
+            }
+            timeout->timeout_ms = timeout_ms;
+            return 0;
+        }
+    }
+
+    if (timeout_ms == 0) {
+        return 0;
+    }
+
+    /* no timeout found. create one. */
+    if (!(timeout = (ec_al_timeout_t *)
+          kmalloc(sizeof(ec_al_timeout_t), GFP_KERNEL))) {
+        EC_CONFIG_ERR(sc, "Failed to allocate memory for"
+                " AL timeout configuration!\n");
+        return -ENOMEM;
+    }
+
+    timeout->from = from_state;
+    timeout->to = to_state;
+    timeout->timeout_ms = timeout_ms;
+
+    down(&sc->master->master_sem);
+    list_add_tail(&timeout->list, &sc->al_timeouts);
+    up(&sc->master->master_sem);
+    return 0;
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecrt_slave_config_sync_manager);
+EXPORT_SYMBOL(ecrt_slave_config_watchdog);
+EXPORT_SYMBOL(ecrt_slave_config_pdo_assign_add);
+EXPORT_SYMBOL(ecrt_slave_config_pdo_assign_clear);
+EXPORT_SYMBOL(ecrt_slave_config_pdo_mapping_add);
+EXPORT_SYMBOL(ecrt_slave_config_pdo_mapping_clear);
+EXPORT_SYMBOL(ecrt_slave_config_pdos);
+EXPORT_SYMBOL(ecrt_slave_config_reg_pdo_entry);
+EXPORT_SYMBOL(ecrt_slave_config_reg_pdo_entry_pos);
+EXPORT_SYMBOL(ecrt_slave_config_dc);
+EXPORT_SYMBOL(ecrt_slave_config_sdo);
+EXPORT_SYMBOL(ecrt_slave_config_sdo8);
+EXPORT_SYMBOL(ecrt_slave_config_sdo16);
+EXPORT_SYMBOL(ecrt_slave_config_sdo32);
+EXPORT_SYMBOL(ecrt_slave_config_complete_sdo);
+EXPORT_SYMBOL(ecrt_slave_config_emerg_size);
+EXPORT_SYMBOL(ecrt_slave_config_emerg_pop);
+EXPORT_SYMBOL(ecrt_slave_config_emerg_clear);
+EXPORT_SYMBOL(ecrt_slave_config_emerg_overruns);
+EXPORT_SYMBOL(ecrt_slave_config_create_sdo_request);
+EXPORT_SYMBOL(ecrt_slave_config_create_soe_request);
+EXPORT_SYMBOL(ecrt_slave_config_create_voe_handler);
+EXPORT_SYMBOL(ecrt_slave_config_create_reg_request);
+EXPORT_SYMBOL(ecrt_slave_config_state);
+EXPORT_SYMBOL(ecrt_slave_config_idn);
+EXPORT_SYMBOL(ecrt_slave_config_flag);
+#ifdef EOE
+EXPORT_SYMBOL(ecrt_slave_config_eoe_mac_address);
+EXPORT_SYMBOL(ecrt_slave_config_eoe_ip_address);
+EXPORT_SYMBOL(ecrt_slave_config_eoe_subnet_mask);
+EXPORT_SYMBOL(ecrt_slave_config_eoe_default_gateway);
+EXPORT_SYMBOL(ecrt_slave_config_eoe_dns_address);
+EXPORT_SYMBOL(ecrt_slave_config_eoe_hostname);
+#endif
+EXPORT_SYMBOL(ecrt_slave_config_state_timeout);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/slave_config.h b/drivers/net/ethercat/master/slave_config.h
new file mode 100644
index 000000000000..124995c28150
--- /dev/null
+++ b/drivers/net/ethercat/master/slave_config.h
@@ -0,0 +1,196 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2024  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT slave configuration structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_SLAVE_CONFIG_H__
+#define __EC_SLAVE_CONFIG_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+#include "slave.h"
+#include "sync_config.h"
+#include "fmmu_config.h"
+#include "coe_emerg_ring.h"
+#include "flag.h"
+
+/****************************************************************************/
+
+/** Convenience macro for printing configuration-specific information to
+ * syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX> <ALIAS>:<POSITION>: ", where INDEX is the master index
+ * and ALIAS and POSITION identify the configuration.
+ *
+ * \param sc EtherCAT slave configuration
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_CONFIG_INFO(sc, fmt, args...) \
+    printk(KERN_INFO "EtherCAT %u %u:%u: " fmt, sc->master->index, \
+            sc->alias, sc->position, ##args)
+
+/** Convenience macro for printing configuration-specific errors to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX> <ALIAS>:<POSITION>: ", where INDEX is the master index
+ * and ALIAS and POSITION identify the configuration.
+ *
+ * \param sc EtherCAT slave configuration
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_CONFIG_ERR(sc, fmt, args...) \
+    printk(KERN_ERR "EtherCAT ERROR %u %u:%u: " fmt, sc->master->index, \
+            sc->alias, sc->position, ##args)
+
+/** Convenience macro for printing configuration-specific warnings to syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX> <ALIAS>:<POSITION>: ", where INDEX is the master index
+ * and ALIAS and POSITION identify the configuration.
+ *
+ * \param sc EtherCAT slave configuration
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_CONFIG_WARN(sc, fmt, args...) \
+    printk(KERN_WARNING "EtherCAT WARNING %u %u:%u: " fmt, \
+            sc->master->index, sc->alias, sc->position, ##args)
+
+/** Convenience macro for printing configuration-specific debug messages to
+ * syslog.
+ *
+ * This will print the message in \a fmt with a prefixed
+ * "EtherCAT <INDEX> <ALIAS>:<POSITION>: ", where INDEX is the master index
+ * and ALIAS and POSITION identify the configuration.
+ *
+ * \param sc EtherCAT slave configuration
+ * \param level Debug level. Master's debug level must be >= \a level for
+ * output.
+ * \param fmt format string (like in printf())
+ * \param args arguments (optional)
+ */
+#define EC_CONFIG_DBG(sc, level, fmt, args...) \
+    do { \
+        if (sc->master->debug_level >= level) { \
+            printk(KERN_DEBUG "EtherCAT DEBUG %u %u:%u: " fmt, \
+                    sc->master->index, sc->alias, sc->position, ##args); \
+        } \
+    } while (0)
+
+/****************************************************************************/
+
+/** EtherCAT slave configuration.
+ */
+struct ec_slave_config {
+    struct list_head list; /**< List item. */
+    ec_master_t *master; /**< Master owning the slave configuration. */
+
+    uint16_t alias; /**< Slave alias. */
+    uint16_t position; /**< Index after alias. If alias is zero, this is the
+                         ring position. */
+    uint32_t vendor_id; /**< Slave vendor ID. */
+    uint32_t product_code; /**< Slave product code. */
+
+    uint16_t watchdog_divider; /**< Watchdog divider as a number of 40ns
+                                 intervals (see spec. reg. 0x0400). */
+    uint16_t watchdog_intervals; /**< Process data watchdog intervals (see
+                                   spec. reg. 0x0420). */
+
+    ec_slave_t *slave; /**< Slave pointer. This is \a NULL, if the slave is
+                         offline. */
+
+    ec_sync_config_t sync_configs[EC_MAX_SYNC_MANAGERS]; /**< Sync manager
+                                                   configurations. */
+    ec_fmmu_config_t fmmu_configs[EC_MAX_FMMUS]; /**< FMMU configurations. */
+    uint8_t used_fmmus; /**< Number of FMMUs used. */
+    uint16_t dc_assign_activate; /**< Vendor-specific AssignActivate word. */
+    ec_sync_signal_t dc_sync[EC_SYNC_SIGNAL_COUNT]; /**< DC sync signals. */
+
+    struct list_head sdo_configs; /**< List of SDO configurations. */
+    struct list_head sdo_requests; /**< List of SDO requests. */
+    struct list_head soe_requests; /**< List of SoE requests. */
+    struct list_head voe_handlers; /**< List of VoE handlers. */
+    struct list_head reg_requests; /**< List of register requests. */
+    struct list_head soe_configs; /**< List of SoE configurations. */
+    struct list_head flags; /**< List of feature flags. */
+    struct list_head al_timeouts; /**< List of specific AL state timeouts. */
+
+#ifdef EC_EOE
+    ec_eoe_request_t eoe_ip_param_request; /**< EoE IP parameters. */
+#endif
+
+    ec_coe_emerg_ring_t emerg_ring; /**< CoE emergency ring buffer. */
+};
+
+/****************************************************************************/
+
+void ec_slave_config_init(ec_slave_config_t *, ec_master_t *, uint16_t,
+        uint16_t, uint32_t, uint32_t);
+void ec_slave_config_clear(ec_slave_config_t *);
+
+int ec_slave_config_attach(ec_slave_config_t *);
+void ec_slave_config_detach(ec_slave_config_t *);
+
+void ec_slave_config_load_default_sync_config(ec_slave_config_t *);
+
+unsigned int ec_slave_config_sdo_count(const ec_slave_config_t *);
+const ec_sdo_request_t *ec_slave_config_get_sdo_by_pos_const(
+        const ec_slave_config_t *, unsigned int);
+unsigned int ec_slave_config_idn_count(const ec_slave_config_t *);
+const ec_soe_request_t *ec_slave_config_get_idn_by_pos_const(
+        const ec_slave_config_t *, unsigned int);
+unsigned int ec_slave_config_flag_count(const ec_slave_config_t *);
+const ec_flag_t *ec_slave_config_get_flag_by_pos_const(
+        const ec_slave_config_t *, unsigned int);
+ec_sdo_request_t *ec_slave_config_find_sdo_request(ec_slave_config_t *,
+        unsigned int);
+ec_soe_request_t *ec_slave_config_find_soe_request(ec_slave_config_t *,
+        unsigned int);
+ec_reg_request_t *ec_slave_config_find_reg_request(ec_slave_config_t *,
+        unsigned int);
+ec_voe_handler_t *ec_slave_config_find_voe_handler(ec_slave_config_t *,
+        unsigned int);
+ec_flag_t *ec_slave_config_find_flag(ec_slave_config_t *, const char *);
+
+ec_sdo_request_t *ecrt_slave_config_create_sdo_request_err(
+        ec_slave_config_t *, uint16_t, uint8_t, size_t);
+ec_soe_request_t *ecrt_slave_config_create_soe_request_err(
+        ec_slave_config_t *, uint8_t, uint16_t, size_t);
+ec_voe_handler_t *ecrt_slave_config_create_voe_handler_err(
+        ec_slave_config_t *, size_t);
+ec_reg_request_t *ecrt_slave_config_create_reg_request_err(
+        ec_slave_config_t *, size_t);
+
+unsigned int ec_slave_config_al_timeout(const ec_slave_config_t *,
+        ec_slave_state_t, ec_slave_state_t);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/soe_errors.c b/drivers/net/ethercat/master/soe_errors.c
new file mode 100644
index 000000000000..937803b2e6ca
--- /dev/null
+++ b/drivers/net/ethercat/master/soe_errors.c
@@ -0,0 +1,88 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT SoE errors.
+*/
+
+/****************************************************************************/
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** SoE error codes.
+ */
+const ec_code_msg_t soe_error_codes[] = {
+    {0x1001, "No IDN"},
+    {0x1009, "Invalid access to element 1"},
+    {0x2001, "No name"},
+    {0x2002, "Name transmission too short"},
+    {0x2003, "Name transmission too long"},
+    {0x2004, "Name cannot be changed, read only"},
+    {0x2005, "Name is write protected at this time"},
+    {0x3002, "Attribute transmission too short"},
+    {0x3003, "Attribute transmission too long"},
+    {0x3004, "Attribute cannot be changed, read only"},
+    {0x3005, "Attribute is write protected at this time"},
+    {0x4001, "No unit"},
+    {0x4002, "Unit transmission too short"},
+    {0x4003, "Unit transmission too long"},
+    {0x4004, "Unit cannot be changed, read only"},
+    {0x4005, "Unit is write proteced at this time"},
+    {0x5001, "No minimum input value"},
+    {0x5002, "Minimum input value transmission too short"},
+    {0x5003, "Minimum input value transmission too long"},
+    {0x5004, "Minimum input value cannot be changed, read only"},
+    {0x5005, "Minimum input value is write protected at this time"},
+    {0x6001, "No maximum input value"},
+    {0x6002, "Maximum input value transmission too short"},
+    {0x6003, "Maximum input value transmission too long"},
+    {0x6004, "Maximum input value cannot be changed, read only"},
+    {0x6005, "Maximum input value is write protected at this time"},
+    {0x7002, "Operation data value transmission too short"},
+    {0x7003, "Operation data value transmission too long"},
+    {0x7004, "Operation data value cannot be changed, read only"},
+    {0x7005, "Operation data value is write protected at this time"},
+    {0x7006, "Operation data value is smaller than the minimum input value"},
+    {0x7007, "Operation data value is greater than the minimum input value"},
+    {0x7008, "Invalid operation data"},
+    {0x7009, "Operation data is write protected by a password"},
+    {0x700A, "Operation data is write protected"},
+    {0x700B, "Invalid indirect addressing"},
+    {0x700C, "Operation data is write protected due to other settings"},
+    {0x700D, "Reserved"},
+    {0x7010, "Procedure command already active"},
+    {0x7011, "Procedure command not interruptible"},
+    {0x7012, "Procedure command is at this time not executable"},
+    {0x7013, "Procedure command not executable"},
+    {0x7014, "No data state"},
+    {0x8001, "No default value"},
+    {0x8002, "Default value transmission too long"},
+    {0x8004, "Default value cannot be changed, read only"},
+    {0x800A, "Invalid drive number"},
+    {0x800B, "General error"},
+    {0x800C, "No element addressed"},
+    {}
+};
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/soe_request.c b/drivers/net/ethercat/master/soe_request.c
new file mode 100644
index 000000000000..ef90124ba35a
--- /dev/null
+++ b/drivers/net/ethercat/master/soe_request.c
@@ -0,0 +1,333 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2023  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * Sercos-over-EtherCAT request functions.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/jiffies.h>
+#include <linux/slab.h>
+
+#include "soe_request.h"
+
+/****************************************************************************/
+
+/** Default timeout in ms to wait for SoE responses.
+ */
+#define EC_SOE_REQUEST_RESPONSE_TIMEOUT 1000
+
+/****************************************************************************/
+
+void ec_soe_request_clear_data(ec_soe_request_t *);
+
+/****************************************************************************/
+
+/** SoE request constructor.
+ */
+void ec_soe_request_init(
+        ec_soe_request_t *req /**< SoE request. */
+        )
+{
+    INIT_LIST_HEAD(&req->list);
+    req->drive_no = 0x00;
+    req->idn = 0x0000;
+    req->al_state = EC_AL_STATE_INIT;
+    req->data = NULL;
+    req->mem_size = 0;
+    req->data_size = 0;
+    req->issue_timeout = 0; // no timeout
+    req->dir = EC_DIR_INVALID;
+    req->state = EC_INT_REQUEST_INIT;
+    req->jiffies_start = 0U;
+    req->jiffies_sent = 0U;
+    req->error_code = 0x0000;
+}
+
+/****************************************************************************/
+
+/** SoE request destructor.
+ */
+void ec_soe_request_clear(
+        ec_soe_request_t *req /**< SoE request. */
+        )
+{
+    ec_soe_request_clear_data(req);
+}
+
+/****************************************************************************/
+
+/** Copy another SoE request.
+ *
+ * \return Zero on success, otherwise a negative error code.
+ */
+int ec_soe_request_copy(
+        ec_soe_request_t *req, /**< SoE request. */
+        const ec_soe_request_t *other /**< Other SoE request to copy from. */
+        )
+{
+    req->drive_no = other->drive_no;
+    req->idn = other->idn;
+    req->al_state = other->al_state;
+    return ec_soe_request_copy_data(req, other->data, other->data_size);
+}
+
+/****************************************************************************/
+
+/** Set drive number.
+ */
+void ec_soe_request_set_drive_no(
+        ec_soe_request_t *req, /**< SoE request. */
+        uint8_t drive_no /** Drive Number. */
+        )
+{
+    req->drive_no = drive_no;
+}
+
+/****************************************************************************/
+
+/** Set IDN.
+ */
+void ec_soe_request_set_idn(
+        ec_soe_request_t *req, /**< SoE request. */
+        uint16_t idn /** IDN. */
+        )
+{
+    req->idn = idn;
+}
+
+/****************************************************************************/
+
+/** Free allocated memory.
+ */
+void ec_soe_request_clear_data(
+        ec_soe_request_t *req /**< SoE request. */
+        )
+{
+    if (req->data) {
+        kfree(req->data);
+        req->data = NULL;
+    }
+
+    req->mem_size = 0;
+    req->data_size = 0;
+}
+
+/****************************************************************************/
+
+/** Pre-allocates the data memory.
+ *
+ * If the \a mem_size is already bigger than \a size, nothing is done.
+ *
+ * \return 0 on success, otherwise -ENOMEM.
+ */
+int ec_soe_request_alloc(
+        ec_soe_request_t *req, /**< SoE request. */
+        size_t size /**< Data size to allocate. */
+        )
+{
+    if (size <= req->mem_size)
+        return 0;
+
+    ec_soe_request_clear_data(req);
+
+    if (!(req->data = (uint8_t *) kmalloc(size, GFP_KERNEL))) {
+        EC_ERR("Failed to allocate %zu bytes of SoE memory.\n", size);
+        return -ENOMEM;
+    }
+
+    req->mem_size = size;
+    req->data_size = 0;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Copies SoE data from an external source.
+ *
+ * If the \a mem_size is to small, new memory is allocated.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_soe_request_copy_data(
+        ec_soe_request_t *req, /**< SoE request. */
+        const uint8_t *source, /**< Source data. */
+        size_t size /**< Number of bytes in \a source. */
+        )
+{
+    int ret = ec_soe_request_alloc(req, size);
+    if (ret < 0)
+        return ret;
+
+    memcpy(req->data, source, size);
+    req->data_size = size;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Copies SoE data from an external source.
+ *
+ * If the \a mem_size is to small, new memory is allocated.
+ *
+ * \retval  0 Success.
+ * \retval <0 Error code.
+ */
+int ec_soe_request_append_data(
+        ec_soe_request_t *req, /**< SoE request. */
+        const uint8_t *source, /**< Source data. */
+        size_t size /**< Number of bytes in \a source. */
+        )
+{
+    if (req->data_size + size > req->mem_size) {
+        size_t new_size = req->mem_size ? req->mem_size * 2 : size;
+        uint8_t *new_data = (uint8_t *) kmalloc(new_size, GFP_KERNEL);
+        if (!new_data) {
+            EC_ERR("Failed to allocate %zu bytes of SoE memory.\n",
+                    new_size);
+            return -ENOMEM;
+        }
+        memcpy(new_data, req->data, req->data_size);
+        kfree(req->data);
+        req->data = new_data;
+        req->mem_size = new_size;
+    }
+
+    memcpy(req->data + req->data_size, source, size);
+    req->data_size += size;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Request a read operation.
+ */
+int ec_soe_request_read(
+        ec_soe_request_t *req /**< SoE request. */
+       )
+{
+    req->dir = EC_DIR_INPUT;
+    req->state = EC_INT_REQUEST_QUEUED;
+    req->error_code = 0x0000;
+    req->jiffies_start = jiffies;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Request a write operation.
+ */
+int ec_soe_request_write(
+        ec_soe_request_t *req /**< SoE request. */
+        )
+{
+    req->dir = EC_DIR_OUTPUT;
+    req->state = EC_INT_REQUEST_QUEUED;
+    req->error_code = 0x0000;
+    req->jiffies_start = jiffies;
+    return 0;
+}
+
+/****************************************************************************/
+
+/** Checks, if the timeout was exceeded.
+ *
+ * \return non-zero if the timeout was exceeded, else zero.
+ */
+int ec_soe_request_timed_out(const ec_soe_request_t *req /**< SDO request. */)
+{
+    return req->issue_timeout
+        && jiffies - req->jiffies_start > HZ * req->issue_timeout / 1000;
+}
+
+/*****************************************************************************
+ * Application interface.
+ ****************************************************************************/
+
+int ecrt_soe_request_idn(ec_soe_request_t *req, uint8_t drive_no,
+        uint16_t idn)
+{
+    req->drive_no = drive_no;
+    req->idn = idn;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_soe_request_timeout(ec_soe_request_t *req, uint32_t timeout)
+{
+    req->issue_timeout = timeout;
+    return 0;
+}
+
+/****************************************************************************/
+
+uint8_t *ecrt_soe_request_data(const ec_soe_request_t *req)
+{
+    return req->data;
+}
+
+/****************************************************************************/
+
+size_t ecrt_soe_request_data_size(const ec_soe_request_t *req)
+{
+    return req->data_size;
+}
+
+/****************************************************************************/
+
+ec_request_state_t ecrt_soe_request_state(const ec_soe_request_t *req)
+{
+   return ec_request_state_translation_table[req->state];
+}
+
+/****************************************************************************/
+
+int ecrt_soe_request_read(ec_soe_request_t *req)
+{
+    return ec_soe_request_read(req);
+}
+
+/****************************************************************************/
+
+int ecrt_soe_request_write(ec_soe_request_t *req)
+{
+    return ec_soe_request_write(req);
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecrt_soe_request_idn);
+EXPORT_SYMBOL(ecrt_soe_request_timeout);
+EXPORT_SYMBOL(ecrt_soe_request_data);
+EXPORT_SYMBOL(ecrt_soe_request_data_size);
+EXPORT_SYMBOL(ecrt_soe_request_state);
+EXPORT_SYMBOL(ecrt_soe_request_read);
+EXPORT_SYMBOL(ecrt_soe_request_write);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/soe_request.h b/drivers/net/ethercat/master/soe_request.h
new file mode 100644
index 000000000000..ced4c9a196c0
--- /dev/null
+++ b/drivers/net/ethercat/master/soe_request.h
@@ -0,0 +1,76 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   EtherCAT SoE request structure.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_SOE_REQUEST_H__
+#define __EC_SOE_REQUEST_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+
+/****************************************************************************/
+
+/** Sercos-over-EtherCAT request.
+ */
+struct ec_soe_request {
+    struct list_head list; /**< List item. */
+    uint8_t drive_no; /**< Drive number. */
+    uint16_t idn; /**< Sercos ID-Number. */
+    ec_al_state_t al_state; /**< AL state (only valid for IDN config). */
+    uint8_t *data; /**< Pointer to SDO data. */
+    size_t mem_size; /**< Size of SDO data memory. */
+    size_t data_size; /**< Size of SDO data. */
+    uint32_t issue_timeout; /**< Maximum time in ms, the processing of the
+                              request may take. */
+    ec_direction_t dir; /**< Direction. EC_DIR_OUTPUT means writing to the
+                          slave, EC_DIR_INPUT means reading from the slave. */
+    ec_internal_request_state_t state; /**< Request state. */
+    unsigned long jiffies_start; /**< Jiffies, when the request was issued. */
+    unsigned long jiffies_sent; /**< Jiffies, when the upload/download
+                                     request was sent. */
+    uint16_t error_code; /**< SoE error code. */
+};
+
+/****************************************************************************/
+
+void ec_soe_request_init(ec_soe_request_t *);
+void ec_soe_request_clear(ec_soe_request_t *);
+
+int ec_soe_request_copy(ec_soe_request_t *, const ec_soe_request_t *);
+void ec_soe_request_set_drive_no(ec_soe_request_t *, uint8_t);
+void ec_soe_request_set_idn(ec_soe_request_t *, uint16_t);
+int ec_soe_request_alloc(ec_soe_request_t *, size_t);
+int ec_soe_request_copy_data(ec_soe_request_t *, const uint8_t *, size_t);
+int ec_soe_request_append_data(ec_soe_request_t *, const uint8_t *, size_t);
+int ec_soe_request_read(ec_soe_request_t *);
+int ec_soe_request_write(ec_soe_request_t *);
+int ec_soe_request_timed_out(const ec_soe_request_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/sync.c b/drivers/net/ethercat/master/sync.c
new file mode 100644
index 000000000000..bcfe63fcafe6
--- /dev/null
+++ b/drivers/net/ethercat/master/sync.c
@@ -0,0 +1,170 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT sync manager methods.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "slave.h"
+#include "master.h"
+#include "pdo.h"
+#include "sync.h"
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_sync_init(
+        ec_sync_t *sync, /**< EtherCAT sync manager. */
+        ec_slave_t *slave /**< EtherCAT slave. */
+        )
+{
+    sync->slave = slave;
+    sync->physical_start_address = 0x0000;
+    sync->default_length = 0x0000;
+    sync->control_register = 0x00;
+    sync->enable = 0x00;
+    ec_pdo_list_init(&sync->pdos);
+}
+
+/****************************************************************************/
+
+/** Copy constructor.
+ */
+void ec_sync_init_copy(
+        ec_sync_t *sync, /**< EtherCAT sync manager. */
+        const ec_sync_t *other /**< Sync manager to copy from. */
+        )
+{
+   sync->slave = other->slave;
+   sync->physical_start_address = other->physical_start_address;
+   sync->default_length = other->default_length;
+   sync->control_register = other->control_register;
+   sync->enable = other->enable;
+   ec_pdo_list_init(&sync->pdos);
+   ec_pdo_list_copy(&sync->pdos, &other->pdos);
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_sync_clear(
+        ec_sync_t *sync /**< EtherCAT sync manager. */
+        )
+{
+    ec_pdo_list_clear(&sync->pdos);
+}
+
+/****************************************************************************/
+
+/** Initializes a sync manager configuration page.
+ *
+ * The referenced memory (\a data) must be at least \a EC_SYNC_SIZE bytes.
+ */
+void ec_sync_page(
+        const ec_sync_t *sync, /**< Sync manager. */
+        uint8_t sync_index, /**< Index of the sync manager. */
+        uint16_t data_size, /**< Data size. */
+        const ec_sync_config_t *sync_config, /**< Configuration. */
+        uint8_t pdo_xfer, /**< Non-zero, if PDOs will be transferred via this
+                            sync manager. */
+        uint8_t *data /**> Configuration memory. */
+        )
+{
+    // enable only if (SII enable is set or PDO xfer)
+    // and size is > 0 and SM is not virtual
+    uint16_t enable = ((sync->enable & 0x01) || pdo_xfer)
+                        && data_size
+                        && ((sync->enable & 0x04) == 0);
+    uint8_t control = sync->control_register;
+
+    if (sync_config) {
+
+        switch (sync_config->dir) {
+            case EC_DIR_OUTPUT:
+            case EC_DIR_INPUT:
+                EC_WRITE_BIT(&control, 2,
+                        sync_config->dir == EC_DIR_OUTPUT ? 1 : 0);
+                EC_WRITE_BIT(&control, 3, 0);
+                break;
+            default:
+                break;
+        }
+
+        switch (sync_config->watchdog_mode) {
+            case EC_WD_ENABLE:
+            case EC_WD_DISABLE:
+                EC_WRITE_BIT(&control, 6,
+                        sync_config->watchdog_mode == EC_WD_ENABLE);
+                break;
+            default:
+                break;
+        }
+    }
+
+    EC_SLAVE_DBG(sync->slave, 1, "SM%u: Addr 0x%04X, Size %3u,"
+            " Ctrl 0x%02X, En %u\n",
+            sync_index, sync->physical_start_address,
+            data_size, control, enable);
+
+    EC_WRITE_U16(data,     sync->physical_start_address);
+    EC_WRITE_U16(data + 2, data_size);
+    EC_WRITE_U8 (data + 4, control);
+    EC_WRITE_U8 (data + 5, 0x00); // status byte (read only)
+    EC_WRITE_U16(data + 6, enable);
+}
+
+/****************************************************************************/
+
+/** Adds a PDO to the list of known mapped PDOs.
+ *
+ * \return 0 on success, else < 0
+ */
+int ec_sync_add_pdo(
+        ec_sync_t *sync, /**< EtherCAT sync manager. */
+        const ec_pdo_t *pdo /**< PDO to map. */
+        )
+{
+    return ec_pdo_list_add_pdo_copy(&sync->pdos, pdo);
+}
+
+/****************************************************************************/
+
+/** Determines the default direction from the control register.
+ *
+ * \return Direction.
+ */
+ec_direction_t ec_sync_default_direction(
+        const ec_sync_t *sync /**< EtherCAT sync manager. */
+        )
+{
+    switch ((sync->control_register & 0x0C) >> 2) {
+        case 0x0: return EC_DIR_INPUT;
+        case 0x1: return EC_DIR_OUTPUT;
+        default: return EC_DIR_INVALID;
+    }
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/sync.h b/drivers/net/ethercat/master/sync.h
new file mode 100644
index 000000000000..9d0ddced610f
--- /dev/null
+++ b/drivers/net/ethercat/master/sync.h
@@ -0,0 +1,60 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT sync manager.
+ */
+
+/****************************************************************************/
+
+#ifndef __EC_SYNC_H__
+#define __EC_SYNC_H__
+
+#include "globals.h"
+#include "pdo_list.h"
+#include "sync_config.h"
+
+/****************************************************************************/
+
+/** Sync manager.
+ */
+typedef struct {
+    ec_slave_t *slave; /**< Slave, the sync manager belongs to. */
+    uint16_t physical_start_address; /**< Physical start address. */
+    uint16_t default_length; /**< Data length in bytes. */
+    uint8_t control_register; /**< Control register value. */
+    uint8_t enable; /**< Enable bit. */
+    ec_pdo_list_t pdos; /**< Current PDO assignment. */
+} ec_sync_t;
+
+/****************************************************************************/
+
+void ec_sync_init(ec_sync_t *, ec_slave_t *);
+void ec_sync_init_copy(ec_sync_t *, const ec_sync_t *);
+void ec_sync_clear(ec_sync_t *);
+void ec_sync_page(const ec_sync_t *, uint8_t, uint16_t,
+        const ec_sync_config_t *, uint8_t, uint8_t *);
+int ec_sync_add_pdo(ec_sync_t *, const ec_pdo_t *);
+ec_direction_t ec_sync_default_direction(const ec_sync_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/sync_config.c b/drivers/net/ethercat/master/sync_config.c
new file mode 100644
index 000000000000..d2d87a74948d
--- /dev/null
+++ b/drivers/net/ethercat/master/sync_config.c
@@ -0,0 +1,55 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT sync manager configuration methods.
+ */
+
+/****************************************************************************/
+
+#include "globals.h"
+#include "sync_config.h"
+
+/****************************************************************************/
+
+/** Constructor.
+ */
+void ec_sync_config_init(
+        ec_sync_config_t *sync_config /**< Sync manager configuration. */
+        )
+{
+    sync_config->dir = EC_DIR_INVALID;
+    sync_config->watchdog_mode = EC_WD_DEFAULT;
+    ec_pdo_list_init(&sync_config->pdos);
+}
+
+/****************************************************************************/
+
+/** Destructor.
+ */
+void ec_sync_config_clear(
+        ec_sync_config_t *sync_config /**< Sync manager configuration. */
+        )
+{
+    ec_pdo_list_clear(&sync_config->pdos);
+}
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/sync_config.h b/drivers/net/ethercat/master/sync_config.h
new file mode 100644
index 000000000000..ca31aa720141
--- /dev/null
+++ b/drivers/net/ethercat/master/sync_config.h
@@ -0,0 +1,51 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * EtherCAT sync manager.
+ */
+
+/****************************************************************************/
+
+#ifndef __EC_SYNC_CONFIG_H__
+#define __EC_SYNC_CONFIG_H__
+
+#include "globals.h"
+#include "pdo_list.h"
+
+/****************************************************************************/
+
+/** Sync manager configuration.
+ */
+typedef struct {
+    ec_direction_t dir; /**< Sync manager direction. */
+    ec_watchdog_mode_t watchdog_mode; /**< Watchdog mode. */
+    ec_pdo_list_t pdos; /**< Current PDO assignment. */
+} ec_sync_config_t;
+
+/****************************************************************************/
+
+void ec_sync_config_init(ec_sync_config_t *);
+void ec_sync_config_clear(ec_sync_config_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethercat/master/voe_handler.c b/drivers/net/ethercat/master/voe_handler.c
new file mode 100644
index 000000000000..fc8cf307b34d
--- /dev/null
+++ b/drivers/net/ethercat/master/voe_handler.c
@@ -0,0 +1,553 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/** \file
+ * Vendor specific over EtherCAT protocol handler functions.
+ */
+
+/****************************************************************************/
+
+#include <linux/module.h>
+
+#include "master.h"
+#include "slave_config.h"
+#include "mailbox.h"
+#include "voe_handler.h"
+
+/** VoE header size.
+ */
+#define EC_VOE_HEADER_SIZE 6
+
+/** VoE response timeout in [ms].
+ */
+#define EC_VOE_RESPONSE_TIMEOUT 500
+
+/****************************************************************************/
+
+void ec_voe_handler_state_write_start(ec_voe_handler_t *);
+void ec_voe_handler_state_write_response(ec_voe_handler_t *);
+
+void ec_voe_handler_state_read_start(ec_voe_handler_t *);
+void ec_voe_handler_state_read_check(ec_voe_handler_t *);
+void ec_voe_handler_state_read_response(ec_voe_handler_t *);
+
+void ec_voe_handler_state_read_nosync_start(ec_voe_handler_t *);
+void ec_voe_handler_state_read_nosync_response(ec_voe_handler_t *);
+
+void ec_voe_handler_state_end(ec_voe_handler_t *);
+void ec_voe_handler_state_error(ec_voe_handler_t *);
+
+/****************************************************************************/
+
+/** VoE handler constructor.
+ *
+ * \return Return value of ec_datagram_prealloc().
+ */
+int ec_voe_handler_init(
+        ec_voe_handler_t *voe, /**< VoE handler. */
+        ec_slave_config_t *sc, /**< Parent slave configuration. */
+        size_t size /**< Size of memory to reserve. */
+        )
+{
+    voe->config = sc;
+    voe->vendor_id = 0x00000000;
+    voe->vendor_type = 0x0000;
+    voe->data_size = 0;
+    voe->dir = EC_DIR_INVALID;
+    voe->state = ec_voe_handler_state_error;
+    voe->request_state = EC_INT_REQUEST_INIT;
+
+    ec_datagram_init(&voe->datagram);
+    return ec_datagram_prealloc(&voe->datagram,
+            size + EC_MBOX_HEADER_SIZE + EC_VOE_HEADER_SIZE);
+}
+
+/****************************************************************************/
+
+/** VoE handler destructor.
+ */
+void ec_voe_handler_clear(
+        ec_voe_handler_t *voe /**< VoE handler. */
+        )
+{
+    ec_datagram_clear(&voe->datagram);
+}
+
+/****************************************************************************/
+
+/** Get usable memory size.
+ *
+ * \return Memory size.
+ */
+size_t ec_voe_handler_mem_size(
+        const ec_voe_handler_t *voe /**< VoE handler. */
+        )
+{
+    if (voe->datagram.mem_size >= EC_MBOX_HEADER_SIZE + EC_VOE_HEADER_SIZE)
+        return voe->datagram.mem_size -
+            (EC_MBOX_HEADER_SIZE + EC_VOE_HEADER_SIZE);
+    else
+        return 0;
+}
+
+/*****************************************************************************
+ * Application interface.
+ ****************************************************************************/
+
+int ecrt_voe_handler_send_header(ec_voe_handler_t *voe, uint32_t vendor_id,
+        uint16_t vendor_type)
+{
+    voe->vendor_id = vendor_id;
+    voe->vendor_type = vendor_type;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_voe_handler_received_header(const ec_voe_handler_t *voe,
+        uint32_t *vendor_id, uint16_t *vendor_type)
+{
+    uint8_t *header = voe->datagram.data + EC_MBOX_HEADER_SIZE;
+
+    if (vendor_id)
+        *vendor_id = EC_READ_U32(header);
+    if (vendor_type)
+        *vendor_type = EC_READ_U16(header + 4);
+    return 0;
+}
+
+/****************************************************************************/
+
+uint8_t *ecrt_voe_handler_data(const ec_voe_handler_t *voe)
+{
+    return voe->datagram.data + EC_MBOX_HEADER_SIZE + EC_VOE_HEADER_SIZE;
+}
+
+/****************************************************************************/
+
+size_t ecrt_voe_handler_data_size(const ec_voe_handler_t *voe)
+{
+    return voe->data_size;
+}
+
+/****************************************************************************/
+
+int ecrt_voe_handler_read(ec_voe_handler_t *voe)
+{
+    voe->dir = EC_DIR_INPUT;
+    voe->state = ec_voe_handler_state_read_start;
+    voe->request_state = EC_INT_REQUEST_BUSY;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_voe_handler_read_nosync(ec_voe_handler_t *voe)
+{
+    voe->dir = EC_DIR_INPUT;
+    voe->state = ec_voe_handler_state_read_nosync_start;
+    voe->request_state = EC_INT_REQUEST_BUSY;
+    return 0;
+}
+
+/****************************************************************************/
+
+int ecrt_voe_handler_write(ec_voe_handler_t *voe, size_t size)
+{
+    voe->dir = EC_DIR_OUTPUT;
+    voe->data_size = size;
+    voe->state = ec_voe_handler_state_write_start;
+    voe->request_state = EC_INT_REQUEST_BUSY;
+    return 0;
+}
+
+/****************************************************************************/
+
+ec_request_state_t ecrt_voe_handler_execute(ec_voe_handler_t *voe)
+{
+    if (voe->config->slave) { // FIXME locking?
+        voe->state(voe);
+        if (voe->request_state == EC_INT_REQUEST_BUSY) {
+            ec_master_queue_datagram(voe->config->master, &voe->datagram);
+        }
+    } else {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+    }
+
+    return ec_request_state_translation_table[voe->request_state];
+}
+
+/*****************************************************************************
+ * State functions.
+ ****************************************************************************/
+
+/** Start writing VoE data.
+ */
+void ec_voe_handler_state_write_start(ec_voe_handler_t *voe)
+{
+    ec_slave_t *slave = voe->config->slave;
+    uint8_t *data;
+
+    if (slave->master->debug_level) {
+        EC_SLAVE_DBG(slave, 0, "Writing %zu bytes of VoE data.\n",
+               voe->data_size);
+        ec_print_data(ecrt_voe_handler_data(voe), voe->data_size);
+    }
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_VOE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support VoE!\n");
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        return;
+    }
+
+    data = ec_slave_mbox_prepare_send(slave, &voe->datagram,
+            EC_MBOX_TYPE_VOE, EC_VOE_HEADER_SIZE + voe->data_size);
+    if (IS_ERR(data)) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        return;
+    }
+
+    EC_WRITE_U32(data,     voe->vendor_id);
+    EC_WRITE_U16(data + 4, voe->vendor_type);
+    /* data already in datagram */
+
+    voe->retries = EC_FSM_RETRIES;
+    voe->jiffies_start = jiffies;
+    voe->state = ec_voe_handler_state_write_response;
+}
+
+/****************************************************************************/
+
+/** Wait for the mailbox response.
+ */
+void ec_voe_handler_state_write_response(ec_voe_handler_t *voe)
+{
+    ec_datagram_t *datagram = &voe->datagram;
+    ec_slave_t *slave = voe->config->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && voe->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Failed to receive VoE write request datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        if (!datagram->working_counter) {
+            unsigned long diff_ms =
+                (jiffies - voe->jiffies_start) * 1000 / HZ;
+            if (diff_ms < EC_VOE_RESPONSE_TIMEOUT) {
+                EC_SLAVE_DBG(slave, 1, "Slave did not respond to"
+                        " VoE write request. Retrying after %lu ms...\n",
+                        diff_ms);
+                // no response; send request datagram again
+                return;
+            }
+        }
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Reception of VoE write request failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    EC_CONFIG_DBG(voe->config, 1, "VoE write request successful.\n");
+
+    voe->request_state = EC_INT_REQUEST_SUCCESS;
+    voe->state = ec_voe_handler_state_end;
+}
+
+/****************************************************************************/
+
+/** Start reading VoE data.
+ */
+void ec_voe_handler_state_read_start(ec_voe_handler_t *voe)
+{
+    ec_datagram_t *datagram = &voe->datagram;
+    ec_slave_t *slave = voe->config->slave;
+
+    EC_SLAVE_DBG(slave, 1, "Reading VoE data.\n");
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_VOE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support VoE!\n");
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        return;
+    }
+
+    ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+
+    voe->jiffies_start = jiffies;
+    voe->retries = EC_FSM_RETRIES;
+    voe->state = ec_voe_handler_state_read_check;
+}
+
+/****************************************************************************/
+
+/** Check for new data in the mailbox.
+ */
+void ec_voe_handler_state_read_check(ec_voe_handler_t *voe)
+{
+    ec_datagram_t *datagram = &voe->datagram;
+    ec_slave_t *slave = voe->config->slave;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && voe->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Failed to receive VoE mailbox check datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Reception of VoE mailbox check"
+                " datagram failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    if (!ec_slave_mbox_check(datagram)) {
+        unsigned long diff_ms =
+            (datagram->jiffies_received - voe->jiffies_start) * 1000 / HZ;
+        if (diff_ms >= EC_VOE_RESPONSE_TIMEOUT) {
+            voe->state = ec_voe_handler_state_error;
+            voe->request_state = EC_INT_REQUEST_FAILURE;
+            EC_SLAVE_ERR(slave, "Timeout while waiting for VoE data.\n");
+            return;
+        }
+
+        ec_slave_mbox_prepare_check(slave, datagram); // can not fail.
+        voe->retries = EC_FSM_RETRIES;
+        return;
+    }
+
+    // Fetch response
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+    voe->retries = EC_FSM_RETRIES;
+    voe->state = ec_voe_handler_state_read_response;
+}
+
+/****************************************************************************/
+
+/** Read the pending mailbox data.
+ */
+void ec_voe_handler_state_read_response(ec_voe_handler_t *voe)
+{
+    ec_datagram_t *datagram = &voe->datagram;
+    ec_slave_t *slave = voe->config->slave;
+    ec_master_t *master = voe->config->master;
+    uint8_t *data, mbox_prot;
+    size_t rec_size;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && voe->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Failed to receive VoE read datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Reception of VoE read response failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    data = ec_slave_mbox_fetch(slave, datagram, &mbox_prot, &rec_size);
+    if (IS_ERR(data)) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_VOE) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_WARN(slave, "Received mailbox protocol 0x%02X"
+                " as response.\n", mbox_prot);
+        ec_print_data(data, rec_size);
+        return;
+    }
+
+    if (rec_size < EC_VOE_HEADER_SIZE) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Received VoE header is"
+                " incomplete (%zu bytes)!\n", rec_size);
+        return;
+    }
+
+    if (master->debug_level) {
+        EC_CONFIG_DBG(voe->config, 0, "VoE data:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    voe->data_size = rec_size - EC_VOE_HEADER_SIZE;
+    voe->request_state = EC_INT_REQUEST_SUCCESS;
+    voe->state = ec_voe_handler_state_end; // success
+}
+
+/****************************************************************************/
+
+/** Start reading VoE data without sending a sync message before.
+ */
+void ec_voe_handler_state_read_nosync_start(ec_voe_handler_t *voe)
+{
+    ec_datagram_t *datagram = &voe->datagram;
+    ec_slave_t *slave = voe->config->slave;
+
+    EC_SLAVE_DBG(slave, 1, "Reading VoE data.\n");
+
+    if (!(slave->sii.mailbox_protocols & EC_MBOX_VOE)) {
+        EC_SLAVE_ERR(slave, "Slave does not support VoE!\n");
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        return;
+    }
+
+    ec_slave_mbox_prepare_fetch(slave, datagram); // can not fail.
+
+    voe->jiffies_start = jiffies;
+    voe->retries = EC_FSM_RETRIES;
+    voe->state = ec_voe_handler_state_read_nosync_response;
+}
+
+/****************************************************************************/
+
+/** Read the pending mailbox data without sending a sync message before. This
+ *  might lead to an empty reponse from the client.
+ */
+void ec_voe_handler_state_read_nosync_response(ec_voe_handler_t *voe)
+{
+    ec_datagram_t *datagram = &voe->datagram;
+    ec_slave_t *slave = voe->config->slave;
+    ec_master_t *master = voe->config->master;
+    uint8_t *data, mbox_prot;
+    size_t rec_size;
+
+    if (datagram->state == EC_DATAGRAM_TIMED_OUT && voe->retries--)
+        return;
+
+    if (datagram->state != EC_DATAGRAM_RECEIVED) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Failed to receive VoE read datagram: ");
+        ec_datagram_print_state(datagram);
+        return;
+    }
+
+    if (datagram->working_counter == 0) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_DBG(slave, 1, "Slave did not send VoE data.\n");
+        return;
+    }
+
+    if (datagram->working_counter != 1) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_WARN(slave, "Reception of VoE read response failed: ");
+        ec_datagram_print_wc_error(datagram);
+        return;
+    }
+
+    if (!(data = ec_slave_mbox_fetch(slave, datagram,
+                    &mbox_prot, &rec_size))) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        return;
+    }
+
+    if (mbox_prot != EC_MBOX_TYPE_VOE) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_WARN(slave, "Received mailbox protocol 0x%02X"
+                " as response.\n", mbox_prot);
+        ec_print_data(data, rec_size);
+        return;
+    }
+
+    if (rec_size < EC_VOE_HEADER_SIZE) {
+        voe->state = ec_voe_handler_state_error;
+        voe->request_state = EC_INT_REQUEST_FAILURE;
+        EC_SLAVE_ERR(slave, "Received VoE header is"
+                " incomplete (%zu bytes)!\n", rec_size);
+        return;
+    }
+
+    if (master->debug_level) {
+        EC_CONFIG_DBG(voe->config, 1, "VoE data:\n");
+        ec_print_data(data, rec_size);
+    }
+
+    voe->data_size = rec_size - EC_VOE_HEADER_SIZE;
+    voe->request_state = EC_INT_REQUEST_SUCCESS;
+    voe->state = ec_voe_handler_state_end; // success
+}
+
+/****************************************************************************/
+
+/** Successful termination state function.
+ */
+void ec_voe_handler_state_end(ec_voe_handler_t *voe)
+{
+}
+
+/****************************************************************************/
+
+/** Failure termination state function.
+ */
+void ec_voe_handler_state_error(ec_voe_handler_t *voe)
+{
+}
+
+/****************************************************************************/
+
+/** \cond */
+
+EXPORT_SYMBOL(ecrt_voe_handler_send_header);
+EXPORT_SYMBOL(ecrt_voe_handler_received_header);
+EXPORT_SYMBOL(ecrt_voe_handler_data);
+EXPORT_SYMBOL(ecrt_voe_handler_data_size);
+EXPORT_SYMBOL(ecrt_voe_handler_read);
+EXPORT_SYMBOL(ecrt_voe_handler_write);
+EXPORT_SYMBOL(ecrt_voe_handler_execute);
+
+/** \endcond */
+
+/****************************************************************************/
diff --git a/drivers/net/ethercat/master/voe_handler.h b/drivers/net/ethercat/master/voe_handler.h
new file mode 100644
index 000000000000..8e85372ccf05
--- /dev/null
+++ b/drivers/net/ethercat/master/voe_handler.h
@@ -0,0 +1,65 @@
+/*****************************************************************************
+ *
+ *  Copyright (C) 2006-2008  Florian Pose, Ingenieurgemeinschaft IgH
+ *
+ *  This file is part of the IgH EtherCAT Master.
+ *
+ *  The IgH EtherCAT Master is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License version 2, as
+ *  published by the Free Software Foundation.
+ *
+ *  The IgH EtherCAT Master is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+ *  Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with the IgH EtherCAT Master; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ ****************************************************************************/
+
+/**
+   \file
+   Vendor specific over EtherCAT protocol handler.
+*/
+
+/****************************************************************************/
+
+#ifndef __EC_VOE_HANDLER_H__
+#define __EC_VOE_HANDLER_H__
+
+#include <linux/list.h>
+
+#include "globals.h"
+#include "datagram.h"
+
+/****************************************************************************/
+
+/** Vendor specific over EtherCAT handler.
+ */
+struct ec_voe_handler {
+    struct list_head list; /**< List item. */
+    ec_slave_config_t *config; /**< Parent slave configuration. */
+    ec_datagram_t datagram; /**< State machine datagram. */
+    uint32_t vendor_id; /**< Vendor ID for the header. */
+    uint16_t vendor_type; /**< Vendor type for the header. */
+    size_t data_size; /**< Size of VoE data. */
+    ec_direction_t dir; /**< Direction. EC_DIR_OUTPUT means writing to
+                          the slave, EC_DIR_INPUT means reading from the
+                          slave. */
+    void (*state)(ec_voe_handler_t *); /**< State function */
+    ec_internal_request_state_t request_state; /**< Handler state. */
+    unsigned int retries; /**< retries upon datagram timeout */
+    unsigned long jiffies_start; /**< Timestamp for timeout calculation. */
+};
+
+/****************************************************************************/
+
+int ec_voe_handler_init(ec_voe_handler_t *, ec_slave_config_t *, size_t);
+void ec_voe_handler_clear(ec_voe_handler_t *);
+size_t ec_voe_handler_mem_size(const ec_voe_handler_t *);
+
+/****************************************************************************/
+
+#endif
diff --git a/drivers/net/ethernet/realtek/Kconfig b/drivers/net/ethernet/realtek/Kconfig
index 272c83bfdc6c..cd33a7bc36f2 100644
--- a/drivers/net/ethernet/realtek/Kconfig
+++ b/drivers/net/ethernet/realtek/Kconfig
@@ -121,6 +121,12 @@ config R8169_LEDS
 	  Optional support for controlling the NIC LED's with the netdev
 	  LED trigger.
 
+config R8127
+	tristate "Realtek RTL8127 10GbE PCIe Ethernet support (vendor driver)"
+	depends on PCI
+	help
+	  This driver supports Realtek RTL8127 10 Gigabit Ethernet controllers.
+
 config RTASE
 	tristate "Realtek Automotive Switch 9054/9068/9072/9075/9068/9071 PCIe Interface support"
 	depends on PCI
diff --git a/drivers/net/ethernet/realtek/Makefile b/drivers/net/ethernet/realtek/Makefile
index 046adf503ff4..a9675e8a23c2 100644
--- a/drivers/net/ethernet/realtek/Makefile
+++ b/drivers/net/ethernet/realtek/Makefile
@@ -3,6 +3,7 @@
 # Makefile for the Realtek network device drivers.
 #
 
+obj-$(CONFIG_R8127) += r8127/
 obj-$(CONFIG_8139CP) += 8139cp.o
 obj-$(CONFIG_8139TOO) += 8139too.o
 obj-$(CONFIG_ATP) += atp.o
diff --git a/drivers/net/ethernet/realtek/r8127/Makefile b/drivers/net/ethernet/realtek/r8127/Makefile
new file mode 100644
index 000000000000..114d823e68d4
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/Makefile
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Kbuild file for Realtek r8127 (vendor driver)
+
+# --- vendor feature toggles (keep same defaults as vendor src/Makefile) ---
+CONFIG_SOC_LAN            := y
+ENABLE_REALWOW_SUPPORT    := n
+ENABLE_DASH_SUPPORT       := n
+CONFIG_DOWN_SPEED_100     := n
+CONFIG_ASPM               := y
+ENABLE_S5WOL              := y
+ENABLE_S5_KEEP_CURR_MAC   := n
+ENABLE_EEE                := y
+ENABLE_S0_MAGIC_PACKET    := n
+ENABLE_TX_NO_CLOSE        := y
+ENABLE_MULTIPLE_TX_QUEUE  := y
+ENABLE_PTP_SUPPORT        := y
+ENABLE_RSS_SUPPORT        := y
+ENABLE_LIB_SUPPORT        := n
+ENABLE_USE_FIRMWARE_FILE  := n
+DISABLE_WOL_SUPPORT       := n
+DISABLE_MULTI_MSIX_VECTOR := n
+ENABLE_DOUBLE_VLAN        := n
+ENABLE_PAGE_REUSE         := n
+ENABLE_RX_PACKET_FRAGMENT := n
+ENABLE_GIGA_LITE          := y
+
+# --- module build ---
+obj-$(CONFIG_R8127) += r8127.o
+
+r8127-objs := r8127_n.o rtl_eeprom.o rtltool.o r8127_fiber.o
+
+# --- flags/objects following vendor rules ---
+ifeq ($(CONFIG_SOC_LAN),y)
+  ccflags-y += -DCONFIG_SOC_LAN
+endif
+
+ifeq ($(ENABLE_REALWOW_SUPPORT),y)
+  r8127-objs += r8127_realwow.o
+  ccflags-y  += -DENABLE_REALWOW_SUPPORT
+endif
+
+ifeq ($(ENABLE_DASH_SUPPORT),y)
+  r8127-objs += r8127_dash.o
+  ccflags-y  += -DENABLE_DASH_SUPPORT
+endif
+
+ccflags-y += -DCONFIG_R8127_NAPI
+ccflags-y += -DCONFIG_R8127_VLAN
+
+ifeq ($(CONFIG_DOWN_SPEED_100),y)
+  ccflags-y += -DCONFIG_DOWN_SPEED_100
+endif
+ifeq ($(CONFIG_ASPM),y)
+  ccflags-y += -DCONFIG_ASPM
+endif
+ifeq ($(ENABLE_S5WOL),y)
+  ccflags-y += -DENABLE_S5WOL
+endif
+ifeq ($(ENABLE_S5_KEEP_CURR_MAC),y)
+  ccflags-y += -DENABLE_S5_KEEP_CURR_MAC
+endif
+ifeq ($(ENABLE_EEE),y)
+  ccflags-y += -DENABLE_EEE
+endif
+ifeq ($(ENABLE_S0_MAGIC_PACKET),y)
+  ccflags-y += -DENABLE_S0_MAGIC_PACKET
+endif
+ifeq ($(ENABLE_TX_NO_CLOSE),y)
+  ccflags-y += -DENABLE_TX_NO_CLOSE
+endif
+ifeq ($(ENABLE_MULTIPLE_TX_QUEUE),y)
+  ccflags-y += -DENABLE_MULTIPLE_TX_QUEUE
+endif
+
+ifeq ($(ENABLE_PTP_SUPPORT),y)
+  r8127-objs += r8127_ptp.o
+  ccflags-y  += -DENABLE_PTP_SUPPORT
+endif
+ifeq ($(ENABLE_RSS_SUPPORT),y)
+  r8127-objs += r8127_rss.o
+  ccflags-y  += -DENABLE_RSS_SUPPORT
+endif
+ifeq ($(ENABLE_LIB_SUPPORT),y)
+  r8127-objs += r8127_lib.o
+  ccflags-y  += -DENABLE_LIB_SUPPORT
+endif
+ifeq ($(ENABLE_USE_FIRMWARE_FILE),y)
+  r8127-objs += r8127_firmware.o
+  ccflags-y  += -DENABLE_USE_FIRMWARE_FILE
+endif
+
+ifeq ($(DISABLE_WOL_SUPPORT),y)
+  ccflags-y += -DDISABLE_WOL_SUPPORT
+endif
+ifeq ($(DISABLE_MULTI_MSIX_VECTOR),y)
+  ccflags-y += -DDISABLE_MULTI_MSIX_VECTOR
+endif
+ifeq ($(ENABLE_DOUBLE_VLAN),y)
+  ccflags-y += -DENABLE_DOUBLE_VLAN
+endif
+ifeq ($(ENABLE_PAGE_REUSE),y)
+  ccflags-y += -DENABLE_PAGE_REUSE
+endif
+ifeq ($(ENABLE_RX_PACKET_FRAGMENT),y)
+  ccflags-y += -DENABLE_RX_PACKET_FRAGMENT
+endif
+ifeq ($(ENABLE_RX_PACKET_FRAGMENT),y)
+  ccflags-y += -DENABLE_RX_PACKET_FRAGMENT
+endif
+ifeq ($(ENABLE_GIGA_LITE),y)
+  ccflags-y += -DENABLE_GIGA_LITE
+endif
diff --git a/drivers/net/ethernet/realtek/r8127/r8127.h b/drivers/net/ethernet/realtek/r8127/r8127.h
new file mode 100644
index 000000000000..f7a384870fc0
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127.h
@@ -0,0 +1,3121 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef __R8127_H
+#define __R8127_H
+
+//#include <linux/pci.h>
+#include <linux/ethtool.h>
+#include <linux/interrupt.h>
+#include <linux/version.h>
+#include "r8127_dash.h"
+#include "r8127_realwow.h"
+#include "r8127_fiber.h"
+#ifdef ENABLE_PTP_SUPPORT
+#include "r8127_ptp.h"
+#endif
+#include "r8127_rss.h"
+#ifdef ENABLE_LIB_SUPPORT
+#include "r8127_lib.h"
+#endif
+
+#ifndef fallthrough
+#define fallthrough
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+#define netif_xmit_stopped netif_tx_queue_stopped
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0)
+#ifndef MDIO_AN_EEE_ADV_100TX
+#define MDIO_AN_EEE_ADV_100TX	0x0002	/* Advertise 100TX EEE cap */
+#endif
+#ifndef MDIO_AN_EEE_ADV_1000T
+#define MDIO_AN_EEE_ADV_1000T	0x0004	/* Advertise 1000T EEE cap */
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0)
+#define MDIO_EEE_100TX		MDIO_AN_EEE_ADV_100TX	/* 100TX EEE cap */
+#define MDIO_EEE_1000T		MDIO_AN_EEE_ADV_1000T	/* 1000T EEE cap */
+#define MDIO_EEE_10GT		0x0008	/* 10GT EEE cap */
+#define MDIO_EEE_1000KX		0x0010	/* 1000KX EEE cap */
+#define MDIO_EEE_10GKX4		0x0020	/* 10G KX4 EEE cap */
+#define MDIO_EEE_10GKR		0x0040	/* 10G KR EEE cap */
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) */
+
+static inline u32 mmd_eee_adv_to_ethtool_adv_t(u16 eee_adv)
+{
+        u32 adv = 0;
+
+        if (eee_adv & MDIO_EEE_100TX)
+                adv |= ADVERTISED_100baseT_Full;
+        if (eee_adv & MDIO_EEE_1000T)
+                adv |= ADVERTISED_1000baseT_Full;
+        if (eee_adv & MDIO_EEE_10GT)
+                adv |= ADVERTISED_10000baseT_Full;
+        if (eee_adv & MDIO_EEE_1000KX)
+                adv |= ADVERTISED_1000baseKX_Full;
+        if (eee_adv & MDIO_EEE_10GKX4)
+                adv |= ADVERTISED_10000baseKX4_Full;
+        if (eee_adv & MDIO_EEE_10GKR)
+                adv |= ADVERTISED_10000baseKR_Full;
+
+        return adv;
+}
+
+static inline u16 ethtool_adv_to_mmd_eee_adv_t(u32 adv)
+{
+        u16 reg = 0;
+
+        if (adv & ADVERTISED_100baseT_Full)
+                reg |= MDIO_EEE_100TX;
+        if (adv & ADVERTISED_1000baseT_Full)
+                reg |= MDIO_EEE_1000T;
+        if (adv & ADVERTISED_10000baseT_Full)
+                reg |= MDIO_EEE_10GT;
+        if (adv & ADVERTISED_1000baseKX_Full)
+                reg |= MDIO_EEE_1000KX;
+        if (adv & ADVERTISED_10000baseKX4_Full)
+                reg |= MDIO_EEE_10GKX4;
+        if (adv & ADVERTISED_10000baseKR_Full)
+                reg |= MDIO_EEE_10GKR;
+
+        return reg;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,7,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0)
+static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
+{
+        return skb->transport_header != ~0U;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,9,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,20,0)
+static inline void linkmode_set_bit(int nr, volatile unsigned long *addr)
+{
+        __set_bit(nr, addr);
+}
+
+static inline void linkmode_clear_bit(int nr, volatile unsigned long *addr)
+{
+        __clear_bit(nr, addr);
+}
+
+static inline int linkmode_test_bit(int nr, volatile unsigned long *addr)
+{
+        return test_bit(nr, addr);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,20,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,0,0)
+static inline void linkmode_mod_bit(int nr, volatile unsigned long *addr,
+                                    int set)
+{
+        if (set)
+                linkmode_set_bit(nr, addr);
+        else
+                linkmode_clear_bit(nr, addr);
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,0,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,3,0)
+static inline
+ssize_t strscpy(char *dest, const char *src, size_t count)
+{
+        long res = 0;
+
+        if (count == 0)
+                return -E2BIG;
+
+        while (count) {
+                char c;
+
+                c = src[res];
+                dest[res] = c;
+                if (!c)
+                        return res;
+                res++;
+                count--;
+        }
+
+        /* Hit buffer length without finding a NUL; force NUL-termination. */
+        if (res)
+                dest[res-1] = '\0';
+
+        return -E2BIG;
+}
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0))
+static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22))
+        return skb->head + skb->csum_start;
+#else /* < 2.6.22 */
+        return skb_transport_header(skb);
+#endif
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
+                                        unsigned int bytes)
+{}
+static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
+                unsigned int pkts,
+                unsigned int bytes)
+{}
+static inline void netdev_tx_reset_queue(struct netdev_queue *q) {}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0)
+static inline void fsleep(unsigned long usecs)
+{
+        if (usecs <= 10)
+                udelay(usecs);
+        else if (usecs <= 20000)
+                usleep_range(usecs, 2 * usecs);
+        else
+                msleep(DIV_ROUND_UP(usecs, 1000));
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,2,0)
+#define netdev_xmit_more() (0)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0)
+#define netif_testing_on(dev)
+#define netif_testing_off(dev)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,2,0)
+#define netdev_sw_irq_coalesce_default_on(dev)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,2,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,32)
+typedef int netdev_tx_t;
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,12,0)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,1,9)
+static inline bool page_is_pfmemalloc(struct page *page)
+{
+        /*
+         * Page index cannot be this large so this must be
+         * a pfmemalloc page.
+         */
+        return page->index == -1UL;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4,1,9) */
+static inline bool dev_page_is_reusable(struct page *page)
+{
+        return likely(page_to_nid(page) == numa_mem_id() &&
+                      !page_is_pfmemalloc(page));
+}
+#endif
+
+/*
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)&& !defined(ENABLE_LIB_SUPPORT)
+#define RTL_USE_NEW_INTR_API
+#endif
+*/
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0)
+#define dma_map_page_attrs(dev, page, offset, size, dir, attrs) \
+	dma_map_page(dev, page, offset, size, dir)
+#define dma_unmap_page_attrs(dev, page, size, dir, attrs) \
+	 dma_unmap_page(dev, page, size, dir)
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+#define page_ref_inc(page) atomic_inc(&page->_count)
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,4,216)
+#define page_ref_count(page) atomic_read(&page->_count)
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,4,216)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+#define skb_transport_offset(skb) (skb->h.raw - skb->data)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,26)
+#define device_set_wakeup_enable(dev, val)	do {} while (0)
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0)
+static inline void ether_addr_copy(u8 *dst, const u8 *src)
+{
+        u16 *a = (u16 *)dst;
+        const u16 *b = (const u16 *)src;
+
+        a[0] = b[0];
+        a[1] = b[1];
+        a[2] = b[2];
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,15,0)
+#define IS_ERR_OR_NULL(ptr)			(!ptr)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,13,0)
+#define reinit_completion(x)			((x)->done = 0)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,39)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32)
+#define pm_runtime_mark_last_busy(x)
+#define pm_runtime_put_autosuspend(x)		pm_runtime_put(x)
+#define pm_runtime_put_sync_autosuspend(x)	pm_runtime_put_sync(x)
+
+static inline bool pm_runtime_suspended(struct device *dev)
+{
+        return dev->power.runtime_status == RPM_SUSPENDED
+               && !dev->power.disable_depth;
+}
+
+static inline bool pm_runtime_active(struct device *dev)
+{
+        return dev->power.runtime_status == RPM_ACTIVE
+               || dev->power.disable_depth;
+}
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,36)
+#define queue_delayed_work(long_wq, work, delay)	schedule_delayed_work(work, delay)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34)
+#define netif_printk(priv, type, level, netdev, fmt, args...)	\
+	do {								\
+		if (netif_msg_##type(priv))				\
+			printk(level "%s: " fmt,(netdev)->name , ##args); \
+	} while (0)
+
+#define netif_emerg(priv, type, netdev, fmt, args...)		\
+		netif_printk(priv, type, KERN_EMERG, netdev, fmt, ##args)
+#define netif_alert(priv, type, netdev, fmt, args...)		\
+		netif_printk(priv, type, KERN_ALERT, netdev, fmt, ##args)
+#define netif_crit(priv, type, netdev, fmt, args...)		\
+		netif_printk(priv, type, KERN_CRIT, netdev, fmt, ##args)
+#define netif_err(priv, type, netdev, fmt, args...)		\
+		netif_printk(priv, type, KERN_ERR, netdev, fmt, ##args)
+#define netif_warn(priv, type, netdev, fmt, args...)		\
+		netif_printk(priv, type, KERN_WARNING, netdev, fmt, ##args)
+#define netif_notice(priv, type, netdev, fmt, args...)		\
+		netif_printk(priv, type, KERN_NOTICE, netdev, fmt, ##args)
+#define netif_info(priv, type, netdev, fmt, args...)		\
+		netif_printk(priv, type, KERN_INFO, (netdev), fmt, ##args)
+#endif
+#endif
+#endif
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
+#define setup_timer(_timer, _function, _data) \
+do { \
+	(_timer)->function = _function; \
+	(_timer)->data = _data; \
+	init_timer(_timer); \
+} while (0)
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0)
+#if defined(skb_vlan_tag_present) && !defined(vlan_tx_tag_present)
+#define vlan_tx_tag_present skb_vlan_tag_present
+#endif
+#if defined(skb_vlan_tag_get) && !defined(vlan_tx_tag_get)
+#define vlan_tx_tag_get skb_vlan_tag_get
+#endif
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0)
+
+#define RTL_ALLOC_SKB_INTR(napi, length) dev_alloc_skb(length)
+#define R8127_USE_NAPI_ALLOC_SKB 0
+#ifdef CONFIG_R8127_NAPI
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0)
+#undef RTL_ALLOC_SKB_INTR
+#define RTL_ALLOC_SKB_INTR(napi, length) napi_alloc_skb(napi, length)
+#undef R8127_USE_NAPI_ALLOC_SKB
+#define R8127_USE_NAPI_ALLOC_SKB 1
+#endif
+#endif
+
+#define RTL_BUILD_SKB_INTR(data, frag_size) build_skb(data, frag_size)
+#ifdef CONFIG_R8127_NAPI
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0)
+#undef RTL_BUILD_SKB_INTR
+#define RTL_BUILD_SKB_INTR(data, frag_size) napi_build_skb(data, frag_size)
+#endif
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0)
+#define eth_random_addr(addr) random_ether_addr(addr)
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0)
+#define netdev_features_t  u32
+#endif
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,5,0)
+#define NETIF_F_ALL_CSUM        NETIF_F_CSUM_MASK
+#else
+#ifndef NETIF_F_ALL_CSUM
+#define NETIF_F_ALL_CSUM        NETIF_F_CSUM_MASK
+#endif
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37)
+#define ENABLE_R8127_PROCFS
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,11,0)
+#define ENABLE_R8127_SYSFS
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+#define NETIF_F_HW_VLAN_RX	NETIF_F_HW_VLAN_CTAG_RX
+#define NETIF_F_HW_VLAN_TX	NETIF_F_HW_VLAN_CTAG_TX
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,8,0)
+#define __devinit
+#define __devexit
+#define __devexit_p(func)   func
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+#define CHECKSUM_PARTIAL CHECKSUM_HW
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
+#define irqreturn_t void
+#define IRQ_HANDLED    1
+#define IRQ_NONE   0
+#define IRQ_RETVAL(x)
+#endif
+
+#ifndef NETIF_F_RXALL
+#define NETIF_F_RXALL  0
+#endif
+
+#ifndef NETIF_F_RXFCS
+#define NETIF_F_RXFCS  0
+#endif
+
+#if !defined(HAVE_FREE_NETDEV) && (LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0))
+#define free_netdev(x)  kfree(x)
+#endif
+
+#ifndef SET_NETDEV_DEV
+#define SET_NETDEV_DEV(net, pdev)
+#endif
+
+#ifndef SET_MODULE_OWNER
+#define SET_MODULE_OWNER(dev)
+#endif
+
+#ifndef SA_SHIRQ
+#define SA_SHIRQ IRQF_SHARED
+#endif
+
+#ifndef NETIF_F_GSO
+#define gso_size    tso_size
+#define gso_segs    tso_segs
+#endif
+
+#ifndef PCI_VENDOR_ID_DLINK
+#define PCI_VENDOR_ID_DLINK 0x1186
+#endif
+
+#ifndef dma_mapping_error
+#define dma_mapping_error(a,b) 0
+#endif
+
+#ifndef netif_err
+#define netif_err(a,b,c,d)
+#endif
+
+#ifndef AUTONEG_DISABLE
+#define AUTONEG_DISABLE   0x00
+#endif
+
+#ifndef AUTONEG_ENABLE
+#define AUTONEG_ENABLE    0x01
+#endif
+
+#ifndef BMCR_SPEED1000
+#define BMCR_SPEED1000  0x0040
+#endif
+
+#ifndef BMCR_SPEED100
+#define BMCR_SPEED100   0x2000
+#endif
+
+#ifndef BMCR_SPEED10
+#define BMCR_SPEED10    0x0000
+#endif
+
+#ifndef SPEED_UNKNOWN
+#define SPEED_UNKNOWN   -1
+#endif
+
+#ifndef DUPLEX_UNKNOWN
+#define DUPLEX_UNKNOWN  0xff
+#endif
+
+#ifndef SUPPORTED_Pause
+#define SUPPORTED_Pause  (1 << 13)
+#endif
+
+#ifndef SUPPORTED_Asym_Pause
+#define SUPPORTED_Asym_Pause  (1 << 14)
+#endif
+
+#ifndef  MDIO_EEE_100TX
+#define  MDIO_EEE_100TX  0x0002
+#endif
+
+#ifndef  MDIO_EEE_1000T
+#define  MDIO_EEE_1000T  0x0004
+#endif
+
+#ifndef  MDIO_EEE_2_5GT
+#define  MDIO_EEE_2_5GT  0x0001
+#endif
+
+#ifndef  MDIO_EEE_5GT
+#define  MDIO_EEE_5GT  0x0002
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
+#define ethtool_keee ethtool_eee
+#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t ethtool_adv_to_mmd_eee_adv_t
+static inline u32 rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t(u32 adv)
+{
+        u32 result = 0;
+
+        if (adv & SUPPORTED_2500baseX_Full)
+                result |= MDIO_EEE_2_5GT;
+
+        return result;
+}
+#else
+#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t linkmode_to_mii_eee_cap1_t
+#define rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t linkmode_to_mii_eee_cap2_t
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29)
+#ifdef CONFIG_NET_POLL_CONTROLLER
+#define RTL_NET_POLL_CONTROLLER dev->poll_controller=rtl8127_netpoll
+#else
+#define RTL_NET_POLL_CONTROLLER
+#endif
+
+#ifdef CONFIG_R8127_VLAN
+#define RTL_SET_VLAN dev->vlan_rx_register=rtl8127_vlan_rx_register
+#else
+#define RTL_SET_VLAN
+#endif
+
+#define RTL_NET_DEVICE_OPS(ops) dev->open=rtl8127_open; \
+                    dev->hard_start_xmit=rtl8127_start_xmit; \
+                    dev->get_stats=rtl8127_get_stats; \
+                    dev->stop=rtl8127_close; \
+                    dev->tx_timeout=rtl8127_tx_timeout; \
+                    dev->set_multicast_list=rtl8127_set_rx_mode; \
+                    dev->change_mtu=rtl8127_change_mtu; \
+                    dev->set_mac_address=rtl8127_set_mac_address; \
+                    dev->do_ioctl=rtl8127_do_ioctl; \
+                    RTL_NET_POLL_CONTROLLER; \
+                    RTL_SET_VLAN;
+#else
+#define RTL_NET_DEVICE_OPS(ops) dev->netdev_ops=&ops
+#endif
+
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#ifndef TRUE
+#define TRUE  1
+#endif
+
+#ifndef false
+#define false 0
+#endif
+
+#ifndef true
+#define true  1
+#endif
+
+//Hardware will continue interrupt 10 times after interrupt finished.
+#define RTK_KEEP_INTERRUPT_COUNT (10)
+
+//the low 32 bit address of receive buffer must be 8-byte alignment.
+#ifndef NET_IP_ALIGN
+#define NET_IP_ALIGN        2
+#endif
+#define R8127_RX_ALIGN        NET_IP_ALIGN
+
+#ifdef CONFIG_R8127_NAPI
+#define NAPI_SUFFIX "-NAPI"
+#else
+#define NAPI_SUFFIX ""
+#endif
+
+#if defined(ENABLE_REALWOW_SUPPORT)
+#define REALWOW_SUFFIX "-REALWOW"
+#else
+#define REALWOW_SUFFIX ""
+#endif
+
+#if defined(ENABLE_DASH_SUPPORT)
+#define DASH_SUFFIX "-DASH"
+#else
+#define DASH_SUFFIX ""
+#endif
+
+#if defined(ENABLE_PTP_SUPPORT)
+#define PTP_SUFFIX "-PTP"
+#else
+#define PTP_SUFFIX ""
+#endif
+
+#if defined(ENABLE_RSS_SUPPORT)
+#define RSS_SUFFIX "-RSS"
+#else
+#define RSS_SUFFIX ""
+#endif
+
+#define RTL8127_VERSION "11.015.00" NAPI_SUFFIX DASH_SUFFIX REALWOW_SUFFIX PTP_SUFFIX RSS_SUFFIX
+#define MODULENAME "r8127"
+#define PFX MODULENAME ": "
+
+#define GPL_CLAIM "\
+r8127  Copyright (C) 2025 Realtek NIC software team <nicfae@realtek.com> \n \
+This program comes with ABSOLUTELY NO WARRANTY; for details, please see <http://www.gnu.org/licenses/>. \n \
+This is free software, and you are welcome to redistribute it under certain conditions; see <http://www.gnu.org/licenses/>. \n"
+
+#ifdef RTL8127_DEBUG
+#define assert(expr) \
+        if(!(expr)) {                   \
+            printk("Assertion failed! %s,%s,%s,line=%d\n", \
+            #expr,__FILE__,__FUNCTION__,__LINE__);      \
+        }
+#define dprintk(fmt, args...)   do { printk(PFX fmt, ## args); } while (0)
+#else
+#define assert(expr) do {} while (0)
+#define dprintk(fmt, args...)   do {} while (0)
+#endif /* RTL8127_DEBUG */
+
+#define R8127_MSG_DEFAULT \
+    (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | NETIF_MSG_IFDOWN)
+
+#ifdef CONFIG_R8127_NAPI
+#define rtl8127_rx_hwaccel_skb      vlan_hwaccel_receive_skb
+#define rtl8127_rx_quota(count, quota)  min(count, quota)
+#else
+#define rtl8127_rx_hwaccel_skb      vlan_hwaccel_rx
+#define rtl8127_rx_quota(count, quota)  count
+#endif
+
+#ifdef CONFIG_R8127_NAPI
+#define r8127_spin_lock(lock, flags)  (void)flags;spin_lock_bh(lock)
+#define r8127_spin_unlock(lock, flags)  (void)flags;spin_unlock_bh(lock)
+#else
+#define r8127_spin_lock(lock, flags)  spin_lock_irqsave(lock, flags)
+#define r8127_spin_unlock(lock, flags)  spin_unlock_irqrestore(lock, flags)
+#endif
+
+/* MAC address length */
+#ifndef MAC_ADDR_LEN
+#define MAC_ADDR_LEN    6
+#endif
+
+#ifndef MAC_PROTOCOL_LEN
+#define MAC_PROTOCOL_LEN    2
+#endif
+
+#ifndef ETH_FCS_LEN
+#define ETH_FCS_LEN	  4
+#endif
+
+#ifndef NETIF_F_TSO6
+#define NETIF_F_TSO6  0
+#endif
+
+#define Reserved2_data  7
+#define RX_DMA_BURST_unlimited  7   /* Maximum PCI burst, '7' is unlimited */
+#define RX_DMA_BURST_512    5
+#define RX_DMA_BURST_256    4
+#define TX_DMA_BURST_unlimited  7
+#define TX_DMA_BURST_1024   6
+#define TX_DMA_BURST_512    5
+#define TX_DMA_BURST_256    4
+#define TX_DMA_BURST_128    3
+#define TX_DMA_BURST_64     2
+#define TX_DMA_BURST_32     1
+#define TX_DMA_BURST_16     0
+#define Reserved1_data  0x3F
+#define RxPacketMaxSize 0x3FE8  /* 16K - 1 - ETH_HLEN - VLAN - CRC... */
+#define Jumbo_Frame_1k  ETH_DATA_LEN
+#define Jumbo_Frame_2k  (2*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define Jumbo_Frame_3k  (3*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define Jumbo_Frame_4k  (4*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define Jumbo_Frame_5k  (5*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define Jumbo_Frame_6k  (6*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define Jumbo_Frame_7k  (7*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define Jumbo_Frame_8k  (8*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define Jumbo_Frame_9k  (9*1024 - ETH_HLEN - VLAN_HLEN - ETH_FCS_LEN)
+#define InterFrameGap   0x03    /* 3 means InterFrameGap = the shortest one */
+#define RxEarly_off_V1 (0x07 << 11)
+#define RxEarly_off_V2 (1 << 11)
+#define Rx_Single_fetch_V2 (1 << 14)
+#define Rx_Close_Multiple (1 << 21)
+#define Rx_Fetch_Number_8 (1 << 30)
+
+#define R8127_REGS_SIZE     (256)
+#define R8127_MAC_REGS_SIZE     (256)
+#define R8127_PHY_REGS_SIZE     (16*2)
+#define R8127_EPHY_REGS_SIZE  	(31*2)
+#define R8127_ERI_REGS_SIZE  	(0x100)
+#define R8127_REGS_DUMP_SIZE     (0x400)
+#define R8127_PCI_REGS_SIZE  	(0x100)
+#define R8127_NAPI_WEIGHT   64
+
+#define R8127_MAX_MSIX_VEC_8125A   4
+#define R8127_MAX_MSIX_VEC_8125B   32
+#define R8127_MAX_MSIX_VEC_8127    32
+#define R8127_MAX_MSIX_VEC_8127AP  64
+#define R8127_MAX_MSIX_VEC_8125D   32
+#define R8127_MIN_MSIX_VEC_8125B   22
+#define R8127_MIN_MSIX_VEC_8125BP  31
+#define R8127_MIN_MSIX_VEC_8125D   20
+#define R8127_MIN_MSIX_VEC_8127    30
+#define R8127_MIN_MSIX_VEC_8127AP  49
+#define R8127_MAX_MSIX_VEC   64
+#define R8127_MAX_RX_QUEUES_VEC_V3 (16)
+#define R8127_MAX_RX_QUEUES_VEC_V4 (8)
+
+#define RTL8127_TX_TIMEOUT  (6 * HZ)
+#define RTL8127_LINK_TIMEOUT    (1 * HZ)
+#define RTL8127_ESD_TIMEOUT (2 * HZ)
+#define RTL8127_DASH_TIMEOUT    (0)
+
+#define rtl8127_rx_page_size(order) (PAGE_SIZE << order)
+
+#define MAX_NUM_TX_DESC 1024    /* Maximum number of Tx descriptor registers */
+#define MAX_NUM_RX_DESC 1024    /* Maximum number of Rx descriptor registers */
+
+#define MIN_NUM_TX_DESC 256    /* Minimum number of Tx descriptor registers */
+#define MIN_NUM_RX_DESC 256    /* Minimum number of Rx descriptor registers */
+
+#define NUM_TX_DESC MAX_NUM_TX_DESC    /* Number of Tx descriptor registers */
+#define NUM_RX_DESC MAX_NUM_RX_DESC    /* Number of Rx descriptor registers */
+
+#ifdef ENABLE_DOUBLE_VLAN
+#define RX_BUF_SIZE 0x05F6  /* 0x05F6(1526) = 1514 + 8(double vlan) + 4(crc) bytes */
+#define RT_VALN_HLEN 8      /* 8(double vlan) bytes */
+#else
+#define RX_BUF_SIZE 0x05F2  /* 0x05F2(1522) = 1514 + 4(single vlan) + 4(crc) bytes */
+#define RT_VALN_HLEN 4      /* 4(single vlan) bytes */
+#endif
+
+#define R8127_MAX_TX_QUEUES (2)
+#define R8127_MAX_RX_QUEUES_V2 (4)
+#define R8127_MAX_RX_QUEUES_V3 (16)
+#define R8127_MAX_RX_QUEUES R8127_MAX_RX_QUEUES_V3
+#define R8127_MAX_QUEUES R8127_MAX_RX_QUEUES
+
+#define OCP_STD_PHY_BASE	0xa400
+
+//Channel Wait Count
+#define R8127_CHANNEL_WAIT_COUNT (20000)
+#define R8127_CHANNEL_WAIT_TIME (1)  // 1us
+#define R8127_CHANNEL_EXIT_DELAY_TIME (20)  //20us
+
+#ifdef ENABLE_LIB_SUPPORT
+#define R8127_MULTI_RX_Q(tp) 0
+#else
+#define R8127_MULTI_RX_Q(tp) (tp->num_rx_rings > 1)
+#endif
+
+#define NODE_ADDRESS_SIZE 6
+
+#define SHORT_PACKET_PADDING_BUF_SIZE 256
+
+#define RTK_MAGIC_DEBUG_VALUE 0x0badbeef
+
+/* write/read MMIO register */
+#define RTL_W8(tp, reg, val8)	writeb((val8), tp->mmio_addr + (reg))
+#define RTL_W16(tp, reg, val16)	writew((val16), tp->mmio_addr + (reg))
+#define RTL_W32(tp, reg, val32)	writel((val32), tp->mmio_addr + (reg))
+#define RTL_R8(tp, reg)		readb(tp->mmio_addr + (reg))
+#define RTL_R16(tp, reg)		readw(tp->mmio_addr + (reg))
+#define RTL_R32(tp, reg)		((unsigned long) readl(tp->mmio_addr + (reg)))
+
+#ifndef DMA_64BIT_MASK
+#define DMA_64BIT_MASK  0xffffffffffffffffULL
+#endif
+
+#ifndef DMA_32BIT_MASK
+#define DMA_32BIT_MASK  0x00000000ffffffffULL
+#endif
+
+#ifndef NETDEV_TX_OK
+#define NETDEV_TX_OK 0      /* driver took care of packet */
+#endif
+
+#ifndef NETDEV_TX_BUSY
+#define NETDEV_TX_BUSY 1    /* driver tx path was busy*/
+#endif
+
+#ifndef NETDEV_TX_LOCKED
+#define NETDEV_TX_LOCKED -1t /* driver tx lock was already taken */
+#endif
+
+#ifndef ADVERTISED_Pause
+#define ADVERTISED_Pause    (1 << 13)
+#endif
+
+#ifndef ADVERTISED_Asym_Pause
+#define ADVERTISED_Asym_Pause   (1 << 14)
+#endif
+
+#ifndef ADVERTISE_PAUSE_CAP
+#define ADVERTISE_PAUSE_CAP 0x400
+#endif
+
+#ifndef ADVERTISE_PAUSE_ASYM
+#define ADVERTISE_PAUSE_ASYM    0x800
+#endif
+
+#ifndef MII_CTRL1000
+#define MII_CTRL1000        0x09
+#endif
+
+#ifndef ADVERTISE_1000FULL
+#define ADVERTISE_1000FULL  0x200
+#endif
+
+#ifndef ADVERTISE_1000HALF
+#define ADVERTISE_1000HALF  0x100
+#endif
+
+#ifndef BIT_ULL
+#define BIT_ULL(nr)		(1ULL << (nr))
+#endif
+
+#ifndef ADVERTISED_2500baseX_Full
+#define ADVERTISED_2500baseX_Full  0x8000
+#endif
+#define RTK_ADVERTISED_5000baseX_Full  BIT_ULL(48)
+#define RTK_SUPPORTED_5000baseX_Full BIT_ULL(48)
+
+#define RTK_ADVERTISE_2500FULL  0x80
+#define RTK_ADVERTISE_5000FULL  0x100
+#define RTK_ADVERTISE_10000FULL  0x1000
+#define RTK_LPA_ADVERTISE_2500FULL  0x20
+#define RTK_LPA_ADVERTISE_5000FULL  0x40
+#define RTK_LPA_ADVERTISE_10000FULL  0x800
+
+#define RTK_EEE_ADVERTISE_2500FULL  BIT(0)
+#define RTK_EEE_ADVERTISE_5000FULL  BIT(1)
+#define RTK_LPA_EEE_ADVERTISE_2500FULL  BIT(0)
+#define RTK_LPA_EEE_ADVERTISE_5000FULL  BIT(1)
+
+/* Tx NO CLOSE */
+#define MAX_TX_NO_CLOSE_DESC_PTR_V2 0x10000
+#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V2 0xFFFF
+#define MAX_TX_NO_CLOSE_DESC_PTR_V3 0x100000000
+#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V3 0xFFFFFFFF
+#define MAX_TX_NO_CLOSE_DESC_PTR_V4 0x80000000
+#define MAX_TX_NO_CLOSE_DESC_PTR_MASK_V4 0x7FFFFFFF
+#define TX_NO_CLOSE_SW_PTR_MASK_V2 0x1FFFF
+
+#ifndef ETH_MIN_MTU
+#define ETH_MIN_MTU  68
+#endif
+
+#define D0_SPEED_UP_SPEED_DISABLE    0
+#define D0_SPEED_UP_SPEED_1000       1
+#define D0_SPEED_UP_SPEED_2500       2
+#define D0_SPEED_UP_SPEED_5000       3
+#define D0_SPEED_UP_SPEED_10000      4
+
+#define RTL8127_MAC_MCU_PAGE_SIZE 256 //256 words
+
+#ifndef WRITE_ONCE
+#define WRITE_ONCE(var, val) (*((volatile typeof(val) *)(&(var))) = (val))
+#endif
+#ifndef READ_ONCE
+#define READ_ONCE(var) (*((volatile typeof(var) *)(&(var))))
+#endif
+
+#ifndef SPEED_5000
+#define SPEED_5000		5000
+#endif
+
+#ifndef SPEED_10000
+#define SPEED_10000		10000
+#endif
+
+#define R8127_LINK_STATE_OFF 0
+#define R8127_LINK_STATE_ON 1
+#define R8127_LINK_STATE_UNKNOWN 2
+
+/*****************************************************************************/
+
+//#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3)
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,4,27)) || \
+     ((LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)) && \
+      (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3))))
+/* copied from linux kernel 2.6.20 include/linux/netdev.h */
+#define NETDEV_ALIGN        32
+#define NETDEV_ALIGN_CONST  (NETDEV_ALIGN - 1)
+
+static inline void *netdev_priv(struct net_device *dev)
+{
+        return (char *)dev + ((sizeof(struct net_device)
+                               + NETDEV_ALIGN_CONST)
+                              & ~NETDEV_ALIGN_CONST);
+}
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,3)
+
+/*****************************************************************************/
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+#define RTLDEV  tp
+#else
+#define RTLDEV  dev
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+/*****************************************************************************/
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
+typedef struct net_device *napi_ptr;
+typedef int *napi_budget;
+
+#define napi dev
+#define RTL_NAPI_CONFIG(ndev, priv, function, weig) ndev->poll=function;    \
+                                ndev->weight=weig;
+#define RTL_NAPI_QUOTA(budget, ndev)            min(*budget, ndev->quota)
+#define RTL_GET_PRIV(stuct_ptr, priv_struct)        netdev_priv(stuct_ptr)
+#define RTL_GET_NETDEV(priv_ptr)
+#define RTL_RX_QUOTA(budget)          *budget
+#define RTL_NAPI_QUOTA_UPDATE(ndev, work_done, budget)  *budget -= work_done;   \
+                                ndev->quota -= work_done;
+#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done)        netif_rx_complete(dev)
+#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi)       netif_rx_schedule_prep(dev)
+#define __RTL_NETIF_RX_SCHEDULE(dev, napi)      __netif_rx_schedule(dev)
+#define RTL_NAPI_RETURN_VALUE               work_done >= work_to_do
+#define RTL_NAPI_ENABLE(dev, napi)          netif_poll_enable(dev)
+#define RTL_NAPI_DISABLE(dev, napi)         netif_poll_disable(dev)
+#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
+#else
+typedef struct napi_struct *napi_ptr;
+typedef int napi_budget;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,1,0)
+#define RTL_NAPI_CONFIG(ndev, priv, function, weight)   netif_napi_add_weight(ndev, &priv->napi, function, weight)
+#else
+#define RTL_NAPI_CONFIG(ndev, priv, function, weight)   netif_napi_add(ndev, &priv->napi, function, weight)
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(6,1,0)
+#define RTL_NAPI_QUOTA(budget, ndev)            min(budget, budget)
+#define RTL_GET_PRIV(stuct_ptr, priv_struct)        container_of(stuct_ptr, priv_struct, stuct_ptr)
+#define RTL_GET_NETDEV(priv_ptr)            struct net_device *dev = priv_ptr->dev;
+#define RTL_RX_QUOTA(budget)          budget
+#define RTL_NAPI_QUOTA_UPDATE(ndev, work_done, budget)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29)
+#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done)        netif_rx_complete(dev, napi)
+#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi)       netif_rx_schedule_prep(dev, napi)
+#define __RTL_NETIF_RX_SCHEDULE(dev, napi)      __netif_rx_schedule(dev, napi)
+#endif
+#if LINUX_VERSION_CODE == KERNEL_VERSION(2,6,29)
+#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done)        netif_rx_complete(napi)
+#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi)       netif_rx_schedule_prep(napi)
+#define __RTL_NETIF_RX_SCHEDULE(dev, napi)      __netif_rx_schedule(napi)
+#endif
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,29)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0)
+#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done)        napi_complete_done(napi, work_done)
+#else
+#define RTL_NETIF_RX_COMPLETE(dev, napi, work_done)        napi_complete(napi)
+#endif
+#define RTL_NETIF_RX_SCHEDULE_PREP(dev, napi)       napi_schedule_prep(napi)
+#define __RTL_NETIF_RX_SCHEDULE(dev, napi)      __napi_schedule(napi)
+#endif
+#define RTL_NAPI_RETURN_VALUE work_done
+#define RTL_NAPI_ENABLE(dev, napi)          napi_enable(napi)
+#define RTL_NAPI_DISABLE(dev, napi)         napi_disable(napi)
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27)
+#define RTL_NAPI_DEL(priv)
+#else
+#define RTL_NAPI_DEL(priv)   netif_napi_del(&priv->napi)
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,27)
+
+/*****************************************************************************/
+#ifdef CONFIG_R8127_NAPI
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget)          napi_consume_skb(skb, budget)
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)
+#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget)          dev_consume_skb_any(skb);
+#else
+#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget)          dev_kfree_skb_any(skb);
+#endif  //LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#else   //CONFIG_R8127_NAPI
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)
+#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget)          dev_consume_skb_any(skb);
+#else
+#define RTL_NAPI_CONSUME_SKB_ANY(skb, budget)          dev_kfree_skb_any(skb);
+#endif
+#endif  //CONFIG_R8127_NAPI
+
+/*****************************************************************************/
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9)
+#ifdef __CHECKER__
+#define __iomem __attribute__((noderef, address_space(2)))
+extern void __chk_io_ptr(void __iomem *);
+#define __bitwise __attribute__((bitwise))
+#else
+#define __iomem
+#define __chk_io_ptr(x) (void)0
+#define __bitwise
+#endif
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9)
+
+/*****************************************************************************/
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,8)
+#ifdef __CHECKER__
+#define __force __attribute__((force))
+#else
+#define __force
+#endif
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,8)
+
+#ifndef module_param
+#define module_param(v,t,p) MODULE_PARM(v, "i");
+#endif
+
+#ifndef PCI_DEVICE
+#define PCI_DEVICE(vend,dev) \
+    .vendor = (vend), .device = (dev), \
+    .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID
+#endif
+
+/*****************************************************************************/
+/* 2.5.28 => 2.4.23 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,28))
+
+static inline void _kc_synchronize_irq(void)
+{
+        synchronize_irq();
+}
+#undef synchronize_irq
+#define synchronize_irq(X) _kc_synchronize_irq()
+
+#include <linux/tqueue.h>
+#define work_struct tq_struct
+#undef INIT_WORK
+#define INIT_WORK(a,b,c) INIT_TQUEUE(a,(void (*)(void *))b,c)
+#undef container_of
+#define container_of list_entry
+#define schedule_work schedule_task
+#define flush_scheduled_work flush_scheduled_tasks
+#endif /* 2.5.28 => 2.4.17 */
+
+/*****************************************************************************/
+/* 2.6.4 => 2.6.0 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,4))
+#define MODULE_VERSION(_version) MODULE_INFO(version, _version)
+#endif /* 2.6.4 => 2.6.0 */
+/*****************************************************************************/
+/* 2.6.0 => 2.5.28 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0))
+#define MODULE_INFO(version, _version)
+#ifndef CONFIG_E1000_DISABLE_PACKET_SPLIT
+#define CONFIG_E1000_DISABLE_PACKET_SPLIT 1
+#endif
+
+#define pci_set_consistent_dma_mask(dev,mask) 1
+
+#undef dev_put
+#define dev_put(dev) __dev_put(dev)
+
+#ifndef skb_fill_page_desc
+#define skb_fill_page_desc _kc_skb_fill_page_desc
+extern void _kc_skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, int off, int size);
+#endif
+
+#ifndef pci_dma_mapping_error
+#define pci_dma_mapping_error _kc_pci_dma_mapping_error
+static inline int _kc_pci_dma_mapping_error(dma_addr_t dma_addr)
+{
+        return dma_addr == 0;
+}
+#endif
+
+#undef ALIGN
+#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1))
+
+#endif /* 2.6.0 => 2.5.28 */
+
+/*****************************************************************************/
+/* 2.4.22 => 2.4.17 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,4,22))
+#define pci_name(x) ((x)->slot_name)
+#endif /* 2.4.22 => 2.4.17 */
+
+/*****************************************************************************/
+/* 2.6.5 => 2.6.0 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5))
+#define pci_dma_sync_single_for_cpu pci_dma_sync_single
+#define pci_dma_sync_single_for_device  pci_dma_sync_single_for_cpu
+#endif /* 2.6.5 => 2.6.0 */
+
+/*****************************************************************************/
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
+/*
+ * initialize a work-struct's func and data pointers:
+ */
+#define PREPARE_WORK(_work, _func, _data)           \
+    do {                            \
+        (_work)->func = _func;              \
+        (_work)->data = _data;              \
+    } while (0)
+
+#endif
+/*****************************************************************************/
+/* 2.6.4 => 2.6.0 */
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(2,4,25) && \
+     LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)) || \
+    (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) && \
+      LINUX_VERSION_CODE < KERNEL_VERSION(2,6,4)))
+#define ETHTOOL_OPS_COMPAT
+#endif /* 2.6.4 => 2.6.0 */
+
+/*****************************************************************************/
+/* Installations with ethtool version without eeprom, adapter id, or statistics
+ * support */
+
+#ifndef ETH_GSTRING_LEN
+#define ETH_GSTRING_LEN 32
+#endif
+
+#ifndef ETHTOOL_GSTATS
+#define ETHTOOL_GSTATS 0x1d
+#undef ethtool_drvinfo
+#define ethtool_drvinfo k_ethtool_drvinfo
+struct k_ethtool_drvinfo {
+        u32 cmd;
+        char driver[32];
+        char version[32];
+        char fw_version[32];
+        char bus_info[32];
+        char reserved1[32];
+        char reserved2[16];
+        u32 n_stats;
+        u32 testinfo_len;
+        u32 eedump_len;
+        u32 regdump_len;
+};
+
+struct ethtool_stats {
+        u32 cmd;
+        u32 n_stats;
+        u64 data[0];
+};
+#endif /* ETHTOOL_GSTATS */
+
+#ifndef ETHTOOL_PHYS_ID
+#define ETHTOOL_PHYS_ID 0x1c
+#endif /* ETHTOOL_PHYS_ID */
+
+#ifndef ETHTOOL_GSTRINGS
+#define ETHTOOL_GSTRINGS 0x1b
+enum ethtool_stringset {
+        ETH_SS_TEST             = 0,
+        ETH_SS_STATS,
+};
+struct ethtool_gstrings {
+        u32 cmd;            /* ETHTOOL_GSTRINGS */
+        u32 string_set;     /* string set id e.c. ETH_SS_TEST, etc*/
+        u32 len;            /* number of strings in the string set */
+        u8 data[0];
+};
+#endif /* ETHTOOL_GSTRINGS */
+
+#ifndef ETHTOOL_TEST
+#define ETHTOOL_TEST 0x1a
+enum ethtool_test_flags {
+        ETH_TEST_FL_OFFLINE = (1 << 0),
+        ETH_TEST_FL_FAILED  = (1 << 1),
+};
+struct ethtool_test {
+        u32 cmd;
+        u32 flags;
+        u32 reserved;
+        u32 len;
+        u64 data[0];
+};
+#endif /* ETHTOOL_TEST */
+
+#ifndef ETHTOOL_GEEPROM
+#define ETHTOOL_GEEPROM 0xb
+#undef ETHTOOL_GREGS
+struct ethtool_eeprom {
+        u32 cmd;
+        u32 magic;
+        u32 offset;
+        u32 len;
+        u8 data[0];
+};
+
+struct ethtool_value {
+        u32 cmd;
+        u32 data;
+};
+#endif /* ETHTOOL_GEEPROM */
+
+#ifndef ETHTOOL_GLINK
+#define ETHTOOL_GLINK 0xa
+#endif /* ETHTOOL_GLINK */
+
+#ifndef ETHTOOL_GREGS
+#define ETHTOOL_GREGS       0x00000004 /* Get NIC registers */
+#define ethtool_regs _kc_ethtool_regs
+/* for passing big chunks of data */
+struct _kc_ethtool_regs {
+        u32 cmd;
+        u32 version; /* driver-specific, indicates different chips/revs */
+        u32 len; /* bytes */
+        u8 data[0];
+};
+#endif /* ETHTOOL_GREGS */
+
+#ifndef ETHTOOL_GMSGLVL
+#define ETHTOOL_GMSGLVL     0x00000007 /* Get driver message level */
+#endif
+#ifndef ETHTOOL_SMSGLVL
+#define ETHTOOL_SMSGLVL     0x00000008 /* Set driver msg level, priv. */
+#endif
+#ifndef ETHTOOL_NWAY_RST
+#define ETHTOOL_NWAY_RST    0x00000009 /* Restart autonegotiation, priv */
+#endif
+#ifndef ETHTOOL_GLINK
+#define ETHTOOL_GLINK       0x0000000a /* Get link status */
+#endif
+#ifndef ETHTOOL_GEEPROM
+#define ETHTOOL_GEEPROM     0x0000000b /* Get EEPROM data */
+#endif
+#ifndef ETHTOOL_SEEPROM
+#define ETHTOOL_SEEPROM     0x0000000c /* Set EEPROM data */
+#endif
+#ifndef ETHTOOL_GCOALESCE
+#define ETHTOOL_GCOALESCE   0x0000000e /* Get coalesce config */
+/* for configuring coalescing parameters of chip */
+#define ethtool_coalesce _kc_ethtool_coalesce
+struct _kc_ethtool_coalesce {
+        u32 cmd;    /* ETHTOOL_{G,S}COALESCE */
+
+        /* How many usecs to delay an RX interrupt after
+         * a packet arrives.  If 0, only rx_max_coalesced_frames
+         * is used.
+         */
+        u32 rx_coalesce_usecs;
+
+        /* How many packets to delay an RX interrupt after
+         * a packet arrives.  If 0, only rx_coalesce_usecs is
+         * used.  It is illegal to set both usecs and max frames
+         * to zero as this would cause RX interrupts to never be
+         * generated.
+         */
+        u32 rx_max_coalesced_frames;
+
+        /* Same as above two parameters, except that these values
+         * apply while an IRQ is being serviced by the host.  Not
+         * all cards support this feature and the values are ignored
+         * in that case.
+         */
+        u32 rx_coalesce_usecs_irq;
+        u32 rx_max_coalesced_frames_irq;
+
+        /* How many usecs to delay a TX interrupt after
+         * a packet is sent.  If 0, only tx_max_coalesced_frames
+         * is used.
+         */
+        u32 tx_coalesce_usecs;
+
+        /* How many packets to delay a TX interrupt after
+         * a packet is sent.  If 0, only tx_coalesce_usecs is
+         * used.  It is illegal to set both usecs and max frames
+         * to zero as this would cause TX interrupts to never be
+         * generated.
+         */
+        u32 tx_max_coalesced_frames;
+
+        /* Same as above two parameters, except that these values
+         * apply while an IRQ is being serviced by the host.  Not
+         * all cards support this feature and the values are ignored
+         * in that case.
+         */
+        u32 tx_coalesce_usecs_irq;
+        u32 tx_max_coalesced_frames_irq;
+
+        /* How many usecs to delay in-memory statistics
+         * block updates.  Some drivers do not have an in-memory
+         * statistic block, and in such cases this value is ignored.
+         * This value must not be zero.
+         */
+        u32 stats_block_coalesce_usecs;
+
+        /* Adaptive RX/TX coalescing is an algorithm implemented by
+         * some drivers to improve latency under low packet rates and
+         * improve throughput under high packet rates.  Some drivers
+         * only implement one of RX or TX adaptive coalescing.  Anything
+         * not implemented by the driver causes these values to be
+         * silently ignored.
+         */
+        u32 use_adaptive_rx_coalesce;
+        u32 use_adaptive_tx_coalesce;
+
+        /* When the packet rate (measured in packets per second)
+         * is below pkt_rate_low, the {rx,tx}_*_low parameters are
+         * used.
+         */
+        u32 pkt_rate_low;
+        u32 rx_coalesce_usecs_low;
+        u32 rx_max_coalesced_frames_low;
+        u32 tx_coalesce_usecs_low;
+        u32 tx_max_coalesced_frames_low;
+
+        /* When the packet rate is below pkt_rate_high but above
+         * pkt_rate_low (both measured in packets per second) the
+         * normal {rx,tx}_* coalescing parameters are used.
+         */
+
+        /* When the packet rate is (measured in packets per second)
+         * is above pkt_rate_high, the {rx,tx}_*_high parameters are
+         * used.
+         */
+        u32 pkt_rate_high;
+        u32 rx_coalesce_usecs_high;
+        u32 rx_max_coalesced_frames_high;
+        u32 tx_coalesce_usecs_high;
+        u32 tx_max_coalesced_frames_high;
+
+        /* How often to do adaptive coalescing packet rate sampling,
+         * measured in seconds.  Must not be zero.
+         */
+        u32 rate_sample_interval;
+};
+#endif /* ETHTOOL_GCOALESCE */
+
+#ifndef ETHTOOL_SCOALESCE
+#define ETHTOOL_SCOALESCE   0x0000000f /* Set coalesce config. */
+#endif
+#ifndef ETHTOOL_GRINGPARAM
+#define ETHTOOL_GRINGPARAM  0x00000010 /* Get ring parameters */
+/* for configuring RX/TX ring parameters */
+#define ethtool_ringparam _kc_ethtool_ringparam
+struct _kc_ethtool_ringparam {
+        u32 cmd;    /* ETHTOOL_{G,S}RINGPARAM */
+
+        /* Read only attributes.  These indicate the maximum number
+         * of pending RX/TX ring entries the driver will allow the
+         * user to set.
+         */
+        u32 rx_max_pending;
+        u32 rx_mini_max_pending;
+        u32 rx_jumbo_max_pending;
+        u32 tx_max_pending;
+
+        /* Values changeable by the user.  The valid values are
+         * in the range 1 to the "*_max_pending" counterpart above.
+         */
+        u32 rx_pending;
+        u32 rx_mini_pending;
+        u32 rx_jumbo_pending;
+        u32 tx_pending;
+};
+#endif /* ETHTOOL_GRINGPARAM */
+
+#ifndef ETHTOOL_SRINGPARAM
+#define ETHTOOL_SRINGPARAM  0x00000011 /* Set ring parameters, priv. */
+#endif
+#ifndef ETHTOOL_GPAUSEPARAM
+#define ETHTOOL_GPAUSEPARAM 0x00000012 /* Get pause parameters */
+/* for configuring link flow control parameters */
+#define ethtool_pauseparam _kc_ethtool_pauseparam
+struct _kc_ethtool_pauseparam {
+        u32 cmd;    /* ETHTOOL_{G,S}PAUSEPARAM */
+
+        /* If the link is being auto-negotiated (via ethtool_cmd.autoneg
+         * being true) the user may set 'autonet' here non-zero to have the
+         * pause parameters be auto-negotiated too.  In such a case, the
+         * {rx,tx}_pause values below determine what capabilities are
+         * advertised.
+         *
+         * If 'autoneg' is zero or the link is not being auto-negotiated,
+         * then {rx,tx}_pause force the driver to use/not-use pause
+         * flow control.
+         */
+        u32 autoneg;
+        u32 rx_pause;
+        u32 tx_pause;
+};
+#endif /* ETHTOOL_GPAUSEPARAM */
+
+#ifndef ETHTOOL_SPAUSEPARAM
+#define ETHTOOL_SPAUSEPARAM 0x00000013 /* Set pause parameters. */
+#endif
+#ifndef ETHTOOL_GRXCSUM
+#define ETHTOOL_GRXCSUM     0x00000014 /* Get RX hw csum enable (ethtool_value) */
+#endif
+#ifndef ETHTOOL_SRXCSUM
+#define ETHTOOL_SRXCSUM     0x00000015 /* Set RX hw csum enable (ethtool_value) */
+#endif
+#ifndef ETHTOOL_GTXCSUM
+#define ETHTOOL_GTXCSUM     0x00000016 /* Get TX hw csum enable (ethtool_value) */
+#endif
+#ifndef ETHTOOL_STXCSUM
+#define ETHTOOL_STXCSUM     0x00000017 /* Set TX hw csum enable (ethtool_value) */
+#endif
+#ifndef ETHTOOL_GSG
+#define ETHTOOL_GSG     0x00000018 /* Get scatter-gather enable
+* (ethtool_value) */
+#endif
+#ifndef ETHTOOL_SSG
+#define ETHTOOL_SSG     0x00000019 /* Set scatter-gather enable
+* (ethtool_value). */
+#endif
+#ifndef ETHTOOL_TEST
+#define ETHTOOL_TEST        0x0000001a /* execute NIC self-test, priv. */
+#endif
+#ifndef ETHTOOL_GSTRINGS
+#define ETHTOOL_GSTRINGS    0x0000001b /* get specified string set */
+#endif
+#ifndef ETHTOOL_PHYS_ID
+#define ETHTOOL_PHYS_ID     0x0000001c /* identify the NIC */
+#endif
+#ifndef ETHTOOL_GSTATS
+#define ETHTOOL_GSTATS      0x0000001d /* get NIC-specific statistics */
+#endif
+#ifndef ETHTOOL_GTSO
+#define ETHTOOL_GTSO        0x0000001e /* Get TSO enable (ethtool_value) */
+#endif
+#ifndef ETHTOOL_STSO
+#define ETHTOOL_STSO        0x0000001f /* Set TSO enable (ethtool_value) */
+#endif
+
+#ifndef ETHTOOL_BUSINFO_LEN
+#define ETHTOOL_BUSINFO_LEN 32
+#endif
+
+/*****************************************************************************/
+
+enum RTL8127_registers {
+        MAC0            = 0x00,     /* Ethernet hardware address. */
+        MAC4            = 0x04,
+        MAR0            = 0x08,     /* Multicast filter. */
+        CounterAddrLow      = 0x10,
+        CounterAddrHigh     = 0x14,
+        CustomLED       = 0x18,
+        TxDescStartAddrLow  = 0x20,
+        TxDescStartAddrHigh = 0x24,
+        TxHDescStartAddrLow = 0x28,
+        TxHDescStartAddrHigh    = 0x2c,
+        FLASH           = 0x30,
+        INT_CFG0_8125   = 0x34,
+        ERSR            = 0x36,
+        ChipCmd         = 0x37,
+        TxPoll          = 0x38,
+        IntrMask        = 0x3C,
+        IntrStatus      = 0x3E,
+        TxConfig        = 0x40,
+        RxConfig        = 0x44,
+        TCTR            = 0x48,
+        Cfg9346         = 0x50,
+        Config0         = 0x51,
+        Config1         = 0x52,
+        Config2         = 0x53,
+        Config3         = 0x54,
+        Config4         = 0x55,
+        Config5         = 0x56,
+        TDFNR           = 0x57,
+        TimeInt0        = 0x58,
+        TimeInt1        = 0x5C,
+        PHYAR           = 0x60,
+        CSIDR           = 0x64,
+        CSIAR           = 0x68,
+        PHYstatus       = 0x6C,
+        MACDBG          = 0x6D,
+        GPIO            = 0x6E,
+        PMCH            = 0x6F,
+        ERIDR           = 0x70,
+        ERIAR           = 0x74,
+        INT_CFG1_8125   = 0x7A,
+        EPHY_RXER_NUM   = 0x7C,
+        EPHYAR          = 0x80,
+        TimeInt2        = 0x8C,
+        OCPDR           = 0xB0,
+        MACOCP          = 0xB0,
+        OCPAR           = 0xB4,
+        SecMAC0         = 0xB4,
+        SecMAC4         = 0xB8,
+        PHYOCP          = 0xB8,
+        DBG_reg         = 0xD1,
+        TwiCmdReg       = 0xD2,
+        MCUCmd_reg      = 0xD3,
+        RxMaxSize       = 0xDA,
+        EFUSEAR         = 0xDC,
+        CPlusCmd        = 0xE0,
+        IntrMitigate    = 0xE2,
+        RxDescAddrLow   = 0xE4,
+        RxDescAddrHigh  = 0xE8,
+        MTPS            = 0xEC,
+        FuncEvent       = 0xF0,
+        PPSW            = 0xF2,
+        FuncEventMask   = 0xF4,
+        TimeInt3        = 0xF4,
+        FuncPresetState = 0xF8,
+        CMAC_IBCR0      = 0xF8,
+        CMAC_IBCR2      = 0xF9,
+        CMAC_IBIMR0     = 0xFA,
+        CMAC_IBISR0     = 0xFB,
+        FuncForceEvent  = 0xFC,
+        //8125
+        IMR0_8125          = 0x38,
+        ISR0_8125          = 0x3C,
+        TPPOLL_8125        = 0x90,
+        IMR1_8125          = 0x800,
+        ISR1_8125          = 0x802,
+        IMR2_8125          = 0x804,
+        ISR2_8125          = 0x806,
+        IMR3_8125          = 0x808,
+        ISR3_8125          = 0x80A,
+        BACKUP_ADDR0_8125  = 0x19E0,
+        BACKUP_ADDR1_8125  = 0X19E4,
+        TCTR0_8125         = 0x0048,
+        TCTR1_8125         = 0x004C,
+        TCTR2_8125         = 0x0088,
+        TCTR3_8125         = 0x001C,
+        TIMER_INT0_8125    = 0x0058,
+        TIMER_INT1_8125    = 0x005C,
+        TIMER_INT2_8125    = 0x008C,
+        TIMER_INT3_8125    = 0x00F4,
+        INT_MITI_V2_0_RX   = 0x0A00,
+        INT_MITI_V2_0_TX   = 0x0A02,
+        INT_MITI_V2_1_RX   = 0x0A08,
+        INT_MITI_V2_1_TX   = 0x0A0A,
+        IMR_V2_CLEAR_REG_8125 = 0x0D00,
+        ISR_V2_8125           = 0x0D04,
+        IMR_V2_SET_REG_8125   = 0x0D0C,
+        TDU_STA_8125       = 0x0D08,
+        RDU_STA_8125       = 0x0D0A,
+        IMR_V4_L2_CLEAR_REG_8125 = 0x0D10,
+        IMR_V4_L2_SET_REG_8125   = 0x0D18,
+        ISR_V4_L2_8125     = 0x0D14,
+        SW_TAIL_PTR0_8125BP = 0x0D30,
+        SW_TAIL_PTR1_8125BP = 0x0D38,
+        HW_CLO_PTR0_8125BP = 0x0D34,
+        HW_CLO_PTR1_8125BP = 0x0D3C,
+        DOUBLE_VLAN_CONFIG = 0x1000,
+        TX_NEW_CTRL        = 0x203E,
+        TNPDS_Q1_LOW_8125  = 0x2100,
+        PLA_TXQ0_IDLE_CREDIT = 0x2500,
+        PLA_TXQ1_IDLE_CREDIT = 0x2504,
+        SW_TAIL_PTR0_8125  = 0x2800,
+        HW_CLO_PTR0_8125   = 0x2802,
+        SW_TAIL_PTR0_8126  = 0x2800,
+        HW_CLO_PTR0_8126   = 0x2800,
+        RDSAR_Q1_LOW_8125  = 0x4000,
+        RSS_CTRL_8125      = 0x4500,
+        Q_NUM_CTRL_8125    = 0x4800,
+        RSS_KEY_8125       = 0x4600,
+        RSS_INDIRECTION_TBL_8125_V2 = 0x4700,
+        EEE_TXIDLE_TIMER_8125   = 0x6048,
+        PTP_CTRL_8125      = 0x6800,
+        PTP_STATUS_8125    = 0x6802,
+        PTP_ISR_8125       = 0x6804,
+        PTP_IMR_8125       = 0x6805,
+        PTP_TIME_CORRECT_CMD_8125    = 0x6806,
+        PTP_SOFT_CONFIG_Time_NS_8125 = 0x6808,
+        PTP_SOFT_CONFIG_Time_S_8125  = 0x680C,
+        PTP_SOFT_CONFIG_Time_Sign    = 0x6812,
+        PTP_LOCAL_Time_SUB_NS_8125   = 0x6814,
+        PTP_LOCAL_Time_NS_8125       = 0x6818,
+        PTP_LOCAL_Time_S_8125        = 0x681C,
+        PTP_Time_SHIFTER_S_8125      = 0x6856,
+        PPS_RISE_TIME_NS_8125        = 0x68A0,
+        PPS_RISE_TIME_S_8125         = 0x68A4,
+        PTP_EGRESS_TIME_BASE_NS_8125 = 0XCF20,
+        PTP_EGRESS_TIME_BASE_S_8125  = 0XCF24,
+        PTP_CTL                 = 0xE400,
+        PTP_INER                = 0xE402,
+        PTP_INSR                = 0xE404,
+        PTP_SYNCE_CTL           = 0xE406,
+        PTP_GEN_CFG             = 0xE408,
+        PTP_CLK_CFG_8126        = 0xE410,
+        PTP_CFG_NS_LO_8126      = 0xE412,
+        PTP_CFG_NS_HI_8126      = 0xE414,
+        PTP_CFG_S_LO_8126       = 0xE416,
+        PTP_CFG_S_MI_8126       = 0xE418,
+        PTP_CFG_S_HI_8126       = 0xE41A,
+        PTP_TAI_CFG             = 0xE420,
+        PTP_TAI_TS_S_LO         = 0xE42A,
+        PTP_TAI_TS_S_HI         = 0xE42C,
+        PTP_TRX_TS_STA          = 0xE430,
+        PTP_TRX_TS_NS_LO        = 0xE446,
+        PTP_TRX_TS_NS_HI        = 0xE448,
+        PTP_TRX_TS_S_LO         = 0xE44A,
+        PTP_TRX_TS_S_MI         = 0xE44C,
+        PTP_TRX_TS_S_HI         = 0xE44E,
+
+        //TCAM
+        TCAM_NOTVALID_ADDR           = 0xA000,
+        TCAM_VALID_ADDR              = 0xA800,
+        TCAM_MAC_ADDR                = 448,
+        TCAM_VLAN_TAG                = 496,
+        //TCAM V2
+        TCAM_NOTVALID_ADDR_V2           = 0xA000,
+        TCAM_VALID_ADDR_V2              = 0xB000,
+        TCAM_MAC_ADDR_V2                = 0x00,
+        TCAM_VLAN_TAG_V2                = 0x03,
+
+        RADMFIFO_PROTECT    = 0x0402,
+
+        RISC_IMR_8127AP     = 0x0D20,
+        RISC_ISR_8127AP     = 0x0D24,
+};
+
+enum RTL8127_register_content {
+        /* InterruptStatusBits */
+        SYSErr      = 0x8000,
+        PCSTimeout  = 0x4000,
+        SWInt       = 0x0100,
+        TxDescUnavail   = 0x0080,
+        RxFIFOOver  = 0x0040,
+        LinkChg     = 0x0020,
+        RxDescUnavail   = 0x0010,
+        TxErr       = 0x0008,
+        TxOK        = 0x0004,
+        RxErr       = 0x0002,
+        RxOK        = 0x0001,
+        RxDU1       = 0x0002,
+        RxOK1       = 0x0001,
+
+        /* RxStatusDesc */
+        RxRWT = (1 << 22),
+        RxRES = (1 << 21),
+        RxRUNT = (1 << 20),
+        RxCRC = (1 << 19),
+
+        RxRWT_V3 = (1 << 18),
+        RxRES_V3 = (1 << 20),
+        RxRUNT_V3 = (1 << 19),
+        RxCRC_V3 = (1 << 17),
+
+        RxRES_V4 = (1 << 22),
+        RxRUNT_V4 = (1 << 21),
+        RxCRC_V4 = (1 << 20),
+
+        /* ChipCmdBits */
+        StopReq  = 0x80,
+        CmdReset = 0x10,
+        CmdRxEnb = 0x08,
+        CmdTxEnb = 0x04,
+        RxBufEmpty = 0x01,
+
+        /* Cfg9346Bits */
+        Cfg9346_EEM_MASK = 0xC0,
+        Cfg9346_Lock = 0x00,
+        Cfg9346_Unlock = 0xC0,
+        Cfg9346_EEDO = (1 << 0),
+        Cfg9346_EEDI = (1 << 1),
+        Cfg9346_EESK = (1 << 2),
+        Cfg9346_EECS = (1 << 3),
+        Cfg9346_EEM0 = (1 << 6),
+        Cfg9346_EEM1 = (1 << 7),
+
+        /* rx_mode_bits */
+        AcceptErr = 0x20,
+        AcceptRunt = 0x10,
+        AcceptBroadcast = 0x08,
+        AcceptMulticast = 0x04,
+        AcceptMyPhys = 0x02,
+        AcceptAllPhys = 0x01,
+        AcceppVlanPhys = 0x8000,
+
+        /* Transmit Priority Polling*/
+        HPQ = 0x80,
+        NPQ = 0x40,
+        FSWInt = 0x01,
+
+        /* RxConfigBits */
+        Reserved2_shift = 13,
+        RxCfgDMAShift = 8,
+        EnableRxDescV3 = (1 << 24),
+        EnableRxDescV4_1 = (1 << 24),
+        EnableOuterVlan = (1 << 23),
+        EnableInnerVlan = (1 << 22),
+        RxCfg_128_int_en = (1 << 15),
+        RxCfg_fet_multi_en = (1 << 14),
+        RxCfg_half_refetch = (1 << 13),
+        RxCfg_pause_slot_en = (1 << 11),
+        RxCfg_9356SEL = (1 << 6),
+        EnableRxDescV4_0 = (1 << 1), //not in rcr
+
+        /* TxConfigBits */
+        TxInterFrameGapShift = 24,
+        TxDMAShift = 8, /* DMA burst value (0-7) is shift this many bits */
+        TxMACLoopBack = (1 << 17),  /* MAC loopback */
+
+        /* Config1 register */
+        LEDS1       = (1 << 7),
+        LEDS0       = (1 << 6),
+        Speed_down  = (1 << 4),
+        MEMMAP      = (1 << 3),
+        IOMAP       = (1 << 2),
+        VPD         = (1 << 1),
+        PMEnable    = (1 << 0), /* Power Management Enable */
+
+        /* Config2 register */
+        PMSTS_En    = (1 << 5),
+
+        /* Config3 register */
+        Isolate_en  = (1 << 12), /* Isolate enable */
+        MagicPacket = (1 << 5), /* Wake up when receives a Magic Packet */
+        LinkUp      = (1 << 4), /* This bit is reserved in RTL8125B.*/
+        /* Wake up when the cable connection is re-established */
+        ECRCEN      = (1 << 3), /* This bit is reserved in RTL8125B*/
+        Jumbo_En0   = (1 << 2), /* This bit is reserved in RTL8125B*/
+        RDY_TO_L23  = (1 << 1), /* This bit is reserved in RTL8125B*/
+        Beacon_en   = (1 << 0), /* This bit is reserved in RTL8125B*/
+
+        /* Config4 register */
+        Jumbo_En1   = (1 << 1), /* This bit is reserved in RTL8125B*/
+
+        /* Config5 register */
+        BWF     = (1 << 6), /* Accept Broadcast wakeup frame */
+        MWF     = (1 << 5), /* Accept Multicast wakeup frame */
+        UWF     = (1 << 4), /* Accept Unicast wakeup frame */
+        LanWake     = (1 << 1), /* LanWake enable/disable */
+        PMEStatus   = (1 << 0), /* PME status can be reset by PCI RST# */
+
+        /* CPlusCmd */
+        EnableBist  = (1 << 15),
+        Macdbgo_oe  = (1 << 14),
+        Normal_mode = (1 << 13),
+        Force_halfdup   = (1 << 12),
+        Force_rxflow_en = (1 << 11),
+        Force_txflow_en = (1 << 10),
+        Cxpl_dbg_sel    = (1 << 9),//This bit is reserved in RTL8125B
+        ASF     = (1 << 8),//This bit is reserved in RTL8125C
+        PktCntrDisable  = (1 << 7),
+        RxVlan      = (1 << 6),
+        RxChkSum    = (1 << 5),
+        Macdbgo_sel = 0x001C,
+        INTT_0      = 0x0000,
+        INTT_1      = 0x0001,
+        INTT_2      = 0x0002,
+        INTT_3      = 0x0003,
+
+        /* rtl8127_PHYstatus */
+        PowerSaveStatus = 0x80,
+        _1000bpsL = 0x80000,
+        _10000bpsF = 0x4000,
+        _10000bpsL = 0x2000,
+        _5000bpsF = 0x1000,
+        _5000bpsL = 0x800,
+        _2500bpsF = 0x400,
+        _2500bpsL = 0x200,
+        TxFlowCtrl = 0x40,
+        RxFlowCtrl = 0x20,
+        _1000bpsF = 0x10,
+        _100bps = 0x08,
+        _10bps = 0x04,
+        LinkStatus = 0x02,
+        FullDup = 0x01,
+
+        /* DBG_reg */
+        Fix_Nak_1 = (1 << 4),
+        Fix_Nak_2 = (1 << 3),
+        DBGPIN_E2 = (1 << 0),
+
+        /* ResetCounterCommand */
+        CounterReset = 0x1,
+        /* DumpCounterCommand */
+        CounterDump = 0x8,
+
+        /* PHY access */
+        PHYAR_Flag = 0x80000000,
+        PHYAR_Write = 0x80000000,
+        PHYAR_Read = 0x00000000,
+        PHYAR_Reg_Mask = 0x1f,
+        PHYAR_Reg_shift = 16,
+        PHYAR_Data_Mask = 0xffff,
+
+        /* EPHY access */
+        EPHYAR_Flag = 0x80000000,
+        EPHYAR_Write = 0x80000000,
+        EPHYAR_Read = 0x00000000,
+        EPHYAR_Reg_Mask = 0x3f,
+        EPHYAR_Reg_Mask_v2 = 0x7f,
+        EPHYAR_Reg_shift = 16,
+        EPHYAR_Data_Mask = 0xffff,
+        EPHYAR_EXT_ADDR = 0x0ffe,
+
+        /* CSI access */
+        CSIAR_Flag = 0x80000000,
+        CSIAR_Write = 0x80000000,
+        CSIAR_Read = 0x00000000,
+        CSIAR_ByteEn = 0x0f,
+        CSIAR_ByteEn_shift = 12,
+        CSIAR_Addr_Mask = 0x0fff,
+
+        /* ERI access */
+        ERIAR_Flag = 0x80000000,
+        ERIAR_Write = 0x80000000,
+        ERIAR_Read = 0x00000000,
+        ERIAR_Addr_Align = 4, /* ERI access register address must be 4 byte alignment */
+        ERIAR_ExGMAC = 0,
+        ERIAR_MSIX = 1,
+        ERIAR_ASF = 2,
+        ERIAR_OOB = 2,
+        ERIAR_Type_shift = 16,
+        ERIAR_ByteEn = 0x0f,
+        ERIAR_ByteEn_shift = 12,
+
+        /* OCP GPHY access */
+        OCPDR_Write = 0x80000000,
+        OCPDR_Read = 0x00000000,
+        OCPDR_Reg_Mask = 0xFF,
+        OCPDR_Data_Mask = 0xFFFF,
+        OCPDR_GPHY_Reg_shift = 16,
+        OCPAR_Flag = 0x80000000,
+        OCPAR_GPHY_Write = 0x8000F060,
+        OCPAR_GPHY_Read = 0x0000F060,
+        OCPR_Write = 0x80000000,
+        OCPR_Read = 0x00000000,
+        OCPR_Addr_Reg_shift = 16,
+        OCPR_Flag = 0x80000000,
+        OCP_STD_PHY_BASE_PAGE = 0x0A40,
+
+        /* MCU Command */
+        Now_is_oob = (1 << 7),
+        Txfifo_empty = (1 << 5),
+        Rxfifo_empty = (1 << 4),
+
+        /* E-FUSE access */
+        EFUSE_WRITE = 0x80000000,
+        EFUSE_WRITE_OK  = 0x00000000,
+        EFUSE_READ  = 0x00000000,
+        EFUSE_READ_OK   = 0x80000000,
+        EFUSE_WRITE_V3 = 0x40000000,
+        EFUSE_WRITE_OK_V3  = 0x00000000,
+        EFUSE_READ_V3  = 0x80000000,
+        EFUSE_READ_OK_V3   = 0x00000000,
+        EFUSE_Reg_Mask  = 0x03FF,
+        EFUSE_Reg_Shift = 8,
+        EFUSE_Check_Cnt = 300,
+        EFUSE_READ_FAIL = 0xFF,
+        EFUSE_Data_Mask = 0x000000FF,
+
+        /* GPIO */
+        GPIO_en = (1 << 0),
+
+        /* PTP */
+        PTP_ISR_TOK = (1 << 1),
+        PTP_ISR_TER = (1 << 2),
+        PTP_EXEC_CMD = (1 << 7),
+        PTP_ADJUST_TIME_NS_NEGATIVE = (1 << 30),
+        PTP_ADJUST_TIME_S_NEGATIVE = (1ULL << 48),
+        PTP_SOFT_CONFIG_TIME_NS_NEGATIVE = (1 << 30),
+        PTP_SOFT_CONFIG_TIME_S_NEGATIVE = (1ULL << 48),
+
+        /* New Interrupt Bits */
+        INT_CFG0_ENABLE_8125 = (1 << 0),
+        INT_CFG0_TIMEOUT0_BYPASS_8125 = (1 << 1),
+        INT_CFG0_MITIGATION_BYPASS_8125 = (1 << 2),
+        INT_CFG0_RDU_BYPASS_8126 = (1 << 4),
+        INT_CFG0_MSIX_ENTRY_NUM_MODE = (1 << 5),
+        INT_CFG0_AUTO_CLEAR_IMR = (1 << 5),
+        INT_CFG0_AVOID_MISS_INTR = (1 << 6),
+        ISRIMR_V2_ROK_Q0     = (1 << 0),
+        ISRIMR_TOK_Q0        = (1 << 16),
+        ISRIMR_TOK_Q1        = (1 << 18),
+        ISRIMR_V2_LINKCHG    = (1 << 21),
+
+        ISRIMR_V4_ROK_Q0     = (1 << 0),
+        ISRIMR_V4_LINKCHG    = (1 << 29),
+
+        ISRIMR_V5_ROK_Q0     = (1 << 0),
+        ISRIMR_V5_TOK_Q0     = (1 << 16),
+        ISRIMR_V5_TOK_Q1     = (1 << 17),
+        ISRIMR_V5_LINKCHG    = (1 << 18),
+
+        ISRIMR_V6_ROK_Q0     = (1 << 0),
+        ISRIMR_V6_TOK_Q0     = (1 << 8),
+        ISRIMR_V6_TOK_Q1     = (1 << 9),
+        ISRIMR_V6_LINKCHG    = (1 << 29),
+        ISRIMR_V6_LAYER2_INTR_STS = (1 << 31),
+        /* MISC interrupt */
+        ISRIMR_V6_L2_MISC_INTR    = (1 << 16),
+        ISRIMR_V6_L3_MISC_IPC2    = (1 << 2),
+
+        /* IPC2 */
+        RISC_IPC2_INTR    = (1 << 1),
+
+        /* Magic Number */
+        RTL8127_MAGIC_NUMBER = 0x0badbadbadbadbadull,
+};
+
+enum RTL8127_msix_id {
+        MSIX_ID_V2_LINKCHG = 21,
+
+        MSIX_ID_V4_LINKCHG = 29,
+
+        MSIX_ID_V5_LINKCHG = 18,
+
+        MSIX_ID_V6_L2_MISC_INTR = 48,
+};
+
+enum _DescStatusBit {
+        DescOwn     = (1 << 31), /* Descriptor is owned by NIC */
+        RingEnd     = (1 << 30), /* End of descriptor ring */
+        FirstFrag   = (1 << 29), /* First segment of a packet */
+        LastFrag    = (1 << 28), /* Final segment of a packet */
+
+        DescOwn_V3     = (DescOwn), /* Descriptor is owned by NIC */
+        RingEnd_V3     = (RingEnd), /* End of descriptor ring */
+        FirstFrag_V3   = (1 << 25), /* First segment of a packet */
+        LastFrag_V3    = (1 << 24), /* Final segment of a packet */
+
+        DescOwn_V4     = (DescOwn), /* Descriptor is owned by NIC */
+        RingEnd_V4     = (RingEnd), /* End of descriptor ring */
+        FirstFrag_V4   = (FirstFrag), /* First segment of a packet */
+        LastFrag_V4    = (LastFrag), /* Final segment of a packet */
+
+        /* Tx private */
+        /*------ offset 0 of tx descriptor ------*/
+        LargeSend   = (1 << 27), /* TCP Large Send Offload (TSO) */
+        GiantSendv4 = (1 << 26), /* TCP Giant Send Offload V4 (GSOv4) */
+        GiantSendv6 = (1 << 25), /* TCP Giant Send Offload V6 (GSOv6) */
+        LargeSend_DP = (1 << 16), /* TCP Large Send Offload (TSO) */
+        MSSShift    = 16,        /* MSS value position */
+        MSSMask     = 0x7FFU,    /* MSS value 11 bits */
+        TxIPCS      = (1 << 18), /* Calculate IP checksum */
+        TxUDPCS     = (1 << 17), /* Calculate UDP/IP checksum */
+        TxTCPCS     = (1 << 16), /* Calculate TCP/IP checksum */
+        TxVlanTag   = (1 << 17), /* Add VLAN tag */
+
+        /*@@@@@@ offset 4 of tx descriptor => bits for RTL8125 only     begin @@@@@@*/
+        TxUDPCS_C   = (1 << 31), /* Calculate UDP/IP checksum */
+        TxTCPCS_C   = (1 << 30), /* Calculate TCP/IP checksum */
+        TxIPCS_C    = (1 << 29), /* Calculate IP checksum */
+        TxIPV6F_C   = (1 << 28), /* Indicate it is an IPv6 packet */
+        /*@@@@@@ offset 4 of tx descriptor => bits for RTL8125 only     end @@@@@@*/
+
+
+        /* Rx private */
+        /*------ offset 0 of rx descriptor ------*/
+        PID1        = (1 << 18), /* Protocol ID bit 1/2 */
+        PID0        = (1 << 17), /* Protocol ID bit 2/2 */
+
+#define RxProtoUDP  (PID1)
+#define RxProtoTCP  (PID0)
+#define RxProtoIP   (PID1 | PID0)
+#define RxProtoMask RxProtoIP
+
+        RxIPF       = (1 << 16), /* IP checksum failed */
+        RxUDPF      = (1 << 15), /* UDP/IP checksum failed */
+        RxTCPF      = (1 << 14), /* TCP/IP checksum failed */
+        RxVlanTag   = (1 << 16), /* VLAN tag available */
+
+        /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only     begin @@@@@@*/
+        RxUDPT      = (1 << 18),
+        RxTCPT      = (1 << 17),
+        /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only     end @@@@@@*/
+
+        /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only     begin @@@@@@*/
+        RxV6F       = (1 << 31),
+        RxV4F       = (1 << 30),
+        /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only     end @@@@@@*/
+
+
+        PID1_v3        = (1 << 29), /* Protocol ID bit 1/2 */
+        PID0_v3        = (1 << 28), /* Protocol ID bit 2/2 */
+
+#define RxProtoUDP_v3  (PID1_v3)
+#define RxProtoTCP_v3  (PID0_v3)
+#define RxProtoIP_v3   (PID1_v3 | PID0_v3)
+#define RxProtoMask_v3 RxProtoIP_v3
+
+        RxIPF_v3       = (1 << 26), /* IP checksum failed */
+        RxUDPF_v3      = (1 << 25), /* UDP/IP checksum failed */
+        RxTCPF_v3      = (1 << 24), /* TCP/IP checksum failed */
+        RxSCTPF_v3     = (1 << 23), /* SCTP  checksum failed */
+        RxVlanTag_v3   = (RxVlanTag), /* VLAN tag available */
+
+        /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only     begin @@@@@@*/
+        RxUDPT_v3      = (1 << 29),
+        RxTCPT_v3      = (1 << 28),
+        RxSCTP_v3      = (1 << 27),
+        /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only     end @@@@@@*/
+
+        /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only     begin @@@@@@*/
+        RxV6F_v3       = (RxV6F),
+        RxV4F_v3       = (RxV4F),
+        /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only     end @@@@@@*/
+
+        RxIPF_v4       = (1 << 17), /* IP checksum failed */
+        RxUDPF_v4      = (1 << 16), /* UDP/IP checksum failed */
+        RxTCPF_v4      = (1 << 15), /* TCP/IP checksum failed */
+        RxSCTPF_v4     = (1 << 19), /* SCTP checksum failed */
+        RxVlanTag_v4   = (RxVlanTag), /* VLAN tag available */
+
+        /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only     begin @@@@@@*/
+        RxUDPT_v4      = (1 << 19),
+        RxTCPT_v4      = (1 << 18),
+        RxSCTP_v4      = (1 << 19),
+        /*@@@@@@ offset 0 of rx descriptor => bits for RTL8125 only     end @@@@@@*/
+
+        /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only     begin @@@@@@*/
+        RxV6F_v4       = (RxV6F),
+        RxV4F_v4       = (RxV4F),
+        /*@@@@@@ offset 4 of rx descriptor => bits for RTL8125 only     end @@@@@@*/
+};
+
+enum features {
+//  RTL_FEATURE_WOL = (1 << 0),
+        RTL_FEATURE_MSI = (1 << 1),
+        RTL_FEATURE_MSIX = (1 << 2),
+};
+
+enum wol_capability {
+        WOL_DISABLED = 0,
+        WOL_ENABLED = 1
+};
+
+enum bits {
+        BIT_0 = (1 << 0),
+        BIT_1 = (1 << 1),
+        BIT_2 = (1 << 2),
+        BIT_3 = (1 << 3),
+        BIT_4 = (1 << 4),
+        BIT_5 = (1 << 5),
+        BIT_6 = (1 << 6),
+        BIT_7 = (1 << 7),
+        BIT_8 = (1 << 8),
+        BIT_9 = (1 << 9),
+        BIT_10 = (1 << 10),
+        BIT_11 = (1 << 11),
+        BIT_12 = (1 << 12),
+        BIT_13 = (1 << 13),
+        BIT_14 = (1 << 14),
+        BIT_15 = (1 << 15),
+        BIT_16 = (1 << 16),
+        BIT_17 = (1 << 17),
+        BIT_18 = (1 << 18),
+        BIT_19 = (1 << 19),
+        BIT_20 = (1 << 20),
+        BIT_21 = (1 << 21),
+        BIT_22 = (1 << 22),
+        BIT_23 = (1 << 23),
+        BIT_24 = (1 << 24),
+        BIT_25 = (1 << 25),
+        BIT_26 = (1 << 26),
+        BIT_27 = (1 << 27),
+        BIT_28 = (1 << 28),
+        BIT_29 = (1 << 29),
+        BIT_30 = (1 << 30),
+        BIT_31 = (1 << 31)
+};
+
+#define RTL8127_CP_NUM 4
+#define RTL8127_MAX_SUPPORT_CP_LEN 110
+
+enum rtl8127_cp_status {
+        rtl8127_cp_normal = 0,
+        rtl8127_cp_short,
+        rtl8127_cp_open,
+        rtl8127_cp_mismatch,
+        rtl8127_cp_unknown
+};
+
+enum efuse {
+        EFUSE_NOT_SUPPORT = 0,
+        EFUSE_SUPPORT_V1,
+        EFUSE_SUPPORT_V2,
+        EFUSE_SUPPORT_V3,
+        EFUSE_SUPPORT_V4,
+};
+#define RsvdMask    0x3fffc000
+#define RsvdMaskV3  0x3fff8000
+#define RsvdMaskV4  RsvdMaskV3
+
+struct TxDesc {
+        u32 opts1;
+        u32 opts2;
+        u64 addr;
+        u32 reserved0;
+        u32 reserved1;
+        u32 reserved2;
+        u32 reserved3;
+};
+
+struct RxDesc {
+        u32 opts1;
+        u32 opts2;
+        u64 addr;
+};
+
+struct RxDescV3 {
+        union {
+                struct {
+                        u32 rsv1;
+                        u32 rsv2;
+                } RxDescDDWord1;
+        };
+
+        union {
+                struct {
+                        u32 RSSResult;
+                        u16 HeaderBufferLen;
+                        u16 HeaderInfo;
+                } RxDescNormalDDWord2;
+
+                struct {
+                        u32 rsv5;
+                        u32 rsv6;
+                } RxDescDDWord2;
+        };
+
+        union {
+                u64   addr;
+
+                struct {
+                        u32 TimeStampLow;
+                        u32 TimeStampHigh;
+                } RxDescTimeStamp;
+
+                struct {
+                        u32 rsv8;
+                        u32 rsv9;
+                } RxDescDDWord3;
+        };
+
+        union {
+                struct {
+                        u32 opts2;
+                        u32 opts1;
+                } RxDescNormalDDWord4;
+
+                struct {
+                        u16 TimeStampHHigh;
+                        u16 rsv11;
+                        u32 opts1;
+                } RxDescPTPDDWord4;
+        };
+};
+
+struct RxDescV4 {
+        union {
+                u64   addr;
+
+                struct {
+                        u32 RSSInfo;
+                        u32 RSSResult;
+                } RxDescNormalDDWord1;
+        };
+
+        struct {
+                u32 opts2;
+                u32 opts1;
+        } RxDescNormalDDWord2;
+};
+
+enum rxdesc_type {
+        RXDESC_TYPE_NORMAL=0,
+        RXDESC_TYPE_NEXT,
+        RXDESC_TYPE_PTP,
+        RXDESC_TYPE_MAX
+};
+
+//Rx Desc Type
+enum rx_desc_ring_type {
+        RX_DESC_RING_TYPE_UNKNOWN=0,
+        RX_DESC_RING_TYPE_1,
+        RX_DESC_RING_TYPE_2,
+        RX_DESC_RING_TYPE_3,
+        RX_DESC_RING_TYPE_4,
+        RX_DESC_RING_TYPE_MAX
+};
+
+enum rx_desc_len {
+        RX_DESC_LEN_TYPE_1 = (sizeof(struct RxDesc)),
+        RX_DESC_LEN_TYPE_3 = (sizeof(struct RxDescV3)),
+        RX_DESC_LEN_TYPE_4 = (sizeof(struct RxDescV4))
+};
+
+struct ring_info {
+        struct sk_buff  *skb;
+        u32     len;
+        unsigned int   bytecount;
+        unsigned short gso_segs;
+        u8      __pad[sizeof(void *) - sizeof(u32)];
+};
+
+struct pci_resource {
+        u8  cmd;
+        u8  cls;
+        u16 io_base_h;
+        u16 io_base_l;
+        u16 mem_base_h;
+        u16 mem_base_l;
+        u8  ilr;
+        u16 resv_0x1c_h;
+        u16 resv_0x1c_l;
+        u16 resv_0x20_h;
+        u16 resv_0x20_l;
+        u16 resv_0x24_h;
+        u16 resv_0x24_l;
+        u16 resv_0x2c_h;
+        u16 resv_0x2c_l;
+        u32 pci_sn_l;
+        u32 pci_sn_h;
+};
+
+enum r8127_dash_req_flag {
+        R8127_RCV_REQ_SYS_OK = 0,
+        R8127_RCV_REQ_DASH_OK,
+        R8127_SEND_REQ_HOST_OK,
+        R8127_CMAC_RESET,
+        R8127_CMAC_DISALE_RX_FLAG_MAX,
+        R8127_DASH_REQ_FLAG_MAX
+};
+
+enum r8127_flag {
+        R8127_FLAG_DOWN = 0,
+        R8127_FLAG_TASK_RESET_PENDING,
+        R8127_FLAG_TASK_ESD_CHECK_PENDING,
+        R8127_FLAG_TASK_LINKCHG_CHECK_PENDING,
+        R8127_FLAG_TASK_DASH_CHECK_PENDING,
+        R8127_FLAG_MAX
+};
+
+enum r8127_sysfs_flag {
+        R8127_SYSFS_RTL_ADV = 0,
+        R8127_SYSFS_FLAG_MAX
+};
+
+struct rtl8127_tx_ring {
+        void* priv;
+        struct net_device *netdev;
+        u32 index;
+        u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
+        u32 dirty_tx;
+        u32 num_tx_desc; /* Number of Tx descriptor registers */
+        struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */
+        dma_addr_t TxPhyAddr;
+        u32 TxDescAllocSize;
+        struct ring_info tx_skb[MAX_NUM_TX_DESC]; /* Tx data buffers */
+
+        u32 NextHwDesCloPtr;
+        u32 BeginHwDesCloPtr;
+
+        u16 hw_clo_ptr_reg;
+        u16 sw_tail_ptr_reg;
+
+        u16 tdsar_reg; /* Transmit Descriptor Start Address */
+};
+
+struct rtl8127_rx_buffer {
+        struct page *page;
+        u32 page_offset;
+        dma_addr_t dma;
+        void* data;
+        struct sk_buff *skb;
+};
+
+struct rtl8127_rx_ring {
+        void* priv;
+        struct net_device *netdev;
+        u32 index;
+        u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
+        u32 dirty_rx;
+        u32 num_rx_desc; /* Number of Rx descriptor registers */
+        struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */
+        u32 RxDescAllocSize;
+        u64 RxDescPhyAddr[MAX_NUM_RX_DESC]; /* Rx desc physical address*/
+        dma_addr_t RxPhyAddr;
+#ifdef ENABLE_PAGE_REUSE
+        struct rtl8127_rx_buffer rx_buffer[MAX_NUM_RX_DESC];
+        u16 rx_offset;
+#else
+        struct sk_buff *Rx_skbuff[MAX_NUM_RX_DESC]; /* Rx data buffers */
+#endif //ENABLE_PAGE_REUSE
+
+        u16 rdsar_reg; /* Receive Descriptor Start Address */
+};
+
+struct r8127_napi {
+#ifdef CONFIG_R8127_NAPI
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24)
+        struct napi_struct napi;
+#endif
+#endif
+        void* priv;
+        int index;
+};
+
+struct r8127_irq {
+        irq_handler_t	handler;
+        unsigned int	vector;
+        u8		requested;
+        char		name[IFNAMSIZ + 10];
+};
+
+#pragma pack(1)
+struct rtl8127_regs {
+        //00
+        u8 mac_id[6];
+        u16 reg_06;
+        u8 mar[8];
+        //10
+        u64 dtccr;
+        u16 ledsel0;
+        u16 legreg;
+        u32 tctr3;
+        //20
+        u32 txq0_dsc_st_addr_0;
+        u32 txq0_dsc_st_addr_2;
+        u64 reg_28;
+        //30
+        u16 rit;
+        u16 ritc;
+        u16 reg_34;
+        u8 reg_36;
+        u8 command;
+        u32 imr0;
+        u32 isr0;
+        //40
+        u32 tcr;
+        u32 rcr;
+        u32 tctr0;
+        u32 tctr1;
+        //50
+        u8 cr93c46;
+        u8 config0;
+        u8 config1;
+        u8 config2;
+        u8 config3;
+        u8 config4;
+        u8 config5;
+        u8 tdfnr;
+        u32 timer_int0;
+        u32 timer_int1;
+        //60
+        u32 gphy_mdcmdio;
+        u32 csidr;
+        u32 csiar;
+        u16 phy_status;
+        u8 config6;
+        u8 pmch;
+        //70
+        u32 eridr;
+        u32 eriar;
+        u16 config7;
+        u16 reg_7a;
+        u32 ephy_rxerr_cnt;
+        //80
+        u32 ephy_mdcmdio;
+        u16 ledsel2;
+        u16 ledsel1;
+        u32 tctr2;
+        u32 timer_int2;
+        //90
+        u8 tppoll0;
+        u8 reg_91;
+        u16 reg_92;
+        u16 led_feature;
+        u16 ledsel3;
+        u16 eee_led_config;
+        u16 reg_9a;
+        u32 reg_9c;
+        //a0
+        u32 reg_a0;
+        u32 reg_a4;
+        u32 reg_a8;
+        u32 reg_ac;
+        //b0
+        u32 patch_dbg;
+        u32 reg_b4;
+        u32 gphy_ocp;
+        u32 reg_bc;
+        //c0
+        u32 reg_c0;
+        u32 reg_c4;
+        u32 reg_c8;
+        u16 otp_cmd;
+        u16 otp_pg_config;
+        //d0
+        u16 phy_pwr;
+        u8 twsi_ctrl;
+        u8 oob_ctrl;
+        u16 mac_dbgo;
+        u16 mac_dbg;
+        u16 reg_d8;
+        u16 rms;
+        u32 efuse_data;
+        //e0
+        u16 cplus_cmd;
+        u16 reg_e2;
+        u32 rxq0_dsc_st_addr_0;
+        u32 rxq0_dsc_st_addr_2;
+        u16 reg_ec;
+        u16 tx10midle_cnt;
+        //f0
+        u16 misc0;
+        u16 misc1;
+        u32 timer_int3;
+        u32 cmac_ib;
+        u16 reg_fc;
+        u16 sw_rst;
+};
+#pragma pack()
+
+struct rtl8127_regs_save {
+        union {
+                u8 mac_io[R8127_MAC_REGS_SIZE];
+
+                struct rtl8127_regs mac_reg;
+        };
+        u16 pcie_phy[R8127_EPHY_REGS_SIZE/2];
+        u16 eth_phy[R8127_PHY_REGS_SIZE/2];
+        u32 eri_reg[R8127_ERI_REGS_SIZE/4];
+        u32 pci_reg[R8127_PCI_REGS_SIZE/4];
+        u16 sw_tail_ptr_reg[R8127_MAX_TX_QUEUES];
+        u16 hw_clo_ptr_reg[R8127_MAX_TX_QUEUES];
+
+        //ktime_t begin_ktime;
+        //ktime_t end_ktime;
+        //u64 duration_ns;
+
+        u16 sw0_tail_ptr;
+        u16 next_hwq0_clo_ptr;
+        u16 sw1_tail_ptr;
+        u16 next_hwq1_clo_ptr;
+
+        u16 int_miti_rxq0;
+        u16 int_miti_txq0;
+        u16 int_miti_rxq1;
+        u16 int_miti_txq1;
+        u8 int_config;
+        u32 imr_new;
+        u32 isr_new;
+
+        u8 tdu_status;
+        u16 rdu_status;
+
+        u16 tc_mode;
+
+        u32 txq1_dsc_st_addr_0;
+        u32 txq1_dsc_st_addr_2;
+
+        u32 pla_tx_q0_idle_credit;
+        u32 pla_tx_q1_idle_credit;
+
+        u32 rxq1_dsc_st_addr_0;
+        u32 rxq1_dsc_st_addr_2;
+
+        u32 rss_ctrl;
+        u8 rss_key[RTL8127_RSS_KEY_SIZE];
+        u8 rss_i_table[RTL8127_MAX_INDIRECTION_TABLE_ENTRIES];
+        u16 rss_queue_num_sel_r;
+};
+
+struct rtl8127_counters {
+        /* legacy */
+        u64 tx_packets;
+        u64 rx_packets;
+        u64 tx_errors;
+        u32 rx_errors;
+        u16 rx_missed;
+        u16 align_errors;
+        u32 tx_one_collision;
+        u32 tx_multi_collision;
+        u64 rx_unicast;
+        u64 rx_broadcast;
+        u32 rx_multicast;
+        u16 tx_aborted;
+        u16 tx_underrun;
+
+        /* extended */
+        u64 tx_octets;
+        u64 rx_octets;
+        u64 rx_multicast64;
+        u64 tx_unicast64;
+        u64 tx_broadcast64;
+        u64 tx_multicast64;
+        u32 tx_pause_on;
+        u32 tx_pause_off;
+        u32 tx_pause_all;
+        u32 tx_deferred;
+        u32 tx_late_collision;
+        u32 tx_all_collision;
+        u32 tx_aborted32;
+        u32 align_errors32;
+        u32 rx_frame_too_long;
+        u32 rx_runt;
+        u32 rx_pause_on;
+        u32 rx_pause_off;
+        u32 rx_pause_all;
+        u32 rx_unknown_opcode;
+        u32 rx_mac_error;
+        u32 tx_underrun32;
+        u32 rx_mac_missed;
+        u32 rx_tcam_dropped;
+        u32 tdu;
+        u32 rdu;
+};
+
+/* Flow Control Settings */
+enum rtl8127_fc_mode {
+        rtl8127_fc_none = 0,
+        rtl8127_fc_rx_pause,
+        rtl8127_fc_tx_pause,
+        rtl8127_fc_full,
+        rtl8127_fc_default
+};
+
+enum rtl8127_state_t {
+        __RTL8127_TESTING = 0,
+        __RTL8127_RESETTING,
+        __RTL8127_DOWN,
+        __RTL8127_PTP_TX_IN_PROGRESS,
+};
+
+#define RTL_FLAG_RX_HWTSTAMP_ENABLED BIT_0
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0)
+struct ethtool_eee {
+        __u32	cmd;
+        __u32	supported;
+        __u32	advertised;
+        __u32	lp_advertised;
+        __u32	eee_active;
+        __u32	eee_enabled;
+        __u32	tx_lpi_enabled;
+        __u32	tx_lpi_timer;
+        __u32	reserved[2];
+};
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0) */
+
+struct rtl8127_private {
+        void __iomem *mmio_addr;    /* memory map physical address */
+        struct pci_dev *pci_dev;    /* Index of PCI device */
+        struct net_device *dev;
+        struct r8127_napi r8127napi[R8127_MAX_MSIX_VEC];
+        struct r8127_irq irq_tbl[R8127_MAX_MSIX_VEC];
+        unsigned int irq_nvecs;
+        unsigned int max_irq_nvecs;
+        unsigned int min_irq_nvecs;
+        unsigned int hw_supp_irq_nvecs;
+        //struct msix_entry msix_entries[R8127_MAX_MSIX_VEC];
+        struct net_device_stats stats;  /* statistics of net device */
+        unsigned long state;
+        u32 flags;
+
+        u32 msg_enable;
+        u32 tx_tcp_csum_cmd;
+        u32 tx_udp_csum_cmd;
+        u32 tx_ip_csum_cmd;
+        u32 tx_ipv6_csum_cmd;
+        int max_jumbo_frame_size;
+        int chipset;
+        u32 mcfg;
+        //u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
+        //u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
+        //u32 dirty_rx;
+        //u32 dirty_tx;
+        //struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */
+        //struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */
+        //dma_addr_t TxPhyAddr;
+        //dma_addr_t RxPhyAddr;
+        //struct sk_buff *Rx_skbuff[MAX_NUM_RX_DESC]; /* Rx data buffers */
+        //struct ring_info tx_skb[MAX_NUM_TX_DESC];   /* Tx data buffers */
+        unsigned rx_buf_sz;
+#ifdef ENABLE_PAGE_REUSE
+        unsigned rx_buf_page_order;
+        unsigned rx_buf_page_size;
+        u32 page_reuse_fail_cnt;
+#endif //ENABLE_PAGE_REUSE
+        u16 HwSuppNumTxQueues;
+        u16 HwSuppNumRxQueues;
+        unsigned int num_tx_rings;
+        unsigned int num_rx_rings;
+        struct rtl8127_tx_ring tx_ring[R8127_MAX_TX_QUEUES];
+        struct rtl8127_rx_ring rx_ring[R8127_MAX_RX_QUEUES];
+#ifdef ENABLE_LIB_SUPPORT
+        struct blocking_notifier_head lib_nh;
+        struct rtl8127_ring lib_tx_ring[R8127_MAX_TX_QUEUES];
+        struct rtl8127_ring lib_rx_ring[R8127_MAX_RX_QUEUES];
+#endif
+        //struct timer_list esd_timer;
+        //struct timer_list link_timer;
+        struct pci_resource pci_cfg_space;
+        unsigned int esd_flag;
+        unsigned int pci_cfg_is_read;
+        unsigned int rtl8127_rx_config;
+        u16 rms;
+        u16 cp_cmd;
+        u32 intr_mask;
+        u32 intr_l2_mask;
+        u32 timer_intr_mask;
+        u16 isr_reg[R8127_MAX_MSIX_VEC];
+        u16 imr_reg[R8127_MAX_MSIX_VEC];
+        int phy_auto_nego_reg;
+        int phy_1000_ctrl_reg;
+        int phy_2500_ctrl_reg;
+        u8 org_mac_addr[NODE_ADDRESS_SIZE];
+        struct rtl8127_counters *tally_vaddr;
+        dma_addr_t tally_paddr;
+
+#ifdef CONFIG_R8127_VLAN
+        struct vlan_group *vlgrp;
+#endif
+        u8  wol_enabled;
+        u32 wol_opts;
+        u8  efuse_ver;
+        u8  eeprom_type;
+        u8  autoneg;
+        u8  duplex;
+        u32 speed;
+        u64 advertising;
+        enum rtl8127_fc_mode fcpause;
+        u32 HwSuppMaxPhyLinkSpeed;
+        u16 eeprom_len;
+        u16 cur_page;
+        u32 bios_setting;
+
+        int (*set_speed)(struct net_device *, u8 autoneg, u32 speed, u8 duplex, u64 adv);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+        void (*get_settings)(struct net_device *, struct ethtool_cmd *);
+#else
+        void (*get_settings)(struct net_device *, struct ethtool_link_ksettings *);
+#endif
+        void (*phy_reset_enable)(struct net_device *);
+        unsigned int (*phy_reset_pending)(struct net_device *);
+        unsigned int (*link_ok)(struct net_device *);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+        struct work_struct reset_task;
+        struct work_struct esd_task;
+        struct work_struct linkchg_task;
+        struct work_struct dash_task;
+#else
+        struct delayed_work reset_task;
+        struct delayed_work esd_task;
+        struct delayed_work linkchg_task;
+        struct delayed_work dash_task;
+#endif
+        DECLARE_BITMAP(task_flags, R8127_FLAG_MAX);
+        unsigned features;
+
+        u8 org_pci_offset_99;
+        u8 org_pci_offset_180;
+        u8 issue_offset_99_event;
+
+        u8 org_pci_offset_80;
+        u8 org_pci_offset_81;
+        u8 use_timer_interrupt;
+
+        u32 keep_intr_cnt;
+
+        u8  HwIcVerUnknown;
+        u8  NotWrRamCodeToMicroP;
+        u8  NotWrMcuPatchCode;
+        u8  HwHasWrRamCodeToMicroP;
+
+        u16 sw_ram_code_ver;
+        u16 hw_ram_code_ver;
+
+        u8 rtk_enable_diag;
+
+        u8 ShortPacketSwChecksum;
+
+        u8 UseSwPaddingShortPkt;
+
+        u8 RequireAdcBiasPatch;
+        u16 AdcBiasPatchIoffset;
+
+        u8 RequireAdjustUpsTxLinkPulseTiming;
+        u16 SwrCnt1msIni;
+
+        u8 HwSuppNowIsOobVer;
+
+        u8 RequiredSecLanDonglePatch;
+
+        u8 RequirePhyMdiSwapPatch;
+
+        u32 HwFiberModeVer;
+        u32 HwFiberStat;
+        u8 HwSwitchMdiToFiber;
+
+        u16 NicCustLedValue;
+
+        u8 HwSuppMagicPktVer;
+
+        u8 HwSuppLinkChgWakeUpVer;
+
+        u8 HwSuppCheckPhyDisableModeVer;
+
+        u8 random_mac;
+
+        u16 phy_reg_aner;
+        u16 phy_reg_anlpar;
+        u16 phy_reg_gbsr;
+        u16 phy_reg_status_2500;
+
+        u32 HwPcieSNOffset;
+
+        u32 MaxTxDescPtrMask;
+        u8 HwSuppTxNoCloseVer;
+        u8 EnableTxNoClose;
+
+        u8 HwSuppIsrVer;
+        u8 HwCurrIsrVer;
+
+        u8 HwSuppIntMitiVer;
+
+        u8 HwSuppExtendTallyCounterVer;
+
+        u8 check_keep_link_speed;
+        u8 resume_not_chg_speed;
+
+        u8 HwSuppD0SpeedUpVer;
+        u8 D0SpeedUpSpeed;
+
+        u8 ring_lib_enabled;
+
+        const char *fw_name;
+        struct rtl8127_fw *rtl_fw;
+        u32 ocp_base;
+
+        //Dash+++++++++++++++++
+        u8 HwSuppDashVer;
+        u8 DASH;
+        u8 HwPkgDet;
+        u8 HwSuppOcpChannelVer;
+        u32 DashFirmwareVersion;
+        u32 SizeOfSendToFwBuffer;
+        u32 SizeOfRecvFromFwBuffer;
+        u8 AllowAccessDashOcp;
+        DECLARE_BITMAP(dash_req_flags, R8127_DASH_REQ_FLAG_MAX);
+
+#ifdef ENABLE_DASH_SUPPORT
+        u16 AfterRecvFromFwBufLen;
+        u8 AfterRecvFromFwBuf[RECV_FROM_FW_BUF_SIZE];
+        u32 RecvFromFwBufErrCnt;
+        u16 AfterSendToFwBufLen;
+        u8 AfterSendToFwBuf[SEND_TO_FW_BUF_SIZE];
+        u16 SendToFwBufferLen;
+
+        u8 OobReq;
+        u8 OobAck;
+        u32 OobReqComplete;
+        u32 OobAckComplete;
+
+        u8 RcvFwReqSysOkEvt;
+        u8 RcvFwDashOkEvt;
+        u8 SendFwHostOkEvt;
+
+        u8 DashFwDisableRx;
+
+        u8 SendingToFw;
+
+        u32 RecvFromDashFwCnt;
+
+        u8 DashReqRegValue;
+        u16 HostReqValue;
+
+        //Dash-----------------
+#endif //ENABLE_DASH_SUPPORT
+
+        //Realwow++++++++++++++
+        u8 HwSuppKCPOffloadVer;
+
+        u8 EnableDhcpTimeoutWake;
+        u8 EnableTeredoOffload;
+        u8 EnableKCPOffload;
+#ifdef ENABLE_REALWOW_SUPPORT
+        u32 DhcpTimeout;
+        MP_KCP_INFO MpKCPInfo;
+        //Realwow--------------
+#endif //ENABLE_REALWOW_SUPPORT
+
+        struct ethtool_keee eee;
+
+#ifdef ENABLE_R8127_PROCFS
+        //Procfs support
+        struct proc_dir_entry *proc_dir;
+        struct proc_dir_entry *proc_dir_debug;
+        struct proc_dir_entry *proc_dir_test;
+#endif
+#ifdef ENABLE_R8127_SYSFS
+        //sysfs support
+        DECLARE_BITMAP(sysfs_flag, R8127_SYSFS_FLAG_MAX);
+        u32 testmode;
+#endif
+        u8 HwSuppRxDescType;
+        u8 InitRxDescType;
+        u16 RxDescLength; //V1 16 Byte V2 32 Bytes
+
+        spinlock_t phy_lock;
+
+        u8 HwSuppPtpVer;
+        u8 EnablePtp;
+#ifdef ENABLE_PTP_SUPPORT
+        u32 tx_hwtstamp_timeouts;
+        u32 tx_hwtstamp_skipped;
+        struct work_struct ptp_tx_work;
+        struct sk_buff *ptp_tx_skb;
+        struct hwtstamp_config hwtstamp_config;
+        unsigned long ptp_tx_start;
+        struct ptp_clock_info ptp_clock_info;
+        struct ptp_clock *ptp_clock;
+        u8 syncE_en;
+        u8 pps_enable;
+        struct hrtimer pps_timer;
+#endif
+
+        u8 HwSuppRssVer;
+        u8 EnableRss;
+        u16 HwSuppIndirTblEntries;
+#ifdef ENABLE_RSS_SUPPORT
+        u32 rss_flags;
+        /* Receive Side Scaling settings */
+        u8 rss_key[RTL8127_RSS_KEY_SIZE];
+        u8 rss_indir_tbl[RTL8127_MAX_INDIRECTION_TABLE_ENTRIES];
+        u32 rss_options;
+#endif
+
+        u8 HwSuppMacMcuVer;
+        u16 MacMcuPageSize;
+        u64 hw_mcu_patch_code_ver;
+        u64 bin_mcu_patch_code_ver;
+
+        u8 HwSuppTcamVer;
+
+        u16 TcamNotValidReg;
+        u16 TcamValidReg;
+        u16 TcamMaAddrcOffset;
+        u16 TcamVlanTagOffset;
+};
+
+#ifdef ENABLE_LIB_SUPPORT
+static inline unsigned int
+rtl8127_num_lib_tx_rings(struct rtl8127_private *tp)
+{
+        int count, i;
+
+        for (count = 0, i = tp->num_tx_rings; i < tp->HwSuppNumTxQueues; i++)
+                if(tp->lib_tx_ring[i].enabled)
+                        count++;
+
+        return count;
+}
+
+static inline unsigned int
+rtl8127_num_lib_rx_rings(struct rtl8127_private *tp)
+{
+        int count, i;
+
+        for (count = 0, i = tp->num_rx_rings; i < tp->HwSuppNumRxQueues; i++)
+                if(tp->lib_rx_ring[i].enabled)
+                        count++;
+
+        return count;
+}
+
+#else
+static inline unsigned int
+rtl8127_num_lib_tx_rings(struct rtl8127_private *tp)
+{
+        return 0;
+}
+
+static inline unsigned int
+rtl8127_num_lib_rx_rings(struct rtl8127_private *tp)
+{
+        return 0;
+}
+#endif
+
+static inline unsigned int
+rtl8127_tot_tx_rings(struct rtl8127_private *tp)
+{
+        return tp->num_tx_rings + rtl8127_num_lib_tx_rings(tp);
+}
+
+static inline unsigned int
+rtl8127_tot_rx_rings(struct rtl8127_private *tp)
+{
+        return tp->num_rx_rings + rtl8127_num_lib_rx_rings(tp);
+}
+
+static inline struct netdev_queue *txring_txq(const struct rtl8127_tx_ring *ring)
+{
+        return netdev_get_tx_queue(ring->netdev, ring->index);
+}
+
+enum eetype {
+        EEPROM_TYPE_NONE=0,
+        EEPROM_TYPE_93C46,
+        EEPROM_TYPE_93C56,
+        EEPROM_TWSI
+};
+
+enum mcfg {
+        CFG_METHOD_1=1,
+        CFG_METHOD_2,
+        CFG_METHOD_DEFAULT,
+        CFG_METHOD_MAX
+};
+
+#define LSO_32K 32000
+#define LSO_64K 64000
+
+#define NIC_MIN_PHYS_BUF_COUNT          (2)
+#define NIC_MAX_PHYS_BUF_COUNT_LSO_64K  (24)
+#define NIC_MAX_PHYS_BUF_COUNT_LSO2     (16*4)
+
+#define GTTCPHO_SHIFT                   18
+#define GTTCPHO_MAX                     0x70U
+#define GTPKTSIZE_MAX                   0x3ffffU
+#define TCPHO_SHIFT                     18
+#define TCPHO_MAX                       0x3ffU
+#define LSOPKTSIZE_MAX                  0xffffU
+#define MSS_MAX                         0x07ffu /* MSS value */
+
+#define OOB_CMD_RESET       0x00
+#define OOB_CMD_DRIVER_START    0x05
+#define OOB_CMD_DRIVER_STOP 0x06
+#define OOB_CMD_SET_IPMAC   0x41
+
+#define WAKEUP_MAGIC_PACKET_NOT_SUPPORT (0)
+#define WAKEUP_MAGIC_PACKET_V1 (1)
+#define WAKEUP_MAGIC_PACKET_V2 (2)
+#define WAKEUP_MAGIC_PACKET_V3 (3)
+
+//Ram Code Version
+#define NIC_RAMCODE_VERSION_CFG_METHOD_1 (0x0015)
+#define NIC_RAMCODE_VERSION_CFG_METHOD_2 (0x0036)
+
+//hwoptimize
+#define HW_PATCH_SOC_LAN (BIT_0)
+#define HW_PATCH_SAMSUNG_LAN_DONGLE (BIT_2)
+
+static const u16 other_q_intr_mask = (RxOK1 | RxDU1);
+
+void rtl8127_mdio_write(struct rtl8127_private *tp, u16 RegAddr, u16 value);
+void rtl8127_mdio_prot_write(struct rtl8127_private *tp, u32 RegAddr, u32 value);
+void rtl8127_mdio_prot_direct_write_phy_ocp(struct rtl8127_private *tp, u32 RegAddr, u32 value);
+u32 rtl8127_mdio_read(struct rtl8127_private *tp, u16 RegAddr);
+u32 rtl8127_mdio_prot_read(struct rtl8127_private *tp, u32 RegAddr);
+u32 rtl8127_mdio_prot_direct_read_phy_ocp(struct rtl8127_private *tp, u32 RegAddr);
+void rtl8127_ephy_write(struct rtl8127_private *tp, int RegAddr, int value);
+void rtl8127_mac_ocp_write(struct rtl8127_private *tp, u16 reg_addr, u16 value);
+u16 rtl8127_mac_ocp_read(struct rtl8127_private *tp, u16 reg_addr);
+void rtl8127_clear_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask);
+void rtl8127_set_eth_phy_bit(struct rtl8127_private *tp,  u8  addr, u16  mask);
+void rtl8127_ocp_write(struct rtl8127_private *tp, u16 addr, u8 len, u32 data);
+void rtl8127_init_ring_indexes(struct rtl8127_private *tp);
+void rtl8127_oob_mutex_lock(struct rtl8127_private *tp);
+u32 rtl8127_ocp_read(struct rtl8127_private *tp, u16 addr, u8 len);
+u32 rtl8127_ocp_read_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 base_address);
+u32 rtl8127_ocp_write_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 value, u32 base_address);
+u32 rtl8127_eri_read(struct rtl8127_private *tp, int addr, int len, int type);
+u32 rtl8127_eri_read_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, int type, u32 base_address);
+int rtl8127_eri_write(struct rtl8127_private *tp, int addr, int len, u32 value, int type);
+int rtl8127_eri_write_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, u32 value, int type, u32 base_address);
+u16 rtl8127_ephy_read(struct rtl8127_private *tp, int RegAddr);
+void rtl8127_wait_txrx_fifo_empty(struct net_device *dev);
+void rtl8127_enable_now_is_oob(struct rtl8127_private *tp);
+void rtl8127_disable_now_is_oob(struct rtl8127_private *tp);
+void rtl8127_oob_mutex_unlock(struct rtl8127_private *tp);
+void rtl8127_dash2_disable_tx(struct rtl8127_private *tp);
+void rtl8127_dash2_enable_tx(struct rtl8127_private *tp);
+void rtl8127_dash2_disable_rx(struct rtl8127_private *tp);
+void rtl8127_dash2_enable_rx(struct rtl8127_private *tp);
+void rtl8127_hw_disable_mac_mcu_bps(struct net_device *dev);
+void rtl8127_mark_to_asic(struct rtl8127_private *tp, struct RxDesc *desc, u32 rx_buf_sz);
+void rtl8127_mark_as_last_descriptor(struct rtl8127_private *tp, struct RxDesc *desc);
+
+static inline void
+rtl8127_make_unusable_by_asic(struct rtl8127_private *tp,
+                              struct RxDesc *desc)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                ((struct RxDescV3 *)desc)->addr = RTL8127_MAGIC_NUMBER;
+                ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts1 &= ~cpu_to_le32(DescOwn | RsvdMaskV3);
+                break;
+        case RX_DESC_RING_TYPE_4:
+                ((struct RxDescV4 *)desc)->addr = RTL8127_MAGIC_NUMBER;
+                ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts1 &= ~cpu_to_le32(DescOwn | RsvdMaskV4);
+                break;
+        default:
+                desc->addr = RTL8127_MAGIC_NUMBER;
+                desc->opts1 &= ~cpu_to_le32(DescOwn | RsvdMask);
+                break;
+        }
+}
+
+static inline struct RxDesc*
+rtl8127_get_rxdesc(struct rtl8127_private *tp, struct RxDesc *RxDescBase, u32 const cur_rx)
+{
+        return (struct RxDesc*)((u8*)RxDescBase + (cur_rx * tp->RxDescLength));
+}
+
+static inline void
+rtl8127_disable_hw_interrupt_v2(struct rtl8127_private *tp,
+                                u32 message_id)
+{
+        if (message_id < 32)
+                RTL_W32(tp, IMR_V2_CLEAR_REG_8125, BIT(message_id));
+        else
+                return;
+}
+
+static inline void
+rtl8127_enable_hw_interrupt_v2(struct rtl8127_private *tp, u32 message_id)
+{
+        if (message_id < 32)
+                RTL_W32(tp, IMR_V2_SET_REG_8125, BIT(message_id));
+        else
+                return;
+}
+
+static inline void
+rtl8127_disable_hw_l2_interrupt_v4(struct rtl8127_private *tp,
+                                   u32 message_id)
+{
+        if (message_id < 32)
+                return;
+        else
+                RTL_W32(tp, IMR_V4_L2_CLEAR_REG_8125, BIT(message_id - 32));
+}
+
+static inline void
+rtl8127_enable_hw_l2_interrupt_v4(struct rtl8127_private *tp, u32 message_id)
+{
+        if (message_id < 32)
+                return;
+        else
+                RTL_W32(tp, IMR_V4_L2_SET_REG_8125, BIT(message_id - 32));
+}
+
+static inline void
+rtl8127_disable_hw_layered_interrupt(struct rtl8127_private *tp,
+                                     u32 message_id)
+{
+        if (message_id < 32)
+                rtl8127_disable_hw_interrupt_v2(tp, message_id);
+        else
+                rtl8127_disable_hw_l2_interrupt_v4(tp, message_id);
+}
+
+static inline void
+rtl8127_enable_hw_layered_interrupt(struct rtl8127_private *tp,
+                                    u32 message_id)
+{
+        if (message_id < 32)
+                rtl8127_enable_hw_interrupt_v2(tp, message_id);
+        else
+                rtl8127_enable_hw_l2_interrupt_v4(tp, message_id);
+}
+
+int rtl8127_open(struct net_device *dev);
+int rtl8127_close(struct net_device *dev);
+void rtl8127_hw_config(struct net_device *dev);
+void rtl8127_hw_set_timer_int(struct rtl8127_private *tp, u32 message_id, u8 timer_intmiti_val);
+void rtl8127_set_rx_q_num(struct rtl8127_private *tp, unsigned int num_rx_queues);
+void rtl8127_set_tx_q_num(struct rtl8127_private *tp, unsigned int num_tx_queues);
+void rtl8127_enable_mcu(struct rtl8127_private *tp, bool enable);
+void rtl8127_hw_start(struct net_device *dev);
+void rtl8127_hw_reset(struct net_device *dev);
+void rtl8127_tx_clear(struct rtl8127_private *tp);
+void rtl8127_rx_clear(struct rtl8127_private *tp);
+int rtl8127_init_ring(struct net_device *dev);
+void rtl8127_hw_set_rx_packet_filter(struct net_device *dev);
+void rtl8127_enable_hw_linkchg_interrupt(struct rtl8127_private *tp);
+int rtl8127_dump_tally_counter(struct rtl8127_private *tp, dma_addr_t paddr);
+void rtl8127_enable_napi(struct rtl8127_private *tp);
+void _rtl8127_wait_for_quiescence(struct net_device *dev);
+
+void rtl8127_mdio_direct_write_phy_ocp(struct rtl8127_private *tp, u16 RegAddr,u16 value);
+u32 rtl8127_mdio_direct_read_phy_ocp(struct rtl8127_private *tp, u16 RegAddr);
+void rtl8127_clear_and_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 clearmask, u16 setmask);
+void rtl8127_clear_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask);
+void rtl8127_set_eth_phy_ocp_bit(struct rtl8127_private *tp,  u16 addr, u16 mask);
+
+void rtl8127_clear_mac_ocp_bit(struct rtl8127_private *tp, u16   addr, u16   mask);
+
+#ifndef ENABLE_LIB_SUPPORT
+static inline void rtl8127_lib_reset_prepare(struct rtl8127_private *tp) { }
+static inline void rtl8127_lib_reset_complete(struct rtl8127_private *tp) { }
+#endif
+
+#define HW_SUPPORT_CHECK_PHY_DISABLE_MODE(_M)        ((_M)->HwSuppCheckPhyDisableModeVer > 0)
+#define HW_HAS_WRITE_PHY_MCU_RAM_CODE(_M)        (((_M)->HwHasWrRamCodeToMicroP == TRUE) ? 1 : 0)
+#define HW_SUPPORT_D0_SPEED_UP(_M)        ((_M)->HwSuppD0SpeedUpVer > 0)
+#define HW_SUPPORT_MAC_MCU(_M)        ((_M)->HwSuppMacMcuVer > 0)
+#define HW_SUPPORT_TCAM(_M)        ((_M)->HwSuppTcamVer > 0)
+
+#define HW_SUPP_PHY_LINK_SPEED_GIGA(_M)        ((_M)->HwSuppMaxPhyLinkSpeed >= 1000)
+#define HW_SUPP_PHY_LINK_SPEED_2500M(_M)        ((_M)->HwSuppMaxPhyLinkSpeed >= 2500)
+#define HW_SUPP_PHY_LINK_SPEED_5000M(_M)        ((_M)->HwSuppMaxPhyLinkSpeed >= 5000)
+#define HW_SUPP_PHY_LINK_SPEED_10000M(_M)        ((_M)->HwSuppMaxPhyLinkSpeed >= 10000)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34)
+#define netdev_mc_count(dev) ((dev)->mc_count)
+#define netdev_mc_empty(dev) (netdev_mc_count(dev) == 0)
+#define netdev_for_each_mc_addr(mclist, dev) \
+    for (mclist = dev->mc_list; mclist; mclist = mclist->next)
+#endif
+
+#endif /* __R8127_H */
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_dash.h b/drivers/net/ethernet/realtek/r8127/r8127_dash.h
new file mode 100644
index 000000000000..ebaccfbfd341
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_dash.h
@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_R8127_DASH_H
+#define _LINUX_R8127_DASH_H
+
+#include <linux/if.h>
+
+#define SIOCDEVPRIVATE_RTLDASH   SIOCDEVPRIVATE+2
+
+enum rtl_dash_cmd {
+        RTL_DASH_ARP_NS_OFFLOAD = 0,
+        RTL_DASH_SET_OOB_IPMAC,
+        RTL_DASH_NOTIFY_OOB,
+
+        RTL_DASH_SEND_BUFFER_DATA_TO_DASH_FW,
+        RTL_DASH_CHECK_SEND_BUFFER_TO_DASH_FW_COMPLETE,
+        RTL_DASH_GET_RCV_FROM_FW_BUFFER_DATA,
+        RTL_DASH_OOB_REQ,
+        RTL_DASH_OOB_ACK,
+        RTL_DASH_DETACH_OOB_REQ,
+        RTL_DASH_DETACH_OOB_ACK,
+
+        RTL_FW_SET_IPV4 = 0x10,
+        RTL_FW_GET_IPV4,
+        RTL_FW_SET_IPV6,
+        RTL_FW_GET_IPV6,
+        RTL_FW_SET_EXT_SNMP,
+        RTL_FW_GET_EXT_SNMP,
+        RTL_FW_SET_WAKEUP_PATTERN,
+        RTL_FW_GET_WAKEUP_PATTERN,
+        RTL_FW_DEL_WAKEUP_PATTERN,
+
+        RTLT_DASH_COMMAND_INVALID,
+};
+
+struct rtl_dash_ioctl_struct {
+        __u32	cmd;
+        __u32	offset;
+        __u32	len;
+        union {
+                __u32	data;
+                void *data_buffer;
+        };
+};
+
+typedef struct _OSOOBHdr {
+        __le32 len;
+        u8 type;
+        u8 flag;
+        u8 hostReqV;
+        u8 res;
+}
+OSOOBHdr, *POSOOBHdr;
+
+typedef struct _RX_DASH_BUFFER_TYPE_2 {
+        OSOOBHdr oobhdr;
+        u8 RxDataBuffer[0];
+}
+RX_DASH_BUFFER_TYPE_2, *PRX_DASH_BUFFER_TYPE_2;
+
+#define ALIGN_8                 (0x7)
+#define ALIGN_16                (0xf)
+#define ALIGN_32                (0x1f)
+#define ALIGN_64                (0x3f)
+#define ALIGN_256               (0xff)
+#define ALIGN_4096              (0xfff)
+
+#define OCP_REG_FIRMWARE_MAJOR_VERSION (0x120)
+
+#define HW_DASH_SUPPORT_DASH(_M)        ((_M)->HwSuppDashVer > 0)
+#define HW_DASH_SUPPORT_TYPE_1(_M)      ((_M)->HwSuppDashVer == 1)
+#define HW_DASH_SUPPORT_TYPE_2(_M)      ((_M)->HwSuppDashVer == 2)
+#define HW_DASH_SUPPORT_TYPE_3(_M)      ((_M)->HwSuppDashVer == 3)
+#define HW_DASH_SUPPORT_TYPE_4(_M)      ((_M)->HwSuppDashVer == 4)
+#define HW_DASH_SUPPORT_CMAC(_M)        (HW_DASH_SUPPORT_TYPE_2(_M) || HW_DASH_SUPPORT_TYPE_3(_M))
+#define HW_DASH_SUPPORT_IPC2(_M)        (HW_DASH_SUPPORT_TYPE_4(_M))
+#define HW_DASH_SUPPORT_GET_FIRMWARE_VERSION(_M) (HW_DASH_SUPPORT_TYPE_2(_M) || \
+                                                  HW_DASH_SUPPORT_TYPE_3(_M) || \
+                                                  HW_DASH_SUPPORT_TYPE_4(_M))
+
+#define RECV_FROM_FW_BUF_SIZE (1520)
+#define SEND_TO_FW_BUF_SIZE (1520)
+
+#define TXS_CC3_0       (BIT_0|BIT_1|BIT_2|BIT_3)
+#define TXS_EXC         BIT_4
+#define TXS_LNKF        BIT_5
+#define TXS_OWC         BIT_6
+#define TXS_TES         BIT_7
+#define TXS_UNF         BIT_9
+#define TXS_LGSEN       BIT_11
+#define TXS_LS          BIT_12
+#define TXS_FS          BIT_13
+#define TXS_EOR         BIT_14
+#define TXS_OWN         BIT_15
+
+#define TPPool_HRDY     0x20
+
+#define RXS_OWN      BIT_15
+#define RXS_EOR      BIT_14
+#define RXS_FS       BIT_13
+#define RXS_LS       BIT_12
+
+#define ISRIMR_DASH_INTR_EN BIT_12
+
+#define NO_BASE_ADDRESS 0x00000000
+
+/* IB2SOC registers */
+#define IPC2_SWISR_DRIVER_READY 0x05
+#define IPC2_SWISR_DRIVER_EXIT 0x06
+#define IPC2_SWISR_CLIENTTOOL_SYNC_HOSTNAME 0x20
+#define IPC2_SWISR_DIS_DASH 0x55
+#define IPC2_SWISR_EN_DASH 0x56
+
+#define IPC2_IB2SOC_SET 0x10
+#define IPC2_IB2SOC_DATA 0x14
+#define IPC2_IB2SOC_CMD 0x18
+#define IPC2_IB2SOC_IMR 0x1C
+
+/* IPC2 registers */
+#define IPC2_PCIE_BASE      0xC100
+#define IPC2_TX_SET_REG     IPC2_PCIE_BASE
+#define IPC2_TX_STATUS_REG  (IPC2_PCIE_BASE+0x04)
+#define IPC2_RX_STATUS_REG  (IPC2_PCIE_BASE+0x08)
+#define IPC2_RX_CLEAR_REG   (IPC2_PCIE_BASE+0x0C)
+#define IPC2_DATA_BASE      0x32000
+#define IPC2_BUFFER_LENGTH  0x1000
+#define IPC2_DATA_MASTER    IPC2_DATA_BASE                        //dash tx buffer base
+#define IPC2_DATA_SLAVE     (IPC2_DATA_BASE+IPC2_BUFFER_LENGTH)   //dash rx buffer base
+#define IPC2_TX_BUFFER      IPC2_DATA_MASTER
+#define IPC2_RX_BUFFER      IPC2_DATA_SLAVE
+
+#define IPC2_TX_SEND_BIT        BIT_0
+#define IPC2_TX_ACK_BIT         BIT_8
+#define IPC2_RX_ROK_BIT         BIT_0
+#define IPC2_RX_ACK_BIT         BIT_8
+
+/* IPC2 write/read MMIO register */
+#define RTL_DASH_IPC2_W8(tp, reg, val8)   RTL_W8(tp, reg, val8)
+#define RTL_DASH_IPC2_W16(tp, reg, val16) RTL_W16(tp, reg, val16)
+#define RTL_DASH_IPC2_W32(tp, reg, val32)  RTL_W32(tp, reg, val32)
+#define RTL_DASH_IPC2_R8(tp, reg)     RTL_R8(tp, reg)
+#define RTL_DASH_IPC2_R16(tp, reg)    RTL_R16(tp, reg)
+#define RTL_DASH_IPC2_R32(tp, reg)    RTL_R32(tp, reg)
+
+/* DASH OOB Header Type */
+#define DASH_OOB_HDR_TYPE_REQ 0x91
+#define DASH_OOB_HDR_TYPE_ACK 0x92
+
+struct  rtl8127_private;
+
+int rtl8127_dash_ioctl(struct net_device *dev, struct ifreq *ifr);
+bool rtl8127_check_dash_interrupt(struct rtl8127_private *tp);
+void rtl8127_handle_dash_interrupt(struct net_device *dev);
+void rtl8127_clear_ipc2_isr(struct rtl8127_private *tp);
+void rtl8127_set_ipc2_soc_imr_bit(struct rtl8127_private *tp, u32 mask);
+void rtl8127_clear_ipc2_soc_imr_bit(struct rtl8127_private *tp, u32 mask);
+
+#endif /* _LINUX_R8127_DASH_H */
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_fiber.c b/drivers/net/ethernet/realtek/r8127/r8127_fiber.c
new file mode 100644
index 000000000000..daf28d75ce86
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_fiber.c
@@ -0,0 +1,278 @@
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#include "r8127.h"
+#include "r8127_fiber.h"
+
+// sds address
+#define R8127_SDS_8127_CMD 0x2348
+#define R8127_SDS_8127_ADDR 0x234A
+#define R8127_SDS_8127_DATA_IN 0x234C
+#define R8127_SDS_8127_DATA_OUT 0x234E
+
+#define R8127_MAKE_SDS_8127_ADDR(_index, _page, _reg) ((_index << 11) | (_page << 5) | _reg)
+
+// sds command
+#define R8127_SDS_8127_CMD_IN BIT_0
+#define R8127_SDS_8127_WE_IN BIT_1
+
+static bool
+rtl8127_wait_8127_sds_cmd_done(struct rtl8127_private *tp)
+{
+        u32 timeout = 0, waitcount = 100;
+
+        do {
+                if (RTL_R16(tp, R8127_SDS_8127_CMD) & R8127_SDS_8127_CMD_IN)
+                        udelay(1);
+                else
+                        return true;
+        } while (++timeout < waitcount);
+
+        return false;
+}
+
+u16
+rtl8127_sds_phy_read_8127(struct rtl8127_private *tp, u16 index, u16 page,
+                          u16 reg)
+{
+        RTL_W16(tp, R8127_SDS_8127_ADDR,
+                R8127_MAKE_SDS_8127_ADDR(index, page, reg));
+        RTL_W16(tp, R8127_SDS_8127_CMD, R8127_SDS_8127_CMD_IN);
+
+        if (rtl8127_wait_8127_sds_cmd_done(tp))
+                return RTL_R16(tp, R8127_SDS_8127_DATA_OUT);
+        else
+                return 0xffff;
+}
+
+void
+rtl8127_sds_phy_write_8127(struct rtl8127_private *tp, u16 index, u16 page,
+                           u16 reg, u16 val)
+{
+        RTL_W16(tp, R8127_SDS_8127_DATA_IN, val);
+        RTL_W16(tp, R8127_SDS_8127_ADDR,
+                R8127_MAKE_SDS_8127_ADDR(index, page, reg));
+        RTL_W16(tp, R8127_SDS_8127_CMD,
+                R8127_SDS_8127_CMD_IN | R8127_SDS_8127_WE_IN);
+
+        rtl8127_wait_8127_sds_cmd_done(tp);
+}
+
+static void
+rtl8127_clear_and_set_sds_phy_bit(struct rtl8127_private *tp, u16 index,
+                                  u16 page, u16 addr, u16 clearmask,
+                                  u16 setmask)
+{
+        u16 val;
+
+        val = rtl8127_sds_phy_read_8127(tp, index, page, addr);
+        val &= ~clearmask;
+        val |= setmask;
+        rtl8127_sds_phy_write_8127(tp, index, page, addr, val);
+}
+
+static void
+rtl8127_clear_sds_phy_bit(struct rtl8127_private *tp, u16 index, u16 page,
+                          u16 addr, u16 mask)
+{
+        rtl8127_clear_and_set_sds_phy_bit(tp,
+                                          index,
+                                          page,
+                                          addr,
+                                          mask,
+                                          0);
+}
+
+static void
+rtl8127_set_sds_phy_bit(struct rtl8127_private *tp, u16 index,
+                        u16 page, u16 addr, u16 mask)
+{
+        rtl8127_clear_and_set_sds_phy_bit(tp,
+                                          index,
+                                          page,
+                                          addr,
+                                          0,
+                                          mask);
+}
+
+static void
+rtl8127_sds_phy_reset_8127(struct rtl8127_private *tp)
+{
+        RTL_W8(tp, 0x2350, RTL_R8(tp, 0x2350) & ~BIT_0);
+        udelay(1);
+
+        RTL_W16(tp, 0x233A, 0x801F);
+        RTL_W8(tp, 0x2350, RTL_R8(tp, 0x2350) | BIT_0);
+        udelay(10);
+}
+
+static void
+rtl8127_sds_phy_reset(struct rtl8127_private *tp)
+{
+        switch (tp->HwFiberModeVer) {
+        case FIBER_MODE_RTL8127ATF:
+                rtl8127_sds_phy_reset_8127(tp);
+                break;
+        default:
+                break;
+        }
+}
+
+static void
+rtl8127_set_sds_phy_caps_1g_8127(struct rtl8127_private *tp)
+{
+        u16 val;
+
+        rtl8127_set_sds_phy_bit(tp, 0, 1, 31, BIT_3);
+        rtl8127_clear_and_set_sds_phy_bit(tp,
+                                          0,
+                                          2,
+                                          0,
+                                          BIT_13 | BIT_12 | BIT_6,
+                                          BIT_12 | BIT_6);
+        RTL_W16(tp, 0x233A, 0x8004);
+
+        val = RTL_R16(tp, 0x233E);
+        val &= (BIT_13 | BIT_12 | BIT_1 | BIT_0);
+        val |= BIT_1;
+        RTL_W16(tp, 0x233E, val);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC40A, 0x0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC466, 0x0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC808, 0x0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC80A, 0x0);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xC804,
+                                              0x000F,
+                                              0x000C);
+}
+
+static void
+rtl8127_sds_phy_exit_1g_8127(struct rtl8127_private *tp)
+{
+        rtl8127_clear_sds_phy_bit(tp, 0, 1, 31, BIT_3);
+        rtl8127_clear_and_set_sds_phy_bit(tp,
+                                          0,
+                                          2,
+                                          0,
+                                          BIT_13 | BIT_12 | BIT_6,
+                                          BIT_6);
+
+        rtl8127_sds_phy_reset(tp);
+}
+
+static void
+rtl8127_set_sds_phy_caps_10g_8127(struct rtl8127_private *tp)
+{
+        u16 val;
+
+        RTL_W16(tp, 0x233A, 0x801A);
+
+        val = RTL_R16(tp, 0x233E);
+        val &= (BIT_13 | BIT_12 | BIT_1 | BIT_0);
+        val |= BIT_12;
+        RTL_W16(tp, 0x233E, val);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC40A, 0x0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC466, 0x3);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC808, 0x0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xC80A, 0x0);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xC804,
+                                              0x000F,
+                                              0x000C);
+}
+
+static void
+rtl8127_set_sds_phy_caps_8127(struct rtl8127_private *tp)
+{
+        rtl8127_sds_phy_exit_1g_8127(tp);
+
+        switch (tp->speed) {
+        case SPEED_10000:
+                rtl8127_set_sds_phy_caps_10g_8127(tp);
+                break;
+        case SPEED_1000:
+                rtl8127_set_sds_phy_caps_1g_8127(tp);
+                break;
+        default:
+                break;
+        }
+}
+
+static void
+rtl8127_set_sds_phy_caps(struct rtl8127_private *tp)
+{
+        switch (tp->HwFiberModeVer) {
+        case FIBER_MODE_RTL8127ATF:
+                rtl8127_set_sds_phy_caps_8127(tp);
+                break;
+        default:
+                break;
+        }
+}
+
+static void
+rtl8127_hw_sds_phy_config(struct rtl8127_private *tp)
+{
+        rtl8127_set_sds_phy_caps(tp);
+}
+
+void
+rtl8127_hw_fiber_phy_config(struct rtl8127_private *tp)
+{
+        switch (tp->HwFiberModeVer) {
+        case FIBER_MODE_RTL8127ATF:
+                rtl8127_hw_sds_phy_config(tp);
+                break;
+        default:
+                break;
+        }
+}
+
+bool
+rtl8127_check_fiber_mode_support(struct rtl8127_private *tp)
+{
+        switch(tp->mcfg) {
+        case CFG_METHOD_2: {
+                u8 tmp = (u8)rtl8127_mac_ocp_read(tp, 0xD006);
+                if (tmp == 0x07)
+                        tp->HwFiberModeVer = FIBER_MODE_RTL8127ATF;
+        }
+        break;
+        default:
+                break;
+        }
+
+        return (HW_FIBER_MODE_ENABLED(tp)) ? true : false;
+}
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_fiber.h b/drivers/net/ethernet/realtek/r8127/r8127_fiber.h
new file mode 100644
index 000000000000..00d59b67d39e
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_fiber.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_R8127_FIBER_H
+#define _LINUX_R8127_FIBER_H
+
+enum {
+        FIBER_MODE_NIC_ONLY = 0,
+        FIBER_MODE_RTL8127ATF,
+        FIBER_MODE_MAX
+};
+
+#define HW_FIBER_MODE_ENABLED(_M)        ((_M)->HwFiberModeVer > 0)
+
+struct rtl8127_private;
+
+u16 rtl8127_sds_phy_read_8127(struct rtl8127_private *tp, u16 index, u16 page,
+                              u16 reg);
+void rtl8127_sds_phy_write_8127(struct rtl8127_private *tp, u16 index, u16 page,
+                                u16 reg, u16 val);
+void rtl8127_hw_fiber_phy_config(struct rtl8127_private *tp);
+bool rtl8127_check_fiber_mode_support(struct rtl8127_private *tp);
+
+#endif /* _LINUX_R8127_FIBER_H */
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_firmware.c b/drivers/net/ethernet/realtek/r8127/r8127_firmware.c
new file mode 100644
index 000000000000..7ab59f641e77
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_firmware.c
@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#include <linux/version.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+
+#include "r8127_firmware.h"
+
+enum rtl_fw_opcode {
+        PHY_READ		= 0x0,
+        PHY_DATA_OR		= 0x1,
+        PHY_DATA_AND		= 0x2,
+        PHY_BJMPN		= 0x3,
+        PHY_MDIO_CHG		= 0x4,
+        PHY_CLEAR_READCOUNT	= 0x7,
+        PHY_WRITE		= 0x8,
+        PHY_READCOUNT_EQ_SKIP	= 0x9,
+        PHY_COMP_EQ_SKIPN	= 0xa,
+        PHY_COMP_NEQ_SKIPN	= 0xb,
+        PHY_WRITE_PREVIOUS	= 0xc,
+        PHY_SKIPN		= 0xd,
+        PHY_DELAY_MS		= 0xe,
+};
+
+struct fw_info {
+        u32	magic;
+        char	version[RTL8127_VER_SIZE];
+        __le32	fw_start;
+        __le32	fw_len;
+        u8	chksum;
+} __packed;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,16,0)
+#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
+#endif
+#define FW_OPCODE_SIZE sizeof_field(struct rtl8127_fw_phy_action, code[0])
+
+static bool rtl8127_fw_format_ok(struct rtl8127_fw *rtl_fw)
+{
+        const struct firmware *fw = rtl_fw->fw;
+        struct fw_info *fw_info = (struct fw_info *)fw->data;
+        struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action;
+
+        if (fw->size < FW_OPCODE_SIZE)
+                return false;
+
+        if (!fw_info->magic) {
+                size_t i, size, start;
+                u8 checksum = 0;
+
+                if (fw->size < sizeof(*fw_info))
+                        return false;
+
+                for (i = 0; i < fw->size; i++)
+                        checksum += fw->data[i];
+                if (checksum != 0)
+                        return false;
+
+                start = le32_to_cpu(fw_info->fw_start);
+                if (start > fw->size)
+                        return false;
+
+                size = le32_to_cpu(fw_info->fw_len);
+                if (size > (fw->size - start) / FW_OPCODE_SIZE)
+                        return false;
+
+                strscpy(rtl_fw->version, fw_info->version, RTL8127_VER_SIZE);
+
+                pa->code = (__le32 *)(fw->data + start);
+                pa->size = size;
+        } else {
+                if (fw->size % FW_OPCODE_SIZE)
+                        return false;
+
+                strscpy(rtl_fw->version, rtl_fw->fw_name, RTL8127_VER_SIZE);
+
+                pa->code = (__le32 *)fw->data;
+                pa->size = fw->size / FW_OPCODE_SIZE;
+        }
+
+        return true;
+}
+
+static bool rtl8127_fw_data_ok(struct rtl8127_fw *rtl_fw)
+{
+        struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action;
+        size_t index;
+
+        for (index = 0; index < pa->size; index++) {
+                u32 action = le32_to_cpu(pa->code[index]);
+                u32 val = action & 0x0000ffff;
+                u32 regno = (action & 0x0fff0000) >> 16;
+
+                switch (action >> 28) {
+                case PHY_READ:
+                case PHY_DATA_OR:
+                case PHY_DATA_AND:
+                case PHY_CLEAR_READCOUNT:
+                case PHY_WRITE:
+                case PHY_WRITE_PREVIOUS:
+                case PHY_DELAY_MS:
+                        break;
+
+                case PHY_MDIO_CHG:
+                        if (val > 1)
+                                goto out;
+                        break;
+
+                case PHY_BJMPN:
+                        if (regno > index)
+                                goto out;
+                        break;
+                case PHY_READCOUNT_EQ_SKIP:
+                        if (index + 2 >= pa->size)
+                                goto out;
+                        break;
+                case PHY_COMP_EQ_SKIPN:
+                case PHY_COMP_NEQ_SKIPN:
+                case PHY_SKIPN:
+                        if (index + 1 + regno >= pa->size)
+                                goto out;
+                        break;
+
+                default:
+                        dev_err(rtl_fw->dev, "Invalid action 0x%08x\n", action);
+                        return false;
+                }
+        }
+
+        return true;
+out:
+        dev_err(rtl_fw->dev, "Out of range of firmware\n");
+        return false;
+}
+
+void rtl8127_fw_write_firmware(struct rtl8127_private *tp, struct rtl8127_fw *rtl_fw)
+{
+        struct rtl8127_fw_phy_action *pa = &rtl_fw->phy_action;
+        rtl8127_fw_write_t fw_write = rtl_fw->phy_write;
+        rtl8127_fw_read_t fw_read = rtl_fw->phy_read;
+        int predata = 0, count = 0;
+        size_t index;
+
+        for (index = 0; index < pa->size; index++) {
+                u32 action = le32_to_cpu(pa->code[index]);
+                u32 data = action & 0x0000ffff;
+                u32 regno = (action & 0x0fff0000) >> 16;
+                enum rtl_fw_opcode opcode = action >> 28;
+
+                if (!action)
+                        break;
+
+                switch (opcode) {
+                case PHY_READ:
+                        predata = fw_read(tp, regno);
+                        count++;
+                        break;
+                case PHY_DATA_OR:
+                        predata |= data;
+                        break;
+                case PHY_DATA_AND:
+                        predata &= data;
+                        break;
+                case PHY_BJMPN:
+                        index -= (regno + 1);
+                        break;
+                case PHY_MDIO_CHG:
+                        if (data) {
+                                fw_write = rtl_fw->mac_mcu_write;
+                                fw_read = rtl_fw->mac_mcu_read;
+                        } else {
+                                fw_write = rtl_fw->phy_write;
+                                fw_read = rtl_fw->phy_read;
+                        }
+
+                        break;
+                case PHY_CLEAR_READCOUNT:
+                        count = 0;
+                        break;
+                case PHY_WRITE:
+                        fw_write(tp, regno, data);
+                        break;
+                case PHY_READCOUNT_EQ_SKIP:
+                        if (count == data)
+                                index++;
+                        break;
+                case PHY_COMP_EQ_SKIPN:
+                        if (predata == data)
+                                index += regno;
+                        break;
+                case PHY_COMP_NEQ_SKIPN:
+                        if (predata != data)
+                                index += regno;
+                        break;
+                case PHY_WRITE_PREVIOUS:
+                        fw_write(tp, regno, predata);
+                        break;
+                case PHY_SKIPN:
+                        index += regno;
+                        break;
+                case PHY_DELAY_MS:
+                        mdelay(data);
+                        break;
+                }
+        }
+}
+
+void rtl8127_fw_release_firmware(struct rtl8127_fw *rtl_fw)
+{
+        release_firmware(rtl_fw->fw);
+}
+
+int rtl8127_fw_request_firmware(struct rtl8127_fw *rtl_fw)
+{
+        int rc;
+
+        rc = request_firmware(&rtl_fw->fw, rtl_fw->fw_name, rtl_fw->dev);
+        if (rc < 0)
+                goto out;
+
+        if (!rtl8127_fw_format_ok(rtl_fw) || !rtl8127_fw_data_ok(rtl_fw)) {
+                release_firmware(rtl_fw->fw);
+                rc = -EINVAL;
+                goto out;
+        }
+
+        return 0;
+out:
+        dev_err(rtl_fw->dev, "Unable to load firmware %s (%d)\n",
+                rtl_fw->fw_name, rc);
+        return rc;
+}
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_firmware.h b/drivers/net/ethernet/realtek/r8127/r8127_firmware.h
new file mode 100644
index 000000000000..6b1acea98a3f
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_firmware.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_R8127_FIRMWARE_H
+#define _LINUX_R8127_FIRMWARE_H
+
+#include <linux/device.h>
+#include <linux/firmware.h>
+
+struct rtl8127_private;
+typedef void (*rtl8127_fw_write_t)(struct rtl8127_private *tp, u16 reg, u16 val);
+typedef u32 (*rtl8127_fw_read_t)(struct rtl8127_private *tp, u16 reg);
+
+#define RTL8127_VER_SIZE		32
+
+struct rtl8127_fw {
+        rtl8127_fw_write_t phy_write;
+        rtl8127_fw_read_t phy_read;
+        rtl8127_fw_write_t mac_mcu_write;
+        rtl8127_fw_read_t mac_mcu_read;
+        const struct firmware *fw;
+        const char *fw_name;
+        struct device *dev;
+
+        char version[RTL8127_VER_SIZE];
+
+        struct rtl8127_fw_phy_action {
+                __le32 *code;
+                size_t size;
+        } phy_action;
+};
+
+int rtl8127_fw_request_firmware(struct rtl8127_fw *rtl_fw);
+void rtl8127_fw_release_firmware(struct rtl8127_fw *rtl_fw);
+void rtl8127_fw_write_firmware(struct rtl8127_private *tp, struct rtl8127_fw *rtl_fw);
+
+#endif /* _LINUX_R8127_FIRMWARE_H */
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_n.c b/drivers/net/ethernet/realtek/r8127/r8127_n.c
new file mode 100644
index 000000000000..86573b697672
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_n.c
@@ -0,0 +1,18003 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+/*
+ * This driver is modified from r8169.c in Linux kernel 2.6.18
+ */
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/delay.h>
+#include <linux/mii.h>
+#include <linux/if_vlan.h>
+#include <linux/crc32.h>
+#include <linux/interrupt.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22)
+#include <linux/ipv6.h>
+#include <net/ip6_checksum.h>
+#endif
+#include <linux/tcp.h>
+#include <linux/init.h>
+#include <linux/rtnetlink.h>
+#include <linux/completion.h>
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
+#include <linux/pci-aspm.h>
+#endif
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37)
+#include <linux/prefetch.h>
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
+#define dev_printk(A,B,fmt,args...) printk(A fmt,##args)
+#else
+#include <linux/dma-mapping.h>
+#include <linux/moduleparam.h>
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,31)
+#include <linux/mdio.h>
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,4,10)
+#include <net/gso.h>
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,4,10) */
+
+#include <asm/io.h>
+#include <asm/irq.h>
+
+#include "r8127.h"
+#include "rtl_eeprom.h"
+#include "rtltool.h"
+#include "r8127_firmware.h"
+
+#ifdef ENABLE_R8127_PROCFS
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#endif
+
+#define FIRMWARE_8127_1		"rtl_nic/rtl8127-1.fw"
+#define FIRMWARE_8127_2		"rtl_nic/rtl8127-2.fw"
+
+static const struct {
+        const char *name;
+        const char *fw_name;
+} rtl_chip_fw_infos[] = {
+        /* PCI-E devices. */
+        [CFG_METHOD_1] = {"RTL8127",		FIRMWARE_8127_1},
+        [CFG_METHOD_2] = {"RTL8127",		FIRMWARE_8127_2},
+        [CFG_METHOD_DEFAULT] = {"Unknown",                  },
+};
+
+#define _R(NAME,MAC,RCR,MASK,JumFrameSz) \
+    { .name = NAME, .mcfg = MAC, .RCR_Cfg = RCR, .RxConfigMask = MASK, .jumbo_frame_sz = JumFrameSz }
+
+static const struct {
+        const char *name;
+        u8 mcfg;
+        u32 RCR_Cfg;
+        u32 RxConfigMask;   /* Clears the bits supported by this chip */
+        u32 jumbo_frame_sz;
+} rtl_chip_info[] = {
+        _R("RTL8127",
+        CFG_METHOD_1,
+        Rx_Fetch_Number_8 | Rx_Close_Multiple | RxCfg_pause_slot_en | EnableInnerVlan | EnableOuterVlan | (RX_DMA_BURST_512 << RxCfgDMAShift),
+        0xff7e5880,
+        Jumbo_Frame_9k),
+
+        _R("RTL8127",
+        CFG_METHOD_2,
+        Rx_Fetch_Number_8 | Rx_Close_Multiple | RxCfg_pause_slot_en | EnableInnerVlan | EnableOuterVlan | (RX_DMA_BURST_512 << RxCfgDMAShift),
+        0xff7e5880,
+        Jumbo_Frame_9k),
+
+        _R("Unknown",
+        CFG_METHOD_DEFAULT,
+        (RX_DMA_BURST_512 << RxCfgDMAShift),
+        0xff7e5880,
+        Jumbo_Frame_1k)
+};
+#undef _R
+
+
+static struct pci_device_id rtl8127_pci_tbl[] = {
+        { PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8127), },
+        { PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x0E10), },
+        {0,},
+};
+
+MODULE_DEVICE_TABLE(pci, rtl8127_pci_tbl);
+
+static int use_dac = 1;
+static int timer_count = 0x2600;
+static int timer_count_v2 = (0x2600 / 0x200);
+
+static struct {
+        u32 msg_enable;
+} debug = { -1 };
+
+static unsigned int speed_mode = SPEED_10000;
+static unsigned int duplex_mode = DUPLEX_FULL;
+static unsigned int autoneg_mode = AUTONEG_ENABLE;
+#ifdef CONFIG_ASPM
+static int aspm = 1;
+#else
+static int aspm = 0;
+#endif
+#ifdef ENABLE_S5WOL
+static int s5wol = 1;
+#else
+static int s5wol = 0;
+#endif
+#ifdef ENABLE_S5_KEEP_CURR_MAC
+static int s5_keep_curr_mac = 1;
+#else
+static int s5_keep_curr_mac = 0;
+#endif
+#ifdef ENABLE_EEE
+static int eee_enable = 1;
+#else
+static int eee_enable = 0;
+#endif
+#ifdef CONFIG_SOC_LAN
+static ulong hwoptimize = HW_PATCH_SOC_LAN;
+#else
+static ulong hwoptimize = 0;
+#endif
+#ifdef ENABLE_S0_MAGIC_PACKET
+static int s0_magic_packet = 1;
+#else
+static int s0_magic_packet = 0;
+#endif
+#ifdef ENABLE_TX_NO_CLOSE
+static int tx_no_close_enable = 1;
+#else
+static int tx_no_close_enable = 0;
+#endif
+#ifdef DISABLE_WOL_SUPPORT
+static int disable_wol_support = 1;
+#else
+static int disable_wol_support = 0;
+#endif
+#ifdef ENABLE_DOUBLE_VLAN
+static int enable_double_vlan = 1;
+#else
+static int enable_double_vlan = 0;
+#endif
+#ifdef ENABLE_GIGA_LITE
+static int eee_giga_lite = 1;
+#else
+static int eee_giga_lite = 0;
+#endif
+
+MODULE_AUTHOR("Realtek and the Linux r8127 crew <netdev@vger.kernel.org>");
+MODULE_DESCRIPTION("Realtek r8127 Ethernet controller driver");
+
+module_param(speed_mode, uint, 0);
+MODULE_PARM_DESC(speed_mode, "force phy operation. Deprecated by ethtool (8).");
+
+module_param(duplex_mode, uint, 0);
+MODULE_PARM_DESC(duplex_mode, "force phy operation. Deprecated by ethtool (8).");
+
+module_param(autoneg_mode, uint, 0);
+MODULE_PARM_DESC(autoneg_mode, "force phy operation. Deprecated by ethtool (8).");
+
+module_param(aspm, int, 0);
+MODULE_PARM_DESC(aspm, "Enable ASPM.");
+
+module_param(s5wol, int, 0);
+MODULE_PARM_DESC(s5wol, "Enable Shutdown Wake On Lan.");
+
+module_param(s5_keep_curr_mac, int, 0);
+MODULE_PARM_DESC(s5_keep_curr_mac, "Enable Shutdown Keep Current MAC Address.");
+
+module_param(use_dac, int, 0);
+MODULE_PARM_DESC(use_dac, "Enable PCI DAC. Unsafe on 32 bit PCI slot.");
+
+module_param(timer_count, int, 0);
+MODULE_PARM_DESC(timer_count, "Timer Interrupt Interval.");
+
+module_param(eee_enable, int, 0);
+MODULE_PARM_DESC(eee_enable, "Enable Energy Efficient Ethernet.");
+
+module_param(hwoptimize, ulong, 0);
+MODULE_PARM_DESC(hwoptimize, "Enable HW optimization function.");
+
+module_param(s0_magic_packet, int, 0);
+MODULE_PARM_DESC(s0_magic_packet, "Enable S0 Magic Packet.");
+
+module_param(tx_no_close_enable, int, 0);
+MODULE_PARM_DESC(tx_no_close_enable, "Enable TX No Close.");
+
+module_param(disable_wol_support, int, 0);
+MODULE_PARM_DESC(disable_wol_support, "Disable PM support.");
+
+module_param(enable_double_vlan, int, 0);
+MODULE_PARM_DESC(enable_double_vlan, "Enable Double VLAN.");
+
+module_param(eee_giga_lite, int, 0);
+MODULE_PARM_DESC(eee_giga_lite, "Enable Giga Lite.");
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+module_param_named(debug, debug.msg_enable, int, 0);
+MODULE_PARM_DESC(debug, "Debug verbosity level (0=none, ..., 16=all)");
+#endif//LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+
+MODULE_LICENSE("GPL");
+#ifdef ENABLE_USE_FIRMWARE_FILE
+MODULE_FIRMWARE(FIRMWARE_8127_1);
+MODULE_FIRMWARE(FIRMWARE_8127_2);
+#endif
+
+MODULE_VERSION(RTL8127_VERSION);
+
+/*
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+static void rtl8127_esd_timer(unsigned long __opaque);
+#else
+static void rtl8127_esd_timer(struct timer_list *t);
+#endif
+*/
+/*
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+static void rtl8127_link_timer(unsigned long __opaque);
+#else
+static void rtl8127_link_timer(struct timer_list *t);
+#endif
+*/
+
+static netdev_tx_t rtl8127_start_xmit(struct sk_buff *skb, struct net_device *dev);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance, struct pt_regs *regs);
+#else
+static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance);
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance, struct pt_regs *regs);
+#else
+static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance);
+#endif
+static void rtl8127_set_rx_mode(struct net_device *dev);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
+static void rtl8127_tx_timeout(struct net_device *dev, unsigned int txqueue);
+#else
+static void rtl8127_tx_timeout(struct net_device *dev);
+#endif
+static int rtl8127_rx_interrupt(struct net_device *, struct rtl8127_private *, struct rtl8127_rx_ring *, napi_budget);
+static int rtl8127_tx_interrupt(struct rtl8127_tx_ring *ring, int budget);
+static int rtl8127_tx_interrupt_with_vector(struct rtl8127_private *tp, const int message_id, int budget);
+static void rtl8127_wait_for_quiescence(struct net_device *dev);
+static int rtl8127_change_mtu(struct net_device *dev, int new_mtu);
+static void rtl8127_down(struct net_device *dev);
+
+static int rtl8127_set_mac_address(struct net_device *dev, void *p);
+static void rtl8127_rar_set(struct rtl8127_private *tp, const u8 *addr);
+static void rtl8127_desc_addr_fill(struct rtl8127_private *);
+static void rtl8127_tx_desc_init(struct rtl8127_private *tp);
+static void rtl8127_rx_desc_init(struct rtl8127_private *tp);
+
+static u16 rtl8127_get_hw_phy_mcu_code_ver(struct rtl8127_private *tp);
+static void rtl8127_phy_power_up(struct net_device *dev);
+static void rtl8127_phy_power_down(struct net_device *dev);
+static int rtl8127_set_speed(struct net_device *dev, u8 autoneg, u32 speed, u8 duplex, u64 adv);
+static bool rtl8127_set_phy_mcu_patch_request(struct rtl8127_private *tp);
+static bool rtl8127_clear_phy_mcu_patch_request(struct rtl8127_private *tp);
+
+#ifdef CONFIG_R8127_NAPI
+static int rtl8127_poll(napi_ptr napi, napi_budget budget);
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+static void rtl8127_reset_task(void *_data);
+static void rtl8127_esd_task(void *_data);
+static void rtl8127_linkchg_task(void *_data);
+static void rtl8127_dash_task(void *_data);
+#else
+static void rtl8127_reset_task(struct work_struct *work);
+static void rtl8127_esd_task(struct work_struct *work);
+static void rtl8127_linkchg_task(struct work_struct *work);
+static void rtl8127_dash_task(struct work_struct *work);
+#endif
+static void rtl8127_schedule_reset_work(struct rtl8127_private *tp);
+static void rtl8127_schedule_esd_work(struct rtl8127_private *tp);
+static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp);
+void rtl8127_schedule_dash_work(struct rtl8127_private *tp);
+static void rtl8127_init_all_schedule_work(struct rtl8127_private *tp);
+static void rtl8127_cancel_all_schedule_work(struct rtl8127_private *tp);
+
+static inline struct device *tp_to_dev(struct rtl8127_private *tp)
+{
+        return &tp->pci_dev->dev;
+}
+
+#if ((LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0) && \
+     LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,00)))
+void ethtool_convert_legacy_u32_to_link_mode(unsigned long *dst,
+                u32 legacy_u32)
+{
+        bitmap_zero(dst, __ETHTOOL_LINK_MODE_MASK_NBITS);
+        dst[0] = legacy_u32;
+}
+
+bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32,
+                const unsigned long *src)
+{
+        bool retval = true;
+
+        /* TODO: following test will soon always be true */
+        if (__ETHTOOL_LINK_MODE_MASK_NBITS > 32) {
+                __ETHTOOL_DECLARE_LINK_MODE_MASK(ext);
+
+                bitmap_zero(ext, __ETHTOOL_LINK_MODE_MASK_NBITS);
+                bitmap_fill(ext, 32);
+                bitmap_complement(ext, ext, __ETHTOOL_LINK_MODE_MASK_NBITS);
+                if (bitmap_intersects(ext, src,
+                                      __ETHTOOL_LINK_MODE_MASK_NBITS)) {
+                        /* src mask goes beyond bit 31 */
+                        retval = false;
+                }
+        }
+        *legacy_u32 = src[0];
+        return retval;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+
+#ifndef LPA_1000FULL
+#define LPA_1000FULL            0x0800
+#endif
+
+#ifndef LPA_1000HALF
+#define LPA_1000HALF            0x0400
+#endif
+
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,4,0)
+static inline void eth_hw_addr_random(struct net_device *dev)
+{
+        random_ether_addr(dev->dev_addr);
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
+#undef ethtool_ops
+#define ethtool_ops _kc_ethtool_ops
+
+struct _kc_ethtool_ops {
+        int  (*get_settings)(struct net_device *, struct ethtool_cmd *);
+        int  (*set_settings)(struct net_device *, struct ethtool_cmd *);
+        void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *);
+        int  (*get_regs_len)(struct net_device *);
+        void (*get_regs)(struct net_device *, struct ethtool_regs *, void *);
+        void (*get_wol)(struct net_device *, struct ethtool_wolinfo *);
+        int  (*set_wol)(struct net_device *, struct ethtool_wolinfo *);
+        u32  (*get_msglevel)(struct net_device *);
+        void (*set_msglevel)(struct net_device *, u32);
+        int  (*nway_reset)(struct net_device *);
+        u32  (*get_link)(struct net_device *);
+        int  (*get_eeprom_len)(struct net_device *);
+        int  (*get_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *);
+        int  (*set_eeprom)(struct net_device *, struct ethtool_eeprom *, u8 *);
+        int  (*get_coalesce)(struct net_device *, struct ethtool_coalesce *);
+        int  (*set_coalesce)(struct net_device *, struct ethtool_coalesce *);
+        void (*get_ringparam)(struct net_device *, struct ethtool_ringparam *);
+        int  (*set_ringparam)(struct net_device *, struct ethtool_ringparam *);
+        void (*get_pauseparam)(struct net_device *,
+                               struct ethtool_pauseparam*);
+        int  (*set_pauseparam)(struct net_device *,
+                               struct ethtool_pauseparam*);
+        u32  (*get_rx_csum)(struct net_device *);
+        int  (*set_rx_csum)(struct net_device *, u32);
+        u32  (*get_tx_csum)(struct net_device *);
+        int  (*set_tx_csum)(struct net_device *, u32);
+        u32  (*get_sg)(struct net_device *);
+        int  (*set_sg)(struct net_device *, u32);
+        u32  (*get_tso)(struct net_device *);
+        int  (*set_tso)(struct net_device *, u32);
+        int  (*self_test_count)(struct net_device *);
+        void (*self_test)(struct net_device *, struct ethtool_test *, u64 *);
+        void (*get_strings)(struct net_device *, u32 stringset, u8 *);
+        int  (*phys_id)(struct net_device *, u32);
+        int  (*get_stats_count)(struct net_device *);
+        void (*get_ethtool_stats)(struct net_device *, struct ethtool_stats *,
+                                  u64 *);
+} *ethtool_ops = NULL;
+
+#undef SET_ETHTOOL_OPS
+#define SET_ETHTOOL_OPS(netdev, ops) (ethtool_ops = (ops))
+
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
+#ifndef SET_ETHTOOL_OPS
+#define SET_ETHTOOL_OPS(netdev,ops) \
+         ((netdev)->ethtool_ops = (ops))
+#endif //SET_ETHTOOL_OPS
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
+
+//#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5)
+#ifndef netif_msg_init
+#define netif_msg_init _kc_netif_msg_init
+/* copied from linux kernel 2.6.20 include/linux/netdevice.h */
+static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
+{
+        /* use default */
+        if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
+                return default_msg_enable_bits;
+        if (debug_value == 0)   /* no output */
+                return 0;
+        /* set low N bits */
+        return (1 << debug_value) - 1;
+}
+
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,5)
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,22)
+static inline void eth_copy_and_sum (struct sk_buff *dest,
+                                     const unsigned char *src,
+                                     int len, int base)
+{
+        skb_copy_to_linear_data(dest, src, len);
+}
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,22)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7)
+/* copied from linux kernel 2.6.20 /include/linux/time.h */
+/* Parameters used to convert the timespec values: */
+#define MSEC_PER_SEC    1000L
+
+/* copied from linux kernel 2.6.20 /include/linux/jiffies.h */
+/*
+ * Change timeval to jiffies, trying to avoid the
+ * most obvious overflows..
+ *
+ * And some not so obvious.
+ *
+ * Note that we don't want to return MAX_LONG, because
+ * for various timeout reasons we often end up having
+ * to wait "jiffies+1" in order to guarantee that we wait
+ * at _least_ "jiffies" - so "jiffies+1" had better still
+ * be positive.
+ */
+#define MAX_JIFFY_OFFSET ((~0UL >> 1)-1)
+
+/*
+ * Convert jiffies to milliseconds and back.
+ *
+ * Avoid unnecessary multiplications/divisions in the
+ * two most common HZ cases:
+ */
+static inline unsigned int _kc_jiffies_to_msecs(const unsigned long j)
+{
+#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ)
+        return (MSEC_PER_SEC / HZ) * j;
+#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC)
+        return (j + (HZ / MSEC_PER_SEC) - 1)/(HZ / MSEC_PER_SEC);
+#else
+        return (j * MSEC_PER_SEC) / HZ;
+#endif
+}
+
+static inline unsigned long _kc_msecs_to_jiffies(const unsigned int m)
+{
+        if (m > _kc_jiffies_to_msecs(MAX_JIFFY_OFFSET))
+                return MAX_JIFFY_OFFSET;
+#if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ)
+        return (m + (MSEC_PER_SEC / HZ) - 1) / (MSEC_PER_SEC / HZ);
+#elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC)
+        return m * (HZ / MSEC_PER_SEC);
+#else
+        return (m * HZ + MSEC_PER_SEC - 1) / MSEC_PER_SEC;
+#endif
+}
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7)
+
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11)
+
+/* copied from linux kernel 2.6.12.6 /include/linux/pm.h */
+typedef int __bitwise pci_power_t;
+
+/* copied from linux kernel 2.6.12.6 /include/linux/pci.h */
+typedef u32 __bitwise pm_message_t;
+
+#define PCI_D0  ((pci_power_t __force) 0)
+#define PCI_D1  ((pci_power_t __force) 1)
+#define PCI_D2  ((pci_power_t __force) 2)
+#define PCI_D3hot   ((pci_power_t __force) 3)
+#define PCI_D3cold  ((pci_power_t __force) 4)
+#define PCI_POWER_ERROR ((pci_power_t __force) -1)
+
+/* copied from linux kernel 2.6.12.6 /drivers/pci/pci.c */
+/**
+ * pci_choose_state - Choose the power state of a PCI device
+ * @dev: PCI device to be suspended
+ * @state: target sleep state for the whole system. This is the value
+ *  that is passed to suspend() function.
+ *
+ * Returns PCI power state suitable for given device and given system
+ * message.
+ */
+
+pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
+{
+        if (!pci_find_capability(dev, PCI_CAP_ID_PM))
+                return PCI_D0;
+
+        switch (state) {
+        case 0:
+                return PCI_D0;
+        case 3:
+                return PCI_D3hot;
+        default:
+                printk("They asked me for state %d\n", state);
+//      BUG();
+        }
+        return PCI_D0;
+}
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9)
+/**
+ * msleep_interruptible - sleep waiting for waitqueue interruptions
+ * @msecs: Time in milliseconds to sleep for
+ */
+#define msleep_interruptible _kc_msleep_interruptible
+unsigned long _kc_msleep_interruptible(unsigned int msecs)
+{
+        unsigned long timeout = _kc_msecs_to_jiffies(msecs);
+
+        while (timeout && !signal_pending(current)) {
+                set_current_state(TASK_INTERRUPTIBLE);
+                timeout = schedule_timeout(timeout);
+        }
+        return _kc_jiffies_to_msecs(timeout);
+}
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7)
+/* copied from linux kernel 2.6.20 include/linux/sched.h */
+#ifndef __sched
+#define __sched     __attribute__((__section__(".sched.text")))
+#endif
+
+/* copied from linux kernel 2.6.20 kernel/timer.c */
+signed long __sched schedule_timeout_uninterruptible(signed long timeout)
+{
+        __set_current_state(TASK_UNINTERRUPTIBLE);
+        return schedule_timeout(timeout);
+}
+
+/* copied from linux kernel 2.6.20 include/linux/mii.h */
+#undef if_mii
+#define if_mii _kc_if_mii
+static inline struct mii_ioctl_data *if_mii(struct ifreq *rq)
+{
+        return (struct mii_ioctl_data *) &rq->ifr_ifru;
+}
+#endif  //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,7)
+
+static u32 rtl8127_read_thermal_sensor(struct rtl8127_private *tp)
+{
+        u16 ts_digout;
+
+        ts_digout = rtl8127_mdio_direct_read_phy_ocp(tp, 0xBD84);
+        ts_digout &= 0x3ff;
+
+        return ts_digout;
+}
+
+int rtl8127_dump_tally_counter(struct rtl8127_private *tp, dma_addr_t paddr)
+{
+        u32 cmd;
+        u32 WaitCnt;
+        int retval = -1;
+
+        RTL_W32(tp, CounterAddrHigh, (u64)paddr >> 32);
+        cmd = (u64)paddr & DMA_BIT_MASK(32);
+        RTL_W32(tp, CounterAddrLow, cmd);
+        RTL_W32(tp, CounterAddrLow, cmd | CounterDump);
+
+        WaitCnt = 0;
+        while (RTL_R32(tp, CounterAddrLow) & CounterDump) {
+                udelay(10);
+
+                WaitCnt++;
+                if (WaitCnt > 20)
+                        break;
+        }
+
+        if (WaitCnt <= 20)
+                retval = 0;
+
+        return retval;
+}
+
+static u32
+rtl8127_get_hw_clo_ptr(struct rtl8127_tx_ring *ring)
+{
+        struct rtl8127_private *tp = ring->priv;
+
+        switch (tp->HwSuppTxNoCloseVer) {
+        case 3:
+                return RTL_R16(tp, ring->hw_clo_ptr_reg);
+        case 4:
+        case 5:
+        case 6:
+                return RTL_R32(tp, ring->hw_clo_ptr_reg);
+        default:
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                WARN_ON(1);
+#endif
+                return 0;
+        }
+}
+
+static u32
+rtl8127_get_sw_tail_ptr(struct rtl8127_tx_ring *ring)
+{
+        struct rtl8127_private *tp = ring->priv;
+
+        switch (tp->HwSuppTxNoCloseVer) {
+        case 3:
+                return RTL_R16(tp, ring->sw_tail_ptr_reg);
+        case 4:
+        case 5:
+        case 6:
+                return RTL_R32(tp, ring->sw_tail_ptr_reg);
+        default:
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                WARN_ON(1);
+#endif
+                return 0;
+        }
+}
+
+static u32
+rtl8127_get_phy_status(struct rtl8127_private *tp)
+{
+        return RTL_R32(tp, PHYstatus);
+}
+
+static bool
+rtl8127_sysfs_testmode_on(struct rtl8127_private *tp)
+{
+#ifdef ENABLE_R8127_SYSFS
+        return !!tp->testmode;
+#else
+        return 1;
+#endif
+}
+
+static u32 rtl8127_convert_link_speed(u32 status)
+{
+        u32 speed = SPEED_UNKNOWN;
+
+        if (status & LinkStatus) {
+                if (status & _10000bpsF)
+                        speed = SPEED_10000;
+                else if (status & (_5000bpsF | _10000bpsL))
+                        speed = SPEED_5000;
+                else if (status & (_2500bpsF | _5000bpsL))
+                        speed = SPEED_2500;
+                else if (status & (_1000bpsF | _2500bpsL | _1000bpsL))
+                        speed = SPEED_1000;
+                else if (status & _100bps)
+                        speed = SPEED_100;
+                else if (status & _10bps)
+                        speed = SPEED_10;
+        }
+
+        return speed;
+}
+
+static void rtl8127_mdi_swap(struct rtl8127_private *tp)
+{
+        int i;
+        u16 reg, val, mdi_reverse;
+        u16 tps_p0, tps_p1, tps_p2, tps_p3, tps_p3_p0;
+
+        switch (tp->mcfg) {
+        default:
+                return;
+        };
+
+        tps_p3_p0 = rtl8127_mac_ocp_read(tp, 0xD440) & 0xF000;
+        tps_p3 = !!(tps_p3_p0 & BIT_15);
+        tps_p2 = !!(tps_p3_p0 & BIT_14);
+        tps_p1 = !!(tps_p3_p0 & BIT_13);
+        tps_p0 = !!(tps_p3_p0 & BIT_12);
+        mdi_reverse = rtl8127_mac_ocp_read(tp, 0xD442);
+
+        if ((mdi_reverse & BIT_5) && tps_p3_p0 == 0xA000)
+                return;
+
+        if (!(mdi_reverse & BIT_5))
+                val = tps_p0 << 8 |
+                      tps_p1 << 9 |
+                      tps_p2 << 10 |
+                      tps_p3 << 11;
+        else
+                val = tps_p3 << 8 |
+                      tps_p2 << 9 |
+                      tps_p1 << 10 |
+                      tps_p0 << 11;
+
+        for (i=8; i<12; i++) {
+                rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, reg);
+                rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                                      0xA438,
+                                                      BIT(i),
+                                                      val & BIT(i));
+        }
+}
+
+static int rtl8127_vcd_test(struct rtl8127_private *tp)
+{
+        u16 val;
+        u32 wait_cnt;
+        int ret = -1;
+
+        rtl8127_mdi_swap(tp);
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA422, BIT(0));
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA422, 0x00F0);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA422, BIT(0));
+
+        wait_cnt = 0;
+        do {
+                mdelay(1);
+                val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA422);
+                wait_cnt++;
+        } while (!(val & BIT_15) && (wait_cnt < 5000));
+
+        if (wait_cnt == 5000)
+                goto exit;
+
+        ret = 0;
+
+exit:
+        return ret;
+}
+
+static void rtl8127_get_cp_len(struct rtl8127_private *tp,
+                               int cp_len[RTL8127_CP_NUM])
+{
+        int i;
+        u32 status;
+        int tmp_cp_len;
+
+        status = rtl8127_get_phy_status(tp);
+        if (status & LinkStatus) {
+                if (status & _10bps) {
+                        tmp_cp_len = -1;
+                } else if (status & (_100bps | _1000bpsF)) {
+                        rtl8127_mdio_write(tp, 0x1f, 0x0a88);
+                        tmp_cp_len = rtl8127_mdio_read(tp, 0x10);
+                } else if (status & _2500bpsF) {
+                        rtl8127_mdio_write(tp, 0x1f, 0x0acb);
+                        tmp_cp_len = rtl8127_mdio_read(tp, 0x15);
+                        tmp_cp_len >>= 2;
+                } else
+                        tmp_cp_len = 0;
+        } else
+                tmp_cp_len = 0;
+
+        if (tmp_cp_len > 0)
+                tmp_cp_len &= 0xff;
+        for (i=0; i<RTL8127_CP_NUM; i++)
+                cp_len[i] = tmp_cp_len;
+
+        rtl8127_mdio_write(tp, 0x1f, 0x0000);
+
+        for (i=0; i<RTL8127_CP_NUM; i++)
+                if (cp_len[i] > RTL8127_MAX_SUPPORT_CP_LEN)
+                        cp_len[i] = RTL8127_MAX_SUPPORT_CP_LEN;
+
+        return;
+}
+
+static int __rtl8127_get_cp_status(u16 val)
+{
+        switch (val) {
+        case 0x0060:
+                return rtl8127_cp_normal;
+        case 0x0048:
+                return rtl8127_cp_open;
+        case 0x0050:
+                return rtl8127_cp_short;
+        case 0x0042:
+        case 0x0044:
+                return rtl8127_cp_mismatch;
+        default:
+                return rtl8127_cp_normal;
+        }
+}
+
+static int _rtl8127_get_cp_status(struct rtl8127_private *tp, u8 pair_num)
+{
+        u16 val;
+        int cp_status = rtl8127_cp_unknown;
+
+        if (pair_num > 3)
+                goto exit;
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8027 + 4 * pair_num);
+        val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438);
+
+        cp_status = __rtl8127_get_cp_status(val);
+
+exit:
+        return cp_status;
+}
+
+static const char * rtl8127_get_cp_status_string(int cp_status)
+{
+        switch(cp_status) {
+        case rtl8127_cp_normal:
+                return "normal  ";
+        case rtl8127_cp_short:
+                return "short   ";
+        case rtl8127_cp_open:
+                return "open    ";
+        case rtl8127_cp_mismatch:
+                return "mismatch";
+        default:
+                return "unknown ";
+        }
+}
+
+static u16 rtl8127_get_cp_pp(struct rtl8127_private *tp, u8 pair_num)
+{
+        u16 pp = 0;
+
+        if (pair_num > 3)
+                goto exit;
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8029 + 4 * pair_num);
+        pp = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438);
+
+        pp &= 0x3fff;
+        pp /= 80;
+
+exit:
+        return pp;
+}
+
+static void rtl8127_get_cp_status(struct rtl8127_private *tp,
+                                  int cp_status[RTL8127_CP_NUM],
+                                  bool poe_mode)
+{
+        u32 status;
+        int i;
+
+        status = rtl8127_get_phy_status(tp);
+        if (status & LinkStatus && !(status & (_10bps | _100bps))) {
+                for (i=0; i<RTL8127_CP_NUM; i++)
+                        cp_status[i] = rtl8127_cp_normal;
+        } else {
+                /* cannot do vcd when link is on */
+                rtl8127_vcd_test(tp);
+
+                for (i=0; i<RTL8127_CP_NUM; i++)
+                        cp_status[i] = _rtl8127_get_cp_status(tp, i);
+        }
+
+        if (poe_mode) {
+                for (i=0; i<RTL8127_CP_NUM; i++) {
+                        if (cp_status[i] == rtl8127_cp_mismatch)
+                                cp_status[i] = rtl8127_cp_normal;
+                }
+        }
+}
+
+#ifdef ENABLE_R8127_PROCFS
+/****************************************************************************
+*   -----------------------------PROCFS STUFF-------------------------
+*****************************************************************************
+*/
+
+static struct proc_dir_entry *rtl8127_proc;
+static int proc_init_num = 0;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+static int proc_get_driver_variable(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        seq_puts(m, "\nDump Driver Variable\n");
+
+        rtnl_lock();
+
+        seq_puts(m, "Variable\tValue\n----------\t-----\n");
+        seq_printf(m, "MODULENAME\t%s\n", MODULENAME);
+        seq_printf(m, "driver version\t%s\n", RTL8127_VERSION);
+        seq_printf(m, "mcfg\t%d\n", tp->mcfg);
+        seq_printf(m, "chipset\t%d\n", tp->chipset);
+        seq_printf(m, "chipset_name\t%s\n", rtl_chip_info[tp->chipset].name);
+        seq_printf(m, "mtu\t%d\n", dev->mtu);
+        seq_printf(m, "NUM_RX_DESC\t0x%x\n", tp->rx_ring[0].num_rx_desc);
+        seq_printf(m, "cur_rx0\t0x%x\n", tp->rx_ring[0].cur_rx);
+        seq_printf(m, "dirty_rx0\t0x%x\n", tp->rx_ring[0].dirty_rx);
+        seq_printf(m, "cur_rx1\t0x%x\n", tp->rx_ring[1].cur_rx);
+        seq_printf(m, "dirty_rx1\t0x%x\n", tp->rx_ring[1].dirty_rx);
+        seq_printf(m, "cur_rx2\t0x%x\n", tp->rx_ring[2].cur_rx);
+        seq_printf(m, "dirty_rx2\t0x%x\n", tp->rx_ring[2].dirty_rx);
+        seq_printf(m, "cur_rx3\t0x%x\n", tp->rx_ring[3].cur_rx);
+        seq_printf(m, "dirty_rx3\t0x%x\n", tp->rx_ring[3].dirty_rx);
+        seq_printf(m, "NUM_TX_DESC\t0x%x\n", tp->tx_ring[0].num_tx_desc);
+        seq_printf(m, "cur_tx0\t0x%x\n", tp->tx_ring[0].cur_tx);
+        seq_printf(m, "dirty_tx0\t0x%x\n", tp->tx_ring[0].dirty_tx);
+        seq_printf(m, "cur_tx1\t0x%x\n", tp->tx_ring[1].cur_tx);
+        seq_printf(m, "dirty_tx1\t0x%x\n", tp->tx_ring[1].dirty_tx);
+        seq_printf(m, "rx_buf_sz\t0x%x\n", tp->rx_buf_sz);
+#ifdef ENABLE_PAGE_REUSE
+        seq_printf(m, "rx_buf_page_order\t0x%x\n", tp->rx_buf_page_order);
+        seq_printf(m, "rx_buf_page_size\t0x%x\n", tp->rx_buf_page_size);
+        seq_printf(m, "page_reuse_fail_cnt\t0x%x\n", tp->page_reuse_fail_cnt);
+#endif //ENABLE_PAGE_REUSE
+        seq_printf(m, "esd_flag\t0x%x\n", tp->esd_flag);
+        seq_printf(m, "pci_cfg_is_read\t0x%x\n", tp->pci_cfg_is_read);
+        seq_printf(m, "rtl8127_rx_config\t0x%x\n", tp->rtl8127_rx_config);
+        seq_printf(m, "cp_cmd\t0x%x\n", tp->cp_cmd);
+        seq_printf(m, "intr_mask\t0x%x\n", tp->intr_mask);
+        seq_printf(m, "timer_intr_mask\t0x%x\n", tp->timer_intr_mask);
+        seq_printf(m, "wol_enabled\t0x%x\n", tp->wol_enabled);
+        seq_printf(m, "wol_opts\t0x%x\n", tp->wol_opts);
+        seq_printf(m, "efuse_ver\t0x%x\n", tp->efuse_ver);
+        seq_printf(m, "eeprom_type\t0x%x\n", tp->eeprom_type);
+        seq_printf(m, "autoneg\t0x%x\n", tp->autoneg);
+        seq_printf(m, "duplex\t0x%x\n", tp->duplex);
+        seq_printf(m, "speed\t%d\n", tp->speed);
+        seq_printf(m, "advertising\t0x%llx\n", tp->advertising);
+        seq_printf(m, "eeprom_len\t0x%x\n", tp->eeprom_len);
+        seq_printf(m, "cur_page\t0x%x\n", tp->cur_page);
+        seq_printf(m, "features\t0x%x\n", tp->features);
+        seq_printf(m, "org_pci_offset_99\t0x%x\n", tp->org_pci_offset_99);
+        seq_printf(m, "org_pci_offset_180\t0x%x\n", tp->org_pci_offset_180);
+        seq_printf(m, "issue_offset_99_event\t0x%x\n", tp->issue_offset_99_event);
+        seq_printf(m, "org_pci_offset_80\t0x%x\n", tp->org_pci_offset_80);
+        seq_printf(m, "org_pci_offset_81\t0x%x\n", tp->org_pci_offset_81);
+        seq_printf(m, "use_timer_interrupt\t0x%x\n", tp->use_timer_interrupt);
+        seq_printf(m, "HwIcVerUnknown\t0x%x\n", tp->HwIcVerUnknown);
+        seq_printf(m, "NotWrRamCodeToMicroP\t0x%x\n", tp->NotWrRamCodeToMicroP);
+        seq_printf(m, "NotWrMcuPatchCode\t0x%x\n", tp->NotWrMcuPatchCode);
+        seq_printf(m, "HwHasWrRamCodeToMicroP\t0x%x\n", tp->HwHasWrRamCodeToMicroP);
+        seq_printf(m, "sw_ram_code_ver\t0x%x\n", tp->sw_ram_code_ver);
+        seq_printf(m, "hw_ram_code_ver\t0x%x\n", tp->hw_ram_code_ver);
+        seq_printf(m, "rtk_enable_diag\t0x%x\n", tp->rtk_enable_diag);
+        seq_printf(m, "ShortPacketSwChecksum\t0x%x\n", tp->ShortPacketSwChecksum);
+        seq_printf(m, "UseSwPaddingShortPkt\t0x%x\n", tp->UseSwPaddingShortPkt);
+        seq_printf(m, "RequireAdcBiasPatch\t0x%x\n", tp->RequireAdcBiasPatch);
+        seq_printf(m, "AdcBiasPatchIoffset\t0x%x\n", tp->AdcBiasPatchIoffset);
+        seq_printf(m, "RequireAdjustUpsTxLinkPulseTiming\t0x%x\n", tp->RequireAdjustUpsTxLinkPulseTiming);
+        seq_printf(m, "SwrCnt1msIni\t0x%x\n", tp->SwrCnt1msIni);
+        seq_printf(m, "HwSuppNowIsOobVer\t0x%x\n", tp->HwSuppNowIsOobVer);
+        seq_printf(m, "HwFiberModeVer\t0x%x\n", tp->HwFiberModeVer);
+        seq_printf(m, "HwFiberStat\t0x%x\n", tp->HwFiberStat);
+        seq_printf(m, "HwSwitchMdiToFiber\t0x%x\n", tp->HwSwitchMdiToFiber);
+        seq_printf(m, "NicCustLedValue\t0x%x\n", tp->NicCustLedValue);
+        seq_printf(m, "RequiredSecLanDonglePatch\t0x%x\n", tp->RequiredSecLanDonglePatch);
+        seq_printf(m, "HwSuppDashVer\t0x%x\n", tp->HwSuppDashVer);
+        seq_printf(m, "DASH\t0x%x\n", tp->DASH);
+        seq_printf(m, "DashFirmwareVersion\t0x%x\n", tp->DashFirmwareVersion);
+        seq_printf(m, "HwSuppKCPOffloadVer\t0x%x\n", tp->HwSuppKCPOffloadVer);
+        seq_printf(m, "speed_mode\t0x%x\n", speed_mode);
+        seq_printf(m, "duplex_mode\t0x%x\n", duplex_mode);
+        seq_printf(m, "autoneg_mode\t0x%x\n", autoneg_mode);
+        seq_printf(m, "aspm\t0x%x\n", aspm);
+        seq_printf(m, "s5wol\t0x%x\n", s5wol);
+        seq_printf(m, "s5_keep_curr_mac\t0x%x\n", s5_keep_curr_mac);
+        seq_printf(m, "eee_enable\t0x%x\n", tp->eee.eee_enabled);
+        seq_printf(m, "hwoptimize\t0x%lx\n", hwoptimize);
+        seq_printf(m, "proc_init_num\t0x%x\n", proc_init_num);
+        seq_printf(m, "s0_magic_packet\t0x%x\n", s0_magic_packet);
+        seq_printf(m, "disable_wol_support\t0x%x\n", disable_wol_support);
+        seq_printf(m, "enable_double_vlan\t0x%x\n", enable_double_vlan);
+        seq_printf(m, "eee_giga_lite\t0x%x\n", eee_giga_lite);
+        seq_printf(m, "HwSuppMagicPktVer\t0x%x\n", tp->HwSuppMagicPktVer);
+        seq_printf(m, "HwSuppLinkChgWakeUpVer\t0x%x\n", tp->HwSuppLinkChgWakeUpVer);
+        seq_printf(m, "HwSuppD0SpeedUpVer\t0x%x\n", tp->HwSuppD0SpeedUpVer);
+        seq_printf(m, "D0SpeedUpSpeed\t0x%x\n", tp->D0SpeedUpSpeed);
+        seq_printf(m, "HwSuppCheckPhyDisableModeVer\t0x%x\n", tp->HwSuppCheckPhyDisableModeVer);
+        seq_printf(m, "HwPkgDet\t0x%x\n", tp->HwPkgDet);
+        seq_printf(m, "HwSuppTxNoCloseVer\t0x%x\n", tp->HwSuppTxNoCloseVer);
+        seq_printf(m, "EnableTxNoClose\t0x%x\n", tp->EnableTxNoClose);
+        seq_printf(m, "NextHwDesCloPtr0\t0x%x\n", tp->tx_ring[0].NextHwDesCloPtr);
+        seq_printf(m, "BeginHwDesCloPtr0\t0x%x\n", tp->tx_ring[0].BeginHwDesCloPtr);
+        seq_printf(m, "hw_clo_ptr_reg0\t0x%x\n", rtl8127_get_hw_clo_ptr(&tp->tx_ring[0]));
+        seq_printf(m, "sw_tail_ptr_reg0\t0x%x\n", rtl8127_get_sw_tail_ptr(&tp->tx_ring[0]));
+        seq_printf(m, "NextHwDesCloPtr1\t0x%x\n", tp->tx_ring[1].NextHwDesCloPtr);
+        seq_printf(m, "BeginHwDesCloPtr1\t0x%x\n", tp->tx_ring[1].BeginHwDesCloPtr);
+        seq_printf(m, "hw_clo_ptr_reg1\t0x%x\n", rtl8127_get_hw_clo_ptr(&tp->tx_ring[1]));
+        seq_printf(m, "sw_tail_ptr_reg1\t0x%x\n", rtl8127_get_sw_tail_ptr(&tp->tx_ring[1]));
+        seq_printf(m, "InitRxDescType\t0x%x\n", tp->InitRxDescType);
+        seq_printf(m, "RxDescLength\t0x%x\n", tp->RxDescLength);
+        seq_printf(m, "num_rx_rings\t0x%x\n", tp->num_rx_rings);
+        seq_printf(m, "num_tx_rings\t0x%x\n", tp->num_tx_rings);
+        seq_printf(m, "tot_rx_rings\t0x%x\n", rtl8127_tot_rx_rings(tp));
+        seq_printf(m, "tot_tx_rings\t0x%x\n", rtl8127_tot_tx_rings(tp));
+        seq_printf(m, "HwSuppNumRxQueues\t0x%x\n", tp->HwSuppNumRxQueues);
+        seq_printf(m, "HwSuppNumTxQueues\t0x%x\n", tp->HwSuppNumTxQueues);
+        seq_printf(m, "EnableRss\t0x%x\n", tp->EnableRss);
+        seq_printf(m, "EnablePtp\t0x%x\n", tp->EnablePtp);
+        seq_printf(m, "min_irq_nvecs\t0x%x\n", tp->min_irq_nvecs);
+        seq_printf(m, "irq_nvecs\t0x%x\n", tp->irq_nvecs);
+        seq_printf(m, "hw_supp_irq_nvecs\t0x%x\n", tp->hw_supp_irq_nvecs);
+        seq_printf(m, "ring_lib_enabled\t0x%x\n", tp->ring_lib_enabled);
+        seq_printf(m, "HwSuppIsrVer\t0x%x\n", tp->HwSuppIsrVer);
+        seq_printf(m, "HwCurrIsrVer\t0x%x\n", tp->HwCurrIsrVer);
+        seq_printf(m, "HwSuppMacMcuVer\t0x%x\n", tp->HwSuppMacMcuVer);
+        seq_printf(m, "MacMcuPageSize\t0x%x\n", tp->MacMcuPageSize);
+        seq_printf(m, "hw_mcu_patch_code_ver\t0x%llx\n", tp->hw_mcu_patch_code_ver);
+        seq_printf(m, "bin_mcu_patch_code_ver\t0x%llx\n", tp->bin_mcu_patch_code_ver);
+#ifdef ENABLE_PTP_SUPPORT
+        seq_printf(m, "tx_hwtstamp_timeouts\t0x%x\n", tp->tx_hwtstamp_timeouts);
+        seq_printf(m, "tx_hwtstamp_skipped\t0x%x\n", tp->tx_hwtstamp_skipped);
+#endif
+        seq_printf(m, "random_mac\t0x%x\n", tp->random_mac);
+        seq_printf(m, "org_mac_addr\t%pM\n", tp->org_mac_addr);
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13)
+        seq_printf(m, "perm_addr\t%pM\n", dev->perm_addr);
+#endif
+        seq_printf(m, "dev_addr\t%pM\n", dev->dev_addr);
+
+        rtnl_unlock();
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_get_tally_counter(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct rtl8127_counters *counters;
+        dma_addr_t paddr;
+
+        seq_puts(m, "\nDump Tally Counter\n");
+
+        rtnl_lock();
+
+        counters = tp->tally_vaddr;
+        paddr = tp->tally_paddr;
+        if (!counters) {
+                seq_puts(m, "\nDump Tally Counter Fail\n");
+                goto out_unlock;
+        }
+
+        rtl8127_dump_tally_counter(tp, paddr);
+
+        seq_puts(m, "Statistics\tValue\n----------\t-----\n");
+        seq_printf(m, "tx_packets\t%lld\n", le64_to_cpu(counters->tx_packets));
+        seq_printf(m, "rx_packets\t%lld\n", le64_to_cpu(counters->rx_packets));
+        seq_printf(m, "tx_errors\t%lld\n", le64_to_cpu(counters->tx_errors));
+        seq_printf(m, "rx_errors\t%d\n", le32_to_cpu(counters->rx_errors));
+        seq_printf(m, "rx_missed\t%d\n", le16_to_cpu(counters->rx_missed));
+        seq_printf(m, "align_errors\t%d\n", le16_to_cpu(counters->align_errors));
+        seq_printf(m, "tx_one_collision\t%d\n", le32_to_cpu(counters->tx_one_collision));
+        seq_printf(m, "tx_multi_collision\t%d\n", le32_to_cpu(counters->tx_multi_collision));
+        seq_printf(m, "rx_unicast\t%lld\n", le64_to_cpu(counters->rx_unicast));
+        seq_printf(m, "rx_broadcast\t%lld\n", le64_to_cpu(counters->rx_broadcast));
+        seq_printf(m, "rx_multicast\t%d\n", le32_to_cpu(counters->rx_multicast));
+        seq_printf(m, "tx_aborted\t%d\n", le16_to_cpu(counters->tx_aborted));
+        seq_printf(m, "tx_underrun\t%d\n", le16_to_cpu(counters->tx_underrun));
+
+        seq_printf(m, "tx_octets\t%lld\n", le64_to_cpu(counters->tx_octets));
+        seq_printf(m, "rx_octets\t%lld\n", le64_to_cpu(counters->rx_octets));
+        seq_printf(m, "rx_multicast64\t%lld\n", le64_to_cpu(counters->rx_multicast64));
+        seq_printf(m, "tx_unicast64\t%lld\n", le64_to_cpu(counters->tx_unicast64));
+        seq_printf(m, "tx_broadcast64\t%lld\n", le64_to_cpu(counters->tx_broadcast64));
+        seq_printf(m, "tx_multicast64\t%lld\n", le64_to_cpu(counters->tx_multicast64));
+        seq_printf(m, "tx_pause_on\t%d\n", le32_to_cpu(counters->tx_pause_on));
+        seq_printf(m, "tx_pause_off\t%d\n", le32_to_cpu(counters->tx_pause_off));
+        seq_printf(m, "tx_pause_all\t%d\n", le32_to_cpu(counters->tx_pause_all));
+        seq_printf(m, "tx_deferred\t%d\n", le32_to_cpu(counters->tx_deferred));
+        seq_printf(m, "tx_late_collision\t%d\n", le32_to_cpu(counters->tx_late_collision));
+        seq_printf(m, "tx_all_collision\t%d\n", le32_to_cpu(counters->tx_all_collision));
+        seq_printf(m, "tx_aborted32\t%d\n", le32_to_cpu(counters->tx_aborted32));
+        seq_printf(m, "align_errors32\t%d\n", le32_to_cpu(counters->align_errors32));
+        seq_printf(m, "rx_frame_too_long\t%d\n", le32_to_cpu(counters->rx_frame_too_long));
+        seq_printf(m, "rx_runt\t%d\n", le32_to_cpu(counters->rx_runt));
+        seq_printf(m, "rx_pause_on\t%d\n", le32_to_cpu(counters->rx_pause_on));
+        seq_printf(m, "rx_pause_off\t%d\n", le32_to_cpu(counters->rx_pause_off));
+        seq_printf(m, "rx_pause_all\t%d\n", le32_to_cpu(counters->rx_pause_all));
+        seq_printf(m, "rx_unknown_opcode\t%d\n", le32_to_cpu(counters->rx_unknown_opcode));
+        seq_printf(m, "rx_mac_error\t%d\n", le32_to_cpu(counters->rx_mac_error));
+        seq_printf(m, "tx_underrun32\t%d\n", le32_to_cpu(counters->tx_underrun32));
+        seq_printf(m, "rx_mac_missed\t%d\n", le32_to_cpu(counters->rx_mac_missed));
+        seq_printf(m, "rx_tcam_dropped\t%d\n", le32_to_cpu(counters->rx_tcam_dropped));
+        seq_printf(m, "tdu\t%d\n", le32_to_cpu(counters->tdu));
+        seq_printf(m, "rdu\t%d\n", le32_to_cpu(counters->rdu));
+
+        seq_putc(m, '\n');
+
+out_unlock:
+        rtnl_unlock();
+
+        return 0;
+}
+
+static int proc_get_registers(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        int i, n, max = R8127_MAC_REGS_SIZE;
+        u8 byte_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        void __iomem *ioaddr = tp->mmio_addr;
+
+        seq_puts(m, "\nDump MAC Registers\n");
+        seq_puts(m, "Offset\tValue\n------\t-----\n");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                seq_printf(m, "\n0x%04x:\t", n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        seq_printf(m, "%02x ", byte_rd);
+                }
+        }
+
+        max = 0xB00;
+        for (n = 0xA00; n < max;) {
+                seq_printf(m, "\n0x%04x:\t", n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        seq_printf(m, "%02x ", byte_rd);
+                }
+        }
+
+        max = 0xD40;
+        for (n = 0xD00; n < max;) {
+                seq_printf(m, "\n0x%04x:\t", n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        seq_printf(m, "%02x ", byte_rd);
+                }
+        }
+
+        max = 0x2840;
+        for (n = 0x2800; n < max;) {
+                seq_printf(m, "\n0x%04x:\t", n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        seq_printf(m, "%02x ", byte_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_get_all_registers(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        int i, n, max;
+        u8 byte_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        void __iomem *ioaddr = tp->mmio_addr;
+        struct pci_dev *pdev = tp->pci_dev;
+
+        seq_puts(m, "\nDump All MAC Registers\n");
+        seq_puts(m, "Offset\tValue\n------\t-----\n");
+
+        rtnl_lock();
+
+        max = pci_resource_len(pdev, 2);
+
+        for (n = 0; n < max;) {
+                seq_printf(m, "\n0x%04x:\t", n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        seq_printf(m, "%02x ", byte_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        seq_printf(m, "\nTotal length:0x%X", max);
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_get_pcie_phy(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        int i, n, max = R8127_EPHY_REGS_SIZE/2;
+        u16 word_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        seq_puts(m, "\nDump PCIE PHY\n");
+        seq_puts(m, "\nOffset\tValue\n------\t-----\n ");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                seq_printf(m, "\n0x%02x:\t", n);
+
+                for (i = 0; i < 8 && n < max; i++, n++) {
+                        word_rd = rtl8127_ephy_read(tp, n);
+                        seq_printf(m, "%04x ", word_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_get_eth_phy(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        int i, n, max = R8127_PHY_REGS_SIZE/2;
+        unsigned long flags;
+        u16 word_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        seq_puts(m, "\nDump Ethernet PHY\n");
+        seq_puts(m, "\nOffset\tValue\n------\t-----\n ");
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        seq_puts(m, "\n####################page 0##################\n ");
+        rtl8127_mdio_write(tp, 0x1f, 0x0000);
+        for (n = 0; n < max;) {
+                seq_printf(m, "\n0x%02x:\t", n);
+
+                for (i = 0; i < 8 && n < max; i++, n++) {
+                        word_rd = rtl8127_mdio_read(tp, n);
+                        seq_printf(m, "%04x ", word_rd);
+                }
+        }
+
+        seq_puts(m, "\n####################extra reg##################\n ");
+        n = 0xA400;
+        seq_printf(m, "\n0x%02x:\t", n);
+        for (i = 0; i < 8; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                seq_printf(m, "%04x ", word_rd);
+        }
+
+        n = 0xA410;
+        seq_printf(m, "\n0x%02x:\t", n);
+        for (i = 0; i < 3; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                seq_printf(m, "%04x ", word_rd);
+        }
+
+        n = 0xA434;
+        seq_printf(m, "\n0x%02x:\t", n);
+        word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+        seq_printf(m, "%04x ", word_rd);
+
+        n = 0xA5D0;
+        seq_printf(m, "\n0x%02x:\t", n);
+        for (i = 0; i < 4; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                seq_printf(m, "%04x ", word_rd);
+        }
+
+        n = 0xA61A;
+        seq_printf(m, "\n0x%02x:\t", n);
+        word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+        seq_printf(m, "%04x ", word_rd);
+
+        n = 0xA6D0;
+        seq_printf(m, "\n0x%02x:\t", n);
+        for (i = 0; i < 3; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                seq_printf(m, "%04x ", word_rd);
+        }
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_get_extended_registers(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        int i, n, max = R8127_ERI_REGS_SIZE;
+        u32 dword_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        seq_puts(m, "\nDump Extended Registers\n");
+        seq_puts(m, "\nOffset\tValue\n------\t-----\n ");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                seq_printf(m, "\n0x%02x:\t", n);
+
+                for (i = 0; i < 4 && n < max; i++, n+=4) {
+                        dword_rd = rtl8127_eri_read(tp, n, 4, ERIAR_ExGMAC);
+                        seq_printf(m, "%08x ", dword_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_get_pci_registers(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        int i, n, max = R8127_PCI_REGS_SIZE;
+        u32 dword_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        seq_puts(m, "\nDump PCI Registers\n");
+        seq_puts(m, "\nOffset\tValue\n------\t-----\n ");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                seq_printf(m, "\n0x%03x:\t", n);
+
+                for (i = 0; i < 4 && n < max; i++, n+=4) {
+                        pci_read_config_dword(tp->pci_dev, n, &dword_rd);
+                        seq_printf(m, "%08x ", dword_rd);
+                }
+        }
+
+        n = 0x110;
+        pci_read_config_dword(tp->pci_dev, n, &dword_rd);
+        seq_printf(m, "\n0x%03x:\t%08x ", n, dword_rd);
+        n = 0x70c;
+        pci_read_config_dword(tp->pci_dev, n, &dword_rd);
+        seq_printf(m, "\n0x%03x:\t%08x ", n, dword_rd);
+
+        rtnl_unlock();
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_get_temperature(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u16 ts_digout, tj, fah;
+        unsigned long flags;
+
+        seq_puts(m, "\nChip Temperature\n");
+
+        rtnl_lock();
+
+        if (!rtl8127_sysfs_testmode_on(tp)) {
+                seq_puts(m, "\nPlease turn on ""/sys/class/net/<iface>/rtk_adv/testmode"".\n\n");
+                rtnl_unlock();
+                return 0;
+        }
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        netif_testing_on(dev);
+        ts_digout = rtl8127_read_thermal_sensor(tp);
+        netif_testing_off(dev);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        rtnl_unlock();
+
+        if (ts_digout <= 512) {
+                tj = ts_digout / 2;
+                seq_printf(m, "Cel:%d\n", tj);
+                fah = tj * (9/5) + 32;
+                seq_printf(m, "Fah:%d\n", fah);
+        } else {
+                tj = (512 - (ts_digout - 512)) / 2;
+                seq_printf(m, "Cel:-%d\n", tj);
+                fah = tj * (9/5) + 32;
+                seq_printf(m, "Fah:-%d\n", fah);
+        }
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int _proc_get_cable_info(struct seq_file *m, void *v, bool poe_mode)
+{
+        int i;
+        u32 status;
+        int cp_status[RTL8127_CP_NUM];
+        int cp_len[RTL8127_CP_NUM] = {0};
+        struct net_device *dev = m->private;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        const char *pair_str[RTL8127_CP_NUM] = {"1-2", "3-6", "4-5", "7-8"};
+        unsigned long flags;
+        int ret;
+
+        switch (tp->mcfg) {
+        default:
+                ret = -EOPNOTSUPP;
+                goto error_out;
+        }
+
+        rtnl_lock();
+
+        if (!rtl8127_sysfs_testmode_on(tp)) {
+                seq_puts(m, "\nPlease turn on ""/sys/class/net/<iface>/rtk_adv/testmode"".\n\n");
+                ret = 0;
+                goto error_unlock;
+        }
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        if (rtl8127_mdio_read(tp, MII_BMCR) & BMCR_PDOWN) {
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                ret = -EIO;
+                goto error_unlock;
+        }
+
+        netif_testing_on(dev);
+
+        status = rtl8127_get_phy_status(tp);
+        if (status & LinkStatus)
+                seq_printf(m, "\nlink speed:%d",
+                           rtl8127_convert_link_speed(status));
+        else
+                seq_puts(m, "\nlink status:off");
+
+        rtl8127_get_cp_len(tp, cp_len);
+
+        rtl8127_get_cp_status(tp, cp_status, poe_mode);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        seq_puts(m, "\npair\tlength\tstatus   \tpp\n");
+
+        for (i=0; i<RTL8127_CP_NUM; i++) {
+                if (cp_len[i] < 0)
+                        seq_printf(m, "%s\t%s\t%s\t",
+                                   pair_str[i], "none",
+                                   rtl8127_get_cp_status_string(cp_status[i]));
+                else
+                        seq_printf(m, "%s\t%d\t%s\t",
+                                   pair_str[i], cp_len[i],
+                                   rtl8127_get_cp_status_string(cp_status[i]));
+                if (cp_status[i] == rtl8127_cp_normal)
+                        seq_printf(m, "none\n");
+                else
+                        seq_printf(m, "%dm\n", rtl8127_get_cp_pp(tp, i));
+        }
+
+        netif_testing_off(dev);
+
+        seq_putc(m, '\n');
+
+        ret = 0;
+
+error_unlock:
+        rtnl_unlock();
+
+error_out:
+        return ret;
+}
+
+static int proc_get_cable_info(struct seq_file *m, void *v)
+{
+        return _proc_get_cable_info(m, v, 0);
+}
+
+static int proc_get_poe_cable_info(struct seq_file *m, void *v)
+{
+        return _proc_get_cable_info(m, v, 1);
+}
+
+static void _proc_dump_desc(struct seq_file *m, void *desc_base, u32 alloc_size)
+{
+        u32 *pdword;
+        int i;
+
+        if (desc_base == NULL ||
+            alloc_size == 0)
+                return;
+
+        pdword = (u32*)desc_base;
+        for (i=0; i<(alloc_size/4); i++) {
+                if (!(i % 4))
+                        seq_printf(m, "\n%04x ", i);
+                seq_printf(m, "%08x ", pdword[i]);
+        }
+
+        seq_putc(m, '\n');
+        return;
+}
+
+static int proc_dump_rx_desc(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        rtnl_lock();
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                struct rtl8127_rx_ring *ring = &tp->rx_ring[i];
+
+                if (!ring)
+                        continue;
+
+                seq_printf(m, "\ndump rx %d desc:%d\n", i, ring->num_rx_desc);
+
+                _proc_dump_desc(m, (void*)ring->RxDescArray, ring->RxDescAllocSize);
+        }
+
+#ifdef ENABLE_LIB_SUPPORT
+        if (rtl8127_num_lib_rx_rings(tp) > 0) {
+                for (i = 0; i < tp->HwSuppNumRxQueues; i++) {
+                        struct rtl8127_ring *lib_ring = &tp->lib_rx_ring[i];
+                        if (lib_ring->enabled) {
+                                seq_printf(m, "\ndump lib rx %d desc:%d\n", i,
+                                           lib_ring->ring_size);
+                                _proc_dump_desc(m, (void*)lib_ring->desc_addr,
+                                                lib_ring->desc_size);
+                        }
+                }
+        }
+#endif //ENABLE_LIB_SUPPORT
+
+        rtnl_unlock();
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_dump_tx_desc(struct seq_file *m, void *v)
+{
+        struct net_device *dev = m->private;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        rtnl_lock();
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                struct rtl8127_tx_ring *ring = &tp->tx_ring[i];
+
+                if (!ring)
+                        continue;
+
+                seq_printf(m, "\ndump tx %d desc:%d\n", i, ring->num_tx_desc);
+
+                _proc_dump_desc(m, (void*)ring->TxDescArray, ring->TxDescAllocSize);
+        }
+
+#ifdef ENABLE_LIB_SUPPORT
+        if (rtl8127_num_lib_tx_rings(tp) > 0) {
+                for (i = 0; i < tp->HwSuppNumTxQueues; i++) {
+                        struct rtl8127_ring *lib_ring = &tp->lib_tx_ring[i];
+                        if (lib_ring->enabled) {
+                                seq_printf(m, "\ndump lib tx %d desc:%d\n", i,
+                                           lib_ring->ring_size);
+                                _proc_dump_desc(m, (void*)lib_ring->desc_addr,
+                                                lib_ring->desc_size);
+                        }
+                }
+        }
+#endif //ENABLE_LIB_SUPPORT
+
+        rtnl_unlock();
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+static int proc_dump_msix_tbl(struct seq_file *m, void *v)
+{
+        int i, j;
+        void __iomem *ioaddr;
+        struct net_device *dev = m->private;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        /* ioremap MMIO region */
+        ioaddr = ioremap(pci_resource_start(tp->pci_dev, 4), pci_resource_len(tp->pci_dev, 4));
+        if (!ioaddr)
+                return -EFAULT;
+
+        rtnl_lock();
+
+        seq_printf(m, "\ndump MSI-X Table. Total Entry %d. \n", tp->hw_supp_irq_nvecs);
+
+        for (i=0; i<tp->hw_supp_irq_nvecs; i++) {
+                seq_printf(m, "\n%04x ", i);
+                for (j=0; j<4; j++)
+                        seq_printf(m, "%08x ",
+                                   readl(ioaddr + i*0x10 + 4*j));
+        }
+
+        rtnl_unlock();
+
+        iounmap(ioaddr);
+
+        seq_putc(m, '\n');
+        return 0;
+}
+
+#else //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+
+static int proc_get_driver_variable(char *page, char **start,
+                                    off_t offset, int count,
+                                    int *eof, void *data)
+{
+        struct net_device *dev = data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump Driver Driver\n");
+
+        rtnl_lock();
+
+        len += snprintf(page + len, count - len,
+                        "Variable\tValue\n----------\t-----\n");
+
+        len += snprintf(page + len, count - len,
+                        "MODULENAME\t%s\n"
+                        "driver version\t%s\n"
+                        "mcfg\t%d\n"
+                        "chipset\t%d\n"
+                        "chipset_name\t%s\n"
+                        "mtu\t%d\n"
+                        "NUM_RX_DESC\t0x%x\n"
+                        "cur_rx0\t0x%x\n"
+                        "dirty_rx0\t0x%x\n"
+                        "cur_rx1\t0x%x\n"
+                        "dirty_rx1\t0x%x\n"
+                        "cur_rx2\t0x%x\n"
+                        "dirty_rx2\t0x%x\n"
+                        "cur_rx3\t0x%x\n"
+                        "dirty_rx3\t0x%x\n"
+                        "NUM_TX_DESC\t0x%x\n"
+                        "cur_tx0\t0x%x\n"
+                        "dirty_tx0\t0x%x\n"
+                        "cur_tx1\t0x%x\n"
+                        "dirty_tx1\t0x%x\n"
+                        "rx_buf_sz\t0x%x\n"
+#ifdef ENABLE_PAGE_REUSE
+                        "rx_buf_page_order\t0x%x\n"
+                        "rx_buf_page_size\t0x%x\n"
+                        "page_reuse_fail_cnt\t0x%x\n"
+#endif //ENABLE_PAGE_REUSE
+                        "esd_flag\t0x%x\n"
+                        "pci_cfg_is_read\t0x%x\n"
+                        "rtl8127_rx_config\t0x%x\n"
+                        "cp_cmd\t0x%x\n"
+                        "intr_mask\t0x%x\n"
+                        "timer_intr_mask\t0x%x\n"
+                        "wol_enabled\t0x%x\n"
+                        "wol_opts\t0x%x\n"
+                        "efuse_ver\t0x%x\n"
+                        "eeprom_type\t0x%x\n"
+                        "autoneg\t0x%x\n"
+                        "duplex\t0x%x\n"
+                        "speed\t%d\n"
+                        "advertising\t0x%llx\n"
+                        "eeprom_len\t0x%x\n"
+                        "cur_page\t0x%x\n"
+                        "features\t0x%x\n"
+                        "org_pci_offset_99\t0x%x\n"
+                        "org_pci_offset_180\t0x%x\n"
+                        "issue_offset_99_event\t0x%x\n"
+                        "org_pci_offset_80\t0x%x\n"
+                        "org_pci_offset_81\t0x%x\n"
+                        "use_timer_interrupt\t0x%x\n"
+                        "HwIcVerUnknown\t0x%x\n"
+                        "NotWrRamCodeToMicroP\t0x%x\n"
+                        "NotWrMcuPatchCode\t0x%x\n"
+                        "HwHasWrRamCodeToMicroP\t0x%x\n"
+                        "sw_ram_code_ver\t0x%x\n"
+                        "hw_ram_code_ver\t0x%x\n"
+                        "rtk_enable_diag\t0x%x\n"
+                        "ShortPacketSwChecksum\t0x%x\n"
+                        "UseSwPaddingShortPkt\t0x%x\n"
+                        "RequireAdcBiasPatch\t0x%x\n"
+                        "AdcBiasPatchIoffset\t0x%x\n"
+                        "RequireAdjustUpsTxLinkPulseTiming\t0x%x\n"
+                        "SwrCnt1msIni\t0x%x\n"
+                        "HwSuppNowIsOobVer\t0x%x\n"
+                        "HwFiberModeVer\t0x%x\n"
+                        "HwFiberStat\t0x%x\n"
+                        "HwSwitchMdiToFiber\t0x%x\n"
+                        "NicCustLedValue\t0x%x\n"
+                        "RequiredSecLanDonglePatch\t0x%x\n"
+                        "HwSuppDashVer\t0x%x\n"
+                        "DASH\t0x%x\n"
+                        "DashFirmwareVersion\t0x%x\n"
+                        "HwSuppKCPOffloadVer\t0x%x\n"
+                        "speed_mode\t0x%x\n"
+                        "duplex_mode\t0x%x\n"
+                        "autoneg_mode\t0x%x\n"
+                        "aspm\t0x%x\n"
+                        "s5wol\t0x%x\n"
+                        "s5_keep_curr_mac\t0x%x\n"
+                        "eee_enable\t0x%x\n"
+                        "hwoptimize\t0x%lx\n"
+                        "proc_init_num\t0x%x\n"
+                        "s0_magic_packet\t0x%x\n"
+                        "disable_wol_support\t0x%x\n"
+                        "enable_double_vlan\t0x%x\n"
+                        "eee_giga_lite\t0x%x\n"
+                        "HwSuppMagicPktVer\t0x%x\n"
+                        "HwSuppLinkChgWakeUpVer\t0x%x\n"
+                        "HwSuppD0SpeedUpVer\t0x%x\n"
+                        "D0SpeedUpSpeed\t0x%x\n"
+                        "HwSuppCheckPhyDisableModeVer\t0x%x\n"
+                        "HwPkgDet\t0x%x\n"
+                        "HwSuppTxNoCloseVer\t0x%x\n"
+                        "EnableTxNoClose\t0x%x\n"
+                        "NextHwDesCloPtr0\t0x%x\n"
+                        "BeginHwDesCloPtr0\t0x%x\n"
+                        "hw_clo_ptr_reg0\t0x%x\n"
+                        "sw_tail_ptr_reg0\t0x%x\n"
+                        "NextHwDesCloPtr1\t0x%x\n"
+                        "BeginHwDesCloPtr1\t0x%x\n"
+                        "hw_clo_ptr_reg1\t0x%x\n"
+                        "sw_tail_ptr_reg1\t0x%x\n"
+                        "InitRxDescType\t0x%x\n"
+                        "RxDescLength\t0x%x\n"
+                        "num_rx_rings\t0x%x\n"
+                        "num_tx_rings\t0x%x\n"
+                        "tot_rx_rings\t0x%x\n"
+                        "tot_tx_rings\t0x%x\n"
+                        "HwSuppNumRxQueues\t0x%x\n"
+                        "HwSuppNumTxQueues\t0x%x\n"
+                        "EnableRss\t0x%x\n"
+                        "EnablePtp\t0x%x\n"
+                        "min_irq_nvecs\t0x%x\n"
+                        "irq_nvecs\t0x%x\n"
+                        "hw_supp_irq_nvecs\t0x%x\n"
+                        "ring_lib_enabled\t0x%x\n"
+                        "HwSuppIsrVer\t0x%x\n"
+                        "HwCurrIsrVer\t0x%x\n"
+                        "HwSuppMacMcuVer\t0x%x\n"
+                        "MacMcuPageSize\t0x%x\n"
+                        "hw_mcu_patch_code_ver\t0x%llx\n"
+                        "bin_mcu_patch_code_ver\t0x%llx\n"
+#ifdef ENABLE_PTP_SUPPORT
+                        "tx_hwtstamp_timeouts\t0x%x\n"
+                        "tx_hwtstamp_skipped\t0x%x\n"
+#endif
+                        "random_mac\t0x%x\n"
+                        "org_mac_addr\t%pM\n"
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13)
+                        "perm_addr\t%pM\n"
+#endif
+                        "dev_addr\t%pM\n",
+                        MODULENAME,
+                        RTL8127_VERSION,
+                        tp->mcfg,
+                        tp->chipset,
+                        rtl_chip_info[tp->chipset].name,
+                        dev->mtu,
+                        tp->rx_ring[0].num_rx_desc,
+                        tp->rx_ring[0].cur_rx,
+                        tp->rx_ring[0].dirty_rx,
+                        tp->rx_ring[1].cur_rx,
+                        tp->rx_ring[1].dirty_rx,
+                        tp->rx_ring[2].cur_rx,
+                        tp->rx_ring[2].dirty_rx,
+                        tp->rx_ring[3].cur_rx,
+                        tp->rx_ring[3].dirty_rx,
+                        tp->tx_ring[0].num_tx_desc,
+                        tp->tx_ring[0].cur_tx,
+                        tp->tx_ring[0].dirty_tx,
+                        tp->tx_ring[1].cur_tx,
+                        tp->tx_ring[1].dirty_tx,
+                        tp->rx_buf_sz,
+#ifdef ENABLE_PAGE_REUSE
+                        tp->rx_buf_page_order,
+                        tp->rx_buf_page_size,
+                        tp->page_reuse_fail_cnt,
+#endif //ENABLE_PAGE_REUSE
+                        tp->esd_flag,
+                        tp->pci_cfg_is_read,
+                        tp->rtl8127_rx_config,
+                        tp->cp_cmd,
+                        tp->intr_mask,
+                        tp->timer_intr_mask,
+                        tp->wol_enabled,
+                        tp->wol_opts,
+                        tp->efuse_ver,
+                        tp->eeprom_type,
+                        tp->autoneg,
+                        tp->duplex,
+                        tp->speed,
+                        tp->advertising,
+                        tp->eeprom_len,
+                        tp->cur_page,
+                        tp->features,
+                        tp->org_pci_offset_99,
+                        tp->org_pci_offset_180,
+                        tp->issue_offset_99_event,
+                        tp->org_pci_offset_80,
+                        tp->org_pci_offset_81,
+                        tp->use_timer_interrupt,
+                        tp->HwIcVerUnknown,
+                        tp->NotWrRamCodeToMicroP,
+                        tp->NotWrMcuPatchCode,
+                        tp->HwHasWrRamCodeToMicroP,
+                        tp->sw_ram_code_ver,
+                        tp->hw_ram_code_ver,
+                        tp->rtk_enable_diag,
+                        tp->ShortPacketSwChecksum,
+                        tp->UseSwPaddingShortPkt,
+                        tp->RequireAdcBiasPatch,
+                        tp->AdcBiasPatchIoffset,
+                        tp->RequireAdjustUpsTxLinkPulseTiming,
+                        tp->SwrCnt1msIni,
+                        tp->HwSuppNowIsOobVer,
+                        tp->HwFiberModeVer,
+                        tp->HwFiberStat,
+                        tp->HwSwitchMdiToFiber,
+                        tp->NicCustLedValue,
+                        tp->RequiredSecLanDonglePatch,
+                        tp->HwSuppDashVer,
+                        tp->DASH,
+                        tp->DashFirmwareVersion,
+                        tp->HwSuppKCPOffloadVer,
+                        speed_mode,
+                        duplex_mode,
+                        autoneg_mode,
+                        aspm,
+                        s5wol,
+                        s5_keep_curr_mac,
+                        tp->eee.eee_enabled,
+                        hwoptimize,
+                        proc_init_num,
+                        s0_magic_packet,
+                        disable_wol_support,
+                        enable_double_vlan,
+                        eee_giga_lite,
+                        tp->HwSuppMagicPktVer,
+                        tp->HwSuppLinkChgWakeUpVer,
+                        tp->HwSuppD0SpeedUpVer,
+                        tp->D0SpeedUpSpeed,
+                        tp->HwSuppCheckPhyDisableModeVer,
+                        tp->HwPkgDet,
+                        tp->HwSuppTxNoCloseVer,
+                        tp->EnableTxNoClose,
+                        tp->tx_ring[0].NextHwDesCloPtr,
+                        tp->tx_ring[0].BeginHwDesCloPtr,
+                        rtl8127_get_hw_clo_ptr(&tp->tx_ring[0]),
+                        rtl8127_get_sw_tail_ptr(&tp->tx_ring[0]),
+                        tp->tx_ring[1].NextHwDesCloPtr,
+                        tp->tx_ring[1].BeginHwDesCloPtr,
+                        rtl8127_get_hw_clo_ptr(&tp->tx_ring[1]),
+                        rtl8127_get_sw_tail_ptr(&tp->tx_ring[1]),
+                        tp->InitRxDescType,
+                        tp->RxDescLength,
+                        tp->num_rx_rings,
+                        tp->num_tx_rings,
+                        rtl8127_tot_rx_rings(tp),
+                        rtl8127_tot_tx_rings(tp),
+                        tp->HwSuppNumRxQueues,
+                        tp->HwSuppNumTxQueues,
+                        tp->EnableRss,
+                        tp->EnablePtp,
+                        tp->min_irq_nvecs,
+                        tp->irq_nvecs,
+                        tp->hw_supp_irq_nvecs,
+                        tp->ring_lib_enabled,
+                        tp->HwSuppIsrVer,
+                        tp->HwCurrIsrVer,
+                        tp->HwSuppMacMcuVer,
+                        tp->MacMcuPageSize,
+                        tp->hw_mcu_patch_code_ver,
+                        tp->bin_mcu_patch_code_ver,
+#ifdef ENABLE_PTP_SUPPORT
+                        tp->tx_hwtstamp_timeouts,
+                        tp->tx_hwtstamp_skipped,
+#endif
+                        tp->random_mac,
+                        tp->org_mac_addr,
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13)
+                        dev->perm_addr,
+#endif
+                        dev->dev_addr);
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_tally_counter(char *page, char **start,
+                                  off_t offset, int count,
+                                  int *eof, void *data)
+{
+        struct net_device *dev = data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct rtl8127_counters *counters;
+        dma_addr_t paddr;
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump Tally Counter\n");
+
+        rtnl_lock();
+
+        counters = tp->tally_vaddr;
+        paddr = tp->tally_paddr;
+        if (!counters) {
+                len += snprintf(page + len, count - len,
+                                "\nDump Tally Counter Fail\n");
+                goto out_unlock;
+        }
+
+        rtl8127_dump_tally_counter(tp, paddr);
+
+        len += snprintf(page + len, count - len,
+                        "Statistics\tValue\n----------\t-----\n");
+
+        len += snprintf(page + len, count - len,
+                        "tx_packets\t%lld\n"
+                        "rx_packets\t%lld\n"
+                        "tx_errors\t%lld\n"
+                        "rx_errors\t%d\n"
+                        "rx_missed\t%d\n"
+                        "align_errors\t%d\n"
+                        "tx_one_collision\t%d\n"
+                        "tx_multi_collision\t%d\n"
+                        "rx_unicast\t%lld\n"
+                        "rx_broadcast\t%lld\n"
+                        "rx_multicast\t%d\n"
+                        "tx_aborted\t%d\n"
+                        "tx_underrun\t%d\n"
+
+                        "tx_octets\t%lld\n"
+                        "rx_octets\t%lld\n"
+                        "rx_multicast64\t%lld\n"
+                        "tx_unicast64\t%lld\n"
+                        "tx_broadcast64\t%lld\n"
+                        "tx_multicast64\t%lld\n"
+                        "tx_pause_on\t%d\n"
+                        "tx_pause_off\t%d\n"
+                        "tx_pause_all\t%d\n"
+                        "tx_deferred\t%d\n"
+                        "tx_late_collision\t%d\n"
+                        "tx_all_collision\t%d\n"
+                        "tx_aborted32\t%d\n"
+                        "align_errors32\t%d\n"
+                        "rx_frame_too_long\t%d\n"
+                        "rx_runt\t%d\n"
+                        "rx_pause_on\t%d\n"
+                        "rx_pause_off\t%d\n"
+                        "rx_pause_all\t%d\n"
+                        "rx_unknown_opcode\t%d\n"
+                        "rx_mac_error\t%d\n"
+                        "tx_underrun32\t%d\n"
+                        "rx_mac_missed\t%d\n"
+                        "rx_tcam_dropped\t%d\n"
+                        "tdu\t%d\n"
+                        "rdu\t%d\n",
+                        le64_to_cpu(counters->tx_packets),
+                        le64_to_cpu(counters->rx_packets),
+                        le64_to_cpu(counters->tx_errors),
+                        le32_to_cpu(counters->rx_errors),
+                        le16_to_cpu(counters->rx_missed),
+                        le16_to_cpu(counters->align_errors),
+                        le32_to_cpu(counters->tx_one_collision),
+                        le32_to_cpu(counters->tx_multi_collision),
+                        le64_to_cpu(counters->rx_unicast),
+                        le64_to_cpu(counters->rx_broadcast),
+                        le32_to_cpu(counters->rx_multicast),
+                        le16_to_cpu(counters->tx_aborted),
+                        le16_to_cpu(counters->tx_underrun),
+
+                        le64_to_cpu(counters->tx_octets),
+                        le64_to_cpu(counters->rx_octets),
+                        le64_to_cpu(counters->rx_multicast64),
+                        le64_to_cpu(counters->tx_unicast64),
+                        le64_to_cpu(counters->tx_broadcast64),
+                        le64_to_cpu(counters->tx_multicast64),
+                        le32_to_cpu(counters->tx_pause_on),
+                        le32_to_cpu(counters->tx_pause_off),
+                        le32_to_cpu(counters->tx_pause_all),
+                        le32_to_cpu(counters->tx_deferred),
+                        le32_to_cpu(counters->tx_late_collision),
+                        le32_to_cpu(counters->tx_all_collision),
+                        le32_to_cpu(counters->tx_aborted32),
+                        le32_to_cpu(counters->align_errors32),
+                        le32_to_cpu(counters->rx_frame_too_long),
+                        le32_to_cpu(counters->rx_runt),
+                        le32_to_cpu(counters->rx_pause_on),
+                        le32_to_cpu(counters->rx_pause_off),
+                        le32_to_cpu(counters->rx_pause_all),
+                        le32_to_cpu(counters->rx_unknown_opcode),
+                        le32_to_cpu(counters->rx_mac_error),
+                        le32_to_cpu(counters->tx_underrun32),
+                        le32_to_cpu(counters->rx_mac_missed),
+                        le32_to_cpu(counters->rx_tcam_dropped),
+                        le32_to_cpu(counters->tdu),
+                        le32_to_cpu(counters->rdu));
+
+        len += snprintf(page + len, count - len, "\n");
+out_unlock:
+        rtnl_unlock();
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_registers(char *page, char **start,
+                              off_t offset, int count,
+                              int *eof, void *data)
+{
+        struct net_device *dev = data;
+        int i, n, max = R8127_MAC_REGS_SIZE;
+        u8 byte_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        void __iomem *ioaddr = tp->mmio_addr;
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump MAC Registers\n"
+                        "Offset\tValue\n------\t-----\n");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%04x:\t",
+                                n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        len += snprintf(page + len, count - len,
+                                        "%02x ",
+                                        byte_rd);
+                }
+        }
+
+        max = 0xB00;
+        for (n = 0xA00; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%04x:\t",
+                                n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        len += snprintf(page + len, count - len,
+                                        "%02x ",
+                                        byte_rd);
+                }
+        }
+
+        max = 0xD40;
+        for (n = 0xD00; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%04x:\t",
+                                n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        len += snprintf(page + len, count - len,
+                                        "%02x ",
+                                        byte_rd);
+                }
+        }
+
+        max = 0x2840;
+        for (n = 0x2800; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%04x:\t",
+                                n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        len += snprintf(page + len, count - len,
+                                        "%02x ",
+                                        byte_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_all_registers(char *page, char **start,
+                                  off_t offset, int count,
+                                  int *eof, void *data)
+{
+        struct net_device *dev = data;
+        int i, n, max;
+        u8 byte_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        void __iomem *ioaddr = tp->mmio_addr;
+        struct pci_dev *pdev = tp->pci_dev;
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump All MAC Registers\n"
+                        "Offset\tValue\n------\t-----\n");
+
+        rtnl_lock();
+
+        max = pci_resource_len(pdev, 2);
+
+        for (n = 0; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%04x:\t",
+                                n);
+
+                for (i = 0; i < 16 && n < max; i++, n++) {
+                        byte_rd = readb(ioaddr + n);
+                        len += snprintf(page + len, count - len,
+                                        "%02x ",
+                                        byte_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\nTotal length:0x%X", max);
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_pcie_phy(char *page, char **start,
+                             off_t offset, int count,
+                             int *eof, void *data)
+{
+        struct net_device *dev = data;
+        int i, n, max = R8127_EPHY_REGS_SIZE/2;
+        u16 word_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump PCIE PHY\n"
+                        "Offset\tValue\n------\t-----\n");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%02x:\t",
+                                n);
+
+                for (i = 0; i < 8 && n < max; i++, n++) {
+                        word_rd = rtl8127_ephy_read(tp, n);
+                        len += snprintf(page + len, count - len,
+                                        "%04x ",
+                                        word_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_eth_phy(char *page, char **start,
+                            off_t offset, int count,
+                            int *eof, void *data)
+{
+        struct net_device *dev = data;
+        int i, n, max = R8127_PHY_REGS_SIZE/2;
+        unsigned long flags;
+        u16 word_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump Ethernet PHY\n"
+                        "Offset\tValue\n------\t-----\n");
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        len += snprintf(page + len, count - len,
+                        "\n####################page 0##################\n");
+        rtl8127_mdio_write(tp, 0x1f, 0x0000);
+        for (n = 0; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%02x:\t",
+                                n);
+
+                for (i = 0; i < 8 && n < max; i++, n++) {
+                        word_rd = rtl8127_mdio_read(tp, n);
+                        len += snprintf(page + len, count - len,
+                                        "%04x ",
+                                        word_rd);
+                }
+        }
+
+        len += snprintf(page + len, count - len,
+                        "\n####################extra reg##################\n");
+        n = 0xA400;
+        len += snprintf(page + len, count - len,
+                        "\n0x%02x:\t",
+                        n);
+        for (i = 0; i < 8; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                len += snprintf(page + len, count - len,
+                                "%04x ",
+                                word_rd);
+        }
+
+        n = 0xA410;
+        len += snprintf(page + len, count - len,
+                        "\n0x%02x:\t",
+                        n);
+        for (i = 0; i < 3; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                len += snprintf(page + len, count - len,
+                                "%04x ",
+                                word_rd);
+        }
+
+        n = 0xA434;
+        len += snprintf(page + len, count - len,
+                        "\n0x%02x:\t",
+                        n);
+        word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+        len += snprintf(page + len, count - len,
+                        "%04x ",
+                        word_rd);
+
+        n = 0xA5D0;
+        len += snprintf(page + len, count - len,
+                        "\n0x%02x:\t",
+                        n);
+        for (i = 0; i < 4; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                len += snprintf(page + len, count - len,
+                                "%04x ",
+                                word_rd);
+        }
+
+        n = 0xA61A;
+        len += snprintf(page + len, count - len,
+                        "\n0x%02x:\t",
+                        n);
+        word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+        len += snprintf(page + len, count - len,
+                        "%04x ",
+                        word_rd);
+
+        n = 0xA6D0;
+        len += snprintf(page + len, count - len,
+                        "\n0x%02x:\t",
+                        n);
+        for (i = 0; i < 3; i++, n+=2) {
+                word_rd = rtl8127_mdio_direct_read_phy_ocp(tp, n);
+                len += snprintf(page + len, count - len,
+                                "%04x ",
+                                word_rd);
+        }
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_extended_registers(char *page, char **start,
+                                       off_t offset, int count,
+                                       int *eof, void *data)
+{
+        struct net_device *dev = data;
+        int i, n, max = R8127_ERI_REGS_SIZE;
+        u32 dword_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump Extended Registers\n"
+                        "Offset\tValue\n------\t-----\n");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%02x:\t",
+                                n);
+
+                for (i = 0; i < 4 && n < max; i++, n+=4) {
+                        dword_rd = rtl8127_eri_read(tp, n, 4, ERIAR_ExGMAC);
+                        len += snprintf(page + len, count - len,
+                                        "%08x ",
+                                        dword_rd);
+                }
+        }
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_pci_registers(char *page, char **start,
+                                  off_t offset, int count,
+                                  int *eof, void *data)
+{
+        struct net_device *dev = data;
+        int i, n, max = R8127_PCI_REGS_SIZE;
+        u32 dword_rd;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nDump PCI Registers\n"
+                        "Offset\tValue\n------\t-----\n");
+
+        rtnl_lock();
+
+        for (n = 0; n < max;) {
+                len += snprintf(page + len, count - len,
+                                "\n0x%03x:\t",
+                                n);
+
+                for (i = 0; i < 4 && n < max; i++, n+=4) {
+                        pci_read_config_dword(tp->pci_dev, n, &dword_rd);
+                        len += snprintf(page + len, count - len,
+                                        "%08x ",
+                                        dword_rd);
+                }
+        }
+
+        n = 0x110;
+        pci_read_config_dword(tp->pci_dev, n, &dword_rd);
+        len += snprintf(page + len, count - len,
+                        "\n0x%03x:\t%08x ",
+                        n,
+                        dword_rd);
+        n = 0x70c;
+        pci_read_config_dword(tp->pci_dev, n, &dword_rd);
+        len += snprintf(page + len, count - len,
+                        "\n0x%03x:\t%08x ",
+                        n,
+                        dword_rd);
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_temperature(char *page, char **start,
+                                off_t offset, int count,
+                                int *eof, void *data)
+{
+        struct net_device *dev = data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u16 ts_digout, tj, fah;
+        unsigned long flags;
+        int len = 0;
+
+        len += snprintf(page + len, count - len,
+                        "\nChip Temperature\n");
+
+        rtnl_lock();
+
+        if (!rtl8127_sysfs_testmode_on(tp)) {
+                len += snprintf(page + len, count - len,
+                                "\nPlease turn on ""/sys/class/net/<iface>/rtk_adv/testmode"".\n\n");
+                goto out_unlock;
+        }
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+        ts_digout = rtl8127_read_thermal_sensor(tp);
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        if (ts_digout <= 512) {
+                tj = ts_digout / 2;
+                len += snprintf(page + len, count - len,
+                                "Cel:%d\n",
+                                tj);
+                fah = tj * (9/5) + 32;
+                len += snprintf(page + len, count - len,
+                                "Fah:%d\n",
+                                fah);
+
+        } else {
+                tj = (512 - (ts_digout - 512)) / 2;
+                len += snprintf(page + len, count - len,
+                                "Cel:-%d\n",
+                                tj);
+                fah = tj * (9/5) + 32;
+                len += snprintf(page + len, count - len,
+                                "Fah:-%d\n",
+                                fah);
+        }
+
+        len += snprintf(page + len, count - len, "\n");
+
+out_unlock:
+        rtnl_unlock();
+
+        *eof = 1;
+        return len;
+}
+
+static int _proc_get_cable_info(char *page, char **start,
+                                off_t offset, int count,
+                                int *eof, void *data,
+                                bool poe_mode)
+{
+        int i;
+        u32 status;
+        int len = 0;
+        struct net_device *dev = data;
+        int cp_status[RTL8127_CP_NUM] = {0};
+        int cp_len[RTL8127_CP_NUM] = {0};
+        struct rtl8127_private *tp = netdev_priv(dev);
+        const char *pair_str[RTL8127_CP_NUM] = {"1-2", "3-6", "4-5", "7-8"};
+
+        switch (tp->mcfg) {
+        default:
+                return -EOPNOTSUPP;
+        }
+
+        rtnl_lock();
+
+        if (!rtl8127_sysfs_testmode_on(tp)) {
+                len += snprintf(page + len, count - len,
+                                "\nPlease turn on ""/sys/class/net/<iface>/rtk_adv/testmode"".\n\n");
+                goto out_unlock;
+        }
+
+        status = rtl8127_get_phy_status(tp);
+        if (status & LinkStatus)
+                len += snprintf(page + len, count - len,
+                                "\nlink speed:%d",
+                                rtl8127_convert_link_speed(status));
+        else
+                len += snprintf(page + len, count - len,
+                                "\nlink status:off");
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        rtl8127_get_cp_len(tp, cp_len);
+
+        rtl8127_get_cp_status(tp, cp_status, poe_mode);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        len += snprintf(page + len, count - len,
+                        "\npair\tlength\tstatus   \tpp\n");
+
+        for (i=0; i<RTL8127_CP_NUM; i++) {
+                if (cp_len[i] < 0)
+                        len += snprintf(page + len, count - len,
+                                        "%s\t%s\t%s\t",
+                                        pair_str[i], "none",
+                                        rtl8127_get_cp_status_string(cp_status[i]));
+                else
+                        len += snprintf(page + len, count - len,
+                                        "%s\t%d\t%s\t",
+                                        pair_str[i], cp_len[i],
+                                        rtl8127_get_cp_status_string(cp_status[i]));
+                if (cp_status[i] == rtl8127_cp_normal)
+                        len += snprintf(page + len, count - len, "none\n");
+                else
+                        len += snprintf(page + len, count - len, "%dm\n",
+                                        rtl8127_get_cp_pp(tp, i));
+        }
+
+        len += snprintf(page + len, count - len, "\n");
+
+out_unlock:
+        rtnl_unlock();
+
+        *eof = 1;
+        return len;
+}
+
+static int proc_get_cable_info(char *page, char **start,
+                               off_t offset, int count,
+                               int *eof, void *data)
+{
+        return _proc_get_cable_info(page, start, offset, count, eof, data, 0);
+}
+
+static int proc_get_poe_cable_info(char *page, char **start,
+                                   off_t offset, int count,
+                                   int *eof, void *data)
+{
+        return _proc_get_cable_info(page, start, offset, count, eof, data, 1);
+}
+
+static void _proc_dump_desc(char *page, int *page_len, int *count, void *desc_base,
+                            u32 alloc_size)
+{
+        u32 *pdword;
+        int i, len;
+
+        if (desc_base == NULL ||
+            alloc_size == 0)
+                return;
+
+        len = *page_len;
+        pdword = (u32*)desc_base;
+        for (i=0; i<(alloc_size/4); i++) {
+                if (!(i % 4))
+                        len += snprintf(page + len, *count - len,
+                                        "\n%04x ",
+                                        i);
+                len += snprintf(page + len, *count - len,
+                                "%08x ",
+                                pdword[i]);
+        }
+
+        len += snprintf(page + len, *count - len, "\n");
+
+        *page_len = len;
+        return;
+}
+
+static int proc_dump_rx_desc(char *page, char **start,
+                             off_t offset, int count,
+                             int *eof, void *data)
+{
+        int i;
+        int len = 0;
+        struct net_device *dev = data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtnl_lock();
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                struct rtl8127_rx_ring *ring = &tp->rx_ring[i];
+
+                if (!ring)
+                        continue;
+
+                len += snprintf(page + len, count - len,
+                                "\ndump rx %d desc:%d",
+                                i, ring->num_rx_desc);
+
+                _proc_dump_desc(page, &len, &count,
+                                ring->RxDescArray,
+                                ring->RxDescAllocSize);
+        }
+
+#ifdef ENABLE_LIB_SUPPORT
+        if (rtl8127_num_lib_rx_rings(tp) > 0) {
+                for (i = 0; i < tp->HwSuppNumRxQueues; i++) {
+                        struct rtl8127_ring *lib_ring = &tp->lib_rx_ring[i];
+                        if (lib_ring->enabled) {
+                                len += snprintf(page + len, count - len,
+                                                "\ndump lib rx %d desc:%d",
+                                                i,
+                                                ring->ring_size);
+                                _proc_dump_desc(page, &len, &count,
+                                                (void*)lib_ring->desc_addr,
+                                                lib_ring->desc_size);
+                        }
+                }
+        }
+#endif //ENABLE_LIB_SUPPORT
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+
+        return len;
+}
+
+static int proc_dump_tx_desc(char *page, char **start,
+                             off_t offset, int count,
+                             int *eof, void *data)
+{
+        int len = 0;
+        struct net_device *dev = data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        rtnl_lock();
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                struct rtl8127_tx_ring *ring = &tp->tx_ring[i];
+
+                if (!ring)
+                        continue;
+
+                len += snprintf(page + len, count - len,
+                                "\ndump tx desc:%d",
+                                ring->num_tx_desc);
+
+                _proc_dump_desc(page, &len, &count,
+                                ring->TxDescArray,
+                                ring->TxDescAllocSize);
+        }
+
+#ifdef ENABLE_LIB_SUPPORT
+        if (rtl8127_num_lib_tx_rings(tp) > 0) {
+                for (i = 0; i < tp->HwSuppNumTxQueues; i++) {
+                        struct rtl8127_ring *lib_ring = &tp->lib_tx_ring[i];
+                        if (lib_ring->enabled) {
+                                len += snprintf(page + len, count - len,
+                                                "\ndump lib tx %d desc:%d",
+                                                i,
+                                                ring->ring_size);
+                                _proc_dump_desc(page, &len, &count,
+                                                (void*)lib_ring->desc_addr,
+                                                lib_ring->desc_size);
+                        }
+                }
+        }
+#endif //ENABLE_LIB_SUPPORT
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+
+        return len;
+}
+
+static int proc_dump_msix_tbl(char *page, char **start,
+                              off_t offset, int count,
+                              int *eof, void *data)
+{
+        int i, j;
+        int len = 0;
+        void __iomem *ioaddr;
+        struct net_device *dev = data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        /* ioremap MMIO region */
+        ioaddr = ioremap(pci_resource_start(tp->pci_dev, 4), pci_resource_len(tp->pci_dev, 4));
+        if (!ioaddr)
+                return -EFAULT;
+
+        rtnl_lock();
+
+        len += snprintf(page + len, count - len,
+                        "\ndump MSI-X Table. Total Entry %d. \n",
+                        tp->hw_supp_irq_nvecs);
+
+        for (i=0; i<tp->hw_supp_irq_nvecs; i++) {
+                len += snprintf(page + len, count - len,
+                                "\n%04x ", i);
+                for (j=0; j<4; j++)
+                        len += snprintf(page + len, count - len, "%08x ",
+                                        readl(ioaddr + i*0x10 + 4*j));
+        }
+
+        rtnl_unlock();
+
+        len += snprintf(page + len, count - len, "\n");
+
+        *eof = 1;
+        return 0;
+}
+
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+
+static void rtl8127_proc_module_init(void)
+{
+        //create /proc/net/r8127
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32)
+        rtl8127_proc = proc_mkdir(MODULENAME, init_net.proc_net);
+#else
+        rtl8127_proc = proc_mkdir(MODULENAME, proc_net);
+#endif
+        if (!rtl8127_proc)
+                dprintk("cannot create %s proc entry \n", MODULENAME);
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+/*
+ * seq_file wrappers for procfile show routines.
+ */
+static int rtl8127_proc_open(struct inode *inode, struct file *file)
+{
+        struct net_device *dev = proc_get_parent_data(inode);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+        int (*show)(struct seq_file *, void *) = pde_data(inode);
+#else
+        int (*show)(struct seq_file *, void *) = PDE_DATA(inode);
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+
+        return single_open(file, show, dev);
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
+static const struct proc_ops rtl8127_proc_fops = {
+        .proc_open           = rtl8127_proc_open,
+        .proc_read           = seq_read,
+        .proc_lseek          = seq_lseek,
+        .proc_release        = single_release,
+};
+#else
+static const struct file_operations rtl8127_proc_fops = {
+        .open           = rtl8127_proc_open,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = single_release,
+};
+#endif
+
+#endif
+
+/*
+ * Table of proc files we need to create.
+ */
+struct rtl8127_proc_file {
+        char name[16];
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+        int (*show)(struct seq_file *, void *);
+#else
+        int (*show)(char *, char **, off_t, int, int *, void *);
+#endif
+};
+
+static const struct rtl8127_proc_file rtl8127_debug_proc_files[] = {
+        { "driver_var", &proc_get_driver_variable },
+        { "tally", &proc_get_tally_counter },
+        { "registers", &proc_get_registers },
+        { "registers2", &proc_get_all_registers },
+        { "pcie_phy", &proc_get_pcie_phy },
+        { "eth_phy", &proc_get_eth_phy },
+        { "ext_regs", &proc_get_extended_registers },
+        { "pci_regs", &proc_get_pci_registers },
+        { "tx_desc", &proc_dump_tx_desc },
+        { "rx_desc", &proc_dump_rx_desc },
+        { "msix_tbl", &proc_dump_msix_tbl },
+        { "", NULL }
+};
+
+static const struct rtl8127_proc_file rtl8127_test_proc_files[] = {
+        { "temp", &proc_get_temperature },
+        { "cdt", &proc_get_cable_info },
+        { "cdt_poe", &proc_get_poe_cable_info },
+        { "", NULL }
+};
+
+#define R8127_PROC_DEBUG_DIR "debug"
+#define R8127_PROC_TEST_DIR "test"
+
+static void rtl8127_proc_init(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        const struct rtl8127_proc_file *f;
+        struct proc_dir_entry *dir;
+
+        if (!rtl8127_proc)
+                return;
+
+        if (tp->proc_dir_debug || tp->proc_dir_test)
+                return;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+        dir = proc_mkdir_data(dev->name, 0, rtl8127_proc, dev);
+        if (!dir) {
+                printk("Unable to initialize /proc/net/%s/%s\n",
+                       MODULENAME, dev->name);
+                return;
+        }
+        tp->proc_dir = dir;
+        proc_init_num++;
+
+        /* create debug entry */
+        dir = proc_mkdir_data(R8127_PROC_DEBUG_DIR, 0, tp->proc_dir, dev);
+        if (!dir) {
+                printk("Unable to initialize /proc/net/%s/%s/%s\n",
+                       MODULENAME, dev->name, R8127_PROC_DEBUG_DIR);
+                return;
+        }
+
+        tp->proc_dir_debug = dir;
+        for (f = rtl8127_debug_proc_files; f->name[0]; f++) {
+                if (!proc_create_data(f->name, S_IFREG | S_IRUGO, dir,
+                                      &rtl8127_proc_fops, f->show)) {
+                        printk("Unable to initialize "
+                               "/proc/net/%s/%s/%s/%s\n",
+                               MODULENAME, dev->name, R8127_PROC_DEBUG_DIR,
+                               f->name);
+                        return;
+                }
+        }
+
+        /* create test entry */
+        dir = proc_mkdir_data(R8127_PROC_TEST_DIR, 0, tp->proc_dir, dev);
+        if (!dir) {
+                printk("Unable to initialize /proc/net/%s/%s/%s\n",
+                       MODULENAME, dev->name, R8127_PROC_TEST_DIR);
+                return;
+        }
+
+        tp->proc_dir_test = dir;
+        for (f = rtl8127_test_proc_files; f->name[0]; f++) {
+                if (!proc_create_data(f->name, S_IFREG | S_IRUGO, dir,
+                                      &rtl8127_proc_fops, f->show)) {
+                        printk("Unable to initialize "
+                               "/proc/net/%s/%s/%s/%s\n",
+                               MODULENAME, dev->name, R8127_PROC_TEST_DIR,
+                               f->name);
+                        return;
+                }
+        }
+#else
+        dir = proc_mkdir(dev->name, rtl8127_proc);
+        if (!dir) {
+                printk("Unable to initialize /proc/net/%s/%s\n",
+                       MODULENAME, dev->name);
+                return;
+        }
+
+        tp->proc_dir = dir;
+        proc_init_num++;
+
+        /* create debug entry */
+        dir = proc_mkdir(R8127_PROC_DEBUG_DIR, tp->proc_dir);
+        if (!dir) {
+                printk("Unable to initialize /proc/net/%s/%s/%s\n",
+                       MODULENAME, dev->name, R8127_PROC_DEBUG_DIR);
+                return;
+        }
+
+        tp->proc_dir_debug = dir;
+        for (f = rtl8127_debug_proc_files; f->name[0]; f++) {
+                if (!create_proc_read_entry(f->name, S_IFREG | S_IRUGO,
+                                            dir, f->show, dev)) {
+                        printk("Unable to initialize "
+                               "/proc/net/%s/%s/%s/%s\n",
+                               MODULENAME, dev->name, R8127_PROC_DEBUG_DIR,
+                               f->name);
+                        return;
+                }
+        }
+
+        /* create test entry */
+        dir = proc_mkdir(R8127_PROC_TEST_DIR, tp->proc_dir);
+        if (!dir) {
+                printk("Unable to initialize /proc/net/%s/%s/%s\n",
+                       MODULENAME, dev->name, R8127_PROC_TEST_DIR);
+                return;
+        }
+
+        tp->proc_dir_test = dir;
+        for (f = rtl8127_test_proc_files; f->name[0]; f++) {
+                if (!create_proc_read_entry(f->name, S_IFREG | S_IRUGO,
+                                            dir, f->show, dev)) {
+                        printk("Unable to initialize "
+                               "/proc/net/%s/%s/%s/%s\n",
+                               MODULENAME, dev->name, R8127_PROC_TEST_DIR,
+                               f->name);
+                        return;
+                }
+        }
+#endif
+}
+
+static void rtl8127_proc_remove(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (tp->proc_dir) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+                remove_proc_subtree(dev->name, rtl8127_proc);
+#else
+                const struct rtl8127_proc_file *f;
+                struct rtl8127_private *tp = netdev_priv(dev);
+
+                if (tp->proc_dir_debug) {
+                        for (f = rtl8127_debug_proc_files; f->name[0]; f++)
+                                remove_proc_entry(f->name, tp->proc_dir_debug);
+                        remove_proc_entry(R8127_PROC_DEBUG_DIR, tp->proc_dir);
+                }
+
+                if (tp->proc_dir_test) {
+                        for (f = rtl8127_test_proc_files; f->name[0]; f++)
+                                remove_proc_entry(f->name, tp->proc_dir_test);
+                        remove_proc_entry(R8127_PROC_TEST_DIR, tp->proc_dir);
+                }
+
+                remove_proc_entry(dev->name, rtl8127_proc);
+#endif
+                proc_init_num--;
+
+                tp->proc_dir_debug = NULL;
+                tp->proc_dir_test = NULL;
+                tp->proc_dir = NULL;
+        }
+}
+
+#endif //ENABLE_R8127_PROCFS
+
+#ifdef ENABLE_R8127_SYSFS
+/****************************************************************************
+*   -----------------------------SYSFS STUFF-------------------------
+*****************************************************************************
+*/
+static ssize_t testmode_show(struct device *dev,
+                             struct device_attribute *attr, char *buf)
+{
+        struct net_device *netdev = to_net_dev(dev);
+        struct rtl8127_private *tp = netdev_priv(netdev);
+
+        sprintf(buf, "%u\n", tp->testmode);
+
+        return strlen(buf);
+}
+
+static ssize_t testmode_store(struct device *dev,
+                              struct device_attribute *attr,
+                              const char *buf, size_t count)
+{
+        struct net_device *netdev = to_net_dev(dev);
+        struct rtl8127_private *tp = netdev_priv(netdev);
+        u32 testmode;
+
+        if (sscanf(buf, "%u\n", &testmode) != 1)
+                return -EINVAL;
+
+        if (tp->testmode != testmode) {
+                rtnl_lock();
+                tp->testmode = testmode;
+                rtnl_unlock();
+        }
+
+        return count;
+}
+
+static DEVICE_ATTR_RW(testmode);
+
+static struct attribute *rtk_adv_attrs[] = {
+        &dev_attr_testmode.attr,
+        NULL
+};
+
+static struct attribute_group rtk_adv_grp = {
+        .name = "rtl_adv",
+        .attrs = rtk_adv_attrs,
+};
+
+static void rtl8127_sysfs_init(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ret;
+
+        /* init rtl_adv */
+#ifdef ENABLE_LIB_SUPPORT
+        tp->testmode = 0;
+#else
+        tp->testmode = 1;
+#endif //ENABLE_LIB_SUPPORT
+
+        ret = sysfs_create_group(&dev->dev.kobj, &rtk_adv_grp);
+        if (ret < 0)
+                netif_warn(tp, probe, dev, "create rtk_adv_grp fail\n");
+        else
+                set_bit(R8127_SYSFS_RTL_ADV, tp->sysfs_flag);
+}
+
+static void rtl8127_sysfs_remove(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (test_and_clear_bit(R8127_SYSFS_RTL_ADV, tp->sysfs_flag))
+                sysfs_remove_group(&dev->dev.kobj, &rtk_adv_grp);
+}
+#endif //ENABLE_R8127_SYSFS
+
+static inline u16 map_phy_ocp_addr(u16 PageNum, u8 RegNum)
+{
+        u16 OcpPageNum = 0;
+        u8 OcpRegNum = 0;
+        u16 OcpPhyAddress = 0;
+
+        if (PageNum == 0) {
+                OcpPageNum = OCP_STD_PHY_BASE_PAGE + (RegNum / 8);
+                OcpRegNum = 0x10 + (RegNum % 8);
+        } else {
+                OcpPageNum = PageNum;
+                OcpRegNum = RegNum;
+        }
+
+        OcpPageNum <<= 4;
+
+        if (OcpRegNum < 16) {
+                OcpPhyAddress = 0;
+        } else {
+                OcpRegNum -= 16;
+                OcpRegNum <<= 1;
+
+                OcpPhyAddress = OcpPageNum + OcpRegNum;
+        }
+
+
+        return OcpPhyAddress;
+}
+
+static void mdio_real_direct_write_phy_ocp(struct rtl8127_private *tp,
+                u16 RegAddr,
+                u16 value)
+{
+        u32 data32;
+        int i;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18)
+        WARN_ON_ONCE(RegAddr % 2);
+#endif
+        data32 = RegAddr/2;
+        data32 <<= OCPR_Addr_Reg_shift;
+        data32 |= OCPR_Write | value;
+
+        RTL_W32(tp, PHYOCP, data32);
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                udelay(R8127_CHANNEL_WAIT_TIME);
+
+                if (!(RTL_R32(tp, PHYOCP) & OCPR_Flag))
+                        break;
+        }
+}
+
+void rtl8127_mdio_direct_write_phy_ocp(struct rtl8127_private *tp,
+                                       u16 RegAddr,
+                                       u16 value)
+{
+        if (tp->rtk_enable_diag)
+                return;
+
+        mdio_real_direct_write_phy_ocp(tp, RegAddr, value);
+}
+
+/*
+static void rtl8127_mdio_write_phy_ocp(struct rtl8127_private *tp,
+                                       u16 PageNum,
+                                       u32 RegAddr,
+                                       u32 value)
+{
+        u16 ocp_addr;
+
+        ocp_addr = map_phy_ocp_addr(PageNum, RegAddr);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, ocp_addr, value);
+}
+*/
+
+static void rtl8127_mdio_real_write_phy_ocp(struct rtl8127_private *tp,
+                u16 PageNum,
+                u32 RegAddr,
+                u32 value)
+{
+        u16 ocp_addr;
+
+        ocp_addr = map_phy_ocp_addr(PageNum, RegAddr);
+
+        mdio_real_direct_write_phy_ocp(tp, ocp_addr, value);
+}
+
+static void mdio_real_write(struct rtl8127_private *tp,
+                            u16 RegAddr,
+                            u16 value)
+{
+        if (RegAddr == 0x1F) {
+                tp->cur_page = value;
+                return;
+        }
+        rtl8127_mdio_real_write_phy_ocp(tp, tp->cur_page, RegAddr, value);
+}
+
+void rtl8127_mdio_write(struct rtl8127_private *tp,
+                        u16 RegAddr,
+                        u16 value)
+{
+        if (tp->rtk_enable_diag)
+                return;
+
+        mdio_real_write(tp, RegAddr, value);
+}
+
+void rtl8127_mdio_prot_write(struct rtl8127_private *tp,
+                             u32 RegAddr,
+                             u32 value)
+{
+        mdio_real_write(tp, RegAddr, value);
+}
+
+void rtl8127_mdio_prot_direct_write_phy_ocp(struct rtl8127_private *tp,
+                u32 RegAddr,
+                u32 value)
+{
+        mdio_real_direct_write_phy_ocp(tp, RegAddr, value);
+}
+
+static u32 mdio_real_direct_read_phy_ocp(struct rtl8127_private *tp,
+                u16 RegAddr)
+{
+        u32 data32;
+        int i, value = 0;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18)
+        WARN_ON_ONCE(RegAddr % 2);
+#endif
+        data32 = RegAddr/2;
+        data32 <<= OCPR_Addr_Reg_shift;
+
+        RTL_W32(tp, PHYOCP, data32);
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                udelay(R8127_CHANNEL_WAIT_TIME);
+
+                if (RTL_R32(tp, PHYOCP) & OCPR_Flag)
+                        break;
+        }
+        value = RTL_R32(tp, PHYOCP) & OCPDR_Data_Mask;
+
+        return value;
+}
+
+u32 rtl8127_mdio_direct_read_phy_ocp(struct rtl8127_private *tp,
+                                     u16 RegAddr)
+{
+        if (tp->rtk_enable_diag)
+                return 0xffffffff;
+
+        return mdio_real_direct_read_phy_ocp(tp, RegAddr);
+}
+
+/*
+static u32 rtl8127_mdio_read_phy_ocp(struct rtl8127_private *tp,
+                                     u16 PageNum,
+                                     u32 RegAddr)
+{
+        u16 ocp_addr;
+
+        ocp_addr = map_phy_ocp_addr(PageNum, RegAddr);
+
+        return rtl8127_mdio_direct_read_phy_ocp(tp, ocp_addr);
+}
+*/
+
+static u32 rtl8127_mdio_real_read_phy_ocp(struct rtl8127_private *tp,
+                u16 PageNum,
+                u32 RegAddr)
+{
+        u16 ocp_addr;
+
+        ocp_addr = map_phy_ocp_addr(PageNum, RegAddr);
+
+        return mdio_real_direct_read_phy_ocp(tp, ocp_addr);
+}
+
+static u32 mdio_real_read(struct rtl8127_private *tp,
+                          u16 RegAddr)
+{
+        return rtl8127_mdio_real_read_phy_ocp(tp, tp->cur_page, RegAddr);
+}
+
+u32 rtl8127_mdio_read(struct rtl8127_private *tp,
+                      u16 RegAddr)
+{
+        if (tp->rtk_enable_diag)
+                return 0xffffffff;
+
+        return mdio_real_read(tp, RegAddr);
+}
+
+u32 rtl8127_mdio_prot_read(struct rtl8127_private *tp,
+                           u32 RegAddr)
+{
+        return mdio_real_read(tp, RegAddr);
+}
+
+u32 rtl8127_mdio_prot_direct_read_phy_ocp(struct rtl8127_private *tp,
+                u32 RegAddr)
+{
+        return mdio_real_direct_read_phy_ocp(tp, RegAddr);
+}
+
+static void rtl8127_clear_and_set_eth_phy_bit(struct rtl8127_private *tp, u8  addr, u16 clearmask, u16 setmask)
+{
+        u16 PhyRegValue;
+
+        PhyRegValue = rtl8127_mdio_read(tp, addr);
+        PhyRegValue &= ~clearmask;
+        PhyRegValue |= setmask;
+        rtl8127_mdio_write(tp, addr, PhyRegValue);
+}
+
+void rtl8127_clear_eth_phy_bit(struct rtl8127_private *tp, u8 addr, u16 mask)
+{
+        rtl8127_clear_and_set_eth_phy_bit(tp,
+                                          addr,
+                                          mask,
+                                          0);
+}
+
+void rtl8127_set_eth_phy_bit(struct rtl8127_private *tp,  u8  addr, u16  mask)
+{
+        rtl8127_clear_and_set_eth_phy_bit(tp,
+                                          addr,
+                                          0,
+                                          mask);
+}
+
+void rtl8127_clear_and_set_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 clearmask, u16 setmask)
+{
+        u16 PhyRegValue;
+
+        PhyRegValue = rtl8127_mdio_direct_read_phy_ocp(tp, addr);
+        PhyRegValue &= ~clearmask;
+        PhyRegValue |= setmask;
+        rtl8127_mdio_direct_write_phy_ocp(tp, addr, PhyRegValue);
+}
+
+void rtl8127_clear_eth_phy_ocp_bit(struct rtl8127_private *tp, u16 addr, u16 mask)
+{
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              addr,
+                                              mask,
+                                              0);
+}
+
+void rtl8127_set_eth_phy_ocp_bit(struct rtl8127_private *tp,  u16 addr, u16 mask)
+{
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              addr,
+                                              0,
+                                              mask);
+}
+
+void rtl8127_mac_ocp_write(struct rtl8127_private *tp, u16 reg_addr, u16 value)
+{
+        u32 data32;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18)
+        WARN_ON_ONCE(reg_addr % 2);
+#endif
+
+        data32 = reg_addr/2;
+        data32 <<= OCPR_Addr_Reg_shift;
+        data32 += value;
+        data32 |= OCPR_Write;
+
+        RTL_W32(tp, MACOCP, data32);
+}
+
+u16 rtl8127_mac_ocp_read(struct rtl8127_private *tp, u16 reg_addr)
+{
+        u32 data32;
+        u16 data16 = 0;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18)
+        WARN_ON_ONCE(reg_addr % 2);
+#endif
+
+        data32 = reg_addr/2;
+        data32 <<= OCPR_Addr_Reg_shift;
+
+        RTL_W32(tp, MACOCP, data32);
+        data16 = (u16)RTL_R32(tp, MACOCP);
+
+        return data16;
+}
+
+#ifdef ENABLE_USE_FIRMWARE_FILE
+static void mac_mcu_write(struct rtl8127_private *tp, u16 reg, u16 value)
+{
+        if (reg == 0x1f) {
+                tp->ocp_base = value << 4;
+                return;
+        }
+
+        rtl8127_mac_ocp_write(tp, tp->ocp_base + reg, value);
+}
+
+static u32 mac_mcu_read(struct rtl8127_private *tp, u16 reg)
+{
+        return rtl8127_mac_ocp_read(tp, tp->ocp_base + reg);
+}
+#endif
+
+static void
+rtl8127_clear_set_mac_ocp_bit(
+        struct rtl8127_private *tp,
+        u16   addr,
+        u16   clearmask,
+        u16   setmask
+)
+{
+        u16 PhyRegValue;
+
+        PhyRegValue = rtl8127_mac_ocp_read(tp, addr);
+        PhyRegValue &= ~clearmask;
+        PhyRegValue |= setmask;
+        rtl8127_mac_ocp_write(tp, addr, PhyRegValue);
+}
+
+void
+rtl8127_clear_mac_ocp_bit(
+        struct rtl8127_private *tp,
+        u16   addr,
+        u16   mask
+)
+{
+        rtl8127_clear_set_mac_ocp_bit(tp,
+                                      addr,
+                                      mask,
+                                      0);
+}
+
+static void
+rtl8127_set_mac_ocp_bit(
+        struct rtl8127_private *tp,
+        u16   addr,
+        u16   mask
+)
+{
+        rtl8127_clear_set_mac_ocp_bit(tp,
+                                      addr,
+                                      0,
+                                      mask);
+}
+
+u32 rtl8127_ocp_read_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, const u32 base_address)
+{
+        return rtl8127_eri_read_with_oob_base_address(tp, addr, len, ERIAR_OOB, base_address);
+}
+
+u32 rtl8127_ocp_read(struct rtl8127_private *tp, u16 addr, u8 len)
+{
+        if (!tp->AllowAccessDashOcp || tp->HwSuppOcpChannelVer != 2)
+                return 0xffffffff;
+
+        return rtl8127_ocp_read_with_oob_base_address(tp, addr, len,
+                        NO_BASE_ADDRESS);
+}
+
+u32 rtl8127_ocp_write_with_oob_base_address(struct rtl8127_private *tp, u16 addr, u8 len, u32 value, const u32 base_address)
+{
+        return rtl8127_eri_write_with_oob_base_address(tp, addr, len, value,
+                        ERIAR_OOB, base_address);
+}
+
+void rtl8127_ocp_write(struct rtl8127_private *tp, u16 addr, u8 len, u32 value)
+{
+        if (!tp->AllowAccessDashOcp || tp->HwSuppOcpChannelVer != 2)
+                return;
+
+        rtl8127_ocp_write_with_oob_base_address(tp, addr, len, value, NO_BASE_ADDRESS);
+}
+
+void rtl8127_oob_mutex_lock(struct rtl8127_private *tp)
+{
+        u8 reg_16, reg_a0;
+        u32 wait_cnt_0, wait_Cnt_1;
+        u16 ocp_reg_mutex_ib;
+        u16 ocp_reg_mutex_oob;
+        u16 ocp_reg_mutex_prio;
+
+        if (!HW_DASH_SUPPORT_DASH(tp))
+                return;
+
+        if (!tp->DASH)
+                return;
+
+        ocp_reg_mutex_oob = 0x110;
+        ocp_reg_mutex_ib = 0x114;
+        ocp_reg_mutex_prio = 0x11C;
+
+        rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, BIT_0);
+        reg_16 = rtl8127_ocp_read(tp, ocp_reg_mutex_oob, 1);
+        wait_cnt_0 = 0;
+        while(reg_16) {
+                reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1);
+                if (reg_a0) {
+                        rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, 0x00);
+                        reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1);
+                        wait_Cnt_1 = 0;
+                        while(reg_a0) {
+                                reg_a0 = rtl8127_ocp_read(tp, ocp_reg_mutex_prio, 1);
+
+                                wait_Cnt_1++;
+
+                                if (wait_Cnt_1 > 2000)
+                                        break;
+                        };
+                        rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, BIT_0);
+
+                }
+                reg_16 = rtl8127_ocp_read(tp, ocp_reg_mutex_oob, 1);
+
+                wait_cnt_0++;
+
+                if (wait_cnt_0 > 2000)
+                        break;
+        };
+}
+
+void rtl8127_oob_mutex_unlock(struct rtl8127_private *tp)
+{
+        u16 ocp_reg_mutex_ib;
+        u16 ocp_reg_mutex_prio;
+
+        if (!HW_DASH_SUPPORT_DASH(tp))
+                return;
+
+        if (!tp->DASH)
+                return;
+
+        ocp_reg_mutex_ib = 0x114;
+        ocp_reg_mutex_prio = 0x11C;
+
+        rtl8127_ocp_write(tp, ocp_reg_mutex_prio, 1, BIT_0);
+        rtl8127_ocp_write(tp, ocp_reg_mutex_ib, 1, 0x00);
+}
+
+static bool
+rtl8127_is_allow_access_dash_ocp(struct rtl8127_private *tp)
+{
+        bool allow_access = false;
+        u16 mac_ocp_data;
+
+        if (!HW_DASH_SUPPORT_DASH(tp))
+                goto exit;
+
+        allow_access = true;
+        switch (tp->mcfg) {
+        case CFG_METHOD_2:
+                mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xd4c0);
+                if (mac_ocp_data == 0xffff || (mac_ocp_data & BIT_3))
+                        allow_access = false;
+                break;
+        default:
+                goto exit;
+        }
+exit:
+        return allow_access;
+}
+
+static u32 rtl8127_get_dash_fw_ver(struct rtl8127_private *tp)
+{
+        u32 ver = 0xffffffff;
+
+        if (FALSE == HW_DASH_SUPPORT_GET_FIRMWARE_VERSION(tp))
+                goto exit;
+
+        ver = rtl8127_ocp_read(tp, OCP_REG_FIRMWARE_MAJOR_VERSION, 4);
+
+exit:
+        return ver;
+}
+
+static int _rtl8127_check_dash(struct rtl8127_private *tp)
+{
+        if (!tp->AllowAccessDashOcp)
+                return 0;
+
+        if (!HW_DASH_SUPPORT_IPC2(tp))
+                return 0;
+
+        if (rtl8127_ocp_read(tp, 0x128, 1) & BIT_0)
+                return 1;
+
+        return 0;
+}
+
+static int rtl8127_check_dash(struct rtl8127_private *tp)
+{
+        if (HW_DASH_SUPPORT_DASH(tp) && _rtl8127_check_dash(tp)) {
+                u32 ver = rtl8127_get_dash_fw_ver(tp);
+                if (!(ver == 0 || ver == 0xffffffff))
+                        return 1;
+        }
+
+        return 0;
+}
+
+static int rtl8127_wait_dash_fw_ready(struct rtl8127_private *tp)
+{
+        int rc = -1;
+        int timeout;
+
+        if (!tp->DASH)
+                goto out;
+
+        for (timeout = 0; timeout < 10; timeout++) {
+                fsleep(10000);
+                if (rtl8127_ocp_read(tp, 0x124, 1) & BIT_0) {
+                        rc = 1;
+                        goto out;
+                }
+        }
+
+        rc = 0;
+
+out:
+        return rc;
+}
+
+static void
+rtl8127_notify_dash_oob_ipc2(struct rtl8127_private *tp, u32 cmd)
+{
+        if (FALSE == HW_DASH_SUPPORT_IPC2(tp))
+                return;
+
+        rtl8127_ocp_write(tp, IPC2_IB2SOC_DATA, 4, cmd);
+        rtl8127_ocp_write(tp, IPC2_IB2SOC_CMD, 4, 0x00);
+        rtl8127_ocp_write(tp, IPC2_IB2SOC_SET, 4, 0x01);
+}
+
+static void
+rtl8127_notify_dash_oob(struct rtl8127_private *tp, u32 cmd)
+{
+        if (HW_DASH_SUPPORT_IPC2(tp))
+                return rtl8127_notify_dash_oob_ipc2(tp, cmd);
+        else
+                return;
+}
+
+static void rtl8127_driver_start(struct rtl8127_private *tp)
+{
+        if (!tp->AllowAccessDashOcp)
+                return;
+
+        rtl8127_notify_dash_oob(tp, OOB_CMD_DRIVER_START);
+
+        rtl8127_wait_dash_fw_ready(tp);
+}
+
+static void rtl8127_driver_stop(struct rtl8127_private *tp)
+{
+        if (!tp->AllowAccessDashOcp)
+                return;
+
+        rtl8127_notify_dash_oob(tp, OOB_CMD_DRIVER_STOP);
+
+        rtl8127_wait_dash_fw_ready(tp);
+}
+
+static void _rtl8127_ephy_write(struct rtl8127_private *tp, int addr, int data)
+{
+        int i;
+
+        RTL_W32(tp, EPHYAR,
+                EPHYAR_Write |
+                (addr & EPHYAR_Reg_Mask_v2) << EPHYAR_Reg_shift |
+                (data & EPHYAR_Data_Mask));
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                udelay(R8127_CHANNEL_WAIT_TIME);
+
+                /* Check if the RTL8125 has completed EPHY write */
+                if (!(RTL_R32(tp, EPHYAR) & EPHYAR_Flag))
+                        break;
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+}
+
+static void rtl8127_set_ephy_ext_addr(struct rtl8127_private *tp, int addr)
+{
+        _rtl8127_ephy_write(tp, EPHYAR_EXT_ADDR, addr);
+}
+
+static int rtl8127_check_ephy_ext_addr(struct rtl8127_private *tp, int addr)
+{
+        int data;
+
+        data = ((u16)addr >> 12);
+
+        rtl8127_set_ephy_ext_addr(tp, data);
+
+        return (addr & 0xfff);
+}
+
+void rtl8127_ephy_write(struct rtl8127_private *tp, int addr, int data)
+{
+        _rtl8127_ephy_write(tp, rtl8127_check_ephy_ext_addr(tp, addr), data);
+}
+
+static u16 _rtl8127_ephy_read(struct rtl8127_private *tp, int addr)
+{
+        int i;
+        u16 data = 0xffff;
+
+        RTL_W32(tp, EPHYAR,
+                EPHYAR_Read | (addr & EPHYAR_Reg_Mask_v2) << EPHYAR_Reg_shift);
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                udelay(R8127_CHANNEL_WAIT_TIME);
+
+                /* Check if the RTL8125 has completed EPHY read */
+                if (RTL_R32(tp, EPHYAR) & EPHYAR_Flag) {
+                        data = (u16) (RTL_R32(tp, EPHYAR) & EPHYAR_Data_Mask);
+                        break;
+                }
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+
+        return data;
+}
+
+u16 rtl8127_ephy_read(struct rtl8127_private *tp, int addr)
+{
+        return  _rtl8127_ephy_read(tp, rtl8127_check_ephy_ext_addr(tp, addr));
+}
+
+/*
+static void ClearAndSetPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 clearmask, u16 setmask)
+{
+        u16 EphyValue;
+
+        EphyValue = rtl8127_ephy_read(tp, addr);
+        EphyValue &= ~clearmask;
+        EphyValue |= setmask;
+        rtl8127_ephy_write(tp, addr, EphyValue);
+}
+
+static void ClearPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 mask)
+{
+        ClearAndSetPCIePhyBit(tp,
+                              addr,
+                              mask,
+                              0);
+}
+
+static void SetPCIePhyBit(struct rtl8127_private *tp, u8 addr, u16 mask)
+{
+        ClearAndSetPCIePhyBit(tp,
+                              addr,
+                              0,
+                              mask);
+}
+*/
+
+static u32
+rtl8127_csi_other_fun_read(struct rtl8127_private *tp,
+                           u8 multi_fun_sel_bit,
+                           u32 addr)
+{
+        u32 cmd;
+        int i;
+        u32 value = 0xffffffff;
+
+        cmd = CSIAR_Read | CSIAR_ByteEn << CSIAR_ByteEn_shift | (addr & CSIAR_Addr_Mask);
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT)
+                multi_fun_sel_bit = 0;
+
+        if (multi_fun_sel_bit > 7)
+                goto exit;
+
+        cmd |= multi_fun_sel_bit << 16;
+
+        RTL_W32(tp, CSIAR, cmd);
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                udelay(R8127_CHANNEL_WAIT_TIME);
+
+                /* Check if the RTL8125 has completed CSI read */
+                if (RTL_R32(tp, CSIAR) & CSIAR_Flag) {
+                        value = (u32)RTL_R32(tp, CSIDR);
+                        break;
+                }
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+
+exit:
+        return value;
+}
+
+static void
+rtl8127_csi_other_fun_write(struct rtl8127_private *tp,
+                            u8 multi_fun_sel_bit,
+                            u32 addr,
+                            u32 value)
+{
+        u32 cmd;
+        int i;
+
+        RTL_W32(tp, CSIDR, value);
+        cmd = CSIAR_Write | CSIAR_ByteEn << CSIAR_ByteEn_shift | (addr & CSIAR_Addr_Mask);
+        if (tp->mcfg == CFG_METHOD_DEFAULT)
+                multi_fun_sel_bit = 0;
+
+        if (multi_fun_sel_bit > 7)
+                return;
+
+        cmd |= multi_fun_sel_bit << 16;
+
+        RTL_W32(tp, CSIAR, cmd);
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                udelay(R8127_CHANNEL_WAIT_TIME);
+
+                /* Check if the RTL8125 has completed CSI write */
+                if (!(RTL_R32(tp, CSIAR) & CSIAR_Flag))
+                        break;
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+}
+
+static u32
+rtl8127_csi_read(struct rtl8127_private *tp,
+                 u32 addr)
+{
+        u8 multi_fun_sel_bit;
+
+        multi_fun_sel_bit = 0;
+
+        return rtl8127_csi_other_fun_read(tp, multi_fun_sel_bit, addr);
+}
+
+static void
+rtl8127_csi_write(struct rtl8127_private *tp,
+                  u32 addr,
+                  u32 value)
+{
+        u8 multi_fun_sel_bit;
+
+        multi_fun_sel_bit = 0;
+
+        rtl8127_csi_other_fun_write(tp, multi_fun_sel_bit, addr, value);
+}
+
+static u8
+rtl8127_csi_fun0_read_byte(struct rtl8127_private *tp,
+                           u32 addr)
+{
+        u8 RetVal = 0;
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT) {
+                struct pci_dev *pdev = tp->pci_dev;
+
+                pci_read_config_byte(pdev, addr, &RetVal);
+        } else {
+                u32 TmpUlong;
+                u16 RegAlignAddr;
+                u8 ShiftByte;
+
+                RegAlignAddr = addr & ~(0x3);
+                ShiftByte = addr & (0x3);
+                TmpUlong = rtl8127_csi_other_fun_read(tp, 0, RegAlignAddr);
+                TmpUlong >>= (8*ShiftByte);
+                RetVal = (u8)TmpUlong;
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+
+        return RetVal;
+}
+
+static void
+rtl8127_csi_fun0_write_byte(struct rtl8127_private *tp,
+                            u32 addr,
+                            u8 value)
+{
+        if (tp->mcfg == CFG_METHOD_DEFAULT) {
+                struct pci_dev *pdev = tp->pci_dev;
+
+                pci_write_config_byte(pdev, addr, value);
+        } else {
+                u32 TmpUlong;
+                u16 RegAlignAddr;
+                u8 ShiftByte;
+
+                RegAlignAddr = addr & ~(0x3);
+                ShiftByte = addr & (0x3);
+                TmpUlong = rtl8127_csi_other_fun_read(tp, 0, RegAlignAddr);
+                TmpUlong &= ~(0xFF << (8*ShiftByte));
+                TmpUlong |= (value << (8*ShiftByte));
+                rtl8127_csi_other_fun_write(tp, 0, RegAlignAddr, TmpUlong);
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+}
+
+u32 rtl8127_eri_read_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, int type, const u32 base_address)
+{
+        int i, val_shift, shift = 0;
+        u32 value1 = 0, value2 = 0, mask;
+        u32 eri_cmd;
+        const u32 transformed_base_address = ((base_address & 0x00FFF000) << 6) | (base_address & 0x000FFF);
+
+        if (len > 4 || len <= 0)
+                return -1;
+
+        while (len > 0) {
+                val_shift = addr % ERIAR_Addr_Align;
+                addr = addr & ~0x3;
+
+                eri_cmd = ERIAR_Read |
+                          transformed_base_address |
+                          type << ERIAR_Type_shift |
+                          ERIAR_ByteEn << ERIAR_ByteEn_shift |
+                          (addr & 0x0FFF);
+                if (addr & 0xF000) {
+                        u32 tmp;
+
+                        tmp = addr & 0xF000;
+                        tmp >>= 12;
+                        eri_cmd |= (tmp << 20) & 0x00F00000;
+                }
+
+                RTL_W32(tp, ERIAR, eri_cmd);
+
+                for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                        udelay(R8127_CHANNEL_WAIT_TIME);
+
+                        /* Check if the RTL8125 has completed ERI read */
+                        if (RTL_R32(tp, ERIAR) & ERIAR_Flag)
+                                break;
+                }
+
+                if (len == 1)       mask = (0xFF << (val_shift * 8)) & 0xFFFFFFFF;
+                else if (len == 2)  mask = (0xFFFF << (val_shift * 8)) & 0xFFFFFFFF;
+                else if (len == 3)  mask = (0xFFFFFF << (val_shift * 8)) & 0xFFFFFFFF;
+                else            mask = (0xFFFFFFFF << (val_shift * 8)) & 0xFFFFFFFF;
+
+                value1 = RTL_R32(tp, ERIDR) & mask;
+                value2 |= (value1 >> val_shift * 8) << shift * 8;
+
+                if (len <= 4 - val_shift) {
+                        len = 0;
+                } else {
+                        len -= (4 - val_shift);
+                        shift = 4 - val_shift;
+                        addr += 4;
+                }
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+
+        return value2;
+}
+
+u32 rtl8127_eri_read(struct rtl8127_private *tp, int addr, int len, int type)
+{
+        return rtl8127_eri_read_with_oob_base_address(tp, addr, len, type, 0);
+}
+
+int rtl8127_eri_write_with_oob_base_address(struct rtl8127_private *tp, int addr, int len, u32 value, int type, const u32 base_address)
+{
+        int i, val_shift, shift = 0;
+        u32 value1 = 0, mask;
+        u32 eri_cmd;
+        const u32 transformed_base_address = ((base_address & 0x00FFF000) << 6) | (base_address & 0x000FFF);
+
+        if (len > 4 || len <= 0)
+                return -1;
+
+        while (len > 0) {
+                val_shift = addr % ERIAR_Addr_Align;
+                addr = addr & ~0x3;
+
+                if (len == 1)       mask = (0xFF << (val_shift * 8)) & 0xFFFFFFFF;
+                else if (len == 2)  mask = (0xFFFF << (val_shift * 8)) & 0xFFFFFFFF;
+                else if (len == 3)  mask = (0xFFFFFF << (val_shift * 8)) & 0xFFFFFFFF;
+                else            mask = (0xFFFFFFFF << (val_shift * 8)) & 0xFFFFFFFF;
+
+                value1 = rtl8127_eri_read_with_oob_base_address(tp, addr, 4, type, base_address) & ~mask;
+                value1 |= ((value << val_shift * 8) >> shift * 8);
+
+                RTL_W32(tp, ERIDR, value1);
+
+                eri_cmd = ERIAR_Write |
+                          transformed_base_address |
+                          type << ERIAR_Type_shift |
+                          ERIAR_ByteEn << ERIAR_ByteEn_shift |
+                          (addr & 0x0FFF);
+                if (addr & 0xF000) {
+                        u32 tmp;
+
+                        tmp = addr & 0xF000;
+                        tmp >>= 12;
+                        eri_cmd |= (tmp << 20) & 0x00F00000;
+                }
+
+                RTL_W32(tp, ERIAR, eri_cmd);
+
+                for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                        udelay(R8127_CHANNEL_WAIT_TIME);
+
+                        /* Check if the RTL8125 has completed ERI write */
+                        if (!(RTL_R32(tp, ERIAR) & ERIAR_Flag))
+                                break;
+                }
+
+                if (len <= 4 - val_shift) {
+                        len = 0;
+                } else {
+                        len -= (4 - val_shift);
+                        shift = 4 - val_shift;
+                        addr += 4;
+                }
+        }
+
+        udelay(R8127_CHANNEL_EXIT_DELAY_TIME);
+
+        return 0;
+}
+
+int rtl8127_eri_write(struct rtl8127_private *tp, int addr, int len, u32 value, int type)
+{
+        return rtl8127_eri_write_with_oob_base_address(tp, addr, len, value, type, NO_BASE_ADDRESS);
+}
+
+static void
+rtl8127_enable_rxdvgate(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) | BIT_3);
+}
+
+static void
+rtl8127_disable_rxdvgate(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_3);
+}
+
+static u8
+rtl8127_is_gpio_low(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u8 gpio_low = FALSE;
+
+        switch (tp->HwSuppCheckPhyDisableModeVer) {
+        case 3:
+                if (!(rtl8127_mac_ocp_read(tp, 0xDC04) & BIT_13))
+                        gpio_low = TRUE;
+                break;
+        }
+
+        if (gpio_low)
+                dprintk("gpio is low.\n");
+
+        return gpio_low;
+}
+
+static u8
+rtl8127_is_phy_disable_mode_enabled(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u8 phy_disable_mode_enabled = FALSE;
+
+        switch (tp->HwSuppCheckPhyDisableModeVer) {
+        case 3:
+                if (RTL_R8(tp, 0xF2) & BIT_5)
+                        phy_disable_mode_enabled = TRUE;
+                break;
+        }
+
+        if (phy_disable_mode_enabled)
+                dprintk("phy disable mode enabled.\n");
+
+        return phy_disable_mode_enabled;
+}
+
+static u8
+rtl8127_is_in_phy_disable_mode(struct net_device *dev)
+{
+        u8 in_phy_disable_mode = FALSE;
+
+        if (rtl8127_is_phy_disable_mode_enabled(dev) && rtl8127_is_gpio_low(dev))
+                in_phy_disable_mode = TRUE;
+
+        if (in_phy_disable_mode)
+                dprintk("Hardware is in phy disable mode.\n");
+
+        return in_phy_disable_mode;
+}
+
+static void
+rtl8127_stop_all_request(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
+        udelay(200);
+}
+
+static void
+rtl8127_clear_stop_all_request(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & (CmdTxEnb | CmdRxEnb));
+}
+
+void
+rtl8127_wait_txrx_fifo_empty(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        /* Txfifo_empty require StopReq been set */
+        for (i = 0; i < 3000; i++) {
+                udelay(50);
+                if ((RTL_R8(tp, MCUCmd_reg) & (Txfifo_empty | Rxfifo_empty)) == (Txfifo_empty | Rxfifo_empty))
+                        break;
+        }
+
+        for (i = 0; i < 3000; i++) {
+                udelay(50);
+                if ((RTL_R16(tp, IntrMitigate) & (BIT_0 | BIT_1 | BIT_8)) == (BIT_0 | BIT_1 | BIT_8))
+                        break;
+        }
+}
+
+#ifdef ENABLE_DASH_SUPPORT
+
+static inline void
+rtl8127_enable_dash2_interrupt(struct rtl8127_private *tp)
+{
+        if (!HW_DASH_SUPPORT_IPC2(tp))
+                return;
+
+        if (!tp->DASH)
+                return;
+
+        rtl8127_set_ipc2_soc_imr_bit(tp, RISC_IPC2_INTR);
+}
+
+static inline void
+rtl8127_disable_dash2_interrupt(struct rtl8127_private *tp)
+{
+        if (!HW_DASH_SUPPORT_IPC2(tp))
+                return;
+
+        rtl8127_clear_ipc2_soc_imr_bit(tp, RISC_IPC2_INTR);
+}
+#endif
+
+void
+rtl8127_enable_hw_linkchg_interrupt(struct rtl8127_private *tp)
+{
+        switch (tp->HwCurrIsrVer) {
+        case 6:
+                RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V6_LINKCHG);
+                break;
+        case 5:
+                RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V5_LINKCHG);
+                break;
+        case 4:
+                RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V4_LINKCHG);
+                break;
+        case 2:
+        case 3:
+                RTL_W32(tp, IMR_V2_SET_REG_8125, ISRIMR_V2_LINKCHG);
+                break;
+        case 1:
+                RTL_W32(tp, tp->imr_reg[0], LinkChg | RTL_R32(tp, tp->imr_reg[0]));
+                break;
+        }
+
+#ifdef ENABLE_DASH_SUPPORT
+        if (tp->DASH)
+                rtl8127_enable_dash2_interrupt(tp);
+#endif
+}
+
+static inline void
+rtl8127_enable_hw_interrupt(struct rtl8127_private *tp)
+{
+        switch (tp->HwCurrIsrVer) {
+        case 2:
+        case 3:
+        case 4:
+        case 5:
+        case 6:
+                RTL_W32(tp, IMR_V2_SET_REG_8125, tp->intr_mask);
+                break;
+        case 1:
+                RTL_W32(tp, tp->imr_reg[0], tp->intr_mask);
+
+                if (R8127_MULTI_RX_Q(tp)) {
+                        int i;
+                        for (i=1; i<tp->num_rx_rings; i++)
+                                RTL_W16(tp, tp->imr_reg[i], other_q_intr_mask);
+                }
+                break;
+        }
+
+#ifdef ENABLE_DASH_SUPPORT
+        if (tp->DASH)
+                rtl8127_enable_dash2_interrupt(tp);
+#endif
+}
+
+static inline void rtl8127_clear_hw_isr_v2(struct rtl8127_private *tp,
+                u32 message_id)
+{
+        RTL_W32(tp, ISR_V2_8125, BIT(message_id));
+}
+
+static inline void
+rtl8127_disable_hw_interrupt(struct rtl8127_private *tp)
+{
+        if (tp->HwCurrIsrVer > 1) {
+                RTL_W32(tp, IMR_V2_CLEAR_REG_8125, 0xFFFFFFFF);
+                if (tp->HwCurrIsrVer > 3)
+                        RTL_W32(tp, IMR_V4_L2_CLEAR_REG_8125, 0xFFFFFFFF);
+        } else {
+                RTL_W32(tp, tp->imr_reg[0], 0x0000);
+
+                if (R8127_MULTI_RX_Q(tp)) {
+                        int i;
+                        for (i=1; i<tp->num_rx_rings; i++)
+                                RTL_W16(tp, tp->imr_reg[i], 0);
+                }
+
+#ifdef ENABLE_DASH_SUPPORT
+                rtl8127_disable_dash2_interrupt(tp);
+#endif
+        }
+}
+
+static inline void
+rtl8127_switch_to_hw_interrupt(struct rtl8127_private *tp)
+{
+        RTL_W32(tp, TIMER_INT0_8125, 0x0000);
+
+        rtl8127_enable_hw_interrupt(tp);
+}
+
+static inline void
+rtl8127_switch_to_timer_interrupt(struct rtl8127_private *tp)
+{
+        if (tp->use_timer_interrupt) {
+                RTL_W32(tp, TIMER_INT0_8125, timer_count);
+                RTL_W32(tp, TCTR0_8125, timer_count);
+                RTL_W32(tp, tp->imr_reg[0], tp->timer_intr_mask);
+        } else {
+                rtl8127_switch_to_hw_interrupt(tp);
+        }
+}
+
+static void
+rtl8127_irq_mask_and_ack(struct rtl8127_private *tp)
+{
+        rtl8127_disable_hw_interrupt(tp);
+
+        if (tp->HwCurrIsrVer > 1) {
+                RTL_W32(tp, ISR_V2_8125, 0xFFFFFFFF);
+                if (tp->HwCurrIsrVer > 3)
+                        RTL_W32(tp, ISR_V4_L2_8125, 0xFFFFFFFF);
+        } else {
+                if (R8127_MULTI_RX_Q(tp)) {
+                        int i;
+                        for (i=1; i<tp->num_rx_rings; i++)
+                                RTL_W16(tp, tp->isr_reg[i], RTL_R16(tp, tp->isr_reg[i]));
+                }
+        }
+
+#ifdef ENABLE_DASH_SUPPORT
+        rtl8127_clear_ipc2_isr(tp);
+#endif
+}
+
+static void
+rtl8127_disable_rx_packet_filter(struct rtl8127_private *tp)
+{
+        RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) &
+                ~(AcceptErr | AcceptRunt |AcceptBroadcast | AcceptMulticast |
+                  AcceptMyPhys |  AcceptAllPhys));
+}
+
+static void
+rtl8127_nic_reset(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        rtl8127_disable_rx_packet_filter(tp);
+
+        rtl8127_enable_rxdvgate(dev);
+
+        rtl8127_stop_all_request(dev);
+
+        rtl8127_wait_txrx_fifo_empty(dev);
+
+        rtl8127_clear_stop_all_request(dev);
+
+        /* Soft reset the chip. */
+        RTL_W8(tp, ChipCmd, CmdReset);
+
+        /* Check that the chip has finished the reset. */
+        for (i = 100; i > 0; i--) {
+                udelay(100);
+                if ((RTL_R8(tp, ChipCmd) & CmdReset) == 0)
+                        break;
+        }
+
+        /* reset rcr */
+        RTL_W32(tp, RxConfig, (RX_DMA_BURST_512 << RxCfgDMAShift));
+}
+
+static void
+rtl8127_hw_set_interrupt_type(struct rtl8127_private *tp, u8 isr_ver)
+{
+        u8 tmp;
+
+        if (tp->HwSuppIsrVer < 2)
+                return;
+
+        tmp = RTL_R8(tp, INT_CFG0_8125);
+
+        switch (tp->HwSuppIsrVer) {
+        case 6:
+                tmp &= ~INT_CFG0_AVOID_MISS_INTR;
+                fallthrough;
+        case 4:
+        case 5:
+                if (tp->HwSuppIsrVer == 6)
+                        tmp &= ~INT_CFG0_AUTO_CLEAR_IMR;
+                else
+                        tmp &= ~INT_CFG0_MSIX_ENTRY_NUM_MODE;
+                fallthrough;
+        case 2:
+        case 3:
+                tmp &= ~(INT_CFG0_ENABLE_8125);
+                if (isr_ver > 1)
+                        tmp |= INT_CFG0_ENABLE_8125;
+                break;
+        default:
+                return;
+        }
+
+        RTL_W8(tp, INT_CFG0_8125, tmp);
+}
+
+static void
+rtl8127_hw_clear_timer_int(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        RTL_W32(tp, TIMER_INT0_8125, 0x0000);
+        RTL_W32(tp, TIMER_INT1_8125, 0x0000);
+        RTL_W32(tp, TIMER_INT2_8125, 0x0000);
+        RTL_W32(tp, TIMER_INT3_8125, 0x0000);
+}
+
+static void
+rtl8127_hw_clear_int_miti(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        switch (tp->HwSuppIntMitiVer) {
+        case 3:
+        case 6:
+                //IntMITI_0-IntMITI_31
+                for (i=0xA00; i<0xB00; i+=4)
+                        RTL_W32(tp, i, 0x0000);
+                break;
+        case 4:
+        case 5:
+                //IntMITI_0-IntMITI_15
+                for (i = 0xA00; i < 0xA80; i += 4)
+                        RTL_W32(tp, i, 0x0000);
+
+                if (tp->HwSuppIntMitiVer == 5)
+                        RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) &
+                               ~(INT_CFG0_TIMEOUT0_BYPASS_8125 |
+                                 INT_CFG0_MITIGATION_BYPASS_8125 |
+                                 INT_CFG0_RDU_BYPASS_8126));
+                else
+                        RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) &
+                               ~(INT_CFG0_TIMEOUT0_BYPASS_8125 | INT_CFG0_MITIGATION_BYPASS_8125));
+
+                RTL_W16(tp, INT_CFG1_8125, 0x0000);
+                break;
+        }
+}
+
+static bool
+rtl8127_vec_2_tx_q_num(
+        struct rtl8127_private *tp,
+        u32 messageId,
+        u32 *qnum
+)
+{
+        u32 whichQ = 0xffffffff;
+        bool rc = false;
+
+        switch (tp->HwSuppIsrVer) {
+        case 2:
+                if (messageId == 0x10)
+                        whichQ = 0;
+                else if (messageId == 0x12 && tp->num_tx_rings > 1)
+                        whichQ = 1;
+                break;
+        case 3:
+        case 4:
+                if (messageId == 0x00)
+                        whichQ = 0;
+                else if (messageId == 0x01 && tp->num_tx_rings > 1)
+                        whichQ = 1;
+                break;
+        case 5:
+                if (messageId == 0x10)
+                        whichQ = 0;
+                else if (messageId == 0x11 && tp->num_tx_rings > 1)
+                        whichQ = 1;
+                break;
+        case 6:
+                if (messageId == 0x08)
+                        whichQ = 0;
+                else if (messageId == 0x09 && tp->num_tx_rings > 1)
+                        whichQ = 1;
+                break;
+        case 7:
+                if (messageId == 0x1B)
+                        whichQ = 0;
+                else if (messageId == 0x1C && tp->num_tx_rings > 1)
+                        whichQ = 1;
+                break;
+        }
+
+        if (whichQ != 0xffffffff) {
+                *qnum = whichQ;
+                rc = true;
+        }
+
+        return rc;
+}
+
+static bool
+rtl8127_vec_2_rx_q_num(
+        struct rtl8127_private *tp,
+        u32 messageId,
+        u32 *qnum
+)
+{
+        u32 whichQ = 0xffffffff;
+        bool rc = false;
+
+        switch (tp->HwSuppIsrVer) {
+        case 2:
+        case 3:
+        case 4:
+        case 5:
+        case 6:
+        case 7:
+                if (messageId < tp->HwSuppNumRxQueues)
+                        whichQ = messageId;
+                break;
+        }
+
+        if (whichQ != 0xffffffff) {
+                *qnum = whichQ;
+                rc = true;
+        }
+
+        return rc;
+}
+
+void
+rtl8127_hw_set_timer_int(struct rtl8127_private *tp,
+                         u32 message_id,
+                         u8 timer_intmiti_val)
+{
+        u32 qnum;
+
+        switch (tp->HwSuppIntMitiVer) {
+        case 4:
+        case 5:
+        case 6:
+                //ROK
+                if (rtl8127_vec_2_rx_q_num(tp, message_id, &qnum))
+                        RTL_W8(tp,INT_MITI_V2_0_RX + 8 * qnum, timer_intmiti_val);
+                //TOK
+                if (rtl8127_vec_2_tx_q_num(tp, message_id, &qnum))
+                        RTL_W8(tp,INT_MITI_V2_0_TX + 8 * qnum, timer_intmiti_val);
+                break;
+        }
+}
+
+void
+rtl8127_hw_reset(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_lib_reset_prepare(tp);
+
+        /* Disable interrupts */
+        rtl8127_irq_mask_and_ack(tp);
+
+        rtl8127_hw_clear_timer_int(dev);
+
+        rtl8127_nic_reset(dev);
+}
+
+static unsigned int
+rtl8127_xmii_reset_pending(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned int retval;
+        unsigned long flags;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+        rtl8127_mdio_write(tp, 0x1f, 0x0000);
+        retval = rtl8127_mdio_read(tp, MII_BMCR) & BMCR_RESET;
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return retval;
+}
+
+static unsigned int
+rtl8127_xmii_link_ok(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u32 status;
+
+        status = rtl8127_get_phy_status(tp);
+        if (status == UINT_MAX)
+                return 0;
+
+        return (status & LinkStatus) ? 1 : 0;
+}
+
+static int
+rtl8127_wait_phy_reset_complete(struct rtl8127_private *tp)
+{
+        int i, val;
+
+        for (i = 0; i < 2500; i++) {
+                val = rtl8127_mdio_read(tp, MII_BMCR) & BMCR_RESET;
+                if (!val)
+                        return 0;
+
+                mdelay(1);
+        }
+
+        return -1;
+}
+
+static void
+rtl8127_xmii_reset_enable(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+        int ret;
+
+        if (rtl8127_is_in_phy_disable_mode(dev))
+                return;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        rtl8127_mdio_write(tp, 0x1f, 0x0000);
+        rtl8127_mdio_write(tp, MII_ADVERTISE, rtl8127_mdio_read(tp, MII_ADVERTISE) &
+                           ~(ADVERTISE_10HALF | ADVERTISE_10FULL |
+                             ADVERTISE_100HALF | ADVERTISE_100FULL));
+        rtl8127_mdio_write(tp, MII_CTRL1000, rtl8127_mdio_read(tp, MII_CTRL1000) &
+                           ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL));
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4) &
+                                          ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL |
+                                            RTK_ADVERTISE_10000FULL));
+        rtl8127_mdio_write(tp, MII_BMCR, BMCR_RESET | BMCR_ANENABLE);
+
+        ret = rtl8127_wait_phy_reset_complete(tp);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        if (ret != 0 && netif_msg_link(tp))
+                printk(KERN_ERR "%s: PHY reset failed.\n", dev->name);
+}
+
+void
+rtl8127_init_ring_indexes(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < tp->HwSuppNumTxQueues; i++) {
+                struct rtl8127_tx_ring *ring = &tp->tx_ring[i];
+                ring->dirty_tx = ring->cur_tx = 0;
+                ring->NextHwDesCloPtr = 0;
+                ring->BeginHwDesCloPtr = 0;
+                ring->index = i;
+                ring->priv = tp;
+                ring->netdev = tp->dev;
+
+                /* reset BQL for queue */
+                netdev_tx_reset_queue(txring_txq(ring));
+        }
+
+        for (i = 0; i < tp->HwSuppNumRxQueues; i++) {
+                struct rtl8127_rx_ring *ring = &tp->rx_ring[i];
+                ring->dirty_rx = ring->cur_rx = 0;
+                ring->index = i;
+                ring->priv = tp;
+                ring->netdev = tp->dev;
+        }
+
+#ifdef ENABLE_LIB_SUPPORT
+        for (i = 0; i < tp->HwSuppNumTxQueues; i++) {
+                struct rtl8127_ring *ring = &tp->lib_tx_ring[i];
+                ring->direction = RTL8127_CH_DIR_TX;
+                ring->queue_num = i;
+                ring->private = tp;
+        }
+
+        for (i = 0; i < tp->HwSuppNumRxQueues; i++) {
+                struct rtl8127_ring *ring = &tp->lib_rx_ring[i];
+                ring->direction = RTL8127_CH_DIR_RX;
+                ring->queue_num = i;
+                ring->private = tp;
+        }
+#endif
+}
+
+static void
+rtl8127_issue_offset_99_event(struct rtl8127_private *tp)
+{
+        rtl8127_mac_ocp_write(tp, 0xE09A,  rtl8127_mac_ocp_read(tp, 0xE09A) | BIT_0);
+}
+
+#ifdef ENABLE_DASH_SUPPORT
+static void
+rtl8127_check_and_enable_dash_interrupt(struct rtl8127_private *tp)
+{
+        if (!HW_DASH_SUPPORT_IPC2(tp))
+                return;
+
+        if (!tp->DASH)
+                return;
+
+        //
+        // even disconnected, enable dash interrupt mask bits for in-band/out-band communication
+        //
+        rtl8127_enable_dash2_interrupt(tp);
+        if (tp->HwCurrIsrVer > 1)
+                RTL_W32(tp, IMR_V4_L2_SET_REG_8125, ISRIMR_V6_L2_MISC_INTR);
+        else
+                RTL_W16(tp, tp->imr_reg[0], ISRIMR_DASH_INTR_EN);
+}
+#endif
+
+static int rtl8127_enable_eee_plus(struct rtl8127_private *tp)
+{
+        rtl8127_mac_ocp_write(tp, 0xE080, rtl8127_mac_ocp_read(tp, 0xE080)|BIT_1);
+
+        return 0;
+}
+
+static int rtl8127_disable_eee_plus(struct rtl8127_private *tp)
+{
+        rtl8127_mac_ocp_write(tp, 0xE080, rtl8127_mac_ocp_read(tp, 0xE080)&~BIT_1);
+
+        return 0;
+}
+
+static void rtl8127_enable_double_vlan(struct rtl8127_private *tp)
+{
+        RTL_W16(tp, DOUBLE_VLAN_CONFIG, 0xf002);
+}
+
+static void rtl8127_disable_double_vlan(struct rtl8127_private *tp)
+{
+        RTL_W16(tp, DOUBLE_VLAN_CONFIG, 0);
+}
+
+static void rtl8127_set_radm_fifo_prot(struct rtl8127_private *tp, bool enable)
+{
+        switch (tp->mcfg) {
+        case CFG_METHOD_2:
+                RTL_W16(tp, RADMFIFO_PROTECT, enable ? 0x2001 : 0);
+                break;
+        default:
+                break;
+        }
+}
+
+static void
+rtl8127_link_on_patch(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+        u32 status;
+
+        rtl8127_hw_config(dev);
+
+        status = rtl8127_get_phy_status(tp);
+        if (status & _10bps)
+                rtl8127_enable_eee_plus(tp);
+
+        if (status & (_1000bpsL | _2500bpsL | _10bps | _100bps | _1000bpsF))
+                rtl8127_set_radm_fifo_prot(tp, 1);
+        else
+                rtl8127_set_radm_fifo_prot(tp, 0);
+
+        rtl8127_hw_start(dev);
+
+        netif_carrier_on(dev);
+
+        netif_tx_wake_all_queues(dev);
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+        tp->phy_reg_aner = rtl8127_mdio_read(tp, MII_EXPANSION);
+        tp->phy_reg_anlpar = rtl8127_mdio_read(tp, MII_LPA);
+        tp->phy_reg_gbsr = rtl8127_mdio_read(tp, MII_STAT1000);
+        tp->phy_reg_status_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D6);
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+#ifdef ENABLE_PTP_SUPPORT
+        if (tp->EnablePtp)
+                rtl8127_set_local_time(tp);
+#endif // ENABLE_PTP_SUPPORT
+}
+
+static void
+rtl8127_link_down_patch(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        tp->phy_reg_aner = 0;
+        tp->phy_reg_anlpar = 0;
+        tp->phy_reg_gbsr = 0;
+        tp->phy_reg_status_2500 = 0;
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        rtl8127_disable_eee_plus(tp);
+
+        netif_carrier_off(dev);
+
+        netif_tx_disable(dev);
+
+        rtl8127_hw_reset(dev);
+
+        rtl8127_tx_clear(tp);
+
+        rtl8127_rx_clear(tp);
+
+        rtl8127_init_ring(dev);
+
+        rtl8127_enable_hw_linkchg_interrupt(tp);
+
+        //rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising);
+
+#ifdef ENABLE_DASH_SUPPORT
+        rtl8127_check_and_enable_dash_interrupt(tp);
+#endif
+}
+
+static void
+_rtl8127_check_link_status(struct net_device *dev, unsigned int link_state)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (link_state != R8127_LINK_STATE_OFF &&
+            link_state != R8127_LINK_STATE_ON)
+                link_state = tp->link_ok(dev);
+
+        if (link_state == R8127_LINK_STATE_ON) {
+                rtl8127_link_on_patch(dev);
+
+                if (netif_msg_ifup(tp))
+                        printk(KERN_INFO PFX "%s: link up\n", dev->name);
+        } else {
+                if (netif_msg_ifdown(tp))
+                        printk(KERN_INFO PFX "%s: link down\n", dev->name);
+
+                rtl8127_link_down_patch(dev);
+        }
+}
+
+static void
+rtl8127_check_link_status(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned int link_status_on;
+
+        tp->resume_not_chg_speed = 0;
+
+        link_status_on = tp->link_ok(dev);
+        if (netif_carrier_ok(dev) == link_status_on)
+                return;
+
+        _rtl8127_check_link_status(dev, link_status_on);
+}
+
+static bool
+rtl8127_is_autoneg_mode_valid(u32 autoneg)
+{
+        switch(autoneg) {
+        case AUTONEG_ENABLE:
+        case AUTONEG_DISABLE:
+                return true;
+        default:
+                return false;
+        }
+}
+
+static bool
+rtl8127_is_speed_mode_valid(struct rtl8127_private *tp, u32 speed)
+{
+        if (HW_FIBER_MODE_ENABLED(tp)) {
+                switch(speed) {
+                case SPEED_10000:
+                case SPEED_1000:
+                        return true;
+                default:
+                        return false;
+                }
+        } else {
+                switch(speed) {
+                case SPEED_10000:
+                case SPEED_5000:
+                case SPEED_2500:
+                case SPEED_1000:
+                case SPEED_100:
+                case SPEED_10:
+                        return true;
+                default:
+                        return false;
+                }
+        }
+}
+
+static bool
+rtl8127_is_duplex_mode_valid(u8 duplex)
+{
+        switch(duplex) {
+        case DUPLEX_FULL:
+        case DUPLEX_HALF:
+                return true;
+        default:
+                return false;
+        }
+}
+
+static void
+rtl8127_set_link_option(struct rtl8127_private *tp,
+                        u8 autoneg,
+                        u32 speed,
+                        u8 duplex,
+                        enum rtl8127_fc_mode fc)
+{
+        u64 adv;
+
+        if (!rtl8127_is_speed_mode_valid(tp, speed))
+                speed = SPEED_10000;
+
+        if (!rtl8127_is_duplex_mode_valid(duplex))
+                duplex = DUPLEX_FULL;
+
+        if (!rtl8127_is_autoneg_mode_valid(autoneg))
+                autoneg = AUTONEG_ENABLE;
+
+        speed = min(speed, tp->HwSuppMaxPhyLinkSpeed);
+
+        adv = 0;
+        if (HW_FIBER_MODE_ENABLED(tp)) {
+                switch(speed) {
+                case SPEED_10000:
+                        adv |= ADVERTISED_10000baseT_Full;
+                        fallthrough;
+                case SPEED_1000:
+                        adv |= ADVERTISED_1000baseT_Full;
+                        break;
+                default:
+                        break;
+                }
+        } else {
+                switch(speed) {
+                case SPEED_10000:
+                        adv |= ADVERTISED_10000baseT_Full;
+                        fallthrough;
+                case SPEED_5000:
+                        adv |= RTK_ADVERTISED_5000baseX_Full;
+                        fallthrough;
+                case SPEED_2500:
+                        adv |= ADVERTISED_2500baseX_Full;
+                        fallthrough;
+                default:
+                        adv |= (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
+                                ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
+                                ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full);
+                        break;
+                }
+        }
+
+        tp->autoneg = autoneg;
+        tp->speed = speed;
+        tp->duplex = duplex;
+        tp->advertising = adv;
+        tp->fcpause = fc;
+}
+
+static void
+rtl8127_wait_ll_share_fifo_ready(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        for (i = 0; i < 10; i++) {
+                udelay(100);
+                if (RTL_R16(tp, 0xD2) & BIT_9)
+                        break;
+        }
+}
+
+static void
+rtl8127_disable_pci_offset_99(struct rtl8127_private *tp)
+{
+        rtl8127_mac_ocp_write(tp, 0xE032, rtl8127_mac_ocp_read(tp, 0xE032) & ~(BIT_0 | BIT_1));
+
+        rtl8127_csi_fun0_write_byte(tp, 0x99, 0x00);
+}
+
+static void
+rtl8127_enable_pci_offset_99(struct rtl8127_private *tp)
+{
+        u32 csi_tmp;
+
+        rtl8127_csi_fun0_write_byte(tp, 0x99, tp->org_pci_offset_99);
+
+        csi_tmp = rtl8127_mac_ocp_read(tp, 0xE032);
+        csi_tmp &= ~(BIT_0 | BIT_1);
+        if (tp->org_pci_offset_99 & (BIT_5 | BIT_6))
+                csi_tmp |= BIT_1;
+        if (tp->org_pci_offset_99 & BIT_2)
+                csi_tmp |= BIT_0;
+        rtl8127_mac_ocp_write(tp, 0xE032, csi_tmp);
+}
+
+static void
+rtl8127_init_pci_offset_99(struct rtl8127_private *tp)
+{
+        rtl8127_mac_ocp_write(tp, 0xCDD0, 0x9003);
+        rtl8127_set_mac_ocp_bit(tp, 0xE034, (BIT_15 | BIT_14));
+        rtl8127_mac_ocp_write(tp, 0xCDD2, 0x8C17);
+        rtl8127_mac_ocp_write(tp, 0xCDD8, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDD4, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDDA, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDD6, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDDC, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDE8, 0x8C08);
+        rtl8127_mac_ocp_write(tp, 0xCDEA, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDEC, 0x8C12);
+        rtl8127_mac_ocp_write(tp, 0xCDEE, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDF0, 0x8C2E);
+        rtl8127_mac_ocp_write(tp, 0xCDF2, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDF4, 0x8892);
+        rtl8127_mac_ocp_write(tp, 0xCDF6, 0x9003);
+        rtl8127_mac_ocp_write(tp, 0xCDF4, 0x8849);
+        rtl8127_mac_ocp_write(tp, 0xCDF6, 0x9003);
+        rtl8127_set_mac_ocp_bit(tp, 0xE032, BIT_14);
+        rtl8127_set_mac_ocp_bit(tp, 0xE0A2, BIT_0);
+
+        rtl8127_enable_pci_offset_99(tp);
+}
+
+static void
+rtl8127_disable_pci_offset_180(struct rtl8127_private *tp)
+{
+        rtl8127_clear_mac_ocp_bit(tp, 0xE092, 0x00FF);
+}
+
+static void
+rtl8127_enable_pci_offset_180(struct rtl8127_private *tp)
+{
+        rtl8127_clear_mac_ocp_bit(tp, 0xE094, 0xFF00);
+
+        rtl8127_clear_set_mac_ocp_bit(tp, 0xE092, 0x00FF, BIT_2);
+}
+
+static void
+rtl8127_init_pci_offset_180(struct rtl8127_private *tp)
+{
+        rtl8127_enable_pci_offset_180(tp);
+}
+
+static void
+rtl8127_set_pci_99_exit_driver_para(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (tp->org_pci_offset_99 & BIT_2)
+                rtl8127_issue_offset_99_event(tp);
+        rtl8127_disable_pci_offset_99(tp);
+}
+
+static void
+rtl8127_enable_cfg9346_write(struct rtl8127_private *tp)
+{
+        RTL_W8(tp, Cfg9346, RTL_R8(tp, Cfg9346) | Cfg9346_Unlock);
+}
+
+static void
+rtl8127_disable_cfg9346_write(struct rtl8127_private *tp)
+{
+        RTL_W8(tp, Cfg9346, RTL_R8(tp, Cfg9346) & ~Cfg9346_Unlock);
+}
+
+static void
+rtl8127_enable_exit_l1_mask(struct rtl8127_private *tp)
+{
+        //(1)ERI(0xD4)(OCP 0xC0AC).bit[7:12]=6'b111111, L1 Mask
+        rtl8127_set_mac_ocp_bit(tp, 0xC0AC, (BIT_7 | BIT_8 | BIT_9 | BIT_10 | BIT_11 | BIT_12));
+}
+
+static void
+rtl8127_disable_exit_l1_mask(struct rtl8127_private *tp)
+{
+        //(1)ERI(0xD4)(OCP 0xC0AC).bit[7:12]=6'b000000, L1 Mask
+        rtl8127_clear_mac_ocp_bit(tp, 0xC0AC, (BIT_7 | BIT_8 | BIT_9 | BIT_10 | BIT_11 | BIT_12));
+}
+
+static void
+rtl8127_enable_extend_tally_couter(struct rtl8127_private *tp)
+{
+        switch (tp->HwSuppExtendTallyCounterVer) {
+        case 1:
+                rtl8127_set_mac_ocp_bit(tp, 0xEA84, (BIT_1 | BIT_0));
+                break;
+        }
+}
+
+static void
+rtl8127_disable_extend_tally_couter(struct rtl8127_private *tp)
+{
+        switch (tp->HwSuppExtendTallyCounterVer) {
+        case 1:
+                rtl8127_clear_mac_ocp_bit(tp, 0xEA84, (BIT_1 | BIT_0));
+                break;
+        }
+}
+
+static void
+rtl8127_enable_force_clkreq(struct rtl8127_private *tp, bool enable)
+{
+        if (enable)
+                RTL_W8(tp, 0xF1, RTL_R8(tp, 0xF1) | BIT_7);
+        else
+                RTL_W8(tp, 0xF1, RTL_R8(tp, 0xF1) & ~BIT_7);
+}
+
+static void
+rtl8127_enable_aspm_clkreq_lock(struct rtl8127_private *tp, bool enable)
+{
+        bool unlock_cfg_wr;
+
+        if ((RTL_R8(tp, Cfg9346) & Cfg9346_EEM_MASK) == Cfg9346_Unlock)
+                unlock_cfg_wr = false;
+        else
+                unlock_cfg_wr = true;
+
+        if (unlock_cfg_wr)
+                rtl8127_enable_cfg9346_write(tp);
+
+        if (enable) {
+                RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) | BIT_3);
+                RTL_W8(tp, Config5, RTL_R8(tp, Config5) | BIT_0);
+        } else {
+                RTL_W8(tp, INT_CFG0_8125, RTL_R8(tp, INT_CFG0_8125) & ~BIT_3);
+                RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~BIT_0);
+        }
+
+        if (unlock_cfg_wr)
+                rtl8127_disable_cfg9346_write(tp);
+}
+
+static void
+rtl8127_hw_d3_para(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        RTL_W16(tp, RxMaxSize, RX_BUF_SIZE);
+
+        rtl8127_enable_force_clkreq(tp, 0);
+        rtl8127_enable_aspm_clkreq_lock(tp, 0);
+
+        rtl8127_disable_exit_l1_mask(tp);
+
+#ifdef ENABLE_REALWOW_SUPPORT
+        rtl8127_set_realwow_d3_para(dev);
+#endif
+
+        rtl8127_set_pci_99_exit_driver_para(dev);
+
+        rtl8127_disable_rxdvgate(dev);
+
+        rtl8127_disable_extend_tally_couter(tp);
+
+        rtl8127_set_radm_fifo_prot(tp, 1);
+}
+
+static void
+rtl8127_enable_magic_packet(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        switch (tp->HwSuppMagicPktVer) {
+        case WAKEUP_MAGIC_PACKET_V3:
+                rtl8127_mac_ocp_write(tp, 0xC0B6, rtl8127_mac_ocp_read(tp, 0xC0B6) | BIT_0);
+                break;
+        }
+}
+static void
+rtl8127_disable_magic_packet(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        switch (tp->HwSuppMagicPktVer) {
+        case WAKEUP_MAGIC_PACKET_V3:
+                rtl8127_mac_ocp_write(tp, 0xC0B6, rtl8127_mac_ocp_read(tp, 0xC0B6) & ~BIT_0);
+                break;
+        }
+}
+
+static void
+rtl8127_enable_linkchg_wakeup(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        switch (tp->HwSuppLinkChgWakeUpVer) {
+        case 3:
+                RTL_W8(tp, Config3, RTL_R8(tp, Config3) | LinkUp);
+                rtl8127_clear_set_mac_ocp_bit(tp, 0xE0C6, (BIT_5 | BIT_3 | BIT_2), (BIT_4 | BIT_1 | BIT_0));
+                break;
+        }
+}
+
+static void
+rtl8127_disable_linkchg_wakeup(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        switch (tp->HwSuppLinkChgWakeUpVer) {
+        case 3:
+                RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~LinkUp);
+                if (!(rtl8127_mac_ocp_read(tp, 0xE0C6) & BIT_0))
+                        rtl8127_clear_set_mac_ocp_bit(tp, 0xE0C6, (BIT_5 | BIT_3 | BIT_2 | BIT_1), BIT_4);
+                break;
+        }
+}
+
+#define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST)
+
+static u32
+rtl8127_get_hw_wol(struct rtl8127_private *tp)
+{
+        u8 options;
+        u32 csi_tmp;
+        u32 wol_opts = 0;
+
+        if (disable_wol_support)
+                goto out;
+
+        options = RTL_R8(tp, Config1);
+        if (!(options & PMEnable))
+                goto out;
+
+        options = RTL_R8(tp, Config3);
+        if (options & LinkUp)
+                wol_opts |= WAKE_PHY;
+
+        switch (tp->HwSuppMagicPktVer) {
+        case WAKEUP_MAGIC_PACKET_V3:
+                csi_tmp = rtl8127_mac_ocp_read(tp, 0xC0B6);
+                if (csi_tmp & BIT_0)
+                        wol_opts |= WAKE_MAGIC;
+                break;
+        }
+
+        options = RTL_R8(tp, Config5);
+        if (options & UWF)
+                wol_opts |= WAKE_UCAST;
+        if (options & BWF)
+                wol_opts |= WAKE_BCAST;
+        if (options & MWF)
+                wol_opts |= WAKE_MCAST;
+
+out:
+        return wol_opts;
+}
+
+static void
+rtl8127_enable_d0_speedup(struct rtl8127_private *tp)
+{
+        u16 clearmask;
+        u16 setmask;
+
+        if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp))
+                return;
+
+        if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_DISABLE)
+                return;
+
+        if (tp->HwSuppD0SpeedUpVer == 1 || tp->HwSuppD0SpeedUpVer == 2) {
+                //speed up speed
+                clearmask = (BIT_10 | BIT_9 | BIT_8 | BIT_7);
+                if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_2500)
+                        setmask = BIT_7;
+                else if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_5000)
+                        setmask = BIT_8;
+                else if (tp->D0SpeedUpSpeed == D0_SPEED_UP_SPEED_10000)
+                        setmask = BIT_7 | BIT_8;
+                else
+                        setmask = 0;
+                rtl8127_clear_set_mac_ocp_bit(tp, 0xE10A, clearmask, setmask);
+
+                //speed up flowcontrol
+                clearmask = (BIT_15 | BIT_14);
+                if (tp->HwSuppD0SpeedUpVer == 2)
+                        clearmask |= BIT_13;
+
+                if (tp->fcpause == rtl8127_fc_full) {
+                        setmask = (BIT_15 | BIT_14);
+                        if (tp->HwSuppD0SpeedUpVer == 2)
+                                setmask |= BIT_13;
+                } else
+                        setmask = 0;
+                rtl8127_clear_set_mac_ocp_bit(tp, 0xE860, clearmask, setmask);
+        }
+
+        RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) | BIT_3);
+}
+
+static void
+rtl8127_disable_d0_speedup(struct rtl8127_private *tp)
+{
+        if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp))
+                return;
+
+        RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) & ~BIT_3);
+}
+
+static void
+rtl8127_set_hw_wol(struct net_device *dev, u32 wolopts)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i,tmp = 0;
+        static struct {
+                u32 opt;
+                u16 reg;
+                u8  mask;
+        } cfg[] = {
+                { WAKE_PHY,   Config3, LinkUp },
+                { WAKE_UCAST, Config5, UWF },
+                { WAKE_BCAST, Config5, BWF },
+                { WAKE_MCAST, Config5, MWF },
+                { WAKE_ANY,   Config5, LanWake },
+                { WAKE_MAGIC, Config3, MagicPacket },
+        };
+
+        switch (tp->HwSuppMagicPktVer) {
+        case WAKEUP_MAGIC_PACKET_V3:
+                tmp = ARRAY_SIZE(cfg) - 1;
+
+                if (wolopts & WAKE_MAGIC)
+                        rtl8127_enable_magic_packet(dev);
+                else
+                        rtl8127_disable_magic_packet(dev);
+                break;
+        default:
+                break;
+        }
+
+        rtl8127_enable_cfg9346_write(tp);
+
+        for (i = 0; i < tmp; i++) {
+                u8 options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask;
+                if (wolopts & cfg[i].opt)
+                        options |= cfg[i].mask;
+                RTL_W8(tp, cfg[i].reg, options);
+        }
+
+        switch (tp->HwSuppLinkChgWakeUpVer) {
+        case 3:
+                if (wolopts & WAKE_PHY)
+                        rtl8127_enable_linkchg_wakeup(dev);
+                else
+                        rtl8127_disable_linkchg_wakeup(dev);
+                break;
+        }
+
+        rtl8127_disable_cfg9346_write(tp);
+}
+
+static void
+rtl8127_phy_restart_nway(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (rtl8127_is_in_phy_disable_mode(dev))
+                return;
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE | BMCR_ANRESTART);
+}
+
+static void
+rtl8127_phy_setup_force_mode(struct net_device *dev, u32 speed, u8 duplex)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u16 bmcr_true_force = 0;
+
+        if (rtl8127_is_in_phy_disable_mode(dev))
+                return;
+
+        if ((speed == SPEED_10) && (duplex == DUPLEX_HALF)) {
+                bmcr_true_force = BMCR_SPEED10;
+        } else if ((speed == SPEED_10) && (duplex == DUPLEX_FULL)) {
+                bmcr_true_force = BMCR_SPEED10 | BMCR_FULLDPLX;
+        } else if ((speed == SPEED_100) && (duplex == DUPLEX_HALF)) {
+                bmcr_true_force = BMCR_SPEED100;
+        } else if ((speed == SPEED_100) && (duplex == DUPLEX_FULL)) {
+                bmcr_true_force = BMCR_SPEED100 | BMCR_FULLDPLX;
+        } else {
+                netif_err(tp, drv, dev, "Failed to set phy force mode!\n");
+                return;
+        }
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        rtl8127_mdio_write(tp, MII_BMCR, bmcr_true_force);
+}
+
+static void
+rtl8127_set_pci_pme(struct rtl8127_private *tp, int set)
+{
+        struct pci_dev *pdev = tp->pci_dev;
+        u16 pmc;
+
+        if (!pdev->pm_cap)
+                return;
+
+        pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmc);
+        pmc |= PCI_PM_CTRL_PME_STATUS;
+        if (set)
+                pmc |= PCI_PM_CTRL_PME_ENABLE;
+        else
+                pmc &= ~PCI_PM_CTRL_PME_ENABLE;
+        pci_write_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, pmc);
+}
+
+static void
+rtl8127_enable_giga_lite(struct rtl8127_private *tp, u64 adv)
+{
+        if (adv & ADVERTISED_1000baseT_Full)
+                rtl8127_set_eth_phy_ocp_bit(tp, 0xA428, BIT_9);
+        else
+                rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_9);
+
+        if (adv & ADVERTISED_2500baseX_Full)
+                rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0);
+        else
+                rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0);
+
+        if (adv & RTK_ADVERTISED_5000baseX_Full)
+                rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_1);
+        else
+                rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_1);
+
+        if (adv & ADVERTISED_10000baseT_Full)
+                rtl8127_set_eth_phy_ocp_bit(tp, 0xA5EA, BIT_2);
+        else
+                rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_2);
+}
+
+static void
+rtl8127_disable_giga_lite(struct rtl8127_private *tp)
+{
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_9);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5EA, BIT_0 | BIT_1 | BIT_2);
+}
+
+static void
+rtl8127_set_wol_link_speed(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+        int auto_nego;
+        int giga_ctrl;
+        int ctrl_2500;
+        u64 adv;
+        u16 anlpar;
+        u16 gbsr;
+        u16 status_2500;
+        u16 aner;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        if (tp->autoneg != AUTONEG_ENABLE)
+                goto exit;
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+
+        auto_nego = rtl8127_mdio_read(tp, MII_ADVERTISE);
+        auto_nego &= ~(ADVERTISE_10HALF | ADVERTISE_10FULL
+                       | ADVERTISE_100HALF | ADVERTISE_100FULL);
+
+        giga_ctrl = rtl8127_mdio_read(tp, MII_CTRL1000);
+        giga_ctrl &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL);
+
+        ctrl_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4);
+        ctrl_2500 &= ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL |
+                       RTK_ADVERTISE_10000FULL);
+
+        aner = tp->phy_reg_aner;
+        anlpar = tp->phy_reg_anlpar;
+        gbsr = tp->phy_reg_gbsr;
+        status_2500 = tp->phy_reg_status_2500;
+        if (tp->link_ok(dev)) {
+                aner = rtl8127_mdio_read(tp, MII_EXPANSION);
+                anlpar = rtl8127_mdio_read(tp, MII_LPA);
+                gbsr = rtl8127_mdio_read(tp, MII_STAT1000);
+                status_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D6);
+        }
+
+        adv = tp->advertising;
+        if ((aner | anlpar | gbsr | status_2500) == 0) {
+                int auto_nego_tmp = 0;
+                if (adv & ADVERTISED_10baseT_Half)
+                        auto_nego_tmp |= ADVERTISE_10HALF;
+                if (adv & ADVERTISED_10baseT_Full)
+                        auto_nego_tmp |= ADVERTISE_10FULL;
+                if (adv & ADVERTISED_100baseT_Half)
+                        auto_nego_tmp |= ADVERTISE_100HALF;
+                if (adv & ADVERTISED_100baseT_Full)
+                        auto_nego_tmp |= ADVERTISE_100FULL;
+
+                if (auto_nego_tmp == 0)
+                        goto exit;
+
+                auto_nego |= auto_nego_tmp;
+                goto skip_check_lpa;
+        }
+        if (!(aner & EXPANSION_NWAY))
+                goto exit;
+
+        if ((adv & ADVERTISED_10baseT_Half) && (anlpar & LPA_10HALF))
+                auto_nego |= ADVERTISE_10HALF;
+        else if ((adv & ADVERTISED_10baseT_Full) && (anlpar & LPA_10FULL))
+                auto_nego |= ADVERTISE_10FULL;
+        else if ((adv & ADVERTISED_100baseT_Half) && (anlpar & LPA_100HALF))
+                auto_nego |= ADVERTISE_100HALF;
+        else if ((adv & ADVERTISED_100baseT_Full) && (anlpar & LPA_100FULL))
+                auto_nego |= ADVERTISE_100FULL;
+        else if (adv & ADVERTISED_1000baseT_Half && (gbsr & LPA_1000HALF))
+                giga_ctrl |= ADVERTISE_1000HALF;
+        else if (adv & ADVERTISED_1000baseT_Full && (gbsr & LPA_1000FULL))
+                giga_ctrl |= ADVERTISE_1000FULL;
+        else if (adv & ADVERTISED_2500baseX_Full && (status_2500 & RTK_LPA_ADVERTISE_2500FULL))
+                ctrl_2500 |= RTK_ADVERTISE_2500FULL;
+        else if (adv & RTK_ADVERTISED_5000baseX_Full && (status_2500 & RTK_LPA_ADVERTISE_5000FULL))
+                ctrl_2500 |= RTK_ADVERTISE_5000FULL;
+        else if (adv & ADVERTISED_10000baseT_Full && (status_2500 & RTK_LPA_ADVERTISE_10000FULL))
+                ctrl_2500 |= RTK_ADVERTISE_10000FULL;
+        else
+                goto exit;
+
+skip_check_lpa:
+        if (tp->DASH)
+                auto_nego |= (ADVERTISE_100FULL | ADVERTISE_100HALF | ADVERTISE_10HALF | ADVERTISE_10FULL);
+
+#ifdef CONFIG_DOWN_SPEED_100
+        auto_nego |= (ADVERTISE_100FULL | ADVERTISE_100HALF | ADVERTISE_10HALF | ADVERTISE_10FULL);
+#endif
+
+        rtl8127_mdio_write(tp, MII_ADVERTISE, auto_nego);
+        rtl8127_mdio_write(tp, MII_CTRL1000, giga_ctrl);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, ctrl_2500);
+
+        rtl8127_disable_giga_lite(tp);
+
+        rtl8127_phy_restart_nway(dev);
+
+exit:
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return;
+}
+
+static bool
+rtl8127_keep_wol_link_speed(struct net_device *dev, u8 from_suspend)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (from_suspend && tp->link_ok(dev) && (tp->wol_opts & WAKE_PHY))
+                return 1;
+
+        if (!from_suspend && tp->resume_not_chg_speed)
+                return 1;
+
+        return 0;
+}
+
+static void
+rtl8127_set_wol_rx_packet_filter(struct rtl8127_private *tp)
+{
+        RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) | AcceptBroadcast |
+                AcceptMulticast | AcceptMyPhys);
+}
+
+static void
+rtl8127_powerdown_pll(struct net_device *dev, u8 from_suspend)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        /* Reboot not set wol link speed */
+        if (system_state == SYSTEM_RESTART)
+                return;
+
+        tp->check_keep_link_speed = 0;
+        if (tp->wol_enabled == WOL_ENABLED || tp->DASH || tp->EnableKCPOffload) {
+                rtl8127_set_hw_wol(dev, tp->wol_opts);
+
+                rtl8127_enable_cfg9346_write(tp);
+                RTL_W8(tp, Config2, RTL_R8(tp, Config2) | PMSTS_En);
+                rtl8127_disable_cfg9346_write(tp);
+
+                /* Enable the PME and clear the status */
+                rtl8127_set_pci_pme(tp, 1);
+
+                rtl8127_set_wol_rx_packet_filter(tp);
+
+                if (rtl8127_keep_wol_link_speed(dev, from_suspend)) {
+                        tp->check_keep_link_speed = 1;
+                } else {
+                        if (tp->D0SpeedUpSpeed != D0_SPEED_UP_SPEED_DISABLE) {
+                                rtl8127_enable_d0_speedup(tp);
+                                tp->check_keep_link_speed = 1;
+                        }
+
+                        rtl8127_set_wol_link_speed(dev);
+                }
+
+                return;
+        }
+
+        if (tp->DASH)
+                return;
+
+        rtl8127_phy_power_down(dev);
+
+        RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_6);
+}
+
+static void rtl8127_powerup_pll(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | BIT_7 | BIT_6);
+
+        if (tp->resume_not_chg_speed)
+                return;
+
+        rtl8127_phy_power_up(dev);
+}
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+static void
+rtl8127_get_wol(struct net_device *dev,
+                struct ethtool_wolinfo *wol)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u8 options;
+
+        wol->wolopts = 0;
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT || disable_wol_support) {
+                wol->supported = 0;
+                return;
+        } else {
+                wol->supported = WAKE_ANY;
+        }
+
+        options = RTL_R8(tp, Config1);
+        if (!(options & PMEnable))
+                return;
+
+        wol->wolopts = tp->wol_opts;
+}
+
+static int
+rtl8127_set_wol(struct net_device *dev,
+                struct ethtool_wolinfo *wol)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT || disable_wol_support)
+                return -EOPNOTSUPP;
+
+        tp->wol_opts = wol->wolopts;
+
+        tp->wol_enabled = (tp->wol_opts) ? WOL_ENABLED : WOL_DISABLED;
+
+        device_set_wakeup_enable(tp_to_dev(tp), wol->wolopts);
+
+        return 0;
+}
+
+static void
+rtl8127_get_drvinfo(struct net_device *dev,
+                    struct ethtool_drvinfo *info)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct rtl8127_fw *rtl_fw = tp->rtl_fw;
+
+        strscpy(info->driver, MODULENAME, sizeof(info->driver));
+        strscpy(info->version, RTL8127_VERSION, sizeof(info->version));
+        strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
+        info->regdump_len = R8127_REGS_DUMP_SIZE;
+        info->eedump_len = tp->eeprom_len;
+        BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version));
+        if (rtl_fw)
+                strscpy(info->fw_version, rtl_fw->version,
+                        sizeof(info->fw_version));
+}
+
+static int
+rtl8127_get_regs_len(struct net_device *dev)
+{
+        return R8127_REGS_DUMP_SIZE;
+}
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+
+static void
+rtl8127_set_d0_speedup_speed(struct rtl8127_private *tp)
+{
+        if (FALSE == HW_SUPPORT_D0_SPEED_UP(tp))
+                return;
+
+        tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_DISABLE;
+        if (tp->autoneg == AUTONEG_ENABLE) {
+                if (tp->speed == SPEED_10000)
+                        tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_10000;
+                else if (tp->speed == SPEED_5000)
+                        tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_5000;
+                else if (tp->speed == SPEED_2500)
+                        tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_2500;
+                else if (tp->speed == SPEED_1000)
+                        tp->D0SpeedUpSpeed = D0_SPEED_UP_SPEED_1000;
+        }
+}
+
+static int
+rtl8127_set_speed_xmii(struct net_device *dev,
+                       u8 autoneg,
+                       u32 speed,
+                       u8 duplex,
+                       u64 adv)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+        int auto_nego = 0;
+        int giga_ctrl = 0;
+        int ctrl_2500 = 0;
+        int rc = -EINVAL;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        if (!rtl8127_is_speed_mode_valid(tp, speed)) {
+                speed = SPEED_10000;
+                duplex = DUPLEX_FULL;
+                adv |= tp->advertising;
+        }
+
+        if (eee_giga_lite && (autoneg == AUTONEG_ENABLE))
+                rtl8127_enable_giga_lite(tp, adv);
+        else
+                rtl8127_disable_giga_lite(tp);
+
+        if (HW_FIBER_MODE_ENABLED(tp))
+                goto set_speed;
+
+        giga_ctrl = rtl8127_mdio_read(tp, MII_CTRL1000);
+        giga_ctrl &= ~(ADVERTISE_1000HALF | ADVERTISE_1000FULL);
+        ctrl_2500 = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D4);
+        ctrl_2500 &= ~(RTK_ADVERTISE_2500FULL | RTK_ADVERTISE_5000FULL |
+                       RTK_ADVERTISE_10000FULL);
+
+        if (autoneg == AUTONEG_ENABLE) {
+                /*n-way force*/
+                auto_nego = rtl8127_mdio_read(tp, MII_ADVERTISE);
+                auto_nego &= ~(ADVERTISE_10HALF | ADVERTISE_10FULL |
+                               ADVERTISE_100HALF | ADVERTISE_100FULL |
+                               ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
+
+                if (adv & ADVERTISED_10baseT_Half)
+                        auto_nego |= ADVERTISE_10HALF;
+                if (adv & ADVERTISED_10baseT_Full)
+                        auto_nego |= ADVERTISE_10FULL;
+                if (adv & ADVERTISED_100baseT_Half)
+                        auto_nego |= ADVERTISE_100HALF;
+                if (adv & ADVERTISED_100baseT_Full)
+                        auto_nego |= ADVERTISE_100FULL;
+                if (adv & ADVERTISED_1000baseT_Half)
+                        giga_ctrl |= ADVERTISE_1000HALF;
+                if (adv & ADVERTISED_1000baseT_Full)
+                        giga_ctrl |= ADVERTISE_1000FULL;
+                if (adv & ADVERTISED_2500baseX_Full)
+                        ctrl_2500 |= RTK_ADVERTISE_2500FULL;
+                if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) {
+                        if (adv & RTK_ADVERTISED_5000baseX_Full)
+                                ctrl_2500 |= RTK_ADVERTISE_5000FULL;
+                }
+                if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) {
+                        if (adv & ADVERTISED_10000baseT_Full)
+                                ctrl_2500 |= RTK_ADVERTISE_10000FULL;
+                }
+
+                //flow control
+                if (tp->fcpause == rtl8127_fc_full)
+                        auto_nego |= ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
+
+                tp->phy_auto_nego_reg = auto_nego;
+                tp->phy_1000_ctrl_reg = giga_ctrl;
+
+                tp->phy_2500_ctrl_reg = ctrl_2500;
+
+                rtl8127_mdio_write(tp, 0x1f, 0x0000);
+                rtl8127_mdio_write(tp, MII_ADVERTISE, auto_nego);
+                rtl8127_mdio_write(tp, MII_CTRL1000, giga_ctrl);
+                rtl8127_mdio_direct_write_phy_ocp(tp, 0xA5D4, ctrl_2500);
+                rtl8127_phy_restart_nway(dev);
+        } else {
+                /*true force*/
+                if (speed == SPEED_10 || speed == SPEED_100)
+                        rtl8127_phy_setup_force_mode(dev, speed, duplex);
+                else
+                        goto out;
+        }
+
+set_speed:
+        tp->autoneg = autoneg;
+        tp->speed = speed;
+        tp->duplex = duplex;
+        tp->advertising = adv;
+
+        rtl8127_set_d0_speedup_speed(tp);
+
+        rtl8127_hw_fiber_phy_config(tp);
+
+        rc = 0;
+out:
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return rc;
+}
+
+static int
+rtl8127_set_speed(struct net_device *dev,
+                  u8 autoneg,
+                  u32 speed,
+                  u8 duplex,
+                  u64 adv)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ret;
+
+        if (tp->resume_not_chg_speed)
+                return 0;
+
+        ret = tp->set_speed(dev, autoneg, speed, duplex, adv);
+
+        return ret;
+}
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+static int
+rtl8127_set_settings(struct net_device *dev,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+                     struct ethtool_cmd *cmd
+#else
+                     const struct ethtool_link_ksettings *cmd
+#endif
+                    )
+{
+        int ret;
+        u8 autoneg;
+        u32 speed;
+        u8 duplex;
+        u64 supported = 0, advertising = 0;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+        autoneg = cmd->autoneg;
+        speed = cmd->speed;
+        duplex = cmd->duplex;
+        supported = cmd->supported;
+        advertising = cmd->advertising;
+#else
+        struct rtl8127_private *tp = netdev_priv(dev);
+        const struct ethtool_link_settings *base = &cmd->base;
+        autoneg = base->autoneg;
+        speed = base->speed;
+        duplex = base->duplex;
+        ethtool_convert_link_mode_to_legacy_u32((u32*)&supported,
+                                                cmd->link_modes.supported);
+        ethtool_convert_link_mode_to_legacy_u32((u32*)&advertising,
+                                                cmd->link_modes.advertising);
+        if (test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT,
+                     cmd->link_modes.supported))
+                supported |= ADVERTISED_2500baseX_Full;
+        if (test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT,
+                     cmd->link_modes.advertising))
+                advertising |= ADVERTISED_2500baseX_Full;
+        if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) {
+                if (test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
+                             cmd->link_modes.supported))
+                        supported |= RTK_ADVERTISED_5000baseX_Full;
+                if (test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
+                             cmd->link_modes.advertising))
+                        advertising |= RTK_ADVERTISED_5000baseX_Full;
+        }
+        if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) {
+                if (test_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+                             cmd->link_modes.supported))
+                        supported |= ADVERTISED_10000baseT_Full;
+                if (test_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+                             cmd->link_modes.advertising))
+                        advertising |= ADVERTISED_10000baseT_Full;
+        }
+#endif
+        if (advertising & ~supported)
+                return -EINVAL;
+
+        ret = rtl8127_set_speed(dev, autoneg, speed, duplex, advertising);
+
+        return ret;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+static u32
+rtl8127_get_tx_csum(struct net_device *dev)
+{
+        u32 ret;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+        ret = ((dev->features & NETIF_F_IP_CSUM) != 0);
+#else
+        ret = ((dev->features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) != 0);
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+
+        return ret;
+}
+
+static u32
+rtl8127_get_rx_csum(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u32 ret;
+
+        ret = tp->cp_cmd & RxChkSum;
+
+        return ret;
+}
+
+static int
+rtl8127_set_tx_csum(struct net_device *dev,
+                    u32 data)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT)
+                return -EOPNOTSUPP;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+        if (data)
+                dev->features |= NETIF_F_IP_CSUM;
+        else
+                dev->features &= ~NETIF_F_IP_CSUM;
+#else
+        if (data)
+                dev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
+        else
+                dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+
+        return 0;
+}
+
+static int
+rtl8127_set_rx_csum(struct net_device *dev,
+                    u32 data)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT)
+                return -EOPNOTSUPP;
+
+        if (data)
+                tp->cp_cmd |= RxChkSum;
+        else
+                tp->cp_cmd &= ~RxChkSum;
+
+        RTL_W16(tp, CPlusCmd, tp->cp_cmd);
+
+        return 0;
+}
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+
+static u32
+rtl8127_rx_desc_opts1(struct rtl8127_private *tp,
+                      struct RxDesc *desc)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                return READ_ONCE(((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts1);
+        case RX_DESC_RING_TYPE_4:
+                return READ_ONCE(((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts1);
+        default:
+                return READ_ONCE(desc->opts1);
+        }
+}
+
+static u32
+rtl8127_rx_desc_opts2(struct rtl8127_private *tp,
+                      struct RxDesc *desc)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                return ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts2;
+        case RX_DESC_RING_TYPE_4:
+                return ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts2;
+        default:
+                return desc->opts2;
+        }
+}
+
+#ifdef CONFIG_R8127_VLAN
+
+static void
+rtl8127_clear_rx_desc_opts2(struct rtl8127_private *tp,
+                            struct RxDesc *desc)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                ((struct RxDescV3 *)desc)->RxDescNormalDDWord4.opts2 = 0;
+                break;
+        case RX_DESC_RING_TYPE_4:
+                ((struct RxDescV4 *)desc)->RxDescNormalDDWord2.opts2 = 0;
+                break;
+        default:
+                desc->opts2 = 0;
+                break;
+        }
+}
+
+
+static inline u32
+rtl8127_tx_vlan_tag(struct rtl8127_private *tp,
+                    struct sk_buff *skb)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+        return (tp->vlgrp && vlan_tx_tag_present(skb)) ?
+               TxVlanTag | swab16(vlan_tx_tag_get(skb)) : 0x00;
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(4,0,0)
+        return (vlan_tx_tag_present(skb)) ?
+               TxVlanTag | swab16(vlan_tx_tag_get(skb)) : 0x00;
+#else
+        return (skb_vlan_tag_present(skb)) ?
+               TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00;
+#endif
+
+        return 0;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+
+static void
+rtl8127_vlan_rx_register(struct net_device *dev,
+                         struct vlan_group *grp)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        tp->vlgrp = grp;
+
+        if (tp->vlgrp) {
+                tp->rtl8127_rx_config |= (EnableInnerVlan | EnableOuterVlan);
+                RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) | (EnableInnerVlan | EnableOuterVlan))
+        } else {
+                tp->rtl8127_rx_config &= ~(EnableInnerVlan | EnableOuterVlan);
+                RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & ~(EnableInnerVlan | EnableOuterVlan))
+        }
+}
+
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+static void
+rtl8127_vlan_rx_kill_vid(struct net_device *dev,
+                         unsigned short vid)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21)
+        if (tp->vlgrp)
+                tp->vlgrp->vlan_devices[vid] = NULL;
+#else
+        vlan_group_set_device(tp->vlgrp, vid, NULL);
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,21)
+}
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+
+static int
+rtl8127_rx_vlan_skb(struct rtl8127_private *tp,
+                    struct RxDesc *desc,
+                    struct sk_buff *skb)
+{
+        u32 opts2 = le32_to_cpu(rtl8127_rx_desc_opts2(tp, desc));
+        int ret = -1;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+        if (tp->vlgrp && (opts2 & RxVlanTag)) {
+                rtl8127_rx_hwaccel_skb(skb, tp->vlgrp,
+                                       swab16(opts2 & 0xffff));
+                ret = 0;
+        }
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)
+        if (opts2 & RxVlanTag)
+                __vlan_hwaccel_put_tag(skb, swab16(opts2 & 0xffff));
+#else
+        if (opts2 & RxVlanTag)
+                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff));
+#endif
+
+        rtl8127_clear_rx_desc_opts2(tp, desc);
+        return ret;
+}
+
+#else /* !CONFIG_R8127_VLAN */
+
+static inline u32
+rtl8127_tx_vlan_tag(struct rtl8127_private *tp,
+                    struct sk_buff *skb)
+{
+        return 0;
+}
+
+static int
+rtl8127_rx_vlan_skb(struct rtl8127_private *tp,
+                    struct RxDesc *desc,
+                    struct sk_buff *skb)
+{
+        return -1;
+}
+
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0)
+
+static netdev_features_t rtl8127_fix_features(struct net_device *dev,
+                netdev_features_t features)
+{
+        if (dev->mtu > MSS_MAX || dev->mtu > ETH_DATA_LEN)
+                features &= ~NETIF_F_ALL_TSO;
+#ifndef CONFIG_R8127_VLAN
+        features &= ~NETIF_F_ALL_CSUM;
+#endif
+
+        return features;
+}
+
+static int rtl8127_hw_set_features(struct net_device *dev,
+                                   netdev_features_t features)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u32 rx_config;
+
+        rx_config = RTL_R32(tp, RxConfig);
+        if (features & NETIF_F_RXALL) {
+                tp->rtl8127_rx_config |= (AcceptErr | AcceptRunt);
+                rx_config |= (AcceptErr | AcceptRunt);
+        } else {
+                tp->rtl8127_rx_config &= ~(AcceptErr | AcceptRunt);
+                rx_config &= ~(AcceptErr | AcceptRunt);
+        }
+
+        if (features & NETIF_F_HW_VLAN_RX) {
+                tp->rtl8127_rx_config |= (EnableInnerVlan | EnableOuterVlan);
+                rx_config |= (EnableInnerVlan | EnableOuterVlan);
+        } else {
+                tp->rtl8127_rx_config &= ~(EnableInnerVlan | EnableOuterVlan);
+                rx_config &= ~(EnableInnerVlan | EnableOuterVlan);
+        }
+
+        RTL_W32(tp, RxConfig, rx_config);
+
+        if (features & NETIF_F_RXCSUM)
+                tp->cp_cmd |= RxChkSum;
+        else
+                tp->cp_cmd &= ~RxChkSum;
+
+        RTL_W16(tp, CPlusCmd, tp->cp_cmd);
+        RTL_R16(tp, CPlusCmd);
+
+        return 0;
+}
+
+static int rtl8127_set_features(struct net_device *dev,
+                                netdev_features_t features)
+{
+        features &= NETIF_F_RXALL | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_RX;
+
+        rtl8127_hw_set_features(dev, features);
+
+        return 0;
+}
+
+#endif
+
+static u8 rtl8127_get_mdi_status(struct rtl8127_private *tp)
+{
+        if (!tp->link_ok(tp->dev))
+                return ETH_TP_MDI_INVALID;
+
+        if (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA444) & BIT_1)
+                return ETH_TP_MDI;
+        else
+                return ETH_TP_MDI_X;
+}
+
+static void rtl8127_gset_xmii(struct net_device *dev,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+                              struct ethtool_cmd *cmd
+#else
+                              struct ethtool_link_ksettings *cmd
+#endif
+                             )
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u16 aner = tp->phy_reg_aner;
+        u16 anlpar = tp->phy_reg_anlpar;
+        u16 gbsr = tp->phy_reg_gbsr;
+        u16 status_2500 = tp->phy_reg_status_2500;
+        unsigned long flags;
+        u64 lpa_adv = 0;
+        u32 status;
+        u8 autoneg, duplex;
+        u32 speed = 0;
+        u16 bmcr;
+        u64 supported, advertising;
+        u8 report_lpa = 0;
+
+        if (HW_FIBER_MODE_ENABLED(tp)) {
+                supported = SUPPORTED_1000baseT_Full |
+                            SUPPORTED_10000baseT_Full |
+                            SUPPORTED_TP;
+        } else {
+                supported = SUPPORTED_10baseT_Half |
+                            SUPPORTED_10baseT_Full |
+                            SUPPORTED_100baseT_Half |
+                            SUPPORTED_100baseT_Full |
+                            SUPPORTED_1000baseT_Full |
+                            SUPPORTED_2500baseX_Full |
+                            RTK_SUPPORTED_5000baseX_Full |
+                            SUPPORTED_10000baseT_Full |
+                            SUPPORTED_Autoneg |
+                            SUPPORTED_TP |
+                            SUPPORTED_Pause |
+                            SUPPORTED_Asym_Pause;
+        }
+
+        if (!HW_SUPP_PHY_LINK_SPEED_2500M(tp))
+                supported &= ~SUPPORTED_2500baseX_Full;
+
+        if (!HW_SUPP_PHY_LINK_SPEED_5000M(tp))
+                supported &= ~RTK_SUPPORTED_5000baseX_Full;
+
+        if (!HW_SUPP_PHY_LINK_SPEED_10000M(tp))
+                supported &= ~SUPPORTED_10000baseT_Full;
+
+        advertising = tp->advertising;
+        if (tp->phy_auto_nego_reg || tp->phy_1000_ctrl_reg ||
+            tp->phy_2500_ctrl_reg) {
+                advertising = 0;
+                if (tp->phy_auto_nego_reg & ADVERTISE_10HALF)
+                        advertising |= ADVERTISED_10baseT_Half;
+                if (tp->phy_auto_nego_reg & ADVERTISE_10FULL)
+                        advertising |= ADVERTISED_10baseT_Full;
+                if (tp->phy_auto_nego_reg & ADVERTISE_100HALF)
+                        advertising |= ADVERTISED_100baseT_Half;
+                if (tp->phy_auto_nego_reg & ADVERTISE_100FULL)
+                        advertising |= ADVERTISED_100baseT_Full;
+                if (tp->phy_1000_ctrl_reg & ADVERTISE_1000FULL)
+                        advertising |= ADVERTISED_1000baseT_Full;
+                if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_2500FULL)
+                        advertising |= ADVERTISED_2500baseX_Full;
+                if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_5000FULL)
+                        advertising |= RTK_ADVERTISED_5000baseX_Full;
+                if (tp->phy_2500_ctrl_reg & RTK_ADVERTISE_10000FULL)
+                        advertising |= ADVERTISED_10000baseT_Full;
+        }
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        bmcr = rtl8127_mdio_read(tp, MII_BMCR);
+        if (bmcr & BMCR_ANENABLE) {
+                autoneg = AUTONEG_ENABLE;
+                advertising |= ADVERTISED_Autoneg;
+        } else {
+                autoneg = AUTONEG_DISABLE;
+        }
+
+        advertising |= ADVERTISED_TP;
+
+        status = rtl8127_get_phy_status(tp);
+        if (netif_running(dev) && (status & LinkStatus))
+                report_lpa = 1;
+
+        if (report_lpa) {
+                /*link on*/
+                speed = rtl8127_convert_link_speed(status);
+
+                if (status & TxFlowCtrl)
+                        advertising |= ADVERTISED_Asym_Pause;
+
+                if (status & RxFlowCtrl)
+                        advertising |= ADVERTISED_Pause;
+
+                duplex = ((status & (_1000bpsF | _2500bpsF | _5000bpsF | _10000bpsF)) ||
+                          (status & FullDup)) ?
+                         DUPLEX_FULL : DUPLEX_HALF;
+
+                /*link partner*/
+                if (aner & EXPANSION_NWAY)
+                        lpa_adv |= ADVERTISED_Autoneg;
+                if (anlpar & LPA_10HALF)
+                        lpa_adv |= ADVERTISED_10baseT_Half;
+                if (anlpar & LPA_10FULL)
+                        lpa_adv |= ADVERTISED_10baseT_Full;
+                if (anlpar & LPA_100HALF)
+                        lpa_adv |= ADVERTISED_100baseT_Half;
+                if (anlpar & LPA_100FULL)
+                        lpa_adv |= ADVERTISED_100baseT_Full;
+                if (anlpar & LPA_PAUSE_CAP)
+                        lpa_adv |= ADVERTISED_Pause;
+                if (anlpar & LPA_PAUSE_ASYM)
+                        lpa_adv |= ADVERTISED_Asym_Pause;
+                if (gbsr & LPA_1000HALF)
+                        lpa_adv |= ADVERTISED_1000baseT_Half;
+                if (gbsr & LPA_1000FULL)
+                        lpa_adv |= ADVERTISED_1000baseT_Full;
+                if (status_2500 & RTK_LPA_ADVERTISE_2500FULL)
+                        lpa_adv |= ADVERTISED_2500baseX_Full;
+                if (status_2500 & RTK_LPA_ADVERTISE_5000FULL)
+                        lpa_adv |= RTK_ADVERTISED_5000baseX_Full;
+                if (status_2500 & RTK_LPA_ADVERTISE_10000FULL)
+                        lpa_adv |= ADVERTISED_10000baseT_Full;
+        } else {
+                /*link down*/
+                speed = SPEED_UNKNOWN;
+                duplex = DUPLEX_UNKNOWN;
+                lpa_adv = 0;
+        }
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+        cmd->supported = (u32)supported;
+        cmd->advertising = (u32)advertising;
+        cmd->autoneg = autoneg;
+        cmd->speed = speed;
+        cmd->duplex = duplex;
+        cmd->port = PORT_TP;
+        cmd->lp_advertising = (u32)lpa_adv;
+        cmd->eth_tp_mdix = rtl8127_get_mdi_status(tp);
+#else
+        ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
+                                                supported);
+        ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
+                                                advertising);
+        ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.lp_advertising,
+                                                lpa_adv);
+
+        if (supported & SUPPORTED_2500baseX_Full) {
+                linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT,
+                                 cmd->link_modes.supported, 1);
+        }
+        if (advertising & ADVERTISED_2500baseX_Full) {
+                linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT,
+                                 cmd->link_modes.advertising, 1);
+        }
+        if (supported & RTK_SUPPORTED_5000baseX_Full) {
+                linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
+                                 cmd->link_modes.supported, 1);
+        }
+        if (advertising & RTK_ADVERTISED_5000baseX_Full) {
+                linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
+                                 cmd->link_modes.advertising, 1);
+        }
+        if (supported & SUPPORTED_10000baseT_Full) {
+                linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+                                 cmd->link_modes.supported, 1);
+        }
+        if (advertising & ADVERTISED_10000baseT_Full) {
+                linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+                                 cmd->link_modes.advertising, 1);
+        }
+        if (report_lpa) {
+                if (lpa_adv & ADVERTISED_2500baseX_Full) {
+                        linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT,
+                                         cmd->link_modes.lp_advertising, 1);
+                }
+                if (lpa_adv & RTK_ADVERTISED_5000baseX_Full)
+                        linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
+                                         cmd->link_modes.lp_advertising, 1);
+                if (lpa_adv & ADVERTISED_10000baseT_Full)
+                        linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+                                         cmd->link_modes.lp_advertising, 1);
+        }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0)
+        /* Use ETHTOOL_LINK_MODE_2500baseT_Full_BIT instead of
+           ETHTOOL_LINK_MODE_2500baseX_Full_BIT. */
+        linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT,
+                         cmd->link_modes.supported, 0);
+
+        linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT,
+                         cmd->link_modes.advertising, 0);
+
+        linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT,
+                         cmd->link_modes.lp_advertising, 0);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */
+
+        cmd->base.autoneg = autoneg;
+        cmd->base.speed = speed;
+        cmd->base.duplex = duplex;
+        cmd->base.port = PORT_TP;
+        cmd->base.eth_tp_mdix = rtl8127_get_mdi_status(tp);
+#endif
+        r8127_spin_unlock(&tp->phy_lock, flags);
+}
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+static int
+rtl8127_get_settings(struct net_device *dev,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+                     struct ethtool_cmd *cmd
+#else
+                     struct ethtool_link_ksettings *cmd
+#endif
+                    )
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        tp->get_settings(dev, cmd);
+
+        return 0;
+}
+
+static void rtl8127_get_regs(struct net_device *dev, struct ethtool_regs *regs,
+                             void *p)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        void __iomem *ioaddr = tp->mmio_addr;
+        unsigned int i;
+        u8 *data = p;
+
+        if (regs->len < R8127_REGS_DUMP_SIZE)
+                return /* -EINVAL */;
+
+        memset(p, 0, regs->len);
+
+        for (i = 0; i < R8127_MAC_REGS_SIZE; i++)
+                *data++ = readb(ioaddr + i);
+        data = (u8*)p + 256;
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        for (i = 0; i < R8127_PHY_REGS_SIZE/2; i++) {
+                *(u16*)data = rtl8127_mdio_read(tp, i);
+                data += 2;
+        }
+        data = (u8*)p + 256 * 2;
+
+        for (i = 0; i < R8127_EPHY_REGS_SIZE/2; i++) {
+                *(u16*)data = rtl8127_ephy_read(tp, i);
+                data += 2;
+        }
+        data = (u8*)p + 256 * 3;
+
+        for (i = 0; i < R8127_ERI_REGS_SIZE; i+=4) {
+                *(u32*)data = rtl8127_eri_read(tp, i , 4, ERIAR_ExGMAC);
+                data += 4;
+        }
+}
+
+static void rtl8127_get_pauseparam(struct net_device *dev,
+                                   struct ethtool_pauseparam *pause)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        pause->autoneg = (tp->autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE);
+        if (tp->fcpause == rtl8127_fc_rx_pause)
+                pause->rx_pause = 1;
+        else if (tp->fcpause == rtl8127_fc_tx_pause)
+                pause->tx_pause = 1;
+        else if (tp->fcpause == rtl8127_fc_full) {
+                pause->rx_pause = 1;
+                pause->tx_pause = 1;
+        }
+}
+
+static int rtl8127_set_pauseparam(struct net_device *dev,
+                                  struct ethtool_pauseparam *pause)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        enum rtl8127_fc_mode newfc;
+
+        if (pause->tx_pause || pause->rx_pause)
+                newfc = rtl8127_fc_full;
+        else
+                newfc = rtl8127_fc_none;
+
+        if (tp->fcpause != newfc) {
+                tp->fcpause = newfc;
+
+                rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising);
+        }
+
+        return 0;
+
+}
+
+static u32
+rtl8127_get_msglevel(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        return tp->msg_enable;
+}
+
+static void
+rtl8127_set_msglevel(struct net_device *dev,
+                     u32 value)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        tp->msg_enable = value;
+}
+
+static const char rtl8127_gstrings[][ETH_GSTRING_LEN] = {
+        /* legacy */
+        "tx_packets",
+        "rx_packets",
+        "tx_errors",
+        "rx_errors",
+        "rx_missed",
+        "align_errors",
+        "tx_single_collisions",
+        "tx_multi_collisions",
+        "unicast",
+        "broadcast",
+        "multicast",
+        "tx_aborted",
+        "tx_underrun",
+
+        /* extended */
+        "tx_octets",
+        "rx_octets",
+        "rx_multicast64",
+        "tx_unicast64",
+        "tx_broadcast64",
+        "tx_multicast64",
+        "tx_pause_on",
+        "tx_pause_off",
+        "tx_pause_all",
+        "tx_deferred",
+        "tx_late_collision",
+        "tx_all_collision",
+        "tx_aborted32",
+        "align_errors32",
+        "rx_frame_too_long",
+        "rx_runt",
+        "rx_pause_on",
+        "rx_pause_off",
+        "rx_pause_all",
+        "rx_unknown_opcode",
+        "rx_mac_error",
+        "tx_underrun32",
+        "rx_mac_missed",
+        "rx_tcam_dropped",
+        "tdu",
+        "rdu",
+};
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33)
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+static int rtl8127_get_stats_count(struct net_device *dev)
+{
+        return ARRAY_SIZE(rtl8127_gstrings);
+}
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+#else
+static int rtl8127_get_sset_count(struct net_device *dev, int sset)
+{
+        switch (sset) {
+        case ETH_SS_STATS:
+                return ARRAY_SIZE(rtl8127_gstrings);
+        default:
+                return -EOPNOTSUPP;
+        }
+}
+#endif
+
+static void
+rtl8127_set_ring_size(struct rtl8127_private *tp, u32 rx, u32 tx)
+{
+        int i;
+
+        for (i = 0; i < R8127_MAX_RX_QUEUES; i++)
+                tp->rx_ring[i].num_rx_desc = rx;
+
+        for (i = 0; i < R8127_MAX_TX_QUEUES; i++)
+                tp->tx_ring[i].num_tx_desc = tx;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+static void rtl8127_get_ringparam(struct net_device *dev,
+                                  struct ethtool_ringparam *ring,
+                                  struct kernel_ethtool_ringparam *kernel_ring,
+                                  struct netlink_ext_ack *extack)
+#else
+static void rtl8127_get_ringparam(struct net_device *dev,
+                                  struct ethtool_ringparam *ring)
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        ring->rx_max_pending = MAX_NUM_TX_DESC;
+        ring->tx_max_pending = MAX_NUM_RX_DESC;
+        ring->rx_pending = tp->rx_ring[0].num_rx_desc;
+        ring->tx_pending = tp->tx_ring[0].num_tx_desc;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+static int rtl8127_set_ringparam(struct net_device *dev,
+                                 struct ethtool_ringparam *ring,
+                                 struct kernel_ethtool_ringparam *kernel_ring,
+                                 struct netlink_ext_ack *extack)
+#else
+static int rtl8127_set_ringparam(struct net_device *dev,
+                                 struct ethtool_ringparam *ring)
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u32 new_rx_count, new_tx_count;
+        int rc = 0;
+
+        if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
+                return -EINVAL;
+
+        new_tx_count = clamp_t(u32, ring->tx_pending,
+                               MIN_NUM_TX_DESC, MAX_NUM_TX_DESC);
+
+        new_rx_count = clamp_t(u32, ring->rx_pending,
+                               MIN_NUM_RX_DESC, MAX_NUM_RX_DESC);
+
+        if ((new_rx_count == tp->rx_ring[0].num_rx_desc) &&
+            (new_tx_count == tp->tx_ring[0].num_tx_desc)) {
+                /* nothing to do */
+                return 0;
+        }
+
+        if (netif_running(dev)) {
+                rtl8127_wait_for_quiescence(dev);
+                rtl8127_close(dev);
+        }
+
+        rtl8127_set_ring_size(tp, new_rx_count, new_tx_count);
+
+        if (netif_running(dev))
+                rc = rtl8127_open(dev);
+
+        return rc;
+}
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+static void
+rtl8127_get_ethtool_stats(struct net_device *dev,
+                          struct ethtool_stats *stats,
+                          u64 *data)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct rtl8127_counters *counters;
+        dma_addr_t paddr;
+
+        ASSERT_RTNL();
+
+        counters = tp->tally_vaddr;
+        paddr = tp->tally_paddr;
+        if (!counters)
+                return;
+
+        rtl8127_dump_tally_counter(tp, paddr);
+
+        data[0] = le64_to_cpu(counters->tx_packets);
+        data[1] = le64_to_cpu(counters->rx_packets);
+        data[2] = le64_to_cpu(counters->tx_errors);
+        data[3] = le32_to_cpu(counters->rx_errors);
+        data[4] = le16_to_cpu(counters->rx_missed);
+        data[5] = le16_to_cpu(counters->align_errors);
+        data[6] = le32_to_cpu(counters->tx_one_collision);
+        data[7] = le32_to_cpu(counters->tx_multi_collision);
+        data[8] = le64_to_cpu(counters->rx_unicast);
+        data[9] = le64_to_cpu(counters->rx_broadcast);
+        data[10] = le32_to_cpu(counters->rx_multicast);
+        data[11] = le16_to_cpu(counters->tx_aborted);
+        data[12] = le16_to_cpu(counters->tx_underrun);
+
+        data[13] = le64_to_cpu(counters->tx_octets);
+        data[14] = le64_to_cpu(counters->rx_octets);
+        data[15] = le64_to_cpu(counters->rx_multicast64);
+        data[16] = le64_to_cpu(counters->tx_unicast64);
+        data[17] = le64_to_cpu(counters->tx_broadcast64);
+        data[18] = le64_to_cpu(counters->tx_multicast64);
+        data[19] = le32_to_cpu(counters->tx_pause_on);
+        data[20] = le32_to_cpu(counters->tx_pause_off);
+        data[21] = le32_to_cpu(counters->tx_pause_all);
+        data[22] = le32_to_cpu(counters->tx_deferred);
+        data[23] = le32_to_cpu(counters->tx_late_collision);
+        data[24] = le32_to_cpu(counters->tx_all_collision);
+        data[25] = le32_to_cpu(counters->tx_aborted32);
+        data[26] = le32_to_cpu(counters->align_errors32);
+        data[27] = le32_to_cpu(counters->rx_frame_too_long);
+        data[28] = le32_to_cpu(counters->rx_runt);
+        data[29] = le32_to_cpu(counters->rx_pause_on);
+        data[30] = le32_to_cpu(counters->rx_pause_off);
+        data[31] = le32_to_cpu(counters->rx_pause_all);
+        data[32] = le32_to_cpu(counters->rx_unknown_opcode);
+        data[33] = le32_to_cpu(counters->rx_mac_error);
+        data[34] = le32_to_cpu(counters->tx_underrun32);
+        data[35] = le32_to_cpu(counters->rx_mac_missed);
+        data[36] = le32_to_cpu(counters->rx_tcam_dropped);
+        data[37] = le32_to_cpu(counters->tdu);
+        data[38] = le32_to_cpu(counters->rdu);
+}
+
+static void
+rtl8127_get_strings(struct net_device *dev,
+                    u32 stringset,
+                    u8 *data)
+{
+        switch (stringset) {
+        case ETH_SS_STATS:
+                memcpy(data, rtl8127_gstrings, sizeof(rtl8127_gstrings));
+                break;
+        }
+}
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+
+static int rtl_get_eeprom_len(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        return tp->eeprom_len;
+}
+
+static int rtl_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *buf)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i,j,ret;
+        int start_w, end_w;
+        int VPD_addr, VPD_data;
+        u32 *eeprom_buff;
+        u16 tmp;
+
+        if (tp->eeprom_type == EEPROM_TYPE_NONE) {
+                dev_printk(KERN_DEBUG, tp_to_dev(tp), "Detect none EEPROM\n");
+                return -EOPNOTSUPP;
+        } else if (eeprom->len == 0 || (eeprom->offset+eeprom->len) > tp->eeprom_len) {
+                dev_printk(KERN_DEBUG, tp_to_dev(tp), "Invalid parameter\n");
+                return -EINVAL;
+        }
+
+        VPD_addr = 0xD2;
+        VPD_data = 0xD4;
+
+        start_w = eeprom->offset >> 2;
+        end_w = (eeprom->offset + eeprom->len - 1) >> 2;
+
+        eeprom_buff = kmalloc(sizeof(u32)*(end_w - start_w + 1), GFP_KERNEL);
+        if (!eeprom_buff)
+                return -ENOMEM;
+
+        rtl8127_enable_cfg9346_write(tp);
+        ret = -EFAULT;
+        for (i=start_w; i<=end_w; i++) {
+                pci_write_config_word(tp->pci_dev, VPD_addr, (u16)i*4);
+                ret = -EFAULT;
+                for (j = 0; j < 10; j++) {
+                        udelay(400);
+                        pci_read_config_word(tp->pci_dev, VPD_addr, &tmp);
+                        if (tmp&0x8000) {
+                                ret = 0;
+                                break;
+                        }
+                }
+
+                if (ret)
+                        break;
+
+                pci_read_config_dword(tp->pci_dev, VPD_data, &eeprom_buff[i-start_w]);
+        }
+        rtl8127_disable_cfg9346_write(tp);
+
+        if (!ret)
+                memcpy(buf, (u8 *)eeprom_buff + (eeprom->offset & 3), eeprom->len);
+
+        kfree(eeprom_buff);
+
+        return ret;
+}
+
+#undef ethtool_op_get_link
+#define ethtool_op_get_link _kc_ethtool_op_get_link
+static u32 _kc_ethtool_op_get_link(struct net_device *dev)
+{
+        return netif_carrier_ok(dev) ? 1 : 0;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+#undef ethtool_op_get_sg
+#define ethtool_op_get_sg _kc_ethtool_op_get_sg
+static u32 _kc_ethtool_op_get_sg(struct net_device *dev)
+{
+#ifdef NETIF_F_SG
+        return (dev->features & NETIF_F_SG) != 0;
+#else
+        return 0;
+#endif
+}
+
+#undef ethtool_op_set_sg
+#define ethtool_op_set_sg _kc_ethtool_op_set_sg
+static int _kc_ethtool_op_set_sg(struct net_device *dev, u32 data)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT)
+                return -EOPNOTSUPP;
+
+#ifdef NETIF_F_SG
+        if (data)
+                dev->features |= NETIF_F_SG;
+        else
+                dev->features &= ~NETIF_F_SG;
+#endif
+
+        return 0;
+}
+#endif
+
+static void
+rtl8127_set_eee_lpi_timer(struct rtl8127_private *tp)
+{
+        u16 dev_lpi_timer;
+
+        dev_lpi_timer = tp->eee.tx_lpi_timer;
+
+        RTL_W16(tp, EEE_TXIDLE_TIMER_8125, dev_lpi_timer);
+}
+
+static bool rtl8127_is_adv_eee_enabled(struct rtl8127_private *tp)
+{
+        if (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA430) & BIT_15)
+                return true;
+        else
+                return false;
+}
+
+static void _rtl8127_disable_adv_eee(struct rtl8127_private *tp)
+{
+        bool lock;
+
+        if (rtl8127_is_adv_eee_enabled(tp))
+                lock = true;
+        else
+                lock = false;
+
+        if (lock)
+                rtl8127_set_phy_mcu_patch_request(tp);
+
+        rtl8127_clear_mac_ocp_bit(tp, 0xE052, BIT_0);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA442, BIT_12 | BIT_13);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA430, BIT_15);
+
+        if (lock)
+                rtl8127_clear_phy_mcu_patch_request(tp);
+}
+
+static void rtl8127_disable_adv_eee(struct rtl8127_private *tp)
+{
+        rtl8127_oob_mutex_lock(tp);
+
+        _rtl8127_disable_adv_eee(tp);
+
+        rtl8127_oob_mutex_unlock(tp);
+}
+
+static int rtl8127_enable_eee(struct rtl8127_private *tp)
+{
+        struct ethtool_keee *eee = &tp->eee;
+        u16 eee_adv_cap1_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t(eee->advertised);
+        u16 eee_adv_cap2_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap2_t(eee->advertised);
+
+        if (!HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp))
+                return 0;
+
+        rtl8127_set_mac_ocp_bit(tp, 0xE040, (BIT_1|BIT_0));
+
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA5D0,
+                                              MDIO_EEE_100TX | MDIO_EEE_1000T | MDIO_EEE_10GT,
+                                              eee_adv_cap1_t);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA6D4,
+                                              MDIO_EEE_2_5GT | MDIO_EEE_5GT,
+                                              eee_adv_cap2_t);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D8, BIT_4);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_7);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4A2, BIT_9);
+
+        /*Advanced EEE*/
+        rtl8127_disable_adv_eee(tp);
+
+        return 0;
+}
+
+static int rtl8127_disable_eee(struct rtl8127_private *tp)
+{
+        rtl8127_clear_mac_ocp_bit(tp, 0xE040, (BIT_1|BIT_0));
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5D0,
+                                      (MDIO_EEE_100TX | MDIO_EEE_1000T | MDIO_EEE_10GT));
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D4,
+                                      (MDIO_EEE_2_5GT | MDIO_EEE_5GT));
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA6D8, BIT_4);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA428, BIT_7);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4A2, BIT_9);
+
+        /*Advanced EEE*/
+        rtl8127_disable_adv_eee(tp);
+
+        return 0;
+}
+
+static int rtl_nway_reset(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ret, bmcr;
+
+        if (unlikely(tp->rtk_enable_diag))
+                return -EBUSY;
+
+        /* if autoneg is off, it's an error */
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        bmcr = rtl8127_mdio_read(tp, MII_BMCR);
+
+        if (bmcr & BMCR_ANENABLE) {
+                bmcr |= BMCR_ANRESTART;
+                rtl8127_mdio_write(tp, MII_BMCR, bmcr);
+                ret = 0;
+        } else {
+                ret = -EINVAL;
+        }
+
+        return ret;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0)
+static u32
+rtl8127_device_lpi_t_to_ethtool_lpi_t(struct rtl8127_private *tp , u32 lpi_timer)
+{
+        u32 to_us;
+        u32 status;
+
+        to_us = lpi_timer * 80;
+        status = rtl8127_get_phy_status(tp);
+        if (status & LinkStatus) {
+                /*link on*/
+                if (HW_SUPP_PHY_LINK_SPEED_10000M(tp)) {
+                        //5G : lpi_timer * 12.8ns
+                        //2.5G : lpi_timer * 25.6ns
+                        //Giga: lpi_timer * 8ns
+                        //100M : lpi_timer * 80ns
+                        if (status & (_10000bpsF))
+                                to_us = (lpi_timer * 128) / 10;
+                        else if (status & (_5000bpsF))
+                                to_us = (lpi_timer * 128) / 10;
+                        else if (status & _2500bpsF)
+                                to_us = (lpi_timer * 256) / 10;
+                        else if (status & _1000bpsF)
+                                to_us = lpi_timer * 8;
+                } else if (HW_SUPP_PHY_LINK_SPEED_5000M(tp)) {
+                        //5G : lpi_timer * 12.8ns
+                        //2.5G : lpi_timer * 25.6ns
+                        //Giga: lpi_timer * 8ns
+                        //100M : lpi_timer * 80ns
+                        if (status & (_5000bpsF))
+                                to_us = (lpi_timer * 128) / 10;
+                        else if (status & _2500bpsF)
+                                to_us = (lpi_timer * 256) / 10;
+                        else if (status & _1000bpsF)
+                                to_us = lpi_timer * 8;
+                } else {
+                        //2.5G : lpi_timer * 3.2ns
+                        //Giga: lpi_timer * 8ns
+                        //100M : lpi_timer * 80ns
+                        if (status & _2500bpsF)
+                                to_us = (lpi_timer * 32) / 10;
+                        else if (status & _1000bpsF)
+                                to_us = lpi_timer * 8;
+                }
+        }
+
+        //ns to us
+        to_us /= 1000;
+
+        return to_us;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0)
+static void
+rtl8127_adv_to_linkmode(unsigned long *mode, u64 adv)
+{
+        linkmode_zero(mode);
+
+        if (adv & ADVERTISED_10baseT_Half)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT, mode);
+        if (adv & ADVERTISED_10baseT_Full)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT, mode);
+        if (adv & ADVERTISED_100baseT_Half)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, mode);
+        if (adv & ADVERTISED_100baseT_Full)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, mode);
+        if (adv & ADVERTISED_1000baseT_Half)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT, mode);
+        if (adv & ADVERTISED_1000baseT_Full)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, mode);
+        if (adv & ADVERTISED_2500baseX_Full)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, mode);
+        if (adv & RTK_ADVERTISED_5000baseX_Full)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, mode);
+        if (adv & ADVERTISED_10000baseT_Full)
+                linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, mode);
+}
+
+static int
+rtl_ethtool_get_eee(struct net_device *net, struct ethtool_keee *edata)
+{
+        __ETHTOOL_DECLARE_LINK_MODE_MASK(common);
+        struct rtl8127_private *tp = netdev_priv(net);
+        struct ethtool_keee *eee = &tp->eee;
+        unsigned long flags;
+        u32 tx_lpi_timer;
+        u16 val;
+
+        if (unlikely(tp->rtk_enable_diag))
+                return -EBUSY;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        /* Get LP advertisement EEE */
+        val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D2);
+        mii_eee_cap1_mod_linkmode_t(edata->lp_advertised, val);
+        val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA6D0);
+        mii_eee_cap2_mod_linkmode_sup_t(edata->lp_advertised, val);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        /* Get EEE Tx LPI timer*/
+        tx_lpi_timer = rtl8127_device_lpi_t_to_ethtool_lpi_t(tp, eee->tx_lpi_timer);
+
+        val = rtl8127_mac_ocp_read(tp, 0xE040);
+        val &= BIT_1 | BIT_0;
+
+        edata->eee_enabled = !!val;
+        linkmode_copy(edata->supported, eee->supported);
+        linkmode_copy(edata->advertised, eee->advertised);
+        edata->tx_lpi_enabled = edata->eee_enabled;
+        edata->tx_lpi_timer = tx_lpi_timer;
+        linkmode_and(common, edata->advertised, edata->lp_advertised);
+        edata->eee_active = !linkmode_empty(common);
+
+        return 0;
+}
+
+static int
+rtl_ethtool_set_eee(struct net_device *net, struct ethtool_keee *edata)
+{
+        __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising);
+        __ETHTOOL_DECLARE_LINK_MODE_MASK(tmp);
+        struct rtl8127_private *tp = netdev_priv(net);
+        struct ethtool_keee *eee = &tp->eee;
+        unsigned long flags;
+        int rc = 0;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        if (!HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp) ||
+            tp->DASH) {
+                rc = -EOPNOTSUPP;
+                goto out;
+        }
+
+        if (unlikely(tp->rtk_enable_diag)) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "Diag Enabled\n");
+                rc = -EBUSY;
+                goto out;
+        }
+
+        if (tp->autoneg != AUTONEG_ENABLE) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE requires autoneg\n");
+                rc = -EINVAL;
+                goto out;
+        }
+
+        /*
+        if (edata->tx_lpi_enabled) {
+        if (edata->tx_lpi_timer > tp->max_jumbo_frame_size ||
+            edata->tx_lpi_timer < ETH_MIN_MTU) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "Valid LPI timer range is %d to %d. \n",
+                           ETH_MIN_MTU, tp->max_jumbo_frame_size);
+                rc = -EINVAL;
+                goto out;
+        }
+        }
+        */
+
+        rtl8127_adv_to_linkmode(advertising, tp->advertising);
+        if (linkmode_empty(edata->advertised)) {
+                linkmode_and(edata->advertised, advertising, eee->supported);
+        } else if (linkmode_andnot(tmp, edata->advertised, advertising)) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised must be a subset of autoneg advertised speeds\n");
+                rc = -EINVAL;
+                goto out;
+        }
+
+        if (linkmode_andnot(tmp, edata->advertised, eee->supported)) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised must be a subset of support \n");
+                rc = -EINVAL;
+                goto out;
+        }
+
+        //tp->eee.eee_enabled = edata->eee_enabled;
+        //tp->eee_adv_t = rtl8127_ethtool_adv_to_mmd_eee_adv_cap1_t(edata->advertised);
+
+        linkmode_copy(eee->advertised, edata->advertised);
+        //eee->tx_lpi_enabled = edata->tx_lpi_enabled;
+        //eee->tx_lpi_timer = edata->tx_lpi_timer;
+        eee->eee_enabled = edata->eee_enabled;
+
+        if (eee->eee_enabled)
+                rtl8127_enable_eee(tp);
+        else
+                rtl8127_disable_eee(tp);
+
+        rtl_nway_reset(net);
+
+out:
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return rc;
+}
+#else
+static int
+rtl_ethtool_get_eee(struct net_device *net, struct ethtool_eee *edata)
+{
+        struct rtl8127_private *tp = netdev_priv(net);
+        struct ethtool_eee *eee = &tp->eee;
+        u32 lp, adv, tx_lpi_timer, supported = 0;
+        unsigned long flags;
+        u16 val;
+
+        if (unlikely(tp->rtk_enable_diag))
+                return -EBUSY;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        /* Get Supported EEE */
+        //val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5C4);
+        //supported = mmd_eee_cap_to_ethtool_sup_t(val);
+        supported = eee->supported;
+
+        /* Get advertisement EEE */
+        adv = eee->advertised;
+
+        /* Get LP advertisement EEE */
+        val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA5D2);
+        lp = mmd_eee_adv_to_ethtool_adv_t(val);
+        val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA6D0);
+        if (val & RTK_LPA_EEE_ADVERTISE_2500FULL)
+                lp |= ADVERTISED_2500baseX_Full;
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        /* Get EEE Tx LPI timer*/
+        tx_lpi_timer = rtl8127_device_lpi_t_to_ethtool_lpi_t(tp, eee->tx_lpi_timer);
+
+        val = rtl8127_mac_ocp_read(tp, 0xE040);
+        val &= BIT_1 | BIT_0;
+
+        edata->eee_enabled = !!val;
+        edata->eee_active = !!(supported & adv & lp);
+        edata->supported = supported;
+        edata->advertised = adv;
+        edata->lp_advertised = lp;
+        edata->tx_lpi_enabled = edata->eee_enabled;
+        edata->tx_lpi_timer = tx_lpi_timer;
+
+        return 0;
+}
+
+static int
+rtl_ethtool_set_eee(struct net_device *net, struct ethtool_eee *edata)
+{
+        struct rtl8127_private *tp = netdev_priv(net);
+        struct ethtool_eee *eee = &tp->eee;
+        unsigned long flags;
+        u32 advertising;
+        int rc = 0;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        if (!HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp) ||
+            tp->DASH) {
+                rc = -EOPNOTSUPP;
+                goto out;
+        }
+
+        if (unlikely(tp->rtk_enable_diag)) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "Diag Enabled\n");
+                rc = -EBUSY;
+                goto out;
+        }
+
+        if (tp->autoneg != AUTONEG_ENABLE) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE requires autoneg\n");
+                rc = -EINVAL;
+                goto out;
+        }
+
+        /*
+        if (edata->tx_lpi_enabled) {
+        if (edata->tx_lpi_timer > tp->max_jumbo_frame_size ||
+            edata->tx_lpi_timer < ETH_MIN_MTU) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "Valid LPI timer range is %d to %d. \n",
+                           ETH_MIN_MTU, tp->max_jumbo_frame_size);
+                rc = -EINVAL;
+                goto out;
+        }
+        }
+        */
+
+        advertising = tp->advertising;
+        if (!edata->advertised) {
+                edata->advertised = advertising & eee->supported;
+        } else if (edata->advertised & ~advertising) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised %x must be a subset of autoneg advertised speeds %x\n",
+                           edata->advertised, advertising);
+                rc = -EINVAL;
+                goto out;
+        }
+
+        if (edata->advertised & ~eee->supported) {
+                dev_printk(KERN_WARNING, tp_to_dev(tp), "EEE advertised %x must be a subset of support %x\n",
+                           edata->advertised, eee->supported);
+                rc = -EINVAL;
+                goto out;
+        }
+
+        //tp->eee.eee_enabled = edata->eee_enabled;
+        //tp->eee_adv_t = ethtool_adv_to_mmd_eee_adv_t(edata->advertised);
+
+        eee->advertised = edata->advertised;
+        //eee->tx_lpi_enabled = edata->tx_lpi_enabled;
+        //eee->tx_lpi_timer = edata->tx_lpi_timer;
+        eee->eee_enabled = edata->eee_enabled;
+
+        if (eee->eee_enabled)
+                rtl8127_enable_eee(tp);
+        else
+                rtl8127_disable_eee(tp);
+
+        rtl_nway_reset(net);
+
+out:
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return rc;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0) */
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) */
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0)
+static void rtl8127_get_channels(struct net_device *dev,
+                                 struct ethtool_channels *channel)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        channel->max_rx = tp->HwSuppNumRxQueues;
+        channel->max_tx = tp->HwSuppNumTxQueues;
+        channel->rx_count = tp->num_rx_rings;
+        channel->tx_count = tp->num_tx_rings;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) */
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+static const struct ethtool_ops rtl8127_ethtool_ops = {
+        .get_drvinfo        = rtl8127_get_drvinfo,
+        .get_regs_len       = rtl8127_get_regs_len,
+        .get_link       = ethtool_op_get_link,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+        .get_ringparam      = rtl8127_get_ringparam,
+        .set_ringparam      = rtl8127_set_ringparam,
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+        .get_settings       = rtl8127_get_settings,
+        .set_settings       = rtl8127_set_settings,
+#else
+        .get_link_ksettings       = rtl8127_get_settings,
+        .set_link_ksettings       = rtl8127_set_settings,
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+        .get_pauseparam     = rtl8127_get_pauseparam,
+        .set_pauseparam     = rtl8127_set_pauseparam,
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
+        .get_msglevel       = rtl8127_get_msglevel,
+        .set_msglevel       = rtl8127_set_msglevel,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+        .get_rx_csum        = rtl8127_get_rx_csum,
+        .set_rx_csum        = rtl8127_set_rx_csum,
+        .get_tx_csum        = rtl8127_get_tx_csum,
+        .set_tx_csum        = rtl8127_set_tx_csum,
+        .get_sg         = ethtool_op_get_sg,
+        .set_sg         = ethtool_op_set_sg,
+#ifdef NETIF_F_TSO
+        .get_tso        = ethtool_op_get_tso,
+        .set_tso        = ethtool_op_set_tso,
+#endif //NETIF_F_TSO
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,3,0)
+        .get_regs       = rtl8127_get_regs,
+        .get_wol        = rtl8127_get_wol,
+        .set_wol        = rtl8127_set_wol,
+        .get_strings        = rtl8127_get_strings,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33)
+        .get_stats_count    = rtl8127_get_stats_count,
+#else
+        .get_sset_count     = rtl8127_get_sset_count,
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33)
+        .get_ethtool_stats  = rtl8127_get_ethtool_stats,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23)
+#ifdef ETHTOOL_GPERMADDR
+        .get_perm_addr      = ethtool_op_get_perm_addr,
+#endif //ETHTOOL_GPERMADDR
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,23)
+        .get_eeprom     = rtl_get_eeprom,
+        .get_eeprom_len     = rtl_get_eeprom_len,
+#ifdef ENABLE_RSS_SUPPORT
+        .get_rxnfc		= rtl8127_get_rxnfc,
+        .set_rxnfc		= rtl8127_set_rxnfc,
+        .get_rxfh_indir_size	= rtl8127_rss_indir_size,
+        .get_rxfh_key_size	= rtl8127_get_rxfh_key_size,
+        .get_rxfh		= rtl8127_get_rxfh,
+        .set_rxfh		= rtl8127_set_rxfh,
+#endif //ENABLE_RSS_SUPPORT
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0)
+#ifdef ENABLE_PTP_SUPPORT
+        .get_ts_info        = rtl8127_get_ts_info,
+#else
+        .get_ts_info        = ethtool_op_get_ts_info,
+#endif //ENABLE_PTP_SUPPORT
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0)
+        .get_eee = rtl_ethtool_get_eee,
+        .set_eee = rtl_ethtool_set_eee,
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,6,0) */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0)
+        .get_channels		= rtl8127_get_channels,
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0) */
+        .nway_reset = rtl_nway_reset,
+
+};
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+
+static void rtl8127_get_mac_version(struct rtl8127_private *tp)
+{
+        u32 reg,val32;
+        u32 ICVerID;
+
+        val32 = RTL_R32(tp, TxConfig);
+        reg = val32 & 0x7c800000;
+        ICVerID = val32 & 0x00700000;
+
+        switch (reg) {
+        case 0x6C800000:
+                if (ICVerID == 0x00000000) {
+                        tp->mcfg = CFG_METHOD_1;
+                } else if (ICVerID == 0x100000) {
+                        tp->mcfg = CFG_METHOD_2;
+                } else {
+                        tp->mcfg = CFG_METHOD_2;
+                        tp->HwIcVerUnknown = TRUE;
+                }
+
+                tp->efuse_ver = EFUSE_SUPPORT_V4;
+                break;
+        default:
+                printk("unknown chip version (%x)\n",reg);
+                tp->mcfg = CFG_METHOD_DEFAULT;
+                tp->HwIcVerUnknown = TRUE;
+                tp->efuse_ver = EFUSE_NOT_SUPPORT;
+                break;
+        }
+}
+
+static void
+rtl8127_print_mac_version(struct rtl8127_private *tp)
+{
+        int i;
+        for (i = ARRAY_SIZE(rtl_chip_info) - 1; i >= 0; i--) {
+                if (tp->mcfg == rtl_chip_info[i].mcfg) {
+                        dprintk("Realtek %s Ethernet controller mcfg = %04d\n",
+                                MODULENAME, rtl_chip_info[i].mcfg);
+                        return;
+                }
+        }
+
+        dprintk("mac_version == Unknown\n");
+}
+
+static void
+rtl8127_tally_counter_addr_fill(struct rtl8127_private *tp)
+{
+        if (!tp->tally_paddr)
+                return;
+
+        RTL_W32(tp, CounterAddrHigh, (u64)tp->tally_paddr >> 32);
+        RTL_W32(tp, CounterAddrLow, (u64)tp->tally_paddr & (DMA_BIT_MASK(32)));
+}
+
+static void
+rtl8127_tally_counter_clear(struct rtl8127_private *tp)
+{
+        if (!tp->tally_paddr)
+                return;
+
+        RTL_W32(tp, CounterAddrHigh, (u64)tp->tally_paddr >> 32);
+        RTL_W32(tp, CounterAddrLow, ((u64)tp->tally_paddr & (DMA_BIT_MASK(32))) | CounterReset);
+}
+
+static void
+rtl8127_clear_phy_ups_reg(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA466, BIT_0);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA468, BIT_3 | BIT_1);
+}
+
+static int
+rtl8127_is_ups_resume(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        return (rtl8127_mac_ocp_read(tp, 0xD42C) & BIT_8);
+}
+
+static void
+rtl8127_clear_ups_resume_bit(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_clear_mac_ocp_bit(tp, 0xD42C, BIT_8);
+}
+
+static u8
+rtl8127_get_phy_state(struct rtl8127_private *tp)
+{
+        return (rtl8127_mdio_direct_read_phy_ocp(tp, 0xA420) & 0x7);
+}
+
+static void
+rtl8127_wait_phy_ups_resume(struct net_device *dev, u16 PhyState)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        for (i=0; i< 100; i++) {
+                if (rtl8127_get_phy_state(tp) == PhyState)
+                        break;
+                else
+                        mdelay(1);
+        }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18)
+        WARN_ON_ONCE(i == 100);
+#endif
+}
+
+static void
+rtl8127_set_mcu_d3_stack(struct rtl8127_private *tp)
+{
+        switch (tp->mcfg) {
+        case CFG_METHOD_2:
+                rtl8127_mac_ocp_write(tp, 0xD018, 0xC896);
+                rtl8127_mac_ocp_write(tp, 0xC896, 0x661A);
+                break;
+        default:
+                return;
+        }
+}
+
+static void
+_rtl8127_enable_now_is_oob(struct rtl8127_private *tp)
+{
+        if (tp->HwSuppNowIsOobVer == 1)
+                RTL_W8(tp, MCUCmd_reg, RTL_R8(tp, MCUCmd_reg) | Now_is_oob);
+}
+
+void
+rtl8127_enable_now_is_oob(struct rtl8127_private *tp)
+{
+        rtl8127_set_mcu_d3_stack(tp);
+        _rtl8127_enable_now_is_oob(tp);
+}
+
+void
+rtl8127_disable_now_is_oob(struct rtl8127_private *tp)
+{
+        if (tp->HwSuppNowIsOobVer == 1)
+                RTL_W8(tp, MCUCmd_reg, RTL_R8(tp, MCUCmd_reg) & ~Now_is_oob);
+}
+
+static void
+rtl8127_exit_oob(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u16 data16;
+
+        rtl8127_disable_rx_packet_filter(tp);
+
+        if (HW_DASH_SUPPORT_DASH(tp))
+                rtl8127_driver_start(tp);
+
+#ifdef ENABLE_REALWOW_SUPPORT
+        rtl8127_realwow_hw_init(dev);
+#else
+        rtl8127_mac_ocp_write(tp, 0xC0BC, 0x00FF);
+#endif //ENABLE_REALWOW_SUPPORT
+
+        rtl8127_nic_reset(dev);
+
+        rtl8127_disable_now_is_oob(tp);
+
+        data16 = rtl8127_mac_ocp_read(tp, 0xE8DE) & ~BIT_14;
+        rtl8127_mac_ocp_write(tp, 0xE8DE, data16);
+        rtl8127_wait_ll_share_fifo_ready(dev);
+
+        rtl8127_mac_ocp_write(tp, 0xC0AA, 0x07D0);
+#ifdef ENABLE_LIB_SUPPORT
+        rtl8127_mac_ocp_write(tp, 0xC0A6, 0x04E2);
+#else
+        rtl8127_mac_ocp_write(tp, 0xC0A6, 0x01B5);
+#endif
+        rtl8127_mac_ocp_write(tp, 0xC01E, 0x5555);
+
+        rtl8127_wait_ll_share_fifo_ready(dev);
+
+        //wait ups resume (phy state 2)
+        if (rtl8127_is_ups_resume(dev)) {
+                rtl8127_wait_phy_ups_resume(dev, 2);
+                rtl8127_clear_ups_resume_bit(dev);
+                rtl8127_clear_phy_ups_reg(dev);
+        }
+}
+
+void
+rtl8127_hw_disable_mac_mcu_bps(struct net_device *dev)
+{
+        u16 regAddr;
+
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_enable_aspm_clkreq_lock(tp, 0);
+
+        rtl8127_mac_ocp_write(tp, 0xFC48, 0x0000);
+
+        for (regAddr = 0xFC28; regAddr < 0xFC48; regAddr += 2) {
+                rtl8127_mac_ocp_write(tp, regAddr, 0x0000);
+        }
+
+        fsleep(3000);
+
+        rtl8127_mac_ocp_write(tp, 0xFC26, 0x0000);
+}
+
+#ifndef ENABLE_USE_FIRMWARE_FILE
+static void
+rtl8127_switch_mac_mcu_ram_code_page(struct rtl8127_private *tp, u16 page)
+{
+        u16 tmpUshort;
+
+        page &= (BIT_1 | BIT_0);
+        tmpUshort = rtl8127_mac_ocp_read(tp, 0xE446);
+        tmpUshort &= ~(BIT_1 | BIT_0);
+        tmpUshort |= page;
+        rtl8127_mac_ocp_write(tp, 0xE446, tmpUshort);
+}
+
+static void
+_rtl8127_set_hw_mcu_patch_code_ver(struct rtl8127_private *tp, u64 ver)
+{
+        int i;
+
+        /* Switch to page 2 */
+        rtl8127_switch_mac_mcu_ram_code_page(tp, 2);
+
+        for (i = 0; i < 8; i += 2) {
+                rtl8127_mac_ocp_write(tp, 0xF9F8 + 6 - i, (u16)ver);
+                ver >>= 16;
+        }
+
+        /* Switch back to page 0 */
+        rtl8127_switch_mac_mcu_ram_code_page(tp, 0);
+}
+
+static void
+rtl8127_set_hw_mcu_patch_code_ver(struct rtl8127_private *tp, u64 ver)
+{
+        _rtl8127_set_hw_mcu_patch_code_ver(tp, ver);
+
+        tp->hw_mcu_patch_code_ver = ver;
+}
+
+static u64
+rtl8127_get_hw_mcu_patch_code_ver(struct rtl8127_private *tp)
+{
+        u64 ver;
+        int i;
+
+        /* Switch to page 2 */
+        rtl8127_switch_mac_mcu_ram_code_page(tp, 2);
+
+        ver = 0;
+        for (i = 0; i < 8; i += 2) {
+                ver <<= 16;
+                ver |= rtl8127_mac_ocp_read(tp, 0xF9F8 + i);
+        }
+
+        /* Switch back to page 0 */
+        rtl8127_switch_mac_mcu_ram_code_page(tp, 0);
+
+        return ver;
+}
+
+static u64
+rtl8127_get_bin_mcu_patch_code_ver(const u16 *entry, u16 entry_cnt)
+{
+        u64 ver;
+        int i;
+
+        if (entry == NULL || entry_cnt == 0 || entry_cnt < 4)
+                return 0;
+
+        ver = 0;
+        for (i = 0; i < 4; i++) {
+                ver <<= 16;
+                ver |= entry[entry_cnt - 4 + i];
+        }
+
+        return ver;
+}
+
+static void
+_rtl8127_write_mac_mcu_ram_code(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt)
+{
+        u16 i;
+
+        for (i = 0; i < entry_cnt; i++)
+                rtl8127_mac_ocp_write(tp, 0xF800 + i * 2, entry[i]);
+}
+
+static void
+_rtl8127_write_mac_mcu_ram_code_with_page(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt, u16 page_size)
+{
+        u16 i;
+        u16 offset;
+
+        if (page_size == 0)
+                return;
+
+        for (i = 0; i < entry_cnt; i++) {
+                offset = i % page_size;
+                if (offset == 0) {
+                        u16 page = (i / page_size);
+                        rtl8127_switch_mac_mcu_ram_code_page(tp, page);
+                }
+                rtl8127_mac_ocp_write(tp, 0xF800 + offset * 2, entry[i]);
+        }
+}
+
+static void
+rtl8127_write_mac_mcu_ram_code(struct rtl8127_private *tp, const u16 *entry, u16 entry_cnt)
+{
+        if (FALSE == HW_SUPPORT_MAC_MCU(tp))
+                return;
+
+        if (entry == NULL || entry_cnt == 0)
+                return;
+
+        if (tp->MacMcuPageSize > 0)
+                _rtl8127_write_mac_mcu_ram_code_with_page(tp, entry, entry_cnt, tp->MacMcuPageSize);
+        else
+                _rtl8127_write_mac_mcu_ram_code(tp, entry, entry_cnt);
+
+        if (tp->bin_mcu_patch_code_ver > 0)
+                rtl8127_set_hw_mcu_patch_code_ver(tp, tp->bin_mcu_patch_code_ver);
+}
+
+static void
+rtl8127_set_mac_mcu_8127a_tc(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        static const u16 mcu_patch_code[] = {
+                0xE010, 0xE019, 0xE01B, 0xE01D, 0xE029, 0xE02C, 0xE0E1, 0xE192, 0xE194,
+                0xE196, 0xE198, 0xE19A, 0xE19C, 0xE19E, 0xE1A0, 0xE1A2, 0xC008, 0x7100,
+                0x4897, 0x9900, 0xC005, 0xC602, 0xBE00, 0x3D8E, 0xD428, 0xD400, 0xC602,
+                0xBE00, 0x0000, 0xC502, 0xBD00, 0x0000, 0x48C3, 0x4847, 0x48C1, 0x8CF8,
+                0x74F8, 0x74F8, 0x74F8, 0x4842, 0x8CF8, 0x1E10, 0xC502, 0xBD00, 0x14BA,
+                0x1E10, 0xC502, 0xBD00, 0x14EE, 0xC643, 0x76C0, 0x49E1, 0xF13F, 0xC140,
+                0x7720, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006,
+                0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A0E, 0x44DA, 0xE893,
+                0x481C, 0xE884, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003,
+                0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A12,
+                0x44DA, 0xE87F, 0x481F, 0xE870, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A,
+                0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C,
+                0x21B8, 0x1A1C, 0x44DA, 0xE86B, 0x481F, 0xE85C, 0xE004, 0xE04F, 0xDD98,
+                0xD450, 0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006,
+                0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A0E, 0x44DA, 0xE854,
+                0x489E, 0x481F, 0xE844, 0xE001, 0x1908, 0xE83E, 0x49E0, 0xF003, 0x1B00,
+                0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002,
+                0x1B0C, 0x21B8, 0x1A8A, 0x44DA, 0xE83D, 0x4813, 0xE82E, 0x49F9, 0xF106,
+                0x4838, 0xE837, 0x4813, 0xE828, 0xE001, 0x49E0, 0xF003, 0x1B00, 0xE00A,
+                0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002, 0x1B0C,
+                0x21B8, 0x1A84, 0x44DA, 0xE823, 0x4890, 0x4811, 0xE813, 0x49F9, 0xF106,
+                0x4838, 0xE81C, 0x4890, 0x4811, 0xE80C, 0xC207, 0x7440, 0xC602, 0xBE00,
+                0x14CC, 0x0FFE, 0xDE20, 0xE092, 0xC3FD, 0xE802, 0xFF80, 0xC0FB, 0x7202,
+                0x49AE, 0xF1FE, 0x9900, 0x44D3, 0x4413, 0x482F, 0x9A02, 0x7202, 0x49AE,
+                0xF1FE, 0xFF80, 0xC0EE, 0x7202, 0x49AE, 0xF1FE, 0x44D3, 0x4413, 0x48AF,
+                0x9A02, 0x7202, 0x49AE, 0xF1FE, 0x7100, 0xFF80, 0xB401, 0xB402, 0xB404,
+                0xB407, 0xC61F, 0x76C0, 0x49E1, 0xF164, 0xC11C, 0x7720, 0x1906, 0xE88A,
+                0x1B0C, 0x21B8, 0x1A40, 0x44DA, 0xE895, 0x4810, 0xE886, 0x190C, 0xE881,
+                0x1B08, 0x21B8, 0x1A26, 0x44DA, 0xE88C, 0x4890, 0x4891, 0xE87C, 0x49F9,
+                0xF107, 0x4898, 0x4899, 0xE877, 0xE003, 0xDD98, 0xD450, 0x1908, 0xE86F,
+                0x49E0, 0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E2,
+                0xF003, 0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A5C, 0x44DA, 0xE86E, 0x4897,
+                0x4898, 0x4819, 0x481A, 0xE85C, 0x49F9, 0xF109, 0x4838, 0xE865, 0x4897,
+                0x4898, 0x4819, 0x481A, 0xE853, 0xE001, 0x190A, 0xE84D, 0x1B00, 0xE85B,
+                0x44E1, 0x4838, 0xE858, 0x44E9, 0x1908, 0xE845, 0x49E0, 0xF003, 0x1B00,
+                0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003, 0x1B08, 0xE002,
+                0x1B0C, 0x21B8, 0x1A86, 0x44DA, 0xE844, 0x44CC, 0xE835, 0x49F9, 0xF108,
+                0x4838, 0xE83E, 0x44CD, 0xE82F, 0xE003, 0xE021, 0xFFC0, 0x190A, 0xE827,
+                0x1B00, 0x4839, 0xE834, 0x249A, 0x1C00, 0x44E1, 0x1909, 0xE81F, 0x49E0,
+                0xF003, 0x1B00, 0xE00A, 0x49E2, 0xF003, 0x1B04, 0xE006, 0x49E4, 0xF003,
+                0x1B08, 0xE002, 0x1B0C, 0x21B8, 0x1A1A, 0x44DA, 0xE81E, 0xC5E4, 0x414D,
+                0x418C, 0xE80D, 0xB007, 0xB004, 0xB002, 0xB001, 0xC602, 0xBE00, 0x14B2,
+                0x0FFE, 0xDE20, 0xC3FE, 0xE802, 0xFF80, 0xC0FC, 0x7202, 0x49AE, 0xF1FE,
+                0x9900, 0x44D3, 0x4413, 0x482F, 0x9A02, 0x7202, 0x49AE, 0xF1FE, 0xFF80,
+                0xC0EF, 0x7202, 0x49AE, 0xF1FE, 0x44D3, 0x4413, 0x48AF, 0x9A02, 0x7202,
+                0x49AE, 0xF1FE, 0x7100, 0xFF80, 0xC502, 0xBD00, 0x0000, 0xC502, 0xBD00,
+                0x0000, 0xC502, 0xBD00, 0x0000, 0xC302, 0xBB00, 0x0000, 0xC602, 0xBE00,
+                0x0000, 0xC102, 0xB900, 0x0000, 0xC102, 0xB900, 0x0000, 0xC602, 0xBE00,
+                0x0000, 0xC602, 0xBE00, 0x0000, 0x1332, 0x0018, 0x0C05, 0x140D
+        };
+
+        /* Get BIN mac mcu patch code version */
+        tp->bin_mcu_patch_code_ver = rtl8127_get_bin_mcu_patch_code_ver(mcu_patch_code, ARRAY_SIZE(mcu_patch_code));
+
+        if (tp->hw_mcu_patch_code_ver != tp->bin_mcu_patch_code_ver)
+                rtl8127_write_mac_mcu_ram_code(tp, mcu_patch_code, ARRAY_SIZE(mcu_patch_code));
+
+        rtl8127_mac_ocp_write(tp, 0xFC26, 0x8000);
+
+        rtl8127_mac_ocp_write(tp, 0xFC2E, 0x14B8);
+        rtl8127_mac_ocp_write(tp, 0xFC30, 0x14EC);
+
+        rtl8127_mac_ocp_write(tp, 0xFC48, 0x0018);
+}
+
+static void
+rtl8127_set_mac_mcu_8127a_1(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        static const u16 mcu_patch_code[] = {
+                0xE010, 0xE023, 0xE036, 0xE049, 0xE05C, 0xE075, 0xE0B1, 0xE117, 0xE11B,
+                0xE11D, 0xE11F, 0xE121, 0xE123, 0xE125, 0xE127, 0xE129, 0x7020, 0xB405,
+                0xB404, 0xC50F, 0x74A0, 0xC50E, 0x4025, 0xF005, 0x4850, 0x4025, 0xF002,
+                0xE002, 0x4809, 0xB004, 0xB005, 0xC502, 0xBD00, 0x1522, 0xD006, 0x0004,
+                0x7760, 0xB405, 0xB404, 0xC50F, 0x74A0, 0xC50E, 0x4025, 0xF005, 0x4850,
+                0x4025, 0xF002, 0xE002, 0x4879, 0xB004, 0xB005, 0xC002, 0xB800, 0x41E2,
+                0xD006, 0x0004, 0x7160, 0xB405, 0xB404, 0xC50F, 0x74A0, 0xC50E, 0x4025,
+                0xF005, 0x4850, 0x4025, 0xF002, 0xE002, 0x4819, 0xB004, 0xB005, 0xC302,
+                0xBB00, 0x508E, 0xD006, 0x0004, 0x7720, 0xB405, 0xB404, 0xC50F, 0x74A0,
+                0xC50E, 0x4025, 0xF005, 0x4850, 0x4025, 0xF002, 0xE002, 0x4879, 0xB004,
+                0xB005, 0xC102, 0xB900, 0x50F8, 0xD006, 0x0004, 0x61A9, 0xB403, 0xB404,
+                0xC313, 0x7460, 0xC312, 0x4023, 0xF005, 0x4830, 0x4023, 0xF002, 0xE003,
+                0x4997, 0xF003, 0xC00A, 0xE002, 0xC009, 0xB004, 0xB003, 0xC102, 0xB900,
+                0x34FC, 0xD006, 0x0004, 0x02AF, 0x041F, 0xB407, 0xB406, 0xB405, 0xB404,
+                0xB403, 0xB402, 0xB401, 0xB400, 0x49D2, 0xF116, 0xC62F, 0x77C0, 0x49F9,
+                0xF020, 0x49FA, 0xF11E, 0x49F2, 0xF102, 0xE01B, 0x48F2, 0x9FC0, 0xC625,
+                0x75C2, 0x4852, 0x9DC2, 0xC122, 0x7020, 0x4801, 0x4802, 0x9820, 0xE00F,
+                0xC61A, 0x77C0, 0x49F2, 0xF10B, 0xC618, 0x77C0, 0x49F2, 0xF007, 0x48F2,
+                0x9FC0, 0xC611, 0x75C2, 0x48D2, 0x9DC2, 0xB000, 0xB001, 0xB002, 0xB003,
+                0xB004, 0xB005, 0xB006, 0xB007, 0x9D6C, 0xC502, 0xBD00, 0x0168, 0xE024,
+                0xC010, 0xD410, 0xD460, 0xB407, 0xB406, 0xB405, 0xB404, 0xB403, 0xB402,
+                0xB401, 0xB400, 0xC152, 0x7020, 0x4981, 0xF043, 0xC050, 0x7100, 0xB401,
+                0xC14C, 0x489E, 0x481D, 0x9900, 0xC24A, 0x7340, 0x49B7, 0xF135, 0xC144,
+                0x9900, 0xC245, 0x7340, 0xC447, 0x401C, 0xF109, 0x734C, 0x1301, 0xF12B,
+                0xC343, 0x9B40, 0xC33E, 0x9B40, 0xE022, 0xC63A, 0x77C0, 0x48F4, 0x48F5,
+                0x48F9, 0x48FA, 0x9FC0, 0xC231, 0x7344, 0x48B4, 0x9B44, 0xC22E, 0x7340,
+                0x4830, 0x48B1, 0x4832, 0x483C, 0x48BD, 0x48BE, 0x48BF, 0x9B40, 0xC223,
+                0xC32A, 0x9B48, 0xC327, 0x9B46, 0xC324, 0x9B40, 0xC321, 0x9B42, 0xC31E,
+                0x9B40, 0xE005, 0xC113, 0x7020, 0x4881, 0x9820, 0xB001, 0xC010, 0x9900,
+                0xB000, 0xB001, 0xB002, 0xB003, 0xB004, 0xB005, 0xB006, 0xB007, 0x2125,
+                0xC102, 0xB900, 0x1A6C, 0xD410, 0xC000, 0xE86C, 0xB600, 0xB800, 0xB40A,
+                0xE024, 0x5A00, 0x5A81, 0x0073, 0x5A80, 0x0042, 0x0001, 0xC104, 0xC202,
+                0xBA00, 0x1A2E, 0xC896, 0xC302, 0xBB00, 0x0000, 0xC002, 0xB800, 0x0000,
+                0xC002, 0xB800, 0x0000, 0xC502, 0xBD00, 0x0000, 0xC102, 0xB900, 0x0000,
+                0xC102, 0xB900, 0x0000, 0xC602, 0xBE00, 0x0000, 0xC602, 0xBE00, 0x0000,
+                0x6961, 0x0019, 0x050C, 0x140C
+        };
+
+        /* Get BIN mac mcu patch code version */
+        tp->bin_mcu_patch_code_ver = rtl8127_get_bin_mcu_patch_code_ver(mcu_patch_code, ARRAY_SIZE(mcu_patch_code));
+
+        if (tp->hw_mcu_patch_code_ver != tp->bin_mcu_patch_code_ver)
+                rtl8127_write_mac_mcu_ram_code(tp, mcu_patch_code, ARRAY_SIZE(mcu_patch_code));
+
+        rtl8127_mac_ocp_write(tp, 0xFC26, 0x8000);
+
+        rtl8127_mac_ocp_write(tp, 0xFC28, 0x1520);
+        rtl8127_mac_ocp_write(tp, 0xFC2A, 0x41E0);
+        rtl8127_mac_ocp_write(tp, 0xFC2C, 0x508C);
+        rtl8127_mac_ocp_write(tp, 0xFC2E, 0x50F6);
+        rtl8127_mac_ocp_write(tp, 0xFC30, 0x34FA);
+        rtl8127_mac_ocp_write(tp, 0xFC32, 0x0166);
+        rtl8127_mac_ocp_write(tp, 0xFC34, 0x1A6A);
+        rtl8127_mac_ocp_write(tp, 0xFC36, 0x1A2C);
+
+        rtl8127_mac_ocp_write(tp, 0xFC48, 0x00FF);
+}
+
+static void
+rtl8127_hw_mac_mcu_config(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (tp->NotWrMcuPatchCode == TRUE)
+                return;
+
+        rtl8127_hw_disable_mac_mcu_bps(dev);
+
+        /* Get H/W mac mcu patch code version */
+        tp->hw_mcu_patch_code_ver = rtl8127_get_hw_mcu_patch_code_ver(tp);
+
+        switch (tp->mcfg) {
+        case CFG_METHOD_1:
+                rtl8127_set_mac_mcu_8127a_tc(dev);
+                break;
+        case CFG_METHOD_2:
+                rtl8127_set_mac_mcu_8127a_1(dev);
+                break;
+        default:
+                break;
+        }
+}
+#endif
+
+#ifdef ENABLE_USE_FIRMWARE_FILE
+static void rtl8127_release_firmware(struct rtl8127_private *tp)
+{
+        if (tp->rtl_fw) {
+                rtl8127_fw_release_firmware(tp->rtl_fw);
+                kfree(tp->rtl_fw);
+                tp->rtl_fw = NULL;
+        }
+}
+
+static void rtl8127_apply_firmware(struct rtl8127_private *tp)
+{
+        unsigned long flags;
+
+        /* TODO: release firmware if rtl_fw_write_firmware signals failure. */
+        if (tp->rtl_fw) {
+                r8127_spin_lock(&tp->phy_lock, flags);
+
+                rtl8127_fw_write_firmware(tp, tp->rtl_fw);
+                /* At least one firmware doesn't reset tp->ocp_base. */
+                tp->ocp_base = OCP_STD_PHY_BASE;
+
+                /* PHY soft reset may still be in progress */
+                //phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
+                //		      !(val & BMCR_RESET),
+                //		      50000, 600000, true);
+                rtl8127_wait_phy_reset_complete(tp);
+
+                tp->hw_ram_code_ver = rtl8127_get_hw_phy_mcu_code_ver(tp);
+                tp->sw_ram_code_ver = tp->hw_ram_code_ver;
+                tp->HwHasWrRamCodeToMicroP = TRUE;
+
+                r8127_spin_unlock(&tp->phy_lock, flags);
+        }
+}
+#endif
+
+static void
+rtl8127_hw_init(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u32 csi_tmp;
+
+        rtl8127_enable_aspm_clkreq_lock(tp, 0);
+        rtl8127_enable_force_clkreq(tp, 0);
+
+        //Disable UPS
+        rtl8127_mac_ocp_write(tp, 0xD40A, rtl8127_mac_ocp_read(tp, 0xD40A) & ~(BIT_4));
+
+#ifndef ENABLE_USE_FIRMWARE_FILE
+        if (!tp->rtl_fw)
+                rtl8127_hw_mac_mcu_config(dev);
+#endif
+
+        //Set PCIE uncorrectable error status mask pcie 0x108
+        csi_tmp = rtl8127_csi_read(tp, 0x108);
+        csi_tmp |= BIT_20;
+        rtl8127_csi_write(tp, 0x108, csi_tmp);
+
+        rtl8127_enable_cfg9346_write(tp);
+        rtl8127_disable_linkchg_wakeup(dev);
+        rtl8127_disable_cfg9346_write(tp);
+        rtl8127_disable_magic_packet(dev);
+        rtl8127_disable_d0_speedup(tp);
+        rtl8127_set_pci_pme(tp, 0);
+        if (s0_magic_packet == 1)
+                rtl8127_enable_magic_packet(dev);
+
+#ifdef ENABLE_USE_FIRMWARE_FILE
+        if (tp->rtl_fw && !tp->resume_not_chg_speed)
+                rtl8127_apply_firmware(tp);
+#endif
+}
+
+static void
+rtl8127_clear_ephy_ext_addr(struct rtl8127_private *tp)
+{
+        rtl8127_set_ephy_ext_addr(tp, 0x0000);
+}
+
+static void
+rtl8127_hw_ephy_config_8127_1(struct rtl8127_private *tp)
+{
+        rtl8127_ephy_write(tp, 0x8088, 0x0064);
+        rtl8127_ephy_write(tp, 0x8488, 0x0064);
+        rtl8127_ephy_write(tp, 0x8888, 0x0064);
+        rtl8127_ephy_write(tp, 0x8C88, 0x0064);
+        rtl8127_ephy_write(tp, 0x8188, 0x0064);
+        rtl8127_ephy_write(tp, 0x8588, 0x0064);
+        rtl8127_ephy_write(tp, 0x8988, 0x0064);
+        rtl8127_ephy_write(tp, 0x8D88, 0x0064);
+        rtl8127_ephy_write(tp, 0x808C, 0x09B0);
+        rtl8127_ephy_write(tp, 0x848C, 0x09B0);
+        rtl8127_ephy_write(tp, 0x888C, 0x0F90);
+        rtl8127_ephy_write(tp, 0x8C8C, 0x0F90);
+        rtl8127_ephy_write(tp, 0x818C, 0x09B0);
+        rtl8127_ephy_write(tp, 0x858C, 0x09B0);
+        rtl8127_ephy_write(tp, 0x898C, 0x0F90);
+        rtl8127_ephy_write(tp, 0x8D8C, 0x0F90);
+        rtl8127_ephy_write(tp, 0x808A, 0x09B8);
+        rtl8127_ephy_write(tp, 0x848A, 0x09B8);
+        rtl8127_ephy_write(tp, 0x888A, 0x0F98);
+        rtl8127_ephy_write(tp, 0x8C8A, 0x0F98);
+        rtl8127_ephy_write(tp, 0x818A, 0x09B8);
+        rtl8127_ephy_write(tp, 0x858A, 0x09B8);
+        rtl8127_ephy_write(tp, 0x898A, 0x0F98);
+        rtl8127_ephy_write(tp, 0x8D8A, 0x0F98);
+        rtl8127_ephy_write(tp, 0x9020, 0x0080);
+        rtl8127_ephy_write(tp, 0x9420, 0x0080);
+        rtl8127_ephy_write(tp, 0x9820, 0x0080);
+        rtl8127_ephy_write(tp, 0x9C20, 0x0080);
+        rtl8127_ephy_write(tp, 0x901E, 0x0190);
+        rtl8127_ephy_write(tp, 0x941E, 0x0190);
+        rtl8127_ephy_write(tp, 0x981E, 0x0140);
+        rtl8127_ephy_write(tp, 0x9C1E, 0x0140);
+        rtl8127_ephy_write(tp, 0x901C, 0x0190);
+        rtl8127_ephy_write(tp, 0x941C, 0x0190);
+        rtl8127_ephy_write(tp, 0x981C, 0x0140);
+        rtl8127_ephy_write(tp, 0x9C1C, 0x0140);
+
+        /* Clear extended address */
+        rtl8127_clear_ephy_ext_addr(tp);
+}
+
+static void
+rtl8127_hw_ephy_config(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        switch (tp->mcfg) {
+        case CFG_METHOD_2:
+                rtl8127_hw_ephy_config_8127_1(tp);
+                break;
+        default:
+                /* nothing to do */
+                break;
+        }
+}
+
+static u16
+rtl8127_get_hw_phy_mcu_code_ver(struct rtl8127_private *tp)
+{
+        u16 hw_ram_code_ver;
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x801E);
+        hw_ram_code_ver = rtl8127_mdio_direct_read_phy_ocp(tp, 0xA438);
+
+        return hw_ram_code_ver;
+}
+
+static int
+rtl8127_check_hw_phy_mcu_code_ver(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ram_code_ver_match = 0;
+
+        tp->hw_ram_code_ver = rtl8127_get_hw_phy_mcu_code_ver(tp);
+
+        if (tp->hw_ram_code_ver == tp->sw_ram_code_ver) {
+                ram_code_ver_match = 1;
+                tp->HwHasWrRamCodeToMicroP = TRUE;
+        }
+
+        return ram_code_ver_match;
+}
+
+bool
+rtl8127_set_phy_mcu_patch_request(struct rtl8127_private *tp)
+{
+        u16 gphy_val;
+        u16 WaitCount;
+        bool bSuccess = TRUE;
+
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB820, BIT_4);
+
+        WaitCount = 0;
+        do {
+                gphy_val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xB800);
+                udelay(100);
+                WaitCount++;
+        } while (!(gphy_val & BIT_6) && (WaitCount < 1000));
+
+        if (!(gphy_val & BIT_6) && (WaitCount == 1000))
+                bSuccess = FALSE;
+
+        if (!bSuccess)
+                dprintk("rtl8127_set_phy_mcu_patch_request fail.\n");
+
+        return bSuccess;
+}
+
+bool
+rtl8127_clear_phy_mcu_patch_request(struct rtl8127_private *tp)
+{
+        u16 gphy_val;
+        u16 WaitCount;
+        bool bSuccess = TRUE;
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB820, BIT_4);
+
+        WaitCount = 0;
+        do {
+                gphy_val = rtl8127_mdio_direct_read_phy_ocp(tp, 0xB800);
+                udelay(100);
+                WaitCount++;
+        } while ((gphy_val & BIT_6) && (WaitCount < 1000));
+
+        if ((gphy_val & BIT_6) && (WaitCount == 1000))
+                bSuccess = FALSE;
+
+        if (!bSuccess)
+                dprintk("rtl8127_clear_phy_mcu_patch_request fail.\n");
+
+        return bSuccess;
+}
+
+#ifndef ENABLE_USE_FIRMWARE_FILE
+static void
+rtl8127_write_hw_phy_mcu_code_ver(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x801E);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, tp->sw_ram_code_ver);
+        tp->hw_ram_code_ver = tp->sw_ram_code_ver;
+}
+
+static void
+rtl8127_set_phy_mcu_ram_code(struct net_device *dev, const u16 *ramcode, u16 codesize)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u16 i;
+        u16 addr;
+        u16 val;
+
+        if (ramcode == NULL || codesize % 2)
+                goto out;
+
+        for (i = 0; i < codesize; i += 2) {
+                addr = ramcode[i];
+                val = ramcode[i + 1];
+                if (addr == 0xFFFF && val == 0xFFFF)
+                        break;
+                rtl8127_mdio_direct_write_phy_ocp(tp, addr, val);
+        }
+
+out:
+        return;
+}
+
+static void
+rtl8127_enable_phy_disable_mode(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        switch (tp->HwSuppCheckPhyDisableModeVer) {
+        case 3:
+                RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) | BIT_5);
+                break;
+        }
+
+        dprintk("enable phy disable mode.\n");
+}
+
+static void
+rtl8127_disable_phy_disable_mode(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        switch (tp->HwSuppCheckPhyDisableModeVer) {
+        case 3:
+                RTL_W8(tp, 0xF2, RTL_R8(tp, 0xF2) & ~BIT_5);
+                break;
+        }
+
+        mdelay(1);
+
+        dprintk("disable phy disable mode.\n");
+}
+
+static const u16 phy_mcu_ram_code_8127a_tc_1[] = {
+        0xa436, 0x8023, 0xa438, 0x3200, 0xa436, 0xB82E, 0xa438, 0x0001,
+        0xBF8E, 0x1410, 0xBF8E, 0x1410, 0xBF90, 0xC20E, 0xBF9E, 0xBFFC,
+        0xBFAA, 0x1924, 0xBFB4, 0x0B1E, 0xBFB6, 0x3740, 0xBFB8, 0x460E,
+        0xBFBE, 0x000D, 0xBF8A, 0x3FF7, 0xBF9A, 0x0007, 0xBF1E, 0x01FF,
+        0xBF1E, 0x01FF, 0xBF1E, 0x01FF, 0xBF2E, 0x454D, 0xbc10, 0xD50C,
+        0xbc10, 0x950C, 0xbc10, 0xD50C, 0xbc10, 0x550C, 0xbddE, 0xEF00,
+        0xbd32, 0xF000, 0xbd2C, 0x0800, 0xbdc8, 0x04B0, 0xbdc8, 0x0CB0,
+        0xBD92, 0x0003, 0xBD94, 0x8000, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xBD92, 0x0016, 0xBD94, 0x1000, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xBD92, 0x0016, 0xBD94, 0x1200, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xBD92, 0x0000, 0xBD94, 0x1F1F, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xBD92, 0x0001, 0xBD94, 0x1F00, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xBD92, 0x0003, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xBD92, 0x0016, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xBD92, 0x0000, 0xBD94, 0x0000, 0xBD96, 0x000f, 0xBD96, 0x0000,
+        0xbdc8, 0x08B0, 0xbdc8, 0x00B0, 0xbd32, 0x0000, 0xbd2C, 0x0000,
+        0xbc10, 0x750C, 0xbc10, 0x650C, 0xbc10, 0x750C, 0xbc10, 0x550C,
+        0xb820, 0x0090, 0xa436, 0xA016, 0xa438, 0x0000, 0xa436, 0xA012,
+        0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010,
+        0xa438, 0x1800, 0xa438, 0x809b, 0xa438, 0x1800, 0xa438, 0x8145,
+        0xa438, 0x1800, 0xa438, 0x8197, 0xa438, 0x1800, 0xa438, 0x81d4,
+        0xa438, 0x1800, 0xa438, 0x8214, 0xa438, 0x1800, 0xa438, 0x8226,
+        0xa438, 0x1800, 0xa438, 0x8232, 0xa438, 0xd707, 0xa438, 0x4141,
+        0xa438, 0xd70a, 0xa438, 0x4115, 0xa438, 0xd705, 0xa438, 0x40da,
+        0xa438, 0xb808, 0xa438, 0xd028, 0xa438, 0xd1c1, 0xa438, 0x1800,
+        0xa438, 0x801e, 0xa438, 0x9808, 0xa438, 0xd07b, 0xa438, 0xd1c5,
+        0xa438, 0xbe10, 0xa438, 0xd503, 0xa438, 0xa108, 0xa438, 0xd505,
+        0xa438, 0x8103, 0xa438, 0xd504, 0xa438, 0xa002, 0xa438, 0xa302,
+        0xa438, 0xd707, 0xa438, 0x4061, 0xa438, 0xd503, 0xa438, 0x8b01,
+        0xa438, 0xd500, 0xa438, 0xc48a, 0xa438, 0xd503, 0xa438, 0xcc09,
+        0xa438, 0xcd58, 0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0x1000,
+        0xa438, 0x1764, 0xa438, 0xd719, 0xa438, 0x606c, 0xa438, 0xd704,
+        0xa438, 0x645c, 0xa438, 0xd75e, 0xa438, 0x604d, 0xa438, 0xfff8,
+        0xa438, 0x9e10, 0xa438, 0x1000, 0xa438, 0x1764, 0xa438, 0xd719,
+        0xa438, 0x606c, 0xa438, 0xd704, 0xa438, 0x631c, 0xa438, 0xd75e,
+        0xa438, 0x404d, 0xa438, 0xfff8, 0xa438, 0xd504, 0xa438, 0xaa18,
+        0xa438, 0xa001, 0xa438, 0xa1e0, 0xa438, 0xd500, 0xa438, 0x1000,
+        0xa438, 0x1764, 0xa438, 0xd719, 0xa438, 0x7fac, 0xa438, 0xd504,
+        0xa438, 0xa001, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x1764,
+        0xa438, 0xd704, 0xa438, 0x5f5c, 0xa438, 0xd719, 0xa438, 0x3aaf,
+        0xa438, 0x8058, 0xa438, 0xf016, 0xa438, 0xd707, 0xa438, 0x6121,
+        0xa438, 0x1000, 0xa438, 0x16f8, 0xa438, 0xd503, 0xa438, 0xcd59,
+        0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x0e49,
+        0xa438, 0xd503, 0xa438, 0x8040, 0xa438, 0xd500, 0xa438, 0x1000,
+        0xa438, 0x16f8, 0xa438, 0xd503, 0xa438, 0xcd5a, 0xa438, 0xaf01,
+        0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x0e2f, 0xa438, 0xd504,
+        0xa438, 0xa008, 0xa438, 0xa204, 0xa438, 0xd500, 0xa438, 0x1000,
+        0xa438, 0x1764, 0xa438, 0xd701, 0xa438, 0x5fa0, 0xa438, 0xd503,
+        0xa438, 0xa082, 0xa438, 0xd500, 0xa438, 0xd71e, 0xa438, 0x4097,
+        0xa438, 0xd078, 0xa438, 0xd1aa, 0xa438, 0xf003, 0xa438, 0xd078,
+        0xa438, 0xd1aa, 0xa438, 0xd707, 0xa438, 0x4081, 0xa438, 0xd70a,
+        0xa438, 0x4055, 0xa438, 0xf014, 0xa438, 0xd706, 0xa438, 0x6065,
+        0xa438, 0xcc89, 0xa438, 0xf002, 0xa438, 0xcc8b, 0xa438, 0x1000,
+        0xa438, 0x0bb2, 0xa438, 0xd705, 0xa438, 0x2ad0, 0xa438, 0x808f,
+        0xa438, 0xf003, 0xa438, 0x1000, 0xa438, 0x0bb8, 0xa438, 0x1000,
+        0xa438, 0x0bbe, 0xa438, 0x607a, 0xa438, 0x9c01, 0xa438, 0xf002,
+        0xa438, 0xbc01, 0xa438, 0x1000, 0xa438, 0x0cc9, 0xa438, 0x1800,
+        0xa438, 0x132f, 0xa438, 0x9a10, 0xa438, 0x9d02, 0xa438, 0xd706,
+        0xa438, 0x629a, 0xa438, 0x61bb, 0xa438, 0xd707, 0xa438, 0x60d7,
+        0xa438, 0xd70a, 0xa438, 0x6196, 0xa438, 0xd0e5, 0xa438, 0xd1e7,
+        0xa438, 0xf012, 0xa438, 0xd70a, 0xa438, 0x5f55, 0xa438, 0xd060,
+        0xa438, 0xd1e8, 0xa438, 0xf00d, 0xa438, 0xd056, 0xa438, 0xd1e8,
+        0xa438, 0xf00a, 0xa438, 0xd043, 0xa438, 0xd1e8, 0xa438, 0xf007,
+        0xa438, 0x609b, 0xa438, 0xd078, 0xa438, 0xd1e9, 0xa438, 0xf003,
+        0xa438, 0xd07f, 0xa438, 0xd1e9, 0xa438, 0xd503, 0xa438, 0xab01,
+        0xa438, 0xd500, 0xa438, 0xd706, 0xa438, 0x6139, 0xa438, 0xd503,
+        0xa438, 0x6065, 0xa438, 0xa7f0, 0xa438, 0xf003, 0xa438, 0x0cf0,
+        0xa438, 0x0750, 0xa438, 0x8908, 0xa438, 0xf004, 0xa438, 0xd503,
+        0xa438, 0xa908, 0xa438, 0x87f0, 0xa438, 0xd503, 0xa438, 0xa040,
+        0xa438, 0xd500, 0xa438, 0xd705, 0xa438, 0x407b, 0xa438, 0x1000,
+        0xa438, 0x1ac8, 0xa438, 0xd503, 0xa438, 0x0c07, 0xa438, 0x0902,
+        0xa438, 0xa008, 0xa438, 0xd500, 0xa438, 0x9d80, 0xa438, 0xc48c,
+        0xa438, 0xd73e, 0xa438, 0x6000, 0xa438, 0xd706, 0xa438, 0x419b,
+        0xa438, 0xd503, 0xa438, 0x0c87, 0xa438, 0x0981, 0xa438, 0xd500,
+        0xa438, 0xd073, 0xa438, 0xd1b7, 0xa438, 0xc490, 0xa438, 0x1000,
+        0xa438, 0x1764, 0xa438, 0xd704, 0xa438, 0x5fbb, 0xa438, 0xd503,
+        0xa438, 0xd706, 0xa438, 0x607a, 0xa438, 0x617c, 0xa438, 0xf013,
+        0xa438, 0xd70b, 0xa438, 0x40b4, 0xa438, 0xd706, 0xa438, 0x6309,
+        0xa438, 0x6348, 0xa438, 0xf025, 0xa438, 0xd706, 0xa438, 0x6408,
+        0xa438, 0xf022, 0xa438, 0xd70b, 0xa438, 0x40b4, 0xa438, 0xd706,
+        0xa438, 0x6429, 0xa438, 0x6468, 0xa438, 0xf028, 0xa438, 0xd706,
+        0xa438, 0x6468, 0xa438, 0xf025, 0xa438, 0xd70b, 0xa438, 0x67d4,
+        0xa438, 0xd706, 0xa438, 0x6488, 0xa438, 0x6589, 0xa438, 0x2c69,
+        0xa438, 0x8135, 0xa438, 0x66ab, 0xa438, 0xf037, 0xa438, 0xc320,
+        0xa438, 0xc420, 0xa438, 0xf03c, 0xa438, 0xd707, 0xa438, 0x6077,
+        0xa438, 0xc310, 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5,
+        0xa438, 0xc340, 0xa438, 0xc410, 0xa438, 0xf033, 0xa438, 0xc340,
+        0xa438, 0xc420, 0xa438, 0xf030, 0xa438, 0xc340, 0xa438, 0xc440,
+        0xa438, 0xf02d, 0xa438, 0xc304, 0xa438, 0xc408, 0xa438, 0xf02a,
+        0xa438, 0xc302, 0xa438, 0xc404, 0xa438, 0xf027, 0xa438, 0xc308,
+        0xa438, 0xc408, 0xa438, 0xf024, 0xa438, 0xc308, 0xa438, 0xc410,
+        0xa438, 0xf021, 0xa438, 0xd707, 0xa438, 0x6077, 0xa438, 0xc304,
+        0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, 0xa438, 0xc308,
+        0xa438, 0xc408, 0xa438, 0xf018, 0xa438, 0xd707, 0xa438, 0x6077,
+        0xa438, 0xc308, 0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5,
+        0xa438, 0xc310, 0xa438, 0xc410, 0xa438, 0xf00f, 0xa438, 0xc304,
+        0xa438, 0xc408, 0xa438, 0xf00c, 0xa438, 0xc302, 0xa438, 0xc404,
+        0xa438, 0xf009, 0xa438, 0xd707, 0xa438, 0x6077, 0xa438, 0xc308,
+        0xa438, 0xf004, 0xa438, 0xd70a, 0xa438, 0x5fb5, 0xa438, 0xc310,
+        0xa438, 0xc410, 0xa438, 0x1800, 0xa438, 0x14e7, 0xa438, 0xd07b,
+        0xa438, 0xd1c5, 0xa438, 0xd503, 0xa438, 0xa028, 0xa438, 0x8970,
+        0xa438, 0x880f, 0xa438, 0x0c0f, 0xa438, 0x0909, 0xa438, 0x87f0,
+        0xa438, 0xc600, 0xa438, 0xa521, 0xa438, 0xd501, 0xa438, 0xce01,
+        0xa438, 0xa202, 0xa438, 0xa201, 0xa438, 0x8201, 0xa438, 0xce00,
+        0xa438, 0xd500, 0xa438, 0xd706, 0xa438, 0x4425, 0xa438, 0xd503,
+        0xa438, 0xab80, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c1d,
+        0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xaac0, 0xa438, 0xd00d,
+        0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xa60c,
+        0xa438, 0xd010, 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a,
+        0xa438, 0xa70c, 0xa438, 0xd09e, 0xa438, 0xd1a2, 0xa438, 0xd700,
+        0xa438, 0x401a, 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505,
+        0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30,
+        0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0xbc02,
+        0xa438, 0xc482, 0xa438, 0xd503, 0xa438, 0xcc01, 0xa438, 0xcd0c,
+        0xa438, 0xaf01, 0xa438, 0xd500, 0xa438, 0xd75e, 0xa438, 0x4000,
+        0xa438, 0xd706, 0xa438, 0x4245, 0xa438, 0x1000, 0xa438, 0x0c1d,
+        0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0x8ac0, 0xa438, 0x860c,
+        0xa438, 0x870c, 0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505,
+        0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30,
+        0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0x1800,
+        0xa438, 0x0e74, 0xa438, 0xd707, 0xa438, 0x43cf, 0xa438, 0x1000,
+        0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xaac0,
+        0xa438, 0xd00d, 0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a,
+        0xa438, 0xa60c, 0xa438, 0xd010, 0xa438, 0xd1a2, 0xa438, 0xd700,
+        0xa438, 0x401a, 0xa438, 0xa70c, 0xa438, 0xd09e, 0xa438, 0xd1a2,
+        0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xce00, 0xa438, 0xd500,
+        0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000,
+        0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500,
+        0xa438, 0xd501, 0xa438, 0xce00, 0xa438, 0xab10, 0xa438, 0xbb10,
+        0xa438, 0x9c02, 0xa438, 0xd75e, 0xa438, 0x6000, 0xa438, 0xd707,
+        0xa438, 0x424f, 0xa438, 0x1000, 0xa438, 0x0c1d, 0xa438, 0xd501,
+        0xa438, 0xce01, 0xa438, 0x8ac0, 0xa438, 0x860c, 0xa438, 0x870c,
+        0xa438, 0xce00, 0xa438, 0xd500, 0xa438, 0xd505, 0xa438, 0xab01,
+        0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0xd505,
+        0xa438, 0x8b01, 0xa438, 0xd500, 0xa438, 0x9808, 0xa438, 0x9c01,
+        0xa438, 0x1800, 0xa438, 0x172a, 0xa438, 0xd707, 0xa438, 0x43cf,
+        0xa438, 0x1000, 0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01,
+        0xa438, 0xaac0, 0xa438, 0xd00d, 0xa438, 0xd1a2, 0xa438, 0xd700,
+        0xa438, 0x401a, 0xa438, 0xa60c, 0xa438, 0xd010, 0xa438, 0xd1a2,
+        0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xa70c, 0xa438, 0xd09e,
+        0xa438, 0xd1a2, 0xa438, 0xd700, 0xa438, 0x401a, 0xa438, 0xce00,
+        0xa438, 0xd500, 0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500,
+        0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01,
+        0xa438, 0xd500, 0xa438, 0x1000, 0xa438, 0x1a6b, 0xa438, 0x9808,
+        0xa438, 0x9c01, 0xa438, 0xd707, 0xa438, 0x424f, 0xa438, 0x1000,
+        0xa438, 0x0c1d, 0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0x8ac0,
+        0xa438, 0x860c, 0xa438, 0x870c, 0xa438, 0xce00, 0xa438, 0xd500,
+        0xa438, 0xd505, 0xa438, 0xab01, 0xa438, 0xd500, 0xa438, 0x1000,
+        0xa438, 0x0c30, 0xa438, 0xd505, 0xa438, 0x8b01, 0xa438, 0xd500,
+        0xa438, 0xd503, 0xa438, 0x8b80, 0xa438, 0xd500, 0xa438, 0x1000,
+        0xa438, 0x1aa5, 0xa438, 0xd503, 0xa438, 0xcda3, 0xa438, 0xaf01,
+        0xa438, 0x1800, 0xa438, 0x181c, 0xa438, 0xd700, 0xa438, 0x41f6,
+        0xa438, 0xd703, 0xa438, 0x41a3, 0xa438, 0x1000, 0xa438, 0x0c1d,
+        0xa438, 0xd501, 0xa438, 0xa580, 0xa438, 0xa701, 0xa438, 0xd500,
+        0xa438, 0xd014, 0xa438, 0xd1c3, 0xa438, 0xd703, 0xa438, 0x401c,
+        0xa438, 0x1000, 0xa438, 0x0c30, 0xa438, 0x1800, 0xa438, 0x069b,
+        0xa438, 0xba10, 0xa438, 0xd70c, 0xa438, 0x4107, 0xa438, 0xd702,
+        0xa438, 0x40d0, 0xa438, 0xd504, 0xa438, 0xa110, 0xa438, 0xd500,
+        0xa438, 0x1800, 0xa438, 0x1447, 0xa438, 0x1800, 0xa438, 0x1420,
+        0xa438, 0xd70c, 0xa438, 0x60a6, 0xa438, 0xd501, 0xa438, 0xce01,
+        0xa438, 0x840f, 0xa438, 0xce00, 0xa438, 0xd503, 0xa438, 0x8008,
+        0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x1360, 0xa436, 0xA026,
+        0xa438, 0x1359, 0xa436, 0xA024, 0xa438, 0x141f, 0xa436, 0xA022,
+        0xa438, 0x068f, 0xa436, 0xA020, 0xa438, 0x1815, 0xa436, 0xA006,
+        0xa438, 0x1723, 0xa436, 0xA004, 0xa438, 0x0e59, 0xa436, 0xA002,
+        0xa438, 0x1452, 0xa436, 0xA000, 0xa438, 0x12bb, 0xa436, 0xA008,
+        0xa438, 0xff00, 0xa436, 0xA016, 0xa438, 0x0010, 0xa436, 0xA012,
+        0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010,
+        0xa438, 0x1800, 0xa438, 0x8010, 0xa438, 0x1800, 0xa438, 0x8019,
+        0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0x1800, 0xa438, 0x8019,
+        0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0x1800, 0xa438, 0x8019,
+        0xa438, 0x1800, 0xa438, 0x8019, 0xa438, 0xd700, 0xa438, 0x2a59,
+        0xa438, 0x0101, 0xa438, 0x2841, 0xa438, 0x0122, 0xa438, 0x2d69,
+        0xa438, 0x00e6, 0xa438, 0x1800, 0xa438, 0x00e0, 0xa436, 0xA08E,
+        0xa438, 0x0000, 0xa436, 0xA08C, 0xa438, 0x0000, 0xa436, 0xA08A,
+        0xa438, 0x0000, 0xa436, 0xA088, 0xa438, 0x0000, 0xa436, 0xA086,
+        0xa438, 0x0000, 0xa436, 0xA084, 0xa438, 0x0000, 0xa436, 0xA082,
+        0xa438, 0x00e2, 0xa436, 0xA080, 0xa438, 0x0000, 0xa436, 0xA090,
+        0xa438, 0x0002, 0xa436, 0xA016, 0xa438, 0x0020, 0xa436, 0xA012,
+        0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010,
+        0xa438, 0x1800, 0xa438, 0x9dfb, 0xa438, 0x1800, 0xa438, 0x9e02,
+        0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0x1800, 0xa438, 0x9e02,
+        0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0x1800, 0xa438, 0x9e02,
+        0xa438, 0x1800, 0xa438, 0x9e02, 0xa438, 0xd71f, 0xa438, 0x626d,
+        0xa438, 0xd71e, 0xa438, 0x4103, 0xa438, 0xa70c, 0xa438, 0xb801,
+        0xa438, 0xba04, 0xa438, 0xd71f, 0xa438, 0x6001, 0xa438, 0x1800,
+        0xa438, 0x001c, 0xa438, 0xb801, 0xa438, 0xd704, 0xa438, 0x2211,
+        0xa438, 0x9d54, 0xa438, 0xd700, 0xa438, 0x2739, 0xa438, 0x837a,
+        0xa438, 0x1800, 0xa438, 0x814b, 0xa438, 0xcd70, 0xa438, 0xb801,
+        0xa438, 0xa708, 0xa438, 0xd700, 0xa438, 0x60d7, 0xa438, 0x6073,
+        0xa438, 0xce02, 0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002,
+        0xa438, 0xce01, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc22,
+        0xa438, 0xcd01, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0xa501,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd,
+        0xa438, 0x8501, 0xa438, 0x1000, 0xa438, 0x9c02, 0xa438, 0xa610,
+        0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xcd71, 0xa438, 0xa501,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd,
+        0xa438, 0x8501, 0xa438, 0xd707, 0xa438, 0x61cf, 0xa438, 0xa502,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd702, 0xa438, 0x5fbe,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503,
+        0xa438, 0xd403, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x8502,
+        0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108,
+        0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, 0xa438, 0xa8c0,
+        0xa438, 0xa2fc, 0xa438, 0xa120, 0xa438, 0x0ca0, 0xa438, 0x0480,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503,
+        0xa438, 0xcd72, 0xa438, 0xd1f5, 0xa438, 0xd057, 0xa438, 0xd1c4,
+        0xa438, 0xd066, 0xa438, 0xd1c4, 0xa438, 0xd077, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x607c, 0xa438, 0x613d,
+        0xa438, 0xfffb, 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fbd, 0xa438, 0xa607, 0xa438, 0xf007,
+        0xa438, 0xa607, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fbc, 0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0x8840, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0cf8, 0xa438, 0x0d48, 0xa438, 0x8320,
+        0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb,
+        0xa438, 0xd706, 0xa438, 0x6227, 0xa438, 0xd700, 0xa438, 0x5f3a,
+        0xa438, 0x88c0, 0xa438, 0x82fc, 0xa438, 0x8120, 0xa438, 0x8350,
+        0xa438, 0x84a0, 0xa438, 0x8607, 0xa438, 0xa510, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8df8, 0xa438, 0x8370, 0xa438, 0x8180,
+        0xa438, 0x9503, 0xa438, 0xff96, 0xa438, 0x8510, 0xa438, 0xa508,
+        0xa438, 0x8508, 0xa438, 0xcd73, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065,
+        0xa438, 0x5f94, 0xa438, 0xffdf, 0xa438, 0xb820, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820,
+        0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108,
+        0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x8480,
+        0xa438, 0x8606, 0xa438, 0xcd74, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8b0f, 0xa438, 0x8c3f, 0xa438, 0x9503, 0xa438, 0xa810,
+        0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0xa4a0, 0xa438, 0xa606,
+        0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x80e5, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8310, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x80f0, 0xa438, 0x33a9, 0xa438, 0x80ed,
+        0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf006, 0xa438, 0xd199,
+        0xa438, 0xd04c, 0xa438, 0xf003, 0xa438, 0xd1d6, 0xa438, 0xd04c,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xd706, 0xa438, 0x5f69, 0xa438, 0xd700, 0xa438, 0x60d7,
+        0xa438, 0x6073, 0xa438, 0xce05, 0xa438, 0xf004, 0xa438, 0xce04,
+        0xa438, 0xf002, 0xa438, 0xce04, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b02,
+        0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503, 0xa438, 0xa180,
+        0xa438, 0xa680, 0xa438, 0xcd75, 0xa438, 0xd199, 0xa438, 0xd04b,
+        0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xa302, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706,
+        0xa438, 0x5f6a, 0xa438, 0xbb50, 0xa438, 0xcd76, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x9b10,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5,
+        0xa438, 0xa120, 0xa438, 0x0c12, 0xa438, 0x0310, 0xa438, 0x8480,
+        0xa438, 0x0c84, 0xa438, 0x0604, 0xa438, 0xcd77, 0xa438, 0xd1a0,
+        0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x422a, 0xa438, 0xa1a0,
+        0xa438, 0xa312, 0xa438, 0xa480, 0xa438, 0xa684, 0xa438, 0xcd78,
+        0xa438, 0xd148, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xcd79, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x7f8a, 0xa438, 0x9b40,
+        0xa438, 0xcd7f, 0xa438, 0xd71f, 0xa438, 0x7fe1, 0xa438, 0x1800,
+        0xa438, 0x001c, 0xa438, 0xa708, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce02, 0xa438, 0xf00a, 0xa438, 0xce02,
+        0xa438, 0xf008, 0xa438, 0xce01, 0xa438, 0xf006, 0xa438, 0xce01,
+        0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, 0xa438, 0xce01,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc22, 0xa438, 0xcd01,
+        0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0xcd15, 0xa438, 0xa501,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd,
+        0xa438, 0x8501, 0xa438, 0xba20, 0xa438, 0xd701, 0xa438, 0x4154,
+        0xa438, 0xd115, 0xa438, 0xd04f, 0xa438, 0x1000, 0xa438, 0x9c02,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xf003, 0xa438, 0x1000, 0xa438, 0x9c02, 0xa438, 0xa610,
+        0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xcd16, 0xa438, 0xa501,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd,
+        0xa438, 0x8501, 0xa438, 0xd707, 0xa438, 0x61cf, 0xa438, 0xa502,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd702, 0xa438, 0x5fbe,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503,
+        0xa438, 0xd403, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x8502,
+        0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108,
+        0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440, 0xa438, 0xa8c0,
+        0xa438, 0xd705, 0xa438, 0x40d3, 0xa438, 0xd707, 0xa438, 0x4082,
+        0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, 0xa438, 0xa2fc,
+        0xa438, 0xa120, 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, 0xa438, 0xcd17,
+        0xa438, 0xd1f5, 0xa438, 0xd057, 0xa438, 0xd1c4, 0xa438, 0xd066,
+        0xa438, 0xd1c4, 0xa438, 0xd077, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x607c, 0xa438, 0x613d, 0xa438, 0xfffb,
+        0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fbd, 0xa438, 0xa607, 0xa438, 0xf007, 0xa438, 0xa607,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbc,
+        0xa438, 0xa310, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fbb, 0xa438, 0x8840, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0cf8, 0xa438, 0x0d48, 0xa438, 0x8320, 0xa438, 0xa180,
+        0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xd706,
+        0xa438, 0x6227, 0xa438, 0xd700, 0xa438, 0x5f3a, 0xa438, 0x88c0,
+        0xa438, 0x82fc, 0xa438, 0x8120, 0xa438, 0x8350, 0xa438, 0x84a0,
+        0xa438, 0x8607, 0xa438, 0xa510, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8df8, 0xa438, 0x8370, 0xa438, 0x8180, 0xa438, 0x9503,
+        0xa438, 0xff8f, 0xa438, 0x8510, 0xa438, 0xa508, 0xa438, 0x8508,
+        0xa438, 0xcd18, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9a20,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065,
+        0xa438, 0x5f94, 0xa438, 0xffde, 0xa438, 0xb820, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820,
+        0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108,
+        0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x8480,
+        0xa438, 0x8606, 0xa438, 0xcd19, 0xa438, 0xd705, 0xa438, 0x40d3,
+        0xa438, 0xd707, 0xa438, 0x4082, 0xa438, 0x8310, 0xa438, 0x1000,
+        0xa438, 0x9c0c, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8b0f,
+        0xa438, 0x8c3f, 0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa120,
+        0xa438, 0xd705, 0xa438, 0x40b3, 0xa438, 0xd707, 0xa438, 0x4062,
+        0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa4a0,
+        0xa438, 0xa606, 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x8236,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8310, 0xa438, 0x9503,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8241, 0xa438, 0x33a9,
+        0xa438, 0x823e, 0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf006,
+        0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf003, 0xa438, 0xd1d6,
+        0xa438, 0xd04c, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706,
+        0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce05, 0xa438, 0xf00a, 0xa438, 0xce05, 0xa438, 0xf008,
+        0xa438, 0xce04, 0xa438, 0xf006, 0xa438, 0xce04, 0xa438, 0xf004,
+        0xa438, 0xce04, 0xa438, 0xf002, 0xa438, 0xce04, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f,
+        0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503,
+        0xa438, 0xa180, 0xa438, 0xa680, 0xa438, 0xcd1a, 0xa438, 0xd199,
+        0xa438, 0xd04b, 0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7b, 0xa438, 0xa302, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f2a, 0xa438, 0xbb10, 0xa438, 0xcd1b,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0xa704, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a,
+        0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006,
+        0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002,
+        0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a,
+        0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0xa686,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503,
+        0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fb5, 0xa438, 0xcd2a, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x5fa4, 0xa438, 0xd17b, 0xa438, 0xd04a,
+        0xa438, 0xa980, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0x800a,
+        0xa438, 0x8604, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8801,
+        0xa438, 0x9503, 0xa438, 0xd40c, 0xa438, 0x1000, 0xa438, 0x9bbf,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503,
+        0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xa00a,
+        0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x39b9, 0xa438, 0x82e4,
+        0xa438, 0xd707, 0xa438, 0x432f, 0xa438, 0xd700, 0xa438, 0x608f,
+        0xa438, 0x60b1, 0xa438, 0x60d3, 0xa438, 0x60f5, 0xa438, 0xce08,
+        0xa438, 0xf007, 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08,
+        0xa438, 0xf003, 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xa1a0, 0xa438, 0xa302, 0xa438, 0xa480,
+        0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8310,
+        0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd17b, 0xa438, 0xd049,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0xd700, 0xa438, 0x37cd,
+        0xa438, 0x8302, 0xa438, 0x800a, 0xa438, 0x8604, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xa00a, 0xa438, 0xa604,
+        0xa438, 0xd17b, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08,
+        0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08,
+        0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08,
+        0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xa1a0, 0xa438, 0xd707, 0xa438, 0x40df, 0xa438, 0x40a2,
+        0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0x8310, 0xa438, 0xf004,
+        0xa438, 0x0cfc, 0xa438, 0x02fc, 0xa438, 0xa310, 0xa438, 0xa302,
+        0xa438, 0xa480, 0xa438, 0xa686, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0xcd2b, 0xa438, 0xd199,
+        0xa438, 0xd04a, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f68, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503,
+        0xa438, 0xcd2c, 0xa438, 0xd199, 0xa438, 0xd04b, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706,
+        0xa438, 0x5f6a, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312,
+        0xa438, 0x8604, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380,
+        0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf,
+        0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd707, 0xa438, 0x409f,
+        0xa438, 0x4062, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310,
+        0xa438, 0xa302, 0xa438, 0xa604, 0xa438, 0xd409, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0xcd2d, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920,
+        0xa438, 0xbb10, 0xa438, 0xcd2e, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x800a, 0xa438, 0x81a0,
+        0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x0c86, 0xa438, 0x0680,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa140,
+        0xa438, 0x9503, 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0x1800, 0xa438, 0x8740,
+        0xa438, 0xa70c, 0xa438, 0x8510, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03,
+        0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03,
+        0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0ccf, 0xa438, 0x0b04, 0xa438, 0xcc21, 0xa438, 0xcd10,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x406d, 0xa438, 0x0cc0,
+        0xa438, 0x0080, 0xa438, 0xd198, 0xa438, 0xd07f, 0xa438, 0xcd11,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fa4,
+        0xa438, 0xd102, 0xa438, 0xd040, 0xa438, 0xcd12, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x1000,
+        0xa438, 0x9c02, 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52,
+        0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8110,
+        0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa224,
+        0xa438, 0xa00a, 0xa438, 0xa802, 0xa438, 0xa980, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa370, 0xa438, 0x9503, 0xa438, 0xcd13,
+        0xa438, 0xd700, 0xa438, 0x2469, 0xa438, 0x83db, 0xa438, 0x634b,
+        0xa438, 0x39b9, 0xa438, 0x83c5, 0xa438, 0xf017, 0xa438, 0xd17a,
+        0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f64, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503,
+        0xa438, 0xcd14, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706,
+        0xa438, 0x5f65, 0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8340, 0xa438, 0xd700, 0xa438, 0x37cd, 0xa438, 0x83e3,
+        0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2469,
+        0xa438, 0x8409, 0xa438, 0xd700, 0xa438, 0x65eb, 0xa438, 0xd700,
+        0xa438, 0x6609, 0xa438, 0xd705, 0xa438, 0x6051, 0xa438, 0xf009,
+        0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd705, 0xa438, 0x7fb1,
+        0xa438, 0xd700, 0xa438, 0x39b9, 0xa438, 0x83fa, 0xa438, 0xf01e,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa801, 0xa438, 0x9503,
+        0xa438, 0xd40c, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xf00f, 0xa438, 0xd700,
+        0xa438, 0x39b9, 0xa438, 0x840d, 0xa438, 0xf00e, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa801, 0xa438, 0x9503, 0xa438, 0xd40c,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x6044,
+        0xa438, 0xf004, 0xa438, 0xd407, 0xa438, 0x1000, 0xa438, 0x9bbf,
+        0xa438, 0xa508, 0xa438, 0x8508, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0xd704,
+        0xa438, 0x6091, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003,
+        0xa438, 0xd1c4, 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x6a8b, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f24, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503, 0xa438, 0xd704,
+        0xa438, 0x6091, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003,
+        0xa438, 0xd1c4, 0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f65,
+        0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340,
+        0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2d49,
+        0xa438, 0x848d, 0xa438, 0x37c9, 0xa438, 0x8459, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xf023, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xd17a,
+        0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f64, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503,
+        0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x5f65,
+        0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380,
+        0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf,
+        0xa438, 0xf00b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x5fab, 0xa438, 0xba08, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7f8b, 0xa438, 0x9a08, 0xa438, 0xa00a,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4,
+        0xa438, 0x800a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340,
+        0xa438, 0xa180, 0xa438, 0x9503, 0xa438, 0xb920, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920,
+        0xa438, 0xd700, 0xa438, 0x296d, 0xa438, 0x84bc, 0xa438, 0x1000,
+        0xa438, 0x9c52, 0xa438, 0xa004, 0xa438, 0x1000, 0xa438, 0x9be6,
+        0xa438, 0x8004, 0xa438, 0xa001, 0xa438, 0x1000, 0xa438, 0x9be6,
+        0xa438, 0x8001, 0xa438, 0xa020, 0xa438, 0x1000, 0xa438, 0x9be6,
+        0xa438, 0x8020, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xd120,
+        0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0x8704, 0xa438, 0xcd21, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fad, 0xa438, 0xa501,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd,
+        0xa438, 0x8501, 0xa438, 0xba20, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x84d5, 0xa438, 0xd700, 0xa438, 0x612b, 0xa438, 0xd701,
+        0xa438, 0x40f4, 0xa438, 0xd196, 0xa438, 0xd04d, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8802,
+        0xa438, 0xcd22, 0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xa501,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fbd,
+        0xa438, 0x8501, 0xa438, 0xa502, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd702, 0xa438, 0x5fbe, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd403, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0x8502, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8801, 0xa438, 0x9503, 0xa438, 0xd40c, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0xd707, 0xa438, 0x428f, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf01d,
+        0xa438, 0xce06, 0xa438, 0xf01b, 0xa438, 0xce06, 0xa438, 0xf019,
+        0xa438, 0xce06, 0xa438, 0xf017, 0xa438, 0xce06, 0xa438, 0xf015,
+        0xa438, 0xce06, 0xa438, 0xf013, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce02, 0xa438, 0xf00a, 0xa438, 0xce02,
+        0xa438, 0xf008, 0xa438, 0xce01, 0xa438, 0xf006, 0xa438, 0xce01,
+        0xa438, 0xf004, 0xa438, 0xce01, 0xa438, 0xf002, 0xa438, 0xce01,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa340, 0xa438, 0x1000,
+        0xa438, 0x9c52, 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6,
+        0xa438, 0x8110, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304,
+        0xa438, 0xa440, 0xa438, 0xa8c0, 0xa438, 0xd707, 0xa438, 0x40cf,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8d07, 0xa438, 0x9503,
+        0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0x0ca0,
+        0xa438, 0x0480, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x853b,
+        0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0xa308,
+        0xa438, 0xf002, 0xa438, 0x8308, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c60, 0xa438, 0x0340, 0xa438, 0x9503, 0xa438, 0xcd23,
+        0xa438, 0xd162, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4,
+        0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x6327, 0xa438, 0xd700,
+        0xa438, 0x5f3b, 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120,
+        0xa438, 0x8358, 0xa438, 0x8308, 0xa438, 0x84a0, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8360, 0xa438, 0x9503, 0xa438, 0xd707,
+        0xa438, 0x2f7d, 0xa438, 0x84d7, 0xa438, 0xd17a, 0xa438, 0xd05a,
+        0xa438, 0xa501, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701,
+        0xa438, 0x5fbd, 0xa438, 0x8501, 0xa438, 0xff8b, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920,
+        0xa438, 0xcd24, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9a20, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94,
+        0xa438, 0xffd9, 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a,
+        0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108,
+        0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8598,
+        0xa438, 0x6144, 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707,
+        0xa438, 0x417f, 0xa438, 0x4140, 0xa438, 0xf006, 0xa438, 0x82fc,
+        0xa438, 0xa201, 0xa438, 0xf007, 0xa438, 0xa2fc, 0xa438, 0xf005,
+        0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002, 0xa438, 0xa2fc,
+        0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x0cc0, 0xa438, 0x0440,
+        0xa438, 0xcd25, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x85b7,
+        0xa438, 0x6224, 0xa438, 0xd704, 0xa438, 0x605d, 0xa438, 0xf004,
+        0xa438, 0xd704, 0xa438, 0x613b, 0xa438, 0xf00b, 0xa438, 0xd700,
+        0xa438, 0x2969, 0xa438, 0x85b7, 0xa438, 0x60eb, 0xa438, 0xd707,
+        0xa438, 0x40bf, 0xa438, 0x4080, 0xa438, 0x8310, 0xa438, 0x1000,
+        0xa438, 0x9c0c, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008,
+        0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004,
+        0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8c07,
+        0xa438, 0x0c07, 0xa438, 0x0d03, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x85d9, 0xa438, 0x60c4, 0xa438, 0xd704, 0xa438, 0x409d,
+        0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0xf003, 0xa438, 0xab07,
+        0xa438, 0xac38, 0xa438, 0x0c07, 0xa438, 0x0d03, 0xa438, 0x9503,
+        0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd700,
+        0xa438, 0x2969, 0xa438, 0x85e7, 0xa438, 0x6064, 0xa438, 0xd704,
+        0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa605, 0xa438, 0xd17a,
+        0xa438, 0xd049, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x85f9,
+        0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a,
+        0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310,
+        0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4046, 0xa438, 0xf014,
+        0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000,
+        0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310,
+        0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380,
+        0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0x8d38,
+        0xa438, 0x9503, 0xa438, 0xcd26, 0xa438, 0xd704, 0xa438, 0x6291,
+        0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x862e, 0xa438, 0x37c9,
+        0xa438, 0x862b, 0xa438, 0x33a9, 0xa438, 0x8628, 0xa438, 0xd18a,
+        0xa438, 0xd04b, 0xa438, 0xf00c, 0xa438, 0xd17a, 0xa438, 0xd04b,
+        0xa438, 0xf009, 0xa438, 0xd1c6, 0xa438, 0xd04b, 0xa438, 0xf006,
+        0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd1c4,
+        0xa438, 0xd046, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8641,
+        0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a,
+        0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310,
+        0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014,
+        0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000,
+        0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310,
+        0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380,
+        0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d01, 0xa438, 0x9503,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08,
+        0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08,
+        0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08,
+        0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xa180, 0xa438, 0xcd27, 0xa438, 0xd704, 0xa438, 0x61f1,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8685, 0xa438, 0x33a9,
+        0xa438, 0x8682, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf009,
+        0xa438, 0xd1b7, 0xa438, 0xd049, 0xa438, 0xf006, 0xa438, 0xd17a,
+        0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd128, 0xa438, 0xd044,
+        0xa438, 0xd13b, 0xa438, 0xd055, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x869a, 0xa438, 0x6164, 0xa438, 0xd704, 0xa438, 0x413d,
+        0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b,
+        0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd700, 0xa438, 0x405b, 0xa438, 0xf014, 0xa438, 0xd704,
+        0xa438, 0x6055, 0xa438, 0xfff7, 0xa438, 0x1000, 0xa438, 0x9c52,
+        0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa380,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xffdf, 0xa438, 0xa302, 0xa438, 0xd700,
+        0xa438, 0x2969, 0xa438, 0x86c4, 0xa438, 0x6184, 0xa438, 0xd704,
+        0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704,
+        0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310,
+        0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x409a,
+        0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014, 0xa438, 0xd704,
+        0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52,
+        0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0xa380,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xffdc, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c0f, 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10,
+        0xa438, 0x9503, 0xa438, 0xcd28, 0xa438, 0xd704, 0xa438, 0x61f1,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x86f5, 0xa438, 0x33a9,
+        0xa438, 0x86f2, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf009,
+        0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd199,
+        0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd128, 0xa438, 0xd044,
+        0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8709, 0xa438, 0x6184,
+        0xa438, 0xd704, 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0,
+        0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002,
+        0xa438, 0xa310, 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x404a, 0xa438, 0xf014,
+        0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000,
+        0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8312,
+        0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380,
+        0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdc, 0xa438, 0x8403,
+        0xa438, 0xd409, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10,
+        0xa438, 0xcd29, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0x800a,
+        0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x8604,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa140,
+        0xa438, 0x9503, 0xa438, 0x9b10, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0xcd30, 0xa438, 0xaa80,
+        0xa438, 0xd704, 0xa438, 0x2319, 0xa438, 0x9d9a, 0xa438, 0xd700,
+        0xa438, 0x40bd, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xaf01,
+        0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x5fb4, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8798,
+        0xa438, 0xd700, 0xa438, 0x273d, 0xa438, 0x8794, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x876a, 0xa438, 0x33a9, 0xa438, 0x8762,
+        0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1b7, 0xa438, 0xd05c,
+        0xa438, 0xf013, 0xa438, 0xd1b7, 0xa438, 0xd05c, 0xa438, 0xf010,
+        0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1b7, 0xa438, 0xd05c,
+        0xa438, 0xf00b, 0xa438, 0xd1b7, 0xa438, 0xd05c, 0xa438, 0xf008,
+        0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd199, 0xa438, 0xd05d,
+        0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd05d, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x8786, 0xa438, 0x33a9, 0xa438, 0x877e,
+        0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1bf, 0xa438, 0xd06d,
+        0xa438, 0xf013, 0xa438, 0xd1de, 0xa438, 0xd06d, 0xa438, 0xf010,
+        0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd1bf, 0xa438, 0xd06d,
+        0xa438, 0xf00b, 0xa438, 0xd1bf, 0xa438, 0xd06d, 0xa438, 0xf008,
+        0xa438, 0xd702, 0xa438, 0x6099, 0xa438, 0xd199, 0xa438, 0xd06e,
+        0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd06e, 0xa438, 0xd703,
+        0xa438, 0x60d0, 0xa438, 0x1000, 0xa438, 0x9c20, 0xa438, 0xd41a,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd408, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0xcd31, 0xa438, 0xd700, 0xa438, 0x2fa9,
+        0xa438, 0x879f, 0xa438, 0x33c9, 0xa438, 0x87a2, 0xa438, 0x6117,
+        0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043, 0xa438, 0xf009,
+        0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, 0xa438, 0xd122,
+        0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, 0xa438, 0xd043,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8817, 0xa438, 0xd700,
+        0xa438, 0x6cab, 0xa438, 0xd700, 0xa438, 0x6d47, 0xa438, 0x800a,
+        0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0x8480, 0xa438, 0x0c86,
+        0xa438, 0x0680, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702,
+        0xa438, 0x9503, 0xa438, 0xd407, 0xa438, 0x1000, 0xa438, 0x9bbf,
+        0xa438, 0xd703, 0xa438, 0x60d0, 0xa438, 0x1000, 0xa438, 0x9c20,
+        0xa438, 0xd41a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd406,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xcd39, 0xa438, 0xd404,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03,
+        0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03,
+        0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x8d07, 0xa438, 0x9503,
+        0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302, 0xa438, 0x0ca0,
+        0xa438, 0x0480, 0xa438, 0xa684, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810,
+        0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xfff0, 0xa438, 0xb820,
+        0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706, 0xa438, 0x5f69,
+        0xa438, 0xf010, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xae80,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2b6d, 0xa438, 0x88c0,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503,
+        0xa438, 0xd406, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xcd32,
+        0xa438, 0xd701, 0xa438, 0x6191, 0xa438, 0xa504, 0xa438, 0xcd3a,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x6067,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f3a,
+        0xa438, 0x8504, 0xa438, 0xd700, 0xa438, 0x2739, 0xa438, 0x88b6,
+        0xa438, 0xd707, 0xa438, 0x6061, 0xa438, 0x1800, 0xa438, 0x88a8,
+        0xa438, 0xd193, 0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06,
+        0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06,
+        0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c0f, 0xa438, 0x0b01, 0xa438, 0x0c3f, 0xa438, 0x0c08,
+        0xa438, 0x9503, 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062,
+        0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa120,
+        0xa438, 0xa420, 0xa438, 0xd193, 0xa438, 0xd048, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f29, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a,
+        0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006,
+        0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002,
+        0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x9503,
+        0xa438, 0x8420, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8312,
+        0xa438, 0x8480, 0xa438, 0x8604, 0xa438, 0xd419, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0xd702, 0xa438, 0x4080, 0xa438, 0xbb20,
+        0xa438, 0x1800, 0xa438, 0x8c32, 0xa438, 0xa00a, 0xa438, 0xa302,
+        0xa438, 0xa480, 0xa438, 0xa604, 0xa438, 0xd193, 0xa438, 0xd047,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f29,
+        0xa438, 0x1800, 0xa438, 0x888b, 0xa438, 0x800a, 0xa438, 0x8302,
+        0xa438, 0x8480, 0xa438, 0x8604, 0xa438, 0xd405, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0xbb20, 0xa438, 0xa00a, 0xa438, 0xa302,
+        0xa438, 0xa480, 0xa438, 0xa604, 0xa438, 0x1800, 0xa438, 0x8c32,
+        0xa438, 0xd405, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd404,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x2b69,
+        0xa438, 0x88c0, 0xa438, 0xf06d, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce03, 0xa438, 0xf00a, 0xa438, 0xce03,
+        0xa438, 0xf008, 0xa438, 0xce03, 0xa438, 0xf006, 0xa438, 0xce03,
+        0xa438, 0xf004, 0xa438, 0xce03, 0xa438, 0xf002, 0xa438, 0xce03,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c07, 0xa438, 0x0c01, 0xa438, 0x0c3f, 0xa438, 0x0d08,
+        0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302,
+        0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x88e8, 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d,
+        0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0xd700, 0xa438, 0x37cd,
+        0xa438, 0x88f0, 0xa438, 0x8310, 0xa438, 0x9503, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4,
+        0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xfff0,
+        0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0xd700,
+        0xa438, 0x2d59, 0xa438, 0x8c32, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8c07, 0xa438, 0x9503, 0xa438, 0xa120, 0xa438, 0xd700,
+        0xa438, 0x2969, 0xa438, 0x891f, 0xa438, 0x60e4, 0xa438, 0xd704,
+        0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310,
+        0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x8928, 0xa438, 0x60a4, 0xa438, 0xd704, 0xa438, 0x407d,
+        0xa438, 0xa202, 0xa438, 0xf002, 0xa438, 0x8202, 0xa438, 0xa420,
+        0xa438, 0x1800, 0xa438, 0x8d48, 0xa438, 0x8810, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fbb, 0xa438, 0xd17a,
+        0xa438, 0xd05a, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008,
+        0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004,
+        0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f,
+        0xa438, 0x0b04, 0xa438, 0x0c3f, 0xa438, 0x0c21, 0xa438, 0x8d07,
+        0xa438, 0x9503, 0xa438, 0xa340, 0xa438, 0x1000, 0xa438, 0x9c52,
+        0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8110,
+        0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304, 0xa438, 0xa440,
+        0xa438, 0xa8c0, 0xa438, 0x8810, 0xa438, 0xa00a, 0xa438, 0xa120,
+        0xa438, 0xa310, 0xa438, 0xd704, 0xa438, 0x405d, 0xa438, 0xa308,
+        0xa438, 0x0cfc, 0xa438, 0x0224, 0xa438, 0x0ca0, 0xa438, 0x0480,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0xd700,
+        0xa438, 0x37cd, 0xa438, 0x896d, 0xa438, 0x8310, 0xa438, 0x9503,
+        0xa438, 0xd162, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4,
+        0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd706, 0xa438, 0x6127, 0xa438, 0xd700,
+        0xa438, 0x5f3b, 0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120,
+        0xa438, 0x8350, 0xa438, 0x84a0, 0xa438, 0xffad, 0xa438, 0xb920,
+        0xa438, 0xcd33, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94, 0xa438, 0xffee,
+        0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a, 0xa438, 0x8120,
+        0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108, 0xa438, 0x1000,
+        0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000, 0xa438, 0x9c5b,
+        0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707, 0xa438, 0x413f,
+        0xa438, 0x4100, 0xa438, 0xf004, 0xa438, 0x0cfd, 0xa438, 0x0201,
+        0xa438, 0xf005, 0xa438, 0x0cfc, 0xa438, 0x02bc, 0xa438, 0xf002,
+        0xa438, 0xa2fc, 0xa438, 0xa304, 0xa438, 0x8880, 0xa438, 0x0cc0,
+        0xa438, 0x0440, 0xa438, 0xcd34, 0xa438, 0xd704, 0xa438, 0x407d,
+        0xa438, 0x405b, 0xa438, 0xf006, 0xa438, 0xd704, 0xa438, 0x60fd,
+        0xa438, 0xd707, 0xa438, 0x40bf, 0xa438, 0x4080, 0xa438, 0x8310,
+        0xa438, 0x1000, 0xa438, 0x9c0c, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8c07, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x8b0f,
+        0xa438, 0x8c38, 0xa438, 0xf004, 0xa438, 0x0c0f, 0xa438, 0x0b07,
+        0xa438, 0xac38, 0xa438, 0x0c38, 0xa438, 0x0d10, 0xa438, 0x9503,
+        0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704,
+        0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa604, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x89db, 0xa438, 0xd17a, 0xa438, 0xd049,
+        0xa438, 0xf003, 0xa438, 0xd19f, 0xa438, 0xd049, 0xa438, 0xd704,
+        0xa438, 0x413d, 0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704,
+        0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310,
+        0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x6a14, 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706,
+        0xa438, 0x4988, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055,
+        0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a,
+        0xa438, 0x8120, 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705,
+        0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b,
+        0xa438, 0xffdd, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8b0f,
+        0xa438, 0x8c38, 0xa438, 0x8d38, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x8a14, 0xa438, 0x33a9, 0xa438, 0x8a11,
+        0xa438, 0xd1f4, 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd1b7,
+        0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd1c6, 0xa438, 0xd04c,
+        0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a, 0xa438, 0xa120,
+        0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002,
+        0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x62f4, 0xa438, 0xd700, 0xa438, 0x405a,
+        0xa438, 0xf01a, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff7,
+        0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120,
+        0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77,
+        0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdf,
+        0xa438, 0x8403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x629c, 0xa438, 0xfffb, 0xa438, 0x8403, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6096, 0xa438, 0xd700,
+        0xa438, 0x619c, 0xa438, 0xfffa, 0xa438, 0xd706, 0xa438, 0x4128,
+        0xa438, 0xd702, 0xa438, 0x60b0, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xae80, 0xa438, 0x9503, 0xa438, 0x1800, 0xa438, 0x8b62,
+        0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06,
+        0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06,
+        0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c0f, 0xa438, 0x0b04, 0xa438, 0x0c3f, 0xa438, 0x0c21,
+        0xa438, 0x8d07, 0xa438, 0x9503, 0xa438, 0xa340, 0xa438, 0x1000,
+        0xa438, 0x9c52, 0xa438, 0xa110, 0xa438, 0x1000, 0xa438, 0x9be6,
+        0xa438, 0x8110, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xa304,
+        0xa438, 0xa440, 0xa438, 0xa8c0, 0xa438, 0x8810, 0xa438, 0xa00a,
+        0xa438, 0xa120, 0xa438, 0xa310, 0xa438, 0xd704, 0xa438, 0x405d,
+        0xa438, 0xa308, 0xa438, 0x0cfc, 0xa438, 0x0224, 0xa438, 0x0ca0,
+        0xa438, 0x0480, 0xa438, 0x8604, 0xa438, 0xcd35, 0xa438, 0xd162,
+        0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0x8840, 0xa438, 0xd1c4, 0xa438, 0xd045,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xd706, 0xa438, 0x6127, 0xa438, 0xd700, 0xa438, 0x5f3b,
+        0xa438, 0x88c0, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8350,
+        0xa438, 0x84a0, 0xa438, 0xffb3, 0xa438, 0xbb80, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920,
+        0xa438, 0xcd36, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0x9b80, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x6065, 0xa438, 0x5f94,
+        0xa438, 0xffe8, 0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x800a,
+        0xa438, 0x8120, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0xa108,
+        0xa438, 0x1000, 0xa438, 0x9be6, 0xa438, 0x8108, 0xa438, 0x1000,
+        0xa438, 0x9c5b, 0xa438, 0xd704, 0xa438, 0x60bd, 0xa438, 0xd707,
+        0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0xf004, 0xa438, 0x0cfd,
+        0xa438, 0x0201, 0xa438, 0xf005, 0xa438, 0x0cfc, 0xa438, 0x02bc,
+        0xa438, 0xf002, 0xa438, 0xa2fc, 0xa438, 0xa304, 0xa438, 0x8880,
+        0xa438, 0x0cc0, 0xa438, 0x0440, 0xa438, 0xcd37, 0xa438, 0xd704,
+        0xa438, 0x407d, 0xa438, 0x405b, 0xa438, 0xf006, 0xa438, 0xd704,
+        0xa438, 0x60fd, 0xa438, 0xd707, 0xa438, 0x40bf, 0xa438, 0x4080,
+        0xa438, 0x8310, 0xa438, 0x1000, 0xa438, 0x9c0c, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8c07, 0xa438, 0xd704, 0xa438, 0x409d,
+        0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0xf004, 0xa438, 0x0c0f,
+        0xa438, 0x0b07, 0xa438, 0xac38, 0xa438, 0x0c38, 0xa438, 0x0d10,
+        0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa120,
+        0xa438, 0xd704, 0xa438, 0x607d, 0xa438, 0xa4a0, 0xa438, 0xa604,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8b01, 0xa438, 0x33a9,
+        0xa438, 0x8afe, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf006,
+        0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd17a,
+        0xa438, 0xd048, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a,
+        0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310,
+        0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048, 0xa438, 0xf014,
+        0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5, 0xa438, 0x1000,
+        0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120, 0xa438, 0x8310,
+        0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77, 0xa438, 0x8380,
+        0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8b0f, 0xa438, 0x8c38, 0xa438, 0x8d38,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8b3a,
+        0xa438, 0x33a9, 0xa438, 0x8b37, 0xa438, 0xd199, 0xa438, 0xd04b,
+        0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003,
+        0xa438, 0xd189, 0xa438, 0xd04c, 0xa438, 0xd704, 0xa438, 0x413d,
+        0xa438, 0xa00a, 0xa438, 0xa120, 0xa438, 0xd704, 0xa438, 0x407b,
+        0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706, 0xa438, 0x4048,
+        0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff5,
+        0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x8120,
+        0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77,
+        0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdd,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d01,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008,
+        0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004,
+        0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xa180, 0xa438, 0xcd38, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x8b92, 0xa438, 0x33a9, 0xa438, 0x8b8a,
+        0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, 0xa438, 0xd04a,
+        0xa438, 0xf013, 0xa438, 0xd100, 0xa438, 0xd049, 0xa438, 0xf010,
+        0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd1b7, 0xa438, 0xd049,
+        0xa438, 0xf00b, 0xa438, 0xd100, 0xa438, 0xd048, 0xa438, 0xf008,
+        0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a, 0xa438, 0xd04b,
+        0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xd13b,
+        0xa438, 0xd055, 0xa438, 0xd704, 0xa438, 0x413d, 0xa438, 0xa00a,
+        0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310,
+        0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa403, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x405b, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055,
+        0xa438, 0xfff7, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a,
+        0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa380, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705,
+        0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b,
+        0xa438, 0xffdf, 0xa438, 0xa302, 0xa438, 0xd704, 0xa438, 0x415d,
+        0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xd704, 0xa438, 0x407b,
+        0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xa302,
+        0xa438, 0xa403, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x409a, 0xa438, 0xd706,
+        0xa438, 0x4048, 0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055,
+        0xa438, 0xfff5, 0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a,
+        0xa438, 0x81a0, 0xa438, 0x8312, 0xa438, 0xa380, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705,
+        0xa438, 0x5f77, 0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b,
+        0xa438, 0xffdc, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f,
+        0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10, 0xa438, 0x9503,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8c03, 0xa438, 0x33a9,
+        0xa438, 0x8bfb, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd199,
+        0xa438, 0xd04a, 0xa438, 0xf013, 0xa438, 0xd100, 0xa438, 0xd04a,
+        0xa438, 0xf010, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a,
+        0xa438, 0xd04a, 0xa438, 0xf00b, 0xa438, 0xd100, 0xa438, 0xd048,
+        0xa438, 0xf008, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd199,
+        0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd16b, 0xa438, 0xd04b,
+        0xa438, 0xd704, 0xa438, 0x415d, 0xa438, 0xa00a, 0xa438, 0xa1a0,
+        0xa438, 0xd704, 0xa438, 0x407b, 0xa438, 0x8310, 0xa438, 0xf002,
+        0xa438, 0xa310, 0xa438, 0xa302, 0xa438, 0xa403, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x40ba, 0xa438, 0xd706, 0xa438, 0x406a, 0xa438, 0x8403,
+        0xa438, 0xf014, 0xa438, 0xd704, 0xa438, 0x6055, 0xa438, 0xfff4,
+        0xa438, 0x1000, 0xa438, 0x9c52, 0xa438, 0x800a, 0xa438, 0x81a0,
+        0xa438, 0x8312, 0xa438, 0xa380, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd705, 0xa438, 0x5f77,
+        0xa438, 0x8380, 0xa438, 0x1000, 0xa438, 0x9c5b, 0xa438, 0xffdb,
+        0xa438, 0x81a0, 0xa438, 0x8310, 0xa438, 0xa302, 0xa438, 0xa00a,
+        0xa438, 0xa480, 0xa438, 0x8420, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x8c42, 0xa438, 0x60e4, 0xa438, 0xd700, 0xa438, 0x40a7,
+        0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002,
+        0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa602,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce03,
+        0xa438, 0xf00a, 0xa438, 0xce03, 0xa438, 0xf008, 0xa438, 0xce03,
+        0xa438, 0xf006, 0xa438, 0xce03, 0xa438, 0xf004, 0xa438, 0xce03,
+        0xa438, 0xf002, 0xa438, 0xce03, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0c01,
+        0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x646d, 0xa438, 0x37c9, 0xa438, 0x8c83, 0xa438, 0x33a9,
+        0xa438, 0x8c79, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702,
+        0xa438, 0x6098, 0xa438, 0xd100, 0xa438, 0xd048, 0xa438, 0xf01a,
+        0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0xf017, 0xa438, 0xd700,
+        0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd100,
+        0xa438, 0xd049, 0xa438, 0xf010, 0xa438, 0xd17a, 0xa438, 0xd04c,
+        0xa438, 0xf00d, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702,
+        0xa438, 0x6098, 0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf006,
+        0xa438, 0xd17a, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd17a,
+        0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706,
+        0xa438, 0x5f29, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008,
+        0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004,
+        0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f,
+        0xa438, 0x0b01, 0xa438, 0x0c3f, 0xa438, 0x0c08, 0xa438, 0x9503,
+        0xa438, 0xa120, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8cc7,
+        0xa438, 0x6224, 0xa438, 0xd700, 0xa438, 0x4147, 0xa438, 0xd704,
+        0xa438, 0x409d, 0xa438, 0xd704, 0xa438, 0x613b, 0xa438, 0xf00a,
+        0xa438, 0xd707, 0xa438, 0x411f, 0xa438, 0x40e0, 0xa438, 0xf004,
+        0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8310,
+        0xa438, 0xf002, 0xa438, 0xa310, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x8cd2, 0xa438, 0x60e4, 0xa438, 0xd700, 0xa438, 0x40a7,
+        0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0xa202, 0xa438, 0xf002,
+        0xa438, 0x8202, 0xa438, 0xa420, 0xa438, 0xcd3b, 0xa438, 0xd700,
+        0xa438, 0x65ad, 0xa438, 0x43c7, 0xa438, 0xd700, 0xa438, 0x37c9,
+        0xa438, 0x8ced, 0xa438, 0x33a9, 0xa438, 0x8ce5, 0xa438, 0xd702,
+        0xa438, 0x4098, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf024,
+        0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf021, 0xa438, 0xd702,
+        0xa438, 0x4098, 0xa438, 0xd1c6, 0xa438, 0xd04c, 0xa438, 0xf01c,
+        0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf019, 0xa438, 0xd702,
+        0xa438, 0x4098, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf014,
+        0xa438, 0xd17a, 0xa438, 0xd04c, 0xa438, 0xf011, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x8d00, 0xa438, 0x33a9, 0xa438, 0x8cfd,
+        0xa438, 0xd1e5, 0xa438, 0xd04c, 0xa438, 0xf009, 0xa438, 0xd191,
+        0xa438, 0xd04d, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04d,
+        0xa438, 0xf003, 0xa438, 0xd16b, 0xa438, 0xd04c, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2c, 0xa438, 0xd700,
+        0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07,
+        0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503,
+        0xa438, 0xcd3c, 0xa438, 0xd700, 0xa438, 0x644d, 0xa438, 0x43c7,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8d33, 0xa438, 0x33a9,
+        0xa438, 0x8d2b, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a,
+        0xa438, 0xd04a, 0xa438, 0xf019, 0xa438, 0xd17a, 0xa438, 0xd048,
+        0xa438, 0xf016, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a,
+        0xa438, 0xd04b, 0xa438, 0xf011, 0xa438, 0xd17a, 0xa438, 0xd048,
+        0xa438, 0xf00e, 0xa438, 0xd702, 0xa438, 0x4098, 0xa438, 0xd17a,
+        0xa438, 0xd04a, 0xa438, 0xf009, 0xa438, 0xd17a, 0xa438, 0xd049,
+        0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf003,
+        0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f2d, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08,
+        0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08,
+        0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c0f, 0xa438, 0x0b02, 0xa438, 0x0c38, 0xa438, 0x0c10,
+        0xa438, 0x9503, 0xa438, 0xa180, 0xa438, 0xcd3d, 0xa438, 0xd700,
+        0xa438, 0x2969, 0xa438, 0x8d8a, 0xa438, 0x37c9, 0xa438, 0x8d80,
+        0xa438, 0x33a9, 0xa438, 0x8d76, 0xa438, 0xd700, 0xa438, 0x40c7,
+        0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd15c, 0xa438, 0xd04a,
+        0xa438, 0xf01a, 0xa438, 0xd199, 0xa438, 0xd04b, 0xa438, 0xf017,
+        0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098,
+        0xa438, 0xd13e, 0xa438, 0xd04a, 0xa438, 0xf010, 0xa438, 0xd199,
+        0xa438, 0xd04b, 0xa438, 0xf00d, 0xa438, 0xd700, 0xa438, 0x40c7,
+        0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a, 0xa438, 0xd04c,
+        0xa438, 0xf006, 0xa438, 0xd199, 0xa438, 0xd04c, 0xa438, 0xf003,
+        0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xd700, 0xa438, 0x37c9,
+        0xa438, 0x8da5, 0xa438, 0x33a9, 0xa438, 0x8d9b, 0xa438, 0xd700,
+        0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd100,
+        0xa438, 0xd050, 0xa438, 0xf017, 0xa438, 0xd17a, 0xa438, 0xd05a,
+        0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702,
+        0xa438, 0x6098, 0xa438, 0xd100, 0xa438, 0xd050, 0xa438, 0xf00d,
+        0xa438, 0xd17a, 0xa438, 0xd05a, 0xa438, 0xf00a, 0xa438, 0xd700,
+        0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a,
+        0xa438, 0xd05a, 0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd05a,
+        0xa438, 0xd707, 0xa438, 0x3ad1, 0xa438, 0x8e73, 0xa438, 0xd705,
+        0xa438, 0x36b1, 0xa438, 0x8ddd, 0xa438, 0xd700, 0xa438, 0x2969,
+        0xa438, 0x8dd5, 0xa438, 0x63c4, 0xa438, 0xd700, 0xa438, 0x40e7,
+        0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x60dc,
+        0xa438, 0x63fb, 0xa438, 0xf016, 0xa438, 0xd707, 0xa438, 0x429f,
+        0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7b, 0xa438, 0xd700,
+        0xa438, 0x40e7, 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704,
+        0xa438, 0x407c, 0xa438, 0x1800, 0xa438, 0x8e73, 0xa438, 0xd706,
+        0xa438, 0x2b59, 0xa438, 0x8e73, 0xa438, 0xf009, 0xa438, 0xaa20,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd701, 0xa438, 0x5f72, 0xa438, 0x8a20, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700,
+        0xa438, 0x4287, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce08, 0xa438, 0xf01e, 0xa438, 0xce08, 0xa438, 0xf01c,
+        0xa438, 0xce08, 0xa438, 0xf01a, 0xa438, 0xce08, 0xa438, 0xf018,
+        0xa438, 0xce08, 0xa438, 0xf016, 0xa438, 0xce08, 0xa438, 0xf014,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08,
+        0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf009, 0xa438, 0xce08,
+        0xa438, 0xf007, 0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08,
+        0xa438, 0xf003, 0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8e23,
+        0xa438, 0x6224, 0xa438, 0xd700, 0xa438, 0x4147, 0xa438, 0xd704,
+        0xa438, 0x409d, 0xa438, 0xd704, 0xa438, 0x417b, 0xa438, 0xf008,
+        0xa438, 0xd707, 0xa438, 0x411f, 0xa438, 0x40e0, 0xa438, 0xf004,
+        0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062, 0xa438, 0x8320,
+        0xa438, 0xf002, 0xa438, 0xa320, 0xa438, 0x8310, 0xa438, 0xcd3e,
+        0xa438, 0xd700, 0xa438, 0x2969, 0xa438, 0x8e4b, 0xa438, 0x37c9,
+        0xa438, 0x8e41, 0xa438, 0x33a9, 0xa438, 0x8e37, 0xa438, 0xd700,
+        0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd15d,
+        0xa438, 0xd04b, 0xa438, 0xf01a, 0xa438, 0xd17a, 0xa438, 0xd04b,
+        0xa438, 0xf017, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702,
+        0xa438, 0x6098, 0xa438, 0xd16b, 0xa438, 0xd04b, 0xa438, 0xf010,
+        0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf00d, 0xa438, 0xd700,
+        0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd17a,
+        0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd04b,
+        0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700,
+        0xa438, 0x2969, 0xa438, 0x8e6b, 0xa438, 0x6264, 0xa438, 0xd700,
+        0xa438, 0x4187, 0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704,
+        0xa438, 0x41bb, 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0xd707,
+        0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0x1800, 0xa438, 0x8f25,
+        0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4060, 0xa438, 0x1800,
+        0xa438, 0x8f25, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f77, 0xa438, 0x1800,
+        0xa438, 0x8f25, 0xa438, 0xd700, 0xa438, 0x4287, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf01e,
+        0xa438, 0xce08, 0xa438, 0xf01c, 0xa438, 0xce08, 0xa438, 0xf01a,
+        0xa438, 0xce08, 0xa438, 0xf018, 0xa438, 0xce08, 0xa438, 0xf016,
+        0xa438, 0xce08, 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x60cf,
+        0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157,
+        0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00b, 0xa438, 0xce08,
+        0xa438, 0xf009, 0xa438, 0xce08, 0xa438, 0xf007, 0xa438, 0xce08,
+        0xa438, 0xf005, 0xa438, 0xce08, 0xa438, 0xf003, 0xa438, 0xce08,
+        0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa102, 0xa438, 0xa802, 0xa438, 0x9503,
+        0xa438, 0xaa02, 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0xac38,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd704,
+        0xa438, 0x60fd, 0xa438, 0xd707, 0xa438, 0x40a0, 0xa438, 0xf003,
+        0xa438, 0xd707, 0xa438, 0x4042, 0xa438, 0xa308, 0xa438, 0xa310,
+        0xa438, 0xcd3f, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8ed0,
+        0xa438, 0x33a9, 0xa438, 0x8ec6, 0xa438, 0xd700, 0xa438, 0x40c7,
+        0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd1d6, 0xa438, 0xd04a,
+        0xa438, 0xf017, 0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf014,
+        0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098,
+        0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf00d, 0xa438, 0xd1d6,
+        0xa438, 0xd04a, 0xa438, 0xf00a, 0xa438, 0xd700, 0xa438, 0x40c7,
+        0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd199, 0xa438, 0xd04a,
+        0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0x0c30, 0xa438, 0x0320, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x8efa, 0xa438, 0x33a9, 0xa438, 0x8ef0,
+        0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098,
+        0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf017, 0xa438, 0xd17a,
+        0xa438, 0xd04b, 0xa438, 0xf014, 0xa438, 0xd700, 0xa438, 0x40c7,
+        0xa438, 0xd702, 0xa438, 0x6098, 0xa438, 0xd199, 0xa438, 0xd04a,
+        0xa438, 0xf00d, 0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf00a,
+        0xa438, 0xd700, 0xa438, 0x40c7, 0xa438, 0xd702, 0xa438, 0x6098,
+        0xa438, 0xd17a, 0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd17a,
+        0xa438, 0xd04b, 0xa438, 0xd700, 0xa438, 0x4287, 0xa438, 0xd704,
+        0xa438, 0x425d, 0xa438, 0xd704, 0xa438, 0x421c, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701,
+        0xa438, 0x5f77, 0xa438, 0x1000, 0xa438, 0x9c16, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xf009, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd701, 0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8102, 0xa438, 0x9503, 0xa438, 0x8a02, 0xa438, 0xcd40,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8380, 0xa438, 0x9503,
+        0xa438, 0xd40a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10,
+        0xa438, 0xcd41, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd71f, 0xa438, 0x5f75, 0xa438, 0x800a,
+        0xa438, 0x81a0, 0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0xd700,
+        0xa438, 0x6047, 0xa438, 0xa680, 0xa438, 0x8606, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xd700, 0xa438, 0x6060,
+        0xa438, 0xa120, 0xa438, 0xa302, 0xa438, 0x9503, 0xa438, 0x8c40,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8702,
+        0xa438, 0x9503, 0xa438, 0x9b30, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8011, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0xcd42, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5f94, 0xa438, 0xd700,
+        0xa438, 0x2969, 0xa438, 0x8f67, 0xa438, 0xd700, 0xa438, 0x6140,
+        0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0xd703, 0xa438, 0x40d0,
+        0xa438, 0x1000, 0xa438, 0x9c20, 0xa438, 0xd41a, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0x8340, 0xa438, 0xa801, 0xa438, 0xd700,
+        0xa438, 0x2fa9, 0xa438, 0x8f70, 0xa438, 0x33c9, 0xa438, 0x8f73,
+        0xa438, 0x6117, 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043,
+        0xa438, 0xf009, 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006,
+        0xa438, 0xd122, 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181,
+        0xa438, 0xd043, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008,
+        0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004,
+        0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0ccf, 0xa438, 0x0b02, 0xa438, 0x8cc7,
+        0xa438, 0x9503, 0xa438, 0xf008, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0ccf, 0xa438, 0x0b46, 0xa438, 0x0cc7, 0xa438, 0x0c03,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d09, 0xa438, 0x9503,
+        0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f,
+        0xa438, 0x0d0a, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x607e,
+        0xa438, 0xa302, 0xa438, 0xf00b, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd14f, 0xa438, 0xd043,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xa810, 0xa438, 0x8240, 0xa438, 0xa00a, 0xa438, 0xa1a0,
+        0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704,
+        0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604,
+        0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa682, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920,
+        0xa438, 0xcd43, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7,
+        0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820,
+        0xa438, 0xbb20, 0xa438, 0xcd44, 0xa438, 0xd700, 0xa438, 0x4060,
+        0xa438, 0x1800, 0xa438, 0x9072, 0xa438, 0xd700, 0xa438, 0x37c9,
+        0xa438, 0x9003, 0xa438, 0x33a9, 0xa438, 0x9000, 0xa438, 0xd17a,
+        0xa438, 0xd047, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048,
+        0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700,
+        0xa438, 0x2d49, 0xa438, 0x9014, 0xa438, 0xd701, 0xa438, 0x60b0,
+        0xa438, 0x1800, 0xa438, 0x90c4, 0xa438, 0x1800, 0xa438, 0x911b,
+        0xa438, 0x0c06, 0xa438, 0x0a06, 0xa438, 0x0cc0, 0xa438, 0x0cc0,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa7a0, 0xa438, 0x9503,
+        0xa438, 0xcd45, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x902a,
+        0xa438, 0x33a9, 0xa438, 0x9027, 0xa438, 0xd17a, 0xa438, 0xd047,
+        0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003,
+        0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x8c40, 0xa438, 0xd418,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xac40, 0xa438, 0xaa01,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd706, 0xa438, 0x5f6f, 0xa438, 0xd700, 0xa438, 0x37c9,
+        0xa438, 0x904b, 0xa438, 0x33a9, 0xa438, 0x9048, 0xa438, 0xd17a,
+        0xa438, 0xd047, 0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048,
+        0xa438, 0xf003, 0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xab20,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503,
+        0xa438, 0xcd46, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9066,
+        0xa438, 0x33a9, 0xa438, 0x9063, 0xa438, 0xd1b7, 0xa438, 0xd04d,
+        0xa438, 0xf006, 0xa438, 0xd1b7, 0xa438, 0xd04d, 0xa438, 0xf003,
+        0xa438, 0xd1b7, 0xa438, 0xd04d, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x1800, 0xa438, 0x90c4,
+        0xa438, 0xd701, 0xa438, 0x4830, 0xa438, 0xd700, 0xa438, 0x47e7,
+        0xa438, 0xd704, 0xa438, 0x47bd, 0xa438, 0xd704, 0xa438, 0x477c,
+        0xa438, 0x0c06, 0xa438, 0x0a06, 0xa438, 0x0cc0, 0xa438, 0x0cc0,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9089, 0xa438, 0x33a9,
+        0xa438, 0x9086, 0xa438, 0xd17a, 0xa438, 0xd047, 0xa438, 0xf006,
+        0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd17a,
+        0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706,
+        0xa438, 0x5f2f, 0xa438, 0x8c40, 0xa438, 0xd418, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0xac40, 0xa438, 0xaa01, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706,
+        0xa438, 0x5f6f, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x90aa,
+        0xa438, 0x33a9, 0xa438, 0x90a7, 0xa438, 0xd17a, 0xa438, 0xd047,
+        0xa438, 0xf006, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0xf003,
+        0xa438, 0xd17a, 0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd701, 0xa438, 0x40f0,
+        0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750,
+        0xa438, 0x9503, 0xa438, 0xab20, 0xa438, 0xd1c4, 0xa438, 0xd046,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x424f,
+        0xa438, 0x8c40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750,
+        0xa438, 0x8380, 0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503,
+        0xa438, 0xcd47, 0xa438, 0xd700, 0xa438, 0x686b, 0xa438, 0x6060,
+        0xa438, 0x1800, 0xa438, 0x90f3, 0xa438, 0xd700, 0xa438, 0x37c9,
+        0xa438, 0x90e7, 0xa438, 0x33a9, 0xa438, 0x90e4, 0xa438, 0xd15c,
+        0xa438, 0xd04c, 0xa438, 0xf006, 0xa438, 0xd182, 0xa438, 0xd04c,
+        0xa438, 0xf003, 0xa438, 0xd191, 0xa438, 0xd04c, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0x1800,
+        0xa438, 0x911b, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf,
+        0xa438, 0x0b03, 0xa438, 0x8cc7, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x60e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f,
+        0xa438, 0x0d1a, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d19, 0xa438, 0x9503,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9111, 0xa438, 0x33a9,
+        0xa438, 0x910e, 0xa438, 0xd19f, 0xa438, 0xd049, 0xa438, 0xf006,
+        0xa438, 0xd199, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd199,
+        0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706,
+        0xa438, 0x5f2f, 0xa438, 0xd416, 0xa438, 0x1000, 0xa438, 0x9bbf,
+        0xa438, 0xbb10, 0xa438, 0xcd4f, 0xa438, 0xcd50, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd71f,
+        0xa438, 0x5f75, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302,
+        0xa438, 0x8480, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682,
+        0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d,
+        0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0x8818,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340, 0xa438, 0xa120,
+        0xa438, 0xa302, 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0x9b30,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8011, 0xa438, 0x9503,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08,
+        0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08,
+        0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08,
+        0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xcd51, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x5f94, 0xa438, 0xd71e, 0xa438, 0x6103, 0xa438, 0xd700,
+        0xa438, 0x2fa9, 0xa438, 0x916f, 0xa438, 0x33c9, 0xa438, 0x9172,
+        0xa438, 0x6177, 0xa438, 0x61b1, 0xa438, 0xd101, 0xa438, 0xd040,
+        0xa438, 0xf00c, 0xa438, 0xd141, 0xa438, 0xd043, 0xa438, 0xf009,
+        0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006, 0xa438, 0xd122,
+        0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181, 0xa438, 0xd043,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xd705, 0xa438, 0x60fe, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8c07, 0xa438, 0x9503,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8cc0, 0xa438, 0x9503,
+        0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d1a, 0xa438, 0x9503,
+        0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f,
+        0xa438, 0x0d19, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2f99,
+        0xa438, 0x919c, 0xa438, 0xa804, 0xa438, 0xd705, 0xa438, 0x607e,
+        0xa438, 0xa302, 0xa438, 0xf00b, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa017, 0xa438, 0x9503, 0xa438, 0xd14f, 0xa438, 0xd043,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480, 0xa438, 0xd700,
+        0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8644,
+        0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604, 0xa438, 0xd701,
+        0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x6047,
+        0xa438, 0xa682, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340,
+        0xa438, 0xc5aa, 0xa438, 0x9503, 0xa438, 0xab80, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5faf, 0xa438, 0xb920,
+        0xa438, 0xcd52, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7fb4, 0xa438, 0x9920, 0xa438, 0xa00a, 0xa438, 0xa1a0,
+        0xa438, 0xd705, 0xa438, 0x605e, 0xa438, 0xa302, 0xa438, 0xa480,
+        0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d,
+        0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604,
+        0xa438, 0xa902, 0xa438, 0x8920, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa480, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9c2a,
+        0xa438, 0xd707, 0xa438, 0x40ae, 0xa438, 0x0c18, 0xa438, 0x0a08,
+        0xa438, 0x1000, 0xa438, 0x9c30, 0xa438, 0xcd60, 0xa438, 0xd101,
+        0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa230,
+        0xa438, 0x9503, 0xa438, 0xd703, 0xa438, 0x68d1, 0xa438, 0xcd62,
+        0xa438, 0x1000, 0xa438, 0x9a89, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd707, 0xa438, 0x4058, 0xa438, 0xa901, 0xa438, 0xd700,
+        0xa438, 0x62dc, 0xa438, 0xd71f, 0xa438, 0x628e, 0xa438, 0xd704,
+        0xa438, 0x4067, 0xa438, 0x1800, 0xa438, 0x96ac, 0xa438, 0xd704,
+        0xa438, 0x40ab, 0xa438, 0xd705, 0xa438, 0x607f, 0xa438, 0x1800,
+        0xa438, 0x974b, 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0xd705,
+        0xa438, 0x405d, 0xa438, 0xf005, 0xa438, 0xd704, 0xa438, 0x5c96,
+        0xa438, 0xd75f, 0xa438, 0x5c40, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xc5aa, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x60de,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c38, 0xa438, 0x0d18,
+        0xa438, 0x9503, 0xa438, 0xd702, 0xa438, 0x4357, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8f80, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00b,
+        0xa438, 0xce08, 0xa438, 0xf009, 0xa438, 0xce08, 0xa438, 0xf007,
+        0xa438, 0xce08, 0xa438, 0xf005, 0xa438, 0xce08, 0xa438, 0xf003,
+        0xa438, 0xce08, 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xcd61, 0xa438, 0x1000, 0xa438, 0x9a89, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd707, 0xa438, 0x4058, 0xa438, 0xa901,
+        0xa438, 0xd71f, 0xa438, 0x2e71, 0xa438, 0x925e, 0xa438, 0xd704,
+        0xa438, 0x2739, 0xa438, 0x96ac, 0xa438, 0xd704, 0xa438, 0x40ab,
+        0xa438, 0xd705, 0xa438, 0x607f, 0xa438, 0x1800, 0xa438, 0x974b,
+        0xa438, 0xd704, 0xa438, 0x60bf, 0xa438, 0xd705, 0xa438, 0x407d,
+        0xa438, 0x1800, 0xa438, 0x97da, 0xa438, 0xd704, 0xa438, 0x5cb6,
+        0xa438, 0xd75f, 0xa438, 0x5c60, 0xa438, 0x1800, 0xa438, 0x997c,
+        0xa438, 0xaa10, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8480,
+        0xa438, 0x9503, 0xa438, 0x9904, 0xa438, 0xcd80, 0xa438, 0x1000,
+        0xa438, 0x9c99, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302,
+        0xa438, 0x8480, 0xa438, 0x8646, 0xa438, 0x1000, 0xa438, 0x9d44,
+        0xa438, 0xd707, 0xa438, 0x605d, 0xa438, 0x8320, 0xa438, 0x8c40,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8340,
+        0xa438, 0x8120, 0xa438, 0x8302, 0xa438, 0xa61c, 0xa438, 0x9503,
+        0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0x8b80,
+        0xa438, 0x0c0c, 0xa438, 0x0808, 0xa438, 0xd702, 0xa438, 0x4191,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f02, 0xa438, 0x9503,
+        0xa438, 0xd400, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xb910,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fae,
+        0xa438, 0x9910, 0xa438, 0xd71f, 0xa438, 0x409e, 0xa438, 0xd1b8,
+        0xa438, 0xd049, 0xa438, 0xf012, 0xa438, 0xd700, 0xa438, 0x2fa9,
+        0xa438, 0x92a1, 0xa438, 0x6131, 0xa438, 0x33c9, 0xa438, 0x92a7,
+        0xa438, 0xd15d, 0xa438, 0xd040, 0xa438, 0xf009, 0xa438, 0xd193,
+        0xa438, 0xd040, 0xa438, 0xf006, 0xa438, 0xd1db, 0xa438, 0xd040,
+        0xa438, 0xf003, 0xa438, 0xd16f, 0xa438, 0xd040, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa380, 0xa438, 0x9503, 0xa438, 0xd417,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd700, 0xa438, 0x4127,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8bc0, 0xa438, 0x8cc7,
+        0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503, 0xa438, 0xf007,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8bc0, 0xa438, 0x8cc7,
+        0xa438, 0xcd48, 0xa438, 0x9503, 0xa438, 0x0c06, 0xa438, 0x0102,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xc555, 0xa438, 0x9503,
+        0xa438, 0xcd81, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd707, 0xa438, 0x40fd, 0xa438, 0xd193,
+        0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xa810, 0xa438, 0x8902, 0xa438, 0xd707,
+        0xa438, 0x407b, 0xa438, 0x8801, 0xa438, 0xa340, 0xa438, 0xd71f,
+        0xa438, 0x3ffd, 0xa438, 0x9521, 0xa438, 0xcd82, 0xa438, 0xd704,
+        0xa438, 0x407f, 0xa438, 0x1800, 0xa438, 0x9390, 0xa438, 0x8810,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa204, 0xa438, 0xa108,
+        0xa438, 0x9503, 0xa438, 0xa604, 0xa438, 0xd131, 0xa438, 0xd046,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba,
+        0xa438, 0xd707, 0xa438, 0x5f75, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8108, 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b02,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c03, 0xa438, 0x0200,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce0b, 0xa438, 0xf00a, 0xa438, 0xce0b, 0xa438, 0xf008,
+        0xa438, 0xce0a, 0xa438, 0xf006, 0xa438, 0xce0a, 0xa438, 0xf004,
+        0xa438, 0xce09, 0xa438, 0xf002, 0xa438, 0xce09, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xab08, 0xa438, 0xcdc0, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd704, 0xa438, 0x5fb7, 0xa438, 0x8b08,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0xa380,
+        0xa438, 0x9503, 0xa438, 0xd414, 0xa438, 0xb308, 0xa438, 0xd202,
+        0xa438, 0xb302, 0xa438, 0xb301, 0xa438, 0xcdc1, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xd400,
+        0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xd71f, 0xa438, 0x6060,
+        0xa438, 0xd704, 0xa438, 0x6074, 0xa438, 0x1800, 0xa438, 0x9390,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c91, 0xa438, 0x0080,
+        0xa438, 0xa060, 0xa438, 0xa101, 0xa438, 0xa408, 0xa438, 0xa240,
+        0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c26,
+        0xa438, 0x0022, 0xa438, 0xa110, 0xa438, 0xa011, 0xa438, 0x9503,
+        0xa438, 0xd17a, 0xa438, 0xd046, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c38, 0xa438, 0x0c18, 0xa438, 0x9503, 0xa438, 0xa318,
+        0xa438, 0xd131, 0xa438, 0xd046, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c83, 0xa438, 0x0282, 0xa438, 0x9503, 0xa438, 0x0c07,
+        0xa438, 0x0b02, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce11, 0xa438, 0xf00a, 0xa438, 0xce11, 0xa438, 0xf008,
+        0xa438, 0xce10, 0xa438, 0xf006, 0xa438, 0xce10, 0xa438, 0xf004,
+        0xa438, 0xce0f, 0xa438, 0xf002, 0xa438, 0xce0f, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd704, 0xa438, 0x3ebc, 0xa438, 0x9370, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8284, 0xa438, 0x9503, 0xa438, 0x8b08,
+        0xa438, 0xd704, 0xa438, 0x61b9, 0xa438, 0xd705, 0xa438, 0x407c,
+        0xa438, 0x1800, 0xa438, 0x938e, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa402, 0xa438, 0x9503, 0xa438, 0x8310, 0xa438, 0xcdc3,
+        0xa438, 0x1800, 0xa438, 0x9337, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8402, 0xa438, 0x9503, 0xa438, 0xcdc2, 0xa438, 0xf003,
+        0xa438, 0x1000, 0xa438, 0x9c99, 0xa438, 0x8320, 0xa438, 0xd705,
+        0xa438, 0x429e, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce02, 0xa438, 0xf01d, 0xa438, 0xce02, 0xa438, 0xf01b,
+        0xa438, 0xce01, 0xa438, 0xf019, 0xa438, 0xce01, 0xa438, 0xf017,
+        0xa438, 0xce01, 0xa438, 0xf015, 0xa438, 0xce01, 0xa438, 0xf013,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06,
+        0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008, 0xa438, 0xce06,
+        0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004, 0xa438, 0xce06,
+        0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xd705, 0xa438, 0x61be, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0x0c38, 0xa438, 0x0c28,
+        0xa438, 0x9503, 0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0xa302,
+        0xa438, 0xa4a0, 0xa438, 0xf012, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c0f, 0xa438, 0x0b06, 0xa438, 0x0c3f, 0xa438, 0x0c1b,
+        0xa438, 0xd700, 0xa438, 0x6067, 0xa438, 0x0cc0, 0xa438, 0x0d80,
+        0xa438, 0x9503, 0xa438, 0x8810, 0xa438, 0x800a, 0xa438, 0x8302,
+        0xa438, 0xa8c0, 0xa438, 0x8420, 0xa438, 0xa480, 0xa438, 0xd700,
+        0xa438, 0x40c7, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x0ce3,
+        0xa438, 0x0203, 0xa438, 0xf003, 0xa438, 0x0ce0, 0xa438, 0x02a0,
+        0xa438, 0xa120, 0xa438, 0xd700, 0xa438, 0x4187, 0xa438, 0xd704,
+        0xa438, 0x40dd, 0xa438, 0xd704, 0xa438, 0x617b, 0xa438, 0xd704,
+        0xa438, 0x613c, 0xa438, 0xf00a, 0xa438, 0xd707, 0xa438, 0x411f,
+        0xa438, 0x40e0, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x409f,
+        0xa438, 0x4062, 0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa318,
+        0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d,
+        0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0xd700,
+        0xa438, 0x6047, 0xa438, 0xa682, 0xa438, 0x1000, 0xa438, 0x9cc6,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4,
+        0xa438, 0xb920, 0xa438, 0xcd83, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5,
+        0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706,
+        0xa438, 0x5fa7, 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5,
+        0xa438, 0x9820, 0xa438, 0xbb20, 0xa438, 0xd705, 0xa438, 0x605e,
+        0xa438, 0xf018, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008,
+        0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004,
+        0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa420, 0xa438, 0x88c0,
+        0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x9450, 0xa438, 0x33a9, 0xa438, 0x944d,
+        0xa438, 0xd1b7, 0xa438, 0xd04b, 0xa438, 0xf006, 0xa438, 0xd1b7,
+        0xa438, 0xd04b, 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04b,
+        0xa438, 0xd707, 0xa438, 0x3ad1, 0xa438, 0x94d8, 0xa438, 0xd700,
+        0xa438, 0x40c7, 0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0xd704,
+        0xa438, 0x465c, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x605f,
+        0xa438, 0xf02e, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x946a,
+        0xa438, 0x33a9, 0xa438, 0x9467, 0xa438, 0xd199, 0xa438, 0xd05a,
+        0xa438, 0xf006, 0xa438, 0xd199, 0xa438, 0xd05a, 0xa438, 0xf003,
+        0xa438, 0xd1d6, 0xa438, 0xd05a, 0xa438, 0xcd84, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7b, 0xa438, 0xd700, 0xa438, 0x42c7, 0xa438, 0xd704,
+        0xa438, 0x429d, 0xa438, 0xd704, 0xa438, 0x425c, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd706,
+        0xa438, 0x5f6f, 0xa438, 0x8a04, 0xa438, 0x8b20, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa380, 0xa438, 0x9503, 0xa438, 0xd401,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x1800, 0xa438, 0x94d8,
+        0xa438, 0xd706, 0xa438, 0x69ab, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a,
+        0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700, 0xa438, 0x4167,
+        0xa438, 0xd704, 0xa438, 0x40bd, 0xa438, 0xd704, 0xa438, 0x615b,
+        0xa438, 0x1800, 0xa438, 0x94a5, 0xa438, 0xd707, 0xa438, 0x411f,
+        0xa438, 0x40e0, 0xa438, 0xf004, 0xa438, 0xd707, 0xa438, 0x409f,
+        0xa438, 0x4062, 0xa438, 0x8320, 0xa438, 0xf002, 0xa438, 0xa320,
+        0xa438, 0x8310, 0xa438, 0xcd85, 0xa438, 0xd700, 0xa438, 0x37c9,
+        0xa438, 0x94b3, 0xa438, 0x33a9, 0xa438, 0x94b0, 0xa438, 0xd1b7,
+        0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd1b7, 0xa438, 0xd04a,
+        0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd700,
+        0xa438, 0x41a7, 0xa438, 0xd704, 0xa438, 0x40dd, 0xa438, 0xd704,
+        0xa438, 0x605b, 0xa438, 0xf00d, 0xa438, 0x1800, 0xa438, 0x8f25,
+        0xa438, 0xd707, 0xa438, 0x413f, 0xa438, 0x4100, 0xa438, 0x1800,
+        0xa438, 0x8f25, 0xa438, 0xd707, 0xa438, 0x409f, 0xa438, 0x4062,
+        0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd701, 0xa438, 0x5f77,
+        0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0xd705, 0xa438, 0x413e,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f, 0xa438, 0x0b03,
+        0xa438, 0x0c38, 0xa438, 0x0c00, 0xa438, 0xa104, 0xa438, 0x9503,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa102, 0xa438, 0x9503,
+        0xa438, 0xaa02, 0xa438, 0xa310, 0xa438, 0xcd86, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x94f4, 0xa438, 0x33a9, 0xa438, 0x94f1,
+        0xa438, 0xd1d6, 0xa438, 0xd04a, 0xa438, 0xf006, 0xa438, 0xd1d6,
+        0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd199, 0xa438, 0xd04a,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36,
+        0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0x0c30, 0xa438, 0x0320,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9509, 0xa438, 0x33a9,
+        0xa438, 0x9506, 0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf006,
+        0xa438, 0xd1b7, 0xa438, 0xd04a, 0xa438, 0xf003, 0xa438, 0xd1b7,
+        0xa438, 0xd04a, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd701,
+        0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8102,
+        0xa438, 0x8104, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x40b0,
+        0xa438, 0xaa02, 0xa438, 0xac80, 0xa438, 0x1800, 0xa438, 0x8f25,
+        0xa438, 0x8a02, 0xa438, 0x1800, 0xa438, 0x8f25, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa708, 0xa438, 0x9503, 0xa438, 0xcd87,
+        0xa438, 0xd704, 0xa438, 0x615f, 0xa438, 0x8810, 0xa438, 0xd702,
+        0xa438, 0x40e6, 0xa438, 0xd17a, 0xa438, 0xd048, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce06, 0xa438, 0xf00a,
+        0xa438, 0xce06, 0xa438, 0xf008, 0xa438, 0xce06, 0xa438, 0xf006,
+        0xa438, 0xce06, 0xa438, 0xf004, 0xa438, 0xce06, 0xa438, 0xf002,
+        0xa438, 0xce06, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xd707,
+        0xa438, 0x413b, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f,
+        0xa438, 0x0b03, 0xa438, 0x0c38, 0xa438, 0x0c18, 0xa438, 0x9503,
+        0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f,
+        0xa438, 0x0b07, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x40c7,
+        0xa438, 0xd704, 0xa438, 0x409d, 0xa438, 0x0ce3, 0xa438, 0x0203,
+        0xa438, 0xf003, 0xa438, 0x0ce0, 0xa438, 0x02a0, 0xa438, 0xa120,
+        0xa438, 0xd707, 0xa438, 0x425b, 0xa438, 0xd700, 0xa438, 0x4167,
+        0xa438, 0xd704, 0xa438, 0x40dd, 0xa438, 0xd704, 0xa438, 0x613b,
+        0xa438, 0xd704, 0xa438, 0x60fc, 0xa438, 0xf008, 0xa438, 0xd707,
+        0xa438, 0x40df, 0xa438, 0xf003, 0xa438, 0xd707, 0xa438, 0x407f,
+        0xa438, 0x8310, 0xa438, 0xf002, 0xa438, 0xa318, 0xa438, 0xa00a,
+        0xa438, 0xa302, 0xa438, 0xa4a0, 0xa438, 0xd700, 0xa438, 0x40a7,
+        0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002,
+        0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682,
+        0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0xcd88,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4,
+        0xa438, 0x8810, 0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x60e5, 0xa438, 0x5f94, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xffef,
+        0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0xbb20,
+        0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x95be, 0xa438, 0x33a9, 0xa438, 0x95b6,
+        0xa438, 0xd702, 0xa438, 0x4086, 0xa438, 0xd161, 0xa438, 0xd049,
+        0xa438, 0xf00e, 0xa438, 0xd19e, 0xa438, 0xd049, 0xa438, 0xf00b,
+        0xa438, 0xd702, 0xa438, 0x4086, 0xa438, 0xd186, 0xa438, 0xd049,
+        0xa438, 0xf006, 0xa438, 0xd1c3, 0xa438, 0xd049, 0xa438, 0xf003,
+        0xa438, 0xd1cf, 0xa438, 0xd049, 0xa438, 0xcd89, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x5f2f, 0xa438, 0xd707,
+        0xa438, 0x40fb, 0xa438, 0x8310, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c0f, 0xa438, 0x0b05, 0xa438, 0x9503, 0xa438, 0xcd8a,
+        0xa438, 0xd707, 0xa438, 0x419b, 0xa438, 0xd700, 0xa438, 0x33ad,
+        0xa438, 0x95df, 0xa438, 0xd189, 0xa438, 0xd045, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xcd8b, 0xa438, 0xd40a, 0xa438, 0x1000,
+        0xa438, 0x9bbf, 0xa438, 0xbb10, 0xa438, 0xcd8c, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd71f,
+        0xa438, 0x5f75, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302,
+        0xa438, 0x8480, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa680,
+        0xa438, 0x8606, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8340,
+        0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x9b30,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5,
+        0xa438, 0xcd8d, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x5f94, 0xa438, 0x8340, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8708, 0xa438, 0x9503, 0xa438, 0xa801, 0xa438, 0xd700,
+        0xa438, 0x2fa9, 0xa438, 0x9610, 0xa438, 0x33c9, 0xa438, 0x9613,
+        0xa438, 0x6117, 0xa438, 0xf00a, 0xa438, 0xd141, 0xa438, 0xd043,
+        0xa438, 0xf009, 0xa438, 0xd121, 0xa438, 0xd043, 0xa438, 0xf006,
+        0xa438, 0xd122, 0xa438, 0xd042, 0xa438, 0xf003, 0xa438, 0xd181,
+        0xa438, 0xd043, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce06, 0xa438, 0xf00a, 0xa438, 0xce06, 0xa438, 0xf008,
+        0xa438, 0xce06, 0xa438, 0xf006, 0xa438, 0xce06, 0xa438, 0xf004,
+        0xa438, 0xce06, 0xa438, 0xf002, 0xa438, 0xce06, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c0f,
+        0xa438, 0x0b02, 0xa438, 0x8cc7, 0xa438, 0x9503, 0xa438, 0xd700,
+        0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f,
+        0xa438, 0x0d09, 0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d08, 0xa438, 0x9503,
+        0xa438, 0xa810, 0xa438, 0xa00a, 0xa438, 0x0ca0, 0xa438, 0x0120,
+        0xa438, 0xa302, 0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7,
+        0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002,
+        0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7,
+        0xa438, 0xb920, 0xa438, 0xcd8e, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x8810, 0xa438, 0x9920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x60e5,
+        0xa438, 0x5f94, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706,
+        0xa438, 0x5fa7, 0xa438, 0xffef, 0xa438, 0xb820, 0xa438, 0xa810,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fa5,
+        0xa438, 0x9820, 0xa438, 0xbb20, 0xa438, 0xd701, 0xa438, 0x40d0,
+        0xa438, 0xac40, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750,
+        0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd046, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x9c36, 0xa438, 0xd700,
+        0xa438, 0x5f7a, 0xa438, 0xd706, 0xa438, 0x422f, 0xa438, 0x8c40,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8750, 0xa438, 0x8380,
+        0xa438, 0x9503, 0xa438, 0xd417, 0xa438, 0x1000, 0xa438, 0x9bbf,
+        0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503, 0xa438, 0xd192,
+        0xa438, 0xd047, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x9c36, 0xa438, 0xd700, 0xa438, 0x5f7a, 0xa438, 0xd706,
+        0xa438, 0x5f2f, 0xa438, 0xd707, 0xa438, 0x409b, 0xa438, 0xd416,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xbb10, 0xa438, 0xcd8f,
+        0xa438, 0x1800, 0xa438, 0x9120, 0xa438, 0x1000, 0xa438, 0x9c99,
+        0xa438, 0x8302, 0xa438, 0x8646, 0xa438, 0xd700, 0xa438, 0x60c7,
+        0xa438, 0xa680, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702,
+        0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa680,
+        0xa438, 0x850f, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4050,
+        0xa438, 0x8b20, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf,
+        0xa438, 0x0bc8, 0xa438, 0x0cc7, 0xa438, 0x0c44, 0xa438, 0x9503,
+        0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0xd700, 0xa438, 0x40e7,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d24,
+        0xa438, 0x9503, 0xa438, 0xf005, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xcde4, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9cdd,
+        0xa438, 0xcd90, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa601,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x2fa9, 0xa438, 0x96e1,
+        0xa438, 0x60d1, 0xa438, 0x6113, 0xa438, 0x6157, 0xa438, 0xd150,
+        0xa438, 0xd040, 0xa438, 0xf009, 0xa438, 0xd1a0, 0xa438, 0xd040,
+        0xa438, 0xf006, 0xa438, 0xd128, 0xa438, 0xd040, 0xa438, 0xf003,
+        0xa438, 0xd114, 0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x99a8,
+        0xa438, 0xd700, 0xa438, 0x5fba, 0xa438, 0xd704, 0xa438, 0x7f6b,
+        0xa438, 0xa81a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8603,
+        0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9d31, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8480,
+        0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, 0xa438, 0x4058,
+        0xa438, 0xa901, 0xa438, 0xd704, 0xa438, 0x2b59, 0xa438, 0x974b,
+        0xa438, 0xd704, 0xa438, 0x604c, 0xa438, 0xfff6, 0xa438, 0xd701,
+        0xa438, 0x6056, 0xa438, 0xa704, 0xa438, 0xa302, 0xa438, 0xd700,
+        0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d, 0xa438, 0x8604,
+        0xa438, 0xf002, 0xa438, 0xa604, 0xa438, 0xd700, 0xa438, 0x60c7,
+        0xa438, 0xa602, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702,
+        0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4055, 0xa438, 0xa940,
+        0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0xcd91,
+        0xa438, 0x1000, 0xa438, 0x9cf4, 0xa438, 0x1000, 0xa438, 0x99a8,
+        0xa438, 0xd704, 0xa438, 0x7f6c, 0xa438, 0xa120, 0xa438, 0xd707,
+        0xa438, 0x4051, 0xa438, 0xa180, 0xa438, 0xa480, 0xa438, 0x8302,
+        0xa438, 0x8606, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702,
+        0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x6056, 0xa438, 0x8704,
+        0xa438, 0x1000, 0xa438, 0x9cf4, 0xa438, 0xce93, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20,
+        0xa438, 0xcd92, 0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707,
+        0xa438, 0x7fb7, 0xa438, 0x81a0, 0xa438, 0x8480, 0xa438, 0x1000,
+        0xa438, 0x9d31, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x1800,
+        0xa438, 0x96fe, 0xa438, 0xd705, 0xa438, 0x40d9, 0xa438, 0xd701,
+        0xa438, 0x6056, 0xa438, 0xa704, 0xa438, 0xa010, 0xa438, 0xf002,
+        0xa438, 0x8010, 0xa438, 0xd701, 0xa438, 0x4095, 0xa438, 0xd705,
+        0xa438, 0x4059, 0xa438, 0xa940, 0xa438, 0xd705, 0xa438, 0x6119,
+        0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c07, 0xa438, 0x0d04, 0xa438, 0x9503, 0xa438, 0xd701,
+        0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0xcd93, 0xa438, 0x1000,
+        0xa438, 0x99a8, 0xa438, 0xd704, 0xa438, 0x464b, 0xa438, 0xd707,
+        0xa438, 0x5f78, 0xa438, 0x1000, 0xa438, 0x9d13, 0xa438, 0x800a,
+        0xa438, 0x81a0, 0xa438, 0xd700, 0xa438, 0x6117, 0xa438, 0x6093,
+        0xa438, 0xd15e, 0xa438, 0xd040, 0xa438, 0xf006, 0xa438, 0xd132,
+        0xa438, 0xd040, 0xa438, 0xf003, 0xa438, 0xd119, 0xa438, 0xd040,
+        0xa438, 0x1000, 0xa438, 0x9d08, 0xa438, 0x1000, 0xa438, 0x99a8,
+        0xa438, 0xd73e, 0xa438, 0x6065, 0xa438, 0xd700, 0xa438, 0x5f3a,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x85f0, 0xa438, 0x9503,
+        0xa438, 0xa008, 0xa438, 0xd707, 0xa438, 0x4052, 0xa438, 0xa002,
+        0xa438, 0x8010, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0xd701,
+        0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, 0xa438, 0xcd94,
+        0xa438, 0x1000, 0xa438, 0x99a8, 0xa438, 0xd707, 0xa438, 0x7fb8,
+        0xa438, 0x8010, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0xd701,
+        0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0xd705, 0xa438, 0x4099,
+        0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0x8b20, 0xa438, 0xd705,
+        0xa438, 0x61f9, 0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02,
+        0xa438, 0x9503, 0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c07, 0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0x800a,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0cf0, 0xa438, 0x05a0,
+        0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x4099, 0xa438, 0x1000,
+        0xa438, 0x9d31, 0xa438, 0xf014, 0xa438, 0xa1a0, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a,
+        0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006,
+        0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002,
+        0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xd705,
+        0xa438, 0x39cc, 0xa438, 0x91f3, 0xa438, 0x1800, 0xa438, 0x96fe,
+        0xa438, 0xd75e, 0xa438, 0x60f4, 0xa438, 0x9b08, 0xa438, 0x9920,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8403, 0xa438, 0x9503,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8208, 0xa438, 0x8404,
+        0xa438, 0x9503, 0xa438, 0xcda1, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4,
+        0xa438, 0xd704, 0xa438, 0x5f37, 0xa438, 0x8b08, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0x9503, 0xa438, 0xd707,
+        0xa438, 0x6536, 0xa438, 0x0c07, 0xa438, 0x0b00, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c03, 0xa438, 0x0200, 0xa438, 0x9503,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce0b,
+        0xa438, 0xf00a, 0xa438, 0xce0b, 0xa438, 0xf008, 0xa438, 0xce0a,
+        0xa438, 0xf006, 0xa438, 0xce0a, 0xa438, 0xf004, 0xa438, 0xce09,
+        0xa438, 0xf002, 0xa438, 0xce09, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704,
+        0xa438, 0x5f37, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8204,
+        0xa438, 0x9503, 0xa438, 0x8b08, 0xa438, 0xd414, 0xa438, 0xd202,
+        0xa438, 0xb308, 0xa438, 0xb302, 0xa438, 0xb301, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xaf02, 0xa438, 0x9503, 0xa438, 0xcdaf,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0,
+        0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd701, 0xa438, 0x5f21,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f02, 0xa438, 0x9503,
+        0xa438, 0xd400, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0xd704,
+        0xa438, 0x67b4, 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xbb08,
+        0xa438, 0x9a10, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008,
+        0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004,
+        0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302,
+        0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704,
+        0xa438, 0x407d, 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44,
+        0xa438, 0x0604, 0xa438, 0xd700, 0xa438, 0x6047, 0xa438, 0xa682,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8230, 0xa438, 0x8410,
+        0xa438, 0x9503, 0xa438, 0xd707, 0xa438, 0x404e, 0xa438, 0x8a10,
+        0xa438, 0xcda9, 0xa438, 0xd110, 0xa438, 0xd040, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000,
+        0xa438, 0x99a4, 0xa438, 0xd700, 0xa438, 0x5f3a, 0xa438, 0x1800,
+        0xa438, 0x91f3, 0xa438, 0xba10, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c97, 0xa438, 0x0086, 0xa438, 0xa060, 0xa438, 0xa101,
+        0xa438, 0xa408, 0xa438, 0xa240, 0xa438, 0x9503, 0xa438, 0xcda2,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa020, 0xa438, 0xa110,
+        0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd17a, 0xa438, 0xd046,
+        0xa438, 0xcda3, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd700,
+        0xa438, 0x5f3a, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c83,
+        0xa438, 0x0282, 0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b02,
+        0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113,
+        0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce11,
+        0xa438, 0xf00a, 0xa438, 0xce11, 0xa438, 0xf008, 0xa438, 0xce10,
+        0xa438, 0xf006, 0xa438, 0xce10, 0xa438, 0xf004, 0xa438, 0xce0f,
+        0xa438, 0xf002, 0xa438, 0xce0f, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0xab08, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704,
+        0xa438, 0x3ebc, 0xa438, 0x98b7, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8284, 0xa438, 0x9503, 0xa438, 0x8b08, 0xa438, 0xcda4,
+        0xa438, 0xd704, 0xa438, 0x6419, 0xa438, 0x3ad1, 0xa438, 0x9969,
+        0xa438, 0xd705, 0xa438, 0x60dc, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa402, 0xa438, 0x9503, 0xa438, 0xf012, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8402, 0xa438, 0x9503, 0xa438, 0xd705,
+        0xa438, 0x60db, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa401,
+        0xa438, 0x9503, 0xa438, 0xf007, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c21, 0xa438, 0x0420, 0xa438, 0x9503, 0xa438, 0x9a10,
+        0xa438, 0xd705, 0xa438, 0x34a4, 0xa438, 0x9886, 0xa438, 0x1800,
+        0xa438, 0x9969, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c10,
+        0xa438, 0x0010, 0xa438, 0x8410, 0xa438, 0xa250, 0xa438, 0x9503,
+        0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704, 0xa438, 0x407d,
+        0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44, 0xa438, 0x0604,
+        0xa438, 0xd700, 0xa438, 0x40e7, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c38, 0xa438, 0x0d28, 0xa438, 0x9503, 0xa438, 0xf007,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0cf8, 0xa438, 0x0d90,
+        0xa438, 0x9503, 0xa438, 0xa682, 0xa438, 0xd1f1, 0xa438, 0xd046,
+        0xa438, 0xcda5, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd704,
+        0xa438, 0x60ef, 0xa438, 0xd706, 0xa438, 0x40b1, 0xa438, 0xd700,
+        0xa438, 0x5eba, 0xa438, 0x1800, 0xa438, 0x9925, 0xa438, 0xd162,
+        0xa438, 0xd055, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0x1000,
+        0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4, 0xa438, 0xd700,
+        0xa438, 0x5f3b, 0xa438, 0xd704, 0xa438, 0x692f, 0xa438, 0xd706,
+        0xa438, 0x48f1, 0xa438, 0x1800, 0xa438, 0x9907, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a,
+        0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006,
+        0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002,
+        0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa00a,
+        0xa438, 0xd179, 0xa438, 0xd047, 0xa438, 0xcda6, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000,
+        0xa438, 0x99a4, 0xa438, 0xd704, 0xa438, 0x60ef, 0xa438, 0xd706,
+        0xa438, 0x40b1, 0xa438, 0xd700, 0xa438, 0x5eba, 0xa438, 0x1800,
+        0xa438, 0x995b, 0xa438, 0xd162, 0xa438, 0xd055, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000,
+        0xa438, 0x99a4, 0xa438, 0xd700, 0xa438, 0x5f3b, 0xa438, 0xd704,
+        0xa438, 0x626f, 0xa438, 0xd706, 0xa438, 0x4231, 0xa438, 0x1800,
+        0xa438, 0x993d, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf,
+        0xa438, 0x0b46, 0xa438, 0x0c07, 0xa438, 0x0c03, 0xa438, 0x9503,
+        0xa438, 0xa1a0, 0xa438, 0xa480, 0xa438, 0xb920, 0xa438, 0x9a10,
+        0xa438, 0xcda7, 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8091, 0xa438, 0x8101, 0xa438, 0x8250,
+        0xa438, 0x0c28, 0xa438, 0x0420, 0xa438, 0xd700, 0xa438, 0x4087,
+        0xa438, 0x0c38, 0xa438, 0x0d18, 0xa438, 0xf003, 0xa438, 0x0cf8,
+        0xa438, 0x0d58, 0xa438, 0x9503, 0xa438, 0x8106, 0xa438, 0xcda8,
+        0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8090, 0xa438, 0x8101, 0xa438, 0x8248, 0xa438, 0x9503,
+        0xa438, 0x1000, 0xa438, 0x9cb7, 0xa438, 0x1000, 0xa438, 0x9c90,
+        0xa438, 0x8106, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0ccf,
+        0xa438, 0x0b03, 0xa438, 0x8c07, 0xa438, 0xd700, 0xa438, 0x4087,
+        0xa438, 0x0c38, 0xa438, 0x0d18, 0xa438, 0xf003, 0xa438, 0x0cf8,
+        0xa438, 0x0d98, 0xa438, 0x8408, 0xa438, 0x9503, 0xa438, 0x9c01,
+        0xa438, 0xa1a0, 0xa438, 0xa302, 0xa438, 0xd707, 0xa438, 0x404e,
+        0xa438, 0x8a10, 0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xcdab,
+        0xa438, 0x1800, 0xa438, 0x923e, 0xa438, 0xd71f, 0xa438, 0x2e71,
+        0xa438, 0x925e, 0xa438, 0x0800, 0xa438, 0xd704, 0xa438, 0x2739,
+        0xa438, 0x96ac, 0xa438, 0x0800, 0xa438, 0xd71f, 0xa438, 0x62ae,
+        0xa438, 0xd706, 0xa438, 0x6400, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd705, 0xa438, 0x61d9, 0xa438, 0xd704, 0xa438, 0x4187,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa601, 0xa438, 0x9503,
+        0xa438, 0x800a, 0xa438, 0x1000, 0xa438, 0x9d31, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0x1800, 0xa438, 0x96ac, 0xa438, 0x0800,
+        0xa438, 0xcd99, 0xa438, 0xa70c, 0xa438, 0x881a, 0xa438, 0x8010,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8683, 0xa438, 0x9503,
+        0xa438, 0xd701, 0xa438, 0x4050, 0xa438, 0xab20, 0xa438, 0x1800,
+        0xa438, 0x925e, 0xa438, 0x8010, 0xa438, 0xa704, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa60c, 0xa438, 0xa603, 0xa438, 0x9503,
+        0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302, 0xa438, 0x8480,
+        0xa438, 0x8604, 0xa438, 0x8602, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8702, 0xa438, 0x9503, 0xa438, 0xd701, 0xa438, 0x4050,
+        0xa438, 0xab20, 0xa438, 0xcd96, 0xa438, 0xa901, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x7ce0, 0xa438, 0xd706,
+        0xa438, 0x2109, 0xa438, 0x9a75, 0xa438, 0xd71f, 0xa438, 0x7aae,
+        0xa438, 0xd704, 0xa438, 0x7ea7, 0xa438, 0xd706, 0xa438, 0x5e62,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, 0xa438, 0x9503,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa610, 0xa438, 0x9503,
+        0xa438, 0xa708, 0xa438, 0x881a, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8cc7, 0xa438, 0x9503,
+        0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xcd5a, 0xa438, 0x9503, 0xa438, 0xf006,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c3f, 0xa438, 0x0d19,
+        0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302,
+        0xa438, 0xa480, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704,
+        0xa438, 0x407d, 0xa438, 0x8604, 0xa438, 0xf002, 0xa438, 0xa604,
+        0xa438, 0xd700, 0xa438, 0x60c7, 0xa438, 0xa602, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa702, 0xa438, 0x9503, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8680, 0xa438, 0x9503, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xc500, 0xa438, 0x9503, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x72ce, 0xa438, 0xd704,
+        0xa438, 0x5f65, 0xa438, 0xd705, 0xa438, 0x4378, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce1d, 0xa438, 0xf00b,
+        0xa438, 0xce1d, 0xa438, 0xf009, 0xa438, 0xce1d, 0xa438, 0xf007,
+        0xa438, 0xce1d, 0xa438, 0xf005, 0xa438, 0xce1c, 0xa438, 0xf003,
+        0xa438, 0xce1c, 0xa438, 0xf001, 0xa438, 0x1000, 0xa438, 0x9bde,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8640, 0xa438, 0x9503,
+        0xa438, 0xf01a, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce05, 0xa438, 0xf00b, 0xa438, 0xce05, 0xa438, 0xf009,
+        0xa438, 0xce04, 0xa438, 0xf007, 0xa438, 0xce04, 0xa438, 0xf005,
+        0xa438, 0xce04, 0xa438, 0xf003, 0xa438, 0xce04, 0xa438, 0xf001,
+        0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa640, 0xa438, 0x9503, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8602, 0xa438, 0x8601, 0xa438, 0x9503, 0xa438, 0x8640,
+        0xa438, 0x1000, 0xa438, 0x9cc6, 0xa438, 0xd703, 0xa438, 0x6131,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xaf80, 0xa438, 0x0c38,
+        0xa438, 0x0d30, 0xa438, 0x9503, 0xa438, 0xd193, 0xa438, 0xd067,
+        0xa438, 0xcd98, 0xa438, 0x1800, 0xa438, 0x91f3, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa608, 0xa438, 0x9503, 0xa438, 0xa708,
+        0xa438, 0xd701, 0xa438, 0x6056, 0xa438, 0x8704, 0xa438, 0x1000,
+        0xa438, 0x9d31, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa81a,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8603, 0xa438, 0x9503,
+        0xa438, 0xcd97, 0xa438, 0x1800, 0xa438, 0x96fe, 0xa438, 0xd706,
+        0xa438, 0x4103, 0xa438, 0xd705, 0xa438, 0x407f, 0xa438, 0x1800,
+        0xa438, 0x9b77, 0xa438, 0xd105, 0xa438, 0xd056, 0xa438, 0xf03f,
+        0xa438, 0xd705, 0xa438, 0x3fa7, 0xa438, 0x9a9a, 0xa438, 0xd705,
+        0xa438, 0x605a, 0xa438, 0xf092, 0xa438, 0x1800, 0xa438, 0x9b1e,
+        0xa438, 0xd704, 0xa438, 0x4146, 0xa438, 0x800a, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d04, 0xa438, 0x9503,
+        0xa438, 0x0c06, 0xa438, 0x0102, 0xa438, 0xf010, 0xa438, 0xa00a,
+        0xa438, 0x8106, 0xa438, 0xd700, 0xa438, 0x60e7, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c07, 0xa438, 0x0d02, 0xa438, 0x9503,
+        0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c07,
+        0xa438, 0x0d01, 0xa438, 0x9503, 0xa438, 0xd705, 0xa438, 0x61d4,
+        0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0x6170, 0xa438, 0x2d71,
+        0xa438, 0x9ac3, 0xa438, 0xd702, 0xa438, 0x60d7, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0cf0, 0xa438, 0x05a0, 0xa438, 0x9503,
+        0xa438, 0xf0fc, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa108,
+        0xa438, 0xa204, 0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4,
+        0xa438, 0xd707, 0xa438, 0x5f35, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa410, 0xa438, 0x9503, 0xa438, 0x800a, 0xa438, 0x81a0,
+        0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0x8642, 0xa438, 0x1000,
+        0xa438, 0x9d44, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8108,
+        0xa438, 0x9503, 0xa438, 0x8c40, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8750, 0xa438, 0x9503, 0xa438, 0xaa10, 0xa438, 0xd706,
+        0xa438, 0x4063, 0xa438, 0x8604, 0xa438, 0xf022, 0xa438, 0xa604,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8220, 0xa438, 0x8301,
+        0xa438, 0x8420, 0xa438, 0x8108, 0xa438, 0x0c03, 0xa438, 0x0201,
+        0xa438, 0x9503, 0xa438, 0x0c07, 0xa438, 0x0b01, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce0e, 0xa438, 0xf00a,
+        0xa438, 0xce0e, 0xa438, 0xf008, 0xa438, 0xce0d, 0xa438, 0xf006,
+        0xa438, 0xce0d, 0xa438, 0xf004, 0xa438, 0xce0c, 0xa438, 0xf002,
+        0xa438, 0xce0c, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xab08,
+        0xa438, 0xcda0, 0xa438, 0xd706, 0xa438, 0x4043, 0xa438, 0xf0b1,
+        0xa438, 0xd705, 0xa438, 0x411e, 0xa438, 0xd162, 0xa438, 0xd045,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa008, 0xa438, 0x9503,
+        0xa438, 0xf0a8, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8008,
+        0xa438, 0x9503, 0xa438, 0xd1c4, 0xa438, 0xd055, 0xa438, 0xf0a1,
+        0xa438, 0xd700, 0xa438, 0x605a, 0xa438, 0xf09e, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8008, 0xa438, 0x8210, 0xa438, 0xa210,
+        0xa438, 0x9503, 0xa438, 0xd162, 0xa438, 0xd055, 0xa438, 0xd704,
+        0xa438, 0x608f, 0xa438, 0xd706, 0xa438, 0x4051, 0xa438, 0xf04a,
+        0xa438, 0xd705, 0xa438, 0x415e, 0xa438, 0xd705, 0xa438, 0x6112,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c06, 0xa438, 0x0002,
+        0xa438, 0xa110, 0xa438, 0xa305, 0xa438, 0x9503, 0xa438, 0xd705,
+        0xa438, 0x40de, 0xa438, 0xd706, 0xa438, 0x6771, 0xa438, 0xd700,
+        0xa438, 0x60bb, 0xa438, 0xf07f, 0xa438, 0xd700, 0xa438, 0x661b,
+        0xa438, 0xf07c, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8090,
+        0xa438, 0x8101, 0xa438, 0x0ccf, 0xa438, 0x0b03, 0xa438, 0x8c07,
+        0xa438, 0xd700, 0xa438, 0x4087, 0xa438, 0x0c38, 0xa438, 0x0d18,
+        0xa438, 0xf003, 0xa438, 0x0cf8, 0xa438, 0x0d58, 0xa438, 0x8270,
+        0xa438, 0x8408, 0xa438, 0x8301, 0xa438, 0x9503, 0xa438, 0x8106,
+        0xa438, 0x9c01, 0xa438, 0xd705, 0xa438, 0x40b5, 0xa438, 0x1000,
+        0xa438, 0x9cb7, 0xa438, 0xd601, 0xa438, 0xd628, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8208, 0xa438, 0x9503, 0xa438, 0xcdac,
+        0xa438, 0xd1c4, 0xa438, 0xd054, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0x1000, 0xa438, 0x99a0, 0xa438, 0x1000, 0xa438, 0x99a4,
+        0xa438, 0xd700, 0xa438, 0x5f3b, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0xa230, 0xa438, 0x9503, 0xa438, 0xf04e, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa404, 0xa438, 0x8230, 0xa438, 0x9503,
+        0xa438, 0xf048, 0xa438, 0xd700, 0xa438, 0x48db, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8204, 0xa438, 0x8410, 0xa438, 0x9503,
+        0xa438, 0x8b08, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008,
+        0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004,
+        0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480,
+        0xa438, 0xd705, 0xa438, 0x611e, 0xa438, 0xa302, 0xa438, 0x1000,
+        0xa438, 0x9cc6, 0xa438, 0xd707, 0xa438, 0x404e, 0xa438, 0x8a10,
+        0xa438, 0xf006, 0xa438, 0x8302, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8301, 0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x6047,
+        0xa438, 0xa682, 0xa438, 0xd700, 0xa438, 0x40a7, 0xa438, 0xd704,
+        0xa438, 0x407d, 0xa438, 0x8644, 0xa438, 0xf003, 0xa438, 0x0c44,
+        0xa438, 0x0604, 0xa438, 0xd701, 0xa438, 0x40d0, 0xa438, 0xac40,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa750, 0xa438, 0x9503,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa220, 0xa438, 0x8210,
+        0xa438, 0xa210, 0xa438, 0xa620, 0xa438, 0x9503, 0xa438, 0xcdaa,
+        0xa438, 0x0800, 0xa438, 0xd202, 0xa438, 0xb309, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xd400,
+        0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd71f,
+        0xa438, 0x6261, 0xa438, 0xcdff, 0xa438, 0xd705, 0xa438, 0x41d5,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, 0xa438, 0x0e12,
+        0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x5ff3, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x8e1f, 0xa438, 0x9503, 0xa438, 0xd70c,
+        0xa438, 0x7f73, 0xa438, 0x1800, 0xa438, 0x001c, 0xa438, 0x0800,
+        0xa438, 0xab10, 0xa438, 0xd71f, 0xa438, 0x210c, 0xa438, 0x001c,
+        0xa438, 0xd701, 0xa438, 0x5f98, 0xa438, 0x8b10, 0xa438, 0x0800,
+        0xa438, 0xd71f, 0xa438, 0x210c, 0xa438, 0x001c, 0xa438, 0xd701,
+        0xa438, 0x5f99, 0xa438, 0x0800, 0xa438, 0xa110, 0xa438, 0x1000,
+        0xa438, 0x9be6, 0xa438, 0x8110, 0xa438, 0xa140, 0xa438, 0x1000,
+        0xa438, 0x9be6, 0xa438, 0x8140, 0xa438, 0xa004, 0xa438, 0x1000,
+        0xa438, 0x9be6, 0xa438, 0x8004, 0xa438, 0xa001, 0xa438, 0x1000,
+        0xa438, 0x9be6, 0xa438, 0x8001, 0xa438, 0xa020, 0xa438, 0x1000,
+        0xa438, 0x9be6, 0xa438, 0x8020, 0xa438, 0xa101, 0xa438, 0x0800,
+        0xa438, 0xd202, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa1, 0xa438, 0xb302,
+        0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd204, 0xa438, 0x0c09,
+        0xa438, 0x1301, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701,
+        0xa438, 0x5fa2, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800,
+        0xa438, 0xd208, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fa3, 0xa438, 0xb302,
+        0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd210, 0xa438, 0x0c09,
+        0xa438, 0x1301, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701,
+        0xa438, 0x5fa4, 0xa438, 0xb302, 0xa438, 0xd200, 0xa438, 0x0800,
+        0xa438, 0xac01, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd707,
+        0xa438, 0x5fad, 0xa438, 0x0800, 0xa438, 0xac04, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd707, 0xa438, 0x5fab, 0xa438, 0x0800,
+        0xa438, 0xd71f, 0xa438, 0x4354, 0xa438, 0x8810, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5fa7, 0xa438, 0xb920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4,
+        0xa438, 0x9920, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x6085, 0xa438, 0xd71f, 0xa438, 0x7e34, 0xa438, 0xfffa,
+        0xa438, 0xb820, 0xa438, 0xa810, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x7fa5, 0xa438, 0x9820, 0xa438, 0x0800,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa308, 0xa438, 0x9503,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701, 0xa438, 0x5fb3,
+        0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8308,
+        0xa438, 0x9503, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd701,
+        0xa438, 0x7fb3, 0xa438, 0x0800, 0xa438, 0xc000, 0xa438, 0xc100,
+        0xa438, 0xc200, 0xa438, 0xc300, 0xa438, 0xc400, 0xa438, 0xc500,
+        0xa438, 0xc600, 0xa438, 0xc700, 0xa438, 0xc828, 0xa438, 0xc904,
+        0xa438, 0xca00, 0xa438, 0xcb00, 0xa438, 0xcc00, 0xa438, 0xce00,
+        0xa438, 0xcf00, 0xa438, 0xd000, 0xa438, 0xd100, 0xa438, 0xd200,
+        0xa438, 0xd300, 0xa438, 0xd400, 0xa438, 0xd500, 0xa438, 0xd700,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xc000, 0xa438, 0xc100,
+        0xa438, 0xc200, 0xa438, 0xc300, 0xa438, 0xc400, 0xa438, 0xc500,
+        0xa438, 0xc600, 0xa438, 0xa63c, 0xa438, 0xc700, 0xa438, 0xc800,
+        0xa438, 0xc900, 0xa438, 0xca00, 0xa438, 0xcb00, 0xa438, 0xcc00,
+        0xa438, 0xcd00, 0xa438, 0xce00, 0xa438, 0xcf00, 0xa438, 0x9503,
+        0xa438, 0xcd00, 0xa438, 0x0800, 0xa438, 0xd601, 0xa438, 0xd608,
+        0xa438, 0xd610, 0xa438, 0xd618, 0xa438, 0xd620, 0xa438, 0xd628,
+        0xa438, 0xd630, 0xa438, 0xd638, 0xa438, 0x0800, 0xa438, 0x8b0f,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8097, 0xa438, 0x8109,
+        0xa438, 0x82ff, 0xa438, 0x843f, 0xa438, 0x9503, 0xa438, 0x9920,
+        0xa438, 0x9b08, 0xa438, 0x9a10, 0xa438, 0xd705, 0xa438, 0x4235,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f, 0xa438, 0x0e12,
+        0xa438, 0x8440, 0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x5ff3,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8e1f, 0xa438, 0x9503,
+        0xa438, 0xd70c, 0xa438, 0x7f73, 0xa438, 0xd601, 0xa438, 0xd628,
+        0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c1f,
+        0xa438, 0x0e12, 0xa438, 0x8440, 0xa438, 0x9503, 0xa438, 0xd70c,
+        0xa438, 0x5ff3, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8e1f,
+        0xa438, 0x9503, 0xa438, 0xd70c, 0xa438, 0x7f73, 0xa438, 0x0800,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x9ccf, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0350, 0xa438, 0x9503,
+        0xa438, 0xf00e, 0xa438, 0xd702, 0xa438, 0x40f9, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0340, 0xa438, 0x9503,
+        0xa438, 0xf006, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c70,
+        0xa438, 0x0340, 0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0xd700,
+        0xa438, 0x37c9, 0xa438, 0x9ce6, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c70, 0xa438, 0x0370, 0xa438, 0x9503, 0xa438, 0xf00e,
+        0xa438, 0xd702, 0xa438, 0x40f9, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x0c70, 0xa438, 0x0350, 0xa438, 0x9503, 0xa438, 0xf006,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x0c70, 0xa438, 0x0350,
+        0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0x8910, 0xa438, 0xd704,
+        0xa438, 0x61a8, 0xa438, 0xd702, 0xa438, 0x60d6, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xaf40, 0xa438, 0x9503, 0xa438, 0xf00a,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40, 0xa438, 0x9503,
+        0xa438, 0xf005, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8f40,
+        0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0x8910, 0xa438, 0xd705,
+        0xa438, 0x4059, 0xa438, 0xa910, 0xa438, 0xd704, 0xa438, 0x6068,
+        0xa438, 0x0000, 0xa438, 0xf002, 0xa438, 0x0000, 0xa438, 0x8910,
+        0xa438, 0x0800, 0xa438, 0xd703, 0xa438, 0x6080, 0xa438, 0x6121,
+        0xa438, 0x61c2, 0xa438, 0x6263, 0xa438, 0xd707, 0xa438, 0x4070,
+        0xa438, 0xce98, 0xa438, 0xf015, 0xa438, 0xce94, 0xa438, 0xf013,
+        0xa438, 0xd707, 0xa438, 0x4070, 0xa438, 0xce99, 0xa438, 0xf00f,
+        0xa438, 0xce95, 0xa438, 0xf00d, 0xa438, 0xd707, 0xa438, 0x4070,
+        0xa438, 0xce9a, 0xa438, 0xf009, 0xa438, 0xce96, 0xa438, 0xf007,
+        0xa438, 0xd707, 0xa438, 0x4070, 0xa438, 0xce9b, 0xa438, 0xf003,
+        0xa438, 0xce97, 0xa438, 0xf001, 0xa438, 0x0800, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce00, 0xa438, 0xf00a,
+        0xa438, 0xce00, 0xa438, 0xf008, 0xa438, 0xce00, 0xa438, 0xf006,
+        0xa438, 0xce00, 0xa438, 0xf004, 0xa438, 0xce00, 0xa438, 0xf002,
+        0xa438, 0xce00, 0xa438, 0x0800, 0xa438, 0x0c03, 0xa438, 0x1502,
+        0xa438, 0x8702, 0xa438, 0x9503, 0xa438, 0xd101, 0xa438, 0xd040,
+        0xa438, 0xd700, 0xa438, 0x5ffa, 0xa438, 0x8680, 0xa438, 0xd700,
+        0xa438, 0x60a7, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702,
+        0xa438, 0x9503, 0xa438, 0x0800, 0xa438, 0xcdfe, 0xa438, 0xa708,
+        0xa438, 0xa2fc, 0xa438, 0xba20, 0xa438, 0xa980, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4,
+        0xa438, 0x9920, 0xa438, 0x0ca0, 0xa438, 0x0480, 0xa438, 0xd706,
+        0xa438, 0x5fe7, 0xa438, 0x84a0, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4, 0xa438, 0x9920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fa5,
+        0xa438, 0xb820, 0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f,
+        0xa438, 0x7f85, 0xa438, 0x9820, 0xa438, 0xa810, 0xa438, 0xd700,
+        0xa438, 0x60cf, 0xa438, 0x60f1, 0xa438, 0x6113, 0xa438, 0x6135,
+        0xa438, 0x6157, 0xa438, 0xf00b, 0xa438, 0xce08, 0xa438, 0xf00a,
+        0xa438, 0xce08, 0xa438, 0xf008, 0xa438, 0xce08, 0xa438, 0xf006,
+        0xa438, 0xce08, 0xa438, 0xf004, 0xa438, 0xce08, 0xa438, 0xf002,
+        0xa438, 0xce08, 0xa438, 0x1000, 0xa438, 0x9bde, 0xa438, 0xa120,
+        0xa438, 0xa4a0, 0xa438, 0xa980, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd706, 0xa438, 0x5faf, 0xa438, 0x1800, 0xa438, 0x9de8,
+        0xa438, 0xcdfd, 0xa438, 0xd700, 0xa438, 0x60cf, 0xa438, 0x60f1,
+        0xa438, 0x6113, 0xa438, 0x6135, 0xa438, 0x6157, 0xa438, 0xf00b,
+        0xa438, 0xce08, 0xa438, 0xf00a, 0xa438, 0xce08, 0xa438, 0xf008,
+        0xa438, 0xce08, 0xa438, 0xf006, 0xa438, 0xce08, 0xa438, 0xf004,
+        0xa438, 0xce08, 0xa438, 0xf002, 0xa438, 0xce08, 0xa438, 0x1000,
+        0xa438, 0x9bde, 0xa438, 0xa980, 0xa438, 0xa810, 0xa438, 0xa00a,
+        0xa438, 0xa1a0, 0xa438, 0xa312, 0xa438, 0xa4a0, 0xa438, 0xa604,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa340, 0xa438, 0x9503,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd706, 0xa438, 0x5faf,
+        0xa438, 0x0c30, 0xa438, 0x0320, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd704, 0xa438, 0x5fa4, 0xa438, 0x800a, 0xa438, 0x81a0,
+        0xa438, 0x8312, 0xa438, 0x84a0, 0xa438, 0x8604, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x5fb4, 0xa438, 0xb920,
+        0xa438, 0x1000, 0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb4,
+        0xa438, 0x9920, 0xa438, 0x8340, 0xa438, 0xa801, 0xa438, 0xa980,
+        0xa438, 0x8240, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa302,
+        0xa438, 0xa4a0, 0xa438, 0xa604, 0xa438, 0xd1c8, 0xa438, 0xd045,
+        0xa438, 0xd700, 0xa438, 0x5ffa, 0xa438, 0xd706, 0xa438, 0x5faf,
+        0xa438, 0xd40a, 0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0xd416,
+        0xa438, 0x1000, 0xa438, 0x9bbf, 0xa438, 0x800a, 0xa438, 0x81a0,
+        0xa438, 0x8302, 0xa438, 0x8480, 0xa438, 0xa604, 0xa438, 0xa801,
+        0xa438, 0x8818, 0xa438, 0xbb10, 0xa438, 0x1000, 0xa438, 0x9bc9,
+        0xa438, 0xd71f, 0xa438, 0x5fb5, 0xa438, 0x9b10, 0xa438, 0x1000,
+        0xa438, 0x9bc9, 0xa438, 0xd71f, 0xa438, 0x7fb5, 0xa438, 0x1800,
+        0xa438, 0x915e, 0xa438, 0x1000, 0xa438, 0x1c45, 0xa438, 0x1000,
+        0xa438, 0x9c64, 0xa438, 0xac10, 0xa438, 0x1800, 0xa438, 0x0021,
+        0xa436, 0xA10E, 0xa438, 0xffff, 0xa436, 0xA10C, 0xa438, 0xffff,
+        0xa436, 0xA10A, 0xa438, 0xffff, 0xa436, 0xA108, 0xa438, 0xffff,
+        0xa436, 0xA106, 0xa438, 0xffff, 0xa436, 0xA104, 0xa438, 0xffff,
+        0xa436, 0xA102, 0xa438, 0x001c, 0xa436, 0xA100, 0xa438, 0x0073,
+        0xa436, 0xA110, 0xa438, 0x0003, 0xa436, 0xA016, 0xa438, 0x0020,
+        0xa436, 0xA012, 0xa438, 0x1ff8, 0xa436, 0xA014, 0xa438, 0xb904,
+        0xa438, 0xd18a, 0xa438, 0xd17a, 0xa438, 0x9c10, 0xa438, 0x0000,
+        0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA164,
+        0xa438, 0x1CAC, 0xa436, 0xA166, 0xa438, 0x0666, 0xa436, 0xA168,
+        0xa438, 0x0669, 0xa436, 0xA16A, 0xa438, 0x001f, 0xa436, 0xA16C,
+        0xa438, 0x3fff, 0xa436, 0xA16E, 0xa438, 0x3fff, 0xa436, 0xA170,
+        0xa438, 0x3fff, 0xa436, 0xA172, 0xa438, 0x3fff, 0xa436, 0xA162,
+        0xa438, 0x000f, 0xa436, 0xb87c, 0xa438, 0x8a23, 0xa436, 0xb87e,
+        0xa438, 0xaf8a, 0xa438, 0x3baf, 0xa438, 0x8a4c, 0xa438, 0xaf8a,
+        0xa438, 0x5caf, 0xa438, 0x8a62, 0xa438, 0xaf8a, 0xa438, 0x68af,
+        0xa438, 0x8a9e, 0xa438, 0xaf8b, 0xa438, 0x4daf, 0xa438, 0x8ba8,
+        0xa438, 0xe080, 0xa438, 0x15ad, 0xa438, 0x2505, 0xa438, 0xd101,
+        0xa438, 0xaf58, 0xa438, 0xcc02, 0xa438, 0x1e44, 0xa438, 0xaf59,
+        0xa438, 0x05ee, 0xa438, 0x88af, 0xa438, 0x0102, 0xa438, 0x7eb6,
+        0xa438, 0xad2a, 0xa438, 0x03af, 0xa438, 0x5b41, 0xa438, 0xaf5b,
+        0xa438, 0x4a02, 0xa438, 0x8c4c, 0xa438, 0xaf69, 0xa438, 0x4f02,
+        0xa438, 0x8d51, 0xa438, 0xaf65, 0xa438, 0xaf02, 0xa438, 0x7b73,
+        0xa438, 0xbf7a, 0xa438, 0xa302, 0xa438, 0x7b73, 0xa438, 0xbf8e,
+        0xa438, 0x5b02, 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x5e02,
+        0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6102, 0xa438, 0x7b73,
+        0xa438, 0xbf7a, 0xa438, 0xa602, 0xa438, 0x7b73, 0xa438, 0xbf8e,
+        0xa438, 0x6402, 0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6702,
+        0xa438, 0x7b73, 0xa438, 0xbf8e, 0xa438, 0x6a02, 0xa438, 0x7b73,
+        0xa438, 0xaf64, 0xa438, 0xcfe0, 0xa438, 0x898a, 0xa438, 0xe189,
+        0xa438, 0x8bc4, 0xa438, 0xef67, 0xa438, 0xe089, 0xa438, 0x8ce1,
+        0xa438, 0x898e, 0xa438, 0x1b01, 0xa438, 0xef20, 0xa438, 0xad27,
+        0xa438, 0x04c7, 0xa438, 0x78ff, 0xa438, 0x109e, 0xa438, 0x050d,
+        0xa438, 0x8180, 0xa438, 0xaef9, 0xa438, 0xac37, 0xa438, 0x09cf,
+        0xa438, 0x027a, 0xa438, 0xcdac, 0xa438, 0x5030, 0xa438, 0xae13,
+        0xa438, 0xe089, 0xa438, 0x8ae1, 0xa438, 0x898b, 0xa438, 0xef74,
+        0xa438, 0xef46, 0xa438, 0xce02, 0xa438, 0x7acd, 0xa438, 0xef64,
+        0xa438, 0xac50, 0xa438, 0x1be0, 0xa438, 0x8991, 0xa438, 0x3802,
+        0xa438, 0xac27, 0xa438, 0x13ef, 0xa438, 0x46e4, 0xa438, 0x898a,
+        0xa438, 0xe589, 0xa438, 0x8b3b, 0xa438, 0x02e7, 0xa438, 0x898d,
+        0xa438, 0xe289, 0xa438, 0x8ee6, 0xa438, 0x898c, 0xa438, 0x1f44,
+        0xa438, 0xe189, 0xa438, 0x68e2, 0xa438, 0x898e, 0xa438, 0x3a09,
+        0xa438, 0xef32, 0xa438, 0xad37, 0xa438, 0x06c6, 0xa438, 0xef64,
+        0xa438, 0x7aff, 0xa438, 0x129e, 0xa438, 0x050d, 0xa438, 0x6182,
+        0xa438, 0xaef9, 0xa438, 0xac3f, 0xa438, 0x04ef, 0xa438, 0x74ae,
+        0xa438, 0x03ef, 0xa438, 0x76ce, 0xa438, 0x027a, 0xa438, 0xcdad,
+        0xa438, 0x501e, 0xa438, 0xe089, 0xa438, 0x9138, 0xa438, 0x02ac,
+        0xa438, 0x2709, 0xa438, 0xe089, 0xa438, 0x8f10, 0xa438, 0xe489,
+        0xa438, 0x8fae, 0xa438, 0x0de0, 0xa438, 0x8991, 0xa438, 0x2801,
+        0xa438, 0xe189, 0xa438, 0x921e, 0xa438, 0x10e5, 0xa438, 0x8992,
+        0xa438, 0xe089, 0xa438, 0x9110, 0xa438, 0xe489, 0xa438, 0x91af,
+        0xa438, 0x68c5, 0xa438, 0xcffb, 0xa438, 0xee89, 0xa438, 0x6700,
+        0xa438, 0xe389, 0xa438, 0x671f, 0xa438, 0x77d8, 0xa438, 0xa000,
+        0xa438, 0x1213, 0xa438, 0xe789, 0xa438, 0x67d2, 0xa438, 0x031a,
+        0xa438, 0x92e1, 0xa438, 0x8966, 0xa438, 0x1b13, 0xa438, 0x9fed,
+        0xa438, 0xaf8b, 0xa438, 0x87a0, 0xa438, 0x040b, 0xa438, 0xbcbf,
+        0xa438, 0x7ab2, 0xa438, 0x027d, 0xa438, 0x5abd, 0xa438, 0xac28,
+        0xa438, 0xe0d8, 0xa438, 0x1f11, 0xa438, 0x0267, 0xa438, 0xc3d7,
+        0xa438, 0x0001, 0xa438, 0xaed5, 0xa438, 0xad50, 0xa438, 0x02ae,
+        0xa438, 0x12e0, 0xa438, 0x8987, 0xa438, 0xad20, 0xa438, 0x06ee,
+        0xa438, 0x8987, 0xa438, 0x00ae, 0xa438, 0x06ee, 0xa438, 0x8989,
+        0xa438, 0x00ae, 0xa438, 0x06e0, 0xa438, 0x895c, 0xa438, 0xe489,
+        0xa438, 0x89ff, 0xa438, 0xaf6b, 0xa438, 0x3cd4, 0xa438, 0x000f,
+        0xa438, 0xbf79, 0xa438, 0x3202, 0xa438, 0x7b9f, 0xa438, 0xd400,
+        0xa438, 0x04bf, 0xa438, 0x5075, 0xa438, 0xd700, 0xa438, 0x0802,
+        0xa438, 0x7da0, 0xa438, 0xbf8e, 0xa438, 0x6d02, 0xa438, 0x7b73,
+        0xa438, 0xd402, 0xa438, 0x00bf, 0xa438, 0x8e70, 0xa438, 0x027b,
+        0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, 0xa438, 0x7302,
+        0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, 0xa438, 0x7b73,
+        0xa438, 0xd41b, 0xa438, 0x3ebf, 0xa438, 0x8e6d, 0xa438, 0x027b,
+        0xa438, 0x9fd4, 0xa438, 0x00dc, 0xa438, 0xbf8e, 0xa438, 0x7002,
+        0xa438, 0x7b9f, 0xa438, 0xd400, 0xa438, 0x0fbf, 0xa438, 0x8e73,
+        0xa438, 0x027b, 0xa438, 0x9fbf, 0xa438, 0x8e73, 0xa438, 0x027b,
+        0xa438, 0x73d4, 0xa438, 0x1bbe, 0xa438, 0xbf8e, 0xa438, 0x6d02,
+        0xa438, 0x7b9f, 0xa438, 0xd400, 0xa438, 0xdcbf, 0xa438, 0x8e70,
+        0xa438, 0x027b, 0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e,
+        0xa438, 0x7302, 0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302,
+        0xa438, 0x7b73, 0xa438, 0xd700, 0xa438, 0x86b7, 0xa438, 0xfed4,
+        0xa438, 0x003e, 0xa438, 0xbf8e, 0xa438, 0x6d02, 0xa438, 0x7b9f,
+        0xa438, 0xd400, 0xa438, 0xdcbf, 0xa438, 0x8e70, 0xa438, 0x027b,
+        0xa438, 0x9fd4, 0xa438, 0x000f, 0xa438, 0xbf8e, 0xa438, 0x7302,
+        0xa438, 0x7b9f, 0xa438, 0xbf8e, 0xa438, 0x7302, 0xa438, 0x7b73,
+        0xa438, 0xbf79, 0xa438, 0x3202, 0xa438, 0x7b73, 0xa438, 0xaf51,
+        0xa438, 0x4ff8, 0xa438, 0xf9fa, 0xa438, 0xcefb, 0xa438, 0xef79,
+        0xa438, 0xfbe0, 0xa438, 0x8967, 0xa438, 0xef10, 0xa438, 0x4803,
+        0xa438, 0xbf89, 0xa438, 0x721a, 0xa438, 0x90ef, 0xa438, 0x79bf,
+        0xa438, 0x8982, 0xa438, 0x1a91, 0xa438, 0xec01, 0xa438, 0x0702,
+        0xa438, 0x7eb6, 0xa438, 0xad29, 0xa438, 0x10e0, 0xa438, 0x898f,
+        0xa438, 0x3804, 0xa438, 0x9f06, 0xa438, 0xef97, 0xa438, 0x1919,
+        0xa438, 0xec02, 0xa438, 0xaf8d, 0xa438, 0x48ef, 0xa438, 0x97ec,
+        0xa438, 0x04e0, 0xa438, 0x898f, 0xa438, 0x380c, 0xa438, 0x9e3d,
+        0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x03ac, 0xa438, 0x273d,
+        0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x039e, 0xa438, 0x3f07,
+        0xa438, 0xec03, 0xa438, 0x07e0, 0xa438, 0x898d, 0xa438, 0x3804,
+        0xa438, 0x9e53, 0xa438, 0x19ec, 0xa438, 0x02e0, 0xa438, 0x898d,
+        0xa438, 0x3807, 0xa438, 0xac27, 0xa438, 0x6089, 0xa438, 0xec03,
+        0xa438, 0xe089, 0xa438, 0x8d38, 0xa438, 0x079e, 0xa438, 0x5807,
+        0xa438, 0xec04, 0xa438, 0x07e0, 0xa438, 0x898d, 0xa438, 0x3808,
+        0xa438, 0x9e61, 0xa438, 0xaf8d, 0xa438, 0x4819, 0xa438, 0xec03,
+        0xa438, 0x19ec, 0xa438, 0x02ae, 0xa438, 0x7819, 0xa438, 0xec03,
+        0xa438, 0x07ec, 0xa438, 0x0207, 0xa438, 0xae6f, 0xa438, 0xe089,
+        0xa438, 0x5da0, 0xa438, 0x010e, 0xa438, 0xe089, 0xa438, 0x92a0,
+        0xa438, 0x0308, 0xa438, 0x1f55, 0xa438, 0x07ec, 0xa438, 0x0307,
+        0xa438, 0xae55, 0xa438, 0x07ec, 0xa438, 0x0207, 0xa438, 0xd103,
+        0xa438, 0x026b, 0xa438, 0x74ae, 0xa438, 0x50e0, 0xa438, 0x895d,
+        0xa438, 0xa001, 0xa438, 0x0be0, 0xa438, 0x8992, 0xa438, 0xad21,
+        0xa438, 0x05d5, 0xa438, 0x010f, 0xa438, 0xae39, 0xa438, 0xd102,
+        0xa438, 0x026b, 0xa438, 0x74ae, 0xa438, 0x38ae, 0xa438, 0x36e0,
+        0xa438, 0x895d, 0xa438, 0xa001, 0xa438, 0x1fe0, 0xa438, 0x8992,
+        0xa438, 0xa003, 0xa438, 0x191f, 0xa438, 0x5507, 0xa438, 0xec04,
+        0xa438, 0x07ae, 0xa438, 0x17e0, 0xa438, 0x895d, 0xa438, 0xa001,
+        0xa438, 0x0be0, 0xa438, 0x8992, 0xa438, 0xad21, 0xa438, 0x05d5,
+        0xa438, 0x010f, 0xa438, 0xae06, 0xa438, 0xef97, 0xa438, 0xec00,
+        0xa438, 0xae0b, 0xa438, 0x0266, 0xa438, 0xa7ae, 0xa438, 0x0602,
+        0xa438, 0x66a7, 0xa438, 0x19ec, 0xa438, 0x02ff, 0xa438, 0xef97,
+        0xa438, 0xffc6, 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xf8f9,
+        0xa438, 0xfaef, 0xa438, 0x69fb, 0xa438, 0xcfd3, 0xa438, 0x00e7,
+        0xa438, 0x8967, 0xa438, 0xbf89, 0xa438, 0x6e1a, 0xa438, 0x93bc,
+        0xa438, 0xdaad, 0xa438, 0x302d, 0xa438, 0xbf76, 0xa438, 0x111f,
+        0xa438, 0x4402, 0xa438, 0x7d66, 0xa438, 0x0266, 0xa438, 0xfcbf,
+        0xa438, 0x8982, 0xa438, 0xe189, 0xa438, 0x67a1, 0xa438, 0x0001,
+        0xa438, 0x191f, 0xa438, 0x44d9, 0xa438, 0x81bf, 0xa438, 0x7aa0,
+        0xa438, 0x027d, 0xa438, 0x66bf, 0xa438, 0x7611, 0xa438, 0xd400,
+        0xa438, 0x0102, 0xa438, 0x7d66, 0xa438, 0xbdae, 0xa438, 0x02ae,
+        0xa438, 0xc9da, 0xa438, 0xad31, 0xa438, 0x31bf, 0xa438, 0x761d,
+        0xa438, 0x1f44, 0xa438, 0x027d, 0xa438, 0x6602, 0xa438, 0x66fc,
+        0xa438, 0xbf89, 0xa438, 0x8219, 0xa438, 0xe189, 0xa438, 0x67a1,
+        0xa438, 0x0202, 0xa438, 0xae06, 0xa438, 0xa103, 0xa438, 0x02ae,
+        0xa438, 0x0119, 0xa438, 0x1f44, 0xa438, 0xd981, 0xa438, 0xbf7a,
+        0xa438, 0xa302, 0xa438, 0x7d66, 0xa438, 0xbf76, 0xa438, 0x1dd4,
+        0xa438, 0x0001, 0xa438, 0x027d, 0xa438, 0x66bd, 0xa438, 0xdaad,
+        0xa438, 0x322c, 0xa438, 0xbf76, 0xa438, 0x291f, 0xa438, 0x4402,
+        0xa438, 0x7d66, 0xa438, 0x0266, 0xa438, 0xfcbf, 0xa438, 0x8982,
+        0xa438, 0x1919, 0xa438, 0xe189, 0xa438, 0x67a1, 0xa438, 0x0302,
+        0xa438, 0xae01, 0xa438, 0x191f, 0xa438, 0x44d9, 0xa438, 0x81bf,
+        0xa438, 0x7aa6, 0xa438, 0x027d, 0xa438, 0x66bf, 0xa438, 0x7629,
+        0xa438, 0xd400, 0xa438, 0x0102, 0xa438, 0x7d66, 0xa438, 0x028e,
+        0xa438, 0x0f13, 0xa438, 0xe789, 0xa438, 0x67e0, 0xa438, 0x8966,
+        0xa438, 0x1b03, 0xa438, 0x9f8b, 0xa438, 0xc7ff, 0xa438, 0xef96,
+        0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xf8f9, 0xa438, 0xfaef,
+        0xa438, 0x69fb, 0xa438, 0xcfe3, 0xa438, 0x8967, 0xa438, 0xbf89,
+        0xa438, 0x7e1a, 0xa438, 0x93bc, 0xa438, 0xda6a, 0xa438, 0x009e,
+        0xa438, 0x2fbf, 0xa438, 0x7653, 0xa438, 0xe389, 0xa438, 0x671f,
+        0xa438, 0x4402, 0xa438, 0x7d66, 0xa438, 0x0267, 0xa438, 0x1abd,
+        0xa438, 0x1f00, 0xa438, 0xd959, 0xa438, 0x010c, 0xa438, 0x1269,
+        0xa438, 0x08a3, 0xa438, 0x0104, 0xa438, 0x6902, 0xa438, 0xae02,
+        0xa438, 0x6901, 0xa438, 0xbf7a, 0xa438, 0xb202, 0xa438, 0x7d66,
+        0xa438, 0xd101, 0xa438, 0xbf76, 0xa438, 0x5302, 0xa438, 0x7d66,
+        0xa438, 0xc7ff, 0xa438, 0xef96, 0xa438, 0xfefd, 0xa438, 0xfc04,
+        0xa438, 0x32b1, 0xa438, 0x4a32, 0xa438, 0xb24a, 0xa438, 0x32b3,
+        0xa438, 0x4a10, 0xa438, 0xb14a, 0xa438, 0x10b2, 0xa438, 0x4a10,
+        0xa438, 0xb34a, 0xa438, 0xf0bd, 0xa438, 0x94f0, 0xa438, 0xbd92,
+        0xa438, 0x74bd, 0xa438, 0x9600, 0xa436, 0xb85e, 0xa438, 0x58ca,
+        0xa436, 0xb860, 0xa438, 0x5b3d, 0xa436, 0xb862, 0xa438, 0x694c,
+        0xa436, 0xb864, 0xa438, 0x65ac, 0xa436, 0xb886, 0xa438, 0x64cc,
+        0xa436, 0xb888, 0xa438, 0x683e, 0xa436, 0xb88a, 0xa438, 0x6ae6,
+        0xa436, 0xb88c, 0xa438, 0x5119, 0xa436, 0xb838, 0xa438, 0x00ff,
+        0xb820, 0x0010, 0xa464, 0x0001, 0xa436, 0x8474, 0xa438, 0x0000,
+        0xa436, 0x8608, 0xa438, 0xaf86, 0xa438, 0x20af, 0xa438, 0x865e,
+        0xa438, 0xaf86, 0xa438, 0xacaf, 0xa438, 0x86b7, 0xa438, 0xaf88,
+        0xa438, 0xbdaf, 0xa438, 0x88d4, 0xa438, 0xaf88, 0xa438, 0xe3af,
+        0xa438, 0x893f, 0xa438, 0xbf8a, 0xa438, 0x1102, 0xa438, 0x7589,
+        0xa438, 0xbf8a, 0xa438, 0x1702, 0xa438, 0x756a, 0xa438, 0xbf8a,
+        0xa438, 0x1d02, 0xa438, 0x756a, 0xa438, 0xbf8a, 0xa438, 0x1402,
+        0xa438, 0x7589, 0xa438, 0xbf8a, 0xa438, 0x1a02, 0xa438, 0x756a,
+        0xa438, 0xbf8a, 0xa438, 0x2002, 0xa438, 0x756a, 0xa438, 0xa200,
+        0xa438, 0x08bf, 0xa438, 0x6e23, 0xa438, 0x0275, 0xa438, 0x3eae,
+        0xa438, 0x0ca2, 0xa438, 0x0609, 0xa438, 0xe08f, 0xa438, 0x7dad,
+        0xa438, 0x2003, 0xa438, 0xaf0f, 0xa438, 0xc9af, 0xa438, 0x0fd5,
+        0xa438, 0xe084, 0xa438, 0x68a0, 0xa438, 0x0005, 0xa438, 0x0264,
+        0xa438, 0xe6ae, 0xa438, 0x0ba0, 0xa438, 0x0105, 0xa438, 0x0265,
+        0xa438, 0xb4ae, 0xa438, 0x0302, 0xa438, 0x8677, 0xa438, 0xaf64,
+        0xa438, 0xe1f8, 0xa438, 0xf9e0, 0xa438, 0x8469, 0xa438, 0xe184,
+        0xa438, 0x6a14, 0xa438, 0xe484, 0xa438, 0x69e5, 0xa438, 0x846a,
+        0xa438, 0xe283, 0xa438, 0xade3, 0xa438, 0x83ae, 0xa438, 0x1b45,
+        0xa438, 0x9f11, 0xa438, 0xee84, 0xa438, 0x6900, 0xa438, 0xee84,
+        0xa438, 0x6a00, 0xa438, 0xee84, 0xa438, 0x6800, 0xa438, 0x0264,
+        0xa438, 0xe6ae, 0xa438, 0x08e0, 0xa438, 0x8043, 0xa438, 0xf626,
+        0xa438, 0xe480, 0xa438, 0x43fd, 0xa438, 0xfc04, 0xa438, 0xe080,
+        0xa438, 0x43f6, 0xa438, 0x26e4, 0xa438, 0x8043, 0xa438, 0xaf65,
+        0xa438, 0xa8ee, 0xa438, 0x8468, 0xa438, 0x0202, 0xa438, 0x86c9,
+        0xa438, 0xe080, 0xa438, 0x43f6, 0xa438, 0x26e4, 0xa438, 0x8043,
+        0xa438, 0xaf66, 0xa438, 0x0bf8, 0xa438, 0xf9ef, 0xa438, 0x59f9,
+        0xa438, 0xfafb, 0xa438, 0xe18f, 0xa438, 0x7ea1, 0xa438, 0x0003,
+        0xa438, 0xaf88, 0xa438, 0x64bf, 0xa438, 0x6c43, 0xa438, 0x0275,
+        0xa438, 0x89d0, 0xa438, 0x003c, 0xa438, 0x008a, 0xa438, 0xad27,
+        0xa438, 0x03af, 0xa438, 0x86fe, 0xa438, 0xee8f, 0xa438, 0x7e00,
+        0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd401, 0xa438, 0x28d6,
+        0xa438, 0x0010, 0xa438, 0x0288, 0xa438, 0x6caf, 0xa438, 0x8864,
+        0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x8f02,
+        0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600,
+        0xa438, 0xc002, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01,
+        0xa438, 0xd600, 0xa438, 0x8f02, 0xa438, 0x889b, 0xa438, 0xee84,
+        0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc002, 0xa438, 0x886c,
+        0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x8f02,
+        0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600,
+        0xa438, 0xc002, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03,
+        0xa438, 0xd600, 0xa438, 0x8f02, 0xa438, 0x889b, 0xa438, 0xee84,
+        0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc002, 0xa438, 0x886c,
+        0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x9002,
+        0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600,
+        0xa438, 0xc102, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01,
+        0xa438, 0xd600, 0xa438, 0x9002, 0xa438, 0x889b, 0xa438, 0xee84,
+        0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc102, 0xa438, 0x886c,
+        0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x9002,
+        0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600,
+        0xa438, 0xc102, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03,
+        0xa438, 0xd600, 0xa438, 0x9002, 0xa438, 0x889b, 0xa438, 0xee84,
+        0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc102, 0xa438, 0x886c,
+        0xa438, 0xee84, 0xa438, 0x6d00, 0xa438, 0xd600, 0xa438, 0x9102,
+        0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d01, 0xa438, 0xd600,
+        0xa438, 0xc202, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d01,
+        0xa438, 0xd600, 0xa438, 0x9102, 0xa438, 0x889b, 0xa438, 0xee84,
+        0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0xc202, 0xa438, 0x886c,
+        0xa438, 0xee84, 0xa438, 0x6d02, 0xa438, 0xd600, 0xa438, 0x9102,
+        0xa438, 0x889b, 0xa438, 0xee84, 0xa438, 0x6d04, 0xa438, 0xd600,
+        0xa438, 0xc202, 0xa438, 0x886c, 0xa438, 0xee84, 0xa438, 0x6d03,
+        0xa438, 0xd600, 0xa438, 0x9102, 0xa438, 0x889b, 0xa438, 0xee84,
+        0xa438, 0x6d08, 0xa438, 0xd600, 0xa438, 0xc202, 0xa438, 0x886c,
+        0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd414, 0xa438, 0x00d6,
+        0xa438, 0x000d, 0xa438, 0x0288, 0xa438, 0x6cbf, 0xa438, 0x8a2c,
+        0xa438, 0x0275, 0xa438, 0x3ebf, 0xa438, 0x8a2f, 0xa438, 0x0275,
+        0xa438, 0x3ebf, 0xa438, 0x8a32, 0xa438, 0x0275, 0xa438, 0x3ebf,
+        0xa438, 0x8a35, 0xa438, 0x0275, 0xa438, 0x3ebf, 0xa438, 0x8a38,
+        0xa438, 0x0275, 0xa438, 0x3ed4, 0xa438, 0x000f, 0xa438, 0xbf71,
+        0xa438, 0x5c02, 0xa438, 0x756a, 0xa438, 0xbf71, 0xa438, 0x5f02,
+        0xa438, 0x7589, 0xa438, 0xef31, 0xa438, 0xe783, 0xa438, 0x9fa3,
+        0xa438, 0x0ff2, 0xa438, 0xd400, 0xa438, 0x00bf, 0xa438, 0x715c,
+        0xa438, 0x0275, 0xa438, 0x6abf, 0xa438, 0x8a2c, 0xa438, 0x0275,
+        0xa438, 0x47bf, 0xa438, 0x8a2f, 0xa438, 0x0275, 0xa438, 0x47bf,
+        0xa438, 0x8a32, 0xa438, 0x0275, 0xa438, 0x47bf, 0xa438, 0x8a35,
+        0xa438, 0x0275, 0xa438, 0x47bf, 0xa438, 0x8a38, 0xa438, 0x0275,
+        0xa438, 0x47ee, 0xa438, 0x846d, 0xa438, 0x0fd4, 0xa438, 0x1000,
+        0xa438, 0xd600, 0xa438, 0x0d02, 0xa438, 0x886c, 0xa438, 0xfffe,
+        0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04, 0xa438, 0xf8f9,
+        0xa438, 0xef59, 0xa438, 0xf9fa, 0xa438, 0xef44, 0xa438, 0xbf73,
+        0xa438, 0xbf02, 0xa438, 0x756a, 0xa438, 0xef46, 0xa438, 0xbf73,
+        0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xe184, 0xa438, 0x6dbf,
+        0xa438, 0x8a29, 0xa438, 0x0275, 0xa438, 0x6ad4, 0xa438, 0x0000,
+        0xa438, 0xbf8a, 0xa438, 0x2902, 0xa438, 0x756a, 0xa438, 0xfefd,
+        0xa438, 0xef95, 0xa438, 0xfdfc, 0xa438, 0x04f9, 0xa438, 0xef59,
+        0xa438, 0xf9fa, 0xa438, 0xef46, 0xa438, 0xbf73, 0xa438, 0xc202,
+        0xa438, 0x756a, 0xa438, 0xe184, 0xa438, 0x6dbf, 0xa438, 0x8a23,
+        0xa438, 0x0275, 0xa438, 0x6abf, 0xa438, 0x8a26, 0xa438, 0x0275,
+        0xa438, 0x89fe, 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0x04e4,
+        0xa438, 0x8044, 0xa438, 0xee84, 0xa438, 0x6d0f, 0xa438, 0xd401,
+        0xa438, 0x20d6, 0xa438, 0x0010, 0xa438, 0x0288, 0xa438, 0x6cee,
+        0xa438, 0x8f7e, 0xa438, 0x01af, 0xa438, 0x6879, 0xa438, 0xee84,
+        0xa438, 0x6900, 0xa438, 0xee84, 0xa438, 0x6a00, 0xa438, 0xee8f,
+        0xa438, 0x7e00, 0xa438, 0xaf64, 0xa438, 0x8702, 0xa438, 0x6a2a,
+        0xa438, 0xe384, 0xa438, 0xf302, 0xa438, 0x6a4c, 0xa438, 0xac28,
+        0xa438, 0x08e0, 0xa438, 0x84f6, 0xa438, 0xf722, 0xa438, 0xe484,
+        0xa438, 0xf602, 0xa438, 0x6af9, 0xa438, 0xbf73, 0xa438, 0x4d02,
+        0xa438, 0x753e, 0xa438, 0xbf73, 0xa438, 0x5002, 0xa438, 0x753e,
+        0xa438, 0xbf73, 0xa438, 0x5302, 0xa438, 0x753e, 0xa438, 0xbf73,
+        0xa438, 0x5602, 0xa438, 0x753e, 0xa438, 0xd500, 0xa438, 0x0002,
+        0xa438, 0x6b1d, 0xa438, 0xbf73, 0xa438, 0x4402, 0xa438, 0x7547,
+        0xa438, 0xbf73, 0xa438, 0x4702, 0xa438, 0x7547, 0xa438, 0xbf73,
+        0xa438, 0x7702, 0xa438, 0x753e, 0xa438, 0xbf73, 0xa438, 0x4a02,
+        0xa438, 0x7547, 0xa438, 0xbf73, 0xa438, 0x5302, 0xa438, 0x753e,
+        0xa438, 0xbf73, 0xa438, 0x5602, 0xa438, 0x7547, 0xa438, 0xaf69,
+        0xa438, 0x4d02, 0xa438, 0x8948, 0xa438, 0x021e, 0xa438, 0x40af,
+        0xa438, 0x1e3f, 0xa438, 0xf8fa, 0xa438, 0xef69, 0xa438, 0xe080,
+        0xa438, 0x4fac, 0xa438, 0x2417, 0xa438, 0xe080, 0xa438, 0x44ad,
+        0xa438, 0x2417, 0xa438, 0x0289, 0xa438, 0x74e0, 0xa438, 0x8044,
+        0xa438, 0xac24, 0xa438, 0x0ebf, 0xa438, 0x8a3b, 0xa438, 0x0275,
+        0xa438, 0x47ae, 0xa438, 0x0602, 0xa438, 0x8a06, 0xa438, 0x0289,
+        0xa438, 0xfbef, 0xa438, 0x96fe, 0xa438, 0xfc04, 0xa438, 0xf8f9,
+        0xa438, 0xfaef, 0xa438, 0x69fa, 0xa438, 0xfbd2, 0xa438, 0x00a2,
+        0xa438, 0x0403, 0xa438, 0xaf89, 0xa438, 0xeabf, 0xa438, 0x6af1,
+        0xa438, 0x0277, 0xa438, 0x24ef, 0xa438, 0x010d, 0xa438, 0x11d0,
+        0xa438, 0x00ef, 0xa438, 0x640c, 0xa438, 0x66ef, 0xa438, 0x12bf,
+        0xa438, 0x8a23, 0xa438, 0x0275, 0xa438, 0x6ad3, 0xa438, 0x01a3,
+        0xa438, 0x4302, 0xa438, 0xae44, 0xa438, 0x1f00, 0xa438, 0xef13,
+        0xa438, 0xbf73, 0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xbf8a,
+        0xa438, 0x3e02, 0xa438, 0x7589, 0xa438, 0xd100, 0xa438, 0x0d01,
+        0xa438, 0x0c01, 0xa438, 0x1a46, 0xa438, 0xbf73, 0xa438, 0xbf02,
+        0xa438, 0x756a, 0xa438, 0x1f00, 0xa438, 0xef13, 0xa438, 0xbf73,
+        0xa438, 0xc202, 0xa438, 0x756a, 0xa438, 0xd101, 0xa438, 0xef02,
+        0xa438, 0x10b0, 0xa438, 0x02ae, 0xa438, 0x0449, 0xa438, 0x02ae,
+        0xa438, 0xf8bf, 0xa438, 0x73c5, 0xa438, 0x0275, 0xa438, 0x6abf,
+        0xa438, 0x73c5, 0xa438, 0x0275, 0xa438, 0x3e13, 0xa438, 0xaeb7,
+        0xa438, 0x12af, 0xa438, 0x897d, 0xa438, 0xd500, 0xa438, 0x0102,
+        0xa438, 0x6b1d, 0xa438, 0x0289, 0xa438, 0xfbff, 0xa438, 0xfeef,
+        0xa438, 0x96fe, 0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xe080,
+        0xa438, 0x44f6, 0xa438, 0x24e4, 0xa438, 0x8044, 0xa438, 0xfc04,
+        0xa438, 0xf8e0, 0xa438, 0x804f, 0xa438, 0xf624, 0xa438, 0xe480,
+        0xa438, 0x4ffc, 0xa438, 0x0455, 0xa438, 0xa6fe, 0xa438, 0x44a6,
+        0xa438, 0xfe66, 0xa438, 0xa4b6, 0xa438, 0x55a4, 0xa438, 0xb666,
+        0xa438, 0xac0e, 0xa438, 0x55ac, 0xa438, 0x0efe, 0xa438, 0xbda4,
+        0xa438, 0xf0bd, 0xa438, 0x9830, 0xa438, 0xbd96, 0xa438, 0xffbd,
+        0xa438, 0xdeee, 0xa438, 0xbdde, 0xa438, 0xddbd, 0xa438, 0xdebb,
+        0xa438, 0xbdde, 0xa438, 0xaabd, 0xa438, 0xde44, 0xa438, 0xac00,
+        0xa438, 0xf0bd, 0xa438, 0x9a00, 0xa436, 0xb818, 0xa438, 0x0fb9,
+        0xa436, 0xb81a, 0xa438, 0x64c3, 0xa436, 0xb81c, 0xa438, 0x64f1,
+        0xa436, 0xb81e, 0xa438, 0x6607, 0xa436, 0xb850, 0xa438, 0x6876,
+        0xa436, 0xb852, 0xa438, 0x647f, 0xa436, 0xb878, 0xa438, 0x68e8,
+        0xa436, 0xb884, 0xa438, 0x1e3c, 0xa436, 0xb832, 0xa438, 0x00df,
+        0xB82E, 0x0000, 0xa436, 0x8023, 0xa438, 0x0000, 0xB820, 0x0000,
+        0xFFFF, 0xFFFF
+};
+
+static const u16 phy_mcu_ram_code_8127a_1[] = {
+        0xa436, 0x8023, 0xa438, 0x6100, 0xa436, 0xB82E, 0xa438, 0x0001,
+        0xb820, 0x0090, 0xa436, 0xA016, 0xa438, 0x0000, 0xa436, 0xA012,
+        0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010,
+        0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a,
+        0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a,
+        0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0x1800, 0xa438, 0x801a,
+        0xa438, 0x1800, 0xa438, 0x801a, 0xa438, 0xce00, 0xa438, 0x2941,
+        0xa438, 0x8017, 0xa438, 0x2c59, 0xa438, 0x8017, 0xa438, 0x1800,
+        0xa438, 0x0e11, 0xa438, 0x8aff, 0xa438, 0x1800, 0xa438, 0x0e11,
+        0xa436, 0xA026, 0xa438, 0xffff, 0xa436, 0xA024, 0xa438, 0xffff,
+        0xa436, 0xA022, 0xa438, 0xffff, 0xa436, 0xA020, 0xa438, 0xffff,
+        0xa436, 0xA006, 0xa438, 0xffff, 0xa436, 0xA004, 0xa438, 0xffff,
+        0xa436, 0xA002, 0xa438, 0xffff, 0xa436, 0xA000, 0xa438, 0x0e10,
+        0xa436, 0xA008, 0xa438, 0x0100, 0xa436, 0xA016, 0xa438, 0x0000,
+        0xa436, 0xA012, 0xa438, 0x0ff8, 0xa436, 0xA014, 0xa438, 0x219a,
+        0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000,
+        0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA152,
+        0xa438, 0x21a4, 0xa436, 0xA154, 0xa438, 0x3fff, 0xa436, 0xA156,
+        0xa438, 0x3fff, 0xa436, 0xA158, 0xa438, 0x3fff, 0xa436, 0xA15A,
+        0xa438, 0x3fff, 0xa436, 0xA15C, 0xa438, 0x3fff, 0xa436, 0xA15E,
+        0xa438, 0x3fff, 0xa436, 0xA160, 0xa438, 0x3fff, 0xa436, 0xA150,
+        0xa438, 0x0001, 0xa436, 0xA016, 0xa438, 0x0010, 0xa436, 0xA012,
+        0xa438, 0x0000, 0xa436, 0xA014, 0xa438, 0x1800, 0xa438, 0x8010,
+        0xa438, 0x1800, 0xa438, 0x8014, 0xa438, 0x1800, 0xa438, 0x801a,
+        0xa438, 0x1800, 0xa438, 0x801e, 0xa438, 0x1800, 0xa438, 0x8026,
+        0xa438, 0x1800, 0xa438, 0x802e, 0xa438, 0x1800, 0xa438, 0x8036,
+        0xa438, 0x1800, 0xa438, 0x803a, 0xa438, 0xce01, 0xa438, 0x8208,
+        0xa438, 0x1800, 0xa438, 0x0028, 0xa438, 0x1000, 0xa438, 0x02c5,
+        0xa438, 0x1000, 0xa438, 0x0304, 0xa438, 0x1800, 0xa438, 0x0119,
+        0xa438, 0xce01, 0xa438, 0x8208, 0xa438, 0x1800, 0xa438, 0x009e,
+        0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208,
+        0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x015b,
+        0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208,
+        0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x01a9,
+        0xa438, 0xd501, 0xa438, 0xce01, 0xa438, 0xa50f, 0xa438, 0x8208,
+        0xa438, 0xd500, 0xa438, 0xaa0f, 0xa438, 0x1800, 0xa438, 0x01f4,
+        0xa438, 0x8208, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x02a5,
+        0xa438, 0xa208, 0xa438, 0xd500, 0xa438, 0x1800, 0xa438, 0x02b8,
+        0xa436, 0xA08E, 0xa438, 0x02b7, 0xa436, 0xA08C, 0xa438, 0x02a4,
+        0xa436, 0xA08A, 0xa438, 0x01e7, 0xa436, 0xA088, 0xa438, 0x019c,
+        0xa436, 0xA086, 0xa438, 0x014e, 0xa436, 0xA084, 0xa438, 0x009d,
+        0xa436, 0xA082, 0xa438, 0x0117, 0xa436, 0xA080, 0xa438, 0x0027,
+        0xa436, 0xA090, 0xa438, 0x00ff, 0xa436, 0xa016, 0xa438, 0x0020,
+        0xa436, 0xa012, 0xa438, 0x0000, 0xa436, 0xa014, 0xa438, 0x1800,
+        0xa438, 0x8010, 0xa438, 0x1800, 0xa438, 0x801d, 0xa438, 0x1800,
+        0xa438, 0x803b, 0xa438, 0x1800, 0xa438, 0x8087, 0xa438, 0x1800,
+        0xa438, 0x808e, 0xa438, 0x1800, 0xa438, 0x809d, 0xa438, 0x1800,
+        0xa438, 0x80b7, 0xa438, 0x1800, 0xa438, 0x80c4, 0xa438, 0xd1bc,
+        0xa438, 0xd040, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd700, 0xa438, 0x273d, 0xa438, 0x801b,
+        0xa438, 0x1800, 0xa438, 0x07d1, 0xa438, 0x1800, 0xa438, 0x080e,
+        0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8032, 0xa438, 0x33a9,
+        0xa438, 0x802a, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4,
+        0xa438, 0xd048, 0xa438, 0xf013, 0xa438, 0xd1b7, 0xa438, 0xd04b,
+        0xa438, 0xf010, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4,
+        0xa438, 0xd048, 0xa438, 0xf00b, 0xa438, 0xd1b7, 0xa438, 0xd04b,
+        0xa438, 0xf008, 0xa438, 0xd705, 0xa438, 0x4084, 0xa438, 0xd1f4,
+        0xa438, 0xd048, 0xa438, 0xf003, 0xa438, 0xd1b7, 0xa438, 0xd04b,
+        0xa438, 0x1800, 0xa438, 0x14cc, 0xa438, 0xd700, 0xa438, 0x2b59,
+        0xa438, 0x803f, 0xa438, 0xf003, 0xa438, 0x1800, 0xa438, 0x118f,
+        0xa438, 0x6060, 0xa438, 0x1800, 0xa438, 0x1167, 0xa438, 0xd700,
+        0xa438, 0x60c7, 0xa438, 0xd704, 0xa438, 0x609f, 0xa438, 0xd705,
+        0xa438, 0x4043, 0xa438, 0xf003, 0xa438, 0x1800, 0xa438, 0x1150,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0x8702, 0xa438, 0x8011,
+        0xa438, 0x9503, 0xa438, 0x800a, 0xa438, 0x81a0, 0xa438, 0x8302,
+        0xa438, 0x8480, 0xa438, 0x8686, 0xa438, 0xcde0, 0xa438, 0xd1ff,
+        0xa438, 0xd049, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0xd705, 0xa438, 0x417e, 0xa438, 0x0c03,
+        0xa438, 0x1502, 0xa438, 0xa011, 0xa438, 0x9503, 0xa438, 0xd1c8,
+        0xa438, 0xd045, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa702,
+        0xa438, 0x9503, 0xa438, 0xa00a, 0xa438, 0xa1a0, 0xa438, 0xa480,
+        0xa438, 0xa686, 0xa438, 0xd705, 0xa438, 0x605e, 0xa438, 0xa302,
+        0xa438, 0x9503, 0xa438, 0xd700, 0xa438, 0x37c9, 0xa438, 0x8083,
+        0xa438, 0x33a9, 0xa438, 0x807f, 0xa438, 0xd178, 0xa438, 0xd04b,
+        0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd1c8, 0xa438, 0xd04b,
+        0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd1e6, 0xa438, 0xd04b,
+        0xa438, 0x1800, 0xa438, 0x115d, 0xa438, 0xd71f, 0xa438, 0x6080,
+        0xa438, 0xd704, 0xa438, 0x1800, 0xa438, 0x1bc0, 0xa438, 0x1800,
+        0xa438, 0x1bc4, 0xa438, 0x4134, 0xa438, 0xd115, 0xa438, 0xd04f,
+        0xa438, 0x1000, 0xa438, 0x1d0b, 0xa438, 0x1000, 0xa438, 0x80ad,
+        0xa438, 0x1800, 0xa438, 0x01f2, 0xa438, 0x1000, 0xa438, 0x1d0b,
+        0xa438, 0x1000, 0xa438, 0x80ad, 0xa438, 0x1800, 0xa438, 0x01f9,
+        0xa438, 0x2969, 0xa438, 0x80a3, 0xa438, 0xd700, 0xa438, 0x606b,
+        0xa438, 0xd701, 0xa438, 0x60b4, 0xa438, 0x1000, 0xa438, 0x80ad,
+        0xa438, 0x1800, 0xa438, 0x0551, 0xa438, 0xd196, 0xa438, 0xd04d,
+        0xa438, 0x1000, 0xa438, 0x80ad, 0xa438, 0x1800, 0xa438, 0x054d,
+        0xa438, 0xd208, 0xa438, 0x0c09, 0xa438, 0x1301, 0xa438, 0x1000,
+        0xa438, 0x1cd2, 0xa438, 0xd701, 0xa438, 0x5fa3, 0xa438, 0xb302,
+        0xa438, 0xd200, 0xa438, 0x0800, 0xa438, 0xd705, 0xa438, 0x6064,
+        0xa438, 0x1800, 0xa438, 0x140a, 0xa438, 0x8810, 0xa438, 0xd199,
+        0xa438, 0xd04b, 0xa438, 0x1000, 0xa438, 0x1cd2, 0xa438, 0xd700,
+        0xa438, 0x5fba, 0xa438, 0x1800, 0xa438, 0x140a, 0xa438, 0x8910,
+        0xa438, 0xd704, 0xa438, 0x61a8, 0xa438, 0x8480, 0xa438, 0x8604,
+        0xa438, 0x8302, 0xa438, 0x81a0, 0xa438, 0xd703, 0xa438, 0x4112,
+        0xa438, 0x0c03, 0xa438, 0x1502, 0xa438, 0xa990, 0xa438, 0x9503,
+        0xa438, 0xb904, 0xa438, 0xf002, 0xa438, 0x0000, 0xa438, 0x0800,
+        0xa436, 0xa10e, 0xa438, 0x1de8, 0xa436, 0xa10c, 0xa438, 0x1352,
+        0xa436, 0xa10a, 0xa438, 0x0545, 0xa436, 0xa108, 0xa438, 0x01ed,
+        0xa436, 0xa106, 0xa438, 0x1bbf, 0xa436, 0xa104, 0xa438, 0x114b,
+        0xa436, 0xa102, 0xa438, 0x14bf, 0xa436, 0xa100, 0xa438, 0x07ce,
+        0xa436, 0xa110, 0xa438, 0x00ff, 0xa436, 0xA016, 0xa438, 0x0020,
+        0xa436, 0xA012, 0xa438, 0x1ff8, 0xa436, 0xA014, 0xa438, 0xd1ce,
+        0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000,
+        0xa438, 0x0000, 0xa438, 0x0000, 0xa438, 0x0000, 0xa436, 0xA164,
+        0xa438, 0x07fc, 0xa436, 0xA166, 0xa438, 0x143d, 0xa436, 0xA168,
+        0xa438, 0x3fff, 0xa436, 0xA16A, 0xa438, 0x3fff, 0xa436, 0xA16C,
+        0xa438, 0x3fff, 0xa436, 0xA16E, 0xa438, 0x3fff, 0xa436, 0xA170,
+        0xa438, 0x3fff, 0xa436, 0xA172, 0xa438, 0x3fff, 0xa436, 0xA162,
+        0xa438, 0x0003, 0xa436, 0xb87c, 0xa438, 0x8994, 0xa436, 0xb87e,
+        0xa438, 0xaf89, 0xa438, 0xacaf, 0xa438, 0x89e4, 0xa438, 0xaf89,
+        0xa438, 0xecaf, 0xa438, 0x8a04, 0xa438, 0xaf8a, 0xa438, 0x2eaf,
+        0xa438, 0x8a4a, 0xa438, 0xaf8d, 0xa438, 0x31af, 0xa438, 0x8dc6,
+        0xa438, 0x1f55, 0xa438, 0xe18f, 0xa438, 0xe3a1, 0xa438, 0x0007,
+        0xa438, 0xee86, 0xa438, 0xe900, 0xa438, 0xaf4f, 0xa438, 0x9ead,
+        0xa438, 0x281b, 0xa438, 0xe18f, 0xa438, 0xfcef, 0xa438, 0x71bf,
+        0xa438, 0x74f6, 0xa438, 0x027e, 0xa438, 0xd2ef, 0xa438, 0x641c,
+        0xa438, 0x670d, 0xa438, 0x67ef, 0xa438, 0x461f, 0xa438, 0x00bf,
+        0xa438, 0x74f6, 0xa438, 0x027e, 0xa438, 0xdee1, 0xa438, 0x8fe3,
+        0xa438, 0x0d11, 0xa438, 0xe58f, 0xa438, 0xe313, 0xa438, 0xaeca,
+        0xa438, 0x028d, 0xa438, 0xd1d3, 0xa438, 0x01af, 0xa438, 0x40d1,
+        0xa438, 0xbf7a, 0xa438, 0x6102, 0xa438, 0x7d44, 0xa438, 0xa100,
+        0xa438, 0x09e0, 0xa438, 0x8ffa, 0xa438, 0xe18f, 0xa438, 0xfbaf,
+        0xa438, 0x683d, 0xa438, 0x027f, 0xa438, 0xa9af, 0xa438, 0x682c,
+        0xa438, 0xbf8e, 0xa438, 0x4102, 0xa438, 0x7d44, 0xa438, 0xe58f,
+        0xa438, 0xecbf, 0xa438, 0x74cc, 0xa438, 0x027d, 0xa438, 0x44e3,
+        0xa438, 0x8fed, 0xa438, 0x0d31, 0xa438, 0xf63f, 0xa438, 0x0d11,
+        0xa438, 0xf62f, 0xa438, 0x1b13, 0xa438, 0xad2f, 0xa438, 0x06bf,
+        0xa438, 0x8e41, 0xa438, 0x027c, 0xa438, 0xf9d1, 0xa438, 0x01af,
+        0xa438, 0x5974, 0xa438, 0xee88, 0xa438, 0x8600, 0xa438, 0xe08f,
+        0xa438, 0xebad, 0xa438, 0x200b, 0xa438, 0xe18f, 0xa438, 0xecbf,
+        0xa438, 0x8e41, 0xa438, 0x027d, 0xa438, 0x25ae, 0xa438, 0x04ee,
+        0xa438, 0x8feb, 0xa438, 0x01af, 0xa438, 0x5945, 0xa438, 0xad28,
+        0xa438, 0x2ce0, 0xa438, 0x8fea, 0xa438, 0xa000, 0xa438, 0x0502,
+        0xa438, 0x8af0, 0xa438, 0xae1e, 0xa438, 0xa001, 0xa438, 0x0502,
+        0xa438, 0x8b9f, 0xa438, 0xae16, 0xa438, 0xa002, 0xa438, 0x0502,
+        0xa438, 0x8c0f, 0xa438, 0xae0e, 0xa438, 0xa003, 0xa438, 0x0502,
+        0xa438, 0x8c95, 0xa438, 0xae06, 0xa438, 0xa004, 0xa438, 0x0302,
+        0xa438, 0x8d08, 0xa438, 0xaf63, 0xa438, 0x8902, 0xa438, 0x8a7f,
+        0xa438, 0xaf63, 0xa438, 0x81f8, 0xa438, 0xef49, 0xa438, 0xf8e0,
+        0xa438, 0x8015, 0xa438, 0xad21, 0xa438, 0x19bf, 0xa438, 0x7bd8,
+        0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7bf3, 0xa438, 0x027d,
+        0xa438, 0x44bf, 0xa438, 0x7bf6, 0xa438, 0x027c, 0xa438, 0xf902,
+        0xa438, 0x638e, 0xa438, 0xee8f, 0xa438, 0xea00, 0xa438, 0xe080,
+        0xa438, 0x16ad, 0xa438, 0x233d, 0xa438, 0xbf7b, 0xa438, 0xf302,
+        0xa438, 0x7d44, 0xa438, 0xbf7a, 0xa438, 0x9402, 0xa438, 0x7cf9,
+        0xa438, 0xbf8e, 0xa438, 0x4402, 0xa438, 0x7cf9, 0xa438, 0xbf7a,
+        0xa438, 0xa602, 0xa438, 0x7cf9, 0xa438, 0xbf7a, 0xa438, 0xa302,
+        0xa438, 0x7cf9, 0xa438, 0xbf7a, 0xa438, 0xa902, 0xa438, 0x7cf9,
+        0xa438, 0xbf7a, 0xa438, 0xac02, 0xa438, 0x7cf9, 0xa438, 0xbf8e,
+        0xa438, 0x4702, 0xa438, 0x7cf9, 0xa438, 0xbf8e, 0xa438, 0x4a02,
+        0xa438, 0x7cf9, 0xa438, 0x0263, 0xa438, 0x8eee, 0xa438, 0x8fea,
+        0xa438, 0x00bf, 0xa438, 0x7c02, 0xa438, 0x027c, 0xa438, 0xf9fc,
+        0xa438, 0xef94, 0xa438, 0xfc04, 0xa438, 0xf8f9, 0xa438, 0xfbef,
+        0xa438, 0x79fb, 0xa438, 0xe080, 0xa438, 0x15ac, 0xa438, 0x2103,
+        0xa438, 0xaf8b, 0xa438, 0x70ee, 0xa438, 0x8888, 0xa438, 0x00ee,
+        0xa438, 0x888a, 0xa438, 0x00ee, 0xa438, 0x888b, 0xa438, 0x00bf,
+        0xa438, 0x7bd8, 0xa438, 0x027d, 0xa438, 0x02bf, 0xa438, 0x6000,
+        0xa438, 0xd788, 0xa438, 0x881f, 0xa438, 0x44d4, 0xa438, 0x000c,
+        0xa438, 0x0273, 0xa438, 0x3b02, 0xa438, 0x7fa9, 0xa438, 0xac28,
+        0xa438, 0x05ac, 0xa438, 0x290d, 0xa438, 0xae18, 0xa438, 0xe188,
+        0xa438, 0x98bf, 0xa438, 0x7be1, 0xa438, 0x027d, 0xa438, 0x25ae,
+        0xa438, 0x18e1, 0xa438, 0x8898, 0xa438, 0x0d11, 0xa438, 0xbf7b,
+        0xa438, 0xe102, 0xa438, 0x7d25, 0xa438, 0xae0b, 0xa438, 0xe188,
+        0xa438, 0x980d, 0xa438, 0x12bf, 0xa438, 0x7be1, 0xa438, 0x027d,
+        0xa438, 0x25bf, 0xa438, 0x88a0, 0xa438, 0xda19, 0xa438, 0xdb19,
+        0xa438, 0xd819, 0xa438, 0xd91f, 0xa438, 0x77bf, 0xa438, 0x88b1,
+        0xa438, 0xde19, 0xa438, 0xdf19, 0xa438, 0xdc19, 0xa438, 0xdd19,
+        0xa438, 0x17a7, 0xa438, 0x0004, 0xa438, 0xf302, 0xa438, 0x63cd,
+        0xa438, 0xee8f, 0xa438, 0xea01, 0xa438, 0xe080, 0xa438, 0x16ad,
+        0xa438, 0x2319, 0xa438, 0xee88, 0xa438, 0x8800, 0xa438, 0xee88,
+        0xa438, 0x8a00, 0xa438, 0xee88, 0xa438, 0x8b00, 0xa438, 0xbf8e,
+        0xa438, 0x4402, 0xa438, 0x7d02, 0xa438, 0x0263, 0xa438, 0xcdee,
+        0xa438, 0x8fea, 0xa438, 0x0102, 0xa438, 0x70de, 0xa438, 0xbf7c,
+        0xa438, 0x0202, 0xa438, 0x7d02, 0xa438, 0xffef, 0xa438, 0x97ff,
+        0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xf9fa, 0xa438, 0xef69,
+        0xa438, 0xfae0, 0xa438, 0x888a, 0xa438, 0xe188, 0xa438, 0x8b14,
+        0xa438, 0xe488, 0xa438, 0x8ae5, 0xa438, 0x888b, 0xa438, 0xbf88,
+        0xa438, 0x94d8, 0xa438, 0x19d9, 0xa438, 0xef64, 0xa438, 0xe088,
+        0xa438, 0x8ae1, 0xa438, 0x888b, 0xa438, 0x1b46, 0xa438, 0x9f30,
+        0xa438, 0x1f44, 0xa438, 0xe488, 0xa438, 0x8ae5, 0xa438, 0x888b,
+        0xa438, 0xe080, 0xa438, 0x15ad, 0xa438, 0x211a, 0xa438, 0x0260,
+        0xa438, 0xece0, 0xa438, 0x8016, 0xa438, 0xad23, 0xa438, 0x1602,
+        0xa438, 0x7c86, 0xa438, 0xef47, 0xa438, 0xe48f, 0xa438, 0xe9e5,
+        0xa438, 0x8fe8, 0xa438, 0xee8f, 0xa438, 0xea02, 0xa438, 0xae0b,
+        0xa438, 0x028c, 0xa438, 0x2eae, 0xa438, 0x0602, 0xa438, 0x8bfe,
+        0xa438, 0x0270, 0xa438, 0xdefe, 0xa438, 0xef96, 0xa438, 0xfefd,
+        0xa438, 0xfc04, 0xa438, 0xf8e1, 0xa438, 0x8888, 0xa438, 0x11e5,
+        0xa438, 0x8888, 0xa438, 0xad2a, 0xa438, 0x04ee, 0xa438, 0x8888,
+        0xa438, 0x00fc, 0xa438, 0x04f8, 0xa438, 0xfafb, 0xa438, 0xe08f,
+        0xa438, 0xe9e1, 0xa438, 0x8fe8, 0xa438, 0xef64, 0xa438, 0x1f00,
+        0xa438, 0xe18f, 0xa438, 0xe6ef, 0xa438, 0x7402, 0xa438, 0x7ca1,
+        0xa438, 0xad50, 0xa438, 0x0302, 0xa438, 0x8c2e, 0xa438, 0xfffe,
+        0xa438, 0xfc04, 0xa438, 0xf8fa, 0xa438, 0xef69, 0xa438, 0xfbbf,
+        0xa438, 0x7bf3, 0xa438, 0x027d, 0xa438, 0x44ac, 0xa438, 0x284c,
+        0xa438, 0x0264, 0xa438, 0x1cbf, 0xa438, 0x8e47, 0xa438, 0x027d,
+        0xa438, 0x02bf, 0xa438, 0x8e4a, 0xa438, 0x027d, 0xa438, 0x02d1,
+        0xa438, 0x43b1, 0xa438, 0xfebf, 0xa438, 0x7aa6, 0xa438, 0x027c,
+        0xa438, 0xf9bf, 0xa438, 0x7aa3, 0xa438, 0x027c, 0xa438, 0xf9bf,
+        0xa438, 0x7aa9, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aac,
+        0xa438, 0x027d, 0xa438, 0x02d1, 0xa438, 0x80e0, 0xa438, 0x8888,
+        0xa438, 0x100e, 0xa438, 0x11b0, 0xa438, 0xfcbf, 0xa438, 0x7a94,
+        0xa438, 0x027d, 0xa438, 0x2502, 0xa438, 0x7c86, 0xa438, 0xef47,
+        0xa438, 0xe48f, 0xa438, 0xe9e5, 0xa438, 0x8fe8, 0xa438, 0xee8f,
+        0xa438, 0xea03, 0xa438, 0xae07, 0xa438, 0xee8f, 0xa438, 0xea01,
+        0xa438, 0x0270, 0xa438, 0xdeff, 0xa438, 0xef96, 0xa438, 0xfefc,
+        0xa438, 0x04f8, 0xa438, 0xf9fa, 0xa438, 0xfbef, 0xa438, 0x79fb,
+        0xa438, 0xbf7a, 0xa438, 0x9402, 0xa438, 0x7d44, 0xa438, 0xef21,
+        0xa438, 0xbf7a, 0xa438, 0xb802, 0xa438, 0x7d44, 0xa438, 0x1f21,
+        0xa438, 0x9e19, 0xa438, 0xe08f, 0xa438, 0xe9e1, 0xa438, 0x8fe8,
+        0xa438, 0xef64, 0xa438, 0x1f00, 0xa438, 0xe18f, 0xa438, 0xe4ef,
+        0xa438, 0x7402, 0xa438, 0x7ca1, 0xa438, 0xad50, 0xa438, 0x3dee,
+        0xa438, 0x8fe7, 0xa438, 0x01bf, 0xa438, 0x7a94, 0xa438, 0x027c,
+        0xa438, 0xf9bf, 0xa438, 0x7aa6, 0xa438, 0x027c, 0xa438, 0xf9bf,
+        0xa438, 0x7aa3, 0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aa9,
+        0xa438, 0x027c, 0xa438, 0xf9bf, 0xa438, 0x7aac, 0xa438, 0x027d,
+        0xa438, 0x02bf, 0xa438, 0x8e47, 0xa438, 0x027c, 0xa438, 0xf9bf,
+        0xa438, 0x8e4a, 0xa438, 0x027c, 0xa438, 0xf902, 0xa438, 0x7c86,
+        0xa438, 0xef47, 0xa438, 0xe48f, 0xa438, 0xe9e5, 0xa438, 0x8fe8,
+        0xa438, 0xee8f, 0xa438, 0xea04, 0xa438, 0xffef, 0xa438, 0x97ff,
+        0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xf8fa, 0xa438, 0xfbe0,
+        0xa438, 0x8fe9, 0xa438, 0xe18f, 0xa438, 0xe8ef, 0xa438, 0x641f,
+        0xa438, 0x00e1, 0xa438, 0x8fe5, 0xa438, 0xef74, 0xa438, 0x027c,
+        0xa438, 0xa1ad, 0xa438, 0x500d, 0xa438, 0x0263, 0xa438, 0x8e02,
+        0xa438, 0x8bfe, 0xa438, 0xee8f, 0xa438, 0xea01, 0xa438, 0x0270,
+        0xa438, 0xdeff, 0xa438, 0xfefc, 0xa438, 0x04e3, 0xa438, 0x8fd8,
+        0xa438, 0xe787, 0xa438, 0x75e4, 0xa438, 0x8fe1, 0xa438, 0xe58f,
+        0xa438, 0xe2bf, 0xa438, 0x8fd9, 0xa438, 0xef32, 0xa438, 0x0c31,
+        0xa438, 0x1a93, 0xa438, 0xdc19, 0xa438, 0xdd02, 0xa438, 0x7fa9,
+        0xa438, 0xac2a, 0xa438, 0x18e0, 0xa438, 0x8fe1, 0xa438, 0xe18f,
+        0xa438, 0xe2ef, 0xa438, 0x74e1, 0xa438, 0x8775, 0xa438, 0x1f00,
+        0xa438, 0xef64, 0xa438, 0xe18f, 0xa438, 0xd8e5, 0xa438, 0x8775,
+        0xa438, 0xaf4d, 0xa438, 0x72bf, 0xa438, 0x7b3c, 0xa438, 0xef32,
+        0xa438, 0x4b03, 0xa438, 0x1a93, 0xa438, 0x027d, 0xa438, 0x44ef,
+        0xa438, 0x64e1, 0xa438, 0x8fff, 0xa438, 0x1f00, 0xa438, 0xef74,
+        0xa438, 0x1b67, 0xa438, 0xac4f, 0xa438, 0xcee0, 0xa438, 0x8ffd,
+        0xa438, 0xe18f, 0xa438, 0xfeef, 0xa438, 0x64e0, 0xa438, 0x8fe1,
+        0xa438, 0xe18f, 0xa438, 0xe2ef, 0xa438, 0x7402, 0xa438, 0x7c53,
+        0xa438, 0xac50, 0xa438, 0x02ae, 0xa438, 0xb6e1, 0xa438, 0x8775,
+        0xa438, 0x1f00, 0xa438, 0xef64, 0xa438, 0xe18f, 0xa438, 0xfcef,
+        0xa438, 0x711c, 0xa438, 0x670d, 0xa438, 0x67ef, 0xa438, 0x46e5,
+        0xa438, 0x8775, 0xa438, 0xef32, 0xa438, 0xd101, 0xa438, 0xa300,
+        0xa438, 0x02ae, 0xa438, 0x050c, 0xa438, 0x1183, 0xa438, 0xaef6,
+        0xa438, 0xe08f, 0xa438, 0xe31e, 0xa438, 0x10e5, 0xa438, 0x8fe3,
+        0xa438, 0xae89, 0xa438, 0xe287, 0xa438, 0x75e6, 0xa438, 0x8fd8,
+        0xa438, 0x1f22, 0xa438, 0xaf4d, 0xa438, 0x42f8, 0xa438, 0xf9ef,
+        0xa438, 0x59fa, 0xa438, 0xfbbf, 0xa438, 0x8fee, 0xa438, 0x027f,
+        0xa438, 0xa90d, 0xa438, 0x1149, 0xa438, 0x041a, 0xa438, 0x91d7,
+        0xa438, 0x8df3, 0xa438, 0xd68e, 0xa438, 0x2302, 0xa438, 0x72aa,
+        0xa438, 0xfffe, 0xa438, 0xef95, 0xa438, 0xfdfc, 0xa438, 0x0400,
+        0xa438, 0x7591, 0xa438, 0x0275, 0xa438, 0x4404, 0xa438, 0x758e,
+        0xa438, 0x2675, 0xa438, 0x4100, 0xa438, 0x8e26, 0xa438, 0x028e,
+        0xa438, 0x2304, 0xa438, 0x759d, 0xa438, 0x2675, 0xa438, 0x4700,
+        0xa438, 0x8e32, 0xa438, 0x028e, 0xa438, 0x2f04, 0xa438, 0x8e2c,
+        0xa438, 0x268e, 0xa438, 0x2900, 0xa438, 0x8e3e, 0xa438, 0x028e,
+        0xa438, 0x3b04, 0xa438, 0x8e38, 0xa438, 0x268e, 0xa438, 0x35fe,
+        0xa438, 0xad96, 0xa438, 0xdcad, 0xa438, 0x96ba, 0xa438, 0xad96,
+        0xa438, 0x98ad, 0xa438, 0x9676, 0xa438, 0xad98, 0xa438, 0x54ad,
+        0xa438, 0x9876, 0xa438, 0xae38, 0xa438, 0x54ae, 0xa438, 0x38fe,
+        0xa438, 0xae3a, 0xa438, 0xdcae, 0xa438, 0x3abb, 0xa438, 0xbf14,
+        0xa438, 0x99bd, 0xa438, 0xe0cc, 0xa438, 0xbdc8, 0xa438, 0xddbd,
+        0xa438, 0xc800, 0xa436, 0xb85e, 0xa438, 0x4f9a, 0xa436, 0xb860,
+        0xa438, 0x40cf, 0xa436, 0xb862, 0xa438, 0x6829, 0xa436, 0xb864,
+        0xa438, 0x5972, 0xa436, 0xb886, 0xa438, 0x5941, 0xa436, 0xb888,
+        0xa438, 0x636b, 0xa436, 0xb88a, 0xa438, 0x4d6b, 0xa436, 0xb88c,
+        0xa438, 0x4d40, 0xa436, 0xb838, 0xa438, 0x00ff, 0xb820, 0x0010,
+        0xa436, 0x8608, 0xa438, 0xaf86, 0xa438, 0xdaaf, 0xa438, 0x894c,
+        0xa438, 0xaf8a, 0xa438, 0xf8af, 0xa438, 0x8bf3, 0xa438, 0xaf8b,
+        0xa438, 0xf3af, 0xa438, 0x8bf3, 0xa438, 0xaf8b, 0xa438, 0xf3af,
+        0xa438, 0x8bf3, 0xa438, 0x006f, 0xa438, 0x4a03, 0xa438, 0x6f47,
+        0xa438, 0x266f, 0xa438, 0x5900, 0xa438, 0x6f4d, 0xa438, 0x016f,
+        0xa438, 0x5004, 0xa438, 0x6f56, 0xa438, 0x056f, 0xa438, 0x5f06,
+        0xa438, 0x6f5c, 0xa438, 0x2774, 0xa438, 0x7800, 0xa438, 0x6f68,
+        0xa438, 0x246f, 0xa438, 0x6b20, 0xa438, 0x6f6e, 0xa438, 0x206f,
+        0xa438, 0x7410, 0xa438, 0x7469, 0xa438, 0x1074, 0xa438, 0x6c10,
+        0xa438, 0x746f, 0xa438, 0x1074, 0xa438, 0x7225, 0xa438, 0x8bfc,
+        0xa438, 0x008c, 0xa438, 0x0802, 0xa438, 0x8c02, 0xa438, 0x038b,
+        0xa438, 0xff04, 0xa438, 0x6eed, 0xa438, 0x278c, 0xa438, 0x0520,
+        0xa438, 0x74da, 0xa438, 0x2074, 0xa438, 0xdd20, 0xa438, 0x74e0,
+        0xa438, 0x0074, 0xa438, 0xe300, 0xa438, 0x6ef3, 0xa438, 0x006e,
+        0xa438, 0xf600, 0xa438, 0x6ef9, 0xa438, 0x006e, 0xa438, 0xfc00,
+        0xa438, 0x6eff, 0xa438, 0x006f, 0xa438, 0x0200, 0xa438, 0x6f05,
+        0xa438, 0x026f, 0xa438, 0x0802, 0xa438, 0x6f0b, 0xa438, 0x026f,
+        0xa438, 0x0e02, 0xa438, 0x6f11, 0xa438, 0x026f, 0xa438, 0x1402,
+        0xa438, 0x6f17, 0xa438, 0x226f, 0xa438, 0x1a00, 0xa438, 0x723e,
+        0xa438, 0x016e, 0xa438, 0xed24, 0xa438, 0x6f50, 0xa438, 0x0072,
+        0xa438, 0x4701, 0xa438, 0x724a, 0xa438, 0x0272, 0xa438, 0x4d23,
+        0xa438, 0x7250, 0xa438, 0x1074, 0xa438, 0x6910, 0xa438, 0x746c,
+        0xa438, 0x1074, 0xa438, 0x6f00, 0xa438, 0x7472, 0xa438, 0x158c,
+        0xa438, 0x0b15, 0xa438, 0x8c0e, 0xa438, 0x158c, 0xa438, 0x1105,
+        0xa438, 0x8c14, 0xa438, 0x006f, 0xa438, 0x4a03, 0xa438, 0x6f47,
+        0xa438, 0x266f, 0xa438, 0x5900, 0xa438, 0x731f, 0xa438, 0x0273,
+        0xa438, 0x2203, 0xa438, 0x8c08, 0xa438, 0xee84, 0xa438, 0x7100,
+        0xa438, 0x0286, 0xa438, 0xece0, 0xa438, 0x8043, 0xa438, 0xf626,
+        0xa438, 0xe480, 0xa438, 0x43af, 0xa438, 0x6611, 0xa438, 0xf8e0,
+        0xa438, 0x8012, 0xa438, 0xac26, 0xa438, 0x03af, 0xa438, 0x86ff,
+        0xa438, 0x0287, 0xa438, 0x0102, 0xa438, 0x8906, 0xa438, 0x0289,
+        0xa438, 0x29fc, 0xa438, 0x04f8, 0xa438, 0xf9ef, 0xa438, 0x59f9,
+        0xa438, 0xfaee, 0xa438, 0x8476, 0xa438, 0x00d6, 0xa438, 0x008f,
+        0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200,
+        0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8fe7, 0xa438, 0xe58f,
+        0xa438, 0xe8ae, 0xa438, 0x06e0, 0xa438, 0x8fe7, 0xa438, 0xe18f,
+        0xa438, 0xe8ee, 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x00c0,
+        0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x00d6,
+        0xa438, 0x0090, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e,
+        0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8fe9,
+        0xa438, 0xe58f, 0xa438, 0xeaae, 0xa438, 0x06e0, 0xa438, 0x8fe9,
+        0xa438, 0xe18f, 0xa438, 0xeaee, 0xa438, 0x8476, 0xa438, 0x01d6,
+        0xa438, 0x00c1, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476,
+        0xa438, 0x00d6, 0xa438, 0x0091, 0xa438, 0x0266, 0xa438, 0x53ef,
+        0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4,
+        0xa438, 0x8feb, 0xa438, 0xe58f, 0xa438, 0xecae, 0xa438, 0x06e0,
+        0xa438, 0x8feb, 0xa438, 0xe18f, 0xa438, 0xecee, 0xa438, 0x8476,
+        0xa438, 0x01d6, 0xa438, 0x00c2, 0xa438, 0x0266, 0xa438, 0x71ee,
+        0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x008f, 0xa438, 0x0266,
+        0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f,
+        0xa438, 0x08e4, 0xa438, 0x8fed, 0xa438, 0xe58f, 0xa438, 0xeeae,
+        0xa438, 0x06e0, 0xa438, 0x8fed, 0xa438, 0xe18f, 0xa438, 0xeeee,
+        0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x00c0, 0xa438, 0x0266,
+        0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x01d6, 0xa438, 0x0090,
+        0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200,
+        0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8fef, 0xa438, 0xe58f,
+        0xa438, 0xf0ae, 0xa438, 0x06e0, 0xa438, 0x8fef, 0xa438, 0xe18f,
+        0xa438, 0xf0ee, 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x00c1,
+        0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x01d6,
+        0xa438, 0x0091, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e,
+        0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff1,
+        0xa438, 0xe58f, 0xa438, 0xf2ae, 0xa438, 0x06e0, 0xa438, 0x8ff1,
+        0xa438, 0xe18f, 0xa438, 0xf2ee, 0xa438, 0x8476, 0xa438, 0x02d6,
+        0xa438, 0x00c2, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476,
+        0xa438, 0x02d6, 0xa438, 0x008f, 0xa438, 0x0266, 0xa438, 0x53ef,
+        0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4,
+        0xa438, 0x8ff3, 0xa438, 0xe58f, 0xa438, 0xf4ae, 0xa438, 0x06e0,
+        0xa438, 0x8ff3, 0xa438, 0xe18f, 0xa438, 0xf4ee, 0xa438, 0x8476,
+        0xa438, 0x04d6, 0xa438, 0x00c0, 0xa438, 0x0266, 0xa438, 0x71ee,
+        0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x0090, 0xa438, 0x0266,
+        0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f,
+        0xa438, 0x08e4, 0xa438, 0x8ff5, 0xa438, 0xe58f, 0xa438, 0xf6ae,
+        0xa438, 0x06e0, 0xa438, 0x8ff5, 0xa438, 0xe18f, 0xa438, 0xf6ee,
+        0xa438, 0x8476, 0xa438, 0x04d6, 0xa438, 0x00c1, 0xa438, 0x0266,
+        0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x02d6, 0xa438, 0x0091,
+        0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200,
+        0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff7, 0xa438, 0xe58f,
+        0xa438, 0xf8ae, 0xa438, 0x06e0, 0xa438, 0x8ff7, 0xa438, 0xe18f,
+        0xa438, 0xf8ee, 0xa438, 0x8476, 0xa438, 0x04d6, 0xa438, 0x00c2,
+        0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476, 0xa438, 0x03d6,
+        0xa438, 0x008f, 0xa438, 0x0266, 0xa438, 0x53ef, 0xa438, 0x643e,
+        0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4, 0xa438, 0x8ff9,
+        0xa438, 0xe58f, 0xa438, 0xfaae, 0xa438, 0x06e0, 0xa438, 0x8ff9,
+        0xa438, 0xe18f, 0xa438, 0xfaee, 0xa438, 0x8476, 0xa438, 0x08d6,
+        0xa438, 0x00c0, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x8476,
+        0xa438, 0x03d6, 0xa438, 0x0090, 0xa438, 0x0266, 0xa438, 0x53ef,
+        0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f, 0xa438, 0x08e4,
+        0xa438, 0x8ffb, 0xa438, 0xe58f, 0xa438, 0xfcae, 0xa438, 0x06e0,
+        0xa438, 0x8ffb, 0xa438, 0xe18f, 0xa438, 0xfcee, 0xa438, 0x8476,
+        0xa438, 0x08d6, 0xa438, 0x00c1, 0xa438, 0x0266, 0xa438, 0x71ee,
+        0xa438, 0x8476, 0xa438, 0x03d6, 0xa438, 0x0091, 0xa438, 0x0266,
+        0xa438, 0x53ef, 0xa438, 0x643e, 0xa438, 0x1200, 0xa438, 0xac4f,
+        0xa438, 0x08e4, 0xa438, 0x8ffd, 0xa438, 0xe58f, 0xa438, 0xfeae,
+        0xa438, 0x06e0, 0xa438, 0x8ffd, 0xa438, 0xe18f, 0xa438, 0xfeee,
+        0xa438, 0x8476, 0xa438, 0x08d6, 0xa438, 0x00c2, 0xa438, 0x0266,
+        0xa438, 0x71fe, 0xa438, 0xfdef, 0xa438, 0x95fd, 0xa438, 0xfc04,
+        0xa438, 0xf8f9, 0xa438, 0xfad4, 0xa438, 0x0400, 0xa438, 0xd600,
+        0xa438, 0x0dd3, 0xa438, 0x0fe7, 0xa438, 0x8476, 0xa438, 0x0266,
+        0xa438, 0x71d4, 0xa438, 0x1400, 0xa438, 0xd600, 0xa438, 0x0dd3,
+        0xa438, 0x0fe7, 0xa438, 0x8476, 0xa438, 0x0266, 0xa438, 0x71fe,
+        0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xf9fa, 0xa438, 0xd410,
+        0xa438, 0x00d6, 0xa438, 0x000d, 0xa438, 0xd30f, 0xa438, 0xe784,
+        0xa438, 0x7602, 0xa438, 0x6671, 0xa438, 0xd400, 0xa438, 0x00d6,
+        0xa438, 0x000d, 0xa438, 0xd30f, 0xa438, 0xe784, 0xa438, 0x7602,
+        0xa438, 0x6671, 0xa438, 0xfefd, 0xa438, 0xfc04, 0xa438, 0xe080,
+        0xa438, 0x4fac, 0xa438, 0x2317, 0xa438, 0xe080, 0xa438, 0x44ad,
+        0xa438, 0x231a, 0xa438, 0x0289, 0xa438, 0x75e0, 0xa438, 0x8044,
+        0xa438, 0xac23, 0xa438, 0x11bf, 0xa438, 0x6ecf, 0xa438, 0x0276,
+        0xa438, 0x74ae, 0xa438, 0x0902, 0xa438, 0x8adb, 0xa438, 0x021f,
+        0xa438, 0xe702, 0xa438, 0x1fbb, 0xa438, 0xaf1f, 0xa438, 0x95f8,
+        0xa438, 0xf9ef, 0xa438, 0x59f9, 0xa438, 0xfafb, 0xa438, 0xe080,
+        0xa438, 0x12ac, 0xa438, 0x2303, 0xa438, 0xaf8a, 0xa438, 0xd0d4,
+        0xa438, 0x0120, 0xa438, 0xd600, 0xa438, 0x10d2, 0xa438, 0x0fe6,
+        0xa438, 0x8476, 0xa438, 0x0266, 0xa438, 0x71ee, 0xa438, 0x846f,
+        0xa438, 0x00d4, 0xa438, 0x000f, 0xa438, 0xbf72, 0xa438, 0x9e02,
+        0xa438, 0x7697, 0xa438, 0x0275, 0xa438, 0xbeef, 0xa438, 0x47e4,
+        0xa438, 0x8474, 0xa438, 0xe584, 0xa438, 0x75bf, 0xa438, 0x729b,
+        0xa438, 0x0276, 0xa438, 0xb6e5, 0xa438, 0x846f, 0xa438, 0xef31,
+        0xa438, 0xbf6e, 0xa438, 0x0602, 0xa438, 0x76b6, 0xa438, 0xef64,
+        0xa438, 0xbf6e, 0xa438, 0x0902, 0xa438, 0x76b6, 0xa438, 0x1e64,
+        0xa438, 0xbf6e, 0xa438, 0x0f02, 0xa438, 0x76b6, 0xa438, 0x1e64,
+        0xa438, 0xac40, 0xa438, 0x05a3, 0xa438, 0x0f0c, 0xa438, 0xae26,
+        0xa438, 0xa303, 0xa438, 0x02ae, 0xa438, 0x21a3, 0xa438, 0x0c02,
+        0xa438, 0xae1c, 0xa438, 0xe084, 0xa438, 0x74e1, 0xa438, 0x8475,
+        0xa438, 0xef64, 0xa438, 0xd000, 0xa438, 0xd196, 0xa438, 0xef74,
+        0xa438, 0x0275, 0xa438, 0xd9ad, 0xa438, 0x50b7, 0xa438, 0xe083,
+        0xa438, 0xecf7, 0xa438, 0x23e4, 0xa438, 0x83ec, 0xa438, 0xbf72,
+        0xa438, 0x9e02, 0xa438, 0x766b, 0xa438, 0x0287, 0xa438, 0x0102,
+        0xa438, 0x8906, 0xa438, 0xee83, 0xa438, 0xe800, 0xa438, 0xbf72,
+        0xa438, 0x6b02, 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x6e02,
+        0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x7102, 0xa438, 0x766b,
+        0xa438, 0xbf72, 0xa438, 0x7402, 0xa438, 0x766b, 0xa438, 0xbf72,
+        0xa438, 0x7702, 0xa438, 0x766b, 0xa438, 0xbf72, 0xa438, 0x7a02,
+        0xa438, 0x766b, 0xa438, 0xd400, 0xa438, 0x0fbf, 0xa438, 0x7295,
+        0xa438, 0x0276, 0xa438, 0x97d7, 0xa438, 0x0400, 0xa438, 0xbf6e,
+        0xa438, 0x0602, 0xa438, 0x76b6, 0xa438, 0xef64, 0xa438, 0xbf6e,
+        0xa438, 0x0902, 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xbf6e,
+        0xa438, 0x0f02, 0xa438, 0x76b6, 0xa438, 0x1e64, 0xa438, 0xac40,
+        0xa438, 0x0fbf, 0xa438, 0x7298, 0xa438, 0x0276, 0xa438, 0xb6e5,
+        0xa438, 0x83e8, 0xa438, 0xa10f, 0xa438, 0x28af, 0xa438, 0x8a95,
+        0xa438, 0xbf8b, 0xa438, 0xf302, 0xa438, 0x76b6, 0xa438, 0xac28,
+        0xa438, 0x02ae, 0xa438, 0x0bbf, 0xa438, 0x8bf9, 0xa438, 0x0276,
+        0xa438, 0xb6e5, 0xa438, 0x83e8, 0xa438, 0xae09, 0xa438, 0xbf8b,
+        0xa438, 0xf602, 0xa438, 0x76b6, 0xa438, 0xe583, 0xa438, 0xe8a1,
+        0xa438, 0x0303, 0xa438, 0xaf8a, 0xa438, 0x95b7, 0xa438, 0xafe2,
+        0xa438, 0x83ec, 0xa438, 0xf735, 0xa438, 0xe683, 0xa438, 0xecbf,
+        0xa438, 0x7295, 0xa438, 0x0276, 0xa438, 0x6bbf, 0xa438, 0x726b,
+        0xa438, 0x0276, 0xa438, 0x74bf, 0xa438, 0x726e, 0xa438, 0x0276,
+        0xa438, 0x74bf, 0xa438, 0x7271, 0xa438, 0x0276, 0xa438, 0x74bf,
+        0xa438, 0x7274, 0xa438, 0x0276, 0xa438, 0x74bf, 0xa438, 0x7277,
+        0xa438, 0x0276, 0xa438, 0x74bf, 0xa438, 0x727a, 0xa438, 0x0276,
+        0xa438, 0x7402, 0xa438, 0x8929, 0xa438, 0xd401, 0xa438, 0x28d6,
+        0xa438, 0x0010, 0xa438, 0xd20f, 0xa438, 0xe684, 0xa438, 0x7602,
+        0xa438, 0x6671, 0xa438, 0x021f, 0xa438, 0xbbff, 0xa438, 0xfefd,
+        0xa438, 0xef95, 0xa438, 0xfdfc, 0xa438, 0x04f8, 0xa438, 0xf9ef,
+        0xa438, 0x59f9, 0xa438, 0xe080, 0xa438, 0x12ad, 0xa438, 0x230c,
+        0xa438, 0xbf72, 0xa438, 0x9e02, 0xa438, 0x766b, 0xa438, 0xbf72,
+        0xa438, 0x9502, 0xa438, 0x766b, 0xa438, 0xfdef, 0xa438, 0x95fd,
+        0xa438, 0xfc04, 0xa438, 0xbf6e, 0xa438, 0x0602, 0xa438, 0x76b6,
+        0xa438, 0xef64, 0xa438, 0xbf6e, 0xa438, 0x0902, 0xa438, 0x76b6,
+        0xa438, 0x1e64, 0xa438, 0xbf6e, 0xa438, 0x0f02, 0xa438, 0x76b6,
+        0xa438, 0x1e64, 0xa438, 0xac40, 0xa438, 0x0ebf, 0xa438, 0x7298,
+        0xa438, 0x0276, 0xa438, 0xb6e5, 0xa438, 0x8478, 0xa438, 0xa10f,
+        0xa438, 0x26ae, 0xa438, 0x47bf, 0xa438, 0x8bf3, 0xa438, 0x0276,
+        0xa438, 0xb6ac, 0xa438, 0x2802, 0xa438, 0xae0b, 0xa438, 0xbf8b,
+        0xa438, 0xf902, 0xa438, 0x76b6, 0xa438, 0xe584, 0xa438, 0x78ae,
+        0xa438, 0x09bf, 0xa438, 0x8bf6, 0xa438, 0x0276, 0xa438, 0xb6e5,
+        0xa438, 0x8478, 0xa438, 0xa103, 0xa438, 0x02ae, 0xa438, 0x23e0,
+        0xa438, 0x8474, 0xa438, 0xe184, 0xa438, 0x75ef, 0xa438, 0x64e0,
+        0xa438, 0x83fc, 0xa438, 0xe183, 0xa438, 0xfdef, 0xa438, 0x7402,
+        0xa438, 0x75d9, 0xa438, 0xad50, 0xa438, 0x0ae0, 0xa438, 0x83ec,
+        0xa438, 0xf721, 0xa438, 0xe483, 0xa438, 0xecae, 0xa438, 0x03af,
+        0xa438, 0x68e4, 0xa438, 0xbf72, 0xa438, 0x9502, 0xa438, 0x766b,
+        0xa438, 0xe083, 0xa438, 0xebad, 0xa438, 0x2170, 0xa438, 0xbf73,
+        0xa438, 0x7f02, 0xa438, 0x766b, 0xa438, 0xd700, 0xa438, 0x64bf,
+        0xa438, 0x73c4, 0xa438, 0x0276, 0xa438, 0xb6a4, 0xa438, 0x0000,
+        0xa438, 0x02ae, 0xa438, 0x0d87, 0xa438, 0xa700, 0xa438, 0x00ef,
+        0xa438, 0xe183, 0xa438, 0xecf7, 0xa438, 0x2ae5, 0xa438, 0x83ec,
+        0xa438, 0xbf73, 0xa438, 0xbe02, 0xa438, 0x766b, 0xa438, 0xbf73,
+        0xa438, 0xb802, 0xa438, 0x766b, 0xa438, 0xbf73, 0xa438, 0xc102,
+        0xa438, 0x766b, 0xa438, 0xbf73, 0xa438, 0xbb02, 0xa438, 0x766b,
+        0xa438, 0xe084, 0xa438, 0x9ee1, 0xa438, 0x849f, 0xa438, 0xbf72,
+        0xa438, 0x7d02, 0xa438, 0x7697, 0xa438, 0xbf72, 0xa438, 0x8002,
+        0xa438, 0x7697, 0xa438, 0xbf72, 0xa438, 0x8302, 0xa438, 0x7697,
+        0xa438, 0xbf72, 0xa438, 0x8602, 0xa438, 0x7697, 0xa438, 0xbf72,
+        0xa438, 0x8902, 0xa438, 0x7674, 0xa438, 0xbf72, 0xa438, 0x8c02,
+        0xa438, 0x7674, 0xa438, 0xbf72, 0xa438, 0x8f02, 0xa438, 0x7674,
+        0xa438, 0xbf72, 0xa438, 0x9202, 0xa438, 0x7674, 0xa438, 0xee84,
+        0xa438, 0x7700, 0xa438, 0xe080, 0xa438, 0x44f6, 0xa438, 0x21e4,
+        0xa438, 0x8044, 0xa438, 0xaf68, 0xa438, 0xe411, 0xa438, 0xd1a4,
+        0xa438, 0x10bc, 0xa438, 0x7432, 0xa438, 0xbc74, 0xa438, 0xbbbf,
+        0xa438, 0x14cc, 0xa438, 0xbfaa, 0xa438, 0x00bf, 0xa438, 0x9055,
+        0xa438, 0xbf06, 0xa438, 0x10bf, 0xa438, 0xb876, 0xa438, 0xbe02,
+        0xa438, 0x54be, 0xa438, 0x0232, 0xa438, 0xbe02, 0xa438, 0x10be,
+        0xa438, 0x0200, 0xa436, 0x8fe7, 0xa438, 0x1200, 0xa436, 0x8fe9,
+        0xa438, 0x1200, 0xa436, 0x8feb, 0xa438, 0x1200, 0xa436, 0x8fed,
+        0xa438, 0x1200, 0xa436, 0x8fef, 0xa438, 0x1200, 0xa436, 0x8ff1,
+        0xa438, 0x1200, 0xa436, 0x8ff3, 0xa438, 0x1200, 0xa436, 0x8ff5,
+        0xa438, 0x1200, 0xa436, 0x8ff7, 0xa438, 0x1200, 0xa436, 0x8ff9,
+        0xa438, 0x1200, 0xa436, 0x8ffb, 0xa438, 0x1200, 0xa436, 0x8ffd,
+        0xa438, 0x1200, 0xa436, 0xb818, 0xa438, 0x6602, 0xa436, 0xb81a,
+        0xa438, 0x1f75, 0xa436, 0xb81c, 0xa438, 0x67eb, 0xa436, 0xb81e,
+        0xa438, 0xffff, 0xa436, 0xb850, 0xa438, 0xffff, 0xa436, 0xb852,
+        0xa438, 0xffff, 0xa436, 0xb878, 0xa438, 0xffff, 0xa436, 0xb884,
+        0xa438, 0xffff, 0xa436, 0xb832, 0xa438, 0x0007, 0xB82E, 0x0000,
+        0xa436, 0x8023, 0xa438, 0x0000, 0xB820, 0x0000, 0xFFFF, 0xFFFF
+};
+
+static const u16 phy_mcu_ram_code_8127a_2[] = {
+        0xb892, 0x0000, 0xB88E, 0xc07c, 0xB890, 0x0203, 0xB890, 0x0304,
+        0xB890, 0x0405, 0xB890, 0x0607, 0xB890, 0x0809, 0xB890, 0x0B0D,
+        0xB890, 0x0F11, 0xB890, 0x1418, 0xB890, 0x1B20, 0xB890, 0x252B,
+        0xB890, 0x343E, 0xB890, 0x4854, 0xB890, 0x6203, 0xB890, 0x0304,
+        0xB890, 0x0506, 0xB890, 0x080A, 0xB890, 0x0C0E, 0xB890, 0x1216,
+        0xB890, 0x1B22, 0xB890, 0x2A34, 0xB890, 0x404F, 0xB890, 0x6171,
+        0xB890, 0x7884, 0xB890, 0x9097, 0xB890, 0x0203, 0xB890, 0x0406,
+        0xB890, 0x080B, 0xB890, 0x0E13, 0xB890, 0x1820, 0xB890, 0x2A39,
+        0xB890, 0x4856, 0xB890, 0xE060, 0xB890, 0xE050, 0xB890, 0xD080,
+        0xB890, 0x8070, 0xB890, 0x70A0, 0xB890, 0x1000, 0xB890, 0x60D0,
+        0xB890, 0xB010, 0xB890, 0xE0B0, 0xB890, 0x80C0, 0xB890, 0xE000,
+        0xB890, 0x2020, 0xB890, 0x1020, 0xB890, 0xE090, 0xB890, 0x80C0,
+        0xB890, 0x3020, 0xB890, 0x00E0, 0xB890, 0x40A0, 0xB890, 0xE020,
+        0xB890, 0x5060, 0xB890, 0xE0D0, 0xB890, 0xA000, 0xB890, 0x3030,
+        0xB890, 0x4070, 0xB890, 0xE0E0, 0xB890, 0xD080, 0xB890, 0xA010,
+        0xB890, 0xE040, 0xB890, 0x80B0, 0xB890, 0x50B0, 0xB890, 0x2090,
+        0xB820, 0x0000, 0xFFFF, 0xFFFF
+};
+
+static void
+rtl8127_real_set_phy_mcu_8127a_tc_1(struct net_device *dev)
+{
+        rtl8127_set_phy_mcu_ram_code(dev,
+                                     phy_mcu_ram_code_8127a_tc_1,
+                                     ARRAY_SIZE(phy_mcu_ram_code_8127a_tc_1));
+}
+
+static void
+rtl8127_set_phy_mcu_8127a_tc(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_set_phy_mcu_patch_request(tp);
+
+        rtl8127_real_set_phy_mcu_8127a_tc_1(dev);
+
+        rtl8127_clear_phy_mcu_patch_request(tp);
+}
+
+static void
+rtl8127_real_set_phy_mcu_8127a_1(struct net_device *dev)
+{
+        rtl8127_set_phy_mcu_ram_code(dev,
+                                     phy_mcu_ram_code_8127a_1,
+                                     ARRAY_SIZE(phy_mcu_ram_code_8127a_1));
+}
+
+static void
+rtl8127_real_set_phy_mcu_8127a_2(struct net_device *dev)
+{
+        rtl8127_set_phy_mcu_ram_code(dev,
+                                     phy_mcu_ram_code_8127a_2,
+                                     ARRAY_SIZE(phy_mcu_ram_code_8127a_2));
+}
+
+static void
+rtl8127_set_phy_mcu_8127a_1(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_set_phy_mcu_patch_request(tp);
+
+        rtl8127_real_set_phy_mcu_8127a_1(dev);
+
+        rtl8127_clear_phy_mcu_patch_request(tp);
+
+        rtl8127_set_phy_mcu_patch_request(tp);
+
+        rtl8127_real_set_phy_mcu_8127a_2(dev);
+
+        rtl8127_clear_phy_mcu_patch_request(tp);
+}
+
+static void
+rtl8127_init_hw_phy_mcu(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u8 require_disable_phy_disable_mode = FALSE;
+
+        if (tp->NotWrRamCodeToMicroP == TRUE)
+                return;
+
+        if (rtl8127_check_hw_phy_mcu_code_ver(dev))
+                return;
+
+        if (HW_SUPPORT_CHECK_PHY_DISABLE_MODE(tp) && rtl8127_is_in_phy_disable_mode(dev))
+                require_disable_phy_disable_mode = TRUE;
+
+        if (require_disable_phy_disable_mode)
+                rtl8127_disable_phy_disable_mode(dev);
+
+        switch (tp->mcfg) {
+        case CFG_METHOD_1:
+                rtl8127_set_phy_mcu_8127a_tc(dev);
+                break;
+        case CFG_METHOD_2:
+                rtl8127_set_phy_mcu_8127a_1(dev);
+                break;
+        default:
+                break;
+        }
+
+        if (require_disable_phy_disable_mode)
+                rtl8127_enable_phy_disable_mode(dev);
+
+        rtl8127_write_hw_phy_mcu_code_ver(dev);
+
+        rtl8127_mdio_write(tp,0x1F, 0x0000);
+
+        tp->HwHasWrRamCodeToMicroP = TRUE;
+}
+#endif
+
+static void
+rtl8127_enable_phy_aldps(struct rtl8127_private *tp)
+{
+        //enable aldps
+        //GPHY OCP 0xA430 bit[2] = 0x1 (en_aldps)
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA430, BIT_2);
+}
+
+static void
+rtl8127_tgphy_irq_mask_and_ack(struct rtl8127_private *tp)
+{
+        switch (tp->mcfg) {
+        case CFG_METHOD_2:
+                rtl8127_mdio_direct_write_phy_ocp(tp, 0xA4D2, 0x0000);
+                (void)rtl8127_mdio_direct_read_phy_ocp(tp, 0xA4D4);
+                break;
+        default:
+                break;
+        }
+}
+
+static void
+rtl8127_hw_phy_config_8127a_tc_1(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA442, BIT_11);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x815E);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xa438,
+                                              0xFF00,
+                                              0x8600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8169);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xa438,
+                                              0xFF00,
+                                              0x8600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8174);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xa438,
+                                              0xFF00,
+                                              0xA100);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83BF);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xa438,
+                                              0xFF00,
+                                              0x5A00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83C5);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xa438,
+                                              0xFF00,
+                                              0x5A00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x83CB);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xa438,
+                                              0xFF00,
+                                              0x8B00);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8238);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xC000,
+                                              0x4000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x823A);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xA000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xa436, 0x8148);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xa438,
+                                              0xFF00,
+                                              0x0100);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84AD);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0C00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B2);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0800);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B7);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x1400);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84BC);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0040);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00D6);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84BE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00A0);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84AE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0C0C);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84B0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0C0C);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xBD7A, 0xAAAA);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xBCE0, 0x6666);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85FC);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0AAA);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85FF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0AAA);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC32, BIT_3);
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC32, BIT_11);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xADDC,
+                                              0x3FFF,
+                                              0x2000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8111);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0F00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E9);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0F00);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAEC4,
+                                              0xFF00,
+                                              0x4600);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAC56,
+                                              0x0007,
+                                              0x0005);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x825B);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0D00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8283);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0334);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8289);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xADB8, 0x0190);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAE3A,
+                                              0x00FF,
+                                              0x0026);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAE4A,
+                                              0x0FF0,
+                                              0x0150);
+
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xAEC2, BIT_12);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAE22, 0x0352);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC0, 0x00FA);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8188);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xF500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8203);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xF500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x827E);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xF500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81CB);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8246);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x818A);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xF500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8205);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xF500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8280);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xF500);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83DD);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x840B);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83BC);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83BE);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C0);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x0700,
+                                              0x0400);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C2);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C4);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_10);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x83C6);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x0700,
+                                              0x0400);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C2);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0xFEFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0003);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C6);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0116);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C8);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x6300);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x822A);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x4FFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81AF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x4067);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8134);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x5069);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x822C);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x1A00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B1);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x3A00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8136);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x810E);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x9000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8114);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x9000);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB63C, BIT_9);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80B4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xB63B);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E4);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0700);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E5);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x7000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80EA);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80EB);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x5000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8291);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0416);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8015);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_11);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x895E);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8960);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826B);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8175);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF0AF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826D);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0100);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F2);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0100);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8177);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0100);
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1C, BIT_8 | BIT_7);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1E, BIT_13 | BIT_12);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAD96, 0xAAFF);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAD98,
+                                              0x00FF,
+                                              0x00AA);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAE38,
+                                              0x3FFF,
+                                              0x2554);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAE3A,
+                                              0xF000,
+                                              0xA000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8932);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0900);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x892F);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0900);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x892C);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0900);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85B6);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, 0xFF00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x85B4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8905);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0xFF00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8853);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x2800);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x884B);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x3F00);
+
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xBDE6, 0x3FFF);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xBDE8, 0x3FFF);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xBF0E, 0x0003);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8156);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x1600);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80AB);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x7500);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80C3);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x090D);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80C6);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0xC600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x80BF);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x5500);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8096);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x4500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x809D);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0200);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x809B);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0xE50A);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8099);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x9906);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x831F);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x5000);
+
+
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB648, BIT_14);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA4E0, BIT_15);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x849A);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0004);
+
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xAC1C, 0x0C00);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA42C, BIT_6);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xACBA, 0xFC00);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8122);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0xC000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8123);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0xC000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80FA);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x2000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x825B);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0xB000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80D2);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0xD000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80D3);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0xD000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80C8);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0200);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80CA);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0300);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80E2);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x2300);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80A9);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x0F00,
+                                              0x0A00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80AA);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x5000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80AB);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0xA000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x805A);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x2000);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8106);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x40CC);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x812C);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x40CC);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8096);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x7500);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x809C);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x6300);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x859E);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x1F00);
+
+
+        if (aspm && HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp))
+                rtl8127_enable_phy_aldps(tp);
+}
+
+static void
+rtl8127_hw_phy_config_8127a_1(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtl8127_tgphy_irq_mask_and_ack(tp);
+
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA442, BIT_11);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8415);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x9300);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A3);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0F00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81AE);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0F00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B9);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0xB900);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83B0);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83C5);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83DA);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x83EF);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0x0E00);
+
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xBF38,
+                                              0x01F0,
+                                              0x0160);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xBF3A,
+                                              0x001F,
+                                              0x0014);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xBF28, 0x6000);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xBF2C, 0xC000);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xBF28,
+                                              0x1FFF,
+                                              0x0187);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xBF2A,
+                                              0x003F,
+                                              0x0003);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8173);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8620);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8175);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8671);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x817C);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8187);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8192);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x819D);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A8);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA438, BIT_13);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B3);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA438, BIT_13);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81BE);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_13);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x817D);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0xA600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8188);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0xA600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8193);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0xA600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x819E);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0xA600);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81A9);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x1400);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B4);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x1400);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81BF);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0xA600);
+
+
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xAEAA, (BIT_5 | BIT_3));
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x201C);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F2);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x3117);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC6, 0x0000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAE20, 0xFFFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAECE, 0xFFFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAED2, 0xFFFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xAEC8, 0x0000);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xAED0, BIT_0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xADB8, 0x0150);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8197);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8231);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CB);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CD);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5700);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8233);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5700);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8199);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x5700);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x815A);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81F4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x828E);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0150);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B1);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x824B);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82E5);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84F7);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x2800);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xAEC2, BIT_12);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x81B3);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xAD00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x824D);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xAD00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82E7);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xAD00);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xAE4E,
+                                              0x000F,
+                                              0x0001);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x82CE);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xF000,
+                                              0x4000);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84AC);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84AE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84B0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF818);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x84B2);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x6000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFC);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x6008);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xF450);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8015);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB87E, BIT_9);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8016);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, BIT_11);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FE6);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x0800);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FE4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x2114);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8647);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xA7B1);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8649);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xBBCA);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x864B);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0xDC00);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8154);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xC000,
+                                              0x4000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8158);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0xC000);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826C);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x826E);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8872);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x0E00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8012);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_11);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8012);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA438, BIT_14);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xB576, BIT_0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x834A);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0700);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8217);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0x3F00,
+                                              0x2A00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81B1);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0B00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FED);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x4E00);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88AC);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x2300);
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xBF0C, 0x3800);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88DE);
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xB87E, 0xFF00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x80B4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x5195);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8370);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x8671);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8372);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86C8);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8401);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86C8);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8403);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x86DA);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8406);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8408);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840A);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840C);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x840E);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8410);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8412);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8414);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x8416);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0x1800,
+                                              0x1000);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x82BD);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x1F40);
+
+
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xBFB4,
+                                              0x07FF,
+                                              0x0328);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xBFB6, 0x3E14);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x81C4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x003B);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0086);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00B7);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00DB);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00FE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x00C3);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0078);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0047);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA438, 0x0023);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D7);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D9);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x01A0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FFA);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x002A);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FEE);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFDF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF0);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xFFFF);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF2);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0A4A);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF4);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xAA5A);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF6);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0x0A4A);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x8FF8);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87E, 0xAA5A);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xB87C, 0x88D5);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xB87E,
+                                              0xFF00,
+                                              0x0200);
+
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84BB);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x0A00);
+        rtl8127_mdio_direct_write_phy_ocp(tp, 0xA436, 0x84C0);
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              0xA438,
+                                              0xFF00,
+                                              0x1600);
+
+
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA430, BIT_1 | BIT_0);
+
+
+        if (aspm && HW_HAS_WRITE_PHY_MCU_RAM_CODE(tp))
+                rtl8127_enable_phy_aldps(tp);
+}
+
+static void
+rtl8127_hw_phy_config(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+
+        if (tp->resume_not_chg_speed)
+                return;
+
+        tp->phy_reset_enable(dev);
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+#ifndef ENABLE_USE_FIRMWARE_FILE
+        if (!tp->rtl_fw)
+                rtl8127_init_hw_phy_mcu(dev);
+#endif
+
+        switch (tp->mcfg) {
+        case CFG_METHOD_1:
+                rtl8127_hw_phy_config_8127a_tc_1(dev);
+                break;
+        case CFG_METHOD_2:
+                rtl8127_hw_phy_config_8127a_1(dev);
+                break;
+        default:
+                break;
+        }
+
+        //legacy force mode(Chap 22)
+        rtl8127_clear_eth_phy_ocp_bit(tp, 0xA5B4, BIT_15);
+
+        rtl8127_hw_fiber_phy_config(tp);
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+
+        if (tp->eee.eee_enabled)
+                rtl8127_enable_eee(tp);
+        else
+                rtl8127_disable_eee(tp);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+}
+
+static void
+rtl8127_up(struct net_device *dev)
+{
+        rtl8127_hw_init(dev);
+        rtl8127_hw_reset(dev);
+        rtl8127_powerup_pll(dev);
+        rtl8127_hw_ephy_config(dev);
+        rtl8127_hw_phy_config(dev);
+        rtl8127_hw_config(dev);
+}
+
+/*
+static inline void rtl8127_delete_esd_timer(struct net_device *dev, struct timer_list *timer)
+{
+        del_timer_sync(timer);
+}
+
+static inline void rtl8127_request_esd_timer(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct timer_list *timer = &tp->esd_timer;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+        setup_timer(timer, rtl8127_esd_timer, (unsigned long)dev);
+#else
+        timer_setup(timer, rtl8127_esd_timer, 0);
+#endif
+        mod_timer(timer, jiffies + RTL8127_ESD_TIMEOUT);
+}
+*/
+
+/*
+static inline void rtl8127_delete_link_timer(struct net_device *dev, struct timer_list *timer)
+{
+        del_timer_sync(timer);
+}
+
+static inline void rtl8127_request_link_timer(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct timer_list *timer = &tp->link_timer;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+        setup_timer(timer, rtl8127_link_timer, (unsigned long)dev);
+#else
+        timer_setup(timer, rtl8127_link_timer, 0);
+#endif
+        mod_timer(timer, jiffies + RTL8127_LINK_TIMEOUT);
+}
+*/
+
+#ifdef CONFIG_NET_POLL_CONTROLLER
+/*
+ * Polling 'interrupt' - used by things like netconsole to send skbs
+ * without having to re-enable interrupts. It's not called while
+ * the interrupt routine is executing.
+ */
+static void
+rtl8127_netpoll(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+        for (i = 0; i < tp->irq_nvecs; i++) {
+                struct r8127_irq *irq = &tp->irq_tbl[i];
+                struct r8127_napi *r8127napi = &tp->r8127napi[i];
+
+                disable_irq(irq->vector);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0)
+                irq->handler(irq->vector, r8127napi);
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+                irq->handler(irq->vector, r8127napi, NULL);
+#else
+                irq->handler(irq->vector, r8127napi);
+#endif
+
+                enable_irq(irq->vector);
+        }
+}
+#endif //CONFIG_NET_POLL_CONTROLLER
+
+static void
+rtl8127_setup_interrupt_mask(struct rtl8127_private *tp)
+{
+        int i;
+
+        if (tp->HwCurrIsrVer == 6) {
+                tp->intr_mask = ISRIMR_V6_LINKCHG | ISRIMR_V6_TOK_Q0;
+                if (tp->num_tx_rings > 1)
+                        tp->intr_mask |= ISRIMR_V6_TOK_Q1;
+                for (i = 0; i < tp->num_rx_rings; i++)
+                        tp->intr_mask |= ISRIMR_V6_ROK_Q0 << i;
+
+                if (tp->DASH)
+                        tp->intr_l2_mask |= ISRIMR_V6_L2_MISC_INTR;
+        } else if (tp->HwCurrIsrVer == 5) {
+                tp->intr_mask = ISRIMR_V5_LINKCHG | ISRIMR_V5_TOK_Q0;
+                if (tp->num_tx_rings > 1)
+                        tp->intr_mask |= ISRIMR_V5_TOK_Q1;
+                for (i = 0; i < tp->num_rx_rings; i++)
+                        tp->intr_mask |= ISRIMR_V5_ROK_Q0 << i;
+        } else if (tp->HwCurrIsrVer == 4) {
+                tp->intr_mask = ISRIMR_V4_LINKCHG;
+                for (i = 0; i < tp->num_rx_rings; i++)
+                        tp->intr_mask |= ISRIMR_V4_ROK_Q0 << i;
+        } else if (tp->HwCurrIsrVer == 3) {
+                tp->intr_mask = ISRIMR_V2_LINKCHG;
+                for (i = 0; i < max(tp->num_tx_rings, tp->num_rx_rings); i++)
+                        tp->intr_mask |= ISRIMR_V2_ROK_Q0 << i;
+        } else if (tp->HwCurrIsrVer == 2) {
+                tp->intr_mask = ISRIMR_V2_LINKCHG | ISRIMR_TOK_Q0;
+                if (tp->num_tx_rings > 1)
+                        tp->intr_mask |= ISRIMR_TOK_Q1;
+
+                for (i = 0; i < tp->num_rx_rings; i++)
+                        tp->intr_mask |= ISRIMR_V2_ROK_Q0 << i;
+        } else {
+                tp->intr_mask = LinkChg | RxDescUnavail | TxOK | RxOK | SWInt;
+                tp->timer_intr_mask = LinkChg | PCSTimeout;
+
+#ifdef ENABLE_DASH_SUPPORT
+                if (tp->DASH) {
+                        if (HW_DASH_SUPPORT_IPC2(tp)) {
+                                tp->timer_intr_mask |= ISRIMR_DASH_INTR_EN;
+                                tp->intr_mask |= ISRIMR_DASH_INTR_EN;
+                        }
+                }
+#endif
+        }
+}
+
+static void
+rtl8127_setup_mqs_reg(struct rtl8127_private *tp)
+{
+        u16 hw_clo_ptr0_reg, sw_tail_ptr0_reg;
+        u16 reg_len;
+        int i;
+
+        //tx
+        tp->tx_ring[0].tdsar_reg = TxDescStartAddrLow;
+        for (i = 1; i < tp->HwSuppNumTxQueues; i++)
+                tp->tx_ring[i].tdsar_reg = (u16)(TNPDS_Q1_LOW_8125 + (i - 1) * 8);
+
+        switch (tp->HwSuppTxNoCloseVer) {
+        case 4:
+        case 5:
+                hw_clo_ptr0_reg = HW_CLO_PTR0_8126;
+                sw_tail_ptr0_reg = SW_TAIL_PTR0_8126;
+                reg_len = 4;
+                break;
+        case 6:
+                hw_clo_ptr0_reg = HW_CLO_PTR0_8125BP;
+                sw_tail_ptr0_reg = SW_TAIL_PTR0_8125BP;
+                reg_len = 8;
+                break;
+        default:
+                hw_clo_ptr0_reg = HW_CLO_PTR0_8125;
+                sw_tail_ptr0_reg = SW_TAIL_PTR0_8125;
+                reg_len = 4;
+                break;
+        }
+
+        for (i = 0; i < tp->HwSuppNumTxQueues; i++) {
+                tp->tx_ring[i].hw_clo_ptr_reg = (u16)(hw_clo_ptr0_reg + i * reg_len);
+                tp->tx_ring[i].sw_tail_ptr_reg = (u16)(sw_tail_ptr0_reg + i * reg_len);
+        }
+
+        //rx
+        tp->rx_ring[0].rdsar_reg = RxDescAddrLow;
+        for (i = 1; i < tp->HwSuppNumRxQueues; i++)
+                tp->rx_ring[i].rdsar_reg = (u16)(RDSAR_Q1_LOW_8125 + (i - 1) * 8);
+
+        tp->isr_reg[0] = ISR0_8125;
+        for (i = 1; i < tp->hw_supp_irq_nvecs; i++)
+                tp->isr_reg[i] = (u16)(ISR1_8125 + (i - 1) * 4);
+
+        tp->imr_reg[0] = IMR0_8125;
+        for (i = 1; i < tp->hw_supp_irq_nvecs; i++)
+                tp->imr_reg[i] = (u16)(IMR1_8125 + (i - 1) * 4);
+}
+
+static void
+rtl8127_init_software_variable(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct pci_dev *pdev = tp->pci_dev;
+
+#ifdef ENABLE_LIB_SUPPORT
+        tp->ring_lib_enabled = 1;
+#endif
+
+        switch (tp->mcfg) {
+        case CFG_METHOD_2: {
+                u8 tmp = (u8)rtl8127_mac_ocp_read(tp, 0xD006);
+                if (tmp == 0x03)
+                        tp->HwSuppDashVer = 4;
+        }
+        break;
+        }
+
+        switch (tp->mcfg) {
+        case CFG_METHOD_2:
+                if (HW_DASH_SUPPORT_DASH(tp))
+                        tp->HwSuppOcpChannelVer = 2;
+                break;
+        }
+        tp->AllowAccessDashOcp = rtl8127_is_allow_access_dash_ocp(tp);
+
+        tp->HwPkgDet = rtl8127_mac_ocp_read(tp, 0xDC00);
+        tp->HwPkgDet = (tp->HwPkgDet >> 3) & 0x07;
+
+        tp->HwSuppNowIsOobVer = 1;
+
+        tp->HwPcieSNOffset = 0x168;
+
+#ifdef ENABLE_REALWOW_SUPPORT
+        rtl8127_get_realwow_hw_version(dev);
+#endif //ENABLE_REALWOW_SUPPORT
+
+        tp->DASH = rtl8127_check_dash(tp);
+
+        if (tp->DASH) {
+                eee_enable = 0;
+
+                tp->SizeOfSendToFwBuffer = SEND_TO_FW_BUF_SIZE;
+                tp->SizeOfRecvFromFwBuffer = RECV_FROM_FW_BUF_SIZE;
+
+                tp->DashFirmwareVersion = rtl8127_get_dash_fw_ver(tp);
+        }
+
+        if (aspm) {
+                tp->org_pci_offset_99 = rtl8127_csi_fun0_read_byte(tp, 0x99);
+                tp->org_pci_offset_99 &= ~(BIT_5|BIT_6);
+
+                tp->org_pci_offset_180 = rtl8127_csi_fun0_read_byte(tp, 0x254);
+        }
+
+        pci_read_config_byte(pdev, 0x80, &tp->org_pci_offset_80);
+        pci_read_config_byte(pdev, 0x81, &tp->org_pci_offset_81);
+
+        tp->use_timer_interrupt = TRUE;
+
+        tp->HwSuppMaxPhyLinkSpeed = 10000;
+
+        if (timer_count == 0 || tp->mcfg == CFG_METHOD_DEFAULT)
+                tp->use_timer_interrupt = FALSE;
+
+        tp->ShortPacketSwChecksum = TRUE;
+        tp->UseSwPaddingShortPkt = TRUE;
+
+        rtl8127_check_fiber_mode_support(tp);
+        if (HW_FIBER_MODE_ENABLED(tp)) {
+                eee_enable = 0;
+                eee_giga_lite = 0;
+        }
+
+        tp->HwSuppMagicPktVer = WAKEUP_MAGIC_PACKET_V3;
+
+        tp->HwSuppLinkChgWakeUpVer = 3;
+
+        tp->HwSuppD0SpeedUpVer = 2;
+
+        tp->HwSuppCheckPhyDisableModeVer = 3;
+
+        tp->HwSuppTxNoCloseVer = 6;
+
+        switch (tp->HwSuppTxNoCloseVer) {
+        case 5:
+        case 6:
+                tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V4;
+                break;
+        case 4:
+                tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V3;
+                break;
+        case 3:
+                tp->MaxTxDescPtrMask = MAX_TX_NO_CLOSE_DESC_PTR_MASK_V2;
+                break;
+        default:
+                tx_no_close_enable = 0;
+                break;
+        }
+
+        if (tp->HwSuppTxNoCloseVer > 0 && tx_no_close_enable == 1)
+                tp->EnableTxNoClose = TRUE;
+
+        switch (tp->mcfg) {
+        case CFG_METHOD_1:
+                tp->sw_ram_code_ver = NIC_RAMCODE_VERSION_CFG_METHOD_1;
+                break;
+        case CFG_METHOD_2:
+                tp->sw_ram_code_ver = NIC_RAMCODE_VERSION_CFG_METHOD_2;
+                break;
+        default:
+                break;
+        }
+
+        if (tp->HwIcVerUnknown) {
+                tp->NotWrRamCodeToMicroP = TRUE;
+                tp->NotWrMcuPatchCode = TRUE;
+        }
+
+        tp->HwSuppMacMcuVer = 2;
+
+        tp->MacMcuPageSize = RTL8127_MAC_MCU_PAGE_SIZE;
+
+        tp->HwSuppNumTxQueues = 2;
+        tp->HwSuppNumRxQueues = 4;
+
+        //init interrupt
+        tp->HwSuppIsrVer = 6;
+
+        tp->HwCurrIsrVer = tp->HwSuppIsrVer;
+        if (tp->HwCurrIsrVer > 1) {
+                if (!(tp->features & RTL_FEATURE_MSIX) ||
+                    tp->irq_nvecs < tp->min_irq_nvecs)
+                        tp->HwCurrIsrVer = 1;
+        }
+
+        tp->num_tx_rings = 1;
+#ifdef ENABLE_MULTIPLE_TX_QUEUE
+#ifndef ENABLE_LIB_SUPPORT
+        tp->num_tx_rings = tp->HwSuppNumTxQueues;
+#endif
+#endif
+        if (tp->HwCurrIsrVer < 2 ||
+            (tp->HwCurrIsrVer == 2 && tp->irq_nvecs < 19))
+                tp->num_tx_rings = 1;
+
+        //RSS
+        tp->HwSuppRssVer = 5;
+        tp->HwSuppIndirTblEntries = 128;
+
+        tp->num_rx_rings = 1;
+#ifdef ENABLE_RSS_SUPPORT
+#ifdef ENABLE_LIB_SUPPORT
+        if (tp->HwSuppRssVer > 0)
+                tp->EnableRss = 1;
+#else
+        if (tp->HwSuppRssVer > 0 && tp->HwCurrIsrVer > 1) {
+                u8 rss_queue_num = netif_get_num_default_rss_queues();
+                tp->num_rx_rings = (tp->HwSuppNumRxQueues > rss_queue_num)?
+                                   rss_queue_num : tp->HwSuppNumRxQueues;
+
+                if (!(tp->num_rx_rings >= 2 && tp->irq_nvecs >= tp->num_rx_rings))
+                        tp->num_rx_rings = 1;
+
+                if (tp->num_rx_rings >= 2)
+                        tp->EnableRss = 1;
+        }
+#endif
+#endif
+
+        //interrupt mask
+        rtl8127_setup_interrupt_mask(tp);
+
+        rtl8127_setup_mqs_reg(tp);
+
+        rtl8127_set_ring_size(tp, NUM_RX_DESC, NUM_TX_DESC);
+
+        tp->HwSuppPtpVer = 2;
+#ifdef ENABLE_PTP_SUPPORT
+        if (tp->HwSuppPtpVer > 0)
+                tp->EnablePtp = 1;
+#endif
+
+        tp->HwSuppIntMitiVer = 6;
+
+        tp->HwSuppTcamVer = 2;
+
+        tp->TcamNotValidReg = TCAM_NOTVALID_ADDR_V2;
+        tp->TcamValidReg = TCAM_VALID_ADDR_V2;
+        tp->TcamMaAddrcOffset = TCAM_MAC_ADDR_V2;
+        tp->TcamVlanTagOffset = TCAM_VLAN_TAG_V2;
+
+        tp->HwSuppExtendTallyCounterVer = 1;
+
+        timer_count_v2 = (timer_count / 0x200);
+
+        tp->HwSuppRxDescType = RX_DESC_RING_TYPE_4;
+
+        tp->InitRxDescType = RX_DESC_RING_TYPE_1;
+        tp->RxDescLength = RX_DESC_LEN_TYPE_1;
+        switch (tp->HwSuppRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                if (tp->EnableRss) {
+                        tp->InitRxDescType = RX_DESC_RING_TYPE_3;
+                        tp->RxDescLength = RX_DESC_LEN_TYPE_3;
+                }
+                break;
+        case RX_DESC_RING_TYPE_4:
+                if (tp->EnableRss) {
+                        tp->InitRxDescType = RX_DESC_RING_TYPE_4;
+                        tp->RxDescLength = RX_DESC_LEN_TYPE_4;
+                }
+                break;
+        }
+
+        tp->rtl8127_rx_config = rtl_chip_info[tp->chipset].RCR_Cfg;
+        if (tp->InitRxDescType == RX_DESC_RING_TYPE_3)
+                tp->rtl8127_rx_config |= EnableRxDescV3;
+        else if (tp->InitRxDescType == RX_DESC_RING_TYPE_4)
+                tp->rtl8127_rx_config &= ~EnableRxDescV4_1;
+
+        tp->NicCustLedValue = RTL_R16(tp, CustomLED);
+
+        tp->wol_opts = rtl8127_get_hw_wol(tp);
+        tp->wol_enabled = (tp->wol_opts) ? WOL_ENABLED : WOL_DISABLED;
+
+        rtl8127_set_link_option(tp, autoneg_mode, speed_mode, duplex_mode,
+                                rtl8127_fc_full);
+
+        tp->max_jumbo_frame_size = rtl_chip_info[tp->chipset].jumbo_frame_sz;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+        /* MTU range: 60 - hw-specific max */
+        dev->min_mtu = ETH_MIN_MTU;
+        dev->max_mtu = tp->max_jumbo_frame_size;
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+
+        if (tp->mcfg != CFG_METHOD_DEFAULT) {
+                struct ethtool_keee *eee = &tp->eee;
+
+                eee->eee_enabled = eee_enable;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0)
+                eee->supported  = SUPPORTED_100baseT_Full |
+                                  SUPPORTED_1000baseT_Full |
+                                  SUPPORTED_2500baseX_Full;
+                eee->advertised = mmd_eee_adv_to_ethtool_adv_t(MDIO_EEE_1000T | MDIO_EEE_100TX);
+                eee->advertised |= SUPPORTED_2500baseX_Full;
+#else
+                linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, eee->supported);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, eee->supported);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, eee->supported);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, eee->supported);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, eee->supported);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, eee->advertised);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, eee->advertised);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, eee->advertised);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, eee->advertised);
+                linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, eee->advertised);
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,9,0) */
+                eee->tx_lpi_enabled = eee_enable;
+                eee->tx_lpi_timer = dev->mtu + ETH_HLEN + 0x20;
+        }
+
+#ifdef ENABLE_RSS_SUPPORT
+        if (tp->EnableRss)
+                rtl8127_init_rss(tp);
+#endif
+}
+
+static void
+rtl8127_release_board(struct pci_dev *pdev,
+                      struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        void __iomem *ioaddr = tp->mmio_addr;
+
+        rtl8127_rar_set(tp, tp->org_mac_addr);
+        tp->wol_enabled = WOL_DISABLED;
+
+        if (!tp->DASH)
+                rtl8127_phy_power_down(dev);
+
+        iounmap(ioaddr);
+        pci_release_regions(pdev);
+        pci_clear_mwi(pdev);
+        pci_disable_device(pdev);
+        free_netdev(dev);
+}
+
+static void
+rtl8127_hw_address_set(struct net_device *dev, u8 mac_addr[MAC_ADDR_LEN])
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+        eth_hw_addr_set(dev, mac_addr);
+#else
+        memcpy(dev->dev_addr, mac_addr, MAC_ADDR_LEN);
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,17,0)
+}
+
+static int
+rtl8127_get_mac_address(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+        u8 mac_addr[MAC_ADDR_LEN];
+
+        for (i = 0; i < MAC_ADDR_LEN; i++)
+                mac_addr[i] = RTL_R8(tp, MAC0 + i);
+
+        *(u32*)&mac_addr[0] = RTL_R32(tp, BACKUP_ADDR0_8125);
+        *(u16*)&mac_addr[4] = RTL_R16(tp, BACKUP_ADDR1_8125);
+
+        if (!is_valid_ether_addr(mac_addr)) {
+                netif_err(tp, probe, dev, "Invalid ether addr %pM\n",
+                          mac_addr);
+                eth_random_addr(mac_addr);
+                dev->addr_assign_type = NET_ADDR_RANDOM;
+                netif_info(tp, probe, dev, "Random ether addr %pM\n",
+                           mac_addr);
+                tp->random_mac = 1;
+        }
+
+        rtl8127_hw_address_set(dev, mac_addr);
+        rtl8127_rar_set(tp, mac_addr);
+
+        /* keep the original MAC address */
+        memcpy(tp->org_mac_addr, dev->dev_addr, MAC_ADDR_LEN);
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13)
+        memcpy(dev->perm_addr, dev->dev_addr, MAC_ADDR_LEN);
+#endif
+        return 0;
+}
+
+/**
+ * rtl8127_set_mac_address - Change the Ethernet Address of the NIC
+ * @dev: network interface device structure
+ * @p:   pointer to an address structure
+ *
+ * Return 0 on success, negative on failure
+ **/
+static int
+rtl8127_set_mac_address(struct net_device *dev,
+                        void *p)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct sockaddr *addr = p;
+
+        if (!is_valid_ether_addr(addr->sa_data))
+                return -EADDRNOTAVAIL;
+
+        rtl8127_hw_address_set(dev, addr->sa_data);
+
+        rtl8127_rar_set(tp, dev->dev_addr);
+
+        return 0;
+}
+
+/******************************************************************************
+ * rtl8127_rar_set - Puts an ethernet address into a receive address register.
+ *
+ * tp - The private data structure for driver
+ * addr - Address to put into receive address register
+ *****************************************************************************/
+void
+rtl8127_rar_set(struct rtl8127_private *tp,
+                const u8 *addr)
+{
+        uint32_t rar_low = 0;
+        uint32_t rar_high = 0;
+
+        rar_low = ((uint32_t) addr[0] |
+                   ((uint32_t) addr[1] << 8) |
+                   ((uint32_t) addr[2] << 16) |
+                   ((uint32_t) addr[3] << 24));
+
+        rar_high = ((uint32_t) addr[4] |
+                    ((uint32_t) addr[5] << 8));
+
+        rtl8127_enable_cfg9346_write(tp);
+        RTL_W32(tp, MAC0, rar_low);
+        RTL_W32(tp, MAC4, rar_high);
+
+        rtl8127_disable_cfg9346_write(tp);
+}
+
+#ifdef ETHTOOL_OPS_COMPAT
+static int ethtool_get_settings(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_cmd cmd = { ETHTOOL_GSET };
+        int err;
+
+        if (!ethtool_ops->get_settings)
+                return -EOPNOTSUPP;
+
+        err = ethtool_ops->get_settings(dev, &cmd);
+        if (err < 0)
+                return err;
+
+        if (copy_to_user(useraddr, &cmd, sizeof(cmd)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_settings(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_cmd cmd;
+
+        if (!ethtool_ops->set_settings)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
+                return -EFAULT;
+
+        return ethtool_ops->set_settings(dev, &cmd);
+}
+
+static int ethtool_get_drvinfo(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_drvinfo info;
+        struct ethtool_ops *ops = ethtool_ops;
+
+        if (!ops->get_drvinfo)
+                return -EOPNOTSUPP;
+
+        memset(&info, 0, sizeof(info));
+        info.cmd = ETHTOOL_GDRVINFO;
+        ops->get_drvinfo(dev, &info);
+
+        if (ops->self_test_count)
+                info.testinfo_len = ops->self_test_count(dev);
+        if (ops->get_stats_count)
+                info.n_stats = ops->get_stats_count(dev);
+        if (ops->get_regs_len)
+                info.regdump_len = ops->get_regs_len(dev);
+        if (ops->get_eeprom_len)
+                info.eedump_len = ops->get_eeprom_len(dev);
+
+        if (copy_to_user(useraddr, &info, sizeof(info)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_get_regs(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_regs regs;
+        struct ethtool_ops *ops = ethtool_ops;
+        void *regbuf;
+        int reglen, ret;
+
+        if (!ops->get_regs || !ops->get_regs_len)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&regs, useraddr, sizeof(regs)))
+                return -EFAULT;
+
+        reglen = ops->get_regs_len(dev);
+        if (regs.len > reglen)
+                regs.len = reglen;
+
+        regbuf = kmalloc(reglen, GFP_USER);
+        if (!regbuf)
+                return -ENOMEM;
+
+        ops->get_regs(dev, &regs, regbuf);
+
+        ret = -EFAULT;
+        if (copy_to_user(useraddr, &regs, sizeof(regs)))
+                goto out;
+        useraddr += offsetof(struct ethtool_regs, data);
+        if (copy_to_user(useraddr, regbuf, reglen))
+                goto out;
+        ret = 0;
+
+out:
+        kfree(regbuf);
+        return ret;
+}
+
+static int ethtool_get_wol(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_wolinfo wol = { ETHTOOL_GWOL };
+
+        if (!ethtool_ops->get_wol)
+                return -EOPNOTSUPP;
+
+        ethtool_ops->get_wol(dev, &wol);
+
+        if (copy_to_user(useraddr, &wol, sizeof(wol)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_wol(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_wolinfo wol;
+
+        if (!ethtool_ops->set_wol)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&wol, useraddr, sizeof(wol)))
+                return -EFAULT;
+
+        return ethtool_ops->set_wol(dev, &wol);
+}
+
+static int ethtool_get_msglevel(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata = { ETHTOOL_GMSGLVL };
+
+        if (!ethtool_ops->get_msglevel)
+                return -EOPNOTSUPP;
+
+        edata.data = ethtool_ops->get_msglevel(dev);
+
+        if (copy_to_user(useraddr, &edata, sizeof(edata)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_msglevel(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata;
+
+        if (!ethtool_ops->set_msglevel)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&edata, useraddr, sizeof(edata)))
+                return -EFAULT;
+
+        ethtool_ops->set_msglevel(dev, edata.data);
+        return 0;
+}
+
+static int ethtool_nway_reset(struct net_device *dev)
+{
+        if (!ethtool_ops->nway_reset)
+                return -EOPNOTSUPP;
+
+        return ethtool_ops->nway_reset(dev);
+}
+
+static int ethtool_get_link(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_value edata = { ETHTOOL_GLINK };
+
+        if (!ethtool_ops->get_link)
+                return -EOPNOTSUPP;
+
+        edata.data = ethtool_ops->get_link(dev);
+
+        if (copy_to_user(useraddr, &edata, sizeof(edata)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_get_eeprom(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_eeprom eeprom;
+        struct ethtool_ops *ops = ethtool_ops;
+        u8 *data;
+        int ret;
+
+        if (!ops->get_eeprom || !ops->get_eeprom_len)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&eeprom, useraddr, sizeof(eeprom)))
+                return -EFAULT;
+
+        /* Check for wrap and zero */
+        if (eeprom.offset + eeprom.len <= eeprom.offset)
+                return -EINVAL;
+
+        /* Check for exceeding total eeprom len */
+        if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev))
+                return -EINVAL;
+
+        data = kmalloc(eeprom.len, GFP_USER);
+        if (!data)
+                return -ENOMEM;
+
+        ret = -EFAULT;
+        if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len))
+                goto out;
+
+        ret = ops->get_eeprom(dev, &eeprom, data);
+        if (ret)
+                goto out;
+
+        ret = -EFAULT;
+        if (copy_to_user(useraddr, &eeprom, sizeof(eeprom)))
+                goto out;
+        if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len))
+                goto out;
+        ret = 0;
+
+out:
+        kfree(data);
+        return ret;
+}
+
+static int ethtool_set_eeprom(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_eeprom eeprom;
+        struct ethtool_ops *ops = ethtool_ops;
+        u8 *data;
+        int ret;
+
+        if (!ops->set_eeprom || !ops->get_eeprom_len)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&eeprom, useraddr, sizeof(eeprom)))
+                return -EFAULT;
+
+        /* Check for wrap and zero */
+        if (eeprom.offset + eeprom.len <= eeprom.offset)
+                return -EINVAL;
+
+        /* Check for exceeding total eeprom len */
+        if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev))
+                return -EINVAL;
+
+        data = kmalloc(eeprom.len, GFP_USER);
+        if (!data)
+                return -ENOMEM;
+
+        ret = -EFAULT;
+        if (copy_from_user(data, useraddr + sizeof(eeprom), eeprom.len))
+                goto out;
+
+        ret = ops->set_eeprom(dev, &eeprom, data);
+        if (ret)
+                goto out;
+
+        if (copy_to_user(useraddr + sizeof(eeprom), data, eeprom.len))
+                ret = -EFAULT;
+
+out:
+        kfree(data);
+        return ret;
+}
+
+static int ethtool_get_coalesce(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_coalesce coalesce = { ETHTOOL_GCOALESCE };
+
+        if (!ethtool_ops->get_coalesce)
+                return -EOPNOTSUPP;
+
+        ethtool_ops->get_coalesce(dev, &coalesce);
+
+        if (copy_to_user(useraddr, &coalesce, sizeof(coalesce)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_coalesce(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_coalesce coalesce;
+
+        if (!ethtool_ops->get_coalesce)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&coalesce, useraddr, sizeof(coalesce)))
+                return -EFAULT;
+
+        return ethtool_ops->set_coalesce(dev, &coalesce);
+}
+
+static int ethtool_get_ringparam(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_ringparam ringparam = { ETHTOOL_GRINGPARAM };
+
+        if (!ethtool_ops->get_ringparam)
+                return -EOPNOTSUPP;
+
+        ethtool_ops->get_ringparam(dev, &ringparam);
+
+        if (copy_to_user(useraddr, &ringparam, sizeof(ringparam)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_ringparam(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_ringparam ringparam;
+
+        if (!ethtool_ops->get_ringparam)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&ringparam, useraddr, sizeof(ringparam)))
+                return -EFAULT;
+
+        return ethtool_ops->set_ringparam(dev, &ringparam);
+}
+
+static int ethtool_get_pauseparam(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_pauseparam pauseparam = { ETHTOOL_GPAUSEPARAM };
+
+        if (!ethtool_ops->get_pauseparam)
+                return -EOPNOTSUPP;
+
+        ethtool_ops->get_pauseparam(dev, &pauseparam);
+
+        if (copy_to_user(useraddr, &pauseparam, sizeof(pauseparam)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_pauseparam(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_pauseparam pauseparam;
+
+        if (!ethtool_ops->get_pauseparam)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&pauseparam, useraddr, sizeof(pauseparam)))
+                return -EFAULT;
+
+        return ethtool_ops->set_pauseparam(dev, &pauseparam);
+}
+
+static int ethtool_get_rx_csum(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata = { ETHTOOL_GRXCSUM };
+
+        if (!ethtool_ops->get_rx_csum)
+                return -EOPNOTSUPP;
+
+        edata.data = ethtool_ops->get_rx_csum(dev);
+
+        if (copy_to_user(useraddr, &edata, sizeof(edata)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_rx_csum(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata;
+
+        if (!ethtool_ops->set_rx_csum)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&edata, useraddr, sizeof(edata)))
+                return -EFAULT;
+
+        ethtool_ops->set_rx_csum(dev, edata.data);
+        return 0;
+}
+
+static int ethtool_get_tx_csum(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata = { ETHTOOL_GTXCSUM };
+
+        if (!ethtool_ops->get_tx_csum)
+                return -EOPNOTSUPP;
+
+        edata.data = ethtool_ops->get_tx_csum(dev);
+
+        if (copy_to_user(useraddr, &edata, sizeof(edata)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_tx_csum(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata;
+
+        if (!ethtool_ops->set_tx_csum)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&edata, useraddr, sizeof(edata)))
+                return -EFAULT;
+
+        return ethtool_ops->set_tx_csum(dev, edata.data);
+}
+
+static int ethtool_get_sg(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata = { ETHTOOL_GSG };
+
+        if (!ethtool_ops->get_sg)
+                return -EOPNOTSUPP;
+
+        edata.data = ethtool_ops->get_sg(dev);
+
+        if (copy_to_user(useraddr, &edata, sizeof(edata)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_sg(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata;
+
+        if (!ethtool_ops->set_sg)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&edata, useraddr, sizeof(edata)))
+                return -EFAULT;
+
+        return ethtool_ops->set_sg(dev, edata.data);
+}
+
+static int ethtool_get_tso(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata = { ETHTOOL_GTSO };
+
+        if (!ethtool_ops->get_tso)
+                return -EOPNOTSUPP;
+
+        edata.data = ethtool_ops->get_tso(dev);
+
+        if (copy_to_user(useraddr, &edata, sizeof(edata)))
+                return -EFAULT;
+        return 0;
+}
+
+static int ethtool_set_tso(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_value edata;
+
+        if (!ethtool_ops->set_tso)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&edata, useraddr, sizeof(edata)))
+                return -EFAULT;
+
+        return ethtool_ops->set_tso(dev, edata.data);
+}
+
+static int ethtool_self_test(struct net_device *dev, char *useraddr)
+{
+        struct ethtool_test test;
+        struct ethtool_ops *ops = ethtool_ops;
+        u64 *data;
+        int ret;
+
+        if (!ops->self_test || !ops->self_test_count)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&test, useraddr, sizeof(test)))
+                return -EFAULT;
+
+        test.len = ops->self_test_count(dev);
+        data = kmalloc(test.len * sizeof(u64), GFP_USER);
+        if (!data)
+                return -ENOMEM;
+
+        ops->self_test(dev, &test, data);
+
+        ret = -EFAULT;
+        if (copy_to_user(useraddr, &test, sizeof(test)))
+                goto out;
+        useraddr += sizeof(test);
+        if (copy_to_user(useraddr, data, test.len * sizeof(u64)))
+                goto out;
+        ret = 0;
+
+out:
+        kfree(data);
+        return ret;
+}
+
+static int ethtool_get_strings(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_gstrings gstrings;
+        struct ethtool_ops *ops = ethtool_ops;
+        u8 *data;
+        int ret;
+
+        if (!ops->get_strings)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&gstrings, useraddr, sizeof(gstrings)))
+                return -EFAULT;
+
+        switch (gstrings.string_set) {
+        case ETH_SS_TEST:
+                if (!ops->self_test_count)
+                        return -EOPNOTSUPP;
+                gstrings.len = ops->self_test_count(dev);
+                break;
+        case ETH_SS_STATS:
+                if (!ops->get_stats_count)
+                        return -EOPNOTSUPP;
+                gstrings.len = ops->get_stats_count(dev);
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        data = kmalloc(gstrings.len * ETH_GSTRING_LEN, GFP_USER);
+        if (!data)
+                return -ENOMEM;
+
+        ops->get_strings(dev, gstrings.string_set, data);
+
+        ret = -EFAULT;
+        if (copy_to_user(useraddr, &gstrings, sizeof(gstrings)))
+                goto out;
+        useraddr += sizeof(gstrings);
+        if (copy_to_user(useraddr, data, gstrings.len * ETH_GSTRING_LEN))
+                goto out;
+        ret = 0;
+
+out:
+        kfree(data);
+        return ret;
+}
+
+static int ethtool_phys_id(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_value id;
+
+        if (!ethtool_ops->phys_id)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&id, useraddr, sizeof(id)))
+                return -EFAULT;
+
+        return ethtool_ops->phys_id(dev, id.data);
+}
+
+static int ethtool_get_stats(struct net_device *dev, void *useraddr)
+{
+        struct ethtool_stats stats;
+        struct ethtool_ops *ops = ethtool_ops;
+        u64 *data;
+        int ret;
+
+        if (!ops->get_ethtool_stats || !ops->get_stats_count)
+                return -EOPNOTSUPP;
+
+        if (copy_from_user(&stats, useraddr, sizeof(stats)))
+                return -EFAULT;
+
+        stats.n_stats = ops->get_stats_count(dev);
+        data = kmalloc(stats.n_stats * sizeof(u64), GFP_USER);
+        if (!data)
+                return -ENOMEM;
+
+        ops->get_ethtool_stats(dev, &stats, data);
+
+        ret = -EFAULT;
+        if (copy_to_user(useraddr, &stats, sizeof(stats)))
+                goto out;
+        useraddr += sizeof(stats);
+        if (copy_to_user(useraddr, data, stats.n_stats * sizeof(u64)))
+                goto out;
+        ret = 0;
+
+out:
+        kfree(data);
+        return ret;
+}
+
+static int ethtool_ioctl(struct ifreq *ifr)
+{
+        struct net_device *dev = __dev_get_by_name(ifr->ifr_name);
+        void *useraddr = (void *) ifr->ifr_data;
+        u32 ethcmd;
+
+        /*
+         * XXX: This can be pushed down into the ethtool_* handlers that
+         * need it.  Keep existing behaviour for the moment.
+         */
+        if (!capable(CAP_NET_ADMIN))
+                return -EPERM;
+
+        if (!dev || !netif_device_present(dev))
+                return -ENODEV;
+
+        if (copy_from_user(&ethcmd, useraddr, sizeof (ethcmd)))
+                return -EFAULT;
+
+        switch (ethcmd) {
+        case ETHTOOL_GSET:
+                return ethtool_get_settings(dev, useraddr);
+        case ETHTOOL_SSET:
+                return ethtool_set_settings(dev, useraddr);
+        case ETHTOOL_GDRVINFO:
+                return ethtool_get_drvinfo(dev, useraddr);
+        case ETHTOOL_GREGS:
+                return ethtool_get_regs(dev, useraddr);
+        case ETHTOOL_GWOL:
+                return ethtool_get_wol(dev, useraddr);
+        case ETHTOOL_SWOL:
+                return ethtool_set_wol(dev, useraddr);
+        case ETHTOOL_GMSGLVL:
+                return ethtool_get_msglevel(dev, useraddr);
+        case ETHTOOL_SMSGLVL:
+                return ethtool_set_msglevel(dev, useraddr);
+        case ETHTOOL_NWAY_RST:
+                return ethtool_nway_reset(dev);
+        case ETHTOOL_GLINK:
+                return ethtool_get_link(dev, useraddr);
+        case ETHTOOL_GEEPROM:
+                return ethtool_get_eeprom(dev, useraddr);
+        case ETHTOOL_SEEPROM:
+                return ethtool_set_eeprom(dev, useraddr);
+        case ETHTOOL_GCOALESCE:
+                return ethtool_get_coalesce(dev, useraddr);
+        case ETHTOOL_SCOALESCE:
+                return ethtool_set_coalesce(dev, useraddr);
+        case ETHTOOL_GRINGPARAM:
+                return ethtool_get_ringparam(dev, useraddr);
+        case ETHTOOL_SRINGPARAM:
+                return ethtool_set_ringparam(dev, useraddr);
+        case ETHTOOL_GPAUSEPARAM:
+                return ethtool_get_pauseparam(dev, useraddr);
+        case ETHTOOL_SPAUSEPARAM:
+                return ethtool_set_pauseparam(dev, useraddr);
+        case ETHTOOL_GRXCSUM:
+                return ethtool_get_rx_csum(dev, useraddr);
+        case ETHTOOL_SRXCSUM:
+                return ethtool_set_rx_csum(dev, useraddr);
+        case ETHTOOL_GTXCSUM:
+                return ethtool_get_tx_csum(dev, useraddr);
+        case ETHTOOL_STXCSUM:
+                return ethtool_set_tx_csum(dev, useraddr);
+        case ETHTOOL_GSG:
+                return ethtool_get_sg(dev, useraddr);
+        case ETHTOOL_SSG:
+                return ethtool_set_sg(dev, useraddr);
+        case ETHTOOL_GTSO:
+                return ethtool_get_tso(dev, useraddr);
+        case ETHTOOL_STSO:
+                return ethtool_set_tso(dev, useraddr);
+        case ETHTOOL_TEST:
+                return ethtool_self_test(dev, useraddr);
+        case ETHTOOL_GSTRINGS:
+                return ethtool_get_strings(dev, useraddr);
+        case ETHTOOL_PHYS_ID:
+                return ethtool_phys_id(dev, useraddr);
+        case ETHTOOL_GSTATS:
+                return ethtool_get_stats(dev, useraddr);
+        default:
+                return -EOPNOTSUPP;
+        }
+
+        return -EOPNOTSUPP;
+}
+#endif //ETHTOOL_OPS_COMPAT
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0)
+static int rtl8127_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
+                                  void __user *data, int cmd)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ret = 0;
+
+        switch (cmd) {
+#ifdef ENABLE_DASH_SUPPORT
+        case SIOCDEVPRIVATE_RTLDASH:
+                if (!netif_running(dev)) {
+                        ret = -ENODEV;
+                        break;
+                }
+                if (!capable(CAP_NET_ADMIN)) {
+                        ret = -EPERM;
+                        break;
+                }
+
+                ret = rtl8127_dash_ioctl(dev, ifr);
+                break;
+#endif
+
+#ifdef ENABLE_REALWOW_SUPPORT
+        case SIOCDEVPRIVATE_RTLREALWOW:
+                if (!netif_running(dev)) {
+                        ret = -ENODEV;
+                        break;
+                }
+
+                ret = rtl8127_realwow_ioctl(dev, ifr);
+                break;
+#endif
+
+        case SIOCRTLTOOL:
+                if (!capable(CAP_NET_ADMIN)) {
+                        ret = -EPERM;
+                        break;
+                }
+
+                ret = rtl8127_tool_ioctl(tp, ifr);
+                break;
+
+        default:
+                ret = -EOPNOTSUPP;
+        }
+
+        return ret;
+}
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,15,0)
+
+static int
+rtl8127_do_ioctl(struct net_device *dev,
+                 struct ifreq *ifr,
+                 int cmd)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct mii_ioctl_data *data = if_mii(ifr);
+        int ret = 0;
+
+        switch (cmd) {
+        case SIOCGMIIPHY:
+                data->phy_id = 32; /* Internal PHY */
+                break;
+
+        case SIOCGMIIREG:
+                rtl8127_mdio_write(tp, 0x1F, 0x0000);
+                data->val_out = rtl8127_mdio_read(tp, data->reg_num);
+                break;
+
+        case SIOCSMIIREG:
+                if (!capable(CAP_NET_ADMIN))
+                        return -EPERM;
+                rtl8127_mdio_write(tp, 0x1F, 0x0000);
+                rtl8127_mdio_write(tp, data->reg_num, data->val_in);
+                break;
+
+#ifdef ETHTOOL_OPS_COMPAT
+        case SIOCETHTOOL:
+                ret = ethtool_ioctl(ifr);
+                break;
+#endif
+
+#ifdef ENABLE_PTP_SUPPORT
+        case SIOCSHWTSTAMP:
+        case SIOCGHWTSTAMP:
+                if (tp->EnablePtp)
+                        ret = rtl8127_ptp_ioctl(dev, ifr, cmd);
+                else
+                        ret = -EOPNOTSUPP;
+                break;
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0)
+#ifdef ENABLE_DASH_SUPPORT
+        case SIOCDEVPRIVATE_RTLDASH:
+                if (!netif_running(dev)) {
+                        ret = -ENODEV;
+                        break;
+                }
+                if (!capable(CAP_NET_ADMIN)) {
+                        ret = -EPERM;
+                        break;
+                }
+
+                ret = rtl8127_dash_ioctl(dev, ifr);
+                break;
+#endif
+
+#ifdef ENABLE_REALWOW_SUPPORT
+        case SIOCDEVPRIVATE_RTLREALWOW:
+                if (!netif_running(dev)) {
+                        ret = -ENODEV;
+                        break;
+                }
+
+                if (!capable(CAP_NET_ADMIN)) {
+                        ret = -EPERM;
+                        break;
+                }
+
+                ret = rtl8127_realwow_ioctl(dev, ifr);
+                break;
+#endif
+
+        case SIOCRTLTOOL:
+                if (!capable(CAP_NET_ADMIN)) {
+                        ret = -EPERM;
+                        break;
+                }
+
+                ret = rtl8127_tool_ioctl(tp, ifr);
+                break;
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0)
+
+        default:
+                ret = -EOPNOTSUPP;
+                break;
+        }
+
+        return ret;
+}
+
+static void
+rtl8127_phy_power_up(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+
+        if (rtl8127_is_in_phy_disable_mode(dev))
+                return;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE);
+
+        //wait ups resume (phy state 3)
+        rtl8127_wait_phy_ups_resume(dev, 3);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+}
+
+static void
+rtl8127_phy_power_down(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long flags;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+        rtl8127_mdio_write(tp, 0x1F, 0x0000);
+        rtl8127_mdio_write(tp, MII_BMCR, BMCR_ANENABLE | BMCR_PDOWN);
+        r8127_spin_unlock(&tp->phy_lock, flags);
+}
+
+static int __devinit
+rtl8127_init_board(struct pci_dev *pdev,
+                   struct net_device **dev_out,
+                   void __iomem **ioaddr_out)
+{
+        void __iomem *ioaddr;
+        struct net_device *dev;
+        struct rtl8127_private *tp;
+        int rc = -ENOMEM, i, pm_cap;
+
+        assert(ioaddr_out != NULL);
+
+        /* dev zeroed in alloc_etherdev */
+        dev = alloc_etherdev_mq(sizeof (*tp), R8127_MAX_QUEUES);
+        if (dev == NULL) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_drv(&debug))
+                        dev_err(&pdev->dev, "unable to alloc new ethernet\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                goto err_out;
+        }
+
+        SET_MODULE_OWNER(dev);
+        SET_NETDEV_DEV(dev, &pdev->dev);
+        tp = netdev_priv(dev);
+        tp->dev = dev;
+        tp->pci_dev = pdev;
+        tp->msg_enable = netif_msg_init(debug.msg_enable, R8127_MSG_DEFAULT);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26)
+        if (!aspm)
+                pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1 |
+                                       PCIE_LINK_STATE_CLKPM);
+#endif
+
+        /* enable device (incl. PCI PM wakeup and hotplug setup) */
+        rc = pci_enable_device(pdev);
+        if (rc < 0) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_probe(tp))
+                        dev_err(&pdev->dev, "enable failure\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                goto err_out_free_dev;
+        }
+
+        if (pci_set_mwi(pdev) < 0) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_drv(&debug))
+                        dev_info(&pdev->dev, "Mem-Wr-Inval unavailable.\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        }
+
+        /* save power state before pci_enable_device overwrites it */
+        pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
+        if (pm_cap) {
+                u16 pwr_command;
+
+                pci_read_config_word(pdev, pm_cap + PCI_PM_CTRL, &pwr_command);
+        } else {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_probe(tp))
+                        dev_err(&pdev->dev, "PowerManagement capability not found.\n");
+#else
+                printk("PowerManagement capability not found.\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+
+        }
+
+        /* make sure PCI base addr 1 is MMIO */
+        if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_probe(tp))
+                        dev_err(&pdev->dev, "region #1 not an MMIO resource, aborting\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                rc = -ENODEV;
+                goto err_out_mwi;
+        }
+        /* check for weird/broken PCI region reporting */
+        if (pci_resource_len(pdev, 2) < R8127_REGS_SIZE) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_probe(tp))
+                        dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                rc = -ENODEV;
+                goto err_out_mwi;
+        }
+
+        rc = pci_request_regions(pdev, MODULENAME);
+        if (rc < 0) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_probe(tp))
+                        dev_err(&pdev->dev, "could not request regions.\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                goto err_out_mwi;
+        }
+
+        if ((sizeof(dma_addr_t) > 4) &&
+            use_dac &&
+            !dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
+            !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
+                dev->features |= NETIF_F_HIGHDMA;
+        } else {
+                rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
+                if (rc < 0) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                        if (netif_msg_probe(tp))
+                                dev_err(&pdev->dev, "DMA configuration failed.\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                        goto err_out_free_res;
+                }
+        }
+
+        /* ioremap MMIO region */
+        ioaddr = ioremap(pci_resource_start(pdev, 2), pci_resource_len(pdev, 2));
+        if (ioaddr == NULL) {
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_probe(tp))
+                        dev_err(&pdev->dev, "cannot remap MMIO, aborting\n");
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                rc = -EIO;
+                goto err_out_free_res;
+        }
+
+        tp->mmio_addr = ioaddr;
+
+        /* Identify chip attached to board */
+        rtl8127_get_mac_version(tp);
+
+        rtl8127_print_mac_version(tp);
+
+        for (i = ARRAY_SIZE(rtl_chip_info) - 1; i >= 0; i--) {
+                if (tp->mcfg == rtl_chip_info[i].mcfg)
+                        break;
+        }
+
+        if (i < 0) {
+                /* Unknown chip: assume array element #0, original RTL-8125 */
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                if (netif_msg_probe(tp))
+                        dev_printk(KERN_DEBUG, &pdev->dev, "unknown chip version, assuming %s\n", rtl_chip_info[0].name);
+#else
+                printk("Realtek unknown chip version, assuming %s\n", rtl_chip_info[0].name);
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
+                i++;
+        }
+
+        tp->chipset = i;
+
+        *ioaddr_out = ioaddr;
+        *dev_out = dev;
+out:
+        return rc;
+
+err_out_free_res:
+        pci_release_regions(pdev);
+err_out_mwi:
+        pci_clear_mwi(pdev);
+        pci_disable_device(pdev);
+err_out_free_dev:
+        free_netdev(dev);
+err_out:
+        *ioaddr_out = NULL;
+        *dev_out = NULL;
+        goto out;
+}
+
+static void
+rtl8127_esd_checker(struct rtl8127_private *tp)
+{
+        struct net_device *dev = tp->dev;
+        struct pci_dev *pdev = tp->pci_dev;
+        u8 cmd;
+        u16 io_base_l;
+        u16 mem_base_l;
+        u16 mem_base_h;
+        u8 ilr;
+        u16 resv_0x1c_h;
+        u16 resv_0x1c_l;
+        u16 resv_0x20_l;
+        u16 resv_0x20_h;
+        u16 resv_0x24_l;
+        u16 resv_0x24_h;
+        u16 resv_0x2c_h;
+        u16 resv_0x2c_l;
+        u32 pci_sn_l;
+        u32 pci_sn_h;
+
+        if (unlikely(tp->rtk_enable_diag))
+                goto exit;
+
+        tp->esd_flag = 0;
+
+        pci_read_config_byte(pdev, PCI_COMMAND, &cmd);
+        if (cmd != tp->pci_cfg_space.cmd) {
+                printk(KERN_ERR "%s: cmd = 0x%02x, should be 0x%02x \n.", dev->name, cmd, tp->pci_cfg_space.cmd);
+                pci_write_config_byte(pdev, PCI_COMMAND, tp->pci_cfg_space.cmd);
+                tp->esd_flag |= BIT_0;
+
+                pci_read_config_byte(pdev, PCI_COMMAND, &cmd);
+                if (cmd == 0xff) {
+                        printk(KERN_ERR "%s: pci link is down \n.", dev->name);
+                        goto exit;
+                }
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_0, &io_base_l);
+        if (io_base_l != tp->pci_cfg_space.io_base_l) {
+                printk(KERN_ERR "%s: io_base_l = 0x%04x, should be 0x%04x \n.", dev->name, io_base_l, tp->pci_cfg_space.io_base_l);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_0, tp->pci_cfg_space.io_base_l);
+                tp->esd_flag |= BIT_1;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_2, &mem_base_l);
+        if (mem_base_l != tp->pci_cfg_space.mem_base_l) {
+                printk(KERN_ERR "%s: mem_base_l = 0x%04x, should be 0x%04x \n.", dev->name, mem_base_l, tp->pci_cfg_space.mem_base_l);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_2, tp->pci_cfg_space.mem_base_l);
+                tp->esd_flag |= BIT_2;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, &mem_base_h);
+        if (mem_base_h!= tp->pci_cfg_space.mem_base_h) {
+                printk(KERN_ERR "%s: mem_base_h = 0x%04x, should be 0x%04x \n.", dev->name, mem_base_h, tp->pci_cfg_space.mem_base_h);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, tp->pci_cfg_space.mem_base_h);
+                tp->esd_flag |= BIT_3;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_3, &resv_0x1c_l);
+        if (resv_0x1c_l != tp->pci_cfg_space.resv_0x1c_l) {
+                printk(KERN_ERR "%s: resv_0x1c_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x1c_l, tp->pci_cfg_space.resv_0x1c_l);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_3, tp->pci_cfg_space.resv_0x1c_l);
+                tp->esd_flag |= BIT_4;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, &resv_0x1c_h);
+        if (resv_0x1c_h != tp->pci_cfg_space.resv_0x1c_h) {
+                printk(KERN_ERR "%s: resv_0x1c_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x1c_h, tp->pci_cfg_space.resv_0x1c_h);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, tp->pci_cfg_space.resv_0x1c_h);
+                tp->esd_flag |= BIT_5;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_4, &resv_0x20_l);
+        if (resv_0x20_l != tp->pci_cfg_space.resv_0x20_l) {
+                printk(KERN_ERR "%s: resv_0x20_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x20_l, tp->pci_cfg_space.resv_0x20_l);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_4, tp->pci_cfg_space.resv_0x20_l);
+                tp->esd_flag |= BIT_6;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, &resv_0x20_h);
+        if (resv_0x20_h != tp->pci_cfg_space.resv_0x20_h) {
+                printk(KERN_ERR "%s: resv_0x20_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x20_h, tp->pci_cfg_space.resv_0x20_h);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, tp->pci_cfg_space.resv_0x20_h);
+                tp->esd_flag |= BIT_7;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_5, &resv_0x24_l);
+        if (resv_0x24_l != tp->pci_cfg_space.resv_0x24_l) {
+                printk(KERN_ERR "%s: resv_0x24_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x24_l, tp->pci_cfg_space.resv_0x24_l);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_5, tp->pci_cfg_space.resv_0x24_l);
+                tp->esd_flag |= BIT_8;
+        }
+
+        pci_read_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, &resv_0x24_h);
+        if (resv_0x24_h != tp->pci_cfg_space.resv_0x24_h) {
+                printk(KERN_ERR "%s: resv_0x24_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x24_h, tp->pci_cfg_space.resv_0x24_h);
+                pci_write_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, tp->pci_cfg_space.resv_0x24_h);
+                tp->esd_flag |= BIT_9;
+        }
+
+        pci_read_config_byte(pdev, PCI_INTERRUPT_LINE, &ilr);
+        if (ilr != tp->pci_cfg_space.ilr) {
+                printk(KERN_ERR "%s: ilr = 0x%02x, should be 0x%02x \n.", dev->name, ilr, tp->pci_cfg_space.ilr);
+                pci_write_config_byte(pdev, PCI_INTERRUPT_LINE, tp->pci_cfg_space.ilr);
+                tp->esd_flag |= BIT_10;
+        }
+
+        pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &resv_0x2c_l);
+        if (resv_0x2c_l != tp->pci_cfg_space.resv_0x2c_l) {
+                printk(KERN_ERR "%s: resv_0x2c_l = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x2c_l, tp->pci_cfg_space.resv_0x2c_l);
+                pci_write_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, tp->pci_cfg_space.resv_0x2c_l);
+                tp->esd_flag |= BIT_11;
+        }
+
+        pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, &resv_0x2c_h);
+        if (resv_0x2c_h != tp->pci_cfg_space.resv_0x2c_h) {
+                printk(KERN_ERR "%s: resv_0x2c_h = 0x%04x, should be 0x%04x \n.", dev->name, resv_0x2c_h, tp->pci_cfg_space.resv_0x2c_h);
+                pci_write_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, tp->pci_cfg_space.resv_0x2c_h);
+                tp->esd_flag |= BIT_12;
+        }
+
+        if (tp->HwPcieSNOffset > 0) {
+                pci_sn_l = rtl8127_csi_read(tp, tp->HwPcieSNOffset);
+                if (pci_sn_l != tp->pci_cfg_space.pci_sn_l) {
+                        printk(KERN_ERR "%s: pci_sn_l = 0x%08x, should be 0x%08x \n.", dev->name, pci_sn_l, tp->pci_cfg_space.pci_sn_l);
+                        rtl8127_csi_write(tp, tp->HwPcieSNOffset, tp->pci_cfg_space.pci_sn_l);
+                        tp->esd_flag |= BIT_13;
+                }
+
+                pci_sn_h = rtl8127_csi_read(tp, tp->HwPcieSNOffset + 4);
+                if (pci_sn_h != tp->pci_cfg_space.pci_sn_h) {
+                        printk(KERN_ERR "%s: pci_sn_h = 0x%08x, should be 0x%08x \n.", dev->name, pci_sn_h, tp->pci_cfg_space.pci_sn_h);
+                        rtl8127_csi_write(tp, tp->HwPcieSNOffset + 4, tp->pci_cfg_space.pci_sn_h);
+                        tp->esd_flag |= BIT_14;
+                }
+        }
+
+        if (tp->esd_flag != 0) {
+                printk(KERN_ERR "%s: esd_flag = 0x%04x\n.\n", dev->name, tp->esd_flag);
+                netif_carrier_off(dev);
+                netif_tx_disable(dev);
+                rtl8127_hw_reset(dev);
+                rtl8127_tx_clear(tp);
+                rtl8127_rx_clear(tp);
+                rtl8127_init_ring(dev);
+                rtl8127_up(dev);
+                rtl8127_enable_hw_linkchg_interrupt(tp);
+                rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising);
+                tp->esd_flag = 0;
+        }
+exit:
+        return;
+}
+/*
+static void
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+rtl8127_esd_timer(unsigned long __opaque)
+#else
+rtl8127_esd_timer(struct timer_list *t)
+#endif
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+        struct net_device *dev = (struct net_device *)__opaque;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct timer_list *timer = &tp->esd_timer;
+#else
+        struct rtl8127_private *tp = from_timer(tp, t, esd_timer);
+        //struct net_device *dev = tp->dev;
+        struct timer_list *timer = t;
+#endif
+        rtl8127_esd_checker(tp);
+
+        mod_timer(timer, jiffies + timeout);
+}
+*/
+
+/*
+static void
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+rtl8127_link_timer(unsigned long __opaque)
+#else
+rtl8127_link_timer(struct timer_list *t)
+#endif
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
+        struct net_device *dev = (struct net_device *)__opaque;
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct timer_list *timer = &tp->link_timer;
+#else
+        struct rtl8127_private *tp = from_timer(tp, t, link_timer);
+        struct net_device *dev = tp->dev;
+        struct timer_list *timer = t;
+#endif
+        rtl8127_check_link_status(dev);
+
+        mod_timer(timer, jiffies + RTL8127_LINK_TIMEOUT);
+}
+*/
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0)
+static int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
+                                 int minvec, int maxvec)
+{
+        int nvec = maxvec;
+        int rc;
+
+        if (maxvec < minvec)
+                return -ERANGE;
+
+        do {
+                rc = pci_enable_msix(dev, entries, nvec);
+                if (rc < 0) {
+                        return rc;
+                } else if (rc > 0) {
+                        if (rc < minvec)
+                                return -ENOSPC;
+                        nvec = rc;
+                }
+        } while (rc);
+
+        return nvec;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3,14,0) */
+
+static int rtl8127_enable_msix(struct rtl8127_private *tp)
+{
+        int i, nvecs = 0;
+        struct msix_entry msix_ent[R8127_MAX_MSIX_VEC];
+        //struct net_device *dev = tp->dev;
+        //const int len = sizeof(tp->irq_tbl[0].name);
+
+        for (i = 0; i < R8127_MAX_MSIX_VEC; i++) {
+                msix_ent[i].entry = i;
+                msix_ent[i].vector = 0;
+        }
+
+        nvecs = pci_enable_msix_range(tp->pci_dev, msix_ent,
+                                      tp->min_irq_nvecs, tp->max_irq_nvecs);
+        if (nvecs < 0)
+                goto out;
+
+        for (i = 0; i < nvecs; i++) {
+                struct r8127_irq *irq = &tp->irq_tbl[i];
+                irq->vector = msix_ent[i].vector;
+                //snprintf(irq->name, len, "%s-%d", dev->name, i);
+                //irq->handler = rtl8127_interrupt_msix;
+        }
+
+out:
+        return nvecs;
+}
+
+/* Cfg9346_Unlock assumed. */
+static int rtl8127_try_msi(struct rtl8127_private *tp)
+{
+        struct pci_dev *pdev = tp->pci_dev;
+        unsigned int hw_supp_irq_nvecs;
+        unsigned msi = 0;
+        int nvecs = 1;
+
+        if (HW_DASH_SUPPORT_DASH(tp))
+                hw_supp_irq_nvecs = R8127_MAX_MSIX_VEC_8127AP;
+        else
+                hw_supp_irq_nvecs = R8127_MAX_MSIX_VEC_8127;
+        tp->hw_supp_irq_nvecs = clamp_val(hw_supp_irq_nvecs, 1,
+                                          R8127_MAX_MSIX_VEC);
+
+        tp->max_irq_nvecs = tp->hw_supp_irq_nvecs;
+        if (HW_DASH_SUPPORT_DASH(tp))
+                tp->min_irq_nvecs = R8127_MIN_MSIX_VEC_8127AP;
+        else
+                tp->min_irq_nvecs = R8127_MIN_MSIX_VEC_8127;
+#ifdef DISABLE_MULTI_MSIX_VECTOR
+        tp->max_irq_nvecs = 1;
+#endif
+
+#if defined(RTL_USE_NEW_INTR_API)
+        if ((nvecs = pci_alloc_irq_vectors(pdev, tp->min_irq_nvecs, tp->max_irq_nvecs, PCI_IRQ_MSIX)) > 0)
+                msi |= RTL_FEATURE_MSIX;
+        else if ((nvecs = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES)) > 0 &&
+                 pci_dev_msi_enabled(pdev))
+                msi |= RTL_FEATURE_MSI;
+#elif LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13)
+        if ((nvecs = rtl8127_enable_msix(tp)) > 0)
+                msi |= RTL_FEATURE_MSIX;
+        else if (!pci_enable_msi(pdev))
+                msi |= RTL_FEATURE_MSI;
+#endif
+        if (!(msi & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX)))
+                dev_info(&pdev->dev, "no MSI/MSI-X. Back to INTx.\n");
+
+        if (!(msi & RTL_FEATURE_MSIX) || nvecs < 1)
+                nvecs = 1;
+
+        tp->irq_nvecs = nvecs;
+
+        tp->features |= msi;
+
+        return nvecs;
+}
+
+static void rtl8127_disable_msi(struct pci_dev *pdev, struct rtl8127_private *tp)
+{
+#if defined(RTL_USE_NEW_INTR_API)
+        if (tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX))
+                pci_free_irq_vectors(pdev);
+#elif LINUX_VERSION_CODE > KERNEL_VERSION(2,6,13)
+        if (tp->features & (RTL_FEATURE_MSIX))
+                pci_disable_msix(pdev);
+        else if (tp->features & (RTL_FEATURE_MSI))
+                pci_disable_msi(pdev);
+#endif
+        tp->features &= ~(RTL_FEATURE_MSI | RTL_FEATURE_MSIX);
+}
+
+static int rtl8127_get_irq(struct pci_dev *pdev)
+{
+#if defined(RTL_USE_NEW_INTR_API)
+        return pci_irq_vector(pdev, 0);
+#else
+        return pdev->irq;
+#endif
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0)
+static void
+rtl8127_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct rtl8127_counters *counters = tp->tally_vaddr;
+        dma_addr_t paddr = tp->tally_paddr;
+
+        if (!counters)
+                return;
+
+        netdev_stats_to_stats64(stats, &dev->stats);
+        dev_fetch_sw_netstats(stats, dev->tstats);
+
+        /*
+         * Fetch additional counter values missing in stats collected by driver
+         * from tally counters.
+         */
+        rtl8127_dump_tally_counter(tp, paddr);
+
+        stats->tx_errors = le64_to_cpu(counters->tx_errors);
+        stats->collisions = le32_to_cpu(counters->tx_multi_collision);
+        stats->tx_aborted_errors = le16_to_cpu(counters->tx_aborted);
+        stats->rx_missed_errors = le16_to_cpu(counters->rx_missed);
+}
+#else
+/**
+ *  rtl8127_get_stats - Get rtl8127 read/write statistics
+ *  @dev: The Ethernet Device to get statistics for
+ *
+ *  Get TX/RX statistics for rtl8127
+ */
+static struct
+net_device_stats *rtl8127_get_stats(struct net_device *dev)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+        struct rtl8127_private *tp = netdev_priv(dev);
+#endif
+        return &RTLDEV->stats;
+}
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,36)
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)
+static const struct net_device_ops rtl8127_netdev_ops = {
+        .ndo_open       = rtl8127_open,
+        .ndo_stop       = rtl8127_close,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0)
+        .ndo_get_stats64    = rtl8127_get_stats64,
+#else
+        .ndo_get_stats      = rtl8127_get_stats,
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0)
+        .ndo_start_xmit     = rtl8127_start_xmit,
+        .ndo_tx_timeout     = rtl8127_tx_timeout,
+        .ndo_change_mtu     = rtl8127_change_mtu,
+        .ndo_set_mac_address    = rtl8127_set_mac_address,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0)
+        .ndo_do_ioctl       = rtl8127_do_ioctl,
+#else
+        .ndo_siocdevprivate = rtl8127_siocdevprivate,
+        .ndo_eth_ioctl      = rtl8127_do_ioctl,
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(5,15,0)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,1,0)
+        .ndo_set_multicast_list = rtl8127_set_rx_mode,
+#else
+        .ndo_set_rx_mode    = rtl8127_set_rx_mode,
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+#ifdef CONFIG_R8127_VLAN
+        .ndo_vlan_rx_register   = rtl8127_vlan_rx_register,
+#endif
+#else
+        .ndo_fix_features   = rtl8127_fix_features,
+        .ndo_set_features   = rtl8127_set_features,
+#endif
+#ifdef CONFIG_NET_POLL_CONTROLLER
+        .ndo_poll_controller    = rtl8127_netpoll,
+#endif
+};
+#endif
+
+
+#ifdef  CONFIG_R8127_NAPI
+
+static int rtl8127_poll(napi_ptr napi, napi_budget budget)
+{
+        struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi);
+        struct rtl8127_private *tp = r8127napi->priv;
+        RTL_GET_NETDEV(tp)
+        unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev);
+        unsigned int work_done = 0;
+        int i;
+
+        for (i = 0; i < tp->num_tx_rings; i++)
+                rtl8127_tx_interrupt(&tp->tx_ring[i], budget);
+
+        for (i = 0; i < tp->num_rx_rings; i++)
+                work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[i], budget);
+
+        work_done = min(work_done, work_to_do);
+
+        RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget);
+
+        if (work_done < work_to_do) {
+#ifdef ENABLE_DASH_SUPPORT
+                if (rtl8127_check_dash_interrupt(tp))
+                        rtl8127_schedule_dash_work(tp);
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+                if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE)
+                        return RTL_NAPI_RETURN_VALUE;
+#else
+                RTL_NETIF_RX_COMPLETE(dev, napi, work_done);
+#endif
+                /*
+                 * 20040426: the barrier is not strictly required but the
+                 * behavior of the irq handler could be less predictable
+                 * without it. Btw, the lack of flush for the posted pci
+                 * write is safe - FR
+                 */
+                smp_wmb();
+
+                rtl8127_switch_to_timer_interrupt(tp);
+        }
+
+        return RTL_NAPI_RETURN_VALUE;
+}
+
+static int rtl8127_poll_msix_ring(napi_ptr napi, napi_budget budget)
+{
+        struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi);
+        struct rtl8127_private *tp = r8127napi->priv;
+        RTL_GET_NETDEV(tp)
+        unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev);
+        unsigned int work_done = 0;
+        const int message_id = r8127napi->index;
+
+        if (message_id < tp->num_tx_rings)
+                rtl8127_tx_interrupt_with_vector(tp, message_id, budget);
+
+        if (message_id < tp->num_rx_rings)
+                work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget);
+
+        RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget);
+
+        if (work_done < work_to_do) {
+#ifdef ENABLE_DASH_SUPPORT
+                if (message_id == MSIX_ID_V6_L2_MISC_INTR)
+                        if (rtl8127_check_dash_interrupt(tp))
+                                rtl8127_schedule_dash_work(tp);
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+                if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE)
+                        return RTL_NAPI_RETURN_VALUE;
+#else
+                RTL_NETIF_RX_COMPLETE(dev, napi, work_done);
+#endif
+                /*
+                 * 20040426: the barrier is not strictly required but the
+                 * behavior of the irq handler could be less predictable
+                 * without it. Btw, the lack of flush for the posted pci
+                 * write is safe - FR
+                 */
+                smp_wmb();
+
+                rtl8127_enable_hw_layered_interrupt(tp, message_id);
+        }
+
+        return RTL_NAPI_RETURN_VALUE;
+}
+
+static int rtl8127_poll_msix_tx(napi_ptr napi, napi_budget budget)
+{
+        struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi);
+        struct rtl8127_private *tp = r8127napi->priv;
+        RTL_GET_NETDEV(tp)
+        unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev);
+        unsigned int work_done = 0;
+        const int message_id = r8127napi->index;
+
+        //suppress unused variable
+        (void)(dev);
+
+        rtl8127_tx_interrupt_with_vector(tp, message_id, budget);
+
+        RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget);
+
+        if (work_done < work_to_do) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+                if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE)
+                        return RTL_NAPI_RETURN_VALUE;
+#else
+                RTL_NETIF_RX_COMPLETE(dev, napi, work_done);
+#endif
+                /*
+                 * 20040426: the barrier is not strictly required but the
+                 * behavior of the irq handler could be less predictable
+                 * without it. Btw, the lack of flush for the posted pci
+                 * write is safe - FR
+                 */
+                smp_wmb();
+
+                rtl8127_enable_hw_layered_interrupt(tp, message_id);
+        }
+
+        return RTL_NAPI_RETURN_VALUE;
+}
+
+static int rtl8127_poll_msix_other(napi_ptr napi, napi_budget budget)
+{
+        struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi);
+        struct rtl8127_private *tp = r8127napi->priv;
+        RTL_GET_NETDEV(tp)
+        unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev);
+        const int message_id = r8127napi->index;
+
+        //suppress unused variable
+        (void)(dev);
+        (void)(work_to_do);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+        RTL_NETIF_RX_COMPLETE(dev, napi, work_to_do);
+#else
+        RTL_NETIF_RX_COMPLETE(dev, napi, work_to_do);
+#endif
+
+        rtl8127_enable_hw_layered_interrupt(tp, message_id);
+
+        return 1;
+}
+
+static int rtl8127_poll_msix_rx(napi_ptr napi, napi_budget budget)
+{
+        struct r8127_napi *r8127napi = RTL_GET_PRIV(napi, struct r8127_napi);
+        struct rtl8127_private *tp = r8127napi->priv;
+        RTL_GET_NETDEV(tp)
+        unsigned int work_to_do = RTL_NAPI_QUOTA(budget, dev);
+        unsigned int work_done = 0;
+        const int message_id = r8127napi->index;
+
+        if (message_id < tp->num_rx_rings)
+                work_done += rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget);
+
+        RTL_NAPI_QUOTA_UPDATE(dev, work_done, budget);
+
+        if (work_done < work_to_do) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)
+                if (RTL_NETIF_RX_COMPLETE(dev, napi, work_done) == FALSE)
+                        return RTL_NAPI_RETURN_VALUE;
+#else
+                RTL_NETIF_RX_COMPLETE(dev, napi, work_done);
+#endif
+                /*
+                 * 20040426: the barrier is not strictly required but the
+                 * behavior of the irq handler could be less predictable
+                 * without it. Btw, the lack of flush for the posted pci
+                 * write is safe - FR
+                 */
+                smp_wmb();
+
+                rtl8127_enable_hw_layered_interrupt(tp, message_id);
+        }
+
+        return RTL_NAPI_RETURN_VALUE;
+}
+
+void rtl8127_enable_napi(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        int i;
+
+        for (i = 0; i < tp->irq_nvecs; i++)
+                RTL_NAPI_ENABLE(tp->dev, &tp->r8127napi[i].napi);
+#endif
+}
+
+static void rtl8127_disable_napi(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        int i;
+
+        for (i = 0; i < tp->irq_nvecs; i++)
+                RTL_NAPI_DISABLE(tp->dev, &tp->r8127napi[i].napi);
+#endif
+}
+
+static void rtl8127_del_napi(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        int i;
+
+        for (i = 0; i < tp->irq_nvecs; i++)
+                RTL_NAPI_DEL((&tp->r8127napi[i]));
+#endif
+}
+#endif //CONFIG_R8127_NAPI
+
+static void rtl8127_init_napi(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i=0; i<tp->irq_nvecs; i++) {
+                struct r8127_napi *r8127napi = &tp->r8127napi[i];
+#ifdef CONFIG_R8127_NAPI
+                int (*poll)(struct napi_struct *, int);
+
+                poll = rtl8127_poll;
+                if (tp->features & RTL_FEATURE_MSIX) {
+                        switch (tp->HwCurrIsrVer) {
+                        case 6:
+                                if (i < R8127_MAX_RX_QUEUES_VEC_V4)
+                                        poll = rtl8127_poll_msix_rx;
+                                else if (i == 8 || i == 9)
+                                        poll = rtl8127_poll_msix_tx;
+                                else
+                                        poll = rtl8127_poll_msix_other;
+                                break;
+                        case 5:
+                                if (i < R8127_MAX_RX_QUEUES_VEC_V3)
+                                        poll = rtl8127_poll_msix_rx;
+                                else if (i == 16 || i == 17)
+                                        poll = rtl8127_poll_msix_tx;
+                                else
+                                        poll = rtl8127_poll_msix_other;
+                                break;
+                        case 2:
+                                if (i < R8127_MAX_RX_QUEUES_VEC_V3)
+                                        poll = rtl8127_poll_msix_rx;
+                                else if (i == 16 || i == 18)
+                                        poll = rtl8127_poll_msix_tx;
+                                else
+                                        poll = rtl8127_poll_msix_other;
+                                break;
+                        case 3:
+                        case 4:
+                                if (i < R8127_MAX_RX_QUEUES_VEC_V3)
+                                        poll = rtl8127_poll_msix_ring;
+                                else
+                                        poll = rtl8127_poll_msix_other;
+                                break;
+                        }
+                }
+
+                RTL_NAPI_CONFIG(tp->dev, r8127napi, poll, R8127_NAPI_WEIGHT);
+#endif
+
+                r8127napi->priv = tp;
+                r8127napi->index = i;
+        }
+}
+
+static int
+rtl8127_set_real_num_queue(struct rtl8127_private *tp)
+{
+        int retval = 0;
+
+        retval = netif_set_real_num_tx_queues(tp->dev, tp->num_tx_rings);
+        if (retval < 0)
+                goto exit;
+
+        retval = netif_set_real_num_rx_queues(tp->dev, tp->num_rx_rings);
+        if (retval < 0)
+                goto exit;
+
+exit:
+        return retval;
+}
+
+static int __devinit
+rtl8127_init_one(struct pci_dev *pdev,
+                 const struct pci_device_id *ent)
+{
+        struct net_device *dev = NULL;
+        struct rtl8127_private *tp;
+        void __iomem *ioaddr = NULL;
+        static int board_idx = -1;
+
+        int rc;
+
+        assert(pdev != NULL);
+        assert(ent != NULL);
+
+        board_idx++;
+
+        if (netif_msg_drv(&debug))
+                printk(KERN_INFO "%s Ethernet controller driver %s loaded\n",
+                       MODULENAME, RTL8127_VERSION);
+
+        rc = rtl8127_init_board(pdev, &dev, &ioaddr);
+        if (rc)
+                goto out;
+
+        tp = netdev_priv(dev);
+        assert(ioaddr != NULL);
+
+        spin_lock_init(&tp->phy_lock);
+
+        tp->set_speed = rtl8127_set_speed_xmii;
+        tp->get_settings = rtl8127_gset_xmii;
+        tp->phy_reset_enable = rtl8127_xmii_reset_enable;
+        tp->phy_reset_pending = rtl8127_xmii_reset_pending;
+        tp->link_ok = rtl8127_xmii_link_ok;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0)
+        dev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev,
+                        struct pcpu_sw_netstats);
+        if (!dev->tstats)
+                goto err_out_1;
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,11,0)
+
+        rc = rtl8127_try_msi(tp);
+        if (rc < 0) {
+                dev_err(&pdev->dev, "Can't allocate interrupt\n");
+                goto err_out_1;
+        }
+
+        rtl8127_init_software_variable(dev);
+
+        RTL_NET_DEVICE_OPS(rtl8127_netdev_ops);
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,4,22)
+        SET_ETHTOOL_OPS(dev, &rtl8127_ethtool_ops);
+#endif
+
+        dev->watchdog_timeo = RTL8127_TX_TIMEOUT;
+        dev->irq = rtl8127_get_irq(pdev);
+        dev->base_addr = (unsigned long) ioaddr;
+
+        rtl8127_init_napi(tp);
+
+#ifdef CONFIG_R8127_VLAN
+        if (tp->mcfg != CFG_METHOD_DEFAULT) {
+                dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+                dev->vlan_rx_kill_vid = rtl8127_vlan_rx_kill_vid;
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+        }
+#endif
+
+        /* There has been a number of reports that using SG/TSO results in
+         * tx timeouts. However for a lot of people SG/TSO works fine.
+         * Therefore disable both features by default, but allow users to
+         * enable them. Use at own risk!
+         */
+        tp->cp_cmd |= RTL_R16(tp, CPlusCmd);
+        if (tp->mcfg != CFG_METHOD_DEFAULT) {
+                dev->features |= NETIF_F_IP_CSUM;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+                tp->cp_cmd |= RxChkSum;
+#else
+                dev->features |= NETIF_F_RXCSUM;
+                dev->features |= NETIF_F_SG | NETIF_F_TSO;
+                dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO |
+                                   NETIF_F_RXCSUM | NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
+                dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO |
+                                     NETIF_F_HIGHDMA;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,15,0)
+                dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,15,0)
+                dev->hw_features |= NETIF_F_RXALL;
+                dev->hw_features |= NETIF_F_RXFCS;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22)
+                dev->hw_features |= NETIF_F_IPV6_CSUM | NETIF_F_TSO6;
+                dev->features |= NETIF_F_IPV6_CSUM;
+                dev->features |= NETIF_F_TSO6;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0)
+                netif_set_tso_max_size(dev, LSO_64K);
+                netif_set_tso_max_segs(dev, NIC_MAX_PHYS_BUF_COUNT_LSO2);
+#else //LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0)
+                netif_set_gso_max_size(dev, LSO_64K);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,0)
+                dev->gso_max_segs = NIC_MAX_PHYS_BUF_COUNT_LSO2;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0)
+                dev->gso_min_segs = NIC_MIN_PHYS_BUF_COUNT;
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,7,0)
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,0)
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(5,19,0)
+
+#endif //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22)
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+
+#ifdef ENABLE_RSS_SUPPORT
+                if (tp->EnableRss) {
+                        dev->hw_features |= NETIF_F_RXHASH;
+                        dev->features |= NETIF_F_RXHASH;
+                }
+#endif
+        }
+
+        netdev_sw_irq_coalesce_default_on(dev);
+
+#ifdef ENABLE_LIB_SUPPORT
+        BLOCKING_INIT_NOTIFIER_HEAD(&tp->lib_nh);
+#endif
+        rtl8127_init_all_schedule_work(tp);
+
+        rc = rtl8127_set_real_num_queue(tp);
+        if (rc < 0)
+                goto err_out;
+
+        rtl8127_exit_oob(dev);
+
+        rtl8127_powerup_pll(dev);
+
+        rtl8127_hw_init(dev);
+
+        rtl8127_hw_reset(dev);
+
+        /* Get production from EEPROM */
+        rtl8127_eeprom_type(tp);
+
+        if (tp->eeprom_type == EEPROM_TYPE_93C46 || tp->eeprom_type == EEPROM_TYPE_93C56)
+                rtl8127_set_eeprom_sel_low(tp);
+
+        rtl8127_get_mac_address(dev);
+
+        tp->fw_name = rtl_chip_fw_infos[tp->mcfg].fw_name;
+
+        tp->tally_vaddr = dma_alloc_coherent(&pdev->dev, sizeof(*tp->tally_vaddr),
+                                             &tp->tally_paddr, GFP_KERNEL);
+        if (!tp->tally_vaddr) {
+                rc = -ENOMEM;
+                goto err_out;
+        }
+
+        rtl8127_tally_counter_clear(tp);
+
+        pci_set_drvdata(pdev, dev);
+
+        rc = register_netdev(dev);
+        if (rc)
+                goto err_out;
+
+        printk(KERN_INFO "%s: This product is covered by one or more of the following patents: US6,570,884, US6,115,776, and US6,327,625.\n", MODULENAME);
+
+        rtl8127_disable_rxdvgate(dev);
+
+        device_set_wakeup_enable(&pdev->dev, tp->wol_enabled);
+
+        netif_carrier_off(dev);
+
+#ifdef ENABLE_R8127_SYSFS
+        rtl8127_sysfs_init(dev);
+#endif /* ENABLE_R8127_SYSFS */
+
+        printk("%s", GPL_CLAIM);
+
+out:
+        return rc;
+
+err_out:
+        if (tp->tally_vaddr != NULL) {
+                dma_free_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), tp->tally_vaddr,
+                                  tp->tally_paddr);
+
+                tp->tally_vaddr = NULL;
+        }
+#ifdef  CONFIG_R8127_NAPI
+        rtl8127_del_napi(tp);
+#endif
+        rtl8127_disable_msi(pdev, tp);
+
+err_out_1:
+        rtl8127_release_board(pdev, dev);
+
+        goto out;
+}
+
+static void __devexit
+rtl8127_remove_one(struct pci_dev *pdev)
+{
+        struct net_device *dev = pci_get_drvdata(pdev);
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        assert(dev != NULL);
+        assert(tp != NULL);
+
+        set_bit(R8127_FLAG_DOWN, tp->task_flags);
+
+        rtl8127_cancel_all_schedule_work(tp);
+
+        if (HW_DASH_SUPPORT_DASH(tp))
+                rtl8127_driver_stop(tp);
+
+        rtl8127_disable_pci_offset_180(tp);
+
+#ifdef ENABLE_R8127_SYSFS
+        rtl8127_sysfs_remove(dev);
+#endif //ENABLE_R8127_SYSFS
+
+        unregister_netdev(dev);
+#ifdef  CONFIG_R8127_NAPI
+        rtl8127_del_napi(tp);
+#endif
+        rtl8127_disable_msi(pdev, tp);
+#ifdef ENABLE_R8127_PROCFS
+        rtl8127_proc_remove(dev);
+#endif
+        if (tp->tally_vaddr != NULL) {
+                dma_free_coherent(&pdev->dev, sizeof(*tp->tally_vaddr), tp->tally_vaddr, tp->tally_paddr);
+                tp->tally_vaddr = NULL;
+        }
+
+        rtl8127_release_board(pdev, dev);
+
+#ifdef ENABLE_USE_FIRMWARE_FILE
+        rtl8127_release_firmware(tp);
+#endif
+
+        pci_set_drvdata(pdev, NULL);
+}
+
+#ifdef ENABLE_PAGE_REUSE
+static inline unsigned int rtl8127_rx_page_order(unsigned rx_buf_sz, unsigned page_size)
+{
+        unsigned truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
+                            SKB_DATA_ALIGN(rx_buf_sz + R8127_RX_ALIGN);
+
+        return get_order(truesize * 2);
+}
+#endif //ENABLE_PAGE_REUSE
+
+static void
+rtl8127_set_rxbufsize(struct rtl8127_private *tp,
+                      struct net_device *dev)
+{
+        unsigned int mtu = dev->mtu;
+
+        tp->rms = (mtu > ETH_DATA_LEN) ?
+                  mtu + ETH_HLEN + RT_VALN_HLEN + ETH_FCS_LEN:
+                  RX_BUF_SIZE;
+        tp->rx_buf_sz = tp->rms;
+#ifdef ENABLE_RX_PACKET_FRAGMENT
+        tp->rx_buf_sz =  SKB_DATA_ALIGN(RX_BUF_SIZE);
+#endif //ENABLE_RX_PACKET_FRAGMENT
+#ifdef ENABLE_PAGE_REUSE
+        tp->rx_buf_page_order = rtl8127_rx_page_order(tp->rx_buf_sz, PAGE_SIZE);
+        tp->rx_buf_page_size = rtl8127_rx_page_size(tp->rx_buf_page_order);
+#endif //ENABLE_PAGE_REUSE
+}
+
+static void
+rtl8127_set_rms(struct rtl8127_private *tp, u16 rms)
+{
+        RTL_W16(tp, RxMaxSize, rms | AcceppVlanPhys);
+}
+
+static void rtl8127_free_irq(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i=0; i<tp->irq_nvecs; i++) {
+                struct r8127_irq *irq = &tp->irq_tbl[i];
+                struct r8127_napi *r8127napi = &tp->r8127napi[i];
+
+                if (irq->requested) {
+                        irq->requested = 0;
+#if defined(RTL_USE_NEW_INTR_API)
+                        pci_free_irq(tp->pci_dev, i, r8127napi);
+#else
+                        free_irq(irq->vector, r8127napi);
+#endif
+                }
+        }
+}
+
+static int rtl8127_alloc_irq(struct rtl8127_private *tp)
+{
+        struct net_device *dev = tp->dev;
+        int rc = 0;
+        struct r8127_irq *irq;
+        struct r8127_napi *r8127napi;
+        int i = 0;
+        const int len = sizeof(tp->irq_tbl[0].name);
+
+#if defined(RTL_USE_NEW_INTR_API)
+        for (i=0; i<tp->irq_nvecs; i++) {
+                irq = &tp->irq_tbl[i];
+                if (tp->features & RTL_FEATURE_MSIX &&
+                    tp->HwCurrIsrVer > 1)
+                        irq->handler = rtl8127_interrupt_msix;
+                else
+                        irq->handler = rtl8127_interrupt;
+
+                r8127napi = &tp->r8127napi[i];
+                snprintf(irq->name, len, "%s-%d", dev->name, i);
+                rc = pci_request_irq(tp->pci_dev, i, irq->handler, NULL, r8127napi,
+                                     irq->name);
+                if (rc)
+                        break;
+
+                irq->vector = pci_irq_vector(tp->pci_dev, i);
+                irq->requested = 1;
+        }
+#else
+        unsigned long irq_flags = 0;
+#ifdef ENABLE_LIB_SUPPORT
+        irq_flags |= IRQF_NO_SUSPEND;
+#endif
+        if (tp->features & RTL_FEATURE_MSIX &&
+            tp->HwCurrIsrVer > 1) {
+                for (i=0; i<tp->irq_nvecs; i++) {
+                        irq = &tp->irq_tbl[i];
+                        irq->handler = rtl8127_interrupt_msix;
+                        r8127napi = &tp->r8127napi[i];
+                        snprintf(irq->name, len, "%s-%d", dev->name, i);
+                        rc = request_irq(irq->vector, irq->handler, irq_flags, irq->name, r8127napi);
+
+                        if (rc)
+                                break;
+
+                        irq->requested = 1;
+                }
+        } else {
+                irq = &tp->irq_tbl[0];
+                irq->handler = rtl8127_interrupt;
+                r8127napi = &tp->r8127napi[0];
+                snprintf(irq->name, len, "%s-0", dev->name);
+                if (!(tp->features & RTL_FEATURE_MSIX))
+                        irq->vector = dev->irq;
+                irq_flags |= (tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX)) ? 0 : SA_SHIRQ;
+                rc = request_irq(irq->vector, irq->handler, irq_flags, irq->name, r8127napi);
+
+                if (rc == 0)
+                        irq->requested = 1;
+        }
+#endif
+        if (rc)
+                rtl8127_free_irq(tp);
+
+        return rc;
+}
+
+static int rtl8127_alloc_tx_desc(struct rtl8127_private *tp)
+{
+        struct rtl8127_tx_ring *ring;
+        struct pci_dev *pdev = tp->pci_dev;
+        int i;
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                ring = &tp->tx_ring[i];
+                ring->TxDescAllocSize = (ring->num_tx_desc + 1) * sizeof(struct TxDesc);
+                ring->TxDescArray = dma_alloc_coherent(&pdev->dev,
+                                                       ring->TxDescAllocSize,
+                                                       &ring->TxPhyAddr,
+                                                       GFP_KERNEL);
+
+                if (!ring->TxDescArray)
+                        return -1;
+        }
+
+        return 0;
+}
+
+static int rtl8127_alloc_rx_desc(struct rtl8127_private *tp)
+{
+        struct rtl8127_rx_ring *ring;
+        struct pci_dev *pdev = tp->pci_dev;
+        int i;
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                ring = &tp->rx_ring[i];
+                ring->RxDescAllocSize = (ring->num_rx_desc + 1) * tp->RxDescLength;
+                ring->RxDescArray = dma_alloc_coherent(&pdev->dev,
+                                                       ring->RxDescAllocSize,
+                                                       &ring->RxPhyAddr,
+                                                       GFP_KERNEL);
+
+                if (!ring->RxDescArray)
+                        return -1;
+        }
+
+        return 0;
+}
+
+static void rtl8127_free_tx_desc(struct rtl8127_private *tp)
+{
+        struct rtl8127_tx_ring *ring;
+        struct pci_dev *pdev = tp->pci_dev;
+        int i;
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                ring = &tp->tx_ring[i];
+                if (ring->TxDescArray) {
+                        dma_free_coherent(&pdev->dev,
+                                          ring->TxDescAllocSize,
+                                          ring->TxDescArray,
+                                          ring->TxPhyAddr);
+                        ring->TxDescArray = NULL;
+                }
+        }
+}
+
+static void rtl8127_free_rx_desc(struct rtl8127_private *tp)
+{
+        struct rtl8127_rx_ring *ring;
+        struct pci_dev *pdev = tp->pci_dev;
+        int i;
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                ring = &tp->rx_ring[i];
+                if (ring->RxDescArray) {
+                        dma_free_coherent(&pdev->dev,
+                                          ring->RxDescAllocSize,
+                                          ring->RxDescArray,
+                                          ring->RxPhyAddr);
+                        ring->RxDescArray = NULL;
+                }
+        }
+}
+
+static void rtl8127_free_alloc_resources(struct rtl8127_private *tp)
+{
+        rtl8127_free_rx_desc(tp);
+
+        rtl8127_free_tx_desc(tp);
+}
+
+#ifdef ENABLE_USE_FIRMWARE_FILE
+static void rtl8127_request_firmware(struct rtl8127_private *tp)
+{
+        struct rtl8127_fw *rtl_fw;
+
+        /* firmware loaded already or no firmware available */
+        if (tp->rtl_fw || !tp->fw_name)
+                return;
+
+        rtl_fw = kzalloc(sizeof(*rtl_fw), GFP_KERNEL);
+        if (!rtl_fw)
+                return;
+
+        rtl_fw->phy_write = rtl8127_mdio_write;
+        rtl_fw->phy_read = rtl8127_mdio_read;
+        rtl_fw->mac_mcu_write = mac_mcu_write;
+        rtl_fw->mac_mcu_read = mac_mcu_read;
+        rtl_fw->fw_name = tp->fw_name;
+        rtl_fw->dev = tp_to_dev(tp);
+
+        if (rtl8127_fw_request_firmware(rtl_fw))
+                kfree(rtl_fw);
+        else
+                tp->rtl_fw = rtl_fw;
+}
+#endif
+
+int rtl8127_open(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int retval;
+
+        retval = -ENOMEM;
+
+#ifdef ENABLE_R8127_PROCFS
+        rtl8127_proc_init(dev);
+#endif
+        rtl8127_set_rxbufsize(tp, dev);
+        /*
+         * Rx and Tx descriptors needs 256 bytes alignment.
+         * pci_alloc_consistent provides more.
+         */
+        if (rtl8127_alloc_tx_desc(tp) < 0 || rtl8127_alloc_rx_desc(tp) < 0)
+                goto err_free_all_allocated_mem;
+
+        retval = rtl8127_init_ring(dev);
+        if (retval < 0)
+                goto err_free_all_allocated_mem;
+
+        retval = rtl8127_alloc_irq(tp);
+        if (retval < 0)
+                goto err_free_all_allocated_mem;
+
+        if (netif_msg_probe(tp)) {
+                printk(KERN_INFO "%s: 0x%lx, "
+                       "%2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x, "
+                       "IRQ %d\n",
+                       dev->name,
+                       dev->base_addr,
+                       dev->dev_addr[0], dev->dev_addr[1],
+                       dev->dev_addr[2], dev->dev_addr[3],
+                       dev->dev_addr[4], dev->dev_addr[5], dev->irq);
+        }
+
+#ifdef ENABLE_USE_FIRMWARE_FILE
+        rtl8127_request_firmware(tp);
+#endif
+        pci_set_master(tp->pci_dev);
+
+#ifdef  CONFIG_R8127_NAPI
+        rtl8127_enable_napi(tp);
+#endif
+
+        rtl8127_exit_oob(dev);
+
+        rtl8127_up(dev);
+
+#ifdef ENABLE_PTP_SUPPORT
+        if (tp->EnablePtp)
+                rtl8127_ptp_init(tp);
+#endif
+        clear_bit(R8127_FLAG_DOWN, tp->task_flags);
+
+        if (tp->resume_not_chg_speed)
+                _rtl8127_check_link_status(dev, R8127_LINK_STATE_UNKNOWN);
+        else
+                rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising);
+
+        if (tp->esd_flag == 0) {
+                //rtl8127_request_esd_timer(dev);
+
+                rtl8127_schedule_esd_work(tp);
+        }
+
+        //rtl8127_request_link_timer(dev);
+
+        rtl8127_enable_hw_linkchg_interrupt(tp);
+
+out:
+
+        return retval;
+
+err_free_all_allocated_mem:
+        rtl8127_free_alloc_resources(tp);
+
+        goto out;
+}
+
+static void
+_rtl8127_set_l1_l0s_entry_latency(struct rtl8127_private *tp, u8 setting)
+{
+        u32 csi_tmp;
+        u32 temp;
+
+        temp = setting & 0x3f;
+        temp <<= 24;
+        /*set PCI configuration space offset 0x70F to setting*/
+        /*When the register offset of PCI configuration space larger than 0xff, use CSI to access it.*/
+
+        csi_tmp = rtl8127_csi_read(tp, 0x70c) & 0xc0ffffff;
+        rtl8127_csi_write(tp, 0x70c, csi_tmp | temp);
+}
+
+static void
+rtl8127_set_l1_l0s_entry_latency(struct rtl8127_private *tp)
+{
+        _rtl8127_set_l1_l0s_entry_latency(tp, 0x27);
+}
+
+static void
+_rtl8127_set_mrrs(struct rtl8127_private *tp, u8 setting)
+{
+        //Set PCI configuration space offset 0x79 to setting
+
+        struct pci_dev *pdev = tp->pci_dev;
+        u8 device_control;
+
+        pci_read_config_byte(pdev, 0x79, &device_control);
+        device_control &= ~0x70;
+        device_control |= setting;
+        pci_write_config_byte(pdev, 0x79, device_control);
+}
+
+static void
+rtl8127_set_mrrs(struct rtl8127_private *tp)
+{
+        if (hwoptimize & HW_PATCH_SOC_LAN)
+                return;
+
+        _rtl8127_set_mrrs(tp, 0x40);
+}
+
+static void
+rtl8127_disable_l1_timeout(struct rtl8127_private *tp)
+{
+        rtl8127_csi_write(tp, 0x890, rtl8127_csi_read(tp, 0x890) & ~BIT(0));
+}
+
+void
+rtl8127_hw_set_rx_packet_filter(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        u32 mc_filter[2];   /* Multicast hash filter */
+        int rx_mode;
+        u32 tmp = 0;
+
+        if (dev->flags & IFF_PROMISC) {
+                /* Unconditionally log net taps. */
+                if (netif_msg_link(tp))
+                        printk(KERN_NOTICE "%s: Promiscuous mode enabled.\n",
+                               dev->name);
+
+                rx_mode =
+                        AcceptBroadcast | AcceptMulticast | AcceptMyPhys |
+                        AcceptAllPhys;
+                mc_filter[1] = mc_filter[0] = 0xffffffff;
+        } else if (dev->flags & IFF_ALLMULTI) {
+                /* accept all multicasts. */
+                rx_mode = AcceptBroadcast | AcceptMulticast | AcceptMyPhys;
+                mc_filter[1] = mc_filter[0] = 0xffffffff;
+        } else {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35)
+                struct dev_mc_list *mclist;
+                unsigned int i;
+
+                rx_mode = AcceptBroadcast | AcceptMyPhys;
+                mc_filter[1] = mc_filter[0] = 0;
+                for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count;
+                     i++, mclist = mclist->next) {
+                        int bit_nr = ether_crc(ETH_ALEN, mclist->dmi_addr) >> 26;
+                        mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
+                        rx_mode |= AcceptMulticast;
+                }
+#else
+                struct netdev_hw_addr *ha;
+
+                rx_mode = AcceptBroadcast | AcceptMyPhys;
+                mc_filter[1] = mc_filter[0] = 0;
+                netdev_for_each_mc_addr(ha, dev) {
+                        int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26;
+                        mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
+                        rx_mode |= AcceptMulticast;
+                }
+#endif
+        }
+
+        if (dev->features & NETIF_F_RXALL)
+                rx_mode |= (AcceptErr | AcceptRunt);
+
+        tmp = mc_filter[0];
+        mc_filter[0] = swab32(mc_filter[1]);
+        mc_filter[1] = swab32(tmp);
+
+        tmp = tp->rtl8127_rx_config | rx_mode | (RTL_R32(tp, RxConfig) & rtl_chip_info[tp->chipset].RxConfigMask);
+
+        RTL_W32(tp, RxConfig, tmp);
+        RTL_W32(tp, MAR0 + 0, mc_filter[0]);
+        RTL_W32(tp, MAR0 + 4, mc_filter[1]);
+}
+
+static void
+rtl8127_set_rx_mode(struct net_device *dev)
+{
+        rtl8127_hw_set_rx_packet_filter(dev);
+}
+
+void
+rtl8127_set_rx_q_num(struct rtl8127_private *tp,
+                     unsigned int num_rx_queues)
+{
+        u16 q_ctrl;
+        u16 rx_q_num;
+
+        rx_q_num = (u16)ilog2(num_rx_queues);
+        rx_q_num &= (BIT_0 | BIT_1 | BIT_2);
+        rx_q_num <<= 2;
+        q_ctrl = RTL_R16(tp, Q_NUM_CTRL_8125);
+        q_ctrl &= ~(BIT_2 | BIT_3 | BIT_4);
+        q_ctrl |= rx_q_num;
+        RTL_W16(tp, Q_NUM_CTRL_8125, q_ctrl);
+}
+
+void
+rtl8127_set_tx_q_num(struct rtl8127_private *tp,
+                     unsigned int num_tx_queues)
+{
+        u16 mac_ocp_data;
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE63E);
+        mac_ocp_data &= ~(BIT_11 | BIT_10);
+        mac_ocp_data |= ((ilog2(num_tx_queues) & 0x03) << 10);
+        rtl8127_mac_ocp_write(tp, 0xE63E, mac_ocp_data);
+}
+
+void
+rtl8127_enable_mcu(struct rtl8127_private *tp, bool enable)
+{
+        if (FALSE == HW_SUPPORT_MAC_MCU(tp))
+                return;
+
+        if (enable)
+                rtl8127_set_mac_ocp_bit(tp, 0xC0B4, BIT_0);
+        else
+                rtl8127_clear_mac_ocp_bit(tp, 0xC0B4, BIT_0);
+}
+
+static void
+rtl8127_clear_tcam_entries(struct rtl8127_private *tp)
+{
+        if (FALSE == HW_SUPPORT_TCAM(tp))
+                return;
+
+        rtl8127_set_mac_ocp_bit(tp, 0xEB54, BIT_0);
+        udelay(1);
+        rtl8127_clear_mac_ocp_bit(tp, 0xEB54, BIT_0);
+}
+
+static u8
+rtl8127_get_l1off_cap_bits(struct rtl8127_private *tp)
+{
+        u8 l1offCapBits = 0;
+
+        l1offCapBits = (BIT_0 | BIT_1);
+        l1offCapBits |= (BIT_2 | BIT_3);
+
+        return l1offCapBits;
+}
+
+void
+rtl8127_hw_config(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        struct pci_dev *pdev = tp->pci_dev;
+        u16 mac_ocp_data;
+
+        rtl8127_disable_rx_packet_filter(tp);
+
+        rtl8127_hw_reset(dev);
+
+        rtl8127_enable_cfg9346_write(tp);
+        rtl8127_enable_force_clkreq(tp, 0);
+        rtl8127_enable_aspm_clkreq_lock(tp, 0);
+
+        rtl8127_set_eee_lpi_timer(tp);
+
+        //keep magic packet only
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xC0B6);
+        mac_ocp_data &= BIT_0;
+        rtl8127_mac_ocp_write(tp, 0xC0B6, mac_ocp_data);
+
+        rtl8127_tally_counter_addr_fill(tp);
+
+        rtl8127_enable_extend_tally_couter(tp);
+
+        rtl8127_desc_addr_fill(tp);
+
+        /* Set DMA burst size and Interframe Gap Time */
+        RTL_W32(tp, TxConfig, (TX_DMA_BURST_unlimited << TxDMAShift) |
+                (InterFrameGap << TxInterFrameGapShift));
+
+        if (tp->EnableTxNoClose)
+                RTL_W32(tp, TxConfig, (RTL_R32(tp, TxConfig) | BIT_6));
+
+        if (enable_double_vlan)
+                rtl8127_enable_double_vlan(tp);
+        else
+                rtl8127_disable_double_vlan(tp);
+
+        rtl8127_set_l1_l0s_entry_latency(tp);
+
+        rtl8127_set_mrrs(tp);
+
+        rtl8127_disable_l1_timeout(tp);
+
+#ifdef ENABLE_RSS_SUPPORT
+        rtl8127_config_rss(tp);
+#else
+        RTL_W32(tp, RSS_CTRL_8125, 0x00);
+#endif
+        rtl8127_set_rx_q_num(tp, rtl8127_tot_rx_rings(tp));
+
+        RTL_W8(tp, Config1, RTL_R8(tp, Config1) & ~0x10);
+
+        rtl8127_mac_ocp_write(tp, 0xC140, 0xFFFF);
+        rtl8127_mac_ocp_write(tp, 0xC142, 0xFFFF);
+
+        //new tx desc format
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB58);
+        mac_ocp_data &= ~(BIT_0 | BIT_1);
+        mac_ocp_data |= (BIT_0);
+        rtl8127_mac_ocp_write(tp, 0xEB58, mac_ocp_data);
+
+        if (tp->EnableTxNoClose)
+                RTL_W8(tp, 0x20E4, RTL_R8(tp, 0x20E4) | BIT_2);
+        else
+                RTL_W8(tp, 0x20E4, RTL_R8(tp, 0x20E4) & ~BIT_2);
+
+        if (tp->HwSuppRxDescType == RX_DESC_RING_TYPE_4) {
+                if (tp->InitRxDescType == RX_DESC_RING_TYPE_4)
+                        RTL_W8(tp, 0xd8, RTL_R8(tp, 0xd8) |
+                               EnableRxDescV4_0);
+                else
+                        RTL_W8(tp, 0xd8, RTL_R8(tp, 0xd8) &
+                               ~EnableRxDescV4_0);
+        }
+
+        if (tp->mcfg == CFG_METHOD_2) {
+                rtl8127_clear_mac_ocp_bit(tp, 0xE00C, BIT_12);
+
+                rtl8127_clear_mac_ocp_bit(tp, 0xC0C2, BIT_6);
+        }
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE614);
+        mac_ocp_data &= ~(BIT_11 | BIT_10 | BIT_9 | BIT_8);
+        mac_ocp_data |= (15 << 8);
+        rtl8127_mac_ocp_write(tp, 0xE614, mac_ocp_data);
+
+        rtl8127_set_tx_q_num(tp, rtl8127_tot_tx_rings(tp));
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE63E);
+        mac_ocp_data &= ~(BIT_5 | BIT_4);
+        mac_ocp_data |= ((0x02 & 0x03) << 4);
+        rtl8127_mac_ocp_write(tp, 0xE63E, mac_ocp_data);
+
+        rtl8127_enable_mcu(tp, 0);
+        rtl8127_enable_mcu(tp, 1);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xC0B4);
+        mac_ocp_data |= (BIT_3 | BIT_2);
+        rtl8127_mac_ocp_write(tp, 0xC0B4, mac_ocp_data);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB6A);
+        mac_ocp_data &= ~(BIT_7 | BIT_6 | BIT_5 | BIT_4 | BIT_3 | BIT_2 | BIT_1 | BIT_0);
+        mac_ocp_data |= (BIT_5 | BIT_4 | BIT_1 | BIT_0);
+        rtl8127_mac_ocp_write(tp, 0xEB6A, mac_ocp_data);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEB50);
+        mac_ocp_data &= ~(BIT_9 | BIT_8 | BIT_7 | BIT_6 | BIT_5);
+        mac_ocp_data |= (BIT_6);
+        rtl8127_mac_ocp_write(tp, 0xEB50, mac_ocp_data);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE056);
+        mac_ocp_data &= ~(BIT_7 | BIT_6 | BIT_5 | BIT_4);
+        //mac_ocp_data |= (BIT_4 | BIT_5);
+        rtl8127_mac_ocp_write(tp, 0xE056, mac_ocp_data);
+
+        RTL_W8(tp, TDFNR, 0x10);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xE040);
+        mac_ocp_data &= ~(BIT_12);
+        rtl8127_mac_ocp_write(tp, 0xE040, mac_ocp_data);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEA1C);
+        mac_ocp_data &= ~(BIT_1 | BIT_0);
+        mac_ocp_data |= (BIT_0);
+        rtl8127_mac_ocp_write(tp, 0xEA1C, mac_ocp_data);
+
+        rtl8127_mac_ocp_write(tp, 0xE0C0, 0x4000);
+
+        rtl8127_set_mac_ocp_bit(tp, 0xE052, (BIT_6 | BIT_5));
+        rtl8127_clear_mac_ocp_bit(tp, 0xE052, BIT_3 | BIT_7);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xD430);
+        mac_ocp_data &= ~(BIT_11 | BIT_10 | BIT_9 | BIT_8 | BIT_7 | BIT_6 | BIT_5 | BIT_4 | BIT_3 | BIT_2 | BIT_1 | BIT_0);
+        mac_ocp_data |= 0x45F;
+        rtl8127_mac_ocp_write(tp, 0xD430, mac_ocp_data);
+
+        //rtl8127_mac_ocp_write(tp, 0xE0C0, 0x4F87);
+        if (!tp->DASH)
+                RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) | BIT_6 | BIT_7);
+        else
+                RTL_W8(tp, 0xD0, RTL_R8(tp, 0xD0) & ~(BIT_6 | BIT_7));
+
+        rtl8127_disable_eee_plus(tp);
+
+        mac_ocp_data = rtl8127_mac_ocp_read(tp, 0xEA1C);
+        mac_ocp_data &= ~(BIT_2);
+        mac_ocp_data &= ~(BIT_9 | BIT_8);
+        rtl8127_mac_ocp_write(tp, 0xEA1C, mac_ocp_data);
+
+        rtl8127_clear_tcam_entries(tp);
+
+        RTL_W16(tp, 0x1880, RTL_R16(tp, 0x1880) & ~(BIT_4 | BIT_5));
+
+        rtl8127_clear_set_mac_ocp_bit(tp, 0xD40C, 0xE038, 0x8020);
+
+        /* csum offload command for RTL8125 */
+        tp->tx_tcp_csum_cmd = TxTCPCS_C;
+        tp->tx_udp_csum_cmd = TxUDPCS_C;
+        tp->tx_ip_csum_cmd = TxIPCS_C;
+        tp->tx_ipv6_csum_cmd = TxIPV6F_C;
+
+        /* config interrupt type for RTL8125B */
+        if (tp->HwSuppIsrVer > 1)
+                rtl8127_hw_set_interrupt_type(tp, tp->HwCurrIsrVer);
+
+        //other hw parameters
+        rtl8127_hw_clear_timer_int(dev);
+
+        rtl8127_hw_clear_int_miti(dev);
+
+        if (tp->use_timer_interrupt &&
+            (tp->HwCurrIsrVer > 1) &&
+            (tp->HwSuppIntMitiVer > 3) &&
+            (tp->features & RTL_FEATURE_MSIX)) {
+                int i;
+                for (i = 0; i < tp->irq_nvecs; i++)
+                        rtl8127_hw_set_timer_int(tp, i, timer_count_v2);
+        }
+
+        rtl8127_enable_exit_l1_mask(tp);
+
+        rtl8127_mac_ocp_write(tp, 0xE098, 0xC302);
+
+        if (aspm && (tp->org_pci_offset_99 & (BIT_2 | BIT_5 | BIT_6)))
+                rtl8127_init_pci_offset_99(tp);
+        else
+                rtl8127_disable_pci_offset_99(tp);
+
+        if (aspm && (tp->org_pci_offset_180 & rtl8127_get_l1off_cap_bits(tp)))
+                rtl8127_init_pci_offset_180(tp);
+        else
+                rtl8127_disable_pci_offset_180(tp);
+
+        tp->cp_cmd &= ~(EnableBist | Macdbgo_oe | Force_halfdup |
+                        Force_rxflow_en | Force_txflow_en | Cxpl_dbg_sel |
+                        ASF | Macdbgo_sel);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,0,0)
+        RTL_W16(tp, CPlusCmd, tp->cp_cmd);
+#else
+        rtl8127_hw_set_features(dev, dev->features);
+#endif
+        rtl8127_set_rms(tp, tp->rms);
+
+        rtl8127_disable_rxdvgate(dev);
+
+        if (!tp->pci_cfg_is_read) {
+                pci_read_config_byte(pdev, PCI_COMMAND, &tp->pci_cfg_space.cmd);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_0, &tp->pci_cfg_space.io_base_l);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_0 + 2, &tp->pci_cfg_space.io_base_h);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_2, &tp->pci_cfg_space.mem_base_l);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_2 + 2, &tp->pci_cfg_space.mem_base_h);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_3, &tp->pci_cfg_space.resv_0x1c_l);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_3 + 2, &tp->pci_cfg_space.resv_0x1c_h);
+                pci_read_config_byte(pdev, PCI_INTERRUPT_LINE, &tp->pci_cfg_space.ilr);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_4, &tp->pci_cfg_space.resv_0x20_l);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_4 + 2, &tp->pci_cfg_space.resv_0x20_h);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_5, &tp->pci_cfg_space.resv_0x24_l);
+                pci_read_config_word(pdev, PCI_BASE_ADDRESS_5 + 2, &tp->pci_cfg_space.resv_0x24_h);
+                pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &tp->pci_cfg_space.resv_0x2c_l);
+                pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID + 2, &tp->pci_cfg_space.resv_0x2c_h);
+                if (tp->HwPcieSNOffset > 0) {
+                        tp->pci_cfg_space.pci_sn_l = rtl8127_csi_read(tp, tp->HwPcieSNOffset);
+                        tp->pci_cfg_space.pci_sn_h = rtl8127_csi_read(tp, tp->HwPcieSNOffset + 4);
+                }
+
+                tp->pci_cfg_is_read = 1;
+        }
+
+        /* Set Rx packet filter */
+        rtl8127_hw_set_rx_packet_filter(dev);
+
+#ifdef ENABLE_DASH_SUPPORT
+        rtl8127_check_and_enable_dash_interrupt(tp);
+#endif
+
+        rtl8127_set_radm_fifo_prot(tp, 1);
+
+        rtl8127_enable_aspm_clkreq_lock(tp, aspm ? 1 : 0);
+
+        rtl8127_disable_cfg9346_write(tp);
+
+        udelay(10);
+}
+
+void
+rtl8127_hw_start(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+#ifdef ENABLE_LIB_SUPPORT
+        rtl8127_init_lib_ring(tp);
+#endif
+
+        RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
+
+        rtl8127_enable_hw_interrupt(tp);
+
+        rtl8127_lib_reset_complete(tp);
+}
+
+static int
+rtl8127_change_mtu(struct net_device *dev,
+                   int new_mtu)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ret = 0;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0)
+        if (new_mtu < ETH_MIN_MTU)
+                return -EINVAL;
+        else if (new_mtu > tp->max_jumbo_frame_size)
+                new_mtu = tp->max_jumbo_frame_size;
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,10,0)
+
+        dev->mtu = new_mtu;
+
+        tp->eee.tx_lpi_timer = dev->mtu + ETH_HLEN + 0x20;
+
+        if (!netif_running(dev))
+                goto out;
+
+        rtl8127_down(dev);
+
+        rtl8127_set_rxbufsize(tp, dev);
+
+        ret = rtl8127_init_ring(dev);
+
+        if (ret < 0)
+                goto err_out;
+
+#ifdef CONFIG_R8127_NAPI
+        rtl8127_enable_napi(tp);
+#endif//CONFIG_R8127_NAPI
+
+        if (tp->link_ok(dev))
+                rtl8127_link_on_patch(dev);
+        else
+                rtl8127_link_down_patch(dev);
+
+        //mod_timer(&tp->esd_timer, jiffies + RTL8127_ESD_TIMEOUT);
+        //mod_timer(&tp->link_timer, jiffies + RTL8127_LINK_TIMEOUT);
+out:
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,0)
+        netdev_update_features(dev);
+#endif
+
+err_out:
+        return ret;
+}
+
+static inline void
+rtl8127_set_desc_dma_addr(struct rtl8127_private *tp,
+                          struct RxDesc *desc,
+                          dma_addr_t mapping)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                ((struct RxDescV3 *)desc)->addr = cpu_to_le64(mapping);
+                break;
+        case RX_DESC_RING_TYPE_4:
+                ((struct RxDescV4 *)desc)->addr = cpu_to_le64(mapping);
+                break;
+        default:
+                desc->addr = cpu_to_le64(mapping);
+                break;
+        }
+}
+
+static inline void
+rtl8127_mark_to_asic_v1(struct RxDesc *desc,
+                        u32 rx_buf_sz)
+{
+        u32 eor = le32_to_cpu(desc->opts1) & RingEnd;
+
+        WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz));
+}
+
+static inline void
+rtl8127_mark_to_asic_v3(struct RxDescV3 *descv3,
+                        u32 rx_buf_sz)
+{
+        u32 eor = le32_to_cpu(descv3->RxDescNormalDDWord4.opts1) & RingEnd;
+
+        WRITE_ONCE(descv3->RxDescNormalDDWord4.opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz));
+}
+
+static inline void
+rtl8127_mark_to_asic_v4(struct RxDescV4 *descv4,
+                        u32 rx_buf_sz)
+{
+        u32 eor = le32_to_cpu(descv4->RxDescNormalDDWord2.opts1) & RingEnd;
+
+        WRITE_ONCE(descv4->RxDescNormalDDWord2.opts1, cpu_to_le32(DescOwn | eor | rx_buf_sz));
+}
+
+void
+rtl8127_mark_to_asic(struct rtl8127_private *tp,
+                     struct RxDesc *desc,
+                     u32 rx_buf_sz)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                rtl8127_mark_to_asic_v3((struct RxDescV3 *)desc, rx_buf_sz);
+                break;
+        case RX_DESC_RING_TYPE_4:
+                rtl8127_mark_to_asic_v4((struct RxDescV4 *)desc, rx_buf_sz);
+                break;
+        default:
+                rtl8127_mark_to_asic_v1(desc, rx_buf_sz);
+                break;
+        }
+}
+
+static inline void
+rtl8127_map_to_asic(struct rtl8127_private *tp,
+                    struct rtl8127_rx_ring *ring,
+                    struct RxDesc *desc,
+                    dma_addr_t mapping,
+                    u32 rx_buf_sz,
+                    const u32 cur_rx)
+{
+        ring->RxDescPhyAddr[cur_rx] = mapping;
+        rtl8127_set_desc_dma_addr(tp, desc, mapping);
+        wmb();
+        rtl8127_mark_to_asic(tp, desc, rx_buf_sz);
+}
+
+#ifdef ENABLE_PAGE_REUSE
+
+static int
+rtl8127_alloc_rx_page(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring,
+                      struct rtl8127_rx_buffer *rxb)
+{
+        struct page *page;
+        dma_addr_t dma;
+        unsigned int order = tp->rx_buf_page_order;
+
+        //get free page
+        page = dev_alloc_pages(order);
+
+        if (unlikely(!page))
+                return -ENOMEM;
+
+        dma = dma_map_page_attrs(&tp->pci_dev->dev, page, 0,
+                                 tp->rx_buf_page_size,
+                                 DMA_FROM_DEVICE,
+                                 (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING));
+
+        if (unlikely(dma_mapping_error(&tp->pci_dev->dev, dma))) {
+                __free_pages(page, order);
+                return -ENOMEM;
+        }
+
+        rxb->page = page;
+        rxb->data = page_address(page);
+        rxb->page_offset = ring->rx_offset;
+        rxb->dma = dma;
+
+        //after page alloc, page refcount already = 1
+
+        return 0;
+}
+
+static void
+rtl8127_free_rx_page(struct rtl8127_private *tp, struct rtl8127_rx_buffer *rxb)
+{
+        if (!rxb->page)
+                return;
+
+        dma_unmap_page_attrs(&tp->pci_dev->dev, rxb->dma,
+                             tp->rx_buf_page_size,
+                             DMA_FROM_DEVICE,
+                             (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING));
+        __free_pages(rxb->page, tp->rx_buf_page_order);
+        rxb->page = NULL;
+}
+
+static void
+_rtl8127_rx_clear(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring)
+{
+        int i;
+        struct rtl8127_rx_buffer *rxb;
+
+        for (i = 0; i < ring->num_rx_desc; i++) {
+                rxb = &ring->rx_buffer[i];
+                if (rxb->skb) {
+                        dev_kfree_skb(rxb->skb);
+                        rxb->skb = NULL;
+                }
+                rtl8127_free_rx_page(tp, rxb);
+        }
+}
+
+static u32
+rtl8127_rx_fill(struct rtl8127_private *tp,
+                struct rtl8127_rx_ring *ring,
+                struct net_device *dev,
+                u32 start,
+                u32 end,
+                u8 in_intr)
+{
+        u32 cur;
+        struct rtl8127_rx_buffer *rxb;
+
+        for (cur = start; end - cur > 0; cur++) {
+                int ret, i = cur % ring->num_rx_desc;
+
+                rxb = &ring->rx_buffer[i];
+                if (rxb->page)
+                        continue;
+
+                ret = rtl8127_alloc_rx_page(tp, ring, rxb);
+                if (ret)
+                        break;
+
+                dma_sync_single_range_for_device(tp_to_dev(tp),
+                                                 rxb->dma,
+                                                 rxb->page_offset,
+                                                 tp->rx_buf_sz,
+                                                 DMA_FROM_DEVICE);
+
+                rtl8127_map_to_asic(tp, ring,
+                                    rtl8127_get_rxdesc(tp, ring->RxDescArray, i),
+                                    rxb->dma + rxb->page_offset,
+                                    tp->rx_buf_sz, i);
+        }
+        return cur - start;
+}
+
+#else //ENABLE_PAGE_REUSE
+
+static void
+rtl8127_free_rx_skb(struct rtl8127_private *tp,
+                    struct rtl8127_rx_ring *ring,
+                    struct sk_buff **sk_buff,
+                    struct RxDesc *desc,
+                    const u32 cur_rx)
+{
+        struct pci_dev *pdev = tp->pci_dev;
+
+        dma_unmap_single(&pdev->dev, ring->RxDescPhyAddr[cur_rx], tp->rx_buf_sz,
+                         DMA_FROM_DEVICE);
+        dev_kfree_skb(*sk_buff);
+        *sk_buff = NULL;
+        rtl8127_make_unusable_by_asic(tp, desc);
+}
+
+static int
+rtl8127_alloc_rx_skb(struct rtl8127_private *tp,
+                     struct rtl8127_rx_ring *ring,
+                     struct sk_buff **sk_buff,
+                     struct RxDesc *desc,
+                     int rx_buf_sz,
+                     const u32 cur_rx,
+                     u8 in_intr)
+{
+        struct sk_buff *skb;
+        dma_addr_t mapping;
+        int ret = 0;
+
+        if (in_intr)
+                skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, rx_buf_sz + R8127_RX_ALIGN);
+        else
+                skb = dev_alloc_skb(rx_buf_sz + R8127_RX_ALIGN);
+
+        if (unlikely(!skb))
+                goto err_out;
+
+        if (!in_intr || !R8127_USE_NAPI_ALLOC_SKB)
+                skb_reserve(skb, R8127_RX_ALIGN);
+
+        mapping = dma_map_single(tp_to_dev(tp), skb->data, rx_buf_sz,
+                                 DMA_FROM_DEVICE);
+        if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) {
+                if (unlikely(net_ratelimit()))
+                        netif_err(tp, drv, tp->dev, "Failed to map RX DMA!\n");
+                goto err_out;
+        }
+
+        *sk_buff = skb;
+        rtl8127_map_to_asic(tp, ring, desc, mapping, rx_buf_sz, cur_rx);
+out:
+        return ret;
+
+err_out:
+        if (skb)
+                dev_kfree_skb(skb);
+        ret = -ENOMEM;
+        rtl8127_make_unusable_by_asic(tp, desc);
+        goto out;
+}
+
+static void
+_rtl8127_rx_clear(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring)
+{
+        int i;
+
+        for (i = 0; i < ring->num_rx_desc; i++) {
+                if (ring->Rx_skbuff[i]) {
+                        rtl8127_free_rx_skb(tp,
+                                            ring,
+                                            ring->Rx_skbuff + i,
+                                            rtl8127_get_rxdesc(tp, ring->RxDescArray, i),
+                                            i);
+                        ring->Rx_skbuff[i] = NULL;
+                }
+        }
+}
+
+static u32
+rtl8127_rx_fill(struct rtl8127_private *tp,
+                struct rtl8127_rx_ring *ring,
+                struct net_device *dev,
+                u32 start,
+                u32 end,
+                u8 in_intr)
+{
+        u32 cur;
+
+        for (cur = start; end - cur > 0; cur++) {
+                int ret, i = cur % ring->num_rx_desc;
+
+                if (ring->Rx_skbuff[i])
+                        continue;
+
+                ret = rtl8127_alloc_rx_skb(tp,
+                                           ring,
+                                           ring->Rx_skbuff + i,
+                                           rtl8127_get_rxdesc(tp, ring->RxDescArray, i),
+                                           tp->rx_buf_sz,
+                                           i,
+                                           in_intr);
+                if (ret < 0)
+                        break;
+        }
+        return cur - start;
+}
+
+#endif //ENABLE_PAGE_REUSE
+
+void
+rtl8127_rx_clear(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                struct rtl8127_rx_ring *ring = &tp->rx_ring[i];
+
+                _rtl8127_rx_clear(tp, ring);
+        }
+}
+
+static void
+rtl8127_mark_as_last_descriptor_v1(struct RxDesc *desc)
+{
+        desc->opts1 |= cpu_to_le32(RingEnd);
+}
+
+static void
+rtl8127_mark_as_last_descriptor_v3(struct RxDescV3 *descv3)
+{
+        descv3->RxDescNormalDDWord4.opts1 |= cpu_to_le32(RingEnd);
+}
+
+static void
+rtl8127_mark_as_last_descriptor_v4(struct RxDescV4 *descv4)
+{
+        descv4->RxDescNormalDDWord2.opts1 |= cpu_to_le32(RingEnd);
+}
+
+void
+rtl8127_mark_as_last_descriptor(struct rtl8127_private *tp,
+                                struct RxDesc *desc)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                rtl8127_mark_as_last_descriptor_v3((struct RxDescV3 *)desc);
+                break;
+        case RX_DESC_RING_TYPE_4:
+                rtl8127_mark_as_last_descriptor_v4((struct RxDescV4 *)desc);
+                break;
+        default:
+                rtl8127_mark_as_last_descriptor_v1(desc);
+                break;
+        }
+}
+
+static void
+rtl8127_desc_addr_fill(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                struct rtl8127_tx_ring *ring = &tp->tx_ring[i];
+                RTL_W32(tp, ring->tdsar_reg, ((u64)ring->TxPhyAddr & DMA_BIT_MASK(32)));
+                RTL_W32(tp, ring->tdsar_reg + 4, ((u64)ring->TxPhyAddr >> 32));
+        }
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                struct rtl8127_rx_ring *ring = &tp->rx_ring[i];
+                RTL_W32(tp, ring->rdsar_reg, ((u64)ring->RxPhyAddr & DMA_BIT_MASK(32)));
+                RTL_W32(tp, ring->rdsar_reg + 4, ((u64)ring->RxPhyAddr >> 32));
+        }
+}
+
+static void
+rtl8127_tx_desc_init(struct rtl8127_private *tp)
+{
+        int i = 0;
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                struct rtl8127_tx_ring *ring = &tp->tx_ring[i];
+                memset(ring->TxDescArray, 0x0, ring->TxDescAllocSize);
+
+                ring->TxDescArray[ring->num_tx_desc - 1].opts1 = cpu_to_le32(RingEnd);
+        }
+}
+
+static void
+rtl8127_rx_desc_init(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                struct rtl8127_rx_ring *ring = &tp->rx_ring[i];
+                memset(ring->RxDescArray, 0x0, ring->RxDescAllocSize);
+        }
+}
+
+int
+rtl8127_init_ring(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+
+        rtl8127_init_ring_indexes(tp);
+
+        rtl8127_tx_desc_init(tp);
+        rtl8127_rx_desc_init(tp);
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                struct rtl8127_tx_ring *ring = &tp->tx_ring[i];
+                memset(ring->tx_skb, 0x0, sizeof(ring->tx_skb));
+        }
+
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                struct rtl8127_rx_ring *ring = &tp->rx_ring[i];
+#ifdef ENABLE_PAGE_REUSE
+                ring->rx_offset = R8127_RX_ALIGN;
+#else
+                memset(ring->Rx_skbuff, 0x0, sizeof(ring->Rx_skbuff));
+#endif //ENABLE_PAGE_REUSE
+                if (rtl8127_rx_fill(tp, ring, dev, 0, ring->num_rx_desc, 0) != ring->num_rx_desc)
+                        goto err_out;
+
+                rtl8127_mark_as_last_descriptor(tp, rtl8127_get_rxdesc(tp, ring->RxDescArray, ring->num_rx_desc - 1));
+        }
+
+        return 0;
+
+err_out:
+        rtl8127_rx_clear(tp);
+        return -ENOMEM;
+}
+
+static void
+rtl8127_unmap_tx_skb(struct pci_dev *pdev,
+                     struct ring_info *tx_skb,
+                     struct TxDesc *desc)
+{
+        unsigned int len = tx_skb->len;
+
+        dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len, DMA_TO_DEVICE);
+
+        desc->opts1 = cpu_to_le32(RTK_MAGIC_DEBUG_VALUE);
+        desc->opts2 = 0x00;
+        desc->addr = RTL8127_MAGIC_NUMBER;
+        tx_skb->len = 0;
+}
+
+static void
+rtl8127_tx_clear_range(struct rtl8127_private *tp,
+                       struct rtl8127_tx_ring *ring,
+                       u32 start,
+                       unsigned int n)
+{
+        unsigned int i;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,22)
+        struct net_device *dev = tp->dev;
+#endif
+
+        for (i = 0; i < n; i++) {
+                unsigned int entry = (start + i) % ring->num_tx_desc;
+                struct ring_info *tx_skb = ring->tx_skb + entry;
+                unsigned int len = tx_skb->len;
+
+                if (len) {
+                        struct sk_buff *skb = tx_skb->skb;
+
+                        rtl8127_unmap_tx_skb(tp->pci_dev, tx_skb,
+                                             ring->TxDescArray + entry);
+                        if (skb) {
+                                RTLDEV->stats.tx_dropped++;
+                                dev_kfree_skb_any(skb);
+                                tx_skb->skb = NULL;
+                        }
+                }
+        }
+}
+
+void
+rtl8127_tx_clear(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < tp->num_tx_rings; i++) {
+                struct rtl8127_tx_ring *ring = &tp->tx_ring[i];
+                rtl8127_tx_clear_range(tp, ring, ring->dirty_tx, ring->num_tx_desc);
+                ring->cur_tx = ring->dirty_tx = 0;
+        }
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+static void rtl8127_schedule_reset_work(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        set_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->reset_task, 4);
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+}
+
+static void rtl8127_schedule_esd_work(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        set_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->esd_task, RTL8127_ESD_TIMEOUT);
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+}
+
+static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        set_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->linkchg_task, 4);
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+}
+
+static void rtl8127_schedule_dash_work(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        set_bit(R8125_FLAG_TASK_DASH_CHECK_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->dash_task, RTL8127_DASH_TIMEOUT);
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+}
+
+#define rtl8127_cancel_schedule_reset_work(a)
+#define rtl8127_cancel_schedule_esd_work(a)
+#define rtl8127_cancel_schedule_linkchg_work(a)
+#define rtl8127_cancel_schedule_dash_work(a)
+
+#else
+static void rtl8127_schedule_reset_work(struct rtl8127_private *tp)
+{
+        set_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->reset_task, 4);
+}
+
+static void rtl8127_cancel_schedule_reset_work(struct rtl8127_private *tp)
+{
+        struct work_struct *work = &tp->reset_task.work;
+
+        if (!work->func)
+                return;
+
+        cancel_delayed_work_sync(&tp->reset_task);
+}
+
+static void rtl8127_schedule_esd_work(struct rtl8127_private *tp)
+{
+        set_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->esd_task, RTL8127_ESD_TIMEOUT);
+}
+
+static void rtl8127_cancel_schedule_esd_work(struct rtl8127_private *tp)
+{
+        struct work_struct *work = &tp->esd_task.work;
+
+        if (!work->func)
+                return;
+
+        cancel_delayed_work_sync(&tp->esd_task);
+}
+
+static void rtl8127_schedule_linkchg_work(struct rtl8127_private *tp)
+{
+        set_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->linkchg_task, 4);
+}
+
+static void rtl8127_cancel_schedule_linkchg_work(struct rtl8127_private *tp)
+{
+        struct work_struct *work = &tp->linkchg_task.work;
+
+        if (!work->func)
+                return;
+
+        cancel_delayed_work_sync(&tp->linkchg_task);
+}
+
+void rtl8127_schedule_dash_work(struct rtl8127_private *tp)
+{
+        set_bit(R8127_FLAG_TASK_DASH_CHECK_PENDING, tp->task_flags);
+        schedule_delayed_work(&tp->dash_task, RTL8127_DASH_TIMEOUT);
+}
+
+static void rtl8127_cancel_schedule_dash_work(struct rtl8127_private *tp)
+{
+        struct work_struct *work = &tp->dash_task.work;
+
+        if (!work->func)
+                return;
+
+        cancel_delayed_work_sync(&tp->dash_task);
+}
+#endif
+
+static void rtl8127_init_all_schedule_work(struct rtl8127_private *tp)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+        INIT_WORK(&tp->reset_task, rtl8127_reset_task, dev);
+        INIT_WORK(&tp->esd_task, rtl8127_esd_task, dev);
+        INIT_WORK(&tp->linkchg_task, rtl8127_linkchg_task, dev);
+        INIT_WORK(&tp->dash_task, rtl8127_dash_task, dev);
+#else
+        INIT_DELAYED_WORK(&tp->reset_task, rtl8127_reset_task);
+        INIT_DELAYED_WORK(&tp->esd_task, rtl8127_esd_task);
+        INIT_DELAYED_WORK(&tp->linkchg_task, rtl8127_linkchg_task);
+        INIT_DELAYED_WORK(&tp->dash_task, rtl8127_dash_task);
+#endif
+}
+
+static void rtl8127_cancel_all_schedule_work(struct rtl8127_private *tp)
+{
+        rtl8127_cancel_schedule_reset_work(tp);
+        rtl8127_cancel_schedule_esd_work(tp);
+        rtl8127_cancel_schedule_linkchg_work(tp);
+        rtl8127_cancel_schedule_dash_work(tp);
+}
+
+static void
+rtl8127_wait_for_irq_complete(struct rtl8127_private *tp)
+{
+        if (tp->features & RTL_FEATURE_MSIX) {
+                int i;
+                for (i = 0; i < tp->irq_nvecs; i++)
+                        synchronize_irq(tp->irq_tbl[i].vector);
+        } else {
+                synchronize_irq(tp->dev->irq);
+        }
+}
+
+void
+_rtl8127_wait_for_quiescence(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        /* Wait for any pending NAPI task to complete */
+#ifdef CONFIG_R8127_NAPI
+        rtl8127_disable_napi(tp);
+#endif//CONFIG_R8127_NAPI
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,5,67)
+        /* Give a racing hard_start_xmit a few cycles to complete. */
+        synchronize_net();
+#endif
+
+        rtl8127_irq_mask_and_ack(tp);
+
+        rtl8127_wait_for_irq_complete(tp);
+}
+
+static void
+rtl8127_wait_for_quiescence(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        //suppress unused variable
+        (void)(tp);
+
+        _rtl8127_wait_for_quiescence(dev);
+
+#ifdef CONFIG_R8127_NAPI
+        rtl8127_enable_napi(tp);
+#endif//CONFIG_R8127_NAPI
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+static void rtl8127_reset_task(void *_data)
+{
+        struct net_device *dev = _data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+#else
+static void rtl8127_reset_task(struct work_struct *work)
+{
+        struct rtl8127_private *tp =
+                container_of(work, struct rtl8127_private, reset_task.work);
+        struct net_device *dev = tp->dev;
+#endif
+        int i;
+
+        rtnl_lock();
+
+        if (!netif_running(dev) ||
+            test_bit(R8127_FLAG_DOWN, tp->task_flags) ||
+            !test_and_clear_bit(R8127_FLAG_TASK_RESET_PENDING, tp->task_flags))
+                goto out_unlock;
+
+        netdev_err(dev, "Device reseting!\n");
+
+        netif_carrier_off(dev);
+        netif_tx_disable(dev);
+        _rtl8127_wait_for_quiescence(dev);
+        rtl8127_hw_reset(dev);
+
+        rtl8127_tx_clear(tp);
+
+        rtl8127_init_ring_indexes(tp);
+
+        rtl8127_tx_desc_init(tp);
+        for (i = 0; i < tp->num_rx_rings; i++) {
+                struct rtl8127_rx_ring *ring;
+                u32 entry;
+
+                ring = &tp->rx_ring[i];
+                for (entry = 0; entry < ring->num_rx_desc; entry++) {
+                        struct RxDesc *desc;
+
+                        desc = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry);
+                        rtl8127_mark_to_asic(tp, desc, tp->rx_buf_sz);
+                }
+        }
+
+#ifdef ENABLE_PTP_SUPPORT
+        rtl8127_ptp_reset(tp);
+#endif
+
+#ifdef CONFIG_R8127_NAPI
+        rtl8127_enable_napi(tp);
+#endif //CONFIG_R8127_NAPI
+
+        if (tp->resume_not_chg_speed) {
+                _rtl8127_check_link_status(dev, R8127_LINK_STATE_UNKNOWN);
+
+                tp->resume_not_chg_speed = 0;
+        } else {
+                rtl8127_enable_hw_linkchg_interrupt(tp);
+
+                rtl8127_set_speed(dev, tp->autoneg, tp->speed, tp->duplex, tp->advertising);
+        }
+
+out_unlock:
+        rtnl_unlock();
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+static void rtl8127_esd_task(void *_data)
+{
+        struct net_device *dev = _data;
+        struct rtl8127_private *tp = netdev_priv(dev);
+#else
+static void rtl8127_esd_task(struct work_struct *work)
+{
+        struct rtl8127_private *tp =
+                container_of(work, struct rtl8127_private, esd_task.work);
+        struct net_device *dev = tp->dev;
+#endif
+        rtnl_lock();
+
+        if (!netif_running(dev) ||
+            test_bit(R8127_FLAG_DOWN, tp->task_flags) ||
+            !test_and_clear_bit(R8127_FLAG_TASK_ESD_CHECK_PENDING, tp->task_flags))
+                goto out_unlock;
+
+        rtl8127_esd_checker(tp);
+
+        rtl8127_schedule_esd_work(tp);
+
+out_unlock:
+        rtnl_unlock();
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+static void rtl8127_linkchg_task(void *_data)
+{
+        struct net_device *dev = _data;
+        //struct rtl8127_private *tp = netdev_priv(dev);
+#else
+static void rtl8127_linkchg_task(struct work_struct *work)
+{
+        struct rtl8127_private *tp =
+                container_of(work, struct rtl8127_private, linkchg_task.work);
+        struct net_device *dev = tp->dev;
+#endif
+        rtnl_lock();
+
+        if (!netif_running(dev) ||
+            test_bit(R8127_FLAG_DOWN, tp->task_flags) ||
+            !test_and_clear_bit(R8127_FLAG_TASK_LINKCHG_CHECK_PENDING, tp->task_flags))
+                goto out_unlock;
+
+        rtl8127_check_link_status(dev);
+
+out_unlock:
+        rtnl_unlock();
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,20)
+static void rtl8127_dash_task(void *_data)
+{
+        struct net_device *dev = _data;
+        //struct rtl8127_private *tp = netdev_priv(dev);
+#else
+static void rtl8127_dash_task(struct work_struct *work)
+{
+        struct rtl8127_private *tp =
+                container_of(work, struct rtl8127_private, dash_task.work);
+        struct net_device *dev = tp->dev;
+#endif
+        rtnl_lock();
+
+        if (!netif_running(dev) ||
+            test_bit(R8127_FLAG_DOWN, tp->task_flags) ||
+            !test_and_clear_bit(R8127_FLAG_TASK_DASH_CHECK_PENDING, tp->task_flags))
+                goto out_unlock;
+
+#ifdef ENABLE_DASH_SUPPORT
+        rtl8127_handle_dash_interrupt(dev);
+#endif
+
+out_unlock:
+        rtnl_unlock();
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,6,0)
+static void
+rtl8127_tx_timeout(struct net_device *dev, unsigned int txqueue)
+#else
+static void
+rtl8127_tx_timeout(struct net_device *dev)
+#endif
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        netdev_err(dev, "Transmit timeout reset Device!\n");
+
+        /* Let's wait a bit while any (async) irq lands on */
+        rtl8127_schedule_reset_work(tp);
+}
+
+static u32
+rtl8127_get_txd_opts1(struct rtl8127_tx_ring *ring,
+                      u32 opts1,
+                      u32 len,
+                      unsigned int entry)
+{
+        u32 status = opts1 | len;
+
+        if (entry == ring->num_tx_desc - 1)
+                status |= RingEnd;
+
+        return status;
+}
+
+static int
+rtl8127_xmit_frags(struct rtl8127_private *tp,
+                   struct rtl8127_tx_ring *ring,
+                   struct sk_buff *skb,
+                   const u32 *opts)
+{
+        struct skb_shared_info *info = skb_shinfo(skb);
+        unsigned int cur_frag, entry;
+        struct TxDesc *txd = NULL;
+        const unsigned char nr_frags = info->nr_frags;
+        unsigned long PktLenCnt = 0;
+        bool LsoPatchEnabled = FALSE;
+
+        entry = ring->cur_tx;
+        for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) {
+                skb_frag_t *frag = info->frags + cur_frag;
+                dma_addr_t mapping;
+                u32 status, len;
+                void *addr;
+
+                entry = (entry + 1) % ring->num_tx_desc;
+
+                txd = ring->TxDescArray + entry;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,2,0)
+                len = frag->size;
+                addr = ((void *) page_address(frag->page)) + frag->page_offset;
+#else
+                len = skb_frag_size(frag);
+                addr = skb_frag_address(frag);
+#endif
+                mapping = dma_map_single(tp_to_dev(tp), addr, len, DMA_TO_DEVICE);
+
+                if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) {
+                        if (unlikely(net_ratelimit()))
+                                netif_err(tp, drv, tp->dev,
+                                          "Failed to map TX fragments DMA!\n");
+                        goto err_out;
+                }
+
+                /* anti gcc 2.95.3 bugware (sic) */
+                status = rtl8127_get_txd_opts1(ring, opts[0], len, entry);
+                if (cur_frag == (nr_frags - 1) || LsoPatchEnabled == TRUE)
+                        status |= LastFrag;
+
+                txd->addr = cpu_to_le64(mapping);
+
+                ring->tx_skb[entry].len = len;
+
+                txd->opts2 = cpu_to_le32(opts[1]);
+                wmb();
+                txd->opts1 = cpu_to_le32(status);
+
+                PktLenCnt += len;
+        }
+
+        return cur_frag;
+
+err_out:
+        rtl8127_tx_clear_range(tp, ring, ring->cur_tx + 1, cur_frag);
+        return -EIO;
+}
+
+static inline
+__be16 get_protocol(struct sk_buff *skb)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37)
+        return vlan_get_protocol(skb);
+#else
+        __be16 protocol;
+
+        if (skb->protocol == htons(ETH_P_8021Q))
+                protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
+        else
+                protocol = skb->protocol;
+
+        return protocol;
+#endif
+}
+
+static inline
+u8 rtl8127_get_l4_protocol(struct sk_buff *skb)
+{
+        int no = skb_network_offset(skb);
+        struct ipv6hdr *i6h, _i6h;
+        struct iphdr *ih, _ih;
+        u8 ip_protocol = IPPROTO_RAW;
+
+        switch (get_protocol(skb)) {
+        case  __constant_htons(ETH_P_IP):
+                ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih);
+                if (ih)
+                        ip_protocol = ih->protocol;
+                break;
+        case  __constant_htons(ETH_P_IPV6):
+                i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h);
+                if (i6h)
+                        ip_protocol = i6h->nexthdr;
+                break;
+        }
+
+        return ip_protocol;
+}
+
+static bool rtl8127_skb_pad_with_len(struct sk_buff *skb, unsigned int len)
+{
+        if (skb_padto(skb, len))
+                return false;
+        skb_put(skb, len - skb->len);
+        return true;
+}
+
+static bool rtl8127_skb_pad(struct sk_buff *skb)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0)
+        return rtl8127_skb_pad_with_len(skb, ETH_ZLEN);
+#else
+        return !eth_skb_pad(skb);
+#endif
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)
+/* msdn_giant_send_check()
+ * According to the document of microsoft, the TCP Pseudo Header excludes the
+ * packet length for IPv6 TCP large packets.
+ */
+static int msdn_giant_send_check(struct sk_buff *skb)
+{
+        const struct ipv6hdr *ipv6h;
+        struct tcphdr *th;
+        int ret;
+
+        ret = skb_cow_head(skb, 0);
+        if (ret)
+                return ret;
+
+        ipv6h = ipv6_hdr(skb);
+        th = tcp_hdr(skb);
+
+        th->check = 0;
+        th->check = ~tcp_v6_check(0, &ipv6h->saddr, &ipv6h->daddr, 0);
+
+        return ret;
+}
+#endif
+
+static bool rtl8127_require_pad_ptp_pkt(struct rtl8127_private *tp)
+{
+        return false;
+}
+
+#define MIN_PATCH_LEN (47)
+static u32
+rtl8127_get_patch_pad_len(struct rtl8127_private *tp,
+                          struct sk_buff *skb)
+{
+        u32 pad_len = 0;
+        int trans_data_len;
+        u32 hdr_len;
+        u32 pkt_len = skb->len;
+        u8 ip_protocol;
+        bool has_trans = skb_transport_header_was_set(skb);
+
+        if (!rtl8127_require_pad_ptp_pkt(tp))
+                goto no_padding;
+
+        if (!(has_trans && (pkt_len < 175))) //128 + MIN_PATCH_LEN
+                goto no_padding;
+
+        ip_protocol = rtl8127_get_l4_protocol(skb);
+        if (!(ip_protocol == IPPROTO_TCP || ip_protocol == IPPROTO_UDP))
+                goto no_padding;
+
+        trans_data_len = pkt_len -
+                         (skb->transport_header -
+                          skb_headroom(skb));
+        if (ip_protocol == IPPROTO_UDP) {
+                if (trans_data_len > 3 && trans_data_len < MIN_PATCH_LEN) {
+                        u16 dest_port = 0;
+
+                        skb_copy_bits(skb, skb->transport_header - skb_headroom(skb) + 2, &dest_port, 2);
+                        dest_port = ntohs(dest_port);
+
+                        if (dest_port == 0x13f ||
+                            dest_port == 0x140) {
+                                pad_len = MIN_PATCH_LEN - trans_data_len;
+                                goto out;
+                        }
+                }
+        }
+
+        hdr_len = 0;
+        if (ip_protocol == IPPROTO_TCP)
+                hdr_len = 20;
+        else if (ip_protocol == IPPROTO_UDP)
+                hdr_len = 8;
+        if (trans_data_len < hdr_len)
+                pad_len = hdr_len - trans_data_len;
+
+out:
+        if ((pkt_len + pad_len) < ETH_ZLEN)
+                pad_len = ETH_ZLEN - pkt_len;
+
+        return pad_len;
+
+no_padding:
+
+        return 0;
+}
+
+static bool
+rtl8127_tso_csum(struct sk_buff *skb,
+                 struct net_device *dev,
+                 u32 *opts,
+                 unsigned int *bytecount,
+                 unsigned short *gso_segs)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned long large_send = 0;
+        u32 csum_cmd = 0;
+        u8 sw_calc_csum = false;
+        u8 check_patch_required = true;
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+        if (dev->features & (NETIF_F_TSO | NETIF_F_TSO6)) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
+                u32 mss = skb_shinfo(skb)->tso_size;
+#else
+                u32 mss = skb_shinfo(skb)->gso_size;
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(2,6,18)
+
+                /* TCP Segmentation Offload (or TCP Large Send) */
+                if (mss) {
+                        union {
+                                struct iphdr *v4;
+                                struct ipv6hdr *v6;
+                                unsigned char *hdr;
+                        } ip;
+                        union {
+                                struct tcphdr *tcp;
+                                struct udphdr *udp;
+                                unsigned char *hdr;
+                        } l4;
+                        u32 l4_offset, hdr_len;
+
+                        ip.hdr = skb_network_header(skb);
+                        l4.hdr = skb_checksum_start(skb);
+
+                        l4_offset = skb_transport_offset(skb);
+                        assert((l4_offset%2) == 0);
+                        switch (get_protocol(skb)) {
+                        case __constant_htons(ETH_P_IP):
+                                if (l4_offset <= GTTCPHO_MAX) {
+                                        opts[0] |= GiantSendv4;
+                                        opts[0] |= l4_offset << GTTCPHO_SHIFT;
+                                        opts[1] |= min(mss, MSS_MAX) << 18;
+                                        large_send = 1;
+                                }
+                                break;
+                        case __constant_htons(ETH_P_IPV6):
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)
+                                if (msdn_giant_send_check(skb))
+                                        return false;
+#endif
+                                if (l4_offset <= GTTCPHO_MAX) {
+                                        opts[0] |= GiantSendv6;
+                                        opts[0] |= l4_offset << GTTCPHO_SHIFT;
+                                        opts[1] |= min(mss, MSS_MAX) << 18;
+                                        large_send = 1;
+                                }
+                                break;
+                        default:
+                                if (unlikely(net_ratelimit()))
+                                        dprintk("tso proto=%x!\n", skb->protocol);
+                                break;
+                        }
+
+                        if (large_send == 0)
+                                return false;
+
+
+                        /* compute length of segmentation header */
+                        hdr_len = (l4.tcp->doff * 4) + l4_offset;
+                        /* update gso size and bytecount with header size */
+                        *gso_segs = skb_shinfo(skb)->gso_segs;
+                        *bytecount += (*gso_segs - 1) * hdr_len;
+
+                        return true;
+                }
+        }
+#endif //LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+
+        if (skb->ip_summed == CHECKSUM_PARTIAL) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
+                const struct iphdr *ip = skb->nh.iph;
+
+                if (dev->features & NETIF_F_IP_CSUM) {
+                        if (ip->protocol == IPPROTO_TCP)
+                                csum_cmd = tp->tx_ip_csum_cmd | tp->tx_tcp_csum_cmd;
+                        else if (ip->protocol == IPPROTO_UDP)
+                                csum_cmd = tp->tx_ip_csum_cmd | tp->tx_udp_csum_cmd;
+                        else if (ip->protocol == IPPROTO_IP)
+                                csum_cmd = tp->tx_ip_csum_cmd;
+                }
+#else
+                u8 ip_protocol = IPPROTO_RAW;
+
+                switch (get_protocol(skb)) {
+                case  __constant_htons(ETH_P_IP):
+                        if (dev->features & NETIF_F_IP_CSUM) {
+                                ip_protocol = ip_hdr(skb)->protocol;
+                                csum_cmd = tp->tx_ip_csum_cmd;
+                        }
+                        break;
+                case  __constant_htons(ETH_P_IPV6):
+                        if (dev->features & NETIF_F_IPV6_CSUM) {
+                                if (skb_transport_offset(skb) > 0 && skb_transport_offset(skb) <= TCPHO_MAX) {
+                                        ip_protocol = ipv6_hdr(skb)->nexthdr;
+                                        csum_cmd = tp->tx_ipv6_csum_cmd;
+                                        csum_cmd |= skb_transport_offset(skb) << TCPHO_SHIFT;
+                                }
+                        }
+                        break;
+                default:
+                        if (unlikely(net_ratelimit()))
+                                dprintk("checksum_partial proto=%x!\n", skb->protocol);
+                        break;
+                }
+
+                if (ip_protocol == IPPROTO_TCP)
+                        csum_cmd |= tp->tx_tcp_csum_cmd;
+                else if (ip_protocol == IPPROTO_UDP)
+                        csum_cmd |= tp->tx_udp_csum_cmd;
+#endif
+                if (csum_cmd == 0) {
+                        sw_calc_csum = true;
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+                        WARN_ON(1); /* we need a WARN() */
+#endif
+                }
+
+                if (ip_protocol == IPPROTO_TCP)
+                        check_patch_required = false;
+        }
+
+        if (check_patch_required) {
+                u32 pad_len = rtl8127_get_patch_pad_len(tp, skb);
+
+                if (pad_len > 0) {
+                        if (!rtl8127_skb_pad_with_len(skb, skb->len + pad_len))
+                                return false;
+
+                        if (csum_cmd != 0)
+                                sw_calc_csum = true;
+                }
+        }
+
+        if (skb->len < ETH_ZLEN) {
+                if (tp->UseSwPaddingShortPkt ||
+                    (tp->ShortPacketSwChecksum && csum_cmd != 0)) {
+                        if (!rtl8127_skb_pad(skb))
+                                return false;
+
+                        if (csum_cmd != 0)
+                                sw_calc_csum = true;
+                }
+        }
+
+        if (sw_calc_csum) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10) && LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,7)
+                skb_checksum_help(&skb, 0);
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19) && LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10)
+                skb_checksum_help(skb, 0);
+#else
+                skb_checksum_help(skb);
+#endif
+        } else
+                opts[1] |= csum_cmd;
+
+        return true;
+}
+
+static bool rtl8127_tx_slots_avail(struct rtl8127_private *tp,
+                                   struct rtl8127_tx_ring *ring)
+{
+        unsigned int slots_avail = READ_ONCE(ring->dirty_tx) + ring->num_tx_desc
+                                   - READ_ONCE(ring->cur_tx);
+
+        /* A skbuff with nr_frags needs nr_frags+1 entries in the tx queue */
+        return slots_avail > MAX_SKB_FRAGS;
+}
+
+static inline u32
+rtl8127_fast_mod_mask(const u32 input, const u32 mask)
+{
+        return input > mask ? input & mask : input;
+}
+
+static void rtl8127_doorbell(struct rtl8127_private *tp,
+                             struct rtl8127_tx_ring *ring)
+{
+        if (tp->EnableTxNoClose) {
+                if (tp->HwSuppTxNoCloseVer > 3)
+                        RTL_W32(tp, ring->sw_tail_ptr_reg, ring->cur_tx);
+                else
+                        RTL_W16(tp, ring->sw_tail_ptr_reg, ring->cur_tx);
+        } else {
+                /* set polling bit */
+                RTL_W32(tp, TPPOLL_8125, BIT(ring->index));
+        }
+}
+
+static netdev_tx_t
+rtl8127_start_xmit(struct sk_buff *skb,
+                   struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        unsigned int   bytecount;
+        unsigned short gso_segs;
+        struct ring_info *last;
+        unsigned int last_entry;
+        unsigned int entry;
+        struct TxDesc *txd;
+        dma_addr_t mapping;
+        u32 len;
+        u32 opts[2];
+        netdev_tx_t ret = NETDEV_TX_OK;
+        int frags;
+        u8 EnableTxNoClose = tp->EnableTxNoClose;
+        const u16 queue_mapping = skb_get_queue_mapping(skb);
+        struct rtl8127_tx_ring *ring;
+        bool stop_queue;
+
+        assert(queue_mapping < tp->num_tx_rings);
+
+        ring = &tp->tx_ring[queue_mapping];
+
+        if (unlikely(!rtl8127_tx_slots_avail(tp, ring))) {
+                if (netif_msg_drv(tp)) {
+                        printk(KERN_ERR
+                               "%s: BUG! Tx Ring[%d] full when queue awake!\n",
+                               dev->name,
+                               queue_mapping);
+                }
+                goto err_stop;
+        }
+
+        entry = ring->cur_tx % ring->num_tx_desc;
+        txd = ring->TxDescArray + entry;
+
+        if (!EnableTxNoClose) {
+                if (unlikely(le32_to_cpu(txd->opts1) & DescOwn)) {
+                        if (netif_msg_drv(tp)) {
+                                printk(KERN_ERR
+                                       "%s: BUG! Tx Desc is own by hardware!\n",
+                                       dev->name);
+                        }
+                        goto err_stop;
+                }
+        }
+
+        bytecount = skb->len;
+        gso_segs = 1;
+
+        opts[0] = DescOwn;
+        opts[1] = rtl8127_tx_vlan_tag(tp, skb);
+
+        if (unlikely(!rtl8127_tso_csum(skb, dev, opts, &bytecount, &gso_segs)))
+                goto err_dma_0;
+
+        frags = rtl8127_xmit_frags(tp, ring, skb, opts);
+        if (unlikely(frags < 0))
+                goto err_dma_0;
+        if (frags) {
+                len = skb_headlen(skb);
+                opts[0] |= FirstFrag;
+        } else {
+                len = skb->len;
+                opts[0] |= FirstFrag | LastFrag;
+        }
+
+        opts[0] = rtl8127_get_txd_opts1(ring, opts[0], len, entry);
+        mapping = dma_map_single(tp_to_dev(tp), skb->data, len, DMA_TO_DEVICE);
+        if (unlikely(dma_mapping_error(tp_to_dev(tp), mapping))) {
+                if (unlikely(net_ratelimit()))
+                        netif_err(tp, drv, dev, "Failed to map TX DMA!\n");
+                goto err_dma_1;
+        }
+
+#ifdef ENABLE_PTP_SUPPORT
+        if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
+                if (!test_and_set_bit_lock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state)) {
+                        if (tp->hwtstamp_config.tx_type == HWTSTAMP_TX_ON &&
+                            !tp->ptp_tx_skb) {
+                                skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+
+                                tp->ptp_tx_skb = skb_get(skb);
+                                tp->ptp_tx_start = jiffies;
+                                schedule_work(&tp->ptp_tx_work);
+                        } else
+                                tp->tx_hwtstamp_skipped++;
+                }
+        }
+#endif
+        /* set first fragment's length */
+        ring->tx_skb[entry].len = len;
+
+        /* set skb to last fragment */
+        last_entry = (entry + frags) % ring->num_tx_desc;
+        last = &ring->tx_skb[last_entry];
+        last->skb = skb;
+        last->gso_segs = gso_segs;
+        last->bytecount = bytecount;
+
+        txd->addr = cpu_to_le64(mapping);
+        txd->opts2 = cpu_to_le32(opts[1]);
+        wmb();
+        txd->opts1 = cpu_to_le32(opts[0]);
+
+        netdev_tx_sent_queue(txring_txq(ring), bytecount);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
+        dev->trans_start = jiffies;
+#else
+        skb_tx_timestamp(skb);
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(3,5,0)
+
+        /* rtl_tx needs to see descriptor changes before updated tp->cur_tx */
+        smp_wmb();
+
+        WRITE_ONCE(ring->cur_tx, ring->cur_tx + frags + 1);
+
+        stop_queue = !rtl8127_tx_slots_avail(tp, ring);
+        if (unlikely(stop_queue)) {
+                /* Avoid wrongly optimistic queue wake-up: rtl_tx thread must
+                 * not miss a ring update when it notices a stopped queue.
+                 */
+                smp_wmb();
+                netif_stop_subqueue(dev, queue_mapping);
+        }
+
+        if (netif_xmit_stopped(txring_txq(ring)) || !netdev_xmit_more())
+                rtl8127_doorbell(tp, ring);
+
+        if (unlikely(stop_queue)) {
+                /* Sync with rtl_tx:
+                 * - publish queue status and cur_tx ring index (write barrier)
+                 * - refresh dirty_tx ring index (read barrier).
+                 * May the current thread have a pessimistic view of the ring
+                 * status and forget to wake up queue, a racing rtl_tx thread
+                 * can't.
+                 */
+                smp_mb();
+                if (rtl8127_tx_slots_avail(tp, ring))
+                        netif_start_subqueue(dev, queue_mapping);
+        }
+out:
+        return ret;
+err_dma_1:
+        rtl8127_tx_clear_range(tp, ring, ring->cur_tx + 1, frags);
+err_dma_0:
+        RTLDEV->stats.tx_dropped++;
+        dev_kfree_skb_any(skb);
+        ret = NETDEV_TX_OK;
+        goto out;
+err_stop:
+        netif_stop_subqueue(dev, queue_mapping);
+        ret = NETDEV_TX_BUSY;
+        RTLDEV->stats.tx_dropped++;
+        goto out;
+}
+
+/* recycle tx no close desc*/
+static int
+rtl8127_tx_interrupt_noclose(struct rtl8127_tx_ring *ring, int budget)
+{
+        unsigned int total_bytes = 0, total_packets = 0;
+        struct rtl8127_private *tp = ring->priv;
+        struct net_device *dev = tp->dev;
+        unsigned int dirty_tx, tx_left;
+        unsigned int tx_desc_closed;
+        unsigned int count = 0;
+
+        dirty_tx = ring->dirty_tx;
+        ring->NextHwDesCloPtr = rtl8127_get_hw_clo_ptr(ring);
+        tx_desc_closed = rtl8127_fast_mod_mask(ring->NextHwDesCloPtr -
+                                               ring->BeginHwDesCloPtr,
+                                               tp->MaxTxDescPtrMask);
+        tx_left = min((READ_ONCE(ring->cur_tx) - dirty_tx), tx_desc_closed);
+        ring->BeginHwDesCloPtr += tx_left;
+
+        while (tx_left > 0) {
+                unsigned int entry = dirty_tx % ring->num_tx_desc;
+                struct ring_info *tx_skb = ring->tx_skb + entry;
+
+                rtl8127_unmap_tx_skb(tp->pci_dev,
+                                     tx_skb,
+                                     ring->TxDescArray + entry);
+
+                if (tx_skb->skb != NULL) {
+                        /* update the statistics for this packet */
+                        total_bytes += tx_skb->bytecount;
+                        total_packets += tx_skb->gso_segs;
+
+                        RTL_NAPI_CONSUME_SKB_ANY(tx_skb->skb, budget);
+                        tx_skb->skb = NULL;
+                }
+                dirty_tx++;
+                tx_left--;
+        }
+
+        if (total_packets) {
+                netdev_tx_completed_queue(txring_txq(ring),
+                                          total_packets, total_bytes);
+
+                RTLDEV->stats.tx_bytes += total_bytes;
+                RTLDEV->stats.tx_packets+= total_packets;
+        }
+
+        if (ring->dirty_tx != dirty_tx) {
+                count = dirty_tx - ring->dirty_tx;
+                WRITE_ONCE(ring->dirty_tx, dirty_tx);
+                smp_wmb();
+                if (__netif_subqueue_stopped(dev, ring->index) &&
+                    rtl8127_tx_slots_avail(tp, ring) && netif_carrier_ok(dev)) {
+                        netif_start_subqueue(dev, ring->index);
+                }
+        }
+
+        return count;
+}
+
+/* recycle tx close desc*/
+static int
+rtl8127_tx_interrupt_close(struct rtl8127_tx_ring *ring, int budget)
+{
+        unsigned int total_bytes = 0, total_packets = 0;
+        struct rtl8127_private *tp = ring->priv;
+        struct net_device *dev = tp->dev;
+        unsigned int dirty_tx, tx_left;
+        unsigned int count = 0;
+
+        dirty_tx = ring->dirty_tx;
+        tx_left = READ_ONCE(ring->cur_tx) - dirty_tx;
+
+        while (tx_left > 0) {
+                unsigned int entry = dirty_tx % ring->num_tx_desc;
+                struct ring_info *tx_skb = ring->tx_skb + entry;
+
+                if (le32_to_cpu(READ_ONCE(ring->TxDescArray[entry].opts1)) & DescOwn)
+                        break;
+
+                rtl8127_unmap_tx_skb(tp->pci_dev,
+                                     tx_skb,
+                                     ring->TxDescArray + entry);
+
+                if (tx_skb->skb != NULL) {
+                        /* update the statistics for this packet */
+                        total_bytes += tx_skb->bytecount;
+                        total_packets += tx_skb->gso_segs;
+
+                        RTL_NAPI_CONSUME_SKB_ANY(tx_skb->skb, budget);
+                        tx_skb->skb = NULL;
+                }
+                dirty_tx++;
+                tx_left--;
+        }
+
+        if (total_packets) {
+                netdev_tx_completed_queue(txring_txq(ring),
+                                          total_packets, total_bytes);
+
+                RTLDEV->stats.tx_bytes += total_bytes;
+                RTLDEV->stats.tx_packets+= total_packets;
+        }
+
+        if (ring->dirty_tx != dirty_tx) {
+                count = dirty_tx - ring->dirty_tx;
+                WRITE_ONCE(ring->dirty_tx, dirty_tx);
+                smp_wmb();
+                if (__netif_subqueue_stopped(dev, ring->index) &&
+                    rtl8127_tx_slots_avail(tp, ring) && netif_carrier_ok(dev)) {
+                        netif_start_subqueue(dev, ring->index);
+                }
+
+                if (READ_ONCE(ring->cur_tx) != dirty_tx)
+                        rtl8127_doorbell(tp, ring);
+        }
+
+        return count;
+}
+
+static int
+rtl8127_tx_interrupt(struct rtl8127_tx_ring *ring, int budget)
+{
+        struct rtl8127_private *tp = ring->priv;
+
+        if (tp->EnableTxNoClose)
+                return rtl8127_tx_interrupt_noclose(ring, budget);
+        else
+                return rtl8127_tx_interrupt_close(ring, budget);
+}
+
+static int
+rtl8127_tx_interrupt_with_vector(struct rtl8127_private *tp,
+                                 const int message_id,
+                                 int budget)
+{
+        int count = 0;
+
+        switch (tp->HwCurrIsrVer) {
+        case 3:
+        case 4:
+                if (message_id < tp->num_tx_rings)
+                        count += rtl8127_tx_interrupt(&tp->tx_ring[message_id], budget);
+                break;
+        case 5:
+                if (message_id == 16)
+                        count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget);
+#ifdef ENABLE_MULTIPLE_TX_QUEUE
+                else if (message_id == 17 && tp->num_tx_rings > 1)
+                        count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget);
+#endif
+                break;
+        case 6:
+                if (message_id == 8)
+                        count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget);
+#ifdef ENABLE_MULTIPLE_TX_QUEUE
+                else if (message_id == 9 && tp->num_tx_rings > 1)
+                        count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget);
+#endif
+                break;
+        default:
+                if (message_id == 16)
+                        count += rtl8127_tx_interrupt(&tp->tx_ring[0], budget);
+#ifdef ENABLE_MULTIPLE_TX_QUEUE
+                else if (message_id == 18 && tp->num_tx_rings > 1)
+                        count += rtl8127_tx_interrupt(&tp->tx_ring[1], budget);
+#endif
+                break;
+        }
+
+        return count;
+}
+
+static inline int
+rtl8127_fragmented_frame(struct rtl8127_private *tp, u32 status)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                return (status & (FirstFrag_V3 | LastFrag_V3)) != (FirstFrag_V3 | LastFrag_V3);
+        case RX_DESC_RING_TYPE_4:
+                return (status & (FirstFrag_V4 | LastFrag_V4)) != (FirstFrag_V4 | LastFrag_V4);
+        default:
+                return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag);
+        }
+}
+
+static inline int
+rtl8127_is_non_eop(struct rtl8127_private *tp, u32 status)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                return !(status & LastFrag_V3);
+        case RX_DESC_RING_TYPE_4:
+                return !(status & LastFrag_V4);
+        default:
+                return !(status & LastFrag);
+        }
+}
+
+static inline int
+rtl8127_rx_desc_type(u32 status)
+{
+        return ((status >> 26) & 0x0F);
+}
+
+static inline void
+rtl8127_rx_v1_csum(struct rtl8127_private *tp,
+                   struct sk_buff *skb,
+                   struct RxDesc *desc)
+{
+        u32 opts1 = le32_to_cpu(desc->opts1);
+
+        if (((opts1 & RxTCPT) && !(opts1 & RxTCPF)) ||
+            ((opts1 & RxUDPT) && !(opts1 & RxUDPF)))
+                skb->ip_summed = CHECKSUM_UNNECESSARY;
+        else
+                skb_checksum_none_assert(skb);
+}
+
+static inline void
+rtl8127_rx_v3_csum(struct rtl8127_private *tp,
+                   struct sk_buff *skb,
+                   struct RxDescV3 *descv3)
+{
+        u32 opts2 = le32_to_cpu(descv3->RxDescNormalDDWord4.opts2);
+
+        /* rx csum offload for RTL8125 */
+        if (((opts2 & RxTCPT_v3) && !(opts2 & RxTCPF_v3)) ||
+            ((opts2 & RxUDPT_v3) && !(opts2 & RxUDPF_v3)))
+                skb->ip_summed = CHECKSUM_UNNECESSARY;
+        else
+                skb_checksum_none_assert(skb);
+}
+
+static inline void
+rtl8127_rx_v4_csum(struct rtl8127_private *tp,
+                   struct sk_buff *skb,
+                   struct RxDescV4 *descv4)
+{
+        u32 opts1 = le32_to_cpu(descv4->RxDescNormalDDWord2.opts1);
+
+        /* rx csum offload for RTL8125 */
+        if (((opts1 & RxTCPT_v4) && !(opts1 & RxTCPF_v4)) ||
+            ((opts1 & RxUDPT_v4) && !(opts1 & RxUDPF_v4)))
+                skb->ip_summed = CHECKSUM_UNNECESSARY;
+        else
+                skb_checksum_none_assert(skb);
+}
+
+static inline void
+rtl8127_rx_csum(struct rtl8127_private *tp,
+                struct sk_buff *skb,
+                struct RxDesc *desc)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                rtl8127_rx_v3_csum(tp, skb, (struct RxDescV3 *)desc);
+                break;
+        case RX_DESC_RING_TYPE_4:
+                rtl8127_rx_v4_csum(tp, skb, (struct RxDescV4 *)desc);
+                break;
+        default:
+                rtl8127_rx_v1_csum(tp, skb, desc);
+                break;
+        }
+}
+
+/*
+static inline int
+rtl8127_try_rx_copy(struct rtl8127_private *tp,
+                    struct rtl8127_rx_ring *ring,
+                    struct sk_buff **sk_buff,
+                    int pkt_size,
+                    struct RxDesc *desc,
+                    int rx_buf_sz)
+{
+        int ret = -1;
+
+        struct sk_buff *skb;
+
+        skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, pkt_size + R8127_RX_ALIGN);
+        if (skb) {
+                u8 *data;
+
+                data = sk_buff[0]->data;
+                if (!R8127_USE_NAPI_ALLOC_SKB)
+                    skb_reserve(skb, R8127_RX_ALIGN);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37)
+                prefetch(data - R8127_RX_ALIGN);
+#endif
+                eth_copy_and_sum(skb, data, pkt_size, 0);
+                *sk_buff = skb;
+                rtl8127_mark_to_asic(tp, desc, rx_buf_sz);
+                ret = 0;
+        }
+
+        return ret;
+}
+*/
+
+static inline void
+rtl8127_rx_skb(struct rtl8127_private *tp,
+               struct sk_buff *skb,
+               u32 ring_index)
+{
+#ifdef CONFIG_R8127_NAPI
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29)
+        netif_receive_skb(skb);
+#else
+        napi_gro_receive(&tp->r8127napi[ring_index].napi, skb);
+#endif
+#else
+        netif_rx(skb);
+#endif
+}
+
+static int
+rtl8127_check_rx_desc_error(struct net_device *dev,
+                            struct rtl8127_private *tp,
+                            u32 status)
+{
+        int ret = 0;
+
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                if (unlikely(status & RxRES_V3)) {
+                        if (status & (RxRWT_V3 | RxRUNT_V3))
+                                RTLDEV->stats.rx_length_errors++;
+                        if (status & RxCRC_V3)
+                                RTLDEV->stats.rx_crc_errors++;
+
+                        ret = -1;
+                }
+                break;
+        case RX_DESC_RING_TYPE_4:
+                if (unlikely(status & RxRES_V4)) {
+                        if (status & RxRUNT_V4)
+                                RTLDEV->stats.rx_length_errors++;
+                        if (status & RxCRC_V4)
+                                RTLDEV->stats.rx_crc_errors++;
+
+                        ret = -1;
+                }
+                break;
+        default:
+                if (unlikely(status & RxRES)) {
+                        if (status & (RxRWT | RxRUNT))
+                                RTLDEV->stats.rx_length_errors++;
+                        if (status & RxCRC)
+                                RTLDEV->stats.rx_crc_errors++;
+
+                        ret = -1;
+                }
+                break;
+        }
+
+        return ret;
+}
+
+#ifdef ENABLE_PAGE_REUSE
+
+static inline bool
+rtl8127_reuse_rx_ok(struct page *page)
+{
+        /* avoid re-using remote pages */
+        if (!dev_page_is_reusable(page)) {
+                //printk(KERN_INFO "r8127 page pfmemalloc, can't reuse!\n");
+                return false;
+        }
+        /* if we are only owner of page we can reuse it */
+        if (unlikely(page_ref_count(page) != 1)) {
+                //printk(KERN_INFO "r8127 page refcnt %d, can't reuse!\n", page_ref_count(page));
+                return false;
+        }
+
+        return true;
+}
+
+static void
+rtl8127_reuse_rx_buffer(struct rtl8127_private *tp, struct rtl8127_rx_ring *ring, u32 cur_rx, struct rtl8127_rx_buffer *rxb)
+{
+        struct page *page = rxb->page;
+
+        u32 dirty_rx = ring->dirty_rx;
+        u32 entry = dirty_rx % ring->num_rx_desc;
+        struct rtl8127_rx_buffer *nrxb = &ring->rx_buffer[entry];
+
+        u32 noffset;
+
+        //the page gonna be shared by us and kernel, keep page ref = 2
+        page_ref_inc(page);
+
+        //flip the buffer in page to use next
+        noffset = rxb->page_offset ^ (tp->rx_buf_page_size / 2); //one page, two buffer, ping-pong
+
+        nrxb->dma = rxb->dma;
+        nrxb->page_offset = noffset;
+        nrxb->data = rxb->data;
+
+        if (cur_rx != dirty_rx) {
+                //move the buffer to other slot
+                nrxb->page = page;
+                rxb->page = NULL;
+        }
+}
+
+static void rtl8127_put_rx_buffer(struct rtl8127_private *tp,
+                                  struct rtl8127_rx_ring *ring,
+                                  u32 cur_rx,
+                                  struct rtl8127_rx_buffer *rxb)
+{
+        struct rtl8127_rx_buffer *nrxb;
+        struct page *page = rxb->page;
+        u32 entry;
+
+        entry = ring->dirty_rx % ring->num_rx_desc;
+        nrxb = &ring->rx_buffer[entry];
+        if (likely(rtl8127_reuse_rx_ok(page))) {
+                /* hand second half of page back to the ring */
+                rtl8127_reuse_rx_buffer(tp, ring, cur_rx, rxb);
+        } else {
+                tp->page_reuse_fail_cnt++;
+
+                dma_unmap_page_attrs(&tp->pci_dev->dev, rxb->dma,
+                                     tp->rx_buf_page_size,
+                                     DMA_FROM_DEVICE,
+                                     (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING));
+                //the page ref is kept 1, uniquely owned by kernel now
+                rxb->page = NULL;
+
+                return;
+        }
+
+        dma_sync_single_range_for_device(tp_to_dev(tp),
+                                         nrxb->dma,
+                                         nrxb->page_offset,
+                                         tp->rx_buf_sz,
+                                         DMA_FROM_DEVICE);
+
+        rtl8127_map_to_asic(tp, ring,
+                            rtl8127_get_rxdesc(tp, ring->RxDescArray, entry),
+                            nrxb->dma + nrxb->page_offset,
+                            tp->rx_buf_sz, entry);
+
+        ring->dirty_rx++;
+}
+
+#endif //ENABLE_PAGE_REUSE
+
+static int
+rtl8127_rx_interrupt(struct net_device *dev,
+                     struct rtl8127_private *tp,
+                     struct rtl8127_rx_ring *ring,
+                     napi_budget budget)
+{
+        unsigned int cur_rx, rx_left;
+        unsigned int delta, count = 0;
+        unsigned int entry;
+        struct RxDesc *desc;
+        struct sk_buff *skb;
+        u32 status;
+        u32 rx_quota;
+        u32 ring_index = ring->index;
+#ifdef ENABLE_PAGE_REUSE
+        struct rtl8127_rx_buffer *rxb;
+#else //ENABLE_PAGE_REUSE
+        u64 rx_buf_phy_addr;
+#endif //ENABLE_PAGE_REUSE
+        unsigned int total_rx_multicast_packets = 0;
+        unsigned int total_rx_bytes = 0, total_rx_packets = 0;
+
+        assert(dev != NULL);
+        assert(tp != NULL);
+
+        if (ring->RxDescArray == NULL)
+                goto rx_out;
+
+        rx_quota = RTL_RX_QUOTA(budget);
+        cur_rx = ring->cur_rx;
+        rx_left = ring->num_rx_desc + ring->dirty_rx - cur_rx;
+        rx_left = rtl8127_rx_quota(rx_left, (u32)rx_quota);
+
+        for (; rx_left > 0; rx_left--, cur_rx++) {
+#ifndef ENABLE_PAGE_REUSE
+                const void *rx_buf;
+#endif //!ENABLE_PAGE_REUSE
+                u32 pkt_size;
+
+                entry = cur_rx % ring->num_rx_desc;
+                desc = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry);
+                status = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc));
+                if (status & DescOwn) {
+                        RTL_R8(tp, tp->imr_reg[0]);
+                        status = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc));
+                        if (status & DescOwn)
+                                break;
+                }
+
+                rmb();
+
+                if (unlikely(rtl8127_check_rx_desc_error(dev, tp, status) < 0)) {
+                        if (netif_msg_rx_err(tp)) {
+                                printk(KERN_INFO
+                                       "%s: Rx ERROR. status = %08x\n",
+                                       dev->name, status);
+                        }
+
+                        RTLDEV->stats.rx_errors++;
+
+                        if (!(dev->features & NETIF_F_RXALL))
+                                goto release_descriptor;
+                }
+                pkt_size = status & 0x00003fff;
+                if (likely(!(dev->features & NETIF_F_RXFCS))) {
+#ifdef ENABLE_RX_PACKET_FRAGMENT
+                        if (rtl8127_is_non_eop(tp, status) &&
+                            pkt_size == tp->rx_buf_sz) {
+                                struct RxDesc *desc_next;
+                                unsigned int entry_next;
+                                int pkt_size_next;
+                                u32 status_next;
+
+                                entry_next = (cur_rx + 1) % ring->num_rx_desc;
+                                desc_next = rtl8127_get_rxdesc(tp, ring->RxDescArray, entry_next);
+                                status_next = le32_to_cpu(rtl8127_rx_desc_opts1(tp, desc_next));
+                                if (!(status_next & DescOwn)) {
+                                        pkt_size_next = status_next & 0x00003fff;
+                                        if (pkt_size_next < ETH_FCS_LEN)
+                                                pkt_size -= (ETH_FCS_LEN - pkt_size_next);
+                                }
+                        }
+#endif //ENABLE_RX_PACKET_FRAGMENT
+                        if (!rtl8127_is_non_eop(tp, status)) {
+                                if (pkt_size < ETH_FCS_LEN) {
+#ifdef ENABLE_RX_PACKET_FRAGMENT
+                                        pkt_size = 0;
+#else
+                                        goto drop_packet;
+#endif //ENABLE_RX_PACKET_FRAGMENT
+                                } else
+                                        pkt_size -= ETH_FCS_LEN;
+                        }
+                }
+
+                if (unlikely(pkt_size > tp->rx_buf_sz))
+                        goto drop_packet;
+
+#if !defined(ENABLE_RX_PACKET_FRAGMENT) || !defined(ENABLE_PAGE_REUSE)
+                /*
+                 * The driver does not support incoming fragmented
+                 * frames. They are seen as a symptom of over-mtu
+                 * sized frames.
+                 */
+                if (unlikely(rtl8127_fragmented_frame(tp, status)))
+                        goto drop_packet;
+#endif //!ENABLE_RX_PACKET_FRAGMENT || !ENABLE_PAGE_REUSE
+
+#ifdef ENABLE_PAGE_REUSE
+                rxb = &ring->rx_buffer[entry];
+                skb = rxb->skb;
+                rxb->skb = NULL;
+                if (!skb) {
+                        skb = RTL_BUILD_SKB_INTR(rxb->data + rxb->page_offset - ring->rx_offset, tp->rx_buf_page_size / 2);
+                        if (!skb) {
+                                //netdev_err(tp->dev, "Failed to allocate RX skb!\n");
+                                goto drop_packet;
+                        }
+
+                        skb->dev = dev;
+                        if (!R8127_USE_NAPI_ALLOC_SKB)
+                                skb_reserve(skb, R8127_RX_ALIGN);
+                        skb_put(skb, pkt_size);
+#ifdef ENABLE_RSS_SUPPORT
+                        rtl8127_rx_hash(tp, desc, skb);
+#endif
+                        rtl8127_rx_csum(tp, skb, desc);
+                } else
+                        skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rxb->page,
+                                        rxb->page_offset, pkt_size, tp->rx_buf_page_size / 2);
+                //recycle desc
+                rtl8127_put_rx_buffer(tp, ring, cur_rx, rxb);
+
+                dma_sync_single_range_for_cpu(tp_to_dev(tp),
+                                              rxb->dma,
+                                              rxb->page_offset,
+                                              tp->rx_buf_sz,
+                                              DMA_FROM_DEVICE);
+#else //ENABLE_PAGE_REUSE
+                skb = RTL_ALLOC_SKB_INTR(&tp->r8127napi[ring->index].napi, pkt_size + R8127_RX_ALIGN);
+                if (!skb) {
+                        //netdev_err(tp->dev, "Failed to allocate RX skb!\n");
+                        goto drop_packet;
+                }
+
+                skb->dev = dev;
+                if (!R8127_USE_NAPI_ALLOC_SKB)
+                        skb_reserve(skb, R8127_RX_ALIGN);
+                skb_put(skb, pkt_size);
+
+                rx_buf_phy_addr = ring->RxDescPhyAddr[entry];
+                dma_sync_single_for_cpu(tp_to_dev(tp),
+                                        rx_buf_phy_addr, tp->rx_buf_sz,
+                                        DMA_FROM_DEVICE);
+                rx_buf = ring->Rx_skbuff[entry]->data;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,37)
+                prefetch(rx_buf - R8127_RX_ALIGN);
+#endif
+                eth_copy_and_sum(skb, rx_buf, pkt_size, 0);
+
+                dma_sync_single_for_device(tp_to_dev(tp), rx_buf_phy_addr,
+                                           tp->rx_buf_sz, DMA_FROM_DEVICE);
+#endif //ENABLE_PAGE_REUSE
+
+#ifdef ENABLE_PTP_SUPPORT
+                if (tp->flags & RTL_FLAG_RX_HWTSTAMP_ENABLED)
+                        rtl8127_rx_ptp_timestamp(tp, skb);
+#endif // ENABLE_PTP_SUPPORT
+
+#ifdef ENABLE_RX_PACKET_FRAGMENT
+                if (rtl8127_is_non_eop(tp, status)) {
+                        unsigned int entry_next;
+                        entry_next = (entry + 1) % ring->num_rx_desc;
+                        rxb = &ring->rx_buffer[entry_next];
+                        rxb->skb = skb;
+                        continue;
+                }
+#endif //ENABLE_RX_PACKET_FRAGMENT
+
+#ifndef ENABLE_PAGE_REUSE
+#ifdef ENABLE_RSS_SUPPORT
+                rtl8127_rx_hash(tp, desc, skb);
+#endif
+                rtl8127_rx_csum(tp, skb, desc);
+#endif /* !ENABLE_PAGE_REUSE */
+
+                skb->protocol = eth_type_trans(skb, dev);
+
+                total_rx_bytes += skb->len;
+
+                if (skb->pkt_type == PACKET_MULTICAST)
+                        total_rx_multicast_packets++;
+
+                if (rtl8127_rx_vlan_skb(tp, desc, skb) < 0)
+                        rtl8127_rx_skb(tp, skb, ring_index);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0)
+                dev->last_rx = jiffies;
+#endif //LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0)
+                total_rx_packets++;
+
+#ifdef ENABLE_PAGE_REUSE
+                rxb->skb = NULL;
+                continue;
+#endif
+
+release_descriptor:
+                switch (tp->InitRxDescType) {
+                case RX_DESC_RING_TYPE_3:
+                case RX_DESC_RING_TYPE_4:
+                        rtl8127_set_desc_dma_addr(tp, desc,
+                                                  ring->RxDescPhyAddr[entry]);
+                        wmb();
+                        break;
+                }
+                rtl8127_mark_to_asic(tp, desc, tp->rx_buf_sz);
+                continue;
+drop_packet:
+                RTLDEV->stats.rx_dropped++;
+                RTLDEV->stats.rx_length_errors++;
+                goto release_descriptor;
+        }
+
+        count = cur_rx - ring->cur_rx;
+        ring->cur_rx = cur_rx;
+
+        delta = rtl8127_rx_fill(tp, ring, dev, ring->dirty_rx, ring->cur_rx, 1);
+        if (!delta && count && netif_msg_intr(tp))
+                printk(KERN_INFO "%s: no Rx buffer allocated\n", dev->name);
+        ring->dirty_rx += delta;
+
+        RTLDEV->stats.rx_bytes += total_rx_bytes;
+        RTLDEV->stats.rx_packets += total_rx_packets;
+        RTLDEV->stats.multicast += total_rx_multicast_packets;
+
+        /*
+         * FIXME: until there is periodic timer to try and refill the ring,
+         * a temporary shortage may definitely kill the Rx process.
+         * - disable the asic to try and avoid an overflow and kick it again
+         *   after refill ?
+         * - how do others driver handle this condition (Uh oh...).
+         */
+        if ((ring->dirty_rx + ring->num_rx_desc == ring->cur_rx) && netif_msg_intr(tp))
+                printk(KERN_EMERG "%s: Rx buffers exhausted\n", dev->name);
+
+rx_out:
+        return total_rx_packets;
+}
+
+static bool
+rtl8127_linkchg_interrupt(struct rtl8127_private *tp, u32 status)
+{
+        switch (tp->HwCurrIsrVer) {
+        case 2:
+        case 3:
+                return status & ISRIMR_V2_LINKCHG;
+        case 4:
+                return status & ISRIMR_V4_LINKCHG;
+        case 5:
+                return status & ISRIMR_V5_LINKCHG;
+        case 6:
+                return status & ISRIMR_V6_LINKCHG;
+        default:
+                return status & LinkChg;
+        }
+}
+
+static u32
+rtl8127_get_linkchg_message_id(struct rtl8127_private *tp)
+{
+        switch (tp->HwCurrIsrVer) {
+        case 4:
+        case 6:
+                return MSIX_ID_V4_LINKCHG;
+        case 5:
+                return MSIX_ID_V5_LINKCHG;
+        default:
+                return MSIX_ID_V2_LINKCHG;
+        }
+}
+
+/*
+ *The interrupt handler does all of the Rx thread work and cleans up after
+ *the Tx thread.
+ */
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance, struct pt_regs *regs)
+#else
+static irqreturn_t rtl8127_interrupt(int irq, void *dev_instance)
+#endif
+{
+        struct r8127_napi *r8127napi = dev_instance;
+        struct rtl8127_private *tp = r8127napi->priv;
+        struct net_device *dev = tp->dev;
+        u32 status;
+        int handled = 0;
+
+        do {
+                status = RTL_R32(tp, tp->isr_reg[0]);
+
+                if (!(tp->features & (RTL_FEATURE_MSI | RTL_FEATURE_MSIX))) {
+                        /* hotplug/major error/no more work/shared irq */
+                        if (!status)
+                                break;
+
+                        if (status == 0xFFFFFFFF)
+                                break;
+
+                        if (!(status & (tp->intr_mask | tp->timer_intr_mask)))
+                                break;
+                }
+
+                handled = 1;
+
+#if defined(RTL_USE_NEW_INTR_API)
+                if (!tp->irq_tbl[0].requested)
+                        break;
+#endif
+                rtl8127_disable_hw_interrupt(tp);
+
+                RTL_W32(tp, tp->isr_reg[0], status&~RxFIFOOver);
+
+                if (rtl8127_linkchg_interrupt(tp, status))
+                        rtl8127_schedule_linkchg_work(tp);
+
+#ifdef ENABLE_DASH_SUPPORT
+                if ((status & ISRIMR_V6_LAYER2_INTR_STS) &&
+                    rtl8127_check_dash_interrupt(tp))
+                        rtl8127_schedule_dash_work(tp);
+#endif
+
+#ifdef CONFIG_R8127_NAPI
+                if (status & tp->intr_mask || tp->keep_intr_cnt-- > 0) {
+                        if (status & tp->intr_mask)
+                                tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT;
+
+                        if (likely(RTL_NETIF_RX_SCHEDULE_PREP(dev, &tp->r8127napi[0].napi)))
+                                __RTL_NETIF_RX_SCHEDULE(dev, &tp->r8127napi[0].napi);
+                        else if (netif_msg_intr(tp))
+                                printk(KERN_INFO "%s: interrupt %04x in poll\n",
+                                       dev->name, status);
+                } else {
+                        tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT;
+                        rtl8127_switch_to_hw_interrupt(tp);
+                }
+#else
+                if (status & tp->intr_mask || tp->keep_intr_cnt-- > 0) {
+                        u32 budget = ~(u32)0;
+                        int i;
+
+                        if (status & tp->intr_mask)
+                                tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT;
+
+                        for (i = 0; i < tp->num_tx_rings; i++)
+                                rtl8127_tx_interrupt(&tp->tx_ring[i], ~(u32)0);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
+                        rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[0], &budget);
+#else
+                        rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[0], budget);
+#endif	//LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
+
+#ifdef ENABLE_DASH_SUPPORT
+                        if ((status & ISRIMR_V6_LAYER2_INTR_STS) &&
+                            rtl8127_check_dash_interrupt(tp))
+                                rtl8127_schedule_dash_work(tp);
+#endif
+
+                        rtl8127_switch_to_timer_interrupt(tp);
+                } else {
+                        tp->keep_intr_cnt = RTK_KEEP_INTERRUPT_COUNT;
+                        rtl8127_switch_to_hw_interrupt(tp);
+                }
+#endif
+        } while (false);
+
+        return IRQ_RETVAL(handled);
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance, struct pt_regs *regs)
+#else
+static irqreturn_t rtl8127_interrupt_msix(int irq, void *dev_instance)
+#endif
+{
+        struct r8127_napi *r8127napi = dev_instance;
+        struct rtl8127_private *tp = r8127napi->priv;
+        struct net_device *dev = tp->dev;
+        int message_id = r8127napi->index;
+#ifndef CONFIG_R8127_NAPI
+        u32 budget = ~(u32)0;
+#endif
+
+        do {
+#if defined(RTL_USE_NEW_INTR_API)
+                if (!tp->irq_tbl[message_id].requested)
+                        break;
+#endif
+                //link change
+                if (message_id == rtl8127_get_linkchg_message_id(tp)) {
+                        rtl8127_disable_hw_layered_interrupt(tp, message_id);
+                        rtl8127_clear_hw_isr_v2(tp, message_id);
+                        rtl8127_schedule_linkchg_work(tp);
+                        break;
+                }
+
+#ifdef ENABLE_DASH_SUPPORT
+                if (message_id == MSIX_ID_V6_L2_MISC_INTR) {
+                        if (rtl8127_check_dash_interrupt(tp))
+                                rtl8127_disable_hw_layered_interrupt(tp, message_id);
+                        rtl8127_clear_hw_isr_v2(tp, message_id);
+                        rtl8127_schedule_dash_work(tp);
+                        rtl8127_enable_hw_layered_interrupt(tp, message_id);
+                        break;
+                }
+#endif
+
+#ifdef CONFIG_R8127_NAPI
+                if (likely(RTL_NETIF_RX_SCHEDULE_PREP(dev, &r8127napi->napi))) {
+                        rtl8127_disable_hw_layered_interrupt(tp, message_id);
+                        __RTL_NETIF_RX_SCHEDULE(dev, &r8127napi->napi);
+                } else if (netif_msg_intr(tp))
+                        printk(KERN_INFO "%s: interrupt message id %d in poll_msix\n",
+                               dev->name, message_id);
+                rtl8127_clear_hw_isr_v2(tp, message_id);
+#else
+                rtl8127_disable_hw_layered_interrupt(tp, message_id);
+
+                rtl8127_clear_hw_isr_v2(tp, message_id);
+
+                rtl8127_tx_interrupt_with_vector(tp, message_id, ~(u32)0);
+
+                if (message_id < tp->num_rx_rings) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
+                        rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], &budget);
+#else
+                        rtl8127_rx_interrupt(dev, tp, &tp->rx_ring[message_id], budget);
+#endif	//LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24)
+                }
+
+                rtl8127_enable_hw_layered_interrupt(tp, message_id);
+#endif
+
+        } while (false);
+
+        return IRQ_HANDLED;
+}
+
+static void rtl8127_down(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        //rtl8127_delete_esd_timer(dev, &tp->esd_timer);
+
+        //rtl8127_delete_link_timer(dev, &tp->link_timer);
+
+        netif_carrier_off(dev);
+
+        netif_tx_disable(dev);
+
+        _rtl8127_wait_for_quiescence(dev);
+
+        rtl8127_hw_reset(dev);
+
+        rtl8127_tx_clear(tp);
+
+        rtl8127_rx_clear(tp);
+}
+
+static int rtl8127_resource_freed(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < tp->num_tx_rings; i++)
+                if (tp->tx_ring[i].TxDescArray)
+                        return 0;
+
+        for (i = 0; i < tp->num_rx_rings; i++)
+                if (tp->rx_ring[i].RxDescArray)
+                        return 0;
+
+        return 1;
+}
+
+int rtl8127_close(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (!rtl8127_resource_freed(tp)) {
+                set_bit(R8127_FLAG_DOWN, tp->task_flags);
+
+                rtl8127_down(dev);
+
+                pci_clear_master(tp->pci_dev);
+
+#ifdef ENABLE_PTP_SUPPORT
+                rtl8127_ptp_stop(tp);
+#endif
+                rtl8127_hw_d3_para(dev);
+
+                rtl8127_powerdown_pll(dev, 0);
+
+                rtl8127_free_irq(tp);
+
+                rtl8127_free_alloc_resources(tp);
+        } else {
+                rtl8127_hw_d3_para(dev);
+
+                rtl8127_powerdown_pll(dev, 0);
+        }
+
+        return 0;
+}
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,11)
+static void rtl8127_shutdown(struct pci_dev *pdev)
+{
+        struct net_device *dev = pci_get_drvdata(pdev);
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        rtnl_lock();
+
+        if (HW_DASH_SUPPORT_DASH(tp))
+                rtl8127_driver_stop(tp);
+
+        rtl8127_disable_pci_offset_180(tp);
+
+        if (s5_keep_curr_mac == 0 && tp->random_mac == 0)
+                rtl8127_rar_set(tp, tp->org_mac_addr);
+
+        if (s5wol == 0)
+                tp->wol_enabled = WOL_DISABLED;
+
+        rtl8127_close(dev);
+        rtl8127_disable_msi(pdev, tp);
+
+        rtnl_unlock();
+
+        if (system_state == SYSTEM_POWER_OFF) {
+                pci_clear_master(tp->pci_dev);
+                pci_wake_from_d3(pdev, tp->wol_enabled);
+                pci_set_power_state(pdev, PCI_D3hot);
+        }
+}
+#endif
+
+#ifdef CONFIG_PM
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11)
+static int
+rtl8127_suspend(struct pci_dev *pdev, u32 state)
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)
+static int
+rtl8127_suspend(struct device *device)
+#else
+static int
+rtl8127_suspend(struct pci_dev *pdev, pm_message_t state)
+#endif
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)
+        struct pci_dev *pdev = to_pci_dev(device);
+        struct net_device *dev = pci_get_drvdata(pdev);
+#else
+        struct net_device *dev = pci_get_drvdata(pdev);
+#endif
+        struct rtl8127_private *tp = netdev_priv(dev);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10)
+        u32 pci_pm_state = pci_choose_state(pdev, state);
+#endif
+        rtnl_lock();
+
+        if (!netif_running(dev))
+                goto out;
+
+        set_bit(R8127_FLAG_DOWN, tp->task_flags);
+
+        netif_carrier_off(dev);
+
+        netif_tx_disable(dev);
+
+        netif_device_detach(dev);
+
+#ifdef ENABLE_PTP_SUPPORT
+        rtl8127_ptp_suspend(tp);
+#endif
+        rtl8127_hw_reset(dev);
+
+        pci_clear_master(pdev);
+
+        rtl8127_hw_d3_para(dev);
+
+        rtl8127_powerdown_pll(dev, 1);
+out:
+        if (HW_DASH_SUPPORT_DASH(tp))
+                rtl8127_driver_stop(tp);
+
+        rtnl_unlock();
+
+        pci_disable_device(pdev);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10)
+        pci_save_state(pdev, &pci_pm_state);
+#else
+        pci_save_state(pdev);
+#endif
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29)
+        pci_enable_wake(pdev, pci_choose_state(pdev, state), tp->wol_enabled);
+#endif
+
+        pci_prepare_to_sleep(pdev);
+
+        return 0;
+}
+
+static int
+rtl8127_hw_d3_not_power_off(struct net_device *dev)
+{
+        return rtl8127_check_hw_phy_mcu_code_ver(dev);
+}
+
+static int rtl8127_wait_phy_nway_complete_sleep(struct rtl8127_private *tp)
+{
+        int i, val;
+
+        for (i = 0; i < 30; i++) {
+                val = rtl8127_mdio_read(tp, MII_BMSR) & BMSR_ANEGCOMPLETE;
+                if (val)
+                        return 0;
+
+                mdelay(100);
+        }
+
+        return -1;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29)
+static int
+rtl8127_resume(struct pci_dev *pdev)
+#else
+static int
+rtl8127_resume(struct device *device)
+#endif
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)
+        struct pci_dev *pdev = to_pci_dev(device);
+        struct net_device *dev = pci_get_drvdata(pdev);
+#else
+        struct net_device *dev = pci_get_drvdata(pdev);
+#endif
+        struct rtl8127_private *tp = netdev_priv(dev);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10)
+        u32 pci_pm_state = PCI_D0;
+#endif
+        unsigned long flags;
+        u32 err;
+
+        rtnl_lock();
+
+        err = pci_enable_device(pdev);
+        if (err) {
+                dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
+                goto out_unlock;
+        }
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,10)
+        pci_restore_state(pdev, &pci_pm_state);
+#else
+        pci_restore_state(pdev);
+#endif
+        pci_enable_wake(pdev, PCI_D0, 0);
+
+        /* restore last modified mac address */
+        rtl8127_rar_set(tp, dev->dev_addr);
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        tp->resume_not_chg_speed = 0;
+        if (tp->check_keep_link_speed &&
+            //tp->link_ok(dev) &&
+            rtl8127_hw_d3_not_power_off(dev) &&
+            rtl8127_wait_phy_nway_complete_sleep(tp) == 0)
+                tp->resume_not_chg_speed = 1;
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        if (!netif_running(dev))
+                goto out_unlock;
+
+        pci_set_master(pdev);
+
+        rtl8127_exit_oob(dev);
+
+        rtl8127_up(dev);
+
+        clear_bit(R8127_FLAG_DOWN, tp->task_flags);
+
+        rtl8127_schedule_reset_work(tp);
+
+        rtl8127_schedule_esd_work(tp);
+
+        //mod_timer(&tp->esd_timer, jiffies + RTL8127_ESD_TIMEOUT);
+        //mod_timer(&tp->link_timer, jiffies + RTL8127_LINK_TIMEOUT);
+out_unlock:
+        netif_device_attach(dev);
+
+        rtnl_unlock();
+
+        return err;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)
+
+static struct dev_pm_ops rtl8127_pm_ops = {
+        .suspend = rtl8127_suspend,
+        .resume = rtl8127_resume,
+        .freeze = rtl8127_suspend,
+        .thaw = rtl8127_resume,
+        .poweroff = rtl8127_suspend,
+        .restore = rtl8127_resume,
+};
+
+#define RTL8127_PM_OPS	(&rtl8127_pm_ops)
+
+#endif
+
+#else /* !CONFIG_PM */
+
+#define RTL8127_PM_OPS	NULL
+
+#endif /* CONFIG_PM */
+
+static struct pci_driver rtl8127_pci_driver = {
+        .name       = MODULENAME,
+        .id_table   = rtl8127_pci_tbl,
+        .probe      = rtl8127_init_one,
+        .remove     = __devexit_p(rtl8127_remove_one),
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,11)
+        .shutdown   = rtl8127_shutdown,
+#endif
+#ifdef CONFIG_PM
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,29)
+        .suspend    = rtl8127_suspend,
+        .resume     = rtl8127_resume,
+#else
+        .driver.pm	= RTL8127_PM_OPS,
+#endif
+#endif
+};
+
+static int __init
+rtl8127_init_module(void)
+{
+        int ret = 0;
+#ifdef ENABLE_R8127_PROCFS
+        rtl8127_proc_module_init();
+#endif
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0)
+
+        ret = pci_register_driver(&rtl8127_pci_driver);
+#else
+        ret = pci_module_init(&rtl8127_pci_driver);
+#endif
+
+        return ret;
+}
+
+static void __exit
+rtl8127_cleanup_module(void)
+{
+        pci_unregister_driver(&rtl8127_pci_driver);
+
+#ifdef ENABLE_R8127_PROCFS
+        if (rtl8127_proc) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+                remove_proc_subtree(MODULENAME, init_net.proc_net);
+#else
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32)
+                remove_proc_entry(MODULENAME, init_net.proc_net);
+#else
+                remove_proc_entry(MODULENAME, proc_net);
+#endif  //LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32)
+#endif  //LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0)
+                rtl8127_proc = NULL;
+        }
+#endif
+}
+
+module_init(rtl8127_init_module);
+module_exit(rtl8127_cleanup_module);
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_ptp.c b/drivers/net/ethernet/realtek/r8127/r8127_ptp.c
new file mode 100644
index 000000000000..129feb1f383d
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_ptp.c
@@ -0,0 +1,960 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/delay.h>
+#include <linux/mii.h>
+#include <linux/in.h>
+#include <linux/ethtool.h>
+#include <linux/rtnetlink.h>
+
+#include "r8127.h"
+#include "r8127_ptp.h"
+
+static void rtl8127_wait_clkadj_ready(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++)
+                if (!(rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CLK_CFG_8126) & CLKADJ_MODE_SET))
+                        break;
+}
+
+static void rtl8127_set_clkadj_mode(struct rtl8127_private *tp, u16 cmd)
+{
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              PTP_CLK_CFG_8126,
+                                              BIT_3 | BIT_2 | BIT_1,
+                                              CLKADJ_MODE_SET | cmd);
+
+        rtl8127_wait_clkadj_ready(tp);
+}
+
+static int _rtl8127_phc_gettime(struct rtl8127_private *tp, struct timespec64 *ts64)
+{
+        unsigned long flags;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        //Direct Read
+        rtl8127_set_clkadj_mode(tp, DIRECT_READ);
+
+        /* nanoseconds */
+        //Ns[29:16] E414[13:0]
+        ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_NS_HI_8126) & 0x3fff;
+        ts64->tv_nsec <<= 16;
+        //Ns[15:0]  E412[15:0]
+        ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_NS_LO_8126);
+
+
+        /* seconds */
+        //S[47:32] E41A[15:0]
+        ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_HI_8126);
+        ts64->tv_sec <<= 16;
+        //S[31:16] E418[15:0]
+        ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_MI_8126);
+        ts64->tv_sec <<= 16;
+        //S[15:0]  E416[15:0]
+        ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_LO_8126);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return 0;
+}
+
+static int _rtl8127_phc_settime(struct rtl8127_private *tp, const struct timespec64 *ts64)
+{
+        unsigned long flags;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        /* nanoseconds */
+        //Ns[15:0]  E412[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, ts64->tv_nsec);
+        //Ns[29:16] E414[13:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (ts64->tv_nsec & 0x3fff0000) >> 16);
+
+        /* seconds */
+        //S[15:0]  E416[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_LO_8126, ts64->tv_sec);
+        //S[31:16] E418[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_MI_8126, (ts64->tv_sec >> 16));
+        //S[47:32] E41A[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_HI_8126, (ts64->tv_sec >> 32));
+
+        //Direct Write
+        rtl8127_set_clkadj_mode(tp, DIRECT_WRITE);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return 0;
+}
+
+static int _rtl8127_phc_adjtime(struct rtl8127_private *tp, s64 delta)
+{
+        unsigned long flags;
+        struct timespec64 d;
+        bool negative;
+        u64 tohw;
+        u32 nsec;
+        u64 sec;
+
+        if (delta < 0) {
+                negative = true;
+                tohw = -delta;
+        } else {
+                negative = false;
+                tohw = delta;
+        }
+
+        d = ns_to_timespec64(tohw);
+
+        nsec = d.tv_nsec;
+        sec = d.tv_sec;
+
+        nsec &= 0x3fffffff;
+        sec &= 0x0000ffffffffffff;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        /* nanoseconds */
+        //Ns[15:0]  E412[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, nsec);
+        //Ns[29:16] E414[13:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (nsec >> 16));
+
+        /* seconds */
+        //S[15:0]  E416[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_LO_8126, sec);
+        //S[31:16] E418[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_MI_8126, (sec >> 16));
+        //S[47:32] E41A[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_S_HI_8126, (sec >> 32));
+
+        if (negative)
+                rtl8127_set_clkadj_mode(tp, DECREMENT_STEP);
+        else
+                rtl8127_set_clkadj_mode(tp, INCREMENT_STEP);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return 0;
+}
+
+static int rtl8127_phc_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+        struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info);
+        int ret;
+
+        //netif_info(tp, drv, tp->dev, "phc adjust time\n");
+
+        ret = _rtl8127_phc_adjtime(tp, delta);
+
+        return ret;
+}
+
+/*
+ * delta = delta * 10^6 ppm = delta * 10^9 ppb (in this equation ppm and ppb are not variable)
+ *
+ * in adjfreq ppb is a variable
+ * ppb = delta * 10^9
+ * delta = ppb / 10^9
+ * rate_value = |delta| * 2^32 = |ppb| / 10^9 * 2^32 = (|ppb| << 32) / 10^9
+ */
+static int _rtl8127_phc_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
+{
+        struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info);
+        unsigned long flags;
+        u32 rate_value;
+
+        if (ppb < 0) {
+                rate_value = ((u64)-ppb << 32) / 1000000000;
+                rate_value = ~rate_value + 1;
+        } else
+                rate_value = ((u64)ppb << 32) / 1000000000;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        /* nanoseconds */
+        //Ns[15:0]  E412[15:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_LO_8126, rate_value);
+        //Ns[22:16] E414[13:0]
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CFG_NS_HI_8126, (rate_value & 0x003f0000) >> 16);
+
+        rtl8127_set_clkadj_mode(tp, RATE_WRITE);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return 0;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0)
+static int rtl8127_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
+{
+        s32 ppb = scaled_ppm_to_ppb(scaled_ppm);
+
+        if (ppb > ptp->max_adj || ppb < -ptp->max_adj)
+                return -EINVAL;
+
+        _rtl8127_phc_adjfreq(ptp, ppb);
+
+        return 0;
+}
+
+#else
+static int rtl8127_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta)
+{
+        //struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info);
+
+        //netif_info(tp, drv, tp->dev, "phc adjust freq\n");
+
+        if (delta > ptp->max_adj || delta < -ptp->max_adj)
+                return -EINVAL;
+
+        _rtl8127_phc_adjfreq(ptp, delta);
+
+        return 0;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) */
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0)
+static int rtl8127_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts64,
+                               struct ptp_system_timestamp *sts)
+{
+        struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info);
+        int ret;
+
+        //netif_info(tp, drv, tp->dev, "phc get ts\n");
+
+        ptp_read_system_prets(sts);
+        ret = _rtl8127_phc_gettime(tp, ts64);
+        ptp_read_system_postts(sts);
+
+        return ret;
+}
+#else
+static int rtl8127_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts64)
+{
+        struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info);
+        int ret;
+
+        //netif_info(tp, drv, tp->dev, "phc get ts\n");
+
+        ret = _rtl8127_phc_gettime(tp, ts64);
+
+        return ret;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */
+
+static int rtl8127_phc_settime(struct ptp_clock_info *ptp,
+                               const struct timespec64 *ts64)
+{
+        struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info);
+        int ret;
+
+        //netif_info(tp, drv, tp->dev, "phc set ts\n");
+
+        ret = _rtl8127_phc_settime(tp, ts64);
+
+        return ret;
+}
+
+static void _rtl8127_phc_enable(struct ptp_clock_info *ptp,
+                                struct ptp_clock_request *rq, int on)
+{
+        struct rtl8127_private *tp = container_of(ptp, struct rtl8127_private, ptp_clock_info);
+        unsigned long flags;
+        u16 phy_ocp_data;
+
+        if (on) {
+                tp->pps_enable = 1;
+                rtl8127_clear_mac_ocp_bit(tp, 0xDC00, BIT_6);
+                rtl8127_clear_mac_ocp_bit(tp, 0xDC20, BIT_1);
+
+                r8127_spin_lock(&tp->phy_lock, flags);
+
+                /* Set periodic pulse 1pps */
+                /* E432[8:0] = 0x017d */
+                phy_ocp_data = rtl8127_mdio_direct_read_phy_ocp(tp, 0xE432);
+                phy_ocp_data &= 0xFE00;
+                phy_ocp_data |= 0x017d;
+                rtl8127_mdio_direct_write_phy_ocp(tp, 0xE432, phy_ocp_data);
+
+                rtl8127_mdio_direct_write_phy_ocp(tp, 0xE434, 0x7840);
+
+                /* E436[8:0] = 0xbe */
+                phy_ocp_data = rtl8127_mdio_direct_read_phy_ocp(tp, 0xE436);
+                phy_ocp_data &= 0xFE00;
+                phy_ocp_data |= 0xbe;
+                rtl8127_mdio_direct_write_phy_ocp(tp, 0xE436, phy_ocp_data);
+
+                rtl8127_mdio_direct_write_phy_ocp(tp, 0xE438, 0xbc20);
+
+                r8127_spin_unlock(&tp->phy_lock, flags);
+
+                /* start hrtimer */
+                hrtimer_start(&tp->pps_timer, 1000000000, HRTIMER_MODE_REL);
+        } else
+                tp->pps_enable = 0;
+}
+
+static int rtl8127_phc_enable(struct ptp_clock_info *ptp,
+                              struct ptp_clock_request *rq, int on)
+{
+        switch (rq->type) {
+        case PTP_CLK_REQ_PPS:
+                _rtl8127_phc_enable(ptp, rq, on);
+                return 0;
+        default:
+                return -EOPNOTSUPP;
+        }
+}
+
+static void rtl8127_ptp_enable_config(struct rtl8127_private *tp)
+{
+        if (tp->syncE_en)
+                rtl8127_set_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0);
+        else
+                rtl8127_clear_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0);
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_CTL, PTP_CTL_TYPE_3 | BIT_12);
+
+        rtl8127_set_eth_phy_ocp_bit(tp, 0xA640, BIT_15);
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,11,0)
+int rtl8127_get_ts_info(struct net_device *netdev,
+                        struct ethtool_ts_info *info)
+#else
+int rtl8127_get_ts_info(struct net_device *netdev,
+                        struct kernel_ethtool_ts_info *info)
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,11,0) */
+{
+        struct rtl8127_private *tp = netdev_priv(netdev);
+
+        /* we always support timestamping disabled */
+        info->rx_filters = BIT(HWTSTAMP_FILTER_NONE);
+
+        if (tp->HwSuppPtpVer == 0)
+                return ethtool_op_get_ts_info(netdev, info);
+
+        info->so_timestamping =  SOF_TIMESTAMPING_TX_SOFTWARE |
+                                 SOF_TIMESTAMPING_RX_SOFTWARE |
+                                 SOF_TIMESTAMPING_SOFTWARE |
+                                 SOF_TIMESTAMPING_TX_HARDWARE |
+                                 SOF_TIMESTAMPING_RX_HARDWARE |
+                                 SOF_TIMESTAMPING_RAW_HARDWARE;
+
+        if (tp->ptp_clock)
+                info->phc_index = ptp_clock_index(tp->ptp_clock);
+        else
+                info->phc_index = -1;
+
+        info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON);
+
+        info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
+                           BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
+                           BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) |
+                           BIT(HWTSTAMP_FILTER_PTP_V2_SYNC) |
+                           BIT(HWTSTAMP_FILTER_PTP_V2_L4_SYNC) |
+                           BIT(HWTSTAMP_FILTER_PTP_V2_DELAY_REQ) |
+                           BIT(HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ);
+
+        return 0;
+}
+
+static const struct ptp_clock_info rtl_ptp_clock_info = {
+        .owner      = THIS_MODULE,
+        .n_alarm    = 0,
+        .n_ext_ts   = 0,
+        .n_per_out  = 0,
+        .n_pins     = 0,
+        .pps        = 1,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0)
+        .adjfine   = rtl8127_ptp_adjfine,
+#else
+        .adjfreq    = rtl8127_phc_adjfreq,
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,2,0) */
+        .adjtime    = rtl8127_phc_adjtime,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0)
+        .gettimex64 = rtl8127_phc_gettime,
+#else
+        .gettime64  = rtl8127_phc_gettime,
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5,0,0) */
+
+        .settime64  = rtl8127_phc_settime,
+        .enable     = rtl8127_phc_enable,
+};
+
+static u16 rtl8127_ptp_get_tx_msgtype(struct rtl8127_private *tp)
+{
+        u16 tx_ts_ready = 0;
+        int i;
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                tx_ts_ready = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & 0xF000;
+                if (tx_ts_ready)
+                        break;
+        }
+
+        switch (tx_ts_ready) {
+        case TX_TS_PDLYRSP_RDY:
+                return PTP_MSGTYPE_PDELAY_RESP;
+        case TX_TS_PDLYREQ_RDY:
+                return PTP_MSGTYPE_PDELAY_REQ;
+        case TX_TS_DLYREQ_RDY:
+                return PTP_MSGTYPE_DELAY_REQ;
+        case TX_TS_SYNC_RDY:
+        default:
+                return PTP_MSGTYPE_SYNC;
+        }
+}
+
+/*
+static u16 rtl8127_ptp_get_rx_msgtype(struct rtl8127_private *tp)
+{
+        u16 rx_ts_ready = 0;
+        int i;
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++) {
+                rx_ts_ready = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & 0x0F00;
+                if (rx_ts_ready)
+                        break;
+        }
+
+        switch (rx_ts_ready) {
+        case RX_TS_PDLYRSP_RDY:
+                return PTP_MSGTYPE_PDELAY_RESP;
+        case RX_TS_PDLYREQ_RDY:
+                return PTP_MSGTYPE_PDELAY_REQ;
+        case RX_TS_DLYREQ_RDY:
+                return PTP_MSGTYPE_DELAY_REQ;
+        case RX_TS_SYNC_RDY:
+        default:
+                return PTP_MSGTYPE_SYNC;
+        }
+}
+*/
+
+static void rtl8127_wait_trx_ts_ready(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < R8127_CHANNEL_WAIT_COUNT; i++)
+                if (!(rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_STA) & TRX_TS_RD))
+                        break;
+}
+
+static void rtl8127_set_trx_ts_cmd(struct rtl8127_private *tp, u16 cmd)
+{
+        rtl8127_clear_and_set_eth_phy_ocp_bit(tp,
+                                              PTP_TRX_TS_STA,
+                                              TRXTS_SEL | BIT_3 | BIT_2,
+                                              TRX_TS_RD | cmd);
+
+        rtl8127_wait_trx_ts_ready(tp);
+}
+
+static void rtl8127_ptp_egresstime(struct rtl8127_private *tp, struct timespec64 *ts64)
+{
+        u16 msgtype;
+
+        msgtype = rtl8127_ptp_get_tx_msgtype(tp);
+
+        msgtype <<= 2;
+
+        rtl8127_set_trx_ts_cmd(tp, (msgtype | BIT_4));
+
+        /* nanoseconds */
+        //Ns[29:16] E448[13:0]
+        ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_HI) & 0x3fff;
+        ts64->tv_nsec <<= 16;
+        //Ns[15:0]  E446[15:0]
+        ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_LO);
+
+        /* seconds */
+        //S[47:32] E44E[15:0]
+        ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_HI);
+        ts64->tv_sec <<= 16;
+        //S[31:16] E44C[15:0]
+        ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_MI);
+        ts64->tv_sec <<= 16;
+        //S[15:0]  E44A[15:0]
+        ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_LO);
+}
+
+static void rtl8127_ptp_ingresstime(struct rtl8127_private *tp, struct timespec64 *ts64, u8 type)
+{
+        u16 msgtype;
+
+        switch (type) {
+        case PTP_MSGTYPE_PDELAY_RESP:
+        case PTP_MSGTYPE_PDELAY_REQ:
+        case PTP_MSGTYPE_DELAY_REQ:
+        case PTP_MSGTYPE_SYNC:
+                msgtype = type << 2;
+                break;
+        default:
+                return;
+        }
+
+        rtl8127_set_trx_ts_cmd(tp, (TRXTS_SEL | msgtype | BIT_4));
+
+        /* nanoseconds */
+        //Ns[29:16] E448[13:0]
+        ts64->tv_nsec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_HI) & 0x3fff;
+        ts64->tv_nsec <<= 16;
+        //Ns[15:0]  E446[15:0]
+        ts64->tv_nsec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_NS_LO);
+
+        /* seconds */
+        //S[47:32] E44E[15:0]
+        ts64->tv_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_HI);
+        ts64->tv_sec <<= 16;
+        //S[31:16] E44C[15:0]
+        ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_MI);
+        ts64->tv_sec <<= 16;
+        //S[15:0]  E44A[15:0]
+        ts64->tv_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_TRX_TS_S_LO);
+}
+
+static void rtl8127_ptp_tx_hwtstamp(struct rtl8127_private *tp)
+{
+        struct sk_buff *skb = tp->ptp_tx_skb;
+        struct skb_shared_hwtstamps shhwtstamps = { 0 };
+        struct timespec64 ts64;
+
+        rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, TX_TX_INTR);
+
+        rtl8127_ptp_egresstime(tp, &ts64);
+
+        /* Upper 32 bits contain s, lower 32 bits contain ns. */
+        shhwtstamps.hwtstamp = ktime_set(ts64.tv_sec,
+                                         ts64.tv_nsec);
+
+        /* Clear the lock early before calling skb_tstamp_tx so that
+         * applications are not woken up before the lock bit is clear. We use
+         * a copy of the skb pointer to ensure other threads can't change it
+         * while we're notifying the stack.
+         */
+        tp->ptp_tx_skb = NULL;
+        clear_bit_unlock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state);
+
+        /* Notify the stack and free the skb after we've unlocked */
+        skb_tstamp_tx(skb, &shhwtstamps);
+        dev_kfree_skb_any(skb);
+}
+
+#define RTL8127_PTP_TX_TIMEOUT      (HZ * 15)
+static void rtl8127_ptp_tx_work(struct work_struct *work)
+{
+        struct rtl8127_private *tp = container_of(work, struct rtl8127_private,
+                                     ptp_tx_work);
+        unsigned long flags;
+        bool tx_intr;
+
+        if (!tp->ptp_tx_skb)
+                return;
+
+        if (time_is_before_jiffies(tp->ptp_tx_start +
+                                   RTL8127_PTP_TX_TIMEOUT)) {
+                dev_kfree_skb_any(tp->ptp_tx_skb);
+                tp->ptp_tx_skb = NULL;
+                clear_bit_unlock(__RTL8127_PTP_TX_IN_PROGRESS, &tp->state);
+                tp->tx_hwtstamp_timeouts++;
+                /* Clear the tx valid bit in TSYNCTXCTL register to enable
+                 * interrupt
+                 */
+                r8127_spin_lock(&tp->phy_lock, flags);
+                rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, TX_TX_INTR);
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                return;
+        }
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+        if (rtl8127_mdio_direct_read_phy_ocp(tp, PTP_INSR) & TX_TX_INTR) {
+                tx_intr = true;
+                rtl8127_ptp_tx_hwtstamp(tp);
+        } else {
+                tx_intr = false;
+        }
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        if (!tx_intr) {
+                /* reschedule to check later */
+                schedule_work(&tp->ptp_tx_work);
+        }
+}
+
+static int rtl8127_hwtstamp_enable(struct rtl8127_private *tp, bool enable)
+{
+        unsigned long flags;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        if (enable) {
+                //trx timestamp interrupt enable
+                rtl8127_set_eth_phy_ocp_bit(tp, PTP_INER, BIT_2 | BIT_3);
+
+                //set isr clear mode
+                rtl8127_set_eth_phy_ocp_bit(tp, PTP_GEN_CFG, BIT_0);
+
+                //clear ptp isr
+                rtl8127_mdio_direct_write_phy_ocp(tp, PTP_INSR, 0xFFFF);
+
+                //enable ptp
+                rtl8127_ptp_enable_config(tp);
+
+                //rtl8127_set_local_time(tp);
+        } else {
+                /* trx timestamp interrupt disable */
+                rtl8127_clear_eth_phy_ocp_bit(tp, PTP_INER, BIT_2 | BIT_3);
+
+                /* disable ptp */
+                rtl8127_clear_eth_phy_ocp_bit(tp, PTP_SYNCE_CTL, BIT_0);
+                rtl8127_clear_eth_phy_ocp_bit(tp, PTP_CTL, BIT_0);
+                rtl8127_set_eth_phy_ocp_bit(tp, 0xA640, BIT_15);
+        }
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        return 0;
+}
+
+void rtl8127_set_local_time(struct rtl8127_private *tp)
+{
+        struct timespec64 ts64;
+        //set system time
+        ktime_get_real_ts64(&ts64);
+        _rtl8127_phc_settime(tp, &ts64);
+}
+
+static long rtl8127_ptp_create_clock(struct rtl8127_private *tp)
+{
+        struct net_device *netdev = tp->dev;
+        long err;
+
+        if (!IS_ERR_OR_NULL(tp->ptp_clock))
+                return 0;
+
+        if (tp->HwSuppPtpVer == 0) {
+                tp->ptp_clock = NULL;
+                return -EOPNOTSUPP;
+        }
+
+        tp->ptp_clock_info = rtl_ptp_clock_info;
+        tp->ptp_clock_info.max_adj = 488281;//0x1FFFFF * 10^9 / 2^32
+
+        snprintf(tp->ptp_clock_info.name, sizeof(tp->ptp_clock_info.name),
+                 "%pm", tp->dev->dev_addr);
+        tp->ptp_clock = ptp_clock_register(&tp->ptp_clock_info, &tp->pci_dev->dev);
+        if (IS_ERR(tp->ptp_clock)) {
+                err = PTR_ERR(tp->ptp_clock);
+                tp->ptp_clock = NULL;
+                netif_err(tp, drv, tp->dev, "ptp_clock_register failed\n");
+                return err;
+        } else
+                netif_info(tp, drv, tp->dev, "registered PHC device on %s\n", netdev->name);
+
+        return 0;
+}
+
+static enum hrtimer_restart
+rtl8127_hrtimer_for_pps(struct hrtimer *timer) {
+        struct rtl8127_private *tp = container_of(timer, struct rtl8127_private, pps_timer);
+        u16 tai_cfg = BIT_8 | BIT_3 | BIT_1 | BIT_0;
+        s64 pps_sec;
+
+        if (tp->pps_enable)
+        {
+                unsigned long flags;
+
+                r8127_spin_lock(&tp->phy_lock, flags);
+
+                //Direct Read
+                rtl8127_set_clkadj_mode(tp, DIRECT_READ);
+
+                pps_sec = rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_HI_8126);
+                pps_sec <<= 16;
+                pps_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_MI_8126);
+                pps_sec <<= 16;
+                pps_sec |= rtl8127_mdio_direct_read_phy_ocp(tp, PTP_CFG_S_LO_8126);
+                pps_sec++;
+
+                //E42A[15:0]
+                rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_TS_S_LO, pps_sec & 0xffff);
+                //E42C[31:16]
+                rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_TS_S_HI, (pps_sec & 0xffff0000) >> 16);
+                //Periodic Tai start
+                rtl8127_mdio_direct_write_phy_ocp(tp, PTP_TAI_CFG, tai_cfg);
+
+                r8127_spin_unlock(&tp->phy_lock, flags);
+
+                hrtimer_forward_now(&tp->pps_timer, 1000000000); //rekick
+                return HRTIMER_RESTART;
+        } else
+                return HRTIMER_NORESTART;
+}
+
+void rtl8127_ptp_reset(struct rtl8127_private *tp)
+{
+        if (!tp->ptp_clock)
+                return;
+
+        netif_info(tp, drv, tp->dev, "reset PHC clock\n");
+
+        rtl8127_hwtstamp_enable(tp, false);
+}
+
+void rtl8127_ptp_init(struct rtl8127_private *tp)
+{
+        /* obtain a PTP device, or re-use an existing device */
+        if (rtl8127_ptp_create_clock(tp))
+                return;
+
+        /* we have a clock so we can initialize work now */
+        INIT_WORK(&tp->ptp_tx_work, rtl8127_ptp_tx_work);
+
+        /* init a hrtimer for pps */
+        tp->pps_enable = 0;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 18, 0)
+	hrtimer_init(&tp->pps_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	tp->pps_timer.function = rtl8127_hrtimer_for_pps;
+#else
+	hrtimer_setup(&tp->pps_timer, rtl8127_hrtimer_for_pps,
+			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+#endif
+
+        /* reset the PTP related hardware bits */
+        rtl8127_ptp_reset(tp);
+
+        return;
+}
+
+void rtl8127_ptp_suspend(struct rtl8127_private *tp)
+{
+        if (!tp->ptp_clock)
+                return;
+
+        netif_info(tp, drv, tp->dev, "suspend PHC clock\n");
+
+        rtl8127_hwtstamp_enable(tp, false);
+
+        /* ensure that we cancel any pending PTP Tx work item in progress */
+        cancel_work_sync(&tp->ptp_tx_work);
+
+        hrtimer_cancel(&tp->pps_timer);
+}
+
+void rtl8127_ptp_stop(struct rtl8127_private *tp)
+{
+        struct net_device *netdev = tp->dev;
+
+        netif_info(tp, drv, tp->dev, "stop PHC clock\n");
+
+        /* first, suspend PTP activity */
+        rtl8127_ptp_suspend(tp);
+
+        /* disable the PTP clock device */
+        if (tp->ptp_clock) {
+                ptp_clock_unregister(tp->ptp_clock);
+                tp->ptp_clock = NULL;
+                netif_info(tp, drv, tp->dev, "removed PHC on %s\n",
+                           netdev->name);
+        }
+}
+
+static int rtl8127_set_tstamp(struct net_device *netdev, struct ifreq *ifr)
+{
+        struct rtl8127_private *tp = netdev_priv(netdev);
+        struct hwtstamp_config config;
+        bool hwtstamp = 0;
+
+        //netif_info(tp, drv, tp->dev, "ptp set ts\n");
+
+        if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
+                return -EFAULT;
+
+        if (config.flags)
+                return -EINVAL;
+
+        switch (config.tx_type) {
+        case HWTSTAMP_TX_ON:
+                hwtstamp = 1;
+                break;
+        case HWTSTAMP_TX_OFF:
+                break;
+        case HWTSTAMP_TX_ONESTEP_SYNC:
+        default:
+                return -ERANGE;
+        }
+
+        switch (config.rx_filter) {
+        case HWTSTAMP_FILTER_PTP_V2_EVENT:
+        case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
+        case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+        case HWTSTAMP_FILTER_PTP_V2_SYNC:
+        case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
+        case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+        case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
+        case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
+        case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+                config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
+                hwtstamp = 1;
+                tp->flags |= RTL_FLAG_RX_HWTSTAMP_ENABLED;
+                break;
+        case HWTSTAMP_FILTER_NONE:
+                tp->flags &= ~RTL_FLAG_RX_HWTSTAMP_ENABLED;
+                break;
+        default:
+                tp->flags &= ~RTL_FLAG_RX_HWTSTAMP_ENABLED;
+                return -ERANGE;
+        }
+
+        if (tp->hwtstamp_config.tx_type != config.tx_type ||
+            tp->hwtstamp_config.rx_filter != config.rx_filter) {
+                tp->hwtstamp_config = config;
+
+                rtl8127_hwtstamp_enable(tp, hwtstamp);
+        }
+
+        return copy_to_user(ifr->ifr_data, &config,
+                            sizeof(config)) ? -EFAULT : 0;
+}
+
+static int rtl8127_get_tstamp(struct net_device *netdev, struct ifreq *ifr)
+{
+        struct rtl8127_private *tp = netdev_priv(netdev);
+
+        //netif_info(tp, drv, tp->dev, "ptp get ts\n");
+
+        return copy_to_user(ifr->ifr_data, &tp->hwtstamp_config,
+                            sizeof(tp->hwtstamp_config)) ? -EFAULT : 0;
+}
+
+int rtl8127_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
+{
+        int ret;
+
+        //netif_info(tp, drv, tp->dev, "ptp ioctl\n");
+
+        switch (cmd) {
+#ifdef ENABLE_PTP_SUPPORT
+        case SIOCSHWTSTAMP:
+                ret = rtl8127_set_tstamp(netdev, ifr);
+                break;
+        case SIOCGHWTSTAMP:
+                ret = rtl8127_get_tstamp(netdev, ifr);
+                break;
+#endif
+        default:
+                ret = -EOPNOTSUPP;
+                break;
+        }
+
+        return ret;
+}
+
+static void rtl8127_rx_ptp_pktstamp(struct rtl8127_private *tp, struct sk_buff *skb, u8 type)
+{
+        struct timespec64 ts64;
+        unsigned long flags;
+
+        r8127_spin_lock(&tp->phy_lock, flags);
+
+        rtl8127_ptp_ingresstime(tp, &ts64, type);
+
+        r8127_spin_unlock(&tp->phy_lock, flags);
+
+        skb_hwtstamps(skb)->hwtstamp = ktime_set(ts64.tv_sec, ts64.tv_nsec);
+
+        return;
+}
+
+void rtl8127_rx_ptp_timestamp(struct rtl8127_private *tp, struct sk_buff *skb)
+{
+        unsigned int ptp_class;
+        struct ptp_header *hdr;
+        u8 msgtype;
+
+        ptp_class = ptp_classify_raw(skb);
+        if (ptp_class == PTP_CLASS_NONE)
+                return;
+
+        skb_reset_mac_header(skb);
+        hdr = ptp_parse_header(skb, ptp_class);
+        if (unlikely(!hdr))
+                return;
+
+        msgtype = ptp_get_msgtype(hdr, ptp_class);
+        rtl8127_rx_ptp_pktstamp(tp, skb, msgtype);
+
+        return;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)
+struct ptp_header *ptp_parse_header(struct sk_buff *skb, unsigned int type)
+{
+        u8 *ptr = skb_mac_header(skb);
+
+        if (type & PTP_CLASS_VLAN)
+                //ptr += VLAN_HLEN;
+                ptr += 4;
+
+        switch (type & PTP_CLASS_PMASK) {
+        case PTP_CLASS_IPV4:
+                ptr += IPV4_HLEN(ptr) + UDP_HLEN;
+                break;
+        case PTP_CLASS_IPV6:
+                ptr += IP6_HLEN + UDP_HLEN;
+                break;
+        case PTP_CLASS_L2:
+                break;
+        default:
+                return NULL;
+        }
+
+        ptr += ETH_HLEN;
+
+        /* Ensure that the entire header is present in this packet. */
+        if (ptr + sizeof(struct ptp_header) > skb->data + skb->len)
+                return NULL;
+
+        return (struct ptp_header *)ptr;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) */
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_ptp.h b/drivers/net/ethernet/realtek/r8127/r8127_ptp.h
new file mode 100644
index 000000000000..3365c0b0578a
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_ptp.h
@@ -0,0 +1,207 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_R8127_PTP_H
+#define _LINUX_R8127_PTP_H
+
+#include <linux/ktime.h>
+#include <linux/timecounter.h>
+#include <linux/net_tstamp.h>
+#include <linux/ptp_clock_kernel.h>
+#include <linux/ptp_classify.h>
+
+#ifndef PTP_CLASS_NONE
+#define PTP_CLASS_NONE  0x00
+#endif
+
+#ifndef PTP_MSGTYPE_SYNC
+#define PTP_MSGTYPE_SYNC        0x0
+#endif
+#ifndef PTP_MSGTYPE_DELAY_REQ
+#define PTP_MSGTYPE_DELAY_REQ   0x1
+#endif
+#ifndef PTP_MSGTYPE_PDELAY_REQ
+#define PTP_MSGTYPE_PDELAY_REQ  0x2
+#endif
+#ifndef PTP_MSGTYPE_PDELAY_RESP
+#define PTP_MSGTYPE_PDELAY_RESP 0x3
+#endif
+
+struct rtl8127_ptp_info {
+        s64 time_sec;
+        u32 time_ns;
+        u16 ts_info;
+};
+
+#ifndef _STRUCT_TIMESPEC
+#define _STRUCT_TIMESPEC
+struct timespec {
+        __kernel_old_time_t tv_sec;     /* seconds */
+        long            tv_nsec;    /* nanoseconds */
+};
+#endif
+
+enum PTP_CMD_TYPE {
+        PTP_CMD_SET_LOCAL_TIME = 0,
+        PTP_CMD_DRIFT_LOCAL_TIME,
+        PTP_CMD_LATCHED_LOCAL_TIME,
+};
+
+enum PTP_CLKADJ_MOD_TYPE {
+        NO_FUNCTION     = 0,
+        CLKADJ_MODE_SET = 1,
+        RESERVED        = 2,
+        DIRECT_READ     = 4,
+        DIRECT_WRITE    = 6,
+        INCREMENT_STEP  = 8,
+        DECREMENT_STEP  = 10,
+        RATE_READ       = 12,
+        RATE_WRITE      = 14,
+};
+
+enum PTP_INSR_TYPE {
+        EVENT_CAP_INTR   = (1 << 0),
+        TRIG_GEN_INTR    = (1 << 1),
+        RX_TS_INTR       = (1 << 2),
+        TX_TX_INTR       = (1 << 3),
+};
+
+enum PTP_TRX_TS_STA_REG {
+        TRX_TS_RD               = (1 << 0),
+        TRXTS_SEL               = (1 << 1),
+        RX_TS_PDLYRSP_RDY       = (1 << 8),
+        RX_TS_PDLYREQ_RDY       = (1 << 9),
+        RX_TS_DLYREQ_RDY        = (1 << 10),
+        RX_TS_SYNC_RDY          = (1 << 11),
+        TX_TS_PDLYRSP_RDY       = (1 << 12),
+        TX_TS_PDLYREQ_RDY       = (1 << 13),
+        TX_TS_DLYREQ_RDY        = (1 << 14),
+        TX_TS_SYNC_RDY          = (1 << 15),
+};
+
+#define PTP_CTL_TYPE_0 (0xF3F)
+#define PTP_CTL_TYPE_1 (0x2FF)
+#define PTP_CTL_TYPE_2 (0x0FF)
+#define PTP_CTL_TYPE_3 (0x03F)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)
+struct clock_identity {
+        u8 id[8];
+} __packed;
+
+struct port_identity {
+        struct clock_identity	clock_identity;
+        __be16			port_number;
+} __packed;
+
+struct ptp_header {
+        u8			tsmt;  /* transportSpecific | messageType */
+        u8			ver;   /* reserved          | versionPTP  */
+        __be16			message_length;
+        u8			domain_number;
+        u8			reserved1;
+        u8			flag_field[2];
+        __be64			correction;
+        __be32			reserved2;
+        struct port_identity	source_port_identity;
+        __be16			sequence_id;
+        u8			control;
+        u8			log_message_interval;
+} __packed;
+
+/**
+ * ptp_parse_header - Get pointer to the PTP v2 header
+ * @skb: packet buffer
+ * @type: type of the packet (see ptp_classify_raw())
+ *
+ * This function takes care of the VLAN, UDP, IPv4 and IPv6 headers. The length
+ * is checked.
+ *
+ * Note, internally skb_mac_header() is used. Make sure that the @skb is
+ * initialized accordingly.
+ *
+ * Return: Pointer to the ptp v2 header or NULL if not found
+ */
+struct ptp_header *ptp_parse_header(struct sk_buff *skb, unsigned int type);
+
+/**
+ * ptp_get_msgtype - Extract ptp message type from given header
+ * @hdr: ptp header
+ * @type: type of the packet (see ptp_classify_raw())
+ *
+ * This function returns the message type for a given ptp header. It takes care
+ * of the different ptp header versions (v1 or v2).
+ *
+ * Return: The message type
+ */
+static inline u8 ptp_get_msgtype(const struct ptp_header *hdr,
+                                 unsigned int type)
+{
+        u8 msgtype;
+
+        if (unlikely(type & PTP_CLASS_V1)) {
+                /* msg type is located at the control field for ptp v1 */
+                msgtype = hdr->control;
+        } else {
+                msgtype = hdr->tsmt & 0x0f;
+        }
+
+        return msgtype;
+}
+
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0) */
+
+struct rtl8127_private;
+struct RxDescV3;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,11,0)
+int rtl8127_get_ts_info(struct net_device *netdev,
+                        struct ethtool_ts_info *info);
+#else
+int rtl8127_get_ts_info(struct net_device *netdev,
+                        struct kernel_ethtool_ts_info *info);
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(6,11,0) */
+
+void rtl8127_ptp_reset(struct rtl8127_private *tp);
+void rtl8127_ptp_init(struct rtl8127_private *tp);
+void rtl8127_ptp_suspend(struct rtl8127_private *tp);
+void rtl8127_ptp_stop(struct rtl8127_private *tp);
+
+int rtl8127_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
+
+void rtl8127_rx_ptp_timestamp(struct rtl8127_private *tp, struct sk_buff *skb);
+
+void rtl8127_set_local_time(struct rtl8127_private *tp);
+
+#endif /* _LINUX_R8127_PTP_H */
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_realwow.h b/drivers/net/ethernet/realtek/r8127/r8127_realwow.h
new file mode 100644
index 000000000000..a869b6c532b3
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_realwow.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_R8127_REALWOW_H
+#define _LINUX_R8127_REALWOW_H
+
+#define SIOCDEVPRIVATE_RTLREALWOW   SIOCDEVPRIVATE+3
+
+#define MAX_RealWoW_KCP_SIZE (100)
+#define MAX_RealWoW_Payload (64)
+
+#define KA_TX_PACKET_SIZE (100)
+#define KA_WAKEUP_PATTERN_SIZE (120)
+
+//HwSuppKeepAliveOffloadVer
+#define HW_SUPPORT_KCP_OFFLOAD(_M)        ((_M)->HwSuppKCPOffloadVer > 0)
+
+enum rtl_realwow_cmd {
+
+        RTL_REALWOW_SET_KCP_DISABLE=0,
+        RTL_REALWOW_SET_KCP_INFO,
+        RTL_REALWOW_SET_KCP_CONTENT,
+
+        RTL_REALWOW_SET_KCP_ACKPKTINFO,
+        RTL_REALWOW_SET_KCP_WPINFO,
+        RTL_REALWOW_SET_KCPDHCP_TIMEOUT,
+
+        RTLT_REALWOW_COMMAND_INVALID
+};
+
+struct rtl_realwow_ioctl_struct {
+        __u32	cmd;
+        __u32	offset;
+        __u32	len;
+        union {
+                __u32	data;
+                void *data_buffer;
+        };
+};
+
+typedef struct _MP_KCPInfo {
+        u8 DIPv4[4];
+        u8 MacID[6];
+        u16 UdpPort[2];
+        u8 PKTLEN[2];
+
+        u16 ackLostCnt;
+        u8 KCP_WakePattern[MAX_RealWoW_Payload];
+        u8 KCP_AckPacket[MAX_RealWoW_Payload];
+        u32 KCP_interval;
+        u8 KCP_WakePattern_Len;
+        u8 KCP_AckPacket_Len;
+        u8 KCP_TxPacket[2][KA_TX_PACKET_SIZE];
+} MP_KCP_INFO, *PMP_KCP_INFO;
+
+typedef struct _KCPInfo {
+        u32 nId; // = id
+        u8 DIPv4[4];
+        u8 MacID[6];
+        u16 UdpPort;
+        u16 PKTLEN;
+} KCPInfo, *PKCPInfo;
+
+typedef struct _KCPContent {
+        u32 id; // = id
+        u32 mSec; // = msec
+        u32 size; // =size
+        u8 bPacket[MAX_RealWoW_KCP_SIZE]; // put packet here
+} KCPContent, *PKCPContent;
+
+typedef struct _RealWoWAckPktInfo {
+        u16 ackLostCnt;
+        u16 patterntSize;
+        u8 pattern[MAX_RealWoW_Payload];
+} RealWoWAckPktInfo,*PRealWoWAckPktInfo;
+
+typedef struct _RealWoWWPInfo {
+        u16 patterntSize;
+        u8 pattern[MAX_RealWoW_Payload];
+} RealWoWWPInfo,*PRealWoWWPInfo;
+
+int rtl8127_realwow_ioctl(struct net_device *dev, struct ifreq *ifr);
+void rtl8127_realwow_hw_init(struct net_device *dev);
+void rtl8127_get_realwow_hw_version(struct net_device *dev);
+void rtl8127_set_realwow_d3_para(struct net_device *dev);
+
+#endif /* _LINUX_R8127_REALWOW_H */
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_rss.c b/drivers/net/ethernet/realtek/r8127/r8127_rss.c
new file mode 100644
index 000000000000..e36462191005
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_rss.c
@@ -0,0 +1,583 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#include <linux/version.h>
+#include "r8127.h"
+
+enum rtl8127_rss_register_content {
+        /* RSS */
+        RSS_CTRL_TCP_IPV4_SUPP = (1 << 0),
+        RSS_CTRL_IPV4_SUPP  = (1 << 1),
+        RSS_CTRL_TCP_IPV6_SUPP  = (1 << 2),
+        RSS_CTRL_IPV6_SUPP  = (1 << 3),
+        RSS_CTRL_IPV6_EXT_SUPP  = (1 << 4),
+        RSS_CTRL_TCP_IPV6_EXT_SUPP  = (1 << 5),
+        RSS_HALF_SUPP  = (1 << 7),
+        RSS_CTRL_UDP_IPV4_SUPP  = (1 << 11),
+        RSS_CTRL_UDP_IPV6_SUPP  = (1 << 12),
+        RSS_CTRL_UDP_IPV6_EXT_SUPP  = (1 << 13),
+        RSS_QUAD_CPU_EN  = (1 << 16),
+        RSS_HQ_Q_SUP_R  = (1 << 31),
+};
+
+static int rtl8127_get_rss_hash_opts(struct rtl8127_private *tp,
+                                     struct ethtool_rxnfc *cmd)
+{
+        cmd->data = 0;
+
+        /* Report default options for RSS */
+        switch (cmd->flow_type) {
+        case TCP_V4_FLOW:
+                cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+                fallthrough;
+        case UDP_V4_FLOW:
+                if (tp->rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4)
+                        cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+                fallthrough;
+        case IPV4_FLOW:
+                cmd->data |= RXH_IP_SRC | RXH_IP_DST;
+                break;
+        case TCP_V6_FLOW:
+                cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+                fallthrough;
+        case UDP_V6_FLOW:
+                if (tp->rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6)
+                        cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+                fallthrough;
+        case IPV6_FLOW:
+                cmd->data |= RXH_IP_SRC | RXH_IP_DST;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+int rtl8127_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
+                      u32 *rule_locs)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ret = -EOPNOTSUPP;
+
+        if (!(dev->features & NETIF_F_RXHASH))
+                return ret;
+
+        switch (cmd->cmd) {
+        case ETHTOOL_GRXRINGS:
+                cmd->data = rtl8127_tot_rx_rings(tp);
+                ret = 0;
+                break;
+        case ETHTOOL_GRXFH:
+                ret = rtl8127_get_rss_hash_opts(tp, cmd);
+                break;
+        default:
+                break;
+        }
+
+        return ret;
+}
+
+u32 rtl8127_rss_indir_tbl_entries(struct rtl8127_private *tp)
+{
+        return tp->HwSuppIndirTblEntries;
+}
+
+#define RSS_MASK_BITS_OFFSET (8)
+#define RSS_CPU_NUM_OFFSET (16)
+#define RTL8127_UDP_RSS_FLAGS (RTL_8125_RSS_FLAG_HASH_UDP_IPV4 | \
+		       RTL_8125_RSS_FLAG_HASH_UDP_IPV6)
+static int _rtl8127_set_rss_hash_opt(struct rtl8127_private *tp)
+{
+        u32 rss_flags = tp->rss_flags;
+        u32 hash_mask_len;
+        u32 rss_ctrl;
+
+        rss_ctrl = ilog2(rtl8127_tot_rx_rings(tp));
+        rss_ctrl &= (BIT_0 | BIT_1 | BIT_2);
+        rss_ctrl <<= RSS_CPU_NUM_OFFSET;
+
+        /* Perform hash on these packet types */
+        rss_ctrl |= RSS_CTRL_TCP_IPV4_SUPP
+                    | RSS_CTRL_IPV4_SUPP
+                    | RSS_CTRL_IPV6_SUPP
+                    | RSS_CTRL_IPV6_EXT_SUPP
+                    | RSS_CTRL_TCP_IPV6_SUPP
+                    | RSS_CTRL_TCP_IPV6_EXT_SUPP;
+
+        if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4)
+                rss_ctrl |= RSS_CTRL_UDP_IPV4_SUPP;
+
+        if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6)
+                rss_ctrl |= RSS_CTRL_UDP_IPV6_SUPP |
+                            RSS_CTRL_UDP_IPV6_EXT_SUPP;
+
+        hash_mask_len = ilog2(rtl8127_rss_indir_tbl_entries(tp));
+        hash_mask_len &= (BIT_0 | BIT_1 | BIT_2);
+        rss_ctrl |= hash_mask_len << RSS_MASK_BITS_OFFSET;
+
+        RTL_W32(tp, RSS_CTRL_8125, rss_ctrl);
+
+        return 0;
+}
+
+static int rtl8127_set_rss_hash_opt(struct rtl8127_private *tp,
+                                    struct ethtool_rxnfc *nfc)
+{
+        u32 rss_flags = tp->rss_flags;
+
+        /*
+         * RSS does not support anything other than hashing
+         * to queues on src and dst IPs and ports
+         */
+        if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST |
+                          RXH_L4_B_0_1 | RXH_L4_B_2_3))
+                return -EINVAL;
+
+        switch (nfc->flow_type) {
+        case TCP_V4_FLOW:
+        case TCP_V6_FLOW:
+                if (!(nfc->data & RXH_IP_SRC) ||
+                    !(nfc->data & RXH_IP_DST) ||
+                    !(nfc->data & RXH_L4_B_0_1) ||
+                    !(nfc->data & RXH_L4_B_2_3))
+                        return -EINVAL;
+                break;
+        case UDP_V4_FLOW:
+                if (!(nfc->data & RXH_IP_SRC) ||
+                    !(nfc->data & RXH_IP_DST))
+                        return -EINVAL;
+                switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
+                case 0:
+                        rss_flags &= ~RTL_8125_RSS_FLAG_HASH_UDP_IPV4;
+                        break;
+                case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
+                        rss_flags |= RTL_8125_RSS_FLAG_HASH_UDP_IPV4;
+                        break;
+                default:
+                        return -EINVAL;
+                }
+                break;
+        case UDP_V6_FLOW:
+                if (!(nfc->data & RXH_IP_SRC) ||
+                    !(nfc->data & RXH_IP_DST))
+                        return -EINVAL;
+                switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
+                case 0:
+                        rss_flags &= ~RTL_8125_RSS_FLAG_HASH_UDP_IPV6;
+                        break;
+                case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
+                        rss_flags |= RTL_8125_RSS_FLAG_HASH_UDP_IPV6;
+                        break;
+                default:
+                        return -EINVAL;
+                }
+                break;
+        case SCTP_V4_FLOW:
+        case AH_ESP_V4_FLOW:
+        case AH_V4_FLOW:
+        case ESP_V4_FLOW:
+        case SCTP_V6_FLOW:
+        case AH_ESP_V6_FLOW:
+        case AH_V6_FLOW:
+        case ESP_V6_FLOW:
+        case IP_USER_FLOW:
+        case ETHER_FLOW:
+                /* RSS is not supported for these protocols */
+                if (nfc->data) {
+                        netif_err(tp, drv, tp->dev, "Command parameters not supported\n");
+                        return -EINVAL;
+                }
+                return 0;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        /* if we changed something we need to update flags */
+        if (rss_flags != tp->rss_flags) {
+                u32 rss_ctrl = RTL_R32(tp, RSS_CTRL_8125);
+
+                if ((rss_flags & RTL8127_UDP_RSS_FLAGS) &&
+                    !(tp->rss_flags & RTL8127_UDP_RSS_FLAGS))
+                        netdev_warn(tp->dev,
+                                    "enabling UDP RSS: fragmented packets may "
+                                    "arrive out of order to the stack above\n");
+
+                tp->rss_flags = rss_flags;
+
+                /* Perform hash on these packet types */
+                rss_ctrl |= RSS_CTRL_TCP_IPV4_SUPP
+                            | RSS_CTRL_IPV4_SUPP
+                            | RSS_CTRL_IPV6_SUPP
+                            | RSS_CTRL_IPV6_EXT_SUPP
+                            | RSS_CTRL_TCP_IPV6_SUPP
+                            | RSS_CTRL_TCP_IPV6_EXT_SUPP;
+
+                rss_ctrl &= ~(RSS_CTRL_UDP_IPV4_SUPP |
+                              RSS_CTRL_UDP_IPV6_SUPP |
+                              RSS_CTRL_UDP_IPV6_EXT_SUPP);
+
+                if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV4)
+                        rss_ctrl |= RSS_CTRL_UDP_IPV4_SUPP;
+
+                if (rss_flags & RTL_8125_RSS_FLAG_HASH_UDP_IPV6)
+                        rss_ctrl |= RSS_CTRL_UDP_IPV6_SUPP |
+                                    RSS_CTRL_UDP_IPV6_EXT_SUPP;
+
+                RTL_W32(tp, RSS_CTRL_8125, rss_ctrl);
+        }
+
+        return 0;
+}
+
+int rtl8127_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int ret = -EOPNOTSUPP;
+
+        if (!(dev->features & NETIF_F_RXHASH))
+                return ret;
+
+        switch (cmd->cmd) {
+        case ETHTOOL_SRXFH:
+                ret = rtl8127_set_rss_hash_opt(tp, cmd);
+                break;
+        default:
+                break;
+        }
+
+        return ret;
+}
+
+static u32 _rtl8127_get_rxfh_key_size(struct rtl8127_private *tp)
+{
+        return sizeof(tp->rss_key);
+}
+
+u32 rtl8127_get_rxfh_key_size(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (!(dev->features & NETIF_F_RXHASH))
+                return 0;
+
+        return _rtl8127_get_rxfh_key_size(tp);
+}
+
+u32 rtl8127_rss_indir_size(struct net_device *dev)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (!(dev->features & NETIF_F_RXHASH))
+                return 0;
+
+        return rtl8127_rss_indir_tbl_entries(tp);
+}
+
+static void rtl8127_get_reta(struct rtl8127_private *tp, u32 *indir)
+{
+        int i, reta_size = rtl8127_rss_indir_tbl_entries(tp);
+
+        for (i = 0; i < reta_size; i++)
+                indir[i] = tp->rss_indir_tbl[i];
+}
+
+static u32 rtl8127_rss_key_reg(struct rtl8127_private *tp)
+{
+        return RSS_KEY_8125;
+}
+
+static u32 rtl8127_rss_indir_tbl_reg(struct rtl8127_private *tp)
+{
+        return RSS_INDIRECTION_TBL_8125_V2;
+}
+
+static void rtl8127_store_reta(struct rtl8127_private *tp)
+{
+        u16 indir_tbl_reg = rtl8127_rss_indir_tbl_reg(tp);
+        u32 i, reta_entries = rtl8127_rss_indir_tbl_entries(tp);
+        u32 reta = 0;
+        u8 *indir_tbl = tp->rss_indir_tbl;
+
+        /* Write redirection table to HW */
+        for (i = 0; i < reta_entries; i++) {
+                reta |= indir_tbl[i] << (i & 0x3) * 8;
+                if ((i & 3) == 3) {
+                        RTL_W32(tp, indir_tbl_reg, reta);
+
+                        indir_tbl_reg += 4;
+                        reta = 0;
+                }
+        }
+}
+
+static void rtl8127_store_rss_key(struct rtl8127_private *tp)
+{
+        const u16 rss_key_reg = rtl8127_rss_key_reg(tp);
+        u32 i, rss_key_size = _rtl8127_get_rxfh_key_size(tp);
+        u32 *rss_key = (u32*)tp->rss_key;
+
+        /* Write redirection table to HW */
+        for (i = 0; i < rss_key_size; i+=4)
+                RTL_W32(tp, rss_key_reg + i, *rss_key++);
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0)
+int rtl8127_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (!(dev->features & NETIF_F_RXHASH))
+                return -EOPNOTSUPP;
+
+        rxfh->hfunc = ETH_RSS_HASH_TOP;
+
+        if (rxfh->indir)
+                rtl8127_get_reta(tp, rxfh->indir);
+
+        if (rxfh->key)
+                memcpy(rxfh->key, tp->rss_key, RTL8127_RSS_KEY_SIZE);
+
+        return 0;
+}
+
+int rtl8127_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh,
+                     struct netlink_ext_ack *extack)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+        u32 reta_entries = rtl8127_rss_indir_tbl_entries(tp);
+
+        /* We require at least one supported parameter to be changed and no
+         * change in any of the unsupported parameters
+         */
+        if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && rxfh->hfunc != ETH_RSS_HASH_TOP)
+                return -EOPNOTSUPP;
+
+        /* Fill out the redirection table */
+        if (rxfh->indir) {
+                int max_queues = tp->num_rx_rings;
+
+                /* Verify user input. */
+                for (i = 0; i < reta_entries; i++)
+                        if (rxfh->indir[i] >= max_queues)
+                                return -EINVAL;
+
+                for (i = 0; i < reta_entries; i++)
+                        tp->rss_indir_tbl[i] = rxfh->indir[i];
+        }
+
+        /* Fill out the rss hash key */
+        if (rxfh->key)
+                memcpy(tp->rss_key, rxfh->key, RTL8127_RSS_KEY_SIZE);
+
+        rtl8127_store_reta(tp);
+
+        rtl8127_store_rss_key(tp);
+
+        return 0;
+}
+#else
+int rtl8127_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
+                     u8 *hfunc)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+
+        if (!(dev->features & NETIF_F_RXHASH))
+                return -EOPNOTSUPP;
+
+        if (hfunc)
+                *hfunc = ETH_RSS_HASH_TOP;
+
+        if (indir)
+                rtl8127_get_reta(tp, indir);
+
+        if (key)
+                memcpy(key, tp->rss_key, RTL8127_RSS_KEY_SIZE);
+
+        return 0;
+}
+
+int rtl8127_set_rxfh(struct net_device *dev, const u32 *indir,
+                     const u8 *key, const u8 hfunc)
+{
+        struct rtl8127_private *tp = netdev_priv(dev);
+        int i;
+        u32 reta_entries = rtl8127_rss_indir_tbl_entries(tp);
+
+        /* We require at least one supported parameter to be changed and no
+         * change in any of the unsupported parameters
+         */
+        if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
+                return -EOPNOTSUPP;
+
+        /* Fill out the redirection table */
+        if (indir) {
+                int max_queues = tp->num_rx_rings;
+
+                /* Verify user input. */
+                for (i = 0; i < reta_entries; i++)
+                        if (indir[i] >= max_queues)
+                                return -EINVAL;
+
+                for (i = 0; i < reta_entries; i++)
+                        tp->rss_indir_tbl[i] = indir[i];
+        }
+
+        /* Fill out the rss hash key */
+        if (key)
+                memcpy(tp->rss_key, key, RTL8127_RSS_KEY_SIZE);
+
+        rtl8127_store_reta(tp);
+
+        rtl8127_store_rss_key(tp);
+
+        return 0;
+}
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) */
+
+static u32 rtl8127_get_rx_desc_hash(struct rtl8127_private *tp,
+                                    struct RxDesc *desc)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                return le32_to_cpu(((struct RxDescV3 *)desc)->RxDescNormalDDWord2.RSSResult);
+        case RX_DESC_RING_TYPE_4:
+                return le32_to_cpu(((struct RxDescV4 *)desc)->RxDescNormalDDWord1.RSSResult);
+        default:
+                return 0;
+        }
+}
+
+#define RXS_8125B_RSS_UDP BIT(9)
+#define RXS_8125_RSS_IPV4 BIT(10)
+#define RXS_8125_RSS_IPV6 BIT(12)
+#define RXS_8125_RSS_TCP BIT(13)
+#define RTL8127_RXS_RSS_L3_TYPE_MASK (RXS_8125_RSS_IPV4 | RXS_8125_RSS_IPV6)
+#define RTL8127_RXS_RSS_L4_TYPE_MASK (RXS_8125_RSS_TCP | RXS_8125B_RSS_UDP)
+
+#define RXS_8125B_RSS_UDP_V4 BIT(27)
+#define RXS_8125_RSS_IPV4_V4 BIT(28)
+#define RXS_8125_RSS_IPV6_V4 BIT(29)
+#define RXS_8125_RSS_TCP_V4 BIT(30)
+#define RTL8127_RXS_RSS_L3_TYPE_MASK_V4 (RXS_8125_RSS_IPV4_V4 | RXS_8125_RSS_IPV6_V4)
+#define RTL8127_RXS_RSS_L4_TYPE_MASK_V4 (RXS_8125_RSS_TCP_V4 | RXS_8125B_RSS_UDP_V4)
+static void rtl8127_rx_hash_v3(struct rtl8127_private *tp,
+                               struct RxDescV3 *descv3,
+                               struct sk_buff *skb)
+{
+        u16 rss_header_info;
+
+        if (!(tp->dev->features & NETIF_F_RXHASH))
+                return;
+
+        rss_header_info = le16_to_cpu(descv3->RxDescNormalDDWord2.HeaderInfo);
+
+        if (!(rss_header_info & RTL8127_RXS_RSS_L3_TYPE_MASK))
+                return;
+
+        skb_set_hash(skb, rtl8127_get_rx_desc_hash(tp, (struct RxDesc *)descv3),
+                     (RTL8127_RXS_RSS_L4_TYPE_MASK & rss_header_info) ?
+                     PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
+}
+
+static void rtl8127_rx_hash_v4(struct rtl8127_private *tp,
+                               struct RxDescV4 *descv4,
+                               struct sk_buff *skb)
+{
+        u32 rss_header_info;
+
+        if (!(tp->dev->features & NETIF_F_RXHASH))
+                return;
+
+        rss_header_info = le32_to_cpu(descv4->RxDescNormalDDWord1.RSSInfo);
+
+        if (!(rss_header_info & RTL8127_RXS_RSS_L3_TYPE_MASK_V4))
+                return;
+
+        skb_set_hash(skb, rtl8127_get_rx_desc_hash(tp, (struct RxDesc *)descv4),
+                     (RTL8127_RXS_RSS_L4_TYPE_MASK_V4 & rss_header_info) ?
+                     PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
+}
+
+void rtl8127_rx_hash(struct rtl8127_private *tp,
+                     struct RxDesc *desc,
+                     struct sk_buff *skb)
+{
+        switch (tp->InitRxDescType) {
+        case RX_DESC_RING_TYPE_3:
+                rtl8127_rx_hash_v3(tp, (struct RxDescV3 *)desc, skb);
+                break;
+        case RX_DESC_RING_TYPE_4:
+                rtl8127_rx_hash_v4(tp, (struct RxDescV4 *)desc, skb);
+                break;
+        default:
+                return;
+        }
+}
+
+void rtl8127_disable_rss(struct rtl8127_private *tp)
+{
+        RTL_W32(tp, RSS_CTRL_8125, 0x00);
+}
+
+void _rtl8127_config_rss(struct rtl8127_private *tp)
+{
+        _rtl8127_set_rss_hash_opt(tp);
+
+        rtl8127_store_reta(tp);
+
+        rtl8127_store_rss_key(tp);
+}
+
+void rtl8127_config_rss(struct rtl8127_private *tp)
+{
+        if (!tp->EnableRss) {
+                rtl8127_disable_rss(tp);
+                return;
+        }
+
+        _rtl8127_config_rss(tp);
+}
+
+void rtl8127_init_rss(struct rtl8127_private *tp)
+{
+        int i;
+
+        for (i = 0; i < rtl8127_rss_indir_tbl_entries(tp); i++)
+                tp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, tp->num_rx_rings);
+
+        netdev_rss_key_fill(tp->rss_key, RTL8127_RSS_KEY_SIZE);
+}
diff --git a/drivers/net/ethernet/realtek/r8127/r8127_rss.h b/drivers/net/ethernet/realtek/r8127/r8127_rss.h
new file mode 100644
index 000000000000..8e92bb830c96
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/r8127_rss.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_R8127_RSS_H
+#define _LINUX_R8127_RSS_H
+
+#include <linux/netdevice.h>
+#include <linux/types.h>
+
+#define RTL8127_RSS_KEY_SIZE     40  /* size of RSS Hash Key in bytes */
+#define RTL8127_MAX_INDIRECTION_TABLE_ENTRIES 128
+
+enum rtl8127_rss_flag {
+        RTL_8125_RSS_FLAG_HASH_UDP_IPV4  = (1 << 0),
+        RTL_8125_RSS_FLAG_HASH_UDP_IPV6  = (1 << 1),
+};
+
+struct rtl8127_private;
+struct RxDesc;
+
+int rtl8127_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
+                      u32 *rule_locs);
+int rtl8127_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd);
+u32 rtl8127_get_rxfh_key_size(struct net_device *netdev);
+u32 rtl8127_rss_indir_size(struct net_device *netdev);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0)
+int rtl8127_get_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh);
+int rtl8127_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh,
+                     struct netlink_ext_ack *extack);
+#else
+int rtl8127_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key,
+                     u8 *hfunc);
+int rtl8127_set_rxfh(struct net_device *netdev, const u32 *indir,
+                     const u8 *key, const u8 hfunc);
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0) */
+void rtl8127_rx_hash(struct rtl8127_private *tp,
+                     struct RxDesc *desc,
+                     struct sk_buff *skb);
+void _rtl8127_config_rss(struct rtl8127_private *tp);
+void rtl8127_config_rss(struct rtl8127_private *tp);
+void rtl8127_init_rss(struct rtl8127_private *tp);
+u32 rtl8127_rss_indir_tbl_entries(struct rtl8127_private *tp);
+void rtl8127_disable_rss(struct rtl8127_private *tp);
+
+#endif /* _LINUX_R8127_RSS_H */
diff --git a/drivers/net/ethernet/realtek/r8127/rtl_eeprom.c b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.c
new file mode 100644
index 000000000000..094b31c10bfb
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.c
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/ethtool.h>
+#include <linux/netdevice.h>
+#include <linux/delay.h>
+
+#include <asm/io.h>
+
+#include "r8127.h"
+#include "rtl_eeprom.h"
+
+//-------------------------------------------------------------------
+//rtl8127_eeprom_type():
+//  tell the eeprom type
+//return value:
+//  0: the eeprom type is 93C46
+//  1: the eeprom type is 93C56 or 93C66
+//-------------------------------------------------------------------
+void rtl8127_eeprom_type(struct rtl8127_private *tp)
+{
+        u16 magic = 0;
+
+        if (tp->mcfg == CFG_METHOD_DEFAULT)
+                goto out_no_eeprom;
+
+        if(RTL_R8(tp, 0xD2)&0x04) {
+                //not support
+                //tp->eeprom_type = EEPROM_TWSI;
+                //tp->eeprom_len = 256;
+                goto out_no_eeprom;
+        } else if(RTL_R32(tp, RxConfig) & RxCfg_9356SEL) {
+                tp->eeprom_type = EEPROM_TYPE_93C56;
+                tp->eeprom_len = 256;
+        } else {
+                tp->eeprom_type = EEPROM_TYPE_93C46;
+                tp->eeprom_len = 128;
+        }
+
+        magic = rtl8127_eeprom_read_sc(tp, 0);
+
+out_no_eeprom:
+        if ((magic != 0x8129) && (magic != 0x8128)) {
+                tp->eeprom_type = EEPROM_TYPE_NONE;
+                tp->eeprom_len = 0;
+        }
+}
+
+void rtl8127_eeprom_cleanup(struct rtl8127_private *tp)
+{
+        u8 x;
+
+        x = RTL_R8(tp, Cfg9346);
+        x &= ~(Cfg9346_EEDI | Cfg9346_EECS);
+
+        RTL_W8(tp, Cfg9346, x);
+
+        rtl8127_raise_clock(tp, &x);
+        rtl8127_lower_clock(tp, &x);
+}
+
+static int rtl8127_eeprom_cmd_done(struct rtl8127_private *tp)
+{
+        u8 x;
+        int i;
+
+        rtl8127_stand_by(tp);
+
+        for (i = 0; i < 50000; i++) {
+                x = RTL_R8(tp, Cfg9346);
+
+                if (x & Cfg9346_EEDO) {
+                        udelay(RTL_CLOCK_RATE * 2 * 3);
+                        return 0;
+                }
+                udelay(1);
+        }
+
+        return -1;
+}
+
+//-------------------------------------------------------------------
+//rtl8127_eeprom_read_sc():
+//  read one word from eeprom
+//-------------------------------------------------------------------
+u16 rtl8127_eeprom_read_sc(struct rtl8127_private *tp, u16 reg)
+{
+        int addr_sz = 6;
+        u8 x;
+        u16 data;
+
+        if(tp->eeprom_type == EEPROM_TYPE_NONE)
+                return -1;
+
+        if (tp->eeprom_type==EEPROM_TYPE_93C46)
+                addr_sz = 6;
+        else if (tp->eeprom_type==EEPROM_TYPE_93C56)
+                addr_sz = 8;
+
+        x = Cfg9346_EEM1 | Cfg9346_EECS;
+        RTL_W8(tp, Cfg9346, x);
+
+        rtl8127_shift_out_bits(tp, RTL_EEPROM_READ_OPCODE, 3);
+        rtl8127_shift_out_bits(tp, reg, addr_sz);
+
+        data = rtl8127_shift_in_bits(tp);
+
+        rtl8127_eeprom_cleanup(tp);
+
+        RTL_W8(tp, Cfg9346, 0);
+
+        return data;
+}
+
+//-------------------------------------------------------------------
+//rtl8127_eeprom_write_sc():
+//  write one word to a specific address in the eeprom
+//-------------------------------------------------------------------
+void rtl8127_eeprom_write_sc(struct rtl8127_private *tp, u16 reg, u16 data)
+{
+        u8 x;
+        int addr_sz = 6;
+        int w_dummy_addr = 4;
+
+        if(tp->eeprom_type == EEPROM_TYPE_NONE)
+                return;
+
+        if (tp->eeprom_type==EEPROM_TYPE_93C46) {
+                addr_sz = 6;
+                w_dummy_addr = 4;
+        } else if (tp->eeprom_type==EEPROM_TYPE_93C56) {
+                addr_sz = 8;
+                w_dummy_addr = 6;
+        }
+
+        x = Cfg9346_EEM1 | Cfg9346_EECS;
+        RTL_W8(tp, Cfg9346, x);
+
+        rtl8127_shift_out_bits(tp, RTL_EEPROM_EWEN_OPCODE, 5);
+        rtl8127_shift_out_bits(tp, reg, w_dummy_addr);
+        rtl8127_stand_by(tp);
+
+        rtl8127_shift_out_bits(tp, RTL_EEPROM_ERASE_OPCODE, 3);
+        rtl8127_shift_out_bits(tp, reg, addr_sz);
+        if (rtl8127_eeprom_cmd_done(tp) < 0)
+                return;
+        rtl8127_stand_by(tp);
+
+        rtl8127_shift_out_bits(tp, RTL_EEPROM_WRITE_OPCODE, 3);
+        rtl8127_shift_out_bits(tp, reg, addr_sz);
+        rtl8127_shift_out_bits(tp, data, 16);
+        if (rtl8127_eeprom_cmd_done(tp) < 0)
+                return;
+        rtl8127_stand_by(tp);
+
+        rtl8127_shift_out_bits(tp, RTL_EEPROM_EWDS_OPCODE, 5);
+        rtl8127_shift_out_bits(tp, reg, w_dummy_addr);
+
+        rtl8127_eeprom_cleanup(tp);
+        RTL_W8(tp, Cfg9346, 0);
+}
+
+void rtl8127_raise_clock(struct rtl8127_private *tp, u8 *x)
+{
+        *x = *x | Cfg9346_EESK;
+        RTL_W8(tp, Cfg9346, *x);
+        udelay(RTL_CLOCK_RATE);
+}
+
+void rtl8127_lower_clock(struct rtl8127_private *tp, u8 *x)
+{
+
+        *x = *x & ~Cfg9346_EESK;
+        RTL_W8(tp, Cfg9346, *x);
+        udelay(RTL_CLOCK_RATE);
+}
+
+void rtl8127_shift_out_bits(struct rtl8127_private *tp, int data, int count)
+{
+        u8 x;
+        int  mask;
+
+        mask = 0x01 << (count - 1);
+        x = RTL_R8(tp, Cfg9346);
+        x &= ~(Cfg9346_EEDI | Cfg9346_EEDO);
+
+        do {
+                if (data & mask)
+                        x |= Cfg9346_EEDI;
+                else
+                        x &= ~Cfg9346_EEDI;
+
+                RTL_W8(tp, Cfg9346, x);
+                udelay(RTL_CLOCK_RATE);
+                rtl8127_raise_clock(tp, &x);
+                rtl8127_lower_clock(tp, &x);
+                mask = mask >> 1;
+        } while(mask);
+
+        x &= ~Cfg9346_EEDI;
+        RTL_W8(tp, Cfg9346, x);
+}
+
+u16 rtl8127_shift_in_bits(struct rtl8127_private *tp)
+{
+        u8 x;
+        u16 d, i;
+
+        x = RTL_R8(tp, Cfg9346);
+        x &= ~(Cfg9346_EEDI | Cfg9346_EEDO);
+
+        d = 0;
+
+        for (i = 0; i < 16; i++) {
+                d = d << 1;
+                rtl8127_raise_clock(tp, &x);
+
+                x = RTL_R8(tp, Cfg9346);
+                x &= ~Cfg9346_EEDI;
+
+                if (x & Cfg9346_EEDO)
+                        d |= 1;
+
+                rtl8127_lower_clock(tp, &x);
+        }
+
+        return d;
+}
+
+void rtl8127_stand_by(struct rtl8127_private *tp)
+{
+        u8 x;
+
+        x = RTL_R8(tp, Cfg9346);
+        x &= ~(Cfg9346_EECS | Cfg9346_EESK);
+        RTL_W8(tp, Cfg9346, x);
+        udelay(RTL_CLOCK_RATE);
+
+        x |= Cfg9346_EECS;
+        RTL_W8(tp, Cfg9346, x);
+}
+
+void rtl8127_set_eeprom_sel_low(struct rtl8127_private *tp)
+{
+        RTL_W8(tp, Cfg9346, Cfg9346_EEM1);
+        RTL_W8(tp, Cfg9346, Cfg9346_EEM1 | Cfg9346_EESK);
+
+        udelay(20);
+
+        RTL_W8(tp, Cfg9346, Cfg9346_EEM1);
+}
diff --git a/drivers/net/ethernet/realtek/r8127/rtl_eeprom.h b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.h
new file mode 100644
index 000000000000..e4d8c6c3f676
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/rtl_eeprom.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_RTLEEPROM_H
+#define _LINUX_RTLEEPROM_H
+
+//EEPROM opcodes
+#define RTL_EEPROM_READ_OPCODE      06
+#define RTL_EEPROM_WRITE_OPCODE     05
+#define RTL_EEPROM_ERASE_OPCODE     07
+#define RTL_EEPROM_EWEN_OPCODE      19
+#define RTL_EEPROM_EWDS_OPCODE      16
+
+#define RTL_CLOCK_RATE  3
+
+void rtl8127_eeprom_type(struct rtl8127_private *tp);
+void rtl8127_eeprom_cleanup(struct rtl8127_private *tp);
+u16 rtl8127_eeprom_read_sc(struct rtl8127_private *tp, u16 reg);
+void rtl8127_eeprom_write_sc(struct rtl8127_private *tp, u16 reg, u16 data);
+void rtl8127_shift_out_bits(struct rtl8127_private *tp, int data, int count);
+u16 rtl8127_shift_in_bits(struct rtl8127_private *tp);
+void rtl8127_raise_clock(struct rtl8127_private *tp, u8 *x);
+void rtl8127_lower_clock(struct rtl8127_private *tp, u8 *x);
+void rtl8127_stand_by(struct rtl8127_private *tp);
+void rtl8127_set_eeprom_sel_low(struct rtl8127_private *tp);
+
+#endif /* _LINUX_RTLEEPROM_H */
diff --git a/drivers/net/ethernet/realtek/r8127/rtltool.c b/drivers/net/ethernet/realtek/r8127/rtltool.c
new file mode 100644
index 000000000000..a37495bbb8ab
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/rtltool.c
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/delay.h>
+#include <linux/in.h>
+#include <linux/ethtool.h>
+#include <asm/uaccess.h>
+#include "r8127.h"
+#include "r8127_fiber.h"
+#include "rtl_eeprom.h"
+#include "rtltool.h"
+
+int rtl8127_tool_ioctl(struct rtl8127_private *tp, struct ifreq *ifr)
+{
+        struct rtltool_sds_cmd *sds_cmd;
+        struct rtltool_cmd my_cmd;
+        unsigned long flags;
+        int ret;
+
+        if (copy_from_user(&my_cmd, ifr->ifr_data, sizeof(my_cmd)))
+                return -EFAULT;
+
+        sds_cmd = (struct rtltool_sds_cmd*)&my_cmd;
+        ret = 0;
+        switch (my_cmd.cmd) {
+        case RTLTOOL_READ_MAC:
+                if ((my_cmd.offset + my_cmd.len) > pci_resource_len(tp->pci_dev, 2)) {
+                        ret = -EINVAL;
+                        break;
+                }
+
+                if (my_cmd.len==1)
+                        my_cmd.data = readb(tp->mmio_addr+my_cmd.offset);
+                else if (my_cmd.len==2)
+                        my_cmd.data = readw(tp->mmio_addr+(my_cmd.offset&~1));
+                else if (my_cmd.len==4)
+                        my_cmd.data = readl(tp->mmio_addr+(my_cmd.offset&~3));
+                else {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+                break;
+
+        case RTLTOOL_WRITE_MAC:
+                if ((my_cmd.offset + my_cmd.len) > pci_resource_len(tp->pci_dev, 2)) {
+                        ret = -EINVAL;
+                        break;
+                }
+
+                if (my_cmd.len==1)
+                        writeb(my_cmd.data, tp->mmio_addr+my_cmd.offset);
+                else if (my_cmd.len==2)
+                        writew(my_cmd.data, tp->mmio_addr+(my_cmd.offset&~1));
+                else if (my_cmd.len==4)
+                        writel(my_cmd.data, tp->mmio_addr+(my_cmd.offset&~3));
+                else {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+
+                break;
+
+        case RTLTOOL_READ_PHY:
+                r8127_spin_lock(&tp->phy_lock, flags);
+                my_cmd.data = rtl8127_mdio_prot_read(tp, my_cmd.offset);
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+
+                break;
+
+        case RTLTOOL_WRITE_PHY:
+                r8127_spin_lock(&tp->phy_lock, flags);
+                rtl8127_mdio_prot_write(tp, my_cmd.offset, my_cmd.data);
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                break;
+
+        case RTLTOOL_READ_EPHY:
+                my_cmd.data = rtl8127_ephy_read(tp, my_cmd.offset);
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+
+                break;
+
+        case RTLTOOL_WRITE_EPHY:
+                rtl8127_ephy_write(tp, my_cmd.offset, my_cmd.data);
+                break;
+
+        case RTLTOOL_READ_ERI:
+                my_cmd.data = 0;
+                if (my_cmd.len==1 || my_cmd.len==2 || my_cmd.len==4) {
+                        my_cmd.data = rtl8127_eri_read(tp, my_cmd.offset, my_cmd.len, ERIAR_ExGMAC);
+                } else {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+
+                break;
+
+        case RTLTOOL_WRITE_ERI:
+                if (my_cmd.len==1 || my_cmd.len==2 || my_cmd.len==4) {
+                        rtl8127_eri_write(tp, my_cmd.offset, my_cmd.len, my_cmd.data, ERIAR_ExGMAC);
+                } else {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+                break;
+
+        case RTLTOOL_READ_PCI:
+                my_cmd.data = 0;
+                if (my_cmd.len==1)
+                        pci_read_config_byte(tp->pci_dev, my_cmd.offset,
+                                             (u8 *)&my_cmd.data);
+                else if (my_cmd.len==2)
+                        pci_read_config_word(tp->pci_dev, my_cmd.offset,
+                                             (u16 *)&my_cmd.data);
+                else if (my_cmd.len==4)
+                        pci_read_config_dword(tp->pci_dev, my_cmd.offset,
+                                              &my_cmd.data);
+                else {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+                break;
+
+        case RTLTOOL_WRITE_PCI:
+                if (my_cmd.len==1)
+                        pci_write_config_byte(tp->pci_dev, my_cmd.offset,
+                                              my_cmd.data);
+                else if (my_cmd.len==2)
+                        pci_write_config_word(tp->pci_dev, my_cmd.offset,
+                                              my_cmd.data);
+                else if (my_cmd.len==4)
+                        pci_write_config_dword(tp->pci_dev, my_cmd.offset,
+                                               my_cmd.data);
+                else {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+
+                break;
+
+        case RTLTOOL_READ_EEPROM:
+                my_cmd.data = rtl8127_eeprom_read_sc(tp, my_cmd.offset);
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+
+                break;
+
+        case RTLTOOL_WRITE_EEPROM:
+                rtl8127_eeprom_write_sc(tp, my_cmd.offset, my_cmd.data);
+                break;
+
+        case RTL_READ_OOB_MAC:
+                rtl8127_oob_mutex_lock(tp);
+                my_cmd.data = rtl8127_ocp_read(tp, my_cmd.offset, 4);
+                rtl8127_oob_mutex_unlock(tp);
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+                break;
+
+        case RTL_WRITE_OOB_MAC:
+                if (my_cmd.len == 0 || my_cmd.len > 4)
+                        return -EOPNOTSUPP;
+
+                rtl8127_oob_mutex_lock(tp);
+                rtl8127_ocp_write(tp, my_cmd.offset, my_cmd.len, my_cmd.data);
+                rtl8127_oob_mutex_unlock(tp);
+                break;
+
+        case RTL_ENABLE_PCI_DIAG:
+                r8127_spin_lock(&tp->phy_lock, flags);
+                tp->rtk_enable_diag = 1;
+                r8127_spin_unlock(&tp->phy_lock, flags);
+
+                dprintk("enable rtk diag\n");
+                break;
+
+        case RTL_DISABLE_PCI_DIAG:
+                r8127_spin_lock(&tp->phy_lock, flags);
+                tp->rtk_enable_diag = 0;
+                r8127_spin_unlock(&tp->phy_lock, flags);
+
+                dprintk("disable rtk diag\n");
+                break;
+
+        case RTL_READ_MAC_OCP:
+                if (my_cmd.offset % 2)
+                        return -EOPNOTSUPP;
+
+                my_cmd.data = rtl8127_mac_ocp_read(tp, my_cmd.offset);
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+                break;
+
+        case RTL_WRITE_MAC_OCP:
+                if ((my_cmd.offset % 2) || (my_cmd.len != 2))
+                        return -EOPNOTSUPP;
+
+                rtl8127_mac_ocp_write(tp, my_cmd.offset, (u16)my_cmd.data);
+                break;
+
+        case RTL_DIRECT_READ_PHY_OCP:
+                r8127_spin_lock(&tp->phy_lock, flags);
+                my_cmd.data = rtl8127_mdio_prot_direct_read_phy_ocp(tp, my_cmd.offset);
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+
+                break;
+
+        case RTL_DIRECT_WRITE_PHY_OCP:
+                r8127_spin_lock(&tp->phy_lock, flags);
+                rtl8127_mdio_prot_direct_write_phy_ocp(tp, my_cmd.offset, my_cmd.data);
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                break;
+
+        case RTL_READ_SDS_PHY:
+                if (!HW_FIBER_MODE_ENABLED(tp)) {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+
+                r8127_spin_lock(&tp->phy_lock, flags);
+                sds_cmd->data = rtl8127_sds_phy_read_8127(tp, sds_cmd->index,
+                                sds_cmd->page, sds_cmd->addr);
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                if (copy_to_user(ifr->ifr_data, &my_cmd, sizeof(my_cmd))) {
+                        ret = -EFAULT;
+                        break;
+                }
+
+                break;
+
+        case RTL_WRITE_SDS_PHY:
+                if (!HW_FIBER_MODE_ENABLED(tp)) {
+                        ret = -EOPNOTSUPP;
+                        break;
+                }
+
+                r8127_spin_lock(&tp->phy_lock, flags);
+                rtl8127_sds_phy_write_8127(tp, sds_cmd->index, sds_cmd->page,
+                                           sds_cmd->addr, sds_cmd->data);
+                r8127_spin_unlock(&tp->phy_lock, flags);
+                break;
+
+        default:
+                ret = -EOPNOTSUPP;
+                break;
+        }
+
+        return ret;
+}
diff --git a/drivers/net/ethernet/realtek/r8127/rtltool.h b/drivers/net/ethernet/realtek/r8127/rtltool.h
new file mode 100644
index 000000000000..42e64054ab85
--- /dev/null
+++ b/drivers/net/ethernet/realtek/r8127/rtltool.h
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+################################################################################
+#
+# r8127 is the Linux device driver released for Realtek 10 Gigabit Ethernet
+# controllers with PCI-Express interface.
+#
+# Copyright(c) 2025 Realtek Semiconductor Corp. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 2 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+# Realtek NIC software team <nicfae@realtek.com>
+# No. 2, Innovation Road II, Hsinchu Science Park, Hsinchu 300, Taiwan
+#
+################################################################################
+*/
+
+/************************************************************************************
+ *  This product is covered by one or more of the following patents:
+ *  US6,570,884, US6,115,776, and US6,327,625.
+ ***********************************************************************************/
+
+#ifndef _LINUX_RTLTOOL_H
+#define _LINUX_RTLTOOL_H
+
+#define SIOCRTLTOOL		SIOCDEVPRIVATE+1
+
+enum rtl_cmd {
+        RTLTOOL_READ_MAC=0,
+        RTLTOOL_WRITE_MAC,
+        RTLTOOL_READ_PHY,
+        RTLTOOL_WRITE_PHY,
+        RTLTOOL_READ_EPHY,
+        RTLTOOL_WRITE_EPHY,
+        RTLTOOL_READ_ERI,
+        RTLTOOL_WRITE_ERI,
+        RTLTOOL_READ_PCI,
+        RTLTOOL_WRITE_PCI,
+        RTLTOOL_READ_EEPROM,
+        RTLTOOL_WRITE_EEPROM,
+
+        RTL_READ_OOB_MAC,
+        RTL_WRITE_OOB_MAC,
+
+        RTL_ENABLE_PCI_DIAG,
+        RTL_DISABLE_PCI_DIAG,
+
+        RTL_READ_MAC_OCP,
+        RTL_WRITE_MAC_OCP,
+
+        RTL_DIRECT_READ_PHY_OCP,
+        RTL_DIRECT_WRITE_PHY_OCP,
+
+        RTL_READ_FIBER_PHY,
+        RTL_WRITE_FIBER_PHY,
+
+        RTL_READ_SDS_PHY,
+        RTL_WRITE_SDS_PHY,
+
+        RTLTOOL_INVALID
+};
+
+struct rtltool_cmd {
+        __u32	cmd;
+        __u32	offset;
+        __u32	len;
+        __u32	data;
+};
+
+struct rtltool_sds_cmd {
+        __u32	cmd;
+        __u16	index;
+        __u16	page;
+        __u16	addr;
+        __u16	len;
+        __u32	data;
+};
+
+enum mode_access {
+        MODE_NONE=0,
+        MODE_READ,
+        MODE_WRITE
+};
+
+#ifdef __KERNEL__
+int rtl8127_tool_ioctl(struct rtl8127_private *tp, struct ifreq *ifr);
+#endif
+
+#endif /* _LINUX_RTLTOOL_H */
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index d407b031cc2a..92071a8f83d8 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -224,7 +224,7 @@ static const struct pci_device_id rtl8169_pci_tbl[] = {
 	{ 0x0001, 0x8168, PCI_ANY_ID, 0x2410 },
 	{ PCI_VDEVICE(REALTEK,	0x8125) },
 	{ PCI_VDEVICE(REALTEK,	0x8126) },
-	{ PCI_VDEVICE(REALTEK,	0x8127) },
+//	{ PCI_VDEVICE(REALTEK,	0x8127) },
 	{ PCI_VDEVICE(REALTEK,	0x3000) },
 	{ PCI_VDEVICE(REALTEK,	0x5000) },
 	{ PCI_VDEVICE(REALTEK,	0x0e10) },
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index 9507131875b2..c8c3681d0df0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -206,6 +206,15 @@ config DWMAC_SOPHGO
 	  for the stmmac device driver. This driver is used for the
 	  ethernet controllers on various Sophgo SoCs.
 
+config DWMAC_SPACEMIT_ETHQOS
+	tristate "Spacemit ETHQOS support"
+	depends on OF && (SOC_SPACEMIT || COMPILE_TEST)
+	help
+	  Support for the Spacemit ETHQOS core.
+
+	  This selects the Spacemit ETHQOS glue layer support for the
+	  stmmac device driver.
+
 config DWMAC_STARFIVE
 	tristate "StarFive dwmac support"
 	depends on OF && (ARCH_STARFIVE || COMPILE_TEST)
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index 51e068e26ce4..a4539aa7e047 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_DWMAC_RZN1)	+= dwmac-rzn1.o
 obj-$(CONFIG_DWMAC_S32)		+= dwmac-s32.o
 obj-$(CONFIG_DWMAC_SOCFPGA)	+= dwmac-altr-socfpga.o
 obj-$(CONFIG_DWMAC_SOPHGO)	+= dwmac-sophgo.o
+obj-$(CONFIG_DWMAC_SPACEMIT_ETHQOS) += dwmac-spacemit-ethqos.o
 obj-$(CONFIG_DWMAC_STARFIVE)	+= dwmac-starfive.o
 obj-$(CONFIG_DWMAC_STI)		+= dwmac-sti.o
 obj-$(CONFIG_DWMAC_STM32)	+= dwmac-stm32.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit-ethqos.c
new file mode 100644
index 000000000000..d145cc94fd95
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit-ethqos.c
@@ -0,0 +1,704 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Spacemit Ethernet QoS glue driver
+ *
+ * Copyright (c) 2025, Spacemit Corporation.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/platform_device.h>
+#include <linux/phy.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/regmap.h>
+#include <linux/bitfield.h>
+#include <linux/mfd/syscon.h>
+#ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#endif
+#include "stmmac.h"
+#include "stmmac_platform.h"
+
+#define DRIVER_NAME			"dwmac-spacemit-ethqos"
+
+#define TUNING_CMD_LEN			50
+#define CLK_PHASE_CNT			256
+#define CLK_PHASE_REVERT		180
+
+#define TXCLK_PHASE_DEFAULT		0
+#define RXCLK_PHASE_DEFAULT		0
+
+struct spacemit_ethqos {
+	struct platform_device *pdev;
+	struct plat_stmmacenet_data *plat;
+	struct regmap *apmu;
+	u32 ctrl_off;
+	u32 dline_off;
+	phy_interface_t phy_iface;
+	u8 tx_clk_phase;
+	u8 rx_clk_phase;
+	u8 clk_tuning_way;
+	struct clk *tx_clk;
+	struct clk *phy_clk;
+	int speed;
+	bool clk_tuning_enable;
+	bool tx_clk_from_soc;
+	bool phy_clk_from_soc;
+	bool wol_irq_enable;
+#ifdef CONFIG_DEBUG_FS
+	struct dentry *dbg_dir;
+	struct dentry *dbg_clk_tuning;
+#endif
+};
+
+/**
+ * struct spacemit_ethqos_ops - Spacemit platform glue ops
+ * @devm_glue_parse_dt:        Parse platform data from DT (devm-managed)
+ * @devm_glue_bind_ops:        Install platform callbacks (devm-rollback)
+ * @devm_glue_config_plat:     Program platform registers (devm-managed)
+ */
+struct spacemit_ethqos_ops {
+	int (*devm_glue_parse_dt)(struct platform_device *pdev,
+				  struct spacemit_ethqos *eqos);
+	int (*devm_glue_bind_ops)(struct spacemit_ethqos *eqos);
+	int (*devm_glue_config_plat)(struct spacemit_ethqos *eqos);
+};
+
+static int devm_spacemit_glue_init(struct platform_device *pdev,
+				   struct plat_stmmacenet_data *plat_dat,
+				   const struct spacemit_ethqos_ops *ops)
+{
+	struct device *dev = &pdev->dev;
+	struct spacemit_ethqos *eqos;
+	int ret;
+
+	eqos = devm_kzalloc(dev, sizeof(*eqos), GFP_KERNEL);
+	if (!eqos)
+		return -ENOMEM;
+
+	eqos->pdev = pdev;
+	eqos->plat = plat_dat;
+	plat_dat->bsp_priv = eqos;
+
+	if (ops->devm_glue_parse_dt) {
+		ret = ops->devm_glue_parse_dt(pdev, eqos);
+		if (ret)
+			return dev_err_probe(dev, ret, "glue layer: dt parse failed\n");
+	}
+
+	if (ops->devm_glue_bind_ops) {
+		ret = ops->devm_glue_bind_ops(eqos);
+		if (ret)
+			return dev_err_probe(dev, ret, "glue layer: bind ops failed\n");
+	}
+
+	if (ops->devm_glue_config_plat) {
+		ret = ops->devm_glue_config_plat(eqos);
+		if (ret)
+			return dev_err_probe(dev, ret, "glue layer: config failed\n");
+	}
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * K3 SoC-specific macros
+ * -----------------------------------------------------------------------------
+ */
+#define TX_PHASE			1
+#define RX_PHASE			0
+
+#define PHY_INTF_MODE_MASK		GENMASK(4, 3)
+
+#define PHY_INTF_RMII			FIELD_PREP(PHY_INTF_MODE_MASK, 0x0)
+#define PHY_INTF_RGMII			FIELD_PREP(PHY_INTF_MODE_MASK, 0x1)
+#define PHY_INTF_MII			FIELD_PREP(PHY_INTF_MODE_MASK, 0x3)
+
+/* only valid for rmii, invert tx clk */
+#define RMII_TX_CLK_SEL			BIT(6)
+/* only valid for rmii, invert rx clk */
+#define RMII_RX_CLK_SEL			BIT(7)
+
+#define LPI_IRQ_EN			BIT(9)
+#define WAKE_IRQ_EN			BIT(12)
+#define AXI_SINGLE_ID			BIT(13)
+
+/* dline register bits */
+#define EMAC_RX_DLINE_EN		BIT(0)
+#define EMAC_TX_DLINE_EN		BIT(16)
+
+#define RMII_TX_PHASE_MASK		GENMASK(18, 16)
+#define RMII_RX_PHASE_MASK		GENMASK(22, 20)
+
+#define RGMII_RX_PHASE_MASK		GENMASK(22, 20)
+#define RGMII_TX_PHASE_MASK		GENMASK(26, 24)
+
+#define EMAC_RX_DLINE_CODE_MASK		GENMASK(15, 8)
+#define EMAC_TX_DLINE_CODE_MASK		GENMASK(31, 24)
+
+enum clk_tuning_way {
+	/* fpga clk tuning register */
+	CLK_TUNING_BY_REG,
+	/* zebu/evb rgmii delayline register */
+	CLK_TUNING_BY_DLINE,
+	/* evb rmii only revert tx/rx clock for clk tuning */
+	CLK_TUNING_BY_CLK_REVERT,
+	CLK_TUNING_MAX,
+};
+
+static int clk_phase_rmii_set(struct spacemit_ethqos *eqos, bool is_tx)
+{
+	struct device *dev = &eqos->pdev->dev;
+	u32 mask, val, phase;
+	int ret;
+
+	switch (eqos->clk_tuning_way) {
+	case CLK_TUNING_BY_REG:
+		if (is_tx) {
+			mask = RMII_TX_PHASE_MASK;
+			val  = FIELD_PREP(RMII_TX_PHASE_MASK, eqos->tx_clk_phase);
+		} else {
+			mask = RMII_RX_PHASE_MASK;
+			val  = FIELD_PREP(RMII_RX_PHASE_MASK, eqos->rx_clk_phase);
+		}
+		ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off, mask, val);
+		break;
+
+	case CLK_TUNING_BY_CLK_REVERT: {
+		mask = is_tx ? RMII_TX_CLK_SEL : RMII_RX_CLK_SEL;
+		phase = is_tx ? eqos->tx_clk_phase : eqos->rx_clk_phase;
+		val = (phase == CLK_PHASE_REVERT) ? mask : 0;
+		ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off, mask, val);
+		break;
+	}
+
+	default:
+		dev_err(dev, "invalid clk tuning way: %d\n", eqos->clk_tuning_way);
+		return -EINVAL;
+	}
+
+	if (ret < 0)
+		dev_err(dev, "failed to update RMII %s phase (ret=%d)\n",
+			is_tx ? "tx" : "rx", ret);
+
+	return ret;
+}
+
+static int clk_phase_rgmii_set(struct spacemit_ethqos *eqos, bool is_tx)
+{
+	struct device *dev = &eqos->pdev->dev;
+	u32 mask, val;
+	int ret;
+
+	switch (eqos->clk_tuning_way) {
+	case CLK_TUNING_BY_REG:
+		if (is_tx) {
+			mask = RGMII_TX_PHASE_MASK;
+			val  = FIELD_PREP(RGMII_TX_PHASE_MASK, eqos->tx_clk_phase);
+		} else {
+			mask = RGMII_RX_PHASE_MASK;
+			val  = FIELD_PREP(RGMII_RX_PHASE_MASK, eqos->rx_clk_phase);
+		}
+		ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off, mask, val);
+		break;
+
+	case CLK_TUNING_BY_DLINE:
+		if (is_tx) {
+			mask = EMAC_TX_DLINE_CODE_MASK;
+			val  = FIELD_PREP(EMAC_TX_DLINE_CODE_MASK, eqos->tx_clk_phase);
+		} else {
+			mask = EMAC_RX_DLINE_CODE_MASK;
+			val  = FIELD_PREP(EMAC_RX_DLINE_CODE_MASK, eqos->rx_clk_phase);
+		}
+		ret = regmap_update_bits(eqos->apmu, eqos->dline_off, mask, val);
+		break;
+
+	default:
+		dev_err(dev, "invalid clk tuning way: %d\n", eqos->clk_tuning_way);
+		return -EINVAL;
+	}
+
+	if (ret < 0)
+		dev_err(dev, "failed to update RGMII %s phase (ret=%d)\n",
+			is_tx ? "tx" : "rx", ret);
+
+	return ret;
+}
+
+static int clk_phase_set(struct spacemit_ethqos *eqos, bool is_tx)
+{
+	if (!eqos->clk_tuning_enable)
+		return 0;
+
+	if (eqos->phy_iface == PHY_INTERFACE_MODE_MII)
+		return 0;
+
+	if (phy_interface_mode_is_rgmii(eqos->phy_iface))
+		return clk_phase_rgmii_set(eqos, is_tx);
+	else
+		return clk_phase_rmii_set(eqos, is_tx);
+}
+
+static int k3_delayline_init(struct spacemit_ethqos *eqos)
+{
+	u32 mask, val;
+	int ret;
+
+	/*
+	 * On K3, TX/RX delayline must be enabled for reliable DMA init.
+	 * This is required for all phy-modes (rgmii/rmii/mii).
+	 * Also clear delay codes to remove any prior configuration residue.
+	 */
+	mask = EMAC_TX_DLINE_EN | EMAC_RX_DLINE_EN |
+	       EMAC_TX_DLINE_CODE_MASK | EMAC_RX_DLINE_CODE_MASK;
+
+	val = EMAC_TX_DLINE_EN | EMAC_RX_DLINE_EN;
+
+	ret = regmap_update_bits(eqos->apmu, eqos->dline_off,
+				 mask, val);
+	if (ret)
+		dev_err(&eqos->pdev->dev, "failed to init delayline\n");
+
+	return ret;
+}
+
+static int k3_eqos_iface_config(struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &eqos->pdev->dev;
+	phy_interface_t iface = eqos->phy_iface;
+	u32 val;
+	int ret;
+
+	switch (iface) {
+	case PHY_INTERFACE_MODE_MII:
+		val = PHY_INTF_MII;
+		break;
+
+	case PHY_INTERFACE_MODE_RMII:
+		val = PHY_INTF_RMII;
+		break;
+
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		val = PHY_INTF_RGMII;
+		break;
+
+	default:
+		dev_warn(dev, "unsupported phy-mode: %s\n", phy_modes(iface));
+		return -EINVAL; /* don't write unexpected bits */
+	}
+	ret = regmap_update_bits(eqos->apmu, eqos->ctrl_off,
+				 PHY_INTF_MODE_MASK, val);
+	if (!ret)
+		dev_info(dev, "phy-mode=%s val=0x%08x\n", phy_modes(iface), val);
+
+	return ret;
+}
+
+static int k3_eqos_wol_config(struct spacemit_ethqos *eqos)
+{
+	u32 val = eqos->wol_irq_enable ? WAKE_IRQ_EN : 0;
+
+	return regmap_update_bits(eqos->apmu, eqos->ctrl_off, WAKE_IRQ_EN, val);
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int clk_phase_show(struct seq_file *s, void *data)
+{
+	struct spacemit_ethqos *eqos = s->private;
+
+	seq_printf(s, "phy-mode : %s\n", phy_modes(eqos->phy_iface));
+	seq_printf(s, "rx phase : %d\n", eqos->rx_clk_phase);
+	seq_printf(s, "tx phase : %d\n", eqos->tx_clk_phase);
+
+	return 0;
+}
+
+static ssize_t clk_tuning_write(struct file *file,
+				const char __user *user_buf,
+				size_t count, loff_t *ppos)
+{
+	struct spacemit_ethqos *eqos =
+				((struct seq_file *)(file->private_data))->private;
+	char buff[TUNING_CMD_LEN];
+	char mode_str[20];
+	size_t len = min_t(size_t, count, TUNING_CMD_LEN - 1);
+	int err, clk_phase;
+
+	if (copy_from_user(buff, user_buf, len))
+		return -EFAULT;
+	buff[len] = '\0';
+
+	err = sscanf(buff, "%19s %d", mode_str, &clk_phase);
+	if (err != 2)
+		return -EINVAL;
+	if (clk_phase < 0 || clk_phase >= CLK_PHASE_CNT)
+		return -EINVAL;
+
+	if (!strcmp(mode_str, "tx")) {
+		eqos->tx_clk_phase = clk_phase;
+		clk_phase_set(eqos, TX_PHASE);
+	} else if (!strcmp(mode_str, "rx")) {
+		eqos->rx_clk_phase = clk_phase;
+		clk_phase_set(eqos, RX_PHASE);
+	} else {
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static int clk_tuning_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, clk_phase_show, inode->i_private);
+}
+
+static const struct file_operations clk_tuning_fops = {
+	.open		= clk_tuning_open,
+	.write		= clk_tuning_write,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+#endif
+
+static int k3_parse_dt(struct platform_device *pdev, struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	u32 tx_phase, rx_phase;
+	int ret;
+
+	eqos->phy_iface = eqos->plat->phy_interface;
+
+	eqos->tx_clk = NULL;
+	if (phy_interface_mode_is_rgmii(eqos->phy_iface)) {
+		eqos->tx_clk = devm_clk_get_optional(dev, "tx_clk");
+		if (IS_ERR(eqos->tx_clk))
+			return dev_err_probe(dev, PTR_ERR(eqos->tx_clk), "tx clock");
+		if (!eqos->tx_clk)
+			dev_info(dev, "rgmii tx clk is derived from the rx clk\n");
+	}
+
+	eqos->tx_clk_from_soc = !!eqos->tx_clk;
+
+	eqos->phy_clk = devm_clk_get_optional(dev, "phy_clk");
+	if (IS_ERR(eqos->phy_clk))
+		return dev_err_probe(dev, PTR_ERR(eqos->phy_clk), "phy clock");
+
+	eqos->phy_clk_from_soc = !!eqos->phy_clk;
+	if (!eqos->phy_clk_from_soc)
+		dev_info(dev, "phy clk is provided by a external crystal oscillator\n");
+
+	eqos->apmu = syscon_regmap_lookup_by_phandle(np, "spacemit,apmu");
+	if (IS_ERR(eqos->apmu))
+		return dev_err_probe(dev, PTR_ERR(eqos->apmu), "spacemit,apmu lookup failed");
+
+	ret = of_property_read_u32(np, "spacemit,ctrl-offset", &eqos->ctrl_off);
+	if (ret)
+		return dev_err_probe(dev, ret, "missing spacemit,ctrl-offset");
+
+	ret = of_property_read_u32(np, "spacemit,dline-offset", &eqos->dline_off);
+	if (ret)
+		return dev_err_probe(dev, ret, "missing spacemit,dline-offset");
+
+	eqos->wol_irq_enable = of_property_read_bool(np, "spacemit,wake-irq-enable");
+
+	eqos->clk_tuning_enable = of_property_read_bool(np, "spacemit,clk-tuning-enable");
+	if (!eqos->clk_tuning_enable)
+		return 0;
+
+	if (of_property_read_bool(np, "spacemit,clk-tuning-by-reg")) {
+		eqos->clk_tuning_way = CLK_TUNING_BY_REG;
+	} else if (of_property_read_bool(np, "spacemit,clk-tuning-by-clk-revert")) {
+		eqos->clk_tuning_way = CLK_TUNING_BY_CLK_REVERT;
+	} else if (of_property_read_bool(np, "spacemit,clk-tuning-by-delayline")) {
+		eqos->clk_tuning_way = CLK_TUNING_BY_DLINE;
+	} else {
+		eqos->clk_tuning_way = CLK_TUNING_BY_REG;
+	}
+
+	if (of_property_read_u32(np, "spacemit,tx-phase", &tx_phase))
+		eqos->tx_clk_phase = TXCLK_PHASE_DEFAULT;
+	else
+		eqos->tx_clk_phase = tx_phase;
+
+	if (of_property_read_u32(np, "spacemit,rx-phase", &rx_phase))
+		eqos->rx_clk_phase = RXCLK_PHASE_DEFAULT;
+	else
+		eqos->rx_clk_phase = rx_phase;
+
+#ifdef CONFIG_DEBUG_FS
+	if (!eqos->dbg_dir) {
+		eqos->dbg_dir = debugfs_create_dir(dev_name(dev), NULL);
+
+		if (IS_ERR_OR_NULL(eqos->dbg_dir)) {
+			dev_err(dev, "debugfs: failed to create dir\n");
+		} else {
+			eqos->dbg_clk_tuning = debugfs_create_file("clk_tuning", 0644,
+								   eqos->dbg_dir, eqos,
+								   &clk_tuning_fops);
+			if (IS_ERR_OR_NULL(eqos->dbg_clk_tuning))
+				dev_err(dev, "debugfs: failed to create file\n");
+		}
+	}
+#endif
+
+	return 0;
+}
+
+static void devm_k3_release_dt(void *arg)
+{
+	struct spacemit_ethqos *eqos = arg;
+
+#ifdef CONFIG_DEBUG_FS
+	debugfs_remove_recursive(eqos->dbg_dir);
+	eqos->dbg_dir = NULL;
+	eqos->dbg_clk_tuning = NULL;
+#endif
+}
+
+static int devm_k3_parse_dt(struct platform_device *pdev,
+			    struct spacemit_ethqos *eqos)
+{
+	int ret;
+
+	ret = k3_parse_dt(pdev, eqos);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(&pdev->dev,
+					devm_k3_release_dt, eqos);
+}
+
+static void k3_fix_mac_speed(void *bsp_priv, int speed, unsigned int mode)
+{
+	struct spacemit_ethqos *eqos = bsp_priv;
+	struct device *dev = &eqos->pdev->dev;
+	phy_interface_t iface = eqos->phy_iface;
+
+	eqos->speed = speed;
+
+	if (!eqos->clk_tuning_enable)
+		return;
+
+	switch (iface) {
+	case PHY_INTERFACE_MODE_RGMII_ID:
+		/* PHY already provides TX+RX delay */
+		return;
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		/* PHY provides TX delay; only adjust RX */
+		clk_phase_set(eqos, RX_PHASE);
+		return;
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+		/* PHY provides RX delay; only adjust TX */
+		clk_phase_set(eqos, TX_PHASE);
+		return;
+	case PHY_INTERFACE_MODE_RMII:
+	case PHY_INTERFACE_MODE_RGMII:
+		/* rgmii/rmii: adjust both TX and RX phases */
+		clk_phase_set(eqos, TX_PHASE);
+		clk_phase_set(eqos, RX_PHASE);
+		return;
+	default:
+		dev_warn(dev, "clk tuning skipped for phy-mode: %s\n", phy_modes(iface));
+		return;
+	}
+}
+
+static int k3_clks_config(void *bsp_priv, bool enabled)
+{
+	struct spacemit_ethqos *eqos = bsp_priv;
+	int ret = 0;
+
+	if (enabled) {
+		if (eqos->tx_clk_from_soc) {
+			ret = clk_prepare_enable(eqos->tx_clk);
+			if (ret)
+				return ret;
+		}
+
+		if (eqos->phy_clk_from_soc) {
+			ret = clk_prepare_enable(eqos->phy_clk);
+			if (ret) {
+				if (eqos->tx_clk_from_soc)
+					clk_disable_unprepare(eqos->tx_clk);
+				return ret;
+			}
+		}
+	} else {
+		if (eqos->phy_clk_from_soc)
+			clk_disable_unprepare(eqos->phy_clk);
+		if (eqos->tx_clk_from_soc)
+			clk_disable_unprepare(eqos->tx_clk);
+	}
+
+	return ret;
+}
+
+static int devm_k3_bind_plat_ops(struct spacemit_ethqos *eqos)
+{
+	struct plat_stmmacenet_data *plat_dat = eqos->plat;
+
+	plat_dat->fix_mac_speed = k3_fix_mac_speed;
+	plat_dat->clks_config = k3_clks_config;
+
+	return 0;
+}
+
+static void k3_eqos_config_caps(struct spacemit_ethqos *eqos)
+{
+	struct plat_stmmacenet_data *plat_dat = eqos->plat;
+	int i;
+
+	/*
+	 * If GMAC TX clock is derived from PHY RXC, stopping RXC in EEE/LPI
+	 * will remove the TX clock and cause TX timeouts. Require RXC to run
+	 * in LPI.
+	 */
+	if (phy_interface_mode_is_rgmii(eqos->phy_iface) && !eqos->tx_clk_from_soc)
+		plat_dat->flags |= STMMAC_FLAG_RX_CLK_RUNS_IN_LPI;
+
+	/* Fall back to PHY WOL when GMAC WOL IRQ is not enabled */
+	if (!eqos->wol_irq_enable)
+		plat_dat->flags |= STMMAC_FLAG_USE_PHY_WOL;
+
+	/* Enable TSO on queue0 and enable TBS on rest of the queues */
+	for (i = 1; i < plat_dat->tx_queues_to_use; i++)
+		plat_dat->tx_queues_cfg[i].tbs_en = 1;
+}
+
+static int k3_setup_plat(struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &eqos->pdev->dev;
+	int ret;
+
+	if (eqos->phy_clk_from_soc) {
+		ret = clk_prepare_enable(eqos->phy_clk);
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to enable phy_clk\n");
+	}
+
+	if (eqos->tx_clk_from_soc) {
+		ret = clk_prepare_enable(eqos->tx_clk);
+		if (ret) {
+			ret = dev_err_probe(dev, ret, "failed to enable tx_clk\n");
+			goto err_disable_phy_clk;
+		}
+	}
+
+	ret = k3_eqos_iface_config(eqos);
+	if (ret)
+		goto err_disable_tx_clk;
+
+	ret = k3_eqos_wol_config(eqos);
+	if (ret)
+		goto err_disable_tx_clk;
+
+	ret = k3_delayline_init(eqos);
+	if (ret)
+		goto err_disable_tx_clk;
+
+	k3_eqos_config_caps(eqos);
+
+	return 0;
+
+err_disable_tx_clk:
+	if (eqos->tx_clk_from_soc)
+		clk_disable_unprepare(eqos->tx_clk);
+
+err_disable_phy_clk:
+	if (eqos->phy_clk_from_soc)
+		clk_disable_unprepare(eqos->phy_clk);
+
+	return ret;
+}
+
+static void devm_k3_cleanup_plat(void *arg)
+{
+	struct spacemit_ethqos *eqos = arg;
+
+	if (eqos->tx_clk_from_soc)
+		clk_disable_unprepare(eqos->tx_clk);
+
+	if (eqos->phy_clk_from_soc)
+		clk_disable_unprepare(eqos->phy_clk);
+}
+
+static int devm_k3_setup_plat(struct spacemit_ethqos *eqos)
+{
+	struct device *dev = &eqos->pdev->dev;
+	int ret;
+
+	ret = k3_setup_plat(eqos);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, devm_k3_cleanup_plat, eqos);
+}
+
+static const struct spacemit_ethqos_ops k3_gmac_ops = {
+	.devm_glue_parse_dt = devm_k3_parse_dt,
+	.devm_glue_bind_ops = devm_k3_bind_plat_ops,
+	.devm_glue_config_plat = devm_k3_setup_plat,
+};
+
+/* TODO: add K4/K5/K6 SoC-specific GMAC macros/ops here in future */
+
+static int spacemit_ethqos_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct stmmac_resources stmmac_res;
+	struct plat_stmmacenet_data *plat_dat;
+	const struct spacemit_ethqos_ops *ops;
+	int ret;
+
+	ops = of_device_get_match_data(dev);
+	if (!ops)
+		return dev_err_probe(dev, -EINVAL, "no of_match data");
+
+	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (ret)
+		return ret;
+
+	plat_dat = devm_stmmac_probe_config_dt(pdev, stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return PTR_ERR(plat_dat);
+
+	ret = devm_spacemit_glue_init(pdev, plat_dat, ops);
+	if (ret)
+		return ret;
+
+	ret = devm_stmmac_pltfr_probe(pdev, plat_dat, &stmmac_res);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id spacemit_ethqos_match[] = {
+	{ .compatible = "spacemit,k3-gmac", .data = &k3_gmac_ops },
+	{}
+};
+MODULE_DEVICE_TABLE(of, spacemit_ethqos_match);
+
+static struct platform_driver spacemit_ethqos_driver = {
+	.probe  = spacemit_ethqos_probe,
+	.driver = {
+		.name           = DRIVER_NAME,
+		.pm             = &stmmac_pltfr_pm_ops,
+		.of_match_table = spacemit_ethqos_match,
+	},
+};
+module_platform_driver(spacemit_ethqos_driver);
+
+MODULE_DESCRIPTION("Spacemit dwmac ethqos specific glue layer");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/wireless/realtek/Kconfig b/drivers/net/wireless/realtek/Kconfig
index 4a1f0e64df03..35a7d25b5922 100644
--- a/drivers/net/wireless/realtek/Kconfig
+++ b/drivers/net/wireless/realtek/Kconfig
@@ -17,5 +17,6 @@ source "drivers/net/wireless/realtek/rtlwifi/Kconfig"
 source "drivers/net/wireless/realtek/rtl8xxxu/Kconfig"
 source "drivers/net/wireless/realtek/rtw88/Kconfig"
 source "drivers/net/wireless/realtek/rtw89/Kconfig"
+source "drivers/net/wireless/realtek/rtl8852bs/Kconfig"
 
 endif # WLAN_VENDOR_REALTEK
diff --git a/drivers/net/wireless/realtek/Makefile b/drivers/net/wireless/realtek/Makefile
index ab25419f56c6..9bd5bbf841ab 100644
--- a/drivers/net/wireless/realtek/Makefile
+++ b/drivers/net/wireless/realtek/Makefile
@@ -9,4 +9,4 @@ obj-$(CONFIG_RTLWIFI)		+= rtlwifi/
 obj-$(CONFIG_RTL8XXXU)		+= rtl8xxxu/
 obj-$(CONFIG_RTW88)		+= rtw88/
 obj-$(CONFIG_RTW89)		+= rtw89/
-
+obj-$(CONFIG_RTL8852BS)		+= rtl8852bs/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/Kconfig b/drivers/net/wireless/realtek/rtl8852bs/Kconfig
new file mode 100644
index 000000000000..e3b8101b5352
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/Kconfig
@@ -0,0 +1,5 @@
+config RTL8852BS
+	tristate "Realtek 8852B SDIO Wireless driver"
+	depends on WLAN && MMC && CFG80211
+	help
+	This option enables support for RTL8852BS SDIO drivers
diff --git a/drivers/net/wireless/realtek/rtl8852bs/Makefile b/drivers/net/wireless/realtek/rtl8852bs/Makefile
new file mode 100644
index 000000000000..cf3636e2af14
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/Makefile
@@ -0,0 +1,987 @@
+EXTRA_CFLAGS += $(USER_EXTRA_CFLAGS)
+EXTRA_CFLAGS += -O2
+#EXTRA_CFLAGS += -O3
+#EXTRA_CFLAGS += -Wall
+#EXTRA_CFLAGS += -Wextra
+#EXTRA_CFLAGS += -Werror
+#EXTRA_CFLAGS += -pedantic
+#EXTRA_CFLAGS += -Wshadow -Wpointer-arith -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes
+
+EXTRA_CFLAGS += -Wno-unused-variable
+#EXTRA_CFLAGS += -Wno-unused-value
+EXTRA_CFLAGS += -Wno-unused-label
+#EXTRA_CFLAGS += -Wno-unused-parameter
+#EXTRA_CFLAGS += -Wno-unused-function
+EXTRA_CFLAGS += -Wno-unused
+#EXTRA_CFLAGS += -Wno-uninitialized
+
+############ ANDROID COMMON KERNEL ############
+# clang
+ifeq ($(CC), clang)
+EXTRA_CFLAGS += -Wno-uninitialized
+EXTRA_CFLAGS += -Wno-enum-conversion
+EXTRA_CFLAGS += -Wno-fortify-source
+EXTRA_CFLAGS += -Wno-invalid-source-encoding
+EXTRA_CFLAGS += -Wno-tautological-pointer-compare
+EXTRA_CFLAGS += -Wno-tautological-overlap-compare
+EXTRA_CFLAGS += -Wno-pointer-bool-conversion
+EXTRA_CFLAGS += -Wno-parentheses-equality
+EXTRA_CFLAGS += -Wno-self-assign
+EXTRA_CFLAGS += -Wno-header-guard
+endif
+
+GCC_VER_49 := $(shell echo `$(CC) -dumpversion | cut -f1-2 -d.` \>= 4.9 | bc )
+ifeq ($(GCC_VER_49),1)
+EXTRA_CFLAGS += -Wno-date-time	# Fix compile error && warning on gcc 4.9 and later
+endif
+
+EXTRA_CFLAGS += -Wno-error=designated-init -Wno-incompatible-pointer-types
+
+EXTRA_CFLAGS += -I$(src)/include
+
+EXTRA_LDFLAGS += --strip-debug
+
+CONFIG_AUTOCFG_CP = n
+
+########################## WIFI IC ############################
+CONFIG_RTL8852B = y
+CONFIG_RTL8852BP = n
+CONFIG_RTL8852BT = n
+CONFIG_RTL8851B = n
+CONFIG_RTL8852C = n
+CONFIG_RTL8852D = n
+CONFIG_RTL8842A = n
+######################### Interface ###########################
+CONFIG_USB_HCI = n
+CONFIG_PCI_HCI = n
+CONFIG_SDIO_HCI = y
+CONFIG_GSPI_HCI = n
+########################## Features ###########################
+CONFIG_MP_INCLUDED = y
+CONFIG_CONCURRENT_MODE = n
+CONFIG_POWER_SAVE = y
+CONFIG_IPS_MODE = default
+CONFIG_LPS_MODE = default
+CONFIG_BTC = y
+CONFIG_WAPI_SUPPORT = n
+CONFIG_EFUSE_CONFIG_FILE = y
+CONFIG_EXT_CLK = n
+CONFIG_TRAFFIC_PROTECT = n
+CONFIG_LOAD_PHY_PARA_FROM_FILE = y
+# Remember to set CONFIG_FILE_FWIMG when set CONFIG_FILE_FWIMG to y,
+# or driver will fail on ifconfig up because can't find firmware file
+CONFIG_FILE_FWIMG = n
+CONFIG_TXPWR_BY_RATE = y
+CONFIG_TXPWR_BY_RATE_EN = y
+CONFIG_TXPWR_LIMIT = y
+CONFIG_TXPWR_LIMIT_EN = n
+CONFIG_PCI_ASPM = n
+CONFIG_PCI_RTW_MSI = y
+CONFIG_PCI_64BIT_DMA = n
+
+####################### RTW regulatory DB selection  ######################
+# RTW regulatory database version to select
+# *MUST* config to match the certification status of shipping product,
+# otherwise regression issue on regulatory may happen.
+CONFIG_RTW_REGDB = rtk_68-47
+
+########################## Initial Channel Plan  ##########################
+# XX: unspecified
+CONFIG_RTW_COUNTRY_CODE = XX
+# 0xFFFF: unspecified
+CONFIG_RTW_CHPLAN = 0xFFFF
+# 0xFFFF: unspecified
+CONFIG_RTW_CHPLAN_6G = 0xFFFF
+
+########################## 802.11d (country IE slave) ##########################
+CONFIG_80211D = y
+# 0: disable, 1: enable, 2: enable when INIT/USER set world wide mode
+CONFIG_RTW_COUNTRY_IE_SLAVE_EN_MODE = 0
+# BIT0: deprecated BIT
+# BIT1: consider all environment BSSs, otherwise associated BSSs only
+CONFIG_RTW_COUNTRY_IE_SLAVE_FLAGS = 0x00
+
+########################## EDCCA for regulatory ##########################
+# NORMAL: Without regulatory consideration
+# CS    : Force Carrier Sense
+# ADAPT : Force Adaptivity
+# CBP   : Force Contention Based Protocol
+# AUTO  : According to regulatory
+CONFIG_RTW_EDCCA_MODE_SEL = NORMAL
+
+CONFIG_SIGNAL_SCALE_MAPPING = n
+CONFIG_80211W = y
+CONFIG_REDUCE_TX_CPU_LOADING = n
+CONFIG_BR_EXT = y
+CONFIG_TDLS = n
+CONFIG_WIFI_MONITOR = n
+CONFIG_MCC_MODE = n
+CONFIG_APPEND_VENDOR_IE_ENABLE = n
+CONFIG_RTW_NAPI = y
+CONFIG_RTW_GRO = y
+CONFIG_RTW_NETIF_SG = y
+CONFIG_RTW_IPCAM_APPLICATION = n
+CONFIG_ICMP_VOQ = n
+CONFIG_IP_R_MONITOR = n #arp VOQ and high rate
+# user priority mapping rule : tos, dscp
+CONFIG_RTW_UP_MAPPING_RULE = tos
+
+CONFIG_PHL_ARCH = y
+CONFIG_RTW_FSM = y
+
+CONFIG_HWSIM = n
+
+CONFIG_PHL_TEST_SUITE = n
+CONFIG_WIFI_6 = y
+
+RTW_PHL_RX = y
+RTW_PHL_TX = y
+RTW_PHL_BCN = y
+DIRTY_FOR_WORK = y
+
+CONFIG_DRV_FAKE_AP = n
+
+CONFIG_DBG_AX_CAM = y
+
+USE_TRUE_PHY = y
+CONFIG_RTW_MBO = y
+# CONFIG_RTKM - n/m/y for not support / standalone / built-in
+CONFIG_RTKM ?= n
+########################## Debug ###########################
+CONFIG_RTW_DEBUG = y
+# default log level is _DRV_INFO_ = 4,
+# please refer to "How_to_set_driver_debug_log_level.doc" to set the available level.
+CONFIG_RTW_LOG_LEVEL = 2
+CONFIG_RTW_PHL_LOG_LEVEL = 0
+
+# CONFIG_RTW_APPEND_LOGLEVEL decide if append kernel log level to each messages.
+# default "n" for don't append.
+CONFIG_RTW_APPEND_LOGLEVEL = n
+
+# Add "RTW_LL_*" define to change default mapping of driver debug log level
+# to Linux kernel log level.
+# NOTICE: "RTW_LL_*" would be valid only when "CONFIG_RTW_APPEND_LOGLEVEL = y"
+#RTW_LL_PRINT = KERN_CRIT
+#RTW_LL_ERR = KERN_ERR
+#RTW_LL_WARN = KERN_WARNING
+#RTW_LL_INFO = KERN_INFO
+#RTW_LL_DBG = KERN_DEBUG
+#RTW_LL_DEFAULT = KERN_DEFAULT
+
+# enable /proc/net/rtlxxxx/ debug interfaces
+CONFIG_PROC_DEBUG = y
+CONFIG_SELF_DIAG_INFO = y
+############################# MLO #############################
+CONFIG_80211BE_EHT = n
+ifeq ($(CONFIG_80211BE_EHT), y)
+EXTRA_CFLAGS += -DCONFIG_80211BE_EHT
+endif
+CONFIG_MLD_TEST = n # used for test MLD capability
+RTW_NETWORK_LINK_MAX = 3
+DTYPE = DEV_TYPE_LEGACY # DEV_TYPE_MLD for MLD device
+ifeq ($(CONFIG_MLD_TEST), y)
+EXTRA_CFLAGS += -DCONFIG_MLD_TEST
+endif
+EXTRA_CFLAGS += -DRTW_NETWORK_LINK_MAX=$(RTW_NETWORK_LINK_MAX)
+EXTRA_CFLAGS += -DDTYPE=$(DTYPE)
+######################## Wake On Lan ##########################
+CONFIG_WOWLAN = y
+# CONFIG_WAKE_TYPE definition:
+#    bit0: magic packet
+#    bit1: unicast packet (default pattern match)
+#    bit2: disconnect (beacon loss & deauth/dissociation)
+#    bit3: customized pattern match
+#    bit4: pairwise key rekey
+CONFIG_WAKEUP_TYPE = 0x0f
+CONFIG_GOOGLE_CAST_WAKEUP = n
+CONFIG_WOW_IPS_MODE = default
+CONFIG_WOW_LPS_MODE = default
+#bit0: disBBRF off, #bit1: Wireless remote controller (WRC)
+CONFIG_SUSPEND_TYPE = 0
+CONFIG_WOW_STA_MIX = n
+CONFIG_GPIO_WAKEUP = y
+# Please contact with RTK support team first. After getting the agreement from RTK support team, 
+# you are just able to modify the CONFIG_WAKEUP_GPIO_IDX with customized requirement.
+CONFIG_WAKEUP_GPIO_IDX = default
+CONFIG_HIGH_ACTIVE_DEV2HST = n
+#0:TOGGLE 1:PULSE
+CONFIG_TOGGLE_PULSE = 0
+CONFIG_PULSE_COUNT = 3
+CONFIG_MDNS_OFFLOAD = n
+CONFIG_WOW_APF = n
+######### only for USB #########
+CONFIG_ONE_PIN_GPIO = n
+CONFIG_HIGH_ACTIVE_HST2DEV = n
+CONFIG_PNO_SUPPORT = n
+CONFIG_AP_WOWLAN = n
+######### Notify SDIO Host Keep Power During Syspend ##########
+CONFIG_RTW_SDIO_PM_KEEP_POWER = y
+###################### MP HW TX MODE FOR VHT #######################
+CONFIG_MP_VHT_HW_TX_MODE = n
+
+###################### ROAMING #####################################
+CONFIG_LAYER2_ROAMING = y
+#bit0: ROAM_ON_EXPIRED, #bit1: ROAM_ON_RESUME, #bit2: ROAM_ACTIVE
+#bit3: ROAM_QUICK_SCAN
+CONFIG_ROAMING_FLAG = 0x3
+# 2:2G, 5:5G, 6:6G, 52:5G-2G, 526:5G-2G-6G, 562:5G-6G-2G, ...
+CONFIG_ROAM_SCAN_ORDER = 52
+
+###################### BTC #####################################
+# CONFIG_BTC_EXT_CFG_ESOC_TYPE definition:
+#    0: NONE
+#    1: BT_8761
+#    2: BT_8771
+#    3: SILAB_MG21
+#    4: NORDI_NRF52840
+CONFIG_BTC_EXT_CFG_ESOC_TYPE = 0
+
+# CONFIG_BTC_EXT_CFG_ANT_ISO_DB definition:
+#    default: use halbtc default antenna isolation value
+#    other decimal value: use antenna isolation value actually measured
+#                         from customer platform (unit: dB)
+CONFIG_BTC_EXT_CFG_ANT_ISO_DB = default
+
+#### Trigger bcn comparison or disconnect after N consecutive mismatched beacons ####
+CONFIG_BCN_CNT_CONFIRM_HDL = y
+# Number of consecutive mismatched beacon keys before beacon comparison and disconnect
+# CONFIG_BCN_NEW_MAX_CNT = 0 will skip the beacon check
+CONFIG_BCN_NEW_MAX_CNT = 3
+########################## Android ###########################
+# CONFIG_RTW_ANDROID - 0: no Android, 4/5/6/7/8/9/10/11/12 : Android version
+CONFIG_RTW_ANDROID = 0
+
+ifeq ($(shell test $(CONFIG_RTW_ANDROID) -gt 0; echo $$?), 0)
+EXTRA_CFLAGS += -DCONFIG_RTW_ANDROID=$(CONFIG_RTW_ANDROID)
+CONFIG_GOOGLE_CAST_WAKEUP = y
+endif
+###################### Platform Related #######################
+CONFIG_PLATFORM_I386_PC = n
+CONFIG_PLATFORM_RTL8198D = n
+CONFIG_PLATFORM_ANDROID_X86 = n
+CONFIG_PLATFORM_ANDROID_INTEL_X86 = n
+CONFIG_PLATFORM_NV_TK1 = n
+CONFIG_PLATFORM_NV_TK1_UBUNTU = n
+CONFIG_PLATFORM_ARM_SUNxI = n
+CONFIG_PLATFORM_RTK1319 = n
+CONFIG_PLATFORM_RTK16XXB = n
+CONFIG_PLATFORM_AML_S905 = n
+CONFIG_PLATFORM_AML_S905_L3A = n
+CONFIG_PLATFORM_HUANGLONG = n
+CONFIG_PLATFORM_ARM_ROCKCHIP = n
+CONFIG_PLATFORM_SPACEMIT = y
+
+########### CUSTOMER ################################
+
+CONFIG_DRVEXT_MODULE = n
+
+export TopDIR ?= $(shell pwd)
+
+########### COMMON  #################################
+ifeq ($(CONFIG_GSPI_HCI), y)
+HCI_NAME = gspi
+endif
+
+ifeq ($(CONFIG_SDIO_HCI), y)
+HCI_NAME = sdio
+endif
+
+ifeq ($(CONFIG_USB_HCI), y)
+HCI_NAME = usb
+endif
+
+ifeq ($(CONFIG_PCI_HCI), y)
+HCI_NAME = pci
+endif
+
+ifeq ($(CONFIG_HWSIM), y)
+	HAL = hal_sim
+else
+	HAL = phl
+endif
+
+DRV_PATH = $(src)
+
+########### HAL_RTL8852B #################################
+ifeq ($(CONFIG_RTL8852B), y)
+IC_NAME := rtl8852b
+ifeq ($(CONFIG_USB_HCI), y)
+MODULE_NAME = 8852bu
+endif
+ifeq ($(CONFIG_PCI_HCI), y)
+MODULE_NAME = 8852be
+endif
+ifeq ($(CONFIG_SDIO_HCI), y)
+MODULE_NAME = 8852bs
+endif
+
+endif
+
+########### HAL_RTL8852BP #################################
+ifeq ($(CONFIG_RTL8852BP), y)
+IC_NAME := rtl8852bp
+ifeq ($(CONFIG_USB_HCI), y)
+MODULE_NAME = 8852bpu
+endif
+ifeq ($(CONFIG_PCI_HCI), y)
+MODULE_NAME = 8852bpe
+endif
+ifeq ($(CONFIG_SDIO_HCI), y)
+MODULE_NAME = 8852bps
+endif
+
+endif
+
+########### HAL_RTL8852BT #################################
+ifeq ($(CONFIG_RTL8852BT), y)
+IC_NAME := rtl8852bt
+ifeq ($(CONFIG_USB_HCI), y)
+MODULE_NAME = 8852btu
+endif
+ifeq ($(CONFIG_PCI_HCI), y)
+MODULE_NAME = 8852bte
+endif
+ifeq ($(CONFIG_SDIO_HCI), y)
+MODULE_NAME = 8852bts
+endif
+
+endif
+
+########### HAL_RTL8851B #################################
+ifeq ($(CONFIG_RTL8851B), y)
+IC_NAME := rtl8851b
+ifeq ($(CONFIG_USB_HCI), y)
+MODULE_NAME = 8851bu
+endif
+ifeq ($(CONFIG_PCI_HCI), y)
+MODULE_NAME = 8851be
+endif
+ifeq ($(CONFIG_SDIO_HCI), y)
+MODULE_NAME = 8851bs
+endif
+
+endif
+
+########### HAL_RTL8852C #################################
+ifeq ($(CONFIG_RTL8852C), y)
+IC_NAME := rtl8852c
+ifeq ($(CONFIG_USB_HCI), y)
+MODULE_NAME = 8852cu
+endif
+ifeq ($(CONFIG_PCI_HCI), y)
+MODULE_NAME = 8852ce
+endif
+endif
+
+########### HAL_RTL8852D #################################
+ifeq ($(CONFIG_RTL8852D), y)
+IC_NAME := rtl8852d
+ifeq ($(CONFIG_USB_HCI), y)
+MODULE_NAME = 8852du
+endif
+ifeq ($(CONFIG_PCI_HCI), y)
+MODULE_NAME = 8852de
+endif
+endif
+
+########### HAL_RTL8842A #################################
+ifeq ($(CONFIG_RTL8842A), y)
+IC_NAME := rtl8842a
+ifeq ($(CONFIG_USB_HCI), y)
+MODULE_NAME = 8842au
+endif
+endif
+
+########### AUTO_CFG  #################################
+
+ifeq ($(CONFIG_AUTOCFG_CP), y)
+$(shell cp $(DRV_PATH)/autoconf_$(IC_NAME)_$(HCI_NAME)_linux.h $(DRV_PATH)/include/autoconf.h)
+endif
+
+########### END OF PATH  #################################
+ifeq ($(CONFIG_MP_INCLUDED), y)
+#MODULE_NAME := $(MODULE_NAME)_mp
+EXTRA_CFLAGS += -DCONFIG_MP_INCLUDED
+CONFIG_PHL_TEST_SUITE = y
+endif
+
+ifeq ($(CONFIG_RTW_FSM), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_FSM
+#EXTRA_CFLAGS += -DCONFIG_RTW_FSM_XXX
+EXTRA_CFLAGS += -DCONFIG_RTW_FSM_RRM
+#EXTRA_CFLAGS += -DCONFIG_RTW_FSM_BTM
+endif
+
+ifeq ($(CONFIG_PHL_TEST_SUITE), y)
+EXTRA_CFLAGS += -DCONFIG_PHL_TEST_SUITE
+endif
+
+ifeq ($(CONFIG_CONCURRENT_MODE), y)
+EXTRA_CFLAGS += -DCONFIG_CONCURRENT_MODE
+endif
+
+ifeq ($(CONFIG_POWER_SAVE), y)
+EXTRA_CFLAGS += -DCONFIG_POWER_SAVE
+ifneq ($(CONFIG_IPS_MODE), default)
+EXTRA_CFLAGS += -DRTW_IPS_MODE=$(CONFIG_IPS_MODE)
+endif
+ifneq ($(CONFIG_LPS_MODE), default)
+EXTRA_CFLAGS += -DRTW_LPS_MODE=$(CONFIG_LPS_MODE)
+endif
+ifneq ($(CONFIG_WOW_IPS_MODE), default)
+EXTRA_CFLAGS += -DRTW_WOW_IPS_MODE=$(CONFIG_WOW_IPS_MODE)
+endif
+ifneq ($(CONFIG_WOW_LPS_MODE), default)
+EXTRA_CFLAGS += -DRTW_WOW_LPS_MODE=$(CONFIG_WOW_LPS_MODE)
+endif
+endif
+
+ifeq ($(CONFIG_BTC), y)
+EXTRA_CFLAGS += -DCONFIG_BTC
+EXTRA_CFLAGS += -DCONFIG_BTC_EXT_CFG_ESOC_TYPE=$(CONFIG_BTC_EXT_CFG_ESOC_TYPE)
+ifeq ($(CONFIG_BTC_EXT_CFG_ANT_ISO_DB), default)
+EXTRA_CFLAGS += -DCONFIG_BTC_EXT_CFG_ANT_ISO_DB=0xFF
+else
+EXTRA_CFLAGS += -DCONFIG_BTC_EXT_CFG_ANT_ISO_DB=$(CONFIG_BTC_EXT_CFG_ANT_ISO_DB)
+endif
+endif
+
+ifeq ($(CONFIG_WAPI_SUPPORT), y)
+EXTRA_CFLAGS += -DCONFIG_WAPI_SUPPORT
+endif
+
+ifeq ($(CONFIG_WIFI_6), y)
+EXTRA_CFLAGS += -DCONFIG_WIFI_6
+endif
+
+ifeq ($(CONFIG_EFUSE_CONFIG_FILE), y)
+EXTRA_CFLAGS += -DCONFIG_EFUSE_CONFIG_FILE
+
+#EFUSE_MAP_PATH
+USER_EFUSE_MAP_PATH ?=
+ifneq ($(USER_EFUSE_MAP_PATH),)
+EXTRA_CFLAGS += -DEFUSE_MAP_PATH=\"$(USER_EFUSE_MAP_PATH)\"
+else
+EXTRA_CFLAGS += -DEFUSE_MAP_PATH=\"/system/etc/wifi/wifi_efuse_$(MODULE_NAME).map\"
+endif
+
+#WIFIMAC_PATH
+USER_WIFIMAC_PATH ?=
+ifneq ($(USER_WIFIMAC_PATH),)
+EXTRA_CFLAGS += -DWIFIMAC_PATH=\"$(USER_WIFIMAC_PATH)\"
+else
+EXTRA_CFLAGS += -DWIFIMAC_PATH=\"/data/wifimac.txt\"
+endif
+
+endif
+
+ifeq ($(CONFIG_EXT_CLK), y)
+EXTRA_CFLAGS += -DCONFIG_EXT_CLK
+endif
+
+ifeq ($(CONFIG_TRAFFIC_PROTECT), y)
+EXTRA_CFLAGS += -DCONFIG_TRAFFIC_PROTECT
+endif
+
+ifeq ($(CONFIG_LOAD_PHY_PARA_FROM_FILE), y)
+EXTRA_CFLAGS += -DCONFIG_LOAD_PHY_PARA_FROM_FILE
+#EXTRA_CFLAGS += -DREALTEK_CONFIG_PATH_WITH_IC_NAME_FOLDER
+EXTRA_CFLAGS += -DREALTEK_CONFIG_PATH=\"/lib/firmware/\"
+endif
+
+ifeq ($(CONFIG_FILE_FWIMG), y)
+EXTRA_CFLAGS += -DCONFIG_FILE_FWIMG
+# default external firmware path is [CONFIG_FIRMWARE_PATH][ic_name]/[fw_name]
+# ex. Take 8852AE as example:
+#     normal firmware is [CONFIG_FIRMWARE_PATH]rtl8852ae/rtl8852afw.bin
+#     WOW firmware is [CONFIG_FIRMWARE_PATH]rtl8852ae/rtl8852afw_wowlan.bin
+EXTRA_CFLAGS += -DCONFIG_FIRMWARE_PATH=\"\"
+# EXTRA_CFLAGS += -DCONFIG_FIRMWARE_PATH=\"/lib/firmware/\"
+endif
+
+ifeq ($(CONFIG_TXPWR_BY_RATE), n)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_BY_RATE=0
+else ifeq ($(CONFIG_TXPWR_BY_RATE), y)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_BY_RATE=1
+endif
+ifeq ($(CONFIG_TXPWR_BY_RATE_EN), n)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_BY_RATE_EN=0
+else ifeq ($(CONFIG_TXPWR_BY_RATE_EN), y)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_BY_RATE_EN=1
+else ifeq ($(CONFIG_TXPWR_BY_RATE_EN), auto)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_BY_RATE_EN=2
+endif
+
+ifeq ($(CONFIG_TXPWR_LIMIT), n)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_LIMIT=0
+else ifeq ($(CONFIG_TXPWR_LIMIT), y)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_LIMIT=1
+endif
+ifeq ($(CONFIG_TXPWR_LIMIT_EN), n)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_LIMIT_EN=0
+else ifeq ($(CONFIG_TXPWR_LIMIT_EN), y)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_LIMIT_EN=1
+else ifeq ($(CONFIG_TXPWR_LIMIT_EN), auto)
+EXTRA_CFLAGS += -DCONFIG_TXPWR_LIMIT_EN=2
+endif
+
+ifeq ($(CONFIG_PCI_ASPM), y)
+EXTRA_CFLAGS += -DCONFIG_PCI_ASPM
+endif
+
+ifeq ($(CONFIG_PCI_RTW_MSI), n)
+EXTRA_CFLAGS += -DCONFIG_RTW_FORCE_PCI_MSI_DISABLE
+endif
+
+ifeq ($(CONFIG_PCI_64BIT_DMA), y)
+EXTRA_CFLAGS += -DCONFIG_64BIT_DMA
+endif
+
+ifneq ($(CONFIG_RTW_COUNTRY_CODE), XX)
+EXTRA_CFLAGS += -DCONFIG_RTW_COUNTRY_CODE=\"$(CONFIG_RTW_COUNTRY_CODE)\"
+endif
+ifneq ($(CONFIG_RTW_CHPLAN), 0xFFFF)
+EXTRA_CFLAGS += -DCONFIG_RTW_CHPLAN=$(CONFIG_RTW_CHPLAN)
+endif
+ifneq ($(CONFIG_RTW_CHPLAN_6G), 0xFFFF)
+EXTRA_CFLAGS += -DCONFIG_RTW_CHPLAN_6G=$(CONFIG_RTW_CHPLAN_6G)
+endif
+
+ifeq ($(CONFIG_CALIBRATE_TX_POWER_BY_REGULATORY), y)
+EXTRA_CFLAGS += -DCONFIG_CALIBRATE_TX_POWER_BY_REGULATORY
+endif
+
+ifeq ($(CONFIG_CALIBRATE_TX_POWER_TO_MAX), y)
+EXTRA_CFLAGS += -DCONFIG_CALIBRATE_TX_POWER_TO_MAX
+endif
+
+ifeq ($(CONFIG_RTW_EDCCA_MODE_SEL), NORMAL)
+EXTRA_CFLAGS += -DCONFIG_RTW_EDCCA_MODE_SEL=0
+else ifeq ($(CONFIG_RTW_EDCCA_MODE_SEL), CS)
+EXTRA_CFLAGS += -DCONFIG_RTW_EDCCA_MODE_SEL=1
+else ifeq ($(CONFIG_RTW_EDCCA_MODE_SEL), ADAPT)
+EXTRA_CFLAGS += -DCONFIG_RTW_EDCCA_MODE_SEL=2
+else ifeq ($(CONFIG_RTW_EDCCA_MODE_SEL), CBP)
+EXTRA_CFLAGS += -DCONFIG_RTW_EDCCA_MODE_SEL=3
+else ifeq ($(CONFIG_RTW_EDCCA_MODE_SEL), AUTO)
+EXTRA_CFLAGS += -DCONFIG_RTW_EDCCA_MODE_SEL=255
+endif
+
+ifeq ($(CONFIG_RTW_ADAPTIVITY_EN), disable)
+EXTRA_CFLAGS += -DCONFIG_RTW_ADAPTIVITY_EN=0
+else ifeq ($(CONFIG_RTW_ADAPTIVITY_EN), enable)
+EXTRA_CFLAGS += -DCONFIG_RTW_ADAPTIVITY_EN=1
+else ifeq ($(CONFIG_RTW_ADAPTIVITY_EN), auto)
+EXTRA_CFLAGS += -DCONFIG_RTW_ADAPTIVITY_EN=2
+endif
+
+ifeq ($(CONFIG_RTW_ADAPTIVITY_MODE), normal)
+EXTRA_CFLAGS += -DCONFIG_RTW_ADAPTIVITY_MODE=0
+else ifeq ($(CONFIG_RTW_ADAPTIVITY_MODE), carrier_sense)
+EXTRA_CFLAGS += -DCONFIG_RTW_ADAPTIVITY_MODE=1
+endif
+
+ifeq ($(CONFIG_80211D), y)
+EXTRA_CFLAGS += -DCONFIG_80211D
+ifneq ($(CONFIG_RTW_COUNTRY_IE_SLAVE_EN_MODE), )
+EXTRA_CFLAGS += -DCONFIG_RTW_COUNTRY_IE_SLAVE_EN_MODE=$(CONFIG_RTW_COUNTRY_IE_SLAVE_EN_MODE)
+endif
+ifneq ($(CONFIG_RTW_COUNTRY_IE_SLAVE_FLAGS), )
+EXTRA_CFLAGS += -DCONFIG_RTW_COUNTRY_IE_SLAVE_FLAGS=$(CONFIG_RTW_COUNTRY_IE_SLAVE_FLAGS)
+endif
+endif
+
+ifeq ($(CONFIG_SIGNAL_SCALE_MAPPING), y)
+EXTRA_CFLAGS += -DCONFIG_SIGNAL_SCALE_MAPPING
+endif
+
+ifeq ($(CONFIG_80211W), y)
+EXTRA_CFLAGS += -DCONFIG_IEEE80211W
+endif
+
+ifeq ($(CONFIG_WOWLAN), y)
+EXTRA_CFLAGS += -DCONFIG_WOWLAN -DRTW_WAKEUP_EVENT=$(CONFIG_WAKEUP_TYPE)
+EXTRA_CFLAGS += -DRTW_SUSPEND_TYPE=$(CONFIG_SUSPEND_TYPE)
+ifeq ($(CONFIG_GOOGLE_CAST_WAKEUP), y)
+EXTRA_CFLAGS += -DCONFIG_GOOGLE_CAST_WAKEUP
+endif
+ifeq ($(CONFIG_WOW_STA_MIX), y)
+EXTRA_CFLAGS += -DRTW_WOW_STA_MIX
+endif
+ifeq ($(CONFIG_SDIO_HCI), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_SDIO_PM_KEEP_POWER
+endif
+endif
+
+ifeq ($(CONFIG_AP_WOWLAN), y)
+EXTRA_CFLAGS += -DCONFIG_AP_WOWLAN
+ifeq ($(CONFIG_SDIO_HCI), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_SDIO_PM_KEEP_POWER
+endif
+endif
+
+ifeq ($(CONFIG_BCN_CNT_CONFIRM_HDL), y)
+EXTRA_CFLAGS += -DCONFIG_BCN_CNT_CONFIRM_HDL
+EXTRA_CFLAGS += -DRTW_BCN_NEW_MAX_CNT=$(CONFIG_BCN_NEW_MAX_CNT)
+endif
+
+ifeq ($(CONFIG_LAYER2_ROAMING), y)
+	EXTRA_CFLAGS += -DCONFIG_LAYER2_ROAMING -DCONFIG_ROAMING_FLAG=$(CONFIG_ROAMING_FLAG)
+	EXTRA_CFLAGS += -DCONFIG_ROAM_SCAN_ORDER=$(CONFIG_ROAM_SCAN_ORDER)
+endif
+
+ifeq ($(CONFIG_PNO_SUPPORT), y)
+EXTRA_CFLAGS += -DCONFIG_PNO_SUPPORT
+endif
+
+ifeq ($(CONFIG_MDNS_OFFLOAD), y)
+EXTRA_CFLAGS += -DCONFIG_MDNS_OFFLOAD
+endif
+
+ifeq ($(CONFIG_WOW_APF), y)
+EXTRA_CFLAGS += -DCONFIG_WOW_APF
+EXTRA_CFLAGS += -DCONFIG_WOW_APF_DBG
+endif
+
+ifeq ($(CONFIG_GPIO_WAKEUP), y)
+EXTRA_CFLAGS += -DCONFIG_GPIO_WAKEUP
+EXTRA_CFLAGS += -DCONFIG_TOGGLE_PULSE=$(CONFIG_TOGGLE_PULSE)
+EXTRA_CFLAGS += -DCONFIG_PULSE_COUNT=$(CONFIG_PULSE_COUNT)
+ifeq ($(CONFIG_ONE_PIN_GPIO), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_ONE_PIN_GPIO
+endif
+ifeq ($(CONFIG_HIGH_ACTIVE_DEV2HST), y)
+EXTRA_CFLAGS += -DHIGH_ACTIVE_DEV2HST=1
+else
+EXTRA_CFLAGS += -DHIGH_ACTIVE_DEV2HST=0
+endif
+endif
+
+ifeq ($(CONFIG_HIGH_ACTIVE_HST2DEV), y)
+EXTRA_CFLAGS += -DHIGH_ACTIVE_HST2DEV=1
+else
+EXTRA_CFLAGS += -DHIGH_ACTIVE_HST2DEV=0
+endif
+
+ifneq ($(CONFIG_WAKEUP_GPIO_IDX), default)
+EXTRA_CFLAGS += -DWAKEUP_GPIO_IDX=$(CONFIG_WAKEUP_GPIO_IDX)
+endif
+
+ifeq ($(CONFIG_RTW_SDIO_PM_KEEP_POWER), y)
+ifeq ($(CONFIG_SDIO_HCI), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_SDIO_PM_KEEP_POWER
+endif
+endif
+
+ifeq ($(CONFIG_REDUCE_TX_CPU_LOADING), y)
+EXTRA_CFLAGS += -DCONFIG_REDUCE_TX_CPU_LOADING
+endif
+
+ifeq ($(CONFIG_BR_EXT), y)
+BR_NAME = br0
+EXTRA_CFLAGS += -DCONFIG_BR_EXT
+EXTRA_CFLAGS += '-DCONFIG_BR_EXT_BRNAME="'$(BR_NAME)'"'
+endif
+
+
+ifeq ($(CONFIG_TDLS), y)
+EXTRA_CFLAGS += -DCONFIG_TDLS
+endif
+
+ifeq ($(CONFIG_WIFI_MONITOR), y)
+EXTRA_CFLAGS += -DCONFIG_WIFI_MONITOR
+endif
+
+ifeq ($(CONFIG_MCC_MODE), y)
+EXTRA_CFLAGS += -DCONFIG_MCC_MODE
+endif
+
+ifeq ($(CONFIG_RTW_NAPI), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_NAPI
+endif
+
+ifeq ($(CONFIG_RTW_GRO), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_GRO
+endif
+
+ifeq ($(CONFIG_RTW_IPCAM_APPLICATION), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_IPCAM_APPLICATION
+ifeq ($(CONFIG_WIFI_MONITOR), n)
+EXTRA_CFLAGS += -DCONFIG_WIFI_MONITOR
+endif
+endif
+
+ifeq ($(CONFIG_RTW_NETIF_SG), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_NETIF_SG
+endif
+
+ifeq ($(CONFIG_ICMP_VOQ), y)
+EXTRA_CFLAGS += -DCONFIG_ICMP_VOQ
+endif
+
+ifeq ($(CONFIG_IP_R_MONITOR), y)
+EXTRA_CFLAGS += -DCONFIG_IP_R_MONITOR
+endif
+
+ifeq ($(CONFIG_MP_VHT_HW_TX_MODE), y)
+EXTRA_CFLAGS += -DCONFIG_MP_VHT_HW_TX_MODE
+ifeq ($(CONFIG_PLATFORM_I386_PC), y)
+## For I386 X86 ToolChain use Hardware FLOATING
+EXTRA_CFLAGS += -mhard-float
+else
+## For ARM ToolChain use Hardware FLOATING
+EXTRA_CFLAGS += -mfloat-abi=hard
+endif
+endif
+
+ifeq ($(CONFIG_APPEND_VENDOR_IE_ENABLE), y)
+EXTRA_CFLAGS += -DCONFIG_APPEND_VENDOR_IE_ENABLE
+endif
+
+ifeq ($(CONFIG_RTW_DEBUG), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_DEBUG
+EXTRA_CFLAGS += -DRTW_LOG_LEVEL=$(CONFIG_RTW_LOG_LEVEL)
+EXTRA_CFLAGS += -DRTW_PHL_LOG_LEVEL=$(CONFIG_RTW_PHL_LOG_LEVEL)
+EXTRA_CFLAGS += -DDRV_BB_DBG_TRACE_DISABLE
+ifeq ($(CONFIG_RTW_APPEND_LOGLEVEL), y)
+EXTRA_CFLAGS += -DRTW_APPEND_LOGLEVEL
+ifdef RTW_LL_PRINT
+EXTRA_CFLAGS += -DRTW_LL_PRINT=$(RTW_LL_PRINT)
+endif
+ifdef RTW_LL_ERR
+EXTRA_CFLAGS += -DRTW_LL_ERR=$(RTW_LL_ERR)
+endif
+ifdef RTW_LL_WARN
+EXTRA_CFLAGS += -DRTW_LL_WARN=$(RTW_LL_WARN)
+endif
+ifdef RTW_LL_INFO
+EXTRA_CFLAGS += -DRTW_LL_INFO=$(RTW_LL_INFO)
+endif
+ifdef RTW_LL_DBG
+EXTRA_CFLAGS += -DRTW_LL_DBG=$(RTW_LL_DBG)
+endif
+ifdef RTW_LL_DEFAULT
+EXTRA_CFLAGS += -DRTW_LL_DEFAULT=$(RTW_LL_DEFAULT)
+endif
+endif # CONFIG_RTW_APPEND_LOGLEVEL
+endif # CONFIG_RTW_DEBUG
+
+ifeq ($(CONFIG_PROC_DEBUG), y)
+EXTRA_CFLAGS += -DCONFIG_PROC_DEBUG
+endif
+
+ifeq ($(CONFIG_SELF_DIAG_INFO), y)
+EXTRA_CFLAGS += -DCONFIG_SELF_DIAG_INFO
+endif
+
+ifeq ($(CONFIG_RTW_UP_MAPPING_RULE), dscp)
+EXTRA_CFLAGS += -DCONFIG_RTW_UP_MAPPING_RULE=1
+else
+EXTRA_CFLAGS += -DCONFIG_RTW_UP_MAPPING_RULE=0
+endif
+
+EXTRA_CFLAGS += -DPLATFORM_LINUX
+
+ifeq ($(USE_TRUE_PHY), y)
+EXTRA_CFLAGS += -DUSE_TRUE_PHY
+endif
+
+ifeq ($(CONFIG_HWSIM), y)
+EXTRA_CFLAGS += -DCONFIG_HWSIM
+
+# To use pure sw beacon
+EXTRA_CFLAGS += -DCONFIG_SWTIMER_BASED_TXBCN
+EXTRA_CFLAGS += -DCONFIG_SUPPORT_MULTI_BCN
+endif
+
+ifeq ($(CONFIG_DRV_FAKE_AP), y)
+EXTRA_CFLAGS += -DCONFIG_DRV_FAKE_AP
+OBJS += core/rtw_fake_ap.o
+endif
+
+ifeq ($(CONFIG_DBG_AX_CAM), y)
+EXTRA_CFLAGS += -DCONFIG_DBG_AX_CAM
+endif
+
+ifeq ($(CONFIG_RTW_MBO), y)
+EXTRA_CFLAGS += -DCONFIG_RTW_MBO -DCONFIG_RTW_WNM -DCONFIG_RTW_BTM_ROAM
+#EXTRA_CFLAGS += -DCONFIG_RTW_80211K
+EXTRA_CFLAGS += -DCONFIG_RTW_80211R
+EXTRA_CFLAGS += -DRTW_FT_DBG=0 -DRTW_WNM_DBG=0 -DRTW_MBO_DBG=0
+endif
+
+########### PLATFORM OPS  ##########################
+# Import platform assigned KSRC and CROSS_COMPILE
+include $(wildcard $(DRV_PATH)/platform/*.mk)
+
+# Import platform specific compile options
+EXTRA_CFLAGS += -I$(src)/platform
+#_PLATFORM_FILES := platform/platform_ops.o
+OBJS += $(_PLATFORM_FILES)
+
+########### CUSTOMER ################################
+USER_MODULE_NAME ?=
+ifneq ($(USER_MODULE_NAME),)
+MODULE_NAME := $(USER_MODULE_NAME)
+endif
+
+############ ANDROID COMMON KERNEL ############
+export M ?= $(shell pwd)
+export OUT_DIR ?= $(shell pwd)
+ifneq ($(LLVM),)
+export CC_STRIP = llvm-strip
+else
+export CC_STRIP = $(CROSS_COMPILE)strip
+endif
+
+ifneq ($(KERNELRELEASE),)
+########### COMMON #################################
+include $(src)/common.mk
+
+EXTRA_CFLAGS += -DPHL_PLATFORM_LINUX
+EXTRA_CFLAGS += -DCONFIG_PHL_ARCH
+
+ifeq ($(RTW_PHL_RX), y)
+EXTRA_CFLAGS += -DRTW_PHL_RX
+endif
+
+ifeq ($(RTW_PHL_TX), y)
+EXTRA_CFLAGS += -DRTW_PHL_TX
+endif
+
+ifeq ($(RTW_PHL_BCN), y)
+EXTRA_CFLAGS += -DRTW_PHL_BCN
+endif
+
+ifeq ($(DIRTY_FOR_WORK), y)
+EXTRA_CFLAGS += -DDIRTY_FOR_WORK
+endif
+
+include $(src)/phl/phl.mk
+
+
+obj-$(CONFIG_RTL8852BS) := $(MODULE_NAME).o
+obj-$(CONFIG_RTL8852BS) := $(MODULE_NAME).o
+$(MODULE_NAME)-y = $(OBJS)
+
+############# MEMORY MANAGMENT #############
+ifneq ($(CONFIG_RTKM), n)
+_MEMM_FILES = core/rtw_prealloc.o
+ifeq ($(CONFIG_RTKM), y)
+EXTRA_CFLAGS += -DCONFIG_RTKM -DCONFIG_RTKM_BUILT_IN
+$(MODULE_NAME)-y += $(_MEMM_FILES)
+else ifeq ($(CONFIG_RTKM), m)
+RTKM_MODULE = rtkm
+EXTRA_CFLAGS += -DCONFIG_RTKM -DCONFIG_RTKM_STANDALONE
+_MEMM_FILES += core/rtw_mem.o
+$(RTKM_MODULE)-y += $(_MEMM_FILES)
+obj-$(CONFIG_RTL8852BS) += $(RTKM_MODULE).o
+endif
+endif
+ccflags-y += $(EXTRA_CFLAGS)
+else
+
+export CONFIG_RTL8852BS = m
+
+all: modules
+
+modules:
+#	rm -f .symvers.$(MODULE_NAME)
+
+	$(MAKE) ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) -C $(KSRC) M=$(M)  modules
+
+#	$(CC_STRIP) --strip-unneeded ${OUT_DIR}/$(M)/$(MODULE_NAME).ko
+#	cp Module.symvers .symvers.$(MODULE_NAME)
+
+strip:
+	$(CC_STRIP) $(MODULE_NAME).ko --strip-unneeded
+
+install:
+	install -p -m 644 $(MODULE_NAME).ko  $(MODDESTDIR)
+	/sbin/depmod -a ${KVER}
+
+uninstall:
+	rm -f $(MODDESTDIR)/$(MODULE_NAME).ko
+	/sbin/depmod -a ${KVER}
+
+modules_install:
+	$(MAKE) INSTALL_MOD_STRIP=1 M=$(M) -C $(KSRC) modules_install
+#	mkdir -p ${OUT_DIR}/../vendor_lib/modules
+#	cd ${OUT_DIR}/$(M)/; find -name $(MODULE_NAME).ko -exec cp {} ${OUT_DIR}/../vendor_lib/modules/ \;
+
+backup_rtlwifi:
+	@echo "Making backup rtlwifi drivers"
+ifneq (,$(wildcard $(STAGINGMODDIR)/rtl*))
+	@tar cPf $(wildcard $(STAGINGMODDIR))/backup_rtlwifi_driver.tar $(wildcard $(STAGINGMODDIR)/rtl*)
+	@rm -rf $(wildcard $(STAGINGMODDIR)/rtl*)
+endif
+ifneq (,$(wildcard $(MODDESTDIR)realtek))
+	@tar cPf $(MODDESTDIR)backup_rtlwifi_driver.tar $(MODDESTDIR)realtek
+	@rm -fr $(MODDESTDIR)realtek
+endif
+ifneq (,$(wildcard $(MODDESTDIR)rtl*))
+	@tar cPf $(MODDESTDIR)../backup_rtlwifi_driver.tar $(wildcard $(MODDESTDIR)rtl*)
+	@rm -fr $(wildcard $(MODDESTDIR)rtl*)
+endif
+	@/sbin/depmod -a ${KVER}
+	@echo "Please reboot your system"
+
+restore_rtlwifi:
+	@echo "Restoring backups"
+ifneq (,$(wildcard $(STAGINGMODDIR)/backup_rtlwifi_driver.tar))
+	@tar xPf $(STAGINGMODDIR)/backup_rtlwifi_driver.tar
+	@rm $(STAGINGMODDIR)/backup_rtlwifi_driver.tar
+endif
+ifneq (,$(wildcard $(MODDESTDIR)backup_rtlwifi_driver.tar))
+	@tar xPf $(MODDESTDIR)backup_rtlwifi_driver.tar
+	@rm $(MODDESTDIR)backup_rtlwifi_driver.tar
+endif
+ifneq (,$(wildcard $(MODDESTDIR)../backup_rtlwifi_driver.tar))
+	@tar xPf $(MODDESTDIR)../backup_rtlwifi_driver.tar
+	@rm $(MODDESTDIR)../backup_rtlwifi_driver.tar
+endif
+	@/sbin/depmod -a ${KVER}
+	@echo "Please reboot your system"
+
+config_r:
+	@echo "make config"
+	/bin/bash script/Configure script/config.in
+
+
+.PHONY: modules clean
+
+clean:
+#	$(MAKE) -C $(KSRC) M=$(shell pwd) clean
+	cd $(HAL) ; rm -fr */*/*/*/*.mod.c */*/*/*/*.mod */*/*/*/*.o */*/*/*/.*.cmd */*/*/*/*.ko
+	cd $(HAL) ; rm -fr */*/*/*.mod.c */*/*/*.mod */*/*/*.o */*/*/.*.cmd */*/*/*.ko
+	cd $(HAL) ; rm -fr */*/*.mod.c */*/*.mod */*/*.o */*/.*.cmd */*/*.ko
+	cd $(HAL) ; rm -fr */*.mod.c */*.mod */*.o */.*.cmd */*.ko
+	cd $(HAL) ; rm -fr *.mod.c *.mod *.o .*.cmd *.ko
+	cd core ; rm -fr */*.mod.c */*.mod */*.o */.*.cmd */*.ko
+	cd core ; rm -fr *.mod.c *.mod *.o .*.cmd *.ko
+	cd os_dep/linux ; rm -fr *.mod.c *.mod *.o .*.cmd *.ko
+	cd os_dep/linux/hwsim ; rm -fr *.mod.c *.mod *.o .*.cmd *.ko
+	cd os_dep ; rm -fr *.mod.c *.mod *.o .*.cmd *.ko
+	cd platform ; rm -fr *.mod.c *.mod *.o .*.cmd *.ko
+	rm -fr Module.symvers ; rm -fr Module.markers ; rm -fr modules.order
+	rm -fr *.mod.c *.mod *.o .*.cmd *.ko *~
+	rm -fr .tmp_versions
+endif
+
+############ ANDROID COMMON KERNEL ############
+# Convert to absolute path
+ifneq ($(srctree),)
+_EXTRA_CFLAGS :=
+_INC_CFLAGS :=
+$(foreach flag,$(EXTRA_CFLAGS),\
+ $(if $(shell echo $(flag) | grep "\-I"),\
+  $(eval _INC_CFLAGS += $(flag)),\
+  $(eval _EXTRA_CFLAGS += $(flag))\
+ )\
+)
+_INC_CFLAGS := \
+$(foreach flag,$(subst -I,,$(_INC_CFLAGS)),\
+ $(shell if test -d $(srctree)/$(flag); then echo -I$$(cd $(srctree)/$(flag) && pwd); else echo -I$(flag); fi)\
+)
+EXTRA_CFLAGS := $(_EXTRA_CFLAGS) $(_INC_CFLAGS)
+endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/clean b/drivers/net/wireless/realtek/rtl8852bs/clean
new file mode 100755
index 000000000000..87664218b889
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/clean
@@ -0,0 +1,5 @@
+#!/bin/bash
+rmmod 8192cu
+rmmod 8192ce
+rmmod 8192du
+rmmod 8192de
diff --git a/drivers/net/wireless/realtek/rtl8852bs/common.mk b/drivers/net/wireless/realtek/rtl8852bs/common.mk
new file mode 100755
index 000000000000..912304169c06
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/common.mk
@@ -0,0 +1,157 @@
+########### OS_DEP PATH  #################################
+_OS_INTFS_FILES :=	os_dep/osdep_service.o \
+			os_dep/osdep_service_linux.o \
+			os_dep/linux/rtw_cfg.o \
+			os_dep/linux/os_intfs.o \
+			os_dep/linux/ioctl_linux.o \
+			os_dep/linux/xmit_linux.o \
+			os_dep/linux/mlme_linux.o \
+			os_dep/linux/recv_linux.o \
+			os_dep/linux/ioctl_cfg80211.o \
+			os_dep/linux/rtw_cfgvendor.o \
+			os_dep/linux/os_ch_utils.o \
+			os_dep/linux/wifi_regd.o \
+			os_dep/linux/rtw_android.o \
+			os_dep/linux/rtw_proc.o \
+			os_dep/linux/nlrtw.o \
+			os_dep/linux/rtw_rhashtable.o
+
+ifeq ($(CONFIG_HWSIM), y)
+	_OS_INTFS_FILES += os_dep/linux/hwsim/medium/local.o
+	_OS_INTFS_FILES += os_dep/linux/hwsim/medium/sock_udp.o
+	_OS_INTFS_FILES += os_dep/linux/hwsim/medium/loopback.o
+	_OS_INTFS_FILES += os_dep/linux/hwsim/core.o
+	_OS_INTFS_FILES += os_dep/linux/hwsim/txrx.o
+	_OS_INTFS_FILES += os_dep/linux/hwsim/netdev.o
+	_OS_INTFS_FILES += os_dep/linux/hwsim/cfg80211.o
+	_OS_INTFS_FILES += os_dep/linux/hwsim/platform_dev.o
+
+	_OS_INTFS_FILES += os_dep/linux/$(HCI_NAME)_ops_linux.o
+else
+	_OS_INTFS_FILES += os_dep/linux/$(HCI_NAME)_intf.o
+	_OS_INTFS_FILES += os_dep/linux/$(HCI_NAME)_ops_linux.o
+endif
+
+ifeq ($(CONFIG_MP_INCLUDED), y)
+_OS_INTFS_FILES += os_dep/linux/ioctl_mp.o \
+		os_dep/linux/ioctl_efuse.o
+endif
+
+ifeq ($(CONFIG_SDIO_HCI), y)
+_OS_INTFS_FILES += os_dep/linux/custom_gpio_linux.o
+endif
+
+ifeq ($(CONFIG_GSPI_HCI), y)
+_OS_INTFS_FILES += os_dep/linux/custom_gpio_linux.o
+endif
+
+########### CORE PATH  #################################
+_CORE_FILES :=	core/rtw_fsm.o \
+		core/rtw_fsm_xxx.o \
+		core/rtw_fsm_rrm.o \
+		core/rtw_fsm_wnm.o \
+		core/rtw_cmd.o \
+		core/rtw_security.o \
+		core/rtw_debug.o \
+		core/rtw_io.o \
+		core/rtw_io_records.o \
+		core/rtw_ioctl_query.o \
+		core/rtw_ioctl_set.o \
+		core/rtw_ieee80211.o \
+		core/rtw_mlme.o \
+		core/rtw_mlme_ext.o \
+		core/rtw_sec_cam.o \
+		core/rtw_mi.o \
+		core/rtw_wlan_util.o \
+		core/rtw_vht.o \
+		core/rtw_he.o \
+		core/rtw_eht.o \
+		core/rtw_pwrctrl.o \
+		core/rtw_rf.o \
+		core/rtw_chplan.o \
+		core/regdb/rtw_regdb_$(CONFIG_RTW_REGDB).o \
+		core/rtw_chset.o \
+		core/rtw_dfs.o \
+		core/rtw_txpwr.o \
+		core/monitor/rtw_radiotap.o \
+		core/rtw_recv.o \
+		core/rtw_recv_shortcut.o \
+		core/rtw_sta_mgt.o \
+		core/rtw_ap.o \
+		core/rtw_csa.o \
+		core/wds/rtw_wds.o \
+		core/mesh/rtw_mesh.o \
+		core/mesh/rtw_mesh_pathtbl.o \
+		core/mesh/rtw_mesh_hwmp.o \
+		core/rtw_xmit.o	\
+		core/rtw_xmit_shortcut.o \
+		core/rtw_p2p.o \
+		core/rtw_tdls.o \
+		core/rtw_br_ext.o \
+		core/rtw_rm.o \
+		core/rtw_rm_fsm.o \
+		core/rtw_rm_util.o \
+		core/rtw_trx.o \
+		core/rtw_beamforming.o \
+		core/rtw_scan.o \
+		core/rtw_twt.o \
+		#core/efuse/rtw_efuse.o
+
+_CORE_FILES +=	core/rtw_phl.o \
+		core/rtw_phl_cmd.o
+
+EXTRA_CFLAGS += -I$(src)/core/crypto
+_CORE_FILES += core/crypto/aes-internal.o \
+		core/crypto/aes-internal-enc.o \
+		core/crypto/aes-gcm.o \
+		core/crypto/aes-ccm.o \
+		core/crypto/aes-omac1.o \
+		core/crypto/ccmp.o \
+		core/crypto/gcmp.o \
+		core/crypto/aes-siv.o \
+		core/crypto/aes-ctr.o \
+		core/crypto/sha256-internal.o \
+		core/crypto/sha256.o \
+		core/crypto/sha256-prf.o \
+		core/crypto/rtw_crypto_wrap.o \
+		core/rtw_swcrypto.o \
+		core/rtw_csi.o
+
+ifeq ($(CONFIG_WOWLAN), y)
+_CORE_FILES += core/rtw_wow.o
+endif
+
+ifeq ($(CONFIG_PCI_HCI), y)
+_CORE_FILES += core/rtw_trx_pci.o
+endif
+
+ifeq ($(CONFIG_USB_HCI), y)
+_CORE_FILES += core/rtw_trx_usb.o
+endif
+
+ifeq ($(CONFIG_SDIO_HCI), y)
+_CORE_FILES += core/rtw_sdio.o
+endif
+
+ifeq ($(CONFIG_FPGA_INCLUDED), y)
+_CORE_FILES += core/rtw_fpga.o
+endif
+
+ifeq ($(CONFIG_MP_INCLUDED), y)
+_CORE_FILES += core/rtw_mp.o
+endif
+
+ifeq ($(CONFIG_WAPI_SUPPORT), y)
+_CORE_FILES += core/rtw_wapi.o	\
+					core/rtw_wapi_sms4.o
+endif
+
+ifeq ($(CONFIG_BTC), y)
+_CORE_FILES += core/rtw_btc.o
+endif
+
+ifeq ($(CONFIG_RTW_MBO), y)
+_CORE_FILES +=	core/rtw_mbo.o core/rtw_ft.o core/rtw_wnm.o
+endif
+
+OBJS += $(_OS_INTFS_FILES) $(_CORE_FILES)
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-ccm.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-ccm.c
new file mode 100644
index 000000000000..f208eb2f4b38
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-ccm.c
@@ -0,0 +1,211 @@
+/*
+ * Counter with CBC-MAC (CCM) with AES
+ *
+ * Copyright (c) 2010-2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes.h"
+#include "aes_wrap.h"
+
+
+static void xor_aes_block(u8 *dst, const u8 *src)
+{
+	u32 *d = (u32 *) dst;
+	u32 *s = (u32 *) src;
+	*d++ ^= *s++;
+	*d++ ^= *s++;
+	*d++ ^= *s++;
+	*d++ ^= *s++;
+}
+
+
+static void aes_ccm_auth_start(void *aes, size_t M, size_t L, const u8 *nonce,
+			       const u8 *aad, size_t aad_len, size_t plain_len,
+			       u8 *x)
+{
+	u8 aad_buf[2 * AES_BLOCK_SIZE];
+	u8 b[AES_BLOCK_SIZE];
+
+	/* Authentication */
+	/* B_0: Flags | Nonce N | l(m) */
+	b[0] = aad_len ? 0x40 : 0 /* Adata */;
+	b[0] |= (((M - 2) / 2) /* M' */ << 3);
+	b[0] |= (L - 1) /* L' */;
+	os_memcpy(&b[1], nonce, 15 - L);
+	WPA_PUT_BE16(&b[AES_BLOCK_SIZE - L], plain_len);
+
+	wpa_hexdump_key(_MSG_EXCESSIVE_, "CCM B_0", b, AES_BLOCK_SIZE);
+	wpa_aes_encrypt(aes, b, x); /* X_1 = E(K, B_0) */
+
+	if (!aad_len)
+		return;
+
+	WPA_PUT_BE16(aad_buf, aad_len);
+	os_memcpy(aad_buf + 2, aad, aad_len);
+	os_memset(aad_buf + 2 + aad_len, 0, sizeof(aad_buf) - 2 - aad_len);
+
+	xor_aes_block(aad_buf, x);
+	wpa_aes_encrypt(aes, aad_buf, x); /* X_2 = E(K, X_1 XOR B_1) */
+
+	if (aad_len > AES_BLOCK_SIZE - 2) {
+		xor_aes_block(&aad_buf[AES_BLOCK_SIZE], x);
+		/* X_3 = E(K, X_2 XOR B_2) */
+		wpa_aes_encrypt(aes, &aad_buf[AES_BLOCK_SIZE], x);
+	}
+}
+
+
+static void aes_ccm_auth(void *aes, const u8 *data, size_t len, u8 *x)
+{
+	size_t last = len % AES_BLOCK_SIZE;
+	size_t i;
+
+	for (i = 0; i < len / AES_BLOCK_SIZE; i++) {
+		/* X_i+1 = E(K, X_i XOR B_i) */
+		xor_aes_block(x, data);
+		data += AES_BLOCK_SIZE;
+		wpa_aes_encrypt(aes, x, x);
+	}
+	if (last) {
+		/* XOR zero-padded last block */
+		for (i = 0; i < last; i++)
+			x[i] ^= *data++;
+		wpa_aes_encrypt(aes, x, x);
+	}
+}
+
+
+static void aes_ccm_encr_start(size_t L, const u8 *nonce, u8 *a)
+{
+	/* A_i = Flags | Nonce N | Counter i */
+	a[0] = L - 1; /* Flags = L' */
+	os_memcpy(&a[1], nonce, 15 - L);
+}
+
+
+static void aes_ccm_encr(void *aes, size_t L, const u8 *in, size_t len, u8 *out,
+			 u8 *a)
+{
+	size_t last = len % AES_BLOCK_SIZE;
+	size_t i;
+
+	/* crypt = msg XOR (S_1 | S_2 | ... | S_n) */
+	for (i = 1; i <= len / AES_BLOCK_SIZE; i++) {
+		WPA_PUT_BE16(&a[AES_BLOCK_SIZE - 2], i);
+		/* S_i = E(K, A_i) */
+		wpa_aes_encrypt(aes, a, out);
+		xor_aes_block(out, in);
+		out += AES_BLOCK_SIZE;
+		in += AES_BLOCK_SIZE;
+	}
+	if (last) {
+		WPA_PUT_BE16(&a[AES_BLOCK_SIZE - 2], i);
+		wpa_aes_encrypt(aes, a, out);
+		/* XOR zero-padded last block */
+		for (i = 0; i < last; i++)
+			*out++ ^= *in++;
+	}
+}
+
+
+static void aes_ccm_encr_auth(void *aes, size_t M, u8 *x, u8 *a, u8 *auth)
+{
+	size_t i;
+	u8 tmp[AES_BLOCK_SIZE];
+
+	wpa_hexdump_key(_MSG_EXCESSIVE_, "CCM T", x, M);
+	/* U = T XOR S_0; S_0 = E(K, A_0) */
+	WPA_PUT_BE16(&a[AES_BLOCK_SIZE - 2], 0);
+	wpa_aes_encrypt(aes, a, tmp);
+	for (i = 0; i < M; i++)
+		auth[i] = x[i] ^ tmp[i];
+	wpa_hexdump_key(_MSG_EXCESSIVE_, "CCM U", auth, M);
+}
+
+
+static void aes_ccm_decr_auth(void *aes, size_t M, u8 *a, const u8 *auth, u8 *t)
+{
+	size_t i;
+	u8 tmp[AES_BLOCK_SIZE];
+
+	wpa_hexdump_key(_MSG_EXCESSIVE_, "CCM U", auth, M);
+	/* U = T XOR S_0; S_0 = E(K, A_0) */
+	WPA_PUT_BE16(&a[AES_BLOCK_SIZE - 2], 0);
+	wpa_aes_encrypt(aes, a, tmp);
+	for (i = 0; i < M; i++)
+		t[i] = auth[i] ^ tmp[i];
+	wpa_hexdump_key(_MSG_EXCESSIVE_, "CCM T", t, M);
+}
+
+
+/* AES-CCM with fixed L=2 and aad_len <= 30 assumption */
+int aes_ccm_ae(const u8 *key, size_t key_len, const u8 *nonce,
+	       size_t M, const u8 *plain, size_t plain_len,
+	       const u8 *aad, size_t aad_len, u8 *crypt, u8 *auth)
+{
+	const size_t L = 2;
+	void *aes;
+	u8 x[AES_BLOCK_SIZE], a[AES_BLOCK_SIZE];
+
+	if (aad_len > 30 || M > AES_BLOCK_SIZE)
+		return -1;
+
+	aes = aes_encrypt_init(key, key_len);
+	if (aes == NULL)
+		return -1;
+
+	aes_ccm_auth_start(aes, M, L, nonce, aad, aad_len, plain_len, x);
+	aes_ccm_auth(aes, plain, plain_len, x);
+
+	/* Encryption */
+	aes_ccm_encr_start(L, nonce, a);
+	aes_ccm_encr(aes, L, plain, plain_len, crypt, a);
+	aes_ccm_encr_auth(aes, M, x, a, auth);
+
+	aes_encrypt_deinit(aes);
+
+	return 0;
+}
+
+
+/* AES-CCM with fixed L=2 and aad_len <= 30 assumption */
+int aes_ccm_ad(const u8 *key, size_t key_len, const u8 *nonce,
+	       size_t M, const u8 *crypt, size_t crypt_len,
+	       const u8 *aad, size_t aad_len, const u8 *auth, u8 *plain)
+{
+	const size_t L = 2;
+	void *aes;
+	u8 x[AES_BLOCK_SIZE], a[AES_BLOCK_SIZE];
+	u8 t[AES_BLOCK_SIZE];
+
+	if (aad_len > 30 || M > AES_BLOCK_SIZE)
+		return -1;
+
+	aes = aes_encrypt_init(key, key_len);
+	if (aes == NULL)
+		return -1;
+
+	/* Decryption */
+	aes_ccm_encr_start(L, nonce, a);
+	aes_ccm_decr_auth(aes, M, a, auth, t);
+
+	/* plaintext = msg XOR (S_1 | S_2 | ... | S_n) */
+	aes_ccm_encr(aes, L, crypt, crypt_len, plain, a);
+
+	aes_ccm_auth_start(aes, M, L, nonce, aad, aad_len, crypt_len, x);
+	aes_ccm_auth(aes, plain, crypt_len, x);
+
+	aes_encrypt_deinit(aes);
+
+	if (os_memcmp_const(x, t, M) != 0) {
+		wpa_printf(_MSG_EXCESSIVE_, "CCM: Auth mismatch");
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-ctr.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-ctr.c
new file mode 100644
index 000000000000..c7e235e4e064
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-ctr.c
@@ -0,0 +1,70 @@
+/*
+ * AES-128/192/256 CTR
+ *
+ * Copyright (c) 2003-2007, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes.h"
+#include "aes_wrap.h"
+
+/**
+ * aes_ctr_encrypt - AES-128/192/256 CTR mode encryption
+ * @key: Key for encryption (key_len bytes)
+ * @key_len: Length of the key (16, 24, or 32 bytes)
+ * @nonce: Nonce for counter mode (16 bytes)
+ * @data: Data to encrypt in-place
+ * @data_len: Length of data in bytes
+ * Returns: 0 on success, -1 on failure
+ */
+int aes_ctr_encrypt(const u8 *key, size_t key_len, const u8 *nonce,
+		    u8 *data, size_t data_len)
+{
+	void *ctx;
+	size_t j, len, left = data_len;
+	int i;
+	u8 *pos = data;
+	u8 counter[AES_BLOCK_SIZE], buf[AES_BLOCK_SIZE];
+
+	ctx = aes_encrypt_init(key, key_len);
+	if (ctx == NULL)
+		return -1;
+	os_memcpy(counter, nonce, AES_BLOCK_SIZE);
+
+	while (left > 0) {
+		wpa_aes_encrypt(ctx, counter, buf);
+
+		len = (left < AES_BLOCK_SIZE) ? left : AES_BLOCK_SIZE;
+		for (j = 0; j < len; j++)
+			pos[j] ^= buf[j];
+		pos += len;
+		left -= len;
+
+		for (i = AES_BLOCK_SIZE - 1; i >= 0; i--) {
+			counter[i]++;
+			if (counter[i])
+				break;
+		}
+	}
+	aes_encrypt_deinit(ctx);
+	return 0;
+}
+
+
+/**
+ * aes_128_ctr_encrypt - AES-128 CTR mode encryption
+ * @key: Key for encryption (key_len bytes)
+ * @nonce: Nonce for counter mode (16 bytes)
+ * @data: Data to encrypt in-place
+ * @data_len: Length of data in bytes
+ * Returns: 0 on success, -1 on failure
+ */
+int aes_128_ctr_encrypt(const u8 *key, const u8 *nonce,
+			u8 *data, size_t data_len)
+{
+	return aes_ctr_encrypt(key, 16, nonce, data, data_len);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-gcm.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-gcm.c
new file mode 100644
index 000000000000..27b4bbf0ac5a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-gcm.c
@@ -0,0 +1,326 @@
+/*
+ * Galois/Counter Mode (GCM) and GMAC with AES
+ *
+ * Copyright (c) 2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes.h"
+#include "aes_wrap.h"
+
+static void inc32(u8 *block)
+{
+	u32 val;
+	val = WPA_GET_BE32(block + AES_BLOCK_SIZE - 4);
+	val++;
+	WPA_PUT_BE32(block + AES_BLOCK_SIZE - 4, val);
+}
+
+
+static void xor_block(u8 *dst, const u8 *src)
+{
+	u32 *d = (u32 *) dst;
+	u32 *s = (u32 *) src;
+	*d++ ^= *s++;
+	*d++ ^= *s++;
+	*d++ ^= *s++;
+	*d++ ^= *s++;
+}
+
+
+static void shift_right_block(u8 *v)
+{
+	u32 val;
+
+	val = WPA_GET_BE32(v + 12);
+	val >>= 1;
+	if (v[11] & 0x01)
+		val |= 0x80000000;
+	WPA_PUT_BE32(v + 12, val);
+
+	val = WPA_GET_BE32(v + 8);
+	val >>= 1;
+	if (v[7] & 0x01)
+		val |= 0x80000000;
+	WPA_PUT_BE32(v + 8, val);
+
+	val = WPA_GET_BE32(v + 4);
+	val >>= 1;
+	if (v[3] & 0x01)
+		val |= 0x80000000;
+	WPA_PUT_BE32(v + 4, val);
+
+	val = WPA_GET_BE32(v);
+	val >>= 1;
+	WPA_PUT_BE32(v, val);
+}
+
+
+/* Multiplication in GF(2^128) */
+static void gf_mult(const u8 *x, const u8 *y, u8 *z)
+{
+	u8 v[16];
+	int i, j;
+
+	os_memset(z, 0, 16); /* Z_0 = 0^128 */
+	os_memcpy(v, y, 16); /* V_0 = Y */
+
+	for (i = 0; i < 16; i++) {
+		for (j = 0; j < 8; j++) {
+			if (x[i] & BIT(7 - j)) {
+				/* Z_(i + 1) = Z_i XOR V_i */
+				xor_block(z, v);
+			} else {
+				/* Z_(i + 1) = Z_i */
+			}
+
+			if (v[15] & 0x01) {
+				/* V_(i + 1) = (V_i >> 1) XOR R */
+				shift_right_block(v);
+				/* R = 11100001 || 0^120 */
+				v[0] ^= 0xe1;
+			} else {
+				/* V_(i + 1) = V_i >> 1 */
+				shift_right_block(v);
+			}
+		}
+	}
+}
+
+
+static void ghash_start(u8 *y)
+{
+	/* Y_0 = 0^128 */
+	os_memset(y, 0, 16);
+}
+
+
+static void ghash(const u8 *h, const u8 *x, size_t xlen, u8 *y)
+{
+	size_t m, i;
+	const u8 *xpos = x;
+	u8 tmp[16];
+
+	m = xlen / 16;
+
+	for (i = 0; i < m; i++) {
+		/* Y_i = (Y^(i-1) XOR X_i) dot H */
+		xor_block(y, xpos);
+		xpos += 16;
+
+		/* dot operation:
+		 * multiplication operation for binary Galois (finite) field of
+		 * 2^128 elements */
+		gf_mult(y, h, tmp);
+		os_memcpy(y, tmp, 16);
+	}
+
+	if (x + xlen > xpos) {
+		/* Add zero padded last block */
+		size_t last = x + xlen - xpos;
+		os_memcpy(tmp, xpos, last);
+		os_memset(tmp + last, 0, sizeof(tmp) - last);
+
+		/* Y_i = (Y^(i-1) XOR X_i) dot H */
+		xor_block(y, tmp);
+
+		/* dot operation:
+		 * multiplication operation for binary Galois (finite) field of
+		 * 2^128 elements */
+		gf_mult(y, h, tmp);
+		os_memcpy(y, tmp, 16);
+	}
+
+	/* Return Y_m */
+}
+
+
+static void aes_gctr(void *aes, const u8 *icb, const u8 *x, size_t xlen, u8 *y)
+{
+	size_t i, n, last;
+	u8 cb[AES_BLOCK_SIZE], tmp[AES_BLOCK_SIZE];
+	const u8 *xpos = x;
+	u8 *ypos = y;
+
+	if (xlen == 0)
+		return;
+
+	n = xlen / 16;
+
+	os_memcpy(cb, icb, AES_BLOCK_SIZE);
+	/* Full blocks */
+	for (i = 0; i < n; i++) {
+		wpa_aes_encrypt(aes, cb, ypos);
+		xor_block(ypos, xpos);
+		xpos += AES_BLOCK_SIZE;
+		ypos += AES_BLOCK_SIZE;
+		inc32(cb);
+	}
+
+	last = x + xlen - xpos;
+	if (last) {
+		/* Last, partial block */
+		wpa_aes_encrypt(aes, cb, tmp);
+		for (i = 0; i < last; i++)
+			*ypos++ = *xpos++ ^ tmp[i];
+	}
+}
+
+
+static void * aes_gcm_init_hash_subkey(const u8 *key, size_t key_len, u8 *H)
+{
+	void *aes;
+
+	aes = aes_encrypt_init(key, key_len);
+	if (aes == NULL)
+		return NULL;
+
+	/* Generate hash subkey H = AES_K(0^128) */
+	os_memset(H, 0, AES_BLOCK_SIZE);
+	wpa_aes_encrypt(aes, H, H);
+	wpa_hexdump_key(_MSG_EXCESSIVE_, "Hash subkey H for GHASH",
+			H, AES_BLOCK_SIZE);
+	return aes;
+}
+
+
+static void aes_gcm_prepare_j0(const u8 *iv, size_t iv_len, const u8 *H, u8 *J0)
+{
+	u8 len_buf[16];
+
+	if (iv_len == 12) {
+		/* Prepare block J_0 = IV || 0^31 || 1 [len(IV) = 96] */
+		os_memcpy(J0, iv, iv_len);
+		os_memset(J0 + iv_len, 0, AES_BLOCK_SIZE - iv_len);
+		J0[AES_BLOCK_SIZE - 1] = 0x01;
+	} else {
+		/*
+		 * s = 128 * ceil(len(IV)/128) - len(IV)
+		 * J_0 = GHASH_H(IV || 0^(s+64) || [len(IV)]_64)
+		 */
+		ghash_start(J0);
+		ghash(H, iv, iv_len, J0);
+		WPA_PUT_BE64(len_buf, 0);
+		WPA_PUT_BE64(len_buf + 8, iv_len * 8);
+		ghash(H, len_buf, sizeof(len_buf), J0);
+	}
+}
+
+
+static void aes_gcm_gctr(void *aes, const u8 *J0, const u8 *in, size_t len,
+			 u8 *out)
+{
+	u8 J0inc[AES_BLOCK_SIZE];
+
+	if (len == 0)
+		return;
+
+	os_memcpy(J0inc, J0, AES_BLOCK_SIZE);
+	inc32(J0inc);
+	aes_gctr(aes, J0inc, in, len, out);
+}
+
+
+static void aes_gcm_ghash(const u8 *H, const u8 *aad, size_t aad_len,
+			  const u8 *crypt, size_t crypt_len, u8 *S)
+{
+	u8 len_buf[16];
+
+	/*
+	 * u = 128 * ceil[len(C)/128] - len(C)
+	 * v = 128 * ceil[len(A)/128] - len(A)
+	 * S = GHASH_H(A || 0^v || C || 0^u || [len(A)]64 || [len(C)]64)
+	 * (i.e., zero padded to block size A || C and lengths of each in bits)
+	 */
+	ghash_start(S);
+	ghash(H, aad, aad_len, S);
+	ghash(H, crypt, crypt_len, S);
+	WPA_PUT_BE64(len_buf, aad_len * 8);
+	WPA_PUT_BE64(len_buf + 8, crypt_len * 8);
+	ghash(H, len_buf, sizeof(len_buf), S);
+
+	wpa_hexdump_key(_MSG_EXCESSIVE_, "S = GHASH_H(...)", S, 16);
+}
+
+
+/**
+ * aes_gcm_ae - GCM-AE_K(IV, P, A)
+ */
+int aes_gcm_ae(const u8 *key, size_t key_len, const u8 *iv, size_t iv_len,
+	       const u8 *plain, size_t plain_len,
+	       const u8 *aad, size_t aad_len, u8 *crypt, u8 *tag)
+{
+	u8 H[AES_BLOCK_SIZE];
+	u8 J0[AES_BLOCK_SIZE];
+	u8 S[16];
+	void *aes;
+
+	aes = aes_gcm_init_hash_subkey(key, key_len, H);
+	if (aes == NULL)
+		return -1;
+
+	aes_gcm_prepare_j0(iv, iv_len, H, J0);
+
+	/* C = GCTR_K(inc_32(J_0), P) */
+	aes_gcm_gctr(aes, J0, plain, plain_len, crypt);
+
+	aes_gcm_ghash(H, aad, aad_len, crypt, plain_len, S);
+
+	/* T = MSB_t(GCTR_K(J_0, S)) */
+	aes_gctr(aes, J0, S, sizeof(S), tag);
+
+	/* Return (C, T) */
+
+	aes_encrypt_deinit(aes);
+
+	return 0;
+}
+
+
+/**
+ * aes_gcm_ad - GCM-AD_K(IV, C, A, T)
+ */
+int aes_gcm_ad(const u8 *key, size_t key_len, const u8 *iv, size_t iv_len,
+	       const u8 *crypt, size_t crypt_len,
+	       const u8 *aad, size_t aad_len, const u8 *tag, u8 *plain)
+{
+	u8 H[AES_BLOCK_SIZE];
+	u8 J0[AES_BLOCK_SIZE];
+	u8 S[16], T[16];
+	void *aes;
+
+	aes = aes_gcm_init_hash_subkey(key, key_len, H);
+	if (aes == NULL)
+		return -1;
+
+	aes_gcm_prepare_j0(iv, iv_len, H, J0);
+
+	/* P = GCTR_K(inc_32(J_0), C) */
+	aes_gcm_gctr(aes, J0, crypt, crypt_len, plain);
+
+	aes_gcm_ghash(H, aad, aad_len, crypt, crypt_len, S);
+
+	/* T' = MSB_t(GCTR_K(J_0, S)) */
+	aes_gctr(aes, J0, S, sizeof(S), T);
+
+	aes_encrypt_deinit(aes);
+
+	if (os_memcmp_const(tag, T, 16) != 0) {
+		wpa_printf(_MSG_EXCESSIVE_, "GCM: Tag mismatch");
+		return -1;
+	}
+
+	return 0;
+}
+
+
+int aes_gmac(const u8 *key, size_t key_len, const u8 *iv, size_t iv_len,
+	     const u8 *aad, size_t aad_len, u8 *tag)
+{
+	return aes_gcm_ae(key, key_len, iv, iv_len, NULL, 0, aad, aad_len, NULL,
+			  tag);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-internal-enc.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-internal-enc.c
new file mode 100644
index 000000000000..c5521b876922
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-internal-enc.c
@@ -0,0 +1,129 @@
+/*
+ * AES (Rijndael) cipher - encrypt
+ *
+ * Modifications to public domain implementation:
+ * - cleanup
+ * - use C pre-processor to make it easier to change S table access
+ * - added option (AES_SMALL_TABLES) for reducing code size by about 8 kB at
+ *   cost of reduced throughput (quite small difference on Pentium 4,
+ *   10-25% when using -O1 or -O2 optimization)
+ *
+ * Copyright (c) 2003-2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes_i.h"
+
+static void rijndaelEncrypt(const u32 rk[], int Nr, const u8 pt[16], u8 ct[16])
+{
+	u32 s0, s1, s2, s3, t0, t1, t2, t3;
+#ifndef FULL_UNROLL
+	int r;
+#endif /* ?FULL_UNROLL */
+
+	/*
+	 * map byte array block to cipher state
+	 * and add initial round key:
+	 */
+	s0 = GETU32(pt     ) ^ rk[0];
+	s1 = GETU32(pt +  4) ^ rk[1];
+	s2 = GETU32(pt +  8) ^ rk[2];
+	s3 = GETU32(pt + 12) ^ rk[3];
+
+#define ROUND(i,d,s) \
+d##0 = TE0(s##0) ^ TE1(s##1) ^ TE2(s##2) ^ TE3(s##3) ^ rk[4 * i]; \
+d##1 = TE0(s##1) ^ TE1(s##2) ^ TE2(s##3) ^ TE3(s##0) ^ rk[4 * i + 1]; \
+d##2 = TE0(s##2) ^ TE1(s##3) ^ TE2(s##0) ^ TE3(s##1) ^ rk[4 * i + 2]; \
+d##3 = TE0(s##3) ^ TE1(s##0) ^ TE2(s##1) ^ TE3(s##2) ^ rk[4 * i + 3]
+
+#ifdef FULL_UNROLL
+
+	ROUND(1,t,s);
+	ROUND(2,s,t);
+	ROUND(3,t,s);
+	ROUND(4,s,t);
+	ROUND(5,t,s);
+	ROUND(6,s,t);
+	ROUND(7,t,s);
+	ROUND(8,s,t);
+	ROUND(9,t,s);
+	if (Nr > 10) {
+		ROUND(10,s,t);
+		ROUND(11,t,s);
+		if (Nr > 12) {
+			ROUND(12,s,t);
+			ROUND(13,t,s);
+		}
+	}
+
+	rk += Nr << 2;
+
+#else  /* !FULL_UNROLL */
+
+	/* Nr - 1 full rounds: */
+	r = Nr >> 1;
+	for (;;) {
+		ROUND(1,t,s);
+		rk += 8;
+		if (--r == 0)
+			break;
+		ROUND(0,s,t);
+	}
+
+#endif /* ?FULL_UNROLL */
+
+#undef ROUND
+
+	/*
+	 * apply last round and
+	 * map cipher state to byte array block:
+	 */
+	s0 = TE41(t0) ^ TE42(t1) ^ TE43(t2) ^ TE44(t3) ^ rk[0];
+	PUTU32(ct     , s0);
+	s1 = TE41(t1) ^ TE42(t2) ^ TE43(t3) ^ TE44(t0) ^ rk[1];
+	PUTU32(ct +  4, s1);
+	s2 = TE41(t2) ^ TE42(t3) ^ TE43(t0) ^ TE44(t1) ^ rk[2];
+	PUTU32(ct +  8, s2);
+	s3 = TE41(t3) ^ TE42(t0) ^ TE43(t1) ^ TE44(t2) ^ rk[3];
+	PUTU32(ct + 12, s3);
+}
+
+
+void * aes_encrypt_init(const u8 *key, size_t len)
+{
+	u32 *rk;
+	int res;
+
+	if (TEST_FAIL())
+		return NULL;
+
+	rk = os_malloc(AES_PRIV_SIZE);
+	if (rk == NULL)
+		return NULL;
+	res = rijndaelKeySetupEnc(rk, key, len * 8);
+	if (res < 0) {
+		rtw_mfree(rk, AES_PRIV_SIZE);
+		return NULL;
+	}
+	rk[AES_PRIV_NR_POS] = res;
+	return rk;
+}
+
+
+int wpa_aes_encrypt(void *ctx, const u8 *plain, u8 *crypt)
+{
+	u32 *rk = ctx;
+	rijndaelEncrypt(ctx, rk[AES_PRIV_NR_POS], plain, crypt);
+	return 0;
+}
+
+
+void aes_encrypt_deinit(void *ctx)
+{
+	os_memset(ctx, 0, AES_PRIV_SIZE);
+	rtw_mfree(ctx, AES_PRIV_SIZE);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-internal.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-internal.c
new file mode 100644
index 000000000000..57d653949991
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-internal.c
@@ -0,0 +1,843 @@
+/*
+ * AES (Rijndael) cipher
+ *
+ * Modifications to public domain implementation:
+ * - cleanup
+ * - use C pre-processor to make it easier to change S table access
+ * - added option (AES_SMALL_TABLES) for reducing code size by about 8 kB at
+ *   cost of reduced throughput (quite small difference on Pentium 4,
+ *   10-25% when using -O1 or -O2 optimization)
+ *
+ * Copyright (c) 2003-2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes_i.h"
+
+/*
+ * rijndael-alg-fst.c
+ *
+ * @version 3.0 (December 2000)
+ *
+ * Optimised ANSI C code for the Rijndael cipher (now AES)
+ *
+ * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
+ * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
+ * @author Paulo Barreto <paulo.barreto@terra.com.br>
+ *
+ * This code is hereby placed in the public domain.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+/*
+Te0[x] = S [x].[02, 01, 01, 03];
+Te1[x] = S [x].[03, 02, 01, 01];
+Te2[x] = S [x].[01, 03, 02, 01];
+Te3[x] = S [x].[01, 01, 03, 02];
+Te4[x] = S [x].[01, 01, 01, 01];
+
+Td0[x] = Si[x].[0e, 09, 0d, 0b];
+Td1[x] = Si[x].[0b, 0e, 09, 0d];
+Td2[x] = Si[x].[0d, 0b, 0e, 09];
+Td3[x] = Si[x].[09, 0d, 0b, 0e];
+Td4[x] = Si[x].[01, 01, 01, 01];
+*/
+
+const u32 Te0[256] = {
+    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
+    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
+    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
+    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
+    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
+    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
+    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
+    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
+    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
+    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
+    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
+    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
+    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
+    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
+    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
+    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
+    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
+    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
+    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
+    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
+    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
+    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
+    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
+    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
+    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
+    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
+    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
+    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
+    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
+    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
+    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
+    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
+    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
+    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
+    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
+    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
+    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
+    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
+    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
+    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
+    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
+    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
+    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
+    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
+    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
+    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
+    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
+    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
+    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
+    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
+    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
+    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
+    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
+    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
+    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
+    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
+    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
+    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
+    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
+    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
+    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
+    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
+    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
+    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
+};
+#ifndef AES_SMALL_TABLES
+const u32 Te1[256] = {
+    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
+    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
+    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
+    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
+    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
+    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
+    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
+    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
+    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
+    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
+    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
+    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
+    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
+    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
+    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
+    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
+    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
+    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
+    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
+    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
+    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
+    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
+    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
+    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
+    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
+    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
+    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
+    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
+    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
+    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
+    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
+    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
+    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
+    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
+    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
+    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
+    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
+    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
+    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
+    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
+    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
+    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
+    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
+    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
+    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
+    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
+    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
+    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
+    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
+    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
+    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
+    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
+    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
+    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
+    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
+    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
+    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
+    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
+    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
+    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
+    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
+    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
+    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
+    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
+};
+const u32 Te2[256] = {
+    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
+    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
+    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
+    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
+    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
+    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
+    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
+    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
+    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
+    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
+    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
+    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
+    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
+    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
+    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
+    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
+    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
+    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
+    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
+    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
+    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
+    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
+    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
+    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
+    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
+    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
+    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
+    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
+    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
+    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
+    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
+    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
+    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
+    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
+    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
+    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
+    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
+    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
+    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
+    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
+    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
+    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
+    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
+    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
+    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
+    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
+    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
+    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
+    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
+    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
+    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
+    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
+    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
+    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
+    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
+    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
+    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
+    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
+    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
+    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
+    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
+    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
+    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
+    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
+};
+const u32 Te3[256] = {
+
+    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
+    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
+    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
+    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
+    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
+    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
+    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
+    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
+    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
+    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
+    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
+    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
+    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
+    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
+    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
+    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
+    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
+    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
+    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
+    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
+    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
+    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
+    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
+    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
+    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
+    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
+    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
+    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
+    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
+    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
+    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
+    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
+    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
+    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
+    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
+    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
+    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
+    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
+    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
+    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
+    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
+    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
+    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
+    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
+    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
+    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
+    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
+    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
+    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
+    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
+    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
+    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
+    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
+    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
+    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
+    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
+    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
+    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
+    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
+    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
+    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
+    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
+    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
+    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
+};
+const u32 Te4[256] = {
+    0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
+    0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
+    0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
+    0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
+    0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
+    0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
+    0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
+    0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
+    0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
+    0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
+    0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
+    0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
+    0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
+    0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
+    0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
+    0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
+    0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
+    0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
+    0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
+    0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
+    0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
+    0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
+    0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
+    0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
+    0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
+    0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
+    0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
+    0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
+    0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
+    0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
+    0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
+    0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
+    0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
+    0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
+    0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
+    0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
+    0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
+    0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
+    0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
+    0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
+    0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
+    0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
+    0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
+    0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
+    0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
+    0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
+    0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
+    0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
+    0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
+    0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
+    0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
+    0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
+    0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
+    0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
+    0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
+    0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
+    0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
+    0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
+    0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
+    0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
+    0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
+    0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
+    0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
+    0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U,
+};
+#endif /* AES_SMALL_TABLES */
+const u32 Td0[256] = {
+    0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
+    0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
+    0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
+    0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
+    0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
+    0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
+    0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
+    0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
+    0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
+    0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
+    0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
+    0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
+    0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
+    0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
+    0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
+    0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
+    0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
+    0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
+    0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
+    0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
+    0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
+    0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
+    0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
+    0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
+    0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
+    0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
+    0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
+    0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
+    0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
+    0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
+    0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
+    0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
+    0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
+    0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
+    0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
+    0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
+    0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
+    0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
+    0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
+    0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
+    0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
+    0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
+    0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
+    0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
+    0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
+    0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
+    0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
+    0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
+    0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
+    0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
+    0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
+    0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
+    0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
+    0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
+    0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
+    0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
+    0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
+    0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
+    0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
+    0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
+    0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
+    0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
+    0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
+    0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
+};
+#ifndef AES_SMALL_TABLES
+const u32 Td1[256] = {
+    0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
+    0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
+    0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
+    0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
+    0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
+    0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
+    0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
+    0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
+    0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
+    0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
+    0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
+    0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
+    0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
+    0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
+    0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
+    0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
+    0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
+    0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
+    0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
+    0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
+    0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
+    0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
+    0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
+    0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
+    0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
+    0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
+    0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
+    0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
+    0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
+    0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
+    0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
+    0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
+    0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
+    0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
+    0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
+    0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
+    0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
+    0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
+    0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
+    0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
+    0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
+    0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
+    0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
+    0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
+    0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
+    0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
+    0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
+    0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
+    0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
+    0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
+    0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
+    0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
+    0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
+    0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
+    0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
+    0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
+    0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
+    0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
+    0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
+    0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
+    0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
+    0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
+    0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
+    0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
+};
+const u32 Td2[256] = {
+    0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
+    0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
+    0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
+    0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
+    0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
+    0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
+    0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
+    0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
+    0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
+    0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
+    0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
+    0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
+    0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
+    0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
+    0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
+    0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
+    0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
+    0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
+    0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
+    0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
+
+    0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
+    0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
+    0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
+    0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
+    0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
+    0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
+    0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
+    0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
+    0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
+    0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
+    0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
+    0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
+    0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
+    0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
+    0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
+    0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
+    0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
+    0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
+    0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
+    0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
+    0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
+    0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
+    0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
+    0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
+    0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
+    0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
+    0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
+    0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
+    0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
+    0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
+    0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
+    0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
+    0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
+    0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
+    0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
+    0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
+    0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
+    0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
+    0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
+    0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
+    0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
+    0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
+    0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
+    0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
+};
+const u32 Td3[256] = {
+    0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
+    0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
+    0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
+    0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
+    0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
+    0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
+    0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
+    0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
+    0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
+    0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
+    0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
+    0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
+    0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
+    0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
+    0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
+    0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
+    0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
+    0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
+    0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
+    0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
+    0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
+    0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
+    0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
+    0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
+    0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
+    0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
+    0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
+    0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
+    0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
+    0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
+    0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
+    0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
+    0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
+    0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
+    0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
+    0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
+    0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
+    0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
+    0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
+    0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
+    0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
+    0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
+    0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
+    0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
+    0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
+    0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
+    0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
+    0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
+    0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
+    0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
+    0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
+    0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
+    0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
+    0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
+    0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
+    0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
+    0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
+    0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
+    0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
+    0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
+    0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
+    0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
+    0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
+    0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
+};
+const u32 Td4[256] = {
+    0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U,
+    0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U,
+    0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU,
+    0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU,
+    0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U,
+    0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U,
+    0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U,
+    0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU,
+    0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U,
+    0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU,
+    0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU,
+    0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU,
+    0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U,
+    0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U,
+    0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U,
+    0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U,
+    0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U,
+    0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U,
+    0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU,
+    0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U,
+    0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U,
+    0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU,
+    0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U,
+    0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U,
+    0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U,
+    0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU,
+    0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U,
+    0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U,
+    0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU,
+    0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U,
+    0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U,
+    0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU,
+    0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U,
+    0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU,
+    0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU,
+    0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U,
+    0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U,
+    0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U,
+    0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U,
+    0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU,
+    0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U,
+    0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U,
+    0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU,
+    0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU,
+    0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU,
+    0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U,
+    0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU,
+    0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U,
+    0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U,
+    0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U,
+    0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U,
+    0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU,
+    0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U,
+    0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU,
+    0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU,
+    0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU,
+    0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU,
+    0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U,
+    0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU,
+    0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U,
+    0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU,
+    0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U,
+    0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U,
+    0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU,
+};
+const u32 rcon[] = {
+	0x01000000, 0x02000000, 0x04000000, 0x08000000,
+	0x10000000, 0x20000000, 0x40000000, 0x80000000,
+	0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
+};
+#else /* AES_SMALL_TABLES */
+const u8 Td4s[256] = {
+    0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U,
+    0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU,
+    0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U,
+    0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU,
+    0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU,
+    0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU,
+    0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U,
+    0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U,
+    0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U,
+    0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U,
+    0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU,
+    0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U,
+    0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU,
+    0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U,
+    0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U,
+    0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU,
+    0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU,
+    0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U,
+    0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U,
+    0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU,
+    0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U,
+    0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU,
+    0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U,
+    0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U,
+    0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U,
+    0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU,
+    0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU,
+    0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU,
+    0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U,
+    0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U,
+    0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U,
+    0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU,
+};
+const u8 rcons[] = {
+	0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
+	/* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
+};
+#endif /* AES_SMALL_TABLES */
+/**
+ * Expand the cipher key into the encryption key schedule.
+ *
+ * @return	the number of rounds for the given cipher key size.
+ */
+int rijndaelKeySetupEnc(u32 rk[], const u8 cipherKey[], int keyBits)
+{
+	int i;
+	u32 temp;
+
+	rk[0] = GETU32(cipherKey     );
+	rk[1] = GETU32(cipherKey +  4);
+	rk[2] = GETU32(cipherKey +  8);
+	rk[3] = GETU32(cipherKey + 12);
+
+	if (keyBits == 128) {
+		for (i = 0; i < 10; i++) {
+			temp  = rk[3];
+			rk[4] = rk[0] ^ TE421(temp) ^ TE432(temp) ^
+				TE443(temp) ^ TE414(temp) ^ RCON(i);
+			rk[5] = rk[1] ^ rk[4];
+			rk[6] = rk[2] ^ rk[5];
+			rk[7] = rk[3] ^ rk[6];
+			rk += 4;
+		}
+		return 10;
+	}
+
+	rk[4] = GETU32(cipherKey + 16);
+	rk[5] = GETU32(cipherKey + 20);
+
+	if (keyBits == 192) {
+		for (i = 0; i < 8; i++) {
+			temp  = rk[5];
+			rk[6] = rk[0] ^ TE421(temp) ^ TE432(temp) ^
+				TE443(temp) ^ TE414(temp) ^ RCON(i);
+			rk[7] = rk[1] ^ rk[6];
+			rk[8] = rk[2] ^ rk[7];
+			rk[9] = rk[3] ^ rk[8];
+			if (i == 7)
+				return 12;
+			rk[10] = rk[4] ^ rk[9];
+			rk[11] = rk[5] ^ rk[10];
+			rk += 6;
+		}
+	}
+
+	rk[6] = GETU32(cipherKey + 24);
+	rk[7] = GETU32(cipherKey + 28);
+
+	if (keyBits == 256) {
+		for (i = 0; i < 7; i++) {
+			temp  = rk[7];
+			rk[8] = rk[0] ^ TE421(temp) ^ TE432(temp) ^
+				TE443(temp) ^ TE414(temp) ^ RCON(i);
+			rk[9] = rk[1] ^ rk[8];
+			rk[10] = rk[2] ^ rk[9];
+			rk[11] = rk[3] ^ rk[10];
+			if (i == 6)
+				return 14;
+			temp  = rk[11];
+			rk[12] = rk[4] ^ TE411(temp) ^ TE422(temp) ^
+				TE433(temp) ^ TE444(temp);
+			rk[13] = rk[5] ^ rk[12];
+			rk[14] = rk[6] ^ rk[13];
+			rk[15] = rk[7] ^ rk[14];
+			rk += 8;
+		}
+	}
+
+	return -1;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-omac1.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-omac1.c
new file mode 100644
index 000000000000..c55645fb0c49
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-omac1.c
@@ -0,0 +1,172 @@
+/*
+ * One-key CBC MAC (OMAC1) hash with AES
+ *
+ * Copyright (c) 2003-2007, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes.h"
+#include "aes_wrap.h"
+
+static void gf_mulx(u8 *pad)
+{
+	int i, carry;
+
+	carry = pad[0] & 0x80;
+	for (i = 0; i < AES_BLOCK_SIZE - 1; i++)
+		pad[i] = (pad[i] << 1) | (pad[i + 1] >> 7);
+	pad[AES_BLOCK_SIZE - 1] <<= 1;
+	if (carry)
+		pad[AES_BLOCK_SIZE - 1] ^= 0x87;
+}
+
+
+/**
+ * omac1_aes_vector - One-Key CBC MAC (OMAC1) hash with AES
+ * @key: Key for the hash operation
+ * @key_len: Key length in octets
+ * @num_elem: Number of elements in the data vector
+ * @addr: Pointers to the data areas
+ * @len: Lengths of the data blocks
+ * @mac: Buffer for MAC (128 bits, i.e., 16 bytes)
+ * Returns: 0 on success, -1 on failure
+ *
+ * This is a mode for using block cipher (AES in this case) for authentication.
+ * OMAC1 was standardized with the name CMAC by NIST in a Special Publication
+ * (SP) 800-38B.
+ */
+int omac1_aes_vector(const u8 *key, size_t key_len, size_t num_elem,
+		     const u8 *addr[], const size_t *len, u8 *mac)
+{
+	void *ctx;
+	u8 cbc[AES_BLOCK_SIZE], pad[AES_BLOCK_SIZE];
+	const u8 *pos, *end;
+	size_t i, e, left, total_len;
+
+	if (TEST_FAIL())
+		return -1;
+
+	ctx = aes_encrypt_init(key, key_len);
+	if (ctx == NULL)
+		return -1;
+	os_memset(cbc, 0, AES_BLOCK_SIZE);
+
+	total_len = 0;
+	for (e = 0; e < num_elem; e++)
+		total_len += len[e];
+	left = total_len;
+
+	e = 0;
+	pos = addr[0];
+	end = pos + len[0];
+
+	while (left >= AES_BLOCK_SIZE) {
+		for (i = 0; i < AES_BLOCK_SIZE; i++) {
+			cbc[i] ^= *pos++;
+			if (pos >= end) {
+				/*
+				 * Stop if there are no more bytes to process
+				 * since there are no more entries in the array.
+				 */
+				if (i + 1 == AES_BLOCK_SIZE &&
+				    left == AES_BLOCK_SIZE)
+					break;
+				e++;
+				pos = addr[e];
+				end = pos + len[e];
+			}
+		}
+		if (left > AES_BLOCK_SIZE)
+			wpa_aes_encrypt(ctx, cbc, cbc);
+		left -= AES_BLOCK_SIZE;
+	}
+
+	os_memset(pad, 0, AES_BLOCK_SIZE);
+	wpa_aes_encrypt(ctx, pad, pad);
+	gf_mulx(pad);
+
+	if (left || total_len == 0) {
+		for (i = 0; i < left; i++) {
+			cbc[i] ^= *pos++;
+			if (pos >= end) {
+				/*
+				 * Stop if there are no more bytes to process
+				 * since there are no more entries in the array.
+				 */
+				if (i + 1 == left)
+					break;
+				e++;
+				pos = addr[e];
+				end = pos + len[e];
+			}
+		}
+		cbc[left] ^= 0x80;
+		gf_mulx(pad);
+	}
+
+	for (i = 0; i < AES_BLOCK_SIZE; i++)
+		pad[i] ^= cbc[i];
+	wpa_aes_encrypt(ctx, pad, mac);
+	aes_encrypt_deinit(ctx);
+	return 0;
+}
+
+
+/**
+ * omac1_aes_128_vector - One-Key CBC MAC (OMAC1) hash with AES-128
+ * @key: 128-bit key for the hash operation
+ * @num_elem: Number of elements in the data vector
+ * @addr: Pointers to the data areas
+ * @len: Lengths of the data blocks
+ * @mac: Buffer for MAC (128 bits, i.e., 16 bytes)
+ * Returns: 0 on success, -1 on failure
+ *
+ * This is a mode for using block cipher (AES in this case) for authentication.
+ * OMAC1 was standardized with the name CMAC by NIST in a Special Publication
+ * (SP) 800-38B.
+ */
+int omac1_aes_128_vector(const u8 *key, size_t num_elem,
+			 const u8 *addr[], const size_t *len, u8 *mac)
+{
+	return omac1_aes_vector(key, 16, num_elem, addr, len, mac);
+}
+
+
+/**
+ * omac1_aes_128 - One-Key CBC MAC (OMAC1) hash with AES-128 (aka AES-CMAC)
+ * @key: 128-bit key for the hash operation
+ * @data: Data buffer for which a MAC is determined
+ * @data_len: Length of data buffer in bytes
+ * @mac: Buffer for MAC (128 bits, i.e., 16 bytes)
+ * Returns: 0 on success, -1 on failure
+ *
+ * This is a mode for using block cipher (AES in this case) for authentication.
+ * OMAC1 was standardized with the name CMAC by NIST in a Special Publication
+ * (SP) 800-38B.
+ */
+int omac1_aes_128(const u8 *key, const u8 *data, size_t data_len, u8 *mac)
+{
+	return omac1_aes_128_vector(key, 1, &data, &data_len, mac);
+}
+
+
+/**
+ * omac1_aes_256 - One-Key CBC MAC (OMAC1) hash with AES-256 (aka AES-CMAC)
+ * @key: 256-bit key for the hash operation
+ * @data: Data buffer for which a MAC is determined
+ * @data_len: Length of data buffer in bytes
+ * @mac: Buffer for MAC (128 bits, i.e., 16 bytes)
+ * Returns: 0 on success, -1 on failure
+ *
+ * This is a mode for using block cipher (AES in this case) for authentication.
+ * OMAC1 was standardized with the name CMAC by NIST in a Special Publication
+ * (SP) 800-38B.
+ */
+int omac1_aes_256(const u8 *key, const u8 *data, size_t data_len, u8 *mac)
+{
+	return omac1_aes_vector(key, 32, 1, &data, &data_len, mac);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-siv.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-siv.c
new file mode 100644
index 000000000000..58656f2b5de4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes-siv.c
@@ -0,0 +1,207 @@
+/*
+ * AES SIV (RFC 5297)
+ * Copyright (c) 2013 Cozybit, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes.h"
+#include "aes_wrap.h"
+#include "aes_siv.h"
+
+
+static const u8 zero[AES_BLOCK_SIZE];
+
+
+static void dbl(u8 *pad)
+{
+	int i, carry;
+
+	carry = pad[0] & 0x80;
+	for (i = 0; i < AES_BLOCK_SIZE - 1; i++)
+		pad[i] = (pad[i] << 1) | (pad[i + 1] >> 7);
+	pad[AES_BLOCK_SIZE - 1] <<= 1;
+	if (carry)
+		pad[AES_BLOCK_SIZE - 1] ^= 0x87;
+}
+
+
+static void xor(u8 *a, const u8 *b)
+{
+	int i;
+
+	for (i = 0; i < AES_BLOCK_SIZE; i++)
+		*a++ ^= *b++;
+}
+
+
+static void xorend(u8 *a, int alen, const u8 *b, int blen)
+{
+	int i;
+
+	if (alen < blen)
+		return;
+
+	for (i = 0; i < blen; i++)
+		a[alen - blen + i] ^= b[i];
+}
+
+
+static void pad_block(u8 *pad, const u8 *addr, size_t len)
+{
+	os_memset(pad, 0, AES_BLOCK_SIZE);
+	os_memcpy(pad, addr, len);
+
+	if (len < AES_BLOCK_SIZE)
+		pad[len] = 0x80;
+}
+
+
+static int aes_s2v(const u8 *key, size_t key_len,
+		   size_t num_elem, const u8 *addr[], size_t *len, u8 *mac)
+{
+	u8 tmp[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
+	u8 *buf = NULL;
+	int ret;
+	size_t i;
+	const u8 *data[1];
+	size_t data_len[1];
+
+	if (!num_elem) {
+		os_memcpy(tmp, zero, sizeof(zero));
+		tmp[AES_BLOCK_SIZE - 1] = 1;
+		data[0] = tmp;
+		data_len[0] = sizeof(tmp);
+		return omac1_aes_vector(key, key_len, 1, data, data_len, mac);
+	}
+
+	data[0] = zero;
+	data_len[0] = sizeof(zero);
+	ret = omac1_aes_vector(key, key_len, 1, data, data_len, tmp);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < num_elem - 1; i++) {
+		ret = omac1_aes_vector(key, key_len, 1, &addr[i], &len[i],
+				       tmp2);
+		if (ret)
+			return ret;
+
+		dbl(tmp);
+		xor(tmp, tmp2);
+	}
+	if (len[i] >= AES_BLOCK_SIZE) {
+		buf = os_memdup(addr[i], len[i]);
+		if (!buf)
+			return -ENOMEM;
+
+		xorend(buf, len[i], tmp, AES_BLOCK_SIZE);
+		data[0] = buf;
+		ret = omac1_aes_vector(key, key_len, 1, data, &len[i], mac);
+		bin_clear_free(buf, len[i]);
+		return ret;
+	}
+
+	dbl(tmp);
+	pad_block(tmp2, addr[i], len[i]);
+	xor(tmp, tmp2);
+
+	data[0] = tmp;
+	data_len[0] = sizeof(tmp);
+	return omac1_aes_vector(key, key_len, 1, data, data_len, mac);
+}
+
+
+int aes_siv_encrypt(const u8 *key, size_t key_len,
+		    const u8 *pw, size_t pwlen,
+		    size_t num_elem, const u8 *addr[], const size_t *len,
+		    u8 *out)
+{
+	const u8 *_addr[6];
+	size_t _len[6];
+	const u8 *k1, *k2;
+	u8 v[AES_BLOCK_SIZE];
+	size_t i;
+	u8 *iv, *crypt_pw;
+
+	if (num_elem > ARRAY_SIZE(_addr) - 1 ||
+	    (key_len != 32 && key_len != 48 && key_len != 64))
+		return -1;
+
+	key_len /= 2;
+	k1 = key;
+	k2 = key + key_len;
+
+	for (i = 0; i < num_elem; i++) {
+		_addr[i] = addr[i];
+		_len[i] = len[i];
+	}
+	_addr[num_elem] = pw;
+	_len[num_elem] = pwlen;
+
+	if (aes_s2v(k1, key_len, num_elem + 1, _addr, _len, v))
+		return -1;
+
+	iv = out;
+	crypt_pw = out + AES_BLOCK_SIZE;
+
+	os_memcpy(iv, v, AES_BLOCK_SIZE);
+	os_memcpy(crypt_pw, pw, pwlen);
+
+	/* zero out 63rd and 31st bits of ctr (from right) */
+	v[8] &= 0x7f;
+	v[12] &= 0x7f;
+	return aes_ctr_encrypt(k2, key_len, v, crypt_pw, pwlen);
+}
+
+
+int aes_siv_decrypt(const u8 *key, size_t key_len,
+		    const u8 *iv_crypt, size_t iv_c_len,
+		    size_t num_elem, const u8 *addr[], const size_t *len,
+		    u8 *out)
+{
+	const u8 *_addr[6];
+	size_t _len[6];
+	const u8 *k1, *k2;
+	size_t crypt_len;
+	size_t i;
+	int ret;
+	u8 iv[AES_BLOCK_SIZE];
+	u8 check[AES_BLOCK_SIZE];
+
+	if (iv_c_len < AES_BLOCK_SIZE || num_elem > ARRAY_SIZE(_addr) - 1 ||
+	    (key_len != 32 && key_len != 48 && key_len != 64))
+		return -1;
+	crypt_len = iv_c_len - AES_BLOCK_SIZE;
+	key_len /= 2;
+	k1 = key;
+	k2 = key + key_len;
+
+	for (i = 0; i < num_elem; i++) {
+		_addr[i] = addr[i];
+		_len[i] = len[i];
+	}
+	_addr[num_elem] = out;
+	_len[num_elem] = crypt_len;
+
+	os_memcpy(iv, iv_crypt, AES_BLOCK_SIZE);
+	os_memcpy(out, iv_crypt + AES_BLOCK_SIZE, crypt_len);
+
+	iv[8] &= 0x7f;
+	iv[12] &= 0x7f;
+
+	ret = aes_ctr_encrypt(k2, key_len, iv, out, crypt_len);
+	if (ret)
+		return ret;
+
+	ret = aes_s2v(k1, key_len, num_elem + 1, _addr, _len, check);
+	if (ret)
+		return ret;
+	if (os_memcmp(check, iv_crypt, AES_BLOCK_SIZE) == 0)
+		return 0;
+
+	return -1;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes.h
new file mode 100644
index 000000000000..d105d54db5c6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes.h
@@ -0,0 +1,21 @@
+/*
+ * AES functions
+ * Copyright (c) 2003-2006, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef AES_H
+#define AES_H
+
+#define AES_BLOCK_SIZE 16
+
+void * aes_encrypt_init(const u8 *key, size_t len);
+int wpa_aes_encrypt(void *ctx, const u8 *plain, u8 *crypt);
+void aes_encrypt_deinit(void *ctx);
+void * aes_decrypt_init(const u8 *key, size_t len);
+int aes_decrypt(void *ctx, const u8 *crypt, u8 *plain);
+void aes_decrypt_deinit(void *ctx);
+
+#endif /* AES_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_i.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_i.h
new file mode 100644
index 000000000000..54375cf35583
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_i.h
@@ -0,0 +1,125 @@
+/*
+ * AES (Rijndael) cipher
+ * Copyright (c) 2003-2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef AES_I_H
+#define AES_I_H
+
+#include "aes.h"
+
+/* #define FULL_UNROLL */
+#define AES_SMALL_TABLES
+
+extern const u32 Te0[256];
+extern const u32 Te1[256];
+extern const u32 Te2[256];
+extern const u32 Te3[256];
+extern const u32 Te4[256];
+extern const u32 Td0[256];
+extern const u32 Td1[256];
+extern const u32 Td2[256];
+extern const u32 Td3[256];
+extern const u32 Td4[256];
+extern const u32 rcon[10];
+extern const u8 Td4s[256];
+extern const u8 rcons[10];
+
+#ifndef AES_SMALL_TABLES
+
+#define RCON(i) rcon[(i)]
+
+#define TE0(i) Te0[((i) >> 24) & 0xff]
+#define TE1(i) Te1[((i) >> 16) & 0xff]
+#define TE2(i) Te2[((i) >> 8) & 0xff]
+#define TE3(i) Te3[(i) & 0xff]
+#define TE41(i) (Te4[((i) >> 24) & 0xff] & 0xff000000)
+#define TE42(i) (Te4[((i) >> 16) & 0xff] & 0x00ff0000)
+#define TE43(i) (Te4[((i) >> 8) & 0xff] & 0x0000ff00)
+#define TE44(i) (Te4[(i) & 0xff] & 0x000000ff)
+#define TE421(i) (Te4[((i) >> 16) & 0xff] & 0xff000000)
+#define TE432(i) (Te4[((i) >> 8) & 0xff] & 0x00ff0000)
+#define TE443(i) (Te4[(i) & 0xff] & 0x0000ff00)
+#define TE414(i) (Te4[((i) >> 24) & 0xff] & 0x000000ff)
+#define TE411(i) (Te4[((i) >> 24) & 0xff] & 0xff000000)
+#define TE422(i) (Te4[((i) >> 16) & 0xff] & 0x00ff0000)
+#define TE433(i) (Te4[((i) >> 8) & 0xff] & 0x0000ff00)
+#define TE444(i) (Te4[(i) & 0xff] & 0x000000ff)
+#define TE4(i) (Te4[(i)] & 0x000000ff)
+
+#define TD0(i) Td0[((i) >> 24) & 0xff]
+#define TD1(i) Td1[((i) >> 16) & 0xff]
+#define TD2(i) Td2[((i) >> 8) & 0xff]
+#define TD3(i) Td3[(i) & 0xff]
+#define TD41(i) (Td4[((i) >> 24) & 0xff] & 0xff000000)
+#define TD42(i) (Td4[((i) >> 16) & 0xff] & 0x00ff0000)
+#define TD43(i) (Td4[((i) >> 8) & 0xff] & 0x0000ff00)
+#define TD44(i) (Td4[(i) & 0xff] & 0x000000ff)
+#define TD0_(i) Td0[(i) & 0xff]
+#define TD1_(i) Td1[(i) & 0xff]
+#define TD2_(i) Td2[(i) & 0xff]
+#define TD3_(i) Td3[(i) & 0xff]
+
+#else /* AES_SMALL_TABLES */
+
+#define RCON(i) (rcons[(i)] << 24)
+
+static inline u32 rotr(u32 val, int bits)
+{
+	return (val >> bits) | (val << (32 - bits));
+}
+
+#define TE0(i) Te0[((i) >> 24) & 0xff]
+#define TE1(i) rotr(Te0[((i) >> 16) & 0xff], 8)
+#define TE2(i) rotr(Te0[((i) >> 8) & 0xff], 16)
+#define TE3(i) rotr(Te0[(i) & 0xff], 24)
+#define TE41(i) ((Te0[((i) >> 24) & 0xff] << 8) & 0xff000000)
+#define TE42(i) (Te0[((i) >> 16) & 0xff] & 0x00ff0000)
+#define TE43(i) (Te0[((i) >> 8) & 0xff] & 0x0000ff00)
+#define TE44(i) ((Te0[(i) & 0xff] >> 8) & 0x000000ff)
+#define TE421(i) ((Te0[((i) >> 16) & 0xff] << 8) & 0xff000000)
+#define TE432(i) (Te0[((i) >> 8) & 0xff] & 0x00ff0000)
+#define TE443(i) (Te0[(i) & 0xff] & 0x0000ff00)
+#define TE414(i) ((Te0[((i) >> 24) & 0xff] >> 8) & 0x000000ff)
+#define TE411(i) ((Te0[((i) >> 24) & 0xff] << 8) & 0xff000000)
+#define TE422(i) (Te0[((i) >> 16) & 0xff] & 0x00ff0000)
+#define TE433(i) (Te0[((i) >> 8) & 0xff] & 0x0000ff00)
+#define TE444(i) ((Te0[(i) & 0xff] >> 8) & 0x000000ff)
+#define TE4(i) ((Te0[(i)] >> 8) & 0x000000ff)
+
+#define TD0(i) Td0[((i) >> 24) & 0xff]
+#define TD1(i) rotr(Td0[((i) >> 16) & 0xff], 8)
+#define TD2(i) rotr(Td0[((i) >> 8) & 0xff], 16)
+#define TD3(i) rotr(Td0[(i) & 0xff], 24)
+#define TD41(i) (Td4s[((i) >> 24) & 0xff] << 24)
+#define TD42(i) (Td4s[((i) >> 16) & 0xff] << 16)
+#define TD43(i) (Td4s[((i) >> 8) & 0xff] << 8)
+#define TD44(i) (Td4s[(i) & 0xff])
+#define TD0_(i) Td0[(i) & 0xff]
+#define TD1_(i) rotr(Td0[(i) & 0xff], 8)
+#define TD2_(i) rotr(Td0[(i) & 0xff], 16)
+#define TD3_(i) rotr(Td0[(i) & 0xff], 24)
+
+#endif /* AES_SMALL_TABLES */
+
+#ifdef _MSC_VER
+#define SWAP(x) (_lrotl(x, 8) & 0x00ff00ff | _lrotr(x, 8) & 0xff00ff00)
+#define GETU32(p) SWAP(*((u32 *)(p)))
+#define PUTU32(ct, st) { *((u32 *)(ct)) = SWAP((st)); }
+#else
+#define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ \
+((u32)(pt)[2] <<  8) ^ ((u32)(pt)[3]))
+#define PUTU32(ct, st) { \
+(ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); \
+(ct)[2] = (u8)((st) >>  8); (ct)[3] = (u8)(st); }
+#endif
+
+#define AES_PRIV_SIZE (4 * 4 * 15 + 4)
+#define AES_PRIV_NR_POS (4 * 15)
+
+int rijndaelKeySetupEnc(u32 rk[], const u8 cipherKey[], int keyBits);
+
+#endif /* AES_I_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_siv.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_siv.h
new file mode 100644
index 000000000000..fb05d80c1f12
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_siv.h
@@ -0,0 +1,21 @@
+/*
+ * AES SIV (RFC 5297)
+ * Copyright (c) 2013 Cozybit, Inc.
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef AES_SIV_H
+#define AES_SIV_H
+
+int aes_siv_encrypt(const u8 *key, size_t key_len,
+		    const u8 *pw, size_t pwlen,
+		    size_t num_elem, const u8 *addr[], const size_t *len,
+		    u8 *out);
+int aes_siv_decrypt(const u8 *key, size_t key_len,
+		    const u8 *iv_crypt, size_t iv_c_len,
+		    size_t num_elem, const u8 *addr[], const size_t *len,
+		    u8 *out);
+
+#endif /* AES_SIV_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_wrap.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_wrap.h
new file mode 100644
index 000000000000..b70b1d26e550
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/aes_wrap.h
@@ -0,0 +1,73 @@
+/*
+ * AES-based functions
+ *
+ * - AES Key Wrap Algorithm (RFC3394)
+ * - One-Key CBC MAC (OMAC1) hash with AES-128 and AES-256
+ * - AES-128/192/256 CTR mode encryption
+ * - AES-128 EAX mode encryption/decryption
+ * - AES-128 CBC
+ * - AES-GCM
+ * - AES-CCM
+ *
+ * Copyright (c) 2003-2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef AES_WRAP_H
+#define AES_WRAP_H
+
+int __must_check aes_wrap(const u8 *kek, size_t kek_len, int n, const u8 *plain,
+			  u8 *cipher);
+int __must_check aes_unwrap(const u8 *kek, size_t kek_len, int n,
+			    const u8 *cipher, u8 *plain);
+int __must_check omac1_aes_vector(const u8 *key, size_t key_len,
+				  size_t num_elem, const u8 *addr[],
+				  const size_t *len, u8 *mac);
+int __must_check omac1_aes_128_vector(const u8 *key, size_t num_elem,
+				      const u8 *addr[], const size_t *len,
+				      u8 *mac);
+int __must_check omac1_aes_128(const u8 *key, const u8 *data, size_t data_len,
+			       u8 *mac);
+int __must_check omac1_aes_256(const u8 *key, const u8 *data, size_t data_len,
+			       u8 *mac);
+int __must_check aes_128_encrypt_block(const u8 *key, const u8 *in, u8 *out);
+int __must_check aes_ctr_encrypt(const u8 *key, size_t key_len, const u8 *nonce,
+				 u8 *data, size_t data_len);
+int __must_check aes_128_ctr_encrypt(const u8 *key, const u8 *nonce,
+				     u8 *data, size_t data_len);
+int __must_check aes_128_eax_encrypt(const u8 *key,
+				     const u8 *nonce, size_t nonce_len,
+				     const u8 *hdr, size_t hdr_len,
+				     u8 *data, size_t data_len, u8 *tag);
+int __must_check aes_128_eax_decrypt(const u8 *key,
+				     const u8 *nonce, size_t nonce_len,
+				     const u8 *hdr, size_t hdr_len,
+				     u8 *data, size_t data_len, const u8 *tag);
+int __must_check aes_128_cbc_encrypt(const u8 *key, const u8 *iv, u8 *data,
+				     size_t data_len);
+int __must_check aes_128_cbc_decrypt(const u8 *key, const u8 *iv, u8 *data,
+				     size_t data_len);
+int __must_check aes_gcm_ae(const u8 *key, size_t key_len,
+			    const u8 *iv, size_t iv_len,
+			    const u8 *plain, size_t plain_len,
+			    const u8 *aad, size_t aad_len,
+			    u8 *crypt, u8 *tag);
+int __must_check aes_gcm_ad(const u8 *key, size_t key_len,
+			    const u8 *iv, size_t iv_len,
+			    const u8 *crypt, size_t crypt_len,
+			    const u8 *aad, size_t aad_len, const u8 *tag,
+			    u8 *plain);
+int __must_check aes_gmac(const u8 *key, size_t key_len,
+			  const u8 *iv, size_t iv_len,
+			  const u8 *aad, size_t aad_len, u8 *tag);
+int __must_check aes_ccm_ae(const u8 *key, size_t key_len, const u8 *nonce,
+			    size_t M, const u8 *plain, size_t plain_len,
+			    const u8 *aad, size_t aad_len, u8 *crypt, u8 *auth);
+int __must_check aes_ccm_ad(const u8 *key, size_t key_len, const u8 *nonce,
+			    size_t M, const u8 *crypt, size_t crypt_len,
+			    const u8 *aad, size_t aad_len, const u8 *auth,
+			    u8 *plain);
+
+#endif /* AES_WRAP_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/ccmp.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/ccmp.c
new file mode 100644
index 000000000000..13975e1453f6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/ccmp.c
@@ -0,0 +1,510 @@
+/*
+ * CTR with CBC-MAC Protocol (CCMP)
+ * Copyright (c) 2010-2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes.h"
+#include "aes_wrap.h"
+#include "wlancrypto_wrap.h"
+
+
+
+static void ccmp_aad_nonce(_adapter *padapter, const struct ieee80211_hdr *hdr, const u8 *data,
+			   u8 *aad, size_t *aad_len, u8 *nonce)
+{
+	u16 fc, stype, seq;
+	int qos = 0, addr4 = 0;
+	u8 *pos;
+
+	nonce[0] = 0;
+
+	fc = le_to_host16(hdr->frame_control);
+	stype = WLAN_FC_GET_STYPE(fc);
+	if ((fc & (WLAN_FC_TODS | WLAN_FC_FROMDS)) ==
+	    (WLAN_FC_TODS | WLAN_FC_FROMDS))
+		addr4 = 1;
+
+	if (WLAN_FC_GET_TYPE(fc) == WLAN_FC_TYPE_DATA) {
+		fc &= ~0x0070; /* Mask subtype bits */
+		if (stype & WLAN_FC_STYPE_QOS_DATA) {
+			const u8 *qc;
+			qos = 1;
+			fc &= ~WLAN_FC_ORDER;
+			qc = (const u8 *)hdr + 24;
+			if (addr4)
+				qc += ETH_ALEN;
+			nonce[0] = qc[0] & 0x0f;
+		}
+	} else if (WLAN_FC_GET_TYPE(fc) == WLAN_FC_TYPE_MGMT)
+		nonce[0] |= 0x10; /* Management */
+
+	fc &= ~(WLAN_FC_RETRY | WLAN_FC_PWRMGT | WLAN_FC_MOREDATA);
+	fc |= WLAN_FC_ISWEP;
+	WPA_PUT_LE16(aad, fc);
+	pos = aad + 2;
+	os_memcpy(pos, GetAddr1Ptr((u8 *)hdr), 3 * ETH_ALEN);
+	pos += 3 * ETH_ALEN;
+	seq = le_to_host16(hdr->seq_ctrl);
+	seq &= ~0xfff0; /* Mask Seq#; do not modify Frag# */
+	WPA_PUT_LE16(pos, seq);
+	pos += 2;
+
+	os_memcpy(pos, (u8 *)hdr + 24, addr4 * ETH_ALEN + qos * 2);
+	pos += addr4 * ETH_ALEN;
+	if (qos) {
+		pos[0] &= ~0x70;
+		/* only spp mode need to refer QoS bit7 */
+		if (padapter->registrypriv.amsdu_mode != RTW_AMSDU_MODE_SPP)
+			pos[0] &= ~0x80;
+		pos++;
+		*pos++ = 0x00;
+	}
+
+	*aad_len = pos - aad;
+
+	os_memcpy(nonce + 1, hdr->addr2, ETH_ALEN);
+	nonce[7] = data[7]; /* PN5 */
+	nonce[8] = data[6]; /* PN4 */
+	nonce[9] = data[5]; /* PN3 */
+	nonce[10] = data[4]; /* PN2 */
+	nonce[11] = data[1]; /* PN1 */
+	nonce[12] = data[0]; /* PN0 */
+}
+
+
+static void ccmp_aad_nonce_pv1(const u8 *hdr, const u8 *a1, const u8 *a2,
+			       const u8 *a3, const u8 *pn,
+			       u8 *aad, size_t *aad_len, u8 *nonce)
+{
+	u16 fc, type;
+	u8 *pos;
+
+	nonce[0] = BIT(5); /* PV1 */
+	/* TODO: Priority for QMF; 0 is used for Data frames */
+
+	fc = WPA_GET_LE16(hdr);
+	type = (fc & (BIT(2) | BIT(3) | BIT(4))) >> 2;
+
+	if (type == 1)
+		nonce[0] |= 0x10; /* Management */
+
+	fc &= ~(BIT(10) | BIT(11) | BIT(13) | BIT(14) | BIT(15));
+	fc |= BIT(12);
+	WPA_PUT_LE16(aad, fc);
+	pos = aad + 2;
+	if (type == 0 || type == 3) {
+		const u8 *sc;
+
+		os_memcpy(pos, a1, ETH_ALEN);
+		pos += ETH_ALEN;
+		os_memcpy(pos, a2, ETH_ALEN);
+		pos += ETH_ALEN;
+
+		if (type == 0) {
+			/* Either A1 or A2 contains SID */
+			sc = hdr + 2 + 2 + ETH_ALEN;
+		} else {
+			/* Both A1 and A2 contain full addresses */
+			sc = hdr + 2 + 2 * ETH_ALEN;
+		}
+		/* SC with Sequence Number subfield (bits 4-15 of the Sequence
+		 * Control field) masked to 0. */
+		*pos++ = *sc & 0x0f;
+		*pos++ = 0;
+
+		if (a3) {
+			os_memcpy(pos, a3, ETH_ALEN);
+			pos += ETH_ALEN;
+		}
+	}
+
+	*aad_len = pos - aad;
+
+	os_memcpy(nonce + 1, a2, ETH_ALEN);
+	nonce[7] = pn[5]; /* PN5 */
+	nonce[8] = pn[4]; /* PN4 */
+	nonce[9] = pn[3]; /* PN3 */
+	nonce[10] = pn[2]; /* PN2 */
+	nonce[11] = pn[1]; /* PN1 */
+	nonce[12] = pn[0]; /* PN0 */
+}
+
+
+u8 * ccmp_decrypt(_adapter *padapter, const u8 *tk, const struct ieee80211_hdr *hdr,
+		  const u8 *data, size_t data_len, size_t *decrypted_len)
+{
+	u8 aad[30], nonce[13];
+	size_t aad_len;
+	size_t mlen;
+	u8 *plain;
+
+	if (data_len < 8 + 8)
+		return NULL;
+
+	plain = os_malloc(data_len + AES_BLOCK_SIZE);
+	if (plain == NULL)
+		return NULL;
+
+	mlen = data_len - 8 - 8;
+
+	os_memset(aad, 0, sizeof(aad));
+	ccmp_aad_nonce(padapter, hdr, data, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP nonce", nonce, 13);
+
+	if (aes_ccm_ad(tk, 16, nonce, 8, data + 8, mlen, aad, aad_len,
+		       data + 8 + mlen, plain) < 0) {
+		u16 seq_ctrl = le_to_host16(hdr->seq_ctrl);
+		wpa_printf(_MSG_INFO_, "Invalid CCMP MIC in frame: A1=" MACSTR
+			   " A2=" MACSTR " A3=" MACSTR " seq=%u frag=%u",
+			   MAC2STR(hdr->addr1), MAC2STR(hdr->addr2),
+			   MAC2STR(hdr->addr3),
+			   WLAN_GET_SEQ_SEQ(seq_ctrl),
+			   WLAN_GET_SEQ_FRAG(seq_ctrl));
+		rtw_mfree(plain, data_len + AES_BLOCK_SIZE);
+		return NULL;
+	}
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP decrypted", plain, mlen);
+
+	*decrypted_len = mlen;
+	return plain;
+}
+
+
+void ccmp_get_pn(u8 *pn, const u8 *data)
+{
+	pn[0] = data[7]; /* PN5 */
+	pn[1] = data[6]; /* PN4 */
+	pn[2] = data[5]; /* PN3 */
+	pn[3] = data[4]; /* PN2 */
+	pn[4] = data[1]; /* PN1 */
+	pn[5] = data[0]; /* PN0 */
+}
+
+
+u8 * ccmp_encrypt(_adapter *padapter, const u8 *tk, u8 *frame, size_t len, size_t hdrlen, u8 *qos,
+		  u8 *pn, int keyid, size_t *encrypted_len)
+{
+	u8 aad[30], nonce[13];
+	size_t aad_len, plen;
+	u8 *crypt, *pos, *pdata;
+	struct ieee80211_hdr *hdr;
+
+	if (len < hdrlen || hdrlen < 24)
+		return NULL;
+	plen = len - hdrlen;
+
+	crypt = os_malloc(hdrlen + 8 + plen + 8 + AES_BLOCK_SIZE);
+	if (crypt == NULL)
+		return NULL;
+
+	if (pn == NULL) {
+		os_memcpy(crypt, frame, hdrlen + 8);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen + 8;
+		pdata = frame + hdrlen + 8;
+	} else {
+		os_memcpy(crypt, frame, hdrlen);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen;
+		*pos++ = pn[5]; /* PN0 */
+		*pos++ = pn[4]; /* PN1 */
+		*pos++ = 0x00; /* Rsvd */
+		*pos++ = 0x20 | (keyid << 6);
+		*pos++ = pn[3]; /* PN2 */
+		*pos++ = pn[2]; /* PN3 */
+		*pos++ = pn[1]; /* PN4 */
+		*pos++ = pn[0]; /* PN5 */
+		pdata = frame + hdrlen;
+	}
+
+	os_memset(aad, 0, sizeof(aad));
+	ccmp_aad_nonce(padapter, hdr, crypt + hdrlen, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP nonce", nonce, 13);
+
+	if (aes_ccm_ae(tk, 16, nonce, 8, pdata, plen, aad, aad_len,
+		       pos, pos + plen) < 0) {
+		rtw_mfree(crypt, hdrlen + 8 + plen + 8 + AES_BLOCK_SIZE);
+		return NULL;
+	}
+
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP encrypted", crypt + hdrlen + 8, plen);
+
+	*encrypted_len = hdrlen + 8 + plen + 8;
+
+	return crypt;
+}
+
+
+u8 * ccmp_encrypt_pv1(const u8 *tk, const u8 *a1, const u8 *a2, const u8 *a3,
+		      const u8 *frame, size_t len,
+		      size_t hdrlen, const u8 *pn, int keyid,
+		      size_t *encrypted_len)
+{
+	u8 aad[24], nonce[13];
+	size_t aad_len, plen;
+	u8 *crypt, *pos;
+	struct ieee80211_hdr *hdr;
+
+	if (len < hdrlen || hdrlen < 12)
+		return NULL;
+	plen = len - hdrlen;
+
+	crypt = os_malloc(hdrlen + plen + 8 + AES_BLOCK_SIZE);
+	if (crypt == NULL)
+		return NULL;
+
+	os_memcpy(crypt, frame, hdrlen);
+	hdr = (struct ieee80211_hdr *) crypt;
+	hdr->frame_control |= host_to_le16(BIT(12)); /* Protected Frame */
+	pos = crypt + hdrlen;
+
+	os_memset(aad, 0, sizeof(aad));
+	ccmp_aad_nonce_pv1(crypt, a1, a2, a3, pn, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP nonce", nonce, sizeof(nonce));
+
+	if (aes_ccm_ae(tk, 16, nonce, 8, frame + hdrlen, plen, aad, aad_len,
+		       pos, pos + plen) < 0) {
+		rtw_mfree(crypt, hdrlen + plen + 8 + AES_BLOCK_SIZE);
+		return NULL;
+	}
+
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP encrypted", crypt + hdrlen, plen);
+
+	*encrypted_len = hdrlen + plen + 8;
+
+	return crypt;
+}
+
+
+u8 * ccmp_256_decrypt(_adapter *padapter, const u8 *tk, const struct ieee80211_hdr *hdr,
+		      const u8 *data, size_t data_len, size_t *decrypted_len)
+{
+	u8 aad[30], nonce[13];
+	size_t aad_len;
+	size_t mlen;
+	u8 *plain;
+
+	if (data_len < 8 + 16)
+		return NULL;
+
+	plain = os_malloc(data_len + AES_BLOCK_SIZE);
+	if (plain == NULL)
+		return NULL;
+
+	mlen = data_len - 8 - 16;
+
+	os_memset(aad, 0, sizeof(aad));
+	ccmp_aad_nonce(padapter, hdr, data, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 nonce", nonce, 13);
+
+	if (aes_ccm_ad(tk, 32, nonce, 16, data + 8, mlen, aad, aad_len,
+		       data + 8 + mlen, plain) < 0) {
+		u16 seq_ctrl = le_to_host16(hdr->seq_ctrl);
+		wpa_printf(_MSG_INFO_, "Invalid CCMP-256 MIC in frame: A1=" MACSTR
+			   " A2=" MACSTR " A3=" MACSTR " seq=%u frag=%u",
+			   MAC2STR(hdr->addr1), MAC2STR(hdr->addr2),
+			   MAC2STR(hdr->addr3),
+			   WLAN_GET_SEQ_SEQ(seq_ctrl),
+			   WLAN_GET_SEQ_FRAG(seq_ctrl));
+		rtw_mfree(plain, data_len + AES_BLOCK_SIZE);
+		return NULL;
+	}
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 decrypted", plain, mlen);
+
+	*decrypted_len = mlen;
+	return plain;
+}
+
+
+u8 * ccmp_256_encrypt(_adapter *padapter, const u8 *tk, u8 *frame, size_t len, size_t hdrlen,
+		      u8 *qos, u8 *pn, int keyid, size_t *encrypted_len)
+{
+	u8 aad[30], nonce[13];
+	size_t aad_len, plen;
+	u8 *crypt, *pos, *pdata;
+	struct ieee80211_hdr *hdr;
+
+	if (len < hdrlen || hdrlen < 24)
+		return NULL;
+	plen = len - hdrlen;
+
+	crypt = os_malloc(hdrlen + 8 + plen + 16 + AES_BLOCK_SIZE);
+	if (crypt == NULL)
+		return NULL;
+
+	if (pn == NULL) {
+		os_memcpy(crypt, frame, hdrlen + 8);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen + 8;
+		pdata = frame + hdrlen + 8;
+	} else {
+		os_memcpy(crypt, frame, hdrlen);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen;
+		*pos++ = pn[5]; /* PN0 */
+		*pos++ = pn[4]; /* PN1 */
+		*pos++ = 0x00; /* Rsvd */
+		*pos++ = 0x20 | (keyid << 6);
+		*pos++ = pn[3]; /* PN2 */
+		*pos++ = pn[2]; /* PN3 */
+		*pos++ = pn[1]; /* PN4 */
+		*pos++ = pn[0]; /* PN5 */
+		pdata = frame + hdrlen;
+	}
+
+	os_memset(aad, 0, sizeof(aad));
+	ccmp_aad_nonce(padapter, hdr, crypt + hdrlen, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 nonce", nonce, 13);
+
+	if (aes_ccm_ae(tk, 32, nonce, 16, pdata, plen, aad, aad_len,
+		       pos, pos + plen) < 0) {
+		rtw_mfree(crypt, hdrlen + 8 + plen + 16 + AES_BLOCK_SIZE);
+		return NULL;
+	}
+
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 encrypted", crypt + hdrlen + 8,
+		    plen);
+
+	*encrypted_len = hdrlen + 8 + plen + 16;
+
+	return crypt;
+}
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+void core_ccmp_encrypt(const u8 *tk, uint hdrlen, u8 *phdr, uint datalen, u8 *pdata, 
+	u8 *qos, u8 *pn, int keyid, size_t *encrypted_len)
+{
+	u8 aad[30], nonce[13];
+	size_t aad_len;
+	u8 *crypt, *pos;
+	struct ieee80211_hdr *hdr;
+	size_t enc_hdrlen, enc_datalen = 0;
+
+	if (hdrlen < 24)
+		return;
+
+	enc_hdrlen = hdrlen + 8;
+	enc_datalen = datalen + 8;
+
+	crypt = os_malloc(enc_hdrlen + enc_hdrlen + AES_BLOCK_SIZE);
+	if (crypt == NULL)
+		return;
+
+	if (pn == NULL) {
+		os_memcpy(crypt, phdr, hdrlen + 8);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen + 8;
+	} else {
+		os_memcpy(crypt, phdr, hdrlen);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen;
+		*pos++ = pn[5]; /* PN0 */
+		*pos++ = pn[4]; /* PN1 */
+		*pos++ = 0x00; /* Rsvd */
+		*pos++ = 0x20 | (keyid << 6);
+		*pos++ = pn[3]; /* PN2 */
+		*pos++ = pn[2]; /* PN3 */
+		*pos++ = pn[1]; /* PN4 */
+		*pos++ = pn[0]; /* PN5 */
+	}
+
+	os_memset(aad, 0, sizeof(aad));
+	ccmp_aad_nonce(hdr, crypt + hdrlen, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP nonce", nonce, 13);
+
+	if (aes_ccm_ae(tk, 16, nonce, 8, pdata, datalen, aad, aad_len,
+		       pos, pos + datalen) < 0) {
+		rtw_mfree(crypt, hdrlen + 8 + datalen + 8 + AES_BLOCK_SIZE);
+		return;
+	}
+
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP encrypted", crypt + enc_hdrlen, datalen);
+
+	/* Copy @enc back to @frame and free @enc */
+	_rtw_memcpy(phdr, crypt, enc_hdrlen);
+	_rtw_memcpy(pdata, crypt + enc_hdrlen, enc_datalen);
+	
+	rtw_mfree(crypt, enc_hdrlen+enc_datalen+AES_BLOCK_SIZE);
+
+	return;
+}
+
+
+void core_ccmp_256_encrypt(const u8 *tk, uint hdrlen, u8 *phdr, uint datalen, u8 *pdata, 
+		      u8 *qos, u8 *pn, int keyid, size_t *encrypted_len)
+{
+	u8 aad[30], nonce[13];
+	size_t aad_len;
+	u8 *crypt, *pos;
+	struct ieee80211_hdr *hdr;
+	size_t enc_hdrlen, enc_datalen = 0;
+
+	if (hdrlen < 24)
+		return;
+
+	enc_hdrlen = hdrlen + 8;
+	enc_datalen = datalen + 16;
+
+	crypt = os_malloc(enc_hdrlen + enc_hdrlen + AES_BLOCK_SIZE);
+	
+	if (crypt == NULL)
+		return;
+
+	if (pn == NULL) {
+		os_memcpy(crypt, phdr, hdrlen + 8);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen + 8;
+	} else {
+		os_memcpy(crypt, phdr, hdrlen);
+		hdr = (struct ieee80211_hdr *) crypt;
+		hdr->frame_control |= host_to_le16(WLAN_FC_ISWEP);
+		pos = crypt + hdrlen;
+		*pos++ = pn[5]; /* PN0 */
+		*pos++ = pn[4]; /* PN1 */
+		*pos++ = 0x00; /* Rsvd */
+		*pos++ = 0x20 | (keyid << 6);
+		*pos++ = pn[3]; /* PN2 */
+		*pos++ = pn[2]; /* PN3 */
+		*pos++ = pn[1]; /* PN4 */
+		*pos++ = pn[0]; /* PN5 */
+	}
+
+	os_memset(aad, 0, sizeof(aad));
+	ccmp_aad_nonce(hdr, crypt + hdrlen, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 nonce", nonce, 13);
+
+	if (aes_ccm_ae(tk, 32, nonce, 16, pdata, datalen, aad, aad_len,
+		       pos, pos + datalen) < 0) {
+		rtw_mfree(crypt, enc_hdrlen + enc_datalen + AES_BLOCK_SIZE);
+		return;
+	}
+
+	wpa_hexdump(_MSG_EXCESSIVE_, "CCMP-256 encrypted", crypt + enc_hdrlen, datalen);
+
+	_rtw_memcpy(phdr, crypt, enc_hdrlen);
+	_rtw_memcpy(pdata, crypt + enc_hdrlen, enc_datalen);
+	
+	rtw_mfree(crypt, enc_hdrlen+enc_datalen+AES_BLOCK_SIZE);
+}
+
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/gcmp.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/gcmp.c
new file mode 100644
index 000000000000..9141d55202b0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/gcmp.c
@@ -0,0 +1,194 @@
+/*
+ * GCM with GMAC Protocol (GCMP)
+ * Copyright (c) 2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "aes.h"
+#include "aes_wrap.h"
+#include "wlancrypto_wrap.h"
+
+
+static void gcmp_aad_nonce(_adapter * padapter, const struct ieee80211_hdr *hdr, const u8 *data,
+			   u8 *aad, size_t *aad_len, u8 *nonce)
+{
+	u16 fc, stype, seq;
+	int qos = 0, addr4 = 0;
+	u8 *pos;
+
+	fc = le_to_host16(hdr->frame_control);
+	stype = WLAN_FC_GET_STYPE(fc);
+	if ((fc & (WLAN_FC_TODS | WLAN_FC_FROMDS)) ==
+	    (WLAN_FC_TODS | WLAN_FC_FROMDS))
+		addr4 = 1;
+
+	if (WLAN_FC_GET_TYPE(fc) == WLAN_FC_TYPE_DATA) {
+		fc &= ~0x0070; /* Mask subtype bits */
+		if (stype & WLAN_FC_STYPE_QOS_DATA) {
+			const u8 *qc;
+			qos = 1;
+			fc &= ~WLAN_FC_ORDER;
+			qc = (const u8 *)hdr + 24;
+			if (addr4)
+				qc += ETH_ALEN;
+		}
+	}
+
+	fc &= ~(WLAN_FC_RETRY | WLAN_FC_PWRMGT | WLAN_FC_MOREDATA);
+	WPA_PUT_LE16(aad, fc);
+	pos = aad + 2;
+	os_memcpy(pos, GetAddr1Ptr((u8 *)hdr), 3 * ETH_ALEN);
+	pos += 3 * ETH_ALEN;
+	seq = le_to_host16(hdr->seq_ctrl);
+	seq &= ~0xfff0; /* Mask Seq#; do not modify Frag# */
+	WPA_PUT_LE16(pos, seq);
+	pos += 2;
+
+	wpa_printf(_MSG_INFO_, "pos - aad = %u, qos(%d)\n", (pos - aad), qos);
+
+	os_memcpy(pos, (u8 *)hdr + 24, addr4 * ETH_ALEN + qos * 2);
+	pos += addr4 * ETH_ALEN;
+	if (qos) {
+		pos[0] &= ~0x70;
+		/* only spp mode need to refer QoS bit7 */
+		if (padapter->registrypriv.amsdu_mode != RTW_AMSDU_MODE_SPP)
+			pos[0] &= ~0x80;
+		pos++;
+		*pos++ = 0x00;
+	}
+
+	wpa_printf(_MSG_INFO_, "pos - aad = %u\n", (pos - aad));
+	*aad_len = pos - aad;
+
+	os_memcpy(nonce, hdr->addr2, ETH_ALEN);
+	nonce[6] = data[7]; /* PN5 */
+	nonce[7] = data[6]; /* PN4 */
+	nonce[8] = data[5]; /* PN3 */
+	nonce[9] = data[4]; /* PN2 */
+	nonce[10] = data[1]; /* PN1 */
+	nonce[11] = data[0]; /* PN0 */
+}
+
+/**
+ * gcmp_decrypt -
+ * @tk: the temporal key
+ * @tk_len: length of @tk
+ * @hdr: the mac header
+ * @data: payload after mac header (PN + enc_data + MIC)
+ * @data_len: length of @data (PN + enc_data + MIC)
+ * @decrypted_len: length of the data decrypted
+ */
+u8 * gcmp_decrypt(_adapter *padapter, const u8 *tk, size_t tk_len, const struct ieee80211_hdr *hdr,
+		  const u8 *data, size_t data_len, size_t *decrypted_len)
+{
+	u8 aad[30], nonce[12], *plain;
+	size_t aad_len, mlen;
+	const u8 *m;
+
+	if (data_len < 8 + 16)
+		return NULL;
+
+	plain = os_malloc(data_len + AES_BLOCK_SIZE);
+	if (plain == NULL)
+		return NULL;
+
+	m = data + 8;
+	mlen = data_len - 8 - 16;
+
+	os_memset(aad, 0, sizeof(aad));
+	gcmp_aad_nonce(padapter, hdr, data, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "GCMP AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "GCMP nonce", nonce, sizeof(nonce));
+
+	if (aes_gcm_ad(tk, tk_len, nonce, sizeof(nonce), m, mlen, aad, aad_len,
+		       m + mlen, plain) < 0) {
+		u16 seq_ctrl = le_to_host16(hdr->seq_ctrl);
+		wpa_printf(_MSG_INFO_, "Invalid GCMP frame: A1=" MACSTR
+			   " A2=" MACSTR " A3=" MACSTR " seq=%u frag=%u",
+			   MAC2STR(hdr->addr1), MAC2STR(hdr->addr2),
+			   MAC2STR(hdr->addr3),
+			   WLAN_GET_SEQ_SEQ(seq_ctrl),
+			   WLAN_GET_SEQ_FRAG(seq_ctrl));
+		rtw_mfree(plain, data_len + AES_BLOCK_SIZE);
+		return NULL;
+	}
+
+	*decrypted_len = mlen;
+	return plain;
+}
+
+/**
+ * gcmp_encrypt - 
+ * @tk: the temporal key
+ * @tk_len: length of @tk
+ * @frame: the point to mac header, the frame including mac header and payload, 
+ *         if @pn is NULL, then the frame including pn
+ * @len: length of @frame 
+ *         length = mac header + payload
+ * @hdrlen: length of the mac header
+ * @qos: pointer to the QOS field of the frame
+ * @pn: packet number
+ * @keyid: key id
+ * @encrypted_len: length of the encrypted frame 
+ *                 including mac header, pn, payload and MIC
+ */
+u8 * gcmp_encrypt(_adapter *padapter, const u8 *tk, size_t tk_len, const u8 *frame, size_t len,
+		  size_t hdrlen, const u8 *qos,
+		  const u8 *pn, int keyid, size_t *encrypted_len)
+{
+	u8 aad[30], nonce[12], *crypt, *pos;
+	const u8 *pdata;
+	size_t aad_len, plen;
+	struct ieee80211_hdr *hdr;
+
+	if (len < hdrlen || hdrlen < 24)
+		return NULL;
+	plen = len - hdrlen;
+
+	crypt = os_malloc(hdrlen + 8 + plen + 16 + AES_BLOCK_SIZE);
+	if (crypt == NULL)
+		return NULL;
+
+	if (pn == NULL) {
+		os_memcpy(crypt, frame, hdrlen + 8);
+		hdr = (struct ieee80211_hdr *)crypt;
+		pos = crypt + hdrlen + 8;
+		pdata = frame + hdrlen + 8;
+	} else {
+		os_memcpy(crypt, frame, hdrlen);
+		hdr = (struct ieee80211_hdr *)crypt;
+		pos = crypt + hdrlen;
+
+		*pos++ = pn[5]; /* PN0 */
+		*pos++ = pn[4]; /* PN1 */
+		*pos++ = 0x00; /* Rsvd */
+		*pos++ = 0x20 | (keyid << 6);
+		*pos++ = pn[3]; /* PN2 */
+		*pos++ = pn[2]; /* PN3 */
+		*pos++ = pn[1]; /* PN4 */
+		*pos++ = pn[0]; /* PN5 */
+		pdata = frame + hdrlen;
+	}
+
+	os_memset(aad, 0, sizeof(aad));
+	gcmp_aad_nonce(padapter, hdr, crypt + hdrlen, aad, &aad_len, nonce);
+	wpa_hexdump(_MSG_EXCESSIVE_, "GCMP AAD", aad, aad_len);
+	wpa_hexdump(_MSG_EXCESSIVE_, "GCMP nonce", nonce, sizeof(nonce));
+
+	if (aes_gcm_ae(tk, tk_len, nonce, sizeof(nonce), pdata, plen,
+			aad, aad_len, pos, pos + plen) < 0) {
+		rtw_mfree(crypt, hdrlen + 8 + plen + 16 + AES_BLOCK_SIZE);
+		return NULL;
+	}
+
+	wpa_hexdump(_MSG_EXCESSIVE_, "GCMP MIC", pos + plen, 16);
+	wpa_hexdump(_MSG_EXCESSIVE_, "GCMP encrypted", pos, plen);
+
+	*encrypted_len = hdrlen + 8 + plen + 16;
+
+	return crypt;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/rtw_crypto_wrap.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/rtw_crypto_wrap.c
new file mode 100644
index 000000000000..8fdb3c903bb7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/rtw_crypto_wrap.c
@@ -0,0 +1,85 @@
+#include "rtw_crypto_wrap.h"
+
+#ifndef DEBUG_CRYPTO
+#define DEBUG_CRYPTO 0
+#endif /* DEBUG_CRYTO */
+
+int os_memcmp(const void *s1, const void *s2, size_t n)
+{
+	return _rtw_memcmp2(s1, s2, n);
+}
+
+int os_memcmp_const(const void *a, const void *b, size_t len)
+{
+	const u8 *aa = a;
+	const u8 *bb = b;
+	size_t i;
+	u8 res;
+
+	for (res = 0, i = 0; i < len; i++)
+		res |= aa[i] ^ bb[i];
+
+	return res;
+}
+
+void* os_memdup(const void *src, u32 sz)
+{
+	void *r = rtw_malloc(sz);
+
+	if (r && src)
+		_rtw_memcpy(r, src, sz);
+	return r;
+}
+
+size_t os_strlen(const char *s)
+{
+	const char *p = s;
+	while (*p)
+		p++;
+	return p - s;
+}
+
+
+void forced_memzero(void *ptr, size_t len)
+{
+	_rtw_memset(ptr, 0, len);
+}
+
+void bin_clear_free(void *bin, size_t len)
+{
+	if (bin) {
+		forced_memzero(bin, len);
+		rtw_mfree(bin, len);
+	}
+}
+
+void wpa_printf(int level, const char *fmt, ...)
+{
+#if DEBUG_CRYPTO
+#define MSG_LEN 100
+	va_list args;
+	u8 buf[MSG_LEN] = { 0 };
+	int err;
+
+	va_start(args, fmt);
+	err = vsnprintf(buf, MSG_LEN, fmt, args);
+	va_end(args);
+
+	RTW_INFO("%s", buf);
+#undef MSG_LEN
+#endif /* DEBUG_CRYPTO */
+}
+
+void wpa_hexdump(int level, const char *title, const void *buf, size_t len)
+{
+#if DEBUG_CRYPTO
+	RTW_INFO_DUMP((u8 *)title, buf, len);
+#endif /* DEBUG_CRYPTO */
+}
+
+void wpa_hexdump_key(int level, const char *title, const void *buf, size_t len)
+{
+#if DEBUG_CRYPTO
+	RTW_INFO_DUMP((u8 *)title, buf, len);
+#endif /* DEBUG_CRYPTO */
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/rtw_crypto_wrap.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/rtw_crypto_wrap.h
new file mode 100644
index 000000000000..9b64a14194ef
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/rtw_crypto_wrap.h
@@ -0,0 +1,64 @@
+#ifndef RTW_CRYTO_WRAP_H
+#define RTW_CRYTO_WRAP_H
+
+#include <drv_types.h>
+
+#define TEST_FAIL() 0
+
+#define os_memset _rtw_memset
+#define os_memcpy _rtw_memcpy
+#define os_malloc rtw_malloc
+
+#define le_to_host16 le16_to_cpu
+#define host_to_le16 cpu_to_le16
+
+#define WPA_PUT_LE16 RTW_PUT_LE16
+#define WPA_GET_LE16 RTW_GET_LE16
+#define WPA_PUT_LE32 RTW_PUT_LE32
+#define WPA_GET_LE32 RTW_GET_LE32
+#define WPA_PUT_LE64 RTW_PUT_LE64
+#define WPA_GET_LE64 RTW_GET_LE64
+#define WPA_PUT_BE16 RTW_PUT_BE16
+#define WPA_GET_BE16 RTW_GET_BE16
+#define WPA_PUT_BE32 RTW_PUT_BE32
+#define WPA_GET_BE32 RTW_GET_BE32
+#define WPA_PUT_BE64 RTW_PUT_BE64
+#define WPA_GET_BE64 RTW_GET_BE64
+
+#ifndef MAC2STR
+#define MAC2STR(a) (a)[0], (a)[1], (a)[2], (a)[3], (a)[4], (a)[5]
+#define MACSTR "%02x:%02x:%02x:%02x:%02x:%02x"
+#endif
+
+#define WLAN_FC_PVER		0x0003
+#define WLAN_FC_TODS		0x0100
+#define WLAN_FC_FROMDS		0x0200
+#define WLAN_FC_MOREFRAG	0x0400
+#define WLAN_FC_RETRY		0x0800
+#define WLAN_FC_PWRMGT		0x1000
+#define WLAN_FC_MOREDATA	0x2000
+#define WLAN_FC_ISWEP		0x4000
+#define WLAN_FC_ORDER		0x8000
+
+#define WLAN_FC_TYPE_DATA RTW_IEEE80211_FTYPE_DATA
+#define WLAN_FC_TYPE_MGMT RTW_IEEE80211_FTYPE_MGMT
+
+#define WLAN_FC_STYPE_QOS_DATA RTW_IEEE80211_STYPE_QOS_DATA
+
+enum {
+	_MSG_EXCESSIVE_, _MSG_MSGDUMP_, _MSG_DEBUG_, _MSG_INFO_, _MSG_WARNING_, _MSG_ERROR_
+};
+
+int os_memcmp(const void *s1, const void *s2, size_t n);
+int os_memcmp_const(const void *a, const void *b, size_t len);
+void* os_memdup(const void *src, u32 sz);
+size_t os_strlen(const char *s);
+
+void forced_memzero(void *ptr, size_t len);
+void bin_clear_free(void *bin, size_t len);
+
+void wpa_printf(int level, const char *fmt, ...);
+void wpa_hexdump(int level, const char *title, const void *buf, size_t len);
+void wpa_hexdump_key(int level, const char *title, const void *buf, size_t len);
+
+#endif /* RTW_CRYTO_WRAP_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256-internal.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256-internal.c
new file mode 100644
index 000000000000..98228ea3378c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256-internal.c
@@ -0,0 +1,230 @@
+/*
+ * SHA-256 hash implementation and interface functions
+ * Copyright (c) 2003-2011, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+//#include "common.h"
+#include "sha256.h"
+#include "sha256_i.h"
+//#include "crypto.h"
+#include "wlancrypto_wrap.h"
+
+
+/**
+ * sha256_vector - SHA256 hash for data vector
+ * @num_elem: Number of elements in the data vector
+ * @addr: Pointers to the data areas
+ * @len: Lengths of the data blocks
+ * @mac: Buffer for the hash
+ * Returns: 0 on success, -1 of failure
+ */
+int sha256_vector(size_t num_elem, const u8 *addr[], const size_t *len,
+		  u8 *mac)
+{
+	struct _sha256_state ctx;
+	size_t i;
+
+	if (TEST_FAIL())
+		return -1;
+
+	_sha256_init(&ctx);
+	for (i = 0; i < num_elem; i++)
+		if (sha256_process(&ctx, addr[i], len[i]))
+			return -1;
+	if (sha256_done(&ctx, mac))
+		return -1;
+	return 0;
+}
+
+
+/* ===== start - public domain SHA256 implementation ===== */
+
+/* This is based on SHA256 implementation in LibTomCrypt that was released into
+ * public domain by Tom St Denis. */
+
+/* the K array */
+static const unsigned long K[64] = {
+	0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL,
+	0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL,
+	0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL,
+	0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL,
+	0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL,
+	0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL,
+	0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL,
+	0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL,
+	0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL,
+	0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL,
+	0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL,
+	0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL,
+	0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL
+};
+
+
+/* Various logical functions */
+#define RORc(x, y) \
+( ((((unsigned long) (x) & 0xFFFFFFFFUL) >> (unsigned long) ((y) & 31)) | \
+   ((unsigned long) (x) << (unsigned long) (32 - ((y) & 31)))) & 0xFFFFFFFFUL)
+#define Ch(x,y,z)       (z ^ (x & (y ^ z)))
+#define Maj(x,y,z)      (((x | y) & z) | (x & y))
+#define S(x, n)         RORc((x), (n))
+#define R(x, n)         (((x)&0xFFFFFFFFUL)>>(n))
+#define Sigma0(x)       (S(x, 2) ^ S(x, 13) ^ S(x, 22))
+#define Sigma1(x)       (S(x, 6) ^ S(x, 11) ^ S(x, 25))
+#define Gamma0(x)       (S(x, 7) ^ S(x, 18) ^ R(x, 3))
+#define Gamma1(x)       (S(x, 17) ^ S(x, 19) ^ R(x, 10))
+#ifndef MIN
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+#endif
+
+/* compress 512-bits */
+static int sha256_compress(struct _sha256_state *md, unsigned char *buf)
+{
+	u32 S[8], W[64], t0, t1;
+	u32 t;
+	int i;
+
+	/* copy state into S */
+	for (i = 0; i < 8; i++) {
+		S[i] = md->state[i];
+	}
+
+	/* copy the state into 512-bits into W[0..15] */
+	for (i = 0; i < 16; i++)
+		W[i] = WPA_GET_BE32(buf + (4 * i));
+
+	/* fill W[16..63] */
+	for (i = 16; i < 64; i++) {
+		W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) +
+			W[i - 16];
+	}
+
+	/* Compress */
+#define RND(a,b,c,d,e,f,g,h,i)                          \
+	t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i];	\
+	t1 = Sigma0(a) + Maj(a, b, c);			\
+	d += t0;					\
+	h  = t0 + t1;
+
+	for (i = 0; i < 64; ++i) {
+		RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i);
+		t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4];
+		S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t;
+	}
+
+	/* feedback */
+	for (i = 0; i < 8; i++) {
+		md->state[i] = md->state[i] + S[i];
+	}
+	return 0;
+}
+
+
+/* Initialize the hash state */
+void _sha256_init(struct _sha256_state *md)
+{
+	md->curlen = 0;
+	md->length = 0;
+	md->state[0] = 0x6A09E667UL;
+	md->state[1] = 0xBB67AE85UL;
+	md->state[2] = 0x3C6EF372UL;
+	md->state[3] = 0xA54FF53AUL;
+	md->state[4] = 0x510E527FUL;
+	md->state[5] = 0x9B05688CUL;
+	md->state[6] = 0x1F83D9ABUL;
+	md->state[7] = 0x5BE0CD19UL;
+}
+
+/**
+   Process a block of memory though the hash
+   @param md     The hash state
+   @param in     The data to hash
+   @param inlen  The length of the data (octets)
+   @return CRYPT_OK if successful
+*/
+int sha256_process(struct _sha256_state *md, const unsigned char *in,
+		   unsigned long inlen)
+{
+	unsigned long n;
+
+	if (md->curlen >= sizeof(md->buf))
+		return -1;
+
+	while (inlen > 0) {
+		if (md->curlen == 0 && inlen >= SHA256_BLOCK_SIZE) {
+			if (sha256_compress(md, (unsigned char *) in) < 0)
+				return -1;
+			md->length += SHA256_BLOCK_SIZE * 8;
+			in += SHA256_BLOCK_SIZE;
+			inlen -= SHA256_BLOCK_SIZE;
+		} else {
+			n = MIN(inlen, (SHA256_BLOCK_SIZE - md->curlen));
+			os_memcpy(md->buf + md->curlen, in, n);
+			md->curlen += n;
+			in += n;
+			inlen -= n;
+			if (md->curlen == SHA256_BLOCK_SIZE) {
+				if (sha256_compress(md, md->buf) < 0)
+					return -1;
+				md->length += 8 * SHA256_BLOCK_SIZE;
+				md->curlen = 0;
+			}
+		}
+	}
+
+	return 0;
+}
+
+
+/**
+   Terminate the hash to get the digest
+   @param md  The hash state
+   @param out [out] The destination of the hash (32 bytes)
+   @return CRYPT_OK if successful
+*/
+int sha256_done(struct _sha256_state *md, unsigned char *out)
+{
+	int i;
+
+	if (md->curlen >= sizeof(md->buf))
+		return -1;
+
+	/* increase the length of the message */
+	md->length += md->curlen * 8;
+
+	/* append the '1' bit */
+	md->buf[md->curlen++] = (unsigned char) 0x80;
+
+	/* if the length is currently above 56 bytes we append zeros
+	 * then compress.  Then we can fall back to padding zeros and length
+	 * encoding like normal.
+	 */
+	if (md->curlen > 56) {
+		while (md->curlen < SHA256_BLOCK_SIZE) {
+			md->buf[md->curlen++] = (unsigned char) 0;
+		}
+		sha256_compress(md, md->buf);
+		md->curlen = 0;
+	}
+
+	/* pad up to 56 bytes of zeroes */
+	while (md->curlen < 56) {
+		md->buf[md->curlen++] = (unsigned char) 0;
+	}
+
+	/* store length */
+	WPA_PUT_BE64(md->buf + 56, md->length);
+	sha256_compress(md, md->buf);
+
+	/* copy output */
+	for (i = 0; i < 8; i++)
+		WPA_PUT_BE32(out + (4 * i), md->state[i]);
+
+	return 0;
+}
+
+/* ===== end - public domain SHA256 implementation ===== */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256-prf.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256-prf.c
new file mode 100644
index 000000000000..642b38ff618c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256-prf.c
@@ -0,0 +1,109 @@
+/*
+ * SHA256-based PRF (IEEE 802.11r)
+ * Copyright (c) 2003-2016, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+//#include "common.h"
+#include "sha256.h"
+//#include "crypto.h"
+#include "wlancrypto_wrap.h"
+
+
+/**
+ * sha256_prf - SHA256-based Pseudo-Random Function (IEEE 802.11r, 8.5.1.5.2)
+ * @key: Key for PRF
+ * @key_len: Length of the key in bytes
+ * @label: A unique label for each purpose of the PRF
+ * @data: Extra data to bind into the key
+ * @data_len: Length of the data
+ * @buf: Buffer for the generated pseudo-random key
+ * @buf_len: Number of bytes of key to generate
+ * Returns: 0 on success, -1 on failure
+ *
+ * This function is used to derive new, cryptographically separate keys from a
+ * given key.
+ */
+int sha256_prf(const u8 *key, size_t key_len, const char *label,
+		const u8 *data, size_t data_len, u8 *buf, size_t buf_len)
+{
+	return sha256_prf_bits(key, key_len, label, data, data_len, buf,
+			       buf_len * 8);
+}
+
+
+/**
+ * sha256_prf_bits - IEEE Std 802.11-2012, 11.6.1.7.2 Key derivation function
+ * @key: Key for KDF
+ * @key_len: Length of the key in bytes
+ * @label: A unique label for each purpose of the PRF
+ * @data: Extra data to bind into the key
+ * @data_len: Length of the data
+ * @buf: Buffer for the generated pseudo-random key
+ * @buf_len: Number of bits of key to generate
+ * Returns: 0 on success, -1 on failure
+ *
+ * This function is used to derive new, cryptographically separate keys from a
+ * given key. If the requested buf_len is not divisible by eight, the least
+ * significant 1-7 bits of the last octet in the output are not part of the
+ * requested output.
+ */
+int sha256_prf_bits(const u8 *key, size_t key_len, const char *label,
+		    const u8 *data, size_t data_len, u8 *buf,
+		    size_t buf_len_bits)
+{
+	u16 counter = 1;
+	size_t pos, plen;
+	u8 hash[SHA256_MAC_LEN];
+	const u8 *addr[4];
+	size_t len[4];
+	u8 counter_le[2], length_le[2];
+	size_t buf_len = (buf_len_bits + 7) / 8;
+
+	addr[0] = counter_le;
+	len[0] = 2;
+	addr[1] = (u8 *) label;
+	len[1] = os_strlen(label);
+	addr[2] = data;
+	len[2] = data_len;
+	addr[3] = length_le;
+	len[3] = sizeof(length_le);
+
+	WPA_PUT_LE16(length_le, buf_len_bits);
+	pos = 0;
+	while (pos < buf_len) {
+		plen = buf_len - pos;
+		WPA_PUT_LE16(counter_le, counter);
+		if (plen >= SHA256_MAC_LEN) {
+			if (hmac_sha256_vector(key, key_len, 4, addr, len,
+					       &buf[pos]) < 0)
+				return -1;
+			pos += SHA256_MAC_LEN;
+		} else {
+			if (hmac_sha256_vector(key, key_len, 4, addr, len,
+					       hash) < 0)
+				return -1;
+			os_memcpy(&buf[pos], hash, plen);
+			pos += plen;
+			break;
+		}
+		counter++;
+	}
+
+	/*
+	 * Mask out unused bits in the last octet if it does not use all the
+	 * bits.
+	 */
+	if (buf_len_bits % 8) {
+		u8 mask = 0xff << (8 - buf_len_bits % 8);
+		buf[pos - 1] &= mask;
+	}
+
+	forced_memzero(hash, sizeof(hash));
+
+	return 0;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256.c b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256.c
new file mode 100644
index 000000000000..26ca0a6e171c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256.c
@@ -0,0 +1,104 @@
+/*
+ * SHA-256 hash implementation and interface functions
+ * Copyright (c) 2003-2012, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#include "rtw_crypto_wrap.h"
+
+#include "sha256.h"
+//#include "crypto.h"
+#include "wlancrypto_wrap.h"
+
+
+/**
+ * hmac_sha256_vector - HMAC-SHA256 over data vector (RFC 2104)
+ * @key: Key for HMAC operations
+ * @key_len: Length of the key in bytes
+ * @num_elem: Number of elements in the data vector
+ * @addr: Pointers to the data areas
+ * @len: Lengths of the data blocks
+ * @mac: Buffer for the hash (32 bytes)
+ * Returns: 0 on success, -1 on failure
+ */
+int hmac_sha256_vector(const u8 *key, size_t key_len, size_t num_elem,
+		       const u8 *addr[], const size_t *len, u8 *mac)
+{
+	unsigned char k_pad[64]; /* padding - key XORd with ipad/opad */
+	unsigned char tk[32];
+	const u8 *_addr[6];
+	size_t _len[6], i;
+
+	if (num_elem > 5) {
+		/*
+		 * Fixed limit on the number of fragments to avoid having to
+		 * allocate memory (which could fail).
+		 */
+		return -1;
+	}
+
+        /* if key is longer than 64 bytes reset it to key = SHA256(key) */
+        if (key_len > 64) {
+		if (sha256_vector(1, &key, &key_len, tk) < 0)
+			return -1;
+		key = tk;
+		key_len = 32;
+        }
+
+	/* the HMAC_SHA256 transform looks like:
+	 *
+	 * SHA256(K XOR opad, SHA256(K XOR ipad, text))
+	 *
+	 * where K is an n byte key
+	 * ipad is the byte 0x36 repeated 64 times
+	 * opad is the byte 0x5c repeated 64 times
+	 * and text is the data being protected */
+
+	/* start out by storing key in ipad */
+	os_memset(k_pad, 0, sizeof(k_pad));
+	os_memcpy(k_pad, key, key_len);
+	/* XOR key with ipad values */
+	for (i = 0; i < 64; i++)
+		k_pad[i] ^= 0x36;
+
+	/* perform inner SHA256 */
+	_addr[0] = k_pad;
+	_len[0] = 64;
+	for (i = 0; i < num_elem; i++) {
+		_addr[i + 1] = addr[i];
+		_len[i + 1] = len[i];
+	}
+	if (sha256_vector(1 + num_elem, _addr, _len, mac) < 0)
+		return -1;
+
+	os_memset(k_pad, 0, sizeof(k_pad));
+	os_memcpy(k_pad, key, key_len);
+	/* XOR key with opad values */
+	for (i = 0; i < 64; i++)
+		k_pad[i] ^= 0x5c;
+
+	/* perform outer SHA256 */
+	_addr[0] = k_pad;
+	_len[0] = 64;
+	_addr[1] = mac;
+	_len[1] = SHA256_MAC_LEN;
+	return sha256_vector(2, _addr, _len, mac);
+}
+
+
+/**
+ * hmac_sha256 - HMAC-SHA256 over data buffer (RFC 2104)
+ * @key: Key for HMAC operations
+ * @key_len: Length of the key in bytes
+ * @data: Pointers to the data area
+ * @data_len: Length of the data area
+ * @mac: Buffer for the hash (32 bytes)
+ * Returns: 0 on success, -1 on failure
+ */
+int _hmac_sha256(const u8 *key, size_t key_len, const u8 *data,
+		size_t data_len, u8 *mac)
+{
+	return hmac_sha256_vector(key, key_len, 1, &data, &data_len, mac);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256.h
new file mode 100644
index 000000000000..8f0f6f43ad10
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256.h
@@ -0,0 +1,30 @@
+/*
+ * SHA256 hash implementation and interface functions
+ * Copyright (c) 2003-2016, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef SHA256_H
+#define SHA256_H
+
+#define SHA256_MAC_LEN 32
+
+int hmac_sha256_vector(const u8 *key, size_t key_len, size_t num_elem,
+		       const u8 *addr[], const size_t *len, u8 *mac);
+int _hmac_sha256(const u8 *key, size_t key_len, const u8 *data,
+		size_t data_len, u8 *mac);
+int sha256_prf(const u8 *key, size_t key_len, const char *label,
+	       const u8 *data, size_t data_len, u8 *buf, size_t buf_len);
+int sha256_prf_bits(const u8 *key, size_t key_len, const char *label,
+		    const u8 *data, size_t data_len, u8 *buf,
+		    size_t buf_len_bits);
+void tls_prf_sha256(const u8 *secret, size_t secret_len,
+		    const char *label, const u8 *seed, size_t seed_len,
+		    u8 *out, size_t outlen);
+int hmac_sha256_kdf(const u8 *secret, size_t secret_len,
+		    const char *label, const u8 *seed, size_t seed_len,
+		    u8 *out, size_t outlen);
+
+#endif /* SHA256_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256_i.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256_i.h
new file mode 100644
index 000000000000..11ddd6b14716
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/sha256_i.h
@@ -0,0 +1,25 @@
+/*
+ * SHA-256 internal definitions
+ * Copyright (c) 2003-2011, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef SHA256_I_H
+#define SHA256_I_H
+
+#define SHA256_BLOCK_SIZE 64
+
+struct _sha256_state {
+	u64 length;
+	u32 state[8], curlen;
+	u8 buf[SHA256_BLOCK_SIZE];
+};
+
+void _sha256_init(struct _sha256_state *md);
+int sha256_process(struct _sha256_state *md, const unsigned char *in,
+		   unsigned long inlen);
+int sha256_done(struct _sha256_state *md, unsigned char *out);
+
+#endif /* SHA256_I_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/crypto/wlancrypto_wrap.h b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/wlancrypto_wrap.h
new file mode 100644
index 000000000000..d39ce99e5ed1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/crypto/wlancrypto_wrap.h
@@ -0,0 +1,41 @@
+/*
+ * wlantest - IEEE 802.11 protocol monitoring and testing tool
+ * Copyright (c) 2010-2013, Jouni Malinen <j@w1.fi>
+ *
+ * This software may be distributed under the terms of the BSD license.
+ * See README for more details.
+ */
+
+#ifndef WLANCRYPTO_WRAP_H
+#define WLANCRYPTO_WRAP_H
+
+int sha256_vector(size_t num_elem, const u8 *addr[], const size_t *len,
+	u8 *mac);
+
+u8* ccmp_decrypt(_adapter *padapter, const u8 *tk, const struct ieee80211_hdr *hdr,
+	const u8 *data, size_t data_len, size_t *decrypted_len);
+u8* ccmp_encrypt(_adapter *padapter, const u8 *tk, u8 *frame, size_t len, size_t hdrlen, u8 *qos,
+	u8 *pn, int keyid, size_t *encrypted_len);
+u8* ccmp_encrypt_pv1(const u8 *tk, const u8 *a1, const u8 *a2, const u8 *a3,
+	const u8 *frame, size_t len,
+	size_t hdrlen, const u8 *pn, int keyid,
+	size_t *encrypted_len);
+u8* ccmp_256_decrypt(_adapter *padapter, const u8 *tk, const struct ieee80211_hdr *hdr,
+	const u8 *data, size_t data_len, size_t *decrypted_len);
+u8* ccmp_256_encrypt(_adapter *padapter, const u8 *tk, u8 *frame, size_t len, size_t hdrlen,
+	u8 *qos, u8 *pn, int keyid, size_t *encrypted_len);
+
+u8* gcmp_decrypt(_adapter *padapter, const u8 *tk, size_t tk_len, const struct ieee80211_hdr *hdr,
+	const u8 *data, size_t data_len, size_t *decrypted_len);
+u8* gcmp_encrypt(_adapter *padapter, const u8 *tk, size_t tk_len, const u8 *frame, size_t len,
+	size_t hdrlen, const u8 *qos,
+	const u8 *pn, int keyid, size_t *encrypted_len);
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+void core_ccmp_encrypt(const u8 *tk, uint hdrlen, u8 *phdr, uint datalen, u8 *pdata, 
+	u8 *qos, u8 *pn, int keyid, size_t *encrypted_len);
+
+void core_ccmp_256_encrypt(const u8 *tk, uint hdrlen, u8 *phdr, uint datalen, u8 *pdata, 
+		      u8 *qos, u8 *pn, int keyid, size_t *encrypted_len);
+#endif
+#endif /* WLANCRYPTO_WRAP_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/def_module_country_chplan.h b/drivers/net/wireless/realtek/rtl8852bs/core/def_module_country_chplan.h
new file mode 100644
index 000000000000..b993d9e4bd8d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/def_module_country_chplan.h
@@ -0,0 +1,1519 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#define RTW_MODULE_RTL8821AE_HMC_M2		BIT0	/* RTL8821AE(HMC + M.2) */
+#define RTW_MODULE_RTL8821AU			BIT1	/* RTL8821AU */
+#define RTW_MODULE_RTL8812AENF_NGFF		BIT2	/* RTL8812AENF(8812AE+8761)_NGFF */
+#define RTW_MODULE_RTL8812AEBT_HMC		BIT3	/* RTL8812AEBT(8812AE+8761)_HMC */
+#define RTW_MODULE_RTL8188EE_HMC_M2		BIT4	/* RTL8188EE(HMC + M.2) */
+#define RTW_MODULE_RTL8723BE_HMC_M2		BIT5	/* RTL8723BE(HMC + M.2) */
+#define RTW_MODULE_RTL8723BS_NGFF1216	BIT6	/* RTL8723BS(NGFF1216) */
+#define RTW_MODULE_RTL8192EEBT_HMC_M2	BIT7	/* RTL8192EEBT(8192EE+8761AU)_(HMC + M.2) */
+#define RTW_MODULE_RTL8723DE_NGFF1630	BIT8	/* RTL8723DE(NGFF1630) */
+#define RTW_MODULE_RTL8822BE			BIT9	/* RTL8822BE */
+#define RTW_MODULE_RTL8821CE			BIT10	/* RTL8821CE */
+#define RTW_MODULE_RTL8822CE			BIT11	/* RTL8822CE */
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8821AE_HMC_M2) /* 2013 certify */
+static const struct country_chplan RTL8821AE_HMC_M2_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x3D, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x36, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8821AU) /* 2014 certify */
+static const struct country_chplan RTL8821AU_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x3D, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x36, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8812AENF_NGFF) /* 2014 certify */
+static const struct country_chplan RTL8812AENF_NGFF_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8812AEBT_HMC) /* 2013 certify */
+static const struct country_chplan RTL8812AEBT_HMC_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x36, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8188EE_HMC_M2) /* 2012 certify */
+static const struct country_chplan RTL8188EE_HMC_M2_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AW", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BB", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x20, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ET", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HT", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ML", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NP", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UZ", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8723BE_HMC_M2) /* 2013 certify */
+static const struct country_chplan RTL8723BE_HMC_M2_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AO", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AW", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BS", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x20, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ET", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ML", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NP", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UZ", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8723BS_NGFF1216) /* 2014 certify */
+static const struct country_chplan RTL8723BS_NGFF1216_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AO", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BB", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x20, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HT", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NP", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UZ", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("YE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8192EEBT_HMC_M2) /* 2013 certify */
+static const struct country_chplan RTL8192EEBT_HMC_M2_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AO", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AW", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x20, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EH", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ET", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ML", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NP", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SC", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ST", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x39, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UZ", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("YT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8723DE_NGFF1630) /* 2016 certify */
+static const struct country_chplan RTL8723DE_NGFF1630_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x2A, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x34, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8822BE) /* 2016 certify */
+static const struct country_chplan RTL8822BE_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AO", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EH", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GP", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x3D, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ML", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MO", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NP", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, UKRAINE, 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UZ", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("YT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8821CE) /* 2016 certify */
+static const struct country_chplan RTL8821CE_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AO", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EH", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ET", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GP", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ID", 0x3D, 0x00, DEF    , 0, 0, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x28, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ML", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MO", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NP", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, UKRAINE, 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UZ", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("YT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+#if (RTW_DEF_MODULE_REGULATORY_CERT & RTW_MODULE_RTL8822CE) /* 2018 certify */
+static const struct country_chplan RTL8822CE_country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AE", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AO", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AR", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AU", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("AZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BH", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BO", 0x73, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BW", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("BY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CL", 0x2D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CN", 0x48, 0x00, CN     , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CY", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("CZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EG", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("EH", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ES", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ET", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("FR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GP", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GT", 0x61, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("GW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HK", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("HU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IL", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IN", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("IT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JM", 0x32, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JO", 0x49, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("JP", 0x27, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KE", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KH", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("KZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LB", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("LV", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MA", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MC", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ME", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ML", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MO", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MQ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MU", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MW", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MY", 0x63, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("MZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NA", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NP", 0x48, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("NZ", 0x45, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("OM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("QA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RO", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RS", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("RU", 0x59, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SE", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SG", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SI", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SK", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SL", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SN", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SV", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("SZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TD", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TF", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TH", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TJ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TN", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TR", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("TZ", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UA", 0x35, 0x00, UKRAINE, 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UG", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("US", 0x76, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("UZ", 0x47, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("VN", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("YT", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZA", 0x35, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZM", 0x26, 0x00, DEF    , 0, 1, 1),
+	COUNTRY_CHPLAN_ENT("ZW", 0x26, 0x00, DEF    , 0, 1, 1),
+};
+#endif
+
+/**
+ * rtw_def_module_country_chplan_map -
+ * @hal_map: returned map
+ * @return: size of map
+ */
+static u16 rtw_def_module_country_chplan_map(const struct country_chplan **hal_map)
+{
+	u16 hal_map_sz = 0;
+
+	/* TODO: runtime selection for multi driver */
+#if (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8821AE_HMC_M2)
+	*hal_map = RTL8821AE_HMC_M2_country_chplan_map;
+	hal_map_sz = sizeof(RTL8821AE_HMC_M2_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8821AU)
+	*hal_map = RTL8821AU_country_chplan_map;
+	hal_map_sz = sizeof(RTL8821AU_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8812AENF_NGFF)
+	*hal_map = RTL8812AENF_NGFF_country_chplan_map;
+	hal_map_sz = sizeof(RTL8812AENF_NGFF_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8812AEBT_HMC)
+	*hal_map = RTL8812AEBT_HMC_country_chplan_map;
+	hal_map_sz = sizeof(RTL8812AEBT_HMC_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8188EE_HMC_M2)
+	*hal_map = RTL8188EE_HMC_M2_country_chplan_map;
+	hal_map_sz = sizeof(RTL8188EE_HMC_M2_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8723BE_HMC_M2)
+	*hal_map = RTL8723BE_HMC_M2_country_chplan_map;
+	hal_map_sz = sizeof(RTL8723BE_HMC_M2_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8723BS_NGFF1216)
+	*hal_map = RTL8723BS_NGFF1216_country_chplan_map;
+	hal_map_sz = sizeof(RTL8723BS_NGFF1216_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8192EEBT_HMC_M2)
+	*hal_map = RTL8192EEBT_HMC_M2_country_chplan_map;
+	hal_map_sz = sizeof(RTL8192EEBT_HMC_M2_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8723DE_NGFF1630)
+	*hal_map = RTL8723DE_NGFF1630_country_chplan_map;
+	hal_map_sz = sizeof(RTL8723DE_NGFF1630_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8822BE)
+	*hal_map = RTL8822BE_country_chplan_map;
+	hal_map_sz = sizeof(RTL8822BE_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8821CE)
+	*hal_map = RTL8821CE_country_chplan_map;
+	hal_map_sz = sizeof(RTL8821CE_country_chplan_map) / sizeof(struct country_chplan);
+#elif (RTW_DEF_MODULE_REGULATORY_CERT == RTW_MODULE_RTL8822CE)
+	*hal_map = RTL8822CE_country_chplan_map;
+	hal_map_sz = sizeof(RTL8822CE_country_chplan_map) / sizeof(struct country_chplan);
+#endif
+
+	return hal_map_sz;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh.c b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh.c
new file mode 100644
index 000000000000..c213adb81b6c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh.c
@@ -0,0 +1,4343 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_MESH_C_
+
+#ifdef CONFIG_RTW_MESH
+#include <drv_types.h>
+
+const char *_rtw_mesh_plink_str[] = {
+	"UNKNOWN",
+	"LISTEN",
+	"OPN_SNT",
+	"OPN_RCVD",
+	"CNF_RCVD",
+	"ESTAB",
+	"HOLDING",
+	"BLOCKED",
+};
+
+const char *_rtw_mesh_ps_str[] = {
+	"UNKNOWN",
+	"ACTIVE",
+	"LSLEEP",
+	"DSLEEP",
+};
+
+const char *_action_self_protected_str[] = {
+	"ACT_SELF_PROTECTED_RSVD",
+	"MESH_OPEN",
+	"MESH_CONF",
+	"MESH_CLOSE",
+	"MESH_GK_INFORM",
+	"MESH_GK_ACK",
+};
+
+inline u8 *rtw_set_ie_mesh_id(u8 *buf, u32 *buf_len, const char *mesh_id, u8 id_len)
+{
+	return rtw_set_ie(buf, WLAN_EID_MESH_ID, id_len, mesh_id, buf_len);
+}
+
+inline u8 *rtw_set_ie_mesh_config(u8 *buf, u32 *buf_len
+	, u8 path_sel_proto, u8 path_sel_metric, u8 congest_ctl_mode, u8 sync_method, u8 auth_proto
+	, u8 num_of_peerings, bool cto_mgate, bool cto_as
+	, bool accept_peerings, bool mcca_sup, bool mcca_en, bool forwarding
+	, bool mbca_en, bool tbtt_adj, bool ps_level)
+{
+
+	u8 conf[7] = {0};
+
+	SET_MESH_CONF_ELE_PATH_SEL_PROTO_ID(conf, path_sel_proto);
+	SET_MESH_CONF_ELE_PATH_SEL_METRIC_ID(conf, path_sel_metric);
+	SET_MESH_CONF_ELE_CONGEST_CTRL_MODE_ID(conf, congest_ctl_mode);
+	SET_MESH_CONF_ELE_SYNC_METHOD_ID(conf, sync_method);
+	SET_MESH_CONF_ELE_AUTH_PROTO_ID(conf, auth_proto);
+
+	SET_MESH_CONF_ELE_CTO_MGATE(conf, cto_mgate);
+	SET_MESH_CONF_ELE_NUM_OF_PEERINGS(conf, num_of_peerings);
+	SET_MESH_CONF_ELE_CTO_AS(conf, cto_as);
+
+	SET_MESH_CONF_ELE_ACCEPT_PEERINGS(conf, accept_peerings);
+	SET_MESH_CONF_ELE_MCCA_SUP(conf, mcca_sup);
+	SET_MESH_CONF_ELE_MCCA_EN(conf, mcca_en);
+	SET_MESH_CONF_ELE_FORWARDING(conf, forwarding);
+	SET_MESH_CONF_ELE_MBCA_EN(conf, mbca_en);
+	SET_MESH_CONF_ELE_TBTT_ADJ(conf, tbtt_adj);
+	SET_MESH_CONF_ELE_PS_LEVEL(conf, ps_level);
+
+	return rtw_set_ie(buf, WLAN_EID_MESH_CONFIG, 7, conf, buf_len);
+}
+
+inline u8 *rtw_set_ie_mpm(u8 *buf, u32 *buf_len
+	, u8 proto_id, u16 llid, u16 *plid, u16 *reason, u8 *chosen_pmk)
+{
+	u8 data[24] = {0};
+	u8 *pos = data;
+
+	RTW_PUT_LE16(pos, proto_id);
+	pos += 2;
+
+	RTW_PUT_LE16(pos, llid);
+	pos += 2;
+
+	if (plid) {
+		RTW_PUT_LE16(pos, *plid);
+		pos += 2;
+	}
+
+	if (reason) {
+		RTW_PUT_LE16(pos, *reason);
+		pos += 2;
+	}
+
+	if (chosen_pmk) {
+		_rtw_memcpy(pos, chosen_pmk, 16);
+		pos += 16;
+	}
+
+	return rtw_set_ie(buf, WLAN_EID_MPM, pos - data, data, buf_len);
+}
+
+bool rtw_bss_is_forwarding(WLAN_BSSID_EX *bss)
+{
+	u8 *ie;
+	int ie_len;
+	bool ret = 0;
+
+	ie = rtw_get_ie(BSS_EX_TLV_IES(bss), WLAN_EID_MESH_CONFIG, &ie_len,
+			BSS_EX_TLV_IES_LEN(bss));
+	if (!ie || ie_len != 7)
+		goto exit;
+
+	ret = GET_MESH_CONF_ELE_FORWARDING(ie + 2);
+
+exit:
+	return ret;
+}
+
+bool rtw_bss_is_cto_mgate(WLAN_BSSID_EX *bss)
+{
+	u8 *ie;
+	int ie_len;
+	bool ret = 0;
+
+	ie = rtw_get_ie(BSS_EX_TLV_IES(bss), WLAN_EID_MESH_CONFIG, &ie_len,
+			BSS_EX_TLV_IES_LEN(bss));
+	if (!ie || ie_len != 7)
+		goto exit;
+
+	ret = GET_MESH_CONF_ELE_CTO_MGATE(ie + 2);
+
+exit:
+	return ret;
+}
+
+int _rtw_bss_is_same_mbss(WLAN_BSSID_EX *a, WLAN_BSSID_EX *b, u8 **a_mconf_ie_r, u8 **b_mconf_ie_r)
+{
+	int ret = 0;
+	u8 *a_mconf_ie, *b_mconf_ie;
+	sint a_mconf_ie_len, b_mconf_ie_len;
+
+	if (a->InfrastructureMode != Ndis802_11_mesh)
+		goto exit;
+	a_mconf_ie = rtw_get_ie(BSS_EX_TLV_IES(a), WLAN_EID_MESH_CONFIG, &a_mconf_ie_len, BSS_EX_TLV_IES_LEN(a));
+	if (!a_mconf_ie || a_mconf_ie_len != 7)
+		goto exit;
+	if (a_mconf_ie_r)
+		*a_mconf_ie_r = a_mconf_ie;
+
+	if (b->InfrastructureMode != Ndis802_11_mesh)
+		goto exit;
+	b_mconf_ie = rtw_get_ie(BSS_EX_TLV_IES(b), WLAN_EID_MESH_CONFIG, &b_mconf_ie_len, BSS_EX_TLV_IES_LEN(b));
+	if (!b_mconf_ie || b_mconf_ie_len != 7)
+		goto exit;
+	if (b_mconf_ie_r)
+		*b_mconf_ie_r = b_mconf_ie;
+
+	if (a->mesh_id.SsidLength != b->mesh_id.SsidLength
+		|| _rtw_memcmp(a->mesh_id.Ssid, b->mesh_id.Ssid, a->mesh_id.SsidLength) == _FALSE)
+		goto exit;
+
+	if (_rtw_memcmp(a_mconf_ie + 2, b_mconf_ie + 2, 5) == _FALSE)
+		goto exit;
+
+	ret = 1;
+
+exit:
+	return ret;
+}
+
+int rtw_bss_is_same_mbss(WLAN_BSSID_EX *a, WLAN_BSSID_EX *b)
+{
+	return _rtw_bss_is_same_mbss(a, b, NULL, NULL);
+}
+
+int rtw_bss_is_candidate_mesh_peer(_adapter *adapter, WLAN_BSSID_EX *target, u8 ch, u8 add_peer)
+{
+	int ret = 0;
+	WLAN_BSSID_EX *self = &adapter->mlmepriv.cur_network.network;
+	u8 *s_mconf_ie, *t_mconf_ie;
+	u8 auth_pid;
+	int i, j;
+
+	if (ch && self->Configuration.DSConfig != target->Configuration.DSConfig)
+		goto exit;
+
+	if (!_rtw_bss_is_same_mbss(self, target, &s_mconf_ie, &t_mconf_ie))
+		goto exit;
+
+	if (add_peer) {
+		/* Accept additional mesh peerings */
+		if (GET_MESH_CONF_ELE_ACCEPT_PEERINGS(t_mconf_ie + 2) == 0)
+			goto exit;
+	}
+
+	/* BSSBasicRateSet */
+	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
+		if (target->SupportedRates[i] == 0)
+			break;	
+		if (target->SupportedRates[i] & 0x80) {
+			u8 match = 0;
+
+			if (!ch) {
+				/* off-channel, check target with our hardcode capability */
+				if (BSS_EX_OP_BAND(target) != BAND_ON_24G)
+					match = rtw_is_basic_rate_ofdm(target->SupportedRates[i]);
+				else
+					match = rtw_is_basic_rate_mix(target->SupportedRates[i]);
+			} else { 
+				for (j = 0; j < NDIS_802_11_LENGTH_RATES_EX; j++) {
+					if (self->SupportedRates[j] == 0)
+						break;
+					if (self->SupportedRates[j] == target->SupportedRates[i]) {
+						match = 1;
+						break;
+					}
+				}
+			}
+			if (!match)
+				goto exit;
+		}
+	}
+
+	/* BSSBasicMCSSet */
+
+
+	auth_pid = GET_MESH_CONF_ELE_AUTH_PROTO_ID(s_mconf_ie + 2);
+	if (auth_pid && auth_pid <= 2) {
+		struct security_priv *sec = &adapter->securitypriv;
+		u8 *rsn_ie;
+		int rsn_ie_len;
+		int group_cipher = 0, pairwise_cipher = 0, gmcs = 0;
+		u8 mfp_opt = MFP_NO;
+
+		/* 802.1X connected to AS ? */
+
+		/* RSN */
+		rsn_ie = rtw_get_wpa2_ie(BSS_EX_TLV_IES(target), &rsn_ie_len, BSS_EX_TLV_IES_LEN(target));
+		if (!rsn_ie || rsn_ie_len == 0)
+			goto exit;
+		if (rtw_parse_wpa2_ie(rsn_ie, rsn_ie_len + 2, &group_cipher, &pairwise_cipher, &gmcs, NULL, &mfp_opt, NULL) != _SUCCESS)
+			goto exit;
+		if ((sec->mfp_opt == MFP_REQUIRED && mfp_opt < MFP_OPTIONAL)
+			|| (mfp_opt == MFP_REQUIRED && sec->mfp_opt < MFP_OPTIONAL))
+			goto exit;
+		if (!(sec->wpa2_group_cipher & group_cipher))
+			goto exit;
+		if (!(sec->wpa2_pairwise_cipher & pairwise_cipher))
+			goto exit;
+		#ifdef CONFIG_IEEE80211W
+		if ((sec->mfp_opt >= MFP_OPTIONAL && mfp_opt >= MFP_OPTIONAL)
+			&& security_type_bip_to_gmcs(sec->dot11wCipher) != gmcs)
+			goto exit;
+		#endif
+	}
+
+	ret = 1;
+
+exit:
+	return ret;
+}
+
+void rtw_mesh_bss_peering_status(WLAN_BSSID_EX *bss, u8 *nop, u8 *accept)
+{
+	u8 *ie;
+	int ie_len;
+
+	if (nop)
+		*nop = 0;
+	if (accept)
+		*accept = 0;
+
+	ie = rtw_get_ie(BSS_EX_TLV_IES(bss), WLAN_EID_MESH_CONFIG, &ie_len,
+			BSS_EX_TLV_IES_LEN(bss));
+	if (!ie || ie_len != 7)
+		goto exit;
+
+	if (nop)
+		*nop = GET_MESH_CONF_ELE_NUM_OF_PEERINGS(ie + 2);
+	if (accept)
+		*accept = GET_MESH_CONF_ELE_ACCEPT_PEERINGS(ie + 2);
+
+exit:
+	return;
+}
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+void rtw_mesh_update_scanned_acnode_status(_adapter *adapter, struct wlan_network *scanned)
+{
+	bool acnode;
+	u8 nop, accept;
+
+	rtw_mesh_bss_peering_status(&scanned->network, &nop, &accept);
+
+	acnode = !nop && accept;
+
+	if (acnode && scanned->acnode_stime == 0) {
+		scanned->acnode_stime = rtw_get_current_time();
+		if (scanned->acnode_stime == 0)
+			scanned->acnode_stime++;
+	} else if (!acnode) {
+		scanned->acnode_stime = 0;
+		scanned->acnode_notify_etime = 0;
+	}
+}
+
+bool rtw_mesh_scanned_is_acnode_confirmed(_adapter *adapter, struct wlan_network *scanned)
+{
+	return scanned->acnode_stime
+			&& rtw_get_passing_time_ms(scanned->acnode_stime)
+				> adapter->mesh_cfg.peer_sel_policy.acnode_conf_timeout_ms;
+}
+
+static bool rtw_mesh_scanned_is_acnode_allow_notify(_adapter *adapter, struct wlan_network *scanned)
+{
+	return scanned->acnode_notify_etime
+			&& rtw_time_after(scanned->acnode_notify_etime, rtw_get_current_time());
+}
+
+bool rtw_mesh_acnode_prevent_allow_sacrifice(_adapter *adapter)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	bool allow = 0;
+
+	if (!mcfg->peer_sel_policy.acnode_prevent
+		|| mcfg->max_peer_links <= 1
+		|| stapriv->asoc_list_cnt < mcfg->max_peer_links)
+		goto exit;
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	if (rtw_mesh_cto_mgate_required(adapter))
+		goto exit;
+#endif
+
+	allow = 1;
+
+exit:
+	return allow;
+}
+
+static bool rtw_mesh_acnode_candidate_exist(_adapter *adapter)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	_queue *queue = &(mlme->scanned_queue);
+	_list *head, *list;
+	struct wlan_network *scanned = NULL;
+	struct sta_info *sta = NULL;
+	bool need = 0;
+
+	_rtw_spinlock_bh(&(mlme->scanned_queue.lock));
+
+	head = get_list_head(queue);
+	list = get_next(head);
+	while (!rtw_end_of_queue_search(head, list)) {
+		scanned = LIST_CONTAINOR(list, struct wlan_network, list);
+		list = get_next(list);
+
+		if (rtw_get_passing_time_ms(scanned->last_scanned) < mcfg->peer_sel_policy.scanr_exp_ms
+			&& rtw_mesh_scanned_is_acnode_confirmed(adapter, scanned)
+			&& (!mcfg->rssi_threshold || mcfg->rssi_threshold <= scanned->network.PhyInfo.rssi)
+			#if CONFIG_RTW_MACADDR_ACL
+			&& rtw_access_ctrl(adapter, scanned->network.MacAddress) == _TRUE
+			#endif
+			&& rtw_bss_is_candidate_mesh_peer(adapter, &scanned->network, 1, 1)
+			#if CONFIG_RTW_MESH_PEER_BLACKLIST
+			&& !rtw_mesh_peer_blacklist_search(adapter, scanned->network.MacAddress)
+			#endif
+			#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+			&& rtw_mesh_cto_mgate_network_filter(adapter, scanned)
+			#endif
+		) {
+			need = 1;
+			break;
+		}
+	}
+
+	_rtw_spinunlock_bh(&(mlme->scanned_queue.lock));
+
+	return need;
+}
+
+static int rtw_mesh_acnode_prevent_sacrifice_chk(_adapter *adapter, struct sta_info **sac, struct sta_info *com)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	int updated = 0;
+
+	/*
+	* TODO: compare next_hop reference cnt of forwarding info
+	* don't sacrifice working next_hop or choose sta with least cnt
+	*/
+
+	if (*sac == NULL) {
+		updated = 1;
+		goto exit;
+	}
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	if (mcfg->peer_sel_policy.cto_mgate_require
+		&& !mcfg->dot11MeshGateAnnouncementProtocol
+	) {
+		if (IS_CTO_MGATE_CONF_TIMEOUT(com->plink)) {
+			if (!IS_CTO_MGATE_CONF_TIMEOUT((*sac)->plink)) {
+				/* blacklist > not blacklist */
+				updated = 1;
+				goto exit;
+			}
+		} else if (!IS_CTO_MGATE_CONF_DISABLED(com->plink)) {
+			if (IS_CTO_MGATE_CONF_DISABLED((*sac)->plink)) {
+				/* confirming > disabled */
+				updated = 1;
+				goto exit;
+			}
+		}
+	}
+#endif
+
+exit:
+	if (updated)
+		*sac = com;
+
+	return updated;
+}
+
+struct sta_info *_rtw_mesh_acnode_prevent_pick_sacrifice(_adapter *adapter)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	_list *head, *list;
+	struct sta_info *sta, *sacrifice = NULL;
+	u8 nop;
+
+	head = &stapriv->asoc_list;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		sta = LIST_CONTAINOR(list, struct sta_info, asoc_list);
+		list = get_next(list);
+
+		if (!sta->plink || !sta->plink->scanned) {
+			rtw_warn_on(1);
+			continue;
+		}
+
+		rtw_mesh_bss_peering_status(&sta->plink->scanned->network, &nop, NULL);
+		if (nop < 2)
+			continue;
+
+		rtw_mesh_acnode_prevent_sacrifice_chk(adapter, &sacrifice, sta);
+	}
+
+	return sacrifice;
+}
+
+struct sta_info *rtw_mesh_acnode_prevent_pick_sacrifice(_adapter *adapter)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct sta_info *sacrifice = NULL;
+
+	rtw_stapriv_asoc_list_lock(stapriv);
+
+	sacrifice = _rtw_mesh_acnode_prevent_pick_sacrifice(adapter);
+
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+	return sacrifice;
+}
+
+static void rtw_mesh_acnode_rsvd_chk(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	u8 acnode_rsvd = 0;
+
+	if (rtw_mesh_acnode_prevent_allow_sacrifice(adapter)
+		&& rtw_mesh_acnode_prevent_pick_sacrifice(adapter)
+		&& rtw_mesh_acnode_candidate_exist(adapter))
+		acnode_rsvd = 1;
+
+	if (plink_ctl->acnode_rsvd != acnode_rsvd) {
+		plink_ctl->acnode_rsvd = acnode_rsvd;
+		RTW_INFO(FUNC_ADPT_FMT" acnode_rsvd = %d\n", FUNC_ADPT_ARG(adapter), plink_ctl->acnode_rsvd);
+		rtw_update_beacon(adapter, WLAN_EID_MESH_CONFIG, NULL, 1, 0);
+	}
+}
+
+static void rtw_mesh_acnode_set_notify_etime(_adapter *adapter, u8 *rframe_whdr)
+{
+	if (adapter->mesh_info.plink_ctl.acnode_rsvd) {
+		struct wlan_network *scanned = rtw_find_network(&adapter->mlmepriv.scanned_queue, get_addr2_ptr(rframe_whdr));
+
+		if (rtw_mesh_scanned_is_acnode_confirmed(adapter, scanned)) {
+			scanned->acnode_notify_etime = rtw_get_current_time()
+				+ rtw_ms_to_systime(adapter->mesh_cfg.peer_sel_policy.acnode_notify_timeout_ms);
+			if (scanned->acnode_notify_etime == 0)
+				scanned->acnode_notify_etime++;
+		}
+	}
+}
+
+void dump_mesh_acnode_prevent_settings(void *sel, _adapter *adapter)
+{
+	struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+
+	RTW_PRINT_SEL(sel, "%-6s %-12s %-14s\n"
+		, "enable", "conf_timeout", "nofity_timeout");
+	RTW_PRINT_SEL(sel, "%6u %12u %14u\n"
+		, peer_sel_policy->acnode_prevent
+		, peer_sel_policy->acnode_conf_timeout_ms
+		, peer_sel_policy->acnode_notify_timeout_ms);
+}
+#endif /* CONFIG_RTW_MESH_ACNODE_PREVENT */
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+int rtw_mesh_peer_blacklist_add(_adapter *adapter, const u8 *addr)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	return rtw_blacklist_add(&plink_ctl->peer_blacklist, addr
+		, mcfg->peer_sel_policy.peer_blacklist_timeout_ms);
+}
+
+int rtw_mesh_peer_blacklist_del(_adapter *adapter, const u8 *addr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	return rtw_blacklist_del(&plink_ctl->peer_blacklist, addr);
+}
+
+int rtw_mesh_peer_blacklist_search(_adapter *adapter, const u8 *addr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	return rtw_blacklist_search(&plink_ctl->peer_blacklist, addr);
+}
+
+void rtw_mesh_peer_blacklist_flush(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	rtw_blacklist_flush(&plink_ctl->peer_blacklist);
+}
+
+void dump_mesh_peer_blacklist(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	dump_blacklist(sel, &plink_ctl->peer_blacklist, "blacklist");
+}
+
+void dump_mesh_peer_blacklist_settings(void *sel, _adapter *adapter)
+{
+	struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+
+	RTW_PRINT_SEL(sel, "%-12s %-17s\n"
+		, "conf_timeout", "blacklist_timeout");
+	RTW_PRINT_SEL(sel, "%12u %17u\n"
+		, peer_sel_policy->peer_conf_timeout_ms
+		, peer_sel_policy->peer_blacklist_timeout_ms);
+}
+#endif /* CONFIG_RTW_MESH_PEER_BLACKLIST */
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+u8 rtw_mesh_cto_mgate_required(_adapter *adapter)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	return mcfg->peer_sel_policy.cto_mgate_require
+		&& !rtw_bss_is_cto_mgate(&(mlmeext->mlmext_info.network));
+}
+
+u8 rtw_mesh_cto_mgate_network_filter(_adapter *adapter, struct wlan_network *scanned)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	return !rtw_mesh_cto_mgate_required(adapter)
+			|| (rtw_bss_is_cto_mgate(&scanned->network)
+				&& !rtw_mesh_cto_mgate_blacklist_search(adapter, scanned->network.MacAddress));
+}
+
+int rtw_mesh_cto_mgate_blacklist_add(_adapter *adapter, const u8 *addr)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	return rtw_blacklist_add(&plink_ctl->cto_mgate_blacklist, addr
+		, mcfg->peer_sel_policy.cto_mgate_blacklist_timeout_ms);
+}
+
+int rtw_mesh_cto_mgate_blacklist_del(_adapter *adapter, const u8 *addr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	return rtw_blacklist_del(&plink_ctl->cto_mgate_blacklist, addr);
+}
+
+int rtw_mesh_cto_mgate_blacklist_search(_adapter *adapter, const u8 *addr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	return rtw_blacklist_search(&plink_ctl->cto_mgate_blacklist, addr);
+}
+
+void rtw_mesh_cto_mgate_blacklist_flush(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	rtw_blacklist_flush(&plink_ctl->cto_mgate_blacklist);
+}
+
+void dump_mesh_cto_mgate_blacklist(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	dump_blacklist(sel, &plink_ctl->cto_mgate_blacklist, "blacklist");
+}
+
+void dump_mesh_cto_mgate_blacklist_settings(void *sel, _adapter *adapter)
+{
+	struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+
+	RTW_PRINT_SEL(sel, "%-12s %-17s\n"
+		, "conf_timeout", "blacklist_timeout");
+	RTW_PRINT_SEL(sel, "%12u %17u\n"
+		, peer_sel_policy->cto_mgate_conf_timeout_ms
+		, peer_sel_policy->cto_mgate_blacklist_timeout_ms);
+}
+
+static void rtw_mesh_cto_mgate_blacklist_chk(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	_queue *blist = &plink_ctl->cto_mgate_blacklist;
+	_list *list, *head;
+	struct blacklist_ent *ent = NULL;
+	struct wlan_network *scanned = NULL;
+
+	_rtw_spinlock_bh(&blist->lock);
+	head = &blist->queue;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		ent = LIST_CONTAINOR(list, struct blacklist_ent, list);
+		list = get_next(list);
+
+		if (rtw_time_after(rtw_get_current_time(), ent->exp_time)) {
+			rtw_list_delete(&ent->list);
+			rtw_mfree(ent, sizeof(struct blacklist_ent));
+			continue;
+		}
+
+		scanned = rtw_find_network(&adapter->mlmepriv.scanned_queue, ent->addr);
+		if (!scanned)
+			continue;
+
+		if (rtw_bss_is_forwarding(&scanned->network)) {
+			rtw_list_delete(&ent->list);
+			rtw_mfree(ent, sizeof(struct blacklist_ent));
+		}
+	}
+
+	_rtw_spinunlock_bh(&blist->lock);
+}
+#endif /* CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST */
+
+void rtw_chk_candidate_peer_notify(_adapter *adapter, struct wlan_network *scanned)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	bool acnode = 0;
+
+	if (IS_CH_WAITING(rfctl) && !IS_UNDER_CAC(rfctl))
+		goto exit;
+
+	if (plink_ctl->num >= RTW_MESH_MAX_PEER_CANDIDATES)
+		goto exit;
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	if (plink_ctl->acnode_rsvd) {
+		acnode = rtw_mesh_scanned_is_acnode_confirmed(adapter, scanned);
+		if (acnode && !rtw_mesh_scanned_is_acnode_allow_notify(adapter, scanned))
+			goto exit;
+	}
+#endif
+
+	/* wpa_supplicant's auto peer will initiate peering when candidate peer is reported without max_peer_links consideration */
+	if (plink_ctl->num >= mcfg->max_peer_links + acnode ? 1 : 0)
+		goto exit;
+
+	if (rtw_get_passing_time_ms(scanned->last_scanned) >= mcfg->peer_sel_policy.scanr_exp_ms
+		|| (mcfg->rssi_threshold && mcfg->rssi_threshold > scanned->network.PhyInfo.rssi)
+		|| !rtw_bss_is_candidate_mesh_peer(adapter, &scanned->network, 1, 1)
+		#if CONFIG_RTW_MACADDR_ACL
+		|| rtw_access_ctrl(adapter, scanned->network.MacAddress) == _FALSE
+		#endif
+		|| rtw_mesh_plink_get(adapter, scanned->network.MacAddress)
+		#if CONFIG_RTW_MESH_PEER_BLACKLIST
+		|| rtw_mesh_peer_blacklist_search(adapter, scanned->network.MacAddress)
+		#endif
+		#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+		|| !rtw_mesh_cto_mgate_network_filter(adapter, scanned)
+		#endif
+	)
+		goto exit;
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	if (acnode) {
+		scanned->acnode_notify_etime = 0;
+		RTW_INFO(FUNC_ADPT_FMT" acnode "MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(scanned->network.MacAddress));
+	}
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_notify_new_peer_candidate(adapter->rtw_wdev
+		, scanned->network.MacAddress
+		, BSS_EX_TLV_IES(&scanned->network)
+		, BSS_EX_TLV_IES_LEN(&scanned->network)
+		, scanned->network.PhyInfo.rssi
+		, GFP_ATOMIC
+	);
+#endif
+
+exit:
+	return;
+}
+
+void rtw_mesh_peer_status_chk(_adapter *adapter)
+{
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *plink;
+	_list *head, *list;
+	struct sta_info *sta = NULL;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	int stainfo_offset;
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	u8 cto_mgate, forwarding, mgate;
+#endif
+	u8 flush;
+	s8 flush_list[NUM_STA];
+	u8 flush_num = 0;
+	int i;
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	if (rtw_mesh_cto_mgate_required(adapter)) {
+		/* active scan on operating channel */
+		issue_probereq_ex(adapter, &adapter->mlmepriv.cur_network.network.mesh_id, NULL, 0, 0, 0, 0);
+	}
+#endif
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+
+	/* check established peers */
+	rtw_stapriv_asoc_list_lock(stapriv);
+
+	head = &stapriv->asoc_list;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		sta = LIST_CONTAINOR(list, struct sta_info, asoc_list);
+		list = get_next(list);
+
+		if (!sta->plink || !sta->plink->scanned) {
+			rtw_warn_on(1);
+			continue;
+		}
+		plink = sta->plink;
+		flush = 0;
+
+		/* remove unsuitable peer */
+		if (!rtw_bss_is_candidate_mesh_peer(adapter, &plink->scanned->network, 1, 0)
+			#if CONFIG_RTW_MACADDR_ACL
+			|| rtw_access_ctrl(adapter, plink->addr) == _FALSE
+			#endif
+		) {
+			flush = 1;
+			goto flush_add;
+		}
+
+		#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+		cto_mgate = rtw_bss_is_cto_mgate(&(plink->scanned->network));
+		forwarding = rtw_bss_is_forwarding(&(plink->scanned->network));
+		mgate = rtw_mesh_gate_search(minfo->mesh_paths, sta->phl_sta->mac_addr);
+
+		/* CTO_MGATE required, remove peer without CTO_MGATE */
+		if (rtw_mesh_cto_mgate_required(adapter) && !cto_mgate) {
+			flush = 1;
+			goto flush_add;
+		}
+
+		/* cto_mgate_conf status update */
+		if (IS_CTO_MGATE_CONF_DISABLED(plink)) {
+			if (cto_mgate && !forwarding && !mgate)
+				SET_CTO_MGATE_CONF_END_TIME(plink, mcfg->peer_sel_policy.cto_mgate_conf_timeout_ms);
+			else
+				rtw_mesh_cto_mgate_blacklist_del(adapter, sta->phl_sta->mac_addr);
+		} else {
+			/* cto_mgate_conf ongoing */
+			if (cto_mgate && !forwarding && !mgate) {
+				if (IS_CTO_MGATE_CONF_TIMEOUT(plink)) {
+					rtw_mesh_cto_mgate_blacklist_add(adapter, sta->phl_sta->mac_addr);
+
+					/* CTO_MGATE required, remove peering can't achieve CTO_MGATE */
+					if (rtw_mesh_cto_mgate_required(adapter)) {
+						flush = 1;
+						goto flush_add;
+					}	
+				}
+			} else {
+				SET_CTO_MGATE_CONF_DISABLED(plink);
+				rtw_mesh_cto_mgate_blacklist_del(adapter, sta->phl_sta->mac_addr);
+			}
+		}
+		#endif /* CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST */
+
+flush_add:
+		if (flush) {
+			rtw_stapriv_asoc_list_del(stapriv, sta);
+			stainfo_offset = rtw_stainfo_offset(stapriv, sta);
+			if (stainfo_offset_valid(stainfo_offset))
+				flush_list[flush_num++] = stainfo_offset;
+			else
+				rtw_warn_on(1);
+		}
+	}
+
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+	/* check non-established peers */
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++) {
+		plink = &plink_ctl->ent[i];
+		if (plink->valid != _TRUE || plink->plink_state == RTW_MESH_PLINK_ESTAB)
+			continue;
+
+		/* remove unsuitable peer */
+		if (!rtw_bss_is_candidate_mesh_peer(adapter, &plink->scanned->network, 1, 1)
+			#if CONFIG_RTW_MACADDR_ACL
+			|| rtw_access_ctrl(adapter, plink->addr) == _FALSE
+			#endif
+		) {
+			_rtw_mesh_expire_peer_ent(adapter, plink);
+			continue;
+		}
+
+		#if CONFIG_RTW_MESH_PEER_BLACKLIST
+		/* peer confirm check timeout, add to black list */
+		if (IS_PEER_CONF_TIMEOUT(plink)) {
+			rtw_mesh_peer_blacklist_add(adapter, plink->addr);
+			_rtw_mesh_expire_peer_ent(adapter, plink);
+		}
+		#endif
+	}
+
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	if (flush_num) {
+		u8 sta_addr[ETH_ALEN];
+		u8 updated = _FALSE;
+
+		for (i = 0; i < flush_num; i++) {
+			sta = rtw_get_stainfo_by_offset(stapriv, flush_list[i]);
+			_rtw_memcpy(sta_addr, sta->phl_sta->mac_addr, ETH_ALEN);
+
+			updated |= ap_free_sta(adapter, sta, _TRUE, 0, WLAN_REASON_DEAUTH_LEAVING, _FALSE);
+			rtw_mesh_expire_peer(adapter, sta_addr);
+		}
+
+		associated_clients_update(adapter, updated, STA_INFO_UPDATE_ALL);
+	}
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	/* loop cto_mgate_blacklist to remove ent according to scan_r */
+	rtw_mesh_cto_mgate_blacklist_chk(adapter);
+#endif
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	rtw_mesh_acnode_rsvd_chk(adapter);
+#endif
+
+	return;
+}
+
+#if CONFIG_RTW_MESH_OFFCH_CAND
+static u8 rtw_mesh_offch_cto_mgate_required(_adapter *adapter)
+{
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	_queue *queue = &(mlme->scanned_queue);
+	_list *head, *pos;
+	struct wlan_network *scanned = NULL;
+	u8 ret = 0;
+
+	if (!rtw_mesh_cto_mgate_required(adapter))
+		goto exit;
+
+	_rtw_spinlock_bh(&(mlme->scanned_queue.lock));
+
+	head = get_list_head(queue);
+	pos = get_next(head);
+	while (!rtw_end_of_queue_search(head, pos)) {
+		scanned = LIST_CONTAINOR(pos, struct wlan_network, list);
+
+		if (rtw_get_passing_time_ms(scanned->last_scanned) < mcfg->peer_sel_policy.scanr_exp_ms
+			&& (!mcfg->rssi_threshold || mcfg->rssi_threshold <= scanned->network.PhyInfo.rssi)
+			#if CONFIG_RTW_MACADDR_ACL
+			&& rtw_access_ctrl(adapter, scanned->network.MacAddress) == _TRUE
+			#endif
+			&& rtw_bss_is_candidate_mesh_peer(adapter, &scanned->network, 1, 1)
+			&& rtw_bss_is_cto_mgate(&scanned->network)
+			#if CONFIG_RTW_MESH_PEER_BLACKLIST
+			&& !rtw_mesh_peer_blacklist_search(adapter, scanned->network.MacAddress)
+			#endif
+			&& !rtw_mesh_cto_mgate_blacklist_search(adapter, scanned->network.MacAddress)
+		)
+			break;
+
+		pos = get_next(pos);
+	}
+
+	if (rtw_end_of_queue_search(head, pos))
+		ret = 1;
+
+	_rtw_spinunlock_bh(&(mlme->scanned_queue.lock));
+
+exit:
+	return ret;
+#else
+	return 0;
+#endif /* CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST */
+}
+
+u8 rtw_mesh_offch_candidate_accepted(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	u8 ret = 0;
+
+	if (!adapter->mesh_cfg.peer_sel_policy.offch_cand)
+		goto exit;
+
+	ret = MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter)
+		&& (!plink_ctl->num || rtw_mesh_offch_cto_mgate_required(adapter))
+		;
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (ret) {
+		struct mi_state mstate_no_self;
+
+		rtw_mi_status_no_self(adapter, &mstate_no_self);
+		if (MSTATE_STA_LD_NUM(&mstate_no_self))
+			ret = 0;
+	}
+#endif
+
+exit:
+	return ret;
+}
+
+/*
+ * this function is called under off channel candidate is required 
+ * the channel with maximum candidate count is selected
+*/
+u8 rtw_mesh_select_operating_ch(_adapter *adapter)
+{
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	_queue *queue = &(mlme->scanned_queue);
+	_list *head, *pos;
+	struct wlan_network *scanned = NULL;
+	int i;
+	/* statistics for candidate accept peering */
+	u8 cand_ap_cnt[MAX_CHANNEL_NUM] = {0};
+	u8 max_cand_ap_ch = 0;
+	u8 max_cand_ap_cnt = 0;
+	/* statistics for candidate including not accept peering */
+	u8 cand_cnt[MAX_CHANNEL_NUM] = {0};
+	u8 max_cand_ch = 0;
+	u8 max_cand_cnt = 0;
+
+	_rtw_spinlock_bh(&(mlme->scanned_queue.lock));
+
+	head = get_list_head(queue);
+	pos = get_next(head);
+	while (!rtw_end_of_queue_search(head, pos)) {
+		scanned = LIST_CONTAINOR(pos, struct wlan_network, list);
+		pos = get_next(pos);
+
+		if (rtw_get_passing_time_ms(scanned->last_scanned) < mcfg->peer_sel_policy.scanr_exp_ms
+			&& (!mcfg->rssi_threshold || mcfg->rssi_threshold <= scanned->network.PhyInfo.rssi)
+			#if CONFIG_RTW_MACADDR_ACL
+			&& rtw_access_ctrl(adapter, scanned->network.MacAddress) == _TRUE
+			#endif
+			&& rtw_bss_is_candidate_mesh_peer(adapter, &scanned->network, 0, 0)
+			#if CONFIG_RTW_MESH_PEER_BLACKLIST
+			&& !rtw_mesh_peer_blacklist_search(adapter, scanned->network.MacAddress)
+			#endif
+			#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+			&& rtw_mesh_cto_mgate_network_filter(adapter, scanned)
+			#endif
+		) {
+			int ch_set_idx = rtw_chset_search_ch(chset, scanned->network.Configuration.DSConfig);
+
+			if (ch_set_idx >= 0
+				&& !(chset->chs[ch_set_idx].flags & RTW_CHF_NO_IR)
+				&& !CH_IS_NON_OCP(&chset->chs[ch_set_idx])
+			) {
+				u8 nop, accept;
+
+				rtw_mesh_bss_peering_status(&scanned->network, &nop, &accept);
+				cand_cnt[ch_set_idx]++;
+				if (max_cand_cnt < cand_cnt[ch_set_idx]) {
+					max_cand_cnt = cand_cnt[ch_set_idx];
+					max_cand_ch = chset->chs[ch_set_idx].ChannelNum;
+				}
+				if (accept) {
+					cand_ap_cnt[ch_set_idx]++;
+					if (max_cand_ap_cnt < cand_ap_cnt[ch_set_idx]) {
+						max_cand_ap_cnt = cand_ap_cnt[ch_set_idx];
+						max_cand_ap_ch = chset->chs[ch_set_idx].ChannelNum;
+					}
+				}
+			}
+		}
+	}
+
+	_rtw_spinunlock_bh(&(mlme->scanned_queue.lock));
+
+	return max_cand_ap_ch ? max_cand_ap_ch : max_cand_ch;
+}
+
+void dump_mesh_offch_cand_settings(void *sel, _adapter *adapter)
+{
+	struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+
+	RTW_PRINT_SEL(sel, "%-6s %-11s\n"
+		, "enable", "find_int_ms");
+	RTW_PRINT_SEL(sel, "%6u %11u\n"
+		, peer_sel_policy->offch_cand, peer_sel_policy->offch_find_int_ms);
+}
+#endif /* CONFIG_RTW_MESH_OFFCH_CAND */
+
+void dump_mesh_peer_sel_policy(void *sel, _adapter *adapter)
+{
+	struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+
+	RTW_PRINT_SEL(sel, "%-12s\n", "scanr_exp_ms");
+	RTW_PRINT_SEL(sel, "%12u\n", peer_sel_policy->scanr_exp_ms);
+}
+
+void dump_mesh_networks(void *sel, _adapter *adapter)
+{
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+#define NSTATE_TITLE_FMT_ACN " %-5s"
+#define NSTATE_VALUE_FMT_ACN " %5d"
+#define NSTATE_TITLE_ARG_ACN , "acn"
+#define NSTATE_VALUE_ARG_ACN , (acn_ms < 99999 ? acn_ms : 99999)
+#else
+#define NSTATE_TITLE_FMT_ACN ""
+#define NSTATE_VALUE_FMT_ACN ""
+#define NSTATE_TITLE_ARG_ACN
+#define NSTATE_VALUE_ARG_ACN
+#endif
+
+	struct mlme_priv *mlme = &(adapter->mlmepriv);
+	_queue *queue = &(mlme->scanned_queue);
+	struct wlan_network	*network;
+	_list *list, *head;
+	u8 same_mbss;
+	u8 candidate;
+	struct mesh_plink_ent *plink;
+	u8 blocked;
+	u8 established;
+	s32 age_ms;
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	s32 acn_ms;
+#endif
+	u8 *mesh_conf_ie;
+	sint mesh_conf_ie_len;
+	u8 auth_pid;
+	u8 *rsn_ie;
+	int rsn_ie_len;
+	int gcs, pcs, gmcs;
+	u8 mfp_opt;
+	struct wlan_network **mesh_networks;
+	u8 mesh_network_cnt = 0;
+	int i;
+
+	mesh_networks = rtw_zvmalloc(mlme->max_bss_cnt * sizeof(struct wlan_network *));
+	if (!mesh_networks)
+		return;
+
+	_rtw_spinlock_bh(&queue->lock);
+	head = get_list_head(queue);
+	list = get_next(head);
+
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		network = LIST_CONTAINOR(list, struct wlan_network, list);
+		list = get_next(list);
+
+		if (network->network.InfrastructureMode != Ndis802_11_mesh)
+			continue;
+
+		mesh_conf_ie = rtw_get_ie(BSS_EX_TLV_IES(&network->network), WLAN_EID_MESH_CONFIG
+			, &mesh_conf_ie_len, BSS_EX_TLV_IES_LEN(&network->network));
+		if (!mesh_conf_ie || mesh_conf_ie_len != 7)
+			continue;
+
+		mesh_networks[mesh_network_cnt++] = network;
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+
+	RTW_PRINT_SEL(sel, "  %-17s %-3s %-4s %-5s %-32s %-10s"
+		" %-3s %-3s %-4s"
+		" %-3s %-3s %-3s"
+		NSTATE_TITLE_FMT_ACN
+		"\n"
+		, "bssid", "ch", "rssi", "age", "mesh_id", "P M C S A "
+		, "pcs", "gcs", "gmcs"
+		, "nop", "fwd", "cto"
+		NSTATE_TITLE_ARG_ACN
+	);
+
+	if (MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter)) {
+		network = &mlme->cur_network;
+		mesh_conf_ie = rtw_get_ie(BSS_EX_TLV_IES(&network->network), WLAN_EID_MESH_CONFIG
+			, &mesh_conf_ie_len, BSS_EX_TLV_IES_LEN(&network->network));
+		if (mesh_conf_ie && mesh_conf_ie_len == 7) {
+			gcs = pcs = gmcs = 0;
+			mfp_opt = MFP_NO;
+			auth_pid = GET_MESH_CONF_ELE_AUTH_PROTO_ID(mesh_conf_ie + 2);
+			if (auth_pid && auth_pid <= 2) {
+				rsn_ie = rtw_get_wpa2_ie(BSS_EX_TLV_IES(&network->network)
+							, &rsn_ie_len, BSS_EX_TLV_IES_LEN(&network->network));
+				if (rsn_ie && rsn_ie_len)
+					rtw_parse_wpa2_ie(rsn_ie, rsn_ie_len + 2, &gcs, &pcs, &gmcs, NULL, &mfp_opt, NULL);
+			}
+			RTW_PRINT_SEL(sel, "* "MAC_FMT" %3d            %-32s %2x%2x%2x%2x%2x"
+				" %03x %03x %c%03x"
+				" %c%2u %3u %c%c "
+				"\n"
+				, MAC_ARG(network->network.MacAddress)
+				, network->network.Configuration.DSConfig
+				, network->network.mesh_id.Ssid
+				, GET_MESH_CONF_ELE_PATH_SEL_PROTO_ID(mesh_conf_ie + 2)
+				, GET_MESH_CONF_ELE_PATH_SEL_METRIC_ID(mesh_conf_ie + 2)
+				, GET_MESH_CONF_ELE_CONGEST_CTRL_MODE_ID(mesh_conf_ie + 2)
+				, GET_MESH_CONF_ELE_SYNC_METHOD_ID(mesh_conf_ie + 2)
+				, auth_pid
+				, pcs, gcs, mfp_opt == MFP_REQUIRED ? 'R' : (mfp_opt == MFP_OPTIONAL ? 'C' : ' ')
+				, mfp_opt >= MFP_OPTIONAL ? gmcs : 0
+				, GET_MESH_CONF_ELE_ACCEPT_PEERINGS(mesh_conf_ie + 2) ? '+' : ' '
+				, GET_MESH_CONF_ELE_NUM_OF_PEERINGS(mesh_conf_ie + 2)
+				, GET_MESH_CONF_ELE_FORWARDING(mesh_conf_ie + 2)
+				, GET_MESH_CONF_ELE_CTO_MGATE(mesh_conf_ie + 2) ? 'G' : ' '
+				, GET_MESH_CONF_ELE_CTO_AS(mesh_conf_ie + 2) ? 'A' : ' '
+			);
+		}
+	}
+
+	for (i = 0; i < mesh_network_cnt; i++) {
+		network = mesh_networks[i];
+
+		if (network->network.InfrastructureMode != Ndis802_11_mesh)
+			continue;
+
+		mesh_conf_ie = rtw_get_ie(BSS_EX_TLV_IES(&network->network), WLAN_EID_MESH_CONFIG
+			, &mesh_conf_ie_len, BSS_EX_TLV_IES_LEN(&network->network));
+		if (!mesh_conf_ie || mesh_conf_ie_len != 7)
+			continue;
+
+		gcs = pcs = gmcs = 0;
+		mfp_opt = MFP_NO;
+		auth_pid = GET_MESH_CONF_ELE_AUTH_PROTO_ID(mesh_conf_ie + 2);
+		if (auth_pid && auth_pid <= 2) {
+			rsn_ie = rtw_get_wpa2_ie(BSS_EX_TLV_IES(&network->network), &rsn_ie_len, BSS_EX_TLV_IES_LEN(&network->network));
+			if (rsn_ie && rsn_ie_len)
+				rtw_parse_wpa2_ie(rsn_ie, rsn_ie_len + 2, &gcs, &pcs, &gmcs, NULL, &mfp_opt, NULL);
+		}
+		age_ms = rtw_get_passing_time_ms(network->last_scanned);
+		#if CONFIG_RTW_MESH_ACNODE_PREVENT
+		if (network->acnode_stime == 0)
+			acn_ms = 0;
+		else
+			acn_ms = rtw_get_passing_time_ms(network->acnode_stime);
+		#endif
+		same_mbss = 0;
+		candidate = 0;
+		plink = NULL;
+		blocked = 0;
+		established = 0;
+
+		if (MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter)) {
+			plink = rtw_mesh_plink_get(adapter, network->network.MacAddress);
+			if (plink && plink->plink_state == RTW_MESH_PLINK_ESTAB)
+				established = 1;
+			else if (plink && plink->plink_state == RTW_MESH_PLINK_BLOCKED)
+				blocked = 1;
+			else if (plink)
+				;
+			else if (rtw_bss_is_candidate_mesh_peer(adapter, &network->network, 0, 1))
+				candidate = 1;
+			else if (rtw_bss_is_same_mbss(&mlme->cur_network.network, &network->network))
+				same_mbss = 1;
+		}
+
+		RTW_PRINT_SEL(sel, "%c "MAC_FMT" %3d %4ld %5d %-32s %2x%2x%2x%2x%2x"
+			" %03x %03x %c%03x"
+			" %c%2u %3u %c%c "
+			NSTATE_VALUE_FMT_ACN
+			"\n"
+			, established ? 'E' : (blocked ? 'B' : (plink ? 'N' : (candidate ? 'C' : (same_mbss ? 'S' : ' '))))
+			, MAC_ARG(network->network.MacAddress)
+			, network->network.Configuration.DSConfig
+			, network->network.PhyInfo.rssi
+			, age_ms < 99999 ? age_ms : 99999
+			, network->network.mesh_id.Ssid
+			, GET_MESH_CONF_ELE_PATH_SEL_PROTO_ID(mesh_conf_ie + 2)
+			, GET_MESH_CONF_ELE_PATH_SEL_METRIC_ID(mesh_conf_ie + 2)
+			, GET_MESH_CONF_ELE_CONGEST_CTRL_MODE_ID(mesh_conf_ie + 2)
+			, GET_MESH_CONF_ELE_SYNC_METHOD_ID(mesh_conf_ie + 2)
+			, auth_pid
+			, pcs, gcs, mfp_opt == MFP_REQUIRED ? 'R' : (mfp_opt == MFP_OPTIONAL ? 'C' : ' ')
+			, mfp_opt >= MFP_OPTIONAL ? gmcs : 0
+			, GET_MESH_CONF_ELE_ACCEPT_PEERINGS(mesh_conf_ie + 2) ? '+' : ' '
+			, GET_MESH_CONF_ELE_NUM_OF_PEERINGS(mesh_conf_ie + 2)
+			, GET_MESH_CONF_ELE_FORWARDING(mesh_conf_ie + 2)
+			, GET_MESH_CONF_ELE_CTO_MGATE(mesh_conf_ie + 2) ? 'G' : ' '
+			, GET_MESH_CONF_ELE_CTO_AS(mesh_conf_ie + 2) ? 'A' : ' '
+			NSTATE_VALUE_ARG_ACN
+		);
+	}
+
+	rtw_vmfree(mesh_networks, mlme->max_bss_cnt * sizeof(struct wlan_network *));
+}
+
+void rtw_mesh_adjust_bchbw(enum band_type req_band, u8 req_ch, u8 *req_bw, u8 *req_offset)
+{
+	if (req_band == BAND_ON_24G && req_ch >= 5 && req_ch <= 9) {
+		/* prevent secondary channel offset mismatch */
+		if (*req_bw > CHANNEL_WIDTH_20) {
+			*req_bw = CHANNEL_WIDTH_20;
+			*req_offset = CHAN_OFFSET_NO_EXT;
+		}
+	}
+}
+
+void rtw_mesh_sae_check_frames(_adapter *adapter, const u8 *buf, u32 len, u8 tx, u16 alg, u16 seq, u16 status)
+{
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	if (tx && seq == 1)
+		rtw_mesh_plink_set_peer_conf_timeout(adapter, GetAddr1Ptr(buf));
+#endif
+}
+
+#if CONFIG_RTW_MPM_TX_IES_SYNC_BSS
+#ifdef CONFIG_RTW_MESH_AEK
+static int rtw_mpm_ampe_dec(_adapter *adapter, struct mesh_plink_ent *plink
+	, u8 *fhead, size_t flen, u8* fbody, u8 *mic_ie, u8 *ampe_buf)
+{	
+	int ret = _FAIL, verify_ret;
+	const u8 *aad[] = {adapter_mac_addr(adapter), plink->addr, fbody};
+	const size_t aad_len[] = {ETH_ALEN, ETH_ALEN, mic_ie - fbody};
+	u8 *iv_crypt;
+	size_t iv_crypt_len = flen - (mic_ie + 2 - fhead);
+
+	iv_crypt = rtw_malloc(iv_crypt_len);
+	if (!iv_crypt)
+		goto exit;
+
+	_rtw_memcpy(iv_crypt, mic_ie + 2, iv_crypt_len);
+
+	verify_ret = rtw_aes_siv_decrypt(plink->aek, 32, iv_crypt, iv_crypt_len
+		, 3, aad, aad_len, ampe_buf);
+
+	rtw_mfree(iv_crypt, iv_crypt_len);
+
+	if (verify_ret) {
+		RTW_WARN("verify error, aek_valid=%u\n", plink->aek_valid);
+		goto exit;
+	} else if (*ampe_buf != WLAN_EID_AMPE) {
+		RTW_WARN("plaintext is not AMPE IE\n");
+		goto exit;
+	} else if ( 16 /* AES_BLOCK_SIZE*/ + 2 + *(ampe_buf + 1) > iv_crypt_len) {
+		RTW_WARN("plaintext AMPE IE length is not valid\n");
+		goto exit;
+	}
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+static int rtw_mpm_ampe_enc(_adapter *adapter, struct mesh_plink_ent *plink
+	, u8* fbody, u8 *mic_ie, u8 *ampe_buf, bool inverse)
+{
+	int ret = _FAIL, protect_ret;
+	const u8 *aad[3];
+	const size_t aad_len[3] = {ETH_ALEN, ETH_ALEN, mic_ie - fbody};
+	u8 *ampe_ie;
+	size_t ampe_ie_len = *(ampe_buf + 1) + 2; /* including id & len */
+
+	if (inverse) {
+		aad[0] = plink->addr;
+		aad[1] = adapter_mac_addr(adapter);
+	} else {
+		aad[0] = adapter_mac_addr(adapter);
+		aad[1] = plink->addr;
+	}
+	aad[2] = fbody;
+
+	ampe_ie = rtw_malloc(ampe_ie_len);
+	if (!ampe_ie)
+		goto exit;
+
+	_rtw_memcpy(ampe_ie, ampe_buf, ampe_ie_len);
+
+	protect_ret = rtw_aes_siv_encrypt(plink->aek, 32, ampe_ie, ampe_ie_len
+		, 3, aad, aad_len, mic_ie + 2);
+
+	rtw_mfree(ampe_ie, ampe_ie_len);
+
+	if (protect_ret) {
+		RTW_WARN("protect error, aek_valid=%u\n", plink->aek_valid);
+		goto exit;
+	}
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_RTW_MESH_AEK */
+
+static int rtw_mpm_tx_ies_sync_bss(_adapter *adapter, struct mesh_plink_ent *plink
+	, u8 *fhead, size_t flen, u8* fbody, u8 tlv_ies_offset, u8 *mpm_ie, u8 *mic_ie
+	, u8 **nbuf, size_t *nlen)
+{
+	int ret = _FAIL;
+	struct mlme_priv *mlme = &(adapter->mlmepriv);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *mlmeinfo = &(mlmeext->mlmext_info);
+	WLAN_BSSID_EX *network = &(mlmeinfo->network);
+	uint left;
+	u8 *pos;
+
+	uint mpm_ielen = *(mpm_ie + 1);
+	u8 *fpos;
+	u8 *new_buf = NULL;
+	size_t new_len = 0;
+
+	u8 *new_fhead;
+	size_t new_flen;
+	u8 *new_fbody;
+	u8 *new_mic_ie;
+
+#ifdef CONFIG_RTW_MESH_AEK
+	u8 *ampe_buf = NULL;
+	size_t ampe_buf_len = 0;
+
+	/* decode */
+	if (mic_ie) {
+		ampe_buf_len = flen - (mic_ie + 2 + 16 /* AES_BLOCK_SIZE */ - fhead);
+		ampe_buf = rtw_malloc(ampe_buf_len);
+		if (!ampe_buf)
+			goto exit;
+
+		if (rtw_mpm_ampe_dec(adapter, plink, fhead, flen, fbody, mic_ie, ampe_buf) != _SUCCESS)
+			goto exit;
+
+		if (*(ampe_buf + 1) >= 68) {
+			_rtw_memcpy(plink->sel_pcs, ampe_buf + 2, 4);
+			_rtw_memcpy(plink->l_nonce, ampe_buf + 6, 32);
+			_rtw_memcpy(plink->p_nonce, ampe_buf + 38, 32);
+		}
+	}
+#endif
+
+	/* count for new frame length  */
+	new_len = sizeof(struct rtw_ieee80211_hdr_3addr) + tlv_ies_offset;
+	left = BSS_EX_TLV_IES_LEN(network);
+	pos = BSS_EX_TLV_IES(network);
+	while (left >= 2) {
+		u8 id, elen;
+	
+		id = *pos++;
+		elen = *pos++;
+		left -= 2;
+
+		if (elen > left)
+			break;
+
+		switch (id) {
+		case WLAN_EID_SSID:
+		case WLAN_EID_DS_PARAMS:
+		case WLAN_EID_TIM:
+			break;
+		default:
+			new_len += 2 + elen;
+		}
+
+		left -= elen;
+		pos += elen;
+	}
+	new_len += mpm_ielen + 2;
+	if (mic_ie)
+		new_len += 16 /* AES_BLOCK_SIZE*/ + 2 + ampe_buf_len;
+
+	/* alloc new frame */
+	new_buf = rtw_malloc(new_len);
+	if (!new_buf) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	/* build new frame  */
+	_rtw_memcpy(new_buf, fhead, sizeof(struct rtw_ieee80211_hdr_3addr) + tlv_ies_offset);
+	new_fhead = new_buf;
+	new_flen = new_len;
+	new_fbody = new_fhead + sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	fpos = new_fbody + tlv_ies_offset;
+	left = BSS_EX_TLV_IES_LEN(network);
+	pos = BSS_EX_TLV_IES(network);
+	while (left >= 2) {
+		u8 id, elen;
+	
+		id = *pos++;
+		elen = *pos++;
+		left -= 2;
+
+		if (elen > left)
+			break;
+
+		switch (id) {
+		case WLAN_EID_SSID:
+		case WLAN_EID_DS_PARAMS:
+		case WLAN_EID_TIM:
+			break;
+		default:
+			fpos = rtw_set_ie(fpos, id, elen, pos, NULL);
+			if (id == WLAN_EID_MESH_CONFIG)
+				fpos = rtw_set_ie(fpos, WLAN_EID_MPM, mpm_ielen, mpm_ie + 2, NULL);
+		}
+
+		left -= elen;
+		pos += elen;
+	}
+	if (mic_ie) {
+		new_mic_ie = fpos;
+		*fpos++ = WLAN_EID_MIC;
+		*fpos++ = 16 /* AES_BLOCK_SIZE */;
+	}
+
+#ifdef CONFIG_RTW_MESH_AEK
+	/* encode */
+	if (mic_ie) {
+		int enc_ret = rtw_mpm_ampe_enc(adapter, plink, new_fbody, new_mic_ie, ampe_buf, 0);
+		if (enc_ret != _SUCCESS)
+			goto exit;
+	}
+#endif
+
+	*nlen = new_len;
+	*nbuf = new_buf;
+
+	ret = _SUCCESS;
+
+exit:
+	if (ret != _SUCCESS && new_buf)
+		rtw_mfree(new_buf, new_len);
+
+#ifdef CONFIG_RTW_MESH_AEK
+	if (ampe_buf)
+		rtw_mfree(ampe_buf, ampe_buf_len);
+#endif
+
+	return ret;
+}
+#endif /* CONFIG_RTW_MPM_TX_IES_SYNC_BSS */
+
+struct mpm_frame_info {
+	u8 *aid;
+	u16 aid_v;
+	u8 *pid;
+	u16 pid_v;
+	u8 *llid;
+	u16 llid_v;
+	u8 *plid;
+	u16 plid_v;
+	u8 *reason;
+	u16 reason_v;
+	u8 *chosen_pmk;
+};
+
+/*
+* pid:00000 llid:00000 chosen_pmk:0x00000000000000000000000000000000
+* aid:00000 pid:00000 llid:00000 plid:00000 chosen_pmk:0x00000000000000000000000000000000
+* pid:00000 llid:00000 plid:00000 reason:00000 chosen_pmk:0x00000000000000000000000000000000
+*/
+#define MPM_LOG_BUF_LEN 92 /* this length is limited for legal combination */
+static void rtw_mpm_info_msg(struct mpm_frame_info *mpm_info, u8 *mpm_log_buf)
+{
+	int cnt = 0;
+
+	if (mpm_info->aid) {
+		cnt += snprintf(mpm_log_buf + cnt, MPM_LOG_BUF_LEN - cnt - 1, "aid:%u ", mpm_info->aid_v);
+		if (cnt >= MPM_LOG_BUF_LEN - 1)
+			goto exit;
+	}
+	if (mpm_info->pid) {
+		cnt += snprintf(mpm_log_buf + cnt, MPM_LOG_BUF_LEN - cnt - 1, "pid:%u ", mpm_info->pid_v);
+		if (cnt >= MPM_LOG_BUF_LEN - 1)
+			goto exit;
+	}
+	if (mpm_info->llid) {
+		cnt += snprintf(mpm_log_buf + cnt, MPM_LOG_BUF_LEN - cnt - 1, "llid:%u ", mpm_info->llid_v);
+		if (cnt >= MPM_LOG_BUF_LEN - 1)
+			goto exit;
+	}
+	if (mpm_info->plid) {
+		cnt += snprintf(mpm_log_buf + cnt, MPM_LOG_BUF_LEN - cnt - 1, "plid:%u ", mpm_info->plid_v);
+		if (cnt >= MPM_LOG_BUF_LEN - 1)
+			goto exit;
+	}
+	if (mpm_info->reason) {
+		cnt += snprintf(mpm_log_buf + cnt, MPM_LOG_BUF_LEN - cnt - 1, "reason:%u ", mpm_info->reason_v);
+		if (cnt >= MPM_LOG_BUF_LEN - 1)
+			goto exit;
+	}
+	if (mpm_info->chosen_pmk) {
+		cnt += snprintf(mpm_log_buf + cnt, MPM_LOG_BUF_LEN - cnt - 1, "chosen_pmk:0x"KEY_FMT, KEY_ARG(mpm_info->chosen_pmk));
+		if (cnt >= MPM_LOG_BUF_LEN - 1)
+			goto exit;
+	}
+
+exit:
+	return;
+}
+
+static int rtw_mpm_check_frames(_adapter *adapter, u8 action, const u8 **buf, size_t *len, u8 tx)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *plink = NULL;
+	u8 *nbuf = NULL;
+	size_t nlen = 0;
+	u8 *fhead = (u8 *)*buf;
+	size_t flen = *len;
+	u8 *peer_addr = tx ? GetAddr1Ptr(fhead) : get_addr2_ptr(fhead);
+	u8 *frame_body = fhead + sizeof(struct rtw_ieee80211_hdr_3addr);
+	struct mpm_frame_info mpm_info;
+	u8 tlv_ies_offset;
+	u8 *mpm_ie = NULL;
+	uint mpm_ielen = 0;
+	u8 *mic_ie = NULL;
+	uint mic_ielen = 0;
+	int ret = 0;
+	u8 mpm_log_buf[MPM_LOG_BUF_LEN] = {0};
+
+	if (action == RTW_ACT_SELF_PROTECTED_MESH_OPEN)
+		tlv_ies_offset = 4;
+	else if (action == RTW_ACT_SELF_PROTECTED_MESH_CONF)
+		tlv_ies_offset = 6;
+	else if (action == RTW_ACT_SELF_PROTECTED_MESH_CLOSE)
+		tlv_ies_offset = 2;
+	else {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	plink = rtw_mesh_plink_get(adapter, peer_addr);
+	if (!plink && (tx == _TRUE || action == RTW_ACT_SELF_PROTECTED_MESH_CONF)) {
+		/* warning message if no plink when: 1.TX all MPM or 2.RX CONF */
+		RTW_WARN("RTW_%s:%s without plink of "MAC_FMT"\n"
+			, (tx == _TRUE) ? "Tx" : "Rx", action_self_protected_str(action), MAC_ARG(peer_addr));
+		goto exit;
+	}
+
+	_rtw_memset(&mpm_info, 0, sizeof(struct mpm_frame_info));
+
+	if (action == RTW_ACT_SELF_PROTECTED_MESH_CONF) {
+		mpm_info.aid = (u8 *)frame_body + 4;
+		mpm_info.aid_v = RTW_GET_LE16(mpm_info.aid);
+	}
+
+	mpm_ie = rtw_get_ie(fhead + sizeof(struct rtw_ieee80211_hdr_3addr) + tlv_ies_offset
+		, WLAN_EID_MPM, &mpm_ielen
+		, flen - sizeof(struct rtw_ieee80211_hdr_3addr) - tlv_ies_offset);
+	if (!mpm_ie || mpm_ielen < 2 + 2)
+		goto exit;
+
+	mpm_info.pid = mpm_ie + 2;
+	mpm_info.pid_v = RTW_GET_LE16(mpm_info.pid);
+	mpm_info.llid = mpm_info.pid + 2;
+	mpm_info.llid_v = RTW_GET_LE16(mpm_info.llid);
+
+	switch (action) {
+	case RTW_ACT_SELF_PROTECTED_MESH_OPEN:
+		/* pid:2, llid:2, (chosen_pmk:16) */
+		if (mpm_info.pid_v == 0 && mpm_ielen == 4)
+			;
+		else if (mpm_info.pid_v == 1 && mpm_ielen == 20)
+			mpm_info.chosen_pmk = mpm_info.llid + 2;
+		else
+			goto exit;
+		break;
+	case RTW_ACT_SELF_PROTECTED_MESH_CONF:
+		/* pid:2, llid:2, plid:2, (chosen_pmk:16) */
+		mpm_info.plid = mpm_info.llid + 2;
+		mpm_info.plid_v = RTW_GET_LE16(mpm_info.plid);
+		if (mpm_info.pid_v == 0 && mpm_ielen == 6)
+			;
+		else if (mpm_info.pid_v == 1 && mpm_ielen == 22)
+			mpm_info.chosen_pmk = mpm_info.plid + 2;
+		else
+			goto exit;
+		break;
+	case RTW_ACT_SELF_PROTECTED_MESH_CLOSE:
+		/* pid:2, llid:2, (plid:2), reason:2, (chosen_pmk:16) */
+		if (mpm_info.pid_v == 0 && mpm_ielen == 6) {
+			/* MPM, without plid */
+			mpm_info.reason = mpm_info.llid + 2;
+			mpm_info.reason_v = RTW_GET_LE16(mpm_info.reason);
+		} else if (mpm_info.pid_v == 0 && mpm_ielen == 8) {
+			/* MPM, with plid */
+			mpm_info.plid = mpm_info.llid + 2;
+			mpm_info.plid_v = RTW_GET_LE16(mpm_info.plid);
+			mpm_info.reason = mpm_info.plid + 2;
+			mpm_info.reason_v = RTW_GET_LE16(mpm_info.reason);
+		} else if (mpm_info.pid_v == 1 && mpm_ielen == 22) {
+			/* AMPE, without plid */
+			mpm_info.reason = mpm_info.llid + 2;
+			mpm_info.reason_v = RTW_GET_LE16(mpm_info.reason);
+			mpm_info.chosen_pmk = mpm_info.reason + 2;
+		} else if (mpm_info.pid_v == 1 && mpm_ielen == 24) {
+			/* AMPE, with plid */
+			mpm_info.plid = mpm_info.llid + 2;
+			mpm_info.plid_v = RTW_GET_LE16(mpm_info.plid);
+			mpm_info.reason = mpm_info.plid + 2;
+			mpm_info.reason_v = RTW_GET_LE16(mpm_info.reason);
+			mpm_info.chosen_pmk = mpm_info.reason + 2;
+		} else
+			goto exit;
+		break;
+	};
+
+	if (mpm_info.pid_v == 1) {
+		mic_ie = rtw_get_ie(fhead + sizeof(struct rtw_ieee80211_hdr_3addr) + tlv_ies_offset
+			, WLAN_EID_MIC, &mic_ielen
+			, flen - sizeof(struct rtw_ieee80211_hdr_3addr) - tlv_ies_offset);
+		if (!mic_ie || mic_ielen != 16 /* AES_BLOCK_SIZE */)
+			goto exit;
+	}
+
+#if CONFIG_RTW_MPM_TX_IES_SYNC_BSS
+	if ((action == RTW_ACT_SELF_PROTECTED_MESH_OPEN || action == RTW_ACT_SELF_PROTECTED_MESH_CONF)
+		&& tx == _TRUE
+	) {
+#define DBG_RTW_MPM_TX_IES_SYNC_BSS 0
+
+		if (mpm_info.pid_v == 1 && (!plink || !MESH_PLINK_AEK_VALID(plink))) {
+			RTW_WARN("AEK not ready, IEs can't sync with BSS\n");
+			goto bypass_sync_bss;
+		}
+
+		if (DBG_RTW_MPM_TX_IES_SYNC_BSS) {
+			RTW_INFO(FUNC_ADPT_FMT" before:\n", FUNC_ADPT_ARG(adapter));
+			dump_ies(RTW_DBGDUMP
+				, fhead + sizeof(struct rtw_ieee80211_hdr_3addr) + tlv_ies_offset
+				, flen - sizeof(struct rtw_ieee80211_hdr_3addr) - tlv_ies_offset);
+		}
+
+		rtw_mpm_tx_ies_sync_bss(adapter, plink
+			, fhead, flen, frame_body, tlv_ies_offset, mpm_ie, mic_ie
+			, &nbuf, &nlen);
+		if (!nbuf)
+			goto exit;
+
+		/* update pointer & len for new frame */
+		fhead = nbuf;
+		flen = nlen;
+		frame_body = fhead + sizeof(struct rtw_ieee80211_hdr_3addr);
+		if (mpm_info.pid_v == 1) {
+			mic_ie = rtw_get_ie(fhead + sizeof(struct rtw_ieee80211_hdr_3addr) + tlv_ies_offset
+				, WLAN_EID_MIC, &mic_ielen
+				, flen - sizeof(struct rtw_ieee80211_hdr_3addr) - tlv_ies_offset);
+		}
+
+		if (DBG_RTW_MPM_TX_IES_SYNC_BSS) {
+			RTW_INFO(FUNC_ADPT_FMT" after:\n", FUNC_ADPT_ARG(adapter));
+			dump_ies(RTW_DBGDUMP
+				, fhead + sizeof(struct rtw_ieee80211_hdr_3addr) + tlv_ies_offset
+				, flen - sizeof(struct rtw_ieee80211_hdr_3addr) - tlv_ies_offset);
+		}
+	}
+bypass_sync_bss:
+#endif /* CONFIG_RTW_MPM_TX_IES_SYNC_BSS */
+
+	if (!plink)
+		goto mpm_log;
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	if (action == RTW_ACT_SELF_PROTECTED_MESH_OPEN) {
+		if (tx)
+			rtw_mesh_plink_set_peer_conf_timeout(adapter, peer_addr);
+
+	} else
+#endif
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	if (action == RTW_ACT_SELF_PROTECTED_MESH_CLOSE) {
+		if (tx && mpm_info.reason && mpm_info.reason_v == WLAN_REASON_MESH_MAX_PEERS) {
+			if (rtw_mesh_scanned_is_acnode_confirmed(adapter, plink->scanned)
+				&& rtw_mesh_acnode_prevent_allow_sacrifice(adapter)
+			) {
+				struct sta_info *sac = rtw_mesh_acnode_prevent_pick_sacrifice(adapter);
+
+				if (sac) {
+					struct sta_priv *stapriv = &adapter->stapriv;
+					u8 sta_addr[ETH_ALEN];
+					u8 updated = _FALSE;
+
+					rtw_stapriv_asoc_list_lock(stapriv);
+					if (!rtw_is_list_empty(&sac->asoc_list))
+						rtw_stapriv_asoc_list_del(stapriv, sac);
+					rtw_stapriv_asoc_list_unlock(stapriv);
+					RTW_INFO(FUNC_ADPT_FMT" sacrifice "MAC_FMT" for acnode\n"
+						, FUNC_ADPT_ARG(adapter), MAC_ARG(sac->phl_sta->mac_addr));
+
+					_rtw_memcpy(sta_addr, sac->phl_sta->mac_addr, ETH_ALEN);
+					updated = ap_free_sta(adapter, sac, 0, 0, 0, 1);
+					rtw_mesh_expire_peer(stapriv->padapter, sta_addr);
+
+					associated_clients_update(adapter, updated, STA_INFO_UPDATE_ALL);
+				}
+			}
+		}
+	} else
+#endif
+	if (action == RTW_ACT_SELF_PROTECTED_MESH_CONF) {
+		u8 *ies = NULL;
+		u16 ies_len = 0;
+
+		_rtw_spinlock_bh(&(plink_ctl->lock));
+
+		plink = _rtw_mesh_plink_get(adapter, peer_addr);
+		if (!plink)
+			goto release_plink_ctl;
+
+		if (tx == _FALSE) {
+			ies = plink->rx_conf_ies;
+			ies_len = plink->rx_conf_ies_len;
+			plink->rx_conf_ies = NULL;
+			plink->rx_conf_ies_len = 0;
+
+			plink->llid = mpm_info.plid_v;
+			plink->plid = mpm_info.llid_v;
+			plink->peer_aid = mpm_info.aid_v;
+			if (mpm_info.pid_v == 1)
+				_rtw_memcpy(plink->chosen_pmk, mpm_info.chosen_pmk, 16);
+		}
+		#ifdef CONFIG_RTW_MESH_DRIVER_AID
+		else {
+			ies = plink->tx_conf_ies;
+			ies_len = plink->tx_conf_ies_len;
+			plink->tx_conf_ies = NULL;
+			plink->tx_conf_ies_len = 0;
+		}
+		#endif
+
+		if (ies && ies_len)
+			rtw_mfree(ies, ies_len);
+
+		#ifndef CONFIG_RTW_MESH_DRIVER_AID
+		if (tx == _TRUE)
+			goto release_plink_ctl; /* no need to copy tx conf ies */
+		#endif
+
+		/* copy mesh confirm IEs */
+		if (mpm_info.pid_v == 1) /* not include MIC & encrypted AMPE */
+			ies_len = (mic_ie - fhead) - sizeof(struct rtw_ieee80211_hdr_3addr) - 2;
+		else
+			ies_len = flen - sizeof(struct rtw_ieee80211_hdr_3addr) - 2;
+
+		ies = rtw_zmalloc(ies_len);
+		if (ies) {
+			_rtw_memcpy(ies, fhead + sizeof(struct rtw_ieee80211_hdr_3addr) + 2, ies_len);
+			if (tx == _FALSE) {
+				plink->rx_conf_ies = ies;
+				plink->rx_conf_ies_len = ies_len;
+			}
+			#ifdef CONFIG_RTW_MESH_DRIVER_AID	
+			else {
+				plink->tx_conf_ies = ies;
+				plink->tx_conf_ies_len = ies_len;
+			}
+			#endif
+		}
+
+release_plink_ctl:
+		_rtw_spinunlock_bh(&(plink_ctl->lock));
+	}
+
+mpm_log:
+	rtw_mpm_info_msg(&mpm_info, mpm_log_buf);
+	RTW_INFO("RTW_%s:%s %s\n"
+		, (tx == _TRUE) ? "Tx" : "Rx"
+		, action_self_protected_str(action)
+		, mpm_log_buf
+	);
+
+	ret = 1;
+
+exit:
+	if (nbuf) {
+		if (ret == 1) {
+			*buf = nbuf;
+			*len = nlen;
+		} else
+			rtw_mfree(nbuf, nlen);
+	}
+
+	return ret;
+}
+
+static int rtw_mesh_check_frames(_adapter *adapter, const u8 **buf, size_t *len, u8 tx)
+{
+	int is_mesh_frame = -1;
+	const u8 *frame_body;
+	u8 category, action;
+
+	frame_body = *buf + sizeof(struct rtw_ieee80211_hdr_3addr);
+	category = frame_body[0];
+
+	if (category == RTW_WLAN_CATEGORY_SELF_PROTECTED) {
+		action = frame_body[1];
+		switch (action) {
+		case RTW_ACT_SELF_PROTECTED_MESH_OPEN:
+		case RTW_ACT_SELF_PROTECTED_MESH_CONF:
+		case RTW_ACT_SELF_PROTECTED_MESH_CLOSE:
+			rtw_mpm_check_frames(adapter, action, buf, len, tx);
+			is_mesh_frame = action;
+			break;
+		case RTW_ACT_SELF_PROTECTED_MESH_GK_INFORM:
+		case RTW_ACT_SELF_PROTECTED_MESH_GK_ACK:
+			RTW_INFO("RTW_%s:%s\n", (tx == _TRUE) ? "Tx" : "Rx", action_self_protected_str(action));
+			is_mesh_frame = action;
+			break;
+		default:
+			break;
+		};
+	}
+
+	return is_mesh_frame;
+}
+
+int rtw_mesh_check_frames_tx(_adapter *adapter, const u8 **buf, size_t *len)
+{
+	return rtw_mesh_check_frames(adapter, buf, len, _TRUE);
+}
+
+int rtw_mesh_check_frames_rx(_adapter *adapter, const u8 *buf, size_t len)
+{
+	return rtw_mesh_check_frames(adapter, &buf, &len, _FALSE);
+}
+
+int rtw_mesh_on_auth(_adapter *adapter, union recv_frame *rframe)
+{
+	u8 *whdr = rframe->u.hdr.rx_data;
+
+#if CONFIG_RTW_MACADDR_ACL
+	if (rtw_access_ctrl(adapter, get_addr2_ptr(whdr)) == _FALSE)
+		return _SUCCESS;
+#endif
+
+	if (!rtw_mesh_plink_get(adapter, get_addr2_ptr(whdr))) {
+		#if CONFIG_RTW_MESH_ACNODE_PREVENT
+		rtw_mesh_acnode_set_notify_etime(adapter, whdr);
+		#endif
+
+		if (adapter_to_rfctl(adapter)->offch_state == OFFCHS_NONE)
+			issue_probereq(adapter, &adapter->mlmepriv.cur_network.network.mesh_id, get_addr2_ptr(whdr));
+
+		/* only peer being added (checked by notify conditions) is allowed */
+		return _SUCCESS;
+	}
+
+	rtw_cfg80211_rx_mframe(adapter, rframe, NULL);
+	return _SUCCESS;
+}
+
+unsigned int on_action_self_protected(_adapter *adapter, union recv_frame *rframe)
+{
+	unsigned int ret = _FAIL;
+	struct sta_info *sta = NULL;
+	u8 *pframe = rframe->u.hdr.rx_data;
+	uint frame_len = rframe->u.hdr.len;
+	u8 *frame_body = (u8 *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 category;
+	u8 action;
+
+	/* check RA matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(adapter), GetAddr1Ptr(pframe), ETH_ALEN))
+		goto exit;
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_SELF_PROTECTED)
+		goto exit;
+
+	action = frame_body[1];
+	switch (action) {
+	case RTW_ACT_SELF_PROTECTED_MESH_OPEN:
+	case RTW_ACT_SELF_PROTECTED_MESH_CONF:
+	case RTW_ACT_SELF_PROTECTED_MESH_CLOSE:
+	case RTW_ACT_SELF_PROTECTED_MESH_GK_INFORM:
+	case RTW_ACT_SELF_PROTECTED_MESH_GK_ACK:
+		if (!(MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter)))
+			goto exit;
+#ifdef CONFIG_IOCTL_CFG80211
+		#if CONFIG_RTW_MACADDR_ACL
+		if (rtw_access_ctrl(adapter, get_addr2_ptr(pframe)) == _FALSE)
+			goto exit;
+		#endif
+		#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+		if (rtw_mesh_cto_mgate_required(adapter)
+			/* only peer being added (checked by notify conditions) is allowed */
+			&& !rtw_mesh_plink_get(adapter, get_addr2_ptr(pframe)))
+			goto exit;
+		#endif
+		rtw_cfg80211_rx_action(adapter, rframe, NULL);
+		ret = _SUCCESS;
+#endif /* CONFIG_IOCTL_CFG80211 */
+		break;
+	default:
+		break;
+	}
+
+exit:
+	return ret;
+}
+
+const u8 ae_to_mesh_ctrl_len[] = {
+	6,
+	12, /* MESH_FLAGS_AE_A4 */
+	18, /* MESH_FLAGS_AE_A5_A6 */
+	0,
+};
+
+unsigned int on_action_mesh(_adapter *adapter, union recv_frame *rframe)
+{
+	unsigned int ret = _FAIL;
+	struct sta_info *sta = NULL;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	u8 *pframe = rframe->u.hdr.rx_data;
+	uint frame_len = rframe->u.hdr.len;
+	u8 *frame_body = (u8 *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 category;
+	u8 action;
+
+	if (!MLME_IS_MESH(adapter))
+		goto exit;
+
+	/* check stainfo exist? */
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_MESH)
+		goto exit;
+
+	action = frame_body[1];
+	switch (action) {
+	case RTW_ACT_MESH_HWMP_PATH_SELECTION:
+		rtw_mesh_rx_path_sel_frame(adapter, rframe);
+		ret = _SUCCESS;
+		break;
+	default:
+		break;
+	}
+
+exit:
+	return ret;
+}
+
+bool rtw_mesh_update_bss_peering_status(_adapter *adapter, WLAN_BSSID_EX *bss)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	u8 num_of_peerings = stapriv->asoc_list_cnt;
+	bool accept_peerings = stapriv->asoc_list_cnt < mcfg->max_peer_links;
+	u8 *ie;
+	int ie_len;
+	bool updated = 0;
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	accept_peerings |= plink_ctl->acnode_rsvd;
+#endif
+
+	ie = rtw_get_ie(BSS_EX_TLV_IES(bss), WLAN_EID_MESH_CONFIG, &ie_len, BSS_EX_TLV_IES_LEN(bss));
+	if (!ie || ie_len != 7) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (GET_MESH_CONF_ELE_NUM_OF_PEERINGS(ie + 2) != num_of_peerings) {
+		SET_MESH_CONF_ELE_NUM_OF_PEERINGS(ie + 2, num_of_peerings);
+		updated = 1;
+	}
+
+	if (GET_MESH_CONF_ELE_ACCEPT_PEERINGS(ie + 2) != accept_peerings) {
+		SET_MESH_CONF_ELE_ACCEPT_PEERINGS(ie + 2, accept_peerings);
+		updated = 1;
+	}
+
+exit:
+	return updated;
+}
+
+bool rtw_mesh_update_bss_formation_info(_adapter *adapter, WLAN_BSSID_EX *bss)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	u8 cto_mgate = (minfo->num_gates || mcfg->dot11MeshGateAnnouncementProtocol);
+	u8 cto_as = 0;
+	u8 *ie;
+	int ie_len;
+	bool updated = 0;
+
+	ie = rtw_get_ie(BSS_EX_TLV_IES(bss), WLAN_EID_MESH_CONFIG, &ie_len,
+			BSS_EX_TLV_IES_LEN(bss));
+	if (!ie || ie_len != 7) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (GET_MESH_CONF_ELE_CTO_MGATE(ie + 2) != cto_mgate) {
+		SET_MESH_CONF_ELE_CTO_MGATE(ie + 2, cto_mgate);
+		updated = 1;
+	}
+
+	if (GET_MESH_CONF_ELE_CTO_AS(ie + 2) != cto_as) {
+		SET_MESH_CONF_ELE_CTO_AS(ie + 2, cto_as);
+		updated = 1;
+	}
+
+exit:
+	return updated;
+}
+
+bool rtw_mesh_update_bss_forwarding_state(_adapter *adapter, WLAN_BSSID_EX *bss)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	u8 forward = mcfg->dot11MeshForwarding;
+	u8 *ie;
+	int ie_len;
+	bool updated = 0;
+
+	ie = rtw_get_ie(BSS_EX_TLV_IES(bss), WLAN_EID_MESH_CONFIG, &ie_len,
+			BSS_EX_TLV_IES_LEN(bss));
+	if (!ie || ie_len != 7) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (GET_MESH_CONF_ELE_FORWARDING(ie + 2) != forward) {
+		SET_MESH_CONF_ELE_FORWARDING(ie + 2, forward);
+		updated = 1;
+	}
+
+exit:
+	return updated;
+}
+
+struct mesh_plink_ent *_rtw_mesh_plink_get(_adapter *adapter, const u8 *hwaddr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+	int i;
+
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++) {
+		if (plink_ctl->ent[i].valid == _TRUE
+			&& _rtw_memcmp(plink_ctl->ent[i].addr, hwaddr, ETH_ALEN) == _TRUE
+		) {
+			ent = &plink_ctl->ent[i];
+			break;
+		}
+	}
+
+	return ent;
+}
+
+struct mesh_plink_ent *rtw_mesh_plink_get(_adapter *adapter, const u8 *hwaddr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	ent = _rtw_mesh_plink_get(adapter, hwaddr);
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	return ent;
+}
+
+struct mesh_plink_ent *rtw_mesh_plink_get_no_estab_by_idx(_adapter *adapter, u8 idx)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+	int i, j = 0;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++) {
+		if (plink_ctl->ent[i].valid == _TRUE
+			&& plink_ctl->ent[i].plink_state != RTW_MESH_PLINK_ESTAB
+		) {
+			if (j == idx) {
+				ent = &plink_ctl->ent[i];
+				break;
+			}
+			j++;
+		}
+	}
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	return ent;
+}
+
+int _rtw_mesh_plink_add(_adapter *adapter, const u8 *hwaddr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+	u8 exist = _FALSE;
+	int i;
+
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++) {
+		if (plink_ctl->ent[i].valid == _TRUE
+			&& _rtw_memcmp(plink_ctl->ent[i].addr, hwaddr, ETH_ALEN) == _TRUE
+		) {
+			ent = &plink_ctl->ent[i];
+			exist = _TRUE;
+			break;
+		}
+
+		if (ent == NULL && plink_ctl->ent[i].valid == _FALSE)
+			ent = &plink_ctl->ent[i];
+	}
+
+	if (exist == _FALSE && ent) {
+		_rtw_memcpy(ent->addr, hwaddr, ETH_ALEN);
+		ent->valid = _TRUE;
+		#ifdef CONFIG_RTW_MESH_AEK
+		ent->aek_valid = 0;
+		#endif
+		ent->llid = 0;
+		ent->plid = 0;
+		_rtw_memset(ent->chosen_pmk, 0, 16);
+		#ifdef CONFIG_RTW_MESH_AEK
+		_rtw_memset(ent->sel_pcs, 0, 4);
+		_rtw_memset(ent->l_nonce, 0, 32);
+		_rtw_memset(ent->p_nonce, 0, 32);
+		#endif
+		ent->plink_state = RTW_MESH_PLINK_LISTEN;
+		#ifndef CONFIG_RTW_MESH_DRIVER_AID
+		ent->aid = 0;
+		#endif
+		ent->peer_aid = 0;
+		SET_PEER_CONF_DISABLED(ent);
+		SET_CTO_MGATE_CONF_DISABLED(ent);
+		plink_ctl->num++;
+	}
+
+	return exist == _TRUE ? RTW_ALREADY : (ent ? _SUCCESS : _FAIL);
+}
+
+int rtw_mesh_plink_add(_adapter *adapter, const u8 *hwaddr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	int ret;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	ret = _rtw_mesh_plink_add(adapter, hwaddr);
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	return ret;
+}
+
+int rtw_mesh_plink_set_state(_adapter *adapter, const u8 *hwaddr, u8 state)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	ent = _rtw_mesh_plink_get(adapter, hwaddr);
+	if (ent)
+		ent->plink_state = state;
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	return ent ? _SUCCESS : _FAIL;
+}
+
+#ifdef CONFIG_RTW_MESH_AEK
+int rtw_mesh_plink_set_aek(_adapter *adapter, const u8 *hwaddr, const u8 *aek)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	ent = _rtw_mesh_plink_get(adapter, hwaddr);
+	if (ent) {
+		_rtw_memcpy(ent->aek, aek, 32);
+		ent->aek_valid = 1;
+	}
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	return ent ? _SUCCESS : _FAIL;
+}
+#endif
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+int rtw_mesh_plink_set_peer_conf_timeout(_adapter *adapter, const u8 *hwaddr)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	ent = _rtw_mesh_plink_get(adapter, hwaddr);
+	if (ent) {
+		if (IS_PEER_CONF_DISABLED(ent))
+			SET_PEER_CONF_END_TIME(ent, mcfg->peer_sel_policy.peer_conf_timeout_ms);
+	}
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	return ent ? _SUCCESS : _FAIL;
+}
+#endif
+
+void _rtw_mesh_plink_del_ent(_adapter *adapter, struct mesh_plink_ent *ent)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+
+	ent->valid = _FALSE;
+	#ifdef CONFIG_RTW_MESH_DRIVER_AID
+	if (ent->tx_conf_ies && ent->tx_conf_ies_len)
+		rtw_mfree(ent->tx_conf_ies, ent->tx_conf_ies_len);
+	ent->tx_conf_ies = NULL;
+	ent->tx_conf_ies_len = 0;
+	#endif
+	if (ent->rx_conf_ies && ent->rx_conf_ies_len)
+		rtw_mfree(ent->rx_conf_ies, ent->rx_conf_ies_len);
+	ent->rx_conf_ies = NULL;
+	ent->rx_conf_ies_len = 0;
+	if (ent->scanned)
+		ent->scanned = NULL;
+	plink_ctl->num--;
+}
+
+int rtw_mesh_plink_del(_adapter *adapter, const u8 *hwaddr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent = NULL;
+	u8 exist = _FALSE;
+	int i;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++) {
+		if (plink_ctl->ent[i].valid == _TRUE
+			&& _rtw_memcmp(plink_ctl->ent[i].addr, hwaddr, ETH_ALEN) == _TRUE
+		) {
+			ent = &plink_ctl->ent[i];
+			exist = _TRUE;
+			break;
+		}
+	}
+
+	if (exist == _TRUE)
+		_rtw_mesh_plink_del_ent(adapter, ent);
+
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	return exist == _TRUE ? _SUCCESS : RTW_ALREADY;
+}
+
+void rtw_mesh_plink_ctl_init(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	int i;
+
+	_rtw_spinlock_init(&plink_ctl->lock);
+	plink_ctl->num = 0;
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++)
+		plink_ctl->ent[i].valid = _FALSE;
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	_rtw_init_queue(&plink_ctl->peer_blacklist);
+#endif
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	_rtw_init_queue(&plink_ctl->cto_mgate_blacklist);
+#endif
+}
+
+void rtw_mesh_plink_ctl_deinit(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent;
+	int i;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++) {
+		ent = &plink_ctl->ent[i];
+		#ifdef CONFIG_RTW_MESH_DRIVER_AID
+		if (ent->tx_conf_ies && ent->tx_conf_ies_len)
+			rtw_mfree(ent->tx_conf_ies, ent->tx_conf_ies_len);
+		#endif
+		if (ent->rx_conf_ies && ent->rx_conf_ies_len)
+			rtw_mfree(ent->rx_conf_ies, ent->rx_conf_ies_len);
+	}
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+	_rtw_spinlock_free(&plink_ctl->lock);
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	rtw_mesh_peer_blacklist_flush(adapter);
+	_rtw_deinit_queue(&plink_ctl->peer_blacklist);
+#endif
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	rtw_mesh_cto_mgate_blacklist_flush(adapter);
+	_rtw_deinit_queue(&plink_ctl->cto_mgate_blacklist);
+#endif
+}
+
+void dump_mesh_plink_ctl(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *ent;
+	int i;
+
+	RTW_PRINT_SEL(sel, "num:%u\n", plink_ctl->num);
+	#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	RTW_PRINT_SEL(sel, "acnode_rsvd:%u\n", plink_ctl->acnode_rsvd);
+	#endif
+
+	for (i = 0; i < RTW_MESH_MAX_PEER_CANDIDATES; i++)  {
+		ent = &plink_ctl->ent[i];
+		if (!ent->valid)
+			continue;
+
+		RTW_PRINT_SEL(sel, "\n");
+		RTW_PRINT_SEL(sel, "peer:"MAC_FMT"\n", MAC_ARG(ent->addr));
+		RTW_PRINT_SEL(sel, "plink_state:%s\n", rtw_mesh_plink_str(ent->plink_state));
+
+		#ifdef CONFIG_RTW_MESH_AEK
+		if (ent->aek_valid)
+			RTW_PRINT_SEL(sel, "aek:"KEY_FMT KEY_FMT"\n", KEY_ARG(ent->aek), KEY_ARG(ent->aek + 16));
+		#endif
+
+		RTW_PRINT_SEL(sel, "llid:%u, plid:%u\n", ent->llid, ent->plid);
+		#ifndef CONFIG_RTW_MESH_DRIVER_AID
+		RTW_PRINT_SEL(sel, "aid:%u\n", ent->aid);
+		#endif
+		RTW_PRINT_SEL(sel, "peer_aid:%u\n", ent->peer_aid);
+
+		RTW_PRINT_SEL(sel, "chosen_pmk:"KEY_FMT"\n", KEY_ARG(ent->chosen_pmk));
+
+		#ifdef CONFIG_RTW_MESH_AEK
+		RTW_PRINT_SEL(sel, "sel_pcs:%02x%02x%02x%02x\n"
+			, ent->sel_pcs[0], ent->sel_pcs[1], ent->sel_pcs[2], ent->sel_pcs[3]);
+		RTW_PRINT_SEL(sel, "l_nonce:"KEY_FMT KEY_FMT"\n", KEY_ARG(ent->l_nonce), KEY_ARG(ent->l_nonce + 16));
+		RTW_PRINT_SEL(sel, "p_nonce:"KEY_FMT KEY_FMT"\n", KEY_ARG(ent->p_nonce), KEY_ARG(ent->p_nonce + 16));
+		#endif
+
+		#ifdef CONFIG_RTW_MESH_DRIVER_AID
+		RTW_PRINT_SEL(sel, "tx_conf_ies:%p, len:%u\n", ent->tx_conf_ies, ent->tx_conf_ies_len);
+		#endif
+		RTW_PRINT_SEL(sel, "rx_conf_ies:%p, len:%u\n", ent->rx_conf_ies, ent->rx_conf_ies_len);
+		RTW_PRINT_SEL(sel, "scanned:%p\n", ent->scanned);
+
+		#if CONFIG_RTW_MESH_PEER_BLACKLIST
+		if (!IS_PEER_CONF_DISABLED(ent)) {
+			if (!IS_PEER_CONF_TIMEOUT(ent))
+				RTW_PRINT_SEL(sel, "peer_conf:%d\n", rtw_systime_to_ms(ent->peer_conf_end_time - rtw_get_current_time()));
+			else
+				RTW_PRINT_SEL(sel, "peer_conf:TIMEOUT\n");
+		}
+		#endif
+
+		#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+		if (!IS_CTO_MGATE_CONF_DISABLED(ent)) {
+			if (!IS_CTO_MGATE_CONF_TIMEOUT(ent))
+				RTW_PRINT_SEL(sel, "cto_mgate_conf:%d\n", rtw_systime_to_ms(ent->cto_mgate_conf_end_time - rtw_get_current_time()));
+			else
+				RTW_PRINT_SEL(sel, "cto_mgate_conf:TIMEOUT\n");
+		}
+		#endif
+	}
+}
+
+/* this function is called with plink_ctl being locked */
+static int rtw_mesh_peer_establish(_adapter *adapter, struct mesh_plink_ent *plink, struct sta_info *sta)
+{
+#ifndef DBG_RTW_MESH_PEER_ESTABLISH
+#define DBG_RTW_MESH_PEER_ESTABLISH 0
+#endif
+
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	u8 *tlv_ies;
+	u16 tlv_ieslen;
+	struct rtw_ieee802_11_elems elems;
+	int i;
+	u16 status = 0;
+	int ret = _FAIL;
+
+	if (!plink->rx_conf_ies || !plink->rx_conf_ies_len) {
+		RTW_INFO(FUNC_ADPT_FMT" no rx confirm from sta "MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+		goto exit;
+	}
+
+	if (plink->rx_conf_ies_len < 4) {
+		RTW_INFO(FUNC_ADPT_FMT" confirm from sta "MAC_FMT" too short\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+		goto exit;
+	}
+
+#ifdef CONFIG_RTW_MESH_DRIVER_AID
+	if (!plink->tx_conf_ies || !plink->tx_conf_ies_len) {
+		RTW_INFO(FUNC_ADPT_FMT" no tx confirm to sta "MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+		goto exit;
+	}
+
+	if (plink->tx_conf_ies_len < 4) {
+		RTW_INFO(FUNC_ADPT_FMT" confirm to sta "MAC_FMT" too short\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+		goto exit;
+	}
+#endif
+
+	tlv_ies = plink->rx_conf_ies + 4;
+	tlv_ieslen = plink->rx_conf_ies_len - 4;
+
+	if (DBG_RTW_MESH_PEER_ESTABLISH)
+		dump_ies(RTW_DBGDUMP, tlv_ies, tlv_ieslen);
+
+	if (rtw_ieee802_11_parse_elems(tlv_ies, tlv_ieslen, &elems, 1) == ParseFailed) {
+		RTW_INFO(FUNC_ADPT_FMT" sta "MAC_FMT" sent invalid confirm\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+		goto exit;
+	}
+
+	SET_PEER_CONF_DISABLED(plink);
+	if (rtw_bss_is_cto_mgate(&plink->scanned->network)
+		&& !rtw_bss_is_forwarding(&plink->scanned->network))
+		SET_CTO_MGATE_CONF_END_TIME(plink, mcfg->peer_sel_policy.cto_mgate_conf_timeout_ms);
+	else
+		SET_CTO_MGATE_CONF_DISABLED(plink);
+
+	sta->state &= (~WIFI_FW_AUTH_SUCCESS);
+	sta->state |= WIFI_FW_ASSOC_STATE;
+
+	rtw_ap_parse_sta_capability(adapter, sta, plink->rx_conf_ies);
+
+	status = rtw_ap_parse_sta_supported_rates(adapter, sta, tlv_ies, tlv_ieslen);
+	if (status != _STATS_SUCCESSFUL_)
+		goto exit;
+
+	status = rtw_ap_parse_sta_security_ie(adapter, sta, &elems);
+	if (status != _STATS_SUCCESSFUL_) {
+		RTW_INFO(FUNC_ADPT_FMT" security check fail, status=%u\n", FUNC_ADPT_ARG(adapter), status);
+		goto exit;
+	}
+
+	rtw_ap_parse_sta_wmm_ie(adapter, sta, tlv_ies, tlv_ieslen);
+#ifdef CONFIG_RTS_FULL_BW
+	/*check vendor IE*/
+	rtw_parse_sta_vendor_ie_8812(adapter, sta, tlv_ies, tlv_ieslen);
+#endif/*CONFIG_RTS_FULL_BW*/
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	if (elems.tbtx_cap && elems.tbtx_cap_len != 0) {
+		if(rtw_is_tbtx_capabilty(elems.tbtx_cap, elems.tbtx_cap_len))
+			sta->tbtx_enable = _TRUE;
+	}
+#endif
+
+	rtw_ap_parse_sta_ht_ie(adapter, sta, &elems);
+	rtw_ap_parse_sta_vht_ie(adapter, sta, &elems);
+
+	/* AID */
+#ifdef CONFIG_RTW_MESH_DRIVER_AID
+	sta->phl_sta->aid = RTW_GET_LE16(plink->tx_conf_ies + 2);
+#else
+	sta->phl_sta->aid = plink->aid;
+#endif
+	stapriv->sta_aid[sta->phl_sta->aid - 1] = sta;
+	RTW_INFO(FUNC_ADPT_FMT" sta "MAC_FMT" aid:%u\n"
+		, FUNC_ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr), sta->phl_sta->aid);
+
+	sta->state &= (~WIFI_FW_ASSOC_STATE);
+	sta->state |= WIFI_FW_ASSOC_SUCCESS;
+
+	sta->local_mps = RTW_MESH_PS_ACTIVE;
+
+	rtw_ewma_err_rate_init(&sta->metrics.err_rate);
+	rtw_ewma_err_rate_add(&sta->metrics.err_rate, 1);
+	/* init data_rate to 1M */
+	sta->metrics.data_rate = 10;
+	sta->alive = _TRUE;
+
+	rtw_stapriv_asoc_list_lock(stapriv);
+	if (rtw_is_list_empty(&sta->asoc_list)) {
+		STA_SET_MESH_PLINK(sta, plink);
+		/* TBD: up layer timeout mechanism */
+		/* sta->expire_to = mcfg->plink_timeout / 2; */
+		rtw_stapriv_asoc_list_add(stapriv, sta);
+	}
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+	bss_cap_update_on_sta_join(adapter, sta);
+	sta_info_update(adapter, sta);
+	report_add_sta_event(adapter, sta->phl_sta->mac_addr);
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+int rtw_mesh_set_plink_state(_adapter *adapter, const u8 *mac, u8 plink_state)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *plink = NULL;
+	_irqL irqL2;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct sta_info *sta = NULL;
+	struct sta_info *del_sta = NULL;
+	int ret = _SUCCESS;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock), &irqL2);
+
+	plink = _rtw_mesh_plink_get(adapter, mac);
+	if (!plink) {
+		ret = _FAIL;
+		goto release_plink_ctl;
+	}
+
+	plink->plink_state = plink_state;
+
+	#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	if (plink_state == RTW_MESH_PLINK_OPN_SNT) {
+		if (rtw_mesh_scanned_is_acnode_confirmed(adapter, plink->scanned)
+			&& rtw_mesh_acnode_prevent_allow_sacrifice(adapter)
+		) {
+			struct sta_info *sac = rtw_mesh_acnode_prevent_pick_sacrifice(adapter);
+
+			if (sac) {
+				del_sta = sac;
+				rtw_stapriv_asoc_list_lock(stapriv);
+				if (!rtw_is_list_empty(&del_sta->asoc_list))
+					rtw_stapriv_asoc_list_del(stapriv, del_sta);
+				rtw_stapriv_asoc_list_unlock(stapriv);
+				RTW_INFO(FUNC_ADPT_FMT" sacrifice "MAC_FMT" for acnode\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(del_sta->phl_sta->mac_addr));
+			}
+		}
+	} else
+	#endif
+	if (plink_state == RTW_MESH_PLINK_OPN_RCVD
+		|| plink_state == RTW_MESH_PLINK_CNF_RCVD
+		|| plink_state == RTW_MESH_PLINK_ESTAB
+	) {
+		sta = rtw_get_stainfo(stapriv, mac);
+		if (!sta) {
+			sta = rtw_alloc_stainfo(stapriv, mac, PHL_CMD_DIRECTLY);
+			if (!sta)
+				goto release_plink_ctl;
+		}
+
+		if (plink_state == RTW_MESH_PLINK_ESTAB) {
+			if (rtw_mesh_peer_establish(adapter, plink, sta) != _SUCCESS) {
+				del_sta = sta;
+				ret = _FAIL;
+				goto release_plink_ctl;
+			}
+		}
+	}
+	else if (plink_state == RTW_MESH_PLINK_HOLDING) {
+		del_sta = rtw_get_stainfo(stapriv, mac);
+		if (!del_sta)
+			goto release_plink_ctl;
+
+		rtw_stapriv_asoc_list_lock(stapriv);
+		if (!rtw_is_list_empty(&del_sta->asoc_list))
+			rtw_stapriv_asoc_list_del(stapriv, del_sta);
+		rtw_stapriv_asoc_list_unlock(stapriv);
+	}
+
+release_plink_ctl:
+	_rtw_spinunlock_bh(&(plink_ctl->lock), &irqL2);
+
+	if (del_sta) {
+		u8 sta_addr[ETH_ALEN];
+		u8 updated = _FALSE;
+
+		_rtw_memcpy(sta_addr, del_sta->phl_sta->mac_addr, ETH_ALEN);
+		updated = ap_free_sta(adapter, del_sta, 0, 0, 0, 1);
+		rtw_mesh_expire_peer(stapriv->padapter, sta_addr);
+
+		associated_clients_update(adapter, updated, STA_INFO_UPDATE_ALL);
+	}
+
+	return ret;
+}
+
+struct mesh_set_plink_cmd_parm {
+	const u8 *mac;
+	u8 plink_state;
+};
+
+u8 rtw_mesh_set_plink_state_cmd_hdl(_adapter *adapter, u8 *parmbuf)
+{
+	struct mesh_set_plink_cmd_parm *parm = (struct mesh_set_plink_cmd_parm *)parmbuf;
+
+	if (rtw_mesh_set_plink_state(adapter, parm->mac, parm->plink_state) == _SUCCESS)
+		return	H2C_SUCCESS;
+
+	return H2C_CMD_FAIL;
+}
+
+u8 rtw_mesh_set_plink_state_cmd(_adapter *adapter, const u8 *mac, u8 plink_state)
+{
+	struct cmd_obj *cmdobj;
+	struct mesh_set_plink_cmd_parm *parm;
+	struct cmd_priv *cmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 res = _SUCCESS;
+
+	/* prepare cmd parameter */
+	parm = rtw_zmalloc(sizeof(*parm));
+	if (parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	parm->mac = mac;
+	parm->plink_state = plink_state;
+
+	/* need enqueue, prepare cmd_obj and enqueue */
+	cmdobj = rtw_zmalloc(sizeof(*cmdobj));
+	if (cmdobj == NULL) {
+		res = _FAIL;
+		rtw_mfree(parm, sizeof(*parm));
+		goto exit;
+	}
+	cmdobj->padapter = adapter;
+
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_MESH_PLINK_STATE);
+	cmdobj->sctx = &sctx;
+	rtw_sctx_init(&sctx, 2000);
+
+	res = rtw_enqueue_cmd(cmdpriv, cmdobj);
+	if (res == _SUCCESS) {
+		rtw_sctx_wait(&sctx, __func__);
+		_rtw_mutex_lock_interruptible(&cmdpriv->sctx_mutex, NULL);
+		if (sctx.status == RTW_SCTX_SUBMITTED)
+			cmdobj->sctx = NULL;
+		_rtw_mutex_unlock(&cmdpriv->sctx_mutex, NULL);
+		if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+			res = _FAIL;
+	}
+
+exit:
+	return res;
+}
+
+void rtw_mesh_expire_peer_notify(_adapter *adapter, const u8 *peer_addr)
+{
+	u8 null_ssid[2] = {0, 0};
+
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_notify_new_peer_candidate(adapter->rtw_wdev
+		, peer_addr
+		, null_ssid
+		, 2
+		, 0
+		, GFP_ATOMIC
+	);
+#endif
+
+	return;
+}
+
+static u8 *rtw_mesh_construct_peer_mesh_close(_adapter *adapter, struct mesh_plink_ent *plink, u16 reason, u32 *len)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	u8 *frame = NULL, *pos;
+	u32 flen;
+	struct rtw_ieee80211_hdr *whdr;
+
+	if (minfo->mesh_auth_id && !MESH_PLINK_AEK_VALID(plink))
+		goto exit;
+
+	flen = sizeof(struct rtw_ieee80211_hdr_3addr)
+		+ 2 /* category, action */
+		+ 2 + minfo->mesh_id_len /* mesh id */
+		+ 2 + 8 + (minfo->mesh_auth_id ? 16 : 0) /* mpm */
+		+ (minfo->mesh_auth_id ? 2 + 16 /* AES_BLOCK_SIZE */ : 0) /* mic */
+		+ (minfo->mesh_auth_id ? 70 : 0) /* ampe */
+		;
+
+	pos = frame = rtw_zmalloc(flen);
+	if (!frame)
+		goto exit;
+
+	whdr = (struct rtw_ieee80211_hdr *)frame;
+	_rtw_memcpy(whdr->addr1, adapter_mac_addr(adapter), ETH_ALEN);
+	_rtw_memcpy(whdr->addr2, plink->addr, ETH_ALEN);
+	_rtw_memcpy(whdr->addr3, adapter_mac_addr(adapter), ETH_ALEN);
+
+	set_frame_sub_type(frame, WIFI_ACTION);
+
+	pos += sizeof(struct rtw_ieee80211_hdr_3addr);
+	*(pos++) = RTW_WLAN_CATEGORY_SELF_PROTECTED;
+	*(pos++) = RTW_ACT_SELF_PROTECTED_MESH_CLOSE;
+
+	pos = rtw_set_ie_mesh_id(pos, NULL, minfo->mesh_id, minfo->mesh_id_len);
+
+	pos = rtw_set_ie_mpm(pos, NULL
+		, minfo->mesh_auth_id ? 1 : 0
+		, plink->plid
+		, &plink->llid
+		, &reason
+		, minfo->mesh_auth_id ? plink->chosen_pmk : NULL);
+
+#ifdef CONFIG_RTW_MESH_AEK
+	if (minfo->mesh_auth_id) {
+		u8 ampe_buf[70];
+		int enc_ret;
+
+		*pos = WLAN_EID_MIC;
+		*(pos + 1) = 16 /* AES_BLOCK_SIZE */;
+
+		ampe_buf[0] = WLAN_EID_AMPE;
+		ampe_buf[1] = 68;
+		_rtw_memcpy(ampe_buf + 2, plink->sel_pcs, 4);
+		_rtw_memcpy(ampe_buf + 6, plink->p_nonce, 32);
+		_rtw_memcpy(ampe_buf + 38, plink->l_nonce, 32);
+
+		enc_ret = rtw_mpm_ampe_enc(adapter, plink
+			, frame + sizeof(struct rtw_ieee80211_hdr_3addr)
+			, pos, ampe_buf, 1);
+		if (enc_ret != _SUCCESS) {
+			rtw_mfree(frame, flen);
+			frame = NULL;
+			goto exit;
+		}
+	}
+#endif
+
+	*len = flen;
+
+exit:
+	return frame;
+}
+
+void _rtw_mesh_expire_peer_ent(_adapter *adapter, struct mesh_plink_ent *plink)
+{
+#if defined(CONFIG_RTW_MESH_STA_DEL_DISASOC)
+	_rtw_mesh_plink_del_ent(adapter, plink);
+	rtw_cfg80211_indicate_sta_disassoc(adapter, plink->addr, 0);
+#else
+	u8 *frame = NULL;
+	u32 flen;
+
+	if (plink->plink_state == RTW_MESH_PLINK_ESTAB)
+		frame = rtw_mesh_construct_peer_mesh_close(adapter, plink, WLAN_REASON_MESH_CLOSE, &flen);
+
+	if (frame) {
+		struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+		struct wireless_dev *wdev = adapter->rtw_wdev;
+		s32 freq = rtw_ch2freq(mlmeext->chandef.chan);
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+		rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, flen, GFP_ATOMIC);
+		#else
+		cfg80211_rx_action(adapter->pnetdev, freq, frame, flen, GFP_ATOMIC);
+		#endif
+
+		rtw_mfree(frame, flen);
+	} else {
+		rtw_mesh_expire_peer_notify(adapter, plink->addr);
+		RTW_INFO(FUNC_ADPT_FMT" set "MAC_FMT" plink unknown\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(plink->addr));
+		plink->plink_state = RTW_MESH_PLINK_UNKNOWN;
+	}
+#endif
+}
+
+void rtw_mesh_expire_peer(_adapter *adapter, const u8 *peer_addr)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+	struct mesh_plink_ent *plink;
+
+	_rtw_spinlock_bh(&(plink_ctl->lock));
+
+	plink = _rtw_mesh_plink_get(adapter, peer_addr);
+	if (!plink)
+		goto exit;
+
+	_rtw_mesh_expire_peer_ent(adapter, plink);
+
+exit:
+	_rtw_spinunlock_bh(&(plink_ctl->lock));
+}
+
+u8 rtw_mesh_ps_annc(_adapter *adapter, u8 ps)
+{
+	_list *head, *list;
+	struct sta_info *sta;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	u8 sta_alive_num = 0, i;
+	char sta_alive_list[NUM_STA];
+	u8 annc_cnt = 0;
+
+	if (rtw_linked_check(adapter) == _FALSE)
+		goto exit;
+
+	rtw_stapriv_asoc_list_lock(stapriv);
+
+	head = &stapriv->asoc_list;
+	list = get_next(head);
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		int stainfo_offset;
+
+		sta = LIST_CONTAINOR(list, struct sta_info, asoc_list);
+		list = get_next(list);
+
+		stainfo_offset = rtw_stainfo_offset(stapriv, sta);
+		if (stainfo_offset_valid(stainfo_offset))
+			sta_alive_list[sta_alive_num++] = stainfo_offset;
+	}
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+	for (i = 0; i < sta_alive_num; i++) {
+		sta = rtw_get_stainfo_by_offset(stapriv, sta_alive_list[i]);
+		if (!sta)
+			continue;
+
+		issue_qos_nulldata(adapter, sta->phl_sta->mac_addr, 7, ps, 3, PS_ANNC_DRV_RETRY_INT_MS, _FALSE);
+		annc_cnt++;
+	}
+
+exit:
+	return annc_cnt;
+}
+
+static void mpath_tx_tasklet_hdl(void *priv)
+{
+	_adapter *adapter = (_adapter *)priv;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct xmit_frame *xframe;
+	_list *list, *head;
+	_list tmp;
+	u32 tmp_len;
+	s32 res;
+
+	_rtw_init_listhead(&tmp);
+
+	while (1) {
+		tmp_len = 0;
+		_rtw_spinlock_bh(&minfo->mpath_tx_queue.lock);
+		if (minfo->mpath_tx_queue_len) {
+			rtw_list_splice_init(&minfo->mpath_tx_queue.queue, &tmp);
+			tmp_len = minfo->mpath_tx_queue_len;
+			minfo->mpath_tx_queue_len = 0;
+		}
+		_rtw_spinunlock_bh(&minfo->mpath_tx_queue.lock);
+
+		if (!tmp_len)
+			break;
+
+		head = &tmp;
+		list = get_next(head);
+		while (rtw_end_of_queue_search(head, list) == _FALSE) {
+			xframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+			list = get_next(list);
+			rtw_list_delete(&xframe->list);
+			res = rtw_xmit_posthandle(adapter, xframe, xframe->pkt);
+			if (res < 0) {
+				#ifdef DBG_TX_DROP_FRAME
+				RTW_INFO("DBG_TX_DROP_FRAME %s rtw_xmit fail\n", __FUNCTION__);
+				#endif
+				adapter->xmitpriv.tx_drop++;
+			}
+		}
+	}
+}
+
+static void rtw_mpath_tx_queue_flush(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct xmit_frame *xframe;
+	_list *list, *head;
+	_list tmp;
+
+	_rtw_init_listhead(&tmp);
+
+	_rtw_spinlock_bh(&minfo->mpath_tx_queue.lock);
+	rtw_list_splice_init(&minfo->mpath_tx_queue.queue, &tmp);
+	minfo->mpath_tx_queue_len = 0;
+	_rtw_spinunlock_bh(&minfo->mpath_tx_queue.lock);
+
+	head = &tmp;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		xframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+		list = get_next(list);
+		rtw_list_delete(&xframe->list);
+		rtw_free_xmitframe(&adapter->xmitpriv, xframe);
+	}
+}
+
+#ifdef PLATFORM_LINUX /* 3.10 ~ 4.13 checked */
+#if defined(CONFIG_SLUB)
+#include <linux/slub_def.h>
+#elif defined(CONFIG_SLAB)
+#include <linux/slab_def.h>
+#endif
+typedef struct kmem_cache rtw_mcache;
+#endif
+
+rtw_mcache *rtw_mcache_create(const char *name, size_t size)
+{
+#ifdef PLATFORM_LINUX /* 3.10 ~ 4.13 checked */
+	return kmem_cache_create(name, size, 0, 0, NULL);
+#else
+	#error "TBD\n";
+#endif
+}
+
+void rtw_mcache_destroy(rtw_mcache *s)
+{
+#ifdef PLATFORM_LINUX /* 3.10 ~ 4.13 checked */
+	kmem_cache_destroy(s);
+#else
+	#error "TBD\n";
+#endif
+}
+
+void *_rtw_mcache_alloc(rtw_mcache *cachep)
+{
+#ifdef PLATFORM_LINUX /* 3.10 ~ 4.13 checked */
+	return kmem_cache_alloc(cachep, GFP_ATOMIC);
+#else
+	#error "TBD\n";
+#endif
+}
+
+void _rtw_mcache_free(rtw_mcache *cachep, void *objp)
+{
+#ifdef PLATFORM_LINUX /* 3.10 ~ 4.13 checked */
+	kmem_cache_free(cachep, objp);
+#else
+	#error "TBD\n";
+#endif
+}
+
+#ifdef DBG_MEM_ALLOC
+inline void *dbg_rtw_mcache_alloc(rtw_mcache *cachep, const enum mstat_f flags, const char *func, const int line)
+{
+	void *p;
+	u32 sz = cachep->size;
+
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%u)\n", func, line, __func__, sz);
+
+	p = _rtw_mcache_alloc(cachep);
+
+	rtw_mstat_update(
+		flags
+		, p ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, sz
+	);
+
+	return p;
+}
+
+inline void dbg_rtw_mcache_free(rtw_mcache *cachep, void *pbuf, const enum mstat_f flags, const char *func, const int line)
+{
+	u32 sz = cachep->size;
+
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%u)\n", func, line, __func__, sz);
+
+	_rtw_mcache_free(cachep, pbuf);
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, sz
+	);
+}
+
+#define rtw_mcache_alloc(cachep) dbg_rtw_mcache_alloc(cachep, MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_mcache_free(cachep, objp) dbg_rtw_mcache_free(cachep, objp, MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#else
+#define rtw_mcache_alloc(cachep) _rtw_mcache_alloc(cachep)
+#define rtw_mcache_free(cachep, objp) _rtw_mcache_free(cachep, objp)
+#endif /* DBG_MEM_ALLOC */
+
+/* Mesh Received Cache */
+#define RTW_MRC_BUCKETS			256 /* must be a power of 2 */
+#define RTW_MRC_QUEUE_MAX_LEN	4
+#define RTW_MRC_TIMEOUT_MS		(3 * 1000)
+
+/**
+ * struct rtw_mrc_entry - entry in the Mesh Received Cache
+ *
+ * @seqnum: mesh sequence number of the frame
+ * @exp_time: expiration time of the entry
+ * @msa: mesh source address of the frame
+ * @list: hashtable list pointer
+ *
+ * The Mesh Received Cache keeps track of the latest received frames that
+ * have been received by a mesh interface and discards received frames
+ * that are found in the cache.
+ */
+struct rtw_mrc_entry {
+	rtw_hlist_node list;
+	systime exp_time;
+	u32 seqnum;
+	u8 msa[ETH_ALEN];
+};
+
+struct rtw_mrc {
+	rtw_hlist_head bucket[RTW_MRC_BUCKETS];
+	u32 idx_mask;
+	rtw_mcache *cache;
+};
+
+static int rtw_mrc_init(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	char cache_name[IFNAMSIZ + 8 + 1];
+	int i;
+
+	minfo->mrc = rtw_malloc(sizeof(struct rtw_mrc));
+	if (!minfo->mrc)
+		return -ENOMEM;
+	minfo->mrc->idx_mask = RTW_MRC_BUCKETS - 1;
+	for (i = 0; i < RTW_MRC_BUCKETS; i++)
+		rtw_hlist_head_init(&minfo->mrc->bucket[i]);
+
+	sprintf(cache_name, "rtw_mrc_%s", ADPT_ARG(adapter));
+	minfo->mrc->cache = rtw_mcache_create(cache_name, sizeof(struct rtw_mrc_entry));
+
+	return 0;
+}
+
+static void rtw_mrc_free(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mrc *mrc = minfo->mrc;
+	struct rtw_mrc_entry *p;
+	rtw_hlist_node *np, *n;
+	int i;
+
+	if (!mrc)
+		return;
+
+	for (i = 0; i < RTW_MRC_BUCKETS; i++) {
+		rtw_hlist_for_each_entry_safe(p, np, n, &mrc->bucket[i], list) {
+			rtw_hlist_del(&p->list);
+			rtw_mcache_free(mrc->cache, p);
+		}
+	}
+
+	rtw_mcache_destroy(mrc->cache);
+
+	rtw_mfree(mrc, sizeof(struct rtw_mrc));
+	minfo->mrc = NULL;
+}
+
+/**
+ * rtw_mrc_check - Check frame in mesh received cache and add if absent.
+ *
+ * @adapter:	interface
+ * @msa:		mesh source address
+ * @seq:		mesh seq number
+ *
+ * Returns: 0 if the frame is not in the cache, nonzero otherwise.
+ *
+ * Checks using the mesh source address and the mesh sequence number if we have
+ * received this frame lately. If the frame is not in the cache, it is added to
+ * it.
+ */
+static int rtw_mrc_check(_adapter *adapter, const u8 *msa, u32 seq)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mrc *mrc = minfo->mrc;
+	int entries = 0;
+	u8 idx;
+	struct rtw_mrc_entry *p;
+	rtw_hlist_node *np, *n;
+	u8 timeout;
+
+	if (!mrc)
+		return -1;
+
+	idx = seq & mrc->idx_mask;
+	rtw_hlist_for_each_entry_safe(p, np, n, &mrc->bucket[idx], list) {
+		++entries;
+		timeout = rtw_time_after(rtw_get_current_time(), p->exp_time);
+		if (timeout || entries == RTW_MRC_QUEUE_MAX_LEN) {
+			if (!timeout)
+				minfo->mshstats.mrc_del_qlen++;
+
+			rtw_hlist_del(&p->list);
+			rtw_mcache_free(mrc->cache, p);
+			--entries;
+		} else if ((seq == p->seqnum) && _rtw_memcmp(msa, p->msa, ETH_ALEN) == _TRUE)
+			return -1;
+	}
+
+	p = rtw_mcache_alloc(mrc->cache);
+	if (!p)
+		return 0;
+
+	p->seqnum = seq;
+	p->exp_time = rtw_get_current_time() + rtw_ms_to_systime(RTW_MRC_TIMEOUT_MS);
+	_rtw_memcpy(p->msa, msa, ETH_ALEN);
+	rtw_hlist_add_head(&p->list, &mrc->bucket[idx]);
+	return 0;
+}
+
+static int rtw_mesh_decache(_adapter *adapter, const u8 *msa, u32 seq)
+{
+	return rtw_mrc_check(adapter, msa, seq);
+}
+
+#ifndef RTW_MESH_SCAN_RESULT_EXP_MS
+#define RTW_MESH_SCAN_RESULT_EXP_MS (10 * 1000)
+#endif
+
+#ifndef RTW_MESH_ACNODE_PREVENT
+#define RTW_MESH_ACNODE_PREVENT 0
+#endif
+#ifndef RTW_MESH_ACNODE_CONF_TIMEOUT_MS
+#define RTW_MESH_ACNODE_CONF_TIMEOUT_MS (20 * 1000)
+#endif
+#ifndef RTW_MESH_ACNODE_NOTIFY_TIMEOUT_MS
+#define RTW_MESH_ACNODE_NOTIFY_TIMEOUT_MS (2 * 1000)
+#endif
+
+#ifndef RTW_MESH_OFFCH_CAND
+#define RTW_MESH_OFFCH_CAND 1
+#endif
+#ifndef RTW_MESH_OFFCH_CAND_FIND_INT_MS
+#define RTW_MESH_OFFCH_CAND_FIND_INT_MS (10 * 1000)
+#endif
+
+#ifndef RTW_MESH_PEER_CONF_TIMEOUT_MS
+#define RTW_MESH_PEER_CONF_TIMEOUT_MS (20 * 1000)
+#endif
+#ifndef RTW_MESH_PEER_BLACKLIST_TIMEOUT_MS
+#define RTW_MESH_PEER_BLACKLIST_TIMEOUT_MS (20 * 1000)
+#endif
+
+#ifndef RTW_MESH_CTO_MGATE_REQUIRE
+#define RTW_MESH_CTO_MGATE_REQUIRE 0
+#endif
+#ifndef RTW_MESH_CTO_MGATE_CONF_TIMEOUT_MS
+#define RTW_MESH_CTO_MGATE_CONF_TIMEOUT_MS (20 * 1000)
+#endif
+#ifndef RTW_MESH_CTO_MGATE_BLACKLIST_TIMEOUT_MS
+#define RTW_MESH_CTO_MGATE_BLACKLIST_TIMEOUT_MS (20 * 1000)
+#endif
+
+void rtw_mesh_cfg_init_peer_sel_policy(struct rtw_mesh_cfg *mcfg)
+{
+	struct mesh_peer_sel_policy *sel_policy = &mcfg->peer_sel_policy;
+
+	sel_policy->scanr_exp_ms = RTW_MESH_SCAN_RESULT_EXP_MS;
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	sel_policy->acnode_prevent = RTW_MESH_ACNODE_PREVENT;
+	sel_policy->acnode_conf_timeout_ms = RTW_MESH_ACNODE_CONF_TIMEOUT_MS;
+	sel_policy->acnode_notify_timeout_ms = RTW_MESH_ACNODE_NOTIFY_TIMEOUT_MS;
+#endif
+
+#if CONFIG_RTW_MESH_OFFCH_CAND
+	sel_policy->offch_cand = RTW_MESH_OFFCH_CAND;
+	sel_policy->offch_find_int_ms = RTW_MESH_OFFCH_CAND_FIND_INT_MS;
+#endif
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	sel_policy->peer_conf_timeout_ms = RTW_MESH_PEER_CONF_TIMEOUT_MS;
+	sel_policy->peer_blacklist_timeout_ms = RTW_MESH_PEER_BLACKLIST_TIMEOUT_MS;
+#endif
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	sel_policy->cto_mgate_require = RTW_MESH_CTO_MGATE_REQUIRE;
+	sel_policy->cto_mgate_conf_timeout_ms = RTW_MESH_CTO_MGATE_CONF_TIMEOUT_MS;
+	sel_policy->cto_mgate_blacklist_timeout_ms = RTW_MESH_CTO_MGATE_BLACKLIST_TIMEOUT_MS;
+#endif
+}
+
+void rtw_mesh_cfg_init(_adapter *adapter)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+
+	mcfg->max_peer_links = RTW_MESH_MAX_PEER_LINKS;
+	mcfg->plink_timeout = RTW_MESH_PEER_LINK_TIMEOUT;
+
+	mcfg->dot11MeshTTL = RTW_MESH_TTL;
+	mcfg->element_ttl = RTW_MESH_DEFAULT_ELEMENT_TTL;
+	mcfg->dot11MeshHWMPmaxPREQretries = RTW_MESH_MAX_PREQ_RETRIES;
+	mcfg->path_refresh_time = RTW_MESH_PATH_REFRESH_TIME;
+	mcfg->min_discovery_timeout = RTW_MESH_MIN_DISCOVERY_TIMEOUT;
+	mcfg->dot11MeshHWMPactivePathTimeout = RTW_MESH_PATH_TIMEOUT;
+	mcfg->dot11MeshHWMPpreqMinInterval = RTW_MESH_PREQ_MIN_INT;
+	mcfg->dot11MeshHWMPperrMinInterval = RTW_MESH_PERR_MIN_INT;
+	mcfg->dot11MeshHWMPnetDiameterTraversalTime = RTW_MESH_DIAM_TRAVERSAL_TIME;
+	mcfg->dot11MeshHWMPRootMode = RTW_IEEE80211_ROOTMODE_NO_ROOT;
+	mcfg->dot11MeshHWMPRannInterval = RTW_MESH_RANN_INTERVAL;
+	mcfg->dot11MeshGateAnnouncementProtocol = _FALSE;
+	mcfg->dot11MeshForwarding = _TRUE;
+	mcfg->rssi_threshold = 0;
+	mcfg->dot11MeshHWMPactivePathToRootTimeout = RTW_MESH_PATH_TO_ROOT_TIMEOUT;
+	mcfg->dot11MeshHWMProotInterval = RTW_MESH_ROOT_INTERVAL;
+	mcfg->dot11MeshHWMPconfirmationInterval = RTW_MESH_ROOT_CONFIRMATION_INTERVAL;
+	mcfg->path_gate_timeout_factor = 3;
+	rtw_mesh_cfg_init_peer_sel_policy(mcfg);
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+	mcfg->sane_metric_delta = RTW_MESH_SANE_METRIC_DELTA;
+	mcfg->max_root_add_chk_cnt = RTW_MESH_MAX_ROOT_ADD_CHK_CNT;
+#endif
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	mcfg->b2u_flags_msrc = 0;
+	mcfg->b2u_flags_mfwd = RTW_MESH_B2U_GA_UCAST;
+#endif
+}
+
+void rtw_mesh_cfg_init_max_peer_links(_adapter *adapter, u8 stack_conf)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+
+	mcfg->max_peer_links = RTW_MESH_MAX_PEER_LINKS;
+
+	if (mcfg->max_peer_links > stack_conf)
+		mcfg->max_peer_links = stack_conf;
+}
+
+void rtw_mesh_cfg_init_plink_timeout(_adapter *adapter, u32 stack_conf)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+
+	mcfg->plink_timeout = stack_conf;
+}
+
+void rtw_mesh_init_mesh_info(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+
+	_rtw_memset(minfo, 0, sizeof(struct rtw_mesh_info));
+
+	rtw_mesh_plink_ctl_init(adapter);
+	
+	minfo->last_preq = rtw_get_current_time();
+	/* minfo->last_sn_update = rtw_get_current_time(); */
+	minfo->next_perr = rtw_get_current_time();
+	
+	ATOMIC_SET(&minfo->mpaths, 0);
+	rtw_mesh_pathtbl_init(adapter);
+
+	_rtw_init_queue(&minfo->mpath_tx_queue);
+	tasklet_init(&minfo->mpath_tx_tasklet
+		, (void(*)(unsigned long))mpath_tx_tasklet_hdl
+		, (unsigned long)adapter);
+
+	rtw_mrc_init(adapter);
+
+	_rtw_init_listhead(&minfo->preq_queue.list);
+	_rtw_spinlock_init(&minfo->mesh_preq_queue_lock);
+	
+	rtw_init_timer(&adapter->mesh_path_timer, rtw_ieee80211_mesh_path_timer, adapter);
+	rtw_init_timer(&adapter->mesh_path_root_timer, rtw_ieee80211_mesh_path_root_timer, adapter);
+	rtw_init_timer(&adapter->mesh_atlm_param_req_timer, rtw_mesh_atlm_param_req_timer, adapter);
+	_init_workitem(&adapter->mesh_work, rtw_mesh_work_hdl, NULL);
+}
+
+void rtw_mesh_deinit_mesh_info(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+
+	tasklet_kill(&minfo->mpath_tx_tasklet);
+	rtw_mpath_tx_queue_flush(adapter);
+	_rtw_deinit_queue(&adapter->mesh_info.mpath_tx_queue);
+
+	rtw_mrc_free(adapter);
+
+	rtw_mesh_pathtbl_unregister(adapter);
+
+	rtw_mesh_plink_ctl_deinit(adapter);
+
+	_cancel_workitem_sync(&adapter->mesh_work);
+	_cancel_timer_ex(&adapter->mesh_path_timer);
+	_cancel_timer_ex(&adapter->mesh_path_root_timer);
+	_cancel_timer_ex(&adapter->mesh_atlm_param_req_timer);
+}
+
+/**
+ * rtw_mesh_nexthop_resolve - lookup next hop; conditionally start path discovery
+ *
+ * @skb: 802.11 frame to be sent
+ * @sdata: network subif the frame will be sent through
+ *
+ * Lookup next hop for given skb and start path discovery if no
+ * forwarding information is found.
+ *
+ * Returns: 0 if the next hop was found and -ENOENT if the frame was queued.
+ * skb is freeed here if no mpath could be allocated.
+ */
+int rtw_mesh_nexthop_resolve(_adapter *adapter,
+			struct xmit_frame *xframe)
+{
+	struct pkt_attrib *attrib = &xframe->attrib;
+	struct rtw_mesh_path *mpath;
+	struct xmit_frame *xframe_to_free = NULL;
+	u8 *target_addr = attrib->mda;
+	int err = 0;
+	int ret = _SUCCESS;
+
+	rtw_rcu_read_lock();
+	err = rtw_mesh_nexthop_lookup(adapter, target_addr, attrib->msa, attrib->ra);
+	if (!err)
+		goto endlookup;
+
+	/* no nexthop found, start resolving */
+	mpath = rtw_mesh_path_lookup(adapter, target_addr);
+	if (!mpath) {
+		mpath = rtw_mesh_path_add(adapter, target_addr);
+		if (IS_ERR(mpath)) {
+			xframe->pkt = NULL; /* free pkt outside */
+			rtw_mesh_path_discard_frame(adapter, xframe);
+			err = PTR_ERR(mpath);
+			ret = _FAIL;
+			goto endlookup;
+		}
+	}
+
+	if (!(mpath->flags & RTW_MESH_PATH_RESOLVING))
+		rtw_mesh_queue_preq(mpath, RTW_PREQ_Q_F_START);
+
+	_rtw_spinlock_bh(&mpath->frame_queue.lock);
+
+	if (mpath->frame_queue_len >= RTW_MESH_FRAME_QUEUE_LEN) {
+		xframe_to_free = LIST_CONTAINOR(get_next(get_list_head(&mpath->frame_queue)), struct xmit_frame, list);
+		rtw_list_delete(&(xframe_to_free->list));
+		mpath->frame_queue_len--;
+	}
+
+	rtw_list_insert_tail(&xframe->list, get_list_head(&mpath->frame_queue));
+	mpath->frame_queue_len++;
+
+	_rtw_spinunlock_bh(&mpath->frame_queue.lock);
+
+	ret = RTW_RA_RESOLVING;
+	if (xframe_to_free)
+		rtw_mesh_path_discard_frame(adapter, xframe_to_free);
+
+endlookup:
+	rtw_rcu_read_unlock();
+	return ret;
+}
+
+/**
+ * rtw_mesh_nexthop_lookup - put the appropriate next hop on a mesh frame. Calling
+ * this function is considered "using" the associated mpath, so preempt a path
+ * refresh if this mpath expires soon.
+ *
+ * @skb: 802.11 frame to be sent
+ * @sdata: network subif the frame will be sent through
+ *
+ * Returns: 0 if the next hop was found. Nonzero otherwise.
+ */
+int rtw_mesh_nexthop_lookup(_adapter *adapter,
+	const u8 *mda, const u8 *msa, u8 *ra)
+{
+	struct rtw_mesh_path *mpath;
+	struct sta_info *next_hop;
+	const u8 *target_addr = mda;
+	int err = -ENOENT;
+	struct registry_priv  *registry_par = &adapter->registrypriv;
+	u8 peer_alive_based_preq = registry_par->peer_alive_based_preq;
+	BOOLEAN nexthop_alive = _TRUE;
+
+	rtw_rcu_read_lock();
+	mpath = rtw_mesh_path_lookup(adapter, target_addr);
+
+	if (!mpath || !(mpath->flags & RTW_MESH_PATH_ACTIVE))
+		goto endlookup;
+
+	next_hop = rtw_rcu_dereference(mpath->next_hop);
+	if (next_hop) {
+		_rtw_memcpy(ra, next_hop->phl_sta->mac_addr, ETH_ALEN);
+		err = 0;
+	}
+
+	if (peer_alive_based_preq && next_hop)
+		nexthop_alive = next_hop->alive;
+
+	if (_rtw_memcmp(adapter_mac_addr(adapter), msa, ETH_ALEN) == _TRUE &&
+	    !(mpath->flags & RTW_MESH_PATH_RESOLVING) &&
+	    !(mpath->flags & RTW_MESH_PATH_FIXED)) {
+		u8 flags = RTW_PREQ_Q_F_START | RTW_PREQ_Q_F_REFRESH;
+
+		if (peer_alive_based_preq && nexthop_alive == _FALSE) {
+			flags |= RTW_PREQ_Q_F_BCAST_PREQ;
+			rtw_mesh_queue_preq(mpath, flags);
+		} else if (rtw_time_after(rtw_get_current_time(),
+			mpath->exp_time -
+			rtw_ms_to_systime(adapter->mesh_cfg.path_refresh_time))) {
+			rtw_mesh_queue_preq(mpath, flags);
+		}
+	/* Avoid keeping trying unicast PREQ toward root,
+	   when next_hop leaves */
+	} else if (peer_alive_based_preq &&
+		   _rtw_memcmp(adapter_mac_addr(adapter), msa, ETH_ALEN) == _TRUE &&
+		   (mpath->flags & RTW_MESH_PATH_RESOLVING) &&
+		   !(mpath->flags & RTW_MESH_PATH_FIXED) &&
+		   !(mpath->flags & RTW_MESH_PATH_BCAST_PREQ) &&
+		   mpath->is_root && nexthop_alive == _FALSE) {
+		_rtw_spinlock_bh(&mpath->state_lock);
+		mpath->flags |= RTW_MESH_PATH_BCAST_PREQ;
+		_rtw_spinunlock_bh(&mpath->state_lock);
+	}
+
+endlookup:
+	rtw_rcu_read_unlock();
+	return err;
+}
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+static bool rtw_mesh_data_bmc_to_uc(_adapter *adapter
+	, const u8 *da, const u8 *sa, const u8 *mda, const u8 *msa
+	, u8 ae_need, const u8 *ori_ta, u8 mfwd_ttl
+	, u16 os_qid, _list *b2u_list, u8 *b2u_num, u32 *b2u_mseq)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct xmit_priv *xmitpriv = &adapter->xmitpriv;
+	_list *head, *list;
+	struct sta_info *sta;
+	char b2u_sta_id[NUM_STA];
+	u8 b2u_sta_num = 0;
+	bool bmc_need = _FALSE;
+	int i;
+
+	rtw_stapriv_asoc_list_lock(stapriv);
+	head = &stapriv->asoc_list;
+	list = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		int stainfo_offset;
+
+		sta = LIST_CONTAINOR(list, struct sta_info, asoc_list);
+		list = get_next(list);
+	
+		stainfo_offset = rtw_stainfo_offset(stapriv, sta);
+		if (stainfo_offset_valid(stainfo_offset))
+			b2u_sta_id[b2u_sta_num++] = stainfo_offset;
+	}
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+	if (!b2u_sta_num)
+		goto exit;
+
+	for (i = 0; i < b2u_sta_num; i++) {
+		struct xmit_frame *b2uframe;
+		struct pkt_attrib *attrib;
+
+		sta = rtw_get_stainfo_by_offset(stapriv, b2u_sta_id[i]);
+		if (!(sta->state & WIFI_ASOC_STATE)
+			|| _rtw_memcmp(sta->phl_sta->mac_addr, msa, ETH_ALEN) == _TRUE
+			|| (ori_ta && _rtw_memcmp(sta->phl_sta->mac_addr, ori_ta, ETH_ALEN) == _TRUE)
+			|| is_broadcast_mac_addr(sta->phl_sta->mac_addr)
+			|| is_zero_mac_addr(sta->phl_sta->mac_addr))
+			continue;
+
+		b2uframe = rtw_alloc_xmitframe(xmitpriv, os_qid);
+		if (!b2uframe) {
+			bmc_need = _TRUE;
+			break;
+		}
+
+		if ((*b2u_num)++ == 0 && !ori_ta) {
+			*b2u_mseq = (cpu_to_le32(adapter->mesh_info.mesh_seqnum));
+			adapter->mesh_info.mesh_seqnum++;
+		}
+
+		attrib = &b2uframe->attrib;
+
+		attrib->mb2u = 1;
+		attrib->mseq = *b2u_mseq;
+		attrib->mfwd_ttl = ori_ta ? mfwd_ttl : 0;
+		_rtw_memcpy(attrib->ra, sta->phl_sta->mac_addr, ETH_ALEN);
+		_rtw_memcpy(attrib->ta, adapter_mac_addr(adapter), ETH_ALEN);
+		_rtw_memcpy(attrib->mda, mda, ETH_ALEN);
+		_rtw_memcpy(attrib->msa, msa, ETH_ALEN);
+		_rtw_memcpy(attrib->dst, da, ETH_ALEN);
+		_rtw_memcpy(attrib->src, sa, ETH_ALEN);
+		attrib->mesh_frame_mode = ae_need ? MESH_UCAST_PX_DATA : MESH_UCAST_DATA;
+
+		rtw_list_insert_tail(&b2uframe->list, b2u_list);
+	}
+
+exit:
+	return bmc_need;
+}
+
+void dump_mesh_b2u_flags(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+
+	RTW_PRINT_SEL(sel, "%4s %4s\n", "msrc", "mfwd");
+	RTW_PRINT_SEL(sel, "0x%02x 0x%02x\n", mcfg->b2u_flags_msrc, mcfg->b2u_flags_mfwd);
+}
+#endif /* CONFIG_RTW_MESH_DATA_BMC_TO_UC */
+
+int rtw_mesh_addr_resolve(_adapter *adapter, u16 os_qid, struct xmit_frame *xframe, _pkt *pkt, _list *b2u_list)
+{
+	struct pkt_file pktfile;
+	struct ethhdr etherhdr;
+	struct pkt_attrib *attrib;
+	struct rtw_mesh_path *mpath = NULL, *mppath = NULL;
+	u8 is_da_mcast;
+	u8 ae_need;
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	bool bmc_need = _TRUE;
+	u8 b2u_num = 0;
+	u32 b2u_mseq = 0;
+#endif
+	int res = _SUCCESS;
+
+	_rtw_open_pktfile(pkt, &pktfile);
+	if (_rtw_pktfile_read(&pktfile, (u8 *)&etherhdr, ETH_HLEN) != ETH_HLEN) {
+		res = _FAIL;
+		goto exit;
+	}
+	
+	xframe->pkt = pkt;
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	_rtw_init_listhead(b2u_list);
+#endif
+
+	is_da_mcast = IS_MCAST(etherhdr.h_dest);
+	if (!is_da_mcast) {
+		struct sta_info *next_hop; 
+		bool mpp_lookup = 1;
+	
+		mpath = rtw_mesh_path_lookup(adapter, etherhdr.h_dest);
+		if (mpath) {
+			mpp_lookup = 0;
+			next_hop = rtw_rcu_dereference(mpath->next_hop);
+			if (!next_hop
+				|| !(mpath->flags & (RTW_MESH_PATH_ACTIVE | RTW_MESH_PATH_RESOLVING))
+			) {
+				/* mpath is not valid, search mppath */
+				mpp_lookup = 1;
+			}
+		}
+
+		if (mpp_lookup) {
+			mppath = rtw_mpp_path_lookup(adapter, etherhdr.h_dest);
+			if (mppath)
+				mppath->exp_time = rtw_get_current_time();
+		}
+
+		if (mppath && mpath)
+			rtw_mesh_path_del(adapter, mpath->dst);
+
+		ae_need = _rtw_memcmp(adapter_mac_addr(adapter), etherhdr.h_source, ETH_ALEN) == _FALSE
+			|| (mppath && _rtw_memcmp(mppath->mpp, etherhdr.h_dest, ETH_ALEN) == _FALSE);
+	} else {
+		ae_need = _rtw_memcmp(adapter_mac_addr(adapter), etherhdr.h_source, ETH_ALEN) == _FALSE;
+
+		#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+		if (rtw_msrc_b2u_policy_chk(adapter->mesh_cfg.b2u_flags_msrc, etherhdr.h_dest)) {
+			bmc_need = rtw_mesh_data_bmc_to_uc(adapter
+				, etherhdr.h_dest, etherhdr.h_source
+				, etherhdr.h_dest, adapter_mac_addr(adapter), ae_need, NULL, 0
+				, os_qid, b2u_list, &b2u_num, &b2u_mseq);
+			if (bmc_need == _FALSE) {
+				res = RTW_ORI_NO_NEED;
+				goto exit;
+			}
+		}
+		#endif
+	}
+
+	attrib = &xframe->attrib;
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	if (b2u_num) {
+		attrib->mb2u = 1;
+		attrib->mseq = b2u_mseq;
+	} else
+		attrib->mb2u = 0;
+#endif
+
+	attrib->mfwd_ttl = 0;
+	_rtw_memcpy(attrib->dst, etherhdr.h_dest, ETH_ALEN);
+	_rtw_memcpy(attrib->src, etherhdr.h_source, ETH_ALEN);
+	_rtw_memcpy(attrib->ta, adapter_mac_addr(adapter), ETH_ALEN);
+
+	if (is_da_mcast) {
+		attrib->mesh_frame_mode = ae_need ? MESH_BMCAST_PX_DATA : MESH_BMCAST_DATA;
+		_rtw_memcpy(attrib->ra, attrib->dst, ETH_ALEN);
+		_rtw_memcpy(attrib->msa, adapter_mac_addr(adapter), ETH_ALEN);
+	} else {
+		attrib->mesh_frame_mode = ae_need ? MESH_UCAST_PX_DATA : MESH_UCAST_DATA;
+		_rtw_memcpy(attrib->mda, (mppath && ae_need) ? mppath->mpp : attrib->dst, ETH_ALEN);
+		_rtw_memcpy(attrib->msa, adapter_mac_addr(adapter), ETH_ALEN);
+		/* RA needs to be resolved */
+		res = rtw_mesh_nexthop_resolve(adapter, xframe);
+	}
+
+exit:
+	return res;
+}
+
+s8 rtw_mesh_tx_set_whdr_mctrl_len(u8 mesh_frame_mode, struct pkt_attrib *attrib)
+{
+	u8 ret = 0;
+	switch (mesh_frame_mode) {
+	case MESH_UCAST_DATA:
+		attrib->hdrlen = WLAN_HDR_A4_QOS_LEN;
+		attrib->a4_hdr = 1;
+		/* mesh flag + mesh TTL + Mesh SN. no ext addr. */
+		attrib->meshctrl_len = 6;
+		break;
+	case MESH_BMCAST_DATA:
+		attrib->hdrlen = WLAN_HDR_A3_QOS_LEN;
+		/* mesh flag + mesh TTL + Mesh SN. no ext addr. */
+		attrib->meshctrl_len = 6;
+		break;
+	case MESH_UCAST_PX_DATA:
+		attrib->hdrlen = WLAN_HDR_A4_QOS_LEN;
+		attrib->a4_hdr = 1;
+		/* mesh flag + mesh TTL + Mesh SN + extaddr1 + extaddr2. */
+		attrib->meshctrl_len = 18;
+		break;
+	case MESH_BMCAST_PX_DATA:
+		attrib->hdrlen = WLAN_HDR_A3_QOS_LEN;
+		/* mesh flag + mesh TTL + Mesh SN + extaddr1 */
+		attrib->meshctrl_len = 12;
+		break;
+	default:
+		RTW_WARN("Invalid mesh frame mode:%u\n", mesh_frame_mode);
+		ret = -1;
+		break;
+	}				
+
+	return ret;
+}
+
+void rtw_mesh_tx_build_mctrl(_adapter *adapter, struct pkt_attrib *attrib, u8 *buf)
+{
+	struct rtw_ieee80211s_hdr *mctrl = (struct rtw_ieee80211s_hdr *)buf;
+
+	_rtw_memset(mctrl, 0, XATTRIB_GET_MCTRL_LEN(attrib));
+
+	if (attrib->mfwd_ttl
+		#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+		|| attrib->mb2u
+		#endif
+	) {
+		#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+		if (!attrib->mfwd_ttl)
+			mctrl->ttl = adapter->mesh_cfg.dot11MeshTTL;
+		else
+		#endif
+			mctrl->ttl = attrib->mfwd_ttl;
+
+		mctrl->seqnum = (cpu_to_le32(attrib->mseq));
+	} else {
+		mctrl->ttl = adapter->mesh_cfg.dot11MeshTTL;
+		mctrl->seqnum = (cpu_to_le32(adapter->mesh_info.mesh_seqnum));
+		adapter->mesh_info.mesh_seqnum++;
+	}
+
+	switch (attrib->mesh_frame_mode){
+	case MESH_UCAST_DATA:
+	case MESH_BMCAST_DATA:
+		break;
+	case MESH_UCAST_PX_DATA:
+		mctrl->flags |= MESH_FLAGS_AE_A5_A6;
+		_rtw_memcpy(mctrl->eaddr1, attrib->dst, ETH_ALEN);
+		_rtw_memcpy(mctrl->eaddr2, attrib->src, ETH_ALEN);
+		break;
+	case MESH_BMCAST_PX_DATA:
+		mctrl->flags |= MESH_FLAGS_AE_A4;
+		_rtw_memcpy(mctrl->eaddr1, attrib->src, ETH_ALEN);
+		break;
+	case MESH_MHOP_UCAST_ACT:
+		/* TBD */
+		break;
+	case MESH_MHOP_BMCAST_ACT:
+		/* TBD */
+		break;
+	default:
+		break;
+	}
+}
+
+u8 rtw_mesh_tx_build_whdr(_adapter *adapter, struct pkt_attrib *attrib
+	, u16 *fctrl, struct rtw_ieee80211_hdr *whdr)
+{
+	switch (attrib->mesh_frame_mode) {
+	case MESH_UCAST_DATA:		/* 1, 1, RA, TA, mDA(=DA),	mSA(=SA) */
+	case MESH_UCAST_PX_DATA:	/* 1, 1, RA, TA, mDA,		mSA,		[DA, SA] */
+		SetToDs(fctrl);
+		SetFrDs(fctrl);
+		_rtw_memcpy(whdr->addr1, attrib->ra, ETH_ALEN);
+		_rtw_memcpy(whdr->addr2, attrib->ta, ETH_ALEN);
+		_rtw_memcpy(whdr->addr3, attrib->mda, ETH_ALEN);
+		_rtw_memcpy(whdr->addr4, attrib->msa, ETH_ALEN);
+		break;
+	case MESH_BMCAST_DATA:		/* 0, 1, RA(DA), TA, mSA(SA) */
+	case MESH_BMCAST_PX_DATA:	/* 0, 1, RA(DA), TA, mSA,		[SA] */
+		SetFrDs(fctrl);
+		_rtw_memcpy(whdr->addr1, attrib->ra, ETH_ALEN);
+		_rtw_memcpy(whdr->addr2, attrib->ta, ETH_ALEN);
+		_rtw_memcpy(whdr->addr3, attrib->msa, ETH_ALEN);
+		break;
+	case MESH_MHOP_UCAST_ACT:
+		/* TBD */
+		RTW_INFO("MESH_MHOP_UCAST_ACT\n");
+		break;
+	case MESH_MHOP_BMCAST_ACT:
+		/* TBD */
+		RTW_INFO("MESH_MHOP_BMCAST_ACT\n");
+		break;
+	default:
+		RTW_WARN("Invalid mesh frame mode\n");
+		break;
+	}
+	
+	return 0;
+}
+
+int rtw_mesh_rx_data_validate_hdr(_adapter *adapter, union recv_frame *rframe, struct sta_info **sta)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct rx_pkt_attrib *rattrib = &rframe->u.hdr.attrib;
+	u8 *whdr = get_recvframe_data(rframe);
+	u8 is_ra_bmc = 0;
+	u8 a4_shift = 0;
+	u8 ps;
+	u8 *qc;
+	u8 mps_mode = RTW_MESH_PS_UNKNOWN;
+	sint ret = _FAIL;
+
+	if (!(MLME_STATE(adapter) & WIFI_ASOC_STATE))
+		goto exit;
+
+	if (!rattrib->qos)
+		goto exit;
+
+	switch (rattrib->to_fr_ds) {
+	case 1:
+		if (!IS_MCAST(GetAddr1Ptr(whdr)))
+			goto exit;
+		*sta = rtw_get_stainfo(stapriv, get_addr2_ptr(whdr));
+		if (*sta == NULL) {
+			ret = _SUCCESS; /* return _SUCCESS to drop at sta checking */
+			goto exit;
+		}
+		_rtw_memcpy(rattrib->ra, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->ta, get_addr2_ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->mda, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->msa, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->dst, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->src, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking mesh ctrl field */
+		_rtw_memcpy(rattrib->bssid, get_addr2_ptr(whdr), ETH_ALEN);
+		is_ra_bmc = 1;
+		break;
+	case 3:
+		if (IS_MCAST(GetAddr1Ptr(whdr)))
+			goto exit;
+		*sta = rtw_get_stainfo(stapriv, get_addr2_ptr(whdr));
+		if (*sta == NULL) {
+			ret = _SUCCESS; /* return _SUCCESS to drop at sta checking */
+			goto exit;
+		}
+		_rtw_memcpy(rattrib->ra, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->ta, get_addr2_ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->mda, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->msa, GetAddr4Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->dst, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking mesh ctrl field */
+		_rtw_memcpy(rattrib->src, GetAddr4Ptr(whdr), ETH_ALEN); /* may change after checking mesh ctrl field */
+		_rtw_memcpy(rattrib->bssid, get_addr2_ptr(whdr), ETH_ALEN);
+		a4_shift = ETH_ALEN;
+		break;
+	default:
+		goto exit;
+	}
+
+	qc = whdr + WLAN_HDR_A3_LEN + a4_shift;
+	ps = GetPwrMgt(whdr);
+	mps_mode = ps ? (is_ra_bmc || (get_mps_lv(qc)) ? RTW_MESH_PS_DSLEEP : RTW_MESH_PS_LSLEEP) : RTW_MESH_PS_ACTIVE;
+
+	if (ps) {
+		if (!((*sta)->state & WIFI_SLEEP_STATE))
+			stop_sta_xmit(adapter, *sta);
+	} else {
+		if ((*sta)->state & WIFI_SLEEP_STATE)
+			wakeup_sta_to_xmit(adapter, *sta);
+	}
+
+	if (is_ra_bmc)
+		(*sta)->nonpeer_mps = mps_mode;
+	else {
+		(*sta)->peer_mps = mps_mode;
+		if (mps_mode != RTW_MESH_PS_ACTIVE && (*sta)->nonpeer_mps == RTW_MESH_PS_ACTIVE)
+			(*sta)->nonpeer_mps = RTW_MESH_PS_DSLEEP;
+	}
+
+	if (get_frame_sub_type(whdr) & BIT(6)) {
+		/* No data, will not indicate to upper layer, temporily count it here */
+		count_rx_stats(adapter, rframe, *sta);
+		ret = RTW_RX_HANDLED;
+		goto exit;
+	}
+
+	rattrib->mesh_ctrl_present = get_mctrl_present(qc) ? 1 : 0;
+	if (!rattrib->mesh_ctrl_present)
+		goto exit;
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+int rtw_mesh_rx_data_validate_mctrl(_adapter *adapter, union recv_frame *rframe
+	, const struct rtw_ieee80211s_hdr *mctrl, const u8 *mda, const u8 *msa
+	, u8 *mctrl_len
+	, const u8 **da, const u8 **sa)
+{
+	struct rx_pkt_attrib *rattrib = &rframe->u.hdr.attrib;
+	u8 mlen;
+	u8 ae;
+	int ret = _SUCCESS;
+
+	ae = mctrl->flags & MESH_FLAGS_AE;
+	mlen = ae_to_mesh_ctrl_len[ae];
+	switch (rattrib->to_fr_ds) {
+	case 1:
+		*da = mda;
+		if (ae == MESH_FLAGS_AE_A4)
+			*sa = mctrl->eaddr1;
+		else if (ae == 0)
+			*sa = msa;
+		else
+			ret = _FAIL;
+		break;
+	case 3:
+		if (ae == MESH_FLAGS_AE_A5_A6) {
+			*da = mctrl->eaddr1;
+			*sa = mctrl->eaddr2;
+		} else if (ae == 0) {
+			*da = mda;
+			*sa = msa;
+		} else
+			ret = _FAIL;
+		break;
+	default:
+		ret = _FAIL;
+	}
+
+	if (ret == _FAIL) {
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" invalid tfDS:%u AE:%u combination ra="MAC_FMT" ta="MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), rattrib->to_fr_ds, ae, MAC_ARG(rattrib->ra), MAC_ARG(rattrib->ta));
+		#endif
+		*mctrl_len = 0;
+	} else
+		*mctrl_len = mlen;
+
+	return ret;	
+}
+
+inline int rtw_mesh_rx_validate_mctrl_non_amsdu(_adapter *adapter, union recv_frame *rframe)
+{
+	struct rx_pkt_attrib *rattrib = &rframe->u.hdr.attrib;
+	const u8 *da, *sa;
+	int ret;
+
+	ret = rtw_mesh_rx_data_validate_mctrl(adapter, rframe
+			, (struct rtw_ieee80211s_hdr *)(get_recvframe_data(rframe) + rattrib->hdrlen + rattrib->iv_len)
+			, rattrib->mda, rattrib->msa
+			, &rattrib->mesh_ctrl_len
+			, &da, &sa);
+
+	if (ret == _SUCCESS) {
+		_rtw_memcpy(rattrib->dst, da, ETH_ALEN);
+		_rtw_memcpy(rattrib->src, sa, ETH_ALEN);
+	}
+
+	return ret;
+}
+
+/**
+ * rtw_mesh_rx_nexthop_resolve - lookup next hop; conditionally start path discovery
+ *
+ * @skb: 802.11 frame to be sent
+ * @sdata: network subif the frame will be sent through
+ *
+ * Lookup next hop for given skb and start path discovery if no
+ * forwarding information is found.
+ *
+ * Returns: 0 if the next hop was found and -ENOENT if the frame was queued.
+ * skb is freeed here if no mpath could be allocated.
+ */
+static int rtw_mesh_rx_nexthop_resolve(_adapter *adapter,
+	const u8 *mda, const u8 *msa, u8 *ra)
+{
+	struct rtw_mesh_path *mpath;
+	struct xmit_frame *xframe_to_free = NULL;
+	int err = 0;
+	int ret = _SUCCESS;
+
+	rtw_rcu_read_lock();
+	err = rtw_mesh_nexthop_lookup(adapter, mda, msa, ra);
+	if (!err)
+		goto endlookup;
+
+	/* no nexthop found, start resolving */
+	mpath = rtw_mesh_path_lookup(adapter, mda);
+	if (!mpath) {
+		mpath = rtw_mesh_path_add(adapter, mda);
+		if (IS_ERR(mpath)) {
+			err = PTR_ERR(mpath);
+			ret = _FAIL;
+			goto endlookup;
+		}
+	}
+
+	if (!(mpath->flags & RTW_MESH_PATH_RESOLVING))
+		rtw_mesh_queue_preq(mpath, RTW_PREQ_Q_F_START);
+
+	ret = _FAIL;
+
+endlookup:
+	rtw_rcu_read_unlock();
+	return ret;
+}
+
+#define RTW_MESH_DECACHE_BMC 1
+#define RTW_MESH_DECACHE_UC 0
+
+#define RTW_MESH_FORWARD_MDA_SELF_COND 0
+#define DBG_RTW_MESH_FORWARD_MDA_SELF_COND 0
+int rtw_mesh_rx_msdu_act_check(union recv_frame *rframe
+	, const u8 *mda, const u8 *msa
+	, const u8 *da, const u8 *sa
+	, struct rtw_ieee80211s_hdr *mctrl
+	, u8 *msdu, enum rtw_rx_llc_hdl llc_hdl
+	, struct xmit_frame **fwd_frame, _list *b2u_list)
+{
+	_adapter *adapter = rframe->u.hdr.adapter;
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rx_pkt_attrib *rattrib = &rframe->u.hdr.attrib;
+	struct rtw_mesh_path *mppath;
+	u8 is_mda_bmc = IS_MCAST(mda); 
+	u8 is_mda_self = !is_mda_bmc && _rtw_memcmp(mda, adapter_mac_addr(adapter), ETH_ALEN);
+	u16 os_qid;
+	struct xmit_frame *xframe;
+	struct pkt_attrib *xattrib;
+	u8 fwd_ra[ETH_ALEN] = {0};
+	u8 fwd_mpp[ETH_ALEN] = {0}; /* forward to other gate */
+	u32 fwd_mseq;
+	int act = 0;
+	u8 ae_need;
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	bool ori_need = _TRUE;
+	u8 b2u_num = 0;
+#endif
+
+	/* fwd info lifetime update */
+	#if 0
+	if (!is_mda_self)
+		mDA(A3) fwinfo.lifetime
+	mSA(A4) fwinfo.lifetime
+	Precursor-to-mDA(A2) fwinfo.lifetime
+	#endif
+
+	/* update/create pxoxy info for SA, mSA */
+	if ((mctrl->flags & MESH_FLAGS_AE)
+		&& sa != msa && _rtw_memcmp(sa, msa, ETH_ALEN) == _FALSE
+	) {
+		const u8 *proxied_addr = sa;
+		const u8 *mpp_addr = msa;
+
+		rtw_rcu_read_lock();
+		mppath = rtw_mpp_path_lookup(adapter, proxied_addr);
+		if (!mppath)
+			rtw_mpp_path_add(adapter, proxied_addr, mpp_addr);
+		else {
+			_rtw_spinlock_bh(&mppath->state_lock);
+			if (_rtw_memcmp(mppath->mpp, mpp_addr, ETH_ALEN) == _FALSE)
+				_rtw_memcpy(mppath->mpp, mpp_addr, ETH_ALEN);
+			mppath->exp_time = rtw_get_current_time();
+			_rtw_spinunlock_bh(&mppath->state_lock);
+		}
+		rtw_rcu_read_unlock();
+	}
+
+	/* mSA is self, need no further process */
+	if (_rtw_memcmp(msa, adapter_mac_addr(adapter), ETH_ALEN) == _TRUE)
+		goto exit;
+
+	fwd_mseq = le32_to_cpu(mctrl->seqnum);
+
+	/* check duplicate MSDU from mSA */
+	if (((RTW_MESH_DECACHE_BMC && is_mda_bmc)
+			|| (RTW_MESH_DECACHE_UC && !is_mda_bmc))
+		&& rtw_mesh_decache(adapter, msa, fwd_mseq)
+	) {
+		minfo->mshstats.dropped_frames_duplicate++;
+		goto exit;
+	}
+
+	if (is_mda_bmc) {
+		/* mDA is bmc addr */
+		act |= RTW_RX_MSDU_ACT_INDICATE;
+		if (!mcfg->dot11MeshForwarding)
+			goto exit;
+		goto fwd_chk;
+
+	} else if (!is_mda_self) {
+		/* mDA is unicast but not self */
+		if (!mcfg->dot11MeshForwarding) {
+			rtw_mesh_path_error_tx(adapter
+				, adapter->mesh_cfg.element_ttl
+				, mda, 0
+				, WLAN_REASON_MESH_PATH_NOFORWARD
+				, rattrib->ta
+			);
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" mDA("MAC_FMT") not self, !dot11MeshForwarding\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(mda));
+			#endif
+			goto exit;
+		}
+
+		if (rtw_mesh_rx_nexthop_resolve(adapter, mda, msa, fwd_ra) != _SUCCESS) {
+			/* mDA is unknown */
+			rtw_mesh_path_error_tx(adapter
+				, adapter->mesh_cfg.element_ttl
+				, mda, 0
+				, WLAN_REASON_MESH_PATH_NOFORWARD
+				, rattrib->ta
+			);
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" mDA("MAC_FMT") unknown\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(mda));
+			#endif
+			minfo->mshstats.dropped_frames_no_route++;
+			goto exit;
+
+		} else {
+			/* mDA is known in fwd info */
+			#if 0
+			if	(TA is not in precursors)
+				goto exit;
+			#endif
+			goto fwd_chk;
+		}
+
+	} else {
+		/* mDA is self */
+		#if RTW_MESH_FORWARD_MDA_SELF_COND
+		if (da == mda
+			|| _rtw_memcmp(da, adapter_mac_addr(adapter), ETH_ALEN)
+		) {
+			/* DA is self, indicate */
+			act |= RTW_RX_MSDU_ACT_INDICATE;
+			goto exit;
+		}
+
+		if (rtw_get_iface_by_macddr(adapter, da)) {
+			/* DA is buddy, indicate */
+			act |= RTW_RX_MSDU_ACT_INDICATE;
+			#if DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+			RTW_INFO(FUNC_ADPT_FMT" DA("MAC_FMT") is buddy("ADPT_FMT")\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(da), ADPT_ARG(rtw_get_iface_by_macddr(adapter, da)));
+			#endif
+			goto exit;
+		}
+
+		/* DA is not self or buddy */
+		if (rtw_mesh_nexthop_lookup(adapter, da, msa, fwd_ra) == 0) {
+			/* DA is known in fwd info */
+			if (!mcfg->dot11MeshForwarding) {
+				/* path error to? */
+				#if defined(DBG_RX_DROP_FRAME) || DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+				RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" DA("MAC_FMT") not self, !dot11MeshForwarding\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(da));
+				#endif
+				goto exit;
+			}
+			mda = da;
+			#if DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+			RTW_INFO(FUNC_ADPT_FMT" fwd to DA("MAC_FMT"), fwd_RA("MAC_FMT")\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(da), MAC_ARG(fwd_ra));
+			#endif
+			goto fwd_chk;
+		}
+
+		rtw_rcu_read_lock();
+		mppath = rtw_mpp_path_lookup(adapter, da);
+		if (mppath) {
+			if (_rtw_memcmp(mppath->mpp, adapter_mac_addr(adapter), ETH_ALEN) == _FALSE) {
+				/* DA is proxied by others */
+				if (!mcfg->dot11MeshForwarding) {
+					/* path error to? */
+					#if defined(DBG_RX_DROP_FRAME) || DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" DA("MAC_FMT") is proxied by ("MAC_FMT"), !dot11MeshForwarding\n"
+						, FUNC_ADPT_ARG(adapter), MAC_ARG(da), MAC_ARG(mppath->mpp));
+					#endif
+					rtw_rcu_read_unlock();
+					goto exit;
+				}
+				_rtw_memcpy(fwd_mpp, mppath->mpp, ETH_ALEN);
+				mda = fwd_mpp;
+				msa = adapter_mac_addr(adapter);
+				rtw_rcu_read_unlock();
+
+				/* resolve RA */
+				if (rtw_mesh_nexthop_lookup(adapter, mda, msa, fwd_ra) != 0) {
+					minfo->mshstats.dropped_frames_no_route++;
+					#if defined(DBG_RX_DROP_FRAME) || DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" DA("MAC_FMT") is proxied by ("MAC_FMT"), RA resolve fail\n"
+						, FUNC_ADPT_ARG(adapter), MAC_ARG(da), MAC_ARG(mppath->mpp));
+					#endif
+					goto exit;
+				}
+				#if DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+				RTW_INFO(FUNC_ADPT_FMT" DA("MAC_FMT") is proxied by ("MAC_FMT"), fwd_RA("MAC_FMT")\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(da), MAC_ARG(mppath->mpp), MAC_ARG(fwd_ra));
+				#endif
+				goto fwd_chk; /*  forward to other gate */
+			} else {
+				#if DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+				RTW_INFO(FUNC_ADPT_FMT" DA("MAC_FMT") is proxied by self\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(da));
+				#endif
+			}
+		}
+		rtw_rcu_read_unlock();
+
+		if (!mppath) {
+			#if DBG_RTW_MESH_FORWARD_MDA_SELF_COND
+			RTW_INFO(FUNC_ADPT_FMT" DA("MAC_FMT") unknown\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(da));
+			#endif
+			/* DA is unknown */
+			#if 0 /* TODO: flags with AE bit */
+			rtw_mesh_path_error_tx(adapter
+				, adapter->mesh_cfg.element_ttl
+				, mda, adapter->mesh_info.last_sn_update
+				, WLAN_REASON_MESH_PATH_NOPROXY
+				, msa
+			);
+			#endif
+		}
+
+		/*
+		* indicate to DS for both cases:
+		* 1.) DA is proxied by self
+		* 2.) DA is unknown
+		*/
+		#endif /* RTW_MESH_FORWARD_MDA_SELF_COND */
+		act |= RTW_RX_MSDU_ACT_INDICATE;
+		goto exit;
+	}
+
+fwd_chk:
+
+	if (adapter->stapriv.asoc_list_cnt <= 1)
+		goto exit;
+
+	if (mctrl->ttl == 1) {
+		minfo->mshstats.dropped_frames_ttl++;
+		if (!act) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" ttl reaches 0, not forwarding\n"
+				, FUNC_ADPT_ARG(adapter));
+			#endif
+		}
+		goto exit;
+	}
+
+	os_qid = rtw_os_recv_select_queue(msdu, llc_hdl);
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	_rtw_init_listhead(b2u_list);
+#endif
+
+	ae_need = _rtw_memcmp(da , mda, ETH_ALEN) == _FALSE
+		|| _rtw_memcmp(sa , msa, ETH_ALEN) == _FALSE;
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	if (is_mda_bmc
+		&& rtw_mfwd_b2u_policy_chk(mcfg->b2u_flags_mfwd, mda, rattrib->to_fr_ds == 3)
+	) {
+		ori_need = rtw_mesh_data_bmc_to_uc(adapter
+			, da, sa, mda, msa, ae_need, rframe->u.hdr.psta->phl_sta->mac_addr, mctrl->ttl - 1
+			, os_qid, b2u_list, &b2u_num, &fwd_mseq);
+	}
+
+	if (ori_need == _TRUE)
+#endif
+	{
+		xframe = rtw_alloc_xmitframe(&adapter->xmitpriv, os_qid);
+		if (!xframe) {
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME "FUNC_ADPT_FMT" rtw_alloc_xmitframe fail\n"
+				, FUNC_ADPT_ARG(adapter));
+			#endif
+			goto exit;
+		}
+
+		xattrib = &xframe->attrib;
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+		if (b2u_num)
+			xattrib->mb2u = 1;
+		else
+			xattrib->mb2u = 0;
+#endif
+		xattrib->mfwd_ttl = mctrl->ttl - 1;
+		xattrib->mseq = fwd_mseq;
+		_rtw_memcpy(xattrib->dst, da, ETH_ALEN);
+		_rtw_memcpy(xattrib->src, sa, ETH_ALEN);
+		_rtw_memcpy(xattrib->mda, mda, ETH_ALEN);
+		_rtw_memcpy(xattrib->msa, msa, ETH_ALEN);
+		_rtw_memcpy(xattrib->ta, adapter_mac_addr(adapter), ETH_ALEN);
+
+		if (is_mda_bmc) {
+			xattrib->mesh_frame_mode = ae_need ? MESH_BMCAST_PX_DATA : MESH_BMCAST_DATA;
+			_rtw_memcpy(xattrib->ra, mda, ETH_ALEN);
+		} else {
+			xattrib->mesh_frame_mode = ae_need ? MESH_UCAST_PX_DATA : MESH_UCAST_DATA;
+			_rtw_memcpy(xattrib->ra, fwd_ra, ETH_ALEN);
+		}
+
+		*fwd_frame = xframe;
+	}
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	if (ori_need || b2u_num)
+#endif
+	{
+		act |= RTW_RX_MSDU_ACT_FORWARD;
+		if (is_mda_bmc)
+			minfo->mshstats.fwded_mcast++;
+		else
+			minfo->mshstats.fwded_unicast++;
+		minfo->mshstats.fwded_frames++;
+	}
+
+exit:
+	return act;
+}
+
+void dump_mesh_stats(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mesh_stats *stats = &minfo->mshstats;
+
+	RTW_PRINT_SEL(sel, "fwd_bmc:%u\n", stats->fwded_mcast);
+	RTW_PRINT_SEL(sel, "fwd_uc:%u\n", stats->fwded_unicast);
+
+	RTW_PRINT_SEL(sel, "drop_ttl:%u\n", stats->dropped_frames_ttl);
+	RTW_PRINT_SEL(sel, "drop_no_route:%u\n", stats->dropped_frames_no_route);
+	RTW_PRINT_SEL(sel, "drop_congestion:%u\n", stats->dropped_frames_congestion);
+	RTW_PRINT_SEL(sel, "drop_dup:%u\n", stats->dropped_frames_duplicate);
+
+	RTW_PRINT_SEL(sel, "mrc_del_qlen:%u\n", stats->mrc_del_qlen);
+}
+#endif /* CONFIG_RTW_MESH */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh.h b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh.h
new file mode 100644
index 000000000000..56d1f4b93b37
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh.h
@@ -0,0 +1,537 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_MESH_H_
+#define __RTW_MESH_H_
+
+#ifndef CONFIG_AP_MODE
+	#error "CONFIG_RTW_MESH can't be enabled when CONFIG_AP_MODE is not defined\n"
+#endif
+
+#define RTW_MESH_TTL				31
+#define RTW_MESH_PERR_MIN_INT			100
+#define RTW_MESH_DEFAULT_ELEMENT_TTL		31
+#define RTW_MESH_RANN_INTERVAL			5000
+#define RTW_MESH_PATH_TO_ROOT_TIMEOUT		6000
+#define RTW_MESH_DIAM_TRAVERSAL_TIME		50
+#define RTW_MESH_PATH_TIMEOUT			5000
+#define RTW_MESH_PREQ_MIN_INT			10
+#define RTW_MESH_MAX_PREQ_RETRIES		4
+#define RTW_MESH_MIN_DISCOVERY_TIMEOUT 		(2 * RTW_MESH_DIAM_TRAVERSAL_TIME)
+#define RTW_MESH_ROOT_CONFIRMATION_INTERVAL	2000
+#define RTW_MESH_PATH_REFRESH_TIME		1000
+#define RTW_MESH_ROOT_INTERVAL			5000
+
+#define RTW_MESH_SANE_METRIC_DELTA		100
+#define RTW_MESH_MAX_ROOT_ADD_CHK_CNT		2
+
+#define RTW_MESH_PLINK_UNKNOWN	0
+#define RTW_MESH_PLINK_LISTEN	1
+#define RTW_MESH_PLINK_OPN_SNT	2
+#define RTW_MESH_PLINK_OPN_RCVD 3
+#define RTW_MESH_PLINK_CNF_RCVD 4
+#define RTW_MESH_PLINK_ESTAB	5
+#define RTW_MESH_PLINK_HOLDING	6
+#define RTW_MESH_PLINK_BLOCKED	7
+
+extern const char *_rtw_mesh_plink_str[];
+#define rtw_mesh_plink_str(s) ((s <= RTW_MESH_PLINK_BLOCKED) ? _rtw_mesh_plink_str[s] : _rtw_mesh_plink_str[RTW_MESH_PLINK_UNKNOWN])
+
+#define RTW_MESH_PS_UNKNOWN 0
+#define RTW_MESH_PS_ACTIVE 1
+#define RTW_MESH_PS_LSLEEP 2
+#define RTW_MESH_PS_DSLEEP 3
+
+extern const char *_rtw_mesh_ps_str[];
+#define rtw_mesh_ps_str(mps) ((mps <= RTW_MESH_PS_DSLEEP) ? _rtw_mesh_ps_str[mps] : _rtw_mesh_ps_str[RTW_MESH_PS_UNKNOWN])
+
+#define GET_MESH_CONF_ELE_PATH_SEL_PROTO_ID(_iec)		LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 0, 0, 8)
+#define GET_MESH_CONF_ELE_PATH_SEL_METRIC_ID(_iec)		LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 1, 0, 8)
+#define GET_MESH_CONF_ELE_CONGEST_CTRL_MODE_ID(_iec)	LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 2, 0, 8)
+#define GET_MESH_CONF_ELE_SYNC_METHOD_ID(_iec)			LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 3, 0, 8)
+#define GET_MESH_CONF_ELE_AUTH_PROTO_ID(_iec)			LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 4, 0, 8)
+
+#define GET_MESH_CONF_ELE_MESH_FORMATION(_iec)			LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 5, 0, 8)
+#define GET_MESH_CONF_ELE_CTO_MGATE(_iec)				LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 5, 0, 1)
+#define GET_MESH_CONF_ELE_NUM_OF_PEERINGS(_iec)			LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 5, 1, 6)
+#define GET_MESH_CONF_ELE_CTO_AS(_iec)					LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 5, 7, 1)
+
+#define GET_MESH_CONF_ELE_MESH_CAP(_iec)				LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 0, 8)
+#define GET_MESH_CONF_ELE_ACCEPT_PEERINGS(_iec)			LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 0, 1)
+#define GET_MESH_CONF_ELE_MCCA_SUP(_iec)				LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 1, 1)
+#define GET_MESH_CONF_ELE_MCCA_EN(_iec)					LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 2, 1)
+#define GET_MESH_CONF_ELE_FORWARDING(_iec)				LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 3, 1)
+#define GET_MESH_CONF_ELE_MBCA_EN(_iec)					LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 4, 1)
+#define GET_MESH_CONF_ELE_TBTT_ADJ(_iec)				LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 5, 1)
+#define GET_MESH_CONF_ELE_PS_LEVEL(_iec)				LE_BITS_TO_1BYTE(((u8 *)(_iec)) + 6, 6, 1)
+
+#define SET_MESH_CONF_ELE_PATH_SEL_PROTO_ID(_iec, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 0, 0, 8, _val)
+#define SET_MESH_CONF_ELE_PATH_SEL_METRIC_ID(_iec, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 1, 0, 8, _val)
+#define SET_MESH_CONF_ELE_CONGEST_CTRL_MODE_ID(_iec, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 2, 0, 8, _val)
+#define SET_MESH_CONF_ELE_SYNC_METHOD_ID(_iec, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 3, 0, 8, _val)
+#define SET_MESH_CONF_ELE_AUTH_PROTO_ID(_iec, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 4, 0, 8, _val)
+
+#define SET_MESH_CONF_ELE_CTO_MGATE(_iec, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 5, 0, 1, _val)
+#define SET_MESH_CONF_ELE_NUM_OF_PEERINGS(_iec, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 5, 1, 6, _val)
+#define SET_MESH_CONF_ELE_CTO_AS(_iec, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 5, 7, 1, _val)
+
+#define SET_MESH_CONF_ELE_ACCEPT_PEERINGS(_iec, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 6, 0, 1, _val)
+#define SET_MESH_CONF_ELE_MCCA_SUP(_iec, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 6, 1, 1, _val)
+#define SET_MESH_CONF_ELE_MCCA_EN(_iec, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 6, 2, 1, _val)
+#define SET_MESH_CONF_ELE_FORWARDING(_iec, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 6, 3, 1, _val)
+#define SET_MESH_CONF_ELE_MBCA_EN(_iec, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 6, 4, 1, _val)
+#define SET_MESH_CONF_ELE_TBTT_ADJ(_iec, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 6, 5, 1, _val)
+#define SET_MESH_CONF_ELE_PS_LEVEL(_iec, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_iec)) + 6, 6, 1, _val)
+
+/* Mesh flags */
+#define MESH_FLAGS_AE		0x3 /* mask */
+#define MESH_FLAGS_AE_A4 	0x1
+#define MESH_FLAGS_AE_A5_A6	0x2
+
+/* Max number of paths */
+#define RTW_MESH_MAX_PATHS 1024
+
+#define RTW_PREQ_Q_F_START	0x1
+#define RTW_PREQ_Q_F_REFRESH	0x2
+#define RTW_PREQ_Q_F_CHK	0x4
+#define RTW_PREQ_Q_F_PEER_AKA	0x8
+#define RTW_PREQ_Q_F_BCAST_PREQ	0x10 /* force path_dicover using broadcast */
+struct rtw_mesh_preq_queue {
+	_list list;
+	u8 dst[ETH_ALEN];
+	u8 flags;
+};
+
+extern const u8 ae_to_mesh_ctrl_len[];
+
+enum mesh_frame_type {
+	MESH_UCAST_DATA		= 0x0,
+	MESH_BMCAST_DATA	= 0x1,
+	MESH_UCAST_PX_DATA	= 0x2,
+	MESH_BMCAST_PX_DATA	= 0x3,
+	MESH_MHOP_UCAST_ACT	= 0x4,
+	MESH_MHOP_BMCAST_ACT	= 0x5,
+};
+
+enum mpath_sel_frame_type {
+	MPATH_PREQ = 0,
+	MPATH_PREP,
+	MPATH_PERR,
+	MPATH_RANN
+};
+
+/**
+ * enum rtw_mesh_deferred_task_flags - mesh deferred tasks
+ *
+ *
+ *
+ * @RTW_MESH_WORK_HOUSEKEEPING: run the periodic mesh housekeeping tasks
+ * @RTW_MESH_WORK_ROOT: the mesh root station needs to send a frame
+ * @RTW_MESH_WORK_DRIFT_ADJUST: time to compensate for clock drift relative to other
+ * mesh nodes
+ * @RTW_MESH_WORK_MBSS_CHANGED: rebuild beacon and notify driver of BSS changes
+ */
+enum rtw_mesh_deferred_task_flags {
+	RTW_MESH_WORK_HOUSEKEEPING,
+	RTW_MESH_WORK_ROOT,
+	RTW_MESH_WORK_DRIFT_ADJUST,
+	RTW_MESH_WORK_MBSS_CHANGED,
+};
+
+#define RTW_MESH_MAX_PEER_CANDIDATES 15 /* aid consideration */
+#define RTW_MESH_MAX_PEER_LINKS 8
+#define RTW_MESH_PEER_LINK_TIMEOUT 20
+
+#define RTW_MESH_PEER_CONF_DISABLED 0 /* special time value means no confirmation ongoing */
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+#define IS_PEER_CONF_DISABLED(plink) ((plink)->peer_conf_end_time == RTW_MESH_PEER_CONF_DISABLED)
+#define IS_PEER_CONF_TIMEOUT(plink)(!IS_PEER_CONF_DISABLED(plink) && rtw_time_after(rtw_get_current_time(), (plink)->peer_conf_end_time))
+#define SET_PEER_CONF_DISABLED(plink) (plink)->peer_conf_end_time = RTW_MESH_PEER_CONF_DISABLED
+#define SET_PEER_CONF_END_TIME(plink, timeout_ms) \
+	do { \
+		(plink)->peer_conf_end_time = rtw_get_current_time() + rtw_ms_to_systime(timeout_ms); \
+		if ((plink)->peer_conf_end_time == RTW_MESH_PEER_CONF_DISABLED) \
+			(plink)->peer_conf_end_time++; \
+	} while (0)
+#else
+#define IS_PEER_CONF_DISABLED(plink) 1
+#define IS_PEER_CONF_TIMEOUT(plink) 0
+#define SET_PEER_CONF_DISABLED(plink) do {} while (0)
+#define SET_PEER_CONF_END_TIME(plink, timeout_ms) do {} while (0)
+#endif /* CONFIG_RTW_MESH_PEER_BLACKLIST */
+
+#define RTW_MESH_CTO_MGATE_CONF_DISABLED 0 /* special time value means no confirmation ongoing */
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+#define IS_CTO_MGATE_CONF_DISABLED(plink) ((plink)->cto_mgate_conf_end_time == RTW_MESH_CTO_MGATE_CONF_DISABLED)
+#define IS_CTO_MGATE_CONF_TIMEOUT(plink)(!IS_CTO_MGATE_CONF_DISABLED(plink) && rtw_time_after(rtw_get_current_time(), (plink)->cto_mgate_conf_end_time))
+#define SET_CTO_MGATE_CONF_DISABLED(plink) (plink)->cto_mgate_conf_end_time = RTW_MESH_CTO_MGATE_CONF_DISABLED
+#define SET_CTO_MGATE_CONF_END_TIME(plink, timeout_ms) \
+	do { \
+		(plink)->cto_mgate_conf_end_time = rtw_get_current_time() + rtw_ms_to_systime(timeout_ms); \
+		if ((plink)->cto_mgate_conf_end_time == RTW_MESH_CTO_MGATE_CONF_DISABLED) \
+			(plink)->cto_mgate_conf_end_time++; \
+	} while (0)
+#else
+#define IS_CTO_MGATE_CONF_DISABLED(plink) 1
+#define IS_CTO_MGATE_CONF_TIMEOUT(plink) 0
+#define SET_CTO_MGATE_CONF_DISABLED(plink) do {} while (0)
+#define SET_CTO_MGATE_CONF_END_TIME(plink, timeout_ms) do {} while (0)
+#endif /* CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST */
+
+struct mesh_plink_ent {
+	u8 valid;
+	u8 addr[ETH_ALEN];
+	u8 plink_state;
+
+#ifdef CONFIG_RTW_MESH_AEK
+	u8 aek_valid;
+	u8 aek[32];
+#endif
+
+	u16 llid;
+	u16 plid;
+#ifndef CONFIG_RTW_MESH_DRIVER_AID
+	u16 aid; /* aid assigned from upper layer */
+#endif
+	u16 peer_aid; /* aid assigned from peer */
+
+	u8 chosen_pmk[16];
+
+#ifdef CONFIG_RTW_MESH_AEK
+	u8 sel_pcs[4];
+	u8 l_nonce[32];
+	u8 p_nonce[32];
+#endif
+
+#ifdef CONFIG_RTW_MESH_DRIVER_AID
+	u8 *tx_conf_ies;
+	u16 tx_conf_ies_len;
+#endif
+	u8 *rx_conf_ies;
+	u16 rx_conf_ies_len;
+
+	struct wlan_network *scanned;
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	systime peer_conf_end_time;
+#endif
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	systime cto_mgate_conf_end_time;
+#endif
+};
+
+#ifdef CONFIG_RTW_MESH_AEK
+#define MESH_PLINK_AEK_VALID(ent) ent->aek_valid
+#else
+#define MESH_PLINK_AEK_VALID(ent) 0
+#endif
+
+struct mesh_plink_pool {
+	_lock lock;
+	u8 num; /* current ent being used */
+	struct mesh_plink_ent ent[RTW_MESH_MAX_PEER_CANDIDATES];
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	u8 acnode_rsvd;
+#endif
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	_queue peer_blacklist;
+#endif
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	_queue cto_mgate_blacklist;
+#endif
+};
+
+struct mesh_peer_sel_policy {
+	u32 scanr_exp_ms;
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	u8 acnode_prevent;
+	u32 acnode_conf_timeout_ms;
+	u32 acnode_notify_timeout_ms;
+#endif
+
+#if CONFIG_RTW_MESH_OFFCH_CAND
+	u8 offch_cand;
+	u32 offch_find_int_ms; /* 0 means no offch find triggerred by driver self*/
+#endif
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	u32 peer_conf_timeout_ms;
+	u32 peer_blacklist_timeout_ms;
+#endif
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	u8 cto_mgate_require;
+	u32 cto_mgate_conf_timeout_ms;
+	u32 cto_mgate_blacklist_timeout_ms;
+#endif
+};
+
+/* b2u flags */
+#define RTW_MESH_B2U_ALL		BIT0
+#define RTW_MESH_B2U_GA_UCAST	BIT1 /* Group addressed unicast frame, forward only */
+#define RTW_MESH_B2U_BCAST		BIT2
+#define RTW_MESH_B2U_IP_MCAST	BIT3
+
+#define rtw_msrc_b2u_policy_chk(flags, mda) ( \
+	(flags & RTW_MESH_B2U_ALL) \
+	|| ((flags & RTW_MESH_B2U_BCAST) && is_broadcast_mac_addr(mda)) \
+	|| ((flags & RTW_MESH_B2U_IP_MCAST) && (IP_MCAST_MAC(mda) || ICMPV6_MCAST_MAC(mda))) \
+	)
+
+#define rtw_mfwd_b2u_policy_chk(flags, mda, ucst) ( \
+	(flags & RTW_MESH_B2U_ALL) \
+	|| ((flags & RTW_MESH_B2U_GA_UCAST) && ucst) \
+	|| ((flags & RTW_MESH_B2U_BCAST) && is_broadcast_mac_addr(mda)) \
+	|| ((flags & RTW_MESH_B2U_IP_MCAST) && (IP_MCAST_MAC(mda) || ICMPV6_MCAST_MAC(mda))) \
+	)
+
+/**
+ * @sane_metric_delta: Controlling if trigger additional path check mechanism
+ * @max_root_add_chk_cnt: The retry cnt to send additional root confirmation
+ *	PREQ through old(last) path
+ */
+struct rtw_mesh_cfg {
+	u8 max_peer_links; /* peering limit */
+	u32 plink_timeout; /* seconds */
+
+	u8 dot11MeshTTL;
+	u8 element_ttl;
+	u32 path_refresh_time;
+	u16 dot11MeshHWMPpreqMinInterval;
+	u16 dot11MeshHWMPnetDiameterTraversalTime;
+	u32 dot11MeshHWMPactivePathTimeout;
+	u8 dot11MeshHWMPmaxPREQretries;
+	u16 min_discovery_timeout;
+	u16 dot11MeshHWMPconfirmationInterval;
+	u16 dot11MeshHWMPperrMinInterval;
+	u8 dot11MeshHWMPRootMode;
+	BOOLEAN dot11MeshForwarding;
+	s32 rssi_threshold; /* in dBm, 0: no specified */
+	u16 dot11MeshHWMPRannInterval;
+	BOOLEAN dot11MeshGateAnnouncementProtocol;
+	u32 dot11MeshHWMPactivePathToRootTimeout;
+	u16 dot11MeshHWMProotInterval;
+	u8 path_gate_timeout_factor;
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+	u16 sane_metric_delta;
+	u8 max_root_add_chk_cnt;
+#endif
+
+	struct mesh_peer_sel_policy peer_sel_policy;
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	u8 b2u_flags_msrc;
+	u8 b2u_flags_mfwd;
+#endif
+};
+
+struct rtw_mesh_stats {
+	u32 fwded_mcast;		/* Mesh forwarded multicast frames */
+	u32 fwded_unicast;		/* Mesh forwarded unicast frames */
+	u32 fwded_frames;		/* Mesh total forwarded frames */
+	u32 dropped_frames_ttl;	/* Not transmitted since mesh_ttl == 0*/
+	u32 dropped_frames_no_route;	/* Not transmitted, no route found */
+	u32 dropped_frames_congestion;/* Not forwarded due to congestion */
+	u32 dropped_frames_duplicate;
+
+	u32 mrc_del_qlen; /* MRC entry deleted cause by queue length limit */
+};
+
+struct rtw_mrc;
+
+struct rtw_mesh_info {
+	u8 mesh_id[NDIS_802_11_LENGTH_SSID];
+	size_t mesh_id_len;
+	/* Active Path Selection Protocol Identifier */
+	u8 mesh_pp_id;
+	/* Active Path Selection Metric Identifier */
+	u8 mesh_pm_id;
+	/* Congestion Control Mode Identifier */
+	u8 mesh_cc_id;
+	/* Synchronization Protocol Identifier */
+	u8 mesh_sp_id;
+	/* Authentication Protocol Identifier */
+	u8 mesh_auth_id;
+
+	struct mesh_plink_pool plink_ctl;
+
+	u32 mesh_seqnum;
+	/* MSTA's own hwmp sequence number */
+	u32 sn;
+	systime last_preq;
+	systime last_sn_update;
+	systime next_perr;
+	/* Last used Path Discovery ID */
+	u32 preq_id;
+	
+	ATOMIC_T mpaths;
+	struct rtw_mesh_table *mesh_paths;
+	struct rtw_mesh_table *mpp_paths;
+	int mesh_paths_generation;
+	int mpp_paths_generation;
+
+	int num_gates;
+	struct rtw_mesh_path *max_addr_gate;
+	bool max_addr_gate_is_larger_than_self;
+
+	struct rtw_mesh_stats mshstats;
+
+	_queue mpath_tx_queue;
+	u32 mpath_tx_queue_len;
+	_tasklet mpath_tx_tasklet;
+
+	struct rtw_mrc *mrc;
+
+	_lock mesh_preq_queue_lock;
+	struct rtw_mesh_preq_queue preq_queue;
+	int preq_queue_len;
+};
+
+extern const char *_action_self_protected_str[];
+#define action_self_protected_str(action) ((action < RTW_ACT_SELF_PROTECTED_NUM) ? _action_self_protected_str[action] : _action_self_protected_str[0])
+
+u8 *rtw_set_ie_mesh_id(u8 *buf, u32 *buf_len, const char *mesh_id, u8 id_len);
+u8 *rtw_set_ie_mesh_config(u8 *buf, u32 *buf_len
+	, u8 path_sel_proto, u8 path_sel_metric, u8 congest_ctl_mode, u8 sync_method, u8 auth_proto
+	, u8 num_of_peerings, bool cto_mgate, bool cto_as
+	, bool accept_peerings, bool mcca_sup, bool mcca_en, bool forwarding
+	, bool mbca_en, bool tbtt_adj, bool ps_level);
+
+int rtw_bss_is_same_mbss(WLAN_BSSID_EX *a, WLAN_BSSID_EX *b);
+int rtw_bss_is_candidate_mesh_peer(_adapter *adapter, WLAN_BSSID_EX *target, u8 ch, u8 add_peer);
+
+void rtw_chk_candidate_peer_notify(_adapter *adapter, struct wlan_network *scanned);
+
+void rtw_mesh_peer_status_chk(_adapter *adapter);
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+void rtw_mesh_update_scanned_acnode_status(_adapter *adapter, struct wlan_network *scanned);
+bool rtw_mesh_scanned_is_acnode_confirmed(_adapter *adapter, struct wlan_network *scanned);
+bool rtw_mesh_acnode_prevent_allow_sacrifice(_adapter *adapter);
+struct sta_info *rtw_mesh_acnode_prevent_pick_sacrifice(_adapter *adapter);
+void dump_mesh_acnode_prevent_settings(void *sel, _adapter *adapter);
+#endif
+
+#if CONFIG_RTW_MESH_OFFCH_CAND
+u8 rtw_mesh_offch_candidate_accepted(_adapter *adapter);
+u8 rtw_mesh_select_operating_ch(_adapter *adapter);
+void dump_mesh_offch_cand_settings(void *sel, _adapter *adapter);
+#endif
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+int rtw_mesh_peer_blacklist_add(_adapter *adapter, const u8 *addr);
+int rtw_mesh_peer_blacklist_del(_adapter *adapter, const u8 *addr);
+int rtw_mesh_peer_blacklist_search(_adapter *adapter, const u8 *addr);
+void rtw_mesh_peer_blacklist_flush(_adapter *adapter);
+void dump_mesh_peer_blacklist(void *sel, _adapter *adapter);
+void dump_mesh_peer_blacklist_settings(void *sel, _adapter *adapter);
+#endif
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+u8 rtw_mesh_cto_mgate_required(_adapter *adapter);
+u8 rtw_mesh_cto_mgate_network_filter(_adapter *adapter, struct wlan_network *scanned);
+int rtw_mesh_cto_mgate_blacklist_add(_adapter *adapter, const u8 *addr);
+int rtw_mesh_cto_mgate_blacklist_del(_adapter *adapter, const u8 *addr);
+int rtw_mesh_cto_mgate_blacklist_search(_adapter *adapter, const u8 *addr);
+void rtw_mesh_cto_mgate_blacklist_flush(_adapter *adapter);
+void dump_mesh_cto_mgate_blacklist(void *sel, _adapter *adapter);
+void dump_mesh_cto_mgate_blacklist_settings(void *sel, _adapter *adapter);
+#endif
+void dump_mesh_peer_sel_policy(void *sel, _adapter *adapter);
+void dump_mesh_networks(void *sel, _adapter *adapter);
+
+void rtw_mesh_adjust_bchbw(enum band_type req_band, u8 req_ch, u8 *req_bw, u8 *req_offset);
+
+void rtw_mesh_sae_check_frames(_adapter *adapter, const u8 *buf, u32 len, u8 tx, u16 alg, u16 seq, u16 status);
+int rtw_mesh_check_frames_tx(_adapter *adapter, const u8 **buf, size_t *len);
+int rtw_mesh_check_frames_rx(_adapter *adapter, const u8 *buf, size_t len);
+
+int rtw_mesh_on_auth(_adapter *adapter, union recv_frame *rframe);
+unsigned int on_action_self_protected(_adapter *adapter, union recv_frame *rframe);
+
+bool rtw_mesh_update_bss_peering_status(_adapter *adapter, WLAN_BSSID_EX *bss);
+bool rtw_mesh_update_bss_formation_info(_adapter *adapter, WLAN_BSSID_EX *bss);
+bool rtw_mesh_update_bss_forwarding_state(_adapter *adapter, WLAN_BSSID_EX *bss);
+
+struct mesh_plink_ent *_rtw_mesh_plink_get(_adapter *adapter, const u8 *hwaddr);
+struct mesh_plink_ent *rtw_mesh_plink_get(_adapter *adapter, const u8 *hwaddr);
+struct mesh_plink_ent *rtw_mesh_plink_get_no_estab_by_idx(_adapter *adapter, u8 idx);
+int _rtw_mesh_plink_add(_adapter *adapter, const u8 *hwaddr);
+int rtw_mesh_plink_add(_adapter *adapter, const u8 *hwaddr);
+int rtw_mesh_plink_set_state(_adapter *adapter, const u8 *hwaddr, u8 state);
+#ifdef CONFIG_RTW_MESH_AEK
+int rtw_mesh_plink_set_aek(_adapter *adapter, const u8 *hwaddr, const u8 *aek);
+#endif
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+int rtw_mesh_plink_set_peer_conf_timeout(_adapter *adapter, const u8 *hwaddr);
+#endif
+void _rtw_mesh_plink_del_ent(_adapter *adapter, struct mesh_plink_ent *ent);
+int rtw_mesh_plink_del(_adapter *adapter, const u8 *hwaddr);
+void rtw_mesh_plink_ctl_init(_adapter *adapter);
+void rtw_mesh_plink_ctl_deinit(_adapter *adapter);
+void dump_mesh_plink_ctl(void *sel, _adapter *adapter);
+
+u8 rtw_mesh_set_plink_state_cmd(_adapter *adapter, const u8 *mac, u8 plink_state);
+
+void _rtw_mesh_expire_peer_ent(_adapter *adapter, struct mesh_plink_ent *plink);
+void rtw_mesh_expire_peer(_adapter *adapter, const u8 *peer_addr);
+u8 rtw_mesh_ps_annc(_adapter *adapter, u8 ps);
+
+unsigned int on_action_mesh(_adapter *adapter, union recv_frame *rframe);
+
+void rtw_mesh_cfg_init(_adapter *adapter);
+void rtw_mesh_cfg_init_max_peer_links(_adapter *adapter, u8 stack_conf);
+void rtw_mesh_cfg_init_plink_timeout(_adapter *adapter, u32 stack_conf);
+void rtw_mesh_init_mesh_info(_adapter *adapter);
+void rtw_mesh_deinit_mesh_info(_adapter *adapter);
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+void dump_mesh_b2u_flags(void *sel, _adapter *adapter);
+#endif
+
+int rtw_mesh_addr_resolve(_adapter *adapter, u16 os_qid, struct xmit_frame *xframe, _pkt *pkt, _list *b2u_list);
+
+s8 rtw_mesh_tx_set_whdr_mctrl_len(u8 mesh_frame_mode, struct pkt_attrib *attrib);
+void rtw_mesh_tx_build_mctrl(_adapter *adapter, struct pkt_attrib *attrib, u8 *buf);
+u8 rtw_mesh_tx_build_whdr(_adapter *adapter, struct pkt_attrib *attrib
+	, u16 *fctrl, struct rtw_ieee80211_hdr *whdr);
+
+int rtw_mesh_rx_data_validate_hdr(_adapter *adapter, union recv_frame *rframe, struct sta_info **sta);
+int rtw_mesh_rx_data_validate_mctrl(_adapter *adapter, union recv_frame *rframe
+	, const struct rtw_ieee80211s_hdr *mctrl, const u8 *mda, const u8 *msa
+	, u8 *mctrl_len, const u8 **da, const u8 **sa);
+int rtw_mesh_rx_validate_mctrl_non_amsdu(_adapter *adapter, union recv_frame *rframe);
+
+int rtw_mesh_rx_msdu_act_check(union recv_frame *rframe
+	, const u8 *mda, const u8 *msa
+	, const u8 *da, const u8 *sa
+	, struct rtw_ieee80211s_hdr *mctrl
+	, u8 *msdu, enum rtw_rx_llc_hdl llc_hdl
+	, struct xmit_frame **fwd_frame, _list *b2u_list);
+
+void dump_mesh_stats(void *sel, _adapter *adapter);
+
+#if defined(PLATFORM_LINUX) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32))
+#define rtw_lockdep_assert_held(l) lockdep_assert_held(l)
+#define rtw_lockdep_is_held(l) lockdep_is_held(l)
+#else
+#error "TBD\n"
+#endif
+
+#include "rtw_mesh_pathtbl.h"
+#include "rtw_mesh_hwmp.h"
+#endif /* __RTW_MESH_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_hwmp.c b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_hwmp.c
new file mode 100644
index 000000000000..cfff06d35836
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_hwmp.c
@@ -0,0 +1,1519 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_HWMP_C_
+
+#ifdef CONFIG_RTW_MESH
+#include <drv_types.h>
+
+#define RTW_TEST_FRAME_LEN	8192
+#define RTW_MAX_METRIC	0xffffffff
+#define RTW_ARITH_SHIFT	8
+#define RTW_LINK_FAIL_THRESH 95
+#define RTW_MAX_PREQ_QUEUE_LEN	64
+#define RTW_ATLM_REQ_CYCLE 1000
+
+#define rtw_ilog2(n)			\
+(					\
+	(n) < 2 ? 0 :			\
+	(n) & (1ULL << 63) ? 63 :	\
+	(n) & (1ULL << 62) ? 62 :	\
+	(n) & (1ULL << 61) ? 61 :	\
+	(n) & (1ULL << 60) ? 60 :	\
+	(n) & (1ULL << 59) ? 59 :	\
+	(n) & (1ULL << 58) ? 58 :	\
+	(n) & (1ULL << 57) ? 57 :	\
+	(n) & (1ULL << 56) ? 56 :	\
+	(n) & (1ULL << 55) ? 55 :	\
+	(n) & (1ULL << 54) ? 54 :	\
+	(n) & (1ULL << 53) ? 53 :	\
+	(n) & (1ULL << 52) ? 52 :	\
+	(n) & (1ULL << 51) ? 51 :	\
+	(n) & (1ULL << 50) ? 50 :	\
+	(n) & (1ULL << 49) ? 49 :	\
+	(n) & (1ULL << 48) ? 48 :	\
+	(n) & (1ULL << 47) ? 47 :	\
+	(n) & (1ULL << 46) ? 46 :	\
+	(n) & (1ULL << 45) ? 45 :	\
+	(n) & (1ULL << 44) ? 44 :	\
+	(n) & (1ULL << 43) ? 43 :	\
+	(n) & (1ULL << 42) ? 42 :	\
+	(n) & (1ULL << 41) ? 41 :	\
+	(n) & (1ULL << 40) ? 40 :	\
+	(n) & (1ULL << 39) ? 39 :	\
+	(n) & (1ULL << 38) ? 38 :	\
+	(n) & (1ULL << 37) ? 37 :	\
+	(n) & (1ULL << 36) ? 36 :	\
+	(n) & (1ULL << 35) ? 35 :	\
+	(n) & (1ULL << 34) ? 34 :	\
+	(n) & (1ULL << 33) ? 33 :	\
+	(n) & (1ULL << 32) ? 32 :	\
+	(n) & (1ULL << 31) ? 31 :	\
+	(n) & (1ULL << 30) ? 30 :	\
+	(n) & (1ULL << 29) ? 29 :	\
+	(n) & (1ULL << 28) ? 28 :	\
+	(n) & (1ULL << 27) ? 27 :	\
+	(n) & (1ULL << 26) ? 26 :	\
+	(n) & (1ULL << 25) ? 25 :	\
+	(n) & (1ULL << 24) ? 24 :	\
+	(n) & (1ULL << 23) ? 23 :	\
+	(n) & (1ULL << 22) ? 22 :	\
+	(n) & (1ULL << 21) ? 21 :	\
+	(n) & (1ULL << 20) ? 20 :	\
+	(n) & (1ULL << 19) ? 19 :	\
+	(n) & (1ULL << 18) ? 18 :	\
+	(n) & (1ULL << 17) ? 17 :	\
+	(n) & (1ULL << 16) ? 16 :	\
+	(n) & (1ULL << 15) ? 15 :	\
+	(n) & (1ULL << 14) ? 14 :	\
+	(n) & (1ULL << 13) ? 13 :	\
+	(n) & (1ULL << 12) ? 12 :	\
+	(n) & (1ULL << 11) ? 11 :	\
+	(n) & (1ULL << 10) ? 10 :	\
+	(n) & (1ULL <<  9) ?  9 :	\
+	(n) & (1ULL <<  8) ?  8 :	\
+	(n) & (1ULL <<  7) ?  7 :	\
+	(n) & (1ULL <<  6) ?  6 :	\
+	(n) & (1ULL <<  5) ?  5 :	\
+	(n) & (1ULL <<  4) ?  4 :	\
+	(n) & (1ULL <<  3) ?  3 :	\
+	(n) & (1ULL <<  2) ?  2 :	\
+	1				\
+)
+
+enum rtw_mpath_frame_type {
+	RTW_MPATH_PREQ = 0,
+	RTW_MPATH_PREP,
+	RTW_MPATH_PERR,
+	RTW_MPATH_RANN
+};
+
+static inline u32 rtw_u32_field_get(const u8 *preq_elem, int shift, BOOLEAN ae)
+{
+	if (ae)
+		shift += 6;
+	return LE_BITS_TO_4BYTE(preq_elem + shift, 0, 32);
+}
+
+static inline u16 rtw_u16_field_get(const u8 *preq_elem, int shift, BOOLEAN ae)
+{
+	if (ae)
+		shift += 6;
+	return LE_BITS_TO_2BYTE(preq_elem + shift, 0, 16);
+}
+
+/* HWMP IE processing macros */
+#define RTW_AE_F			(1<<6)
+#define RTW_AE_F_SET(x)			(*x & RTW_AE_F)
+#define RTW_PREQ_IE_FLAGS(x)		(*(x))
+#define RTW_PREQ_IE_HOPCOUNT(x)		(*(x + 1))
+#define RTW_PREQ_IE_TTL(x)		(*(x + 2))
+#define RTW_PREQ_IE_PREQ_ID(x)		rtw_u32_field_get(x, 3, 0)
+#define RTW_PREQ_IE_ORIG_ADDR(x)	(x + 7)
+#define RTW_PREQ_IE_ORIG_SN(x)		rtw_u32_field_get(x, 13, 0)
+#define RTW_PREQ_IE_LIFETIME(x)		rtw_u32_field_get(x, 17, RTW_AE_F_SET(x))
+#define RTW_PREQ_IE_METRIC(x) 		rtw_u32_field_get(x, 21, RTW_AE_F_SET(x))
+#define RTW_PREQ_IE_TARGET_F(x)		(*(RTW_AE_F_SET(x) ? x + 32 : x + 26))
+#define RTW_PREQ_IE_TARGET_ADDR(x) 	(RTW_AE_F_SET(x) ? x + 33 : x + 27)
+#define RTW_PREQ_IE_TARGET_SN(x) 	rtw_u32_field_get(x, 33, RTW_AE_F_SET(x))
+
+#define RTW_PREP_IE_FLAGS(x)		RTW_PREQ_IE_FLAGS(x)
+#define RTW_PREP_IE_HOPCOUNT(x)		RTW_PREQ_IE_HOPCOUNT(x)
+#define RTW_PREP_IE_TTL(x)		RTW_PREQ_IE_TTL(x)
+#define RTW_PREP_IE_ORIG_ADDR(x)	(RTW_AE_F_SET(x) ? x + 27 : x + 21)
+#define RTW_PREP_IE_ORIG_SN(x)		rtw_u32_field_get(x, 27, RTW_AE_F_SET(x))
+#define RTW_PREP_IE_LIFETIME(x)		rtw_u32_field_get(x, 13, RTW_AE_F_SET(x))
+#define RTW_PREP_IE_METRIC(x)		rtw_u32_field_get(x, 17, RTW_AE_F_SET(x))
+#define RTW_PREP_IE_TARGET_ADDR(x)	(x + 3)
+#define RTW_PREP_IE_TARGET_SN(x)	rtw_u32_field_get(x, 9, 0)
+
+#define RTW_PERR_IE_TTL(x)		(*(x))
+#define RTW_PERR_IE_TARGET_FLAGS(x)	(*(x + 2))
+#define RTW_PERR_IE_TARGET_ADDR(x)	(x + 3)
+#define RTW_PERR_IE_TARGET_SN(x)	rtw_u32_field_get(x, 9, 0)
+#define RTW_PERR_IE_TARGET_RCODE(x)	rtw_u16_field_get(x, 13, 0)
+
+#define RTW_TU_TO_SYSTIME(x)	(rtw_us_to_systime((x) * 1024))
+#define RTW_TU_TO_EXP_TIME(x)	(rtw_get_current_time() + RTW_TU_TO_SYSTIME(x))
+#define RTW_MSEC_TO_TU(x) (x*1000/1024)
+#define RTW_SN_GT(x, y) ((s32)(y - x) < 0)
+#define RTW_SN_LT(x, y) ((s32)(x - y) < 0)
+#define RTW_MAX_SANE_SN_DELTA 32
+
+static inline u32 RTW_SN_DELTA(u32 x, u32 y)
+{
+	return x >= y ? x - y : y - x;
+}
+
+#define rtw_net_traversal_jiffies(adapter) \
+	rtw_ms_to_systime(adapter->mesh_cfg.dot11MeshHWMPnetDiameterTraversalTime)
+#define rtw_default_lifetime(adapter) \
+	RTW_MSEC_TO_TU(adapter->mesh_cfg.dot11MeshHWMPactivePathTimeout)
+#define rtw_min_preq_int_jiff(adapter) \
+	(rtw_ms_to_systime(adapter->mesh_cfg.dot11MeshHWMPpreqMinInterval))
+#define rtw_max_preq_retries(adapter) (adapter->mesh_cfg.dot11MeshHWMPmaxPREQretries)
+#define rtw_disc_timeout_jiff(adapter) \
+	rtw_ms_to_systime(adapter->mesh_cfg.min_discovery_timeout)
+#define rtw_root_path_confirmation_jiffies(adapter) \
+	rtw_ms_to_systime(adapter->mesh_cfg.dot11MeshHWMPconfirmationInterval)
+
+static inline BOOLEAN rtw_ether_addr_equal(const u8 *addr1, const u8 *addr2)
+{
+	return _rtw_memcmp(addr1, addr2, ETH_ALEN);
+}
+
+#ifdef PLATFORM_LINUX
+#define rtw_print_ratelimit()	printk_ratelimit()
+#define rtw_mod_timer(ptimer, expires) mod_timer(&(ptimer)->timer, expires)
+#else
+
+#endif
+
+#define RTW_MESH_EWMA_PRECISION 20
+#define RTW_MESH_EWMA_WEIGHT_RCP 8
+#define RTW_TOTAL_PKT_MIN_THRESHOLD 1
+inline void rtw_ewma_err_rate_init(struct rtw_ewma_err_rate *e)
+{
+	e->internal = 0;
+}
+inline unsigned long rtw_ewma_err_rate_read(struct rtw_ewma_err_rate *e)
+{
+	return e->internal >> (RTW_MESH_EWMA_PRECISION);
+}
+inline void rtw_ewma_err_rate_add(struct rtw_ewma_err_rate *e,
+				  unsigned long val)
+{
+	unsigned long internal = e->internal;
+	unsigned long weight_rcp = rtw_ilog2(RTW_MESH_EWMA_WEIGHT_RCP);
+	unsigned long precision = RTW_MESH_EWMA_PRECISION;
+
+	(e->internal) = internal ? (((internal << weight_rcp) - internal) +
+			(val << precision)) >> weight_rcp :
+			(val << precision);
+}
+
+static const u8 bcast_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+static int rtw_mesh_path_sel_frame_tx(enum rtw_mpath_frame_type mpath_action, u8 flags,
+				      const u8 *originator_addr, u32 originator_sn,
+				      u8 target_flags, const u8 *target,
+				      u32 target_sn, const u8 *da, u8 hopcount, u8 ttl,
+				      u32 lifetime, u32 metric, u32 preq_id, 
+				      _adapter *adapter)
+{
+	struct xmit_priv *pxmitpriv = &(adapter->xmitpriv);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct xmit_frame *pmgntframe = NULL;
+	struct rtw_ieee80211_hdr *pwlanhdr = NULL;
+	struct pkt_attrib *pattrib = NULL;
+	u8 category = RTW_WLAN_CATEGORY_MESH;
+	u8 action = RTW_ACT_MESH_HWMP_PATH_SELECTION;
+	u16 *fctrl = NULL;
+	u8 *pos, ie_len;
+
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return -1;
+
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(adapter, pattrib);
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pos = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pos;
+
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(adapter), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, adapter_mac_addr(adapter), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pos, WIFI_ACTION);
+
+	pos += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pos = rtw_set_fixed_ie(pos, 1, &(category), &(pattrib->pktlen));
+	pos = rtw_set_fixed_ie(pos, 1, &(action), &(pattrib->pktlen));
+
+	switch (mpath_action) {
+	case RTW_MPATH_PREQ:
+		RTW_HWMP_DBG("sending PREQ to "MAC_FMT"\n", MAC_ARG(target));
+		ie_len = 37;
+		pattrib->pktlen += (ie_len + 2);
+		*pos++ = WLAN_EID_PREQ;
+		break;
+	case RTW_MPATH_PREP:
+		RTW_HWMP_DBG("sending PREP to "MAC_FMT"\n", MAC_ARG(originator_addr));
+		ie_len = 31;
+		pattrib->pktlen += (ie_len + 2);
+		*pos++ = WLAN_EID_PREP;
+		break;
+	case RTW_MPATH_RANN:
+		RTW_HWMP_DBG("sending RANN from "MAC_FMT"\n", MAC_ARG(originator_addr));
+		ie_len = sizeof(struct rtw_ieee80211_rann_ie);
+		pattrib->pktlen += (ie_len + 2);
+		*pos++ = WLAN_EID_RANN;
+		break;
+	default:
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		return _FAIL;
+	}
+	*pos++ = ie_len;
+	*pos++ = flags;
+	*pos++ = hopcount;
+	*pos++ = ttl;
+	if (mpath_action == RTW_MPATH_PREP) {
+		_rtw_memcpy(pos, target, ETH_ALEN);
+		pos += ETH_ALEN;
+		*(u32 *)pos = cpu_to_le32(target_sn);
+		pos += 4;
+	} else {
+		if (mpath_action == RTW_MPATH_PREQ) {
+			*(u32 *)pos = cpu_to_le32(preq_id);
+			pos += 4;
+		}
+		_rtw_memcpy(pos, originator_addr, ETH_ALEN);
+		pos += ETH_ALEN;
+		*(u32 *)pos = cpu_to_le32(originator_sn);
+		pos += 4;
+	}
+	*(u32 *)pos = cpu_to_le32(lifetime);
+	pos += 4;
+	*(u32 *)pos = cpu_to_le32(metric);
+	pos += 4;
+	if (mpath_action == RTW_MPATH_PREQ) {
+		*pos++ = 1; /* support only 1 destination now */
+		*pos++ = target_flags;
+		_rtw_memcpy(pos, target, ETH_ALEN);
+		pos += ETH_ALEN;
+		*(u32 *)pos = cpu_to_le32(target_sn);
+		pos += 4;
+	} else if (mpath_action == RTW_MPATH_PREP) {
+		_rtw_memcpy(pos, originator_addr, ETH_ALEN);
+		pos += ETH_ALEN;
+		*(u32 *)pos = cpu_to_le32(originator_sn);
+		pos += 4;
+	}
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	dump_mgntframe(adapter, pmgntframe);
+	return 0;
+}
+
+int rtw_mesh_path_error_tx(_adapter *adapter,
+			   u8 ttl, const u8 *target, u32 target_sn,
+			   u16 perr_reason_code, const u8 *ra)
+{
+
+	struct xmit_priv *pxmitpriv = &(adapter->xmitpriv);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct xmit_frame *pmgntframe = NULL;
+	struct rtw_ieee80211_hdr *pwlanhdr = NULL;
+	struct pkt_attrib *pattrib = NULL;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	u8 category = RTW_WLAN_CATEGORY_MESH;
+	u8 action = RTW_ACT_MESH_HWMP_PATH_SELECTION;
+	u8 *pos, ie_len;
+	u16 *fctrl = NULL;
+
+	if (rtw_time_before(rtw_get_current_time(), minfo->next_perr))
+		return -1;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return -1;
+
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(adapter, pattrib);
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pos = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pos;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, ra, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(adapter), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, adapter_mac_addr(adapter), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pos, WIFI_ACTION);
+
+	pos += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pos = rtw_set_fixed_ie(pos, 1, &(category), &(pattrib->pktlen));
+	pos = rtw_set_fixed_ie(pos, 1, &(action), &(pattrib->pktlen));
+
+	ie_len = 15;
+	pattrib->pktlen += (2 + ie_len);
+	*pos++ = WLAN_EID_PERR;
+	*pos++ = ie_len;
+	/* ttl */
+	*pos++ = ttl;
+	/* The Number of Destinations N */
+	*pos++ = 1;
+	/* Flags format | B7 | B6 | B5:B0 | = | rsvd | AE | rsvd | */
+	*pos = 0;
+	pos++;
+	_rtw_memcpy(pos, target, ETH_ALEN);
+	pos += ETH_ALEN;
+	*(u32 *)pos = cpu_to_le32(target_sn);
+	pos += 4;
+	*(u16 *)pos = cpu_to_le16(perr_reason_code);
+
+	adapter->mesh_info.next_perr = RTW_TU_TO_EXP_TIME(
+				adapter->mesh_cfg.dot11MeshHWMPperrMinInterval);
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	/* Send directly. Rewrite it if deferred tx is needed */
+	dump_mgntframe(adapter, pmgntframe);
+
+	RTW_HWMP_DBG("TX PERR toward "MAC_FMT", ra = "MAC_FMT"\n", MAC_ARG(target), MAC_ARG(ra));
+	
+	return 0;
+}
+
+static u32 rtw_airtime_link_metric_get(_adapter *adapter, struct sta_info *sta)
+{
+	int device_constant = rtw_hal_phydm_get_plcp(adapter, sta->phl_sta->macid) << RTW_ARITH_SHIFT;
+	u32 test_frame_len = RTW_TEST_FRAME_LEN << RTW_ARITH_SHIFT;
+	u32 s_unit = 1 << RTW_ARITH_SHIFT;
+	u32 err;
+	u16 rate;
+	u32 tx_time, estimated_retx;
+	u64 result;
+	/* The fail_avg should <= 100 here */
+	u32 fail_avg = (u32)rtw_ewma_err_rate_read(&sta->metrics.err_rate);
+
+	if (fail_avg > RTW_LINK_FAIL_THRESH)
+		return RTW_MAX_METRIC;
+
+	rate = sta->metrics.data_rate;
+	/* rate unit is 100Kbps, min rate = 10 */
+	if (rate < 10) {
+		RTW_HWMP_INFO("rate = %d\n", rate);
+		return RTW_MAX_METRIC;
+	}
+
+	err = (fail_avg << RTW_ARITH_SHIFT) / 100;
+
+	/* test_frame_len*10 to adjust the unit of rate(100kbps/unit) */
+	tx_time = (device_constant + 10 * test_frame_len / rate);
+	estimated_retx = ((1 << (2 * RTW_ARITH_SHIFT)) / (s_unit - err));
+	result = (tx_time * estimated_retx) >> (2 * RTW_ARITH_SHIFT);
+	/* Convert us to 0.01 TU(10.24us). x/10.24 = x*100/1024 */
+	result = (result * 100) >> 10;
+
+	return (u32)result;
+}
+
+void rtw_ieee80211s_update_metric(_adapter *adapter, u8 mac_id,
+				  u8 per, u8 rate,
+				  u8 bw, u8 total_pkt)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj);
+	struct sta_info *sta;
+	u8 rate_idx;
+	u8 sgi;
+
+	sta = macid_ctl->sta[mac_id];
+	if (!sta)
+		return;
+
+	/* if RA, use reported rate */
+	if (adapter->fix_rate == 0xff) {
+		rate_idx = rate & 0x7f;
+		sgi = rate >> 7;
+	} else {
+		rate_idx = adapter->fix_rate & 0x7f;
+		sgi = adapter->fix_rate >> 7;
+	}
+	sta->metrics.data_rate = rtw_desc_rate_to_bitrate(bw, rate_idx, sgi);
+
+	if (total_pkt < RTW_TOTAL_PKT_MIN_THRESHOLD)
+		return;
+
+	/* TBD: sta->metrics.overhead = phydm_get_plcp(void *dm_void, u16 macid); */
+	sta->metrics.total_pkt = total_pkt;
+
+	rtw_ewma_err_rate_add(&sta->metrics.err_rate, per);
+	if (rtw_ewma_err_rate_read(&sta->metrics.err_rate) > 
+			RTW_LINK_FAIL_THRESH)
+		rtw_mesh_plink_broken(sta);
+}
+
+static void rtw_hwmp_preq_frame_process(_adapter *adapter,
+					struct rtw_ieee80211_hdr_3addr *mgmt,
+					const u8 *preq_elem, u32 originator_metric)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mesh_cfg *mshcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_path *path = NULL;
+	const u8 *target_addr, *originator_addr;
+	const u8 *da;
+	u8 target_flags, ttl, flags, to_gate_ask = 0;
+	u32 originator_sn, target_sn, lifetime, target_metric = 0;
+	BOOLEAN reply = _FALSE;
+	BOOLEAN forward = _TRUE;
+	BOOLEAN preq_is_gate;
+
+	/* Update target SN, if present */
+	target_addr = RTW_PREQ_IE_TARGET_ADDR(preq_elem);
+	originator_addr = RTW_PREQ_IE_ORIG_ADDR(preq_elem);
+	target_sn = RTW_PREQ_IE_TARGET_SN(preq_elem);
+	originator_sn = RTW_PREQ_IE_ORIG_SN(preq_elem);
+	target_flags = RTW_PREQ_IE_TARGET_F(preq_elem);
+	/* PREQ gate announcements */
+	flags = RTW_PREQ_IE_FLAGS(preq_elem);
+	preq_is_gate = !!(flags & RTW_IEEE80211_PREQ_IS_GATE_FLAG);
+
+	RTW_HWMP_DBG("received PREQ from "MAC_FMT"\n", MAC_ARG(originator_addr));
+
+	if (rtw_ether_addr_equal(target_addr, adapter_mac_addr(adapter))) {
+		RTW_HWMP_DBG("PREQ is for us\n");
+#ifdef CONFIG_RTW_MESH_ON_DMD_GANN
+		rtw_rcu_read_lock();
+		path = rtw_mesh_path_lookup(adapter, originator_addr);
+		if (path) {
+			if (preq_is_gate)
+				rtw_mesh_path_add_gate(path);
+			else if (path->is_gate) {
+				_rtw_spinlock_bh(&path->state_lock);
+				rtw_mesh_gate_del(adapter->mesh_info.mesh_paths, path);
+				_rtw_spinunlock_bh(&path->state_lock);
+			}
+		}
+		path = NULL;
+		rtw_rcu_read_unlock();
+#endif
+		forward = _FALSE;
+		reply = _TRUE;
+		to_gate_ask = 1;
+		target_metric = 0;
+		if (rtw_time_after(rtw_get_current_time(), minfo->last_sn_update +
+					rtw_net_traversal_jiffies(adapter)) ||
+		    rtw_time_before(rtw_get_current_time(), minfo->last_sn_update)) {
+			++minfo->sn;
+			minfo->last_sn_update = rtw_get_current_time();
+		}
+		target_sn = minfo->sn;
+	} else if (is_broadcast_mac_addr(target_addr) &&
+		   (target_flags & RTW_IEEE80211_PREQ_TO_FLAG)) {
+		rtw_rcu_read_lock();
+		path = rtw_mesh_path_lookup(adapter, originator_addr);
+		if (path) {
+			if (flags & RTW_IEEE80211_PREQ_PROACTIVE_PREP_FLAG) {
+				reply = _TRUE;
+				target_addr = adapter_mac_addr(adapter);
+				target_sn = ++minfo->sn;
+				target_metric = 0;
+				minfo->last_sn_update = rtw_get_current_time();
+			}
+
+			if (preq_is_gate) {
+				lifetime = RTW_PREQ_IE_LIFETIME(preq_elem);
+				path->gate_ann_int = lifetime;
+				path->gate_asked = false;
+				rtw_mesh_path_add_gate(path);
+			} else if (path->is_gate) {
+				_rtw_spinlock_bh(&path->state_lock);
+				rtw_mesh_gate_del(adapter->mesh_info.mesh_paths, path);
+				_rtw_spinunlock_bh(&path->state_lock);
+			}
+		}
+		rtw_rcu_read_unlock();
+	} else {
+		rtw_rcu_read_lock();
+#ifdef CONFIG_RTW_MESH_ON_DMD_GANN
+		path = rtw_mesh_path_lookup(adapter, originator_addr);
+		if (path) {
+			if (preq_is_gate)
+				rtw_mesh_path_add_gate(path);
+			else if (path->is_gate) {
+				_rtw_spinlock_bh(&path->state_lock);
+				rtw_mesh_gate_del(adapter->mesh_info.mesh_paths, path);
+				_rtw_spinunlock_bh(&path->state_lock);
+			}
+		}
+		path = NULL;
+#endif
+		path = rtw_mesh_path_lookup(adapter, target_addr);
+		if (path) {
+			if ((!(path->flags & RTW_MESH_PATH_SN_VALID)) ||
+					RTW_SN_LT(path->sn, target_sn)) {
+				path->sn = target_sn;
+				path->flags |= RTW_MESH_PATH_SN_VALID;
+			} else if ((!(target_flags & RTW_IEEE80211_PREQ_TO_FLAG)) &&
+					(path->flags & RTW_MESH_PATH_ACTIVE)) {
+				reply = _TRUE;
+				target_metric = path->metric;
+				target_sn = path->sn;
+				/* Case E2 of sec 13.10.9.3 IEEE 802.11-2012*/
+				target_flags |= RTW_IEEE80211_PREQ_TO_FLAG;
+			}
+		}
+		rtw_rcu_read_unlock();
+	}
+
+	if (reply) {
+		lifetime = RTW_PREQ_IE_LIFETIME(preq_elem);
+		ttl = mshcfg->element_ttl;
+		if (ttl != 0 && !to_gate_ask) {
+			RTW_HWMP_DBG("replying to the PREQ\n");
+			rtw_mesh_path_sel_frame_tx(RTW_MPATH_PREP, 0, originator_addr,
+						   originator_sn, 0, target_addr,
+						   target_sn, mgmt->addr2, 0, ttl,
+						   lifetime, target_metric, 0,
+						   adapter);
+		} else if (ttl != 0 && to_gate_ask) {
+			RTW_HWMP_DBG("replying to the PREQ (PREQ for us)\n");
+			if (mshcfg->dot11MeshGateAnnouncementProtocol) {
+				/* BIT 7 is used to identify the prep is from mesh gate */
+				to_gate_ask = RTW_IEEE80211_PREQ_IS_GATE_FLAG | BIT(7);
+			} else {
+				to_gate_ask = 0;
+			}
+
+			rtw_mesh_path_sel_frame_tx(RTW_MPATH_PREP, to_gate_ask, originator_addr,
+						   originator_sn, 0, target_addr,
+						   target_sn, mgmt->addr2, 0, ttl,
+						   lifetime, target_metric, 0,
+						   adapter);
+		} else {
+			minfo->mshstats.dropped_frames_ttl++;
+		}
+	}
+
+	if (forward && mshcfg->dot11MeshForwarding) {
+		u32 preq_id;
+		u8 hopcount;
+
+		ttl = RTW_PREQ_IE_TTL(preq_elem);
+		lifetime = RTW_PREQ_IE_LIFETIME(preq_elem);
+		if (ttl <= 1) {
+			minfo->mshstats.dropped_frames_ttl++;
+			return;
+		}
+		RTW_HWMP_DBG("forwarding the PREQ from "MAC_FMT"\n", MAC_ARG(originator_addr));
+		--ttl;
+		preq_id = RTW_PREQ_IE_PREQ_ID(preq_elem);
+		hopcount = RTW_PREQ_IE_HOPCOUNT(preq_elem) + 1;
+		da = (path && path->is_root) ?
+			path->rann_snd_addr : bcast_addr;
+
+		if (flags & RTW_IEEE80211_PREQ_PROACTIVE_PREP_FLAG) {
+			target_addr = RTW_PREQ_IE_TARGET_ADDR(preq_elem);
+			target_sn = RTW_PREQ_IE_TARGET_SN(preq_elem);
+		}
+
+		rtw_mesh_path_sel_frame_tx(RTW_MPATH_PREQ, flags, originator_addr,
+					   originator_sn, target_flags, target_addr,
+					   target_sn, da, hopcount, ttl, lifetime,
+					   originator_metric, preq_id, adapter);
+		if (!is_multicast_mac_addr(da))
+			minfo->mshstats.fwded_unicast++;
+		else
+			minfo->mshstats.fwded_mcast++;
+		minfo->mshstats.fwded_frames++;
+	}
+}
+
+static inline struct sta_info *
+rtw_next_hop_deref_protected(struct rtw_mesh_path *path)
+{
+	return rtw_rcu_dereference_protected(path->next_hop,
+					 rtw_lockdep_is_held(&path->state_lock));
+}
+
+static void rtw_hwmp_prep_frame_process(_adapter *adapter,
+					struct rtw_ieee80211_hdr_3addr *mgmt,
+					const u8 *prep_elem, u32 metric)
+{
+	struct rtw_mesh_cfg *mshcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_stats *mshstats = &adapter->mesh_info.mshstats;
+	struct rtw_mesh_path *path;
+	const u8 *target_addr, *originator_addr;
+	u8 ttl, hopcount, flags;
+	u8 next_hop[ETH_ALEN];
+	u32 target_sn, originator_sn, lifetime;
+
+	RTW_HWMP_DBG("received PREP from "MAC_FMT"\n",
+		  MAC_ARG(RTW_PREP_IE_TARGET_ADDR(prep_elem)));
+
+	originator_addr = RTW_PREP_IE_ORIG_ADDR(prep_elem);
+	if (rtw_ether_addr_equal(originator_addr, adapter_mac_addr(adapter))) {
+		/* destination, no forwarding required */
+		rtw_rcu_read_lock();
+		target_addr = RTW_PREP_IE_TARGET_ADDR(prep_elem);
+		path = rtw_mesh_path_lookup(adapter, target_addr);
+		if (path && path->gate_asked) {
+			flags = RTW_PREP_IE_FLAGS(prep_elem);
+			if (flags & BIT(7)) {
+				_rtw_spinlock_bh(&path->state_lock);
+				path->gate_asked = false;
+				_rtw_spinunlock_bh(&path->state_lock);
+				if (!(flags & RTW_IEEE80211_PREQ_IS_GATE_FLAG)) {
+					_rtw_spinlock_bh(&path->state_lock);
+					rtw_mesh_gate_del(adapter->mesh_info.mesh_paths, path);
+					_rtw_spinunlock_bh(&path->state_lock);
+				}
+			}
+		}
+
+		rtw_rcu_read_unlock();
+		return;
+	}
+
+	if (!mshcfg->dot11MeshForwarding)
+		return;
+
+	ttl = RTW_PREP_IE_TTL(prep_elem);
+	if (ttl <= 1) {
+		mshstats->dropped_frames_ttl++;
+		return;
+	}
+
+	rtw_rcu_read_lock();
+	path = rtw_mesh_path_lookup(adapter, originator_addr);
+	if (path)
+		_rtw_spinlock_bh(&path->state_lock);
+	else
+		goto fail;
+	if (!(path->flags & RTW_MESH_PATH_ACTIVE)) {
+		_rtw_spinunlock_bh(&path->state_lock);
+		goto fail;
+	}
+	_rtw_memcpy(next_hop, rtw_next_hop_deref_protected(path)->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_spinunlock_bh(&path->state_lock);
+	--ttl;
+	flags = RTW_PREP_IE_FLAGS(prep_elem);
+	lifetime = RTW_PREP_IE_LIFETIME(prep_elem);
+	hopcount = RTW_PREP_IE_HOPCOUNT(prep_elem) + 1;
+	target_addr = RTW_PREP_IE_TARGET_ADDR(prep_elem);
+	target_sn = RTW_PREP_IE_TARGET_SN(prep_elem);
+	originator_sn = RTW_PREP_IE_ORIG_SN(prep_elem);
+
+	rtw_mesh_path_sel_frame_tx(RTW_MPATH_PREP, flags, originator_addr, originator_sn, 0,
+				   target_addr, target_sn, next_hop, hopcount,
+				   ttl, lifetime, metric, 0, adapter);
+	rtw_rcu_read_unlock();
+
+	mshstats->fwded_unicast++;
+	mshstats->fwded_frames++;
+	return;
+
+fail:
+	rtw_rcu_read_unlock();
+	mshstats->dropped_frames_no_route++;
+}
+
+static void rtw_hwmp_perr_frame_process(_adapter *adapter,
+					struct rtw_ieee80211_hdr_3addr *mgmt,
+					const u8 *perr_elem)
+{
+	struct rtw_mesh_cfg *mshcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_stats *mshstats = &adapter->mesh_info.mshstats;
+	struct rtw_mesh_path *path;
+	u8 ttl;
+	const u8 *ta, *target_addr;
+	u32 target_sn;
+	u16 perr_reason_code;
+
+	ta = mgmt->addr2;
+	ttl = RTW_PERR_IE_TTL(perr_elem);
+	if (ttl <= 1) {
+		mshstats->dropped_frames_ttl++;
+		return;
+	}
+	ttl--;
+	target_addr = RTW_PERR_IE_TARGET_ADDR(perr_elem);
+	target_sn = RTW_PERR_IE_TARGET_SN(perr_elem);
+	perr_reason_code = RTW_PERR_IE_TARGET_RCODE(perr_elem);
+
+	RTW_HWMP_DBG("received PERR toward target "MAC_FMT"\n", MAC_ARG(target_addr));
+
+	rtw_rcu_read_lock();
+	path = rtw_mesh_path_lookup(adapter, target_addr);
+	if (path) {
+		struct sta_info *sta;
+
+		_rtw_spinlock_bh(&path->state_lock);
+		sta = rtw_next_hop_deref_protected(path);
+		if (path->flags & RTW_MESH_PATH_ACTIVE &&
+		    rtw_ether_addr_equal(ta, sta->phl_sta->mac_addr) &&
+		    !(path->flags & RTW_MESH_PATH_FIXED) &&
+		    (!(path->flags & RTW_MESH_PATH_SN_VALID) ||
+		    RTW_SN_GT(target_sn, path->sn)  || target_sn == 0)) {
+			path->flags &= ~RTW_MESH_PATH_ACTIVE;
+			if (target_sn != 0)
+				path->sn = target_sn;
+			else
+				path->sn += 1;
+			_rtw_spinunlock_bh(&path->state_lock);
+			if (!mshcfg->dot11MeshForwarding)
+				goto endperr;
+			rtw_mesh_path_error_tx(adapter, ttl, target_addr,
+					       target_sn, perr_reason_code,
+					       bcast_addr);
+		} else
+			_rtw_spinunlock_bh(&path->state_lock);
+	}
+endperr:
+	rtw_rcu_read_unlock();
+}
+
+static void rtw_hwmp_rann_frame_process(_adapter *adapter,
+					struct rtw_ieee80211_hdr_3addr *mgmt,
+					const struct rtw_ieee80211_rann_ie *rann)
+{
+	struct sta_info *sta;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	struct rtw_mesh_cfg *mshcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_stats *mshstats = &adapter->mesh_info.mshstats;
+	struct rtw_mesh_path *path;
+	u8 ttl, flags, hopcount;
+	const u8 *originator_addr;
+	u32 originator_sn, metric, metric_txsta, interval;
+	BOOLEAN root_is_gate;
+
+	ttl = rann->rann_ttl;
+	flags = rann->rann_flags;
+	root_is_gate = !!(flags & RTW_RANN_FLAG_IS_GATE);
+	originator_addr = rann->rann_addr;
+	originator_sn = le32_to_cpu(rann->rann_seq);
+	interval = le32_to_cpu(rann->rann_interval);
+	hopcount = rann->rann_hopcount;
+	hopcount++;
+	metric = le32_to_cpu(rann->rann_metric);
+
+	/*  Ignore our own RANNs */
+	if (rtw_ether_addr_equal(originator_addr, adapter_mac_addr(adapter)))
+		return;
+
+	RTW_HWMP_DBG("received RANN from "MAC_FMT" via neighbour "MAC_FMT" (is_gate=%d)\n",
+		  MAC_ARG(originator_addr), MAC_ARG(mgmt->addr2), root_is_gate);
+
+	rtw_rcu_read_lock();
+	sta = rtw_get_stainfo(pstapriv, mgmt->addr2);
+	if (!sta) {
+		rtw_rcu_read_unlock();
+		return;
+	}
+
+	metric_txsta = rtw_airtime_link_metric_get(adapter, sta);
+
+	path = rtw_mesh_path_lookup(adapter, originator_addr);
+	if (!path) {
+		path = rtw_mesh_path_add(adapter, originator_addr);
+		if (IS_ERR(path)) {
+			rtw_rcu_read_unlock();
+			mshstats->dropped_frames_no_route++;
+			return;
+		}
+	}
+
+	if (!(RTW_SN_LT(path->sn, originator_sn)) &&
+	    !(path->sn == originator_sn && metric < path->rann_metric)) {
+		rtw_rcu_read_unlock();
+		return;
+	}
+
+	if ((!(path->flags & (RTW_MESH_PATH_ACTIVE | RTW_MESH_PATH_RESOLVING)) ||
+	     (rtw_time_after(rtw_get_current_time(), path->last_preq_to_root +
+				  rtw_root_path_confirmation_jiffies(adapter)) ||
+	     rtw_time_before(rtw_get_current_time(), path->last_preq_to_root))) &&
+	     !(path->flags & RTW_MESH_PATH_FIXED) && (ttl != 0)) {
+		u8 preq_node_flag = RTW_PREQ_Q_F_START | RTW_PREQ_Q_F_REFRESH;
+
+		RTW_HWMP_DBG("time to refresh root path "MAC_FMT"\n",
+			  MAC_ARG(originator_addr));
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+		if (RTW_SN_LT(path->sn, originator_sn) &&
+		    (path->rann_metric + mshcfg->sane_metric_delta < metric) &&
+		    _rtw_memcmp(bcast_addr, path->rann_snd_addr, ETH_ALEN) == _FALSE) {
+			RTW_HWMP_DBG("Trigger additional check for root "
+				     "confirm PREQ. rann_snd_addr = "MAC_FMT
+				     "add_chk_rann_snd_addr= "MAC_FMT"\n",
+					MAC_ARG(mgmt->addr2),
+					MAC_ARG(path->rann_snd_addr));
+			_rtw_memcpy(path->add_chk_rann_snd_addr,
+				    path->rann_snd_addr, ETH_ALEN);
+			preq_node_flag |= RTW_PREQ_Q_F_CHK;
+			
+		}
+#endif
+		rtw_mesh_queue_preq(path, preq_node_flag);
+		path->last_preq_to_root = rtw_get_current_time();
+	}
+
+	path->sn = originator_sn;
+	path->rann_metric = metric + metric_txsta;
+	path->is_root = _TRUE;
+	/* Recording RANNs sender address to send individually
+	 * addressed PREQs destined for root mesh STA */
+	_rtw_memcpy(path->rann_snd_addr, mgmt->addr2, ETH_ALEN);
+
+	if (root_is_gate) {
+		path->gate_ann_int = interval;
+		path->gate_asked = false;
+		rtw_mesh_path_add_gate(path);
+	} else if (path->is_gate) {
+		_rtw_spinlock_bh(&path->state_lock);
+		rtw_mesh_gate_del(adapter->mesh_info.mesh_paths, path);
+		_rtw_spinunlock_bh(&path->state_lock);
+	}
+
+	if (ttl <= 1) {
+		mshstats->dropped_frames_ttl++;
+		rtw_rcu_read_unlock();
+		return;
+	}
+	ttl--;
+
+	if (mshcfg->dot11MeshForwarding) {
+		rtw_mesh_path_sel_frame_tx(RTW_MPATH_RANN, flags, originator_addr,
+					   originator_sn, 0, NULL, 0, bcast_addr,
+					   hopcount, ttl, interval,
+					   metric + metric_txsta, 0, adapter);
+	}
+
+	rtw_rcu_read_unlock();
+}
+
+static u32 rtw_hwmp_route_info_get(_adapter *adapter,
+				   struct rtw_ieee80211_hdr_3addr *mgmt,
+				   const u8 *hwmp_ie, enum rtw_mpath_frame_type action)
+{
+	struct rtw_mesh_path *path;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	struct sta_info *sta;
+	BOOLEAN fresh_info;
+	const u8 *originator_addr, *ta;
+	u32 originator_sn, originator_metric;
+	unsigned long originator_lifetime, exp_time;
+	u32 last_hop_metric, new_metric;
+	BOOLEAN process = _TRUE;
+
+	rtw_rcu_read_lock();
+	sta = rtw_get_stainfo(pstapriv, mgmt->addr2);
+	if (!sta) {
+		rtw_rcu_read_unlock();
+		return 0;
+	}
+
+	last_hop_metric = rtw_airtime_link_metric_get(adapter, sta);
+	/* Update and check originator routing info */
+	fresh_info = _TRUE;
+
+	switch (action) {
+	case RTW_MPATH_PREQ:
+		originator_addr = RTW_PREQ_IE_ORIG_ADDR(hwmp_ie);
+		originator_sn = RTW_PREQ_IE_ORIG_SN(hwmp_ie);
+		originator_lifetime = RTW_PREQ_IE_LIFETIME(hwmp_ie);
+		originator_metric = RTW_PREQ_IE_METRIC(hwmp_ie);
+		break;
+	case RTW_MPATH_PREP:
+		/* Note: For coding, the naming is not consist with spec */
+		originator_addr = RTW_PREP_IE_TARGET_ADDR(hwmp_ie);
+		originator_sn = RTW_PREP_IE_TARGET_SN(hwmp_ie);
+		originator_lifetime = RTW_PREP_IE_LIFETIME(hwmp_ie);
+		originator_metric = RTW_PREP_IE_METRIC(hwmp_ie);
+		break;
+	default:
+		rtw_rcu_read_unlock();
+		return 0;
+	}
+	new_metric = originator_metric + last_hop_metric;
+	if (new_metric < originator_metric)
+		new_metric = RTW_MAX_METRIC;
+	exp_time = RTW_TU_TO_EXP_TIME(originator_lifetime);
+
+	if (rtw_ether_addr_equal(originator_addr, adapter_mac_addr(adapter))) {
+		process = _FALSE;
+		fresh_info = _FALSE;
+	} else {
+		path = rtw_mesh_path_lookup(adapter, originator_addr);
+		if (path) {
+			_rtw_spinlock_bh(&path->state_lock);
+			if (path->flags & RTW_MESH_PATH_FIXED)
+				fresh_info = _FALSE;
+			else if ((path->flags & RTW_MESH_PATH_ACTIVE) &&
+			    (path->flags & RTW_MESH_PATH_SN_VALID)) {
+				if (RTW_SN_GT(path->sn, originator_sn) ||
+				    (path->sn == originator_sn &&
+				     new_metric >= path->metric)) {
+					process = _FALSE;
+					fresh_info = _FALSE;
+				}
+			} else if (!(path->flags & RTW_MESH_PATH_ACTIVE)) {
+				BOOLEAN have_sn, newer_sn, bounced;
+
+				have_sn = path->flags & RTW_MESH_PATH_SN_VALID;
+				newer_sn = have_sn && RTW_SN_GT(originator_sn, path->sn);
+				bounced = have_sn &&
+					  (RTW_SN_DELTA(originator_sn, path->sn) >
+							RTW_MAX_SANE_SN_DELTA);
+
+				if (!have_sn || newer_sn) {
+				} else if (bounced) {
+				} else {
+					process = _FALSE;
+					fresh_info = _FALSE;
+				}
+			}
+		} else {
+			path = rtw_mesh_path_add(adapter, originator_addr);
+			if (IS_ERR(path)) {
+				rtw_rcu_read_unlock();
+				return 0;
+			}
+			_rtw_spinlock_bh(&path->state_lock);
+		}
+
+		if (fresh_info) {
+			rtw_mesh_path_assign_nexthop(path, sta);
+			path->flags |= RTW_MESH_PATH_SN_VALID;
+			path->metric = new_metric;
+			path->sn = originator_sn;
+			path->exp_time = rtw_time_after(path->exp_time, exp_time)
+					  ?  path->exp_time : exp_time;
+			rtw_mesh_path_activate(path);
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+			if (path->is_root && (action == RTW_MPATH_PREP)) {
+				_rtw_memcpy(path->rann_snd_addr, 
+				mgmt->addr2, ETH_ALEN);
+				path->rann_metric = new_metric;
+			}
+#endif
+			_rtw_spinunlock_bh(&path->state_lock);
+			rtw_mesh_path_tx_pending(path);
+		} else
+			_rtw_spinunlock_bh(&path->state_lock);
+	}
+
+	/* Update and check transmitter routing info */
+	ta = mgmt->addr2;
+	if (rtw_ether_addr_equal(originator_addr, ta))
+		fresh_info = _FALSE;
+	else {
+		fresh_info = _TRUE;
+
+		path = rtw_mesh_path_lookup(adapter, ta);
+		if (path) {
+			_rtw_spinlock_bh(&path->state_lock);
+			if ((path->flags & RTW_MESH_PATH_FIXED) ||
+				((path->flags & RTW_MESH_PATH_ACTIVE) &&
+					(last_hop_metric > path->metric)))
+				fresh_info = _FALSE;
+		} else {
+			path = rtw_mesh_path_add(adapter, ta);
+			if (IS_ERR(path)) {
+				rtw_rcu_read_unlock();
+				return 0;
+			}
+			_rtw_spinlock_bh(&path->state_lock);
+		}
+
+		if (fresh_info) {
+			rtw_mesh_path_assign_nexthop(path, sta);
+			path->metric = last_hop_metric;
+			path->exp_time = rtw_time_after(path->exp_time, exp_time)
+					  ?  path->exp_time : exp_time;
+			rtw_mesh_path_activate(path);
+			_rtw_spinunlock_bh(&path->state_lock);
+			rtw_mesh_path_tx_pending(path);
+		} else
+			_rtw_spinunlock_bh(&path->state_lock);
+	}
+
+	rtw_rcu_read_unlock();
+
+	return process ? new_metric : 0;
+}
+
+static void rtw_mesh_rx_hwmp_frame_cnts(_adapter *adapter, u8 *addr)
+{
+	struct sta_info *sta;
+
+	sta = rtw_get_stainfo(&adapter->stapriv, addr);
+	if (sta)
+		sta->sta_stats.rx_hwmp_pkts++;
+}
+
+void rtw_mesh_rx_path_sel_frame(_adapter *adapter, union recv_frame *rframe)
+{
+	struct mesh_plink_ent *plink = NULL;
+	struct rtw_ieee802_11_elems elems;
+	u32 path_metric;
+	struct rx_pkt_attrib *attrib = &rframe->u.hdr.attrib;
+	u8 *pframe = rframe->u.hdr.rx_data, *start;
+	uint frame_len = rframe->u.hdr.len, left;
+	struct rtw_ieee80211_hdr_3addr *frame_hdr = (struct rtw_ieee80211_hdr_3addr *)pframe;
+	u8 *frame_body = (u8 *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+	ParseRes parse_res;
+
+	plink = rtw_mesh_plink_get(adapter, get_addr2_ptr(pframe));
+	if (!plink || plink->plink_state != RTW_MESH_PLINK_ESTAB)
+		return;
+
+	rtw_mesh_rx_hwmp_frame_cnts(adapter, get_addr2_ptr(pframe));
+
+	/* Mesh action frame IE offset = 2 */
+	attrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	left = frame_len - attrib->hdrlen - attrib->iv_len - attrib->icv_len - 2;
+	start = pframe + attrib->hdrlen + 2;
+
+	parse_res = rtw_ieee802_11_parse_elems(start, left, &elems, 1);
+	if (parse_res == ParseFailed)
+		RTW_HWMP_INFO(FUNC_ADPT_FMT" Path Select Frame ParseFailed\n"
+			, FUNC_ADPT_ARG(adapter));
+	else if (parse_res == ParseUnknown)
+		RTW_HWMP_INFO(FUNC_ADPT_FMT" Path Select Frame ParseUnknown\n"
+			, FUNC_ADPT_ARG(adapter));
+
+	if (elems.preq) {
+		if (elems.preq_len != 37)
+			/* Right now we support just 1 destination and no AE */
+			return;
+		path_metric = rtw_hwmp_route_info_get(adapter, frame_hdr, elems.preq,
+						  MPATH_PREQ);
+		if (path_metric)
+			rtw_hwmp_preq_frame_process(adapter, frame_hdr, elems.preq,
+						path_metric);
+	}
+	if (elems.prep) {
+		if (elems.prep_len != 31)
+			/* Right now we support no AE */
+			return;
+		path_metric = rtw_hwmp_route_info_get(adapter, frame_hdr, elems.prep,
+						  MPATH_PREP);
+		if (path_metric)
+			rtw_hwmp_prep_frame_process(adapter, frame_hdr, elems.prep,
+						path_metric);
+	}
+	if (elems.perr) {
+		if (elems.perr_len != 15)
+			/* Right now we support only one destination per PERR */
+			return;
+		rtw_hwmp_perr_frame_process(adapter, frame_hdr, elems.perr);
+	}
+	if (elems.rann)
+		rtw_hwmp_rann_frame_process(adapter, frame_hdr, (struct rtw_ieee80211_rann_ie *)elems.rann);
+}
+
+void rtw_mesh_queue_preq(struct rtw_mesh_path *path, u8 flags)
+{
+	_adapter *adapter = path->adapter;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mesh_preq_queue *preq_node;
+
+	preq_node = rtw_malloc(sizeof(struct rtw_mesh_preq_queue));
+	if (!preq_node) {
+		RTW_HWMP_INFO("could not allocate PREQ node\n");
+		return;
+	}
+
+	_rtw_spinlock_bh(&minfo->mesh_preq_queue_lock);
+	if (minfo->preq_queue_len == RTW_MAX_PREQ_QUEUE_LEN) {
+		_rtw_spinunlock_bh(&minfo->mesh_preq_queue_lock);
+		rtw_mfree(preq_node, sizeof(struct rtw_mesh_preq_queue));
+		if (rtw_print_ratelimit())
+			RTW_HWMP_INFO("PREQ node queue full\n");
+		return;
+	}
+
+	_rtw_spinlock(&path->state_lock);
+	if (path->flags & RTW_MESH_PATH_REQ_QUEUED) {
+		_rtw_spinunlock(&path->state_lock);
+		_rtw_spinunlock_bh(&minfo->mesh_preq_queue_lock);
+		rtw_mfree(preq_node, sizeof(struct rtw_mesh_preq_queue));
+		return;
+	}
+
+	_rtw_memcpy(preq_node->dst, path->dst, ETH_ALEN);
+	preq_node->flags = flags;
+
+	path->flags |= RTW_MESH_PATH_REQ_QUEUED;
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+	if (flags & RTW_PREQ_Q_F_CHK)
+		path->flags |= RTW_MESH_PATH_ROOT_ADD_CHK;
+#endif
+	if (flags & RTW_PREQ_Q_F_PEER_AKA)
+		path->flags |= RTW_MESH_PATH_PEER_AKA;
+	if (flags & RTW_PREQ_Q_F_BCAST_PREQ)
+		path->flags |= RTW_MESH_PATH_BCAST_PREQ;
+	_rtw_spinunlock(&path->state_lock);
+
+	rtw_list_insert_tail(&preq_node->list, &minfo->preq_queue.list);
+	++minfo->preq_queue_len;
+	_rtw_spinunlock_bh(&minfo->mesh_preq_queue_lock);
+
+	if (rtw_time_after(rtw_get_current_time(), minfo->last_preq + rtw_min_preq_int_jiff(adapter)))
+		rtw_mesh_work(&adapter->mesh_work);
+
+	else if (rtw_time_before(rtw_get_current_time(), minfo->last_preq)) {
+		/* systime wrapped around issue */
+		minfo->last_preq = rtw_get_current_time() - rtw_min_preq_int_jiff(adapter) - 1;
+		rtw_mesh_work(&adapter->mesh_work);
+	} else
+		rtw_mod_timer(&adapter->mesh_path_timer, minfo->last_preq +
+					rtw_min_preq_int_jiff(adapter) + 1);
+}
+
+static const u8 *rtw_hwmp_preq_da(struct rtw_mesh_path *path,
+			    BOOLEAN is_root_add_chk, BOOLEAN da_is_peer,
+			    BOOLEAN force_preq_bcast)
+{
+	const u8 *da;
+
+	if (da_is_peer)
+		da = path->dst;
+	else if (force_preq_bcast)
+		da = bcast_addr;
+	else if (path->is_root)
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+		da = is_root_add_chk ? path->add_chk_rann_snd_addr:
+				       path->rann_snd_addr;
+#else
+		da = path->rann_snd_addr;
+#endif
+	else
+		da = bcast_addr;
+
+	return da;
+}
+
+void rtw_mesh_path_start_discovery(_adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mesh_cfg *mshcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_preq_queue *preq_node;
+	struct rtw_mesh_path *path;
+	u8 ttl, target_flags = 0;
+	const u8 *da;
+	u32 lifetime;
+	u8 flags = 0;
+	BOOLEAN is_root_add_chk = _FALSE;
+	BOOLEAN da_is_peer, force_preq_bcast;
+
+	_rtw_spinlock_bh(&minfo->mesh_preq_queue_lock);
+	if (!minfo->preq_queue_len ||
+		rtw_time_before(rtw_get_current_time(), minfo->last_preq +
+				rtw_min_preq_int_jiff(adapter))) {
+		_rtw_spinunlock_bh(&minfo->mesh_preq_queue_lock);
+		return;
+	}
+
+	preq_node = rtw_list_first_entry(&minfo->preq_queue.list,
+			struct rtw_mesh_preq_queue, list);
+	rtw_list_delete(&preq_node->list); /* list_del_init(&preq_node->list); */
+	--minfo->preq_queue_len;
+	_rtw_spinunlock_bh(&minfo->mesh_preq_queue_lock);
+
+	rtw_rcu_read_lock();
+	path = rtw_mesh_path_lookup(adapter, preq_node->dst);
+	if (!path)
+		goto enddiscovery;
+
+	_rtw_spinlock_bh(&path->state_lock);
+	if (path->flags & (RTW_MESH_PATH_DELETED | RTW_MESH_PATH_FIXED)) {
+		_rtw_spinunlock_bh(&path->state_lock);
+		goto enddiscovery;
+	}
+	path->flags &= ~RTW_MESH_PATH_REQ_QUEUED;
+	if (preq_node->flags & RTW_PREQ_Q_F_START) {
+		if (path->flags & RTW_MESH_PATH_RESOLVING) {
+			_rtw_spinunlock_bh(&path->state_lock);
+			goto enddiscovery;
+		} else {
+			path->flags &= ~RTW_MESH_PATH_RESOLVED;
+			path->flags |= RTW_MESH_PATH_RESOLVING;
+			path->discovery_retries = 0;
+			path->discovery_timeout = rtw_disc_timeout_jiff(adapter);
+		}
+	} else if (!(path->flags & RTW_MESH_PATH_RESOLVING) ||
+			path->flags & RTW_MESH_PATH_RESOLVED) {
+		path->flags &= ~RTW_MESH_PATH_RESOLVING;
+		_rtw_spinunlock_bh(&path->state_lock);
+		goto enddiscovery;
+	}
+
+	minfo->last_preq = rtw_get_current_time();
+
+	if (rtw_time_after(rtw_get_current_time(), minfo->last_sn_update +
+				rtw_net_traversal_jiffies(adapter)) ||
+	    rtw_time_before(rtw_get_current_time(), minfo->last_sn_update)) {
+		++minfo->sn;
+		minfo->last_sn_update = rtw_get_current_time();
+	}
+	lifetime = rtw_default_lifetime(adapter);
+	ttl = mshcfg->element_ttl;
+	if (ttl == 0) {
+		minfo->mshstats.dropped_frames_ttl++;
+		_rtw_spinunlock_bh(&path->state_lock);
+		goto enddiscovery;
+	}
+
+	if (preq_node->flags & RTW_PREQ_Q_F_REFRESH)
+		target_flags |= RTW_IEEE80211_PREQ_TO_FLAG;
+	else
+		target_flags &= ~RTW_IEEE80211_PREQ_TO_FLAG;
+
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+	is_root_add_chk = !!(path->flags & RTW_MESH_PATH_ROOT_ADD_CHK);
+#endif
+	da_is_peer = !!(path->flags & RTW_MESH_PATH_PEER_AKA);
+	force_preq_bcast = !!(path->flags & RTW_MESH_PATH_BCAST_PREQ);
+	_rtw_spinunlock_bh(&path->state_lock);
+
+	da = rtw_hwmp_preq_da(path, is_root_add_chk,
+			      da_is_peer, force_preq_bcast);
+
+#ifdef CONFIG_RTW_MESH_ON_DMD_GANN
+	flags = (mshcfg->dot11MeshGateAnnouncementProtocol)
+		? RTW_IEEE80211_PREQ_IS_GATE_FLAG : 0;
+#endif
+	rtw_mesh_path_sel_frame_tx(RTW_MPATH_PREQ, flags, adapter_mac_addr(adapter), minfo->sn,
+				   target_flags, path->dst, path->sn, da, 0,
+				   ttl, lifetime, 0, minfo->preq_id++, adapter);
+	rtw_mod_timer(&path->timer, rtw_get_current_time() + path->discovery_timeout);
+
+enddiscovery:
+	rtw_rcu_read_unlock();
+	rtw_mfree(preq_node, sizeof(struct rtw_mesh_preq_queue));
+}
+
+void rtw_mesh_path_timer(void *ctx)
+{
+	struct rtw_mesh_path *path = (void *) ctx;
+	_adapter *adapter = path->adapter;
+	int ret;
+	u8 retry = 0;
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+	struct rtw_mesh_cfg *mshcfg = &adapter->mesh_cfg;
+#endif
+	/* TBD: Proctect for suspend */
+#if 0
+	if (suspending)
+		return;
+#endif
+	_rtw_spinlock_bh(&path->state_lock);
+	if (path->flags & RTW_MESH_PATH_RESOLVED ||
+			(!(path->flags & RTW_MESH_PATH_RESOLVING))) {
+		path->flags &= ~(RTW_MESH_PATH_RESOLVING |
+				 RTW_MESH_PATH_RESOLVED |
+				 RTW_MESH_PATH_ROOT_ADD_CHK |
+				 RTW_MESH_PATH_PEER_AKA |
+				 RTW_MESH_PATH_BCAST_PREQ);
+		_rtw_spinunlock_bh(&path->state_lock);
+	} else if (path->discovery_retries < rtw_max_preq_retries(adapter)) {
+		++path->discovery_retries;
+		path->discovery_timeout *= 2;
+		path->flags &= ~RTW_MESH_PATH_REQ_QUEUED;
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+		if (path->discovery_retries > mshcfg->max_root_add_chk_cnt)
+			path->flags &= ~RTW_MESH_PATH_ROOT_ADD_CHK;
+#endif
+		if (path->gate_asked)
+			retry |= RTW_PREQ_Q_F_REFRESH;
+
+		_rtw_spinunlock_bh(&path->state_lock);
+		rtw_mesh_queue_preq(path, retry);
+	} else {
+		path->flags &= ~(RTW_MESH_PATH_RESOLVING |
+				  RTW_MESH_PATH_RESOLVED |
+				  RTW_MESH_PATH_REQ_QUEUED |
+				  RTW_MESH_PATH_ROOT_ADD_CHK |
+				  RTW_MESH_PATH_PEER_AKA |
+				  RTW_MESH_PATH_BCAST_PREQ);
+		path->exp_time = rtw_get_current_time();
+		_rtw_spinunlock_bh(&path->state_lock);
+		if (!path->is_gate && rtw_mesh_gate_num(adapter) > 0) {
+			ret = rtw_mesh_path_send_to_gates(path);
+			if (ret)
+				RTW_HWMP_DBG("no gate was reachable\n");
+		} else
+			rtw_mesh_path_flush_pending(path);
+	}
+}
+
+
+void rtw_mesh_path_tx_root_frame(_adapter *adapter)
+{
+	struct rtw_mesh_cfg *mshcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	u32 interval = mshcfg->dot11MeshHWMPRannInterval;
+	u8 flags, target_flags = 0;
+
+	flags = (mshcfg->dot11MeshGateAnnouncementProtocol)
+			? RTW_RANN_FLAG_IS_GATE : 0;
+
+	switch (mshcfg->dot11MeshHWMPRootMode) {
+	case RTW_IEEE80211_PROACTIVE_RANN:
+		rtw_mesh_path_sel_frame_tx(RTW_MPATH_RANN, flags, adapter_mac_addr(adapter),
+					   ++minfo->sn, 0, NULL, 0, bcast_addr,
+					   0, mshcfg->element_ttl,
+					   interval, 0, 0, adapter);
+		break;
+	case RTW_IEEE80211_PROACTIVE_PREQ_WITH_PREP:
+		flags |= RTW_IEEE80211_PREQ_PROACTIVE_PREP_FLAG;
+	case RTW_IEEE80211_PROACTIVE_PREQ_NO_PREP:
+		interval = mshcfg->dot11MeshHWMPactivePathToRootTimeout;
+		target_flags |= RTW_IEEE80211_PREQ_TO_FLAG |
+				RTW_IEEE80211_PREQ_USN_FLAG;
+		rtw_mesh_path_sel_frame_tx(RTW_MPATH_PREQ, flags, adapter_mac_addr(adapter),
+					   ++minfo->sn, target_flags,
+					   (u8 *) bcast_addr, 0, bcast_addr,
+					   0, mshcfg->element_ttl, interval,
+					   0, minfo->preq_id++, adapter);
+		break;
+	default:
+		RTW_HWMP_INFO("Proactive mechanism not supported\n");
+		return;
+	}
+}
+
+void rtw_mesh_work(_workitem *work)
+{
+	/* use kernel global workqueue */
+	_set_workitem(work);
+}
+
+void rtw_ieee80211_mesh_path_timer(void *ctx)
+{
+	_adapter *adapter = (_adapter *)ctx;
+	rtw_mesh_work(&adapter->mesh_work);
+}
+
+void rtw_ieee80211_mesh_path_root_timer(void *ctx)
+{
+	_adapter *adapter = (_adapter *)ctx;
+
+	rtw_set_bit(RTW_MESH_WORK_ROOT, &adapter->wrkq_flags);
+
+	rtw_mesh_work(&adapter->mesh_work);
+}
+
+static void rtw_ieee80211_mesh_rootpath(_adapter *adapter)
+{
+	u32 interval;
+
+	rtw_mesh_path_tx_root_frame(adapter);
+
+	if (adapter->mesh_cfg.dot11MeshHWMPRootMode == RTW_IEEE80211_PROACTIVE_RANN)
+		interval = adapter->mesh_cfg.dot11MeshHWMPRannInterval;
+	else
+		interval = adapter->mesh_cfg.dot11MeshHWMProotInterval;
+
+	rtw_mod_timer(&adapter->mesh_path_root_timer,
+		  RTW_TU_TO_EXP_TIME(interval));
+}
+
+BOOLEAN rtw_ieee80211_mesh_root_setup(_adapter *adapter)
+{
+	BOOLEAN root_enabled = _FALSE;
+
+	if (adapter->mesh_cfg.dot11MeshHWMPRootMode > RTW_IEEE80211_ROOTMODE_ROOT) {
+		rtw_set_bit(RTW_MESH_WORK_ROOT, &adapter->wrkq_flags);
+		root_enabled = _TRUE;
+	}
+	else {
+		rtw_clear_bit(RTW_MESH_WORK_ROOT, &adapter->wrkq_flags);
+		/* stop running timer */
+		_cancel_timer_ex(&adapter->mesh_path_root_timer);
+		root_enabled = _FALSE;
+	}
+
+	return root_enabled;
+}
+
+void rtw_mesh_work_hdl(_workitem *work)
+{
+	_adapter *adapter = container_of(work, _adapter, mesh_work);
+
+	while(adapter->mesh_info.preq_queue_len) {
+		if (rtw_time_after(rtw_get_current_time(),
+		       adapter->mesh_info.last_preq + rtw_min_preq_int_jiff(adapter)))
+		       /* It will consume preq_queue_len */
+		       rtw_mesh_path_start_discovery(adapter);
+		else {
+			struct rtw_mesh_info *minfo = &adapter->mesh_info;
+
+			rtw_mod_timer(&adapter->mesh_path_timer,
+				minfo->last_preq + rtw_min_preq_int_jiff(adapter) + 1);
+			break;
+		}
+	}
+
+	if (rtw_test_and_clear_bit(RTW_MESH_WORK_ROOT, &adapter->wrkq_flags))
+		rtw_ieee80211_mesh_rootpath(adapter);
+}
+
+#ifndef RTW_PER_CMD_SUPPORT_FW
+static void rtw_update_metric_directly(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj);
+	u8 i;
+
+	for (i = 0; i < macid_ctl->num; i++) {
+		u8 role;
+		role = GET_H2CCMD_MSRRPT_PARM_ROLE(&macid_ctl->h2c_msr[i]);
+		if (role == H2C_MSR_ROLE_MESH) {
+			struct sta_info *sta = macid_ctl->sta[i];
+			u8 sgi, bw;
+			u16 rate_idx;
+			u32 rate;
+
+			if (!sta)
+				continue;
+			rate_idx = rtw_get_current_tx_rate(adapter, sta);
+			sgi = rtw_get_current_tx_sgi(adapter, sta);
+			bw = sta->phl_sta->chandef.bw;
+			rate = rtw_desc_rate_to_bitrate(bw, rate_idx, sgi);
+			sta->metrics.data_rate = rate;
+		}
+	}
+}
+#endif
+
+void rtw_mesh_atlm_param_req_timer(void *ctx)
+{
+	_adapter *adapter = (_adapter *)ctx;
+	u8 ret = _FAIL;
+
+#ifdef RTW_PER_CMD_SUPPORT_FW
+	ret = rtw_req_per_cmd(adapter);
+	if (ret == _FAIL)
+		RTW_HWMP_INFO("rtw_req_per_cmd fail\n");
+#else
+	rtw_update_metric_directly(adapter);
+#endif
+	_set_timer(&adapter->mesh_atlm_param_req_timer, RTW_ATLM_REQ_CYCLE);
+}
+
+#endif /* CONFIG_RTW_MESH */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_hwmp.h b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_hwmp.h
new file mode 100644
index 000000000000..466570e33e8c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_hwmp.h
@@ -0,0 +1,60 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_MESH_HWMP_H_
+#define __RTW_MESH_HWMP_H_
+
+#ifndef DBG_RTW_HWMP
+#define DBG_RTW_HWMP 0
+#endif
+#if DBG_RTW_HWMP
+#define RTW_HWMP_DBG(fmt, arg...) RTW_PRINT(fmt, ##arg)
+#else
+#define RTW_HWMP_DBG(fmt, arg...) RTW_DBG(fmt, ##arg)
+#endif
+
+#ifndef INFO_RTW_HWMP
+#define INFO_RTW_HWMP 0
+#endif
+#if INFO_RTW_HWMP
+#define RTW_HWMP_INFO(fmt, arg...) RTW_PRINT(fmt, ##arg)
+#else
+#define RTW_HWMP_INFO(fmt, arg...) RTW_INFO(fmt, ##arg)
+#endif
+
+
+void rtw_ewma_err_rate_init(struct rtw_ewma_err_rate *e);
+unsigned long rtw_ewma_err_rate_read(struct rtw_ewma_err_rate *e);
+void rtw_ewma_err_rate_add(struct rtw_ewma_err_rate *e, unsigned long val);
+int rtw_mesh_path_error_tx(_adapter *adapter,
+			   u8 ttl, const u8 *target, u32 target_sn,
+			   u16 target_rcode, const u8 *ra);
+void rtw_ieee80211s_update_metric(_adapter *adapter, u8 mac_id,
+				  u8 per, u8 rate,
+				  u8 bw, u8 total_pkt);
+void rtw_mesh_rx_path_sel_frame(_adapter *adapter, union recv_frame *rframe);
+void rtw_mesh_queue_preq(struct rtw_mesh_path *mpath, u8 flags);
+void rtw_mesh_path_start_discovery(_adapter *adapter);
+void rtw_mesh_path_timer(void *ctx);
+void rtw_mesh_path_tx_root_frame(_adapter *adapter);
+void rtw_mesh_work_hdl(_workitem *work);
+void rtw_ieee80211_mesh_path_timer(void *ctx);
+void rtw_ieee80211_mesh_path_root_timer(void *ctx);
+BOOLEAN rtw_ieee80211_mesh_root_setup(_adapter *adapter);
+void rtw_mesh_work(_workitem *work);
+void rtw_mesh_atlm_param_req_timer(void *ctx);
+
+#endif /* __RTW_MESH_HWMP_H_ */
+
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_pathtbl.c b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_pathtbl.c
new file mode 100644
index 000000000000..97840a660729
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_pathtbl.c
@@ -0,0 +1,1241 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_MESH_PATHTBL_C_
+
+#ifdef CONFIG_RTW_MESH
+#include <drv_types.h>
+#include <linux/jhash.h>
+
+#ifdef PLATFORM_LINUX
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+static void rtw_mpath_free_rcu(struct rtw_mesh_path *mpath)
+{
+	kfree_rcu(mpath, rcu);
+	rtw_mstat_update(MSTAT_TYPE_PHY, MSTAT_FREE, sizeof(struct rtw_mesh_path));
+}
+#else
+static void rtw_mpath_free_rcu_callback(rtw_rcu_head *head)
+{
+	struct rtw_mesh_path *mpath;
+
+	mpath = container_of(head, struct rtw_mesh_path, rcu);
+	rtw_mfree(mpath, sizeof(struct rtw_mesh_path));
+}
+
+static void rtw_mpath_free_rcu(struct rtw_mesh_path *mpath)
+{
+	call_rcu(&mpath->rcu, rtw_mpath_free_rcu_callback);
+}
+#endif
+#endif /* PLATFORM_LINUX */
+
+static void rtw_mesh_path_free_rcu(struct rtw_mesh_table *tbl, struct rtw_mesh_path *mpath);
+
+static u32 rtw_mesh_table_hash(const void *addr, u32 len, u32 seed)
+{
+	/* Use last four bytes of hw addr as hash index */
+	return jhash_1word(*(u32 *)(addr+2), seed);
+}
+
+static const rtw_rhashtable_params rtw_mesh_rht_params = {
+	.nelem_hint = 2,
+	.automatic_shrinking = true,
+	.key_len = ETH_ALEN,
+	.key_offset = offsetof(struct rtw_mesh_path, dst),
+	.head_offset = offsetof(struct rtw_mesh_path, rhash),
+	.hashfn = rtw_mesh_table_hash,
+};
+
+static inline bool rtw_mpath_expired(struct rtw_mesh_path *mpath)
+{
+	return (mpath->flags & RTW_MESH_PATH_ACTIVE) &&
+	       rtw_time_after(rtw_get_current_time(), mpath->exp_time) &&
+	       !(mpath->flags & RTW_MESH_PATH_FIXED);
+}
+
+static void rtw_mesh_path_rht_free(void *ptr, void *tblptr)
+{
+	struct rtw_mesh_path *mpath = ptr;
+	struct rtw_mesh_table *tbl = tblptr;
+
+	rtw_mesh_path_free_rcu(tbl, mpath);
+}
+
+static struct rtw_mesh_table *rtw_mesh_table_alloc(void)
+{
+	struct rtw_mesh_table *newtbl;
+
+	newtbl = rtw_malloc(sizeof(struct rtw_mesh_table));
+	if (!newtbl)
+		return NULL;
+
+	rtw_hlist_head_init(&newtbl->known_gates);
+	ATOMIC_SET(&newtbl->entries,  0);
+	_rtw_spinlock_init(&newtbl->gates_lock);
+
+	return newtbl;
+}
+
+static void rtw_mesh_table_free(struct rtw_mesh_table *tbl)
+{
+	rtw_rhashtable_free_and_destroy(&tbl->rhead,
+				    rtw_mesh_path_rht_free, tbl);
+	rtw_mfree(tbl, sizeof(struct rtw_mesh_table));
+}
+
+/**
+ *
+ * rtw_mesh_path_assign_nexthop - update mesh path next hop
+ *
+ * @mpath: mesh path to update
+ * @sta: next hop to assign
+ *
+ * Locking: mpath->state_lock must be held when calling this function
+ */
+void rtw_mesh_path_assign_nexthop(struct rtw_mesh_path *mpath, struct sta_info *sta)
+{
+	struct xmit_frame *xframe;
+	_list *list, *head;
+
+	rtw_rcu_assign_pointer(mpath->next_hop, sta);
+
+	_rtw_spinlock_bh(&mpath->frame_queue.lock);
+	head = &mpath->frame_queue.queue;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		xframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+		list = get_next(list);
+		_rtw_memcpy(xframe->attrib.ra, sta->phl_sta->mac_addr, ETH_ALEN);
+	}
+
+	_rtw_spinunlock_bh(&mpath->frame_queue.lock);
+}
+
+static void rtw_prepare_for_gate(struct xmit_frame *xframe, char *dst_addr,
+			     struct rtw_mesh_path *gate_mpath)
+{
+	struct pkt_attrib *attrib = &xframe->attrib;
+	char *next_hop;
+
+	if (attrib->mesh_frame_mode == MESH_UCAST_DATA)
+		attrib->mesh_frame_mode = MESH_UCAST_PX_DATA;
+
+	/* update next hop */
+	rtw_rcu_read_lock();
+	next_hop = rtw_rcu_dereference(gate_mpath->next_hop)->phl_sta->mac_addr;
+	_rtw_memcpy(attrib->ra, next_hop, ETH_ALEN);
+	rtw_rcu_read_unlock();
+	_rtw_memcpy(attrib->mda, dst_addr, ETH_ALEN);
+}
+
+/**
+ *
+ * rtw_mesh_path_move_to_queue - Move or copy frames from one mpath queue to another
+ *
+ * This function is used to transfer or copy frames from an unresolved mpath to
+ * a gate mpath.  The function also adds the Address Extension field and
+ * updates the next hop.
+ *
+ * If a frame already has an Address Extension field, only the next hop and
+ * destination addresses are updated.
+ *
+ * The gate mpath must be an active mpath with a valid mpath->next_hop.
+ *
+ * @mpath: An active mpath the frames will be sent to (i.e. the gate)
+ * @from_mpath: The failed mpath
+ * @copy: When true, copy all the frames to the new mpath queue.  When false,
+ * move them.
+ */
+static void rtw_mesh_path_move_to_queue(struct rtw_mesh_path *gate_mpath,
+				    struct rtw_mesh_path *from_mpath,
+				    bool copy)
+{
+	struct xmit_frame *fskb;
+	_list *list, *head;
+	_list failq;
+	u32 failq_len;
+
+	if (rtw_warn_on(gate_mpath == from_mpath))
+		return;
+	if (rtw_warn_on(!gate_mpath->next_hop))
+		return;
+
+	_rtw_init_listhead(&failq);
+
+	_rtw_spinlock_bh(&from_mpath->frame_queue.lock);
+	rtw_list_splice_init(&from_mpath->frame_queue.queue, &failq);
+	failq_len = from_mpath->frame_queue_len;
+	from_mpath->frame_queue_len = 0;
+	_rtw_spinunlock_bh(&from_mpath->frame_queue.lock);
+
+	head = &failq;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		if (gate_mpath->frame_queue_len >= RTW_MESH_FRAME_QUEUE_LEN) {
+			RTW_MPATH_DBG(FUNC_ADPT_FMT" mpath queue for gate %pM is full!\n"
+				, FUNC_ADPT_ARG(gate_mpath->adapter), gate_mpath->dst);
+			break;
+		}
+
+		fskb = LIST_CONTAINOR(list, struct xmit_frame, list);
+		list = get_next(list);
+
+		rtw_list_delete(&fskb->list);
+		failq_len--;
+		rtw_prepare_for_gate(fskb, gate_mpath->dst, gate_mpath);
+		_rtw_spinlock_bh(&gate_mpath->frame_queue.lock);
+		rtw_list_insert_tail(&fskb->list, get_list_head(&gate_mpath->frame_queue));
+		gate_mpath->frame_queue_len++;
+		_rtw_spinunlock_bh(&gate_mpath->frame_queue.lock);
+
+		#if 0 /* TODO: copy */
+		skb = rtw_skb_copy(fskb);
+		if (rtw_warn_on(!skb))
+			break;
+
+		rtw_prepare_for_gate(skb, gate_mpath->dst, gate_mpath);
+		skb_queue_tail(&gate_mpath->frame_queue, skb);
+
+		if (copy)
+			continue;
+
+		__skb_unlink(fskb, &failq);
+		rtw_skb_free(fskb);
+		#endif
+	}
+
+	RTW_MPATH_DBG(FUNC_ADPT_FMT" mpath queue for gate %pM has %d frames\n"
+		, FUNC_ADPT_ARG(gate_mpath->adapter), gate_mpath->dst, gate_mpath->frame_queue_len);
+
+	if (!copy)
+		return;
+
+	_rtw_spinlock_bh(&from_mpath->frame_queue.lock);
+	rtw_list_splice(&failq, &from_mpath->frame_queue.queue);
+	from_mpath->frame_queue_len += failq_len;
+	_rtw_spinunlock_bh(&from_mpath->frame_queue.lock);
+}
+
+
+static struct rtw_mesh_path *rtw_mpath_lookup(struct rtw_mesh_table *tbl, const u8 *dst)
+{
+	struct rtw_mesh_path *mpath;
+
+	if (!tbl)
+		return NULL;
+
+	mpath = rtw_rhashtable_lookup_fast(&tbl->rhead, dst, rtw_mesh_rht_params);
+
+	if (mpath && rtw_mpath_expired(mpath)) {
+		_rtw_spinlock_bh(&mpath->state_lock);
+		mpath->flags &= ~RTW_MESH_PATH_ACTIVE;
+		_rtw_spinunlock_bh(&mpath->state_lock);
+	}
+	return mpath;
+}
+
+/**
+ * rtw_mesh_path_lookup - look up a path in the mesh path table
+ * @sdata: local subif
+ * @dst: hardware address (ETH_ALEN length) of destination
+ *
+ * Returns: pointer to the mesh path structure, or NULL if not found
+ *
+ * Locking: must be called within a read rcu section.
+ */
+struct rtw_mesh_path *
+rtw_mesh_path_lookup(_adapter *adapter, const u8 *dst)
+{
+	return rtw_mpath_lookup(adapter->mesh_info.mesh_paths, dst);
+}
+
+struct rtw_mesh_path *
+rtw_mpp_path_lookup(_adapter *adapter, const u8 *dst)
+{
+	return rtw_mpath_lookup(adapter->mesh_info.mpp_paths, dst);
+}
+
+static struct rtw_mesh_path *
+__rtw_mesh_path_lookup_by_idx(struct rtw_mesh_table *tbl, int idx)
+{
+	int i = 0, ret;
+	struct rtw_mesh_path *mpath = NULL;
+	rtw_rhashtable_iter iter;
+
+	if (!tbl)
+		return NULL;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return NULL;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto err;
+
+	while ((mpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(mpath) && PTR_ERR(mpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(mpath))
+			break;
+		if (i++ == idx)
+			break;
+	}
+err:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+
+	if (IS_ERR(mpath) || !mpath)
+		return NULL;
+
+	if (rtw_mpath_expired(mpath)) {
+		_rtw_spinlock_bh(&mpath->state_lock);
+		mpath->flags &= ~RTW_MESH_PATH_ACTIVE;
+		_rtw_spinunlock_bh(&mpath->state_lock);
+	}
+	return mpath;
+}
+
+/**
+ * rtw_mesh_path_lookup_by_idx - look up a path in the mesh path table by its index
+ * @idx: index
+ * @sdata: local subif, or NULL for all entries
+ *
+ * Returns: pointer to the mesh path structure, or NULL if not found.
+ *
+ * Locking: must be called within a read rcu section.
+ */
+struct rtw_mesh_path *
+rtw_mesh_path_lookup_by_idx(_adapter *adapter, int idx)
+{
+	return __rtw_mesh_path_lookup_by_idx(adapter->mesh_info.mesh_paths, idx);
+}
+
+void dump_mpath(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_path *mpath;
+	int idx = 0;
+	char dst[ETH_ALEN];
+	char next_hop[ETH_ALEN];
+	u32 sn, metric, qlen;
+	u32 exp_ms = 0, dto_ms;
+	u8 drty;
+	enum rtw_mesh_path_flags flags;
+
+	RTW_PRINT_SEL(sel, "%-17s %-17s %-10s %-10s %-4s %-6s %-6s %-4s flags\n"
+		, "dst", "next_hop", "sn", "metric", "qlen", "exp_ms", "dto_ms", "drty"
+	);
+
+	do {
+		rtw_rcu_read_lock();
+
+		mpath = rtw_mesh_path_lookup_by_idx(adapter, idx);
+		if (mpath) {
+			_rtw_memcpy(dst, mpath->dst, ETH_ALEN);
+			_rtw_memcpy(next_hop, mpath->next_hop->phl_sta->mac_addr, ETH_ALEN);
+			sn = mpath->sn;
+			metric = mpath->metric;
+			qlen = mpath->frame_queue_len;
+			if (rtw_time_after(mpath->exp_time, rtw_get_current_time()))
+				exp_ms = rtw_get_remaining_time_ms(mpath->exp_time);
+			dto_ms = rtw_systime_to_ms(mpath->discovery_timeout);
+			drty = mpath->discovery_retries;
+			flags = mpath->flags;
+		}
+
+		rtw_rcu_read_unlock();
+
+		if (mpath) {
+			RTW_PRINT_SEL(sel, MAC_FMT" "MAC_FMT" %10u %10u %4u %6u %6u %4u%s%s%s%s%s%s%s%s%s%s\n"
+				, MAC_ARG(dst), MAC_ARG(next_hop), sn, metric, qlen
+				, exp_ms < 999999 ? exp_ms : 999999
+				, dto_ms < 999999 ? dto_ms : 999999
+				, drty
+				, (flags & RTW_MESH_PATH_ACTIVE) ? " ACT" : ""
+				, (flags & RTW_MESH_PATH_RESOLVING) ? " RSVING" : ""
+				, (flags & RTW_MESH_PATH_SN_VALID) ? " SN_VALID" : ""
+				, (flags & RTW_MESH_PATH_FIXED) ?  " FIXED" : ""
+				, (flags & RTW_MESH_PATH_RESOLVED) ? " RSVED" : ""
+				, (flags & RTW_MESH_PATH_REQ_QUEUED) ? " REQ_IN_Q" : ""
+				, (flags & RTW_MESH_PATH_DELETED) ? " DELETED" : ""
+				, (flags & RTW_MESH_PATH_ROOT_ADD_CHK) ? " R_ADD_CHK" : ""
+				, (flags & RTW_MESH_PATH_PEER_AKA) ? " PEER_AKA" : ""
+				, (flags & RTW_MESH_PATH_BCAST_PREQ) ? " BC_PREQ" : ""
+			);
+		}
+
+		idx++;
+	} while (mpath);
+}
+
+/**
+ * rtw_mpp_path_lookup_by_idx - look up a path in the proxy path table by its index
+ * @idx: index
+ * @sdata: local subif, or NULL for all entries
+ *
+ * Returns: pointer to the proxy path structure, or NULL if not found.
+ *
+ * Locking: must be called within a read rcu section.
+ */
+struct rtw_mesh_path *
+rtw_mpp_path_lookup_by_idx(_adapter *adapter, int idx)
+{
+	return __rtw_mesh_path_lookup_by_idx(adapter->mesh_info.mpp_paths, idx);
+}
+
+/**
+ * rtw_mesh_path_add_gate - add the given mpath to a mesh gate to our path table
+ * @mpath: gate path to add to table
+ */
+int rtw_mesh_path_add_gate(struct rtw_mesh_path *mpath)
+{
+	struct rtw_mesh_cfg *mcfg;
+	struct rtw_mesh_info *minfo;
+	struct rtw_mesh_table *tbl;
+	int err, ori_num_gates;
+
+	rtw_rcu_read_lock();
+	tbl = mpath->adapter->mesh_info.mesh_paths;
+	if (!tbl) {
+		err = -ENOENT;
+		goto err_rcu;
+	}
+
+	_rtw_spinlock_bh(&mpath->state_lock);
+	mcfg = &mpath->adapter->mesh_cfg;
+	mpath->gate_timeout = rtw_get_current_time() +
+			      rtw_ms_to_systime(mcfg->path_gate_timeout_factor *
+					        mpath->gate_ann_int);
+	if (mpath->is_gate) {
+		err = -EEXIST;
+		_rtw_spinunlock_bh(&mpath->state_lock);
+		goto err_rcu;
+	}
+
+	minfo = &mpath->adapter->mesh_info;
+	mpath->is_gate = true;
+	_rtw_spinlock(&tbl->gates_lock);
+	ori_num_gates = minfo->num_gates;
+	minfo->num_gates++;
+	rtw_hlist_add_head_rcu(&mpath->gate_list, &tbl->known_gates);
+
+	if (ori_num_gates == 0
+		|| rtw_macaddr_is_larger(mpath->dst, minfo->max_addr_gate->dst)
+	) {
+		minfo->max_addr_gate = mpath;
+		minfo->max_addr_gate_is_larger_than_self =
+			rtw_macaddr_is_larger(mpath->dst, adapter_mac_addr(mpath->adapter));
+	}
+
+	_rtw_spinunlock(&tbl->gates_lock);
+
+	_rtw_spinunlock_bh(&mpath->state_lock);
+
+	if (ori_num_gates == 0) {
+		rtw_update_beacon(mpath->adapter, WLAN_EID_MESH_CONFIG, NULL, _TRUE, 0);
+		#if CONFIG_RTW_MESH_CTO_MGATE_CARRIER
+		if (!rtw_mesh_cto_mgate_required(mpath->adapter))
+			rtw_netif_carrier_on(mpath->adapter->pnetdev);
+		#endif
+	}
+
+	RTW_MPATH_DBG(
+		  FUNC_ADPT_FMT" Mesh path: Recorded new gate: %pM. %d known gates\n",
+		  FUNC_ADPT_ARG(mpath->adapter),
+		  mpath->dst, mpath->adapter->mesh_info.num_gates);
+	err = 0;
+err_rcu:
+	rtw_rcu_read_unlock();
+	return err;
+}
+
+/**
+ * rtw_mesh_gate_del - remove a mesh gate from the list of known gates
+ * @tbl: table which holds our list of known gates
+ * @mpath: gate mpath
+ */
+void rtw_mesh_gate_del(struct rtw_mesh_table *tbl, struct rtw_mesh_path *mpath)
+{
+	struct rtw_mesh_cfg *mcfg;
+	struct rtw_mesh_info *minfo;
+	int ori_num_gates;
+
+	rtw_lockdep_assert_held(&mpath->state_lock);
+	if (!mpath->is_gate)
+		return;
+
+	mcfg = &mpath->adapter->mesh_cfg;
+	minfo = &mpath->adapter->mesh_info;
+
+	mpath->is_gate = false;
+	_rtw_spinlock_bh(&tbl->gates_lock);
+	rtw_hlist_del_rcu(&mpath->gate_list);
+	ori_num_gates = minfo->num_gates;
+	minfo->num_gates--;
+
+	if (ori_num_gates == 1) {
+		minfo->max_addr_gate = NULL;
+		minfo->max_addr_gate_is_larger_than_self = 0;
+	} else if (minfo->max_addr_gate == mpath) {
+		struct rtw_mesh_path *gate, *max_addr_gate = NULL;
+		rtw_hlist_node *node;
+
+		rtw_hlist_for_each_entry_rcu(gate, node, &tbl->known_gates, gate_list) {
+			if (!max_addr_gate || rtw_macaddr_is_larger(gate->dst, max_addr_gate->dst))
+				max_addr_gate = gate;
+		}
+		minfo->max_addr_gate = max_addr_gate;
+		minfo->max_addr_gate_is_larger_than_self =
+			rtw_macaddr_is_larger(max_addr_gate->dst, adapter_mac_addr(mpath->adapter));
+	}
+
+	_rtw_spinunlock_bh(&tbl->gates_lock);
+
+	if (ori_num_gates == 1) {
+		rtw_update_beacon(mpath->adapter, WLAN_EID_MESH_CONFIG, NULL, _TRUE, 0);
+		#if CONFIG_RTW_MESH_CTO_MGATE_CARRIER
+		if (rtw_mesh_cto_mgate_required(mpath->adapter))
+			rtw_netif_carrier_off(mpath->adapter->pnetdev);
+		#endif
+	}
+
+	RTW_MPATH_DBG(
+		  FUNC_ADPT_FMT" Mesh path: Deleted gate: %pM. %d known gates\n",
+		  FUNC_ADPT_ARG(mpath->adapter),
+		  mpath->dst, mpath->adapter->mesh_info.num_gates);
+}
+
+/**
+ * rtw_mesh_gate_search - search a mesh gate from the list of known gates
+ * @tbl: table which holds our list of known gates
+ * @addr: address of gate
+ */
+bool rtw_mesh_gate_search(struct rtw_mesh_table *tbl, const u8 *addr)
+{
+	struct rtw_mesh_path *gate;
+	rtw_hlist_node *node;
+	bool exist = 0;
+
+	rtw_rcu_read_lock();
+	rtw_hlist_for_each_entry_rcu(gate, node, &tbl->known_gates, gate_list) {
+		if (_rtw_memcmp(gate->dst, addr, ETH_ALEN) == _TRUE) {
+			exist = 1;
+			break;
+		}
+	}
+
+	rtw_rcu_read_unlock();
+
+	return exist;
+}
+
+/**
+ * rtw_mesh_gate_num - number of gates known to this interface
+ * @sdata: subif data
+ */
+int rtw_mesh_gate_num(_adapter *adapter)
+{
+	return adapter->mesh_info.num_gates;
+}
+
+bool rtw_mesh_is_primary_gate(_adapter *adapter)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+
+	return mcfg->dot11MeshGateAnnouncementProtocol
+		&& !minfo->max_addr_gate_is_larger_than_self;
+}
+
+void dump_known_gates(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mesh_table *tbl;
+	struct rtw_mesh_path *gate;
+	rtw_hlist_node *node;
+
+	if (!rtw_mesh_gate_num(adapter))
+		goto exit;
+
+	rtw_rcu_read_lock();
+
+	tbl = minfo->mesh_paths;
+	if (!tbl)
+		goto unlock;
+
+	RTW_PRINT_SEL(sel, "num:%d\n", rtw_mesh_gate_num(adapter));
+
+	rtw_hlist_for_each_entry_rcu(gate, node, &tbl->known_gates, gate_list) {
+		RTW_PRINT_SEL(sel, "%c"MAC_FMT"\n"
+			, gate == minfo->max_addr_gate ? '*' : ' '
+			, MAC_ARG(gate->dst));
+	}
+
+unlock:
+	rtw_rcu_read_unlock();
+exit:
+	return;
+}
+
+static
+struct rtw_mesh_path *rtw_mesh_path_new(_adapter *adapter,
+				const u8 *dst)
+{
+	struct rtw_mesh_path *new_mpath;
+
+	new_mpath = rtw_zmalloc(sizeof(struct rtw_mesh_path));
+	if (!new_mpath)
+		return NULL;
+
+	_rtw_memcpy(new_mpath->dst, dst, ETH_ALEN);
+	_rtw_memset(new_mpath->rann_snd_addr, 0xFF, ETH_ALEN);
+	new_mpath->is_root = false;
+	new_mpath->adapter = adapter;
+	new_mpath->flags = 0;
+	new_mpath->gate_asked = false;
+	_rtw_init_queue(&new_mpath->frame_queue);
+	new_mpath->frame_queue_len = 0;
+	new_mpath->exp_time = rtw_get_current_time();
+	_rtw_spinlock_init(&new_mpath->state_lock);
+	rtw_init_timer(&new_mpath->timer, rtw_mesh_path_timer, new_mpath);
+
+	return new_mpath;
+}
+
+/**
+ * rtw_mesh_path_add - allocate and add a new path to the mesh path table
+ * @dst: destination address of the path (ETH_ALEN length)
+ * @sdata: local subif
+ *
+ * Returns: 0 on success
+ *
+ * State: the initial state of the new path is set to 0
+ */
+struct rtw_mesh_path *rtw_mesh_path_add(_adapter *adapter,
+				const u8 *dst)
+{
+	struct rtw_mesh_table *tbl = adapter->mesh_info.mesh_paths;
+	struct rtw_mesh_path *mpath, *new_mpath;
+	int ret;
+
+	if (!tbl)
+		return ERR_PTR(-ENOTSUPP);
+
+	if (_rtw_memcmp(dst, adapter_mac_addr(adapter), ETH_ALEN) == _TRUE)
+		/* never add ourselves as neighbours */
+		return ERR_PTR(-ENOTSUPP);
+
+	if (is_multicast_mac_addr(dst))
+		return ERR_PTR(-ENOTSUPP);
+
+	if (ATOMIC_INC_UNLESS(&adapter->mesh_info.mpaths, RTW_MESH_MAX_MPATHS) == 0)
+		return ERR_PTR(-ENOSPC);
+
+	new_mpath = rtw_mesh_path_new(adapter, dst);
+	if (!new_mpath)
+		return ERR_PTR(-ENOMEM);
+
+	do {
+		ret = rtw_rhashtable_lookup_insert_fast(&tbl->rhead,
+						    &new_mpath->rhash,
+						    rtw_mesh_rht_params);
+
+		if (ret == -EEXIST)
+			mpath = rtw_rhashtable_lookup_fast(&tbl->rhead,
+						       dst,
+						       rtw_mesh_rht_params);
+
+	} while (unlikely(ret == -EEXIST && !mpath));
+
+	if (ret && ret != -EEXIST)
+		return ERR_PTR(ret);
+
+	/* At this point either new_mpath was added, or we found a
+	 * matching entry already in the table; in the latter case
+	 * free the unnecessary new entry.
+	 */
+	if (ret == -EEXIST) {
+		rtw_mfree(new_mpath, sizeof(struct rtw_mesh_path));
+		new_mpath = mpath;
+	}
+	adapter->mesh_info.mesh_paths_generation++;
+	return new_mpath;
+}
+
+int rtw_mpp_path_add(_adapter *adapter,
+		 const u8 *dst, const u8 *mpp)
+{
+	struct rtw_mesh_table *tbl = adapter->mesh_info.mpp_paths;
+	struct rtw_mesh_path *new_mpath;
+	int ret;
+
+	if (!tbl)
+		return -ENOTSUPP;
+
+	if (_rtw_memcmp(dst, adapter_mac_addr(adapter), ETH_ALEN) == _TRUE)
+		/* never add ourselves as neighbours */
+		return -ENOTSUPP;
+
+	if (is_multicast_mac_addr(dst))
+		return -ENOTSUPP;
+
+	new_mpath = rtw_mesh_path_new(adapter, dst);
+
+	if (!new_mpath)
+		return -ENOMEM;
+
+	_rtw_memcpy(new_mpath->mpp, mpp, ETH_ALEN);
+	ret = rtw_rhashtable_lookup_insert_fast(&tbl->rhead,
+					    &new_mpath->rhash,
+					    rtw_mesh_rht_params);
+
+	adapter->mesh_info.mpp_paths_generation++;
+	return ret;
+}
+
+void dump_mpp(void *sel, _adapter *adapter)
+{
+	struct rtw_mesh_path *mpath;
+	int idx = 0;
+	char dst[ETH_ALEN];
+	char mpp[ETH_ALEN];
+
+	RTW_PRINT_SEL(sel, "%-17s %-17s\n", "dst", "mpp");
+
+	do {
+		rtw_rcu_read_lock();
+
+		mpath = rtw_mpp_path_lookup_by_idx(adapter, idx);
+		if (mpath) {
+			_rtw_memcpy(dst, mpath->dst, ETH_ALEN);
+			_rtw_memcpy(mpp, mpath->mpp, ETH_ALEN);
+		}
+
+		rtw_rcu_read_unlock();
+
+		if (mpath) {
+			RTW_PRINT_SEL(sel, MAC_FMT" "MAC_FMT"\n"
+				, MAC_ARG(dst), MAC_ARG(mpp));
+		}
+
+		idx++;
+	} while (mpath);
+}
+
+/**
+ * rtw_mesh_plink_broken - deactivates paths and sends perr when a link breaks
+ *
+ * @sta: broken peer link
+ *
+ * This function must be called from the rate control algorithm if enough
+ * delivery errors suggest that a peer link is no longer usable.
+ */
+void rtw_mesh_plink_broken(struct sta_info *sta)
+{
+	_adapter *adapter = sta->padapter;
+	struct rtw_mesh_table *tbl = adapter->mesh_info.mesh_paths;
+	static const u8 bcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	struct rtw_mesh_path *mpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((mpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(mpath) && PTR_ERR(mpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(mpath))
+			break;
+		if (rtw_rcu_access_pointer(mpath->next_hop) == sta &&
+		    mpath->flags & RTW_MESH_PATH_ACTIVE &&
+		    !(mpath->flags & RTW_MESH_PATH_FIXED)) {
+			_rtw_spinlock_bh(&mpath->state_lock);
+			mpath->flags &= ~RTW_MESH_PATH_ACTIVE;
+			++mpath->sn;
+			_rtw_spinunlock_bh(&mpath->state_lock);
+			rtw_mesh_path_error_tx(adapter,
+				adapter->mesh_cfg.element_ttl,
+				mpath->dst, mpath->sn,
+				WLAN_REASON_MESH_PATH_DEST_UNREACHABLE, bcast);
+		}
+	}
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+static void rtw_mesh_path_free_rcu(struct rtw_mesh_table *tbl,
+			       struct rtw_mesh_path *mpath)
+{
+	_adapter *adapter = mpath->adapter;
+
+	_rtw_spinlock_bh(&mpath->state_lock);
+	mpath->flags |= RTW_MESH_PATH_RESOLVING | RTW_MESH_PATH_DELETED;
+	rtw_mesh_gate_del(tbl, mpath);
+	_rtw_spinunlock_bh(&mpath->state_lock);
+	_cancel_timer_ex(&mpath->timer);
+	ATOMIC_DEC(&adapter->mesh_info.mpaths);
+	ATOMIC_DEC(&tbl->entries);
+	_rtw_spinlock_free(&mpath->state_lock);
+
+	rtw_mesh_path_flush_pending(mpath);
+
+	rtw_mpath_free_rcu(mpath);
+}
+
+static void __rtw_mesh_path_del(struct rtw_mesh_table *tbl, struct rtw_mesh_path *mpath)
+{
+	rtw_rhashtable_remove_fast(&tbl->rhead, &mpath->rhash, rtw_mesh_rht_params);
+	rtw_mesh_path_free_rcu(tbl, mpath);
+}
+
+/**
+ * rtw_mesh_path_flush_by_nexthop - Deletes mesh paths if their next hop matches
+ *
+ * @sta: mesh peer to match
+ *
+ * RCU notes: this function is called when a mesh plink transitions from
+ * PLINK_ESTAB to any other state, since PLINK_ESTAB state is the only one that
+ * allows path creation. This will happen before the sta can be freed (because
+ * sta_info_destroy() calls this) so any reader in a rcu read block will be
+ * protected against the plink disappearing.
+ */
+void rtw_mesh_path_flush_by_nexthop(struct sta_info *sta)
+{
+	_adapter *adapter = sta->padapter;
+	struct rtw_mesh_table *tbl = adapter->mesh_info.mesh_paths;
+	struct rtw_mesh_path *mpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((mpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(mpath) && PTR_ERR(mpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(mpath))
+			break;
+
+		if (rtw_rcu_access_pointer(mpath->next_hop) == sta)
+			__rtw_mesh_path_del(tbl, mpath);
+	}
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+static void rtw_mpp_flush_by_proxy(_adapter *adapter,
+			       const u8 *proxy)
+{
+	struct rtw_mesh_table *tbl = adapter->mesh_info.mpp_paths;
+	struct rtw_mesh_path *mpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((mpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(mpath) && PTR_ERR(mpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(mpath))
+			break;
+
+		if (_rtw_memcmp(mpath->mpp, proxy, ETH_ALEN) == _TRUE)
+			__rtw_mesh_path_del(tbl, mpath);
+	}
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+static void rtw_table_flush_by_iface(struct rtw_mesh_table *tbl)
+{
+	struct rtw_mesh_path *mpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+	
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((mpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(mpath) && PTR_ERR(mpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(mpath))
+			break;
+		__rtw_mesh_path_del(tbl, mpath);
+	}
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+/**
+ * rtw_mesh_path_flush_by_iface - Deletes all mesh paths associated with a given iface
+ *
+ * This function deletes both mesh paths as well as mesh portal paths.
+ *
+ * @sdata: interface data to match
+ *
+ */
+void rtw_mesh_path_flush_by_iface(_adapter *adapter)
+{
+	rtw_table_flush_by_iface(adapter->mesh_info.mesh_paths);
+	rtw_table_flush_by_iface(adapter->mesh_info.mpp_paths);
+}
+
+/**
+ * rtw_table_path_del - delete a path from the mesh or mpp table
+ *
+ * @tbl: mesh or mpp path table
+ * @sdata: local subif
+ * @addr: dst address (ETH_ALEN length)
+ *
+ * Returns: 0 if successful
+ */
+static int rtw_table_path_del(struct rtw_mesh_table *tbl,
+			  const u8 *addr)
+{
+	struct rtw_mesh_path *mpath;
+
+	if (!tbl)
+		return -ENXIO;
+
+	rtw_rcu_read_lock();
+	mpath = rtw_rhashtable_lookup_fast(&tbl->rhead, addr, rtw_mesh_rht_params);
+	if (!mpath) {
+		rtw_rcu_read_unlock();
+		return -ENXIO;
+	}
+
+	__rtw_mesh_path_del(tbl, mpath);
+	rtw_rcu_read_unlock();
+	return 0;
+}
+
+
+/**
+ * rtw_mesh_path_del - delete a mesh path from the table
+ *
+ * @addr: dst address (ETH_ALEN length)
+ * @sdata: local subif
+ *
+ * Returns: 0 if successful
+ */
+int rtw_mesh_path_del(_adapter *adapter, const u8 *addr)
+{
+	int err;
+
+	/* flush relevant mpp entries first */
+	rtw_mpp_flush_by_proxy(adapter, addr);
+
+	err = rtw_table_path_del(adapter->mesh_info.mesh_paths, addr);
+	adapter->mesh_info.mesh_paths_generation++;
+	return err;
+}
+
+/**
+ * rtw_mesh_path_tx_pending - sends pending frames in a mesh path queue
+ *
+ * @mpath: mesh path to activate
+ *
+ * Locking: the state_lock of the mpath structure must NOT be held when calling
+ * this function.
+ */
+void rtw_mesh_path_tx_pending(struct rtw_mesh_path *mpath)
+{
+	if (mpath->flags & RTW_MESH_PATH_ACTIVE) {
+		struct rtw_mesh_info *minfo = &mpath->adapter->mesh_info;
+		_list q;
+		u32 q_len = 0;
+
+		_rtw_init_listhead(&q);
+
+		/* move to local queue */
+		_rtw_spinlock_bh(&mpath->frame_queue.lock);
+		if (mpath->frame_queue_len) {
+			rtw_list_splice_init(&mpath->frame_queue.queue, &q);
+			q_len = mpath->frame_queue_len;
+			mpath->frame_queue_len = 0;
+		}
+		_rtw_spinunlock_bh(&mpath->frame_queue.lock);
+
+		if (q_len) {
+			/* move to mpath_tx_queue */
+			_rtw_spinlock_bh(&minfo->mpath_tx_queue.lock);
+			rtw_list_splice_tail(&q, &minfo->mpath_tx_queue.queue);
+			minfo->mpath_tx_queue_len += q_len;
+			_rtw_spinunlock_bh(&minfo->mpath_tx_queue.lock);
+
+			/* schedule mpath_tx_tasklet */
+			tasklet_hi_schedule(&minfo->mpath_tx_tasklet);
+		}
+	}
+}
+
+/**
+ * rtw_mesh_path_send_to_gates - sends pending frames to all known mesh gates
+ *
+ * @mpath: mesh path whose queue will be emptied
+ *
+ * If there is only one gate, the frames are transferred from the failed mpath
+ * queue to that gate's queue.  If there are more than one gates, the frames
+ * are copied from each gate to the next.  After frames are copied, the
+ * mpath queues are emptied onto the transmission queue.
+ */
+int rtw_mesh_path_send_to_gates(struct rtw_mesh_path *mpath)
+{
+	_adapter *adapter = mpath->adapter;
+	struct rtw_mesh_table *tbl;
+	struct rtw_mesh_path *from_mpath = mpath;
+	struct rtw_mesh_path *gate;
+	bool copy = false;
+	rtw_hlist_node *node;
+
+	tbl = adapter->mesh_info.mesh_paths;
+	if (!tbl)
+		return 0;
+
+	rtw_rcu_read_lock();
+	rtw_hlist_for_each_entry_rcu(gate, node, &tbl->known_gates, gate_list) {
+		if (gate->flags & RTW_MESH_PATH_ACTIVE) {
+			RTW_MPATH_DBG(FUNC_ADPT_FMT" Forwarding to %pM\n",
+				FUNC_ADPT_ARG(adapter), gate->dst);
+			rtw_mesh_path_move_to_queue(gate, from_mpath, copy);
+			from_mpath = gate;
+			copy = true;
+		} else {
+			RTW_MPATH_DBG(
+				  FUNC_ADPT_FMT" Not forwarding to %pM (flags %#x)\n",
+				  FUNC_ADPT_ARG(adapter), gate->dst, gate->flags);
+		}
+	}
+
+	rtw_hlist_for_each_entry_rcu(gate, node, &tbl->known_gates, gate_list) {
+		RTW_MPATH_DBG(FUNC_ADPT_FMT" Sending to %pM\n",
+			FUNC_ADPT_ARG(adapter), gate->dst);
+		rtw_mesh_path_tx_pending(gate);
+	}
+	rtw_rcu_read_unlock();
+
+	return (from_mpath == mpath) ? -EHOSTUNREACH : 0;
+}
+
+/**
+ * rtw_mesh_path_discard_frame - discard a frame whose path could not be resolved
+ *
+ * @skb: frame to discard
+ * @sdata: network subif the frame was to be sent through
+ *
+ * Locking: the function must me called within a rcu_read_lock region
+ */
+void rtw_mesh_path_discard_frame(_adapter *adapter,
+			     struct xmit_frame *xframe)
+{
+	rtw_free_xmitframe(&adapter->xmitpriv, xframe);
+	adapter->mesh_info.mshstats.dropped_frames_no_route++;
+}
+
+/**
+ * rtw_mesh_path_flush_pending - free the pending queue of a mesh path
+ *
+ * @mpath: mesh path whose queue has to be freed
+ *
+ * Locking: the function must me called within a rcu_read_lock region
+ */
+void rtw_mesh_path_flush_pending(struct rtw_mesh_path *mpath)
+{
+	struct xmit_frame *xframe;
+	_list *list, *head;
+	_list tmp;
+
+	_rtw_init_listhead(&tmp);
+
+	_rtw_spinlock_bh(&mpath->frame_queue.lock);
+	rtw_list_splice_init(&mpath->frame_queue.queue, &tmp);
+	mpath->frame_queue_len = 0;
+	_rtw_spinunlock_bh(&mpath->frame_queue.lock);
+
+	head = &tmp;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		xframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+		list = get_next(list);
+		rtw_list_delete(&xframe->list);
+		rtw_mesh_path_discard_frame(mpath->adapter, xframe);
+	}
+}
+
+/**
+ * rtw_mesh_path_fix_nexthop - force a specific next hop for a mesh path
+ *
+ * @mpath: the mesh path to modify
+ * @next_hop: the next hop to force
+ *
+ * Locking: this function must be called holding mpath->state_lock
+ */
+void rtw_mesh_path_fix_nexthop(struct rtw_mesh_path *mpath, struct sta_info *next_hop)
+{
+	_rtw_spinlock_bh(&mpath->state_lock);
+	rtw_mesh_path_assign_nexthop(mpath, next_hop);
+	mpath->sn = 0xffff;
+	mpath->metric = 0;
+	mpath->hop_count = 0;
+	mpath->exp_time = 0;
+	mpath->flags = RTW_MESH_PATH_FIXED | RTW_MESH_PATH_SN_VALID;
+	rtw_mesh_path_activate(mpath);
+	_rtw_spinunlock_bh(&mpath->state_lock);
+	rtw_ewma_err_rate_init(&next_hop->metrics.err_rate);
+	/* init it at a low value - 0 start is tricky */
+	rtw_ewma_err_rate_add(&next_hop->metrics.err_rate, 1);
+	rtw_mesh_path_tx_pending(mpath);
+}
+
+int rtw_mesh_pathtbl_init(_adapter *adapter)
+{
+	struct rtw_mesh_table *tbl_path, *tbl_mpp;
+	int ret;
+
+	tbl_path = rtw_mesh_table_alloc();
+	if (!tbl_path)
+		return -ENOMEM;
+
+	tbl_mpp = rtw_mesh_table_alloc();
+	if (!tbl_mpp) {
+		ret = -ENOMEM;
+		goto free_path;
+	}
+
+	rtw_rhashtable_init(&tbl_path->rhead, &rtw_mesh_rht_params);
+	rtw_rhashtable_init(&tbl_mpp->rhead, &rtw_mesh_rht_params);
+
+	adapter->mesh_info.mesh_paths = tbl_path;
+	adapter->mesh_info.mpp_paths = tbl_mpp;
+
+	return 0;
+
+free_path:
+	rtw_mesh_table_free(tbl_path);
+	return ret;
+}
+
+static
+void rtw_mesh_path_tbl_expire(_adapter *adapter,
+			  struct rtw_mesh_table *tbl)
+{
+	struct rtw_mesh_path *mpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((mpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(mpath) && PTR_ERR(mpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(mpath))
+			break;
+		if ((!(mpath->flags & RTW_MESH_PATH_RESOLVING)) &&
+		    (!(mpath->flags & RTW_MESH_PATH_FIXED)) &&
+		     rtw_time_after(rtw_get_current_time(), mpath->exp_time + RTW_MESH_PATH_EXPIRE))
+			__rtw_mesh_path_del(tbl, mpath);
+
+		if (mpath->is_gate &&  /* need not to deal with non-gate case */
+		    rtw_time_after(rtw_get_current_time(), mpath->gate_timeout)) {
+			RTW_MPATH_DBG(FUNC_ADPT_FMT"mpath [%pM] expired systime is %lu systime is %lu\n",
+				      FUNC_ADPT_ARG(adapter), mpath->dst,
+				      mpath->gate_timeout, rtw_get_current_time());
+			_rtw_spinlock_bh(&mpath->state_lock);
+			if (mpath->gate_asked) { /* asked gate before */
+				rtw_mesh_gate_del(tbl, mpath);
+				_rtw_spinunlock_bh(&mpath->state_lock);
+			} else {
+				mpath->gate_asked = true;
+				mpath->gate_timeout = rtw_get_current_time() + rtw_ms_to_systime(mpath->gate_ann_int);
+				_rtw_spinunlock_bh(&mpath->state_lock);
+				rtw_mesh_queue_preq(mpath, RTW_PREQ_Q_F_START | RTW_PREQ_Q_F_REFRESH);
+				RTW_MPATH_DBG(FUNC_ADPT_FMT"mpath [%pM] ask mesh gate existence (is_root=%d)\n",
+				      FUNC_ADPT_ARG(adapter), mpath->dst, mpath->is_root);
+			}
+		}
+	}
+
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+void rtw_mesh_path_expire(_adapter *adapter)
+{
+	rtw_mesh_path_tbl_expire(adapter, adapter->mesh_info.mesh_paths);
+	rtw_mesh_path_tbl_expire(adapter, adapter->mesh_info.mpp_paths);
+}
+
+void rtw_mesh_pathtbl_unregister(_adapter *adapter)
+{
+	if (adapter->mesh_info.mesh_paths) {
+		rtw_mesh_table_free(adapter->mesh_info.mesh_paths);
+		adapter->mesh_info.mesh_paths = NULL;
+	}
+
+	if (adapter->mesh_info.mpp_paths) {
+		rtw_mesh_table_free(adapter->mesh_info.mpp_paths);
+		adapter->mesh_info.mpp_paths = NULL;
+	}
+}
+#endif /* CONFIG_RTW_MESH */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_pathtbl.h b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_pathtbl.h
new file mode 100644
index 000000000000..7ff4d8ba0f9c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/mesh/rtw_mesh_pathtbl.h
@@ -0,0 +1,211 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_MESH_PATHTBL_H_
+#define __RTW_MESH_PATHTBL_H_
+
+#ifndef DBG_RTW_MPATH
+#define DBG_RTW_MPATH 1
+#endif
+#if DBG_RTW_MPATH
+#define RTW_MPATH_DBG(fmt, arg...) RTW_PRINT(fmt, ##arg)
+#else
+#define RTW_MPATH_DBG(fmt, arg...) do {} while (0)
+#endif
+
+/**
+ * enum rtw_mesh_path_flags - mesh path flags
+ *
+ * @RTW_MESH_PATH_ACTIVE: the mesh path can be used for forwarding
+ * @RTW_MESH_PATH_RESOLVING: the discovery process is running for this mesh path
+ * @RTW_MESH_PATH_SN_VALID: the mesh path contains a valid destination sequence
+ *	number
+ * @RTW_MESH_PATH_FIXED: the mesh path has been manually set and should not be
+ *	modified
+ * @RTW_MESH_PATH_RESOLVED: the mesh path can has been resolved
+ * @RTW_MESH_PATH_REQ_QUEUED: there is an unsent path request for this destination
+ *	already queued up, waiting for the discovery process to start.
+ * @RTW_MESH_PATH_DELETED: the mesh path has been deleted and should no longer
+ *	be used
+ * @RTW_MESH_PATH_ROOT_ADD_CHK: root additional check in root mode.
+ *	With this flag, It will try the last used rann_snd_addr
+ * @RTW_MESH_PATH_PEER_AKA: only used toward a peer, only used in active keep
+ *	alive mechanism. PREQ's da = path dst
+ * @RTW_MESH_PATH_BCAST_PREQ: for re-checking next hop resolve toward root.
+ *	Use it to force path_discover sending broadcast PREQ for root.
+ * 
+ * RTW_MESH_PATH_RESOLVED is used by the mesh path timer to
+ * decide when to stop or cancel the mesh path discovery.
+ */
+enum rtw_mesh_path_flags {
+	RTW_MESH_PATH_ACTIVE =		BIT(0),
+	RTW_MESH_PATH_RESOLVING =	BIT(1),
+	RTW_MESH_PATH_SN_VALID =	BIT(2),
+	RTW_MESH_PATH_FIXED	=	BIT(3),
+	RTW_MESH_PATH_RESOLVED =	BIT(4),
+	RTW_MESH_PATH_REQ_QUEUED =	BIT(5),
+	RTW_MESH_PATH_DELETED =		BIT(6),
+	RTW_MESH_PATH_ROOT_ADD_CHK =	BIT(7),
+	RTW_MESH_PATH_PEER_AKA =	BIT(8),
+	RTW_MESH_PATH_BCAST_PREQ =	BIT(9),	
+};
+
+/**
+ * struct rtw_mesh_path - mesh path structure
+ *
+ * @dst: mesh path destination mac address
+ * @mpp: mesh proxy mac address
+ * @rhash: rhashtable list pointer
+ * @gate_list: list pointer for known gates list
+ * @sdata: mesh subif
+ * @next_hop: mesh neighbor to which frames for this destination will be
+ *	forwarded
+ * @timer: mesh path discovery timer
+ * @frame_queue: pending queue for frames sent to this destination while the
+ *	path is unresolved
+ * @rcu: rcu head for freeing mesh path
+ * @sn: target sequence number
+ * @metric: current metric to this destination
+ * @hop_count: hops to destination
+ * @exp_time: in jiffies, when the path will expire or when it expired
+ * @discovery_timeout: timeout (lapse in jiffies) used for the last discovery
+ *	retry
+ * @discovery_retries: number of discovery retries
+ * @flags: mesh path flags, as specified on &enum rtw_mesh_path_flags
+ * @state_lock: mesh path state lock used to protect changes to the
+ * mpath itself.  No need to take this lock when adding or removing
+ * an mpath to a hash bucket on a path table.
+ * @rann_snd_addr: the RANN sender address
+ * @rann_metric: the aggregated path metric towards the root node
+ * @last_preq_to_root: Timestamp of last PREQ sent to root
+ * @is_root: the destination station of this path is a root node
+ * @is_gate: the destination station of this path is a mesh gate
+ *
+ *
+ * The dst address is unique in the mesh path table. Since the mesh_path is
+ * protected by RCU, deleting the next_hop STA must remove / substitute the
+ * mesh_path structure and wait until that is no longer reachable before
+ * destroying the STA completely.
+ */
+struct rtw_mesh_path {
+	u8 dst[ETH_ALEN];
+	u8 mpp[ETH_ALEN];	/* used for MPP or MAP */
+	rtw_rhash_head rhash;
+	rtw_hlist_node gate_list;
+	_adapter *adapter;
+	struct sta_info __rcu *next_hop;
+	_timer timer;
+	_queue frame_queue;
+	u32 frame_queue_len;
+	rtw_rcu_head rcu;
+	u32 sn;
+	u32 metric;
+	u8 hop_count;
+	systime exp_time;
+	systime discovery_timeout;
+	systime gate_timeout;
+	u32 gate_ann_int;    /* gate announce interval */
+	u8 discovery_retries;
+	enum rtw_mesh_path_flags flags;
+	_lock state_lock;
+	u8 rann_snd_addr[ETH_ALEN];
+#ifdef CONFIG_RTW_MESH_ADD_ROOT_CHK
+	u8 add_chk_rann_snd_addr[ETH_ALEN];
+#endif
+	u32 rann_metric;
+	unsigned long last_preq_to_root;
+	bool is_root;
+	bool is_gate;
+	bool gate_asked;
+};
+
+/**
+ * struct rtw_mesh_table
+ *
+ * @known_gates: list of known mesh gates and their mpaths by the station. The
+ * gate's mpath may or may not be resolved and active.
+ * @gates_lock: protects updates to known_gates
+ * @rhead: the rhashtable containing struct mesh_paths, keyed by dest addr
+ * @entries: number of entries in the table
+ */
+struct rtw_mesh_table {
+	rtw_hlist_head known_gates;
+	_lock gates_lock;
+	rtw_rhashtable rhead;
+	ATOMIC_T entries;
+};
+
+#define RTW_MESH_PATH_EXPIRE (600 * HZ)
+
+/* Maximum number of paths per interface */
+#define RTW_MESH_MAX_MPATHS		1024
+
+/* Number of frames buffered per destination for unresolved destinations */
+#define RTW_MESH_FRAME_QUEUE_LEN	10
+
+int rtw_mesh_nexthop_lookup(_adapter *adapter,
+	const u8 *mda, const u8 *msa, u8 *ra);
+int rtw_mesh_nexthop_resolve(_adapter *adapter,
+			 struct xmit_frame *xframe);
+
+struct rtw_mesh_path *rtw_mesh_path_lookup(_adapter *adapter,
+				   const u8 *dst);
+struct rtw_mesh_path *rtw_mpp_path_lookup(_adapter *adapter,
+				  const u8 *dst);
+int rtw_mpp_path_add(_adapter *adapter,
+		 const u8 *dst, const u8 *mpp);
+void dump_mpp(void *sel, _adapter *adapter);
+
+struct rtw_mesh_path *
+rtw_mesh_path_lookup_by_idx(_adapter *adapter, int idx);
+void dump_mpath(void *sel, _adapter *adapter);
+
+struct rtw_mesh_path *
+rtw_mpp_path_lookup_by_idx(_adapter *adapter, int idx);
+void rtw_mesh_path_fix_nexthop(struct rtw_mesh_path *mpath, struct sta_info *next_hop);
+void rtw_mesh_path_expire(_adapter *adapter);
+
+struct rtw_mesh_path *
+rtw_mesh_path_add(_adapter *adapter, const u8 *dst);
+
+int rtw_mesh_path_add_gate(struct rtw_mesh_path *mpath);
+void rtw_mesh_gate_del(struct rtw_mesh_table *tbl, struct rtw_mesh_path *mpath);
+bool rtw_mesh_gate_search(struct rtw_mesh_table *tbl, const u8 *addr);
+int rtw_mesh_path_send_to_gates(struct rtw_mesh_path *mpath);
+int rtw_mesh_gate_num(_adapter *adapter);
+bool rtw_mesh_is_primary_gate(_adapter *adapter);
+void dump_known_gates(void *sel, _adapter *adapter);
+
+void rtw_mesh_plink_broken(struct sta_info *sta);
+
+void rtw_mesh_path_assign_nexthop(struct rtw_mesh_path *mpath, struct sta_info *sta);
+void rtw_mesh_path_flush_pending(struct rtw_mesh_path *mpath);
+void rtw_mesh_path_tx_pending(struct rtw_mesh_path *mpath);
+int rtw_mesh_pathtbl_init(_adapter *adapter);
+void rtw_mesh_pathtbl_unregister(_adapter *adapter);
+int rtw_mesh_path_del(_adapter *adapter, const u8 *addr);
+
+void rtw_mesh_path_flush_by_nexthop(struct sta_info *sta);
+void rtw_mesh_path_discard_frame(_adapter *adapter,
+			     struct xmit_frame *xframe);
+
+static inline void rtw_mesh_path_activate(struct rtw_mesh_path *mpath)
+{
+	mpath->flags |= RTW_MESH_PATH_ACTIVE | RTW_MESH_PATH_RESOLVED;
+}
+
+void rtw_mesh_path_flush_by_iface(_adapter *adapter);
+
+#endif /* __RTW_MESH_PATHTBL_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/monitor/rtw_radiotap.c b/drivers/net/wireless/realtek/rtl8852bs/core/monitor/rtw_radiotap.c
new file mode 100644
index 000000000000..c7b523be6560
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/monitor/rtw_radiotap.c
@@ -0,0 +1,806 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_RADIOTAP_C_
+
+#ifdef CONFIG_WIFI_MONITOR
+
+#include <drv_types.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0))
+#include <linux/bitfield.h>
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0))
+#define __bf_shf(x) (__builtin_ffsll(x) - 1)
+
+/**
+ * FIELD_PREP() - prepare a bitfield element
+ * @_mask: shifted mask defining the field's length and position
+ * @_val:  value to put in the field
+ *
+ * FIELD_PREP() masks and shifts up the value.  The result should
+ * be combined with other fields of the bitfield using logical OR.
+ */
+#define FIELD_PREP(_mask, _val)						\
+	({								\
+		((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask);	\
+	})
+#endif
+
+#define __encode_bits(w, v, field) FIELD_PREP((u##w)(field), v)
+#define __u16_encode_bits(v, field)	__encode_bits(16, v, field)
+#define __le16_encode_bits(v, field)	cpu_to_le16(__encode_bits(16, v, field))
+#define __be16_encode_bits(v, field)	cpu_to_be16(__encode_bits(16, v, field))
+#define __u32_encode_bits(v, field)	__encode_bits(32, v, field)
+#define __le32_encode_bits(v, field)	cpu_to_le32(__encode_bits(32, v, field))
+#define __be32_encode_bits(v, field)	cpu_to_be32(__encode_bits(32, v, field))
+#define __u64_encode_bits(v, field)	__encode_bits(64, v, field)
+#define __le64_encode_bits(v, field)	cpu_to_le64(__encode_bits(64, v, field))
+#define __be64_encode_bits(v, field)	cpu_to_be64(__encode_bits(64, v, field))
+
+
+#define CHAN2FREQ(a) ((a < 14) ? (2407+5*a) : (5000+5*a))
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0))
+#define IEEE80211_RADIOTAP_ZERO_LEN_PSDU 26
+#define IEEE80211_RADIOTAP_LSIG 27
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0))
+#define IEEE80211_RADIOTAP_TIMESTAMP 22
+/* For IEEE80211_RADIOTAP_TIMESTAMP */
+#define IEEE80211_RADIOTAP_TIMESTAMP_UNIT_MASK			0x000F
+#define IEEE80211_RADIOTAP_TIMESTAMP_UNIT_MS			0x0000
+#define IEEE80211_RADIOTAP_TIMESTAMP_UNIT_US			0x0001
+#define IEEE80211_RADIOTAP_TIMESTAMP_UNIT_NS			0x0003
+#define IEEE80211_RADIOTAP_TIMESTAMP_SPOS_MASK			0x00F0
+#define IEEE80211_RADIOTAP_TIMESTAMP_SPOS_BEGIN_MDPU		0x0000
+#define IEEE80211_RADIOTAP_TIMESTAMP_SPOS_EO_MPDU		0x0010
+#define IEEE80211_RADIOTAP_TIMESTAMP_SPOS_EO_PPDU		0x0020
+#define IEEE80211_RADIOTAP_TIMESTAMP_SPOS_PLCP_SIG_ACQ		0x0030
+#define IEEE80211_RADIOTAP_TIMESTAMP_SPOS_UNKNOWN		0x00F0
+
+#define IEEE80211_RADIOTAP_TIMESTAMP_FLAG_64BIT			0x00
+#define IEEE80211_RADIOTAP_TIMESTAMP_FLAG_32BIT			0x01
+#define IEEE80211_RADIOTAP_TIMESTAMP_FLAG_ACCURACY		0x02
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0))
+/* for IEEE80211_RADIOTAP_CHANNEL */
+#define	IEEE80211_CHAN_GSM	0x1000	/* GSM (900 MHz) */
+#define	IEEE80211_CHAN_STURBO	0x2000	/* Static Turbo */
+#define	IEEE80211_CHAN_HALF	0x4000	/* Half channel (10 MHz wide) */
+#define	IEEE80211_CHAN_QUARTER	0x8000	/* Quarter channel (5 MHz wide) */
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0))
+#define IEEE80211_RADIOTAP_HE 23
+
+struct ieee80211_radiotap_he {
+	u16 data1, data2, data3, data4, data5, data6;
+};
+
+enum ieee80211_radiotap_he_bits {
+	IEEE80211_RADIOTAP_HE_DATA1_FORMAT_MASK		= 3,
+	IEEE80211_RADIOTAP_HE_DATA1_FORMAT_SU		= 0,
+	IEEE80211_RADIOTAP_HE_DATA1_FORMAT_EXT_SU	= 1,
+	IEEE80211_RADIOTAP_HE_DATA1_FORMAT_MU		= 2,
+	IEEE80211_RADIOTAP_HE_DATA1_FORMAT_TRIG		= 3,
+
+	IEEE80211_RADIOTAP_HE_DATA1_BSS_COLOR_KNOWN	= 0x0004,
+	IEEE80211_RADIOTAP_HE_DATA1_BEAM_CHANGE_KNOWN	= 0x0008,
+	IEEE80211_RADIOTAP_HE_DATA1_UL_DL_KNOWN		= 0x0010,
+	IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN	= 0x0020,
+	IEEE80211_RADIOTAP_HE_DATA1_DATA_DCM_KNOWN	= 0x0040,
+	IEEE80211_RADIOTAP_HE_DATA1_CODING_KNOWN	= 0x0080,
+	IEEE80211_RADIOTAP_HE_DATA1_LDPC_XSYMSEG_KNOWN	= 0x0100,
+	IEEE80211_RADIOTAP_HE_DATA1_STBC_KNOWN		= 0x0200,
+	IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE_KNOWN	= 0x0400,
+	IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE2_KNOWN	= 0x0800,
+	IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE3_KNOWN	= 0x1000,
+	IEEE80211_RADIOTAP_HE_DATA1_SPTL_REUSE4_KNOWN	= 0x2000,
+	IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN	= 0x4000,
+	IEEE80211_RADIOTAP_HE_DATA1_DOPPLER_KNOWN	= 0x8000,
+
+	IEEE80211_RADIOTAP_HE_DATA2_PRISEC_80_KNOWN	= 0x0001,
+	IEEE80211_RADIOTAP_HE_DATA2_GI_KNOWN		= 0x0002,
+	IEEE80211_RADIOTAP_HE_DATA2_NUM_LTF_SYMS_KNOWN	= 0x0004,
+	IEEE80211_RADIOTAP_HE_DATA2_PRE_FEC_PAD_KNOWN	= 0x0008,
+	IEEE80211_RADIOTAP_HE_DATA2_TXBF_KNOWN		= 0x0010,
+	IEEE80211_RADIOTAP_HE_DATA2_PE_DISAMBIG_KNOWN	= 0x0020,
+	IEEE80211_RADIOTAP_HE_DATA2_TXOP_KNOWN		= 0x0040,
+	IEEE80211_RADIOTAP_HE_DATA2_MIDAMBLE_KNOWN	= 0x0080,
+	IEEE80211_RADIOTAP_HE_DATA2_RU_OFFSET		= 0x3f00,
+	IEEE80211_RADIOTAP_HE_DATA2_RU_OFFSET_KNOWN	= 0x4000,
+	IEEE80211_RADIOTAP_HE_DATA2_PRISEC_80_SEC	= 0x8000,
+
+	IEEE80211_RADIOTAP_HE_DATA3_BSS_COLOR		= 0x003f,
+	IEEE80211_RADIOTAP_HE_DATA3_BEAM_CHANGE		= 0x0040,
+	IEEE80211_RADIOTAP_HE_DATA3_UL_DL		= 0x0080,
+	IEEE80211_RADIOTAP_HE_DATA3_DATA_MCS		= 0x0f00,
+	IEEE80211_RADIOTAP_HE_DATA3_DATA_DCM		= 0x1000,
+	IEEE80211_RADIOTAP_HE_DATA3_CODING		= 0x2000,
+	IEEE80211_RADIOTAP_HE_DATA3_LDPC_XSYMSEG	= 0x4000,
+	IEEE80211_RADIOTAP_HE_DATA3_STBC		= 0x8000,
+
+	IEEE80211_RADIOTAP_HE_DATA4_SU_MU_SPTL_REUSE	= 0x000f,
+	IEEE80211_RADIOTAP_HE_DATA4_MU_STA_ID		= 0x7ff0,
+	IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE1	= 0x000f,
+	IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE2	= 0x00f0,
+	IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE3	= 0x0f00,
+	IEEE80211_RADIOTAP_HE_DATA4_TB_SPTL_REUSE4	= 0xf000,
+
+	IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC	= 0x000f,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_20MHZ	= 0,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_40MHZ	= 1,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_80MHZ	= 2,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_160MHZ	= 3,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_26T	= 4,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_52T	= 5,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_106T	= 6,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_242T	= 7,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_484T	= 8,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_996T	= 9,
+		IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_2x996T	= 10,
+
+	IEEE80211_RADIOTAP_HE_DATA5_GI			= 0x0030,
+		IEEE80211_RADIOTAP_HE_DATA5_GI_0_8			= 0,
+		IEEE80211_RADIOTAP_HE_DATA5_GI_1_6			= 1,
+		IEEE80211_RADIOTAP_HE_DATA5_GI_3_2			= 2,
+
+	IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE		= 0x00c0,
+		IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_UNKNOWN		= 0,
+		IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_1X			= 1,
+		IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X			= 2,
+		IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X			= 3,
+	IEEE80211_RADIOTAP_HE_DATA5_NUM_LTF_SYMS	= 0x0700,
+	IEEE80211_RADIOTAP_HE_DATA5_PRE_FEC_PAD		= 0x3000,
+	IEEE80211_RADIOTAP_HE_DATA5_TXBF		= 0x4000,
+	IEEE80211_RADIOTAP_HE_DATA5_PE_DISAMBIG		= 0x8000,
+
+	IEEE80211_RADIOTAP_HE_DATA6_NSTS		= 0x000f,
+	IEEE80211_RADIOTAP_HE_DATA6_DOPPLER		= 0x0010,
+	IEEE80211_RADIOTAP_HE_DATA6_TXOP		= 0x7f00,
+	IEEE80211_RADIOTAP_HE_DATA6_MIDAMBLE_PDCTY	= 0x8000,
+};
+
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+#define IEEE80211_RADIOTAP_VHT 21
+/* For IEEE80211_RADIOTAP_VHT */
+#define IEEE80211_RADIOTAP_VHT_KNOWN_STBC			0x0001
+#define IEEE80211_RADIOTAP_VHT_KNOWN_TXOP_PS_NA			0x0002
+#define IEEE80211_RADIOTAP_VHT_KNOWN_GI				0x0004
+#define IEEE80211_RADIOTAP_VHT_KNOWN_SGI_NSYM_DIS		0x0008
+#define IEEE80211_RADIOTAP_VHT_KNOWN_LDPC_EXTRA_OFDM_SYM	0x0010
+#define IEEE80211_RADIOTAP_VHT_KNOWN_BEAMFORMED			0x0020
+#define IEEE80211_RADIOTAP_VHT_KNOWN_BANDWIDTH			0x0040
+#define IEEE80211_RADIOTAP_VHT_KNOWN_GROUP_ID			0x0080
+#define IEEE80211_RADIOTAP_VHT_KNOWN_PARTIAL_AID		0x0100
+
+#define IEEE80211_RADIOTAP_VHT_FLAG_STBC			0x01
+#define IEEE80211_RADIOTAP_VHT_FLAG_TXOP_PS_NA			0x02
+#define IEEE80211_RADIOTAP_VHT_FLAG_SGI				0x04
+#define IEEE80211_RADIOTAP_VHT_FLAG_SGI_NSYM_M10_9		0x08
+#define IEEE80211_RADIOTAP_VHT_FLAG_LDPC_EXTRA_OFDM_SYM		0x10
+#define IEEE80211_RADIOTAP_VHT_FLAG_BEAMFORMED			0x20
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0))
+#define IEEE80211_RADIOTAP_CODING_LDPC_USER0			0x01
+#define IEEE80211_RADIOTAP_CODING_LDPC_USER1			0x02
+#define IEEE80211_RADIOTAP_CODING_LDPC_USER2			0x04
+#define IEEE80211_RADIOTAP_CODING_LDPC_USER3			0x08
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0))
+#define IEEE80211_RADIOTAP_AMPDU_STATUS 20
+/* For IEEE80211_RADIOTAP_AMPDU_STATUS */
+#define IEEE80211_RADIOTAP_AMPDU_REPORT_ZEROLEN		0x0001
+#define IEEE80211_RADIOTAP_AMPDU_IS_ZEROLEN		0x0002
+#define IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN		0x0004
+#define IEEE80211_RADIOTAP_AMPDU_IS_LAST		0x0008
+#define IEEE80211_RADIOTAP_AMPDU_DELIM_CRC_ERR		0x0010
+#define IEEE80211_RADIOTAP_AMPDU_DELIM_CRC_KNOWN	0x0020
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0))
+#define IEEE80211_RADIOTAP_AMPDU_EOF			0x0040
+#define IEEE80211_RADIOTAP_AMPDU_EOF_KNOWN		0x0080
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39))
+#define IEEE80211_RADIOTAP_MCS 19
+/* For IEEE80211_RADIOTAP_MCS */
+#define IEEE80211_RADIOTAP_MCS_HAVE_BW		0x01
+#define IEEE80211_RADIOTAP_MCS_HAVE_MCS		0x02
+#define IEEE80211_RADIOTAP_MCS_HAVE_GI		0x04
+#define IEEE80211_RADIOTAP_MCS_HAVE_FMT		0x08
+#define IEEE80211_RADIOTAP_MCS_HAVE_FEC		0x10
+
+#define IEEE80211_RADIOTAP_MCS_BW_MASK		0x03
+#define		IEEE80211_RADIOTAP_MCS_BW_20		0
+#define		IEEE80211_RADIOTAP_MCS_BW_40		1
+#define		IEEE80211_RADIOTAP_MCS_BW_20L		2
+#define		IEEE80211_RADIOTAP_MCS_BW_20U		3
+#define IEEE80211_RADIOTAP_MCS_SGI		0x04
+#define IEEE80211_RADIOTAP_MCS_FMT_GF		0x08
+#define IEEE80211_RADIOTAP_MCS_FEC_LDPC		0x10
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0))
+#define IEEE80211_RADIOTAP_MCS_HAVE_STBC	0x20
+
+#define IEEE80211_RADIOTAP_MCS_STBC_MASK	0x60
+#define		IEEE80211_RADIOTAP_MCS_STBC_1	1
+#define		IEEE80211_RADIOTAP_MCS_STBC_2	2
+#define		IEEE80211_RADIOTAP_MCS_STBC_3	3
+#define IEEE80211_RADIOTAP_MCS_STBC_SHIFT	5
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34))
+#define IEEE80211_RADIOTAP_RADIOTAP_NAMESPACE 29
+#define IEEE80211_RADIOTAP_VENDOR_NAMESPACE 30
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 30))
+#define IEEE80211_RADIOTAP_F_BADFCS 0x40
+#endif
+
+static inline void _rtw_radiotap_fill_flags(struct rx_pkt_attrib *a, u8 *flags)
+{
+	struct moinfo *moif = (struct moinfo *)&a->moif;
+
+	if (0)
+		*flags |= IEEE80211_RADIOTAP_F_CFP;
+
+	if (0)
+		*flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
+
+	if ((a->encrypt == 1) || (a->encrypt == 5))
+		*flags |= IEEE80211_RADIOTAP_F_WEP;
+
+	if (a->mfrag)
+		*flags |= IEEE80211_RADIOTAP_F_FRAG;
+
+	if (1)
+		*flags |= IEEE80211_RADIOTAP_F_FCS;
+
+	if (0)
+		*flags |= IEEE80211_RADIOTAP_F_DATAPAD;
+
+	if (a->crc_err)
+		*flags |= IEEE80211_RADIOTAP_F_BADFCS;
+
+	/* Currently unspecified but used
+	   for short guard interval (HT) */
+	if (moif->u.snif_info.sgi || a->sgi)
+		*flags |= 0x80;
+
+}
+
+sint rtw_fill_radiotap_hdr(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		struct rx_pkt_attrib *a, struct rtw_recv_pkt *rx_req, u8 *buf)
+{
+#define RTAP_HDR_MAX 64
+
+	sint ret = _SUCCESS;
+	struct moinfo *moif = (struct moinfo *)&a->moif;
+
+	u8 rx_cnt = 0;
+
+	int i = 0;
+	u8 tmp_8bit = 0;
+	u16 tmp_16bit = 0;
+	u32 tmp_32bit = 0;
+	u64 tmp_64bit = 0;
+
+	struct sk_buff *pskb = NULL;
+
+	struct ieee80211_radiotap_header *rtap_hdr = NULL;
+	u8 *ptr = NULL;
+
+	/*
+	  radiotap length (include header 8)
+	  11G length: 36 (0x0040002f)
+	  11N length:
+	  11AC length: 60 (0x0070002b)
+	 */
+	u8 hdr_buf[RTAP_HDR_MAX] = { 0 };
+	u16 rt_len = 8;
+
+	/* create header */
+	rtap_hdr = (struct ieee80211_radiotap_header *)&hdr_buf[0];
+	rtap_hdr->it_version = PKTHDR_RADIOTAP_VERSION;
+
+	/* each antenna information */
+	rx_cnt = GET_HAL_RFPATH_NUM(padapter->dvobj);
+#if 0
+	if (rx_cnt > 1) {
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_RADIOTAP_NAMESPACE) |
+		BIT(IEEE80211_RADIOTAP_EXT);
+
+		for (i = 1; i < rx_cnt; i++) {
+			tmp_32bit = (BIT(IEEE80211_RADIOTAP_DBM_ANTSIGNAL) |
+				     BIT(IEEE80211_RADIOTAP_LOCK_QUALITY) |
+				     BIT(IEEE80211_RADIOTAP_ANTENNA) |
+				     BIT(IEEE80211_RADIOTAP_RADIOTAP_NAMESPACE) |
+				     BIT(IEEE80211_RADIOTAP_EXT));
+			_rtw_memcpy(&hdr_buf[rt_len], &tmp_32bit, 4);
+			rt_len += 4;
+		}
+
+		tmp_32bit = (BIT(IEEE80211_RADIOTAP_DBM_ANTSIGNAL) |
+			     BIT(IEEE80211_RADIOTAP_LOCK_QUALITY) |
+			     BIT(IEEE80211_RADIOTAP_ANTENNA));
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_32bit, 4);
+		rt_len += 4;
+	}
+#endif
+
+	/* tsft, Required Alignment: 8 bytes */
+	if (0) { //(a->free_cnt) {
+		/* TSFT + free_cnt */
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_TSFT);
+		if (!IS_ALIGNED(rt_len, 8))
+			rt_len = ((rt_len + 7) & 0xFFF8); /* Alignment */
+
+		tmp_64bit = cpu_to_le64(rx_req->mdata.freerun_cnt);
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_64bit, 8);
+		rt_len += 8;
+	}
+
+	/* flags */
+	rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_FLAGS);
+	_rtw_radiotap_fill_flags(a, &hdr_buf[rt_len]);
+	rt_len += 1;
+
+	/* rate */
+	if (rx_req->mdata.rx_rate <= RTW_DATA_RATE_OFDM54) {
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_RATE);
+		hdr_buf[rt_len] = hwrate_to_mrate(rx_req->mdata.rx_rate);
+		rt_len += 1;
+	}
+
+	/* channel & flags, Required Alignment: 2 bytes */
+	if (1) {
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_CHANNEL);
+		rt_len += (rt_len % 2); /* Alignment */
+
+		tmp_16bit = CHAN2FREQ(rtw_get_oper_ch(padapter, padapter_link));
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_16bit, 2);
+		rt_len += 2;
+
+		/* channel flags */
+		tmp_16bit = 0;
+		if (WIFI_ROLE_LINK_IS_ON_24G(padapter_link))
+			tmp_16bit |= cpu_to_le16(IEEE80211_CHAN_2GHZ);
+		else
+			tmp_16bit |= cpu_to_le16(IEEE80211_CHAN_5GHZ);
+
+		if (rx_req->mdata.rx_rate <= RTW_DATA_RATE_CCK11) {
+			/* CCK */
+			tmp_16bit |= cpu_to_le16(IEEE80211_CHAN_CCK);
+		} else {
+			/* OFDM */
+			tmp_16bit |= cpu_to_le16(IEEE80211_CHAN_OFDM);
+		}
+
+		if (rtw_get_oper_bw(padapter, padapter_link) == CHANNEL_WIDTH_10) {
+			/* 10Mhz Channel Width */
+			tmp_16bit |= cpu_to_le16(IEEE80211_CHAN_HALF);
+		}
+
+		if (rtw_get_oper_bw(padapter, padapter_link) == CHANNEL_WIDTH_5) {
+			/* 5Mhz Channel Width */
+			tmp_16bit |= cpu_to_le16(IEEE80211_CHAN_QUARTER);
+		}
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_16bit, 2);
+		rt_len += 2;
+	}
+
+	/* dBm Antenna Signal */
+	rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_DBM_ANTSIGNAL);
+	hdr_buf[rt_len] = a->phy_info.recv_signal_power;
+	rt_len += 1;
+
+#if 0
+	/* dBm Antenna Noise */
+	rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_DBM_ANTNOISE);
+	hdr_buf[rt_len] = 0;
+	rt_len += 1;
+#endif
+#if 0
+	/* Signal Quality, Required Alignment: 2 bytes */
+	rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_LOCK_QUALITY);
+	if (!IS_ALIGNED(rt_len, 2))
+	rt_len++;
+	hdr_buf[rt_len] = a->phy_info.signal_quality;
+	rt_len += 2;
+
+#endif
+
+#if 0
+	/* Antenna */
+	rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_ANTENNA);
+	hdr_buf[rt_len] = 0; /* pHalData->rf_type; */
+	rt_len += 1;
+#endif
+#if 0
+	/* RX flags, Required Alignment: 2 bytes */
+	rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_RX_FLAGS);
+	tmp_16bit = 0;
+	_rtw_memcpy(&hdr_buf[rt_len], &tmp_16bit, 2);
+	rt_len += 2;
+#endif
+
+	/* MCS information, Required Alignment: 1 bytes */
+	if (rx_req->mdata.rx_rate >= RTW_DATA_RATE_MCS0 && rx_req->mdata.rx_rate <= RTW_DATA_RATE_MCS31) {
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_MCS);
+		/* Structure u8 known, u8 flags, u8 mcs */
+
+		/* known.bandwidth */
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_MCS_HAVE_BW;
+		if (moif->u.snif_info.ofdm_bw)
+			hdr_buf[rt_len + 1] |= IEEE80211_RADIOTAP_MCS_BW_40;
+		if (rx_req->mdata.bw == CHANNEL_WIDTH_40)
+			hdr_buf[rt_len + 1] |= IEEE80211_RADIOTAP_MCS_BW_40;
+		else
+			hdr_buf[rt_len + 1] |= IEEE80211_RADIOTAP_MCS_BW_20;
+
+
+		/* known.guard interval */
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_MCS_HAVE_GI;
+		if (moif->u.snif_info.sgi) {
+			hdr_buf[rt_len + 1] |= IEEE80211_RADIOTAP_MCS_SGI;
+		} else {
+			hdr_buf[rt_len + 1] |= ((a->sgi & 0x01) << 2);
+		}
+
+		/* FEC Type */
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_MCS_HAVE_FEC;
+		if (moif->u.snif_info.ldpc) {
+			hdr_buf[rt_len + 1] |= ((moif->u.snif_info.ldpc & 0x01) << 4);
+		} else {
+			hdr_buf[rt_len + 1] |= ((a->ldpc & 0x01) << 4);
+		}
+
+		/* STBC */
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_MCS_HAVE_STBC;
+		if (moif->u.snif_info.stbc) {
+			hdr_buf[rt_len + 1] |= ((moif->u.snif_info.stbc & 0x03) << 5);
+		} else {
+			hdr_buf[rt_len + 1] |= ((a->stbc & 0x03) << 5);
+		}
+
+		/* known.MCS index */
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_MCS_HAVE_MCS;
+
+		/* u8 mcs */
+		hdr_buf[rt_len + 2] = rx_req->mdata.rx_rate - RTW_DATA_RATE_MCS0;
+
+		rt_len += 3;
+	}
+
+	/* AMPDU, Required Alignment: 4 bytes */
+	if (rx_req->mdata.ampdu) {
+		static u32 ref_num = 0x10000000;
+		static u8 ppdu_cnt = 0;
+
+		/* Structure u32 reference number, u16 flags, u8 delimiter CRC value, u8 reserved */
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_AMPDU_STATUS);
+		if (!IS_ALIGNED(rt_len, 4))
+			rt_len = ((rt_len + 3) & 0xFFFC); /* Alignment */
+
+		/* u32 reference number */
+		if (rx_req->mdata.ppdu_cnt != ppdu_cnt) {
+			ppdu_cnt = rx_req->mdata.ppdu_cnt;
+			ref_num += 1;
+		}
+		tmp_32bit = cpu_to_le32(ref_num);
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_32bit, 4);
+		rt_len += 4;
+
+		/* u16 flags */
+		tmp_16bit = 0;
+		if (0) {
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_REPORT_ZEROLEN);
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_IS_ZEROLEN);
+		}
+
+		if (0) {
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_IS_LAST);
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN);
+		}
+
+		if (0) {
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_DELIM_CRC_ERR);
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_DELIM_CRC_KNOWN);
+		}
+
+		if (rx_req->mdata.ampdu_end_pkt) {
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_EOF_KNOWN);
+			tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_AMPDU_EOF);
+		}
+
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_16bit, 2);
+		rt_len += 2;
+
+		/* u8 delimiter CRC value, u8 reserved */
+		rt_len += 2;
+	}
+
+	/* VHT, Required Alignment: 2 bytes */
+	if (rx_req->mdata.rx_rate >= RTW_DATA_RATE_VHT_NSS1_MCS0 && rx_req->mdata.rx_rate <= RTW_DATA_RATE_VHT_NSS4_MCS9) {
+
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_VHT);
+
+		rt_len += (rt_len % 2); /* Alignment */
+
+		/* Structure
+		   u16 known, u8 flags, u8 bandwidth, u8 mcs_nss[4],
+		   u8 coding, u8 group_id, u16 partial_aid */
+
+		tmp_16bit = 0;
+
+		/* STBC */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_STBC);
+		if (moif->u.snif_info.stbc) {
+			hdr_buf[rt_len + 2] |= IEEE80211_RADIOTAP_VHT_FLAG_STBC;
+		} else {
+			hdr_buf[rt_len + 2] |= (a->stbc & 0x01);
+		}
+
+		/* TXOP_PS_NOT_ALLOWED */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_TXOP_PS_NA);
+		if (moif->u.snif_info.vht_txop_not_allow) {
+			hdr_buf[rt_len + 2] |= IEEE80211_RADIOTAP_VHT_FLAG_TXOP_PS_NA;
+		}
+
+
+		/* Guard interval */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_GI);
+		if (moif->u.snif_info.sgi) {
+			hdr_buf[rt_len + 2] |= IEEE80211_RADIOTAP_VHT_FLAG_SGI;
+		} else {
+			hdr_buf[rt_len + 2] |= ((a->sgi & 0x01) << 2);
+		}
+
+		/* Short GI NSYM */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_SGI_NSYM_DIS);
+		if (moif->u.snif_info.vht_nsym_dis) {
+			hdr_buf[rt_len + 2] |= IEEE80211_RADIOTAP_VHT_FLAG_SGI_NSYM_M10_9;
+		}
+
+		/* LDPC extra OFDM symbol */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_LDPC_EXTRA_OFDM_SYM);
+		if (moif->u.snif_info.vht_ldpc_extra) {
+			hdr_buf[rt_len + 2] |= IEEE80211_RADIOTAP_VHT_FLAG_LDPC_EXTRA_OFDM_SYM;
+		} else {
+			hdr_buf[rt_len + 2] |= ((a->ldpc & 0x01) << 4);
+		}
+
+		/* Short GI NSYM */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_BEAMFORMED);
+		if (moif->u.snif_info.vht_beamformed) {
+			hdr_buf[rt_len + 2] |= IEEE80211_RADIOTAP_VHT_FLAG_BEAMFORMED;
+		}
+
+		/* know.Bandwidth */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_BANDWIDTH);
+
+		/* Group ID */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_GROUP_ID);
+
+		/* Partial AID */
+		tmp_16bit |= cpu_to_le16(IEEE80211_RADIOTAP_VHT_KNOWN_PARTIAL_AID);
+
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_16bit, 2);
+		rt_len += 3;
+
+		/* u8 bandwidth */
+		if (moif->u.snif_info.ofdm_bw)
+			tmp_8bit = moif->u.snif_info.ofdm_bw;
+		else
+			tmp_8bit = rx_req->mdata.bw;
+
+		switch (tmp_8bit) {
+		case CHANNEL_WIDTH_20:
+			hdr_buf[rt_len] |= 0;
+			break;
+		case CHANNEL_WIDTH_40:
+			hdr_buf[rt_len] |= 1;
+			break;
+		case CHANNEL_WIDTH_80:
+			hdr_buf[rt_len] |= 4;
+			break;
+		case CHANNEL_WIDTH_160:
+			hdr_buf[rt_len] |= 11;
+			break;
+		default:
+			hdr_buf[rt_len] |= 0;
+		}
+		rt_len += 1;
+
+		/* u8 mcs_nss[4] */
+		if ((RTW_DATA_RATE_VHT_NSS1_MCS0 <= rx_req->mdata.rx_rate) &&
+			(rx_req->mdata.rx_rate <= RTW_DATA_RATE_VHT_NSS4_MCS9)) {
+			/* User 0 */
+			/* MCS */
+			hdr_buf[rt_len] = cpu_to_le16 (((rx_req->mdata.rx_rate - RTW_DATA_RATE_VHT_NSS1_MCS0) % 10) << 4);
+			/* NSS */
+			hdr_buf[rt_len] |= cpu_to_le16((((rx_req->mdata.rx_rate - RTW_DATA_RATE_VHT_NSS1_MCS0) / 10) + 1));
+		}
+		rt_len += 4;
+
+		/* u8 coding, phystat? */
+		hdr_buf[rt_len] = 0;
+		rt_len += 1;
+
+		/* u8 group_id */
+		hdr_buf[rt_len] = moif->u.snif_info.vht_group_id;
+		rt_len += 1;
+
+		/* u16 partial_aid */
+		tmp_16bit = cpu_to_le16(moif->u.snif_info.vht_nsts_aid);
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_16bit, 2);
+		rt_len += 2;
+	}
+
+	/* HE, Required Alignment: 2 bytes */
+	if (rx_req->mdata.rx_rate >= RTW_DATA_RATE_HE_NSS1_MCS0 && rx_req->mdata.rx_rate <= RTW_DATA_RATE_HE_NSS4_MCS11) {
+
+		struct ieee80211_radiotap_he he_hdr;
+		u8 ltf, he_gi, ss;
+
+		_rtw_memset(&he_hdr, 0, sizeof(struct ieee80211_radiotap_he));
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_HE);
+
+		rt_len += (rt_len % 2); /* Alignment */
+		/* Structure
+		   u16 data1, data2, data3, data4, data5, data6 */
+
+		/* HE PPDU Format: 0=HE_SU, 1=HE_EXT_SU, 2=HE_MU, 3=HE_TRIG */
+		he_hdr.data1 |= 0x0000;
+		he_hdr.data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN);
+		he_hdr.data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN);
+
+		he_hdr.data2 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA2_GI_KNOWN);
+		he_hdr.data2 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA2_NUM_LTF_SYMS_KNOWN);
+
+		he_hdr.data3 |= __le16_encode_bits((rx_req->mdata.rx_rate & 0xf), IEEE80211_RADIOTAP_HE_DATA3_DATA_MCS);
+
+		he_hdr.data5 |= cpu_to_le16((rx_req->mdata.bw));
+
+		switch (rx_req->mdata.rx_gi_ltf) {
+		case 0x0:
+			he_gi = IEEE80211_RADIOTAP_HE_DATA5_GI_3_2;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+			break;
+		case 0x1:
+			he_gi = IEEE80211_RADIOTAP_HE_DATA5_GI_0_8;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_4X;
+			break;
+		case 0x2:
+			he_gi = IEEE80211_RADIOTAP_HE_DATA5_GI_1_6;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+			break;
+		case 0x3:
+			he_gi = IEEE80211_RADIOTAP_HE_DATA5_GI_0_8;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_2X;
+			break;
+		case 0x4:
+			he_gi = IEEE80211_RADIOTAP_HE_DATA5_GI_1_6;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_1X;
+			break;
+		case 0x5:
+			he_gi = IEEE80211_RADIOTAP_HE_DATA5_GI_0_8;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_1X;
+			break;
+		default:
+			he_gi = IEEE80211_RADIOTAP_HE_DATA5_GI_0_8;
+			ltf = IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE_1X;
+			break;
+		}
+
+		he_hdr.data5 |= __le16_encode_bits(he_gi, IEEE80211_RADIOTAP_HE_DATA5_GI);
+		he_hdr.data5 |= __le16_encode_bits(ltf, IEEE80211_RADIOTAP_HE_DATA5_LTF_SIZE);
+
+		ss = cpu_to_le16(((0x70 & rx_req->mdata.rx_rate) >> 4) + 1);
+		he_hdr.data6 |= __le16_encode_bits(ss, IEEE80211_RADIOTAP_HE_DATA6_NSTS);
+
+		_rtw_memcpy(&hdr_buf[rt_len], &he_hdr, sizeof(struct ieee80211_radiotap_he));
+
+		rt_len += sizeof(struct ieee80211_radiotap_he);
+	}
+
+	/* frame timestamp, Required Alignment: 8 bytes */
+	if (0) { //(a->free_cnt) {
+		rtap_hdr->it_present |= BIT(IEEE80211_RADIOTAP_TIMESTAMP);
+		if (!IS_ALIGNED(rt_len, 8))
+			rt_len = ((rt_len + 7) & 0xFFF8); /* Alignment */
+
+		/* u64 timestamp */
+		tmp_64bit = cpu_to_le64(rx_req->mdata.freerun_cnt);
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_64bit, 8);
+		rt_len += 8;
+
+		/* u16 accuracy */
+		tmp_16bit = cpu_to_le16(22);
+		_rtw_memcpy(&hdr_buf[rt_len], &tmp_16bit, 2);
+		rt_len += 2;
+
+		/* u8 unit/position */
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_TIMESTAMP_UNIT_US;
+		rt_len += 1;
+
+		/* u8 flags */
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_TIMESTAMP_FLAG_32BIT;
+		hdr_buf[rt_len] |= IEEE80211_RADIOTAP_TIMESTAMP_FLAG_ACCURACY;
+		rt_len += 1;
+	}
+
+	/* each antenna information */
+#if 0
+	if (rx_cnt > 1) {
+		for (i = 0; i <= rx_cnt; i++) {
+			/* dBm Antenna Signal */
+			hdr_buf[rt_len] = a->phy_info.rx_mimo_signal_strength[i];
+			rt_len += 1;
+
+			/* Signal Quality */
+			if (!IS_ALIGNED(rt_len, 2))
+				rt_len++;
+			hdr_buf[rt_len] = cpu_to_le16(a->phy_info.rx_mimo_signal_quality[i]);
+			rt_len += 2;
+
+			/* Antenna */
+			hdr_buf[rt_len] = i; /* pHalData->rf_type; */
+			rt_len += 1;
+		}
+	}
+#endif
+
+	/* push to skb */
+	pskb = (struct sk_buff *)buf;
+	if (skb_headroom(pskb) < rt_len) {
+		RTW_INFO("%s:%d %s headroom is too small.\n", __FILE__, __LINE__, __func__);
+		ret = _FAIL;
+		return ret;
+	}
+
+	ptr = skb_push(pskb, rt_len);
+	if (ptr) {
+		rtap_hdr->it_len = cpu_to_le16(rt_len);
+		rtap_hdr->it_present = cpu_to_le32(rtap_hdr->it_present);
+		memcpy(ptr, rtap_hdr, rt_len);
+	} else
+		ret = _FAIL;
+
+	return ret;
+
+}
+
+void rx_query_moinfo(struct rx_pkt_attrib *a, u8 *desc)
+{
+	switch (a->drvinfo_sz) {
+	case 40:
+		_rtw_memcpy(a->moif, &desc[32], 8);
+		break;
+	case 48:
+		_rtw_memcpy(a->moif, &desc[32], 12);
+		break;
+	case 32:
+		/* passthrough */
+	default:
+		break;
+	}
+}
+
+#endif /* CONFIG_WIFI_MONITOR */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/monitor/rtw_radiotap.h b/drivers/net/wireless/realtek/rtl8852bs/core/monitor/rtw_radiotap.h
new file mode 100644
index 000000000000..554d73b21326
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/monitor/rtw_radiotap.h
@@ -0,0 +1,64 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_RADIOTAP_H_
+#define __RTW_RADIOTAP_H_
+
+struct mon_reg_backup {
+	/* flags */
+	u8	known_rcr:1;
+	u8	known_drvinfo:1;
+	u8	known_rxfilter:1;
+	u8	known_misc0:1;
+	/* data */
+	u8	drvinfo;
+	u16	rxfilter0;
+	u16	rxfilter1;
+	u16	rxfilter2;
+	u32	rcr;
+	u32	misc0;
+};
+
+struct moinfo {
+	union {
+		struct  {
+			u32	sgi:1;
+			u32	ldpc:1;
+			u32	stbc:2;
+			u32	not_sounding:1;
+			u32	ofdm_bw:2;
+			u32	vht_group_id:2;
+			u32	vht_nsts_aid:12;
+			u32	vht_txop_not_allow:1;
+			u32	vht_nsym_dis:1;
+			u32	vht_ldpc_extra:1;
+			u32	vht_su_mcs:12;
+			u32	vht_beamformed:1;
+		}snif_info;
+
+		struct  {
+			u32	A;
+			u32	B;
+			u32	C;
+		}plcp_info;
+	}u;
+};
+
+sint rtw_fill_radiotap_hdr(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+	struct rx_pkt_attrib *a, struct rtw_recv_pkt *rx_req, u8 *buf);
+
+void rx_query_moinfo(struct rx_pkt_attrib *a, u8 *desc);
+
+#endif /* __RTW_RADIOTAP_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_phl_wrapper.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_phl_wrapper.c
new file mode 100644
index 000000000000..c2056c59856b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_phl_wrapper.c
@@ -0,0 +1,201 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_PHL_C_
+
+#include <drv_types.h>
+
+static enum rtw_regd phl_regulation_to_regd(enum REGULATION reg)
+{
+	switch (reg) {
+	case REGULATION_WW:
+		return RTW_REGD_WW;
+	case REGULATION_ETSI:
+		return RTW_REGD_ETSI;
+	case REGULATION_FCC:
+		return RTW_REGD_FCC;
+	case REGULATION_MKK:
+		return RTW_REGD_MKK;
+	case REGULATION_KCC:
+		return RTW_REGD_KCC;
+	case REGULATION_NCC:
+		return RTW_REGD_NCC;
+	case REGULATION_ACMA:
+		return RTW_REGD_ACMA;
+	case REGULATION_NA:
+		return RTW_REGD_NA;
+	case REGULATION_IC:
+		return RTW_REGD_IC;
+	case REGULATION_CHILE:
+		return RTW_REGD_CHILE;
+	case REGULATION_MEX:
+		return RTW_REGD_MEX;
+	case REGULATION_MAX:
+	default:
+		RTW_WARN("unknown reg:%d\n", reg);
+		rtw_warn_on(1);
+		return RTW_REGD_NA;
+	}
+};
+
+static u8 phl_ch_property_to_flag(enum ch_property prop)
+{
+	return ((prop & CH_PASSIVE) ? RTW_CHF_NO_IR : 0)
+		| ((prop & CH_DFS) ? RTW_CHF_DFS : 0)
+		;
+}
+
+static u8 phl_regdb_get_default_regd_2g(u8 id)
+{
+	return phl_regulation_to_regd(rtw_phl_get_domain_regulation_2g(id));
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 phl_regdb_get_default_regd_5g(u8 id)
+{
+	return phl_regulation_to_regd(rtw_phl_get_domain_regulation_5g(id));
+}
+#endif
+
+static bool phl_regdb_is_domain_code_valid(u8 id)
+{
+	return rtw_phl_valid_regulation_domain(id);
+}
+
+static bool phl_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	enum ch_property prop;
+
+	if (rtw_phl_query_domain_channel(id, ch > 14 ? BAND_ON_5G : BAND_ON_24G, ch, &prop)) {
+		if (flags)
+			*flags = phl_ch_property_to_flag(prop);
+		return true;
+	}
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static u8 phl_regdb_get_default_regd_6g(u8 id)
+{
+	return phl_regulation_to_regd(rtw_phl_get_domain_regulation_6g(id));
+}
+
+static bool phl_regdb_is_domain_code_6g_valid(u8 id)
+{
+	return rtw_phl_valid_regulation_domain_6ghz(id);
+}
+
+static bool phl_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	enum ch_property prop;
+
+	if (rtw_phl_query_domain_6g_channel(id, BAND_ON_6G, ch, &prop)) {
+		if (flags)
+			*flags = phl_ch_property_to_flag(prop);
+		return true;
+	}
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static REGULATION_TXPWR_LMT phl_tpo_to_txpwr_lmt(enum TP_OVERWRITE tpo)
+{
+	switch (tpo) {
+	case TPO_CHILE:
+		return TXPWR_LMT_CHILE;
+	case TPO_UK:
+		return TXPWR_LMT_UK;
+	case TPO_QATAR:
+		return TXPWR_LMT_QATAR;
+	case TPO_UKRAINE:
+		return TXPWR_LMT_UKRAINE;
+	case TPO_CN:
+		return TXPWR_LMT_CN;
+	case TPO_THAILAND:
+		return TXPWR_LMT_THAILAND;
+	case TPO_NA:
+		return TXPWR_LMT_DEF;
+	default:
+		RTW_WARN("unknown tpo:%d\n", tpo);
+		rtw_warn_on(1);
+		return TXPWR_LMT_DEF;
+	}
+}
+
+static bool phl_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	struct rtw_regulation_country_chplan plan;
+
+	_rtw_memset(&plan, 0, sizeof(plan));
+
+	if (!rtw_phl_query_country_chplan((char *)alpha2, &plan)) /* TODO:  PHL use type of 'const char *' for 'country' */
+		return false;
+
+	if (ent) {
+		_rtw_memcpy(ent->alpha2, alpha2, 2);
+		ent->domain_code = plan.domain_code;
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		ent->domain_code_6g = plan.domain_code_6g;
+		#endif
+
+		#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE
+		ent->edcca_2g_override = RTW_EDCCA_DEF;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		ent->edcca_5g_override = RTW_EDCCA_DEF;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		ent->edcca_6g_override = RTW_EDCCA_DEF;
+		#endif
+		#endif
+
+		ent->txpwr_lmt_override = phl_tpo_to_txpwr_lmt(plan.tpo);
+		#ifdef CONFIG_CHPLAN_PROTO_EN
+		ent->proto_en = 0
+			/* TODO: SUPPORT_11BE, wait PHL ready */
+			| ((plan.support_mode & SUPPORT_11AC) ? CHPLAN_PROTO_EN_AC : 0)
+			| ((plan.support_mode & SUPPORT_11AX) ? CHPLAN_PROTO_EN_AX : 0)
+			| ((plan.support_mode & SUPPORT_11A) ? CHPLAN_PROTO_EN_A : 0)
+			;
+		#endif
+
+		/* TODO: cate_6g_map, wait PHL ready */
+	}
+
+	return true;
+}
+
+static void phl_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%d-%d", rtw_phl_get_regu_chplan_ver(), rtw_phl_get_regu_country_ver());
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = phl_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = phl_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = phl_regdb_is_domain_code_valid,
+	.domain_get_ch = phl_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = phl_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = phl_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = phl_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = phl_regdb_get_chplan_from_alpha2,
+
+	.get_ver_str = phl_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_58-31.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_58-31.c
new file mode 100644
index 000000000000..1165cf75eeb6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_58-31.c
@@ -0,0 +1,908 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"58"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"31"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 36),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x00, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x00, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x00, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x00, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x00, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x00, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x00, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x32, 0x00, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x06, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x00, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x00, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x00, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x00, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x00, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x3A, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x00, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x00, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x76, 0x7F, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3C, 0x00, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x3A, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_59-32.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_59-32.c
new file mode 100644
index 000000000000..33f7dbb9f47c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_59-32.c
@@ -0,0 +1,917 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"59"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"32"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 36),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x00, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x05, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x00, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x00, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x00, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x00, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x3A, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x00, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x00, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x76, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x3A, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_60-32.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_60-32.c
new file mode 100644
index 000000000000..dc9d40d0440f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_60-32.c
@@ -0,0 +1,920 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"60"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"32"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 36),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x00, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x05, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x00, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x00, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x00, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x00, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x3A, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x00, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x00, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x76, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x3A, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_60-34.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_60-34.c
new file mode 100644
index 000000000000..70bb265836e9
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_60-34.c
@@ -0,0 +1,920 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"60"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"34"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 36),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x00, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x05, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x00, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x00, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x00, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x3A, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x00, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x05, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x3A, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_61-34.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_61-34.c
new file mode 100644
index 000000000000..18e21face86c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_61-34.c
@@ -0,0 +1,923 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"61"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"34"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x00, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x05, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x00, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x00, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x00, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x3A, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x00, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x05, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x3A, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_62-35.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_62-35.c
new file mode 100644
index 000000000000..02bf9ce40164
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_62-35.c
@@ -0,0 +1,924 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"62"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"35"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x05, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x3A, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x05, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x3A, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-36.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-36.c
new file mode 100644
index 000000000000..69eb60ff3e97
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-36.c
@@ -0,0 +1,925 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"63"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"36"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x3A, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x00, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x00, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x00, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x3A, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x3A, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x76, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x3A, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-37.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-37.c
new file mode 100644
index 000000000000..3c639385b743
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-37.c
@@ -0,0 +1,925 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"63"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"37"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x44, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x00, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x00, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-38.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-38.c
new file mode 100644
index 000000000000..412dfbb8309c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-38.c
@@ -0,0 +1,925 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"63"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"38"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x05, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x05, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x51, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-39.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-39.c
new file mode 100644
index 000000000000..6a53cea997c6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-39.c
@@ -0,0 +1,925 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"63"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"39"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x00, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x05, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x05, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x05, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-40.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-40.c
new file mode 100644
index 000000000000..2692c47f6a20
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_63-40.c
@@ -0,0 +1,925 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"63"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"40"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x05, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x05, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x05, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-40.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-40.c
new file mode 100644
index 000000000000..ec5057257df7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-40.c
@@ -0,0 +1,931 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"64"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"40"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x05, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x05, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x05, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-41.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-41.c
new file mode 100644
index 000000000000..e587eabe16f0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-41.c
@@ -0,0 +1,931 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"64"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"41"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x05, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x3A, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-42.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-42.c
new file mode 100644
index 000000000000..0ae7da23691e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_64-42.c
@@ -0,0 +1,931 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"64"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"42"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF    , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF    , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF    , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF    , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF    , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF    , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x05, DEF    , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF    , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF    , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF    , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF    , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF    , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF    , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF    , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF    , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF    , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x05, DEF    , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF    , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF    , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF    , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF    , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x2B, 0x07, DEF    , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF    , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF    , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE  , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN     , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF    , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF    , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF    , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF    , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF    , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF    , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF    , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF    , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF    , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF    , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF    , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF    , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF    , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF    , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF    , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK     , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF    , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF    , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF    , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF    , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF    , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF    , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF    , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF    , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF    , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF    , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF    , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF    , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF    , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF    , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF    , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF    , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF    , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF    , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF    , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF    , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF    , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF    , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF    , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF    , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF    , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF    , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF    , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF    , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF    , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF    , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF    , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF    , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF    , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF    , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF    , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF    , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF    , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF    , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF    , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF    , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF    , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF    , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF    , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF    , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF    , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF    , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF    , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF    , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF    , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF    , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF    , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR  , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF    , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF    , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF    , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF    , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF    , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF    , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF    , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF    , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF    , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF    , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF    , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF    , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF    , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF    , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF    , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF    , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF    , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF    , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF    , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF    , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x06, DEF    , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF    , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF    , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF    , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF    , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF    , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF    , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF    , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF    , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF    , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF    , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE, 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF    , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF    , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF    , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF    , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF    , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF    , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF    , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF    , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF    , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF    , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF    , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF    , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF    , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF    , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF    , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF    , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x06, DEF    , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF    , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_65-44.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_65-44.c
new file mode 100644
index 000000000000..dc2977933c37
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_65-44.c
@@ -0,0 +1,934 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"65"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"44"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF     , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF     , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF     , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF     , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF     , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF     , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x05, DEF     , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF     , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF     , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF     , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF     , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF     , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF     , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF     , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF     , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x61, 0x05, DEF     , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF     , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF     , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF     , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF     , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x07, DEF     , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF     , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE   , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF     , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF     , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF     , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF     , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF     , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF     , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF     , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF     , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF     , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK      , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF     , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF     , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF     , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF     , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF     , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF     , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF     , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF     , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF     , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF     , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF     , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF     , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF     , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF     , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF     , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF     , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF     , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF     , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF     , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF     , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF     , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF     , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF     , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF     , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF     , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF     , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF     , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF     , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF     , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF     , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF     , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF     , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF     , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF     , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF     , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF     , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF     , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF     , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF     , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF     , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF     , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR   , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF     , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF     , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF     , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF     , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF     , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF     , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF     , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF     , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF     , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF     , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF     , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x06, THAILAND, 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF     , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF     , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x06, DEF     , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF     , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF     , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF     , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF     , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF     , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF     , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF     , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x06, DEF     , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_66-45.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_66-45.c
new file mode 100644
index 000000000000..a19d533ac229
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_66-45.c
@@ -0,0 +1,937 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_DOMAIN_MAP_VER	"66"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"45"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+	RTW_CHD_5G_59,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_59 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x11] = CHPLAN_ENT(FCC  , 02, FCC  , 59),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF     , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF     , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF     , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF     , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF     , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF     , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x05, DEF     , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF     , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF     , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF     , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF     , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF     , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF     , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF     , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF     , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x11, 0x00, DEF     , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF     , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF     , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF     , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF     , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x07, DEF     , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF     , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE   , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF     , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF     , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF     , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF     , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF     , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF     , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF     , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x05, DEF     , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF     , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK      , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF     , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF     , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF     , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF     , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF     , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF     , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x05, DEF     , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF     , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF     , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF     , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF     , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF     , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF     , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF     , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF     , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF     , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF     , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF     , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF     , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF     , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF     , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF     , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF     , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF     , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF     , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF     , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF     , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF     , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF     , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF     , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x05, DEF     , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x05, DEF     , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF     , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x05, DEF     , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF     , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF     , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF     , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF     , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF     , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF     , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF     , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF     , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x05, DEF     , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR   , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF     , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF     , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF     , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF     , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF     , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF     , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF     , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF     , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF     , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF     , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF     , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x05, DEF     , 1, 1, 1), /* Sint Maarten */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF     , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF     , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF     , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x06, THAILAND, 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF     , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF     , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF     , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x06, DEF     , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF     , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF     , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF     , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x05, DEF     , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF     , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF     , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x06, DEF     , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF     , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x06, DEF     , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s-%s%s", RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_67-46.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_67-46.c
new file mode 100644
index 000000000000..ea415f9c2fa9
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_67-46.c
@@ -0,0 +1,947 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_MODULE_NAME		""
+#define RTW_DOMAIN_MAP_VER	"67"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"46"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+	RTW_CHD_5G_59,
+	RTW_CHD_5G_60,
+	RTW_CHD_5G_61,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_59 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_60 */	CH_LIST_ENT(26, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_61 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x11] = CHPLAN_ENT(FCC  , 02, FCC  , 59),
+	[0x12] = CHPLAN_ENT(ETSI , 02, ETSI , 60),
+	[0x13] = CHPLAN_ENT(ETSI , 02, ETSI , 61),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF     , 0, 1, 1, 1, ___), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, _I_), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x11, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF     , 0, 0, 1, 1, ___), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x07, DEF     , 1, 1, 1, 1, SI_), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE   , 1, 1, 1, 1, _IV), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 1, 1, 1, 1, ___), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK      , 1, 1, 1, 1, _IV), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF     , 1, 0, 1, 1, _I_), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 0, 1, 1, 1, ___), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF     , 0, 1, 1, 1, ___), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF     , 0, 0, 0, 0, ___), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF     , 1, 1, 1, 1, _IV), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF     , 0, 1, 1, 1, _IV), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF     , 1, 1, 1, 1, _I_), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR   , 1, 1, 1, 1, _IV), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF     , 0, 1, 1, 1, ___), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Sint Maarten */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x06, THAILAND, 1, 1, 1, 1, _IV), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0, 0, ___), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 0, 1, 1, 1, ___), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF     , 1, 1, 1, 1, SI_), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF     , 0, 1, 1, 1, _I_), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s"
+		"%s%s-%s%s"
+		, RTW_MODULE_NAME, strlen(RTW_MODULE_NAME) ? "-" : ""
+		, RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_68-47.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_68-47.c
new file mode 100644
index 000000000000..a8c2c006616f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_68-47.c
@@ -0,0 +1,949 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_MODULE_NAME		""
+#define RTW_DOMAIN_MAP_VER	"68"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"47"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+	RTW_CHD_5G_59,
+	RTW_CHD_5G_60,
+	RTW_CHD_5G_61,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_59 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_60 */	CH_LIST_ENT(26, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_61 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x11] = CHPLAN_ENT(FCC  , 02, FCC  , 59),
+	[0x12] = CHPLAN_ENT(ETSI , 02, ETSI , 60),
+	[0x13] = CHPLAN_ENT(ETSI , 02, ETSI , 61),
+	[0x14] = CHPLAN_ENT(ETSI , 02, ETSI , 32),
+	[0x15] = CHPLAN_ENT(ETSI , 02, ETSI , 09),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x42, 0x00, DEF     , 0, 1, 1, 1, ___), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, _I_), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x11, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF     , 0, 0, 1, 1, ___), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x07, DEF     , 1, 1, 1, 1, SI_), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE   , 1, 1, 1, 1, _IV), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 1, 1, 1, 1, ___), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK      , 1, 1, 1, 1, _IV), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF     , 1, 0, 1, 1, _I_), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 0, 1, 1, 1, ___), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x05, 0x00, DEF     , 0, 1, 1, 1, ___), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x3A, 0x00, DEF     , 0, 0, 0, 0, ___), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF     , 1, 1, 1, 1, _IV), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x00, DEF     , 1, 1, 1, 1, ___), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF     , 0, 1, 1, 1, _IV), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x76, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF     , 1, 1, 1, 1, _I_), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR   , 1, 1, 1, 1, _IV), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x09, 0x00, DEF     , 0, 1, 1, 1, ___), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* Sint Maarten */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _IV), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x06, THAILAND, 1, 1, 1, 1, _IV), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0, 0, ___), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 0, 1, 1, 1, ___), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF     , 1, 1, 1, 1, SI_), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x76, 0x05, DEF     , 0, 1, 1, 1, _I_), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x05, DEF     , 1, 1, 1, 1, SI_), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s"
+		"%s%s-%s%s"
+		, RTW_MODULE_NAME, strlen(RTW_MODULE_NAME) ? "-" : ""
+		, RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8851BE_M.2_2230-67-3.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8851BE_M.2_2230-67-3.c
new file mode 100644
index 000000000000..d9f826184831
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8851BE_M.2_2230-67-3.c
@@ -0,0 +1,947 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_MODULE_NAME		"8851BE_M.2_2230"
+#define RTW_DOMAIN_MAP_VER	"67"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"3"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+	RTW_CHD_5G_59,
+	RTW_CHD_5G_60,
+	RTW_CHD_5G_61,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_59 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_60 */	CH_LIST_ENT(26, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_61 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x11] = CHPLAN_ENT(FCC  , 02, FCC  , 59),
+	[0x12] = CHPLAN_ENT(ETSI , 02, ETSI , 60),
+	[0x13] = CHPLAN_ENT(ETSI , 02, ETSI , 61),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x00, DEF     , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x0A, 0x00, DEF     , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF     , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF     , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF     , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x0A, 0x00, DEF     , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x00, DEF     , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF     , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF     , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF     , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF     , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF     , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF     , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x11, 0x00, DEF     , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF     , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF     , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x0A, 0x00, DEF     , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x0A, 0x00, DEF     , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x00, DEF     , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x00, CHILE   , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF     , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF     , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF     , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF     , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x0A, 0x00, DEF     , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x00, DEF     , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF     , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x00, DEF     , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x00, UK      , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF     , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x00, DEF     , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF     , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x00, DEF     , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF     , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x00, DEF     , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x0A, 0x00, DEF     , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x0A, 0x00, DEF     , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF     , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF     , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x0A, 0x00, DEF     , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x00, DEF     , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x0A, 0x00, DEF     , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x00, DEF     , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x0A, 0x00, DEF     , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x00, DEF     , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x00, DEF     , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF     , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF     , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF     , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x00, DEF     , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF     , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x00, DEF     , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF     , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF     , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x00, QATAR   , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x0A, 0x00, DEF     , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x0A, 0x00, DEF     , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x0A, 0x00, DEF     , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF     , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF     , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF     , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x0A, 0x00, DEF     , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x00, DEF     , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x0A, 0x00, DEF     , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF     , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x00, DEF     , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x0A, 0x00, DEF     , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x0A, 0x00, DEF     , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF     , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x3A, 0x00, DEF     , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x0A, 0x00, DEF     , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x0A, 0x00, DEF     , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF     , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s"
+		"%s%s-%s%s"
+		, RTW_MODULE_NAME, strlen(RTW_MODULE_NAME) ? "-" : ""
+		, RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852BE_M.2_2230-67-48.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852BE_M.2_2230-67-48.c
new file mode 100644
index 000000000000..1406689848ba
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852BE_M.2_2230-67-48.c
@@ -0,0 +1,947 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_MODULE_NAME		"8852BE_M.2_2230"
+#define RTW_DOMAIN_MAP_VER	"67"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"48"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+	RTW_CHD_5G_59,
+	RTW_CHD_5G_60,
+	RTW_CHD_5G_61,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_59 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_60 */	CH_LIST_ENT(26, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_61 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x11] = CHPLAN_ENT(FCC  , 02, FCC  , 59),
+	[0x12] = CHPLAN_ENT(ETSI , 02, ETSI , 60),
+	[0x13] = CHPLAN_ENT(ETSI , 02, ETSI , 61),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 1, 1, 1), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x00, DEF     , 1, 1, 1), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x0A, 0x00, DEF     , 1, 1, 1), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x00, DEF     , 1, 1, 1), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x00, DEF     , 1, 1, 1), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF     , 1, 1, 1), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x00, DEF     , 1, 1, 1), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x0A, 0x00, DEF     , 1, 1, 1), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x00, DEF     , 1, 1, 1), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF     , 1, 1, 1), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x00, DEF     , 1, 1, 1), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x00, DEF     , 1, 1, 1), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x00, DEF     , 1, 1, 1), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x00, DEF     , 1, 1, 1), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x00, DEF     , 1, 1, 1), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 1, 1, 1), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x11, 0x00, DEF     , 1, 1, 1), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x00, DEF     , 1, 1, 1), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x00, DEF     , 1, 1, 1), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x0A, 0x00, DEF     , 1, 1, 1), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x08, 0x00, DEF     , 0, 1, 1), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 1, 1, 1), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x00, DEF     , 1, 1, 1), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 1, 1, 1), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 1, 1, 1), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x00, CHILE   , 1, 1, 1), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 1, 1, 1), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x00, DEF     , 1, 1, 1), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x00, DEF     , 1, 1, 1), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 1, 1, 1), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF     , 1, 1, 1), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF     , 1, 1, 1), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 1, 1, 1), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF     , 1, 1, 1), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 1, 1, 1), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x0A, 0x00, DEF     , 1, 1, 1), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x00, DEF     , 1, 1, 1), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 1, 1, 1), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 1, 1, 1), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x00, DEF     , 1, 1, 1), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x00, DEF     , 1, 1, 1), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x00, UK      , 1, 1, 1), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x00, DEF     , 1, 1, 1), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 1, 1, 1), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x00, DEF     , 1, 1, 1), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x00, DEF     , 1, 1, 1), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 1, 1, 1), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 1, 1, 1), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x00, DEF     , 1, 1, 1), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x00, DEF     , 0, 1, 1), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x00, DEF     , 1, 1, 1), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 1, 1, 1), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 1, 1, 1), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 1, 1, 1), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x0A, 0x00, DEF     , 1, 1, 1), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x0A, 0x00, DEF     , 0, 0, 0), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x00, DEF     , 1, 1, 1), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x00, DEF     , 1, 1, 1), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x0A, 0x00, DEF     , 1, 1, 1), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x00, DEF     , 1, 1, 1), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x0A, 0x00, DEF     , 1, 1, 1), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x00, DEF     , 1, 1, 1), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x00, DEF     , 1, 1, 1), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x0A, 0x00, DEF     , 1, 1, 1), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x00, DEF     , 1, 1, 1), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x00, DEF     , 1, 1, 1), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x00, DEF     , 1, 1, 1), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x00, DEF     , 1, 1, 1), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x00, DEF     , 1, 1, 1), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x00, DEF     , 1, 1, 1), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 1, 1, 1), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 1, 1, 1), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x00, DEF     , 1, 1, 1), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x00, DEF     , 1, 1, 1), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 1, 1, 1), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 1, 1, 1), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF     , 1, 1, 1), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 1, 1, 1), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 1, 1, 1), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 1, 1, 1), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x00, DEF     , 1, 1, 1), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 1, 1, 1), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x00, DEF     , 1, 1, 1), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 1, 1, 1), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x00, DEF     , 1, 1, 1), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 1, 1, 1), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x00, QATAR   , 1, 1, 1), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x00, DEF     , 1, 1, 1), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x00, DEF     , 1, 1, 1), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x0A, 0x00, DEF     , 1, 1, 1), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x0A, 0x00, DEF     , 1, 1, 1), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 1, 1, 1), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x00, DEF     , 1, 1, 1), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 1, 1, 1), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 1, 1, 1), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x0A, 0x00, DEF     , 1, 1, 1), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x00, DEF     , 1, 1, 1), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 1, 1, 1), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x00, DEF     , 1, 1, 1), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x00, DEF     , 1, 1, 1), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x0A, 0x00, DEF     , 1, 1, 1), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x00, DEF     , 1, 1, 1), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 1, 1, 1), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x00, DEF     , 1, 1, 1), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x00, DEF     , 1, 1, 1), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 1, 1, 1), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 1, 1, 1), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 1, 1, 1), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x0A, 0x00, DEF     , 1, 1, 1), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x00, DEF     , 1, 1, 1), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 1, 1, 1), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x00, DEF     , 1, 1, 1), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 1, 1, 1), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 1, 1, 1), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 1, 1, 1), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x00, DEF     , 1, 1, 1), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 1, 1, 1), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x00, DEF     , 1, 1, 1), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 1, 1, 1), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x0A, 0x00, DEF     , 1, 1, 1), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 1, 1, 1), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x0A, 0x00, DEF     , 1, 1, 1), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x00, DEF     , 1, 1, 1), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 1, 1, 1), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF     , 1, 1, 1), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 1, 1, 1), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x0A, 0x00, DEF     , 1, 1, 1), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x00, DEF     , 1, 1, 1), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x0A, 0x00, DEF     , 1, 1, 1), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 1, 1, 1), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x00, DEF     , 1, 1, 1), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF     , 1, 1, 1), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s"
+		"%s%s-%s%s"
+		, RTW_MODULE_NAME, strlen(RTW_MODULE_NAME) ? "-" : ""
+		, RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852CE_M.2_2230-67-51.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852CE_M.2_2230-67-51.c
new file mode 100644
index 000000000000..e73c0ed19b5f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852CE_M.2_2230-67-51.c
@@ -0,0 +1,947 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_MODULE_NAME		"8852CE_M.2_2230"
+#define RTW_DOMAIN_MAP_VER	"67"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"51"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+	RTW_CHD_5G_59,
+	RTW_CHD_5G_60,
+	RTW_CHD_5G_61,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_59 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_60 */	CH_LIST_ENT(26, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_61 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x11] = CHPLAN_ENT(FCC  , 02, FCC  , 59),
+	[0x12] = CHPLAN_ENT(ETSI , 02, ETSI , 60),
+	[0x13] = CHPLAN_ENT(ETSI , 02, ETSI , 61),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, _I_), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x11, 0x00, DEF     , 0, 1, 1, 1, _I_), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x0A, 0x00, DEF     , 0, 0, 1, 1, ___), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x07, DEF     , 1, 1, 1, 1, SI_), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF     , 1, 1, 1, 1, ___), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE   , 1, 1, 1, 1, _I_), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 0, 1, 1, 1, ___), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF     , 0, 1, 1, 1, ___), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK      , 1, 1, 1, 1, _IV), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF     , 1, 0, 1, 1, _I_), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 0, 1, 1, 1, ___), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x0A, 0x00, DEF     , 0, 0, 0, 0, ___), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF     , 1, 1, 1, 1, _IV), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF     , 0, 1, 1, 1, _IV), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF     , 1, 1, 1, 1, _I_), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR   , 1, 1, 1, 1, _IV), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x06, THAILAND, 1, 1, 1, 1, _IV), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0, 0, ___), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 0, 1, 1, 1, ___), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF     , 1, 1, 1, 1, SI_), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF     , 1, 1, 1, 1, ___), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s"
+		"%s%s-%s%s"
+		, RTW_MODULE_NAME, strlen(RTW_MODULE_NAME) ? "-" : ""
+		, RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852CE_M.2_2230-67-52.c b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852CE_M.2_2230-67-52.c
new file mode 100644
index 000000000000..741c83e89506
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/regdb/rtw_regdb_rtk_8852CE_M.2_2230-67-52.c
@@ -0,0 +1,947 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_REGDB_RTK_C_
+
+#include <drv_types.h>
+
+#define RTW_MODULE_NAME		"8852CE_M.2_2230"
+#define RTW_DOMAIN_MAP_VER	"67"
+#define RTW_DOMAIN_MAP_M_VER	""
+#define RTW_COUNTRY_MAP_VER	"52"
+#define RTW_COUNTRY_MAP_M_VER	""
+
+#define rtw_is_5g_band1(ch) ((ch) >= 36 && (ch) <= 48)
+#define rtw_is_5g_band2(ch) ((ch) >= 52 && (ch) <= 64)
+#define rtw_is_5g_band3(ch) ((ch) >= 100 && (ch) <= 144)
+#define rtw_is_5g_band4(ch) ((ch) >= 149 && (ch) <= 177)
+
+#define rtw_is_6g_band1(ch) ((ch) >= 1 && (ch) <= 93)
+#define rtw_is_6g_band2(ch) ((ch) >= 97 && (ch) <= 117)
+#define rtw_is_6g_band3(ch) ((ch) >= 121 && (ch) <= 189)
+#define rtw_is_6g_band4(ch) ((ch) >= 193 && (ch) <= 237)
+
+struct ch_list_t {
+	u8 *len_ch_attr;
+};
+
+#define CLA_2G_12_14_PASSIVE	BIT0
+
+#define CLA_5G_B1_PASSIVE		BIT0
+#define CLA_5G_B2_PASSIVE		BIT1
+#define CLA_5G_B3_PASSIVE		BIT2
+#define CLA_5G_B4_PASSIVE		BIT3
+#define CLA_5G_B2_DFS			BIT4
+#define CLA_5G_B3_DFS			BIT5
+#define CLA_5G_B4_DFS			BIT6
+
+#define CLA_6G_B1_PASSIVE		BIT0
+#define CLA_6G_B2_PASSIVE		BIT1
+#define CLA_6G_B3_PASSIVE		BIT2
+#define CLA_6G_B4_PASSIVE		BIT3
+
+#define CH_LIST_ENT(_len, arg...) \
+	{.len_ch_attr = (u8[_len + 2]) {_len, ##arg}, }
+
+#define CH_LIST_LEN(_ch_list) (_ch_list.len_ch_attr[0])
+#define CH_LIST_CH(_ch_list, _i) (_ch_list.len_ch_attr[_i + 1])
+#define CH_LIST_ATTRIB(_ch_list) (_ch_list.len_ch_attr[CH_LIST_LEN(_ch_list) + 1])
+
+enum rtw_chd_2g {
+	RTW_CHD_2G_INVALID = 0,
+
+	RTW_CHD_2G_00,
+	RTW_CHD_2G_01,
+	RTW_CHD_2G_02,
+	RTW_CHD_2G_03,
+	RTW_CHD_2G_04,
+	RTW_CHD_2G_05,
+	RTW_CHD_2G_06,
+
+	RTW_CHD_2G_MAX,
+	RTW_CHD_2G_NULL = RTW_CHD_2G_00,
+};
+
+enum rtw_chd_5g {
+	RTW_CHD_5G_INVALID = 0,
+
+	RTW_CHD_5G_00,
+	RTW_CHD_5G_01,
+	RTW_CHD_5G_02,
+	RTW_CHD_5G_03,
+	RTW_CHD_5G_04,
+	RTW_CHD_5G_05,
+	RTW_CHD_5G_06,
+	RTW_CHD_5G_07,
+	RTW_CHD_5G_08,
+	RTW_CHD_5G_09,
+	RTW_CHD_5G_10,
+	RTW_CHD_5G_11,
+	RTW_CHD_5G_12,
+	RTW_CHD_5G_13,
+	RTW_CHD_5G_14,
+	RTW_CHD_5G_15,
+	RTW_CHD_5G_16,
+	RTW_CHD_5G_17,
+	RTW_CHD_5G_18,
+	RTW_CHD_5G_19,
+	RTW_CHD_5G_20,
+	RTW_CHD_5G_21,
+	RTW_CHD_5G_22,
+	RTW_CHD_5G_23,
+	RTW_CHD_5G_24,
+	RTW_CHD_5G_25,
+	RTW_CHD_5G_26,
+	RTW_CHD_5G_27,
+	RTW_CHD_5G_28,
+	RTW_CHD_5G_29,
+	RTW_CHD_5G_30,
+	RTW_CHD_5G_31,
+	RTW_CHD_5G_32,
+	RTW_CHD_5G_33,
+	RTW_CHD_5G_34,
+	RTW_CHD_5G_35,
+	RTW_CHD_5G_36,
+	RTW_CHD_5G_37,
+	RTW_CHD_5G_38,
+	RTW_CHD_5G_39,
+	RTW_CHD_5G_40,
+	RTW_CHD_5G_41,
+	RTW_CHD_5G_42,
+	RTW_CHD_5G_43,
+	RTW_CHD_5G_44,
+	RTW_CHD_5G_45,
+	RTW_CHD_5G_46,
+	RTW_CHD_5G_47,
+	RTW_CHD_5G_48,
+	RTW_CHD_5G_49,
+	RTW_CHD_5G_50,
+	RTW_CHD_5G_51,
+	RTW_CHD_5G_52,
+	RTW_CHD_5G_53,
+	RTW_CHD_5G_54,
+	RTW_CHD_5G_55,
+	RTW_CHD_5G_56,
+	RTW_CHD_5G_57,
+	RTW_CHD_5G_58,
+	RTW_CHD_5G_59,
+	RTW_CHD_5G_60,
+	RTW_CHD_5G_61,
+
+	RTW_CHD_5G_MAX,
+	RTW_CHD_5G_NULL = RTW_CHD_5G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_2g[] = {
+	/* RTW_CHD_2G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_2G_01 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, CLA_2G_12_14_PASSIVE),
+	/* RTW_CHD_2G_02 */	CH_LIST_ENT(13, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_03 */	CH_LIST_ENT(11, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0),
+	/* RTW_CHD_2G_04 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0),
+	/* RTW_CHD_2G_05 */	CH_LIST_ENT(4, 10, 11, 12, 13, 0),
+	/* RTW_CHD_2G_06 */	CH_LIST_ENT(14, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, CLA_2G_12_14_PASSIVE),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static const struct ch_list_t rtw_channel_def_5g[] = {
+	/* RTW_CHD_5G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_5G_01 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_02 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_03 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_04 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_05 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_06 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_07 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_08 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_09 */	CH_LIST_ENT(5, 149, 153, 157, 161, 165, 0),
+	/* RTW_CHD_5G_10 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_11 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_12 */	CH_LIST_ENT(16, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_13 */	CH_LIST_ENT(8, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_14 */	CH_LIST_ENT(4, 36, 40, 44, 48, 0),
+	/* RTW_CHD_5G_15 */	CH_LIST_ENT(4, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_16 */	CH_LIST_ENT(11, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 0),
+	/* RTW_CHD_5G_17 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_18 */	CH_LIST_ENT(17, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_19 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_20 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_21 */	CH_LIST_ENT(11, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_22 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_23 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_24 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_25 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_26 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_27 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_28 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_29 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_30 */	CH_LIST_ENT(9, 36, 40, 44, 48, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_31 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_32 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_33 */	CH_LIST_ENT(22, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_34 */	CH_LIST_ENT(13, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_35 */	CH_LIST_ENT(8, 100, 104, 108, 112, 116, 132, 136, 140, CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_36 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_37 */	CH_LIST_ENT(8, 36, 40, 44, 48, 52, 56, 60, 64, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_38 */	CH_LIST_ENT(16, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_39 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_DFS),
+	/* RTW_CHD_5G_40 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_41 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_42 */	CH_LIST_ENT(24, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_43 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_44 */	CH_LIST_ENT(21, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_45 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_46 */	CH_LIST_ENT(12, 36, 40, 44, 48, 52, 56, 60, 64, 149, 153, 157, 161, CLA_5G_B2_PASSIVE),
+	/* RTW_CHD_5G_47 */	CH_LIST_ENT(19, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE),
+	/* RTW_CHD_5G_48 */	CH_LIST_ENT(20, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_49 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_50 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_51 */	CH_LIST_ENT(13, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_52 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_53 */	CH_LIST_ENT(17, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 149, 153, 157, 161, 165, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_54 */	CH_LIST_ENT(8, 36, 40, 44, 48, 149, 153, 157, 161, 0),
+	/* RTW_CHD_5G_55 */	CH_LIST_ENT(28, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B2_DFS | CLA_5G_B3_PASSIVE | CLA_5G_B3_DFS | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_56 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_57 */	CH_LIST_ENT(25, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_58 */	CH_LIST_ENT(16, 36, 40, 44, 48, 52, 56, 60, 64, 132, 136, 140, 149, 153, 157, 161, 165, CLA_5G_B1_PASSIVE | CLA_5G_B2_PASSIVE | CLA_5G_B3_PASSIVE | CLA_5G_B4_PASSIVE),
+	/* RTW_CHD_5G_59 */	CH_LIST_ENT(9, 52, 56, 60, 64, 149, 153, 157, 161, 165, CLA_5G_B2_DFS),
+	/* RTW_CHD_5G_60 */	CH_LIST_ENT(26, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+	/* RTW_CHD_5G_61 */	CH_LIST_ENT(23, 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, 169, 173, CLA_5G_B2_DFS | CLA_5G_B3_DFS),
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+struct chplan_ent_t {
+	u8 regd_2g; /* value of enum rtw_regd */
+	u8 chd_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 regd_5g; /* value of enum rtw_regd */
+	u8 chd_5g;
+#endif
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g, .regd_5g = RTW_REGD_##_regd_5g, .chd_5g = RTW_CHD_5G_##_chd_5g}
+#else
+#define CHPLAN_ENT(_regd_2g, _chd_2g, _regd_5g, _chd_5g) {.regd_2g = RTW_REGD_##_regd_2g, .chd_2g = RTW_CHD_2G_##_chd_2g}
+#endif
+
+#define CHPLAN_ENT_NOT_DEFINED CHPLAN_ENT(NA, INVALID, NA, INVALID)
+
+static const struct chplan_ent_t RTW_ChannelPlanMap[] = {
+	[0x00] = CHPLAN_ENT(ETSI , 02, ETSI , 49),
+	[0x01] = CHPLAN_ENT(ETSI , 02, ETSI , 50),
+	[0x02] = CHPLAN_ENT(ETSI , 03, ETSI , 07),
+	[0x03] = CHPLAN_ENT(ACMA , 02, ACMA , 33),
+	[0x04] = CHPLAN_ENT(ETSI , 02, ETSI , 51),
+	[0x05] = CHPLAN_ENT(ETSI , 02, ETSI , 06),
+	[0x06] = CHPLAN_ENT(ETSI , 02, ETSI , 07),
+	[0x07] = CHPLAN_ENT(ETSI , 02, ETSI , 23),
+	[0x08] = CHPLAN_ENT(ETSI , 02, ETSI , 21),
+	[0x09] = CHPLAN_ENT(ETSI , 02, ETSI , 17),
+	[0x0A] = CHPLAN_ENT(NA   , 00, NA   , 00),
+	[0x0B] = CHPLAN_ENT(ETSI , 02, ETSI , 22),
+	[0x0C] = CHPLAN_ENT(FCC  , 03, FCC  , 54),
+	[0x0D] = CHPLAN_ENT(MKK  , 04, MKK  , 14),
+	[0x0E] = CHPLAN_ENT(ETSI , 01, ETSI , 57),
+	[0x0F] = CHPLAN_ENT(ETSI , 01, ETSI , 58),
+	[0x10] = CHPLAN_ENT(IC   , 02, IC   , 56),
+	[0x11] = CHPLAN_ENT(FCC  , 02, FCC  , 59),
+	[0x12] = CHPLAN_ENT(ETSI , 02, ETSI , 60),
+	[0x13] = CHPLAN_ENT(ETSI , 02, ETSI , 61),
+	[0x1B] = CHPLAN_ENT(FCC  , 02, FCC  , 52),
+	[0x1C] = CHPLAN_ENT(KCC  , 02, KCC  , 53),
+	[0x20] = CHPLAN_ENT(WW   , 01, NA   , 00),
+	[0x21] = CHPLAN_ENT(ETSI , 02, NA   , 00),
+	[0x22] = CHPLAN_ENT(FCC  , 03, NA   , 00),
+	[0x23] = CHPLAN_ENT(MKK  , 04, NA   , 00),
+	[0x24] = CHPLAN_ENT(ETSI , 05, NA   , 00),
+	[0x25] = CHPLAN_ENT(FCC  , 03, FCC  , 03),
+	[0x26] = CHPLAN_ENT(ETSI , 01, ETSI , 02),
+	[0x27] = CHPLAN_ENT(MKK  , 04, MKK  , 02),
+	[0x28] = CHPLAN_ENT(KCC  , 01, KCC  , 05),
+	[0x29] = CHPLAN_ENT(FCC  , 01, FCC  , 06),
+	[0x2A] = CHPLAN_ENT(FCC  , 02, NA   , 00),
+	[0x2B] = CHPLAN_ENT(IC   , 02, IC   , 33),
+	[0x2C] = CHPLAN_ENT(MKK  , 02, NA   , 00),
+	[0x2D] = CHPLAN_ENT(CHILE, 01, CHILE, 22),
+	[0x2E] = CHPLAN_ENT(WW   , 03, WW   , 37),
+	[0x2F] = CHPLAN_ENT(CHILE, 01, CHILE, 38),
+	[0x30] = CHPLAN_ENT(FCC  , 01, FCC  , 07),
+	[0x31] = CHPLAN_ENT(FCC  , 01, FCC  , 08),
+	[0x32] = CHPLAN_ENT(FCC  , 01, FCC  , 09),
+	[0x33] = CHPLAN_ENT(FCC  , 01, FCC  , 10),
+	[0x34] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x35] = CHPLAN_ENT(ETSI , 01, ETSI , 03),
+	[0x36] = CHPLAN_ENT(ETSI , 01, ETSI , 04),
+	[0x37] = CHPLAN_ENT(MKK  , 04, MKK  , 10),
+	[0x38] = CHPLAN_ENT(MKK  , 04, MKK  , 11),
+	[0x39] = CHPLAN_ENT(NCC  , 03, NCC  , 12),
+	[0x3A] = CHPLAN_ENT(ETSI , 02, ETSI , 02),
+	[0x3B] = CHPLAN_ENT(ACMA , 02, ACMA , 01),
+	[0x3C] = CHPLAN_ENT(ETSI , 02, ETSI , 10),
+	[0x3D] = CHPLAN_ENT(ETSI , 02, ETSI , 15),
+	[0x3E] = CHPLAN_ENT(KCC  , 02, KCC  , 03),
+	[0x3F] = CHPLAN_ENT(FCC  , 03, FCC  , 22),
+	[0x40] = CHPLAN_ENT(NCC  , 03, NCC  , 13),
+	[0x41] = CHPLAN_ENT(WW   , 06, NA   , 00),
+	[0x42] = CHPLAN_ENT(ETSI , 02, ETSI , 14),
+	[0x43] = CHPLAN_ENT(FCC  , 03, FCC  , 06),
+	[0x44] = CHPLAN_ENT(NCC  , 03, NCC  , 09),
+	[0x45] = CHPLAN_ENT(ACMA , 01, ACMA , 01),
+	[0x46] = CHPLAN_ENT(FCC  , 03, FCC  , 15),
+	[0x47] = CHPLAN_ENT(ETSI , 01, ETSI , 10),
+	[0x48] = CHPLAN_ENT(ETSI , 01, ETSI , 07),
+	[0x49] = CHPLAN_ENT(ETSI , 01, ETSI , 06),
+	[0x4A] = CHPLAN_ENT(IC   , 03, IC   , 33),
+	[0x4B] = CHPLAN_ENT(KCC  , 02, KCC  , 22),
+	[0x4C] = CHPLAN_ENT(FCC  , 03, FCC  , 28),
+	[0x4D] = CHPLAN_ENT(MEX  , 02, MEX  , 01),
+	[0x4E] = CHPLAN_ENT(ETSI , 02, ETSI , 42),
+	[0x4F] = CHPLAN_ENT(NA   , 00, MKK  , 43),
+	[0x50] = CHPLAN_ENT(ETSI , 01, ETSI , 16),
+	[0x51] = CHPLAN_ENT(ETSI , 01, ETSI , 09),
+	[0x52] = CHPLAN_ENT(ETSI , 01, ETSI , 17),
+	[0x53] = CHPLAN_ENT(NCC  , 03, NCC  , 18),
+	[0x54] = CHPLAN_ENT(ETSI , 01, ETSI , 15),
+	[0x55] = CHPLAN_ENT(FCC  , 03, FCC  , 01),
+	[0x56] = CHPLAN_ENT(ETSI , 01, ETSI , 19),
+	[0x57] = CHPLAN_ENT(FCC  , 03, FCC  , 20),
+	[0x58] = CHPLAN_ENT(MKK  , 02, MKK  , 14),
+	[0x59] = CHPLAN_ENT(ETSI , 01, ETSI , 21),
+	[0x5A] = CHPLAN_ENT(NA   , 00, FCC  , 44),
+	[0x5B] = CHPLAN_ENT(NA   , 00, FCC  , 45),
+	[0x5C] = CHPLAN_ENT(NA   , 00, FCC  , 43),
+	[0x5D] = CHPLAN_ENT(ETSI , 02, ETSI , 08),
+	[0x5E] = CHPLAN_ENT(ETSI , 02, ETSI , 03),
+	[0x5F] = CHPLAN_ENT(MKK  , 02, MKK  , 47),
+	[0x60] = CHPLAN_ENT(FCC  , 03, FCC  , 09),
+	[0x61] = CHPLAN_ENT(FCC  , 02, FCC  , 01),
+	[0x62] = CHPLAN_ENT(FCC  , 02, FCC  , 03),
+	[0x63] = CHPLAN_ENT(ETSI , 01, ETSI , 23),
+	[0x64] = CHPLAN_ENT(MKK  , 02, MKK  , 24),
+	[0x65] = CHPLAN_ENT(ETSI , 02, ETSI , 24),
+	[0x66] = CHPLAN_ENT(FCC  , 03, FCC  , 27),
+	[0x67] = CHPLAN_ENT(FCC  , 03, FCC  , 25),
+	[0x68] = CHPLAN_ENT(FCC  , 02, FCC  , 27),
+	[0x69] = CHPLAN_ENT(FCC  , 02, FCC  , 25),
+	[0x6A] = CHPLAN_ENT(ETSI , 02, ETSI , 25),
+	[0x6B] = CHPLAN_ENT(FCC  , 01, FCC  , 29),
+	[0x6C] = CHPLAN_ENT(FCC  , 01, FCC  , 26),
+	[0x6D] = CHPLAN_ENT(FCC  , 02, FCC  , 28),
+	[0x6E] = CHPLAN_ENT(FCC  , 01, FCC  , 25),
+	[0x6F] = CHPLAN_ENT(NA   , 00, ETSI , 06),
+	[0x70] = CHPLAN_ENT(NA   , 00, ETSI , 30),
+	[0x71] = CHPLAN_ENT(NA   , 00, ETSI , 25),
+	[0x72] = CHPLAN_ENT(NA   , 00, ETSI , 31),
+	[0x73] = CHPLAN_ENT(FCC  , 01, FCC  , 01),
+	[0x74] = CHPLAN_ENT(FCC  , 02, FCC  , 19),
+	[0x75] = CHPLAN_ENT(ETSI , 01, ETSI , 32),
+	[0x76] = CHPLAN_ENT(FCC  , 02, FCC  , 22),
+	[0x77] = CHPLAN_ENT(ETSI , 01, ETSI , 34),
+	[0x78] = CHPLAN_ENT(FCC  , 03, FCC  , 35),
+	[0x79] = CHPLAN_ENT(MKK  , 02, MKK  , 02),
+	[0x7A] = CHPLAN_ENT(ETSI , 02, ETSI , 28),
+	[0x7B] = CHPLAN_ENT(ETSI , 02, ETSI , 46),
+	[0x7C] = CHPLAN_ENT(ETSI , 02, ETSI , 47),
+	[0x7D] = CHPLAN_ENT(MKK  , 04, MKK  , 48),
+	[0x7E] = CHPLAN_ENT(MKK  , 02, MKK  , 48),
+	[0x7F] = CHPLAN_ENT(WW   , 01, WW   , 55),
+};
+
+static const int RTW_ChannelPlanMap_size = sizeof(RTW_ChannelPlanMap) / sizeof(RTW_ChannelPlanMap[0]);
+
+static u8 rtk_regdb_get_default_regd_2g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_2g;
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtk_regdb_get_default_regd_5g(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size)
+		return RTW_ChannelPlanMap[id].regd_5g;
+	return RTW_REGD_NA;
+}
+#endif
+
+static bool rtk_regdb_is_domain_code_valid(u8 id)
+{
+	if (id < RTW_ChannelPlanMap_size) {
+		const struct chplan_ent_t *chplan_map = &RTW_ChannelPlanMap[id];
+
+		if (chplan_map->chd_2g != RTW_CHD_2G_INVALID
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			&& chplan_map->chd_5g != RTW_CHD_5G_INVALID
+			#endif
+		)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+
+	if (flags)
+		*flags = 0;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (ch > 14) {
+		u8 chd_5g = RTW_ChannelPlanMap[id].chd_5g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_5g[chd_5g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_5g[chd_5g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_5g[chd_5g], index) == ch) {
+				if (flags) {
+					if ((rtw_is_5g_band1(ch) && (attrib & CLA_5G_B1_PASSIVE)) /* band1 passive */
+						|| (rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_PASSIVE)) /* band2 passive */
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_PASSIVE)) /* band3 passive */
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_PASSIVE)) /* band4 passive */
+					)
+						*flags |= RTW_CHF_NO_IR;
+
+					if ((rtw_is_5g_band2(ch) && (attrib & CLA_5G_B2_DFS))
+						|| (rtw_is_5g_band3(ch) && (attrib & CLA_5G_B3_DFS))
+						|| (rtw_is_5g_band4(ch) && (attrib & CLA_5G_B4_DFS)))
+						*flags |= RTW_CHF_DFS;
+				}
+				return true;
+			}
+		}
+	} else
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+	 {
+		u8 chd_2g = RTW_ChannelPlanMap[id].chd_2g;
+
+		attrib = CH_LIST_ATTRIB(rtw_channel_def_2g[chd_2g]);
+
+		for (index = 0; index < CH_LIST_LEN(rtw_channel_def_2g[chd_2g]); index++) {
+			if (CH_LIST_CH(rtw_channel_def_2g[chd_2g], index) == ch) {
+				if (flags) {
+					if (ch >= 12 && ch <= 14 && (attrib & CLA_2G_12_14_PASSIVE))
+						*flags |= RTW_CHF_NO_IR;
+				}
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum rtw_chd_6g {
+	RTW_CHD_6G_INVALID = 0,
+
+	RTW_CHD_6G_00,
+	RTW_CHD_6G_01,
+	RTW_CHD_6G_02,
+	RTW_CHD_6G_03,
+	RTW_CHD_6G_04,
+	RTW_CHD_6G_05,	/* 6G Worldwide */
+	RTW_CHD_6G_06,
+
+	RTW_CHD_6G_MAX,
+	RTW_CHD_6G_NULL = RTW_CHD_6G_00,
+};
+
+static const struct ch_list_t rtw_channel_def_6g[] = {
+	/* RTW_CHD_6G_INVALID */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_00 */	CH_LIST_ENT(0, 0),
+	/* RTW_CHD_6G_01 */	CH_LIST_ENT(24, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 0),
+	/* RTW_CHD_6G_02 */	CH_LIST_ENT(6, 97, 101, 105, 109, 113, 117, 0),
+	/* RTW_CHD_6G_03 */	CH_LIST_ENT(18, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 0),
+	/* RTW_CHD_6G_04 */	CH_LIST_ENT(11, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+	/* RTW_CHD_6G_05 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, CLA_6G_B1_PASSIVE | CLA_6G_B2_PASSIVE | CLA_6G_B3_PASSIVE | CLA_6G_B4_PASSIVE),
+	/* RTW_CHD_6G_06 */	CH_LIST_ENT(59, 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233, 0),
+};
+
+struct chplan_6g_ent_t {
+	u8 regd; /* value of enum rtw_regd */
+	u8 chd;
+};
+
+#define CHPLAN_6G_ENT(_regd, _chd) {.regd = RTW_REGD_##_regd, .chd = RTW_CHD_6G_##_chd}
+
+#define CHPLAN_6G_ENT_NOT_DEFINED CHPLAN_6G_ENT(NA, INVALID)
+
+static const struct chplan_6g_ent_t rtw_chplan_6g_map[] = {
+	[0x00] = CHPLAN_6G_ENT(NA   , 00),
+	[0x01] = CHPLAN_6G_ENT(FCC  , 01),
+	[0x02] = CHPLAN_6G_ENT(FCC  , 02),
+	[0x03] = CHPLAN_6G_ENT(FCC  , 03),
+	[0x04] = CHPLAN_6G_ENT(FCC  , 04),
+	[0x05] = CHPLAN_6G_ENT(FCC  , 06),
+	[0x06] = CHPLAN_6G_ENT(ETSI , 01),
+	[0x07] = CHPLAN_6G_ENT(IC   , 06),
+	[0x08] = CHPLAN_6G_ENT(KCC  , 06),
+	[0x09] = CHPLAN_6G_ENT(KCC  , 01),
+	[0x1B] = CHPLAN_6G_ENT(ACMA , 01),
+	[0x1C] = CHPLAN_6G_ENT(MKK  , 01),
+	[0x7F] = CHPLAN_6G_ENT(WW   , 05),
+};
+
+static const int rtw_chplan_6g_map_size = sizeof(rtw_chplan_6g_map) / sizeof(rtw_chplan_6g_map[0]);
+
+static u8 rtk_regdb_get_default_regd_6g(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size)
+		return rtw_chplan_6g_map[id].regd;
+	return RTW_REGD_NA;
+}
+
+static bool rtk_regdb_is_domain_code_6g_valid(u8 id)
+{
+	if (id < rtw_chplan_6g_map_size) {
+		const struct chplan_6g_ent_t *chplan_map = &rtw_chplan_6g_map[id];
+
+		if (chplan_map->chd != RTW_CHD_6G_INVALID)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtk_regdb_domain_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	u8 index, attrib;
+	u8 chd_6g;
+
+	if (flags)
+		*flags = 0;
+
+	chd_6g = rtw_chplan_6g_map[id].chd;
+
+	attrib = CH_LIST_ATTRIB(rtw_channel_def_6g[chd_6g]);
+
+	for (index = 0; index < CH_LIST_LEN(rtw_channel_def_6g[chd_6g]); index++) {
+		if (CH_LIST_CH(rtw_channel_def_6g[chd_6g], index) == ch) {
+			if (flags) {
+				if ((rtw_is_6g_band1(ch) && (attrib & CLA_6G_B1_PASSIVE)) /* band1 passive */
+					|| (rtw_is_6g_band2(ch) && (attrib & CLA_6G_B2_PASSIVE)) /* band2 passive */
+					|| (rtw_is_6g_band3(ch) && (attrib & CLA_6G_B3_PASSIVE)) /* band3 passive */
+					|| (rtw_is_6g_band4(ch) && (attrib & CLA_6G_B4_PASSIVE)) /* band4 passive */
+				)
+					*flags |= RTW_CHF_NO_IR;
+			}
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static const struct country_chplan country_chplan_map[] = {
+	COUNTRY_CHPLAN_ENT("AD", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Andorra */
+	COUNTRY_CHPLAN_ENT("AE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* United Arab Emirates */
+	COUNTRY_CHPLAN_ENT("AF", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Afghanistan */
+	COUNTRY_CHPLAN_ENT("AG", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Antigua & Barbuda */
+	COUNTRY_CHPLAN_ENT("AI", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Anguilla(UK) */
+	COUNTRY_CHPLAN_ENT("AL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Albania */
+	COUNTRY_CHPLAN_ENT("AM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Armenia */
+	COUNTRY_CHPLAN_ENT("AN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Netherlands Antilles */
+	COUNTRY_CHPLAN_ENT("AO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Angola */
+	COUNTRY_CHPLAN_ENT("AQ", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Antarctica */
+	COUNTRY_CHPLAN_ENT("AR", 0x4D, 0x05, DEF     , 0, 1, 1, 1, _I_), /* Argentina */
+	COUNTRY_CHPLAN_ENT("AS", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* American Samoa */
+	COUNTRY_CHPLAN_ENT("AT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Austria */
+	COUNTRY_CHPLAN_ENT("AU", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* Australia */
+	COUNTRY_CHPLAN_ENT("AW", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Aruba */
+	COUNTRY_CHPLAN_ENT("AZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Azerbaijan */
+	COUNTRY_CHPLAN_ENT("BA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, _I_), /* Bosnia & Herzegovina */
+	COUNTRY_CHPLAN_ENT("BB", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Barbados */
+	COUNTRY_CHPLAN_ENT("BD", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bangladesh */
+	COUNTRY_CHPLAN_ENT("BE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Belgium */
+	COUNTRY_CHPLAN_ENT("BF", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burkina Faso */
+	COUNTRY_CHPLAN_ENT("BG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Bulgaria */
+	COUNTRY_CHPLAN_ENT("BH", 0x06, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Bahrain */
+	COUNTRY_CHPLAN_ENT("BI", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Burundi */
+	COUNTRY_CHPLAN_ENT("BJ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Benin */
+	COUNTRY_CHPLAN_ENT("BM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bermuda (UK) */
+	COUNTRY_CHPLAN_ENT("BN", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Brunei */
+	COUNTRY_CHPLAN_ENT("BO", 0x11, 0x00, DEF     , 0, 1, 1, 1, _I_), /* Bolivia */
+	COUNTRY_CHPLAN_ENT("BR", 0x62, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Brazil */
+	COUNTRY_CHPLAN_ENT("BS", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Bahamas */
+	COUNTRY_CHPLAN_ENT("BT", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bhutan */
+	COUNTRY_CHPLAN_ENT("BV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Bouvet Island (Norway) */
+	COUNTRY_CHPLAN_ENT("BW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Botswana */
+	COUNTRY_CHPLAN_ENT("BY", 0x0A, 0x00, DEF     , 0, 0, 1, 1, ___), /* Belarus */
+	COUNTRY_CHPLAN_ENT("BZ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Belize */
+	COUNTRY_CHPLAN_ENT("CA", 0x10, 0x07, DEF     , 1, 1, 1, 1, SI_), /* Canada */
+	COUNTRY_CHPLAN_ENT("CC", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cocos (Keeling) Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("CD", 0x5E, 0x00, DEF     , 1, 1, 1, 1, ___), /* Congo, Republic of the */
+	COUNTRY_CHPLAN_ENT("CF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Central African Republic */
+	COUNTRY_CHPLAN_ENT("CG", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Congo, Democratic Republic of the. Zaire */
+	COUNTRY_CHPLAN_ENT("CH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Switzerland */
+	COUNTRY_CHPLAN_ENT("CI", 0x42, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cote d'Ivoire */
+	COUNTRY_CHPLAN_ENT("CK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cook Islands */
+	COUNTRY_CHPLAN_ENT("CL", 0x76, 0x01, CHILE   , 1, 1, 1, 1, _I_), /* Chile */
+	COUNTRY_CHPLAN_ENT("CM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cameroon */
+	COUNTRY_CHPLAN_ENT("CN", 0x06, 0x00, CN      , 1, 1, 1, 1, ___), /* China */
+	COUNTRY_CHPLAN_ENT("CO", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Colombia */
+	COUNTRY_CHPLAN_ENT("CR", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Costa Rica */
+	COUNTRY_CHPLAN_ENT("CV", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cape Verde */
+	COUNTRY_CHPLAN_ENT("CX", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Christmas Island (Australia) */
+	COUNTRY_CHPLAN_ENT("CY", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Cyprus */
+	COUNTRY_CHPLAN_ENT("CZ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Czech Republic */
+	COUNTRY_CHPLAN_ENT("DE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Germany */
+	COUNTRY_CHPLAN_ENT("DJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Djibouti */
+	COUNTRY_CHPLAN_ENT("DK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Denmark */
+	COUNTRY_CHPLAN_ENT("DM", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Dominica */
+	COUNTRY_CHPLAN_ENT("DO", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Dominican Republic */
+	COUNTRY_CHPLAN_ENT("DZ", 0x00, 0x00, DEF     , 0, 1, 1, 1, ___), /* Algeria */
+	COUNTRY_CHPLAN_ENT("EC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ecuador */
+	COUNTRY_CHPLAN_ENT("EE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Estonia */
+	COUNTRY_CHPLAN_ENT("EG", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Egypt */
+	COUNTRY_CHPLAN_ENT("EH", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Western Sahara */
+	COUNTRY_CHPLAN_ENT("ER", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Eritrea */
+	COUNTRY_CHPLAN_ENT("ES", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Spain, Canary Islands, Ceuta, Melilla */
+	COUNTRY_CHPLAN_ENT("ET", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Ethiopia */
+	COUNTRY_CHPLAN_ENT("FI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Finland */
+	COUNTRY_CHPLAN_ENT("FJ", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Fiji */
+	COUNTRY_CHPLAN_ENT("FK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Falkland Islands (Islas Malvinas) (UK) */
+	COUNTRY_CHPLAN_ENT("FM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Micronesia, Federated States of (USA) */
+	COUNTRY_CHPLAN_ENT("FO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Faroe Islands (Denmark) */
+	COUNTRY_CHPLAN_ENT("FR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* France */
+	COUNTRY_CHPLAN_ENT("GA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Gabon */
+	COUNTRY_CHPLAN_ENT("GB", 0x0B, 0x06, UK      , 1, 1, 1, 1, _IV), /* Great Britain (United Kingdom; England) */
+	COUNTRY_CHPLAN_ENT("GD", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Grenada */
+	COUNTRY_CHPLAN_ENT("GE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Georgia */
+	COUNTRY_CHPLAN_ENT("GF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Guiana */
+	COUNTRY_CHPLAN_ENT("GG", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guernsey (UK) */
+	COUNTRY_CHPLAN_ENT("GH", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Ghana */
+	COUNTRY_CHPLAN_ENT("GI", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Gibraltar (UK) */
+	COUNTRY_CHPLAN_ENT("GL", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Greenland (Denmark) */
+	COUNTRY_CHPLAN_ENT("GM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Gambia */
+	COUNTRY_CHPLAN_ENT("GN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Guinea */
+	COUNTRY_CHPLAN_ENT("GP", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guadeloupe (France) */
+	COUNTRY_CHPLAN_ENT("GQ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Equatorial Guinea */
+	COUNTRY_CHPLAN_ENT("GR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Greece */
+	COUNTRY_CHPLAN_ENT("GS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* South Georgia and the Sandwich Islands (UK) */
+	COUNTRY_CHPLAN_ENT("GT", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Guatemala */
+	COUNTRY_CHPLAN_ENT("GU", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Guam (USA) */
+	COUNTRY_CHPLAN_ENT("GW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guinea-Bissau */
+	COUNTRY_CHPLAN_ENT("GY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Guyana */
+	COUNTRY_CHPLAN_ENT("HK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hong Kong */
+	COUNTRY_CHPLAN_ENT("HM", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Heard and McDonald Islands (Australia) */
+	COUNTRY_CHPLAN_ENT("HN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Honduras */
+	COUNTRY_CHPLAN_ENT("HR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Croatia */
+	COUNTRY_CHPLAN_ENT("HT", 0x76, 0x01, DEF     , 1, 0, 1, 1, _I_), /* Haiti */
+	COUNTRY_CHPLAN_ENT("HU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Hungary */
+	COUNTRY_CHPLAN_ENT("ID", 0x5D, 0x00, DEF     , 0, 1, 1, 1, ___), /* Indonesia */
+	COUNTRY_CHPLAN_ENT("IE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Ireland */
+	COUNTRY_CHPLAN_ENT("IL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Israel */
+	COUNTRY_CHPLAN_ENT("IM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Isle of Man (UK) */
+	COUNTRY_CHPLAN_ENT("IN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* India */
+	COUNTRY_CHPLAN_ENT("IO", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* British Indian Ocean Territory (UK) */
+	COUNTRY_CHPLAN_ENT("IQ", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Iraq */
+	COUNTRY_CHPLAN_ENT("IR", 0x0A, 0x00, DEF     , 0, 0, 0, 0, ___), /* Iran */
+	COUNTRY_CHPLAN_ENT("IS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Iceland */
+	COUNTRY_CHPLAN_ENT("IT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Italy */
+	COUNTRY_CHPLAN_ENT("JE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Jersey (UK) */
+	COUNTRY_CHPLAN_ENT("JM", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Jamaica */
+	COUNTRY_CHPLAN_ENT("JO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Jordan */
+	COUNTRY_CHPLAN_ENT("JP", 0x7D, 0x1C, DEF     , 1, 1, 1, 1, _IV), /* Japan- Telec */
+	COUNTRY_CHPLAN_ENT("KE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kenya */
+	COUNTRY_CHPLAN_ENT("KG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kyrgyzstan */
+	COUNTRY_CHPLAN_ENT("KH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cambodia */
+	COUNTRY_CHPLAN_ENT("KI", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kiribati */
+	COUNTRY_CHPLAN_ENT("KM", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Comoros */
+	COUNTRY_CHPLAN_ENT("KN", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Kitts and Nevis */
+	COUNTRY_CHPLAN_ENT("KR", 0x4B, 0x08, DEF     , 0, 1, 1, 1, _IV), /* South Korea */
+	COUNTRY_CHPLAN_ENT("KW", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kuwait */
+	COUNTRY_CHPLAN_ENT("KY", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Cayman Islands (UK) */
+	COUNTRY_CHPLAN_ENT("KZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Kazakhstan */
+	COUNTRY_CHPLAN_ENT("LA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Laos */
+	COUNTRY_CHPLAN_ENT("LB", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Lebanon */
+	COUNTRY_CHPLAN_ENT("LC", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Lucia */
+	COUNTRY_CHPLAN_ENT("LI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Liechtenstein */
+	COUNTRY_CHPLAN_ENT("LK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sri Lanka */
+	COUNTRY_CHPLAN_ENT("LR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Liberia */
+	COUNTRY_CHPLAN_ENT("LS", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Lesotho */
+	COUNTRY_CHPLAN_ENT("LT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Lithuania */
+	COUNTRY_CHPLAN_ENT("LU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Luxembourg */
+	COUNTRY_CHPLAN_ENT("LV", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Latvia */
+	COUNTRY_CHPLAN_ENT("LY", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Libya */
+	COUNTRY_CHPLAN_ENT("MA", 0x3C, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Morocco */
+	COUNTRY_CHPLAN_ENT("MC", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Monaco */
+	COUNTRY_CHPLAN_ENT("MD", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Moldova */
+	COUNTRY_CHPLAN_ENT("ME", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montenegro */
+	COUNTRY_CHPLAN_ENT("MF", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Saint Martin */
+	COUNTRY_CHPLAN_ENT("MG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Madagascar */
+	COUNTRY_CHPLAN_ENT("MH", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Marshall Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Republic of Macedonia (FYROM) */
+	COUNTRY_CHPLAN_ENT("ML", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mali */
+	COUNTRY_CHPLAN_ENT("MM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Burma (Myanmar) */
+	COUNTRY_CHPLAN_ENT("MN", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mongolia */
+	COUNTRY_CHPLAN_ENT("MO", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Macau */
+	COUNTRY_CHPLAN_ENT("MP", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Northern Mariana Islands (USA) */
+	COUNTRY_CHPLAN_ENT("MQ", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Martinique (France) */
+	COUNTRY_CHPLAN_ENT("MR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mauritania */
+	COUNTRY_CHPLAN_ENT("MS", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Montserrat (UK) */
+	COUNTRY_CHPLAN_ENT("MT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Malta */
+	COUNTRY_CHPLAN_ENT("MU", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Mauritius */
+	COUNTRY_CHPLAN_ENT("MV", 0x3C, 0x00, DEF     , 0, 1, 1, 1, ___), /* Maldives */
+	COUNTRY_CHPLAN_ENT("MW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Malawi */
+	COUNTRY_CHPLAN_ENT("MX", 0x4D, 0x01, DEF     , 1, 1, 1, 1, _I_), /* Mexico */
+	COUNTRY_CHPLAN_ENT("MY", 0x07, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Malaysia */
+	COUNTRY_CHPLAN_ENT("MZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mozambique */
+	COUNTRY_CHPLAN_ENT("NA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Namibia */
+	COUNTRY_CHPLAN_ENT("NC", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* New Caledonia */
+	COUNTRY_CHPLAN_ENT("NE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niger */
+	COUNTRY_CHPLAN_ENT("NF", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Norfolk Island (Australia) */
+	COUNTRY_CHPLAN_ENT("NG", 0x75, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nigeria */
+	COUNTRY_CHPLAN_ENT("NI", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nicaragua */
+	COUNTRY_CHPLAN_ENT("NL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Netherlands */
+	COUNTRY_CHPLAN_ENT("NO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Norway */
+	COUNTRY_CHPLAN_ENT("NP", 0x06, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nepal */
+	COUNTRY_CHPLAN_ENT("NR", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Nauru */
+	COUNTRY_CHPLAN_ENT("NU", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Niue */
+	COUNTRY_CHPLAN_ENT("NZ", 0x03, 0x1B, DEF     , 1, 1, 1, 1, _IV), /* New Zealand */
+	COUNTRY_CHPLAN_ENT("OM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Oman */
+	COUNTRY_CHPLAN_ENT("PA", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Panama */
+	COUNTRY_CHPLAN_ENT("PE", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Peru */
+	COUNTRY_CHPLAN_ENT("PF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Polynesia (France) */
+	COUNTRY_CHPLAN_ENT("PG", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Papua New Guinea */
+	COUNTRY_CHPLAN_ENT("PH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Philippines */
+	COUNTRY_CHPLAN_ENT("PK", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Pakistan */
+	COUNTRY_CHPLAN_ENT("PL", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Poland */
+	COUNTRY_CHPLAN_ENT("PM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Pierre and Miquelon (France) */
+	COUNTRY_CHPLAN_ENT("PR", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Puerto Rico */
+	COUNTRY_CHPLAN_ENT("PS", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Palestine */
+	COUNTRY_CHPLAN_ENT("PT", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Portugal */
+	COUNTRY_CHPLAN_ENT("PW", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Palau */
+	COUNTRY_CHPLAN_ENT("PY", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Paraguay */
+	COUNTRY_CHPLAN_ENT("QA", 0x5E, 0x06, QATAR   , 1, 1, 1, 1, _IV), /* Qatar */
+	COUNTRY_CHPLAN_ENT("RE", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Reunion (France) */
+	COUNTRY_CHPLAN_ENT("RO", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Romania */
+	COUNTRY_CHPLAN_ENT("RS", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Serbia */
+	COUNTRY_CHPLAN_ENT("RU", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Russia(fac/gost), Kaliningrad */
+	COUNTRY_CHPLAN_ENT("RW", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Rwanda */
+	COUNTRY_CHPLAN_ENT("SA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Saudi Arabia */
+	COUNTRY_CHPLAN_ENT("SB", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Solomon Islands */
+	COUNTRY_CHPLAN_ENT("SC", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Seychelles */
+	COUNTRY_CHPLAN_ENT("SE", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Sweden */
+	COUNTRY_CHPLAN_ENT("SG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Singapore */
+	COUNTRY_CHPLAN_ENT("SH", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Helena (UK) */
+	COUNTRY_CHPLAN_ENT("SI", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovenia */
+	COUNTRY_CHPLAN_ENT("SJ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Svalbard (Norway) */
+	COUNTRY_CHPLAN_ENT("SK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Slovakia */
+	COUNTRY_CHPLAN_ENT("SL", 0x5E, 0x06, DEF     , 0, 1, 1, 1, _I_), /* Sierra Leone */
+	COUNTRY_CHPLAN_ENT("SM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* San Marino */
+	COUNTRY_CHPLAN_ENT("SN", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Senegal */
+	COUNTRY_CHPLAN_ENT("SO", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Somalia */
+	COUNTRY_CHPLAN_ENT("SR", 0x74, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Suriname */
+	COUNTRY_CHPLAN_ENT("ST", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Sao Tome and Principe */
+	COUNTRY_CHPLAN_ENT("SV", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* El Salvador */
+	COUNTRY_CHPLAN_ENT("SX", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* Sint Marteen */
+	COUNTRY_CHPLAN_ENT("SZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Swaziland */
+	COUNTRY_CHPLAN_ENT("TC", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turks and Caicos Islands (UK) */
+	COUNTRY_CHPLAN_ENT("TD", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Chad */
+	COUNTRY_CHPLAN_ENT("TF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* French Southern and Antarctic Lands (FR Southern Territories) */
+	COUNTRY_CHPLAN_ENT("TG", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Togo */
+	COUNTRY_CHPLAN_ENT("TH", 0x5E, 0x06, THAILAND, 1, 1, 1, 1, _IV), /* Thailand */
+	COUNTRY_CHPLAN_ENT("TJ", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Tajikistan */
+	COUNTRY_CHPLAN_ENT("TK", 0x03, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tokelau */
+	COUNTRY_CHPLAN_ENT("TM", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Turkmenistan */
+	COUNTRY_CHPLAN_ENT("TN", 0x04, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tunisia */
+	COUNTRY_CHPLAN_ENT("TO", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tonga */
+	COUNTRY_CHPLAN_ENT("TR", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Turkey, Northern Cyprus */
+	COUNTRY_CHPLAN_ENT("TT", 0x76, 0x00, DEF     , 0, 1, 1, 1, ___), /* Trinidad & Tobago */
+	COUNTRY_CHPLAN_ENT("TV", 0x21, 0x00, DEF     , 0, 0, 0, 0, ___), /* Tuvalu */
+	COUNTRY_CHPLAN_ENT("TW", 0x76, 0x06, DEF     , 1, 1, 1, 1, _IV), /* Taiwan */
+	COUNTRY_CHPLAN_ENT("TZ", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Tanzania */
+	COUNTRY_CHPLAN_ENT("UA", 0x5E, 0x00, UKRAINE , 0, 1, 1, 1, ___), /* Ukraine */
+	COUNTRY_CHPLAN_ENT("UG", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uganda */
+	COUNTRY_CHPLAN_ENT("US", 0x1B, 0x05, DEF     , 1, 1, 1, 1, SI_), /* United States of America (USA) */
+	COUNTRY_CHPLAN_ENT("UY", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Uruguay */
+	COUNTRY_CHPLAN_ENT("UZ", 0x3A, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Uzbekistan */
+	COUNTRY_CHPLAN_ENT("VA", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Holy See (Vatican City) */
+	COUNTRY_CHPLAN_ENT("VC", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Saint Vincent and the Grenadines */
+	COUNTRY_CHPLAN_ENT("VE", 0x30, 0x00, DEF     , 0, 1, 1, 1, ___), /* Venezuela */
+	COUNTRY_CHPLAN_ENT("VG", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* British Virgin Islands (UK) */
+	COUNTRY_CHPLAN_ENT("VI", 0x76, 0x05, DEF     , 1, 1, 1, 1, _I_), /* United States Virgin Islands (USA) */
+	COUNTRY_CHPLAN_ENT("VN", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vietnam */
+	COUNTRY_CHPLAN_ENT("VU", 0x26, 0x00, DEF     , 0, 1, 1, 1, ___), /* Vanuatu */
+	COUNTRY_CHPLAN_ENT("WF", 0x3A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Wallis and Futuna (France) */
+	COUNTRY_CHPLAN_ENT("WS", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Samoa */
+	COUNTRY_CHPLAN_ENT("XK", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* Kosovo */
+	COUNTRY_CHPLAN_ENT("YE", 0x0A, 0x00, DEF     , 0, 1, 1, 1, ___), /* Yemen */
+	COUNTRY_CHPLAN_ENT("YT", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Mayotte (France) */
+	COUNTRY_CHPLAN_ENT("ZA", 0x5E, 0x06, DEF     , 1, 1, 1, 1, _I_), /* South Africa */
+	COUNTRY_CHPLAN_ENT("ZM", 0x5E, 0x00, DEF     , 0, 1, 1, 1, ___), /* Zambia */
+	COUNTRY_CHPLAN_ENT("ZW", 0x5E, 0x00, DEF     , 1, 1, 1, 1, ___), /* Zimbabwe */
+};
+
+static bool rtk_regdb_get_chplan_from_alpha2(const char *alpha2, struct country_chplan *ent)
+{
+	const struct country_chplan *map = country_chplan_map;
+	u16 map_sz = sizeof(country_chplan_map) / sizeof(struct country_chplan);
+	int i;
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(alpha2, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+	return false;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static void rtk_regdb_dump_chplan_test(void *sel)
+{
+	int i, j;
+
+	/* check 2G CHD redundent */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = RTW_CHD_2G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_2g[i]) == CH_LIST_LEN(rtw_channel_def_2g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_2g[i], 0), &CH_LIST_CH(rtw_channel_def_2g[j], 0), CH_LIST_LEN(rtw_channel_def_2g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "2G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 2G CHD invalid channel */
+	for (i = RTW_CHD_2G_00; i < RTW_CHD_2G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_2g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_24G, CH_LIST_CH(rtw_channel_def_2g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "2G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_2g[i], j), i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* check 5G CHD redundent */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = RTW_CHD_5G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_5g[i]) == CH_LIST_LEN(rtw_channel_def_5g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_5g[i], 0), &CH_LIST_CH(rtw_channel_def_5g[j], 0), CH_LIST_LEN(rtw_channel_def_5g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "5G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 5G CHD invalid channel */
+	for (i = RTW_CHD_5G_00; i < RTW_CHD_5G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_5g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_5G, CH_LIST_CH(rtw_channel_def_5g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "5G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_5g[i], j), i, j);
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check 6G CHD redundent */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = RTW_CHD_6G_00; j < i; j++) {
+			if (CH_LIST_LEN(rtw_channel_def_6g[i]) == CH_LIST_LEN(rtw_channel_def_6g[j])
+				&& _rtw_memcmp(&CH_LIST_CH(rtw_channel_def_6g[i], 0), &CH_LIST_CH(rtw_channel_def_6g[j], 0), CH_LIST_LEN(rtw_channel_def_6g[i]) + 1) == _TRUE)
+				RTW_PRINT_SEL(sel, "6G chd:%u and %u is the same\n", i, j);
+		}
+	}
+
+	/* check 6G CHD invalid channel */
+	for (i = RTW_CHD_6G_00; i < RTW_CHD_6G_MAX; i++) {
+		for (j = 0; j < CH_LIST_LEN(rtw_channel_def_6g[i]); j++) {
+			if (rtw_bch2freq(BAND_ON_6G, CH_LIST_CH(rtw_channel_def_6g[i], j)) == 0)
+				RTW_PRINT_SEL(sel, "6G invalid ch:%u at (%d,%d)\n", CH_LIST_CH(rtw_channel_def_6g[i], j), i, j);
+		}
+	}
+#endif
+
+	/* check chplan 2G_5G redundent */
+	for (i = 0; i < RTW_ChannelPlanMap_size; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_valid(j))
+				continue;
+			if (_rtw_memcmp(&RTW_ChannelPlanMap[i], &RTW_ChannelPlanMap[j], sizeof(RTW_ChannelPlanMap[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* check chplan 6G redundent */
+	for (i = 0; i < rtw_chplan_6g_map_size; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+		for (j = 0; j < i; j++) {
+			if (!rtw_is_channel_plan_6g_valid(j))
+				continue;
+			if (_rtw_memcmp(&rtw_chplan_6g_map[i], &rtw_chplan_6g_map[j], sizeof(rtw_chplan_6g_map[i])) == _TRUE)
+				RTW_PRINT_SEL(sel, "channel plan 6g 0x%02x and 0x%02x is the same\n", i, j);
+		}
+	}
+#endif
+
+
+	/* check country invalid chplan/chplan_6g */
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+			if (!rtw_is_channel_plan_valid(ent.domain_code))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code:0x%02X\n", code[0], code[1], ent.domain_code);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (!rtw_is_channel_plan_6g_valid(ent.domain_code_6g))
+				RTW_PRINT_SEL(sel, "country \"%c%c\" has invalid domain_code_6g:0x%02X\n", code[0], code[1], ent.domain_code_6g);
+			#endif
+		}
+	}
+}
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtk_regdb_get_ver_str(char *buf, size_t buf_len)
+{
+	snprintf(buf, buf_len, "%s%s"
+		"%s%s-%s%s"
+		, RTW_MODULE_NAME, strlen(RTW_MODULE_NAME) ? "-" : ""
+		, RTW_DOMAIN_MAP_VER, RTW_DOMAIN_MAP_M_VER, RTW_COUNTRY_MAP_VER, RTW_COUNTRY_MAP_M_VER);
+}
+
+struct rtw_regdb_ops regdb_ops = {
+	.get_default_regd_2g = rtk_regdb_get_default_regd_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	.get_default_regd_5g = rtk_regdb_get_default_regd_5g,
+#endif
+	.is_domain_code_valid = rtk_regdb_is_domain_code_valid,
+	.domain_get_ch = rtk_regdb_domain_get_ch,
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	.get_default_regd_6g = rtk_regdb_get_default_regd_6g,
+	.is_domain_code_6g_valid = rtk_regdb_is_domain_code_6g_valid,
+	.domain_6g_get_ch = rtk_regdb_domain_6g_get_ch,
+#endif
+
+	.get_chplan_from_alpha2 = rtk_regdb_get_chplan_from_alpha2,
+
+#ifdef CONFIG_RTW_DEBUG
+	.dump_chplan_test = rtk_regdb_dump_chplan_test,
+#endif
+	.get_ver_str = rtk_regdb_get_ver_str,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ap.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ap.c
new file mode 100644
index 000000000000..abe1ce6b9e95
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ap.c
@@ -0,0 +1,7777 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_AP_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_AP_MODE
+
+extern unsigned char	RTW_WPA_OUI[];
+extern unsigned char	WMM_OUI[];
+extern unsigned char	WPS_OUI[];
+extern unsigned char	P2P_OUI[];
+extern unsigned char	WFD_OUI[];
+
+void init_mlme_ap_info(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	_rtw_spinlock_init(&pmlmepriv->bcn_update_lock);
+	/* pmlmeext->bstart_bss = _FALSE; */
+}
+
+void free_mlme_ap_info(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	stop_ap_mode(padapter);
+	_rtw_spinlock_free(&pmlmepriv->bcn_update_lock);
+
+}
+
+/*
+* Set TIM IE
+* return length of total TIM IE
+*/
+u8 rtw_set_tim_ie(u8 dtim_cnt, u8 dtim_period
+	, const u8 *tim_bmp, u8 tim_bmp_len, u8 *tim_ie)
+{
+	u8 *p = tim_ie;
+	u8 i, n1, n2;
+	u8 bmp_len;
+
+	if (rtw_bmp_not_empty(tim_bmp, tim_bmp_len)) {
+		/* find the first nonzero octet in tim_bitmap */
+		for (i = 0; i < tim_bmp_len; i++)
+			if (tim_bmp[i])
+				break;
+		n1 = i & 0xFE;
+	
+		/* find the last nonzero octet in tim_bitmap, except octet 0 */
+		for (i = tim_bmp_len - 1; i > 0; i--)
+			if (tim_bmp[i])
+				break;
+		n2 = i;
+		bmp_len = n2 - n1 + 1;
+	} else {
+		n1 = n2 = 0;
+		bmp_len = 1;
+	}
+
+	*p++ = WLAN_EID_TIM;
+	*p++ = 2 + 1 + bmp_len;
+	*p++ = dtim_cnt;
+	*p++ = dtim_period;
+	*p++ = (rtw_bmp_is_set(tim_bmp, tim_bmp_len, 0) ? BIT0 : 0) | n1;
+	_rtw_memcpy(p, tim_bmp + n1, bmp_len);
+
+#if 0
+	RTW_INFO("n1:%u, n2:%u, bmp_offset:%u, bmp_len:%u\n", n1, n2, n1 / 2, bmp_len);
+	RTW_INFO_DUMP("tim_ie: ", tim_ie + 2, 2 + 1 + bmp_len);
+#endif
+	return 2 + 2 + 1 + bmp_len;
+}
+
+static void update_BCNTIM(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *pnetwork_mlmeext = &(pmlmeinfo->network);
+	unsigned char *pie = pnetwork_mlmeext->IEs;
+
+#if 0
+
+
+	/* update TIM IE */
+	/* if(rtw_tim_map_anyone_be_set(padapter, pstapriv->tim_bitmap)) */
+#endif
+	if (_TRUE) {
+		u8 *p, *dst_ie, *premainder_ie = NULL, *pbackup_remainder_ie = NULL;
+		uint offset, tmp_len, tim_ielen, tim_ie_offset, remainder_ielen;
+
+		p = rtw_get_ie(pie + _FIXED_IE_LENGTH_, _TIM_IE_, &tim_ielen, pnetwork_mlmeext->IELength - _FIXED_IE_LENGTH_);
+		if (p != NULL && tim_ielen > 0) {
+			tim_ielen += 2;
+
+			premainder_ie = p + tim_ielen;
+
+			tim_ie_offset = (sint)(p - pie);
+
+			remainder_ielen = pnetwork_mlmeext->IELength - tim_ie_offset - tim_ielen;
+
+			/*append TIM IE from dst_ie offset*/
+			dst_ie = p;
+		} else {
+			tim_ielen = 0;
+
+			/*calculate head_len*/
+			offset = _FIXED_IE_LENGTH_;
+
+			/* get ssid_ie len */
+			p = rtw_get_ie(pie + _BEACON_IE_OFFSET_, _SSID_IE_, &tmp_len, (pnetwork_mlmeext->IELength - _BEACON_IE_OFFSET_));
+			if (p != NULL)
+				offset += tmp_len + 2;
+
+			/*get supported rates len*/
+			p = rtw_get_ie(pie + _BEACON_IE_OFFSET_, _SUPPORTEDRATES_IE_, &tmp_len, (pnetwork_mlmeext->IELength - _BEACON_IE_OFFSET_));
+			if (p !=  NULL)
+				offset += tmp_len + 2;
+
+			/* DS Parameter Set IE */
+			p = rtw_get_ie(pie + _BEACON_IE_OFFSET_, _DSSET_IE_, &tmp_len,
+				pnetwork_mlmeext->IELength - _BEACON_IE_OFFSET_);
+			if (p)
+				offset += tmp_len + 2;
+
+			premainder_ie = pie + offset;
+
+			remainder_ielen = pnetwork_mlmeext->IELength - offset - tim_ielen;
+
+			/*append TIM IE from offset*/
+			dst_ie = pie + offset;
+
+		}
+
+		if (remainder_ielen > 0) {
+			pbackup_remainder_ie = rtw_malloc(remainder_ielen);
+			if (pbackup_remainder_ie && premainder_ie)
+				_rtw_memcpy(pbackup_remainder_ie, premainder_ie, remainder_ielen);
+		}
+
+		/* append TIM IE */
+		dst_ie += rtw_set_tim_ie(0, 1, pstapriv->tim_bitmap, pstapriv->aid_bmp_len, dst_ie);
+
+		/*copy remainder IE*/
+		if (pbackup_remainder_ie) {
+			_rtw_memcpy(dst_ie, pbackup_remainder_ie, remainder_ielen);
+
+			rtw_mfree(pbackup_remainder_ie, remainder_ielen);
+		}
+
+		offset = (uint)(dst_ie - pie);
+		pnetwork_mlmeext->IELength = offset + remainder_ielen;
+
+	}
+}
+
+void rtw_add_bcn_ie(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 index, u8 *data, u8 len)
+{
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8	bmatch = _FALSE;
+	u8	*pie = pnetwork->IEs;
+	u8	*p = NULL, *dst_ie = NULL, *premainder_ie = NULL, *pbackup_remainder_ie = NULL;
+	u32	i, offset, ielen = 0, ie_offset, remainder_ielen = 0;
+
+	if (pnetwork->IELength + len > MAX_IE_SZ) {
+		RTW_ERR("Can't add IE to beacon, because size is more than MAX_IE_SZ\n");
+		return;
+	}
+
+	/* Search element id (index) exits or not */
+	for (i = _FIXED_IE_LENGTH_; i < pnetwork->IELength;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(pnetwork->IEs + i);
+
+		if (pIE->ElementID > index)
+			break;
+		else if (pIE->ElementID == index) { /* already exist the same IE */
+			p = (u8 *)pIE;
+			ielen = pIE->Length;
+			bmatch = _TRUE;
+			break;
+		}
+
+		p = (u8 *)pIE;
+		ielen = pIE->Length;
+		i += (pIE->Length + 2);
+	}
+
+	/* Backup remainder IE */
+	if (p != NULL && ielen > 0) {
+		ielen += 2;
+
+		premainder_ie = p + ielen;
+
+		ie_offset = (sint)(p - pie);
+
+		remainder_ielen = pnetwork->IELength - ie_offset - ielen;
+
+		if (bmatch)
+			dst_ie = p;
+		else
+			dst_ie = (p + ielen);
+	}
+
+	if (dst_ie == NULL)
+		return;
+
+	if (remainder_ielen > 0) {
+		pbackup_remainder_ie = rtw_malloc(remainder_ielen);
+		if (pbackup_remainder_ie && premainder_ie)
+			_rtw_memcpy(pbackup_remainder_ie, premainder_ie, remainder_ielen);
+	}
+
+	*dst_ie++ = index;
+	*dst_ie++ = len;
+
+	_rtw_memcpy(dst_ie, data, len);
+	dst_ie += len;
+
+	/* Append remainder IE */
+	if (pbackup_remainder_ie) {
+		_rtw_memcpy(dst_ie, pbackup_remainder_ie, remainder_ielen);
+
+		rtw_mfree(pbackup_remainder_ie, remainder_ielen);
+	}
+
+	offset = (uint)(dst_ie - pie);
+	pnetwork->IELength = offset + remainder_ielen;
+}
+
+void rtw_remove_bcn_ie(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 index)
+{
+	u8 *p, *dst_ie = NULL, *premainder_ie = NULL, *pbackup_remainder_ie = NULL;
+	uint offset, ielen, ie_offset, remainder_ielen = 0;
+	u8	*pie = pnetwork->IEs;
+
+	p = rtw_get_ie(pie + _FIXED_IE_LENGTH_, index, &ielen, pnetwork->IELength - _FIXED_IE_LENGTH_);
+	if (p != NULL && ielen > 0) {
+		ielen += 2;
+
+		premainder_ie = p + ielen;
+
+		ie_offset = (sint)(p - pie);
+
+		remainder_ielen = pnetwork->IELength - ie_offset - ielen;
+
+		dst_ie = p;
+	} else
+		return;
+
+	if (remainder_ielen > 0) {
+		pbackup_remainder_ie = rtw_malloc(remainder_ielen);
+		if (pbackup_remainder_ie && premainder_ie)
+			_rtw_memcpy(pbackup_remainder_ie, premainder_ie, remainder_ielen);
+	}
+
+	/* copy remainder IE */
+	if (pbackup_remainder_ie) {
+		_rtw_memcpy(dst_ie, pbackup_remainder_ie, remainder_ielen);
+
+		rtw_mfree(pbackup_remainder_ie, remainder_ielen);
+	}
+
+	offset = (uint)(dst_ie - pie);
+	pnetwork->IELength = offset + remainder_ielen;
+}
+
+/**
+ * rtw_add_bcn_ie_ex - Add Element ID Extension into Element ID (0xff)
+ * Frame format: | element id (0xff) | length | element id extension | content |
+ *
+ * @eid_ex: element id extension
+ * @data: eid_ex + content
+ * @len: length
+*/
+void rtw_add_bcn_ie_ex(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 eid_ex, u8 *data, u8 len)
+{
+	struct ieee80211_info_element *pie;
+	bool find_ex_ie = _FALSE;
+	bool bmatch = _FALSE;
+	u8 *ies = pnetwork->IEs;
+	u8 *p = NULL, *dst_ie = NULL, *premainder_ie = NULL, *pbackup_remainder_ie = NULL;
+	u32 i, offset, ielen = 0, ie_offset, remainder_ielen = 0;
+
+	if (pnetwork->IELength + len > MAX_IE_SZ) {
+		RTW_ERR("Can't add IE to beacon, because size is more than MAX_IE_SZ\n");
+		return;
+	}
+
+	/* Search element id extension exits or not */
+	for (i = _FIXED_IE_LENGTH_; i < pnetwork->IELength;) {
+		pie = (struct ieee80211_info_element *)(ies + i);
+
+		if (pie->id == WLAN_EID_EXTENSION) {
+			find_ex_ie = _TRUE;
+			if (pie->data[0] > eid_ex) { /* first byte of data is element id extension */
+				break;
+			} else if (pie->data[0] == eid_ex) { /* already exist the same Element ID Extension */
+				p = (u8 *)pie;
+				ielen = pie->len;
+				bmatch = _TRUE;
+				break;
+			}
+		} else if (find_ex_ie) { /* append to last element id (0xff) */
+			break;
+		}
+
+		p = (u8 *)pie;
+		ielen = pie->len;
+		i += (pie->len + 2);
+	}
+
+	/* Backup remainder IE */
+	if (p != NULL && ielen > 0) {
+		ielen += 2;
+		premainder_ie = p + ielen;
+		ie_offset = (sint)(p - ies);
+		remainder_ielen = pnetwork->IELength - ie_offset - ielen;
+
+		if (bmatch)
+			dst_ie = p;
+		else
+			dst_ie = (p + ielen);
+	}
+
+	if (dst_ie == NULL)
+		return;
+
+	if (remainder_ielen > 0) {
+		pbackup_remainder_ie = rtw_malloc(remainder_ielen);
+		if (pbackup_remainder_ie && premainder_ie)
+			_rtw_memcpy(pbackup_remainder_ie, premainder_ie, remainder_ielen);
+	}
+
+	*dst_ie++ = WLAN_EID_EXTENSION;
+	*dst_ie++ = len;
+
+	_rtw_memcpy(dst_ie, data, len);
+	dst_ie += len;
+
+	/* Append remainder IE */
+	if (pbackup_remainder_ie) {
+		_rtw_memcpy(dst_ie, pbackup_remainder_ie, remainder_ielen);
+
+		rtw_mfree(pbackup_remainder_ie, remainder_ielen);
+	}
+
+	offset = (uint)(dst_ie - ies);
+	pnetwork->IELength = offset + remainder_ielen;
+}
+
+void rtw_remove_bcn_ie_ex(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 index, u8* pindex_ex, u8 index_ex_len)
+{
+	u8 *p, *dst_ie = NULL, *premainder_ie = NULL, *pbackup_remainder_ie = NULL;
+	uint offset, ielen, ie_offset, remainder_ielen = 0;
+	u8	*pie = pnetwork->IEs;
+
+	p = rtw_get_ie_ex(pie + _FIXED_IE_LENGTH_, pnetwork->IELength - _FIXED_IE_LENGTH_, index, pindex_ex, index_ex_len, NULL, &ielen);
+	if (p != NULL && ielen > 0) {
+		premainder_ie = p + ielen;
+
+		ie_offset = (sint)(p - pie);
+
+		remainder_ielen = pnetwork->IELength - ie_offset - ielen;
+
+		dst_ie = p;
+	} else
+		return;
+
+	if (remainder_ielen > 0) {
+		pbackup_remainder_ie = rtw_malloc(remainder_ielen);
+		if (pbackup_remainder_ie && premainder_ie)
+			_rtw_memcpy(pbackup_remainder_ie, premainder_ie, remainder_ielen);
+	}
+
+	/* copy remainder IE */
+	if (pbackup_remainder_ie) {
+		_rtw_memcpy(dst_ie, pbackup_remainder_ie, remainder_ielen);
+
+		rtw_mfree(pbackup_remainder_ie, remainder_ielen);
+	}
+
+	offset = (uint)(dst_ie - pie);
+	pnetwork->IELength = offset + remainder_ielen;
+}
+
+u8 chk_sta_is_alive(struct sta_info *psta);
+u8 chk_sta_is_alive(struct sta_info *psta)
+{
+	u8 ret = _FALSE;
+#ifdef DBG_EXPIRATION_CHK
+	RTW_INFO("sta:"MAC_FMT", rssi:%d, rx:"STA_PKTS_FMT", expire_to:%u, %s%ssq_len:%u\n"
+		 , MAC_ARG(psta->phl_sta->mac_addr)
+		 , 0 /* TODO: psta->phl_sta->hal_sta->rssi_stat.rssi */
+		 /* , STA_RX_PKTS_ARG(psta) */
+		 , STA_RX_PKTS_DIFF_ARG(psta)
+		 , psta->expire_to
+		 , psta->state & WIFI_SLEEP_STATE ? "PS, " : ""
+		 , psta->state & WIFI_STA_ALIVE_CHK_STATE ? "SAC, " : ""
+		 , psta->sleepq_len
+		);
+#endif
+
+	/* if(sta_last_rx_pkts(psta) == sta_rx_pkts(psta)) */
+	if ((psta->sta_stats.last_rx_data_pkts + psta->sta_stats.last_rx_ctrl_pkts) == (psta->sta_stats.rx_data_pkts + psta->sta_stats.rx_ctrl_pkts)) {
+#if 0
+		if (psta->state & WIFI_SLEEP_STATE)
+			ret = _TRUE;
+#endif
+	} else
+		ret = _TRUE;
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(psta->padapter)) {
+		u8 bcn_alive, hwmp_alive;
+
+		hwmp_alive = (psta->sta_stats.rx_hwmp_pkts !=
+			      psta->sta_stats.last_rx_hwmp_pkts);
+		bcn_alive = (psta->sta_stats.rx_beacon_pkts != 
+			     psta->sta_stats.last_rx_beacon_pkts);
+		/* The reference for nexthop_lookup */
+		psta->alive = ret || hwmp_alive || bcn_alive;
+		/* The reference for expire_timeout_chk */
+		/* Exclude bcn_alive to avoid a misjudge condition
+		   that a peer unexpectedly leave and restart quickly*/
+		ret = ret || hwmp_alive;
+	}
+#endif
+
+	sta_update_last_rx_pkts(psta);
+
+	return ret;
+}
+
+/**
+ * issue_aka_chk_frame - issue active keep alive check frame
+ *	aka = active keep alive
+ */
+#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+static int issue_aka_chk_frame(_adapter *adapter, struct sta_info *psta)
+{
+	int ret = _FAIL;
+	u8 *target_addr = psta->phl_sta->mac_addr;
+
+	if (MLME_IS_AP(adapter)) {
+		/* issue null data to check sta alive */
+		if (psta->state & WIFI_SLEEP_STATE)
+			ret = issue_nulldata(adapter,
+					psta->padapter_link,
+					target_addr, 0, 1, 50);
+		else
+			ret = issue_nulldata(adapter,
+					psta->padapter_link,
+					target_addr, 0, 3, 50);
+	}
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		struct rtw_mesh_path *mpath;
+
+		rtw_rcu_read_lock();
+		mpath = rtw_mesh_path_lookup(adapter, target_addr);
+		if (!mpath) {
+			mpath = rtw_mesh_path_add(adapter, target_addr);
+			if (IS_ERR(mpath)) {
+				rtw_rcu_read_unlock();
+				RTW_ERR(FUNC_ADPT_FMT" rtw_mesh_path_add for "MAC_FMT" fail.\n",
+					FUNC_ADPT_ARG(adapter), MAC_ARG(target_addr));
+				return _FAIL;
+			}
+		}
+		if (mpath->flags & RTW_MESH_PATH_ACTIVE)
+			ret = _SUCCESS;
+		else {
+			u8 flags = RTW_PREQ_Q_F_START | RTW_PREQ_Q_F_PEER_AKA;
+			/* issue PREQ to check peer alive */
+			rtw_mesh_queue_preq(mpath, flags);
+			ret = _FALSE;
+		}
+		rtw_rcu_read_unlock();
+	}
+#endif
+	return ret;
+}
+#endif
+
+void expire_timeout_post_chk(_adapter *padapter, u32 ap_chk_sta_bmp)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	u8 updated = _FALSE;
+#if defined(CONFIG_ACTIVE_KEEP_ALIVE_CHECK)
+	u8 switch_channel_by_drv = _TRUE;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_chan_def hw_chdef = {0};
+	struct rtw_chan_def union_chdef = {0};
+	struct rtw_chan_def bk_chdef = {0};
+#endif
+	char del_asoc_list[NUM_STA];
+	u8 del_num = 0;
+	u8 i;
+
+	_rtw_memset(del_asoc_list, NUM_STA, NUM_STA);
+
+	#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+	if (padapter->mlmeextpriv.active_keep_alive_check) {
+		if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role,
+				padapter_link->wrlink,
+				&union_chdef)
+			!= RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("fun:%s line:%d get chandef failed\n", __FUNCTION__, __LINE__);
+			rtw_warn_on(1);
+		}
+
+		if (pmlmeext->chandef.chan != union_chdef.chan)
+			switch_channel_by_drv = _FALSE;
+
+		/* switch to correct channel of current network before issue keep-alive frames */
+		rtw_get_oper_chdef(padapter, padapter_link, &hw_chdef);
+		if (switch_channel_by_drv == _TRUE && hw_chdef.chan != pmlmeext->chandef.chan) {
+			_rtw_memcpy(&bk_chdef, &hw_chdef, sizeof(struct rtw_chan_def));
+			set_bch_bwmode(padapter, padapter_link, union_chdef.band,
+				       union_chdef.chan, union_chdef.offset,
+				       union_chdef.bw, RFK_TYPE_FORCE_NOT_DO);
+		}
+	}
+	#endif /* CONFIG_ACTIVE_KEEP_ALIVE_CHECK */
+
+	/* check loop */
+	for (i = 0; i < NUM_STA; i++) {
+		#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+		int ret = _FAIL;
+		#endif
+
+		if (!(BIT(i) & ap_chk_sta_bmp))
+			continue;
+
+		psta = rtw_get_stainfo_by_offset(pstapriv, i);
+
+		if (psta == NULL)
+			continue;
+
+		#ifdef CONFIG_ATMEL_RC_PATCH
+		if (_rtw_memcmp(pstapriv->atmel_rc_pattern, psta->phl_sta->mac_addr, ETH_ALEN) == _TRUE)
+			continue;
+		if (psta->flag_atmel_rc)
+			continue;
+		#endif
+
+		if (!(psta->state & WIFI_ASOC_STATE))
+			continue;
+
+		#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+		if (padapter->mlmeextpriv.active_keep_alive_check) {
+			/* issue active keep alive frame to check */
+			ret = issue_aka_chk_frame(padapter, psta);
+
+			psta->keep_alive_trycnt++;
+			if (ret == _SUCCESS) {
+				RTW_INFO(FUNC_ADPT_FMT" asoc check, "MAC_FMT" is alive\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr));
+				psta->expire_to = pstapriv->expire_to;
+				psta->keep_alive_trycnt = 0;
+				continue;
+			} else if (psta->keep_alive_trycnt <= 3) {
+				RTW_INFO(FUNC_ADPT_FMT" asoc check, "MAC_FMT" keep_alive_trycnt=%d\n"
+					, FUNC_ADPT_ARG(padapter) , MAC_ARG(psta->phl_sta->mac_addr), psta->keep_alive_trycnt);
+				psta->expire_to = 1;
+				continue;
+			}
+		}
+		#endif /* CONFIG_ACTIVE_KEEP_ALIVE_CHECK */
+
+		psta->keep_alive_trycnt = 0;
+		del_asoc_list[del_num++] = i;
+		rtw_stapriv_asoc_list_lock(pstapriv);
+		if (rtw_is_list_empty(&psta->asoc_list) == _FALSE)
+			rtw_stapriv_asoc_list_del(pstapriv, psta);
+		rtw_stapriv_asoc_list_unlock(pstapriv);
+	}
+
+	/* delete loop */
+	for (i = 0; i < del_num; i++) {
+		u8 sta_addr[ETH_ALEN];
+
+		psta = rtw_get_stainfo_by_offset(pstapriv, del_asoc_list[i]);
+		_rtw_memcpy(sta_addr, psta->phl_sta->mac_addr, ETH_ALEN);
+
+		RTW_INFO(FUNC_ADPT_FMT" asoc expire "MAC_FMT", state=0x%x\n"
+			, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr), psta->state);
+		/* ToDo CONFIG_RTW_MLD: call only once for each MLD */
+		updated |= ap_free_sta(padapter, psta, _FALSE, 0, WLAN_REASON_DEAUTH_LEAVING, _FALSE);
+		#ifdef CONFIG_RTW_MESH
+		if (MLME_IS_MESH(padapter))
+			rtw_mesh_expire_peer(padapter, sta_addr);
+		#endif
+	}
+
+	#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+	if (padapter->mlmeextpriv.active_keep_alive_check) {
+		/* back to the original operation channel */
+		if (switch_channel_by_drv == _TRUE && bk_chdef.chan > 0)
+			set_bch_bwmode(padapter, padapter_link, bk_chdef.band,
+				       bk_chdef.chan, bk_chdef.offset, bk_chdef.bw,
+				       RFK_TYPE_FORCE_NOT_DO);
+	}
+	#endif
+
+	associated_clients_update(padapter, updated, STA_INFO_UPDATE_ALL);
+}
+
+void expire_timeout_chk(_adapter *padapter, u32 *ap_chk_sta_bmp)
+{
+	_list	*phead, *plist;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 chk_alive_num = 0;
+	int i;
+	int stainfo_offset;
+	u8 flush_num = 0;
+	char flush_list[NUM_STA]={0};
+	#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+	#endif
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)
+		&& check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)
+	) {
+		struct rtw_mesh_cfg *mcfg = &padapter->mesh_cfg;
+
+		rtw_mesh_path_expire(padapter);
+
+		/* TBD: up layer timeout mechanism */
+		/* if (!mcfg->plink_timeout)
+			return; */
+	}
+#endif
+
+#ifdef CONFIG_RTW_WDS
+	rtw_wds_path_expire(padapter);
+#endif
+
+	_rtw_spinlock_bh(&pstapriv->auth_list_lock);
+
+	phead = &pstapriv->auth_list;
+	plist = get_next(phead);
+
+	/* check auth_queue */
+#ifdef DBG_EXPIRATION_CHK
+	if (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		RTW_INFO(FUNC_ADPT_FMT" auth_list, cnt:%u\n"
+			, FUNC_ADPT_ARG(padapter), pstapriv->auth_list_cnt);
+	}
+#endif
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		psta = LIST_CONTAINOR(plist, struct sta_info, auth_list);
+
+		plist = get_next(plist);
+
+
+#ifdef CONFIG_ATMEL_RC_PATCH
+		if (_rtw_memcmp((void *)(pstapriv->atmel_rc_pattern), (void *)(psta->phl_sta->mac_addr), ETH_ALEN) == _TRUE)
+			continue;
+		if (psta->flag_atmel_rc)
+			continue;
+#endif
+		if (psta->expire_to > 0) {
+			psta->expire_to--;
+			if (psta->expire_to == 0) {
+				stainfo_offset = rtw_stainfo_offset(pstapriv, psta);
+				if (stainfo_offset_valid(stainfo_offset))
+					flush_list[flush_num++] = stainfo_offset;
+				else
+					rtw_warn_on(1);
+			}
+		}
+
+	}
+
+	_rtw_spinunlock_bh(&pstapriv->auth_list_lock);
+	for (i = 0; i < flush_num; i++) {
+		psta = rtw_get_stainfo_by_offset(pstapriv, flush_list[i]);
+		RTW_INFO(FUNC_ADPT_FMT" auth expire "MAC_FMT"\n"
+			, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr));
+		rtw_free_mld_stainfo(padapter, psta->phl_sta->mld);
+		psta = NULL;
+	}
+
+	#ifdef CONFIG_P2P
+	if(rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO) && padapter->registrypriv.p2p_go_skip_keep_alive == _TRUE)
+		return;
+	#endif
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+
+	/* check asoc_queue */
+#ifdef DBG_EXPIRATION_CHK
+	if (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		RTW_INFO(FUNC_ADPT_FMT" asoc_list, cnt:%u\n"
+			, FUNC_ADPT_ARG(padapter), pstapriv->asoc_list_cnt);
+	}
+#endif
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+#ifdef CONFIG_ATMEL_RC_PATCH
+		RTW_INFO("%s:%d  psta=%p, %02x,%02x||%02x,%02x  \n\n", __func__,  __LINE__,
+			psta, pstapriv->atmel_rc_pattern[0], pstapriv->atmel_rc_pattern[5], psta->phl_sta->mac_addr[0], psta->phl_sta->mac_addr[5]);
+		if (_rtw_memcmp((void *)pstapriv->atmel_rc_pattern, (void *)(psta->phl_sta->mac_addr), ETH_ALEN) == _TRUE)
+			continue;
+		if (psta->flag_atmel_rc)
+			continue;
+		RTW_INFO("%s: debug line:%d\n", __func__, __LINE__);
+#endif
+#ifdef CONFIG_AUTO_AP_MODE
+		if (psta->isrc)
+			continue;
+#endif
+		if (chk_sta_is_alive(psta) || !psta->expire_to) {
+			psta->expire_to = pstapriv->expire_to;
+			psta->keep_alive_trycnt = 0;
+			#if !defined(CONFIG_ACTIVE_KEEP_ALIVE_CHECK) && defined(CONFIG_80211N_HT)
+			psta->under_exist_checking = 0;
+			#endif
+		} else
+			psta->expire_to--;
+
+#if !defined(CONFIG_ACTIVE_KEEP_ALIVE_CHECK) && defined(CONFIG_80211N_HT)
+		if ((psta->flags & WLAN_STA_HT
+#if CONFIG_IEEE80211_BAND_6GHZ
+		    || psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+		    ) && (psta->ampdu_priv.agg_enable_bitmap || psta->under_exist_checking)) {
+			/* check sta by delba(addba) for 11n STA */
+			/* ToDo: use CCX report to check for all STAs */
+			/* RTW_INFO("asoc check by DELBA/ADDBA! (pstapriv->expire_to=%d s)(psta->expire_to=%d s), [%02x, %d]\n", pstapriv->expire_to*2, psta->expire_to*2, psta->ampdu_priv.agg_enable_bitmap, psta->under_exist_checking); */
+
+			if (psta->expire_to <= (pstapriv->expire_to - 50)) {
+				RTW_INFO("asoc expire by DELBA/ADDBA! (%d s)\n", (pstapriv->expire_to - psta->expire_to) * 2);
+				psta->under_exist_checking = 0;
+				psta->expire_to = 0;
+			} else if (psta->expire_to <= (pstapriv->expire_to - 3) && (psta->under_exist_checking == 0)) {
+				RTW_INFO("asoc check by DELBA/ADDBA! (%d s)\n", (pstapriv->expire_to - psta->expire_to) * 2);
+				psta->under_exist_checking = 1;
+				/* tear down TX AMPDU */
+				send_delba(padapter, 1, psta->phl_sta->mac_addr);/*  */ /* originator */
+				psta->ampdu_priv.agg_enable_bitmap = 0x0;/* reset */
+				psta->ampdu_priv.candidate_tid_bitmap = 0x0;/* reset */
+			}
+		}
+#endif /* !defined(CONFIG_ACTIVE_KEEP_ALIVE_CHECK) && defined(CONFIG_80211N_HT) */
+
+		if (psta->expire_to <= 0) {
+			if (padapter->registrypriv.wifi_spec == 1) {
+				psta->expire_to = pstapriv->expire_to;
+				continue;
+			}
+
+#ifndef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+#ifdef CONFIG_80211N_HT
+
+#define KEEP_ALIVE_TRYCNT (3)
+
+			if (psta->keep_alive_trycnt > 0 && psta->keep_alive_trycnt <= KEEP_ALIVE_TRYCNT) {
+				if (psta->state & WIFI_STA_ALIVE_CHK_STATE)
+					psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
+				else
+					psta->keep_alive_trycnt = 0;
+
+			} else if ((psta->keep_alive_trycnt > KEEP_ALIVE_TRYCNT) && !(psta->state & WIFI_STA_ALIVE_CHK_STATE))
+				psta->keep_alive_trycnt = 0;
+			if ((psta->htpriv.ht_option == _TRUE
+#if CONFIG_IEEE80211_BAND_6GHZ
+			    || psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+			    ) && (psta->ampdu_priv.ampdu_enable == _TRUE)) {
+				uint priority = 1; /* test using BK */
+				u8 issued = 0;
+
+				/* issued = (psta->ampdu_priv.agg_enable_bitmap>>priority)&0x1; */
+				issued |= (psta->ampdu_priv.candidate_tid_bitmap >> priority) & 0x1;
+
+				if (0 == issued) {
+					if (!(psta->state & WIFI_STA_ALIVE_CHK_STATE)) {
+						psta->ampdu_priv.candidate_tid_bitmap |= BIT((u8)priority);
+
+						if (psta->state & WIFI_SLEEP_STATE)
+							psta->expire_to = 2; /* 2x2=4 sec */
+						else
+							psta->expire_to = 1; /* 2 sec */
+
+						psta->state |= WIFI_STA_ALIVE_CHK_STATE;
+
+						/* add_ba_hdl(padapter, (u8*)paddbareq_parm); */
+
+						RTW_INFO("issue addba_req to check if sta alive, keep_alive_trycnt=%d\n", psta->keep_alive_trycnt);
+
+						issue_addba_req(padapter, psta->phl_sta->mac_addr, (u8)priority);
+
+						_set_timer(&psta->addba_retry_timer, ADDBA_TO);
+
+						psta->keep_alive_trycnt++;
+
+						continue;
+					}
+				}
+			}
+			if (psta->keep_alive_trycnt > 0 && psta->state & WIFI_STA_ALIVE_CHK_STATE) {
+				psta->keep_alive_trycnt = 0;
+				psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
+				RTW_INFO("change to another methods to check alive if staion is at ps mode\n");
+			}
+
+#endif /* CONFIG_80211N_HT */
+#endif /* CONFIG_ACTIVE_KEEP_ALIVE_CHECK	 */
+			if (psta->state & WIFI_SLEEP_STATE) {
+				if (!(psta->state & WIFI_STA_ALIVE_CHK_STATE)) {
+					/* to check if alive by another methods if staion is at ps mode.					 */
+					psta->expire_to = pstapriv->expire_to;
+					psta->state |= WIFI_STA_ALIVE_CHK_STATE;
+
+					/* RTW_INFO("alive chk, sta:" MAC_FMT " is at ps mode!\n", MAC_ARG(psta->phl_sta->mac_addr)); */
+
+					/* to update bcn with tim_bitmap for this station */
+					rtw_tim_map_set(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid);
+					rtw_update_beacon(padapter,
+							padapter_link,
+							_TIM_IE_, NULL, _TRUE, 0);
+
+					if (!padapter->mlmeextpriv.active_keep_alive_check)
+						continue;
+				}
+			}
+
+			{
+				int stainfo_offset;
+
+				stainfo_offset = rtw_stainfo_offset(pstapriv, psta);
+				if (stainfo_offset_valid(stainfo_offset))
+					*ap_chk_sta_bmp |= BIT(stainfo_offset);
+				continue;
+			}
+		} else {
+			/* TODO: Aging mechanism to digest frames in sleep_q to avoid running out of xmitframe */
+			if (psta->sleepq_len > (NR_XMITFRAME / pstapriv->asoc_list_cnt)
+			    && padapter->xmitpriv.free_xmitframe_cnt < ((NR_XMITFRAME / pstapriv->asoc_list_cnt) / 2)
+			   ) {
+				RTW_INFO(FUNC_ADPT_FMT" sta:"MAC_FMT", sleepq_len:%u, free_xmitframe_cnt:%u, asoc_list_cnt:%u, clear sleep_q\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, psta->sleepq_len, padapter->xmitpriv.free_xmitframe_cnt, pstapriv->asoc_list_cnt);
+				wakeup_sta_to_xmit(padapter, psta);
+			}
+		}
+	}
+
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+#ifndef CONFIG_POST_CORE_KEEP_ALIVE
+	expire_timeout_post_chk(padapter, *ap_chk_sta_bmp);
+#endif
+}
+
+#ifdef CONFIG_BMC_TX_RATE_SELECT
+u8 rtw_ap_find_mini_tx_rate(_adapter *adapter)
+{
+	_list	*phead, *plist;
+	u8 mini_tx_rate = DESC_RATEVHTSS4MCS9, sta_tx_rate;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+
+		sta_tx_rate = psta->phl_sta->ra_info.curr_tx_rate & 0x7F;
+		if (sta_tx_rate < mini_tx_rate)
+			mini_tx_rate = sta_tx_rate;
+	}
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	return mini_tx_rate;
+}
+
+u8 rtw_ap_find_bmc_rate(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, u8 tx_rate)
+{
+	u8 tx_ini_rate = DESC_RATE6M;
+
+	switch (tx_rate) {
+	case DESC_RATEVHTSS3MCS9:
+	case DESC_RATEVHTSS3MCS8:
+	case DESC_RATEVHTSS3MCS7:
+	case DESC_RATEVHTSS3MCS6:
+	case DESC_RATEVHTSS3MCS5:
+	case DESC_RATEVHTSS3MCS4:
+	case DESC_RATEVHTSS3MCS3:
+	case DESC_RATEVHTSS2MCS9:
+	case DESC_RATEVHTSS2MCS8:
+	case DESC_RATEVHTSS2MCS7:
+	case DESC_RATEVHTSS2MCS6:
+	case DESC_RATEVHTSS2MCS5:
+	case DESC_RATEVHTSS2MCS4:
+	case DESC_RATEVHTSS2MCS3:
+	case DESC_RATEVHTSS1MCS9:
+	case DESC_RATEVHTSS1MCS8:
+	case DESC_RATEVHTSS1MCS7:
+	case DESC_RATEVHTSS1MCS6:
+	case DESC_RATEVHTSS1MCS5:
+	case DESC_RATEVHTSS1MCS4:
+	case DESC_RATEVHTSS1MCS3:
+	case DESC_RATEMCS15:
+	case DESC_RATEMCS14:
+	case DESC_RATEMCS13:
+	case DESC_RATEMCS12:
+	case DESC_RATEMCS11:
+	case DESC_RATEMCS7:
+	case DESC_RATEMCS6:
+	case DESC_RATEMCS5:
+	case DESC_RATEMCS4:
+	case DESC_RATEMCS3:
+	case DESC_RATE54M:
+	case DESC_RATE48M:
+	case DESC_RATE36M:
+	case DESC_RATE24M:
+		tx_ini_rate = DESC_RATE24M;
+		break;
+	case DESC_RATEVHTSS3MCS2:
+	case DESC_RATEVHTSS3MCS1:
+	case DESC_RATEVHTSS2MCS2:
+	case DESC_RATEVHTSS2MCS1:
+	case DESC_RATEVHTSS1MCS2:
+	case DESC_RATEVHTSS1MCS1:
+	case DESC_RATEMCS10:
+	case DESC_RATEMCS9:
+	case DESC_RATEMCS2:
+	case DESC_RATEMCS1:
+	case DESC_RATE18M:
+	case DESC_RATE12M:
+		tx_ini_rate = DESC_RATE12M;
+		break;
+	case DESC_RATEVHTSS3MCS0:
+	case DESC_RATEVHTSS2MCS0:
+	case DESC_RATEVHTSS1MCS0:
+	case DESC_RATEMCS8:
+	case DESC_RATEMCS0:
+	case DESC_RATE9M:
+	case DESC_RATE6M:
+		tx_ini_rate = DESC_RATE6M;
+		break;
+	case DESC_RATE11M:
+	case DESC_RATE5_5M:
+	case DESC_RATE2M:
+	case DESC_RATE1M:
+		tx_ini_rate = DESC_RATE1M;
+		break;
+	default:
+		tx_ini_rate = DESC_RATE6M;
+		break;
+	}
+
+	if (WIFI_ROLE_LINK_IS_ON_5G(adapter_link))
+		if (tx_ini_rate < DESC_RATE6M)
+			tx_ini_rate = DESC_RATE6M;
+
+	return tx_ini_rate;
+}
+
+void rtw_update_bmc_sta_tx_rate(_adapter *adapter)
+{
+	struct sta_info *psta = NULL;
+	u8 tx_rate;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	psta = rtw_get_bcmc_stainfo(adapter, adapter_link);
+	if (psta == NULL) {
+		RTW_ERR(ADPT_FMT "could not get bmc_sta !!\n", ADPT_ARG(adapter));
+		return;
+	}
+
+	if (adapter->bmc_tx_rate != MGN_UNKNOWN) {
+		psta->init_rate = adapter->bmc_tx_rate;
+		goto _exit;
+	}
+
+	if (adapter->stapriv.asoc_sta_count <= 2)
+		goto _exit;
+
+	tx_rate = rtw_ap_find_mini_tx_rate(adapter);
+	#ifdef CONFIG_BMC_TX_LOW_RATE
+	tx_rate = rtw_ap_find_bmc_rate(adapter, adapter_link, tx_rate);
+	#endif
+
+	psta->init_rate = hwrate_to_mrate(tx_rate);
+
+_exit:
+	RTW_INFO(ADPT_FMT" BMC Tx rate - %s\n", ADPT_ARG(adapter), MGN_RATE_STR(psta->init_rate));
+}
+#endif
+
+void rtw_init_bmc_sta_tx_rate(_adapter *padapter, struct sta_info *psta)
+{
+/* ToDo: need API to query hal_sta->ra_info.ramask */
+#if 0
+#ifdef CONFIG_BMC_TX_LOW_RATE
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+#endif
+	u8 rate_idx = 0;
+	u8 brate_table[] = {MGN_1M, MGN_2M, MGN_5_5M, MGN_11M,
+		MGN_6M, MGN_9M, MGN_12M, MGN_18M, MGN_24M, MGN_36M, MGN_48M, MGN_54M};
+
+	if (!MLME_IS_AP(padapter) && !MLME_IS_MESH(padapter))
+		return;
+
+	if (padapter->bmc_tx_rate != MGN_UNKNOWN)
+		psta->init_rate = padapter->bmc_tx_rate;
+	else {
+		#ifdef CONFIG_BMC_TX_LOW_RATE
+		if (IsEnableHWOFDM(pmlmeext->cur_wireless_mode) && (psta->phl_sta->ra_info.ramask && 0xFF0))
+			rate_idx = get_lowest_rate_idx_ex(psta->phl_sta->ra_info.ramask, 4); /*from basic rate*/
+		else
+			rate_idx = get_lowest_rate_idx(psta->phl_sta->ra_info.ramask); /*from basic rate*/
+		#else
+		rate_idx = get_highest_rate_idx(psta->phl_sta->ra_info.ramask); /*from basic rate*/
+		#endif
+		if (rate_idx < 12)
+			psta->init_rate = brate_table[rate_idx];
+		else
+			psta->init_rate = MGN_1M;
+	}
+#endif
+	RTW_INFO(ADPT_FMT" BMC Init Tx rate - %s\n", ADPT_ARG(padapter), MGN_RATE_STR(psta->init_rate));
+}
+
+#if defined(CONFIG_80211N_HT) && defined(CONFIG_BEAMFORMING)
+void update_sta_info_apmode_ht_bf_cap(_adapter *padapter, struct sta_info *psta)
+{
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct ht_priv	*phtpriv_ap = &pmlmepriv->htpriv;
+	struct ht_priv	*phtpriv_sta = &psta->htpriv;
+
+	u8 cur_beamform_cap = 0;
+
+	/*Config Tx beamforming setting*/
+	if (TEST_FLAG(phtpriv_ap->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE) &&
+		GET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP((u8 *)(&phtpriv_sta->ht_cap))) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
+		/*Shift to BEAMFORMING_HT_BEAMFORMEE_CHNL_EST_CAP*/
+		SET_FLAG(cur_beamform_cap, GET_HT_CAP_TXBF_CHNL_ESTIMATION_NUM_ANTENNAS((u8 *)(&phtpriv_sta->ht_cap)) << 6);
+	}
+
+	if (TEST_FLAG(phtpriv_ap->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE) &&
+		GET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP((u8 *)(&phtpriv_sta->ht_cap))) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
+		/*Shift to BEAMFORMING_HT_BEAMFORMER_STEER_NUM*/
+		SET_FLAG(cur_beamform_cap, GET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS((u8 *)(&phtpriv_sta->ht_cap)) << 4);
+	}
+	if (cur_beamform_cap)
+		RTW_INFO("Client STA(%d) HT Beamforming Cap = 0x%02X\n", psta->phl_sta->aid, cur_beamform_cap);
+
+	phtpriv_sta->beamform_cap = cur_beamform_cap;
+}
+#endif /*CONFIG_80211N_HT && CONFIG_BEAMFORMING*/
+
+#ifdef CONFIG_80211N_HT
+void update_sta_bw_apmode(_adapter *padapter, struct sta_info *psta)
+{
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct ht_priv	*phtpriv_ap = &pmlmepriv->htpriv;
+	struct ht_priv	*phtpriv_sta = &psta->htpriv;
+
+	if (phtpriv_sta->ht_option) {
+		if ((phtpriv_sta->ht_cap.cap_info & phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH))
+			psta->phl_sta->chandef.bw = CHANNEL_WIDTH_40;
+		else
+			psta->phl_sta->chandef.bw = CHANNEL_WIDTH_20;
+
+		if (phtpriv_sta->op_present
+			&& !GET_HT_OP_ELE_STA_CHL_WIDTH(phtpriv_sta->ht_op))
+			psta->phl_sta->chandef.bw = CHANNEL_WIDTH_20;
+
+		if ((psta->ht_40mhz_intolerant) || (pmlmepriv->sw_to_20mhz))
+			psta->phl_sta->chandef.bw = CHANNEL_WIDTH_20;
+
+		if (pmlmeext->chandef.bw < psta->phl_sta->chandef.bw)
+			psta->phl_sta->chandef.bw = pmlmeext->chandef.bw;
+
+		phtpriv_sta->ch_offset = pmlmepriv->sw_to_20mhz ? CHAN_OFFSET_NO_EXT : pmlmeext->chandef.offset;
+	}
+}
+#endif /* CONFIG_80211N_HT */
+
+/* notes:
+ * AID: 1~MAX for sta and 0 for bc/mc in ap/adhoc mode  */
+void update_sta_info_apmode(_adapter *padapter, struct sta_info *psta)
+{
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct protocol_cap_t *protocol_cap = &(padapter_link->wrlink->protocol_cap);
+#ifdef CONFIG_80211N_HT
+	struct ht_priv	*phtpriv_ap = &pmlmepriv->htpriv;
+	struct ht_priv	*phtpriv_sta = &psta->htpriv;
+	struct ampdu_priv *ampdu_priv_ap = &pmlmepriv->ampdu_priv;
+	struct ampdu_priv *ampdu_priv_sta = &psta->ampdu_priv;
+#endif /* CONFIG_80211N_HT */
+	u8	cur_ldpc_cap = 0, cur_stbc_cap = 0;
+	/* set intf_tag to if1 */
+	/* psta->intf_tag = 0; */
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/*alloc macid when call rtw_alloc_stainfo(),release macid when call rtw_free_stainfo()*/
+
+	if (!MLME_IS_MESH(padapter) && psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+		psta->ieee8021x_blocked = _TRUE;
+	else
+		psta->ieee8021x_blocked = _FALSE;
+
+
+	/* update sta's cap */
+
+	psta->phl_sta->chandef.chan = pmlmeext->chandef.chan;
+	psta->phl_sta->chandef.band = pmlmeext->chandef.band;
+
+	/* ERP */
+	VCS_update(padapter, psta);
+#ifdef CONFIG_80211N_HT
+	/* HT related cap */
+	if (phtpriv_sta->ht_option) {
+		/* check if sta supports rx ampdu */
+		ampdu_priv_sta->ampdu_enable = ampdu_priv_ap->ampdu_enable;
+
+		ampdu_priv_sta->rx_ampdu_min_spacing = (phtpriv_sta->ht_cap.ampdu_params_info & IEEE80211_HT_CAP_AMPDU_DENSITY) >> 2;
+
+		update_sta_bw_apmode(padapter, psta);
+
+		/* check if sta support s Short GI 20M */
+		if ((phtpriv_sta->ht_cap.cap_info & phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SGI_20))
+			phtpriv_sta->sgi_20m = _TRUE;
+
+		/* check if sta support s Short GI 40M */
+		if ((phtpriv_sta->ht_cap.cap_info & phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SGI_40)) {
+			if (psta->phl_sta->chandef.bw == CHANNEL_WIDTH_40) /* according to psta->bw_mode */
+				phtpriv_sta->sgi_40m = _TRUE;
+			else
+				phtpriv_sta->sgi_40m = _FALSE;
+		}
+
+		psta->qos_option = _TRUE;
+
+		/* B0 Config LDPC Coding Capability */
+		if (TEST_FLAG(phtpriv_ap->ldpc_cap, LDPC_HT_ENABLE_TX) &&
+		    GET_HT_CAP_ELE_LDPC_CAP((u8 *)(&phtpriv_sta->ht_cap))) {
+			SET_FLAG(cur_ldpc_cap, (LDPC_HT_ENABLE_TX | LDPC_HT_CAP_TX));
+			RTW_INFO("Enable HT Tx LDPC for STA(%d)\n", psta->phl_sta->aid);
+		}
+
+		/* B7 B8 B9 Config STBC setting */
+		if (TEST_FLAG(phtpriv_ap->stbc_cap, STBC_HT_ENABLE_TX) &&
+		    GET_HT_CAP_ELE_RX_STBC((u8 *)(&phtpriv_sta->ht_cap))) {
+			SET_FLAG(cur_stbc_cap, (STBC_HT_ENABLE_TX | STBC_HT_CAP_TX));
+			RTW_INFO("Enable HT Tx STBC for STA(%d)\n", psta->phl_sta->aid);
+		}
+
+		psta->phl_sta->asoc_cap.stbc_ht_tx =
+			GET_HT_CAP_ELE_TX_STBC((u8 *)(&phtpriv_sta->ht_cap));
+		psta->phl_sta->asoc_cap.stbc_ht_rx =
+			protocol_cap->stbc_ht_tx ?
+			GET_HT_CAP_ELE_RX_STBC((u8 *)(&phtpriv_sta->ht_cap)) : 0;
+
+		#ifdef CONFIG_BEAMFORMING
+		update_sta_info_apmode_ht_bf_cap(padapter, psta);
+		#endif
+	} else {
+		ampdu_priv_sta->ampdu_enable = _FALSE;
+
+		phtpriv_sta->sgi_20m = _FALSE;
+		phtpriv_sta->sgi_40m = _FALSE;
+		psta->phl_sta->chandef.bw = CHANNEL_WIDTH_20;
+		phtpriv_sta->ch_offset = CHAN_OFFSET_NO_EXT;
+	}
+
+	phtpriv_sta->ldpc_cap = cur_ldpc_cap;
+	phtpriv_sta->stbc_cap = cur_stbc_cap;
+
+	/* Rx AMPDU */
+	send_delba(padapter, 0, psta->phl_sta->mac_addr);/* recipient */
+
+	/* TX AMPDU */
+	send_delba(padapter, 1, psta->phl_sta->mac_addr);/*  */ /* originator */
+	ampdu_priv_sta->agg_enable_bitmap = 0x0;/* reset */
+	ampdu_priv_sta->candidate_tid_bitmap = 0x0;/* reset */
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_80211AC_VHT
+	update_sta_vht_info_apmode(padapter, psta);
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	update_sta_he_info_apmode(padapter, psta);
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	update_sta_6ghz_info_apmode(padapter, psta);
+#endif
+
+	psta->phl_sta->chandef.offset = (psta->phl_sta->chandef.bw > CHANNEL_WIDTH_20) ?
+		pmlmeext->chandef.offset : CHAN_OFFSET_NO_EXT;
+
+	/* ToDo: need API to inform hal_sta->ra_info.is_support_sgi */
+	/* psta->phl_sta->ra_info.is_support_sgi = query_ra_short_GI(psta, rtw_get_tx_bw_mode(padapter, psta)); */
+	update_ldpc_stbc_cap(psta);
+
+	/* todo: init other variables */
+
+	_rtw_memset((void *)&psta->sta_stats, 0, sizeof(struct stainfo_stats));
+
+	_rtw_spinlock_bh(&psta->lock);
+
+	/* Check encryption */
+	if (!MLME_IS_MESH(padapter) && psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+		psta->state |= WIFI_UNDER_KEY_HANDSHAKE;
+
+	psta->state |= WIFI_ASOC_STATE;
+
+	_rtw_spinunlock_bh(&psta->lock);
+}
+
+static void update_ap_info(_adapter *padapter, struct sta_info *psta)
+{
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	WLAN_BSSID_EX *pnetwork = (WLAN_BSSID_EX *)&pmlmepriv->cur_network.network;
+#ifdef CONFIG_80211N_HT
+	struct ht_priv	*phtpriv_ap = &pmlmepriv->htpriv;
+	struct ampdu_priv *ampdu_priv_ap = &pmlmepriv->ampdu_priv;
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter))
+		psta->qos_option = 1;
+#endif
+
+	psta->phl_sta->wmode = pmlmeext->cur_wireless_mode;
+
+	psta->bssratelen = rtw_get_rateset_len(pnetwork->SupportedRates);
+	_rtw_memcpy(psta->bssrateset, pnetwork->SupportedRates, psta->bssratelen);
+
+#ifdef CONFIG_80211N_HT
+	/* HT related cap */
+	if (phtpriv_ap->ht_option) {
+		/* check if sta supports rx ampdu */
+		/* ampdu_priv_ap->ampdu_enable = ampdu_priv_ap->ampdu_enable; */
+
+		/* check if sta support s Short GI 20M */
+		if ((phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SGI_20))
+			phtpriv_ap->sgi_20m = _TRUE;
+		/* check if sta support s Short GI 40M */
+		if ((phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SGI_40))
+			phtpriv_ap->sgi_40m = _TRUE;
+
+		psta->qos_option = _TRUE;
+	} else {
+		ampdu_priv_ap->ampdu_enable = _FALSE;
+
+		phtpriv_ap->sgi_20m = _FALSE;
+		phtpriv_ap->sgi_40m = _FALSE;
+	}
+
+	_rtw_memcpy(&psta->phl_sta->chandef, &pmlmeext->chandef, sizeof(struct rtw_chan_def));
+	phtpriv_ap->ch_offset = pmlmeext->chandef.offset;
+
+	ampdu_priv_ap->agg_enable_bitmap = 0x0;/* reset */
+	ampdu_priv_ap->candidate_tid_bitmap = 0x0;/* reset */
+
+	_rtw_memcpy(&psta->htpriv, &pmlmepriv->htpriv, sizeof(struct ht_priv));
+
+#ifdef CONFIG_80211AC_VHT
+	_rtw_memcpy(&psta->vhtpriv, &pmlmepriv->vhtpriv, sizeof(struct vht_priv));
+
+#ifdef CONFIG_80211AX_HE
+	_rtw_memcpy(&psta->hepriv, &pmlmepriv->hepriv, sizeof(struct he_priv));
+#endif /* CONFIG_80211AX_HE */
+
+#endif /* CONFIG_80211AC_VHT */
+
+#endif /* CONFIG_80211N_HT */
+
+	_rtw_spinlock_bh(&psta->lock);
+	psta->state |= WIFI_AP_STATE; /* Aries, add,fix bug of flush_cam_entry at STOP AP mode , 0724 */
+	psta->state |= WIFI_ASOC_STATE;
+	_rtw_spinunlock_bh(&psta->lock);
+
+	rtw_init_bmc_sta_tx_rate(padapter, psta);
+}
+
+void rtw_ap_set_edca(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, enum rtw_ac ac, u32 parm)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta = rtw_get_stainfo(pstapriv, padapter_link->mac_addr);
+
+	if (psta && psta->phl_sta) {
+		psta->phl_sta->asoc_cap.edca[ac].ac = ac;
+		psta->phl_sta->asoc_cap.edca[ac].param = parm;
+	}
+	rtw_hw_set_edca(padapter, padapter_link, ac, parm);
+}
+
+static void rtw_set_hw_wmm_param(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	u8	AIFS, ECWMin, ECWMax, aSifsTime;
+	u8	acm_mask;
+	u16	TXOP;
+	u32	acParm, i;
+	u32	edca[4], inx[4];
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+
+	acm_mask = 0;
+#ifdef CONFIG_80211N_HT
+	if (pregpriv->ht_enable &&
+		(WIFI_ROLE_LINK_IS_ON_5G(padapter_link) || (pmlmeext->cur_wireless_mode & WLAN_MD_11N)))
+		aSifsTime = 16;
+	else
+#endif /* CONFIG_80211N_HT */
+		aSifsTime = 10;
+
+	if (pmlmeinfo->WMM_enable == 0) {
+		padapter->mlmepriv.acm_mask = 0;
+
+		AIFS = aSifsTime + (2 * pmlmeinfo->slotTime);
+
+		if (pmlmeext->cur_wireless_mode & (WLAN_MD_11G | WLAN_MD_11A)) {
+			ECWMin = 4;
+			ECWMax = 10;
+		} else if (pmlmeext->cur_wireless_mode & WLAN_MD_11B) {
+			ECWMin = 5;
+			ECWMax = 10;
+		} else {
+			ECWMin = 4;
+			ECWMax = 10;
+		}
+
+		TXOP = 0;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_ap_set_edca(padapter, padapter_link, 0, acParm);
+		rtw_ap_set_edca(padapter, padapter_link, 1, acParm);
+		rtw_ap_set_edca(padapter, padapter_link, 2, acParm);
+
+		ECWMin = 2;
+		ECWMax = 3;
+		TXOP = 0x2f;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_ap_set_edca(padapter, padapter_link, 3, acParm);
+
+	} else {
+		edca[0] = edca[1] = edca[2] = edca[3] = 0;
+
+		/*TODO:*/
+		acm_mask = 0;
+		padapter->mlmepriv.acm_mask = acm_mask;
+
+#if 0
+		/* BK */
+		/* AIFS = AIFSN * slot time + SIFS - r2t phy delay */
+#endif
+		AIFS = (7 * pmlmeinfo->slotTime) + aSifsTime;
+		ECWMin = 4;
+		ECWMax = 10;
+		TXOP = 0;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_ap_set_edca(padapter, padapter_link, 1, acParm);
+		edca[XMIT_BK_QUEUE] = acParm;
+		RTW_INFO("WMM(BK): %x\n", acParm);
+
+		/* BE */
+		AIFS = (3 * pmlmeinfo->slotTime) + aSifsTime;
+		ECWMin = 4;
+		ECWMax = 6;
+		TXOP = 0;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_ap_set_edca(padapter, padapter_link, 0, acParm);
+		edca[XMIT_BE_QUEUE] = acParm;
+		RTW_INFO("WMM(BE): %x\n", acParm);
+
+		/* VI */
+		AIFS = (1 * pmlmeinfo->slotTime) + aSifsTime;
+		ECWMin = 3;
+		ECWMax = 4;
+		TXOP = 94;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_ap_set_edca(padapter, padapter_link, 2, acParm);
+		edca[XMIT_VI_QUEUE] = acParm;
+		RTW_INFO("WMM(VI): %x\n", acParm);
+
+		/* VO */
+		AIFS = (1 * pmlmeinfo->slotTime) + aSifsTime;
+		ECWMin = 2;
+		ECWMax = 3;
+		TXOP = 47;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_ap_set_edca(padapter, padapter_link, 3, acParm);
+		edca[XMIT_VO_QUEUE] = acParm;
+		RTW_INFO("WMM(VO): %x\n", acParm);
+
+
+		if (padapter->registrypriv.acm_method == 1)
+			rtw_hal_set_hwreg(padapter, HW_VAR_ACM_CTRL, (u8 *)(&acm_mask));
+		else
+			padapter->mlmepriv.acm_mask = acm_mask;
+
+		inx[0] = 0;
+		inx[1] = 1;
+		inx[2] = 2;
+		inx[3] = 3;
+
+		if (padapter->registrypriv.wifi_spec == 1) {
+			u32	j, tmp, change_inx = _FALSE;
+
+			/* entry indx: 0->vo, 1->vi, 2->be, 3->bk. */
+			for (i = 0 ; i < 4 ; i++) {
+				for (j = i + 1 ; j < 4 ; j++) {
+					/* compare CW and AIFS */
+					if ((edca[j] & 0xFFFF) < (edca[i] & 0xFFFF))
+						change_inx = _TRUE;
+					else if ((edca[j] & 0xFFFF) == (edca[i] & 0xFFFF)) {
+						/* compare TXOP */
+						if ((edca[j] >> 16) > (edca[i] >> 16))
+							change_inx = _TRUE;
+					}
+
+					if (change_inx) {
+						tmp = edca[i];
+						edca[i] = edca[j];
+						edca[j] = tmp;
+
+						tmp = inx[i];
+						inx[i] = inx[j];
+						inx[j] = tmp;
+
+						change_inx = _FALSE;
+					}
+				}
+			}
+		}
+
+		for (i = 0 ; i < 4 ; i++) {
+			pxmitpriv->wmm_para_seq[i] = inx[i];
+			RTW_INFO("wmm_para_seq(%d): %d\n", i, pxmitpriv->wmm_para_seq[i]);
+		}
+
+	}
+
+}
+static void rtw_ap_check_scan(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	_list		*plist, *phead;
+	u32	delta_time, lifetime;
+	struct	wlan_network	*pnetwork = NULL;
+	WLAN_BSSID_EX *pbss = NULL;
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	_queue	*queue	= &(pmlmepriv->scanned_queue);
+	u8 do_scan = _FALSE;
+	u8 reason = RTW_AUTO_SCAN_REASON_UNSPECIFIED;
+	struct link_mlme_priv *lmlmepriv = &padapter_link->mlmepriv;
+
+	lifetime = SCANQUEUE_LIFETIME; /* 20 sec */
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+	phead = get_list_head(queue);
+	if (rtw_end_of_queue_search(phead, get_next(phead)) == _TRUE)
+		if (padapter->registrypriv.wifi_spec) {
+			do_scan = _TRUE;
+			reason |= RTW_AUTO_SCAN_REASON_2040_BSS;
+		}
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+#if defined(CONFIG_RTW_ACS) && defined(WKARD_ACS)
+	if (padapter->registrypriv.acs_auto_scan) {
+		do_scan = _TRUE;
+		reason |= RTW_AUTO_SCAN_REASON_ACS;
+		rtw_acs_start(padapter);
+	}
+#endif/*CONFIG_RTW_ACS*/
+
+	if (_TRUE == do_scan) {
+		RTW_INFO("%s : drv scans by itself and wait_completed\n", __func__);
+		rtw_drv_scan_by_self(padapter, reason);
+		rtw_scan_abort(padapter, 0);
+	}
+
+#if defined(CONFIG_RTW_ACS) && defined(WKARD_ACS)
+	if (padapter->registrypriv.acs_auto_scan)
+		rtw_acs_stop(padapter);
+#endif
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	while (1) {
+
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+		if (rtw_chset_search_bch(adapter_to_chset(padapter)
+				, pnetwork->network.Configuration.Band
+				, pnetwork->network.Configuration.DSConfig) >= 0
+		    && rtw_mlme_band_check(padapter, pnetwork->network.Configuration.DSConfig) == _TRUE
+		    && _TRUE == rtw_validate_ssid(&(pnetwork->network.Ssid))) {
+			delta_time = (u32) rtw_get_passing_time_ms(pnetwork->last_scanned);
+
+			if (delta_time < lifetime) {
+
+				uint ie_len = 0;
+				u8 *pbuf = NULL;
+				u8 *ie = NULL;
+
+				pbss = &pnetwork->network;
+				ie = pbss->IEs;
+
+				/*check if HT CAP INFO IE exists or not*/
+				pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_CAPABILITY_IE_, &ie_len, (pbss->IELength - _BEACON_IE_OFFSET_));
+				if (pbuf == NULL) {
+					/* HT CAP INFO IE don't exist, it is b/g mode bss.*/
+
+					if (_FALSE == ATOMIC_READ(&lmlmepriv->olbc))
+						ATOMIC_SET(&lmlmepriv->olbc, _TRUE);
+
+					if (_FALSE == ATOMIC_READ(&lmlmepriv->olbc_ht))
+						ATOMIC_SET(&lmlmepriv->olbc_ht, _TRUE);
+					if (padapter->registrypriv.wifi_spec)
+						RTW_INFO("%s: %s is a/b/g ap\n", __func__, pnetwork->network.Ssid.Ssid);
+				}
+			}
+		}
+
+		plist = get_next(plist);
+
+	}
+
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+#ifdef CONFIG_80211N_HT
+	lmlmepriv->num_sta_no_ht = 0; /* reset to 0 after ap do scanning*/
+#endif
+}
+
+void rtw_start_bss_hdl_after_chbw_decided(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	WLAN_BSSID_EX *pnetwork = &(adapter_link->mlmepriv.cur_network.network);
+	struct sta_info *sta = NULL;
+
+	/* update cur_wireless_mode */
+	update_wireless_mode(adapter, adapter_link);
+
+	/* update RRSR and RTS_INIT_RATE register after set channel and bandwidth */
+	UpdateBrateTbl(adapter, pnetwork->SupportedRates);
+	rtw_hal_set_hwreg(adapter, HW_VAR_BASIC_RATE, pnetwork->SupportedRates);
+
+	/* update capability after cur_wireless_mode updated */
+	update_capinfo(adapter, adapter_link, rtw_get_capability(pnetwork));
+
+	/* update AP's sta info */
+	sta = rtw_get_stainfo(&adapter->stapriv, pnetwork->MacAddress);
+	if (!sta) {
+		RTW_INFO(FUNC_ADPT_FMT" !sta for macaddr="MAC_FMT"\n", FUNC_ADPT_ARG(adapter), MAC_ARG(pnetwork->MacAddress));
+		rtw_warn_on(1);
+		return;
+	}
+
+	update_ap_info(adapter, sta);
+}
+
+static void _rtw_iface_undersurvey_chk(const char *func, _adapter *adapter)
+{
+	int i;
+	_adapter *iface;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct mlme_priv *pmlmepriv;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && rtw_is_adapter_up(iface)) {
+			pmlmepriv = &iface->mlmepriv;
+			if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY))
+				RTW_ERR("%s ("ADPT_FMT") under survey\n", func, ADPT_ARG(iface));
+		}
+	}
+}
+
+void rtw_core_ap_swch_start(_adapter *padapter, struct createbss_parm *parm)
+{
+	struct dvobj_priv *pdvobj = padapter->dvobj;
+	int i;
+	u8 ht_option = 0;
+	_adapter *iface;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct _ADAPTER_LINK *iface_link;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+	if (parm->ch_to_set != 0) {
+		set_bch_bwmode(padapter, padapter_link, parm->req_band,
+			parm->ch_to_set, parm->offset_to_set, parm->bw_to_set, RFK_TYPE_FORCE_DO);
+		rtw_phl_mr_dump_cur_chandef(adapter_to_dvobj(padapter)->phl, padapter->phl_role);
+	}
+
+#if defined(CONFIG_IOCTL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	for (i = 0; i < pdvobj->iface_nums; i++) {
+		iface = pdvobj->padapters[i];
+
+		if (!(parm->ifbmp_ch_changed & BIT(i)) || !iface)
+			continue;
+
+		iface_link = GET_PRIMARY_LINK(iface);
+
+		#ifdef CONFIG_80211N_HT
+		ht_option = iface_link->mlmepriv.htpriv.ht_option;
+		#endif
+
+		rtw_cfg80211_ch_switch_notify(iface,
+						iface_link,
+						&iface_link->mlmeextpriv.chandef,
+						ht_option, 0);
+	}
+#endif /* defined(CONFIG_IOCTL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) */
+
+	rtw_rfctl_update_op_mode(adapter_to_rfctl(padapter), parm->ifbmp, 1, parm->excl_ifbmp);
+}
+
+#ifdef CONFIG_DBCC_SUPPORT
+static void _go_dbcc_update_ie(_adapter *a)
+{
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	struct link_mlme_ext_priv *mlmeext = &(alink->mlmeextpriv);
+	struct link_mlme_priv *mlmepriv = &(alink->mlmepriv);
+	struct link_mlme_ext_info *mlmeinfo = &(mlmeext->mlmext_info);
+	WLAN_BSSID_EX *network = &(mlmeinfo->network);
+	struct HT_caps_element *ht_caps;
+	u32 ht_cap_ie_len = 0;
+	u8 *ht_cap_ie = NULL, *ie = network->IEs;
+
+	if (!rtw_phl_mr_is_db(adapter_to_dvobj(a)->phl))
+		return;
+#ifdef CONFIG_P2P
+	if (!MLME_IS_GO(a))
+		return;
+#ifdef CONFIG_80211N_HT
+	if (!mlmepriv->htpriv.ht_option)
+		return;
+
+	if (!mlmeinfo->HT_caps_enable)
+		return;
+
+	ht_cap_ie = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_CAPABILITY_IE_, &ht_cap_ie_len, (network->IELength - _BEACON_IE_OFFSET_));
+	if (ht_cap_ie && ht_cap_ie_len > 0) {
+		ht_caps = (struct HT_caps_element *)(ht_cap_ie + 2);
+		RTW_INFO(FUNC_ADPT_FMT": orig ht caps info = 0x%02x\n", FUNC_ADPT_ARG(a), ht_caps->u.HT_cap_element.HT_caps_info);
+		SET_HT_CAP_ELE_SM_PS(&(ht_caps->u.HT_cap_element.HT_caps_info), SM_PS_STATIC);
+		RTW_INFO(FUNC_ADPT_FMT": update ht caps info = 0x%02x\n", FUNC_ADPT_ARG(a), ht_caps->u.HT_cap_element.HT_caps_info);
+	}
+#endif
+#endif
+}
+#endif
+
+void rtw_core_ap_start(_adapter *padapter, struct createbss_parm *parm)
+{
+	struct dvobj_priv *pdvobj = padapter->dvobj;
+	int i;
+	_adapter *iface;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct _ADAPTER_LINK *iface_link;
+	struct link_mlme_priv *mlme;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+	if (0) {
+		RTW_INFO(FUNC_ADPT_FMT" done\n", FUNC_ADPT_ARG(padapter));
+		dump_adapters_status(RTW_DBGDUMP , adapter_to_dvobj(padapter));
+	}
+
+	for (i = 0; i < pdvobj->iface_nums; i++) {
+		iface = pdvobj->padapters[i];
+
+		if (!(parm->ifbmp & BIT(i)) || !iface)
+			continue;
+
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		iface_link = GET_PRIMARY_LINK(iface);
+		/* update beacon content only if bstart_bss is _TRUE */
+		if (iface_link->mlmeextpriv.bstart_bss != _TRUE)
+			continue;
+
+		mlme = &(iface_link->mlmepriv);
+
+		#ifdef CONFIG_80211N_HT
+		if ((ATOMIC_READ(&mlme->olbc) == _TRUE) || (ATOMIC_READ(&mlme->olbc_ht) == _TRUE)) {
+			/* AP is not starting a 40 MHz BSS in presence of an 802.11g BSS. */
+			mlme->ht_op_mode &= (~HT_INFO_OPERATION_MODE_OP_MODE_MASK);
+			mlme->ht_op_mode |= OP_MODE_MAY_BE_LEGACY_STAS;
+			rtw_update_beacon(iface, iface_link, _HT_ADD_INFO_IE_, NULL, _FALSE, 0);
+		}
+		#endif
+
+		rtw_update_beacon(iface, iface_link, _TIM_IE_, NULL, _FALSE, 0);
+	}
+
+#ifdef CONFIG_DBCC_SUPPORT
+	_go_dbcc_update_ie(padapter);
+#endif
+
+	/* after chanctx_add & before send bcn */
+
+	rtw_scan_abort(padapter, 0);
+	_rtw_iface_undersurvey_chk(__func__, padapter);
+
+	/* send beacon */
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI) || defined(CONFIG_PCI_BCN_POLLING) || defined(CONFIG_HWSIM)
+	for (i = 0; i < pdvobj->iface_nums; i++) {
+		iface = pdvobj->padapters[i];
+		if (!(parm->ifbmp & BIT(i)) || !iface)
+			continue;
+
+		if (send_beacon(iface) == _FAIL)
+			RTW_INFO(ADPT_FMT" issue_beacon, fail!\n", ADPT_ARG(pdvobj->padapters[i]));
+	}
+#endif
+
+	if (parm->is_change_chbw)
+		rtw_phl_cmd_stop_beacon(adapter_to_dvobj(padapter)->phl, padapter_link->wrlink,
+					_FALSE, PHL_CMD_DIRECTLY, 0);
+
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	if (MLME_IS_AP(padapter) && padapter->tbtx_capability == _TRUE) {
+		_set_timer(&pmlmeext->tbtx_token_dispatch_timer, 1);
+		RTW_INFO("Start token dispatch\n");
+	}
+#endif
+}
+void rtw_core_ap_chan_decision(_adapter *padapter, struct createbss_parm *parm)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	bool is_chctx_add = false;
+	struct rtw_chan_def new_chdef = {0};
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+
+	if (!(parm->req_ch == 0 && padapter->mlmeextpriv.mlmext_info.state == WIFI_FW_AP_STATE))
+		return;
+
+#ifdef CONFIG_DBCC_SUPPORT
+	_rtw_memcpy(&new_chdef, &pmlmepriv->ori_chandef, sizeof(struct rtw_chan_def));
+
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+	/*B0,B1 GO start*/
+	rtw_dbcc_chk_disable_hdl(padapter, &new_chdef, PHL_CMD_DIRECTLY, DBCC_CHK_AP_START);
+#endif
+
+	is_chctx_add = rtw_phl_chanctx_chk(dvobj->phl, padapter->phl_role,
+						padapter_link->wrlink,
+						&new_chdef, &mr_cc_info);
+
+	RTW_INFO("%s => chctx_chk:%s\n", __func__, (is_chctx_add) ? "Y" : "N");
+	RTW_INFO("PHL- CH:%d, BW:%d OFF:%d\n", new_chdef.chan, new_chdef.bw, new_chdef.offset);
+	if (is_chctx_add == false && mr_cc_info.sugg_opmode == MR_OP_DBCC) {
+		_rtw_memcpy(&pmlmeext->chandef, &pmlmepriv->ori_chandef, sizeof(struct rtw_chan_def));
+		rtw_phl_mr_trig_dbcc_enable(dvobj->phl);
+		parm->is_trigger_dbcc = _TRUE;
+	}
+#endif
+}
+
+void rtw_core_ap_prepare(_adapter *padapter, struct createbss_parm *parm)
+{
+	u8 mlme_act = MLME_ACTION_UNKNOWN;
+	u16 bcn_interval;
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	s8 req_band = REQ_BAND_NONE;
+	s16 req_ch = REQ_CH_NONE, req_bw = REQ_BW_NONE, req_offset = REQ_OFFSET_NONE;
+	struct rtw_chan_def chdef_to_set = {0};
+	int i;
+	u8 ifbmp_ch_changed = 0;
+#ifdef CONFIG_MCC_MODE
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	u8 mcc_sup = phl_com->dev_cap.mcc_sup;
+#endif
+	struct rtw_chan_def new_chdef = {0};
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+	_adapter *iface = NULL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct _ADAPTER_LINK *iface_link = NULL;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	WLAN_BSSID_EX *pnetwork = (WLAN_BSSID_EX *)&pmlmepriv->cur_network.network; /* used as input */
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *pnetwork_mlmeext = &(pmlmeinfo->network);
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+	if (parm->req_ch != 0) {
+		/* bypass other setting, go checking ch, bw, offset */
+		mlme_act = MLME_OPCH_SWITCH;
+		req_band = parm->req_band;
+		req_ch = parm->req_ch;
+		req_bw = parm->req_bw;
+		req_offset = parm->req_offset;
+		goto chbw_decision;
+	} else {
+		/* request comes from upper layer */
+		if (MLME_IS_AP(padapter))
+			mlme_act = MLME_AP_STARTED;
+		else if (MLME_IS_MESH(padapter))
+			mlme_act = MLME_MESH_STARTED;
+		else
+			rtw_warn_on(1);
+		req_ch = 0;
+		/* MLD network info copied from adapter->mlmepriv.cur_network */
+		_rtw_memcpy(&padapter->mlmeextpriv.mlmext_info.dev_network,
+			&padapter->mlmepriv.dev_cur_network.network,
+			padapter->mlmepriv.dev_cur_network.network.Length);
+		/* Link network info copied from adapter_link->mlmepriv.cur_network */
+		_rtw_memcpy(pnetwork_mlmeext, pnetwork, pnetwork->Length);
+	}
+
+	bcn_interval = (u16)pnetwork->Configuration.BeaconPeriod;
+	pmlmeinfo->bcn_interval = bcn_interval;
+
+	/* check if there is wps ie, */
+	/* if there is wpsie in beacon, the hostapd will update beacon twice when stating hostapd, */
+	/* and at first time the security ie ( RSN/WPA IE) will not include in beacon. */
+	if (NULL == rtw_get_wps_ie(pnetwork->IEs + _FIXED_IE_LENGTH_, pnetwork->IELength - _FIXED_IE_LENGTH_, NULL, NULL))
+		pmlmeext->bstart_bss = _TRUE;
+
+	/* todo: update wmm, ht cap */
+	/* pmlmeinfo->WMM_enable; */
+	/* pmlmeinfo->HT_enable; */
+	if (pmlmepriv->qospriv.qos_option)
+		pmlmeinfo->WMM_enable = _TRUE;
+
+#ifdef CONFIG_80211N_HT
+	if (pmlmepriv->htpriv.ht_option) {
+		pmlmeinfo->WMM_enable = _TRUE;
+		pmlmeinfo->HT_enable = _TRUE;
+	} else
+		pmlmeinfo->HT_enable = _FALSE;
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	if (pmlmepriv->vhtpriv.vht_option)
+		pmlmeinfo->VHT_enable = _TRUE;
+	else
+		pmlmeinfo->VHT_enable = _FALSE;
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	if (pmlmepriv->hepriv.he_option)
+		pmlmeinfo->HE_enable = _TRUE;
+	else
+		pmlmeinfo->HE_enable = _FALSE;
+#endif
+
+	if (pmlmepriv->cur_network.join_res != _TRUE) { /* setting only at  first time */
+		/* WEP Key will be set before this function, do not clear CAM. */
+		if ((psecuritypriv->dot11PrivacyAlgrthm != _WEP40_) && (psecuritypriv->dot11PrivacyAlgrthm != _WEP104_)
+			&& !MLME_IS_MESH(padapter) /* mesh group key is set before this function */
+		)
+			flush_all_cam_entry(padapter, PHL_CMD_DIRECTLY, 0);	/* clear CAM */
+	}
+
+#ifdef CONFIG_RTW_MULTI_AP
+	rtw_map_config_monitor(padapter, mlme_act);
+#endif
+
+chbw_decision:
+	/*SCC && MCC at the same hw band*/
+	ifbmp_ch_changed = rtw_ap_bchbw_decision(padapter, parm->ifbmp, parm->excl_ifbmp, req_band
+						, req_ch, req_bw, req_offset, &chdef_to_set);
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (!(parm->ifbmp & BIT(i)) || !dvobj->padapters[i])
+			continue;
+		iface = dvobj->padapters[i];
+
+		if (!rtw_iface_at_same_hwband(padapter, iface))
+			continue;
+
+		/* let pnetwork_mlme == pnetwork_mlmeext */
+		/* MLD network info copied from adapter */
+		_rtw_memcpy(&(iface->mlmepriv.dev_cur_network.network)
+			, &(iface->mlmeextpriv.mlmext_info.dev_network)
+			, iface->mlmeextpriv.mlmext_info.dev_network.Length);
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		/* Link network info copied from adapter_link */
+		iface_link = GET_PRIMARY_LINK(dvobj->padapters[i]);
+		_rtw_memcpy(&(iface_link->mlmepriv.cur_network.network)
+			, &(iface_link->mlmeextpriv.mlmext_info.network)
+			, iface_link->mlmeextpriv.mlmext_info.network.Length);
+
+		rtw_start_bss_hdl_after_chbw_decided(iface, iface_link);
+
+		/* Set EDCA param reg after update cur_wireless_mode & update_capinfo */
+		if (pregpriv->wifi_spec == 1)
+			rtw_set_hw_wmm_param(iface, iface_link);
+
+		rtw_ap_link_regu_forbid_update(iface_link, false);
+	}
+
+	new_chdef.band = pmlmeext->chandef.band;
+	new_chdef.chan = pmlmeext->chandef.chan;
+	new_chdef.bw = pmlmeext->chandef.bw;
+	new_chdef.offset = pmlmeext->chandef.offset;
+
+	if (!padapter_link->is_ap_chan_ctx_added) {
+		/* add chandef first time */
+		padapter_link->is_ap_chan_ctx_added = rtw_phl_chanctx_add(dvobj->phl, padapter->phl_role,
+					padapter_link->wrlink, &new_chdef, &mr_cc_info);
+
+	} else if (parm->is_change_chbw ||
+		padapter_link->wrlink->chandef.band != new_chdef.band ||
+		padapter_link->wrlink->chandef.chan != new_chdef.chan ||
+		padapter_link->wrlink->chandef.bw != new_chdef.bw ||
+		padapter_link->wrlink->chandef.offset != new_chdef.offset) {
+		/* change chandef */
+		rtw_phl_cmd_stop_beacon(dvobj->phl, padapter_link->wrlink,
+					_TRUE, PHL_CMD_DIRECTLY, 0);
+		rtw_phl_chanctx_del(dvobj->phl, padapter->phl_role, padapter_link->wrlink, NULL);
+		padapter_link->is_ap_chan_ctx_added = rtw_phl_chanctx_add(dvobj->phl, padapter->phl_role,
+					padapter_link->wrlink, &new_chdef, &mr_cc_info);
+	} else {
+		/* chandef doesn't change, just get the union chandef */
+		rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role, padapter_link->wrlink, &new_chdef);
+	}
+
+	rtw_hw_update_chan_def(padapter, padapter_link);
+
+#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+	rtw_dfs_rd_en_dec_on_mlme_act(padapter, padapter_link, mlme_act, parm->excl_ifbmp);
+#endif
+	RTW_INFO("%s: is_ap_chan_ctx_added:%s\n", __func__, (padapter_link->is_ap_chan_ctx_added) ? "Y" : "N");
+	RTW_INFO("PHL union_chdef - BAND: %d, CH:%d, BW:%d OFF:%d\n", new_chdef.band, new_chdef.chan, new_chdef.bw, new_chdef.offset);
+	if (chdef_to_set.chan != new_chdef.chan || chdef_to_set.band != new_chdef.band) {
+		RTW_INFO("Core - BAND:%d, CH:%d, BW:%d OFF:%d\n", chdef_to_set.band, chdef_to_set.chan, chdef_to_set.bw, chdef_to_set.offset);
+		RTW_ERR("chan set mismatch!!!\n");
+	}
+
+	if (padapter_link->is_ap_chan_ctx_added == false) {
+		RTW_ERR("%s : rtw_phl_chanctx_add fail\n", __func__);
+		if (0)
+			rtw_warn_on(1);
+	}
+
+_exit:
+	parm->ifbmp_ch_changed = ifbmp_ch_changed;
+	parm->req_band = new_chdef.band;
+	parm->ch_to_set = new_chdef.chan;
+	parm->offset_to_set = new_chdef.offset;
+	parm->bw_to_set = new_chdef.bw;
+}
+
+int rtw_check_beacon_data(_adapter *padapter, u8 *pbuf,  int len)
+{
+	int ret = _SUCCESS;
+	u8 *p;
+	u8 *pHT_caps_ie = NULL;
+	u8 *pHT_info_ie = NULL;
+	u16 cap, ht_cap = _FALSE;
+	uint ie_len = 0;
+	int group_cipher, pairwise_cipher, gmcs;
+	u32 akm;
+	u8 mfp_opt = MFP_NO;
+	u8 channel = 0, network_type;
+	u8 OUI1[] = {0x00, 0x50, 0xf2, 0x01};
+	u8 WMM_PARA_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x01, 0x01};
+	u8 WIFI_ALLIANCE_OUI[] = {0x50, 0x6f, 0x9a};
+	HT_CAP_AMPDU_DENSITY best_ampdu_density = 0;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	WLAN_BSSID_EX *pbss_network = (WLAN_BSSID_EX *)&pmlmepriv->cur_network.network;
+	u8 *ie = pbss_network->IEs;
+	u8 vht_cap = _FALSE;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct rtw_chan_def chandef = {0};
+	enum band_type band = BAND_ON_24G;
+	u8 rf_num = 0;
+	int ret_rm;
+	/* SSID */
+	/* Supported rates */
+	/* DS Params */
+	/* WLAN_EID_COUNTRY */
+	/* ERP Information element */
+	/* Extended supported rates */
+	/* WPA/WPA2 */
+	/* Radio Resource Management */
+	/* Wi-Fi Wireless Multimedia Extensions */
+	/* ht_capab, ht_oper */
+	/* WPS IE */
+
+	RTW_INFO("%s, len=%d\n", __FUNCTION__, len);
+
+	if (!MLME_IS_AP(padapter) && !MLME_IS_MESH(padapter))
+		return _FAIL;
+
+
+	if (len > MAX_IE_SZ)
+		return _FAIL;
+
+	pbss_network->IELength = len;
+
+	_rtw_memset(ie, 0, MAX_IE_SZ);
+
+	_rtw_memcpy(ie, pbuf, pbss_network->IELength);
+
+
+	if (padapter->mlmepriv.dev_cur_network.network.InfrastructureMode != Ndis802_11APMode
+		&& padapter->mlmepriv.dev_cur_network.network.InfrastructureMode != Ndis802_11_mesh
+	) {
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+
+
+	rtw_ap_check_scan(padapter, padapter_link);
+
+
+	pbss_network->PhyInfo.rssi= 0;
+
+	/* MLD mac address */
+	_rtw_memcpy(padapter->mlmepriv.dev_cur_network.network.MacAddress, adapter_mac_addr(padapter), ETH_ALEN);
+	/* Link mac address */
+	_rtw_memcpy(pbss_network->MacAddress, padapter_link->mac_addr, ETH_ALEN);
+
+	/* beacon interval */
+	p = rtw_get_beacon_interval_from_ie(ie);/* ie + 8;	 */ /* 8: TimeStamp, 2: Beacon Interval 2:Capability */
+	/* pbss_network->Configuration.BeaconPeriod = le16_to_cpu(*(unsigned short*)p); */
+	pbss_network->Configuration.BeaconPeriod = RTW_GET_LE16(p);
+
+	/* capability */
+	p = rtw_get_capability_from_ie(ie);
+	cap = RTW_GET_LE16(p);
+
+	/* SSID */
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _SSID_IE_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+	if (p && ie_len > 0) {
+		_rtw_memset(&pbss_network->Ssid, 0, sizeof(NDIS_802_11_SSID));
+		_rtw_memcpy(pbss_network->Ssid.Ssid, (p + 2), ie_len);
+		pbss_network->Ssid.SsidLength = ie_len;
+	}
+
+#ifdef CONFIG_RTW_MESH
+	/* Mesh ID */
+	if (MLME_IS_MESH(padapter)) {
+		p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, WLAN_EID_MESH_ID, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+		if (p && ie_len > 0) {
+			_rtw_memset(&pbss_network->mesh_id, 0, sizeof(NDIS_802_11_SSID));
+			_rtw_memcpy(pbss_network->mesh_id.Ssid, (p + 2), ie_len);
+			pbss_network->mesh_id.SsidLength = ie_len;
+		}
+	}
+#endif
+
+	/* Parsing extended capabilities IE */
+	rtw_parse_ext_cap_ie(pmlmepriv->ext_capab_ie_data, &(pmlmepriv->ext_capab_ie_len), ie \
+		, pbss_network->IELength, _BEACON_IE_OFFSET_);
+
+	/* channel and band */
+	rtw_ies_get_bchbw(ie + _BEACON_IE_OFFSET_, pbss_network->IELength - _BEACON_IE_OFFSET_,
+		&chandef.band, &chandef.chan, (u8 *)&chandef.bw, (u8 *)&chandef.offset,
+		(u8 *)&chandef.center_freq1, (u8 *)&chandef.center_freq2, 1, 1, 1);
+
+	if (chandef.chan != 0) {
+		pbss_network->Configuration.Band = band = chandef.band;
+		pbss_network->Configuration.DSConfig = channel = chandef.chan;
+	}
+
+	if (pbss_network->Configuration.Band == BAND_ON_6G)
+		/* DSSET uses in 2.4G and 5G band; Remove DSSET when using 6G band */
+		rtw_remove_bcn_ie(padapter, pbss_network, _DSSET_IE_);
+
+	/*	support rate ie & ext support ie & IElen & SupportedRates	*/
+	network_type = rtw_update_rate_bymode(pbss_network, pregistrypriv->wireless_mode);
+
+	/* parsing ERP_IE */
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _ERPINFO_IE_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+	if (p && ie_len > 0)  {
+		if(pregistrypriv->wireless_mode == WLAN_MD_11B)
+		{
+			pbss_network->IELength = pbss_network->IELength - *(p+1) - 2;
+			ret_rm = rtw_ies_remove_ie(ie , &len, _BEACON_IE_OFFSET_, _ERPINFO_IE_,NULL,0);
+			RTW_DBG("%s, remove_ie of ERP_IE=%d\n", __FUNCTION__, ret_rm);
+		} else
+			ERP_IE_handler(padapter, padapter_link, (PNDIS_802_11_VARIABLE_IEs)p);
+
+	}
+
+	/* update privacy/security */
+	if (cap & BIT(4))
+		pbss_network->Privacy = 1;
+	else
+		pbss_network->Privacy = 0;
+
+	psecuritypriv->wpa_psk = 0;
+
+	/* wpa2 */
+	akm = 0;
+	gmcs = 0;
+	group_cipher = 0;
+	pairwise_cipher = 0;
+	psecuritypriv->wpa2_group_cipher = _NO_PRIVACY_;
+	psecuritypriv->wpa2_pairwise_cipher = _NO_PRIVACY_;
+	psecuritypriv->akmp = 0;
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _RSN_IE_2_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+	if (p && ie_len > 0) {
+		if (rtw_parse_wpa2_ie(p, ie_len + 2, &group_cipher, &pairwise_cipher, &gmcs, &akm, &mfp_opt, NULL) == _SUCCESS) {
+			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPA2PSK;
+			psecuritypriv->dot8021xalg = 1;/* psk,  todo:802.1x */
+			psecuritypriv->wpa_psk |= BIT(1);
+
+			psecuritypriv->wpa2_group_cipher = group_cipher;
+			psecuritypriv->wpa2_pairwise_cipher = pairwise_cipher;
+			psecuritypriv->akmp = akm;
+
+#ifdef CONFIG_IOCTL_CFG80211
+			/**
+			 * Kernel < v5.x, the auth_type set as
+			 * NL80211_AUTHTYPE_AUTOMATIC in
+			 * cfg80211_rtw_start_ap(). if the AKM SAE in the RSN
+			 * IE, we have to update the auth_type for SAE in
+			 * rtw_check_beacon_data()
+			 */
+			if (CHECK_BIT(WLAN_AKM_TYPE_SAE, akm)) {
+				RTW_INFO("%s: Auth type as SAE\n", __func__);
+				psecuritypriv->auth_type = MLME_AUTHTYPE_SAE;
+				psecuritypriv->auth_alg = WLAN_AUTH_SAE;
+			}
+#endif /* CONFIG_IOCTL_CFG80211 */
+#if 0
+			switch (group_cipher) {
+			case WPA_CIPHER_NONE:
+				psecuritypriv->wpa2_group_cipher = _NO_PRIVACY_;
+				break;
+			case WPA_CIPHER_WEP40:
+				psecuritypriv->wpa2_group_cipher = _WEP40_;
+				break;
+			case WPA_CIPHER_TKIP:
+				psecuritypriv->wpa2_group_cipher = _TKIP_;
+				break;
+			case WPA_CIPHER_CCMP:
+				psecuritypriv->wpa2_group_cipher = _AES_;
+				break;
+			case WPA_CIPHER_WEP104:
+				psecuritypriv->wpa2_group_cipher = _WEP104_;
+				break;
+			}
+
+			switch (pairwise_cipher) {
+			case WPA_CIPHER_NONE:
+				psecuritypriv->wpa2_pairwise_cipher = _NO_PRIVACY_;
+				break;
+			case WPA_CIPHER_WEP40:
+				psecuritypriv->wpa2_pairwise_cipher = _WEP40_;
+				break;
+			case WPA_CIPHER_TKIP:
+				psecuritypriv->wpa2_pairwise_cipher = _TKIP_;
+				break;
+			case WPA_CIPHER_CCMP:
+				psecuritypriv->wpa2_pairwise_cipher = _AES_;
+				break;
+			case WPA_CIPHER_WEP104:
+				psecuritypriv->wpa2_pairwise_cipher = _WEP104_;
+				break;
+			}
+#endif
+		}
+
+	}
+
+	/* wpa */
+	ie_len = 0;
+	group_cipher = 0;
+	pairwise_cipher = 0;
+	psecuritypriv->wpa_group_cipher = _NO_PRIVACY_;
+	psecuritypriv->wpa_pairwise_cipher = _NO_PRIVACY_;
+	for (p = ie + _BEACON_IE_OFFSET_; ; p += (ie_len + 2)) {
+		p = rtw_get_ie(p, _SSN_IE_1_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_ - (ie_len + 2)));
+		if ((p) && (_rtw_memcmp(p + 2, OUI1, 4))) {
+			if (rtw_parse_wpa_ie(p, ie_len + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
+				psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+				psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPAPSK;
+				psecuritypriv->dot8021xalg = 1;/* psk,  todo:802.1x */
+
+				psecuritypriv->wpa_psk |= BIT(0);
+
+				psecuritypriv->wpa_group_cipher = group_cipher;
+				psecuritypriv->wpa_pairwise_cipher = pairwise_cipher;
+
+#if 0
+				switch (group_cipher) {
+				case WPA_CIPHER_NONE:
+					psecuritypriv->wpa_group_cipher = _NO_PRIVACY_;
+					break;
+				case WPA_CIPHER_WEP40:
+					psecuritypriv->wpa_group_cipher = _WEP40_;
+					break;
+				case WPA_CIPHER_TKIP:
+					psecuritypriv->wpa_group_cipher = _TKIP_;
+					break;
+				case WPA_CIPHER_CCMP:
+					psecuritypriv->wpa_group_cipher = _AES_;
+					break;
+				case WPA_CIPHER_WEP104:
+					psecuritypriv->wpa_group_cipher = _WEP104_;
+					break;
+				}
+
+				switch (pairwise_cipher) {
+				case WPA_CIPHER_NONE:
+					psecuritypriv->wpa_pairwise_cipher = _NO_PRIVACY_;
+					break;
+				case WPA_CIPHER_WEP40:
+					psecuritypriv->wpa_pairwise_cipher = _WEP40_;
+					break;
+				case WPA_CIPHER_TKIP:
+					psecuritypriv->wpa_pairwise_cipher = _TKIP_;
+					break;
+				case WPA_CIPHER_CCMP:
+					psecuritypriv->wpa_pairwise_cipher = _AES_;
+					break;
+				case WPA_CIPHER_WEP104:
+					psecuritypriv->wpa_pairwise_cipher = _WEP104_;
+					break;
+				}
+#endif
+			}
+
+			break;
+
+		}
+
+		if ((p == NULL) || (ie_len == 0))
+			break;
+
+	}
+
+	if (mfp_opt == MFP_INVALID) {
+		RTW_INFO(FUNC_ADPT_FMT" invalid MFP setting\n", FUNC_ADPT_ARG(padapter));
+		return _FAIL;
+	}
+	psecuritypriv->mfp_opt = mfp_opt;
+
+#ifdef CONFIG_RTW_80211K
+	rm_update_cap(pbuf, padapter, len, _BEACON_IE_OFFSET_);
+#endif
+#ifdef CONFIG_RTW_FSM_RRM
+	rrm_update_cap(pbuf, padapter, len, _BEACON_IE_OFFSET_);
+#endif
+	/* wmm */
+	ie_len = 0;
+	pmlmepriv->qospriv.qos_option = 0;
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter))
+		pmlmepriv->qospriv.qos_option = 1;
+#endif
+	if (pregistrypriv->wmm_enable) {
+		for (p = ie + _BEACON_IE_OFFSET_; ; p += (ie_len + 2)) {
+			p = rtw_get_ie(p, _VENDOR_SPECIFIC_IE_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_ - (ie_len + 2)));
+			if ((p) && _rtw_memcmp(p + 2, WMM_PARA_IE, 6)) {
+				pmlmepriv->qospriv.qos_option = 1;
+
+				*(p + 8) |= BIT(7); /* QoS Info, support U-APSD */
+
+				/* disable all ACM bits since the WMM admission control is not supported */
+				*(p + 10) &= ~BIT(4); /* BE */
+				*(p + 14) &= ~BIT(4); /* BK */
+				*(p + 18) &= ~BIT(4); /* VI */
+				*(p + 22) &= ~BIT(4); /* VO */
+
+				WMM_param_handler(padapter, padapter_link,
+						(PNDIS_802_11_VARIABLE_IEs)p);
+
+				break;
+			}
+
+			if ((p == NULL) || (ie_len == 0))
+				break;
+		}
+	}
+#ifdef CONFIG_80211N_HT
+	if(pregistrypriv->ht_enable && is_supported_ht(pregistrypriv->wireless_mode))
+	{
+		/* parsing HT_CAP_IE */
+		p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_CAPABILITY_IE_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+		if (p && ie_len > 0) {
+			HT_CAP_AMPDU_FACTOR max_rx_ampdu_factor = MAX_AMPDU_FACTOR_64K;
+			struct rtw_ieee80211_ht_cap *pht_cap = (struct rtw_ieee80211_ht_cap *)(p + 2);
+
+			if (0) {
+				RTW_INFO(FUNC_ADPT_FMT" HT_CAP_IE from upper layer:\n", FUNC_ADPT_ARG(padapter));
+				dump_ht_cap_ie_content(RTW_DBGDUMP, p + 2, ie_len);
+			}
+
+			pHT_caps_ie = p;
+
+			ht_cap = _TRUE;
+			network_type |= WLAN_MD_11N;
+
+			rtw_ht_use_default_setting(padapter, padapter_link);
+
+			/* Update HT Capabilities Info field */
+			if (pmlmepriv->htpriv.sgi_20m == _FALSE)
+				pht_cap->cap_info &= ~(IEEE80211_HT_CAP_SGI_20);
+
+			if (pmlmepriv->htpriv.sgi_40m == _FALSE)
+				pht_cap->cap_info &= ~(IEEE80211_HT_CAP_SGI_40);
+
+			if (!TEST_FLAG(pmlmepriv->htpriv.ldpc_cap, LDPC_HT_ENABLE_RX))
+				pht_cap->cap_info &= ~(IEEE80211_HT_CAP_LDPC_CODING);
+
+			if (!TEST_FLAG(pmlmepriv->htpriv.stbc_cap, STBC_HT_ENABLE_TX))
+				pht_cap->cap_info &= ~(IEEE80211_HT_CAP_TX_STBC);
+
+			if (!TEST_FLAG(pmlmepriv->htpriv.stbc_cap, STBC_HT_ENABLE_RX))
+				pht_cap->cap_info &= ~(IEEE80211_HT_CAP_RX_STBC_3R);
+
+			/* Update A-MPDU Parameters field */
+			pht_cap->ampdu_params_info &= ~(IEEE80211_HT_CAP_AMPDU_FACTOR | IEEE80211_HT_CAP_AMPDU_DENSITY);
+
+			if ((psecuritypriv->wpa_pairwise_cipher & WPA_CIPHER_CCMP) ||
+				(psecuritypriv->wpa2_pairwise_cipher & WPA_CIPHER_CCMP)) {
+				rtw_hal_get_def_var(padapter, padapter_link, HW_VAR_BEST_AMPDU_DENSITY, &best_ampdu_density);
+				pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & (best_ampdu_density << 2));
+			} else
+				pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & 0x00);
+
+			rtw_hal_get_def_var(padapter, padapter_link, HW_VAR_MAX_RX_AMPDU_FACTOR, &max_rx_ampdu_factor);
+			pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_FACTOR & max_rx_ampdu_factor); /* set  Max Rx AMPDU size  to 64K */
+
+			_rtw_memcpy(&(pmlmeinfo->HT_caps), pht_cap, sizeof(struct HT_caps_element));
+
+			/* Update Supported MCS Set field */
+			{
+				u8 rx_nss = 0;
+				int i;
+
+				rx_nss = get_phy_rx_nss(padapter, padapter_link);
+
+				/* RX MCS Bitmask */
+				switch (rx_nss) {
+				case 1:
+					set_mcs_rate_by_mask(HT_CAP_ELE_RX_MCS_MAP(pht_cap), MCS_RATE_1R);
+					break;
+				case 2:
+					set_mcs_rate_by_mask(HT_CAP_ELE_RX_MCS_MAP(pht_cap), MCS_RATE_2R);
+					break;
+				case 3:
+					set_mcs_rate_by_mask(HT_CAP_ELE_RX_MCS_MAP(pht_cap), MCS_RATE_3R);
+					break;
+				case 4:
+					set_mcs_rate_by_mask(HT_CAP_ELE_RX_MCS_MAP(pht_cap), MCS_RATE_4R);
+					break;
+				default:
+					RTW_WARN("rf_type:%d or rx_nss:%u is not expected\n",
+						GET_HAL_RFPATH(adapter_to_dvobj(padapter)), rx_nss);
+				}
+				for (i = 0; i < 10; i++)
+					*(HT_CAP_ELE_RX_MCS_MAP(pht_cap) + i) &= pmlmeext->default_supported_mcs_set[i];
+			}
+
+#ifdef CONFIG_BEAMFORMING
+			/* Use registry value to enable HT Beamforming. */
+			/* ToDo: use configure file to set these capability. */
+			pht_cap->tx_BF_cap_info = 0;
+
+			/* HT Beamformer */
+			if (TEST_FLAG(pmlmepriv->htpriv.beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE)) {
+				/* Transmit NDP Capable */
+				SET_HT_CAP_TXBF_TRANSMIT_NDP_CAP(pht_cap, 1);
+				/* Explicit Compressed Steering Capable */
+				SET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP(pht_cap, 1);
+				/* Compressed Steering Number Antennas */
+				SET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(pht_cap, 1);
+				rtw_hal_get_def_var(padapter, padapter_link, HAL_DEF_BEAMFORMER_CAP, (u8 *)&rf_num);
+				if (rf_num > 3)
+					rf_num = 3;
+				SET_HT_CAP_TXBF_CHNL_ESTIMATION_NUM_ANTENNAS(pht_cap, rf_num);
+			}
+
+			/* HT Beamformee */
+			if (TEST_FLAG(pmlmepriv->htpriv.beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE)) {
+				/* Receive NDP Capable */
+				SET_HT_CAP_TXBF_RECEIVE_NDP_CAP(pht_cap, 1);
+				/* Explicit Compressed Beamforming Feedback Capable */
+				SET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP(pht_cap, 2);
+				rtw_hal_get_def_var(padapter, padapter_link, HAL_DEF_BEAMFORMEE_CAP, (u8 *)&rf_num);
+				if (rf_num > 3)
+					rf_num = 3;
+				SET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(pht_cap, rf_num);
+			}
+#endif /* CONFIG_BEAMFORMING */
+
+			_rtw_memcpy(&pmlmepriv->htpriv.ht_cap, p + 2, ie_len);
+
+			if (0) {
+				RTW_INFO(FUNC_ADPT_FMT" HT_CAP_IE driver masked:\n", FUNC_ADPT_ARG(padapter));
+				dump_ht_cap_ie_content(RTW_DBGDUMP, p + 2, ie_len);
+			}
+		}
+
+		/* parsing HT_INFO_IE */
+		p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_ADD_INFO_IE_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+		if (p && ie_len > 0) {
+			pHT_info_ie = p;
+			if (channel == 0)
+				pbss_network->Configuration.DSConfig = GET_HT_OP_ELE_PRI_CHL(pHT_info_ie + 2);
+			else if (channel != GET_HT_OP_ELE_PRI_CHL(pHT_info_ie + 2)) {
+				RTW_INFO(FUNC_ADPT_FMT" ch inconsistent, DSSS:%u, HT primary:%u\n"
+					, FUNC_ADPT_ARG(padapter), channel, GET_HT_OP_ELE_PRI_CHL(pHT_info_ie + 2));
+			}
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+	pmlmepriv->cur_network.network_type = network_type;
+
+#ifdef CONFIG_80211N_HT
+	pmlmepriv->htpriv.ht_option = _FALSE;
+
+	if ((psecuritypriv->wpa2_pairwise_cipher & WPA_CIPHER_TKIP) ||
+	    (psecuritypriv->wpa_pairwise_cipher & WPA_CIPHER_TKIP)) {
+		/* todo: */
+		/* ht_cap = _FALSE; */
+	}
+
+	/* ht_cap	 */
+	if (pregistrypriv->ht_enable &&
+		is_supported_ht(pregistrypriv->wireless_mode) && ht_cap == _TRUE &&
+		(pbss_network->Configuration.Band == BAND_ON_24G ||
+		 pbss_network->Configuration.Band == BAND_ON_5G)) {
+
+		pmlmepriv->htpriv.ht_option = _TRUE;
+		pmlmepriv->qospriv.qos_option = 1;
+
+		pmlmepriv->ampdu_priv.ampdu_enable = pregistrypriv->ampdu_enable ? _TRUE : _FALSE;
+
+		HT_caps_handler(padapter, padapter_link, (PNDIS_802_11_VARIABLE_IEs)pHT_caps_ie);
+
+		HT_info_handler(padapter, padapter_link, (PNDIS_802_11_VARIABLE_IEs)pHT_info_ie);
+	}
+#endif
+
+	pmlmepriv->upper_layer_setting = _FALSE;
+
+#ifdef CONFIG_80211AC_VHT
+	pmlmepriv->vhtpriv.vht_option = _FALSE;
+
+	if (pmlmepriv->htpriv.ht_option == _TRUE
+		&& pbss_network->Configuration.Band == BAND_ON_5G
+		&& REGSTY_IS_11AC_ENABLE(pregistrypriv)
+		&& is_supported_vht(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AC(rfctl)) {
+		/* Parsing VHT_CAP_IE */
+		p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, WLAN_EID_VHT_CAPABILITY,
+			&ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+		if (p && ie_len > 0)
+			vht_cap = _TRUE;
+		else
+			RTW_INFO(FUNC_ADPT_FMT" No vht_capability_ie from hostapd/wpa_supplicant\n", FUNC_ADPT_ARG(padapter));
+
+		/* Update VHT related IEs */
+		if (vht_cap == _TRUE) {
+			RTW_INFO(FUNC_ADPT_FMT" VHT IEs is configured by hostapd/wpa_supplicant\n", FUNC_ADPT_ARG(padapter));
+			pmlmepriv->upper_layer_setting = _TRUE;
+			pmlmepriv->vhtpriv.vht_option = _TRUE;
+
+			rtw_check_for_vht20(padapter, ie + _BEACON_IE_OFFSET_,
+					pbss_network->IELength - _BEACON_IE_OFFSET_);
+			rtw_check_vht_ies(padapter, padapter_link, pbss_network);
+		}
+		else if (REGSTY_IS_11AC_AUTO(pregistrypriv)) {
+			rtw_vht_ies_detach(padapter, padapter_link, pbss_network);
+			rtw_vht_ies_attach(padapter, padapter_link, pbss_network);
+		}
+	}
+
+	if (pmlmepriv->vhtpriv.vht_option == _FALSE)
+		rtw_vht_ies_detach(padapter, padapter_link, pbss_network);
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+	pmlmepriv->hepriv.he_option = _FALSE;
+
+	/* An HE STA is also a VHT STA if operating in the 5 GHz band */
+	/* An HE STA is also a HT STA in the 2GHz band */
+
+	if (REGSTY_IS_11AX_ENABLE(pregistrypriv)
+		&& is_supported_he(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AX(rfctl)
+		&& ((band == BAND_ON_5G && pmlmepriv->vhtpriv.vht_option == _TRUE)
+		|| (band == BAND_ON_24G && pmlmepriv->htpriv.ht_option == _TRUE)
+		|| (band == BAND_ON_6G))) {
+		u8 he_cap = _FALSE;
+		u8 he_cap_eid_ext = WLAN_EID_EXTENSION_HE_CAPABILITY;
+
+		p = rtw_get_ie_ex(ie + _BEACON_IE_OFFSET_, pbss_network->IELength - _BEACON_IE_OFFSET_,
+			WLAN_EID_EXTENSION, &he_cap_eid_ext, 1, NULL, &ie_len);
+		if (p && ie_len > 0)
+			he_cap = _TRUE;
+
+		/* If He capability is in beacon IE, enable he_option */
+		pmlmepriv->hepriv.he_option = he_cap;
+
+		/* Update HE related IEs */
+		if (he_cap == _TRUE) {
+			RTW_INFO(FUNC_ADPT_FMT" HE IEs is configured by hostapd/wpa_supplicant\n", FUNC_ADPT_ARG(padapter));
+			pmlmepriv->upper_layer_setting = _TRUE;
+
+			rtw_update_he_ies(padapter, padapter_link, pbss_network, &chandef);
+		} else if (REGSTY_IS_11AX_AUTO(pregistrypriv)) {
+			rtw_he_ies_detach(padapter, padapter_link, pbss_network);
+			rtw_he_ies_attach(padapter, padapter_link, pbss_network, band);
+		}
+	}
+	if (pmlmepriv->hepriv.he_option == _FALSE)
+		rtw_he_ies_detach(padapter, padapter_link, pbss_network);
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+	padapter->mlmepriv.mlopriv.mlo_option = _FALSE;
+	/* ToDo CONFIG_RTW_MLD: check if hostapd will provide hints to enable mlo */
+#ifdef CONFIG_MLD_TEST
+	padapter->mlmepriv.mlopriv.mlo_option = _TRUE;
+#endif
+#endif
+
+#ifdef CONFIG_80211N_HT
+	if(pregistrypriv->ht_enable && is_supported_ht(pregistrypriv->wireless_mode)
+			&& pbss_network->Configuration.DSConfig <= 14 /*&& padapter->registrypriv.wifi_spec == 1*/)
+	{
+	#ifdef CONFIG_RTW_WNM
+		rtw_wnm_add_btm_ext_cap(pmlmepriv->ext_capab_ie_data,
+				&(pmlmepriv->ext_capab_ie_len));
+	#endif
+	#ifdef CONFIG_RTW_MBO
+		rtw_mbo_add_internw_ext_cap(pmlmepriv->ext_capab_ie_data,
+				&(pmlmepriv->ext_capab_ie_len));
+	#endif
+		rtw_add_ext_cap_info(pmlmepriv->ext_capab_ie_data, &(pmlmepriv->ext_capab_ie_len), BSS_COEXT);
+		rtw_update_ext_cap_ie(pmlmepriv->ext_capab_ie_data, pmlmepriv->ext_capab_ie_len, pbss_network->IEs, \
+			&(pbss_network->IELength), _BEACON_IE_OFFSET_);
+	}
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_RTW_80211K
+	padapter->rmpriv.enable = _FALSE;
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, EID_RMEnabledCapability, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+	if (p && ie_len) {
+		RTW_INFO("[%s]Get EID_RMEnabledCapability, len = %d\n", __func__, ie_len);
+		padapter->rmpriv.enable = _TRUE;
+		_rtw_memcpy(padapter->rmpriv.rm_en_cap_def, p + 2, ie_len);
+	}
+#endif /* CONFIG_RTW_80211K */
+
+#ifdef CONFIG_RTW_FSM_RRM
+	padapter->fsmpriv.rmpriv.enable = _FALSE;
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, EID_RMEnabledCapability, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+	if (p && ie_len) {
+		RTW_INFO("[%s]Get EID_RMEnabledCapability, len = %d\n", __func__, ie_len);
+		padapter->fsmpriv.rmpriv.enable = _TRUE;
+		_rtw_memcpy(padapter->fsmpriv.rmpriv.rm_en_cap_def, p + 2, ie_len);
+	}
+#endif /* CONFIG_RTW_FSM_RRM */
+
+#ifdef CONFIG_RTW_MBO
+	ie_len = 0;
+	padapter->mlmepriv.mbopriv.enable = _FALSE;
+	for (p = ie + _BEACON_IE_OFFSET_; ; p += (ie_len + 2)) {
+		p = rtw_get_ie(p, _SSN_IE_1_, &ie_len, (pbss_network->IELength - _BEACON_IE_OFFSET_ - (ie_len + 2)));
+		if ((p) && (_rtw_memcmp(p + 2, WIFI_ALLIANCE_OUI, 3)) && (*(p+5) == MBO_OUI_TYPE)) {
+			/* find MBO-OCE information element */
+			padapter->mlmepriv.mbopriv.enable = _TRUE;
+			rtw_mbo_ie_handler(padapter, &padapter->mlmepriv.mbopriv, p + 6, ie_len - 4);
+			break;
+		}
+		if ((p == NULL) || (ie_len == 0))
+			break;
+	}
+#endif /* CONFIG_RTW_MBO */
+
+	/* Build supported operating class element if ECSA enabled*/
+#ifdef CONFIG_ECSA_PHL
+	if (rtw_is_ecsa_enabled(padapter)) {
+		u8 buf[32];
+		ie_len = get_supported_op_class(padapter, buf, sizeof(buf));
+		rtw_add_bcn_ie(padapter, pbss_network, WLAN_EID_SUPPORT_OP_CLASS,
+					buf, ie_len);
+
+		rtw_add_ext_cap_info(pmlmepriv->ext_capab_ie_data,
+				&(pmlmepriv->ext_capab_ie_len), EXT_CH_SWITCH);
+		rtw_update_ext_cap_ie(pmlmepriv->ext_capab_ie_data,
+				pmlmepriv->ext_capab_ie_len, pbss_network->IEs,
+				&(pbss_network->IELength), _BEACON_IE_OFFSET_);
+	}
+#endif
+
+	pbss_network->Length = get_WLAN_BSSID_EX_sz((WLAN_BSSID_EX *)pbss_network);
+
+	rtw_ies_get_bchbw(pbss_network->IEs + _BEACON_IE_OFFSET_,
+		pbss_network->IELength - _BEACON_IE_OFFSET_,
+		&pmlmepriv->ori_chandef.band,
+		&pmlmepriv->ori_chandef.chan,
+		(u8 *)&pmlmepriv->ori_chandef.bw,
+		(u8 *)&pmlmepriv->ori_chandef.offset,
+		(u8 *)&pmlmepriv->ori_chandef.center_freq1,
+		(u8 *)&pmlmepriv->ori_chandef.center_freq2,
+		1, 1, 1);
+
+	rtw_warn_on(pmlmepriv->ori_chandef.chan == 0);
+
+	ret = rtw_startbss_cmd(padapter, RTW_CMDF_WAIT_ACK);
+	if (ret == _FAIL)
+		RTW_ERR("%s: start bss cm fail\n", __func__);
+
+	{
+		int sk_band = RTW_GET_SCAN_BAND_SKIP(padapter);
+
+		if (sk_band)
+			RTW_CLR_SCAN_BAND_SKIP(padapter, sk_band);
+	}
+
+	return ret;
+
+}
+
+#if CONFIG_RTW_MACADDR_ACL
+void rtw_macaddr_acl_init(_adapter *adapter, u8 period)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct wlan_acl_pool *acl;
+	_queue *acl_node_q;
+	int i;
+
+	if (period >= RTW_ACL_PERIOD_NUM) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	acl = &stapriv->acl_list[period];
+	acl_node_q = &acl->acl_node_q;
+
+	_rtw_spinlock_init(&(acl_node_q->lock));
+
+	_rtw_spinlock_bh(&(acl_node_q->lock));
+	_rtw_init_listhead(&(acl_node_q->queue));
+	acl->num = 0;
+	acl->mode = RTW_ACL_MODE_DISABLED;
+	for (i = 0; i < NUM_ACL; i++) {
+		_rtw_init_listhead(&acl->aclnode[i].list);
+		acl->aclnode[i].valid = _FALSE;
+	}
+	_rtw_spinunlock_bh(&(acl_node_q->lock));
+}
+
+static void _rtw_macaddr_acl_deinit(_adapter *adapter, u8 period, bool clear_only)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct wlan_acl_pool *acl;
+	_queue *acl_node_q;
+	_list *head, *list;
+	struct rtw_wlan_acl_node *acl_node;
+
+	if (period >= RTW_ACL_PERIOD_NUM) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	acl = &stapriv->acl_list[period];
+	acl_node_q = &acl->acl_node_q;
+
+	_rtw_spinlock_bh(&(acl_node_q->lock));
+	head = get_list_head(acl_node_q);
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		acl_node = LIST_CONTAINOR(list, struct rtw_wlan_acl_node, list);
+		list = get_next(list);
+
+		if (acl_node->valid == _TRUE) {
+			acl_node->valid = _FALSE;
+			rtw_list_delete(&acl_node->list);
+			acl->num--;
+		}
+	}
+	_rtw_spinunlock_bh(&(acl_node_q->lock));
+
+	if (!clear_only)
+		_rtw_spinlock_free(&(acl_node_q->lock));
+
+	rtw_warn_on(acl->num);
+	acl->mode = RTW_ACL_MODE_DISABLED;
+}
+
+void rtw_macaddr_acl_deinit(_adapter *adapter, u8 period)
+{
+	_rtw_macaddr_acl_deinit(adapter, period, 0);
+}
+
+void rtw_macaddr_acl_clear(_adapter *adapter, u8 period)
+{
+	_rtw_macaddr_acl_deinit(adapter, period, 1);
+}
+
+void rtw_set_macaddr_acl(_adapter *adapter, u8 period, int mode)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct wlan_acl_pool *acl;
+
+	if (period >= RTW_ACL_PERIOD_NUM) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	acl = &stapriv->acl_list[period];
+
+	RTW_INFO(FUNC_ADPT_FMT" p=%u, mode=%d\n"
+		, FUNC_ADPT_ARG(adapter), period, mode);
+
+	acl->mode = mode;
+}
+
+int rtw_acl_add_sta(_adapter *adapter, u8 period, const u8 *addr)
+{
+	_list *list, *head;
+	u8 existed = 0;
+	int i = -1, ret = 0;
+	struct rtw_wlan_acl_node *acl_node;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct wlan_acl_pool *acl;
+	_queue *acl_node_q;
+
+	if (period >= RTW_ACL_PERIOD_NUM) {
+		rtw_warn_on(1);
+		ret = -1;
+		goto exit;
+	}
+
+	acl = &stapriv->acl_list[period];
+	acl_node_q = &acl->acl_node_q;
+
+	_rtw_spinlock_bh(&(acl_node_q->lock));
+
+	head = get_list_head(acl_node_q);
+	list = get_next(head);
+
+	/* search for existed entry */
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		acl_node = LIST_CONTAINOR(list, struct rtw_wlan_acl_node, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(acl_node->addr, addr, ETH_ALEN)) {
+			if (acl_node->valid == _TRUE) {
+				existed = 1;
+				break;
+			}
+		}
+	}
+	if (existed)
+		goto release_lock;
+
+	if (acl->num >= NUM_ACL)
+		goto release_lock;
+
+	/* find empty one and use */
+	for (i = 0; i < NUM_ACL; i++) {
+
+		acl_node = &acl->aclnode[i];
+		if (acl_node->valid == _FALSE) {
+
+			_rtw_init_listhead(&acl_node->list);
+			_rtw_memcpy(acl_node->addr, addr, ETH_ALEN);
+			acl_node->valid = _TRUE;
+
+			rtw_list_insert_tail(&acl_node->list, get_list_head(acl_node_q));
+			acl->num++;
+			break;
+		}
+	}
+
+release_lock:
+	_rtw_spinunlock_bh(&(acl_node_q->lock));
+
+	if (!existed && (i < 0 || i >= NUM_ACL))
+		ret = -1;
+
+	RTW_INFO(FUNC_ADPT_FMT" p=%u "MAC_FMT" %s (acl_num=%d)\n"
+		 , FUNC_ADPT_ARG(adapter), period, MAC_ARG(addr)
+		, (existed ? "existed" : ((i < 0 || i >= NUM_ACL) ? "no room" : "added"))
+		 , acl->num);
+exit:
+	return ret;
+}
+
+int rtw_acl_remove_sta(_adapter *adapter, u8 period, const u8 *addr)
+{
+	_list *list, *head;
+	int ret = 0;
+	struct rtw_wlan_acl_node *acl_node;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct wlan_acl_pool *acl;
+	_queue	*acl_node_q;
+	u8 is_baddr = is_broadcast_mac_addr(addr);
+	u8 match = 0;
+
+	if (period >= RTW_ACL_PERIOD_NUM) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	acl = &stapriv->acl_list[period];
+	acl_node_q = &acl->acl_node_q;
+
+	_rtw_spinlock_bh(&(acl_node_q->lock));
+
+	head = get_list_head(acl_node_q);
+	list = get_next(head);
+
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		acl_node = LIST_CONTAINOR(list, struct rtw_wlan_acl_node, list);
+		list = get_next(list);
+
+		if (is_baddr || _rtw_memcmp(acl_node->addr, addr, ETH_ALEN)) {
+			if (acl_node->valid == _TRUE) {
+				acl_node->valid = _FALSE;
+				rtw_list_delete(&acl_node->list);
+				acl->num--;
+				match = 1;
+			}
+		}
+	}
+
+	_rtw_spinunlock_bh(&(acl_node_q->lock));
+
+	RTW_INFO(FUNC_ADPT_FMT" p=%u "MAC_FMT" %s (acl_num=%d)\n"
+		 , FUNC_ADPT_ARG(adapter), period, MAC_ARG(addr)
+		 , is_baddr ? "clear all" : (match ? "match" : "no found")
+		 , acl->num);
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_RTW_MACADDR_ACL */
+
+u8 rtw_ap_set_sta_key(_adapter *adapter, const u8 *addr, u8 alg, const u8 *key, u8 keyid, u8 gk)
+{
+	struct set_stakey_parm param;
+	u8	res = _SUCCESS;
+
+	_rtw_memcpy(param.addr, addr, ETH_ALEN);
+	param.algorithm = alg;
+	param.keyid = keyid;
+	if (!!(alg & _SEC_TYPE_256_))
+		_rtw_memcpy(param.key, key, 32);
+	else
+		_rtw_memcpy(param.key, key, 16);
+	param.gk = gk;
+
+	set_stakey_hdl(adapter, &param, PHL_CMD_NO_WAIT, 0);
+exit:
+	return res;
+}
+
+u8 rtw_ap_set_pairwise_key(_adapter *padapter, struct sta_info *psta)
+{
+	return rtw_ap_set_sta_key(padapter
+		, psta->phl_sta->mac_addr
+		, psta->dot118021XPrivacy
+		, psta->dot118021x_UncstKey.skey
+		, 0
+		, 0
+	);
+}
+
+static int rtw_ap_set_key(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *key, u8 alg, int keyid, u8 set_tx)
+{
+	u8 keylen;
+	struct setkey_parm setkeyparm;
+	int res = _SUCCESS;
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	_rtw_memset(&setkeyparm, 0, sizeof(struct setkey_parm));
+
+	setkeyparm.keyid = (u8)keyid;
+	if (is_wep_enc(alg))
+		padapter->securitypriv.key_mask |= BIT(setkeyparm.keyid);
+
+	setkeyparm.algorithm = alg;
+
+	setkeyparm.set_tx = set_tx;
+
+	switch (alg) {
+	case _WEP40_:
+		keylen = 5;
+		break;
+	case _WEP104_:
+		keylen = 13;
+		break;
+	case _GCMP_256_:
+	case _CCMP_256_:
+		keylen = 32;
+		break;
+	case _TKIP_:
+	case _TKIP_WTMIC_:
+	case _AES_:
+	case _GCMP_:
+	#ifdef CONFIG_IEEE80211W
+	case _BIP_CMAC_128_:
+	#endif
+	default:
+		keylen = 16;
+	}
+
+	_rtw_memcpy(&(setkeyparm.key[0]), key, keylen);
+	setkey_hdl(padapter, padapter_link, &setkeyparm, PHL_CMD_NO_WAIT, 0);
+
+exit:
+	return res;
+}
+
+int rtw_ap_set_group_key(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *key, u8 alg, int keyid)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	return rtw_ap_set_key(padapter, padapter_link, key, alg, keyid, 1);
+}
+
+int rtw_ap_set_wep_key(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 *key, u8 keylen, int keyid, u8 set_tx)
+{
+	u8 alg;
+
+	switch (keylen) {
+	case 5:
+		alg = _WEP40_;
+		break;
+	case 13:
+		alg = _WEP104_;
+		break;
+	default:
+		alg = _NO_PRIVACY_;
+	}
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	return rtw_ap_set_key(padapter, padapter_link, key, alg, keyid, set_tx);
+}
+
+#if 0
+u8 rtw_ap_bmc_frames_hdl(_adapter *padapter)
+{
+#define HIQ_XMIT_COUNTS (6)
+	struct sta_info *psta_bmc;
+	_list	*xmitframe_plist, *xmitframe_phead;
+	struct xmit_frame *pxmitframe = NULL;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct sta_priv  *pstapriv = &padapter->stapriv;
+	bool update_tim = _FALSE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+
+	if (padapter->registrypriv.wifi_spec != 1)
+		return H2C_SUCCESS;
+
+
+	psta_bmc = rtw_get_bcmc_stainfo(padapter, padapter_link);
+	if (!psta_bmc)
+		return H2C_SUCCESS;
+
+
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	if ((rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, 0)) && (psta_bmc->sleepq_len > 0)) {
+		int tx_counts = 0;
+
+		_update_beacon(padapter, padapter_link, _TIM_IE_, NULL, _FALSE, 0, "update TIM with TIB=1");
+
+		RTW_INFO("sleepq_len of bmc_sta = %d\n", psta_bmc->sleepq_len);
+
+		xmitframe_phead = get_list_head(&psta_bmc->sleep_q);
+		xmitframe_plist = get_next(xmitframe_phead);
+
+		while ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE) {
+			pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+			xmitframe_plist = get_next(xmitframe_plist);
+
+			rtw_list_delete(&pxmitframe->list);
+
+			psta_bmc->sleepq_len--;
+			tx_counts++;
+
+			if (psta_bmc->sleepq_len > 0)
+				pxmitframe->attrib.mdata = 1;
+			else
+				pxmitframe->attrib.mdata = 0;
+
+			if (tx_counts == HIQ_XMIT_COUNTS)
+				pxmitframe->attrib.mdata = 0;
+
+			pxmitframe->attrib.triggered = 1;
+			#if 0
+			if (xmitframe_hiq_filter(pxmitframe) == _TRUE)
+				pxmitframe->attrib.qsel = rtw_hal_get_qsel(padapter,QSLT_HIGH_ID);/*HIQ*/
+			#endif
+
+			rtw_intf_xmitframe_enqueue(padapter, pxmitframe);
+
+			if (tx_counts == HIQ_XMIT_COUNTS)
+				break;
+
+		}
+
+	} else {
+		if (psta_bmc->sleepq_len == 0) {
+
+			/*RTW_INFO("sleepq_len of bmc_sta = %d\n", psta_bmc->sleepq_len);*/
+
+			if (rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, 0))
+				update_tim = _TRUE;
+
+			rtw_tim_map_clear(padapter, pstapriv->tim_bitmap, 0);
+			rtw_tim_map_clear(padapter, pstapriv->sta_dz_bitmap, 0);
+
+			if (update_tim == _TRUE) {
+				RTW_INFO("clear TIB\n");
+				_update_beacon(padapter, padapter_link, _TIM_IE_, NULL, _TRUE, 0, "bmc sleepq and HIQ empty");
+			}
+		}
+	}
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+#if 0
+	/* HIQ Check */
+	rtw_hal_get_hwreg(padapter, HW_VAR_CHK_HI_QUEUE_EMPTY, &empty);
+
+	while (_FALSE == empty && rtw_get_passing_time_ms(start) < 3000) {
+		rtw_msleep_os(100);
+		rtw_hal_get_hwreg(padapter, HW_VAR_CHK_HI_QUEUE_EMPTY, &empty);
+	}
+
+
+	printk("check if hiq empty=%d\n", empty);
+#endif
+
+	return H2C_SUCCESS;
+}
+#endif
+
+#ifdef CONFIG_NATIVEAP_MLME
+
+static void associated_stainfo_update(_adapter *padapter, struct sta_info *psta, u32 sta_info_type)
+{
+	struct link_mlme_priv *pmlmepriv = &(psta->padapter_link->mlmepriv);
+
+	RTW_INFO("%s: "MAC_FMT", updated_type=0x%x\n", __func__, MAC_ARG(psta->phl_sta->mac_addr), sta_info_type);
+#ifdef CONFIG_80211N_HT
+	if (sta_info_type & STA_INFO_UPDATE_BW) {
+
+		if ((psta->flags & WLAN_STA_HT) && !psta->ht_20mhz_set) {
+			if ((pmlmepriv->sw_to_20mhz && psta->phl_sta->chandef.bw != CHANNEL_WIDTH_20)
+				|| (!pmlmepriv->sw_to_20mhz && psta->phl_sta->chandef.bw == CHANNEL_WIDTH_20)
+			) {
+				update_sta_bw_apmode(padapter, psta);
+				rtw_sta_hal_ra_mask_update_cmd(padapter, psta, 0);
+			}
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+	/*
+		if (sta_info_type & STA_INFO_UPDATE_RATE) {
+
+		}
+	*/
+
+	if (sta_info_type & STA_INFO_UPDATE_PROTECTION_MODE)
+		VCS_update(padapter, psta);
+
+	/*
+		if (sta_info_type & STA_INFO_UPDATE_CAP) {
+
+		}
+
+		if (sta_info_type & STA_INFO_UPDATE_HT_CAP) {
+
+		}
+
+		if (sta_info_type & STA_INFO_UPDATE_VHT_CAP) {
+
+		}
+	*/
+
+}
+
+static void update_bcn_erpinfo_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *pnetwork = &(pmlmeinfo->network);
+	unsigned char *p, *ie = pnetwork->IEs;
+	u32 len = 0;
+
+	RTW_INFO("%s, ERP_enable=%d\n", __FUNCTION__, pmlmeinfo->ERP_enable);
+
+	if (!pmlmeinfo->ERP_enable)
+		return;
+
+	/* parsing ERP_IE */
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _ERPINFO_IE_, &len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+	if (p && len > 0) {
+		PNDIS_802_11_VARIABLE_IEs pIE = (PNDIS_802_11_VARIABLE_IEs)p;
+
+		if (pmlmepriv->num_sta_non_erp == 1)
+			pIE->data[0] |= RTW_ERP_INFO_NON_ERP_PRESENT | RTW_ERP_INFO_USE_PROTECTION;
+		else
+			pIE->data[0] &= ~(RTW_ERP_INFO_NON_ERP_PRESENT | RTW_ERP_INFO_USE_PROTECTION);
+
+		if (pmlmepriv->num_sta_no_short_preamble > 0)
+			pIE->data[0] |= RTW_ERP_INFO_BARKER_PREAMBLE_MODE;
+		else
+			pIE->data[0] &= ~(RTW_ERP_INFO_BARKER_PREAMBLE_MODE);
+
+		ERP_IE_handler(padapter, padapter_link, pIE);
+	}
+
+}
+
+static void update_bcn_htcap_ie(_adapter *padapter)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+}
+
+static void update_bcn_htinfo_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+#ifdef CONFIG_80211N_HT
+	/*
+	u8 beacon_updated = _FALSE;
+	u32 sta_info_update_type = STA_INFO_UPDATE_NONE;
+	*/
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *pnetwork = &(pmlmeinfo->network);
+	unsigned char *p, *ie = pnetwork->IEs;
+	u32 len = 0;
+
+	if (pmlmepriv->htpriv.ht_option == _FALSE)
+		return;
+
+	if (pmlmeinfo->HT_info_enable != 1)
+		return;
+
+
+	RTW_INFO("%s current operation mode=0x%X\n",
+		 __FUNCTION__, pmlmepriv->ht_op_mode);
+
+	RTW_INFO("num_sta_40mhz_intolerant(%d), 20mhz_width_req(%d), intolerant_ch_rpt(%d), olbc(%d)\n",
+		pmlmepriv->num_sta_40mhz_intolerant, pmlmepriv->ht_20mhz_width_req, pmlmepriv->ht_intolerant_ch_reported, ATOMIC_READ(&pmlmepriv->olbc));
+
+	/*parsing HT_INFO_IE, currently only update ht_op_mode - pht_info->infos[1] & pht_info->infos[2] for wifi logo test*/
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_ADD_INFO_IE_, &len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+	if (p && len > 0) {
+		struct HT_info_element *pht_info = NULL;
+
+		pht_info = (struct HT_info_element *)(p + 2);
+
+		/* for STA Channel Width/Secondary Channel Offset*/
+		if ((pmlmepriv->sw_to_20mhz == 0) && (pmlmeext->chandef.band == BAND_ON_24G)) {
+			if ((pmlmepriv->num_sta_40mhz_intolerant > 0) || (pmlmepriv->ht_20mhz_width_req == _TRUE)
+			    || (pmlmepriv->ht_intolerant_ch_reported == _TRUE) || (ATOMIC_READ(&pmlmepriv->olbc) == _TRUE)) {
+				SET_HT_OP_ELE_2ND_CHL_OFFSET(pht_info, 0);
+				SET_HT_OP_ELE_STA_CHL_WIDTH(pht_info, 0);
+
+				pmlmepriv->sw_to_20mhz = 1;
+				/*
+				sta_info_update_type |= STA_INFO_UPDATE_BW;
+				beacon_updated = _TRUE;
+				*/
+
+				RTW_INFO("%s:switching to 20Mhz\n", __FUNCTION__);
+
+				/*TODO : cur_bwmode/cur_ch_offset switches to 20Mhz*/
+			}
+		} else {
+
+			if ((pmlmepriv->num_sta_40mhz_intolerant == 0) && (pmlmepriv->ht_20mhz_width_req == _FALSE)
+			    && (pmlmepriv->ht_intolerant_ch_reported == _FALSE) && (ATOMIC_READ(&pmlmepriv->olbc) == _FALSE)) {
+
+				if (pmlmeext->chandef.bw >= CHANNEL_WIDTH_40) {
+
+					SET_HT_OP_ELE_STA_CHL_WIDTH(pht_info, 1);
+
+					SET_HT_OP_ELE_2ND_CHL_OFFSET(pht_info,
+						(pmlmeext->chandef.offset == CHAN_OFFSET_UPPER) ?
+						HT_INFO_HT_PARAM_SECONDARY_CHNL_ABOVE : HT_INFO_HT_PARAM_SECONDARY_CHNL_BELOW);
+
+					pmlmepriv->sw_to_20mhz = 0;
+					/*
+					sta_info_update_type |= STA_INFO_UPDATE_BW;
+					beacon_updated = _TRUE;
+					*/
+
+					RTW_INFO("%s:switching back to 40Mhz\n", __FUNCTION__);
+				}
+			}
+		}
+
+		/* to update  ht_op_mode*/
+		*(u16 *)(pht_info->infos + 1) = cpu_to_le16(pmlmepriv->ht_op_mode);
+
+	}
+
+	/*associated_clients_update(padapter, beacon_updated, sta_info_update_type);*/
+#endif /* CONFIG_80211N_HT */
+}
+
+static void update_bcn_rsn_ie(_adapter *padapter)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+}
+
+static void update_bcn_wpa_ie(_adapter *padapter)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+}
+
+static void update_bcn_wmm_ie(_adapter *padapter)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+}
+
+static void update_bcn_wps_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	u8 *pwps_ie = NULL, *pwps_ie_src, *premainder_ie, *pbackup_remainder_ie = NULL;
+	uint wps_ielen = 0, wps_offset, remainder_ielen;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *pnetwork = &(pmlmeinfo->network);
+	unsigned char *ie = pnetwork->IEs;
+	u32 ielen = pnetwork->IELength;
+
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	pwps_ie = rtw_get_wps_ie(ie + _FIXED_IE_LENGTH_, ielen - _FIXED_IE_LENGTH_, NULL, &wps_ielen);
+
+	if (pwps_ie == NULL || wps_ielen == 0)
+		return;
+
+	pwps_ie_src = pmlmepriv->wps_beacon_ie;
+	if (pwps_ie_src == NULL)
+		return;
+
+	wps_offset = (uint)(pwps_ie - ie);
+
+	premainder_ie = pwps_ie + wps_ielen;
+
+	remainder_ielen = ielen - wps_offset - wps_ielen;
+
+	if (remainder_ielen > 0) {
+		pbackup_remainder_ie = rtw_malloc(remainder_ielen);
+		if (pbackup_remainder_ie)
+			_rtw_memcpy(pbackup_remainder_ie, premainder_ie, remainder_ielen);
+	}
+
+	wps_ielen = (uint)pwps_ie_src[1];/* to get ie data len */
+	if ((wps_offset + wps_ielen + 2 + remainder_ielen) <= MAX_IE_SZ) {
+		_rtw_memcpy(pwps_ie, pwps_ie_src, wps_ielen + 2);
+		pwps_ie += (wps_ielen + 2);
+
+		if (pbackup_remainder_ie)
+			_rtw_memcpy(pwps_ie, pbackup_remainder_ie, remainder_ielen);
+
+		/* update IELength */
+		pnetwork->IELength = wps_offset + (wps_ielen + 2) + remainder_ielen;
+	}
+
+	if (pbackup_remainder_ie)
+		rtw_mfree(pbackup_remainder_ie, remainder_ielen);
+
+	/* deal with the case without set_tx_beacon_cmd() in rtw_update_beacon() */
+#if defined(CONFIG_INTERRUPT_BASED_TXBCN) || defined(CONFIG_PCI_HCI)
+	if ((padapter->mlmeextpriv.mlmext_info.state & 0x03) == WIFI_FW_AP_STATE) {
+		u8 sr = 0;
+		rtw_get_wps_attr_content(pwps_ie_src,  wps_ielen, WPS_ATTR_SELECTED_REGISTRAR, (u8 *)(&sr), NULL);
+
+		if (sr) {
+			set_fwstate(pmlmepriv, WIFI_UNDER_WPS);
+			RTW_INFO("%s, set WIFI_UNDER_WPS\n", __func__);
+		} else {
+			clr_fwstate(pmlmepriv, WIFI_UNDER_WPS);
+			RTW_INFO("%s, clr WIFI_UNDER_WPS\n", __func__);
+		}
+	}
+#endif
+}
+
+static void update_bcn_p2p_ie(_adapter *padapter)
+{
+
+}
+
+#ifdef CONFIG_ECSA_PHL
+static bool _update_csa_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct core_ecsa_info *ecsa_info = &(padapter->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	WLAN_BSSID_EX *pnetwork = &(padapter_link->mlmeextpriv.mlmext_info.network);
+	u8 chan = ecsa_param->new_chan_def.chan;
+	u8 count = ecsa_param->count;
+	u8 *ies = pnetwork->IEs + _BEACON_IE_OFFSET_;
+	sint ies_len = pnetwork->IELength - _BEACON_IE_OFFSET_;
+	u8 *csa_ie;
+	sint csa_ie_len;
+
+	csa_ie = rtw_get_ie(ies, WLAN_EID_CHANNEL_SWITCH, &csa_ie_len, ies_len);
+
+	if (csa_ie && csa_ie_len == CSA_IE_LEN) {
+		if (count > 0) {
+			/* update CSA IE */
+			#ifdef DBG_CSA
+			RTW_INFO("CSA : "FUNC_ADPT_FMT" ch=%u, count=%u, update CSA IE\n",
+				FUNC_ADPT_ARG(padapter), chan, count);
+			#endif
+
+			csa_ie = csa_ie + 2;
+			csa_ie[2] = count;
+		} else {
+			/* remove CSA IE */
+			RTW_INFO("CSA : "FUNC_ADPT_FMT" ch=%u, count=%u, remove CSA IE\n",
+				FUNC_ADPT_ARG(padapter), chan, count);
+
+			rtw_remove_bcn_ie(padapter, pnetwork, WLAN_EID_CHANNEL_SWITCH);
+			SET_ECSA_STATE(padapter, ECSA_ST_SW_DONE);
+		}
+		return _TRUE;
+	} else {
+		RTW_ERR("CSA : "FUNC_ADPT_FMT" unexpected case\n",
+				FUNC_ADPT_ARG(padapter));
+		return _FALSE;
+	}
+}
+
+static bool _update_ecsa_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct core_ecsa_info *ecsa_info = &(padapter->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	WLAN_BSSID_EX *pnetwork = &(padapter_link->mlmeextpriv.mlmext_info.network);
+	u8 chan = ecsa_param->new_chan_def.chan;
+	u8 count = ecsa_param->count;
+	u8 *ies = pnetwork->IEs + _BEACON_IE_OFFSET_;
+	sint ies_len = pnetwork->IELength - _BEACON_IE_OFFSET_;
+	u8 *ecsa_ie;
+	sint ecsa_ie_len;
+
+	ecsa_ie = rtw_get_ie(ies, WLAN_EID_ECSA, &ecsa_ie_len, ies_len);
+
+	if (ecsa_ie && ecsa_ie_len == ECSA_IE_LEN) {
+		if (count > 0) {
+			/* update CSA IE */
+			#ifdef DBG_CSA
+			RTW_INFO("CSA : "FUNC_ADPT_FMT" ch=%u, count=%u, update ECSA IE\n",
+				FUNC_ADPT_ARG(padapter), chan, count);
+			#endif
+			ecsa_ie = ecsa_ie + 2;
+			ecsa_ie[3] = count;
+		} else {
+			/* remove ECSA IE */
+			RTW_INFO("CSA : "FUNC_ADPT_FMT" ch=%u, count=%u, remove ECSA IE\n",
+				FUNC_ADPT_ARG(padapter), chan, count);
+
+			rtw_remove_bcn_ie(padapter, pnetwork, WLAN_EID_ECSA);
+			SET_ECSA_STATE(padapter, ECSA_ST_SW_DONE);
+		}
+		return _TRUE;
+	} else {
+		RTW_ERR("CSA : "FUNC_ADPT_FMT" unexpected case\n",
+				FUNC_ADPT_ARG(padapter));
+		return _FALSE;
+	}
+}
+#endif
+
+static u8 update_csa_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	u8 update = _FALSE;
+#ifdef CONFIG_ECSA_PHL
+	struct core_ecsa_info *ecsa_info = &(padapter->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+
+	if (!CHK_ECSA_STATE(padapter, ECSA_ST_SW_START))
+		return _FALSE;
+
+	update = _update_csa_ie(padapter, padapter_link);
+
+	if (rtw_is_ecsa_enabled(padapter))
+		update |= _update_ecsa_ie(padapter, padapter_link);
+
+	if (ecsa_param->count > 0)
+		ecsa_param->count--;
+
+#endif /* CONFIG_ECSA_PHL */
+	return update;
+}
+
+static void update_bcn_vendor_spec_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *oui)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	if (_rtw_memcmp(RTW_WPA_OUI, oui, 4))
+		update_bcn_wpa_ie(padapter);
+	else if (_rtw_memcmp(WMM_OUI, oui, 4))
+		update_bcn_wmm_ie(padapter);
+	else if (_rtw_memcmp(WPS_OUI, oui, 4))
+		update_bcn_wps_ie(padapter, padapter_link);
+	else if (_rtw_memcmp(P2P_OUI, oui, 4))
+		update_bcn_p2p_ie(padapter);
+	else
+		RTW_INFO("unknown OUI type!\n");
+
+
+}
+
+void _update_beacon(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 ie_id, u8 *oui, u8 tx, u8 flags, const char *tag)
+{
+	struct link_mlme_priv *pmlmepriv;
+	struct link_mlme_ext_priv *pmlmeext;
+	bool updated = 1; /* treat as upadated by default */
+
+	if (!padapter)
+		return;
+
+	pmlmepriv = &(padapter_link->mlmepriv);
+	pmlmeext = &(padapter_link->mlmeextpriv);
+
+	if (pmlmeext->bstart_bss == _FALSE)
+		return;
+
+	_rtw_spinlock_bh(&pmlmepriv->bcn_update_lock);
+
+	switch (ie_id) {
+	case _TIM_IE_:
+		update_BCNTIM(padapter, padapter_link);
+		break;
+
+	case _ERPINFO_IE_:
+		update_bcn_erpinfo_ie(padapter, padapter_link);
+		break;
+
+	case _HT_CAPABILITY_IE_:
+		update_bcn_htcap_ie(padapter);
+		break;
+
+	case _RSN_IE_2_:
+		update_bcn_rsn_ie(padapter);
+		break;
+
+	case _HT_ADD_INFO_IE_:
+		update_bcn_htinfo_ie(padapter, padapter_link);
+		break;
+
+#ifdef CONFIG_RTW_MESH
+	case WLAN_EID_MESH_CONFIG:
+		updated = rtw_mesh_update_bss_peering_status(padapter, &(pmlmeext->mlmext_info.network));
+		updated |= rtw_mesh_update_bss_formation_info(padapter, &(pmlmeext->mlmext_info.network));
+		updated |= rtw_mesh_update_bss_forwarding_state(padapter, &(pmlmeext->mlmext_info.network));
+		break;
+#endif
+	case WLAN_EID_CHANNEL_SWITCH:
+		updated = update_csa_ie(padapter, padapter_link);
+		break;
+
+	case _VENDOR_SPECIFIC_IE_:
+		update_bcn_vendor_spec_ie(padapter, padapter_link, oui);
+		break;
+
+	case 0xFF:
+	default:
+		break;
+	}
+
+	if (updated)
+		pmlmepriv->update_bcn = _TRUE;
+
+	_rtw_spinunlock_bh(&pmlmepriv->bcn_update_lock);
+
+#ifndef CONFIG_INTERRUPT_BASED_TXBCN
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI) || defined(CONFIG_PCI_BCN_POLLING)
+	if (tx && updated) {
+		/* send_beacon(padapter); */ /* send_beacon must execute on TSR level */
+		if (0)
+			RTW_INFO(FUNC_ADPT_FMT" ie_id:%u - %s\n", FUNC_ADPT_ARG(padapter), ie_id, tag);
+		if(flags == RTW_CMDF_WAIT_ACK)
+			set_tx_beacon_cmd(padapter, padapter_link, RTW_CMDF_WAIT_ACK);
+		else
+			set_tx_beacon_cmd(padapter, padapter_link, 0);
+	}
+#else
+	{
+		/* PCI will issue beacon when BCN interrupt occurs.		 */
+	}
+#endif
+#endif /* !CONFIG_INTERRUPT_BASED_TXBCN */
+}
+
+#ifdef CONFIG_80211N_HT
+
+void rtw_process_public_act_bsscoex(_adapter *padapter, u8 *pframe, uint frame_len)
+{
+	struct sta_info *psta;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 beacon_updated = _FALSE;
+	struct link_mlme_priv *pmlmepriv;
+	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	uint frame_body_len = frame_len - sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 category, action;
+
+	psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+	if (psta == NULL)
+		return;
+	pmlmepriv = &(psta->padapter_link->mlmepriv);
+
+
+	category = frame_body[0];
+	action = frame_body[1];
+
+	if (frame_body_len > 0) {
+		if ((frame_body[2] == EID_BSSCoexistence) && (frame_body[3] > 0)) {
+			u8 ie_data = frame_body[4];
+
+			if (ie_data & RTW_WLAN_20_40_BSS_COEX_40MHZ_INTOL) {
+				if (psta->ht_40mhz_intolerant == 0) {
+					psta->ht_40mhz_intolerant = 1;
+					pmlmepriv->num_sta_40mhz_intolerant++;
+					beacon_updated = _TRUE;
+				}
+			} else if (ie_data & RTW_WLAN_20_40_BSS_COEX_20MHZ_WIDTH_REQ)	{
+				if (pmlmepriv->ht_20mhz_width_req == _FALSE) {
+					pmlmepriv->ht_20mhz_width_req = _TRUE;
+					beacon_updated = _TRUE;
+				}
+			} else
+				beacon_updated = _FALSE;
+		}
+	}
+
+	if (frame_body_len > 8) {
+		/* if EID_BSSIntolerantChlReport ie exists */
+		if ((frame_body[5] == EID_BSSIntolerantChlReport) && (frame_body[6] > 0)) {
+			/*todo:*/
+			if (pmlmepriv->ht_intolerant_ch_reported == _FALSE) {
+				pmlmepriv->ht_intolerant_ch_reported = _TRUE;
+				beacon_updated = _TRUE;
+			}
+		}
+	}
+
+	if (beacon_updated) {
+
+		rtw_update_beacon(padapter, psta->padapter_link, _HT_ADD_INFO_IE_, NULL, _TRUE, 0);
+		associated_clients_update(padapter, _TRUE, STA_INFO_UPDATE_BW);
+	}
+
+
+
+}
+
+void rtw_process_ht_action_smps(_adapter *padapter, u8 *ta, u8 ctrl_field)
+{
+	u8 e_field, m_field;
+	struct sta_info *psta;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+
+	psta = rtw_get_stainfo(pstapriv, ta);
+	if (psta == NULL)
+		return;
+
+	e_field = (ctrl_field & BIT(0)) ? 1 : 0; /*SM Power Save Enabled*/
+	m_field = (ctrl_field & BIT(1)) ? 1 : 0; /*SM Mode, 0:static SMPS, 1:dynamic SMPS*/
+
+	if (e_field) {
+		if (m_field) { /*mode*/
+			psta->smps_mode = SM_PS_DYNAMIC;
+			RTW_ERR("Don't support dynamic SMPS\n");
+		}
+		else
+			psta->smps_mode = SM_PS_STATIC;
+	} else {
+		/*disable*/
+		psta->smps_mode = SM_PS_DISABLE;
+	}
+
+	if (psta->smps_mode != SM_PS_DYNAMIC)
+		rtw_ssmps_wk_cmd(padapter, psta, e_field, 1);
+}
+
+/*
+op_mode
+Set to 0 (HT pure) under the followign conditions
+	- all STAs in the BSS are 20/40 MHz HT in 20/40 MHz BSS or
+	- all STAs in the BSS are 20 MHz HT in 20 MHz BSS
+Set to 1 (HT non-member protection) if there may be non-HT STAs
+	in both the primary and the secondary channel
+Set to 2 if only HT STAs are associated in BSS,
+	however and at least one 20 MHz HT STA is associated
+Set to 3 (HT mixed mode) when one or more non-HT STAs are associated
+	(currently non-GF HT station is considered as non-HT STA also)
+*/
+int rtw_ht_operation_update(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	u16 cur_op_mode, new_op_mode;
+	int op_mode_changes = 0;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct ht_priv	*phtpriv_ap = &pmlmepriv->htpriv;
+
+	if (pmlmepriv->htpriv.ht_option == _FALSE)
+		return 0;
+
+	/*if (!iface->conf->ieee80211n || iface->conf->ht_op_mode_fixed)
+		return 0;*/
+
+	RTW_INFO("%s current operation mode=0x%X\n",
+		 __FUNCTION__, pmlmepriv->ht_op_mode);
+
+	if (!(pmlmepriv->ht_op_mode & HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT)
+	    && pmlmepriv->num_sta_ht_no_gf) {
+		pmlmepriv->ht_op_mode |=
+			HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT;
+		op_mode_changes++;
+	} else if ((pmlmepriv->ht_op_mode &
+		    HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT) &&
+		   pmlmepriv->num_sta_ht_no_gf == 0) {
+		pmlmepriv->ht_op_mode &=
+			~HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT;
+		op_mode_changes++;
+	}
+
+	if (!(pmlmepriv->ht_op_mode & HT_INFO_OPERATION_MODE_NON_HT_STA_PRESENT) &&
+	    (pmlmepriv->num_sta_no_ht || ATOMIC_READ(&pmlmepriv->olbc_ht))) {
+		pmlmepriv->ht_op_mode |= HT_INFO_OPERATION_MODE_NON_HT_STA_PRESENT;
+		op_mode_changes++;
+	} else if ((pmlmepriv->ht_op_mode &
+		    HT_INFO_OPERATION_MODE_NON_HT_STA_PRESENT) &&
+		   (pmlmepriv->num_sta_no_ht == 0 && !ATOMIC_READ(&pmlmepriv->olbc_ht))) {
+		pmlmepriv->ht_op_mode &=
+			~HT_INFO_OPERATION_MODE_NON_HT_STA_PRESENT;
+		op_mode_changes++;
+	}
+
+	/* Note: currently we switch to the MIXED op mode if HT non-greenfield
+	 * station is associated. Probably it's a theoretical case, since
+	 * it looks like all known HT STAs support greenfield.
+	 */
+	new_op_mode = 0;
+	if (pmlmepriv->num_sta_no_ht /*||
+	    (pmlmepriv->ht_op_mode & HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT)*/)
+		new_op_mode = OP_MODE_MIXED;
+	else if ((phtpriv_ap->ht_cap.cap_info & IEEE80211_HT_CAP_SUP_WIDTH)
+		 && pmlmepriv->num_sta_ht_20mhz)
+		new_op_mode = OP_MODE_20MHZ_HT_STA_ASSOCED;
+	else if (ATOMIC_READ(&pmlmepriv->olbc_ht))
+		new_op_mode = OP_MODE_MAY_BE_LEGACY_STAS;
+	else
+		new_op_mode = OP_MODE_PURE;
+
+	cur_op_mode = pmlmepriv->ht_op_mode & HT_INFO_OPERATION_MODE_OP_MODE_MASK;
+	if (cur_op_mode != new_op_mode) {
+		pmlmepriv->ht_op_mode &= ~HT_INFO_OPERATION_MODE_OP_MODE_MASK;
+		pmlmepriv->ht_op_mode |= new_op_mode;
+		op_mode_changes++;
+	}
+
+	RTW_INFO("%s new operation mode=0x%X changes=%d\n",
+		 __FUNCTION__, pmlmepriv->ht_op_mode, op_mode_changes);
+
+	return op_mode_changes;
+
+}
+
+#endif /* CONFIG_80211N_HT */
+
+void associated_clients_update(_adapter *padapter, u8 updated, u32 sta_info_type)
+{
+	/* update associcated stations cap. */
+	if (updated == _TRUE) {
+		_list	*phead, *plist;
+		struct sta_info *psta = NULL;
+		struct sta_priv *pstapriv = &padapter->stapriv;
+
+		rtw_stapriv_asoc_list_lock(pstapriv);
+
+		phead = &pstapriv->asoc_list;
+		plist = get_next(phead);
+
+		/* check asoc_queue */
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+
+			plist = get_next(plist);
+
+			associated_stainfo_update(padapter, psta, sta_info_type);
+		}
+
+		rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	}
+
+}
+
+/* called > TSR LEVEL for USB or SDIO Interface*/
+void bss_cap_update_on_sta_join(_adapter *padapter, struct sta_info *psta)
+{
+	u8 beacon_updated = _FALSE;
+	struct registry_priv *pregistrypriv = &(padapter->registrypriv);
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+
+
+#if 0
+	if (!(psta->capability & WLAN_CAPABILITY_SHORT_PREAMBLE) &&
+	    !psta->no_short_preamble_set) {
+		psta->no_short_preamble_set = 1;
+		pmlmepriv->num_sta_no_short_preamble++;
+		if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
+		    (pmlmepriv->num_sta_no_short_preamble == 1))
+			ieee802_11_set_beacons(hapd->iface);
+	}
+#endif
+
+
+	if (!(psta->flags & WLAN_STA_SHORT_PREAMBLE)) {
+		if (!psta->no_short_preamble_set) {
+			psta->no_short_preamble_set = 1;
+
+			pmlmepriv->num_sta_no_short_preamble++;
+
+			if ((pmlmeext->cur_wireless_mode > WLAN_MD_11B) &&
+			    (pmlmepriv->num_sta_no_short_preamble == 1))
+				beacon_updated = _TRUE;
+		}
+	} else {
+		if (psta->no_short_preamble_set) {
+			psta->no_short_preamble_set = 0;
+
+			pmlmepriv->num_sta_no_short_preamble--;
+
+			if ((pmlmeext->cur_wireless_mode > WLAN_MD_11B) &&
+			    (pmlmepriv->num_sta_no_short_preamble == 0))
+				beacon_updated = _TRUE;
+		}
+	}
+
+#if 0
+	if (psta->flags & WLAN_STA_NONERP && !psta->nonerp_set) {
+		psta->nonerp_set = 1;
+		pmlmepriv->num_sta_non_erp++;
+		if (pmlmepriv->num_sta_non_erp == 1)
+			ieee802_11_set_beacons(hapd->iface);
+	}
+#endif
+
+	if (psta->flags & WLAN_STA_NONERP) {
+		if (!psta->nonerp_set) {
+			psta->nonerp_set = 1;
+
+			pmlmepriv->num_sta_non_erp++;
+
+			if (pmlmepriv->num_sta_non_erp == 1) {
+				beacon_updated = _TRUE;
+				rtw_update_beacon(padapter, padapter_link, _ERPINFO_IE_, NULL, _FALSE, 0);
+			}
+		}
+
+	} else {
+		if (psta->nonerp_set) {
+			psta->nonerp_set = 0;
+
+			pmlmepriv->num_sta_non_erp--;
+
+			if (pmlmepriv->num_sta_non_erp == 0) {
+				beacon_updated = _TRUE;
+				rtw_update_beacon(padapter, padapter_link, _ERPINFO_IE_, NULL, _FALSE, 0);
+			}
+		}
+
+	}
+
+
+#if 0
+	if (!(psta->capability & WLAN_CAPABILITY_SHORT_SLOT) &&
+	    !psta->no_short_slot_time_set) {
+		psta->no_short_slot_time_set = 1;
+		pmlmepriv->num_sta_no_short_slot_time++;
+		if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
+		    (pmlmepriv->num_sta_no_short_slot_time == 1))
+			ieee802_11_set_beacons(hapd->iface);
+	}
+#endif
+
+	if (!(psta->capability & WLAN_CAPABILITY_SHORT_SLOT)) {
+		if (!psta->no_short_slot_time_set) {
+			psta->no_short_slot_time_set = 1;
+
+			pmlmepriv->num_sta_no_short_slot_time++;
+
+			if ((pmlmeext->cur_wireless_mode > WLAN_MD_11B) &&
+			    (pmlmepriv->num_sta_no_short_slot_time == 1))
+				beacon_updated = _TRUE;
+		}
+	} else {
+		if (psta->no_short_slot_time_set) {
+			psta->no_short_slot_time_set = 0;
+
+			pmlmepriv->num_sta_no_short_slot_time--;
+
+			if ((pmlmeext->cur_wireless_mode > WLAN_MD_11B) &&
+			    (pmlmepriv->num_sta_no_short_slot_time == 0))
+				beacon_updated = _TRUE;
+		}
+	}
+
+#ifdef CONFIG_80211N_HT
+	if(pregistrypriv->ht_enable &&
+		is_supported_ht(pregistrypriv->wireless_mode)) {
+		if (psta->flags & WLAN_STA_HT) {
+			u16 ht_capab = le16_to_cpu(psta->htpriv.ht_cap.cap_info);
+
+			RTW_INFO("HT: STA " MAC_FMT " HT Capabilities Info: 0x%04x\n",
+				MAC_ARG(psta->phl_sta->mac_addr), ht_capab);
+
+			if (psta->no_ht_set) {
+				psta->no_ht_set = 0;
+				pmlmepriv->num_sta_no_ht--;
+			}
+
+			if ((ht_capab & IEEE80211_HT_CAP_GRN_FLD) == 0) {
+				if (!psta->no_ht_gf_set) {
+					psta->no_ht_gf_set = 1;
+					pmlmepriv->num_sta_ht_no_gf++;
+				}
+				RTW_INFO("%s STA " MAC_FMT " - no "
+					 "greenfield, num of non-gf stations %d\n",
+					 __FUNCTION__, MAC_ARG(psta->phl_sta->mac_addr),
+					 pmlmepriv->num_sta_ht_no_gf);
+			}
+
+			if ((ht_capab & IEEE80211_HT_CAP_SUP_WIDTH) == 0) {
+				if (!psta->ht_20mhz_set) {
+					psta->ht_20mhz_set = 1;
+					pmlmepriv->num_sta_ht_20mhz++;
+				}
+				RTW_INFO("%s STA " MAC_FMT " - 20 MHz HT, "
+					 "num of 20MHz HT STAs %d\n",
+					 __FUNCTION__, MAC_ARG(psta->phl_sta->mac_addr),
+					 pmlmepriv->num_sta_ht_20mhz);
+			}
+
+			if (((ht_capab & RTW_IEEE80211_HT_CAP_40MHZ_INTOLERANT) != 0) &&
+				(psta->ht_40mhz_intolerant == 0)) {
+				psta->ht_40mhz_intolerant = 1;
+				pmlmepriv->num_sta_40mhz_intolerant++;
+				RTW_INFO("%s STA " MAC_FMT " - 40MHZ_INTOLERANT, ",
+					   __FUNCTION__, MAC_ARG(psta->phl_sta->mac_addr));
+			}
+
+		} else {
+			if (!psta->no_ht_set) {
+				psta->no_ht_set = 1;
+				pmlmepriv->num_sta_no_ht++;
+			}
+			if (pmlmepriv->htpriv.ht_option == _TRUE) {
+				RTW_INFO("%s STA " MAC_FMT
+					 " - no HT, num of non-HT stations %d\n",
+					 __FUNCTION__, MAC_ARG(psta->phl_sta->mac_addr),
+					 pmlmepriv->num_sta_no_ht);
+			}
+		}
+
+		if (rtw_ht_operation_update(padapter, padapter_link) > 0) {
+			rtw_update_beacon(padapter, padapter_link, _HT_CAPABILITY_IE_, NULL, _FALSE, 0);
+			rtw_update_beacon(padapter, padapter_link, _HT_ADD_INFO_IE_, NULL, _FALSE, 0);
+			beacon_updated = _TRUE;
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		struct sta_priv *pstapriv = &padapter->stapriv;
+
+		rtw_update_beacon(padapter, padapter_link, WLAN_EID_MESH_CONFIG, NULL, _FALSE, 0);
+		if (pstapriv->asoc_list_cnt == 1)
+			_set_timer(&padapter->mesh_atlm_param_req_timer, 0);
+		beacon_updated = _TRUE;
+	}
+#endif
+
+	if (beacon_updated)
+		rtw_update_beacon(padapter, padapter_link, 0xFF, NULL, _TRUE, 0);
+
+	/* update associcated stations cap. */
+	associated_clients_update(padapter,  beacon_updated, STA_INFO_UPDATE_ALL);
+
+	RTW_INFO("%s, updated=%d\n", __func__, beacon_updated);
+
+}
+
+u8 bss_cap_update_on_sta_leave(_adapter *padapter, struct sta_info *psta)
+{
+	u8 beacon_updated = _FALSE;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct link_mlme_priv *pmlmepriv = NULL;
+	struct link_mlme_ext_priv *pmlmeext = NULL;
+
+	if (!psta)
+		return beacon_updated;
+
+	pmlmepriv = &(psta->padapter_link->mlmepriv);
+	pmlmeext = &(psta->padapter_link->mlmeextpriv);
+	if (rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid)) {
+		rtw_tim_map_clear(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid);
+		beacon_updated = _TRUE;
+		rtw_update_beacon(padapter, psta->padapter_link, _TIM_IE_, NULL, _FALSE, 0);
+	}
+
+	if (psta->no_short_preamble_set) {
+		psta->no_short_preamble_set = 0;
+		pmlmepriv->num_sta_no_short_preamble--;
+		if (pmlmeext->cur_wireless_mode > WLAN_MD_11B
+		    && pmlmepriv->num_sta_no_short_preamble == 0)
+			beacon_updated = _TRUE;
+	}
+
+	if (psta->nonerp_set) {
+		psta->nonerp_set = 0;
+		pmlmepriv->num_sta_non_erp--;
+		if (pmlmepriv->num_sta_non_erp == 0) {
+			beacon_updated = _TRUE;
+			rtw_update_beacon(padapter, psta->padapter_link, _ERPINFO_IE_, NULL, _FALSE, 0);
+		}
+	}
+
+	if (psta->no_short_slot_time_set) {
+		psta->no_short_slot_time_set = 0;
+		pmlmepriv->num_sta_no_short_slot_time--;
+		if (pmlmeext->cur_wireless_mode > WLAN_MD_11B
+		    && pmlmepriv->num_sta_no_short_slot_time == 0)
+			beacon_updated = _TRUE;
+	}
+
+#ifdef CONFIG_80211N_HT
+	if (psta->no_ht_gf_set) {
+		psta->no_ht_gf_set = 0;
+		pmlmepriv->num_sta_ht_no_gf--;
+	}
+
+	if (psta->no_ht_set) {
+		psta->no_ht_set = 0;
+		pmlmepriv->num_sta_no_ht--;
+	}
+
+	if (psta->ht_20mhz_set) {
+		psta->ht_20mhz_set = 0;
+		pmlmepriv->num_sta_ht_20mhz--;
+	}
+
+	if (psta->ht_40mhz_intolerant) {
+		psta->ht_40mhz_intolerant = 0;
+		if (pmlmepriv->num_sta_40mhz_intolerant > 0)
+			pmlmepriv->num_sta_40mhz_intolerant--;
+		else
+			rtw_warn_on(1);
+	}
+
+	if (rtw_ht_operation_update(padapter, psta->padapter_link) > 0) {
+		rtw_update_beacon(padapter, psta->padapter_link, _HT_CAPABILITY_IE_, NULL, _FALSE, 0);
+		rtw_update_beacon(padapter, psta->padapter_link, _HT_ADD_INFO_IE_, NULL, _FALSE, 0);
+		beacon_updated = _TRUE;
+	}
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		rtw_update_beacon(padapter, psta->padapter_link, WLAN_EID_MESH_CONFIG, NULL, _FALSE, 0);
+		if (pstapriv->asoc_list_cnt == 0)
+			_cancel_timer_ex(&padapter->mesh_atlm_param_req_timer);
+		beacon_updated = _TRUE;
+	}
+#endif
+
+	if (beacon_updated == _TRUE)
+		rtw_update_beacon(padapter, psta->padapter_link, 0xFF, NULL, _TRUE, 0);
+
+#if 0
+	/* update associated stations cap. */
+	associated_clients_update(padapter,  beacon_updated, STA_INFO_UPDATE_ALL); /* move it to avoid deadlock */
+#endif
+
+	RTW_INFO("%s, updated=%d\n", __func__, beacon_updated);
+
+	return beacon_updated;
+
+}
+
+u8 ap_free_sta(_adapter *padapter, struct sta_info *psta, bool active, u8 subtype, u16 reason, bool enqueue)
+{
+	u8 beacon_updated = _FALSE;
+
+	if (!psta)
+		return beacon_updated;
+
+	RTW_INFO("%s sta "MAC_FMT"\n", __func__, MAC_ARG(psta->phl_sta->mac_addr));
+
+	if (active == _TRUE) {
+#ifdef CONFIG_80211N_HT
+		/* tear down Rx AMPDU */
+		send_delba(padapter, 0, psta->phl_sta->mac_addr);/* recipient */
+
+		/* tear down TX AMPDU */
+		send_delba(padapter, 1, psta->phl_sta->mac_addr);/*  */ /* originator */
+
+#endif /* CONFIG_80211N_HT */
+
+		if (!MLME_IS_MESH(padapter)) {
+			if (subtype == RTW_IEEE80211_STYPE_DISASSOC >> 4)
+				issue_disasoc(padapter, psta->phl_sta->mac_addr, reason);
+			else /* if not specified or not valid, use deauth by default */
+				issue_deauth(padapter, psta->phl_sta->mac_addr, reason);
+		}
+	}
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter))
+		rtw_mesh_path_flush_by_nexthop(psta);
+#endif
+
+#ifdef CONFIG_80211N_HT
+	psta->ampdu_priv.agg_enable_bitmap = 0x0;/* reset */
+	psta->ampdu_priv.candidate_tid_bitmap = 0x0;/* reset */
+#endif
+
+	_rtw_spinlock_bh(&psta->lock);
+	psta->state &= ~(WIFI_ASOC_STATE | WIFI_UNDER_KEY_HANDSHAKE);
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if ((psta->auth_len != 0) && (psta->pauth_frame != NULL)) {
+		rtw_mfree(psta->pauth_frame, psta->auth_len);
+		psta->pauth_frame = NULL;
+		psta->auth_len = 0;
+	}
+
+	if (psta->passoc_req && psta->assoc_req_len > 0) {
+		rtw_mfree(psta->passoc_req, psta->assoc_req_len);
+		psta->passoc_req = NULL;
+		psta->assoc_req_len = 0;
+	}
+#endif /* CONFIG_IOCTL_CFG80211 */
+	_rtw_spinunlock_bh(&psta->lock);
+
+	if (!MLME_IS_MESH(padapter)) {
+		#ifdef CONFIG_RTW_WDS
+		rtw_wds_path_flush_by_nexthop(psta);
+		#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+		#ifdef COMPAT_KERNEL_RELEASE
+		rtw_cfg80211_indicate_sta_disassoc(padapter, psta->phl_sta->mac_addr, reason);
+		#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) && !defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER)
+		rtw_cfg80211_indicate_sta_disassoc(padapter, psta->phl_sta->mac_addr, reason);
+		#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) && !defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER) */
+		/* will call rtw_cfg80211_indicate_sta_disassoc() in cmd_thread for old API context */
+		#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) && !defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER) */
+#else
+		rtw_indicate_sta_disassoc_event(padapter, psta);
+#endif
+	}
+
+	beacon_updated = bss_cap_update_on_sta_leave(padapter, psta);
+
+	report_del_sta_event(padapter, psta->phl_sta->mac_addr, reason, enqueue, _FALSE);
+
+	return beacon_updated;
+
+}
+
+int rtw_sta_flush(_adapter *padapter, bool enqueue)
+{
+	_list	*phead, *plist;
+	int ret = 0;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 flush_num = 0;
+	char flush_list[NUM_STA];
+	int i;
+
+	if (!MLME_IS_AP(padapter) && !MLME_IS_MESH(padapter))
+		return ret;
+
+	/* pick sta from sta asoc_queue */
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		int stainfo_offset;
+
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+
+		rtw_stapriv_asoc_list_del(pstapriv, psta);
+
+		stainfo_offset = rtw_stainfo_offset(pstapriv, psta);
+		if (stainfo_offset_valid(stainfo_offset))
+			flush_list[flush_num++] = stainfo_offset;
+		else
+			rtw_warn_on(1);
+	}
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	RTW_INFO(FUNC_NDEV_FMT" flush_num:%d\n", FUNC_NDEV_ARG(padapter->pnetdev), flush_num);
+
+	/* call ap_free_sta() for each sta picked */
+	for (i = 0; i < flush_num; i++) {
+		u8 sta_addr[ETH_ALEN];
+
+		psta = rtw_get_stainfo_by_offset(pstapriv, flush_list[i]);
+		if (psta != NULL) {
+			_rtw_memcpy(sta_addr, psta->phl_sta->mac_addr, ETH_ALEN);
+			ap_free_sta(padapter, psta, _TRUE, 0, WLAN_REASON_DEAUTH_LEAVING, enqueue);
+		}
+		#ifdef CONFIG_RTW_MESH
+		if (MLME_IS_MESH(padapter))
+			rtw_mesh_expire_peer(padapter, sta_addr);
+		#endif
+	}
+
+	if (MLME_IS_ASOC(padapter) && !MLME_IS_MESH(padapter))
+		issue_deauth(padapter, bc_addr, WLAN_REASON_DEAUTH_LEAVING);
+
+	associated_clients_update(padapter, _TRUE, STA_INFO_UPDATE_ALL);
+
+	return ret;
+}
+
+/* called > TSR LEVEL for USB or SDIO Interface*/
+void sta_info_update(_adapter *padapter, struct sta_info *psta)
+{
+	int flags = psta->flags;
+	struct link_mlme_priv *pmlmepriv = &(psta->padapter_link->mlmepriv);
+
+
+	/* update wmm cap. */
+	if (WLAN_STA_WME & flags)
+		psta->qos_option = 1;
+	else
+		psta->qos_option = 0;
+
+	if (pmlmepriv->qospriv.qos_option == 0)
+		psta->qos_option = 0;
+
+
+#ifdef CONFIG_80211N_HT
+	/* update 802.11n ht cap. */
+	if (WLAN_STA_HT & flags) {
+		psta->htpriv.ht_option = _TRUE;
+		psta->qos_option = 1;
+
+		psta->smps_mode = (psta->htpriv.ht_cap.cap_info & IEEE80211_HT_CAP_SM_PS) >> 2;
+	} else
+		psta->htpriv.ht_option = _FALSE;
+
+	if (pmlmepriv->htpriv.ht_option == _FALSE)
+		psta->htpriv.ht_option = _FALSE;
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	/* update 802.11AC vht cap. */
+	if (WLAN_STA_VHT & flags)
+		psta->vhtpriv.vht_option = _TRUE;
+	else
+		psta->vhtpriv.vht_option = _FALSE;
+
+	if (pmlmepriv->vhtpriv.vht_option == _FALSE)
+		psta->vhtpriv.vht_option = _FALSE;
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	/* update 802.11AX he cap. */
+	if (WLAN_STA_HE & flags) {
+		psta->hepriv.he_option = _TRUE;
+#if CONFIG_IEEE80211_BAND_6GHZ
+		if (flags & WLAN_STA_6G_BAND_CAP)
+			psta->smps_mode = GET_HE_6G_BAND_CAP_SM_PS(psta->hepriv.he_6g_band_cap + 1);
+#endif
+	} else {
+		psta->hepriv.he_option = _FALSE;
+	}
+
+	if (pmlmepriv->hepriv.he_option == _FALSE)
+		psta->hepriv.he_option = _FALSE;
+
+#endif
+
+	update_sta_info_apmode(padapter, psta);
+}
+
+/* restore hw setting from sw data structures */
+void rtw_ap_restore_network(_adapter *padapter)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	_list	*phead, *plist;
+	u8 chk_alive_num = 0;
+	char chk_alive_list[NUM_STA];
+	int i;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_security_priv *psecuritypriv = &(padapter_link->securitypriv);
+
+	rtw_setopmode_cmd(padapter
+		, MLME_IS_AP(padapter) ? Ndis802_11APMode : Ndis802_11_mesh
+		, RTW_CMDF_DIRECTLY
+	);
+
+	set_bch_bwmode(padapter, padapter_link, pmlmeext->chandef.band, pmlmeext->chandef.chan,
+		       pmlmeext->chandef.offset, pmlmeext->chandef.bw, RFK_TYPE_FORCE_NOT_DO);
+
+	rtw_startbss_cmd(padapter, RTW_CMDF_DIRECTLY);
+
+	if ((padapter->securitypriv.dot11PrivacyAlgrthm == _TKIP_) ||
+	    (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)) {
+		/* restore group key, WEP keys is restored in ips_leave() */
+		rtw_set_key(padapter, padapter_link, psecuritypriv->dot118021XGrpKeyid, 0, _FALSE);
+	}
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		int stainfo_offset;
+
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+
+		stainfo_offset = rtw_stainfo_offset(pstapriv, psta);
+		if (stainfo_offset_valid(stainfo_offset))
+			chk_alive_list[chk_alive_num++] = stainfo_offset;
+	}
+
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	for (i = 0; i < chk_alive_num; i++) {
+		psta = rtw_get_stainfo_by_offset(pstapriv, chk_alive_list[i]);
+
+		if (psta == NULL){
+			RTW_INFO(FUNC_ADPT_FMT" sta_info is null\n", FUNC_ADPT_ARG(padapter));
+		} else if (psta->state & WIFI_ASOC_STATE) {
+			rtw_sta_hal_media_status_rpt_cmd(padapter, psta, true, RTW_CMDF_DIRECTLY);
+			/* pairwise key */
+			/* per sta pairwise key and settings */
+			if ((padapter->securitypriv.dot11PrivacyAlgrthm == _TKIP_) ||
+			    (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_))
+				rtw_setstakey_cmd(padapter, psta, UNICAST_KEY, _FALSE);
+		}
+	}
+
+}
+
+void start_ap_mode(_adapter *padapter)
+{
+	int i;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+#ifdef CONFIG_CONCURRENT_MODE
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+#endif
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	pmlmepriv->update_bcn = _FALSE;
+
+	/*init_mlme_ap_info(padapter);*/
+
+	pmlmeext->bstart_bss = _FALSE;
+
+	pmlmepriv->num_sta_non_erp = 0;
+
+	pmlmepriv->num_sta_no_short_slot_time = 0;
+
+	pmlmepriv->num_sta_no_short_preamble = 0;
+
+	pmlmepriv->num_sta_ht_no_gf = 0;
+#ifdef CONFIG_80211N_HT
+	pmlmepriv->num_sta_no_ht = 0;
+#endif /* CONFIG_80211N_HT */
+	pmlmeinfo->HT_info_enable = 0;
+	pmlmeinfo->HT_caps_enable = 0;
+	pmlmeinfo->HT_enable = 0;
+
+	pmlmepriv->num_sta_ht_20mhz = 0;
+	pmlmepriv->num_sta_40mhz_intolerant = 0;
+	ATOMIC_SET(&pmlmepriv->olbc, _FALSE);
+	ATOMIC_SET(&pmlmepriv->olbc_ht, _FALSE);
+
+#ifdef CONFIG_80211N_HT
+	pmlmepriv->ht_20mhz_width_req = _FALSE;
+	pmlmepriv->ht_intolerant_ch_reported = _FALSE;
+	pmlmepriv->ht_op_mode = 0;
+	pmlmepriv->sw_to_20mhz = 0;
+#endif
+
+	_rtw_memset(pmlmepriv->ext_capab_ie_data, 0, sizeof(pmlmepriv->ext_capab_ie_data));
+	pmlmepriv->ext_capab_ie_len = 0;
+
+#ifdef CONFIG_CONCURRENT_MODE
+	psecuritypriv->dot118021x_bmc_cam_id = INVALID_SEC_MAC_CAM_ID;
+#endif
+
+	for (i = 0 ;  i < pstapriv->max_aid; i++)
+		pstapriv->sta_aid[i] = NULL;
+
+#ifdef CONFIG_RTW_WDS
+	if (MLME_IS_AP(padapter))
+		rtw_wds_pathtbl_init(padapter);
+#endif
+	rtw_ap_stop_set_state(padapter, AP_STOP_ST_IDLE);
+
+	if (rtw_mi_check_status(padapter, MI_AP_MODE))
+		RTW_SET_SCAN_BAND_SKIP(padapter, BAND_5G);
+}
+
+void stop_ap_mode(_adapter *padapter)
+{
+	u8 self_action = MLME_ACTION_UNKNOWN;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+#if 0
+	int chanctx_num = 0;
+#endif
+	struct rtw_chan_def chan_def = {0};
+
+	RTW_INFO("%s -"ADPT_FMT"\n", __func__, ADPT_ARG(padapter));
+
+	if (MLME_IS_AP(padapter))
+		self_action = MLME_AP_STOPPED;
+	else if (MLME_IS_MESH(padapter))
+		self_action = MLME_MESH_STOPPED;
+	else
+		rtw_warn_on(1);
+
+	pmlmepriv->update_bcn = _FALSE;
+
+	/* reset and init security priv , this can refine with rtw_reset_securitypriv */
+	_rtw_memset((unsigned char *)&padapter->securitypriv, 0, sizeof(struct security_priv));
+	padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
+	padapter->securitypriv.ndisencryptstatus = Ndis802_11WEPDisabled;
+	_rtw_memset((unsigned char *)&padapter_link->securitypriv, 0, sizeof(struct link_security_priv));
+
+	rtw_rfctl_update_op_mode(adapter_to_rfctl(padapter), BIT(padapter->iface_id), 0, 0);
+
+	/* free scan queue */
+	rtw_free_network_queue(padapter, _TRUE);
+	rtw_free_mld_network_queue(padapter, _TRUE);
+
+#if CONFIG_RTW_MACADDR_ACL
+	rtw_macaddr_acl_clear(padapter, RTW_ACL_PERIOD_BSS);
+#endif
+
+	rtw_sta_flush(padapter, _FALSE);
+
+	padapter->mlmepriv.ap_isolate = 0;
+#ifdef CONFIG_RTW_WDS
+	adapter_set_use_wds(padapter, 0);
+#endif
+#ifdef CONFIG_RTW_MULTI_AP
+	padapter->multi_ap = 0;
+#endif
+	rtw_free_mlme_priv_ie_data(&padapter->mlmepriv);
+
+	pmlmeext->bstart_bss = _FALSE;
+
+#ifdef CONFIG_RTW_MULTI_AP
+	rtw_map_config_monitor(padapter, self_action);
+#endif
+#if 0
+	chanctx_num = rtw_phl_chanctx_del(adapter_to_dvobj(padapter)->phl,
+						padapter->phl_role, &chan_def);
+
+	if (chanctx_num && chan_def.chan != 0)
+		set_bch_bwmode(padapter, chan_def.band, chan_def.chan, chan_def.offset, chan_def.bw, RFK_TYPE_FORCE_NOT_DO);
+#endif
+
+#ifdef CONFIG_DFS_MASTER
+	rtw_indicate_cac_state_on_bss_stop(padapter);
+#endif
+
+#ifdef CONFIG_RTW_WDS
+	if (MLME_IS_AP(padapter))
+		rtw_wds_pathtbl_unregister(padapter);
+#endif
+	_rtw_spinlock_free(&pmlmepriv->bcn_update_lock);
+}
+
+#endif /* CONFIG_NATIVEAP_MLME */
+
+void rtw_ap_update_clients_rainfo(struct _ADAPTER *a, u8 flag)
+{
+	struct link_mlme_ext_priv *pmlmeext;
+	struct sta_priv *pstapriv = &a->stapriv;
+	struct sta_info *psta;
+	const struct sta_info *ap_self_psta = rtw_get_stainfo(pstapriv, a->phl_role->mac_addr);
+	_list *plist, *phead;
+	u8 i;
+
+	/* update RA mask of all clients */
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+
+			/* no need to update RA info of ap self */
+			if (psta && psta != ap_self_psta) {
+				pmlmeext = &psta->padapter_link->mlmeextpriv;
+				psta->phl_sta->chandef.bw = pmlmeext->chandef.bw;
+				rtw_sta_hal_ra_mask_update_cmd(a, psta, flag);
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+}
+
+void rtw_ap_update_bss_bchbw(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+			WLAN_BSSID_EX *bss, struct rtw_chan_def *chandef)
+{
+#define UPDATE_VHT_CAP 1
+#define UPDATE_HT_CAP 1
+	struct link_mlme_priv *mlmepriv = &adapter_link->mlmepriv;
+#ifdef CONFIG_80211N_HT
+	struct ht_priv *htpriv = &mlmepriv->htpriv;
+#endif
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv *vhtpriv = &mlmepriv->vhtpriv;
+#endif
+	u8 *p;
+	uint ie_len = 0;
+
+#ifdef CONFIG_80211AX_HE
+	struct he_priv *hepriv = &mlmepriv->hepriv;
+	u8 *he_op_info = NULL;
+
+	if ((hepriv->he_option == _TRUE) && (chandef->band == BAND_ON_6G)) {
+		he_op_info = rtw_ies_get_he_6g_op_info_ie(
+			bss->IEs + _FIXED_IE_LENGTH_,
+			bss->IELength - _FIXED_IE_LENGTH_);
+
+		/* update channel in IE */
+		if (he_op_info &&
+		    (bss->Configuration.DSConfig != chandef->chan ||
+		     bss->Configuration.Band != chandef->band)) {
+			SET_HE_OP_INFO_PRIMARY_CHAN(he_op_info, chandef->chan);
+			SET_HE_OP_INFO_CHAN_WIDTH(he_op_info, chandef->bw);
+			SET_HE_OP_INFO_CENTER_FREQ_0(he_op_info, (u8)chandef->center_freq1);
+			SET_HE_OP_INFO_CENTER_FREQ_1(he_op_info, (u8)chandef->center_freq2);
+		}
+
+		bss->Configuration.DSConfig = chandef->chan;
+
+		if (bss->Configuration.Band != chandef->band) {
+			bss->Configuration.Band = chandef->band;
+			change_band_update_ie(adapter, adapter_link, bss, chandef);
+		}
+
+		/* TODO remove unnecessary IE ie., HT VHT IE */
+		return;
+	}
+#endif
+	/* update channel in IE */
+	if (bss->Configuration.DSConfig != chandef->chan) {
+		p = rtw_get_ie((bss->IEs + _FIXED_IE_LENGTH_), _DSSET_IE_,
+			&ie_len, (bss->IELength - _FIXED_IE_LENGTH_));
+
+		if (p && ie_len > 0)
+			*(p + 2) = chandef->chan;
+
+		bss->Configuration.DSConfig = chandef->chan;
+	}
+
+	/* band is changed, update ERP, support rate, ext support rate IE */
+	if (bss->Configuration.Band != chandef->band) {
+		bss->Configuration.Band = chandef->band;
+		change_band_update_ie(adapter, adapter_link, bss, chandef);
+	}
+
+#ifdef CONFIG_80211AC_VHT
+	if (vhtpriv->vht_option == _TRUE) {
+		u8 *vht_cap_ie, *vht_op_ie;
+		int vht_cap_ielen, vht_op_ielen;
+		u8	center_freq;
+		struct rtw_chan_def chdef = {0};
+
+		vht_cap_ie = rtw_get_ie((bss->IEs + _FIXED_IE_LENGTH_), WLAN_EID_VHT_CAPABILITY, &vht_cap_ielen, (bss->IELength - _FIXED_IE_LENGTH_));
+		vht_op_ie = rtw_get_ie((bss->IEs + _FIXED_IE_LENGTH_), WLAN_EID_VHT_OPERATION, &vht_op_ielen, (bss->IELength - _FIXED_IE_LENGTH_));
+		chdef.chan = chandef->chan;
+		chdef.bw = chandef->bw;
+		chdef.offset = chandef->offset;
+		chdef.band = chandef->band;
+		center_freq = rtw_phl_get_center_ch(&chdef);
+
+		/* update vht cap ie */
+		if (vht_cap_ie && vht_cap_ielen) {
+			#if UPDATE_VHT_CAP
+			if ((chandef->bw == CHANNEL_WIDTH_160 || chandef->bw == CHANNEL_WIDTH_80_80) && vhtpriv->sgi_160m)
+				SET_VHT_CAPABILITY_ELE_SHORT_GI160M(vht_cap_ie + 2, 1);
+			else
+				SET_VHT_CAPABILITY_ELE_SHORT_GI160M(vht_cap_ie + 2, 0);
+
+			if (chandef->bw >= CHANNEL_WIDTH_80 && vhtpriv->sgi_80m)
+				SET_VHT_CAPABILITY_ELE_SHORT_GI80M(vht_cap_ie + 2, 1);
+			else
+				SET_VHT_CAPABILITY_ELE_SHORT_GI80M(vht_cap_ie + 2, 0);
+			#endif
+		}
+
+		/* update vht op ie */
+		if (vht_op_ie && vht_op_ielen) {
+			if (chandef->bw < CHANNEL_WIDTH_80) {
+				RTW_INFO(FUNC_ADPT_FMT" update VHT 20/40M\n", FUNC_ADPT_ARG(adapter));
+				SET_VHT_OPERATION_ELE_CHL_WIDTH(vht_op_ie + 2, CH_WIDTH_20_40M);
+				SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(vht_op_ie + 2, 0);
+				SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(vht_op_ie + 2, 0);
+			} else if (chandef->bw == CHANNEL_WIDTH_80) {
+				RTW_INFO(FUNC_ADPT_FMT" update VHT 80M, center_freq = %u\n", FUNC_ADPT_ARG(adapter), center_freq);
+				SET_VHT_OPERATION_ELE_CHL_WIDTH(vht_op_ie + 2, CH_WIDTH_80_160M);
+				SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(vht_op_ie + 2, center_freq);
+				SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(vht_op_ie + 2, 0);
+			} else {
+				RTW_ERR(FUNC_ADPT_FMT" unsupported BW:%u\n", FUNC_ADPT_ARG(adapter), chandef->bw);
+				rtw_warn_on(1);
+			}
+		}
+	}
+#endif /* CONFIG_80211AC_VHT */
+#ifdef CONFIG_80211N_HT
+	{
+		u8 *ht_cap_ie, *ht_op_ie;
+		int ht_cap_ielen, ht_op_ielen;
+
+		ht_cap_ie = rtw_get_ie((bss->IEs + _FIXED_IE_LENGTH_), EID_HTCapability, &ht_cap_ielen, (bss->IELength - _FIXED_IE_LENGTH_));
+		ht_op_ie = rtw_get_ie((bss->IEs + _FIXED_IE_LENGTH_), EID_HTInfo, &ht_op_ielen, (bss->IELength - _FIXED_IE_LENGTH_));
+
+		/* update ht cap ie */
+		if (ht_cap_ie && ht_cap_ielen) {
+			#if UPDATE_HT_CAP
+			if (chandef->bw >= CHANNEL_WIDTH_40)
+				SET_HT_CAP_ELE_CHL_WIDTH(ht_cap_ie + 2, 1);
+			else
+				SET_HT_CAP_ELE_CHL_WIDTH(ht_cap_ie + 2, 0);
+
+			if (chandef->bw >= CHANNEL_WIDTH_40 && htpriv->sgi_40m)
+				SET_HT_CAP_ELE_SHORT_GI40M(ht_cap_ie + 2, 1);
+			else
+				SET_HT_CAP_ELE_SHORT_GI40M(ht_cap_ie + 2, 0);
+
+			if (htpriv->sgi_20m)
+				SET_HT_CAP_ELE_SHORT_GI20M(ht_cap_ie + 2, 1);
+			else
+				SET_HT_CAP_ELE_SHORT_GI20M(ht_cap_ie + 2, 0);
+			#endif
+		}
+
+		/* update ht op ie */
+		if (ht_op_ie && ht_op_ielen) {
+			SET_HT_OP_ELE_PRI_CHL(ht_op_ie + 2, chandef->chan);
+			switch (chandef->offset) {
+			case CHAN_OFFSET_UPPER:
+				SET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op_ie + 2, IEEE80211_SCA);
+				break;
+			case CHAN_OFFSET_LOWER:
+				SET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op_ie + 2, IEEE80211_SCB);
+				break;
+			case CHAN_OFFSET_NO_EXT:
+			default:
+				break;
+				SET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op_ie + 2, IEEE80211_SCN);
+			}
+
+			if (chandef->bw >= CHANNEL_WIDTH_40)
+				SET_HT_OP_ELE_STA_CHL_WIDTH(ht_op_ie + 2, 1);
+			else
+				SET_HT_OP_ELE_STA_CHL_WIDTH(ht_op_ie + 2, 0);
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+}
+
+static int rtw_ap_gen_HE_freq(enum band_type band, u8 ch, u8 bw, u8 offset, u8 *freq0, u8 *freq1)
+{
+
+	*freq0 = 0;
+	*freq1 = 0;
+
+	if (band != BAND_ON_6G)
+		return -1;
+
+	if (bw == CHANNEL_WIDTH_160) {
+		*freq1 = rtw_get_center_ch_by_band(band, ch, bw, offset);
+		*freq0 = rtw_get_center_ch_by_band(band, ch, CHANNEL_WIDTH_80, offset);
+	} else {
+		*freq0 = rtw_get_center_ch_by_band(band, ch, bw, offset);
+		*freq1 = 0;
+	}
+	return 0;
+}
+
+static u8 rtw_ap_update_chbw_by_ifbmp(struct dvobj_priv *dvobj, u8 ifbmp
+	, enum band_type cur_ie_band[], u8 cur_ie_ch[], u8 cur_ie_bw[], u8 cur_ie_offset[]
+	, enum band_type dec_band[], u8 dec_ch[], u8 dec_bw[], u8 dec_offset[]
+	, const char *caller)
+{
+	_adapter *iface;
+	WLAN_BSSID_EX *network;
+	u8 ifbmp_ch_changed = 0, freq0 = 0, freq1 = 0;
+	int i;
+	struct _ADAPTER_LINK *iface_link = NULL;
+	struct link_mlme_ext_priv *mlmeext;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+			continue;
+
+		iface = dvobj->padapters[i];
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		iface_link = GET_PRIMARY_LINK(iface);
+		mlmeext = &(iface_link->mlmeextpriv);
+
+		if (MLME_IS_ASOC(iface)) {
+			RTW_INFO(FUNC_ADPT_FMT" %u,%u,%u,%u => %u,%u,%u,%u%s\n", caller, ADPT_ARG(iface)
+				, mlmeext->chandef.band, mlmeext->chandef.chan, mlmeext->chandef.bw, mlmeext->chandef.offset
+				, dec_band[i], dec_ch[i], dec_bw[i], dec_offset[i]
+				, MLME_IS_OPCH_SW(iface) ? " OPCH_SW" : "");
+		} else {
+			RTW_INFO(FUNC_ADPT_FMT" %u,%u,%u,%u => %u,%u,%u,%u%s\n", caller, ADPT_ARG(iface)
+				, cur_ie_band[i], cur_ie_ch[i], cur_ie_bw[i], cur_ie_offset[i]
+				, dec_band[i], dec_ch[i], dec_bw[i], dec_offset[i]
+				, MLME_IS_OPCH_SW(iface) ? " OPCH_SW" : "");
+		}
+	}
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+			continue;
+
+		iface = dvobj->padapters[i];
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		iface_link = GET_PRIMARY_LINK(iface);
+		mlmeext = &(iface_link->mlmeextpriv);
+		network = &(mlmeext->mlmext_info.network);
+
+		/* ch setting differs from mlmeext.network IE */
+		if (cur_ie_ch[i] != dec_ch[i]
+			|| cur_ie_bw[i] != dec_bw[i]
+			|| cur_ie_offset[i] != dec_offset[i])
+			ifbmp_ch_changed |= BIT(i);
+
+		/* ch setting differs from existing one */
+		if (MLME_IS_ASOC(iface)
+			&& (mlmeext->chandef.chan != dec_ch[i]
+				|| mlmeext->chandef.band != dec_band[i]
+				|| mlmeext->chandef.bw != dec_bw[i]
+				|| mlmeext->chandef.offset != dec_offset[i])
+		) {
+			if (rtw_linked_check(iface) == _TRUE) {
+				#ifdef CONFIG_ECSA_PHL
+				if (CHK_ECSA_STATE(iface, ECSA_ST_NONE))
+				#endif
+					rtw_sta_flush(iface, _FALSE);
+			}
+		}
+
+		rtw_ap_gen_HE_freq(dec_band[i], dec_ch[i], dec_bw[i], dec_offset[i], &freq0,  &freq1);
+		mlmeext->chandef.center_freq1 = (u32)freq0;
+		mlmeext->chandef.center_freq2 = (u32)freq1;
+		mlmeext->chandef.band = dec_band[i];
+		mlmeext->chandef.chan = dec_ch[i];
+		mlmeext->chandef.bw = dec_bw[i];
+		mlmeext->chandef.offset = dec_offset[i];
+
+		rtw_ap_update_bss_bchbw(iface, iface_link, network, &mlmeext->chandef);
+	}
+
+	return ifbmp_ch_changed;
+}
+
+static u8 rtw_ap_ch_specific_chk(_adapter *adapter, enum band_type band, u8 ch, u8 *bw, u8 *offset, const char *caller)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	int ch_idx;
+	u8 ret = _SUCCESS;
+
+	if (rtw_rfctl_is_regu_forbid_bss(rfctl, band)) {
+		RTW_WARN("%s %s AP not allowed by regulatory\n", __func__, band_str(band));
+		ret = _FAIL;
+		goto exit;
+	}
+
+	ch_idx = rtw_chset_search_bch(chset, band, ch);
+	if (ch_idx < 0) {
+		RTW_WARN("%s ch:%u doesn't fit in chplan\n", caller, ch);
+		ret = _FAIL;
+		goto exit;
+	}
+	if (chset->chs[ch_idx].flags & RTW_CHF_NO_IR) {
+		RTW_WARN("%s ch:%u is passive\n", caller, ch);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	rtw_adjust_bchbw(adapter, band, ch, bw, offset);
+
+	if (!rtw_get_offset_by_bchbw(band, ch, *bw, offset)) {
+		RTW_WARN("%s %u, %u,%u has no valid offset\n", caller, band, ch, *bw);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	while (!rtw_chset_is_bchbw_valid(chset, band, ch, *bw, *offset, 0, 0)
+		|| (!rtw_rfctl_radar_detect_supported(rfctl) && rtw_chset_is_dfs_bchbw(chset, band, ch, *bw, *offset))
+	) {
+		if (*bw > CHANNEL_WIDTH_20)
+			(*bw)--;
+		if (*bw == CHANNEL_WIDTH_20) {
+			*offset = CHAN_OFFSET_NO_EXT;
+			break;
+		}
+	}
+
+
+	if (!rtw_rfctl_radar_detect_supported(rfctl) && rtw_chset_is_dfs_bchbw(chset, band, ch, *bw, *offset)) {
+		RTW_WARN("%s DFS channel %u can't be used\n", caller, ch);
+		ret = _FAIL;
+		goto exit;
+	}
+
+exit:
+	return ret;
+}
+
+u8 rtw_ap_bchbw_decision(_adapter *adapter, u8 ifbmp, u8 excl_ifbmp, s8 req_band
+	, s16 req_ch, s8 req_bw, s8 req_offset, struct rtw_chan_def *chdef)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	bool ch_avail = _FALSE;
+	enum band_type cur_ie_band[CONFIG_IFACE_NUMBER] = {BAND_ON_24G};
+	u8 cur_ie_freq0[CONFIG_IFACE_NUMBER] = {0};
+	u8 cur_ie_freq1[CONFIG_IFACE_NUMBER] = {0};
+	u8 cur_ie_ch[CONFIG_IFACE_NUMBER] = {0};
+	u8 cur_ie_bw[CONFIG_IFACE_NUMBER] = {0};
+	u8 cur_ie_offset[CONFIG_IFACE_NUMBER] = {0};
+	enum band_type dec_band[CONFIG_IFACE_NUMBER] = {BAND_ON_24G};
+	u8 dec_hwband[CONFIG_IFACE_NUMBER] = {0};
+	u8 dec_ch[CONFIG_IFACE_NUMBER] = {0};
+	u8 dec_bw[CONFIG_IFACE_NUMBER] = {0};
+	u8 dec_offset[CONFIG_IFACE_NUMBER] = {0};
+	u8 u_ch = 0, u_bw = 0, u_offset = 0;
+	enum band_type u_band = BAND_ON_24G;
+	WLAN_BSSID_EX *network;
+	struct mi_state mstate;
+	struct mi_state mstate_others;
+	bool set_u_ch = _FALSE;
+	u8 ifbmp_others = 0xFF & ~ifbmp & ~excl_ifbmp;
+	u8 ifbmp_ch_changed = 0;
+	bool ifbmp_all_mesh = 0;
+	_adapter *iface;
+	int i;
+#ifdef CONFIG_MCC_MODE
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	u8 mcc_sup = phl_com->dev_cap.mcc_sup;
+#endif
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct _ADAPTER_LINK *iface_link;
+	struct link_mlme_ext_priv *mlmeext;
+#ifdef CONFIG_DBCC_SUPPORT
+	enum phl_band_idx hw_band = adapter_link->wrlink->hw_band;
+
+	dec_hwband[adapter->iface_id] = hw_band;
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	for (i = 0; i < dvobj->iface_nums; i++)
+		if ((ifbmp & BIT(i)) && dvobj->padapters[i])
+			if (!MLME_IS_MESH(dvobj->padapters[i]))
+				break;
+	ifbmp_all_mesh = i >= dvobj->iface_nums ? 1 : 0;
+#endif
+
+	RTW_INFO("%s ifbmp:0x%02x excl_ifbmp:0x%02x req:%d,%d,%d\n", __func__
+		, ifbmp, excl_ifbmp, req_ch, req_bw, req_offset);
+	rtw_mi_status_by_ifbmp(adapter, ifbmp, &mstate);
+	rtw_mi_status_by_ifbmp(adapter, ifbmp_others, &mstate_others);
+	RTW_INFO("%s others ld_sta_num:%u, lg_sta_num:%u, ap_num:%u, mesh_num:%u\n"
+		, __func__, MSTATE_STA_LD_NUM(&mstate_others), MSTATE_STA_LG_NUM(&mstate_others)
+		, MSTATE_AP_NUM(&mstate_others), MSTATE_MESH_NUM(&mstate_others));
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+			continue;
+		iface = dvobj->padapters[i];
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		iface_link = GET_PRIMARY_LINK(iface);
+		mlmeext = &(iface_link->mlmeextpriv);
+		network = &(mlmeext->mlmext_info.network);
+
+		/* get current IE channel settings */
+		rtw_ies_get_bchbw(BSS_EX_TLV_IES(network), BSS_EX_TLV_IES_LEN(network)
+			, &cur_ie_band[i] , &cur_ie_ch[i] , &cur_ie_bw[i]
+			, &cur_ie_offset[i], &cur_ie_freq0[i], &cur_ie_freq1[i], 1, 1, 1);
+
+		/* prepare temporary channel setting decision */
+		if (req_ch == 0) {
+			/* request comes from upper layer, use cur_ie values */
+			dec_band[i] = cur_ie_band[i];
+			dec_ch[i] = cur_ie_ch[i];
+			dec_bw[i] = cur_ie_bw[i];
+			dec_offset[i] = cur_ie_offset[i];
+		} else {
+			/* use chbw of cur_ie updated with specifying req as temporary decision */
+
+			if (req_ch <= REQ_CH_NONE) {
+				dec_ch[i] = cur_ie_ch[i];
+				dec_band[i] = cur_ie_band[i];
+			} else {
+				dec_ch[i] = req_ch;
+				dec_band[i] = req_band;
+			}
+
+			if (req_bw <= REQ_BW_NONE) {
+				if (req_bw == REQ_BW_ORI)
+					dec_bw[i] = iface_link->mlmepriv.ori_chandef.bw;
+				else
+					dec_bw[i] = cur_ie_bw[i];
+			} else {
+				dec_bw[i] = req_bw;
+			}
+
+			dec_offset[i] = (req_offset <= REQ_OFFSET_NONE) ? cur_ie_offset[i] : req_offset;
+			#ifdef CONFIG_DBCC_SUPPORT
+			dec_hwband[i] = iface_link->wrlink->hw_band;
+			#endif
+		}
+	}
+
+	if (MSTATE_STA_LD_NUM(&mstate_others) || MSTATE_STA_LG_NUM(&mstate_others)
+		|| MSTATE_AP_NUM(&mstate_others) || MSTATE_MESH_NUM(&mstate_others)
+	) {
+		/* has linked/linking STA or has AP/Mesh mode */
+		rtw_warn_on(!rtw_mi_get_ch_setting_union_by_ifbmp(adapter, ifbmp_others, &u_band, &u_ch, &u_bw, &u_offset));
+		RTW_INFO("%s others union:%u,%u,%u,%u\n", __func__, u_band, u_ch, u_bw, u_offset);
+	}
+
+#ifdef CONFIG_MCC_MODE
+	if (mcc_sup && req_ch == 0) {
+		/* will disable MCC in ap starter prepare after v10 merge */
+		if (rtw_hw_mcc_chk_inprogress(adapter, adapter_link)) {
+			u8 if_id = adapter->iface_id;
+
+			mlmeext = &(adapter_link->mlmeextpriv);
+			/* check channel settings are the same */
+			if (cur_ie_ch[if_id] == mlmeext->chandef.chan
+				&& cur_ie_band[if_id] == mlmeext->chandef.band
+				&& cur_ie_bw[if_id] == mlmeext->chandef.bw
+				&& cur_ie_offset[if_id] == mlmeext->chandef.offset) {
+
+				RTW_INFO(FUNC_ADPT_FMT"req ch settings are the same as current ch setting, go to exit\n"
+					, FUNC_ADPT_ARG(adapter));
+				goto exit;
+			} else {
+				RTW_WARN(FUNC_ADPT_FMT"request channel settings are different from current channel setting(%d,%d,%d,%d,%d,%d), restart MCC\n"
+					, FUNC_ADPT_ARG(adapter)
+					, cur_ie_ch[if_id], cur_ie_bw[if_id], cur_ie_offset[if_id]
+					, mlmeext->chandef.chan, mlmeext->chandef.bw, mlmeext->chandef.offset);
+				#if 0
+				rtw_hal_set_mcc_setting_disconnect(adapter);
+				#endif
+			}
+		}
+	}
+#endif /* CONFIG_MCC_MODE */
+
+	if (MSTATE_STA_LG_NUM(&mstate_others) || MSTATE_STA_LD_NUM(&mstate_others)
+		|| MSTATE_AP_NUM(&mstate_others) || MSTATE_MESH_NUM(&mstate_others)
+	) {
+		/* has linked or linking STA mode or AP/Mesh mode */
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+				continue;
+			iface = dvobj->padapters[i];
+
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+
+			/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+			iface_link = GET_PRIMARY_LINK(iface);
+			#ifdef CONFIG_MCC_MODE
+			if (mcc_sup) {
+				if (!rtw_is_bchbw_grouped(u_band, u_ch, u_bw, u_offset,
+					dec_band[i],  dec_ch[i], dec_bw[i], dec_offset[i])) {
+					mlmeext = &(iface_link->mlmeextpriv);
+					mlmeext->chandef.chan = chdef->chan = dec_ch[i];
+					mlmeext->chandef.bw = chdef->bw = dec_bw[i];
+					mlmeext->chandef.offset = chdef->offset = dec_offset[i];
+					mlmeext->chandef.band = chdef->band = dec_band[i];
+
+					/* Trigger CSA should update ifbmp_ch_changed */
+					#ifdef CONFIG_ECSA_PHL
+					if (CHK_ECSA_STATE(iface, ECSA_ST_SW_START) &&
+					   (cur_ie_ch[i] != dec_ch[i]
+					   || cur_ie_band[i] != dec_band[i]
+					   || cur_ie_bw[i] != dec_bw[i]
+					   || cur_ie_offset[i] != dec_offset[i])) {
+						ifbmp_ch_changed |= BIT(i);
+						RTW_INFO("CSA : "FUNC_ADPT_FMT" : Trigger CSA in mcc mode\n",
+							FUNC_ADPT_ARG(iface));
+					}
+
+					RTW_INFO(FUNC_ADPT_FMT" enable mcc: %u,%u,%u,%u\n", FUNC_ADPT_ARG(iface)
+						 , chdef->band, chdef->chan, chdef->bw, chdef->offset);
+					#endif
+					goto exit;
+				}
+			}
+			#endif /* CONFIG_MCC_MODE */
+
+			rtw_adjust_bchbw(iface, u_band, u_ch, &dec_bw[i], &dec_offset[i]);
+			#ifdef CONFIG_RTW_MESH
+			if (MLME_IS_MESH(iface))
+				rtw_mesh_adjust_bchbw(u_band, u_ch, &dec_bw[i], &dec_offset[i]);
+			#endif
+
+			if (req_ch == 0 && dec_bw[i] > u_bw
+				&& rtw_chset_is_dfs_bchbw(chset, u_band, u_ch, u_bw, u_offset)
+			) {
+				/* request comes from upper layer, prevent from additional channel waiting */
+				dec_bw[i] = u_bw;
+				if (dec_bw[i] == CHANNEL_WIDTH_20)
+					dec_offset[i] = CHAN_OFFSET_NO_EXT;
+			}
+
+			/* follow */
+			rtw_chset_sync_bchbw(chset
+				, &dec_band[i], &dec_ch[i], &dec_bw[i], &dec_offset[i]
+				, &u_band, &u_ch, &u_bw, &u_offset, 1, 0);
+		}
+
+		set_u_ch = _TRUE;
+
+	} else {
+		/* autonomous decision */
+		u8 ori_ch = 0, ori_offset = CHAN_OFFSET_NO_EXT;
+		u8 ori_band = 0;
+		u8 max_bw;
+		bool by_int_info;
+
+		if (req_ch <= REQ_CH_NONE) /* channel is not specified */
+			goto choose_chbw;
+
+		/* get tmp dec union of ifbmp */
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+				continue;
+			#ifdef CONFIG_DBCC_SUPPORT
+			if (dec_hwband[i] != hw_band)
+				continue;
+			#endif
+
+			if (u_ch == 0) {
+				u_band = dec_band[i];
+				u_ch = dec_ch[i];
+				u_bw = dec_bw[i];
+				u_offset = dec_offset[i];
+				rtw_adjust_bchbw(adapter, u_band, u_ch, &u_bw, &u_offset);
+				rtw_get_offset_by_bchbw(u_band, u_ch, u_bw, &u_offset);
+			} else {
+				enum band_type tmp_band = dec_band[i];
+				u8 tmp_ch = dec_ch[i];
+				u8 tmp_bw = dec_bw[i];
+				u8 tmp_offset = dec_offset[i];
+
+				rtw_adjust_bchbw(adapter, tmp_band, tmp_ch, &tmp_bw, &tmp_offset);
+				rtw_get_offset_by_bchbw(tmp_band, tmp_ch, tmp_bw, &u_offset);
+				rtw_warn_on(!rtw_is_bchbw_grouped(u_band, u_ch, u_bw, u_offset, tmp_band, tmp_ch, tmp_bw, tmp_offset));
+				rtw_sync_bchbw(&tmp_band, &tmp_ch, &tmp_bw, &tmp_offset, &u_band, &u_ch, &u_bw, &u_offset);
+			}
+		}
+
+		#ifdef CONFIG_RTW_MESH
+		/* if ifbmp are all mesh, apply bw restriction */
+		if (ifbmp_all_mesh)
+			rtw_mesh_adjust_bchbw(u_band, u_ch, &u_bw, &u_offset);
+		#endif
+
+		RTW_INFO("%s ifbmp:0x%02x tmp union:%u,%u,%u,%u\n", __func__, ifbmp, u_band, u_ch, u_bw, u_offset);
+
+		/* check if tmp dec union is usable */
+		if (rtw_ap_ch_specific_chk(adapter, u_band, u_ch, &u_bw, &u_offset, __func__) == _FAIL) {
+			/* channel can't be used */
+			if (req_ch > 0) {
+				/* specific channel and not from IE => don't change channel setting */
+				goto exit;
+			}
+			goto choose_chbw;
+		} else if (rtw_chset_is_bchbw_non_ocp(chset, u_band, u_ch, u_bw, u_offset)) {
+			RTW_WARN("%s DFS channel %u,%u under non ocp\n", __func__, u_ch, u_bw);
+			if (req_ch > 0 && req_bw > REQ_BW_NONE) {
+				/* change_chbw with specific channel and specific bw, goto update_bss_chbw directly */
+				goto update_bss_chbw;
+			}
+		} else
+			goto update_bss_chbw;
+
+choose_chbw:
+		by_int_info = req_ch == REQ_CH_INT_INFO ? 1 : 0;
+		if (req_band <= REQ_BAND_NONE || req_band >= BAND_MAX) {
+			/* band is unspecified/invalid means channel is also unspecified */
+			req_ch = 0;
+			req_band = BAND_MAX;
+		} else if (req_ch <= REQ_CH_NONE)
+			req_ch = 0;
+		max_bw = req_bw > REQ_BW_NONE ? req_bw : CHANNEL_WIDTH_20;
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+				continue;
+			iface = dvobj->padapters[i];
+
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+
+			/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+			iface_link = GET_PRIMARY_LINK(iface);
+			mlmeext = &(iface_link->mlmeextpriv);
+
+			if (req_bw <= REQ_BW_NONE) {
+				if (req_bw == REQ_BW_ORI) {
+					if (max_bw < iface_link->mlmepriv.ori_chandef.bw)
+						max_bw = iface_link->mlmepriv.ori_chandef.bw;
+				} else {
+					if (max_bw < cur_ie_bw[i])
+						max_bw = cur_ie_bw[i];
+				}
+			}
+
+			if (MSTATE_AP_NUM(&mstate) || MSTATE_MESH_NUM(&mstate)) {
+				if (ori_ch == 0) {
+					ori_band = mlmeext->chandef.band;
+					ori_ch = mlmeext->chandef.chan;
+					ori_offset = mlmeext->chandef.offset;
+				} else if (ori_ch != mlmeext->chandef.chan) {
+					rtw_warn_on(1);
+				} else if (mlmeext->chandef.offset != CHAN_OFFSET_NO_EXT) {
+					if (ori_offset == CHAN_OFFSET_NO_EXT)
+						ori_offset = mlmeext->chandef.offset;
+					else if (ori_offset != mlmeext->chandef.offset)
+						rtw_warn_on(1);
+				}
+			} else {
+				if (ori_ch == 0) {
+					ori_band = cur_ie_band[i];
+					ori_ch = cur_ie_ch[i];
+					ori_offset = cur_ie_offset[i];
+				} else if (ori_ch != cur_ie_ch[i]) {
+					rtw_warn_on(1);
+				} else if (cur_ie_offset[i] != CHAN_OFFSET_NO_EXT) {
+					if (ori_offset == CHAN_OFFSET_NO_EXT)
+						ori_offset = cur_ie_offset[i];
+					else if (ori_offset != cur_ie_offset[i])
+						rtw_warn_on(1);
+				}
+			}
+		}
+		req_offset = req_offset > REQ_OFFSET_NONE ? req_offset : CHAN_OFFSET_NO_EXT;
+
+		ch_avail = rtw_rfctl_choose_bchbw(rfctl
+			, req_band, req_ch, max_bw, req_offset
+			, ori_band, ori_ch, ori_offset
+			, &u_band, &u_ch, &u_bw
+			, &u_offset, by_int_info, ifbmp_all_mesh, __func__);
+		if (ch_avail == _FALSE)
+			goto exit;
+
+update_bss_chbw:
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+				continue;
+			iface = dvobj->padapters[i];
+
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+
+			dec_ch[i] = u_ch;
+			dec_band[i] = u_band;
+			if (dec_bw[i] > u_bw)
+				dec_bw[i] = u_bw;
+			if (dec_bw[i] == CHANNEL_WIDTH_20)
+				dec_offset[i] = CHAN_OFFSET_NO_EXT;
+			else
+				dec_offset[i] = u_offset;
+
+			#ifdef CONFIG_RTW_MESH
+			if (MLME_IS_MESH(iface))
+				rtw_mesh_adjust_bchbw(dec_band[i], dec_ch[i], &dec_bw[i], &dec_offset[i]);
+			#endif
+		}
+
+		set_u_ch = _TRUE;
+	}
+
+	ifbmp_ch_changed = rtw_ap_update_chbw_by_ifbmp(dvobj, ifbmp
+							, cur_ie_band, cur_ie_ch, cur_ie_bw, cur_ie_offset
+							, dec_band, dec_ch, dec_bw, dec_offset
+							, __func__);
+
+	if (u_ch != 0)
+		RTW_INFO("%s union:%u,%u,%u,%u\n", __func__, u_band, u_ch, u_bw, u_offset);
+
+	if (rtw_mi_check_fwstate(adapter, WIFI_UNDER_SURVEY)) {
+		/* scanning, leave ch setting to scan state machine */
+		set_u_ch = _FALSE;
+		RTW_INFO("%s, %d NG union:%u,%u,%u,%u\n"
+		, __FUNCTION__, __LINE__, u_band, u_ch, u_bw, u_offset);
+	}
+
+	if (set_u_ch == _TRUE) {
+		chdef->band = u_band;
+		chdef->chan = u_ch;
+		chdef->bw = u_bw;
+		chdef->offset= u_offset;
+	}
+exit:
+	return ifbmp_ch_changed;
+}
+
+u8 rtw_ap_sta_states_check(_adapter *adapter)
+{
+	struct sta_info *psta;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	_list *plist, *phead;
+	u8 rst = _FALSE;
+
+	if (!MLME_IS_AP(adapter) && !MLME_IS_MESH(adapter))
+		return _FALSE;
+
+	if (pstapriv->auth_list_cnt !=0)
+		return _TRUE;
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+
+		if (!(psta->state & WIFI_ASOC_STATE)) {
+			RTW_INFO(ADPT_FMT"- SoftAP/Mesh - sta under linking, its state = 0x%x\n", ADPT_ARG(adapter), psta->state);
+			rst = _TRUE;
+			break;
+		} else if (psta->state & WIFI_UNDER_KEY_HANDSHAKE) {
+			RTW_INFO(ADPT_FMT"- SoftAP/Mesh - sta under key handshaking, its state = 0x%x\n", ADPT_ARG(adapter), psta->state);
+			rst = _TRUE;
+			break;
+		}
+	}
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+	return rst;
+}
+
+void rtw_ap_parse_sta_capability(_adapter *adapter, struct sta_info *sta, u8 *cap)
+{
+	sta->capability = RTW_GET_LE16(cap);
+	if (sta->capability & WLAN_CAPABILITY_SHORT_PREAMBLE)
+		sta->flags |= WLAN_STA_SHORT_PREAMBLE;
+	else
+		sta->flags &= ~WLAN_STA_SHORT_PREAMBLE;
+}
+
+u16 rtw_ap_parse_sta_supported_rates(_adapter *adapter, struct sta_info *sta, u8 *tlv_ies, u16 tlv_ies_len)
+{
+	u8 rate_set[12];
+	u8 rate_num;
+	int i;
+	u16 status = _STATS_SUCCESSFUL_;
+
+	rtw_ies_get_supported_rate(tlv_ies, tlv_ies_len, rate_set, &rate_num);
+	if (rate_num == 0) {
+		RTW_INFO(FUNC_ADPT_FMT" sta "MAC_FMT" with no supported rate\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+		status = _STATS_FAILURE_;
+		goto exit;
+	}
+
+	_rtw_memcpy(sta->bssrateset, rate_set, rate_num);
+	sta->bssratelen = rate_num;
+
+	if (MLME_IS_AP(adapter)) {
+		/* this function force only CCK rates to be bassic rate... */
+		UpdateBrateTblForSoftAP(sta->bssrateset, sta->bssratelen);
+	}
+
+	/* if (hapd->iface->current_mode->mode == HOSTAPD_MODE_IEEE80211G) */ /* ? */
+	sta->flags |= WLAN_STA_NONERP;
+	for (i = 0; i < sta->bssratelen; i++) {
+		if ((sta->bssrateset[i] & 0x7f) > 22) {
+			sta->flags &= ~WLAN_STA_NONERP;
+			break;
+		}
+	}
+
+exit:
+	return status;
+}
+
+u16 rtw_ap_parse_sta_security_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems)
+{
+	struct security_priv *sec = &adapter->securitypriv;
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	u8 *wpa_ie;
+	int wpa_ie_len;
+	int group_cipher = 0, pairwise_cipher = 0, gmcs = 0;
+	u32 akm = 0;
+	u8 mfp_opt = MFP_NO;
+	u8 spp_opt = 0;
+	u16 status = _STATS_SUCCESSFUL_;
+
+	sta->dot8021xalg = 0;
+	sta->wpa_psk = 0;
+	sta->wpa_group_cipher = 0;
+	sta->wpa2_group_cipher = 0;
+	sta->wpa_pairwise_cipher = 0;
+	sta->wpa2_pairwise_cipher = 0;
+	sta->akm_suite_type = 0;
+	_rtw_memset(sta->wpa_ie, 0, sizeof(sta->wpa_ie));
+
+	if ((sec->wpa_psk & BIT(1)) && elems->rsn_ie) {
+		wpa_ie = elems->rsn_ie;
+		wpa_ie_len = elems->rsn_ie_len;
+
+		if (rtw_parse_wpa2_ie(wpa_ie - 2, wpa_ie_len + 2, &group_cipher, &pairwise_cipher, &gmcs, &akm, &mfp_opt, &spp_opt) == _SUCCESS) {
+			sta->dot8021xalg = 1;/* psk, todo:802.1x */
+			sta->wpa_psk |= BIT(1);
+
+			/* RSN optional field absent; the validation is already checked in rtw_rsne_info_parse() */
+			if (!group_cipher) {
+				RTW_INFO("STA lacks WPA2 Group Suite Cipher --> Default\n");
+				group_cipher = sec->wpa2_group_cipher;
+			}
+			if (!pairwise_cipher) {
+				RTW_INFO("STA lacks WPA2 Pairwise Suite Cipher --> Default\n");
+				pairwise_cipher = sec->wpa2_pairwise_cipher;
+			}
+			if (!akm) {
+				RTW_INFO("STA lacks WPA2 AKM Cipher --> Default\n");
+				akm = sec->akmp;
+			}
+
+			sta->wpa2_group_cipher = group_cipher & sec->wpa2_group_cipher;
+			sta->wpa2_pairwise_cipher = pairwise_cipher & sec->wpa2_pairwise_cipher;
+			sta->akm_suite_type = akm;
+
+			if (MLME_IS_AP(adapter) && (CHECK_BIT(WLAN_AKM_TYPE_SAE, akm)) && (MFP_NO == mfp_opt)) {
+				status = WLAN_STATUS_ROBUST_MGMT_FRAME_POLICY_VIOLATION;
+				goto exit;
+			}
+
+			/* RSN optional field exists but no match */
+			if (!sta->wpa2_group_cipher) {
+				status = WLAN_STATUS_GROUP_CIPHER_NOT_VALID;
+				goto exit;
+			}
+			if (!sta->wpa2_pairwise_cipher) {
+				status = WLAN_STATUS_PAIRWISE_CIPHER_NOT_VALID;
+				goto exit;
+			}
+			if (MLME_IS_AP(adapter) && (!CHECK_BIT(sec->akmp, akm))) {
+				status = WLAN_STATUS_AKMP_NOT_VALID;
+				goto exit;
+			}
+		}
+		else {
+			status = WLAN_STATUS_INVALID_IE;
+			goto exit;
+		}
+	}
+	else if ((sec->wpa_psk & BIT(0)) && elems->wpa_ie) {
+		wpa_ie = elems->wpa_ie;
+		wpa_ie_len = elems->wpa_ie_len;
+
+		if (rtw_parse_wpa_ie(wpa_ie - 2, wpa_ie_len + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
+			sta->dot8021xalg = 1;/* psk, todo:802.1x */
+			sta->wpa_psk |= BIT(0);
+
+			sta->wpa_group_cipher = group_cipher & sec->wpa_group_cipher;
+			sta->wpa_pairwise_cipher = pairwise_cipher & sec->wpa_pairwise_cipher;
+
+			if (!sta->wpa_group_cipher) {
+				status = WLAN_STATUS_GROUP_CIPHER_NOT_VALID;
+				goto exit;
+			}
+
+			if (!sta->wpa_pairwise_cipher) {
+				status = WLAN_STATUS_PAIRWISE_CIPHER_NOT_VALID;
+				goto exit;
+			}
+		} else {
+			status = WLAN_STATUS_INVALID_IE;
+			goto exit;
+		}
+
+	} else {
+		wpa_ie = NULL;
+		wpa_ie_len = 0;
+	}
+
+	if (sec->dot11PrivacyAlgrthm != _NO_PRIVACY_) {
+		/*check if amsdu is allowed */
+		if (rtw_check_amsdu_disable(adapter->registrypriv.amsdu_mode, spp_opt) == _TRUE)
+			sta->flags |= WLAN_STA_AMSDU_DISABLE;
+	}
+
+	/* mfp_opt=0 for WPS connection */
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS) && elems->wps_ie)
+		goto ignore_mfp_opt;
+
+	if ((sec->mfp_opt == MFP_REQUIRED && mfp_opt < MFP_OPTIONAL)
+		|| (mfp_opt == MFP_REQUIRED && sec->mfp_opt < MFP_OPTIONAL)
+	) {
+		status = WLAN_STATUS_ROBUST_MGMT_FRAME_POLICY_VIOLATION;
+		goto exit;
+	}
+ignore_mfp_opt:
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		/* MFP is mandatory for secure mesh */
+		if (adapter->mesh_info.mesh_auth_id)
+			sta->flags |= WLAN_STA_MFP;
+	} else
+#endif
+	if (sec->mfp_opt >= MFP_OPTIONAL && mfp_opt >= MFP_OPTIONAL)
+		sta->flags |= WLAN_STA_MFP;
+
+#ifdef CONFIG_IEEE80211W
+	if ((sta->flags & WLAN_STA_MFP)
+		&& (sec->mfp_opt >= MFP_OPTIONAL && mfp_opt >= MFP_OPTIONAL)
+		&& security_type_bip_to_gmcs(sec->dot11wCipher) != gmcs
+	) {
+		status = WLAN_STATUS_CIPHER_REJECTED_PER_POLICY;
+		goto exit;
+	}
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (MLME_IS_AP(adapter) &&
+		(sec->auth_type == MLME_AUTHTYPE_SAE) &&
+		(CHECK_BIT(WLAN_AKM_TYPE_SAE, sta->akm_suite_type)) &&
+		(WLAN_AUTH_OPEN == sta->authalg)) {
+		/* WPA3-SAE, PMK caching */
+		if (rtw_cached_pmkid(adapter, sta->phl_sta->mac_addr) == -1) {
+			RTW_INFO("SAE: No PMKSA cache entry found\n");
+			status = WLAN_STATUS_INVALID_PMKID;
+			goto exit;
+		} else {
+			RTW_INFO("SAE: PMKSA cache entry found\n");
+		}
+	}
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	if (!MLME_IS_AP(adapter))
+		goto exit;
+
+	sta->flags &= ~(WLAN_STA_WPS | WLAN_STA_MAYBE_WPS);
+	/* if (hapd->conf->wps_state && wpa_ie == NULL) { */ /* todo: to check ap if supporting WPS */
+	if (wpa_ie == NULL) {
+		if (elems->wps_ie) {
+			RTW_INFO("STA included WPS IE in "
+				 "(Re)Association Request - assume WPS is "
+				 "used\n");
+			sta->flags |= WLAN_STA_WPS;
+			/* wpabuf_free(sta->wps_ie); */
+			/* sta->wps_ie = wpabuf_alloc_copy(elems.wps_ie + 4, */
+			/*				elems.wps_ie_len - 4); */
+		} else {
+			RTW_INFO("STA did not include WPA/RSN IE "
+				 "in (Re)Association Request - possible WPS "
+				 "use\n");
+			sta->flags |= WLAN_STA_MAYBE_WPS;
+		}
+
+		/* AP support WPA/RSN, and sta is going to do WPS, but AP is not ready */
+		/* that the selected registrar of AP is _FLASE */
+		if ((sec->wpa_psk > 0)
+			&& (sta->flags & (WLAN_STA_WPS | WLAN_STA_MAYBE_WPS))
+		) {
+			struct mlme_priv *mlme = &adapter->mlmepriv;
+
+			if (mlme->wps_beacon_ie) {
+				u8 selected_registrar = 0;
+
+				rtw_get_wps_attr_content(mlme->wps_beacon_ie, mlme->wps_beacon_ie_len, WPS_ATTR_SELECTED_REGISTRAR, &selected_registrar, NULL);
+
+				if (!selected_registrar) {
+					RTW_INFO("selected_registrar is _FALSE , or AP is not ready to do WPS\n");
+					status = _STATS_UNABLE_HANDLE_STA_;
+					goto exit;
+				}
+			}
+		}
+
+	} else {
+		int copy_len;
+
+		if (sec->wpa_psk == 0) {
+			RTW_INFO("STA " MAC_FMT
+				": WPA/RSN IE in association request, but AP don't support WPA/RSN\n",
+				MAC_ARG(sta->phl_sta->mac_addr));
+			status = WLAN_STATUS_INVALID_IE;
+			goto exit;
+		}
+
+		if (elems->wps_ie) {
+			RTW_INFO("STA included WPS IE in "
+				 "(Re)Association Request - WPS is "
+				 "used\n");
+			sta->flags |= WLAN_STA_WPS;
+			copy_len = 0;
+		} else
+			copy_len = ((wpa_ie_len + 2) > sizeof(sta->wpa_ie)) ? (sizeof(sta->wpa_ie)) : (wpa_ie_len + 2);
+
+		if (copy_len > 0)
+			_rtw_memcpy(sta->wpa_ie, wpa_ie - 2, copy_len);
+	}
+
+exit:
+	return status;
+}
+
+void rtw_ap_parse_sta_wmm_ie(_adapter *adapter, struct sta_info *sta, u8 *tlv_ies, u16 tlv_ies_len)
+{
+	struct link_mlme_priv *mlme = &sta->padapter_link->mlmepriv;
+	unsigned char WMM_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01};
+	u8 *p;
+
+	sta->flags &= ~WLAN_STA_WME;
+	sta->qos_option = 0;
+	sta->qos_info = 0;
+	sta->has_legacy_ac = _TRUE;
+	sta->uapsd_vo = 0;
+	sta->uapsd_vi = 0;
+	sta->uapsd_be = 0;
+	sta->uapsd_bk = 0;
+
+	if (!mlme->qospriv.qos_option)
+		goto exit;
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		/* QoS is mandatory in mesh */
+		sta->flags |= WLAN_STA_WME;
+	}
+#endif
+
+	p = rtw_get_ie_ex(tlv_ies, tlv_ies_len, WLAN_EID_VENDOR_SPECIFIC, WMM_IE, 6, NULL, NULL);
+	if (!p)
+		goto exit;
+
+	sta->flags |= WLAN_STA_WME;
+	sta->qos_option = 1;
+	sta->qos_info = *(p + 8);
+	sta->max_sp_len = (sta->qos_info >> 5) & 0x3;
+
+	if ((sta->qos_info & 0xf) != 0xf)
+		sta->has_legacy_ac = _TRUE;
+	else
+		sta->has_legacy_ac = _FALSE;
+
+	if (sta->qos_info & 0xf) {
+		if (sta->qos_info & BIT(0))
+			sta->uapsd_vo = BIT(0) | BIT(1);
+		else
+			sta->uapsd_vo = 0;
+
+		if (sta->qos_info & BIT(1))
+			sta->uapsd_vi = BIT(0) | BIT(1);
+		else
+			sta->uapsd_vi = 0;
+
+		if (sta->qos_info & BIT(2))
+			sta->uapsd_bk = BIT(0) | BIT(1);
+		else
+			sta->uapsd_bk = 0;
+
+		if (sta->qos_info & BIT(3))
+			sta->uapsd_be = BIT(0) | BIT(1);
+		else
+			sta->uapsd_be = 0;
+	}
+
+exit:
+	return;
+}
+
+void rtw_ap_parse_sta_ht_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems)
+{
+	struct link_mlme_priv *mlme = &sta->padapter_link->mlmepriv;
+
+	sta->flags &= ~WLAN_STA_HT;
+
+#ifdef CONFIG_80211N_HT
+	if (mlme->htpriv.ht_option == _FALSE)
+		goto exit;
+
+	/* save HT capabilities in the sta object */
+	_rtw_memset(&sta->htpriv.ht_cap, 0, sizeof(struct rtw_ieee80211_ht_cap));
+	if (elems->ht_capabilities && elems->ht_capabilities_len >= sizeof(struct rtw_ieee80211_ht_cap)) {
+		sta->flags |= WLAN_STA_HT;
+		sta->flags |= WLAN_STA_WME;
+		_rtw_memcpy(&sta->htpriv.ht_cap, elems->ht_capabilities, sizeof(struct rtw_ieee80211_ht_cap));
+
+		if (elems->ht_operation && elems->ht_operation_len == HT_OP_IE_LEN) {
+			_rtw_memcpy(sta->htpriv.ht_op, elems->ht_operation, HT_OP_IE_LEN);
+			sta->htpriv.op_present = 1;
+		}
+	}
+exit:
+#endif
+
+	return;
+}
+
+void rtw_ap_parse_sta_vht_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems)
+{
+	struct link_mlme_priv *mlme = &sta->padapter_link->mlmepriv;
+
+	sta->flags &= ~WLAN_STA_VHT;
+
+#ifdef CONFIG_80211AC_VHT
+	if (mlme->vhtpriv.vht_option == _FALSE)
+		goto exit;
+
+	_rtw_memset(&sta->vhtpriv, 0, sizeof(struct vht_priv));
+	if (elems->vht_capabilities && elems->vht_capabilities_len == VHT_CAP_IE_LEN) {
+		sta->flags |= WLAN_STA_VHT;
+		_rtw_memcpy(sta->vhtpriv.vht_cap, elems->vht_capabilities, VHT_CAP_IE_LEN);
+
+		if (elems->vht_operation && elems->vht_operation_len== VHT_OP_IE_LEN) {
+			_rtw_memcpy(sta->vhtpriv.vht_op, elems->vht_operation, VHT_OP_IE_LEN);
+			sta->vhtpriv.op_present = 1;
+		}
+
+		if (elems->vht_op_mode_notify && elems->vht_op_mode_notify_len == 1) {
+			_rtw_memcpy(&sta->vhtpriv.vht_op_mode_notify, elems->vht_op_mode_notify, 1);
+			sta->vhtpriv.notify_present = 1;
+		}
+	}
+exit:
+#endif
+
+	return;
+}
+
+void rtw_ap_parse_sta_he_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems)
+{
+	struct link_mlme_priv *mlme = &sta->padapter_link->mlmepriv;
+
+	sta->flags &= ~WLAN_STA_HE;
+
+#ifdef CONFIG_80211AX_HE
+	if (mlme->hepriv.he_option == _FALSE)
+		goto exit;
+
+	_rtw_memset(&sta->hepriv, 0, sizeof(struct he_priv));
+	if (elems->he_capabilities && (elems->he_capabilities_len <= HE_CAP_ELE_MAX_LEN)) {
+		sta->flags |= WLAN_STA_HE;
+		_rtw_memcpy(sta->hepriv.he_cap, elems->he_capabilities, elems->he_capabilities_len);
+
+		if (elems->he_operation && (elems->he_operation_len <= HE_OPER_ELE_MAX_LEN)) {
+			_rtw_memcpy(sta->hepriv.he_op, elems->he_operation, elems->he_operation_len);
+			sta->hepriv.op_present = 1;
+		}
+
+#if 0
+		if (elems->vht_op_mode_notify && elems->vht_op_mode_notify_len == 1) {
+			_rtw_memcpy(&sta->vhtpriv.vht_op_mode_notify, elems->vht_op_mode_notify, 1);
+			sta->vhtpriv.notify_present = 1;
+		}
+#endif
+	}
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (elems->he_6g_band_cap && (elems->he_6g_band_cap_len == HE_6G_BAND_CAP_MAX_LEN)) {
+		sta->flags |= WLAN_STA_6G_BAND_CAP;
+		_rtw_memcpy(sta->hepriv.he_6g_band_cap, elems->he_6g_band_cap, elems->he_6g_band_cap_len);
+	} else {
+		sta->flags &= ~WLAN_STA_6G_BAND_CAP;
+	}
+#endif
+exit:
+#endif
+
+	return;
+}
+
+void rtw_ap_parse_sta_multi_ap_ie(_adapter *adapter, struct sta_info *sta, u8 *ies, int ies_len)
+{
+	sta->flags &= ~WLAN_STA_MULTI_AP;
+
+#ifdef CONFIG_RTW_MULTI_AP
+	if (adapter->multi_ap
+		&& (rtw_get_multi_ap_ie_ext(ies, ies_len) & MULTI_AP_BACKHAUL_STA)
+	) {
+		if (adapter->multi_ap & MULTI_AP_BACKHAUL_BSS) /* with backhaul bss, enable WDS */
+			sta->flags |= WLAN_STA_MULTI_AP | WLAN_STA_WDS;
+		else if (adapter->multi_ap & MULTI_AP_FRONTHAUL_BSS) /* fronthaul bss only */
+			sta->flags |= WLAN_STA_MULTI_AP;
+	}
+#endif
+}
+
+#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+static bool rtw_ap_data_bmc_to_uc(_adapter *adapter
+	, const u8 *da, const u8 *sa, const u8 *ori_ta
+	, u16 os_qid, _list *b2u_list)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct xmit_priv *xmitpriv = &adapter->xmitpriv;
+	_list *head, *list;
+	struct sta_info *sta;
+	char b2u_sta_id[NUM_STA];
+	u8 b2u_sta_num = 0;
+	bool bmc_need = _FALSE;
+	int i;
+
+	 rtw_stapriv_asoc_list_lock(stapriv);
+	head = &stapriv->asoc_list;
+	list = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		int stainfo_offset;
+
+		sta = LIST_CONTAINOR(list, struct sta_info, asoc_list);
+		list = get_next(list);
+
+		stainfo_offset = rtw_stainfo_offset(stapriv, sta);
+		if (stainfo_offset_valid(stainfo_offset))
+			b2u_sta_id[b2u_sta_num++] = stainfo_offset;
+	}
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+	if (!b2u_sta_num)
+		goto exit;
+
+	for (i = 0; i < b2u_sta_num; i++) {
+		struct xmit_frame *b2uframe = NULL;
+		struct pkt_attrib *attrib;
+
+		sta = rtw_get_stainfo_by_offset(stapriv, b2u_sta_id[i]);
+		if (!(sta->state & WIFI_ASOC_STATE)
+			|| _rtw_memcmp(sta->phl_sta->mac_addr, sa, ETH_ALEN) == _TRUE
+			|| (ori_ta && _rtw_memcmp(sta->phl_sta->mac_addr, ori_ta, ETH_ALEN) == _TRUE)
+			|| is_broadcast_mac_addr(sta->phl_sta->mac_addr)
+			|| is_zero_mac_addr(sta->phl_sta->mac_addr))
+			continue;
+
+		if (core_tx_alloc_xmitframe(adapter, &b2uframe, os_qid) == FAIL) {
+			bmc_need = _TRUE;
+			break;
+		}
+
+		attrib = &b2uframe->attrib;
+
+		_rtw_memcpy(attrib->ra, sta->phl_sta->mac_addr, ETH_ALEN);
+		_rtw_memcpy(attrib->ta, sta->padapter_link->mac_addr, ETH_ALEN);
+		#ifdef CONFIG_RTW_WDS
+		if (adapter_use_wds(adapter) && (sta->flags & WLAN_STA_WDS)) {
+			_rtw_memcpy(attrib->dst, da, ETH_ALEN);
+			attrib->wds = 1;
+		} else
+		#endif
+			_rtw_memcpy(attrib->dst, attrib->ra, ETH_ALEN);
+		_rtw_memcpy(attrib->src, sa, ETH_ALEN);
+
+		rtw_list_insert_tail(&b2uframe->list, b2u_list);
+	}
+
+exit:
+	return bmc_need;
+}
+
+void dump_ap_b2u_flags(void *sel, _adapter *adapter)
+{
+	RTW_PRINT_SEL(sel, "%4s %4s\n", "src", "fwd");
+	RTW_PRINT_SEL(sel, "0x%02x 0x%02x\n", adapter->b2u_flags_ap_src, adapter->b2u_flags_ap_fwd);
+}
+#endif /* CONFIG_RTW_AP_DATA_BMC_TO_UC */
+
+static int rtw_ap_nexthop_resolve(_adapter *adapter, struct xmit_frame *xframe)
+{
+	struct pkt_attrib *attrib = &xframe->attrib;
+	int ret = _SUCCESS;
+
+#ifdef CONFIG_RTW_WDS
+	if (adapter_use_wds(adapter)) {
+		if (rtw_wds_nexthop_lookup(adapter, attrib->dst, attrib->ra) == 0) {
+			if (_rtw_memcmp(attrib->dst, attrib->ra, ETH_ALEN) == _FALSE)
+				attrib->wds = 1;
+		} else {
+			ret = _FAIL;
+		}
+	} else
+#endif
+		_rtw_memcpy(attrib->ra, attrib->dst, ETH_ALEN);
+
+	return ret;
+}
+
+#ifdef CONFIG_RTW_WDS
+static void rtw_ap_data_flood_for_unknown_da(_adapter *adapter, const u8 *da,
+					     const u8 *sa, const u8 *ori_ta,
+					     u16 os_qid, _list *f_list)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct xmit_priv *xmitpriv = &adapter->xmitpriv;
+	_list *head, *list;
+	struct sta_info *sta;
+	char sta_id[NUM_STA];
+	u8 sta_num = 0;
+	int i;
+
+	rtw_stapriv_asoc_list_lock(stapriv);
+	head = &stapriv->asoc_list;
+	list = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		int stainfo_offset;
+
+		sta = LIST_CONTAINOR(list, struct sta_info, asoc_list);
+		list = get_next(list);
+
+		stainfo_offset = rtw_stainfo_offset(stapriv, sta);
+		if (stainfo_offset_valid(stainfo_offset))
+			sta_id[sta_num++] = stainfo_offset;
+	}
+
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+	if (!sta_num)
+		goto exit;
+
+	for (i = 0; i < sta_num; i++) {
+		struct xmit_frame *frame;
+		struct pkt_attrib *attrib;
+
+		sta = rtw_get_stainfo_by_offset(stapriv, sta_id[i]);
+		if (!(sta->state & WIFI_ASOC_STATE)
+		    || !(sta->flags & WLAN_STA_WDS)
+		    || _rtw_memcmp(sta->phl_sta->mac_addr, sa, ETH_ALEN) == _TRUE
+		    || (ori_ta && _rtw_memcmp(sta->phl_sta->mac_addr, ori_ta, ETH_ALEN) == _TRUE)
+		    || is_broadcast_mac_addr(sta->phl_sta->mac_addr)
+		    || is_zero_mac_addr(sta->phl_sta->mac_addr))
+			continue;
+
+		if (core_tx_alloc_xmitframe(adapter, &frame, os_qid) == FAIL)
+			break;
+
+		attrib = &frame->attrib;
+
+		_rtw_memcpy(attrib->ra, sta->phl_sta->mac_addr, ETH_ALEN);
+		_rtw_memcpy(attrib->ta, adapter_mac_addr(adapter), ETH_ALEN);
+		_rtw_memcpy(attrib->dst, da, ETH_ALEN);
+		_rtw_memcpy(attrib->src, sa, ETH_ALEN);
+		attrib->wds = 1;
+
+		rtw_list_insert_tail(&frame->list, f_list);
+	}
+
+exit:
+	return;
+}
+#endif /* CONFIG_RTW_WDS */
+
+int rtw_ap_addr_resolve(_adapter *adapter, u16 os_qid, struct xmit_frame *xframe,
+			struct sk_buff *pkt, _list *f_list)
+{
+	struct pkt_file pktfile;
+	struct ethhdr etherhdr;
+	struct pkt_attrib *attrib;
+	u8 is_da_mcast;
+
+	int res = _SUCCESS;
+
+	_rtw_open_pktfile(pkt, &pktfile);
+	if (_rtw_pktfile_read(&pktfile, (u8 *)&etherhdr, ETH_HLEN) != ETH_HLEN) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	xframe->pkt = pkt;
+#if defined(CONFIG_RTW_WDS) || CONFIG_RTW_AP_DATA_BMC_TO_UC
+	_rtw_init_listhead(f_list);
+#endif
+
+	is_da_mcast = IS_MCAST(etherhdr.h_dest);
+	if (is_da_mcast) {
+		#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+		if (rtw_ap_src_b2u_policy_chk(adapter->b2u_flags_ap_src, etherhdr.h_dest)
+		    && adapter->registrypriv.wifi_spec == 0
+		    && adapter->xmitpriv.free_xmitframe_cnt > (NR_XMITFRAME / 4)) {
+			if (rtw_ap_data_bmc_to_uc(adapter, etherhdr.h_dest,
+						  etherhdr.h_source, NULL,
+						  os_qid, f_list) == 0) {
+				res = RTW_ORI_NO_NEED;
+				goto exit;
+			}
+		}
+		#endif
+	}
+
+	attrib = &xframe->attrib;
+
+	_rtw_memcpy(attrib->dst, etherhdr.h_dest, ETH_ALEN);
+	_rtw_memcpy(attrib->src, etherhdr.h_source, ETH_ALEN);
+	_rtw_memcpy(attrib->ta, adapter_mac_addr(adapter), ETH_ALEN);
+
+	if (is_da_mcast) {
+		_rtw_memcpy(attrib->ra, attrib->dst, ETH_ALEN);
+	} else {
+		res = rtw_ap_nexthop_resolve(adapter, xframe);
+		#ifdef CONFIG_RTW_WDS
+		if (res != _SUCCESS) {
+			/* unknown DA, flood frame to every WDS STA */
+			rtw_ap_data_flood_for_unknown_da(adapter, etherhdr.h_dest,
+							 etherhdr.h_source, NULL,
+							 os_qid, f_list);
+			res = RTW_ORI_NO_NEED;
+		}
+		#endif
+	}
+
+exit:
+	return res;
+}
+
+int rtw_ap_rx_data_validate_hdr(_adapter *adapter, union recv_frame *rframe, struct sta_info **sta)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct rx_pkt_attrib *rattrib = &rframe->u.hdr.attrib;
+	u8 *whdr = get_recvframe_data(rframe);
+	u8 is_ra_bmc = 0;
+	sint ret = _FAIL;
+
+	if (!(MLME_STATE(adapter) & WIFI_ASOC_STATE))
+		goto exit;
+
+	switch (rattrib->to_fr_ds) {
+	case 2:
+		if (IS_MCAST(GetAddr1Ptr(whdr)))
+			goto exit;
+		_rtw_memcpy(rattrib->ra, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->ta, get_addr2_ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->dst, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->src, get_addr2_ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->bssid, GetAddr1Ptr(whdr), ETH_ALEN);
+		break;
+	case 3:
+		is_ra_bmc = IS_MCAST(GetAddr1Ptr(whdr)) ? 1 : 0;
+		_rtw_memcpy(rattrib->ra, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->ta, get_addr2_ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->dst, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->src, GetAddr4Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		if (!is_ra_bmc)
+			_rtw_memcpy(rattrib->bssid, GetAddr1Ptr(whdr), ETH_ALEN);
+		break;
+	default:
+		ret = RTW_RX_HANDLED; /* don't count for drop */
+		goto exit;
+	}
+
+	if (rattrib->addr_cam_vld)
+		*sta = rtw_get_stainfo_by_macid(stapriv, rattrib->macid);
+	else
+		*sta = rtw_get_stainfo(stapriv, rattrib->ta);
+
+	if (*sta == NULL) {
+		if (!is_ra_bmc && !IS_RADAR_DETECTED(adapter_to_rfctl(adapter))) {
+			#ifndef CONFIG_CUSTOMER_ALIBABA_GENERAL
+			RTW_INFO(FUNC_ADPT_FMT" issue_deauth to "MAC_FMT" with reason(7), unknown TA\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(rattrib->ta));
+			issue_deauth(adapter, rattrib->ta, WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
+			#endif
+		}
+		ret = RTW_RX_HANDLED;
+		goto exit;
+	}
+
+#ifdef CONFIG_RTW_WDS_AUTO_EN
+	if (rattrib->to_fr_ds == 3 && !(sta->flags & WLAN_STA_WDS))
+		sta->flags |= WLAN_STA_WDS;
+#endif
+
+	_rtw_spinlock_bh(&stapriv->active_time_lock);
+	if ((*sta)->start_active == _FALSE) {
+		(*sta)->start_active = _TRUE;
+		(*sta)->start_active_time = rtw_get_current_time();
+		(*sta)->latest_active_time = (*sta)->start_active_time;
+	} else {
+		(*sta)->latest_active_time = rtw_get_current_time();
+	}
+	_rtw_spinunlock_bh(&stapriv->active_time_lock);
+
+	process_pwrbit_data(adapter, rframe, *sta);
+
+	if ((get_frame_sub_type(whdr) & WIFI_QOS_DATA_TYPE) == WIFI_QOS_DATA_TYPE)
+		process_wmmps_data(adapter, rframe, *sta);
+
+	if (get_frame_sub_type(whdr) & BIT(6)) {
+		/* No data, will not indicate to upper layer, temporily count it here */
+		count_rx_stats(adapter, rframe, *sta);
+		ret = RTW_RX_HANDLED;
+		goto exit;
+	}
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+int rtw_ap_rx_msdu_act_check(union recv_frame *rframe
+	, const u8 *da, const u8 *sa
+	, u8 *msdu, enum rtw_rx_llc_hdl llc_hdl
+	, struct xmit_frame **fwd_frame, _list *f_list)
+{
+	_adapter *adapter = rframe->u.hdr.adapter;
+	struct rx_pkt_attrib *rattrib = &rframe->u.hdr.attrib;
+	struct rtw_wds_path *wpath;
+	u8 is_da_bmc = IS_MCAST(da);
+	u8 is_da_self = !is_da_bmc && _rtw_memcmp(da, adapter_mac_addr(adapter), ETH_ALEN);
+	u8 is_da_peer = 0;
+	int in_wds_tbl = 0;
+	u16 os_qid;
+	struct xmit_frame *xframe;
+	struct pkt_attrib *xattrib;
+	u8 fwd_ra[ETH_ALEN] = {0};
+	int act = 0;
+
+#ifdef CONFIG_RTW_WDS
+	/* update/create wds info for SA, RA */
+	if (adapter_use_wds(adapter)
+		&& (rframe->u.hdr.psta->state & WIFI_ASOC_STATE)
+		&& _rtw_memcmp(sa, rframe->u.hdr.psta->phl_sta->mac_addr, ETH_ALEN) == _FALSE
+	) {
+		rtw_rcu_read_lock();
+		wpath = rtw_wds_path_lookup(adapter, sa);
+		if (!wpath)
+			rtw_wds_path_add(adapter, sa, rframe->u.hdr.psta);
+		else {
+			rtw_wds_path_assign_nexthop(wpath, rframe->u.hdr.psta);
+			wpath->last_update = rtw_get_current_time();
+		}
+		rtw_rcu_read_unlock();
+	}
+#endif
+
+	/* SA is self, need no further process */
+	if (_rtw_memcmp(sa, adapter_mac_addr(adapter), ETH_ALEN) == _TRUE)
+		goto exit;
+
+	if (is_da_bmc) {
+		/* DA is bmc addr */
+		act |= RTW_RX_MSDU_ACT_INDICATE;
+		if (adapter->mlmepriv.ap_isolate)
+			goto exit;
+		goto fwd_chk;
+
+	}
+
+	if (is_da_self) {
+		/* DA is self, indicate */
+		act |= RTW_RX_MSDU_ACT_INDICATE;
+		goto exit;
+	}
+
+	/* DA is not self */
+#ifdef CONFIG_RTW_WDS
+	if (adapter_use_wds(adapter))
+		in_wds_tbl = rtw_wds_nexthop_lookup(adapter, da, fwd_ra) == 0;
+#endif
+	if (!in_wds_tbl)
+		is_da_peer = rtw_get_stainfo(&adapter->stapriv, da) ? 1 : 0;
+
+	if (in_wds_tbl || is_da_peer) {
+		/* DA is known (peer or can be forwarded by peer) */
+		if (adapter->mlmepriv.ap_isolate) {
+			#if defined(DBG_RX_DROP_FRAME)
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" DA("MAC_FMT") through peer, ap_isolate\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(da));
+			#endif
+			goto exit;
+		}
+		goto fwd_chk;
+	}
+
+	/* DA is unknown*/
+	act |= RTW_RX_MSDU_ACT_INDICATE;
+	if (adapter->mlmepriv.ap_isolate) {
+		/*
+		* unknown DA and ap_isolate, indicate only
+		* (bridge will not forward packets to originating port)
+		*/
+		goto exit;
+	}
+
+fwd_chk:
+
+	if (adapter->stapriv.asoc_list_cnt <= 1)
+		goto exit;
+
+	os_qid = rtw_os_recv_select_queue(msdu, llc_hdl);
+
+#if defined(CONFIG_RTW_WDS) || CONFIG_RTW_AP_DATA_BMC_TO_UC
+	_rtw_init_listhead(f_list);
+#endif
+
+#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+	if (is_da_bmc) {
+		if (rtw_ap_fwd_b2u_policy_chk(adapter->b2u_flags_ap_fwd, da, rattrib->to_fr_ds == 3 && !IS_MCAST(rattrib->ra))
+			&& adapter->registrypriv.wifi_spec == 0
+			&& adapter->xmitpriv.free_xmitframe_cnt > (NR_XMITFRAME / 4)
+		) {
+			if (rtw_ap_data_bmc_to_uc(adapter
+				, da, sa, rframe->u.hdr.psta->phl_sta->mac_addr
+				, os_qid, f_list) == 0
+			) {
+				if (!rtw_is_list_empty(f_list))
+					goto set_act_fwd;
+				else
+					goto exit;
+			}
+		}
+	} else
+#endif
+#ifdef CONFIG_RTW_WDS
+	if (adapter_use_wds(adapter) && !in_wds_tbl && !is_da_peer) {
+		/* unknown DA, flood frame to every WDS STA except receiving one */
+		rtw_ap_data_flood_for_unknown_da(adapter
+			, da, sa, rframe->u.hdr.psta->phl_sta->mac_addr
+			, os_qid, f_list);
+		if (!rtw_is_list_empty(f_list))
+			goto set_act_fwd;
+		else
+			goto exit;
+	} else
+#endif
+	;
+
+	if (core_tx_alloc_xmitframe(adapter, &xframe, os_qid) == FAIL) {
+		#ifdef DBG_TX_DROP_FRAME
+		RTW_INFO("DBG_TX_DROP_FRAME "FUNC_ADPT_FMT" core_tx_alloc_xmitframe fail\n"
+			, FUNC_ADPT_ARG(adapter));
+		#endif
+		goto exit;
+	}
+
+	xattrib = &xframe->attrib;
+
+	_rtw_memcpy(xattrib->dst, da, ETH_ALEN);
+	_rtw_memcpy(xattrib->src, sa, ETH_ALEN);
+	_rtw_memcpy(xattrib->ta, adapter_mac_addr(adapter), ETH_ALEN);
+
+	#ifdef CONFIG_RTW_WDS
+	if (in_wds_tbl && _rtw_memcmp(da, fwd_ra, ETH_ALEN) == _FALSE) {
+		_rtw_memcpy(xattrib->ra, fwd_ra, ETH_ALEN);
+		xattrib->wds = 1;
+	} else
+	#endif
+		_rtw_memcpy(xattrib->ra, da, ETH_ALEN);
+
+	*fwd_frame = xframe;
+
+#if defined(CONFIG_RTW_WDS) || CONFIG_RTW_AP_DATA_BMC_TO_UC
+set_act_fwd:
+#endif
+	act |= RTW_RX_MSDU_ACT_FORWARD;
+
+exit:
+	return act;
+}
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+void rtw_issue_action_token_req(_adapter *padapter, struct sta_info *pstat)
+{
+	/* Token Request Format
+	 	Category code :		1 Byte
+		Action code : 		1 Byte
+		Element field: 		4 Bytes, the duration of data transmission requested for the station.
+	*/
+
+	u8 val = 0x0;
+	u8 category = RTW_WLAN_CATEGORY_TBTX;
+	u32 tbtx_duration = TBTX_TX_DURATION*1000;
+	u8 *pframe;
+	unsigned short *fctrl;
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct _ADAPTER_LINK *padapter_link = pstat->padapter_link;
+	struct link_mlme_ext_info	*lmlmeinfo = &(padapter_link->mlmeextpriv,mlmext_info);
+	WLAN_BSSID_EX *pnetwork = &(lmlmeinfo->network);
+
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	RTW_DBG("%s: %6ph\n", __FUNCTION__, pstat->phl_sta->mac_addr);
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgnt_tx_rate(padapter, padapter_link, IEEE80211_OFDM_RATE_24MB); // issue action request using OFDM rate? 20190320 Bruce add
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy((void *)GetAddr1Ptr(pwlanhdr), pstat->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy((void *)get_addr2_ptr(pwlanhdr), padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy((void *)GetAddr3Ptr(pwlanhdr), get_my_bssid(&(pnetwork->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(val), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 4, (unsigned char *)&(tbtx_duration), &(pattrib->pktlen));
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	padapter->stapriv.last_token_holder = pstat;
+	dump_mgntframe(padapter, pmgntframe);
+
+}
+#endif	/* CONFIG_RTW_TOKEN_BASED_XMIT */
+
+void rtw_ap_set_sta_wmode(_adapter *padapter, struct sta_info *sta)
+{
+	struct link_mlme_priv *pmlmepriv = &(sta->padapter_link->mlmepriv);
+	WLAN_BSSID_EX *pcur_network = (WLAN_BSSID_EX *)&pmlmepriv->cur_network.network;
+	enum wlan_mode network_type = WLAN_MD_INVALID;
+
+#ifdef CONFIG_80211AX_HE
+	if (sta->hepriv.he_option == _TRUE)
+		network_type = WLAN_MD_11AX;
+#endif
+#ifdef CONFIG_80211AC_VHT
+	if (network_type == WLAN_MD_INVALID) {
+		if (sta->vhtpriv.vht_option == _TRUE)
+			network_type = WLAN_MD_11AC;
+	}
+#endif
+#ifdef CONFIG_80211N_HT
+	if (network_type == WLAN_MD_INVALID) {
+		if (sta->htpriv.ht_option == _TRUE)
+			network_type =  WLAN_MD_11N;
+	}
+#endif
+
+	if (BSS_EX_OP_BAND(pcur_network) != BAND_ON_24G)
+		network_type |= WLAN_MD_11A;
+	else {
+		if ((cckratesonly_included(sta->bssrateset, sta->bssratelen)) == _TRUE)
+			network_type |= WLAN_MD_11B;
+		else if ((cckrates_included(sta->bssrateset, sta->bssratelen)) == _TRUE)
+			network_type |= WLAN_MD_11BG;
+		else
+			network_type |= WLAN_MD_11G;
+	}
+
+	sta->phl_sta->wmode = network_type;
+}
+
+#if defined(CONFIG_RTW_ACS) && defined(WKARD_ACS)
+void rtw_acs_start(_adapter *padapter)
+{
+	RTW_INFO(FUNC_ADPT_FMT" not support\n", FUNC_ADPT_ARG(padapter));
+}
+
+void rtw_acs_stop(_adapter *padapter)
+{
+	RTW_INFO(FUNC_ADPT_FMT" not support\n", FUNC_ADPT_ARG(padapter));
+}
+#endif /* defined(CONFIG_RTW_ACS) && defined(WKARD_ACS) */
+
+static enum rtw_phl_status _ap_start_swch_start_notify(struct _ADAPTER *padapter)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	enum rtw_phl_status status;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_AP_START);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_SWCH_START);
+
+	msg.band_idx = padapter->ap_start_cmd_bidx;
+	msg.rsvd[0].ptr = (u8*)role;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+					       &msg, &attr, NULL);
+
+	return status;
+}
+
+static void _ap_start_swch_done_notify_cb(void *priv, struct phl_msg *msg)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": ap_start_cmd_state=%u\n",
+		FUNC_ADPT_ARG(a), a->ap_start_cmd_state);
+
+	if (msg->inbuf) {
+		rtw_vmfree(msg->inbuf, msg->inlen);
+		msg->inbuf = NULL;
+	}
+}
+
+static enum rtw_phl_status
+_ap_start_swch_done_notify(struct _ADAPTER *a, struct rtw_chan_def *chandef)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	u8 *info = NULL;
+	enum rtw_phl_status status;
+
+
+	info = rtw_vmalloc(sizeof(struct rtw_chan_def));
+	if (!info) {
+		RTW_ERR(FUNC_ADPT_FMT ": Allocate msg hub buffer fail!\n",
+			FUNC_ADPT_ARG(a));
+		return RTW_PHL_STATUS_RESOURCE;
+	}
+	_rtw_memcpy(info, chandef, sizeof(struct rtw_chan_def));
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_AP_START);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_SWCH_DONE);
+
+	msg.band_idx = a->ap_start_cmd_bidx;
+	msg.inbuf = info;
+	msg.inlen = sizeof(struct rtw_chan_def);
+
+	attr.completion.completion = _ap_start_swch_done_notify_cb;
+	attr.completion.priv = a;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+					       &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		rtw_vmfree(info, sizeof(struct rtw_chan_def));
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+	}
+
+	return status;
+}
+
+static void _ap_start_end_notify_cb(void *priv, struct phl_msg *msg)
+{
+
+	if (msg->inbuf) {
+		rtw_vmfree(msg->inbuf, msg->inlen);
+		msg->inbuf = NULL;
+	}
+}
+
+static enum rtw_phl_status
+_ap_start_end_notify(struct _ADAPTER *padapter, bool success, bool abort)
+{
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	u8 *info = NULL;
+
+	RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_AP_START\n", FUNC_ADPT_ARG(padapter));
+
+	info = rtw_vmalloc(sizeof(u8));
+	if (info == NULL) {
+		RTW_ERR("%s: alloc buffer failed!\n", __func__);
+		return RTW_PHL_STATUS_RESOURCE;
+	}
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_AP_START);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_AP_START_END);
+	if (success) {
+		*info = RTW_PHL_STATUS_SUCCESS;
+	} else {
+		*info = RTW_PHL_STATUS_FAILURE;
+	}
+
+	msg.inbuf = info;
+	msg.inlen = 1;
+	msg.band_idx = padapter->ap_start_cmd_bidx;
+	msg.rsvd[0].ptr = (u8*)role;
+
+	if(abort)
+		attr.opt = MSG_OPT_SEND_IN_ABORT;
+	attr.completion.completion = _ap_start_end_notify_cb;
+	attr.completion.priv = padapter;
+
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+					       &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		rtw_vmfree(info, sizeof(u8));
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n",
+			FUNC_ADPT_ARG(padapter), status);
+	}
+
+	return status;
+}
+
+static void ap_free_cmdobj(struct cmd_obj *pcmd)
+{
+	struct _ADAPTER *padapter = NULL;
+
+	if (!pcmd)
+		return;
+
+	padapter = pcmd->padapter;
+	if (!padapter)
+		return;
+	
+	if (pcmd->sctx) {
+		if (pcmd->res == H2C_SUCCESS)
+			rtw_sctx_done(&pcmd->sctx);
+		else
+			rtw_sctx_done_err(&pcmd->sctx, RTW_SCTX_DONE_CMD_ERROR);
+	}
+
+	rtw_free_cmd_obj(pcmd);
+
+	return;
+}
+
+static void _ap_start_cmd_done(struct cmd_obj *pcmd)
+{
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *mlme = &(alink->mlmepriv);
+	enum rtw_phl_status status;
+
+	if (pcmd->cmd_token == 0){
+		RTW_ERR("%s : token is NULL!\n", __func__);
+		return;
+	}
+
+	status = rtw_phl_free_cmd_token(GET_PHL_INFO(d),
+					padapter->ap_start_cmd_bidx,
+					&(pcmd->cmd_token));
+
+	padapter->ap_start_cmd_state = 0;
+
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": free_cmd_token fail(0x%x)!\n",
+			FUNC_ADPT_ARG(padapter), status);
+
+	ap_free_cmdobj(pcmd);
+
+	rtw_indicate_connect(padapter);
+	mlme->cur_network.join_res = _TRUE; /* for check if already set beacon */
+
+	RTW_INFO("%s\n", __func__);
+}
+
+enum rtw_phl_status rtw_free_bcn_entry(struct _ADAPTER *padapter){
+
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+#ifdef RTW_PHL_BCN
+	status = rtw_phl_free_bcn_entry(d->phl, padapter_link->wrlink);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("%s - phl_free_bcn_entry failed\n", __func__);
+	}
+#endif
+
+	return status;
+}
+
+static enum phl_mdl_ret_code _ap_start_req_acquired(void *dispr, void *priv)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct createbss_parm *parm = (struct createbss_parm *)pcmd->parmbuf;
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+	padapter->ap_start_cmd_bidx = pcmd->band_idx;
+
+	rtw_core_ap_chan_decision(padapter, parm);
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_AP_START);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_AP_START_PREPARE);
+	msg.band_idx = padapter->ap_start_cmd_bidx;
+
+	status = rtw_phl_send_msg_to_dispr(dvobj->phl,
+	                                    &msg,
+	                                    &attr,
+	                                    NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		#ifdef CONFIG_DBCC_SUPPORT
+		if (parm->is_trigger_dbcc == _TRUE)
+			rtw_phl_mr_cancel_dbcc_action(dvobj->phl);
+		#endif
+		return MDL_RET_FAIL;
+	} else {
+		SET_STATUS_FLAG(padapter->ap_start_cmd_state, CMD_APSTART_ACQUIRE);
+		return MDL_RET_SUCCESS;
+	}
+}
+
+static enum phl_mdl_ret_code _ap_start_req_abort(void *dispr, void *priv)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+
+	if (TEST_STATUS_FLAG(padapter->ap_start_cmd_state, CMD_APSTART_ACQUIRE)) {
+		status = _ap_start_end_notify(padapter, false, true);
+	}
+
+	padapter->ap_start_cmd_state = 0;
+	RTW_INFO("%s\n", __func__);
+	return MDL_RET_SUCCESS;
+}
+
+const char *ap_get_evt_str(u32 evt)
+{
+	switch (evt) {
+	/* AP */
+	case MSG_EVT_AP_START_PREPARE:
+		return "MSG_EVT_AP_START_PREPARE";
+	case MSG_EVT_AP_START:
+		return "MSG_EVT_AP_START";
+	case MSG_EVT_AP_START_END:
+		return "MSG_EVT_AP_START_END";
+	case MSG_EVT_AP_STOP_PREPARE:
+		return "MSG_EVT_AP_STOP_PREPARE";
+	case MSG_EVT_AP_STOP:
+		return "MSG_EVT_AP_STOP";
+	case MSG_EVT_AP_STOP_END:
+		return "MSG_EVT_AP_STOP_END";
+
+	case MSG_EVT_SWCH_START:
+		return "MSG_EVT_SWCH_START";
+	case MSG_EVT_SWCH_DONE:
+		return "MSG_EVT_SWCH_DONE";
+	case MSG_EVT_ROLE_NTFY:
+		return "MSG_EVT_ROLE_NTFY";
+
+	/* STA */
+	case MSG_EVT_CONNECT_START:
+		return "MSG_EVT_CONNECT_START";
+	case MSG_EVT_CONNECT_LINKED:
+		return "MSG_EVT_CONNECT_LINKED";
+	case MSG_EVT_CONNECT_END:
+		return "MSG_EVT_CONNECT_END";
+	case MSG_EVT_DISCONNECT_PREPARE:
+		return "MSG_EVT_DISCONNECT_PREPARE";
+	case MSG_EVT_DISCONNECT:
+		return "MSG_EVT_DISCONNECT";
+
+	case MSG_EVT_DBCC_DISABLE:
+		return "MSG_EVT_DBCC_DISABLE";
+	default:
+		printk("%s evt=0x%08x\n",__func__, evt);
+		return "unknown EVT";
+	}
+}
+
+static enum phl_mdl_ret_code _ap_start_req_ev_hdlr(void *dispr, void *priv, struct phl_msg *msg)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct createbss_parm *parm = (struct createbss_parm *)pcmd->parmbuf;
+	struct rtw_wifi_role_t *wifi_role = padapter->phl_role;
+	struct rtw_phl_com_t *phl_com = wifi_role->phl_com;
+	void *d = phlcom_to_drvpriv(phl_com);
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_DBG(FUNC_ADPT_FMT ": + msg_id=0x%08x\n",
+		FUNC_ADPT_ARG(padapter), msg->msg_id);
+
+	if (MSG_MDL_ID_FIELD(msg->msg_id) != PHL_FG_MDL_AP_START) {
+		/*RTW_INFO(FUNC_ADPT_FMT ": Message is not from ap start module, "
+			 "skip msg_id=0x%08x\n", FUNC_ADPT_ARG(padapter), msg->msg_id);
+		RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));*/
+		return MDL_RET_IGNORE;
+	}
+
+	RTW_INFO("[AP_START] %s("ADPT_FMT")\n",
+		ap_get_evt_str(MSG_EVT_ID_FIELD(msg->msg_id)), ADPT_ARG(padapter));
+
+	if (IS_MSG_FAIL(msg->msg_id)) {
+		RTW_WARN(FUNC_ADPT_FMT ": cmd dispatcher notify cmd failure on "
+			 "msg_id=0x%08x\n", FUNC_ADPT_ARG(padapter), msg->msg_id);
+	}
+
+	switch (MSG_EVT_ID_FIELD(msg->msg_id)) {
+	case MSG_EVT_AP_START_PREPARE:
+
+		if ((parm->req_ch == 0 &&
+			pmlmeinfo->state == WIFI_FW_AP_STATE) ||
+			parm->req_ch != 0
+		)
+			rtw_core_ap_prepare(padapter, parm);
+
+		status = _ap_start_swch_start_notify(padapter);
+		if(status != RTW_PHL_STATUS_SUCCESS)
+			RTW_ERR("%s :: [MSG_EVT_AP_START_PREPARE] rtw_phl_send_msg_to_dispr failed\n", __func__);
+		break;
+
+	case MSG_EVT_SWCH_START:
+		rtw_core_ap_swch_start(padapter, parm);
+		status = _ap_start_swch_done_notify(padapter, &padapter_link->wrlink->chandef);
+		if(status != RTW_PHL_STATUS_SUCCESS)
+			RTW_ERR("%s :: [SWCH_START] rtw_phl_send_msg_to_dispr failed\n", __func__);
+		break;
+
+	case MSG_EVT_SWCH_DONE:
+		status = rtw_phl_ap_started(adapter_to_dvobj(padapter)->phl,
+					padapter->ap_start_cmd_bidx, padapter->phl_role);
+		if(status != RTW_PHL_STATUS_SUCCESS)
+			RTW_ERR("%s :: [MSG_EVT_SWCH_DONE] rtw_phl_send_msg_to_dispr failed\n", __func__);
+		break;
+
+	case MSG_EVT_AP_START:
+		rtw_core_ap_start(padapter, parm); /* send beacon */
+		status = _ap_start_end_notify(padapter, true, false);
+		break;
+
+	case MSG_EVT_AP_START_END:
+
+		if (msg->inbuf == NULL) {
+			RTW_WARN("[MSG_EVT_AP_START_END] Msg info buffer NULL!\n");
+			break;
+		}
+
+		if (*(msg->inbuf) == RTW_PHL_STATUS_FAILURE){
+			RTW_WARN("[MSG_EVT_AP_START_END] Msg_inbuf is FAIL!\n");
+			stop_ap_hdl(padapter);
+		}
+
+		_ap_start_cmd_done(pcmd);
+		break;
+
+	default:
+		/* unknown state */
+		break;
+	}
+
+
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		/* Trigger stop AP flow when start AP failure */
+		stop_ap_hdl(padapter);
+		_ap_start_cmd_done(pcmd);
+	}
+
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _ap_start_req_set_info(void *dispr, void *priv, struct phl_module_op_info* info)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+
+	return MDL_RET_IGNORE;
+}
+
+static enum phl_mdl_ret_code _ap_start_req_query_info(void *dispr, void *priv, struct phl_module_op_info* info)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+
+	switch (info->op_code) {
+	case FG_REQ_OP_GET_ROLE:
+		info->outbuf = (u8*)padapter->phl_role;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	case FG_REQ_OP_GET_ROLE_LINK:
+		info->outbuf = (u8*)padapter_link->wrlink;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	default:
+		break;
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+	return ret;
+}
+
+static void rtw_cmd_ap_start_req_init(struct cmd_obj *pcmd, struct phl_cmd_token_req *fgreq)
+{
+	struct _ADAPTER *padapter = pcmd->padapter;
+	u8 res = _SUCCESS;
+
+	/* Fill foreground command request */
+	fgreq->module_id= PHL_FG_MDL_AP_START;
+	fgreq->opt = FG_CMD_OPT_EXCLUSIVE;
+	fgreq->priv = pcmd;
+	fgreq->role = padapter->phl_role;
+
+	fgreq->acquired = _ap_start_req_acquired;
+	fgreq->abort = _ap_start_req_abort;
+	fgreq->msg_hdlr = _ap_start_req_ev_hdlr;
+	fgreq->set_info = _ap_start_req_set_info;
+	fgreq->query_info = _ap_start_req_query_info;
+
+	padapter->ap_start_cmd_state = 0;
+	pcmd->cmd_token = 0;
+
+	RTW_INFO("%s\n", __func__);
+}
+
+enum rtw_phl_status rtw_ap_start_cmd(struct cmd_obj *pcmd)
+{
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct phl_cmd_token_req fgreq={0};
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	enum rtw_phl_status status;
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	rtw_mi_set_scan_deny(padapter, 2000);
+	rtw_cmd_ap_start_req_init(pcmd, &fgreq);
+
+	RTW_DBG(FUNC_ADPT_FMT ": \n", FUNC_ADPT_ARG(padapter));
+
+	/* cmd_dispatcher would copy whole phl_cmd_token_req */
+	status = rtw_phl_add_cmd_token_req(d->phl, padapter_link->wrlink->hw_band, &fgreq, &(pcmd->cmd_token));
+	if ((status != RTW_PHL_STATUS_SUCCESS) &&
+	   (status != RTW_PHL_STATUS_PENDING))
+		goto error;
+
+	status = RTW_PHL_STATUS_SUCCESS;
+
+error:
+	return status;
+
+}
+
+static void _ap_stop_cmd_done(struct cmd_obj *pcmd)
+{
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	enum rtw_phl_status status;
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+
+	if (!padapter->ap_stop_cmd_token){
+		RTW_ERR("%s : token is NULL!\n", __func__);
+		return;
+	}
+
+	status = rtw_phl_free_cmd_token(GET_PHL_INFO(d),
+					padapter->ap_stop_cmd_bidx,
+					&padapter->ap_stop_cmd_token);
+
+	padapter->ap_stop_cmd_token = 0;
+	padapter->ap_stop_cmd_state = 0;
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": free_cmd_token fail(0x%x)!\n",
+			FUNC_ADPT_ARG(padapter), status);
+
+	ap_free_cmdobj(pcmd);
+	rtw_ap_stop_set_state(padapter, AP_STOP_ST_DONE);
+
+	RTW_INFO(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+}
+
+static enum rtw_phl_status _ap_stop_ap_stop_notify(struct _ADAPTER *adapter)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+	struct rtw_wifi_role_t *role = adapter->phl_role;
+	enum rtw_phl_status status;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_AP_STOP);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_AP_STOP);
+
+	msg.band_idx = adapter->ap_stop_cmd_bidx;
+	msg.rsvd[0].ptr = (u8*)role;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+					       &msg, &attr, NULL);
+
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n", FUNC_ADPT_ARG(adapter), status);
+	}
+	return status;
+}
+
+static enum rtw_phl_status _ap_stop_ap_end_notify(struct _ADAPTER *adapter)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+	struct rtw_wifi_role_t *role = adapter->phl_role;
+	enum rtw_phl_status status;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_AP_STOP);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_AP_STOP_END);
+
+	msg.band_idx = adapter->ap_stop_cmd_bidx;
+	msg.inbuf = (u8 *)role;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+					       &msg, &attr, NULL);
+
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n", FUNC_ADPT_ARG(adapter), status);
+
+	return status;
+}
+
+static void _ap_stop_abort_notify_cb(void *priv, struct phl_msg *msg)
+{
+	_adapter *padapter = (_adapter *)priv;
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	struct _ADAPTER *adapter = (struct _ADAPTER *)priv;
+	enum rtw_phl_status phl_status = RTW_PHL_STATUS_SUCCESS;
+	struct rtw_chan_def chan_def = {0};
+	u8 chctx_num = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+#ifdef RTW_PHL_BCN
+	phl_status = rtw_phl_free_bcn_entry(d->phl, adapter_link->wrlink);
+	if (phl_status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("%s - phl_free_bcn_entry failed\n", __func__);
+	}
+#endif
+}
+
+static enum rtw_phl_status _ap_stop_abort_notify(struct _ADAPTER *adapter, bool abort)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+	struct rtw_wifi_role_t *role = adapter->phl_role;
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_AP_STOP);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_AP_STOP);
+	msg.band_idx = adapter->ap_stop_cmd_bidx;
+	msg.rsvd[0].ptr = (u8*)role;
+
+	if(abort)
+		attr.opt = MSG_OPT_SEND_IN_ABORT;
+	attr.completion.completion = _ap_stop_abort_notify_cb;
+	attr.completion.priv = adapter;
+
+	status = rtw_phl_send_msg_to_dispr(d->phl,
+	                            		&msg,
+	                                	&attr,
+	                            		NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("[%s] Abort send msg fail!\n", __func__);
+	}
+
+	return status;
+}
+
+static enum phl_mdl_ret_code _ap_stop_req_acquired(void *dispr, void *priv)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	struct phl_msg msg = {0};
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+
+	rtw_ap_stop_set_state(padapter, AP_STOP_ST_ACQUIRED);
+
+	SET_STATUS_FLAG(padapter->ap_stop_cmd_state, CMD_APSTOP_ACQUIRE);
+	padapter->ap_stop_cmd_bidx = padapter_link->wrlink->hw_band;
+
+	rtw_phl_ap_stop(adapter_to_dvobj(padapter)->phl,
+			padapter->ap_stop_cmd_bidx,role);
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+
+	return MDL_RET_SUCCESS;
+
+}
+
+static enum phl_mdl_ret_code _ap_stop_req_abort(void *dispr, void *priv)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+
+	if(TEST_STATUS_FLAG(padapter->ap_stop_cmd_state, CMD_APSTOP_STARTED)){
+		/* case1: MSG_EVT_AP_STOP has been processed*/
+	}
+	else if(TEST_STATUS_FLAG(padapter->ap_stop_cmd_state, CMD_APSTOP_ACQUIRE)){
+		/* case2: MSG_EVT_AP_STOP in msgQ
+		   case3: MSG_EVT_AP_STOP doesn't back to msg_hdlr yet */
+		if (RTW_PHL_STATUS_SUCCESS != _ap_stop_abort_notify(padapter, true)) {
+			/* do not execute core_mr_handle because do not know if the I/O operation is acceptable */
+			//core_mr_delete_chctx(adapter, &chctx_num, &chan_def);
+		}
+	}
+	else{
+		/* apstop token stay in tokenOpQ */
+	}
+
+	padapter->ap_stop_cmd_state = 0;
+	RTW_INFO("%s\n", __func__);
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _ap_stop_req_ev_hdlr(void *dispr, void *priv, struct phl_msg* msg)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *wifi_role = padapter->phl_role;
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_DBG(FUNC_ADPT_FMT ": + msg_id=0x%08x\n",
+		FUNC_ADPT_ARG(padapter), msg->msg_id);
+
+	if ((MSG_MDL_ID_FIELD(msg->msg_id) != PHL_FG_MDL_AP_STOP) &&
+		(MSG_MDL_ID_FIELD(msg->msg_id) != PHL_MDL_MRC)) {
+		RTW_DBG(FUNC_ADPT_FMT ": Message is not from ap stop/mrc module, "
+			 "skip msg_id=0x%08x\n", FUNC_ADPT_ARG(padapter), msg->msg_id);
+		RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+		return MDL_RET_IGNORE;
+	}
+
+	RTW_INFO("[AP_STOP] %s("ADPT_FMT")\n",
+		ap_get_evt_str(MSG_EVT_ID_FIELD(msg->msg_id)), ADPT_ARG(padapter));
+
+	if (IS_MSG_FAIL(msg->msg_id)) {
+		RTW_WARN(FUNC_ADPT_FMT ": cmd dispatcher notify cmd failure on "
+			 "msg_id=0x%08x\n", FUNC_ADPT_ARG(padapter), msg->msg_id);
+	}
+
+	switch(MSG_EVT_ID_FIELD(msg->msg_id)) {
+		case MSG_EVT_AP_STOP_PREPARE:
+		{
+			SET_STATUS_FLAG(padapter->ap_stop_cmd_state, CMD_APSTOP_STARTED);
+			status = _ap_stop_ap_stop_notify(padapter);
+			if (status != RTW_PHL_STATUS_SUCCESS) {
+				RTW_ERR("%s :: [MSG_EVT_AP_STOP_PREPARE] ap_stop_notify fail\n", __func__);
+				_ap_stop_cmd_done(pcmd);
+			}
+			break;
+		}
+		case MSG_EVT_AP_STOP:
+		{
+			#ifdef CONFIG_DBCC_SUPPORT
+			struct rtw_mr_chctx_info mr_cc_info = {0};
+			#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+			bool is_dis_dbcc = false;
+
+			/*B1 GO stop*/
+			is_dis_dbcc = rtw_dbcc_chk_disable_hdl(padapter, NULL, PHL_CMD_DIRECTLY, DBCC_CHK_AP_STOP);
+			#endif
+			#endif
+
+			stop_ap_hdl(padapter);
+
+			#ifdef CONFIG_DBCC_SUPPORT
+			#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+			if (is_dis_dbcc)
+			#endif
+			{
+				rtw_phl_chanctx_chk(GET_PHL_INFO(dvobj),
+								padapter->phl_role,
+								padapter_link->wrlink,
+								NULL,
+								&mr_cc_info);
+				if (mr_cc_info.sugg_opmode == MR_OP_DBCC) /*current stay in DBCC*/
+					rtw_phl_mr_trig_dbcc_disable(GET_PHL_INFO(dvobj));
+			}
+			#endif
+
+			status = _ap_stop_ap_end_notify(padapter);
+			if (status != RTW_PHL_STATUS_SUCCESS) {
+				_ap_stop_cmd_done(pcmd);
+				#ifdef CONFIG_DBCC_SUPPORT
+				if (mr_cc_info.sugg_opmode == MR_OP_DBCC)
+					rtw_phl_mr_cancel_dbcc_action(GET_PHL_INFO(dvobj));
+				#endif
+			}
+			break;
+		}
+		case MSG_EVT_AP_STOP_END:
+			if (padapter->ap_add_del_sta_cmd_token != 0) {
+				status = rtw_phl_cancel_cmd_token(GET_PHL_INFO(dvobj),
+						padapter_link->wrlink->hw_band,
+						&padapter->ap_add_del_sta_cmd_token);
+				if (status != RTW_PHL_STATUS_SUCCESS)
+					RTW_ERR(FUNC_ADPT_FMT ": cancel_cmd_token fail(0x%x)!\n",
+						FUNC_ADPT_ARG(padapter), status);
+			}
+			RTW_INFO(FUNC_ADPT_FMT ": MSG_EVT_AP_STOP_END\n", FUNC_ADPT_ARG(padapter));
+		#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+			/*B0,GO stop*/
+			if (rtw_dbcc_chk_enable_hdl(padapter, PHL_CMD_DIRECTLY, DBCC_CHK_AP_STOP))
+				rtw_phl_mr_trig_dbcc_enable(GET_PHL_INFO(dvobj));
+
+			rtw_discon_end_dbcc_en_notify(padapter, PHL_FG_MDL_AP_STOP);
+			break;
+		case MSG_EVT_DISCONNECT_END_DBCC_EN:
+			RTW_INFO(FUNC_ADPT_FMT ": MSG_EVT_DISCONNECT_END_DBCC_EN\n",
+						FUNC_ADPT_ARG(padapter));
+		#endif
+			_ap_stop_cmd_done(pcmd);
+			break;
+
+#ifdef CONFIG_DBCC_SUPPORT
+		case MSG_EVT_DBCC_DISABLE:
+			if (MSG_MDL_ID_FIELD(msg->msg_id) == PHL_MDL_MRC) {
+				struct rtw_wifi_role_t *role;
+
+				role = (struct rtw_wifi_role_t *)msg->inbuf;
+				RTW_INFO("[STOP_AP-FG] WR(%d) RX MRC- DBCC DISABLE MSG\n", role->id);
+			}
+			break;
+#endif
+
+		default:
+			/* unknown state */
+		break;
+	}
+
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _ap_stop_req_set_info(void *dispr, void *priv, struct phl_module_op_info* info)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *padapter = pcmd->padapter;
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+
+	return MDL_RET_IGNORE;
+}
+
+static enum phl_mdl_ret_code _ap_stop_req_query_info(void *dispr, void *priv, struct phl_module_op_info* info)
+{
+	struct cmd_obj *pcmd = (struct cmd_obj *)priv;
+	struct _ADAPTER *a = pcmd->padapter;
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+
+	switch (info->op_code) {
+	case FG_REQ_OP_GET_ROLE:
+		info->outbuf = (u8*)a->phl_role;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	case FG_REQ_OP_GET_ROLE_LINK:
+		info->outbuf = (u8*)alink->wrlink;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	default:
+		break;
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+	return ret;
+}
+
+void rtw_ap_stop_set_state(struct _ADAPTER *a, u8 st)
+{
+	_rtw_spinlock_bh(&a->ap_stop_st_lock);
+	a->ap_stop_state = st;
+	_rtw_spinunlock_bh(&a->ap_stop_st_lock);
+}
+
+#define WATI_AP_STOP_TO 10000	/* unit: ms */
+int rtw_ap_stop_wait(struct _ADAPTER *a)
+{
+	u32 timeout = WATI_AP_STOP_TO;
+	systime start_t;
+	u32 pass_t = 0;
+	u32 msg_i;	/* message interval */
+	u32 msg_t = 0;	/* next time point to print message */
+	int err = 0;
+
+	if (!rtw_is_adapter_up(a))
+		return err;
+
+	if (a->ap_stop_state == AP_STOP_ST_IDLE)
+		return err;
+
+	msg_i = timeout / 10;	/* at most print 10 msg in whole waiting time */
+
+	_rtw_spinlock_bh(&a->ap_stop_st_lock);
+	start_t = rtw_get_current_time();
+	while (a->ap_stop_state != AP_STOP_ST_DONE) {
+		if (!rtw_is_adapter_up(a))
+			break;
+
+		if (pass_t >= msg_t) {
+			RTW_INFO(FUNC_ADPT_FMT ": ap stop st=%u, %u ms/%u pass...\n",
+				 FUNC_ADPT_ARG(a), a->ap_stop_state, pass_t, timeout);
+			msg_t += msg_i;
+		}
+		_rtw_spinunlock_bh(&a->ap_stop_st_lock);
+
+		pass_t = rtw_get_passing_time_ms(start_t);
+		if (pass_t > timeout) {
+			RTW_ERR(FUNC_ADPT_FMT ": Timeout, fail to ap stop!"
+				" used:%u > max:%u ms\n",
+				FUNC_ADPT_ARG(a), pass_t, timeout);
+			err = -2; /* Timeout */
+			_rtw_spinlock_bh(&a->ap_stop_st_lock);
+			rtw_warn_on(1);
+			break;
+		}
+		rtw_msleep_os(50);
+
+		_rtw_spinlock_bh(&a->ap_stop_st_lock);
+	}
+	_rtw_spinunlock_bh(&a->ap_stop_st_lock);
+
+	return err;
+}
+
+static void rtw_cmd_ap_stop_req_init(struct cmd_obj *pcmd, struct phl_cmd_token_req *fgreq)
+{
+	struct _ADAPTER *padapter = pcmd->padapter;
+
+	RTW_DBG(FUNC_ADPT_FMT ": \n", FUNC_ADPT_ARG(padapter));
+
+	/* Fill foreground command request */
+	fgreq->module_id= PHL_FG_MDL_AP_STOP;
+	fgreq->opt = FG_CMD_OPT_EXCLUSIVE;
+	fgreq->priv = pcmd;
+	fgreq->role = padapter->phl_role;
+
+	fgreq->acquired = _ap_stop_req_acquired;
+	fgreq->abort = _ap_stop_req_abort;
+	fgreq->msg_hdlr = _ap_stop_req_ev_hdlr;
+	fgreq->set_info = _ap_stop_req_set_info;
+	fgreq->query_info = _ap_stop_req_query_info;
+
+	rtw_ap_stop_set_state(padapter, AP_STOP_ST_REQUESTING);
+	padapter->ap_stop_cmd_state = 0;
+	padapter->ap_stop_cmd_token = 0;
+}
+
+enum rtw_phl_status rtw_ap_stop_cmd(struct cmd_obj *pcmd)
+{
+	struct _ADAPTER *padapter = pcmd->padapter;
+	struct phl_cmd_token_req fgreq={0};
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	enum rtw_phl_status pstatus;
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+
+	rtw_cmd_ap_stop_req_init(pcmd, &fgreq);
+
+	/* cmd_dispatcher would copy whole phl_cmd_token_req */
+	pstatus = rtw_phl_add_cmd_token_req(d->phl, padapter_link->wrlink->hw_band, &fgreq, &padapter->ap_stop_cmd_token);
+	if((pstatus != RTW_PHL_STATUS_SUCCESS) &&
+	   (pstatus != RTW_PHL_STATUS_PENDING))
+		goto error;
+
+	pstatus = RTW_PHL_STATUS_SUCCESS;
+
+error:
+	return pstatus;
+
+}
+
+static void _ap_add_del_sta_cmd_done(struct _ADAPTER *padapter)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	enum rtw_phl_status status;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	padapter->ap_add_del_sta_cmd_state = ADD_DEL_STA_ST_IDLE;
+
+	if (!padapter->ap_add_del_sta_cmd_token){
+		RTW_ERR("%s : token is NULL!\n", __func__);
+		return;
+	}
+
+	status = rtw_phl_free_cmd_token(GET_PHL_INFO(d),
+					padapter_link->wrlink->hw_band,
+					&padapter->ap_add_del_sta_cmd_token);
+	padapter->ap_add_del_sta_cmd_token = 0;
+
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR(FUNC_ADPT_FMT ": free_cmd_token fail(0x%x)!\n",
+			FUNC_ADPT_ARG(padapter), status);
+		return;
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT": free token success\n", FUNC_ADPT_ARG(padapter));
+}
+
+/* use this function must add lock */
+static struct rtw_add_del_sta_obj *rtw_ap_get_stainfo_by_add_sta_list(struct _ADAPTER *padapter)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct rtw_add_del_sta_obj *add_del_sta_obj = NULL;
+	_list	*phead, *plist;
+
+	phead = &pstapriv->add_sta_list;
+	plist = get_next(phead);
+	if (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		add_del_sta_obj = LIST_CONTAINOR(plist, struct rtw_add_del_sta_obj, list);
+		plist = get_next(plist);
+	}
+
+	return add_del_sta_obj;
+}
+
+static enum phl_mdl_ret_code _ap_add_del_sta_req_acquired(void *dispr, void *priv)
+{
+	struct _ADAPTER *padapter = (struct _ADAPTER *)priv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	struct rtw_add_del_sta_obj *add_del_sta_obj;
+	struct sta_info *psta = NULL;
+	struct phl_msg_attribute attr = {0};
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	struct phl_msg msg = {0};
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+
+	_rtw_spinlock_bh(&padapter->ap_add_del_sta_lock);
+	if (padapter->ap_add_del_sta_cmd_state != ADD_DEL_STA_ST_REQUESTING)
+		RTW_ERR(FUNC_ADPT_FMT ": add_del_sta_st=%u, not requesting?!\n",
+			FUNC_ADPT_ARG(padapter), padapter->ap_add_del_sta_cmd_state);
+
+	padapter->ap_add_del_sta_cmd_state = ADD_DEL_STA_ST_ACQUIRED;
+
+	add_del_sta_obj = rtw_ap_get_stainfo_by_add_sta_list(padapter);
+
+	_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+
+	if (!add_del_sta_obj) {
+		RTW_ERR(FUNC_ADPT_FMT": add_del_sta_obj is null\n",
+			FUNC_ADPT_ARG(padapter));
+			return (enum phl_mdl_ret_code)status;
+	}
+
+	psta = add_del_sta_obj->sta;
+	if (!psta) {
+		RTW_ERR(FUNC_ADPT_FMT": psta is null\n", FUNC_ADPT_ARG(padapter));
+		return (enum phl_mdl_ret_code)status;
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT ": %s STA\n",
+		FUNC_ADPT_ARG(padapter), add_del_sta_obj->is_add_sta ? "ADD" : "DEL");
+
+	if (add_del_sta_obj->is_add_sta)
+		status = rtw_phl_connect_prepare(adapter_to_dvobj(padapter)->phl,
+				padapter_link->wrlink->hw_band, padapter->phl_role,
+				padapter_link->wrlink, psta->phl_sta->mac_addr);
+	else
+		status = rtw_phl_disconnect(adapter_to_dvobj(padapter)->phl,
+				padapter_link->wrlink->hw_band,
+				padapter->phl_role, _TRUE);
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+
+	return (status == RTW_PHL_STATUS_SUCCESS ? MDL_RET_SUCCESS : MDL_RET_FAIL);
+
+}
+
+static enum phl_mdl_ret_code _ap_add_del_sta_req_abort(void *dispr, void *priv)
+{
+	struct _ADAPTER *padapter = (struct _ADAPTER *)priv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct rtw_add_del_sta_obj *add_del_sta_obj = NULL;
+	_list *phead = NULL, *plist = NULL;
+
+	RTW_INFO("%s\n", __func__);
+
+	_rtw_spinlock_bh(&padapter->ap_add_del_sta_lock);
+
+	phead = &pstapriv->add_sta_list;
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		add_del_sta_obj = LIST_CONTAINOR(plist, struct rtw_add_del_sta_obj, list);
+		plist = get_next(plist);
+		rtw_list_delete(&add_del_sta_obj->list);
+		rtw_mfree(add_del_sta_obj, sizeof(struct rtw_add_del_sta_obj));
+	}
+
+	pstapriv->add_sta_list_cnt = 0;
+	padapter->ap_add_del_sta_cmd_token = 0;
+	padapter->ap_add_del_sta_cmd_state = ADD_DEL_STA_ST_IDLE;
+
+	_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _ap_add_del_sta_req_ev_hdlr(void *dispr, void *priv, struct phl_msg* msg)
+{
+	struct _ADAPTER *padapter = (struct _ADAPTER *)priv;
+	struct rtw_wifi_role_t *wifi_role = padapter->phl_role;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct rtw_add_del_sta_obj *add_del_sta_obj = NULL;
+	struct sta_info *psta = NULL;
+	struct phl_msg next_msg = {0};
+	_list	*phead, *plist;
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	struct rtw_phl_stainfo_t *phl_stainfo;
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	u8 *passoc_req = NULL;
+	u32 assoc_req_len = 0;
+#endif
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (MLME_IS_AP(padapter) == _FALSE)
+		return MDL_RET_IGNORE;
+
+	RTW_INFO(FUNC_ADPT_FMT ": + msg_id=0x%08x\n",
+		FUNC_ADPT_ARG(padapter), msg->msg_id);
+
+	if (MSG_MDL_ID_FIELD(msg->msg_id) != PHL_FG_MDL_AP_ADD_DEL_STA) {
+		//RTW_INFO(FUNC_ADPT_FMT ": Message is not from ap add sta module, "
+		//	 "skip msg_id=0x%08x\n", FUNC_ADPT_ARG(padapter), msg->msg_id);
+		RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+		return MDL_RET_IGNORE;
+	}
+
+	_rtw_spinlock_bh(&padapter->ap_add_del_sta_lock);
+
+	RTW_INFO("add_sta_list, cnt:%u\n", pstapriv->add_sta_list_cnt);
+
+	add_del_sta_obj = rtw_ap_get_stainfo_by_add_sta_list(padapter);
+	if (!add_del_sta_obj || !add_del_sta_obj->sta) {
+		_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+		RTW_ERR(FUNC_ADPT_FMT": add_del_sta_obj or psta is null\n",
+			FUNC_ADPT_ARG(padapter));
+		goto free_token;
+	}
+	psta = add_del_sta_obj->sta;
+	_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+
+	RTW_INFO("[AP_STA] %s("ADPT_FMT") aid=%d\n",
+		ap_get_evt_str(MSG_EVT_ID_FIELD(msg->msg_id)), ADPT_ARG(padapter), add_del_sta_obj->aid);
+
+
+	if (IS_MSG_FAIL(msg->msg_id)) {
+		RTW_WARN(FUNC_ADPT_FMT ": cmd dispatcher notify cmd failure on "
+			 "msg_id=0x%08x\n", FUNC_ADPT_ARG(padapter), msg->msg_id);
+	}
+
+	switch (MSG_EVT_ID_FIELD(msg->msg_id)) {
+	case MSG_EVT_CONNECT_START:
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+
+		rtw_alloc_stainfo_hw(&padapter->stapriv, psta);
+		rtw_hw_connected_apmode(padapter, psta);
+
+		RTW_INFO("indicate_sta_assoc_event to upper layer - hostapd\n");
+
+		_rtw_spinlock_bh(&psta->lock);
+		if (psta->passoc_req && psta->assoc_req_len > 0) {
+			passoc_req = rtw_zmalloc(psta->assoc_req_len);
+			if (passoc_req) {
+				assoc_req_len = psta->assoc_req_len;
+				_rtw_memcpy(passoc_req, psta->passoc_req, assoc_req_len);
+			}
+		}
+		_rtw_spinunlock_bh(&psta->lock);
+
+		if (passoc_req && assoc_req_len > 0) {
+			rtw_cfg80211_indicate_sta_assoc(padapter, passoc_req, assoc_req_len);
+			rtw_mfree(passoc_req, assoc_req_len);
+		}
+
+		if (is_wep_enc(padapter->securitypriv.dot11PrivacyAlgrthm))
+			rtw_ap_wep_pk_setting(padapter, psta);
+		#ifdef CONFIG_RTS_FULL_BW
+		rtw_set_rts_bw(adapter);
+		#endif/*CONFIG_RTS_FULL_BW*/
+
+		status = rtw_phl_connected(adapter_to_dvobj(padapter)->phl,
+					padapter_link->wrlink->hw_band,
+					padapter->phl_role, psta->phl_sta);
+		if (status == RTW_PHL_STATUS_FAILURE) {
+			RTW_ERR("%s :: [MSG_EVT_CONNECT_START] rtw_phl_send_msg_to_dispr failed\n",
+				__func__);
+
+			goto free_token;
+		}
+		break;
+
+	case MSG_EVT_DISCONNECT_PREPARE:
+
+		rtw_wfd_st_switch(psta, 0);
+		psta->hw_decrypted = _FALSE;
+		#ifdef CONFIG_IOCTL_CFG80211
+		#ifdef COMPAT_KERNEL_RELEASE
+
+		#elif (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)) || defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER)
+		rtw_cfg80211_indicate_sta_disassoc(padapter, pstadel->macaddr, *(u16 *)pstadel->rsvd);
+		#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)) || defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER) */
+		#endif /* CONFIG_IOCTL_CFG80211 */
+
+		rtw_hw_disconnect(padapter, psta);
+
+		rtw_free_mld_stainfo(padapter, psta->phl_sta->mld);
+
+		status = rtw_phl_disconnect(adapter_to_dvobj(padapter)->phl,
+					padapter_link->wrlink->hw_band,
+					padapter->phl_role, _FALSE);
+		if (status == RTW_PHL_STATUS_FAILURE) {
+			RTW_ERR("%s :: [MSG_EVT_DISCONNECT_PREPARE] rtw_phl_send_msg_to_dispr failed\n", __func__);
+			goto free_token;
+		}
+		break;
+#endif
+	case MSG_EVT_CONNECT_END:
+	case MSG_EVT_DISCONNECT:
+
+		_rtw_spinlock_bh(&padapter->ap_add_del_sta_lock);
+
+		rtw_list_delete(&add_del_sta_obj->list);
+		pstapriv->add_sta_list_cnt--;
+		rtw_mfree(add_del_sta_obj, sizeof(struct rtw_add_del_sta_obj));
+		add_del_sta_obj = NULL;
+
+		if (pstapriv->add_sta_list_cnt > 0) {	/* token extend */
+			add_del_sta_obj = rtw_ap_get_stainfo_by_add_sta_list(padapter);
+			if (!add_del_sta_obj || !add_del_sta_obj->sta) {
+				_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+				RTW_ERR(FUNC_ADPT_FMT": add_del_sta_obj or psta is null\n",
+					FUNC_ADPT_ARG(padapter));
+				goto free_token;
+			}
+
+			psta = add_del_sta_obj->sta;
+
+			if (add_del_sta_obj->is_add_sta)
+				status = rtw_phl_connect_prepare(adapter_to_dvobj(padapter)->phl,
+								padapter_link->wrlink->hw_band,
+								padapter->phl_role,
+								padapter_link->wrlink,
+								psta->phl_sta->mac_addr);
+			else
+				status = rtw_phl_disconnect(adapter_to_dvobj(padapter)->phl,
+								padapter_link->wrlink->hw_band,
+								padapter->phl_role,
+								_TRUE);
+
+			if (status != RTW_PHL_STATUS_SUCCESS) {
+				_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+				RTW_ERR("%s :: [%s] rtw_phl_send_msg_to_dispr failed\n",
+					__func__,
+					(MSG_EVT_ID_FIELD(msg->msg_id) == MSG_EVT_CONNECT_END) ?
+					"MSG_EVT_CONNECT_END" : "MSG_EVT_DISCONNECT");
+				goto free_token;
+			}
+
+		} else {
+			_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+			/* no more STA */
+			status = RTW_PHL_STATUS_SUCCESS;
+			goto free_token;
+		}
+		_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+		break;
+
+	default:
+		/* unknown state */
+		RTW_ERR("%s :: [0x%04x] unknown state\n", __func__, MSG_EVT_ID_FIELD(msg->msg_id));
+		break;
+	}
+
+	return MDL_RET_SUCCESS;
+
+free_token:
+	_rtw_spinlock_bh(&padapter->ap_add_del_sta_lock);
+	if (add_del_sta_obj) {
+		rtw_list_delete(&add_del_sta_obj->list);
+		pstapriv->add_sta_list_cnt--;
+		rtw_mfree(add_del_sta_obj, sizeof(struct rtw_add_del_sta_obj));
+		add_del_sta_obj = NULL;
+	}
+
+	_ap_add_del_sta_cmd_done(padapter); /* free token */
+	_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+	return (enum phl_mdl_ret_code)status;
+}
+
+static enum phl_mdl_ret_code _ap_add_del_sta_req_set_info(void *dispr, void *priv, struct phl_module_op_info* info)
+{
+	struct _ADAPTER *padapter = (struct _ADAPTER *)priv;
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(padapter));
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(padapter));
+
+	return MDL_RET_IGNORE;
+}
+
+static enum phl_mdl_ret_code _ap_add_del_sta_req_query_info(void *dispr, void *priv, struct phl_module_op_info* info)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+
+	switch (info->op_code) {
+	case FG_REQ_OP_GET_ROLE:
+		info->outbuf = (u8*)a->phl_role;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	case FG_REQ_OP_GET_ROLE_LINK:
+		info->outbuf = (u8*)alink->wrlink;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	default:
+		break;
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+	return ret;
+}
+
+void rtw_cmd_ap_add_del_sta_req_free(struct _ADAPTER *padapter)
+{
+	_rtw_spinlock_free(&padapter->ap_add_del_sta_lock);
+}
+
+void rtw_cmd_ap_add_del_sta_req_init(struct _ADAPTER *padapter)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct phl_cmd_token_req *req;
+
+	RTW_DBG(FUNC_ADPT_FMT ": \n", FUNC_ADPT_ARG(padapter));
+	/* Fill foreground command request */
+
+	_rtw_spinlock_init(&padapter->ap_add_del_sta_lock);
+
+	req = &padapter->add_del_sta_req;
+	req->module_id= PHL_FG_MDL_AP_ADD_DEL_STA;
+	req->priv = padapter;
+	req->role = padapter->phl_role;
+
+	req->acquired = _ap_add_del_sta_req_acquired;
+	req->abort = _ap_add_del_sta_req_abort;
+	req->msg_hdlr = _ap_add_del_sta_req_ev_hdlr;
+	req->set_info = _ap_add_del_sta_req_set_info;
+	req->query_info = _ap_add_del_sta_req_query_info;
+
+	padapter->ap_add_del_sta_cmd_token = 0;
+	padapter->ap_add_del_sta_cmd_state = ADD_DEL_STA_ST_IDLE;
+}
+
+enum rtw_phl_status rtw_ap_add_del_sta_cmd(struct _ADAPTER *padapter)
+{
+	struct phl_cmd_token_req *fgreq;
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	enum rtw_phl_status status;
+	struct rtw_wifi_role_t *role = padapter->phl_role;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	fgreq = &padapter->add_del_sta_req;
+	/* cmd_dispatcher would copy whole phl_cmd_token_req */
+	status = rtw_phl_add_cmd_token_req(d->phl,
+				padapter_link->wrlink->hw_band,
+				fgreq, &padapter->ap_add_del_sta_cmd_token);
+
+	if ((status != RTW_PHL_STATUS_SUCCESS) &&
+	   (status != RTW_PHL_STATUS_PENDING)) {
+		RTW_ERR(FUNC_ADPT_FMT": add token fail\n", FUNC_ADPT_ARG(padapter));
+		goto exit;
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT ": add token success\n", FUNC_ADPT_ARG(padapter));
+	status = RTW_PHL_STATUS_SUCCESS;
+exit:
+	return status;
+}
+
+bool rtw_add_del_sta_cmd_check(struct _ADAPTER *padapter, unsigned char *MacAddr, bool add_sta)
+{
+	enum rtw_phl_status status = _FAIL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct rtw_add_del_sta_obj *add_del_sta_obj = NULL;
+	struct sta_info *psta;
+
+	if (add_sta)
+		psta = rtw_get_stainfo(&padapter->stapriv, MacAddr);
+	else
+		psta = rtw_get_stainfo_to_free(&padapter->stapriv, MacAddr);
+	if (!psta) {
+		RTW_ERR(FUNC_ADPT_FMT": psta is null\n", FUNC_ADPT_ARG(padapter));
+		return _FAIL;
+	}
+
+	add_del_sta_obj = (struct rtw_add_del_sta_obj *)rtw_zmalloc(sizeof(struct rtw_add_del_sta_obj));
+	if (!add_del_sta_obj) {
+		RTW_ERR(FUNC_ADPT_FMT": add_del_sta_obj is null\n", FUNC_ADPT_ARG(padapter));
+		return _FAIL;
+	}
+
+	_rtw_init_listhead(&add_del_sta_obj->list);
+	add_del_sta_obj->sta = psta;
+	add_del_sta_obj->aid = psta->phl_sta->aid;
+	add_del_sta_obj->is_add_sta = add_sta;
+
+	_rtw_spinlock_bh(&padapter->ap_add_del_sta_lock);
+
+	rtw_list_insert_tail(&add_del_sta_obj->list, &pstapriv->add_sta_list);
+	pstapriv->add_sta_list_cnt++;
+
+	if (padapter->ap_add_del_sta_cmd_state == ADD_DEL_STA_ST_IDLE) {
+		status = rtw_ap_add_del_sta_cmd(padapter);
+		if (status == RTW_PHL_STATUS_SUCCESS)
+			padapter->ap_add_del_sta_cmd_state = ADD_DEL_STA_ST_REQUESTING;
+	} else {
+		RTW_INFO(FUNC_ADPT_FMT": reuse token\n", FUNC_ADPT_ARG(padapter));
+	}
+
+	_rtw_spinunlock_bh(&padapter->ap_add_del_sta_lock);
+	return (status == RTW_PHL_STATUS_SUCCESS) ? _SUCCESS : _FAIL;
+}
+
+void rtw_update_probe_rsp_basic_rate_and_ext(struct xmit_frame *xframe) {
+	struct pkt_attrib *pattrib = &xframe->attrib;
+	struct _ADAPTER *padapter = xframe->padapter;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *lmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *lmlmeinfo = &(lmlmeext->mlmext_info);
+	WLAN_BSSID_EX *cur_network = &(lmlmeinfo->network);
+	u8 *basic_rate_ie = NULL, *ext_rate_ie = NULL;
+	u8 hdr_len = sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *ies = NULL;
+	sint ies_len = 0;
+	sint ie_len = 0;
+	unsigned int rate_len = 0;
+
+	ies = xframe->buf_addr + TXDESC_OFFSET + hdr_len + _BEACON_IE_OFFSET_;
+	ies_len = pattrib->pktlen - hdr_len - _BEACON_IE_OFFSET_;
+
+	basic_rate_ie = rtw_get_ie(ies, WLAN_EID_SUPP_RATES, &ie_len, ies_len);
+	if (basic_rate_ie != NULL) {
+		rate_len = rtw_get_rateset_len(cur_network->SupportedRates);
+		rtw_ies_update_ie(ies, &ies_len, 0, WLAN_EID_SUPP_RATES,
+				  cur_network->SupportedRates,
+				  (rate_len > NDIS_802_11_LENGTH_RATES ?
+				  NDIS_802_11_LENGTH_RATES : rate_len));
+		if (rate_len > NDIS_802_11_LENGTH_RATES)
+			pattrib->pktlen += (NDIS_802_11_LENGTH_RATES - ie_len);
+		else
+			pattrib->pktlen += (rate_len - ie_len);
+	}
+
+	ext_rate_ie = rtw_get_ie(ies, WLAN_EID_EXT_SUPP_RATES,
+				 &ie_len, ies_len);
+	if (ext_rate_ie != NULL) {
+		if (rate_len <= NDIS_802_11_LENGTH_RATES) {
+			rtw_ies_remove_ie(ies, &ies_len, 0,
+					  WLAN_EID_EXT_SUPP_RATES, NULL, 0);
+			pattrib->pktlen -= (ie_len + 2);
+		} else {
+			rtw_ies_update_ie(ies, &ies_len, 0,
+					  WLAN_EID_EXT_SUPP_RATES,
+					  cur_network->SupportedRates +
+					  NDIS_802_11_LENGTH_RATES,
+					  rate_len - NDIS_802_11_LENGTH_RATES);
+			pattrib->pktlen += (rate_len - NDIS_802_11_LENGTH_RATES
+					    - ie_len);
+		}
+	} else if (rate_len > NDIS_802_11_LENGTH_RATES) {
+		rtw_ies_add_ie(ies, &ies_len, 0, WLAN_EID_EXT_SUPP_RATES,
+				  cur_network->SupportedRates +
+				  NDIS_802_11_LENGTH_RATES,
+				  rate_len - NDIS_802_11_LENGTH_RATES);
+		pattrib->pktlen += ((rate_len - NDIS_802_11_LENGTH_RATES) + 2);
+	}
+}
+
+#if CONFIG_AP_REGU_FORBID
+bool rtw_ap_link_regu_forbid_apply(struct _ADAPTER_LINK *alink, bool forbid, bool enqueue)
+{
+	if (forbid) {
+		if (!LINK_MLME_IS_REGU_FORBID(alink)) {
+			SET_LINK_MLME_STATE(alink, WIFI_REGU_FORBID);
+			if (LINK_MLME_IS_ASOC(alink))
+				rtw_sta_flush(alink->adapter, enqueue);
+			rtw_link_hal_core_stop_beacon(alink, forbid, enqueue ? 0 : RTW_CMDF_DIRECTLY);
+			return true;
+		}
+	} else {
+		if (LINK_MLME_IS_REGU_FORBID(alink)) {
+			_CLR_LINK_MLME_STATE_(alink, WIFI_REGU_FORBID);
+			rtw_link_hal_core_stop_beacon(alink, forbid, enqueue ? 0 : RTW_CMDF_DIRECTLY);
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool rtw_ap_link_regu_forbid_update(struct _ADAPTER_LINK *alink, bool enqueue)
+{
+	return rtw_ap_link_regu_forbid_apply(alink
+		, rtw_rfctl_is_regu_forbid_bss(adapter_to_rfctl(alink->adapter), ALINK_GET_BAND(alink))
+		, enqueue);
+}
+#endif
+
+#endif	/* CONFIG_AP_MODE */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_beamforming.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_beamforming.c
new file mode 100644
index 000000000000..7deb6f3fa495
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_beamforming.c
@@ -0,0 +1,2138 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_BEAMFORMING_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_BEAMFORMING
+
+#ifdef RTW_WKARD_TX_DISABLE_BFEE
+void rtw_core_bf_watchdog(_adapter *padapter)
+{
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_phl_stainfo_t *psta = NULL;
+	struct rtw_stats *phl_stats = NULL;
+	bool enable_bfee = false;
+	struct protocol_cap_t *proto_cap;
+	struct _ADAPTER_LINK *padapter_link;
+	u8 lidx;
+
+	do {
+		if (padapter == NULL)
+			break;
+
+		if (1 != pregpriv->dyn_txbf) {
+			break;
+		}
+
+		wrole = padapter->phl_role;
+		//psta  = padapter->stainfo;
+
+		if (wrole == NULL)
+			break;
+
+		if (PHL_RTYPE_STATION != wrole->type || PHL_RTYPE_TDLS != wrole->type)
+			break;
+
+		if (MLME_NO_LINK == wrole->mstate)
+			break;
+
+		for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+			padapter_link = GET_LINK(padapter, lidx);
+
+			if (MLME_NO_LINK == padapter_link->wrlink->mstate)
+				break;
+
+			psta = rtw_phl_get_stainfo_self(adapter_to_dvobj(padapter)->phl, padapter_link->wrlink);
+			proto_cap = &(padapter_link->wrlink->protocol_cap);
+
+			if (psta == NULL)
+				break;
+
+			if ((proto_cap->he_su_bfme ||
+				proto_cap->he_mu_bfme) &&
+				(psta->asoc_cap.he_su_bfmr || psta->asoc_cap.he_mu_bfmr))
+				enable_bfee = true;
+
+			if ((proto_cap->vht_su_bfme ||
+				proto_cap->vht_mu_bfme) &&
+				(psta->asoc_cap.vht_su_bfmr ||
+				psta->asoc_cap.vht_mu_bfmr))
+				enable_bfee = true;
+
+			if (proto_cap->ht_su_bfme &&
+				psta->asoc_cap.ht_su_bfmr)
+				enable_bfee = true;
+
+			if (enable_bfee == false)
+				break;
+
+			phl_stats = &(phl_com->phl_stats);
+			if ((phl_stats->tx_traffic.lvl > phl_stats->rx_traffic.lvl) &&
+				(phl_stats->tx_traffic.lvl >= RTW_TFC_HIGH))
+				enable_bfee = false;
+
+			/*Per wrole->hw_band to control bfee, not per hw_port*/
+			rtw_phl_bfee_ctrl(GET_PHL_INFO(dvobj), wrole, enable_bfee);
+			/*RTW_INFO("[WKARD] Enable/Disable BFee function : BFee %d\n", enable_bfee);*/
+		}
+	} while (0);
+}
+#endif /*RTW_WKARD_TX_DISABLE_BFEE*/
+
+#if 0
+struct ndpa_sta_info {
+	u16 aid:12;
+	u16 feedback_type:1;
+	u16 nc_index:3;
+};
+
+static void _get_txvector_parameter(_adapter *adapter, struct sta_info *sta, u8 *g_id, u16 *p_aid)
+{
+	struct mlme_priv *mlme;
+	u16 aid;
+	u8 *bssid;
+	u16 val16;
+	u8 i;
+
+
+	mlme = &adapter->mlmepriv;
+
+	if (check_fwstate(mlme, WIFI_AP_STATE)) {
+		/*
+		 * Sent by an AP and addressed to a STA associated with that AP
+		 * or sent by a DLS or TDLS STA in a direct path to
+		 * a DLS or TDLS peer STA
+		 */
+
+		aid = sta->phl_sta->aid;
+		bssid = adapter_mac_addr(adapter);
+		RTW_INFO("%s: AID=0x%x BSSID=" MAC_FMT "\n",
+			 __FUNCTION__, sta->phl_sta->aid, MAC_ARG(bssid));
+
+		/* AID[0:8] */
+		aid &= 0x1FF;
+		/* BSSID[44:47] xor BSSID[40:43] */
+		val16 = ((bssid[5] & 0xF0) >> 4) ^ (bssid[5] & 0xF);
+		/* (dec(AID[0:8]) + dec(BSSID)*2^5) mod 2^9 */
+		*p_aid = (aid + (val16 << 5)) & 0x1FF;
+		*g_id = 63;
+	} else if ((check_fwstate(mlme, WIFI_ADHOC_STATE) == _TRUE)
+		   || (check_fwstate(mlme, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+		/*
+		 * Otherwise, includes
+		 * 1. Sent to an IBSS STA
+		 * 2. Sent by an AP to a non associated STA
+		 * 3. Sent to a STA for which it is not known
+		 *    which condition is applicable
+		 */
+		*p_aid = 0;
+		*g_id = 63;
+	} else {
+		/* Addressed to AP */
+		bssid = sta->phl_sta->mac_addr;
+		RTW_INFO("%s: BSSID=" MAC_FMT "\n", __FUNCTION__, MAC_ARG(bssid));
+
+		/* BSSID[39:47] */
+		*p_aid = (bssid[5] << 1) | (bssid[4] >> 7);
+		*g_id = 0;
+	}
+
+	RTW_INFO("%s: GROUP_ID=0x%02x PARTIAL_AID=0x%04x\n",
+		 __FUNCTION__, *g_id, *p_aid);
+}
+
+/*
+ * Parameters
+ *	adapter		_adapter*
+ *	sta		struct sta_info*
+ *	sta_bf_cap	beamforming capabe of sta
+ *	sounding_dim	Number of Sounding Dimensions
+ *	comp_steering	Compressed Steering Number of Beamformer Antennas Supported
+ */
+static void _get_sta_beamform_cap(_adapter *adapter, struct sta_info *sta,
+	u8 *sta_bf_cap, u8 *sounding_dim, u8 *comp_steering)
+{
+	struct beamforming_info *info;
+	struct ht_priv *ht;
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv *vht;
+#endif /* CONFIG_80211AC_VHT */
+#ifdef CONFIG_80211AX_HE
+	struct he_priv *he;
+#endif /* CONFIG_80211AX_HE */
+	u16 bf_cap;
+
+
+	*sta_bf_cap = 0;
+	*sounding_dim = 0;
+	*comp_steering = 0;
+
+	info = GET_BEAMFORM_INFO(adapter);
+	ht = &adapter->mlmepriv.htpriv;
+#ifdef CONFIG_80211AC_VHT
+	vht = &adapter->mlmepriv.vhtpriv;
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+	he = &adapter->mlmepriv.hepriv;
+#endif /* CONFIG_80211AX_HE */
+
+	if (is_supported_ht(sta->phl_sta->wmode) == _TRUE) {
+		/* HT */
+		bf_cap = ht->beamform_cap;
+
+		if (TEST_FLAG(bf_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE)) {
+			info->beamforming_cap |= BEAMFORMEE_CAP_HT_EXPLICIT;
+			*sta_bf_cap |= BEAMFORMER_CAP_HT_EXPLICIT;
+			*sounding_dim = (bf_cap & BEAMFORMING_HT_BEAMFORMEE_CHNL_EST_CAP) >> 6;
+		}
+		if (TEST_FLAG(bf_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE)) {
+			info->beamforming_cap |= BEAMFORMER_CAP_HT_EXPLICIT;
+			*sta_bf_cap |= BEAMFORMEE_CAP_HT_EXPLICIT;
+			*comp_steering = (bf_cap & BEAMFORMING_HT_BEAMFORMER_STEER_NUM) >> 4;
+		}
+	}
+
+#ifdef CONFIG_80211AC_VHT
+	if (is_supported_vht(sta->phl_sta->wmode) == _TRUE) {
+		/* VHT */
+		bf_cap = vht->beamform_cap;
+
+		/* We are SU Beamformee because the STA is SU Beamformer */
+		if (TEST_FLAG(bf_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE)) {
+			info->beamforming_cap |= BEAMFORMEE_CAP_VHT_SU;
+			*sta_bf_cap |= BEAMFORMER_CAP_VHT_SU;
+
+			/* We are MU Beamformee because the STA is MU Beamformer */
+			if (TEST_FLAG(bf_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE)) {
+				info->beamforming_cap |= BEAMFORMEE_CAP_VHT_MU;
+				*sta_bf_cap |= BEAMFORMER_CAP_VHT_MU;
+			}
+
+			*sounding_dim = (bf_cap & BEAMFORMING_VHT_BEAMFORMEE_SOUND_DIM) >> 12;
+		}
+		/* We are SU Beamformer because the STA is SU Beamformee */
+		if (TEST_FLAG(bf_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE)) {
+			info->beamforming_cap |= BEAMFORMER_CAP_VHT_SU;
+			*sta_bf_cap |= BEAMFORMEE_CAP_VHT_SU;
+
+			/* We are MU Beamformer because the STA is MU Beamformee */
+			if (TEST_FLAG(bf_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE)) {
+				info->beamforming_cap |= BEAMFORMER_CAP_VHT_MU;
+				*sta_bf_cap |= BEAMFORMEE_CAP_VHT_MU;
+			}
+
+			*comp_steering = (bf_cap & BEAMFORMING_VHT_BEAMFORMER_STS_CAP) >> 8;
+		}
+	}
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+	/* CONFIG_80211AX_HE_TODO */
+#endif /* CONFIG_80211AX_HE */
+}
+
+static u8 _send_ht_ndpa_packet(_adapter *adapter, u8 *ra, enum channel_width bw)
+{
+	/* General */
+	struct xmit_priv		*pxmitpriv;
+	struct mlme_ext_priv		*pmlmeext;
+	struct mlme_ext_info		*pmlmeinfo;
+	struct xmit_frame		*pmgntframe;
+	/* Beamforming */
+	struct beamforming_info		*info;
+	struct beamformee_entry		*bfee;
+	struct ndpa_sta_info		sta_info;
+	u8 ActionHdr[4] = {ACT_CAT_VENDOR, 0x00, 0xE0, 0x4C};
+	/* MISC */
+	struct pkt_attrib		*attrib;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	enum MGN_RATE txrate;
+	u8 *pframe;
+	u16 duration = 0;
+	u8 aSifsTime = 0;
+
+
+	RTW_INFO("+%s: Send to " MAC_FMT "\n", __FUNCTION__, MAC_ARG(ra));
+
+	pxmitpriv = &adapter->xmitpriv;
+	pmlmeext = &adapter->mlmeextpriv;
+	pmlmeinfo = &pmlmeext->mlmext_info;
+	bfee = rtw_bf_bfee_get_entry_by_addr(adapter, ra);
+	if (!bfee) {
+		RTW_ERR("%s: Cann't find beamformee entry!\n", __FUNCTION__);
+		return _FALSE;
+	}
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (!pmgntframe) {
+		RTW_ERR("%s: alloc mgnt frame fail!\n", __FUNCTION__);
+		return _FALSE;
+	}
+
+	txrate = rtw_hal_get_htndp_tx_rate(adapter, bfee->comp_steering_num_of_bfer);
+
+	/* update attribute */
+	attrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(adapter, attrib);
+	/*attrib->type = WIFI_MGT_TYPE;*/ /* set in update_mgntframe_attrib() */
+	attrib->subtype = WIFI_ACTION_NOACK;
+	attrib->bwmode = bw;
+	/*attrib->qsel = rtw_hal_get_qsel(padapter, QSLT_MGNT_ID);*/ /* set in update_mgntframe_attrib() */
+	attrib->order = 1;
+	attrib->rate = (u8)txrate;
+	attrib->bf_pkt_type = 0;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	/* Frame control */
+	pwlanhdr->frame_ctl = 0;
+	set_frame_sub_type(pframe, attrib->subtype);
+	set_htc_order_bit(pframe);
+
+	/* Duration */
+	if (pmlmeext->cur_wireless_mode == WIRELESS_11B)
+		aSifsTime = 10;
+	else
+		aSifsTime = 16;
+	duration = 2 * aSifsTime + 40;
+	if (bw == CHANNEL_WIDTH_40)
+		duration += 87;
+	else
+		duration += 180;
+	set_duration(pframe, duration);
+
+	/* DA */
+	_rtw_memcpy(pwlanhdr->addr1, ra, ETH_ALEN);
+	/* SA */
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(adapter), ETH_ALEN);
+	/* BSSID */
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+
+	/* HT control field */
+	SET_HT_CTRL_CSI_STEERING(pframe + 24, 3);
+	SET_HT_CTRL_NDP_ANNOUNCEMENT(pframe + 24, 1);
+
+	/*
+	 * Frame Body
+	 * Category field: vender-specific value, 0x7F
+	 * OUI: 0x00E04C
+	 */
+	_rtw_memcpy(pframe + 28, ActionHdr, 4);
+
+	attrib->pktlen = 32;
+	attrib->last_txcmdsz = attrib->pktlen;
+
+	dump_mgntframe(adapter, pmgntframe);
+
+	return _TRUE;
+}
+
+static u8 _send_vht_ndpa_packet(_adapter *adapter, u8 *ra, u16 aid, enum channel_width bw)
+{
+	/* General */
+	struct xmit_priv		*pxmitpriv;
+	struct mlme_ext_priv		*pmlmeext;
+	struct xmit_frame		*pmgntframe;
+	/* Beamforming */
+	struct beamforming_info		*info;
+	struct beamformee_entry		*bfee;
+	struct ndpa_sta_info		sta_info;
+	/* MISC */
+	struct pkt_attrib		*attrib;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	u8 *pframe;
+	enum MGN_RATE txrate;
+	u16 duration = 0;
+	u8 sequence = 0, aSifsTime = 0;
+
+
+	RTW_INFO("+%s: Send to " MAC_FMT "\n", __FUNCTION__, MAC_ARG(ra));
+
+	pxmitpriv = &adapter->xmitpriv;
+	pmlmeext = &adapter->mlmeextpriv;
+	info = GET_BEAMFORM_INFO(adapter);
+	bfee = rtw_bf_bfee_get_entry_by_addr(adapter, ra);
+	if (!bfee) {
+		RTW_ERR("%s: Cann't find beamformee entry!\n", __FUNCTION__);
+		return _FALSE;
+	}
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (!pmgntframe) {
+		RTW_ERR("%s: alloc mgnt frame fail!\n", __FUNCTION__);
+		return _FALSE;
+	}
+
+	txrate = rtw_hal_get_vht_ndp_tx_rate(adapter, bfee->comp_steering_num_of_bfer);
+
+	/* update attribute */
+	attrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(adapter, attrib);
+	/*pattrib->type = WIFI_MGT_TYPE;*/ /* set in update_mgntframe_attrib() */
+	attrib->subtype = WIFI_NDPA;
+	attrib->bwmode = bw;
+	/*attrib->qsel = rtw_hal_get_qsel(padapter, QSLT_MGNT_ID);*/ /* set in update_mgntframe_attrib() */
+	attrib->rate = (u8)txrate;
+	attrib->bf_pkt_type = 0;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, TXDESC_OFFSET + WLANHDR_OFFSET);
+	pframe = pmgntframe->buf_addr + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	/* Frame control */
+	pwlanhdr->frame_ctl = 0;
+	set_frame_sub_type(pframe, attrib->subtype);
+
+	/* Duration */
+	if (WIFI_ROLE_IS_ON_5G(adapter) || is_supported_ht(pmlmeext->cur_wireless_mode))
+		aSifsTime = 16;
+	else
+		aSifsTime = 10;
+	duration = 2 * aSifsTime + 44;
+	if (bw == CHANNEL_WIDTH_80)
+		duration += 40;
+	else if (bw == CHANNEL_WIDTH_40)
+		duration += 87;
+	else
+		duration += 180;
+	set_duration(pframe, duration);
+
+	/* RA */
+	_rtw_memcpy(pwlanhdr->addr1, ra, ETH_ALEN);
+
+	/* TA */
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(adapter), ETH_ALEN);
+
+	/* Sounding Sequence, bit0~1 is reserved */
+	sequence = info->sounding_sequence << 2;
+	if (info->sounding_sequence >= 0x3f)
+		info->sounding_sequence = 0;
+	else
+		info->sounding_sequence++;
+	_rtw_memcpy(pframe + 16, &sequence, 1);
+
+	/* STA Info */
+	/*
+	 * "AID12" Equal to 0 if the STA is an AP, mesh STA or
+	 * STA that is a member of an IBSS
+	 */
+	if (check_fwstate(&adapter->mlmepriv, WIFI_AP_STATE) == _FALSE)
+		aid = 0;
+	sta_info.aid = aid;
+	/* "Feedback Type" set to 0 for SU */
+	sta_info.feedback_type = 0;
+	/* "Nc Index" reserved if the Feedback Type field indicates SU */
+	sta_info.nc_index = 0;
+	_rtw_memcpy(pframe + 17, (u8 *)&sta_info, 2);
+
+	attrib->pktlen = 19;
+	attrib->last_txcmdsz = attrib->pktlen;
+
+	dump_mgntframe(adapter, pmgntframe);
+
+	return _TRUE;
+}
+
+static u8 _send_vht_mu_ndpa_packet(_adapter *adapter, enum channel_width bw)
+{
+	/* General */
+	struct xmit_priv		*pxmitpriv;
+	struct mlme_ext_priv		*pmlmeext;
+	struct xmit_frame		*pmgntframe;
+	/* Beamforming */
+	struct beamforming_info		*info;
+	struct sounding_info		*sounding;
+	struct beamformee_entry		*bfee;
+	struct ndpa_sta_info		sta_info;
+	/* MISC */
+	struct pkt_attrib		*attrib;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	enum MGN_RATE txrate;
+	u8 *pframe;
+	u8 *ra = NULL;
+	u16 duration = 0;
+	u8 sequence = 0, aSifsTime = 0;
+	u8 i;
+
+
+	RTW_INFO("+%s\n", __FUNCTION__);
+
+	pxmitpriv = &adapter->xmitpriv;
+	pmlmeext = &adapter->mlmeextpriv;
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	txrate = MGN_VHT2SS_MCS0;
+
+	/*
+	 * Fill the first MU BFee entry (STA1) MAC addr to destination address then
+	 * HW will change A1 to broadcast addr.
+	 * 2015.05.28. Suggested by SD1 Chunchu.
+	 */
+	bfee = &info->bfee_entry[sounding->mu_sounding_list[0]];
+	ra = bfee->mac_addr;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (!pmgntframe) {
+		RTW_ERR("%s: alloc mgnt frame fail!\n", __FUNCTION__);
+		return _FALSE;
+	}
+
+	/* update attribute */
+	attrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(adapter, attrib);
+	/*attrib->type = WIFI_MGT_TYPE;*/ /* set in update_mgntframe_attrib() */
+	attrib->subtype = WIFI_NDPA;
+	attrib->bwmode = bw;
+	/*attrib->qsel = rtw_hal_get_qsel(padapter, QSLT_MGNT_ID);*/ /* set in update_mgntframe_attrib() */
+	attrib->rate = (u8)txrate;
+	/* Set TxBFPktType of Tx desc to unicast type if there is only one MU STA for HW design */
+	if (info->sounding_info.candidate_mu_bfee_cnt > 1)
+		attrib->bf_pkt_type = 1;
+	else
+		attrib->bf_pkt_type = 0;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, TXDESC_OFFSET + WLANHDR_OFFSET);
+	pframe = pmgntframe->buf_addr + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	/* Frame control */
+	pwlanhdr->frame_ctl = 0;
+	set_frame_sub_type(pframe, attrib->subtype);
+
+	/* Duration */
+	if (WIFI_ROLE_IS_ON_5G(adapter) || is_supported_ht(pmlmeext->cur_wireless_mode))
+		aSifsTime = 16;
+	else
+		aSifsTime = 10;
+	duration = 2 * aSifsTime + 44;
+	if (bw == CHANNEL_WIDTH_80)
+		duration += 40;
+	else if (bw == CHANNEL_WIDTH_40)
+		duration += 87;
+	else
+		duration += 180;
+	set_duration(pframe, duration);
+
+	/* RA */
+	_rtw_memcpy(pwlanhdr->addr1, ra, ETH_ALEN);
+
+	/* TA */
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(adapter), ETH_ALEN);
+
+	/* Sounding Sequence, bit0~1 is reserved */
+	sequence = info->sounding_sequence << 2;
+	if (info->sounding_sequence >= 0x3f)
+		info->sounding_sequence = 0;
+	else
+		info->sounding_sequence++;
+	_rtw_memcpy(pframe + 16, &sequence, 1);
+
+	attrib->pktlen = 17;
+
+	/*
+	 * Construct STA info. for multiple STAs
+	 * STA Info1, ..., STA Info n
+	 */
+	for (i = 0; i < sounding->candidate_mu_bfee_cnt; i++) {
+		bfee = &info->bfee_entry[sounding->mu_sounding_list[i]];
+		sta_info.aid = bfee->aid;
+		sta_info.feedback_type = 1; /* 1'b1: MU */
+		sta_info.nc_index = 0;
+		_rtw_memcpy(pframe + attrib->pktlen, (u8 *)&sta_info, 2);
+		attrib->pktlen += 2;
+	}
+
+	attrib->last_txcmdsz = attrib->pktlen;
+
+	dump_mgntframe(adapter, pmgntframe);
+
+	return _TRUE;
+}
+
+static u8 _send_bf_report_poll(_adapter *adapter, u8 *ra, u8 bFinalPoll)
+{
+	/* General */
+	struct xmit_priv *pxmitpriv;
+	struct xmit_frame *pmgntframe;
+	/* MISC */
+	struct pkt_attrib *attrib;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	u8 *pframe;
+
+
+	RTW_INFO("+%s: Send to " MAC_FMT "\n", __FUNCTION__, MAC_ARG(ra));
+
+	pxmitpriv = &adapter->xmitpriv;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (!pmgntframe) {
+		RTW_ERR("%s: alloc mgnt frame fail!\n", __FUNCTION__);
+		return _FALSE;
+	}
+
+	/* update attribute */
+	attrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(adapter, attrib);
+	/*attrib->type = WIFI_MGT_TYPE;*/ /* set in update_mgntframe_attrib() */
+	attrib->subtype = WIFI_BF_REPORT_POLL;
+	attrib->bwmode = CHANNEL_WIDTH_20;
+	/*attrib->qsel = rtw_hal_get_qsel(padapter, QSLT_MGNT_ID);*/ /* set in update_mgntframe_attrib() */
+	attrib->rate = MGN_6M;
+	if (bFinalPoll)
+		attrib->bf_pkt_type = 3;
+	else
+		attrib->bf_pkt_type = 2;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, TXDESC_OFFSET + WLANHDR_OFFSET);
+	pframe = pmgntframe->buf_addr + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	/* Frame control */
+	pwlanhdr->frame_ctl = 0;
+	set_frame_sub_type(pframe, attrib->subtype);
+
+	/* Duration */
+	set_duration(pframe, 100);
+
+	/* RA */
+	_rtw_memcpy(pwlanhdr->addr1, ra, ETH_ALEN);
+
+	/* TA */
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(adapter), ETH_ALEN);
+
+	/* Feedback Segment Retransmission Bitmap */
+	pframe[16] = 0xFF;
+
+	attrib->pktlen = 17;
+	attrib->last_txcmdsz = attrib->pktlen;
+
+	dump_mgntframe(adapter, pmgntframe);
+
+	return _TRUE;
+}
+
+static void _sounding_update_min_period(_adapter *adapter, u16 period, u8 leave)
+{
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee;
+	u8 i = 0;
+	u16 min_val = 0xFFFF;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	if (_TRUE == leave) {
+		/*
+		 * When a BFee left,
+		 * we need to find the latest min sounding period
+		 * from the remaining BFees
+		 */
+		for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+			bfee = &info->bfee_entry[i];
+			if ((bfee->used == _TRUE)
+			    && (bfee->sound_period < min_val))
+				min_val = bfee->sound_period;
+		}
+
+		if (min_val == 0xFFFF)
+			info->sounding_info.min_sounding_period = 0;
+		else
+			info->sounding_info.min_sounding_period = min_val;
+	} else {
+		if ((info->sounding_info.min_sounding_period == 0)
+		    || (period < info->sounding_info.min_sounding_period))
+			info->sounding_info.min_sounding_period = period;
+	}
+}
+
+static void _sounding_init(struct sounding_info *sounding)
+{
+	_rtw_memset(sounding->su_sounding_list, 0xFF, MAX_NUM_BEAMFORMEE_SU);
+	_rtw_memset(sounding->mu_sounding_list, 0xFF, MAX_NUM_BEAMFORMEE_MU);
+	sounding->state = SOUNDING_STATE_NONE;
+	sounding->su_bfee_curidx = 0xFF;
+	sounding->candidate_mu_bfee_cnt = 0;
+	sounding->min_sounding_period = 0;
+	sounding->sound_remain_cnt_per_period = 0;
+}
+
+static void _sounding_reset_vars(_adapter *adapter)
+{
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+	u8 idx;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	_rtw_memset(sounding->su_sounding_list, 0xFF, MAX_NUM_BEAMFORMEE_SU);
+	_rtw_memset(sounding->mu_sounding_list, 0xFF, MAX_NUM_BEAMFORMEE_MU);
+	sounding->su_bfee_curidx = 0xFF;
+	sounding->candidate_mu_bfee_cnt = 0;
+
+	/* Clear bSound flag for the new period */
+	for (idx = 0; idx < MAX_BEAMFORMEE_ENTRY_NUM; idx++) {
+		if ((info->bfee_entry[idx].used == _TRUE)
+		    && (info->bfee_entry[idx].sounding == _TRUE)) {
+			info->bfee_entry[idx].sounding = _FALSE;
+			info->bfee_entry[idx].bCandidateSoundingPeer = _FALSE;
+		}
+	}
+}
+
+/*
+ * Return
+ *	0	Prepare sounding list OK
+ *	-1	Fail to prepare sounding list, because no beamformee need to souding
+ *	-2	Fail to prepare sounding list, because beamformee state not ready
+ *
+ */
+static int _sounding_get_list(_adapter *adapter)
+{
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+	struct beamformee_entry *bfee;
+	u8 i, mu_idx = 0, su_idx = 0, not_ready = 0;
+	int ret = 0;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	/* Add MU BFee list first because MU priority is higher than SU */
+	for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (bfee->used == _FALSE)
+			continue;
+
+		if (bfee->state != BEAMFORM_ENTRY_HW_STATE_ADDED) {
+			RTW_ERR("%s: Invalid BFee idx(%d) Hw state=%d\n", __FUNCTION__, i, bfee->state);
+			not_ready++;
+			continue;
+		}
+
+		/*
+		 * Decrease BFee's SoundCnt per period
+		 * If the remain count is 0,
+		 * then it can be sounded at this time
+		 */
+		if (bfee->SoundCnt) {
+			bfee->SoundCnt--;
+			if (bfee->SoundCnt)
+				continue;
+		}
+
+		/*
+		 * <tynli_Note>
+		 *	If the STA supports MU BFee capability then we add it to MUSoundingList directly
+		 *	because we can only sound one STA by unicast NDPA with MU cap enabled to get correct channel info.
+		 *	Suggested by BB team Luke Lee. 2015.11.25.
+		 */
+		if (bfee->cap & BEAMFORMEE_CAP_VHT_MU) {
+			/* MU BFee */
+			if (mu_idx >= MAX_NUM_BEAMFORMEE_MU) {
+				RTW_ERR("%s: Too much MU bfee entry(Limit:%d)\n", __FUNCTION__, MAX_NUM_BEAMFORMEE_MU);
+				continue;
+			}
+
+			if (bfee->bApplySounding == _TRUE) {
+				bfee->bCandidateSoundingPeer = _TRUE;
+				bfee->SoundCnt = GetInitSoundCnt(bfee->sound_period, sounding->min_sounding_period);
+				sounding->mu_sounding_list[mu_idx] = i;
+				mu_idx++;
+			}
+		} else if (bfee->cap & (BEAMFORMEE_CAP_VHT_SU|BEAMFORMEE_CAP_HT_EXPLICIT)) {
+			/* SU BFee (HT/VHT) */
+			if (su_idx >= MAX_NUM_BEAMFORMEE_SU) {
+				RTW_ERR("%s: Too much SU bfee entry(Limit:%d)\n", __FUNCTION__, MAX_NUM_BEAMFORMEE_SU);
+				continue;
+			}
+
+			if (bfee->bDeleteSounding == _TRUE) {
+				sounding->su_sounding_list[su_idx] = i;
+				su_idx++;
+			} else if ((bfee->bApplySounding == _TRUE)
+			    && (bfee->bSuspendSUCap == _FALSE)) {
+				bfee->bCandidateSoundingPeer = _TRUE;
+				bfee->SoundCnt = GetInitSoundCnt(bfee->sound_period, sounding->min_sounding_period);
+				sounding->su_sounding_list[su_idx] = i;
+				su_idx++;
+			}
+		}
+	}
+
+	sounding->candidate_mu_bfee_cnt = mu_idx;
+
+	if (su_idx + mu_idx == 0) {
+		ret = -1;
+		if (not_ready)
+			ret = -2;
+	}
+
+	RTW_INFO("-%s: There are %d SU and %d MU BFees in this sounding period\n", __FUNCTION__, su_idx, mu_idx);
+
+	return ret;
+}
+
+static void _sounding_handler(_adapter *adapter)
+{
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+	struct beamformee_entry *bfee;
+	u8 su_idx, i;
+	u32 timeout_period = 0;
+	u8 set_timer = _FALSE;
+	int ret = 0;
+	static u16 wait_cnt = 0;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	RTW_DBG("+%s: state=%d\n", __FUNCTION__, sounding->state);
+	if ((sounding->state != SOUNDING_STATE_INIT)
+	    && (sounding->state != SOUNDING_STATE_SU_SOUNDDOWN)
+	    && (sounding->state != SOUNDING_STATE_MU_SOUNDDOWN)
+	    && (sounding->state != SOUNDING_STATE_SOUNDING_TIMEOUT)) {
+		RTW_WARN("%s: Invalid State(%d) and return!\n", __FUNCTION__, sounding->state);
+		return;
+	}
+
+	if (sounding->state == SOUNDING_STATE_INIT) {
+		RTW_INFO("%s: Sounding start\n", __FUNCTION__);
+
+		/* Init Var */
+		_sounding_reset_vars(adapter);
+
+		/* Get the sounding list of this sounding period */
+		ret = _sounding_get_list(adapter);
+		if (ret == -1) {
+			wait_cnt = 0;
+			sounding->state = SOUNDING_STATE_NONE;
+			RTW_ERR("%s: No BFees found, set to SOUNDING_STATE_NONE\n", __FUNCTION__);
+			info->sounding_running--;
+			return;
+		}
+		if (ret == -2) {
+			RTW_WARN("%s: Temporarily cann't find BFee to sounding\n", __FUNCTION__);
+			if (wait_cnt < 5) {
+				wait_cnt++;
+			} else {
+				wait_cnt = 0;
+				sounding->state = SOUNDING_STATE_NONE;
+				RTW_ERR("%s: Wait changing state timeout!! Set to SOUNDING_STATE_NONE\n", __FUNCTION__);
+			}
+			info->sounding_running--;
+			return;
+		}
+		if (ret != 0) {
+			wait_cnt = 0;
+			RTW_ERR("%s: Unkown state(%d)!\n", __FUNCTION__, ret);
+			info->sounding_running--;
+			return;
+
+		}
+
+		wait_cnt = 0;
+
+		if (check_fwstate(&adapter->mlmepriv, WIFI_UNDER_SURVEY) == _TRUE) {
+			RTW_INFO("%s: Sounding abort! scanning APs...\n", __FUNCTION__);
+			info->sounding_running--;
+			return;
+		}
+	}
+
+	/* Get non-sound SU BFee index */
+	for (i = 0; i < MAX_NUM_BEAMFORMEE_SU; i++) {
+		su_idx = sounding->su_sounding_list[i];
+		if (su_idx >= MAX_BEAMFORMEE_ENTRY_NUM)
+			continue;
+		bfee = &info->bfee_entry[su_idx];
+		if (_FALSE == bfee->sounding)
+			break;
+	}
+	if (i < MAX_NUM_BEAMFORMEE_SU) {
+		sounding->su_bfee_curidx = su_idx;
+		/* Set to sounding start state */
+		sounding->state = SOUNDING_STATE_SU_START;
+		RTW_DBG("%s: Set to SOUNDING_STATE_SU_START\n", __FUNCTION__);
+
+		bfee->sounding = _TRUE;
+		/* Reset sounding timeout flag for the new sounding */
+		bfee->bSoundingTimeout = _FALSE;
+
+		if (_TRUE == bfee->bDeleteSounding) {
+			u8 res = _FALSE;
+			rtw_bf_cmd(adapter, BEAMFORMING_CTRL_END_PERIOD, &res, 1, 0);
+			return;
+		}
+
+		/* Start SU sounding */
+		if (bfee->cap & BEAMFORMEE_CAP_VHT_SU)
+			_send_vht_ndpa_packet(adapter, bfee->mac_addr, bfee->aid, bfee->sound_bw);
+		else if (bfee->cap & BEAMFORMEE_CAP_HT_EXPLICIT)
+			_send_ht_ndpa_packet(adapter, bfee->mac_addr, bfee->sound_bw);
+
+		/* Set sounding timeout timer */
+		_set_timer(&info->sounding_timeout_timer, SU_SOUNDING_TIMEOUT);
+		return;
+	}
+
+	if (sounding->candidate_mu_bfee_cnt > 0) {
+		/*
+		 * If there is no SU BFee then find MU BFee and perform MU sounding
+		 *
+		 * <tynli_note> Need to check the MU starting condition. 2015.12.15.
+		 */
+		sounding->state = SOUNDING_STATE_MU_START;
+		RTW_DBG("%s: Set to SOUNDING_STATE_MU_START\n", __FUNCTION__);
+
+		/* Update MU BFee info */
+		for (i = 0; i < sounding->candidate_mu_bfee_cnt; i++) {
+			bfee = &info->bfee_entry[sounding->mu_sounding_list[i]];
+			bfee->sounding = _TRUE;
+		}
+
+		/* Send MU NDPA */
+		bfee = &info->bfee_entry[sounding->mu_sounding_list[0]];
+		_send_vht_mu_ndpa_packet(adapter, bfee->sound_bw);
+
+		/* Send BF report poll if more than 1 MU STA */
+		for (i = 1; i < sounding->candidate_mu_bfee_cnt; i++) {
+			bfee = &info->bfee_entry[sounding->mu_sounding_list[i]];
+
+			if (i == (sounding->candidate_mu_bfee_cnt - 1))/* The last STA*/
+				_send_bf_report_poll(adapter, bfee->mac_addr, _TRUE);
+			else
+				_send_bf_report_poll(adapter, bfee->mac_addr, _FALSE);
+		}
+
+		sounding->candidate_mu_bfee_cnt = 0;
+
+		/* Set sounding timeout timer */
+		_set_timer(&info->sounding_timeout_timer, MU_SOUNDING_TIMEOUT);
+		return;
+	}
+
+	info->sounding_running--;
+	sounding->state = SOUNDING_STATE_INIT;
+	RTW_INFO("%s: Sounding finished!\n", __FUNCTION__);
+}
+
+static void _sounding_force_stop(_adapter *adapter)
+{
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	if ((sounding->state == SOUNDING_STATE_SU_START)
+	    || (sounding->state == SOUNDING_STATE_MU_START)) {
+		u8 res = _FALSE;
+		_cancel_timer_ex(&info->sounding_timeout_timer);
+		rtw_bf_cmd(adapter, BEAMFORMING_CTRL_END_PERIOD, &res, 1, 1);
+		return;
+	}
+
+	info->sounding_running--;
+	sounding->state = SOUNDING_STATE_INIT;
+	RTW_INFO("%s: Sounding finished!\n", __FUNCTION__);
+}
+
+static void _sounding_timer_handler(void *FunctionContext)
+{
+	_adapter *adapter;
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+	static u8 delay = 0;
+
+
+	RTW_DBG("+%s\n", __FUNCTION__);
+
+	adapter = (_adapter *)FunctionContext;
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	if (SOUNDING_STATE_NONE == sounding->state) {
+		RTW_INFO("%s: Stop!\n", __FUNCTION__);
+		if (info->sounding_running)
+			RTW_WARN("%s: souding_running=%d when thread stop!\n",
+				 __FUNCTION__, info->sounding_running);
+		return;
+	}
+
+	_set_timer(&info->sounding_timer, sounding->min_sounding_period);
+
+	if (!info->sounding_running) {
+		if (SOUNDING_STATE_INIT != sounding->state) {
+			RTW_WARN("%s: state(%d) != SOUNDING_STATE_INIT!!\n", __FUNCTION__, sounding->state);
+			sounding->state = SOUNDING_STATE_INIT;
+		}
+		delay = 0;
+		info->sounding_running++;
+		rtw_bf_cmd(adapter, BEAMFORMING_CTRL_START_PERIOD, NULL, 0, 1);
+	} else {
+		if (delay != 0xFF)
+			delay++;
+		RTW_WARN("%s: souding is still processing...(state:%d, running:%d, delay:%d)\n",
+			 __FUNCTION__, sounding->state, info->sounding_running, delay);
+		if (delay > 3) {
+			RTW_WARN("%s: Stop sounding!!\n", __FUNCTION__);
+			_sounding_force_stop(adapter);
+		}
+	}
+}
+
+static void _sounding_timeout_timer_handler(void *FunctionContext)
+{
+	_adapter *adapter;
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+	struct beamformee_entry *bfee;
+
+
+	RTW_WARN("+%s\n", __FUNCTION__);
+
+	adapter = (_adapter *)FunctionContext;
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	if (SOUNDING_STATE_SU_START == sounding->state) {
+		sounding->state = SOUNDING_STATE_SOUNDING_TIMEOUT;
+		RTW_ERR("%s: Set to SU SOUNDING_STATE_SOUNDING_TIMEOUT\n", __FUNCTION__);
+		/* SU BFee */
+		bfee = &info->bfee_entry[sounding->su_bfee_curidx];
+		bfee->bSoundingTimeout = _TRUE;
+		RTW_WARN("%s: The BFee entry[%d] is Sounding Timeout!\n", __FUNCTION__, sounding->su_bfee_curidx);
+	} else if (SOUNDING_STATE_MU_START == sounding->state) {
+		sounding->state = SOUNDING_STATE_SOUNDING_TIMEOUT;
+		RTW_ERR("%s: Set to MU SOUNDING_STATE_SOUNDING_TIMEOUT\n", __FUNCTION__);
+	} else {
+		RTW_WARN("%s: unexpected sounding state:0x%02x\n", __FUNCTION__, sounding->state);
+		return;
+	}
+
+	rtw_bf_cmd(adapter, BEAMFORMING_CTRL_START_PERIOD, NULL, 0, 1);
+}
+
+static struct beamformer_entry *_bfer_get_free_entry(_adapter *adapter)
+{
+	u8 i = 0;
+	struct beamforming_info *info;
+	struct beamformer_entry *bfer;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	for (i = 0; i < MAX_BEAMFORMER_ENTRY_NUM; i++) {
+		bfer = &info->bfer_entry[i];
+		if (bfer->used == _FALSE)
+			return bfer;
+	}
+
+	return NULL;
+}
+
+static struct beamformer_entry *_bfer_get_entry_by_addr(_adapter *adapter, u8 *ra)
+{
+	u8 i = 0;
+	struct beamforming_info *info;
+	struct beamformer_entry *bfer;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	for (i = 0; i < MAX_BEAMFORMER_ENTRY_NUM; i++) {
+		bfer = &info->bfer_entry[i];
+		if (bfer->used == _FALSE)
+			continue;
+		if (_rtw_memcmp(ra, bfer->mac_addr, ETH_ALEN) == _TRUE)
+			return bfer;
+	}
+
+	return NULL;
+}
+
+static struct beamformer_entry *_bfer_add_entry(_adapter *adapter,
+	struct sta_info *sta, u8 bf_cap, u8 sounding_dim, u8 comp_steering)
+{
+	struct mlme_priv *mlme;
+	struct beamforming_info *info;
+	struct beamformer_entry *bfer;
+	u8 *bssid;
+	u16 val16;
+	u8 i;
+
+
+	mlme = &adapter->mlmepriv;
+	info = GET_BEAMFORM_INFO(adapter);
+
+	bfer = _bfer_get_entry_by_addr(adapter, sta->phl_sta->mac_addr);
+	if (!bfer) {
+		bfer = _bfer_get_free_entry(adapter);
+		if (!bfer)
+			return NULL;
+	}
+
+	bfer->used = _TRUE;
+	_get_txvector_parameter(adapter, sta, &bfer->g_id, &bfer->p_aid);
+	_rtw_memcpy(bfer->mac_addr, sta->phl_sta->mac_addr, ETH_ALEN);
+	bfer->cap = bf_cap;
+	bfer->state = BEAMFORM_ENTRY_HW_STATE_ADD_INIT;
+	bfer->NumofSoundingDim = sounding_dim;
+
+	if (TEST_FLAG(bf_cap, BEAMFORMER_CAP_VHT_MU)) {
+		info->beamformer_mu_cnt += 1;
+		bfer->aid = sta->phl_sta->aid;
+	} else if (TEST_FLAG(bf_cap, BEAMFORMER_CAP_VHT_SU|BEAMFORMER_CAP_HT_EXPLICIT)) {
+		info->beamformer_su_cnt += 1;
+
+		/* Record HW idx info */
+		for (i = 0; i < MAX_NUM_BEAMFORMER_SU; i++) {
+			if ((info->beamformer_su_reg_maping & BIT(i)) == 0) {
+				info->beamformer_su_reg_maping |= BIT(i);
+				bfer->su_reg_index = i;
+				break;
+			}
+		}
+		RTW_INFO("%s: Add BFer entry beamformer_su_reg_maping=%#x, su_reg_index=%d\n",
+			 __FUNCTION__, info->beamformer_su_reg_maping, bfer->su_reg_index);
+	}
+
+	return bfer;
+}
+
+static void _bfer_remove_entry(_adapter *adapter, struct beamformer_entry *entry)
+{
+	struct beamforming_info *info;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	entry->state = BEAMFORM_ENTRY_HW_STATE_DELETE_INIT;
+
+	if (TEST_FLAG(entry->cap, BEAMFORMER_CAP_VHT_MU)) {
+		info->beamformer_mu_cnt -= 1;
+		_rtw_memset(entry->gid_valid, 0, 8);
+		_rtw_memset(entry->user_position, 0, 16);
+	} else if (TEST_FLAG(entry->cap, BEAMFORMER_CAP_VHT_SU|BEAMFORMER_CAP_HT_EXPLICIT)) {
+		info->beamformer_su_cnt -= 1;
+	}
+
+	if (info->beamformer_mu_cnt == 0)
+		info->beamforming_cap &= ~BEAMFORMEE_CAP_VHT_MU;
+	if (info->beamformer_su_cnt == 0)
+		info->beamforming_cap &= ~(BEAMFORMEE_CAP_VHT_SU|BEAMFORMEE_CAP_HT_EXPLICIT);
+}
+
+static u8 _bfer_set_entry_gid(_adapter *adapter, u8 *addr, u8 *gid, u8 *position)
+{
+	struct beamformer_entry bfer;
+
+	_rtw_memset(&bfer, 0, sizeof(bfer));
+	_rtw_memcpy(bfer.mac_addr, addr, ETH_ALEN);
+
+	/* Parsing Membership Status Array */
+	_rtw_memcpy(bfer.gid_valid, gid, 8);
+
+	/* Parsing User Position Array */
+	_rtw_memcpy(bfer.user_position, position, 16);
+
+	/* Config HW GID table */
+	rtw_bf_cmd(adapter, BEAMFORMING_CTRL_SET_GID_TABLE, (u8 *) &bfer,
+			sizeof(bfer), 1);
+
+	return _SUCCESS;
+}
+
+static struct beamformee_entry *_bfee_get_free_entry(_adapter *adapter)
+{
+	u8 i = 0;
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (bfee->used == _FALSE)
+			return bfee;
+	}
+
+	return NULL;
+}
+
+static struct beamformee_entry *_bfee_get_entry_by_addr(_adapter *adapter, u8 *ra)
+{
+	u8 i = 0;
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (bfee->used == _FALSE)
+			continue;
+		if (_rtw_memcmp(ra, bfee->mac_addr, ETH_ALEN) == _TRUE)
+			return bfee;
+	}
+
+	return NULL;
+}
+
+static u8 _bfee_get_first_su_entry_idx(_adapter *adapter, struct beamformee_entry *ignore)
+{
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee;
+	u8 i;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (ignore && (bfee == ignore))
+			continue;
+		if (bfee->used == _FALSE)
+			continue;
+		if ((!TEST_FLAG(bfee->cap, BEAMFORMEE_CAP_VHT_MU))
+		    && TEST_FLAG(bfee->cap, BEAMFORMEE_CAP_VHT_SU|BEAMFORMEE_CAP_HT_EXPLICIT))
+			return i;
+	}
+
+	return 0xFF;
+}
+
+/*
+ * Description:
+ *	Get the first entry index of MU Beamformee.
+ *
+ * Return Value:
+ *	Index of the first MU sta, or 0xFF for invalid index.
+ *
+ * 2015.05.25. Created by tynli.
+ *
+ */
+static u8 _bfee_get_first_mu_entry_idx(_adapter *adapter, struct beamformee_entry *ignore)
+{
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee;
+	u8 i;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (ignore && (bfee == ignore))
+			continue;
+		if (bfee->used == _FALSE)
+			continue;
+		if (TEST_FLAG(bfee->cap, BEAMFORMEE_CAP_VHT_MU))
+			return i;
+	}
+
+	return 0xFF;
+}
+
+static struct beamformee_entry *_bfee_add_entry(_adapter *adapter,
+	struct sta_info *sta, u8 bf_cap, u8 sounding_dim, u8 comp_steering)
+{
+	struct mlme_priv *mlme;
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee;
+	u8 *bssid;
+	u16 val16;
+	u8 i;
+
+
+	mlme = &adapter->mlmepriv;
+	info = GET_BEAMFORM_INFO(adapter);
+
+	bfee = _bfee_get_entry_by_addr(adapter, sta->phl_sta->mac_addr);
+	if (!bfee) {
+		bfee = _bfee_get_free_entry(adapter);
+		if (!bfee)
+			return NULL;
+	}
+
+	bfee->used = _TRUE;
+	bfee->aid = sta->phl_sta->aid;
+	bfee->mac_id = sta->phl_sta->macid;
+	bfee->sound_bw = sta->phl_sta->chandef.bw;
+
+	_get_txvector_parameter(adapter, sta, &bfee->g_id, &bfee->p_aid);
+	sta->phl_sta->bf_info.g_id = bfee->g_id;
+	sta->phl_sta->bf_info.p_aid = bfee->p_aid;
+
+	_rtw_memcpy(bfee->mac_addr, sta->phl_sta->mac_addr, ETH_ALEN);
+	bfee->txbf = _FALSE;
+	bfee->sounding = _FALSE;
+	bfee->sound_period = 40;
+	_sounding_update_min_period(adapter, bfee->sound_period, _FALSE);
+	bfee->SoundCnt = GetInitSoundCnt(bfee->sound_period, info->sounding_info.min_sounding_period);
+	bfee->cap = bf_cap;
+	bfee->state = BEAMFORM_ENTRY_HW_STATE_ADD_INIT;
+
+	bfee->bCandidateSoundingPeer = _FALSE;
+	bfee->bSoundingTimeout = _FALSE;
+	bfee->bDeleteSounding = _FALSE;
+	bfee->bApplySounding = _TRUE;
+
+	bfee->tx_timestamp = 0;
+	bfee->tx_bytes = 0;
+
+	bfee->LogStatusFailCnt = 0;
+	bfee->NumofSoundingDim = sounding_dim;
+	bfee->comp_steering_num_of_bfer = comp_steering;
+	bfee->bSuspendSUCap = _FALSE;
+
+	if (TEST_FLAG(bf_cap, BEAMFORMEE_CAP_VHT_MU)) {
+		info->beamformee_mu_cnt += 1;
+		info->first_mu_bfee_index = _bfee_get_first_mu_entry_idx(adapter, NULL);
+
+		if (_TRUE == info->bEnableSUTxBFWorkAround) {
+			/* When the first MU BFee added, discard SU BFee bfee's capability */
+			if ((info->beamformee_mu_cnt == 1) && (info->beamformee_su_cnt > 0)) {
+				if (info->TargetSUBFee) {
+					info->TargetSUBFee->bSuspendSUCap = _TRUE;
+					info->TargetSUBFee->bDeleteSounding = _TRUE;
+				} else {
+					RTW_ERR("%s: UNEXPECTED!! info->TargetSUBFee is NULL!", __FUNCTION__);
+				}
+				info->TargetSUBFee = NULL;
+				_rtw_memset(&info->TargetCSIInfo, 0, sizeof(struct _RT_CSI_INFO));
+				rtw_bf_cmd(adapter, BEAMFORMING_CTRL_SET_CSI_REPORT, (u8*)&info->TargetCSIInfo, sizeof(struct _RT_CSI_INFO), 0);
+			}
+		}
+
+		/* Record HW idx info */
+		for (i = 0; i < MAX_NUM_BEAMFORMEE_MU; i++) {
+			if ((info->beamformee_mu_reg_maping & BIT(i)) == 0) {
+				info->beamformee_mu_reg_maping |= BIT(i);
+				bfee->mu_reg_index = i;
+				break;
+			}
+		}
+		RTW_INFO("%s: Add BFee entry beamformee_mu_reg_maping=%#x, mu_reg_index=%d\n",
+			 __FUNCTION__, info->beamformee_mu_reg_maping, bfee->mu_reg_index);
+
+	} else if (TEST_FLAG(bf_cap, BEAMFORMEE_CAP_VHT_SU|BEAMFORMEE_CAP_HT_EXPLICIT)) {
+		info->beamformee_su_cnt += 1;
+
+		if (_TRUE == info->bEnableSUTxBFWorkAround) {
+			/* Record the first SU BFee index. We only allow the first SU BFee to be sound */
+			if ((info->beamformee_su_cnt == 1) && (info->beamformee_mu_cnt == 0)) {
+				info->TargetSUBFee = bfee;
+				_rtw_memset(&info->TargetCSIInfo, 0, sizeof(struct _RT_CSI_INFO));
+				bfee->bSuspendSUCap = _FALSE;
+			} else {
+				bfee->bSuspendSUCap = _TRUE;
+			}
+		}
+
+		/* Record HW idx info */
+		for (i = 0; i < MAX_NUM_BEAMFORMEE_SU; i++) {
+			if ((info->beamformee_su_reg_maping & BIT(i)) == 0) {
+				info->beamformee_su_reg_maping |= BIT(i);
+				bfee->su_reg_index = i;
+				break;
+			}
+		}
+		RTW_INFO("%s: Add BFee entry beamformee_su_reg_maping=%#x, su_reg_index=%d\n",
+			 __FUNCTION__, info->beamformee_su_reg_maping, bfee->su_reg_index);
+	}
+
+	return bfee;
+}
+
+static void _bfee_remove_entry(_adapter *adapter, struct beamformee_entry *entry)
+{
+	struct beamforming_info *info;
+	u8 idx;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	entry->state = BEAMFORM_ENTRY_HW_STATE_DELETE_INIT;
+
+	if (TEST_FLAG(entry->cap, BEAMFORMEE_CAP_VHT_MU)) {
+		info->beamformee_mu_cnt -= 1;
+		info->first_mu_bfee_index = _bfee_get_first_mu_entry_idx(adapter, entry);
+
+		if (_TRUE == info->bEnableSUTxBFWorkAround) {
+			if ((info->beamformee_mu_cnt == 0) && (info->beamformee_su_cnt > 0)) {
+				idx = _bfee_get_first_su_entry_idx(adapter, NULL);
+				info->TargetSUBFee = &info->bfee_entry[idx];
+				_rtw_memset(&info->TargetCSIInfo, 0, sizeof(struct _RT_CSI_INFO));
+				info->TargetSUBFee->bSuspendSUCap = _FALSE;
+			}
+		}
+	} else if (TEST_FLAG(entry->cap, BEAMFORMEE_CAP_VHT_SU|BEAMFORMEE_CAP_HT_EXPLICIT)) {
+		info->beamformee_su_cnt -= 1;
+
+		/* When the target SU BFee leaves, disable workaround */
+		if ((_TRUE == info->bEnableSUTxBFWorkAround)
+		    && (entry == info->TargetSUBFee)) {
+			entry->bSuspendSUCap = _TRUE;
+			info->TargetSUBFee = NULL;
+			_rtw_memset(&info->TargetCSIInfo, 0, sizeof(struct _RT_CSI_INFO));
+			rtw_bf_cmd(adapter, BEAMFORMING_CTRL_SET_CSI_REPORT, (u8*)&info->TargetCSIInfo, sizeof(struct _RT_CSI_INFO), 0);
+		}
+	}
+
+	if (info->beamformee_mu_cnt == 0)
+		info->beamforming_cap &= ~BEAMFORMER_CAP_VHT_MU;
+	if (info->beamformee_su_cnt == 0)
+		info->beamforming_cap &= ~(BEAMFORMER_CAP_VHT_SU|BEAMFORMER_CAP_HT_EXPLICIT);
+
+	_sounding_update_min_period(adapter, 0, _TRUE);
+}
+
+static enum beamforming_cap _bfee_get_entry_cap_by_macid(_adapter *adapter, u8 macid)
+{
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee;
+	u8 i;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	for (i = 0; i < MAX_BEAMFORMER_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (bfee->used == _FALSE)
+			continue;
+		if (bfee->mac_id == macid)
+			return bfee->cap;
+	}
+
+	return BEAMFORMING_CAP_NONE;
+}
+
+static void _beamforming_enter(_adapter *adapter, void *p)
+{
+	struct mlme_priv *mlme;
+	struct ht_priv *htpriv;
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv *vhtpriv;
+#endif
+	struct mlme_ext_priv *mlme_ext;
+	struct sta_info *sta, *sta_copy;
+	struct beamforming_info *info;
+	struct beamformer_entry *bfer = NULL;
+	struct beamformee_entry *bfee = NULL;
+	u8 wireless_mode;
+	u8 sta_bf_cap;
+	u8 sounding_dim = 0; /* number of sounding dimensions */
+	u8 comp_steering_num = 0; /* compressed steering number */
+
+
+	mlme = &adapter->mlmepriv;
+	htpriv = &mlme->htpriv;
+#ifdef CONFIG_80211AC_VHT
+	vhtpriv = &mlme->vhtpriv;
+#endif
+	mlme_ext = &adapter->mlmeextpriv;
+	info = GET_BEAMFORM_INFO(adapter);
+
+	sta_copy = (struct sta_info *)p;
+	sta = rtw_get_stainfo(&adapter->stapriv, sta_copy->phl_sta->mac_addr);
+	if (!sta) {
+		RTW_ERR("%s: Cann't find STA info for " MAC_FMT "\n",
+			__FUNCTION__, MAC_ARG(sta_copy->phl_sta->mac_addr));
+		return;
+	}
+	if (sta != sta_copy) {
+		RTW_WARN("%s: Origin sta(fake)=%p realsta=%p for " MAC_FMT "\n",
+		__FUNCTION__, sta_copy, sta, MAC_ARG(sta_copy->phl_sta->mac_addr));
+	}
+
+	/* The current setting does not support Beaforming */
+	wireless_mode = sta->phl_sta->wmode;
+	if ((is_supported_ht(wireless_mode) == _FALSE)
+	    && (is_supported_vht(wireless_mode) == _FALSE)) {
+		RTW_WARN("%s: Not support HT or VHT mode\n", __FUNCTION__);
+		return;
+	}
+
+	if ((0 == htpriv->beamform_cap)
+#ifdef CONFIG_80211AC_VHT
+	    && (0 == vhtpriv->beamform_cap)
+#endif
+#ifdef CONFIG_80211AX_HE
+	/* CONFIG_80211AX_HE_TODO */
+#endif
+	   ) {
+		RTW_INFO("The configuration disabled Beamforming! Skip...\n");
+		return;
+	}
+
+	_get_sta_beamform_cap(adapter, sta,
+			      &sta_bf_cap, &sounding_dim, &comp_steering_num);
+	RTW_INFO("STA Beamforming Capability=0x%02X\n", sta_bf_cap);
+	if (sta_bf_cap == BEAMFORMING_CAP_NONE)
+		return;
+	if ((sta_bf_cap & BEAMFORMEE_CAP_HT_EXPLICIT)
+	    || (sta_bf_cap & BEAMFORMEE_CAP_VHT_SU)
+	    || (sta_bf_cap & BEAMFORMEE_CAP_VHT_MU))
+		sta_bf_cap |= BEAMFORMEE_CAP;
+	if ((sta_bf_cap & BEAMFORMER_CAP_HT_EXPLICIT)
+	    || (sta_bf_cap & BEAMFORMER_CAP_VHT_SU)
+	    || (sta_bf_cap & BEAMFORMER_CAP_VHT_MU))
+		sta_bf_cap |= BEAMFORMER_CAP;
+
+	if (sta_bf_cap & BEAMFORMER_CAP) {
+		/* The other side is beamformer */
+		bfer = _bfer_add_entry(adapter, sta, sta_bf_cap, sounding_dim, comp_steering_num);
+		if (!bfer)
+			RTW_ERR("%s: Fail to allocate bfer entry!\n", __FUNCTION__);
+	}
+	if (sta_bf_cap & BEAMFORMEE_CAP) {
+		/* The other side is beamformee */
+		bfee = _bfee_add_entry(adapter, sta, sta_bf_cap, sounding_dim, comp_steering_num);
+		if (!bfee)
+			RTW_ERR("%s: Fail to allocate bfee entry!\n", __FUNCTION__);
+	}
+	if (!bfer && !bfee)
+		return;
+
+	rtw_hal_set_hwreg(adapter, HW_VAR_SOUNDING_ENTER, (u8*)sta);
+
+	/* Perform sounding if there is BFee */
+	if ((info->beamformee_su_cnt != 0)
+	    || (info->beamformee_mu_cnt != 0)) {
+		if (SOUNDING_STATE_NONE == info->sounding_info.state) {
+			info->sounding_info.state = SOUNDING_STATE_INIT;
+			/* Start sounding after 2 sec */
+			_set_timer(&info->sounding_timer, 2000);
+		}
+	}
+}
+
+static void _beamforming_reset(_adapter *adapter)
+{
+	RTW_ERR("%s: Not ready!!\n", __FUNCTION__);
+}
+
+static void _beamforming_leave(_adapter *adapter, u8 *ra)
+{
+	struct beamforming_info *info;
+	struct beamformer_entry *bfer = NULL;
+	struct beamformee_entry *bfee = NULL;
+	u8 bHwStateAddInit = _FALSE;
+
+
+	RTW_INFO("+%s\n", __FUNCTION__);
+
+	info = GET_BEAMFORM_INFO(adapter);
+	bfer = _bfer_get_entry_by_addr(adapter, ra);
+	bfee = _bfee_get_entry_by_addr(adapter, ra);
+
+	if (!bfer && !bfee) {
+		RTW_WARN("%s: " MAC_FMT " is neither beamforming ee or er!!\n",
+			__FUNCTION__, MAC_ARG(ra));
+		return;
+	}
+
+	if (bfer)
+		_bfer_remove_entry(adapter, bfer);
+
+	if (bfee)
+		_bfee_remove_entry(adapter, bfee);
+
+	rtw_hal_set_hwreg(adapter, HW_VAR_SOUNDING_LEAVE, ra);
+
+	/* Stop sounding if there is no any BFee */
+	if ((info->beamformee_su_cnt == 0)
+	    && (info->beamformee_mu_cnt == 0)) {
+		_cancel_timer_ex(&info->sounding_timer);
+		_sounding_init(&info->sounding_info);
+	}
+
+	RTW_INFO("-%s\n", __FUNCTION__);
+}
+
+static void _beamforming_sounding_down(_adapter *adapter, u8 status)
+{
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+	struct beamformee_entry *bfee;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	RTW_INFO("+%s: sounding=%d, status=0x%02x\n", __FUNCTION__, sounding->state, status);
+
+	if (sounding->state == SOUNDING_STATE_MU_START) {
+		RTW_INFO("%s: MU sounding done\n", __FUNCTION__);
+		sounding->state = SOUNDING_STATE_MU_SOUNDDOWN;
+		RTW_INFO("%s: Set to SOUNDING_STATE_MU_SOUNDDOWN\n", __FUNCTION__);
+		info->SetHalSoundownOnDemandCnt++;
+		rtw_hal_set_hwreg(adapter, HW_VAR_SOUNDING_STATUS, &status);
+	} else if (sounding->state == SOUNDING_STATE_SU_START) {
+		RTW_INFO("%s: SU entry[%d] sounding down\n", __FUNCTION__, sounding->su_bfee_curidx);
+		bfee = &info->bfee_entry[sounding->su_bfee_curidx];
+		sounding->state = SOUNDING_STATE_SU_SOUNDDOWN;
+		RTW_INFO("%s: Set to SOUNDING_STATE_SU_SOUNDDOWN\n", __FUNCTION__);
+
+		/*
+		 * <tynli_note>
+		 *	bfee->bSoundingTimeout this flag still cannot avoid
+		 *	old sound down event happens in the new sounding period.
+		 *	2015.12.10
+		 */
+		if (_TRUE == bfee->bSoundingTimeout) {
+			RTW_WARN("%s: The entry[%d] is bSoundingTimeout!\n", __FUNCTION__, sounding->su_bfee_curidx);
+			bfee->bSoundingTimeout = _FALSE;
+			return;
+		}
+
+		if (_TRUE == status) {
+			/* success */
+			bfee->LogStatusFailCnt = 0;
+			info->SetHalSoundownOnDemandCnt++;
+			rtw_hal_set_hwreg(adapter, HW_VAR_SOUNDING_STATUS, &status);
+		} else if (_TRUE == bfee->bDeleteSounding) {
+			RTW_WARN("%s: Delete entry[%d] sounding info!\n", __FUNCTION__, sounding->su_bfee_curidx);
+			rtw_hal_set_hwreg(adapter, HW_VAR_SOUNDING_STATUS, &status);
+			bfee->bDeleteSounding = _FALSE;
+		} else {
+			bfee->LogStatusFailCnt++;
+			RTW_WARN("%s: LogStatusFailCnt=%d\n", __FUNCTION__, bfee->LogStatusFailCnt);
+			if (bfee->LogStatusFailCnt > 30) {
+				RTW_ERR("%s: LogStatusFailCnt > 30, Stop SOUNDING!!\n", __FUNCTION__);
+				rtw_bf_cmd(adapter, BEAMFORMING_CTRL_LEAVE, bfee->mac_addr, ETH_ALEN, 1);
+			}
+		}
+	} else {
+		RTW_WARN("%s: unexpected sounding state:0x%02x\n", __FUNCTION__, sounding->state);
+		return;
+	}
+
+	rtw_bf_cmd(adapter, BEAMFORMING_CTRL_START_PERIOD, NULL, 0, 0);
+}
+
+static void _c2h_snd_txbf(_adapter *adapter, u8 *buf, u8 buf_len)
+{
+	struct beamforming_info	*info;
+	u8 res;
+
+	info = GET_BEAMFORM_INFO(adapter);
+
+	_cancel_timer_ex(&info->sounding_timeout_timer);
+
+	res = C2H_SND_TXBF_GET_SND_RESULT(buf) ? _TRUE : _FALSE;
+	RTW_INFO("+%s: %s\n", __FUNCTION__, res==_TRUE?"Success":"Fail!");
+
+	rtw_bf_cmd(adapter, BEAMFORMING_CTRL_END_PERIOD, &res, 1, 1);
+}
+
+/*
+ * Description:
+ *	This function is for phydm only
+ */
+enum beamforming_cap rtw_bf_bfee_get_entry_cap_by_macid(void *mlme, u8 macid)
+{
+	_adapter *adapter;
+	enum beamforming_cap cap = BEAMFORMING_CAP_NONE;
+
+
+	adapter = mlme_to_adapter((struct mlme_priv *)mlme);
+	cap = _bfee_get_entry_cap_by_macid(adapter, macid);
+
+	return cap;
+}
+
+struct beamformer_entry *rtw_bf_bfer_get_entry_by_addr(_adapter *adapter, u8 *ra)
+{
+	return _bfer_get_entry_by_addr(adapter, ra);
+}
+
+struct beamformee_entry *rtw_bf_bfee_get_entry_by_addr(_adapter *adapter, u8 *ra)
+{
+	return _bfee_get_entry_by_addr(adapter, ra);
+}
+
+void rtw_bf_get_ndpa_packet(_adapter *adapter, union recv_frame *precv_frame)
+{
+	RTW_DBG("+%s\n", __FUNCTION__);
+}
+
+u32 rtw_bf_get_report_packet(_adapter *adapter, union recv_frame *precv_frame)
+{
+	u32 ret = _SUCCESS;
+	struct beamforming_info *info;
+	struct beamformee_entry *bfee = NULL;
+	u8 *pframe;
+	u32 frame_len;
+	u8 *ta;
+	u8 *frame_body;
+	u8 category, action;
+	u8 *pMIMOCtrlField, *pCSIMatrix;
+	u8 Nc = 0, Nr = 0, CH_W = 0, Ng = 0, CodeBook = 0;
+	u16 CSIMatrixLen = 0;
+
+
+	RTW_INFO("+%s\n", __FUNCTION__);
+
+	info = GET_BEAMFORM_INFO(adapter);
+	pframe = precv_frame->u.hdr.rx_data;
+	frame_len = precv_frame->u.hdr.len;
+
+	/* Memory comparison to see if CSI report is the same with previous one */
+	ta = get_addr2_ptr(pframe);
+	bfee = _bfee_get_entry_by_addr(adapter, ta);
+	if (!bfee)
+		return _FAIL;
+
+	frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	category = frame_body[0];
+	action = frame_body[1];
+
+	if ((category == RTW_WLAN_CATEGORY_VHT)
+	    && (action == RTW_WLAN_ACTION_VHT_COMPRESSED_BEAMFORMING)) {
+		pMIMOCtrlField = pframe + 26;
+		Nc = (*pMIMOCtrlField) & 0x7;
+		Nr = ((*pMIMOCtrlField) & 0x38) >> 3;
+		CH_W =  (((*pMIMOCtrlField) & 0xC0) >> 6);
+		Ng = (*(pMIMOCtrlField+1)) & 0x3;
+		CodeBook = ((*(pMIMOCtrlField+1)) & 0x4) >> 2;
+		/*
+		 * 24+(1+1+3)+2
+		 * ==> MAC header+(Category+ActionCode+MIMOControlField)+SNR(Nc=2)
+		 */
+		pCSIMatrix = pMIMOCtrlField + 3 + Nc;
+		CSIMatrixLen = frame_len - 26 - 3 - Nc;
+		info->TargetCSIInfo.bVHT = _TRUE;
+	} else if ((category == RTW_WLAN_CATEGORY_HT)
+		   && (action == RTW_WLAN_ACTION_HT_COMPRESS_BEAMFORMING)) {
+		pMIMOCtrlField = pframe + 26;
+		Nc = (*pMIMOCtrlField) & 0x3;
+		Nr = ((*pMIMOCtrlField) & 0xC) >> 2;
+		CH_W = ((*pMIMOCtrlField) & 0x10) >> 4;
+		Ng = ((*pMIMOCtrlField) & 0x60) >> 5;
+		CodeBook = ((*(pMIMOCtrlField+1)) & 0x6) >> 1;
+		/*
+		 * 24+(1+1+6)+2
+		 * ==> MAC header+(Category+ActionCode+MIMOControlField)+SNR(Nc=2)
+		 */
+		pCSIMatrix = pMIMOCtrlField + 6 + Nr;
+		CSIMatrixLen = frame_len  - 26 - 6 - Nr;
+		info->TargetCSIInfo.bVHT = _FALSE;
+	}
+
+	/* Update current CSI report info */
+	if ((_TRUE == info->bEnableSUTxBFWorkAround)
+	    && (info->TargetSUBFee == bfee)) {
+		if ((info->TargetCSIInfo.Nc != Nc) || (info->TargetCSIInfo.Nr != Nr) ||
+			(info->TargetCSIInfo.ChnlWidth != CH_W) || (info->TargetCSIInfo.Ng != Ng) ||
+			(info->TargetCSIInfo.CodeBook != CodeBook)) {
+			info->TargetCSIInfo.Nc = Nc;
+			info->TargetCSIInfo.Nr = Nr;
+			info->TargetCSIInfo.ChnlWidth = CH_W;
+			info->TargetCSIInfo.Ng = Ng;
+			info->TargetCSIInfo.CodeBook = CodeBook;
+
+			rtw_bf_cmd(adapter, BEAMFORMING_CTRL_SET_CSI_REPORT, (u8*)&info->TargetCSIInfo, sizeof(struct _RT_CSI_INFO), 1);
+		}
+	}
+
+	RTW_INFO("%s: pkt type=%d-%d, Nc=%d, Nr=%d, CH_W=%d, Ng=%d, CodeBook=%d\n",
+		 __FUNCTION__, category, action, Nc, Nr, CH_W, Ng, CodeBook);
+
+	return ret;
+}
+
+u8 rtw_bf_send_vht_gid_mgnt_packet(_adapter *adapter, u8 *ra, u8 *gid, u8 *position)
+{
+	/* General */
+	struct xmit_priv *xmitpriv;
+	struct mlme_priv *mlmepriv;
+	struct xmit_frame *pmgntframe;
+	/* MISC */
+	struct pkt_attrib *attrib;
+	struct rtw_ieee80211_hdr *wlanhdr;
+	u8 *pframe, *ptr;
+
+
+	xmitpriv = &adapter->xmitpriv;
+	mlmepriv = &adapter->mlmepriv;
+
+	pmgntframe = alloc_mgtxmitframe(xmitpriv);
+	if (!pmgntframe)
+		return _FALSE;
+
+	/* update attribute */
+	attrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(adapter, attrib);
+	attrib->rate = MGN_6M;
+	attrib->bwmode = CHANNEL_WIDTH_20;
+	attrib->subtype = WIFI_ACTION;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)pmgntframe->buf_addr + TXDESC_OFFSET;
+	wlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	wlanhdr->frame_ctl = 0;
+	set_frame_sub_type(pframe, attrib->subtype);
+	set_duration(pframe, 0);
+	SetFragNum(pframe, 0);
+	SetSeqNum(pframe, 0);
+
+	_rtw_memcpy(wlanhdr->addr1, ra, ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr2, adapter_mac_addr(adapter), ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr3, get_bssid(mlmepriv), ETH_ALEN);
+
+	pframe[24] = RTW_WLAN_CATEGORY_VHT;
+	pframe[25] = RTW_WLAN_ACTION_VHT_GROUPID_MANAGEMENT;
+	/* Set Membership Status Array */
+	ptr = pframe + 26;
+	_rtw_memcpy(ptr, gid, 8);
+	/* Set User Position Array */
+	ptr = pframe + 34;
+	_rtw_memcpy(ptr, position, 16);
+
+	attrib->pktlen = 54;
+	attrib->last_txcmdsz = attrib->pktlen;
+
+	dump_mgntframe(adapter, pmgntframe);
+
+	return _TRUE;
+}
+
+/*
+ * Description:
+ *	On VHT GID management frame by an MU beamformee.
+ */
+void rtw_bf_get_vht_gid_mgnt_packet(_adapter *adapter, union recv_frame *precv_frame)
+{
+	u8 *pframe;
+	u8 *ta, *gid, *position;
+
+
+	RTW_DBG("+%s\n", __FUNCTION__);
+
+	pframe = precv_frame->u.hdr.rx_data;
+
+	/* Get address by Addr2 */
+	ta = get_addr2_ptr(pframe);
+	/* Remove signaling TA */
+	ta[0] &= 0xFE;
+
+	/* Membership Status Array */
+	gid = pframe + 26;
+	/* User Position Array */
+	position= pframe + 34;
+
+	_bfer_set_entry_gid(adapter, ta, gid, position);
+}
+
+void rtw_bf_init(_adapter *adapter)
+{
+	struct beamforming_info	*info;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+	info->beamforming_cap = BEAMFORMING_CAP_NONE;
+	info->beamforming_state = BEAMFORMING_STATE_IDLE;
+/*
+	info->bfee_entry[MAX_BEAMFORMEE_ENTRY_NUM];
+	info->bfer_entry[MAX_BEAMFORMER_ENTRY_NUM];
+*/
+	info->sounding_sequence = 0;
+	info->beamformee_su_cnt = 0;
+	info->beamformer_su_cnt = 0;
+	info->beamformee_su_reg_maping = 0;
+	info->beamformer_su_reg_maping = 0;
+	info->beamformee_mu_cnt = 0;
+	info->beamformer_mu_cnt = 0;
+	info->beamformee_mu_reg_maping = 0;
+	info->first_mu_bfee_index = 0xFF;
+	info->mu_bfer_curidx = 0xFF;
+	info->cur_csi_rpt_rate = HALMAC_OFDM24;
+
+	_sounding_init(&info->sounding_info);
+	rtw_init_timer(&info->sounding_timer, _sounding_timer_handler, adapter);
+	rtw_init_timer(&info->sounding_timeout_timer, _sounding_timeout_timer_handler, adapter);
+
+	info->SetHalBFEnterOnDemandCnt = 0;
+	info->SetHalBFLeaveOnDemandCnt = 0;
+	info->SetHalSoundownOnDemandCnt = 0;
+
+	info->bEnableSUTxBFWorkAround = _TRUE;
+	info->TargetSUBFee = NULL;
+
+	info->sounding_running = 0;
+}
+
+void rtw_bf_cmd_hdl(_adapter *adapter, u8 type, u8 *pbuf)
+{
+	switch (type) {
+	case BEAMFORMING_CTRL_ENTER:
+		_beamforming_enter(adapter, pbuf);
+		break;
+
+	case BEAMFORMING_CTRL_LEAVE:
+		if (pbuf == NULL)
+			_beamforming_reset(adapter);
+		else
+			_beamforming_leave(adapter, pbuf);
+		break;
+
+	case BEAMFORMING_CTRL_START_PERIOD:
+		_sounding_handler(adapter);
+		break;
+
+	case BEAMFORMING_CTRL_END_PERIOD:
+		_beamforming_sounding_down(adapter, *pbuf);
+		break;
+
+	case BEAMFORMING_CTRL_SET_GID_TABLE:
+		rtw_hal_set_hwreg(adapter, HW_VAR_SOUNDING_SET_GID_TABLE, pbuf);
+		break;
+
+	case BEAMFORMING_CTRL_SET_CSI_REPORT:
+		rtw_hal_set_hwreg(adapter, HW_VAR_SOUNDING_CSI_REPORT, pbuf);
+		break;
+
+	default:
+		break;
+	}
+}
+
+u8 rtw_bf_cmd(_adapter *adapter, s32 type, u8 *pbuf, s32 size, u8 enqueue)
+{
+	struct cmd_obj *cmd;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	u8 *wk_buf;
+	u8 res = _SUCCESS;
+
+	if (!enqueue) {
+		rtw_bf_cmd_hdl(adapter, type, pbuf);
+		goto exit;
+	}
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = adapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	if (pbuf != NULL) {
+		wk_buf = rtw_zmalloc(size);
+		if (wk_buf == NULL) {
+			rtw_mfree((u8 *)cmd, sizeof(struct cmd_obj));
+			rtw_mfree((u8 *)pdrvextra_cmd_parm, sizeof(struct drvextra_cmd_parm));
+			res = _FAIL;
+			goto exit;
+		}
+
+		_rtw_memcpy(wk_buf, pbuf, size);
+	} else {
+		wk_buf = NULL;
+		size = 0;
+	}
+
+	pdrvextra_cmd_parm->ec_id = BEAMFORMING_WK_CID;
+	pdrvextra_cmd_parm->type = type;
+	pdrvextra_cmd_parm->size = size;
+	pdrvextra_cmd_parm->pbuf = wk_buf;
+
+	init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;
+}
+
+void rtw_bf_update_attrib(_adapter *adapter, struct pkt_attrib *attrib, struct sta_info *sta)
+{
+/* bf_cmn_info move to hal_sta */
+#if 0
+	if (sta) {
+		attrib->txbf_g_id = sta->phl_sta->bf_info.g_id;
+		attrib->txbf_p_aid = sta->phl_sta->bf_info.p_aid;
+	}
+#endif
+}
+
+void rtw_bf_c2h_handler(_adapter *adapter, u8 id, u8 *buf, u8 buf_len)
+{
+	switch (id) {
+	case CMD_ID_C2H_SND_TXBF:
+		_c2h_snd_txbf(adapter, buf, buf_len);
+		break;
+	}
+}
+
+#define toMbps(bytes, secs)	(rtw_division64(bytes >> 17, secs))
+void rtw_bf_update_traffic(_adapter *adapter)
+{
+	struct beamforming_info	*info;
+	struct sounding_info *sounding;
+	struct beamformee_entry *bfee;
+	struct sta_info *sta;
+	u8 bfee_cnt, sounding_idx, i;
+	u16 tp[MAX_BEAMFORMEE_ENTRY_NUM] = {0};
+	u16 tx_rate[MAX_BEAMFORMEE_ENTRY_NUM] = {0};
+	u64 tx_bytes, last_bytes;
+	u32 time;
+	systime last_timestamp;
+	u8 set_timer = _FALSE;
+
+
+	info = GET_BEAMFORM_INFO(adapter);
+	sounding = &info->sounding_info;
+
+	/* Check any bfee exist? */
+	bfee_cnt = info->beamformee_su_cnt + info->beamformee_mu_cnt;
+	if (bfee_cnt == 0)
+		return;
+
+	for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (_FALSE == bfee->used)
+			continue;
+
+		sta = rtw_get_stainfo(&adapter->stapriv, bfee->mac_addr);
+		if (!sta) {
+			RTW_ERR("%s: Cann't find sta_info for " MAC_FMT "!\n", __FUNCTION__, MAC_ARG(bfee->mac_addr));
+			continue;
+		}
+
+		last_timestamp = bfee->tx_timestamp;
+		last_bytes = bfee->tx_bytes;
+		bfee->tx_timestamp = rtw_get_current_time();
+		bfee->tx_bytes = sta->sta_stats.tx_bytes;
+		if (last_timestamp) {
+			if (bfee->tx_bytes >= last_bytes)
+				tx_bytes = bfee->tx_bytes - last_bytes;
+			else
+				tx_bytes = bfee->tx_bytes + (~last_bytes);
+			time = rtw_get_time_interval_ms(last_timestamp, bfee->tx_timestamp);
+			time = (time > 1000) ? time/1000 : 1;
+			tp[i] = toMbps(tx_bytes, time);
+			tx_rate[i] = rtw_get_current_tx_rate(adapter, sta);
+			RTW_INFO("%s: BFee idx(%d), MadId(%d), TxTP=%lld bytes (%d Mbps), txrate=%d\n",
+				 __FUNCTION__, i, bfee->mac_id, tx_bytes, tp[i], tx_rate[i]);
+		}
+	}
+
+	sounding_idx = rtw_hal_get_sounding_info(adapter, tp, MAX_BEAMFORMEE_ENTRY_NUM, tx_rate);
+
+	for (i = 0; i < MAX_BEAMFORMEE_ENTRY_NUM; i++) {
+		bfee = &info->bfee_entry[i];
+		if (_FALSE == bfee->used) {
+			if (sounding_idx & BIT(i))
+				RTW_WARN("%s: bfee(%d) not in used but need sounding?!\n", __FUNCTION__, i);
+			continue;
+		}
+
+		if (sounding_idx & BIT(i)) {
+			if (_FALSE == bfee->bApplySounding) {
+				bfee->bApplySounding = _TRUE;
+				bfee->SoundCnt = 0;
+				set_timer = _TRUE;
+			}
+		} else {
+			if (_TRUE == bfee->bApplySounding) {
+				bfee->bApplySounding = _FALSE;
+				bfee->bDeleteSounding = _TRUE;
+				bfee->SoundCnt = 0;
+				set_timer = _TRUE;
+			}
+		}
+	}
+
+	if (_TRUE == set_timer) {
+		if (SOUNDING_STATE_NONE == info->sounding_info.state) {
+			info->sounding_info.state = SOUNDING_STATE_INIT;
+			_set_timer(&info->sounding_timer, 0);
+		}
+	}
+}
+#else
+/*
+ * rtw_bf_get_vht_gid_mgnt_packet() - Set VHT GID for MU beamformee
+ * @a:		struct _ADAPTER*
+ * @rframe:	union recv_frame*
+ *
+ * Return 0 for success, otherwise fail.
+ *
+ */
+int rtw_bf_get_vht_gid_mgnt_packet(struct _ADAPTER *a, union recv_frame *rframe)
+{
+	struct rtw_phl_gid_pos_tbl tbl = {0};
+	u8 *data, *gid, *pos;
+	enum rtw_phl_status status;
+	int err = 0;
+
+
+	data = rframe->u.hdr.rx_data;
+	RTW_DBG(FUNC_ADPT_FMT ": GID setting for " MAC_FMT "\n",
+		FUNC_ADPT_ARG(a), MAC_ARG(get_addr2_ptr(data)));
+	/* Move to data start */
+	data += 26;
+
+	/* Membership Status Array */
+	gid = data;
+	_rtw_memcpy(tbl.gid_vld, gid, RTW_VHT_GID_MGNT_FRAME_GID_SZ);
+	RTW_DBG_DUMP("Membership Status Array: ", tbl.gid_vld, RTW_VHT_GID_MGNT_FRAME_GID_SZ);
+	/* User Position Array */
+	pos= data + RTW_VHT_GID_MGNT_FRAME_GID_SZ;
+	_rtw_memcpy(tbl.pos, pos, RTW_VHT_GID_MGNT_FRAME_POS_SZ);
+	RTW_DBG_DUMP("User Position Array: ", tbl.pos, RTW_VHT_GID_MGNT_FRAME_POS_SZ);
+
+	/* Config HW GID table */
+	status = rtw_phl_snd_cmd_set_vht_gid(GET_PHL_INFO(adapter_to_dvobj(a)),
+					      a->phl_role, &tbl);
+	if (status == RTW_PHL_STATUS_SUCCESS) {
+		RTW_DBG(FUNC_ADPT_FMT ": Add VHT GID Success\n",
+			FUNC_ADPT_ARG(a));
+	} else {
+		RTW_ERR(FUNC_ADPT_FMT ": Add VHT GID FAIL!(0x%x)\n",
+			FUNC_ADPT_ARG(a), status);
+		err = -1;
+	}
+
+	return err;
+}
+#endif
+#endif /* CONFIG_BEAMFORMING */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_br_ext.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_br_ext.c
new file mode 100644
index 000000000000..48fed925d7f8
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_br_ext.c
@@ -0,0 +1,1582 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_BR_EXT_C_
+
+#ifdef __KERNEL__
+	#include <linux/version.h>
+	#include <linux/if_arp.h>
+	#include <net/ip.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0))
+	#include <net/ipx.h>
+#endif
+	#include <linux/atalk.h>
+	#include <linux/udp.h>
+	#include <linux/if_pppox.h>
+#endif
+
+#if 1	/* rtw_wifi_driver */
+	#include <drv_types.h>
+#else	/* rtw_wifi_driver */
+	#include "./8192cd_cfg.h"
+
+	#ifndef __KERNEL__
+		#include "./sys-support.h"
+	#endif
+
+	#include "./8192cd.h"
+	#include "./8192cd_headers.h"
+	#include "./8192cd_br_ext.h"
+	#include "./8192cd_debug.h"
+#endif /* rtw_wifi_driver */
+
+#ifdef CL_IPV6_PASS
+	#ifdef __KERNEL__
+		#include <linux/ipv6.h>
+		#include <linux/icmpv6.h>
+		#include <net/ndisc.h>
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+			#include <net/ip6_checksum.h>
+		#else
+			#include <net/checksum.h>
+		#endif
+	#endif
+#endif
+
+#ifdef CONFIG_BR_EXT
+
+/* #define BR_EXT_DEBUG */
+
+#define NAT25_IPV4		01
+#define NAT25_IPV6		02
+#define NAT25_IPX		03
+#define NAT25_APPLE		04
+#define NAT25_PPPOE		05
+
+#define RTL_RELAY_TAG_LEN (ETH_ALEN)
+#define TAG_HDR_LEN		4
+
+#define MAGIC_CODE		0x8186
+#define MAGIC_CODE_LEN	2
+#define WAIT_TIME_PPPOE	5	/* waiting time for pppoe server in sec */
+
+/*-----------------------------------------------------------------
+  How database records network address:
+           0    1    2    3    4    5    6    7    8    9   10
+        |----|----|----|----|----|----|----|----|----|----|----|
+  IPv4  |type|                             |      IP addr      |
+  IPX   |type|      Net addr     |          Node addr          |
+  IPX   |type|      Net addr     |Sckt addr|
+  Apple |type| Network |node|
+  PPPoE |type|   SID   |           AC MAC            |
+-----------------------------------------------------------------*/
+
+
+/* Find a tag in pppoe frame and return the pointer */
+static __inline__ unsigned char *__nat25_find_pppoe_tag(struct pppoe_hdr *ph, unsigned short type)
+{
+	unsigned char *cur_ptr, *start_ptr;
+	unsigned short tagLen, tagType;
+
+	start_ptr = cur_ptr = (unsigned char *)ph->tag;
+	while ((cur_ptr - start_ptr) < ntohs(ph->length)) {
+		/* prevent un-alignment access */
+		tagType = (unsigned short)((cur_ptr[0] << 8) + cur_ptr[1]);
+		tagLen  = (unsigned short)((cur_ptr[2] << 8) + cur_ptr[3]);
+		if (tagType == type)
+			return cur_ptr;
+		cur_ptr = cur_ptr + TAG_HDR_LEN + tagLen;
+	}
+	return 0;
+}
+
+
+static __inline__ int __nat25_add_pppoe_tag(struct sk_buff *skb, struct pppoe_tag *tag)
+{
+	struct pppoe_hdr *ph = (struct pppoe_hdr *)(skb->data + ETH_HLEN);
+	int data_len;
+
+	data_len = tag->tag_len + TAG_HDR_LEN;
+	if (skb_tailroom(skb) < data_len) {
+		_DEBUG_ERR("skb_tailroom() failed in add SID tag!\n");
+		return -1;
+	}
+
+	skb_put(skb, data_len);
+	/* have a room for new tag */
+	_rtw_memmove(((unsigned char *)ph->tag + data_len), (unsigned char *)ph->tag, ntohs(ph->length));
+	ph->length = htons(ntohs(ph->length) + data_len);
+	_rtw_memcpy((unsigned char *)ph->tag, tag, data_len);
+	return data_len;
+}
+
+static int skb_pull_and_merge(struct sk_buff *skb, unsigned char *src, int len)
+{
+	int tail_len;
+	unsigned long end, tail;
+
+	if ((src + len) > skb_tail_pointer(skb) || skb->len < len)
+		return -1;
+
+	tail = (unsigned long)skb_tail_pointer(skb);
+	end = (unsigned long)src + len;
+	if (tail < end)
+		return -1;
+
+	tail_len = (int)(tail - end);
+	if (tail_len > 0)
+		_rtw_memmove(src, src + len, tail_len);
+
+	skb_trim(skb, skb->len - len);
+	return 0;
+}
+
+static __inline__ unsigned long __nat25_timeout(_adapter *priv)
+{
+	unsigned long timeout;
+
+	timeout = jiffies - NAT25_AGEING_TIME * HZ;
+
+	return timeout;
+}
+
+
+static __inline__ int  __nat25_has_expired(_adapter *priv,
+		struct nat25_network_db_entry *fdb)
+{
+	if (time_before_eq(fdb->ageing_timer, __nat25_timeout(priv)))
+		return 1;
+
+	return 0;
+}
+
+
+static __inline__ void __nat25_generate_ipv4_network_addr(unsigned char *networkAddr,
+		unsigned int *ipAddr)
+{
+	_rtw_memset(networkAddr, 0, MAX_NETWORK_ADDR_LEN);
+
+	networkAddr[0] = NAT25_IPV4;
+	_rtw_memcpy(networkAddr + 7, (unsigned char *)ipAddr, 4);
+}
+
+
+static __inline__ void __nat25_generate_ipx_network_addr_with_node(unsigned char *networkAddr,
+		unsigned int *ipxNetAddr, unsigned char *ipxNodeAddr)
+{
+	_rtw_memset(networkAddr, 0, MAX_NETWORK_ADDR_LEN);
+
+	networkAddr[0] = NAT25_IPX;
+	_rtw_memcpy(networkAddr + 1, (unsigned char *)ipxNetAddr, 4);
+	_rtw_memcpy(networkAddr + 5, ipxNodeAddr, 6);
+}
+
+
+static __inline__ void __nat25_generate_ipx_network_addr_with_socket(unsigned char *networkAddr,
+		unsigned int *ipxNetAddr, unsigned short *ipxSocketAddr)
+{
+	_rtw_memset(networkAddr, 0, MAX_NETWORK_ADDR_LEN);
+
+	networkAddr[0] = NAT25_IPX;
+	_rtw_memcpy(networkAddr + 1, (unsigned char *)ipxNetAddr, 4);
+	_rtw_memcpy(networkAddr + 5, (unsigned char *)ipxSocketAddr, 2);
+}
+
+
+static __inline__ void __nat25_generate_apple_network_addr(unsigned char *networkAddr,
+		unsigned short *network, unsigned char *node)
+{
+	_rtw_memset(networkAddr, 0, MAX_NETWORK_ADDR_LEN);
+
+	networkAddr[0] = NAT25_APPLE;
+	_rtw_memcpy(networkAddr + 1, (unsigned char *)network, 2);
+	networkAddr[3] = *node;
+}
+
+
+static __inline__ void __nat25_generate_pppoe_network_addr(unsigned char *networkAddr,
+		unsigned char *ac_mac, unsigned short *sid)
+{
+	_rtw_memset(networkAddr, 0, MAX_NETWORK_ADDR_LEN);
+
+	networkAddr[0] = NAT25_PPPOE;
+	_rtw_memcpy(networkAddr + 1, (unsigned char *)sid, 2);
+	_rtw_memcpy(networkAddr + 3, (unsigned char *)ac_mac, 6);
+}
+
+
+#ifdef CL_IPV6_PASS
+static  void __nat25_generate_ipv6_network_addr(unsigned char *networkAddr,
+		unsigned int *ipAddr)
+{
+	_rtw_memset(networkAddr, 0, MAX_NETWORK_ADDR_LEN);
+
+	networkAddr[0] = NAT25_IPV6;
+	_rtw_memcpy(networkAddr + 1, (unsigned char *)ipAddr, 16);
+}
+
+
+static unsigned char *scan_tlv(unsigned char *data, int len, unsigned char tag, unsigned char len8b)
+{
+	while (len > 0) {
+		if (*data == tag && *(data + 1) == len8b && len >= len8b * 8)
+			return data + 2;
+
+		len -= (*(data + 1)) * 8;
+		data += (*(data + 1)) * 8;
+	}
+	return NULL;
+}
+
+
+static int update_nd_link_layer_addr(unsigned char *data, int len, unsigned char *replace_mac)
+{
+	struct icmp6hdr *icmphdr = (struct icmp6hdr *)data;
+	unsigned char *mac;
+
+	if (icmphdr->icmp6_type == NDISC_ROUTER_SOLICITATION) {
+		if (len >= 8) {
+			mac = scan_tlv(&data[8], len - 8, 1, 1);
+			if (mac) {
+				RTW_INFO("Router Solicitation, replace MAC From: "MAC_FMT", To: "MAC_FMT"\n",
+					MAC_ARG(mac),
+					MAC_ARG(replace_mac));
+				_rtw_memcpy(mac, replace_mac, 6);
+				return 1;
+			}
+		}
+	} else if (icmphdr->icmp6_type == NDISC_ROUTER_ADVERTISEMENT) {
+		if (len >= 16) {
+			mac = scan_tlv(&data[16], len - 16, 1, 1);
+			if (mac) {
+				RTW_INFO("Router Advertisement, replace MAC From: "MAC_FMT", To: "MAC_FMT"\n",
+					MAC_ARG(mac),
+					MAC_ARG(replace_mac));
+				_rtw_memcpy(mac, replace_mac, 6);
+				return 1;
+			}
+		}
+	} else if (icmphdr->icmp6_type == NDISC_NEIGHBOUR_SOLICITATION) {
+		if (len >= 24) {
+			mac = scan_tlv(&data[24], len - 24, 1, 1);
+			if (mac) {
+				RTW_INFO("Neighbor Solicitation, replace MAC From: "MAC_FMT", To: "MAC_FMT"\n",
+					MAC_ARG(mac),
+					MAC_ARG(replace_mac));
+				_rtw_memcpy(mac, replace_mac, 6);
+				return 1;
+			}
+		}
+	} else if (icmphdr->icmp6_type == NDISC_NEIGHBOUR_ADVERTISEMENT) {
+		if (len >= 24) {
+			mac = scan_tlv(&data[24], len - 24, 2, 1);
+			if (mac) {
+				RTW_INFO("Neighbor Advertisement, replace MAC From: "MAC_FMT", To: "MAC_FMT"\n",
+					MAC_ARG(mac),
+					MAC_ARG(replace_mac));
+				_rtw_memcpy(mac, replace_mac, 6);
+				return 1;
+			}
+		}
+	} else if (icmphdr->icmp6_type == NDISC_REDIRECT) {
+		if (len >= 40) {
+			mac = scan_tlv(&data[40], len - 40, 2, 1);
+			if (mac) {
+				RTW_INFO("Redirect,  replace MAC From: "MAC_FMT", To: "MAC_FMT"\n",
+					MAC_ARG(mac),
+					MAC_ARG(replace_mac));
+				_rtw_memcpy(mac, replace_mac, 6);
+				return 1;
+			}
+		}
+	}
+	return 0;
+}
+
+#ifdef SUPPORT_RX_UNI2MCAST
+static void convert_ipv6_mac_to_mc(struct sk_buff *skb)
+{
+	struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + ETH_HLEN);
+	unsigned char *dst_mac = skb->data;
+
+	/* dst_mac[0] = 0xff; */
+	/* dst_mac[1] = 0xff; */
+	/*modified by qinjunjie,ipv6 multicast address ix 0x33-33-xx-xx-xx-xx*/
+	dst_mac[0] = 0x33;
+	dst_mac[1] = 0x33;
+	_rtw_memcpy(&dst_mac[2], &iph->daddr.s6_addr32[3], 4);
+#if defined(__LINUX_2_6__)
+	/*modified by qinjunjie,warning:should not remove next line*/
+	skb->pkt_type = PACKET_MULTICAST;
+#endif
+}
+#endif /* CL_IPV6_PASS */
+#endif /* SUPPORT_RX_UNI2MCAST */
+
+
+static __inline__ int __nat25_network_hash(unsigned char *networkAddr)
+{
+	if (networkAddr[0] == NAT25_IPV4) {
+		unsigned long x;
+
+		x = networkAddr[7] ^ networkAddr[8] ^ networkAddr[9] ^ networkAddr[10];
+
+		return x & (NAT25_HASH_SIZE - 1);
+	} else if (networkAddr[0] == NAT25_IPX) {
+		unsigned long x;
+
+		x = networkAddr[1] ^ networkAddr[2] ^ networkAddr[3] ^ networkAddr[4] ^ networkAddr[5] ^
+		    networkAddr[6] ^ networkAddr[7] ^ networkAddr[8] ^ networkAddr[9] ^ networkAddr[10];
+
+		return x & (NAT25_HASH_SIZE - 1);
+	} else if (networkAddr[0] == NAT25_APPLE) {
+		unsigned long x;
+
+		x = networkAddr[1] ^ networkAddr[2] ^ networkAddr[3];
+
+		return x & (NAT25_HASH_SIZE - 1);
+	} else if (networkAddr[0] == NAT25_PPPOE) {
+		unsigned long x;
+
+		x = networkAddr[0] ^ networkAddr[1] ^ networkAddr[2] ^ networkAddr[3] ^ networkAddr[4] ^ networkAddr[5] ^ networkAddr[6] ^ networkAddr[7] ^ networkAddr[8];
+
+		return x & (NAT25_HASH_SIZE - 1);
+	}
+#ifdef CL_IPV6_PASS
+	else if (networkAddr[0] == NAT25_IPV6) {
+		unsigned long x;
+
+		x = networkAddr[1] ^ networkAddr[2] ^ networkAddr[3] ^ networkAddr[4] ^ networkAddr[5] ^
+		    networkAddr[6] ^ networkAddr[7] ^ networkAddr[8] ^ networkAddr[9] ^ networkAddr[10] ^
+		    networkAddr[11] ^ networkAddr[12] ^ networkAddr[13] ^ networkAddr[14] ^ networkAddr[15] ^
+		    networkAddr[16];
+
+		return x & (NAT25_HASH_SIZE - 1);
+	}
+#endif
+	else {
+		unsigned long x = 0;
+		int i;
+
+		for (i = 0; i < MAX_NETWORK_ADDR_LEN; i++)
+			x ^= networkAddr[i];
+
+		return x & (NAT25_HASH_SIZE - 1);
+	}
+}
+
+
+static __inline__ void __network_hash_link(_adapter *priv,
+		struct nat25_network_db_entry *ent, int hash)
+{
+	/* Caller must _rtw_spinlock_bh already! */
+	/* _rtw_spinlock_bh(&priv->br_ext_lock); */
+
+	ent->next_hash = priv->nethash[hash];
+	if (ent->next_hash != NULL)
+		ent->next_hash->pprev_hash = &ent->next_hash;
+	priv->nethash[hash] = ent;
+	ent->pprev_hash = &priv->nethash[hash];
+
+	/* _rtw_spinunlock_bh(&priv->br_ext_lock); */
+}
+
+
+static __inline__ void __network_hash_unlink(struct nat25_network_db_entry *ent)
+{
+	/* Caller must _rtw_spinlock_bh already! */
+	/* _rtw_spinlock_bh(&priv->br_ext_lock); */
+
+	*(ent->pprev_hash) = ent->next_hash;
+	if (ent->next_hash != NULL)
+		ent->next_hash->pprev_hash = ent->pprev_hash;
+	ent->next_hash = NULL;
+	ent->pprev_hash = NULL;
+
+	/* _rtw_spinunlock_bh(&priv->br_ext_lock); */
+}
+
+
+static int __nat25_db_network_lookup_and_replace(_adapter *priv,
+		struct sk_buff *skb, unsigned char *networkAddr)
+{
+	struct nat25_network_db_entry *db;
+	_rtw_spinlock_bh(&priv->br_ext_lock);
+
+	db = priv->nethash[__nat25_network_hash(networkAddr)];
+	while (db != NULL) {
+		if (_rtw_memcmp(db->networkAddr, networkAddr, MAX_NETWORK_ADDR_LEN)) {
+			if (!__nat25_has_expired(priv, db)) {
+				/* replace the destination mac address */
+				_rtw_memcpy(skb->data, db->macAddr, ETH_ALEN);
+				atomic_inc(&db->use_count);
+
+#ifdef CL_IPV6_PASS
+				RTW_INFO("NAT25: Lookup M:%02x%02x%02x%02x%02x%02x N:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+					 "%02x%02x%02x%02x%02x%02x\n",
+					 db->macAddr[0],
+					 db->macAddr[1],
+					 db->macAddr[2],
+					 db->macAddr[3],
+					 db->macAddr[4],
+					 db->macAddr[5],
+					 db->networkAddr[0],
+					 db->networkAddr[1],
+					 db->networkAddr[2],
+					 db->networkAddr[3],
+					 db->networkAddr[4],
+					 db->networkAddr[5],
+					 db->networkAddr[6],
+					 db->networkAddr[7],
+					 db->networkAddr[8],
+					 db->networkAddr[9],
+					 db->networkAddr[10],
+					 db->networkAddr[11],
+					 db->networkAddr[12],
+					 db->networkAddr[13],
+					 db->networkAddr[14],
+					 db->networkAddr[15],
+					 db->networkAddr[16]);
+#else
+				RTW_INFO("NAT25: Lookup M:%02x%02x%02x%02x%02x%02x N:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
+					 db->macAddr[0],
+					 db->macAddr[1],
+					 db->macAddr[2],
+					 db->macAddr[3],
+					 db->macAddr[4],
+					 db->macAddr[5],
+					 db->networkAddr[0],
+					 db->networkAddr[1],
+					 db->networkAddr[2],
+					 db->networkAddr[3],
+					 db->networkAddr[4],
+					 db->networkAddr[5],
+					 db->networkAddr[6],
+					 db->networkAddr[7],
+					 db->networkAddr[8],
+					 db->networkAddr[9],
+					 db->networkAddr[10]);
+#endif
+			}
+			_rtw_spinunlock_bh(&priv->br_ext_lock);
+			return 1;
+		}
+
+		db = db->next_hash;
+	}
+
+	_rtw_spinunlock_bh(&priv->br_ext_lock);
+	return 0;
+}
+
+
+static void __nat25_db_network_insert(_adapter *priv,
+		      unsigned char *macAddr, unsigned char *networkAddr)
+{
+	struct nat25_network_db_entry *db;
+	int hash;
+
+	_rtw_spinlock_bh(&priv->br_ext_lock);
+
+	hash = __nat25_network_hash(networkAddr);
+	db = priv->nethash[hash];
+	while (db != NULL) {
+		if (_rtw_memcmp(db->networkAddr, networkAddr, MAX_NETWORK_ADDR_LEN)) {
+			_rtw_memcpy(db->macAddr, macAddr, ETH_ALEN);
+			db->ageing_timer = jiffies;
+			_rtw_spinunlock_bh(&priv->br_ext_lock);
+			return;
+		}
+
+		db = db->next_hash;
+	}
+
+	db = (struct nat25_network_db_entry *) rtw_malloc(sizeof(*db));
+	if (db == NULL) {
+		_rtw_spinunlock_bh(&priv->br_ext_lock);
+		return;
+	}
+
+	_rtw_memcpy(db->networkAddr, networkAddr, MAX_NETWORK_ADDR_LEN);
+	_rtw_memcpy(db->macAddr, macAddr, ETH_ALEN);
+	atomic_set(&db->use_count, 1);
+	db->ageing_timer = jiffies;
+
+	__network_hash_link(priv, db, hash);
+
+	_rtw_spinunlock_bh(&priv->br_ext_lock);
+}
+
+
+static void __nat25_db_print(_adapter *priv)
+{
+
+	_rtw_spinlock_bh(&priv->br_ext_lock);
+
+#ifdef BR_EXT_DEBUG
+	static int counter = 0;
+	int i, j;
+	struct nat25_network_db_entry *db;
+
+	counter++;
+	if ((counter % 16) != 0)
+		return;
+
+	for (i = 0, j = 0; i < NAT25_HASH_SIZE; i++) {
+		db = priv->nethash[i];
+
+		while (db != NULL) {
+#ifdef CL_IPV6_PASS
+			panic_printk("NAT25: DB(%d) H(%02d) C(%d) M:%02x%02x%02x%02x%02x%02x N:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+				     "%02x%02x%02x%02x%02x%02x\n",
+				     j,
+				     i,
+				     atomic_read(&db->use_count),
+				     db->macAddr[0],
+				     db->macAddr[1],
+				     db->macAddr[2],
+				     db->macAddr[3],
+				     db->macAddr[4],
+				     db->macAddr[5],
+				     db->networkAddr[0],
+				     db->networkAddr[1],
+				     db->networkAddr[2],
+				     db->networkAddr[3],
+				     db->networkAddr[4],
+				     db->networkAddr[5],
+				     db->networkAddr[6],
+				     db->networkAddr[7],
+				     db->networkAddr[8],
+				     db->networkAddr[9],
+				     db->networkAddr[10],
+				     db->networkAddr[11],
+				     db->networkAddr[12],
+				     db->networkAddr[13],
+				     db->networkAddr[14],
+				     db->networkAddr[15],
+				     db->networkAddr[16]);
+#else
+			panic_printk("NAT25: DB(%d) H(%02d) C(%d) M:%02x%02x%02x%02x%02x%02x N:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
+				     j,
+				     i,
+				     atomic_read(&db->use_count),
+				     db->macAddr[0],
+				     db->macAddr[1],
+				     db->macAddr[2],
+				     db->macAddr[3],
+				     db->macAddr[4],
+				     db->macAddr[5],
+				     db->networkAddr[0],
+				     db->networkAddr[1],
+				     db->networkAddr[2],
+				     db->networkAddr[3],
+				     db->networkAddr[4],
+				     db->networkAddr[5],
+				     db->networkAddr[6],
+				     db->networkAddr[7],
+				     db->networkAddr[8],
+				     db->networkAddr[9],
+				     db->networkAddr[10]);
+#endif
+			j++;
+
+			db = db->next_hash;
+		}
+	}
+#endif
+
+	_rtw_spinunlock_bh(&priv->br_ext_lock);
+}
+
+
+
+
+/*
+ *	NAT2.5 interface
+ */
+
+void nat25_db_cleanup(_adapter *priv)
+{
+	int i;
+
+	_rtw_spinlock_bh(&priv->br_ext_lock);
+
+	for (i = 0; i < NAT25_HASH_SIZE; i++) {
+		struct nat25_network_db_entry *f;
+		f = priv->nethash[i];
+		while (f != NULL) {
+			struct nat25_network_db_entry *g;
+
+			g = f->next_hash;
+			if (priv->scdb_entry == f) {
+				_rtw_memset(priv->scdb_mac, 0, ETH_ALEN);
+				_rtw_memset(priv->scdb_ip, 0, 4);
+				priv->scdb_entry = NULL;
+			}
+			__network_hash_unlink(f);
+			rtw_mfree((u8 *) f, sizeof(struct nat25_network_db_entry));
+
+			f = g;
+		}
+	}
+
+	_rtw_spinunlock_bh(&priv->br_ext_lock);
+}
+
+
+void nat25_db_expire(_adapter *priv)
+{
+	int i;
+
+	_rtw_spinlock_bh(&priv->br_ext_lock);
+
+	/* if(!priv->ethBrExtInfo.nat25_disable) */
+	{
+		for (i = 0; i < NAT25_HASH_SIZE; i++) {
+			struct nat25_network_db_entry *f;
+			f = priv->nethash[i];
+
+			while (f != NULL) {
+				struct nat25_network_db_entry *g;
+				g = f->next_hash;
+
+				if (__nat25_has_expired(priv, f)) {
+					if (atomic_dec_and_test(&f->use_count)) {
+#ifdef BR_EXT_DEBUG
+#ifdef CL_IPV6_PASS
+						panic_printk("NAT25 Expire H(%02d) M:%02x%02x%02x%02x%02x%02x N:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+							"%02x%02x%02x%02x%02x%02x\n",
+							     i,
+							     f->macAddr[0],
+							     f->macAddr[1],
+							     f->macAddr[2],
+							     f->macAddr[3],
+							     f->macAddr[4],
+							     f->macAddr[5],
+							     f->networkAddr[0],
+							     f->networkAddr[1],
+							     f->networkAddr[2],
+							     f->networkAddr[3],
+							     f->networkAddr[4],
+							     f->networkAddr[5],
+							     f->networkAddr[6],
+							     f->networkAddr[7],
+							     f->networkAddr[8],
+							     f->networkAddr[9],
+							     f->networkAddr[10],
+							     f->networkAddr[11],
+							     f->networkAddr[12],
+							     f->networkAddr[13],
+							     f->networkAddr[14],
+							     f->networkAddr[15],
+							f->networkAddr[16]);
+#else
+
+						panic_printk("NAT25 Expire H(%02d) M:%02x%02x%02x%02x%02x%02x N:%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x\n",
+							     i,
+							     f->macAddr[0],
+							     f->macAddr[1],
+							     f->macAddr[2],
+							     f->macAddr[3],
+							     f->macAddr[4],
+							     f->macAddr[5],
+							     f->networkAddr[0],
+							     f->networkAddr[1],
+							     f->networkAddr[2],
+							     f->networkAddr[3],
+							     f->networkAddr[4],
+							     f->networkAddr[5],
+							     f->networkAddr[6],
+							     f->networkAddr[7],
+							     f->networkAddr[8],
+							     f->networkAddr[9],
+							f->networkAddr[10]);
+#endif
+#endif
+						if (priv->scdb_entry == f) {
+							_rtw_memset(priv->scdb_mac, 0, ETH_ALEN);
+							_rtw_memset(priv->scdb_ip, 0, 4);
+							priv->scdb_entry = NULL;
+						}
+						__network_hash_unlink(f);
+						rtw_mfree((u8 *) f, sizeof(struct nat25_network_db_entry));
+					}
+				}
+
+				f = g;
+			}
+		}
+	}
+
+	_rtw_spinunlock_bh(&priv->br_ext_lock);
+}
+
+
+#ifdef SUPPORT_TX_MCAST2UNI
+static int checkIPMcAndReplace(_adapter *priv, struct sk_buff *skb, unsigned int *dst_ip)
+{
+	struct stat_info	*pstat;
+	struct list_head	*phead, *plist;
+	int i;
+
+	phead = &priv->asoc_list;
+	plist = phead->next;
+
+	while (plist != phead) {
+		pstat = list_entry(plist, struct stat_info, asoc_list);
+		plist = plist->next;
+
+		if (pstat->ipmc_num == 0)
+			continue;
+
+		for (i = 0; i < MAX_IP_MC_ENTRY; i++) {
+			if (pstat->ipmc[i].used && _rtw_memcmp(&pstat->ipmc[i].mcmac[3], ((unsigned char *)dst_ip) + 1, 3)) {
+				_rtw_memcpy(skb->data, pstat->ipmc[i].mcmac, ETH_ALEN);
+				return 1;
+			}
+		}
+	}
+	return 0;
+}
+#endif
+
+int nat25_db_handle(_adapter *priv, struct sk_buff *skb, int method)
+{
+	unsigned short protocol;
+	unsigned char networkAddr[MAX_NETWORK_ADDR_LEN];
+
+	if (skb == NULL)
+		return -1;
+
+	if ((method <= NAT25_MIN) || (method >= NAT25_MAX))
+		return -1;
+
+	protocol = *((unsigned short *)(skb->data + 2 * ETH_ALEN));
+
+	/*---------------------------------------------------*/
+	/*                 Handle IP frame                  */
+	/*---------------------------------------------------*/
+	if (protocol == __constant_htons(ETH_P_IP)) {
+		struct iphdr *iph = (struct iphdr *)(skb->data + ETH_HLEN);
+
+		if (((unsigned char *)(iph) + (iph->ihl << 2)) >= (skb->data + ETH_HLEN + skb->len)) {
+			DEBUG_WARN("NAT25: malformed IP packet !\n");
+			return -1;
+		}
+
+		switch (method) {
+		case NAT25_CHECK:
+			return -1;
+
+		case NAT25_INSERT: {
+			/* some muticast with source IP is all zero, maybe other case is illegal */
+			/* in class A, B, C, host address is all zero or all one is illegal */
+			if (iph->saddr == 0)
+				return 0;
+			RTW_INFO("NAT25: Insert IP, SA=%08x, DA=%08x\n", iph->saddr, iph->daddr);
+			__nat25_generate_ipv4_network_addr(networkAddr, &iph->saddr);
+			/* record source IP address and , source mac address into db */
+			__nat25_db_network_insert(priv, skb->data + ETH_ALEN, networkAddr);
+
+			__nat25_db_print(priv);
+		}
+		return 0;
+
+		case NAT25_LOOKUP: {
+			RTW_INFO("NAT25: Lookup IP, SA=%08x, DA=%08x\n", iph->saddr, iph->daddr);
+#ifdef SUPPORT_TX_MCAST2UNI
+			if (priv->pshare->rf_ft_var.mc2u_disable ||
+			    ((((OPMODE & (WIFI_STATION_STATE | WIFI_ASOC_STATE))
+			       == (WIFI_STATION_STATE | WIFI_ASOC_STATE)) &&
+			      !checkIPMcAndReplace(priv, skb, &iph->daddr)) ||
+			     (OPMODE & WIFI_ADHOC_STATE)))
+#endif
+			{
+				__nat25_generate_ipv4_network_addr(networkAddr, &iph->daddr);
+
+				if (!__nat25_db_network_lookup_and_replace(priv, skb, networkAddr)) {
+					if (*((unsigned char *)&iph->daddr + 3) == 0xff) {
+						/* L2 is unicast but L3 is broadcast, make L2 bacome broadcast */
+						RTW_INFO("NAT25: Set DA as boardcast\n");
+						_rtw_memset(skb->data, 0xff, ETH_ALEN);
+					} else {
+						/* forward unknow IP packet to upper TCP/IP */
+						RTW_INFO("NAT25: Replace DA with BR's MAC\n");
+						if ((*(u32 *)priv->br_mac) == 0 && (*(u16 *)(priv->br_mac + 4)) == 0) {
+							void netdev_br_init(struct net_device *netdev);
+							printk("Re-init netdev_br_init() due to br_mac==0!\n");
+							netdev_br_init(priv->pnetdev);
+						}
+						_rtw_memcpy(skb->data, priv->br_mac, ETH_ALEN);
+					}
+				}
+			}
+		}
+		return 0;
+
+		default:
+			return -1;
+		}
+	}
+
+	/*---------------------------------------------------*/
+	/*                 Handle ARP frame                 */
+	/*---------------------------------------------------*/
+	else if (protocol == __constant_htons(ETH_P_ARP)) {
+		struct arphdr *arp = (struct arphdr *)(skb->data + ETH_HLEN);
+		unsigned char *arp_ptr = (unsigned char *)(arp + 1);
+		unsigned int *sender, *target;
+
+		if (arp->ar_pro != __constant_htons(ETH_P_IP)) {
+			DEBUG_WARN("NAT25: arp protocol unknown (%4x)!\n", htons(arp->ar_pro));
+			return -1;
+		}
+
+		switch (method) {
+		case NAT25_CHECK:
+			return 0;	/* skb_copy for all ARP frame */
+
+		case NAT25_INSERT: {
+			RTW_INFO("NAT25: Insert ARP, MAC=%02x%02x%02x%02x%02x%02x\n", arp_ptr[0],
+				arp_ptr[1], arp_ptr[2], arp_ptr[3], arp_ptr[4], arp_ptr[5]);
+
+			/* change to ARP sender mac address to wlan STA address */
+			_rtw_memcpy(arp_ptr, GET_MY_HWADDR(priv), ETH_ALEN);
+
+			arp_ptr += arp->ar_hln;
+			sender = (unsigned int *)arp_ptr;
+
+			__nat25_generate_ipv4_network_addr(networkAddr, sender);
+
+			__nat25_db_network_insert(priv, skb->data + ETH_ALEN, networkAddr);
+
+			__nat25_db_print(priv);
+		}
+		return 0;
+
+		case NAT25_LOOKUP: {
+			RTW_INFO("NAT25: Lookup ARP\n");
+
+			arp_ptr += arp->ar_hln;
+			sender = (unsigned int *)arp_ptr;
+			arp_ptr += (arp->ar_hln + arp->ar_pln);
+			target = (unsigned int *)arp_ptr;
+
+			__nat25_generate_ipv4_network_addr(networkAddr, target);
+
+			__nat25_db_network_lookup_and_replace(priv, skb, networkAddr);
+
+			/* change to ARP target mac address to Lookup result */
+			arp_ptr = (unsigned char *)(arp + 1);
+			arp_ptr += (arp->ar_hln + arp->ar_pln);
+			_rtw_memcpy(arp_ptr, skb->data, ETH_ALEN);
+		}
+		return 0;
+
+		default:
+			return -1;
+		}
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0))
+	/*---------------------------------------------------*/
+	/*         Handle IPX and Apple Talk frame          */
+	/*---------------------------------------------------*/
+	else if ((protocol == __constant_htons(ETH_P_IPX)) ||
+		 (protocol == __constant_htons(ETH_P_ATALK)) ||
+		 (protocol == __constant_htons(ETH_P_AARP))) {
+		unsigned char ipx_header[2] = {0xFF, 0xFF};
+		struct ipxhdr	*ipx = NULL;
+		struct elapaarp	*ea = NULL;
+		struct ddpehdr	*ddp = NULL;
+		unsigned char *framePtr = skb->data + ETH_HLEN;
+
+		if (protocol == __constant_htons(ETH_P_IPX)) {
+			RTW_INFO("NAT25: Protocol=IPX (Ethernet II)\n");
+			ipx = (struct ipxhdr *)framePtr;
+		} else { /* if(protocol <= __constant_htons(ETH_FRAME_LEN)) */
+			if (_rtw_memcmp(ipx_header, framePtr, 2)) {
+				RTW_INFO("NAT25: Protocol=IPX (Ethernet 802.3)\n");
+				ipx = (struct ipxhdr *)framePtr;
+			} else {
+				unsigned char ipx_8022_type =  0xE0;
+				unsigned char snap_8022_type = 0xAA;
+
+				if (*framePtr == snap_8022_type) {
+					unsigned char ipx_snap_id[5] = {0x0, 0x0, 0x0, 0x81, 0x37};		/* IPX SNAP ID */
+					unsigned char aarp_snap_id[5] = {0x00, 0x00, 0x00, 0x80, 0xF3};	/* Apple Talk AARP SNAP ID */
+					unsigned char ddp_snap_id[5] = {0x08, 0x00, 0x07, 0x80, 0x9B};	/* Apple Talk DDP SNAP ID */
+
+					framePtr += 3;	/* eliminate the 802.2 header */
+
+					if (_rtw_memcmp(ipx_snap_id, framePtr, 5)) {
+						framePtr += 5;	/* eliminate the SNAP header */
+
+						RTW_INFO("NAT25: Protocol=IPX (Ethernet SNAP)\n");
+						ipx = (struct ipxhdr *)framePtr;
+					} else if (_rtw_memcmp(aarp_snap_id, framePtr, 5)) {
+						framePtr += 5;	/* eliminate the SNAP header */
+
+						ea = (struct elapaarp *)framePtr;
+					} else if (_rtw_memcmp(ddp_snap_id, framePtr, 5)) {
+						framePtr += 5;	/* eliminate the SNAP header */
+
+						ddp = (struct ddpehdr *)framePtr;
+					} else {
+						DEBUG_WARN("NAT25: Protocol=Ethernet SNAP %02x%02x%02x%02x%02x\n", framePtr[0],
+							framePtr[1], framePtr[2], framePtr[3], framePtr[4]);
+						return -1;
+					}
+				} else if (*framePtr == ipx_8022_type) {
+					framePtr += 3;	/* eliminate the 802.2 header */
+
+					if (_rtw_memcmp(ipx_header, framePtr, 2)) {
+						RTW_INFO("NAT25: Protocol=IPX (Ethernet 802.2)\n");
+						ipx = (struct ipxhdr *)framePtr;
+					} else
+						return -1;
+				}
+			}
+		}
+
+		/*   IPX  */
+		if (ipx != NULL) {
+			switch (method) {
+			case NAT25_CHECK:
+				if (_rtw_memcmp(skb->data + ETH_ALEN, ipx->ipx_source.node, ETH_ALEN)) {
+					RTW_INFO("NAT25: Check IPX skb_copy\n");
+					return 0;
+				}
+				return -1;
+
+			case NAT25_INSERT: {
+				RTW_INFO("NAT25: Insert IPX, Dest=%08x,%02x%02x%02x%02x%02x%02x,%04x Source=%08x,%02x%02x%02x%02x%02x%02x,%04x\n",
+					 ipx->ipx_dest.net,
+					 ipx->ipx_dest.node[0],
+					 ipx->ipx_dest.node[1],
+					 ipx->ipx_dest.node[2],
+					 ipx->ipx_dest.node[3],
+					 ipx->ipx_dest.node[4],
+					 ipx->ipx_dest.node[5],
+					 ipx->ipx_dest.sock,
+					 ipx->ipx_source.net,
+					 ipx->ipx_source.node[0],
+					 ipx->ipx_source.node[1],
+					 ipx->ipx_source.node[2],
+					 ipx->ipx_source.node[3],
+					 ipx->ipx_source.node[4],
+					 ipx->ipx_source.node[5],
+					 ipx->ipx_source.sock);
+
+				if (_rtw_memcmp(skb->data + ETH_ALEN, ipx->ipx_source.node, ETH_ALEN)) {
+					RTW_INFO("NAT25: Use IPX Net, and Socket as network addr\n");
+
+					__nat25_generate_ipx_network_addr_with_socket(networkAddr, &ipx->ipx_source.net, &ipx->ipx_source.sock);
+
+					/* change IPX source node addr to wlan STA address */
+					_rtw_memcpy(ipx->ipx_source.node, GET_MY_HWADDR(priv), ETH_ALEN);
+				} else
+					__nat25_generate_ipx_network_addr_with_node(networkAddr, &ipx->ipx_source.net, ipx->ipx_source.node);
+
+				__nat25_db_network_insert(priv, skb->data + ETH_ALEN, networkAddr);
+
+				__nat25_db_print(priv);
+			}
+			return 0;
+
+			case NAT25_LOOKUP: {
+				if (_rtw_memcmp(GET_MY_HWADDR(priv), ipx->ipx_dest.node, ETH_ALEN)) {
+					RTW_INFO("NAT25: Lookup IPX, Modify Destination IPX Node addr\n");
+
+					__nat25_generate_ipx_network_addr_with_socket(networkAddr, &ipx->ipx_dest.net, &ipx->ipx_dest.sock);
+
+					__nat25_db_network_lookup_and_replace(priv, skb, networkAddr);
+
+					/* replace IPX destination node addr with Lookup destination MAC addr */
+					_rtw_memcpy(ipx->ipx_dest.node, skb->data, ETH_ALEN);
+				} else {
+					__nat25_generate_ipx_network_addr_with_node(networkAddr, &ipx->ipx_dest.net, ipx->ipx_dest.node);
+
+					__nat25_db_network_lookup_and_replace(priv, skb, networkAddr);
+				}
+			}
+			return 0;
+
+			default:
+				return -1;
+			}
+		}
+
+		/*   AARP  */
+		else if (ea != NULL) {
+			/* Sanity check fields. */
+			if (ea->hw_len != ETH_ALEN || ea->pa_len != AARP_PA_ALEN) {
+				DEBUG_WARN("NAT25: Appletalk AARP Sanity check fail!\n");
+				return -1;
+			}
+
+			switch (method) {
+			case NAT25_CHECK:
+				return 0;
+
+			case NAT25_INSERT: {
+				/* change to AARP source mac address to wlan STA address */
+				_rtw_memcpy(ea->hw_src, GET_MY_HWADDR(priv), ETH_ALEN);
+
+				RTW_INFO("NAT25: Insert AARP, Source=%d,%d Destination=%d,%d\n",
+					 ea->pa_src_net,
+					 ea->pa_src_node,
+					 ea->pa_dst_net,
+					 ea->pa_dst_node);
+
+				__nat25_generate_apple_network_addr(networkAddr, &ea->pa_src_net, &ea->pa_src_node);
+
+				__nat25_db_network_insert(priv, skb->data + ETH_ALEN, networkAddr);
+
+				__nat25_db_print(priv);
+			}
+			return 0;
+
+			case NAT25_LOOKUP: {
+				RTW_INFO("NAT25: Lookup AARP, Source=%d,%d Destination=%d,%d\n",
+					 ea->pa_src_net,
+					 ea->pa_src_node,
+					 ea->pa_dst_net,
+					 ea->pa_dst_node);
+
+				__nat25_generate_apple_network_addr(networkAddr, &ea->pa_dst_net, &ea->pa_dst_node);
+
+				__nat25_db_network_lookup_and_replace(priv, skb, networkAddr);
+
+				/* change to AARP destination mac address to Lookup result */
+				_rtw_memcpy(ea->hw_dst, skb->data, ETH_ALEN);
+			}
+			return 0;
+
+			default:
+				return -1;
+			}
+		}
+
+		/*   DDP  */
+		else if (ddp != NULL) {
+			switch (method) {
+			case NAT25_CHECK:
+				return -1;
+
+			case NAT25_INSERT: {
+				RTW_INFO("NAT25: Insert DDP, Source=%d,%d Destination=%d,%d\n",
+					 ddp->deh_snet,
+					 ddp->deh_snode,
+					 ddp->deh_dnet,
+					 ddp->deh_dnode);
+
+				__nat25_generate_apple_network_addr(networkAddr, &ddp->deh_snet, &ddp->deh_snode);
+
+				__nat25_db_network_insert(priv, skb->data + ETH_ALEN, networkAddr);
+
+				__nat25_db_print(priv);
+			}
+			return 0;
+
+			case NAT25_LOOKUP: {
+				RTW_INFO("NAT25: Lookup DDP, Source=%d,%d Destination=%d,%d\n",
+					 ddp->deh_snet,
+					 ddp->deh_snode,
+					 ddp->deh_dnet,
+					 ddp->deh_dnode);
+
+				__nat25_generate_apple_network_addr(networkAddr, &ddp->deh_dnet, &ddp->deh_dnode);
+
+				__nat25_db_network_lookup_and_replace(priv, skb, networkAddr);
+			}
+			return 0;
+
+			default:
+				return -1;
+			}
+		}
+
+		return -1;
+	}
+#endif
+
+	/*---------------------------------------------------*/
+	/*                Handle PPPoE frame                */
+	/*---------------------------------------------------*/
+	else if ((protocol == __constant_htons(ETH_P_PPP_DISC)) ||
+		 (protocol == __constant_htons(ETH_P_PPP_SES))) {
+		struct pppoe_hdr *ph = (struct pppoe_hdr *)(skb->data + ETH_HLEN);
+		unsigned short *pMagic;
+
+		switch (method) {
+		case NAT25_CHECK:
+			if (ph->sid == 0)
+				return 0;
+			return 1;
+
+		case NAT25_INSERT:
+			if (ph->sid == 0) {	/* Discovery phase according to tag */
+				if (ph->code == PADI_CODE || ph->code == PADR_CODE) {
+					if (priv->ethBrExtInfo.addPPPoETag) {
+						struct pppoe_tag *tag, *pOldTag;
+						unsigned char tag_buf[40];
+						int old_tag_len = 0;
+
+						tag = (struct pppoe_tag *)tag_buf;
+						pOldTag = (struct pppoe_tag *)__nat25_find_pppoe_tag(ph, ntohs(PTT_RELAY_SID));
+						if (pOldTag) { /* if SID existed, copy old value and delete it */
+							old_tag_len = ntohs(pOldTag->tag_len);
+							if (old_tag_len + TAG_HDR_LEN + MAGIC_CODE_LEN + RTL_RELAY_TAG_LEN > sizeof(tag_buf)) {
+								DEBUG_ERR("SID tag length too long!\n");
+								return -1;
+							}
+
+							_rtw_memcpy(tag->tag_data + MAGIC_CODE_LEN + RTL_RELAY_TAG_LEN,
+							       pOldTag->tag_data, old_tag_len);
+
+							if (skb_pull_and_merge(skb, (unsigned char *)pOldTag, TAG_HDR_LEN + old_tag_len) < 0) {
+								DEBUG_ERR("call skb_pull_and_merge() failed in PADI/R packet!\n");
+								return -1;
+							}
+							ph->length = htons(ntohs(ph->length) - TAG_HDR_LEN - old_tag_len);
+						}
+
+						tag->tag_type = PTT_RELAY_SID;
+						tag->tag_len = htons(MAGIC_CODE_LEN + RTL_RELAY_TAG_LEN + old_tag_len);
+
+						/* insert the magic_code+client mac in relay tag */
+						pMagic = (unsigned short *)tag->tag_data;
+						*pMagic = htons(MAGIC_CODE);
+						_rtw_memcpy(tag->tag_data + MAGIC_CODE_LEN, skb->data + ETH_ALEN, ETH_ALEN);
+
+						/* Add relay tag */
+						if (__nat25_add_pppoe_tag(skb, tag) < 0)
+							return -1;
+
+						RTW_INFO("NAT25: Insert PPPoE, forward %s packet\n",
+							(ph->code == PADI_CODE ? "PADI" : "PADR"));
+					} else { /* not add relay tag */
+						if (priv->pppoe_connection_in_progress &&
+						    !_rtw_memcmp(skb->data + ETH_ALEN, priv->pppoe_addr, ETH_ALEN))	 {
+							DEBUG_ERR("Discard PPPoE packet due to another PPPoE connection is in progress!\n");
+							return -2;
+						}
+
+						if (priv->pppoe_connection_in_progress == 0)
+							_rtw_memcpy(priv->pppoe_addr, skb->data + ETH_ALEN, ETH_ALEN);
+
+						priv->pppoe_connection_in_progress = WAIT_TIME_PPPOE;
+					}
+				} else
+					return -1;
+			} else {	/* session phase */
+				RTW_INFO("NAT25: Insert PPPoE, insert session packet to %s\n", skb->dev->name);
+
+				__nat25_generate_pppoe_network_addr(networkAddr, skb->data, &(ph->sid));
+
+				__nat25_db_network_insert(priv, skb->data + ETH_ALEN, networkAddr);
+
+				__nat25_db_print(priv);
+
+				if (!priv->ethBrExtInfo.addPPPoETag &&
+				    priv->pppoe_connection_in_progress &&
+				    _rtw_memcmp(skb->data + ETH_ALEN, priv->pppoe_addr, ETH_ALEN))
+					priv->pppoe_connection_in_progress = 0;
+			}
+			return 0;
+
+		case NAT25_LOOKUP:
+			if (ph->code == PADO_CODE || ph->code == PADS_CODE) {
+				if (priv->ethBrExtInfo.addPPPoETag) {
+					struct pppoe_tag *tag;
+					unsigned char *ptr;
+					unsigned short tagType, tagLen;
+					int offset = 0;
+
+					ptr = __nat25_find_pppoe_tag(ph, ntohs(PTT_RELAY_SID));
+					if (ptr == 0) {
+						DEBUG_ERR("Fail to find PTT_RELAY_SID in FADO!\n");
+						return -1;
+					}
+
+					tag = (struct pppoe_tag *)ptr;
+					tagType = (unsigned short)((ptr[0] << 8) + ptr[1]);
+					tagLen = (unsigned short)((ptr[2] << 8) + ptr[3]);
+
+					if ((tagType != ntohs(PTT_RELAY_SID)) || (tagLen < (MAGIC_CODE_LEN + RTL_RELAY_TAG_LEN))) {
+						DEBUG_ERR("Invalid PTT_RELAY_SID tag length [%d]!\n", tagLen);
+						return -1;
+					}
+
+					pMagic = (unsigned short *)tag->tag_data;
+					if (ntohs(*pMagic) != MAGIC_CODE) {
+						DEBUG_ERR("Can't find MAGIC_CODE in %s packet!\n",
+							(ph->code == PADO_CODE ? "PADO" : "PADS"));
+						return -1;
+					}
+
+					_rtw_memcpy(skb->data, tag->tag_data + MAGIC_CODE_LEN, ETH_ALEN);
+
+					if (tagLen > MAGIC_CODE_LEN + RTL_RELAY_TAG_LEN)
+						offset = TAG_HDR_LEN;
+
+					if (skb_pull_and_merge(skb, ptr + offset, TAG_HDR_LEN + MAGIC_CODE_LEN + RTL_RELAY_TAG_LEN - offset) < 0) {
+						DEBUG_ERR("call skb_pull_and_merge() failed in PADO packet!\n");
+						return -1;
+					}
+					ph->length = htons(ntohs(ph->length) - (TAG_HDR_LEN + MAGIC_CODE_LEN + RTL_RELAY_TAG_LEN - offset));
+					if (offset > 0)
+						tag->tag_len = htons(tagLen - MAGIC_CODE_LEN - RTL_RELAY_TAG_LEN);
+
+					RTW_INFO("NAT25: Lookup PPPoE, forward %s Packet from %s\n",
+						(ph->code == PADO_CODE ? "PADO" : "PADS"),	skb->dev->name);
+				} else { /* not add relay tag */
+					if (!priv->pppoe_connection_in_progress) {
+						DEBUG_ERR("Discard PPPoE packet due to no connection in progresss!\n");
+						return -1;
+					}
+					_rtw_memcpy(skb->data, priv->pppoe_addr, ETH_ALEN);
+					priv->pppoe_connection_in_progress = WAIT_TIME_PPPOE;
+				}
+			} else {
+				if (ph->sid != 0) {
+					RTW_INFO("NAT25: Lookup PPPoE, lookup session packet from %s\n", skb->dev->name);
+					__nat25_generate_pppoe_network_addr(networkAddr, skb->data + ETH_ALEN, &(ph->sid));
+
+					__nat25_db_network_lookup_and_replace(priv, skb, networkAddr);
+
+					__nat25_db_print(priv);
+				} else
+					return -1;
+
+			}
+			return 0;
+
+		default:
+			return -1;
+		}
+	}
+
+	/*---------------------------------------------------*/
+	/*                 Handle EAP frame                 */
+	/*---------------------------------------------------*/
+	else if (protocol == __constant_htons(0x888e)) {
+		switch (method) {
+		case NAT25_CHECK:
+			return -1;
+
+		case NAT25_INSERT:
+			return 0;
+
+		case NAT25_LOOKUP:
+			return 0;
+
+		default:
+			return -1;
+		}
+	}
+
+	/*---------------------------------------------------*/
+	/*         Handle C-Media proprietary frame         */
+	/*---------------------------------------------------*/
+	else if ((protocol == __constant_htons(0xe2ae)) ||
+		 (protocol == __constant_htons(0xe2af))) {
+		switch (method) {
+		case NAT25_CHECK:
+			return -1;
+
+		case NAT25_INSERT:
+			return 0;
+
+		case NAT25_LOOKUP:
+			return 0;
+
+		default:
+			return -1;
+		}
+	}
+
+	/*---------------------------------------------------*/
+	/*         Handle IPV6 frame      							 */
+	/*---------------------------------------------------*/
+#ifdef CL_IPV6_PASS
+	else if (protocol == __constant_htons(ETH_P_IPV6)) {
+		struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + ETH_HLEN);
+
+		if (sizeof(*iph) >= (skb->len - ETH_HLEN)) {
+			DEBUG_WARN("NAT25: malformed IPv6 packet !\n");
+			return -1;
+		}
+
+		switch (method) {
+		case NAT25_CHECK:
+			if (skb->data[0] & 1)
+				return 0;
+			return -1;
+
+		case NAT25_INSERT: {
+			RTW_INFO("NAT25: Insert IP, SA=%4x:%4x:%4x:%4x:%4x:%4x:%4x:%4x,"
+				" DA=%4x:%4x:%4x:%4x:%4x:%4x:%4x:%4x\n",
+				iph->saddr.s6_addr16[0], iph->saddr.s6_addr16[1], iph->saddr.s6_addr16[2], iph->saddr.s6_addr16[3],
+				iph->saddr.s6_addr16[4], iph->saddr.s6_addr16[5], iph->saddr.s6_addr16[6], iph->saddr.s6_addr16[7],
+				iph->daddr.s6_addr16[0], iph->daddr.s6_addr16[1], iph->daddr.s6_addr16[2], iph->daddr.s6_addr16[3],
+				iph->daddr.s6_addr16[4], iph->daddr.s6_addr16[5], iph->daddr.s6_addr16[6], iph->daddr.s6_addr16[7]);
+
+			if (!_rtw_memcmp(&iph->saddr, "\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0", 16)) {
+				__nat25_generate_ipv6_network_addr(networkAddr, (unsigned int *)&iph->saddr);
+				__nat25_db_network_insert(priv, skb->data + ETH_ALEN, networkAddr);
+				__nat25_db_print(priv);
+
+				if (iph->nexthdr == IPPROTO_ICMPV6 &&
+				    skb->len > (ETH_HLEN +  sizeof(*iph) + 4)) {
+					if (update_nd_link_layer_addr(skb->data + ETH_HLEN + sizeof(*iph),
+						skb->len - ETH_HLEN - sizeof(*iph), GET_MY_HWADDR(priv))) {
+						struct icmp6hdr  *hdr = (struct icmp6hdr *)(skb->data + ETH_HLEN + sizeof(*iph));
+						hdr->icmp6_cksum = 0;
+						hdr->icmp6_cksum = csum_ipv6_magic(&iph->saddr, &iph->daddr,
+							iph->payload_len,
+							IPPROTO_ICMPV6,
+							csum_partial((__u8 *)hdr, iph->payload_len, 0));
+					}
+				}
+			}
+		}
+		return 0;
+
+		case NAT25_LOOKUP:
+			RTW_INFO("NAT25: Lookup IP, SA=%4x:%4x:%4x:%4x:%4x:%4x:%4x:%4x,"
+				 " DA=%4x:%4x:%4x:%4x:%4x:%4x:%4x:%4x\n",
+				iph->saddr.s6_addr16[0], iph->saddr.s6_addr16[1], iph->saddr.s6_addr16[2], iph->saddr.s6_addr16[3],
+				iph->saddr.s6_addr16[4], iph->saddr.s6_addr16[5], iph->saddr.s6_addr16[6], iph->saddr.s6_addr16[7],
+				iph->daddr.s6_addr16[0], iph->daddr.s6_addr16[1], iph->daddr.s6_addr16[2], iph->daddr.s6_addr16[3],
+				iph->daddr.s6_addr16[4], iph->daddr.s6_addr16[5], iph->daddr.s6_addr16[6], iph->daddr.s6_addr16[7]);
+
+
+			__nat25_generate_ipv6_network_addr(networkAddr, (unsigned int *)&iph->daddr);
+			if (!__nat25_db_network_lookup_and_replace(priv, skb, networkAddr)) {
+#ifdef SUPPORT_RX_UNI2MCAST
+				if (iph->daddr.s6_addr[0] == 0xff)
+					convert_ipv6_mac_to_mc(skb);
+#endif
+			}
+			return 0;
+
+		default:
+			return -1;
+		}
+	}
+#endif /* CL_IPV6_PASS */
+
+	return -1;
+}
+
+
+int nat25_handle_frame(_adapter *priv, struct sk_buff *skb)
+{
+#ifdef BR_EXT_DEBUG
+	if ((!priv->ethBrExtInfo.nat25_disable) && (!(skb->data[0] & 1))) {
+		panic_printk("NAT25: Input Frame: DA=%02x%02x%02x%02x%02x%02x SA=%02x%02x%02x%02x%02x%02x\n",
+			     skb->data[0],
+			     skb->data[1],
+			     skb->data[2],
+			     skb->data[3],
+			     skb->data[4],
+			     skb->data[5],
+			     skb->data[6],
+			     skb->data[7],
+			     skb->data[8],
+			     skb->data[9],
+			     skb->data[10],
+			     skb->data[11]);
+	}
+#endif
+
+	if (!(skb->data[0] & 1)) {
+		int is_vlan_tag = 0, i, retval = 0;
+		unsigned short vlan_hdr = 0;
+
+		if (*((unsigned short *)(skb->data + ETH_ALEN * 2)) == __constant_htons(ETH_P_8021Q)) {
+			is_vlan_tag = 1;
+			vlan_hdr = *((unsigned short *)(skb->data + ETH_ALEN * 2 + 2));
+			for (i = 0; i < 6; i++)
+				*((unsigned short *)(skb->data + ETH_ALEN * 2 + 2 - i * 2)) = *((unsigned short *)(skb->data + ETH_ALEN * 2 - 2 - i * 2));
+			skb_pull(skb, 4);
+		}
+
+		if (!priv->ethBrExtInfo.nat25_disable) {
+			_rtw_spinlock_bh(&priv->br_ext_lock);
+			/*
+			 *	This function look up the destination network address from
+			 *	the NAT2.5 database. Return value = -1 means that the
+			 *	corresponding network protocol is NOT support.
+			 */
+			if (!priv->ethBrExtInfo.nat25sc_disable &&
+			    (*((unsigned short *)(skb->data + ETH_ALEN * 2)) == __constant_htons(ETH_P_IP)) &&
+			    _rtw_memcmp(priv->scdb_ip, skb->data + ETH_HLEN + 16, 4)) {
+				_rtw_memcpy(skb->data, priv->scdb_mac, ETH_ALEN);
+
+				_rtw_spinunlock_bh(&priv->br_ext_lock);
+			} else {
+				_rtw_spinunlock_bh(&priv->br_ext_lock);
+
+				retval = nat25_db_handle(priv, skb, NAT25_LOOKUP);
+			}
+		} else {
+			if (((*((unsigned short *)(skb->data + ETH_ALEN * 2)) == __constant_htons(ETH_P_IP)) &&
+			     _rtw_memcmp(priv->br_ip, skb->data + ETH_HLEN + 16, 4)) ||
+			    ((*((unsigned short *)(skb->data + ETH_ALEN * 2)) == __constant_htons(ETH_P_ARP)) &&
+			     _rtw_memcmp(priv->br_ip, skb->data + ETH_HLEN + 24, 4))) {
+				/* for traffic to upper TCP/IP */
+				retval = nat25_db_handle(priv, skb, NAT25_LOOKUP);
+			}
+		}
+
+		if (is_vlan_tag) {
+			skb_push(skb, 4);
+			for (i = 0; i < 6; i++)
+				*((unsigned short *)(skb->data + i * 2)) = *((unsigned short *)(skb->data + 4 + i * 2));
+			*((unsigned short *)(skb->data + ETH_ALEN * 2)) = __constant_htons(ETH_P_8021Q);
+			*((unsigned short *)(skb->data + ETH_ALEN * 2 + 2)) = vlan_hdr;
+		}
+
+		if (retval == -1) {
+			/* DEBUG_ERR("NAT25: Lookup fail!\n"); */
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+#if 0
+void mac_clone(_adapter *priv, unsigned char *addr)
+{
+	struct sockaddr sa;
+
+	_rtw_memcpy(sa.sa_data, addr, ETH_ALEN);
+	RTW_INFO("MAC Clone: Addr=%02x%02x%02x%02x%02x%02x\n",
+		 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
+	rtl8192cd_set_hwaddr(priv->dev, &sa);
+}
+
+
+int mac_clone_handle_frame(_adapter *priv, struct sk_buff *skb)
+{
+	if (priv->ethBrExtInfo.macclone_enable && !priv->macclone_completed) {
+		if (!(skb->data[ETH_ALEN] & 1)) {	/* check any other particular MAC add */
+			if (!_rtw_memcmp(skb->data + ETH_ALEN, GET_MY_HWADDR(priv), ETH_ALEN) &&
+			    ((priv->dev->br_port) &&
+			     !_rtw_memcmp(skb->data + ETH_ALEN, priv->br_mac, ETH_ALEN))) {
+				mac_clone(priv, skb->data + ETH_ALEN);
+				priv->macclone_completed = 1;
+			}
+		}
+	}
+
+	return 0;
+}
+#endif /* 0 */
+
+#define SERVER_PORT			67
+#define CLIENT_PORT			68
+#define DHCP_MAGIC			0x63825363
+#define BROADCAST_FLAG		0x8000
+
+struct dhcpMessage {
+	u_int8_t op;
+	u_int8_t htype;
+	u_int8_t hlen;
+	u_int8_t hops;
+	u_int32_t xid;
+	u_int16_t secs;
+	u_int16_t flags;
+	u_int32_t ciaddr;
+	u_int32_t yiaddr;
+	u_int32_t siaddr;
+	u_int32_t giaddr;
+	u_int8_t chaddr[16];
+	u_int8_t sname[64];
+	u_int8_t file[128];
+	u_int32_t cookie;
+	u_int8_t options[308]; /* 312 - cookie */
+};
+
+void dhcp_flag_bcast(_adapter *priv, struct sk_buff *skb)
+{
+	if (skb == NULL)
+		return;
+
+	if (!priv->ethBrExtInfo.dhcp_bcst_disable) {
+		unsigned short protocol = *((unsigned short *)(skb->data + 2 * ETH_ALEN));
+
+		if (protocol == __constant_htons(ETH_P_IP)) { /* IP */
+			struct iphdr *iph = (struct iphdr *)(skb->data + ETH_HLEN);
+
+			if (iph->protocol == IPPROTO_UDP) { /* UDP */
+				struct udphdr *udph = (struct udphdr *)((SIZE_PTR)iph + (iph->ihl << 2));
+
+				if ((udph->source == __constant_htons(CLIENT_PORT))
+				    && (udph->dest == __constant_htons(SERVER_PORT))) { /* DHCP request */
+					struct dhcpMessage *dhcph =
+						(struct dhcpMessage *)((SIZE_PTR)udph + sizeof(struct udphdr));
+
+					if (dhcph->cookie == __constant_htonl(DHCP_MAGIC)) { /* match magic word */
+						if (!(dhcph->flags & htons(BROADCAST_FLAG))) { /* if not broadcast */
+							register int sum = 0;
+
+							RTW_INFO("DHCP: change flag of DHCP request to broadcast.\n");
+							/* or BROADCAST flag */
+							dhcph->flags |= htons(BROADCAST_FLAG);
+							/* recalculate checksum */
+							sum = ~(udph->check) & 0xffff;
+							sum += dhcph->flags;
+							while (sum >> 16)
+								sum = (sum & 0xffff) + (sum >> 16);
+							udph->check = ~sum;
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+
+void *scdb_findEntry(_adapter *priv, unsigned char *macAddr,
+		     unsigned char *ipAddr)
+{
+	unsigned char networkAddr[MAX_NETWORK_ADDR_LEN];
+	struct nat25_network_db_entry *db;
+	int hash;
+
+	/* _rtw_spinlock_bh(&priv->br_ext_lock); */
+
+	__nat25_generate_ipv4_network_addr(networkAddr, (unsigned int *)ipAddr);
+	hash = __nat25_network_hash(networkAddr);
+	db = priv->nethash[hash];
+	while (db != NULL) {
+		if (_rtw_memcmp(db->networkAddr, networkAddr, MAX_NETWORK_ADDR_LEN)) {
+			/* _rtw_spinunlock_bh(&priv->br_ext_lock); */
+			return (void *)db;
+		}
+
+		db = db->next_hash;
+	}
+
+	/* _rtw_spinunlock_bh(&priv->br_ext_lock); */
+	return NULL;
+}
+
+#endif /* CONFIG_BR_EXT */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_btc.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_btc.c
new file mode 100644
index 000000000000..7cb20d975d8c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_btc.c
@@ -0,0 +1,290 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2013 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifdef CONFIG_BTC
+
+#include <drv_types.h>
+
+enum rtw_btc_esoc_type rtw_btc_update_ext_soc_type(u8 esoc_type)
+{
+	enum rtw_btc_esoc_type phl_esoc_type = BTC_EXT_SOC_NONE;
+
+	switch (esoc_type) {
+	case 0:
+		phl_esoc_type = BTC_EXT_SOC_NONE;
+		break;
+	case 1:
+		phl_esoc_type = BTC_EXT_SOC_8761;
+		break;
+	case 2:
+		phl_esoc_type = BTC_EXT_SOC_8771;
+		break;
+	case 3:
+		phl_esoc_type = BTC_EXT_SOC_SILAB_MG21;
+		break;
+	case 4:
+		phl_esoc_type = BTC_EXT_SOC_NORDI_NRF52840;
+		break;
+	default:
+		RTW_WARN("%s: Unsupported esoc_type %d\n", __func__, esoc_type);
+		break;
+	}
+
+	return phl_esoc_type;
+}
+
+#ifdef CONFIG_BTC_TRXSS_CHG
+static u8 _rtw_btc_trxss_chg_hdl_sta(_adapter *padapter, u8 ss_chg_to, u8 hdl_tx, u8 hdl_rx)
+{
+	struct _ADAPTER_LINK *alink = NULL;
+	struct sta_info *psta = NULL;
+	enum wlan_mode wmode = WLAN_MD_INVALID;
+	u8 final_ss = 0;
+
+	psta = rtw_get_stainfo(&padapter->stapriv, get_bssid(&padapter->mlmepriv));
+	if (psta == NULL) {
+		RTW_ERR("%s: psta = NULL!\n", __func__);
+		return _FAIL;
+	}
+
+	alink = GET_PRIMARY_LINK(padapter);
+	wmode = psta->phl_sta->wmode;
+
+	if ((wmode & WLAN_MD_11AX) || (wmode & WLAN_MD_11AC)) {
+		final_ss = rtw_backup_and_get_final_ss(padapter, psta, ss_chg_to);
+		if (wmode & WLAN_MD_11AX)
+			rtw_he_om_ctrl_trx_ss(padapter, alink, psta, final_ss, hdl_tx);
+		else if (wmode & WLAN_MD_11AC)
+			rtw_vht_op_mode_ctrl_rx_nss(padapter, alink, psta, final_ss, hdl_tx);
+	} else if (wmode & WLAN_MD_11N) {
+		if (ss_chg_to)
+			rtw_ssmps_enter(padapter, psta);
+		else
+			rtw_ssmps_leave(padapter, psta);
+	}
+
+	return _SUCCESS;
+}
+
+static u8 _rtw_btc_trxss_chg_hdl_go(_adapter *padapter, u8 ss_chg_to, u8 hdl_tx, u8 hdl_rx)
+{
+	struct sta_priv *stapriv = NULL;
+	struct sta_info *sta = NULL;
+	u32 i, stainfo_offset;
+	_list *plist, *phead;
+	u8 chk_num = 0;
+	u8 chk_list[NUM_STA];
+	struct _ADAPTER_LINK *a_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *mlmepriv = &(a_link->mlmepriv);
+	struct link_mlme_ext_priv *mlmeext = &(a_link->mlmeextpriv);
+	struct link_mlme_ext_info *mlmeinfo = &(mlmeext->mlmext_info);
+	WLAN_BSSID_EX *network = &(mlmeinfo->network);
+	struct HT_caps_element *ht_caps;
+	u8 *p, *ie = network->IEs;
+	u32 len = 0;
+
+	if (!mlmepriv->htpriv.ht_option)
+		return _FAIL;
+
+	if (!mlmeinfo->HT_caps_enable)
+		return _FAIL;
+
+	stapriv = &padapter->stapriv;
+	_rtw_spinlock_bh(&stapriv->asoc_list_lock);
+
+	phead = &stapriv->asoc_list;
+	plist = get_next(phead);
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		sta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+
+		stainfo_offset = rtw_stainfo_offset(stapriv, sta);
+		if (stainfo_offset_valid(stainfo_offset))
+			chk_list[chk_num++] = stainfo_offset;
+		continue;
+	}
+
+	_rtw_spinunlock_bh(&stapriv->asoc_list_lock);
+
+	for (i = 0; i < chk_num; i++) {
+		sta = rtw_get_stainfo_by_offset(stapriv, chk_list[i]);
+		if (ss_chg_to)
+			rtw_ssmps_enter(padapter, sta);
+		else
+			rtw_ssmps_leave(padapter, sta);
+	}
+
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_CAPABILITY_IE_, &len, (network->IELength - _BEACON_IE_OFFSET_));
+	if (p && len > 0) {
+		ht_caps = (struct HT_caps_element *)(p + 2);
+		RTW_INFO("%s: orig ht caps info = 0x%02x\n", __func__, ht_caps->u.HT_cap_element.HT_caps_info);
+
+		if (ss_chg_to)
+			SET_HT_CAP_ELE_SM_PS(&(ht_caps->u.HT_cap_element.HT_caps_info), SM_PS_STATIC);
+		else
+			SET_HT_CAP_ELE_SM_PS(&(ht_caps->u.HT_cap_element.HT_caps_info), SM_PS_DISABLE);
+
+		RTW_INFO("%s: modify ht caps info = 0x%02x\n", __func__, ht_caps->u.HT_cap_element.HT_caps_info);
+		rtw_update_beacon(padapter, a_link, _HT_CAPABILITY_IE_, NULL, _TRUE, RTW_CMDF_DIRECTLY);
+	}
+
+	return _SUCCESS;
+}
+
+static u8 _rtw_btc_trxss_chg_hdl_ap(_adapter *padapter, u8 ss_chg_to, u8 hdl_tx, u8 hdl_rx)
+{
+	/* to be implemented */
+
+	return _SUCCESS;
+}
+
+u8 rtw_btc_trxss_chg_hdl(struct dvobj_priv *dvobj, struct phl_msg *msg, u16 evt_id)
+{
+	_adapter *padapter = dvobj_get_primary_adapter(dvobj);
+	_adapter *iface = NULL;
+	enum rtw_core_btc_cmd_id btc_cmd_id = RTW_CORE_BTC_CMD_MAX;
+	u8 i = 0;
+	u8 ret = _FAIL;
+
+	if (!rtw_hw_is_mimo_support(padapter)) {
+		RTW_WARN("%s: skip since mimo is not supported!\n", __func__);
+		return _FAIL;
+	}
+
+#ifdef CONFIG_DBCC_SUPPORT
+	if (rtw_phl_mr_is_db(dvobj->phl)) {
+		RTW_WARN("%s: skip since dbcc is running!\n", __func__);
+		return _FAIL;
+	}
+#endif
+
+	if (!(GET_DEV_BTC_CAP(GET_PHL_COM(dvobj)).btc_deg_wifi_cap & BTC_DRG_WIFI_CAP_TRX1SS)) {
+		RTW_WARN("%s: btc trxss change isn't enabled!\n", __func__);
+		return _FAIL;
+	}
+
+	switch (evt_id) {
+	case MSG_EVT_ANN_RX1SS:
+		btc_cmd_id = RTW_CORE_BTC_CMD_TRXSS_LMT;
+		break;
+	case MSG_EVT_ANN_RX_MAXSS:
+		btc_cmd_id = RTW_CORE_BTC_CMD_TRXSS_NO_LMT;
+		break;
+	default:
+		RTW_ERR("%s: unsupported evt_id %d!\n", __func__, evt_id);
+		return _FAIL;
+	}
+
+	RTW_INFO("%s: evt_id = %d and macid_map[0] = 0x%08X, macid_map[1] = 0x%08X!\n",
+		 __func__, evt_id, (u32)msg->rsvd[0].value, (u32)msg->rsvd[1].value);
+
+	ret = rtw_core_btc_cmd(padapter, btc_cmd_id, 0);
+
+	return ret;
+}
+#endif /* CONFIG_BTC_TRXSS_CHG */
+
+u8 rtw_core_btc_hdl(_adapter *padapter, enum rtw_core_btc_cmd_id btc_cmd_id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	_adapter *iface = NULL;
+	u8 i = 0;
+	u8 ss_chg_to = 0;
+	u8 ret = _SUCCESS;
+
+	switch (btc_cmd_id) {
+#ifdef CONFIG_BTC_TRXSS_CHG
+	case RTW_CORE_BTC_CMD_TRXSS_LMT:
+	case RTW_CORE_BTC_CMD_TRXSS_NO_LMT:
+		ss_chg_to = (btc_cmd_id == RTW_CORE_BTC_CMD_TRXSS_LMT) ? 1 : 0;
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if (iface == NULL)
+				continue;
+			if (!rtw_is_adapter_up(iface))
+				continue;
+
+			if (is_client_associated_to_ap(iface)) {
+				RTW_INFO("%s: handle "ADPT_FMT" in sta mode!\n", __func__, ADPT_ARG(iface));
+				ret = _rtw_btc_trxss_chg_hdl_sta(iface, ss_chg_to, _TRUE, _TRUE);
+			} else if (MLME_IS_GO(iface)) {
+				RTW_INFO("%s: handle "ADPT_FMT" in P2P-GO mode!\n", __func__, ADPT_ARG(iface));
+				ret = _rtw_btc_trxss_chg_hdl_go(iface, ss_chg_to, _TRUE, _TRUE);
+			} else if (MLME_IS_AP(iface)) {
+				RTW_INFO("%s: handle "ADPT_FMT" in AP mode!\n", __func__, ADPT_ARG(iface));
+				ret = _rtw_btc_trxss_chg_hdl_ap(iface, ss_chg_to, _TRUE, _TRUE);
+			}
+		}
+
+		break;
+#endif
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+u8 rtw_core_btc_cmd(_adapter *padapter, enum rtw_core_btc_cmd_id btc_cmd_id, u8 flags)
+{
+	u8 ret = _SUCCESS;
+	struct	cmd_obj	*pcmdobj;
+	struct	drvextra_cmd_parm *pcmd_parm;
+	struct	cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct submit_ctx sctx;
+
+	pcmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmdobj == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+	pcmdobj->padapter = padapter;
+
+	pcmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pcmd_parm == NULL) {
+		rtw_mfree((u8 *)pcmdobj, sizeof(struct cmd_obj));
+		ret = _FAIL;
+		goto exit;
+	}
+
+	pcmd_parm->ec_id = CORE_BTC_CID;
+	pcmd_parm->type = (int)btc_cmd_id;
+
+	init_h2fwcmd_w_parm_no_rsp(pcmdobj, pcmd_parm, CMD_SET_DRV_EXTRA);
+
+	if (flags & RTW_CMDF_WAIT_ACK) {
+		pcmdobj->sctx = &sctx;
+		rtw_sctx_init(&sctx, 3000);
+	}
+
+	ret = rtw_enqueue_cmd(pcmdpriv, pcmdobj);
+
+	if (ret == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+		rtw_sctx_wait(&sctx, __func__);
+		_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+		if (sctx.status == RTW_SCTX_SUBMITTED)
+			pcmdobj->sctx = NULL;
+		_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+			ret = _FAIL;
+	}
+
+exit:
+	return ret;
+}
+
+#endif /* CONFIG_BTC */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chplan.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chplan.c
new file mode 100644
index 000000000000..620b85efb0d4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chplan.c
@@ -0,0 +1,4809 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_CHPLAN_C_
+
+#include <drv_types.h>
+
+extern struct rtw_regdb_ops regdb_ops;
+
+u8 rtw_chplan_get_default_regd_2g(u8 id)
+{
+	if (regdb_ops.get_default_regd_2g)
+		return regdb_ops.get_default_regd_2g(id);
+	return RTW_REGD_NA;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+u8 rtw_chplan_get_default_regd_5g(u8 id)
+{
+	if (regdb_ops.get_default_regd_5g)
+		return regdb_ops.get_default_regd_5g(id);
+	return RTW_REGD_NA;
+}
+#endif
+
+bool rtw_is_channel_plan_valid(u8 id)
+{
+	if (regdb_ops.is_domain_code_valid)
+		return regdb_ops.is_domain_code_valid(id);
+	return false;
+}
+
+/*
+ * Search the @param ch in chplan by given @param id
+ * @id: the given channel plan id
+ * @ch: the given channel number
+ *
+ * return the index of channel_num in channel_set, -1 if not found
+ */
+static bool rtw_chplan_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	if (regdb_ops.domain_get_ch)
+		return regdb_ops.domain_get_ch(id, ch, flags);
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+u8 rtw_chplan_get_default_regd_6g(u8 id)
+{
+	if (regdb_ops.get_default_regd_6g)
+		return regdb_ops.get_default_regd_6g(id);
+	return RTW_REGD_NA;
+}
+
+bool rtw_is_channel_plan_6g_valid(u8 id)
+{
+	if (regdb_ops.is_domain_code_6g_valid)
+		return regdb_ops.is_domain_code_6g_valid(id);
+	return false;
+}
+
+/*
+ * Search the @param ch in chplan by given @param id
+ * @id: the given channel plan id
+ * @ch: the given channel number
+ *
+ * return the index of channel_num in channel_set, -1 if not found
+ */
+static bool rtw_chplan_6g_get_ch(u8 id, u32 ch, u8 *flags)
+{
+	if (regdb_ops.domain_6g_get_ch)
+		return regdb_ops.domain_6g_get_ch(id, ch, flags);
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+void rtw_rfctl_addl_ch_disable_conf_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty)
+{
+	int i;
+
+	rfctl->dis_ch_flags = regsty->dis_ch_flags
+		#if !CONFIG_DFS
+		| RTW_CHF_DFS
+		#endif
+		;
+
+	for (i = 0; i < MAX_CHANNEL_NUM_2G_5G; i++)
+		rfctl->excl_chs[i] = regsty->excl_chs[i];
+#if CONFIG_IEEE80211_BAND_6GHZ
+	for (i = 0; i < MAX_CHANNEL_NUM_6G; i++)
+		rfctl->excl_chs_6g[i] = regsty->excl_chs_6g[i];
+#endif
+}
+
+static bool rtw_rfctl_is_excl_chs(struct rf_ctl_t *rfctl, u8 ch)
+{
+	int i;
+
+	for (i = 0; i < MAX_CHANNEL_NUM_2G_5G; i++) {
+		if (rfctl->excl_chs[i] == 0)
+			break;
+		if (rfctl->excl_chs[i] == ch)
+			return true;
+	}
+	return false;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static bool rtw_rfctl_is_excl_chs_6g(struct rf_ctl_t *rfctl, u8 ch)
+{
+	int i;
+
+	for (i = 0; i < MAX_CHANNEL_NUM_6G; i++) {
+		if (rfctl->excl_chs_6g[i] == 0)
+			break;
+		if (rfctl->excl_chs_6g[i] == ch)
+			return true;
+	}
+	return false;
+}
+#endif
+
+/*
+ * Check if the @param ch, bw, offset is valid for the given @param id, id_6g
+ * @ch_set: the given channel set
+ * @ch: the given channel number
+ * @bw: the given bandwidth
+ * @offset: the given channel offset
+ *
+ * return valid (1) or not (0)
+ */
+u8 rtw_chplan_is_bchbw_valid(u8 id, u8 id_6g, enum band_type band, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive, struct rf_ctl_t *rfctl)
+{
+	u8 cch;
+	u8 *op_chs;
+	u8 op_ch_num;
+	u8 op_ch;
+	u8 valid = 0;
+	int i;
+	int ch_idx;
+	u8 flags;
+
+	cch = rtw_get_center_ch_by_band(band, ch, bw, offset);
+
+	if (!rtw_get_op_chs_by_bcch_bw(band, cch, bw, &op_chs, &op_ch_num))
+		goto exit;
+
+	for (i = 0; i < op_ch_num; i++) {
+		op_ch = *(op_chs + i);
+		if (0)
+			RTW_INFO("%u,%u,%u - cch:%u, bw:%u, op_ch:%u\n", ch, bw, offset, cch, bw, op_ch);
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		if (band == BAND_ON_6G) {
+			if (!rtw_chplan_6g_get_ch(id_6g, op_ch, &flags)
+				|| (rfctl && (rfctl->dis_ch_flags & flags))
+				|| (rfctl && rtw_rfctl_is_excl_chs_6g(rfctl, op_ch) == _TRUE))
+				break;
+		} else
+		#endif
+		{
+			if (!rtw_chplan_get_ch(id, op_ch, &flags)
+				|| (rfctl && (rfctl->dis_ch_flags & flags))
+				|| (rfctl && rtw_rfctl_is_excl_chs(rfctl, op_ch) == _TRUE))
+				break;
+		}
+		if (flags & RTW_CHF_NO_IR) {
+			if (!allow_passive
+				|| (!allow_primary_passive && op_ch == ch))
+				break;
+		}
+	}
+
+	if (op_ch_num != 0 && i == op_ch_num)
+		valid = 1;
+
+exit:
+	return valid;
+}
+
+const char *_regd_src_str[] = {
+	[REGD_SRC_RTK_PRIV] = "RTK_PRIV",
+	[REGD_SRC_OS] = "OS",
+	[REGD_SRC_NUM] = "UNKNOWN",
+};
+
+static void rtw_chset_apply_from_rtk_priv(struct rtw_chset *chset, u8 chplan, u8 chplan_6g, u8 d_flags)
+{
+	RT_CHANNEL_INFO *chinfo;
+	u8 i;
+	u8 flags;
+	bool chplan_valid = rtw_is_channel_plan_valid(chplan);
+#if CONFIG_IEEE80211_BAND_6GHZ
+	bool chplan_6g_valid = rtw_is_channel_plan_valid(chplan_6g);
+#endif
+	bool usable_ch;
+
+	RTW_INFO("%s chplan:0x%02X chplan_6g:0x%02X\n", __func__, chplan, chplan_6g);
+	rtw_warn_on(!chplan_valid);
+#if CONFIG_IEEE80211_BAND_6GHZ
+	rtw_warn_on(!chplan_6g_valid);
+#endif
+
+	for (i = 0; i < chset->chs_len; i++) {
+		chinfo = &chset->chs[i];
+		if (chinfo->flags & RTW_CHF_DIS)
+			continue;
+
+		if (chinfo->band == BAND_ON_24G
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			|| chinfo->band == BAND_ON_5G
+			#endif
+		) {
+			if (!chplan_valid)
+				continue;
+			usable_ch = rtw_chplan_get_ch(chplan, chinfo->ChannelNum, &flags);
+			if (usable_ch && (flags & d_flags))
+				usable_ch = false;
+		}
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		else if (chinfo->band == BAND_ON_6G) {
+			if (!chplan_6g_valid)
+				continue;
+			usable_ch = rtw_chplan_6g_get_ch(chplan_6g, chinfo->ChannelNum, &flags);
+			if (usable_ch && (flags & d_flags))
+				usable_ch = false;
+		}
+		#endif
+		else
+			usable_ch = false;
+
+		if (usable_ch)
+			chinfo->flags |= flags;
+		else
+			chinfo->flags = RTW_CHF_DIS;
+	}
+}
+
+static void rtw_rfctl_chset_apply_regd_reqs(struct rf_ctl_t *rfctl, u8 d_flags, bool req_lock)
+{
+	struct regd_req_t *req;
+	_list *cur, *head;
+
+	/* apply regd reqs */
+	if (req_lock)
+		_rtw_mutex_lock_interruptible(&rfctl->regd_req_mutex);
+
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+		if (!req->applied)
+			continue;
+
+		if (req->src == REGD_SRC_RTK_PRIV) {
+			u8 domain_code_6g = RTW_CHPLAN_6G_NULL;
+
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			domain_code_6g = req->chplan.domain_code_6g;
+			#endif
+			rtw_chset_apply_from_rtk_priv(&rfctl->chset, req->chplan.domain_code, domain_code_6g, d_flags);
+		}
+		#ifdef CONFIG_REGD_SRC_FROM_OS
+		else if (req->src == REGD_SRC_OS)
+			rtw_chset_apply_from_os(&rfctl->chset, d_flags);
+		#endif
+		else
+			rtw_warn_on(1);
+	}
+
+	if (req_lock)
+		_rtw_mutex_unlock(&rfctl->regd_req_mutex);
+}
+
+static void rtw_rfctl_chset_apply_regulatory(struct dvobj_priv *dvobj, bool req_lock)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	struct rtw_chset *chset = &rfctl->chset;
+	u8 d_band_bmp = 0;
+	u8 d_flags = rfctl->dis_ch_flags;
+	RT_CHANNEL_INFO *chinfo;
+	int i;
+
+	d_band_bmp |= !RFCTL_REG_EN_11A(rfctl) ? BAND_CAP_5G : 0;
+
+	/* reset flags of all channels */
+	for (i = 0; i < chset->chs_len; i++) {
+		chinfo = &chset->chs[i];
+		if ((band_to_band_cap(chinfo->band) & d_band_bmp)
+			|| ((chinfo->band == BAND_ON_24G || chinfo->band == BAND_ON_5G)
+				&& rtw_rfctl_is_excl_chs(rfctl, chinfo->ChannelNum) == true)
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			|| (chinfo->band == BAND_ON_6G
+				&& rtw_rfctl_is_excl_chs_6g(rfctl, chinfo->ChannelNum) == true)
+			#endif
+		)
+			chinfo->flags = RTW_CHF_DIS;
+		else
+			chinfo->flags = 0;
+	}
+
+	rtw_rfctl_chset_apply_regd_reqs(rfctl, d_flags, req_lock);
+
+	chset->enable_ch_num = 0;
+	for (i = 0; i < BAND_MAX; i++)
+		chset->enable_ch_num_of_band[i] = 0;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		chinfo = &chset->chs[i];
+
+		chinfo->reg_no_ir = !!(chinfo->flags & RTW_CHF_NO_IR);
+
+		if (chinfo->flags & RTW_CHF_DIS)
+			continue;
+		chset->enable_ch_num++;
+		chset->enable_ch_num_of_band[chinfo->band]++;
+
+		if (chinfo->flags & RTW_CHF_NO_IR && CH_IS_BCN_HINT(chinfo)
+			&& rtw_rfctl_reg_allow_beacon_hint(rfctl)
+			&& rtw_chinfo_allow_beacon_hint(chinfo))
+			chinfo->flags &= ~RTW_CHF_NO_IR;
+
+		/* logs for channel with NO_IR but can't be cleared through beacon hint */
+		if (chinfo->flags & RTW_CHF_NO_IR) {
+			if (!rtw_rfctl_reg_allow_beacon_hint(rfctl) || !rtw_chinfo_allow_beacon_hint(chinfo))
+				RTW_INFO("band:%s ch%u is NO_IR%s while beacon hint not allowed\n"
+					, band_str(chinfo->band), chinfo->ChannelNum, chinfo->flags & RTW_CHF_DFS ? " DFS" : "");
+		}
+	}
+
+	if (chset->enable_ch_num) {
+		char buf[] = "xx,xx,xx"; /* buf for BAND_ON_24G ~ BAND_ON_6G */
+		size_t buf_len = strlen(buf) + 1;
+		char *pos = buf;
+
+		for (i = 0; i < BAND_MAX; i++)
+			pos += snprintf(pos, buf_len - (pos - buf), "%u,", chset->enable_ch_num_of_band[i]);
+		if (i)
+			*(pos - 1) = '\0';
+
+		RTW_INFO("%s ch num:%d(%s)\n", __func__, chset->enable_ch_num, buf);
+	} else
+		RTW_WARN("%s final chset has no channel\n", __func__);
+}
+
+/* domain status specific beacon hint rules */
+#ifndef RTW_CHPLAN_BEACON_HINT_SPECIFIC_COUNTRY
+#define RTW_CHPLAN_BEACON_HINT_SPECIFIC_COUNTRY 0
+#endif
+
+bool rtw_rfctl_reg_allow_beacon_hint(struct rf_ctl_t *rfctl)
+{
+	return RTW_CHPLAN_BEACON_HINT_SPECIFIC_COUNTRY || RFCTL_REG_WORLDWIDE(rfctl)
+		|| RFCTL_REG_ALPHA2_UNSPEC(rfctl) || RFCTL_REG_INTERSECTED(rfctl);
+}
+
+/* channel specific beacon hint rules */
+#ifndef RTW_CHPLAN_BEACON_HINT_ON_2G_CH_1_11
+#define RTW_CHPLAN_BEACON_HINT_ON_2G_CH_1_11 0
+#endif
+#ifndef RTW_CHPLAN_BEACON_HINT_ON_DFS_CH
+#define RTW_CHPLAN_BEACON_HINT_ON_DFS_CH 0
+#endif
+
+bool rtw_chinfo_allow_beacon_hint(struct _RT_CHANNEL_INFO *chinfo)
+{
+	return (RTW_CHPLAN_BEACON_HINT_ON_2G_CH_1_11 || !(chinfo->band == BAND_ON_24G && chinfo->ChannelNum <= 11))
+		&& (RTW_CHPLAN_BEACON_HINT_ON_DFS_CH || !(chinfo->flags & RTW_CHF_DFS));
+}
+
+u8 rtw_process_beacon_hint(struct rf_ctl_t *rfctl, struct wlan_network *network)
+{
+	u8 act_cnt = 0;
+
+	if (rtw_network_chk_regu_ies(rfctl, network)) {
+		struct rtw_chset *chset = &rfctl->chset;
+		WLAN_BSSID_EX *bss = &network->network;
+		enum band_type band = BSS_EX_OP_BAND(bss);
+		u8 ch = BSS_EX_OP_CH(bss);
+		RT_CHANNEL_INFO *chinfo = rtw_chset_get_chinfo_by_bch(chset, band, ch, false);
+
+		if (!chinfo)
+			goto exit;
+
+		if (rfctl->regd_src == REGD_SRC_RTK_PRIV) {
+			chinfo->bcn_hint_end_time = rtw_get_current_time() + rtw_ms_to_systime(rfctl->bcn_hint_valid_ms);
+			if (chinfo->bcn_hint_end_time == RTW_BCN_HINT_STOPPED)
+				chinfo->bcn_hint_end_time++;
+		}
+
+		if ((chinfo->flags & RTW_CHF_NO_IR)
+			&& rtw_rfctl_reg_allow_beacon_hint(rfctl)
+			&& rtw_chinfo_allow_beacon_hint(chinfo)
+		) {
+			RTW_INFO("%s: change band:%s ch:%d to active\n", __func__, band_str(band), ch);
+			chinfo->flags &= ~RTW_CHF_NO_IR;
+			act_cnt++;
+		}
+	}
+
+exit:
+	return act_cnt;
+}
+
+/* called at cmd handler */
+void rtw_beacon_hint_ch_change_notifier(struct rf_ctl_t *rfctl)
+{
+	_adapter *adapter = dvobj_get_primary_adapter(rfctl_to_dvobj(rfctl));
+	#ifdef CONFIG_IOCTL_CFG80211
+	struct get_chplan_resp *chplan;
+
+	if (rtw_get_chplan_cmd(adapter, RTW_CMDF_DIRECTLY, &chplan) != _SUCCESS
+		|| rtw_regd_change_complete_async(adapter_to_wiphy(adapter), chplan) != _SUCCESS)
+		rtw_warn_on(1);
+	#endif
+
+	op_class_pref_apply_regulatory(rfctl, REG_BEACON_HINT);
+	rtw_nlrtw_reg_beacon_hint_event(adapter);
+}
+
+static void rtw_beacon_hint_expire(struct rf_ctl_t *rfctl)
+{
+	struct dvobj_priv *dvobj;
+	struct rtw_chset *chset;
+	RT_CHANNEL_INFO *chinfo;
+	int i;
+	bool deactivate = false;
+
+	if (rfctl->regd_src != REGD_SRC_RTK_PRIV)
+		return;
+
+	dvobj = rfctl_to_dvobj(rfctl);
+	chset = &rfctl->chset;
+
+	/* don't expire when device is linking/linked/beaconing */
+	for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+		if (HWBAND_STA_LG_NUM(dvobj, i) || HWBAND_STA_LD_NUM(dvobj, i) || HWBAND_WPS_NUM(dvobj, i)
+			|| HWBAND_AP_NUM(dvobj, i) || HWBAND_MESH_NUM(dvobj, i)|| HWBAND_ADHOC_NUM(dvobj, i))
+			break;
+	if (i < HW_BAND_MAX)
+		return;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		chinfo = &chset->chs[i];
+		if (!CH_IS_BCN_HINT(chinfo) && !CH_IS_BCN_HINT_STOPPED(chinfo)) {
+			chinfo->bcn_hint_end_time = RTW_BCN_HINT_STOPPED;
+			if (chinfo->reg_no_ir && !(chinfo->flags & RTW_CHF_NO_IR)) {
+				RTW_INFO("%s: change band:%s ch:%d to passive\n", __func__
+					, band_str(chinfo->band), chinfo->ChannelNum);
+				chinfo->flags |= RTW_CHF_NO_IR;
+				deactivate = true;
+			}
+		}
+	}
+
+	if (deactivate)
+		rtw_beacon_hint_ch_change_notifier(rfctl);
+}
+
+const char *const _regd_inr_str[] = {
+	[RTW_REGD_SET_BY_INIT]			= "INIT",
+	[RTW_REGD_SET_BY_USER]			= "USER",
+	[RTW_REGD_SET_BY_COUNTRY_IE]	= "COUNTRY_IE",
+	[RTW_REGD_SET_BY_DRIVER]		= "DRIVER",
+	[RTW_REGD_SET_BY_CORE]			= "CORE",
+	[RTW_REGD_SET_BY_NUM]			= "UNKNOWN",
+};
+
+const char *const _regd_str[] = {
+	[RTW_REGD_NA]		= "NA",
+	[RTW_REGD_FCC]		= "FCC",
+	[RTW_REGD_MKK]		= "MKK",
+	[RTW_REGD_ETSI]		= "ETSI",
+	[RTW_REGD_IC]		= "IC",
+	[RTW_REGD_KCC]		= "KCC",
+	[RTW_REGD_NCC]		= "NCC",
+	[RTW_REGD_ACMA]		= "ACMA",
+	[RTW_REGD_CHILE]	= "CHILE",
+	[RTW_REGD_MEX]		= "MEX",
+	[RTW_REGD_WW]		= "WW",
+};
+
+const char *const _env_str[] = {
+	[RTW_ENV_ANY]		= "ANY",
+	[RTW_ENV_INDOOR]	= "IN",
+	[RTW_ENV_OUTDOOR]	= "OUT",
+};
+
+const char *const _rtw_edcca_mode_str[] = {
+	[RTW_EDCCA_NORM]	= "NORMAL",
+	[RTW_EDCCA_CS]		= "CS",
+	[RTW_EDCCA_ADAPT]	= "ADAPT",
+	[RTW_EDCCA_CBP]		= "CBP",
+};
+
+const char *const _rtw_dfs_regd_str[] = {
+	[RTW_DFS_REGD_NONE]	= "NONE",
+	[RTW_DFS_REGD_FCC]	= "FCC",
+	[RTW_DFS_REGD_MKK]	= "MKK",
+	[RTW_DFS_REGD_ETSI]	= "ETSI",
+	[RTW_DFS_REGD_KCC]	= "KCC",
+};
+
+const char *const _txpwr_lmt_str[] = {
+	[TXPWR_LMT_NONE]	= "NONE",
+	[TXPWR_LMT_FCC]		= "FCC",
+	[TXPWR_LMT_MKK]		= "MKK",
+	[TXPWR_LMT_ETSI]	= "ETSI",
+	[TXPWR_LMT_IC]		= "IC",
+	[TXPWR_LMT_KCC]		= "KCC",
+	[TXPWR_LMT_NCC]		= "NCC",
+	[TXPWR_LMT_ACMA]	= "ACMA",
+	[TXPWR_LMT_CHILE]	= "CHILE",
+	[TXPWR_LMT_UKRAINE]	= "UKRAINE",
+	[TXPWR_LMT_MEXICO]	= "MEXICO",
+	[TXPWR_LMT_CN]		= "CN",
+	[TXPWR_LMT_QATAR]	= "QATAR",
+	[TXPWR_LMT_UK]		= "UK",
+	[TXPWR_LMT_THAILAND]	= "THAILAND",
+	[TXPWR_LMT_WW]		= "WW",
+	[TXPWR_LMT_NUM]		= NULL,
+};
+
+const REGULATION_TXPWR_LMT _txpwr_lmt_alternate[] = {
+	[TXPWR_LMT_NONE]	= TXPWR_LMT_NONE,
+	[TXPWR_LMT_FCC]		= TXPWR_LMT_FCC,
+	[TXPWR_LMT_MKK]		= TXPWR_LMT_MKK,
+	[TXPWR_LMT_ETSI]	= TXPWR_LMT_ETSI,
+	[TXPWR_LMT_WW]		= TXPWR_LMT_WW,
+	[TXPWR_LMT_NUM]		= TXPWR_LMT_NUM,
+
+	[TXPWR_LMT_IC]		= TXPWR_LMT_FCC,
+	[TXPWR_LMT_KCC]		= TXPWR_LMT_ETSI,
+	[TXPWR_LMT_NCC]		= TXPWR_LMT_FCC,
+	[TXPWR_LMT_ACMA]	= TXPWR_LMT_ETSI,
+	[TXPWR_LMT_CHILE]	= TXPWR_LMT_FCC,
+	[TXPWR_LMT_UKRAINE]	= TXPWR_LMT_ETSI,
+	[TXPWR_LMT_MEXICO]	= TXPWR_LMT_FCC,
+	[TXPWR_LMT_CN]		= TXPWR_LMT_ETSI,
+	[TXPWR_LMT_QATAR]	= TXPWR_LMT_ETSI,
+	[TXPWR_LMT_UK]		= TXPWR_LMT_ETSI,
+	[TXPWR_LMT_THAILAND]	= TXPWR_LMT_ETSI,
+};
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+const char *const _txpwr_lmt_6g_cate_str[] = {
+	[TXPWR_LMT_6G_CATE_VLP]	= "VLP",
+	[TXPWR_LMT_6G_CATE_LPI]	= "LPI",
+	[TXPWR_LMT_6G_CATE_STD]	= "STD",
+	[TXPWR_LMT_6G_CATE_NUM]	= NULL,
+};
+
+#define TXPWR_LMT_6G_CATE_STR_DECLARE(reg) \
+	[TXPWR_LMT_##reg][TXPWR_LMT_6G_CATE_VLP] = #reg"_VLP", \
+	[TXPWR_LMT_##reg][TXPWR_LMT_6G_CATE_LPI] = #reg"_LPI", \
+	[TXPWR_LMT_##reg][TXPWR_LMT_6G_CATE_STD] = #reg"_STD"
+
+const char *const _txpwr_lmt_6g_str[][TXPWR_LMT_6G_CATE_NUM] = {
+	[TXPWR_LMT_NONE][TXPWR_LMT_6G_CATE_VLP]	= "NONE",
+	[TXPWR_LMT_NONE][TXPWR_LMT_6G_CATE_LPI]	= "NONE",
+	[TXPWR_LMT_NONE][TXPWR_LMT_6G_CATE_STD]	= "NONE",
+	[TXPWR_LMT_WW][TXPWR_LMT_6G_CATE_VLP]	= "WW",
+	[TXPWR_LMT_WW][TXPWR_LMT_6G_CATE_LPI]	= "WW",
+	[TXPWR_LMT_WW][TXPWR_LMT_6G_CATE_STD]	= "WW",
+	TXPWR_LMT_6G_CATE_STR_DECLARE(FCC),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(MKK),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(ETSI),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(IC),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(KCC),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(NCC),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(ACMA),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(CHILE),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(UKRAINE),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(MEXICO),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(CN),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(QATAR),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(UK),
+	TXPWR_LMT_6G_CATE_STR_DECLARE(THAILAND),
+};
+
+u8 _rtw_env_to_txpwr_lmt_6g_cate_map[RTW_ENV_NUM] = {
+	[RTW_ENV_ANY]		= BIT(TXPWR_LMT_6G_CATE_VLP),
+	[RTW_ENV_INDOOR]	= BIT(TXPWR_LMT_6G_CATE_VLP) | BIT(TXPWR_LMT_6G_CATE_LPI),
+	[RTW_ENV_OUTDOOR]	= BIT(TXPWR_LMT_6G_CATE_VLP),
+};
+
+#define rtw_env_to_txpwr_lmt_6g_cate_map(env) (((env) >= RTW_ENV_NUM) ? 0 : _rtw_env_to_txpwr_lmt_6g_cate_map[(env)])
+
+u8 _reg_info_to_txpwr_lmt_6g_cate_map[] = {
+	[CIS_6G_REG_IN_AP]	= BIT(TXPWR_LMT_6G_CATE_VLP) | BIT(TXPWR_LMT_6G_CATE_LPI),
+	[CIS_6G_REG_SP_AP]	= BIT(TXPWR_LMT_6G_CATE_VLP) | BIT(TXPWR_LMT_6G_CATE_LPI) | BIT(TXPWR_LMT_6G_CATE_STD),
+	[CIS_6G_REG_VLP_AP]	= BIT(TXPWR_LMT_6G_CATE_VLP),
+	[CIS_6G_REG_IN_EN_AP]	= BIT(TXPWR_LMT_6G_CATE_VLP) | BIT(TXPWR_LMT_6G_CATE_LPI),
+	[CIS_6G_REG_IN_SP_AP]	= BIT(TXPWR_LMT_6G_CATE_VLP) | BIT(TXPWR_LMT_6G_CATE_LPI) | BIT(TXPWR_LMT_6G_CATE_STD),
+};
+
+#define reg_info_to_txpwr_lmt_6g_cate_map(reg_info) (((reg_info) >= CIS_6G_REG_NUM) ? 0 : _reg_info_to_txpwr_lmt_6g_cate_map[(reg_info)])
+
+u8 _reg_info_to_chplan_6g_cate_map[] = {
+	[CIS_6G_REG_IN_AP]	= CHPLAN_6G_CATE_LPI,
+	[CIS_6G_REG_SP_AP]	= CHPLAN_6G_CATE_STD,
+	[CIS_6G_REG_VLP_AP]	= CHPLAN_6G_CATE_VLP,
+	[CIS_6G_REG_IN_EN_AP]	= CHPLAN_6G_CATE_LPI,
+	[CIS_6G_REG_IN_SP_AP]	= CHPLAN_6G_CATE_STD,
+};
+#define reg_info_to_chplan_6g_cate_map(reg_info) (((reg_info) >= CIS_6G_REG_NUM) ? 0 : _reg_info_to_chplan_6g_cate_map[(reg_info)])
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+const enum rtw_edcca_mode_t _rtw_regd_to_edcca_mode[RTW_REGD_NUM] = {
+	[RTW_REGD_NA]	= RTW_EDCCA_MODE_NUM,
+	[RTW_REGD_MKK]	= RTW_EDCCA_CS,
+	[RTW_REGD_ETSI]	= RTW_EDCCA_ADAPT,
+	[RTW_REGD_WW]	= RTW_EDCCA_ADAPT,
+};
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+const enum rtw_edcca_mode_t _rtw_regd_to_edcca_mode_6g[RTW_REGD_NUM] = {
+	[RTW_REGD_NA]	= RTW_EDCCA_MODE_NUM,
+	[RTW_REGD_FCC]	= RTW_EDCCA_CBP,
+	[RTW_REGD_MKK]	= RTW_EDCCA_CS,
+	[RTW_REGD_ETSI]	= RTW_EDCCA_ADAPT,
+	[RTW_REGD_WW]	= RTW_EDCCA_CBP,
+};
+#endif
+
+const REGULATION_TXPWR_LMT _rtw_regd_to_txpwr_lmt[] = {
+	[RTW_REGD_NA]		= TXPWR_LMT_NUM,
+	[RTW_REGD_FCC]		= TXPWR_LMT_FCC,
+	[RTW_REGD_MKK]		= TXPWR_LMT_MKK,
+	[RTW_REGD_ETSI]		= TXPWR_LMT_ETSI,
+	[RTW_REGD_IC]		= TXPWR_LMT_IC,
+	[RTW_REGD_KCC]		= TXPWR_LMT_KCC,
+	[RTW_REGD_NCC]		= TXPWR_LMT_NCC,
+	[RTW_REGD_ACMA]		= TXPWR_LMT_ACMA,
+	[RTW_REGD_CHILE]	= TXPWR_LMT_CHILE,
+	[RTW_REGD_MEX]		= TXPWR_LMT_MEXICO,
+	[RTW_REGD_WW]		= TXPWR_LMT_WW,
+};
+
+char *rtw_get_regd_inr_bmp_str(char *buf, u8 bmp)
+{
+	char *pos = buf;
+	int i;
+
+	for (i = 0; i < RTW_REGD_SET_BY_NUM; i++) {
+		if (!(bmp & BIT(i)))
+			continue;
+		pos += snprintf(pos, REGD_INR_BMP_STR_LEN - (pos - buf), "%s%s"
+			, pos == buf ? "" : " ", regd_inr_str(i));
+		if (pos >= buf + REGD_INR_BMP_STR_LEN - 1)
+			goto exit;
+	}
+	if (pos == buf)
+		buf[0] = '\0';
+
+exit:
+	return buf;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+char *rtw_get_env_bmp_str(char *buf, u8 bmp)
+{
+	char *pos = buf;
+	int i;
+
+	for (i = 0; i < RTW_ENV_NUM; i++) {
+		if (!(bmp & BIT(i)))
+			continue;
+		pos += snprintf(pos, ENV_BMP_STR_LEN - (pos - buf), "%s%s"
+			, pos == buf ? "" : " ", env_str(i));
+		if (pos >= buf + ENV_BMP_STR_LEN - 1)
+			goto exit;
+	}
+	if (pos == buf)
+		buf[0] = '\0';
+
+exit:
+	return buf;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+char *rtw_get_edcca_mode_of_bands_str(char *buf, u8 mode_of_band[])
+{
+#define EDCCA_MODE_SEQ_COMPARE(result, operand) (result == RTW_EDCCA_MODE_NUM ? operand : (operand == RTW_EDCCA_MODE_NUM ? result : (result != operand ? -1 : result)))
+
+	int mode = RTW_EDCCA_MODE_NUM;
+	char *pos = buf;
+
+	mode = EDCCA_MODE_SEQ_COMPARE(mode, mode_of_band[BAND_ON_24G]);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	mode = EDCCA_MODE_SEQ_COMPARE(mode, mode_of_band[BAND_ON_5G]);
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	mode = EDCCA_MODE_SEQ_COMPARE(mode, mode_of_band[BAND_ON_6G]);
+#endif
+
+	if (mode != -1) { /* all available values are the same */
+		pos += snprintf(pos, EDCCA_MODE_OF_BANDS_STR_LEN - (pos - buf), "%s(%u)", rtw_edcca_mode_str(mode), mode);
+		if (pos >= buf + EDCCA_MODE_OF_BANDS_STR_LEN - 1)
+			goto exit;
+	} else {
+		pos += snprintf(pos, EDCCA_MODE_OF_BANDS_STR_LEN - (pos - buf), "%s(%u)", rtw_edcca_mode_str(mode_of_band[BAND_ON_24G]), mode_of_band[BAND_ON_24G]);
+		if (pos >= buf + EDCCA_MODE_OF_BANDS_STR_LEN - 1)
+			goto exit;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		pos += snprintf(pos, EDCCA_MODE_OF_BANDS_STR_LEN - (pos - buf), " %s(%u)", rtw_edcca_mode_str(mode_of_band[BAND_ON_5G]), mode_of_band[BAND_ON_5G]);
+		if (pos >= buf + EDCCA_MODE_OF_BANDS_STR_LEN - 1)
+			goto exit;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		pos += snprintf(pos, EDCCA_MODE_OF_BANDS_STR_LEN - (pos - buf), " %s(%u)", rtw_edcca_mode_str(mode_of_band[BAND_ON_6G]), mode_of_band[BAND_ON_6G]);
+		if (pos >= buf + EDCCA_MODE_OF_BANDS_STR_LEN - 1)
+			goto exit;
+		#endif
+	}
+
+exit:
+	return buf;
+}
+
+static enum rtw_edcca_mode_t rtw_edcca_mode_get_strictest(enum rtw_edcca_mode_t a, enum rtw_edcca_mode_t b)
+{
+	if (a >= RTW_EDCCA_MODE_NUM)
+		return b < RTW_EDCCA_MODE_NUM ? b : RTW_EDCCA_MODE_NUM;
+	if (b >= RTW_EDCCA_MODE_NUM)
+		return a;
+	return rtw_max(a,b);
+}
+
+static void rtw_edcca_mode_update_by_regd_reqs(struct dvobj_priv *dvobj, bool req_lock)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	struct regd_req_t *req;
+	struct country_chplan *chplan;
+	_list *cur, *head;
+	u8 mode[BAND_MAX];
+	u8 band, tmp_mode;
+	char buf[EDCCA_MODE_OF_BANDS_STR_LEN];
+
+	for (band = 0; band < BAND_MAX; band++)
+		mode[band] = RTW_EDCCA_MODE_NUM;
+
+	if (req_lock)
+		_rtw_mutex_lock_interruptible(&rfctl->regd_req_mutex);
+
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		chplan = &req->chplan;
+		cur = get_next(cur);
+		if (!req->applied)
+			continue;
+
+		tmp_mode = COUNTRY_CHPLAN_EDCCA_2G_OVERRIDE(chplan) != RTW_EDCCA_DEF ? COUNTRY_CHPLAN_EDCCA_2G_OVERRIDE(chplan) :
+			rtw_regd_to_edcca_mode(rtw_chplan_get_default_regd_2g(chplan->domain_code));
+		mode[BAND_ON_24G] = rtw_edcca_mode_get_strictest(mode[BAND_ON_24G], tmp_mode);
+
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		tmp_mode = COUNTRY_CHPLAN_EDCCA_5G_OVERRIDE(chplan) != RTW_EDCCA_DEF ? COUNTRY_CHPLAN_EDCCA_5G_OVERRIDE(chplan) :
+			rtw_regd_to_edcca_mode(rtw_chplan_get_default_regd_5g(chplan->domain_code));
+		mode[BAND_ON_5G] = rtw_edcca_mode_get_strictest(mode[BAND_ON_5G], tmp_mode);
+		#endif
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		tmp_mode = COUNTRY_CHPLAN_EDCCA_6G_OVERRIDE(chplan) != RTW_EDCCA_DEF ? COUNTRY_CHPLAN_EDCCA_6G_OVERRIDE(chplan) :
+			rtw_regd_to_edcca_mode_6g(rtw_chplan_get_default_regd_6g(chplan->domain_code_6g));
+		mode[BAND_ON_6G] = rtw_edcca_mode_get_strictest(mode[BAND_ON_6G], tmp_mode);
+		#endif
+	}
+
+	if (req_lock)
+		_rtw_mutex_unlock(&rfctl->regd_req_mutex);
+
+	rfctl->edcca_mode_2g = mode[BAND_ON_24G];
+	#if CONFIG_IEEE80211_BAND_5GHZ
+	rfctl->edcca_mode_5g = mode[BAND_ON_5G];
+	#endif
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	rfctl->edcca_mode_6g = mode[BAND_ON_6G];
+	#endif
+
+	RTW_PRINT("update edcca_mode:%s\n"
+		, rtw_get_edcca_mode_of_bands_str(buf, mode)
+	);
+}
+
+void rtw_edcca_mode_update(struct dvobj_priv *dvobj, bool req_lock)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+
+	if (rfctl->edcca_mode_sel == RTW_EDCCA_NORM) {
+		/* force disable */
+		rfctl->edcca_mode_2g = RTW_EDCCA_NORM;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		rfctl->edcca_mode_5g = RTW_EDCCA_NORM;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		rfctl->edcca_mode_6g = RTW_EDCCA_NORM;
+		#endif
+
+	} else if (rfctl->edcca_mode_sel == RTW_EDCCA_CS) {
+		/* carrier sense */
+		rfctl->edcca_mode_2g = RTW_EDCCA_CS;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		rfctl->edcca_mode_5g = RTW_EDCCA_CS;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		rfctl->edcca_mode_6g = RTW_EDCCA_CS;
+		#endif
+
+	} else if (rfctl->edcca_mode_sel == RTW_EDCCA_ADAPT) {
+		/* adaptivity */
+		rfctl->edcca_mode_2g = RTW_EDCCA_ADAPT;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		rfctl->edcca_mode_5g = RTW_EDCCA_ADAPT;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		rfctl->edcca_mode_6g = RTW_EDCCA_ADAPT;
+		#endif
+
+	} else if (rfctl->edcca_mode_sel == RTW_EDCCA_CBP) {
+		/* adaptivity */
+		rfctl->edcca_mode_2g = RTW_EDCCA_NORM;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		rfctl->edcca_mode_5g = RTW_EDCCA_NORM;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		rfctl->edcca_mode_6g = RTW_EDCCA_CBP;
+		#endif
+
+	} else {
+		/* by regulatory setting */
+		rtw_edcca_mode_update_by_regd_reqs(dvobj, req_lock);
+	}
+
+	rtw_edcca_hal_update(dvobj);
+}
+
+u8 rtw_get_edcca_mode(struct dvobj_priv *dvobj, enum band_type band)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	u8 edcca_mode = RTW_EDCCA_NORM;
+
+	if (band == BAND_ON_24G)
+		edcca_mode = rfctl->edcca_mode_2g;
+	#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G)
+		edcca_mode = rfctl->edcca_mode_5g;
+	#endif
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G)
+		edcca_mode = rfctl->edcca_mode_6g;
+	#endif
+
+	return edcca_mode;
+}
+
+void rtw_edcca_mode_sel_msg(void *sel, struct dvobj_priv *dvobj)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+
+	RTW_PRINT_SEL(sel, "RTW_EDCCA_%s\n", rtw_edcca_mode_sel_str(rfctl->edcca_mode_sel));
+}
+
+#if CONFIG_TXPWR_LIMIT
+char *rtw_get_txpwr_lmt_name_of_bands_str(char *buf, const char *name_of_band[], u8 unknown_bmp)
+{
+#define NAME_DIFF ((void *)1)
+/* input comes form organized database, string with same content will not have different pointer */
+#define NAME_SEQ_COMPARE(result, operand) ( \
+	(result) == NULL ? (operand) : \
+	(operand) == NULL ? (result) : \
+	(result) != (operand) ? NAME_DIFF : (result) \
+	)
+
+#define BOOL_TO_S8(b) ((b) ? 1 : 0)
+#define BOOL_S_NONE (-1)
+#define BOOL_S_DIFF (-2)
+#define BOOL_S_SEQ_COMPARE(result, operand) ( \
+	(result) == BOOL_S_NONE ? BOOL_TO_S8(operand) : \
+	BOOL_TO_S8(operand) == BOOL_S_NONE ? (result) : \
+	(result) != BOOL_TO_S8(operand) ? BOOL_S_DIFF : (result) \
+	)
+
+	const char *name = NULL;
+	s8 unknown = BOOL_S_NONE;
+	char *pos = buf;
+
+	name = NAME_SEQ_COMPARE(name, name_of_band[BAND_ON_24G]);
+	unknown = BOOL_S_SEQ_COMPARE(unknown, !!(unknown_bmp & BIT(BAND_ON_24G)));
+#if CONFIG_IEEE80211_BAND_5GHZ
+	name = NAME_SEQ_COMPARE(name, name_of_band[BAND_ON_5G]);
+	unknown = BOOL_S_SEQ_COMPARE(unknown, !!(unknown_bmp & BIT(BAND_ON_5G)));
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	name = NAME_SEQ_COMPARE(name, name_of_band[BAND_ON_6G]);
+	unknown = BOOL_S_SEQ_COMPARE(unknown, !!(unknown_bmp & BIT(BAND_ON_6G)));
+#endif
+
+	if (name != NAME_DIFF && unknown != BOOL_S_DIFF) { /* all available values are the same */
+		pos += snprintf(pos, TXPWR_NAME_OF_BANDS_STR_LEN - (pos - buf), "%s%s", (unknown_bmp & BIT(BAND_ON_24G)) ? "?" : "", name);
+		if (pos >= buf + TXPWR_NAME_OF_BANDS_STR_LEN - 1)
+			goto exit;
+	} else {
+		pos += snprintf(pos, TXPWR_NAME_OF_BANDS_STR_LEN - (pos - buf), "%s%s", (unknown_bmp & BIT(BAND_ON_24G)) ? "?" : "", name_of_band[BAND_ON_24G]);
+		if (pos >= buf + TXPWR_NAME_OF_BANDS_STR_LEN - 1)
+			goto exit;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		pos += snprintf(pos, TXPWR_NAME_OF_BANDS_STR_LEN - (pos - buf), " %s%s", (unknown_bmp & BIT(BAND_ON_5G)) ? "?" : "", name_of_band[BAND_ON_5G]);
+		if (pos >= buf + TXPWR_NAME_OF_BANDS_STR_LEN - 1)
+			goto exit;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		pos += snprintf(pos, TXPWR_NAME_OF_BANDS_STR_LEN - (pos - buf), " %s%s", (unknown_bmp & BIT(BAND_ON_6G)) ? "?" : "", name_of_band[BAND_ON_6G]);
+		if (pos >= buf + TXPWR_NAME_OF_BANDS_STR_LEN - 1)
+			goto exit;
+		#endif
+	}
+
+exit:
+	return buf;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static u8 regd_req_get_6g_cate_map(struct regd_req_t *req)
+{
+	return req->txpwr_lmt_6g_cate_map;
+}
+#endif
+
+static void rtw_txpwr_apply_regd_req_reg_exc(struct rf_ctl_t *rfctl, struct regd_req_t *req
+	, char *req_6g_name_buf, u8 req_6g_name_bsz
+	, const char *name_of_band[], u8 *unknown_bmp)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct country_chplan *chplan = &req->chplan;
+	enum txpwr_lmt_reg_exc_match exc = TXPWR_LMT_REG_EXC_MATCH_NONE;
+	const char *name = NULL;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	enum txpwr_lmt_reg_exc_match exc_6g = TXPWR_LMT_REG_EXC_MATCH_NONE;
+	const char *name_6g = NULL;
+#endif
+	u8 band;
+
+	if (*unknown_bmp & (BIT(BAND_ON_24G) | BIT(BAND_ON_5G)))
+		exc = rtw_txpwr_hal_lmt_reg_exc_search(dvobj, chplan->alpha2, chplan->domain_code, &name);
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (*unknown_bmp & BIT(BAND_ON_6G))
+		exc_6g = rtw_txpwr_hal_lmt_reg_exc_6g_search(dvobj, chplan->alpha2, chplan->domain_code_6g, &name_6g);
+#endif
+
+	if (exc
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		|| exc_6g
+		#endif
+	) {
+		char exc_msg[] = " country:XX domain:0xGG"
+				#if CONFIG_IEEE80211_BAND_6GHZ
+				" domain_6g:0xGG"
+				#endif
+				"\0";
+		u8 exc_msg_len = sizeof(exc_msg);
+		char *msg_p = exc_msg;
+		char buf[TXPWR_NAME_OF_BANDS_STR_LEN];
+
+		for (band = 0; band < BAND_MAX; band++) {
+			if (!(*unknown_bmp & BIT(band)))
+				continue;
+
+			if ((band == BAND_ON_24G || band == BAND_ON_5G) && exc) {
+				name_of_band[band] = name;
+				if (strcmp(name, txpwr_lmt_str(TXPWR_LMT_NONE)) == 0
+					|| strcmp(name, txpwr_lmt_str(TXPWR_LMT_WW)) == 0
+					|| rtw_txpwr_hal_lmt_reg_search(dvobj, band, name))
+					*unknown_bmp &= ~BIT(band);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			} else if (band == BAND_ON_6G && exc_6g) {
+				if (strcmp(name_6g, txpwr_lmt_str(TXPWR_LMT_NONE)) == 0
+					|| strcmp(name_6g, txpwr_lmt_str(TXPWR_LMT_WW)) == 0
+				) {
+					/* known limits without category */
+					name_of_band[band] = name_6g;
+					*unknown_bmp &= ~BIT(band);
+				} else {
+					u8 cate_map = regd_req_get_6g_cate_map(req);
+					int i;
+
+					name_of_band[band] = req_6g_name_buf;
+					for (i = TXPWR_LMT_6G_CATE_NUM - 1; i >= TXPWR_LMT_6G_CATE_VLP ; i--) {
+						if (!(cate_map & BIT(i)))
+							continue;
+						snprintf(req_6g_name_buf, req_6g_name_bsz, "%s_%s", name_6g, txpwr_lmt_6g_cate_str(i));
+						if (rtw_txpwr_hal_lmt_reg_search(dvobj, band, req_6g_name_buf)) {
+							*unknown_bmp &= ~BIT(band);
+							break;
+						}
+					}
+				}
+			#endif
+			}
+		}
+
+		if (exc == TXPWR_LMT_REG_EXC_MATCH_COUNTRY
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			|| exc_6g == TXPWR_LMT_REG_EXC_MATCH_COUNTRY
+			#endif
+		) {
+			msg_p += snprintf(msg_p, exc_msg_len - (msg_p - exc_msg), " country:"ALPHA2_FMT, ALPHA2_ARG(chplan->alpha2));
+			if (msg_p >= exc_msg + exc_msg_len - 1)
+				goto msg_dump;
+		}
+		if (exc == TXPWR_LMT_REG_EXC_MATCH_DOMAIN) {
+			msg_p += snprintf(msg_p, exc_msg_len - (msg_p - exc_msg), " domain:0x%02x", chplan->domain_code);
+			if (msg_p >= exc_msg + exc_msg_len - 1)
+				goto msg_dump;
+		}
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		if (exc_6g == TXPWR_LMT_REG_EXC_MATCH_DOMAIN) {
+			msg_p += snprintf(msg_p, exc_msg_len - (msg_p - exc_msg), " domain_6g:0x%02x", chplan->domain_code_6g);
+			if (msg_p >= exc_msg + exc_msg_len - 1)
+				goto msg_dump;
+		}
+		#endif
+
+msg_dump:
+		RTW_PRINT("exception%s applied, txpwr_lmt:%s\n", exc_msg
+			, rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+	}
+}
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+static void rtw_txpwr_apply_regd_req_from_os(struct rf_ctl_t *rfctl, struct regd_req_t *req
+	, const char *req_alpha2_str, char *req_6g_name_buf, u8 req_6g_name_bsz
+	, const char *name_of_band[], u8 *unknown_bmp)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct country_chplan *chplan = &req->chplan;
+
+	if (rfctl->regd_src == REGD_SRC_OS) {
+		bool alpha2_is_ww = IS_ALPHA2_WORLDWIDE(req_alpha2_str);
+		const char *name = NULL;
+		char buf[TXPWR_NAME_OF_BANDS_STR_LEN];
+		u8 band;
+
+		name = alpha2_is_ww ? txpwr_lmt_str(TXPWR_LMT_WW) : req_alpha2_str;
+
+		for (band = 0; band < BAND_MAX; band++) {
+			if (!(*unknown_bmp & BIT(band)))
+				continue;
+
+			if (alpha2_is_ww) {
+				name_of_band[band] = name;
+				*unknown_bmp &= ~BIT(band);
+				continue;
+			}
+
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (band == BAND_ON_6G) {
+				u8 cate_map = regd_req_get_6g_cate_map(req);
+				int i;
+
+				name_of_band[band] = req_6g_name_buf;
+				for (i = TXPWR_LMT_6G_CATE_NUM - 1; i >= TXPWR_LMT_6G_CATE_VLP ; i--) {
+					if (!(cate_map & BIT(i)))
+						continue;
+					snprintf(req_6g_name_buf, req_6g_name_bsz, "%s_%s", name, txpwr_lmt_6g_cate_str(i));
+					if (rtw_txpwr_hal_lmt_reg_search(dvobj, band, req_6g_name_buf)) {
+						*unknown_bmp &= ~BIT(band);
+						break;
+					}
+				}
+			} else
+			#endif
+			{
+				name_of_band[band] = name;
+				if (rtw_txpwr_hal_lmt_reg_search(dvobj, band, name_of_band[band]))
+					*unknown_bmp &= ~BIT(band);
+			}
+		}
+
+		RTW_PRINT("os country:"ALPHA2_FMT" applied, txpwr_lmt:%s\n"
+			, ALPHA2_ARG(req_alpha2_str), rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+
+		if (*unknown_bmp && chplan->domain_code == RTW_CHPLAN_UNSPECIFIED) {
+			for (band = 0; band < BAND_MAX; band++) {
+				if (!(*unknown_bmp & BIT(band)))
+					continue;
+				name_of_band[band] = txpwr_lmt_str(TXPWR_LMT_WW);
+				*unknown_bmp &= ~BIT(band);
+			}
+
+			RTW_PRINT("unsupported os country:"ALPHA2_FMT" applied, txpwr_lmt:%s\n"
+				, ALPHA2_ARG(req_alpha2_str), rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+		}
+	}
+}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static bool rtw_txpwr_search_predef_6g_regd(struct rf_ctl_t *rfctl, struct regd_req_t *req
+	, u8 txpwr_lmt, const char **name)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	u8 cate_map = regd_req_get_6g_cate_map(req);
+	int i;
+
+	for (i = TXPWR_LMT_6G_CATE_NUM - 1; i >= TXPWR_LMT_6G_CATE_VLP ; i--) {
+		if (!(cate_map & BIT(i)))
+			continue;
+		*name = txpwr_lmt_6g_str(txpwr_lmt, i);
+		if (!*name)
+			continue;
+		if (rtw_txpwr_hal_lmt_reg_search(dvobj, BAND_ON_6G, *name))
+			return true;
+	}
+
+	return false;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static void rtw_txpwr_apply_predefined(struct rf_ctl_t *rfctl, struct regd_req_t *req, enum band_type band, u8 txpwr_lmt
+	, const char *name_of_band[], u8 *unknown_bmp)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+
+	if (txpwr_lmt == TXPWR_LMT_NONE || txpwr_lmt == TXPWR_LMT_WW || txpwr_lmt == TXPWR_LMT_NUM) {
+		name_of_band[band] = txpwr_lmt_str(txpwr_lmt);
+		*unknown_bmp &= ~BIT(band);
+	} else if (band == BAND_ON_24G || band == BAND_ON_5G) {
+		name_of_band[band] = txpwr_lmt_str(txpwr_lmt);
+		if (name_of_band[band] && rtw_txpwr_hal_lmt_reg_search(dvobj, band, name_of_band[band]))
+			*unknown_bmp &= ~BIT(band);
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	} else if (band == BAND_ON_6G) {
+		if (rtw_txpwr_search_predef_6g_regd(rfctl, req, txpwr_lmt, &name_of_band[band]))
+			*unknown_bmp &= ~BIT(band);
+	#endif
+	}
+}
+
+static void rtw_txpwr_apply_regd_req_default(struct rf_ctl_t *rfctl, struct regd_req_t *req
+	, const char *name_of_band[], u8 *unknown_bmp)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct country_chplan *chplan = &req->chplan;
+	u8 txpwr_lmt[BAND_MAX];
+	char buf[TXPWR_NAME_OF_BANDS_STR_LEN];
+	u8 band;
+	bool altenate_applied = 0;
+
+	for (band = 0; band < BAND_MAX; band++)
+		txpwr_lmt[band] = TXPWR_LMT_NONE;
+
+	if (chplan->txpwr_lmt_override != TXPWR_LMT_DEF) {
+		for (band = 0; band < BAND_MAX; band++) {
+			if (!(*unknown_bmp & BIT(band)))
+				continue;
+			txpwr_lmt[band] = chplan->txpwr_lmt_override;
+			rtw_txpwr_apply_predefined(rfctl, req, band, txpwr_lmt[band], name_of_band, unknown_bmp);
+		}
+		RTW_PRINT("default country:"ALPHA2_FMT" applied, txpwr_lmt:%s\n"
+			, ALPHA2_ARG(chplan->alpha2), rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+
+	} else {
+		if (*unknown_bmp & (BIT(BAND_ON_24G) | BIT(BAND_ON_5G))) {
+			txpwr_lmt[BAND_ON_24G] = rtw_regd_to_txpwr_lmt(rtw_chplan_get_default_regd_2g(chplan->domain_code));
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			txpwr_lmt[BAND_ON_5G] = rtw_regd_to_txpwr_lmt(rtw_chplan_get_default_regd_5g(chplan->domain_code));
+			#endif
+			for (band = 0; band < BAND_MAX; band++) {
+				if (!(*unknown_bmp & BIT(band)) || (band != BAND_ON_24G && band != BAND_ON_5G))
+					continue;
+				rtw_txpwr_apply_predefined(rfctl, req, band, txpwr_lmt[band], name_of_band, unknown_bmp);
+			}
+			RTW_PRINT("default domain:0x%02x applied, txpwr_lmt:%s\n"
+				, chplan->domain_code, rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+		}
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		if (*unknown_bmp & BIT(BAND_ON_6G)) {
+			txpwr_lmt[BAND_ON_6G] = rtw_regd_to_txpwr_lmt(rtw_chplan_get_default_regd_6g(chplan->domain_code_6g));
+			rtw_txpwr_apply_predefined(rfctl, req, BAND_ON_6G, txpwr_lmt[BAND_ON_6G], name_of_band, unknown_bmp);
+			RTW_PRINT("default domain_6g:0x%02x applied, txpwr_lmt:%s\n"
+				, chplan->domain_code_6g, rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+		}
+		#endif
+	}
+
+	if (*unknown_bmp == 0)
+		return;
+
+	for (band = 0; band < BAND_MAX; band++) {
+		if (!(*unknown_bmp & BIT(band)))
+			continue;
+		if (TXPWR_LMT_ALTERNATE_DEFINED(txpwr_lmt[band])) {
+			/*
+			* To support older chips without new predefined txpwr_lmt:
+			* - use txpwr_lmt_alternate() to get alternate if the selection is  not found
+			*/
+			altenate_applied = 1;
+			txpwr_lmt[band] = txpwr_lmt_alternate(txpwr_lmt[band]);
+			rtw_txpwr_apply_predefined(rfctl, req, band, txpwr_lmt[band], name_of_band, unknown_bmp);
+		}
+	}
+	if (altenate_applied) {
+		RTW_PRINT("alternate applied, txpwr_lmt:%s\n"
+			, rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+		if (*unknown_bmp == 0)
+			return;
+	}
+
+	for (band = 0; band < BAND_MAX; band++) {
+		if (!(*unknown_bmp & BIT(band)))
+			continue;
+		txpwr_lmt[band] = TXPWR_LMT_WW;
+		name_of_band[band] = txpwr_lmt_str(txpwr_lmt[band]);
+		*unknown_bmp &= ~BIT(band);
+	}
+	RTW_PRINT("world wide applied, txpwr_lmt:%s\n"
+		, rtw_get_txpwr_lmt_name_of_bands_str(buf, name_of_band, *unknown_bmp));
+}
+
+static void rtw_txpwr_apply_regd_req(struct rf_ctl_t *rfctl, struct regd_req_t *req
+	, char *names_of_band[], int names_of_band_len[])
+{
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define REG_6G_NAME_BSZ (16 + 4) /* ex: reg_name(up to  16 char)  + "_VLP" */
+#else
+#define REG_6G_NAME_BSZ 0
+#endif
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct rtw_chset *chset = &rfctl->chset;
+	struct country_chplan *chplan = &req->chplan;
+#ifdef CONFIG_REGD_SRC_FROM_OS
+	char req_alpha2_str[3] = {chplan->alpha2[0], chplan->alpha2[1], 0};
+#endif
+	char req_6g_name_buf[REG_6G_NAME_BSZ];
+	const char *name_of_band[BAND_MAX];
+	u8 unknown_bmp; /* unknown bitmap of name_of_band */
+	u8 band;
+
+	for (band = 0; band < BAND_MAX; band++)
+		name_of_band[band] = NULL;
+
+	unknown_bmp = 0
+		| (chset->chs_of_band[BAND_ON_24G] ? BIT(BAND_ON_24G) : 0)
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		| (chset->chs_of_band[BAND_ON_5G] ? BIT(BAND_ON_5G) : 0)
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		| (chset->chs_of_band[BAND_ON_6G] ? BIT(BAND_ON_6G) : 0)
+		#endif
+		;
+
+	/* search from exception mapping */
+	rtw_txpwr_apply_regd_req_reg_exc(rfctl, req
+		, req_6g_name_buf, REG_6G_NAME_BSZ, name_of_band, &unknown_bmp);
+	if (!unknown_bmp)
+		goto exit;
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+	rtw_txpwr_apply_regd_req_from_os(rfctl, req, req_alpha2_str
+		, req_6g_name_buf, REG_6G_NAME_BSZ, name_of_band, &unknown_bmp);
+	if (!unknown_bmp)
+		goto exit;
+#endif
+
+	/* follow default channel plan mapping */
+	rtw_txpwr_apply_regd_req_default(rfctl, req, name_of_band, &unknown_bmp);
+
+exit:
+	for (band = 0; band < BAND_MAX; band++)
+		ustrs_add(&names_of_band[band], &names_of_band_len[band], name_of_band[band]);
+}
+
+void rtw_txpwr_update_cur_lmt_regs(struct dvobj_priv *dvobj, bool req_lock)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	struct regd_req_t *req;
+	_list *cur, *head;
+	char *names[BAND_MAX];
+	int names_len[BAND_MAX];
+	u8 band;
+
+	_rtw_memset(names, 0, sizeof(names));
+	_rtw_memset(names_len, 0, sizeof(names_len));
+
+	if (req_lock)
+		_rtw_mutex_lock_interruptible(&rfctl->regd_req_mutex);
+
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+		if (!req->applied)
+			continue;
+		rtw_txpwr_apply_regd_req(rfctl, req, names, names_len);
+	}
+
+	if (req_lock)
+		_rtw_mutex_unlock(&rfctl->regd_req_mutex);
+
+	/* set to tx power limit regulations to HAL */
+	rtw_txpwr_hal_set_current_lmt_regs_by_name(dvobj, names, names_len);
+
+	for (band = 0; band < BAND_MAX; band++)
+		if (names[band] && names_len[band])
+			rtw_mfree(names[band], names_len[band]);
+}
+#endif /* CONFIG_TXPWR_LIMIT */
+
+static const struct country_chplan world_wide_chplan =
+	COUNTRY_CHPLAN_ENT(WORLDWIDE_ALPHA2, RTW_CHPLAN_WORLDWIDE, RTW_CHPLAN_6G_WORLDWIDE, DEF, 1, 1, 1, 1, ___);
+
+#ifdef CONFIG_CUSTOMIZED_COUNTRY_CHPLAN_MAP
+#include "../platform/custom_country_chplan.h"
+#elif RTW_DEF_MODULE_REGULATORY_CERT
+#include "./def_module_country_chplan.h"
+#endif
+
+/*
+* rtw_get_chplan_worldwide -
+* @ent: the buf to copy country_chplan entry content
+*/
+void rtw_get_chplan_worldwide(struct country_chplan *ent)
+{
+	_rtw_memcpy(ent, &world_wide_chplan, sizeof(*ent));
+}
+struct alpha2_alternate_t {
+	char alpha2[2];
+	char alt[2];
+};
+
+#define ALPHA2_ALT_ENT(_alpha2, _alt) {.alpha2 = _alpha2, .alt = _alt}
+
+static const struct alpha2_alternate_t alpha2_alt_map[] = {
+	ALPHA2_ALT_ENT("EU", "DE"),
+};
+
+static bool rtw_get_chplan_from_alpha2_alt(const char *code, struct country_chplan *ent)
+{
+	int i;
+	bool ret = false;
+
+	for (i = 0; i < ARRAY_SIZE(alpha2_alt_map); i++) {
+		if (code[0] == alpha2_alt_map[i].alpha2[0]
+			&& code[1] == alpha2_alt_map[i].alpha2[1]
+		) {
+			ret = regdb_ops.get_chplan_from_alpha2(alpha2_alt_map[i].alt, ent);
+			if (ret)
+				break;
+		}
+	}
+
+	return ret;
+}
+/*
+* rtw_get_chplan_from_country -
+* @country_code: string of country code
+* @ent: the buf to copy country_chplan entry content
+*
+* Return _TRUE or _FALSE when unsupported country_code is given
+*/
+bool rtw_get_chplan_from_country(const char *country_code, struct country_chplan *ent)
+{
+#if defined(CONFIG_CUSTOMIZED_COUNTRY_CHPLAN_MAP) || RTW_DEF_MODULE_REGULATORY_CERT
+	const struct country_chplan *map = NULL;
+	u16 map_sz = 0;
+	int i;
+#else
+	bool ret;
+#endif
+
+	char code[2] = {alpha_to_upper(country_code[0]), alpha_to_upper(country_code[1])};
+
+#if defined(CONFIG_CUSTOMIZED_COUNTRY_CHPLAN_MAP) || RTW_DEF_MODULE_REGULATORY_CERT
+	#ifdef CONFIG_CUSTOMIZED_COUNTRY_CHPLAN_MAP
+	map = CUSTOMIZED_country_chplan_map;
+	map_sz = sizeof(CUSTOMIZED_country_chplan_map) / sizeof(struct country_chplan);
+	#else
+	map_sz = rtw_def_module_country_chplan_map(&map);
+	#endif
+
+	for (i = 0; i < map_sz; i++) {
+		if (strncmp(code, map[i].alpha2, 2) == 0) {
+			if (ent)
+				_rtw_memcpy(ent, &map[i], sizeof(*ent));
+			return true;
+		}
+	}
+
+	return false;
+#else
+	if (regdb_ops.get_chplan_from_alpha2) {
+		ret = regdb_ops.get_chplan_from_alpha2(code, ent);
+		if (!ret)
+			ret = rtw_get_chplan_from_alpha2_alt(code, ent);
+	}
+	return ret;
+#endif
+}
+
+void rtw_chplan_ioctl_input_mapping(u16 *chplan, u16 *chplan_6g)
+{
+	if (chplan) {
+		if (*chplan == RTW_CHPLAN_IOCTL_UNSPECIFIED)
+			*chplan = RTW_CHPLAN_UNSPECIFIED;
+		else if (*chplan == RTW_CHPLAN_IOCTL_NULL)
+			*chplan = RTW_CHPLAN_NULL;
+	}
+
+	if (chplan_6g) {
+		if (*chplan_6g == RTW_CHPLAN_IOCTL_UNSPECIFIED)
+			*chplan_6g = RTW_CHPLAN_6G_UNSPECIFIED;
+		else if (*chplan_6g == RTW_CHPLAN_IOCTL_NULL)
+			*chplan_6g = RTW_CHPLAN_6G_NULL;
+	}
+}
+
+bool rtw_chplan_ids_is_world_wide(u8 chplan, u8 chplan_6g)
+{
+	return !(chplan == RTW_CHPLAN_NULL
+				#if CONFIG_IEEE80211_BAND_6GHZ
+				&& chplan_6g == RTW_CHPLAN_6G_NULL
+				#endif
+			)
+			&& (chplan == RTW_CHPLAN_WORLDWIDE || chplan == RTW_CHPLAN_NULL)
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			&& (chplan_6g == RTW_CHPLAN_6G_WORLDWIDE || chplan_6g == RTW_CHPLAN_6G_NULL)
+			#endif
+			;
+}
+
+/*
+ * Check if the @param ch, bw, offset is valid for the given @param ent
+ * @ent: the given country chplan ent
+ * @band: the given band
+ * @ch: the given channel number
+ * @bw: the given bandwidth
+ * @offset: the given channel offset
+ * @rfctl: refer to addtional ch enable/disable configurations
+ *
+ * return valid (1) or not (0)
+ */
+u8 rtw_country_chplan_is_bchbw_valid(struct country_chplan *ent, enum band_type band, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive, struct rf_ctl_t *rfctl)
+{
+	u8 domain_code_6g = RTW_CHPLAN_6G_NULL;
+	u8 valid = 0;
+
+	if (band == BAND_ON_5G && !COUNTRY_CHPLAN_EN_11A(ent))
+		goto exit;
+
+	if (bw >= CHANNEL_WIDTH_80 && !COUNTRY_CHPLAN_EN_11AC(ent))
+		goto exit;
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	domain_code_6g = ent->domain_code_6g;
+	#endif
+
+	valid = rtw_chplan_is_bchbw_valid(ent->domain_code, domain_code_6g, band, ch, bw, offset
+		, allow_primary_passive, allow_passive, rfctl);
+
+exit:
+	return valid;
+}
+
+static void rtw_country_chplan_get_edcca_mode_of_bands(const struct country_chplan *ent, u8 mode_of_band[])
+{
+	mode_of_band[BAND_ON_24G] =
+		COUNTRY_CHPLAN_EDCCA_2G_OVERRIDE(ent) != RTW_EDCCA_DEF ? COUNTRY_CHPLAN_EDCCA_2G_OVERRIDE(ent) :
+		rtw_regd_to_edcca_mode(rtw_chplan_get_default_regd_2g(ent->domain_code));
+	#if CONFIG_IEEE80211_BAND_5GHZ
+	mode_of_band[BAND_ON_5G] =
+		COUNTRY_CHPLAN_EDCCA_5G_OVERRIDE(ent) != RTW_EDCCA_DEF ? COUNTRY_CHPLAN_EDCCA_5G_OVERRIDE(ent) :
+		rtw_regd_to_edcca_mode(rtw_chplan_get_default_regd_5g(ent->domain_code));
+	#endif
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	mode_of_band[BAND_ON_6G] =
+		COUNTRY_CHPLAN_EDCCA_6G_OVERRIDE(ent) != RTW_EDCCA_DEF ? COUNTRY_CHPLAN_EDCCA_6G_OVERRIDE(ent) :
+		rtw_regd_to_edcca_mode_6g(rtw_chplan_get_default_regd_6g(ent->domain_code_6g));
+	#endif
+}
+
+static void rtw_country_chplan_get_txpwr_lmt_of_bands(const struct country_chplan *ent, u8 txpwr_lmt_of_band[])
+{
+	txpwr_lmt_of_band[BAND_ON_24G] =
+		ent->txpwr_lmt_override != TXPWR_LMT_DEF ? ent->txpwr_lmt_override :
+		rtw_regd_to_txpwr_lmt(rtw_chplan_get_default_regd_2g(ent->domain_code));
+	#if CONFIG_IEEE80211_BAND_5GHZ
+	txpwr_lmt_of_band[BAND_ON_5G] =
+		ent->txpwr_lmt_override != TXPWR_LMT_DEF ? ent->txpwr_lmt_override :
+		rtw_regd_to_txpwr_lmt(rtw_chplan_get_default_regd_5g(ent->domain_code));
+	#endif
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	txpwr_lmt_of_band[BAND_ON_6G] =
+		ent->txpwr_lmt_override != TXPWR_LMT_DEF ? ent->txpwr_lmt_override :
+		rtw_regd_to_txpwr_lmt(rtw_chplan_get_default_regd_6g(ent->domain_code_6g));
+	#endif
+}
+
+#ifdef CONFIG_80211D
+static const char *const _cis_status_str[] = {
+	[COUNTRY_IE_SLAVE_NOCOUNTRY]	= "NOCOUNTRY",
+	[COUNTRY_IE_SLAVE_APPLICABLE]	= "APPLICABLE",
+	[COUNTRY_IE_SLAVE_UNKNOWN]	= "UNKNOWN",
+	[COUNTRY_IE_SLAVE_OPCH_NOEXIST]	= "OPCH_NOEXIST",
+	[COUNTRY_IE_SLAVE_CATE_6G_NS]	= "CATE_6G_NS",
+	[COUNTRY_IE_SLAVE_STATUS_NUM]	= "INVALID",
+};
+
+#define cis_status_str(s) (((s) >= COUNTRY_IE_SLAVE_STATUS_NUM) ? _cis_status_str[COUNTRY_IE_SLAVE_STATUS_NUM] : _cis_status_str[s])
+
+const char _rtw_env_char[] = {
+	[RTW_ENV_ANY]		= ' ',
+	[RTW_ENV_INDOOR]	= 'I',
+	[RTW_ENV_OUTDOOR]	= 'O',
+};
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static const char *const _cis_6g_reg_info_str[] = {
+	[CIS_6G_REG_IN_AP]	= "IN_AP",
+	[CIS_6G_REG_SP_AP]	= "SP_AP",
+	[CIS_6G_REG_VLP_AP]	= "VLP_AP",
+	[CIS_6G_REG_IN_EN_AP]	= "IN_EN_AP",
+	[CIS_6G_REG_IN_SP_AP]	= "IN_SP_AP",
+	[CIS_6G_REG_NUM]	= "RSVD",
+};
+
+#define cis_6g_reg_info_str(r) (((r) >= CIS_6G_REG_NUM) ? _cis_6g_reg_info_str[CIS_6G_REG_NUM] : _cis_6g_reg_info_str[r])
+
+static const enum rtw_env_t _reg_info_to_env[] = {
+	[CIS_6G_REG_IN_AP]	= RTW_ENV_INDOOR,
+	[CIS_6G_REG_SP_AP]	= RTW_ENV_OUTDOOR,
+	[CIS_6G_REG_VLP_AP]	= RTW_ENV_ANY,
+	[CIS_6G_REG_IN_EN_AP]	= RTW_ENV_INDOOR,
+	[CIS_6G_REG_IN_SP_AP]	= RTW_ENV_INDOOR,
+	[CIS_6G_REG_NUM]	= RTW_ENV_ANY,
+};
+
+#define reg_info_to_env(reg_info) (((reg_info) >= CIS_6G_REG_NUM) ? _reg_info_to_env[CIS_6G_REG_NUM] : _reg_info_to_env[(reg_info)])
+#endif
+
+void dump_country_ie_slave_records(void *sel, struct rf_ctl_t *rfctl, bool skip_noset)
+{
+#define CISR_TITLE_FMT "%-6s %-4s %-4s"
+#define CISR_TITLE_ARG , "alpha2", "band", "opch"
+#define CISR_VALUE_FMT "     "ALPHA2_FMT" %4s %4u"
+#define CISR_VALUE_ARG , ALPHA2_ARG(cisr->alpha2), band_str(cisr->band), cisr->opch
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define CISR_TITLE_FMT_6G " %-3s %-8s"
+#define CISR_TITLE_ARG_6G , "env", "6g_reg_i"
+#define CISR_VALUE_FMT_6G " %3c %8s"
+#define CISR_VALUE_ARG_6G , rtw_env_char(cisr->env), cisr->band == BAND_ON_6G ? cis_6g_reg_info_str(cisr->reg_info) : ""
+#else
+#define CISR_TITLE_FMT_6G ""
+#define CISR_VALUE_FMT_6G ""
+#define CISR_TITLE_ARG_6G
+#define CISR_VALUE_ARG_6G
+#endif
+#define CISR_TITLE_FMT_STATUS " %s"
+#define CISR_VALUE_FMT_STATUS " %s"
+#define CISR_TITLE_ARG_STATUS , "status"
+#define CISR_VALUE_ARG_STATUS , cis_status_str(cisr->status)
+
+	struct country_ie_slave_record *cisr;
+	int i, j;
+
+	RTW_PRINT_SEL(sel,
+		"    "
+		CISR_TITLE_FMT
+		CISR_TITLE_FMT_6G
+		CISR_TITLE_FMT_STATUS
+		"\n"
+		CISR_TITLE_ARG
+		CISR_TITLE_ARG_6G
+		CISR_TITLE_ARG_STATUS
+	);
+
+	for (i = 0; i < CONFIG_IFACE_NUMBER; i++) {
+		for (j = 0; j < RTW_RLINK_MAX; j++) {
+			cisr = &rfctl->cisr[i][j];
+			if (skip_noset && strncmp(cisr->alpha2, "\x00\x00", 2) == 0)
+				continue;
+
+			RTW_PRINT_SEL(sel,
+				"%d %d"
+				CISR_VALUE_FMT
+				CISR_VALUE_FMT_6G
+				CISR_VALUE_FMT_STATUS
+				"\n"
+				, i, j
+				CISR_VALUE_ARG
+				CISR_VALUE_ARG_6G
+				CISR_VALUE_ARG_STATUS
+			);
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	RTW_PRINT_SEL(sel, "\n");
+	RTW_PRINT_SEL(sel, "6g_cate_map_int_link_num:%u\n", rfctl->txpwr_lmt_6g_cate_map_int_link_num);
+	RTW_PRINT_SEL(sel, "6g_cate_map_int_all_link:0x%02x\n", rfctl->txpwr_lmt_6g_cate_map_int_all_link);
+#endif
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static enum rtw_env_t country_str_get_env(const char *country_str)
+{
+	if (country_str) {
+		switch (country_str[2]) {
+		case 'I':
+			return RTW_ENV_INDOOR;
+		case 'O':
+			return RTW_ENV_OUTDOOR;
+		case ' ':
+		default:
+			return RTW_ENV_ANY;
+		}
+	}
+	return RTW_ENV_ANY;
+}
+#endif
+
+static enum country_ie_slave_status rtw_get_cisr_from_recv_regu_ies(struct rf_ctl_t *rfctl
+		, enum band_type band, u8 opch
+		, const u8 *country_ie, enum country_ie_slave_6g_reg_info reg_info
+		, struct country_ie_slave_record *cisr)
+{
+	const char *country_code = country_ie ? country_ie + 2 : NULL;
+	u8 domain_code_6g = RTW_CHPLAN_6G_NULL;
+	struct country_chplan *ent = &cisr->chplan;
+	enum country_ie_slave_status ret;
+
+	_rtw_memcpy(cisr->alpha2, country_code ? country_code : "\x00\x00", 2);
+	cisr->band = band;
+	cisr->opch = opch;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	cisr->env = country_str_get_env(country_code);
+	cisr->reg_info = reg_info;
+#endif
+
+	_rtw_memset(ent, 0, sizeof(*ent));
+
+	if (!country_code || strncmp(country_code, "XX", 2) == 0) {
+		ret = COUNTRY_IE_SLAVE_NOCOUNTRY;
+		goto exit;
+	}
+
+	if (!rtw_get_chplan_from_country(country_code, ent)) {
+		ret = COUNTRY_IE_SLAVE_UNKNOWN;
+		goto exit;
+	}
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	domain_code_6g = ent->domain_code_6g;
+	#endif
+
+	if (!rtw_chplan_is_bchbw_valid(ent->domain_code, domain_code_6g, band, opch
+			, CHANNEL_WIDTH_20, CHAN_OFFSET_NO_EXT, 1, 1, rfctl)
+	) {
+		ret = COUNTRY_IE_SLAVE_OPCH_NOEXIST;
+		goto exit;
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (cisr->band == BAND_ON_6G
+		&& !(ent->cate_6g_map & reg_info_to_chplan_6g_cate_map(reg_info))
+	) {
+		ret = COUNTRY_IE_SLAVE_CATE_6G_NS;
+		goto exit;
+	}
+#endif
+	ret = COUNTRY_IE_SLAVE_APPLICABLE;
+
+exit:
+	cisr->status = ret;
+
+	return ret;
+}
+#endif /* CONFIG_80211D */
+
+void dump_country_chplan(void *sel, const struct country_chplan *ent, bool regd_info)
+{
+	char buf[16];
+	char *pos = buf;
+
+	if (ent->domain_code == RTW_CHPLAN_UNSPECIFIED)
+		pos += sprintf(pos, "UNSPEC");
+	else
+		pos += sprintf(pos, "0x%02X", ent->domain_code);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (ent->domain_code_6g == RTW_CHPLAN_6G_UNSPECIFIED)
+		pos += sprintf(pos, " UNSPEC");
+	else
+		pos += sprintf(pos, " 0x%02X", ent->domain_code_6g);
+#endif
+
+	RTW_PRINT_SEL(sel, "\"%c%c\", %s"
+		, ent->alpha2[0], ent->alpha2[1], buf);
+
+	if (regd_info) {
+		u8 edcca_mode[BAND_MAX];
+		u8 txpwr_lmt[BAND_MAX];
+
+		rtw_country_chplan_get_edcca_mode_of_bands(ent, edcca_mode);
+		_RTW_PRINT_SEL(sel, " {%-6s", rtw_edcca_mode_str(edcca_mode[BAND_ON_24G]));
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		_RTW_PRINT_SEL(sel, " %-6s", rtw_edcca_mode_str(edcca_mode[BAND_ON_5G]));
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		_RTW_PRINT_SEL(sel, " %-6s", rtw_edcca_mode_str(edcca_mode[BAND_ON_6G]));
+		#endif
+		_RTW_PRINT_SEL(sel, "}");
+
+		rtw_country_chplan_get_txpwr_lmt_of_bands(ent, txpwr_lmt);
+		_RTW_PRINT_SEL(sel, " {%-8s", txpwr_lmt_str(txpwr_lmt[BAND_ON_24G]));
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		_RTW_PRINT_SEL(sel, " %-8s", txpwr_lmt_str(txpwr_lmt[BAND_ON_5G]));
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		_RTW_PRINT_SEL(sel, " %-8s", txpwr_lmt_str(txpwr_lmt[BAND_ON_6G]));
+		#endif
+		_RTW_PRINT_SEL(sel, "}");
+	}
+
+	_RTW_PRINT_SEL(sel, " %s", COUNTRY_CHPLAN_EN_11BE(ent) ? "be" : "  ");
+	_RTW_PRINT_SEL(sel, " %s", COUNTRY_CHPLAN_EN_11AX(ent) ? "ax" : "  ");
+	_RTW_PRINT_SEL(sel, " %s", COUNTRY_CHPLAN_EN_11AC(ent) ? "ac" : "  ");
+	_RTW_PRINT_SEL(sel, " %s", COUNTRY_CHPLAN_EN_11A(ent) ? "a" : " ");
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	_RTW_PRINT_SEL(sel, " %c%c%c"
+		, (ent->cate_6g_map & CHPLAN_6G_CATE_STD) ? 'S' : '_'
+		, (ent->cate_6g_map & CHPLAN_6G_CATE_LPI) ? 'I' : '_'
+		, (ent->cate_6g_map & CHPLAN_6G_CATE_VLP) ? 'V' : '_'
+	);
+	#endif
+
+	_RTW_PRINT_SEL(sel, "\n");
+}
+
+void dump_country_chplan_map(void *sel, bool regd_info)
+{
+	struct country_chplan ent;
+	u8 code[2];
+
+#if RTW_DEF_MODULE_REGULATORY_CERT
+	RTW_PRINT_SEL(sel, "RTW_DEF_MODULE_REGULATORY_CERT:0x%x\n", RTW_DEF_MODULE_REGULATORY_CERT);
+#endif
+#ifdef CONFIG_CUSTOMIZED_COUNTRY_CHPLAN_MAP
+	RTW_PRINT_SEL(sel, "CONFIG_CUSTOMIZED_COUNTRY_CHPLAN_MAP\n");
+#endif
+
+	rtw_get_chplan_worldwide(&ent);
+	dump_country_chplan(sel, &ent, regd_info);
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, &ent))
+				continue;
+
+			dump_country_chplan(sel, &ent, regd_info);
+		}
+	}
+}
+
+void dump_country_list(void *sel)
+{
+	u8 code[2];
+
+	RTW_PRINT_SEL(sel, "%s ", WORLDWIDE_ALPHA2);
+
+	for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+		for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+			if (!rtw_get_chplan_from_country(code, NULL))
+				continue;
+			_RTW_PRINT_SEL(sel, "%c%c ", code[0], code[1]);
+		}
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+}
+
+void dump_chplan_id_list(void *sel)
+{
+	u8 id_search_max = 255;
+	u8 first = 1;
+	int i;
+
+	for (i = 0; i <= id_search_max; i++) {
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+
+		if (first) {
+			RTW_PRINT_SEL(sel, "0x%02X ", i);
+			first = 0;
+		} else
+			_RTW_PRINT_SEL(sel, "0x%02X ", i);
+	}
+	if (first == 0)
+		_RTW_PRINT_SEL(sel, "\n");
+}
+
+void dump_chplan_country_list(void *sel)
+{
+	u8 id_search_max = 255;
+	int i;
+
+	for (i = 0; i <= id_search_max; i++) {
+		struct country_chplan ent;
+		u8 code[2];
+		u8 first;
+
+		if (!rtw_is_channel_plan_valid(i))
+			continue;
+
+		first = 1;
+		for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+			for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+				if (!rtw_get_chplan_from_country(code, &ent) || ent.domain_code != i)
+					continue;
+
+				if (first) {
+					RTW_PRINT_SEL(sel, "0x%02X %c%c ", i, code[0], code[1]);
+					first = 0;
+				} else
+					_RTW_PRINT_SEL(sel, "%c%c ", code[0], code[1]);
+			}
+		}
+		if (first == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+void dump_chplan_6g_id_list(void *sel)
+{
+	u8 id_search_max = 255;
+	u8 first = 1;
+	int i;
+
+	for (i = 0; i <= id_search_max; i++) {
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+
+		if (first) {
+			RTW_PRINT_SEL(sel, "0x%02X ", i);
+			first = 0;
+		} else
+			_RTW_PRINT_SEL(sel, "0x%02X ", i);
+	}
+	if (first == 0)
+		_RTW_PRINT_SEL(sel, "\n");
+}
+
+void dump_chplan_6g_country_list(void *sel)
+{
+	u8 id_search_max = 255;
+	int i;
+
+	for (i = 0; i <= id_search_max; i++) {
+		struct country_chplan ent;
+		u8 code[2];
+		u8 first;
+
+		if (!rtw_is_channel_plan_6g_valid(i))
+			continue;
+
+		first = 1;
+		for (code[0] = 'A'; code[0] <= 'Z'; code[0]++) {
+			for (code[1] = 'A'; code[1] <= 'Z'; code[1]++) {
+				if (!rtw_get_chplan_from_country(code, &ent) || ent.domain_code_6g != i)
+					continue;
+
+				if (first) {
+					RTW_PRINT_SEL(sel, "0x%02X %c%c ", i, code[0], code[1]);
+					first = 0;
+				} else
+					_RTW_PRINT_SEL(sel, "%c%c ", code[0], code[1]);
+			}
+		}
+		if (first == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+#ifdef CONFIG_RTW_DEBUG
+void dump_chplan_test(void *sel)
+{
+	if (regdb_ops.dump_chplan_test)
+		regdb_ops.dump_chplan_test(sel);
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+void dump_chplan_ver(void *sel)
+{
+	char buf[CHPLAN_VER_STR_BUF_LEN] = {0};
+
+	if (regdb_ops.get_ver_str)
+		regdb_ops.get_ver_str(buf, CHPLAN_VER_STR_BUF_LEN);
+	RTW_PRINT_SEL(sel, "%s\n", buf);
+}
+
+static struct regd_req_t *rtw_regd_req_alloc(void)
+{
+	struct regd_req_t *req;
+
+	req = rtw_zmalloc(sizeof(struct regd_req_t));
+	if (req)
+		_rtw_init_listhead(&req->list);
+
+	return req;
+}
+
+static struct regd_req_t *rtw_regd_req_alloc_with_country_chplan(struct country_chplan *ent)
+{
+	struct regd_req_t *req;
+
+	req = rtw_regd_req_alloc();
+	if (req)
+		_rtw_memcpy(&req->chplan, ent, sizeof(req->chplan));
+
+	return req;
+}
+
+static void rtw_regd_req_free(struct regd_req_t *req)
+{
+	rtw_mfree(req, sizeof(*req));
+}
+
+static void rtw_regd_req_list_insert(struct rf_ctl_t *rfctl, struct regd_req_t *req)
+{
+	rtw_list_insert_tail(&req->list, &rfctl->regd_req_list);
+	rfctl->regd_req_num++;
+}
+
+#if defined(CONFIG_80211D) || defined(CONFIG_REGD_SRC_FROM_OS)
+static void rtw_regd_req_list_delete(struct rf_ctl_t *rfctl, struct regd_req_t *req)
+{
+	rtw_list_delete(&req->list);
+	rfctl->regd_req_num--;
+}
+#endif
+
+void rtw_regd_req_list_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty)
+{
+	_rtw_mutex_init(&rfctl->regd_req_mutex);
+	_rtw_init_listhead(&rfctl->regd_req_list);
+	rfctl->init_regd_always_apply = regsty->init_regd_always_apply;
+	rfctl->user_regd_always_apply = regsty->user_regd_always_apply;
+}
+
+void rtw_regd_req_list_free(struct rf_ctl_t *rfctl)
+{
+	struct regd_req_t *req;
+	_list *cur, *head;
+
+	_rtw_mutex_lock_interruptible(&rfctl->regd_req_mutex);
+
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+		rtw_list_delete(&req->list);
+		if (req != &rfctl->init_req)
+			rtw_regd_req_free(req);
+	}
+	rfctl->user_req = NULL;
+	rfctl->regd_req_num = 0;
+
+	_rtw_mutex_unlock(&rfctl->regd_req_mutex);
+
+	_rtw_mutex_free(&rfctl->regd_req_mutex);
+}
+
+void rtw_regd_watchdog_hdl(struct dvobj_priv *dvobj)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+
+	rtw_beacon_hint_expire(rfctl);
+
+#ifdef CONFIG_DFS_MASTER
+	rtw_rfctl_chset_chk_non_ocp_finish(rfctl);
+#endif
+}
+
+static enum channel_width rtw_regd_adjust_linking_bw(struct rf_ctl_t *rfctl
+	, enum band_type band, u8 ch, enum channel_width bw, enum chan_offset offset)
+{
+#ifndef DBG_REGD_ADJUST_LINKING_BW
+#define DBG_REGD_ADJUST_LINKING_BW 0
+#endif
+
+	struct rtw_chset *chset = &rfctl->chset;
+
+	#if DBG_REGD_ADJUST_LINKING_BW
+	RTW_INFO("%s %s ch:%u,%u,%u\n"
+		, __func__, band_str(band), ch, bw, offset);
+	#endif
+
+	if (bw == CHANNEL_WIDTH_20)
+		goto exit;
+
+	for (; bw > CHANNEL_WIDTH_20; bw--) {
+		if (rtw_chset_is_bchbw_non_ocp(chset, band, ch, bw, offset)) {
+			#if DBG_REGD_ADJUST_LINKING_BW
+			RTW_INFO("%s %s ch:%u,%u,%u not allowed by non_ocp\n", __func__, band_str(band), ch, bw, offset);
+			#endif
+			continue;
+		}
+
+		if (!rtw_chset_is_bchbw_valid(chset, band, ch, bw, offset, true, true)) {
+			#if DBG_REGD_ADJUST_LINKING_BW
+			RTW_INFO("%s %s ch:%u,%u,%u not allowed by chset\n", __func__, band_str(band), ch, bw, offset);
+			#endif
+			continue;
+		}
+
+		break;
+	}
+
+	if (bw == CHANNEL_WIDTH_20)
+		offset = CHAN_OFFSET_NO_EXT;
+
+exit:
+	rtw_warn_on(!rtw_chset_is_bchbw_valid(chset, band, ch, bw, offset, true, true));
+	rtw_warn_on(rtw_chset_is_bchbw_non_ocp(chset, band, ch, bw, offset));
+
+	return bw;
+}
+
+enum channel_width alink_adjust_linking_bw_by_regd(struct _ADAPTER_LINK *alink
+	, enum band_type band, u8 ch, enum channel_width bw, enum chan_offset offset)
+{
+	_adapter *adapter = alink->adapter;
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	return rtw_regd_adjust_linking_bw(rfctl, band, ch, bw, offset);
+}
+
+enum channel_width adapter_adjust_linking_bw_by_regd(_adapter *adapter
+	, enum band_type band, u8 ch, enum channel_width bw, enum chan_offset offset)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	return rtw_regd_adjust_linking_bw(rfctl, band, ch, bw, offset);
+}
+
+static void rtw_chplan_rtk_priv_domain_code_get_country_chplan(struct country_chplan *chplan
+	, u8 domain_code, u8 domain_code_6g)
+{
+	if (rtw_chplan_ids_is_world_wide(domain_code, domain_code_6g)) {
+		rtw_get_chplan_worldwide(chplan);
+		chplan->domain_code = domain_code;
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		chplan->domain_code_6g = domain_code_6g;
+		#endif
+	} else {
+		SET_UNSPEC_ALPHA2(chplan->alpha2);
+		chplan->domain_code = domain_code;
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		chplan->domain_code_6g = domain_code_6g;
+		#endif
+
+		#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE
+		chplan->edcca_2g_override = RTW_EDCCA_DEF;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		chplan->edcca_5g_override = RTW_EDCCA_DEF;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		chplan->edcca_6g_override = RTW_EDCCA_DEF;
+		#endif
+		#endif
+
+		chplan->txpwr_lmt_override = TXPWR_LMT_DEF;
+		#ifdef CONFIG_CHPLAN_PROTO_EN
+		chplan->proto_en = CHPLAN_PROTO_EN_ALL;
+		#endif
+	}
+}
+
+static void rtw_rfctl_regd_status_init_by_req(struct rf_ctl_t *rfctl, struct regd_req_t *req)
+{
+	struct country_chplan *chplan = &req->chplan;
+
+	rfctl->regd_src = req->src;
+	rfctl->regd_inr_bmp = BIT(req->inr);
+	_rtw_memcpy(rfctl->alpha2, chplan->alpha2, 2);
+	rfctl->domain_code = chplan->domain_code;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	rfctl->domain_code_6g = chplan->domain_code_6g;
+	rfctl->env_bmp = BIT(req->env);
+#endif
+#ifdef CONFIG_CHPLAN_PROTO_EN
+	rfctl->proto_en = chplan->proto_en;
+#endif
+}
+
+#ifdef CONFIG_80211D
+static bool rtw_rfctl_is_init_user_req_world_wide(struct rf_ctl_t *rfctl)
+{
+	return (!rfctl->user_req && IS_ALPHA2_WORLDWIDE(rfctl->init_req.chplan.alpha2)) /* INIT is WW (when no USER) */
+		|| (rfctl->user_req && IS_ALPHA2_WORLDWIDE(rfctl->user_req->chplan.alpha2)); /* USER is WW */
+}
+
+static bool rtw_rfctl_should_cis_enable(struct rf_ctl_t *rfctl)
+{
+	if (rtw_rfctl_is_disable_sw_channel_plan(rfctl_to_dvobj(rfctl)))
+		return false;
+	if (rfctl->cis_en_mode == CISEM_ENABLE)
+		return true;
+	if (rfctl->cis_en_mode == CISEM_ENABLE_WW)
+		return rtw_rfctl_is_init_user_req_world_wide(rfctl);
+	return false;
+}
+
+static bool rtw_rfctl_update_cis_enabled(struct rf_ctl_t *rfctl, bool init, const char *caller)
+{
+	bool enabled = rtw_rfctl_should_cis_enable(rfctl);
+
+	if (init || rfctl->cis_enabled != enabled) {
+		rfctl->cis_enabled = enabled;
+		RTW_INFO("%s set cis_enabled to %d\n", caller, enabled);
+		return true;
+	}
+	return false;
+}
+
+static bool rtw_rfctl_forbid_unknown_country_opch(bool cis_enable, bool init_user_ww, bool regd_src_os)
+{
+	return (cis_enable || init_user_ww) && !regd_src_os;
+}
+
+static void rtw_rfctl_update_cisr_collect_mode(struct rf_ctl_t *rfctl, bool init, const char *caller)
+{
+	bool collect_link_cisr;
+	bool collect_network_cisr;
+	bool forbid_unknown_country_opch = rtw_rfctl_forbid_unknown_country_opch(
+		rfctl->cis_enabled, rtw_rfctl_is_init_user_req_world_wide(rfctl),
+		RFCTL_REGD_SRC_FROM_OS(rfctl));
+
+	collect_link_cisr = (rfctl->cis_enabled && !RFCTL_REGD_SRC_FROM_OS(rfctl))
+		|| rtw_chset_has_6g_enabled(&rfctl->chset);
+
+	collect_network_cisr = (rfctl->cis_enabled && (rfctl->cis_flags & CISF_ENV_BSS))
+		|| (rtw_txpwr_hal_is_txpwr_limit_needed(rfctl_to_dvobj(rfctl))
+			&& (forbid_unknown_country_opch || rtw_chset_has_6g_enabled(&rfctl->chset)));
+
+	if (init || rfctl->collect_link_cisr != collect_link_cisr) {
+		rfctl->collect_link_cisr = collect_link_cisr;
+		RTW_INFO("%s set collect_link_cisr to %d\n", caller, collect_link_cisr);
+	}
+	if (init || rfctl->collect_network_cisr != collect_network_cisr) {
+		rfctl->collect_network_cisr = collect_network_cisr;
+		RTW_INFO("%s set collect_network_cisr to %d\n", caller, collect_network_cisr);
+	}
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static void rtw_rfctl_update_default_chplan_cate_6g_map(struct rf_ctl_t *rfctl, bool init, const char *caller)
+{
+	struct regd_req_t *user_req = rfctl->user_req;
+	u8 bmp = 0xFF;
+
+	if (user_req)
+		bmp &= user_req->chplan.cate_6g_map;
+	if (!user_req || rfctl->init_regd_always_apply)
+		bmp &= rfctl->init_req.chplan.cate_6g_map;
+
+	if (init || rfctl->default_chplan_cate_6g_map != bmp) {
+		rfctl->default_chplan_cate_6g_map = bmp;
+		RTW_INFO("%s set default_chplan_cate_6g_map to 0x%02x\n", caller, bmp);
+		if (!bmp && !rfctl->cis_enabled && rtw_txpwr_hal_is_txpwr_limit_needed(rfctl_to_dvobj(rfctl)))
+			RTW_INFO("%s !cis_enabled, 6G connection not allowed\n", caller);
+	}
+}
+
+/* 
+* Update txpwr_lmt_6g_cate_map for 
+* a. REGD_SRC_RTK_PRIV's INIT/USER req
+* b. REGD_SRC_OS's all req
+*/
+static void rtw_rfctl_update_regd_req_txpwr_6g_cate_map(struct rf_ctl_t *rfctl)
+{
+	struct regd_req_t *req;
+	_list *cur, *head;
+
+	/* apply txpwr lmt 6g cate map of req */
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+
+		if (rfctl->regd_src == REGD_SRC_RTK_PRIV) {
+			/* apply txpwr lmt 6g cate map only for INIT/USER req */
+			if (req->inr > RTW_REGD_SET_BY_USER)
+				continue;
+		}
+
+		if (rfctl->txpwr_lmt_6g_cate_map_int_link_num) /* from per_link cisr */
+			req->txpwr_lmt_6g_cate_map = rfctl->txpwr_lmt_6g_cate_map_int_all_link;
+		else {
+			if (req->inr == RTW_REGD_SET_BY_COUNTRY_IE) /* REGD_SRC_OS, without 6G link */
+				req->txpwr_lmt_6g_cate_map = BIT(TXPWR_LMT_6G_CATE_VLP);
+			else /* get from self's env */
+				req->txpwr_lmt_6g_cate_map = rtw_env_to_txpwr_lmt_6g_cate_map(req->env);
+		}
+	}
+}
+#endif
+#endif /* CONFIG_80211D */
+
+/*
+ * Description:
+ *	Use hardware(efuse), driver parameter(registry) and default channel plan
+ *	to decide which one should be used.
+ *
+ * Parameters:
+ *	rfctl				pointer of rfctl
+ *	hw_alpha2		country code from HW (efuse/eeprom/mapfile)
+ *	hw_chplan		domain code from HW (efuse/eeprom/mapfile)
+ *	hw_chplan_6g	6g domain code from HW (efuse/eeprom/mapfile)
+ *	hw_force_chplan	if forcing HW channel plan setting (efuse/eeprom/mapfile)
+ *					will modified tif HW channel plan setting is invlid, will
+ */
+void rtw_rfctl_decide_init_chplan(struct rf_ctl_t *rfctl,
+	const char *hw_alpha2, u8 hw_chplan, u8 hw_chplan_6g, u8 hw_force_chplan)
+{
+	struct registry_priv *regsty;
+	char *sw_alpha2;
+	const struct country_chplan *country_ent = NULL;
+	struct country_chplan ent;
+	int chplan = -1;
+	int chplan_6g = -1;
+
+	u8 sw_chplan;
+	u8 def_chplan = RTW_CHPLAN_WORLDWIDE; /* worldwide,  used when HW, SW both invalid */
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 sw_chplan_6g;
+	u8 def_chplan_6g = RTW_CHPLAN_6G_WORLDWIDE; /* worldwide,  used when HW, SW both invalid */
+#endif
+
+	if (hw_alpha2) {
+		if (strlen(hw_alpha2) != 2 || !is_alpha(hw_alpha2[0]) || is_alpha(hw_alpha2[1]))
+			RTW_PRINT("%s hw_alpha2 is not valid alpha2\n", __func__);
+		else if (rtw_get_chplan_from_country(hw_alpha2, &ent)) {
+			/* get chplan from hw country code, by pass hw chplan setting */
+			country_ent = &ent;
+			chplan = ent.domain_code;
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			chplan_6g = ent.domain_code_6g;
+			#endif
+			goto chk_sw_config;
+		} else
+			RTW_PRINT("%s unsupported hw_alpha2:\"%c%c\"\n", __func__, hw_alpha2[0], hw_alpha2[1]);
+	}
+
+	if (rtw_is_channel_plan_valid(hw_chplan))
+		chplan = hw_chplan;
+	else if (hw_force_chplan == _TRUE) {
+		RTW_PRINT("%s unsupported hw_chplan:0x%02X\n", __func__, hw_chplan);
+		/* hw infomaton invalid, refer to sw information */
+		hw_force_chplan = _FALSE;
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (rtw_is_channel_plan_6g_valid(hw_chplan_6g))
+		chplan_6g = hw_chplan_6g;
+	else if (hw_force_chplan == _TRUE) {
+		RTW_PRINT("%s unsupported hw_chplan_6g:0x%02X\n", __func__, hw_chplan_6g);
+		/* hw infomaton invalid, refer to sw information */
+		hw_force_chplan = _FALSE;
+	}
+#endif
+
+chk_sw_config:
+	regsty = dvobj_to_regsty(rfctl_to_dvobj(rfctl));
+
+	if (hw_force_chplan == _TRUE)
+		goto done;
+
+	sw_alpha2 = regsty->alpha2;
+	sw_chplan = regsty->channel_plan;
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	sw_chplan_6g = regsty->channel_plan_6g;
+	#endif
+
+	if (sw_alpha2 && !IS_ALPHA2_UNSPEC(sw_alpha2)) {
+		if (IS_ALPHA2_WORLDWIDE(sw_alpha2)
+			|| rtw_get_chplan_from_country(sw_alpha2, &ent)
+		) {
+			/* get chplan from sw country code, by pass sw chplan setting */
+			if (IS_ALPHA2_WORLDWIDE(sw_alpha2))
+				rtw_get_chplan_worldwide(&ent);
+			country_ent = &ent;
+			chplan = ent.domain_code;
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			chplan_6g = ent.domain_code_6g;
+			#endif
+			goto done;
+		} else
+			RTW_PRINT("%s unsupported sw_alpha2:\"%c%c\"\n", __func__, sw_alpha2[0], sw_alpha2[1]);
+	}
+
+	if (rtw_is_channel_plan_valid(sw_chplan)) {
+		/* cancel hw_alpha2 because chplan is specified by sw_chplan */
+		country_ent = NULL;
+		chplan = sw_chplan;
+	} else if (sw_chplan != RTW_CHPLAN_UNSPECIFIED)
+		RTW_PRINT("%s unsupported sw_chplan:0x%02X\n", __func__, sw_chplan);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (rtw_is_channel_plan_6g_valid(sw_chplan_6g)) {
+		/* cancel hw_alpha2 because chplan_6g is specified by sw_chplan_6g */
+		country_ent = NULL;
+		chplan_6g = sw_chplan_6g;
+	} else if (sw_chplan_6g != RTW_CHPLAN_6G_UNSPECIFIED)
+		RTW_PRINT("%s unsupported sw_chplan_6g:0x%02X\n", __func__, sw_chplan_6g);
+#endif
+
+done:
+	if (chplan == -1) {
+		RTW_PRINT("%s use def_chplan:0x%02X\n", __func__, def_chplan);
+		chplan = def_chplan;
+	} else
+		RTW_PRINT("%s chplan:0x%02X\n", __func__, chplan);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (chplan_6g == -1) {
+		RTW_PRINT("%s use def_chplan_6g:0x%02X\n", __func__, def_chplan_6g);
+		chplan_6g = def_chplan_6g;
+	} else
+		RTW_PRINT("%s chplan_6g:0x%02X\n", __func__, chplan_6g);
+#endif
+
+	if (!country_ent)
+		rtw_chplan_rtk_priv_domain_code_get_country_chplan(&ent, chplan, chplan_6g);
+	else {
+		RTW_PRINT("%s country code:\"%c%c\"\n", __func__
+			, country_ent->alpha2[0], country_ent->alpha2[1]);
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	RTW_PRINT("%s env:%s\n", __func__, env_str(regsty->env));
+#endif
+
+	rfctl->disable_sw_chplan = hw_force_chplan;
+
+	rfctl->init_req.src = REGD_SRC_RTK_PRIV;
+	rfctl->init_req.inr = RTW_REGD_SET_BY_INIT;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	rfctl->init_req.env = regsty->env;
+#endif
+	rfctl->init_req.applied = true;
+	_rtw_memcpy(&rfctl->init_req.chplan, &ent, sizeof(ent));
+	rtw_regd_req_list_insert(rfctl, &rfctl->init_req);
+
+	rtw_rfctl_regd_status_init_by_req(rfctl, &rfctl->init_req);
+}
+
+void rtw_rfctl_apply_init_chplan(struct rf_ctl_t *rfctl, bool req_lock)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+
+	rtw_edcca_mode_update(dvobj, req_lock);
+	rtw_rfctl_chset_apply_regulatory(dvobj, req_lock);
+
+#ifdef CONFIG_80211D
+	rtw_rfctl_update_cis_enabled(rfctl, true, __func__);
+	rtw_rfctl_update_cisr_collect_mode(rfctl, true, __func__);
+	rfctl->init_user_req_is_ww = rtw_rfctl_is_init_user_req_world_wide(rfctl);
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	rtw_rfctl_update_default_chplan_cate_6g_map(rfctl, true, __func__);
+	rtw_rfctl_update_regd_req_txpwr_6g_cate_map(rfctl);
+	#endif
+#endif
+}
+
+bool rtw_rfctl_is_disable_sw_channel_plan(struct dvobj_priv *dvobj)
+{
+	return dvobj_to_rfctl(dvobj)->disable_sw_chplan;
+}
+
+static void get_str_of_set_chplan_keys(char *buf, u8 buf_len, struct SetChannelPlan_param *param)
+{
+	char *pos = buf;
+
+#ifdef CONFIG_80211D
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define CHPLAN_KEYS_FMT_80211D_6G " env:%s reg_info:%s"
+#define CHPLAN_KEYS_ARG_80211D_6G , env_str(param->cisr.env), cis_6g_reg_info_str(param->cisr.reg_info)
+#else
+#define CHPLAN_KEYS_FMT_80211D_6G
+#define CHPLAN_KEYS_ARG_80211D_6G
+#endif
+
+	if (param->regd_src == REGD_SRC_RTK_PRIV && param->inr == RTW_REGD_SET_BY_COUNTRY_IE) {
+		if (param->has_cisr) {
+			pos += snprintf(pos, buf_len - (pos - buf), "alid:%c alpha2:"ALPHA2_FMT
+				CHPLAN_KEYS_FMT_80211D_6G
+				" %s"
+				, param->cisr_alink_id >= RTW_RLINK_MAX ? '-' : '0' + param->cisr_alink_id
+				, ALPHA2_ARG(param->cisr.alpha2)
+				CHPLAN_KEYS_ARG_80211D_6G
+				, cis_status_str(param->cisr.status));
+		} else
+			*buf = '\0';
+	} else
+#endif
+	{
+		if (param->has_country)
+			pos += snprintf(pos, buf_len - (pos - buf), "alpha2:"ALPHA2_FMT, ALPHA2_ARG(param->country_ent.alpha2));
+		else {
+			if (param->channel_plan == RTW_CHPLAN_UNSPECIFIED)
+				pos += snprintf(pos, buf_len - (pos - buf), "chplan:UNSPEC");
+			else
+				pos += snprintf(pos, buf_len - (pos - buf), "chplan:0x%02X", param->channel_plan);
+
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (param->channel_plan_6g == RTW_CHPLAN_6G_UNSPECIFIED)
+				pos += snprintf(pos, buf_len - (pos - buf), " chplan_6g:UNSPEC");
+			else
+				pos += snprintf(pos, buf_len - (pos - buf), " chplan_6g:0x%02X", param->channel_plan_6g);
+			#endif
+		}
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		if (param->env < RTW_ENV_NUM)
+			pos += snprintf(pos, buf_len - (pos - buf), " env:%s", env_str(param->env));
+		else
+			pos += snprintf(pos, buf_len - (pos - buf), " env:UNSPEC");
+		#endif
+	}
+}
+
+#define EXCL_CHS_STR_LEN (MAX_CHANNEL_NUM_2G_5G * 4)
+#define EXCL_CHS_6G_STR_LEN (MAX_CHANNEL_NUM_6G * 4)
+#if defined(CONFIG_RTW_DEBUG) || defined(CONFIG_PROC_DEBUG)
+static char *get_str_of_u8_array(char *buf, size_t buf_len, u8 array[], size_t array_len, char delim, bool zero_end)
+{
+	char *pos = buf;
+	char d_str[2] = {delim, '\0'};
+	int i;
+
+	for (i = 0; i < array_len && (!zero_end || array[i]); i++) {
+		pos += snprintf(pos, buf_len - (pos - buf), "%s%u"
+			, pos == buf ? "" : d_str, array[i]);
+		if (pos >= buf + buf_len - 1)
+			break;
+	}
+	if (pos == buf)
+		buf[0] = '\0';
+
+	return buf;
+}
+#endif
+
+#define RTW_PRIV_USER_SET_DOMAIN	BIT0
+#define RTW_PRIV_USER_SET_DOMAIN_6G	BIT1
+#define RTW_PRIV_USER_SET_COUNTRY	BIT2
+#define RTW_PRIV_USER_SET_ENV		BIT3
+
+static int rtw_chplan_rtk_priv_req_prehdl_domain_code(struct rf_ctl_t *rfctl, struct SetChannelPlan_param *param, const char *caller)
+{
+	if (param->priv_user_set_bmp & RTW_PRIV_USER_SET_DOMAIN) {
+		/* disallow invalid input */
+		if (!rtw_is_channel_plan_valid(param->channel_plan)) {
+			RTW_WARN("%s invalid chplan:0x%02X\n", caller, param->channel_plan);
+			return _FAIL;
+		}
+	} else {
+		/* use original value when unspecified */
+		if (rfctl->user_req)
+			param->channel_plan = rfctl->user_req->chplan.domain_code;
+		else
+			param->channel_plan = rfctl->init_req.chplan.domain_code;
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (param->priv_user_set_bmp & RTW_PRIV_USER_SET_DOMAIN_6G) {
+		if (!rtw_is_channel_plan_6g_valid(param->channel_plan_6g)) {
+			RTW_WARN("%s invalid chplan_6g:0x%02X\n", caller, param->channel_plan_6g);
+			return _FAIL;
+		}
+	} else {
+		/* use original value when unspecified */
+		if (rfctl->user_req)
+			param->channel_plan_6g = rfctl->user_req->chplan.domain_code_6g;
+		else
+			param->channel_plan_6g = rfctl->init_req.chplan.domain_code_6g;
+	}
+#endif
+
+	return _SUCCESS;
+}
+
+static void rtw_chplan_rtk_priv_req_prehdl_country_ent(struct rf_ctl_t *rfctl, struct SetChannelPlan_param *param)
+{
+	if (!param->has_country) {
+		struct country_chplan *ent = &param->country_ent;
+
+		if (!(param->priv_user_set_bmp & (RTW_PRIV_USER_SET_DOMAIN | RTW_PRIV_USER_SET_DOMAIN_6G))) {
+			/* no domain/_6g specified, keep original country setting */
+			if (rfctl->user_req)
+				_rtw_memcpy(ent, &rfctl->user_req->chplan, sizeof(*ent));
+			else
+				_rtw_memcpy(ent, &rfctl->init_req.chplan, sizeof(*ent));
+		} else {
+			/* set world wide or unspecified */
+			u8 chplan_6g = RTW_CHPLAN_6G_NULL;
+
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			chplan_6g = param->channel_plan_6g;
+			#endif
+
+			rtw_chplan_rtk_priv_domain_code_get_country_chplan(ent, param->channel_plan, chplan_6g);
+		}
+
+		param->has_country = 1;
+	}
+}
+
+static void rtw_chplan_rtk_priv_req_prehdl_confs(struct rf_ctl_t *rfctl, struct chplan_confs *confs, const char *caller)
+{
+	if (confs->set_types & BIT(CHPLAN_CONFS_DIS_CH_FLAGS)) {
+		#if !CONFIG_DFS
+		/* force disable DFS channel because no DFS capability */
+		confs->dis_ch_flags |= RTW_CHF_DFS;
+		#endif
+		if (rfctl->dis_ch_flags != confs->dis_ch_flags) {
+			char buf[RTW_CH_FLAGS_STR_LEN];
+
+			rfctl->dis_ch_flags = confs->dis_ch_flags;
+			RTW_INFO("%s set dis_ch_flags to %s\n", caller
+				, rtw_get_ch_flags_str(buf, confs->dis_ch_flags, ','));
+		} else
+			confs->set_types &= ~BIT(CHPLAN_CONFS_DIS_CH_FLAGS);
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_EXCL_CHS)) {
+		if (_rtw_memcmp(rfctl->excl_chs, confs->excl_chs, sizeof(rfctl->excl_chs)) == _FALSE) {
+			char buf[EXCL_CHS_STR_LEN];
+
+			_rtw_memcpy(rfctl->excl_chs, confs->excl_chs, sizeof(rfctl->excl_chs));
+
+			RTW_INFO("%s set excl_chs to %s\n", caller
+				, get_str_of_u8_array(buf, sizeof(buf), confs->excl_chs, MAX_CHANNEL_NUM_2G_5G, ',', true));
+		} else
+			confs->set_types &= ~BIT(CHPLAN_CONFS_EXCL_CHS);
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (confs->set_types & BIT(CHPLAN_CONFS_EXCL_CHS_6G)) {
+		if (_rtw_memcmp(rfctl->excl_chs_6g, confs->excl_chs_6g, sizeof(rfctl->excl_chs_6g)) == _FALSE) {
+			char buf[EXCL_CHS_6G_STR_LEN];
+
+			_rtw_memcpy(rfctl->excl_chs_6g, confs->excl_chs_6g, sizeof(rfctl->excl_chs_6g));
+
+			RTW_INFO("%s set excl_chs_6g to %s\n", caller
+				, get_str_of_u8_array(buf, sizeof(buf), confs->excl_chs_6g, MAX_CHANNEL_NUM_6G, ',', true));
+		} else
+			confs->set_types &= ~BIT(CHPLAN_CONFS_EXCL_CHS_6G);
+	}
+#endif
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_INIT_REGD_ALWAYS_APPLY)) {
+		bool val_changed = false;
+
+		if (rfctl->init_regd_always_apply != confs->init_regd_always_apply) {
+			rfctl->init_regd_always_apply = confs->init_regd_always_apply;
+			val_changed = true;
+			RTW_INFO("%s set init_regd_always_apply to %d\n"
+				, caller, confs->init_regd_always_apply);
+		}
+
+		if (!val_changed || rfctl->regd_src != REGD_SRC_RTK_PRIV
+			|| rfctl->regd_req_num == 1 /* only INIT */
+		)
+			confs->set_types &= ~BIT(CHPLAN_CONFS_INIT_REGD_ALWAYS_APPLY);
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_USER_REGD_ALWAYS_APPLY)) {
+		bool val_changed = false;
+
+		if (rfctl->user_regd_always_apply != confs->user_regd_always_apply) {
+			rfctl->user_regd_always_apply = confs->user_regd_always_apply;
+			val_changed = true;
+			RTW_INFO("%s set user_regd_always_apply to %d\n"
+				, caller, confs->user_regd_always_apply);
+		}
+
+		if (!val_changed || rfctl->regd_src != REGD_SRC_RTK_PRIV
+			|| !rfctl->user_req
+			|| rfctl->regd_req_num == 2 /* only INIT & USER */
+		)
+			confs->set_types &= ~BIT(CHPLAN_CONFS_USER_REGD_ALWAYS_APPLY);
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_BCN_HINT_VALID_MS)) {
+		if (rfctl->bcn_hint_valid_ms != confs->bcn_hint_valid_ms) {
+			rfctl->bcn_hint_valid_ms = confs->bcn_hint_valid_ms;
+			RTW_INFO("%s set bcn_hint_valid_ms to %u\n", caller, confs->bcn_hint_valid_ms);
+		}
+		confs->set_types &= ~BIT(CHPLAN_CONFS_BCN_HINT_VALID_MS); /* setting done here */
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_EDCCA_MODE_SEL)) {
+		if (RTW_EDCCA_MODE_SEL_VALID(confs->edcca_mode_sel)
+			&& rfctl->edcca_mode_sel != confs->edcca_mode_sel
+		) {
+			rfctl->edcca_mode_sel = confs->edcca_mode_sel;
+			RTW_INFO("%s set edcca_mode_sel to %s\n", caller
+				, rtw_edcca_mode_sel_str(confs->edcca_mode_sel));
+			if (rfctl->edcca_mode_sel != RTW_EDCCA_AUTO) {
+				/* force edcca mode, not rely on regd reqs, update here */
+				rtw_edcca_mode_update(rfctl_to_dvobj(rfctl), false);
+				confs->set_types &= ~BIT(CHPLAN_CONFS_EDCCA_MODE_SEL); /* setting done here */
+			}
+		} else
+			confs->set_types &= ~BIT(CHPLAN_CONFS_EDCCA_MODE_SEL); /* setting done here */
+	}
+
+#ifdef CONFIG_80211D
+	#ifdef CONFIG_REGD_SRC_FROM_OS
+	if (RFCTL_REGD_SRC_FROM_OS(rfctl)) {
+		/* regd_src from OS has its own 802.11d behavior, do not allow change */
+		confs->set_types &= ~(BIT(CHPLAN_CONFS_CIS_EN_MODE)
+					| BIT(CHPLAN_CONFS_CIS_FLAGS)
+					| BIT(CHPLAN_CONFS_CIS_EN_ROLE)
+					| BIT(CHPLAN_CONFS_CIS_EN_IFBMP)
+					| BIT(CHPLAN_CONFS_CIS_SCAN_INT_MS)
+					);
+	}
+	#endif
+	if (confs->set_types & BIT(CHPLAN_CONFS_CIS_EN_MODE)) {
+		if (CIS_EN_MODE_IS_VALID(confs->cis_en_mode)
+			&& rfctl->cis_en_mode != confs->cis_en_mode
+		) {
+			rfctl->cis_en_mode = confs->cis_en_mode;
+			RTW_INFO("%s set cis_en_mode to %u\n", caller, confs->cis_en_mode);
+		} else
+			confs->set_types &= ~BIT(CHPLAN_CONFS_CIS_EN_MODE);
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_CIS_FLAGS)) {
+		if (confs->cis_flags & ~CISF_VALIDS) {
+			RTW_WARN("%s cis_flags:0x%02x has undefined bits, apply valid bits only\n", caller, confs->cis_flags);
+			confs->cis_flags &= CISF_VALIDS;
+		}
+		if (rfctl->cis_flags != confs->cis_flags) {
+			rfctl->cis_flags = confs->cis_flags;
+			RTW_INFO("%s set cis_flags to 0x%02x\n", caller, confs->cis_flags);
+		} else
+			confs->set_types &= ~BIT(CHPLAN_CONFS_CIS_FLAGS);
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_CIS_EN_ROLE)) {
+		if (rfctl->cis_en_role != confs->cis_en_role) {
+			rfctl->cis_en_role = confs->cis_en_role;
+			RTW_INFO("%s set cis_en_role to 0x%02x\n", caller, confs->cis_en_role);
+		}
+		confs->set_types &= ~BIT(CHPLAN_CONFS_CIS_EN_ROLE); /* setting done here */
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_CIS_EN_IFBMP)) {
+		if (rfctl->cis_en_ifbmp != confs->cis_en_ifbmp) {
+			rfctl->cis_en_ifbmp = confs->cis_en_ifbmp;
+			RTW_INFO("%s set cis_en_ifbmp to 0x%02x\n", caller, confs->cis_en_ifbmp);
+		}
+		confs->set_types &= ~BIT(CHPLAN_CONFS_CIS_EN_IFBMP); /* setting done here */
+	}
+
+	if (confs->set_types & BIT(CHPLAN_CONFS_CIS_SCAN_INT_MS)) {
+		if (rfctl->cis_scan_int_ms != confs->cis_scan_int_ms) {
+			rfctl->cis_scan_int_ms = confs->cis_scan_int_ms;
+			RTW_INFO("%s set cis_scan_int_ms to %u\n", caller, confs->cis_scan_int_ms);
+		}
+		confs->set_types &= ~BIT(CHPLAN_CONFS_CIS_SCAN_INT_MS); /* setting done here */
+	}
+#endif
+}
+
+static bool rtw_chplan_rtk_priv_req_prehdl_user(struct rf_ctl_t *rfctl, struct SetChannelPlan_param *param, const char *caller)
+{
+	bool effected = false;
+
+	if (param->channel_plan != RTW_CHPLAN_UNSPECIFIED)
+		param->priv_user_set_bmp |= RTW_PRIV_USER_SET_DOMAIN;
+	if (param->has_country)
+		param->priv_user_set_bmp |= RTW_PRIV_USER_SET_COUNTRY;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (param->channel_plan_6g != RTW_CHPLAN_6G_UNSPECIFIED)
+		param->priv_user_set_bmp |= RTW_PRIV_USER_SET_DOMAIN_6G;
+	if (param->env < RTW_ENV_NUM)
+		param->priv_user_set_bmp |= RTW_PRIV_USER_SET_ENV;
+#endif
+
+	if (!param->priv_user_set_bmp && !param->confs.set_types) {
+		/* meaningless input */
+		RTW_WARN("%s meaningless input\n", caller);
+		goto exit;
+	}
+
+	/* check input parameter */
+	if (rtw_chplan_rtk_priv_req_prehdl_domain_code(rfctl, param, __func__) != _SUCCESS)
+		goto exit;
+
+	if (param->priv_user_set_bmp & (RTW_PRIV_USER_SET_DOMAIN | RTW_PRIV_USER_SET_DOMAIN_6G
+		| RTW_PRIV_USER_SET_COUNTRY | RTW_PRIV_USER_SET_ENV)
+	) {
+		rtw_chplan_rtk_priv_req_prehdl_country_ent(rfctl, param);
+
+		/* allows only one user request */
+		if (!rfctl->user_req) {
+			rfctl->user_req = rtw_regd_req_alloc_with_country_chplan(&param->country_ent);
+			if (rfctl->user_req) {
+				rfctl->user_req->src = param->regd_src;
+				rfctl->user_req->inr = param->inr;
+				#if CONFIG_IEEE80211_BAND_6GHZ
+				if (param->env >= RTW_ENV_NUM) /* no env specified by first user_req, get from init_req */
+					rfctl->user_req->env = rfctl->init_req.env;
+				else {
+					rfctl->user_req->env = param->env;
+					if (rfctl->init_req.env == param->env)
+						param->priv_user_set_bmp &= ~RTW_PRIV_USER_SET_ENV;
+				}
+				#endif
+				rtw_regd_req_list_insert(rfctl, rfctl->user_req);
+				effected = true;
+			} else
+				rtw_warn_on(1);
+		} else {
+			if (!_rtw_memcmp(&rfctl->user_req->chplan, &param->country_ent, sizeof(param->country_ent))) {
+				_rtw_memcpy(&rfctl->user_req->chplan, &param->country_ent, sizeof(param->country_ent));
+				effected = true;
+			}
+
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (param->env < RTW_ENV_NUM && rfctl->user_req->env != param->env) {
+				rfctl->user_req->env = param->env;
+				effected = true;
+			} else
+				param->priv_user_set_bmp &= ~RTW_PRIV_USER_SET_ENV;
+			#endif
+		}
+	}
+
+	/* check and update confs */
+	rtw_chplan_rtk_priv_req_prehdl_confs(rfctl, &param->confs, caller);
+
+exit:
+	return effected || param->confs.set_types;
+}
+
+#ifdef CONFIG_80211D
+#if CONFIG_IEEE80211_BAND_6GHZ
+static u8 cisr_get_txpwr_lmt_6g_cate_map(struct country_ie_slave_record *cisr)
+{
+	if (cisr->band == BAND_ON_6G) {
+		if (!IS_6G_REG_INFO_RSVD(cisr->reg_info))
+			return reg_info_to_txpwr_lmt_6g_cate_map(cisr->reg_info);
+		return BIT(TXPWR_LMT_6G_CATE_VLP);
+	}
+	return 0;
+}
+#endif
+
+enum cisr_match {
+	CISR_MATCH		= 0, /* identically match */
+	CISR_MATCH_CHPLAN	= 1, /* same chplan result (including txpwr_lmt_6g_cate_bmp) */
+	CISR_DIFF		/* different (not above cases) */
+};
+
+static enum cisr_match rtw_cisr_compare(struct country_ie_slave_record *a, struct country_ie_slave_record *b)
+{
+	if (_rtw_memcmp(a, b, sizeof(*a)) == true)
+		return CISR_MATCH;
+	if (_rtw_memcmp(&a->chplan, &b->chplan, sizeof(a->chplan)) == true
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		&& cisr_get_txpwr_lmt_6g_cate_map(a) == cisr_get_txpwr_lmt_6g_cate_map(b)
+		#endif
+	)
+		return CISR_MATCH_CHPLAN;
+	return CISR_DIFF;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static void rtw_rfctl_update_6g_cate_map_int_all_link(struct rf_ctl_t *rfctl)
+{
+	struct regd_req_t *user_req = rfctl->user_req;
+	int i, j;
+	u8 bmp;
+	u8 applied_link_num;
+	bool include_nocountry = !rfctl->cis_enabled; /* if cis is not enabled, include nocountry directly */
+
+search:
+	bmp = 0;
+	applied_link_num = 0;
+	for (i = 0; i < CONFIG_IFACE_NUMBER; i++) {
+		for (j = 0; j < RTW_RLINK_MAX; j++) {
+			if (rfctl->cisr[i][j].band != BAND_ON_6G
+				|| (!include_nocountry && rfctl->cisr[i][j].status == COUNTRY_IE_SLAVE_NOCOUNTRY))
+				continue;
+
+			if (bmp == 0)
+				bmp = cisr_get_txpwr_lmt_6g_cate_map(&rfctl->cisr[i][j]);
+			else
+				bmp &= cisr_get_txpwr_lmt_6g_cate_map(&rfctl->cisr[i][j]);
+
+			applied_link_num++;
+		}
+	}
+
+	if (!applied_link_num && !include_nocountry) {
+		/* if no applied link, include 6G link with NOCOUNTRY status*/
+		include_nocountry = true;
+		goto search;
+	}
+
+	rfctl->txpwr_lmt_6g_cate_map_int_all_link = bmp;
+	rfctl->txpwr_lmt_6g_cate_map_int_link_num = applied_link_num;
+}
+
+static void rtw_rfctl_get_txpwr_lmt_6g_cate_map_from_init_user(struct rf_ctl_t *rfctl
+	, enum rtw_env_t *r_env, enum country_ie_slave_6g_reg_info *r_reg_info
+	, u8 *r_txpwr_lmt_6g_cate_map)
+{
+	struct regd_req_t *user_req = rfctl->user_req;
+	enum rtw_env_t env = RTW_ENV_NUM;
+	u8 bmp;
+
+	if (user_req)
+		env = user_req->env;
+	if (!user_req || rfctl->init_regd_always_apply) {
+		if (env == RTW_ENV_NUM || rfctl->init_req.env == RTW_ENV_ANY)
+			env = rfctl->init_req.env;
+		else if (env != rfctl->init_req.env)
+			env = RTW_ENV_ANY;
+	}
+
+	bmp = rtw_env_to_txpwr_lmt_6g_cate_map(env);
+
+	*r_env = env;
+	*r_reg_info = CIS_6G_REG_RSVD;
+	*r_txpwr_lmt_6g_cate_map = bmp;
+}
+#endif /*CONFIG_IEEE80211_BAND_6GHZ  */
+
+static bool rtw_chplan_update_per_link_cisr(struct rf_ctl_t *rfctl, u8 iface_id
+	, u8 cisr_alink_id, struct country_ie_slave_record *cisr)
+{
+	struct country_ie_slave_record ori_cisr_cont[RTW_RLINK_MAX];
+	u8 alink_id_s, alink_id_e, alink_id;
+	int i, j;
+	bool effected = false;
+
+	if (cisr_alink_id < RTW_RLINK_MAX) {
+		/* specific alink */
+		alink_id_s = cisr_alink_id;
+		alink_id_e = alink_id_s + 1;
+	} else {
+		/* all alinks of specific iface */
+		alink_id_s = 0;
+		alink_id_e = RTW_RLINK_MAX;
+	}
+
+	/* compare original record with same iface_id & spcified alink_id range */
+	for (alink_id = alink_id_s; alink_id < alink_id_e; alink_id++)
+		if (rtw_cisr_compare(&rfctl->cisr[iface_id][alink_id], cisr) != CISR_MATCH)
+			break;
+	if (alink_id >= alink_id_e) {
+		/* record no change  */
+		goto exit;
+	}
+
+	/* backup original content */
+	for (alink_id = alink_id_s; alink_id < alink_id_e; alink_id++)
+		_rtw_memcpy(&ori_cisr_cont[alink_id], &rfctl->cisr[iface_id][alink_id], sizeof(ori_cisr_cont[alink_id]));
+
+	/* update record */
+	for (alink_id = alink_id_s; alink_id < alink_id_e; alink_id++)
+		_rtw_memcpy(&rfctl->cisr[iface_id][alink_id], cisr, sizeof(*cisr));
+
+	/* compare original record with same iface_id & spcified alink_id range for chplan change */
+	for (alink_id = alink_id_s; alink_id < alink_id_e; alink_id++)
+		if (rtw_cisr_compare(&ori_cisr_cont[alink_id], cisr) > CISR_MATCH_CHPLAN)
+			break;
+	if (alink_id >= alink_id_e) {
+		/* chplan no change  */
+		goto exit;
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	rtw_rfctl_update_6g_cate_map_int_all_link(rfctl);
+#endif
+
+	effected = true;
+
+exit:
+	return effected;
+}
+
+static bool rtw_regd_req_list_add_country_ie_req(struct rf_ctl_t *rfctl
+	, struct country_ie_slave_record *cisr, bool link)
+{
+	struct country_chplan *chplan = &cisr->chplan;
+	struct regd_req_t *req;
+	_list *cur, *head;
+	bool insert = false;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	enum rtw_env_t env;
+	enum country_ie_slave_6g_reg_info reg_info;
+	u8 txpwr_lmt_6g_cate_map;
+
+	if (link) {
+		env = cisr->env;
+		reg_info = cisr->reg_info;
+		txpwr_lmt_6g_cate_map = cisr_get_txpwr_lmt_6g_cate_map(cisr);
+	} else {
+		/*  ENV_BSS doesn't affect env and reg_info, assign value acording to per link status or init/user req */
+		if (rfctl->txpwr_lmt_6g_cate_map_int_link_num) {
+			env = RTW_ENV_NUM;
+			reg_info = CIS_6G_REG_RSVD;
+			txpwr_lmt_6g_cate_map = rfctl->txpwr_lmt_6g_cate_map_int_all_link;
+		} else {
+			rtw_rfctl_get_txpwr_lmt_6g_cate_map_from_init_user(rfctl
+				, &env, &reg_info, &txpwr_lmt_6g_cate_map);
+		}
+	}
+#endif
+
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while (rtw_end_of_queue_search(head, cur) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		if (req->inr == RTW_REGD_SET_BY_COUNTRY_IE
+			&& _rtw_memcmp(&req->chplan, chplan, sizeof(*chplan)) == true
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			/* only compare txpwr_lmt_6g_cate_map */
+			&& req->txpwr_lmt_6g_cate_map == txpwr_lmt_6g_cate_map
+			#endif
+		) {
+			req->ref_cnt++;
+			break;
+		}
+		cur = get_next(cur);
+	}
+	if (rtw_end_of_queue_search(head, cur)) {
+		req = rtw_regd_req_alloc_with_country_chplan(chplan);
+		if (req) {
+			req->src = REGD_SRC_RTK_PRIV;
+			req->inr = RTW_REGD_SET_BY_COUNTRY_IE;
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			req->env = env;
+			req->reg_info = reg_info;
+			req->txpwr_lmt_6g_cate_map = txpwr_lmt_6g_cate_map;
+			#endif
+			req->ref_cnt = 1;
+			rtw_regd_req_list_insert(rfctl, req);
+			insert = true;
+		} else
+			rtw_warn_on(1);
+	}
+
+	return insert;
+}
+
+static bool rtw_regd_req_list_add_country_ie_req_from_per_link_cisr(struct rf_ctl_t *rfctl)
+{
+	int i, j;
+	bool effected = false;
+
+	for (i = 0; i < CONFIG_IFACE_NUMBER; i++) {
+		for (j = 0; j < RTW_RLINK_MAX; j++) {
+			if (rfctl->cisr[i][j].status != COUNTRY_IE_SLAVE_APPLICABLE)
+				continue;
+			effected |= rtw_regd_req_list_add_country_ie_req(rfctl, &rfctl->cisr[i][j], true);
+		}
+	}
+
+	return effected;
+}
+
+static bool rtw_regd_req_list_add_country_ie_req_from_scanned_network_cisr(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	_queue *queue = &mlme->scanned_queue;
+	_list *list, *head;
+	struct wlan_network *scanned;
+	bool effected = false;
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	head = get_list_head(queue);
+	list = get_next(head);
+	while (!rtw_end_of_queue_search(head, list)) {
+		scanned = LIST_CONTAINOR(list, struct wlan_network, list);
+		list = get_next(list);
+
+		if (scanned->cisr.status != COUNTRY_IE_SLAVE_APPLICABLE)
+			continue;
+		effected |= rtw_regd_req_list_add_country_ie_req(rfctl, &scanned->cisr, false);
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+
+	return effected;
+}
+
+static void rtw_regd_req_list_clear_ref_cnt_by_inr(struct rf_ctl_t *rfctl, enum rtw_regd_inr inr)
+{
+	struct regd_req_t *req;
+	_list *cur, *head;
+
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while (rtw_end_of_queue_search(head, cur) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+		if (req->inr == inr)
+			req->ref_cnt = 0;
+	}
+}
+
+static bool rtw_regd_req_list_clear_zero_ref_req_by_inr(struct rf_ctl_t *rfctl, enum rtw_regd_inr inr)
+{
+	struct regd_req_t *req;
+	_list *cur, *head;
+	bool del = false;
+
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while (rtw_end_of_queue_search(head, cur) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+		if (req->inr == inr && req->ref_cnt == 0) {
+			rtw_regd_req_list_delete(rfctl, req);
+			rtw_regd_req_free(req);
+			del = true;
+		}
+	}
+	return del;
+}
+
+static bool rtw_chplan_rtk_priv_req_prehdl_country_ie(_adapter *adapter, struct SetChannelPlan_param *param, const char *caller)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct chplan_confs *confs = &param->confs;
+	u16 apply_confs_bmp = BIT(CHPLAN_CONFS_CIS_EN_MODE) | BIT(CHPLAN_CONFS_CIS_FLAGS);
+	bool confs_change = confs->set_types & apply_confs_bmp;
+	bool effected = false;
+
+	if (param->has_cisr) {
+		bool per_link_cisr_changed;
+
+		#ifdef CONFIG_RTW_DEBUG
+		if (rtw_drv_log_level >= _DRV_DEBUG_) {
+			RTW_PRINT("%s cisr before\n", __func__);
+			dump_country_ie_slave_records(RTW_DBGDUMP, rfctl, 0);
+		}
+		#endif
+
+		per_link_cisr_changed = rtw_chplan_update_per_link_cisr(rfctl
+			, adapter->iface_id, param->cisr_alink_id, &param->cisr);
+		if (!per_link_cisr_changed && !confs_change)
+			goto exit;
+
+		if (!rfctl->cis_enabled || rfctl->regd_src != REGD_SRC_RTK_PRIV)
+			effected = per_link_cisr_changed;
+	}
+
+	if (param->has_cisr) {
+		#ifdef CONFIG_RTW_DEBUG
+		if (rtw_drv_log_level >= _DRV_DEBUG_) {
+			RTW_PRINT("%s cisr after\n", __func__);
+			dump_country_ie_slave_records(RTW_DBGDUMP, rfctl, 0);
+		}
+		#endif
+	}
+
+	if (rfctl->regd_src == REGD_SRC_RTK_PRIV) {
+		rtw_regd_req_list_clear_ref_cnt_by_inr(rfctl, RTW_REGD_SET_BY_COUNTRY_IE);
+
+		if (rfctl->cis_enabled) {
+			effected |= rtw_regd_req_list_add_country_ie_req_from_per_link_cisr(rfctl);
+			if (rfctl->cis_flags & CISF_ENV_BSS)
+				effected |= rtw_regd_req_list_add_country_ie_req_from_scanned_network_cisr(adapter);
+		}
+
+		effected |= rtw_regd_req_list_clear_zero_ref_req_by_inr(rfctl, RTW_REGD_SET_BY_COUNTRY_IE);
+	}
+	confs->set_types &= ~apply_confs_bmp;
+
+exit:
+	return effected;
+}
+#endif /* CONFIG_80211D */
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+static bool rtw_chplan_req_prehdl_from_os(_adapter *adapter, struct SetChannelPlan_param *param, const char *caller)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	bool effected = false;
+	struct regd_req_t *req;
+	_list *cur, *head;
+
+#ifdef CONFIG_80211D
+	/* adjust cis settings */
+	rfctl->cis_en_mode = rtw_os_get_cis_en_mode(adapter);
+	rfctl->cis_flags = 0;
+	rfctl->cis_en_role = COUNTRY_IE_SLAVE_EN_ROLE_STA | COUNTRY_IE_SLAVE_EN_ROLE_GC;
+	rfctl->cis_en_ifbmp = 0xFF;
+	rtw_rfctl_update_cis_enabled(rfctl, false, __func__);
+	rtw_rfctl_update_cisr_collect_mode(rfctl, false, __func__);
+#endif
+
+	/*
+	* two reqs only:
+	* 1. init req and
+	* 2. another recent req
+	*/
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	if (param->inr == RTW_REGD_SET_BY_USER
+		&& param->channel_plan == RTW_CHPLAN_UNSPECIFIED
+		&& param->channel_plan_6g == RTW_CHPLAN_6G_UNSPECIFIED
+		&& !param->has_country
+	) {
+		/* special OS-USER req to set default environment */
+		if (param->env < RTW_ENV_NUM) {
+			bool has_country_ie = false;
+			bool non_country_ie_env_changed = false;
+
+			head = &rfctl->regd_req_list;
+			cur = get_next(head);
+			while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+				req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+				cur = get_next(cur);
+				if (req->inr != RTW_REGD_SET_BY_COUNTRY_IE) {
+					if (req->env != param->env) {
+						req->env = param->env;
+						non_country_ie_env_changed = true;
+					}
+				} else
+					has_country_ie = true;
+			}
+			if (!has_country_ie && non_country_ie_env_changed) {
+				effected = true;
+				goto exit;
+			}
+		}
+		goto exit;
+	} else
+	#endif
+	{
+		head = &rfctl->regd_req_list;
+		cur = get_next(head);
+		while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+			req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+			cur = get_next(cur);
+			if (req->inr == RTW_REGD_SET_BY_INIT)
+				continue;
+			if (req->inr == RTW_REGD_SET_BY_USER)
+				rfctl->user_req = NULL;
+			rtw_regd_req_list_delete(rfctl, req);
+			rtw_regd_req_free(req);
+		}
+
+		req = rtw_regd_req_alloc_with_country_chplan(&param->country_ent);
+		if (req) {
+			req->src = param->regd_src;
+			req->inr = param->inr;
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (req->inr == RTW_REGD_SET_BY_COUNTRY_IE && param->env < RTW_ENV_NUM)
+				req->env = param->env;
+			else
+				req->env = rfctl->init_req.env;
+			#endif
+			rtw_regd_req_list_insert(rfctl, req);
+			if (req->inr == RTW_REGD_SET_BY_USER)
+				rfctl->user_req = req;
+		} else {
+			rtw_warn_on(1);
+			goto exit;
+		}
+	}
+
+	effected = true;
+
+exit:
+	return effected;
+}
+#endif /* CONFIG_REGD_SRC_FROM_OS */
+
+static bool rtw_chplan_req_prehdl(_adapter *adapter, struct SetChannelPlan_param *param, const char *caller)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	bool effected = false;
+	char buf[64];
+
+	get_str_of_set_chplan_keys(buf, 64, param);
+	RTW_INFO("%s iface_id:%u src:%s inr:%s %s\n", caller, adapter->iface_id
+		, regd_src_str(param->regd_src), regd_inr_str(param->inr), buf);
+
+	if (param->inr == RTW_REGD_SET_BY_INIT) {
+		/* init should not done here */
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (param->regd_src == REGD_SRC_RTK_PRIV) {
+		if (param->inr == RTW_REGD_SET_BY_USER) {
+			#ifdef CONFIG_80211D
+			struct chplan_confs *confs = &param->confs;
+			#endif
+
+			if (!rtw_chplan_rtk_priv_req_prehdl_user(rfctl, param, __func__))
+				goto exit;
+
+			#ifdef CONFIG_80211D
+			/*
+			* rfctl.regd_src is not allowed to change from REGD_SRC_OS to  REGD_SRC_RTK_PRIV
+			* get new status and update to rfctl.cis_enabled here is safe and necessary for logic below
+			*/
+			if (rtw_rfctl_update_cis_enabled(rfctl, false, __func__)) {
+				rtw_rfctl_update_cisr_collect_mode(rfctl, false, __func__);
+				/* enable status change */
+				confs->set_types |= BIT(CHPLAN_CONFS_CIS_EN_MODE);
+			}
+
+			if ((confs->set_types & BIT(CHPLAN_CONFS_CIS_FLAGS))
+				|| (confs->set_types & BIT(CHPLAN_CONFS_CIS_EN_MODE))
+				|| (param->priv_user_set_bmp & RTW_PRIV_USER_SET_ENV)
+			) {
+				if (!rtw_chplan_rtk_priv_req_prehdl_country_ie(adapter, param, __func__)
+					&& !confs->set_types && !param->priv_user_set_bmp)
+					goto exit;
+			}
+			#endif
+		}
+		#ifdef CONFIG_80211D
+		else if (param->inr == RTW_REGD_SET_BY_COUNTRY_IE) {
+			if (!rtw_chplan_rtk_priv_req_prehdl_country_ie(adapter, param, __func__))
+				goto exit;
+		}
+		#endif
+		else {
+			rtw_warn_on(1);
+			goto exit;
+		}
+	}
+#ifdef CONFIG_REGD_SRC_FROM_OS
+	else if (param->regd_src == REGD_SRC_OS) {
+		if (!rtw_chplan_req_prehdl_from_os(adapter, param, caller))
+			goto exit;
+	}
+#endif
+	else {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	effected = true;
+
+exit:
+#ifdef CONFIG_80211D
+	rfctl->init_user_req_is_ww = rtw_rfctl_is_init_user_req_world_wide(rfctl);
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	rtw_rfctl_update_default_chplan_cate_6g_map(rfctl, false, caller);
+	rtw_rfctl_update_regd_req_txpwr_6g_cate_map(rfctl);
+	#endif
+#endif
+
+	return effected;
+}
+
+static void rtw_rfctl_regd_req_sel_and_status_update(struct rf_ctl_t *rfctl)
+{
+	struct regd_req_t *req;
+	struct country_chplan *chplan;
+	_list *cur, *head;
+	enum regd_src_t applied_src = REGD_SRC_NUM;
+	enum rtw_regd_inr applied_inr = RTW_REGD_SET_BY_NUM;
+
+	/* decide applied_src (highest src) */
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+		if (applied_src == REGD_SRC_NUM || req->src > applied_src)
+			applied_src = req->src;
+	}
+
+	/* decide applied_inr (highest inr) */
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+		if (req->src != applied_src)
+			continue;
+		if (applied_inr == RTW_REGD_SET_BY_NUM || req->inr > applied_inr)
+			applied_inr = req->inr;
+	}
+
+	rfctl->regd_src = REGD_SRC_NUM;
+	head = &rfctl->regd_req_list;
+	cur = get_next(head);
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		req = LIST_CONTAINOR(cur, struct regd_req_t, list);
+		cur = get_next(cur);
+
+		/* mark req with different src as not applied*/
+		if (applied_src != req->src) {
+			req->applied = false;
+			continue;
+		}
+		/* mark req with target inr or always_apply INIT/USER req as applied*/
+		req->applied = applied_inr == req->inr
+			|| (req->inr == RTW_REGD_SET_BY_INIT && rfctl->init_regd_always_apply)
+			|| (req->inr == RTW_REGD_SET_BY_USER && rfctl->user_regd_always_apply);
+		if (!req->applied)
+			continue;
+
+		chplan = &req->chplan;
+		if (rfctl->regd_src == REGD_SRC_NUM) {
+			/* first applied req */
+			rtw_rfctl_regd_status_init_by_req(rfctl, req);
+
+		} else if (rfctl->regd_src == req->src) {
+			rfctl->regd_inr_bmp |= BIT(req->inr);
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			rfctl->env_bmp |= BIT(req->env);
+			#endif
+			if (_rtw_memcmp(rfctl->alpha2, chplan->alpha2, 2) != _TRUE
+				|| rfctl->domain_code != chplan->domain_code
+				#if CONFIG_IEEE80211_BAND_6GHZ
+				|| rfctl->domain_code_6g != chplan->domain_code_6g
+				#endif
+			)
+				SET_INTERSECTEDC_ALPHA2(rfctl->alpha2);
+			if (rfctl->domain_code != chplan->domain_code)
+				rfctl->domain_code = RTW_CHPLAN_UNSPECIFIED;
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			if (rfctl->domain_code_6g != chplan->domain_code_6g)
+				rfctl->domain_code_6g = RTW_CHPLAN_UNSPECIFIED;
+			#endif
+			#ifdef CONFIG_CHPLAN_PROTO_EN
+			rfctl->proto_en &= chplan->proto_en;
+			#endif
+		} else {
+			RTW_WARN("%s req->src:%s != applied src:%s\n", __func__, regd_src_str(req->src), regd_src_str(rfctl->regd_src));
+			rtw_warn_on(1);
+		}
+	}
+
+	#ifdef CONFIG_80211D
+	if (rtw_rfctl_update_cis_enabled(rfctl, false, __func__))
+		rtw_rfctl_update_cisr_collect_mode(rfctl, false, __func__);
+	#endif
+}
+
+#if defined(CONFIG_AP_MODE) && CONFIG_AP_REGU_FORBID
+bool rtw_rfctl_is_regu_forbid_bss(struct rf_ctl_t *rfctl, enum band_type band)
+{
+	if (!rtw_txpwr_hal_is_txpwr_limit_needed(rfctl_to_dvobj(rfctl)))
+		return false;
+
+	/* forbid before 6G AP regulatory ready */
+	return band == BAND_ON_6G;
+}
+
+static void rtw_bss_regu_status_update(struct rf_ctl_t *rfctl)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	u8 band_idx;
+
+	/* TODO: update possible AP cate */
+
+	for (band_idx = HW_BAND_0; band_idx < HW_BAND_MAX; band_idx++) {
+		_adapter *iface;
+		struct _ADAPTER_LINK *alink;
+		int i, j;
+		bool forbid;
+		u8 ifbmp_m = rtw_mi_get_ap_mesh_ifbmp_by_hwband(dvobj, band_idx);
+		u8 forbid_cnt = 0;
+		bool sta_link = false;
+
+		/* loop all bss to apply regu status (reg_info, forbid) */
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if (!iface || !(ifbmp_m & BIT(iface->iface_id)))
+				continue;
+
+			for (j = 0; j < ADAPTER_GET_LINK_NUM(iface); j++) {
+				alink = GET_LINK(iface, j);
+				if (ALINK_GET_HWBAND(alink) != band_idx)
+					continue;
+
+				forbid = rtw_rfctl_is_regu_forbid_bss(rfctl, ALINK_GET_BAND(alink));
+				if (forbid)
+					forbid_cnt++;
+
+				/* TODO: update reg_info for 6G AP */
+
+				/* update forbid status */
+				rtw_ap_link_regu_forbid_apply(alink, forbid, false);
+			}
+		}
+
+		if (!forbid_cnt || DEV_MCC_CAPABLE(dvobj))
+			continue;
+
+		/* check if having sta link exist */
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if (!iface || (ifbmp_m & BIT(iface->iface_id)) || !rtw_is_adapter_up(iface))
+				continue;
+
+			for (j = 0; j < ADAPTER_GET_LINK_NUM(iface); j++) {
+				alink = GET_LINK(iface, j);
+				if (ALINK_GET_HWBAND(alink) != band_idx)
+					continue;
+				if (rfctl->cisr[i][j].opch) {
+					sta_link = true;
+					i = dvobj->iface_nums;
+					break;
+				}
+			}
+		}
+
+		if (!sta_link) {
+			/* no sta link, trigger bss ch select to get out of forbid status */
+			_adapter *m_iface = rtw_mi_get_ap_mesh_iface_by_hwband(dvobj, band_idx);
+			u8 ifbmp_s = rtw_mi_get_lgd_sta_ifbmp_by_hwband(dvobj, band_idx);
+
+			RTW_INFO(FUNC_HWBAND_FMT" trigger ch select for forbid bss\n", FUNC_HWBAND_ARG(band_idx));
+
+			if (!m_iface) {
+				rtw_warn_on(1);
+				continue;
+			}
+
+			rtw_change_bss_bchbw_cmd(m_iface, RTW_CMDF_DIRECTLY
+				, ifbmp_m, ifbmp_s, REQ_BAND_NONE, REQ_CH_NONE, REQ_BW_ORI, REQ_OFFSET_NONE);
+		}
+	}
+}
+#endif /* defined(CONFIG_AP_MODE) && CONFIG_AP_REGU_FORBID */
+
+u8 rtw_set_chplan_hdl(_adapter *adapter, u8 *pbuf)
+{
+	struct SetChannelPlan_param *param;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+#ifdef CONFIG_IOCTL_CFG80211
+	struct get_chplan_resp *chplan;
+#endif
+
+	if (!pbuf)
+		return H2C_PARAMETERS_ERROR;
+
+	param = (struct SetChannelPlan_param *)pbuf;
+
+	_rtw_mutex_lock_interruptible(&rfctl->regd_req_mutex);
+
+	if (!rtw_chplan_req_prehdl(adapter, param, __func__)) {
+		_rtw_mutex_unlock(&rfctl->regd_req_mutex);
+		goto exit;
+	}
+
+	rtw_rfctl_regd_req_sel_and_status_update(rfctl);
+
+#if CONFIG_TXPWR_LIMIT
+	rtw_txpwr_update_cur_lmt_regs(dvobj, false);
+#endif
+
+	rtw_edcca_mode_update(dvobj, false);
+
+	rtw_rfctl_chset_apply_regulatory(dvobj, false);
+#ifdef CONFIG_80211D
+	rtw_rfctl_update_cisr_collect_mode(rfctl, false, __func__);
+#endif
+
+	_rtw_mutex_unlock(&rfctl->regd_req_mutex);
+
+	op_class_pref_apply_regulatory(rfctl, REG_CHANGE);
+	init_channel_list(adapter);
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (rtw_get_chplan_cmd(adapter, RTW_CMDF_DIRECTLY, &chplan) == _SUCCESS) {
+		if (!param->rtnl_lock_needed)
+			rtw_regd_change_complete_sync(adapter_to_wiphy(adapter), chplan, 0);
+		else
+			rtw_warn_on(rtw_regd_change_complete_async(adapter_to_wiphy(adapter), chplan) != _SUCCESS);
+	} else
+		rtw_warn_on(1);
+#endif
+
+	rtw_nlrtw_reg_change_event(adapter);
+
+	if (rtw_txpwr_hal_get_pwr_lmt_en(dvobj) && rtw_hw_is_init_completed(dvobj))
+		rtw_update_txpwr_level(dvobj, HW_BAND_MAX);
+
+exit:
+
+#if defined(CONFIG_AP_MODE) && CONFIG_AP_REGU_FORBID
+	rtw_bss_regu_status_update(rfctl);
+#endif
+
+	return	H2C_SUCCESS;
+}
+
+static u8 _rtw_set_chplan_cmd(_adapter *adapter, int flags
+	, u8 chplan, u8 chplan_6g, const struct country_chplan *country_ent
+	, enum rtw_env_t env, enum regd_src_t regd_src, enum rtw_regd_inr inr
+	, const struct country_ie_slave_record *cisr, u8 cisr_alink_id
+	, struct chplan_confs *confs)
+{
+	struct cmd_obj *cmdobj;
+	struct SetChannelPlan_param *parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+#ifdef PLATFORM_LINUX
+	bool rtnl_lock_needed = rtw_rtnl_lock_needed(adapter_to_dvobj(adapter));
+#endif
+	u8 res = _SUCCESS;
+
+	/* check if allow software config */
+	if (rtw_rfctl_is_disable_sw_channel_plan(adapter_to_dvobj(adapter)) == _TRUE) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	if (country_ent) {
+		/* if country_entry is provided, replace chplan */
+		chplan = country_ent->domain_code;
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		chplan_6g = country_ent->domain_code_6g;
+		#endif
+	}
+
+	/* prepare cmd parameter */
+	parm = rtw_zmalloc(sizeof(*parm));
+	if (parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	parm->regd_src = regd_src;
+	parm->inr = inr;
+	if (country_ent) {
+		_rtw_memcpy(&parm->country_ent, country_ent, sizeof(parm->country_ent));
+		parm->has_country = true;
+	}
+	parm->channel_plan = chplan;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	parm->channel_plan_6g = chplan_6g;
+	parm->env = env;
+#endif
+#ifdef CONFIG_80211D
+	if (cisr) {
+		_rtw_memcpy(&parm->cisr, cisr, sizeof(*cisr));
+		parm->cisr_alink_id = cisr_alink_id;
+		parm->has_cisr = true;
+	}
+#endif
+
+	if (confs)
+		_rtw_memcpy(&parm->confs, confs, sizeof(parm->confs));
+
+#ifdef PLATFORM_LINUX
+	if (flags & (RTW_CMDF_DIRECTLY | RTW_CMDF_WAIT_ACK))
+		parm->rtnl_lock_needed = rtnl_lock_needed; /* synchronous call, follow caller's */
+	else
+		parm->rtnl_lock_needed = 1; /* asynchronous call, always needed */
+#endif
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != rtw_set_chplan_hdl(adapter, (u8 *)parm))
+			res = _FAIL;
+		rtw_mfree(parm, sizeof(*parm));
+	} else {
+		/* need enqueue, prepare cmd_obj and enqueue */
+		cmdobj = rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree(parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_CHANPLAN);
+		CMD_OBJ_SET_HWBAND(cmdobj, HW_BAND_0);
+
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 2000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+			if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+				res = _FAIL;
+		}
+
+		/* allow set channel plan when cmd_thread is not running */
+		if (res != _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			parm = rtw_zmalloc(sizeof(*parm));
+			if (parm == NULL) {
+				res = _FAIL;
+				goto exit;
+			}
+			parm->regd_src = regd_src;
+			parm->inr = inr;
+			if (country_ent) {
+				_rtw_memcpy(&parm->country_ent, country_ent, sizeof(parm->country_ent));
+				parm->has_country = true;
+			}
+			parm->channel_plan = chplan;
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			parm->channel_plan_6g = chplan_6g;
+			parm->env = env;
+			#endif
+			#ifdef CONFIG_80211D
+			if (cisr) {
+				_rtw_memcpy(&parm->cisr, cisr, sizeof(*cisr));
+				parm->cisr_alink_id = cisr_alink_id;
+				parm->has_cisr = true;
+			}
+			#endif
+			if (confs)
+				_rtw_memcpy(&parm->confs, confs, sizeof(parm->confs));
+			#ifdef PLATFORM_LINUX
+			parm->rtnl_lock_needed = rtnl_lock_needed; /* synchronous call, follow caller's */
+			#endif
+
+			if (H2C_SUCCESS != rtw_set_chplan_hdl(adapter, (u8 *)parm))
+				res = _FAIL;
+			else
+				res = _SUCCESS;
+			rtw_mfree(parm, sizeof(*parm));
+		}
+	}
+
+exit:
+	return res;
+}
+
+u8 rtw_set_chplan_cmd(_adapter *adapter, int flags, u8 chplan, u8 chplan_6g
+	, enum rtw_env_t env, enum rtw_regd_inr inr)
+{
+	return _rtw_set_chplan_cmd(adapter, flags
+		, chplan, chplan_6g, NULL, env
+		, REGD_SRC_RTK_PRIV, inr
+		, NULL, RTW_RLINK_MAX
+		, NULL);
+}
+
+u8 rtw_set_country_cmd(_adapter *adapter, int flags, const char *country_code
+	, enum rtw_env_t env, enum rtw_regd_inr inr)
+{
+	struct country_chplan ent;
+
+	if (IS_ALPHA2_WORLDWIDE(country_code)) {
+		rtw_get_chplan_worldwide(&ent);
+		goto cmd;
+	}
+
+	if (is_alpha(country_code[0]) == _FALSE
+	    || is_alpha(country_code[1]) == _FALSE
+	   ) {
+		RTW_PRINT("%s input country_code is not alpha2\n", __func__);
+		return _FAIL;
+	}
+
+	if (!rtw_get_chplan_from_country(country_code, &ent)) {
+		RTW_PRINT("%s unsupported country_code:\"%c%c\"\n", __func__, country_code[0], country_code[1]);
+		return _FAIL;
+	}
+
+cmd:
+	RTW_PRINT("%s country_code:\"%c%c\"\n", __func__, country_code[0], country_code[1]);
+
+	return _rtw_set_chplan_cmd(adapter, flags
+		, RTW_CHPLAN_UNSPECIFIED, RTW_CHPLAN_6G_UNSPECIFIED, &ent, env
+		, REGD_SRC_RTK_PRIV, inr
+		, NULL, RTW_RLINK_MAX
+		, NULL);
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+u8 rtw_set_env_cmd(_adapter *adapter, int flags, enum rtw_env_t env
+	, enum regd_src_t regd_src, enum rtw_regd_inr inr)
+{
+	return _rtw_set_chplan_cmd(adapter, flags
+		, RTW_CHPLAN_UNSPECIFIED, RTW_CHPLAN_6G_UNSPECIFIED, NULL, env
+		, regd_src, inr
+		, NULL, RTW_RLINK_MAX
+		, NULL);
+}
+#endif
+
+u8 rtw_set_chplan_confs_cmd(_adapter *adapter, int flags, struct chplan_confs *confs)
+{
+	return _rtw_set_chplan_cmd(adapter, flags
+		, RTW_CHPLAN_UNSPECIFIED, RTW_CHPLAN_6G_UNSPECIFIED, NULL, RTW_ENV_NUM
+		, REGD_SRC_RTK_PRIV, RTW_REGD_SET_BY_USER
+		, NULL, RTW_RLINK_MAX
+		, confs);
+}
+
+#ifdef CONFIG_PROC_DEBUG
+static const char *const chplan_confs_type_str[] = {
+	[CHPLAN_CONFS_DIS_CH_FLAGS]		= "dis_ch_flags",
+	[CHPLAN_CONFS_EXCL_CHS]			= "excl_chs",
+	[CHPLAN_CONFS_EXCL_CHS_6G]		= "excl_chs_6g",
+	[CHPLAN_CONFS_INIT_REGD_ALWAYS_APPLY]	= "init_regd_always_apply",
+	[CHPLAN_CONFS_USER_REGD_ALWAYS_APPLY]	= "user_regd_always_apply",
+	[CHPLAN_CONFS_BCN_HINT_VALID_MS]	= "bcn_hint_valid_ms",
+	[CHPLAN_CONFS_EDCCA_MODE_SEL]		= "edcca_mode_sel",
+	[CHPLAN_CONFS_CIS_EN_MODE]		= "cis_en_mode",
+	[CHPLAN_CONFS_CIS_FLAGS]		= "cis_flags",
+	[CHPLAN_CONFS_CIS_EN_IFBMP]		= "cis_en_ifbmp",
+	[CHPLAN_CONFS_CIS_EN_ROLE]		= "cis_en_role",
+	[CHPLAN_CONFS_CIS_SCAN_INT_MS]		= "cis_scan_int_ms",
+};
+
+static enum chplan_confs_type get_chplan_confs_type_from_str(const char *str, size_t str_len)
+{
+	u8 i;
+
+	for (i = 0; i < CHPLAN_CONFS_NUM; i++)
+		if (str_len == strlen(chplan_confs_type_str[i])
+			&& strncmp(str, chplan_confs_type_str[i], str_len) == 0)
+			return i;
+	return CHPLAN_CONFS_NUM;
+}
+
+static void rtw_parse_chplan_confs_type_arg_str(struct chplan_confs *confs, enum chplan_confs_type type, char *str)
+{
+	if (type == CHPLAN_CONFS_DIS_CH_FLAGS) {
+		confs->set_types |= BIT(type);
+		if (strlen(str)) {
+			char *c;
+			enum rtw_ch_type ch_type;
+
+			for (c = strsep(&str, ","); c; c = strsep(&str, ",")) {
+				ch_type = get_ch_type_from_str(c, strlen(c));
+				if (ch_type != RTW_CHT_NUM)
+					confs->dis_ch_flags |= BIT(ch_type);
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_EXCL_CHS) {
+		confs->set_types |= BIT(type);
+		if (strlen(str)) {
+			char *c;
+			int ch, ch_num = 0;
+
+			for (c = strsep(&str, ","); c && ch_num <  MAX_CHANNEL_NUM_2G_5G; c = strsep(&str, ",")) {
+				if (sscanf(c, "%d", &ch) == 1 && ch > 0 && ch < 256)
+					confs->excl_chs[ch_num++] = ch;
+			}
+		}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	} else if (type == CHPLAN_CONFS_EXCL_CHS_6G) {
+		confs->set_types |= BIT(type);
+		if (strlen(str)) {
+			char *c;
+			int ch, ch_num = 0;
+
+			for (c = strsep(&str, ","); c && ch_num < MAX_CHANNEL_NUM_6G; c = strsep(&str, ",")) {
+				if (sscanf(c, "%d", &ch) == 1 && ch > 0 && ch < 256)
+					confs->excl_chs_6g[ch_num++] = ch;
+			}
+		}
+#endif
+
+	} else if (type == CHPLAN_CONFS_INIT_REGD_ALWAYS_APPLY) {
+		if (strlen(str)) {
+			int val;
+
+			if (sscanf(str, "%d", &val) == 1) {
+				confs->set_types |= BIT(type);
+				confs->init_regd_always_apply = !!val;
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_USER_REGD_ALWAYS_APPLY) {
+		if (strlen(str)) {
+			int val;
+
+			if (sscanf(str, "%d", &val) == 1) {
+				confs->set_types |= BIT(type);
+				confs->user_regd_always_apply = !!val;
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_BCN_HINT_VALID_MS) {
+		if (strlen(str)) {
+			u32 ms;
+
+			if (sscanf(str, "%u", &ms) == 1) {
+				confs->set_types |= BIT(type);
+				confs->bcn_hint_valid_ms = ms;
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_EDCCA_MODE_SEL) {
+		if (strlen(str)) {
+			u8 mode;
+
+			if (sscanf(str, "%hhu", &mode) == 1) {
+				confs->set_types |= BIT(type);
+				confs->edcca_mode_sel = mode;
+			}
+		}
+
+#ifdef CONFIG_80211D
+	} else if (type == CHPLAN_CONFS_CIS_EN_MODE) {
+		if (strlen(str)) {
+			u8 mode;
+
+			if (sscanf(str, "%hhu", &mode) == 1) {
+				confs->set_types |= BIT(type);
+				confs->cis_en_mode = mode;
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_CIS_FLAGS) {
+		if (strlen(str)) {
+			u8 flags;
+
+			if (sscanf(str, "%hhx", &flags) == 1) {
+				confs->set_types |= BIT(type);
+				confs->cis_flags = flags;
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_CIS_EN_ROLE) {
+		if (strlen(str)) {
+			u8 role;
+
+			if (sscanf(str, "%hhx", &role) == 1) {
+				confs->set_types |= BIT(type);
+				confs->cis_en_role = role;
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_CIS_EN_IFBMP) {
+		if (strlen(str)) {
+			u8 ifbmp;
+
+			if (sscanf(str, "%hhx", &ifbmp) == 1) {
+				confs->set_types |= BIT(type);
+				confs->cis_en_ifbmp = ifbmp;
+			}
+		}
+
+	} else if (type == CHPLAN_CONFS_CIS_SCAN_INT_MS) {
+		if (strlen(str)) {
+			u32 ms;
+
+			if (sscanf(str, "%u", &ms) == 1) {
+				confs->set_types |= BIT(type);
+				confs->cis_scan_int_ms = ms;
+			}
+		}
+#endif /* CONFIG_80211D */
+
+	}
+}
+
+u16 rtw_parse_chplan_confs_cmd_str(struct chplan_confs *confs, char *str)
+{
+	char *next = str, *c, *equal;
+	enum chplan_confs_type type;
+
+	_rtw_memset(confs, 0, sizeof(*confs));
+
+	for (c = strsep(&next, " \t\n\r"); c; c = strsep(&next, " \t\n\r")) {
+		equal = strchr(c, '=');
+		if (!equal || c == equal)
+			continue;
+		type = get_chplan_confs_type_from_str(c, equal - c);
+		if (type == CHPLAN_CONFS_NUM)
+			continue;
+		rtw_parse_chplan_confs_type_arg_str(confs, type, equal + 1);
+	}
+
+	return confs->set_types;
+}
+#endif /* CONFIG_PROC_DEBUG */
+
+#ifdef CONFIG_80211D
+u8 rtw_alink_apply_recv_regu_ies_cmd(struct _ADAPTER_LINK *alink, int flags, enum band_type band,u8 opch
+	, const u8 *country_ie, enum country_ie_slave_6g_reg_info reg_info)
+{
+	struct country_ie_slave_record cisr;
+
+	rtw_get_cisr_from_recv_regu_ies(adapter_to_rfctl(alink->adapter), band, opch
+		, country_ie, reg_info
+		, &cisr);
+
+	return _rtw_set_chplan_cmd(alink->adapter, flags
+		, RTW_CHPLAN_UNSPECIFIED, RTW_CHPLAN_6G_UNSPECIFIED, NULL, RTW_ENV_NUM
+		, REGD_SRC_RTK_PRIV, RTW_REGD_SET_BY_COUNTRY_IE
+		, &cisr, rtw_adapter_link_get_id(alink)
+		, NULL);
+}
+
+u8 rtw_apply_recv_regu_ies_cmd(_adapter *adapter, int flags, enum band_type band,u8 opch
+	, const u8 *country_ie, enum country_ie_slave_6g_reg_info reg_info)
+{
+	struct country_ie_slave_record cisr;
+
+	rtw_get_cisr_from_recv_regu_ies(adapter_to_rfctl(adapter), band, opch
+		, country_ie, reg_info
+		, &cisr);
+
+	return _rtw_set_chplan_cmd(adapter, flags
+		, RTW_CHPLAN_UNSPECIFIED, RTW_CHPLAN_6G_UNSPECIFIED, NULL, RTW_ENV_NUM
+		, REGD_SRC_RTK_PRIV, RTW_REGD_SET_BY_COUNTRY_IE
+		, &cisr, RTW_RLINK_MAX
+		, NULL);
+}
+
+u8 rtw_apply_scan_network_country_ie_cmd(_adapter *adapter, int flags)
+{
+	return _rtw_set_chplan_cmd(adapter, flags
+		, RTW_CHPLAN_UNSPECIFIED, RTW_CHPLAN_6G_UNSPECIFIED, NULL, RTW_ENV_NUM
+		, REGD_SRC_RTK_PRIV, RTW_REGD_SET_BY_COUNTRY_IE
+		, NULL, RTW_RLINK_MAX
+		, NULL);
+}
+#endif /* CONFIG_80211D */
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+u8 rtw_sync_os_regd_cmd(_adapter *adapter, int flags, const char *country_code
+	, u8 dfs_region, enum rtw_env_t env, enum rtw_regd_inr inr)
+{
+	struct country_chplan ent;
+	struct country_chplan rtk_ent;
+	bool rtk_ent_exist;
+
+	rtk_ent_exist = rtw_get_chplan_from_country(country_code, &rtk_ent);
+
+	_rtw_memcpy(ent.alpha2, country_code, 2);
+
+	/*
+	* Regulation follows OS, the internal txpwr limit selection is searched by alpha2
+	*     "00" => WW, others use string mapping
+	* When  no matching txpwr limit selection is found, use
+	*     1. txpwr lmit selection associated with alpha2 inside driver regulation database
+	*     2. WW when driver has no support of this alpha2
+	*/
+
+	ent.domain_code = rtk_ent_exist ? rtk_ent.domain_code : RTW_CHPLAN_UNSPECIFIED;
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	ent.domain_code_6g = rtk_ent_exist ? rtk_ent.domain_code_6g : RTW_CHPLAN_6G_UNSPECIFIED;
+	#endif
+
+	#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE
+	ent.edcca_2g_override = rtk_ent_exist ? rtk_ent.edcca_2g_override : RTW_EDCCA_ADAPT;
+	#if CONFIG_IEEE80211_BAND_5GHZ
+	ent.edcca_5g_override = rtk_ent_exist ? rtk_ent.edcca_5g_override : RTW_EDCCA_ADAPT;
+	#endif
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	ent.edcca_6g_override = rtk_ent_exist ? rtk_ent.edcca_6g_override : RTW_EDCCA_ADAPT;
+	#endif
+	#endif
+
+	ent.txpwr_lmt_override = rtk_ent_exist ? rtk_ent.txpwr_lmt_override : TXPWR_LMT_DEF;
+	#ifdef CONFIG_CHPLAN_PROTO_EN
+	ent.proto_en = CHPLAN_PROTO_EN_ALL;
+	#endif
+
+	/* TODO: dfs_region */
+
+	return _rtw_set_chplan_cmd(adapter, flags
+		, RTW_CHPLAN_UNSPECIFIED, RTW_CHPLAN_6G_UNSPECIFIED, &ent, env
+		, REGD_SRC_OS, inr
+		, NULL, RTW_RLINK_MAX
+		, NULL);
+}
+#endif /* CONFIG_REGD_SRC_FROM_OS */
+
+u8 rtw_get_chplan_hdl(_adapter *adapter, u8 *pbuf)
+{
+	struct get_channel_plan_param *param;
+	struct get_chplan_resp *chplan;
+	struct rf_ctl_t *rfctl;
+	struct rtw_chset *chset;
+#if CONFIG_TXPWR_LIMIT
+	char *tl_reg_names[BAND_MAX];
+	int tl_reg_names_len[BAND_MAX];
+#endif
+	int tl_reg_names_len_total = 0;
+	int i;
+
+	if (!pbuf)
+		return H2C_PARAMETERS_ERROR;
+
+	rfctl = adapter_to_rfctl(adapter);
+	chset = adapter_to_chset(adapter);
+	param = (struct get_channel_plan_param *)pbuf;
+
+#if CONFIG_TXPWR_LIMIT
+	rtw_txpwr_hal_get_current_lmt_regs_name(adapter_to_dvobj(adapter), tl_reg_names, tl_reg_names_len);
+	for (i = 0; i < BAND_MAX; i++)
+		tl_reg_names_len_total += tl_reg_names_len[i];
+#endif
+
+	chplan = rtw_vmalloc(sizeof(struct get_chplan_resp) + sizeof(RT_CHANNEL_INFO) * chset->chs_len + tl_reg_names_len_total);
+	if (!chplan)
+		return H2C_CMD_FAIL;
+
+	chplan->regd_src = rfctl->regd_src;
+	chplan->regd_inr_bmp = rfctl->regd_inr_bmp;
+
+	chplan->alpha2[0] = rfctl->alpha2[0];
+	chplan->alpha2[1] = rfctl->alpha2[1];
+
+	chplan->channel_plan = rfctl->domain_code;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	chplan->chplan_6g = rfctl->domain_code_6g;
+	chplan->env_bmp = rfctl->env_bmp;
+#endif
+#if CONFIG_TXPWR_LIMIT
+	chplan->txpwr_lmt_names_len_total = tl_reg_names_len_total;
+	for (i = 0; i < BAND_MAX; i++) {
+		if (i == 0)
+			chplan->txpwr_lmt_names[i] = ((u8 *)(chplan->chs)) + sizeof(RT_CHANNEL_INFO) * chset->chs_len;
+		else
+			chplan->txpwr_lmt_names[i] = chplan->txpwr_lmt_names[i - 1] + chplan->txpwr_lmt_names_len[i - 1];
+
+		chplan->txpwr_lmt_names_len[i] = tl_reg_names_len[i];
+		if (tl_reg_names[i] && tl_reg_names_len[i]) {
+			_rtw_memcpy((void *)chplan->txpwr_lmt_names[i], tl_reg_names[i], tl_reg_names_len[i]);
+			rtw_mfree(tl_reg_names[i], tl_reg_names_len[i]);
+		}
+	}
+#endif
+	chplan->edcca_mode_2g = rfctl->edcca_mode_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	chplan->edcca_mode_5g = rfctl->edcca_mode_5g;
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	chplan->edcca_mode_6g = rfctl->edcca_mode_6g;
+#endif
+#ifdef CONFIG_DFS_MASTER
+	chplan->dfs_domain = rtw_rfctl_get_dfs_domain(rfctl);
+#endif
+
+	chplan->proto_en = 0
+		#ifdef CONFIG_CHPLAN_PROTO_EN
+		| rfctl->proto_en
+		#endif
+		;
+
+	chplan->confs.dis_ch_flags = rfctl->dis_ch_flags;
+	_rtw_memcpy(chplan->confs.excl_chs, rfctl->excl_chs, MAX_CHANNEL_NUM_2G_5G);
+#if CONFIG_IEEE80211_BAND_6GHZ
+	_rtw_memcpy(chplan->confs.excl_chs_6g, rfctl->excl_chs_6g, MAX_CHANNEL_NUM_6G);
+#endif
+	chplan->confs.init_regd_always_apply = rfctl->init_regd_always_apply;
+	chplan->confs.user_regd_always_apply = rfctl->user_regd_always_apply;
+	chplan->confs.bcn_hint_valid_ms = rfctl->bcn_hint_valid_ms;
+	chplan->confs.edcca_mode_sel = rfctl->edcca_mode_sel;
+#ifdef CONFIG_80211D
+	chplan->confs.cis_en_mode = rfctl->cis_en_mode;
+	chplan->confs.cis_flags = rfctl->cis_flags;
+	chplan->confs.cis_en_role = rfctl->cis_en_role;
+	chplan->confs.cis_en_ifbmp = rfctl->cis_en_ifbmp;
+	chplan->confs.cis_scan_int_ms = rfctl->cis_scan_int_ms;
+#endif
+
+	chplan->chs_len = chset->chs_len;
+	_rtw_memcpy(chplan->chs, chset->chs, sizeof(RT_CHANNEL_INFO) * chset->chs_len);
+	param->chplan = chplan;
+
+	return	H2C_SUCCESS;
+}
+
+void rtw_get_chplan_callback(_adapter *adapter, struct cmd_obj *cmdobj)
+{
+	struct get_channel_plan_param *param = (struct get_channel_plan_param *)cmdobj->parmbuf;
+
+	cmdobj->sctx_rsp_buf = param->chplan;
+}
+
+u8 rtw_get_chplan_cmd(_adapter *adapter, int flags, struct get_chplan_resp **chplan)
+{
+	struct cmd_obj *cmdobj;
+	struct get_channel_plan_param *parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 res = _FAIL;
+
+	if (!(flags & (RTW_CMDF_DIRECTLY | RTW_CMDF_WAIT_ACK))) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	/* prepare cmd parameter */
+	parm = rtw_zmalloc(sizeof(*parm));
+	if (parm == NULL)
+		goto exit;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS == rtw_get_chplan_hdl(adapter, (u8 *)parm)) {
+			*chplan = parm->chplan;
+			res = _SUCCESS;
+		}
+		rtw_mfree((u8 *)parm, sizeof(*parm));
+	} else { /* case of RTW_CMDF_WAIT_ACK */
+		/* need enqueue, prepare cmd_obj and enqueue */
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_GET_CHANPLAN);
+		CMD_OBJ_SET_HWBAND(cmdobj, HW_BAND_0);
+		cmdobj->no_io = true;
+
+		cmdobj->sctx = &sctx;
+		rtw_sctx_init(&sctx, 2000);
+		cmdobj->sctx_rsp_buf_free = (void *)rtw_free_get_chplan_resp;
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+		if (res == _SUCCESS) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_DONE_SUCCESS)
+				*chplan = sctx.rsp;
+			else
+				res = _FAIL;
+		}
+
+		/* allow get channel plan when cmd_thread is not running */
+		if (res != _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			parm = rtw_zmalloc(sizeof(*parm));
+			if (parm == NULL)
+				goto exit;
+
+			if (H2C_SUCCESS == rtw_get_chplan_hdl(adapter, (u8 *)parm)) {
+				*chplan = parm->chplan;
+				res = _SUCCESS;
+			}
+
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+		}
+	}
+
+exit:
+	return res;
+}
+
+void rtw_free_get_chplan_resp(struct get_chplan_resp *chplan)
+{
+	size_t sz = sizeof(struct get_chplan_resp) + sizeof(RT_CHANNEL_INFO) * chplan->chs_len
+		#if CONFIG_TXPWR_LIMIT
+		+ chplan->txpwr_lmt_names_len_total
+		#endif
+		;
+
+	rtw_vmfree(chplan, sz);
+}
+
+bool rtw_network_chk_opch_status(struct rf_ctl_t *rfctl
+	, struct wlan_network *network)
+{
+	RT_CHANNEL_INFO *chinfo;
+	bool ret = false;
+
+	chinfo = rtw_chset_get_chinfo_by_bch(&rfctl->chset
+		, BSS_EX_OP_BAND(&network->network), BSS_EX_OP_CH(&network->network), false);
+	if (!chinfo)
+		goto exit;
+
+	if (CH_IS_NON_OCP(chinfo))
+		goto exit;
+
+	ret = true;
+
+exit:
+	return ret;
+}
+
+#ifdef CONFIG_80211D
+static bool rtw_iface_accept_country_ie(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	if (!(BIT(adapter->iface_id) & rfctl->cis_en_ifbmp))
+		return false;
+	if (!MLME_IS_STA(adapter))
+		return false;
+	if (!MLME_IS_GC(adapter)) {
+		if (!(rfctl->cis_en_role & COUNTRY_IE_SLAVE_EN_ROLE_STA))
+			return false;
+	} else {
+		if (!(rfctl->cis_en_role & COUNTRY_IE_SLAVE_EN_ROLE_GC))
+			return false;
+	}
+	return true;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static enum country_ie_slave_6g_reg_info rtw_ies_get_6g_reg_info(const u8 *ies, uint ies_len)
+{
+	const u8 *he_op_ie;
+	sint he_op_ielen;
+
+	he_op_ie = rtw_get_ext_ie(ies, WLAN_EID_EXTENSION_HE_OPERATION, &he_op_ielen, ies_len);
+	if (he_op_ie && he_op_ielen >= 1 + HE_OPER_PARAMS_LEN && GET_HE_OP_PARA_6GHZ_OP_INFO_PRESENT(he_op_ie + 3)) {
+		u8 offset = HE_OPER_PARAMS_LEN + HE_OPER_BSS_COLOR_INFO_LEN + HE_OPER_BASIC_MCS_LEN
+			+ (GET_HE_OP_PARA_VHT_OP_INFO_PRESENT(he_op_ie + 3) ? HE_OPER_VHT_OPER_INFO_LEN : 0)
+			+ (GET_HE_OP_PARA_CO_HOSTED_BSS(he_op_ie + 3) ? HE_OPER_MAX_COHOST_BSSID_LEN : 0);
+
+		if (he_op_ielen >= offset)
+			return GET_HE_OP_INFO_REG_INFO(he_op_ie + 3 + offset);
+	}
+	return CIS_6G_REG_RSVD;
+}
+#endif
+
+void rtw_alink_joinbss_update_regulatory(struct _ADAPTER_LINK *alink, const WLAN_BSSID_EX *network)
+{
+	_adapter *adapter = alink->adapter;
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	if (rfctl->collect_link_cisr) {
+		u8 iface_id = adapter->iface_id;
+		u8 alink_id = rtw_adapter_link_get_id(alink);
+		const u8 *country_ie = NULL;
+		sint country_ie_len = 0;
+		enum country_ie_slave_6g_reg_info reg_info = CIS_6G_REG_RSVD;
+
+		if (rtw_iface_accept_country_ie(adapter)) {
+			country_ie = rtw_get_ie(BSS_EX_TLV_IES(network)
+				, WLAN_EID_COUNTRY, &country_ie_len, BSS_EX_TLV_IES_LEN(network));
+			if (country_ie) {
+				if (country_ie_len < 3) {
+					country_ie = NULL;
+					country_ie_len = 0;
+				} else
+					country_ie_len += 2;
+			}
+		}
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		if (BSS_EX_OP_BAND(network) == BAND_ON_6G)
+			reg_info = rtw_ies_get_6g_reg_info(BSS_EX_TLV_IES(network), BSS_EX_TLV_IES_LEN(network));
+		#endif
+
+		if (country_ie) {
+			rtw_buf_update(&rfctl->recv_country_ie[iface_id][alink_id]
+				, &rfctl->recv_country_ie_len[iface_id][alink_id], country_ie, country_ie_len);
+		} else {
+			rtw_buf_free(&rfctl->recv_country_ie[iface_id][alink_id]
+				, &rfctl->recv_country_ie_len[iface_id][alink_id]);
+		}
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		rfctl->recv_6g_reg_info[iface_id][alink_id] = reg_info;
+		#endif
+
+		if (rtw_alink_apply_recv_regu_ies_cmd(alink, RTW_CMDF_DIRECTLY
+			, BSS_EX_OP_BAND(network), BSS_EX_OP_CH(network), country_ie, reg_info) != _SUCCESS)
+			RTW_WARN(FUNC_ADPT_FMT" id:%u rtw_alink_apply_recv_regu_ies_cmd() fail\n", FUNC_ADPT_ARG(adapter), alink_id);
+	}
+}
+
+static void _rtw_alink_leavebss_update_regulatory(_adapter *adapter, u8 alink_id)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	if (rfctl->collect_link_cisr) {
+		u8 iface_id = adapter->iface_id;
+
+		if (alink_id < RTW_RLINK_MAX) {
+			struct _ADAPTER_LINK * alink = GET_LINK(adapter, alink_id);
+
+			rtw_buf_free(&rfctl->recv_country_ie[iface_id][alink_id]
+				, &rfctl->recv_country_ie_len[iface_id][alink_id]);
+			if (rtw_alink_apply_recv_regu_ies_cmd(alink, RTW_CMDF_DIRECTLY, 0, 0, NULL, CIS_6G_REG_RSVD) != _SUCCESS)
+				RTW_WARN(FUNC_ADPT_FMT" id:%u rtw_alink_apply_recv_regu_ies_cmd() fail\n", FUNC_ADPT_ARG(adapter), alink_id);
+		} else {
+			u8 i;
+
+			for (i = 0; i < RTW_RLINK_MAX; i++)
+				rtw_buf_free(&rfctl->recv_country_ie[iface_id][i]
+				, &rfctl->recv_country_ie_len[iface_id][i]);
+			if (rtw_apply_recv_regu_ies_cmd(adapter, RTW_CMDF_DIRECTLY, 0, 0, NULL, CIS_6G_REG_RSVD) != _SUCCESS)
+				RTW_WARN(FUNC_ADPT_FMT" rtw_apply_recv_regu_ies_cmd() fail\n", FUNC_ADPT_ARG(adapter));
+		}
+	}
+}
+
+void rtw_alink_leavebss_update_regulatory(struct _ADAPTER_LINK * alink)
+{
+	_rtw_alink_leavebss_update_regulatory(alink->adapter, rtw_adapter_link_get_id(alink));
+}
+
+void rtw_alink_csa_update_regulatory(struct _ADAPTER_LINK *alink, enum band_type req_band, u8 req_ch)
+{
+	_adapter *adapter = alink->adapter;
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	if (rfctl->collect_link_cisr) {
+		u8 iface_id = adapter->iface_id;
+		u8 alink_id = rtw_adapter_link_get_id(alink);
+
+		if (rtw_alink_apply_recv_regu_ies_cmd(alink, RTW_CMDF_DIRECTLY
+				, req_band, req_ch, rfctl->recv_country_ie[iface_id][alink_id]
+				, RFCTL_RECV_6G_REG_INFO(rfctl, iface_id, alink_id)) != _SUCCESS)
+			RTW_WARN(FUNC_ADPT_FMT" id:%u rtw_alink_apply_recv_regu_ies_cmd() fail\n", FUNC_ADPT_ARG(adapter), alink_id);
+	}
+}
+
+void alink_process_regu_ies(struct _ADAPTER_LINK *alink, u8 *ies, uint ies_len)
+{
+	_adapter *adapter = alink->adapter;
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	u8 iface_id;
+	u8 alink_id;
+
+#ifdef CONFIG_ECSA_PHL
+	/* don't process country ie when under CSA processing */
+	if (rtw_mr_is_ecsa_running(adapter))
+		return;
+#endif
+
+	iface_id = adapter->iface_id;
+	alink_id = rtw_adapter_link_get_id(alink);
+
+	if (rfctl->collect_link_cisr) {
+		const u8 *country_ie = NULL;
+		sint country_ie_len = 0;
+		bool country_ie_changed;
+		bool country_str_changed = false;
+		enum country_ie_slave_6g_reg_info reg_info = CIS_6G_REG_RSVD;
+		bool reg_info_changed = false;
+
+		if (rtw_iface_accept_country_ie(adapter)) {
+			country_ie = rtw_get_ie(ies, WLAN_EID_COUNTRY, &country_ie_len, ies_len);
+			if (country_ie) {
+				if (country_ie_len < 3) {
+					country_ie = NULL;
+					country_ie_len = 0;
+				} else
+					country_ie_len += 2;
+			}
+		}
+		country_ie_changed = ((!!rfctl->recv_country_ie[iface_id][alink_id]) ^ (!!country_ie))
+			|| rfctl->recv_country_ie_len[iface_id][alink_id] != country_ie_len
+			|| _rtw_memcmp(rfctl->recv_country_ie[iface_id][alink_id], country_ie, country_ie_len) == _FALSE;
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		if (ALINK_GET_BAND(alink) == BAND_ON_6G)
+			reg_info = rtw_ies_get_6g_reg_info(ies, ies_len);
+		reg_info_changed = rfctl->recv_6g_reg_info[iface_id][alink_id] != reg_info;
+		#endif
+
+		if (!country_ie_changed && !reg_info_changed)
+			return;
+
+		if (country_ie_changed) {
+			country_str_changed = ((!!rfctl->recv_country_ie[iface_id][alink_id]) ^ (!!country_ie))
+				|| (country_ie && _rtw_memcmp(rfctl->recv_country_ie[iface_id][alink_id] + 2, country_ie + 2, 3));
+			if (!country_ie) {
+				rtw_buf_free(&rfctl->recv_country_ie[iface_id][alink_id]
+					, &rfctl->recv_country_ie_len[iface_id][alink_id]);
+			} else {
+				rtw_buf_update(&rfctl->recv_country_ie[iface_id][alink_id]
+					, &rfctl->recv_country_ie_len[iface_id][alink_id], country_ie, country_ie_len);
+			}
+		}
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		rfctl->recv_6g_reg_info[iface_id][alink_id] = reg_info;
+		#endif
+
+		if (country_str_changed /* for now only country str is used */
+			|| reg_info_changed
+		) {
+			RTW_INFO(FUNC_ADPT_FMT" id:%u%s%s changed\n", FUNC_ADPT_ARG(adapter), alink_id
+				, country_str_changed ? " country_str" : ""
+				#if CONFIG_IEEE80211_BAND_6GHZ
+				, reg_info_changed ? " 6g_reg_info" : ""
+				#else
+				, ""
+				#endif
+				);
+			rtw_alink_apply_recv_regu_ies_cmd(alink, 0, ALINK_GET_BAND(alink), ALINK_GET_CH(alink)
+				, rfctl->recv_country_ie[iface_id][alink_id], RFCTL_RECV_6G_REG_INFO(rfctl, iface_id, alink_id));
+		}
+	}
+	else if (rfctl->recv_country_ie[iface_id][alink_id]) {
+		rtw_buf_free(&rfctl->recv_country_ie[iface_id][alink_id]
+			, &rfctl->recv_country_ie_len[iface_id][alink_id]);
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		rfctl->recv_6g_reg_info[iface_id][alink_id] = CIS_6G_REG_RSVD;
+		#endif
+		rtw_alink_apply_recv_regu_ies_cmd(alink, 0, ALINK_GET_BAND(alink), ALINK_GET_CH(alink), NULL, CIS_6G_REG_RSVD);
+	}
+}
+
+void rtw_joinbss_update_regulatory(_adapter *adapter, const WLAN_BSSID_EX *network)
+{
+	rtw_alink_joinbss_update_regulatory(GET_PRIMARY_LINK(adapter), network);
+}
+
+void rtw_leavebss_update_regulatory(_adapter *adapter)
+{
+	_rtw_alink_leavebss_update_regulatory(adapter, RTW_RLINK_MAX);
+}
+
+void rtw_csa_update_regulatory(_adapter *adapter, enum band_type req_band, u8 req_ch)
+{
+	rtw_alink_csa_update_regulatory(GET_PRIMARY_LINK(adapter), req_band, req_ch);
+}
+
+void process_regu_ies(_adapter *adapter, u8 *ies, uint ies_len)
+{
+	alink_process_regu_ies(GET_PRIMARY_LINK(adapter), ies, ies_len);
+}
+
+bool rtw_update_scanned_network_cisr(struct rf_ctl_t *rfctl, struct wlan_network *network)
+{
+	if (rfctl->collect_network_cisr) {
+		const u8 *country_ie = NULL;
+		sint country_ie_len = 0;
+		enum country_ie_slave_6g_reg_info reg_info = CIS_6G_REG_RSVD;
+		struct country_ie_slave_record *cisr = &network->cisr;
+		struct country_chplan *chplan = &network->cisr.chplan;
+		struct country_chplan ori_chplan;
+
+		_rtw_memcpy(&ori_chplan, chplan, sizeof(*chplan));
+
+		country_ie = rtw_get_ie(BSS_EX_TLV_IES(&network->network), WLAN_EID_COUNTRY, &country_ie_len, BSS_EX_TLV_IES_LEN(&network->network));
+		if (country_ie) {
+			if (country_ie_len < 3)
+				country_ie = NULL;
+		}
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		if (BSS_EX_OP_BAND(&network->network) == BAND_ON_6G)
+			reg_info = rtw_ies_get_6g_reg_info(BSS_EX_TLV_IES(&network->network), BSS_EX_TLV_IES_LEN(&network->network));
+		#endif
+
+		rtw_get_cisr_from_recv_regu_ies(rfctl
+			, BSS_EX_OP_BAND(&network->network), BSS_EX_OP_CH(&network->network)
+			, country_ie, reg_info, cisr);
+
+		return _rtw_memcmp(&ori_chplan, chplan, sizeof(*chplan)) == _TRUE ? false : true;
+	}
+	else if (network->cisr.alpha2[0] != '\0' || network->cisr.alpha2[1] != '\0') {
+		network->cisr.alpha2[0] = network->cisr.alpha2[1] = '\0';
+		network->cisr.band = BSS_EX_OP_BAND(&network->network);
+		network->cisr.opch = BSS_EX_OP_CH(&network->network);
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		network->cisr.reg_info = CIS_6G_REG_RSVD;
+		#endif
+		network->cisr.status = COUNTRY_IE_SLAVE_NOCOUNTRY;
+		return true;
+	}
+
+	return false;
+}
+
+bool rtw_network_chk_regu_ies(struct rf_ctl_t *rfctl, struct wlan_network *network)
+{
+	bool forbid_unknown_country_opch;
+	bool ret = false;
+
+	if (!rtw_txpwr_hal_is_txpwr_limit_needed(rfctl_to_dvobj(rfctl)))
+		goto bypass;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (network->cisr.band == BAND_ON_6G) {
+		/* forbid before std-client ready  */
+		if (network->cisr.reg_info == CIS_6G_REG_SP_AP
+			|| network->cisr.reg_info == CIS_6G_REG_IN_SP_AP)
+			goto exit;
+
+		if (!rfctl->cis_enabled
+			|| network->cisr.status == COUNTRY_IE_SLAVE_NOCOUNTRY
+			|| network->cisr.status == COUNTRY_IE_SLAVE_UNKNOWN
+		) {
+			if (!(rfctl->default_chplan_cate_6g_map & reg_info_to_chplan_6g_cate_map(network->cisr.reg_info)))
+				goto exit;
+		} else {
+			if (network->cisr.status == COUNTRY_IE_SLAVE_CATE_6G_NS)
+				goto exit;
+		}
+	}
+#endif
+
+	forbid_unknown_country_opch = rtw_rfctl_forbid_unknown_country_opch(
+		rfctl->cis_enabled, rfctl->init_user_req_is_ww, RFCTL_REGD_SRC_FROM_OS(rfctl));
+	if (forbid_unknown_country_opch) {
+		if (network->cisr.status == COUNTRY_IE_SLAVE_UNKNOWN
+			|| network->cisr.status == COUNTRY_IE_SLAVE_OPCH_NOEXIST)
+			goto exit;
+	}
+
+bypass:
+	ret = true;
+
+exit:
+	return ret;
+}
+
+void rtw_rfctl_cis_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty)
+{
+#if CONFIG_IEEE80211_BAND_6GHZ
+	int i, j;
+#endif
+
+	rfctl->cis_en_mode = regsty->country_ie_slave_en_mode;
+	if (!CIS_EN_MODE_IS_VALID(rfctl->cis_en_mode)) {
+		RTW_WARN("%s cis_en_mode %u is not supported, set to disable\n", __func__, rfctl->cis_en_mode);
+		rfctl->cis_en_mode = CISEM_DISABLE;
+	}
+
+	rfctl->cis_flags = regsty->country_ie_slave_flags;
+	if (rfctl->cis_flags & ~CISF_VALIDS) {
+		RTW_WARN("%s cis_flags:0x%02x has undefined bits, apply valid bits only\n", __func__, rfctl->cis_flags);
+		rfctl->cis_flags &= CISF_VALIDS;
+	}
+
+	rfctl->cis_en_role = regsty->country_ie_slave_en_role;
+	rfctl->cis_en_ifbmp = regsty->country_ie_slave_en_ifbmp;
+	rfctl->cis_scan_int_ms = regsty->country_ie_slave_scan_int_ms;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	for (i = 0; i < CONFIG_IFACE_NUMBER; i++) {
+		for (j = 0; j < RTW_RLINK_MAX; j++) {
+			rfctl->recv_6g_reg_info[i][j] = CIS_6G_REG_RSVD;
+			rfctl->cisr[i][j].reg_info = CIS_6G_REG_RSVD;
+		}
+	}
+#endif
+}
+
+void rtw_rfctl_cis_deinit(struct rf_ctl_t *rfctl)
+{
+	int i, j;
+
+	for (i = 0; i < CONFIG_IFACE_NUMBER; i++)
+		for (j = 0; j < RTW_RLINK_MAX; j++)
+			rtw_buf_free(&rfctl->recv_country_ie[i][j], &rfctl->recv_country_ie_len[i][j]);
+}
+#endif /* CONFIG_80211D */
+
+#ifdef CONFIG_PROC_DEBUG
+void dump_cur_chplan_confs(void *sel, struct rf_ctl_t *rfctl)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct get_chplan_resp *chplan;
+	struct chplan_confs *confs;
+	size_t buf_len = rtw_max(RTW_CH_FLAGS_STR_LEN, EXCL_CHS_STR_LEN);
+	char *buf;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	buf_len = rtw_max(buf_len, EXCL_CHS_6G_STR_LEN);
+#endif
+
+	buf = rtw_vmalloc(buf_len);
+	if (!buf)
+		return;
+
+	if (rtw_get_chplan_cmd(dvobj_get_primary_adapter(dvobj), RTW_CMDF_WAIT_ACK, &chplan) == _FAIL)
+		goto free_buf;
+
+	confs = &chplan->confs;
+
+	RTW_PRINT_SEL(sel, "dis_ch_flags=%s\n", rtw_get_ch_flags_str(buf, confs->dis_ch_flags, ','));
+	RTW_PRINT_SEL(sel, "excl_chs=%s\n"
+		, get_str_of_u8_array(buf, buf_len, confs->excl_chs, MAX_CHANNEL_NUM_2G_5G, ',', true));
+#if CONFIG_IEEE80211_BAND_6GHZ
+	RTW_PRINT_SEL(sel, "excl_chs_6g=%s\n"
+		, get_str_of_u8_array(buf, buf_len, confs->excl_chs_6g, MAX_CHANNEL_NUM_6G, ',', true));
+#endif
+	RTW_PRINT_SEL(sel, "init_regd_always_apply=%d\n", confs->init_regd_always_apply);
+	RTW_PRINT_SEL(sel, "user_regd_always_apply=%d\n", confs->user_regd_always_apply);
+	RTW_PRINT_SEL(sel, "bcn_hint_valid_ms=%u\n", confs->bcn_hint_valid_ms);
+	RTW_PRINT_SEL(sel, "edcca_mode_sel=%u\n", confs->edcca_mode_sel);
+#ifdef CONFIG_80211D
+	RTW_PRINT_SEL(sel, "cis_en_mode=%u\n", confs->cis_en_mode);
+	RTW_PRINT_SEL(sel, "cis_flags=0x%02x\n", confs->cis_flags);
+	RTW_PRINT_SEL(sel, "cis_en_role=0x%02x\n", confs->cis_en_role);
+	RTW_PRINT_SEL(sel, "cis_en_ifbmp=0x%02x\n", confs->cis_en_ifbmp);
+	RTW_PRINT_SEL(sel, "cis_scan_int_ms=%u\n", confs->cis_scan_int_ms);
+#endif
+
+	rtw_free_get_chplan_resp(chplan);
+
+free_buf:
+	rtw_vmfree(buf, buf_len);
+}
+
+static void dump_chplan_regd_inrs(void *sel, struct get_chplan_resp *chplan)
+{
+	char buf[REGD_INR_BMP_STR_LEN];
+
+	RTW_PRINT_SEL(sel, "regd_inr:%s\n", rtw_get_regd_inr_bmp_str(buf, chplan->regd_inr_bmp));
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static void dump_chplan_envs(void *sel, struct get_chplan_resp *chplan)
+{
+	char buf[REGD_INR_BMP_STR_LEN];
+
+	RTW_PRINT_SEL(sel, "env:%s\n", rtw_get_env_bmp_str(buf, chplan->env_bmp));
+}
+#endif
+
+#if CONFIG_TXPWR_LIMIT
+static void dump_chplan_txpwr_lmt_regs(void *sel, struct get_chplan_resp *chplan)
+{
+	int band;
+	const char *names, *name;
+	int names_len;
+
+	for (band = 0; band < BAND_MAX; band++) {
+		names = chplan->txpwr_lmt_names[band];
+		names_len = chplan->txpwr_lmt_names_len[band];
+
+		RTW_PRINT_SEL(sel, "txpwr_lmt[%s]:", band_str(band));
+		ustrs_for_each_str(names, names_len, name)
+			_RTW_PRINT_SEL(sel, "%s%s", name == names ? "" : " ", name);
+		_RTW_PRINT_SEL(sel, "\n");
+	}
+}
+#endif
+
+static void dump_chplan_edcca_modes(void *sel, struct get_chplan_resp *chplan)
+{
+	u8 mode[BAND_MAX];
+	char buf[EDCCA_MODE_OF_BANDS_STR_LEN];
+
+	mode[BAND_ON_24G] = chplan->edcca_mode_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	mode[BAND_ON_5G] = chplan->edcca_mode_5g;
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	mode[BAND_ON_6G] = chplan->edcca_mode_6g;
+#endif
+
+	RTW_PRINT_SEL(sel, "edcca_mode:%s\n", rtw_get_edcca_mode_of_bands_str(buf, mode));
+}
+
+static void dump_addl_ch_disable_conf(void *sel, struct get_chplan_resp *chplan)
+{
+	struct chplan_confs *confs = &chplan->confs;
+	int i;
+
+	if (confs->dis_ch_flags) {
+		char buf[RTW_CH_FLAGS_STR_LEN];
+
+		RTW_PRINT_SEL(sel, "dis_ch_flags:%s\n", rtw_get_ch_flags_str(buf, confs->dis_ch_flags, ' '));
+	}
+
+	if (confs->excl_chs[0] != 0) {
+		char buf[EXCL_CHS_STR_LEN];
+
+		RTW_PRINT_SEL(sel, "excl_chs:%s\n"
+			, get_str_of_u8_array(buf, sizeof(buf), confs->excl_chs, MAX_CHANNEL_NUM_2G_5G, ' ', true));
+	}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	for (i = 0; i < MAX_CHANNEL_NUM_6G; i++)
+		if (confs->excl_chs_6g[i] != 0)
+			break;
+
+	if (confs->excl_chs_6g[0] != 0) {
+		char buf[EXCL_CHS_6G_STR_LEN];
+
+		RTW_PRINT_SEL(sel, "excl_chs_6g:%s\n"
+			, get_str_of_u8_array(buf, sizeof(buf), confs->excl_chs_6g, MAX_CHANNEL_NUM_6G, ' ', true));
+	}
+#endif
+}
+
+void dump_cur_country(void *sel, struct rf_ctl_t *rfctl)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct get_chplan_resp *chplan;
+
+	if (rtw_get_chplan_cmd(dvobj_get_primary_adapter(dvobj), RTW_CMDF_WAIT_ACK, &chplan) == _FAIL)
+		return;
+
+	RTW_PRINT_SEL(sel, "%c%c\n", chplan->alpha2[0], chplan->alpha2[1]);
+
+	rtw_free_get_chplan_resp(chplan);
+}
+
+void dump_cur_chplan(void *sel, struct rf_ctl_t *rfctl)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct get_chplan_resp *chplan;
+
+	if (rtw_get_chplan_cmd(dvobj_get_primary_adapter(dvobj), RTW_CMDF_WAIT_ACK, &chplan) == _FAIL)
+		return;
+
+	RTW_PRINT_SEL(sel, "regd_src:%s(%d)\n", regd_src_str(chplan->regd_src), chplan->regd_src);
+	dump_chplan_regd_inrs(sel, chplan);
+
+	RTW_PRINT_SEL(sel, "alpha2:%c%c\n", chplan->alpha2[0], chplan->alpha2[1]);
+
+#ifdef CONFIG_80211BE_EHT
+	RTW_PRINT_SEL(sel, "be:%d\n", (chplan->proto_en & CHPLAN_PROTO_EN_BE) ? 1 : 0);
+#endif
+#ifdef CONFIG_80211AX_HE
+	RTW_PRINT_SEL(sel, "ax:%d\n", (chplan->proto_en & CHPLAN_PROTO_EN_AX) ? 1 : 0);
+#endif
+#ifdef CONFIG_80211AC_VHT
+	RTW_PRINT_SEL(sel, "ac:%d\n", (chplan->proto_en & CHPLAN_PROTO_EN_AC) ? 1 : 0);
+#endif
+#if CONFIG_IEEE80211_BAND_5GHZ
+	RTW_PRINT_SEL(sel, "a:%d\n", (chplan->proto_en & CHPLAN_PROTO_EN_A) ? 1 : 0);
+#endif
+
+	if (chplan->channel_plan == RTW_CHPLAN_UNSPECIFIED)
+		RTW_PRINT_SEL(sel, "chplan:UNSPEC\n");
+	else
+		RTW_PRINT_SEL(sel, "chplan:0x%02X\n", chplan->channel_plan);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (chplan->chplan_6g == RTW_CHPLAN_6G_UNSPECIFIED)
+		RTW_PRINT_SEL(sel, "chplan_6g:UNSPEC\n");
+	else
+		RTW_PRINT_SEL(sel, "chplan_6g:0x%02X\n", chplan->chplan_6g);
+
+	dump_chplan_envs(sel, chplan);
+#endif
+
+#if CONFIG_TXPWR_LIMIT
+	dump_chplan_txpwr_lmt_regs(sel, chplan);
+#endif
+
+	dump_chplan_edcca_modes(sel, chplan);
+
+#ifdef CONFIG_DFS_MASTER
+	RTW_PRINT_SEL(sel, "dfs_domain:%s(%u)\n", rtw_dfs_regd_str(chplan->dfs_domain), chplan->dfs_domain);
+#endif
+
+	dump_addl_ch_disable_conf(sel, chplan);
+
+	dump_chinfos(sel, chplan->chs, chplan->chs_len);
+
+	rtw_free_get_chplan_resp(chplan);
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+void dump_cur_env(void *sel, struct rf_ctl_t *rfctl)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct get_chplan_resp *chplan;
+
+	if (rtw_get_chplan_cmd(dvobj_get_primary_adapter(dvobj), RTW_CMDF_WAIT_ACK, &chplan) == _FAIL)
+		return;
+
+	dump_chplan_envs(sel, chplan);
+
+	rtw_free_get_chplan_resp(chplan);
+}
+#endif
+#endif /* CONFIG_PROC_DEBUG */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chplan.h b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chplan.h
new file mode 100644
index 000000000000..89217c7a45f8
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chplan.h
@@ -0,0 +1,691 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_CHPLAN_H__
+#define __RTW_CHPLAN_H__
+
+#define RTW_CHPLAN_IOCTL_NULL		0xFFFE /* special value by ioctl: null(empty) chplan */
+#define RTW_CHPLAN_IOCTL_UNSPECIFIED	0xFFFF /* special value by ioctl: no change (keep original) */
+
+#define RTW_CHPLAN_NULL		0x0A
+#define RTW_CHPLAN_WORLDWIDE	0x7F
+#define RTW_CHPLAN_UNSPECIFIED	0xFF
+#define RTW_CHPLAN_6G_NULL		0x00
+#define RTW_CHPLAN_6G_WORLDWIDE		0x7F
+#define RTW_CHPLAN_6G_UNSPECIFIED	0xFF
+
+u8 rtw_chplan_get_default_regd_2g(u8 id);
+#if CONFIG_IEEE80211_BAND_5GHZ
+u8 rtw_chplan_get_default_regd_5g(u8 id);
+#endif
+bool rtw_is_channel_plan_valid(u8 id);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+u8 rtw_chplan_get_default_regd_6g(u8 id);
+bool rtw_is_channel_plan_6g_valid(u8 id);
+#endif
+
+void rtw_rfctl_addl_ch_disable_conf_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty);
+
+u8 rtw_chplan_is_bchbw_valid(u8 id, u8 id_6g, enum band_type band, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive, struct rf_ctl_t *rfctl);
+
+enum regd_src_t {
+	REGD_SRC_RTK_PRIV = 0, /* Regulatory settings from Realtek framework (Realtek defined or customized) */
+	REGD_SRC_OS = 1, /* Regulatory settings from OS */
+	REGD_SRC_NUM,
+};
+
+#define regd_src_is_valid(src) ((src) < REGD_SRC_NUM)
+
+extern const char *_regd_src_str[];
+#define regd_src_str(src) ((src) >= REGD_SRC_NUM ? _regd_src_str[REGD_SRC_NUM] : _regd_src_str[src])
+
+bool rtw_rfctl_reg_allow_beacon_hint(struct rf_ctl_t *rfctl);
+bool rtw_chinfo_allow_beacon_hint(struct _RT_CHANNEL_INFO *chinfo);
+u8 rtw_process_beacon_hint(struct rf_ctl_t *rfctl, struct wlan_network *network);
+void rtw_beacon_hint_ch_change_notifier(struct rf_ctl_t *rfctl);
+
+#define ALPHA2_FMT "%c%c"
+#define ALPHA2_ARG(a2) ((is_alpha(a2[0]) || is_decimal(a2[0])) ? a2[0] : '-'), ((is_alpha(a2[1]) || is_decimal(a2[1])) ? a2[1] : '-')
+
+#define WORLDWIDE_ALPHA2	"00"
+#define UNSPEC_ALPHA2		"99"
+#define INTERSECTED_ALPHA2	"98"
+
+#define IS_ALPHA2_WORLDWIDE(_alpha2)	(strncmp(_alpha2, WORLDWIDE_ALPHA2, 2) == 0)
+#define IS_ALPHA2_UNSPEC(_alpha2)	(strncmp(_alpha2, UNSPEC_ALPHA2, 2) == 0)
+#define IS_ALPHA2_INTERSECTED(_alpha2)	(strncmp(_alpha2, INTERSECTED_ALPHA2, 2) == 0)
+#define SET_UNSPEC_ALPHA2(_alpha2)		do { _rtw_memcpy(_alpha2, UNSPEC_ALPHA2, 2); } while (0)
+#define SET_INTERSECTEDC_ALPHA2(_alpha2)	do { _rtw_memcpy(_alpha2, INTERSECTED_ALPHA2, 2); } while (0)
+
+enum rtw_regd_inr {
+	RTW_REGD_SET_BY_INIT = 0,
+	RTW_REGD_SET_BY_USER = 1,
+	RTW_REGD_SET_BY_COUNTRY_IE = 2,
+
+	/* below is not used for REGD_SRC_RTK_PRIV */
+	RTW_REGD_SET_BY_DRIVER = 3,
+	RTW_REGD_SET_BY_CORE = 4,
+
+	RTW_REGD_SET_BY_NUM,
+};
+
+extern const char *const _regd_inr_str[];
+#define regd_inr_str(inr) (((inr) >= RTW_REGD_SET_BY_NUM) ? _regd_inr_str[RTW_REGD_SET_BY_NUM] : _regd_inr_str[(inr)])
+
+enum rtw_regd {
+	RTW_REGD_NA = 0,
+	RTW_REGD_FCC = 1,
+	RTW_REGD_MKK = 2,
+	RTW_REGD_ETSI = 3,
+	RTW_REGD_IC = 4,
+	RTW_REGD_KCC = 5,
+	RTW_REGD_NCC = 6,
+	RTW_REGD_ACMA = 7,
+	RTW_REGD_CHILE = 8,
+	RTW_REGD_MEX = 9,
+	RTW_REGD_WW,
+	RTW_REGD_NUM,
+};
+
+extern const char *const _regd_str[];
+#define regd_str(regd) (((regd) >= RTW_REGD_NUM) ? _regd_str[RTW_REGD_NA] : _regd_str[(regd)])
+
+enum rtw_env_t {
+	RTW_ENV_ANY	= 0,
+	RTW_ENV_INDOOR	= 1,
+	RTW_ENV_OUTDOOR	= 2,
+	RTW_ENV_NUM,
+};
+
+extern const char *const _env_str[];
+#define env_str(env) (((env) >= RTW_ENV_NUM) ? _env_str[RTW_ENV_ANY] : _env_str[(env)])
+
+enum rtw_edcca_mode_t {
+	RTW_EDCCA_NORM		= 0, /* normal */
+	RTW_EDCCA_CS		= 1, /* carrier sense */
+	RTW_EDCCA_ADAPT		= 2, /* adaptivity */
+	RTW_EDCCA_CBP		= 3, /* contention based protocol */
+	RTW_EDCCA_MODE_NUM,
+	RTW_EDCCA_DEF		= RTW_EDCCA_MODE_NUM, /* default (ref to domain code), used at country chplan map's override field */
+	RTW_EDCCA_AUTO		= 255, /* follow channel plan */
+};
+
+extern const char *const _rtw_edcca_mode_str[];
+#define rtw_edcca_mode_str(mode) (((mode) >= RTW_EDCCA_MODE_NUM) ? _rtw_edcca_mode_str[RTW_EDCCA_NORM] : _rtw_edcca_mode_str[(mode)])
+
+#define RTW_EDCCA_MODE_SEL_VALID(mode) ((mode) < RTW_EDCCA_MODE_NUM || (mode) == RTW_EDCCA_AUTO)
+#define rtw_edcca_mode_sel_str(mode) (((mode) == RTW_EDCCA_AUTO) ? "AUTO" : rtw_edcca_mode_str(mode))
+
+enum rtw_dfs_regd {
+	RTW_DFS_REGD_NONE	= 0,
+	RTW_DFS_REGD_FCC	= 1,
+	RTW_DFS_REGD_MKK	= 2,
+	RTW_DFS_REGD_ETSI	= 3,
+	RTW_DFS_REGD_KCC	= 4,
+	RTW_DFS_REGD_NUM,
+	RTW_DFS_REGD_AUTO	= 0xFF, /* follow channel plan */
+};
+
+#define RTW_DFS_REGD_IS_UNKNOWN(regd) ((regd) == RTW_DFS_REGD_NONE || (regd) >= RTW_DFS_REGD_NUM)
+
+extern const char *const _rtw_dfs_regd_str[];
+#define rtw_dfs_regd_str(region) (((region) >= RTW_DFS_REGD_NUM) ? _rtw_dfs_regd_str[RTW_DFS_REGD_NONE] : _rtw_dfs_regd_str[(region)])
+
+typedef enum _REGULATION_TXPWR_LMT {
+	TXPWR_LMT_NONE = 0, /* no limit */
+	TXPWR_LMT_FCC = 1,
+	TXPWR_LMT_MKK = 2,
+	TXPWR_LMT_ETSI = 3,
+	TXPWR_LMT_IC = 4,
+	TXPWR_LMT_KCC = 5,
+	TXPWR_LMT_NCC = 6,
+	TXPWR_LMT_ACMA = 7,
+	TXPWR_LMT_CHILE = 8,
+	TXPWR_LMT_UKRAINE = 9,
+	TXPWR_LMT_MEXICO = 10,
+	TXPWR_LMT_CN = 11,
+	TXPWR_LMT_QATAR = 12,
+	TXPWR_LMT_UK = 13,
+	TXPWR_LMT_THAILAND = 14,
+	TXPWR_LMT_WW, /* smallest of all available limit, keep last */
+
+	TXPWR_LMT_NUM,
+	TXPWR_LMT_DEF = TXPWR_LMT_NUM, /* default (ref to domain code), used at country chplan map's override field */
+} REGULATION_TXPWR_LMT;
+
+extern const char *const _txpwr_lmt_str[];
+#define txpwr_lmt_str(regd) (((regd) >= TXPWR_LMT_NUM) ? _txpwr_lmt_str[TXPWR_LMT_NUM] : _txpwr_lmt_str[(regd)])
+
+extern const REGULATION_TXPWR_LMT _txpwr_lmt_alternate[];
+#define txpwr_lmt_alternate(ori) (((ori) > TXPWR_LMT_NUM) ? _txpwr_lmt_alternate[TXPWR_LMT_WW] : _txpwr_lmt_alternate[(ori)])
+
+#define TXPWR_LMT_ALTERNATE_DEFINED(txpwr_lmt) (txpwr_lmt_alternate(txpwr_lmt) != txpwr_lmt)
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum txpwr_lmt_6g_cate_t {
+	TXPWR_LMT_6G_CATE_VLP,
+	TXPWR_LMT_6G_CATE_LPI,
+	TXPWR_LMT_6G_CATE_STD,
+	TXPWR_LMT_6G_CATE_NUM,
+};
+
+extern const char *const _txpwr_lmt_6g_cate_str[];
+#define txpwr_lmt_6g_cate_str(cate) (((cate) >= TXPWR_LMT_6G_CATE_NUM) ? _txpwr_lmt_6g_cate_str[TXPWR_LMT_6G_CATE_NUM] : _txpwr_lmt_6g_cate_str[(cate)])
+
+extern const char *const _txpwr_lmt_6g_str[][TXPWR_LMT_6G_CATE_NUM];
+#define txpwr_lmt_6g_str(regd, cate) (((regd) >= TXPWR_LMT_NUM || (cate) >= TXPWR_LMT_6G_CATE_NUM) ? _txpwr_lmt_str[TXPWR_LMT_NUM] : _txpwr_lmt_6g_str[(regd)][(cate)])
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+extern const enum rtw_edcca_mode_t _rtw_regd_to_edcca_mode[RTW_REGD_NUM];
+#define rtw_regd_to_edcca_mode(regd) (((regd) >= RTW_REGD_NUM) ? RTW_EDCCA_NORM : _rtw_regd_to_edcca_mode[(regd)])
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+extern const enum rtw_edcca_mode_t _rtw_regd_to_edcca_mode_6g[RTW_REGD_NUM];
+#define rtw_regd_to_edcca_mode_6g(regd) (((regd) >= RTW_REGD_NUM) ? RTW_EDCCA_NORM : _rtw_regd_to_edcca_mode_6g[(regd)])
+#endif
+
+extern const REGULATION_TXPWR_LMT _rtw_regd_to_txpwr_lmt[];
+#define rtw_regd_to_txpwr_lmt(regd) (((regd) >= RTW_REGD_NUM) ? TXPWR_LMT_WW : _rtw_regd_to_txpwr_lmt[(regd)])
+
+#define REGD_INR_BMP_STR_LEN (33)
+char *rtw_get_regd_inr_bmp_str(char *buf, u8 bmp);
+
+#define ENV_BMP_STR_LEN (11)
+char *rtw_get_env_bmp_str(char *buf, u8 bmp);
+
+#define EDCCA_MODE_OF_BANDS_STR_LEN (((6 + 3 + 1) * BAND_MAX) + 1)
+char *rtw_get_edcca_mode_of_bands_str(char *buf, u8 mode_of_band[]);
+void rtw_edcca_mode_update(struct dvobj_priv *dvobj, bool req_lock);
+u8 rtw_get_edcca_mode(struct dvobj_priv *dvobj, enum band_type band);
+void rtw_edcca_mode_sel_msg(void *sel, struct dvobj_priv *dvobj);
+
+#if CONFIG_TXPWR_LIMIT
+#define TXPWR_NAME_OF_BANDS_STR_LEN (((1 + 8 + 1) * BAND_MAX) + (4 * CONFIG_IEEE80211_BAND_6GHZ) + 1)
+char *rtw_get_txpwr_lmt_name_of_bands_str(char *buf, const char *name_of_band[], u8 unknown_bmp);
+void rtw_txpwr_update_cur_lmt_regs(struct dvobj_priv *dvobj, bool req_lock);
+#endif
+
+#define CHPLAN_PROTO_EN_A	BIT0
+#define CHPLAN_PROTO_EN_AC	BIT1
+#define CHPLAN_PROTO_EN_AX	BIT2
+#define CHPLAN_PROTO_EN_BE	BIT3
+#define CHPLAN_PROTO_EN_ALL	0xFF
+
+#if defined(CONFIG_80211BE_EHT) || defined(CONFIG_80211AX_HE) || defined(CONFIG_80211AC_VHT) || CONFIG_IEEE80211_BAND_5GHZ
+#define CONFIG_CHPLAN_PROTO_EN
+#endif
+
+#define CHPLAN_6G_CATE_VLP	BIT0
+#define CHPLAN_6G_CATE_LPI	BIT1
+#define CHPLAN_6G_CATE_STD	BIT2
+
+struct country_chplan {
+	char alpha2[2];
+
+	u8 domain_code;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 domain_code_6g;
+#endif
+
+#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE
+	/* will override edcca mode get by domain code (/6g) */
+	u8 edcca_2g_override:3;
+	#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 edcca_5g_override:3;
+	#endif
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 edcca_6g_override:3;
+	#endif
+#endif
+
+	/* will override txpwr_lmt get by domain code (/6g) */
+	u8 txpwr_lmt_override;
+
+#ifdef CONFIG_CHPLAN_PROTO_EN
+	u8 proto_en;
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 cate_6g_map;
+#endif
+};
+
+#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE
+#define COUNTRY_CHPLAN_ASSIGN_EDCCA_2G_OVERRIDE(_val) , .edcca_2g_override = (_val)
+#else
+#define COUNTRY_CHPLAN_ASSIGN_EDCCA_2G_OVERRIDE(_val)
+#endif
+
+#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE && CONFIG_IEEE80211_BAND_5GHZ
+#define COUNTRY_CHPLAN_ASSIGN_EDCCA_5G_OVERRIDE(_val) , .edcca_5g_override = (_val)
+#else
+#define COUNTRY_CHPLAN_ASSIGN_EDCCA_5G_OVERRIDE(_val)
+#endif
+
+#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE && CONFIG_IEEE80211_BAND_6GHZ
+#define COUNTRY_CHPLAN_ASSIGN_EDCCA_6G_OVERRIDE(_val) , .edcca_6g_override = (_val)
+#else
+#define COUNTRY_CHPLAN_ASSIGN_EDCCA_6G_OVERRIDE(_val)
+#endif
+
+#define CHPLAN_6G_CATE_MAP____	0
+#define CHPLAN_6G_CATE_MAP___V	CHPLAN_6G_CATE_VLP
+#define CHPLAN_6G_CATE_MAP__I_	CHPLAN_6G_CATE_LPI
+#define CHPLAN_6G_CATE_MAP__IV	(CHPLAN_6G_CATE_LPI | CHPLAN_6G_CATE_VLP)
+#define CHPLAN_6G_CATE_MAP_S__	CHPLAN_6G_CATE_STD
+#define CHPLAN_6G_CATE_MAP_S_V	(CHPLAN_6G_CATE_STD | CHPLAN_6G_CATE_VLP)
+#define CHPLAN_6G_CATE_MAP_SI_	(CHPLAN_6G_CATE_STD | CHPLAN_6G_CATE_LPI)
+#define CHPLAN_6G_CATE_MAP_SIV	(CHPLAN_6G_CATE_STD | CHPLAN_6G_CATE_LPI | CHPLAN_6G_CATE_VLP)
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define COUNTRY_CHPLAN_ASSIGN_CHPLAN_6G(_val) , .domain_code_6g = (_val)
+#define COUNTRY_CHPLAN_ASSIGN_6G_CATE_MAP(_val) , .cate_6g_map = CHPLAN_6G_CATE_MAP_##_val
+#else
+#define COUNTRY_CHPLAN_ASSIGN_CHPLAN_6G(_val)
+#define COUNTRY_CHPLAN_ASSIGN_6G_CATE_MAP(_val)
+#endif
+
+#ifdef CONFIG_CHPLAN_PROTO_EN
+#define COUNTRY_CHPLAN_ASSIGN_PROTO_EN(_val) , .proto_en = (_val)
+#else
+#define COUNTRY_CHPLAN_ASSIGN_PROTO_EN(_val)
+#endif
+
+#define COUNTRY_CHPLAN_ENT_ARGC_9(_alpha2, _chplan, _chplan_6g, _txpwr_lmt_override, _en_11be, _en_11ax, _en_11ac, _en_11a, _cate_6g_map) \
+	{ \
+		.alpha2 = (_alpha2), .domain_code = (_chplan) \
+		COUNTRY_CHPLAN_ASSIGN_CHPLAN_6G(_chplan_6g) \
+		COUNTRY_CHPLAN_ASSIGN_EDCCA_2G_OVERRIDE(RTW_EDCCA_DEF) \
+		COUNTRY_CHPLAN_ASSIGN_EDCCA_5G_OVERRIDE(RTW_EDCCA_DEF) \
+		COUNTRY_CHPLAN_ASSIGN_EDCCA_6G_OVERRIDE(RTW_EDCCA_DEF) \
+		, .txpwr_lmt_override = TXPWR_LMT_##_txpwr_lmt_override \
+		COUNTRY_CHPLAN_ASSIGN_PROTO_EN( \
+			(_en_11be ? CHPLAN_PROTO_EN_BE : 0) | (_en_11ax ? CHPLAN_PROTO_EN_AX : 0) \
+			| (_en_11ac ? CHPLAN_PROTO_EN_AC : 0) | (_en_11a ? CHPLAN_PROTO_EN_A : 0)) \
+		COUNTRY_CHPLAN_ASSIGN_6G_CATE_MAP(_cate_6g_map) \
+	}
+
+#define COUNTRY_CHPLAN_ENT_ARGC_7(_alpha2, _chplan, _chplan_6g, _txpwr_lmt_override, _en_11ax, _en_11ac, _en_11a) \
+	COUNTRY_CHPLAN_ENT_ARGC_9(_alpha2, _chplan, _chplan_6g, _txpwr_lmt_override, 0, _en_11ax, _en_11ac, _en_11a, ___)
+
+#define __COUNTRY_CHPLAN_ENT_HDL(argc9, argc8, argc7, argc6, argc5, argc4, argc3, argc2, argc1, hdl, ...) hdl
+
+#define COUNTRY_CHPLAN_ENT(...) \
+	__COUNTRY_CHPLAN_ENT_HDL(__VA_ARGS__, \
+		COUNTRY_CHPLAN_ENT_ARGC_9, \
+		macro_argc8, \
+		COUNTRY_CHPLAN_ENT_ARGC_7, \
+		macro_argc6, \
+		macro_argc5, \
+		macro_argc4, \
+		macro_argc3, \
+		macro_argc2, \
+		macro_argc1) \
+		(__VA_ARGS__)
+
+#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE
+#define COUNTRY_CHPLAN_EDCCA_2G_OVERRIDE(_ent) (_ent)->edcca_2g_override
+#else
+#define COUNTRY_CHPLAN_EDCCA_2G_OVERRIDE(_ent) RTW_EDCCA_DEF
+#endif
+
+#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE && CONFIG_IEEE80211_BAND_5GHZ
+#define COUNTRY_CHPLAN_EDCCA_5G_OVERRIDE(_ent) (_ent)->edcca_5g_override
+#else
+#define COUNTRY_CHPLAN_EDCCA_5G_OVERRIDE(_ent) RTW_EDCCA_DEF
+#endif
+
+#if CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE && CONFIG_IEEE80211_BAND_6GHZ
+#define COUNTRY_CHPLAN_EDCCA_6G_OVERRIDE(_ent) (_ent)->edcca_6g_override
+#else
+#define COUNTRY_CHPLAN_EDCCA_6G_OVERRIDE(_ent) RTW_EDCCA_DEF
+#endif
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define COUNTRY_CHPLAN_EN_11A(_ent) (((_ent)->proto_en & CHPLAN_PROTO_EN_A) ? true : false)
+#else
+#define COUNTRY_CHPLAN_EN_11A(_ent) false
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+#define COUNTRY_CHPLAN_EN_11AC(_ent) (((_ent)->proto_en & CHPLAN_PROTO_EN_AC) ? true : false)
+#else
+#define COUNTRY_CHPLAN_EN_11AC(_ent) false
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#define COUNTRY_CHPLAN_EN_11AX(_ent) (((_ent)->proto_en & CHPLAN_PROTO_EN_AX) ? true : false)
+#else
+#define COUNTRY_CHPLAN_EN_11AX(_ent) false
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+#define COUNTRY_CHPLAN_EN_11BE(_ent) (((_ent)->proto_en & CHPLAN_PROTO_EN_BE) ? true : false)
+#else
+#define COUNTRY_CHPLAN_EN_11BE(_ent) false
+#endif
+
+void rtw_get_chplan_worldwide(struct country_chplan *ent);
+bool rtw_get_chplan_from_country(const char *country_code, struct country_chplan *ent);
+
+void rtw_chplan_ioctl_input_mapping(u16 *chplan, u16 *chplan_6g);
+bool rtw_chplan_ids_is_world_wide(u8 chplan, u8 chplan_6g);
+
+u8 rtw_country_chplan_is_bchbw_valid(struct country_chplan *ent, enum band_type band, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive, struct rf_ctl_t *rfctl);
+
+enum country_ie_slave_en_mode {
+	CISEM_DISABLE	= 0, /* disable */
+	CISEM_ENABLE	= 1, /* enable */
+	CISEM_ENABLE_WW	= 2, /* enable when INIT/USER set world wide mode */
+	CISEM_NUM,
+};
+
+#define CIS_EN_MODE_IS_VALID(mode) ((mode) < CISEM_NUM)
+
+enum country_ie_slave_flags {
+	_CISF_RSVD_0	= BIT0, /* deprecated BIT0 */
+	CISF_ENV_BSS	= BIT1, /* if not set, consider IEs of associated BSSs only */
+
+	CISF_VALIDS = CISF_ENV_BSS,
+};
+
+enum country_ie_slave_6g_reg_info {
+	CIS_6G_REG_IN_AP	= 0,
+	CIS_6G_REG_SP_AP	= 1,
+	CIS_6G_REG_VLP_AP	= 2,
+	CIS_6G_REG_IN_EN_AP	= 3,
+	CIS_6G_REG_IN_SP_AP	= 4,
+	CIS_6G_REG_NUM,
+	CIS_6G_REG_RSVD		= CIS_6G_REG_NUM,
+};
+
+#define IS_6G_REG_INFO_RSVD(reg) ((reg) >= CIS_6G_REG_NUM)
+
+enum country_ie_slave_status {
+	COUNTRY_IE_SLAVE_NOCOUNTRY	= 0, /* no country IE or 'XX' alpha2 */
+	COUNTRY_IE_SLAVE_APPLICABLE	= 1,
+	COUNTRY_IE_SLAVE_UNKNOWN	= 2, /* unknown country */
+	COUNTRY_IE_SLAVE_OPCH_NOEXIST	= 3, /* bss's op ch not exist */
+	COUNTRY_IE_SLAVE_CATE_6G_NS	= 4, /* 6G category not support */
+
+	COUNTRY_IE_SLAVE_STATUS_NUM,
+};
+
+struct country_ie_slave_record {
+	char alpha2[2]; /* country code get from connected AP of STA ifaces, "\x00\x00" is not set */
+	enum band_type band;
+	u8 opch;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	enum rtw_env_t env;
+	enum country_ie_slave_6g_reg_info reg_info;
+#endif
+	enum country_ie_slave_status status;
+	struct country_chplan chplan;
+};
+
+#ifdef CONFIG_80211D
+extern const char _rtw_env_char[];
+#define rtw_env_char(e) (((e) >= RTW_ENV_NUM) ? _rtw_env_char[RTW_ENV_ANY] : _rtw_env_char[e])
+
+void dump_country_ie_slave_records(void *sel, struct rf_ctl_t *rfctl, bool skip_noset);
+#endif
+
+void dump_country_chplan(void *sel, const struct country_chplan *ent, bool regd_info);
+void dump_country_chplan_map(void *sel, bool regd_info);
+void dump_country_list(void *sel);
+void dump_chplan_id_list(void *sel);
+void dump_chplan_country_list(void *sel);
+#if CONFIG_IEEE80211_BAND_6GHZ
+void dump_chplan_6g_id_list(void *sel);
+void dump_chplan_6g_country_list(void *sel);
+#endif
+#ifdef CONFIG_RTW_DEBUG
+void dump_chplan_test(void *sel);
+#endif
+void dump_chplan_ver(void *sel);
+
+struct regd_req_t {
+	_list list;
+	int ref_cnt; /* used by RTK_PRIV's COUNTRY_IE req */
+	bool applied;
+
+	enum regd_src_t src;
+	enum rtw_regd_inr inr;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	enum rtw_env_t env;
+	enum country_ie_slave_6g_reg_info reg_info;
+	u8 txpwr_lmt_6g_cate_map;
+#endif
+
+	struct country_chplan chplan;
+};
+
+void rtw_regd_req_list_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty);
+void rtw_regd_req_list_free(struct rf_ctl_t *rfctl);
+
+void rtw_regd_watchdog_hdl(struct dvobj_priv *dvobj);
+
+struct _ADAPTER_LINK;
+enum channel_width alink_adjust_linking_bw_by_regd(struct _ADAPTER_LINK *alink
+	, enum band_type band, u8 ch, enum channel_width bw, enum chan_offset offset);
+enum channel_width adapter_adjust_linking_bw_by_regd(_adapter *adapter
+	, enum band_type band, u8 ch, enum channel_width bw, enum chan_offset offset);
+
+void rtw_rfctl_decide_init_chplan(struct rf_ctl_t *rfctl,
+	const char *hw_alpha2, u8 hw_chplan, u8 hw_chplan_6g, u8 hw_force_chplan);
+void rtw_rfctl_apply_init_chplan(struct rf_ctl_t *rfctl, bool req_lock);
+
+bool rtw_rfctl_is_disable_sw_channel_plan(struct dvobj_priv *dvobj);
+
+#define alink_regu_block_tx(alink) (LINK_MLME_IS_REGU_FORBID(alink) || alink_is_tx_blocked_by_ch_waiting(alink))
+#define adapter_regu_block_tx(adapter) alink_regu_block_tx(GET_PRIMARY_LINK(adapter))
+
+#if defined(CONFIG_AP_MODE) && CONFIG_AP_REGU_FORBID
+bool rtw_rfctl_is_regu_forbid_bss(struct rf_ctl_t *rfctl, enum band_type band);
+#else
+#define rtw_rfctl_is_regu_forbid_bss(rfctl, band) false
+#endif
+
+enum chplan_confs_type {
+	CHPLAN_CONFS_DIS_CH_FLAGS,
+	CHPLAN_CONFS_EXCL_CHS,
+	CHPLAN_CONFS_EXCL_CHS_6G,
+	CHPLAN_CONFS_INIT_REGD_ALWAYS_APPLY,
+	CHPLAN_CONFS_USER_REGD_ALWAYS_APPLY,
+	CHPLAN_CONFS_BCN_HINT_VALID_MS,
+	CHPLAN_CONFS_EDCCA_MODE_SEL,
+	CHPLAN_CONFS_CIS_EN_MODE,
+	CHPLAN_CONFS_CIS_FLAGS,
+	CHPLAN_CONFS_CIS_EN_ROLE,
+	CHPLAN_CONFS_CIS_EN_IFBMP,
+	CHPLAN_CONFS_CIS_SCAN_INT_MS,
+	CHPLAN_CONFS_NUM,
+};
+
+struct chplan_confs {
+	u16 set_types; /* bitmap of chplan_confs_type */
+	u8 dis_ch_flags;
+	u8 excl_chs[MAX_CHANNEL_NUM_2G_5G];
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 excl_chs_6g[MAX_CHANNEL_NUM_6G];
+#endif
+	bool init_regd_always_apply;
+	bool user_regd_always_apply;
+	u32 bcn_hint_valid_ms;
+	u8 edcca_mode_sel;
+#ifdef CONFIG_80211D
+	u8 cis_en_mode;
+	u8 cis_flags;
+	u8 cis_en_role;
+	u8 cis_en_ifbmp;
+	u32 cis_scan_int_ms;
+#endif
+};
+
+struct SetChannelPlan_param {
+	enum regd_src_t regd_src;
+	enum rtw_regd_inr inr;
+	struct country_chplan country_ent;
+	bool has_country;
+	u8 channel_plan;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 channel_plan_6g;
+	enum rtw_env_t env;
+#endif
+	/* used for regd_src == RTK_PRIV and inr == USER, bitmap of  RTW_PRIV_USER_SET_XXX */
+	u8 priv_user_set_bmp;
+
+#ifdef CONFIG_80211D
+	/* used for regd_src == RTK_PRIV and inr == COUNTRY_IE */
+	struct country_ie_slave_record cisr;
+	u8 cisr_alink_id;
+	bool has_cisr;
+#endif
+
+	struct chplan_confs confs;
+
+#ifdef PLATFORM_LINUX
+	bool rtnl_lock_needed;
+#endif
+};
+
+u8 rtw_set_chplan_hdl(_adapter *adapter, u8 *pbuf);
+
+u8 rtw_set_chplan_cmd(_adapter *adapter, int flags, u8 chplan, u8 chplan_6g
+	, enum rtw_env_t env, enum rtw_regd_inr inr);
+u8 rtw_set_country_cmd(_adapter *adapter, int flags, const char *country_code
+	, enum rtw_env_t env, enum rtw_regd_inr inr);
+#if CONFIG_IEEE80211_BAND_6GHZ
+u8 rtw_set_env_cmd(_adapter *adapter, int flags, enum rtw_env_t env
+	, enum regd_src_t regd_src, enum rtw_regd_inr inr);
+#endif
+#ifdef CONFIG_80211D
+u8 rtw_alink_apply_recv_regu_ies_cmd(struct _ADAPTER_LINK *alink, int flags, enum band_type band,u8 opch
+	, const u8 *country_ie, enum country_ie_slave_6g_reg_info reg_info);
+u8 rtw_apply_recv_regu_ies_cmd(_adapter *adapter, int flags, enum band_type band,u8 opch
+	, const u8 *country_ie, enum country_ie_slave_6g_reg_info reg_info);
+u8 rtw_apply_scan_network_country_ie_cmd(_adapter *adapter, int flags);
+#endif
+#ifdef CONFIG_REGD_SRC_FROM_OS
+u8 rtw_sync_os_regd_cmd(_adapter *adapter, int flags, const char *country_code
+	, u8 dfs_region, enum rtw_env_t env, enum rtw_regd_inr inr);
+#endif
+u8 rtw_set_chplan_confs_cmd(_adapter *adapter, int flags, struct chplan_confs *confs);
+#ifdef CONFIG_PROC_DEBUG
+u16 rtw_parse_chplan_confs_cmd_str(struct chplan_confs *confs, char *str);
+#endif
+
+struct get_chplan_resp {
+	enum regd_src_t regd_src;
+	u8 regd_inr_bmp;
+	char alpha2[2];
+	u8 channel_plan;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 chplan_6g;
+	u8 env_bmp;
+#endif
+
+#if CONFIG_TXPWR_LIMIT
+	/* point to content of txpwr_lmt_names of specific band (after content of chset) */
+	const char *txpwr_lmt_names[BAND_MAX];
+	int txpwr_lmt_names_len[BAND_MAX];
+	int txpwr_lmt_names_len_total; /* ease for free operation */
+#endif
+	u8 edcca_mode_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 edcca_mode_5g;
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 edcca_mode_6g;
+#endif
+#ifdef CONFIG_DFS_MASTER
+	u8 dfs_domain;
+#endif
+	u8 proto_en;
+
+	struct chplan_confs confs;
+
+	u8 chs_len;
+	RT_CHANNEL_INFO chs[];
+};
+
+struct get_channel_plan_param {
+	struct get_chplan_resp *chplan;
+};
+
+u8 rtw_get_chplan_hdl(_adapter *adapter, u8 *pbuf);
+struct cmd_obj;
+void rtw_get_chplan_callback(_adapter *adapter, struct cmd_obj *cmdobj);
+
+u8 rtw_get_chplan_cmd(_adapter *adapter, int flags, struct get_chplan_resp **chplan);
+void rtw_free_get_chplan_resp(struct get_chplan_resp *chplan);
+
+bool rtw_network_chk_opch_status(struct rf_ctl_t *rfctl, struct wlan_network *network);
+
+#ifdef CONFIG_80211D
+void rtw_alink_joinbss_update_regulatory(struct _ADAPTER_LINK *alink, const WLAN_BSSID_EX *network);
+void rtw_alink_leavebss_update_regulatory(struct _ADAPTER_LINK * alink);
+void rtw_alink_csa_update_regulatory(struct _ADAPTER_LINK *alink, enum band_type req_band, u8 req_ch);
+void alink_process_regu_ies(struct _ADAPTER_LINK *alink, u8 *ies, uint ies_len);
+
+void rtw_joinbss_update_regulatory(_adapter *adapter, const WLAN_BSSID_EX *network);
+void rtw_leavebss_update_regulatory(_adapter *adapter);
+void rtw_csa_update_regulatory(_adapter *adapter, enum band_type req_band, u8 req_ch);
+void process_regu_ies(_adapter *adapter, u8 *ies, uint ies_len);
+
+bool rtw_update_scanned_network_cisr(struct rf_ctl_t *rfctl, struct wlan_network *network);
+bool rtw_network_chk_regu_ies(struct rf_ctl_t *rfctl, struct wlan_network *network);
+
+void rtw_rfctl_cis_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty);
+void rtw_rfctl_cis_deinit(struct rf_ctl_t *rfctl);
+#else
+#define rtw_network_chk_regu_ies(rfctl, network) true
+#endif /* CONFIG_80211D */
+
+#ifdef CONFIG_PROC_DEBUG
+void dump_cur_chplan_confs(void *sel, struct rf_ctl_t *rfctl);
+void dump_cur_country(void *sel, struct rf_ctl_t *rfctl);
+void dump_cur_chplan(void *sel, struct rf_ctl_t *rfctl);
+#if CONFIG_IEEE80211_BAND_6GHZ
+void dump_cur_env(void *sel, struct rf_ctl_t *rfctl);
+#endif
+#endif /* CONFIG_PROC_DEBUG */
+
+#define CHPLAN_VER_STR_BUF_LEN 64
+
+struct rtw_regdb_ops {
+	u8 (*get_default_regd_2g)(u8 id);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 (*get_default_regd_5g)(u8 id);
+#endif
+	bool (*is_domain_code_valid)(u8 id);
+	bool (*domain_get_ch)(u8 id, u32 ch, u8 *flags);
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 (*get_default_regd_6g)(u8 id);
+	bool (*is_domain_code_6g_valid)(u8 id);
+	bool (*domain_6g_get_ch)(u8 id, u32 ch, u8 *flags);
+#endif
+
+	bool (*get_chplan_from_alpha2)(const char *alpha2, struct country_chplan *ent);
+
+#ifdef CONFIG_RTW_DEBUG
+	void (*dump_chplan_test)(void *sel);
+#endif
+	void (*get_ver_str)(char *buf, size_t buf_len);
+};
+
+#endif /* __RTW_CHPLAN_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chset.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chset.c
new file mode 100644
index 000000000000..2cd409f47a3c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chset.c
@@ -0,0 +1,382 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_CHSET_C_
+
+#include <drv_types.h>
+
+const char *const _rtw_ch_type_str[] = {
+	[RTW_CHT_DIS]		= "DIS",
+	[RTW_CHT_NO_IR]		= "NO_IR",
+	[RTW_CHT_DFS]		= "DFS",
+	[RTW_CHT_NO_HT40U]	= "NO_40M+",
+	[RTW_CHT_NO_HT40L]	= "NO_40M-",
+	[RTW_CHT_NO_80MHZ]	= "NO_80M",
+	[RTW_CHT_NO_160MHZ]	= "NO_160M",
+	[RTW_CHT_NUM]		= "UNKNOWN",
+};
+
+enum rtw_ch_type get_ch_type_from_str(const char *str, size_t str_len)
+{
+	u8 i;
+
+	for (i = 0; i < RTW_CHT_NUM; i++)
+		if (str_len == strlen(rtw_ch_type_str(i))
+			&& strncmp(str, rtw_ch_type_str(i), str_len) == 0)
+			return i;
+	return RTW_CHT_NUM;
+}
+
+char *rtw_get_ch_flags_str(char *buf, u8 flags, char delim)
+{
+	char *pos = buf;
+	char d_str[2] = {delim, '\0'};
+	int i;
+
+	for (i = 0; i < RTW_CHT_NUM; i++) {
+		if (!(flags & BIT(i)))
+			continue;
+		pos += snprintf(pos, RTW_CH_FLAGS_STR_LEN - (pos - buf), "%s%s"
+			, pos == buf ? "" : d_str, rtw_ch_type_str(i));
+		if (pos >= buf + RTW_CH_FLAGS_STR_LEN - 1)
+			break;
+	}
+	if (pos == buf)
+		*buf = '\0';
+
+	return buf;
+}
+
+int rtw_chset_init(struct rtw_chset *chset, u8 band_bmp)
+{
+	u8 ch_num = 0;
+	int band, i;
+
+	_rtw_memset(chset->chs, 0, sizeof(RT_CHANNEL_INFO) * MAX_CHANNEL_NUM);
+
+	for (band = 0; band < BAND_MAX; band++) {
+		u8 center_ch_num;
+		u8 (*center_chs)(u8, u8);
+
+		if (!(band_bmp & band_to_band_cap(band)))
+			continue;
+
+		center_ch_num = center_chs_num_of_band[band](CHANNEL_WIDTH_20);
+		center_chs = center_chs_of_band[band];
+
+		chset->chs_of_band[band] = &chset->chs[ch_num];
+		chset->chs_len_of_band[band] = 0;
+
+		for (i = 0; i < center_ch_num; i++) {
+			chset->chs[ch_num].band = band;
+			chset->chs[ch_num].ChannelNum = center_chs(CHANNEL_WIDTH_20, i);
+			chset->chs_len_of_band[band]++;;
+			ch_num++;
+		}
+	}
+
+	chset->chs_len = ch_num;
+
+	return _SUCCESS;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+/*
+ * Search enabled channel with the @param ch in given @param ch_set
+ * @ch_set: the given channel set
+ * @ch: the given channel number
+ *
+ * return the index of channel_num in channel_set, -1 if not found
+ */
+RTW_FUNC_2G_5G_ONLY static int _rtw_chset_search_ch(const struct rtw_chset *chset, u32 ch, bool include_dis)
+{
+	int i;
+
+	if (ch == 0)
+		return -1;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (ch == chset->chs[i].ChannelNum
+			&& (include_dis || !(chset->chs[i].flags & RTW_CHF_DIS)))
+			return i;
+	}
+
+	return -1;
+}
+
+RTW_FUNC_2G_5G_ONLY int rtw_chset_search_ch(const struct rtw_chset *chset, u32 ch)
+{
+	return _rtw_chset_search_ch(chset, ch, false);
+}
+#endif
+
+/*
+ * Search enabled channel with the @param ch of @param band in given @param ch_set
+ * @ch_set: the given channel set
+ * @band: the given band
+ * @ch: the given channel number
+ *
+ * return the index of channel_num in channel_set, -1 if not found
+ */
+static int _rtw_chset_search_bch(const struct rtw_chset *chset, enum band_type band, u32 ch, bool include_dis)
+{
+	int i;
+
+	if (ch == 0)
+		return -1;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (band == chset->chs[i].band && ch == chset->chs[i].ChannelNum
+			&& (include_dis || !(chset->chs[i].flags & RTW_CHF_DIS)))
+			return i;
+	}
+
+	return -1;
+}
+
+int rtw_chset_search_bch(const struct rtw_chset *chset, enum band_type band, u32 ch)
+{
+	return _rtw_chset_search_bch(chset, band, ch, false);
+}
+
+RT_CHANNEL_INFO *rtw_chset_get_chinfo_by_bch(struct rtw_chset *chset, enum band_type band, u32 ch, bool include_dis)
+{
+	int i = _rtw_chset_search_bch(chset, band, ch, include_dis);
+
+	return i >= 0 ? &chset->chs[i] : NULL;
+}
+
+/*
+ * Check if the @param ch, bw, offset is valid for the given @param ch_set
+ * @ch_set: the given channel set
+ * @ch: the given channel number
+ * @bw: the given bandwidth
+ * @offset: the given channel offset
+ *
+ * return valid (1) or not (0)
+ */
+static u8 _rtw_chset_is_bchbw_valid(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive)
+{
+	u8 cch;
+	u8 *op_chs;
+	u8 op_ch_num;
+	u8 valid = 0;
+	int i;
+	int ch_idx;
+
+	cch = rtw_get_center_ch_by_band(band, ch, bw, offset);
+
+	if (!rtw_get_op_chs_by_bcch_bw(band, cch, bw, &op_chs, &op_ch_num))
+		goto exit;
+
+	for (i = 0; i < op_ch_num; i++) {
+		if (0)
+			RTW_INFO("%u,%u,%u,%u - cch:%u, bw:%u, op_ch:%u\n", band, ch, bw, offset, cch, bw, *(op_chs + i));
+		ch_idx = rtw_chset_search_bch(chset, band, *(op_chs + i));
+		if (ch_idx == -1)
+			break;
+		if (!allow_passive && chset->chs[ch_idx].flags & RTW_CHF_NO_IR) {
+			/* all sub chs are passive is not allowed and one of sub ch is NO_IR */
+			if (!allow_primary_passive) /* even primary ch is not allow to be NO_IR */
+				break;
+			if (chset->chs[ch_idx].ChannelNum != ch) /* allow primary ch NO_IR, but this is not primary ch */
+				break;
+		}
+		if (bw >= CHANNEL_WIDTH_40) {
+			if ((chset->chs[ch_idx].flags & RTW_CHF_NO_HT40U) && i % 2 == 0)
+				break;
+			if ((chset->chs[ch_idx].flags & RTW_CHF_NO_HT40L) && i % 2 == 1)
+				break;
+		}
+		if (bw >= CHANNEL_WIDTH_80 && (chset->chs[ch_idx].flags & RTW_CHF_NO_80MHZ))
+			break;
+		if (bw >= CHANNEL_WIDTH_160 && (chset->chs[ch_idx].flags & RTW_CHF_NO_160MHZ))
+			break;
+	}
+
+	if (op_ch_num != 0 && i == op_ch_num)
+		valid = 1;
+
+exit:
+	return valid;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY u8 rtw_chset_is_chbw_valid(const struct rtw_chset *chset, u8 ch, u8 bw, u8 offset, bool allow_primary_passive, bool allow_passive)
+{
+	return _rtw_chset_is_bchbw_valid(chset, rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch, bw, offset, allow_primary_passive, allow_passive);
+}
+#endif
+
+u8 rtw_chset_is_bchbw_valid(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive)
+{
+	return _rtw_chset_is_bchbw_valid(chset, band, ch, bw, offset, allow_primary_passive, allow_passive);
+}
+
+/**
+ * rtw_chset_sync_bchbw - obey g_ch, adjust g_bw, g_offset, bw, offset to fit in channel plan
+ * @ch_set: channel plan to check
+ * @req_ch: pointer of the request ch, may be modified further
+ * @req_bw: pointer of the request bw, may be modified further
+ * @req_offset: pointer of the request offset, may be modified further
+ * @g_ch: pointer of the ongoing group ch
+ * @g_bw: pointer of the ongoing group bw, may be modified further
+ * @g_offset: pointer of the ongoing group offset, may be modified further
+ * @allow_primary_passive: if allow passive primary ch when deciding chbw
+ * @allow_passive: if allow passive ch (not primary) when deciding chbw
+ */
+void rtw_chset_sync_bchbw(const struct rtw_chset *chset, enum band_type *req_band, u8 *req_ch, u8 *req_bw, u8 *req_offset
+	, enum band_type *g_band, u8 *g_ch, u8 *g_bw, u8 *g_offset, bool allow_primary_passive, bool allow_passive)
+{
+	enum band_type r_band;
+	u8 r_ch, r_bw, r_offset;
+	enum band_type u_band;
+	u8 u_ch, u_bw, u_offset;
+	u8 cur_bw = *req_bw;
+
+	while (1) {
+		r_band = *req_band;
+		r_ch = *req_ch;
+		r_bw = cur_bw;
+		r_offset = *req_offset;
+		u_band = *g_band;
+		u_ch = *g_ch;
+		u_bw = *g_bw;
+		u_offset = *g_offset;
+
+		rtw_sync_bchbw(&r_band, &r_ch, &r_bw, &r_offset, &u_band, &u_ch, &u_bw, &u_offset);
+
+		if (rtw_chset_is_bchbw_valid(chset, r_band, r_ch, r_bw, r_offset, allow_primary_passive, allow_passive))
+			break;
+		if (cur_bw == CHANNEL_WIDTH_20) {
+			rtw_warn_on(1);
+			break;
+		}
+		cur_bw--;
+	};
+
+	*req_band = r_band;
+	*req_ch = r_ch;
+	*req_bw = r_bw;
+	*req_offset = r_offset;
+	*g_band = u_band;
+	*g_ch = u_ch;
+	*g_bw = u_bw;
+	*g_offset = u_offset;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY void rtw_chset_sync_chbw(const struct rtw_chset *chset, u8 *req_ch, u8 *req_bw, u8 *req_offset
+	, u8 *g_ch, u8 *g_bw, u8 *g_offset, bool allow_primary_passive, bool allow_passive)
+{
+	enum band_type band = rtw_is_2g_ch(*g_ch) ? BAND_ON_24G : BAND_ON_5G; /* follow g_ch's band */
+
+	rtw_chset_sync_bchbw(chset, &band, req_ch, req_bw, req_offset, &band, g_ch, g_bw, g_offset, allow_primary_passive, allow_passive);
+}
+#endif
+
+u8 *rtw_chset_set_spt_chs_ie(struct rtw_chset *chset, u8 *buf_pos, uint *buf_len)
+{
+	u8 i = 0;
+	u8 fch = 0, lch = 0, ch;
+	u8 *cont = buf_pos + 2;
+	RT_CHANNEL_INFO *chinfo;
+
+	while (i < chset->chs_len) {
+		chinfo = &chset->chs[i++];
+
+		if (chinfo->flags & RTW_CHF_DIS)
+			continue;
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		/* don't appnd 6G chs now, how to distinguish 2G/5G chs with 6G? */
+		if (chinfo->band == BAND_ON_6G)
+			continue;
+		#endif
+
+		ch = chinfo->ChannelNum;
+		if (fch == 0) {
+			fch = ch;
+			lch = ch;
+			continue;
+		}
+
+		if (lch + 1 != ch) {
+			*cont = fch;
+			*(cont + 1) = lch - fch + 1;
+			cont += 2;
+			fch = ch;
+		}
+		lch = ch;
+	}
+	if (fch) {
+		/* last subband */
+		*cont = fch;
+		*(cont + 1) = lch - fch + 1;
+		cont += 2;
+	}
+
+	if (cont > buf_pos + 2) {
+		*buf_pos = WLAN_EID_SUPPORTED_CHANNELS;
+		*(buf_pos + 1) = cont - buf_pos - 2;
+		*buf_len += cont - buf_pos;
+		return cont;
+	}
+	return buf_pos;
+}
+
+#ifdef CONFIG_PROC_DEBUG
+void dump_chinfos(void *sel, const RT_CHANNEL_INFO *chinfos, u8 chinfo_num)
+{
+	u32 bhint_sec;
+	char bhint_buf[8];
+	u16 non_ocp_sec;
+	char non_ocp_buf[8];
+	char flags_buf[RTW_CH_FLAGS_STR_LEN];
+	u8 enable_ch_num = 0;
+	u8 i;
+
+	RTW_PRINT_SEL(sel, "%-3s %-4s %-5s %-4s flags\n", "ch", "freq", "bhint", "nocp");
+
+	for (i = 0; i < chinfo_num; i++) {
+		if (chinfos[i].flags & RTW_CHF_DIS)
+			continue;
+		enable_ch_num++;
+
+		bhint_sec = 0;
+		if (CH_IS_BCN_HINT(&chinfos[i])) {
+			bhint_sec = rtw_systime_to_ms(chinfos[i].bcn_hint_end_time - rtw_get_current_time()) / 1000;
+			if (bhint_sec > 99999)
+				bhint_sec = 99999;
+		}
+		snprintf(bhint_buf, 8, "%d", bhint_sec);
+
+		non_ocp_sec = 0;
+		#ifdef CONFIG_DFS_MASTER
+		if (CH_IS_NON_OCP(&chinfos[i]))
+			non_ocp_sec = rtw_systime_to_ms(chinfos[i].non_ocp_end_time - rtw_get_current_time()) / 1000;
+		#endif
+		snprintf(non_ocp_buf, 8, "%d", non_ocp_sec);
+
+		RTW_PRINT_SEL(sel, "%3u %4u %5s %4s %s\n"
+			, chinfos[i].ChannelNum, rtw_bch2freq(chinfos[i].band, chinfos[i].ChannelNum)
+			, bhint_buf, non_ocp_buf, rtw_get_ch_flags_str(flags_buf, chinfos[i].flags, ' ')
+		);
+	}
+
+	RTW_PRINT_SEL(sel, "total ch number:%d\n", enable_ch_num);
+}
+#endif /* CONFIG_PROC_DEBUG */
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chset.h b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chset.h
new file mode 100644
index 000000000000..1e253b079d1a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_chset.h
@@ -0,0 +1,119 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_CHSET_H__
+#define __RTW_CHSET_H__
+
+enum rtw_ch_type {
+	RTW_CHT_DIS		= 0,
+	RTW_CHT_NO_IR		= 1,
+	RTW_CHT_DFS		= 2,
+	RTW_CHT_NO_HT40U	= 3,
+	RTW_CHT_NO_HT40L	= 4,
+	RTW_CHT_NO_80MHZ	= 5,
+	RTW_CHT_NO_160MHZ	= 6,
+	RTW_CHT_NUM,
+};
+
+extern const char *const _rtw_ch_type_str[];
+#define rtw_ch_type_str(type) (((type) >= RTW_CHT_NUM) ? _rtw_ch_type_str[RTW_CHT_NUM] : _rtw_ch_type_str[(type)])
+
+enum rtw_ch_type get_ch_type_from_str(const char *str, size_t str_len);
+
+enum {
+	RTW_CHF_DIS 		= BIT(RTW_CHT_DIS),
+	RTW_CHF_NO_IR		= BIT(RTW_CHT_NO_IR),
+	RTW_CHF_DFS		= BIT(RTW_CHT_DFS),
+	RTW_CHF_NO_HT40U	= BIT(RTW_CHT_NO_HT40U),
+	RTW_CHF_NO_HT40L	= BIT(RTW_CHT_NO_HT40L),
+	RTW_CHF_NO_80MHZ	= BIT(RTW_CHT_NO_80MHZ),
+	RTW_CHF_NO_160MHZ	= BIT(RTW_CHT_NO_160MHZ),
+};
+
+#define RTW_CH_FLAGS_STR_LEN (45)
+char *rtw_get_ch_flags_str(char *buf, u8 flags, char delim);
+
+/* The channel information about this channel including joining, scanning, and power constraints. */
+typedef struct _RT_CHANNEL_INFO {
+	u8 band; /* enum band_type */
+	u8 ChannelNum; /* The channel number. */
+
+	/*
+	* Bitmap and its usage:
+	* RTW_CHF_DIS, RTW_CHF_NO_IR, RTW_CHF_DFS: is used to check for status
+	* RTW_CHF_NO_HT40U, RTW_CHF_NO_HT40L, RTW_CHF_NO_80MHZ, RTW_CHF_NO_160MHZ: extra bandwidth limitation (ex: from regulatory)
+	*/
+	u8 flags;
+
+	bool reg_no_ir;
+	systime bcn_hint_end_time;
+
+#ifdef CONFIG_FIND_BEST_CHANNEL
+	u32 rx_count;
+#endif
+
+#if CONFIG_IEEE80211_BAND_5GHZ && CONFIG_DFS
+	#ifdef CONFIG_DFS_MASTER
+	bool cac_done;
+	systime non_ocp_end_time;
+	#endif
+#endif
+
+	u8 hidden_bss_cnt; /* per scan count */
+
+#if defined(CONFIG_IOCTL_CFG80211) && !RTW_PER_ADAPTER_WIPHY
+	void *os_chan;
+#endif
+} RT_CHANNEL_INFO, *PRT_CHANNEL_INFO;
+
+#define RTW_BCN_HINT_STOPPED 0 /* used by bcn_hint_end_time time stamps */
+#define CH_IS_BCN_HINT_STOPPED(rt_ch_info) ((rt_ch_info)->bcn_hint_end_time == RTW_BCN_HINT_STOPPED)
+#define CH_IS_BCN_HINT(rt_ch_info) (!CH_IS_BCN_HINT_STOPPED(rt_ch_info) && rtw_time_after((rt_ch_info)->bcn_hint_end_time, rtw_get_current_time()))
+
+struct rtw_chset {
+	RT_CHANNEL_INFO chs[MAX_CHANNEL_NUM];
+	u8 chs_len;
+	RT_CHANNEL_INFO *chs_of_band[BAND_MAX];
+	u8 chs_len_of_band[BAND_MAX];
+	u8 enable_ch_num;
+	u8 enable_ch_num_of_band[BAND_MAX];
+};
+
+int rtw_chset_init(struct rtw_chset *chset, u8 band_bmp);
+
+RTW_FUNC_2G_5G_ONLY int rtw_chset_search_ch(const struct rtw_chset *chset, u32 ch);
+RTW_FUNC_2G_5G_ONLY u8 rtw_chset_is_chbw_valid(const struct rtw_chset *chset, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive);
+RTW_FUNC_2G_5G_ONLY void rtw_chset_sync_chbw(const struct rtw_chset *chset, u8 *req_ch, u8 *req_bw, u8 *req_offset
+	, u8 *g_ch, u8 *g_bw, u8 *g_offset, bool allow_primary_passive, bool allow_passive);
+int rtw_chset_search_bch(const struct rtw_chset *chset, enum band_type band, u32 ch);
+RT_CHANNEL_INFO *rtw_chset_get_chinfo_by_bch(struct rtw_chset *chset, enum band_type band, u32 ch, bool include_dis);
+u8 rtw_chset_is_bchbw_valid(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset
+	, bool allow_primary_passive, bool allow_passive);
+void rtw_chset_sync_bchbw(const struct rtw_chset *chset, enum band_type *req_band, u8 *req_ch, u8 *req_bw, u8 *req_offset
+	, enum band_type *g_band, u8 *g_ch, u8 *g_bw, u8 *g_offset, bool allow_primary_passive, bool allow_passive);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define rtw_chset_has_6g_enabled(chset) (!!(chset)->enable_ch_num_of_band[BAND_ON_6G])
+#else
+#define rtw_chset_has_6g_enabled(chset) false
+#endif
+
+u8 *rtw_chset_set_spt_chs_ie(struct rtw_chset *chset, u8 *buf_pos, uint *buf_len);
+
+#ifdef CONFIG_PROC_DEBUG
+void dump_chinfos(void *sel, const RT_CHANNEL_INFO *chinfos, u8 chinfo_num);
+#endif
+
+#endif /* __RTW_CHSET_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_cmd.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_cmd.c
new file mode 100644
index 000000000000..66ca4d7fd0ad
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_cmd.c
@@ -0,0 +1,5007 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_CMD_C_
+
+#include <drv_types.h>
+
+#ifndef DBG_CMD_EXECUTE
+	#define DBG_CMD_EXECUTE 0
+#endif
+
+/*
+Caller and the rtw_cmd_thread can protect cmd_q by spin_lock.
+No irqsave is necessary.
+*/
+u32 rtw_init_cmd_priv(struct dvobj_priv *dvobj)
+{
+	u32 res = _SUCCESS;
+	struct cmd_priv *pcmdpriv = &dvobj->cmdpriv;
+
+	pcmdpriv->dvobj = dvobj;
+	#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	_rtw_init_sema(&(pcmdpriv->cmd_queue_sema), 0);
+	_rtw_init_sema(&(pcmdpriv->start_cmdthread_sema), 0);
+	_rtw_init_queue(&(pcmdpriv->cmd_queue));
+	#endif
+
+	pcmdpriv->cmd_seq = 1;
+	pcmdpriv->cmd_issued_cnt = 0;
+
+	_rtw_mutex_init(&pcmdpriv->sctx_mutex);
+
+	ATOMIC_SET(&pcmdpriv->event_seq, 0);
+	pcmdpriv->evt_done_cnt = 0;
+
+	return res;
+
+}
+
+void rtw_free_cmd_priv(struct dvobj_priv *dvobj)
+{
+	struct cmd_priv *pcmdpriv = &dvobj->cmdpriv;
+
+	#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	_rtw_spinlock_free(&(pcmdpriv->cmd_queue.lock));
+	_rtw_free_sema(&(pcmdpriv->cmd_queue_sema));
+	_rtw_free_sema(&(pcmdpriv->start_cmdthread_sema));
+	#endif
+
+	_rtw_mutex_free(&pcmdpriv->sctx_mutex);
+}
+
+static int rtw_cmd_filter(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
+{
+	u8 bAllow = _FALSE; /* set to _TRUE to allow enqueuing cmd when hw_init_completed is _FALSE */
+	struct dvobj_priv *dvobj = pcmdpriv->dvobj;
+
+
+	if (cmd_obj->cmdcode == CMD_SET_CHANPLAN)
+		bAllow = _TRUE;
+
+	if (cmd_obj->no_io)
+		bAllow = _TRUE;
+
+	return _SUCCESS;
+}
+
+/*
+Calling Context:
+
+rtw_enqueue_cmd can only be called between kernel thread,
+since only spin_lock is used.
+
+ISR/Call-Back functions can't call this sub-function.
+
+*/
+#ifdef DBG_CMD_QUEUE
+extern u8 dump_cmd_id;
+#endif
+
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+static sint _rtw_enqueue_cmd(_queue *queue, struct cmd_obj *obj, bool to_head)
+{
+	unsigned long sp_flags;
+
+	if (obj == NULL)
+		goto exit;
+
+	/* _rtw_spinlock_bh(&queue->lock); */
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+
+	if (to_head)
+		rtw_list_insert_head(&obj->list, &queue->queue);
+	else
+		rtw_list_insert_tail(&obj->list, &queue->queue);
+
+#ifdef DBG_CMD_QUEUE
+	if (dump_cmd_id) {
+		RTW_INFO("%s===> cmdcode:0x%02x\n", __FUNCTION__, obj->cmdcode);
+		if (obj->cmdcode == CMD_SET_MLME_EVT) {
+			if (obj->parmbuf) {
+				struct rtw_evt_header *evt_hdr = (struct rtw_evt_header *)(obj->parmbuf);
+				RTW_INFO("evt_hdr->id:%d\n", evt_hdr->id);
+			}
+		}
+		if (obj->cmdcode == CMD_SET_DRV_EXTRA) {
+			if (obj->parmbuf) {
+				struct drvextra_cmd_parm *pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)(obj->parmbuf);
+				RTW_INFO("pdrvextra_cmd_parm->ec_id:0x%02x\n", pdrvextra_cmd_parm->ec_id);
+			}
+		}
+	}
+
+	if (queue->queue.prev->next != &queue->queue) {
+		RTW_INFO("[%d] head %p, tail %p, tail->prev->next %p[tail], tail->next %p[head]\n", __LINE__,
+			&queue->queue, queue->queue.prev, queue->queue.prev->prev->next, queue->queue.prev->next);
+
+		RTW_INFO("==========%s============\n", __FUNCTION__);
+		RTW_INFO("head:%p,obj_addr:%p\n", &queue->queue, obj);
+		RTW_INFO("padapter: %p\n", obj->padapter);
+		RTW_INFO("cmdcode: 0x%02x\n", obj->cmdcode);
+		RTW_INFO("res: %d\n", obj->res);
+		RTW_INFO("parmbuf: %p\n", obj->parmbuf);
+		RTW_INFO("cmdsz: %d\n", obj->cmdsz);
+		RTW_INFO("rsp: %p\n", obj->rsp);
+		RTW_INFO("rspsz: %d\n", obj->rspsz);
+		RTW_INFO("sctx: %p\n", obj->sctx);
+		RTW_INFO("list->next: %p\n", obj->list.next);
+		RTW_INFO("list->prev: %p\n", obj->list.prev);
+	}
+#endif /* DBG_CMD_QUEUE */
+
+	/* _rtw_spinunlock_bh(&queue->lock);	 */
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+exit:
+
+
+	return _SUCCESS;
+}
+#else
+static sint _rtw_enqueue_cmd(struct cmd_obj *obj, bool to_head)
+{
+	u32 res;
+
+	res = rtw_enqueue_phl_cmd(obj);
+
+#ifdef DBG_CMD_QUEUE
+	if (dump_cmd_id) {
+		RTW_INFO("%s===> cmdcode:0x%02x\n", __FUNCTION__, obj->cmdcode);
+		if (obj->cmdcode == CMD_SET_MLME_EVT) {
+			if (obj->parmbuf) {
+				struct rtw_evt_header *evt_hdr = (struct rtw_evt_header *)(obj->parmbuf);
+				RTW_INFO("evt_hdr->id:%d\n", evt_hdr->id);
+			}
+		}
+		if (obj->cmdcode == CMD_SET_DRV_EXTRA) {
+			if (obj->parmbuf) {
+				struct drvextra_cmd_parm *pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)(obj->parmbuf);
+				RTW_INFO("pdrvextra_cmd_parm->ec_id:0x%02x\n", pdrvextra_cmd_parm->ec_id);
+			}
+		}
+	}
+#endif /* DBG_CMD_QUEUE */
+	return res;
+}
+#endif
+
+u32 rtw_enqueue_cmd(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
+{
+	int res = _FAIL;
+
+	if (cmd_obj == NULL)
+		goto exit;
+
+	res = rtw_cmd_filter(pcmdpriv, cmd_obj);
+	if (_FAIL == res) {
+		rtw_free_cmd_obj(cmd_obj);
+		goto exit;
+	}
+
+
+	res = _rtw_enqueue_cmd(cmd_obj, 0);
+	#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	if (res == _SUCCESS)
+		_rtw_up_sema(&pcmdpriv->cmd_queue_sema);
+	#endif
+
+exit:
+	return res;
+}
+
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+struct	cmd_obj	*_rtw_dequeue_cmd(_queue *queue)
+{
+	struct cmd_obj *obj;
+	unsigned long sp_flags;
+
+	/* _rtw_spinlock_bh(&(queue->lock)); */
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+
+#ifdef DBG_CMD_QUEUE
+	if (queue->queue.prev->next != &queue->queue) {
+		RTW_INFO("[%d] head %p, tail %p, tail->prev->next %p[tail], tail->next %p[head]\n", __LINE__,
+			&queue->queue, queue->queue.prev, queue->queue.prev->prev->next, queue->queue.prev->next);
+	}
+#endif /* DBG_CMD_QUEUE */
+
+
+	if (rtw_is_list_empty(&(queue->queue)))
+		obj = NULL;
+	else {
+		obj = LIST_CONTAINOR(get_next(&(queue->queue)), struct cmd_obj, list);
+
+#ifdef DBG_CMD_QUEUE
+		if (queue->queue.prev->next != &queue->queue) {
+			RTW_INFO("==========%s============\n", __FUNCTION__);
+			RTW_INFO("head:%p,obj_addr:%p\n", &queue->queue, obj);
+			RTW_INFO("padapter: %p\n", obj->padapter);
+			RTW_INFO("cmdcode: 0x%02x\n", obj->cmdcode);
+			RTW_INFO("res: %d\n", obj->res);
+			RTW_INFO("parmbuf: %p\n", obj->parmbuf);
+			RTW_INFO("cmdsz: %d\n", obj->cmdsz);
+			RTW_INFO("rsp: %p\n", obj->rsp);
+			RTW_INFO("rspsz: %d\n", obj->rspsz);
+			RTW_INFO("sctx: %p\n", obj->sctx);
+			RTW_INFO("list->next: %p\n", obj->list.next);
+			RTW_INFO("list->prev: %p\n", obj->list.prev);
+		}
+
+		if (dump_cmd_id) {
+			RTW_INFO("%s===> cmdcode:0x%02x\n", __FUNCTION__, obj->cmdcode);
+			if (obj->cmdcode == CMD_SET_DRV_EXTRA) {
+				if (obj->parmbuf) {
+					struct drvextra_cmd_parm *pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)(obj->parmbuf);
+					printk("pdrvextra_cmd_parm->ec_id:0x%02x\n", pdrvextra_cmd_parm->ec_id);
+				}
+			}
+
+		}
+#endif /* DBG_CMD_QUEUE */
+
+		rtw_list_delete(&obj->list);
+	}
+
+	/* _rtw_spinunlock_bh(&(queue->lock)); */
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+
+	return obj;
+}
+
+struct	cmd_obj	*rtw_dequeue_cmd(struct cmd_priv *pcmdpriv)
+{
+	struct cmd_obj *cmd_obj;
+
+	cmd_obj = _rtw_dequeue_cmd(&pcmdpriv->cmd_queue);
+
+	return cmd_obj;
+}
+#endif
+void rtw_free_cmd_obj(struct cmd_obj *pcmd)
+{
+	if (pcmd->cmdcode == CMD_SET_DRV_EXTRA) {
+		struct drvextra_cmd_parm *extra_parm = (struct drvextra_cmd_parm *)pcmd->parmbuf;
+
+		if (extra_parm->pbuf && extra_parm->size > 0)
+			rtw_mfree(extra_parm->pbuf, extra_parm->size);
+	}
+#ifdef CONFIG_80211BE_EHT
+	else if (pcmd->cmdcode == CMD_JOINBSS) {
+		WLAN_BSSID_EX *network = (WLAN_BSSID_EX *)pcmd->parmbuf;
+
+		if (network && network->is_mld)
+			rtw_free_cloned_mld_network(network->mld_network);
+	}
+#endif
+	if (pcmd->parmbuf != NULL) {
+		/* free parmbuf in cmd_obj */
+		rtw_mfree((unsigned char *)pcmd->parmbuf, pcmd->cmdsz);
+	}
+	if (pcmd->rsp != NULL) {
+		if (pcmd->rspsz != 0) {
+			/* free rsp in cmd_obj */
+			rtw_mfree((unsigned char *)pcmd->rsp, pcmd->rspsz);
+		}
+	}
+
+	/* free cmd_obj */
+	rtw_mfree((unsigned char *)pcmd, sizeof(struct cmd_obj));
+}
+void rtw_run_cmd(_adapter *padapter, struct cmd_obj *pcmd, bool discard)
+{
+	u8 ret;
+	systime cmd_start_time;
+	u32 cmd_process_time;
+	u8(*cmd_hdl)(_adapter *padapter, u8 *pbuf);
+	void (*pcmd_callback)(_adapter *dev, struct cmd_obj *pcmd);
+	struct cmd_priv *pcmdpriv = &(adapter_to_dvobj(padapter)->cmdpriv);
+	struct drvextra_cmd_parm *extra_parm = NULL;
+
+	cmd_start_time = rtw_get_current_time();
+	pcmdpriv->cmd_issued_cnt++;
+
+	if (discard)
+		goto post_process;
+
+	if (pcmd->cmdcode >= (sizeof(wlancmds) / sizeof(struct rtw_cmd))) {
+		RTW_ERR("%s undefined cmdcode:%d\n", __func__, pcmd->cmdcode);
+		pcmd->res = H2C_PARAMETERS_ERROR;
+		goto post_process;
+	}
+
+	cmd_hdl = wlancmds[pcmd->cmdcode].cmd_hdl;
+	if (!cmd_hdl) {
+		RTW_ERR("%s no cmd_hdl for cmdcode:%d\n", __func__, pcmd->cmdcode);
+		pcmd->res = H2C_PARAMETERS_ERROR;
+		goto post_process;
+	}
+
+	if (DBG_CMD_EXECUTE)
+		RTW_INFO(ADPT_FMT" "CMD_FMT" %sexecute\n", ADPT_ARG(pcmd->padapter), CMD_ARG(pcmd)
+			, pcmd->res == H2C_ENQ_HEAD ? "ENQ_HEAD " : (pcmd->res == H2C_ENQ_HEAD_FAIL ? "ENQ_HEAD_FAIL " : ""));
+
+	ret = cmd_hdl(pcmd->padapter, pcmd->parmbuf);
+	pcmd->res = ret;
+
+	pcmdpriv->cmd_seq++;
+
+post_process:
+	pcmd->sctx_rsp_buf = NULL; /* init to NULL, may point to buffer allocated in cmd_thread after callback */
+
+	/* call callback function for post-processed */
+	if (pcmd->cmdcode < (sizeof(wlancmds) / sizeof(struct rtw_cmd)))
+		pcmd_callback = wlancmds[pcmd->cmdcode].callback;
+	else
+		pcmd_callback = NULL;
+
+	if (pcmd_callback)
+		pcmd_callback(pcmd->padapter, pcmd);
+
+	_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+	if (pcmd->sctx) {
+		if (0)
+			RTW_PRINT(FUNC_ADPT_FMT" pcmd->sctx\n", FUNC_ADPT_ARG(pcmd->padapter));
+		if (pcmd->res == H2C_SUCCESS) {
+			if (pcmd->sctx_rsp_buf)
+				pcmd->sctx->rsp = pcmd->sctx_rsp_buf;
+			rtw_sctx_done(&pcmd->sctx);
+		} else
+			rtw_sctx_done_err(&pcmd->sctx, RTW_SCTX_DONE_CMD_ERROR);
+	} else {
+		if (pcmd->sctx_rsp_buf && pcmd->sctx_rsp_buf_free) {
+			if (0)
+				RTW_PRINT(FUNC_ADPT_FMT" free pcmd->sctx_rsp_buf\n", FUNC_ADPT_ARG(pcmd->padapter));
+			pcmd->sctx_rsp_buf_free(pcmd->sctx_rsp_buf);
+		}
+	}
+	_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+
+	cmd_process_time = rtw_get_passing_time_ms(cmd_start_time);
+	if (cmd_process_time > 1000) {
+		RTW_INFO(ADPT_FMT" "CMD_FMT" process_time=%d\n", ADPT_ARG(pcmd->padapter), CMD_ARG(pcmd), cmd_process_time);
+		if (0)
+			rtw_warn_on(1);
+	}
+	rtw_free_cmd_obj(pcmd);
+}
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+void rtw_stop_cmd_thread(_adapter *adapter)
+{
+	if (adapter->cmdThread) {
+		_rtw_up_sema(&adapter->cmdpriv.cmd_queue_sema);
+		rtw_thread_stop(adapter->cmdThread);
+		adapter->cmdThread = NULL;
+	}
+}
+
+thread_return rtw_cmd_thread(thread_context context)
+{
+	u8 ret;
+	struct cmd_obj *pcmd;
+	u8 *pcmdbuf, *prspbuf;
+	systime cmd_start_time;
+	u32 cmd_process_time;
+	u8(*cmd_hdl)(_adapter *padapter, u8 *pbuf);
+	void (*pcmd_callback)(_adapter *dev, struct cmd_obj *pcmd);
+	_adapter *padapter = (_adapter *)context;
+	struct cmd_priv *pcmdpriv = &(padapter->cmdpriv);
+	struct drvextra_cmd_parm *extra_parm = NULL;
+	unsigned long sp_flags;
+
+	rtw_thread_enter("RTW_CMD_THREAD");
+
+	pcmdbuf = pcmdpriv->cmd_buf;
+	prspbuf = pcmdpriv->rsp_buf;
+	ATOMIC_SET(&(pcmdpriv->cmdthd_running), _TRUE);
+	_rtw_up_sema(&pcmdpriv->start_cmdthread_sema);
+
+
+	while (1) {
+		if (_rtw_down_sema(&pcmdpriv->cmd_queue_sema) == _FAIL) {
+			RTW_PRINT(FUNC_ADPT_FMT" _rtw_down_sema(&pcmdpriv->cmd_queue_sema) return _FAIL, break\n", FUNC_ADPT_ARG(padapter));
+			break;
+		}
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+			RTW_DBG(FUNC_ADPT_FMT "- bDriverStopped(%s) bSurpriseRemoved(%s)\n",
+				FUNC_ADPT_ARG(padapter),
+				dev_is_drv_stopped(adapter_to_dvobj(padapter)) ? "True" : "False",
+				dev_is_surprise_removed(adapter_to_dvobj(padapter)) ? "True" : "False");
+			break;
+		}
+
+		_rtw_spinlock_irq(&pcmdpriv->cmd_queue.lock, &sp_flags);
+		if (rtw_is_list_empty(&(pcmdpriv->cmd_queue.queue))) {
+			/* RTW_INFO("%s: cmd queue is empty!\n", __func__); */
+			_rtw_spinunlock_irq(&pcmdpriv->cmd_queue.lock, &sp_flags);
+			continue;
+		}
+		_rtw_spinunlock_irq(&pcmdpriv->cmd_queue.lock, &sp_flags);
+
+_next:
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+			RTW_PRINT("%s: DriverStopped(%s) SurpriseRemoved(%s) break at line %d\n",
+				  __func__
+				, dev_is_drv_stopped(adapter_to_dvobj(padapter)) ? "True" : "False"
+				, dev_is_surprise_removed(adapter_to_dvobj(padapter)) ? "True" : "False"
+				  , __LINE__);
+			break;
+		}
+
+		pcmd = rtw_dequeue_cmd(pcmdpriv);
+		if (!pcmd) {
+			continue;
+		}
+
+		cmd_start_time = rtw_get_current_time();
+		pcmdpriv->cmd_issued_cnt++;
+
+		if (pcmd->cmdsz > MAX_CMDSZ) {
+			RTW_ERR("%s cmdsz:%d > MAX_CMDSZ:%d\n", __func__, pcmd->cmdsz, MAX_CMDSZ);
+			pcmd->res = H2C_PARAMETERS_ERROR;
+			goto post_process;
+		}
+
+		if (pcmd->cmdcode >= (sizeof(wlancmds) / sizeof(struct rtw_cmd))) {
+			RTW_ERR("%s undefined cmdcode:%d\n", __func__, pcmd->cmdcode);
+			pcmd->res = H2C_PARAMETERS_ERROR;
+			goto post_process;
+		}
+
+		cmd_hdl = wlancmds[pcmd->cmdcode].cmd_hdl;
+		if (!cmd_hdl) {
+			RTW_ERR("%s no cmd_hdl for cmdcode:%d\n", __func__, pcmd->cmdcode);
+			pcmd->res = H2C_PARAMETERS_ERROR;
+			goto post_process;
+		}
+
+		if (_FAIL == rtw_cmd_filter(pcmdpriv, pcmd)) {
+			pcmd->res = H2C_DROPPED;
+			if (pcmd->cmdcode == CMD_SET_DRV_EXTRA) {
+				extra_parm = (struct drvextra_cmd_parm *)pcmd->parmbuf;
+				if (extra_parm && extra_parm->pbuf && extra_parm->size > 0)
+					rtw_mfree(extra_parm->pbuf, extra_parm->size);
+			}
+			goto post_process;
+		}
+
+		if (DBG_CMD_EXECUTE)
+			RTW_INFO(ADPT_FMT" "CMD_FMT" %sexecute\n", ADPT_ARG(pcmd->padapter), CMD_ARG(pcmd)
+				, pcmd->res == H2C_ENQ_HEAD ? "ENQ_HEAD " : (pcmd->res == H2C_ENQ_HEAD_FAIL ? "ENQ_HEAD_FAIL " : ""));
+
+		_rtw_memcpy(pcmdbuf, pcmd->parmbuf, pcmd->cmdsz);
+		ret = cmd_hdl(pcmd->padapter, pcmdbuf);
+		pcmd->res = ret;
+
+		pcmdpriv->cmd_seq++;
+
+post_process:
+
+		_rtw_mutex_lock_interruptible(&(pcmd->padapter->cmdpriv.sctx_mutex));
+		if (pcmd->sctx) {
+			if (0)
+				RTW_PRINT(FUNC_ADPT_FMT" pcmd->sctx\n", FUNC_ADPT_ARG(pcmd->padapter));
+			if (pcmd->res == H2C_SUCCESS)
+				rtw_sctx_done(&pcmd->sctx);
+			else
+				rtw_sctx_done_err(&pcmd->sctx, RTW_SCTX_DONE_CMD_ERROR);
+		}
+		_rtw_mutex_unlock(&(pcmd->padapter->cmdpriv.sctx_mutex));
+
+		cmd_process_time = rtw_get_passing_time_ms(cmd_start_time);
+		if (cmd_process_time > 1000) {
+			RTW_INFO(ADPT_FMT" "CMD_FMT" process_time=%d\n", ADPT_ARG(pcmd->padapter), CMD_ARG(pcmd), cmd_process_time);
+			if (0)
+				rtw_warn_on(1);
+		}
+
+		/* call callback function for post-processed */
+		if (pcmd->cmdcode >= (sizeof(wlancmds) / sizeof(struct rtw_cmd)))
+			pcmd_callback = wlancmds[pcmd->cmdcode].callback;
+		else
+			pcmd_callback = NULL;
+
+		if (pcmd_callback == NULL) {
+			rtw_free_cmd_obj(pcmd);
+		} else {
+			/* todo: !!! fill rsp_buf to pcmd->rsp if (pcmd->rsp!=NULL) */
+			pcmd_callback(pcmd->padapter, pcmd);/* need conider that free cmd_obj in rtw_cmd_callback */
+		}
+
+		flush_signals_thread();
+
+		goto _next;
+
+	}
+
+	/* to avoid enqueue cmd after free all cmd_obj */
+	ATOMIC_SET(&(pcmdpriv->cmdthd_running), _FALSE);
+
+	/* free all cmd_obj resources */
+	do {
+		pcmd = rtw_dequeue_cmd(pcmdpriv);
+		if (pcmd == NULL)
+			break;
+
+		if (0)
+			RTW_INFO("%s: leaving... drop "CMD_FMT"\n", __func__, CMD_ARG(pcmd));
+
+		if (pcmd->cmdcode == CMD_SET_DRV_EXTRA) {
+			extra_parm = (struct drvextra_cmd_parm *)pcmd->parmbuf;
+			if (extra_parm->pbuf && extra_parm->size > 0)
+				rtw_mfree(extra_parm->pbuf, extra_parm->size);
+		}
+
+		_rtw_mutex_lock_interruptible(&(pcmd->padapter->cmdpriv.sctx_mutex));
+		if (pcmd->sctx) {
+			if (0)
+				RTW_PRINT(FUNC_ADPT_FMT" pcmd->sctx\n", FUNC_ADPT_ARG(pcmd->padapter));
+			rtw_sctx_done_err(&pcmd->sctx, RTW_SCTX_DONE_CMD_DROP);
+		}
+		_rtw_mutex_unlock(&(pcmd->padapter->cmdpriv.sctx_mutex));
+
+		rtw_free_cmd_obj(pcmd);
+	} while (1);
+
+	RTW_INFO(FUNC_ADPT_FMT " Exit\n", FUNC_ADPT_ARG(padapter));
+
+	rtw_thread_wait_stop();
+
+	return 0;
+}
+#endif
+
+void rtw_readtssi_cmdrsp_callback(_adapter	*padapter,  struct cmd_obj *pcmd)
+{
+
+	rtw_mfree((unsigned char *) pcmd->parmbuf, pcmd->cmdsz);
+	rtw_mfree((unsigned char *) pcmd, sizeof(struct cmd_obj));
+
+#ifdef CONFIG_MP_INCLUDED
+	if (padapter->registrypriv.mp_mode == 1)
+		padapter->mppriv.workparam.bcompleted = _TRUE;
+#endif
+
+}
+
+static u8 rtw_createbss_cmd(_adapter  *adapter, int flags, bool adhoc
+	, u8 ifbmp, u8 excl_ifbmp, s8 req_band, s16 req_ch, s8 req_bw, s8 req_offset, u8 is_change_chbw)
+{
+	struct cmd_obj *cmdobj;
+	struct createbss_parm *parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	u8 res = _SUCCESS;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(adapter);
+
+	if (req_ch > 0 && req_bw >= 0 && req_offset >= 0) {
+		if (!rtw_chset_is_bchbw_valid(adapter_to_chset(adapter), req_band, req_ch, req_bw, req_offset, 0, 0)) {
+			res = _FAIL;
+			goto exit;
+		}
+	}
+
+	/* prepare cmd parameter */
+	parm = (struct createbss_parm *)rtw_zmalloc(sizeof(*parm));
+	if (parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	if (adhoc) {
+		/* for now, adhoc doesn't support ch,bw,offset request */
+		parm->adhoc = 1;
+	} else {
+		parm->adhoc = 0;
+		parm->ifbmp = ifbmp;
+		parm->excl_ifbmp = excl_ifbmp;
+		parm->req_band = req_band;
+		parm->req_ch = req_ch;
+		parm->req_bw = req_bw;
+		parm->req_offset = req_offset;
+		parm->ifbmp_ch_changed = 0;
+		parm->ch_to_set = 0;
+		parm->bw_to_set = 0;
+		parm->offset_to_set = 0;
+		parm->is_change_chbw = is_change_chbw;
+	}
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != createbss_hdl(adapter, (u8 *)parm))
+			res = _FAIL;
+		rtw_mfree((u8 *)parm, sizeof(*parm));
+	} else {
+		/* need enqueue, prepare cmd_obj and enqueue */
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_CREATE_BSS);
+		cmdobj->band_idx = padapter_link->wrlink->hw_band;
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+	}
+
+exit:
+	return res;
+}
+
+inline u8 rtw_create_ibss_cmd(_adapter *adapter, int flags)
+{
+	return rtw_createbss_cmd(adapter, flags
+		, 1
+		, 0, 0, REQ_BAND_NONE
+		, 0, REQ_BW_NONE, REQ_OFFSET_NONE /* for now, adhoc doesn't support ch,bw,offset request */
+		, _FALSE
+	);
+}
+
+inline u8 rtw_startbss_cmd(_adapter *adapter, int flags)
+{
+	return rtw_createbss_cmd(adapter, flags
+		, 0
+		, BIT(adapter->iface_id), 0, REQ_BAND_NONE
+		, 0, REQ_BW_NONE, REQ_OFFSET_NONE /* excute entire AP setup cmd */
+		, _FALSE
+	);
+}
+
+inline u8 rtw_change_bss_bchbw_cmd(_adapter *adapter, int flags
+	, u32 iflbmp, u32 excl_iflbmp, s8 req_band, s16 req_ch, s8 req_bw, s8 req_offset)
+{
+	/* single link only */
+	return rtw_createbss_cmd(adapter, flags
+		, 0
+		, iflbmp_to_ifbmp(iflbmp), iflbmp_to_ifbmp(excl_iflbmp), req_band
+		, req_ch, req_bw, req_offset
+		, _TRUE
+	);
+}
+
+u8 rtw_joinbss_cmd(_adapter *padapter, struct wlan_network *pnetwork)
+{
+	u8 res = _SUCCESS;
+	WLAN_BSSID_EX		*psecnetwork;
+	struct cmd_obj		*pcmd;
+	struct cmd_priv		*pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct joinbss_parm *param;
+	NDIS_802_11_NETWORK_INFRASTRUCTURE ndis_network_mode = pnetwork->network.InfrastructureMode;
+#ifdef CONFIG_80211BE_EHT
+	struct wlan_mld_network *pmld_network = NULL;
+	struct wlan_mld_network *rec_mld_network = NULL;
+#endif
+
+	rtw_led_control(padapter, LED_CTL_START_TO_LINK);
+
+	pcmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pcmd->padapter = padapter;
+
+	/* for hidden ap to set fw_state here */
+	if (!MLME_IS_STA(padapter) || !MLME_IS_ADHOC(padapter)) {
+		switch (ndis_network_mode) {
+		case Ndis802_11IBSS:
+			set_fwstate(&padapter->mlmepriv, WIFI_ADHOC_STATE);
+			break;
+
+		case Ndis802_11Infrastructure:
+			set_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE);
+			break;
+
+		default:
+			rtw_warn_on(1);
+			break;
+		}
+	}
+
+	param = rtw_malloc(sizeof(struct joinbss_parm));
+	if (param == NULL) {
+		if (pcmd != NULL)
+			rtw_mfree((u8 *)pcmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+#ifdef CONFIG_80211BE_EHT
+	if (pnetwork->network.is_mld) {
+		pmld_network = pnetwork->network.mld_network;
+		rec_mld_network = rtw_clone_to_join_mld_network(padapter, pmld_network);
+		if (rec_mld_network == NULL) {
+			rtw_mfree((u8 *)pcmd, sizeof(struct cmd_obj));
+			rtw_mfree((u8 *)param, sizeof(struct joinbss_parm));
+			res = _FAIL;
+			goto exit;
+		}
+
+		_rtw_memcpy(&param->network, &pnetwork->network, get_WLAN_BSSID_EX_sz(&pnetwork->network));
+		param->network.mld_network = rec_mld_network;
+	} else
+#endif
+	{
+		_rtw_memset(&param->network, 0, sizeof(WLAN_BSSID_EX));
+		_rtw_memcpy(&param->network, &pnetwork->network, get_WLAN_BSSID_EX_sz(&pnetwork->network));
+	}
+
+	init_h2fwcmd_w_parm_no_rsp(pcmd, param, CMD_JOINBSS);
+
+	res = rtw_enqueue_cmd(pcmdpriv, pcmd);
+
+exit:
+	return res;
+}
+
+/* for sta_mode only */
+static u8 sta_disassoc_cmd(struct _ADAPTER *a, u32 deauth_timeout_ms, int flags)
+{
+	struct cmd_priv *cmdpriv = &adapter_to_dvobj(a)->cmdpriv;
+	struct cmd_obj *cmd = NULL;
+	struct disconnect_parm *param = NULL;
+	struct submit_ctx sctx;
+	enum rtw_phl_status status;
+	int ret;
+	u8 res = _FAIL;
+
+
+	if (!MLME_IS_ASOC(a))
+		return _SUCCESS;
+
+	param = (struct disconnect_parm *)rtw_zmalloc(sizeof(*param));
+	if (!param) {
+		RTW_ERR(FUNC_ADPT_FMT ": alloc param FAIL!", FUNC_ADPT_ARG(a));
+		goto exit;
+	}
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmd));
+	if (!cmd) {
+		RTW_ERR(FUNC_ADPT_FMT ": alloc cmd FAIL!", FUNC_ADPT_ARG(a));
+		rtw_mfree((u8 *)param, sizeof(*param));
+		goto exit;
+	}
+
+	param->deauth_timeout_ms = deauth_timeout_ms;
+	init_h2fwcmd_w_parm_no_rsp(cmd, param, CMD_DISCONNECT);
+	cmd->padapter = a;
+	if (flags & RTW_CMDF_WAIT_ACK) {
+		cmd->sctx = &sctx;
+		rtw_sctx_init(&sctx, 2000);
+	}
+
+	status = rtw_disconnect_cmd(a, cmd);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		rtw_mfree((u8 *)param, sizeof(*param));
+		rtw_mfree((u8 *)cmd, sizeof(*cmd));
+		if (status == RTW_PHL_STATUS_RESOURCE) {
+			RTW_INFO(FUNC_ADPT_FMT ": disconnect is on-going%s...\n",
+				 FUNC_ADPT_ARG(a),
+				 (flags & RTW_CMDF_WAIT_ACK) ? ", wait to complete" : "");
+			if (flags & RTW_CMDF_WAIT_ACK) {
+				if (rtw_disconnect_wait_complete(a, deauth_timeout_ms)) {
+					res = _SUCCESS;
+				} else {
+					RTW_WARN(FUNC_ADPT_FMT ": Fail to wait disconnect complete!\n",
+						 FUNC_ADPT_ARG(a));
+				}
+			} else {
+				res = _SUCCESS;
+			}
+		} else {
+			RTW_ERR(FUNC_ADPT_FMT ": send disconnect cmd FAIL!(0x%x)\n",
+				FUNC_ADPT_ARG(a), status);
+		}
+		goto exit;
+	}
+	res = _SUCCESS;
+
+	if (flags & RTW_CMDF_WAIT_ACK) {
+		ret = rtw_sctx_wait(&sctx, __func__);
+		if (ret == _FAIL)
+			res = _FAIL;
+		_rtw_spinlock_bh(&a->disconnect_lock);
+		if (a->discon_cmd) {
+			a->discon_cmd->sctx = NULL;
+			/*
+			 * a->discon_param would be
+			 * freed by disconnect cmd dispatcher.
+			 */
+		}
+		_rtw_spinunlock_bh(&a->disconnect_lock);
+	}
+
+exit:
+	return res;
+}
+
+u8 rtw_disassoc_cmd(_adapter *padapter, u32 deauth_timeout_ms, int flags) /* for sta_mode */
+{
+	struct cmd_obj *cmdobj = NULL;
+	struct disconnect_parm *param = NULL;
+	struct cmd_priv *cmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+	struct submit_ctx sctx;
+	u8 res = _SUCCESS;
+
+	if (MLME_IS_STA(padapter))
+		return sta_disassoc_cmd(padapter, deauth_timeout_ms, flags);
+
+	/* prepare cmd parameter */
+	param = (struct disconnect_parm *)rtw_zmalloc(sizeof(*param));
+	if (param == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	param->deauth_timeout_ms = deauth_timeout_ms;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (disconnect_hdl(padapter, (u8 *)param) != H2C_SUCCESS)
+			res = _FAIL;
+		rtw_mfree((u8 *)param, sizeof(*param));
+
+	} else {
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree((u8 *)param, sizeof(*param));
+			goto exit;
+		}
+		cmdobj->padapter = padapter;
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, param, CMD_DISCONNECT);
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 2000);
+		}
+		res = rtw_enqueue_cmd(cmdpriv, cmdobj);
+		if (res == _FAIL) {
+			RTW_ERR(FUNC_ADPT_FMT ": enqueue disconnect cmd FAIL!\n",
+				FUNC_ADPT_ARG(padapter));
+			goto exit;
+		}
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&cmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&cmdpriv->sctx_mutex);
+		}
+	}
+
+exit:
+	return res;
+}
+
+
+u8 rtw_stop_ap_cmd(_adapter *adapter, u8 flags)
+{
+#ifdef CONFIG_AP_MODE
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 res = _SUCCESS;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != stop_ap_hdl(adapter))
+			res = _FAIL;
+	} else {
+		parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+		if (parm == NULL) {
+			res = _FAIL;
+			goto exit;
+		}
+
+		parm->ec_id = STOP_AP_WK_CID;
+		parm->type = 0;
+		parm->size = 0;
+		parm->pbuf = NULL;
+
+		/* need enqueue, prepare cmd_obj and enqueue */
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 2000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		}
+	}
+exit:
+	return res;
+#endif
+}
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+u8 rtw_tx_control_cmd(_adapter *adapter)
+{
+	struct cmd_obj *cmd;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+
+	u8 res = _SUCCESS;
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL){
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = adapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->ec_id = TBTX_CONTROL_TX_WK_CID;
+	pdrvextra_cmd_parm->type = 0;
+	pdrvextra_cmd_parm->size = 0;
+	pdrvextra_cmd_parm->pbuf = NULL;
+	init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;	
+}
+#endif
+
+u8 rtw_setopmode_cmd(_adapter  *adapter, NDIS_802_11_NETWORK_INFRASTRUCTURE networktype, u8 flags)
+{
+	struct cmd_obj *cmdobj;
+	struct setopmode_parm *parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 res = _SUCCESS;
+
+	/* prepare cmd parameter */
+	parm = (struct setopmode_parm *)rtw_zmalloc(sizeof(*parm));
+	if (parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	parm->mode = (u8)networktype;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != setopmode_hdl(adapter, (u8 *)parm))
+			res = _FAIL;
+		rtw_mfree((u8 *)parm, sizeof(*parm));
+	} else {
+		/* need enqueue, prepare cmd_obj and enqueue */
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_OPMODE);
+
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 2000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		}
+	}
+
+exit:
+	return res;
+}
+
+u8 rtw_setstakey_cmd(_adapter *padapter, struct sta_info *sta, u8 key_type, bool enqueue)
+{
+	struct set_stakey_parm	setstakey_para;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct _ADAPTER_LINK *padapter_link = sta->padapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+	u8 key_len =16;
+	u8 res = _SUCCESS;
+
+	_rtw_memset(&setstakey_para, 0, sizeof(struct set_stakey_parm));
+	_rtw_memcpy(setstakey_para.addr, sta->phl_sta->mac_addr, ETH_ALEN);
+
+	if (MLME_IS_STA(padapter))
+		setstakey_para.algorithm = (unsigned char) psecuritypriv->dot11PrivacyAlgrthm;
+	else
+		GET_ENCRY_ALGO(psecuritypriv, sta, setstakey_para.algorithm, _FALSE);
+
+	if ((setstakey_para.algorithm == _GCMP_256_) || (setstakey_para.algorithm == _CCMP_256_))
+		key_len = 32;
+
+	if (key_type == GROUP_KEY) {
+		_rtw_memcpy(&setstakey_para.key, &lsecuritypriv->dot118021XGrpKey[lsecuritypriv->dot118021XGrpKeyid].skey, key_len);
+		setstakey_para.gk = 1;
+		setstakey_para.keyid = (u8)lsecuritypriv->dot118021XGrpKeyid;
+	} else if (key_type == UNICAST_KEY)
+		_rtw_memcpy(&setstakey_para.key, &sta->dot118021x_UncstKey, key_len);
+#ifdef CONFIG_TDLS
+	else if (key_type == TDLS_KEY) {
+		_rtw_memcpy(&setstakey_para.key, sta->tpk.tk, key_len);
+		setstakey_para.algorithm = (u8)sta->dot118021XPrivacy;
+	}
+#endif /* CONFIG_TDLS */
+
+	/* jeff: set this becasue at least sw key is ready */
+	padapter->securitypriv.busetkipkey = _TRUE;
+
+	if (enqueue) {
+		set_stakey_hdl(padapter, &setstakey_para, PHL_CMD_NO_WAIT, 0);
+	} else {
+		set_stakey_hdl(padapter, &setstakey_para, PHL_CMD_DIRECTLY, 0);
+	}
+exit:
+	return res;
+}
+
+u8 rtw_clearstakey_cmd(_adapter *padapter, struct sta_info *sta, u8 enqueue)
+{
+	struct cmd_obj *cmd;
+	struct set_stakey_parm	*psetstakey_para;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct set_stakey_rsp *psetstakey_rsp = NULL;
+	s16 cam_id = 0;
+	u8 res = _SUCCESS;
+
+	if (!sta) {
+		RTW_ERR("%s sta == NULL\n", __func__);
+		goto exit;
+	}
+
+	if (!enqueue)
+		rtw_hw_del_all_key(padapter, sta, PHL_CMD_DIRECTLY, 0);
+	else
+		rtw_hw_del_all_key(padapter, sta, PHL_CMD_NO_WAIT, 0);
+
+exit:
+
+
+	return res;
+}
+
+u8 rtw_addbareq_cmd(_adapter *padapter, u8 tid, u8 *addr)
+{
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct cmd_obj *cmd;
+	struct addBaReq_parm *paddbareq_parm;
+
+	u8	res = _SUCCESS;
+
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	paddbareq_parm = (struct addBaReq_parm *)rtw_zmalloc(sizeof(struct addBaReq_parm));
+	if (paddbareq_parm == NULL) {
+		rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	paddbareq_parm->tid = tid;
+	_rtw_memcpy(paddbareq_parm->addr, addr, ETH_ALEN);
+
+	init_h2fwcmd_w_parm_no_rsp(cmd, paddbareq_parm, CMD_ADD_BAREQ);
+
+	/* RTW_INFO("rtw_addbareq_cmd, tid=%d\n", tid); */
+
+	/* rtw_enqueue_cmd(pcmdpriv, ph2c);	 */
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;
+}
+
+u8 rtw_addbarsp_cmd(_adapter *padapter, u8 *addr, u16 tid,
+		    struct ADDBA_request *paddba_req, u8 status,
+		    u16 size, u16 start_seq)
+{
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct cmd_obj *cmd;
+	struct addBaRsp_parm *paddBaRsp_parm;
+	u8 res = _SUCCESS;
+
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	paddBaRsp_parm = (struct addBaRsp_parm *)rtw_zmalloc(sizeof(struct addBaRsp_parm));
+
+	if (paddBaRsp_parm == NULL) {
+		rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	_rtw_memcpy(paddBaRsp_parm->addr, addr, ETH_ALEN);
+	_rtw_memcpy(&(paddBaRsp_parm->preq), paddba_req, sizeof(struct ADDBA_request));
+	paddBaRsp_parm->tid = tid;
+	paddBaRsp_parm->status = status;
+	paddBaRsp_parm->size = size;
+	paddBaRsp_parm->start_seq = start_seq;
+
+	init_h2fwcmd_w_parm_no_rsp(cmd, paddBaRsp_parm, CMD_ADD_BARSP);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+
+
+	return res;
+}
+
+u8 rtw_delba_cmd(struct _ADAPTER *a, u8 *addr, u16 tid)
+{
+	struct cmd_priv *cmdpriv = &adapter_to_dvobj(a)->cmdpriv;
+	struct cmd_obj *cmd = NULL;
+	struct addBaReq_parm *parm = NULL;
+
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (!cmd)
+		return _FAIL;
+	cmd->padapter = a;
+
+	parm = (struct addBaReq_parm *)rtw_zmalloc(sizeof(struct addBaReq_parm));
+	if (!parm) {
+		rtw_mfree(cmd, sizeof(struct cmd_obj));
+		return _FAIL;
+	}
+
+	parm->tid = tid;
+	_rtw_memcpy(parm->addr, addr, ETH_ALEN);
+	init_h2fwcmd_w_parm_no_rsp(cmd, parm, CMD_DELBA);
+
+	return rtw_enqueue_cmd(cmdpriv, cmd);
+}
+
+/* add for CONFIG_IEEE80211W, none 11w can use it */
+u8 rtw_reset_securitypriv_cmd(_adapter *padapter)
+{
+	struct cmd_obj *cmd;
+	struct drvextra_cmd_parm  *pdrvextra_cmd_parm;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8 res = _SUCCESS;
+
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->ec_id = RESET_SECURITYPRIV;
+	pdrvextra_cmd_parm->type = 0;
+	pdrvextra_cmd_parm->size = 0;
+	pdrvextra_cmd_parm->pbuf = NULL;
+
+	init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+
+	/* rtw_enqueue_cmd(pcmdpriv, ph2c);	 */
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;
+
+}
+
+void free_assoc_resources_hdl(_adapter *padapter, u8 lock_scanned_queue)
+{
+	rtw_free_assoc_resources(padapter, lock_scanned_queue);
+}
+
+u8 rtw_free_assoc_resources_cmd(_adapter *padapter, u8 lock_scanned_queue, int flags)
+{
+	struct cmd_obj *cmd;
+	struct drvextra_cmd_parm  *pdrvextra_cmd_parm;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8	res = _SUCCESS;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		free_assoc_resources_hdl(padapter, lock_scanned_queue);
+	}
+	else {
+		cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+		if (cmd == NULL) {
+			res = _FAIL;
+			goto exit;
+		}
+		cmd->padapter = padapter;
+
+		pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+		if (pdrvextra_cmd_parm == NULL) {
+			rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+			res = _FAIL;
+			goto exit;
+		}
+
+		pdrvextra_cmd_parm->ec_id = FREE_ASSOC_RESOURCES;
+		pdrvextra_cmd_parm->type = lock_scanned_queue;
+		pdrvextra_cmd_parm->size = 0;
+		pdrvextra_cmd_parm->pbuf = NULL;
+
+		init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmd->sctx = &sctx;
+			rtw_sctx_init(&sctx, 2000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmd->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		}
+	}
+exit:
+	return res;
+
+}
+#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+u8 rtw_dynamic_chk_wk_cmd(_adapter *padapter)
+{
+	struct cmd_obj *cmd;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8	res = _SUCCESS;
+
+
+	/* only  primary padapter does this cmd */
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->ec_id = DYNAMIC_CHK_WK_CID;
+	pdrvextra_cmd_parm->type = 0;
+	pdrvextra_cmd_parm->size = 0;
+	pdrvextra_cmd_parm->pbuf = NULL;
+	init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+
+	/* rtw_enqueue_cmd(pcmdpriv, ph2c);	 */
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;
+}
+#endif
+
+u8 rtw_set_chbw_cmd(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				struct rtw_chan_def *chdef, u8 flags, enum rfk_tri_type rt_type)
+{
+	struct cmd_obj *pcmdobj;
+	struct set_ch_parm *set_ch_parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 res = _SUCCESS;
+
+
+	RTW_INFO(FUNC_NDEV_FMT" band:%d, ch:%u, bw:%u, ch_offset:%u\n",
+		 FUNC_NDEV_ARG(padapter->pnetdev),
+		 chdef->band, chdef->chan, chdef->bw, chdef->offset);
+
+	/* check input parameter */
+
+	/* prepare cmd parameter */
+	set_ch_parm = (struct set_ch_parm *)rtw_zmalloc(sizeof(*set_ch_parm));
+	if (set_ch_parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	set_ch_parm->band = chdef->band;
+	set_ch_parm->ch = chdef->chan;
+	set_ch_parm->bw = chdef->bw;
+	set_ch_parm->ch_offset = chdef->offset;
+	set_ch_parm->rt_type = rt_type;
+	set_ch_parm->link_idx = padapter_link->wrlink->id;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != rtw_set_chbw_hdl(padapter, (u8 *)set_ch_parm))
+			res = _FAIL;
+
+		rtw_mfree((u8 *)set_ch_parm, sizeof(*set_ch_parm));
+	} else {
+		/* need enqueue, prepare cmd_obj and enqueue */
+		pcmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+		if (pcmdobj == NULL) {
+			rtw_mfree((u8 *)set_ch_parm, sizeof(*set_ch_parm));
+			res = _FAIL;
+			goto exit;
+		}
+		pcmdobj->padapter = padapter;
+
+		init_h2fwcmd_w_parm_no_rsp(pcmdobj, set_ch_parm, CMD_SET_CHANNEL);
+
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			pcmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 10 * 1000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, pcmdobj);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				pcmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		}
+	}
+
+	/* do something based on res... */
+exit:
+	RTW_INFO(FUNC_NDEV_FMT" res:%u\n", FUNC_NDEV_ARG(padapter->pnetdev), res);
+	return res;
+}
+
+#ifdef CONFIG_RTW_LED_HANDLED_BY_CMD_THREAD
+u8 rtw_led_blink_cmd(_adapter *padapter, void *pLed)
+{
+	struct cmd_obj	*pcmdobj;
+	struct LedBlink_param *ledBlink_param;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8 res = _SUCCESS;
+
+	pcmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pcmdobj->padapter = padapter;
+
+	ledBlink_param = (struct LedBlink_param *)rtw_zmalloc(sizeof(struct LedBlink_param));
+	if (ledBlink_param == NULL) {
+		rtw_mfree((u8 *)pcmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	ledBlink_param->pLed = pLed;
+
+	init_h2fwcmd_w_parm_no_rsp(pcmdobj, ledBlink_param, CMD_LEDBLINK);
+	res = rtw_enqueue_cmd(pcmdpriv, pcmdobj);
+
+exit:
+	return res;
+}
+#endif /*CONFIG_RTW_LED_HANDLED_BY_CMD_THREAD*/
+
+u8 rtw_tdls_cmd(_adapter *padapter, u8 *addr, u8 option, u8 flags)
+{
+	u8 res = _SUCCESS;
+#ifdef CONFIG_TDLS
+	struct	cmd_obj	*pcmdobj;
+	struct	TDLSoption_param *TDLSoption;
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct	cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	struct submit_ctx sctx;
+
+	pcmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pcmdobj->padapter = padapter;
+
+	TDLSoption = (struct TDLSoption_param *)rtw_zmalloc(sizeof(struct TDLSoption_param));
+	if (TDLSoption == NULL) {
+		rtw_mfree((u8 *)pcmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	_rtw_spinlock_bh(&(padapter->tdlsinfo.cmd_lock));
+	if (addr != NULL)
+		_rtw_memcpy(TDLSoption->addr, addr, 6);
+	TDLSoption->option = option;
+	_rtw_spinunlock_bh(&(padapter->tdlsinfo.cmd_lock));
+	init_h2fwcmd_w_parm_no_rsp(pcmdobj, TDLSoption, CMD_TDLS);
+
+	if (flags & RTW_CMDF_WAIT_ACK) {
+		pcmdobj->sctx = &sctx;
+		rtw_sctx_init(&sctx, 3000);
+	}
+
+	res = rtw_enqueue_cmd(pcmdpriv, pcmdobj);
+
+	if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+		rtw_sctx_wait(&sctx, __func__);
+		_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+		if (sctx.status == RTW_SCTX_SUBMITTED)
+			pcmdobj->sctx = NULL;
+		_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+			res = _FAIL;
+	}
+
+exit:
+#endif /* CONFIG_TDLS */
+
+	return res;
+}
+
+u8 rtw_ssmps_wk_hdl(_adapter *adapter, struct ssmps_cmd_parm *ssmp_param)
+{
+	u8 res = _SUCCESS;
+	struct sta_info *sta = ssmp_param->sta;
+	u8 smps = ssmp_param->smps;
+
+	if (sta == NULL)
+		return _FALSE;
+
+	if (smps)
+		rtw_ssmps_enter(adapter, sta);
+	else
+		rtw_ssmps_leave(adapter, sta);
+	return res;
+}
+
+u8 rtw_ssmps_wk_cmd(_adapter *adapter, struct sta_info *sta, u8 smps, u8 enqueue)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *cmd_parm;
+	struct ssmps_cmd_parm *ssmp_param;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	u8 res = _SUCCESS;
+
+	if (enqueue) {
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+		if (cmd_parm == NULL) {
+			rtw_mfree((unsigned char *)cmdobj, sizeof(struct cmd_obj));
+			res = _FAIL;
+			goto exit;
+		}
+
+		ssmp_param = (struct ssmps_cmd_parm *)rtw_zmalloc(sizeof(struct ssmps_cmd_parm));
+		if (ssmp_param == NULL) {
+			rtw_mfree((u8 *)cmdobj, sizeof(struct cmd_obj));
+			rtw_mfree((u8 *)cmd_parm, sizeof(struct drvextra_cmd_parm));
+			res = _FAIL;
+			goto exit;
+		}
+
+		ssmp_param->smps = smps;
+		ssmp_param->sta = sta;
+
+		cmd_parm->ec_id = SSMPS_WK_CID;
+		cmd_parm->type = 0;
+		cmd_parm->size = sizeof(struct ssmps_cmd_parm);
+		cmd_parm->pbuf = (u8 *)ssmp_param;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, cmd_parm, CMD_SET_DRV_EXTRA);
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+	} else {
+		struct ssmps_cmd_parm tmp_ssmp_param;
+
+		tmp_ssmp_param.smps = smps;
+		tmp_ssmp_param.sta = sta;
+		rtw_ssmps_wk_hdl(adapter, &tmp_ssmp_param);
+	}
+
+exit:
+	return res;
+}
+
+#ifdef CONFIG_SUPPORT_STATIC_SMPS
+u8 _ssmps_chk_by_tp(_adapter *adapter)
+{
+	u8 enter_smps = _FALSE;
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	struct sta_info *psta;
+	u32 tx_tp_mbits, rx_tp_mbits;
+
+	if (!MLME_IS_STA(adapter) ||
+		!rtw_hw_is_mimo_support(adapter) ||
+		!pmlmeext->ssmps_en ||
+		(pmlmeext->chandef.chan > 14)
+	)
+		return enter_smps;
+
+	psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+	if (psta == NULL) {
+		RTW_ERR(ADPT_FMT" sta == NULL\n", ADPT_ARG(adapter));
+		rtw_warn_on(1);
+		return enter_smps;
+	}
+	/*TODO*/
+	if (psta->phl_sta->asoc_cap.nss_tx == 1)
+		return enter_smps;
+
+	tx_tp_mbits = psta->sta_stats.tx_tp_kbits >> 10;
+	rx_tp_mbits = psta->sta_stats.rx_tp_kbits >> 10;
+
+	#ifdef DBG_STATIC_SMPS
+	if (pmlmeext->ssmps_test) {
+		enter_smps = (pmlmeext->ssmps_test_en == 1) ? _TRUE : _FALSE;
+	}
+	else
+	#endif
+	{
+		if ((tx_tp_mbits <= pmlmeext->ssmps_tx_tp_th) &&
+			(rx_tp_mbits <= pmlmeext->ssmps_rx_tp_th))
+			enter_smps = _TRUE;
+		else
+			enter_smps = _FALSE;
+	}
+
+	if (1) {
+		RTW_INFO(FUNC_ADPT_FMT" tx_tp:%d [%d], rx_tp:%d [%d] , SSMPS enter :%s\n",
+			FUNC_ADPT_ARG(adapter),
+			tx_tp_mbits, pmlmeext->ssmps_tx_tp_th,
+			rx_tp_mbits, pmlmeext->ssmps_rx_tp_th,
+			(enter_smps == _TRUE) ? "True" : "False");
+		#ifdef DBG_STATIC_SMPS
+		RTW_INFO(FUNC_ADPT_FMT" test:%d test_en:%d\n",
+			FUNC_ADPT_ARG(adapter),
+			pmlmeext->ssmps_test,
+			pmlmeext->ssmps_test_en);
+		#endif
+	}
+
+	if (enter_smps) {
+		if (psta->phl_sta->sm_ps != SM_PS_STATIC)
+			rtw_ssmps_enter(adapter, psta);
+	} else {
+		if (psta->phl_sta->sm_ps != SM_PS_DISABLE)
+			rtw_ssmps_leave(adapter, psta);
+		else {
+			u8 ps_change = _FALSE;
+
+			if (enter_smps && psta->phl_sta->sm_ps != SM_PS_STATIC)
+				ps_change = _TRUE;
+			else if (!enter_smps && psta->phl_sta->sm_ps != SM_PS_DISABLE)
+				ps_change = _TRUE;
+
+			if (ps_change)
+				rtw_ssmps_wk_cmd(adapter, psta, enter_smps, 1);
+		}
+	}
+
+	return enter_smps;
+}
+#endif /*CONFIG_SUPPORT_STATIC_SMPS*/
+
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+void rtw_ctrl_txss_update(_adapter *adapter, struct sta_info *sta)
+{
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+
+	pmlmeext->txss_bk = sta->phl_sta->asoc_cap.nss_rx;
+}
+
+u8 rtw_ctrl_txss(_adapter *adapter, struct sta_info *sta, bool tx_1ss)
+{
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(adapter);
+	u8 lps_changed = _FALSE;
+	u8 rst = _SUCCESS;
+
+	if (pmlmeext->txss_1ss == tx_1ss)
+		return _FALSE;
+	/*
+	if (pwrpriv->bLeisurePs && pwrpriv->pwr_mode != PM_PS_MODE_ACTIVE) {
+		lps_changed = _TRUE;
+		LPS_Leave(adapter, "LPS_CTRL_TXSS");
+	}
+	*/
+
+	RTW_INFO(ADPT_FMT" STA [" MAC_FMT "] set tx to %d ss\n",
+		ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr),
+		(tx_1ss) ? 1 : rtw_get_sta_tx_nss(adapter, sta));
+
+	/*update ra*/
+	if (tx_1ss)
+		sta->phl_sta->asoc_cap.nss_rx = 1;
+	else
+		sta->phl_sta->asoc_cap.nss_rx = pmlmeext->txss_bk;
+	rtw_sta_hal_ra_mask_update_cmd(adapter, sta, RTW_CMDF_DIRECTLY);
+
+	/*configure trx mode*/
+	/*rtw_phydm_trx_cfg(adapter, tx_1ss);*/
+	pmlmeext->txss_1ss = tx_1ss;
+	/*
+	if (lps_changed)
+		LPS_Enter(adapter, "LPS_CTRL_TXSS");
+	*/
+
+	return rst;
+}
+
+u8 rtw_ctrl_txss_wk_hdl(_adapter *adapter, struct txss_cmd_parm *txss_param)
+{
+	if (!txss_param->sta)
+		return _FALSE;
+
+	return rtw_ctrl_txss(adapter, txss_param->sta, txss_param->tx_1ss);
+}
+
+u8 rtw_ctrl_txss_wk_cmd(_adapter *adapter, struct sta_info *sta, bool tx_1ss, u8 flag)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *cmd_parm;
+	struct txss_cmd_parm *txss_param;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8	res = _SUCCESS;
+
+	txss_param = (struct txss_cmd_parm *)rtw_zmalloc(sizeof(struct txss_cmd_parm));
+	if (txss_param == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	txss_param->tx_1ss = tx_1ss;
+	txss_param->sta = sta;
+
+	if (flag & RTW_CMDF_DIRECTLY) {
+		res = rtw_ctrl_txss_wk_hdl(adapter, txss_param);
+		rtw_mfree((u8 *)txss_param, sizeof(*txss_param));
+	} else {
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+		if (cmd_parm == NULL) {
+			rtw_mfree((u8 *)cmdobj, sizeof(struct cmd_obj));
+			res = _FAIL;
+			goto exit;
+		}
+
+		cmd_parm->ec_id = TXSS_WK_CID;
+		cmd_parm->type = 0;
+		cmd_parm->size = sizeof(struct txss_cmd_parm);
+		cmd_parm->pbuf = (u8 *)txss_param;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, cmd_parm, CMD_SET_DRV_EXTRA);
+
+		if (flag & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 10 * 1000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+		if (res == _SUCCESS && (flag & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+			if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+				res = _FAIL;
+		}
+	}
+
+exit:
+	return res;
+}
+
+void rtw_ctrl_tx_ss_by_tp(_adapter *adapter)
+{
+	bool tx_1ss  = _FALSE; /*change tx from 2ss to 1ss*/
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	struct sta_info *psta;
+	u32 tx_tp_mbits;
+
+	if (!MLME_IS_STA(adapter) ||
+		!rtw_hw_is_mimo_support(adapter) ||
+		!pmlmeext->txss_ctrl_en
+	)
+		return;
+
+	psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+	if (psta == NULL) {
+		RTW_ERR(ADPT_FMT" sta == NULL\n", ADPT_ARG(adapter));
+		rtw_warn_on(1);
+		return;
+	}
+
+	tx_tp_mbits = psta->sta_stats.tx_tp_kbits >> 10;
+	if (tx_tp_mbits >= pmlmeext->txss_tp_th) {
+		tx_1ss = _FALSE;
+	} else {
+		if (pmlmeext->txss_tp_chk_cnt && --pmlmeext->txss_tp_chk_cnt)
+			tx_1ss = _FALSE;
+		else
+			tx_1ss = _TRUE;
+	}
+
+	if (1) {
+		RTW_INFO(FUNC_ADPT_FMT" tx_tp:%d [%d] tx_1ss(%d):%s\n",
+			FUNC_ADPT_ARG(adapter),
+			tx_tp_mbits, pmlmeext->txss_tp_th,
+			pmlmeext->txss_tp_chk_cnt,
+			(tx_1ss == _TRUE) ? "True" : "False");
+	}
+
+	if (pmlmeext->txss_1ss != tx_1ss)
+			rtw_ctrl_txss(adapter, psta, tx_1ss);
+}
+#ifdef DBG_CTRL_TXSS
+void dbg_ctrl_txss(_adapter *adapter, bool tx_1ss)
+{
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	struct sta_info *psta;
+
+	if (!MLME_IS_STA(adapter) ||
+		!rtw_hw_is_mimo_support(adapter)
+	)
+		return;
+
+	psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+	if (psta == NULL) {
+		RTW_ERR(ADPT_FMT" sta == NULL\n", ADPT_ARG(adapter));
+		rtw_warn_on(1);
+		return;
+	}
+
+	rtw_ctrl_txss(adapter, psta, tx_1ss);
+}
+#endif
+#endif /*CONFIG_CTRL_TXSS_BY_TP*/
+
+u8 traffic_status_watchdog(_adapter *padapter)
+{
+	u8	bEnterPS = _FALSE;
+	u16 BusyThresholdHigh;
+	u16	BusyThresholdLow;
+	u16	BusyThreshold;
+	u8	bBusyTraffic = _FALSE, bTxBusyTraffic = _FALSE, bRxBusyTraffic = _FALSE;
+	u8	bHigherBusyTraffic = _FALSE, bHigherBusyRxTraffic = _FALSE, bHigherBusyTxTraffic = _FALSE;
+
+	struct mlme_priv		*pmlmepriv = &(padapter->mlmepriv);
+#ifdef CONFIG_TDLS
+	struct tdls_info *ptdlsinfo = &(padapter->tdlsinfo);
+	struct tdls_txmgmt txmgmt;
+	u8 baddr[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+#endif /* CONFIG_TDLS */
+#ifdef CONFIG_TRAFFIC_PROTECT
+	RT_LINK_DETECT_T *link_detect = &pmlmepriv->LinkDetectInfo;
+#endif
+
+#ifdef CONFIG_BTC
+	if (padapter->registrypriv.wifi_spec != 1) {
+		BusyThresholdHigh = 25;
+		BusyThresholdLow = 10;
+	} else
+#endif /* CONFIG_BTC */
+	{
+		BusyThresholdHigh = 100;
+		BusyThresholdLow = 75;
+	}
+	BusyThreshold = BusyThresholdHigh;
+
+
+	/*  */
+	/* Determine if our traffic is busy now */
+	/*  */
+	if ((check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+	    /*&& !MgntInitAdapterInProgress(pMgntInfo)*/) {
+		/* if we raise bBusyTraffic in last watchdog, using lower threshold. */
+		if (pmlmepriv->LinkDetectInfo.bBusyTraffic)
+			BusyThreshold = BusyThresholdLow;
+
+		if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > BusyThreshold ||
+		    pmlmepriv->LinkDetectInfo.NumTxOkInPeriod > BusyThreshold) {
+			bBusyTraffic = _TRUE;
+
+			if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > pmlmepriv->LinkDetectInfo.NumTxOkInPeriod)
+				bRxBusyTraffic = _TRUE;
+			else
+				bTxBusyTraffic = _TRUE;
+		}
+
+		/* Higher Tx/Rx data. */
+		if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > 4000 ||
+		    pmlmepriv->LinkDetectInfo.NumTxOkInPeriod > 4000) {
+			bHigherBusyTraffic = _TRUE;
+
+			if (pmlmepriv->LinkDetectInfo.NumRxOkInPeriod > pmlmepriv->LinkDetectInfo.NumTxOkInPeriod)
+				bHigherBusyRxTraffic = _TRUE;
+			else
+				bHigherBusyTxTraffic = _TRUE;
+		}
+
+#ifdef CONFIG_TRAFFIC_PROTECT
+#define TX_ACTIVE_TH 10
+#define RX_ACTIVE_TH 20
+#define TRAFFIC_PROTECT_PERIOD_MS 4500
+
+		if (link_detect->NumTxOkInPeriod > TX_ACTIVE_TH
+		    || link_detect->NumRxUnicastOkInPeriod > RX_ACTIVE_TH) {
+
+			RTW_INFO(FUNC_ADPT_FMT" acqiure wake_lock for %u ms(tx:%d,rx_unicast:%d)\n",
+				 FUNC_ADPT_ARG(padapter),
+				 TRAFFIC_PROTECT_PERIOD_MS,
+				 link_detect->NumTxOkInPeriod,
+				 link_detect->NumRxUnicastOkInPeriod);
+
+			rtw_lock_traffic_suspend_timeout(TRAFFIC_PROTECT_PERIOD_MS);
+		}
+#endif
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_AUTOSETUP
+		/* TDLS_WATCHDOG_PERIOD * 2sec, periodically send */
+		if (rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_TDLS) == _TRUE) {
+			if ((ptdlsinfo->watchdog_count % TDLS_WATCHDOG_PERIOD) == 0) {
+				_rtw_memcpy(txmgmt.peer, baddr, ETH_ALEN);
+				issue_tdls_dis_req(padapter, &txmgmt);
+			}
+			ptdlsinfo->watchdog_count++;
+		}
+#endif /* CONFIG_TDLS_AUTOSETUP */
+#endif /* CONFIG_TDLS */
+
+#ifdef CONFIG_SUPPORT_STATIC_SMPS
+		_ssmps_chk_by_tp(padapter);
+#endif
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+		rtw_ctrl_tx_ss_by_tp(padapter);
+#endif
+
+	}
+
+#if 0
+	session_tracker_chk_cmd(padapter, NULL);
+#endif
+
+#ifdef CONFIG_BEAMFORMING
+#ifdef RTW_WKARD_TX_DISABLE_BFEE
+	/*For each padapter*/
+	rtw_core_bf_watchdog(padapter);
+#endif
+#endif
+
+	pmlmepriv->LinkDetectInfo.NumRxOkInPeriod = 0;
+	pmlmepriv->LinkDetectInfo.NumTxOkInPeriod = 0;
+	pmlmepriv->LinkDetectInfo.NumRxUnicastOkInPeriod = 0;
+	pmlmepriv->LinkDetectInfo.bBusyTraffic = bBusyTraffic;
+	pmlmepriv->LinkDetectInfo.bTxBusyTraffic = bTxBusyTraffic;
+	pmlmepriv->LinkDetectInfo.bRxBusyTraffic = bRxBusyTraffic;
+	pmlmepriv->LinkDetectInfo.bHigherBusyTraffic = bHigherBusyTraffic;
+	pmlmepriv->LinkDetectInfo.bHigherBusyRxTraffic = bHigherBusyRxTraffic;
+	pmlmepriv->LinkDetectInfo.bHigherBusyTxTraffic = bHigherBusyTxTraffic;
+
+	return bEnterPS;
+
+}
+
+
+/* for 11n Logo 4.2.31/4.2.32 */
+void dynamic_update_bcn_check(_adapter *padapter)
+{
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+
+	if (!padapter->registrypriv.wifi_spec)
+		return;
+
+	if (!pregistrypriv->ht_enable || !is_supported_ht(pregistrypriv->wireless_mode))
+		return;
+
+	if (!MLME_IS_AP(padapter))
+		return;
+
+	if (pmlmeext->bstart_bss) {
+		/* In 10 * 2 = 20s, there are no legacy AP, update HT info  */
+		static u8 count = 1;
+
+		if (count % 10 == 0) {
+			count = 1;
+#ifdef CONFIG_80211N_HT
+			if (_FALSE == ATOMIC_READ(&pmlmepriv->olbc)
+				&& _FALSE == ATOMIC_READ(&pmlmepriv->olbc_ht)) {
+
+				if (rtw_ht_operation_update(padapter, padapter_link) > 0)
+				{
+					rtw_update_beacon(padapter, padapter_link,
+							_HT_CAPABILITY_IE_, NULL, _FALSE, 0);
+					rtw_update_beacon(padapter, padapter_link,
+							_HT_ADD_INFO_IE_, NULL, _TRUE, 0);
+				}
+			}
+#endif /* CONFIG_80211N_HT */
+		}
+
+#ifdef CONFIG_80211N_HT
+		/* In 2s, there are any legacy AP, update HT info, and then reset count  */
+
+		if (_FALSE != ATOMIC_READ(&pmlmepriv->olbc)
+			&& _FALSE != ATOMIC_READ(&pmlmepriv->olbc_ht)) {
+
+			if (rtw_ht_operation_update(padapter, padapter_link) > 0)
+			{
+				rtw_update_beacon(padapter, padapter_link,
+						_HT_CAPABILITY_IE_, NULL, _FALSE, 0);
+				rtw_update_beacon(padapter, padapter_link,
+						_HT_ADD_INFO_IE_, NULL, _TRUE, 0);
+
+			}
+			ATOMIC_SET(&pmlmepriv->olbc, _FALSE);
+			ATOMIC_SET(&pmlmepriv->olbc_ht, _FALSE);
+			count = 0;
+		}
+#endif /* CONFIG_80211N_HT */
+		count ++;
+	}
+}
+
+struct turbo_edca_setting{
+	u32 edca_ul; /* uplink, tx */
+	u32 edca_dl; /* downlink, rx */
+};
+
+#define TURBO_EDCA_ENT(UL, DL) {UL, DL}
+
+#define TURBO_EDCA_MODE_NUM 8
+static struct turbo_edca_setting ctrl_turbo_edca[TURBO_EDCA_MODE_NUM] = {
+	/* { UL, DL } */
+	TURBO_EDCA_ENT(0x5e431c, 0x431c), /* mode 0 */
+
+	TURBO_EDCA_ENT(0x431c, 0x431c), /* mode 1 */
+
+	TURBO_EDCA_ENT(0x5e431c, 0x5e431c), /* mode 2 */
+
+	TURBO_EDCA_ENT(0x5ea42b, 0x5ea42b), /* mode 3 */
+
+	TURBO_EDCA_ENT(0x5ea42b, 0x431c), /* mode 4 */
+
+	TURBO_EDCA_ENT(0x6ea42b, 0x6ea42b), /* mode 5 */
+
+	TURBO_EDCA_ENT(0xa42b, 0xa42b), /* mode 6 */
+
+	TURBO_EDCA_ENT(0x5e431c, 0xa42b), /* mode 7 */
+};
+
+void rtw_turbo_edca(_adapter *padapter)
+{
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	u32	edca_be_ul = padapter->last_edca;
+	u32	edca_be_dl = padapter->last_edca;
+	u32	ac_parm = padapter->last_edca;
+	u8	ac;
+	u8	is_linked = _FALSE;
+	/* ToDo CONFIG_RTW_MLD: currently primary link only */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	u8	set_edca = _FALSE;
+
+	if (padapter->dis_turboedca == DIS_TURBO)
+		return;
+
+	#ifdef CONFIG_THERMAL_PROTECT
+	if (phl_com->drv_mode == RTW_DRV_MODE_HIGH_THERMAL) {
+		padapter->edca_reset = _TRUE;
+		return;
+	}
+	#endif/*CONFIG_THERMAL_PROTECT*/
+
+	if (padapter->edca_reset == _TRUE ) {
+		padapter->edca_reset = _FALSE;
+		set_edca = _TRUE;
+	}
+
+	if (rtw_mi_check_status(padapter, MI_ASSOC))
+		is_linked = _TRUE;
+
+	if (is_linked != _TRUE)
+		return;
+
+	if (pregpriv->wifi_spec == 1)
+		return;
+
+	/* keep this condition at last check */
+	if (padapter->dis_turboedca == DIS_TURBO_USE_MANUAL) {
+		if (padapter->edca_param_mode < TURBO_EDCA_MODE_NUM) {
+
+				struct turbo_edca_setting param;
+
+				param = ctrl_turbo_edca[padapter->edca_param_mode];
+				edca_be_ul = param.edca_ul;
+				edca_be_dl = param.edca_dl;
+			} else {
+				edca_be_ul = padapter->edca_param_mode;
+				edca_be_dl = padapter->edca_param_mode;
+		}
+	}
+
+	if (phl_com->phl_stats.tx_traffic.lvl == RTW_TFC_HIGH)
+		ac_parm = edca_be_ul;
+	else if (phl_com->phl_stats.tx_traffic.lvl != RTW_TFC_HIGH)
+		ac_parm = edca_be_dl;
+	else
+		return;
+
+	if ((padapter->last_edca != ac_parm) || (set_edca == _TRUE)) {
+		ac = 0;
+		/*RTW_INFO("%s, edca(0x%08x), lvl(%d), sts(%d)\n", __func__, ac_parm,
+				phl_com->phl_stats.tx_traffic.lvl, phl_com->phl_stats.tx_traffic.sts);*/
+		rtw_hw_set_edca(padapter, padapter_link, ac, ac_parm);
+		padapter->last_edca = ac_parm;
+	}
+}
+
+u32 rtw_get_turbo_edca(_adapter *padapter, u8 aifs, u8 ecwmin, u8 ecwmax, u8 txop)
+{
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	u32 ret = 0;
+	u8 decide_txop = txop;
+	u8 default_txop = 0x5e;
+	u32 ac_parm = padapter->last_edca;
+	u8 ac = 0;/*BE*/
+
+	if (padapter->dis_turboedca == DIS_TURBO)
+		return ret;
+
+	if (pregpriv->wifi_spec == 1)
+		return ret;
+
+	if (default_txop > txop)
+		decide_txop = default_txop;
+	else
+		decide_txop = txop;
+
+	ac_parm = aifs | (ecwmin << 8) | (ecwmax << 12) | (decide_txop << 16);
+
+	return  ac_parm;
+
+}
+
+void rtw_iface_dynamic_chk_wk_hdl(_adapter *padapter)
+{
+	rtw_turbo_edca(padapter);
+
+	/* for debug purpose */
+	_linked_info_dump(padapter);
+}
+
+void rtw_dynamic_chk_wk_sw_hdl(_adapter *padapter)
+{
+#ifdef CONFIG_RTW_MULTI_AP
+	rtw_ch_util_rpt(padapter);
+#endif
+
+	rtw_edcca_hal_update(adapter_to_dvobj(padapter));
+}
+
+void rtw_dynamic_chk_wk_hw_hdl(_adapter *padapter)
+{
+	rtw_mi_dynamic_chk_wk_hdl(padapter);
+
+	/* if(check_fwstate(pmlmepriv, WIFI_UNDER_LINKING|WIFI_UNDER_SURVEY)==_FALSE) */
+	{
+#ifdef DBG_RX_COUNTER_DUMP
+		rtw_dump_rx_counters(padapter);
+#endif
+	}
+
+#ifdef CONFIG_IPS_CHECK_IN_WD
+	/* always call rtw_ps_processor() at last one. */
+	rtw_ps_processor(padapter);
+#endif
+
+#ifdef RTW_DETECT_HANG
+	rtw_is_hang_check(padapter);
+#endif
+}
+
+/* add for CONFIG_IEEE80211W, none 11w can use it */
+void reset_securitypriv_hdl(_adapter *padapter)
+{
+	rtw_reset_securitypriv(padapter);
+}
+
+#ifdef CONFIG_IOCTL_CFG80211
+#if 0 /*!CONFIG_PHL_ARCH*/
+static u8 _p2p_roch_cmd(_adapter *adapter
+	, u64 cookie, struct wireless_dev *wdev
+	, struct ieee80211_channel *ch, enum nl80211_channel_type ch_type
+	, unsigned int duration
+	, u8 flags
+)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	struct p2p_roch_parm *roch_parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 cancel = duration ? 0 : 1;
+	u8 res = _SUCCESS;
+
+	roch_parm = (struct p2p_roch_parm *)rtw_zmalloc(sizeof(struct p2p_roch_parm));
+	if (roch_parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	roch_parm->cookie = cookie;
+	roch_parm->wdev = wdev;
+	if (!cancel) {
+		_rtw_memcpy(&roch_parm->ch, ch, sizeof(struct ieee80211_channel));
+		roch_parm->ch_type = ch_type;
+		roch_parm->duration = duration;
+	}
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != p2p_protocol_wk_hdl(adapter, cancel ? P2P_CANCEL_RO_CH_WK : P2P_RO_CH_WK, (u8 *)roch_parm))
+			res = _FAIL;
+		rtw_mfree((u8 *)roch_parm, sizeof(*roch_parm));
+	} else {
+		/* need enqueue, prepare cmd_obj and enqueue */
+		parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+		if (parm == NULL) {
+			rtw_mfree((u8 *)roch_parm, sizeof(*roch_parm));
+			res = _FAIL;
+			goto exit;
+		}
+
+		parm->ec_id = P2P_PROTO_WK_CID;
+		parm->type = cancel ? P2P_CANCEL_RO_CH_WK : P2P_RO_CH_WK;
+		parm->size = sizeof(*roch_parm);
+		parm->pbuf = (u8 *)roch_parm;
+
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree((u8 *)roch_parm, sizeof(*roch_parm));
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 10 * 1000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+			if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+				res = _FAIL;
+		}
+	}
+
+exit:
+	return res;
+}
+#endif /*!CONFIG_PHL_ARCH*/
+
+inline u8 rtw_mgnt_tx_cmd(_adapter *adapter, u8 tx_band, u8 tx_ch, u8 no_cck, const u8 *buf, size_t len, int wait_ack, u8 flags)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	struct mgnt_tx_parm *mgnt_parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8	res = _SUCCESS;
+
+	mgnt_parm = (struct mgnt_tx_parm *)rtw_zmalloc(sizeof(struct mgnt_tx_parm));
+	if (mgnt_parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	mgnt_parm->tx_band = tx_band;
+	mgnt_parm->tx_ch = tx_ch;
+	mgnt_parm->no_cck = no_cck;
+	mgnt_parm->buf = buf;
+	mgnt_parm->len = len;
+	mgnt_parm->wait_ack = wait_ack;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != rtw_mgnt_tx_handler(adapter, (u8 *)mgnt_parm))
+			res = _FAIL;
+		rtw_mfree((u8 *)mgnt_parm, sizeof(*mgnt_parm));
+	} else {
+		/* need enqueue, prepare cmd_obj and enqueue */
+		parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+		if (parm == NULL) {
+			rtw_mfree((u8 *)mgnt_parm, sizeof(*mgnt_parm));
+			res = _FAIL;
+			goto exit;
+		}
+
+		parm->ec_id = MGNT_TX_WK_CID;
+		parm->type = 0;
+		parm->size = sizeof(*mgnt_parm);
+		parm->pbuf = (u8 *)mgnt_parm;
+
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree((u8 *)mgnt_parm, sizeof(*mgnt_parm));
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 10 * 1000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+			if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+				res = _FAIL;
+		}
+	}
+
+exit:
+	return res;
+}
+#endif
+
+#ifdef CONFIG_AP_MODE
+
+static void rtw_chk_hi_queue_hdl(_adapter *padapter)
+{
+	struct sta_info *psta_bmc;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	systime start = rtw_get_current_time();
+	u8 empty = _FALSE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	psta_bmc = rtw_get_bcmc_stainfo(padapter, padapter_link);
+	if (!psta_bmc)
+		return;
+
+	rtw_hal_get_hwreg(padapter, HW_VAR_CHK_HI_QUEUE_EMPTY, &empty);
+
+	while (_FALSE == empty && rtw_get_passing_time_ms(start) < rtw_get_wait_hiq_empty_ms()) {
+		rtw_msleep_os(100);
+		rtw_hal_get_hwreg(padapter, HW_VAR_CHK_HI_QUEUE_EMPTY, &empty);
+	}
+
+	if (psta_bmc->sleepq_len == 0) {
+		if (empty == _SUCCESS) {
+			bool update_tim = _FALSE;
+
+			if (rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, 0))
+				update_tim = _TRUE;
+
+			rtw_tim_map_clear(padapter, pstapriv->tim_bitmap, 0);
+			rtw_tim_map_clear(padapter, pstapriv->sta_dz_bitmap, 0);
+
+			if (update_tim == _TRUE)
+				_update_beacon(padapter, padapter_link,
+						_TIM_IE_, NULL, _TRUE, 0,"bmc sleepq and HIQ empty");
+		} else /* re check again */
+			rtw_chk_hi_queue_cmd(padapter);
+
+	}
+
+}
+
+u8 rtw_chk_hi_queue_cmd(_adapter *padapter)
+{
+	struct cmd_obj	*cmd;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8	res = _SUCCESS;
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->ec_id = CHECK_HIQ_WK_CID;
+	pdrvextra_cmd_parm->type = 0;
+	pdrvextra_cmd_parm->size = 0;
+	pdrvextra_cmd_parm->pbuf = NULL;
+
+	init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;
+
+}
+#endif /* CONFIG_AP_MODE */
+
+#if 0 /*def RTW_PHL_DBG_CMD*/
+
+u8 sample_txwd[] = 
+{
+/*dword 00*/	0x80, 0x00, 0x40, 0x00,
+/*dword 01*/	0x00, 0x00, 0x00, 0x00,  
+/*dword 02*/	0xF2, 0x05, 0x00, 0x00, 
+/*dword 03*/	0x3E, 0x11, 0x00, 0x00,  
+/*dword 04*/	0x00, 0x00, 0x00, 0x00, 
+/*dword 05*/	0x00, 0x00, 0x00, 0x00,  
+/*dword 06*/	0x00, 0x07, 0x9B, 0x63, 
+/*dword 07*/	0x3F, 0x00, 0x00, 0x00,  
+/*dword 08*/	0x00, 0x00, 0x00, 0x00, 
+/*dword 09*/	0x00, 0x00, 0x00, 0x00,  
+/*dword 10*/	0x0C, 0x00, 0x00, 0x00, 
+/*dword 11*/	0x00, 0x00, 0x00, 0x00,  
+};
+
+#define WD_SPEC(_name, _idx_dw, _bit_start, _bit_end) {              \
+	.name        	= (_name),                   \
+	.idx_dw         = (_idx_dw),                     \
+	.bit_start      = (_bit_start),                   \
+	.bit_end 		= (_bit_end)  \
+}
+
+struct parse_wd {
+	char name[32];
+	u8 idx_dw;
+	u8 bit_start;
+	u8 bit_end;
+};
+
+#define MAX_PHL_CMD_LEN 200
+#define MAX_PHL_CMD_NUM 10
+#define MAX_PHL_CMD_STR_LEN 200
+
+char *get_next_para_str(char *para)
+{
+	return (para+MAX_PHL_CMD_STR_LEN);
+}
+
+static struct parse_wd parse_txwd_8852ae_full[] = {
+	WD_SPEC("EN_HWSEQ_MODE"			, 0, 0, 1),
+	WD_SPEC("HW_SSN_SEL"			, 0, 2, 3),
+	WD_SPEC("SMH_EN"				, 0, 4, 4),
+	WD_SPEC("HWAMSDU"				, 0, 5, 5),
+	WD_SPEC("HW_AES_IV"				, 0, 6, 6),
+	WD_SPEC("WD page"				, 0, 7, 7),
+	WD_SPEC("CHK_EN"				, 0, 8, 8),
+	WD_SPEC("WP_INT"				, 0, 9, 9),
+	WD_SPEC("STF mode"				, 0, 10, 10),
+	WD_SPEC("HEADERwLLC_LEN"		, 0, 11, 15),
+	WD_SPEC("CHANNEL_DMA"			, 0, 16, 19),
+	WD_SPEC("FW_download"			, 0, 20, 20),
+	WD_SPEC("PKT_OFFSET"			, 0, 21, 21),
+	WD_SPEC("WDINFO_EN"				, 0, 22, 22),
+	WD_SPEC("MOREDATA"				, 0, 23, 23),
+	WD_SPEC("WP_OFFSET" 			, 0, 24, 31),
+
+	WD_SPEC("SHCUT_CAMID"			, 1, 0, 7),
+	WD_SPEC("DMA_TXAGG_NUM"			, 1, 8, 15),
+	WD_SPEC("PLD(Packet ID)"		, 1, 16, 31),
+
+	WD_SPEC("TXPKTSIZE"			, 2, 0, 13),
+	WD_SPEC("RU_TC"				, 2, 14, 16),
+	WD_SPEC("QSEL"				, 2, 17, 22),
+	WD_SPEC("TID_indicate"		, 2, 23, 23),
+	WD_SPEC("MACID"				, 2, 24, 30),
+	WD_SPEC("RSVD"				, 2, 31, 31),
+
+	WD_SPEC("Wifi_SEQ"			, 3, 0, 11),
+	WD_SPEC("AGG_EN"			, 3, 12, 12),
+	WD_SPEC("BK"				, 3, 13, 13),
+	WD_SPEC("RTS_TC"			, 3, 14, 19),
+	WD_SPEC("DATA_TC"			, 3, 20, 25),
+	WD_SPEC("MU_2nd_TC"			, 3, 26, 28),
+	WD_SPEC("MU_TC"				, 3, 29, 31),
+
+	WD_SPEC("TIMESTAMP"			, 4, 0, 15),
+	WD_SPEC("AES_IV_L"			, 4, 16, 31),
+
+	WD_SPEC("AES_IV_H"			, 5, 0, 31),	
+
+	WD_SPEC("MBSSID"			, 6, 0, 3),
+	WD_SPEC("Multiport_ID"		, 6, 4, 6),
+	WD_SPEC("RSVD"				, 6, 7, 7),
+	WD_SPEC("DATA_BW_ER"		, 6, 8, 8),
+	WD_SPEC("DISRTSFB"			, 6, 9, 9),
+	WD_SPEC("DISDATAFB"			, 6, 10, 10),
+	WD_SPEC("DATA_LDPC"			, 6, 11, 11),
+	WD_SPEC("DATA_STBC"			, 6, 12, 13),
+	WD_SPEC("DATA_DCM"			, 6, 14, 14),
+	WD_SPEC("DATA_ER"			, 6, 15, 15),
+	WD_SPEC("DataRate"			, 6, 16, 24),
+	WD_SPEC("GI_LTF"			, 6, 25, 27),
+	WD_SPEC("DATA_BW"			, 6, 28, 29),
+	WD_SPEC("USERATE_SEL" 		, 6, 30, 30),
+	WD_SPEC("ACK_CH_INFO"		, 6, 31, 31),
+
+	WD_SPEC("MAX_AGG_NUM"				, 7, 0, 7),
+	WD_SPEC("BCN_SRCH_SEQ"				, 7, 8, 9),
+	WD_SPEC("NAVUSEHDR"					, 7, 10, 10),
+	WD_SPEC("BMC"						, 7, 11, 11),
+	WD_SPEC("A_CTRL_BQR"				, 7, 12, 12),
+	WD_SPEC("A_CTRL_UPH"				, 7, 13, 13),
+	WD_SPEC("A_CTRL_BSR"				, 7, 14, 14),
+	WD_SPEC("A_CTRL_CAS"				, 7, 15, 15),
+	WD_SPEC("DATA_RTY_LOWEST_RATE"		, 7, 16, 24),
+	WD_SPEC("DATA_TXCNT_LMT"			, 7, 25, 30),
+	WD_SPEC("DATA_TXCNT_LMT_SEL"		, 7, 31, 31),
+
+	WD_SPEC("SEC_CAM_IDX"				, 8, 0, 7),
+	WD_SPEC("SEC_HW_ENC"				, 8, 8, 8),
+	WD_SPEC("SECTYPE"					, 8, 9, 12),
+	WD_SPEC("lifetime_sel"				, 8, 13, 15),
+	WD_SPEC("RSVD"						, 8, 16, 16),
+	WD_SPEC("FORCE_TXOP"				, 8, 17, 17),
+	WD_SPEC("AMPDU_DENSITY"				, 8, 18, 20),
+	WD_SPEC("LSIG_TXOP_EN"				, 8, 21, 21),
+	WD_SPEC("TXPWR_OFSET_TYPE"			, 8, 22, 24),
+	WD_SPEC("RSVD"						, 8, 25, 25),	
+	WD_SPEC("obw_cts2self_dup_type"		, 8, 26, 29),
+	WD_SPEC("RSVD"						, 8, 30, 31),
+
+	WD_SPEC("Signaling_TA_PKT_EN"		, 9, 0, 0),
+	WD_SPEC("NDPA"						, 9, 1, 2),
+	WD_SPEC("SND_PKT_SEL"				, 9, 3, 5),
+	WD_SPEC("SIFS_Tx"					, 9, 6, 6),
+	WD_SPEC("HT_DATA_SND" 				, 9, 7, 7),
+	WD_SPEC("RSVD"						, 9, 8, 8),
+	WD_SPEC("RTT_EN" 					, 9, 9, 9),
+	WD_SPEC("SPE_RPT" 					, 9, 10, 10),
+	WD_SPEC("BT_NULL" 					, 9, 11, 11),
+	WD_SPEC("TRI_FRAME"					, 9, 12, 12),
+	WD_SPEC("NULL_1" 					, 9, 13, 13),
+	WD_SPEC("NULL_0" 					, 9, 14, 14),
+	WD_SPEC("RAW" 						, 9, 15, 15),
+	WD_SPEC("Group_bit"					, 9, 16, 23),
+	WD_SPEC("RSVD" 						, 9, 24, 25),
+	WD_SPEC("BCNPKT_TSF_CTRL" 			, 9, 26, 26),
+	WD_SPEC("Signaling_TA_PKT_SC" 		, 9, 27, 30),
+	WD_SPEC("FORCE_BSS_CLR" 			, 9, 31, 31),
+
+	WD_SPEC("SW_DEFINE"					, 10, 0, 3),
+	WD_SPEC("RSVD"						, 10, 4, 26),
+	WD_SPEC("RTS_EN"					, 10, 27, 27),
+	WD_SPEC("CTS2SELF"					, 10, 28, 28),
+	WD_SPEC("CCA_RTS" 					, 10, 29, 30),
+	WD_SPEC("HW_RTS_EN" 				, 10, 31, 31),
+
+	WD_SPEC("RSVD"						, 11, 0, 3),
+	WD_SPEC("NDPA_duration"				, 11, 4, 26),
+};
+
+static struct parse_wd parse_rxwd_8852ae[] = {
+	WD_SPEC("PKT_LEN"				, 0, 0, 13),
+	WD_SPEC("SHIFT"					, 0, 14, 15),
+	WD_SPEC("WL_HD_IV_LEN"			, 0, 16, 21),
+	WD_SPEC("BB_SEL"				, 0, 22, 22),
+	WD_SPEC("MAC_INFO_VLD"			, 0, 23, 23),
+	WD_SPEC("RPKT_TYPE"				, 0, 24, 27),
+	WD_SPEC("DRV_INFO_SIZE"			, 0, 28, 30),
+	WD_SPEC("LONG_RXD"				, 0, 31, 31),
+	
+	WD_SPEC("PPDU_TYPE"			, 1, 0, 3),
+	WD_SPEC("PPDU_CNT"			, 1, 4, 6),
+	WD_SPEC("SR_EN"				, 1, 7, 7),
+	WD_SPEC("USER_ID"			, 1, 8, 15),
+	WD_SPEC("RX_DATARATE" 		, 1, 16, 24),
+	WD_SPEC("RX_GI_LTF"			, 1, 25, 27),
+	WD_SPEC("NON_SRG_PPDU"		, 1, 28, 28),
+	WD_SPEC("INTER_PPDU" 		, 1, 29, 29),
+	WD_SPEC("BW"				, 1, 30, 31),
+
+	WD_SPEC("FREERUN_CNT"		, 2, 0, 31),
+
+	WD_SPEC("A1_MATCH"			, 3, 1, 1),
+	WD_SPEC("SW_DEC"			, 3, 2, 2),
+	WD_SPEC("HW_DEC"			, 3, 3, 3),
+	WD_SPEC("AMPDU"				, 3, 4, 4),
+	WD_SPEC("AMPDU_END_PKT"		, 3, 5, 5),
+	WD_SPEC("AMSDU"				, 3, 6, 6),
+	WD_SPEC("AMSDU_CUT"			, 3, 7, 7),
+	WD_SPEC("LAST_MSDU"			, 3, 8, 8),
+	WD_SPEC("BYPASS"			, 3, 9, 9),
+	WD_SPEC("CRC32" 			, 3, 10, 10),
+	WD_SPEC("MAGIC_WAKE" 		, 3, 11, 11),
+	WD_SPEC("UNICAST_WAKE"		, 3, 12, 12),
+	WD_SPEC("PATTERN_WAKE"		, 3, 13, 13),
+	WD_SPEC("GET_CH_INFO" 		, 3, 14, 14),
+	WD_SPEC("RX_STATISTICS" 	, 3, 15, 15),
+	WD_SPEC("PATTERN_IDX"		, 3, 16, 20),
+	WD_SPEC("TARGET_IDC"		, 3, 21, 23),
+	WD_SPEC("CHKSUM_OFFLOAD_EN"	, 3, 24, 24),
+	WD_SPEC("WITH_LLC"			, 3, 25, 25),
+	WD_SPEC("RSVD" 				, 3, 26, 31),
+
+	WD_SPEC("TYPE"			, 4, 0, 1),
+	WD_SPEC("MC"			, 4, 2, 2),
+	WD_SPEC("BC" 			, 4, 3, 3),
+	WD_SPEC("MD"			, 4, 4, 4),
+	WD_SPEC("MF" 			, 4, 5, 5),
+	WD_SPEC("PWR"			, 4, 6, 6),
+	WD_SPEC("QOS" 			, 4, 7, 7),
+	WD_SPEC("TID"			, 4, 8, 11),
+	WD_SPEC("EOSP" 			, 4, 12, 12),
+	WD_SPEC("HTC"			, 4, 13, 13),
+	WD_SPEC("QNULL"			, 4, 14, 14),
+	WD_SPEC("RSVD" 			, 4, 15, 15),
+	WD_SPEC("SEQ"			, 4, 16, 27),
+	WD_SPEC("FRAG" 			, 4, 28, 31),
+	
+	WD_SPEC("SEC_CAM_IDX"		, 5, 0, 7),	
+	WD_SPEC("ADDR_CAM"			, 5, 8, 15),	
+	WD_SPEC("MAC_ID"			, 5, 16, 23),	
+	WD_SPEC("RX_PL_ID"			, 5, 24, 27),	
+	WD_SPEC("ADDR_CAM_VLD"		, 5, 28, 28),	
+	WD_SPEC("ADDR_FWD_EN"		, 5, 29, 29),	
+	WD_SPEC("RX_PL_MATCH"		, 5, 30, 30),	
+	WD_SPEC("RSVD"				, 5, 31, 31),	
+};
+
+
+enum WD_TYPE {
+	TXWD_INFO = 0,
+	TXWD_INFO_BODY,
+	RXWD, 
+
+};
+
+u32 get_txdesc_element_val(u32 val_dw, u8 bit_start, u8 bit_end)
+{
+	u32 mask = 0;
+	u32 i = 0;
+
+	if(bit_start>31 
+		|| bit_end>31
+		|| (bit_start>bit_end)){
+		printk("[%s] error %d %d\n", __FUNCTION__, bit_start, bit_end);
+		return 0;
+	}
+
+	for(i = bit_start; i<=bit_end; i++){
+		mask |= (1<<i);
+	}
+
+	return ((val_dw & mask)>>bit_start);
+}
+
+
+void parse_wd_8852ae(_adapter *adapter, u32 type, u32 idx_wd, u8 *wd)
+{
+	u32 i, val = 0;
+	u32 cur_dw = 0xFF;
+	u32 idx, val_dw = 0;
+	u32 array_size = 0;
+	struct parse_wd *parser = NULL;
+
+	if(wd==NULL)
+		return;
+
+	if(type == TXWD_INFO_BODY){
+		parser = parse_txwd_8852ae_full;
+		array_size = ARRAY_SIZE(parse_txwd_8852ae_full);
+	}
+	else if(type == RXWD){
+		parser = parse_rxwd_8852ae;
+		array_size = ARRAY_SIZE(parse_rxwd_8852ae);
+	}
+
+	for(i = 0; i<array_size; i++){
+			if(cur_dw != parser[i].idx_dw){
+				cur_dw = parser[i].idx_dw;
+				idx = (parser[i].idx_dw*4);
+				val_dw = wd[idx] + (wd[idx+1]<<8) + (wd[idx+2]<<16) + (wd[idx+3]<<24);
+				printk(">>>> WD[%03d].dw%02d = 0x%08x \n", idx_wd, cur_dw, val_dw);
+			}
+
+			val = get_txdesc_element_val(val_dw, 
+				parser[i].bit_start, parser[i].bit_end);
+				
+			printk("%s[%d:%d] = (0x%x)\n",  
+				parser[i].name, 
+				parser[i].bit_end, parser[i].bit_start, val);
+		}
+		printk("\n");
+	
+}
+
+
+void compare_wd_8852ae(_adapter *adapter, u32 type, u8 *wd1, u8 *wd2)
+{
+	u32 i, val1, val2 = 0;
+	u32 cur_dw = 0xFF;
+	u32 idx, val_dw1, val_dw2 = 0;
+	u32 array_size = 0;
+	struct parse_wd *parser = NULL;
+
+	if((wd1==NULL) ||(wd2==NULL))
+		return;
+
+	if(type == TXWD_INFO_BODY){
+		parser = parse_txwd_8852ae_full;
+		array_size = ARRAY_SIZE(parse_txwd_8852ae_full);
+	}
+
+	for(i = 0; i<array_size; i++){
+			if(cur_dw != parser[i].idx_dw){
+				cur_dw = parser[i].idx_dw;
+				idx = (parser[i].idx_dw*4);
+				val_dw1 = wd1[idx] + (wd1[idx+1]<<8) + (wd1[idx+2]<<16) + (wd1[idx+3]<<24);
+				val_dw2 = wd2[idx] + (wd2[idx+1]<<8) + (wd2[idx+2]<<16) + (wd2[idx+3]<<24);
+			}
+
+			val1 = get_txdesc_element_val(val_dw1, 
+				parser[i].bit_start, parser[i].bit_end);
+			val2 = get_txdesc_element_val(val_dw2, 
+				parser[i].bit_start, parser[i].bit_end);
+
+			if(val1 != val2){
+				printk("Diff dw%02d: %s[%d:%d] = (0x%x) vs (0x%x)\n", cur_dw,
+					parser[i].name, 
+					parser[i].bit_end, parser[i].bit_start, val1, val2);
+			}
+		}
+		printk("\n");
+	
+}
+
+void core_dump_map_tx(_adapter *adapter)
+{
+	struct core_logs *log = &adapter->core_logs;
+	u32 idx = 0;
+
+ 	printk("drvTx MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct core_record *record = &log->drvTx[idx];
+		if(idx >= log->txCnt_all)
+			break;
+		printk("[drvTx %03d]\n", idx);
+		printk("type=%d totalSz=%d virtAddr=%p\n", record->type, record->totalSz, record->virtAddr[0]);
+	}
+
+	printk("========= \n\n");
+	printk("phlTx MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct core_record *record = &log->phlTx[idx];
+		u32 idx1 = 0;
+		if(idx >= log->txCnt_phl)
+			break;
+		printk("[phlTx %03d]\n", idx);
+		printk("type=%d totalSz=%d fragNum=%d\n", record->type, record->totalSz, record->fragNum);
+		for(idx1=0; idx1<record->fragNum; idx1++){
+			printk("frag#%d: len=%d virtaddr=%p \n", idx1, 
+				record->fragLen[idx1], record->virtAddr[idx1]);
+			printk("frag#%d: phyAddrH=%d phyAddrL=%p \n", idx1, 
+				record->phyAddrH[idx1], record->phyAddrL[idx1]);			
+		}
+	}
+
+	printk("========= \n\n");
+	printk("TxRcycle MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct core_record *record = &log->txRcycle[idx];
+		u32 idx1 = 0;
+		if(idx >= log->txCnt_recycle)
+			break;
+		printk("[TxRcycle %03d]\n", idx);
+		printk("type=%d totalSz=%d fragNum=%d\n", record->type, record->totalSz, record->fragNum);
+		for(idx1=0; idx1<record->fragNum; idx1++){
+			printk("frag#%d: len=%d virtaddr=%p \n", idx1, 
+				record->fragLen[idx1], record->virtAddr[idx1]);
+			printk("frag#%d: phyAddrH=%d phyAddrL=%p \n", idx1, 
+				record->phyAddrH[idx1], record->phyAddrL[idx1]);			
+		}
+	}
+}
+
+void core_dump_map_rx(_adapter *adapter)
+{
+	struct core_logs *log = &adapter->core_logs;
+	u32 idx = 0;
+
+ 	printk("drvRx MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct core_record *record = &log->drvRx[idx];
+		if(idx >= log->rxCnt_all)
+			break;
+		printk("[drvRx %03d]\n", idx);
+		printk("type=%d totalSz=%d virtAddr=%p\n", record->type, record->totalSz, record->virtAddr[0]);
+		printk("wl_seq=%d wl_type=0x%x wl_subtype=0x%x\n", record->wl_seq, record->wl_type, record->wl_subtype);
+	}
+
+	printk("========= \n\n");
+	printk("phlRx MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct core_record *record = &log->phlRx[idx];
+		if(idx >= log->rxCnt_phl)
+			break;
+		printk("[phlRx %03d]\n", idx);
+		printk("type=%d totalSz=%d virtAddr=%p\n", record->type, record->totalSz, record->virtAddr[0]);
+	}
+
+	printk("========= \n\n");
+	printk("rxRcycle MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct core_record *record = &log->rxRcycle[idx];
+		if(idx >= log->rxCnt_recycle)
+			break;
+		printk("[rxRcycle %03d]\n", idx);
+		printk("type=%d totalSz=%d virtAddr=%p\n", record->type, record->totalSz, record->virtAddr[0]);
+	}
+
+}
+
+
+void core_dump_record(_adapter *adapter, u8 dump_type)
+{
+	struct core_logs *log = &adapter->core_logs;
+	u32 idx = 0;
+
+	printk("txCnt_all: %d (%d) \n", log->txCnt_all, log->txCnt_all%CORE_LOG_NUM);
+	printk("txCnt_data: %d \n", log->txCnt_data);
+	printk("txCnt_mgmt: %d \n", log->txCnt_mgmt);
+	printk("txCnt_phl: %d (%d), Sz=%d \n", log->txCnt_phl, log->txCnt_phl%CORE_LOG_NUM, log->txSize_phl);
+	printk("txCnt_recycle: %d (%d), Sz=%d \n", log->txCnt_recycle, log->txCnt_recycle%CORE_LOG_NUM, log->txSize_recycle);
+
+	printk("rxCnt_all: %d (%d) \n", log->rxCnt_all, log->rxCnt_all%CORE_LOG_NUM);
+	printk("rxCnt_data: %d (retry=%d)\n", log->rxCnt_data, log->rxCnt_data_retry);
+	printk("rxCnt_mgmt: %d \n", log->rxCnt_mgmt);
+	printk("rxCnt_phl: %d (%d), Sz=%d \n", log->rxCnt_phl, log->rxCnt_phl%CORE_LOG_NUM, log->rxSize_phl);
+	printk("rxCnt_recycle: %d (%d), Sz=%d\n", log->rxCnt_recycle, log->rxCnt_recycle%CORE_LOG_NUM, log->rxSize_recycle);
+#ifdef CONFIG_RTW_CORE_RXSC
+	printk("enable_rxsc: %d \n", adapter->enable_rxsc);
+	printk("rxCnt_data: orig=%d shortcut=%d(ratio=%d)\n",
+		log->rxCnt_data_orig, log->rxCnt_data_shortcut,
+		log->rxCnt_data_shortcut*100/(log->rxCnt_data_orig+log->rxCnt_data_shortcut));
+	printk("rxCnt_coreInd: %d \n", log->rxCnt_coreInd);
+#endif
+
+	if(dump_type == REC_DUMP_TX){
+		core_dump_map_tx(adapter);
+	}
+	else if(dump_type == REC_DUMP_RX){
+		core_dump_map_rx(adapter);
+	}
+	else if(dump_type == REC_DUMP_ALL){
+		core_dump_map_tx(adapter);
+		core_dump_map_rx(adapter);
+	}
+}
+
+void core_add_record(_adapter *adapter, u8 type, void *p)
+{
+	struct core_logs *log = &adapter->core_logs;
+
+	if(!adapter->record_enable)
+		return;
+
+	if(type == REC_TX_DATA){
+		u32 idx = log->txCnt_all%CORE_LOG_NUM;
+		struct core_record *record = &(log->drvTx[idx]);
+		struct sk_buff *skb = p;
+		
+		log->txCnt_data++;
+		record->type = type;
+		record->totalSz = skb->len;
+		record->virtAddr[0] = skb->data;
+	}
+
+	if(type == REC_TX_MGMT){
+		u32 idx = log->txCnt_all%CORE_LOG_NUM;
+		struct core_record *record = &(log->drvTx[idx]);
+		struct xmit_frame *pxframe = p;
+		
+		log->txCnt_mgmt++;
+		record->type = type;
+		record->totalSz = pxframe->attrib.pktlen;
+		record->virtAddr[0] = pxframe->buf_addr;
+	}
+
+	if(type == REC_TX_PHL || type == REC_TX_PHL_RCC){
+		u32 idx = 0;
+		struct core_record *record = NULL;
+		struct rtw_xmit_req *txreq = p;
+
+		if(type == REC_TX_PHL){
+			idx = log->txCnt_phl%CORE_LOG_NUM;
+			record = &(log->phlTx[idx]);
+			log->txCnt_phl++;
+		}
+		
+		if(type == REC_TX_PHL_RCC){
+			idx = log->txCnt_recycle%CORE_LOG_NUM;
+			record = &(log->txRcycle[idx]);
+			log->txCnt_recycle++;
+		}
+		
+		record->type = type;
+		record->totalSz = 0;
+		record->fragNum = txreq->pkt_cnt;
+
+		{
+			struct rtw_pkt_buf_list *pkt_list =(struct rtw_pkt_buf_list *)txreq->pkt_list;
+			u32 idx1 = 0;
+			for(idx1=0; idx1<txreq->pkt_cnt; idx1++){
+				if(idx1 >= MAX_FRAG){
+					printk("!! WARN[%s][%d] type=%d frag>= %d \n", 
+						__FUNCTION__, __LINE__, type, MAX_FRAG);
+					break;
+				}
+				record->totalSz += pkt_list->length;
+				record->fragLen[idx1] = pkt_list->length;
+				record->virtAddr[idx1] = pkt_list->vir_addr;
+				record->phyAddrL[idx1] = pkt_list->phy_addr_l;
+				record->phyAddrH[idx1] = pkt_list->phy_addr_h;
+				pkt_list++;
+			}
+		}
+
+		if(type == REC_TX_PHL)
+			log->txSize_phl += record->totalSz;
+		else if(type == REC_TX_PHL_RCC)
+			log->txSize_recycle += record->totalSz;
+		
+	}
+
+	if(type == REC_RX_PHL || type == REC_RX_PHL_RCC){
+		u32 idx = 0;
+		struct core_record *record = NULL;
+		struct rtw_recv_pkt *rx_req = p;
+		struct rtw_pkt_buf_list *pkt = rx_req->pkt_list;
+
+		if(type == REC_RX_PHL){
+			idx = log->rxCnt_phl%CORE_LOG_NUM;
+			record = &(log->phlRx[idx]);
+			log->rxCnt_phl++;
+		}
+
+		if(type == REC_RX_PHL_RCC){
+			idx = log->rxCnt_recycle%CORE_LOG_NUM;
+			record = &(log->rxRcycle[idx]);
+			log->rxCnt_recycle++;
+		}
+
+		record->type = type;
+		record->totalSz = pkt->length;
+		record->virtAddr[0] = pkt->vir_addr;
+
+		if(type == REC_RX_PHL)
+			log->rxSize_phl += record->totalSz;
+		else if(type == REC_RX_PHL_RCC)
+			log->rxSize_recycle += record->totalSz;
+	}
+
+	if(type == REC_RX_DATA || type == REC_RX_MGMT){
+		u32 idx = log->rxCnt_all%CORE_LOG_NUM;
+		struct core_record *record = &(log->drvRx[idx]);
+		union recv_frame *prframe = p;
+		struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+		
+		if(type == REC_RX_DATA){
+			log->rxCnt_data++;
+		}
+		
+		if(type == REC_RX_MGMT){
+			log->rxCnt_mgmt++;
+		}
+
+		record->type = type;
+		record->totalSz = prframe->u.hdr.len;
+		record->virtAddr[0] = prframe->u.hdr.rx_data;
+
+		record->wl_seq = pattrib->seq_num;
+		record->wl_type = pattrib->wl_type;
+		record->wl_subtype = pattrib->wl_subtype;
+
+	}
+
+	if(type == REC_RX_DATA_RETRY){
+		log->rxCnt_data_retry++;
+	}
+
+	log->txCnt_all = log->txCnt_mgmt + log->txCnt_data;
+	log->rxCnt_all = log->rxCnt_mgmt + log->rxCnt_data;
+}
+
+
+void phl_dump_map_tx(_adapter *adapter)
+{
+	struct phl_logs *log = &adapter->phl_logs;
+	u32 idx = 0;
+
+ 	printk("txBd MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct record_txbd *record = &log->txBd[idx];
+		if(idx >= log->txCnt_bd)
+			break;
+		printk("[txBd %03d]\n", idx);
+		{
+			u8 *tmp=record->bd_buf;
+			u32 len = record->bd_len;
+			u32 idx1 = 0;
+			if(tmp == NULL)
+				break;
+			for(idx1=0; idx1<len; idx1++){
+				if(idx1%8==0) {
+					printk("[%03d] %02x %02x %02x %02x %02x %02x %02x %02x \n", 
+						idx1, tmp[idx1], tmp[idx1+1], tmp[idx1+2], tmp[idx1+3], 
+						tmp[idx1+4], tmp[idx1+5], tmp[idx1+6], tmp[idx1+7]);
+				}
+			}
+			printk("\n");
+		}
+	}
+	
+	printk("========= \n\n");
+	printk("txWd MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct record_txwd *record = &log->txWd[idx];
+		if(idx >= log->txCnt_wd)
+			break;
+		printk("[txWd %03d]\n", idx);
+		{
+			u8 *tmp=record->wd_buf;
+			u32 len = record->wd_len;
+			u32 idx1 = 0;
+			if(tmp == NULL)
+				break;
+			for(idx1=0; idx1<len; idx1++){
+				if(idx1%8==0) {
+					printk("[%03d] %02x %02x %02x %02x %02x %02x %02x %02x \n", 
+						idx1, tmp[idx1], tmp[idx1+1], tmp[idx1+2], tmp[idx1+3], 
+						tmp[idx1+4], tmp[idx1+5], tmp[idx1+6], tmp[idx1+7]);
+				}
+			}
+			printk("\n");
+		}
+		parse_wd_8852ae(adapter, TXWD_INFO_BODY, idx, record->wd_buf);
+		//compare_wd_8852ae(adapter, TXWD_INFO_BODY, record->wd_buf, sample_txwd);
+	}
+	
+	printk("========= \n\n");
+ 	printk("wpRecycle MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct record_wp_rcc *record = &log->wpRecycle[idx];
+		if(idx >= log->txCnt_recycle)
+			break;
+		printk("[txRecycle %03d]\n", idx);
+		printk("wp_seq=%d \n", record->wp_seq);
+	}
+}
+
+void phl_dump_map_rx(_adapter *adapter)
+{
+	struct phl_logs *log = &adapter->phl_logs;
+	u32 idx = 0;
+
+	printk("rxPciMap MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct record_pci *record = &log->rxPciMap[idx];
+		if(idx >= log->rxCnt_map)
+			break;
+		printk("[rxPciMap %03d]\n", idx);
+		printk("phyAddrL=%p len=%d\n", record->phyAddrL, record->map_len);
+	}
+
+
+	printk("========= \n\n");
+	printk("rxPciUnmap MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct record_pci *record = &log->rxPciUnmap[idx];
+		if(idx >= log->rxCnt_unmap)
+			break;
+		printk("[rxPciUnmap %03d]\n", idx);
+		printk("phyAddrL=%p len=%d\n", record->phyAddrL, record->map_len);
+	}
+
+	printk("========= \n\n");
+	printk("rxWd MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		struct record_rxwd *record = &log->rxWd[idx];
+		if(idx >= log->rxCnt_wd)
+			break;
+		printk("[rxWd %03d]\n", idx);
+		{
+			u8 *tmp = record->wd_buf;
+			u32 len = record->wd_len;
+			u32 idx1 = 0;
+			if(tmp == NULL)
+				break;
+			for(idx1=0; idx1<len; idx1++){
+				if(idx1%8==0) {
+					printk("[%03d] %02x %02x %02x %02x %02x %02x %02x %02x \n", 
+						idx1, tmp[idx1], tmp[idx1+1], tmp[idx1+2], tmp[idx1+3], 
+						tmp[idx1+4], tmp[idx1+5], tmp[idx1+6], tmp[idx1+7]);
+				}
+			}
+			printk("\n");
+		}
+		parse_wd_8852ae(adapter, RXWD, idx, record->wd_buf);
+	}
+
+	printk("========= \n\n");
+	printk("rxAmpdu MAP");
+	for(idx=0; idx<CORE_LOG_NUM; idx++){
+		if(idx >= log->rxCnt_ampdu)
+			break;
+		printk("[rxAmpdu %03d] = %d\n", idx, log->rxAmpdu[idx]);
+	}
+
+}
+
+void phl_dump_record(_adapter *adapter, u8 dump_type)
+{
+	struct phl_logs *log = &adapter->phl_logs;
+	u32 idx = 0;
+
+	printk("txBd: %d (%d) \n", log->txCnt_bd, log->txCnt_bd%CORE_LOG_NUM);
+	printk("txWd: %d (%d) \n", log->txCnt_wd, log->txCnt_wd%CORE_LOG_NUM);
+	printk("wpCnt_recycle: %d (%d) \n", log->txCnt_recycle, log->txCnt_recycle%CORE_LOG_NUM);
+
+	printk("rxMap: %d (%d), Sz=%d \n", log->rxCnt_map, log->rxCnt_map%CORE_LOG_NUM, log->rxSize_map);
+	printk("rxUnmap: %d (%d), Sz=%d \n", log->rxCnt_unmap, log->txCnt_wd%CORE_LOG_NUM, log->rxSize_map);
+	printk("rxWd: %d (%d) \n", log->rxCnt_wd, log->rxCnt_wd%CORE_LOG_NUM);
+	printk("rxCnt_ampdu: %d (%d) \n", log->rxCnt_ampdu, log->rxCnt_ampdu%CORE_LOG_NUM);
+
+	if(dump_type == REC_DUMP_TX)
+		phl_dump_map_tx(adapter);
+	else if(dump_type == REC_DUMP_RX)
+		phl_dump_map_rx(adapter);
+	else if(dump_type == REC_DUMP_ALL){
+		phl_dump_map_tx(adapter);
+		phl_dump_map_rx(adapter);
+	}
+}
+
+u32 tmp_rx_last_ppdu = 0;
+
+void phl_add_record(void *d, u8 type, void *p, u32 num)
+{
+	struct dvobj_priv *pobj = (struct dvobj_priv *)d;
+	_adapter *adapter = dvobj_get_primary_adapter(pobj);
+	struct phl_logs *log = &adapter->phl_logs;
+
+	if(!adapter->record_enable)
+		return;
+
+	if(type == REC_TXWD){
+		u32 idx = log->txCnt_wd%CORE_LOG_NUM;
+		struct record_txwd *record = &(log->txWd[idx]);
+		
+		log->txCnt_wd++;
+		record->wd_len = num;
+		memset((u8 *)record->wd_buf, 0, MAX_TXWD_SIZE);
+		memcpy((u8 *)record->wd_buf, p, num);
+	}
+
+	if(type == REC_TXBD){
+		u32 idx = log->txCnt_bd%CORE_LOG_NUM;
+		struct record_txbd *record = &(log->txBd[idx]);
+	
+		log->txCnt_bd++;
+		record->bd_len = num;
+		memset((u8 *)record->bd_buf, 0, MAX_TXBD_SIZE);
+		memcpy((u8 *)record->bd_buf, p, num);
+	}
+
+	if(type == REC_WP_RCC){
+		u32 idx = log->txCnt_recycle%CORE_LOG_NUM;
+		struct record_wp_rcc *record = &(log->wpRecycle[idx]);
+
+		log->txCnt_recycle++;
+		record->wp_seq = num;
+	}
+
+	if(type == REC_RX_MAP || type == REC_RX_UNMAP){
+		struct record_pci *record = NULL;
+		if(type == REC_RX_MAP) {
+			u32 idx = log->rxCnt_map%CORE_LOG_NUM;
+			record = &(log->rxPciMap[idx]);
+			log->rxCnt_map++;
+			log->rxSize_map+=num;
+		}
+		else if(type == REC_RX_UNMAP) {
+			u32 idx = log->rxCnt_unmap%CORE_LOG_NUM;
+			record = &(log->rxPciUnmap[idx]);
+			log->rxCnt_unmap++;
+			log->rxSize_map+=num;
+		}
+		record->phyAddrL = p;
+		record->map_len = num;
+	}
+
+	if(type == REC_RXWD){
+		u32 idx = log->rxCnt_wd%CORE_LOG_NUM;
+		struct record_rxwd *record = &(log->rxWd[idx]);
+	
+		log->rxCnt_wd++;
+		record->wd_len = num;
+		memset((u8 *)record->wd_buf, 0, MAX_RXWD_SIZE);
+		memcpy((u8 *)record->wd_buf, p, num);
+	}
+
+	if(type == REC_RX_AMPDU){
+		u32 idx = 0; 
+
+		if(log->rxCnt_ampdu == 0 && (log->rxAmpdu[0] == 0))
+			tmp_rx_last_ppdu = num;
+
+		if(tmp_rx_last_ppdu != num){
+			tmp_rx_last_ppdu = num;
+
+			log->rxCnt_ampdu ++;
+			idx = log->rxCnt_ampdu%CORE_LOG_NUM;
+			log->rxAmpdu[idx] = 1;
+		}
+		else{
+			idx = log->rxCnt_ampdu%CORE_LOG_NUM;
+			log->rxAmpdu[idx]++;
+	}
+}
+
+}
+
+void core_cmd_record_trx(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	u32 idx = 0;
+	char *para = (char *)cmd_para;
+	
+	if(para_num<=0)
+		return;
+	
+	if(!strcmp(para, "start")){
+		u8 *log = NULL;
+		log = (u8*)&adapter->core_logs;
+		memset(log, 0, sizeof(struct core_logs));
+		log = (u8*)&adapter->phl_logs;
+		memset(log, 0, sizeof(struct phl_logs));
+		adapter->record_enable = 1;
+	}else if(!strcmp(para, "stop")){
+		adapter->record_enable = 0;
+	}else if(!strcmp(para, "dump")){
+		u32 dump_type = 0;
+		para=get_next_para_str(para);
+		sscanf(para, "%d", &dump_type);
+		phl_dump_record(adapter, (u8)dump_type);
+	}
+}
+
+void reset_txforce_para(_adapter *adapter)
+{
+	adapter->txForce_rate 	= INV_TXFORCE_VAL;
+	adapter->txForce_agg 	= INV_TXFORCE_VAL;
+	adapter->txForce_aggnum = INV_TXFORCE_VAL;
+	adapter->txForce_gi 	= INV_TXFORCE_VAL;
+}
+
+void core_cmd_txforce(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	u32 idx = 0;
+	char *para = (char *)cmd_para;
+
+	if(para_num<=0)
+	return;
+
+	if(!strcmp(para, "start")){
+		adapter->txForce_enable = 1;
+		reset_txforce_para(adapter);
+	}else if(!strcmp(para, "stop")){
+		adapter->txForce_enable = 0;
+		reset_txforce_para(adapter);
+	}else if(!strcmp(para, "rate")){
+		u32 rate = 0;
+		para=get_next_para_str(para);
+		sscanf(para, "%x", &rate);
+		adapter->txForce_rate = rate;
+	}else if(!strcmp(para, "agg")){
+		u32 agg = 0;
+		para=get_next_para_str(para);
+		sscanf(para, "%x", &agg);
+		adapter->txForce_agg = agg;
+	}else if(!strcmp(para, "aggnum")){
+		u32 aggnum = 0;
+		para=get_next_para_str(para);
+		sscanf(para, "%d", &aggnum);
+		adapter->txForce_aggnum = aggnum;
+	}else if(!strcmp(para, "gi")){
+		u32 gi = 0;
+		para=get_next_para_str(para);
+		sscanf(para, "%d", &gi);
+		adapter->txForce_gi = gi;
+	}else if(!strcmp(para, "macid")){
+
+	}else if(!strcmp(para, "retry")){
+	
+	}
+}
+
+#ifdef CONFIG_RTW_CORE_RXSC
+void core_cmd_rxsc(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	u32 idx = 0;
+	char *para = (char *)cmd_para;
+
+	printk("core_cmd_rxsc \n");
+
+	if(para_num<=0)
+		return;
+
+	if(!strcmp(para, "enable")){
+		printk("enable");
+		adapter->enable_rxsc = 1;
+	}else if(!strcmp(para, "disable")){
+		printk("disable");
+		adapter->enable_rxsc = 0;
+	}else if(!strcmp(para, "dump")){
+		struct core_logs *log = &adapter->core_logs;
+		printk("dump");
+		printk("enable_rxsc: %d \n", adapter->enable_rxsc);
+		printk("rxCnt_data: orig=%d shortcut=%d(ratio=%d)\n",
+			log->rxCnt_data_orig, log->rxCnt_data_shortcut,
+			log->rxCnt_data_shortcut*100/(log->rxCnt_data_orig+log->rxCnt_data_shortcut));
+	}
+
+}
+#endif
+
+void core_sniffer_rx(_adapter *adapter, u8 *pkt, u32 pktlen)
+{
+	struct sk_buff* pskb = NULL;
+
+	if(!adapter->sniffer_enable)
+		return;
+	
+	if(pkt==NULL)
+		return;
+
+	pskb = dev_alloc_skb(pktlen+200);
+	
+	if(pskb == NULL){
+	return;
+}
+
+	_rtw_memcpy(pskb->data, pkt, pktlen);
+	pskb->len = pktlen;
+
+	skb_reset_mac_header(pskb);
+	pskb->dev = adapter->pnetdev;
+	pskb->dev->type = ARPHRD_IEEE80211;
+	pskb->ip_summed = CHECKSUM_UNNECESSARY;
+	pskb->pkt_type = PACKET_OTHERHOST;
+	pskb->protocol = htons(ETH_P_802_2);
+	netif_receive_skb(pskb);
+
+	return;
+}
+	
+void core_cmd_sniffer(_adapter *adapter, void *cmd_para, u32 para_num)
+	{
+		u32 idx=0;
+	char *para = (char *)cmd_para;
+	
+	if(para_num<=0)
+	return;
+
+	if(!strcmp(para, "start")){
+		adapter->sniffer_enable = 1;
+	}else if(!strcmp(para, "stop")){
+		adapter->sniffer_enable = 0;
+	}
+}
+
+
+#define LEN_TEST_BUF 2000
+u8 test_buf[LEN_TEST_BUF];
+
+#ifdef CONFIG_PCI_HCI
+#include <rtw_trx_pci.h>
+#endif
+
+#include "../phl/phl_headers.h"
+#include "../phl/phl_api.h"
+#include "../phl/hal_g6/hal_api_mac.h"
+#include "../phl/hal_g6/mac/mac_reg.h"
+
+
+#define SHOW_REG32(adapter, reg) \
+	do { \
+		printk("\t%04X = %08X\n", \
+			  (reg), rtw_phl_read32(adapter->dvobj->phl, reg)); \
+	} while (0)
+
+#define SHOW_REG32_MSG(adapter, reg, msg) \
+	do { \
+		printk("\t%04X = %08X - %s\n", \
+			  (reg), rtw_phl_read32(adapter->dvobj->phl, reg), msg); \
+	} while (0)
+
+#define SHOW_REG16(adapter, reg) \
+	do { \
+		printk("\t%04X = %04X\n", \
+			  (reg), rtw_phl_read16(adapter->dvobj->phl, reg)); \
+	} while (0)
+
+#define SHOW_REG16_MSG(adapter, reg, msg) \
+	do { \
+		printk("\t%04X = %04X - %s\n", \
+			  (reg), rtw_phl_read16(adapter->dvobj->phl, reg), msg); \
+	} while (0)
+
+
+static inline void _show_RX_counter(_adapter *adapter)
+{
+	/* Show RX PPDU counters */
+	int i;
+	u32 reg32 = rtw_phl_read32(adapter->dvobj->phl, R_AX_RX_DBG_CNT_SEL);
+	static const char *cnt_name[] = {"Invalid packet",
+	                                 "RE-CCA",
+	                                 "RX FIFO overflow",
+	                                 "RX packet full drop",
+	                                 "RX packet dma OK",
+	                                 "UD 0",
+	                                 "UD 1",
+	                                 "UD 2",
+	                                 "UD 3",
+	                                 "continuous FCS error",
+	                                 "RX packet filter drop",
+	                                 "CSI packet DMA OK",
+	                                 "CSI packet DMA drop",
+	                                 "RX MAC stop"
+	};
+
+	printk("CMAC0 RX PPDU Counters @%04X:\n", R_AX_RX_DBG_CNT_SEL);
+	reg32 &= ~(B_AX_RX_CNT_IDX_MSK << B_AX_RX_CNT_IDX_SH);
+	for (i = 30; i < 44; i++) {
+		rtw_phl_write32(adapter->dvobj->phl, R_AX_RX_DBG_CNT_SEL,
+		            reg32 | (i << B_AX_RX_CNT_IDX_SH));
+		printk("    %02X: %d - %s\n", i,
+		         (
+		            (   rtw_phl_read32(adapter->dvobj->phl, R_AX_RX_DBG_CNT_SEL)
+		             >> B_AX_RX_DBG_CNT_SH)
+		          & B_AX_RX_DBG_CNT_MSK),
+		          cnt_name[i - 30]);
+	}
+
+	reg32 = rtw_phl_read32(adapter->dvobj->phl, R_AX_RX_DBG_CNT_SEL_C1);
+	printk("CMAC1 RX PPDU Counters @%04X:\n", R_AX_RX_DBG_CNT_SEL_C1);
+	reg32 &= ~(B_AX_RX_CNT_IDX_MSK << B_AX_RX_CNT_IDX_SH);
+	for (i = 30; i < 44; i++) {
+		rtw_phl_write32(adapter->dvobj->phl, R_AX_RX_DBG_CNT_SEL_C1,
+		            reg32 | (i << B_AX_RX_CNT_IDX_SH));
+		printk("    %02X: %d - %s\n", i,
+		         (
+		            (   rtw_phl_read32(adapter->dvobj->phl, R_AX_RX_DBG_CNT_SEL_C1)
+		             >> B_AX_RX_DBG_CNT_SH)
+		          & B_AX_RX_DBG_CNT_MSK),
+		          cnt_name[i - 30]);
+	}
+} /* _show_RX_counter */
+
+void _show_TX_dbg_status(_adapter *adapter)
+{
+	u32	reg32 = rtw_phl_read32(adapter->dvobj->phl, 0x9F1C);
+	printk("TX Debug: 0x%08X\n", reg32);
+}
+
+void _show_BCN_dbg_status(_adapter *adapter)
+{
+	SHOW_REG32_MSG(adapter, R_AX_PORT_CFG_P0,		"PORT_CFG_P0");
+	SHOW_REG32_MSG(adapter, R_AX_TBTT_PROHIB_P0,	"TBTT_PROHIB_P0");
+	SHOW_REG32_MSG(adapter, R_AX_EN_HGQ_NOLIMIT,	"EN_HGQ_NOLIMIT");
+	SHOW_REG32_MSG(adapter, R_AX_TBTT_AGG_P0,		"TBTT_AGG_P0");
+
+	SHOW_REG32_MSG(adapter, R_AX_PORT_CFG_P0_C1,	"PORT_CFG_P0_C1");
+	SHOW_REG32_MSG(adapter, R_AX_TBTT_PROHIB_P0_C1,	"TBTT_PROHIB_P0_C1");
+	SHOW_REG32_MSG(adapter, R_AX_EN_HGQ_NOLIMIT_C1,	"EN_HGQ_NOLIMIT_C1");
+	SHOW_REG32_MSG(adapter, R_AX_TBTT_AGG_P0_C1,	"TBTT_AGG_P0_C1");
+
+	SHOW_REG32_MSG(adapter, R_AX_WCPU_FW_CTRL,		"R_AX_WCPU_FW_CTRL");
+}
+
+
+void core_cmd_dump_debug(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	printk("TX path registers: \n");
+
+	SHOW_REG32_MSG(adapter, R_AX_RXQ_RXBD_IDX, "RX_BD_IDX");
+	SHOW_REG32_MSG(adapter, R_AX_RPQ_RXBD_IDX, "RP_BD_IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH0_TXBD_IDX, "ACH0 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH1_TXBD_IDX, "ACH1 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH2_TXBD_IDX, "ACH2 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH3_TXBD_IDX, "ACH3 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH4_TXBD_IDX, "ACH4 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH5_TXBD_IDX, "ACH5 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH6_TXBD_IDX, "ACH6 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_ACH7_TXBD_IDX, "ACH7 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_CH8_TXBD_IDX, "CH8 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_CH9_TXBD_IDX, "CH9 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_CH10_TXBD_IDX, "CH10 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_CH11_TXBD_IDX, "CH11 IDX");
+	SHOW_REG32_MSG(adapter, R_AX_CH12_TXBD_IDX, "CH12 IDX");
+#ifdef R_AX_PCIE_DBG_CTRL
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_DBG_CTRL, "DBG_CTRL");
+#else
+	SHOW_REG32_MSG(adapter, 0x11C0, "DBG_CTRL");
+#endif
+	SHOW_REG32_MSG(adapter, R_AX_DBG_ERR_FLAG, "DBG_ERR");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_HIMR00, "IMR0");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_HISR00, "ISR0");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_HIMR10, "IMR1");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_HISR10, "IMR1");
+	SHOW_REG16_MSG(adapter, R_AX_ACH0_BDRAM_RWPTR, "CH0");
+	SHOW_REG16_MSG(adapter, R_AX_ACH1_BDRAM_RWPTR, "CH1");
+	SHOW_REG16_MSG(adapter, R_AX_ACH2_BDRAM_RWPTR, "CH2");
+	SHOW_REG16_MSG(adapter, R_AX_ACH3_BDRAM_RWPTR, "CH3");
+	SHOW_REG16_MSG(adapter, R_AX_ACH4_BDRAM_RWPTR, "CH4");
+	SHOW_REG16_MSG(adapter, R_AX_ACH5_BDRAM_RWPTR, "CH5");
+	SHOW_REG16_MSG(adapter, R_AX_ACH6_BDRAM_RWPTR, "CH6");
+	SHOW_REG16_MSG(adapter, R_AX_ACH7_BDRAM_RWPTR, "CH7");
+	SHOW_REG16_MSG(adapter, R_AX_CH8_BDRAM_RWPTR, "CH8");
+	SHOW_REG16_MSG(adapter, R_AX_CH9_BDRAM_RWPTR, "CH9");
+	SHOW_REG16_MSG(adapter, R_AX_CH10_BDRAM_RWPTR, "CH10");
+	SHOW_REG16_MSG(adapter, R_AX_CH11_BDRAM_RWPTR, "CH11");
+	SHOW_REG16_MSG(adapter, R_AX_CH12_BDRAM_RWPTR, "CH12");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_DMA_STOP1, "DMA_STOP1");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_DMA_BUSY1, "DMA_BUSY1");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_DMA_STOP2, "DMA_STOP2");
+	SHOW_REG32_MSG(adapter, R_AX_PCIE_DMA_BUSY2, "DMA_BUSY2");
+	SHOW_REG32(adapter, 0x8840);
+	SHOW_REG32(adapter, 0x8844);
+	SHOW_REG32(adapter, 0x8854);
+	SHOW_REG16(adapter, 0xCA22);
+	SHOW_REG32(adapter, 0x8AA8);
+
+	/* Show TX PPDU counters */
+	do {
+		int i;
+		u32 reg32 = rtw_phl_read32(adapter->dvobj->phl, R_AX_TX_PPDU_CNT);
+
+		printk("CMAC0 TX PPDU Counters @%04X:\n", R_AX_TX_PPDU_CNT);
+
+		reg32 &= ~(B_AX_PPDU_CNT_IDX_MSK << B_AX_PPDU_CNT_IDX_SH);
+		for (i = 0; i < 11; i++) {
+			rtw_phl_write32(adapter->dvobj->phl, R_AX_TX_PPDU_CNT,
+			            reg32 | (i << B_AX_PPDU_CNT_IDX_SH));
+			printk("    %02X: %d\n", i,
+			         (
+			            (   rtw_phl_read32(adapter->dvobj->phl, R_AX_TX_PPDU_CNT)
+			             >> B_AX_TX_PPDU_CNT_SH)
+			          & B_AX_TX_PPDU_CNT_MSK));
+		}
+
+		reg32 = rtw_phl_read32(adapter->dvobj->phl, R_AX_TX_PPDU_CNT_C1);
+
+		printk("CMAC1 TX PPDU Counters @%04X:\n", R_AX_TX_PPDU_CNT_C1);
+
+		reg32 &= ~(B_AX_PPDU_CNT_IDX_MSK << B_AX_PPDU_CNT_IDX_SH);
+		for (i = 0; i < 11; i++) {
+			rtw_phl_write32(adapter->dvobj->phl, R_AX_TX_PPDU_CNT_C1,
+			            reg32 | (i << B_AX_PPDU_CNT_IDX_SH));
+			printk("    %02X: %d\n", i,
+			         (
+			            (   rtw_phl_read32(adapter->dvobj->phl, R_AX_TX_PPDU_CNT_C1)
+			             >> B_AX_TX_PPDU_CNT_SH)
+			          & B_AX_TX_PPDU_CNT_MSK));
+		}
+	} while (0);
+
+	/* Show RX PPDU counters */
+	_show_RX_counter(adapter);
+
+	_show_TX_dbg_status(adapter);
+
+	_show_BCN_dbg_status(adapter);
+
+}
+
+void core_cmd_dump_reg(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	u32 *para = (u32 *)cmd_para;
+	void *phl = adapter->dvobj->phl;
+	u32 reg_start, reg_end;
+	u32 idx = 0;
+
+	reg_start = para[0];
+	reg_end = reg_start + para[1];
+
+	while(1) {
+		if((reg_start>=reg_end) || (reg_start >= 0xffff))
+			break;
+		
+		printk("[%04x] %08x %08x %08x %08x \n", 
+			reg_start, 
+			rtw_phl_read32(phl, reg_start), rtw_phl_read32(phl, reg_start+4),
+			rtw_phl_read32(phl, reg_start+8), rtw_phl_read32(phl, reg_start+12));
+
+		reg_start+=16;
+	}
+}
+
+enum _CORE_CMD_PARA_TYPE {
+	CMD_PARA_DEC = 0,
+	CMD_PARA_HEX,
+	CMD_PARA_STR,
+};
+
+struct test_cmd_list {
+	const char *name;
+	void (*fun)(_adapter *, void *, u32);
+	enum _CORE_CMD_PARA_TYPE para_type;
+};
+
+void test_dump_dec(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	u32 idx = 0;
+	u32 *para = (u32 *)cmd_para;
+	DBGP("para_num=%d\n", para_num);
+
+	for(idx=0; idx<para_num; idx++)
+		DBGP("para[%d]=%d\n", para_num, para[idx]);
+	}
+
+void test_dump_hex(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	u32 idx = 0;
+	u32 *para = (u32 *)cmd_para;
+	DBGP("para_num=%d\n", para_num);
+
+	for(idx=0; idx<para_num; idx++)
+		DBGP("para[%d]=0x%x\n", para_num, para[idx]);
+}
+
+void test_dump_str(_adapter *adapter, void *cmd_para, u32 para_num)
+{
+	u32 idx = 0;
+	char *para = (char *)cmd_para;
+	DBGP("para_num=%d\n", para_num);
+
+	for(idx=0; idx<para_num; idx++, para+=MAX_PHL_CMD_STR_LEN)
+		DBGP("para[%d]=%s\n", para_num, para);
+}
+
+void get_all_cmd_para_value(_adapter *adapter, char *buf, u32 len, u32 *para, u8 type, u32 *num)
+{
+	u8 *tmp = NULL;
+
+	if(!buf || !len)
+		return;
+
+		DBGP("type=%d buf=%s para=%p num=%d\n", type, buf, para, *num);
+
+	if(len > 0){
+		tmp = strsep(&buf, ",");
+
+		if(tmp){
+			if(type == CMD_PARA_HEX)
+				sscanf(tmp, "%x", para);
+			else if(type == CMD_PARA_DEC)
+				sscanf(tmp, "%d", para);
+			para += 1;
+			*num = *num+1;
+		}
+		else
+			return;
+
+		if(buf && (len>strlen(tmp)))
+			get_all_cmd_para_value(adapter, buf, strlen(buf), para, type, num);
+		else
+			return;
+	}
+	}
+
+void get_all_cmd_para_str(_adapter *adapter, char *buf, u32 len, char *para, u8 type, u32* num)
+{
+	u8 *tmp = NULL;
+
+	if(!buf || !len)
+		return;
+
+	DBGP("type=%d buf=%s para=%p num=%d\n", type, buf, para, *num);
+
+	if(len > 0){
+		tmp = strsep(&buf, ",");
+
+		if(tmp){
+			strcpy(para, tmp);
+			para += MAX_PHL_CMD_STR_LEN;
+			*num = *num+1;
+	}
+		else
+			return;
+
+		if(buf && (len>strlen(tmp)))
+			get_all_cmd_para_str(adapter, buf, strlen(buf), para, type, num);
+		else
+			return;
+	}
+	}
+
+struct test_cmd_list core_test_cmd_list[] = {
+	{"dec", test_dump_dec, CMD_PARA_DEC},
+	{"hex", test_dump_hex, CMD_PARA_HEX},
+	{"str", test_dump_str, CMD_PARA_STR},
+	{"dump_reg", 		core_cmd_dump_reg, 		CMD_PARA_HEX},
+	{"dump_debug", 		core_cmd_dump_debug, 	CMD_PARA_DEC},
+	{"record", 			core_cmd_record_trx, 	CMD_PARA_STR},
+	{"txforce",			core_cmd_txforce, 		CMD_PARA_STR},
+	{"sniffer",			core_cmd_sniffer, 		CMD_PARA_STR},
+#ifdef CONFIG_RTW_CORE_RXSC
+	{"rxsc",	core_cmd_rxsc,		CMD_PARA_STR},
+#endif
+};
+
+void core_cmd_phl_handler(_adapter *adapter, char *extra)
+{
+	u32 para[MAX_PHL_CMD_NUM]={0};
+	char para_str[MAX_PHL_CMD_NUM][MAX_PHL_CMD_STR_LEN]={0};
+	char *cmd_name, *cmd_para = NULL;
+	struct test_cmd_list *cmd = &core_test_cmd_list;
+	u32 array_size = ARRAY_SIZE(core_test_cmd_list);
+	u32 i = 0;
+	
+	cmd_name = strsep(&extra, ",");
+
+	if(!cmd_name){
+		for(i = 0; i<array_size; i++, cmd++)
+			printk(" - %s\n", cmd->name);
+		return;
+	}
+
+	for(i = 0; i<array_size; i++, cmd++){
+		if(!strcmp(cmd->name, cmd_name)){
+			void *cmd_para = NULL;
+			u32 cmd_para_num = 0;
+			if(cmd->para_type == CMD_PARA_DEC || cmd->para_type == CMD_PARA_HEX){
+				cmd_para = para;
+				if(extra)
+				get_all_cmd_para_value(adapter, extra, strlen(extra), para, cmd->para_type, &cmd_para_num);
+		}
+		else{
+				cmd_para = para_str;
+				if(extra)
+				get_all_cmd_para_str(adapter, extra, strlen(extra), para_str, cmd->para_type, &cmd_para_num);
+	}
+
+			cmd->fun(adapter, cmd_para, cmd_para_num);
+			break;
+	}
+	}
+	
+}
+
+#endif
+
+u8 rtw_test_h2c_cmd(_adapter *adapter, u8 *buf, u8 len)
+{
+	struct cmd_obj *pcmdobj;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	u8 *ph2c_content;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	u8	res = _SUCCESS;
+
+	pcmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pcmdobj->padapter = adapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((u8 *)pcmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	ph2c_content = rtw_zmalloc(len);
+	if (ph2c_content == NULL) {
+		rtw_mfree((u8 *)pcmdobj, sizeof(struct cmd_obj));
+		rtw_mfree((u8 *)pdrvextra_cmd_parm, sizeof(struct drvextra_cmd_parm));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->ec_id = TEST_H2C_CID;
+	pdrvextra_cmd_parm->type = 0;
+	pdrvextra_cmd_parm->size = len;
+	pdrvextra_cmd_parm->pbuf = ph2c_content;
+
+	_rtw_memcpy(ph2c_content, buf, len);
+
+	init_h2fwcmd_w_parm_no_rsp(pcmdobj, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, pcmdobj);
+
+exit:
+	return res;
+}
+
+#ifdef CONFIG_MP_INCLUDED
+static s32 rtw_mp_cmd_hdl(_adapter *padapter, u8 mp_cmd_id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	int ret = H2C_SUCCESS;
+	uint status = _FALSE;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	struct rtw_test_module_info test_module_info;
+	struct rtw_mp_test_rpt my_mp_test_rpt;
+
+	if (mp_cmd_id == MP_START) {
+		if (padapter->registrypriv.mp_mode == 0) {
+
+			test_module_info.tm_mode = RTW_DRV_MODE_MP;
+			test_module_info.tm_type = RTW_TEST_SUB_MODULE_MP;
+			pmppriv->keep_ips_status = dvobj->phl_com->dev_sw_cap.ps_cap.ips_en;
+			pmppriv->keep_lps_status = dvobj->phl_com->dev_sw_cap.ps_cap.lps_en;
+			dvobj->phl_com->dev_sw_cap.ps_cap.ips_en = PS_OP_MODE_DISABLED;
+			dvobj->phl_com->dev_sw_cap.ps_cap.lps_en = PS_OP_MODE_DISABLED;
+
+			rtw_phl_test_submodule_init(dvobj->phl_com, &test_module_info);
+#ifdef CONFIG_BTC
+			if (dvobj->phl_com->dev_cap.btc_mode == BTC_MODE_NORMAL ||\
+					dvobj->phl_com->dev_cap.btc_mode == BTC_MODE_BT) {
+
+				pmppriv->mp_keep_btc_mode = dvobj->phl_com->dev_cap.btc_mode;
+				rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SWITCH_BT_PATH);
+				RTW_INFO("config BTC to WL Only\n");
+			}
+#endif
+			rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_PHY_INDEX);
+			status = rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_START_DUT);
+
+			if (status == _TRUE) {
+				padapter->registrypriv.mp_mode = 1;
+				MPT_InitializeAdapter(padapter, 1);
+			}
+		}
+
+		if (padapter->registrypriv.mp_mode == 0) {
+			ret = H2C_REJECTED;
+			goto exit;
+		}
+
+		if (padapter->mppriv.mode == MP_OFF) {
+			if (mp_start_test(padapter) == _FAIL) {
+				ret = H2C_REJECTED;
+				goto exit;
+			}
+			padapter->mppriv.mode = MP_ON;
+		}
+		padapter->mppriv.bmac_filter = _FALSE;
+
+	} else if (mp_cmd_id == MP_STOP) {
+		if (padapter->registrypriv.mp_mode == 1) {
+
+			status = rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_STOP_DUT);
+			RTW_INFO("RTW_MP_CONFIG_CMD_STOP_DUT %s\n", status == _TRUE ? "ok" : "fail");
+			rtw_phl_test_get_rpt(dvobj->phl_com, (u8*)&my_mp_test_rpt, sizeof(my_mp_test_rpt));
+
+			if (my_mp_test_rpt.status == _TRUE)
+				RTW_INFO("TM Sub finished OK!!!\n");
+#ifdef CONFIG_BTC
+			if (pmppriv->mp_keep_btc_mode != BTC_MODE_MAX) {
+				pmppriv->btc_path = pmppriv->mp_keep_btc_mode;
+				rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SWITCH_BT_PATH);
+			}
+#endif
+			test_module_info.tm_mode = RTW_DRV_MODE_NORMAL;
+			test_module_info.tm_type = RTW_TEST_SUB_MODULE_MP;
+			dvobj->phl_com->dev_sw_cap.ps_cap.ips_en = pmppriv->keep_ips_status;
+			dvobj->phl_com->dev_sw_cap.ps_cap.lps_en = pmppriv->keep_lps_status;
+
+			rtw_phl_test_submodule_deinit(dvobj->phl_com, &test_module_info);
+
+			MPT_DeInitAdapter(padapter);
+			padapter->registrypriv.mp_mode = 0;
+		}
+
+		if (padapter->mppriv.mode != MP_OFF) {
+			mp_stop_test(padapter);
+			padapter->mppriv.mode = MP_OFF;
+		}
+
+	} else {
+		RTW_INFO(FUNC_ADPT_FMT"invalid id:%d\n", FUNC_ADPT_ARG(padapter), mp_cmd_id);
+		ret = H2C_PARAMETERS_ERROR;
+		rtw_warn_on(1);
+	}
+
+exit:
+	return ret;
+}
+
+u8 rtw_mp_cmd(_adapter *adapter, u8 mp_cmd_id, u8 flags)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8	res = _SUCCESS;
+
+	parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	parm->ec_id = MP_CMD_WK_CID;
+	parm->type = mp_cmd_id;
+	parm->size = 0;
+	parm->pbuf = NULL;
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
+		if (H2C_SUCCESS != rtw_mp_cmd_hdl(adapter, mp_cmd_id))
+			res = _FAIL;
+		rtw_mfree((u8 *)parm, sizeof(*parm));
+	} else {
+		/* need enqueue, prepare cmd_obj and enqueue */
+		cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+		if (cmdobj == NULL) {
+			res = _FAIL;
+			rtw_mfree((u8 *)parm, sizeof(*parm));
+			goto exit;
+		}
+		cmdobj->padapter = adapter;
+
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+
+		if (flags & RTW_CMDF_WAIT_ACK) {
+			cmdobj->sctx = &sctx;
+			rtw_sctx_init(&sctx, 10 * 1000);
+		}
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+		if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+			rtw_sctx_wait(&sctx, __func__);
+			_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+			if (sctx.status == RTW_SCTX_SUBMITTED)
+				cmdobj->sctx = NULL;
+			_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+			if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+				res = _FAIL;
+		}
+	}
+
+exit:
+	return res;
+}
+#endif	/*CONFIG_MP_INCLUDED*/
+
+#ifdef CONFIG_RTW_CUSTOMER_STR
+static s32 rtw_customer_str_cmd_hdl(_adapter *adapter, u8 write, const u8 *cstr)
+{
+	int ret = H2C_SUCCESS;
+
+	if (write)
+		ret = rtw_hal_h2c_customer_str_write(adapter, cstr);
+	else
+		ret = rtw_hal_h2c_customer_str_req(adapter);
+
+	return ret == _SUCCESS ? H2C_SUCCESS : H2C_REJECTED;
+}
+
+static u8 rtw_customer_str_cmd(_adapter *adapter, u8 write, const u8 *cstr)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	u8 *str = NULL;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 res = _SUCCESS;
+
+	parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	if (write) {
+		str = rtw_zmalloc(RTW_CUSTOMER_STR_LEN);
+		if (str == NULL) {
+			rtw_mfree((u8 *)parm, sizeof(struct drvextra_cmd_parm));
+			res = _FAIL;
+			goto exit;
+		}
+	}
+
+	parm->ec_id = CUSTOMER_STR_WK_CID;
+	parm->type = write;
+	parm->size = write ? RTW_CUSTOMER_STR_LEN : 0;
+	parm->pbuf = write ? str : NULL;
+
+	if (write)
+		_rtw_memcpy(str, cstr, RTW_CUSTOMER_STR_LEN);
+
+	/* need enqueue, prepare cmd_obj and enqueue */
+	cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+	if (cmdobj == NULL) {
+		res = _FAIL;
+		rtw_mfree((u8 *)parm, sizeof(*parm));
+		if (write)
+			rtw_mfree(str, RTW_CUSTOMER_STR_LEN);
+		goto exit;
+	}
+	cmdobj->padapter = adapter;
+
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+
+	cmdobj->sctx = &sctx;
+	rtw_sctx_init(&sctx, 2 * 1000);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+	if (res == _SUCCESS) {
+		rtw_sctx_wait(&sctx, __func__);
+		_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+		if (sctx.status == RTW_SCTX_SUBMITTED)
+			cmdobj->sctx = NULL;
+		_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+			res = _FAIL;
+	}
+
+exit:
+	return res;
+}
+
+inline u8 rtw_customer_str_req_cmd(_adapter *adapter)
+{
+	return rtw_customer_str_cmd(adapter, 0, NULL);
+}
+
+inline u8 rtw_customer_str_write_cmd(_adapter *adapter, const u8 *cstr)
+{
+	return rtw_customer_str_cmd(adapter, 1, cstr);
+}
+#endif /* CONFIG_RTW_CUSTOMER_STR */
+
+u8 rtw_c2h_wk_cmd(_adapter *padapter, u8 *pbuf, u16 length, u8 type)
+{
+	struct cmd_obj *cmd;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8 *extra_cmd_buf;
+	u8 res = _SUCCESS;
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((u8 *)cmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	extra_cmd_buf = rtw_zmalloc(length);
+	if (extra_cmd_buf == NULL) {
+		rtw_mfree((u8 *)cmd, sizeof(struct cmd_obj));
+		rtw_mfree((u8 *)pdrvextra_cmd_parm, sizeof(struct drvextra_cmd_parm));
+		res = _FAIL;
+		goto exit;
+	}
+
+	_rtw_memcpy(extra_cmd_buf, pbuf, length);
+	pdrvextra_cmd_parm->ec_id = C2H_WK_CID;
+	pdrvextra_cmd_parm->type = type;
+	pdrvextra_cmd_parm->size = length;
+	pdrvextra_cmd_parm->pbuf = extra_cmd_buf;
+
+	init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;
+}
+
+#define C2H_TYPE_PKT 1
+inline u8 rtw_c2h_packet_wk_cmd(_adapter *adapter, u8 *c2h_evt, u16 length)
+{
+	return rtw_c2h_wk_cmd(adapter, c2h_evt, length, C2H_TYPE_PKT);
+}
+
+static u8 _rtw_run_in_thread_cmd(_adapter *adapter, void (*func)(void *), void *context, s32 timeout_ms)
+{
+	struct cmd_priv *cmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct cmd_obj *cmdobj;
+	struct RunInThread_param *parm;
+	struct submit_ctx sctx;
+	s32 res = _SUCCESS;
+
+	cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (NULL == cmdobj) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmdobj->padapter = adapter;
+
+	parm = (struct RunInThread_param *)rtw_zmalloc(sizeof(struct RunInThread_param));
+	if (NULL == parm) {
+		rtw_mfree((u8 *)cmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	parm->func = func;
+	parm->context = context;
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_RUN_INTHREAD);
+
+	if (timeout_ms >= 0) {
+		cmdobj->sctx = &sctx;
+		rtw_sctx_init(&sctx, timeout_ms);
+	}
+
+	res = rtw_enqueue_cmd(cmdpriv, cmdobj);
+
+	if (res == _SUCCESS && timeout_ms >= 0) {
+		rtw_sctx_wait(&sctx, __func__);
+		_rtw_mutex_lock_interruptible(&cmdpriv->sctx_mutex);
+		if (sctx.status == RTW_SCTX_SUBMITTED)
+			cmdobj->sctx = NULL;
+		_rtw_mutex_unlock(&cmdpriv->sctx_mutex);
+		if (sctx.status != RTW_SCTX_DONE_SUCCESS)
+			res = _FAIL;
+	}
+
+exit:
+	return res;
+}
+u8 rtw_run_in_thread_cmd(_adapter *adapter, void (*func)(void *), void *context)
+{
+	return _rtw_run_in_thread_cmd(adapter, func, context, -1);
+}
+
+u8 rtw_run_in_thread_cmd_wait(_adapter *adapter, void (*func)(void *), void *context, s32 timeout_ms)
+{
+	return _rtw_run_in_thread_cmd(adapter, func, context, timeout_ms);
+}
+
+
+u8 session_tracker_cmd(_adapter *adapter, u8 cmd, struct sta_info *sta, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port)
+{
+	struct cmd_priv	*cmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *cmd_parm;
+	struct st_cmd_parm *st_parm;
+	u8	res = _SUCCESS;
+
+	cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmdobj->padapter = adapter;
+
+	cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (cmd_parm == NULL) {
+		rtw_mfree((u8 *)cmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	st_parm = (struct st_cmd_parm *)rtw_zmalloc(sizeof(struct st_cmd_parm));
+	if (st_parm == NULL) {
+		rtw_mfree((u8 *)cmdobj, sizeof(struct cmd_obj));
+		rtw_mfree((u8 *)cmd_parm, sizeof(struct drvextra_cmd_parm));
+		res = _FAIL;
+		goto exit;
+	}
+
+	st_parm->cmd = cmd;
+	st_parm->sta = sta;
+	if (cmd != ST_CMD_CHK) {
+		_rtw_memcpy(&st_parm->local_naddr, local_naddr, 4);
+		_rtw_memcpy(&st_parm->local_port, local_port, 2);
+		_rtw_memcpy(&st_parm->remote_naddr, remote_naddr, 4);
+		_rtw_memcpy(&st_parm->remote_port, remote_port, 2);
+	}
+
+	cmd_parm->ec_id = SESSION_TRACKER_WK_CID;
+	cmd_parm->type = 0;
+	cmd_parm->size = sizeof(struct st_cmd_parm);
+	cmd_parm->pbuf = (u8 *)st_parm;
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, cmd_parm, CMD_SET_DRV_EXTRA);
+	cmdobj->no_io = 1;
+
+	res = rtw_enqueue_cmd(cmdpriv, cmdobj);
+
+exit:
+	return res;
+}
+
+inline u8 session_tracker_chk_cmd(_adapter *adapter, struct sta_info *sta)
+{
+	return session_tracker_cmd(adapter, ST_CMD_CHK, sta, NULL, NULL, NULL, NULL);
+}
+
+inline u8 session_tracker_add_cmd(_adapter *adapter, struct sta_info *sta, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port)
+{
+	return session_tracker_cmd(adapter, ST_CMD_ADD, sta, local_naddr, local_port, remote_naddr, remote_port);
+}
+
+inline u8 session_tracker_del_cmd(_adapter *adapter, struct sta_info *sta, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port)
+{
+	return session_tracker_cmd(adapter, ST_CMD_DEL, sta, local_naddr, local_port, remote_naddr, remote_port);
+}
+
+void session_tracker_chk_for_sta(_adapter *adapter, struct sta_info *sta)
+{
+	struct st_ctl_t *st_ctl = &sta->st_ctl;
+	int i;
+	_list *plist, *phead, *pnext;
+	_list dlist;
+	struct session_tracker *st = NULL;
+	u8 op_wfd_mode = MIRACAST_DISABLED;
+
+	if (DBG_SESSION_TRACKER)
+		RTW_INFO(FUNC_ADPT_FMT" sta:%p\n", FUNC_ADPT_ARG(adapter), sta);
+
+	if (!(sta->state & WIFI_ASOC_STATE))
+		goto exit;
+
+	for (i = 0; i < SESSION_TRACKER_REG_ID_NUM; i++) {
+		if (st_ctl->reg[i].s_proto != 0)
+			break;
+	}
+	if (i >= SESSION_TRACKER_REG_ID_NUM)
+		goto chk_sta;
+
+	_rtw_init_listhead(&dlist);
+
+	_rtw_spinlock_bh(&st_ctl->tracker_q.lock);
+
+	phead = &st_ctl->tracker_q.queue;
+	plist = get_next(phead);
+	pnext = get_next(plist);
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		st = LIST_CONTAINOR(plist, struct session_tracker, list);
+		plist = pnext;
+		pnext = get_next(pnext);
+
+		if (st->status != ST_STATUS_ESTABLISH
+			&& rtw_get_passing_time_ms(st->set_time) > ST_EXPIRE_MS
+		) {
+			rtw_list_delete(&st->list);
+			rtw_list_insert_tail(&st->list, &dlist);
+		}
+
+		/* TODO: check OS for status update */
+		if (st->status == ST_STATUS_CHECK)
+			st->status = ST_STATUS_ESTABLISH;
+
+		if (st->status != ST_STATUS_ESTABLISH)
+			continue;
+
+		#ifdef CONFIG_WFD
+		if (0)
+			RTW_INFO(FUNC_ADPT_FMT" local:%u, remote:%u, rtsp:%u, %u, %u\n", FUNC_ADPT_ARG(adapter)
+				, ntohs(st->local_port), ntohs(st->remote_port), adapter->wfd_info.rtsp_ctrlport, adapter->wfd_info.tdls_rtsp_ctrlport
+				, adapter->wfd_info.peer_rtsp_ctrlport);
+		if (ntohs(st->local_port) == adapter->wfd_info.rtsp_ctrlport)
+			op_wfd_mode |= MIRACAST_SINK;
+		if (ntohs(st->local_port) == adapter->wfd_info.tdls_rtsp_ctrlport)
+			op_wfd_mode |= MIRACAST_SINK;
+		if (ntohs(st->remote_port) == adapter->wfd_info.peer_rtsp_ctrlport)
+			op_wfd_mode |= MIRACAST_SOURCE;
+		#endif
+	}
+
+	_rtw_spinunlock_bh(&st_ctl->tracker_q.lock);
+
+	plist = get_next(&dlist);
+	while (rtw_end_of_queue_search(&dlist, plist) == _FALSE) {
+		st = LIST_CONTAINOR(plist, struct session_tracker, list);
+		plist = get_next(plist);
+		rtw_mfree((u8 *)st, sizeof(struct session_tracker));
+	}
+
+chk_sta:
+	if (STA_OP_WFD_MODE(sta) != op_wfd_mode) {
+		STA_SET_OP_WFD_MODE(sta, op_wfd_mode);
+		/* TODO: op_wfd_mode inform */
+		/* rtw_sta_hal_media_status_rpt_cmd(adapter, sta, 1, 0); */
+	}
+
+exit:
+	return;
+}
+
+void session_tracker_chk_for_adapter(_adapter *adapter)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct sta_info *sta;
+	int i;
+	_list *plist, *phead;
+	u8 op_wfd_mode = MIRACAST_DISABLED;
+
+	_rtw_spinlock_bh(&stapriv->sta_hash_lock);
+
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(stapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			sta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+
+			session_tracker_chk_for_sta(adapter, sta);
+
+			op_wfd_mode |= STA_OP_WFD_MODE(sta);
+		}
+	}
+
+	_rtw_spinunlock_bh(&stapriv->sta_hash_lock);
+
+#ifdef CONFIG_WFD
+	adapter->wfd_info.op_wfd_mode = MIRACAST_MODE_REVERSE(op_wfd_mode);
+#endif
+}
+
+void session_tracker_cmd_hdl(_adapter *adapter, struct st_cmd_parm *parm)
+{
+	u8 cmd = parm->cmd;
+	struct sta_info *sta = parm->sta;
+
+	if (cmd == ST_CMD_CHK) {
+		if (sta)
+			session_tracker_chk_for_sta(adapter, sta);
+		else
+			session_tracker_chk_for_adapter(adapter);
+
+		goto exit;
+
+	} else if (cmd == ST_CMD_ADD || cmd == ST_CMD_DEL) {
+		struct st_ctl_t *st_ctl;
+		u32 local_naddr = parm->local_naddr;
+		u16 local_port = parm->local_port;
+		u32 remote_naddr = parm->remote_naddr;
+		u16 remote_port = parm->remote_port;
+		struct session_tracker *st = NULL;
+		_list *plist, *phead;
+		u8 free_st = 0;
+		u8 alloc_st = 0;
+
+		if (DBG_SESSION_TRACKER)
+			RTW_INFO(FUNC_ADPT_FMT" cmd:%u, sta:%p, local:"IP_FMT":"PORT_FMT", remote:"IP_FMT":"PORT_FMT"\n"
+				, FUNC_ADPT_ARG(adapter), cmd, sta
+				, IP_ARG(&local_naddr), PORT_ARG(&local_port)
+				, IP_ARG(&remote_naddr), PORT_ARG(&remote_port)
+			);
+
+		if (!(sta->state & WIFI_ASOC_STATE))
+			goto exit;
+
+		st_ctl = &sta->st_ctl;
+
+		_rtw_spinlock_bh(&st_ctl->tracker_q.lock);
+
+		phead = &st_ctl->tracker_q.queue;
+		plist = get_next(phead);
+		while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+			st = LIST_CONTAINOR(plist, struct session_tracker, list);
+
+			if (st->local_naddr == local_naddr
+				&& st->local_port == local_port
+				&& st->remote_naddr == remote_naddr
+				&& st->remote_port == remote_port)
+				break;
+
+			plist = get_next(plist);
+		}
+
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			st = NULL;
+
+		switch (cmd) {
+		case ST_CMD_DEL:
+			if (st) {
+				rtw_list_delete(plist);
+				free_st = 1;
+			}
+			goto unlock;
+		case ST_CMD_ADD:
+			if (!st)
+				alloc_st = 1;
+		}
+
+unlock:
+		_rtw_spinunlock_bh(&st_ctl->tracker_q.lock);
+
+		if (free_st) {
+			rtw_mfree((u8 *)st, sizeof(struct session_tracker));
+			goto exit;
+		}
+
+		if (alloc_st) {
+			st = (struct session_tracker *)rtw_zmalloc(sizeof(struct session_tracker));
+			if (!st)
+				goto exit;
+
+			st->local_naddr = local_naddr;
+			st->local_port = local_port;
+			st->remote_naddr = remote_naddr;
+			st->remote_port = remote_port;
+			st->set_time = rtw_get_current_time();
+			st->status = ST_STATUS_CHECK;
+
+			_rtw_spinlock_bh(&st_ctl->tracker_q.lock);
+			rtw_list_insert_tail(&st->list, phead);
+			_rtw_spinunlock_bh(&st_ctl->tracker_q.lock);
+		}
+	}
+
+exit:
+	return;
+}
+
+#if defined(CONFIG_RTW_MESH) && defined(RTW_PER_CMD_SUPPORT_FW)
+static s32 rtw_req_per_cmd_hdl(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj);
+	struct macid_bmp req_macid_bmp, *macid_bmp;
+	u8 i, ret = _FAIL;
+
+	macid_bmp = &macid_ctl->if_g[adapter->iface_id];
+	_rtw_memcpy(&req_macid_bmp, macid_bmp, sizeof(struct macid_bmp));
+
+	/* Clear none mesh's macid */
+	for (i = 0; i < macid_ctl->num; i++) {
+		u8 role;
+		role = GET_H2CCMD_MSRRPT_PARM_ROLE(&macid_ctl->h2c_msr[i]);
+		if (role != H2C_MSR_ROLE_MESH)
+			rtw_macid_map_clr(&req_macid_bmp, i);
+	}
+
+	/* group_macid: always be 0 in NIC, so only pass macid_bitmap.m0
+	 * rpt_type: 0 includes all info in 1, use 0 for now 
+	 * macid_bitmap: pass m0 only for NIC
+	 */
+	ret = rtw_hal_set_req_per_rpt_cmd(adapter, 0, 0, req_macid_bmp.m0);
+
+	return ret;
+}
+
+u8 rtw_req_per_cmd(_adapter *adapter)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	struct submit_ctx sctx;
+	u8 res = _SUCCESS;
+
+	parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	parm->ec_id = REQ_PER_CMD_WK_CID;
+	parm->type = 0;
+	parm->size = 0;
+	parm->pbuf = NULL;
+
+	cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(*cmdobj));
+	if (cmdobj == NULL) {
+		res = _FAIL;
+		rtw_mfree((u8 *)parm, sizeof(*parm));
+		goto exit;
+	}
+	cmdobj->padapter = adapter;
+
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+exit:
+	return res;
+}
+#endif
+
+
+void rtw_ac_parm_cmd_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *_ac_parm_buf, int ac_type)
+{
+
+	u32 ac_parm_buf;
+
+	_rtw_memcpy(&ac_parm_buf, _ac_parm_buf, sizeof(ac_parm_buf));
+	switch (ac_type) {
+	case XMIT_VO_QUEUE:
+		RTW_INFO(FUNC_NDEV_FMT" AC_VO = 0x%08x\n", FUNC_ADPT_ARG(padapter), (unsigned int) ac_parm_buf);
+		rtw_ap_set_edca(padapter, padapter_link, 3, ac_parm_buf);
+		break;
+
+	case XMIT_VI_QUEUE:
+		RTW_INFO(FUNC_NDEV_FMT" AC_VI = 0x%08x\n", FUNC_ADPT_ARG(padapter), (unsigned int) ac_parm_buf);
+		rtw_ap_set_edca(padapter, padapter_link, 2, ac_parm_buf);
+		break;
+
+	case XMIT_BE_QUEUE:
+		RTW_INFO(FUNC_NDEV_FMT" AC_BE = 0x%08x\n", FUNC_ADPT_ARG(padapter), (unsigned int) ac_parm_buf);
+		rtw_ap_set_edca(padapter, padapter_link, 0, ac_parm_buf);
+		break;
+
+	case XMIT_BK_QUEUE:
+		RTW_INFO(FUNC_NDEV_FMT" AC_BK = 0x%08x\n", FUNC_ADPT_ARG(padapter), (unsigned int) ac_parm_buf);
+		rtw_ap_set_edca(padapter, padapter_link, 1, ac_parm_buf);
+		break;
+
+	default:
+		break;
+	}
+
+}
+
+u8 rtw_drvextra_cmd_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	int ret = H2C_SUCCESS;
+	struct drvextra_cmd_parm *pdrvextra_cmd;
+
+	if (!pbuf)
+		return H2C_PARAMETERS_ERROR;
+
+	pdrvextra_cmd = (struct drvextra_cmd_parm *)pbuf;
+
+	switch (pdrvextra_cmd->ec_id) {
+#ifdef CONFIG_ANTENNA_DIVERSITY
+	case ANT_SELECT_WK_CID:
+		antenna_select_wk_hdl(padapter, pdrvextra_cmd->type);
+		break;
+#endif
+#ifdef CONFIG_P2P_PS
+	case P2P_PS_WK_CID:
+		p2p_ps_wk_hdl(padapter, pdrvextra_cmd->type);
+		break;
+#endif
+#ifdef CONFIG_AP_MODE
+	case CHECK_HIQ_WK_CID:
+		rtw_chk_hi_queue_hdl(padapter);
+		break;
+#endif
+	/* add for CONFIG_IEEE80211W, none 11w can use it */
+	case RESET_SECURITYPRIV:
+		reset_securitypriv_hdl(padapter);
+		break;
+	case FREE_ASSOC_RESOURCES:
+		free_assoc_resources_hdl(padapter, (u8)pdrvextra_cmd->type);
+		break;
+	case C2H_WK_CID:
+		switch (pdrvextra_cmd->type) {
+		case C2H_TYPE_PKT:
+			rtw_hal_c2h_pkt_hdl(padapter, pdrvextra_cmd->pbuf, pdrvextra_cmd->size);
+			break;
+		default:
+			RTW_ERR("unknown C2H type:%d\n", pdrvextra_cmd->type);
+			rtw_warn_on(1);
+			break;
+		}
+		break;
+
+#ifdef CONFIG_DFS_MASTER
+	case DFS_RADAR_DETECT_WK_CID:
+		rtw_dfs_rd_hdl(adapter_to_dvobj(padapter), (enum phl_band_idx)pdrvextra_cmd->type, 0, CHANNEL_WIDTH_MAX);
+		break;
+	case DFS_RADAR_DETECT_EN_DEC_WK_CID:
+		rtw_dfs_rd_en_dec_update(adapter_to_dvobj(padapter), (enum phl_band_idx)pdrvextra_cmd->type);
+		break;
+#endif
+	case SESSION_TRACKER_WK_CID:
+		session_tracker_cmd_hdl(padapter, (struct st_cmd_parm *)pdrvextra_cmd->pbuf);
+		break;
+	case TEST_H2C_CID:
+		rtw_hal_fill_h2c_cmd(padapter, pdrvextra_cmd->pbuf[0], pdrvextra_cmd->size - 1, &pdrvextra_cmd->pbuf[1]);
+		break;
+	case MP_CMD_WK_CID:
+#ifdef CONFIG_MP_INCLUDED
+		ret = rtw_mp_cmd_hdl(padapter, pdrvextra_cmd->type);
+#endif
+		break;
+#ifdef CONFIG_RTW_CUSTOMER_STR
+	case CUSTOMER_STR_WK_CID:
+		ret = rtw_customer_str_cmd_hdl(padapter, pdrvextra_cmd->type, pdrvextra_cmd->pbuf);
+		break;
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	case MGNT_TX_WK_CID:
+		ret = rtw_mgnt_tx_handler(padapter, pdrvextra_cmd->pbuf);
+		break;
+#endif /* CONFIG_IOCTL_CFG80211 */
+#if defined(CONFIG_RTW_MESH) && defined(RTW_PER_CMD_SUPPORT_FW)
+	case REQ_PER_CMD_WK_CID:
+		ret = rtw_req_per_cmd_hdl(padapter);
+		break;
+#endif
+	case SSMPS_WK_CID :
+		rtw_ssmps_wk_hdl(padapter, (struct ssmps_cmd_parm *)pdrvextra_cmd->pbuf);
+		break;
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	case TXSS_WK_CID :
+		rtw_ctrl_txss_wk_hdl(padapter, (struct txss_cmd_parm *)pdrvextra_cmd->pbuf);
+		break;
+#endif
+	case AC_PARM_CMD_WK_CID:
+		rtw_ac_parm_cmd_hdl(padapter, pdrvextra_cmd->adapter_link, pdrvextra_cmd->pbuf, pdrvextra_cmd->type);
+		break;
+#ifdef CONFIG_AP_MODE
+	case STOP_AP_WK_CID:
+		stop_ap_hdl(padapter);
+		break;
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	case TBTX_CONTROL_TX_WK_CID:
+		tx_control_hdl(padapter);
+		break;
+#endif
+#ifdef PRIVATE_R
+	case FIND_REMOTE_WK_CID:
+		ret = issue_action_find_remote(padapter);
+		break;
+#ifdef CONFIG_P2P
+	case HIDE_SSID_WK_CID:
+		issue_beacon(padapter, 0);
+		break;
+#endif
+#endif
+#ifdef CONFIG_BTC
+	case CORE_BTC_CID:
+		ret = rtw_core_btc_hdl(padapter, (enum rtw_core_btc_cmd_id)pdrvextra_cmd->type);
+		break;
+#endif
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+
+void rtw_disassoc_cmd_callback(_adapter *padapter,  struct cmd_obj *pcmd)
+{
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+
+	if (pcmd->res != H2C_SUCCESS) {
+		_rtw_spinlock_bh(&pmlmepriv->lock);
+		set_fwstate(pmlmepriv, WIFI_ASOC_STATE);
+		_rtw_spinunlock_bh(&pmlmepriv->lock);
+		goto exit;
+	}
+#ifdef CONFIG_BR_EXT
+	else /* clear bridge database */
+		nat25_db_cleanup(padapter);
+#endif /* CONFIG_BR_EXT */
+
+exit:
+	return;
+}
+
+void rtw_create_ibss_post_hdl(_adapter *padapter, int status)
+{
+	struct wlan_network *pwlan = NULL;
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	WLAN_BSSID_EX *pdev_network = &padapter->registrypriv.dev_network;
+	struct wlan_network *mlme_cur_network = &(pmlmepriv->dev_cur_network);
+
+	if (status != H2C_SUCCESS)
+		set_assoc_timer(pmlmepriv, 1); /*_set_timer(&pmlmepriv->assoc_timer, 1);*/
+
+	/*_cancel_timer_ex(&pmlmepriv->assoc_timer);*/
+	cancel_assoc_timer(pmlmepriv);
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	{
+		pwlan = _rtw_alloc_network(pmlmepriv);
+		_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+		if (pwlan == NULL) {
+			pwlan = rtw_get_oldest_wlan_network(&pmlmepriv->scanned_queue);
+			if (pwlan == NULL) {
+				_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+				goto createbss_cmd_fail;
+			}
+			pwlan->last_scanned = rtw_get_current_time();
+		} else
+			rtw_list_insert_tail(&(pwlan->list), &pmlmepriv->scanned_queue.queue);
+
+		pdev_network->Length = get_WLAN_BSSID_EX_sz(pdev_network);
+		_rtw_memcpy(&(pwlan->network), pdev_network, pdev_network->Length);
+		/* pwlan->fixed = _TRUE; */
+
+		/* copy pdev_network information to pmlmepriv->cur_network */
+		_rtw_memcpy(&mlme_cur_network->network, pdev_network, (get_WLAN_BSSID_EX_sz(pdev_network)));
+
+#if 0
+		/* reset DSConfig */
+		mlme_cur_network->network.Configuration.DSConfig = (u32)rtw_ch2freq(pdev_network->Configuration.DSConfig);
+#endif
+
+		_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+		_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+		/* we will set WIFI_ASOC_STATE when there is one more sat to join us (rtw_stassoc_event_callback) */
+	}
+
+createbss_cmd_fail:
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+	return;
+}
+
+void rtw_getrttbl_cmd_cmdrsp_callback(_adapter *padapter,  struct cmd_obj *pcmd)
+{
+
+	rtw_free_cmd_obj(pcmd);
+#ifdef CONFIG_MP_INCLUDED
+	if (padapter->registrypriv.mp_mode == 1)
+		padapter->mppriv.workparam.bcompleted = _TRUE;
+#endif
+
+
+}
+
+u8 set_txq_params_cmd(_adapter *adapter, struct _ADAPTER_LINK *padapter_link, u32 ac_parm, u8 ac_type)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	u8 *ac_parm_buf = NULL;
+	u8 sz;
+	u8 res = _SUCCESS;
+
+
+	cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmdobj->padapter = adapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((u8 *)cmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	sz = sizeof(ac_parm);
+	ac_parm_buf = rtw_zmalloc(sz);
+	if (ac_parm_buf == NULL) {
+		rtw_mfree((u8 *)cmdobj, sizeof(struct cmd_obj));
+		rtw_mfree((u8 *)pdrvextra_cmd_parm, sizeof(struct drvextra_cmd_parm));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->adapter_link = padapter_link;
+	pdrvextra_cmd_parm->ec_id = AC_PARM_CMD_WK_CID;
+	pdrvextra_cmd_parm->type = ac_type;
+	pdrvextra_cmd_parm->size = sz;
+	pdrvextra_cmd_parm->pbuf = ac_parm_buf;
+
+	_rtw_memcpy(ac_parm_buf, &ac_parm, sz);
+
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+	res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+exit:
+	return res;
+}
+
+char UNKNOWN_CID[16] = "UNKNOWN_EXTRA";
+char *rtw_extra_name(struct drvextra_cmd_parm *pdrvextra_cmd)
+{
+	switch(pdrvextra_cmd->ec_id) {
+	case NONE_WK_CID:
+		return "NONE_WK_CID";
+		break;
+	#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+	case DYNAMIC_CHK_WK_CID:
+		return "DYNAMIC_CHK_WK_CID";
+		break;
+	#endif
+	case DM_CTRL_WK_CID:
+		return "DM_CTRL_WK_CID";
+		break;
+	case PBC_POLLING_WK_CID:
+		return "PBC_POLLING_WK_CID";
+		break;
+	case LPS_CTRL_WK_CID:
+		return "LPS_CTRL_WK_CID";
+		break;
+	case ANT_SELECT_WK_CID:
+		return "ANT_SELECT_WK_CID";
+		break;
+	case P2P_PS_WK_CID:
+		return "P2P_PS_WK_CID";
+		break;
+	case CHECK_HIQ_WK_CID:
+		return "CHECK_HIQ_WK_CID";
+		break;
+	case C2H_WK_CID:
+		return "C2H_WK_CID";
+		break;
+	case RESET_SECURITYPRIV:
+		return "RESET_SECURITYPRIV";
+		break;
+	case FREE_ASSOC_RESOURCES:
+		return "FREE_ASSOC_RESOURCES";
+		break;
+	case DM_IN_LPS_WK_CID:
+		return "DM_IN_LPS_WK_CID";
+		break;
+	case LPS_CHANGE_DTIM_CID:
+		return "LPS_CHANGE_DTIM_CID";
+		break;
+	case DFS_RADAR_DETECT_WK_CID:
+		return "DFS_RADAR_DETECT_WK_CID";
+		break;
+	case DFS_RADAR_DETECT_EN_DEC_WK_CID:
+		return "DFS_RADAR_DETECT_EN_DEC_WK_CID";
+		break;
+	case SESSION_TRACKER_WK_CID:
+		return "SESSION_TRACKER_WK_CID";
+		break;
+	case TEST_H2C_CID:
+		return "TEST_H2C_CID";
+		break;
+	case MP_CMD_WK_CID:
+		return "MP_CMD_WK_CID";
+		break;
+	case CUSTOMER_STR_WK_CID:
+		return "CUSTOMER_STR_WK_CID";
+		break;
+	case MGNT_TX_WK_CID:
+		return "MGNT_TX_WK_CID";
+		break;
+	case REQ_PER_CMD_WK_CID:
+		return "REQ_PER_CMD_WK_CID";
+		break;
+	case SSMPS_WK_CID:
+		return "SSMPS_WK_CID";
+		break;
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	case TXSS_WK_CID:
+		return "TXSS_WK_CID";
+		break;
+#endif
+	case AC_PARM_CMD_WK_CID:
+		return "AC_PARM_CMD_WK_CID";
+		break;
+#ifdef CONFIG_AP_MODE
+	case STOP_AP_WK_CID:
+		return "STOP_AP_WK_CID";
+		break;
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	case TBTX_CONTROL_TX_WK_CID:
+		return "TBTX_CONTROL_TX_WK_CID";
+		break;
+#endif
+#ifdef PRIVATE_R
+	case FIND_REMOTE_WK_CID:
+		return "FIND_REMOTE_WK_CID";
+		break;
+#ifdef CONFIG_P2P
+	case HIDE_SSID_WK_CID:
+		return "HIDE_SSID_WK_CID";
+		break;
+#endif
+#endif
+	case MAX_WK_CID:
+		return "MAX_WK_CID";
+		break;
+	default:
+		return UNKNOWN_CID;
+		break;
+	}
+	return UNKNOWN_CID;
+}
+
+char UNKNOWN_CMD[16] = "UNKNOWN_CMD";
+char *rtw_cmd_name(struct cmd_obj *pcmd)
+{
+	struct rtw_evt_header *pev;
+
+	if (pcmd->cmdcode >= (sizeof(wlancmds) / sizeof(struct rtw_cmd)))
+		return UNKNOWN_CMD;
+
+	if (pcmd->cmdcode == CMD_SET_MLME_EVT)
+		return rtw_evt_name((struct rtw_evt_header*)pcmd->parmbuf);
+
+	if (pcmd->cmdcode == CMD_SET_DRV_EXTRA)
+		return rtw_extra_name((struct drvextra_cmd_parm*)pcmd->parmbuf);
+
+	return wlancmds[pcmd->cmdcode].name;
+}
+
+#ifdef PRIVATE_R
+u8 rtw_find_remote_wk_cmd(_adapter *adapter)
+{
+	struct cmd_obj		*cmdobj;
+	struct drvextra_cmd_parm  *pdrvextra_cmd_parm;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	u8	res = _SUCCESS;
+
+	cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	cmdobj->padapter = adapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((unsigned char *)cmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->ec_id = FIND_REMOTE_WK_CID;
+	pdrvextra_cmd_parm->type = 0;
+	pdrvextra_cmd_parm->size = 0;
+	pdrvextra_cmd_parm->pbuf = NULL;
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+exit:
+	return res;
+}
+
+#ifdef CONFIG_P2P
+u8 rtw_hide_ssid_wk_cmd(_adapter *adapter)
+{
+	struct cmd_obj		*cmdobj;
+	struct drvextra_cmd_parm  *pdrvextra_cmd_parm;
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(adapter)->cmdpriv;
+	u8	res = _SUCCESS;
+
+	cmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	cmdobj->padapter = adapter;
+
+	pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (pdrvextra_cmd_parm == NULL) {
+		rtw_mfree((unsigned char *)cmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	pdrvextra_cmd_parm->ec_id = HIDE_SSID_WK_CID;
+	pdrvextra_cmd_parm->type = 0;
+	pdrvextra_cmd_parm->size = 0;
+	pdrvextra_cmd_parm->pbuf = NULL;
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmdobj);
+
+exit:
+	return res;
+
+}
+#endif
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csa.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csa.c
new file mode 100644
index 000000000000..99f622d3e628
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csa.c
@@ -0,0 +1,1272 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_CSA_C_
+#include <drv_types.h>
+
+#if CONFIG_DFS
+#ifdef CONFIG_ECSA_PHL
+void reset_all_ecsa_param(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct _ADAPTER *iface;
+	u8 i;
+
+	for (i = 0; i < d->iface_nums; i++) {
+		iface = d->padapters[i];
+		if (iface->ecsa_info.core_ecsa_type != ECSA_CORE_TYPE_NONE)
+			reset_ecsa_param(iface);
+	}
+}
+
+void reset_ecsa_param(struct _ADAPTER *a)
+{
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+
+	SET_ECSA_STATE(a, ECSA_ST_NONE);
+	ecsa_info->core_ecsa_type = ECSA_CORE_TYPE_NONE;
+	ecsa_info->ecsa_allow_case = 0xff;
+	ecsa_info->ecsa_delay_time = 0;
+	ecsa_info->channel_width = 255;
+	ecsa_info->bss_param = NULL;
+	ecsa_info->trigger_iface = NULL;
+	ecsa_info->execute_iface = NULL;
+	_rtw_memset(ecsa_param, 0, sizeof(struct rtw_phl_ecsa_param));
+}
+
+bool rtw_is_ecsa_enabled(struct _ADAPTER *a)
+{
+	return a->registrypriv.en_ecsa;
+}
+
+bool rtw_mr_is_ecsa_running(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct _ADAPTER *iface;
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	u8 i;
+
+	for (i = 0; i < d->iface_nums; i++) {
+		iface = d->padapters[i];
+		if (!iface)
+			continue;
+		if (!CHK_ECSA_STATE(iface, ECSA_ST_NONE))
+			return _TRUE;
+	}
+	return _FALSE;
+}
+
+bool rtw_hal_is_csa_support(struct _ADAPTER *a)
+{
+	/* AX series are all supported */
+	return true;
+}
+
+static void rtw_ecsa_update_sta_cur_network(struct _ADAPTER_LINK *alink, u8 *pframe, u32 packet_len)
+{
+	WLAN_BSSID_EX *cur_network = &(alink->mlmepriv.cur_network.network);
+
+	cur_network->IELength = packet_len - sizeof(struct rtw_ieee80211_hdr_3addr);
+	_rtw_memcpy(cur_network->IEs, (pframe + sizeof(struct rtw_ieee80211_hdr_3addr)), cur_network->IELength);
+}
+
+static void rtw_ecsa_parsing_bcn_ie(struct _ADAPTER_LINK *alink)
+{
+	struct _ADAPTER *a = alink->adapter;
+	WLAN_BSSID_EX *cur_network = &(alink->mlmepriv.cur_network.network);
+	u8 *bcn_ie = cur_network->IEs + _BEACON_IE_OFFSET_;
+	u32 bcn_len = cur_network->IELength - _BEACON_IE_OFFSET_;
+
+	rtw_ie_handler(a, alink, bcn_ie, bcn_len);
+}
+
+/* Reference from rtw_joinbss_update_stainfo() and update_sta_info() */
+static void rtw_ecsa_sta_update_stainfo(struct _ADAPTER_LINK *alink)
+{
+	struct _ADAPTER *a = alink->adapter;
+	struct sta_info *psta = NULL;
+	struct link_mlme_ext_priv *pmlmeext = &(alink->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct link_mlme_priv *pmlmepriv = &(alink->mlmepriv);
+	struct wlan_network *cur_network = &(alink->mlmepriv.cur_network);
+
+	psta = rtw_get_stainfo(&a->stapriv, get_link_bssid(&alink->mlmepriv));
+	if (!psta) {
+		RTW_ERR("CSA : %s sta_info is null\n", __func__);
+		return;
+	}
+
+	update_wireless_mode(a, alink);
+	psta->phl_sta->wmode = pmlmeext->cur_wireless_mode;
+
+	/* update supported rate */
+	psta->bssratelen = rtw_get_rateset_len(cur_network->network.SupportedRates);
+	_rtw_memcpy(psta->bssrateset, cur_network->network.SupportedRates, psta->bssratelen);
+
+#ifdef CONFIG_80211N_HT
+	if (pmlmeinfo->HT_caps_enable) {
+		_rtw_memcpy(psta->htpriv.ht_cap.supp_mcs_set,
+				pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, 16);
+
+		psta->htpriv.ht_option = true;
+		if (support_short_GI(a, alink, &(pmlmeinfo->HT_caps), CHANNEL_WIDTH_20))
+			psta->htpriv.sgi_20m = true;
+		else
+			psta->htpriv.sgi_20m = false;
+
+		if (support_short_GI(a, alink, &(pmlmeinfo->HT_caps), CHANNEL_WIDTH_40))
+			psta->htpriv.sgi_40m = true;
+		else
+			psta->htpriv.sgi_40m = false;
+	} else {
+		_rtw_memset(&(psta->htpriv), 0, sizeof(struct ht_priv));
+	}
+	psta->htpriv.ch_offset = pmlmeext->chandef.offset;
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	if (pmlmeinfo->VHT_enable) {
+		_rtw_memcpy(&psta->vhtpriv, &pmlmepriv->vhtpriv, sizeof(struct vht_priv));
+
+		psta->vhtpriv.vht_option = true;
+		#ifdef CONFIG_BEAMFORMING
+		psta->vhtpriv.beamform_cap = pmlmepriv->vhtpriv.beamform_cap;
+		#endif
+	} else {
+		_rtw_memset(&(psta->vhtpriv), 0, sizeof(struct vht_priv));
+	}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	if (pmlmeinfo->HE_enable)
+		_rtw_memcpy(&psta->hepriv, &pmlmepriv->hepriv, sizeof(struct he_priv));
+	else
+		_rtw_memset(&(psta->hepriv), 0, sizeof(struct he_priv));
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (psta->phl_sta->chandef.band == BAND_ON_6G) {
+		psta->ampdu_priv.ampdu_enable = pmlmepriv->ampdu_priv.ampdu_enable;
+		/* update mpdu_min_spacing in HE_6g_bandcap_handler */
+		psta->ampdu_priv.rx_ampdu_min_spacing = psta->hepriv.mpdu_min_spacing;
+	}
+#endif
+}
+
+void rtw_ecsa_update_sta_mlme(struct _ADAPTER_LINK *alink, u8 *pframe, u32 packet_len)
+{
+	struct link_mlme_priv *pmlmepriv = &(alink->mlmepriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(alink->mlmeextpriv.mlmext_info);
+	struct beacon_keys *cur_beacon = &(pmlmepriv->cur_beacon_keys);
+
+#ifdef CONFIG_80211N_HT
+	if (cur_beacon->proto_cap & PROTO_CAP_11N) {
+		pmlmepriv->htpriv.ht_option = _TRUE;
+		pmlmeinfo->HT_enable = _TRUE;
+	} else {
+		pmlmepriv->htpriv.ht_option = _FALSE;
+		pmlmeinfo->HT_enable = _FALSE;
+	}
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	if (cur_beacon->proto_cap & PROTO_CAP_11AC) {
+		pmlmepriv->vhtpriv.vht_option = _TRUE;
+		pmlmeinfo->VHT_enable = _TRUE;
+	} else {
+		pmlmepriv->vhtpriv.vht_option = _FALSE;
+		pmlmeinfo->VHT_enable = _FALSE;
+	}
+#endif
+
+	rtw_ecsa_update_sta_cur_network(alink, pframe, packet_len);
+
+	/* parse new beacon ie from current network to update protocol information of link_mlme_ext_info */
+	rtw_ecsa_parsing_bcn_ie(alink);
+
+	/* update sta_info from protocol information */
+	rtw_ecsa_sta_update_stainfo(alink);
+}
+
+void rtw_build_csa_ie(struct _ADAPTER *a, struct rtw_phl_ecsa_param *ecsa_param)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	WLAN_BSSID_EX *pnetwork = &(alink->mlmeextpriv.mlmext_info.network);
+	u8 csa_data[CSA_IE_LEN] = {0};
+
+	/*
+	* [0] : Channel Switch Mode
+	* [1] : New Channel Number
+	* [2] : Channel Switch Count
+	*/
+	csa_data[0] = ecsa_param->mode;
+	csa_data[1] = ecsa_param->new_chan_def.chan;
+	csa_data[2] = ecsa_param->count;
+	rtw_add_bcn_ie(a, pnetwork, WLAN_EID_CHANNEL_SWITCH, csa_data, CSA_IE_LEN);
+	RTW_INFO("CSA : build channel switch announcement IE by driver\n");
+	RTW_INFO("CSA : mode = %u, ch = %u, switch count = %u\n",
+			csa_data[0], csa_data[1], csa_data[2]);
+}
+
+void rtw_build_ecsa_ie(struct _ADAPTER *a, struct rtw_phl_ecsa_param *ecsa_param)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	WLAN_BSSID_EX *pnetwork = &(alink->mlmeextpriv.mlmext_info.network);
+	u8 ecsa_data[ECSA_IE_LEN] = {0};
+
+	/*
+	* [0] : Channel Switch Mode
+	* [1] : New Operating Class
+	* [2] : New Channel Number
+	* [3] : Channel Switch Count
+	*/
+	ecsa_data[0] = ecsa_param->mode;
+	ecsa_data[1] = ecsa_param->op_class;
+	ecsa_data[2] = ecsa_param->new_chan_def.chan;
+	ecsa_data[3] = ecsa_param->count;
+	rtw_add_bcn_ie(a, pnetwork, WLAN_EID_ECSA, ecsa_data, ECSA_IE_LEN);
+	RTW_INFO("CSA : build extended channel switch announcement IE by driver\n");
+	RTW_INFO("CSA : mode = %u, op_class = %u, ch = %u, switch count = %u\n",
+			ecsa_data[0], ecsa_data[1], ecsa_data[2], ecsa_data[3]);
+}
+
+void rtw_build_sec_offset_ie(struct _ADAPTER *a, u8 seconday_offset)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	WLAN_BSSID_EX *pnetwork = &(alink->mlmeextpriv.mlmext_info.network);
+
+	rtw_add_bcn_ie(a, pnetwork, WLAN_EID_SECONDARY_CHANNEL_OFFSET, &seconday_offset, 1);
+	RTW_INFO("CSA : build secondary channel offset IE by driver, sec_offset = %u\n", seconday_offset);
+}
+
+void rtw_build_wide_bw_cs_ie(struct _ADAPTER *a, struct rtw_chan_def new_chandef)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	WLAN_BSSID_EX *pnetwork = &(alink->mlmeextpriv.mlmext_info.network);
+	u8 csw_data[CS_WR_DATA_LEN] = {0};
+	u8 ch_width, seg_0;
+
+	switch (new_chandef.bw) {
+	case CHANNEL_WIDTH_40:
+		ch_width = 0;
+		break;
+	case CHANNEL_WIDTH_80:
+		ch_width = 1;
+		break;
+	default:
+		ch_width = 1;
+		break;
+	}
+
+	seg_0 = rtw_phl_get_center_ch(&new_chandef);
+
+	/*
+	* subfields of Wide Bandwidth Channel Switch subelement
+	* [1] : Length
+	* [2] : New Channel Width
+	* [3] : New Channel Center Frequency Segment 0
+	* [4] : New Channel Center Frequency Segment 1
+	*/
+	csw_data[0] = WLAN_EID_VHT_WIDE_BW_CHSWITCH;
+	csw_data[1] = 3;
+	csw_data[2] = ch_width;
+	csw_data[3] = seg_0;
+	csw_data[4] = 0;
+	rtw_add_bcn_ie(a, pnetwork, WLAN_EID_CHANNEL_SWITCH_WRAPPER, csw_data, CS_WR_DATA_LEN);
+	RTW_INFO("CSA : build channel switch wrapper IE by driver\n");
+	RTW_INFO("CSA : channel width = %u, segment_0 = %u, segment_1 = %u\n",
+				csw_data[2], csw_data[3], csw_data[4]);
+}
+
+void rtw_core_build_ecsa_beacon(struct _ADAPTER *a,
+			struct rtw_phl_ecsa_param *ecsa_param,
+			u8 is_vht)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	WLAN_BSSID_EX *pnetwork = &(alink->mlmeextpriv.mlmext_info.network);
+	struct rtw_chan_def new_chandef = ecsa_param->new_chan_def;
+	struct ieee80211_info_element *ie;
+	u8 *ies;
+	uint ies_len;
+	bool cs_add = false, ecs_add = false, sco_add = false, csw_add = false;
+
+	ies = pnetwork->IEs + _BEACON_IE_OFFSET_;
+	ies_len = pnetwork->IELength - _BEACON_IE_OFFSET_;
+
+	/* Check the missing IE of beacon, then build it by driver */
+	for_each_ie(ie, ies, ies_len) {
+		switch (ie->id) {
+		case WLAN_EID_CHANNEL_SWITCH:
+			#ifdef DBG_CSA
+			RTW_INFO("CSA : find CSA IE\n");
+			#endif
+			cs_add = true;
+			break;
+
+		case WLAN_EID_ECSA:
+			#ifdef DBG_CSA
+			RTW_INFO("CSA : find ECSA IE\n");
+			#endif
+			ecs_add = true;
+			break;
+
+		case WLAN_EID_SECONDARY_CHANNEL_OFFSET:
+			#ifdef DBG_CSA
+			RTW_INFO("CSA : find SEC_CH_OFFSET IE\n");
+			#endif
+			sco_add = true;
+			break;
+
+		case WLAN_EID_CHANNEL_SWITCH_WRAPPER:
+			#ifdef DBG_CSA
+			RTW_INFO("CSA : find CH_SW_WRAPPER IE\n");
+			#endif
+			csw_add = true;
+			break;
+
+		default:
+			break;
+		}
+	}
+
+	/* Build Channel Switch Announcement element */
+	if (!cs_add)
+		rtw_build_csa_ie(a, ecsa_param);
+
+	/* Build Extended Channel Switch Announcement element */
+	if (!ecs_add && rtw_is_ecsa_enabled(a))
+		rtw_build_ecsa_ie(a, ecsa_param);
+
+	/* Build Secondary Channel Offset element */
+	if (!sco_add && new_chandef.offset != CHAN_OFFSET_NO_EXT)
+		rtw_build_sec_offset_ie(a, new_chandef.offset);
+
+	/* Build Channel Switch Wrapper element which only include Wide Bandwidth Channel Switch subelement */
+	if (!csw_add && is_vht && (new_chandef.bw >= CHANNEL_WIDTH_40 && new_chandef.bw <= CHANNEL_WIDTH_80_80))
+		rtw_build_wide_bw_cs_ie(a, new_chandef);
+}
+
+void rtw_cfg80211_build_csa_beacon(struct _ADAPTER *a,
+	struct cfg80211_csa_settings *params, struct rtw_chan_def csa_chdef)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	WLAN_BSSID_EX *pnetwork = &(alink->mlmeextpriv.mlmext_info.network);
+	struct ieee80211_info_element *ie;
+	u8 *ies;
+	uint ies_len;
+
+	if (params->beacon_csa.tail) {
+		ies = (u8 *)params->beacon_csa.tail;
+		ies_len = params->beacon_csa.tail_len;
+		RTW_INFO("CSA : Parsing beacon content from cfg80211_csa_settings\n");
+
+		for_each_ie(ie, ies, ies_len) {
+			#ifdef DBG_CSA
+			RTW_INFO("CSA : for each IE, element id = %u, len = %u\n", ie->id, ie->len);
+			#endif
+			switch (ie->id) {
+			case WLAN_EID_CHANNEL_SWITCH:
+				RTW_INFO("CSA : add channel switch announcement IE to beacon\n");
+				RTW_INFO("CSA : mode = %u, ch = %u, switch count = %u\n",
+						ie->data[0], ie->data[1], ie->data[2]);
+				rtw_add_bcn_ie(a, pnetwork, ie->id, ie->data, ie->len);
+				break;
+
+			case WLAN_EID_ECSA:
+				RTW_INFO("CSA : add extended channel switch announcement IE to beacon\n");
+				RTW_INFO("CSA : mode = %u, op_class = %u, ch = %u, switch count = %u\n",
+						ie->data[0], ie->data[1], ie->data[2], ie->data[3]);
+				rtw_add_bcn_ie(a, pnetwork, ie->id, ie->data, ie->len);
+				break;
+
+			/* Secondary channel offset element is not necessary for channel switching to BW_20M */
+			case WLAN_EID_SECONDARY_CHANNEL_OFFSET:
+				RTW_INFO("CSA : add secondary channel offset IE to beacon, sec_offset = %u\n",
+						ie->data[0]);
+				rtw_add_bcn_ie(a, pnetwork, ie->id, ie->data, ie->len);
+				break;
+
+			case WLAN_EID_CHANNEL_SWITCH_WRAPPER:
+				RTW_INFO("CSA : add channel switch wrapper IE to beacon\n");
+				RTW_INFO("CSA : channel width = %u, segment_0 = %u, segment_1 = %u\n",
+						ie->data[2], ie->data[3], ie->data[4]);
+				rtw_add_bcn_ie(a, pnetwork, ie->id, ie->data, ie->len);
+				break;
+
+			default:
+				break;
+			}
+		}
+	}
+}
+
+static void rtw_ecsa_update_sta_chan_info(struct _ADAPTER *a,
+			struct _ADAPTER_LINK *alink,
+			struct rtw_chan_def new_chan_def)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_chset *chset = dvobj_to_chset(d);
+	u8 new_band = new_chan_def.band;
+	u8 new_ch = new_chan_def.chan;
+	u8 new_bw = (u8)new_chan_def.bw;
+	u8 new_offset = (u8)new_chan_def.offset;
+	bool is_chctx_add = _FALSE;
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+	struct link_mlme_ext_priv *pmlmeext = &alink->mlmeextpriv;
+	struct link_mlme_priv *pmlmepriv = &alink->mlmepriv;
+	u32 csa_wait_bcn_ms;
+
+	pmlmeext->chandef.band= new_band;
+	pmlmeext->chandef.chan= new_ch;
+	pmlmeext->chandef.bw = new_bw;
+	pmlmeext->chandef.offset = new_offset;
+	pmlmepriv->cur_network.network.Configuration.DSConfig = new_ch;
+
+	/* update wifi role link chandef */
+	rtw_hw_update_chan_def(a, alink);
+	/* update chanctx */
+	rtw_phl_chanctx_del(d->phl, a->phl_role, alink->wrlink, NULL);
+	is_chctx_add = rtw_phl_chanctx_add(d->phl, a->phl_role,
+						alink->wrlink,
+						&new_chan_def, &mr_cc_info);
+	if (is_chctx_add == _FALSE)
+		RTW_ERR("CSA : "FUNC_ADPT_FMT" chan_ctx add fail!", FUNC_ADPT_ARG(a));
+
+	set_fwstate(&a->mlmepriv, WIFI_CSA_UPDATE_BEACON);
+
+	#ifdef CONFIG_80211D
+	rtw_alink_csa_update_regulatory(alink, new_band, new_ch);
+	#endif
+
+	if (rtw_chset_is_dfs_bchbw(chset, new_band, new_ch, new_bw, new_offset))
+		csa_wait_bcn_ms = CAC_TIME_MS + 10000;
+	else
+		csa_wait_bcn_ms = WAIT_BCN_TIMES * 2 * 1000;
+
+	RTW_INFO("CSA : set csa_wait_bcn_timer to %u ms\n", csa_wait_bcn_ms);
+	_set_timer(&a->mlmeextpriv.csa_wait_bcn_timer, csa_wait_bcn_ms);
+
+	#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+	rtw_dfs_rd_en_dec_on_mlme_act(a, alink, MLME_OPCH_SWITCH, 0);
+	#endif
+}
+
+void rtw_ap_update_beacon_by_role(void *priv, struct rtw_wifi_role_t *role , struct rtw_wifi_role_link_t *rlink)
+{
+	/* TODO */
+}
+static void rtw_ecsa_update_ap_chan_info(struct _ADAPTER *a, struct rtw_chan_def new_chan_def)
+{
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct createbss_parm *parm;
+
+	ecsa_info->bss_param = (struct createbss_parm *)rtw_zmalloc(sizeof(struct createbss_parm));
+	if (ecsa_info->bss_param) {
+		parm = ecsa_info->bss_param;
+		parm->adhoc = 0;
+		parm->ifbmp = BIT(a->iface_id);
+		parm->excl_ifbmp = 0;
+		parm->req_band = new_chan_def.band;
+		parm->req_ch = new_chan_def.chan;
+		parm->req_bw = new_chan_def.bw;
+		parm->req_offset = new_chan_def.offset;
+		parm->ifbmp_ch_changed = 0;
+		parm->ch_to_set = 0;
+		parm->bw_to_set = 0;
+		parm->offset_to_set = 0;
+		parm->is_change_chbw = _TRUE;
+
+		rtw_core_ap_prepare(a, parm);
+	} else {
+		RTW_ERR("CSA : can't allocate memory for bss_param\n");
+	}
+}
+
+/**
+ * rtw_ecsa_update_sta_ap_chan_info - update STA role and AP role chandef
+ * Flow :
+ * del AP role chandef -> del STA role chandef -> add STA role chandef -> add AP role chandef
+ */
+static void rtw_ecsa_update_sta_ap_chan_info(struct _ADAPTER *ap_iface,
+			struct _ADAPTER_LINK *ap_iface_link,
+			struct rtw_chan_def ap_new_chan_def)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(ap_iface);
+	struct _ADAPTER *sta_iface = ap_iface->ecsa_info.trigger_iface;
+	struct _ADAPTER_LINK *sta_iface_link = GET_PRIMARY_LINK(sta_iface);
+	struct rtw_chan_def sta_new_chan_def = sta_iface->ecsa_info.phl_ecsa_param.new_chan_def;
+
+	rtw_phl_chanctx_del(d->phl, ap_iface->phl_role, ap_iface_link->wrlink, NULL);
+	ap_iface_link->is_ap_chan_ctx_added = false;
+
+	rtw_ecsa_update_sta_chan_info(sta_iface, sta_iface_link, sta_new_chan_def);
+
+	rtw_ecsa_update_ap_chan_info(ap_iface, ap_new_chan_def);
+}
+
+void rtw_ecsa_update_probe_resp(struct xmit_frame *xframe)
+{
+	struct _ADAPTER *a = xframe->padapter;
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	struct pkt_attrib *pattrib = &xframe->attrib;
+	u8 hdr_len = sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *ies;
+	sint ies_len;
+	u8 *csa_ie, *ecsa_ie;
+	sint csa_ie_len, ecsa_ie_len;
+
+	ies = xframe->buf_addr + TXDESC_OFFSET + hdr_len + _BEACON_IE_OFFSET_;
+	ies_len = pattrib->pktlen - hdr_len - _BEACON_IE_OFFSET_;
+
+	csa_ie = rtw_get_ie(ies, WLAN_EID_CHANNEL_SWITCH, &csa_ie_len, ies_len);
+
+	if (csa_ie == NULL)
+		return;
+
+	csa_ie[2 + CSA_SWITCH_COUNT] = ecsa_param->count;
+
+	#ifdef DBG_CSA
+	RTW_INFO("CSA : update csa count of probe response = %u\n",
+		csa_ie[2 + CSA_SWITCH_COUNT]);
+	#endif
+
+	if (rtw_is_ecsa_enabled(a)) {
+		ecsa_ie = rtw_get_ie(ies, WLAN_EID_ECSA, &ecsa_ie_len, ies_len);
+		if (ecsa_ie == NULL)
+			return;
+
+		/*
+		* [0] : Channel Switch Mode
+		* [1] : New Operating Class
+		* [2] : New Channel Number
+		* [3] : Channel Switch Count
+		*/
+		ecsa_ie[2 + 3] = ecsa_param->count;
+
+		#ifdef DBG_CSA
+		RTW_INFO("CSA : update ecsa count of probe response = %u\n",
+			ecsa_ie[2 + 3]);
+		#endif
+	}
+}
+
+static void rtw_check_phl_ecsa_request(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	u8 is_vht = alink->mlmepriv.vhtpriv.vht_option;
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	struct rtw_phl_ecsa_param *phl_ecsa_param;
+
+	/* ECSA request comes from PHL MR module, so we need to fill ecsa_param */
+	if (CHK_ECSA_STATE(a, ECSA_ST_SW_START) &&
+	   ecsa_info->core_ecsa_type == ECSA_CORE_TYPE_NONE) {
+		/* Get pointer of ecsa_parameter from PHL, then copy value to ecsa_parameter of core */
+		rtw_phl_ecsa_get_param(d->phl, &phl_ecsa_param);
+		_rtw_memcpy(ecsa_param, phl_ecsa_param, sizeof(struct rtw_phl_ecsa_param));
+		rtw_core_build_ecsa_beacon(a, ecsa_param, is_vht);
+	}
+}
+
+static bool _chk_csa_groupd(struct rtw_chan_def *chandef_a,
+	struct rtw_chan_def *chandef_b)
+{
+	if (chandef_a->band != chandef_b->band)
+		return _FALSE;
+
+	if ((chandef_a->bw == CHANNEL_WIDTH_40 || chandef_a->bw == CHANNEL_WIDTH_80) &&
+	   (chandef_b->bw == CHANNEL_WIDTH_40 || chandef_b->bw == CHANNEL_WIDTH_80)) {
+		if (chandef_a->offset != chandef_b->offset)
+			return _FALSE;
+	}
+	return _TRUE;
+}
+
+void rtw_ecsa_update_beacon(void *priv, struct rtw_wifi_role_t *role , struct rtw_wifi_role_link_t *rlink)
+{
+#ifdef CONFIG_AP_MODE
+	struct dvobj_priv *d = (struct dvobj_priv *)priv;
+	struct _ADAPTER *a = d->padapters[role->id];
+	struct _ADAPTER_LINK *alink = GET_LINK(a, rlink->id);
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	WLAN_BSSID_EX *pnetwork = &(alink->mlmeextpriv.mlmext_info.network);
+
+	rtw_check_phl_ecsa_request(a);
+
+	/* update CSA and ECSA IE at the same time */
+	_update_beacon(a, alink, WLAN_EID_CHANNEL_SWITCH, NULL, _TRUE, 0, "update CSA count");
+
+	if (ecsa_param->count == 0) {
+		rtw_remove_bcn_ie(a, pnetwork, WLAN_EID_SECONDARY_CHANNEL_OFFSET);
+		rtw_remove_bcn_ie(a, pnetwork, WLAN_EID_CHANNEL_SWITCH_WRAPPER);
+	}
+#endif
+}
+
+/* PHL MR module check core layer whether AP mode can switch channel now */
+bool rtw_ap_check_ecsa_allow(
+	void *priv,
+	struct rtw_wifi_role_t *role,
+	struct rtw_chan_def sta_chdef,
+	enum phl_ecsa_start_reason reason,
+#ifdef CONFIG_ECSA_EXTEND_OPTION
+	u32 *extend_option,
+#endif
+	u32 *delay_start_ms
+)
+{
+	struct dvobj_priv *d = (struct dvobj_priv *)priv;
+	struct _ADAPTER *a = d->padapters[role->id];
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	struct rtw_chan_def ap_chdef = alink->wrlink->chandef;
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(d);
+	RT_CHANNEL_INFO *chinfo = rtw_chset_get_chinfo_by_bch(&rfctl->chset, sta_chdef.band, sta_chdef.chan, 0);
+	bool ecsa_allow = _TRUE;
+	u8 i;
+
+#ifdef CONFIG_MCC_MODE
+	if (rtw_hw_mcc_chk_inprogress(a, alink)) {
+		RTW_INFO("CSA : "FUNC_ADPT_FMT" : Don't switch channel if MCC enabled\n",
+			FUNC_ADPT_ARG(a));
+		ecsa_allow = _FALSE;
+		goto exit;
+	}
+#endif
+
+	if (!(ecsa_info->ecsa_allow_case & BIT(reason))) {
+		RTW_INFO("CSA : "FUNC_ADPT_FMT" : Case %u not support!\n", FUNC_ADPT_ARG(a), reason);
+		ecsa_allow = _FALSE;
+		goto exit;
+	}
+
+	/* Check DFS channel */
+	if (!chinfo || (chinfo->flags & (RTW_CHF_NO_IR | RTW_CHF_DFS))) {
+		RTW_ERR("CSA : "FUNC_ADPT_FMT" : DFS channel (%u) not support!\n",
+				FUNC_ADPT_ARG(a), sta_chdef.chan);
+		ecsa_allow = _FALSE;
+		goto exit;
+	}
+
+	if (ecsa_info->ecsa_delay_time != 0)
+		*delay_start_ms = ecsa_info->ecsa_delay_time;
+	else if (reason <= ECSA_START_MCC_5G_TO_24G)
+		*delay_start_ms = MCC_ECSA_DELAY_START_TIME;
+	RTW_INFO("CSA : "FUNC_ADPT_FMT" ECSA will delay %u ms to start\n", FUNC_ADPT_ARG(a), *delay_start_ms);
+
+	SET_ECSA_STATE(a, ECSA_ST_SW_START);
+
+	/*
+	* AP mode use the its bandwidth/offset to switch channel if its bandwidth/offset can group with STA mode.
+	* Otherwise, AP mode use chandef of STA mode.
+	*/
+	if (_chk_csa_groupd(&ap_chdef, &sta_chdef)) {
+		#ifdef CONFIG_ECSA_EXTEND_OPTION
+		*extend_option = ECSA_EX_OPTION_USE_AP_CHANDEF;
+		#endif
+		RTW_INFO("CSA : AP mode use its bandwidth/offset to switch channel\n");
+	} else {
+		RTW_INFO("CSA : bandwidth/offset of AP mode can't group with STA mode, so use bandwidth/offset of STA mode\n");
+	}
+
+	RTW_INFO("CSA : Switch AP to STA's channel. AP:%u,%u,%u  STA:%u,%u,%u\n",
+		ap_chdef.chan, ap_chdef.bw, ap_chdef.offset,
+		sta_chdef.chan, sta_chdef.bw, sta_chdef.offset);
+exit:
+	return ecsa_allow;
+}
+
+void rtw_ecsa_mr_update_chan_info_by_role(
+	void *priv,
+	struct rtw_wifi_role_t *role,
+	struct rtw_wifi_role_link_t *rlink,
+	struct rtw_chan_def new_chan_def)
+{
+	struct dvobj_priv *d = (struct dvobj_priv *)priv;
+	struct _ADAPTER *a = d->padapters[role->id];
+	struct _ADAPTER_LINK *alink = GET_LINK(a, rlink->id);
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+
+	RTW_INFO("CSA : "FUNC_ADPT_FMT", new ch/bw/offset = %u,%u,%u\n",
+		FUNC_ADPT_ARG(a), new_chan_def.chan, new_chan_def.bw,
+		new_chan_def.offset);
+
+	if (ecsa_info->core_ecsa_type == ECSA_CORE_TYPE_SCC_STA_AP &&
+	   (role->type == PHL_RTYPE_AP || role->type == PHL_RTYPE_P2P_GO)) {
+		/* multi-role STA + AP in SCC */
+		rtw_ecsa_update_sta_ap_chan_info(a, alink, new_chan_def);
+	} else {
+		 /* single-role or multi-role in MCC or DBCC */
+		if (role->type == PHL_RTYPE_STATION || role->type == PHL_RTYPE_P2P_GC)
+			rtw_ecsa_update_sta_chan_info(a, alink, new_chan_def);
+		else if (role->type == PHL_RTYPE_AP ||role->type == PHL_RTYPE_P2P_GO)
+			rtw_ecsa_update_ap_chan_info(a, new_chan_def);
+	}
+}
+
+bool rtw_ecsa_check_tx_resume_allow(void *priv, struct rtw_wifi_role_t *role)
+{
+	/* TODO */
+	/* Is DFS slave still monitoring channel ?
+	If Yes, return False to PHL; If no, return True to PHL */
+	struct dvobj_priv *d = (struct dvobj_priv *)priv;
+	struct _ADAPTER *a = d->padapters[role->id];
+
+	RTW_INFO("CSA : "FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(a));
+	return 1;
+}
+
+void rtw_ecsa_complete(void *priv, struct rtw_wifi_role_t *role)
+{
+	struct dvobj_priv *d = (struct dvobj_priv *)priv;
+	struct _ADAPTER *a = d->padapters[role->id];
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(d);
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct createbss_parm *parm;
+	u8 ht_option = 0;
+
+	RTW_INFO("CSA : "FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(a));
+	rtw_phl_mr_dump_cur_chandef(d->phl, role);
+
+	 /* single-rol or multi-role in MCC or DBCC */
+	if (role->type == PHL_RTYPE_STATION || role->type == PHL_RTYPE_P2P_GC) {
+		/*
+		* TODO
+		* STA mode need to update RA if it receive CHANNEL_SWITCH_WRAPPER IE
+		* STA mode update its RA at rtw_check_bcn_info() now
+		*/
+		rtw_rfctl_update_op_mode(rfctl, 0, 0, 0);
+
+		clr_fwstate(&(a->mlmepriv), WIFI_CSA_SKIP_CHECK_BEACON);
+	} else if (role->type == PHL_RTYPE_AP ||role->type == PHL_RTYPE_P2P_GO) {
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+		#ifdef CONFIG_80211N_HT
+		ht_option = alink->mlmepriv.htpriv.ht_option;
+		#endif
+		rtw_cfg80211_ch_switch_notify(a,
+			GET_PRIMARY_LINK(a),
+			&alink->mlmeextpriv.chandef,
+			ht_option, 0);
+		#endif
+
+		parm = ecsa_info->bss_param;
+		rtw_rfctl_update_op_mode(adapter_to_rfctl(a),
+			parm->ifbmp, 1, parm->excl_ifbmp);
+		rtw_core_ap_start(a, parm);
+		rtw_ap_update_clients_rainfo(a, 0);
+		rtw_mfree((u8 *)parm, sizeof(struct createbss_parm));
+	}
+
+	rtw_mi_os_xmit_schedule(a);
+	reset_all_ecsa_param(a);
+}
+
+static void rtw_ap_send_csa_action_frame(struct _ADAPTER *a, struct _ADAPTER_LINK *alink)
+{
+	_list	*phead, *plist;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &(a->stapriv);
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+
+	/* for each sta in asoc_queue */
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+
+		issue_action_spct_ch_switch(a, alink, psta->phl_sta->mac_addr);
+	}
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	issue_action_spct_ch_switch(a, alink, bc_addr);
+}
+
+/* Get ch/bw/offset of CSA from adapter, and check these parameters is valid or not */
+static bool rtw_sta_get_ecsa_setting(struct _ADAPTER *a)
+{
+	struct rtw_chset *chset = adapter_to_chset(a);
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	struct rtw_chan_def chan_def;
+	u8 csa_band = ecsa_param->new_chan_def.band;
+	u8 csa_chan = ecsa_param->new_chan_def.chan;
+	u8 csa_offset = ecsa_param->new_chan_def.offset;
+	u8 csa_op_class = ecsa_param->op_class;
+	bool get_bw_offset_from_opclass;
+	u8 req_band, req_ch, req_bw, req_offset;
+	u8 i, r_offset[2], r_offset_num;
+	bool valid_offset = false;
+
+	req_band = csa_band;
+	req_ch = csa_chan;
+	req_bw = CHANNEL_WIDTH_20;
+	req_offset = CHAN_OFFSET_NO_EXT;
+
+	if (rtw_chset_search_bch(chset, csa_band, csa_chan) < 0
+		|| rtw_chset_is_bch_non_ocp(chset, csa_band, csa_chan)) {
+		RTW_INFO("CSA : channel %u is not supported, so do not switching channel\n", csa_chan);
+		return false;
+	}
+
+	get_bw_offset_from_opclass = rtw_phl_get_chandef_from_operating_class(req_ch,
+			csa_op_class, &chan_def);
+
+	if (get_bw_offset_from_opclass) {
+		req_band = chan_def.band;
+		req_bw = chan_def.bw;
+		req_offset = chan_def.offset;
+	} else {
+		/* Transform channel_width to bandwidth 20/40/80M */
+		switch (ecsa_info->channel_width) {
+		case CH_WIDTH_80_160M:
+			req_bw = CHANNEL_WIDTH_80;
+			req_offset = csa_offset;
+			break;
+		case CH_WIDTH_20_40M:
+			/*
+			* We don't know the actual offset of channel 5 to 9
+			* if offset is CHAN_OFFSET_NO_EXT and bandwidth is 40MHz,
+			* so force its bandwidth to 20MHz
+			*/
+			if ((csa_band == BAND_ON_24G && req_ch >= 5 && req_ch <=9) &&
+			   csa_offset == CHAN_OFFSET_NO_EXT)
+				req_bw = CHANNEL_WIDTH_20;
+			else
+				req_bw = CHANNEL_WIDTH_40;
+			req_offset = csa_offset;
+			break;
+		default:
+			req_bw = CHANNEL_WIDTH_20;
+			req_offset = CHAN_OFFSET_NO_EXT;
+			break;
+		}
+	}
+
+	/* Get correct offset and check ch/bw/offset is valid or not */
+	if (req_bw > CHANNEL_WIDTH_20 &&
+	  (req_offset == CHAN_OFFSET_NO_EXT || req_offset == CHAN_OFFSET_NO_DEF)) {
+		if (rtw_get_offsets_by_bchbw(req_band, req_ch, req_bw, r_offset, &r_offset_num)) {
+			for (i = 0; i < r_offset_num; i++) {
+				req_offset = r_offset[i];
+				/* choose first valid offset */
+				if (rtw_chset_is_bchbw_valid(chset, req_band, req_ch, req_bw, req_offset, 0, 0)) {
+					valid_offset = true;
+					break;
+				}
+			}
+		}
+
+		if (!valid_offset) {
+			req_bw = CHANNEL_WIDTH_20;
+			req_offset = CHAN_OFFSET_NO_EXT;
+		}
+	}
+
+	ecsa_param->new_chan_def.band = req_band;
+	ecsa_param->new_chan_def.chan = req_ch;
+	ecsa_param->new_chan_def.bw = req_bw;
+	ecsa_param->new_chan_def.offset = req_offset;
+
+	/* bw/offset is limited by SW capability */
+	if (rtw_adjust_bchbw(a, ecsa_param->new_chan_def.band,
+		ecsa_param->new_chan_def.chan,
+		(u8 *)&ecsa_param->new_chan_def.bw,
+		(u8 *)&ecsa_param->new_chan_def.offset)) {
+			RTW_INFO("CSA : "FUNC_ADPT_FMT" limit by cap bw (%u) to (%u)\n",
+				FUNC_ADPT_ARG(a), req_bw, ecsa_param->new_chan_def.bw);
+	}
+
+	return true;
+}
+
+static void rtw_sta_ecsa_invalid_hdl(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(d);
+	struct mlme_ext_priv *pmlmeext = &a->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	u8 ifbmp_s = rtw_mi_get_ld_sta_ifbmp(a);
+	struct rtw_chan_def mr_chdef = {0};
+
+	if (!ifbmp_s)
+		return;
+
+	set_fwstate(&a->mlmepriv,  WIFI_OP_CH_SWITCHING);
+	issue_deauth(a, get_bssid(&a->mlmepriv), WLAN_REASON_DEAUTH_LEAVING);
+
+	/* TODO : DFS slave may need to switch channel as soon as possible before disconnect */
+
+	/* This context can't I/O, so use RTW_CMDF_DIRECTLY */
+	rtw_disassoc_cmd(a, 0, RTW_CMDF_DIRECTLY);
+	rtw_indicate_disconnect(a, 0, _FALSE);
+	rtw_free_network_queue(a, _TRUE);
+	rtw_free_mld_network_queue(a, _TRUE);
+	RTW_INFO("CSA : "FUNC_ADPT_FMT" disconnect with AP\n", FUNC_ADPT_ARG(a));
+
+	pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+	pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_RECEIVE_INVALID_CSA;
+	pmlmeinfo->wifi_reason_code = WLAN_REASON_DEAUTH_LEAVING;
+
+	rtw_mi_os_xmit_schedule(a);
+
+	reset_ecsa_param(a);
+}
+
+static enum ecsa_mr_case rtw_ecsa_mr_check(struct _ADAPTER *execute_iface)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(execute_iface);
+	struct rtw_wifi_role_t *role = execute_iface->phl_role;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(execute_iface);
+	struct rtw_wifi_role_link_t *rlink = alink->wrlink;
+	struct core_ecsa_info *ecsa_info = &(execute_iface->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	struct rtw_chan_def n_chdef;
+	struct mi_state mstate = {0};
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+
+	n_chdef = ecsa_param->new_chan_def;
+	rtw_mi_status(execute_iface, &mstate);
+
+	/* Check whether channel, bandwidth, offset can group under multi-role */
+	if ((MSTATE_STA_LD_NUM(&mstate) +
+	   MSTATE_AP_NUM(&mstate) +
+	   MSTATE_MESH_NUM(&mstate) >= 2) &&
+	   !rtw_phl_chanctx_chk(GET_PHL_INFO(d), role, rlink, &n_chdef, &mr_cc_info)) {
+		#ifdef CONFIG_DBCC_SUPPORT
+		if (mr_cc_info.sugg_opmode == MR_OP_DBCC) {
+			rtw_phl_mr_trig_dbcc_enable(GET_PHL_INFO(d));
+		} else
+		#endif
+		{
+			/* chandef of multi-role can't group */
+			RTW_INFO("CSA : "FUNC_ADPT_FMT ": "
+				"chandef can't group, use SCC mode to switch channel! "
+				"ld_sta_num:%u, ap_num:%u, mesh_num:%u\n",
+				FUNC_ADPT_ARG(execute_iface), MSTATE_STA_LD_NUM(&mstate),
+				MSTATE_AP_NUM(&mstate), MSTATE_MESH_NUM(&mstate));
+
+			/* Support ECSA of one STA + one AP/GO */
+			if (MSTATE_STA_LD_NUM(&mstate) == 1 &&
+			   (MSTATE_AP_NUM(&mstate) + MSTATE_MESH_NUM(&mstate) == 1))
+				return ECSA_MR_SCC_STA_AP;
+			else
+				return ECSA_MR_UNSUPPORTED;
+		}
+	}
+
+	return ECSA_SINGLE_ROLE;
+}
+
+static struct _ADAPTER *rtw_scc_sta_ap_init_ap_role(struct _ADAPTER *sta_iface)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(sta_iface);
+	struct core_ecsa_info *sta_ecsa_info = &(sta_iface->ecsa_info);
+	struct rtw_phl_ecsa_param *sta_ecsa_param = &(sta_ecsa_info->phl_ecsa_param);
+	struct core_ecsa_info *ap_ecsa_info;
+	struct rtw_phl_ecsa_param *ap_ecsa_param;
+	struct _ADAPTER *ap_iface = NULL;
+	struct _ADAPTER_LINK *alink;
+	struct link_mlme_ext_priv *lmlmeext;
+	struct rtw_chan_def *ap_new_chan_def;
+	u8 is_vht;
+	u8 i;
+
+	sta_ecsa_info->trigger_iface = sta_iface;
+	for (i = 0; i < d->iface_nums; i++) {
+		ap_iface = d->padapters[i];
+
+		if (!ap_iface || ap_iface == sta_iface || !rtw_is_adapter_up(ap_iface))
+			continue;
+
+		if ((MLME_IS_AP(ap_iface) || MLME_IS_MESH(ap_iface))
+			&& check_fwstate(&ap_iface->mlmepriv, WIFI_ASOC_STATE)) {
+
+			sta_ecsa_info->execute_iface = ap_iface;
+			alink = GET_PRIMARY_LINK(ap_iface);
+			lmlmeext = &(alink->mlmeextpriv);
+			ap_ecsa_info = &(ap_iface->ecsa_info);
+			ap_ecsa_param = &(ap_ecsa_info->phl_ecsa_param);
+
+			SET_ECSA_STATE(ap_iface, ECSA_ST_SW_START);
+			ap_ecsa_info->core_ecsa_type = ECSA_CORE_TYPE_SCC_STA_AP;
+			ap_ecsa_info->trigger_iface = sta_iface;
+			ap_ecsa_info->execute_iface = ap_iface;
+			_rtw_memcpy(ap_ecsa_param, sta_ecsa_param, sizeof(struct rtw_phl_ecsa_param));
+
+			ap_new_chan_def = &(ap_ecsa_param->new_chan_def);
+			ap_new_chan_def->bw = lmlmeext->chandef.bw;
+			ap_new_chan_def->offset = lmlmeext->chandef.offset;
+
+			/* adjust bw according registery and hw cap*/
+			rtw_adjust_bchbw(ap_iface, ap_new_chan_def->band,
+						ap_new_chan_def->chan,
+						(u8 *)&ap_new_chan_def->bw,
+						(u8 *)&ap_new_chan_def->offset);
+
+			/* adjust bw and offset according chanctx to avoid wrong offset */
+			rtw_phl_adjust_chandef(d->phl, alink->wrlink, ap_new_chan_def);
+
+			is_vht = alink->mlmepriv.vhtpriv.vht_option;
+			rtw_core_build_ecsa_beacon(ap_iface, ap_ecsa_param, is_vht);
+			break;
+		}
+	}
+
+	return ap_iface;
+}
+
+void rtw_fill_phl_ecsa_type(struct core_ecsa_info *ecsa_info)
+{
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+
+	switch (ecsa_info->core_ecsa_type) {
+	case ECSA_CORE_TYPE_NONE:
+		ecsa_param->ecsa_type = ECSA_TYPE_NONE;
+		break;
+	case ECSA_CORE_TYPE_AP:
+		ecsa_param->ecsa_type = ECSA_TYPE_AP;
+		break;
+	case ECSA_CORE_TYPE_STA:
+		ecsa_param->ecsa_type = ECSA_TYPE_STA;
+		break;
+	case ECSA_CORE_TYPE_SCC_STA_AP:
+		ecsa_param->ecsa_type = ECSA_TYPE_AP;
+		break;
+	}
+}
+
+/**
+ * @trigger_iface : interface which trigger core layer ECSA flow
+ * @execute_iface : interface which actually execute PHL ECSA FG
+ */
+bool rtw_trigger_phl_ecsa_start(struct _ADAPTER *trigger_iface,
+	enum csa_trigger_type trigger_type)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(trigger_iface);
+	struct _ADAPTER *execute_iface = trigger_iface;
+	struct rtw_wifi_role_t *role = execute_iface->phl_role;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(execute_iface);
+	struct rtw_wifi_role_link_t *rlink = alink->wrlink;
+	struct rtw_chan_def c_chdef = alink->mlmeextpriv.chandef;
+	struct core_ecsa_info *ecsa_info = &(execute_iface->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+	struct rtw_chan_def n_chdef;
+	enum ecsa_mr_case mr_case;
+	u8 is_vht = alink->mlmepriv.vhtpriv.vht_option;
+
+	switch (trigger_type) {
+	case CSA_AP_CORE_SWITCH_CH:
+		ecsa_info->core_ecsa_type = ECSA_CORE_TYPE_AP;
+
+		/* Build CSA beacon by self */
+		rtw_core_build_ecsa_beacon(trigger_iface, ecsa_param, is_vht);
+		break;
+
+	case CSA_AP_CFG80211_SWITCH_CH:
+		ecsa_info->core_ecsa_type = ECSA_CORE_TYPE_AP;
+
+		/* Build CSA beacon by self */
+		rtw_core_build_ecsa_beacon(trigger_iface, ecsa_param, is_vht);
+		break;
+
+	case CSA_STA_RX_CSA_IE:
+		if (!rtw_sta_get_ecsa_setting(execute_iface)) {
+			/* we should handle error case by core layer self */
+			rtw_sta_ecsa_invalid_hdl(execute_iface);
+			return false;
+		}
+
+		mr_case = rtw_ecsa_mr_check(execute_iface);
+		switch (mr_case) {
+		case ECSA_SINGLE_ROLE:
+			ecsa_info->core_ecsa_type = ECSA_CORE_TYPE_STA;
+			break;
+
+		case ECSA_MR_SCC_STA_AP:
+			ecsa_info->core_ecsa_type = ECSA_CORE_TYPE_SCC_STA_AP;
+
+			/* Use AP interface to execute ECSA FG */
+			execute_iface = rtw_scc_sta_ap_init_ap_role(trigger_iface);
+			if (execute_iface == NULL)
+				goto err_hdl;
+			break;
+
+		case ECSA_MR_UNSUPPORTED:
+			RTW_ERR("CSA : only support ECSA of one STA + one AP/GO for now\n");
+			goto err_hdl;
+			break;
+		}
+
+		set_fwstate(&execute_iface->mlmepriv, WIFI_CSA_SKIP_CHECK_BEACON);
+		break;
+
+	case CSA_STA_DISCONNECT_ON_DFS:
+		ecsa_info->core_ecsa_type = ECSA_CORE_TYPE_AP;
+
+		/* Build CSA beacon by self */
+		rtw_core_build_ecsa_beacon(trigger_iface, ecsa_param, is_vht);
+
+		/* Send CSA action frame to clients */
+		rtw_ap_send_csa_action_frame(trigger_iface, alink);
+		break;
+
+	default:
+		goto err_hdl;
+	}
+
+	n_chdef = ecsa_param->new_chan_def;
+	RTW_INFO("CSA : "FUNC_ADPT_FMT" core_ecsa_type = %u\n",
+		FUNC_ADPT_ARG(trigger_iface), ecsa_info->core_ecsa_type);
+	RTW_INFO("CSA : Trigger interface("ADPT_FMT")"
+			"current:%u,%u,%u,%u ==> new:%u,%u,%u,%u\n",
+		ADPT_ARG(trigger_iface),
+		c_chdef.band, c_chdef.chan, c_chdef.bw, c_chdef.offset,
+		n_chdef.band, n_chdef.chan, n_chdef.bw, n_chdef.offset);
+
+	/**
+	  * Update all parameters to parameters of interface which execute ECSA FG
+	  * if execute_iface doesn't equal to trigger_iface.
+	  */
+	if (execute_iface != trigger_iface) {
+		role = execute_iface->phl_role;
+		alink = GET_PRIMARY_LINK(execute_iface);
+		rlink = alink->wrlink;
+		ecsa_info = &(execute_iface->ecsa_info);
+		ecsa_param = &(ecsa_info->phl_ecsa_param);
+
+		c_chdef = alink->mlmeextpriv.chandef;
+		n_chdef = ecsa_param->new_chan_def;
+		RTW_INFO("CSA : Execute interface("ADPT_FMT")"
+				"current:%u,%u,%u,%u ==> new:%u,%u,%u,%u\n",
+			ADPT_ARG(execute_iface),
+			c_chdef.band, c_chdef.chan, c_chdef.bw, c_chdef.offset,
+			n_chdef.band, n_chdef.chan, n_chdef.bw, n_chdef.offset);
+	}
+
+	/* Only execute interface needs to fill PHL ECSA type */
+	rtw_fill_phl_ecsa_type(ecsa_info);
+	if (rtw_phl_ecsa_start(GET_PHL_INFO(d), role, rlink, ecsa_param) != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("CSA : "FUNC_ADPT_FMT" Start PHL ECSA fail\n", FUNC_ADPT_ARG(execute_iface));
+		goto err_hdl;
+	}
+
+	return true;
+
+err_hdl:
+	reset_all_ecsa_param(execute_iface);
+	return false;
+}
+
+/*
+ * @csa_ch_width gets from Channel Switch Wrapper IE
+ * 0 for 40 MHz
+ * 1 for 80 MHz, 160 MHz or 80+80 MHz
+ * 255 for initial value, defined by driver
+ *
+ * @ecsa_op_class gets from ECSA IE
+ */
+bool rtw_hal_trigger_csa_start(struct _ADAPTER *a, enum csa_trigger_type trigger_type,
+	u8 csa_mode, u8 ecsa_op_class, u8 csa_switch_cnt,
+	u8 csa_band, u8 csa_ch, u8 csa_bw, u8 csa_offset,
+	u8 csa_ch_width, u8 csa_ch_freq_seg0, u8 csa_ch_freq_seg1)
+{
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	struct rtw_phl_ecsa_param *ecsa_param = &(ecsa_info->phl_ecsa_param);
+
+	RTW_INFO("CSA : "FUNC_ADPT_FMT" csa trigger type = %u\n", FUNC_ADPT_ARG(a), trigger_type);
+	SET_ECSA_STATE(a, ECSA_ST_SW_START);
+	ecsa_param->mode = csa_mode;
+	ecsa_param->op_class = ecsa_op_class;
+	ecsa_param->count = csa_switch_cnt;
+	ecsa_param->new_chan_def.band = csa_band;
+	ecsa_param->new_chan_def.chan = csa_ch;
+	ecsa_param->new_chan_def.bw = csa_bw;
+	ecsa_param->new_chan_def.offset = csa_offset;
+	/* The channel width defined in 802.11 spec */
+	ecsa_info->channel_width = csa_ch_width;
+	ecsa_param->new_chan_def.center_freq1 = csa_ch_freq_seg0;
+	ecsa_param->new_chan_def.center_freq2 = csa_ch_freq_seg1;
+	ecsa_param->flag = 0;
+	ecsa_param->delay_start_ms = 0;
+
+	return rtw_trigger_phl_ecsa_start(a, trigger_type);
+}
+
+bool rtw_hal_dfs_trigger_csa(struct _ADAPTER *a,
+	enum csa_trigger_type trigger_type,
+	u8 band_idx, u8 u_ch, u8 u_bw, u8 u_offset)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(a);
+	enum band_type band;
+	u8 ch, bw, offset;
+	bool ch_avail, ret;
+	struct rtw_chan_def new_chandef = {0};
+	u8 csa_mode, ecsa_op_class, csa_switch_cnt;
+	u8 csa_ch_width = 0, csa_ch_freq_seg0 = 0, csa_ch_freq_seg1 = 0;
+
+	#ifndef PRIVATE_R
+	/* select new chan_def */
+	ch_avail = rtw_rfctl_choose_bchbw(rfctl, BAND_MAX, 0, u_bw, 0,
+				BAND_ON_5G, u_ch, u_offset,
+				&band, &ch, &bw, &offset,
+				false, false, __func__);
+	if (!ch_avail)
+		return false;
+	#else
+	/* only select CH36 or CH149 */
+	band = BAND_ON_5G;
+	ch = rfctl->p2p_park_ch;
+	bw = CHANNEL_WIDTH_20;
+	offset = CHAN_OFFSET_NO_EXT;
+	#endif
+
+	RTW_INFO(FUNC_HWBAND_FMT" CSA to %s-%u,%u,%u\n",
+		FUNC_HWBAND_ARG(band_idx), band_str(band), ch, bw, offset);
+
+	csa_mode = 1;
+	new_chandef.band = band;
+	new_chandef.chan = ch;
+	new_chandef.bw = bw;
+	new_chandef.offset = offset;
+	ecsa_op_class = rtw_phl_get_operating_class(new_chandef);
+	csa_switch_cnt = 2;
+
+	ret = rtw_hal_trigger_csa_start(a, CSA_STA_DISCONNECT_ON_DFS,
+		csa_mode, ecsa_op_class, csa_switch_cnt,
+		band, ch, bw, offset,
+		csa_ch_width, csa_ch_freq_seg0, csa_ch_freq_seg1);
+
+	return ret;
+}
+#endif /* CONFIG_ECSA_PHL */
+#endif /* CONFIG_DFS */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csi.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csi.c
new file mode 100644
index 000000000000..0ef6f5d7c8a4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csi.c
@@ -0,0 +1,320 @@
+
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#define _RTW_CSI_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_CSI_CHANNEL_INFO
+
+#ifdef CONFIG_RTW_CSI_NETLINK
+#include <linux/module.h>
+#include <net/sock.h>
+#include <linux/netlink.h>
+#include <linux/skbuff.h>
+
+#define NETLINK_CSI_FAMILY 19
+
+static void csi_nl_recv_msg(struct sk_buff *skb)
+{
+	struct nlmsghdr *nlh;
+	int pid;
+	struct sk_buff *skb_out;
+	int msg_size;
+	char *msg = "Hello from kernel";
+	int res;
+
+	RTW_INFO("Entering: %s\n", __FUNCTION__);
+
+	msg_size = strlen(msg);
+
+	nlh = (struct nlmsghdr *)skb->data;
+	RTW_INFO("Kernel: Netlink received msg payload:%s\n", (char *)nlmsg_data(nlh));
+#if 1
+	pid = nlh->nlmsg_pid; /*pid of sending process */
+#else
+	usr_pid = nlh->nlmsg_pid; /*pid of sending process */
+#endif
+
+#if 0//no send back
+	skb_out = nlmsg_new(msg_size, 0);
+	if (!skb_out) {
+		RTW_ERR("Failed to allocate new skb\n");
+		return;
+	}
+
+	nlh = nlmsg_put(skb_out, 0, 0, NLMSG_DONE, msg_size, 0);
+	NETLINK_CB(skb_out).dst_group = 0; /* not in mcast group */
+	strncpy(nlmsg_data(nlh), msg, msg_size);
+
+	res = nlmsg_unicast(nl_sk, skb_out, pid);
+	if (res < 0)
+		RTW_ERR("Error while sending bak to user\n");
+#endif
+}
+
+int rtw_csi_nl_init(struct dvobj_priv *dvobj)
+{
+	struct sock *nl_sk;
+	struct netlink_kernel_cfg cfg;
+
+	RTW_INFO("Entering: %s\n", __func__);
+
+	nl_sk = NULL;
+	dvobj->csi_nl_sk = NULL;
+	cfg.input = csi_nl_recv_msg;
+
+	nl_sk = netlink_kernel_create(&init_net, NETLINK_CSI_FAMILY, &cfg);
+	if (!nl_sk) {
+		RTW_ERR("%s: Error creating socket.\n", __func__);
+		return -10;
+	}
+	dvobj->csi_nl_sk = nl_sk;
+
+	return 0;
+}
+
+void rtw_csi_nl_exit(struct dvobj_priv *dvobj)
+{
+	RTW_INFO("Exiting: %s\n", __func__);
+	netlink_kernel_release(dvobj->csi_nl_sk);
+	dvobj->csi_nl_sk = NULL;
+}
+
+#endif /*CONFIG_RTW_CSI_NETLINK*/
+
+#ifdef CONFIG_CSI_TIMER_POLLING
+
+/******CSI header for OWI******/
+
+static int ow_pid = 123;
+
+#define IEEE80211_ADDR_LEN  6
+
+struct ow_csi_netlink_header {
+    u8    phynetdevidx;
+    u32   sequence_number;
+    u16   fragment_index;
+} __attribute__ ((__packed__));
+
+/*
+  src_macaddr         received frame source MAC address
+  csi_capture_status  1 - CSI capture successful; 0 - CSI capture not successful
+  csi_timestamp       received frame TSF timestamp
+  csi_nc	      Nrx
+  csi_nr	      Nsts
+  csi_bw	      received frame bandwidth
+  csi_rx_rate	      received frame date rate
+  csi_dump_length     Length of the CSI dump, it excludes the header size
+*/
+struct ow_csi_frame_header {
+	u32 rsvd1[4];
+	u8 src_macaddr[IEEE80211_ADDR_LEN];
+	u8 csi_capture_status;
+	u8 rsvd2[13];
+	u32 csi_timestamp;
+	u8 csi_nc;
+	u8 csi_nr;
+	u8 csi_bw;
+	u16 csi_rx_rate;
+	u8 csi_rssi[2];
+	u8 csi_subc_num;
+	u8 csi_tone_bit;
+	u32 csi_data_length;
+};
+/******************************/
+
+
+#define CSI_MSG_LEN 1000
+#define CSI_POLL_TIME 30 /*msec*/
+static u8 msg_buf[CSI_MSG_LEN] = {0};
+static u8 buff[CSI_MSG_LEN] = {0};
+
+void rtw_csi_poll_timer_set(struct dvobj_priv *dvobj, u32 delay)
+{
+	_timer *timer = NULL;
+
+	timer = &dvobj->csi_poll_timer;
+	_set_timer(timer, delay);
+}
+
+void rtw_csi_poll_timer_cancel(struct dvobj_priv *dvobj)
+{
+	_timer *timer = NULL;
+
+	timer = &dvobj->csi_poll_timer;
+	_cancel_timer_ex(timer);
+	timer->function = NULL;
+}
+
+void rtw_nl_send_csi(struct dvobj_priv* dvobj, u8* buffer, u32 buf_len, struct csi_header_t *csi_header)
+{
+	struct nlmsghdr *nlh;
+	struct sk_buff *skb_out;
+	int msg_size = 0;
+	//char *msg = "NL msg send from kernel";
+	//char *msg_send;
+	int res;
+	struct ow_csi_netlink_header ow_csi_nlh;
+	struct ow_csi_frame_header ow_csi_h;
+
+	if ((buf_len + sizeof(struct csi_header_t)) > CSI_MSG_LEN) {
+		RTW_ERR("%s: CSI Buffer len is not enought.\n", __func__);
+		return;
+	}
+	else {
+		//RTW_INFO("%s: CSI total len = %d\n", __func__, buf_len + sizeof(struct csi_header_t));
+	}
+
+	/* Fill ow_csi_netlink_header */
+	RTW_DBG("%s: len=%ld\n", __func__, sizeof(struct ow_csi_netlink_header));
+	_rtw_memset(&ow_csi_nlh, 0, sizeof(struct ow_csi_netlink_header));
+	ow_csi_nlh.phynetdevidx = 0; // ToDo
+	ow_csi_nlh.sequence_number = 0; //ToDo
+	ow_csi_nlh.fragment_index = 0;
+
+	/* Fill ow_csi_frame_header */
+	RTW_DBG("%s: len=%ld\n", __func__, sizeof(struct ow_csi_frame_header));
+	_rtw_memset(&ow_csi_h, 0, sizeof(struct ow_csi_frame_header));
+	_rtw_memcpy(ow_csi_h.src_macaddr, csi_header->mac_addr, 6);
+	ow_csi_h.csi_capture_status = csi_header->csi_valid;
+	ow_csi_h.csi_timestamp = csi_header->hw_assigned_timestamp;
+	ow_csi_h.csi_data_length = csi_header->csi_data_length;
+	ow_csi_h.csi_nc = csi_header->nc;
+	ow_csi_h.csi_nr = csi_header->nr;
+	ow_csi_h.csi_bw = csi_header->bandwidth;
+	ow_csi_h.csi_rx_rate = csi_header->rx_data_rate;
+	ow_csi_h.csi_subc_num = csi_header->num_sub_carrier;
+	ow_csi_h.csi_tone_bit = csi_header->num_bit_per_tone;
+	ow_csi_h.csi_rssi[0] = csi_header->rssi[0];
+	ow_csi_h.csi_rssi[1] = csi_header->rssi[1];
+
+	/* Prepare NL msg */
+	_rtw_memcpy(msg_buf, &ow_csi_nlh, sizeof(struct ow_csi_netlink_header));
+	msg_size += sizeof(struct ow_csi_netlink_header);
+
+	_rtw_memcpy(msg_buf + msg_size, &ow_csi_h, sizeof(struct ow_csi_frame_header));
+	msg_size += sizeof(struct ow_csi_frame_header);
+
+	/* Fill csi_raw */
+#if 1
+	RTW_DBG(":%s: len=%d\n", __func__, csi_header->csi_data_length);
+	_rtw_memcpy(msg_buf + msg_size, buffer, csi_header->csi_data_length);
+	msg_size += csi_header->csi_data_length;
+#else
+	RTW_DBG(":%s: len=%d\n", __func__, buf_len);
+	_rtw_memcpy(msg_buf + msg_size, buffer, buf_len);
+	msg_size += buf_len;
+#endif
+	RTW_DBG("%s: msg_size = %d\n", __func__, msg_size);
+
+	skb_out = nlmsg_new(msg_size, 0);
+
+	if (!skb_out) {
+		RTW_ERR("%s: Failed to allocate new skb\n", __func__);
+		return;
+	}
+
+	nlh = nlmsg_put(skb_out, 0, 0, NLMSG_DONE, msg_size, 0);
+	NETLINK_CB(skb_out).dst_group = 0; /* not in mcast group */
+
+	_rtw_memcpy(nlmsg_data(nlh), msg_buf, msg_size);
+
+	if (!dvobj->csi_nl_sk) {
+		RTW_ERR("%s fail: csi_nl_sk is NULL\n", __func__);
+		return;
+	}
+
+	res = nlmsg_unicast(dvobj->csi_nl_sk, skb_out, ow_pid);
+	if (res < 0) {
+		if (printk_ratelimit())
+			RTW_ERR("%s: Error while sending bak to user\n", __func__);
+	}
+}
+
+void rtw_csi_poll_timeout_handler(void *FunctionContext)
+{
+	struct dvobj_priv *d;
+	struct csi_header_t csi_header;
+	void *phl;
+	u32 buf_len;
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	int i;
+
+	d = (struct dvobj_priv *)FunctionContext;
+	phl = GET_PHL_INFO(d);
+
+	rtw_csi_poll_timer_set(d, CSI_POLL_TIME);
+
+	_rtw_memset(&csi_header, 0, sizeof(csi_header));
+	status = rtw_phl_query_chan_info(phl, CSI_MSG_LEN, buff, &buf_len, &csi_header);
+	if (status == RTW_PHL_STATUS_SUCCESS) {
+
+		/* send to user app by NL */
+		rtw_nl_send_csi(d, buff, buf_len, &csi_header);
+
+#if 0 //debug dump
+		//RTW_DUMP_SEL(m, buff, buf_len);
+		RTW_INFO("csi_valid=%d\n", csi_header.csi_valid);
+		RTW_INFO("csi_data_length=%d\n", csi_header.csi_data_length);
+		RTW_INFO("rxsc=%d\n", csi_header.rxsc);
+		RTW_INFO("nc=%d\n", csi_header.nc);
+		RTW_INFO("nr=%d\n", csi_header.nr);
+		RTW_INFO("avg_idle_noise_pwr=%d\n", csi_header.avg_idle_noise_pwr);
+		RTW_INFO("evm[0]=%d\n", csi_header.evm[0]);
+		RTW_INFO("evm[1]=%d\n", csi_header.evm[1]);
+		RTW_INFO("rssi[0]=%d%%\n", csi_header.rssi[0] >> 1);
+		RTW_INFO("rssi[1]=%d%%\n", csi_header.rssi[1] >> 1);
+		RTW_INFO("timestamp=%u\n", csi_header.hw_assigned_timestamp);
+		RTW_INFO("rx_data_rate=%d\n", csi_header.rx_data_rate);
+		RTW_INFO("channel=%d\n", csi_header.channel);
+		RTW_INFO("bandwidth=%d\n", csi_header.bandwidth);
+		RTW_INFO("num_sub_carrier=%d\n", csi_header.num_sub_carrier);
+		RTW_INFO("num_bit_per_tone=%d\n", csi_header.num_bit_per_tone);
+		printk(KERN_CONT "srcmac= ");
+		for (i = 0; i < 6; i++)
+			printk(KERN_CONT "%x ", csi_header.mac_addr[i]);
+		RTW_INFO("\n");
+
+#if 0
+		RTW_INFO("CSI RAW(len=%d):\n", buf_len);
+		for (i = 0; i < buf_len; i++) {
+			printk(KERN_CONT "%x ", buff[i]);
+			if ((i%8)==7)
+				printk(KERN_CONT "\n");
+		}
+		printk("\n");
+#endif
+#endif
+	}
+}
+
+void rtw_csi_poll_init(struct dvobj_priv *dvobj)
+{
+	_timer *timer = &dvobj->csi_poll_timer;
+
+	if (timer->function != NULL) {
+		RTW_INFO("CSI polling timer has been init.\n");
+		return;
+	}
+
+	rtw_init_timer(timer, rtw_csi_poll_timeout_handler, dvobj);
+	rtw_csi_poll_timer_set(dvobj, CSI_POLL_TIME);
+	RTW_INFO("CSI polling timer init!\n");
+}
+#endif /*CONFIG_CSI_TIMER_POLLING*/
+#endif /*CONFIG_RTW_CSI_CHANNEL_INFO*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csi.h b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csi.h
new file mode 100644
index 000000000000..8a18c34e50f3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_csi.h
@@ -0,0 +1,29 @@
+
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_CSI_H_
+#define __RTW_CSI_H_
+
+#ifdef CONFIG_RTW_CSI_NETLINK
+int rtw_csi_nl_init(struct dvobj_priv *dvobj);
+void rtw_csi_nl_exit(struct dvobj_priv *dvobj);
+#endif
+
+#ifdef CONFIG_CSI_TIMER_POLLING
+void rtw_csi_poll_init(struct dvobj_priv *dvobj);
+void rtw_csi_poll_timer_cancel(struct dvobj_priv *dvobj);
+#endif
+
+#endif /*__RTW_CSI_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_debug.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_debug.c
new file mode 100644
index 000000000000..1f04580d05ce
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_debug.c
@@ -0,0 +1,8393 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_DEBUG_C_
+
+#include <drv_types.h>
+#include "_hal_rate.h"
+
+#ifdef CONFIG_MP_INCLUDED
+#include <rtw_mp.h>
+#endif
+
+#ifdef CONFIG_MDNS_OFFLOAD
+#include <linux/ctype.h>	/* tolower() */
+#endif
+
+#ifdef CONFIG_RTW_DEBUG
+const char *rtw_log_level_str[] = {
+	"_DRV_NONE_ = 0",
+	"_DRV_ALWAYS_ = 1",
+	"_DRV_ERR_ = 2",
+	"_DRV_WARNING_ = 3",
+	"_DRV_INFO_ = 4",
+	"_DRV_DEBUG_ = 5",
+	"_DRV_MAX_ = 6",
+};
+#endif
+
+#ifdef CONFIG_DEBUG_RTL871X
+	u64 GlobalDebugComponents = 0;
+#endif /* CONFIG_DEBUG_RTL871X */
+
+#include <rtw_version.h>
+
+#ifdef CONFIG_TDLS
+	#define TDLS_DBG_INFO_SPACE_BTWN_ITEM_AND_VALUE	41
+#endif
+
+void dump_drv_version(void *sel)
+{
+	RTW_PRINT_SEL(sel, "%s %s\n", DRV_NAME, DRIVERVERSION);
+	RTW_PRINT_SEL(sel, "build time: %s %s\n", __DATE__, __TIME__);
+}
+
+void dump_drv_cfg(void *sel)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+	char *kernel_version = utsname()->release;
+
+	RTW_PRINT_SEL(sel, "\nKernel Version: %s\n", kernel_version);
+#endif
+
+#ifdef CONFIG_RTW_ANDROID
+	RTW_PRINT_SEL(sel, "Android Driver: %d\n", CONFIG_RTW_ANDROID);
+#else
+	RTW_PRINT_SEL(sel, "Linux Driver: \n");
+#endif /* CONFIG_RTW_ANDROID */
+
+	RTW_PRINT_SEL(sel, "Driver Version: %s\n", DRIVERVERSION);
+	RTW_PRINT_SEL(sel, "------------------------------------------------\n");
+
+#ifdef CONFIG_IOCTL_CFG80211
+	RTW_PRINT_SEL(sel, "CFG80211\n");
+#ifdef RTW_USE_CFG80211_STA_EVENT
+	RTW_PRINT_SEL(sel, "RTW_USE_CFG80211_STA_EVENT\n");
+#endif
+	#ifdef CONFIG_RADIO_WORK
+	RTW_PRINT_SEL(sel, "CONFIG_RADIO_WORK\n");
+	#endif
+#else
+	RTW_PRINT_SEL(sel, "WEXT\n");
+#endif
+
+	RTW_PRINT_SEL(sel, "DBG:%d\n", DBG);
+#ifdef CONFIG_RTW_DEBUG
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_DEBUG\n");
+#endif
+
+#ifdef CONFIG_CONCURRENT_MODE
+	RTW_PRINT_SEL(sel, "CONFIG_CONCURRENT_MODE\n");
+
+	#ifdef CONFIG_RTW_STATIC_NDEV_NUM
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_STATIC_NDEV_NUM = %d\n", CONFIG_RTW_STATIC_NDEV_NUM);
+	#endif
+	#if (RTW_P2P_GROUP_INTERFACE == 1)
+	RTW_PRINT_SEL(sel, "RTW_P2P_GROUP_INTERFACE\n");
+	#endif
+	#ifdef RTW_DEDICATED_P2P_DEVICE
+	RTW_PRINT_SEL(sel, "RTW_DEDICATED_P2P_DEVICE\n");
+	#endif
+	#if defined(CONFIG_P2P) && defined(CONFIG_SEL_P2P_IFACE)
+	RTW_PRINT_SEL(sel, "CONFIG_SEL_P2P_IFACE = %d\n", CONFIG_SEL_P2P_IFACE);
+	#endif
+#endif
+
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	RTW_PRINT_SEL(sel, "LOAD_PHY_PARA_FROM_FILE - REALTEK_CONFIG_PATH=%s\n", REALTEK_CONFIG_PATH);
+	#if defined(CONFIG_MULTIDRV) || defined(REALTEK_CONFIG_PATH_WITH_IC_NAME_FOLDER)
+	RTW_PRINT_SEL(sel, "LOAD_PHY_PARA_FROM_FILE - REALTEK_CONFIG_PATH_WITH_IC_NAME_FOLDER\n");
+	#endif
+
+/* configurations about TX power */
+#ifdef CONFIG_CALIBRATE_TX_POWER_BY_REGULATORY
+	RTW_PRINT_SEL(sel, "CONFIG_CALIBRATE_TX_POWER_BY_REGULATORY\n");
+#endif
+#ifdef CONFIG_CALIBRATE_TX_POWER_TO_MAX
+	RTW_PRINT_SEL(sel, "CONFIG_CALIBRATE_TX_POWER_TO_MAX\n");
+#endif
+#endif
+	RTW_PRINT_SEL(sel, "RTW_DEF_MODULE_REGULATORY_CERT=0x%02x\n", RTW_DEF_MODULE_REGULATORY_CERT);
+
+	RTW_PRINT_SEL(sel, "CONFIG_TXPWR_BY_RATE=%d\n", CONFIG_TXPWR_BY_RATE);
+	RTW_PRINT_SEL(sel, "CONFIG_TXPWR_BY_RATE_EN=%d\n", CONFIG_TXPWR_BY_RATE_EN);
+	RTW_PRINT_SEL(sel, "CONFIG_TXPWR_LIMIT=%d\n", CONFIG_TXPWR_LIMIT);
+	RTW_PRINT_SEL(sel, "CONFIG_TXPWR_LIMIT_EN=%d\n", CONFIG_TXPWR_LIMIT_EN);
+
+
+#ifdef CONFIG_DISABLE_ODM
+	RTW_PRINT_SEL(sel, "CONFIG_DISABLE_ODM\n");
+#endif
+
+#ifdef CONFIG_MINIMAL_MEMORY_USAGE
+	RTW_PRINT_SEL(sel, "CONFIG_MINIMAL_MEMORY_USAGE\n");
+#endif
+
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_EDCCA_MODE_SEL = %d\n", CONFIG_RTW_EDCCA_MODE_SEL);
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_ADAPTIVITY_EN = %d\n", CONFIG_RTW_ADAPTIVITY_EN);
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_ADAPTIVITY_MODE = %d\n", CONFIG_RTW_ADAPTIVITY_MODE);
+
+#ifdef CONFIG_POWER_SAVE
+	RTW_PRINT_SEL(sel, "\n===CONFIG_POWER_SAVING===\n");
+	#ifdef RTW_IPS_MODE
+		RTW_PRINT_SEL(sel, "CONFIG_IPS_MODE=%d\n", RTW_IPS_MODE);
+	#endif /* RTW_IPS_MODE */
+	#ifdef RTW_LPS_MODE
+		RTW_PRINT_SEL(sel, "CONFIG_LPS_MODE=%d\n", RTW_LPS_MODE);
+	#endif /* RTW_LPS_MODE */
+	#ifdef RTW_WOW_IPS_MODE
+		RTW_PRINT_SEL(sel, "CONFIG_WOW_IPS_MODE=%d\n", RTW_WOW_IPS_MODE);;
+	#endif /* RTW_WOW_IPS_MODE */
+	#ifdef RTW_WOW_LPS_MODE
+		RTW_PRINT_SEL(sel, "CONFIG_WOW_LPS_MODE=%d\n\n", RTW_WOW_LPS_MODE);
+	#endif /* RTW_WOW_LPS_MODE */
+#endif /* CONFIG_POWER_SAVE */
+
+#ifdef CONFIG_WOWLAN
+	RTW_PRINT_SEL(sel, "CONFIG_WOWLAN - ");
+
+#ifdef CONFIG_GPIO_WAKEUP
+	RTW_PRINT_SEL(sel, "CONFIG_GPIO_WAKEUP - WAKEUP_GPIO_IDX:%d\n", WAKEUP_GPIO_IDX);
+#endif
+#endif
+
+#ifdef CONFIG_TDLS
+	RTW_PRINT_SEL(sel, "CONFIG_TDLS\n");
+#endif
+
+#ifdef CONFIG_RTW_80211R
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_80211R\n");
+#endif
+
+#ifdef CONFIG_RTW_NETIF_SG
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_NETIF_SG\n");
+#endif
+
+#ifdef CONFIG_RTW_WIFI_HAL
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_WIFI_HAL\n");
+#endif
+
+#ifdef RTW_BUSY_DENY_SCAN
+	RTW_PRINT_SEL(sel, "RTW_BUSY_DENY_SCAN\n");
+	RTW_PRINT_SEL(sel, "BUSY_TRAFFIC_SCAN_DENY_PERIOD = %u ms\n", \
+		      BUSY_TRAFFIC_SCAN_DENY_PERIOD);
+#endif
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	RTW_PRINT_SEL(sel, "CONFIG_TCP_CSUM_OFFLOAD_TX\n");
+#endif
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_RX
+	RTW_PRINT_SEL(sel, "CONFIG_TCP_CSUM_OFFLOAD_RX\n");
+#endif
+
+#ifdef CONFIG_USB_HCI
+#ifdef CONFIG_SUPPORT_USB_INT
+	RTW_PRINT_SEL(sel, "CONFIG_SUPPORT_USB_INT\n");
+#endif
+#ifdef CONFIG_USB_INTERRUPT_IN_PIPE
+	RTW_PRINT_SEL(sel, "CONFIG_USB_INTERRUPT_IN_PIPE\n");
+#endif
+#ifdef CONFIG_USB_TX_AGGREGATION
+	RTW_PRINT_SEL(sel, "CONFIG_USB_TX_AGGREGATION\n");
+#endif
+#ifdef CONFIG_USB_RX_AGGREGATION
+	RTW_PRINT_SEL(sel, "CONFIG_USB_RX_AGGREGATION\n");
+#endif
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_TX
+	RTW_PRINT_SEL(sel, "CONFIG_USE_USB_BUFFER_ALLOC_TX\n");
+#endif
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_RX
+	RTW_PRINT_SEL(sel, "CONFIG_USE_USB_BUFFER_ALLOC_RX\n");
+#endif
+
+#ifdef CONFIG_FIX_NR_BULKIN_BUFFER
+	RTW_PRINT_SEL(sel, "CONFIG_FIX_NR_BULKIN_BUFFER\n");
+#endif
+#endif /*CONFIG_USB_HCI*/
+
+#ifdef CONFIG_SDIO_HCI
+#ifdef DBG_SDIO
+	RTW_PRINT_SEL(sel, "DBG_SDIO = %d\n", DBG_SDIO);
+#endif
+
+#ifdef RTW_WKARD_SDIO_TX_USE_YIELD
+	RTW_PRINT_SEL(sel, "RTW_WKARD_SDIO_TX_USE_YIELD\n");
+#endif
+
+#ifdef RTW_XMIT_THREAD_HIGH_PRIORITY
+	RTW_PRINT_SEL(sel, "RTW_XMIT_THREAD_HIGH_PRIORITY\n");
+#endif
+#ifdef RTW_XMIT_THREAD_CB_HIGH_PRIORITY
+	RTW_PRINT_SEL(sel, "RTW_XMIT_THREAD_CB_HIGH_PRIORITY\n");
+#endif
+#ifdef RTW_RECV_THREAD_HIGH_PRIORITY
+	RTW_PRINT_SEL(sel, "RTW_RECV_THREAD_HIGH_PRIORITY\n");
+#endif
+
+#endif /*CONFIG_SDIO_HCI*/
+
+#ifdef CONFIG_PCI_HCI
+#endif
+
+	RTW_PRINT_SEL(sel, "CONFIG_IFACE_NUMBER = %d\n", CONFIG_IFACE_NUMBER);
+
+
+#ifdef CONFIG_PCI_TX_POLLING
+	RTW_PRINT_SEL(sel, "CONFIG_PCI_TX_POLLING\n");
+#endif
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_UP_MAPPING_RULE = %s\n", (CONFIG_RTW_UP_MAPPING_RULE == 1) ? "dscp" : "tos");
+
+	/*GEORGIA_TODO_TRX - need get trx buff accroding to IC spec*/
+	RTW_PRINT_SEL(sel, "\n=== XMIT-INFO ===\n");
+	RTW_PRINT_SEL(sel, "NR_XMITFRAME = %d\n", NR_XMITFRAME);
+	RTW_PRINT_SEL(sel, "MAX_TX_PHL_RING_ENTRY_NUM = %d\n", MAX_PHL_TX_RING_ENTRY_NUM);
+#ifdef CONFIG_QUOTA_TURBO_ENABLE
+	RTW_PRINT_SEL(sel, "CONFIG_QUOTA_TURBO_ENABLE\n");
+#endif
+#ifdef CONFIG_USB_HCI
+	RTW_PRINT_SEL(sel, "MAX_XMITBUF_SZ = %d\n", MAX_XMITBUF_SZ);
+	RTW_PRINT_SEL(sel, "NR_XMITBUFF = %d\n", NR_XMITBUFF);
+	RTW_PRINT_SEL(sel, "MAX_MGNT_XMITBUF_SZ = %d\n", MAX_MGNT_XMITBUF_SZ);
+	RTW_PRINT_SEL(sel, "NR_MGNT_XMITBUFF = %d\n", NR_MGNT_XMITBUFF);
+#endif
+
+	RTW_PRINT_SEL(sel, "\n=== RECV-INFO ===\n");
+	RTW_PRINT_SEL(sel, "NR_RECVFRAME = %d\n", NR_RECVFRAME);
+	RTW_PRINT_SEL(sel, "MAX_RX_PHL_RING_ENTRY_NUM = %d\n", MAX_PHL_RX_RING_ENTRY_NUM);
+#ifdef CONFIG_USB_HCI
+	RTW_PRINT_SEL(sel, "MAX_RECVBUF_SZ = %d\n", MAX_RECVBUF_SZ);
+	RTW_PRINT_SEL(sel, "NR_RECVBUFF = %d\n", NR_RECVBUFF);
+	RTW_PRINT_SEL(sel, "NR_RECV_URB = %d\n", NR_RECV_URB);
+#endif
+}
+
+void dump_log_level(void *sel)
+{
+#ifdef CONFIG_RTW_DEBUG
+	int i;
+
+	RTW_PRINT_SEL(sel, "drv_log_level:%d\n", rtw_drv_log_level);
+	for (i = 0; i <= _DRV_MAX_; i++) {
+		if (rtw_log_level_str[i])
+			RTW_PRINT_SEL(sel, "%c %s = %d\n",
+				(rtw_drv_log_level == i) ? '+' : ' ', rtw_log_level_str[i], i);
+	}
+#else
+	RTW_PRINT_SEL(sel, "CONFIG_RTW_DEBUG is disabled\n");
+#endif
+}
+
+
+void rtw_sink_rtp_seq_dbg(_adapter *adapter, u8 *ehdr_pos)
+{
+	struct recv_info *precvinfo = &(adapter->recvinfo);
+	if (precvinfo->sink_udpport > 0) {
+		if (*((u16 *)(ehdr_pos + 0x24)) == cpu_to_be16(precvinfo->sink_udpport)) {
+			precvinfo->pre_rtp_rxseq = precvinfo->cur_rtp_rxseq;
+			precvinfo->cur_rtp_rxseq = be16_to_cpu(*((u16 *)(ehdr_pos + 0x2C)));
+			if (precvinfo->pre_rtp_rxseq + 1 != precvinfo->cur_rtp_rxseq) {
+				if(precvinfo->pre_rtp_rxseq == 65535 ) {
+					if( precvinfo->cur_rtp_rxseq != 0) {
+						RTW_INFO("%s : RTP Seq num from %d to %d\n", __FUNCTION__, precvinfo->pre_rtp_rxseq, precvinfo->cur_rtp_rxseq);
+					}
+				} else {
+					RTW_INFO("%s : RTP Seq num from %d to %d\n", __FUNCTION__, precvinfo->pre_rtp_rxseq, precvinfo->cur_rtp_rxseq);
+				}
+			}
+		}
+	}
+}
+
+void sta_rx_reorder_ctl_dump(void *sel, struct sta_info *sta)
+{
+	struct recv_reorder_ctrl *reorder_ctl;
+	int i;
+
+	for (i = 0; i < 16; i++) {
+		reorder_ctl = &sta->recvreorder_ctrl[i];
+		if (reorder_ctl->ampdu_size != RX_AMPDU_SIZE_INVALID || reorder_ctl->indicate_seq != 0xFFFF) {
+			RTW_PRINT_SEL(sel, "tid=%d, enable=%d, ampdu_size=%u, indicate_seq=%u\n"
+				, i, reorder_ctl->enable, reorder_ctl->ampdu_size, reorder_ctl->indicate_seq
+				     );
+		}
+	}
+}
+
+void dump_tx_rate_bmp(void *sel, struct dvobj_priv *dvobj)
+{
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	u8 bw;
+
+	RTW_PRINT_SEL(sel, "%-6s", "bw");
+	if (rtw_hw_chk_proto_cap(dvobj, WLAN_MD_11AC))
+		_RTW_PRINT_SEL(sel, " %-15s", "vht");
+
+	_RTW_PRINT_SEL(sel, " %-11s %-4s %-3s\n", "ht", "ofdm", "cck");
+
+	for (bw = CHANNEL_WIDTH_20; bw <= CHANNEL_WIDTH_160; bw++) {
+		if (!rtw_hw_is_bw_support(adapter_to_dvobj(adapter), bw))
+			continue;
+
+		RTW_PRINT_SEL(sel, "%6s", ch_width_str(bw));
+		if (rtw_hw_chk_proto_cap(dvobj, WLAN_MD_11AC)) {
+			_RTW_PRINT_SEL(sel, " %03x %03x %03x %03x"
+				, RATE_BMP_GET_VHT_4SS(rfctl->rate_bmp_vht_by_bw[bw])
+				, RATE_BMP_GET_VHT_3SS(rfctl->rate_bmp_vht_by_bw[bw])
+				, RATE_BMP_GET_VHT_2SS(rfctl->rate_bmp_vht_by_bw[bw])
+				, RATE_BMP_GET_VHT_1SS(rfctl->rate_bmp_vht_by_bw[bw])
+			);
+		}
+
+		_RTW_PRINT_SEL(sel, " %02x %02x %02x %02x"
+			, bw <= CHANNEL_WIDTH_40 ? RATE_BMP_GET_HT_4SS(rfctl->rate_bmp_ht_by_bw[bw]) : 0
+			, bw <= CHANNEL_WIDTH_40 ? RATE_BMP_GET_HT_3SS(rfctl->rate_bmp_ht_by_bw[bw]) : 0
+			, bw <= CHANNEL_WIDTH_40 ? RATE_BMP_GET_HT_2SS(rfctl->rate_bmp_ht_by_bw[bw]) : 0
+			, bw <= CHANNEL_WIDTH_40 ? RATE_BMP_GET_HT_1SS(rfctl->rate_bmp_ht_by_bw[bw]) : 0
+		);
+
+		_RTW_PRINT_SEL(sel, "  %03x   %01x\n"
+			, bw <= CHANNEL_WIDTH_20 ? RATE_BMP_GET_OFDM(rfctl->rate_bmp_cck_ofdm) : 0
+			, bw <= CHANNEL_WIDTH_20 ? RATE_BMP_GET_CCK(rfctl->rate_bmp_cck_ofdm) : 0
+		);
+	}
+}
+
+void dump_adapters_status(void *sel, struct dvobj_priv *dvobj)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	int i;
+	_adapter *iface;
+	u8 u_ch, u_bw, u_offset;
+	struct _ADAPTER_LINK *iface_link;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	dump_mi_status(sel, dvobj);
+	RTW_PRINT_SEL(sel, "dev status:%s%s\n\n"
+		, dev_is_surprise_removed(dvobj) ? " SR" : ""
+		, dev_is_drv_stopped(dvobj) ? " DS" : ""
+	);
+
+#ifdef CONFIG_P2P
+#define P2P_INFO_TITLE_FMT	" %-3s %-4s"
+#define P2P_INFO_TITLE_ARG	, "lch", "p2ps"
+#ifdef CONFIG_IOCTL_CFG80211
+#define P2P_INFO_VALUE_FMT	" %3u %4c"
+#define P2P_INFO_VALUE_ARG	, iface->wdinfo.listen_channel, iface->wdev_data.p2p_enabled ? 'e' : ' '
+#else
+#define P2P_INFO_VALUE_FMT	" %3u"
+#define P2P_INFO_VALUE_ARG	, iface->wdinfo.listen_channel
+#endif
+#define P2P_INFO_DASH		"---------"
+#else
+#define P2P_INFO_TITLE_FMT	""
+#define P2P_INFO_TITLE_ARG
+#define P2P_INFO_VALUE_FMT	""
+#define P2P_INFO_VALUE_ARG
+#define P2P_INFO_DASH
+#endif
+
+#define INFO_FMT	""
+#define INFO_ARG
+#define INFO_CNT_FMT	""
+#define INFO_CNT_ARG
+
+
+	RTW_PRINT_SEL(sel, "%-2s %-15s %c %-8s %-17s %-6s %-4s %-4s %-7s %-5s"
+		P2P_INFO_TITLE_FMT
+		" %s"INFO_FMT"\n"
+		, "id", "ifname", ' ', "netif_up", "macaddr", "hwband", "port", "band", "ch", "class"
+		P2P_INFO_TITLE_ARG
+		, "status"INFO_ARG);
+
+	RTW_PRINT_SEL(sel, "------------------------------------------------------------------------------"
+		P2P_INFO_DASH
+		"-------\n");
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (iface) {
+			iface_link = GET_PRIMARY_LINK(iface);
+			RTW_PRINT_SEL(sel, "%2d %-15s %c %8u %s %6hhu %4hhu %-4s %3u,%u,%u %5u"
+				P2P_INFO_VALUE_FMT
+				" "MLME_STATE_FMT" " INFO_CNT_FMT"\n"
+				, i, iface->registered ? ADPT_ARG(iface) : NULL
+				, iface->registered ? 'R' : ' '
+				, iface->netif_up
+				, get_macaddr_str(mac_addr_str, sel, adapter_mac_addr(iface))
+				, (iface_link->wrlink) ? iface_link->wrlink->hw_band : HW_BAND_MAX
+				, (iface_link->wrlink) ? iface_link->wrlink->hw_port : HW_PORT_MAX
+				, band_str(iface_link->mlmeextpriv.chandef.band)
+				, iface_link->mlmeextpriv.chandef.chan
+				, iface_link->mlmeextpriv.chandef.bw
+				, iface_link->mlmeextpriv.chandef.offset
+				, rtw_get_op_class_by_bchbw(iface_link->mlmeextpriv.chandef.band
+					, iface_link->mlmeextpriv.chandef.chan
+					, iface_link->mlmeextpriv.chandef.bw
+					, iface_link->mlmeextpriv.chandef.offset)
+				P2P_INFO_VALUE_ARG
+				, MLME_STATE_ARG(iface)
+				INFO_CNT_ARG
+			);
+		}
+	}
+
+	RTW_PRINT_SEL(sel, "------------------------------------------------------------------------------"
+		P2P_INFO_DASH
+		"-------\n");
+#if 0
+	if (rtw_mi_get_ch_setting_union(dvobj_get_primary_adapter(dvobj), &u_ch, &u_bw, &u_offset))
+		RTW_PRINT_SEL(sel, "%55s %3u,%u,%u %5u\n"
+			, "union:"
+			, u_ch, u_bw, u_offset, rtw_get_op_class_by_chbw(u_ch, u_bw, u_offset));
+
+	RTW_PRINT_SEL(sel, "%55s %3u,%u,%u offch_state:%d\n"
+		, "oper:"
+		, dvobj->chandef.chan
+		, dvobj->chandef.bw
+		, dvobj->chandef.offset
+		, rfctl->offch_state
+	);
+#endif
+
+	#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+	dump_radar_detect_status(sel, rfctl, "                                          radar_detect:");
+	#endif
+}
+
+static u8 del_rx_ampdu_test_no_tx_fail = 0;
+
+bool rtw_del_rx_ampdu_test_trigger_no_tx_fail(void)
+{
+	if (del_rx_ampdu_test_no_tx_fail) {
+		RTW_PRINT("del_rx_ampdu test case: trigger no_tx_fail\n");
+		del_rx_ampdu_test_no_tx_fail--;
+		return _TRUE;
+	}
+	return _FALSE;
+}
+
+static u32 g_wait_hiq_empty_ms = 0;
+
+u32 rtw_get_wait_hiq_empty_ms(void)
+{
+	return g_wait_hiq_empty_ms;
+}
+
+static systime sta_linking_test_start_time = 0;
+static u32 sta_linking_test_wait_ms = 0;
+static u8 sta_linking_test_force_fail = 0;
+
+void rtw_sta_linking_test_set_start(void)
+{
+	sta_linking_test_start_time = rtw_get_current_time();
+}
+
+bool rtw_sta_linking_test_wait_done(void)
+{
+	return rtw_get_passing_time_ms(sta_linking_test_start_time) >= sta_linking_test_wait_ms;
+}
+
+bool rtw_sta_linking_test_force_fail(void)
+{
+	return sta_linking_test_force_fail;
+}
+
+#ifdef CONFIG_AP_MODE
+static u16 ap_linking_test_force_auth_fail = 0;
+static u16 ap_linking_test_force_asoc_fail = 0;
+
+u16 rtw_ap_linking_test_force_auth_fail(void)
+{
+	return ap_linking_test_force_auth_fail;
+}
+
+u16 rtw_ap_linking_test_force_asoc_fail(void)
+{
+	return ap_linking_test_force_asoc_fail;
+}
+#endif
+
+int proc_get_defs_param(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+
+	RTW_PRINT_SEL(m, "%s %15s\n", "lmt_sta", "lmt_time");
+	RTW_PRINT_SEL(m, "%-15u %-15u\n"
+		, mlme->defs_lmt_sta
+		, mlme->defs_lmt_time
+	);
+
+	return 0;
+}
+
+ssize_t proc_set_defs_param(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+
+	char tmp[32];
+	u32 defs_lmt_sta;
+	u32 defs_lmt_time;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%u %u", &defs_lmt_sta, &defs_lmt_time);
+
+		if (num >= 1)
+			mlme->defs_lmt_sta = defs_lmt_sta;
+		if (num >= 2)
+			mlme->defs_lmt_time = defs_lmt_time;
+	}
+
+	return count;
+
+}
+
+ssize_t proc_set_write_reg(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	char tmp[32];
+	u32 addr, val, len;
+
+	if (count < 3) {
+		RTW_INFO("argument size is less than 3\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%x %x %x", &addr, &val, &len);
+
+		if (num !=  3) {
+			RTW_INFO("invalid write_reg parameter!\n");
+			return count;
+		}
+
+		switch (len) {
+		case 1:
+			rtw_phl_write8(dvobj->phl, addr, (u8)val);
+			break;
+		case 2:
+			rtw_phl_write16(dvobj->phl, addr, (u16)val);
+			break;
+		case 4:
+			rtw_phl_write32(dvobj->phl, addr, val);
+			break;
+		default:
+			RTW_INFO("error write length=%d", len);
+			break;
+		}
+
+	}
+	return count;
+
+}
+
+static u32 proc_get_read_addr = 0xeeeeeeee;
+static u32 proc_get_read_len = 0x4;
+
+int proc_get_read_reg(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	if (proc_get_read_addr == 0xeeeeeeee) {
+		RTW_PRINT_SEL(m, "address not initialized\n");
+		return 0;
+	}
+
+	switch (proc_get_read_len) {
+	case 1:
+		RTW_PRINT_SEL(m, "reg_read8(0x%x)=0x%x\n", proc_get_read_addr,
+			rtw_phl_read8(dvobj->phl, proc_get_read_addr));
+		break;
+	case 2:
+		RTW_PRINT_SEL(m, "reg_read16(0x%x)=0x%x\n", proc_get_read_addr,
+			rtw_phl_read16(dvobj->phl, proc_get_read_addr));
+		break;
+	case 4:
+		RTW_PRINT_SEL(m, "reg_read32(0x%x)=0x%x\n", proc_get_read_addr,
+			rtw_phl_read32(dvobj->phl, proc_get_read_addr));
+		break;
+	default:
+		RTW_PRINT_SEL(m, "error read length=%d\n", proc_get_read_len);
+		break;
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_read_reg(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[16];
+	u32 addr, len;
+
+	if (count < 2) {
+		RTW_INFO("argument size is less than 2\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%x %x", &addr, &len);
+
+		if (num !=  2) {
+			RTW_INFO("invalid read_reg parameter!\n");
+			return count;
+		}
+
+		proc_get_read_addr = addr;
+
+		proc_get_read_len = len;
+	}
+
+	return count;
+
+}
+
+ssize_t proc_set_mac_dbg_status_dump(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	char tmp[32] = {0};
+	u32 val[2] = {0};
+	u32 en;
+	u8 en_u8;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%x", &en);
+
+		if (num !=  1) {
+			RTW_INFO("invalid mac_dbg_status_dump parameter!\n");
+			return count;
+		}
+		en_u8 = (u8)en;
+		rtw_phl_mac_dbg_status_dump(dvobj->phl, val, &en_u8);
+		if (en & BIT0)
+			RTW_INFO_DUMP("ss_dbgpkg: ", (const u8 *)val, 8);
+	}
+	return count;
+
+}
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+int proc_get_ignore_go_and_low_rssi_in_scan(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+
+	RTW_PRINT_SEL(m, "ignore_go_in_scan=%d, ignore_low_rssi_in_scan=%d\n",
+		pregpriv->ignore_go_in_scan, pregpriv->ignore_low_rssi_in_scan);
+	return 0;
+}
+
+ssize_t proc_set_ignore_go_and_low_rssi_in_scan(struct file *file,
+	const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+
+	char tmp[32] = {0};
+	int enable = 0;
+	int num = 0, low_rssi=0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num	= sscanf(tmp, "%d %d", &enable, &low_rssi);
+		RTW_INFO("num=%d enable=%d low_rssi=%d\n", num, enable, low_rssi);
+
+		if (num != 2) {
+			RTW_INFO("argument number is wrong\n");
+			return -EFAULT;
+		}
+
+		pregpriv->ignore_go_in_scan = enable;
+		pregpriv->ignore_low_rssi_in_scan = low_rssi;
+	}
+	return count;
+}
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+
+#ifdef CONFIG_PROC_DEBUG
+int proc_get_rx_stat(struct seq_file *m, void *v)
+{
+	_list	*plist, *phead;
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_info *psta = NULL;
+	struct stainfo_stats	*pstats = NULL;
+	struct sta_priv		*pstapriv = &(adapter->stapriv);
+	u32 i, j;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+			pstats = &psta->sta_stats;
+
+			if (pstats == NULL)
+				continue;
+			if ((_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, ETH_ALEN) !=  _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, ETH_ALEN) != _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, psta->padapter_link->mac_addr, ETH_ALEN) != _TRUE)) {
+				RTW_PRINT_SEL(m, "MAC :\t\t"MAC_FMT "\n", MAC_ARG(psta->phl_sta->mac_addr));
+				RTW_PRINT_SEL(m, "data_rx_cnt :\t%llu\n", sta_rx_data_uc_pkts(psta) - pstats->last_rx_data_uc_pkts);
+				pstats->last_rx_data_uc_pkts = sta_rx_data_uc_pkts(psta);
+				RTW_PRINT_SEL(m, "duplicate_cnt :\t%u\n", pstats->duplicate_cnt);
+				pstats->duplicate_cnt = 0;
+				RTW_PRINT_SEL(m, "rx_per_rate_cnt :\n");
+
+				for (j = 0; j < 0x60; j++) {
+					RTW_PRINT_SEL(m, "%08u  ", pstats->rxratecnt[j]);
+					pstats->rxratecnt[j] = 0;
+					if ((j%8) == 7)
+						RTW_PRINT_SEL(m, "\n");
+				}
+				RTW_PRINT_SEL(m, "\n");
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	return 0;
+}
+
+int proc_get_sta_tx_stat(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv	*stapriv = &(adapter->stapriv);
+	struct sta_priv	*stapriv_primary = &(GET_PRIMARY_ADAPTER(adapter))->stapriv;
+	struct sta_info *sta;
+	struct stainfo_stats *stats;
+	u8 mac[ETH_ALEN] = {0};
+	int ret = 0;
+
+	_rtw_memcpy(mac, stapriv_primary->c2h_sta_mac, ETH_ALEN);
+	sta = rtw_get_stainfo(stapriv, mac);
+	if (sta) {
+		ret = rtw_get_sta_tx_stat(adapter, sta);
+		stats = &sta->sta_stats;
+		RTW_PRINT_SEL(m, "MAC: " MAC_FMT " fail: %u retry: %u\n",
+			      MAC_ARG(mac), stats->tx_fail_cnt_sum, stats->tx_retry_cnt_sum);
+		if (ret == _SUCCESS)
+			ret = 0;
+		else
+			ret = -ENODATA;
+	} else
+		ret = -ENODATA;
+	return ret;
+}
+
+ssize_t proc_set_sta_tx_stat(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv	*stapriv_primary = &(GET_PRIMARY_ADAPTER(adapter))->stapriv;
+	char tmp[18] = {0};
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		if (sscanf(tmp, MAC_SFMT, MAC_SARG(stapriv_primary->c2h_sta_mac)) != 6) {
+			_rtw_memset(stapriv_primary->c2h_sta_mac, 0, 6);
+			RTW_PRINT(FUNC_ADPT_FMT" Invalid format\n",
+				  FUNC_ADPT_ARG(adapter));
+		}
+
+	}
+
+	return count;
+}
+
+int proc_get_tx_stat(struct seq_file *m, void *v)
+{
+	_list	*plist, *phead;
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_info *psta = NULL;
+	u8 sta_mac[NUM_STA][ETH_ALEN] = {{0}};
+	uint mac_id[NUM_STA];
+	struct stainfo_stats	*pstats = NULL;
+	struct sta_priv	*pstapriv = &(adapter->stapriv);
+	struct sta_priv	*pstapriv_primary = &(GET_PRIMARY_ADAPTER(adapter))->stapriv;
+	u32 i, macid_rec_idx = 0;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+#if defined(PRIVATE_R) || defined(CONFIG_RTW_MULTI_AP)
+	char mac_addr_str[MAC_FMT_LEN];
+#endif
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+			if ((_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, ETH_ALEN) !=  _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, ETH_ALEN) != _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, psta->padapter_link->mac_addr, ETH_ALEN) != _TRUE)) {
+				_rtw_memcpy(&sta_mac[macid_rec_idx][0], psta->phl_sta->mac_addr, ETH_ALEN);
+				mac_id[macid_rec_idx] = psta->phl_sta->macid;
+				macid_rec_idx++;
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < macid_rec_idx; i++) {
+		psta = rtw_get_stainfo(pstapriv, &sta_mac[i][0]);
+		if(psta) {
+			rtw_get_sta_tx_stat(adapter, psta);
+			pstats = &psta->sta_stats;
+#if defined(PRIVATE_R) || defined(CONFIG_RTW_MULTI_AP)
+			RTW_PRINT_SEL(m, "MAC: %s sent: %u fail: %u retry: %u\n",
+				get_macaddr_str(mac_addr_str, m, &sta_mac[i][0]),
+				pstats->tx_ok_cnt, pstats->tx_fail_cnt, pstats->tx_retry_cnt);
+#else
+			RTW_PRINT_SEL(m, "data_sent_cnt :\t%u\n", pstats->tx_ok_cnt + pstats->tx_fail_cnt);
+			RTW_PRINT_SEL(m, "success_cnt :\t%u\n", pstats->tx_ok_cnt);
+			RTW_PRINT_SEL(m, "failure_cnt :\t%u\n", pstats->tx_fail_cnt);
+			RTW_PRINT_SEL(m, "retry_cnt :\t%u\n\n", pstats->tx_retry_cnt);
+#endif
+
+		} else
+			RTW_PRINT_SEL(m, "STA is gone\n");
+	}
+	return 0;
+}
+
+int proc_get_fwstate(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	RTW_PRINT_SEL(m, "fwstate=0x%x\n", get_fwstate(pmlmepriv));
+
+	return 0;
+}
+
+int proc_get_sec_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct security_priv *sec = &padapter->securitypriv;
+
+	RTW_PRINT_SEL(m, "auth_alg=0x%x, enc_alg=0x%x, auth_type=0x%x, enc_type=0x%x\n",
+		sec->dot11AuthAlgrthm, sec->dot11PrivacyAlgrthm,
+		sec->ndisauthtype, sec->ndisencryptstatus);
+
+	RTW_PRINT_SEL(m, "hw_decrypted=%d\n", sec->hw_decrypted);
+	RTW_PRINT_SEL(m, "wpa_psk=");
+	if(sec->wpa_psk & BIT(0))
+		RTW_PRINT_SEL(m, "WPA ");
+	if(sec->wpa_psk & BIT(1))
+		RTW_PRINT_SEL(m, "WPA2 ");
+	RTW_PRINT_SEL(m, "\n");
+	RTW_PRINT_SEL(m, "wpa2_group_cipher=%d\n", sec->wpa2_group_cipher);
+	RTW_PRINT_SEL(m, "wpa2_pairwise_cipher=%d\n", sec->wpa2_pairwise_cipher);
+	RTW_PRINT_SEL(m, "wpa_group_cipher=%d\n", sec->wpa_group_cipher);
+	RTW_PRINT_SEL(m, "wpa_pairwise_cipher=%d\n", sec->wpa_pairwise_cipher);
+	RTW_PRINT_SEL(m, "80211W_enable=%d\n", sec->mfp_opt);
+	RTW_PRINT_SEL(m, "akm_suite_bitmap=0x%x\n", sec->akmp);
+
+#ifdef DBG_SW_SEC_CNT
+	RTW_PRINT_SEL(m, "==sw counters(bc, mc, uc)==\n");
+	RTW_PRINT_SEL(m, "wep_sw_enc_cnt=%llu, %llu, %llu\n"
+		, sec->wep_sw_enc_cnt_bc , sec->wep_sw_enc_cnt_mc, sec->wep_sw_enc_cnt_uc);
+	RTW_PRINT_SEL(m, "wep_sw_dec_cnt=%llu, %llu, %llu\n"
+		, sec->wep_sw_dec_cnt_bc , sec->wep_sw_dec_cnt_mc, sec->wep_sw_dec_cnt_uc);
+
+	RTW_PRINT_SEL(m, "tkip_sw_enc_cnt=%llu, %llu, %llu\n"
+		, sec->tkip_sw_enc_cnt_bc , sec->tkip_sw_enc_cnt_mc, sec->tkip_sw_enc_cnt_uc);
+	RTW_PRINT_SEL(m, "tkip_sw_dec_cnt=%llu, %llu, %llu\n"
+		, sec->tkip_sw_dec_cnt_bc , sec->tkip_sw_dec_cnt_mc, sec->tkip_sw_dec_cnt_uc);
+
+	RTW_PRINT_SEL(m, "aes_sw_enc_cnt=%llu, %llu, %llu\n"
+		, sec->aes_sw_enc_cnt_bc , sec->aes_sw_enc_cnt_mc, sec->aes_sw_enc_cnt_uc);
+	RTW_PRINT_SEL(m, "aes_sw_dec_cnt=%llu, %llu, %llu\n"
+		, sec->aes_sw_dec_cnt_bc , sec->aes_sw_dec_cnt_mc, sec->aes_sw_dec_cnt_uc);
+
+	RTW_PRINT_SEL(m, "gcmp_sw_enc_cnt=%llu, %llu, %llu\n"
+		, sec->gcmp_sw_enc_cnt_bc , sec->gcmp_sw_enc_cnt_mc, sec->gcmp_sw_enc_cnt_uc);
+	RTW_PRINT_SEL(m, "gcmp_sw_dec_cnt=%llu, %llu, %llu\n"
+		, sec->gcmp_sw_dec_cnt_bc , sec->gcmp_sw_dec_cnt_mc, sec->gcmp_sw_dec_cnt_uc);
+#endif /* DBG_SW_SEC_CNT */
+
+	return 0;
+}
+
+int proc_get_mlmext_state(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	RTW_PRINT_SEL(m, "pmlmeinfo->state=0x%x\n", pmlmeinfo->state);
+
+	return 0;
+}
+
+#ifdef CONFIG_LAYER2_ROAMING
+int proc_get_roam_flags(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	int i;
+	char str[4], *band = adapter->mlmepriv.roam_scan_order;
+
+	for (i = 0; i < sizeof(str); i++)
+		str[i] = (band[i]==BAND_ON_24G)?'2':(band[i]==BAND_ON_5G)?'5':(band[i]==BAND_ON_6G)?'6':0;
+
+	RTW_PRINT_SEL(m, "0x%02x, order:%s\n", rtw_roam_flags(adapter), str);
+
+	return 0;
+}
+
+ssize_t proc_set_roam_flags(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	char tmp[32], str[8];
+	int num, i;
+	u8 flags;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num = sscanf(tmp, "%hhx %d", &flags, &i);
+
+		if (num >= 1)
+			rtw_assign_roam_flags(adapter, flags);
+
+		if (i > 0) {
+			sprintf(str, "%d", i);
+			for (i = 0; i < sizeof(adapter->mlmepriv.roam_scan_order); i++)
+				adapter->mlmepriv.roam_scan_order[i] = \
+					(str[i]=='2') ? BAND_ON_24G: \
+					(str[i]=='5') ? BAND_ON_5G: \
+					(str[i]=='6') ? BAND_ON_6G:BAND_MAX;
+		}
+	}
+	return count;
+}
+
+int proc_get_roam_param(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+
+	RTW_PRINT_SEL(m, "%12s %15s %26s %16s\n", "rssi_diff_th", "scanr_exp_ms", "scan_interval(unit:2 sec)", "rssi_threshold");
+	RTW_PRINT_SEL(m, "%-15u %-13u %-27u %-11u\n"
+		, mlme->roam_rssi_diff_th
+		, mlme->roam_scanr_exp_ms
+		, mlme->roam_scan_int
+		, mlme->roam_rssi_threshold
+	);
+
+	return 0;
+}
+
+ssize_t proc_set_roam_param(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+
+	char tmp[32];
+	u8 rssi_diff_th;
+	u32 scanr_exp_ms;
+	u32 scan_int;
+	u8 rssi_threshold;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %u %u %hhu", &rssi_diff_th, &scanr_exp_ms, &scan_int, &rssi_threshold);
+
+		if (num >= 1)
+			mlme->roam_rssi_diff_th = rssi_diff_th;
+		if (num >= 2)
+			mlme->roam_scanr_exp_ms = scanr_exp_ms;
+		if (num >= 3)
+			mlme->roam_scan_int = scan_int;
+		if (num >= 4)
+			mlme->roam_rssi_threshold = rssi_threshold;
+	}
+
+	return count;
+
+}
+
+ssize_t proc_set_roam_tgt_addr(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	char tmp[32];
+	u8 addr[ETH_ALEN];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", addr, addr + 1, addr + 2, addr + 3, addr + 4, addr + 5);
+		if (num == 6)
+			_rtw_memcpy(adapter->mlmepriv.roam_tgt_addr, addr, ETH_ALEN);
+
+		RTW_INFO("set roam_tgt_addr to "MAC_FMT"\n", MAC_ARG(adapter->mlmepriv.roam_tgt_addr));
+	}
+
+	return count;
+}
+#endif /* CONFIG_LAYER2_ROAMING */
+
+
+int proc_get_qos_option(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+
+	RTW_PRINT_SEL(m, "qos_option=%d\n", pmlmepriv->qospriv.qos_option);
+
+	return 0;
+}
+
+int proc_get_ht_option(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+
+#ifdef CONFIG_80211N_HT
+	RTW_PRINT_SEL(m, "ht_option=%d\n", pmlmepriv->htpriv.ht_option);
+#endif /* CONFIG_80211N_HT */
+
+	return 0;
+}
+
+int proc_get_rf_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+
+	RTW_PRINT_SEL(m, "cur_ch=%d, cur_bw=%d, cur_ch_offet=%d\n",
+		pmlmeext->chandef.chan, pmlmeext->chandef.bw, pmlmeext->chandef.offset);
+
+	RTW_PRINT_SEL(m, "oper_ch=%d, oper_bw=%d, oper_ch_offet=%d\n",
+		rtw_get_oper_ch(padapter, padapter_link), rtw_get_oper_bw(padapter, padapter_link),
+				rtw_get_oper_choffset(padapter, padapter_link));
+
+	return 0;
+}
+
+int proc_get_scan_param(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+	struct ss_res *ss = &mlmeext->sitesurvey_res;
+
+#define SCAN_PARAM_TITLE_FMT "%10s"
+#define SCAN_PARAM_VALUE_FMT "%-10u"
+#define SCAN_PARAM_TITLE_ARG , "scan_ch_ms"
+#define SCAN_PARAM_VALUE_ARG , ss->scan_ch_ms
+#ifdef CONFIG_80211N_HT
+#define SCAN_PARAM_TITLE_FMT_HT " %15s %13s"
+#define SCAN_PARAM_VALUE_FMT_HT " %-15u %-13u"
+#define SCAN_PARAM_TITLE_ARG_HT , "rx_ampdu_accept", "rx_ampdu_size"
+#define SCAN_PARAM_VALUE_ARG_HT , ss->rx_ampdu_accept, ss->rx_ampdu_size
+#else
+#define SCAN_PARAM_TITLE_FMT_HT ""
+#define SCAN_PARAM_VALUE_FMT_HT ""
+#define SCAN_PARAM_TITLE_ARG_HT
+#define SCAN_PARAM_VALUE_ARG_HT
+#endif
+#ifdef CONFIG_SCAN_BACKOP
+#define SCAN_PARAM_TITLE_FMT_BACKOP " %9s %12s"
+#define SCAN_PARAM_VALUE_FMT_BACKOP " %-9u %-12u"
+#define SCAN_PARAM_TITLE_ARG_BACKOP , "backop_ms", "scan_cnt_max"
+#define SCAN_PARAM_VALUE_ARG_BACKOP , ss->backop_ms, ss->scan_cnt_max
+#else
+#define SCAN_PARAM_TITLE_FMT_BACKOP ""
+#define SCAN_PARAM_VALUE_FMT_BACKOP ""
+#define SCAN_PARAM_TITLE_ARG_BACKOP
+#define SCAN_PARAM_VALUE_ARG_BACKOP
+#endif
+
+	RTW_PRINT_SEL(m,
+		SCAN_PARAM_TITLE_FMT
+		SCAN_PARAM_TITLE_FMT_HT
+		SCAN_PARAM_TITLE_FMT_BACKOP
+		"\n"
+		SCAN_PARAM_TITLE_ARG
+		SCAN_PARAM_TITLE_ARG_HT
+		SCAN_PARAM_TITLE_ARG_BACKOP
+	);
+
+	RTW_PRINT_SEL(m,
+		SCAN_PARAM_VALUE_FMT
+		SCAN_PARAM_VALUE_FMT_HT
+		SCAN_PARAM_VALUE_FMT_BACKOP
+		"\n"
+		SCAN_PARAM_VALUE_ARG
+		SCAN_PARAM_VALUE_ARG_HT
+		SCAN_PARAM_VALUE_ARG_BACKOP
+	);
+
+	return 0;
+}
+
+ssize_t proc_set_scan_param(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+	struct ss_res *ss = &mlmeext->sitesurvey_res;
+
+	char tmp[32] = {0};
+
+	u16 scan_ch_ms;
+#define SCAN_PARAM_INPUT_FMT "%hu"
+#define SCAN_PARAM_INPUT_ARG , &scan_ch_ms
+#ifdef CONFIG_80211N_HT
+	u8 rx_ampdu_accept;
+	u16 rx_ampdu_size;
+#define SCAN_PARAM_INPUT_FMT_HT " %hhu %hu"
+#define SCAN_PARAM_INPUT_ARG_HT , &rx_ampdu_accept, &rx_ampdu_size
+#else
+#define SCAN_PARAM_INPUT_FMT_HT ""
+#define SCAN_PARAM_INPUT_ARG_HT
+#endif
+#ifdef CONFIG_SCAN_BACKOP
+	u16 backop_ms;
+	u8 scan_cnt_max;
+#define SCAN_PARAM_INPUT_FMT_BACKOP " %hu %hhu"
+#define SCAN_PARAM_INPUT_ARG_BACKOP , &backop_ms, &scan_cnt_max
+#else
+#define SCAN_PARAM_INPUT_FMT_BACKOP ""
+#define SCAN_PARAM_INPUT_ARG_BACKOP
+#endif
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp,
+			SCAN_PARAM_INPUT_FMT
+			SCAN_PARAM_INPUT_FMT_HT
+			SCAN_PARAM_INPUT_FMT_BACKOP
+			SCAN_PARAM_INPUT_ARG
+			SCAN_PARAM_INPUT_ARG_HT
+			SCAN_PARAM_INPUT_ARG_BACKOP
+		);
+
+		if (num-- > 0)
+			ss->scan_ch_ms = scan_ch_ms;
+#ifdef CONFIG_80211N_HT
+		if (num-- > 0)
+			ss->rx_ampdu_accept = rx_ampdu_accept;
+		if (num-- > 0)
+			ss->rx_ampdu_size = rx_ampdu_size;
+#endif
+#ifdef CONFIG_SCAN_BACKOP
+		if (num-- > 0)
+			ss->backop_ms = backop_ms;
+		if (num-- > 0)
+			ss->scan_cnt_max = scan_cnt_max;
+#endif
+	}
+
+	return count;
+}
+ssize_t proc_set_scan_abort(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u32 timeout = 0;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = 0;
+
+		num = sscanf(tmp, "%d", &timeout);
+		if (num != 1) {
+			RTW_INFO("invalid parameter!\n");
+			return count;
+		}
+
+		rtw_scan_abort(padapter, timeout);
+	}
+
+	return count;
+}
+
+int proc_get_scan_abort(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	return 0;
+}
+
+int proc_get_survey_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	dump_scanned_queue(m, adapter);
+
+	return 0;
+}
+
+ssize_t proc_set_survey_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 _status = _FALSE;
+	u8 ssc_chk;
+	char tmp[32] = {0};
+	char cmd[8] = {0};
+	bool acs = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%s", cmd);
+
+		if (num < 1)
+			return count;
+
+		if (strcmp("acs", cmd) == 0)
+			acs = 1;
+	}
+
+#if 1
+	ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+	if (ssc_chk != SS_ALLOW)
+		goto exit;
+
+	if (!rtw_is_adapter_up(padapter)) {
+		RTW_INFO("scan abort!! adapter cannot use\n");
+		goto exit;
+	}
+#else
+#ifdef CONFIG_MP_INCLUDED
+	if (rtw_mp_mode_check(padapter)) {
+		RTW_INFO("MP mode block Scan request\n");
+		goto exit;
+	}
+#endif
+	if (rtw_is_scan_deny(padapter)) {
+		RTW_INFO(FUNC_ADPT_FMT  ": scan deny\n", FUNC_ADPT_ARG(padapter));
+		goto exit;
+	}
+
+	if (!rtw_is_adapter_up(padapter)) {
+		RTW_INFO("scan abort!! adapter cannot use\n");
+		goto exit;
+	}
+
+	if (rtw_mi_busy_traffic_check(padapter)) {
+		RTW_INFO("scan abort!! BusyTraffic == _TRUE\n");
+		goto exit;
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) && check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+		RTW_INFO("scan abort!! AP mode process WPS\n");
+		goto exit;
+	}
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING) == _TRUE) {
+		RTW_INFO("scan abort!! fwstate=0x%x\n", pmlmepriv->fw_state);
+		goto exit;
+	}
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_mi_buddy_check_fwstate(padapter,
+		       WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING | WIFI_UNDER_WPS)) {
+		RTW_INFO("scan abort!! buddy_fwstate check failed\n");
+		goto exit;
+	}
+#endif
+#endif
+
+	if (acs) {
+		#ifdef CONFIG_RTW_ACS
+		_status = rtw_set_acs_sitesurvey(padapter);
+		#endif
+	} else
+		_status = rtw_sitesurvey_cmd(padapter, NULL);
+
+exit:
+	return count;
+}
+#ifdef PRIVATE_R
+int proc_get_infra_ap(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	struct sta_info *psta;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct ht_priv_infra_ap *phtpriv = &pmlmepriv->htpriv_infra_ap;
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv_infra_ap *pvhtpriv = &pmlmepriv->vhtpriv_infra_ap;
+#endif
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct wlan_network *cur_network = &(pmlmepriv->cur_network);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	if (MLME_IS_STA(padapter)) {
+		psta = rtw_get_stainfo(pstapriv, cur_network->network.MacAddress);
+		if (psta) {
+			unsigned int i, j;
+			unsigned int Rx_ss = 0, Tx_ss = 0;
+
+			RTW_PRINT_SEL(m, "SSID=%s\n", pmlmeinfo->network.Ssid.Ssid);
+			RTW_PRINT_SEL(m, "sta's macaddr:%s\n", get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr));
+			RTW_PRINT_SEL(m, "Supported rate=");
+			for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
+				if (pmlmeinfo->SupportedRates_infra_ap[i] == 0)
+					break;
+				RTW_PRINT_SEL(m, " 0x%x", pmlmeinfo->SupportedRates_infra_ap[i]);
+			}
+			RTW_PRINT_SEL(m, "\n");
+#ifdef CONFIG_80211N_HT
+			if (pmlmeinfo->ht_vht_received & BIT(0)) {
+				RTW_PRINT_SEL(m, "Supported MCS set=");
+				for (i = 0; i < 16 ; i++)
+					RTW_PRINT_SEL(m, " 0x%02x",  phtpriv->MCS_set_infra_ap[i]);
+				RTW_PRINT_SEL(m, "\n");
+				RTW_PRINT_SEL(m, "highest supported data rate=0x%x\n", phtpriv->rx_highest_data_rate_infra_ap);
+				RTW_PRINT_SEL(m, "HT_supported_channel_width_set=0x%x\n", phtpriv->channel_width_infra_ap);
+				RTW_PRINT_SEL(m, "sgi_20m=%d, sgi_40m=%d\n", phtpriv->sgi_20m_infra_ap, phtpriv->sgi_40m_infra_ap);
+				RTW_PRINT_SEL(m, "ldpc_cap=0x%x, stbc_cap=0x%x\n", phtpriv->ldpc_cap_infra_ap, phtpriv->stbc_cap_infra_ap);
+				RTW_PRINT_SEL(m, "HT_number_of_stream=%d\n", phtpriv->Rx_ss_infra_ap);
+			}
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+			if (pmlmeinfo->ht_vht_received & BIT(1)) {
+				RTW_PRINT_SEL(m, "VHT_supported_channel_width_set=0x%x\n", pvhtpriv->channel_width_infra_ap);
+				RTW_PRINT_SEL(m, "vht_ldpc_cap=0x%x, vht_stbc_cap=0x%x, vht_beamform_cap=0x%x\n", pvhtpriv->ldpc_cap_infra_ap, pvhtpriv->stbc_cap_infra_ap, pvhtpriv->beamform_cap_infra_ap);
+				RTW_PRINT_SEL(m, "Rx_vht_mcs_map=0x%x, Tx_vht_mcs_map=0x%x\n", *(u16 *)pvhtpriv->vht_mcs_map_infra_ap, *(u16 *)pvhtpriv->vht_mcs_map_tx_infra_ap);
+				RTW_PRINT_SEL(m, "VHT_number_of_stream=%d\n", pvhtpriv->number_of_streams_infra_ap);
+			}
+#endif
+		} else {
+			RTW_PRINT_SEL(m, "can't get sta's macaddr, cur_network's macaddr:%s\n",
+				get_macaddr_str(mac_addr_str, m, cur_network->network.MacAddress));
+		}
+	} else
+		RTW_PRINT_SEL(m, "this only applies to STA mode\n");
+	return 0;
+}
+
+#endif /* PRIVATE_R */
+
+static int wireless_mode_to_str(u32 mode, char *str)
+{
+	str[0]='\0';
+	if (mode & WLAN_MD_11A)
+		sprintf(str+strlen(str),"%s","A/");
+	if (mode & WLAN_MD_11B)
+		sprintf(str+strlen(str),"%s","B/");
+	if (mode & WLAN_MD_11G)
+		sprintf(str+strlen(str),"%s","G/");
+	#ifdef CONFIG_80211N_HT
+	if (mode & WLAN_MD_11N)
+		sprintf(str+strlen(str),"%s","N/");
+	#endif
+	#ifdef CONFIG_80211AC_VHT
+	if (mode & WLAN_MD_11AC)
+		sprintf(str+strlen(str),"%s","AC/");
+	#endif
+	#ifdef CONFIG_80211AX_HE
+	if (mode & WLAN_MD_11AX)
+		sprintf(str+strlen(str),"%s","AX/");
+	#endif
+	if (strlen(str)>1)
+		str[strlen(str)-1]='\0';
+
+	return strlen(str);
+}
+
+static void dump_ap_channel_rpt_ie(void *sel, struct link_mlme_ext_priv *pmlmeext)
+{
+	int i = 0;
+	if (pmlmeext->op_class != 0) {
+		RTW_PRINT_SEL(sel, "Operating Class=%d\n", pmlmeext->op_class);
+		while (i < MAX_CHANNEL_NUM && pmlmeext->channel_list[i] != 0) {
+			RTW_PRINT_SEL(sel, "Channel List = %d\n", pmlmeext->channel_list[i]);
+			i++;
+		}
+	}
+}
+
+static bool country_env_is_op_class(char env)
+{
+	switch (env) {
+	case ' ':
+	case 'O':
+	case 'I':
+	case 'X':
+		return false;
+	}
+
+	return true;;
+}
+
+static const char *country_env_str(char env)
+{
+	switch (env) {
+	case ' ':
+		return "ANY";
+	case 'O':
+		return "INDOOR";
+	case 'I':
+		return "OUTDOOR";
+	case 'X':
+		return "NONCOUNTRY";
+	default:
+		return "OPERATING_ClASS";
+	}
+}
+
+static void dump_country_element_ie(void *sel, struct link_mlme_ext_priv *pmlmeext)
+{
+	int i = 0;
+	/* check the country code valid or not.*/
+	if (is_alpha(pmlmeext->country[0]) == _FALSE
+	    || is_alpha(pmlmeext->country[1]) == _FALSE)
+		return;
+
+	RTW_PRINT_SEL(sel, "country=%c%c\n", pmlmeext->country[0], pmlmeext->country[1]);
+	if (country_env_is_op_class(pmlmeext->country[2]))
+		RTW_PRINT_SEL(sel, "Operating class=0x%02x\n", pmlmeext->country[2]);
+	else
+		RTW_PRINT_SEL(sel, "Environment=%s\n", country_env_str(pmlmeext->country[2]));
+
+	while (i < MAX_CHANNEL_NUM && pmlmeext->ch_info[i].first_ch != 0) {
+		RTW_PRINT_SEL(sel, "First channel number = %d ", pmlmeext->ch_info[i].first_ch);
+		RTW_PRINT_SEL(sel, "Number of channel = %d ", pmlmeext->ch_info[i].num_of_ch);
+		RTW_PRINT_SEL(sel, "Maximum Transmit power level = %d dBm\n", pmlmeext->ch_info[i].power_level);
+		i++;
+	}
+}
+
+int proc_get_ap_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	struct sta_info *psta;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct wlan_network *cur_network = &(pmlmepriv->cur_network);
+	struct mlme_ext_info	*pmlmeinfo = &(padapter->mlmeextpriv.mlmext_info);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	char wl_mode[16];
+	u32 mbps = 0;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	/* ap vendor */
+	char vendor[VENDOR_NAME_LEN] = {0};
+	get_assoc_AP_Vendor(vendor, pmlmeinfo->assoc_AP_vendor);
+	RTW_PRINT_SEL(m,"AP Vendor %s\n", vendor);
+
+	psta = rtw_get_stainfo(pstapriv, cur_network->network.MacAddress);
+	if (psta) {
+		mbps = ((u32)rtw_get_cur_max_rate(padapter) / 10);
+		wireless_mode_to_str(psta->phl_sta->wmode, wl_mode);
+		RTW_PRINT_SEL(m, "SSID=%s\n", cur_network->network.Ssid.Ssid);
+		RTW_PRINT_SEL(m, "sta's macaddr:%s\n", get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr));
+		RTW_PRINT_SEL(m, "Max Bit Rate=%u (Mbps)\n", mbps);
+		RTW_PRINT_SEL(m, "cur_channel=%d, cur_bwmode=%d(%s), cur_ch_offset=%d\n", pmlmeext->chandef.chan, pmlmeext->chandef.bw, ch_width_str(pmlmeext->chandef.bw), pmlmeext->chandef.offset);
+		RTW_PRINT_SEL(m, "wireless_mode=0x%x(%s), rtsen=%d, cts2slef=%d hw_rts_en=%d\n",
+				psta->phl_sta->wmode, wl_mode, psta->rtsen, psta->cts2self, psta->hw_rts_en);
+		/* ToDo: need API to query hal_sta->ra_info.rate_id */
+		/* RTW_PRINT_SEL(m, "state=0x%x, aid=%d, macid=%d, raid=%d\n",
+			psta->state, psta->phl_sta->aid, psta->phl_sta->macid, psta->phl_sta->ra_info.rate_id); */
+		RTW_PRINT_SEL(m, "state=0x%x, aid=%d, macid=%d\n",
+			psta->state, psta->phl_sta->aid, psta->phl_sta->macid);
+		if (MLME_IS_STA(padapter)) {
+			dump_country_element_ie(m, pmlmeext);
+			dump_ap_channel_rpt_ie(m, pmlmeext);
+		}
+#ifdef CONFIG_80211N_HT
+		RTW_PRINT_SEL(m, "qos_en=%d, ht_en=%d, init_rate=%d\n", psta->qos_option, psta->htpriv.ht_option, psta->init_rate);
+		RTW_PRINT_SEL(m, "bwmode=%d, ch_offset=%d, sgi_20m=%d,sgi_40m=%d\n"
+			, psta->phl_sta->chandef.bw, psta->htpriv.ch_offset, psta->htpriv.sgi_20m, psta->htpriv.sgi_40m);
+		RTW_PRINT_SEL(m, "ampdu_enable = %d\n", psta->ampdu_priv.ampdu_enable);
+		RTW_PRINT_SEL(m, "agg_enable_bitmap=%x, candidate_tid_bitmap=%x\n", psta->ampdu_priv.agg_enable_bitmap, psta->ampdu_priv.candidate_tid_bitmap);
+		RTW_PRINT_SEL(m, "ldpc_cap=0x%x, stbc_cap=0x%x, beamform_cap=0x%x\n", psta->htpriv.ldpc_cap, psta->htpriv.stbc_cap, psta->htpriv.beamform_cap);
+#endif /* CONFIG_80211N_HT */
+		if (MLME_IS_AP(padapter))
+			RTW_PRINT_SEL(m, " VHT or HE IE is configured by upper layer : %s\n", pmlmepriv->upper_layer_setting ? "True" : "False");
+#ifdef CONFIG_80211AC_VHT
+		RTW_PRINT_SEL(m, "vht_en=%u, vht_sgi_80m=%u\n", psta->vhtpriv.vht_option, psta->vhtpriv.sgi_80m);
+		RTW_PRINT_SEL(m, "vht_ldpc_cap=0x%x, vht_stbc_cap=0x%x, vht_beamform_cap=0x%x\n", psta->vhtpriv.ldpc_cap, psta->vhtpriv.stbc_cap, psta->vhtpriv.beamform_cap);
+		RTW_PRINT_SEL(m, "vht_mcs_map=0x%x, vht_highest_rate=0x%x, vht_ampdu_len=%d\n", *(u16 *)psta->vhtpriv.vht_mcs_map, psta->vhtpriv.vht_highest_rate, psta->vhtpriv.ampdu_len);
+		if (psta->vhtpriv.vht_option) {
+			RTW_MAP_DUMP_SEL_ALWAYS(m, "vht_cap=", psta->vhtpriv.vht_cap, 32);
+		} else {
+			RTW_PRINT_SEL(m, "vht_cap=N/A\n");
+		}
+#endif
+#ifdef CONFIG_80211AX_HE
+		RTW_PRINT_SEL(m, "he_en=%d\n", psta->hepriv.he_option);
+		if (psta->hepriv.he_option) {
+			RTW_MAP_DUMP_SEL_ALWAYS(m, "he_cap=", psta->hepriv.he_cap, HE_CAP_ELE_MAX_LEN);
+		} else {
+			RTW_PRINT_SEL(m, "he_cap=N/A\n");
+		}
+#endif
+#ifdef CONFIG_RECV_REORDERING_CTRL
+		sta_rx_reorder_ctl_dump(m, psta);
+#endif
+	} else {
+		RTW_PRINT_SEL(m, "can't get sta's macaddr, cur_network's macaddr:%s\n",
+			get_macaddr_str(mac_addr_str, m, cur_network->network.MacAddress));
+	}
+
+	return 0;
+}
+
+ssize_t proc_reset_trx_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct recv_info  *precvinfo = &padapter->recvinfo;
+	char cmd[32] = {0};
+	u8 cnt = 0;
+
+	if (count > sizeof(cmd)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(cmd, buffer, count)) {
+		int num = sscanf(cmd, "%hhx", &cnt);
+
+		if (num == 1 && cnt == 0) {
+			precvinfo->dbg_rx_ampdu_drop_count = 0;
+			precvinfo->dbg_rx_ampdu_forced_indicate_count = 0;
+			precvinfo->dbg_rx_ampdu_loss_count = 0;
+			precvinfo->dbg_rx_dup_mgt_frame_drop_count = 0;
+			precvinfo->dbg_rx_ampdu_window_shift_cnt = 0;
+			precvinfo->dbg_rx_conflic_mac_addr_cnt = 0;
+			precvinfo->dbg_rx_drop_count = 0;
+		}
+	}
+
+	return count;
+}
+
+int proc_get_trx_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	int i;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct recv_info *precvinfo = &padapter->recvinfo;
+	struct recv_priv  *precvpriv = &adapter_to_dvobj(padapter)->recvpriv;
+	struct hw_xmit *phwxmit;
+	struct trx_data_buf_q  *litexmitbuf_q = &dvobj->litexmitbuf_q;
+	struct trx_data_buf_q  *literecvbuf_q = &dvobj->literecvbuf_q;
+#ifdef CONFIG_USB_HCI
+	struct trx_urb_buf_q *xmit_urb_q = &dvobj->xmit_urb_q;
+	struct trx_urb_buf_q *recv_urb_q = &dvobj->recv_urb_q;
+#endif
+	u16 vo_params[4]={0}, vi_params[4]={0}, be_params[4]={0}, bk_params[4]={0};
+
+	rtw_hal_read_edca(padapter, vo_params, vi_params, be_params, bk_params);
+
+	RTW_PRINT_SEL(m, "wmm_edca_vo, aifs = %u us, cw_min = %u, cw_max = %u, txop_limit = %u us\n", vo_params[0], vo_params[1], vo_params[2], vo_params[3]);
+	RTW_PRINT_SEL(m, "wmm_edca_vi, aifs = %u us, cw_min = %u, cw_max = %u, txop_limit = %u us\n", vi_params[0], vi_params[1], vi_params[2], vi_params[3]);
+	RTW_PRINT_SEL(m, "wmm_edca_be, aifs = %u us, cw_min = %u, cw_max = %u, txop_limit = %u us\n", be_params[0], be_params[1], be_params[2], be_params[3]);
+	RTW_PRINT_SEL(m, "wmm_edca_bk, aifs = %u us, cw_min = %u, cw_max = %u, txop_limit = %u us\n", bk_params[0], bk_params[1], bk_params[2], bk_params[3]);
+
+	dump_os_queue(m, padapter);
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	RTW_PRINT_SEL(m, "free_xmitbuf_cnt=%d\n"
+		, pxmitpriv->free_xmitbuf_cnt);
+	RTW_PRINT_SEL(m, "free_ext_xmitbuf_cnt=%d\n"
+		, pxmitpriv->free_xmit_extbuf_cnt);
+	#endif
+
+	RTW_PRINT_SEL(m, "free_xmitframe_cnt=%d\n"
+		, pxmitpriv->free_xmitframe_cnt);
+	RTW_PRINT_SEL(m, "free_xframe_ext_cnt=%d\n"
+		, pxmitpriv->free_xframe_ext_cnt);
+	RTW_PRINT_SEL(m, "free_recvframe_cnt=%d\n"
+		      , precvpriv->free_recvframe_cnt);
+
+	RTW_PRINT_SEL(m, "free_litexmitbuf_cnt=%d\n"
+		, litexmitbuf_q->free_data_buf_cnt);
+	RTW_PRINT_SEL(m, "free_recvbuf_cnt=%d\n"
+		, literecvbuf_q->free_data_buf_cnt);
+
+	for (i = 0; i < 4; i++) {
+		phwxmit = pxmitpriv->hwxmits + i;
+		RTW_PRINT_SEL(m, "%d, hwq.accnt=%d\n", i, phwxmit->accnt);
+	}
+
+	rtw_hal_get_hwreg(padapter, HW_VAR_DUMP_MAC_TXFIFO, (u8 *)m);
+
+#ifdef CONFIG_USB_HCI
+	RTW_PRINT_SEL(m, "tx_urb_cnt=%d\n", xmit_urb_q->free_urb_buf_cnt);
+	RTW_PRINT_SEL(m, "rx_urb_cnt=%d\n", recv_urb_q->free_urb_buf_cnt);
+	RTW_PRINT_SEL(m, "rx_urb_pending_cn=%d\n", ATOMIC_READ(&(dvobj->rx_pending_cnt)));
+#endif
+
+	/* Folowing are RX info */
+	RTW_PRINT_SEL(m, "RX: Count of Packets dropped by Driver: %llu\n", (unsigned long long)precvinfo->dbg_rx_drop_count);
+	/* Counts of packets whose seq_num is less than preorder_ctrl->indicate_seq, Ex delay, retransmission, redundant packets and so on */
+	RTW_PRINT_SEL(m, "Rx: Counts of Packets Whose Seq_Num Less Than Reorder Control Seq_Num: %llu\n", (unsigned long long)precvinfo->dbg_rx_ampdu_drop_count);
+	/* How many times the Rx Reorder Timer is triggered. */
+	RTW_PRINT_SEL(m, "Rx: Reorder Time-out Trigger Counts: %llu\n", (unsigned long long)precvinfo->dbg_rx_ampdu_forced_indicate_count);
+	/* Total counts of packets loss */
+	RTW_PRINT_SEL(m, "Rx: Packet Loss Counts: %llu\n", (unsigned long long)precvinfo->dbg_rx_ampdu_loss_count);
+	RTW_PRINT_SEL(m, "Rx: Duplicate Management Frame Drop Count: %llu\n", (unsigned long long)precvinfo->dbg_rx_dup_mgt_frame_drop_count);
+	RTW_PRINT_SEL(m, "Rx: AMPDU BA window shift Count: %llu\n", (unsigned long long)precvinfo->dbg_rx_ampdu_window_shift_cnt);
+	/*The same mac addr counts*/
+	RTW_PRINT_SEL(m, "Rx: Conflict MAC Address Frames Count: %llu\n", (unsigned long long)precvinfo->dbg_rx_conflic_mac_addr_cnt);
+	return 0;
+}
+
+static const char *rtw_data_rate_str(enum rtw_data_rate rate)
+{
+	if (rate >= RTW_DATA_RATE_CCK1 && rate <= RTW_DATA_RATE_CCK11) {
+		switch (rate) {
+#define CASE_CCK_RATE(cck)	case RTW_DATA_RATE_CCK ## cck: return "CCK_" # cck "M"
+		CASE_CCK_RATE(1);
+		CASE_CCK_RATE(2);
+		CASE_CCK_RATE(5_5);
+		CASE_CCK_RATE(11);
+#undef CASE_CCK_RATE
+		default:
+			return "CCK_UNKNOWN";
+		}
+	} else if (rate >= RTW_DATA_RATE_OFDM6 && rate <= RTW_DATA_RATE_OFDM54) {
+		switch (rate) {
+#define CASE_OFDM_RATE(ofdm)	case RTW_DATA_RATE_OFDM ## ofdm: return "OFDM_" # ofdm "M"
+		CASE_OFDM_RATE(6);
+		CASE_OFDM_RATE(9);
+		CASE_OFDM_RATE(12);
+		CASE_OFDM_RATE(18);
+		CASE_OFDM_RATE(24);
+		CASE_OFDM_RATE(36);
+		CASE_OFDM_RATE(48);
+		CASE_OFDM_RATE(54);
+#undef CASE_OFDM_RATE
+		default:
+			return "OFDM_UNKNOWN";
+		}
+	} else if (rate >= RTW_DATA_RATE_MCS0 && rate <= RTW_DATA_RATE_MCS31) {
+		switch (rate) {
+#define CASE_HT_RATE(mcs)	case RTW_DATA_RATE_MCS ## mcs: return "MCS_" # mcs
+		CASE_HT_RATE(0);
+		CASE_HT_RATE(1);
+		CASE_HT_RATE(2);
+		CASE_HT_RATE(3);
+		CASE_HT_RATE(4);
+		CASE_HT_RATE(5);
+		CASE_HT_RATE(6);
+		CASE_HT_RATE(7);
+		CASE_HT_RATE(8);
+		CASE_HT_RATE(9);
+		CASE_HT_RATE(10);
+		CASE_HT_RATE(11);
+		CASE_HT_RATE(12);
+		CASE_HT_RATE(13);
+		CASE_HT_RATE(14);
+		CASE_HT_RATE(15);
+		CASE_HT_RATE(16);
+		CASE_HT_RATE(17);
+		CASE_HT_RATE(18);
+		CASE_HT_RATE(19);
+		CASE_HT_RATE(20);
+		CASE_HT_RATE(21);
+		CASE_HT_RATE(22);
+		CASE_HT_RATE(23);
+		CASE_HT_RATE(24);
+		CASE_HT_RATE(25);
+		CASE_HT_RATE(26);
+		CASE_HT_RATE(27);
+		CASE_HT_RATE(28);
+		CASE_HT_RATE(29);
+		CASE_HT_RATE(30);
+		CASE_HT_RATE(31);
+#undef CASE_HT_RATE
+		default:
+			return "HT_UNKNOWN";
+		}
+	} else if (rate >= RTW_DATA_RATE_VHT_NSS1_MCS0 && rate <= RTW_DATA_RATE_VHT_NSS4_MCS9) {
+		switch (rate) {
+#define CASE_VHT_RATE(ss, mcs)	case RTW_DATA_RATE_VHT_NSS ## ss ##_MCS ##mcs: return "VHT_SS" #ss "MCS" #mcs
+		CASE_VHT_RATE(1, 0);
+		CASE_VHT_RATE(1, 1);
+		CASE_VHT_RATE(1, 2);
+		CASE_VHT_RATE(1, 3);
+		CASE_VHT_RATE(1, 4);
+		CASE_VHT_RATE(1, 5);
+		CASE_VHT_RATE(1, 6);
+		CASE_VHT_RATE(1, 7);
+		CASE_VHT_RATE(1, 8);
+		CASE_VHT_RATE(1, 9);
+		CASE_VHT_RATE(2, 0);
+		CASE_VHT_RATE(2, 1);
+		CASE_VHT_RATE(2, 2);
+		CASE_VHT_RATE(2, 3);
+		CASE_VHT_RATE(2, 4);
+		CASE_VHT_RATE(2, 5);
+		CASE_VHT_RATE(2, 6);
+		CASE_VHT_RATE(2, 7);
+		CASE_VHT_RATE(2, 8);
+		CASE_VHT_RATE(2, 9);
+		CASE_VHT_RATE(3, 0);
+		CASE_VHT_RATE(3, 1);
+		CASE_VHT_RATE(3, 2);
+		CASE_VHT_RATE(3, 3);
+		CASE_VHT_RATE(3, 4);
+		CASE_VHT_RATE(3, 5);
+		CASE_VHT_RATE(3, 6);
+		CASE_VHT_RATE(3, 7);
+		CASE_VHT_RATE(3, 8);
+		CASE_VHT_RATE(3, 9);
+		CASE_VHT_RATE(4, 0);
+		CASE_VHT_RATE(4, 1);
+		CASE_VHT_RATE(4, 2);
+		CASE_VHT_RATE(4, 3);
+		CASE_VHT_RATE(4, 4);
+		CASE_VHT_RATE(4, 5);
+		CASE_VHT_RATE(4, 6);
+		CASE_VHT_RATE(4, 7);
+		CASE_VHT_RATE(4, 8);
+		CASE_VHT_RATE(4, 9);
+#undef CASE_VHT_RATE
+		default:
+			return "VHT_UNKNOWN";
+		}
+	} else if (rate >= RTW_DATA_RATE_HE_NSS1_MCS0 && rate <= RTW_DATA_RATE_HE_NSS4_MCS11) {
+		switch (rate) {
+#define CASE_HE_RATE(ss, mcs)	case RTW_DATA_RATE_HE_NSS ## ss ##_MCS ##mcs: return "HE_SS" #ss "MCS" #mcs
+		CASE_HE_RATE(1, 0);
+		CASE_HE_RATE(1, 1);
+		CASE_HE_RATE(1, 2);
+		CASE_HE_RATE(1, 3);
+		CASE_HE_RATE(1, 4);
+		CASE_HE_RATE(1, 5);
+		CASE_HE_RATE(1, 6);
+		CASE_HE_RATE(1, 7);
+		CASE_HE_RATE(1, 8);
+		CASE_HE_RATE(1, 9);
+		CASE_HE_RATE(1, 10);
+		CASE_HE_RATE(1, 11);
+		CASE_HE_RATE(2, 0);
+		CASE_HE_RATE(2, 1);
+		CASE_HE_RATE(2, 2);
+		CASE_HE_RATE(2, 3);
+		CASE_HE_RATE(2, 4);
+		CASE_HE_RATE(2, 5);
+		CASE_HE_RATE(2, 6);
+		CASE_HE_RATE(2, 7);
+		CASE_HE_RATE(2, 8);
+		CASE_HE_RATE(2, 9);
+		CASE_HE_RATE(2, 10);
+		CASE_HE_RATE(2, 11);
+		CASE_HE_RATE(3, 0);
+		CASE_HE_RATE(3, 1);
+		CASE_HE_RATE(3, 2);
+		CASE_HE_RATE(3, 3);
+		CASE_HE_RATE(3, 4);
+		CASE_HE_RATE(3, 5);
+		CASE_HE_RATE(3, 6);
+		CASE_HE_RATE(3, 7);
+		CASE_HE_RATE(3, 8);
+		CASE_HE_RATE(3, 9);
+		CASE_HE_RATE(3, 10);
+		CASE_HE_RATE(3, 11);
+		CASE_HE_RATE(4, 0);
+		CASE_HE_RATE(4, 1);
+		CASE_HE_RATE(4, 2);
+		CASE_HE_RATE(4, 3);
+		CASE_HE_RATE(4, 4);
+		CASE_HE_RATE(4, 5);
+		CASE_HE_RATE(4, 6);
+		CASE_HE_RATE(4, 7);
+		CASE_HE_RATE(4, 8);
+		CASE_HE_RATE(4, 9);
+		CASE_HE_RATE(4, 10);
+		CASE_HE_RATE(4, 11);
+#undef CASE_HE_RATE
+		default:
+			return "HE_UNKNOWN";
+		}
+	}
+
+	return "ALL_UNKNOWN";
+}
+
+int proc_get_rate_ctl(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	u16 data_rate = 0;
+	u8 sgi = 0, data_fb = 0;
+
+	if (adapter->fix_rate != NO_FIX_RATE) {
+		data_rate = GET_FIX_RATE(adapter->fix_rate);
+		sgi = GET_FIX_RATE_SGI(adapter->fix_rate);
+		data_fb = adapter->data_fb ? 1 : 0;
+		RTW_PRINT_SEL(m, "FIXED %s%s%s\n"
+			, rtw_data_rate_str(data_rate)
+			, data_rate >= RTW_DATA_RATE_MCS0 ? (sgi ? " SGI" : " LGI") : ""
+			, data_fb ? " FB" : ""
+		);
+		RTW_PRINT_SEL(m, "0x%02x %u\n", adapter->fix_rate, adapter->data_fb);
+	} else
+		RTW_PRINT_SEL(m, "RA\n");
+
+	return 0;
+}
+
+ssize_t proc_set_rate_ctl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u16 fix_rate = NO_FIX_RATE;
+	u8 data_fb = 0;
+
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hx %hhu", &fix_rate, &data_fb);
+
+		if (num >= 1) {
+			u16 fix_rate_ori = adapter->fix_rate;
+
+			adapter->fix_rate = fix_rate;
+			#if 0 /*GEORGIA_TODO_FIXIT*/
+			if (fix_rate == 0xFF)
+				hal_data->ForcedDataRate = 0;
+			else
+				hal_data->ForcedDataRate = hwrate_to_mrate(fix_rate & 0x7F);
+			#endif
+			if (adapter->fix_bw != NO_FIX_BW && fix_rate_ori != fix_rate)
+				rtw_run_in_thread_cmd(adapter, ((void *)(rtw_update_tx_rate_bmp)), adapter_to_dvobj(adapter));
+		}
+		if (num >= 2)
+			adapter->data_fb = data_fb ? 1 : 0;
+	}
+
+	return count;
+}
+
+#ifdef CONFIG_AP_MODE
+int proc_get_bmc_tx_rate(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (!MLME_IS_AP(adapter) && !MLME_IS_MESH(adapter)) {
+		RTW_PRINT_SEL(m, "[ERROR] Not in SoftAP/Mesh mode !!\n");
+		return 0;
+	}
+
+	RTW_PRINT_SEL(m, " BMC Tx rate - %s\n", MGN_RATE_STR(adapter->bmc_tx_rate));
+	return 0;
+}
+
+ssize_t proc_set_bmc_tx_rate(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 bmc_tx_rate;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx", &bmc_tx_rate);
+
+		if (num >= 1)
+			/*adapter->bmc_tx_rate = hwrate_to_mrate(bmc_tx_rate);*/
+			adapter->bmc_tx_rate = bmc_tx_rate;
+	}
+
+	return count;
+}
+#endif /*CONFIG_AP_MODE*/
+
+
+int proc_get_tx_power_offset(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "Tx power offset - %u\n", adapter->power_offset);
+	return 0;
+}
+
+ssize_t proc_set_tx_power_offset(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 power_offset = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu", &power_offset);
+
+		if (num >= 1) {
+			if (power_offset > 5)
+				power_offset = 0;
+
+			adapter->power_offset = power_offset;
+		}
+	}
+
+	return count;
+}
+
+int proc_get_bw_ctl(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 data_bw = 0;
+
+	if (adapter->fix_bw != NO_FIX_BW) {
+		data_bw = adapter->fix_bw;
+		RTW_PRINT_SEL(m, "FIXED %s\n", ch_width_str(data_bw));
+	} else
+		RTW_PRINT_SEL(m, "Auto\n");
+
+	return 0;
+}
+
+ssize_t proc_set_bw_ctl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 fix_bw;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhu", &fix_bw);
+
+		if (num >= 1) {
+			u8 fix_bw_ori = adapter->fix_bw;
+
+			adapter->fix_bw = fix_bw;
+
+			if (adapter->fix_rate != NO_FIX_RATE && fix_bw_ori != fix_bw)
+				rtw_run_in_thread_cmd(adapter, ((void *)(rtw_update_tx_rate_bmp)), adapter_to_dvobj(adapter));
+		}
+	}
+
+	return count;
+}
+
+#ifdef DBG_RX_COUNTER_DUMP
+int proc_get_rx_cnt_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	int i;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "BIT0- Dump RX counters of DRV\n");
+	RTW_PRINT_SEL(m, "BIT1- Dump RX counters of MAC\n");
+	RTW_PRINT_SEL(m, "BIT2- Dump RX counters of PHY\n");
+	RTW_PRINT_SEL(m, "BIT3- Dump TRX data frame of DRV\n");
+	RTW_PRINT_SEL(m, "dump_rx_cnt_mode = 0x%02x\n", adapter->dump_rx_cnt_mode);
+
+	return 0;
+}
+ssize_t proc_set_rx_cnt_dump(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 dump_rx_cnt_mode;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx", &dump_rx_cnt_mode);
+
+		if (num == 1) {
+			rtw_dump_phy_rxcnts_preprocess(adapter, dump_rx_cnt_mode);
+			adapter->dump_rx_cnt_mode = dump_rx_cnt_mode;
+		}
+	}
+
+	return count;
+}
+#endif
+
+
+ssize_t proc_set_del_rx_ampdu_test_case(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count))
+		sscanf(tmp, "%hhu", &del_rx_ampdu_test_no_tx_fail);
+
+	return count;
+}
+
+ssize_t proc_set_wait_hiq_empty(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count))
+		sscanf(tmp, "%u", &g_wait_hiq_empty_ms);
+
+	return count;
+}
+
+ssize_t proc_set_sta_linking_test(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u32 wait_ms = 0;
+		u8 force_fail = 0;
+		int num = sscanf(tmp, "%u %hhu", &wait_ms, &force_fail);
+
+		if (num >= 1)
+			sta_linking_test_wait_ms = wait_ms;
+		if (num >= 2)
+			sta_linking_test_force_fail = force_fail;
+	}
+
+	return count;
+}
+
+#ifdef CONFIG_AP_MODE
+ssize_t proc_set_ap_linking_test(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u16 force_auth_fail = 0;
+		u16 force_asoc_fail = 0;
+		int num = sscanf(tmp, "%hu %hu", &force_auth_fail, &force_asoc_fail);
+
+		if (num >= 1)
+			ap_linking_test_force_auth_fail = force_auth_fail;
+		if (num >= 2)
+			ap_linking_test_force_asoc_fail = force_asoc_fail;
+	}
+
+	return count;
+}
+#endif /* CONFIG_AP_MODE */
+
+int proc_get_ps_dbg_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &dvobj->drv_dbg;
+
+	RTW_PRINT_SEL(m, "dbg_sdio_alloc_irq_cnt=%d\n", pdbgpriv->dbg_sdio_alloc_irq_cnt);
+	RTW_PRINT_SEL(m, "dbg_sdio_free_irq_cnt=%d\n", pdbgpriv->dbg_sdio_free_irq_cnt);
+	RTW_PRINT_SEL(m, "dbg_sdio_alloc_irq_error_cnt=%d\n", pdbgpriv->dbg_sdio_alloc_irq_error_cnt);
+	RTW_PRINT_SEL(m, "dbg_sdio_free_irq_error_cnt=%d\n", pdbgpriv->dbg_sdio_free_irq_error_cnt);
+	RTW_PRINT_SEL(m, "dbg_sdio_init_error_cnt=%d\n", pdbgpriv->dbg_sdio_init_error_cnt);
+	RTW_PRINT_SEL(m, "dbg_sdio_deinit_error_cnt=%d\n", pdbgpriv->dbg_sdio_deinit_error_cnt);
+	RTW_PRINT_SEL(m, "dbg_suspend_error_cnt=%d\n", pdbgpriv->dbg_suspend_error_cnt);
+	RTW_PRINT_SEL(m, "dbg_suspend_cnt=%d\n", pdbgpriv->dbg_suspend_cnt);
+	RTW_PRINT_SEL(m, "dbg_resume_cnt=%d\n", pdbgpriv->dbg_resume_cnt);
+	RTW_PRINT_SEL(m, "dbg_resume_error_cnt=%d\n", pdbgpriv->dbg_resume_error_cnt);
+	RTW_PRINT_SEL(m, "dbg_deinit_fail_cnt=%d\n", pdbgpriv->dbg_deinit_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_carddisable_cnt=%d\n", pdbgpriv->dbg_carddisable_cnt);
+	RTW_PRINT_SEL(m, "dbg_ps_insuspend_cnt=%d\n", pdbgpriv->dbg_ps_insuspend_cnt);
+	RTW_PRINT_SEL(m, "dbg_dev_unload_inIPS_cnt=%d\n", pdbgpriv->dbg_dev_unload_inIPS_cnt);
+	RTW_PRINT_SEL(m, "dbg_scan_pwr_state_cnt=%d\n", pdbgpriv->dbg_scan_pwr_state_cnt);
+	RTW_PRINT_SEL(m, "dbg_downloadfw_pwr_state_cnt=%d\n", pdbgpriv->dbg_downloadfw_pwr_state_cnt);
+	RTW_PRINT_SEL(m, "dbg_carddisable_error_cnt=%d\n", pdbgpriv->dbg_carddisable_error_cnt);
+	RTW_PRINT_SEL(m, "dbg_fw_read_ps_state_fail_cnt=%d\n", pdbgpriv->dbg_fw_read_ps_state_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_leave_ips_fail_cnt=%d\n", pdbgpriv->dbg_leave_ips_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_leave_lps_fail_cnt=%d\n", pdbgpriv->dbg_leave_lps_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_h2c_leave32k_fail_cnt=%d\n", pdbgpriv->dbg_h2c_leave32k_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_diswow_dload_fw_fail_cnt=%d\n", pdbgpriv->dbg_diswow_dload_fw_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_enwow_dload_fw_fail_cnt=%d\n", pdbgpriv->dbg_enwow_dload_fw_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_ips_drvopen_fail_cnt=%d\n", pdbgpriv->dbg_ips_drvopen_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_poll_fail_cnt=%d\n", pdbgpriv->dbg_poll_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_rpwm_toogle_cnt=%d\n", pdbgpriv->dbg_rpwm_toogle_cnt);
+	RTW_PRINT_SEL(m, "dbg_rpwm_timeout_fail_cnt=%d\n", pdbgpriv->dbg_rpwm_timeout_fail_cnt);
+	RTW_PRINT_SEL(m, "dbg_fw_mem_dl_error_cnt=%d\n", pdbgpriv->dbg_fw_mem_dl_error_cnt);
+
+	return 0;
+}
+ssize_t proc_set_ps_dbg_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter->dvobj;
+	struct debug_priv *pdbgpriv = &dvobj->drv_dbg;
+	char tmp[32];
+	u8 ps_dbg_cmd_id;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx", &ps_dbg_cmd_id);
+
+		if (num == 1 && ps_dbg_cmd_id == 1) /*Clean all*/
+			_rtw_memset(pdbgpriv, 0, sizeof(struct debug_priv));
+
+	}
+
+	return count;
+}
+
+
+#ifdef CONFIG_DBG_COUNTER
+
+int proc_get_rx_logs(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rx_logs *rx_logs = &padapter->rx_logs;
+
+	RTW_PRINT_SEL(m,
+		      "intf_rx=%d\n"
+		      "intf_rx_err_recvframe=%d\n"
+		      "intf_rx_err_skb=%d\n"
+		      "intf_rx_report=%d\n"
+		      "core_rx=%d\n"
+		      "core_rx_pre=%d\n"
+		      "core_rx_pre_ver_err=%d\n"
+		      "core_rx_pre_mgmt=%d\n"
+		      "core_rx_pre_mgmt_err_80211w=%d\n"
+		      "core_rx_pre_mgmt_err=%d\n"
+		      "core_rx_pre_ctrl=%d\n"
+		      "core_rx_pre_ctrl_err=%d\n"
+		      "core_rx_pre_data=%d\n"
+		      "core_rx_pre_data_wapi_seq_err=%d\n"
+		      "core_rx_pre_data_wapi_key_err=%d\n"
+		      "core_rx_pre_data_handled=%d\n"
+		      "core_rx_pre_data_err=%d\n"
+		      "core_rx_pre_data_unknown=%d\n"
+		      "core_rx_pre_unknown=%d\n"
+		      "core_rx_enqueue=%d\n"
+		      "core_rx_dequeue=%d\n"
+		      "core_rx_post=%d\n"
+		      "core_rx_post_decrypt=%d\n"
+		      "core_rx_post_decrypt_wep=%d\n"
+		      "core_rx_post_decrypt_tkip=%d\n"
+		      "core_rx_post_decrypt_aes=%d\n"
+		      "core_rx_post_decrypt_wapi=%d\n"
+		      "core_rx_post_decrypt_hw=%d\n"
+		      "core_rx_post_decrypt_unknown=%d\n"
+		      "core_rx_post_decrypt_err=%d\n"
+		      "core_rx_post_defrag_err=%d\n"
+		      "core_rx_post_portctrl_err=%d\n"
+		      "core_rx_post_indicate=%d\n"
+		      "core_rx_post_indicate_in_oder=%d\n"
+		      "core_rx_post_indicate_reoder=%d\n"
+		      "core_rx_post_indicate_err=%d\n"
+		      "os_indicate=%d\n"
+		      "os_indicate_ap_mcast=%d\n"
+		      "os_indicate_ap_forward=%d\n"
+		      "os_indicate_ap_self=%d\n"
+		      "os_indicate_err=%d\n"
+		      "os_netif_ok=%d\n"
+		      "os_netif_err=%d\n",
+		      rx_logs->intf_rx,
+		      rx_logs->intf_rx_err_recvframe,
+		      rx_logs->intf_rx_err_skb,
+		      rx_logs->intf_rx_report,
+		      rx_logs->core_rx,
+		      rx_logs->core_rx_pre,
+		      rx_logs->core_rx_pre_ver_err,
+		      rx_logs->core_rx_pre_mgmt,
+		      rx_logs->core_rx_pre_mgmt_err_80211w,
+		      rx_logs->core_rx_pre_mgmt_err,
+		      rx_logs->core_rx_pre_ctrl,
+		      rx_logs->core_rx_pre_ctrl_err,
+		      rx_logs->core_rx_pre_data,
+		      rx_logs->core_rx_pre_data_wapi_seq_err,
+		      rx_logs->core_rx_pre_data_wapi_key_err,
+		      rx_logs->core_rx_pre_data_handled,
+		      rx_logs->core_rx_pre_data_err,
+		      rx_logs->core_rx_pre_data_unknown,
+		      rx_logs->core_rx_pre_unknown,
+		      rx_logs->core_rx_enqueue,
+		      rx_logs->core_rx_dequeue,
+		      rx_logs->core_rx_post,
+		      rx_logs->core_rx_post_decrypt,
+		      rx_logs->core_rx_post_decrypt_wep,
+		      rx_logs->core_rx_post_decrypt_tkip,
+		      rx_logs->core_rx_post_decrypt_aes,
+		      rx_logs->core_rx_post_decrypt_wapi,
+		      rx_logs->core_rx_post_decrypt_hw,
+		      rx_logs->core_rx_post_decrypt_unknown,
+		      rx_logs->core_rx_post_decrypt_err,
+		      rx_logs->core_rx_post_defrag_err,
+		      rx_logs->core_rx_post_portctrl_err,
+		      rx_logs->core_rx_post_indicate,
+		      rx_logs->core_rx_post_indicate_in_oder,
+		      rx_logs->core_rx_post_indicate_reoder,
+		      rx_logs->core_rx_post_indicate_err,
+		      rx_logs->os_indicate,
+		      rx_logs->os_indicate_ap_mcast,
+		      rx_logs->os_indicate_ap_forward,
+		      rx_logs->os_indicate_ap_self,
+		      rx_logs->os_indicate_err,
+		      rx_logs->os_netif_ok,
+		      rx_logs->os_netif_err
+		     );
+
+	return 0;
+}
+
+int proc_get_tx_logs(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tx_logs *tx_logs = &padapter->tx_logs;
+
+	RTW_PRINT_SEL(m,
+		      "os_tx=%d\n"
+		      "os_tx_err_up=%d\n"
+		      "os_tx_err_xmit=%d\n"
+		      "os_tx_m2u=%d\n"
+		      "os_tx_m2u_ignore_fw_linked=%d\n"
+		      "os_tx_m2u_ignore_self=%d\n"
+		      "os_tx_m2u_entry=%d\n"
+		      "os_tx_m2u_entry_err_xmit=%d\n"
+		      "os_tx_m2u_entry_err_skb=%d\n"
+		      "os_tx_m2u_stop=%d\n"
+		      "core_tx=%d\n"
+		      "core_tx_err_pxmitframe=%d\n"
+		      "core_tx_err_brtx=%d\n"
+		      "core_tx_upd_attrib=%d\n"
+		      "core_tx_upd_attrib_adhoc=%d\n"
+		      "core_tx_upd_attrib_sta=%d\n"
+		      "core_tx_upd_attrib_ap=%d\n"
+		      "core_tx_upd_attrib_unknown=%d\n"
+		      "core_tx_upd_attrib_dhcp=%d\n"
+		      "core_tx_upd_attrib_icmp=%d\n"
+		      "core_tx_upd_attrib_active=%d\n"
+		      "core_tx_upd_attrib_err_ucast_sta=%d\n"
+		      "core_tx_upd_attrib_err_ucast_ap_link=%d\n"
+		      "core_tx_upd_attrib_err_sta=%d\n"
+		      "core_tx_upd_attrib_err_link=%d\n"
+		      "core_tx_upd_attrib_err_sec=%d\n"
+		      "core_tx_ap_enqueue_warn_fwstate=%d\n"
+		      "core_tx_ap_enqueue_warn_sta=%d\n"
+		      "core_tx_ap_enqueue_warn_nosta=%d\n"
+		      "core_tx_ap_enqueue_warn_link=%d\n"
+		      "core_tx_ap_enqueue_warn_trigger=%d\n"
+		      "core_tx_ap_enqueue_mcast=%d\n"
+		      "core_tx_ap_enqueue_ucast=%d\n"
+		      "core_tx_ap_enqueue=%d\n"
+		      "intf_tx=%d\n"
+		      "intf_tx_pending_ac=%d\n"
+		      "intf_tx_pending_fw_under_survey=%d\n"
+		      "intf_tx_pending_fw_under_linking=%d\n"
+		      "intf_tx_pending_xmitbuf=%d\n"
+		      "intf_tx_enqueue=%d\n"
+		      "core_tx_enqueue=%d\n"
+		      "core_tx_enqueue_class=%d\n"
+		      "core_tx_enqueue_class_err_sta=%d\n"
+		      "core_tx_enqueue_class_err_nosta=%d\n"
+		      "core_tx_enqueue_class_err_fwlink=%d\n"
+		      "intf_tx_direct=%d\n"
+		      "intf_tx_direct_err_coalesce=%d\n"
+		      "intf_tx_dequeue=%d\n"
+		      "intf_tx_dequeue_err_coalesce=%d\n"
+		      "intf_tx_dump_xframe=%d\n"
+		      "intf_tx_dump_xframe_err_txdesc=%d\n"
+		      "intf_tx_dump_xframe_err_port=%d\n",
+		      tx_logs->os_tx,
+		      tx_logs->os_tx_err_up,
+		      tx_logs->os_tx_err_xmit,
+		      tx_logs->os_tx_m2u,
+		      tx_logs->os_tx_m2u_ignore_fw_linked,
+		      tx_logs->os_tx_m2u_ignore_self,
+		      tx_logs->os_tx_m2u_entry,
+		      tx_logs->os_tx_m2u_entry_err_xmit,
+		      tx_logs->os_tx_m2u_entry_err_skb,
+		      tx_logs->os_tx_m2u_stop,
+		      tx_logs->core_tx,
+		      tx_logs->core_tx_err_pxmitframe,
+		      tx_logs->core_tx_err_brtx,
+		      tx_logs->core_tx_upd_attrib,
+		      tx_logs->core_tx_upd_attrib_adhoc,
+		      tx_logs->core_tx_upd_attrib_sta,
+		      tx_logs->core_tx_upd_attrib_ap,
+		      tx_logs->core_tx_upd_attrib_unknown,
+		      tx_logs->core_tx_upd_attrib_dhcp,
+		      tx_logs->core_tx_upd_attrib_icmp,
+		      tx_logs->core_tx_upd_attrib_active,
+		      tx_logs->core_tx_upd_attrib_err_ucast_sta,
+		      tx_logs->core_tx_upd_attrib_err_ucast_ap_link,
+		      tx_logs->core_tx_upd_attrib_err_sta,
+		      tx_logs->core_tx_upd_attrib_err_link,
+		      tx_logs->core_tx_upd_attrib_err_sec,
+		      tx_logs->core_tx_ap_enqueue_warn_fwstate,
+		      tx_logs->core_tx_ap_enqueue_warn_sta,
+		      tx_logs->core_tx_ap_enqueue_warn_nosta,
+		      tx_logs->core_tx_ap_enqueue_warn_link,
+		      tx_logs->core_tx_ap_enqueue_warn_trigger,
+		      tx_logs->core_tx_ap_enqueue_mcast,
+		      tx_logs->core_tx_ap_enqueue_ucast,
+		      tx_logs->core_tx_ap_enqueue,
+		      tx_logs->intf_tx,
+		      tx_logs->intf_tx_pending_ac,
+		      tx_logs->intf_tx_pending_fw_under_survey,
+		      tx_logs->intf_tx_pending_fw_under_linking,
+		      tx_logs->intf_tx_pending_xmitbuf,
+		      tx_logs->intf_tx_enqueue,
+		      tx_logs->core_tx_enqueue,
+		      tx_logs->core_tx_enqueue_class,
+		      tx_logs->core_tx_enqueue_class_err_sta,
+		      tx_logs->core_tx_enqueue_class_err_nosta,
+		      tx_logs->core_tx_enqueue_class_err_fwlink,
+		      tx_logs->intf_tx_direct,
+		      tx_logs->intf_tx_direct_err_coalesce,
+		      tx_logs->intf_tx_dequeue,
+		      tx_logs->intf_tx_dequeue_err_coalesce,
+		      tx_logs->intf_tx_dump_xframe,
+		      tx_logs->intf_tx_dump_xframe_err_txdesc,
+		      tx_logs->intf_tx_dump_xframe_err_port
+		     );
+
+	return 0;
+}
+
+int proc_get_int_logs(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m,
+		      "all=%d\n"
+		      "known=%d\n"
+		      "err=%d\n",
+		      padapter->int_logs.all,
+		      padapter->int_logs.known,
+		      padapter->int_logs.err
+		     );
+
+	return 0;
+}
+
+#endif /* CONFIG_DBG_COUNTER */
+
+int proc_get_hw_status(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &dvobj->drv_dbg;
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+
+	if (regsty->check_hw_status == 0)
+		RTW_PRINT_SEL(m, "RX FIFO full count: not check in watch dog\n");
+	else if (pdbgpriv->dbg_rx_fifo_last_overflow == 1
+	    && pdbgpriv->dbg_rx_fifo_curr_overflow == 1
+	    && pdbgpriv->dbg_rx_fifo_diff_overflow == 1
+	   )
+		RTW_PRINT_SEL(m, "RX FIFO full count: no implementation\n");
+	else {
+		RTW_PRINT_SEL(m, "RX FIFO full count: last_time=%llu, current_time=%llu, differential=%llu\n"
+			, pdbgpriv->dbg_rx_fifo_last_overflow, pdbgpriv->dbg_rx_fifo_curr_overflow, pdbgpriv->dbg_rx_fifo_diff_overflow);
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_hw_status(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = padapter->dvobj;
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+	char tmp[32];
+	u32 enable;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &enable);
+
+		if (num == 1 && regsty && enable <= 1) {
+			regsty->check_hw_status = enable;
+			RTW_INFO("check_hw_status=%d\n", regsty->check_hw_status);
+		}
+	}
+
+	return count;
+}
+
+int proc_get_trx_info_debug(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	/*============  tx info ============	*/
+	rtw_hal_get_def_var(padapter, padapter_link, HW_DEF_RA_INFO_DUMP, m);
+
+	/*============  rx info ============	*/
+	/*rtw_hal_set_phydm_var(padapter, HAL_PHYDM_RX_INFO_DUMP, m, _FALSE);*/
+
+	return 0;
+}
+
+int proc_get_rx_signal(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+#ifdef CONFIG_SNR_RPT
+	_list	*plist, *phead;
+	struct sta_info *psta = NULL;
+	u8 sta_mac[NUM_STA][ETH_ALEN] = {{0}};
+	uint mac_id[NUM_STA];
+	struct stainfo_stats	*pstats = NULL;
+	struct sta_priv	*pstapriv = &(padapter->stapriv);
+	u32 i, j, macid_rec_idx = 0;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	char mac_addr_str[MAC_FMT_LEN];
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+			if ((_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, 6) !=  _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, 6) != _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, psta->padapter_link->mac_addr, 6) != _TRUE)) {
+				_rtw_memcpy(&sta_mac[macid_rec_idx][0], psta->phl_sta->mac_addr, ETH_ALEN);
+				mac_id[macid_rec_idx] = psta->phl_sta->macid;
+				macid_rec_idx++;
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < macid_rec_idx; i++) {
+		psta = rtw_get_stainfo(pstapriv, &sta_mac[i][0]);
+		if(psta) {
+			/*ToDo, base on the real number of rf path to show the information*/
+			RTW_PRINT_SEL(m, "STA:%s SNR:{%d, %d, %d, %d}\n",
+				get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr),
+				psta->snr_fd_avg[0], psta->snr_fd_avg[1], psta->snr_fd_avg[2], psta->snr_fd_avg[3]);
+		} else {
+			RTW_INFO("STA is gone\n");
+		}
+	}
+#endif /* CONFIG_SNR_RPT */
+
+	RTW_PRINT_SEL(m, "rssi:%d\n", padapter->recvinfo.rssi);
+#ifdef CONFIG_MP_INCLUDED
+	if (padapter->registrypriv.mp_mode == 1) {
+		if (padapter->mppriv.antenna_trx == MP_ANTENNA_A)
+			RTW_PRINT_SEL(m, "Antenna: A\n");
+		else if (padapter->mppriv.antenna_trx == MP_ANTENNA_B)
+			RTW_PRINT_SEL(m, "Antenna: B\n");
+		else if (padapter->mppriv.antenna_trx == MP_ANTENNA_C)
+			RTW_PRINT_SEL(m, "Antenna: C\n");
+		else if (padapter->mppriv.antenna_trx == MP_ANTENNA_D)
+			RTW_PRINT_SEL(m, "Antenna: D\n");
+		else if (padapter->mppriv.antenna_trx == MP_ANTENNA_AB)
+			RTW_PRINT_SEL(m, "Antenna: AB\n");
+		else if (padapter->mppriv.antenna_trx == MP_ANTENNA_BC)
+			RTW_PRINT_SEL(m, "Antenna: BC\n");
+		else if (padapter->mppriv.antenna_trx == MP_ANTENNA_CD)
+			RTW_PRINT_SEL(m, "Antenna: CD\n");
+		else
+			RTW_PRINT_SEL(m, "Antenna: __\n");
+		RTW_PRINT_SEL(m, "rx_rate = %s\n", MGN_RATE_STR(hwrate_to_mrate(padapter->mppriv.rx_rate)));
+		return 0;
+	} else
+#endif
+	{
+		/* RTW_PRINT_SEL(m, "rxpwdb:%d\n", padapter->recvinfo.rxpwdb); */
+		RTW_PRINT_SEL(m, "signal_strength:%u\n", padapter->recvinfo.signal_strength);
+		RTW_PRINT_SEL(m, "signal_qual:%u\n", padapter->recvinfo.signal_qual);
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_rx_signal(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 is_signal_dbg, signal_strength;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%u %u", &is_signal_dbg, &signal_strength);
+
+		if (num < 1)
+			return count;
+
+		is_signal_dbg = is_signal_dbg == 0 ? 0 : 1;
+
+		if (is_signal_dbg && num < 2)
+			return count;
+
+		signal_strength = signal_strength > PHL_MAX_RSSI ? PHL_MAX_RSSI : signal_strength;
+
+		padapter->recvinfo.is_signal_dbg = is_signal_dbg;
+		padapter->recvinfo.signal_strength_dbg = signal_strength;
+
+		if (is_signal_dbg)
+			RTW_INFO("set %s %u\n", "DBG_SIGNAL_STRENGTH", signal_strength);
+		else
+			RTW_INFO("set %s\n", "HW_SIGNAL_STRENGTH");
+
+	}
+
+	return count;
+
+}
+
+int proc_get_mac_rptbuf(struct seq_file *m, void *v)
+{
+	return 0;
+}
+
+#ifdef CONFIG_80211N_HT
+
+int proc_get_ht_enable(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "%d\n", pregpriv->ht_enable);
+
+	return 0;
+}
+
+ssize_t proc_set_ht_enable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 mode;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &mode);
+
+		if ( num == 1 && pregpriv && mode < 2) {
+			pregpriv->ht_enable = mode;
+			RTW_INFO("ht_enable=%d\n", pregpriv->ht_enable);
+		}
+	}
+
+	return count;
+
+}
+
+int proc_get_bw_mode(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "0x%02x\n", pregpriv->bw_mode);
+
+	return 0;
+}
+
+ssize_t proc_set_bw_mode(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 mode;
+	u8 bw_2g;
+	u8 bw_5g;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%x ", &mode);
+		bw_5g = mode >> 4;
+		bw_2g = mode & 0x0f;
+
+		if (num == 1 && pregpriv && bw_2g <= 4 && bw_5g <= 4) {
+			pregpriv->bw_mode = mode;
+			printk("bw_mode=0x%x\n", mode);
+		}
+	}
+
+	return count;
+
+}
+
+int proc_get_ampdu_enable(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "%d\n", pregpriv->ampdu_enable);
+
+	return 0;
+}
+
+ssize_t proc_set_ampdu_enable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 mode;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &mode);
+
+		if (num == 1 && pregpriv && mode < 2) {
+			pregpriv->ampdu_enable = mode;
+			printk("ampdu_enable=%d\n", mode);
+		}
+
+	}
+
+	return count;
+
+}
+
+
+void dump_regsty_rx_ampdu_size_limit(void *sel, _adapter *adapter)
+{
+	struct registry_priv *regsty = adapter_to_regsty(adapter);
+	int i;
+
+	RTW_PRINT_SEL(sel, "%-3s %-5s %-5s %-5s %-5s\n"
+		, "", "20M", "40M", "80M", "160M");
+	for (i = 0; i < 4; i++)
+		RTW_PRINT_SEL(sel, "%dSS %5u %5u %5u %5u\n", i + 1
+			, regsty->rx_ampdu_sz_limit_by_nss_bw[i][0]
+			, regsty->rx_ampdu_sz_limit_by_nss_bw[i][1]
+			, regsty->rx_ampdu_sz_limit_by_nss_bw[i][2]
+			, regsty->rx_ampdu_sz_limit_by_nss_bw[i][3]);
+}
+
+int proc_get_rx_ampdu(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	_RTW_PRINT_SEL(m, "accept: ");
+	if (padapter->fix_rx_ampdu_accept == RX_AMPDU_ACCEPT_INVALID)
+		RTW_PRINT_SEL(m, "%u%s\n", rtw_rx_ampdu_is_accept(padapter), "(auto)");
+	else
+		RTW_PRINT_SEL(m, "%u%s\n", padapter->fix_rx_ampdu_accept, "(fixed)");
+
+	_RTW_PRINT_SEL(m, "size: ");
+	if (padapter->fix_rx_ampdu_size == RX_AMPDU_SIZE_INVALID) {
+		RTW_PRINT_SEL(m, "%u%s\n", rtw_rx_ampdu_size(padapter), "(auto) with conditional limit:");
+		dump_regsty_rx_ampdu_size_limit(m, padapter);
+	} else
+		RTW_PRINT_SEL(m, "%u%s\n", padapter->fix_rx_ampdu_size, "(fixed)");
+	RTW_PRINT_SEL(m, "\n");
+
+	RTW_PRINT_SEL(m, "%19s %17s\n", "fix_rx_ampdu_accept", "fix_rx_ampdu_size");
+
+	_RTW_PRINT_SEL(m, "%-19d %-17u\n"
+		, padapter->fix_rx_ampdu_accept
+		, padapter->fix_rx_ampdu_size);
+
+	return 0;
+}
+
+ssize_t proc_set_rx_ampdu(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 accept;
+	u16 size;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hu", &accept, &size);
+
+		if (num >= 1)
+			rtw_rx_ampdu_set_accept(padapter, accept, RX_AMPDU_DRV_FIXED);
+		if (num >= 2)
+			rtw_rx_ampdu_set_size(padapter, size, RX_AMPDU_DRV_FIXED);
+
+		rtw_rx_ampdu_apply(padapter);
+	}
+
+	return count;
+}
+
+int proc_get_rx_ampdu_factor(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	if (padapter)
+		RTW_PRINT_SEL(m, "rx ampdu factor = %x\n", padapter->driver_rx_ampdu_factor);
+
+	return 0;
+}
+
+ssize_t proc_set_rx_ampdu_factor(struct file *file, const char __user *buffer
+				 , size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 factor;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &factor);
+
+		if (padapter && (num == 1)) {
+			RTW_INFO("padapter->driver_rx_ampdu_factor = %x\n", factor);
+
+			if (factor  > 0x03)
+				padapter->driver_rx_ampdu_factor = 0xFF;
+			else
+				padapter->driver_rx_ampdu_factor = factor;
+		}
+	}
+
+	return count;
+}
+
+int proc_get_tx_ampdu_num(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj;
+	struct rtw_phl_com_t *phl_com;
+	u8 hw_band_num = 1;
+	int i;
+
+	if (padapter) {
+		dvobj = adapter_to_dvobj(padapter);
+		phl_com = GET_PHL_COM(dvobj);
+
+#ifdef CONFIG_DBCC_SUPPORT
+		if (is_dbcc_sup(phl_com))
+			hw_band_num = HW_BAND_MAX;
+#endif
+
+		for (i = HW_BAND_0 ; i < hw_band_num ; i++) {
+			RTW_PRINT_SEL(m, "===== HW band index %d =====\n", i);
+			RTW_PRINT_SEL(m, "[phy_cap] tx ampdu num = %s",
+						  phl_com->phy_cap[i].txagg_num ? "":"not yet set");
+			if (phl_com->phy_cap[i].txagg_num)
+				RTW_PRINT_SEL(m, "%d\n", phl_com->phy_cap[i].txagg_num);
+			else
+				RTW_PRINT_SEL(m, "\n");
+
+			RTW_PRINT_SEL(m, "[phy_sw_cap] tx ampdu num = %s",
+						  phl_com->phy_sw_cap[i].txagg_num ? "":"default by HW");
+			if (phl_com->phy_sw_cap[i].txagg_num)
+				RTW_PRINT_SEL(m, "%d\n", phl_com->phy_sw_cap[i].txagg_num);
+			else
+				RTW_PRINT_SEL(m, "\n");
+		}
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_tx_ampdu_num(struct file *file, const char __user *buffer
+				 , size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_phl_com_t *phl_com = NULL;
+	char tmp[32];
+	u8 hw_band_idx;
+	u32 tx_ampdu_num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %u", &hw_band_idx, &tx_ampdu_num);
+
+		if (padapter && (num == 2)) {
+			if (hw_band_idx < HW_BAND_MAX && hw_band_idx >= HW_BAND_0) {
+				phl_com = GET_PHL_COM(adapter_to_dvobj(padapter));
+				phl_com->phy_cap[hw_band_idx].txagg_num = tx_ampdu_num;
+				RTW_INFO("[HW Band %d] set phy_cap tx ampdu num = %u\n",
+					 hw_band_idx, tx_ampdu_num);
+                        } else {
+				RTW_INFO("The input of HW Band index (%u) is invalid !\n",
+                                         hw_band_idx);
+			}
+		}
+	}
+
+	return count;
+}
+
+int proc_get_rx_ampdu_density(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	if (padapter)
+		RTW_PRINT_SEL(m, "rx ampdu densityg = %x\n", padapter->driver_rx_ampdu_spacing);
+
+	return 0;
+}
+
+ssize_t proc_set_rx_ampdu_density(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 density;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &density);
+
+		if (padapter && (num == 1)) {
+			RTW_INFO("padapter->driver_rx_ampdu_spacing = %x\n", density);
+
+			if (density > 0x07)
+				padapter->driver_rx_ampdu_spacing = 0xFF;
+			else
+				padapter->driver_rx_ampdu_spacing = density;
+		}
+	}
+
+	return count;
+}
+
+int proc_get_tx_ampdu_density(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	if (padapter)
+		RTW_PRINT_SEL(m, "tx ampdu density = %x\n", padapter->driver_ampdu_spacing);
+
+	return 0;
+}
+
+ssize_t proc_set_tx_ampdu_density(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 density;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &density);
+
+		if (padapter && (num == 1)) {
+			RTW_INFO("padapter->driver_ampdu_spacing = %x\n", density);
+
+			if (density > 0x07)
+				padapter->driver_ampdu_spacing = 0xFF;
+			else
+				padapter->driver_ampdu_spacing = density;
+		}
+	}
+
+	return count;
+}
+
+int proc_get_tx_quick_addba_req(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (padapter)
+		RTW_PRINT_SEL(m, "tx_quick_addba_req = %x\n", pregpriv->tx_quick_addba_req);
+
+	return 0;
+}
+
+ssize_t proc_set_tx_quick_addba_req(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 enable;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &enable);
+
+		if (padapter && (num == 1)) {
+			pregpriv->tx_quick_addba_req = enable;
+			RTW_INFO("tx_quick_addba_req = %d\n", pregpriv->tx_quick_addba_req);
+		}
+	}
+
+	return count;
+}
+#ifdef CONFIG_TX_AMSDU
+int proc_get_tx_amsdu(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	int i;
+
+	if (padapter)
+	{
+		RTW_PRINT_SEL(m, "tx amsdu = %d\n", padapter->tx_amsdu);
+		RTW_PRINT_SEL(m, "amsdu set timer conut = %u\n", pxmitpriv->amsdu_debug_set_timer);
+		RTW_PRINT_SEL(m, "amsdu time out count = %u\n", pxmitpriv->amsdu_debug_timeout);
+		for (i = 0; i < (AMSDU_DEBUG_MAX_COUNT - 1); i++) {
+			RTW_PRINT_SEL(m, "amsdu coalesce %d count = %u\n",
+					i + 1, pxmitpriv->amsdu_debug_coalesce[i]);
+		}
+		RTW_PRINT_SEL(m, "amsdu coalesce >%d count = %u\n",
+				i, pxmitpriv->amsdu_debug_coalesce[i]);
+		RTW_PRINT_SEL(m, "amsdu tasklet count = %u\n", pxmitpriv->amsdu_debug_tasklet);
+		RTW_PRINT_SEL(m, "amsdu enqueue count = %u\n", pxmitpriv->amsdu_debug_enqueue);
+		RTW_PRINT_SEL(m, "amsdu dequeue count = %u\n", pxmitpriv->amsdu_debug_dequeue);
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_tx_amsdu(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	char tmp[32];
+	u32 i, amsdu;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &amsdu);
+
+		if (padapter && (num == 1)) {
+			RTW_INFO("padapter->tx_amsdu = %x\n", amsdu);
+
+			if (amsdu == 255) {
+				pxmitpriv->amsdu_debug_set_timer = 0;
+				pxmitpriv->amsdu_debug_timeout = 0;
+				pxmitpriv->amsdu_debug_tasklet = 0;
+				pxmitpriv->amsdu_debug_enqueue = 0;
+				pxmitpriv->amsdu_debug_dequeue = 0;
+				for (i = 0; i < AMSDU_DEBUG_MAX_COUNT; i++)
+					pxmitpriv->amsdu_debug_coalesce[i] = 0;
+			} else {
+				padapter->tx_amsdu = amsdu;
+			}
+		}
+	}
+
+	return count;
+}
+
+int proc_get_tx_amsdu_rate(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (padapter)
+		RTW_PRINT_SEL(m, "tx amsdu rate = %d Mbps\n", padapter->tx_amsdu_rate);
+
+	return 0;
+}
+
+ssize_t proc_set_tx_amsdu_rate(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 amsdu_rate;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &amsdu_rate);
+
+		if (padapter && (num == 1)) {
+			RTW_INFO("padapter->tx_amsdu_rate = %x\n", amsdu_rate);
+			padapter->tx_amsdu_rate = amsdu_rate;
+		}
+	}
+
+	return count;
+}
+#endif /* CONFIG_TX_AMSDU */
+
+int proc_get_tx_max_agg_time(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	if (padapter)
+	{
+		RTW_PRINT_SEL(m, "max agg time = 0x%x\n", pxmitpriv->max_agg_time);
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_tx_max_agg_time(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	char tmp[32];
+	u16 agg_t;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hx ", &agg_t);
+
+		if (padapter && (num == 1)) {
+			RTW_INFO("max_agg_time = 0x%x\n", agg_t);
+
+			pxmitpriv->max_agg_time = agg_t;
+		}
+	}
+
+	return count;
+}
+
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_80211AC_VHT
+int proc_get_vht_24g_enable(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	struct _ADAPTER *a = (struct _ADAPTER *)rtw_netdev_priv(dev);
+	struct registry_priv *regpriv = &a->registrypriv;
+
+
+	if (regpriv)
+		RTW_PRINT_SEL(m, "%d\n", regpriv->vht_24g_enable);
+
+	return 0;
+}
+
+ssize_t proc_set_vht_24g_enable(struct file *file, const char __user *buffer,
+				size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	struct _ADAPTER *a = (struct _ADAPTER *)rtw_netdev_priv(dev);
+	struct registry_priv *regpriv = &a->registrypriv;
+	char tmp[32];
+	u32 mode;
+
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &mode);
+
+		if ((num == 1) && regpriv && (mode < 2)) {
+			regpriv->vht_24g_enable = mode;
+			RTW_INFO("vht_24g_enable = %d\n", regpriv->vht_24g_enable);
+		}
+	}
+
+	return count;
+}
+#endif /* CONFIG_80211AC_VHT */
+
+ssize_t proc_set_dyn_rrsr(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+
+	char tmp[32] = {0};
+	u32 num = 0, enable = 0, rrsr_val = 0; /* gpio_mode:0 input  1:output; */
+
+	if (count < 2)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num	= sscanf(tmp, "%d 0x%x", &enable, &rrsr_val);
+		RTW_INFO("num=%u enable=%d rrsr_val=0x%x\n", num, enable, rrsr_val);
+		pregpriv->en_dyn_rrsr = enable;
+		pregpriv->set_rrsr_value = rrsr_val;
+		/*rtw_phydm_dyn_rrsr_en(padapter, enable);*/
+		/*rtw_phydm_set_rrsr(padapter, rrsr_val, _TRUE);*/
+
+	}
+	return count;
+
+}
+int proc_get_dyn_rrsr(struct seq_file *m, void *v) {
+
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	u32 init_rrsr =0xFFFFFFFF;
+
+	if (padapter)
+		RTW_PRINT_SEL(m, "en_dyn_rrsr = %d fixed_rrsr_value =0x%x %s\n"
+			, pregpriv->en_dyn_rrsr
+			, pregpriv->set_rrsr_value
+			, (pregpriv->set_rrsr_value == init_rrsr)?"(default)":"(fixed)"
+		);
+
+	return 0;
+}
+
+#ifdef CONFIG_80211N_HT
+void rtw_dump_dft_phy_cap(void *sel, _adapter *adapter)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_priv *pmlmepriv = &adapter_link->mlmepriv;
+	struct ht_priv	*phtpriv = &pmlmepriv->htpriv;
+	#ifdef CONFIG_80211AC_VHT
+	struct vht_priv *pvhtpriv = &pmlmepriv->vhtpriv;
+	#endif
+
+	#ifdef CONFIG_80211AC_VHT
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT STBC Tx : %s\n", (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_TX)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT STBC Rx : %s\n", (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_RX)) ? "V" : "X");
+	#endif
+	RTW_PRINT_SEL(sel, "[DFT CAP] HT STBC Tx : %s\n", (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] HT STBC Rx : %s\n\n", (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX)) ? "V" : "X");
+
+	#ifdef CONFIG_80211AC_VHT
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT LDPC Tx : %s\n", (TEST_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_TX)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT LDPC Rx : %s\n", (TEST_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_RX)) ? "V" : "X");
+	#endif
+	RTW_PRINT_SEL(sel, "[DFT CAP] HT LDPC Tx : %s\n", (TEST_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_TX)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] HT LDPC Rx : %s\n\n", (TEST_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_RX)) ? "V" : "X");
+
+	#ifdef CONFIG_BEAMFORMING
+	#ifdef CONFIG_80211AX_HE
+        RTW_PRINT_SEL(sel, "[DFT CAP] HE MU Bfer : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_HE_MU_MIMO_AP_ENABLE)) ? "V" : "X");
+        RTW_PRINT_SEL(sel, "[DFT CAP] HE MU Bfee : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_HE_MU_MIMO_STA_ENABLE)) ? "V" : "X");
+        RTW_PRINT_SEL(sel, "[DFT CAP] HE SU Bfer : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_HE_BEAMFORMER_ENABLE)) ? "V" : "X");
+        RTW_PRINT_SEL(sel, "[DFT CAP] HE SU Bfee : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_HE_BEAMFORMEE_ENABLE)) ? "V" : "X");
+        #endif
+	#ifdef CONFIG_80211AC_VHT
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT MU Bfer : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT MU Bfee : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT SU Bfer : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] VHT SU Bfee : %s\n", (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE)) ? "V" : "X");
+	#endif
+	RTW_PRINT_SEL(sel, "[DFT CAP] HT Bfer : %s\n", (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE))  ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DFT CAP] HT Bfee : %s\n", (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE)) ? "V" : "X");
+	#endif
+}
+
+void rtw_get_dft_phy_cap(void *sel, _adapter *adapter)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	RTW_PRINT_SEL(sel, "\n ======== PHY CAP protocol ========\n");
+	rtw_ht_use_default_setting(adapter, adapter_link);
+	#ifdef CONFIG_80211AC_VHT
+	rtw_vht_get_real_setting(adapter, adapter_link);
+	#endif
+	#ifdef CONFIG_80211N_HT
+	rtw_dump_dft_phy_cap(sel, adapter);
+	#endif
+}
+
+void rtw_dump_drv_phy_cap(void *sel, _adapter *adapter)
+{
+	struct registry_priv	*pregistry_priv = &adapter->registrypriv;
+
+	RTW_PRINT_SEL(sel, "\n ======== DRV's configuration ========\n");
+	#if 0
+	RTW_PRINT_SEL(sel, "[DRV CAP] TRx Capability : 0x%08x\n", phy_spec->trx_cap);
+	RTW_PRINT_SEL(sel, "[DRV CAP] Tx Stream Num Index : %d\n", (phy_spec->trx_cap >> 24) & 0xFF); /*Tx Stream Num Index [31:24]*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] Rx Stream Num Index : %d\n", (phy_spec->trx_cap >> 16) & 0xFF); /*Rx Stream Num Index [23:16]*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] Tx Path Num Index : %d\n", (phy_spec->trx_cap >> 8) & 0xFF);/*Tx Path Num Index	[15:8]*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] Rx Path Num Index : %d\n", (phy_spec->trx_cap & 0xFF));/*Rx Path Num Index	[7:0]*/
+	#endif
+	#ifdef CONFIG_80211N_HT
+	RTW_PRINT_SEL(sel, "[DRV CAP] STBC Capability : 0x%04x\n", pregistry_priv->stbc_cap);
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT STBC Tx : %s\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT5)) ? "V" : "X"); /*BIT5: Enable HT STBC Tx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT STBC Rx : %s\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT4)) ? "V" : "X"); /*BIT4: Enable HT STBC Rx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT STBC Tx : %s\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT1)) ? "V" : "X"); /*BIT1: Enable VHT STBC Tx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT STBC Rx : %s\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT0)) ? "V" : "X"); /*BIT0: Enable VHT STBC Rx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE STBC Tx : %s\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT10)) ? "V" : "X"); /*BIT10: Enable HE STBC Tx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE STBC Rx : %s\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT8)) ? "V" : "X"); /*BIT8: Enable HE STBC Rx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE STBC Tx(>80M) : %s\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT11)) ? "V" : "X"); /*BIT11: Enable HE STBC Tx(>80M)*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE STBC Rx(>80M) : %s\n\n", (TEST_FLAG(pregistry_priv->stbc_cap, BIT9)) ? "V" : "X"); /*BIT9: Enable HE STBC Rx(>80M)*/
+
+
+	RTW_PRINT_SEL(sel, "[DRV CAP] LDPC Capability : 0x%02x\n", pregistry_priv->ldpc_cap);
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT LDPC Tx : %s\n", (TEST_FLAG(pregistry_priv->ldpc_cap, BIT1)) ? "V" : "X"); /*BIT1: Enable VHT LDPC Tx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT LDPC Rx : %s\n", (TEST_FLAG(pregistry_priv->ldpc_cap, BIT0)) ? "V" : "X"); /*BIT0: Enable VHT LDPC Rx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT LDPC Tx : %s\n", (TEST_FLAG(pregistry_priv->ldpc_cap, BIT5)) ? "V" : "X"); /*BIT5: Enable HT LDPC Tx*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT LDPC Rx : %s\n\n", (TEST_FLAG(pregistry_priv->ldpc_cap, BIT4)) ? "V" : "X"); /*BIT4: Enable HT LDPC Rx*/
+	#endif /* CONFIG_80211N_HT */
+	#ifdef CONFIG_BEAMFORMING
+	#if 0
+	RTW_PRINT_SEL(sel, "[DRV CAP] TxBF parameter : 0x%08x\n", phy_spec->txbf_param);
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT Sounding Dim : %d\n", (phy_spec->txbf_param >> 24) & 0xFF); /*VHT Sounding Dim [31:24]*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT Steering Ant : %d\n", (phy_spec->txbf_param >> 16) & 0xFF); /*VHT Steering Ant [23:16]*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT Sounding Dim : %d\n", (phy_spec->txbf_param >> 8) & 0xFF); /*HT Sounding Dim [15:8]*/
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT Steering Ant : %d\n", phy_spec->txbf_param & 0xFF); /*HT Steering Ant [7:0]*/
+	#endif
+
+	/*
+	 * BIT0: Enable VHT SU Beamformer
+	 * BIT1: Enable VHT SU Beamformee
+	 * BIT2: Enable VHT MU Beamformer, depend on VHT SU Beamformer
+	 * BIT3: Enable VHT MU Beamformee, depend on VHT SU Beamformee
+	 * BIT4: Enable HT Beamformer
+	 * BIT5: Enable HT Beamformee
+	 * BIT6: Enable HE SU Beamformer
+	 * BIT7: Enable HE SU Beamformee
+	 * BIT8: Enable HE MU Beamformer
+	 * BIT9: Enable HE MU Beamformee
+	 */
+	RTW_PRINT_SEL(sel, "[DRV CAP] TxBF Capability : 0x%04x\n", pregistry_priv->beamform_cap);
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT MU Bfer : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT2)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT MU Bfee : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT3)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT SU Bfer : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT0)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] VHT SU Bfee : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT1)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT Bfer : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT4))  ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] HT Bfee : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT5)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE SU Bfer : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT6)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE SU Bfee : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT7)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE MU Bfer : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT8)) ? "V" : "X");
+	RTW_PRINT_SEL(sel, "[DRV CAP] HE MU Bfee : %s\n", (TEST_FLAG(pregistry_priv->beamform_cap, BIT9)) ? "V" : "X");
+
+	RTW_PRINT_SEL(sel, "[DRV CAP] Tx Bfer rf_num : %d\n", pregistry_priv->beamformer_rf_num);
+	RTW_PRINT_SEL(sel, "[DRV CAP] Tx Bfee rf_num : %d\n", pregistry_priv->beamformee_rf_num);
+	#endif
+}
+
+void rtw_dump_macaddr(void *sel, _adapter *adapter)
+{
+	int i;
+	_adapter *iface;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	u8 mac_addr[ETH_ALEN];
+	char mac_addr_str[MAC_FMT_LEN];
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (iface) {
+			RTW_PRINT_SEL(sel, ADPT_FMT"- mac_addr =%s\n", ADPT_ARG(iface),
+				get_macaddr_str(mac_addr_str, sel, adapter_mac_addr(iface)));
+		}
+	}
+}
+
+int proc_get_stbc_cap(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "0x%04x\n", pregpriv->stbc_cap);
+
+	return 0;
+}
+
+ssize_t proc_set_stbc_cap(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 mode;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &mode);
+
+		if (num == 1 && pregpriv) {
+			pregpriv->stbc_cap = mode;
+			RTW_INFO("stbc_cap = 0x%02x\n", mode);
+		}
+	}
+
+	return count;
+}
+
+int proc_get_ldpc_cap(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "0x%02x\n", pregpriv->ldpc_cap);
+
+	return 0;
+}
+
+ssize_t proc_set_ldpc_cap(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 mode;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &mode);
+
+		if (num == 1 && pregpriv) {
+			pregpriv->ldpc_cap = mode;
+			RTW_INFO("ldpc_cap = 0x%02x\n", mode);
+		}
+	}
+
+	return count;
+}
+#ifdef CONFIG_BEAMFORMING
+int proc_get_txbf_cap(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "0x%04x\n", pregpriv->beamform_cap);
+
+	return 0;
+}
+
+ssize_t proc_set_txbf_cap(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 mode;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &mode);
+
+		if (num == 1 && pregpriv) {
+			pregpriv->beamform_cap = mode;
+			RTW_INFO("beamform_cap = 0x%04x\n", mode);
+		}
+	}
+
+	return count;
+}
+#endif
+#endif /* CONFIG_80211N_HT */
+
+/*int proc_get_rssi_disp(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	return 0;
+}
+*/
+
+/*ssize_t proc_set_rssi_disp(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 enable=0;
+
+	if (count < 1)
+	{
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%x", &enable);
+
+		if (num !=  1) {
+			RTW_INFO("invalid set_rssi_disp parameter!\n");
+			return count;
+		}
+
+		if(enable)
+		{
+			RTW_INFO("Linked info Function Enable\n");
+			padapter->bLinkInfoDump = enable ;
+		}
+		else
+		{
+			RTW_INFO("Linked info Function Disable\n");
+			padapter->bLinkInfoDump = 0 ;
+		}
+
+	}
+
+	return count;
+
+}
+
+*/
+#ifdef CONFIG_AP_MODE
+
+void dump_phl_tring_status(struct seq_file *m, _adapter *padapter, struct sta_info *psta)
+{
+	int i = 0;
+	u16 tring_len = 0;
+
+	RTW_PRINT_SEL(m, "PHL_tring_len=");
+	for (i = 0; i < MAX_PHL_RING_CAT_NUM; i++) {
+		tring_len = rtw_phl_tring_rsc(padapter->dvobj->phl, psta->phl_sta->macid, i);
+		RTW_PRINT_SEL(m, "%d ", tring_len);
+	}
+	RTW_PRINT_SEL(m, "\n");
+}
+
+int proc_get_sta_active_time(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_list *phead, *plist;
+	struct sta_info *psta;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u32 i, macid_rec_idx = 0;
+	u8 sta_mac[NUM_STA][ETH_ALEN]={{0}};
+	char mac_addr_str[MAC_FMT_LEN];
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+
+	while((rtw_end_of_queue_search(phead, plist) == _FALSE)){
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+		_rtw_memcpy(&sta_mac[macid_rec_idx][0], psta->phl_sta->mac_addr, ETH_ALEN);
+		macid_rec_idx++;
+	}
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	for(i = 0; i < macid_rec_idx; i++){
+		psta = rtw_get_stainfo(pstapriv, &sta_mac[i][0]);
+		if(psta){
+			_rtw_spinlock_bh(&pstapriv->active_time_lock);
+			RTW_PRINT_SEL(m, "==============================\n");
+			RTW_PRINT_SEL(m, "sta's macaddr:%s\n", get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr));
+			RTW_PRINT_SEL(m, "start active time: %u\n", psta->start_active_time);
+			RTW_PRINT_SEL(m, "latest active time: %u\n", psta->latest_active_time);
+			RTW_PRINT_SEL(m, "==============================\n");
+			_rtw_spinunlock_bh(&pstapriv->active_time_lock);
+		}
+	}
+	return 0;
+}
+
+int proc_get_all_sta_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	struct sta_info *psta;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	int i;
+	_list	*plist, *phead;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	RTW_MAP_DUMP_SEL_ALWAYS(m, "sta_dz_bitmap=", pstapriv->sta_dz_bitmap, pstapriv->aid_bmp_len);
+	RTW_MAP_DUMP_SEL_ALWAYS(m, "tim_bitmap=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len);
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+
+			plist = get_next(plist);
+
+			/* if(extra_arg == psta->phl_sta->aid) */
+			{
+				RTW_PRINT_SEL(m, "==============================\n");
+				RTW_PRINT_SEL(m, "sta's macaddr:%s\n", get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr));
+				RTW_PRINT_SEL(m, "ieee8021x_blocked=%d\n", psta->ieee8021x_blocked);
+				RTW_PRINT_SEL(m, "rtsen=%d, cts2slef=%d, hw_rts_en=%d\n",
+						psta->rtsen, psta->cts2self, psta->hw_rts_en);
+				/* ToDo: need API to query hal_sta->ra_info.rate_id */
+				/* RTW_PRINT_SEL(m, "state=0x%x, aid=%d, macid=%d, raid=%d\n",
+					psta->state, psta->phl_sta->aid, psta->phl_sta->macid, psta->phl_sta->hal_sta->ra_info.rate_id); */
+				RTW_PRINT_SEL(m, "state=0x%x, aid=%d, macid=%d\n",
+					psta->state, psta->phl_sta->aid, psta->phl_sta->macid);
+#ifdef CONFIG_RTS_FULL_BW
+				if(psta->vendor_8812)
+					RTW_PRINT_SEL(m,"Vendor Realtek 8812\n");
+#endif/*CONFIG_RTS_FULL_BW*/
+#ifdef CONFIG_80211N_HT
+				RTW_PRINT_SEL(m, "qos_en=%d, ht_en=%d, vht_en=%d, he_en=%d, init_rate=%d, bwmode=%d\n",
+					psta->qos_option, psta->htpriv.ht_option, psta->vhtpriv.vht_option, psta->hepriv.he_option,
+					psta->init_rate, psta->phl_sta->chandef.bw);
+
+				RTW_PRINT_SEL(m, "ampdu_enable = %d\n", psta->ampdu_priv.ampdu_enable);
+				RTW_PRINT_SEL(m, "tx_amsdu_enable = %d\n", psta->ampdu_priv.tx_amsdu_enable);
+				RTW_PRINT_SEL(m, "agg_enable_bitmap=%x, candidate_tid_bitmap=%x\n", psta->ampdu_priv.agg_enable_bitmap, psta->ampdu_priv.candidate_tid_bitmap);
+
+				if (psta->htpriv.ht_option) {
+					RTW_PRINT_SEL(m, "ht_ch_offset=%d, ht_is_8K_amsdu=%d\n",
+						psta->htpriv.ch_offset,
+						(psta->htpriv.ht_cap.cap_info & IEEE80211_HT_CAP_MAX_AMSDU) ? 1 : 0);
+
+					RTW_PRINT_SEL(m, "ht_sgi_20/40m=(%d, %d)\n",
+						psta->htpriv.sgi_20m, psta->htpriv.sgi_40m);
+
+					RTW_PRINT_SEL(m, "ht_max_bitrate_20/40m=(%u, %u)\n",
+						rtw_ht_mcs_rate(0, psta->htpriv.sgi_20m, psta->htpriv.ht_cap.supp_mcs_set) / 10,
+						rtw_ht_mcs_rate(1, psta->htpriv.sgi_40m, psta->htpriv.ht_cap.supp_mcs_set) / 10);
+				}
+#ifdef CONFIG_80211AC_VHT
+				if (psta->vhtpriv.vht_option) {
+					RTW_PRINT_SEL(m, "vht_sgi_80/160m=(%d, %d)\n",
+						psta->vhtpriv.sgi_80m, psta->vhtpriv.sgi_160m);
+
+					RTW_PRINT_SEL(m, "vht_max_bitrate_20/40/80/160m=(%u, %u, %u, %u)\n",
+						(rtw_vht_mcs_to_data_rate(0, psta->htpriv.sgi_20m, psta->vhtpriv.vht_highest_rate) + 1) >> 1,
+						(rtw_vht_mcs_to_data_rate(1, psta->htpriv.sgi_40m, psta->vhtpriv.vht_highest_rate) + 1) >> 1,
+						(rtw_vht_mcs_to_data_rate(2, psta->vhtpriv.sgi_80m, psta->vhtpriv.vht_highest_rate) + 1) >> 1,
+						(rtw_vht_mcs_to_data_rate(3, psta->vhtpriv.sgi_160m, psta->vhtpriv.vht_highest_rate) + 1) >> 1);
+
+					RTW_PRINT_SEL(m, "vht_ldpc_cap=0x%x, vht_stbc_cap=0x%x, vht_beamform_cap=0x%x\n", psta->vhtpriv.ldpc_cap, psta->vhtpriv.stbc_cap, psta->vhtpriv.beamform_cap);
+					RTW_PRINT_SEL(m, "vht_mcs_map=0x%x, vht_highest_rate=0x%x, vht_ampdu_len=%d\n", *(u16 *)psta->vhtpriv.vht_mcs_map, psta->vhtpriv.vht_highest_rate, psta->vhtpriv.ampdu_len);
+					RTW_MAP_DUMP_SEL_ALWAYS(m, "vht_cap=", psta->vhtpriv.vht_cap, 32);
+				}
+#ifdef CONFIG_80211AX_HE
+				if (psta->hepriv.he_option) {
+					/* NOTE: ltf_gi is unused in rtw_he_mcs_to_data_rate currently */
+					RTW_PRINT_SEL(m, "he_max_bitrate_20/40/80/160m=(%u, %u, %u, %u)\n",
+						(rtw_he_mcs_to_data_rate(0, psta->phl_sta->asoc_cap.ltf_gi, psta->hepriv.he_highest_rate) + 1) >> 1,
+						(rtw_he_mcs_to_data_rate(1, psta->phl_sta->asoc_cap.ltf_gi, psta->hepriv.he_highest_rate) + 1) >> 1,
+						(rtw_he_mcs_to_data_rate(2, psta->phl_sta->asoc_cap.ltf_gi, psta->hepriv.he_highest_rate) + 1) >> 1,
+						(rtw_he_mcs_to_data_rate(3, psta->phl_sta->asoc_cap.ltf_gi, psta->hepriv.he_highest_rate) + 1) >> 1);
+
+					RTW_MAP_DUMP_SEL_ALWAYS(m, "he_cap=", psta->hepriv.he_cap, HE_CAP_ELE_MAX_LEN);
+				}
+#endif
+#endif /* CONFIG_80211N_VHT */
+#endif /* CONFIG_80211N_HT */
+				RTW_PRINT_SEL(m, "tx_nss=%d\n", rtw_get_sta_tx_nss(padapter, psta));
+				RTW_PRINT_SEL(m, "rx_nss=%d\n", rtw_get_sta_rx_nss(padapter, psta));
+				RTW_PRINT_SEL(m, "sleepq_len=%d\n", psta->sleepq_len);
+				RTW_PRINT_SEL(m, "sta_xmitpriv.vo_q_qcnt=%d\n", psta->sta_xmitpriv.vo_q.qcnt);
+				RTW_PRINT_SEL(m, "sta_xmitpriv.vi_q_qcnt=%d\n", psta->sta_xmitpriv.vi_q.qcnt);
+				RTW_PRINT_SEL(m, "sta_xmitpriv.be_q_qcnt=%d\n", psta->sta_xmitpriv.be_q.qcnt);
+				RTW_PRINT_SEL(m, "sta_xmitpriv.bk_q_qcnt=%d\n", psta->sta_xmitpriv.bk_q.qcnt);
+
+				RTW_PRINT_SEL(m, "capability=0x%x\n", psta->capability);
+				RTW_PRINT_SEL(m, "flags=0x%x\n", psta->flags);
+				RTW_PRINT_SEL(m, "isPMF=%d\n", (psta->flags & WLAN_STA_MFP)?1:0);
+				RTW_PRINT_SEL(m, "wpa_psk=0x%x\n", psta->wpa_psk);
+				RTW_PRINT_SEL(m, "wpa2_group_cipher=0x%x\n", psta->wpa2_group_cipher);
+				RTW_PRINT_SEL(m, "wpa2_pairwise_cipher=0x%x\n", psta->wpa2_pairwise_cipher);
+				RTW_PRINT_SEL(m, "qos_info=0x%x\n", psta->qos_info);
+				RTW_PRINT_SEL(m, "dot118021XPrivacy=0x%x\n", psta->dot118021XPrivacy);
+				RTW_PRINT_SEL(m, "AuthAlgrthm=0x%x\n", psta->authalg);
+#ifdef CONFIG_RECV_REORDERING_CTRL
+				sta_rx_reorder_ctl_dump(m, psta);
+#endif
+#ifdef CONFIG_TDLS
+				RTW_PRINT_SEL(m, "tdls_sta_state=0x%08x\n", psta->tdls_sta_state);
+				RTW_PRINT_SEL(m, "PeerKey_Lifetime=%d\n", psta->TDLS_PeerKey_Lifetime);
+#endif /* CONFIG_TDLS */
+				if(psta->bssratelen >= 0 && psta->bssratelen <= 16)
+					RTW_MAP_DUMP_SEL_ALWAYS(m, "bssrateset=", psta->bssrateset, psta->bssratelen);
+				RTW_PRINT_SEL(m, "rx_data_uc_pkts=%llu\n", sta_rx_data_uc_pkts(psta));
+				RTW_PRINT_SEL(m, "rx_data_mc_pkts=%llu\n", psta->sta_stats.rx_data_mc_pkts);
+				RTW_PRINT_SEL(m, "rx_data_bc_pkts=%llu\n", psta->sta_stats.rx_data_bc_pkts);
+				RTW_PRINT_SEL(m, "rx_uc_bytes=%llu\n", sta_rx_uc_bytes(psta));
+				RTW_PRINT_SEL(m, "rx_mc_bytes=%llu\n", psta->sta_stats.rx_mc_bytes);
+				RTW_PRINT_SEL(m, "rx_bc_bytes=%llu\n", psta->sta_stats.rx_bc_bytes);
+				if (psta->sta_stats.rx_tp_kbits >> 10)
+					RTW_PRINT_SEL(m, "rx_tp =%d (Mbps)\n", psta->sta_stats.rx_tp_kbits >> 10);
+				else
+					RTW_PRINT_SEL(m, "rx_tp =%d (Kbps)\n", psta->sta_stats.rx_tp_kbits);
+
+				RTW_PRINT_SEL(m, "tx_data_pkts=%llu\n", psta->sta_stats.tx_pkts);
+				RTW_PRINT_SEL(m, "tx_bytes=%llu\n", psta->sta_stats.tx_bytes);
+				if (psta->sta_stats.tx_tp_kbits >> 10)
+					RTW_PRINT_SEL(m, "tx_tp =%d (Mbps)\n", psta->sta_stats.tx_tp_kbits >> 10);
+				else
+					RTW_PRINT_SEL(m, "tx_tp =%d (Kbps)\n", psta->sta_stats.tx_tp_kbits);
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+				RTW_PRINT_SEL(m, "rm_en_cap="RM_CAP_FMT"\n", RM_CAP_ARG(psta->rm_en_cap));
+#endif
+				dump_st_ctl(m, &psta->st_ctl);
+
+				if (STA_OP_WFD_MODE(psta))
+					RTW_PRINT_SEL(m, "op_wfd_mode:0x%02x\n", STA_OP_WFD_MODE(psta));
+
+				RTW_PRINT_SEL(m, "tx_bitrate_100kbps=%u\n",
+						rtw_desc_rate_to_bitrate(psta->phl_sta->rlink->chandef.bw,
+								       rtw_get_current_tx_rate(padapter, psta),
+								       rtw_get_current_tx_sgi(padapter, psta)));
+				RTW_PRINT_SEL(m, "rx_bitrate_100kbps=%u\n",
+						rtw_desc_rate_to_bitrate(psta->phl_sta->rlink->chandef.bw,
+					    psta->curr_rx_rate, psta->curr_rx_gi_ltf));
+
+				RTW_PRINT_SEL(m, "rssi=%d\n", rtw_phl_get_sta_rssi(psta->phl_sta));
+
+				dump_phl_tring_status(m, padapter, psta);
+				RTW_PRINT_SEL(m, "==============================\n");
+			}
+
+		}
+
+	}
+
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	return 0;
+}
+
+#endif
+
+#ifdef CONFIG_RTKM
+int proc_get_rtkm_info(struct seq_file *m, void *v)
+{
+	rtkm_dump_mstatus(m);
+	return 0;
+}
+
+ssize_t proc_set_rtkm_info(struct file *file, const char __user *buffer,
+			   size_t count, loff_t *pos, void *data)
+{
+	char tmp[32] = { 0 };
+	unsigned int mask = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%x", &mask);
+
+		if (num == 1)
+			rtkm_set_trace(mask);
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTKM */
+
+#ifdef DBG_MEMORY_LEAK
+#include <asm/atomic.h>
+extern ATOMIC_T _malloc_cnt;
+extern ATOMIC_T _malloc_size;
+
+int proc_get_malloc_cnt(struct seq_file *m, void *v)
+{
+	RTW_PRINT_SEL(m, "_malloc_cnt=%d\n", atomic_read(&_malloc_cnt));
+	RTW_PRINT_SEL(m, "_malloc_size=%d\n", atomic_read(&_malloc_size));
+
+	return 0;
+}
+#endif /* DBG_MEMORY_LEAK */
+
+#ifdef CONFIG_FIND_BEST_CHANNEL
+int proc_get_best_channel(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	u32 i, best_channel_24G = 1, best_channel_5G = 36, index_24G = 0, index_5G = 0;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].flags & RTW_CHF_DIS)
+			continue;
+		if (chset->chs[i].ChannelNum == 1)
+			index_24G = i;
+		if (chset->chs[i].ChannelNum == 36)
+			index_5G = i;
+	}
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].flags & RTW_CHF_DIS)
+			continue;
+		/* 2.4G */
+		if (chset->chs[i].ChannelNum == 6) {
+			if (chset->chs[i].rx_count < chset->chs[index_24G].rx_count) {
+				index_24G = i;
+				best_channel_24G = chset->chs[i].ChannelNum;
+			}
+		}
+
+		/* 5G */
+		if (chset->chs[i].ChannelNum >= 36
+		    && chset->chs[i].ChannelNum < 140) {
+			/* Find primary channel */
+			if (((chset->chs[i].ChannelNum - 36) % 8 == 0)
+			    && (chset->chs[i].rx_count < chset->chs[index_5G].rx_count)) {
+				index_5G = i;
+				best_channel_5G = chset->chs[i].ChannelNum;
+			}
+		}
+
+		if (chset->chs[i].ChannelNum >= 149
+		    && chset->chs[i].ChannelNum < 165) {
+			/* find primary channel */
+			if (((chset->chs[i].ChannelNum - 149) % 8 == 0)
+			    && (chset->chs[i].rx_count < chset->chs[index_5G].rx_count)) {
+				index_5G = i;
+				best_channel_5G = chset->chs[i].ChannelNum;
+			}
+		}
+#if 1 /* debug */
+		RTW_PRINT_SEL(m, "The rx cnt of channel %3d = %d\n",
+			chset->chs[i].ChannelNum, chset->chs[i].rx_count);
+#endif
+	}
+
+	RTW_PRINT_SEL(m, "best_channel_5G = %d\n", best_channel_5G);
+	RTW_PRINT_SEL(m, "best_channel_24G = %d\n", best_channel_24G);
+
+	return 0;
+}
+
+ssize_t proc_set_best_channel(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int i;
+		for (i = 0; i < chset->chs_len; i++)
+			chset->chs[i].rx_count = 0;
+
+		RTW_INFO("set %s\n", "Clean Best Channel Count");
+	}
+
+	return count;
+}
+#endif /* CONFIG_FIND_BEST_CHANNEL */
+
+
+#ifdef CONFIG_PCI_HCI
+
+ssize_t proc_set_pci_bridge_conf_space(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv       *pdvobjpriv = adapter_to_dvobj(padapter);
+	PPCI_DATA pci_data = dvobj_to_pci(pdvobjpriv);
+	struct pci_dev  *pdev = pci_data->ppcidev;
+	struct pci_dev  *bridge_pdev = pdev->bus->self;
+
+	char tmp[32] = { 0 };
+	int num;
+
+	u32 reg = 0, value = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num = sscanf(tmp, "%x %x", &reg, &value);
+		if (num != 2) {
+			RTW_INFO("invalid parameter!\n");
+			return count;
+		}
+
+		if (reg >= 0x1000) {
+			RTW_INFO("invalid register!\n");
+			return count;
+		}
+
+		if (value > 0xFF) {
+			RTW_INFO("invalid value! Only one byte\n");
+			return count;
+		}
+
+		RTW_INFO(FUNC_ADPT_FMT ": register 0x%x value 0x%x\n",
+			FUNC_ADPT_ARG(padapter), reg, value);
+
+		pci_write_config_byte(bridge_pdev, reg, value);
+	}
+	return count;
+}
+
+
+int proc_get_pci_bridge_conf_space(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *) rtw_netdev_priv(dev);
+	struct dvobj_priv       *pdvobjpriv = adapter_to_dvobj(padapter);
+	PPCI_DATA pci_data = dvobj_to_pci(pdvobjpriv);
+	struct pci_dev  *pdev = pci_data->ppcidev;
+	struct pci_dev  *bridge_pdev = pdev->bus->self;
+
+	u32 tmp[4] = { 0 };
+	u32 i, j;
+
+	RTW_PRINT_SEL(m, "\n*****  PCI Host Device Configuration Space*****\n\n");
+
+	for (i = 0; i < 0x1000; i += 0x10) {
+		for (j = 0 ; j < 4 ; j++)
+			pci_read_config_dword(bridge_pdev, i + j * 4, tmp+j);
+
+		RTW_PRINT_SEL(m, "%03x: %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
+			i, tmp[0] & 0xFF, (tmp[0] >> 8) & 0xFF, (tmp[0] >> 16) & 0xFF, (tmp[0] >> 24) & 0xFF,
+			tmp[1] & 0xFF, (tmp[1] >> 8) & 0xFF, (tmp[1] >> 16) & 0xFF, (tmp[1] >> 24) & 0xFF,
+			tmp[2] & 0xFF, (tmp[2] >> 8) & 0xFF, (tmp[2] >> 16) & 0xFF, (tmp[2] >> 24) & 0xFF,
+			tmp[3] & 0xFF, (tmp[3] >> 8) & 0xFF, (tmp[3] >> 16) & 0xFF, (tmp[3] >> 24) & 0xFF);
+	}
+	return 0;
+}
+
+
+ssize_t proc_set_pci_conf_space(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv       *pdvobjpriv = adapter_to_dvobj(padapter);
+	PPCI_DATA pci_data = dvobj_to_pci(pdvobjpriv);
+	struct pci_dev  *pdev = pci_data->ppcidev;
+
+	char tmp[32] = { 0 };
+	int num;
+
+	u32 reg = 0, value = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num = sscanf(tmp, "%x %x", &reg, &value);
+
+		if (num != 2) {
+			RTW_INFO("invalid parameter!\n");
+			return count;
+		}
+
+
+		if (reg >= 0x1000) {
+			RTW_INFO("invalid register!\n");
+			return count;
+		}
+
+		if (value > 0xFF) {
+			RTW_INFO("invalid value! Only one byte\n");
+			return count;
+		}
+
+		RTW_INFO(FUNC_ADPT_FMT ": register 0x%x value 0x%x\n",
+			FUNC_ADPT_ARG(padapter), reg, value);
+
+		pci_write_config_byte(pdev, reg, value);
+
+
+	}
+	return count;
+}
+
+
+int proc_get_pci_conf_space(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *) rtw_netdev_priv(dev);
+	struct dvobj_priv       *pdvobjpriv = adapter_to_dvobj(padapter);
+	PPCI_DATA pci_data = dvobj_to_pci(pdvobjpriv);
+	struct pci_dev *pdev = pci_data->ppcidev;
+	struct pci_dev *bridge_pdev = pdev->bus->self;
+
+	u32 tmp[4] = { 0 };
+	u32 i, j;
+
+	RTW_PRINT_SEL(m, "\n*****  PCI Device Configuration Space *****\n\n");
+
+	for (i = 0; i < 0x1000; i += 0x10) {
+		for (j = 0 ; j < 4 ; j++)
+			pci_read_config_dword(pdev, i + j * 4, tmp+j);
+
+		RTW_PRINT_SEL(m, "%03x: %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
+			i, tmp[0] & 0xFF, (tmp[0] >> 8) & 0xFF, (tmp[0] >> 16) & 0xFF, (tmp[0] >> 24) & 0xFF,
+			tmp[1] & 0xFF, (tmp[1] >> 8) & 0xFF, (tmp[1] >> 16) & 0xFF, (tmp[1] >> 24) & 0xFF,
+			tmp[2] & 0xFF, (tmp[2] >> 8) & 0xFF, (tmp[2] >> 16) & 0xFF, (tmp[2] >> 24) & 0xFF,
+			tmp[3] & 0xFF, (tmp[3] >> 8) & 0xFF, (tmp[3] >> 16) & 0xFF, (tmp[3] >> 24) & 0xFF);
+	}
+
+	return 0;
+}
+
+
+int proc_get_pci_aspm(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *) rtw_netdev_priv(dev);
+	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
+	PPCI_DATA pci_data = dvobj_to_pci(pdvobjpriv);
+	struct pci_priv	*pcipriv = &(pci_data->pcipriv);
+	u8 tmp8 = 0;
+	u16 tmp16 = 0;
+	u32 tmp32 = 0;
+	u8 l1_idle = 0;
+	u32 l1ss_cap_ptr = 0;
+
+
+	RTW_PRINT_SEL(m, "***** ASPM Capability *****\n");
+
+	pci_read_config_dword(pci_data->ppcidev, pcipriv->pciehdr_offset + PCI_EXP_LNKCAP, &tmp32);
+
+	RTW_PRINT_SEL(m, "CLK REQ:	%s\n", (tmp32&PCI_EXP_LNKCAP_CLKPM) ? "Enable" : "Disable");
+	RTW_PRINT_SEL(m, "ASPM L0s:	%s\n", (tmp32&BIT10) ? "Enable" : "Disable");
+	RTW_PRINT_SEL(m, "ASPM L1:	%s\n", (tmp32&BIT11) ? "Enable" : "Disable");
+
+	l1ss_cap_ptr = pci_find_ext_capability(pci_data->ppcidev, PCI_EXT_CAP_ID_L1SS);
+	pci_read_config_byte(pci_data->ppcidev, l1ss_cap_ptr + PCI_L1SS_CAP, &tmp8);
+	RTW_PRINT_SEL(m, "ASPM L1OFF:	%s\n", (tmp8 & (PCI_L1SS_CAP_PCIPM_L1_2 |
+				 PCI_L1SS_CAP_PCIPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2 |
+				 PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_L1_PM_SS)) ?
+				 "Enable" : "Disable");
+
+	RTW_PRINT_SEL(m, "***** ASPM CTRL Reg *****\n");
+
+	pci_read_config_word(pci_data->ppcidev, pcipriv->pciehdr_offset + PCI_EXP_LNKCTL, &tmp16);
+
+	RTW_PRINT_SEL(m, "CLK REQ:	%s\n", (tmp16&PCI_EXP_LNKCTL_CLKREQ_EN) ? "Enable" : "Disable");
+	RTW_PRINT_SEL(m, "ASPM L0s:	%s\n", (tmp16&BIT0) ? "Enable" : "Disable");
+	RTW_PRINT_SEL(m, "ASPM L1:	%s\n", (tmp16&BIT1) ? "Enable" : "Disable");
+
+	pci_read_config_byte(pci_data->ppcidev, l1ss_cap_ptr + PCI_L1SS_CTL1, &tmp8);
+	RTW_PRINT_SEL(m, "ASPM L1OFF:	%s\n", (tmp8 & (PCI_L1SS_CTL1_L1SS_MASK)) ? "Enable" : "Disable");
+
+	RTW_PRINT_SEL(m, "***** ASPM Backdoor *****\n");
+
+	pci_read_config_byte(pci_data->ppcidev, 0x719, &tmp8);
+	RTW_PRINT_SEL(m, "CLK REQ:	%s\n", (tmp8 & BIT4) ? "Enable" : "Disable");
+
+	pci_read_config_byte(pci_data->ppcidev, 0x70f, &tmp8);
+	l1_idle = tmp8 & 0x38;
+	RTW_PRINT_SEL(m, "ASPM L0s:	%s\n", (tmp8 & BIT7) ? "Enable" : "Disable");
+
+	pci_read_config_byte(pci_data->ppcidev, 0x719, &tmp8);
+	RTW_PRINT_SEL(m, "ASPM L1:	%s\n", (tmp8 & BIT3) ? "Enable" : "Disable");
+
+	pci_read_config_byte(pci_data->ppcidev, 0x718, &tmp8);
+	RTW_PRINT_SEL(m, "ASPM L1OFF:	%s\n", (tmp8 & BIT5) ? "Enable" : "Disable");
+
+	RTW_PRINT_SEL(m, "********* MISC **********\n");
+	RTW_PRINT_SEL(m, "ASPM L1 Idel Time: 0x%x\n", l1_idle>>3);
+	RTW_PRINT_SEL(m, "*************************\n");
+
+	return 0;
+}
+
+int proc_get_rx_ring(struct seq_file *m, void *v)
+{
+
+	return 0;
+}
+
+int proc_get_tx_ring(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *) rtw_netdev_priv(dev);
+	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	int i, j, k;
+	unsigned long sp_flags;
+
+	_rtw_spinlock_irq(&dvobj_to_pci(pdvobjpriv)->irq_th_lock, &sp_flags);
+	for (i = 0; i < PCI_MAX_TX_QUEUE_COUNT; i++) {
+		struct rtw_tx_ring *tx_ring = &pxmitpriv->tx_ring[i];
+
+		RTW_PRINT_SEL(m, "tx ring[%d] (%p)\n", i, tx_ring);
+		RTW_PRINT_SEL(m, "  dma: 0x%08x\n", (int) tx_ring->dma);
+		RTW_PRINT_SEL(m, "  idx: %d\n", tx_ring->idx);
+		RTW_PRINT_SEL(m, "  entries: %d\n", tx_ring->entries);
+		/*		RTW_PRINT_SEL(m, "  queue: %d\n", tx_ring->queue); */
+		RTW_PRINT_SEL(m, "  qlen: %d\n", tx_ring->qlen);
+
+		for (j = 0; j < pxmitpriv->txringcount[i]; j++) {
+#ifdef CONFIG_TRX_BD_ARCH
+			struct tx_buf_desc *entry = &tx_ring->buf_desc[j];
+			RTW_PRINT_SEL(m, "  buf_desc[%03d]: %p\n", j, entry);
+#else
+			struct tx_desc *entry = &tx_ring->desc[j];
+			RTW_PRINT_SEL(m, "  desc[%03d]: %p\n", j, entry);
+#endif
+
+			for (k = 0; k < sizeof(*entry) / 4; k++) {
+				if ((k % 4) == 0)
+					RTW_PRINT_SEL(m, "  0x%03x", k);
+
+				RTW_PRINT_SEL(m, " 0x%08x ", ((int *) entry)[k]);
+
+				if ((k % 4) == 3)
+					RTW_PRINT_SEL(m, "\n");
+			}
+		}
+	}
+	_rtw_spinunlock_irq(&dvobj_to_pci(pdvobjpriv)->irq_th_lock, &sp_flags);
+
+	return 0;
+}
+
+#ifdef DBG_TXBD_DESC_DUMP
+int proc_get_tx_ring_ext(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *) rtw_netdev_priv(dev);
+	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct rtw_tx_desc_backup *pbuf;
+	int i, j, k, idx;
+	unsigned long sp_flags;
+
+	RTW_PRINT_SEL(m, "<<<< tx ring ext dump settings >>>>\n");
+	RTW_PRINT_SEL(m, " - backup frame num: %d\n", TX_BAK_FRMAE_CNT);
+	RTW_PRINT_SEL(m, " - backup max. desc size: %d bytes\n", TX_BAK_DESC_LEN);
+	RTW_PRINT_SEL(m, " - backup data size: %d bytes\n\n", TX_BAK_DATA_LEN);
+
+	if (!pxmitpriv->dump_txbd_desc) {
+		RTW_PRINT_SEL(m, "Dump function is disabled.\n");
+		return 0;
+	}
+
+	_rtw_spinlock_irq(&dvobj_to_pci(pdvobjpriv)->irq_th_lock, &sp_flags);
+	for (i = 0; i < HW_QUEUE_ENTRY; i++) {
+		struct rtw_tx_ring *tx_ring = &pxmitpriv->tx_ring[i];
+
+		idx = rtw_get_tx_desc_backup(padapter, i, &pbuf);
+
+		RTW_PRINT_SEL(m, "Tx ring[%d]", i);
+		switch (i) {
+		case 0:
+			RTW_PRINT_SEL(m, " (VO)\n");
+			break;
+		case 1:
+			RTW_PRINT_SEL(m, " (VI)\n");
+			break;
+		case 2:
+			RTW_PRINT_SEL(m, " (BE)\n");
+			break;
+		case 3:
+			RTW_PRINT_SEL(m, " (BK)\n");
+			break;
+		case 4:
+			RTW_PRINT_SEL(m, " (BCN)\n");
+			break;
+		case 5:
+			RTW_PRINT_SEL(m, " (MGT)\n");
+			break;
+		case 6:
+			RTW_PRINT_SEL(m, " (HIGH)\n");
+			break;
+		case 7:
+			RTW_PRINT_SEL(m, " (TXCMD)\n");
+			break;
+		default:
+			RTW_PRINT_SEL(m, " (?)\n");
+			break;
+		}
+
+		RTW_PRINT_SEL(m, "  Entries: %d\n", TX_BAK_FRMAE_CNT);
+		RTW_PRINT_SEL(m, "  Last idx: %d\n", idx);
+
+		for (j = 0; j < TX_BAK_FRMAE_CNT; j++) {
+			RTW_PRINT_SEL(m, "  desc[%03d]:\n", j);
+
+			for (k = 0; k < (pbuf->tx_desc_size) / 4; k++) {
+				if ((k % 4) == 0)
+					RTW_PRINT_SEL(m, "  0x%03x", k);
+
+				RTW_PRINT_SEL(m, " 0x%08x ", ((int *)pbuf->tx_bak_desc)[k]);
+
+				if ((k % 4) == 3)
+					RTW_PRINT_SEL(m, "\n");
+			}
+
+#if 1 /* data dump */
+			if (pbuf->tx_desc_size) {
+				RTW_PRINT_SEL(m, "  data[%03d]:\n", j);
+
+				for (k = 0; k < (TX_BAK_DATA_LEN) / 4; k++) {
+					if ((k % 4) == 0)
+						RTW_PRINT_SEL(m, "  0x%03x", k);
+
+					RTW_PRINT_SEL(m, " 0x%08x ", ((int *)pbuf->tx_bak_data_hdr)[k]);
+
+					if ((k % 4) == 3)
+						RTW_PRINT_SEL(m, "\n");
+				}
+				RTW_PRINT_SEL(m, "\n");
+			}
+#endif
+
+			RTW_PRINT_SEL(m, "  R/W pointer: %d/%d\n", pbuf->tx_bak_rp, pbuf->tx_bak_wp);
+
+			pbuf = pbuf + 1;
+		}
+		RTW_PRINT_SEL(m, "\n");
+	}
+	_rtw_spinunlock_irq(&dvobj_to_pci(pdvobjpriv)->irq_th_lock, &sp_flags);
+
+	return 0;
+}
+
+ssize_t proc_set_tx_ring_ext(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
+	char tmp[32];
+	u32 reset = 0;
+	u32 dump = 0;
+	unsigned long sp_flags;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%u %u", &dump, &reset);
+
+		if (num != 2) {
+			RTW_INFO("invalid parameter!\n");
+			return count;
+		}
+
+		_rtw_spinlock_irq(&dvobj_to_pci(pdvobjpriv)->irq_th_lock, &sp_flags);
+		pxmitpriv->dump_txbd_desc = (BOOLEAN) dump;
+
+		if (reset == 1)
+			rtw_tx_desc_backup_reset();
+
+		_rtw_spinunlock_irq(&dvobj_to_pci(pdvobjpriv)->irq_th_lock, &sp_flags);
+
+	}
+
+	return count;
+}
+
+#endif
+
+#endif
+
+#ifdef CONFIG_WOWLAN
+int proc_get_wow_enable(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *registry_pair = &padapter->registrypriv;
+
+	RTW_PRINT_SEL(m, "wow - %s\n", (registry_pair->wowlan_enable)? "enable" : "disable");
+	return 0;
+}
+
+ssize_t proc_set_wow_enable(struct file *file, const char __user *buffer,
+			    size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *registry_pair = &padapter->registrypriv;
+	char tmp[8];
+	int num = 0;
+	int mode = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count))
+		num = sscanf(tmp, "%d", &mode);
+	else
+		return -EFAULT;
+
+	if (num != 1) {
+		RTW_ERR("%s: %s - invalid parameter!\n", __func__, tmp);
+		return -EINVAL;
+	}
+
+	if (mode == 1) {
+		RTW_PRINT("%s: wowlan - enable\n", __func__);
+	} else if (mode == 0) {
+		RTW_PRINT("%s: wowlan - disable\n", __func__);
+	} else {
+		RTW_ERR("%s: %s - invalid parameter!, mode=%d\n",
+			__func__, tmp, mode);
+		return -EINVAL;
+	}
+
+	registry_pair->wowlan_enable = mode;
+
+	return count;
+}
+
+int proc_get_pattern_info(struct seq_file *m, void *v)
+{
+// TODO(jw.hou): Get pattern info from PHL.
+#if 0
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	u8 val8;
+	char str_1[128];
+	char *p_str;
+	int i = 0 , j = 0, k = 0;
+	int len = 0, max_len = 0, total = 0;
+
+	p_str = str_1;
+	max_len = sizeof(str_1);
+
+	total = pwrpriv->wowlan_pattern_idx;
+
+	rtw_set_default_pattern(padapter);
+
+	/*show pattern*/
+	RTW_PRINT_SEL(m, "\n======[Pattern Info.]======\n");
+	RTW_PRINT_SEL(m, "pattern number: %d\n", total);
+	RTW_PRINT_SEL(m, "support default patterns: %c\n",
+		      (pwrpriv->default_patterns_en) ? 'Y' : 'N');
+
+	for (k = 0; k < total ; k++) {
+		RTW_PRINT_SEL(m, "\npattern idx: %d\n", k);
+		RTW_PRINT_SEL(m, "pattern content:\n");
+
+		p_str = str_1;
+		max_len = sizeof(str_1);
+		for (i = 0 ; i < MAX_WKFM_PATTERN_SIZE / 8 ; i++) {
+			_rtw_memset(p_str, 0, max_len);
+			len = 0;
+			for (j = 0 ; j < 8 ; j++) {
+				val8 = pwrpriv->patterns[k].content[i * 8 + j];
+				len += snprintf(p_str + len, max_len - len,
+						"%02x ", val8);
+			}
+			RTW_PRINT_SEL(m, "%s\n", p_str);
+		}
+		RTW_PRINT_SEL(m, "\npattern mask:\n");
+		for (i = 0 ; i < MAX_WKFM_SIZE / 8 ; i++) {
+			_rtw_memset(p_str, 0, max_len);
+			len = 0;
+			for (j = 0 ; j < 8 ; j++) {
+				val8 = pwrpriv->patterns[k].mask[i * 8 + j];
+				len += snprintf(p_str + len, max_len - len,
+						"%02x ", val8);
+			}
+			RTW_PRINT_SEL(m, "%s\n", p_str);
+		}
+
+		RTW_PRINT_SEL(m, "\npriv_pattern_len:\n");
+		RTW_PRINT_SEL(m, "pattern_len: %d\n", pwrpriv->patterns[k].len);
+		RTW_PRINT_SEL(m, "*****************\n");
+	}
+#endif
+
+	return 0;
+}
+
+ssize_t proc_set_pattern_info(struct file *file, const char __user *buffer,
+			      size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv  *registry_par = &adapter->registrypriv;
+	struct rtw_wowcam_upd_info wowcam_info = {0};
+	u8 tmp[MAX_WKFM_PATTERN_STR_LEN + 1] = {0};
+
+	if (!(registry_par->wakeup_event & BIT(3))) {
+		RTW_INFO("%s: customized pattern disabled, wakeup_event: %#2x\n",
+			 __func__, registry_par->wakeup_event);
+		return -EOPNOTSUPP;
+	}
+
+	if (count < 1)
+		return -EINVAL;
+
+	if (count >= sizeof(tmp)) {
+		RTW_ERR("%s: pattern string is too long, count=%zu\n",
+			__func__, count);
+		return -EINVAL;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		if (strncmp(tmp, "clean", 5) == 0) {
+			rtw_wow_pattern_clean(adapter, RTW_CUSTOMIZED_PATTERN);
+		} else {
+			if (rtw_wowlan_parser_pattern_cmd(tmp,
+							  wowcam_info.ptrn,
+							  &wowcam_info.ptrn_len,
+							  wowcam_info.mask)) {
+				if (_FAIL == rtw_wow_pattern_set(adapter,
+								 &wowcam_info,
+								 RTW_CUSTOMIZED_PATTERN))
+					return -EFAULT;
+			} else {
+				return -EINVAL;
+			}
+		}
+	} else {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	return count;
+}
+
+int proc_get_wakeup_event(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+
+	RTW_PRINT_SEL(m, "wakeup event: %#02x\n", registry_par->wakeup_event);
+	return 0;
+}
+
+ssize_t proc_set_wakeup_event(struct file *file, const char __user *buffer,
+			      size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+	u32 wakeup_event = 0;
+
+	u8 tmp[8] = {0};
+	int num = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count))
+		num = sscanf(tmp, "%u", &wakeup_event);
+	else
+		return -EFAULT;
+
+	if (num == 1 && wakeup_event <= 0x1f) {
+		registry_par->wakeup_event = wakeup_event;
+
+		if (!(wakeup_event & BIT(1)))
+			rtw_wow_pattern_clean(padapter, RTW_DEFAULT_PATTERN);
+
+		if (!(wakeup_event & BIT(3)))
+			rtw_wow_pattern_clean(padapter, RTW_CUSTOMIZED_PATTERN);
+
+		RTW_INFO("%s: wakeup_event: %#2x\n",
+			 __func__, registry_par->wakeup_event);
+	} else {
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+int proc_get_wakeup_reason(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+
+	RTW_PRINT_SEL(m, "last wake reason: 0x%02x\n", wowpriv->wow_wake_reason);
+
+	return 0;
+}
+
+int proc_get_wake_indication(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+#ifdef RTW_DETECT_HANG
+	struct dvobj_priv *pdvobjpriv = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &pdvobjpriv->drv_dbg;
+	struct hang_info *phang_info = &pdbgpriv->dbg_hang_info;
+	struct fw_hang_info *pfw_hang_info = &phang_info->dbg_fw_hang_info;
+	struct rxff_hang_info *prxff_hang_info = &phang_info->dbg_rxff_hang_info;
+#endif /* RTW_DETECT_HANG */
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+
+#ifdef RTW_DETECT_HANG
+	if (pfw_hang_info->dbg_is_fw_hang ||
+	    pfw_hang_info->dbg_is_fw_gone ||
+	    prxff_hang_info->dbg_is_rxff_hang)
+		RTW_PRINT_SEL(m, "wake indication: -1\n");
+	else
+#endif /* RTW_DETECT_HANG */
+		RTW_PRINT_SEL(m, "wake indication: %#02x\n", !pwrpriv->bInSuspend);
+
+	return 0;
+}
+
+#ifdef CONFIG_MDNS_OFFLOAD
+#define MDNS_RESP_PROC_BUF_LEN 1027 /* MAX_MDNS_RESP_LEN * 2 + 3 */
+#define MDNS_RESP_STR_BUF_LEN 1025  /* MAX_MDNS_RESP_LEN * 2 + 1 */
+#define MDNS_PASSTHRU_PROC_BUF_LEN 511 /* MAX_MDNS_DOMAIN_NAME_LEN * 2 + 1 */
+static void _parse_mdns_domain_name(struct seq_file *m, u8 *buf, u16 buf_len, u16 cur_offset, u8 *name_len)
+{
+	u8 name[255] = {0};
+	u8 lable_remain_len = 0;
+	u16 name_ptr = 0;
+	u16 len = 0;
+
+	while (1) {
+		if (cur_offset >= buf_len || len >= 255) {
+			len = 0;
+			goto exit;
+		}
+
+		if (!lable_remain_len) {
+			/* compression label */
+			if ((buf[cur_offset] & 0xc0) == 0xc0) {
+				if (cur_offset + 1 >= buf_len) {
+					len = 0;
+					goto exit;
+				}
+
+				name_ptr = ((buf[cur_offset] & 0x3f) << 8) +
+					   buf[cur_offset + 1];
+				if (name_ptr >= cur_offset) {
+					len = 0;
+					goto exit;
+				}
+				cur_offset = name_ptr;
+			}
+
+			/* label length */
+			if (buf[cur_offset] > 63) {
+				len = 0;
+				goto exit;
+			}
+
+			if (buf[cur_offset] == 0) {
+				name[len] = '\0';
+				len += 1;
+				goto exit;
+			} else {
+				name[len] = '.';
+				len += 1;
+				lable_remain_len = buf[cur_offset];
+				cur_offset++;
+			}
+		} else {
+			buf[cur_offset] = tolower(buf[cur_offset]);
+			name[len] = buf[cur_offset];
+			len += 1;
+			lable_remain_len = lable_remain_len - 1;
+			cur_offset++;
+		}
+	}
+
+exit:
+	if (name_len)
+		*name_len = (u8)len;
+
+	if (len)
+		RTW_PRINT_SEL(m, "domain name=%s\n", name + 1);
+	else
+		RTW_PRINT_SEL(m, "domain name parsing failed\n");
+}
+
+ssize_t proc_set_wow_mdns_resp(struct file *file, const char __user *buffer,
+			       size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char *tmp = NULL;
+	u8 resp_idx = 0;
+	u8 *mdns_resp = NULL;
+	int err = 0;
+	int num, i;
+	size_t resp_str_len = 0;
+	u16 resp_hex_len = 0;
+
+	if (!padapter) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	tmp = rtw_zvmalloc(MDNS_RESP_PROC_BUF_LEN);
+	mdns_resp = rtw_zvmalloc(MDNS_RESP_STR_BUF_LEN);
+	if (!tmp || !mdns_resp) {
+		RTW_INFO("alloc buffer failed\n");
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (count < 1 || count > MDNS_RESP_PROC_BUF_LEN) {
+		RTW_INFO("argument size not available\n");
+		err = -EINVAL;
+		goto exit;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count)) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (strncmp(tmp, "clear", 5) == 0) {
+		RTW_INFO("clear all mdns resp\n");
+		for (i = 0; i < MAX_MDNS_RESP_NUM; i++)
+			rtw_wow_del_mdns_resp(padapter, i);
+		goto exit;
+	}
+
+	num = sscanf(tmp, "%hhu %s", &resp_idx, mdns_resp);
+	if ((num < 1) || (resp_idx > (MAX_MDNS_RESP_NUM - 1))) {
+		RTW_INFO("argument unavailable\n");
+		err = -EINVAL;
+		goto exit;
+	} else if (num == 1) {
+		RTW_INFO("remove mdns resp(%d)\n", resp_idx);
+		rtw_wow_del_mdns_resp(padapter, resp_idx);
+	} else {
+		resp_str_len = strlen(mdns_resp);
+		if (resp_str_len % 2 != 0) {
+			err = -EINVAL;
+			RTW_INFO("mdns_resp content format is incorrect\n");
+			goto exit;
+		}
+
+		for (i = 0; i < resp_str_len; i += 2) {
+			mdns_resp[resp_hex_len] = key_2char2num(mdns_resp[i], mdns_resp[i + 1]);
+			resp_hex_len++;
+		}
+
+		rtw_wow_add_mdns_resp(padapter, resp_idx, mdns_resp, resp_hex_len);
+	}
+
+exit:
+	if (tmp)
+		rtw_vmfree(tmp, MDNS_RESP_PROC_BUF_LEN);
+	if (mdns_resp)
+		rtw_vmfree(mdns_resp, MDNS_RESP_STR_BUF_LEN);
+
+	return err ? err : count;
+}
+
+int proc_get_wow_mdns_resp(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+	u8 resp_idx = 0;
+	u16 i = 0;
+
+	if (!padapter)
+		goto exit;
+
+	for (resp_idx = 0; resp_idx < MAX_MDNS_RESP_NUM; resp_idx++) {
+		rtw_wow_get_mdns_resp_ent(padapter, resp_idx, &resp_entry);
+
+		RTW_PRINT_SEL(m, "mdns resp(%d):", resp_idx);
+
+		if (resp_entry->content_len == 0) {
+			RTW_PRINT_SEL(m, "\n");
+			continue;
+		}
+
+		for (i = 0; i < resp_entry->content_len; i++) {
+			if (i % 32 == 0)
+				RTW_PRINT_SEL(m, "\n");
+			RTW_PRINT_SEL(m, "%02x", resp_entry->content[i]);
+		}
+		RTW_PRINT_SEL(m, "\n");
+	}
+
+exit:
+	return 0;
+}
+
+ssize_t proc_set_wow_mdns_match_criteria(struct file *file,
+					 const char __user *buffer,
+					 size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+	char tmp[13] = {0};
+	u8 resp_idx = 0;
+	u16 match_type = 0;
+	u16 name_offset = 0;
+	u8 name_len = 0;
+	int err = 0;
+	int num = 0;
+
+	if (!padapter) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (count < 1 || count > sizeof(tmp)) {
+		RTW_INFO("argument size not available\n");
+		err = -EINVAL;
+		goto exit;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count)) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	num = sscanf(tmp, "%hhu %hu %hu", &resp_idx, &match_type, &name_offset);
+	if ((num < 1) || num == 2 || (resp_idx > (MAX_MDNS_RESP_NUM - 1))) {
+		RTW_INFO("argument unavailable\n");
+		err = -EINVAL;
+	} else if (num == 1) {
+		RTW_INFO("remove mdns resp(%d) match criteria\n", resp_idx);
+		rtw_wow_del_mdns_match_crit(padapter, resp_idx);
+	} else {
+		rtw_wow_get_mdns_resp_ent(padapter, resp_idx, &resp_entry);
+		if (resp_entry->content_len == 0) {
+			RTW_INFO("mdns resp(%d) is empty\n", resp_idx);
+			err = -EFAULT;
+			goto exit;
+		}
+
+		_parse_mdns_domain_name(NULL, resp_entry->content,
+				       resp_entry->content_len,
+				       name_offset, &name_len);
+		if (name_len == 0) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		RTW_INFO("name_len=%3d\n", name_len);
+
+		rtw_wow_add_mdns_match_crit(padapter, resp_idx, match_type, name_offset, name_len);
+	}
+
+exit:
+	return err ? err : count;
+}
+
+int proc_get_wow_mdns_match_criteria(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+	u8 resp_idx = 0;
+	u8 i = 0;
+
+	if (!padapter)
+		goto exit;
+
+	for (resp_idx = 0; resp_idx < MAX_MDNS_RESP_NUM; resp_idx++) {
+		rtw_wow_get_mdns_resp_ent(padapter, resp_idx, &resp_entry);
+
+		RTW_PRINT_SEL(m, "mdns resp(%d) match criteria:\n", resp_idx);
+
+		for (i = 0; i < resp_entry->match_ct_num; i++) {
+			RTW_PRINT_SEL(m, "type=%5d, name_offset=%3d, name_len=%3d\n",
+				      resp_entry->match_ct[i].type,
+				      resp_entry->match_ct[i].name_offset,
+				      resp_entry->match_ct[i].name_len);
+			_parse_mdns_domain_name(m, resp_entry->content,
+						resp_entry->content_len,
+						resp_entry->match_ct[i].name_offset,
+						NULL);
+		}
+	}
+
+exit:
+	return 0;
+}
+
+ssize_t proc_set_wow_mdns_passthru_list(struct file *file,
+					const char __user *buffer,
+					size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[MDNS_PASSTHRU_PROC_BUF_LEN] = {0};
+	u16 name_buf_len = 0;
+	u8 name_len = 0;
+	int err = 0;
+	u16 i = 0;
+
+	if (!padapter) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (count < 1 || count > sizeof(tmp)) {
+		RTW_INFO("argument size not available\n");
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count)) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (strncmp(tmp, "clear", 5) == 0) {
+		RTW_INFO("clear passthru list\n");
+		rtw_wow_clr_mdns_passthru_name(padapter);
+	} else {
+		if ((count - 1) % 2 != 0) {
+			err = -EFAULT;
+			RTW_INFO("passthru name format is incorrect\n");
+			goto exit;
+		}
+
+		for (i = 0; i < count - 1 ; i += 2) {
+			tmp[name_buf_len] = key_2char2num(tmp[i], tmp[i+1]);
+			name_buf_len++;
+		}
+		_parse_mdns_domain_name(NULL, tmp, name_buf_len, 0, &name_len);
+
+		if (name_len == 0) {
+			err = -EFAULT;
+			goto exit;
+		}
+
+		if (_FAIL == rtw_wow_add_mdns_passthru_name(padapter, tmp, name_len)) {
+			err = -EFAULT;
+			goto exit;
+		}
+	}
+exit:
+	return err ? err : count;
+}
+
+int proc_get_wow_mdns_passthru_list(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mdns_passthru_list *passthru_list = NULL;
+	u8 i = 0;
+
+	if (!padapter)
+		goto exit;
+
+	rtw_wow_get_mdns_passthru_list(padapter, &passthru_list);
+
+	RTW_PRINT_SEL(m, "mdns passthru list:\n");
+
+	for (i = 0; i < passthru_list->passthru_name_num; i++) {
+		RTW_PRINT_SEL(m, "name_len=%3d, ", passthru_list->passthru_name[i].name_len);
+		_parse_mdns_domain_name(m, passthru_list->passthru_name[i].name,
+					passthru_list->passthru_name[i].name_len,
+					0, NULL);
+	}
+
+exit:
+	return 0;
+}
+
+ssize_t proc_set_wow_mdns_offload_state(struct file *file,
+					    const char __user *buffer,
+					    size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	//struct pwrctrl_priv *pwrpriv;//ryan
+	struct wow_priv *wowpriv;//ryan
+	//struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);//ryan
+	//struct phl_info_t *phl_info = GET_PHL_INFO(dvobj);//ryan
+	//struct phl_wow_info *wow_info = &phl_info->wow_info;//ryan
+	//struct rtw_mdns_ofld_info *ofld_info = &wow_info->mdns_ofld_info;//ryan
+	char tmp;
+	int err = 0;
+
+	if (!padapter) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	//pwrpriv = adapter_to_pwrctl(padapter);//ryan
+	wowpriv = adapter_to_wowlan(padapter);//ryan
+
+	if (count != 2) {
+		RTW_INFO("argument size not available\n");
+		err = -EINVAL;
+		goto exit;
+	}
+
+	if (!buffer || copy_from_user(&tmp, buffer, 1)) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	tmp = key_char2num(tmp);
+	if (tmp < 0 || tmp > 1)
+		RTW_INFO("Invalid mdns offload state!\n");
+	else
+		wowpriv->mdns_ofld_info.offload_state = tmp;//ryan
+		//mdns_info.offload_state = tmp;//ryan
+		//pwrpriv->mdns_ofld_info.offload_state = tmp;//ryan
+
+exit:
+	return err ? err : count;
+}
+
+int proc_get_wow_mdns_offload_state(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	//struct pwrctrl_priv *pwrpriv;//ryan
+	struct wow_priv *wowpriv;//ryan
+	//struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);//ryan
+	//struct phl_info_t *phl_info = GET_PHL_INFO(dvobj);//ryan
+	//struct phl_wow_info *wow_info = &phl_info->wow_info;//ryan
+	//struct rtw_mdns_ofld_info *ofld_info = &wow_info->mdns_ofld_info;//ryan
+
+	if (!padapter)
+		goto exit;
+
+	//pwrpriv = adapter_to_pwrctl(padapter);//ryan
+	wowpriv = adapter_to_wowlan(padapter);//ryan
+
+	//RTW_PRINT_SEL(m, "%d\n", pwrpriv->mdns_ofld_info.offload_state);//ryan
+	RTW_PRINT_SEL(m, "%d\n", wowpriv->mdns_ofld_info.offload_state);//ryan
+	//RTW_PRINT_SEL(m, "%d\n", ofld_info.offload_state);//ryan
+
+exit:
+	return 0;
+}
+
+ssize_t proc_set_wow_mdns_passthru_behavior(struct file *file,
+					    const char __user *buffer,
+					    size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	//struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);//ryan
+	//struct wow_priv *wowpriv = adapter_to_wowlan(padapter);//ryan
+	struct rtw_mdns_passthru_list *passthru_list = NULL;
+	char tmp;
+	int err = 0;
+
+	if (!padapter) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (count != 2) {
+		RTW_INFO("argument size not available\n");
+		err = -EINVAL;
+		goto exit;
+	}
+
+	if (!buffer || copy_from_user(&tmp, buffer, 1)) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	tmp = key_char2num(tmp);
+	if (tmp < 0 || tmp > 2) {
+		RTW_INFO("Invalid passthrough behavior!\n");
+	} else {
+		rtw_wow_get_mdns_passthru_list(padapter, &passthru_list);
+		passthru_list->passthru_behavior = tmp;
+	}
+
+exit:
+	return err ? err : count;
+}
+
+int proc_get_wow_mdns_passthru_behavior(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mdns_passthru_list *passthru_list;
+
+	if (!padapter)
+		goto exit;
+
+	rtw_wow_get_mdns_passthru_list(padapter, &passthru_list);
+
+	RTW_PRINT_SEL(m, "%d\n", passthru_list->passthru_behavior);
+
+exit:
+	return 0;
+}
+#endif /* CONFIG_MDNS_OFFLOAD */
+
+#ifdef CONFIG_WOW_APF
+#define APF_PROC_BUF_LEN (3 + MAX_APF_PROG_SIZE * 2) /* set + apf_prog_str_size */
+ssize_t proc_set_wow_apf(struct file *file, const char __user *buffer,
+			       size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char *tmp = NULL;
+	char cmd[8];
+	u8 *apf_prog = NULL;
+	int err = 0;
+	int num, i;
+	size_t apf_str_len = 0;
+	u16 apf_hex_len = 0;
+
+	tmp = rtw_zvmalloc(APF_PROC_BUF_LEN);
+	apf_prog = rtw_zvmalloc(MAX_APF_PROG_SIZE);
+	if (!tmp || !apf_prog) {
+		RTW_INFO("alloc buffer failed\n");
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (count < 1 || count > APF_PROC_BUF_LEN) {
+		RTW_INFO("argument size not available\n");
+		err = -EINVAL;
+		goto exit;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count)) {
+		err = -EFAULT;
+		goto exit;
+	}
+#ifdef CONFIG_WOW_APF_DBG
+	RTW_INFO("[APFDBG]%s() %s\n", __func__, tmp);
+#endif
+	if (strncmp(tmp, "help", 4) == 0) {
+		RTW_INFO("apf cmd:\n");
+		RTW_INFO("clear all apf prog		: echo \"clear\" > wow_apf\n");
+		RTW_INFO("$set apf prog				: echo set $APF_PROG > wow_apf\n");
+		RTW_INFO("$get apf prog				: cat wow_apf\n");
+	} else if (strncmp(tmp, "clear", 5) == 0) {
+		num = sscanf(tmp, "%s", cmd);
+		if (num != 1) {
+			RTW_INFO("argument unavailable\n");
+			err = -EINVAL;
+			goto exit;
+		} else {
+			RTW_INFO("clear apf program\n");
+			rtw_wow_clear_apf(padapter);
+			goto exit;
+		}
+	} else if (strncmp(tmp, "set", 3) == 0) {
+		num = sscanf(tmp, "%s %s", cmd, apf_prog);
+		if (num != 2) {
+			RTW_INFO("argument unavailable\n");
+			err = -EINVAL;
+			goto exit;
+		} else {
+			RTW_INFO("set apf program\n");
+			apf_str_len = strlen(apf_prog);
+			if (apf_str_len % 2 != 0) {
+				err = -EINVAL;
+				RTW_INFO("apf prog content format is incorrect\n");
+				goto exit;
+			}
+
+			for (i = 0; i < apf_str_len; i += 2) {
+				apf_prog[apf_hex_len] = key_2char2num(apf_prog[i], apf_prog[i + 1]);
+				apf_hex_len++;
+			}
+			if (apf_hex_len > 2048) {
+				RTW_ERR("%s() APF program size exceed %u\n", __func__, MAX_APF_PROG_SIZE);
+				goto exit;
+			}
+			rtw_wow_set_apf(padapter, apf_prog, apf_hex_len);
+			goto exit;
+
+		}
+	} else {
+		RTW_INFO("Incorrect apf cmd, use \"echo help > set_apf\" to get help\n");
+		goto exit;
+	}
+
+exit:
+	if (tmp)
+		rtw_vmfree(tmp, APF_PROC_BUF_LEN);
+	if (apf_prog)
+		rtw_vmfree(apf_prog, MAX_APF_PROG_SIZE);
+
+	return err ? err : count;
+}
+
+int proc_get_wow_apf(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_apf_info *info = &wowpriv->apf_info;
+	u8 ret = _SUCCESS, *apf_prog = NULL;
+	u16 apf_prog_len = 0, i = 0;
+	int err = 0;
+
+	apf_prog = rtw_zvmalloc(MAX_APF_PROG_SIZE);
+	if (!apf_prog) {
+		RTW_ERR("%s() alloc buffer failed\n", __func__);
+		err = -EFAULT;
+		goto exit;
+	}
+
+	RTW_PRINT_SEL(m, "[APF CMD]\n");
+	RTW_PRINT_SEL(m, "Set apf prog			: echo set [$APF_PROG] > wow_apf\n");
+	RTW_PRINT_SEL(m, "Get apf prog			: cat wow_apf\n");
+	RTW_PRINT_SEL(m, "Clear apf prog		: echo \"clear\" > wow_apf\n");
+
+	RTW_PRINT_SEL(m, "\n==== apf_prog_num(%u) total_size(%u)====\n",
+				  info->apf_prog_num, info->apf_prog_total_size);
+	
+	ret = rtw_wow_get_apf(padapter, apf_prog, &apf_prog_len);
+	if (ret != _SUCCESS)
+		return -EFAULT;
+
+	RTW_PRINT_SEL(m, "\n==== apf_prog_len(%u) ram_len(%u)====\n",
+				  apf_prog_len, MAX_APF_PROG_SIZE);
+	for (i = 0; i < MAX_APF_PROG_SIZE; i++) {
+		if (i % 16 == 0)
+			RTW_PRINT_SEL(m, "\n");
+		RTW_PRINT_SEL(m, "%02x ", apf_prog[i]);
+	}
+	RTW_PRINT_SEL(m, "\n");
+
+exit:
+	return err;
+}
+#endif /* CONFIG_WOW_APF */
+
+#ifdef CONFIG_GPIO_WAKEUP
+int proc_get_wowlan_gpio_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_wow_gpio_info *wow_gpio = &wowpriv->wow_gpio;
+	struct rtw_dev2hst_gpio_info *d2h_gpio_info = &wow_gpio->d2h_gpio_info;
+
+	RTW_PRINT_SEL(m, "wakeup_gpio_idx: %d\n", WAKEUP_GPIO_IDX);
+	RTW_PRINT_SEL(m, "high_active: %d\n", d2h_gpio_info->gpio_active);
+
+	return 0;
+}
+
+ssize_t proc_set_wowlan_gpio_info(struct file *file, const char __user *buffer,
+				  size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	void *phl = GET_PHL_INFO(dvobj);
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_wow_gpio_info *wow_gpio = &wowpriv->wow_gpio;
+	struct rtw_dev2hst_gpio_info *d2h_gpio_info = &wow_gpio->d2h_gpio_info;
+	char tmp[32] = {0};
+	int num = 0;
+	u32 is_high_active = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num = sscanf(tmp, "%u", &is_high_active);
+
+		if (num != 1) {
+			RTW_INFO("Invalid format\n");
+			return count;
+		}
+
+		wow_gpio->dev2hst_high = is_high_active == 0 ? 1 : 0;
+		d2h_gpio_info->gpio_active = is_high_active;
+
+		status = rtw_phl_cfg_wow_sw_gpio_ctrl(phl, wow_gpio);
+
+		RTW_INFO("set %s %d\n", "gpio_high_active",
+			 d2h_gpio_info->gpio_active);
+		RTW_INFO("%s: set GPIO_%d %d as default. status=%d\n",
+			 __func__, WAKEUP_GPIO_IDX, wow_gpio->dev2hst_high, status);
+	}
+
+	return count;
+}
+
+ssize_t proc_set_wow_gpio_duration(struct file *file, const char __user *buffer,
+				size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_wow_gpio_info *wow_gpio = &wowpriv->wow_gpio;
+	struct rtw_dev2hst_gpio_info *d2h_gpio_info = &wow_gpio->d2h_gpio_info;
+	u32 wow_cmn_gpio_dur;
+	u32 wow_custom_gpio_dur;
+	u8 tmp[8] = {0};
+	int ret = 0, num = 0;
+	u8 index = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count))
+		num = sscanf(tmp, "%u %u", &wow_cmn_gpio_dur, &wow_custom_gpio_dur);
+	else
+		return -EFAULT;
+
+	if (num == 2 && wow_cmn_gpio_dur <= 127 && wow_custom_gpio_dur <= 127) {
+		d2h_gpio_info->gpio_pulse_dura = wow_cmn_gpio_dur;
+		d2h_gpio_info->gpio_pulse_period = wow_cmn_gpio_dur * 2;
+		d2h_gpio_info->rsn_a_pulse_duration = wow_custom_gpio_dur;
+		d2h_gpio_info->rsn_a_pulse_period = wow_custom_gpio_dur * 2;
+
+		RTW_INFO("%s: wow_cmn_gpio_dur: %u, wow_custom_gpio_dur: %u\n",
+			 __func__, d2h_gpio_info->gpio_pulse_dura,
+			 d2h_gpio_info->rsn_a_pulse_duration);
+	} else {
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+int proc_get_wow_gpio_duration(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_wow_gpio_info *wow_gpio = &wowpriv->wow_gpio;
+	struct rtw_dev2hst_gpio_info *d2h_gpio_info = &wow_gpio->d2h_gpio_info;
+
+	RTW_PRINT_SEL(m, "wow_cmn_gpio_dur = %u, wow_custom_gpio_dur = %u\n"
+		"wow_cmn_gpio_period = %u, wow_custom_gpio_period = %u\n",
+		d2h_gpio_info->gpio_pulse_dura,
+		d2h_gpio_info->rsn_a_pulse_duration,
+		d2h_gpio_info->gpio_pulse_period,
+		d2h_gpio_info->rsn_a_pulse_period);
+	return 0;
+}
+#endif /* CONFIG_GPIO_WAKEUP */
+
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+int proc_get_wow_pw_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_periodic_wake_info *wow_periodic_wake;
+	struct wow_priv *wowpriv;
+
+	wowpriv = adapter_to_wowlan(padapter);
+	wow_periodic_wake = &wowpriv->wow_periodic_wake;
+
+	RTW_PRINT_SEL(m, "wake_peirod=%d, wake_duration=%d\n",
+		      wow_periodic_wake->wake_period,
+		      wow_periodic_wake->wake_duration);
+
+	return 0;
+}
+
+ssize_t proc_set_wow_pw_info(struct file *file, const char __user *buffer,
+			     size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_periodic_wake_info *wow_periodic_wake;
+	struct wow_priv *wowpriv;
+	u32 wake_period;
+	u32 wake_duration;
+
+	wowpriv = adapter_to_wowlan(padapter);
+	wow_periodic_wake = &wowpriv->wow_periodic_wake;
+
+	if (count < 2) {
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%u %u", &wake_period, &wake_duration);
+		if (num != 2)
+			return -EFAULT;
+
+		if (wake_duration >= wake_period) {
+			RTW_INFO("wake_duration should be less than wake_period\n");
+			return -EINVAL;
+		}
+
+		wow_periodic_wake->wake_period = wake_period;
+		wow_periodic_wake->wake_duration = wake_duration;
+
+		RTW_INFO("wake_peirod=%d, wake_duration=%d\n",
+			 wow_periodic_wake->wake_period,
+			 wow_periodic_wake->wake_duration);
+	}
+
+	return count;
+}
+#endif /* CONFIG_WOW_PERIODIC_WAKE */
+#endif /*CONFIG_WOWLAN*/
+
+#ifdef CONFIG_P2P_WOWLAN
+int proc_get_p2p_wowlan_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	struct p2p_wowlan_info	 peerinfo = pwdinfo->p2p_wow_info;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	if (_TRUE == peerinfo.is_trigger) {
+		RTW_PRINT_SEL(m, "is_trigger: TRUE\n");
+		switch (peerinfo.wowlan_recv_frame_type) {
+		case P2P_WOWLAN_RECV_NEGO_REQ:
+			RTW_PRINT_SEL(m, "Frame Type: Nego Request\n");
+			break;
+		case P2P_WOWLAN_RECV_INVITE_REQ:
+			RTW_PRINT_SEL(m, "Frame Type: Invitation Request\n");
+			break;
+		case P2P_WOWLAN_RECV_PROVISION_REQ:
+			RTW_PRINT_SEL(m, "Frame Type: Provision Request\n");
+			break;
+		default:
+			break;
+		}
+
+		RTW_PRINT_SEL(m, "Peer Addr: %s\n", get_macaddr_str(mac_addr_str, m, peerinfo.wowlan_peer_addr));
+		RTW_PRINT_SEL(m, "Peer WPS Config: %x\n", peerinfo.wowlan_peer_wpsconfig);
+		RTW_PRINT_SEL(m, "Persistent Group: %d\n", peerinfo.wowlan_peer_is_persistent);
+		RTW_PRINT_SEL(m, "Intivation Type: %d\n", peerinfo.wowlan_peer_invitation_type);
+	} else
+		RTW_PRINT_SEL(m, "is_trigger: False\n");
+	return 0;
+}
+#endif /* CONFIG_P2P_WOWLAN */
+
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+int proc_get_new_bcn_max(struct seq_file *m, void *v)
+{
+	extern int new_bcn_max;
+
+	RTW_PRINT_SEL(m, "%d\n", new_bcn_max);
+	return 0;
+}
+
+ssize_t proc_set_new_bcn_max(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	extern int new_bcn_max;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count))
+		sscanf(tmp, "%d ", &new_bcn_max);
+
+	return count;
+}
+#endif
+
+#ifdef CONFIG_POWER_SAVE
+int proc_get_ps_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rtw_phl_com_t *phl_com = dvobj->phl_com;
+	struct rtw_ps_cap_t *ps_cap = &phl_com->dev_cap.ps_cap;
+	u8 ips_mode = ps_cap->ips_en;
+	u8 lps_mode = ps_cap->lps_en;
+	u8 lps_cap = ps_cap->lps_cap;
+#ifdef CONFIG_RTW_LPS
+	u8 lps_listen_bcn_mode = ps_cap->lps_listen_bcn_mode;
+#endif /* CONFIG_RTW_LPS */
+#ifdef CONFIG_RTW_LPS_WOW
+	u8 lps_wow_listen_bcn_mode = ps_cap->lps_wow_listen_bcn_mode;
+#endif /* CONFIG_RTW_LPS_WOW */
+	char str[80] = {0};
+
+	RTW_PRINT_SEL(m, "======Power Saving Info:======\n");
+
+	RTW_PRINT_SEL(m, "*IPS:\n");
+
+	if (ips_mode == PS_OP_MODE_DISABLED)
+		strcpy(str, "IPS_NONE");
+	else if (ips_mode == PS_OP_MODE_FORCE_ENABLED)
+		strcpy(str, "IPS_FORCE_ENABLED");
+	else if (ips_mode == PS_OP_MODE_AUTO)
+		strcpy(str, "IPS_NORMAL");
+	else
+		strcpy(str, "Invalid");
+
+	RTW_PRINT_SEL(m, " IPS mode: %s\n", str);
+
+	RTW_PRINT_SEL(m, "*LPS:\n");
+
+	if (lps_mode == PS_OP_MODE_DISABLED)
+		strcpy(str, "LPS_NONE");
+	else if (ips_mode == PS_OP_MODE_FORCE_ENABLED)
+		strcpy(str, "LPS_FORCE_ENABLED");
+	else if (ips_mode == PS_OP_MODE_AUTO)
+		strcpy(str, "LPS_NORMAL");
+	else
+		strcpy(str, "Invalid");
+
+	RTW_PRINT_SEL(m, " LPS mode: %s\n", str);
+
+	if (lps_cap & PS_CAP_PWR_GATED)
+		strcpy(str, "LPS_PG");
+	else if (lps_cap & PS_CAP_CLK_GATED)
+		strcpy(str, "LPS_LCLK");
+	else if (lps_cap & PS_CAP_RF_OFF)
+		strcpy(str, "LPS_NORMAL");
+	else
+		strcpy(str, "Invalid");
+
+	RTW_PRINT_SEL(m, " LPS level: %s\n", str);
+
+#ifdef CONFIG_RTW_LPS
+	if (lps_listen_bcn_mode == RTW_LPS_RLBM_MIN)
+		strcpy(str, "RTW_LPS_RLBM_MIN");
+	else if (lps_listen_bcn_mode == RTW_LPS_RLBM_MAX)
+		strcpy(str, "RTW_LPS_RLBM_MAX");
+	else if (lps_listen_bcn_mode == RTW_LPS_RLBM_USERDEFINE)
+		strcpy(str, "RTW_LPS_RLBM_USERDEFINE");
+	else
+		strcpy(str, "Invalid");
+
+	RTW_PRINT_SEL(m, " LPS listen beacon mode: %s\n", str);
+
+	if (lps_listen_bcn_mode == RTW_LPS_RLBM_USERDEFINE)
+		RTW_PRINT_SEL(m, " LPS awake interval: %d\n",
+			      ps_cap->lps_awake_interval);
+#endif /* CONFIG_RTW_LPS */
+#ifdef CONFIG_RTW_LPS_WOW
+	if (lps_wow_listen_bcn_mode == RTW_LPS_RLBM_MIN)
+		strcpy(str, "RTW_LPS_RLBM_MIN");
+	else if (lps_wow_listen_bcn_mode == RTW_LPS_RLBM_MAX)
+		strcpy(str, "RTW_LPS_RLBM_MAX");
+	else if (lps_wow_listen_bcn_mode == RTW_LPS_RLBM_USERDEFINE)
+		strcpy(str, "RTW_LPS_RLBM_USERDEFINE");
+	else
+		strcpy(str, "Invalid");
+
+	RTW_PRINT_SEL(m, " LPS wowlan listen beacon mode: %s\n", str);
+
+	if (lps_wow_listen_bcn_mode == RTW_LPS_RLBM_USERDEFINE)
+		RTW_PRINT_SEL(m, " LPS wowlan awake interval: %d\n",
+			      ps_cap->lps_wow_awake_interval);
+#endif /* CONFIG_RTW_LPS_WOW */
+
+	RTW_PRINT_SEL(m, "=============================\n");
+	return 0;
+}
+
+ssize_t proc_set_ps_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	struct _ADAPTER *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rtw_phl_com_t *phl_com = dvobj->phl_com;
+	struct rtw_ps_cap_t *ps_cap_p = &phl_com->dev_cap.ps_cap;
+	struct registry_priv  *registry_par = &adapter->registrypriv;
+	char tmp[32];
+	int num = 0, ps_mode = 0, ps_cap = 0;
+	u8 lps_cap = 0;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%d %d", &ps_mode, &ps_cap);
+
+	if (num > 2) {
+		RTW_ERR("%s: invalid parameter!\n", __FUNCTION__);
+		goto exit;
+	}
+
+	if (num == 1 && ps_mode == 0) {
+		/* back to original LPS/IPS Mode */
+		RTW_INFO("%s: back to original LPS/IPS Mode\n", __FUNCTION__);
+#ifdef CONFIG_RTW_IPS
+		if (ps_cap_p->ips_en != registry_par->ips_mode)
+			rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_IPS, registry_par->ips_mode);
+#endif /* CONFIG_RTW_IPS */
+#ifdef CONFIG_RTW_LPS
+		if (ps_cap_p->lps_en != registry_par->lps_mode)
+			rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, registry_par->lps_mode);
+
+		if (ps_cap_p->lps_cap != registry_par->lps_cap)
+			rtw_phl_dbg_ps_cap(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, registry_par->lps_cap);
+#endif /* CONFIG_RTW_LPS */
+		goto exit;
+	}
+
+	if (ps_mode == 1) {
+#ifdef CONFIG_RTW_LPS
+		switch (ps_cap) {
+			case 0:
+				if (ps_cap_p->lps_en == PS_OP_MODE_DISABLED)
+					goto exit;
+				break;
+			case 1:
+				lps_cap = PS_CAP_PWRON | PS_CAP_RF_OFF;
+				break;
+			case 2:
+				lps_cap = PS_CAP_PWRON | PS_CAP_RF_OFF | PS_CAP_CLK_GATED;
+				break;
+			case 3:
+				lps_cap = PS_CAP_PWRON | PS_CAP_RF_OFF | PS_CAP_CLK_GATED | PS_CAP_PWR_GATED;
+				break;
+			default:
+				goto exit;
+		}
+
+		if (ps_cap == 0) {
+			rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, PS_OP_MODE_DISABLED);
+		} else {
+			if (ps_cap_p->lps_en == PS_OP_MODE_DISABLED)
+				rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, PS_OP_MODE_AUTO);
+			if (ps_cap_p->lps_cap != lps_cap)
+				rtw_phl_dbg_ps_cap(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, lps_cap);
+		}
+#endif /* CONFIG_RTW_LPS */
+	} else if (ps_mode == 2) {
+#ifdef CONFIG_RTW_IPS
+		if (ps_cap_p->ips_en != ps_cap) {
+			if (ps_cap != 0)
+				ps_cap = registry_par->ips_mode;
+
+			rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_IPS, ps_cap);
+		}
+#endif /* #ifdef CONFIG_RTW_IPS */
+	}
+	else {
+		RTW_ERR("%s: invalid parameter, mode = %d!\n", __FUNCTION__, ps_mode);
+	}
+exit:
+	return count;
+}
+
+#if defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW)
+ssize_t proc_set_ps_dtim(struct file *file, const char __user *buffer,
+			 size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	struct _ADAPTER *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rtw_phl_com_t *phl_com = dvobj->phl_com;
+	struct rtw_ps_cap_t *ps_cap_p = &phl_com->dev_cap.ps_cap;
+	struct rtw_ps_cap_t *ps_sw_cap_p = &phl_com->dev_sw_cap.ps_cap;
+	struct registry_priv *registry_par = &adapter->registrypriv;
+	char tmp[32];
+	u8 num = 0, set = 0, bcn_mode = 0, awake_interval = 0;
+	u8 final_bcn_mode = 0, final_awake_interval = 0;
+	bool is_wowlan = _FALSE, leave_lps = _FALSE;
+	u8 ori_lps_en = ps_cap_p->lps_en;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%hhu %hhu %hhu", &set, &bcn_mode, &awake_interval);
+
+	if (num > 3) {
+		RTW_ERR("%s: invalid parameter!\n", __func__);
+		goto exit;
+	}
+
+	/* back to original ps dtim */
+	if (set == 0) {
+#ifdef CONFIG_RTW_LPS
+		if (ps_cap_p->lps_listen_bcn_mode !=
+		    registry_par->lps_listen_bcn_mode) {
+			RTW_INFO("%s: back to original LPS DTIM\n", __func__);
+			rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0,
+					       PS_MODE_LPS,
+					       PS_OP_MODE_DISABLED);
+			rtw_update_lps_proto_cap_setting(
+				RTW_PS_PROTO_LISTEN_BEACON_MODE,
+				registry_par->lps_listen_bcn_mode,
+				registry_par->lps_awake_interval, ps_cap_p,
+				_FALSE);
+			rtw_update_lps_proto_cap_setting(
+				RTW_PS_PROTO_LISTEN_BEACON_MODE,
+				registry_par->lps_listen_bcn_mode,
+				registry_par->lps_awake_interval, ps_sw_cap_p,
+				_FALSE);
+			rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0,
+					       PS_MODE_LPS, PS_OP_MODE_AUTO);
+		}
+#else
+		RTW_INFO("%s: LPS is not enabled\n", __func__);
+#endif /* COFNIG_RTW_LPS */
+#ifdef CONFIG_RTW_LPS_WOW
+		if (ps_cap_p->lps_wow_listen_bcn_mode !=
+		    registry_par->lps_wow_listen_bcn_mode) {
+			RTW_INFO("%s: back to original LPS WOWLAN DTIM\n",
+				 __func__);
+			rtw_update_lps_proto_cap_setting(
+				RTW_PS_PROTO_LISTEN_BEACON_MODE,
+				registry_par->lps_wow_listen_bcn_mode,
+				registry_par->lps_wow_awake_interval, ps_cap_p,
+				_TRUE);
+			rtw_update_lps_proto_cap_setting(
+				RTW_PS_PROTO_LISTEN_BEACON_MODE,
+				registry_par->lps_wow_listen_bcn_mode,
+				registry_par->lps_wow_awake_interval,
+				ps_sw_cap_p, _TRUE);
+		}
+		goto exit;
+#else
+		RTW_INFO("%s: WOWLAN is not enabled\n", __func__);
+		goto exit;
+#endif /* CONFIG_RTW_LPS_WOW */
+	}
+	/* set lps dtim */
+	else if (set == 1) {
+#ifdef CONFIG_RTW_LPS
+		RTW_INFO("%s: Config LPS DTIM\n", __func__);
+		if (num == 1) {
+			RTW_ERR("%s: Please enter beacon mode or awake interval(optional) for setting dtim\n",
+				__func__);
+			goto exit;
+		} else if (num == 2) {
+			if (bcn_mode == RTW_LPS_RLBM_MIN ||
+			    bcn_mode == RTW_LPS_RLBM_MAX) {
+				final_bcn_mode = bcn_mode;
+				final_awake_interval = awake_interval;
+				leave_lps = _TRUE;
+			} else if (bcn_mode == RTW_LPS_RLBM_USERDEFINE) {
+				RTW_ERR("%s: Please enter awake interval \n",
+					__func__);
+				goto exit;
+			} else {
+				RTW_ERR("%s: Invalid beacon mode\n", __func__);
+				goto exit;
+			}
+		} else if (num == 3) {
+			if (bcn_mode == RTW_LPS_RLBM_MIN ||
+			    bcn_mode == RTW_LPS_RLBM_MAX ||
+			    bcn_mode == RTW_LPS_RLBM_USERDEFINE) {
+				final_bcn_mode = bcn_mode;
+				final_awake_interval = awake_interval;
+				leave_lps = _TRUE;
+			} else {
+				RTW_ERR("%s: Invalid beacon mode\n", __func__);
+				goto exit;
+			}
+		}
+#else
+		RTW_INFO("%s: LPS is not enabled\n", __func__);
+		goto exit;
+#endif /* CONFIG_RTW_LPS */
+	} else if (set == 2) {
+#ifdef CONFIG_RTW_LPS_WOW
+		RTW_INFO("%s: Config LPS WOWLAN DTIM\n", __func__);
+		is_wowlan = _TRUE;
+		if (num == 1) {
+			RTW_ERR("%s: Please enter beacon mode or awake interval(optional) for setting dtim\n",
+				__func__);
+			goto exit;
+		} else if (num == 2) {
+			if (bcn_mode == RTW_LPS_RLBM_MIN ||
+			    bcn_mode == RTW_LPS_RLBM_MAX) {
+				final_bcn_mode = bcn_mode;
+				final_awake_interval = awake_interval;
+			} else if (bcn_mode == RTW_LPS_RLBM_USERDEFINE) {
+				RTW_ERR("%s: Please enter awake interval \n",
+					__func__);
+				goto exit;
+			} else {
+				RTW_ERR("%s: Invalid beacon mode\n", __func__);
+				goto exit;
+			}
+		} else if (num == 3) {
+			if (bcn_mode == RTW_LPS_RLBM_MIN ||
+			    bcn_mode == RTW_LPS_RLBM_MAX ||
+			    bcn_mode == RTW_LPS_RLBM_USERDEFINE) {
+				final_bcn_mode = bcn_mode;
+				final_awake_interval = awake_interval;
+			} else {
+				RTW_ERR("%s: Invalid beacon mode\n", __func__);
+				goto exit;
+			}
+		}
+#else
+		RTW_INFO("%s: WOWLAN is not enabled\n", __func__);
+		goto exit;
+#endif /* CONFIG_RTW_LPS_WOW */
+	}
+
+	if (leave_lps && ori_lps_en)
+		rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0,
+				       PS_MODE_LPS, PS_OP_MODE_DISABLED);
+
+	rtw_update_lps_proto_cap_setting(RTW_PS_PROTO_LISTEN_BEACON_MODE,
+					 final_bcn_mode, final_awake_interval,
+					 ps_cap_p, is_wowlan);
+	rtw_update_lps_proto_cap_setting(RTW_PS_PROTO_LISTEN_BEACON_MODE,
+					 final_bcn_mode, final_awake_interval,
+					 ps_sw_cap_p, is_wowlan);
+
+	if (leave_lps && ori_lps_en)
+		rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0,
+				       PS_MODE_LPS, PS_OP_MODE_AUTO);
+
+	if (is_wowlan)
+		RTW_INFO(
+			"%s: set wow listen bcn mode = %d and awake interval = %hhu successfully.",
+			__func__, ps_cap_p->lps_wow_listen_bcn_mode,
+			ps_cap_p->lps_wow_awake_interval);
+	else
+		RTW_INFO(
+			"%s: set listen bcn mode = %d and awake interval = %hhu successfully.",
+			__func__, ps_cap_p->lps_listen_bcn_mode,
+			ps_cap_p->lps_awake_interval);
+
+exit:
+	return count;
+}
+#endif /* defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW) */
+#endif /* CONFIG_POWER_SAVE */
+
+#ifdef CONFIG_TDLS
+int proc_get_tdls_enable(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "TDLS is %s !\n", (rtw_is_tdls_enabled(padapter) == _TRUE) ? "enabled" : "disabled");
+
+	return 0;
+}
+
+ssize_t proc_set_tdls_enable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 en_tdls = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &en_tdls);
+
+		if (num == 1 && pregpriv) {
+			if (en_tdls > 0)
+				rtw_enable_tdls_func(padapter);
+			else
+				rtw_disable_tdls_func(padapter, _TRUE);
+		}
+	}
+
+	return count;
+}
+
+static int proc_tdls_display_tdls_function_info(struct seq_file *m)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	u8 SpaceBtwnItemAndValue = TDLS_DBG_INFO_SPACE_BTWN_ITEM_AND_VALUE;
+	u8 SpaceBtwnItemAndValueTmp = 0;
+	BOOLEAN FirstMatchFound = _FALSE;
+	int j = 0;
+
+	RTW_PRINT_SEL(m, "============[TDLS Function Info]============\n");
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS Enable", (rtw_is_tdls_enabled(padapter) == _TRUE) ? "_TRUE" : "_FALSE");
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS Driver Setup", (ptdlsinfo->driver_setup == _TRUE) ? "_TRUE" : "_FALSE");
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS Prohibited", (ptdlsinfo->ap_prohibited == _TRUE) ? "_TRUE" : "_FALSE");
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS Channel Switch Prohibited", (ptdlsinfo->ch_switch_prohibited == _TRUE) ? "_TRUE" : "_FALSE");
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS Link Established", (ptdlsinfo->link_established == _TRUE) ? "_TRUE" : "_FALSE");
+	RTW_PRINT_SEL(m, "%-*s = %d/%d\n", SpaceBtwnItemAndValue, "TDLS STA Num (Linked/Allowed)", ptdlsinfo->sta_cnt, MAX_ALLOWED_TDLS_STA_NUM);
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS Allowed STA Num Reached", (ptdlsinfo->sta_maximum == _TRUE) ? "_TRUE" : "_FALSE");
+
+#ifdef CONFIG_TDLS_CH_SW
+	RTW_PRINT_SEL(m, "%-*s =", SpaceBtwnItemAndValue, "TDLS CH SW State");
+	if (ptdlsinfo->chsw_info.ch_sw_state == TDLS_STATE_NONE)
+		RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_STATE_NONE");
+	else {
+		for (j = 0; j < 32; j++) {
+			if (ptdlsinfo->chsw_info.ch_sw_state & BIT(j)) {
+				if (FirstMatchFound ==  _FALSE) {
+					SpaceBtwnItemAndValueTmp = 1;
+					FirstMatchFound = _TRUE;
+				} else
+					SpaceBtwnItemAndValueTmp = SpaceBtwnItemAndValue + 3;
+				switch (BIT(j)) {
+				case TDLS_INITIATOR_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_INITIATOR_STATE");
+					break;
+				case TDLS_RESPONDER_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_RESPONDER_STATE");
+					break;
+				case TDLS_LINKED_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_LINKED_STATE");
+					break;
+				case TDLS_WAIT_PTR_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_WAIT_PTR_STATE");
+					break;
+				case TDLS_ALIVE_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_ALIVE_STATE");
+					break;
+				case TDLS_CH_SWITCH_ON_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_CH_SWITCH_ON_STATE");
+					break;
+				case TDLS_PEER_AT_OFF_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_PEER_AT_OFF_STATE");
+					break;
+				case TDLS_CH_SW_INITIATOR_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_CH_SW_INITIATOR_STATE");
+					break;
+				case TDLS_WAIT_CH_RSP_STATE:
+					RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValue, " ", "TDLS_WAIT_CH_RSP_STATE");
+					break;
+				default:
+					RTW_PRINT_SEL(m, "%-*sBIT(%d)\n", SpaceBtwnItemAndValueTmp, " ", j);
+					break;
+				}
+			}
+		}
+	}
+
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS CH SW On", (ATOMIC_READ(&ptdlsinfo->chsw_info.chsw_on) == _TRUE) ? "_TRUE" : "_FALSE");
+	RTW_PRINT_SEL(m, "%-*s = %d\n", SpaceBtwnItemAndValue, "TDLS CH SW Off-Channel Num", ptdlsinfo->chsw_info.off_ch_num);
+	RTW_PRINT_SEL(m, "%-*s = %d\n", SpaceBtwnItemAndValue, "TDLS CH SW Channel Offset", ptdlsinfo->chsw_info.ch_offset);
+	RTW_PRINT_SEL(m, "%-*s = %d\n", SpaceBtwnItemAndValue, "TDLS CH SW Current Time", ptdlsinfo->chsw_info.cur_time);
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS CH SW Delay Switch Back", (ptdlsinfo->chsw_info.delay_switch_back == _TRUE) ? "_TRUE" : "_FALSE");
+	RTW_PRINT_SEL(m, "%-*s = %d\n", SpaceBtwnItemAndValue, "TDLS CH SW Dump Back", ptdlsinfo->chsw_info.dump_stack);
+#endif
+
+	RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "TDLS Device Discovered", (ptdlsinfo->dev_discovered == _TRUE) ? "_TRUE" : "_FALSE");
+
+	return 0;
+}
+
+static int proc_tdls_display_network_info(struct seq_file *m)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct wlan_network *cur_network = &(pmlmepriv->cur_network);
+	int i = 0;
+	u8 SpaceBtwnItemAndValue = TDLS_DBG_INFO_SPACE_BTWN_ITEM_AND_VALUE;
+	char wl_mode[16];
+	char mac_addr_str[MAC_FMT_LEN];
+
+	/* Display the linked AP/GO info */
+	RTW_PRINT_SEL(m, "============[Associated AP/GO Info]============\n");
+
+	if ((padapter->mlmepriv.fw_state & WIFI_STATION_STATE) && (padapter->mlmepriv.fw_state & WIFI_ASOC_STATE)) {
+		wireless_mode_to_str(pmlmeext->cur_wireless_mode, wl_mode);
+		RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "BSSID", cur_network->network.Ssid.Ssid);
+		RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "Mac Address",
+			get_macaddr_str(mac_addr_str, m, cur_network->network.MacAddress));
+		RTW_PRINT_SEL(m, "%-*s = ", SpaceBtwnItemAndValue, "Wireless Mode");
+		RTW_PRINT_SEL(m, "0x%x(%s)\n", pmlmeext->cur_wireless_mode, wl_mode);
+
+		RTW_PRINT_SEL(m, "%-*s = ", SpaceBtwnItemAndValue, "Privacy");
+		switch (padapter->securitypriv.dot11PrivacyAlgrthm) {
+		case _NO_PRIVACY_:
+			RTW_PRINT_SEL(m, "%s\n", "NO PRIVACY");
+			break;
+		case _WEP40_:
+			RTW_PRINT_SEL(m, "%s\n", "WEP 40");
+			break;
+		case _TKIP_:
+			RTW_PRINT_SEL(m, "%s\n", "TKIP");
+			break;
+		case _TKIP_WTMIC_:
+			RTW_PRINT_SEL(m, "%s\n", "TKIP WTMIC");
+			break;
+		case _AES_:
+			RTW_PRINT_SEL(m, "%s\n", "AES");
+			break;
+		case _WEP104_:
+			RTW_PRINT_SEL(m, "%s\n", "WEP 104");
+			break;
+#if 0 /* no this setting */
+		case _WEP_WPA_MIXED_:
+			RTW_PRINT_SEL(m, "%s\n", "WEP/WPA Mixed");
+			break;
+#endif
+		case _SMS4_:
+			RTW_PRINT_SEL(m, "%s\n", "SMS4");
+			break;
+#ifdef CONFIG_IEEE80211W
+		case _BIP_CMAC_128_:
+			RTW_PRINT_SEL(m, "%s\n", "BIP");
+			break;
+#endif /* CONFIG_IEEE80211W */
+		}
+
+		RTW_PRINT_SEL(m, "%-*s = %d\n", SpaceBtwnItemAndValue, "Channel", pmlmeext->chandef.chan);
+		RTW_PRINT_SEL(m, "%-*s = %u\n", SpaceBtwnItemAndValue, "Channel Offset", pmlmeext->chandef.offset);
+		RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "Bandwidth Mode", ch_width_str(pmlmeext->chandef.bw));
+
+	} else
+		RTW_PRINT_SEL(m, "No association with AP/GO exists!\n");
+
+	return 0;
+}
+
+static int proc_tdls_display_tdls_sta_info(struct seq_file *m)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct sta_info *psta;
+	int i = 0, j = 0;
+	_list	*plist, *phead;
+	u8 SpaceBtwnItemAndValue = TDLS_DBG_INFO_SPACE_BTWN_ITEM_AND_VALUE;
+	u8 SpaceBtwnItemAndValueTmp = 0;
+	u8 NumOfTdlsStaToShow = 0;
+	BOOLEAN FirstMatchFound = _FALSE;
+	char wl_mode[16];
+	char mac_addr_str[MAC_FMT_LEN];
+
+	/* Search for TDLS sta info to display */
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+			if (psta->tdls_sta_state != TDLS_STATE_NONE) {
+				/* We got one TDLS sta info to show */
+				RTW_PRINT_SEL(m, "============[TDLS Peer STA Info: STA %d]============\n", ++NumOfTdlsStaToShow);
+				RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "Mac Address",
+					get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr));
+				RTW_PRINT_SEL(m, "%-*s =", SpaceBtwnItemAndValue, "TDLS STA State");
+				SpaceBtwnItemAndValueTmp = 0;
+				FirstMatchFound = _FALSE;
+				for (j = 0; j < 32; j++) {
+					if (psta->tdls_sta_state & BIT(j)) {
+						if (FirstMatchFound ==  _FALSE) {
+							SpaceBtwnItemAndValueTmp = 1;
+							FirstMatchFound = _TRUE;
+						} else
+							SpaceBtwnItemAndValueTmp = SpaceBtwnItemAndValue + 3;
+						switch (BIT(j)) {
+						case TDLS_INITIATOR_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_INITIATOR_STATE");
+							break;
+						case TDLS_RESPONDER_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_RESPONDER_STATE");
+							break;
+						case TDLS_LINKED_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_LINKED_STATE");
+							break;
+						case TDLS_WAIT_PTR_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_WAIT_PTR_STATE");
+							break;
+						case TDLS_ALIVE_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_ALIVE_STATE");
+							break;
+						case TDLS_CH_SWITCH_ON_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_CH_SWITCH_ON_STATE");
+							break;
+						case TDLS_PEER_AT_OFF_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_PEER_AT_OFF_STATE");
+							break;
+						case TDLS_CH_SW_INITIATOR_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValueTmp, " ", "TDLS_CH_SW_INITIATOR_STATE");
+							break;
+						case TDLS_WAIT_CH_RSP_STATE:
+							RTW_PRINT_SEL(m, "%-*s%s\n", SpaceBtwnItemAndValue, " ", "TDLS_WAIT_CH_RSP_STATE");
+							break;
+						default:
+							RTW_PRINT_SEL(m, "%-*sBIT(%d)\n", SpaceBtwnItemAndValueTmp, " ", j);
+							break;
+						}
+					}
+				}
+
+				wireless_mode_to_str(psta->phl_sta->wmode, wl_mode);
+				RTW_PRINT_SEL(m, "%-*s = ", SpaceBtwnItemAndValue, "Wireless Mode");
+				RTW_PRINT_SEL(m, "0x%x(%s)\n", psta->phl_sta->wmode, wl_mode);
+
+				RTW_PRINT_SEL(m, "%-*s = %s\n", SpaceBtwnItemAndValue, "Bandwidth Mode", ch_width_str(psta->phl_sta->chandef.bw));
+
+				RTW_PRINT_SEL(m, "%-*s = ", SpaceBtwnItemAndValue, "Privacy");
+				switch (psta->dot118021XPrivacy) {
+				case _NO_PRIVACY_:
+					RTW_PRINT_SEL(m, "%s\n", "NO PRIVACY");
+					break;
+				case _WEP40_:
+					RTW_PRINT_SEL(m, "%s\n", "WEP 40");
+					break;
+				case _TKIP_:
+					RTW_PRINT_SEL(m, "%s\n", "TKIP");
+					break;
+				case _TKIP_WTMIC_:
+					RTW_PRINT_SEL(m, "%s\n", "TKIP WTMIC");
+					break;
+				case _AES_:
+					RTW_PRINT_SEL(m, "%s\n", "AES");
+					break;
+				case _WEP104_:
+					RTW_PRINT_SEL(m, "%s\n", "WEP 104");
+					break;
+#if 0 /* no this setting */
+				case _WEP_WPA_MIXED_:
+					RTW_PRINT_SEL(m, "%s\n", "WEP/WPA Mixed");
+					break;
+#endif
+				case _SMS4_:
+					RTW_PRINT_SEL(m, "%s\n", "SMS4");
+					break;
+#ifdef CONFIG_IEEE80211W
+				case _BIP_CMAC_128_:
+					RTW_PRINT_SEL(m, "%s\n", "BIP");
+					break;
+#endif /* CONFIG_IEEE80211W */
+				}
+
+				RTW_PRINT_SEL(m, "%-*s = %d sec/%d sec\n", SpaceBtwnItemAndValue, "TPK Lifetime (Current/Expire)", psta->TPK_count, psta->TDLS_PeerKey_Lifetime);
+				RTW_PRINT_SEL(m, "%-*s = %llu\n", SpaceBtwnItemAndValue, "Tx Packets Over Direct Link", psta->sta_stats.tx_pkts);
+				RTW_PRINT_SEL(m, "%-*s = %llu\n", SpaceBtwnItemAndValue, "Rx Packets Over Direct Link", psta->sta_stats.rx_data_pkts);
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	if (NumOfTdlsStaToShow == 0) {
+		RTW_PRINT_SEL(m, "============[TDLS Peer STA Info]============\n");
+		RTW_PRINT_SEL(m, "No TDLS direct link exists!\n");
+	}
+
+	return 0;
+}
+
+int proc_get_tdls_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct wlan_network *cur_network = &(pmlmepriv->dev_cur_network);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct sta_info *psta;
+	int i = 0, j = 0;
+	_list	*plist, *phead;
+	u8 SpaceBtwnItemAndValue = 41;
+	u8 SpaceBtwnItemAndValueTmp = 0;
+	u8 NumOfTdlsStaToShow = 0;
+	BOOLEAN FirstMatchFound = _FALSE;
+
+	if (rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_TDLS) == _FALSE) {
+		RTW_PRINT_SEL(m, "No tdls info can be shown since hal doesn't support tdls\n");
+		return 0;
+	}
+
+	proc_tdls_display_tdls_function_info(m);
+	proc_tdls_display_network_info(m);
+	proc_tdls_display_tdls_sta_info(m);
+
+	return 0;
+}
+#endif
+
+int proc_get_monitor(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLDL: [currentlty primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (MLME_IS_MONITOR(padapter)) {
+		RTW_PRINT_SEL(m, "Monitor mode : Enable\n");
+		RTW_PRINT_SEL(m, "Device type  : %u\n", dev->type);
+
+		RTW_PRINT_SEL(m, "ch=%d, ch_offset=%d, bw=%d\n",
+			rtw_get_oper_ch(padapter, padapter_link),
+			rtw_get_oper_choffset(padapter, padapter_link),
+			rtw_get_oper_bw(padapter, padapter_link));
+	} else
+		RTW_PRINT_SEL(m, "Monitor mode : Disable\n");
+
+	return 0;
+}
+
+ssize_t proc_set_monitor(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u16 target_type;
+	u8 target_band, target_ch, target_offset, target_bw;
+	/* ToDo CONFIG_RTW_MLDL: [currentlty primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (count < 3) {
+		RTW_INFO("argument size is less than 3\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = 0;
+
+		num = sscanf(tmp, "type %hu", &target_type);
+		if ((num == 1) &&
+			((target_type != ARPHRD_IEEE80211) &&
+			(target_type != ARPHRD_IEEE80211_RADIOTAP))) {
+			dev->type = ARPHRD_IEEE80211_RADIOTAP;
+			return count;
+		}
+
+		num = sscanf(tmp, "%hhu %hhu %hhu %hhu", &target_band, &target_ch, &target_offset, &target_bw);
+		if (num != 4) {
+			RTW_INFO("invalid write_reg parameter!\n");
+			return count;
+		}
+
+		padapter_link->mlmeextpriv.chandef.chan = target_ch;
+		set_bch_bwmode(padapter, padapter_link, target_band, target_ch,
+			       target_offset, target_bw, RFK_TYPE_FORCE_DO);
+	}
+
+	return count;
+}
+#ifdef DBG_XMIT_BLOCK
+int proc_get_xmit_block(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_xmit_block(m, padapter);
+
+	return 0;
+}
+
+ssize_t proc_set_xmit_block(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 xb_mode, xb_reason;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx %hhx", &xb_mode, &xb_reason);
+
+		if (num != 2) {
+			RTW_INFO("invalid parameter!\n");
+			return count;
+		}
+
+		if (xb_mode == 0)/*set*/
+			rtw_set_xmit_block(padapter, xb_reason);
+		else if (xb_mode == 1)/*clear*/
+			rtw_clr_xmit_block(padapter, xb_reason);
+		else
+			RTW_INFO("invalid parameter!\n");
+	}
+
+	return count;
+}
+#endif
+
+int proc_get_efuse_map(struct seq_file *m, void *v)
+{
+#if 0
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	HAL_DATA_TYPE *pHalData = GET_PHL_COM(adapter_to_dvobj(padapter));
+	struct pwrctrl_priv *pwrctrlpriv  = adapter_to_pwrctl(padapter);
+	PEFUSE_HAL pEfuseHal = &pHalData->EfuseHal;
+	struct efuse_info *efuse = adapter_to_efuse(padapter);
+	int i, j;
+	u16 mapLen;
+
+	EFUSE_GetEfuseDefinition(padapter, EFUSE_WIFI, TYPE_EFUSE_MAP_LEN, (void *)&mapLen, _FALSE);
+	if (mapLen > EFUSE_MAX_MAP_LEN)
+		mapLen = EFUSE_MAX_MAP_LEN;
+
+	if (efuse->file_status == EFUSE_FILE_LOADED) {
+		RTW_PRINT_SEL(m, "File eFuse Map loaded! file path:%s\nDriver eFuse Map From File\n", EFUSE_MAP_PATH);
+		if (efuse->is_autoload_fail)
+			RTW_PRINT_SEL(m, "File Autoload fail!!!\n");
+	} else if (efuse->file_status ==  EFUSE_FILE_FAILED) {
+		RTW_PRINT_SEL(m, "Open File eFuse Map Fail ! file path:%s\nDriver eFuse Map From Default\n", EFUSE_MAP_PATH);
+		if (efuse->is_autoload_fail)
+			RTW_PRINT_SEL(m, "HW Autoload fail!!!\n");
+	} else {
+		RTW_PRINT_SEL(m, "Driver eFuse Map From HW\n");
+		if (efuse->is_autoload_fail)
+			RTW_PRINT_SEL(m, "HW Autoload fail!!!\n");
+	}
+	for (i = 0; i < mapLen; i += 16) {
+		RTW_PRINT_SEL(m, "0x%02x\t", i);
+		for (j = 0; j < 8; j++)
+			RTW_PRINT_SEL(m, "%02X ", efuse->data[i + j]);
+		RTW_PRINT_SEL(m, "\t");
+		for (; j < 16; j++)
+			RTW_PRINT_SEL(m, "%02X ", efuse->data[i + j]);
+		RTW_PRINT_SEL(m, "\n");
+	}
+
+	if (rtw_efuse_map_read(padapter, 0, mapLen, pEfuseHal->fakeEfuseInitMap) == _FAIL) {
+		RTW_PRINT_SEL(m, "WARN - Read Realmap Failed\n");
+		return 0;
+	}
+
+	RTW_PRINT_SEL(m, "\n");
+	RTW_PRINT_SEL(m, "HW eFuse Map\n");
+	for (i = 0; i < mapLen; i += 16) {
+		RTW_PRINT_SEL(m, "0x%02x\t", i);
+		for (j = 0; j < 8; j++)
+			RTW_PRINT_SEL(m, "%02X ", pEfuseHal->fakeEfuseInitMap[i + j]);
+		RTW_PRINT_SEL(m, "\t");
+		for (; j < 16; j++)
+			RTW_PRINT_SEL(m, "%02X ", pEfuseHal->fakeEfuseInitMap[i + j]);
+		RTW_PRINT_SEL(m, "\n");
+	}
+
+#endif
+	return 0;
+}
+
+ssize_t proc_set_efuse_map(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+#if 0
+	char tmp[256] = {0};
+	u32 addr, cnts;
+	u8 efuse_data;
+
+	int jj, kk;
+
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct pwrctrl_priv *pwrctrlpriv  = adapter_to_pwrctl(padapter);
+
+	if (count < 3) {
+		RTW_INFO("argument size is less than 3\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%x %d %x", &addr, &cnts, &efuse_data);
+
+		if (num != 3) {
+			RTW_INFO("invalid write_reg parameter!\n");
+			return count;
+		}
+	}
+	if (rtw_efuse_map_write(padapter, addr, cnts, &efuse_data) == _FAIL)
+		RTW_INFO("WARN - rtw_efuse_map_write error!!\n");
+#endif
+	return count;
+}
+
+#ifdef CONFIG_IEEE80211W
+ssize_t proc_set_tx_sa_query(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct	sta_priv *pstapriv = &padapter->stapriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	/* macid_ctl move to phl */
+	/* struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj); */
+	struct sta_info *psta;
+	_list	*plist, *phead;
+	char tmp[16];
+	u8	mac_addr[NUM_STA][ETH_ALEN];
+	u32 key_type;
+	u8 index;
+	void *phl = GET_PHL_INFO(dvobj);
+	u16 macid_num = rtw_phl_get_macid_max_num(phl);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (count > 2) {
+		RTW_INFO("argument size is more than 2\n");
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
+
+		int num = sscanf(tmp, "%x", &key_type);
+
+		if (num !=  1) {
+			RTW_INFO("invalid read_reg parameter!\n");
+			return count;
+		}
+		RTW_INFO("0: set sa query request , key_type=%d\n", key_type);
+	} else {
+		return -EFAULT;
+	}
+
+	if (MLME_IS_STA(padapter)
+	    && (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) && SEC_IS_BIP_KEY_INSTALLED(&padapter_link->securitypriv) == _TRUE) {
+		RTW_INFO("STA:"MAC_FMT"\n", MAC_ARG(get_my_bssid(&(pmlmeinfo->network))));
+		/* TX unicast sa_query to AP */
+		issue_action_SA_Query(padapter, padapter_link,
+				get_my_bssid(&(pmlmeinfo->network)), 0, 0, (u8)key_type);
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE && SEC_IS_BIP_KEY_INSTALLED(&padapter_link->securitypriv) == _TRUE) {
+		/* TX unicast sa_query to every client STA */
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+		for (index = 0; index < NUM_STA; index++) {
+			psta = NULL;
+
+			phead = &(pstapriv->sta_hash[index]);
+			plist = get_next(phead);
+
+			while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+				psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+				plist = get_next(plist);
+				_rtw_memcpy(&mac_addr[psta->phl_sta->macid][0], psta->phl_sta->mac_addr, ETH_ALEN);
+			}
+		}
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+		for (index = 0; index < macid_num && index < NUM_STA; index++) {
+			if (rtw_phl_macid_is_used(phl, index) && !rtw_phl_macid_is_bmc(phl, index)) {
+				if (!_rtw_memcmp(get_my_bssid(&(pmlmeinfo->network)), &mac_addr[index][0], ETH_ALEN)
+				    && !IS_MCAST(&mac_addr[index][0])) {
+					issue_action_SA_Query(padapter, padapter_link,
+						&mac_addr[index][0], 0, 0, (u8)key_type);
+					RTW_INFO("STA[%u]:"MAC_FMT"\n", index , MAC_ARG(&mac_addr[index][0]));
+				}
+			}
+		}
+	}
+
+	return count;
+}
+
+int proc_get_tx_sa_query(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "%s\n", __func__);
+	return 0;
+}
+
+ssize_t proc_set_tx_deauth(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct	sta_priv *pstapriv = &padapter->stapriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	/* macid_ctl move to phl */
+	/* struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj); */
+	struct sta_info *psta;
+	_list	*plist, *phead;
+	char tmp[16];
+	u8	mac_addr[NUM_STA][ETH_ALEN];
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u32 key_type = 0;
+	u8 index;
+	void *phl = GET_PHL_INFO(dvobj);
+	u16 macid_num = rtw_phl_get_macid_max_num(phl);
+
+
+	if (count > 2) {
+		RTW_INFO("argument size is more than 2\n");
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
+
+		int num = sscanf(tmp, "%x", &key_type);
+
+		if (num !=  1) {
+			RTW_INFO("invalid read_reg parameter!\n");
+			return count;
+		}
+		RTW_INFO("key_type=%d\n", key_type);
+	} else {
+		return -EFAULT;
+	}
+	if (key_type > 4)
+		return count;
+
+	if (MLME_IS_STA(padapter)
+	    && (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)) {
+		if (key_type == 3) /* key_type 3 only for AP mode */
+			return count;
+		/* TX unicast deauth to AP */
+		issue_deauth_11w(padapter, get_my_bssid(&(pmlmeinfo->network)), 0, (u8)key_type);
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE) {
+		u8 updated = _FALSE;
+
+		if (key_type == 3)
+			issue_deauth_11w(padapter, bc_addr, 0, IEEE80211W_RIGHT_KEY);
+
+		/* TX unicast deauth to every client STA */
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+		for (index = 0; index < NUM_STA; index++) {
+			psta = NULL;
+
+			phead = &(pstapriv->sta_hash[index]);
+			plist = get_next(phead);
+
+			while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+				psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+				plist = get_next(plist);
+				_rtw_memcpy(&mac_addr[psta->phl_sta->macid][0], psta->phl_sta->mac_addr, ETH_ALEN);
+			}
+		}
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+		for (index = 0; index < macid_num && index < NUM_STA; index++) {
+			if (rtw_phl_macid_is_used(phl, index) && !rtw_phl_macid_is_bmc(phl, index)) {
+				if (!_rtw_memcmp(get_my_bssid(&(pmlmeinfo->network)), &mac_addr[index][0], ETH_ALEN)) {
+					if (key_type != 3)
+						issue_deauth_11w(padapter, &mac_addr[index][0], 0, (u8)key_type);
+
+					psta = rtw_get_stainfo(pstapriv, &mac_addr[index][0]);
+					if (psta && key_type != IEEE80211W_WRONG_KEY && key_type != IEEE80211W_NO_KEY) {
+						rtw_stapriv_asoc_list_lock(pstapriv);
+						if (rtw_is_list_empty(&psta->asoc_list) == _FALSE) {
+							rtw_stapriv_asoc_list_del(pstapriv, psta);
+							updated |= ap_free_sta(padapter, psta, _FALSE, 0, WLAN_REASON_PREV_AUTH_NOT_VALID, _TRUE);
+
+						}
+						rtw_stapriv_asoc_list_unlock(pstapriv);
+					}
+
+					RTW_INFO("STA[%u]:"MAC_FMT"\n", index , MAC_ARG(&mac_addr[index][0]));
+				}
+			}
+		}
+
+		associated_clients_update(padapter, updated, STA_INFO_UPDATE_ALL);
+	}
+
+	return count;
+}
+
+int proc_get_tx_deauth(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "%s\n", __func__);
+	return 0;
+}
+
+ssize_t proc_set_tx_auth(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct	sta_priv *pstapriv = &padapter->stapriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	/* macid_ctl move to phl */
+	/*struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj); */
+	struct sta_info *psta;
+	_list	*plist, *phead;
+	char tmp[16];
+	u8	mac_addr[NUM_STA][ETH_ALEN];
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u32 tx_auth = 0;
+	u8 index;
+
+
+	if (count > 2) {
+		RTW_INFO("argument size is more than 2\n");
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
+
+		int num = sscanf(tmp, "%x", &tx_auth);
+
+		if (num !=  1) {
+			RTW_INFO("invalid read_reg parameter!\n");
+			return count;
+		}
+		RTW_INFO("1: setnd auth, 2: send assoc request. tx_auth=%d\n", tx_auth);
+	}
+
+	if (MLME_IS_STA(padapter)
+	    && (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)) {
+		if (tx_auth == 1) {
+			/* TX unicast auth to AP */
+			issue_auth(padapter, NULL, 0);
+		} else if (tx_auth == 2) {
+			/* TX unicast auth to AP */
+			issue_assocreq(padapter);
+		}
+	}
+
+	return count;
+}
+
+int proc_get_tx_auth(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "%s\n", __func__);
+	return 0;
+}
+#endif /* CONFIG_IEEE80211W */
+
+int proc_get_ack_timeout(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 ack_timeout_val = 0;
+	u8 ack_timeout_val_cck = 0;
+
+	rtw_hal_get_hwreg(padapter, HW_VAR_ACKTO, &ack_timeout_val);
+	rtw_hal_get_hwreg(padapter, HW_VAR_ACKTO_CCK, &ack_timeout_val_cck);
+
+	RTW_PRINT_SEL(m, "Current CCK packet ACK Timeout = %d us (0x%x).\n", ack_timeout_val_cck, ack_timeout_val_cck);
+	RTW_PRINT_SEL(m, "Current non-CCK packet ACK Timeout = %d us (0x%x).\n", ack_timeout_val, ack_timeout_val);
+	return 0;
+}
+
+ssize_t proc_set_ack_timeout(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 tmp_v1 = 0, tmp_v2 = 0;
+	u8 ack_to_ms = 0, ack_to_cck_ms = 0;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%u %u", &tmp_v1, &tmp_v2);
+
+		if (num < 1) {
+			RTW_INFO(FUNC_ADPT_FMT ": input parameters < 1\n", FUNC_ADPT_ARG(padapter));
+			return -EINVAL;
+		}
+
+		if (tmp_v1) {
+			ack_to_ms = (u8)tmp_v1;
+			rtw_hal_set_hwreg(padapter, HW_VAR_ACKTO, &ack_to_ms);
+		}
+
+		if (tmp_v2) {
+			ack_to_cck_ms = (u8)tmp_v2;
+			rtw_hal_set_hwreg(padapter, HW_VAR_ACKTO_CCK, &ack_to_cck_ms);
+		}
+		/* This register sets the Ack time out value after Tx unicast packet. It is in units of us. */
+		/* This register sets the Ack time out value after Tx unicast CCK packet. It is in units of us. */
+		RTW_INFO("Set CCK packet ACK Timeout to %d us.\n", ack_to_cck_ms);
+		RTW_INFO("Set non-CCK packet ACK Timeout to %d us.\n", ack_to_ms);
+	}
+
+	return count;
+}
+
+int proc_get_fw_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char fw_ver[20] = {0};
+
+	rtw_phl_get_fw_ver(GET_PHL_INFO(adapter_to_dvobj(adapter)), fw_ver, 20);
+	RTW_PRINT_SEL(m, "FW VER %s\n", fw_ver);
+
+	return 0;
+}
+
+ssize_t proc_set_fw_offload(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+
+#if 0 /*GEORGIA_TODO_FIXIT*/
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	_adapter *pri_adapter = GET_PRIMARY_ADAPTER(adapter);
+	HAL_DATA_TYPE *hal = GET_PHL_COM(adapter_to_dvobj(adapter));
+	char tmp[32];
+	u32 iqk_offload_enable = 0, ch_switch_offload_enable = 0;
+
+	if (buffer == NULL) {
+		RTW_INFO("input buffer is NULL!\n");
+		return -EFAULT;
+	}
+
+	if (count < 1) {
+		RTW_INFO("input length is 0!\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		RTW_INFO("input length is too large\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%d %d", &iqk_offload_enable, &ch_switch_offload_enable);
+
+		if (num < 2) {
+			RTW_INFO("input parameters < 1\n");
+			return -EINVAL;
+		}
+
+		if (hal->RegIQKFWOffload != iqk_offload_enable) {
+			hal->RegIQKFWOffload = iqk_offload_enable;
+			rtw_run_in_thread_cmd(pri_adapter, ((void *)(rtw_hal_update_iqk_fw_offload_cap)), pri_adapter);
+		}
+
+		if (hal->ch_switch_offload != ch_switch_offload_enable)
+			hal->ch_switch_offload = ch_switch_offload_enable;
+	}
+
+	return count;
+#endif
+	return 0;
+}
+
+int proc_get_fw_offload(struct seq_file *m, void *v)
+{
+#if 0 /*GEORGIA_TODO_FIXIT*/
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	HAL_DATA_TYPE *hal = GET_PHL_COM(adapter_to_dvobj(adapter));
+
+
+	RTW_PRINT_SEL(m, "IQK FW offload:%s\n", hal->RegIQKFWOffload?"enable":"disable");
+	RTW_PRINT_SEL(m, "Channel switch FW offload:%s\n", hal->ch_switch_offload?"enable":"disable");
+#endif
+	return 0;
+}
+
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+ssize_t proc_set_txss_tp(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+
+	char tmp[32];
+	u32 enable = 0;
+	u32 txss_tx_tp = 0;
+	int txss_chk_cnt = 0;
+
+	if (buffer == NULL) {
+		RTW_INFO("input buffer is NULL!\n");
+		return -EFAULT;
+	}
+
+	if (count < 1) {
+		RTW_INFO("input length is 0!\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		RTW_INFO("input length is too large\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%u %u %d",
+			&enable, &txss_tx_tp, &txss_chk_cnt);
+
+		if (num < 1) {
+			RTW_INFO("input parameters < 1\n");
+			return -EINVAL;
+		}
+		pmlmeext->txss_ctrl_en = enable;
+
+		if (txss_tx_tp)
+			pmlmeext->txss_tp_th = txss_tx_tp;
+		if (txss_chk_cnt)
+			pmlmeext->txss_tp_chk_cnt = txss_chk_cnt;
+
+		RTW_INFO("%s txss_ctl_en :%s , txss_tp_th:%d, tp_chk_cnt:%d\n",
+			__func__, pmlmeext->txss_tp_th ? "Y" : "N",
+			pmlmeext->txss_tp_th, pmlmeext->txss_tp_chk_cnt);
+
+	}
+
+	return count;
+}
+
+int proc_get_txss_tp(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+
+	RTW_PRINT_SEL(m, "TXSS  Control - %s\n", pmlmeext->txss_ctrl_en ? "enable" : "disable");
+	RTW_PRINT_SEL(m, "TXSS  Tx TP TH - %d\n", pmlmeext->txss_tp_th);
+	RTW_PRINT_SEL(m, "TXSS  check cnt - %d\n", pmlmeext->txss_tp_chk_cnt);
+
+	return 0;
+}
+#ifdef DBG_CTRL_TXSS
+ssize_t proc_set_txss_ctrl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+
+	char tmp[32];
+	u32 tx_1ss = 0;
+
+	if (buffer == NULL) {
+		RTW_INFO("input buffer is NULL!\n");
+		return -EFAULT;
+	}
+
+	if (count < 1) {
+		RTW_INFO("input length is 0!\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		RTW_INFO("input length is too large\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%u",	&tx_1ss);
+
+		if (num < 1) {
+			RTW_INFO("input parameters < 1\n");
+			return -EINVAL;
+		}
+
+		pmlmeext->txss_ctrl_en = _FALSE;
+
+		dbg_ctrl_txss(adapter, tx_1ss);
+
+		RTW_INFO("%s set tx to  1ss :%s\n", __func__, tx_1ss ? "Y" : "N");
+	}
+
+	return count;
+}
+
+int proc_get_txss_ctrl(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+
+	RTW_PRINT_SEL(m, "TXSS  1ss - %s\n", pmlmeext->txss_1ss ? "Y" : "N");
+
+	return 0;
+}
+#endif
+#endif
+
+#ifdef CONFIG_DBG_RF_CAL
+int proc_get_iqk_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	return 0;
+}
+
+ssize_t proc_set_iqk(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 recovery, clear, segment;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d %d %d", &recovery, &clear, &segment);
+
+		if (num != 3) {
+			RTW_INFO("Invalid format\n");
+			return count;
+		}
+
+		rtw_hal_iqk_test(padapter, recovery, clear, segment);
+	}
+
+	return count;
+
+}
+
+int proc_get_lck_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	return 0;
+}
+
+ssize_t proc_set_lck(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u32 trigger;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d", &trigger);
+
+		if (num != 1) {
+			RTW_INFO("Invalid format\n");
+			return count;
+		}
+
+		rtw_hal_lck_test(padapter);
+	}
+
+	return count;
+}
+#endif /* CONFIG_DBG_RF_CAL */
+
+#ifdef CONFIG_SUPPORT_STATIC_SMPS
+ssize_t proc_set_smps(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	char tmp[32];
+	u32 enable = 0;
+	u32 smps_en, smps_tx_tp = 0, smps_rx_tp = 0;
+	u32 smps_test = 0, smps_test_en = 0;
+
+	if (buffer == NULL) {
+		RTW_INFO("input buffer is NULL!\n");
+		return -EFAULT;
+	}
+
+	if (count < 1) {
+		RTW_INFO("input length is 0!\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		RTW_INFO("input length is too large\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%u %u %u %u %u", &smps_en, &smps_tx_tp, &smps_rx_tp,
+			&smps_test, &smps_test_en);
+
+		if (num < 1) {
+			RTW_INFO("input parameters < 1\n");
+			return -EINVAL;
+		}
+
+		pmlmeext->ssmps_en = smps_en;
+		if (smps_tx_tp) {
+			pmlmeext->ssmps_tx_tp_th= smps_tx_tp;
+			pmlmeext->ssmps_rx_tp_th= smps_tx_tp;
+		}
+		if (smps_rx_tp)
+			pmlmeext->ssmps_rx_tp_th = smps_rx_tp;
+
+		#ifdef DBG_STATIC_SMPS
+		if (num > 3) {
+			pmlmeext->ssmps_test = smps_test;
+			pmlmeext->ssmps_test_en = smps_test_en;
+		}
+		#endif
+		RTW_INFO("SM PS : %s tx_tp_th:%d, rx_tp_th:%d\n",
+			(smps_en) ? "Enable" : "Disable",
+			pmlmeext->ssmps_tx_tp_th,
+			pmlmeext->ssmps_rx_tp_th);
+		#ifdef DBG_STATIC_SMPS
+		RTW_INFO("SM PS : %s ssmps_test_en:%d\n",
+			(smps_test) ? "Enable" : "Disable",
+			pmlmeext->ssmps_test_en);
+		#endif
+	}
+
+	return count;
+}
+
+int proc_get_smps(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+
+	RTW_PRINT_SEL(m, "Static SMPS %s\n", pmlmeext->ssmps_en ? "enable" : "disable");
+	RTW_PRINT_SEL(m, "Tx TP TH %d\n", pmlmeext->ssmps_tx_tp_th);
+	RTW_PRINT_SEL(m, "Rx TP TH %d\n", pmlmeext->ssmps_rx_tp_th);
+	#ifdef DBG_STATIC_SMPS
+	RTW_PRINT_SEL(m, "test %d, test_en:%d\n", pmlmeext->ssmps_test, pmlmeext->ssmps_test_en);
+	#endif
+	return 0;
+}
+#endif /*CONFIG_SUPPORT_STATIC_SMPS*/
+
+#ifdef RTW_DETECT_HANG
+ssize_t proc_set_hang_info(struct file *file, const char __user *buffer,
+			   size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *pdvobjpriv = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &pdvobjpriv->drv_dbg;
+	struct hang_info *phang_info = &pdbgpriv->dbg_hang_info;
+	char tmp[32] = {0};
+	int level = 0;
+	int stop = 0;
+	int num = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num = sscanf(tmp, "%d %d", &level, &stop);
+		if (num == 1) {
+			RTW_INFO("level configuration is not available now\n");
+		} else if (num == 2) {
+			phang_info->is_stop = stop;
+			RTW_INFO("level=%d, stop=%d\n", level,
+				 phang_info->is_stop);
+		} else {
+			RTW_INFO("argument number is wrong\n");
+			return -EFAULT;
+		}
+	}
+
+	return count;
+}
+
+int proc_get_hang_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *pdvobjpriv = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &pdvobjpriv->drv_dbg;
+	struct hang_info *phang_info = &pdbgpriv->dbg_hang_info;
+	struct fw_hang_info *pfw_hang_info = &phang_info->dbg_fw_hang_info;
+	struct rxff_hang_info *prxff_hang_info = &phang_info->dbg_rxff_hang_info;
+	u8 is_hang = _FALSE;
+	u8 fw_gone = _FALSE;
+
+	if (phang_info->is_stop) {
+		RTW_PRINT_SEL(m, "This function is disabled\n");
+		goto exit;
+	}
+
+	if (dev_is_surprise_removed(pdvobjpriv))
+		is_hang = _TRUE;
+
+	if (pfw_hang_info->dbg_is_fw_gone &&
+	    rtw_hw_get_init_completed(pdvobjpriv) &&
+	    !padapter->registrypriv.mp_mode)
+		fw_gone = _TRUE;
+
+	if (pfw_hang_info->dbg_is_fw_hang || is_hang || fw_gone ||
+	    prxff_hang_info->dbg_is_rxff_hang) {
+		RTW_PRINT_SEL(m, "1\n");
+
+		if (pfw_hang_info->dbg_is_fw_hang)
+			RTW_PRINT_SEL(m, "FW hang\n");
+		if (prxff_hang_info->dbg_is_rxff_hang)
+			RTW_PRINT_SEL(m, "RXFF hang\n");
+		if (is_hang)
+			RTW_PRINT_SEL(m, "surprise remove\n");
+		if (fw_gone)
+			RTW_PRINT_SEL(m, "FW gone\n");
+	} else {
+		RTW_PRINT_SEL(m, "0\n");
+	}
+
+	RTW_PRINT_SEL(m, "Enter cnt = %d\n", phang_info->enter_cnt);
+exit:
+	return 0;
+}
+#endif /* RTW_DETECT_HANG */
+
+int proc_get_disconnect_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	char *disconnect_reason;
+	char *illegal_beacon_reason;
+
+	if (pmlmeinfo) {
+		switch (pmlmeinfo->disconnect_code) {
+		case DISCONNECTION_NOT_YET_OCCUR:
+			disconnect_reason = "Disconnection has not yet occurred.";
+			break;
+		case DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND:
+			disconnect_reason = "System requests to disconnect by command.";
+			break;
+		case DISCONNECTION_BY_SYSTEM_DUE_TO_NET_DEVICE_DOWN:
+			disconnect_reason = "System makes net_device down to cause disconnection.";
+			break;
+		case DISCONNECTION_BY_SYSTEM_DUE_TO_SYSTEM_IN_SUSPEND:
+			disconnect_reason = "System enters suspend state to cause disconnection.";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_CONNECTION_EXIST:
+			disconnect_reason = "The connection is exist so it triggers disconnection when joinbss.";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_EACH_IFACE_CHBW_NOT_SYNC:
+			disconnect_reason = "Multiple interfaces do not synchronize channel and bandwidth when joinbss.";
+			break;
+		/*
+		case DISCONNECTION_BY_DRIVER_DUE_TO_DFS_DETECTION:
+			disconnect_reason = "DFS Slave mechanism detects radar to cause disconnection.";
+			break;
+		*/
+		case DISCONNECTION_BY_DRIVER_DUE_TO_IOCTL_DBG_PORT:
+			disconnect_reason = "Using ioctl dbg port command to accomplish disconnection.";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_AP_BEACON_CHANGED:
+			disconnect_reason = "AP's beacon content was changed to cause disconnection.";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_KEEPALIVE_TIMEOUT:
+			disconnect_reason = "Maybe AP disappears to trigger station keepalive timeout and connection break.";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_LAYER2_ROAMING_TERMINATE:
+			disconnect_reason = "Layer2 roaming terminated to trigger disconnection.";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_JOINBSS_TIMEOUT:
+			disconnect_reason = "Station joinbss process is timeout to trigger disconnection.";
+			break;
+		case DISCONNECTION_BY_FW_DUE_TO_FW_DECISION_IN_WOW_RESUME:
+			disconnect_reason = "FW decides to disconnect in WOW resume flow.";
+			break;
+		case DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DISASSOC_IN_WOW_RESUME:
+			disconnect_reason = "Receiving Disassociation frame from AP to cause disconnection in WOW resume flow.";
+			break;
+		case DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DEAUTH_IN_WOW_RESUME:
+			disconnect_reason = "Receiving Deauth frame from AP to cause disconnection in WOW resume flow.";
+			break;
+		case DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DEAUTH:
+			disconnect_reason = "Receiving Deauth frame from AP to cause disconnection.";
+			break;
+		case DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DISASSOC:
+			disconnect_reason = "Receiving Disassociation frame from AP to cause disconnection.";
+			break;
+		/*
+		case DISCONNECTION_BY_DRIVER_DUE_TO_RECEIVE_CSA_NON_DFS:
+			disconnect_reason = "Disconnection due to AP switched to unsupported channel.";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_RECEIVE_CSA_DFS:
+			disconnect_reason = "Disconnection due to AP switched to unsupported DFS channel.";
+			break;
+		*/
+		case DISCONNECTION_BY_DRIVER_DUE_TO_RECEIVE_INVALID_CSA:
+			disconnect_reason = "Disconnection due to received invalid CSA IE";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_JOIN_WRONG_CHANNEL:
+			disconnect_reason = "Disconnection due to joined wrong channel";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_FT:
+			disconnect_reason = "Disconnection due to FT";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_ROAMING:
+			disconnect_reason = "Disconnection due to roaming";
+			break;
+		case DISCONNECTION_BY_DRIVER_DUE_TO_SA_QUERY_TIMEOUT:
+			disconnect_reason = "Disconnection due to SA Query timeout";
+			break;
+		default:
+			disconnect_reason = "Unspecified";
+			break;
+		}
+
+		RTW_PRINT_SEL(m, "occurred disconnection time = %d\n", pmlmeinfo->disconnect_occurred_time);
+		RTW_PRINT_SEL(m, "disconnect code = %d\n", pmlmeinfo->disconnect_code);
+		RTW_PRINT_SEL(m, "last disconnect reason: %s\n", disconnect_reason);
+		RTW_PRINT_SEL(m, "beacon code: %d\n", pmlmeinfo->illegal_beacon_code);
+		if (pmlmeinfo->disconnect_code == DISCONNECTION_BY_DRIVER_DUE_TO_AP_BEACON_CHANGED) {
+			if(pmlmeinfo->illegal_beacon_code & SSID_CHANGED)
+				RTW_PRINT_SEL(m, "illegal beacon reason: The SSID of beacon is changed.\n");
+			if(pmlmeinfo->illegal_beacon_code & SSID_LENGTH_CHANGED)
+				RTW_PRINT_SEL(m, "illegal beacon reason: The SSID length of beacon is changed.\n");
+			if(pmlmeinfo->illegal_beacon_code & BEACON_CHANNEL_CHANGED)
+				RTW_PRINT_SEL(m, "illegal beacon reason: The Beacon channel of beacon is changed.\n");
+			if(pmlmeinfo->illegal_beacon_code & ENCRYPT_PROTOCOL_CHANGED)
+				RTW_PRINT_SEL(m, "illegal beacon reason: The Encrypt protocol of beacon is changed.\n");
+			if(pmlmeinfo->illegal_beacon_code & PAIRWISE_CIPHER_CHANGED)
+				RTW_PRINT_SEL(m, "illegal beacon reason: The Pairwise cipheris of beacon changed.\n");
+			if(pmlmeinfo->illegal_beacon_code & GROUP_CIPHER_CHANGED)
+				RTW_PRINT_SEL(m, "illegal beacon reason: The Group cipher of beacon is changed.\n");
+			if(pmlmeinfo->illegal_beacon_code & IS_8021X_CHANGED)
+				RTW_PRINT_SEL(m, "illegal beacon reason: The 802.1x of beacon is changed.\n");
+		} else {
+			RTW_PRINT_SEL(m, "illegal beacon reason: NA.\n");
+		}
+
+		RTW_PRINT_SEL(m, "Wi-Fi reason code: %d\n",
+			      pmlmeinfo->disconnect_code ? pmlmeinfo->wifi_reason_code : 0);
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_disconnect_info(struct file *file, const char __user *buffer,
+				 size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	char tmp[32];
+	u8 clear;
+
+	if (!pmlmeinfo)
+		return -EPERM;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhd", &clear);
+		if (num == 1 && clear == 0) {
+			pmlmeinfo->disconnect_occurred_time = 0;
+			pmlmeinfo->disconnect_code = DISCONNECTION_NOT_YET_OCCUR;
+			pmlmeinfo->illegal_beacon_code = 0;
+		}
+	}
+
+	return count;
+}
+
+int proc_get_chan(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct _ADAPTER_LINK *padapter_link;
+	u8 lidx;
+
+	for (lidx = 0; lidx < padapter->phl_role->rlink_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		RTW_PRINT_SEL(m, "ch=%d, ch_offset=%d, bw=%d\n",
+				rtw_get_oper_ch(padapter, padapter_link),
+				rtw_get_oper_choffset(padapter, padapter_link),
+				rtw_get_oper_bw(padapter, padapter_link));
+
+		RTW_PRINT_SEL(m, "PHL rlink %d ch=%d\n", lidx,
+			rtw_phl_get_cur_ch(padapter->phl_role,
+							padapter_link->wrlink));
+	}
+	return 0;
+}
+
+ssize_t proc_set_chan(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 target_band, target_ch, target_offset, target_bw;
+#ifdef DBG_CONFIG_CMD_DISP
+	u8 cmd_type = 0xFF;
+	u8 cmd_to = 0;
+#endif
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (count < 3) {
+		RTW_INFO("argument size is less than 3\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = 0;
+#ifdef DBG_CONFIG_CMD_DISP
+		num = sscanf(tmp, "%hhu %hhu %hhu %hhu %hhu %hhu",
+			&target_band, &target_ch, &target_offset, &target_bw,
+			&cmd_type, &cmd_to);
+		if (num != 6) {
+			RTW_INFO("invalid write_reg parameter!\n");
+			return count;
+		}
+
+		padapter->cmd_type = cmd_type;
+		padapter->cmd_timeout = cmd_to;
+#else
+		num = sscanf(tmp, "%hhu %hhu %hhu %hhu",
+			&target_band, &target_ch, &target_offset, &target_bw);
+		if (num != 4) {
+			RTW_INFO("invalid write_reg parameter!\n");
+			return count;
+		}
+#endif
+		set_bch_bwmode(padapter, padapter_link, target_band, target_ch,
+			       target_offset, target_bw, RFK_TYPE_FORCE_NOT_DO);
+	}
+
+	return count;
+}
+
+int proc_get_mr_test(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	return 0;
+}
+
+ssize_t proc_set_mr_test(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u8 mode, param, param_1, param_2, param_3;
+	bool show_caller;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (count < 2) {
+		RTW_INFO("argument size is less than 3\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = 0;
+
+		num = sscanf(tmp, "%hhu %hhu %hhu %hhu %hhu", &mode, &param, &param_1, &param_2, &param_3);
+		if (num < 2) {
+			RTW_INFO("invalid write_reg parameter!\n");
+			return count;
+		}
+#ifdef PHL_MR_PROC_CMD
+		if (mode == 1) {
+			show_caller = (param) ? _TRUE : _FALSE;
+			rtw_phl_mr_dump_info(dvobj->phl, show_caller);
+		}
+		else if (mode == 2) {
+			show_caller = (param) ? _TRUE : _FALSE;
+			rtw_phl_mr_dump_band_ctl(dvobj->phl, show_caller);
+		}
+		else if (mode == 3) {
+			if (param == 1)
+				rtw_phl_mr_dump_mac_addr(dvobj->phl, padapter->phl_role);
+			else if (param == 2)
+				rtw_phl_mr_buddy_dump_mac_addr(dvobj->phl, padapter->phl_role);
+			else if (param == 3)
+				rtw_mi_dump_mac_addr(padapter);
+		}
+		else if (mode == 4) {
+			show_caller = (param) ? _TRUE : _FALSE;
+			rtw_phl_sta_dump_info(dvobj->phl, show_caller, padapter->phl_role, param_1);
+		}
+		else if (mode == 5 || mode == 6 || mode == 7 || mode == 8 || mode == 9) {
+			u8 test_mode = 0;/*0-chk, 1-add, 2-del*/
+			struct rtw_chan_def new_chdef = {0};
+			struct rtw_mr_chctx_info mr_cc_info = {0};
+			bool rst;
+
+			if (mode == 5)
+				test_mode = 0;
+			else if (mode == 6)
+				test_mode = 1;
+			else if (mode == 7)
+				test_mode = 2;
+			else if (mode == 8)
+				test_mode = 3;
+			else if (mode == 9)
+				test_mode = 4;
+			new_chdef.band = param;
+			new_chdef.chan = param_1;
+			new_chdef.bw = param_2;
+			new_chdef.offset = param_3;
+
+			rst = rtw_phl_chanctx_test(dvobj->phl, padapter->phl_role, padapter_link->wrlink,
+						   test_mode, &new_chdef, &mr_cc_info);
+			RTW_INFO("%s s_band:%d,s_ch:%d,s_bw:%d,s_offset:%d\n",
+				__func__, param, param_1, param_2, param_3);
+		}
+		else if (mode == 10) {
+			u8 new_port = param;
+			u8 org_port = padapter_link->wrlink->hw_port;
+
+			rtw_phl_wifi_role_realloc_port(dvobj->phl, padapter->phl_role, padapter_link->wrlink, new_port);
+			RTW_INFO("%s realloc port from %d to %d\n",
+					__func__, org_port, new_port);
+		}
+		else if (mode == 11) {
+			rtw_phl_get_addr_cam(dvobj->phl, param, NULL, 0);
+		}
+		else if (mode == 12) {
+			struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+			struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+			struct sta_priv *pstapriv = &padapter->stapriv;
+			struct sta_info *psta;
+
+			psta = rtw_get_stainfo(pstapriv, get_link_bssid(pmlmepriv));
+			if (psta) {
+				if (param == 1)
+					rtw_ssmps_enter(padapter, psta);
+				else if (param == 0)
+					rtw_ssmps_leave(padapter, psta);
+			}
+		}
+		else if (mode == 13) {
+			#ifdef CONFIG_DBCC_SUPPORT
+			dvobj->dis_dbcc_scan = param;
+			RTW_INFO("%s dis_dbcc_scan %d\n",
+					__func__, dvobj->dis_dbcc_scan);
+			#endif
+		}
+		#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+		else if (mode == 14) {
+			/*mode
+			param - 0: rtw_phl_cmd_dbcc_enable, 1 :rtw_phl_cmd_dbcc_disable
+			param_1 - iface_id
+			param_2 - cmd_type : 0:PHL_CMD_DIRECTLY,1:PHL_CMD_NO_WAIT,2:PHL_CMD_WAIT*/
+
+			_adapter *target_adp = NULL;
+			struct _ADAPTER_LINK *target_alink;
+			struct rtw_wifi_role_link_t *target_rlink;
+
+			if (param == 0)
+				RTW_INFO("%s dbcc enable cmd test - iface_id %d cmd_type:%d\n",
+					__func__, param_1, param_2);
+			else /* if (param == 1)*/
+				RTW_INFO("%s dbcc disable cmd test - iface_id %d cmd_type:%d\n",
+					__func__, param_1, param_2);
+
+			target_adp = dvobj->padapters[param_1];
+			target_alink = GET_PRIMARY_LINK(target_adp);
+			target_rlink = target_alink->wrlink;
+
+			RTW_INFO(FUNC_ADPT_FMT" at HW_B(%d)\n",
+				FUNC_ADPT_ARG(target_adp), target_rlink->hw_band);
+			if (param == 0) {
+				rtw_phl_cmd_dbcc_enable(target_adp->phl_role,
+							  target_rlink->hw_band,
+							  param_2, 0);
+			} else {
+				rtw_phl_cmd_dbcc_disable(target_adp->phl_role,
+							  target_rlink->hw_band,
+							  param_2, 0);
+			}
+		}
+		#endif
+#endif
+	}
+
+	return count;
+}
+
+int proc_get_deny_legacy(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "deny_legacy       = 0x%x\n", pregpriv->deny_legacy);
+
+	if (pmlmeext)
+		RTW_PRINT_SEL(m, "cur_wireless_mode = 0x%x\n", pmlmeext->cur_wireless_mode);
+
+	return 0;
+}
+
+ssize_t proc_set_deny_legacy(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u8 legacy;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu ", &legacy);
+
+		if (num == 1 && pregpriv) {
+			pregpriv->deny_legacy = legacy;
+			RTW_INFO("deny_legacy=%d\n", legacy);
+		}
+	}
+
+	return count;
+}
+
+#ifdef CONFIG_80211AX_HE
+int proc_get_tx_ul_mu_disable(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct rtw_he_actrl_om *cur_om_info = &(pmlmepriv->hepriv.om_info);
+
+	RTW_PRINT_SEL(m, "OM ul_mu_disable = %s\n", cur_om_info->om_actrl_ele.ul_mu_disable ? "True" : "False");
+	return 0;
+}
+
+ssize_t proc_set_tx_ul_mu_disable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_he_actrl_om om_info = {0};
+	u8 om_mask = 0;
+	char tmp[32];
+	u8 ul_mu_disable;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu ", &ul_mu_disable);
+
+		if (num == 1) {
+			om_mask = om_mask | OM_UL_MU_DIS;
+			/*om_info.actrl_om_normal_tx = _TRUE;*/
+			/*om_info.actrl_om_normal_tx_cnt = 8;*/
+			if (ul_mu_disable == _TRUE)
+				om_info.om_actrl_ele.ul_mu_disable = _TRUE;
+			else
+				om_info.om_actrl_ele.ul_mu_disable = _FALSE;
+
+#if 0
+			/*om others control setting*/
+			om_mask = om_mask | OM_RX_NSS;
+			om_info.om_actrl_ele.rx_nss = 1;
+			om_mask = om_mask | OM_CH_BW;
+			om_info.om_actrl_ele.channel_width= 2;
+			om_mask = om_mask | OM_TX_NSTS;
+			om_info.om_actrl_ele.tx_nsts= 2;
+			om_mask = om_mask | OM_ER_SU_DIS;
+			om_info.om_actrl_ele.er_su_disable = _TRUE;
+			om_mask = om_mask | OM_DL_MU_RR;
+			om_info.om_actrl_ele.dl_mu_mimo_rr= _TRUE;
+			om_mask = om_mask | OM_UL_MU_DATA_DIS;
+			om_info.om_actrl_ele.ul_mu_data_disable= _TRUE;
+#endif
+			rtw_he_set_om_info(padapter, padapter_link, om_mask, &om_info);
+			issue_qos_nulldata(padapter, padapter_link,
+					NULL, 0, 0, 3, 10, _TRUE);
+		}
+	}
+
+	return count;
+}
+#endif
+
+#ifdef CONFIG_RTW_CSI_CHANNEL_INFO
+int proc_get_csi_ch_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *d = padapter->dvobj;
+	void *phl = GET_PHL_INFO(d);
+	struct csi_header_t csi_header;
+	u8 buff[512] = {0};
+	u32 buf_len;
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+
+	_rtw_memset(&csi_header, 0, sizeof(csi_header));
+	status = rtw_phl_query_chan_info(phl, 512, buff, &buf_len, &csi_header);
+	if (status == RTW_PHL_STATUS_SUCCESS) {
+		RTW_DUMP_SEL(m, buff, buf_len);
+		RTW_PRINT_SEL(m, "rxsc=%d\n", csi_header.rxsc);
+		RTW_PRINT_SEL(m, "nc=%d\n", csi_header.nc);
+		RTW_PRINT_SEL(m, "nr=%d\n", csi_header.nr);
+		RTW_PRINT_SEL(m, "avg_idle_noise_pwr=%d\n", csi_header.avg_idle_noise_pwr);
+		RTW_PRINT_SEL(m, "csi_data_length=%d\n", csi_header.csi_data_length);
+		RTW_PRINT_SEL(m, "csi_valid=%d\n", csi_header.csi_valid);
+		RTW_PRINT_SEL(m, "evm[0]=%d\n", csi_header.evm[0]);
+		RTW_PRINT_SEL(m, "evm[1]=%d\n", csi_header.evm[1]);
+		RTW_PRINT_SEL(m, "rssi[0]=%d%%\n", csi_header.rssi[0] >> 1);
+		RTW_PRINT_SEL(m, "rssi[1]=%d%%\n", csi_header.rssi[1] >> 1);
+		RTW_PRINT_SEL(m, "timestamp=%d\n", csi_header.hw_assigned_timestamp);
+		RTW_PRINT_SEL(m, "rx_data_rate=%d\n", csi_header.rx_data_rate);
+		RTW_PRINT_SEL(m, "bandwidth=%d\n", csi_header.bandwidth);
+	}
+	return 0;
+}
+
+ssize_t proc_set_csi_ch_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *d = adapter->dvobj;
+	void *phl = GET_PHL_INFO(d);
+	struct _ADAPTER_LINK *adapter_link = NULL;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *wrlink = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL, *peer_sta = NULL;
+	struct rtw_chinfo_action_parm act_parm;
+	char tmp[32];
+	u8 en, group_num, accuracy, mode, en_mode, period, addr[ETH_ALEN];
+	int i, num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	adapter_link = GET_PRIMARY_LINK(adapter);
+	if (adapter_link == NULL) {
+		RTW_ERR("adapter_link = NULL\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	wrole = adapter->phl_role;
+	if (wrole == NULL) {
+		RTW_ERR("wrole = NULL\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	wrlink = adapter_link->wrlink;
+	if (wrlink == NULL) {
+		RTW_ERR("wrlink = NULL\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	phl_sta = rtw_phl_get_stainfo_self(phl, wrlink);
+	if (phl_sta == NULL) {
+		RTW_ERR("phl_sta = NULL\n");
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	RTW_INFO("sta mac addr: "MAC_FMT"\n", MAC_ARG(phl_sta->mac_addr));
+	RTW_INFO("sta->wrole mac addr: "MAC_FMT"\n", MAC_ARG(phl_sta->wrole->mac_addr));
+	RTW_INFO("sta->rlink mac addr: "MAC_FMT"\n", MAC_ARG(phl_sta->rlink->mac_addr));
+
+	_rtw_memset(&act_parm, 0, sizeof(struct rtw_chinfo_action_parm));
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num = sscanf(tmp, "%hhx %hhx %hhx %hhx %hhx %hhd %hhx %hhx %hhx %hhx %hhx %hhx",
+					&en, &group_num, &accuracy, &mode, &en_mode, &period,
+					addr, addr + 1, addr + 2, addr + 3, addr + 4, addr + 5);
+		if ((num == 1) || (num == 6))
+			goto csi_set_next;
+
+		num = sscanf(tmp, "%hhx %hhx %hhx %hhx %hhx %hhd %hhx %hhx %hhx %hhx %hhx %hhx",
+					&en, &group_num, &accuracy, &mode, &en_mode, &period,
+					addr, addr + 1, addr + 2, addr + 3, addr + 4, addr + 5);
+		if (num == 12)
+			goto csi_set_next;
+
+		num = sscanf(tmp, "%hhx %hhx %hhx %hhx %hhx %hhx %hhx",
+					&en, addr, addr + 1, addr + 2, addr + 3, addr + 4, addr + 5);
+		if (num == 7)
+			goto csi_set_next;
+
+csi_set_next:
+		RTW_INFO("%s: arguments num=%d\n", __func__, num);
+		RTW_INFO("%s: client mac addr: "MAC_FMT"\n", __func__, MAC_ARG(addr));
+		RTW_INFO("%s: wrole type = %d\n", __func__, wrole->type);
+
+		/*
+		 * num=1: for STA mode, default setting
+		 * num=6: for STA mode, changed setting
+		 * num=7: for softAP mode, default setting + associated client MAC address
+		 * num=12: for softAP mode, changed setting + associated client MAC address
+		 */
+		if ((num != 1) && (num != 6) && (num !=7) && (num != 12)) {
+			RTW_ERR("%s: Wrong setting (%d arguments), only accept 1, 6, 7, or 12 arguments\n", __func__, num);
+			return -EFAULT;
+		}
+
+		if (rtw_phl_is_ap_category(wrole->type)) {
+			if ((num == 1) || (num == 6)) {
+				RTW_ERR("%s: Current is AP mode, you need assigned associated client MAC address!\n", __func__);
+				return -EFAULT;
+			}
+		} else if (rtw_phl_is_client_category(wrole->type)) {
+			if ((num == 1) || (num == 6)) {
+				if (LINK_MLME_IS_ASOC(adapter_link)) {
+					RTW_INFO("%s: Current is STA mode, no MAC address assigned, use AP address by default\n", __func__);
+				} else {
+					RTW_ERR("%s: Current is STA mode, connected before using CSI\n", __func__);
+					return -EFAULT;
+				}
+			}
+		} else {
+			RTW_ERR("%s: wrole type %d is not supported!\n", __func__, wrole->type);
+			return -EFAULT;
+		}
+
+		if (num == 7 || num == 12) { /* peer address is specified */
+			struct sta_info *sta = rtw_get_stainfo(&adapter->stapriv, addr);
+
+			if (sta)
+				peer_sta = sta->phl_sta;
+			if (!peer_sta) {
+				RTW_ERR("%s: can't get peer sta with addr!\n", __func__);
+				return -EFAULT;
+			}
+		} else
+			peer_sta = phl_sta;
+
+		switch (num) {
+		case 1:
+		case 7:
+			/* default setting */
+
+			act_parm.act = CHINFO_ACT_CFG;
+			act_parm.group_num = CHINFO_GROUP_NUM_16;
+			act_parm.accuracy = CHINFO_ACCU_1BYTE;
+			//act_parm.mode = CHINFO_MODE_MACID;
+			act_parm.mode = CHINFO_MODE_ACK;
+			act_parm.enable_mode = CHINFO_EN_RICH_MODE; //if set cause 100ms CSI
+			act_parm.sta = peer_sta;
+			act_parm.enable = en;
+			act_parm.trig_period = 30;
+			act_parm.tx_nss = 2;
+			act_parm.rx_nss = 2;
+			if (act_parm.mode == CHINFO_MODE_ACK)
+				act_parm.ele_bitmap = 0x0101; /* For ACK, Legacy, only H00, H01 */
+			else
+				act_parm.ele_bitmap = 0x0303;
+			rtw_phl_cmd_cfg_chinfo(phl, &act_parm, PHL_CMD_WAIT, 500);
+			act_parm.act = CHINFO_ACT_EN;
+			rtw_phl_cmd_cfg_chinfo(phl, &act_parm, PHL_CMD_WAIT, 500);
+			break;
+		case 6:
+		case 12:
+			/* changed setting */
+
+			/*default: 1 3 0 0*/
+			act_parm.act = CHINFO_ACT_CFG;
+			act_parm.group_num = group_num; /* 0~3: num 1/2/4/16 */
+			act_parm.accuracy = accuracy; 	/* 0: 1byte, 1: 2 bytes */
+			//act_parm.mode = CHINFO_MODE_MACID;
+			act_parm.mode = mode;		/* 0: ACK mode, 1: MACID, 2: NDP */
+			act_parm.enable_mode = en_mode; /* 0: Light mode, 1: Rich mode, 2: auto mode */
+			act_parm.sta = peer_sta;
+			act_parm.enable = en;
+			act_parm.trig_period = period;
+			act_parm.tx_nss = 2;
+			act_parm.rx_nss = 2;
+			if (act_parm.mode == CHINFO_MODE_ACK)
+				act_parm.ele_bitmap = 0x0101; /* For ACK, Legacy, only H00, H01 */
+			else
+				act_parm.ele_bitmap = 0x0303;
+			rtw_phl_cmd_cfg_chinfo(phl, &act_parm, PHL_CMD_WAIT, 500);
+			act_parm.act = CHINFO_ACT_EN;
+			rtw_phl_cmd_cfg_chinfo(phl, &act_parm, PHL_CMD_WAIT, 500);
+			break;
+		default:
+			RTW_ERR("%s: Wrong setting, only accept 1, 6, 7, or 12 arguments\n", __func__);
+			return -EFAULT;
+			break;
+		}
+	}
+
+	return count;
+}
+#endif
+
+#endif /* CONFIG_PROC_DEBUG */
+#define RTW_BUFDUMP_BSIZE		16
+#if 1
+inline void RTW_BUF_DUMP_SEL(uint _loglevel, void *sel, u8 *_titlestring,
+					bool _idx_show, const u8 *_hexdata, int _hexdatalen)
+{
+#ifdef CONFIG_RTW_DEBUG
+	int __i;
+	u8 *ptr = (u8 *)_hexdata;
+
+	if (_loglevel <= rtw_drv_log_level) {
+		if (_titlestring) {
+			if (sel == RTW_DBGDUMP)
+				RTW_PRINT("");
+			_RTW_PRINT_SEL(sel, "%s", _titlestring);
+			if (_hexdatalen >= RTW_BUFDUMP_BSIZE)
+				_RTW_PRINT_SEL(sel, "\n");
+		}
+
+		for (__i = 0; __i < _hexdatalen; __i++) {
+			if (((__i % RTW_BUFDUMP_BSIZE) == 0) && (_hexdatalen >= RTW_BUFDUMP_BSIZE)) {
+				if (sel == RTW_DBGDUMP)
+					RTW_PRINT("");
+				if (_idx_show)
+					_RTW_PRINT_SEL(sel, "0x%03X: ", __i);
+			}
+			_RTW_PRINT_SEL(sel, "%02X%s", ptr[__i], (((__i + 1) % 4) == 0) ? "  " : " ");
+			if ((__i + 1 < _hexdatalen) && ((__i + 1) % RTW_BUFDUMP_BSIZE) == 0)
+				_RTW_PRINT_SEL(sel, "\n");
+		}
+		_RTW_PRINT_SEL(sel, "\n");
+	}
+#endif
+}
+
+inline void RTW_BUF_DUMP_SEL_ALWAYS(void *sel, u8 *_titlestring,
+					bool _idx_show, const u8 *_hexdata, int _hexdatalen)
+{
+	int __i;
+	u8 *ptr = (u8 *)_hexdata;
+
+	if (_titlestring) {
+		if (sel == RTW_DBGDUMP)
+			RTW_PRINT("");
+		_RTW_PRINT_SEL(sel, "%s", _titlestring);
+		if (_hexdatalen >= RTW_BUFDUMP_BSIZE)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+
+	for (__i = 0; __i < _hexdatalen; __i++) {
+		if (((__i % RTW_BUFDUMP_BSIZE) == 0) && (_hexdatalen >= RTW_BUFDUMP_BSIZE)) {
+			if (sel == RTW_DBGDUMP)
+				RTW_PRINT("");
+			if (_idx_show)
+				_RTW_PRINT_SEL(sel, "0x%03X: ", __i);
+		}
+		_RTW_PRINT_SEL(sel, "%02X%s", ptr[__i], (((__i + 1) % 4) == 0) ? "  " : " ");
+		if ((__i + 1 < _hexdatalen) && ((__i + 1) % RTW_BUFDUMP_BSIZE) == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+}
+#else
+inline void _RTW_STR_DUMP_SEL(void *sel, char *str_out)
+{
+	if (sel == RTW_DBGDUMP)
+		_dbgdump("%s\n", str_out);
+	#if defined(_seqdump)
+	else
+		_seqdump(sel, "%s\n", str_out);
+	#endif /*_seqdump*/
+}
+inline void RTW_BUF_DUMP_SEL(uint _loglevel, void *sel, u8 *_titlestring,
+					bool _idx_show, u8 *_hexdata, int _hexdatalen)
+{
+	int __i, len;
+	int __j, idx;
+	int block_num, remain_byte;
+	char str_out[128] = {'\0'};
+	char str_val[32] = {'\0'};
+	char *p = NULL;
+	u8 *ptr = (u8 *)_hexdata;
+
+	if (_loglevel <= rtw_drv_log_level) {
+		/*dump title*/
+		p = &str_out[0];
+		if (_titlestring) {
+			if (sel == RTW_DBGDUMP) {
+				len = snprintf(str_val, sizeof(str_val), "%s", DRIVER_PREFIX);
+				strncpy(p, str_val, len);
+				p += len;
+			}
+			len = snprintf(str_val, sizeof(str_val), "%s", _titlestring);
+			strncpy(p, str_val, len);
+			p += len;
+		}
+		if (p != &str_out[0]) {
+			_RTW_STR_DUMP_SEL(sel, str_out);
+			_rtw_memset(&str_out, '\0', sizeof(str_out));
+		}
+
+		/*dump buffer*/
+		block_num = _hexdatalen / RTW_BUFDUMP_BSIZE;
+		remain_byte = _hexdatalen % RTW_BUFDUMP_BSIZE;
+		for (__i = 0; __i < block_num; __i++) {
+			p = &str_out[0];
+			if (sel == RTW_DBGDUMP) {
+				len = snprintf(str_val, sizeof(str_val), "%s", DRIVER_PREFIX);
+				strncpy(p, str_val, len);
+				p += len;
+			}
+			if (_idx_show) {
+				len = snprintf(str_val, sizeof(str_val), "0x%03X: ", __i * RTW_BUFDUMP_BSIZE);
+				strncpy(p, str_val, len);
+				p += len;
+			}
+			for (__j =0; __j < RTW_BUFDUMP_BSIZE; __j++) {
+				idx = __i * RTW_BUFDUMP_BSIZE + __j;
+				len = snprintf(str_val, sizeof(str_val), "%02X%s", ptr[idx], (((__j + 1) % 4) == 0) ? "  " : " ");
+				strncpy(p, str_val, len);
+				p += len;
+			}
+			_RTW_STR_DUMP_SEL(sel, str_out);
+			_rtw_memset(&str_out, '\0', sizeof(str_out));
+		}
+
+		p = &str_out[0];
+		if ((sel == RTW_DBGDUMP) && remain_byte) {
+			len = snprintf(str_val, sizeof(str_val), "%s", DRIVER_PREFIX);
+			strncpy(p, str_val, len);
+			p += len;
+		}
+		if (_idx_show && remain_byte) {
+			len = snprintf(str_val, sizeof(str_val), "0x%03X: ", block_num * RTW_BUFDUMP_BSIZE);
+			strncpy(p, str_val, len);
+			p += len;
+		}
+		for (__i = 0; __i < remain_byte; __i++) {
+			idx = block_num * RTW_BUFDUMP_BSIZE + __i;
+			len = snprintf(str_val, sizeof(str_val), "%02X%s", ptr[idx], (((__i + 1) % 4) == 0) ? "  " : " ");
+			strncpy(p, str_val, len);
+			p += len;
+		}
+		_RTW_STR_DUMP_SEL(sel, str_out);
+	}
+}
+
+#endif
+
+int proc_get_vcs(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+
+	if (padapter)
+	{
+		RTW_PRINT_SEL(m, "vrtl_carrier_sense = %d\n", pregpriv->vrtl_carrier_sense);
+		RTW_PRINT_SEL(m, "vcs_type = %u\n", pregpriv->vcs_type);
+		RTW_PRINT_SEL(m, "rts_thresh = %u\n", pregpriv->rts_thresh);
+		RTW_PRINT_SEL(m, "hw_rts_en = %u\n", pregpriv->hw_rts_en);
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_vcs(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u8 vcs, vcs_t, hw_rts;
+	u16 rts_th;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hhu %hhu %hu ", &vcs, &vcs_t, &hw_rts, &rts_th);
+
+		if (padapter && (num == 4)) {
+			pregpriv->vrtl_carrier_sense = vcs;
+			pregpriv->vcs_type = vcs_t;
+			pregpriv->hw_rts_en= hw_rts;
+			pregpriv->rts_thresh = rts_th;
+		}
+		else
+			RTW_WARN("%s: Wrong setting.\n", __func__);
+
+	}
+
+	return count;
+}
+
+int proc_get_cur_tsf(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct rtw_phl_port_tsf tsf = {0};
+	u64 cur_tsf = 0;
+	u32 cmd_timeout = 0;
+	enum rtw_phl_status ret = RTW_PHL_STATUS_FAILURE;
+	enum phl_cmd_type cmd_type = 0;
+
+	tsf.hwband = padapter_link->wrlink->hw_band;
+	tsf.port = padapter_link->wrlink->hw_port;
+
+        if (padapter->cur_tsf_cmd_enqueue) {
+                  cmd_timeout = 500;
+		  cmd_type = PHL_CMD_WAIT;
+	} else {
+		cmd_type = PHL_CMD_DIRECTLY;
+	}
+        ret = rtl_phl_cmd_get_cur_tsf(GET_PHL_INFO(dvobj), &tsf, padapter_link->wrlink, cmd_type, cmd_timeout);
+
+        if (ret != RTW_PHL_STATUS_SUCCESS) {
+                RTW_PRINT_SEL(m, "Failed to get TSF (ret=%d)\n", ret);
+                return 0;
+        }
+
+	cur_tsf = (u64)tsf.tsf_h << 32;
+	cur_tsf = cur_tsf | tsf.tsf_l;
+
+        if (cur_tsf == 0) {
+                RTW_PRINT_SEL(m, "Warning: TSF = 0. Possibly not initialized or failed to read.\n");
+		return 0;
+	}
+
+	RTW_PRINT_SEL(m, "cur_tsf: %llu\n", cur_tsf);
+
+	return 0;
+}
+
+int proc_get_cur_tsf_cmd_enqueue(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	const char *str = padapter->cur_tsf_cmd_enqueue ? "Enqueue" : "No Enqueue";
+
+	RTW_PRINT_SEL(m, "cur_tsf_cmd_enqueue: %s (%d)\n", str, padapter->cur_tsf_cmd_enqueue);
+	return 0;
+}
+
+ssize_t proc_set_cur_tsf_cmd_enqueue(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	int val;
+	int num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num = sscanf(tmp, "%d", &val);
+		if (num == 1 && (val == 0 || val == 1)) {
+			padapter->cur_tsf_cmd_enqueue = (val != 0);
+			RTW_INFO("cur_tsf_cmd_enqueue = %d\n", padapter->cur_tsf_cmd_enqueue);
+		} else {
+			RTW_INFO("Invalid value: must be 0 or 1\n");
+		}
+	}
+
+	return count;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_dfs.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_dfs.c
new file mode 100644
index 000000000000..cfafa65f482a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_dfs.c
@@ -0,0 +1,1652 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_DFS_C_
+
+#include <drv_types.h>
+
+bool rtw_chset_is_dfs_range(const struct rtw_chset *chset, u32 hi, u32 lo)
+{
+	enum band_type band = rtw_freq2band(hi);
+	u8 hi_ch = rtw_freq2ch(hi);
+	u8 lo_ch = rtw_freq2ch(lo);
+	int i;
+
+	for (i = 0; i < chset->chs_len; i++){
+		if (!(chset->chs[i].flags & RTW_CHF_DFS))
+			continue;
+		if (band == chset->chs[i].band && hi_ch > chset->chs[i].ChannelNum && lo_ch < chset->chs[i].ChannelNum)
+			return 1;
+	}
+
+	return 0;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_dfs_ch(const struct rtw_chset *chset, u8 ch)
+{
+	int i;
+
+	for (i = 0; i < chset->chs_len; i++){
+		if (chset->chs[i].ChannelNum == ch)
+			return chset->chs[i].flags & RTW_CHF_DFS ? 1 : 0;
+	}
+
+	return 0;
+}
+
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_dfs_chbw(const struct rtw_chset *chset, u8 ch, u8 bw, u8 offset)
+{
+	u32 hi, lo;
+
+	if (!rtw_chbw_to_freq_range(ch, bw, offset, &hi, &lo))
+		return 0;
+
+	return rtw_chset_is_dfs_range(chset, hi, lo);
+}
+#endif
+
+bool rtw_chset_is_dfs_bch(const struct rtw_chset *chset, enum band_type band, u8 ch)
+{
+	int i;
+
+	for (i = 0; i < chset->chs_len; i++){
+		if (chset->chs[i].band == band && chset->chs[i].ChannelNum == ch)
+			return chset->chs[i].flags & RTW_CHF_DFS ? 1 : 0;
+	}
+
+	return 0;
+}
+
+bool rtw_chset_is_dfs_bchbw(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	u32 hi, lo;
+
+	if (!rtw_bchbw_to_freq_range(band, ch, bw, offset, &hi, &lo))
+		return 0;
+
+	return rtw_chset_is_dfs_range(chset, hi, lo);
+}
+
+#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+static bool rtw_chset_get_dfs_frange(const struct rtw_chset *chset
+	, enum band_type band, u8 ch, u8 bw, u8 offset, u32 *dfs_freq_hi, u32 *dfs_freq_lo)
+{
+	u32 hi, lo;
+	u32 freq;
+	int i;
+
+	*dfs_freq_hi = *dfs_freq_lo = 0;
+
+	if (!rtw_bchbw_to_freq_range(band, ch, bw, offset, &hi, &lo)) {
+		rtw_warn_on(1);
+		return false;
+	}
+
+	for (i = 0; i < chset->chs_len; i++){
+		if (!(chset->chs[i].flags & RTW_CHF_DFS))
+			continue;
+		freq = rtw_bch2freq(chset->chs[i].band, chset->chs[i].ChannelNum);
+		if (!freq) {
+			rtw_warn_on(1);
+			continue;
+		}
+		if (!rtw_is_range_a_in_b(freq + 10, freq - 10, hi, lo))
+			continue;
+		rtw_range_merge(*dfs_freq_hi, *dfs_freq_lo, freq + 10, freq - 10);
+	}
+
+	return *dfs_freq_hi != 0;
+}
+#endif /* CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ */
+
+enum rtw_dfs_regd rtw_rfctl_get_dfs_domain(struct rf_ctl_t *rfctl)
+{
+#ifdef CONFIG_DFS_MASTER
+	return rfctl->dfs_region_domain;
+#else
+	return RTW_DFS_REGD_NONE;
+#endif
+}
+
+bool rtw_rfctl_radar_detect_supported(struct rf_ctl_t *rfctl)
+{
+#ifdef CONFIG_DFS_MASTER
+	enum rtw_dfs_regd domain = rtw_rfctl_get_dfs_domain(rfctl);
+
+	return !RTW_DFS_REGD_IS_UNKNOWN(domain)
+		&& rtw_dfs_hal_region_supported(rfctl_to_dvobj(rfctl), domain);
+#else
+	return false;
+#endif
+}
+
+#ifdef CONFIG_DFS_MASTER
+bool rtw_rfctl_overlap_radar_detect_ch(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	bool ret = false;
+	u8 rd_hwband;
+	u32 hi = 0, lo = 0;
+	u32 r_hi = 0, r_lo = 0;
+
+	if (band != BAND_ON_5G || !rfctl->radar_detect_enabled)
+		goto exit;
+
+	if (rtw_bchbw_to_freq_range(band, ch, bw, offset, &hi, &lo) == _FALSE) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	rd_hwband = rfctl->radar_detect_hwband;
+	if (rtw_is_range_overlap(hi, lo, rfctl->radar_detect_freq_hi[rd_hwband], rfctl->radar_detect_freq_lo[rd_hwband]))
+		ret = true;
+
+exit:
+	return ret;
+}
+
+static bool rtw_rfctl_hwband_is_tx_blocked_by_ch_waiting(struct rf_ctl_t *rfctl, enum phl_band_idx hwband)
+{
+	if (hwband >= HW_BAND_MAX)
+		return false;
+
+	if (IS_CH_WAITING(rfctl)) {
+		enum band_type band;
+		u8 ch, bw, offset;
+
+		if (rtw_get_oper_bchbw_by_hwband(rfctl_to_dvobj(rfctl), hwband
+				, &band, &ch, &bw, &offset) != _SUCCESS)
+			return false;
+
+		return rtw_rfctl_overlap_radar_detect_ch(rfctl
+			, band, ch, bw, offset);
+	}
+	return false;
+}
+
+bool adapter_is_tx_blocked_by_ch_waiting(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(adapter);
+
+	if (!alink)
+		return false;
+
+	return rtw_rfctl_hwband_is_tx_blocked_by_ch_waiting(rfctl, ALINK_GET_HWBAND(alink));
+}
+
+bool alink_is_tx_blocked_by_ch_waiting(struct _ADAPTER_LINK *alink)
+{
+	_adapter *adapter = alink->adapter;
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	return rtw_rfctl_hwband_is_tx_blocked_by_ch_waiting(rfctl, ALINK_GET_HWBAND(alink));
+}
+
+/**
+ * rtw_chset_update_cac_state_of_chs - update cac_done according to the given @band, @chs, @chs_len into @chset
+ * @chset: the given channel set
+ * @band: band of setting range
+ * @chs: channel array of setting range
+ * @chs_len: channel array length
+ * @done: if CAC done
+ */
+static void rtw_chset_update_cac_state_of_chs(struct rtw_chset *chset
+	, u8 chs[], u8 chs_len, bool done)
+{
+	int i, j;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].band != BAND_ON_5G)
+			continue;
+		for (j = 0; j < chs_len; j++) {
+			if (chs[j] == 0)
+				continue;
+			if (chset->chs[i].ChannelNum == chs[j])
+				chset->chs[i].cac_done = done;
+		}
+	}
+}
+
+/**
+ * rtw_chset_update_cac_state_by_freq - update cac_done according to the given @hi, @lo frequency into @chset
+ * @chset: the given channel set
+ * @hi: high frequency of setting range
+ * @lo: low frequency of setting range
+ * @done: if CAC done
+ */
+static void rtw_chset_update_cac_state_by_freq(struct rtw_chset *chset, u32 hi, u32 lo, bool done)
+{
+	enum band_type band;
+	int i;
+
+	band = rtw_freq2band(hi);
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].band != band)
+			continue;
+		if (!rtw_bch2freq(band, chset->chs[i].ChannelNum)) {
+			rtw_warn_on(1);
+			continue;
+		}
+
+		if (lo <= rtw_bch2freq(band, chset->chs[i].ChannelNum)
+			&& rtw_bch2freq(band, chset->chs[i].ChannelNum) <= hi)
+			chset->chs[i].cac_done = done;
+	}
+}
+
+static void rtw_chset_update_cac_state_by_cch(struct rtw_chset *chset, u8 cch, u8 bw, bool done)
+{
+	u32 hi, lo;
+
+	if (rtw_bcchbw_to_freq_range(BAND_ON_5G, cch, bw, &hi, &lo))
+		rtw_chset_update_cac_state_by_freq(chset, hi, lo, done);
+}
+
+/**
+ * rtw_chset_is_frange_require_cac - check cac_done according to the given @hi, @lo frequency into @chset
+ * @chset: the given channel set
+ * @hi: high frequency of checking range
+ * @lo: low frequency of checking range
+ * return: if all channel inside range require CAC (DFS channel and not CAC done)
+ */
+static bool rtw_chset_is_frange_require_cac(struct rtw_chset *chset, u32 hi, u32 lo)
+{
+	enum band_type band;
+	int i;
+
+	band = rtw_freq2band(hi);
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].band != band)
+			continue;
+		if (!(chset->chs[i].flags & RTW_CHF_DFS))
+			continue;
+		if (!rtw_bch2freq(band, chset->chs[i].ChannelNum)) {
+			rtw_warn_on(1);
+			continue;
+		}
+		if (lo <= rtw_bch2freq(band, chset->chs[i].ChannelNum)
+			&& rtw_bch2freq(band, chset->chs[i].ChannelNum) <= hi
+			&& !chset->chs[i].cac_done)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rtw_chset_is_cch_require_cac(struct rtw_chset *chset, u8 cch, u8 bw)
+{
+	u32 hi, lo;
+
+	if (rtw_bcchbw_to_freq_range(BAND_ON_5G, cch, bw, &hi, &lo))
+		return rtw_chset_is_frange_require_cac(chset, hi, lo);
+	return false;
+}
+
+bool rtw_chset_is_bchbw_non_ocp(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	bool ret = _FALSE;
+	u32 hi = 0, lo = 0;
+	int i;
+
+	if (rtw_bchbw_to_freq_range(band, ch, bw, offset, &hi, &lo) == _FALSE)
+		goto exit;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].band != band)
+			continue;
+		if (!rtw_bch2freq(band, chset->chs[i].ChannelNum)) {
+			rtw_warn_on(1);
+			continue;
+		}
+
+		if (!CH_IS_NON_OCP(&chset->chs[i]))
+			continue;
+
+		if (lo <= rtw_bch2freq(band, chset->chs[i].ChannelNum)
+			&& rtw_bch2freq(band, chset->chs[i].ChannelNum) <= hi
+		) {
+			ret = _TRUE;
+			break;
+		}
+	}
+
+exit:
+	return ret;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_chbw_non_ocp(const struct rtw_chset *chset, u8 ch, u8 bw, u8 offset)
+{
+	return rtw_chset_is_bchbw_non_ocp(chset, rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch, bw, offset);
+}
+
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_ch_non_ocp(const struct rtw_chset *chset, u8 ch)
+{
+	return rtw_chset_is_chbw_non_ocp(chset, ch, CHANNEL_WIDTH_20, CHAN_OFFSET_NO_EXT);
+}
+#endif
+
+bool rtw_chset_is_bch_non_ocp(const struct rtw_chset *chset, enum band_type band, u8 ch)
+{
+	return rtw_chset_is_bchbw_non_ocp(chset, band, ch, CHANNEL_WIDTH_20, CHAN_OFFSET_NO_EXT);
+}
+
+static u32 rtw_chset_get_bch_non_ocp_ms(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	int ms = 0;
+	systime current_time;
+	u32 hi = 0, lo = 0;
+	int i;
+
+	if (rtw_bchbw_to_freq_range(band, ch, bw, offset, &hi, &lo) == _FALSE)
+		goto exit;
+
+	current_time = rtw_get_current_time();
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].band != band)
+			continue;
+		if (!rtw_bch2freq(band, chset->chs[i].ChannelNum)) {
+			rtw_warn_on(1);
+			continue;
+		}
+
+		if (!CH_IS_NON_OCP(&chset->chs[i]))
+			continue;
+
+		if (lo <= rtw_bch2freq(band, chset->chs[i].ChannelNum)
+			&& rtw_bch2freq(band, chset->chs[i].ChannelNum) <= hi
+		) {
+			if (rtw_systime_to_ms(chset->chs[i].non_ocp_end_time - current_time) > ms)
+				ms = rtw_systime_to_ms(chset->chs[i].non_ocp_end_time - current_time);
+		}
+	}
+
+exit:
+	return ms;
+}
+
+/**
+ * rtw_chset_update_non_ocp_ms_by_freq - update non_ocp_end_time according to the given @hi, @lo frequency into @ch_set
+ * @chset: the given channel set
+ * @hi: high frequency on which radar is detected
+ * @lo: low frequency on which radar is detected
+ * @ms: ms to add from now to update non_ocp_end_time, ms < 0 means use NON_OCP_TIME_MS
+ */
+static bool rtw_chset_update_non_ocp_ms_by_freq(struct rtw_chset *chset, u32 hi, u32 lo, int ms)
+{
+	enum band_type band;
+	int i;
+	bool updated = 0;
+
+	band = rtw_freq2band(hi);
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].band != band)
+			continue;
+		if (!rtw_bch2freq(band, chset->chs[i].ChannelNum)) {
+			rtw_warn_on(1);
+			continue;
+		}
+
+		if (lo <= rtw_bch2freq(band, chset->chs[i].ChannelNum)
+			&& rtw_bch2freq(band, chset->chs[i].ChannelNum) <= hi
+		) {
+			if (ms >= 0)
+				chset->chs[i].non_ocp_end_time = rtw_get_current_time() + rtw_ms_to_systime(ms);
+			else
+				chset->chs[i].non_ocp_end_time = rtw_get_current_time() + rtw_ms_to_systime(NON_OCP_TIME_MS);
+			if (chset->chs[i].non_ocp_end_time == RTW_NON_OCP_STOPPED)
+				chset->chs[i].non_ocp_end_time++;
+			updated = 1;
+		}
+	}
+
+	return updated;
+}
+
+static bool rtw_chset_update_non_ocp_ms_by_band_cch(struct rtw_chset *chset, enum band_type band, u8 cch, u8 bw, int ms)
+{
+	u32 hi, lo;
+
+	if (rtw_bcchbw_to_freq_range(band, cch, bw, &hi, &lo) == _FALSE)
+		return _FALSE;
+
+	return rtw_chset_update_non_ocp_ms_by_freq(chset, hi, lo, ms);
+}
+
+static bool rtw_chset_update_non_ocp_ms_by_band(struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset, int ms)
+{
+	u32 hi, lo;
+
+	if (rtw_bchbw_to_freq_range(band, ch, bw, offset, &hi, &lo) == _FALSE)
+		return _FALSE;
+
+	return rtw_chset_update_non_ocp_ms_by_freq(chset, hi, lo, ms);
+}
+
+static bool rtw_rfctl_chset_chk_non_ocp_finish_for_bchbw(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	struct rtw_chset *chset = &rfctl->chset;
+	RT_CHANNEL_INFO *chinfo;
+	u8 cch;
+	u8 *op_chs;
+	u8 op_ch_num;
+	int i;
+	bool ret = 0;
+
+	cch = rtw_get_center_ch_by_band(band, ch, bw, offset);
+
+	if (!rtw_get_op_chs_by_bcch_bw(band, cch, bw, &op_chs, &op_ch_num))
+		goto exit;
+
+	for (i = 0; i < op_ch_num; i++) {
+		if (0)
+			RTW_INFO("%u,%u,%u,%u - cch:%u, bw:%u, op_ch:%u\n", band, ch, bw, offset, cch, bw, *(op_chs + i));
+		chinfo = rtw_chset_get_chinfo_by_bch(chset, band, *(op_chs + i), true);
+		if (!chinfo)
+			break;
+		if (CH_IS_NON_OCP_STOPPED(chinfo) || CH_IS_NON_OCP(chinfo))
+			break;
+	}
+
+	if (op_ch_num != 0 && i == op_ch_num) {
+		ret = 1;
+		/* set to RTW_NON_OCP_STOPPED */
+		for (i = 0; i < op_ch_num; i++) {
+			chinfo = rtw_chset_get_chinfo_by_bch(chset, band, *(op_chs + i), true);
+			chinfo->non_ocp_end_time = RTW_NON_OCP_STOPPED;
+		}
+		for (i = HW_BAND_0; i < HW_BAND_MAX; i++) /* single chset shared by all hwband */
+			rtw_os_indicate_nop_finished(rfctl, i, band, cch, bw);
+	}
+
+exit:
+	return ret;
+}
+
+/* called by watchdog to set RTW_NON_OCP_STOPPED and generate NON_OCP finish event */
+void rtw_rfctl_chset_chk_non_ocp_finish(struct rf_ctl_t *rfctl)
+{
+	struct rtw_chset *chset = &rfctl->chset;
+	enum band_type band;
+	u8 ch, bw, offset;
+	int i;
+
+	bw = CHANNEL_WIDTH_160;
+	while (1) {
+		for (i = 0; i < chset->chs_len; i++) {
+			band = chset->chs[i].band;
+			ch = chset->chs[i].ChannelNum;
+			if (CH_IS_NON_OCP_STOPPED(&chset->chs[i]))
+				continue;
+			if (!rtw_get_offset_by_bchbw(band, ch, bw, &offset))
+				continue;
+
+			rtw_rfctl_chset_chk_non_ocp_finish_for_bchbw(rfctl, band, ch, bw, offset);
+		}
+		if (bw-- == CHANNEL_WIDTH_20)
+			break;
+	}
+}
+
+/*
+* For ioctl debug usage, will not change current CAC status
+* Don't use on current operating channels
+*/
+void rtw_rfctl_force_update_non_ocp_ms(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset, int ms)
+{
+	struct rtw_chset *chset = &rfctl->chset;
+	bool updated;
+
+	if (bw == CHANNEL_WIDTH_20)
+		updated = rtw_chset_update_non_ocp_ms_by_band(chset
+			, band, ch, bw, CHAN_OFFSET_NO_EXT, ms);
+	else
+		updated = rtw_chset_update_non_ocp_ms_by_band(chset
+			, band, ch, bw, offset, ms);
+
+	if (updated) {
+		u8 i;
+		u8 cch = rtw_get_center_ch_by_band(band, ch, bw, offset);
+
+		for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+			rtw_os_indicate_nop_started(rfctl, i, band, cch, bw, false);
+	}
+}
+
+static u32 _rtw_get_ch_waiting_ms(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset, bool in_self_rd_range, u32 *r_non_ocp_ms, u32 *r_cac_ms)
+{
+	struct rtw_chset *chset = &rfctl->chset;
+	u32 non_ocp_ms;
+	u32 cac_ms;
+
+	if (rtw_chset_is_bchbw_non_ocp(chset, band, ch, bw, offset))
+		non_ocp_ms = rtw_chset_get_bch_non_ocp_ms(chset, band, ch, bw, offset);
+	else
+		non_ocp_ms = 0;
+
+	if (!rtw_chset_is_dfs_bchbw(chset, band, ch, bw, offset))
+		cac_ms = 0;
+	else if (in_self_rd_range && !non_ocp_ms) {
+		if (IS_CH_WAITING(rfctl))
+			cac_ms = rtw_systime_to_ms(rfctl->cac_end_time - rtw_get_current_time());
+		else
+			cac_ms = 0;
+	} else if (rtw_is_long_cac_bch(band, ch, bw, offset, rtw_rfctl_get_dfs_domain(rfctl)))
+		cac_ms = CAC_TIME_CE_MS;
+	else
+		cac_ms = CAC_TIME_MS;
+
+	if (r_non_ocp_ms)
+		*r_non_ocp_ms = non_ocp_ms;
+	if (r_cac_ms)
+		*r_cac_ms = cac_ms;
+
+	return non_ocp_ms + cac_ms;
+}
+
+u32 rtw_get_ch_waiting_ms(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset, u32 *r_non_ocp_ms, u32 *r_cac_ms)
+{
+	bool in_self_rd_range = false;
+
+	if (rfctl->radar_detect_enabled) {
+		u32 hi, lo;
+		u8 rd_hwband = rfctl->radar_detect_hwband;
+
+		if (rtw_chset_get_dfs_frange(&rfctl->chset, band, ch, bw, offset, &hi, &lo)
+			&& rtw_is_range_a_in_b(hi, lo, rfctl->radar_detect_freq_hi[rd_hwband], rfctl->radar_detect_freq_lo[rd_hwband]))
+			in_self_rd_range = 1;
+	}
+
+	return _rtw_get_ch_waiting_ms(rfctl, band, ch, bw, offset, in_self_rd_range, r_non_ocp_ms, r_cac_ms);
+}
+
+static void rtw_reset_cac(struct rf_ctl_t *rfctl, u8 op_ch, u8 op_bw, u8 op_offset)
+{
+#define RD_CONFIG_INT_MS 2000
+	u32 non_ocp_ms;
+	u32 cac_ms;
+	u32 rd_ready_ms = 0;
+
+	_rtw_get_ch_waiting_ms(rfctl, BAND_ON_5G, op_ch, op_bw, op_offset, false
+		, &non_ocp_ms, &cac_ms);
+	if (non_ocp_ms < RD_CONFIG_INT_MS)
+		rd_ready_ms = RD_CONFIG_INT_MS - non_ocp_ms;
+
+	rfctl->non_ocp_finished = non_ocp_ms ? false : true;
+	rfctl->cac_start_time = rtw_get_current_time() + rtw_ms_to_systime(non_ocp_ms);
+	rfctl->cac_end_time = rfctl->cac_start_time + rtw_ms_to_systime(rd_ready_ms + cac_ms);
+
+	/* skip special value */
+	if (rfctl->cac_start_time == RTW_CAC_STOPPED) {
+		rfctl->cac_start_time++;
+		rfctl->cac_end_time++;
+	}
+	if (rfctl->cac_end_time == RTW_CAC_STOPPED)
+		rfctl->cac_end_time++;
+}
+
+u32 rtw_force_stop_cac(struct rf_ctl_t *rfctl, u32 timeout_ms)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	systime start;
+	u32 pass_ms;
+
+	start = rtw_get_current_time();
+
+	rfctl->cac_force_stop = 1;
+
+	while (rtw_get_passing_time_ms(start) <= timeout_ms
+		&& IS_UNDER_CAC(rfctl)
+	) {
+		if (dev_is_surprise_removed(dvobj) || dev_is_drv_stopped(dvobj))
+			break;
+		rtw_msleep_os(20);
+	}
+
+	if (IS_UNDER_CAC(rfctl)) {
+		if (!dev_is_surprise_removed(dvobj) && !dev_is_drv_stopped(dvobj))
+			RTW_INFO("%s waiting for cac stop timeout!\n", __func__);
+	}
+
+	rfctl->cac_force_stop = 0;
+
+	pass_ms = rtw_get_passing_time_ms(start);
+
+	return pass_ms;
+}
+
+static void rtw_dfs_ch_switch_hdl(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	_adapter *m_iface = rtw_mi_get_ap_mesh_iface_by_hwband(dvobj, band_idx);
+	u8 ifbmp_m;
+	u8 ifbmp_s;
+	u8 i;
+
+	if (!m_iface) {
+		RTW_WARN(FUNC_HWBAND_FMT" can't get ap/mesh iface\n", FUNC_HWBAND_ARG(band_idx));
+		rtw_warn_on(1);
+		return;
+	}
+
+	ifbmp_m = rtw_mi_get_ap_mesh_ifbmp_by_hwband(dvobj, band_idx);
+	ifbmp_s = rtw_mi_get_ld_sta_ifbmp_by_hwband(dvobj, band_idx);
+
+	rtw_dfs_hal_csa_mg_tx_pause(dvobj, band_idx, true);
+
+#ifdef CONFIG_AP_MODE
+	if (ifbmp_m) {
+		RTW_INFO(FUNC_HWBAND_FMT" ch sel by AP/MESH ifaces\n", FUNC_HWBAND_ARG(band_idx));
+		/* trigger channel selection with consideraton of asoc STA ifaces */
+		rtw_change_bss_bchbw_cmd(m_iface, RTW_CMDF_DIRECTLY
+			, ifbmp_m, ifbmp_s, REQ_BAND_NONE, REQ_CH_NONE, REQ_BW_ORI, REQ_OFFSET_NONE);
+	}
+#endif
+
+	rtw_dfs_hal_csa_mg_tx_pause(dvobj, band_idx, false);
+
+	rtw_mi_os_xmit_schedule(m_iface);
+}
+
+u8 rtw_dfs_rd_hdl(struct dvobj_priv *dvobj, enum phl_band_idx hwband, u8 radar_cch, enum channel_width radar_bw)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	bool detected;
+
+	if (!rfctl->radar_detect_enabled || rfctl->radar_detect_hwband != hwband)
+		goto exit;
+
+	if (IS_CH_WAITING(rfctl) && !IS_UNDER_CAC(rfctl)) {
+		/* non_ocp, ignore radar detect */
+		goto cac_status_chk;
+	} else if (IS_UNDER_CAC(rfctl) && !rfctl->non_ocp_finished) {
+		/* from NON_OCP to CAC, indicate CAC started event */
+		rtw_os_indicate_cac_started(rfctl, hwband, 0xFF
+			, rfctl->radar_detect_cch[hwband], rfctl->radar_detect_bw[hwband]);
+		rfctl->non_ocp_finished = true;
+	}
+
+	detected = (radar_cch > 0 && radar_bw != CHANNEL_WIDTH_MAX) ? true : false;
+	if (!rfctl->dbg_dfs_fake_radar_detect_cnt && !detected)
+		goto cac_status_chk;
+
+	if (!rfctl->dbg_dfs_fake_radar_detect_cnt
+		&& rfctl->dbg_dfs_radar_detect_trigger_non
+	) {
+		/* radar detect debug mode, trigger no mlme flow */
+		RTW_INFO(FUNC_HWBAND_FMT" radar detected on test mode, trigger no mlme flow\n", FUNC_HWBAND_ARG(hwband));
+		goto cac_status_chk;
+	}
+
+	if (rfctl->dbg_dfs_fake_radar_detect_cnt != 0) {
+		RTW_INFO(FUNC_HWBAND_FMT" fake radar detected, cnt:%d\n", FUNC_HWBAND_ARG(hwband)
+			, rfctl->dbg_dfs_fake_radar_detect_cnt);
+		rfctl->dbg_dfs_fake_radar_detect_cnt--;
+
+		/* TODO: fack radar detected with specific range */
+		radar_cch = rfctl->radar_detect_cch[hwband];
+		radar_bw = rfctl->radar_detect_bw[hwband];
+
+	} else
+		RTW_INFO(FUNC_HWBAND_FMT" radar detected\n", FUNC_HWBAND_ARG(hwband));
+
+	rfctl->radar_detected = 1;
+
+	rtw_chset_update_non_ocp_ms_by_band_cch(&rfctl->chset, BAND_ON_5G
+		, radar_cch, radar_bw, -1);
+
+	rtw_os_indicate_radar_detected(rfctl, hwband, radar_cch, radar_bw);
+	rtw_os_indicate_nop_started(rfctl, hwband, BAND_ON_5G, radar_cch, radar_bw, true);
+
+	rtw_dfs_ch_switch_hdl(dvobj, hwband);
+
+	if (rfctl->radar_detect_enabled)
+		goto set_timer;
+	goto exit;
+
+cac_status_chk:
+
+	if (!IS_CAC_STOPPED(rfctl)
+		&& ((IS_UNDER_CAC(rfctl) && rfctl->cac_force_stop)
+			|| !IS_CH_WAITING(rfctl)
+			)
+	) {
+		_adapter *m_iface;
+
+		rfctl->cac_start_time = rfctl->cac_end_time = RTW_CAC_STOPPED;
+		rtw_chset_update_cac_state_by_cch(&rfctl->chset
+			, rfctl->radar_detect_cch[hwband], rfctl->radar_detect_bw[hwband], true);
+		rtw_dfs_hal_set_cac_status(dvobj, hwband, false);
+
+		m_iface = rtw_mi_get_ap_mesh_iface_by_hwband(dvobj, hwband);
+		if (m_iface) {
+			if (!rtw_mi_check_fwstate_by_hwband(dvobj, hwband, WIFI_UNDER_LINKING|WIFI_UNDER_SURVEY)) {
+				struct _ADAPTER_LINK *alink = rtw_get_adapter_link_by_hwband(m_iface, hwband);
+				enum band_type u_band;
+				u8 u_ch, u_bw, u_offset;
+
+				if (rtw_mi_get_bch_setting_union_by_hwband(dvobj, hwband, &u_band, &u_ch, &u_bw, &u_offset))
+					set_bch_bwmode(m_iface, alink, u_band, u_ch, u_offset, u_bw, RFK_TYPE_FORCE_DO);
+				else
+					rtw_warn_on(1);
+			}
+		} else
+			RTW_ERR(FUNC_HWBAND_FMT" can't get ap/mesh iface\n", FUNC_HWBAND_ARG(hwband));
+
+		rtw_os_indicate_cac_finished(rfctl, hwband, 0xFF
+			, rfctl->radar_detect_cch[hwband], rfctl->radar_detect_bw[hwband]);
+	}
+
+set_timer:
+	_set_timer(&rfctl->radar_detect_timer
+		, rtw_dfs_hal_radar_detect_polling_int_ms(dvobj));
+
+exit:
+	return H2C_SUCCESS;
+}
+
+static u8 rtw_dfs_rd_cmd(struct dvobj_priv *dvobj, enum phl_band_idx hwband)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	struct cmd_priv *cmdpriv = &dvobj->cmdpriv;
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+	u8 res = _FAIL;
+
+	cmdobj = rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmdobj == NULL)
+		goto exit;
+	cmdobj->padapter = adapter;
+
+	parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (parm == NULL) {
+		rtw_mfree(cmdobj, sizeof(struct cmd_obj));
+		goto exit;
+	}
+
+	parm->ec_id = DFS_RADAR_DETECT_WK_CID;
+	parm->type = hwband;
+	parm->size = 0;
+	parm->pbuf = NULL;
+
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+	CMD_OBJ_SET_HWBAND(cmdobj, hwband);
+	res = rtw_enqueue_cmd(cmdpriv, cmdobj);
+
+exit:
+	return res;
+}
+
+static void rtw_dfs_rd_timer_hdl(void *ctx)
+{
+	struct rf_ctl_t *rfctl = (struct rf_ctl_t *)ctx;
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+
+	rtw_dfs_rd_cmd(dvobj, rfctl->radar_detect_hwband);
+}
+#endif /* CONFIG_DFS_MASTER */
+
+#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+static void rtw_dfs_update_rd_range_status(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 op_ch, enum channel_width op_bw, enum chan_offset op_offset, u32 rd_freq_hi, u32 rd_freq_lo, bool by_others)
+{
+#define RD_RANGE_OPCH_NUM_MAX (160 / 20)
+#ifdef CONFIG_DFS_MASTER
+	bool new_self_rd_ch = false;
+	bool cover_non_ocp = op_ch && rtw_chset_is_bchbw_non_ocp(&rfctl->chset, BAND_ON_5G, op_ch, op_bw, op_offset);
+#endif
+	u8 cch = 0;
+	enum channel_width bw = CHANNEL_WIDTH_MAX;
+
+#ifndef CONFIG_DFS_MASTER
+	if (rd_freq_hi && !by_others) {
+		rtw_warn_on(1);
+		rd_freq_hi = 0;
+	}
+#endif
+
+	if (rd_freq_hi) {
+		cch = rtw_freq2ch((rd_freq_hi + rd_freq_lo) / 2);
+		bw = rtw_frange_to_bw(rd_freq_hi, rd_freq_lo);
+		rtw_warn_on(cch == 0);
+		rtw_warn_on(bw == CHANNEL_WIDTH_MAX);
+	}
+
+#ifdef CONFIG_DFS_MASTER
+	if (rfctl->radar_detect_cch[band_idx] != cch
+		|| rfctl->radar_detect_bw[band_idx] != bw
+	) {
+		/* coverage range changed */
+		u8 *op_chs;
+		u8 op_ch_num;
+		u8 rd_cch;
+		enum channel_width rd_bw;
+		u8 i, j, k;
+
+		if (rfctl->radar_detect_cch[band_idx]) {
+			u8 clr_chs[RD_RANGE_OPCH_NUM_MAX];
+
+			/* init clr_chs from original coverage of this hwband */
+			if (!rtw_get_op_chs_by_bcch_bw(BAND_ON_5G
+				, rfctl->radar_detect_cch[band_idx], rfctl->radar_detect_bw[band_idx], &op_chs, &op_ch_num)
+			) {
+				RTW_WARN(FUNC_HWBAND_FMT" rtw_get_op_chs_by_bcch_bw() fail on cch:%u bw:%u\n", FUNC_HWBAND_ARG(band_idx)
+					, rfctl->radar_detect_cch[band_idx], rfctl->radar_detect_bw[band_idx]);
+				rtw_warn_on(1);
+				goto handle_add_chs;
+			}
+			_rtw_memset(clr_chs, 0, RD_RANGE_OPCH_NUM_MAX);
+			for (i = 0; i < op_ch_num; i++)
+				clr_chs[i] = op_chs[i];
+
+			/* remove op chs of new coverages from clr_chs */
+			for (i = HW_BAND_0; i < HW_BAND_MAX; i++) {
+				if (i == band_idx && cch) {
+					rd_cch = cch;
+					rd_bw = bw;
+				} else if (i != band_idx && rfctl->radar_detect_cch[i]) {
+					rd_cch = rfctl->radar_detect_cch[i];
+					rd_bw = rfctl->radar_detect_bw[i];
+				} else
+					continue;
+
+				if (!rtw_get_op_chs_by_bcch_bw(BAND_ON_5G, rd_cch, rd_bw, &op_chs, &op_ch_num)) {
+					RTW_WARN(FUNC_HWBAND_FMT" rtw_get_op_chs_by_bcch_bw() fail on cch:%u bw:%u\n"
+						, FUNC_HWBAND_ARG(i), rd_cch, rd_bw);
+					rtw_warn_on(1);
+					continue;
+				}
+				for (j = 0; j < RD_RANGE_OPCH_NUM_MAX; j++)
+					for (k = 0; k < op_ch_num; k++)
+						if (clr_chs[j] == op_chs[k])
+							clr_chs[j] = 0;
+			}
+
+			/* clear CAC_DONE of clr_chs */
+			rtw_chset_update_cac_state_of_chs(&rfctl->chset, clr_chs, RD_RANGE_OPCH_NUM_MAX, false);
+		}
+
+handle_add_chs:
+		if (cch && !by_others && !cover_non_ocp) {
+			u8 add_chs[RD_RANGE_OPCH_NUM_MAX];
+
+			/* init add_chs from new coverage of this hwband */
+			if (!rtw_get_op_chs_by_bcch_bw(BAND_ON_5G, cch, bw, &op_chs, &op_ch_num)) {
+				RTW_WARN(FUNC_HWBAND_FMT" rtw_get_op_chs_by_bcch_bw() fail on cch:%u bw:%u\n"
+					, FUNC_HWBAND_ARG(band_idx), cch, bw);
+				rtw_warn_on(1);
+				goto update_ch_cac;
+			}
+			_rtw_memset(add_chs, 0, RD_RANGE_OPCH_NUM_MAX);
+			for (i = 0; i < op_ch_num; i++)
+				add_chs[i] = op_chs[i];
+
+			/* remove op chs of original coverages from add_chs */
+			for (i = HW_BAND_0; i < HW_BAND_MAX; i++) {
+				if (rfctl->radar_detect_cch[i]) {
+					rd_cch = rfctl->radar_detect_cch[i];
+					rd_bw = rfctl->radar_detect_bw[i];
+				} else
+					continue;
+
+				if (!rtw_get_op_chs_by_bcch_bw(BAND_ON_5G, rd_cch, rd_bw, &op_chs, &op_ch_num)) {
+					RTW_WARN(FUNC_HWBAND_FMT" rtw_get_op_chs_by_bcch_bw() fail on cch:%u bw:%u\n"
+						, FUNC_HWBAND_ARG(i), rd_cch, rd_bw);
+					rtw_warn_on(1);
+					continue;
+				}
+				for (j = 0; j < RD_RANGE_OPCH_NUM_MAX; j++)
+					for (k = 0; k < op_ch_num; k++)
+						if (add_chs[j] == op_chs[k])
+							add_chs[j] = 0;
+			}
+
+			for (i = 0; i < RD_RANGE_OPCH_NUM_MAX; i++)
+				if (add_chs[i])
+					break;
+			if (i < RD_RANGE_OPCH_NUM_MAX)
+				new_self_rd_ch = true;
+		}
+	}
+
+update_ch_cac:
+	if (cch && by_others) {
+		/* new coverage of this hwband is detected by others, set CAC_DONE directly */
+		rtw_chset_update_cac_state_by_cch(&rfctl->chset, cch, bw, true);
+	}
+
+
+	/* hwband CAC status update */
+	if (cover_non_ocp) {
+		RTW_INFO(FUNC_HWBAND_FMT" cover NON_OCP channel\n", FUNC_HWBAND_ARG(band_idx));
+		if (IS_UNDER_CAC(rfctl)) {
+			RTW_INFO(FUNC_HWBAND_FMT" CAC abort\n", FUNC_HWBAND_ARG(band_idx));
+			rtw_os_indicate_cac_aborted(rfctl, band_idx, 0xFF
+				, rfctl->radar_detect_cch[band_idx], rfctl->radar_detect_bw[band_idx]);
+		}
+		rtw_reset_cac(rfctl, op_ch, op_bw, op_offset);
+
+	} else if (new_self_rd_ch) {
+		/* if having new rd chs and not detected by others, reset CAC of detecting hwband */
+		RTW_INFO(FUNC_HWBAND_FMT" have new detect range\n", FUNC_HWBAND_ARG(band_idx));
+		rtw_reset_cac(rfctl, op_ch, op_bw, op_offset);
+		rtw_os_indicate_cac_started(rfctl, band_idx, 0xFF, cch, bw);
+
+	} else {
+		u8 rd_cch;
+		enum channel_width rd_bw;
+		u8 i;
+		bool require_cac[HW_BAND_MAX];
+
+		if (IS_UNDER_CAC(rfctl) && cch == 0 && rfctl->radar_detect_cch[band_idx]
+			&& !rfctl->radar_detect_by_others[band_idx]
+		) {
+			/* from detecting by self to no detecting (by self/others), CAC abort  */
+			RTW_INFO(FUNC_HWBAND_FMT" disable detect, CAC abort\n", FUNC_HWBAND_ARG(band_idx));
+			rtw_os_indicate_cac_aborted(rfctl, band_idx, 0xFF
+				, rfctl->radar_detect_cch[band_idx], rfctl->radar_detect_bw[band_idx]);
+		}
+
+		for (i = HW_BAND_0; i < HW_BAND_MAX; i++) {
+			if (i == band_idx && cch) {
+				rd_cch = cch;
+				rd_bw = bw;
+			} else if (i != band_idx && rfctl->radar_detect_cch[i]) {
+				rd_cch = rfctl->radar_detect_cch[i];
+				rd_bw = rfctl->radar_detect_bw[i];
+			} else {
+				require_cac[i] = false;
+				continue;
+			}
+
+			require_cac[i] = rtw_chset_is_cch_require_cac(&rfctl->chset, rd_cch, rd_bw);
+			if (!require_cac[i]) {
+				if (IS_UNDER_CAC(rfctl))
+					RTW_INFO(FUNC_HWBAND_FMT" detect range already CAC done\n", FUNC_HWBAND_ARG(i));
+				/* if new coverage of detecting hwband CAC_DONE, force CAC finish */
+				rtw_os_force_cac_finished(rfctl, i, 0xFF, rd_cch, rd_bw);
+			}
+		}
+
+		if (!IS_CAC_STOPPED(rfctl)) {
+			/* set CAC stop if no hwband needed */
+			for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+				if (require_cac[i])
+					break;
+			if (i >= HW_BAND_MAX)
+				rfctl->cac_start_time = rfctl->cac_end_time = RTW_CAC_STOPPED;
+		}
+	}
+#endif /* CONFIG_DFS_MASTER */
+
+	rfctl->radar_detect_by_others[band_idx] = by_others;
+	rfctl->radar_detect_cch[band_idx] = cch;
+	rfctl->radar_detect_bw[band_idx] = bw;
+	rfctl->radar_detect_freq_hi[band_idx] = rd_freq_hi;
+	rfctl->radar_detect_freq_lo[band_idx] = rd_freq_lo;
+}
+
+static void rtw_dfs_rd_enable(struct rf_ctl_t *rfctl, u8 band_idx, u8 op_ch, u8 op_bw, u8 op_offset
+	, u32 rd_freq_hi, u32 rd_freq_lo, bool by_others)
+{
+#ifdef CONFIG_DFS_MASTER
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+
+	RTW_INFO(FUNC_HWBAND_FMT" on %u,%u,%u\n", FUNC_HWBAND_ARG(band_idx), op_ch, op_bw, op_offset);
+
+	if (rfctl->radar_detect_enabled && rfctl->radar_detect_hwband != band_idx) {
+		RTW_WARN(FUNC_HWBAND_FMT" radar_detect has been enabled for "HWBAND_FMT"\n"
+			, FUNC_HWBAND_ARG(band_idx), HWBAND_ARG(rfctl->radar_detect_hwband));
+		rtw_warn_on(1);
+		return;
+	}
+
+	if (!rfctl->radar_detect_enabled) {
+		RTW_INFO(FUNC_HWBAND_FMT" set radar_detect_enabled\n", FUNC_HWBAND_ARG(band_idx));
+		rfctl->radar_detect_hwband = band_idx;
+		rfctl->radar_detect_enabled = 1;
+		//#ifdef CONFIG_LPS
+		//LPS_Leave(adapter, "RADAR_DETECT_EN");
+		//#endif
+		_set_timer(&rfctl->radar_detect_timer
+			, rtw_dfs_hal_radar_detect_polling_int_ms(dvobj));
+	}
+	rfctl->radar_detected = 0;
+
+	rtw_dfs_update_rd_range_status(rfctl, band_idx, op_ch, op_bw, op_offset
+		, rd_freq_hi, rd_freq_lo, by_others);
+
+	rtw_dfs_hal_update_region(dvobj, band_idx, rtw_rfctl_get_dfs_domain(rfctl));
+	rtw_dfs_hal_radar_detect_enable(dvobj, band_idx, IS_CH_WAITING(rfctl), rd_freq_hi, rd_freq_lo);
+#else
+	rtw_warn_on(1);
+#endif
+}
+
+static void rtw_dfs_rd_disable(struct rf_ctl_t *rfctl, u8 band_idx, u8 op_ch, u8 op_bw, u8 op_offset
+	, u32 rd_freq_hi, u32 rd_freq_lo, bool by_others)
+{
+#ifdef CONFIG_DFS_MASTER
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+
+	if (rfctl->radar_detect_hwband == band_idx) {
+		if (rfctl->radar_detect_enabled) {
+			RTW_INFO(FUNC_HWBAND_FMT" clear radar_detect_enabled\n", FUNC_HWBAND_ARG(band_idx));
+			rfctl->radar_detect_enabled = 0;
+			rfctl->radar_detected = 0;
+			_cancel_timer_ex(&rfctl->radar_detect_timer);
+		}
+
+		rtw_dfs_hal_radar_detect_disable(dvobj, band_idx);
+	}
+#endif
+
+	if (by_others) {
+		rtw_dfs_update_rd_range_status(rfctl, band_idx, op_ch, op_bw, op_offset
+			, rd_freq_hi, rd_freq_lo, true);
+	} else
+		rtw_dfs_update_rd_range_status(rfctl, band_idx, 0, 0, 0, 0, 0, false);
+}
+
+static bool _rtw_dfs_rd_en_decision(struct rf_ctl_t *rfctl, struct mi_state *mstate
+	, enum band_type u_band, u8 u_ch, enum channel_width u_bw, enum chan_offset u_offset
+	, u32 *rd_freq_hi, u32 *rd_freq_lo, bool *lgd_sta_in_dfs)
+{
+	bool overlap_dfs = u_ch ? rtw_chset_get_dfs_frange(&rfctl->chset, u_band, u_ch, u_bw, u_offset, rd_freq_hi, rd_freq_lo) : false;
+	bool needed = false;
+
+	if (MSTATE_STA_LG_NUM(mstate) > 0) {
+		/* STA mode is linking */
+		if (overlap_dfs)
+			*lgd_sta_in_dfs = true;
+		goto exit;
+	}
+
+	if (MSTATE_STA_LD_NUM(mstate) > 0) {
+		if (overlap_dfs) {
+			/*
+			* if operate as slave w/o radar detect,
+			* rely on AP on which STA mode connects
+			*/
+			if (IS_DFS_SLAVE_WITH_RD(rfctl) && rtw_rfctl_radar_detect_supported(rfctl))
+				needed = true;
+			*lgd_sta_in_dfs = true;
+		}
+		goto exit;
+	}
+
+	if (!MSTATE_AP_NUM(mstate) && !MSTATE_MESH_NUM(mstate)) {
+		/* No working AP/Mesh mode */
+		goto exit;
+	}
+
+	if (!overlap_dfs)
+		goto exit;
+
+	needed = true;
+
+exit:
+	return needed;
+}
+
+static void rtw_dfs_rd_en_decision(struct dvobj_priv *dvobj, enum phl_band_idx hwband
+	, _adapter *adapter, struct _ADAPTER_LINK *alink, u8 mlme_act, u8 excl_ifbmp)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	u8 band_idx;
+
+	for (band_idx = HW_BAND_0; band_idx < HW_BAND_MAX; band_idx++) {
+		struct mi_state mstate;
+		u8 ifbmp;
+		enum band_type u_band;
+		u8 u_ch, u_bw, u_offset;
+		u32 rd_freq_hi, rd_freq_lo;
+		bool lgd_sta_in_dfs;
+		bool needed;
+
+		/* hwband < HW_BAND_MAX means specific hwband */
+		if (hwband < HW_BAND_MAX && hwband != band_idx)
+			continue;
+
+		/* alink != NULL means specific hwband */
+		if (alink && band_idx != ALINK_GET_HWBAND(alink))
+			continue;
+
+		rd_freq_hi = rd_freq_lo = 0;
+		lgd_sta_in_dfs = false;
+		needed = false;
+
+		if (mlme_act == MLME_ACTION_NONE
+			|| mlme_act == MLME_OPCH_SWITCH
+		) {
+			ifbmp = ~excl_ifbmp;
+			rtw_mi_status_by_hwband_ifbmp(dvobj, band_idx, ifbmp, &mstate);
+			rtw_mi_get_bch_setting_union_by_hwband_ifbmp(dvobj, band_idx, ifbmp, &u_band, &u_ch, &u_bw, &u_offset);
+
+		} else {
+			struct _ADAPTER_LINK *alink_by_hwband;
+
+			if (alink)
+				alink_by_hwband = alink;
+			else
+				alink_by_hwband = rtw_get_adapter_link_by_hwband(adapter, band_idx);
+
+			ifbmp = ~excl_ifbmp & ~BIT(adapter->iface_id);
+			rtw_mi_status_by_hwband_ifbmp(dvobj, band_idx, ifbmp, &mstate);
+			rtw_mi_get_bch_setting_union_by_hwband_ifbmp(dvobj, band_idx, ifbmp, &u_band, &u_ch, &u_bw, &u_offset);
+
+			switch (mlme_act) {
+			case MLME_STA_CONNECTING:
+				MSTATE_STA_LG_NUM(&mstate)++;
+				break;
+			case MLME_STA_CONNECTED:
+				MSTATE_STA_LD_NUM(&mstate)++;
+				break;
+			case MLME_STA_DISCONNECTED:
+				break;
+			#ifdef CONFIG_AP_MODE
+			case MLME_AP_STARTED:
+				MSTATE_AP_NUM(&mstate)++;
+				break;
+			case MLME_AP_STOPPED:
+				break;
+			#endif
+			#ifdef CONFIG_RTW_MESH
+			case MLME_MESH_STARTED:
+				MSTATE_MESH_NUM(&mstate)++;
+				break;
+			case MLME_MESH_STOPPED:
+				break;
+			#endif
+			default:
+				rtw_warn_on(1);
+				break;
+			}
+
+			if (alink_by_hwband) {
+				enum band_type band = ALINK_GET_BAND(alink_by_hwband);
+				u8 ch = ALINK_GET_CH(alink_by_hwband);
+				u8 bw = ALINK_GET_BW(alink_by_hwband);
+				u8 offset = ALINK_GET_OFFSET(alink_by_hwband);
+
+				if (u_ch) {
+					if (!MLME_IS_OPCH_SW(adapter) && CHK_MLME_STATE(adapter, WIFI_UNDER_LINKING | WIFI_ASOC_STATE)) {
+						if (!rtw_is_bchbw_grouped(band, ch, bw, offset, u_band, u_ch, u_bw, u_offset)) {
+							RTW_WARN(FUNC_HWBAND_FMT" "ADPT_FMT" can't sync %u,%u,%u,%u with %u,%u,%u,%u\n"
+								, FUNC_HWBAND_ARG(band_idx), ADPT_ARG(adapter)
+								, band, ch, bw, offset, u_band, u_ch, u_bw, u_offset);
+							goto apply;
+						}
+
+						rtw_sync_bchbw(&band, &ch, &bw, &offset, &u_band, &u_ch, &u_bw, &u_offset);
+					}
+				} else {
+					u_band = band;
+					u_ch = ch;
+					u_bw = bw;
+					u_offset = offset;
+				}
+			}
+		}
+
+		needed = _rtw_dfs_rd_en_decision(rfctl, &mstate, u_band, u_ch, u_bw, u_offset
+				, &rd_freq_hi, &rd_freq_lo, &lgd_sta_in_dfs);
+
+apply:
+		if (adapter)
+			RTW_INFO(FUNC_HWBAND_FMT" needed:%d, lgd_sta_in_dfs:%d, "ADPT_FMT" mlme_act:%u, excl_ifbmp:0x%02x\n"
+				, FUNC_HWBAND_ARG(band_idx), needed, lgd_sta_in_dfs, ADPT_ARG(adapter), mlme_act, excl_ifbmp);
+		else
+			RTW_INFO(FUNC_HWBAND_FMT" needed:%d, lgd_sta_in_dfs:%d, excl_ifbmp:0x%02x\n"
+				, FUNC_HWBAND_ARG(band_idx), needed, lgd_sta_in_dfs, excl_ifbmp);
+
+		RTW_INFO(FUNC_HWBAND_FMT" ld_sta:%u, lg_sta:%u, ap:%u, mesh:%u, %u,%u,%u,%u, rd_freq:%u~%u\n"
+			, FUNC_HWBAND_ARG(band_idx), MSTATE_STA_LD_NUM(&mstate), MSTATE_STA_LG_NUM(&mstate)
+			, MSTATE_AP_NUM(&mstate), MSTATE_MESH_NUM(&mstate)
+			, u_band, u_ch, u_bw, u_offset, rd_freq_lo, rd_freq_hi);
+
+		if (needed && rtw_rfctl_radar_detect_supported(rfctl))
+			rtw_dfs_rd_enable(rfctl, band_idx, u_ch, u_bw, u_offset, rd_freq_hi, rd_freq_lo, lgd_sta_in_dfs);
+		else {
+			rtw_dfs_rd_disable(rfctl, band_idx, u_ch, u_bw, u_offset, rd_freq_hi, rd_freq_lo, lgd_sta_in_dfs);
+			if (needed) {
+				_adapter *m_iface = rtw_mi_get_ap_mesh_iface_by_hwband(dvobj, band_idx);
+				u8 ifbmp_m = rtw_mi_get_ap_mesh_ifbmp_by_hwband(dvobj, band_idx);
+				u8 ifbmp_s = rtw_mi_get_lgd_sta_ifbmp_by_hwband(dvobj, band_idx);
+
+				RTW_INFO(FUNC_HWBAND_FMT" radart detect for this region not supported\n", FUNC_HWBAND_ARG(band_idx));
+
+				if (!m_iface) {
+					rtw_warn_on(1);
+					return;
+				}
+
+				if (rtw_hal_is_csa_support(m_iface) && !rtw_mr_is_ecsa_running(m_iface)) {
+					if (!rtw_hal_dfs_trigger_csa(m_iface, CSA_STA_DISCONNECT_ON_DFS,
+											band_idx, u_ch, u_bw, u_offset))
+						RTW_ERR(FUNC_HWBAND_FMT" trigger ECSA fail", FUNC_HWBAND_ARG(band_idx));
+				} else {
+					rtw_change_bss_bchbw_cmd(m_iface, RTW_CMDF_DIRECTLY
+						, ifbmp_m, ifbmp_s, REQ_BAND_NONE, REQ_CH_NONE, REQ_BW_ORI, REQ_OFFSET_NONE);
+				}
+			}
+		}
+	}
+}
+
+void rtw_dfs_rd_en_dec_on_mlme_act(_adapter *adapter, struct _ADAPTER_LINK *alink, u8 mlme_act, u8 excl_ifbmp)
+{
+	if (adapter && mlme_act != MLME_ACTION_NONE)
+		rtw_dfs_rd_en_decision(adapter_to_dvobj(adapter), HW_BAND_MAX, adapter, alink, mlme_act, excl_ifbmp);
+
+	rtw_warn_on(!adapter);
+	rtw_warn_on(mlme_act == MLME_ACTION_NONE);
+}
+
+void dump_radar_detect_status(void *sel, struct rf_ctl_t *rfctl, const char *title)
+{
+	u8 hwband;
+
+#ifdef CONFIG_DFS_MASTER
+	for (hwband = HW_BAND_0; hwband < HW_BAND_MAX; hwband++) {
+		if (rfctl->radar_detect_enabled && hwband == rfctl->radar_detect_hwband)
+			break;
+	}
+#else
+	hwband = HW_BAND_MAX;
+#endif
+	if (hwband >= HW_BAND_MAX) {
+		for (hwband = HW_BAND_0; hwband < HW_BAND_MAX; hwband++)
+			if (rfctl->radar_detect_by_others[hwband])
+				break;
+	}
+
+	if (hwband < HW_BAND_MAX) { /* by self or only by others */
+		RTW_PRINT_SEL(sel, "%s cch %u-%s"
+			, title
+			, rfctl->radar_detect_cch[hwband]
+			, ch_width_str(rfctl->radar_detect_bw[hwband])
+		);
+
+#ifdef CONFIG_DFS_MASTER
+		if (rfctl->radar_detect_enabled) {
+			u32 non_ocp_ms = 0;
+			u32 cac_ms = 0;
+			u8 dfs_domain = rtw_rfctl_get_dfs_domain(rfctl);
+
+			_RTW_PRINT_SEL(sel, ", domain:%s(%u)", rtw_dfs_regd_str(dfs_domain), dfs_domain);
+
+			if (IS_CH_WAITING(rfctl)) {
+				if (!IS_UNDER_CAC(rfctl)) {
+					non_ocp_ms = rtw_systime_to_ms(rfctl->cac_start_time - rtw_get_current_time());
+					cac_ms = rtw_systime_to_ms(rfctl->cac_end_time - rfctl->cac_start_time);
+				} else
+					cac_ms = rtw_systime_to_ms(rfctl->cac_end_time - rtw_get_current_time());
+			}
+
+			if (non_ocp_ms)
+				_RTW_PRINT_SEL(sel, ", non_ocp:%d", non_ocp_ms);
+			if (cac_ms)
+				_RTW_PRINT_SEL(sel, ", cac:%d", cac_ms);
+		} else
+#endif
+			_RTW_PRINT_SEL(sel, ", by AP of STA link");
+
+		_RTW_PRINT_SEL(sel, "\n");
+	}
+}
+#endif /* CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ */
+
+#ifdef CONFIG_DFS_MASTER
+void rtw_dfs_rd_en_dec_update(struct dvobj_priv *dvobj, enum phl_band_idx hwband)
+{
+	rtw_dfs_rd_en_decision(dvobj, hwband, NULL, NULL, MLME_ACTION_NONE, 0);
+}
+
+u8 rtw_dfs_rd_en_decision_cmd(struct dvobj_priv *dvobj, enum phl_band_idx hwband)
+{
+	struct cmd_obj *cmdobj;
+	struct drvextra_cmd_parm *parm;
+	struct cmd_priv *cmdpriv = &dvobj->cmdpriv;
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+	u8 res = _FAIL;
+
+	cmdobj = rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmdobj == NULL)
+		goto exit;
+
+	cmdobj->padapter = adapter;
+
+	parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+	if (parm == NULL) {
+		rtw_mfree(cmdobj, sizeof(struct cmd_obj));
+		goto exit;
+	}
+
+	parm->ec_id = DFS_RADAR_DETECT_EN_DEC_WK_CID;
+	parm->type = hwband;
+	parm->size = 0;
+	parm->pbuf = NULL;
+
+	init_h2fwcmd_w_parm_no_rsp(cmdobj, parm, CMD_SET_DRV_EXTRA);
+	CMD_OBJ_SET_HWBAND(cmdobj, hwband);
+	res = rtw_enqueue_cmd(cmdpriv, cmdobj);
+
+exit:
+	return res;
+}
+
+void rtw_indicate_cac_state_on_bss_start(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	if (IS_UNDER_CAC(rfctl)) {
+		u8 rd_hwband = rfctl->radar_detect_hwband;
+
+		if (rtw_iface_is_operate_at_hwband(adapter, rd_hwband))
+			rtw_os_indicate_cac_started(rfctl, rd_hwband, BIT(adapter->iface_id)
+				, rfctl->radar_detect_cch[rd_hwband], rfctl->radar_detect_bw[rd_hwband]);
+	} else {
+		u8 band_idx;
+
+		for (band_idx = HW_BAND_0; band_idx < HW_BAND_MAX; band_idx++) {
+			if (!rfctl->radar_detect_by_others[band_idx])
+				continue;
+			if (!rtw_iface_is_operate_at_hwband(adapter, band_idx))
+				continue;
+			rtw_os_force_cac_finished(rfctl, band_idx, BIT(adapter->iface_id)
+				, rfctl->radar_detect_cch[band_idx], rfctl->radar_detect_bw[band_idx]);
+		}
+	}
+}
+
+void rtw_indicate_cac_state_on_bss_stop(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	if (IS_UNDER_CAC(rfctl)) {
+		u8 rd_hwband = rfctl->radar_detect_hwband;
+
+		if (rtw_iface_is_operate_at_hwband(adapter, rd_hwband))
+			rtw_os_indicate_cac_aborted(rfctl, rd_hwband, BIT(adapter->iface_id)
+				, rfctl->radar_detect_cch[rd_hwband], rfctl->radar_detect_bw[rd_hwband]);
+	}
+}
+#endif /* CONFIG_DFS_MASTER */
+
+/* choose channel with shortest waiting (non ocp + cac) time */
+static bool rtw_choose_shortest_waiting_ch(struct rf_ctl_t *rfctl
+	, enum band_type sel_band, u8 sel_ch, u8 max_bw, enum chan_offset sel_offset
+	, enum band_type *dec_band, u8 *dec_ch, u8 *dec_bw, u8 *dec_offset
+	, u8 e_flags, u8 d_flags, enum band_type cur_band, u8 cur_ch, enum chan_offset cur_offset
+	, bool by_int_info, u8 mesh_only)
+{
+#ifndef DBG_CHOOSE_SHORTEST_WAITING_CH
+#define DBG_CHOOSE_SHORTEST_WAITING_CH 0
+#endif
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct rtw_chset *chset = &rfctl->chset;
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+	enum band_type band;
+	u8 ch, bw, offset, r_offset[2], r_offset_num;
+	enum band_type band_c = BAND_MAX;
+	u8 ch_c = 0, bw_c = 0, offset_c = 0;
+	int i, j;
+	u32 min_waiting_ms = 0;
+	u16 int_factor_c = 0;
+	bool within_same_band = rfctl->ch_sel_within_same_band;
+
+	if (!dec_ch || !dec_bw || !dec_offset) {
+		rtw_warn_on(1);
+		return _FALSE;
+	}
+
+	RTW_INFO("%s: sel_ch:%s-%u(%u) max_bw:%u e_flags:0x%02x d_flags:0x%02x cur_ch:%s-%u(%u) within_sb:%d%s%s\n"
+		, __func__, band_str(sel_band), sel_ch, sel_offset, max_bw, e_flags, d_flags
+		, band_str(cur_band), cur_ch, cur_offset, within_same_band
+		, by_int_info ? " int" : "", mesh_only ? " mesh_only" : "");
+
+	if (sel_band != BAND_MAX && rtw_rfctl_is_regu_forbid_bss(rfctl, sel_band))
+		goto exit;
+	if (sel_band == BAND_MAX && within_same_band && rtw_rfctl_is_regu_forbid_bss(rfctl, cur_band)) {
+		RTW_INFO("%s: cancel within_sb because REGU_FORBID for %s BSS", __func__, band_str(cur_band));
+		within_same_band = false;
+	}
+
+	/* full search and narrow bw judegement first to avoid potetial judegement timing issue */
+	for (bw = CHANNEL_WIDTH_20; bw <= max_bw; bw++) {
+		if (!rtw_hw_is_bw_support(dvobj, bw))
+			continue;
+
+		for (i = 0; i < chset->chs_len; i++) {
+			u32 non_ocp_ms = 0;
+			u32 cac_ms = 0;
+			u32 waiting_ms = 0;
+			u16 int_factor = 0;
+			bool dfs_ch;
+
+			if (chset->chs[i].flags & RTW_CHF_DIS)
+				continue;
+
+			band = chset->chs[i].band;
+			ch = chset->chs[i].ChannelNum;
+			if (rtw_rfctl_is_regu_forbid_bss(rfctl, band))
+				continue;
+			if (sel_band != BAND_MAX) {
+				if (band != sel_band)
+					continue;
+			} else if (within_same_band && cur_band != band)
+				continue;
+			if (sel_ch) {
+				if (ch != sel_ch)
+					continue;
+			}
+
+			if (band == BAND_ON_24G) {
+				if (bw > REGSTY_BW_2G(regsty))
+					continue;
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			} else if (band == BAND_ON_5G) {
+				if (bw > REGSTY_BW_5G(regsty))
+					continue;
+			#endif
+			#if CONFIG_IEEE80211_BAND_6GHZ
+			} else if (band == BAND_ON_6G) {
+				if (bw > REGSTY_BW_6G(regsty))
+					continue;
+			#endif
+			}
+
+			if (mesh_only && band == BAND_ON_24G && ch >= 5 && ch <= 9 && bw > CHANNEL_WIDTH_20)
+				continue;
+
+			if (sel_offset == CHAN_OFFSET_NO_EXT || bw == CHANNEL_WIDTH_20) {
+				if (!rtw_get_offsets_by_bchbw(band, ch, bw, r_offset, &r_offset_num))
+					continue;
+			} else {
+				r_offset[0] = sel_offset;
+				r_offset_num = 1;
+			}
+
+			for (j = 0; j < r_offset_num; j++) {
+				offset = r_offset[j];
+				if (!rtw_chset_is_bchbw_valid(chset, band, ch, bw, offset, 0, 0))
+					continue;
+
+				if ((e_flags & RTW_CHF_DFS) || (d_flags & RTW_CHF_DFS)) {
+					dfs_ch = rtw_chset_is_dfs_bchbw(chset, band, ch, bw, offset);
+					if (((e_flags & RTW_CHF_DFS) && !dfs_ch)
+						|| ((d_flags & RTW_CHF_DFS) && dfs_ch))
+						continue;
+				}
+
+				#ifdef CONFIG_DFS_MASTER
+				waiting_ms = rtw_get_ch_waiting_ms(rfctl, band, ch, bw, offset, &non_ocp_ms, &cac_ms);
+				#endif
+
+				#if 0 /* def CONFIG_RTW_ACS */
+				if (by_int_info) {
+					/* for now, consider only primary channel */
+					int_factor = hal_data->acs.interference_time[i];
+				}
+				#endif
+
+				if (DBG_CHOOSE_SHORTEST_WAITING_CH)
+					RTW_INFO("%s:%s,%u,%u,%u %u(non_ocp:%u, cac:%u), int:%u\n"
+						, __func__, band_str(band), ch, bw, offset, waiting_ms, non_ocp_ms, cac_ms, int_factor);
+
+				if (ch_c == 0
+					/* first: smaller wating time */
+					|| min_waiting_ms > waiting_ms
+					/* then: less interference */
+					|| (min_waiting_ms == waiting_ms && int_factor_c > int_factor)
+					/* then: wider bw */
+					|| (min_waiting_ms == waiting_ms && int_factor_c == int_factor && bw > bw_c)
+					/* if all condition equal, same channel&offset -> same channel -> same band prefer */
+					|| (min_waiting_ms == waiting_ms && int_factor_c == int_factor && bw == bw_c
+						&& (((cur_band != band_c || cur_ch != ch_c || cur_offset != offset_c)
+								&& (cur_band == band && cur_ch == ch && cur_offset == offset))
+							|| ((cur_band != band_c || cur_ch != ch_c) && (cur_band == band && cur_ch == ch))
+							|| (cur_band != band_c && cur_band == band))
+						)
+				) {
+					band_c = band;
+					ch_c = ch;
+					bw_c = bw;
+					offset_c = offset;
+					min_waiting_ms = waiting_ms;
+					int_factor_c = int_factor;
+				}
+			}
+		}
+	}
+
+exit:
+	if (ch_c != 0) {
+		RTW_INFO("%s: select %s,%u,%u,%u waiting_ms:%u\n"
+			, __func__, band_str(band_c), ch_c, bw_c, offset_c, min_waiting_ms);
+		if (dec_band)
+			*dec_band = band_c;
+		*dec_ch = ch_c;
+		*dec_bw = bw_c;
+		*dec_offset = offset_c;
+		return _TRUE;
+	} else {
+		RTW_INFO("%s: not found\n", __func__);
+		if (d_flags == 0)
+			rtw_warn_on(1);
+	}
+
+	return _FALSE;
+}
+
+bool rtw_rfctl_choose_bchbw(struct rf_ctl_t *rfctl
+	, enum band_type sel_band, u8 sel_ch, u8 max_bw, enum chan_offset sel_offset
+	, enum band_type cur_band, u8 cur_ch, enum chan_offset cur_offset
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset
+	, bool by_int_info, u8 mesh_only, const char *caller)
+{
+	bool ch_avail = _FALSE;
+
+#if defined(CONFIG_DFS_MASTER)
+	if (rtw_rfctl_radar_detect_supported(rfctl)) {
+		if (rfctl->radar_detected
+			&& rfctl->dbg_dfs_choose_dfs_ch_first
+		) {
+			ch_avail = rtw_choose_shortest_waiting_ch(rfctl, sel_band, sel_ch, max_bw, sel_offset
+						, band, ch, bw, offset
+						, RTW_CHF_DFS, 0
+						, cur_band, cur_ch, cur_offset, by_int_info, mesh_only);
+			if (ch_avail == _TRUE) {
+				RTW_INFO("%s choose 5G DFS channel for debug\n", caller);
+				goto exit;
+			}
+		}
+
+		if (rfctl->radar_detected
+			&& (rfctl->dfs_ch_sel_e_flags || rfctl->dfs_ch_sel_d_flags)
+		) {
+			ch_avail = rtw_choose_shortest_waiting_ch(rfctl, sel_band, sel_ch, max_bw, sel_offset
+						, band, ch, bw, offset
+						, rfctl->dfs_ch_sel_e_flags, rfctl->dfs_ch_sel_d_flags
+						, cur_band, cur_ch, cur_offset, by_int_info, mesh_only);
+			if (ch_avail == _TRUE) {
+				RTW_INFO("%s choose with dfs_ch_sel_ e_flags:0x%02x d_flags:0x%02x for debug\n"
+					, caller, rfctl->dfs_ch_sel_e_flags, rfctl->dfs_ch_sel_d_flags);
+				goto exit;
+			}
+		}
+
+		ch_avail = rtw_choose_shortest_waiting_ch(rfctl, sel_band, sel_ch, max_bw, sel_offset
+					, band, ch, bw, offset
+					, 0, 0
+					, cur_band, cur_ch, cur_offset, by_int_info, mesh_only);
+	} else
+#endif /* defined(CONFIG_DFS_MASTER) */
+	{
+		ch_avail = rtw_choose_shortest_waiting_ch(rfctl, sel_band, sel_ch, max_bw, sel_offset
+					, band, ch, bw, offset
+					, 0, RTW_CHF_DFS
+					, cur_band, cur_ch, cur_offset, by_int_info, mesh_only);
+	}
+#if defined(CONFIG_DFS_MASTER)
+exit:
+#endif
+	if (ch_avail == _FALSE)
+		RTW_WARN("%s no available channel\n", caller);
+
+	return ch_avail;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY bool rtw_rfctl_choose_chbw(struct rf_ctl_t *rfctl, u8 sel_ch, u8 max_bw, enum chan_offset sel_offset
+	, u8 cur_ch, enum chan_offset cur_offset, u8 *ch, u8 *bw, u8 *offset, bool by_int_info, u8 mesh_only, const char *caller)
+{
+	enum band_type sel_band = sel_ch ? (rtw_is_2g_ch(sel_ch) ? BAND_ON_24G : BAND_ON_5G) : BAND_MAX;
+	enum band_type cur_band = rtw_is_2g_ch(cur_ch) ? BAND_ON_24G : BAND_ON_5G;
+	enum band_type *band = NULL;
+
+	return rtw_rfctl_choose_bchbw(rfctl
+		, sel_band, sel_ch, max_bw, sel_offset
+		, cur_band, cur_ch, cur_offset
+		, band, ch, bw, offset
+		, by_int_info, mesh_only, caller);
+}
+#endif
+
+void rtw_rfctl_dfs_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty)
+{
+	rfctl->ch_sel_within_same_band = 1;
+
+#ifdef CONFIG_DFS_MASTER
+	rfctl->dfs_region_domain = regsty->dfs_region_domain;
+	if (!rtw_dfs_hal_region_supported(rfctl_to_dvobj(rfctl), rfctl->dfs_region_domain)) {
+		RTW_WARN("%s DFS region domain: %s is not supported by HAL, set to %s\n", __func__
+			, rtw_dfs_regd_str(rfctl->dfs_region_domain), rtw_dfs_regd_str(RTW_DFS_REGD_NONE));
+		rfctl->dfs_region_domain = RTW_DFS_REGD_NONE;
+	}
+	rfctl->non_ocp_finished = true;
+	rfctl->cac_start_time = rfctl->cac_end_time = RTW_CAC_STOPPED;
+	rtw_init_timer(&(rfctl->radar_detect_timer), rtw_dfs_rd_timer_hdl, rfctl);
+#endif
+
+#if CONFIG_DFS_SLAVE_WITH_RADAR_DETECT
+	rfctl->dfs_slave_with_rd = 1;
+#endif
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_dfs.h b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_dfs.h
new file mode 100644
index 000000000000..f726c12d9f9f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_dfs.h
@@ -0,0 +1,93 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_DFS_H__
+#define __RTW_DFS_H__
+
+#define CAC_TIME_MS (60*1000)
+#define CAC_TIME_CE_MS (10*60*1000)
+#define NON_OCP_TIME_MS (30*60*1000)
+
+#define RTW_CAC_STOPPED 0 /* used by cac_start_time, cac_end_time time stamps */
+#define RTW_NON_OCP_STOPPED 0 /* used by non_ocp_end_time time stamps */
+
+bool rtw_chset_is_dfs_range(const struct rtw_chset *chset, u32 hi, u32 lo);
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_dfs_ch(const struct rtw_chset *chset, u8 ch);
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_dfs_chbw(const struct rtw_chset *chset, u8 ch, u8 bw, u8 offset);
+bool rtw_chset_is_dfs_bch(const struct rtw_chset *chset, enum band_type band, u8 ch);
+bool rtw_chset_is_dfs_bchbw(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset);
+
+enum rtw_dfs_regd rtw_rfctl_get_dfs_domain(struct rf_ctl_t *rfctl);
+bool rtw_rfctl_radar_detect_supported(struct rf_ctl_t *rfctl);
+
+#ifdef CONFIG_DFS_MASTER
+struct rf_ctl_t;
+enum phl_band_idx;
+
+#define CH_IS_NON_OCP_STOPPED(rt_ch_info) ((rt_ch_info)->non_ocp_end_time == RTW_NON_OCP_STOPPED)
+#define CH_IS_NON_OCP(rt_ch_info) (!CH_IS_NON_OCP_STOPPED(rt_ch_info) && rtw_time_after((rt_ch_info)->non_ocp_end_time, rtw_get_current_time()))
+
+bool rtw_rfctl_overlap_radar_detect_ch(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset);
+bool adapter_is_tx_blocked_by_ch_waiting(_adapter *adapter);
+bool alink_is_tx_blocked_by_ch_waiting(struct _ADAPTER_LINK *alink);
+
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_chbw_non_ocp(const struct rtw_chset *chset, u8 ch, u8 bw, u8 offset);
+RTW_FUNC_2G_5G_ONLY bool rtw_chset_is_ch_non_ocp(const struct rtw_chset *chset, u8 ch);
+bool rtw_chset_is_bchbw_non_ocp(const struct rtw_chset *chset, enum band_type band, u8 ch, u8 bw, u8 offset);
+bool rtw_chset_is_bch_non_ocp(const struct rtw_chset *chset, enum band_type band, u8 ch);
+
+void rtw_rfctl_chset_chk_non_ocp_finish(struct rf_ctl_t *rfctl);
+
+void rtw_rfctl_force_update_non_ocp_ms(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset, int ms);
+
+u32 rtw_get_ch_waiting_ms(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset, u32 *r_non_ocp_ms, u32 *r_cac_ms);
+
+u32 rtw_force_stop_cac(struct rf_ctl_t *rfctl, u32 timeout_ms);
+
+u8 rtw_dfs_rd_hdl(struct dvobj_priv *dvobj, enum phl_band_idx hwband, u8 radar_cch, enum channel_width radar_bw);
+
+void rtw_dfs_rd_en_dec_update(struct dvobj_priv *dvobj, enum phl_band_idx hwband);
+u8 rtw_dfs_rd_en_decision_cmd(struct dvobj_priv *dvobj, enum phl_band_idx hwband);
+
+void rtw_indicate_cac_state_on_bss_start(_adapter *adapter);
+void rtw_indicate_cac_state_on_bss_stop(_adapter *adapter);
+
+#else
+#define CH_IS_NON_OCP(rt_ch_info) 0
+#define adapter_is_tx_blocked_by_ch_waiting(adapter) false
+#define alink_is_tx_blocked_by_ch_waiting(alink) false
+
+RTW_FUNC_2G_5G_ONLY static inline bool rtw_chset_is_chbw_non_ocp(const struct rtw_chset *chset, u8 ch, u8 bw, u8 offset) { return false; }
+RTW_FUNC_2G_5G_ONLY static inline bool rtw_chset_is_ch_non_ocp(const struct rtw_chset *chset, u8 ch) { return false; }
+#define rtw_chset_is_bchbw_non_ocp(chset, band, ch, bw, offset) false
+#define rtw_chset_is_bch_non_ocp(chset, band, ch) false
+#endif /* CONFIG_DFS_MASTER */
+
+#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+void rtw_dfs_rd_en_dec_on_mlme_act(_adapter *adapter, struct _ADAPTER_LINK *alink, u8 mlme_act, u8 excl_ifbmp);
+void dump_radar_detect_status(void *sel, struct rf_ctl_t *rfctl, const char *title);
+#endif
+
+bool rtw_rfctl_choose_bchbw(struct rf_ctl_t *rfctl
+	, enum band_type sel_band, u8 sel_ch, u8 max_bw, enum chan_offset sel_offset
+	, enum band_type cur_band, u8 cur_ch, enum chan_offset cur_offset
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset
+	, bool by_int_info, u8 mesh_only, const char *caller);
+
+RTW_FUNC_2G_5G_ONLY bool rtw_rfctl_choose_chbw(struct rf_ctl_t *rfctl, u8 sel_ch, u8 max_bw, enum chan_offset sel_offset
+	, u8 cur_ch, enum chan_offset cur_offset, u8 *ch, u8 *bw, u8 *offset, bool by_int_info, u8 mesh_only, const char *caller);
+
+void rtw_rfctl_dfs_init(struct rf_ctl_t *rfctl, struct registry_priv *regsty);
+
+#endif /* __RTW_DFS_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_eht.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_eht.c
new file mode 100644
index 000000000000..0d92c764d209
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_eht.c
@@ -0,0 +1,458 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_EHT_C
+
+#include <drv_types.h>
+
+#ifdef CONFIG_80211BE_EHT
+
+void rtw_set_assoc_status(_adapter *padapter, struct rtw_phl_mld_t *pmld, u8 link_id, u16 status)
+{
+	u8 i;
+	for (i = 0; i < padapter->phl_role->rlink_num; i++) {
+		if (pmld->assoc_status[i].used)
+			continue;
+		pmld->assoc_status[i].link_id = link_id;
+		pmld->assoc_status[i].status = status;
+		pmld->assoc_status[i].used = 1;
+		break;
+	}
+}
+
+void rtw_update_ies_with_inheritance(u8 *cur_ies, u32 *cur_ies_len, u8 *inh_ies, u8 inh_ies_len)
+{
+	u8 *non_inh_ele, non_inh_ele_len;
+	u8 i, j;
+	struct _NDIS_802_11_VARIABLE_IEs *pIE = NULL;
+
+	/* Remove the non-inherited elements*/
+	for (i = 0; i < inh_ies_len;) {
+		pIE = (struct _NDIS_802_11_VARIABLE_IEs *)(inh_ies + i);
+
+		if (pIE->ElementID == WLAN_EID_EXTENSION &&
+				pIE->data[0] == EID_EXT_NON_INHERITANCE) {
+			/* Non-inherited elements */
+			non_inh_ele = &pIE->data[0] + 1;
+			non_inh_ele_len = *non_inh_ele++;
+			for (j = 0; j < non_inh_ele_len; j++) {
+				rtw_ies_remove_ie(cur_ies, cur_ies_len, 0, *non_inh_ele, NULL, 0);
+				non_inh_ele++;
+			}
+			/* Non-inherited extension elements */
+			non_inh_ele_len = *non_inh_ele++;
+			for (j = 0; j < non_inh_ele_len; j++) {
+				rtw_ies_remove_ie(cur_ies, cur_ies_len, 0, WLAN_EID_EXTENSION, non_inh_ele, 1);
+				non_inh_ele++;
+			}
+		} else if (pIE->ElementID == WLAN_EID_EXTENSION) { /* Update extension IEs */
+			rtw_ies_remove_ie(cur_ies, cur_ies_len, 0, WLAN_EID_EXTENSION, &pIE->data[0], 1);
+			rtw_set_ie(cur_ies + *cur_ies_len, WLAN_EID_EXTENSION,
+				pIE->Length, pIE->data, cur_ies_len);
+		} else { /* Update IEs */
+			rtw_ies_remove_ie(cur_ies, cur_ies_len, 0, pIE->ElementID, NULL, 0);
+			rtw_set_ie(cur_ies + *cur_ies_len, pIE->ElementID,
+				pIE->Length, pIE->data, cur_ies_len);
+		}
+		i += (pIE->Length + 2);
+	}
+}
+
+void assoc_rsp_per_sta_profile_handler(_adapter *padapter,
+		struct rtw_phl_per_sta_profile_element *profile,
+		u8 *primary_ies, u32 primary_ies_len)
+{
+	struct _ADAPTER_LINK *cur_link = NULL;
+	u8 cur_ies[MAX_IE_SZ];
+	u32 cur_ies_len = 0;
+	struct rtw_phl_mld_t *pmld = NULL;
+	unsigned short status;
+	u8 eid_eht_ml = EID_EXT_MULTI_LINK;
+
+	pmld = rtw_phl_get_mld_self(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter->phl_role);
+	cur_link = rtw_get_adapter_link_by_linkid(padapter, profile->link_id);
+
+	if (!cur_link)
+		goto _exit;
+
+	/* status */
+	status = le16_to_cpu(*(unsigned short *)(profile->sta_profile + 2));
+	rtw_set_assoc_status(padapter, pmld, profile->link_id, status);
+	if (status != _STATS_SUCCESSFUL_) {
+		RTW_INFO("assoc reject for link (link_id = %d), status code: %d\n",
+			profile->link_id, status);
+		goto _exit;
+	}
+	cur_link->mlmepriv.is_accepted = _TRUE;
+	/* get capabilities */
+	cur_link->mlmeextpriv.mlmext_info.capability =
+				le16_to_cpu(*(unsigned short *)(profile->sta_profile));
+	/* set slot time */
+	cur_link->mlmeextpriv.mlmext_info.slotTime =
+		(cur_link->mlmeextpriv.mlmext_info.capability & BIT(10)) ? 9 : 20;
+
+	_rtw_memcpy(cur_ies, primary_ies, primary_ies_len);
+	cur_ies_len = primary_ies_len;
+
+
+	/* Remove ML IEs for other links */
+	rtw_ies_remove_ie(cur_ies, &cur_ies_len, 0, WLAN_EID_EXTENSION, &eid_eht_ml, 1);
+
+	/* Update cur_ies */
+	rtw_update_ies_with_inheritance(cur_ies, &cur_ies_len,
+		profile->sta_profile + 4, profile->sta_profile_len - 4);
+
+	rtw_ie_handler(padapter, cur_link, cur_ies, cur_ies_len);
+
+_exit:
+	return;
+}
+
+u32 rtw_parse_ml_ie(_adapter *padapter, u8 *ies, u32 ies_len,
+		struct rtw_phl_ml_element *ml_ele)
+{
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(adapter_to_dvobj(padapter));
+	u8 eid_eht_ml = EID_EXT_MULTI_LINK;
+	u32 eht_ml_ie_len;
+	u8 *p;
+
+	p = rtw_get_ie_ex(ies, ies_len, WLAN_EID_EXTENSION,
+			&eid_eht_ml, 1, NULL, &eht_ml_ie_len);
+
+	if (p == NULL || eht_ml_ie_len < EHT_ML_BASIC_MIN_LEN)
+		return 0;
+
+	rtw_phl_parse_ml_ie(phl_com, p + 3, eht_ml_ie_len - 3, ml_ele);
+	return eht_ml_ie_len;
+}
+
+void EHT_ml_ie_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 *primary_ies, u32 primary_ies_len)
+{
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct mlo_priv		*pmlopriv = &pmlmepriv->mlopriv;
+	struct rtw_phl_ml_element ml_ele = {0};
+	u8 lidx;
+
+	if (!rtw_parse_ml_ie(padapter, primary_ies, primary_ies_len, &ml_ele))
+		return;
+
+	for (lidx = 0; lidx < ml_ele.profile_num; lidx++)
+		assoc_rsp_per_sta_profile_handler(padapter, &ml_ele.profile[lidx],
+				primary_ies, primary_ies_len);
+
+}
+
+void EHT_tid2link_ie_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 *ies, u32 ies_len)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlo_priv *pmlopriv = &pmlmepriv->mlopriv;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	WLAN_BSSID_EX *cur_network = &pmlmeinfo->network;
+	struct sta_info		*psta = NULL;
+	u8 eid_eht_tid2link = EID_EXT_TID_TO_LINK_MAPPING;
+	u32 eht_tid2link_ie_len;
+	u8 *p;
+
+	if (pmlopriv->tid2link_nego == _FALSE)
+		return;
+
+	psta = rtw_get_stainfo(&padapter->stapriv, cur_network->MacAddress);
+
+	if (!psta)
+		return;
+
+	/* TID-To_Link Mapping IE */
+	p = rtw_get_ie_ex(ies, ies_len, WLAN_EID_EXTENSION,
+			&eid_eht_tid2link, 1, NULL, &eht_tid2link_ie_len);
+
+	if (p == NULL)
+		rtw_phl_tid2link_not_present(psta->phl_sta, pmlopriv->tid2link_nego);
+	else
+		rtw_phl_parse_tid2link(psta->phl_sta, p + 3, eht_tid2link_ie_len - 3);
+}
+
+void rtw_eht_ml_ies_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 *ies, u32 ies_len)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlo_priv *pmlopriv = &pmlmepriv->mlopriv;
+
+	if (pmlopriv->mlo_option == _FALSE)
+		return;
+
+	/* ML IE */
+	EHT_ml_ie_handler(padapter, padapter_link, ies, ies_len);
+
+	/* TID-to-Link IE */
+	EHT_tid2link_ie_handler(padapter, padapter_link, ies, ies_len);
+}
+
+u8 build_inherited_ies(u8 *cur_ies, u32 cur_ies_len, u8 *primary_ies, u32 primary_ies_len, u8 *sta_ies)
+{
+	PNDIS_802_11_VARIABLE_IEs sIE = NULL;
+	u8 *p;
+	u32 i;
+	u32 len;
+	u8 nih_eles[256];
+	u8 nih_eles_len = 0;
+	u8 nih_ext_eles[256];
+	u8 nih_ext_eles_len = 0;
+	u32 sta_ies_len = 0;
+
+	/* Build per sta IEs */
+	for (i = 0; i < cur_ies_len;) {
+		sIE = (struct _NDIS_802_11_VARIABLE_IEs *)(cur_ies + i);
+		if (sIE->ElementID == WLAN_EID_EXTENSION) {
+			p = rtw_get_ie_ex(primary_ies, primary_ies_len,
+				WLAN_EID_EXTENSION, &sIE->data[0], 1, NULL, &len);
+			len -= 2;
+		}
+		else
+			p = rtw_get_ie(primary_ies, sIE->ElementID, &len, primary_ies_len);
+
+		/* Skip the same IEs */
+		if (p == NULL) /* Element only exists in cur_ies */
+			rtw_set_ie(sta_ies + sta_ies_len, sIE->ElementID, sIE->Length, sIE->data, &sta_ies_len);
+		else if ((sIE->Length != len || !_rtw_memcmp(sIE->data, p + 2, len)) /* Element is different between primary_ies and cur_ies */
+				&& (sIE->ElementID != WLAN_EID_VENDOR_SPECIFIC)) /* skip vendor specific */
+			rtw_set_ie(sta_ies + sta_ies_len, sIE->ElementID, sIE->Length, sIE->data, &sta_ies_len);
+
+		i += (sIE->Length + 2);
+	}
+
+	/* Get non-inherited elements */
+	nih_eles[nih_eles_len++] = EID_EXT_NON_INHERITANCE;
+	nih_eles_len++; /* Skip length first */
+	for (i = 0; i < primary_ies_len;) {
+		sIE = (struct _NDIS_802_11_VARIABLE_IEs *)(primary_ies + i);
+		if (sIE->ElementID == WLAN_EID_EXTENSION) {
+			p = rtw_get_ie_ex(cur_ies, cur_ies_len,
+				WLAN_EID_EXTENSION, &sIE->data[0], 1, NULL, &len);
+			if (p == NULL)
+				nih_ext_eles[nih_ext_eles_len++] = sIE->data[0];
+		}
+		else {
+			p = rtw_get_ie(cur_ies, sIE->ElementID, &len, cur_ies_len);
+			if (p == NULL)
+				nih_eles[nih_eles_len++] = sIE->ElementID;
+		}
+		i += (sIE->Length + 2);
+	}
+
+	/* Append non-inherited element IE */
+	nih_eles[1] = nih_eles_len - 2;
+	nih_eles[nih_eles_len++] = nih_ext_eles_len;
+	for (i = 0; i < nih_ext_eles_len; i++)
+		nih_eles[nih_eles_len++] = nih_ext_eles[i];
+	if (nih_eles_len > 3) /* non-inherited elements exist */
+		rtw_set_ie(sta_ies + sta_ies_len, WLAN_EID_EXTENSION, nih_eles_len, nih_eles, &sta_ies_len);
+
+	return (u8)sta_ies_len;
+}
+
+int build_link_assocreq_ies(_adapter *padapter, struct _ADAPTER_LINK *cur_link, u8 *cur_ies, u32 *cur_ies_len, u8 is_reassoc)
+{
+	struct pkt_attrib cur_attrib = {0};
+	int ret;
+
+	cur_attrib.adapter_link = cur_link;
+	ret = rtw_build_assocreq_ies(padapter, cur_link, cur_ies, &cur_attrib, is_reassoc);
+	*cur_ies_len = cur_attrib.pktlen;
+
+	return ret;
+}
+
+void build_link_probersp_ies(_adapter *padapter, struct _ADAPTER_LINK *cur_link, u8 *cur_ies, u32 *cur_ies_len,
+		u8 is_valid_p2p_probereq)
+{
+	struct pkt_attrib cur_attrib = {0};
+
+	cur_attrib.adapter_link = cur_link;
+	rtw_build_probersp_ies(padapter, cur_link, cur_ies, &cur_attrib, is_valid_p2p_probereq);
+	*cur_ies_len = cur_attrib.pktlen;
+
+}
+
+u8 _build_ml_sta_profile(_adapter *padapter, struct _ADAPTER_LINK *cur_link, struct sta_info *cur_sta,
+			enum phl_packet_type pkt_type, u8 *primary_ies, u32 primary_ies_len, u8 *ies, u8 reserved)
+{
+	u8 cur_ies[MAX_IE_SZ] = {0};
+	u32 cur_ies_len = 0;
+	u8 offset;
+	u8 ies_len = 0;
+
+	switch(pkt_type) {
+		case PACKET_ASSOC_REQUEST:
+			if (reserved) /* reassoc_req */
+				offset = _REASOCREQ_IE_OFFSET_;
+			else
+				offset = _ASOCREQ_IE_OFFSET_;
+			if (build_link_assocreq_ies(padapter, cur_link, cur_ies, &cur_ies_len, reserved)
+					!= _SUCCESS) /* don't request the current link if no joint supported rate */
+				goto _exit;
+			/* capability */
+			_rtw_memcpy(ies, cur_ies, 2);
+			ies_len += 2;
+			/* skip listen interval and current AP field */
+			ies_len += build_inherited_ies(cur_ies + offset, cur_ies_len - offset,
+					primary_ies + offset, primary_ies_len - offset,
+					ies + ies_len);
+			break;
+		case PACKET_ASSOC_RESPONSE:
+			/* ToDo */
+			break;
+		case PACKET_PROBE_RESPONSE_ML:
+			offset = _PROBERSP_IE_OFFSET_;
+			build_link_probersp_ies(padapter, cur_link, cur_ies, &cur_ies_len, reserved);
+			/* copy capabilities */
+			_rtw_memcpy(ies, (unsigned char *)(rtw_get_capability_from_ie(cur_ies)) , 2);
+			ies_len += 2;
+			ies_len += build_inherited_ies(cur_ies + offset, cur_ies_len - offset,
+				primary_ies + offset, primary_ies_len - offset,
+				ies + ies_len);
+			break;
+		default:
+			break;
+	}
+
+_exit:
+	return ies_len;
+}
+
+u8 *rtw_build_ml_ies(_adapter *padapter, struct sta_info *psta, u8 *pframe, struct pkt_attrib *pattrib,
+			enum phl_packet_type pkt_type, u8 *primary_ies, u32 primary_ies_len, u8 reserved)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	u32 len = 0;
+	struct rtw_phl_ml_ie_info info = {0};
+	u8 lidx;
+	u8 profiles[256] = {0};
+	struct rtw_phl_mld_t *pmld;
+	u8 critical_update = 0;
+	u32 total_len = 0;
+
+	if (psta)
+		info.mld = psta->phl_sta->mld;
+	else
+		info.mld = rtw_phl_get_mld_self(GET_PHL_INFO(d), padapter->phl_role);
+	info.pkt_type = pkt_type;
+	info.rlink = padapter_link->wrlink;
+
+	/* The following types need to append link info */
+	switch(pkt_type) {
+		case PACKET_BEACON:
+			/* ToDo CONFIG_RTW_MLD: Get the critical update value */
+			info.critical_update = critical_update;
+			break;
+		case PACKET_ASSOC_REQUEST:
+			/* Multilink Element */
+			/* Append the to_join link info */
+			for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+				struct _ADAPTER_LINK *alink = NULL;
+				struct rtw_phl_per_sta_profile_info ml_sta_info = {0};
+				u8 sta_profile[256] = {0};
+
+				alink = GET_LINK(padapter, lidx);
+				if (alink == padapter_link || !alink->mlmepriv.to_join)
+					continue;
+
+				ml_sta_info.pkt_type = pkt_type;
+				ml_sta_info.mld = rtw_phl_get_mld_self(GET_PHL_INFO(d), padapter->phl_role);
+				ml_sta_info.rlink = alink->wrlink;
+				ml_sta_info.link_id = alink->mlmepriv.link_id;
+				ml_sta_info.sta_profile_len = _build_ml_sta_profile(padapter, alink, NULL, pkt_type, primary_ies,
+						primary_ies_len, sta_profile, reserved);
+				ml_sta_info.sta_profile = sta_profile;
+				len = rtw_phl_build_per_sta_profile(GET_PHL_COM(d), &ml_sta_info, profiles + info.opt_len);
+				info.opt_len += len;
+			}
+			info.opt = profiles;
+			total_len += rtw_phl_build_ml_ie(GET_PHL_COM(d), &info, pframe);
+
+			/* TID-to-Link Mapping Element */
+			if (padapter->mlmepriv.mlopriv.tid2link_nego)
+				total_len += rtw_phl_build_tid2link(padapter_link->wrlink, pframe + total_len);
+
+			break;
+		case PACKET_ASSOC_RESPONSE:
+			/* ToDo */
+			break;
+		case PACKET_PROBE_RESPONSE_ML:
+			for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+				struct _ADAPTER_LINK *alink = NULL;
+				struct rtw_phl_per_sta_profile_info ml_sta_info = {0};
+				u8 sta_profile[256] = {0};
+
+				alink = GET_LINK(padapter, lidx);
+				/* Skip the primary adapter_link and the link without ready content */
+				if (alink == padapter_link || !alink->mlmepriv.cur_network.join_res)
+					continue;
+
+				ml_sta_info.pkt_type = pkt_type;
+				ml_sta_info.mld = rtw_phl_get_mld_self(GET_PHL_INFO(d), padapter->phl_role);
+				ml_sta_info.rlink = alink->wrlink;
+				ml_sta_info.link_id = alink->wrlink->id;
+				ml_sta_info.complete_profile = _TRUE;
+				ml_sta_info.sta_profile_len = _build_ml_sta_profile(padapter, alink, NULL, pkt_type, primary_ies,
+						primary_ies_len, sta_profile, reserved);
+				ml_sta_info.sta_profile = sta_profile;
+				len = rtw_phl_build_per_sta_profile(GET_PHL_COM(d), &ml_sta_info, profiles + info.opt_len);
+				info.opt_len += len;
+			}
+			info.opt = profiles;
+			total_len += rtw_phl_build_ml_ie(GET_PHL_COM(d), &info, pframe);
+			break;
+		default:
+			total_len += rtw_phl_build_ml_ie(GET_PHL_COM(d), &info, pframe);
+			break;
+	}
+
+	pattrib->pktlen += total_len;
+
+	return pframe + total_len;
+}
+
+u32 rtw_restructure_ml_ie(_adapter *padapter,
+					struct _ADAPTER_LINK *padapter_link,
+					u8 *in_ie, u8 *out_ie, uint in_len,
+					uint *pout_len)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlo_priv	*pmlopriv = &pmlmepriv->mlopriv;
+	struct rtw_phl_ml_ie_info info = {0};
+	struct rtw_phl_ml_element ml_ele = {0};
+
+	if (!rtw_parse_ml_ie(padapter, in_ie + _FIXED_IE_LENGTH_, in_len - _FIXED_IE_LENGTH_, &ml_ele))
+		goto exit;
+
+	pmlopriv->mlo_option = _TRUE;
+
+	if (ml_ele.common_info.basic_ml.mld_cap.tid_to_link_nego_support)
+		pmlopriv->tid2link_nego = _TRUE;
+
+	info.mld = rtw_phl_get_mld_self(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter->phl_role);
+	info.pkt_type = PACKET_BEACON;
+	info.rlink = padapter_link->wrlink;
+
+	*pout_len += rtw_phl_build_ml_ie(GET_PHL_COM(adapter_to_dvobj(padapter)), &info, out_ie + *pout_len);
+
+exit:
+	return pmlopriv->mlo_option;
+}
+
+#endif /* CONFIG_80211BE_EHT */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fake_ap.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fake_ap.c
new file mode 100644
index 000000000000..98d47dfe740f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fake_ap.c
@@ -0,0 +1,480 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+
+static const u8 bcast_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+#ifdef RTW_PHL_TEST_FPGA
+static const u8 ap_addr[ETH_ALEN] = {0x00, 0xe0, 0x4c, 0x88, 0x52, 0xa0};
+u8 ap_ssid[] = {'p','h','l','-','5','g'};
+#else
+static const u8 ap_addr[ETH_ALEN] = {0x00, 0xe0, 0x4c, 0x12, 0x34, 0x56};
+u8 ap_ssid[] = {'f','a','k','e','a','p'};
+#endif
+u8 ap_ch = 1;
+u16 ap_beacon_interval = 100; /* unit: ms */
+u16 ap_capability_info = 0x0001;
+u8 ap_datarate[] = {0x82, 0x84, 0x8b, 0x96, 0xc, 0x12, 0x18, 0x24};
+
+int rtw_fakeap_tx(struct _ADAPTER *a, struct xmit_frame *frame)
+{
+	u8 *data = frame->buf_addr + TXDESC_OFFSET;
+	u32 data_len = frame->attrib.last_txcmdsz;
+ 	struct sk_buff *skb;
+	u8 *buf;
+
+
+	if (GetFrameType(data) != WIFI_MGT_TYPE)
+		return _FAIL;
+
+	switch (get_frame_sub_type(data)) {
+	case WIFI_AUTH:
+	case WIFI_ASSOCREQ:
+	case WIFI_REASSOCREQ:
+	case WIFI_PROBEREQ:
+		break;
+
+	default:
+		return _FAIL;
+	}
+
+	skb = rtw_skb_alloc(data_len);
+	buf = skb_put(skb, data_len);
+	_rtw_memcpy(buf, data, data_len);
+	rtw_free_xmitbuf(&a->xmitpriv, frame->pxmitbuf);
+	rtw_free_xmitframe(&a->xmitpriv, frame);
+
+	skb_queue_tail(&adapter_to_dvobj(a)->fakeap.rxq, skb);
+	_set_workitem(&adapter_to_dvobj(a)->fakeap.work);
+
+	return _SUCCESS;
+}
+
+static void add_ie(struct sk_buff *skb, u8 index, u8 len, const u8 *source)
+{
+	u8 *buf;
+
+
+	buf = skb_put(skb, 2 + len);
+	*buf = index;
+	*(buf + 1) = len;
+	if (len > 0)
+		_rtw_memcpy(buf + 2, source, len);
+}
+
+void rtw_fakeap_bcn_timer_hdl(void *p)
+{
+	struct dvobj_priv *d;
+	struct sk_buff *skb;
+	struct rtw_ieee80211_hdr *wlanhdr;
+	int len;
+	u8 *buf;
+	u8 hdrlen;
+
+
+	d = (struct dvobj_priv*)p;
+	hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	len = hdrlen + _FIXED_IE_LENGTH_
+	      + 2 + sizeof(ap_ssid)
+	      + 2 + sizeof(ap_datarate)
+	      + 2 + 1;
+	skb = rtw_skb_alloc(len);
+	_rtw_memset(skb->data, 0, len);
+
+	buf = skb_put(skb, hdrlen);
+	wlanhdr = (struct rtw_ieee80211_hdr*)buf;
+	set_frame_sub_type(&wlanhdr->frame_ctl, WIFI_BEACON);
+	_rtw_memcpy(wlanhdr->addr1, bcast_addr, ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr2, ap_addr, ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr3, ap_addr, ETH_ALEN);
+
+	buf = skb_put(skb, _FIXED_IE_LENGTH_);
+	/* timestamp: 8 bytes */
+	/* beacon interval: 2 bytes */
+	*(u16*)(buf + 8) = ap_beacon_interval;
+	/* capability info: 2 bytes */
+	*(u16*)(buf + 10) = ap_capability_info;
+
+	/* SSID */
+	add_ie(skb, _SSID_IE_, sizeof(ap_ssid), ap_ssid);
+
+	/* supported rates... */
+	add_ie(skb, _SUPPORTEDRATES_IE_, sizeof(ap_datarate), ap_datarate);
+
+	/* DS parameter set */
+	add_ie(skb, _DSSET_IE_, 1, &ap_ch);
+
+	skb_queue_tail(&d->fakeap.rxq, skb);
+	_set_workitem(&d->fakeap.work);
+}
+
+static void rx_prepare(struct _ADAPTER *adapter, union recv_frame *r)
+{
+	struct rx_pkt_attrib *a;
+
+
+	/* init recv_frame */
+	_rtw_init_listhead(&r->u.hdr.list);
+	r->u.hdr.len = 0;
+	a = &r->u.hdr.attrib;
+	_rtw_memset(a, 0, sizeof(*a));
+
+	/* fill sec related attrib, iv_len and icv_len will be filled by
+	 * validate_recv_data_frame() */
+	a->crc_err = 0;
+	a->icv_err = 0;
+	a->encrypt = 0;
+
+	/* fill rx pkt attrib */
+	a->hdrlen = 0;
+	a->bw = CHANNEL_WIDTH_MAX;
+	a->pkt_len = 0;
+	a->pkt_rpt_type = NORMAL_RX;
+	a->drvinfo_sz = 0;
+	a->bdecrypted = 0;
+	a->qos = 0;
+	a->priority = 0;
+	a->amsdu = 0;
+	a->mdata = 0;
+	a->mfrag = 0;
+	a->seq_num = 0;
+	a->frag_num = 0;
+	a->data_rate = DESC_RATE6M;
+	a->ppdu_cnt = 1;
+	a->free_cnt = 0;
+}
+
+static union recv_frame* gen_probersp(struct _ADAPTER *a)
+{
+	union recv_frame *r;
+	u32 len;
+	struct sk_buff *skb;
+	struct rtw_ieee80211_hdr *wlanhdr;
+	u8 *buf;
+	u8 hdrlen;
+	u8 ielen;
+
+
+	r = rtw_alloc_recvframe(&a->recvpriv.free_recv_queue);
+	if (!r) {
+		RTW_ERR("%s: alloc recvframe failed!\n", __func__);
+		return NULL;
+	}
+	rtw_init_recvframe(r, &a->recvpriv);
+	rx_prepare(a, r);
+
+	hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	len = RXDESC_OFFSET + hdrlen + _CAPABILITY_ + _STATUS_CODE_ + _ASOC_ID_
+	      + 2 + sizeof(ap_ssid)
+	      + 2 + sizeof(ap_datarate)
+	      + 2 + 1;
+	skb = rtw_skb_alloc(len);
+	_rtw_memset(skb->data, 0, len);
+	skb->dev = a->pnetdev;
+	skb_reserve(skb, RXDESC_OFFSET);
+	r->u.hdr.pkt = skb;
+
+	buf = skb_put(skb, hdrlen);
+	wlanhdr = (struct rtw_ieee80211_hdr*)buf;
+	set_frame_sub_type(&wlanhdr->frame_ctl, WIFI_PROBERSP);
+	_rtw_memcpy(wlanhdr->addr1, adapter_mac_addr(a), ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr2, ap_addr, ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr3, ap_addr, ETH_ALEN);
+
+	buf = skb_put(skb, _FIXED_IE_LENGTH_);
+	/* timestamp: 8 bytes */
+	/* beacon interval: 2 bytes */
+	*(u16*)(buf + 8) = ap_beacon_interval;
+	/* capability info: 2 bytes */
+	*(u16*)(buf + 10) = ap_capability_info;
+
+	/* SSID */
+	add_ie(skb, _SSID_IE_, sizeof(ap_ssid), ap_ssid);
+
+	/* supported rates... */
+	add_ie(skb, _SUPPORTEDRATES_IE_, sizeof(ap_datarate), ap_datarate);
+
+	/* DS parameter set */
+	add_ie(skb, _DSSET_IE_, 1, &ap_ch);
+
+	/* handle r->u.hdr.attrib */
+
+	/* handle recv_frame pointer */
+	r->u.hdr.len = skb->len;
+	r->u.hdr.rx_head = skb->head;
+	r->u.hdr.rx_data = skb->data;
+	r->u.hdr.rx_tail = skb_tail_pointer(skb);
+	r->u.hdr.rx_end = skb_end_pointer(skb);
+
+	return r;
+}
+
+static union recv_frame* gen_beacon(struct _ADAPTER *a, struct sk_buff *beacon)
+{
+	union recv_frame *r;
+	u32 len;
+	struct sk_buff *skb;
+	struct rtw_ieee80211_hdr *wlanhdr;
+	u8 *buf;
+	u8 hdrlen;
+
+
+	r = rtw_alloc_recvframe(&a->recvpriv.free_recv_queue);
+	if (!r) {
+		RTW_ERR("%s: alloc recvframe failed!\n", __func__);
+		return NULL;
+	}
+	rtw_init_recvframe(r, &a->recvpriv);
+	rx_prepare(a, r);
+
+	hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	len = RXDESC_OFFSET + beacon->len;
+	skb = rtw_skb_alloc(len);
+	_rtw_memset(skb->data, 0, len);
+	skb->dev = a->pnetdev;
+	skb_reserve(skb, RXDESC_OFFSET);
+	r->u.hdr.pkt = skb;
+
+	buf = skb_put(skb, beacon->len);
+	_rtw_memcpy(buf, beacon->data, beacon->len);
+
+	/* handle r->u.hdr.attrib */
+
+	/* handle recv_frame pointer */
+	r->u.hdr.len = skb->len;
+	r->u.hdr.rx_head = skb->head;
+	r->u.hdr.rx_data = skb->data;
+	r->u.hdr.rx_tail = skb_tail_pointer(skb);
+	r->u.hdr.rx_end = skb_end_pointer(skb);
+
+	return r;
+}
+
+static union recv_frame *gen_auth(struct _ADAPTER *a, u8 *bssid)
+{
+	union recv_frame *r;
+	u32 len;
+	struct sk_buff *skb;
+	struct rtw_ieee80211_hdr *wlanhdr;
+	u8 *buf;
+	u8 hdrlen;
+
+
+	r = rtw_alloc_recvframe(&a->recvpriv.free_recv_queue);
+	if (!r) {
+		RTW_ERR("%s: alloc recvframe failed!\n", __func__);
+		return NULL;
+	}
+	rtw_init_recvframe(r, &a->recvpriv);
+	rx_prepare(a, r);
+
+	hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	len = RXDESC_OFFSET + hdrlen
+	      + _AUTH_ALGM_NUM_ + _AUTH_SEQ_NUM_ + _STATUS_CODE_;
+	skb = rtw_skb_alloc(len);
+	_rtw_memset(skb->data, 0, len);
+	skb->dev = a->pnetdev;
+	skb_reserve(skb, RXDESC_OFFSET);
+	r->u.hdr.pkt = skb;
+
+	buf = skb_put(skb, hdrlen);
+	wlanhdr = (struct rtw_ieee80211_hdr*)buf;
+	set_frame_sub_type(&wlanhdr->frame_ctl, WIFI_AUTH);
+	_rtw_memcpy(wlanhdr->addr1, adapter_mac_addr(a), ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr2, bssid, ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr3, bssid, ETH_ALEN);
+
+	/* setting auth algo number */
+	buf = skb_put(skb, _AUTH_ALGM_NUM_);
+	*(u16*)buf = cpu_to_le16(0);;
+
+	/* setting auth seq number */
+	buf = skb_put(skb, _AUTH_SEQ_NUM_);
+	*(u16*)buf = cpu_to_le16(2);
+
+	/* setting status code... */
+	buf = skb_put(skb, _STATUS_CODE_);
+	*(u16*)buf = cpu_to_le16(0);
+
+	/* handle r->u.hdr.attrib */
+
+	/* handle recv_frame pointer */
+	r->u.hdr.len = skb->len;
+	r->u.hdr.rx_head = skb->head;
+	r->u.hdr.rx_data = skb->data;
+	r->u.hdr.rx_tail = skb_tail_pointer(skb);
+	r->u.hdr.rx_end = skb_end_pointer(skb);
+
+	return r;
+}
+
+static union recv_frame *gen_assocrsp(struct _ADAPTER *a, u8 *bssid)
+{
+	union recv_frame *r;
+	u32 len;
+	struct sk_buff *skb;
+	struct rtw_ieee80211_hdr *wlanhdr;
+	u8 *buf;
+	u8 hdrlen;
+
+
+	r = rtw_alloc_recvframe(&a->recvpriv.free_recv_queue);
+	if (!r) {
+		RTW_ERR("%s: alloc recvframe failed!\n", __func__);
+		return NULL;
+	}
+	rtw_init_recvframe(r, &a->recvpriv);
+	rx_prepare(a, r);
+
+	hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	len = RXDESC_OFFSET + hdrlen
+	      + _CAPABILITY_ + _STATUS_CODE_ + _ASOC_ID_
+	      + 2 + sizeof(ap_datarate);
+	skb = rtw_skb_alloc(len);
+	_rtw_memset(skb->data, 0, len);
+	skb->dev = a->pnetdev;
+	skb_reserve(skb, RXDESC_OFFSET);
+	r->u.hdr.pkt = skb;
+
+	buf = skb_put(skb, hdrlen);
+	wlanhdr = (struct rtw_ieee80211_hdr*)buf;
+	set_frame_sub_type(&wlanhdr->frame_ctl, WIFI_ASSOCRSP);
+	_rtw_memcpy(wlanhdr->addr1, adapter_mac_addr(a), ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr2, bssid, ETH_ALEN);
+	_rtw_memcpy(wlanhdr->addr3, bssid, ETH_ALEN);
+
+
+	/* capability info: 2 bytes */
+	buf = skb_put(skb, _CAPABILITY_);
+	*(u16*)buf = ap_capability_info;
+
+	/* status code: 2 bytes */
+	buf = skb_put(skb, _STATUS_CODE_);
+	*(u16*)buf = 0;
+
+	/* AID: 2 bytes */
+	buf = skb_put(skb, _ASOC_ID_);
+	*(u16*)buf = 100;
+
+	/* supported rates... */
+	add_ie(skb, _SUPPORTEDRATES_IE_, sizeof(ap_datarate), ap_datarate);
+
+	/* handle r->u.hdr.attrib */
+
+	/* handle recv_frame pointer */
+	r->u.hdr.len = skb->len;
+	r->u.hdr.rx_head = skb->head;
+	r->u.hdr.rx_data = skb->data;
+	r->u.hdr.rx_tail = skb_tail_pointer(skb);
+	r->u.hdr.rx_end = skb_end_pointer(skb);
+
+//	recvframe_put(r, buflen);
+//	pre_recv_entry(r, NULL);
+
+	return r;
+}
+
+static union recv_frame* fakeap_dispatcher(struct _ADAPTER *a,
+					   struct sk_buff *skb)
+{
+	u8 *pframe;
+	union recv_frame *rframe = NULL;
+
+
+	pframe = skb->data;
+	RTW_INFO("A1-" MAC_FMT "\n", MAC_ARG(GetAddr1Ptr(pframe)));
+	RTW_INFO("A2-" MAC_FMT "\n", MAC_ARG(get_addr2_ptr(pframe)));
+	RTW_INFO("A3-" MAC_FMT "\n", MAC_ARG(GetAddr3Ptr(pframe)));
+
+	if (GetFrameType(pframe) != WIFI_MGT_TYPE)
+		return NULL;
+
+	switch (get_frame_sub_type(pframe)) {
+	case WIFI_AUTH:
+		{
+		u16 algo, seq;
+
+		algo = le16_to_cpu(*(u16 *)(pframe + WLAN_HDR_A3_LEN));
+		seq = le16_to_cpu(*(u16 *)(pframe + WLAN_HDR_A3_LEN + 2));
+
+		if ((algo == 0) && (seq == 1))
+			rframe = gen_auth(a, GetAddr1Ptr(pframe));
+		}
+		break;
+
+	case WIFI_ASSOCREQ:
+	case WIFI_REASSOCREQ:
+		rframe = gen_assocrsp(a, GetAddr1Ptr(pframe));
+		break;
+
+	case WIFI_PROBEREQ:
+		rframe = gen_probersp(a);
+		break;
+
+	case WIFI_BEACON:
+		rframe = gen_beacon(a, skb);
+
+	default:
+		break;
+	}
+
+	return rframe;
+}
+
+extern sint validate_recv_frame(struct _ADAPTER*, union recv_frame**);
+void rtw_fakeap_work(struct work_struct *work)
+{
+	struct dvobj_priv *d;
+	struct _ADAPTER *a;
+	struct sk_buff_head *pktq;
+	struct sk_buff *skb, *tmp;
+	u8 *ta, *ra;
+	union recv_frame *rframe;
+	int i;
+
+
+	d = container_of(work, struct dvobj_priv, fakeap.work);
+
+	pktq = &d->fakeap.rxq;
+	skb_queue_walk_safe(pktq, skb, tmp) {
+		__skb_unlink(skb, pktq);
+
+		ta = get_addr2_ptr(skb->data);
+		ra = GetAddr1Ptr(skb->data);
+		if (is_broadcast_mac_addr(ra)
+		    && (_rtw_memcmp(ta, ap_addr, ETH_ALEN)==_TRUE)
+		    && (get_frame_sub_type(skb->data) == WIFI_BEACON)) {
+			for (i = 0; i < d->iface_nums; i++) {
+				a = d->padapters[i];
+				if (a) {
+					rframe = fakeap_dispatcher(a, skb);
+					if (rframe) {
+						validate_recv_frame(a, &rframe);
+						rframe = NULL;
+					}
+				}
+			}
+		} else {
+			a = rtw_get_iface_by_macddr(dvobj_get_primary_adapter(d),
+						    ta);
+			if (a)
+				rframe = fakeap_dispatcher(a, skb);
+		}
+		rtw_skb_free(skb);
+		if (rframe) {
+			validate_recv_frame(a, &rframe);
+			rframe = NULL;
+		}
+	}
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm.c
new file mode 100644
index 000000000000..97f5dddaca5e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm.c
@@ -0,0 +1,2043 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * Author: vincent_fann@realtek.com
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+#include <rtw_fsm.h>
+
+#ifdef CONFIG_RTW_FSM
+
+/* #define USE_PHL_CMD_DISPR */
+
+#define MAX_OBJ_NUM 8
+#define CLOCK_NUM 3
+#define CLOCK_UNIT 10
+#define IS_CLK_OFF(clk) (clk->remain < 0) /* Negative value means disabled */
+#define IS_CLK_ON(clk) (clk->remain >= 0)
+#define IS_CLK_EXP(clk) (clk->remain < (CLOCK_UNIT >> 1)) /* expire */
+
+#ifndef MAX
+#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y))
+#endif
+#define pstr(s) (s +strlen((u8 *)s))
+#define lstr(s, l) (size_t)(l - strlen((u8 *)s))
+
+static struct fsm_obj *fsm_get_obj(struct fsm_main *fsm, u32 oid, u16 cid);
+
+//#define FSM_DBG_MEM_OVERWRITE
+#ifdef FSM_DBG_MEM_OVERWRITE
+void *fsm_kmalloc(u32 sz)
+{
+	char *ptr;
+
+	ptr = kmalloc(sz+4, GFP_KERNEL);
+	memset(ptr+sz, 0xff, 4);
+	RTW_INFO("+AA %p %d\n", ptr, sz);
+	return ptr;
+}
+
+void fsm_kfree(void *ptr, u32 sz)
+{
+	u32 ptn = 0xffffffff;
+	u32 *p = (u32 *)(ptr+sz);
+
+	RTW_INFO("-AA %p %d", ptr, sz);
+	if ((*p&ptn) != ptn) {
+		RTW_ERR("- %p %d", ptr, sz);
+		RTW_ERR("OVER WRITE %x\n", ptn);
+	}
+	kfree(ptr);
+}
+#define _os_kmem_alloc(a, b) fsm_kmalloc(b)
+#define _os_kmem_free(a, b, c) fsm_kfree(b, c)
+#endif
+
+struct fsm_event_ent int_event_tbl[] = {
+	EV_DBG(FSM_INT_EV_MASK),
+	EV_DBG(FSM_EV_ABORT),
+	EV_DBG(FSM_EV_TIMER_EXPIRE),
+	EV_DBG(FSM_EV_END),
+
+	//EV_DBG(FSM_EV_SWITCH_IN),
+	//EV_DBG(FSM_EV_SWITCH_OUT),
+	EV_DBG(FSM_EV_STATE_IN),
+	EV_DBG(FSM_EV_STATE_OUT),
+
+	EV_DBG(FSM_EV_NEW_OBJ),
+	EV_DBG(FSM_EV_DEL_OBJ),
+
+	EV_DBG(FSM_EV_INT_MAX),
+
+	EV_DBG(FSM_GBL_EV_MASK),
+	EV_DBG(FSM_EV_CONNECTED),
+	EV_DBG(FSM_EV_CONNECT_FAIL),
+	EV_DBG(FSM_EV_DISCONNECTED),
+	EV_DBG(FSM_EV_SCAN_START),
+	EV_DBG(FSM_EV_SCAN_DONE),
+
+	EV_DBG(FSM_EV_GBL_MAX)
+};
+
+/*
+ * FSM status
+ */
+enum FSM_STATUS {
+	FSM_STATUS_NONE,	/* default value */
+
+	FSM_STATUS_INITIALIZED, /* insert module ok,
+				 * mem/queue/timer were allocated
+				 */
+
+	FSM_STATUS_ENABLE,	/* run msg */
+
+	FSM_STATUS_DISABLE,	/* Does NOT run and receiv msg,
+				 * thread will remove all NON INT event
+				 * after dequeue msg
+				 */
+};
+
+/* @obj: obj that will be infomred to when time's up
+ * @counter: clock time period
+ * @event: event that will delivered when time's up
+ * @end: end time
+ * @pause: stop countdown
+ */
+struct fsm_clk {
+	u16 event;
+	void *priv;
+	u8 pause;
+	u32 start;
+	u32 end;
+	int remain; /* ms */
+};
+
+struct fsm_queue {
+	struct	list_head q;
+	_lock lock;
+};
+
+struct fsm_main {
+	struct list_head list;
+	char name[FSM_NAME_LEN];
+	u8 status;
+	u8 obj_cnt;
+	u8 en_clock_num;
+
+	_timer fsm_timer; /* unit in ms */
+
+	struct fsm_root *root;
+	struct fsm_queue obj_queue;
+	struct fsm_queue msg_queue;
+	struct fsm_queue clk_queue;
+
+	/* extra custom queue; for fsm private */
+	struct fsm_queue ext_queue;
+
+	struct rtw_fsm_tb tb;
+};
+
+/* @obj_id: object id
+ * @state: current state
+ * @prive: object's private date
+ */
+struct fsm_obj {
+	struct list_head list;
+	u32 oid;
+	u16 cid; /* customer id */
+	u8 state;
+	u8 valid; /* framework */
+	char name[FSM_NAME_LEN];
+	struct fsm_clocks *clocks;
+	struct fsm_main *fsm;
+
+	void *custom_obj;
+	int custom_len; /* custom obj length */
+
+	struct sta_info *psta;
+};
+
+struct fsm_clocks {
+	struct list_head list;
+	struct fsm_clk clk[CLOCK_NUM];
+	struct fsm_obj *obj;
+};
+
+/* Main structure to handle all standalone fsm */
+struct fsm_root {
+	struct task_struct * thread;
+	struct list_head list;
+	struct fsm_queue q_share_thd;
+	u8 thread_should_stop;
+
+	_adapter *a;
+	u32 oid_seq; /* starts from 1 */
+
+	_sema msg_ready;
+
+	u32 status; /* refer to enum FSM_ROOT_STATUS_FLAGS */
+};
+
+/* Static function porto type */
+static int fsm_handler(struct fsm_main *fsm);
+static char *fsm_state_name(struct fsm_main *fsm, u8 state);
+static u8 get_evt_level(struct fsm_main *fsm, u16 event);
+static void fsm_destory_obj(struct fsm_obj *obj);
+static int fsm_start_fsm(struct fsm_main *fsm);
+static int fsm_stop_fsm(struct fsm_main *fsm);
+static void fsm_user_evt_handler(struct fsm_main *fsm);
+static int fsm_enqueue_list(struct fsm_queue *queue, struct list_head *list);
+
+u8 fsm_post_event_hdl(_adapter *padapter, u8 *pbuf)
+{
+#ifdef USE_PHL_CMD_DISPR
+	struct fsm_msg *msg = (struct fsm_msg *)pbuf;
+
+	fsm_enqueue_list(&msg->fsm->msg_queue, &msg->list);
+	fsm_user_evt_handler(msg->fsm);
+#endif
+	return H2C_SUCCESS;
+}
+
+static void fsm_status_set(struct fsm_main *fsm, enum FSM_STATUS status)
+{
+	fsm->status = status;
+}
+
+static enum FSM_STATUS fsm_status(struct fsm_main *fsm)
+{
+	return fsm->status;
+}
+
+/* unit ms */
+u32 rtw_fsm_time_pass(u32 start)
+{
+	u32 now = rtw_systime_to_ms(rtw_get_current_time());
+	u32 pass;
+
+	if (start <= now)
+		pass = now - start;
+	else
+		pass = 0xffffffff - start + now;
+
+	return pass;
+}
+
+u32 rtw_fsm_time_left(u32 start, u32 end)
+{
+	u32 total, pass;
+	int left = 0;
+
+	pass = rtw_fsm_time_pass(start);
+
+	if (end >= start)
+		total = end - start;
+	else
+		total = 0xffffffff - start + end;
+
+	left = total - pass;
+
+	if (left < 0)
+		left = 0;
+
+	return (u32)left;
+}
+
+static struct fsm_msg *fsm_dequeue_msg(struct fsm_main *fsm)
+{
+	struct fsm_msg *msg;
+
+	if (list_empty(&fsm->msg_queue.q))
+		return NULL;
+
+	_rtw_spinlock_bh(&fsm->msg_queue.lock);
+	msg = list_first_entry(&fsm->msg_queue.q, struct fsm_msg, list);
+	list_del(&msg->list);
+	_rtw_spinunlock_bh(&fsm->msg_queue.lock);
+	return msg;
+}
+
+#if 0
+static struct fsm_clk *fsm_dequeue_clk(struct fsm_main *fsm)
+{
+	struct fsm_clk *clk;
+
+	if (list_empty(&fsm->clk_queue.q))
+		return NULL;
+
+	_rtw_spinlock_bh(&fsm->clk_queue.lock);
+	clk = list_first_entry(&fsm->clk_queue.q, struct fsm_clk, list);
+	list_del(&clk->list);
+	_rtw_spinunlock_bh(&fsm->clk_queue.lock);
+	return clk;
+}
+#endif
+
+/* For EXTERNAL application to get adapter (expose) */
+void *fsm_to_priv(struct fsm_main *fsm)
+{
+	return fsm->tb.priv; /* xxx_priv */
+}
+
+struct sta_info *obj_to_sta(struct fsm_obj *obj)
+{
+	return obj->psta;
+}
+
+_adapter *fsm_to_adapter(struct fsm_main *fsm)
+{
+	return fsm->root->a;
+}
+
+/* For EXTERNAL application to enqueue message to extra queue (expose)
+ *
+ * @fsm: fsm that object belonged to
+ * @msg: message to be enqueued
+ * @to_head: enqueue message to the head
+ */
+int rtw_fsm_enqueue_ext(struct fsm_main *fsm, struct fsm_msg *msg, u8 to_head)
+{
+	struct fsm_queue *queue = &fsm->ext_queue;
+
+	_rtw_spinlock_bh(&queue->lock);
+	if (to_head)
+		list_add(&msg->list, &queue->q);
+	else
+		list_add_tail(&msg->list, &queue->q);
+	_rtw_spinunlock_bh(&queue->lock);
+
+	return 0;
+}
+
+/* For EXTERNAL application to dequeue message from extra queue (expose)
+ *
+ * @fsm: fsm that object belonged to
+ */
+struct fsm_msg *rtw_fsm_dequeue_ext(struct fsm_main *fsm)
+{
+	struct fsm_msg *msg;
+
+	if (list_empty(&fsm->ext_queue.q))
+		return NULL;
+
+	_rtw_spinlock_bh(&fsm->ext_queue.lock);
+	msg = list_first_entry(&fsm->ext_queue.q, struct fsm_msg, list);
+	list_del(&msg->list);
+	_rtw_spinunlock_bh(&fsm->ext_queue.lock);
+	return msg;
+}
+
+/* For EXTERNAL application to dequeue message from extra queue (expose)
+ *
+ * @fsm: fsm that object belonged to
+ */
+int rtw_fsm_is_ext_queue_empty(struct fsm_main *fsm)
+{
+	return list_empty(&fsm->ext_queue.q);
+}
+
+static u32 fsm_new_oid(struct fsm_main *fsm)
+{
+	/* TODO: how to handle redendent oid ? */
+	do {
+		fsm->root->oid_seq++;
+	} while (fsm->root->oid_seq == 0);
+
+	return fsm->root->oid_seq;
+}
+
+static int fsm_enqueue_list(struct fsm_queue *queue, struct list_head *list)
+{
+	_rtw_spinlock_bh(&queue->lock);
+	list_add_tail(list, &queue->q);
+	_rtw_spinunlock_bh(&queue->lock);
+	return 0;
+}
+
+static int fsm_state_run(struct fsm_obj *obj, u16 event, void *param)
+{
+	struct fsm_main *fsm = obj->fsm;
+
+	FSM_EV_MSG_(fsm, get_evt_level(fsm, event),
+		"%s %-18s    %s\n", obj->name,
+		fsm_state_name(fsm, obj->state), fsm_evt_name(obj->fsm, event));
+
+	return fsm->tb.state_tbl[obj->state].fsm_func(obj->custom_obj,
+		event, param);
+}
+
+static void fsm_remove_all_queuing_msg(struct fsm_main *fsm)
+{
+	struct fsm_msg *msg;
+	//struct fsm_evt *evt;
+	struct fsm_obj *obj;
+
+	/* go through msg queue and free everything */
+	while ((msg = fsm_dequeue_msg(fsm)) != NULL) {
+		obj = fsm_get_obj(fsm, 0, msg->cid);
+		if (msg->param)
+			rtw_mfree((void *)msg->param, msg->param_sz);
+		rtw_mfree((void *)msg, sizeof(*msg));
+	}
+
+	/* go through event queue and free everything */
+#if 0
+	while ((evt = fsm_dequeue_clk(fsm)) != NULL) {
+		if (evt->param)
+			rtw_mfree((void *)evt->param, evt->param_sz);
+		rtw_mfree((void *)evt, sizeof(*evt));
+	}
+#endif
+
+	/* go through ext queue and free everything */
+	while ((msg = rtw_fsm_dequeue_ext(fsm)) != NULL) {
+		if (msg->param)
+			rtw_mfree((void *)msg->param, msg->param_sz);
+		rtw_mfree((void *)msg, sizeof(*msg));
+	}
+}
+
+static int fsm_abort_all_running_obj(struct fsm_main *fsm)
+{
+	struct fsm_obj *obj;
+
+	if (list_empty(&fsm->obj_queue.q))
+		return 0;
+
+	list_for_each_entry(obj, &fsm->obj_queue.q, list) {
+		fsm_gen_msg(obj, NULL, 0, FSM_EV_ABORT);
+	}
+
+	return 0;
+}
+
+u8 fsm_dbg_lv(struct fsm_main *fsm, u8 level)
+{
+	if (fsm->tb.dbg_level >= level)
+		return fsm->tb.dbg_level;
+
+	return 0;
+}
+
+u8 fsm_evt_lv(struct fsm_main *fsm, u8 level)
+{
+	if (fsm->tb.evt_level >= level)
+		return fsm->tb.evt_level;
+
+	return 0;
+}
+
+static u8 get_evt_level(struct fsm_main *fsm, u16 event)
+{
+	u16 ev;
+
+	/* fsm global event */
+	if (event & FSM_GBL_EV_MASK) {
+		ev = (u8)((event & ~(FSM_GBL_EV_MASK)) + 1 +
+			(FSM_EV_INT_MAX & ~(FSM_INT_EV_MASK)));
+		return int_event_tbl[ev].evt_level;
+	}
+
+	/* fsm internal event */
+	if (event & FSM_INT_EV_MASK) {
+		ev = (u8)(event & ~(FSM_INT_EV_MASK));
+		return int_event_tbl[ev].evt_level;
+	}
+
+	if (event == FSM_EV_UNKNOWN)
+		return FSM_LV_INFO;
+
+	if (event > fsm->tb.max_event)
+		return FSM_LV_INFO;
+
+	/* user event */
+	return fsm->tb.evt_tbl[event].evt_level;
+}
+
+static void fsm_init_queue(struct fsm_queue *queue)
+{
+	INIT_LIST_HEAD(&queue->q);
+	_rtw_spinlock_init(&queue->lock);
+}
+
+static void fsm_deinit_queue(struct fsm_queue *queue)
+{
+	_rtw_spinlock_free(&queue->lock);
+}
+
+
+#ifndef USE_PHL_CMD_DISPR
+int fsm_thread_share(void *param)
+{
+	struct fsm_obj *obj;
+	struct fsm_main *fsm, *fsm_t;
+	struct fsm_root *root = (struct fsm_root *)param;
+	int rtn;
+
+	while (1) {
+
+		//while (down_timeout(&root->msg_ready, 100)) {
+		rtn = down_interruptible(&root->msg_ready);
+
+		if (root->thread_should_stop)
+			goto exit;
+
+		list_for_each_entry(fsm, &root->q_share_thd.q, list) {
+			fsm_handler(fsm);
+		}
+	}
+exit:
+	while (!kthread_should_stop()) {
+		RTW_INFO("fsm: [FSM] thread wait stop\n");
+		rtw_msleep_os(10);
+	}
+	RTW_INFO("fsm: [FSM] thread down\n");
+
+	return 0;
+}
+#endif
+
+/* cid customer cid */
+static struct fsm_obj *fsm_get_obj(struct fsm_main *fsm, u32 oid, u16 cid)
+{
+	struct fsm_obj *obj, *obj_t;
+
+	if (list_empty(&fsm->obj_queue.q))
+		return NULL;
+
+	_rtw_spinlock_bh(&fsm->obj_queue.lock);
+	list_for_each_entry(obj, &fsm->obj_queue.q, list) {
+		if ((oid && (oid == obj->oid)) ||
+		    (cid && (cid == obj->cid))) {
+			_rtw_spinunlock_bh(&fsm->obj_queue.lock);
+			return obj;
+		}
+	}
+	_rtw_spinunlock_bh(&fsm->obj_queue.lock);
+	return NULL;
+}
+
+struct fsm_msg *fsm_new_msg(struct fsm_obj *obj, u16 event)
+{
+	struct fsm_msg *msg = NULL;
+
+	msg = (struct fsm_msg *)rtw_zmalloc(sizeof(*msg));
+
+	if (msg == NULL)
+		return NULL;
+
+	_rtw_memset(msg, 0, sizeof(*msg));
+	msg->event = event;
+
+	if (obj) {
+		msg->fsm = obj->fsm;
+		msg->oid = obj->oid;
+		msg->cid = obj->cid;
+	}
+	return msg;
+}
+
+#ifdef USE_PHL_CMD_DISPR
+static int rtw_fsm_send_cmd_msg(struct fsm_main *fsm, struct fsm_msg *msg)
+{
+	struct cmd_obj *pcmd = NULL;
+	_adapter *padapter = fsm_to_adapter(fsm);
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+	pcmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd == NULL)
+		goto fail;
+
+	pcmd->padapter = padapter;
+	init_h2fwcmd_w_parm_no_rsp(pcmd, msg, CMD_RM_POST_EVENT);
+	return rtw_enqueue_cmd(pcmdpriv, pcmd);
+fail:
+	if (msg->param && msg->param_sz)
+		rtw_mfree(msg->param, msg->param_sz);
+	rtw_mfree(msg, sizeof(*msg));
+
+	if (pcmd)
+		rtw_mfree(pcmd, sizeof(*pcmd));
+	return _FAIL;
+}
+#endif
+
+int fsm_send_msg(struct fsm_main *fsm, struct fsm_msg *msg)
+{
+
+	if ((fsm_status(fsm) != FSM_STATUS_ENABLE) &&
+		!(msg->event & FSM_INT_EV_MASK)) {
+		RTW_WARN("fsm: %s is out of service, drop message %s!\n",
+			fsm->name, fsm_evt_name(fsm, msg->event));
+		if (msg->param && msg->param_sz)
+			rtw_mfree(msg->param, msg->param_sz);
+		rtw_mfree(msg, sizeof(*msg));
+		return _FAIL;
+
+	}
+#ifdef USE_PHL_CMD_DISPR
+	rtw_fsm_send_cmd_msg(fsm, msg);
+#else
+	fsm_enqueue_list(&fsm->msg_queue, &msg->list);
+	_rtw_up_sema(&fsm->root->msg_ready);
+#endif
+	return _SUCCESS;
+}
+
+void fsm_timer_callback(void *context)
+{
+	struct fsm_main *fsm = (struct fsm_main *)context;
+	struct fsm_clk *clk;
+	struct fsm_clocks *clocks;
+	int i = 0;
+
+	_set_timer(&fsm->fsm_timer, CLOCK_UNIT);
+
+	_rtw_spinlock_bh(&fsm->clk_queue.lock);
+
+	if (fsm->en_clock_num == 0)
+		goto done;
+
+	/* go through clock and descrease timer
+	 * if timer was expired, issue event
+	 */
+	list_for_each_entry(clocks, &fsm->clk_queue.q, list) {
+		for (i = 0; i < CLOCK_NUM; i++) {
+
+			clk = &clocks->clk[i];
+
+			if (IS_CLK_OFF(clk) || clk->pause)
+				continue;
+
+			clk->remain = (int)rtw_fsm_time_left(clk->start, clk->end);
+
+			/* timer expired */
+			if (!IS_CLK_EXP(clk))
+				continue;
+
+			FSM_TRACE_(fsm, "%s: expire in %d ms\n",
+				fsm_evt_name(clocks->obj->fsm, clk->event),
+				rtw_fsm_time_pass(clk->start));
+
+			clk->end = 0;
+			clk->remain = -1;
+			/* send message to obj */
+
+			/* check fsm status before posting */
+			if (fsm_status(fsm) == FSM_STATUS_ENABLE) {
+				fsm_gen_msg(clocks->obj, NULL, 0, clk->event);
+				if (--fsm->en_clock_num == 0)
+					break;
+			}
+		}
+	}
+done:
+	_rtw_spinunlock_bh(&fsm->clk_queue.lock);
+}
+
+/* allocate and init fsm resource */
+struct fsm_main *rtw_fsm_register_fsm(struct fsm_root *root,
+	const char *name, struct rtw_fsm_tb *tb)
+{
+	struct fsm_main *fsm;
+
+	/* check event table */
+	if (tb->evt_tbl[tb->max_event-1].event != tb->max_event-1) {
+		RTW_ERR("Event mismatch ? Is max event = %d != %d ?\n",
+			tb->evt_tbl[tb->max_event-1].event,
+			tb->max_event-1);
+		return NULL;
+	}
+
+	/* check state table */
+	if (tb->state_tbl[tb->max_state-1].state != tb->max_state-1) {
+		RTW_ERR("State mismatch ? Is max state = %d != %d) ?\n",
+			tb->state_tbl[tb->max_state-1].state,
+			tb->max_state-1);
+		return NULL;
+	}
+
+	fsm = (struct fsm_main *)rtw_zmalloc(sizeof(*fsm));
+
+	if (fsm == NULL)
+		return NULL;
+
+	_rtw_memset(fsm, 0, sizeof(*fsm));
+	_rtw_memcpy(&fsm->tb, (void *)tb, sizeof(*tb));
+	_rtw_memcpy(&fsm->name, (void *)name,
+		MIN(FSM_NAME_LEN-1, strlen((u8 *)name)));
+
+	fsm->root = root;
+	fsm_init_queue(&(fsm->obj_queue));
+	fsm_init_queue(&(fsm->msg_queue));
+	fsm_init_queue(&(fsm->clk_queue));
+	fsm_init_queue(&(fsm->ext_queue));
+
+	if (tb->init_priv)
+		tb->init_priv(tb->priv);
+
+	rtw_init_timer(&fsm->fsm_timer, fsm_timer_callback, fsm);
+
+	/* link fsm_main to fsm_root */
+	fsm_enqueue_list(&root->q_share_thd, &fsm->list);
+
+	FSM_INFO_(fsm, "fsm: [%s] %s()\n", fsm->name, __func__);
+	fsm_status_set(fsm, FSM_STATUS_INITIALIZED);
+	return fsm;
+}
+
+void fsm_deinit_fsm(struct fsm_main *fsm)
+{
+	_adapter *a;
+	struct fsm_obj *obj, *obj_t;
+
+	if (!fsm)
+		return;
+
+	_cancel_timer_ex(&fsm->fsm_timer);
+	a = fsm_to_adapter(fsm);
+
+	if (!list_empty(&fsm->obj_queue.q)) {
+		list_for_each_entry_safe(obj, obj_t, &fsm->obj_queue.q, list) {
+			fsm_destory_obj(obj);
+		}
+	}
+
+	FSM_INFO_(fsm, "fsm: [%s] "FUNC_NDEV_FMT"\n", fsm->name, FUNC_NDEV_ARG(a->pnetdev));
+
+	fsm_deinit_queue(&(fsm->obj_queue));
+	fsm_deinit_queue(&(fsm->msg_queue));
+	fsm_deinit_queue(&(fsm->clk_queue));
+	fsm_deinit_queue(&(fsm->ext_queue));
+	rtw_mfree(fsm, sizeof(*fsm));
+
+	return;
+}
+
+char *fsm_evt_name(struct fsm_main *fsm, u16 event)
+{
+	u8 ev;
+
+	/* fsm global event */
+	if (event & FSM_GBL_EV_MASK) {
+		ev = (u8)((event & ~(FSM_GBL_EV_MASK)) + 1 +
+			(FSM_EV_INT_MAX & ~(FSM_INT_EV_MASK)));
+		return int_event_tbl[ev].name;
+	}
+
+	/* fsm internal event */
+	if (event & FSM_INT_EV_MASK) {
+		ev = (u8)(event & ~(FSM_INT_EV_MASK));
+		return int_event_tbl[ev].name;
+	}
+
+	if (event == FSM_EV_UNKNOWN)
+		return "FSM_EV_UNKNOWN";
+
+	if (event > fsm->tb.max_event)
+		return "undefine";
+
+	/* user event */
+	return fsm->tb.evt_tbl[event].name;
+}
+
+static char *fsm_state_name(struct fsm_main *fsm, u8 state)
+{
+	if (state > fsm->tb.max_state)
+		return "unknown";
+
+	return fsm->tb.state_tbl[state].name;
+}
+
+/* For EXTERNAL application to get custom obj (expose)
+ *
+ * @obj: cid to get obj
+ */
+void *rtw_fsm_get_obj(struct fsm_main *fsm, u16 cid)
+{
+	struct fsm_obj *obj = NULL;
+
+	obj = fsm_get_obj(fsm, 0, cid);
+
+	if (obj)
+		return obj->custom_obj;
+	return NULL;
+}
+
+/**  init obj internal variable
+ *
+ * @fsm: fsm that object belonged to
+ * default init to the 1st state in state_tbl
+
+ */
+static void fsm_obj_switch_in(struct fsm_obj *obj, struct fsm_msg *msg)
+{
+	struct fsm_main *fsm = obj->fsm;
+
+	if (obj->fsm->tb.active_obj)
+		obj->fsm->tb.active_obj(obj->fsm->tb.priv, obj->custom_obj);
+
+	/* default init to the 1st state in state_tbl */
+	obj->state = fsm->tb.state_tbl[0].state;
+
+	FSM_DBG_(fsm, "%s %-21s %s(cid=0x%04x oid=0x%08x)\n", obj->name,
+		fsm_state_name(fsm, obj->state), "new obj", obj->cid, obj->oid);
+
+	obj->valid = 1;
+	/* Make it alive! Hello obj! */
+	fsm_state_run(obj, FSM_EV_STATE_IN, msg->param);
+}
+
+/**  deinit obj internal variable
+ *
+ * @fsm: fsm that object belonged to
+ * default init to the 1st state in state_tbl
+
+ */
+static void fsm_obj_switch_out(struct fsm_obj *obj, struct fsm_msg *msg)
+{
+	struct fsm_main *fsm = obj->fsm;
+
+	if (fsm->tb.deactive_obj)
+		fsm->tb.deactive_obj(fsm->tb.priv, obj->custom_obj);
+
+	fsm_state_run(obj, FSM_EV_STATE_OUT, msg->param);
+
+	FSM_DBG_(fsm, "%s %-21s %s(cid=0x%04x oid=0x%08x)\n", obj->name,
+		fsm_state_name(fsm, obj->state), "del obj", obj->cid, obj->oid);
+
+	obj->valid = 0;
+	/* Let it go! Goodbye obj! */
+	fsm_destory_obj(obj);
+}
+
+/* For EXTERNAL application to new a fsm object (expose)
+ *
+ * @fsm: fsm that object belonged to
+ * @fsm_obj: obj param when calling FSM framework function
+ * @priv_len: custom obj length
+ *
+ * return value: NULL	:fail
+ *		 other	:cusomer obj handler (success)
+ */
+void *rtw_fsm_new_obj(struct fsm_main *fsm,
+	struct sta_info *psta, u16 cid, void **custom_obj, int sz)
+{
+	struct fsm_obj *obj;
+	struct fsm_clocks *clocks;
+	int i;
+
+	*custom_obj = NULL;
+	obj = (struct fsm_obj *)rtw_zmalloc(sizeof(*obj));
+	if (obj == NULL)
+		return NULL;
+	_rtw_memset(obj, 0, sizeof(*obj));
+
+	clocks = (struct fsm_clocks *)rtw_zmalloc(sizeof(*clocks));
+	if (clocks == NULL) {
+		rtw_mfree(obj, sizeof(*obj));
+		return NULL;
+	}
+	_rtw_memset(clocks, 0, sizeof(*clocks));
+	clocks->obj = obj;
+	for (i = 0; i < CLOCK_NUM; i++)
+		clocks->clk[i].remain = -1; /* Negative means disable */
+
+	obj->custom_obj = rtw_zmalloc(sz);
+	if (obj->custom_obj == NULL) {
+		rtw_mfree(clocks, sizeof(*clocks));
+		rtw_mfree(obj, sizeof(*obj));
+		return NULL;
+	}
+	_rtw_memset(obj->custom_obj, 0, sz);
+	obj->clocks = clocks;
+	obj->custom_len = sz;
+	obj->oid = fsm_new_oid(fsm);
+	obj->cid = cid;
+	obj->fsm = fsm;
+	obj->state = fsm->tb.state_tbl[0].state;
+	obj->psta = psta;
+
+	_rtw_memset(obj->name, 0, FSM_NAME_LEN);
+	snprintf(obj->name, FSM_NAME_LEN,
+		"%s-%04x", obj->fsm->name, obj->cid);
+	*custom_obj = obj->custom_obj;
+	fsm_enqueue_list(&fsm->obj_queue, &obj->list);
+	fsm_enqueue_list(&fsm->clk_queue, &clocks->list);
+
+	return obj;
+}
+
+/* For EXTERNAL application to destory a fsm object (expose)
+ *
+ * @fsm_obj: obj param when calling FSM framework function
+ */
+static void fsm_destory_obj(struct fsm_obj *obj)
+{
+	struct fsm_main *fsm = obj->fsm;
+	struct fsm_clocks *clocks = obj->clocks;
+
+	list_del(&obj->list);
+
+	/* free clocks */
+	_rtw_spinlock_bh(&fsm->clk_queue.lock);
+	list_del(&clocks->list);
+	_rtw_spinunlock_bh(&fsm->clk_queue.lock);
+
+	rtw_mfree(clocks, sizeof(struct fsm_clocks));
+
+	/* free custom_obj */
+	rtw_mfree(obj->custom_obj, obj->custom_len);
+
+	/* free fsm_obj */
+	rtw_mfree(obj, sizeof(*obj));
+}
+
+void fsm_del_obj(struct fsm_obj *obj)
+{
+	struct fsm_main *fsm = obj->fsm;
+
+	/* only allow to del an inactive obj */
+	if (obj->valid != 0) {
+		FSM_ERR_(obj->fsm, "%s: free an actived obj\n", fsm_obj_name(obj));
+		return;
+	}
+	fsm_destory_obj(obj);
+}
+
+void fsm_activate_obj(struct fsm_obj *obj)
+{
+	fsm_gen_msg(obj, NULL, 0, FSM_EV_NEW_OBJ);
+}
+
+void fsm_deactivate_obj(struct fsm_obj *obj)
+{
+	fsm_gen_msg(obj, NULL, 0, FSM_EV_DEL_OBJ);
+}
+
+bool fsm_is_alarm_off_ext(struct fsm_obj *obj, u8 id)
+{
+	struct fsm_clocks *clocks = obj->clocks;
+	struct fsm_clk *clk = &clocks->clk[id];
+
+	return IS_CLK_OFF(clk);
+}
+
+bool fsm_is_alarm_off(struct fsm_obj *obj)
+{
+	struct fsm_clocks *clocks = obj->clocks;
+	struct fsm_clk *clk = &clocks->clk[0];
+
+	return IS_CLK_OFF(clk);
+}
+
+static void _fsm_set_alarm(struct fsm_obj *obj, int ms,
+	u16 event, u8 id, void *priv)
+{
+	struct fsm_clocks *clocks = obj->clocks;
+	struct fsm_clk *clk = &clocks->clk[id];
+	u32 now;
+
+#if 0
+	if (ms == 0)
+		fsm_post_event(obj, event, priv);
+#endif
+
+	now = rtw_systime_to_ms(rtw_get_current_time());
+
+	_rtw_spinlock_bh(&obj->fsm->clk_queue.lock);
+	/* turn on clock from off */
+	if (IS_CLK_OFF(clk))
+		obj->fsm->en_clock_num++;
+
+	clk->event = event;
+	clk->priv = priv;
+	clk->start = now;
+	clk->end = now + ms;
+	clk->remain = (int)rtw_fsm_time_left(clk->start, clk->end);
+	_rtw_spinunlock_bh(&obj->fsm->clk_queue.lock);
+#if 0
+	FSM_DBG_(obj->fsm, "%s:%s now=0x%08x, end=0x%08x, remain=0x%08x\n",
+		fsm_obj_name(obj), fsm_evt_name(obj->fsm, event),
+		clk->start, clk->end, clk->remain);
+#endif
+}
+
+/* For EXTERNAL application to extend alarm time (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ * @event: alarm will issue this event while timer expired
+ * @ms: time period for the alarm
+ *	remain time does not less than 'ms'
+ * @id: alarm id; start from 1
+ */
+void fsm_extend_alarm_ext(struct fsm_obj *obj, int ms, u8 id)
+
+{
+	struct fsm_clocks *clocks = obj->clocks;
+	struct fsm_clk *clk = &clocks->clk[id];
+	int remain = ms;
+
+	if (id == 0 || id >= CLOCK_NUM) {
+		RTW_ERR("%s: %s_%d fail\n",
+			fsm_obj_name(obj), __func__, id);
+		return;
+	}
+
+	if (IS_CLK_OFF(clk))
+		return;
+
+	remain = MAX((int)rtw_fsm_time_left(clk->start, clk->end), ms);
+	fsm_set_alarm_ext(obj, remain, clk->event, id, clk->priv);
+}
+
+/* For EXTERNAL application to setup alarm (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ * @event: alarm will issue this event while timer expired
+ * @ms: time period for the alarm
+ * @id: alarm id; start from 1
+ */
+void fsm_set_alarm(struct fsm_obj *obj, int ms, u16 event)
+{
+	_fsm_set_alarm(obj, ms, event, 0, NULL);
+}
+
+/* For EXTERNAL application to setup alarm_ext (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ * @event: alarm will issue this event while timer expired
+ * @ms: time period for the alarm
+ * @id: alarm id; start from 1
+ * @priv: priv from caller
+ */
+void fsm_set_alarm_ext(struct fsm_obj *obj,
+	int ms, u16 event, u8 id, void *priv)
+{
+	if (id >= CLOCK_NUM) {
+		RTW_ERR("%s: set alarm_ext_%d to %d ms fail\n",
+			fsm_obj_name(obj), id, ms);
+		return;
+	}
+	_fsm_set_alarm(obj, ms, event, id, priv);
+}
+
+static void _fsm_cancel_alarm(struct fsm_obj *obj, u8 id)
+{
+	struct fsm_clocks *clocks = obj->clocks;
+	struct fsm_clk *clk = &clocks->clk[id];
+
+	_rtw_spinlock_bh(&obj->fsm->clk_queue.lock);
+	/* turn off clock from on */
+	if (IS_CLK_ON(clk))
+		obj->fsm->en_clock_num--;
+
+	clk->end = 0;
+	clk->remain = -1;
+	clk->pause = 0;
+	_rtw_spinunlock_bh(&obj->fsm->clk_queue.lock);
+}
+
+/* For EXTERNAL application to cancel alarm (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ */
+void fsm_cancel_alarm(struct fsm_obj *obj)
+{
+	_fsm_cancel_alarm(obj, 0);
+}
+
+/* For EXTERNAL application to cancel alarm_ext (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ * @id: alarm id; start from 1
+ */
+void fsm_cancel_alarm_ext(struct fsm_obj *obj, u8 id)
+{
+	if (id == 0 || id >= CLOCK_NUM) {
+		RTW_ERR("%s: cancel alarm_ext_%d fail\n",
+			fsm_obj_name(obj), id);
+		return;
+	}
+	_fsm_cancel_alarm(obj, id);
+}
+
+static void _fsm_pause_alarm(struct fsm_obj *obj, u8 id)
+{
+	struct fsm_clocks *clocks = obj->clocks;
+	struct fsm_clk *clk = &clocks->clk[id];
+
+	_rtw_spinlock_bh(&obj->fsm->clk_queue.lock);
+	clk[id].pause = 1;
+	_rtw_spinunlock_bh(&obj->fsm->clk_queue.lock);
+}
+
+/* For EXTERNAL application to pause alarm (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ */
+void fsm_pause_alarm(struct fsm_obj *obj)
+{
+	_fsm_pause_alarm(obj, 0);
+}
+
+/* For EXTERNAL application to pause alarm_ext (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ * @id: alarm id; start from 1
+ */
+void fsm_pause_alarm_ext(struct fsm_obj *obj, u8 id)
+{
+	if (id == 0 || id >= CLOCK_NUM) {
+		RTW_ERR("%s: pause alarm_%d fail\n", fsm_obj_name(obj), id);
+		return;
+	}
+	_fsm_pause_alarm(obj, id);
+}
+
+static void _fsm_resume_alarm(struct fsm_obj *obj, u8 id)
+{
+	struct fsm_clocks *clocks = obj->clocks;
+	struct fsm_clk *clk = &clocks->clk[id];
+	u32 cur = rtw_systime_to_ms(rtw_get_current_time());
+
+	/* extrend end time */
+	_rtw_spinlock_bh(&obj->fsm->clk_queue.lock);
+	clk->end = cur + clk->remain;
+	clk->pause = 0;
+	_rtw_spinunlock_bh(&obj->fsm->clk_queue.lock);
+}
+
+/* For EXTERNAL application to resume alarm (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ */
+void fsm_resume_alarm(struct fsm_obj *obj)
+{
+	_fsm_resume_alarm(obj, 0);
+}
+
+/* For EXTERNAL application to resume alarm_ext (expose)
+ *
+ * @obj: obj param when calling FSM framework function
+ * @id: alarm id; start from 1
+ */
+void fsm_resume_alarm_ext(struct fsm_obj *obj, u8 id)
+{
+	if (id == 0 || id >= CLOCK_NUM) {
+		RTW_ERR("%s: resume alarm_ext_%d fail\n",
+			fsm_obj_name(obj), id);
+		return;
+	}
+	_fsm_resume_alarm(obj, id);
+}
+
+/* For EXTERNAL application to change state (expose)
+ *
+ * @obj: obj that changes state
+ * @new_state: new state
+ */
+void fsm_state_goto(struct fsm_obj *obj, u8 new_state)
+{
+	struct fsm_main *fsm = NULL;
+
+	if (obj->state == new_state)
+		return;
+
+	fsm = obj->fsm;
+
+	fsm_state_run(obj, FSM_EV_STATE_OUT, NULL);
+
+	FSM_MSG_(fsm, FSM_LV_DBG, "\n");
+	FSM_MSG_(fsm, FSM_LV_DBG, "%s %-18s -> %s\n", obj->name,
+		fsm_state_name(fsm, obj->state),
+		fsm_state_name(fsm, new_state));
+
+	obj->state = new_state; /* new state */
+	fsm_state_run(obj, FSM_EV_STATE_IN, NULL);
+}
+
+static int fsm_int_evt_handler(struct fsm_obj *obj, struct fsm_msg *msg)
+{
+	switch(msg->event) {
+	case FSM_EV_NEW_OBJ:
+		fsm_obj_switch_in(obj, msg);
+		break;
+	case FSM_EV_DEL_OBJ:
+		fsm_obj_switch_out(obj, msg);
+		break;
+	default:
+		FSM_WARN_(obj->fsm, "unexpect event %04x(%s)\n",
+			msg->event, fsm_evt_name(obj->fsm, msg->event));
+		return _FALSE;
+		break;
+	}
+	return _SUCCESS;
+}
+
+static void fsm_free_msg(struct fsm_msg *msg)
+{
+	if (msg && msg->param && msg->param_sz)
+		rtw_mfree((void *)msg->param, msg->param_sz);
+#ifndef USE_PHL_CMD_DISPR
+	if (msg)
+		rtw_mfree((void *)msg, sizeof(*msg));
+#endif
+
+}
+
+static void fsm_user_evt_handler(struct fsm_main *fsm)
+{
+	struct fsm_msg *msg;
+	struct fsm_obj *obj;
+
+	while ((msg = fsm_dequeue_msg(fsm)) != NULL) {
+
+		obj = fsm_get_obj(fsm, msg->oid, 0);
+
+		if (obj == NULL) {
+			FSM_WARN_(fsm, "%s: obj (cid=0x%04x, oid=0x%8x) not found, ignore %s\n",
+				fsm->name, msg->cid, msg->oid, fsm_evt_name(fsm, msg->event));
+			fsm_free_msg(msg);
+			continue;
+		}
+
+		if (msg->event & FSM_INT_EV_MASK) {
+			/* Allow INT_EV even FSM_STATUS_DISABLE
+			 * for ABORT and DEL_OBJ to run. Free ALL user and
+			 * Global evt to empty msg queue.
+			 */
+			if (fsm_int_evt_handler(obj, msg)) {
+				fsm_free_msg(msg);
+				continue;
+			}
+		}
+
+		if ((fsm_status(fsm) != FSM_STATUS_ENABLE)) {
+			fsm_free_msg(msg);
+			continue;
+		}
+
+		/* pre handle events */
+		switch(msg->event) {
+		case FSM_EV_CONNECTED:
+		case FSM_EV_DISCONNECTED:
+			if (msg->param && !msg->param_sz)
+				obj->psta = (struct sta_info *)msg->param;
+			break;
+		default:
+			break;
+		}
+
+		if (!obj->valid) {
+			FSM_WARN_(fsm, "%s: invalid, ignore %s\n",
+				fsm_obj_name(obj), fsm_evt_name(fsm, msg->event));
+			fsm_free_msg(msg);
+			continue;
+		}
+
+		/* run state machine */
+		fsm_state_run(obj, msg->event, msg->param);
+		fsm_free_msg(msg);
+	}
+	return;
+}
+
+static int fsm_handler(struct fsm_main *fsm)
+{
+	/* USER EVENT */
+	if (list_empty(&fsm->msg_queue.q))
+		return 0;
+
+	fsm_user_evt_handler(fsm);
+
+	return 0;
+}
+
+/* For EXTERNAL application to get fsm name (expose)
+ * @fsm: fsm to be get name
+ */
+char *rtw_fsm_fsm_name(struct fsm_main *fsm)
+{
+	return fsm->name;
+}
+
+/* For EXTERNAL application to get obj name (expose)
+ * @obj: obj to be get name
+ * For example: scan-1 (sacn obj with object id 1)
+ */
+char *fsm_obj_name(struct fsm_obj *obj)
+{
+	return obj->name;
+}
+
+/* For EXTERNAL application to cancel sma (expose)
+ * @obj: obj job will be cancelled
+ */
+int fsm_abort_obj(struct fsm_obj *obj)
+{
+	struct fsm_msg *msg;
+
+	/* NEW message to cancel obj task */
+	msg = fsm_new_msg(obj, FSM_EV_ABORT);
+	if (msg == NULL) {
+		RTW_ERR("%s: alloc msg fail\n", obj->fsm->name);
+		return _FAIL;
+	}
+	return fsm_send_msg(obj->fsm, msg);
+}
+
+int fsm_init_custom(struct fsm_priv *fsmpriv)
+{
+	int rtn = _FAIL;
+
+#ifdef CONFIG_RTW_FSM_XXX
+	if (rtw_xxx_reg_fsm(fsmpriv) == _FAIL)
+		goto exit;
+#endif
+#ifdef CONFIG_RTW_FSM_RRM
+	if (rtw_rrm_reg_fsm(fsmpriv) == _FAIL)
+		goto exit;
+#endif
+#ifdef CONFIG_RTW_FSM_BTM
+	if (rtw_btm_reg_fsm(fsmpriv) == _FAIL)
+		goto exit;
+#endif
+	rtn = _SUCCESS;;
+exit:
+	return rtn;
+}
+
+void fsm_deinit_custom(struct fsm_priv *fsmpriv)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct fsm_main *fsm, *fsm_t;
+
+	if (list_empty(&root->q_share_thd.q))
+		return;
+
+	list_for_each_entry_safe(fsm, fsm_t, &root->q_share_thd.q, list) {
+		list_del(&fsm->list);
+		/* csutom deinit fsm */
+		if (fsm->tb.deinit_priv)
+			fsm->tb.deinit_priv(fsm->tb.priv);
+		fsm_deinit_fsm(fsm);
+	}
+}
+
+/* For EXTERNAL application to init FSM framework (expose) */
+/* @obj: obj job will be cancelled
+ */
+int rtw_fsm_init(struct fsm_priv *fsmpriv, void *priv)
+{
+	struct fsm_root *root;
+	int max, i, g, size;
+
+	/* check size of internal event table */
+	i = FSM_EV_INT_MAX & ~FSM_INT_EV_MASK;
+	g = FSM_EV_GBL_MAX & ~FSM_GBL_EV_MASK;
+	max = i + g + 2;
+	size = sizeof(int_event_tbl)/sizeof(int_event_tbl)[0];
+	if (size != max) {
+		RTW_ERR("fsm: int_event_tbl[%d] != %d size mismatch!!",
+			size, max);
+		WARN_ON(1);
+		return _FAIL;
+	}
+	root = (struct fsm_root *)rtw_zmalloc(sizeof(*root));
+	if (root == NULL)
+		return _FAIL;
+
+	_rtw_memset(root, 0, sizeof(*root));
+	fsm_init_queue(&(root->q_share_thd));
+	_rtw_init_sema(&root->msg_ready, 0);
+	root->a = priv;
+	root->oid_seq = 1;
+
+	fsmpriv->root = root;
+
+	RTW_INFO("fsm: [FSM] %s()\n", __func__);
+
+	return fsm_init_custom(fsmpriv);
+}
+
+/* For EXTERNAL application to deinit FSM framework (expose)
+ * @root: FSM framework handler
+ */
+void rtw_fsm_deinit(struct fsm_priv *fsmpriv)
+{
+	_adapter *a;
+	void *c = NULL;
+	struct fsm_root *root = fsmpriv->root;
+
+	if (!root) {
+		RTW_ERR("fsm: root hasn't initialized!\n");
+		return;
+	}
+	fsm_deinit_custom(fsmpriv);
+	a = (_adapter *)root->a;
+
+	FSM_INFO_(c, "fsm: [FSM] "FUNC_NDEV_FMT"\n",FUNC_NDEV_ARG(a->pnetdev));
+
+	fsm_deinit_queue(&(root->q_share_thd));
+	_rtw_free_sema(&root->msg_ready);
+
+	/* free fsm_root */
+	rtw_mfree(root, sizeof(*root));
+	fsmpriv->root = NULL;
+
+}
+
+void fsm_start_custom(struct fsm_priv *fsmpriv)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct fsm_main *fsm, *fsm_t;
+
+	if (list_empty(&root->q_share_thd.q))
+		return;
+
+	list_for_each_entry_safe(fsm, fsm_t, &root->q_share_thd.q, list) {
+		fsm_start_fsm(fsm);
+	}
+}
+
+void fsm_stop_custom(struct fsm_priv *fsmpriv)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct fsm_main *fsm, *fsm_t;
+	int sleep_count;
+
+	if (list_empty(&root->q_share_thd.q))
+		return;
+
+	list_for_each_entry_safe(fsm, fsm_t, &root->q_share_thd.q, list) {
+		fsm_stop_fsm(fsm);
+	}
+	list_for_each_entry_safe(fsm, fsm_t, &root->q_share_thd.q, list) {
+		sleep_count = 100;
+		/* Max wait 1000ms until queue is empty */
+		while (!list_empty(&fsm->msg_queue.q) && sleep_count--)
+			rtw_msleep_os(10);
+
+		if (sleep_count <= 0) {
+			FSM_WARN_(fsm, "msg queue is not empty!\n");
+		}
+		fsm_remove_all_queuing_msg(fsm);
+	}
+}
+
+/* For EXTERNAL application to start fsm root (expose)
+ * @fsm: see struct fsm_main
+ */
+void rtw_fsm_start(struct fsm_priv *fsmpriv)
+{
+	struct fsm_root *root = fsmpriv->root;
+	_adapter *a = (_adapter *)root->a;
+
+#ifndef USE_PHL_CMD_DISPR
+	if (root->thread) {
+		RTW_WARN("fsm: thread is running\n");
+		return;
+	}
+	root->thread_should_stop = 0;
+	root->thread = kthread_create(fsm_thread_share, root, "fsm_thread");
+	wake_up_process(root->thread);
+#endif
+	RTW_INFO("fsm: [FSM] "FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(a->pnetdev));
+
+	fsm_start_custom(fsmpriv);
+	return;
+}
+
+/* For EXTERNAL application to stop fsm root (expose)
+ * @fsm: see struct fsm_main
+ */
+void rtw_fsm_stop(struct fsm_priv *fsmpriv)
+{
+	void *c = NULL;
+	struct fsm_root *root = fsmpriv->root;
+	_adapter *a = (_adapter *)root->a;
+
+#ifdef USE_PHL_CMD_DISPR
+	fsm_stop_custom(fsmpriv);
+#else
+	if (!root->thread) {
+		RTW_INFO("fsm: thread was down\n");
+		return;
+	}
+	fsm_stop_custom(fsmpriv);
+
+	root->thread_should_stop = 1;
+	_rtw_up_sema(&root->msg_ready);
+
+	kthread_stop(root->thread);
+	root->thread = NULL;
+#endif
+	RTW_INFO("fsm: [FSM] "FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(a->pnetdev));
+
+	return;
+}
+
+static int fsm_start_fsm(struct fsm_main *fsm)
+{
+	_adapter *a = fsm_to_adapter(fsm);
+
+	_set_timer(&fsm->fsm_timer, CLOCK_UNIT);
+	fsm_status_set(fsm, FSM_STATUS_ENABLE);
+
+	FSM_INFO_(fsm, "fsm: [%s] "FUNC_NDEV_FMT"\n", fsm->name, FUNC_NDEV_ARG(a->pnetdev));
+
+	return _SUCCESS;
+}
+
+static int fsm_stop_fsm(struct fsm_main *fsm)
+{
+	struct fsm_obj *obj;
+	_adapter *a = fsm_to_adapter(fsm);
+
+	_cancel_timer_ex(&fsm->fsm_timer);
+
+	/* Stop receiving message */
+	fsm_status_set(fsm, FSM_STATUS_DISABLE);
+
+	/* ABORT all objs within fsm */
+	fsm_abort_all_running_obj(fsm);
+
+	FSM_INFO_(fsm, "fsm: [%s] "FUNC_NDEV_FMT"\n", fsm->name, FUNC_NDEV_ARG(a->pnetdev));
+
+	return _SUCCESS;
+}
+
+/* For EXTERNAL application to generate message buffer (expose)
+ * Generate message quickly and simply
+ * @obj: fsm_obj (msg receiver)
+ * @pbuf: message parameter
+ * @sz: message parameter size
+ * @event: event for the message
+ */
+int fsm_gen_msg(struct fsm_obj *obj, void *pbuf, u32 sz, u16 event)
+{
+	void *param = NULL;
+	struct fsm_msg *msg;
+
+	/* NEW mem for message */
+	msg = fsm_new_msg(obj, event);
+	if (msg == NULL) {
+		FSM_ERR_(obj->fsm, "%s: alloc msg %s fail\n",
+			fsm_obj_name(obj),
+			fsm_evt_name(obj->fsm, event));
+		goto msg_fail;
+	}
+
+	/* NEW mem for param */
+	if (pbuf && sz) {
+		param = rtw_zmalloc(sz);
+		if (param == NULL) {
+			FSM_ERR_(obj->fsm,
+				"%s: alloc param %s fail\n",
+				fsm_obj_name(obj),
+				fsm_evt_name(obj->fsm, event));
+			rtw_mfree(msg, sizeof(*msg));
+			goto msg_fail;
+		}
+		_rtw_memcpy(param, pbuf, sz);
+		msg->param = (void *)param;
+		msg->param_sz = sz;
+	} else {
+		msg->param = (void *)pbuf;
+		msg->param_sz = sz;
+	}
+
+	return fsm_send_msg(obj->fsm, msg);
+
+msg_fail:
+	return _FAIL;
+}
+
+static void fsm_ucast_msg(struct fsm_priv *fsmpriv,
+	struct sta_info *psta, char *pbuf, u32 sz, u16 event)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct fsm_main *fsm;
+	struct fsm_obj *obj;
+	struct fsm_obj *obj_ary[MAX_OBJ_NUM];
+	int i, obj_num = 0;
+
+	if (!psta)
+		return;
+
+	memset(obj_ary, 0, sizeof(obj_ary));
+
+	list_for_each_entry(fsm, &root->q_share_thd.q, list) {
+		if (fsm_status(fsm) != FSM_STATUS_ENABLE)
+			continue;
+
+		_rtw_spinlock_bh(&fsm->obj_queue.lock);
+		list_for_each_entry(obj, &fsm->obj_queue.q, list) {
+			if (obj->psta == psta) {
+				obj_ary[obj_num++] = obj;
+				if (obj_num > MAX_OBJ_NUM) {
+					FSM_WARN_(obj->fsm, "obj num exceed, event %04x(%s)\n",
+						event, fsm_evt_name(obj->fsm, event));
+					_rtw_spinunlock_bh(&fsm->obj_queue.lock);
+					goto done;
+				}
+			}
+		}
+		_rtw_spinunlock_bh(&fsm->obj_queue.lock);
+	}
+done:
+	for (i = 0; i < obj_num; i++)
+		fsm_gen_msg(obj_ary[i], pbuf, sz,  event);
+}
+
+#if 0
+static void fsm_bcast_msg(struct fsm_priv *fsmpriv,
+	struct sta_info *psta, char *pbuf, u32 sz, u16 event)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct fsm_main *fsm;
+	struct fsm_obj *obj;
+	struct fsm_obj *obj_ary[MAX_OBJ_NUM];
+	int i, obj_num = 0;
+
+	if (!psta)
+		return;
+
+	memset(obj_ary, 0, sizeof(obj_ary));
+
+	list_for_each_entry(fsm, &root->q_share_thd.q, list) {
+		if (fsm_status(fsm) != FSM_STATUS_ENABLE)
+			continue;
+
+		_rtw_spinlock_bh(&fsm->obj_queue.lock);
+		list_for_each_entry(obj, &fsm->obj_queue.q, list) {
+			if (!psta || (psta && obj->psta == psta))
+				obj_ary[obj_num++] = obj;
+				if (obj_num > MAX_OBJ_NUM) {
+					FSM_WARN_(obj->fsm, "obj num exceed, event %04x(%s)\n",
+						event, fsm_evt_name(obj->fsm, event));
+					_rtw_spinunlock_bh(&fsm->obj_queue.lock);
+					goto done;
+				}
+			}
+		}
+		_rtw_spinunlock_bh(&fsm->obj_queue.lock);
+	}
+done:
+	for (i = 0; i < obj_num; i++)
+		fsm_gen_msg(obj_ary[i], pbuf, sz,  event);
+}
+#endif
+
+void rtw_fsm_notify_connect(struct fsm_priv *fsmpriv, struct sta_info *psta, int res)
+{
+	_adapter *a = fsmpriv->root->a;
+	struct sta_info *pself;
+
+	pself = rtw_get_stainfo(&a->stapriv, a->phl_role->mac_addr);
+	if (res >= 0) /* success */
+		fsm_ucast_msg(fsmpriv, pself, (char *)psta, 0, FSM_EV_CONNECTED);
+	else
+		fsm_ucast_msg(fsmpriv, pself, (char *)psta, 0, FSM_EV_CONNECT_FAIL);
+}
+
+void rtw_fsm_notify_disconnect(struct fsm_priv *fsmpriv, struct sta_info *psta)
+{
+	_adapter *a = fsmpriv->root->a;
+	struct sta_info *pself;
+
+	pself = rtw_get_stainfo(&a->stapriv, a->phl_role->mac_addr);
+	if (pself)
+		fsm_ucast_msg(fsmpriv, psta, (char *)pself, 0, FSM_EV_DISCONNECTED);
+}
+
+void rtw_fsm_notify_scan_start(struct fsm_priv *fsmpriv, struct sta_info *psta)
+{
+	fsm_ucast_msg(fsmpriv, psta, NULL, 0, FSM_EV_SCAN_START);
+}
+
+void rtw_fsm_notify_scan_done(struct fsm_priv *fsmpriv, struct sta_info *psta)
+{
+	fsm_ucast_msg(fsmpriv, psta, NULL, 0, FSM_EV_SCAN_DONE);
+}
+
+/** Debug funcitons
+ *
+ */
+#ifdef RTW_DEBUG_FSM
+static void fsm_dbg_dump_fsm_queue(struct fsm_queue *fsmq,
+	char *s, int *sz,bool detail)
+{
+	struct fsm_main *fsm, *fsm_t;
+
+	char *ptr = s;
+	int len = *sz;
+
+	list_for_each_entry(fsm, &fsmq->q, list) {
+		_os_snprintf(pstr(ptr), lstr(ptr, len), "\t%4s : %s\n", fsm->name,
+			fsm->tb.mode ? "STANDALONE":"SHARE");
+
+		if (fsm->tb.dump_fsm && detail) {
+			len = lstr(ptr, len);
+			ptr = pstr(ptr);
+			fsm->tb.dump_fsm(fsm, ptr, &len);
+		}
+	}
+	*sz = len;
+}
+
+static void fsm_dbg_help(struct fsm_main *fsm, char *s, int *sz, bool detail);
+static void fsm_dbg_dump_fsm(struct fsm_main *fsm,
+	char *s, int *sz, bool detail)
+{
+	int len = *sz;
+	char *ptr = s;
+
+	_os_snprintf(pstr(ptr), lstr(ptr, len), "\t%4s : %s\n", fsm->name,
+		fsm->tb.mode ? "STANDALONE":"SHARE");
+
+	if (fsm->tb.dump_fsm && detail) {
+		len = lstr(ptr, len);
+		ptr = pstr(ptr);
+		fsm->tb.dump_fsm(fsm, ptr, &len);
+	}
+
+}
+
+static void fsm_dbg_dump_state(struct fsm_main *fsm,
+	char *s, int *sz, bool detail)
+{
+	int i;
+	int len = *sz;
+
+	_os_snprintf(pstr(s), lstr(s, len),
+		"[%s] state table\n", fsm->name);
+	for (i = 0; i < fsm->tb.max_state; i++)
+		_os_snprintf(pstr(s), lstr(s, len), "\t%4d : %s\n",
+			i, fsm->tb.state_tbl[i].name);
+	*sz = len;
+}
+
+static void fsm_dbg_dump_event(struct fsm_main *fsm,
+	char *s, int *sz, bool detail)
+{
+	int i, max;
+	int len = *sz;
+
+	/* internal event */
+	_os_snprintf(pstr(s), lstr(s, len), "[Internal] event table\n");
+
+	max = FSM_EV_END & ~(int_event_tbl[0].event); /* FSM_INT_EV_MASK */
+	for (i = 1; i < max; i++)
+		_os_snprintf(pstr(s), lstr(s, len), "\t0x%4x : %s\n",
+			int_event_tbl[i].event, int_event_tbl[i].name);
+
+	/* user event */
+	_os_snprintf(pstr(s), lstr(s, len), "\n[%s] event table max %d\n", fsm->name, fsm->tb.max_event);
+	for (i = 0; i < fsm->tb.max_event-1; i++)
+		_os_snprintf(pstr(s), lstr(s, len), "\t0x%4x : %s\n",
+			fsm->tb.evt_tbl[i].event, fsm->tb.evt_tbl[i].name);
+	*sz = len;
+}
+
+static void fsm_dbg_dump_obj(struct fsm_main *fsm,
+	char *s, int *sz, bool detail)
+{
+	struct fsm_obj *obj, *obj_t;
+	int len = *sz;
+	char *ptr = s;
+
+	list_for_each_entry(obj, &fsm->obj_queue.q, list) {
+
+		_os_snprintf(pstr(ptr), lstr(ptr, len), "%s : state %s",
+			obj->name, fsm_state_name(fsm, obj->state));
+
+		if (fsm->tb.dump_obj && detail) {
+			len = lstr(ptr, len);
+			ptr = pstr(ptr);
+			fsm->tb.dump_obj(obj->custom_obj, ptr, &len);
+		}
+	}
+	*sz = len;
+}
+
+static void fsm_dbg_max(struct fsm_main *fsm, char *s, int *sz, bool detail)
+{
+	int len = *sz;
+
+	_os_snprintf(pstr(s), lstr(s, len),
+		"ERR: fsm %s sould not run to here!!\n", __func__);
+	*sz = len;
+}
+
+struct fsm_debug_ent {
+	char *opt;
+	void (*func)(struct fsm_main *fsm, char *s, int *sz, bool detail);
+	char *desc;
+};
+
+struct fsm_debug_ent debug_opt[] = {
+	{"help", fsm_dbg_help, "help message"},
+	{"fsm", fsm_dbg_dump_fsm, "all fsm name"},
+	{"st", fsm_dbg_dump_state, "state name"},
+	{"ev", fsm_dbg_dump_event, "event name"},
+	{"obj", fsm_dbg_dump_obj, "obj detail"},
+	{"max", fsm_dbg_max, "max_opt"}
+};
+
+static void _fsm_dbg_help(struct fsm_root *root, char *s, int *sz, bool detail)
+{
+	int i, max_opt;
+	int len = *sz;
+	char *ptr = s;
+
+	_os_snprintf(pstr(ptr), lstr(ptr, len),
+		"usage:\tfsm d <fsm_name> <option>\n");
+	_os_snprintf(pstr(ptr), lstr(ptr, len),
+		"\tfsm p,<obj_name> <priv_dbg_cmd> ....\n");
+	_os_snprintf(pstr(ptr), lstr(ptr, len),
+		"\tfsm s,<obj_name> <EVENT>\n");
+	_os_snprintf(pstr(ptr), lstr(ptr, len),
+		"\tfsm w,<fsm_name> <dbg_level|ev_level> <0-5(dbg)>\n");
+
+	_os_snprintf(pstr(s), lstr(ptr, len), "\nfsm_name:\n");
+
+	len = lstr(ptr, len);
+	ptr = pstr(ptr);
+	fsm_dbg_dump_fsm_queue(&root->q_share_thd, ptr, &len, detail);
+
+	_os_snprintf(pstr(ptr), lstr(ptr, len), "\noption:\n");
+	max_opt = sizeof(debug_opt)/sizeof(debug_opt[0]);
+	for (i = 0; i < max_opt-1; i++)
+		_os_snprintf(pstr(ptr), lstr(ptr, len), "%12s : %s\n",
+			debug_opt[i].opt, debug_opt[i].desc);
+	*sz = len;
+}
+
+static void fsm_dbg_help(struct fsm_main *fsm, char *s, int *sz, bool detail)
+{
+	_fsm_dbg_help(fsm->root, s, sz, false);
+}
+
+struct fsm_main *get_fsm_by_name(struct fsm_root *root, char *name)
+{
+	struct fsm_main *fsm, *fsm_t;
+	u32 len = strlen((u8 *)name);
+
+	if (len > FSM_NAME_LEN)
+		return NULL;
+
+	list_for_each_entry(fsm, &root->q_share_thd.q, list) {
+		if (strlen((u8 *)fsm->name) == len &&
+			_os_mem_cmp(d, fsm->name, name, len) == 0)
+			return fsm;
+	}
+
+	return NULL;
+}
+
+static u16 fsm_get_evt_id(struct fsm_main *fsm, char *event)
+{
+	int i;
+	u32 len = strlen((u8 *)event);
+
+	/* internal event */
+	for (i = 0; i < (sizeof(int_event_tbl)/sizeof(int_event_tbl[0])); i++) {
+		if (strlen((u8 *)int_event_tbl[i].name) == len &&
+			_os_mem_cmp(d, int_event_tbl[i].name, event, len) == 0)
+			return int_event_tbl[i].event;
+	}
+
+	/* user event */
+	for (i = 0; i < fsm->tb.max_event; i++) {
+		if (strlen((u8 *)fsm->tb.evt_tbl[i].name) == len &&
+			_os_mem_cmp(d,
+				fsm->tb.evt_tbl[i].name, event, len) == 0)
+			return fsm->tb.evt_tbl[i].event;
+	}
+	return FSM_EV_UNKNOWN;
+}
+#endif /* RTW_DEBUG_FSM */
+
+/* For EXTERNAL application to debug fsm (expose)
+ * @input: input cmd
+ * @input_num: num of cmd param
+ * @output: output buffer
+ * @out_len: MAX output buffer len
+ *
+ * d: dump fsm info
+ *	fsm <d> <fsm_name> <fsm|st|ev|obj>
+ * p: private cmd to fsm module
+ *	fsm <p> <obj_name> <cmd to fsm module>
+ * s: send event to fsm
+ *	fsm <s> <obj_name> <ev>
+ * w: write debug level
+ *	fsm <w> <fsm_name> <dbg_level|evt_level> <0-5>
+ */
+void rtw_fsm_dbg(void *a, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 out_len)
+{
+#ifdef RTW_DEBUG_FSM
+	//struct phl_info_t *phl = (struct phl_info_t *)phl_info;
+	struct fsm_root *root = a->fsmpriv.root;
+	struct fsm_main *fsm = NULL;
+	struct fsm_obj *obj = NULL;
+	struct fsm_msg *msg;
+	int i, max_opt, len = out_len;
+	char fsm_name[FSM_NAME_LEN], opt[FSM_NAME_LEN], cmd[FSM_NAME_LEN];
+	char c, *ptr, *sp;
+	u8 obj_id = 0;
+	u16 ev_id;
+
+	ptr = output;
+	/* fsm <cmd> <fsm_name> <opt> : fsm d cmd ev
+	 * fsm <cmd> <fsm_name> <evt> : fsm s cmd-1 FSM_EV_ABORT
+	 */
+	if (input_num < 4)
+		goto help;
+
+	_rtw_memset(cmd, 0, FSM_NAME_LEN);
+	_rtw_memcpy(cmd, input[1],
+		MIN(strlen((u8 *)input[1]), FSM_NAME_LEN));
+
+	_rtw_memset(fsm_name, 0, FSM_NAME_LEN);
+	_rtw_memcpy(fsm_name, input[2],
+		MIN(strlen((u8 *)input[2]), FSM_NAME_LEN));
+
+	_rtw_memset(opt, 0, FSM_NAME_LEN);
+	_rtw_memcpy(opt, input[3],
+		MIN(strlen((u8 *)input[3]), FSM_NAME_LEN));
+
+	c = (char)*cmd;
+	/* read obj_id
+	 * if fsm_name is "cmd-1" then obj number is "1"
+	 */
+	sp = _os_strchr((const char *)fsm_name, '-');
+
+	if (sp != NULL) {
+		*sp = '\0';
+		if (_os_sscanf(sp+1, "%hhd", &obj_id) != 1) {
+			_os_snprintf(pstr(ptr), lstr(ptr, len),
+				"ERR: fsm[%s] miss obj_id\n", fsm_name);
+			return;
+		}
+	} else
+		obj_id = 1; /* assume obj-1 */
+
+	/* search fsm by name */
+	fsm = get_fsm_by_name(root, (char *)fsm_name);
+	if (fsm == NULL) {
+		_os_snprintf(pstr(ptr), lstr(ptr, len),
+			"ERR: fsm[%s] not found\n", fsm_name);
+		return;
+	}
+
+	obj = fsm_get_obj(fsm, obj_id, 0);
+	if (obj == NULL) {
+		_os_snprintf(pstr(ptr), lstr(ptr, len),
+			"ERR: fsm[%s] miss obj_%d\n", fsm_name, obj_id);
+		return;
+	}
+
+	switch (c) {
+	case 'd':
+		/* dump status */
+		max_opt = sizeof(debug_opt)/sizeof(debug_opt)[0];
+		for (i = 0; i < max_opt-1; i++) {
+			if (strlen((u8 *)debug_opt[i].opt) == \
+				strlen((u8 *)opt) &&
+				_os_mem_cmp(d, debug_opt[i].opt, opt,
+				strlen((u8 *)opt)) == 0) {
+
+				len = lstr(ptr, len);
+				ptr = pstr(ptr);
+				debug_opt[i].func(fsm, ptr, &len, true);
+				break;
+			}
+		}
+		break;
+
+	case 'p':
+		/* call fsm private degug function */
+		if ((fsm != NULL) && (obj != NULL) && (fsm->tb.debug != NULL)){
+			len = lstr(ptr, len);
+			ptr = pstr(ptr);
+			fsm->tb.debug(obj->custom_obj, &input[3],
+				(input_num - 3), ptr, (u32 *)&len);
+		}
+		break;
+
+	case 's':
+		/* get event id */
+		ev_id = fsm_get_evt_id(fsm, (char *)opt);
+
+		if (ev_id == FSM_EV_UNKNOWN) {
+			_os_snprintf(pstr(ptr), lstr(ptr, len),
+				"\n\nERR: fsm[%s] unknown event %s\n",
+				fsm_name, opt);
+			len = lstr(ptr, len);
+			ptr = pstr(ptr);
+			fsm_dbg_dump_event(fsm, ptr, &len, false);
+			break;
+		}
+
+		if (obj != NULL) {
+			msg = fsm_new_msg(obj, ev_id);
+
+			/* send event */
+			fsm_send_msg(obj->fsm, msg);
+		}
+
+		break;
+
+	case 'w':
+		/* write cfg */
+		/* fsm w,<fsm_name>,<dbg_level|ev_level>,<0-5(dbg)> */
+
+		sp = _os_strchr((const char *)opt, ',');
+		if (sp == NULL)
+			goto help;
+
+		*sp = '\0';
+		if (_os_sscanf(sp+1, "%d", &i) != 1)
+			goto help;
+
+		if ((i<0) || (i>5))
+			goto help;
+
+		if (!_os_strcmp(opt, "dbg_level")) {
+			fsm->tb.dbg_level = (u8)i;
+			_os_snprintf(pstr(ptr), lstr(ptr, len),
+				"\n%s: set debug level to %d\n",
+				rtw_fsm_fsm_name(fsm), i);
+		} else if (!_os_strcmp(opt, "evt_level")) {
+			_os_snprintf(pstr(ptr), lstr(ptr, len),
+				"\n%s: set event level to %d\n",
+				rtw_fsm_fsm_name(fsm), i);
+		} else
+			goto help;
+		break;
+
+	default:
+		goto help;
+	}
+	return;
+help:
+	len = lstr(ptr, len);
+	ptr = pstr(ptr);
+	_fsm_dbg_help(fsm->root, ptr, &len, false);
+#endif /* RTW_DEBUG_FSM */
+}
+#endif /*CONFIG_RTW_FSM*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_rrm.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_rrm.c
new file mode 100644
index 000000000000..838a535dc0e0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_rrm.c
@@ -0,0 +1,4313 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * Author: vincent_fann@realtek.com
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_FSM_RRM
+
+#ifdef FSM_DBG_MEM_OVERWRITE
+#define _os_kmem_alloc(a, b) fsm_kmalloc(b)
+#define _os_kmem_free(a, b, c) fsm_kfree(b, c)
+#endif
+
+#define pstr(s) (s +strlen((u8 *)s))
+#define lstr(s, l) (size_t)(l - strlen((u8 *)s))
+
+#if 1
+#define RM_SUPPORT_IWPRIV_DBG	1
+
+#define DBG_BCN_REQ_DETAIL	0
+#define DBG_BCN_REQ_WILDCARD	0
+#define DBG_BCN_REQ_SSID	0
+#define DBG_BCN_REQ_SSID_NAME	"RealKungFu"
+
+#define RM_REQ_TIMEOUT		10000	/* 10 seconds */
+#define RM_MEAS_TIMEOUT		2000	/*  2 seconds */
+#define RM_REPT_SCAN_INTVL	5000	/*  5 seconds */
+#define RM_REPT_POLL_INTVL	2000	/*  2 seconds */
+#define RM_COND_INTVL		2000	/*  2 seconds */
+#define RM_BUSY_TRAFFIC_TIMES	2
+#define RM_WAIT_BUSY_TIMEOUT	2000	/*  1 seconds */
+
+#define MEAS_REP_MOD_LATE	BIT(0)
+#define MEAS_REP_MOD_INCAP	BIT(1)
+#define MEAS_REP_MOD_REFUSE	BIT(2)
+
+#define RM_MASTER		0x0100	/* STA who issue meas_req */
+#define RM_SLAVE		0x0000	/* STA who do measurement */
+
+/* IEEE 802.11-2012 Table 8-59 Measurement Type definitions
+*  for measurement request
+*  modify rm_meas_type_req_name() when adding new type
+*/
+enum meas_type_of_req {
+	basic_req,	/* spectrum measurement */
+	cca_req,
+	rpi_histo_req,
+	ch_load_req,
+	noise_histo_req,
+	bcn_req,
+	frame_req,
+	sta_statis_req,
+	lci_req,
+	meas_type_req_max,
+};
+
+/* IEEE 802.11-2012 Table 8-81 Measurement Type definitions
+*  for measurement report
+*  modify rm_type_rep_name() when adding new type
+*/
+enum meas_type_of_rep {
+	basic_rep,	/* spectrum measurement */
+	cca_rep,
+	rpi_histo_rep,
+	ch_load_rep,	/* radio measurement */
+	noise_histo_rep,
+	bcn_rep,
+	frame_rep,
+	sta_statis_rep,	/* Radio measurement and WNM */
+	lci_rep,
+	meas_type_rep_max
+};
+
+/*
+* Beacon request
+*/
+/* IEEE 802.11-2012 Table 8-64 Measurement mode for Beacon Request element */
+enum bcn_req_meas_mode {
+	bcn_req_passive,
+	bcn_req_active,
+	bcn_req_bcn_table
+};
+
+/* IEEE 802.11-2012 Table 8-65 optional subelement IDs for Beacon Request */
+enum bcn_req_opt_sub_id{
+	bcn_req_ssid = 0,		/* len 0-32 */
+	bcn_req_rep_info = 1,		/* len 2 */
+	bcn_req_rep_detail = 2,		/* len 1 */
+	bcn_req_req = 10,		/* len 0-237 */
+	bcn_req_ap_ch_rep = 51,		/* len 1-237 */
+	bcn_req_last_bcn_rpt_ind = 164	/* len 1 */
+};
+
+/* IEEE 802.11-2012 Table 8-66 Reporting condition of Beacon Report */
+enum bcn_rep_cound_id{
+	bcn_rep_cond_immediately,	/* default */
+	bcn_req_cond_rcpi_greater,
+	bcn_req_cond_rcpi_less,
+	bcn_req_cond_rsni_greater,
+	bcn_req_cond_rsni_less,
+	bcn_req_cond_max
+};
+
+struct opt_rep_info {
+	u8 cond;
+	u8 threshold;
+};
+
+#define	MAX_CH_NUM_IN_OP_CLASS	11
+typedef struct _RT_OPERATING_CLASS {
+	int	global_op_class;
+	int	Len;
+	u8	Channel[MAX_CH_NUM_IN_OP_CLASS];
+} RT_OPERATING_CLASS, *PRT_OPERATING_CLASS;
+
+#define BCN_REQ_OPT_MAX_NUM		16
+#define BCN_REQ_REQ_OPT_MAX_NUM		16
+#define BCN_REQ_OPT_AP_CH_RPT_MAX_NUM	12
+struct bcn_req_opt {
+	/* all req cmd id */
+	u8 opt_id[BCN_REQ_OPT_MAX_NUM];
+	u8 opt_id_num;
+	u8 req_id_num;
+	u8 req_id[BCN_REQ_REQ_OPT_MAX_NUM];
+	u8 rep_detail;
+	u8 rep_last_bcn_ind;
+	NDIS_802_11_SSID ssid;
+
+	/* bcn report condition */
+	struct opt_rep_info rep_cond;
+
+	u8 ap_ch_rpt_num;
+	struct _RT_OPERATING_CLASS *ap_ch_rpt[BCN_REQ_OPT_AP_CH_RPT_MAX_NUM];
+
+	/* 0:default(Report to be issued after each measurement) */
+	u8 *req_start;	/*id : 10 request;start  */
+	u8 req_len;	/*id : 10 request;length */
+};
+
+/*
+* channel load
+*/
+/* IEEE 802.11-2012 Table 8-60 optional subelement IDs for channel load request */
+enum ch_load_opt_sub_id{
+	ch_load_rsvd,
+	ch_load_rep_info
+};
+
+/* IEEE 802.11-2012 Table 8-61 Reporting condition for channel load Report */
+enum ch_load_cound_id{
+	ch_load_cond_immediately,	/* default */
+	ch_load_cond_anpi_equal_greater,
+	ch_load_cond_anpi_equal_less,
+	ch_load_cond_max
+};
+
+/*
+* Noise histogram
+*/
+/* IEEE 802.11-2012 Table 8-62 optional subelement IDs for noise histogram */
+enum noise_histo_opt_sub_id{
+	noise_histo_rsvd,
+	noise_histo_rep_info
+};
+
+/* IEEE 802.11-2012 Table 8-63 Reporting condition for noise historgarm Report */
+enum noise_histo_cound_id{
+	noise_histo_cond_immediately,	/* default */
+	noise_histo_cond_anpi_equal_greater,
+	noise_histo_cond_anpi_equal_less,
+	noise_histo_cond_max
+};
+
+struct meas_req_opt {
+	/* report condition */
+	struct opt_rep_info rep_cond;
+};
+
+#define MAX_RM_PKT_NUM	32
+#define MAX_RM_AP_NUM	128
+struct data_buf {
+	u8 *pbuf;
+	u16 len;
+};
+
+/*
+* Measurement
+*/
+struct opt_subelement {
+	u8 id;
+	u8 length;
+	u8 *data;
+};
+
+/* 802.11-2012 Table 8-206 Radio Measurment Action field */
+enum rm_action_code {
+	RM_ACT_RADIO_MEAS_REQ,
+	RM_ACT_RADIO_MEAS_REP,
+	RM_ACT_LINK_MEAS_REQ,
+	RM_ACT_LINK_MEAS_REP,
+	RM_ACT_NB_REP_REQ,	/* 4 */
+	RM_ACT_NB_REP_RESP,
+	RM_ACT_RESV,
+	RM_ACT_MAX
+};
+
+/* 802.11-2012 Table 8-119 RM Enabled Capabilities definition */
+enum rm_cap_en {
+	RM_LINK_MEAS_CAP_EN,
+	RM_NB_REP_CAP_EN,		/* neighbor report */
+	RM_PARAL_MEAS_CAP_EN,		/* parallel report */
+	RM_REPEAT_MEAS_CAP_EN,
+	RM_BCN_PASSIVE_MEAS_CAP_EN,
+	RM_BCN_ACTIVE_MEAS_CAP_EN,
+	RM_BCN_TABLE_MEAS_CAP_EN,
+	RM_BCN_MEAS_REP_COND_CAP_EN,	/* conditions */
+
+	RM_FRAME_MEAS_CAP_EN,
+	RM_CH_LOAD_CAP_EN,
+	RM_NOISE_HISTO_CAP_EN,		/* noise historgram */
+	RM_STATIS_MEAS_CAP_EN,		/* statistics */
+	RM_LCI_MEAS_CAP_EN,		/* 12 */
+	RM_LCI_AMIMUTH_CAP_EN,
+	RM_TRANS_STREAM_CAT_MEAS_CAP_EN,
+	RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN,
+
+	RM_AP_CH_REP_CAP_EN,
+	RM_RM_MIB_CAP_EN,
+	RM_OP_CH_MAX_MEAS_DUR0,		/* 18-20 */
+	RM_OP_CH_MAX_MEAS_DUR1,
+	RM_OP_CH_MAX_MEAS_DUR2,
+	RM_NONOP_CH_MAX_MEAS_DUR0,	/* 21-23 */
+	RM_NONOP_CH_MAX_MEAS_DUR1,
+	RM_NONOP_CH_MAX_MEAS_DUR2,
+
+	RM_MEAS_PILOT_CAP0,		/* 24-26 */
+	RM_MEAS_PILOT_CAP1,
+	RM_MEAS_PILOT_CAP2,
+	RM_MEAS_PILOT_TRANS_INFO_CAP_EN,
+	RM_NB_REP_TSF_OFFSET_CAP_EN,
+	RM_RCPI_MEAS_CAP_EN,		/* 29 */
+	RM_RSNI_MEAS_CAP_EN,
+	RM_BSS_AVG_ACCESS_DELAY_CAP_EN,
+
+	RM_AVALB_ADMIS_CAPACITY_CAP_EN,
+	RM_ANT_CAP_EN,
+	RM_RSVD,			/* 34-39 */
+	RM_MAX
+};
+#endif
+
+/*
+* State machine
+*/
+enum RRM_STATE_ST {
+	RRM_ST_IDLE,
+	RRM_ST_DO_MEAS,
+	RRM_ST_WAIT_MEAS,
+	RRM_ST_SEND_REPORT,
+	RRM_ST_RECV_REPORT,
+	RRM_ST_END,
+	RRM_ST_MAX
+};
+
+enum RM_EV_ID {
+	RRM_EV_busy_timer_expire,
+	RRM_EV_delay_timer_expire,
+	RRM_EV_meas_timer_expire,
+	RRM_EV_retry_timer_expire,
+	RRM_EV_repeat_delay_expire,
+	RRM_EV_request_timer_expire,
+	RRM_EV_start_meas,
+	RRM_EV_recv_rep,
+	RRM_EV_max
+};
+
+static int rrm_idle_st_hdl(void *obj, u16 event, void *param);
+static int rrm_do_meas_st_hdl(void *obj, u16 event, void *param);
+static int rrm_wait_meas_st_hdl(void *obj, u16 event, void *param);
+static int rrm_send_report_st_hdl(void *obj, u16 event, void *param);
+static int rrm_recv_report_st_hdl(void *obj, u16 event, void *param);
+static int rrm_end_st_hdl(void *obj, u16 event, void *param);
+
+/* STATE table */
+static struct fsm_state_ent rrm_state_tbl[] = {
+	ST_ENT(RRM_ST_IDLE, rrm_idle_st_hdl),
+	ST_ENT(RRM_ST_DO_MEAS, rrm_do_meas_st_hdl),
+	ST_ENT(RRM_ST_WAIT_MEAS, rrm_wait_meas_st_hdl),
+	ST_ENT(RRM_ST_SEND_REPORT, rrm_send_report_st_hdl),
+	ST_ENT(RRM_ST_RECV_REPORT, rrm_recv_report_st_hdl),
+	ST_ENT(RRM_ST_END, rrm_end_st_hdl),
+};
+
+/* EVENT table */
+static struct fsm_event_ent rrm_event_tbl[] = {
+	EV_DBG(RRM_EV_busy_timer_expire),
+	EV_DBG(RRM_EV_delay_timer_expire),
+	EV_DBG(RRM_EV_meas_timer_expire),
+	EV_DBG(RRM_EV_retry_timer_expire),
+	EV_DBG(RRM_EV_repeat_delay_expire),
+	EV_DBG(RRM_EV_request_timer_expire),
+	EV_DBG(RRM_EV_start_meas),
+	EV_DBG(RRM_EV_recv_rep),
+	EV_DBG(RRM_EV_max) /* EV_MAX for fsm safety checking */
+};
+
+struct rm_meas_req {
+	u8 category;
+	u8 action_code;		/* T8-206  */
+	u8 diag_token;
+	u16 rpt;
+
+	u8 e_id;
+	u8 len;
+	u8 m_token;
+	u8 m_mode;		/* req:F8-105, rep:F8-141 */
+	u8 m_type;		/* T8-59 */
+	u8 op_class;
+	u8 ch_num;
+	u16 rand_intvl;		/* units of TU */
+	u16 meas_dur;		/* units of TU */
+	u8 s_mode;
+
+	u8 bssid[6];		/* for bcn_req */
+
+	u8 *pssid;
+	u8 *opt_s_elem_start;
+	int opt_s_elem_len;
+
+	s8 tx_pwr_used;		/* for link measurement */
+	s8 tx_pwr_max;		/* for link measurement */
+
+	union {
+		struct bcn_req_opt bcn;
+		struct meas_req_opt clm;
+		struct meas_req_opt nhm;
+	}opt;
+
+	struct rtw_ieee80211_channel ch_set[RTW_CHANNEL_SCAN_AMOUNT];
+	u8 ch_set_ch_amount;
+	s8 rx_pwr;		/* in dBm */
+	u8 rx_bw;
+	u8 rx_rsni;
+	u16 rx_rate;
+	u8 band;
+};
+
+struct rm_meas_rep {
+	u8 category;
+	u8 action_code;		/* T8-206  */
+	u8 diag_token;
+
+	u8 e_id;		/* T8-54, 38 request; 39 report */
+	u8 len;
+	u8 m_token;
+	u8 m_mode;		/* req:F8-105, rep:F8-141 */
+	u8 m_type;		/* T8-59 */
+	u8 op_class;
+	u8 ch_num;
+
+	u8 ch_load;
+	u8 anpi;
+	u8 ipi[11];
+
+	u16 rpt;
+	u8 bssid[6];		/* for bcn_req */
+	u8 band;
+};
+
+struct rrm_obj {
+
+	u16 cid;
+
+	enum RRM_STATE_ST state;
+	struct rm_meas_req q;
+	struct rm_meas_rep p;
+
+	/* meas report */
+	u64 meas_start_time;
+	u64 meas_end_time;
+	int wait_busy;
+	u8 poll_mode;
+
+	struct data_buf buf[MAX_RM_PKT_NUM];
+	bool from_ioctl;
+	struct fsm_obj *obj;
+	struct fsm_main *fsm;
+
+	struct wlan_network *ap[MAX_RM_AP_NUM];
+	u8 ap_num;
+
+	_list list;
+};
+
+/*
+ * rrm sub function
+ */
+
+void rrm_update_cap(u8 *frame_head, _adapter *pa, u32 pktlen, int offset)
+{
+	u8 *res;
+	sint len;
+
+	res = rtw_get_ie(frame_head + offset, _EID_RRM_EN_CAP_IE_, &len,
+			 pktlen - offset);
+	if (res != NULL)
+		_rtw_memcpy((void *)pa->fsmpriv.rmpriv.rm_en_cap_def, (res + 2),
+			    MIN(len, sizeof(pa->fsmpriv.rmpriv.rm_en_cap_def)));
+}
+
+struct cmd_meas_type_ {
+	u8 id;
+	char *name;
+};
+
+char *rm_type_req_name(u8 meas_type) {
+
+	switch (meas_type) {
+	case basic_req:
+		return "basic_req";
+	case cca_req:
+		return "cca_req";
+	case rpi_histo_req:
+		return "rpi_histo_req";
+	case ch_load_req:
+		return "ch_load_req";
+	case noise_histo_req:
+		return "noise_histo_req";
+	case bcn_req:
+		return "bcn_req";
+	case frame_req:
+		return "frame_req";
+	case sta_statis_req:
+		return "sta_statis_req";
+	}
+	return "unknown_req";
+};
+
+char *rm_type_rep_name(u8 meas_type) {
+
+	switch (meas_type) {
+	case basic_rep:
+		return "basic_rep";
+	case cca_rep:
+		return "cca_rep";
+	case rpi_histo_rep:
+		return "rpi_histo_rep";
+	case ch_load_rep:
+		return "ch_load_rep";
+	case noise_histo_rep:
+		return "noise_histo_rep";
+	case bcn_rep:
+		return "bcn_rep";
+	case frame_rep:
+		return "frame_rep";
+	case sta_statis_rep:
+		return "sta_statis_rep";
+	}
+	return "unknown_rep";
+};
+
+static char *rm_en_cap_name(enum rm_cap_en en)
+{
+	switch (en) {
+	case RM_LINK_MEAS_CAP_EN:
+		return "RM_LINK_MEAS_CAP_EN";
+	case RM_NB_REP_CAP_EN:
+		return "RM_NB_REP_CAP_EN";
+	case RM_PARAL_MEAS_CAP_EN:
+		return "RM_PARAL_MEAS_CAP_EN";
+	case RM_REPEAT_MEAS_CAP_EN:
+		return "RM_REPEAT_MEAS_CAP_EN";
+	case RM_BCN_PASSIVE_MEAS_CAP_EN:
+		return "RM_BCN_PASSIVE_MEAS_CAP_EN";
+	case RM_BCN_ACTIVE_MEAS_CAP_EN:
+		return "RM_BCN_ACTIVE_MEAS_CAP_EN";
+	case RM_BCN_TABLE_MEAS_CAP_EN:
+		return "RM_BCN_TABLE_MEAS_CAP_EN";
+	case RM_BCN_MEAS_REP_COND_CAP_EN:
+		return "RM_BCN_MEAS_REP_COND_CAP_EN";
+
+	case RM_FRAME_MEAS_CAP_EN:
+		return "RM_FRAME_MEAS_CAP_EN";
+	case RM_CH_LOAD_CAP_EN:
+		return "RM_CH_LOAD_CAP_EN";
+	case RM_NOISE_HISTO_CAP_EN:
+		return "RM_NOISE_HISTO_CAP_EN";
+	case RM_STATIS_MEAS_CAP_EN:
+		return "RM_STATIS_MEAS_CAP_EN";
+	case RM_LCI_MEAS_CAP_EN:
+		return "RM_LCI_MEAS_CAP_EN";
+	case RM_LCI_AMIMUTH_CAP_EN:
+		return "RM_LCI_AMIMUTH_CAP_EN";
+	case RM_TRANS_STREAM_CAT_MEAS_CAP_EN:
+		return "RM_TRANS_STREAM_CAT_MEAS_CAP_EN";
+	case RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN:
+		return "RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN";
+
+	case RM_AP_CH_REP_CAP_EN:
+		return "RM_AP_CH_REP_CAP_EN";
+	case RM_RM_MIB_CAP_EN:
+		return "RM_RM_MIB_CAP_EN";
+	case RM_OP_CH_MAX_MEAS_DUR0:
+		return "RM_OP_CH_MAX_MEAS_DUR0";
+	case RM_OP_CH_MAX_MEAS_DUR1:
+		return "RM_OP_CH_MAX_MEAS_DUR1";
+	case RM_OP_CH_MAX_MEAS_DUR2:
+		return "RM_OP_CH_MAX_MEAS_DUR2";
+	case RM_NONOP_CH_MAX_MEAS_DUR0:
+		return "RM_NONOP_CH_MAX_MEAS_DUR0";
+	case RM_NONOP_CH_MAX_MEAS_DUR1:
+		return "RM_NONOP_CH_MAX_MEAS_DUR1";
+	case RM_NONOP_CH_MAX_MEAS_DUR2:
+		return "RM_NONOP_CH_MAX_MEAS_DUR2";
+
+	case RM_MEAS_PILOT_CAP0:
+		return "RM_MEAS_PILOT_CAP0";		/* 24-26 */
+	case RM_MEAS_PILOT_CAP1:
+		return "RM_MEAS_PILOT_CAP1";
+	case RM_MEAS_PILOT_CAP2:
+		return "RM_MEAS_PILOT_CAP2";
+	case RM_MEAS_PILOT_TRANS_INFO_CAP_EN:
+		return "RM_MEAS_PILOT_TRANS_INFO_CAP_EN";
+	case RM_NB_REP_TSF_OFFSET_CAP_EN:
+		return "RM_NB_REP_TSF_OFFSET_CAP_EN";
+	case RM_RCPI_MEAS_CAP_EN:
+		return "RM_RCPI_MEAS_CAP_EN";		/* 29 */
+	case RM_RSNI_MEAS_CAP_EN:
+		return "RM_RSNI_MEAS_CAP_EN";
+	case RM_BSS_AVG_ACCESS_DELAY_CAP_EN:
+		return "RM_BSS_AVG_ACCESS_DELAY_CAP_EN";
+
+	case RM_AVALB_ADMIS_CAPACITY_CAP_EN:
+		return "RM_AVALB_ADMIS_CAPACITY_CAP_EN";
+	case RM_ANT_CAP_EN:
+		return "RM_ANT_CAP_EN";
+	case RM_RSVD:
+	case RM_MAX:
+	default:
+		break;
+	}
+	return "unknown";
+}
+
+static void rm_set_rep_mode(struct rrm_obj *prm, u8 mode)
+{
+
+	FSM_DBG(prm, "set %s\n",
+		mode&MEAS_REP_MOD_INCAP?"INCAP":
+		mode&MEAS_REP_MOD_REFUSE?"REFUSE":
+		mode&MEAS_REP_MOD_LATE?"LATE":"");
+
+	prm->p.m_mode |= mode;
+}
+
+int rm_en_cap_chk_and_set(struct rrm_obj *prm, enum rm_cap_en en)
+{
+	struct rrm_priv *rmpriv = (struct rrm_priv *)obj2priv(prm);
+	int idx;
+	u8 cap;
+
+
+	if (en >= RM_MAX)
+		return _FALSE;
+
+	idx = en / 8;
+	cap = rmpriv->rm_en_cap_def[idx];
+
+	if (!(cap & BIT(en - (idx*8)))) {
+		FSM_INFO(prm, "%s incapable\n",rm_en_cap_name(en));
+		rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+		return _FALSE;
+	}
+	return _SUCCESS;
+}
+
+static u8 rm_get_ch_set(u8 op_class, struct rtw_ieee80211_channel *pch_set, u8 pch_num)
+{
+	int i, array_idx;
+	const struct op_class_t *opc = get_global_op_class_by_id(op_class);
+
+	if (opc < global_op_class
+		|| (((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t)
+	) {
+		RTW_ERR("Invalid opc pointer:%p (global_op_class:%p, sizeof(struct op_class_t):%zu, %zu)\n"
+			, opc, global_op_class, sizeof(struct op_class_t),
+			(((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t));
+		return 0;
+	}
+
+	array_idx = (((u8 *)opc) - ((u8 *)global_op_class)) / sizeof(struct op_class_t);
+	if (pch_num < OPC_CH_LIST_LEN(global_op_class[array_idx])) {
+		RTW_ERR("Invalid pch len %d < %d\n",pch_num,
+			OPC_CH_LIST_LEN(global_op_class[array_idx]));
+		return 0;
+	}
+
+	for (i = 0; i < OPC_CH_LIST_LEN(global_op_class[array_idx]); i++) {
+		pch_set[i].hw_value = OPC_CH_LIST_CH(global_op_class[array_idx], i);
+		pch_set[i].band = global_op_class[array_idx].band;
+	}
+
+	return i;
+}
+
+
+static u8 rm_get_ch_set_from_bcn_req_opt(struct rrm_obj *prm, struct bcn_req_opt *opt,
+	struct rtw_ieee80211_channel *pch_set, u8 pch_num)
+{
+	int i,j,k,sz;
+	struct _RT_OPERATING_CLASS *ap_ch_rpt;
+	enum band_type band;
+	u8 ch_amount = 0;
+
+	k = 0;
+	for (i = 0; i < opt->ap_ch_rpt_num; i++) {
+		if (opt->ap_ch_rpt[i] == NULL)
+			break;
+
+		ap_ch_rpt = opt->ap_ch_rpt[i];
+		band = rtw_get_band_by_op_class(ap_ch_rpt->global_op_class);
+
+		if ((k + ap_ch_rpt->Len) > pch_num) {
+			RTW_ERR("rrm: ch num exceed %d > %d\n", (k + ap_ch_rpt->Len), pch_num);
+			return k;
+		}
+
+		for (j = 0; j < ap_ch_rpt->Len; j++) {
+			pch_set[k].hw_value =
+				ap_ch_rpt->Channel[j];
+			pch_set[k].band = band;
+			FSM_TRACE(prm, "meas_ch[%d].hw_value = %u\n",
+				k, pch_set[k].hw_value);
+			k++;
+		}
+	}
+	return k;
+}
+
+static int is_wildcard_bssid(u8 *bssid)
+{
+	int i;
+	u8 val8 = 0xff;
+
+
+	for (i=0;i<6;i++)
+		val8 &= bssid[i];
+
+	if (val8 == 0xff)
+		return _SUCCESS;
+	return _FALSE;
+}
+
+static u8 translate_dbm_to_rcpi(s8 SignalPower)
+{
+	/* RCPI = Int{(Power in dBm + 110)*2} for 0dBm > Power > -110dBm
+	 *    0	: power <= -110.0 dBm
+	 *    1	: power =  -109.5 dBm
+	 *    2	: power =  -109.0 dBm
+	 */
+	return (SignalPower + 110)*2;
+}
+
+static u8 translate_percentage_to_rcpi(u32 SignalStrengthIndex)
+{
+	/* Translate to dBm (x=y-100) */
+	return translate_dbm_to_rcpi(SignalStrengthIndex - 100);
+}
+
+static u8 rm_get_bcn_rcpi(struct rrm_obj *prm, struct wlan_network *pnetwork)
+{
+	return translate_percentage_to_rcpi(
+		pnetwork->network.PhyInfo.SignalStrength);
+}
+
+static u8 rm_get_frame_rsni(struct rrm_obj *prm, union recv_frame *pframe)
+{
+#if 0
+	int i;
+	u8 val8 = 0, snr;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(prm->psta->padapter);
+	u8 rf_path = GET_HAL_RFPATH_NUM(dvobj);
+	if (IS_CCK_RATE((hw_rate_to_m_rate(pframe->u.hdr.attrib.data_rate))))
+		val8 = 255;
+	else {
+		snr = 0;
+		for (i = 0; i < rf_path; i++)
+			snr += pframe->u.hdr.attrib.phy_info.rx_snr[i];
+		snr = snr / rf_path;
+		val8 = (u8)(snr + 10)*2;
+	}
+#endif
+	return 0;
+}
+
+static u8 rm_get_bcn_rsni(struct rrm_obj *prm, struct wlan_network *pnetwork)
+{
+	int i;
+	u8 val8, snr;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(obj2adp(prm));
+	u8 rf_path = GET_HAL_RFPATH_NUM(dvobj);
+
+
+	if (pnetwork->network.PhyInfo.is_cck_rate) {
+		/* current HW doesn't have CCK RSNI */
+		/* 255 indicates RSNI is unavailable */
+		val8 = 255;
+	} else {
+		snr = 0;
+		for (i = 0; i < rf_path; i++) {
+			snr += pnetwork->network.PhyInfo.rx_snr[i];
+		}
+		snr = snr / rf_path;
+		val8 = (u8)(snr + 10)*2;
+	}
+	return val8;
+}
+
+/* output: pwr (unit dBm) */
+static int rm_get_tx_power(_adapter *adapter, enum band_type band, enum MGN_RATE rate, s8 *pwr)
+{
+	struct dvobj_priv *devob = adapter_to_dvobj(adapter);
+	u8 rs = mgn_rate_to_rs(rate);
+
+	*pwr = rtw_phl_get_power_by_rate_band(GET_PHL_INFO(devob), HW_BAND_0, _rate_mrate2phl(rate),
+					       IS_DCM_RATE_SECTION(rs) ? 1 : 0, 0, band);
+	return 0;
+}
+
+static u8 rrm_gen_dialog_token(_adapter *padapter)
+{
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	do {
+		pmlmeinfo->dialogToken++;
+	} while (pmlmeinfo->dialogToken == 0);
+	return pmlmeinfo->dialogToken;
+}
+
+static u8 rrm_gen_meas_token(_adapter *padapter)
+{
+	struct rrm_priv *prmpriv = &(padapter->fsmpriv.rmpriv);
+
+	do {
+		prmpriv->meas_token++;
+	} while (prmpriv->meas_token == 0);
+
+	return prmpriv->meas_token;
+}
+
+static u16 rrm_cal_cid(u8 token, u16 role)
+{
+	u16 cid;
+
+	cid = role | token;
+
+	return cid;
+}
+
+static int rrm_get_rx_sensitivity(_adapter *adapter, enum channel_width bw, enum MGN_RATE rate, s8 *pwr)
+{
+	s8 rx_sensitivity = -110;
+
+	switch(rate) {
+	case MGN_1M:
+		rx_sensitivity= -101;
+		break;
+	case MGN_2M:
+		rx_sensitivity= -98;
+		break;
+	case MGN_5_5M:
+		rx_sensitivity= -92;
+		break;
+	case MGN_11M:
+		rx_sensitivity= -89;
+		break;
+	case MGN_6M:
+	case MGN_9M:
+	case MGN_12M:
+		rx_sensitivity = -92;
+		break;
+	case MGN_18M:
+		rx_sensitivity = -90;
+		break;
+	case MGN_24M:
+		rx_sensitivity = -88;
+		break;
+	case MGN_36M:
+		rx_sensitivity = -84;
+		break;
+	case MGN_48M:
+		rx_sensitivity = -79;
+		break;
+	case MGN_54M:
+		rx_sensitivity = -78;
+		break;
+
+	case MGN_MCS0:
+	case MGN_MCS8:
+	case MGN_MCS16:
+	case MGN_MCS24:
+	case MGN_VHT1SS_MCS0:
+	case MGN_VHT2SS_MCS0:
+	case MGN_VHT3SS_MCS0:
+	case MGN_VHT4SS_MCS0:
+		/* BW20 BPSK 1/2 */
+		rx_sensitivity = -82;
+		break;
+
+	case MGN_MCS1:
+	case MGN_MCS9:
+	case MGN_MCS17:
+	case MGN_MCS25:
+	case MGN_VHT1SS_MCS1:
+	case MGN_VHT2SS_MCS1:
+	case MGN_VHT3SS_MCS1:
+	case MGN_VHT4SS_MCS1:
+		/* BW20 QPSK 1/2 */
+		rx_sensitivity = -79;
+		break;
+
+	case MGN_MCS2:
+	case MGN_MCS10:
+	case MGN_MCS18:
+	case MGN_MCS26:
+	case MGN_VHT1SS_MCS2:
+	case MGN_VHT2SS_MCS2:
+	case MGN_VHT3SS_MCS2:
+	case MGN_VHT4SS_MCS2:
+		/* BW20 QPSK 3/4 */
+		rx_sensitivity = -77;
+		break;
+
+	case MGN_MCS3:
+	case MGN_MCS11:
+	case MGN_MCS19:
+	case MGN_MCS27:
+	case MGN_VHT1SS_MCS3:
+	case MGN_VHT2SS_MCS3:
+	case MGN_VHT3SS_MCS3:
+	case MGN_VHT4SS_MCS3:
+		/* BW20 16-QAM 1/2 */
+		rx_sensitivity = -74;
+		break;
+
+	case MGN_MCS4:
+	case MGN_MCS12:
+	case MGN_MCS20:
+	case MGN_MCS28:
+	case MGN_VHT1SS_MCS4:
+	case MGN_VHT2SS_MCS4:
+	case MGN_VHT3SS_MCS4:
+	case MGN_VHT4SS_MCS4:
+		/* BW20 16-QAM 3/4 */
+		rx_sensitivity = -70;
+		break;
+
+	case MGN_MCS5:
+	case MGN_MCS13:
+	case MGN_MCS21:
+	case MGN_MCS29:
+	case MGN_VHT1SS_MCS5:
+	case MGN_VHT2SS_MCS5:
+	case MGN_VHT3SS_MCS5:
+	case MGN_VHT4SS_MCS5:
+		/* BW20 64-QAM 2/3 */
+		rx_sensitivity = -66;
+		break;
+
+	case MGN_MCS6:
+	case MGN_MCS14:
+	case MGN_MCS22:
+	case MGN_MCS30:
+	case MGN_VHT1SS_MCS6:
+	case MGN_VHT2SS_MCS6:
+	case MGN_VHT3SS_MCS6:
+	case MGN_VHT4SS_MCS6:
+		/* BW20 64-QAM 3/4 */
+		rx_sensitivity = -65;
+		break;
+
+	case MGN_MCS7:
+	case MGN_MCS15:
+	case MGN_MCS23:
+	case MGN_MCS31:
+	case MGN_VHT1SS_MCS7:
+	case MGN_VHT2SS_MCS7:
+	case MGN_VHT3SS_MCS7:
+	case MGN_VHT4SS_MCS7:
+		/* BW20 64-QAM 5/6 */
+		rx_sensitivity = -64;
+		break;
+
+	case MGN_VHT1SS_MCS8:
+	case MGN_VHT2SS_MCS8:
+	case MGN_VHT3SS_MCS8:
+	case MGN_VHT4SS_MCS8:
+		/* BW20 256-QAM 3/4 */
+		rx_sensitivity = -59;
+		break;
+
+	case MGN_VHT1SS_MCS9:
+	case MGN_VHT2SS_MCS9:
+	case MGN_VHT3SS_MCS9:
+	case MGN_VHT4SS_MCS9:
+		/* BW20 256-QAM 5/6 */
+		rx_sensitivity = -57;
+		break;
+
+	default:
+		return -1;
+		break;
+
+	}
+
+	switch(bw) {
+	case CHANNEL_WIDTH_20:
+		break;
+	case CHANNEL_WIDTH_40:
+		rx_sensitivity -= 3;
+		break;
+	case CHANNEL_WIDTH_80:
+		rx_sensitivity -= 6;
+		break;
+	case CHANNEL_WIDTH_160:
+		rx_sensitivity -= 9;
+		break;
+	case CHANNEL_WIDTH_5:
+	case CHANNEL_WIDTH_10:
+	case CHANNEL_WIDTH_80_80:
+	default:
+		return -1;
+		break;
+	}
+	*pwr = rx_sensitivity;
+
+	return 0;
+}
+
+/* output: path_a max tx power in dBm */
+static int rrm_get_path_a_max_tx_power(_adapter *adapter, s8 *path_a)
+{
+#if 0 /*GEORGIA_TODO_FIXIT*/
+
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct hal_spec_t *hal_spec = GET_HAL_SPEC(dvobj);
+	HAL_DATA_TYPE *hal_data = GET_PHL_COM(dvobj);
+	int path, tx_num, band, bw, ch, n, rs;
+	u8 rate_num;
+	s8 max_pwr[RF_PATH_MAX], pwr;
+
+
+	band = hal_data->current_band_type;
+	bw = hal_data->current_channel_bw;
+	ch = hal_data->current_channel;
+
+	for (path = 0; path < RF_PATH_MAX; path++) {
+		if (!HAL_SPEC_CHK_RF_PATH(hal_spec, band, path))
+			break;
+
+		max_pwr[path] = -127; /* min value of s8 */
+#if (RM_MORE_DBG_MSG)
+		RTW_INFO("[%s][%c]\n", band_str(band), rf_path_char(path));
+#endif
+		for (rs = 0; rs < RATE_SECTION_NUM; rs++) {
+			tx_num = rate_section_to_tx_num(rs);
+
+			if (tx_num >= hal_spec->tx_nss_num)
+				continue;
+
+			if (band == BAND_ON_5G && IS_CCK_RATE_SECTION(rs))
+				continue;
+
+			if (IS_VHT_RATE_SECTION(rs) && !IS_HARDWARE_TYPE_JAGUAR_ALL(adapter))
+				continue;
+
+			rate_num = rate_section_rate_num(rs);
+
+			/* get power by rate in db */
+			for (n = rate_num - 1; n >= 0; n--) {
+				pwr = phy_get_tx_power_final_absolute_value(adapter, path, rates_by_sections[rs].rates[n], bw, ch);
+				max_pwr[path] = MAX(max_pwr[path], pwr);
+#if (RM_MORE_DBG_MSG)
+				RTW_INFO("%9s = %2d\n",
+					MGN_RATE_STR(rates_by_sections[rs].rates[n]), pwr);
+#endif
+			}
+		}
+	}
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("path_a max_pwr=%ddBm\n", max_pwr[0]);
+#endif
+	*path_a = max_pwr[0];
+#endif
+	return 0;
+}
+
+static u8 *build_wlan_hdr(struct rrm_obj *prm, _adapter *padapter,
+	struct xmit_frame *pmgntframe, struct sta_info *psta, u16 frame_type)
+{
+	u8 *pframe;
+	u16 *fctrl;
+	struct pkt_attrib *pattr;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_ext_info *pmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+
+
+	/* update attribute */
+	pattr = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattr);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, psta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+
+	FSM_TRACE(prm, "dst = " MAC_FMT "\n", MAC_ARG(pwlanhdr->addr1));
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	SetFragNum(pframe, 0);
+
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattr->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	return pframe;
+}
+
+int rrm_issue_null_reply(struct rrm_obj *prm)
+{
+	int len=0, my_len;
+	u8 *pframe, m_mode;
+	_adapter *padapter = obj2adp(prm);
+	struct pkt_attrib *pattr;
+	struct xmit_frame *pmgntframe;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+
+	m_mode = prm->p.m_mode;
+	if (m_mode || prm->q.rpt == 0) {
+		FSM_INFO(prm, "reply %s\n",
+			m_mode&MEAS_REP_MOD_INCAP?"INCAP":
+			m_mode&MEAS_REP_MOD_REFUSE?"REFUSE":
+			m_mode&MEAS_REP_MOD_LATE?"LATE":"no content");
+	}
+
+	switch (prm->q.action_code) {
+	case RM_ACT_RADIO_MEAS_REQ:
+		len = 8;
+		break;
+	case RM_ACT_NB_REP_REQ:
+		len = 3;
+		break;
+	case RM_ACT_LINK_MEAS_REQ:
+		len = 3;
+		break;
+	default:
+		break;
+	}
+
+	if (len==0)
+		return _FALSE;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		FSM_ERR(prm, "%s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(prm, padapter, pmgntframe, obj2sta(prm), WIFI_ACTION);
+	pframe = rtw_set_fixed_ie(pframe, 3, &prm->p.category, &pattr->pktlen);
+
+	my_len = 0;
+	if (len>5) {
+		prm->p.len = len - 3 - 2;
+		pframe = rtw_set_fixed_ie(pframe, len - 3,
+			&prm->p.e_id, &my_len);
+	}
+
+	pattr->pktlen += my_len;
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+static int rm_prepare_scan(struct rrm_obj *prm)
+{
+	_adapter *padapter = obj2adp(prm);
+	u8 ssc_chk;
+
+	ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+
+	if (ssc_chk == SS_ALLOW) {
+		return _SUCCESS;
+	} else if (ssc_chk == SS_DENY_BUSY_TRAFFIC) {
+		FSM_DBG(prm, "Don't care BusyTraffic\n");
+		return _SUCCESS;
+	}
+	return _FALSE;
+}
+
+int rrm_get_chset(struct rrm_obj *prm)
+{
+	int i,meas_ch_amount=0;
+	u8 op_class=0, val8;
+	struct rtw_ieee80211_channel *pch_set;
+
+	pch_set = &prm->q.ch_set[0];
+
+	_rtw_memset(pch_set, 0,
+		sizeof(struct rtw_ieee80211_channel) * RTW_CHANNEL_SCAN_AMOUNT);
+
+	op_class = prm->q.op_class;
+	if (prm->q.ch_num == 0) {
+		/* ch_num=0   : scan all ch in operating class */
+		meas_ch_amount = rm_get_ch_set(op_class, pch_set, RTW_CHANNEL_SCAN_AMOUNT);
+
+	} else if (prm->q.ch_num == 255) {
+		/* 802.11 p.1066 */
+		/* ch_num=255 : If the Channel Number is 255 and includes
+		 * AP Channel Report subelements
+		 */
+		meas_ch_amount = rm_get_ch_set_from_bcn_req_opt(prm,
+			&prm->q.opt.bcn, pch_set, RTW_CHANNEL_SCAN_AMOUNT);
+	} else {
+		pch_set[0].hw_value = prm->q.ch_num;
+		pch_set[0].band = prm->q.band;
+		meas_ch_amount = 1;
+		FSM_INFO(prm, "meas_ch->hw_value = %u\n", pch_set->hw_value);
+	}
+
+	/* get means channel */
+	prm->q.ch_set_ch_amount = meas_ch_amount;
+
+	return 0;
+}
+
+static int rrm_sitesurvey(struct rrm_obj *prm)
+{
+	int meas_ch_amount=0;
+	struct rtw_ieee80211_channel *pch_set;
+	struct sitesurvey_parm *parm;
+
+	parm = rtw_malloc(sizeof(*parm));
+	if (parm == NULL)
+		return _FAIL;
+
+	rrm_get_chset(prm);
+	pch_set = &prm->q.ch_set[0];
+
+	meas_ch_amount = MIN(prm->q.ch_set_ch_amount, RTW_CHANNEL_SCAN_AMOUNT);
+	_rtw_memset(parm, 0, sizeof(struct sitesurvey_parm));
+	_rtw_memcpy(parm->ch, pch_set, sizeof(struct rtw_ieee80211_channel) * meas_ch_amount);
+
+	_rtw_memcpy(&parm->ssid[0], &prm->q.opt.bcn.ssid, IW_ESSID_MAX_SIZE);
+
+	parm->ssid_num = 1;
+	parm->scan_mode = prm->q.s_mode;
+	parm->ch_num = meas_ch_amount;
+	parm->duration = prm->q.meas_dur;
+	parm->scan_type = RTW_SCAN_RRM;
+	parm->psta = obj2sta(prm);
+	/* parm.bw = BW_20M; */
+
+	rtw_sitesurvey_cmd(obj2adp(prm), parm);
+	rtw_mfree(parm, sizeof(*parm));
+
+	if (prm->q.opt.bcn.ssid.SsidLength)
+		FSM_INFO(prm, "%s search %s\n", __func__, prm->q.opt.bcn.ssid.Ssid);
+	else
+		FSM_INFO(prm, "%s\n", __func__);
+
+	return _SUCCESS;
+}
+
+static int rrm_parse_ch_load_s_elem(struct rrm_obj *prm, u8 *pbody, int req_len)
+{
+	u8 *popt_id;
+	int i, p=0; /* position */
+	int len = req_len;
+
+
+	prm->q.opt_s_elem_len = len;
+
+	while (len) {
+
+		switch (pbody[p]) {
+		case ch_load_rep_info:
+			/* check RM_EN */
+			rm_en_cap_chk_and_set(prm, RM_CH_LOAD_CAP_EN);
+
+			_rtw_memcpy(&(prm->q.opt.clm.rep_cond),
+				&pbody[p+2], sizeof(prm->q.opt.clm.rep_cond));
+
+			FSM_INFO(prm, "ch_load_rep_info=%u:%u\n",
+				prm->q.opt.clm.rep_cond.cond,
+				prm->q.opt.clm.rep_cond.threshold);
+			break;
+		default:
+			break;
+
+		}
+		len = len - (int)pbody[p+1] - 2;
+		p = p + (int)pbody[p+1] + 2;
+	}
+	return _SUCCESS;
+}
+
+static int rrm_parse_noise_histo_s_elem(struct rrm_obj *prm,
+	u8 *pbody, int req_len)
+{
+	u8 *popt_id;
+	int i, p=0; /* position */
+	int len = req_len;
+
+
+	prm->q.opt_s_elem_len = len;
+
+	while (len) {
+
+		switch (pbody[p]) {
+		case noise_histo_rep_info:
+			/* check RM_EN */
+			rm_en_cap_chk_and_set(prm, RM_NOISE_HISTO_CAP_EN);
+
+			_rtw_memcpy(&(prm->q.opt.nhm.rep_cond),
+				&pbody[p+2], sizeof(prm->q.opt.nhm.rep_cond));
+
+			FSM_INFO(prm, "noise_histo_rep_info=%u:%u\n",
+				prm->q.opt.nhm.rep_cond.cond,
+				prm->q.opt.nhm.rep_cond.threshold);
+			break;
+		default:
+			break;
+
+       		}
+		len = len - (int)pbody[p+1] - 2;
+		p = p + (int)pbody[p+1] + 2;
+	}
+	return _SUCCESS;
+}
+
+static int rrm_parse_bcn_req_s_elemm(struct rrm_obj *prm, u8 *pbody, int req_len)
+{
+	u8 *popt_id;
+	int i, p=0; /* position */
+	int len = req_len;
+	int ap_ch_rpt_idx = 0;
+	struct _RT_OPERATING_CLASS *op;
+
+
+	/* opt length,2:pbody[0]+ pbody[1] */
+	/* first opt id : pbody[18] */
+
+	prm->q.opt_s_elem_len = len;
+
+	popt_id = prm->q.opt.bcn.opt_id;
+	while (len && prm->q.opt.bcn.opt_id_num < BCN_REQ_OPT_MAX_NUM) {
+
+		switch (pbody[p]) {
+		case bcn_req_ssid:
+#if (DBG_BCN_REQ_WILDCARD)
+			FSM_INFO(prm, "DBG set ssid to WILDCARD\n");
+#else
+#if (DBG_BCN_REQ_SSID)
+			FSM_INFO(prm, "DBG set ssid to %s\n",DBG_BCN_REQ_SSID_NAME);
+			i = strlen(DBG_BCN_REQ_SSID_NAME);
+			prm->q.opt.bcn.ssid.SsidLength = i;
+			_rtw_memcpy(&(prm->q.opt.bcn.ssid.Ssid), DBG_BCN_REQ_SSID_NAME,
+				MIN(i, sizeof(prm->q.opt.bcn.ssid.Ssid)-1));
+
+#else /* original */
+			prm->q.opt.bcn.ssid.SsidLength = pbody[p+1];
+			_rtw_memcpy(&(prm->q.opt.bcn.ssid.Ssid), &pbody[p+2],
+				MIN(pbody[p+1], sizeof(prm->q.opt.bcn.ssid.Ssid)-1));
+#endif
+#endif
+			FSM_DBG(prm, "bcn_req_ssid %s\n", prm->q.opt.bcn.ssid.Ssid);
+
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+			break;
+
+		case bcn_req_rep_info:
+			FSM_TRACE(prm, "bcn_req_rep_info\n");
+			/* check RM_EN */
+			rm_en_cap_chk_and_set(prm, RM_BCN_MEAS_REP_COND_CAP_EN);
+
+			_rtw_memcpy(&(prm->q.opt.bcn.rep_cond),
+				&pbody[p+2], sizeof(prm->q.opt.bcn.rep_cond));
+
+			FSM_TRACE(prm, "bcn_req_rep_info=%u:%u\n",
+				prm->q.opt.bcn.rep_cond.cond,
+				prm->q.opt.bcn.rep_cond.threshold);
+
+			/*popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];*/
+			break;
+
+		case bcn_req_rep_detail:
+			FSM_TRACE(prm, "bcn_req_rep_detail\n");
+#if DBG_BCN_REQ_DETAIL
+			prm->q.opt.bcn.rep_detail = 2; /* all IE in beacon */
+#else
+			prm->q.opt.bcn.rep_detail = pbody[p+2];
+#endif
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+
+			FSM_TRACE(prm, "report_detail=%d\n",
+				prm->q.opt.bcn.rep_detail);
+			break;
+		case bcn_req_last_bcn_rpt_ind:
+			prm->q.opt.bcn.rep_last_bcn_ind = pbody[p+2];
+#if DBG_BCN_REQ_DETAIL
+			RTW_INFO("RM: bcn_req_last_bcn_rpt_ind=%d\n",
+				prm->q.opt.bcn.rep_last_bcn_ind);
+#endif
+			break;
+
+		case bcn_req_req:
+
+			FSM_TRACE(prm, "bcn_req_req\n");
+
+			prm->q.opt.bcn.req_start = rtw_malloc(pbody[p+1]);
+
+			if (prm->q.opt.bcn.req_start == NULL) {
+				FSM_ERR(prm, "req_start malloc fail!!\n");
+				break;
+			}
+
+			for (i = 0; i < pbody[p+1]; i++)
+				*((prm->q.opt.bcn.req_start)+i) =
+					pbody[p+2+i];
+
+			prm->q.opt.bcn.req_len = pbody[p+1];
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+			break;
+
+		case bcn_req_ap_ch_rep:
+
+			FSM_TRACE(prm, "rrm: bcn_req_ap_ch_rep\n");
+
+			if (ap_ch_rpt_idx > BCN_REQ_OPT_AP_CH_RPT_MAX_NUM) {
+				FSM_ERR(prm, "bcn_req_ap_ch_rep over size\n");
+				break;
+			}
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+
+			/* get channel list
+			 * EID:len:op-class:ch-list
+			 */
+			op = rtw_malloc(sizeof (*op));
+			op->global_op_class = pbody[p + 2];
+			i = pbody[p + 1] - 1; /* ch list len; (-1) is op class */
+
+			FSM_DBG(prm, "%d op class %d has %d ch\n",
+				ap_ch_rpt_idx,op->global_op_class,i);
+
+			op->Len = i;
+			memcpy(op->Channel, &pbody[p + 3],
+				MIN(i, MAX_CH_NUM_IN_OP_CLASS));
+			prm->q.opt.bcn.ap_ch_rpt[ap_ch_rpt_idx++] = op;
+			prm->q.opt.bcn.ap_ch_rpt_num = ap_ch_rpt_idx;
+			break;
+
+		default:
+			break;
+
+		}
+		len = len - (int)pbody[p+1] - 2;
+		p = p + (int)pbody[p+1] + 2;
+	}
+
+	return _SUCCESS;
+}
+
+static int rrm_parse_meas_req(struct rrm_obj *prm, u8 *pbody)
+{
+	int p; /* position */
+	int req_len;
+
+
+	req_len = (int)pbody[1];
+	p = 5;
+
+	prm->q.op_class = pbody[p++];
+	prm->q.ch_num = pbody[p++];
+	prm->q.band = rtw_get_band_by_op_class(prm->q.op_class);
+	prm->q.rand_intvl = le16_to_cpu(*(u16*)(&pbody[p]));
+	p+=2;
+	prm->q.meas_dur = le16_to_cpu(*(u16*)(&pbody[p]));
+	p+=2;
+
+	if (prm->q.m_type == bcn_req) {
+		/*
+		 * 0: passive
+		 * 1: active
+		 * 2: bcn_table
+		 */
+		prm->q.s_mode = pbody[p++];
+
+		/* BSSID */
+		_rtw_memcpy(&(prm->q.bssid), &pbody[p], 6);
+		p+=6;
+
+		/*
+		 * default, used when Reporting detail subelement
+		 * is not included in Beacon Request
+		 */
+		prm->q.opt.bcn.rep_detail = 2;
+	}
+
+	if (req_len-(p-2) <= 0) /* without sub-element */
+		return _SUCCESS;
+
+	switch (prm->q.m_type) {
+	case bcn_req:
+		rrm_parse_bcn_req_s_elemm(prm, &pbody[p], req_len-(p-2));
+		break;
+	case ch_load_req:
+		rrm_parse_ch_load_s_elem(prm, &pbody[p], req_len-(p-2));
+		break;
+	case noise_histo_req:
+		rrm_parse_noise_histo_s_elem(prm, &pbody[p], req_len-(p-2));
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int rrm_enqueue_rm(struct rrm_priv *prmpriv , struct rrm_obj *prm, u8 to_head)
+{
+	_queue *queue = &prmpriv->rm_queue;
+
+	_rtw_spinlock_bh(&queue->lock);
+	if (to_head)
+		list_add(&prm->list, &queue->queue);
+	else
+		list_add_tail(&prm->list, &queue->queue);
+	_rtw_spinunlock_bh(&queue->lock);
+
+	return 0;
+}
+
+static int rrm_dequeue_rm(struct rrm_priv *prmpriv , struct rrm_obj *prm)
+{
+	_queue *queue = &prmpriv->rm_queue;
+
+	_rtw_spinlock_bh(&queue->lock);
+	list_del(&prm->list);
+	_rtw_spinunlock_bh(&queue->lock);
+
+	return 0;
+}
+
+static struct rrm_obj *rtw_rrm_new_obj(_adapter *a, struct sta_info *psta, u8 diag_token)
+{
+	struct fsm_main *fsm = a->fsmpriv.rmpriv.fsm;
+	struct rrm_priv *prmpriv = &(a->fsmpriv.rmpriv);
+	struct fsm_obj *obj;
+	struct rrm_obj *prm;
+	u16 cid;
+
+	if (diag_token) {
+		cid = rrm_cal_cid(diag_token, RM_SLAVE);
+	} else {
+		diag_token = rrm_gen_dialog_token(a);
+		cid = rrm_cal_cid(diag_token, RM_MASTER);
+	}
+
+	obj = rtw_fsm_new_obj(fsm, psta, cid, (void **)&prm, sizeof(*prm));
+
+	if (prm == NULL) {
+		FSM_ERR_(fsm, "rrm: malloc obj fail\n");
+		return NULL;
+	}
+
+	prm->fsm = fsm;
+	prm->obj = obj;
+	prm->cid = cid;
+	prm->p.diag_token = diag_token;
+	prm->q.diag_token = diag_token;
+
+	return prm;
+}
+
+/* receive measurement request */
+int rrm_recv_radio_mens_req(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	struct rrm_obj *prm;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 ssc_chk, diag_token = pdiag_body[2];
+	u8 *pmeas_body = &pdiag_body[5];
+	u8 need_to_scan = 1;
+	u16 cid;
+
+#if 0
+	if (pmeas_body[4] == bcn_req) {
+		rtw_cfg80211_rx_rrm_action(padapter, precv_frame);
+		return _SUCCESS;
+	}
+#endif
+
+	/* search existing rrm_obj */
+	cid = rrm_cal_cid(pdiag_body[2], RM_SLAVE);
+	prm = rtw_fsm_get_obj(prmpriv->fsm, cid);
+	if (prm) {
+		FSM_INFO(prm, "Found an exist meas\n");
+		return _FALSE;
+	}
+
+	prm = rtw_rrm_new_obj(padapter, psta, diag_token);
+	if (prm == NULL) {
+		RTW_ERR("rrm: unable to alloc rm obj for requeset\n");
+		return _FALSE;
+	}
+
+	prm->q.diag_token = diag_token;
+	prm->q.rpt = le16_to_cpu(*(u16*)(&pdiag_body[3]));
+
+	/* Figure 8-104 Measurement Requested format */
+	prm->q.e_id = pmeas_body[0];
+	prm->q.m_token = pmeas_body[2];
+	prm->q.m_mode = pmeas_body[3];
+	prm->q.m_type = pmeas_body[4];
+
+	//FSM_INFO(prm, "bssid " MAC_FMT "\n", MAC_ARG(obj2mac(prm)));
+
+	FSM_TRACE(prm, "element_id = %d\n", prm->q.e_id);
+	FSM_TRACE(prm, "length = %d\n", (int)pmeas_body[1]);
+	FSM_TRACE(prm, "meas_token = %d\n", prm->q.m_token);
+	FSM_TRACE(prm, "meas_mode = %d\n", prm->q.m_mode);
+	FSM_TRACE(prm, "meas_type = %d\n", prm->q.m_type);
+
+	if (prm->q.e_id != _MEAS_REQ_IE_) { /* 38 */
+		rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+		RTW_WARN("rrm: Unsupported Action element(%d!=%d)!\n",
+			prm->q.e_id, _MEAS_REQ_IE_);
+		goto fail;
+	}
+
+	FSM_INFO(prm, "rx %s from " MAC_FMT "\n",
+		rm_type_req_name(prm->q.m_type), MAC_ARG(obj2mac(prm)));
+
+	switch (prm->q.m_type) {
+	case bcn_req:
+		rrm_parse_meas_req(prm, pmeas_body);
+		switch (prm->q.s_mode) {
+		case bcn_req_passive:
+			rm_en_cap_chk_and_set(prm, RM_BCN_PASSIVE_MEAS_CAP_EN);
+			break;
+		case bcn_req_active:
+			rm_en_cap_chk_and_set(prm, RM_BCN_ACTIVE_MEAS_CAP_EN);
+			break;
+		case bcn_req_bcn_table:
+			need_to_scan = 0;
+			rm_en_cap_chk_and_set(prm, RM_BCN_TABLE_MEAS_CAP_EN);
+			break;
+		default:
+			rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+			goto fail;
+		}
+		break;
+	case ch_load_req:
+		rrm_parse_meas_req(prm, pmeas_body);
+		rm_en_cap_chk_and_set(prm, RM_CH_LOAD_CAP_EN);
+		break;
+	case noise_histo_req:
+		rrm_parse_meas_req(prm, pmeas_body);
+		rm_en_cap_chk_and_set(prm, RM_NOISE_HISTO_CAP_EN);
+		break;
+	default:
+		rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+		goto fail;
+	}
+
+	if (need_to_scan) {
+		ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+
+		if ((ssc_chk != SS_ALLOW) && (ssc_chk != SS_DENY_BUSY_TRAFFIC)) {
+			rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+			goto fail;
+		}
+	}
+
+	rtw_fsm_activate_obj(prm);
+
+	return _SUCCESS;
+fail:
+	/* reply incap */
+	rrm_issue_null_reply(prm);
+
+	/* del obj */
+	rtw_fsm_del_obj(prm);
+
+	return _FAIL;
+}
+
+void indicate_beacon_report(struct rrm_obj *prm, u8 *sta_addr,
+	u8 n_measure_rpt, u32 elem_len, u8 *elem)
+{
+	FSM_INFO(prm, "recv bcn reprot from mac="MAC_FMT"\n", MAC_ARG(sta_addr));
+}
+
+/* receive measurement report */
+int rrm_recv_radio_mens_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	int ret = _FALSE;
+	struct rrm_obj *prm;
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	u32 len;
+	u16 cid;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = &pdiag_body[3];
+
+	cid = rrm_cal_cid(pdiag_body[2], RM_MASTER);
+	prm = rtw_fsm_get_obj(prmpriv->fsm, cid);
+	if (prm == NULL) {
+		/* not belong to us, report to upper */
+		rtw_cfg80211_rx_rrm_action(psta->padapter, precv_frame);
+		return _TRUE;
+	}
+
+	prm->p.action_code = pdiag_body[1];
+	prm->p.diag_token = pdiag_body[2];
+
+	/* Figure 8-140 Measuremnt Report format */
+	prm->p.e_id = pmeas_body[0];
+	prm->p.m_token = pmeas_body[2];
+	prm->p.m_mode = pmeas_body[3];
+	prm->p.m_type = pmeas_body[4];
+
+	FSM_INFO(prm, "bssid " MAC_FMT "\n", MAC_ARG(obj2mac(prm)));
+
+	FSM_TRACE(prm, "element_id = %d\n", prm->p.e_id);
+	FSM_TRACE(prm, "length = %d\n", (int)pmeas_body[1]);
+	FSM_TRACE(prm, "meas_token = %d\n", prm->p.m_token);
+	FSM_TRACE(prm, "meas_mode = %d\n", prm->p.m_mode);
+	FSM_TRACE(prm, "meas_type = %d\n", prm->p.m_type);
+
+	if (prm->p.e_id != _MEAS_RSP_IE_) /* 39 */
+		return _FALSE;
+
+	FSM_INFO(prm, "rx %s (" MAC_FMT ")\n",
+		rm_type_rep_name(prm->p.m_type), MAC_ARG(obj2mac(prm)));
+
+	rtw_fsm_gen_msg(prm, NULL, 0, RRM_EV_recv_rep);
+
+	/* report to upper via ioctl */
+	if ((prm->from_ioctl == true) &&
+		prm->q.m_type == bcn_req) {
+		len = pmeas_body[1] + 2; /* 2 : EID(1B)  length(1B) */
+		indicate_beacon_report(prm, obj2mac(prm),
+			1, len, pmeas_body);
+	}
+
+	return ret;
+}
+
+/* receive link measurement request */
+int rrm_recv_link_mens_req(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	struct rrm_obj *prm;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 diag_token = pdiag_body[2];
+	u8 *pmeas_body = &pdiag_body[3];
+	int i;
+
+	prm = rtw_rrm_new_obj(padapter, psta, diag_token);
+
+	if (prm == NULL) {
+		RTW_ERR("rrm: unable to alloc rm obj for requeset\n");
+		return _FALSE;
+	}
+
+	prm->q.action_code = pdiag_body[1];
+	prm->q.diag_token = pdiag_body[2];
+	prm->q.tx_pwr_used = pmeas_body[0];
+	prm->q.tx_pwr_max = pmeas_body[1];
+	prm->q.rx_pwr = precv_frame->u.hdr.attrib.phy_info.rx_power;
+	prm->q.rx_rate = precv_frame->u.hdr.attrib.data_rate;
+	prm->q.rx_bw = precv_frame->u.hdr.attrib.bw;
+	prm->q.rx_rsni = rm_get_frame_rsni(prm, precv_frame);
+
+	FSM_INFO(prm, "rx link_mens_req(" MAC_FMT ") rx_pwr=%ddBm, rate=%s\n",
+		MAC_ARG(obj2mac(prm)), prm->q.rx_pwr,
+		MGN_RATE_STR(prm->q.rx_rate));
+
+	FSM_TRACE(prm, "tx_pwr_used =%d dBm\n", prm->q.tx_pwr_used);
+	FSM_TRACE(prm, "tx_pwr_max  =%d dBm\n", prm->q.tx_pwr_max);
+
+	rtw_fsm_activate_obj(prm);
+
+	return _SUCCESS;
+}
+
+/* receive link measurement report */
+int rrm_recv_link_mens_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	int ret = _FALSE;
+	struct rrm_obj *prm;
+	u16 cid;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = pdiag_body + 3;
+	s8 val;
+
+	cid = rrm_cal_cid(pdiag_body[2], RM_MASTER);
+	prm = rtw_fsm_get_obj(prmpriv->fsm, cid);
+	if (prm == NULL) {
+		/* not belong to us, report to upper */
+		rtw_cfg80211_rx_rrm_action(padapter, precv_frame);
+		return _TRUE;
+	}
+
+	//FSM_INFO(prm, "bssid " MAC_FMT "\n", MAC_ARG(obj2mac(prm)));
+
+	prm->p.action_code = pdiag_body[1];
+	prm->p.diag_token = pdiag_body[2];
+
+	FSM_TRACE(prm, "action_code = %d\n", prm->p.action_code);
+	FSM_TRACE(prm, "diag_token  = %d\n", prm->p.diag_token);
+	FSM_TRACE(prm, "xmit_power  = %d dBm\n", pmeas_body[2]);
+	FSM_TRACE(prm, "link_margin = %d dBm\n", pmeas_body[3]);
+	FSM_TRACE(prm, "xmit_ant    = %d\n", pmeas_body[4]);
+	FSM_TRACE(prm, "recv_ant    = %d\n", pmeas_body[5]);
+	FSM_TRACE(prm, "RCPI        = %d\n", pmeas_body[6]);
+	FSM_TRACE(prm, "RSNI        = %d\n", pmeas_body[7]);
+
+	FSM_INFO(prm, "rx  link_meas_report (" MAC_FMT ")\n", MAC_ARG(obj2mac(prm)));
+
+	rtw_fsm_gen_msg(prm, NULL, 0, RRM_EV_recv_rep);
+
+	return ret;
+}
+
+
+int rm_radio_mens_nb_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = &pdiag_body[3];
+	u32 len = precv_frame->u.hdr.len;
+	u16 cid;
+	struct rrm_obj *prm;
+
+	cid = rrm_cal_cid(pdiag_body[2], RM_MASTER);
+	prm = rtw_fsm_get_obj(prmpriv->fsm, cid);
+	if (prm == NULL) {
+		/* not belong to us, report to upper */
+		rtw_cfg80211_rx_rrm_action(padapter, precv_frame);
+		return _TRUE;
+	}
+
+	prm->p.action_code = pdiag_body[1];
+	prm->p.diag_token = pdiag_body[2];
+	prm->p.e_id = pmeas_body[0];
+
+	FSM_INFO(prm, "rx  neighbor report (" MAC_FMT ")\n", MAC_ARG(obj2mac(prm)));
+
+	FSM_TRACE(prm, "element_id = %d\n", prm->p.e_id);
+	FSM_TRACE(prm, "length = %d\n", (int)pmeas_body[1]);
+
+	rtw_fsm_gen_msg(prm, NULL, 0, RRM_EV_recv_rep);
+
+#ifdef CONFIG_RTW_MBO
+#ifdef CONFIG_LAYER2_ROAMING
+	if (rtw_wnm_btm_candidates_survey(padapter
+			,(pdiag_body + 3)
+			,(len - sizeof(struct rtw_ieee80211_hdr_3addr))
+			,_FALSE) == _FAIL)
+		return _FALSE;
+#endif
+#endif
+	rtw_cfg80211_rx_rrm_action(padapter, precv_frame);
+
+	return _TRUE;
+}
+
+unsigned int rrm_on_action(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct mlme_ext_info *pmlmeinfo = &padapter->mlmeextpriv.mlmext_info;
+	u32 ret = _FAIL;
+	u8 *pframe = NULL;
+	u8 *pframe_body = NULL;
+	u8 action_code = 0;
+	u8 diag_token = 0;
+	struct rtw_ieee80211_hdr_3addr *whdr;
+	struct sta_info *psta;
+
+
+	if (!(pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
+		goto exit;
+
+	pframe = precv_frame->u.hdr.rx_data;
+
+	/* check RA matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(padapter),
+		GetAddr1Ptr(pframe), ETH_ALEN))
+		goto exit;
+
+	whdr = (struct rtw_ieee80211_hdr_3addr *)pframe;
+	RTW_DBG("rrm: %s bssid = " MAC_FMT "\n",
+		__func__, MAC_ARG(whdr->addr2));
+
+	psta = rtw_get_stainfo(&padapter->stapriv, whdr->addr2);
+
+        if (!psta) {
+		RTW_ERR("rrm: psta not found\n");
+                goto exit;
+        }
+
+	pframe_body = (unsigned char *)(pframe +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+
+	/* Figure 8-438 radio measurement request frame Action field format */
+	/* Category = pframe_body[0] = 5 (Radio Measurement) */
+	action_code = pframe_body[1];
+	diag_token = pframe_body[2];
+
+	//RTW_DBG("RRM: %s radio_action=%x, diag_token=%x\n", __func__,
+	//	action_code, diag_token);
+
+	switch (action_code) {
+
+	case RM_ACT_RADIO_MEAS_REQ:
+		//RTW_INFO("RRM: RM_ACT_RADIO_MEAS_REQ\n");
+		ret = rrm_recv_radio_mens_req(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_RADIO_MEAS_REP:
+		//RTW_INFO("RRM: RM_ACT_RADIO_MEAS_REP\n");
+		ret = rrm_recv_radio_mens_rep(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_LINK_MEAS_REQ:
+		//RTW_INFO("RRM: RM_ACT_LINK_MEAS_REQ\n");
+		ret = rrm_recv_link_mens_req(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_LINK_MEAS_REP:
+		//RTW_INFO("RRM: RM_ACT_LINK_MEAS_REP\n");
+		ret = rrm_recv_link_mens_rep(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_NB_REP_REQ:
+		//RTW_INFO("RRM: RM_ACT_NB_REP_REQ\n");
+		break;
+
+	case RM_ACT_NB_REP_RESP:
+		//RTW_INFO("RRM: RM_ACT_NB_REP_RESP\n");
+		ret = rm_radio_mens_nb_rep(padapter, precv_frame, psta);
+		break;
+
+	default:
+		RTW_WARN("rrm: unknown specturm management action %2x\n",
+			action_code);
+		break;
+	}
+exit:
+	return ret;
+}
+
+static u8 *rm_gen_bcn_detail_2_elem(_adapter *padapter, u8 *pframe, u8 frag_id,
+	struct rrm_obj *prm, u8 **pie, u32 *ie_len, unsigned int *fr_len)
+{
+	u32 frag_max_len = 255 - *fr_len - 4; /* 4: frag_id */
+	int len = *ie_len;
+	u8 slen = 0, frag_len = 0, more = 0;
+	u8 *ies = *pie;
+	u8 id, val8;
+
+	if (prm->q.opt.bcn.rep_last_bcn_ind)
+		frag_max_len -= 3;
+
+	/* report_detail = 2 */
+	if (frag_id == 0) {
+		frag_len += _FIXED_IE_LENGTH_;
+		len -= _FIXED_IE_LENGTH_;
+	}
+
+	while (len >= 0) {
+
+		id = ies[frag_len];
+		slen = (unsigned int)ies[frag_len + 1] + 2;
+
+		if ((frag_len + slen) >= frag_max_len ||
+		    frag_len == *ie_len) {
+			/* ID */
+			val8 = 1; /* reported frame body */
+			pframe = rtw_set_fixed_ie(pframe, 1, &val8, fr_len);
+
+			/* LEN */
+			pframe = rtw_set_fixed_ie(pframe, 1, &frag_len, fr_len);
+
+			/* frame body */
+			pframe = rtw_set_fixed_ie(pframe, frag_len, ies, fr_len);
+
+			goto done;
+		}
+
+		len -= slen;
+		frag_len += slen;
+	}
+done:
+	*pie = ies + frag_len;
+
+	if (len > 0)
+		*ie_len = len; /* remain len */
+	else
+		*ie_len = 0;
+
+	return pframe;
+}
+
+static u8 *rm_gen_bcn_detail_1_elem(u8 *pframe, u8 frag_id,
+	struct rrm_obj *prm, u8 **ies, u32 *ie_len, unsigned int *fr_len)
+{
+	unsigned int my_len = 0;
+	_adapter *padapter = obj2adp(prm);
+	int j = 0, k, len;
+	u8 *plen;
+	u8 *ptr;
+	u8 val8, eid;
+
+	/* report_detail = 1 */
+
+	/* ID */
+	val8 = 1; /* 1:reported frame body */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	plen = pframe;
+	val8 = 0;
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* report_detail = 1
+	 * All fixed-length fields and any requested elements
+	 * in the Request element if present
+	 */
+	pframe = rtw_set_fixed_ie(pframe,
+		_FIXED_IE_LENGTH_, *ies, &my_len);
+
+	for (j = 0; j < prm->q.opt.bcn.opt_id_num; j++) {
+		switch (prm->q.opt.bcn.opt_id[j]) {
+		case bcn_req_ssid:
+			/* SSID */
+			FSM_TRACE(prm, "bcn_req_ssid\n");
+			ptr = rtw_get_ie(*ies + _FIXED_IE_LENGTH_, _SSID_IE_,
+				&len, *ie_len - _FIXED_IE_LENGTH_);
+
+			if (ptr)
+				pframe = rtw_set_ie(pframe, _SSID_IE_, len, ptr + 2, &my_len);
+			break;
+		case bcn_req_req:
+			if (prm->q.opt.bcn.req_start == NULL)
+				break;
+
+			FSM_TRACE(prm, "bcn_req_req");
+
+			for (k=0; k<prm->q.opt.bcn.req_len; k++) {
+				eid = prm->q.opt.bcn.req_start[k];
+
+				val8 = *ie_len - _FIXED_IE_LENGTH_;
+				ptr = rtw_get_ie(*ies + _FIXED_IE_LENGTH_,
+					eid, &len, val8);
+
+				if (!ptr)
+					continue;
+
+				switch (eid) {
+				case EID_SsId:
+					FSM_TRACE(prm, "EID_SSID\n");
+					break;
+				case EID_QBSSLoad:
+					FSM_TRACE(prm, "EID_QBSSLoad\n");
+					break;
+				case EID_HTCapability:
+					FSM_TRACE(prm, "EID_HTCapability\n");
+					break;
+				case _MDIE_:
+					FSM_TRACE(prm, "EID_MobilityDomain\n");
+					break;
+				case EID_Vendor:
+					FSM_TRACE(prm, "EID_Vendor\n");
+					break;
+				default:
+					FSM_TRACE(prm, "EID %d todo\n",eid);
+					break;
+				}
+				pframe = rtw_set_ie(pframe, eid,
+					len, ptr + 2, &my_len);
+			} /* for() */
+			break;
+		case bcn_req_rep_detail:
+			FSM_DBG(prm, "bcn_req_rep_detail\n");
+			break;
+		case bcn_req_ap_ch_rep:
+			FSM_DBG(prm, "bcn_req_ap_ch_rep\n");
+			break;
+		default:
+			FSM_INFO(prm, "OPT %d not support\n",prm->q.opt.bcn.opt_id[j]);
+			break;
+		}
+	}
+	/*
+	 * update my length
+	 * content length does NOT include ID and LEN
+	 */
+	val8 = my_len - 2;
+	rtw_set_fixed_ie(plen, 1, &val8, &j);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	return pframe;
+}
+
+u8 rm_bcn_req_cond_mach(struct rrm_obj *prm, struct wlan_network *pnetwork)
+{
+	u8 val8;
+
+
+	switch(prm->q.opt.bcn.rep_cond.cond) {
+	case bcn_rep_cond_immediately:
+		return _SUCCESS;
+	case bcn_req_cond_rcpi_greater:
+		val8 = rm_get_bcn_rcpi(prm, pnetwork);
+		if (val8 > prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	case bcn_req_cond_rcpi_less:
+		val8 = rm_get_bcn_rcpi(prm, pnetwork);
+		if (val8 < prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	case bcn_req_cond_rsni_greater:
+		val8 = rm_get_bcn_rsni(prm, pnetwork);
+		if (val8 != 255 && val8 > prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	case bcn_req_cond_rsni_less:
+		val8 = rm_get_bcn_rsni(prm, pnetwork);
+		if (val8 != 255 && val8 < prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	default:
+		RTW_ERR("rrm: bcn_req cond %u not support\n",
+			prm->q.opt.bcn.rep_cond.cond);
+		break;
+	}
+	return _FALSE;
+}
+
+static u8 *rm_gen_bcn_rep_ie (struct rrm_obj *prm, u8 last,
+	u8 *pframe, struct wlan_network *pnetwork, unsigned int *fr_len)
+{
+	int snr, i = 0;
+	u8 val8, *plen, *ies, frag_id = 0, more;
+	u16 val16;
+	u32 val32, ie_len;
+	u64 val64;
+	unsigned int my_len;
+	_adapter *padapter = obj2adp(prm);
+	u8 frag_bcn_idx[4] = {0x02, 0x2, 0x1, 0x0};
+	u8 last_bcn_ind[3] = {0xa4, 0x1, 0x0};
+
+	ies = pnetwork->network.IEs;
+
+	ie_len = pnetwork->network.IELength;
+more:
+	my_len = 0;
+	plen = pframe + 1;
+	pframe = rtw_set_fixed_ie(pframe, 7, &prm->p.e_id, &my_len);
+
+	/* Actual Measurement StartTime */
+	val64 = cpu_to_le64(prm->meas_start_time);
+	pframe = rtw_set_fixed_ie(pframe, 8, (u8 *)&val64, &my_len);
+
+	/* Measurement Duration */
+	val16 = prm->meas_end_time - prm->meas_start_time;
+	val16 = cpu_to_le16(val16);
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8*)&val16, &my_len);
+
+	/* TODO
+	* ReportedFrameInformation:
+	* 0 :beacon or probe rsp
+	* 1 :pilot frame
+	*/
+	val8 = 0; /* report frame info */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RCPI */
+	val8 = rm_get_bcn_rcpi(prm, pnetwork);
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RSNI */
+	val8 = rm_get_bcn_rsni(prm, pnetwork);
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* BSSID */
+	pframe = rtw_set_fixed_ie(pframe, 6,
+		(u8 *)&pnetwork->network.MacAddress, &my_len);
+
+	/*
+	 * AntennaID
+	 * 0: unknown
+	 * 255: multiple antenna (Diversity)
+	 */
+	val8 = 0;
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* ParentTSF */
+	val32 = prm->meas_start_time + pnetwork->network.PhyInfo.free_cnt;
+	pframe = rtw_set_fixed_ie(pframe, 4, (u8 *)&val32, &my_len);
+
+	/* Generate Beacon detail */
+
+	/* Reporting Detail values
+	 * 0: No fixed length fields or elements
+	 * 1: All fixed length fields and any requested elements
+	 *    in the Request info element if present
+	 * 2: All fixed length fields and elements
+	 * 3-255: Reserved
+	 */
+	if (prm->q.opt.bcn.rep_detail == 1) {
+		pframe = rm_gen_bcn_detail_1_elem(pframe, frag_id,
+			prm, &ies, &ie_len, &my_len);
+
+	} else if (prm->q.opt.bcn.rep_detail == 2) {
+		pframe = rm_gen_bcn_detail_2_elem(padapter, pframe, frag_id,
+			prm, &ies, &ie_len, &my_len);
+
+		/* fragment id num */
+		more = (ie_len > 0);
+		val8 = more << 7 | frag_id;
+		frag_bcn_idx[3] = val8;
+
+		pframe = rtw_set_fixed_ie(pframe, sizeof(frag_bcn_idx), frag_bcn_idx, &my_len);
+
+		if (prm->q.opt.bcn.rep_last_bcn_ind) {
+			if (last && !more)
+				last_bcn_ind[2] = 1;
+			pframe = rtw_set_fixed_ie(pframe,
+				sizeof(last_bcn_ind), last_bcn_ind, &my_len);
+		}
+	}
+
+	/*
+	* update my length
+	* content length does NOT include ID and LEN
+	*/
+	val8 = my_len - 2;
+	rtw_set_fixed_ie(plen, 1, &val8, &i);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	if (ie_len && prm->q.opt.bcn.rep_detail == 2 && frag_id < 3) {
+		frag_id++;
+		goto more;
+	}
+
+	return pframe;
+}
+
+static int rm_match_sub_elem(_adapter *padapter,
+	struct rrm_obj *prm, struct wlan_network *pnetwork)
+{
+	WLAN_BSSID_EX *pbss = &pnetwork->network;
+	unsigned int my_len;
+	int i, j, k, len;
+	u8 *plen;
+	u8 *ptr;
+	u8 val8, eid;
+
+
+	my_len = 0;
+	/* Reporting Detail values
+	 * 0: No fixed length fields or elements
+	 * 1: All fixed length fields and any requested elements
+	 *    in the Request info element if present
+	 * 2: All fixed length fields and elements
+	 * 3-255: Reserved
+	 */
+
+	/* report_detail != 1  */
+	if (prm->q.opt.bcn.rep_detail != 1)
+		return _TRUE;
+
+	/* report_detail = 1 */
+
+	for (j = 0; j < prm->q.opt.bcn.opt_id_num; j++) {
+		switch (prm->q.opt.bcn.opt_id[j]) {
+		case bcn_req_ssid:
+			/* SSID */
+
+			FSM_DBG(prm, "bcn_req_ssid\n");
+
+			if (pbss->Ssid.SsidLength == 0)
+				return _FALSE;
+			break;
+		case bcn_req_req:
+			if (prm->q.opt.bcn.req_start == NULL)
+				break;
+
+			FSM_DBG(prm, "RM: bcn_req_req");
+
+			for (k=0; k<prm->q.opt.bcn.req_len; k++) {
+				eid = prm->q.opt.bcn.req_start[k];
+
+				i = pbss->IELength - _FIXED_IE_LENGTH_;
+				ptr = rtw_get_ie(pbss->IEs + _FIXED_IE_LENGTH_,
+					eid, &len, i);
+
+				switch (eid) {
+				case EID_SsId:
+					FSM_TRACE(prm, "EID_SSID\n");
+					break;
+				case EID_QBSSLoad:
+					FSM_TRACE(prm, "EID_QBSSLoad\n");
+					break;
+				case EID_HTCapability:
+					FSM_TRACE(prm, "EID_HTCapability\n");
+					break;
+				case _MDIE_:
+					FSM_TRACE(prm, "EID_MobilityDomain\n");
+					break;
+				case EID_Vendor:
+					FSM_TRACE(prm, "EID_Vendor\n");
+					break;
+				default:
+					FSM_TRACE(prm, "EID %d todo\n",eid);
+					break;
+				}
+				if (!ptr) {
+					FSM_INFO(prm, "EID %d not found\n",eid);
+					return _FALSE;
+				}
+			} /* for() */
+			break;
+		case bcn_req_rep_detail:
+			FSM_DBG(prm, "bcn_req_rep_detail\n");
+			break;
+		case bcn_req_ap_ch_rep:
+			FSM_DBG(prm, "bcn_req_ap_ch_rep\n");
+			break;
+		default:
+			FSM_INFO(prm, "OPT %d not found\n",prm->q.opt.bcn.opt_id[j]);
+			break;
+		}
+	}
+	return _TRUE;
+}
+
+static int retrieve_scan_result(struct rrm_obj *prm)
+{
+	_list *plist, *phead;
+	_queue *queue;
+	_adapter *padapter = obj2adp(prm);
+	struct rtw_ieee80211_channel *pch_set;
+	struct wlan_network *pnetwork = NULL;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	int i, j;
+	PWLAN_BSSID_EX pbss;
+	unsigned int matched_network;
+	int len, my_len;
+	u8 last = 0;
+	u8 buf_idx, *pbuf = NULL, *tmp_buf = NULL;
+	u16 xframe_ext_sz = SZ_XMITFRAME_EXT;
+
+
+	tmp_buf = rtw_malloc(xframe_ext_sz);
+	if (tmp_buf == NULL)
+		return 0;
+
+	matched_network = 0;
+	queue = &(pmlmepriv->scanned_queue);
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	/* get requested measurement channel set */
+	pch_set = prm->q.ch_set;
+
+	prm->ap_num = 0;
+	/* search scan queue to find requested SSID */
+	while (1) {
+
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+		pbss = &pnetwork->network;
+
+#if 0
+		RTW_INFO("RM: ch %u ssid %s bssid "MAC_FMT"\n",
+			pbss->Configuration.DSConfig, pbss->Ssid.Ssid,
+			MAC_ARG(pbss->MacAddress));
+		/*
+		* report network if requested channel set contains
+		* the channel matchs selected network
+		*/
+		if (rtw_chset_search_ch(adapter_to_chset(padapter),
+			pbss->Configuration.DSConfig) < 0) /* not match */
+			goto next;
+
+		if (rtw_mlme_band_check(padapter, pbss->Configuration.DSConfig)
+			== _FALSE)
+			goto next;
+#endif
+		if (rtw_validate_ssid(&(pbss->Ssid)) == _FALSE)
+			goto next;
+
+		/* match bssid */
+		if (is_wildcard_bssid(prm->q.bssid) == _FALSE)
+			if (_rtw_memcmp(prm->q.bssid,
+				pbss->MacAddress, 6) == _FALSE)
+				goto next;
+		/*
+		 * default wildcard SSID. wildcard SSID:
+		 * A SSID value (null) used to represent all SSIDs
+		 */
+
+		/* match ssid */
+		if ((prm->q.opt.bcn.ssid.SsidLength > 0) &&
+			_rtw_memcmp(prm->q.opt.bcn.ssid.Ssid,
+			pbss->Ssid.Ssid,
+			prm->q.opt.bcn.ssid.SsidLength) == _FALSE)
+			goto next;
+
+		/* go through measurement requested channels */
+		for (i = 0; i < prm->q.ch_set_ch_amount; i++) {
+			if ((pch_set[i].hw_value) ==
+				(pbss->Configuration.DSConfig)) /* match ch */
+				break;
+		}
+		if (i >= prm->q.ch_set_ch_amount) /* channel mismatch */
+			goto next;
+
+		/* match condition */
+		if (rm_bcn_req_cond_mach(prm, pnetwork) == _FALSE) {
+			RTW_INFO("RM: condition mismatch ch %u ssid %s bssid "MAC_FMT"\n",
+				pbss->Configuration.DSConfig, pbss->Ssid.Ssid,
+				MAC_ARG(pbss->MacAddress));
+			RTW_INFO("RM: condition %u:%u\n",
+				prm->q.opt.bcn.rep_cond.cond,
+				prm->q.opt.bcn.rep_cond.threshold);
+			goto next;
+		}
+
+		/* match subelement */
+		if (rm_match_sub_elem(padapter, prm, pnetwork) == _FALSE)
+			goto next;
+
+		/* Found a matched SSID */
+		matched_network++;
+
+		RTW_INFO("RM: ch %u Found %s bssid "MAC_FMT"\n",
+			pbss->Configuration.DSConfig, pbss->Ssid.Ssid,
+			MAC_ARG(pbss->MacAddress));
+
+		prm->ap[prm->ap_num++] = pnetwork;
+		if (prm->ap_num >= MAX_RM_AP_NUM)
+			break;
+next:
+		plist = get_next(plist);
+	} /* while() */
+
+	/* generate packet */
+	my_len = 0;
+	buf_idx = 0;
+	for (i = 0; i < prm->ap_num; i++) {
+		len = 0;
+		last = (i == (prm->ap_num - 1));
+		_rtw_memset(tmp_buf, 0, xframe_ext_sz);
+		pnetwork = prm->ap[i];
+		rm_gen_bcn_rep_ie(prm, last, tmp_buf, pnetwork, &len);
+		prm->ap[i] = NULL;
+new_packet:
+		if (my_len == 0) {
+			pbuf = rtw_malloc(xframe_ext_sz);
+			if (pbuf == NULL)
+				goto fail;
+			prm->buf[buf_idx].pbuf = pbuf;
+		}
+
+		if ((xframe_ext_sz - (my_len+len+24+4)) > 0) {
+			pbuf = rtw_set_fixed_ie(pbuf,
+				len, tmp_buf, &my_len);
+			prm->buf[buf_idx].len = my_len;
+		} else {
+			if (my_len == 0) /* not enough space */
+				goto fail;
+
+			my_len = 0;
+			buf_idx++;
+
+			if (buf_idx >= MAX_RM_PKT_NUM) {
+				for (j = i; j < prm->ap_num; j++)
+					prm->ap[i] = NULL;
+				break;
+			}
+
+			goto new_packet;
+		}
+	}
+
+fail:
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	if (tmp_buf)
+		rtw_mfree(tmp_buf, xframe_ext_sz);
+
+	RTW_INFO("RM: Found %d matched %s (%d)\n", matched_network,
+		prm->q.opt.bcn.ssid.Ssid, buf_idx+1);
+
+	if (prm->buf[buf_idx].pbuf)
+		return buf_idx + 1;
+
+	return 0;
+}
+
+int rrm_issue_beacon_rep(struct rrm_obj *prm)
+{
+	int i, my_len;
+	u8 *pframe;
+	_adapter *padapter = obj2adp(prm);
+	struct pkt_attrib *pattr;
+	struct xmit_frame *pmgntframe;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	int pkt_num;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u16 xframe_ext_sz = SZ_XMITFRAME_EXT;
+
+	pkt_num = retrieve_scan_result(prm);
+
+	if (pkt_num == 0) {
+		rrm_issue_null_reply(prm);
+		return _SUCCESS;
+	}
+
+	FSM_INFO(prm, "tx bcn_req report\n");
+
+	for (i=0;i<pkt_num;i++) {
+
+		pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+		if (pmgntframe == NULL) {
+			FSM_ERR(prm, "%s alloc xmit_frame fail\n",__func__);
+			goto fail;
+		}
+		pattr = &pmgntframe->attrib;
+		pframe = build_wlan_hdr(prm, padapter,
+			pmgntframe, obj2sta(prm), WIFI_ACTION);
+		pframe = rtw_set_fixed_ie(pframe,
+			3, &prm->p.category, &pattr->pktlen);
+
+		my_len = 0;
+		pframe = rtw_set_fixed_ie(pframe,
+			prm->buf[i].len, prm->buf[i].pbuf, &my_len);
+
+		pattr->pktlen += my_len;
+		pattr->last_txcmdsz = pattr->pktlen;
+		dump_mgntframe(padapter, pmgntframe);
+	}
+fail:
+	/*GEORGIA_TODO_FIXIT*/
+	for (i = 0; i < pkt_num; i++) {
+		if (prm->buf[i].pbuf) {
+			rtw_mfree(prm->buf[i].pbuf, xframe_ext_sz);
+			prm->buf[i].pbuf = NULL;
+			prm->buf[i].len = 0;
+		}
+	}
+	return _SUCCESS;
+}
+
+/* neighbor request */
+int rrm_issue_nb_req(struct rrm_obj *prm)
+{
+	_adapter *padapter = obj2adp(prm);
+	struct sta_info *psta = obj2sta(prm);
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pmgntframe = NULL;
+	struct pkt_attrib *pattr = NULL;
+	u8 val8;
+	u8 *pframe = NULL;
+
+
+	FSM_DBG(prm, "%s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		FSM_ERR(prm, "%s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(prm, padapter, pmgntframe, psta, WIFI_ACTION);
+	pframe = rtw_set_fixed_ie(pframe,
+		3, &prm->q.category, &pattr->pktlen);
+
+	if (prm->q.pssid) {
+
+		u8 sub_ie[64] = {0};
+		u8 *pie = &sub_ie[2];
+
+		FSM_INFO(prm, "tx NB Req to search %s\n",
+			pmlmepriv->cur_network.network.Ssid.Ssid);
+
+		val8 = strlen(prm->q.pssid);
+		sub_ie[0] = 0; /*SSID*/
+		sub_ie[1] = val8;
+
+		_rtw_memcpy(pie, prm->q.pssid, val8);
+
+		pframe = rtw_set_fixed_ie(pframe, val8 + 2,
+			sub_ie, &pattr->pktlen);
+	} else {
+
+		if (!pmlmepriv->cur_network.network.Ssid.SsidLength) {
+			FSM_INFO(prm, "tx neighbor request\n");
+		} else {
+			u8 sub_ie[64] = {0};
+			u8 *pie = &sub_ie[2];
+
+			FSM_INFO(prm, "tx neighbor request to search %s\n",
+				pmlmepriv->cur_network.network.Ssid.Ssid);
+
+			sub_ie[0] = 0; /*SSID*/
+			sub_ie[1] = pmlmepriv->cur_network.network.Ssid.SsidLength;
+
+			_rtw_memcpy(pie, pmlmepriv->cur_network.network.Ssid.Ssid,
+				pmlmepriv->cur_network.network.Ssid.SsidLength);
+
+			pframe = rtw_set_fixed_ie(pframe,
+				pmlmepriv->cur_network.network.Ssid.SsidLength + 2,
+				sub_ie, &pattr->pktlen);
+		}
+	}
+
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+/* issue link measurement request */
+int rrm_issue_link_meas_req(struct rrm_obj *prm)
+{
+	_adapter *padapter = obj2adp(prm);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct sta_info *psta = obj2sta(prm);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pmgntframe = NULL;
+	struct pkt_attrib *pattr = NULL;
+	u8 *pframe = NULL;
+	s8 pwr_used = 0, path_a_pwr = 0;
+
+
+	FSM_INFO(prm, "%s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		FSM_ERR(prm, "%s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(prm, padapter, pmgntframe, psta, WIFI_ACTION);
+
+	/* Category, Action code, Dialog token */
+	pframe = rtw_set_fixed_ie(pframe,
+		3, &prm->q.category, &pattr->pktlen);
+
+	/* xmit power used */
+	/* we don't know actual TX power due to RA may change TX rate;
+	 * But if we fix TX rate then we can get specific tx power
+	 */
+	pattr->rate = MGN_6M;
+	rm_get_tx_power(padapter, padapter_link->mlmeextpriv.chandef.band, MGN_6M, &pwr_used);
+	pframe = rtw_set_fixed_ie(pframe,
+		1, &pwr_used, &pattr->pktlen);
+
+	/* Max xmit power */
+	rrm_get_path_a_max_tx_power(padapter, &path_a_pwr);
+	pframe = rtw_set_fixed_ie(pframe,
+		1, &path_a_pwr, &pattr->pktlen);
+
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+/* issue link measurement report */
+int rrm_issue_link_meas_rep(struct rrm_obj *prm)
+{
+	u8 val8;
+	u8 *pframe;
+	unsigned int my_len;
+	_adapter *padapter = obj2adp(prm);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattr;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sta_info *psta = obj2sta(prm);
+	int i;
+	u8 tpc[4];
+	s8 pwr_used;
+
+
+	FSM_INFO(prm, "%s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		FSM_ERR(prm, "ERR %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(prm, padapter, pmgntframe, psta, WIFI_ACTION);
+	/* Category, action code, Dialog token */
+	pframe = rtw_set_fixed_ie(pframe, 3,
+		&prm->p.category, &pattr->pktlen);
+
+	my_len = 0;
+
+	/* TPC report */
+	rm_get_tx_power(padapter, padapter_link->mlmeextpriv.chandef.band, MGN_6M, &pwr_used);
+	tpc[0] = EID_TPC;
+	tpc[1] = 2; /* length */
+
+	/* TX power */
+	tpc[2] = pwr_used;
+
+	/* link margin */
+	rrm_get_rx_sensitivity(padapter, prm->q.rx_bw, prm->q.rx_rate, &pwr_used);
+	tpc[3] = prm->q.rx_pwr - pwr_used; /* RX sensitivity */
+	pattr->rate = MGN_6M; /* use fix rate to get fixed RX sensitivity */
+
+	FSM_TRACE(prm, "rx_pwr=%ddBm - rx_sensitivity=%ddBm = link_margin=%ddB\n",
+		prm->q.rx_pwr, pwr_used, tpc[3]);
+
+	pframe = rtw_set_fixed_ie(pframe, 4, tpc, &my_len);
+
+	/* RECV antenna ID */
+	val8 = 0; /* unknown antenna */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* XMIT antenna ID */
+	/* Fix rate 6M(1T) always use main antenna to TX */
+	val8 = 1; /* main antenna */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RCPI */
+	val8 = translate_dbm_to_rcpi(prm->q.rx_pwr);
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RSNI */
+	val8 = prm->q.rx_rsni;
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* length */
+	//val8 = (u8)my_len-2;
+	//rtw_set_fixed_ie(plen, 1, &val8, &i); /* use variable i to ignore it */
+
+	pattr->pktlen += my_len;
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+static u8 *rm_gen_bcn_req_s_elem(_adapter *padapter,
+	struct rrm_obj *prm, u8 *pframe, unsigned int *fr_len)
+{
+	u8 val8, l;
+	int i;
+	unsigned int my_len = 0;
+	struct _RT_OPERATING_CLASS *op;
+
+
+	val8 = bcn_req_active; /* measurement mode T8-64 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* bssid */
+	pframe = rtw_set_fixed_ie(pframe, 6, prm->q.bssid, &my_len);
+
+	/*
+	 * opt ssid (0)
+	 */
+	l = MIN(32, (int)prm->q.opt.bcn.ssid.SsidLength);
+
+	if (l > 0 && l <= 32) {
+		/* Type */
+		val8 = bcn_req_ssid;
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&val8, &my_len);
+		/* Len */
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&l, &my_len);
+		/* Value */
+		pframe = rtw_set_fixed_ie(pframe, l,
+			prm->q.opt.bcn.ssid.Ssid, &my_len);
+	}
+
+	/*
+	 * opt reporting detail (2)
+	 */
+	/* Type */
+	val8 = bcn_req_rep_detail;
+	pframe = rtw_set_fixed_ie(pframe, 1,
+		&val8, &my_len);
+	/* Len */
+	l = 1;
+	pframe = rtw_set_fixed_ie(pframe, 1,
+		&l, &my_len);
+	/* Value */
+	pframe = rtw_set_fixed_ie(pframe, l,
+		&prm->q.opt.bcn.rep_detail, &my_len);
+
+	/*
+	 * opt request (10)
+	 */
+
+	if (prm->q.opt.bcn.req_id_num > 0) {
+		/* Type */
+		val8 = bcn_req_req;
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&val8, &my_len);
+		/* Len */
+		l = prm->q.opt.bcn.req_id_num;
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&l, &my_len);
+		/* Value */
+		pframe = rtw_set_fixed_ie(pframe, l,
+			prm->q.opt.bcn.req_id, &my_len);
+	}
+
+	/*
+	 * opt ap channel report (51)
+	 */
+	for (i = 0; i < prm->q.opt.bcn.ap_ch_rpt_num; i++) {
+		op = prm->q.opt.bcn.ap_ch_rpt[i];
+		if (op == NULL)
+			break;
+		/* Type */
+		val8 = bcn_req_ap_ch_rep;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		l = (u8)op->Len + 1;
+		/* length */
+		pframe = rtw_set_fixed_ie(pframe, 1, &l, &my_len);
+
+		/* op class */
+		val8 = op->global_op_class;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		/* channel */
+		pframe = rtw_set_fixed_ie(pframe, op->Len, op->Channel, &my_len);
+	}
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	/* optional subelements */
+	return pframe;
+}
+
+static u8 *rm_gen_ch_load_req_s_elem(_adapter *padapter,
+	u8 *pframe, unsigned int *fr_len)
+{
+	u8 val8;
+	unsigned int my_len = 0;
+
+
+	val8 = 1; /* 1: channel load T8-60 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 2; /* channel load length = 2 (extensible)  */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* channel load condition : 0 (issue when meas done) T8-61 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* channel load reference value : 0 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	return pframe;
+}
+
+static u8 *rm_gen_noise_histo_req_s_elem(_adapter *padapter,
+	u8 *pframe, unsigned int *fr_len)
+{
+	u8 val8;
+	unsigned int my_len = 0;
+
+
+	val8 = 1; /* 1: noise histogram T8-62 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 2; /* noise histogram length = 2 (extensible)  */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* noise histogram condition : 0 (issue when meas done) T8-63 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* noise histogram reference value : 0 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	return pframe;
+}
+
+int rrm_issue_radio_meas_req(struct rrm_obj *prm)
+{
+	u8 val8;
+	u8 *pframe;
+	u8 *plen;
+	u16 val16;
+	int my_len, i = 0;
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattr;
+	_adapter *padapter = obj2adp(prm);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+
+	FSM_INFO(prm, "%s - %s\n", __func__, rm_type_req_name(prm->q.m_type));
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		FSM_ERR(prm, "%s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(prm, padapter, pmgntframe, obj2sta(prm), WIFI_ACTION);
+
+	/* Category, Action code, Dialog token */
+	pframe = rtw_set_fixed_ie(pframe, 3, &prm->q.category, &pattr->pktlen);
+
+	/* repeat */
+	val16 = cpu_to_le16(prm->q.rpt);
+	pframe = rtw_set_fixed_ie(pframe, 2,
+		(unsigned char *)&(val16), &pattr->pktlen);
+
+	my_len = 0;
+	plen = pframe + 1;
+
+	/* Element ID, Length, Meas token, Meas Mode, Meas type, op class, ch */
+	pframe = rtw_set_fixed_ie(pframe, 7, &prm->q.e_id, &my_len);
+
+	/* random interval */
+	val16 = cpu_to_le16(prm->q.rand_intvl); /* TU */
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8 *)&val16, &my_len);
+
+	/* measurement duration */
+	val16 = cpu_to_le16(prm->q.meas_dur);
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8 *)&val16, &my_len);
+
+	/* optional subelement */
+	switch (prm->q.m_type) {
+	case bcn_req:
+		pframe = rm_gen_bcn_req_s_elem(padapter, prm,  pframe, &my_len);
+		break;
+	case ch_load_req:
+		pframe = rm_gen_ch_load_req_s_elem(padapter, pframe, &my_len);
+		break;
+	case noise_histo_req:
+		pframe = rm_gen_noise_histo_req_s_elem(padapter,
+			pframe, &my_len);
+		break;
+	case basic_req:
+	default:
+		break;
+	}
+
+	/* length */
+	val8 = (u8)my_len - 2;
+	rtw_set_fixed_ie(plen, 1, &val8, &i);
+
+	pattr->pktlen += my_len;
+
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+static int rrm_issue_meas_req(struct rrm_obj *prm)
+{
+	switch (prm->q.action_code) {
+	case RM_ACT_RADIO_MEAS_REQ:
+		switch (prm->q.m_type) {
+		case bcn_req:
+		case ch_load_req:
+		case noise_histo_req:
+			rrm_issue_radio_meas_req(prm);
+			break;
+		default:
+			break;
+		} /* meas_type */
+		break;
+	case RM_ACT_NB_REP_REQ:
+		/* issue neighbor request */
+		rrm_issue_nb_req(prm);
+		break;
+	case RM_ACT_LINK_MEAS_REQ:
+		rrm_issue_link_meas_req(prm);
+		break;
+	default:
+		return _FALSE;
+	} /* action_code */
+
+	return _SUCCESS;
+}
+
+int rm_radio_meas_report_cond(struct rrm_obj *prm)
+{
+	u8 val8;
+	int i = 0, ret = _FAIL;
+
+
+	switch (prm->q.m_type) {
+	case ch_load_req:
+		val8 = prm->p.ch_load;
+		switch (prm->q.opt.clm.rep_cond.cond) {
+		case ch_load_cond_immediately:
+			ret = _SUCCESS;
+			break;
+		case ch_load_cond_anpi_equal_greater:
+			if (val8 >= prm->q.opt.clm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		case ch_load_cond_anpi_equal_less:
+			if (val8 <= prm->q.opt.clm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		default:
+			break;
+		}
+		break;
+	case noise_histo_req:
+		val8 = prm->p.anpi;
+		switch (prm->q.opt.nhm.rep_cond.cond) {
+		case noise_histo_cond_immediately:
+			ret = _SUCCESS;
+			break;
+		case noise_histo_cond_anpi_equal_greater:
+			if (val8 >= prm->q.opt.nhm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		case noise_histo_cond_anpi_equal_less:
+			if (val8 <= prm->q.opt.nhm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+	return ret;
+}
+
+int retrieve_radio_meas_result(struct rrm_obj *prm)
+{
+#ifdef CONFIG_RTW_ACS
+	struct dvobj_priv *dvobj = adapter_to_dvobj(obj2adp(prm));
+#endif
+	int i, ch = -1;
+	u8 val8;
+
+
+	ch = rtw_chset_search_bch(adapter_to_chset(obj2adp(prm)),
+		prm->q.band, prm->q.ch_num);
+
+	if ((ch == -1) || (ch >= MAX_CHANNEL_NUM)) {
+		FSM_ERR(prm, "get ch(CH:%d) fail\n", prm->q.ch_num);
+		ch = 0;
+	}
+
+	switch (prm->q.m_type) {
+	case ch_load_req:
+#if 0 /* def CONFIG_RTW_ACS */
+		val8 = hal_data->acs.clm_ratio[ch];
+#else
+		val8 = 0;
+#endif
+		prm->p.ch_load = val8;
+		break;
+	case noise_histo_req:
+#if 0 /* def CONFIG_RTW_ACS */
+		/* ANPI */
+		prm->p.anpi = hal_data->acs.nhm_ratio[ch];
+
+		/* IPI 0~10 */
+		for (i=0;i<11;i++)
+			prm->p.ipi[i] = hal_data->acs.nhm[ch][i];
+#else
+		val8 = 0;
+		prm->p.anpi = val8;
+		for (i=0;i<11;i++)
+			prm->p.ipi[i] = val8;
+#endif
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+int rrm_issue_radio_meas_rep(struct rrm_obj *prm)
+{
+	u8 val8;
+	u8 *pframe;
+	u8 *plen;
+	u16 val16;
+	u64 val64;
+	unsigned int my_len;
+	_adapter *padapter = obj2adp(prm);
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattr;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct sta_info *psta = obj2sta(prm);
+	int i = 0;
+
+
+	FSM_INFO(prm, "%s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		FSM_ERR(prm, "ERR %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(prm, padapter, pmgntframe, psta, WIFI_ACTION);
+	pframe = rtw_set_fixed_ie(pframe, 3,
+		&prm->p.category, &pattr->pktlen);
+
+	my_len = 0;
+	plen = pframe + 1;
+	pframe = rtw_set_fixed_ie(pframe, 7, &prm->p.e_id, &my_len);
+
+	/* Actual Meas start time - 8 bytes */
+	val64 = cpu_to_le64(prm->meas_start_time);
+	pframe = rtw_set_fixed_ie(pframe, 8, (u8 *)&val64, &my_len);
+
+	/* measurement duration */
+	val16 = prm->meas_end_time - prm->meas_start_time;
+	val16 = cpu_to_le16(val16);
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8 *)&val16, &my_len);
+
+	/* optional subelement */
+	switch (prm->q.m_type) {
+	case ch_load_req:
+		val8 = prm->p.ch_load;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		break;
+	case noise_histo_req:
+		/*
+		 * AntennaID
+		 * 0: unknown
+		 * 255: multiple antenna (Diversity)
+		 */
+		val8 = 0;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		/* ANPI */
+		val8 = prm->p.anpi;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		/* IPI 0~10 */
+		for (i=0;i<11;i++) {
+			val8 = prm->p.ipi[i];
+			pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		}
+		break;
+	default:
+		break;
+	}
+	/* length */
+	val8 = (u8)my_len-2;
+	rtw_set_fixed_ie(plen, 1, &val8, &i); /* use variable i to ignore it */
+
+	pattr->pktlen += my_len;
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+void rtw_ap_parse_sta_rm_en_cap(_adapter *padapter,
+	struct sta_info *psta, struct rtw_ieee802_11_elems *elem)
+{
+	if (elem->rm_en_cap) {
+		RTW_INFO("rrm: assoc.rm_en_cap="RM_CAP_FMT"\n",
+			RM_CAP_ARG(elem->rm_en_cap));
+
+		_rtw_memcpy(psta->rm_en_cap, (elem->rm_en_cap),
+			MIN(elem->rm_en_cap_len, sizeof(psta->rm_en_cap)));
+	}
+}
+
+void RM_IE_handler(_adapter *padapter, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	int i;
+
+	_rtw_memcpy(&padapter->fsmpriv.rmpriv.rm_en_cap_assoc, pIE->data,
+		    MIN(pIE->Length, sizeof(padapter->fsmpriv.rmpriv.rm_en_cap_assoc)));
+	RTW_INFO("rrm: assoc.rm_en_cap="RM_CAP_FMT"\n", RM_CAP_ARG(pIE->data));
+}
+
+/* Debug command */
+
+#if (RM_SUPPORT_IWPRIV_DBG)
+static int hex2num(char c)
+{
+	if (c >= '0' && c <= '9')
+		return c - '0';
+	if (c >= 'a' && c <= 'f')
+		return c - 'a' + 10;
+	if (c >= 'A' && c <= 'F')
+		return c - 'A' + 10;
+	return -1;
+}
+
+int hex2byte(const char *hex)
+{
+	int a, b;
+	a = hex2num(*hex++);
+	if (a < 0)
+		return -1;
+	b = hex2num(*hex++);
+	if (b < 0)
+		return -1;
+	return (a << 4) | b;
+}
+
+static char * hwaddr_parse(char *txt, u8 *addr)
+{
+	size_t i;
+
+	for (i = 0; i < ETH_ALEN; i++) {
+		int a;
+
+		a = hex2byte(txt);
+		if (a < 0)
+			return NULL;
+		txt += 2;
+		addr[i] = a;
+		if (i < ETH_ALEN - 1 && *txt++ != ':')
+			return NULL;
+	}
+	return txt;
+}
+
+void rm_dbg_list_sta(_adapter *padapter, char *s)
+{
+	int i;
+	struct sta_info *psta;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	_list *plist, *phead;
+
+
+	sprintf(pstr(s), "\n");
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist,
+				struct sta_info, hash_list);
+
+			plist = get_next(plist);
+
+			sprintf(pstr(s), "=========================================\n");
+			sprintf(pstr(s), "mac=" MAC_FMT "\n",
+				MAC_ARG(psta->phl_sta->mac_addr));
+			sprintf(pstr(s), "state=0x%x, aid=%d, macid=%d\n",
+				psta->state, psta->phl_sta->aid, psta->phl_sta->macid);
+			sprintf(pstr(s), "rm_cap="RM_CAP_FMT"\n",
+				RM_CAP_ARG(psta->rm_en_cap));
+		}
+
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	sprintf(pstr(s), "=========================================\n");
+}
+
+#if 0
+void rm_dbg_help(_adapter *padapter, char *s)
+{
+	int i;
+
+
+	sprintf(pstr(s), "\n");
+	sprintf(pstr(s), "rrm list_sta\n");
+	sprintf(pstr(s), "rrm list_meas\n");
+
+	sprintf(pstr(s), "rrm add_meas <aid=1|mac=>,m=<bcn|clm|nhm|nb|link>,rpt=\n");
+	sprintf(pstr(s), "rrm run_meas <aid=1|evid=>\n");
+	sprintf(pstr(s), "rrm del_meas\n");
+
+	sprintf(pstr(s), "rrm run_meas cid=xxxx,ev=xx\n");
+	sprintf(pstr(s), "rrm activate\n");
+
+	for (i=0;i<RRM_EV_max;i++)
+		sprintf(pstr(s), "\t%2d %s\n",i, rtw_fsm_event_name(i) );
+	sprintf(pstr(s), "\n");
+}
+#endif
+
+struct sta_info *rm_get_sta(_adapter *padapter, u16 aid, u8* pbssid)
+{
+	int i;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	_list *plist, *phead;
+
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist,
+				struct sta_info, hash_list);
+
+			plist = get_next(plist);
+
+			if (psta->phl_sta->aid == aid)
+				goto done;
+
+			if (pbssid && _rtw_memcmp(psta->phl_sta->mac_addr,
+				pbssid, 6))
+				goto done;
+		}
+
+	}
+	psta = NULL;
+done:
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	return psta;
+}
+
+static int rm_dbg_modify_meas(_adapter *padapter, char *s)
+{
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	struct mlme_ext_info *pmlmeinfo = &padapter->mlmeextpriv.mlmext_info;
+	struct rrm_obj *prm;
+	struct sta_info *psta;
+	char *pmac, *ptr, *paid, *prpt, *pnbp, *pclm, *pnhm, *pbcn, *plnk;
+	unsigned val;
+	u8 bssid[ETH_ALEN];
+	int i;
+
+	/* example :
+	* rrm add_meas <aid=1|mac=>,m=<nb|clm|nhm|bcn|link>,<rept=>
+	* rrm run_meas <aid=1|evid=>
+	*/
+	paid = strstr(s, "aid=");
+	pmac = strstr(s, "mac=");
+	pbcn = strstr(s, "m=bcn");
+	pclm = strstr(s, "m=clm");
+	pnhm = strstr(s, "m=nhm");
+	pnbp = strstr(s, "m=nb");
+	plnk = strstr(s, "m=link");
+	prpt = strstr(s, "rpt=");
+
+	/* set all ',' to NULL (end of line) */
+	ptr = s;
+	while (ptr) {
+		ptr = strchr(ptr, ',');
+		if (ptr) {
+			*(ptr) = 0x0;
+			ptr++;
+		}
+	}
+	prm = (struct rrm_obj *)prmpriv->prm_sel;
+	prm->q.m_token = rrm_gen_meas_token(padapter);
+	psta = obj2sta(prm);
+
+	for (i=0;i<6;i++)
+		prm->q.bssid[i] = 0xff; /* wildcard bssid */
+	if (paid) { /* find sta_info according to aid */
+		paid += 4; /* skip aid= */
+		sscanf(paid, "%u", &val); /* aid=x */
+		psta = rm_get_sta(padapter, val, NULL);
+
+	} else if (pmac) { /* find sta_info according to bssid */
+		pmac += 4; /* skip mac= */
+		if (hwaddr_parse(pmac, bssid) == NULL) {
+			sprintf(pstr(s), "Err: \nincorrect mac format\n");
+			return _FAIL;
+		}
+		psta = rm_get_sta(padapter, 0xff, bssid);
+	}
+
+	if (psta) {
+		prm->q.diag_token = rrm_gen_dialog_token(padapter);
+		prm->cid = rrm_cal_cid(prm->q.diag_token, RM_MASTER);
+	} else
+		return _FAIL;
+
+	prm->q.action_code = RM_ACT_RADIO_MEAS_REQ;
+	if (pbcn) {
+		prm->q.m_type = bcn_req;
+		prm->q.rand_intvl = le16_to_cpu(100);
+		prm->q.meas_dur = le16_to_cpu(100);
+	} else if (pnhm) {
+		prm->q.m_type = noise_histo_req;
+	} else if (pclm) {
+		prm->q.m_type = ch_load_req;
+	} else if (pnbp) {
+		prm->q.action_code = RM_ACT_NB_REP_REQ;
+	} else if (plnk) {
+		prm->q.action_code = RM_ACT_LINK_MEAS_REQ;
+	} else
+		return _FAIL;
+
+	if (prpt) {
+		prpt += 4; /* skip rpt= */
+		sscanf(prpt, "%u", &val);
+		prm->q.rpt = (u8)val;
+	}
+
+	return _SUCCESS;
+}
+
+static void rm_dbg_activate_meas(_adapter *padapter, char *s)
+{
+	struct rrm_priv *prmpriv = &(padapter->fsmpriv.rmpriv);
+	struct rrm_obj *prm;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+
+	if (prmpriv->prm_sel == NULL) {
+		sprintf(pstr(s), "\nErr: No inActivate measurement\n");
+		return;
+	}
+	prm = (struct rrm_obj *)prmpriv->prm_sel;
+
+	/* measure current channel */
+	prm->q.ch_num = padapter_link->mlmeextpriv.chandef.chan;
+	prm->q.band = padapter_link->mlmeextpriv.chandef.band;
+	prm->q.op_class = rtw_get_op_class_by_bchbw(prm->q.band,
+		prm->q.ch_num, CHANNEL_WIDTH_20, CHAN_OFFSET_NO_EXT);
+
+	rtw_fsm_activate_obj(prm);
+
+#if 0
+	sprintf(pstr(s), "\nActivate cid=%x, state=%s, meas_type=%s\n",
+		prm->cid, rm_state_name(prm->pstate),
+		rm_type_req_name(prm->q.m_type));
+#endif
+
+	sprintf(pstr(s), "aid=%d, mac=" MAC_FMT "\n",
+		obj2sta(prm)->phl_sta->aid, MAC_ARG(obj2mac(prm)));
+
+	/* clearn inActivate prm info */
+	prmpriv->prm_sel = NULL;
+}
+
+/* for ioctl */
+int rm_send_bcn_reqs(_adapter *padapter, u8 *sta_addr, u8 op_class, u8 ch,
+	u16 measure_duration, u8 measure_mode, u8 *bssid, u8 *ssid,
+	u8 reporting_detail,
+	u8 n_ap_ch_rpt, struct _RT_OPERATING_CLASS *rpt,
+	u8 n_elem_id, u8 *elem_id_list)
+{
+	struct rrm_obj *prm;
+	char *pact;
+	struct sta_info *psta;
+	struct _RT_OPERATING_CLASS *prpt;
+	void *ptr;
+	int i,j,sz;
+	u8 bcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+	if (n_ap_ch_rpt > BCN_REQ_OPT_AP_CH_RPT_MAX_NUM) {
+		RTW_ERR("rrm: chset num %d > %d\n",
+			n_ap_ch_rpt, BCN_REQ_OPT_AP_CH_RPT_MAX_NUM);
+		return -1;
+	}
+	/* dest sta */
+	psta = rtw_get_stainfo(&padapter->stapriv, sta_addr);
+        if (!psta) {
+		RTW_ERR("rrm: psta not found\n");
+		return -2;
+        }
+	prm = rtw_rrm_new_obj(padapter, psta, 0);
+	if (prm == NULL) {
+		RTW_ERR("rrm: unable to alloc rm obj for requeset\n");
+		return -3;
+	}
+
+	prm->q.meas_dur = measure_duration;
+
+	/* Figure 8-104 Measurement Requested format */
+	prm->q.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+	prm->q.action_code = RM_ACT_RADIO_MEAS_REQ;
+	prm->q.m_mode = measure_mode; /* TODO */
+	prm->q.m_type = bcn_req;
+	prm->q.m_token = rrm_gen_meas_token(padapter);
+
+	prm->q.e_id = _MEAS_REQ_IE_; /* 38 */
+	prm->q.ch_num = ch;
+	prm->q.op_class = op_class;
+	prm->q.band = rtw_get_band_by_op_class(op_class);
+	prm->from_ioctl = true;
+
+	if (bssid != NULL)
+		memcpy(prm->q.bssid, bssid, ETH_ALEN);
+	else
+		memcpy(prm->q.bssid, bcast, ETH_ALEN);
+
+	if (ssid != NULL) {
+		i = MIN(32, strlen(ssid));
+		prm->q.opt.bcn.ssid.SsidLength = i;
+		memcpy(prm->q.opt.bcn.ssid.Ssid, ssid, i);
+	}
+
+	if (n_ap_ch_rpt > 0) {
+		prm->q.opt.bcn.ap_ch_rpt_num = n_ap_ch_rpt;
+		j = 0;
+		for (i = 0; i < n_ap_ch_rpt; i++) {
+			prpt = rpt++;
+			if (prpt == NULL)
+				break;
+
+			sz = sizeof(struct _RT_OPERATING_CLASS) * prpt->Len;
+			ptr = rtw_malloc(sz);
+			_rtw_memcpy(ptr, prpt, sz);
+			prm->q.opt.bcn.ap_ch_rpt[i] = (struct _RT_OPERATING_CLASS *)ptr;
+		}
+	}
+	prm->q.opt.bcn.rep_detail = reporting_detail;
+
+	if ((n_elem_id > 0) && (n_elem_id < BCN_REQ_REQ_OPT_MAX_NUM)) {
+		prm->q.opt.bcn.req_id_num = n_elem_id;
+		_rtw_memcpy(prm->q.opt.bcn.req_id, elem_id_list, n_elem_id);
+	}
+
+	/* enquee rmobj */
+	rtw_fsm_activate_obj(prm);
+
+	FSM_INFO(prm, "\nAdd meas_type=%s ok\n",
+		rm_type_req_name(prm->q.m_type));
+
+	FSM_INFO(prm, "mac="MAC_FMT"\n", MAC_ARG(obj2mac(prm)));
+	return 0;
+}
+
+static void rm_dbg_add_meas(_adapter *padapter, char *s)
+{
+#if 0
+	struct rrm_priv *prmpriv = &(padapter->fsmpriv.rmpriv);
+	struct rrm_obj *prm;
+	char *pact;
+
+
+	/* example :
+	* rrm add_meas <aid=1|mac=>,m=<nb|clm|nhm|link>
+	* rrm run_meas <aid=1|evid=>
+	*/
+	prm = (struct rrm_obj *)prmpriv->prm_sel;
+	if (prm == NULL) {
+		prm = rtw_rrm_new_obj(padapter, psta);
+	}
+
+	if (prm == NULL) {
+		sprintf(pstr(s), "\nErr: alloc meas fail\n");
+		return;
+	}
+
+	prm->obj = obj;
+        prmpriv->prm_sel = prm;
+
+	pact = strstr(s, "act");
+	if (rm_dbg_modify_meas(padapter, s) == _FAIL) {
+
+		sprintf(pstr(s), "\nErr: add meas fail\n");
+		rtw_fsm_destory_obj(prm->obj);
+		prmpriv->prm_sel = NULL;
+		return;
+	}
+	prm->q.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+	prm->q.e_id = _MEAS_REQ_IE_; /* 38 */
+
+	sprintf(pstr(s), "\nAdd cid=%x, meas_type=%s ok\n",
+		prm->cid, rm_type_req_name(prm->q.m_type));
+
+	//if (prm->psta)
+	sprintf(pstr(s), "mac="MAC_FMT"\n",
+		MAC_ARG(obj2sta(prm)->phl_sta->mac_addr));
+
+	if (pact)
+#endif
+		rm_dbg_activate_meas(padapter, pstr(s));
+}
+
+static void rm_dbg_del_meas(_adapter *padapter, char *s)
+{
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	struct rrm_obj *prm = (struct rrm_obj *)prmpriv->prm_sel;
+
+
+	if (prm) {
+		sprintf(pstr(s), "\ndelete cid=%x\n",prm->cid);
+
+		/* free inActivate meas - enqueue yet  */
+		prmpriv->prm_sel = NULL;
+		rtw_mfree(prmpriv->prm_sel, sizeof(struct rrm_obj));
+	} else
+		sprintf(pstr(s), "Err: no inActivate measurement\n");
+}
+
+static void rm_dbg_run_meas(_adapter *padapter, char *s)
+{
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	struct rrm_obj *prm;
+	char *pevid, *pcid;
+	u32 evid;
+	u16 cid;
+
+	pcid = strstr(s, "cid="); /* hex */
+	pevid = strstr(s, "evid="); /* dec */
+
+	if (pcid && pevid) {
+		pcid += 4; /* cid= */
+		sscanf(pcid, "%hx", &cid);
+
+		pevid += 5; /* evid= */
+		sscanf(pevid, "%u", &evid);
+	} else {
+		sprintf(pstr(s), "\nErr: incorrect attribute\n");
+		return;
+	}
+
+	prm = rtw_fsm_get_obj(prmpriv->fsm, cid);
+	if (!prm) {
+		sprintf(pstr(s), "\nErr: measurement not found\n");
+		return;
+	}
+
+	if (evid >= RRM_EV_max) {
+		sprintf(pstr(s), "\nErr: wrong event id\n");
+		return;
+	}
+
+	rtw_fsm_gen_msg(prm, NULL, 0, evid);
+	sprintf(pstr(s), "\npost %s to cid=%x\n", rtw_fsm_evt_name(prm, evid), cid);
+}
+
+static void rm_dbg_show_meas(struct rrm_obj *prm, char *s)
+{
+	struct sta_info *psta = obj2sta(prm);
+
+	if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+
+		sprintf(pstr(s), "\ncid=%x, meas_type=%s\n",
+			prm->cid, rm_type_req_name(prm->q.m_type));
+
+	} else  if (prm->q.action_code == RM_ACT_NB_REP_REQ) {
+
+		sprintf(pstr(s), "\ncid=%x, action=neighbor_req\n",
+			prm->cid);
+	} else
+		sprintf(pstr(s), "\ncid=%x, action=unknown\n",
+			prm->cid);
+
+	if (psta)
+		sprintf(pstr(s), "aid=%d, mac="MAC_FMT"\n",
+			psta->phl_sta->aid, MAC_ARG(obj2mac(prm)));
+
+#if 0
+	sprintf(pstr(s), "clock=%d, state=%s, rpt=%u/%u\n",
+		(int)ATOMIC_READ(&prm->pclock->counter),
+		rm_state_name(prm->pstate), prm->p.rpt, prm->q.rpt);
+#endif
+}
+
+static void rm_dbg_list_meas(_adapter *padapter, char *s)
+{
+	int meas_amount;
+	struct rrm_obj *prm;
+	struct sta_info *psta;
+	struct rrm_priv *prmpriv = &padapter->fsmpriv.rmpriv;
+	_queue *queue = &prmpriv->rm_queue;
+	_list *plist, *phead;
+	unsigned long sp_flags;
+
+	sprintf(pstr(s), "\n");
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+	meas_amount = 0;
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		prm = LIST_CONTAINOR(plist, struct rrm_obj, list);
+		meas_amount++;
+		plist = get_next(plist);
+		psta = obj2sta(prm);
+		sprintf(pstr(s), "=========================================\n");
+
+		rm_dbg_show_meas(prm, s);
+	}
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+	sprintf(pstr(s), "=========================================\n");
+
+	if (meas_amount==0) {
+		sprintf(pstr(s), "No Activate measurement\n");
+		sprintf(pstr(s), "=========================================\n");
+	}
+
+	if (prmpriv->prm_sel == NULL)
+		sprintf(pstr(s), "\nNo inActivate measurement\n");
+	else {
+		sprintf(pstr(s), "\ninActivate measurement\n");
+		rm_dbg_show_meas((struct rrm_obj *)prmpriv->prm_sel, s);
+	}
+}
+#endif /* RM_SUPPORT_IWPRIV_DBG */
+
+int verify_bcn_req(_adapter *padapter, struct sta_info *psta)
+{
+	char *bssid =  NULL;
+	char ssid[] = "RealKungFu";
+	u8 op_class = 0;
+	u8 ch = 255;
+	u16 measure_duration = 100;
+	u8 reporting_detaial = 0;
+	u8 n_ap_ch_rpt = 6;
+	u8 measure_mode = bcn_req_active;
+	u8 req[] = {1,2,3};
+	u8 req_len = sizeof(req);
+
+
+	static RT_OPERATING_CLASS US[] = {
+	/* 0, OP_CLASS_NULL */	//{  0,  0, {}},
+	/* 1, OP_CLASS_1 */	{115,  4, {36, 40, 44, 48}},
+	/* 2, OP_CLASS_2 */	{118,  4, {52, 56, 60, 64}},
+	/* 3, OP_CLASS_3 */	{124,  4, {149, 153, 157, 161}},
+	/* 4, OP_CLASS_4 */	{121, 11, {100, 104, 108, 112, 116, 120, 124,
+						128, 132, 136, 140}},
+	/* 5, OP_CLASS_5 */	{125,  5, {149, 153, 157, 161, 165}},
+	/* 6, OP_CLASS_12 */	{ 81, 11, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}}
+	};
+
+	rm_send_bcn_reqs(padapter, psta->phl_sta->mac_addr, op_class, ch,
+		measure_duration, measure_mode, bssid, ssid,
+		reporting_detaial, n_ap_ch_rpt, US, req_len, req);
+	return 0;
+}
+
+/*
+ * RM state machine
+ */
+static int rrm_idle_st_hdl(void *obj, u16 event, void *param)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+	_adapter *padapter = obj2adp(prm);
+	struct _ADAPTER_LINK *padapter_link = obj2sta(prm)->padapter_link;
+	u8 val8;
+	u32 val32;
+
+	prm->p.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+
+		rrm_enqueue_rm(obj2priv(prm) ,prm, 0);
+
+		switch (prm->q.action_code) {
+		case RM_ACT_RADIO_MEAS_REQ:
+			/* copy attrib from meas_req to meas_rep */
+			prm->p.action_code = RM_ACT_RADIO_MEAS_REP;
+			prm->p.diag_token = prm->q.diag_token;
+			prm->p.e_id = _MEAS_RSP_IE_;
+			prm->p.m_token = prm->q.m_token;
+			prm->p.m_type = prm->q.m_type;
+			prm->p.rpt = prm->q.rpt;
+			prm->p.ch_num = prm->q.ch_num;
+			prm->p.op_class = prm->q.op_class;
+			prm->p.band = prm->q.band;
+
+			if (prm->q.m_type == ch_load_req
+				|| prm->q.m_type == noise_histo_req) {
+				/*
+				 * phydm measure current ch periodically
+				 * scan current ch is not necessary
+				 */
+				val8 = padapter_link->mlmeextpriv.chandef.chan;
+				if (prm->q.ch_num == val8)
+					prm->poll_mode = 1;
+			}
+			FSM_DBG(prm, "%s repeat=%u\n",
+				rm_type_req_name(prm->q.m_type),
+				prm->q.rpt);
+			break;
+		case RM_ACT_NB_REP_REQ:
+			prm->p.action_code = RM_ACT_NB_REP_RESP;
+			FSM_DBG(prm, "Neighbor request\n");
+			break;
+		case RM_ACT_LINK_MEAS_REQ:
+			prm->p.diag_token = prm->q.diag_token;
+			prm->p.action_code = RM_ACT_LINK_MEAS_REP;
+			FSM_DBG(prm, "Link meas\n");
+			break;
+		default:
+			prm->p.action_code = prm->q.action_code;
+			rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+			FSM_WARN(prm, "recv unknown action%d\n", prm->p.action_code);
+			break;
+		} /* switch() */
+		if (prm->cid & RM_MASTER) {
+			if (rrm_issue_meas_req(prm) == _SUCCESS)
+				rtw_fsm_st_goto(prm, RRM_ST_WAIT_MEAS);
+			else
+				rtw_fsm_st_goto(prm, RRM_ST_END);
+			return _SUCCESS;
+		} else {
+			rtw_fsm_st_goto(prm, RRM_ST_DO_MEAS);
+			return _SUCCESS;
+		}
+
+#if 0
+		/* TODO below are dead code */
+		if (prm->p.m_mode) {
+			rrm_issue_null_reply(prm);
+			rtw_fsm_st_goto(prm, RRM_ST_END);
+			return _SUCCESS;
+		}
+		if (prm->q.rand_intvl) {
+		#if 0 /*GEORGIA_TODO_REDEFINE_IO*/
+			/* get low tsf to generate random interval */
+			val32 = rtw_read32(padapter, REG_TSFTR);
+		#endif
+			val32 = rtw_hal_get_ltsf(padapter);
+			val32 = val32 % prm->q.rand_intvl;
+			FSM_INFO(prm, "rand_intval=%d, rand=%d\n", (int)prm->q.rand_intvl,val32);
+			rtw_fsm_set_alarm(prm, prm->q.rand_intvl, RRM_EV_delay_timer_expire);
+			return _SUCCESS;
+		}
+#endif
+		break;
+	case RRM_EV_delay_timer_expire:
+		rtw_fsm_st_goto(prm, RRM_ST_DO_MEAS);
+		break;
+	case FSM_EV_ABORT:
+	case FSM_EV_DISCONNECTED:
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+/* we do the measuring */
+static int rrm_do_meas_st_hdl(void *obj, u16 event, void *param)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+	_adapter *padapter = obj2adp(prm);
+	u8 val8;
+	u64 val64;
+
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			switch (prm->q.m_type) {
+			case bcn_req:
+				if (prm->q.s_mode == bcn_req_bcn_table) {
+					FSM_INFO(prm, "Beacon table\n");
+					rrm_get_chset(prm);
+					rtw_fsm_st_goto(prm, RRM_ST_SEND_REPORT);
+					return _SUCCESS;
+				}
+				break;
+			case ch_load_req:
+			case noise_histo_req:
+				/* TODO DO not send event */
+				if (prm->poll_mode)
+					rtw_fsm_gen_msg(prm, NULL, 0, FSM_EV_SCAN_DONE);
+				return _SUCCESS;
+			default:
+				rtw_fsm_st_goto(prm, RRM_ST_END);
+				return _SUCCESS;
+			}
+
+			if (!rm_prepare_scan(prm)) {
+#if 0
+				prm->wait_busy = RM_BUSY_TRAFFIC_TIMES;
+				FSM_DBG(prm, "wait busy traffic - %d\n", prm->wait_busy);
+				rtw_fsm_set_alarm(prm, RM_WAIT_BUSY_TIMEOUT, RRM_EV_busy_timer_expire);
+#endif
+				rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+				rrm_issue_null_reply(prm);
+				rtw_fsm_st_goto(prm, RRM_ST_END);
+				return _SUCCESS;
+			}
+		} else if (prm->q.action_code == RM_ACT_LINK_MEAS_REQ) {
+			; /* do nothing */
+			rtw_fsm_st_goto(prm, RRM_ST_SEND_REPORT);
+			return _SUCCESS;
+		}
+		fallthrough;
+	case RRM_EV_retry_timer_expire:
+		/* expired due to meas condition mismatch, meas again */
+		fallthrough;
+	case RRM_EV_start_meas:
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			/* resotre measurement start time */
+			prm->meas_start_time =
+				rtw_hal_get_tsftr_by_port(padapter, rtw_hal_get_port(padapter));
+
+			switch (prm->q.m_type) {
+			case bcn_req:
+				val8 = 1; /* Enable free run counter */
+				rtw_hal_set_hwreg(padapter,
+					HW_VAR_FREECNT, &val8);
+				rrm_sitesurvey(prm);
+				break;
+			case ch_load_req:
+			case noise_histo_req:
+				rrm_sitesurvey(prm);
+				break;
+			default:
+				rtw_fsm_st_goto(prm, RRM_ST_END);
+				return _SUCCESS;
+				break;
+			}
+		}
+		/* handle measurement timeout */
+		rtw_fsm_set_alarm(prm, RM_MEAS_TIMEOUT, RRM_EV_meas_timer_expire);
+		break;
+	case FSM_EV_SCAN_DONE:
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			switch (prm->q.m_type) {
+			case bcn_req:
+				rtw_fsm_st_goto(prm, RRM_ST_SEND_REPORT);
+				return _SUCCESS;
+			case ch_load_req:
+			case noise_histo_req:
+				retrieve_radio_meas_result(prm);
+
+				if (rm_radio_meas_report_cond(prm) == _SUCCESS)
+					rtw_fsm_st_goto(prm, RRM_ST_SEND_REPORT);
+				else
+					rtw_fsm_set_alarm(prm, RM_COND_INTVL,
+						RRM_EV_retry_timer_expire);
+				break;
+			default:
+				rtw_fsm_st_goto(prm, RRM_ST_END);
+				return _SUCCESS;
+			}
+		}
+		break;
+	case RRM_EV_meas_timer_expire:
+		FSM_INFO(prm, "measurement timeount\n");
+		rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+		rrm_issue_null_reply(prm);
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+#if 0
+	case RRM_EV_busy_timer_expire:
+		if (!rm_prepare_scan(prm) && prm->wait_busy--) {
+			FSM_DBG(prm, "wait busy - %d\n",prm->wait_busy);
+			rtw_fsm_set_alarm(prm, RM_WAIT_BUSY_TIMEOUT,
+				RRM_EV_busy_timer_expire);
+			break;
+		}
+		else if (prm->wait_busy <= 0) {
+			FSM_DBG(prm, "wait busy timeout\n");
+			rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+			rrm_issue_null_reply(prm);
+			rtw_fsm_st_goto(prm, RRM_ST_END);
+			return _SUCCESS;
+		}
+		rtw_fsm_gen_msg(prm, NULL, 0, RRM_EV_start_meas);
+		break;
+#endif
+	case RRM_EV_request_timer_expire:
+		rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+		rrm_issue_null_reply(prm);
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case FSM_EV_ABORT:
+	case FSM_EV_DISCONNECTED:
+		rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(prm);
+		/* resotre measurement end time */
+		prm->meas_end_time = rtw_hal_get_tsftr_by_port(padapter
+								, rtw_hal_get_port(padapter));
+
+		val8 = 0; /* Disable free run counter */
+		rtw_hal_set_hwreg(padapter, HW_VAR_FREECNT, &val8);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int rrm_wait_meas_st_hdl(void *obj, u16 event, void *param)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		/* we create meas_req, waiting for peer report */
+		rtw_fsm_set_alarm(prm, RM_REQ_TIMEOUT,
+			RRM_EV_request_timer_expire);
+		break;
+	case RRM_EV_recv_rep:
+		rtw_fsm_st_goto(prm, RRM_ST_RECV_REPORT);
+		break;
+	case RRM_EV_request_timer_expire:
+		FSM_INFO(prm, "request timeout\n");
+		fallthrough;
+	case FSM_EV_ABORT:
+	case FSM_EV_DISCONNECTED:
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static int rrm_send_report_st_hdl(void *obj, u16 event, void *param)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+	u8 val8;
+
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		/* we have to issue report */
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			switch (prm->q.m_type) {
+			case bcn_req:
+				rrm_issue_beacon_rep(prm);
+				break;
+			case ch_load_req:
+			case noise_histo_req:
+				rrm_issue_radio_meas_rep(prm);
+				break;
+			default:
+				rtw_fsm_st_goto(prm, RRM_ST_END);
+				return _SUCCESS;
+			}
+
+		} else if (prm->q.action_code == RM_ACT_LINK_MEAS_REQ) {
+			rrm_issue_link_meas_rep(prm);
+			rtw_fsm_st_goto(prm, RRM_ST_END);
+			return _SUCCESS;
+
+		} else {
+			rtw_fsm_st_goto(prm, RRM_ST_END);
+			return _SUCCESS;
+		}
+
+		/* check repeat */
+		if (prm->p.rpt) {
+			FSM_INFO(prm, "repeat=%u/%u\n", prm->p.rpt, prm->q.rpt);
+			prm->p.rpt--;
+			/*
+			* we recv meas_req,
+			* delay for a wihile and than meas again
+			*/
+			if (prm->poll_mode)
+				rtw_fsm_set_alarm(prm, RM_REPT_POLL_INTVL,
+					RRM_EV_repeat_delay_expire);
+			else
+				rtw_fsm_set_alarm(prm, RM_REPT_SCAN_INTVL,
+					RRM_EV_repeat_delay_expire);
+			return _SUCCESS;
+		}
+		/* we are done */
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case RRM_EV_repeat_delay_expire:
+		rtw_fsm_st_goto(prm, RRM_ST_DO_MEAS);
+		break;
+	case FSM_EV_ABORT:
+	case FSM_EV_DISCONNECTED:
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static int rrm_recv_report_st_hdl(void *obj, u16 event, void *param)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+	u8 val8;
+
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		/* we issue meas_req, got peer's meas report */
+		switch (prm->p.action_code) {
+		case RM_ACT_RADIO_MEAS_REP:
+			/* check refuse, incapable and repeat */
+			val8 = prm->p.m_mode;
+			if (val8) {
+				FSM_INFO(prm, "peer reject (%s repeat=%d)\n",
+					val8&MEAS_REP_MOD_INCAP?"INCAP":
+					val8&MEAS_REP_MOD_REFUSE?"REFUSE":
+					val8&MEAS_REP_MOD_LATE?"LATE":"",
+					prm->p.rpt);
+				rtw_fsm_st_goto(prm, RRM_ST_END);
+				return _SUCCESS;
+			}
+			break;
+		case RM_ACT_NB_REP_RESP:
+			/* report to upper layer if needing */
+			rtw_fsm_st_goto(prm, RRM_ST_END);
+			return _SUCCESS;
+		default:
+			rtw_fsm_st_goto(prm, RRM_ST_END);
+			return _SUCCESS;
+		}
+		/* check repeat */
+		if (prm->p.rpt) {
+			FSM_INFO(prm, "repeat=%u/%u\n", prm->p.rpt, prm->q.rpt);
+			prm->p.rpt--;
+			/* waitting more report */
+			rtw_fsm_st_goto(prm, RRM_ST_WAIT_MEAS);
+			break;
+		}
+		/* we are done */
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case FSM_EV_ABORT:
+	case FSM_EV_DISCONNECTED:
+		rtw_fsm_st_goto(prm, RRM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static int rrm_end_st_hdl(void *obj, u16 event, void *param)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+	_adapter *a = obj2adp(prm);
+	struct rrm_priv *prmpriv = &a->fsmpriv.rmpriv;
+	int i;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+
+		rrm_dequeue_rm(obj2priv(prm), prm);
+
+		if (prm->q.opt.bcn.req_start)
+			rtw_mfree(prm->q.opt.bcn.req_start, prm->q.opt.bcn.req_len);
+		for (i = 0; i < prm->q.opt.bcn.ap_ch_rpt_num; i++)
+			if (prm->q.opt.bcn.ap_ch_rpt[i])
+				rtw_mfree(prm->q.opt.bcn.ap_ch_rpt[i],
+					sizeof(struct _RT_OPERATING_CLASS));
+		rtw_fsm_deactivate_obj(prm);
+		break;
+
+	case FSM_EV_ABORT:
+	case FSM_EV_STATE_OUT:
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static void rrm_dump_obj_cb(void *obj, char *p, int *sz)
+{
+	/* nothing to do for now */
+}
+
+static void rrm_dump_fsm_cb(void *fsm, char *p, int *sz)
+{
+	/* nothing to do for now */
+}
+
+static void rm_dbg_help(void *obj, char *p, int *sz)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+	int len = *sz;
+
+	snprintf(pstr(p), lstr(p, len),
+		"usage:\n\t<%s> <wdog>,<pause|resume>\n",
+		rtw_fsm_obj_name(prm));
+	*sz = len;
+}
+
+static void rm_debug(void *obj, char input[][MAX_ARGV], u32 input_num,
+	char *output, u32 *out_len)
+{
+	struct rrm_obj *prm = (struct rrm_obj *)obj;
+	char *ptr = output;
+	int len = *out_len;
+
+	if (input_num <  2) {
+		rm_dbg_help(prm, ptr, &len);
+		goto done;
+	}
+
+	if (!strcmp(input[0], "wdog")) {
+		if (!strcmp(input[1], "pause")) {
+
+			snprintf(pstr(ptr), lstr(ptr, len),
+				"\n%s: pause watchdog\n",
+				rtw_fsm_obj_name(prm));
+
+		} else if (!strcmp(input[1], "resume")) {
+
+			/* wdog,resume */
+			snprintf(pstr(ptr), lstr(ptr, len),
+				"\n%s: resume watchdog\n",
+				rtw_fsm_obj_name(prm));
+		}
+	} else
+		rm_dbg_help(prm, ptr, &len);
+done:
+	*out_len = len;
+}
+
+static int rrm_init_priv_cb(void *priv)
+{
+	struct rrm_priv *prmpriv = (struct rrm_priv *)priv;
+
+	/* RRM private queue */
+	_rtw_init_queue(&(prmpriv->rm_queue));
+
+	/* bit 0-7 */
+	prmpriv->rm_en_cap_def[0] = 0
+		| BIT(RM_LINK_MEAS_CAP_EN)
+		| BIT(RM_NB_REP_CAP_EN)
+		/*| BIT(RM_PARAL_MEAS_CAP_EN)*/
+		| BIT(RM_REPEAT_MEAS_CAP_EN)
+		| BIT(RM_BCN_PASSIVE_MEAS_CAP_EN)
+		| BIT(RM_BCN_ACTIVE_MEAS_CAP_EN)
+		| BIT(RM_BCN_TABLE_MEAS_CAP_EN)
+		| BIT(RM_BCN_MEAS_REP_COND_CAP_EN);
+
+	/* bit  8-15 */
+	prmpriv->rm_en_cap_def[1] = 0
+		/*| BIT(RM_FRAME_MEAS_CAP_EN - 8)*/
+#ifdef CONFIG_RTW_ACS
+		| BIT(RM_CH_LOAD_CAP_EN - 8)
+		| BIT(RM_NOISE_HISTO_CAP_EN - 8)
+#endif
+		/*| BIT(RM_STATIS_MEAS_CAP_EN - 8)*/
+		/*| BIT(RM_LCI_MEAS_CAP_EN - 8)*/
+		/*| BIT(RM_LCI_AMIMUTH_CAP_EN - 8)*/
+		/*| BIT(RM_TRANS_STREAM_CAT_MEAS_CAP_EN - 8)*/
+		/*| BIT(RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN - 8)*/;
+
+	/* bit 16-23 */
+	prmpriv->rm_en_cap_def[2] = 0
+		/*| BIT(RM_AP_CH_REP_CAP_EN - 16)*/
+		/*| BIT(RM_RM_MIB_CAP_EN - 16)*/
+		/*| BIT(RM_OP_CH_MAX_MEAS_DUR0 - 16)*/
+		/*| BIT(RM_OP_CH_MAX_MEAS_DUR1 - 16)*/
+		/*| BIT(RM_OP_CH_MAX_MEAS_DUR2 - 16)*/
+		/*| BIT(RM_NONOP_CH_MAX_MEAS_DUR0 - 16)*/
+		/*| BIT(RM_NONOP_CH_MAX_MEAS_DUR1 - 16)*/
+		/*| BIT(RM_NONOP_CH_MAX_MEAS_DUR2 - 16)*/;
+
+	/* bit 24-31 */
+	prmpriv->rm_en_cap_def[3] = 0
+		/*| BIT(RM_MEAS_PILOT_CAP0 - 24)*/
+		/*| BIT(RM_MEAS_PILOT_CAP1 - 24)*/
+		/*| BIT(RM_MEAS_PILOT_CAP2 - 24)*/
+		/*| BIT(RM_MEAS_PILOT_TRANS_INFO_CAP_EN - 24)*/
+		/*| BIT(RM_NB_REP_TSF_OFFSET_CAP_EN - 24)*/
+		| BIT(RM_RCPI_MEAS_CAP_EN - 24)
+		| BIT(RM_RSNI_MEAS_CAP_EN - 24)
+		/*| BIT(RM_BSS_AVG_ACCESS_DELAY_CAP_EN - 24)*/;
+
+	/* bit 32-39 */
+	prmpriv->rm_en_cap_def[4] = 0
+		/*| BIT(RM_BSS_AVG_ACCESS_DELAY_CAP_EN - 32)*/
+		/*| BIT(RM_AVALB_ADMIS_CAPACITY_CAP_EN - 32)*/
+		/*| BIT(RM_ANT_CAP_EN - 32)*/;
+
+	return _SUCCESS;
+}
+
+static int rrm_deinit_priv_cb(void *priv)
+{
+	struct rrm_priv *prmpriv = (struct rrm_priv *)priv;
+
+	/* RRM private queue */
+	_rtw_deinit_queue(&(prmpriv->rm_queue));
+
+	return _SUCCESS;
+}
+
+u8 rm_add_nb_req(_adapter *padapter, struct sta_info *psta)
+{
+	struct rrm_obj *prm;
+
+	prm = rtw_rrm_new_obj(padapter, psta, 0);
+
+	if (prm == NULL) {
+		RTW_ERR("rrm: unable to alloc rm obj for requeset\n");
+		return _FALSE;
+	}
+
+	prm->q.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+	prm->q.action_code = RM_ACT_NB_REP_REQ;
+
+	rtw_fsm_activate_obj(prm);
+
+	FSM_INFO(prm, "%s to " MAC_FMT "\n", __func__, MAC_ARG(obj2mac(prm)));
+
+	return _SUCCESS;
+}
+
+/* For EXTERNAL application to create RRM FSM */
+int rtw_rrm_reg_fsm(struct fsm_priv *fsmpriv)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct rrm_priv *prmpriv = &fsmpriv->rmpriv;
+	struct fsm_main *fsm = NULL;
+	struct rtw_fsm_tb tb;
+
+	memset(&tb, 0, sizeof(tb));
+	tb.max_state = sizeof(rrm_state_tbl)/sizeof(rrm_state_tbl[0]);
+	tb.max_event = sizeof(rrm_event_tbl)/sizeof(rrm_event_tbl[0]);
+	tb.state_tbl = rrm_state_tbl;
+	tb.evt_tbl = rrm_event_tbl;
+	tb.priv = prmpriv;
+	tb.init_priv = rrm_init_priv_cb;
+	tb.deinit_priv = rrm_deinit_priv_cb;
+	tb.dump_obj = rrm_dump_obj_cb;
+	tb.dump_fsm = rrm_dump_fsm_cb;
+	tb.dbg_level = FSM_LV_TRACE;
+	tb.evt_level = FSM_LV_TRACE;
+	tb.debug = rm_debug;
+
+	fsm = rtw_fsm_register_fsm(root, "rrm", &tb);
+	prmpriv->fsm = fsm;
+
+	if (fsm)
+		return _SUCCESS;
+	return _FAIL;
+}
+#endif /* CONFIG_RTW_FSM_RRM */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_wnm.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_wnm.c
new file mode 100644
index 000000000000..54f047369ac0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_wnm.c
@@ -0,0 +1,763 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2023 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_FSM_BTM
+
+#define BTM_CONNECT_WAIT 2000
+#define BTM_DISASSOC_WAIT 1000
+#ifdef CONFIG_IEEE80211_BAND_6GHZ
+#define BTM_SCAN_TIMEOUT 15000
+#else
+#define BTM_SCAN_TIMEOUT 10000
+#endif
+#define BTM_SCAN_DENY_WAIT 500
+#define BTM_FT_ACTION_TIMEOUT 1000
+#define BTM_SCAN_DWELL_TIME 30
+/*
+* State machine
+*/
+enum BTM_STATE_ST {
+	BTM_ST_START,
+	BTM_ST_SCAN,
+	BTM_ST_OTD,
+	BTM_ST_ROAM,
+	BTM_ST_END,
+	BTM_ST_MAX
+};
+
+enum BTM_EV_ID {
+	BTM_EV_scan_deny_expire,
+	BTM_EV_scan_timeout,
+	BTM_EV_scan_found_candidate,
+	BTM_EV_scan_opch_reported,
+	BTM_EV_connect_timeout,
+	BTM_EV_disassoc_timeout,
+	BTM_EV_ft_action_timeout,
+	BTM_EV_ft_action_resp,
+	BTM_EV_max
+};
+
+static int btm_start_st_hdl(void *obj, u16 event, void *param);
+static int btm_scan_st_hdl(void *obj, u16 event, void *param);
+static int btm_otd_st_hdl(void *obj, u16 event, void *param);
+static int btm_roam_st_hdl(void *obj, u16 event, void *param);
+static int btm_end_st_hdl(void *obj, u16 event, void *param);
+
+/* STATE table */
+static struct fsm_state_ent btm_state_tbl[] = {
+	ST_ENT(BTM_ST_START, btm_start_st_hdl),
+	ST_ENT(BTM_ST_SCAN, btm_scan_st_hdl),
+	ST_ENT(BTM_ST_OTD, btm_otd_st_hdl),
+	ST_ENT(BTM_ST_ROAM, btm_roam_st_hdl),
+	ST_ENT(BTM_ST_END, btm_end_st_hdl),
+};
+
+/* EVENT table */
+static struct fsm_event_ent btm_event_tbl[] = {
+	EV_DBG(BTM_EV_scan_deny_expire),
+	EV_DBG(BTM_EV_scan_timeout),
+	EV_DBG(BTM_EV_scan_found_candidate),
+	EV_DBG(BTM_EV_scan_opch_reported),
+	EV_DBG(BTM_EV_connect_timeout),
+	EV_DBG(BTM_EV_disassoc_timeout),
+	EV_DBG(BTM_EV_ft_action_timeout),
+	EV_DBG(BTM_EV_ft_action_resp),
+	EV_DBG(BTM_EV_max) /* EV_MAX for fsm safety checking */
+};
+
+struct btm_obj {
+
+	u32 cid;
+	u8 resp_reason;
+	u8 roam_reason;
+	u8 ft_req_retry_cnt;
+	u8 op_ch_reported;
+	struct roam_nb_info pnb;
+	struct wlan_network *candidate;
+	int finish;
+
+	/* obj default */
+	struct fsm_obj *obj;
+	struct fsm_main *fsm;
+	_list list;
+};
+
+/*
+ * btm sub function
+ */
+#if CONFIG_IEEE80211_BAND_6GHZ
+/* save PSC channel and remove other channel */
+static int btm_scan_ch_psc(struct btm_obj *pbtm, struct rtw_ieee80211_channel *out, u32 out_num)
+{
+	int i, ch_num = 0;
+	struct rtw_ieee80211_channel *chs;
+
+	chs = rtw_zmalloc(sizeof(struct rtw_ieee80211_channel) * out_num);
+	if (!chs)
+		return out_num;
+
+	_rtw_memcpy(chs, out, sizeof(struct rtw_ieee80211_channel) * out_num);
+	_rtw_memset(out, 0, sizeof(struct rtw_ieee80211_channel) * out_num);
+
+	for (i = 0; i < out_num; i ++) {
+		if (chs[i].band != BAND_ON_6G)
+			continue;
+
+		 /* PSC = 16*n+5 */
+		if ((chs[i].hw_value-5)%16 != 0)
+			continue;
+
+		/* Save PSC channel */
+		_rtw_memcpy(&out[ch_num++], &chs[i], sizeof(struct rtw_ieee80211_channel));
+	}
+	rtw_mfree(chs, sizeof(struct rtw_ieee80211_channel) * out_num);
+	return ch_num;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static int btm_scan_ch_sort(struct btm_obj *pbtm, struct rtw_ieee80211_channel *out, u32 out_num)
+{
+	int i, j, k, l;
+	struct rtw_ieee80211_channel *chs;
+	_adapter *a = obj2adp(pbtm);
+	u8 *band_order = a->mlmepriv.roam_scan_order;
+
+	chs = rtw_zmalloc(sizeof(struct rtw_ieee80211_channel) * out_num);
+	if (!chs)
+		return out_num;
+
+	_rtw_memcpy(chs, out, sizeof(struct rtw_ieee80211_channel) * out_num);
+	_rtw_memset(out, 0, sizeof(struct rtw_ieee80211_channel) * out_num);
+
+	k = 0;
+	for (i = 0; i < 3; i++) {
+		if (band_order[i] >= BAND_MAX)
+			break;
+		for (j = 0; j < out_num; j++) {
+			if (chs[j].hw_value == 0)
+				break;
+			if (chs[j].band != band_order[i])
+				continue;
+			if (chs[j].flags & RTW_IEEE80211_CHAN_PASSIVE_SCAN)
+				continue;
+
+			/* ACTIVE CH */
+			_rtw_memcpy(&out[k++], &chs[j], sizeof(struct rtw_ieee80211_channel));
+		}
+
+		for (j = 0; j < out_num; j++) {
+			if (chs[j].hw_value == 0)
+				break;
+			if (chs[j].band != band_order[i])
+				continue;
+			if (!(chs[j].flags & RTW_IEEE80211_CHAN_PASSIVE_SCAN))
+				continue;
+
+			/* PASSIVE CH */
+			_rtw_memcpy(&out[k++], &chs[j], sizeof(struct rtw_ieee80211_channel));
+
+		}
+	}
+	rtw_mfree(chs, sizeof(struct rtw_ieee80211_channel) * out_num);
+	return k;
+}
+
+/* extend roam channel into full scan channel */
+static int btm_roam_ch_decision(struct btm_obj *pbtm, struct rtw_ieee80211_channel *out,
+		u32 out_num, struct rtw_ieee80211_channel *in_ch, u32 in_num)
+{
+	_adapter *a = obj2adp(pbtm);
+	struct rtw_ieee80211_channel *roam_ch = NULL, *full_ch = NULL;
+	int i, j, k, roam_ch_num = 0, full_ch_num = 0;
+
+	/* sort original ROAM channel */
+	if (in_num) {
+		roam_ch = rtw_zmalloc(sizeof(struct rtw_ieee80211_channel) * in_num);
+		if (!roam_ch)
+			goto roam_exit;
+		roam_ch_num = rtw_scan_ch_decision(a, roam_ch,
+					in_num, in_ch, in_num, _TRUE);
+		roam_ch_num = btm_scan_ch_sort(pbtm, roam_ch, roam_ch_num);
+	}
+
+	full_ch = rtw_zmalloc(sizeof(struct rtw_ieee80211_channel) * RTW_CHANNEL_SCAN_AMOUNT);
+	if (!full_ch)
+		goto full_exit;
+
+	/* get full channel */
+	full_ch_num = rtw_scan_ch_decision(a, full_ch,
+				RTW_CHANNEL_SCAN_AMOUNT, NULL, 0, _TRUE);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* only accept 6G PSC channel */
+	full_ch_num = btm_scan_ch_psc(pbtm, full_ch, full_ch_num);
+#endif
+	/* sort full channel */
+	full_ch_num = btm_scan_ch_sort(pbtm, full_ch, full_ch_num);
+
+	_rtw_memset(out, 0, sizeof(struct rtw_ieee80211_channel) * out_num);
+
+	if (roam_ch_num) {
+		/* combine ROAM ch and full ch */
+		_rtw_memcpy(out, roam_ch, sizeof(struct rtw_ieee80211_channel) * roam_ch_num);
+
+		k = roam_ch_num;
+		for (i = 0; i < full_ch_num; i++) {
+			for (j = 0; j < k; j++) {
+				if (roam_ch[j].hw_value == full_ch[i].hw_value)
+					goto next;
+			}
+			_rtw_memcpy(&out[roam_ch_num++] ,&full_ch[i], sizeof(struct rtw_ieee80211_channel));
+next:
+			; /* make compile happy:label at end of compound statement */
+		}
+	} else {
+		roam_ch_num = full_ch_num;
+		_rtw_memcpy(out ,full_ch, sizeof(struct rtw_ieee80211_channel) * full_ch_num);
+	}
+
+	if (full_ch)
+		rtw_mfree(full_ch, sizeof(struct rtw_ieee80211_channel) * RTW_CHANNEL_SCAN_AMOUNT);
+full_exit:
+	if (roam_ch)
+		rtw_mfree(roam_ch, sizeof(struct rtw_ieee80211_channel) * in_num);
+roam_exit:
+#if 0
+	for (i = 0; i < roam_ch_num; i++) {
+		RTW_INFO("roam ch: %d\n", out[i].hw_value);
+	}
+#endif
+	return roam_ch_num;
+}
+
+static void btm_nb_scan_list_set(struct btm_obj *pbtm, struct sitesurvey_parm *pparm)
+{
+	u32 i;
+	struct roam_nb_info *pnb = &pbtm->pnb;
+	_adapter *a = obj2adp(pbtm);
+	struct mlme_priv *pmlmepriv = &a->mlmepriv;
+
+	if (pmlmepriv->cur_network_scanned) {
+		_rtw_memcpy(&pparm->ssid[0], &pmlmepriv->cur_network_scanned->network.Ssid,
+			sizeof(struct _NDIS_802_11_SSID));
+		pparm->ssid_num = 1;
+	}
+
+	for (i = 0; i < pnb->nb_rpt_ch_list_num; i++) {
+		_rtw_memcpy(pparm->nb[i].bssid, pnb->nb_rpt[i].bssid, ETH_ALEN);
+		pparm->nb[i].ch = pnb->nb_rpt[i].ch_num;
+		pparm->nb_num++;
+	}
+
+	pparm->scan_mode = RTW_PHL_SCAN_ACTIVE;
+	pparm->duration = BTM_SCAN_DWELL_TIME;
+	pparm->ch_num = btm_roam_ch_decision(pbtm, pparm->ch, RTW_CHANNEL_SCAN_AMOUNT,
+		pnb->nb_rpt_ch_list, pnb->nb_rpt_ch_list_num);
+	pnb->nb_rpt_valid = _FALSE;
+}
+
+static int btm_roam_scan(struct btm_obj *pbtm)
+{
+	_adapter *a = obj2adp(pbtm);
+	struct sitesurvey_parm *parm;
+	u8 ret = _FAIL;
+
+	parm = (struct sitesurvey_parm*)rtw_zmalloc(sizeof(*parm));
+
+	if (!parm)
+		return ret;
+
+	rtw_init_sitesurvey_parm(a, parm);
+	btm_nb_scan_list_set(pbtm, parm);
+	parm->scan_type = RTW_SCAN_ROAM;
+	parm->psta = obj2sta(pbtm);
+
+	ret = rtw_sitesurvey_cmd(a, parm);
+	rtw_mfree(parm, sizeof(*parm));
+
+	return ret;
+}
+
+static int btm_del_obj(struct btm_obj *pbtm)
+{
+	struct btm_priv *btmpriv = obj2priv(pbtm);
+	struct sta_info *psta = obj2sta(pbtm);
+
+	psta->btm = NULL;
+	btmpriv->btm = NULL;
+
+	rtw_fsm_deactivate_obj(pbtm);
+
+	return _SUCCESS;
+}
+
+/*
+ * BTM state handler
+ */
+static int btm_start_st_hdl(void *obj, u16 event, void *param)
+{
+	struct btm_obj *pbtm = (struct btm_obj *)obj;
+	_adapter *a = obj2adp(pbtm);
+	struct btm_priv *btmpriv = (struct btm_priv *)obj2priv(pbtm);
+	struct roam_nb_info *pnb = &pbtm->pnb;
+	u8 ssc_chk;
+	int i;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		pbtm->resp_reason = 1; /* unspecified reject reason */
+		for (i = 0; i < pnb->last_nb_rpt_entries; i++)
+			FSM_INFO(pbtm, "roam list "MAC_FMT"\n",
+				MAC_ARG(pnb->nb_rpt[i].bssid));
+		fallthrough;
+	case BTM_EV_scan_deny_expire:
+		ssc_chk = rtw_sitesurvey_condition_check(a, _FALSE);
+		if (ssc_chk == SS_DENY_BUSY_TRAFFIC) {
+			FSM_INFO(pbtm, "need to roam, don't care BusyTraffic\n");
+		} else if (ssc_chk != SS_ALLOW) {
+			rtw_fsm_set_alarm(pbtm, BTM_SCAN_DENY_WAIT, BTM_EV_scan_deny_expire);
+			return _SUCCESS;
+		}
+		/* Able to scan */
+		rtw_fsm_st_goto(pbtm, BTM_ST_SCAN);
+		break;
+	case FSM_EV_ABORT:
+	case FSM_EV_DISCONNECTED:
+		if (pbtm->roam_reason & RTW_ROAM_BTM)
+			rtw_wnm_issue_action(a, &pbtm->pnb, RTW_WLAN_ACTION_WNM_BTM_RSP,
+				pbtm->resp_reason, pbtm->pnb.btm_cache.dialog_token);
+		rtw_fsm_st_goto(pbtm, BTM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(pbtm);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int btm_scan_st_hdl(void *obj, u16 event, void *param)
+{
+	struct btm_obj *pbtm = (struct btm_obj *)obj;
+	_adapter *a = obj2adp(pbtm);
+	struct mlme_priv *pmlmepriv = &a->mlmepriv;
+	struct roam_nb_info *pnb = &pbtm->pnb;
+	struct wlan_network *pnetwork = NULL;
+	struct wlan_network *cnetwork = pmlmepriv->cur_network_scanned;
+	struct roam_nb_info *mlmepriv_pnb = &(pmlmepriv->nb_info);
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		if (!btm_roam_scan(pbtm)) {
+			rtw_fsm_st_goto(pbtm, BTM_ST_END);
+			return _SUCCESS;
+		}
+		pbtm->op_ch_reported = 0;
+		/* Set scan timeout */
+		rtw_fsm_set_alarm(pbtm, BTM_SCAN_TIMEOUT, BTM_EV_scan_timeout);
+		break;
+	case BTM_EV_scan_opch_reported:
+		pbtm->op_ch_reported = 1;
+		break;
+	case BTM_EV_scan_found_candidate:
+		pnetwork = (struct wlan_network *)param;
+
+		FSM_DBG(pbtm, "Recv %s ("MAC_FMT")\n",pnetwork->network.Ssid.Ssid,
+			MAC_ARG(pnetwork->network.MacAddress));
+
+		if ((pnetwork->network.Configuration.DSConfig ==\
+			cnetwork->network.Configuration.DSConfig) &&
+			(pnetwork->network.Configuration.Band ==\
+			 cnetwork->network.Configuration.Band)) {
+
+			if (pbtm->op_ch_reported) /* already reported */
+				return _SUCCESS;
+			rtw_fsm_set_alarm_ext(pbtm, 100, BTM_EV_scan_opch_reported, 1, NULL);
+		}
+
+		if (pbtm->roam_reason & RTW_ROAM_ACTIVE)
+			rtw_check_roaming_candidate(pmlmepriv, &pbtm->candidate, pnetwork);
+		else if (pbtm->roam_reason & RTW_ROAM_BTM)
+			pbtm->candidate = rtw_wnm_btm_candidate_check(a, &pbtm->pnb, pnetwork);
+		else if (pbtm->roam_reason & RTW_ROAM_ON_RESUME)
+			rtw_check_join_candidate(pmlmepriv, &pbtm->candidate, pnetwork);
+		else
+			return _SUCCESS;
+
+		if (!pbtm->candidate)
+			return _SUCCESS;
+
+		a->mlmepriv.roam_network = pbtm->candidate;
+		rtw_scan_abort_no_wait(a);
+		/* wait scan done */
+		break;
+	case FSM_EV_SCAN_DONE:
+		rtw_fsm_cancel_alarm(pbtm);
+
+		if (pbtm->roam_reason & RTW_ROAM_ACTIVE) {
+			if (!pbtm->candidate) {
+				FSM_INFO(pbtm, "Pick a candidate from scan queue\n");
+				pbtm->candidate = rtw_select_roaming_candidate(pmlmepriv);
+			}
+
+		} else if (pbtm->roam_reason & RTW_ROAM_BTM) {
+
+			pbtm->resp_reason = rtw_wmn_btm_rsp_reason_decision(a, pnb);
+			if (pbtm->resp_reason > 0) {
+				pbtm->candidate = NULL;
+				rtw_fsm_st_goto(pbtm, BTM_ST_END);
+				return _SUCCESS;
+			}
+			if (!pbtm->candidate) {
+				FSM_INFO(pbtm, "Pick a candidate from scan queue\n");
+#ifdef DBG_WNM_UNIT_TEST
+				pbtm->candidate = rtw_wnm_btm_candidate_select(a, pnb);
+#else
+				if (!rtw_mbo_wifi_logo_test(a))
+					pbtm->candidate = rtw_select_roaming_candidate(pmlmepriv);
+				else
+					pbtm->candidate = rtw_wnm_btm_candidate_select(a, pnb);
+#endif
+			}
+			if (pbtm->candidate) {
+				_rtw_memcpy(pnb->roam_target_addr,
+					(u8 *)&pbtm->candidate->network.MacAddress,
+					ETH_ALEN);
+
+				/* for ReAssocreq check in rtw_wnm_btm_reassoc_req() */
+				/* TODO Do NOT use global variable */
+				_rtw_memcpy(mlmepriv_pnb->roam_target_addr,
+					(u8 *)&pbtm->candidate->network.MacAddress,
+					ETH_ALEN);
+			}
+
+		} else if (pbtm->roam_reason & RTW_ROAM_ON_RESUME) {
+			if (rtw_select_and_join_from_scanned_queue(pmlmepriv))
+				pbtm->finish = _SUCCESS;
+
+			rtw_fsm_st_goto(pbtm, BTM_ST_END);
+			return _SUCCESS;
+		}
+
+		/* RTW_ROAM_ACTIVE */
+		if (!pbtm->candidate) {
+			if (pbtm->resp_reason == 0)
+				pbtm->resp_reason = 7;
+
+			rtw_fsm_st_goto(pbtm, BTM_ST_END);
+			return _SUCCESS;
+		}
+		a->mlmepriv.roam_network = pbtm->candidate;
+
+		if (rtw_ft_otd_roam(a))
+			rtw_fsm_st_goto(pbtm, BTM_ST_OTD); /* Over The DS */
+		else
+			rtw_fsm_st_goto(pbtm, BTM_ST_ROAM);
+		break;
+	case BTM_EV_scan_timeout:
+	case FSM_EV_ABORT:
+	case FSM_EV_DISCONNECTED:
+		rtw_scan_abort_no_wait(a);
+		rtw_fsm_st_goto(pbtm, BTM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(pbtm);
+		rtw_fsm_cancel_alarm_ext(pbtm, 1);
+		if (pbtm->roam_reason & RTW_ROAM_BTM)
+			rtw_wnm_issue_action(a, pnb, RTW_WLAN_ACTION_WNM_BTM_RSP,
+				pbtm->resp_reason, pnb->btm_cache.dialog_token);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int btm_otd_st_hdl(void *obj, u16 event, void *param)
+{
+	struct btm_obj *pbtm = (struct btm_obj *)obj;
+	_adapter *a = obj2adp(pbtm);
+	struct mlme_priv *pmlmepriv = &a->mlmepriv;
+	struct ft_roam_info *pft_roam = &(pmlmepriv->ft_roam);
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		FSM_INFO(pbtm, "FT OTD roam\n");
+		fallthrough;
+	case BTM_EV_ft_action_timeout:
+		if (pbtm->ft_req_retry_cnt++ >= RTW_FT_ACTION_REQ_LMT) {
+			FSM_WARN(pbtm, "RX FT action timeout, try OTA roam\n");
+			rtw_ft_clr_flags(a, RTW_FT_PEER_OTD_EN);
+			rtw_fsm_st_goto(pbtm, BTM_ST_ROAM);
+			return _SUCCESS;
+		}
+		/* send ft action frame */
+		rtw_ft_issue_action_req(a, pbtm->candidate->network.MacAddress);
+		rtw_fsm_set_alarm(pbtm, BTM_FT_ACTION_TIMEOUT, BTM_EV_ft_action_timeout);
+		break;
+	case BTM_EV_ft_action_resp:
+		rtw_fsm_cancel_alarm(pbtm);
+		rtw_fsm_st_goto(pbtm, BTM_ST_ROAM);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(pbtm);
+		break;
+	case FSM_EV_DISCONNECTED:
+	case FSM_EV_ABORT:
+		rtw_fsm_st_goto(pbtm, BTM_ST_END);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static int btm_roam_st_hdl(void *obj, u16 event, void *param)
+{
+	struct btm_obj *pbtm = (struct btm_obj *)obj;
+	_adapter *a = obj2adp(pbtm);
+	struct mlme_priv *pmlmepriv = &a->mlmepriv;
+	struct mlme_ext_info *pmlmeinfo = &a->mlmeextpriv.mlmext_info;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		RTW_INFO("roaming from %s("MAC_FMT"), length:%d\n",
+			pmlmepriv->dev_cur_network.network.Ssid.Ssid,
+			MAC_ARG(pmlmepriv->dev_cur_network.network.MacAddress),
+			pmlmepriv->dev_cur_network.network.Ssid.SsidLength);
+
+		if (pbtm->pnb.btm_cache.req_mode & DISASSOC_IMMINENT) {
+			/* indicates that the STA is to be disassociated
+			 * from the current AP */
+			rtw_fsm_set_alarm(pbtm, BTM_DISASSOC_WAIT, BTM_EV_disassoc_timeout);
+			return _SUCCESS;
+		}
+		fallthrough;
+	case BTM_EV_disassoc_timeout:
+		if (rtw_ft_otd_roam(a)) {
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_FT;
+		} else {
+			rtw_set_to_roam(a, a->registrypriv.max_roaming_times);
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_ROAMING;
+		}
+
+		receive_disconnect(a, pmlmepriv->dev_cur_network.network.MacAddress
+			, WLAN_REASON_ACTIVE_ROAM, _FALSE);
+		break;
+	case FSM_EV_DISCONNECTED:
+		FSM_INFO(pbtm, "roaming to %s(" MAC_FMT ")\n",
+			pbtm->candidate->network.Ssid.Ssid,
+			MAC_ARG(pbtm->candidate->network.MacAddress));
+
+		_rtw_memcpy(&pmlmepriv->assoc_ssid,
+			&pmlmepriv->dev_cur_network.network.Ssid, sizeof(NDIS_802_11_SSID));
+		pmlmepriv->assoc_ch = 0;
+		pmlmepriv->assoc_band = BAND_MAX;
+		pmlmepriv->assoc_by_bssid = _FALSE;
+
+		/* Start to connect */
+		if (!rtw_do_join(a)) {
+			rtw_fsm_st_goto(pbtm, BTM_ST_END);
+		}
+		rtw_fsm_set_alarm(pbtm, BTM_CONNECT_WAIT, BTM_EV_connect_timeout);
+		break;
+	case FSM_EV_CONNECTED:
+		pbtm->finish = _SUCCESS;
+		rtw_set_to_roam(a, 0);
+		rtw_fsm_st_goto(pbtm, BTM_ST_END);
+		break;
+	case FSM_EV_CONNECT_FAIL:
+		/* don't trigger disconnect. Connect module will do it! */
+		pbtm->finish = _SUCCESS;
+		fallthrough;
+	case FSM_EV_ABORT:
+	case BTM_EV_connect_timeout:
+		rtw_fsm_st_goto(pbtm, BTM_ST_END);
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(pbtm);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int btm_end_st_hdl(void *obj, u16 event, void *param)
+{
+	struct btm_obj *pbtm = (struct btm_obj *)obj;
+	_adapter *a = obj2adp(pbtm);
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+
+		if (pbtm->finish == _FAIL) {
+			rtw_set_to_roam(a, 0);
+			rtw_ft_reset_status(a);
+			if (pbtm->candidate) /* driver disconnected */
+				rtw_indicate_disconnect(a, 0, _FALSE);
+		}
+		btm_del_obj(pbtm);
+		break;
+	case FSM_EV_ABORT:
+		break;
+	case FSM_EV_STATE_OUT:
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static void btm_dump_obj_cb(void *obj, char *p, int *sz)
+{
+	/* nothing to do for now */
+}
+
+static void btm_dump_fsm_cb(void *fsm, char *p, int *sz)
+{
+	/* nothing to do for now */
+}
+
+static int btm_init_priv_cb(void *priv)
+{
+	struct btm_priv *pbtmpriv = (struct btm_priv *)priv;
+
+	return _SUCCESS;
+}
+
+static int btm_deinit_priv_cb(void *priv)
+{
+	struct btm_priv *pbtmpriv = (struct btm_priv *)priv;
+
+	return _SUCCESS;
+}
+
+static void btm_debug(void *obj, char input[][MAX_ARGV], u32 input_num,
+	char *output, u32 *out_len)
+{
+	return;
+}
+
+/* For EXTERNAL application to create a btm object
+ * return
+ * btm_obj: ptr to new btm object
+ */
+int rtw_btm_new_obj(_adapter *a, struct sta_info *psta,
+	struct roam_nb_info *pnb, u8 roam_reason)
+{
+	struct btm_priv *btmpriv = &a->fsmpriv.btmpriv;
+	struct fsm_main *fsm = btmpriv->fsm;
+	struct fsm_obj *obj;
+	struct btm_obj *pbtm;
+	struct mlme_priv *pmlmepriv = &a->mlmepriv;
+	u16 cid = 0;
+
+	if (btmpriv->btm) {
+		FSM_WARN_(fsm, "BTM request is on going ...\n");
+		return _FAIL;
+	}
+
+	if (!psta)
+		psta = rtw_get_stainfo(&a->stapriv, get_bssid(pmlmepriv));
+
+	if (!psta)
+		psta = rtw_get_stainfo(&a->stapriv, a->mac_addr);
+
+	if (!psta) {
+		FSM_WARN_(fsm, "psta NOT found\n");
+		return _FAIL;
+	}
+
+	if (roam_reason == RTW_ROAM_BTM)
+		cid = pnb->btm_cache.dialog_token;
+
+	obj = rtw_fsm_new_obj(fsm, psta, cid, (void **)&pbtm, sizeof(*pbtm));
+
+	if (pbtm == NULL) {
+		FSM_ERR_(fsm, "malloc obj fail\n");
+		return _FAIL;
+	}
+	pbtm->fsm = fsm;
+	pbtm->obj = obj;
+	pbtm->cid = cid;
+	pbtm->roam_reason = roam_reason;
+	pbtm->finish = _FAIL;
+
+	_rtw_memcpy(&pbtm->pnb, pnb, sizeof(*pnb));
+
+	/* activate it immediately */
+	rtw_fsm_activate_obj(pbtm);
+	pmlmepriv->need_to_roam = _TRUE;
+	btmpriv->btm = pbtm;
+	psta->btm = pbtm;
+
+	return _SUCCESS;
+}
+
+void rtw_btm_notify_action_resp(struct btm_obj *pbtm)
+{
+	rtw_fsm_gen_msg(pbtm, NULL, 0, BTM_EV_ft_action_resp);
+}
+
+void rtw_btm_notify_scan_found_candidate(struct btm_obj *pbtm, struct wlan_network *pnetwork)
+{
+	_adapter *a = obj2adp(pbtm);
+
+	if (!rtw_chk_roam_flags(a, RTW_ROAM_QUICK_SCAN))
+		return;
+
+	if (!pnetwork)
+		return;
+	rtw_fsm_gen_msg(pbtm, (void *)pnetwork, 0, BTM_EV_scan_found_candidate);
+}
+
+/* For EXTERNAL application to create RRM FSM */
+int rtw_btm_reg_fsm(struct fsm_priv *fsmpriv)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct fsm_main *fsm = NULL;
+	struct rtw_fsm_tb tb;
+	struct btm_priv *btmpriv = &fsmpriv->btmpriv;
+
+	memset(&tb, 0, sizeof(tb));
+	tb.max_state = sizeof(btm_state_tbl)/sizeof(btm_state_tbl[0]);
+	tb.max_event = sizeof(btm_event_tbl)/sizeof(btm_event_tbl[0]);
+	tb.state_tbl = btm_state_tbl;
+	tb.evt_tbl = btm_event_tbl;
+	tb.priv = btmpriv;
+	tb.init_priv = btm_init_priv_cb;
+	tb.deinit_priv = btm_deinit_priv_cb;
+	tb.dump_obj = btm_dump_obj_cb;
+	tb.dump_fsm = btm_dump_fsm_cb;
+	tb.dbg_level = FSM_LV_INFO;
+	tb.evt_level = FSM_LV_INFO;
+	tb.debug = btm_debug;
+
+	fsm = rtw_fsm_register_fsm(root, "wnm", &tb);
+	btmpriv->fsm = fsm;
+
+	if (!fsm)
+		return _FAIL;
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_RTW_FSM_BTM */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_xxx.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_xxx.c
new file mode 100644
index 000000000000..ae960c475a42
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_fsm_xxx.c
@@ -0,0 +1,234 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2023 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_FSM_XXX
+/*
+* State machine
+*/
+enum XXX_STATE_ST {
+	XXX_ST_IDLE,
+	XXX_ST_SCAN,
+	XXX_ST_END,
+	XXX_ST_MAX
+};
+
+enum XXX_EV_ID {
+	XXX_EV_request_timer_expire,
+	XXX_EV_start_scan,
+	XXX_EV_survey_done,
+	XXX_EV_recv_rep,
+	XXX_EV_max
+};
+
+static int xxx_idle_st_hdl(void *obj, u16 event, void *param);
+static int xxx_scan_st_hdl(void *obj, u16 event, void *param);
+static int xxx_end_st_hdl(void *obj, u16 event, void *param);
+
+/* STATE table */
+static struct fsm_state_ent xxx_state_tbl[] = {
+	ST_ENT(XXX_ST_IDLE, xxx_idle_st_hdl),
+	ST_ENT(XXX_ST_SCAN, xxx_scan_st_hdl),
+	ST_ENT(XXX_ST_END, xxx_end_st_hdl),
+};
+
+/* EVENT table */
+static struct fsm_event_ent xxx_event_tbl[] = {
+	EV_DBG(XXX_EV_request_timer_expire),
+	EV_DBG(XXX_EV_start_scan),
+	EV_DBG(XXX_EV_survey_done),
+	EV_DBG(XXX_EV_recv_rep),
+	EV_DBG(XXX_EV_max) /* EV_MAX for fsm safety checking */
+};
+
+struct xxx_obj {
+
+	u32 cid;
+
+	/* obj default */
+	struct fsm_obj *obj;
+	struct fsm_main *fsm;
+
+	_list list;
+};
+
+/*
+ * xxx sub function
+ */
+
+
+/*
+ * XXX state handler
+ */
+static int xxx_idle_st_hdl(void *obj, u16 event, void *param)
+{
+	struct xxx_obj *pxxx = (struct xxx_obj *)obj;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		break;
+	case FSM_EV_ABORT:
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(pxxx);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int xxx_scan_st_hdl(void *obj, u16 event, void *param)
+{
+	struct xxx_obj *pxxx = (struct xxx_obj *)obj;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		break;
+	case FSM_EV_ABORT:
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(pxxx);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int xxx_end_st_hdl(void *obj, u16 event, void *param)
+{
+	struct xxx_obj *pxxx = (struct xxx_obj *)obj;
+
+	switch (event) {
+	case FSM_EV_STATE_IN:
+		break;
+	case FSM_EV_ABORT:
+		break;
+	case FSM_EV_STATE_OUT:
+		rtw_fsm_cancel_alarm(pxxx);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static void xxx_dump_obj_cb(void *obj, char *p, int *sz)
+{
+	/* nothing to do for now */
+}
+
+static void xxx_dump_fsm_cb(void *fsm, char *p, int *sz)
+{
+	/* nothing to do for now */
+}
+
+static int xxx_init_priv_cb(void *priv)
+{
+	struct xxx_priv *pxxxpriv = (struct xxx_priv *)priv;
+
+	RTW_INFO("%s\n", __func__);
+
+	return _SUCCESS;
+}
+
+static int xxx_deinit_priv_cb(void *priv)
+{
+	struct xxx_priv *pxxxpriv = (struct xxx_priv *)priv;
+
+	RTW_INFO("%s\n", __func__);
+
+	return _SUCCESS;
+}
+
+static void xxx_debug(void *obj, char input[][MAX_ARGV], u32 input_num,
+	char *output, u32 *out_len)
+{
+	return;
+}
+
+/* For EXTERNAL application to create a xxx object
+ * return
+ * xxx_obj: ptr to new xxx object
+ */
+struct xxx_obj *rtw_xxx_new_obj(_adapter *a, struct sta_info *psta, u16 cid)
+{
+	struct fsm_main *fsm = a->fsmpriv.xxxpriv.fsm;
+	struct fsm_obj *obj;
+	struct xxx_obj *pxxx;
+
+	obj = rtw_fsm_new_obj(fsm, psta, cid, (void **)&pxxx, sizeof(*pxxx));
+
+	if (pxxx == NULL) {
+		FSM_ERR_(fsm, "malloc obj fail\n");
+		return NULL;
+	}
+	pxxx->fsm = fsm;
+	pxxx->obj = obj;
+
+	FSM_TRACE(pxxx, "%s\n", __func__);
+
+	return pxxx;
+}
+
+/* For EXTERNAL application to abort xxx section
+ */
+void rtw_xxx_abort(struct xxx_priv *pxxxpriv)
+{
+	struct xxx_obj *pxxx = pxxxpriv->xxx;
+
+	if (!pxxx)
+		return;
+
+	FSM_TRACE(pxxx, "%s\n", __func__);
+
+	rtw_fsm_gen_msg(pxxx, NULL, 0, FSM_EV_ABORT);
+}
+
+/* For EXTERNAL application to create RRM FSM */
+int rtw_xxx_reg_fsm(struct fsm_priv *fsmpriv)
+{
+	struct fsm_root *root = fsmpriv->root;
+	struct fsm_main *fsm = NULL;
+	struct rtw_fsm_tb tb;
+	struct xxx_priv *xxxpriv = &fsmpriv->xxxpriv;
+
+	memset(&tb, 0, sizeof(tb));
+	tb.max_state = sizeof(xxx_state_tbl)/sizeof(xxx_state_tbl[0]);
+	tb.max_event = sizeof(xxx_event_tbl)/sizeof(xxx_event_tbl[0]);
+	tb.state_tbl = xxx_state_tbl;
+	tb.evt_tbl = xxx_event_tbl;
+	tb.priv = xxxpriv;
+	tb.init_priv = xxx_init_priv_cb;
+	tb.deinit_priv = xxx_deinit_priv_cb;
+	tb.dump_obj = xxx_dump_obj_cb;
+	tb.dump_fsm = xxx_dump_fsm_cb;
+	tb.dbg_level = FSM_LV_TRACE;
+	tb.evt_level = FSM_LV_TRACE;
+	tb.debug = xxx_debug;
+
+	fsm = rtw_fsm_register_fsm(root, "XXX", &tb);
+	xxxpriv->fsm = fsm;
+
+	if (!fsm)
+		return _FAIL;
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_RTW_FSM_XXX */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ft.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ft.c
new file mode 100644
index 000000000000..ac393158cf61
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ft.c
@@ -0,0 +1,951 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_80211R
+
+#ifndef RTW_FT_DBG
+	#define RTW_FT_DBG	0
+#endif
+#if RTW_FT_DBG
+	#define RTW_FT_INFO(fmt, arg...)	\
+		RTW_INFO(fmt, arg)
+	#define RTW_FT_DUMP(str, data, len)	\
+		RTW_INFO_DUMP(str, data, len)
+#else
+	#define RTW_FT_INFO(fmt, arg...) do {} while (0)
+	#define RTW_FT_DUMP(str, data, len) do {} while (0)
+#endif
+
+void rtw_ft_info_init(struct ft_roam_info *pft)
+{
+	_rtw_memset(pft, 0, sizeof(struct ft_roam_info));
+	pft->ft_flags = 0
+		| RTW_FT_EN
+		| RTW_FT_OTD_EN
+#ifdef CONFIG_RTW_BTM_ROAM
+		| RTW_FT_BTM_ROAM
+#endif
+		;
+	pft->ft_updated_bcn = _FALSE;
+	RTW_FT_INFO("%s : ft_flags=0x%02x\n", __func__, pft->ft_flags);
+}
+
+ssize_t rtw_ft_proc_flags_set(struct file *file,
+	const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	char tmp[32];
+	u8 flags;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhx", &flags);
+		if (num == 1)
+			adapter->mlmepriv.ft_roam.ft_flags = flags;
+	}
+
+	return count;
+
+}
+
+int rtw_ft_proc_flags_get(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "0x%02x\n", adapter->mlmepriv.ft_roam.ft_flags);
+
+	return 0;
+}
+
+u8 rtw_ft_chk_roaming_candidate(
+	_adapter *padapter, struct wlan_network *competitor)
+{
+	u8 *pmdie;
+	u32 mdie_len = 0;
+	struct ft_roam_info *pft_roam = &(padapter->mlmepriv.ft_roam);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (!(pmdie = rtw_get_ie(&competitor->network.IEs[12], _MDIE_,
+		&mdie_len, competitor->network.IELength-12))) {
+		RTW_INFO("FT : MDIE not foud in competitor!\n");
+		return _FALSE;
+	}
+
+	if (!_rtw_memcmp(&pft_roam->mdid, (pmdie+2), 2)) {
+		RTW_INFO("FT : unmatched MDIE!\n");
+		return _FALSE;
+	}
+
+	/*The candidate don't support over-the-DS*/
+	if (rtw_ft_valid_otd_candidate(padapter, pmdie)) {
+		RTW_INFO("FT: ignore the candidate("
+			MAC_FMT ") for over-the-DS\n",
+			MAC_ARG(competitor->network.MacAddress));
+		/* rtw_ft_clr_flags(padapter, RTW_FT_PEER_OTD_EN); */
+		return _FALSE;
+	}
+
+	if (rtw_ft_chk_flags(padapter, RTW_FT_TEST_RSSI_ROAM)) {
+		if (!_rtw_memcmp(
+			padapter_link->mlmepriv.cur_network.network.MacAddress,
+			competitor->network.MacAddress, ETH_ALEN)) {
+			competitor->network.PhyInfo.rssi +=20;
+			RTW_FT_INFO("%s : update "MAC_FMT
+				" RSSI to %d for RTW_FT_TEST_RSSI_ROAM\n",
+				__func__, MAC_ARG(competitor->network.MacAddress),
+				(int)competitor->network.PhyInfo.rssi);
+			rtw_ft_clr_flags(padapter, RTW_FT_TEST_RSSI_ROAM);
+		}
+	}
+
+	return _TRUE;
+}
+
+void rtw_ft_update_stainfo(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork)
+{
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct sta_info		*psta = NULL;
+	struct rtw_phl_mld_t	*pmld = NULL;
+
+	psta = rtw_get_stainfo(pstapriv, pnetwork->MacAddress);
+	if (psta == NULL) {
+		pmld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)),
+					padapter->phl_role, pnetwork->MacAddress, DTYPE);
+		/* main_id is don't care for self sta */
+		psta = rtw_alloc_stainfo(pstapriv, pnetwork->MacAddress, DTYPE, 0,
+					padapter_link->wrlink->id, PHL_CMD_DIRECTLY);
+	}
+
+	if (padapter->securitypriv.dot11AuthAlgrthm ==
+		dot11AuthAlgrthm_8021X) {
+		padapter_link->securitypriv.binstallGrpkey = _FALSE;
+		padapter_link->securitypriv.bgrpkey_handshake = _FALSE;
+		padapter->securitypriv.busetkipkey = _FALSE;
+
+		psta->ieee8021x_blocked = _TRUE;
+		psta->dot118021XPrivacy = \
+			padapter->securitypriv.dot11PrivacyAlgrthm;
+		_rtw_memset((u8 *)&psta->dot118021x_UncstKey,
+				0, sizeof(union Keytype));
+		_rtw_memset((u8 *)&psta->dot11tkiprxmickey,
+				0, sizeof(union Keytype));
+		_rtw_memset((u8 *)&psta->dot11tkiptxmickey,
+				0, sizeof(union Keytype));
+	}
+
+}
+
+void rtw_ft_reassoc_event_callback(_adapter *padapter, u8 *pbuf)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct stassoc_event *pstassoc = (struct stassoc_event *)pbuf;
+	struct ft_roam_info *pft_roam = &(pmlmepriv->ft_roam);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *pnetwork = (WLAN_BSSID_EX *)&(pmlmeinfo->network);
+	struct cfg80211_ft_event_params ft_evt_parms;
+
+	_rtw_memset(&ft_evt_parms, 0, sizeof(ft_evt_parms));
+	rtw_ft_update_stainfo(padapter, padapter_link, pnetwork);
+	ft_evt_parms.ies_len = pft_roam->ft_event.ies_len;
+	ft_evt_parms.ies =  rtw_zmalloc(ft_evt_parms.ies_len);
+	if (ft_evt_parms.ies) {
+		_rtw_memcpy((void *)ft_evt_parms.ies,
+			pft_roam->ft_event.ies, ft_evt_parms.ies_len);
+	 } else
+		goto err_2;
+
+	ft_evt_parms.target_ap = rtw_zmalloc(ETH_ALEN);
+	if (ft_evt_parms.target_ap) {
+		_rtw_memcpy((void *)ft_evt_parms.target_ap,
+			pstassoc->macaddr, ETH_ALEN);
+	} else
+		goto err_1;
+
+	ft_evt_parms.ric_ies = pft_roam->ft_event.ric_ies;
+	ft_evt_parms.ric_ies_len = pft_roam->ft_event.ric_ies_len;
+
+	/* It's a KERNEL issue between v4.11 ~ v4.16,
+	* <= v4.10, NLMSG_DEFAULT_SIZE is used for nlmsg_new().
+	* v4.11 ~ v4.16, only used "100 + >ric_ies_len" for nlmsg_new()
+	*	even then DRIVER don't support RIC.
+	* >= v4.17, issue should correct as "100 + ies_len + ric_ies_len".
+	*/
+	#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) && \
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)))
+		if (!ft_evt_parms.ric_ies_len)
+			ft_evt_parms.ric_ies_len = ft_evt_parms.ies_len;
+		else
+			ft_evt_parms.ric_ies_len += ft_evt_parms.ies_len;
+	#endif
+
+	rtw_ft_lock_set_status(padapter, RTW_FT_AUTHENTICATED_STA);
+	rtw_cfg80211_ft_event(padapter, &ft_evt_parms);
+	RTW_INFO("%s: to "MAC_FMT"\n", __func__,
+			MAC_ARG(ft_evt_parms.target_ap));
+
+	rtw_mfree((u8 *)pft_roam->ft_event.target_ap, ETH_ALEN);
+err_1:
+	rtw_mfree((u8 *)ft_evt_parms.ies, ft_evt_parms.ies_len);
+err_2:
+	return;
+}
+
+void rtw_ft_validate_akm_type(_adapter  *padapter,
+	WLAN_BSSID_EX *network)
+{
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	struct ft_roam_info *pft_roam = &(padapter->mlmepriv.ft_roam);
+	u32 tmp_len;
+	u8 *ptmp;
+
+	/*IEEE802.11-2012 Std. Table 8-101-AKM suite selectors*/
+	if (rtw_ft_valid_akm(padapter, psecuritypriv->rsn_akm_suite_type)) {
+		ptmp = rtw_get_ie(&network->IEs[12],
+				_MDIE_, &tmp_len,
+				(network->IELength-12));
+		if (ptmp) {
+			pft_roam->mdid = *(u16 *)(ptmp+2);
+			pft_roam->ft_cap = *(ptmp+4);
+
+			RTW_INFO("FT: target "MAC_FMT
+				" mdid=(0x%2x), capacity=(0x%2x)\n",
+				MAC_ARG(network->MacAddress),
+				pft_roam->mdid, pft_roam->ft_cap);
+
+			rtw_ft_set_flags(padapter, RTW_FT_PEER_EN);
+
+			RTW_FT_INFO("%s : peer support FTOTA(0x%02x)\n",
+				__func__, pft_roam->ft_flags);
+
+			if (rtw_ft_otd_roam_en(padapter)) {
+				rtw_ft_set_flags(padapter, RTW_FT_PEER_OTD_EN);
+				RTW_FT_INFO("%s : peer support FTOTD(0x%02x)\n",
+					__func__, pft_roam->ft_flags);
+			} else {
+				pft_roam->ft_cap &= ~BIT(0);
+				rtw_ft_clr_flags(padapter, RTW_FT_PEER_OTD_EN);
+				RTW_FT_INFO("%s : update mdid=0x%02x, "
+					"cap=0x%02x, flag=0x%02x\n",
+					__func__, pft_roam->mdid,
+					pft_roam->ft_cap,
+					pft_roam->ft_flags);
+			}
+		} else {
+			/* Don't use FT roaming if target AP cannot support FT */
+			rtw_ft_clr_flags(padapter,
+				(RTW_FT_PEER_EN|RTW_FT_PEER_OTD_EN));
+			rtw_ft_reset_status(padapter);
+		}
+	} else {
+		/* It could be a non-FT connection */
+		rtw_ft_clr_flags(padapter,
+				(RTW_FT_PEER_EN|RTW_FT_PEER_OTD_EN));
+		rtw_ft_reset_status(padapter);
+	}
+
+	RTW_FT_INFO("%s : ft_flags=0x%02x\n", __func__, pft_roam->ft_flags);
+}
+
+void rtw_ft_update_bcn(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint len = precv_frame->u.hdr.len;
+	WLAN_BSSID_EX *pbss;
+
+	if (rtw_ft_chk_status(padapter,RTW_FT_ASSOCIATED_STA)
+		&& (pmlmepriv->ft_roam.ft_updated_bcn == _FALSE)) {
+		pbss = (WLAN_BSSID_EX*)rtw_malloc(sizeof(WLAN_BSSID_EX));
+		if (pbss) {
+			if (collect_bss_info(
+				padapter, precv_frame, pbss) == _SUCCESS) {
+				struct beacon_keys recv_beacon;
+
+				/* Update adapter network info */
+				rtw_update_adapter_network(
+					&(pmlmepriv->dev_cur_network.network),
+					pbss, padapter, _TRUE);
+
+				rtw_update_network(
+					&(padapter_link->mlmepriv.cur_network.network),
+					pbss, padapter, _TRUE);
+
+				/* Move into rtw_get_bcn_keys */
+				/* rtw_get_bcn_info(&(pmlmepriv->cur_network)); */
+
+				/* update bcn keys */
+				if (rtw_get_bcn_keys(padapter, padapter_link, pframe, len,
+						&recv_beacon) == _TRUE) {
+
+					RTW_FT_INFO("%s: beacon keys ready\n",
+						__func__);
+
+					_rtw_memcpy(
+						&padapter_link->mlmepriv.cur_beacon_keys,
+						&recv_beacon,
+						sizeof(recv_beacon));
+
+					if (is_hidden_ssid(
+						recv_beacon.ssid,
+						recv_beacon.ssid_len)) {
+
+						_rtw_memcpy(
+						padapter_link->mlmepriv.cur_beacon_keys.ssid,
+						pmlmeinfo->network.Ssid.Ssid,
+						IW_ESSID_MAX_SIZE);
+
+						padapter_link->mlmepriv.cur_beacon_keys.ssid_len = \
+						pmlmeinfo->network.Ssid.SsidLength;
+					}
+				} else {
+					RTW_ERR("%s: get beacon keys failed\n",
+						__func__);
+					_rtw_memset(
+						&padapter_link->mlmepriv.cur_beacon_keys,
+						0, sizeof(recv_beacon));
+				}
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+				padapter_link->mlmepriv.new_beacon_cnts = 0;
+#endif
+			}
+			rtw_mfree((u8*)pbss, sizeof(WLAN_BSSID_EX));
+		}
+
+		/* check the vendor of the assoc AP */
+		padapter->mlmeextpriv.mlmext_info.assoc_AP_vendor =
+			check_assoc_AP(
+				pframe + sizeof(struct rtw_ieee80211_hdr_3addr),
+				(len - sizeof(struct rtw_ieee80211_hdr_3addr))
+				);
+
+		/* update TSF Value */
+		update_TSF(pmlmeext, pframe, len);
+		pmlmeext->bcn_cnt = 0;
+		pmlmeext->last_bcn_cnt = 0;
+		pmlmepriv->ft_roam.ft_updated_bcn = _TRUE;
+	}
+}
+
+void rtw_ft_start_clnt_join(_adapter *padapter)
+{
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct	mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct ft_roam_info *pft_roam = &(pmlmepriv->ft_roam);
+
+	if (rtw_ft_otd_roam(padapter)) {
+		pmlmeinfo->state = WIFI_FW_AUTH_SUCCESS | WIFI_FW_STATION_STATE;
+		pft_roam->ft_event.ies =
+			(pft_roam->ft_action + \
+			sizeof(struct rtw_ieee80211_hdr_3addr) + 16);
+		pft_roam->ft_event.ies_len =
+			(pft_roam->ft_action_len - \
+			sizeof(struct rtw_ieee80211_hdr_3addr));
+
+		/*Not support RIC*/
+		pft_roam->ft_event.ric_ies =  NULL;
+		pft_roam->ft_event.ric_ies_len = 0;
+		rtw_ft_report_evt(padapter);
+		return;
+	}
+
+	pmlmeinfo->state = WIFI_FW_AUTH_NULL | WIFI_FW_STATION_STATE;
+	start_clnt_auth(padapter);
+}
+
+u8 rtw_ft_update_rsnie(
+	_adapter *padapter, u8 bwrite,
+	struct pkt_attrib *pattrib, u8 **pframe)
+{
+	struct ft_roam_info *pft_roam = &(padapter->mlmepriv.ft_roam);
+	u8 *pie;
+	u32 len;
+
+	pie = rtw_get_ie(pft_roam->updated_ft_ies, EID_WPA2, &len,
+			pft_roam->updated_ft_ies_len);
+
+	if (!bwrite)
+		return (pie)?_SUCCESS:_FAIL;
+
+	if (pie) {
+		*pframe = rtw_set_ie(((u8 *)*pframe), EID_WPA2, len,
+				pie+2, &(pattrib->pktlen));
+	} else
+		return _FAIL;
+
+	return _SUCCESS;
+}
+
+static u8 rtw_ft_update_mdie(
+	_adapter *padapter, struct pkt_attrib *pattrib, u8 **pframe)
+{
+	struct ft_roam_info *pft_roam = &(padapter->mlmepriv.ft_roam);
+	u8 *pie, mdie[3];
+	u32 len = 3;
+
+	if (rtw_ft_roam(padapter)) {
+		if ((pie = rtw_get_ie(pft_roam->updated_ft_ies, _MDIE_,
+				&len, pft_roam->updated_ft_ies_len))) {
+			pie = (pie + 2); /* ignore md-id & length */
+		} else
+			return _FAIL;
+	} else {
+		*((u16 *)&mdie[0]) = pft_roam->mdid;
+		mdie[2] = pft_roam->ft_cap;
+		pie = &mdie[0];
+	}
+
+	*pframe = rtw_set_ie(((u8 *)*pframe), _MDIE_,
+			len, pie, &(pattrib->pktlen));
+	return _SUCCESS;
+}
+
+static u8 rtw_ft_update_ftie(
+	_adapter *padapter, struct pkt_attrib *pattrib, u8 **pframe)
+{
+	struct ft_roam_info *pft_roam = &(padapter->mlmepriv.ft_roam);
+	u8 *pie;
+	u32 len;
+
+	if ((pie = rtw_get_ie(pft_roam->updated_ft_ies, _FTIE_, &len,
+				pft_roam->updated_ft_ies_len)) != NULL) {
+		*pframe = rtw_set_ie(*pframe, _FTIE_, len,
+				(pie+2), &(pattrib->pktlen));
+	} else
+		return _FAIL;
+
+	return _SUCCESS;
+}
+
+void rtw_ft_build_auth_req_ies(_adapter *padapter,
+	struct pkt_attrib *pattrib, u8 **pframe)
+{
+	u8 ftie_append = _TRUE;
+
+	if (!pattrib || !(*pframe))
+		return;
+
+	if (!rtw_ft_roam(padapter))
+		return;
+
+	ftie_append = rtw_ft_update_rsnie(padapter, _TRUE, pattrib, pframe);
+	rtw_ft_update_mdie(padapter, pattrib, pframe);
+	if (ftie_append)
+		rtw_ft_update_ftie(padapter, pattrib, pframe);
+}
+
+void rtw_ft_build_assoc_req_ies(_adapter *padapter,
+	u8 is_reassoc, struct pkt_attrib *pattrib, u8 **pframe)
+{
+	if (!pattrib || !(*pframe))
+		return;
+
+	if (rtw_ft_chk_flags(padapter, RTW_FT_PEER_EN))
+		rtw_ft_update_mdie(padapter, pattrib, pframe);
+
+	if ((!is_reassoc) || (!rtw_ft_roam(padapter)))
+		return;
+
+	if (rtw_ft_update_rsnie(padapter, _FALSE, pattrib, pframe))
+		rtw_ft_update_ftie(padapter, pattrib, pframe);
+}
+
+u8 rtw_ft_update_auth_rsp_ies(_adapter *padapter, u8 *pframe, u32 len)
+{
+	u8 ret = _SUCCESS;
+	u8 target_ap_addr[ETH_ALEN] = {0};
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct ft_roam_info *pft_roam = &(pmlmepriv->ft_roam);
+
+	if (!rtw_ft_roam(padapter))
+		return _FAIL;
+
+	/*rtw_ft_report_reassoc_evt already,
+	 * and waiting for cfg80211_rtw_update_ft_ies */
+	if (rtw_ft_authed_sta(padapter))
+		return ret;
+
+	if (!pframe || !len)
+		return _FAIL;
+
+	rtw_buf_update(&pmlmepriv->auth_rsp,
+		&pmlmepriv->auth_rsp_len, pframe, len);
+	pft_roam->ft_event.ies =
+		(pmlmepriv->auth_rsp + \
+		sizeof(struct rtw_ieee80211_hdr_3addr) + 6);
+	pft_roam->ft_event.ies_len =
+		(pmlmepriv->auth_rsp_len - \
+		sizeof(struct rtw_ieee80211_hdr_3addr) - 6);
+
+	/*Not support RIC*/
+	pft_roam->ft_event.ric_ies =  NULL;
+	pft_roam->ft_event.ric_ies_len =  0;
+	_rtw_memcpy(target_ap_addr, pmlmepriv->assoc_bssid, ETH_ALEN);
+	rtw_ft_report_reassoc_evt(padapter, target_ap_addr);
+
+	return ret;
+}
+
+void rtw_ft_start_clnt_action(_adapter *padapter, u8 *pTargetAddr)
+{
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+
+	rtw_ft_set_status(padapter, RTW_FT_REQUESTING_STA);
+	rtw_ft_issue_action_req(padapter, pTargetAddr);
+	_set_timer(&pmlmeext->ft_link_timer, REASSOC_TO);
+}
+
+void rtw_ft_start_roam(_adapter *padapter, u8 *pTargetAddr)
+{
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+
+	if (rtw_ft_otd_roam(padapter)) {
+		RTW_FT_INFO("%s : try OTD roaming\n", __func__);
+		rtw_ft_start_clnt_action(padapter, pTargetAddr);
+	} else {
+		/* wait a little time to retrieve packets buffered
+			in the current ap while scan*/
+		RTW_FT_INFO("%s : start roaming timer\n", __func__);
+		_set_timer(&pmlmeext->ft_roam_timer, 30);
+	}
+}
+
+void rtw_ft_issue_action_req(_adapter *padapter, u8 *pTargetAddr)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct xmit_frame *pmgntframe;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct pkt_attrib *pattrib;
+	u8 *pframe;
+	u8 category = RTW_WLAN_CATEGORY_FT;
+	u8 action = RTW_WLAN_ACTION_FT_REQ;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	pwlanhdr->frame_ctl = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, padapter->mlmeextpriv.mgnt_seq);
+	padapter->mlmeextpriv.mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	/* ToDo CONFIG_RTW_MLD: MLD address or link address */
+	_rtw_memcpy(pframe, adapter_mac_addr(padapter), ETH_ALEN);
+	pframe += ETH_ALEN;
+	pattrib->pktlen += ETH_ALEN;
+
+	_rtw_memcpy(pframe, pTargetAddr, ETH_ALEN);
+	pframe += ETH_ALEN;
+	pattrib->pktlen += ETH_ALEN;
+
+	rtw_ft_update_mdie(padapter, pattrib, &pframe);
+	if (rtw_ft_update_rsnie(padapter, _TRUE, pattrib, &pframe))
+		rtw_ft_update_ftie(padapter, pattrib, &pframe);
+
+	RTW_INFO("FT : issue RTW_WLAN_ACTION_FT_REQ\n");
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+}
+
+void rtw_ft_report_evt(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct ft_roam_info *pft_roam = &(pmlmepriv->ft_roam);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *pnetwork = (WLAN_BSSID_EX *)&(pmlmeinfo->network);
+	struct cfg80211_ft_event_params ft_evt_parms;
+
+	_rtw_memset(&ft_evt_parms, 0, sizeof(ft_evt_parms));
+	rtw_ft_update_stainfo(padapter, padapter_link, pnetwork);
+
+	if (!pnetwork)
+		goto err_2;
+
+	ft_evt_parms.ies_len = pft_roam->ft_event.ies_len;
+	ft_evt_parms.ies =  rtw_zmalloc(ft_evt_parms.ies_len);
+	if (ft_evt_parms.ies) {
+		_rtw_memcpy((void *)ft_evt_parms.ies,
+			pft_roam->ft_event.ies, ft_evt_parms.ies_len);
+	 } else
+		goto err_2;
+
+	ft_evt_parms.target_ap = rtw_zmalloc(ETH_ALEN);
+	if (ft_evt_parms.target_ap) {
+		_rtw_memcpy((void *)ft_evt_parms.target_ap,
+			pnetwork->MacAddress, ETH_ALEN);
+	} else
+		goto err_1;
+
+	ft_evt_parms.ric_ies = pft_roam->ft_event.ric_ies;
+	ft_evt_parms.ric_ies_len = pft_roam->ft_event.ric_ies_len;
+
+	/* It's a KERNEL issue between v4.11 ~ v4.16,
+	* <= v4.10, NLMSG_DEFAULT_SIZE is used for nlmsg_new().
+	* v4.11 ~ v4.16, only used "100 + >ric_ies_len" for nlmsg_new()
+	*	even then DRIVER don't support RIC.
+	* >= v4.17, issue should correct as "100 + ies_len + ric_ies_len".
+	*/
+	#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) && \
+	(LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0)))
+		ft_evt_parms.ric_ies_len = (ft_evt_parms.ies_len <= 100 )?
+			(0):(ft_evt_parms.ies_len - 100);
+	#endif
+
+	rtw_ft_lock_set_status(padapter, RTW_FT_AUTHENTICATED_STA);
+	rtw_cfg80211_ft_event(padapter, &ft_evt_parms);
+	RTW_INFO("FT: rtw_ft_report_evt\n");
+	rtw_mfree((u8 *)pft_roam->ft_event.target_ap, ETH_ALEN);
+err_1:
+	rtw_mfree((u8 *)ft_evt_parms.ies, ft_evt_parms.ies_len);
+err_2:
+	return;
+}
+
+void rtw_ft_report_reassoc_evt(_adapter *padapter, u8 *pMacAddr)
+{
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct cmd_priv *pcmdpriv = &(adapter_to_dvobj(padapter)->cmdpriv);
+	struct cmd_obj *pcmd_obj = NULL;
+	struct stassoc_event *passoc_sta_evt = NULL;
+	struct rtw_evt_header *evt_hdr = NULL;
+	u8 *pevtcmd = NULL;
+	u32 cmdsz = 0;
+
+	pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd_obj == NULL)
+		return;
+
+	cmdsz = (sizeof(struct stassoc_event) + sizeof(struct rtw_evt_header));
+	pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+	if (pevtcmd == NULL) {
+		rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+		return;
+	}
+
+	_rtw_init_listhead(&pcmd_obj->list);
+	pcmd_obj->padapter = padapter;
+	pcmd_obj->band_idx = HW_BAND_MAX;
+	pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+	pcmd_obj->cmdsz = cmdsz;
+	pcmd_obj->parmbuf = pevtcmd;
+	pcmd_obj->rsp = NULL;
+	pcmd_obj->rspsz  = 0;
+
+	evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+	evt_hdr->len = sizeof(struct stassoc_event);
+	evt_hdr->id = EVT_FT_REASSOC;
+	evt_hdr->seq = ATOMIC_INC_RETURN(&pmlmeext->event_seq);
+
+	passoc_sta_evt = \
+			(struct stassoc_event *)(pevtcmd + \
+			sizeof(struct rtw_evt_header));
+	_rtw_memcpy((unsigned char *)(&(passoc_sta_evt->macaddr)),
+			pMacAddr, ETH_ALEN);
+	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+}
+
+void rtw_ft_link_timer_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct ft_roam_info *pft_roam = &(pmlmepriv->ft_roam);
+
+	if (rtw_ft_chk_status(padapter, RTW_FT_REQUESTING_STA)) {
+		if (pft_roam->ft_req_retry_cnt < RTW_FT_ACTION_REQ_LMT) {
+			pft_roam->ft_req_retry_cnt++;
+			rtw_ft_issue_action_req(padapter,
+			(u8 *)pmlmepriv->roam_network->network.MacAddress);
+			_set_timer(&pmlmeext->ft_link_timer, REASSOC_TO);
+		} else {
+			pft_roam->ft_req_retry_cnt = 0;
+			if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
+				rtw_ft_set_status(padapter,
+					RTW_FT_ASSOCIATED_STA);
+			} else
+				rtw_ft_reset_status(padapter);
+		}
+	}
+}
+
+void rtw_ft_roam_timer_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	RTW_FT_INFO("%s : try roaming\n", __func__);
+	receive_disconnect(padapter,
+			pmlmepriv->cur_network.network.MacAddress,
+			WLAN_REASON_ACTIVE_ROAM, _FALSE);
+	pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+	pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_FT;
+	pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+}
+
+void rtw_ft_roam_status_reset(_adapter *padapter)
+{
+	struct ft_roam_info *pft_roam = &(padapter->mlmepriv.ft_roam);
+
+	if ((rtw_to_roam(padapter) > 0) &&
+		(!rtw_ft_chk_status(padapter, RTW_FT_REQUESTED_STA))) {
+		rtw_ft_reset_status(padapter);
+	}
+
+	padapter->mlmepriv.ft_roam.ft_updated_bcn = _FALSE;
+}
+
+void rtw_ft_peer_info_init(struct sta_info *psta)
+{
+	struct rtw_sta_ft_info_t *peer = &(psta->ft_peer);
+	_rtw_memset(peer, 0, sizeof(struct rtw_sta_ft_info_t));
+}
+
+void rtw_ft_peer_info_free(struct sta_info *psta)
+{
+	struct rtw_sta_ft_info_t *peer = &(psta->ft_peer);
+
+	rtw_buf_free(&peer->md_ie, &peer->md_len);
+	rtw_buf_free(&peer->rsn_ie, &peer->rsn_len);
+	rtw_buf_free(&peer->ft_ie, &peer->ft_len);
+}
+
+int rtw_ft_update_sta_ies(_adapter *padapter,
+	struct cfg80211_update_ft_ies_params *ie)
+{
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	struct sta_priv *pstapriv = &(padapter->stapriv);
+	struct sta_info *psta;
+	struct rtw_sta_ft_info_t *peer_info;
+	u8 *ptr, *paddr;
+	u32 len;
+	int ret = 0;
+
+	if (!CHECK_BIT(WLAN_AKM_TYPE_FT_PSK, psecuritypriv->akmp)
+		&& !CHECK_BIT(WLAN_AKM_TYPE_FT_8021X, psecuritypriv->akmp)
+		&& !CHECK_BIT(WLAN_AKM_TYPE_FT_OVER_SAE, psecuritypriv->akmp)) {
+		RTW_FT_INFO("%s : AKM=0x%x\n", __func__, psecuritypriv->akmp);
+		goto exit;
+	}
+
+	if ((ie->ie_len == 0) ||
+		(ie->ie_len >= RTW_FT_MAX_IE_SZ)) {
+		ret = - EINVAL;
+		goto exit;
+	}
+
+	if ((ptr = rtw_get_ie(ie->ie, WLAN_EID_VENDOR_SPECIFIC,
+			&len , ie->ie_len)) == NULL) {
+		RTW_ERR("FT : no station mac address found in ies\n");
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (len != ETH_ALEN) {
+		RTW_ERR("FT : invalid station mac length(%u)\n", len);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	paddr = ptr + 2;
+	if ((psta = rtw_get_stainfo(pstapriv, paddr)) == NULL) {
+		RTW_ERR("FT : sta "MAC_FMT" not found!\n", MAC_ARG(paddr));
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	RTW_FT_INFO("%s : update sta "MAC_FMT" ie, len=%lu\n",
+			__func__, MAC_ARG(paddr), ie->ie_len);
+	RTW_FT_DUMP("ie = ", ie->ie, ie->ie_len);
+
+	peer_info = &(psta->ft_peer);
+	if ((ptr = rtw_get_ie(ie->ie, EID_WPA2, &len , ie->ie_len)) != NULL) {
+		rtw_buf_update(&peer_info->rsn_ie,
+			&peer_info->rsn_len, ptr + 2, len);
+		RTW_FT_DUMP("rsn_ie = ", peer_info->rsn_ie,
+			peer_info->rsn_len);
+	}
+
+	if ((ptr = rtw_get_ie(ie->ie, _MDIE_, &len , ie->ie_len)) != NULL) {
+		rtw_buf_update(&peer_info->md_ie,
+			&peer_info->md_len, ptr + 2, len);
+		RTW_FT_DUMP("md_ie = ",
+			peer_info->md_ie, peer_info->md_len);
+	}
+
+	if ((ptr = rtw_get_ie(ie->ie, _FTIE_, &len , ie->ie_len)) != NULL) {
+		rtw_buf_update(&peer_info->ft_ie,
+			&peer_info->ft_len, ptr + 2, len);
+		RTW_FT_DUMP("ft_ie = ",
+			peer_info->ft_ie, peer_info->ft_len);
+	}
+
+	if (psta->authalg == WLAN_AUTH_FT)
+		issue_asocrsp(padapter, 0, psta, WIFI_REASSOCRSP);
+exit:
+	return ret;
+}
+
+void rtw_ft_update_assocresp_ies(struct net_device *net,
+	struct cfg80211_ap_settings *settings)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct ft_roam_info *pinfo = &(pmlmepriv->ft_roam);
+	u8 *ie, *bcn_tail_ies, *assocrsp_ies;
+	int ielen, bcn_tail_len, assocrsp_ies_len, ret = 0;
+
+	bcn_tail_ies = (u8 *)settings->beacon.tail;
+	bcn_tail_len = (u32)settings->beacon.tail_len;
+	assocrsp_ies = (u8 *)settings->beacon.assocresp_ies;
+	assocrsp_ies_len = (u32)settings->beacon.assocresp_ies_len;
+
+	if ((bcn_tail_len > 0) &&
+		((ie = rtw_get_ie(bcn_tail_ies, _MDIE_,
+			&ielen , bcn_tail_len)) != NULL)) {
+		pinfo->mdid = *(u16 *)(ie + 2);
+		pinfo->ft_cap = *(ie + 4);
+		RTW_FT_INFO("%s : mdid = 0x%04x, cap=0x%02x\n",
+			__func__, pinfo->mdid, pinfo->ft_cap);
+	}
+
+	if ((assocrsp_ies_len > 0) &&
+		((ie = rtw_get_ie(assocrsp_ies, _FTIE_,
+			&ielen , assocrsp_ies_len)) != NULL)) {
+			_rtw_memset(pinfo->updated_ft_ies, 0, RTW_FT_MAX_IE_SZ);
+			_rtw_memcpy(pinfo->updated_ft_ies, ie + 2, ielen);
+			pinfo->updated_ft_ies_len  = ielen;
+			RTW_FT_DUMP("FT : ft ie = ", ie + 2, ielen);
+	}
+
+}
+
+void rtw_ft_process_ft_auth_rsp(_adapter *padapter, u8 *pframe, u32 len)
+{
+	struct sta_priv *pstapriv = &(padapter->stapriv);
+	struct sta_info *psta = NULL;
+	u8 *ptr;
+	u32 plen;
+
+	ptr = pframe + IEEE80211_3ADDR_LEN + _AUTH_IE_OFFSET_;
+	plen = len - IEEE80211_3ADDR_LEN - _AUTH_IE_OFFSET_;
+
+	psta = rtw_get_stainfo(pstapriv, GetAddr1Ptr(pframe));
+	if (psta) {
+		_rtw_spinlock_bh(&psta->lock);
+		psta->state &= ~WIFI_FW_AUTH_NULL;
+		psta->state |= WIFI_FW_AUTH_SUCCESS;
+		psta->expire_to = pstapriv->assoc_to;
+		_rtw_spinunlock_bh(&psta->lock);
+	}
+}
+
+void rtw_ft_build_assoc_rsp_ies(_adapter *padapter,
+	struct sta_info *psta, struct pkt_attrib *pattrib, u8 **pframe)
+{
+	struct ft_roam_info *prinfo = &(padapter->mlmepriv.ft_roam);
+	struct rtw_sta_ft_info_t *peer_info;
+	u8 *pie, mdie[3] = {0};
+	u32 ielen;
+
+	if (psta == NULL)
+		return;
+
+	if (psta->authalg != WLAN_AUTH_FT) {
+		*((u16 *)&mdie[0]) = prinfo->mdid;
+		mdie[2] = prinfo->ft_cap;
+		pie = mdie;
+		ielen = 3;
+
+		if (prinfo->mdid > 0) {
+		*pframe = rtw_set_ie(((u8 *)*pframe),
+			_MDIE_, ielen , pie, &(pattrib->pktlen));
+		}
+
+		if (prinfo->updated_ft_ies_len > 0) {
+		*pframe = rtw_set_ie(((u8 *)*pframe),
+			_FTIE_, prinfo->updated_ft_ies_len,
+			prinfo->updated_ft_ies, &(pattrib->pktlen));
+		}
+	} else {
+		peer_info = &(psta->ft_peer);
+		if (peer_info->rsn_len > 0) {
+			*pframe = rtw_set_ie(((u8 *)*pframe), EID_WPA2,
+					peer_info->rsn_len, peer_info->rsn_ie,
+					&(pattrib->pktlen));
+		}
+
+		if (peer_info->md_len > 0) {
+			*pframe = rtw_set_ie(((u8 *)*pframe), _MDIE_,
+					peer_info->md_len, peer_info->md_ie,
+					&(pattrib->pktlen));
+		}
+
+		if (peer_info->ft_len > 0) {
+			*pframe = rtw_set_ie(((u8 *)*pframe), _FTIE_,
+					peer_info->ft_len, peer_info->ft_ie,
+					&(pattrib->pktlen));
+		}
+	}
+
+}
+#endif /* CONFIG_RTW_80211R */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_he.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_he.c
new file mode 100644
index 000000000000..e1671eceff50
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_he.c
@@ -0,0 +1,1971 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_HE_C
+
+#include <drv_types.h>
+
+#ifdef CONFIG_80211AX_HE
+
+/* for now cover BW 20/40/80/160 bounded in 2SS */
+const u16 HE_MCS_DATA_RATE[CHANNEL_WIDTH_MAX][MAX_HE_GI_TYPE][MAX_HE_MCS_INDEX] = {
+	{	/* 20M */
+		{	/* 3.2us */
+			14, 29, 43, 58, 87, 117, 131, 146, 175, 195, 219, 243,
+			29, 58, 87, 117, 175, 234, 263, 292, 351, 390, 438, 487,
+		},
+		{	/* 1.6us */
+			16, 32, 48, 65, 97, 130, 146, 162, 195, 216, 243, 270,
+			32, 65, 97, 130, 195, 260, 292, 325, 390, 433, 487, 541,
+		},
+		{	/* 0.8us */
+			17, 34, 51, 68, 103, 137, 154, 172, 206, 229, 258, 286,
+			34, 68, 103, 137, 206, 275, 309, 344, 413, 458, 516, 573,
+		}
+	},
+	{	/* 40M */
+		{	/* 3.2us */
+			29, 58, 87, 117, 175, 234, 263, 292, 351, 390, 438, 487,
+			58, 117, 175, 234, 351, 468, 526, 585, 702, 780, 877, 975,
+		},
+		{	/* 1.6us */
+			32, 65, 97, 130, 195, 260, 292, 325, 390, 433, 487, 541,
+			65, 130, 195, 260, 390, 520, 585, 650, 780, 866, 975, 1083,
+		},
+		{	/* 0.8us */
+			34, 68, 103, 138, 206, 275, 309, 344, 413, 458, 516, 573,
+			68, 137, 206, 275, 413, 550, 619, 688, 825, 917, 1032, 1147,
+		}
+	},
+	{	/* 80M */
+		{	/* 3.2us */
+			61, 122, 183, 245, 367, 490, 551, 612, 735, 816, 918, 1020,
+			122, 245, 367, 490, 735, 980, 1102, 1225, 1470, 1633, 1839, 2041,
+		},
+		{	/* 1.6us */
+			68, 136, 204, 272, 408, 544, 612, 680, 816, 907, 1020, 1134,
+			136, 272, 408, 544, 816, 1088, 1225, 1361, 1633, 1814, 2041, 2268,
+		},
+		{	/* 0.8us */
+			72, 144, 216, 288, 432, 576, 648, 720, 864, 960, 1080, 1200,
+			144, 288, 432, 576, 864, 1153, 1297, 1441, 1729, 1921, 2161, 2402,
+		}
+	},
+	{	/* 160M */
+		{	/* 3.2us */
+			123, 245, 368, 490, 735, 980, 1103, 1225, 1470, 1633, 1838, 2042,
+			245, 490, 735, 980, 1470, 1960, 2205, 2450, 2940, 3267, 3675, 4083,
+		},
+		{	/* 1.6us */
+			136, 272, 408, 544, 817, 1089, 1225, 1361, 1633, 1815, 2042, 2269,
+			272, 544, 817, 1089, 1633, 2178, 2450, 2722, 3267, 3630, 4083, 4537,
+		},
+		{	/* 0.8us */
+			144, 288, 432, 576, 865, 1153, 1297, 1441, 1729, 1922, 2162, 2402,
+			288, 576, 865, 1153, 1729, 2306, 2594, 2882, 3459, 3843, 4324, 4804,
+		}
+	}
+};
+
+u16 rtw_he_mcs_to_data_rate(u8 bw, u8 guard_int, u8 he_mcs_rate)
+{
+	u8 gi = 2; /* use 0.8us GI since 2XLTF_0.8us GI is mandatory in HE */
+	u8 mcs_idx = he_mcs_rate - MGN_HE1SS_MCS0;
+
+	return HE_MCS_DATA_RATE[bw][gi][mcs_idx];
+}
+
+u8 rtw_he_get_highest_rate(u8 *he_mcs_map)
+{
+	u8 i, j;
+	u8 bit_map;
+	u8 he_mcs_rate = 0;
+
+	/* currently only consider the BW 80M */
+	for (i = 0; i < 2; i++) {
+		if (he_mcs_map[i] != 0xff) {
+			/* max to 4SS, each SS contains 2 bit */
+			for (j = 0; j < 8; j += 2) {
+				bit_map = (he_mcs_map[i] >> j) & 3;
+
+				if (bit_map != 3)
+					he_mcs_rate = MGN_HE1SS_MCS7 + 12 * j / 2 + i * 48 + 2 * bit_map;
+			}
+		}
+	}
+
+	/*RTW_INFO("####### HighestHEMCSRate is %x\n", he_mcs_rate);*/
+	return he_mcs_rate;
+}
+
+void	rtw_he_use_default_setting(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct protocol_cap_t *role_cap = &(padapter_link->wrlink->protocol_cap);
+	struct he_priv *phepriv = &pmlmepriv->hepriv;
+
+	phepriv->he_highest_rate = rtw_he_get_highest_rate(role_cap->he_rx_mcs);
+}
+
+void rtw_he_set_asoc_cap_supp_mcs(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 supp_mcs_len)
+{
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct protocol_cap_t *role_cap = &(phl_sta->rlink->protocol_cap);
+	int nss = 0, nss_tx = 0, nss_rx = 0;
+	u8 mcs_from_role = HE_MSC_NOT_SUPP;
+	u8 mcs_from_ie = HE_MSC_NOT_SUPP;
+	u8 mcs_val_rx = HE_MSC_NOT_SUPP;
+	u8 mcs_val_tx = HE_MSC_NOT_SUPP;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+
+	_rtw_memset(phl_sta->asoc_cap.he_rx_mcs, HE_MSC_NOT_SUPP_BYTE, HE_CAP_ELE_SUPP_MCS_LEN_RX_80M);
+	_rtw_memset(phl_sta->asoc_cap.he_tx_mcs, HE_MSC_NOT_SUPP_BYTE, HE_CAP_ELE_SUPP_MCS_LEN_TX_80M);
+	_rtw_memset(phl_sta->asoc_cap.he_rx_mcs + 2, HE_MSC_NOT_SUPP_BYTE, HE_CAP_ELE_SUPP_MCS_LEN_RX_160M);
+	_rtw_memset(phl_sta->asoc_cap.he_tx_mcs + 2, HE_MSC_NOT_SUPP_BYTE, HE_CAP_ELE_SUPP_MCS_LEN_TX_160M);
+	_rtw_memset(phl_sta->asoc_cap.he_rx_mcs + 4, HE_MSC_NOT_SUPP_BYTE, HE_CAP_ELE_SUPP_MCS_LEN_RX_80M_80M);
+	_rtw_memset(phl_sta->asoc_cap.he_tx_mcs + 4, HE_MSC_NOT_SUPP_BYTE, HE_CAP_ELE_SUPP_MCS_LEN_TX_80M_80M);
+
+	/* only deal with <= 80MHz & 160M now (ToDo 80M+80M)*/
+	for (nss = 1; nss <= 8; nss++) {
+
+		mcs_val_rx = HE_MSC_NOT_SUPP;
+		mcs_val_tx = HE_MSC_NOT_SUPP;
+
+		switch (nss) {
+		case 1:
+			mcs_from_role = GET_HE_CAP_MCS_1SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_1SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_1SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_1SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_1SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_1SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_1SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_1SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_1SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_1SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_1SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_1SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		case 2:
+			mcs_from_role = GET_HE_CAP_MCS_2SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_2SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_2SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_2SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_2SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_2SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_2SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_2SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_2SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_2SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_2SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_2SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		case 3:
+			mcs_from_role = GET_HE_CAP_MCS_3SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_3SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_3SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_3SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_3SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_3SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_3SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_3SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_3SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_3SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_3SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_3SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		case 4:
+			mcs_from_role = GET_HE_CAP_MCS_4SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_4SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_4SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_4SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_4SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_4SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_4SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_4SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_4SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_4SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_4SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_4SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		case 5:
+			mcs_from_role = GET_HE_CAP_MCS_5SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_5SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_5SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_5SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_5SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_5SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_5SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_5SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_5SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_5SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_5SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_5SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		case 6:
+			mcs_from_role = GET_HE_CAP_MCS_6SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_6SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_6SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_6SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_6SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_6SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_6SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_6SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_6SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_6SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_6SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_6SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		case 7:
+			mcs_from_role = GET_HE_CAP_MCS_7SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_7SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_7SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_7SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_7SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_7SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_7SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_7SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_7SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_7SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_7SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_7SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		case 8:
+			mcs_from_role = GET_HE_CAP_MCS_8SS(role_cap->he_tx_mcs);
+			mcs_from_ie = GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_8SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+			mcs_from_role = GET_HE_CAP_MCS_8SS(role_cap->he_rx_mcs);
+			mcs_from_ie = GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_8SS(ele_start);
+			if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+				mcs_val_tx = mcs_from_ie;
+
+			SET_HE_CAP_MCS_8SS(phl_sta->asoc_cap.he_rx_mcs, mcs_val_rx);
+			SET_HE_CAP_MCS_8SS(phl_sta->asoc_cap.he_tx_mcs, mcs_val_tx);
+			if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+			    ((phl_sta->chandef.band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+			     (phl_sta->chandef.band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+			   ) {
+				mcs_from_role = GET_HE_CAP_MCS_8SS(role_cap->he_tx_mcs);
+				mcs_from_ie = GET_HE_CAP_RX_MCS_160MHZ_8SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_rx = (mcs_from_role < mcs_from_ie) ? mcs_from_role : mcs_from_ie;
+
+				mcs_from_role = GET_HE_CAP_MCS_8SS(role_cap->he_rx_mcs);
+				mcs_from_ie = GET_HE_CAP_TX_MCS_160MHZ_8SS(ele_start);
+				if ((mcs_from_role != HE_MSC_NOT_SUPP) && (mcs_from_ie != HE_MSC_NOT_SUPP))
+					mcs_val_tx = mcs_from_ie;
+
+				SET_HE_CAP_MCS_8SS(phl_sta->asoc_cap.he_rx_mcs + 2, mcs_val_rx);
+				SET_HE_CAP_MCS_8SS(phl_sta->asoc_cap.he_tx_mcs + 2, mcs_val_tx);
+			}
+			break;
+		}
+
+		if (mcs_val_rx != HE_MSC_NOT_SUPP)
+			nss_rx++;
+
+		if (mcs_val_tx != HE_MSC_NOT_SUPP)
+			nss_tx++;
+	}
+
+	phl_sta->asoc_cap.nss_rx = nss_rx;
+	phl_sta->asoc_cap.nss_tx = nss_tx;
+}
+
+static void rtw_he_set_asoc_cap_ppe_thre(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start)
+{
+	u8 nsts, rumsk, i, j, offset, shift;
+	u16 ppe8, ppe16;
+
+	if (phl_sta->asoc_cap.pkt_padding != 3)
+		return;
+
+	nsts = GET_HE_CAP_PPE_NSTS(ele_start);
+	rumsk = GET_HE_CAP_PPE_PU_IDX_BITMASK(ele_start);
+	shift = 7;
+
+	for (i = 0; i <= nsts; i ++) {
+		for (j = 0; j < 4; j++) {
+			if (rumsk & (BIT(0) << j)) {
+				offset = shift / 8;
+				ppe16 = LE_BITS_TO_2BYTE(ele_start + offset, shift % 8, 3);
+				shift += 3;
+				offset = shift / 8;
+				ppe8 = LE_BITS_TO_2BYTE(ele_start + offset, shift % 8, 3);
+				shift += 3;
+				phl_sta->asoc_cap.ppe_thr[i][j] = ((ppe16 & 0x07) | ((ppe8 & 0x07) << 3));
+			} else {
+				phl_sta->asoc_cap.ppe_thr[i][j] = 0;
+			}
+		}
+	}
+}
+
+static void update_sta_he_mac_cap_apmode(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start)
+{
+	/* CONFIG_80211AX_HE_TODO - we may need to refer to role_cap when setting some of asoc_cap  */
+#if 0
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct protocol_cap_t *role_cap = &(wrole->protocol_cap);
+#endif
+
+	phl_sta->asoc_cap.htc_rx = GET_HE_MAC_CAP_HTC_HE_SUPPORT(ele_start);
+	phl_sta->asoc_cap.twt = GET_HE_MAC_CAP_TWT_REQUESTER_SUPPORT(ele_start);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_TWT_RESPONDER_SUPPORT(ele_start)) << 1);
+	phl_sta->asoc_cap.trig_padding = GET_HE_MAC_CAP_TRI_FRAME_PADDING_DUR(ele_start);
+	phl_sta->asoc_cap.all_ack = GET_HE_MAC_CAP_ALL_ACK_SUPPORT(ele_start);
+	phl_sta->asoc_cap.a_ctrl = GET_HE_MAC_CAP_TRS_SUPPORT(ele_start);
+	phl_sta->asoc_cap.a_ctrl |= ((GET_HE_MAC_CAP_BRS_SUPPORT(ele_start)) << 1);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_BC_TWT_SUPPORT(ele_start)) << 2);
+	phl_sta->asoc_cap.a_ctrl |= ((GET_HE_MAC_CAP_OM_CTRL_SUPPORT(ele_start)) << 2);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_FLEX_TWT_SCHED_SUPPORT(ele_start)) << 3);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_PSR_RESPONDER(ele_start)) << 4);
+	phl_sta->asoc_cap.ops = GET_HE_MAC_CAP_OPS_SUPPORT(ele_start);
+	phl_sta->asoc_cap.amsdu_in_ampdu =
+		GET_HE_MAC_CAP_AMSDU_NOT_UNDER_BA_IN_ACK_EN_AMPDU(ele_start);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_HE_SUB_CH_SELECTIVE_TX(ele_start)) << 5);
+	phl_sta->asoc_cap.ht_vht_trig_rx =
+		GET_HE_MAC_CAP_HT_VHT_TRIG_FRAME_RX(ele_start);
+}
+
+static void update_sta_he_phy_cap_apmode(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 *supp_mcs_len)
+{
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct rtw_wifi_role_link_t *rlink = phl_sta->rlink;
+	struct protocol_cap_t *role_cap = &(rlink->protocol_cap);
+	struct role_link_cap_t *cap = &(rlink->cap);
+
+	if (phl_sta->chandef.band == BAND_ON_24G) {
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(0))
+			phl_sta->chandef.bw = (rlink->chandef.bw < CHANNEL_WIDTH_40) ?
+			rlink->chandef.bw : CHANNEL_WIDTH_40;
+	} else if ((phl_sta->chandef.band == BAND_ON_5G) || (phl_sta->chandef.band == BAND_ON_6G)) {
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(1))
+			phl_sta->chandef.bw = (rlink->chandef.bw < CHANNEL_WIDTH_80) ?
+			rlink->chandef.bw : CHANNEL_WIDTH_80;
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(2)) {
+			phl_sta->chandef.bw = (rlink->chandef.bw < CHANNEL_WIDTH_160) ?
+			rlink->chandef.bw : CHANNEL_WIDTH_160;
+			*supp_mcs_len += 4;
+		}
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(3))
+			*supp_mcs_len += 4;
+	}
+
+	phl_sta->asoc_cap.he_ldpc = (GET_HE_PHY_CAP_LDPC_IN_PAYLOAD(ele_start) & role_cap->he_ldpc);
+
+	if (role_cap->ltf_gi) {
+		if (phl_sta->asoc_cap.er_su) {
+			phl_sta->asoc_cap.ltf_gi = (BIT(RTW_GILTF_2XHE16) |
+				BIT(RTW_GILTF_2XHE08) | BIT(RTW_GILTF_1XHE16) |
+				(GET_HE_PHY_CAP_NDP_4X_LTF_3_POINT_2_GI(ele_start) ?
+				BIT(RTW_GILTF_LGI_4XHE32) : 0) |
+				(GET_HE_PHY_CAP_ERSU_PPDU_4X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_SGI_4XHE08) : 0) |
+				(GET_HE_PHY_CAP_ERSU_PPDU_1X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_1XHE08) : 0));
+		} else {
+			phl_sta->asoc_cap.ltf_gi = (BIT(RTW_GILTF_2XHE16) |
+				BIT(RTW_GILTF_2XHE08) | BIT(RTW_GILTF_1XHE16) |
+				(GET_HE_PHY_CAP_NDP_4X_LTF_3_POINT_2_GI(ele_start) ?
+				BIT(RTW_GILTF_LGI_4XHE32) : 0) |
+				(GET_HE_PHY_CAP_SU_MU_PPDU_4X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_SGI_4XHE08) : 0) |
+				(GET_HE_PHY_CAP_SU_PPDU_1X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_1XHE08) : 0));
+		}
+	}
+
+	phl_sta->asoc_cap.stbc_he_tx = GET_HE_PHY_CAP_STBC_TX_LESS_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.stbc_he_rx = (GET_HE_PHY_CAP_STBC_RX_LESS_THAN_80MHZ(ele_start) & role_cap->stbc_he_tx);
+	phl_sta->asoc_cap.doppler_tx = GET_HE_PHY_CAP_DOPPLER_TX(ele_start);
+	phl_sta->asoc_cap.doppler_rx = (GET_HE_PHY_CAP_DOPPLER_RX(ele_start) & role_cap->doppler_tx);
+
+	phl_sta->asoc_cap.dcm_max_const_tx =
+		GET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_TX(ele_start);
+	if (phl_sta->asoc_cap.dcm_max_const_tx > role_cap->dcm_max_const_rx)
+		phl_sta->asoc_cap.dcm_max_const_tx = role_cap->dcm_max_const_rx;
+
+	phl_sta->asoc_cap.dcm_max_nss_tx = (GET_HE_PHY_CAP_DCM_MAX_NSS_TX(ele_start) & role_cap->dcm_max_nss_rx);
+
+	phl_sta->asoc_cap.dcm_max_const_rx =
+		GET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_RX(ele_start);
+	if (phl_sta->asoc_cap.dcm_max_const_rx > role_cap->dcm_max_const_tx)
+		phl_sta->asoc_cap.dcm_max_const_rx = role_cap->dcm_max_const_tx;
+
+	phl_sta->asoc_cap.dcm_max_nss_rx = (GET_HE_PHY_CAP_DCM_MAX_NSS_RX(ele_start) & role_cap->dcm_max_nss_tx);
+
+	phl_sta->asoc_cap.partial_bw_su_er =
+		GET_HE_PHY_CAP_RX_PARTIAL_BW_SU_IN_20MHZ_MUPPDU(ele_start);
+	phl_sta->asoc_cap.he_su_bfmr = GET_HE_PHY_CAP_SU_BFER(ele_start);
+	phl_sta->asoc_cap.he_su_bfme = GET_HE_PHY_CAP_SU_BFEE(ele_start);
+	phl_sta->asoc_cap.he_mu_bfmr = GET_HE_PHY_CAP_MU_BFER(ele_start);
+	phl_sta->asoc_cap.bfme_sts =
+		GET_HE_PHY_CAP_BFEE_STS_LESS_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.bfme_sts_greater_80mhz =
+		GET_HE_PHY_CAP_BFEE_STS_GREATER_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.num_snd_dim =
+		GET_HE_PHY_CAP_NUM_SND_DIMEN_LESS_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.num_snd_dim_greater_80mhz =
+		GET_HE_PHY_CAP_NUM_SND_DIMEN_GREATER_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.ng_16_su_fb = GET_HE_PHY_CAP_NG_16_SU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.ng_16_mu_fb = GET_HE_PHY_CAP_NG_16_MU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.cb_sz_su_fb =
+		GET_HE_PHY_CAP_CODEBOOK_4_2_SU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.cb_sz_mu_fb =
+		GET_HE_PHY_CAP_CODEBOOK_7_5_MU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.trig_su_bfm_fb =
+		GET_HE_PHY_CAP_TRIG_SUBF_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.trig_mu_bfm_fb =
+		GET_HE_PHY_CAP_TRIG_MUBF_PARTIAL_BW_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.trig_cqi_fb = GET_HE_PHY_CAP_TRIG_CQI_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.partial_bw_su_er =
+		GET_HE_PHY_CAP_PARTIAL_BW_EXT_RANGE(ele_start);
+	phl_sta->asoc_cap.pwr_bst_factor =
+		GET_HE_PHY_CAP_PWR_BOOST_FACTOR_SUPPORT(ele_start);
+	phl_sta->asoc_cap.max_nc = GET_HE_PHY_CAP_MAX_NC(ele_start);
+	phl_sta->asoc_cap.stbc_tx_greater_80mhz =
+		(GET_HE_PHY_CAP_STBC_TX_GREATER_THAN_80MHZ(ele_start) & role_cap->stbc_rx_greater_80mhz);
+	phl_sta->asoc_cap.stbc_rx_greater_80mhz =
+		(GET_HE_PHY_CAP_STBC_RX_GREATER_THAN_80MHZ(ele_start) & role_cap->stbc_tx_greater_80mhz);
+	phl_sta->asoc_cap.dcm_max_ru = GET_HE_PHY_CAP_DCM_MAX_RU(ele_start);
+	phl_sta->asoc_cap.long_sigb_symbol =
+		GET_HE_PHY_CAP_LONGER_THAN_16_HESIGB_OFDM_SYM(ele_start);
+	phl_sta->asoc_cap.non_trig_cqi_fb =
+		GET_HE_PHY_CAP_NON_TRIGGER_CQI_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.tx_1024q_ru =
+		(GET_HE_PHY_CAP_TX_1024_QAM_LESS_THAN_242_TONE_RU(ele_start) & role_cap->rx_1024q_ru);
+	phl_sta->asoc_cap.rx_1024q_ru =
+		(GET_HE_PHY_CAP_RX_1024_QAM_LESS_THAN_242_TONE_RU(ele_start) & role_cap->tx_1024q_ru);
+	phl_sta->asoc_cap.fbw_su_using_mu_cmprs_sigb =
+		GET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_CMP_SIGB(ele_start);
+	phl_sta->asoc_cap.fbw_su_using_mu_non_cmprs_sigb =
+		GET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_NONCMP_SIGB(ele_start);
+
+	if (GET_HE_PHY_CAP_PPE_THRESHOLD_PRESENT(ele_start))
+		phl_sta->asoc_cap.pkt_padding = 3;
+	else
+		phl_sta->asoc_cap.pkt_padding = GET_HE_PHY_CAP_NOMINAL_PACKET_PADDING(ele_start);
+}
+
+static void update_sta_he_supp_mcs_apmode(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 supp_mcs_len)
+{
+	rtw_he_set_asoc_cap_supp_mcs(padapter, phl_sta, ele_start, supp_mcs_len);
+}
+
+static void update_sta_he_ppe_thre_apmode(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start)
+{
+	rtw_he_set_asoc_cap_ppe_thre(padapter, phl_sta, ele_start);
+}
+
+void	update_sta_he_info_apmode(_adapter *padapter, void *sta)
+{
+	struct sta_info	*psta = (struct sta_info *)sta;
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct he_priv	*phepriv_sta = &psta->hepriv;
+	u8 *ele_start = NULL;
+	u8 supp_mcs_len = 4;
+
+	if (phepriv_sta->he_option == _FALSE)
+		return;
+
+	ele_start = &(phepriv_sta->he_cap[1]);
+	update_sta_he_mac_cap_apmode(padapter, phl_sta, ele_start);
+
+	ele_start += HE_CAP_ELE_MAC_CAP_LEN;
+	update_sta_he_phy_cap_apmode(padapter, phl_sta, ele_start, &supp_mcs_len);
+
+	ele_start += HE_CAP_ELE_PHY_CAP_LEN;
+	update_sta_he_supp_mcs_apmode(padapter, phl_sta, ele_start, supp_mcs_len);
+
+	ele_start += supp_mcs_len;
+	update_sta_he_ppe_thre_apmode(padapter, phl_sta, ele_start);
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+void update_sta_6ghz_info_apmode(_adapter *padapter, void *sta)
+{
+	struct sta_info	*psta = (struct sta_info *)sta;
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct ampdu_priv *ampdu_priv_sta = &psta->ampdu_priv;
+
+	/* 6G = HE only = AMPDU always */
+	if (phl_sta->chandef.band == BAND_ON_6G)
+		ampdu_priv_sta->ampdu_enable = _TRUE;
+}
+#endif
+
+void HE_mac_caps_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start)
+{
+	phl_sta->asoc_cap.htc_rx = GET_HE_MAC_CAP_HTC_HE_SUPPORT(ele_start);
+	phl_sta->asoc_cap.twt = GET_HE_MAC_CAP_TWT_REQUESTER_SUPPORT(ele_start);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_TWT_RESPONDER_SUPPORT(ele_start)) << 1);
+	phl_sta->asoc_cap.trig_padding = GET_HE_MAC_CAP_TRI_FRAME_PADDING_DUR(ele_start);
+	phl_sta->asoc_cap.all_ack = GET_HE_MAC_CAP_ALL_ACK_SUPPORT(ele_start);
+	phl_sta->asoc_cap.a_ctrl = GET_HE_MAC_CAP_TRS_SUPPORT(ele_start);
+	phl_sta->asoc_cap.a_ctrl |= ((GET_HE_MAC_CAP_BRS_SUPPORT(ele_start)) << 1);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_BC_TWT_SUPPORT(ele_start)) << 2);
+	phl_sta->asoc_cap.a_ctrl |= ((GET_HE_MAC_CAP_OM_CTRL_SUPPORT(ele_start)) << 2);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_FLEX_TWT_SCHED_SUPPORT(ele_start)) << 3);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_PSR_RESPONDER(ele_start)) << 4);
+	phl_sta->asoc_cap.ops = GET_HE_MAC_CAP_OPS_SUPPORT(ele_start);
+	phl_sta->asoc_cap.amsdu_in_ampdu =
+		GET_HE_MAC_CAP_AMSDU_NOT_UNDER_BA_IN_ACK_EN_AMPDU(ele_start);
+	phl_sta->asoc_cap.twt |= ((GET_HE_MAC_CAP_HE_SUB_CH_SELECTIVE_TX(ele_start)) << 5);
+	phl_sta->asoc_cap.ht_vht_trig_rx =
+		GET_HE_MAC_CAP_HT_VHT_TRIG_FRAME_RX(ele_start);
+}
+
+void HE_phy_caps_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 *supp_mcs_len)
+{
+	struct rtw_wifi_role_t 	*wrole = padapter->phl_role;
+	struct rtw_wifi_role_link_t *rlink = phl_sta->rlink;
+	struct protocol_cap_t *role_cap = &(phl_sta->rlink->protocol_cap);
+
+	if (phl_sta->chandef.band == BAND_ON_24G) {
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(0))
+			phl_sta->chandef.bw = (rlink->chandef.bw < CHANNEL_WIDTH_40) ?
+			rlink->chandef.bw : CHANNEL_WIDTH_40;
+	} else if ((phl_sta->chandef.band == BAND_ON_5G) || (phl_sta->chandef.band == BAND_ON_6G)) {
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(1))
+			phl_sta->chandef.bw = (rlink->chandef.bw < CHANNEL_WIDTH_80) ?
+			rlink->chandef.bw : CHANNEL_WIDTH_80;
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(2)) {
+			phl_sta->chandef.bw = (rlink->chandef.bw < CHANNEL_WIDTH_160) ?
+			rlink->chandef.bw : CHANNEL_WIDTH_160;
+			*supp_mcs_len += 4;
+		}
+		if (GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(ele_start) & BIT(3))
+			*supp_mcs_len += 4;
+	}
+	phl_sta->asoc_cap.he_ldpc = (GET_HE_PHY_CAP_LDPC_IN_PAYLOAD(ele_start) & role_cap->he_ldpc);
+	if (role_cap->ltf_gi) {
+		if (phl_sta->asoc_cap.er_su) {
+			phl_sta->asoc_cap.ltf_gi = (BIT(RTW_GILTF_2XHE16) |
+				BIT(RTW_GILTF_2XHE08) | BIT(RTW_GILTF_1XHE16) |
+				(GET_HE_PHY_CAP_NDP_4X_LTF_3_POINT_2_GI(ele_start) ?
+				BIT(RTW_GILTF_LGI_4XHE32) : 0) |
+				(GET_HE_PHY_CAP_ERSU_PPDU_4X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_SGI_4XHE08) : 0) |
+				(GET_HE_PHY_CAP_ERSU_PPDU_1X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_1XHE08) : 0));
+		} else {
+			phl_sta->asoc_cap.ltf_gi = (BIT(RTW_GILTF_2XHE16) |
+				BIT(RTW_GILTF_2XHE08) | BIT(RTW_GILTF_1XHE16) |
+				(GET_HE_PHY_CAP_NDP_4X_LTF_3_POINT_2_GI(ele_start) ?
+				BIT(RTW_GILTF_LGI_4XHE32) : 0) |
+				(GET_HE_PHY_CAP_SU_MU_PPDU_4X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_SGI_4XHE08) : 0) |
+				(GET_HE_PHY_CAP_SU_PPDU_1X_LTF_0_POINT_8_GI(ele_start) ?
+				BIT(RTW_GILTF_1XHE08) : 0));
+		}
+	}
+	phl_sta->asoc_cap.stbc_he_tx = GET_HE_PHY_CAP_STBC_TX_LESS_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.stbc_he_rx = (GET_HE_PHY_CAP_STBC_RX_LESS_THAN_80MHZ(ele_start) & role_cap->stbc_he_tx);
+	phl_sta->asoc_cap.doppler_tx = GET_HE_PHY_CAP_DOPPLER_TX(ele_start);
+	phl_sta->asoc_cap.doppler_rx = (GET_HE_PHY_CAP_DOPPLER_RX(ele_start) & role_cap->doppler_tx);
+	phl_sta->asoc_cap.dcm_max_const_tx =
+		GET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_TX(ele_start);
+	phl_sta->asoc_cap.dcm_max_nss_tx = GET_HE_PHY_CAP_DCM_MAX_NSS_TX(ele_start);
+	phl_sta->asoc_cap.dcm_max_const_rx =
+		GET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_RX(ele_start);
+	phl_sta->asoc_cap.dcm_max_nss_rx = GET_HE_PHY_CAP_DCM_MAX_NSS_RX(ele_start);
+	phl_sta->asoc_cap.partial_bw_su_er =
+		GET_HE_PHY_CAP_RX_PARTIAL_BW_SU_IN_20MHZ_MUPPDU(ele_start);
+	phl_sta->asoc_cap.he_su_bfmr = GET_HE_PHY_CAP_SU_BFER(ele_start);
+	phl_sta->asoc_cap.he_su_bfme = GET_HE_PHY_CAP_SU_BFEE(ele_start);
+	phl_sta->asoc_cap.he_mu_bfmr = GET_HE_PHY_CAP_MU_BFER(ele_start);
+	phl_sta->asoc_cap.bfme_sts =
+		GET_HE_PHY_CAP_BFEE_STS_LESS_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.bfme_sts_greater_80mhz =
+		GET_HE_PHY_CAP_BFEE_STS_GREATER_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.num_snd_dim =
+		GET_HE_PHY_CAP_NUM_SND_DIMEN_LESS_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.num_snd_dim_greater_80mhz =
+		GET_HE_PHY_CAP_NUM_SND_DIMEN_GREATER_THAN_80MHZ(ele_start);
+
+	RTW_INFO("%s: HE STA assoc_cap:\n", __func__);
+	RTW_INFO("- SU BFer: %d\n", phl_sta->asoc_cap.he_su_bfmr);
+	RTW_INFO("- SU BFee: %d\n", phl_sta->asoc_cap.he_su_bfme);
+	RTW_INFO("- MU BFer: %d\n", phl_sta->asoc_cap.he_mu_bfmr);
+	RTW_INFO("- BFee STS: %d\n", phl_sta->asoc_cap.bfme_sts);
+	RTW_INFO("- BFee STS(>80MHz): %d\n", phl_sta->asoc_cap.bfme_sts_greater_80mhz);
+	RTW_INFO("- BFer SND DIM number: %d\n", phl_sta->asoc_cap.num_snd_dim);
+	RTW_INFO("- BFer SND DIM number(>80MHz): %d\n", phl_sta->asoc_cap.num_snd_dim_greater_80mhz);
+
+	phl_sta->asoc_cap.ng_16_su_fb = GET_HE_PHY_CAP_NG_16_SU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.ng_16_mu_fb = GET_HE_PHY_CAP_NG_16_MU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.cb_sz_su_fb =
+		GET_HE_PHY_CAP_CODEBOOK_4_2_SU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.cb_sz_mu_fb =
+		GET_HE_PHY_CAP_CODEBOOK_7_5_MU_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.trig_su_bfm_fb =
+		GET_HE_PHY_CAP_TRIG_SUBF_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.trig_mu_bfm_fb =
+		GET_HE_PHY_CAP_TRIG_MUBF_PARTIAL_BW_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.trig_cqi_fb = GET_HE_PHY_CAP_TRIG_CQI_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.partial_bw_su_er =
+		GET_HE_PHY_CAP_PARTIAL_BW_EXT_RANGE(ele_start);
+	phl_sta->asoc_cap.pwr_bst_factor =
+		GET_HE_PHY_CAP_PWR_BOOST_FACTOR_SUPPORT(ele_start);
+	phl_sta->asoc_cap.max_nc = GET_HE_PHY_CAP_MAX_NC(ele_start);
+	phl_sta->asoc_cap.stbc_tx_greater_80mhz =
+		GET_HE_PHY_CAP_STBC_TX_GREATER_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.stbc_rx_greater_80mhz =
+		GET_HE_PHY_CAP_STBC_RX_GREATER_THAN_80MHZ(ele_start);
+	phl_sta->asoc_cap.dcm_max_ru = GET_HE_PHY_CAP_DCM_MAX_RU(ele_start);
+	phl_sta->asoc_cap.long_sigb_symbol =
+		GET_HE_PHY_CAP_LONGER_THAN_16_HESIGB_OFDM_SYM(ele_start);
+	phl_sta->asoc_cap.non_trig_cqi_fb =
+		GET_HE_PHY_CAP_NON_TRIGGER_CQI_FEEDBACK(ele_start);
+	phl_sta->asoc_cap.tx_1024q_ru =
+		GET_HE_PHY_CAP_TX_1024_QAM_LESS_THAN_242_TONE_RU(ele_start);
+	phl_sta->asoc_cap.rx_1024q_ru =
+		GET_HE_PHY_CAP_RX_1024_QAM_LESS_THAN_242_TONE_RU(ele_start);
+	phl_sta->asoc_cap.fbw_su_using_mu_cmprs_sigb =
+		GET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_CMP_SIGB(ele_start);
+	phl_sta->asoc_cap.fbw_su_using_mu_non_cmprs_sigb =
+		GET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_NONCMP_SIGB(ele_start);
+
+	if (GET_HE_PHY_CAP_PPE_THRESHOLD_PRESENT(ele_start))
+		phl_sta->asoc_cap.pkt_padding = 3;
+	else
+		phl_sta->asoc_cap.pkt_padding = GET_HE_PHY_CAP_NOMINAL_PACKET_PADDING(ele_start);
+}
+
+void HE_supp_mcs_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 supp_mcs_len)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_LINK(padapter, phl_sta->rlink->id);
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct he_priv *phepriv = &pmlmepriv->hepriv;
+
+	rtw_he_set_asoc_cap_supp_mcs(padapter, phl_sta, ele_start, supp_mcs_len);
+	phepriv->he_highest_rate = rtw_he_get_highest_rate(phl_sta->asoc_cap.he_rx_mcs);
+}
+
+void HE_ppe_thre_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start)
+{
+	rtw_he_set_asoc_cap_ppe_thre(padapter, phl_sta, ele_start);
+}
+
+void HE_caps_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct rtw_wifi_role_t 	*wrole = padapter->phl_role;
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct sta_priv 		*pstapriv = &padapter->stapriv;
+	struct he_priv		*phepriv = &pmlmepriv->hepriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX 		*cur_network = &(pmlmeinfo->network);
+	struct sta_info 		*psta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8 *ele_start = (&(pIE->data[0]) + 1);
+	u8 supp_mcs_len = 4;
+
+	if (pIE == NULL)
+		return;
+
+	if (phepriv->he_option == _FALSE)
+		return;
+
+	psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+	if (psta == NULL)
+		return;
+	if (psta->phl_sta == NULL)
+		return;
+
+	phl_sta = psta->phl_sta;
+
+	/* HE MAC Caps */
+	HE_mac_caps_handler(padapter, phl_sta, ele_start);
+	ele_start += HE_CAP_ELE_MAC_CAP_LEN;
+
+	/* HE PHY Caps */
+	HE_phy_caps_handler(padapter, phl_sta, ele_start, &supp_mcs_len);
+	ele_start += HE_CAP_ELE_PHY_CAP_LEN;
+
+	/* HE Supp MCS Set */
+	HE_supp_mcs_handler(padapter, phl_sta, ele_start, supp_mcs_len);
+	ele_start += supp_mcs_len;
+
+	/* HE PPE Thresholds */
+	HE_ppe_thre_handler(padapter, phl_sta, ele_start);
+
+	pmlmeinfo->HE_enable = 1;
+}
+
+void HE_operation_handler(_adapter *padapter,
+				struct _ADAPTER_LINK *padapter_link,
+				PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct sta_priv 		*pstapriv = &padapter->stapriv;
+	struct he_priv		*phepriv = &pmlmepriv->hepriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX 		*cur_network = &(pmlmeinfo->network);
+	struct sta_info 		*psta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8 *ele_start = (&(pIE->data[0]) + 1);
+	struct dvobj_priv *d;
+	void *phl;
+	u8 pre_bsscolor = 0;
+	u16 pre_rts_th = 0;
+
+	if (pIE == NULL)
+		return;
+
+	if (phepriv->he_option == _FALSE)
+		return;
+
+	d = adapter_to_dvobj(padapter);
+	phl = GET_PHL_INFO(d);
+
+	psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+	if (psta == NULL)
+		return;
+	if (psta->phl_sta == NULL)
+		return;
+
+	phl_sta = psta->phl_sta;
+	phl_sta->tf_trs = _TRUE;
+
+	phl_sta->asoc_cap.er_su = !GET_HE_OP_PARA_ER_SU_DISABLE(ele_start);
+	if (!GET_HE_OP_BSS_COLOR_INFO_BSS_COLOR_DISABLE(ele_start)) {
+
+		pre_bsscolor = phl_sta->asoc_cap.bsscolor;
+		phl_sta->asoc_cap.bsscolor = GET_HE_OP_BSS_COLOR_INFO_BSS_COLOR(ele_start);
+
+		/* rx thread & assoc timer callback, use cmd no_wait */
+		if (pre_bsscolor != phl_sta->asoc_cap.bsscolor) {
+			RTW_INFO("%s, Update BSS Color = %d\n", __func__, phl_sta->asoc_cap.bsscolor);
+			rtw_phl_cmd_wrole_change(phl,
+			                         padapter->phl_role,
+			                         padapter_link->wrlink,
+			                         WR_CHG_BSS_COLOR,
+			                         (u8 *)&phl_sta->asoc_cap.bsscolor,
+			                         sizeof(phl_sta->asoc_cap.bsscolor),
+			                         PHL_CMD_NO_WAIT,
+			                         0);
+		}
+	}
+
+	pre_rts_th = phl_sta->asoc_cap.rts_th;
+	phl_sta->asoc_cap.rts_th =
+		GET_HE_OP_PARA_TXOP_DUR_RTS_THRESHOLD(ele_start);
+
+	if ((phl_sta->asoc_cap.rts_th > 0) &&
+	    (phl_sta->asoc_cap.rts_th != TXOP_DUR_RTS_TH_DISABLED)) {
+		struct rtw_rts_threshold val = {0};
+
+		/* time preference */
+		val.rts_len_th = 0xffff;
+		/* IE field unit 32us, parameter unit 1us */
+		val.rts_time_th = phl_sta->asoc_cap.rts_th * 32;
+		/* rx thread & assoc timer callback, use cmd no_wait */
+		if (pre_rts_th != phl_sta->asoc_cap.rts_th) {
+			RTW_INFO("%s, Update TXOP Duration RTS Threshold =%d\n", __func__, phl_sta->asoc_cap.rts_th);
+			rtw_phl_cmd_wrole_change(phl,
+			                         padapter->phl_role,
+			                         padapter_link->wrlink,
+			                         WR_CHG_RTS_TH,
+			                         (u8 *)&val,
+			                         sizeof(struct rtw_rts_threshold),
+			                         PHL_CMD_NO_WAIT,
+			                         0);
+		}
+	}
+}
+
+void HE_mu_edca_handler(_adapter *padapter,
+			struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE, u8 first)
+{
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct he_priv *phepriv = &pmlmepriv->hepriv;
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *cur_network = &(pmlmeinfo->network);
+	struct sta_info *psta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8 *ele_start = (&(pIE->data[0]) + 1);
+	struct dvobj_priv *d;
+	void *phl;
+	struct rtw_mu_edca_param edca = {0};
+	u8 pre_cnt = 0, cur_cnt = 0;
+	u8 i = 0;
+
+	if (pIE == NULL)
+		return;
+
+	if (phepriv->he_option == _FALSE)
+		return;
+
+	d = adapter_to_dvobj(padapter);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return;
+
+	psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+	if (psta == NULL)
+		return;
+	if (psta->phl_sta == NULL)
+		return;
+
+	phl_sta = psta->phl_sta;
+
+	pre_cnt = phepriv->pre_he_muedca_cnt;
+	cur_cnt = GET_HE_MU_EDCA_QOS_INFO_UPDATE_CNT(ele_start);
+	if (cur_cnt != pre_cnt || first == _TRUE) {
+		phepriv->pre_he_muedca_cnt = cur_cnt;
+		phl_sta->asoc_cap.mu_edca[0].ac =
+			GET_HE_MU_EDCA_BE_ACI(ele_start);
+		phl_sta->asoc_cap.mu_edca[0].aifsn =
+			GET_HE_MU_EDCA_BE_AIFSN(ele_start);
+		phl_sta->asoc_cap.mu_edca[0].cw =
+			GET_HE_MU_EDCA_BE_ECW_MIN_MAX(ele_start);
+		phl_sta->asoc_cap.mu_edca[0].timer =
+			GET_HE_MU_EDCA_BE_TIMER(ele_start);
+		phl_sta->asoc_cap.mu_edca[1].ac =
+			GET_HE_MU_EDCA_BK_ACI(ele_start);
+		phl_sta->asoc_cap.mu_edca[1].aifsn =
+			GET_HE_MU_EDCA_BK_AIFSN(ele_start);
+		phl_sta->asoc_cap.mu_edca[1].cw =
+			GET_HE_MU_EDCA_BK_ECW_MIN_MAX(ele_start);
+		phl_sta->asoc_cap.mu_edca[1].timer =
+			GET_HE_MU_EDCA_BK_TIMER(ele_start);
+		phl_sta->asoc_cap.mu_edca[2].ac =
+			GET_HE_MU_EDCA_VI_ACI(ele_start);
+		phl_sta->asoc_cap.mu_edca[2].aifsn =
+			GET_HE_MU_EDCA_VI_AIFSN(ele_start);
+		phl_sta->asoc_cap.mu_edca[2].cw =
+			GET_HE_MU_EDCA_VI_ECW_MIN_MAX(ele_start);
+		phl_sta->asoc_cap.mu_edca[2].timer =
+			GET_HE_MU_EDCA_VI_TIMER(ele_start);
+		phl_sta->asoc_cap.mu_edca[3].ac =
+			GET_HE_MU_EDCA_VO_ACI(ele_start);
+		phl_sta->asoc_cap.mu_edca[3].aifsn =
+			GET_HE_MU_EDCA_VO_AIFSN(ele_start);
+		phl_sta->asoc_cap.mu_edca[3].cw =
+			GET_HE_MU_EDCA_VO_ECW_MIN_MAX(ele_start);
+		phl_sta->asoc_cap.mu_edca[3].timer =
+			GET_HE_MU_EDCA_VO_TIMER(ele_start);
+		for (i = 0; i < 4; i++) {
+			rtw_phl_cmd_wrole_change(phl,
+						 padapter->phl_role,
+						 padapter_link->wrlink,
+						 WR_CHG_MU_EDCA_PARAM,
+						 (u8 *)&phl_sta->asoc_cap.mu_edca[i],
+						 sizeof(struct rtw_mu_edca_param),
+						 PHL_CMD_NO_WAIT,
+						 0);
+
+			RTW_INFO("%s, Update HE MU EDCA AC(%d) aifsn(%d) cw(0x%x) timer(0x%x)\n",
+					__func__,
+					phl_sta->asoc_cap.mu_edca[i].ac,
+					phl_sta->asoc_cap.mu_edca[i].aifsn,
+					phl_sta->asoc_cap.mu_edca[i].cw,
+					phl_sta->asoc_cap.mu_edca[i].timer);
+		}
+
+		if (first) {
+			rtw_phl_cmd_wrole_change(phl,
+						 padapter->phl_role,
+						 padapter_link->wrlink,
+						 WR_CHG_MU_EDCA_CFG,
+						 (u8 *)&first,
+						 sizeof(first),
+						 PHL_CMD_NO_WAIT,
+						 0);
+		}
+	}
+}
+
+void HE_6g_bandcap_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct sta_priv			*pstapriv = &padapter->stapriv;
+	struct he_priv			*phepriv = &pmlmepriv->hepriv;
+	struct registry_priv		*pregistrypriv = &padapter->registrypriv;
+	struct link_mlme_ext_info		*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX			*cur_network = &(pmlmeinfo->network);
+	struct sta_info			*psta = NULL;
+	struct rtw_phl_stainfo_t	*phl_sta = NULL;
+	u8				*ele_start = (&(pIE->data[0]) + 1);
+
+	if (pIE == NULL)
+		return;
+
+	if (phepriv->he_option == _FALSE)
+		return;
+
+	psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+	if (psta == NULL)
+		return;
+	if (psta->phl_sta == NULL)
+		return;
+
+	phl_sta = psta->phl_sta;
+
+	phl_sta->asoc_cap.ampdu_density = GET_HE_6G_BAND_CAP_MIN_MPDU_SPACING(ele_start);
+	phl_sta->asoc_cap.ampdu_len_exp = GET_HE_6G_BAND_CAP_MAX_AMPDU_LEN_EXP(ele_start);
+	phl_sta->asoc_cap.max_amsdu_len = GET_HE_6G_BAND_CAP_MAX_MPDU_LEN(ele_start);
+	phl_sta->asoc_cap.sm_ps = GET_HE_6G_BAND_CAP_SM_PS(ele_start);
+
+	pmlmeinfo->SM_PS = phl_sta->asoc_cap.sm_ps;
+	if (pmlmeinfo->SM_PS == SM_PS_STATIC)
+		RTW_WARN("%s(): SM_PS_STATIC\n", __FUNCTION__);
+
+	phepriv->mpdu_min_spacing = GET_HE_6G_BAND_CAP_MIN_MPDU_SPACING(ele_start);
+}
+
+static int rtw_build_he_mac_caps(struct protocol_cap_t *proto_cap, u8 *pbuf)
+{
+	/* Set HE MAC Capabilities Information */
+
+	int info_len = HE_CAP_ELE_MAC_CAP_LEN;
+
+	if (proto_cap->htc_rx)
+		SET_HE_MAC_CAP_HTC_HE_SUPPORT(pbuf, 1);
+
+	if (proto_cap->twt & BIT(0))
+		SET_HE_MAC_CAP_TWT_REQUESTER_SUPPORT(pbuf, 1);
+
+	if (proto_cap->twt & BIT(1))
+		SET_HE_MAC_CAP_TWT_RESPONDER_SUPPORT(pbuf, 1);
+
+	if (proto_cap->trig_padding)
+		SET_HE_MAC_CAP_TRI_FRAME_PADDING_DUR(pbuf,
+			proto_cap->trig_padding);
+
+	if (proto_cap->all_ack)
+		SET_HE_MAC_CAP_ALL_ACK_SUPPORT(pbuf, 1);
+
+	if (proto_cap->htc_rx && (proto_cap->a_ctrl & BIT(0)))
+		SET_HE_MAC_CAP_TRS_SUPPORT(pbuf, 1);
+
+	if (proto_cap->a_ctrl & BIT(1))
+		SET_HE_MAC_CAP_BRS_SUPPORT(pbuf, 1);
+
+	if (proto_cap->twt & BIT(2))
+		SET_HE_MAC_CAP_BC_TWT_SUPPORT(pbuf, 1);
+
+	if (proto_cap->htc_rx && (proto_cap->a_ctrl & BIT(2)))
+		SET_HE_MAC_CAP_OM_CTRL_SUPPORT(pbuf, 1);
+
+	SET_HE_MAC_CAP_MAX_AMPDU_LEN_EXP_EXT(pbuf, 2);
+
+	if (proto_cap->twt & BIT(3))
+		SET_HE_MAC_CAP_FLEX_TWT_SCHED_SUPPORT(pbuf, 1);
+
+	if (proto_cap->twt & BIT(4))
+		SET_HE_MAC_CAP_PSR_RESPONDER(pbuf, 1);
+
+	if (proto_cap->ops)
+		SET_HE_MAC_CAP_OPS_SUPPORT(pbuf, 1);
+
+	if (proto_cap->amsdu_in_ampdu)
+		SET_HE_MAC_CAP_AMSDU_NOT_UNDER_BA_IN_ACK_EN_AMPDU(pbuf, 1);
+
+	if (proto_cap->twt & BIT(5))
+		SET_HE_MAC_CAP_HE_SUB_CH_SELECTIVE_TX(pbuf, 1);
+
+	if (proto_cap->ht_vht_trig_rx)
+		SET_HE_MAC_CAP_HT_VHT_TRIG_FRAME_RX(pbuf, 1);
+
+	return info_len;
+}
+
+static int rtw_build_he_phy_caps(_adapter *padapter, struct protocol_cap_t *proto_cap, u8 *pbuf, enum band_type band)
+{
+	/* struct rtw_chan_def *chan_def = &(wrole->chandef); */
+	/* Set HE PHY Capabilities Information */
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	int info_len = HE_CAP_ELE_PHY_CAP_LEN;
+	u8 bw_cap = 0;
+
+	if (band == BAND_ON_24G) {
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_40M) && REGSTY_IS_BW_2G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+			bw_cap |= BIT(0);
+	} else if (band == BAND_ON_5G) {
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_40M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+			bw_cap |= BIT(1);
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_80))
+			bw_cap |= BIT(1);
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160))
+			bw_cap |= BIT(2);
+	} else if (band == BAND_ON_6G) {
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_40M) && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+			bw_cap |= BIT(1);
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80M) && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_80))
+			bw_cap |= BIT(1);
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160))
+			bw_cap |= BIT(2);
+	}
+
+	SET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(pbuf, bw_cap);
+
+	SET_HE_PHY_CAP_DEVICE_CLASS(pbuf, HE_DEV_CLASS_A);
+
+	if (proto_cap->he_ldpc)
+		SET_HE_PHY_CAP_LDPC_IN_PAYLOAD(pbuf, 1);
+
+#ifdef CONFIG_TWT
+	if (padapter->registrypriv.wifi_spec == 1 && padapter->registrypriv.twt_en == 1)
+		SET_HE_PHY_CAP_SU_PPDU_1X_LTF_0_POINT_8_GI(pbuf, 0);
+	else
+#endif
+		SET_HE_PHY_CAP_SU_PPDU_1X_LTF_0_POINT_8_GI(pbuf, 1);
+
+	if (proto_cap->he_rx_ndp_4x32) {
+		SET_HE_PHY_CAP_NDP_4X_LTF_3_POINT_2_GI(pbuf, 1);
+		RTW_DBG("NDP_4x32 is set.\n");;
+	}
+
+	if (proto_cap->stbc_he_tx)
+		SET_HE_PHY_CAP_STBC_TX_LESS_THAN_80MHZ(pbuf, 1);
+
+	if (proto_cap->stbc_he_rx)
+		SET_HE_PHY_CAP_STBC_RX_LESS_THAN_80MHZ(pbuf, 1);
+
+	if (proto_cap->doppler_tx)
+		SET_HE_PHY_CAP_DOPPLER_TX(pbuf, 1);
+
+	if (proto_cap->doppler_rx)
+		SET_HE_PHY_CAP_DOPPLER_RX(pbuf, 1);
+
+#ifdef CONFIG_FULL_BW_UL_MU_MIMO
+	SET_HE_PHY_CAP_FULL_BW_UL_MUMIMO(pbuf, 1);
+#endif
+
+	if (proto_cap->dcm_max_const_tx)
+		SET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_TX(pbuf,
+			proto_cap->dcm_max_const_tx);
+
+	if (proto_cap->dcm_max_nss_tx)
+		SET_HE_PHY_CAP_DCM_MAX_NSS_TX(pbuf, 1);
+
+	if (proto_cap->dcm_max_const_rx)
+		SET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_RX(pbuf,
+			proto_cap->dcm_max_const_rx);
+
+	if (proto_cap->dcm_max_nss_rx)
+		SET_HE_PHY_CAP_DCM_MAX_NSS_RX(pbuf, 1);
+
+	if (proto_cap->partial_bw_su_in_mu)
+		SET_HE_PHY_CAP_RX_PARTIAL_BW_SU_IN_20MHZ_MUPPDU(pbuf, 1);
+
+	if (proto_cap->he_su_bfmr)
+		SET_HE_PHY_CAP_SU_BFER(pbuf, 1);
+
+	if (proto_cap->he_su_bfme)
+		SET_HE_PHY_CAP_SU_BFEE(pbuf, 1);
+
+	if (proto_cap->he_mu_bfmr)
+		SET_HE_PHY_CAP_MU_BFER(pbuf, 1);
+
+	if (proto_cap->bfme_sts)
+		SET_HE_PHY_CAP_BFEE_STS_LESS_THAN_80MHZ(pbuf,
+			proto_cap->bfme_sts);
+
+	if (proto_cap->bfme_sts_greater_80mhz)
+		SET_HE_PHY_CAP_BFEE_STS_GREATER_THAN_80MHZ(pbuf,
+			proto_cap->bfme_sts_greater_80mhz);
+
+	if (proto_cap->num_snd_dim)
+		SET_HE_PHY_CAP_NUM_SND_DIMEN_LESS_THAN_80MHZ(pbuf,
+			proto_cap->num_snd_dim);
+
+	if (proto_cap->num_snd_dim_greater_80mhz)
+		SET_HE_PHY_CAP_NUM_SND_DIMEN_GREATER_THAN_80MHZ(pbuf,
+			proto_cap->num_snd_dim_greater_80mhz);
+
+	if (proto_cap->ng_16_su_fb)
+		SET_HE_PHY_CAP_NG_16_SU_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->ng_16_mu_fb)
+		SET_HE_PHY_CAP_NG_16_MU_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->cb_sz_su_fb)
+		SET_HE_PHY_CAP_CODEBOOK_4_2_SU_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->cb_sz_mu_fb)
+		SET_HE_PHY_CAP_CODEBOOK_7_5_MU_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->trig_su_bfm_fb)
+		SET_HE_PHY_CAP_TRIG_SUBF_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->trig_mu_bfm_fb)
+		SET_HE_PHY_CAP_TRIG_MUBF_PARTIAL_BW_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->trig_cqi_fb)
+		SET_HE_PHY_CAP_TRIG_CQI_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->partial_bw_su_er)
+		SET_HE_PHY_CAP_PARTIAL_BW_EXT_RANGE(pbuf, 1);
+
+	if (proto_cap->pwr_bst_factor)
+		SET_HE_PHY_CAP_PWR_BOOST_FACTOR_SUPPORT(pbuf, 1);
+
+	SET_HE_PHY_CAP_SU_MU_PPDU_4X_LTF_0_POINT_8_GI(pbuf, 1);
+
+	if (proto_cap->max_nc)
+		SET_HE_PHY_CAP_MAX_NC(pbuf, proto_cap->max_nc);
+
+	if (proto_cap->stbc_tx_greater_80mhz)
+		SET_HE_PHY_CAP_STBC_TX_GREATER_THAN_80MHZ(pbuf, 1);
+
+	if (proto_cap->stbc_rx_greater_80mhz)
+		SET_HE_PHY_CAP_STBC_RX_GREATER_THAN_80MHZ(pbuf, 1);
+
+	SET_HE_PHY_CAP_ERSU_PPDU_4X_LTF_0_POINT_8_GI(pbuf, 1);
+	SET_HE_PHY_CAP_ERSU_PPDU_1X_LTF_0_POINT_8_GI(pbuf, 1);
+
+	if (proto_cap->dcm_max_ru)
+		SET_HE_PHY_CAP_DCM_MAX_RU(pbuf, proto_cap->dcm_max_ru);
+
+	if (proto_cap->long_sigb_symbol)
+		SET_HE_PHY_CAP_LONGER_THAN_16_HESIGB_OFDM_SYM(pbuf, 1);
+
+	if (proto_cap->non_trig_cqi_fb)
+		SET_HE_PHY_CAP_NON_TRIGGER_CQI_FEEDBACK(pbuf, 1);
+
+	if (proto_cap->tx_1024q_ru)
+		SET_HE_PHY_CAP_TX_1024_QAM_LESS_THAN_242_TONE_RU(pbuf, 1);
+
+	if (proto_cap->rx_1024q_ru)
+		SET_HE_PHY_CAP_RX_1024_QAM_LESS_THAN_242_TONE_RU(pbuf, 1);
+
+	if (proto_cap->fbw_su_using_mu_cmprs_sigb)
+		SET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_CMP_SIGB(pbuf, 1);
+
+	if (proto_cap->fbw_su_using_mu_non_cmprs_sigb)
+		SET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_NONCMP_SIGB(pbuf, 1);
+
+	if (proto_cap->pkt_padding)
+		SET_HE_PHY_CAP_NOMINAL_PACKET_PADDING(pbuf,
+			proto_cap->pkt_padding);
+
+	return info_len;
+}
+
+static int rtw_build_he_supp_mcs(_adapter *padapter, struct protocol_cap_t *proto_cap, u8 *pbuf, enum band_type band)
+{
+
+	 /* struct rtw_chan_def *chan_def = &(wrole->chandef); */
+
+	/* Set HE Supported MCS and NSS Set */
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	int info_len = 4;
+
+	_rtw_memset(pbuf, HE_MSC_NOT_SUPP_BYTE, info_len);
+
+	_rtw_memcpy(pbuf, proto_cap->he_rx_mcs, HE_CAP_ELE_SUPP_MCS_LEN_RX_80M);
+
+	_rtw_memcpy(pbuf + 2, proto_cap->he_tx_mcs, HE_CAP_ELE_SUPP_MCS_LEN_TX_80M);
+
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) &&
+	    ((band == BAND_ON_5G && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) ||
+	     (band == BAND_ON_6G && REGSTY_IS_BW_6G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)))
+	    ) {
+		_rtw_memcpy(pbuf + 4, proto_cap->he_rx_mcs + 2, HE_CAP_ELE_SUPP_MCS_LEN_RX_160M);
+
+		_rtw_memcpy(pbuf + 6, proto_cap->he_tx_mcs + 2, HE_CAP_ELE_SUPP_MCS_LEN_TX_160M);
+
+		info_len += 4;
+	}
+	return info_len;
+}
+
+static int rtw_build_he_ppe_thre(struct protocol_cap_t *proto_cap, u8 *pbuf)
+{
+	/* Set HE PPE Thresholds (optional) */
+
+	int info_len = 0;
+
+	return info_len;
+}
+
+u32 rtw_get_dft_he_cap_ie(_adapter *padapter, struct phy_cap_t *phy_cap,
+		struct protocol_cap_t *proto_cap, u8 *pbuf, enum band_type band)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct he_priv	*phepriv = &pmlmepriv->dev_hepriv;
+	u32 he_cap_total_len = 0, len = 0;
+	u8* pcap_start = phepriv->he_cap;
+	u8* pcap = pcap_start;
+
+	_rtw_memset(pcap, 0, HE_CAP_ELE_MAX_LEN);
+
+	/* Ele ID Extension */
+	*pcap++ = WLAN_EID_EXTENSION_HE_CAPABILITY;
+
+	/* HE MAC Caps */
+	pcap += rtw_build_he_mac_caps(proto_cap, pcap);
+
+	/* HE PHY Caps */
+	pcap += rtw_build_he_phy_caps(padapter, proto_cap, pcap, band);
+
+	/* HE Supported MCS and NSS Set */
+	pcap += rtw_build_he_supp_mcs(padapter, proto_cap, pcap, band);
+
+	/* HE PPE Thresholds (optional) */
+	pcap += rtw_build_he_ppe_thre(proto_cap, pcap);
+
+	he_cap_total_len = (pcap - pcap_start);
+
+	pbuf = rtw_set_ie(pbuf, WLAN_EID_EXTENSION, he_cap_total_len, pcap_start, &len);
+
+	return len;
+}
+
+u32 rtw_build_he_cap_ie(_adapter *padapter,
+					struct _ADAPTER_LINK *padapter_link,
+					u8 *pbuf, enum band_type band)
+{
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct protocol_cap_t *proto_cap = &(padapter_link->wrlink->protocol_cap);
+	struct he_priv	*phepriv = &pmlmepriv->hepriv;
+	u32 he_cap_total_len = 0, len = 0;
+	u8* pcap_start = phepriv->he_cap;
+	u8* pcap = pcap_start;
+
+	_rtw_memset(pcap, 0, HE_CAP_ELE_MAX_LEN);
+
+	/* Ele ID Extension */
+	*pcap++ = WLAN_EID_EXTENSION_HE_CAPABILITY;
+
+	/* HE MAC Caps */
+	pcap += rtw_build_he_mac_caps(proto_cap, pcap);
+
+	/* HE PHY Caps */
+	pcap += rtw_build_he_phy_caps(padapter, proto_cap, pcap, band);
+
+	/* HE Supported MCS and NSS Set */
+	pcap += rtw_build_he_supp_mcs(padapter, proto_cap, pcap, band);
+
+	/* HE PPE Thresholds (optional) */
+	pcap += rtw_build_he_ppe_thre(proto_cap, pcap);
+
+	he_cap_total_len = (pcap - pcap_start);
+
+	pbuf = rtw_set_ie(pbuf, WLAN_EID_EXTENSION, he_cap_total_len, pcap_start, &len);
+
+	return len;
+}
+
+u32 rtw_build_he_6g_band_cap_ie_by_proto(_adapter *padapter, enum role_type role_type,
+	struct protocol_cap_t *proto_cap, u8 *pbuf)
+{
+	/* pbuf[-2]  = 0xff
+	 * pbuf[-1]  = 0x03
+	 * pbuf[0]   = 59
+	 * pbuf[1-2] = he_6g_cap
+	 *
+	 * This function fills pbuf[0...2], but not guarantee values of
+	 * pbuf[-2...-1]
+	 */
+	u8 ant_pat;
+
+	if (rtw_phl_is_ap_category(role_type))
+		ant_pat = 1;
+	else
+		ant_pat = 0;
+
+	*pbuf++ = WLAN_EID_EXT_HE_6G_CAP;
+	SET_HE_6G_BAND_CAP_MAX_AMPDU_LEN_EXP(pbuf, proto_cap->ampdu_len_exp);
+	SET_HE_6G_BAND_CAP_MAX_MPDU_LEN(pbuf, proto_cap->max_amsdu_len);
+	SET_HE_6G_BAND_CAP_SM_PS(pbuf, proto_cap->sm_ps);
+	SET_HE_6G_BAND_CAP_RX_ANT_PATTERN(pbuf, ant_pat);
+	SET_HE_6G_BAND_CAP_TX_ANT_PATTERN(pbuf, ant_pat);
+
+	return HE_6G_BAND_CAP_MAX_LEN + 2;
+}
+
+u32 rtw_build_he_6g_band_cap_ie(_adapter *padapter,
+					struct _ADAPTER_LINK *padapter_link,
+					u8 *pbuf)
+{
+	struct protocol_cap_t *proto_cap = &(padapter_link->wrlink->protocol_cap);
+	enum role_type role_type = PHL_RTYPE_NONE;
+
+	if (padapter->phl_role)
+		role_type = padapter->phl_role->type;
+
+	return rtw_build_he_6g_band_cap_ie_by_proto(padapter, role_type, proto_cap, pbuf);
+}
+
+u32 rtw_restructure_he_ie(_adapter *padapter,
+                                        struct _ADAPTER_LINK *padapter_link,
+                                        u8 *in_ie, u8 *out_ie, uint in_len,
+                                        uint *pout_len, enum band_type band)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct he_priv	*phepriv = &pmlmepriv->hepriv;
+	u32	ielen;
+	u8 *out_he_op_ie, *he_cap_ie, *he_op_ie;
+	u8 he_cap_eid_ext = WLAN_EID_EXTENSION_HE_CAPABILITY;
+	u8 he_op_eid_ext = WLAN_EID_EXTENSION_HE_OPERATION;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 *he_6g_bcap_ie;
+	u8 he_6g_bcap_eid_ext = WLAN_EID_EXT_HE_6G_CAP;
+#endif
+
+	rtw_he_use_default_setting(padapter, padapter_link);
+
+	he_cap_ie = rtw_get_ie_ex(in_ie + 12, in_len - 12, WLAN_EID_EXTENSION, &he_cap_eid_ext, 1, NULL, &ielen);
+	if (!he_cap_ie || (ielen > (HE_CAP_ELE_MAX_LEN + 2)))
+		goto exit;
+	*pout_len += rtw_build_he_cap_ie(padapter, padapter_link, out_ie + *pout_len, band);
+
+	he_op_ie = rtw_get_ie_ex(in_ie + 12, in_len - 12, WLAN_EID_EXTENSION, &he_op_eid_ext, 1, out_ie + *pout_len, &ielen);
+	if (!he_op_ie || (ielen > (HE_OPER_ELE_MAX_LEN + 2)))
+		goto exit;
+	*pout_len += ielen;
+
+	phepriv->he_option = _TRUE;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	he_6g_bcap_ie = rtw_get_ie_ex(in_ie + 12, in_len - 12, WLAN_EID_EXTENSION, &he_6g_bcap_eid_ext, 1, out_ie + *pout_len, &ielen);
+	if (!he_6g_bcap_ie || (ielen > (HE_6G_BAND_CAP_MAX_LEN + 2)))
+		goto exit;
+	*pout_len += rtw_build_he_6g_band_cap_ie(padapter, padapter_link, out_ie + *pout_len + 2);
+#endif
+
+exit:
+	return phepriv->he_option;
+}
+
+static int rtw_build_he_oper_params(_adapter *padapter, u8 *pbuf, bool he_6g_op_present)
+{
+	/* Set HE Operation Parameters */
+
+	int info_len = HE_OPER_PARAMS_LEN;
+
+	SET_HE_OP_PARA_DEFAULT_PE_DUR(pbuf, 0x4);
+	SET_HE_OP_PARA_TXOP_DUR_RTS_THRESHOLD(pbuf, 0x3ff);
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (he_6g_op_present)
+		SET_HE_OP_PARA_6GHZ_OP_INFO_PRESENT(pbuf, 0x1);
+#endif
+
+	return info_len;
+}
+
+static int rtw_build_he_oper_bss_color_info(_adapter *padapter,
+					struct _ADAPTER_LINK *padapter_link,
+					u8 *pbuf)
+{
+	/* Set BSS Color Information */
+	int info_len = HE_OPER_BSS_COLOR_INFO_LEN;
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct protocol_cap_t *proto_cap = &(padapter_link->wrlink->protocol_cap);
+
+	SET_HE_OP_BSS_COLOR_INFO_BSS_COLOR(pbuf, proto_cap->bsscolor);
+
+	return info_len;
+}
+
+static int rtw_build_he_oper_basic_mcs_set(_adapter *padapter, u8 *pbuf)
+{
+	/* Set Basic HE-MCS and NSS Set */
+
+	int info_len = HE_OPER_BASIC_MCS_LEN;
+
+	_rtw_memset(pbuf, HE_MSC_NOT_SUPP_BYTE, info_len);
+
+	SET_HE_OP_BASIC_MCS_1SS(pbuf, HE_MCS_SUPP_MSC0_TO_MSC11);
+	SET_HE_OP_BASIC_MCS_2SS(pbuf, HE_MCS_SUPP_MSC0_TO_MSC11);
+
+	return info_len;
+}
+
+static int rtw_build_vht_oper_info(_adapter *padapter, u8 *pbuf)
+{
+	/* Set VHT Operation Information (optional) */
+
+	int info_len = 0;
+
+	return info_len;
+}
+
+static int rtw_build_max_cohost_bssid_ind(_adapter *padapter, u8 *pbuf)
+{
+	/* Set Max Co-Hosted BSSID Indicator (optional) */
+
+	int info_len = 0;
+
+	return info_len;
+}
+
+static int rtw_build_6g_oper_info(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+	u8 *pbuf, struct rtw_chan_def *chandef)
+{
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	int info_len = 5;
+
+	/* Set 6GHz Operation Information */
+	SET_HE_OP_INFO_PRIMARY_CHAN(pbuf, chandef->chan);
+
+	if (!chandef->center_freq1)
+		SET_HE_OP_INFO_CENTER_FREQ_0(pbuf, chandef->chan);
+	else
+		SET_HE_OP_INFO_CENTER_FREQ_0(pbuf, chandef->center_freq1);
+	SET_HE_OP_INFO_CENTER_FREQ_1(pbuf, chandef->center_freq2);
+	SET_HE_OP_INFO_CHAN_WIDTH(pbuf, chandef->bw);
+	SET_HE_OP_INFO_MIN_RATE(pbuf, 0x6);
+
+	/* Set 6GHz Operation Information (optional) */
+	return info_len;
+}
+
+u32	rtw_build_he_operation_ie(_adapter *padapter,
+				struct _ADAPTER_LINK *padapter_link,
+				u8 *pbuf, struct rtw_chan_def *chandef)
+{
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct he_priv		*phepriv = &pmlmepriv->hepriv;
+	u32 he_oper_total_len = 0, len = 0;
+	u8* poper_start = phepriv->he_op;
+	u8* poper = poper_start;
+	bool he_6g_op_present = false;
+
+	if (chandef->band == BAND_ON_6G)
+		he_6g_op_present = true;
+
+	_rtw_memset(poper, 0, HE_OPER_ELE_MAX_LEN);
+
+	/* Ele ID Extension */
+	*poper++ = WLAN_EID_EXTENSION_HE_OPERATION;
+
+	/* HE Oper Params */
+	poper += rtw_build_he_oper_params(padapter, poper, he_6g_op_present);
+
+	/* BSS Color Info */
+	poper += rtw_build_he_oper_bss_color_info(padapter, padapter_link, poper);
+
+	/* Basic MCS and NSS Set */
+	poper += rtw_build_he_oper_basic_mcs_set(padapter, poper);
+
+	/* VHT Oper Info */
+	poper += rtw_build_vht_oper_info(padapter, poper);
+
+	/* Max Co-Hosted BSSID Indicator */
+	poper += rtw_build_max_cohost_bssid_ind(padapter, poper);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* 6G Oper Info */
+	if (he_6g_op_present)
+		poper += rtw_build_6g_oper_info(padapter, padapter_link, poper, chandef);
+#endif
+
+	he_oper_total_len = (poper - poper_start);
+
+	pbuf = rtw_set_ie(pbuf, WLAN_EID_EXTENSION, he_oper_total_len, poper_start, &len);
+
+	return len;
+}
+
+void HEOnAssocRsp(_adapter *padapter)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv		*pvhtpriv = &pmlmepriv->vhtpriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8	ht_AMPDU_len;
+
+	if (!pmlmeinfo->VHT_enable)
+		return;
+
+	if (!pmlmeinfo->HE_enable)
+		return;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/* AMPDU related settings here ? */
+}
+
+void rtw_he_ies_attach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork, enum band_type band)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	u8 he_cap_eid_ext = WLAN_EID_EXTENSION_HE_CAPABILITY;
+	u8 cap_len, operation_len;
+	uint len = 0;
+	sint ie_len = 0;
+	u8 *p = NULL;
+
+	p = rtw_get_ie_ex(pnetwork->IEs + _BEACON_IE_OFFSET_, pnetwork->IELength - _BEACON_IE_OFFSET_,
+		WLAN_EID_EXTENSION, &he_cap_eid_ext, 1, NULL, &ie_len);
+	if (p && ie_len > 0)
+		return;
+
+	rtw_he_use_default_setting(padapter, padapter_link);
+	cap_len = rtw_build_he_cap_ie(padapter, padapter_link, pnetwork->IEs + pnetwork->IELength, band);
+
+	pnetwork->IELength += cap_len;
+
+	operation_len = rtw_build_he_operation_ie(padapter, padapter_link, pnetwork->IEs + pnetwork->IELength,
+				&padapter_link->mlmeextpriv.chandef);
+	pnetwork->IELength += operation_len;
+
+	pmlmepriv->hepriv.he_option = _TRUE;
+}
+
+void rtw_he_ies_detach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	u8 he_cap_eid_ext = WLAN_EID_EXTENSION_HE_CAPABILITY;
+	u8 he_op_eid_ext = WLAN_EID_EXTENSION_HE_OPERATION;
+
+	rtw_remove_bcn_ie_ex(padapter, pnetwork, WLAN_EID_EXTENSION, &he_cap_eid_ext, 1);
+	rtw_remove_bcn_ie_ex(padapter, pnetwork, WLAN_EID_EXTENSION, &he_op_eid_ext, 1);
+
+	pmlmepriv->hepriv.he_option = _FALSE;
+}
+
+u8 rtw_he_htc_en(_adapter *padapter, struct sta_info *psta)
+{
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+
+	if (rtw_phl_is_ap_category(wrole->type))
+		return 0;
+
+	return 1;
+}
+
+void rtw_he_fill_htc(_adapter *padapter, struct pkt_attrib *pattrib, u32 *phtc_buf)
+{
+	struct link_mlme_priv *pmlmepriv = &pattrib->adapter_link->mlmepriv;
+	struct rtw_he_actrl_om *cur_om_info = &(pmlmepriv->hepriv.om_info);
+
+	SET_HE_VAR_HTC(phtc_buf);
+	SET_HE_VAR_HTC_CID_CAS(phtc_buf);
+
+	/* CONFIG_80211AX_HE_TODO */
+
+	if ((pattrib->type == WIFI_DATA_TYPE &&
+		cur_om_info->actrl_om_normal_tx &&
+		cur_om_info->actrl_om_normal_tx_cnt != 0) ||
+		pattrib->type == WIFI_MGT_TYPE) {
+
+		SET_HE_VAR_HTC_CID_OM(phtc_buf);
+		SET_HE_VAR_HTC_OM_RX_NSS(phtc_buf, cur_om_info->om_actrl_ele.rx_nss);
+		SET_HE_VAR_HTC_OM_CH_WIDTH(phtc_buf, cur_om_info->om_actrl_ele.channel_width);
+		SET_HE_VAR_HTC_OM_UL_MU_DIS(phtc_buf, cur_om_info->om_actrl_ele.ul_mu_disable);
+		SET_HE_VAR_HTC_OM_TX_NSTS(phtc_buf, cur_om_info->om_actrl_ele.tx_nsts);
+		SET_HE_VAR_HTC_OM_ER_SU_DIS(phtc_buf, cur_om_info->om_actrl_ele.er_su_disable);
+		SET_HE_VAR_HTC_OM_DL_MU_MIMO_RR(phtc_buf, cur_om_info->om_actrl_ele.dl_mu_mimo_rr);
+		SET_HE_VAR_HTC_OM_UL_MU_DATA_DIS(phtc_buf, cur_om_info->om_actrl_ele.ul_mu_data_disable);
+		if (cur_om_info->actrl_om_normal_tx_cnt) {
+			/*RTW_INFO("%s, cur_om_info->actrl_om_normal_tx_cnt=%d\n", __func__, cur_om_info->actrl_om_normal_tx_cnt);*/
+			cur_om_info->actrl_om_normal_tx_cnt --;
+		}
+	}
+
+}
+
+void rtw_he_set_om_info(_adapter *padapter, struct _ADAPTER_LINK *alink, u8 om_mask, struct rtw_he_actrl_om *om_info)
+{
+	struct link_mlme_priv *pmlmepriv = &(alink->mlmepriv);
+	struct rtw_he_actrl_om *cur_om_info = &(pmlmepriv->hepriv.om_info);
+
+	if (om_mask & OM_RX_NSS)
+		cur_om_info->om_actrl_ele.rx_nss = om_info->om_actrl_ele.rx_nss;
+
+	if (om_mask & OM_CH_BW)
+		cur_om_info->om_actrl_ele.channel_width= om_info->om_actrl_ele.channel_width;
+
+	if (om_mask & OM_UL_MU_DIS)
+		cur_om_info->om_actrl_ele.ul_mu_disable= om_info->om_actrl_ele.ul_mu_disable;
+
+	if (om_mask & OM_TX_NSTS)
+		cur_om_info->om_actrl_ele.tx_nsts= om_info->om_actrl_ele.tx_nsts;
+
+	if (om_mask & OM_ER_SU_DIS)
+		cur_om_info->om_actrl_ele.er_su_disable= om_info->om_actrl_ele.er_su_disable;
+
+	if (om_mask & OM_DL_MU_RR)
+		cur_om_info->om_actrl_ele.dl_mu_mimo_rr= om_info->om_actrl_ele.dl_mu_mimo_rr;
+
+	if (om_mask & OM_UL_MU_DATA_DIS)
+		cur_om_info->om_actrl_ele.ul_mu_data_disable= om_info->om_actrl_ele.ul_mu_data_disable;
+
+	cur_om_info->actrl_om_normal_tx = om_info->actrl_om_normal_tx;
+	cur_om_info->actrl_om_normal_tx_cnt = om_info->actrl_om_normal_tx_cnt;
+#if 0
+	RTW_INFO("%s, cur_om_info->om_actrl_ele.rx_nss = %d\n", __func__, cur_om_info->om_actrl_ele.rx_nss);
+	RTW_INFO("%s, cur_om_info->om_actrl_ele.channel_width = %d\n", __func__, cur_om_info->om_actrl_ele.channel_width);
+	RTW_INFO("%s, cur_om_info->om_actrl_ele.ul_mu_disable = %d\n", __func__, cur_om_info->om_actrl_ele.ul_mu_disable);
+	RTW_INFO("%s, cur_om_info->om_actrl_ele.tx_nsts = %d\n", __func__, cur_om_info->om_actrl_ele.tx_nsts);
+	RTW_INFO("%s, cur_om_info->om_actrl_ele.er_su_disable = %d\n", __func__, cur_om_info->om_actrl_ele.er_su_disable);
+	RTW_INFO("%s, cur_om_info->om_actrl_ele.dl_mu_mimo_rr = %d\n", __func__, cur_om_info->om_actrl_ele.dl_mu_mimo_rr);
+	RTW_INFO("%s, cur_om_info->om_actrl_ele.ul_mu_data_disable = %d\n", __func__, cur_om_info->om_actrl_ele.ul_mu_data_disable);
+	RTW_INFO("%s, cur_om_info->actrl_om_normal_tx = %d\n", __func__, cur_om_info->actrl_om_normal_tx);
+	RTW_INFO("%s, cur_om_info->actrl_om_normal_tx_cnt = %d\n", __func__, cur_om_info->actrl_om_normal_tx_cnt);
+#endif
+}
+
+void rtw_he_init_om_info(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct rtw_wifi_role_link_t *rlink = padapter_link->wrlink;
+	struct protocol_cap_t *proto_cap = &(rlink->protocol_cap);
+	struct rtw_he_actrl_om *cur_om_info = &(pmlmepriv->hepriv.om_info);
+
+	cur_om_info->om_actrl_ele.rx_nss = proto_cap->nss_rx - 1;
+
+	switch (rlink->chandef.bw) {
+		case CHANNEL_WIDTH_20:
+			cur_om_info->om_actrl_ele.channel_width = 0;
+			break;
+		case CHANNEL_WIDTH_40:
+			cur_om_info->om_actrl_ele.channel_width = 1;
+			break;
+		case CHANNEL_WIDTH_80:
+			cur_om_info->om_actrl_ele.channel_width = 2;
+			break;
+		case CHANNEL_WIDTH_160:
+		case CHANNEL_WIDTH_80_80:
+			cur_om_info->om_actrl_ele.channel_width = 3;
+			break;
+		default:
+			RTW_WARN("%s, HE OM control not support CH BW (%d), set to 0 (20M)\n", __func__, rlink->chandef.bw);
+			cur_om_info->om_actrl_ele.channel_width = 0;
+			break;
+	}
+
+	cur_om_info->om_actrl_ele.ul_mu_disable = _FALSE;
+	cur_om_info->om_actrl_ele.tx_nsts = proto_cap->nss_tx - 1;
+	cur_om_info->om_actrl_ele.er_su_disable =  _FALSE;
+	cur_om_info->om_actrl_ele.dl_mu_mimo_rr = _FALSE;
+	cur_om_info->om_actrl_ele.ul_mu_data_disable = _FALSE;
+	cur_om_info->actrl_om_normal_tx = _FALSE;
+	cur_om_info->actrl_om_normal_tx_cnt = 0;
+
+}
+
+void rtw_update_he_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork,
+	struct rtw_chan_def *chandef)
+{
+	u8 he_cap_ie_len;
+	u8 he_cap_ie[255];
+	u8 he_cap_eid_ext = WLAN_EID_EXTENSION_HE_CAPABILITY;
+	u8 he_op_ie_len;
+	u8 he_op_ie[255];
+	u8 he_op_eid_ext = WLAN_EID_EXTENSION_HE_OPERATION;
+
+	RTW_INFO("Don't setting HE capability/operation IE from hostap, builded by driver temporarily\n");
+	rtw_he_use_default_setting(padapter, padapter_link);
+
+	rtw_remove_bcn_ie_ex(padapter, pnetwork, WLAN_EID_EXTENSION, &he_cap_eid_ext, 1);
+	he_cap_ie_len = rtw_build_he_cap_ie(padapter, padapter_link, he_cap_ie, chandef->band);
+	rtw_add_bcn_ie_ex(padapter, pnetwork, he_cap_eid_ext, he_cap_ie + 2, he_cap_ie_len - 2);
+
+	rtw_remove_bcn_ie_ex(padapter, pnetwork, WLAN_EID_EXTENSION, &he_op_eid_ext, 1);
+	he_op_ie_len = rtw_build_he_operation_ie(padapter, padapter_link, he_op_ie, chandef);
+	rtw_add_bcn_ie_ex(padapter, pnetwork, he_op_eid_ext, he_op_ie + 2, he_op_ie_len - 2);
+}
+
+void rtw_update_probe_rsp_he_cap_and_op(struct _ADAPTER *a, u8 *ies, sint *ies_len, enum band_type band)
+{
+	struct _ADAPTER_LINK *a_link = GET_PRIMARY_LINK(a);
+	u8 cur_he_cap_ie[255], cur_he_op_ie[255];
+	uint cur_he_cap_ie_len, cur_he_op_ie_len;
+	u8 *he_cap_ie, *he_op_ie;
+	uint ie_len;
+	u8 he_cap_eid_ext = WLAN_EID_EXTENSION_HE_CAPABILITY;
+	u8 he_op_eid_ext = WLAN_EID_EXTENSION_HE_OPERATION;
+
+	he_cap_ie = rtw_get_ie_ex(ies, *ies_len, WLAN_EID_EXTENSION,
+							&he_cap_eid_ext, 1, NULL, &ie_len);
+	if (he_cap_ie) {
+		cur_he_cap_ie_len = rtw_build_he_cap_ie(a, a_link, cur_he_cap_ie, band);
+		rtw_ies_update_ie_ex(ies, ies_len, 0, WLAN_EID_EXTENSION_HE_CAPABILITY,
+						cur_he_cap_ie, cur_he_cap_ie_len);
+	}
+
+	he_op_ie = rtw_get_ie_ex(ies, *ies_len, WLAN_EID_EXTENSION,
+							&he_op_eid_ext, 1, NULL, &ie_len);
+	if (he_op_ie) {
+		cur_he_op_ie_len = rtw_build_he_operation_ie(a, a_link,
+					cur_he_op_ie, &a_link->mlmeextpriv.chandef);
+		rtw_ies_update_ie_ex(ies, ies_len, 0, WLAN_EID_EXTENSION_HE_OPERATION,
+					cur_he_op_ie, cur_he_op_ie_len);
+	}
+}
+
+void rtw_he_om_ctrl_trx_ss(_adapter *adapter, struct _ADAPTER_LINK *alink,
+							struct sta_info *sta, u8 final_ss, bool need_update_ra)
+{
+	struct rtw_he_actrl_om om_info = {0};
+	u8 om_mask;
+
+	om_mask = OM_RX_NSS | OM_TX_NSTS;
+
+	RTW_INFO("%s, issue qos null data to update OMI rxnss\n", __func__);
+	if (alink->mlmeextpriv.chandef.bw == CHANNEL_WIDTH_160 ||
+		alink->mlmeextpriv.chandef.bw == CHANNEL_WIDTH_80_80)
+		/* 11ax D8 9.2.4.6a.2 OM Control */
+		/* 160/80+80 */
+		om_info.om_actrl_ele.rx_nss = final_ss;
+	else
+		/* 20/40/80 */
+		om_info.om_actrl_ele.rx_nss = final_ss - 1;
+
+	om_info.om_actrl_ele.tx_nsts = final_ss - 1;
+
+	rtw_he_set_om_info(adapter, alink, om_mask, &om_info);
+	issue_qos_nulldata(adapter, alink, NULL, 0, 0, 3, 10, _TRUE);
+
+	if (need_update_ra)
+		rtw_sta_hal_ra_mask_update_cmd(adapter, sta, RTW_CMDF_DIRECTLY);
+}
+
+void rtw_process_he_triggerframe(_adapter *padapter,
+				union recv_frame *precv_frame)
+{
+
+	void *phl;
+
+	struct sta_info *psta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	u8 *trigger_frame = precv_frame->u.hdr.rx_data;
+	u16 trigger_length = precv_frame->u.hdr.len;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct wlan_network *cur_network = &(padapter_link->mlmepriv.cur_network);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u16 aid = 0;
+	u8 *user_info;
+	u16 remain_length = 0;
+	u8 trigger_type = 0;
+	bool ra_is_bc = _FALSE;
+	u8 trigger_bw = 0;
+	u16 trigger_rua = 0;
+	phl = GET_PHL_INFO(d);
+
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _FALSE)
+		return;
+
+	/* check length */
+	if (trigger_length < TRIGGER_FRAME_MIN_LENGTH) {
+		RTW_INFO("%s [T_Frame]TRIGGER_FRAME_MIN_LENGTH(%d) trigger_length=%d\n",
+			 __func__,
+			 TRIGGER_FRAME_MIN_LENGTH,
+			 trigger_length);
+		return;
+	}
+
+	/* Check TA : from connected AP*/
+	if(_rtw_memcmp(get_addr2_ptr(trigger_frame), cur_network->network.MacAddress, ETH_ALEN) == _FALSE) {
+		RTW_INFO("%s [T_Frame] Trigger Frame error, not from connected AP\n", __func__);
+		return;
+	}
+
+	/* parsing trigger frame sub-type*/
+	trigger_type = GET_TRIGGER_FRAME_TYPE(trigger_frame);
+	trigger_bw = GET_TRIGGER_FRAME_COM_INFO_BW(trigger_frame);
+	switch (trigger_type) {
+	case TRIGGER_FRAME_T_BASIC:
+	case TRIGGER_FRAME_T_MUBAR:
+		{
+			#ifdef RTW_WKARD_TRIGGER_FRAME_PARSER
+			user_info = trigger_frame + 24;
+			remain_length = trigger_length - 24;
+			/*ToDo CONFIG_RTW_MLD: tigger frame is BC, need check*/
+			psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(trigger_frame));
+			if (psta == NULL)
+				break;
+
+			phl_sta = psta->phl_sta;
+			if(phl_sta == NULL)
+				break;
+
+			/* start from User Info */
+			while (remain_length >= TRIGGER_FRAME_BASIC_USER_INFO_SZ) {
+				aid = GET_TRIGGER_FRAME_USER_INFO_AID12(user_info);
+				trigger_rua =  GET_TRIGGER_FRAME_USER_INFO_RUA(user_info);
+				RTW_DBG("%s [T_Frame] aid=0x%x, UL MCS=0x%x, RU_alloc=0x%x, BW=0x%x \n",
+					__func__, aid, GET_TRIGGER_FRAME_USER_INFO_UL_MCS(user_info), trigger_rua, trigger_bw);
+				if ((aid == phl_sta->aid) && (aid != 0)) {
+					phl_sta->stats.rx_tf_cnt++;
+					RTW_DBG("%s [T_Frame]phl_sta->stats.rx_tf_cnt(%d)\n",
+						 __func__,
+						 phl_sta->stats.rx_tf_cnt);
+					#ifdef RTW_WKARD_TRIGGER_PWR_DIFF_LARGE
+					if (trigger_bw == TRIGGER_UL_BW_80_80_160 && (trigger_rua >> 1) <= RTW_HE_RU106_8) {
+						phl_sta->flag_pwr_diff_large = true;
+						RTW_DBG("%s, phl_sta->flag_pwr_diff_large=%d\n", __func__, phl_sta->flag_pwr_diff_large);
+					}
+					#endif
+					break;
+				}
+				if (aid == 0xfff) {
+					/* padding content, break it */
+					break;
+				}
+				/* shift to next user info */
+				user_info += TRIGGER_FRAME_BASIC_USER_INFO_SZ;
+				remain_length -= TRIGGER_FRAME_BASIC_USER_INFO_SZ;
+			}
+			#endif /*RTW_WKARD_TRIGGER_FRAME_PARSER*/
+		}
+		break;
+	case TRIGGER_FRAME_T_BFRP:
+		/* fall through */
+	case TRIGGER_FRAME_T_MURTS:
+		/* fall through */
+	case TRIGGER_FRAME_T_BSRP:
+		/* fall through */
+	case TRIGGER_FRAME_T_GCR_MUBAR:
+		/* fall through */
+	case TRIGGER_FRAME_T_BQRP:
+		/* fall through */
+	case TRIGGER_FRAME_T_NFRP:
+		/* fall through */
+	case TRIGGER_FRAME_T_RSVD:
+		break;
+	}
+}
+#endif /* CONFIG_80211AX_HE */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ieee80211.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ieee80211.c
new file mode 100644
index 000000000000..50928edd5f93
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ieee80211.c
@@ -0,0 +1,3766 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _IEEE80211_C
+
+#include <drv_types.h>
+
+u8 RTW_WPA_OUI_TYPE[] = { 0x00, 0x50, 0xf2, 1 };
+u16 RTW_WPA_VERSION = 1;
+u8 WPA_AUTH_KEY_MGMT_NONE[] = { 0x00, 0x50, 0xf2, 0 };
+u8 WPA_AUTH_KEY_MGMT_UNSPEC_802_1X[] = { 0x00, 0x50, 0xf2, 1 };
+u8 WPA_AUTH_KEY_MGMT_PSK_OVER_802_1X[] = { 0x00, 0x50, 0xf2, 2 };
+u8 WPA_CIPHER_SUITE_NONE[] = { 0x00, 0x50, 0xf2, 0 };
+u8 WPA_CIPHER_SUITE_WEP40[] = { 0x00, 0x50, 0xf2, 1 };
+u8 WPA_CIPHER_SUITE_TKIP[] = { 0x00, 0x50, 0xf2, 2 };
+u8 WPA_CIPHER_SUITE_WRAP[] = { 0x00, 0x50, 0xf2, 3 };
+u8 WPA_CIPHER_SUITE_CCMP[] = { 0x00, 0x50, 0xf2, 4 };
+u8 WPA_CIPHER_SUITE_WEP104[] = { 0x00, 0x50, 0xf2, 5 };
+u8 WPA_CIPHER_SUITE_AES_CMAC[] = { 0x00, 0x50, 0xf2, 6 };
+u8 WPA_CIPHER_SUITE_GCMP[] = { 0x00, 0x50, 0xf2, 8 };
+u8 WPA_CIPHER_SUITE_GCMP_256[] = { 0x00, 0x50, 0xf2, 9 };
+u8 WPA_CIPHER_SUITE_CCMP_256[] = { 0x00, 0x50, 0xf2, 10 };
+
+u16 RSN_VERSION_BSD = 1;
+u8 RSN_CIPHER_SUITE_NONE[] = { 0x00, 0x0f, 0xac, 0 };
+u8 RSN_CIPHER_SUITE_WEP40[] = { 0x00, 0x0f, 0xac, 1 };
+u8 RSN_CIPHER_SUITE_TKIP[] = { 0x00, 0x0f, 0xac, 2 };
+/*u8 RSN_CIPHER_SUITE_WRAP[] = { 0x00, 0x0f, 0xac, 3 };*/
+u8 RSN_CIPHER_SUITE_CCMP[] = { 0x00, 0x0f, 0xac, 4 };
+u8 RSN_CIPHER_SUITE_AES_128_CMAC[] = { 0x00, 0x0f, 0xac, 6 };
+u8 RSN_CIPHER_SUITE_GCMP[] = { 0x00, 0x0f, 0xac, 8 };
+u8 RSN_CIPHER_SUITE_GCMP_256[] = { 0x00, 0x0f, 0xac, 9 };
+u8 RSN_CIPHER_SUITE_CCMP_256[] = { 0x00, 0x0f, 0xac, 10 };
+u8 RSN_CIPHER_SUITE_BIP_GMAC_128[] = { 0x00, 0x0f, 0xac, 11 };
+u8 RSN_CIPHER_SUITE_BIP_GMAC_256[] = { 0x00, 0x0f, 0xac, 12 };
+u8 RSN_CIPHER_SUITE_BIP_CMAC_256[] = { 0x00, 0x0f, 0xac, 13 };
+u8 RSN_CIPHER_SUITE_WEP104[] = { 0x00, 0x0f, 0xac, 5 };
+
+u8 WLAN_AKM_8021X[] = {0x00, 0x0f, 0xac, 1};
+u8 WLAN_AKM_PSK[] = {0x00, 0x0f, 0xac, 2};
+u8 WLAN_AKM_FT_8021X[] = {0x00, 0x0f, 0xac, 3};
+u8 WLAN_AKM_FT_PSK[] = {0x00, 0x0f, 0xac, 4};
+u8 WLAN_AKM_8021X_SHA256[] = {0x00, 0x0f, 0xac, 5};
+u8 WLAN_AKM_PSK_SHA256[] = {0x00, 0x0f, 0xac, 6};
+u8 WLAN_AKM_TDLS[] = {0x00, 0x0f, 0xac, 7};
+u8 WLAN_AKM_SAE[] = {0x00, 0x0f, 0xac, 8};
+u8 WLAN_AKM_FT_OVER_SAE[] = {0x00, 0x0f, 0xac, 9};
+u8 WLAN_AKM_8021X_SUITE_B[] = {0x00, 0x0f, 0xac, 11};
+u8 WLAN_AKM_8021X_SUITE_B_192[] = {0x00, 0x0f, 0xac, 12};
+u8 WLAN_AKM_FILS_SHA256[] = {0x00, 0x0f, 0xac, 14};
+u8 WLAN_AKM_FILS_SHA384[] = {0x00, 0x0f, 0xac, 15};
+u8 WLAN_AKM_FT_FILS_SHA256[] = {0x00, 0x0f, 0xac, 16};
+u8 WLAN_AKM_FT_FILS_SHA384[] = {0x00, 0x0f, 0xac, 17};
+
+/* -----------------------------------------------------------
+ * for adhoc-master to generate ie and provide supported-rate to fw
+ * ----------------------------------------------------------- */
+
+u8	WIFI_CCKRATES[] = {
+	(IEEE80211_CCK_RATE_1MB | IEEE80211_BASIC_RATE_MASK),
+	(IEEE80211_CCK_RATE_2MB | IEEE80211_BASIC_RATE_MASK),
+	(IEEE80211_CCK_RATE_5MB | IEEE80211_BASIC_RATE_MASK),
+	(IEEE80211_CCK_RATE_11MB | IEEE80211_BASIC_RATE_MASK)
+};
+
+u8	WIFI_OFDMRATES[] = {
+	(IEEE80211_OFDM_RATE_6MB),
+	(IEEE80211_OFDM_RATE_9MB),
+	(IEEE80211_OFDM_RATE_12MB),
+	(IEEE80211_OFDM_RATE_18MB),
+	(IEEE80211_OFDM_RATE_24MB),
+	IEEE80211_OFDM_RATE_36MB,
+	IEEE80211_OFDM_RATE_48MB,
+	IEEE80211_OFDM_RATE_54MB
+};
+
+u8 mgn_rates_cck[4] = {MGN_1M, MGN_2M, MGN_5_5M, MGN_11M};
+u8 mgn_rates_ofdm[8] = {MGN_6M, MGN_9M, MGN_12M, MGN_18M, MGN_24M, MGN_36M, MGN_48M, MGN_54M};
+u8 mgn_rates_mcs0_7[8] = {MGN_MCS0, MGN_MCS1, MGN_MCS2, MGN_MCS3, MGN_MCS4, MGN_MCS5, MGN_MCS6, MGN_MCS7};
+u8 mgn_rates_mcs8_15[8] = {MGN_MCS8, MGN_MCS9, MGN_MCS10, MGN_MCS11, MGN_MCS12, MGN_MCS13, MGN_MCS14, MGN_MCS15};
+u8 mgn_rates_mcs16_23[8] = {MGN_MCS16, MGN_MCS17, MGN_MCS18, MGN_MCS19, MGN_MCS20, MGN_MCS21, MGN_MCS22, MGN_MCS23};
+u8 mgn_rates_mcs24_31[8] = {MGN_MCS24, MGN_MCS25, MGN_MCS26, MGN_MCS27, MGN_MCS28, MGN_MCS29, MGN_MCS30, MGN_MCS31};
+u8 mgn_rates_vht1ss[10] = {MGN_VHT1SS_MCS0, MGN_VHT1SS_MCS1, MGN_VHT1SS_MCS2, MGN_VHT1SS_MCS3, MGN_VHT1SS_MCS4
+	, MGN_VHT1SS_MCS5, MGN_VHT1SS_MCS6, MGN_VHT1SS_MCS7, MGN_VHT1SS_MCS8, MGN_VHT1SS_MCS9
+			  };
+u8 mgn_rates_vht2ss[10] = {MGN_VHT2SS_MCS0, MGN_VHT2SS_MCS1, MGN_VHT2SS_MCS2, MGN_VHT2SS_MCS3, MGN_VHT2SS_MCS4
+	, MGN_VHT2SS_MCS5, MGN_VHT2SS_MCS6, MGN_VHT2SS_MCS7, MGN_VHT2SS_MCS8, MGN_VHT2SS_MCS9
+			  };
+u8 mgn_rates_vht3ss[10] = {MGN_VHT3SS_MCS0, MGN_VHT3SS_MCS1, MGN_VHT3SS_MCS2, MGN_VHT3SS_MCS3, MGN_VHT3SS_MCS4
+	, MGN_VHT3SS_MCS5, MGN_VHT3SS_MCS6, MGN_VHT3SS_MCS7, MGN_VHT3SS_MCS8, MGN_VHT3SS_MCS9
+			  };
+u8 mgn_rates_vht4ss[10] = {MGN_VHT4SS_MCS0, MGN_VHT4SS_MCS1, MGN_VHT4SS_MCS2, MGN_VHT4SS_MCS3, MGN_VHT4SS_MCS4
+	, MGN_VHT4SS_MCS5, MGN_VHT4SS_MCS6, MGN_VHT4SS_MCS7, MGN_VHT4SS_MCS8, MGN_VHT4SS_MCS9
+			  };
+u8 mgn_rates_he1ss[12] = {MGN_HE1SS_MCS0, MGN_HE1SS_MCS1, MGN_HE1SS_MCS2, MGN_HE1SS_MCS3, MGN_HE1SS_MCS4
+	, MGN_HE1SS_MCS5, MGN_HE1SS_MCS6, MGN_HE1SS_MCS7, MGN_HE1SS_MCS8, MGN_HE1SS_MCS9, MGN_HE1SS_MCS10, MGN_HE1SS_MCS11
+			  };
+u8 mgn_rates_he2ss[12] = {MGN_HE2SS_MCS0, MGN_HE2SS_MCS1, MGN_HE2SS_MCS2, MGN_HE2SS_MCS3, MGN_HE2SS_MCS4
+	, MGN_HE2SS_MCS5, MGN_HE2SS_MCS6, MGN_HE2SS_MCS7, MGN_HE2SS_MCS8, MGN_HE2SS_MCS9, MGN_HE2SS_MCS10, MGN_HE2SS_MCS11
+			  };
+u8 mgn_rates_he3ss[12] = {MGN_HE3SS_MCS0, MGN_HE3SS_MCS1, MGN_HE3SS_MCS2, MGN_HE3SS_MCS3, MGN_HE3SS_MCS4
+	, MGN_HE3SS_MCS5, MGN_HE3SS_MCS6, MGN_HE3SS_MCS7, MGN_HE3SS_MCS8, MGN_HE3SS_MCS9, MGN_HE3SS_MCS10, MGN_HE3SS_MCS11
+			  };
+u8 mgn_rates_he4ss[12] = {MGN_HE4SS_MCS0, MGN_HE4SS_MCS1, MGN_HE4SS_MCS2, MGN_HE4SS_MCS3, MGN_HE4SS_MCS4
+	, MGN_HE4SS_MCS5, MGN_HE4SS_MCS6, MGN_HE4SS_MCS7, MGN_HE4SS_MCS8, MGN_HE4SS_MCS9, MGN_HE4SS_MCS10, MGN_HE4SS_MCS11
+			  };
+u8 mgn_rates_dcm1ss[4] = {MGN_HE1SS_MCS0, MGN_HE1SS_MCS1, MGN_HE1SS_MCS3, MGN_HE1SS_MCS4};
+u8 mgn_rates_dcm2ss[4] = {MGN_HE2SS_MCS0, MGN_HE2SS_MCS1, MGN_HE2SS_MCS3, MGN_HE2SS_MCS4};
+u8 mgn_rates_dcm3ss[4] = {MGN_HE3SS_MCS0, MGN_HE3SS_MCS1, MGN_HE3SS_MCS3, MGN_HE3SS_MCS4};
+u8 mgn_rates_dcm4ss[4] = {MGN_HE4SS_MCS0, MGN_HE4SS_MCS1, MGN_HE4SS_MCS3, MGN_HE4SS_MCS4};
+u8 mgn_rates_offset[5] = {MGN_1M, MGN_6M, MGN_MCS0, MGN_VHT1SS_MCS0, MGN_HE1SS_MCS0};
+
+RATE_SECTION mgn_rate_to_rs(enum MGN_RATE rate)
+{
+	RATE_SECTION rs = RATE_SECTION_NUM;
+
+	if (IS_CCK_RATE(rate))
+		rs = CCK;
+	else if (IS_OFDM_RATE(rate))
+		rs = OFDM;
+	else if (IS_HT1SS_RATE(rate))
+		rs = HT_1SS;
+	else if (IS_HT2SS_RATE(rate))
+		rs = HT_2SS;
+	else if (IS_HT3SS_RATE(rate))
+		rs = HT_3SS;
+	else if (IS_HT4SS_RATE(rate))
+		rs = HT_4SS;
+	else if (IS_VHT1SS_RATE(rate))
+		rs = VHT_1SS;
+	else if (IS_VHT2SS_RATE(rate))
+		rs = VHT_2SS;
+	else if (IS_VHT3SS_RATE(rate))
+		rs = VHT_3SS;
+	else if (IS_VHT4SS_RATE(rate))
+		rs = VHT_4SS;
+
+	return rs;
+}
+
+static const char *const _rate_section_str[] = {
+	"CCK",
+	"OFDM",
+	"HT_1SS",
+	"HT_2SS",
+	"HT_3SS",
+	"HT_4SS",
+	"VHT_1SS",
+	"VHT_2SS",
+	"VHT_3SS",
+	"VHT_4SS",
+	"HE_1SS",
+	"HE_2SS",
+	"HE_3SS",
+	"HE_4SS",
+	"DCM_1SS",
+	"DCM_2SS",
+	"DCM_3SS",
+	"DCM_4SS",
+	"RATE_SECTION_UNKNOWN",
+};
+
+const char *rate_section_str(u8 section)
+{
+	section = (section >= RATE_SECTION_NUM) ? RATE_SECTION_NUM : section;
+	return _rate_section_str[section];
+}
+
+struct rate_section_ent rates_by_sections[RATE_SECTION_NUM] = {
+	{RF_1TX, 4, mgn_rates_cck},
+	{RF_1TX, 8, mgn_rates_ofdm},
+	{RF_1TX, 8, mgn_rates_mcs0_7},
+	{RF_2TX, 8, mgn_rates_mcs8_15},
+	{RF_3TX, 8, mgn_rates_mcs16_23},
+	{RF_4TX, 8, mgn_rates_mcs24_31},
+	{RF_1TX, 10, mgn_rates_vht1ss},
+	{RF_2TX, 10, mgn_rates_vht2ss},
+	{RF_3TX, 10, mgn_rates_vht3ss},
+	{RF_4TX, 10, mgn_rates_vht4ss},
+	{RF_1TX, 12, mgn_rates_he1ss},
+	{RF_2TX, 12, mgn_rates_he2ss},
+	{RF_3TX, 12, mgn_rates_he3ss},
+	{RF_4TX, 12, mgn_rates_he4ss},
+	{RF_1TX, 4, mgn_rates_dcm1ss},
+	{RF_2TX, 4, mgn_rates_dcm2ss},
+	{RF_3TX, 4, mgn_rates_dcm3ss},
+	{RF_4TX, 4, mgn_rates_dcm4ss},
+};
+
+int rtw_get_bit_value_from_ieee_value(u8 val)
+{
+	unsigned char dot11_rate_table[] = {2, 4, 11, 22, 12, 18, 24, 36, 48, 72, 96, 108, 0}; /* last element must be zero!! */
+
+	int i = 0;
+	while (dot11_rate_table[i] != 0) {
+		if (dot11_rate_table[i] == val)
+			return BIT(i);
+		i++;
+	}
+	return 0;
+}
+uint rtw_get_cckrate_size(u8 *rate, u32 rate_length)
+{
+	int i = 0;
+	while(i < rate_length){
+		RTW_DBG("%s, rate[%d]=%u\n", __FUNCTION__, i, rate[i]);
+		if (((rate[i] & 0x7f) == 2) || ((rate[i] & 0x7f) == 4) ||
+			((rate[i] & 0x7f) == 11)  || ((rate[i] & 0x7f) == 22))
+			i++;
+		else
+			break;
+	}
+	return i;
+}
+
+uint	rtw_is_cckrates_included(u8 *rate)
+{
+	u32	i = 0;
+
+	while (rate[i] != 0) {
+		if ((((rate[i]) & 0x7f) == 2)	|| (((rate[i]) & 0x7f) == 4) ||
+		    (((rate[i]) & 0x7f) == 11)  || (((rate[i]) & 0x7f) == 22))
+			return _TRUE;
+		i++;
+	}
+
+	return _FALSE;
+}
+
+uint	rtw_is_cckratesonly_included(u8 *rate)
+{
+	u32 i = 0;
+
+
+	while (rate[i] != 0) {
+		if ((((rate[i]) & 0x7f) != 2) && (((rate[i]) & 0x7f) != 4) &&
+		    (((rate[i]) & 0x7f) != 11)  && (((rate[i]) & 0x7f) != 22))
+			return _FALSE;
+
+		i++;
+	}
+
+	return _TRUE;
+
+}
+
+int rtw_check_network_type(unsigned char *rate, int ratelen, int channel)
+{
+	if (channel > 14) {
+		if ((rtw_is_cckrates_included(rate)) == _TRUE)
+			return WLAN_MD_INVALID;
+		else
+			return WLAN_MD_11A;
+	} else { /* could be pure B, pure G, or B/G */
+		if ((rtw_is_cckratesonly_included(rate)) == _TRUE)
+			return WLAN_MD_11B;
+		else if ((rtw_is_cckrates_included(rate)) == _TRUE)
+			return	WLAN_MD_11BG;
+		else
+			return WLAN_MD_11G;
+	}
+
+}
+
+u8 *rtw_set_fixed_ie(unsigned char *pbuf, unsigned int len, unsigned char *source,
+		     unsigned int *frlen)
+{
+	_rtw_memcpy((void *)pbuf, (void *)source, len);
+	*frlen = *frlen + len;
+	return pbuf + len;
+}
+
+/* rtw_set_ie will update frame length */
+u8 *rtw_set_ie
+(
+	u8 *pbuf,
+	sint index,
+	uint len, /* IE content length, not entire IE length */
+	const u8 *source,
+	uint *frlen /* frame length */
+)
+{
+	*pbuf = (u8)index;
+
+	*(pbuf + 1) = (u8)len;
+
+	if (len > 0)
+		_rtw_memcpy((void *)(pbuf + 2), (void *)source, len);
+
+	if (frlen)
+		*frlen = *frlen + (len + 2);
+
+	return pbuf + len + 2;
+}
+
+inline u8 *rtw_set_ie_ch_switch(u8 *buf, u32 *buf_len, u8 ch_switch_mode,
+				u8 new_ch, u8 ch_switch_cnt)
+{
+	u8 ie_data[3];
+
+	ie_data[0] = ch_switch_mode;
+	ie_data[1] = new_ch;
+	ie_data[2] = ch_switch_cnt;
+	return rtw_set_ie(buf, WLAN_EID_CHANNEL_SWITCH,  3, ie_data, buf_len);
+}
+
+inline u8 secondary_ch_offset_to_hal_ch_offset(u8 ch_offset)
+{
+	if (ch_offset == IEEE80211_SCN)
+		return CHAN_OFFSET_NO_EXT;
+	else if (ch_offset == IEEE80211_SCA)
+		return CHAN_OFFSET_UPPER;
+	else if (ch_offset == IEEE80211_SCB)
+		return CHAN_OFFSET_LOWER;
+
+	return CHAN_OFFSET_NO_EXT;
+}
+
+inline u8 hal_ch_offset_to_secondary_ch_offset(u8 ch_offset)
+{
+	if (ch_offset == CHAN_OFFSET_NO_EXT)
+		return IEEE80211_SCN;
+	else if (ch_offset == CHAN_OFFSET_UPPER)
+		return IEEE80211_SCA;
+	else if (ch_offset == CHAN_OFFSET_LOWER)
+		return IEEE80211_SCB;
+
+	return IEEE80211_SCN;
+}
+
+inline u8 *rtw_set_ie_secondary_ch_offset(u8 *buf, u32 *buf_len, u8 secondary_ch_offset)
+{
+	return rtw_set_ie(buf, WLAN_EID_SECONDARY_CHANNEL_OFFSET,  1, &secondary_ch_offset, buf_len);
+}
+
+inline u8 *rtw_set_ie_wide_bw_ch_switch(u8 *buf, u32 *buf_len,
+	u8 ch_width, u8 seg_0, u8 seg_1)
+{
+	u8 csw_data[3] = {0};
+
+	/*
+	* [0] : New Channel Width
+	* [1] : New Channel Center Frequency Segment 0
+	* [2] : New Channel Center Frequency Segment 1
+	*/
+	csw_data[0] = ch_width;
+	csw_data[1] = seg_0;
+	csw_data[2] = seg_1;
+
+	return rtw_set_ie(buf, WLAN_EID_VHT_WIDE_BW_CHSWITCH, 3, csw_data, buf_len);
+}
+
+inline u8 *rtw_set_ie_mesh_ch_switch_parm(u8 *buf, u32 *buf_len, u8 ttl,
+		u8 flags, u16 reason, u16 precedence)
+{
+	u8 ie_data[6];
+
+	ie_data[0] = ttl;
+	ie_data[1] = flags;
+	RTW_PUT_LE16((u8 *)&ie_data[2], reason);
+	RTW_PUT_LE16((u8 *)&ie_data[4], precedence);
+
+	return rtw_set_ie(buf, 0x118,  6, ie_data, buf_len);
+}
+
+/*----------------------------------------------------------------------------
+index: the information element id index, limit is the limit for search
+-----------------------------------------------------------------------------*/
+u8 *rtw_get_ie(const u8 *pbuf, sint index, sint *len, sint limit)
+{
+	sint tmp, i;
+	const u8 *p;
+	if (limit < 1) {
+		return NULL;
+	}
+
+	p = pbuf;
+	i = 0;
+	*len = 0;
+	while (1) {
+		if (*p == index) {
+			*len = *(p + 1);
+			return (u8 *)p;
+		} else {
+			tmp = *(p + 1);
+			p += (tmp + 2);
+			i += (tmp + 2);
+		}
+		if (i >= limit)
+			break;
+	}
+	return NULL;
+}
+
+u8 *rtw_get_ext_ie(const u8 *pbuf, sint ext_id, sint *len, sint limit)
+{
+	sint tmp, i;
+	const u8 *p;
+
+	if (limit < 3)
+		return NULL;
+
+	p = pbuf;
+	i = 0;
+	*len = 0;
+	do {
+		if (*p == WLAN_EID_EXTENSION && *(p + 2) == ext_id) {
+			*len = *(p + 1);
+			return (u8 *)p;
+		} else {
+			tmp = *(p + 1);
+			p += (tmp + 2);
+			i += (tmp + 2);
+		}
+	} while (i + 3 <= limit);
+
+	return NULL;
+}
+
+/**
+ * rtw_get_ie_ex - Search specific IE from a series of IEs
+ * @in_ie: Address of IEs to search
+ * @in_len: Length limit from in_ie
+ * @eid: Element ID to match
+ * @oui: OUI to match
+ * @oui_len: OUI length
+ * @ie: If not NULL and the specific IE is found, the IE will be copied to the buf starting from the specific IE
+ * @ielen: If not NULL and the specific IE is found, will set to the length of the entire IE
+ *
+ * Returns: The address of the specific IE found, or NULL
+ */
+u8 *rtw_get_ie_ex(const u8 *in_ie, uint in_len, u8 eid, const u8 *oui, u8 oui_len, u8 *ie, uint *ielen)
+{
+	uint cnt;
+	const u8 *target_ie = NULL;
+
+
+	if (ielen)
+		*ielen = 0;
+
+	if (!in_ie || in_len <= 0)
+		return (u8 *)target_ie;
+
+	cnt = 0;
+
+	while (cnt < in_len) {
+		if (eid == in_ie[cnt]
+		    && (!oui || _rtw_memcmp(&in_ie[cnt + 2], oui, oui_len) == _TRUE)) {
+			target_ie = &in_ie[cnt];
+
+			if (ie)
+				_rtw_memcpy(ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			if (ielen)
+				*ielen = in_ie[cnt + 1] + 2;
+
+			break;
+		} else {
+			cnt += in_ie[cnt + 1] + 2; /* goto next	 */
+		}
+
+	}
+
+	return (u8 *)target_ie;
+}
+
+/**
+ * rtw_ies_update_ie - Find matching IEs and update it
+ *
+ * @ies: address of IEs to search
+ * @ies_len: address of length of ies, will update to new length
+ * @offset: the offset to start scarch
+ * @eid: element ID to match
+ * @content: new content will update to matching element
+ * @content_len: length of new content
+ * Returns: _SUCCESS: ies is updated, _FAIL: not updated
+ */
+u8 rtw_ies_update_ie(u8 *ies, uint *ies_len, uint ies_offset, u8 eid, const u8 *content, u8 content_len)
+{
+	u8 ret = _FAIL;
+	u8 *target_ie;
+	u32 target_ielen;
+	u8 *start, *remain_ies = NULL, *backup_ies = NULL;
+	uint search_len, remain_len = 0;
+	sint offset;
+
+	if (ies == NULL || *ies_len == 0 || *ies_len <= ies_offset)
+		goto exit;
+
+	start = ies + ies_offset;
+	search_len = *ies_len - ies_offset;
+
+	target_ie = rtw_get_ie(start, eid, &target_ielen, search_len);
+	if (target_ie && target_ielen) {
+		if (target_ielen != content_len) {
+			remain_ies = target_ie + 2 + target_ielen;
+			remain_len = search_len - (remain_ies - start);
+
+			backup_ies = rtw_malloc(remain_len);
+			if (!backup_ies)
+				goto exit;
+
+			_rtw_memcpy(backup_ies, remain_ies, remain_len);
+		}
+
+		_rtw_memcpy(target_ie + 2, content, content_len);
+		*(target_ie + 1) = content_len;
+		ret = _SUCCESS;
+
+		if (target_ielen != content_len) {
+			remain_ies = target_ie + 2 + content_len;
+			_rtw_memcpy(remain_ies, backup_ies, remain_len);
+			rtw_mfree(backup_ies, remain_len);
+			offset = content_len - target_ielen;
+			*ies_len = *ies_len + offset;
+		}
+	}
+exit:
+	return ret;
+}
+
+u8 rtw_ies_update_ie_ex(u8 *ies,
+		uint *ies_len,
+		uint ies_offset,
+		u8 eid_ex,
+		const u8 *content,
+		u8 content_len)
+{
+	u8 ret = _FAIL;
+	u8 *target_ie;
+	u32 target_ielen;
+	u8 *start, *remain_ies = NULL, *backup_ies = NULL;
+	uint search_len, remain_len = 0;
+	sint offset;
+
+	if (ies == NULL || *ies_len == 0 || *ies_len <= ies_offset)
+		goto exit;
+
+	start = ies + ies_offset;
+	search_len = *ies_len - ies_offset;
+
+	/* | Element ID extension | Ext. tag lendth | Ext. tag number | ... | */
+	/* | target_ie | (target_ielen-2) | eid_ex | ... | */
+	target_ie = rtw_get_ie_ex(start, search_len, WLAN_EID_EXTENSION,
+							&eid_ex, 1, NULL, &target_ielen);
+	if (target_ie && target_ielen) {
+		if (target_ielen != content_len) {
+			remain_ies = target_ie + target_ielen;
+			remain_len = search_len - (remain_ies - start);
+
+			backup_ies = rtw_malloc(remain_len);
+			if (!backup_ies)
+				goto exit;
+
+			_rtw_memcpy(backup_ies, remain_ies, remain_len);
+		}
+
+		_rtw_memcpy(target_ie, content, content_len);
+		*(target_ie + 1) = content_len - 2;
+		ret = _SUCCESS;
+
+		if (target_ielen != content_len) {
+			remain_ies = target_ie + content_len;
+			_rtw_memcpy(remain_ies, backup_ies, remain_len);
+			rtw_mfree(backup_ies, remain_len);
+			offset = content_len - target_ielen;
+			*ies_len = *ies_len + offset;
+		}
+	}
+exit:
+	return ret;
+}
+
+/**
+ * rtw_ies_add_ie - append IE to existing IEs
+ *
+ * @ies: address of IEs to search
+ * @ies_len: address of length of ies, will update to new length
+ * @offset: the offset to start scarch
+ * @eid: element ID to add
+ * @content: content will be appended before the first element ID that is
+ * greater than the @eid or overwrite the existing same element ID.
+ * @content_len: length of content
+ * Returns: _SUCCESS: ies is appended, _FAIL: not appended
+ *
+ * Notes: this api does not consider the element ID like
+ * WLAN_EID_VENDOR_SPECIFIC(221), which contains different OUIs with the same
+ * element ID or like WLAN_EID_EXTENSION(255), which contains different
+ * extention tag number with the same IE.
+ */
+u8 rtw_ies_add_ie(u8 *ies, uint *ies_len, uint ies_offset, u8 eid,
+		     const u8 *content, u8 content_len)
+{
+	u8 ret = _FAIL;
+	PNDIS_802_11_VARIABLE_IEs pIE = {0};
+	u8 *pstart = NULL, *pcurrent = NULL, *premainder_id = NULL;
+	u8 *pbackup_remainder_ie = NULL, *dst_ie = NULL;
+	uint search_len = 0, remainder_ielen = 0;
+	u32 offset = 0;
+
+	if (ies == NULL || *ies_len == 0 || *ies_len <= ies_offset)
+		goto exit;
+
+	pstart = ies + ies_offset;
+	search_len = *ies_len - ies_offset;
+
+	for (pcurrent = pstart; pcurrent - pstart <= search_len;
+	     pcurrent += (pIE->Length + 2)) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)pcurrent;
+		dst_ie = pcurrent;
+
+		if (pcurrent - pstart == search_len)
+			break;
+
+		if (pIE->ElementID > eid)
+			break;
+		else if (pIE->ElementID == eid) {
+			pcurrent += (pIE->Length + 2);
+			break;
+		}
+	}
+
+	if (dst_ie == NULL)
+		goto exit;
+
+	remainder_ielen = *ies_len - (pcurrent - ies);
+	if (remainder_ielen > 0) {
+		pbackup_remainder_ie = rtw_malloc(remainder_ielen);
+		_rtw_memcpy(pbackup_remainder_ie, pcurrent, remainder_ielen);
+	}
+
+	*dst_ie++ = eid;
+	*dst_ie++ = content_len;
+
+	_rtw_memcpy(dst_ie, content, content_len);
+	dst_ie += content_len;
+
+	/* Append remainder IE */
+	if (pbackup_remainder_ie) {
+		_rtw_memcpy(dst_ie, pbackup_remainder_ie, remainder_ielen);
+		rtw_mfree(pbackup_remainder_ie, remainder_ielen);
+	}
+
+	offset = (uint)(dst_ie - ies);
+	*ies_len = offset + remainder_ielen;
+	ret = _SUCCESS;
+exit:
+	return ret;
+}
+/**
+ * rtw_ies_remove_ie - Find matching IEs and remove
+ * @ies: Address of IEs to search
+ * @ies_len: Pointer of length of ies, will update to new length
+ * @offset: The offset to start scarch
+ * @eid: Element ID to match
+ * @oui: OUI to match
+ * @oui_len: OUI length
+ *
+ * Returns: _SUCCESS: ies is updated, _FAIL: not updated
+ */
+int rtw_ies_remove_ie(u8 *ies, uint *ies_len, uint offset, u8 eid, u8 *oui, u8 oui_len)
+{
+	int ret = _FAIL;
+	u8 *target_ie;
+	u32 target_ielen;
+	u8 *start;
+	uint search_len;
+
+	if (!ies || !ies_len || *ies_len <= offset)
+		goto exit;
+
+	start = ies + offset;
+	search_len = *ies_len - offset;
+
+	while (1) {
+		target_ie = rtw_get_ie_ex(start, search_len, eid, oui, oui_len, NULL, &target_ielen);
+		if (target_ie && target_ielen) {
+			u8 *remain_ies = target_ie + target_ielen;
+			uint remain_len = search_len - (remain_ies - start);
+
+			_rtw_memmove(target_ie, remain_ies, remain_len);
+			*ies_len = *ies_len - target_ielen;
+			ret = _SUCCESS;
+
+			start = target_ie;
+			search_len = remain_len;
+		} else
+			break;
+	}
+exit:
+	return ret;
+}
+
+void rtw_set_supported_rate(u8 *SupportedRates, uint mode, u8 ch, enum band_type band)
+{
+	int is_2G_band = 1;
+	int cck_len = 0;
+
+	if (ch > 14 || band == BAND_ON_6G)
+		is_2G_band = 0;
+
+	_rtw_memset(SupportedRates, 0, NDIS_802_11_LENGTH_RATES_EX);
+
+	if (is_2G_band) {
+		if (mode & WLAN_MD_11B) {
+			_rtw_memcpy(SupportedRates, WIFI_CCKRATES, IEEE80211_CCK_RATE_LEN);
+			cck_len = IEEE80211_CCK_RATE_LEN;
+		}
+
+		if (mode & ~WLAN_MD_11B) /* NOT B only */
+			_rtw_memcpy(SupportedRates + cck_len,
+				WIFI_OFDMRATES, IEEE80211_NUM_OFDM_RATESLEN);
+	} else {/* 5G and 6G */
+		_rtw_memcpy(SupportedRates, WIFI_OFDMRATES, IEEE80211_NUM_OFDM_RATESLEN);
+	}
+}
+
+void rtw_filter_suppport_rateie(WLAN_BSSID_EX *pbss_network, u8 keep)
+{
+	u8 i, idx = 0, new_rate[NDIS_802_11_LENGTH_RATES_EX], *p;
+	uint iscck, isofdm, ie_orilen = 0, remain_len;
+	u8 *remain_ies;
+
+	p = rtw_get_ie(pbss_network->IEs + _BEACON_IE_OFFSET_, _SUPPORTEDRATES_IE_, &ie_orilen, (pbss_network->IELength - _BEACON_IE_OFFSET_));
+	if (!p)
+		return;
+
+	_rtw_memset(new_rate, 0, NDIS_802_11_LENGTH_RATES_EX);
+	for (i=0; i < ie_orilen; i++) {
+		iscck = rtw_is_cck_rate(p[i+2]);
+		isofdm= rtw_is_ofdm_rate(p[i+2]);
+		if (((keep == CCK) && iscck)
+			|| ((keep == OFDM) && isofdm))
+			new_rate[idx++]= rtw_is_basic_rate_ofdm(p[i+2]) ? p[i+2]|IEEE80211_BASIC_RATE_MASK : p[i+2];
+	}
+	/*	update rate ie	*/
+	p[1] = idx;
+	_rtw_memcpy(p+2, new_rate, idx);
+	/*	update remain ie & IELength*/
+	remain_ies = p + 2 + ie_orilen;
+	remain_len = pbss_network->IELength - (remain_ies - pbss_network->IEs);
+	_rtw_memmove(p+2+idx, remain_ies, remain_len);
+	pbss_network->IELength -= (ie_orilen - idx);
+}
+ 
+
+/*
+	Adjust those items by given wireless_mode
+		1. pbss_network->IELength
+		2. pbss_network->IE (SUPPORTRATE & EXT_SUPPORTRATE)
+		3. pbss_network->SupportedRates
+*/
+
+u8 rtw_update_rate_bymode(WLAN_BSSID_EX *pbss_network, u32 mode)
+{
+	u8 network_type, *p, *ie = pbss_network->IEs;
+	sint ie_len;
+	uint network_ielen = pbss_network->IELength;
+
+	if (mode == WLAN_MD_11B) {
+		/*only keep CCK in support_rate IE and remove whole ext_support_rate IE*/
+		rtw_filter_suppport_rateie(pbss_network, CCK);
+		p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _EXT_SUPPORTEDRATES_IE_, &ie_len, pbss_network->IELength - _BEACON_IE_OFFSET_);
+		if (p) {
+			rtw_ies_remove_ie(ie , &network_ielen, _BEACON_IE_OFFSET_, _EXT_SUPPORTEDRATES_IE_, NULL, 0);
+			pbss_network->IELength -= ie_len;
+		}
+		network_type = WLAN_MD_11B;
+	} else {
+		if (BSS_EX_OP_BAND(pbss_network) != BAND_ON_24G){
+			/* Remove CCK in support_rate IE */
+			rtw_filter_suppport_rateie(pbss_network, OFDM);
+			network_type = WLAN_MD_11A;
+		} else {
+			if ((mode & WLAN_MD_11B) == 0) {
+				/* Remove CCK in support_rate IE */
+				rtw_filter_suppport_rateie(pbss_network, OFDM);
+				network_type = WLAN_MD_11G;
+			} else {
+				network_type = WLAN_MD_11BG;
+			}
+		}
+	}
+
+	rtw_set_supported_rate(pbss_network->SupportedRates, network_type,
+			       pbss_network->Configuration.DSConfig,
+			       pbss_network->Configuration.Band);
+
+	return network_type;
+}
+
+uint	rtw_get_rateset_len(u8	*rateset)
+{
+	uint i = 0;
+	while (1) {
+		if ((rateset[i]) == 0)
+			break;
+
+		if (i > 12)
+			break;
+
+		i++;
+	}
+	return i;
+}
+
+int rtw_generate_ie(struct registry_priv *pregistrypriv)
+{
+	u8	wireless_mode;
+	int	sz = 0, rateLen;
+	WLAN_BSSID_EX	*pdev_network = &pregistrypriv->dev_network;
+	u8	*ie = pdev_network->IEs;
+
+
+	/* timestamp will be inserted by hardware */
+	sz += 8;
+	ie += sz;
+
+	/* beacon interval : 2bytes */
+	*(u16 *)ie = cpu_to_le16((u16)pdev_network->Configuration.BeaconPeriod); /* BCN_INTERVAL; */
+	sz += 2;
+	ie += 2;
+
+	/* capability info */
+	*(u16 *)ie = 0;
+
+	*(u16 *)ie |= cpu_to_le16(cap_IBSS);
+
+	if (pregistrypriv->preamble == PREAMBLE_SHORT)
+		*(u16 *)ie |= cpu_to_le16(cap_ShortPremble);
+
+	if (pdev_network->Privacy)
+		*(u16 *)ie |= cpu_to_le16(cap_Privacy);
+
+	sz += 2;
+	ie += 2;
+
+	/* SSID */
+	ie = rtw_set_ie(ie, _SSID_IE_, pdev_network->Ssid.SsidLength, pdev_network->Ssid.Ssid, &sz);
+
+	/* supported rates */
+	if (pregistrypriv->wireless_mode == WLAN_MD_11ABGN) {
+		if (pdev_network->Configuration.DSConfig > 14)
+			wireless_mode = WLAN_MD_11AN;
+		else
+			wireless_mode = WLAN_MD_11BGN;
+	} else if (pregistrypriv->wireless_mode == WLAN_MD_MAX) { /* WIRELESS_11ABGN | WIRELESS_11AC */
+		if (pdev_network->Configuration.DSConfig > 14)
+			wireless_mode = WLAN_MD_5G_MIX;
+		else
+			wireless_mode = WLAN_MD_24G_MIX;
+	} else
+		wireless_mode = pregistrypriv->wireless_mode;
+
+	rtw_set_supported_rate(pdev_network->SupportedRates, wireless_mode,
+			       pdev_network->Configuration.DSConfig,
+			       pdev_network->Configuration.Band) ;
+
+	rateLen = rtw_get_rateset_len(pdev_network->SupportedRates);
+
+	if (rateLen > 8) {
+		ie = rtw_set_ie(ie, _SUPPORTEDRATES_IE_, 8, pdev_network->SupportedRates, &sz);
+		/* ie = rtw_set_ie(ie, _EXT_SUPPORTEDRATES_IE_, (rateLen - 8), (pdev_network->SupportedRates + 8), &sz); */
+	} else
+		ie = rtw_set_ie(ie, _SUPPORTEDRATES_IE_, rateLen, pdev_network->SupportedRates, &sz);
+
+	/* DS parameter set */
+	ie = rtw_set_ie(ie, _DSSET_IE_, 1, (u8 *)&(pdev_network->Configuration.DSConfig), &sz);
+
+
+	/* IBSS Parameter Set */
+
+	ie = rtw_set_ie(ie, _IBSS_PARA_IE_, 2, (u8 *)&(pdev_network->Configuration.ATIMWindow), &sz);
+
+	if (rateLen > 8)
+		ie = rtw_set_ie(ie, _EXT_SUPPORTEDRATES_IE_, (rateLen - 8), (pdev_network->SupportedRates + 8), &sz);
+
+#ifdef CONFIG_80211N_HT
+	/* HT Cap. */
+	if (is_supported_ht(pregistrypriv->wireless_mode)
+	    && (pregistrypriv->ht_enable == _TRUE)) {
+		/* todo: */
+	}
+#endif /* CONFIG_80211N_HT */
+
+	/* pdev_network->IELength =  sz; */ /* update IELength */
+
+
+	/* return _SUCCESS; */
+
+	return sz;
+
+}
+
+unsigned char *rtw_get_wpa_ie(unsigned char *pie, int *wpa_ie_len, int limit)
+{
+	int len;
+	u16 val16;
+	unsigned char wpa_oui_type[] = {0x00, 0x50, 0xf2, 0x01};
+	u8 *pbuf = pie;
+	int limit_new = limit;
+
+	while (1) {
+		pbuf = rtw_get_ie(pbuf, _WPA_IE_ID_, &len, limit_new);
+
+		if (pbuf) {
+
+			/* check if oui matches... */
+			if (_rtw_memcmp((pbuf + 2), wpa_oui_type, sizeof(wpa_oui_type)) == _FALSE)
+
+				goto check_next_ie;
+
+			/* check version... */
+			_rtw_memcpy((u8 *)&val16, (pbuf + 6), sizeof(val16));
+
+			val16 = le16_to_cpu(val16);
+			if (val16 != 0x0001)
+				goto check_next_ie;
+
+			*wpa_ie_len = *(pbuf + 1);
+
+			return pbuf;
+
+		} else {
+
+			*wpa_ie_len = 0;
+			return NULL;
+		}
+
+check_next_ie:
+
+		limit_new = limit - (pbuf - pie) - 2 - len;
+
+		if (limit_new <= 0)
+			break;
+
+		pbuf += (2 + len);
+
+	}
+
+	*wpa_ie_len = 0;
+
+	return NULL;
+
+}
+
+unsigned char *rtw_get_wpa2_ie(unsigned char *pie, int *rsn_ie_len, int limit)
+{
+
+	return rtw_get_ie(pie, _WPA2_IE_ID_, rsn_ie_len, limit);
+
+}
+
+int rtw_get_wpa_cipher_suite(u8 *s)
+{
+	if (_rtw_memcmp(s, WPA_CIPHER_SUITE_NONE, WPA_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_NONE;
+	if (_rtw_memcmp(s, WPA_CIPHER_SUITE_WEP40, WPA_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_WEP40;
+	if (_rtw_memcmp(s, WPA_CIPHER_SUITE_TKIP, WPA_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_TKIP;
+	if (_rtw_memcmp(s, WPA_CIPHER_SUITE_CCMP, WPA_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_CCMP;
+	if (_rtw_memcmp(s, WPA_CIPHER_SUITE_WEP104, WPA_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_WEP104;
+
+	return 0;
+}
+
+int rtw_get_rsn_cipher_suite(u8 *s)
+{
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_NONE, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_NONE;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_WEP40, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_WEP40;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_TKIP, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_TKIP;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_CCMP, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_CCMP;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_GCMP, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_GCMP;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_GCMP_256, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_GCMP_256;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_CCMP_256, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_CCMP_256;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_WEP104, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_WEP104;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_AES_128_CMAC, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_BIP_CMAC_128;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_BIP_GMAC_128, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_BIP_GMAC_128;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_BIP_GMAC_256, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_BIP_GMAC_256;
+	if (_rtw_memcmp(s, RSN_CIPHER_SUITE_BIP_CMAC_256, RSN_SELECTOR_LEN) == _TRUE)
+		return WPA_CIPHER_BIP_CMAC_256;
+	return 0;
+}
+
+u32 rtw_get_akm_suite_bitmap(u8 *s)
+{
+	if (_rtw_memcmp(s, WLAN_AKM_8021X, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_8021X;
+	if (_rtw_memcmp(s, WLAN_AKM_PSK, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_PSK;
+	if (_rtw_memcmp(s, WLAN_AKM_FT_8021X, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_FT_8021X;
+	if (_rtw_memcmp(s, WLAN_AKM_FT_PSK, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_FT_PSK;
+	if (_rtw_memcmp(s, WLAN_AKM_8021X_SHA256, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_8021X_SHA256;
+	if (_rtw_memcmp(s, WLAN_AKM_PSK_SHA256, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_PSK_SHA256;
+	if (_rtw_memcmp(s, WLAN_AKM_TDLS, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_TDLS;
+	if (_rtw_memcmp(s, WLAN_AKM_SAE, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_SAE;
+	if (_rtw_memcmp(s, WLAN_AKM_FT_OVER_SAE, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_FT_OVER_SAE;
+	if (_rtw_memcmp(s, WLAN_AKM_8021X_SUITE_B, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_8021X_SUITE_B;
+	if (_rtw_memcmp(s, WLAN_AKM_8021X_SUITE_B_192, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_8021X_SUITE_B_192;
+	if (_rtw_memcmp(s, WLAN_AKM_FILS_SHA256, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_FILS_SHA256;
+	if (_rtw_memcmp(s, WLAN_AKM_FILS_SHA384, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_FILS_SHA384;
+	if (_rtw_memcmp(s, WLAN_AKM_FT_FILS_SHA256, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_FT_FILS_SHA256;
+	if (_rtw_memcmp(s, WLAN_AKM_FT_FILS_SHA384, RSN_SELECTOR_LEN) == _TRUE)
+		return WLAN_AKM_TYPE_FT_FILS_SHA384;
+
+	return 0;
+}
+
+int rtw_parse_wpa_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher,
+	int *pairwise_cipher, u32 *akm)
+{
+	int i, ret = _SUCCESS;
+	int left, count;
+	u8 *pos;
+	u8 SUITE_1X[4] = {0x00, 0x50, 0xf2, 1};
+
+	if (wpa_ie_len <= 0) {
+		/* No WPA IE - fail silently */
+		return _FAIL;
+	}
+
+
+	if ((*wpa_ie != _WPA_IE_ID_) || (*(wpa_ie + 1) != (u8)(wpa_ie_len - 2)) ||
+	    (_rtw_memcmp(wpa_ie + 2, RTW_WPA_OUI_TYPE, WPA_SELECTOR_LEN) != _TRUE))
+		return _FAIL;
+
+	pos = wpa_ie;
+
+	pos += 8;
+	left = wpa_ie_len - 8;
+
+
+	/* group_cipher */
+	if (left >= WPA_SELECTOR_LEN) {
+
+		*group_cipher = rtw_get_wpa_cipher_suite(pos);
+
+		pos += WPA_SELECTOR_LEN;
+		left -= WPA_SELECTOR_LEN;
+
+	} else if (left > 0) {
+
+		return _FAIL;
+	}
+
+
+	/* pairwise_cipher */
+	if (left >= 2) {
+		/* count = le16_to_cpu(*(u16*)pos);	 */
+		count = RTW_GET_LE16(pos);
+		pos += 2;
+		left -= 2;
+
+		if (count == 0 || left < count * WPA_SELECTOR_LEN) {
+			return _FAIL;
+		}
+
+		for (i = 0; i < count; i++) {
+			*pairwise_cipher |= rtw_get_wpa_cipher_suite(pos);
+
+			pos += WPA_SELECTOR_LEN;
+			left -= WPA_SELECTOR_LEN;
+		}
+
+	} else if (left == 1) {
+		return _FAIL;
+	}
+
+	if (akm) {
+		if (left >= 6) {
+			pos += 2;
+			if (_rtw_memcmp(pos, SUITE_1X, 4) == 1) {
+				*akm = WLAN_AKM_TYPE_8021X;
+			}
+		}
+	}
+
+	return ret;
+
+}
+
+int rtw_rsne_info_parse(const u8 *ie, uint ie_len, struct rsne_info *info)
+{
+	const u8 *pos = ie;
+	u16 cnt;
+
+	_rtw_memset(info, 0, sizeof(struct rsne_info));
+
+	/* wpa2-imp 4.2.2 F */
+	if (ie + ie_len < pos + RSN_HEADER_LEN) {
+		RTW_INFO("%s: Invalid RSN IE length\n", __func__);
+		goto err;
+	}
+
+	if (*ie != WLAN_EID_RSN || *(ie + 1) != ie_len - 2)
+		goto err;
+
+	/* wpa2-imp 4.2.2 A */
+	if ((u16)(*(ie + 2)) != RSN_VERSION_BSD) {
+		RTW_INFO("%s: Invalid RSN version\n", __func__);
+		goto err;
+	}
+	pos += RSN_HEADER_LEN;
+
+	/* ------------------- */
+	/* Group CS processing */
+	/* ------------------- */
+	/* wpa2-imp 4.2.2 G */
+	if (ie + ie_len < pos + RSN_SELECTOR_LEN) {
+		if (ie + ie_len != pos) {
+			RTW_INFO("%s: Group Cipher Suite truncated\n", __func__);
+			goto err;
+		}
+		goto exit;
+	}
+	info->gcs = (u8 *)pos;
+	pos += RSN_SELECTOR_LEN;
+
+	/* ---------------------- */
+	/* Pairwise CS processing */
+	/* ---------------------- */
+	if (ie + ie_len < pos + 2) {
+		if (ie + ie_len != pos)
+			goto err;
+		goto exit;
+	}
+	cnt = RTW_GET_LE16(pos);
+	pos += 2;
+
+	/* wpa-imp 4.2.2 B D */
+	if (cnt == 0) {
+		RTW_INFO("%s: Pairwise Suite Count Invalid\n", __func__);
+		goto err;
+	}
+
+	/* wpa2-imp 4.2.2 H L */
+	if (ie + ie_len < pos + RSN_SELECTOR_LEN * cnt) {
+		if (ie + ie_len != pos) {
+			RTW_INFO("%s: Pairwise Suite truncated\n", __func__);
+			goto err;
+		}
+		goto exit;
+	}
+	info->pcs_cnt = cnt;
+	info->pcs_list = (u8 *)pos;
+	pos += RSN_SELECTOR_LEN * cnt;
+
+	/* -------------- */
+	/* AKM processing */
+	/* -------------- */
+	if (ie + ie_len < pos + 2) {
+		if (ie + ie_len != pos)
+			goto err;
+		goto exit;
+	}
+	cnt = RTW_GET_LE16(pos);
+	pos += 2;
+
+	/* wpa-imp 4.2.2 C D */
+	if (cnt == 0) {
+		RTW_INFO("%s: AKM Count Invalid\n", __func__);
+		goto err;
+	}
+
+	/* wpa2-imp 4.2.2 I M */
+	if (ie + ie_len < pos + RSN_SELECTOR_LEN * cnt) {
+		if (ie + ie_len != pos) {
+			RTW_INFO("%s: AKM truncated\n", __func__);
+			goto err;
+		}
+		goto exit;
+	}
+	info->akm_cnt = cnt;
+	info->akm_list = (u8 *)pos;
+	pos += RSN_SELECTOR_LEN * cnt;
+
+	/* ------------------------- */
+	/* RSN Capability processing */
+	/* ------------------------- */
+	/* wpa2-imp 4.2.2 J */
+	if (ie + ie_len < pos + 2) {
+		if (ie + ie_len != pos) {
+			RTW_INFO("%s: RSN Capability truncated\n", __func__);
+			goto err;
+		}
+		goto exit;
+	}
+	info->cap = (u8 *)pos;
+	pos += 2;
+
+	/* ---------------- */
+	/* PMKID processing */
+	/* ---------------- */
+	/* wpa2-imp 4.2.2 K */
+	if (ie + ie_len < pos + 2) {
+		if (ie + ie_len != pos) {
+			RTW_INFO("%s: PMKID count truncated\n", __func__);
+			goto err;
+		}
+		goto exit;
+	}
+	cnt = RTW_GET_LE16(pos);
+	pos += 2;
+
+	/* wpa2-imp 4.2.2 N */
+	if (ie + ie_len < pos + RSN_PMKID_LEN * cnt) {
+		if (ie + ie_len != pos || cnt) {
+			RTW_INFO("%s: PMKID list truncated\n", __func__);
+			goto err;
+		}
+		goto exit;
+	}
+	info->pmkid_cnt = cnt;
+	info->pmkid_list = (u8 *)pos;
+	pos += RSN_PMKID_LEN * cnt;
+
+	/* --------------------------------------- */
+	/* Group Management Cipher Suit processing */
+	/* --------------------------------------- */
+	if (ie + ie_len < pos + RSN_SELECTOR_LEN) {
+		if (ie + ie_len != pos)
+			goto err;
+		goto exit;
+	}
+	info->gmcs = (u8 *)pos;
+
+exit:
+	return _SUCCESS;
+
+err:
+	info->err = 1;
+	return _FAIL;
+}
+
+int rtw_parse_wpa2_ie(u8 *rsn_ie, int rsn_ie_len, int *group_cipher,
+	int *pairwise_cipher, int *gmcs, u32 *akm, u8 *mfp_opt, u8 *spp_opt)
+{
+	struct rsne_info info;
+	int i, ret = _SUCCESS;
+
+	ret = rtw_rsne_info_parse(rsn_ie, rsn_ie_len, &info);
+	if (ret != _SUCCESS)
+		goto exit;
+
+	if (group_cipher) {
+		if (info.gcs)
+			*group_cipher = rtw_get_rsn_cipher_suite(info.gcs);
+		else
+			*group_cipher = 0;
+	}
+
+	if (pairwise_cipher) {
+		*pairwise_cipher = 0;
+		for (i = 0; i < info.pcs_cnt; i++)
+			*pairwise_cipher |= rtw_get_rsn_cipher_suite(info.pcs_list + 4 * i);
+	}
+
+	if (gmcs) {
+		if (info.gmcs)
+			*gmcs = rtw_get_rsn_cipher_suite(info.gmcs);
+		else
+			*gmcs = WPA_CIPHER_BIP_CMAC_128; /* default value when absent */
+	}
+
+	if (akm) {
+		*akm = 0;
+		for (i = 0; i < info.akm_cnt; i++)
+			*akm |= rtw_get_akm_suite_bitmap(info.akm_list + 4 * i);
+	}
+
+	if (mfp_opt) {
+		*mfp_opt = MFP_NO;
+		if (info.cap)
+			*mfp_opt = GET_RSN_CAP_MFP_OPTION(info.cap);
+	}
+	if (spp_opt) {
+		*spp_opt = 0;
+		if (info.cap)
+			*spp_opt = GET_RSN_CAP_SPP_OPT(info.cap);
+		}
+exit:
+	return ret;
+}
+
+/* #ifdef CONFIG_WAPI_SUPPORT */
+int rtw_get_wapi_ie(u8 *in_ie, uint in_len, u8 *wapi_ie, u16 *wapi_len)
+{
+	int len = 0;
+	u8 authmode;
+	uint	cnt;
+	u8 wapi_oui1[4] = {0x0, 0x14, 0x72, 0x01};
+	u8 wapi_oui2[4] = {0x0, 0x14, 0x72, 0x02};
+
+
+	if (wapi_len)
+		*wapi_len = 0;
+
+	if (!in_ie || in_len <= 0)
+		return len;
+
+	cnt = (_TIMESTAMP_ + _BEACON_ITERVAL_ + _CAPABILITY_);
+
+	while (cnt < in_len) {
+		authmode = in_ie[cnt];
+
+		/* if(authmode==_WAPI_IE_) */
+		if (authmode == _WAPI_IE_ && (_rtw_memcmp(&in_ie[cnt + 6], wapi_oui1, 4) == _TRUE ||
+			_rtw_memcmp(&in_ie[cnt + 6], wapi_oui2, 4) == _TRUE)) {
+			if (wapi_ie)
+				_rtw_memcpy(wapi_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			if (wapi_len)
+				*wapi_len = in_ie[cnt + 1] + 2;
+
+			cnt += in_ie[cnt + 1] + 2; /* get next */
+		} else {
+			cnt += in_ie[cnt + 1] + 2; /* get next */
+		}
+	}
+
+	if (wapi_len)
+		len = *wapi_len;
+
+
+	return len;
+
+}
+/* #endif */
+
+int rtw_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len, u8 *wpa_ie, u16 *wpa_len)
+{
+	u8 authmode, sec_idx;
+	u8 wpa_oui[4] = {0x0, 0x50, 0xf2, 0x01};
+	uint	cnt;
+
+
+	/* Search required WPA or WPA2 IE and copy to sec_ie[ ] */
+
+	cnt = (_TIMESTAMP_ + _BEACON_ITERVAL_ + _CAPABILITY_);
+
+	sec_idx = 0;
+
+	while (cnt < in_len) {
+		authmode = in_ie[cnt];
+
+		if ((authmode == _WPA_IE_ID_) && (_rtw_memcmp(&in_ie[cnt + 2], &wpa_oui[0], 4) == _TRUE)) {
+
+			if (wpa_ie)
+				_rtw_memcpy(wpa_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			*wpa_len = in_ie[cnt + 1] + 2;
+			cnt += in_ie[cnt + 1] + 2; /* get next */
+		} else {
+			if (authmode == _WPA2_IE_ID_) {
+
+				if (rsn_ie)
+					_rtw_memcpy(rsn_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+				*rsn_len = in_ie[cnt + 1] + 2;
+				cnt += in_ie[cnt + 1] + 2; /* get next */
+			} else {
+				cnt += in_ie[cnt + 1] + 2; /* get next */
+			}
+		}
+
+	}
+
+
+	return *rsn_len + *wpa_len;
+
+}
+
+u8 rtw_is_wps_ie(u8 *ie_ptr, uint *wps_ielen)
+{
+	u8 match = _FALSE;
+	u8 eid, wps_oui[4] = {0x0, 0x50, 0xf2, 0x04};
+
+	if (ie_ptr == NULL)
+		return match;
+
+	eid = ie_ptr[0];
+
+	if ((eid == _WPA_IE_ID_) && (_rtw_memcmp(&ie_ptr[2], wps_oui, 4) == _TRUE)) {
+		/* RTW_INFO("==> found WPS_IE.....\n"); */
+		*wps_ielen = ie_ptr[1] + 2;
+		match = _TRUE;
+	}
+	return match;
+}
+
+u8 *rtw_get_wps_ie_from_scan_queue(u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen, enum bss_type frame_type)
+{
+	u8	*wps = NULL;
+
+	RTW_INFO("[%s] frame_type = %d\n", __FUNCTION__, frame_type);
+	switch (frame_type) {
+	case BSS_TYPE_BCN:
+	case BSS_TYPE_PROB_RSP: {
+		/*	Beacon or Probe Response */
+		wps = rtw_get_wps_ie(in_ie + _PROBERSP_IE_OFFSET_, in_len - _PROBERSP_IE_OFFSET_, wps_ie, wps_ielen);
+		break;
+	}
+	case BSS_TYPE_PROB_REQ: {
+		/*	Probe Request */
+		wps = rtw_get_wps_ie(in_ie + _PROBEREQ_IE_OFFSET_ , in_len - _PROBEREQ_IE_OFFSET_ , wps_ie, wps_ielen);
+		break;
+	}
+	default:
+	case BSS_TYPE_UNDEF:
+		break;
+	}
+	return wps;
+}
+
+/**
+ * rtw_get_wps_ie - Search WPS IE from a series of IEs
+ * @in_ie: Address of IEs to search
+ * @in_len: Length limit from in_ie
+ * @wps_ie: If not NULL and WPS IE is found, WPS IE will be copied to the buf starting from wps_ie
+ * @wps_ielen: If not NULL and WPS IE is found, will set to the length of the entire WPS IE
+ *
+ * Returns: The address of the WPS IE found, or NULL
+ */
+u8 *rtw_get_wps_ie(const u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen)
+{
+	uint cnt;
+	const u8 *wpsie_ptr = NULL;
+	u8 eid, wps_oui[4] = {0x00, 0x50, 0xf2, 0x04};
+
+	if (wps_ielen)
+		*wps_ielen = 0;
+
+	if (!in_ie) {
+		rtw_warn_on(1);
+		return (u8 *)wpsie_ptr;
+	}
+
+	if (in_len <= 0)
+		return (u8 *)wpsie_ptr;
+
+	cnt = 0;
+
+	while (cnt + 1 + 4 < in_len) {
+		eid = in_ie[cnt];
+
+		if (cnt + 1 + 4 >= MAX_IE_SZ) {
+			rtw_warn_on(1);
+			return NULL;
+		}
+
+		if (eid == WLAN_EID_VENDOR_SPECIFIC && _rtw_memcmp(&in_ie[cnt + 2], wps_oui, 4) == _TRUE) {
+			wpsie_ptr = in_ie + cnt;
+
+			if (wps_ie)
+				_rtw_memcpy(wps_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			if (wps_ielen)
+				*wps_ielen = in_ie[cnt + 1] + 2;
+
+			break;
+		} else
+			cnt += in_ie[cnt + 1] + 2;
+
+	}
+
+	return (u8 *)wpsie_ptr;
+}
+
+/**
+ * rtw_get_wps_attr - Search a specific WPS attribute from a given WPS IE
+ * @wps_ie: Address of WPS IE to search
+ * @wps_ielen: Length limit from wps_ie
+ * @target_attr_id: The attribute ID of WPS attribute to search
+ * @buf_attr: If not NULL and the WPS attribute is found, WPS attribute will be copied to the buf starting from buf_attr
+ * @len_attr: If not NULL and the WPS attribute is found, will set to the length of the entire WPS attribute
+ *
+ * Returns: the address of the specific WPS attribute found, or NULL
+ */
+u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id , u8 *buf_attr, u32 *len_attr)
+{
+	u8 *attr_ptr = NULL;
+	u8 *target_attr_ptr = NULL;
+	u8 wps_oui[4] = {0x00, 0x50, 0xF2, 0x04};
+
+	if (len_attr)
+		*len_attr = 0;
+
+	if ((wps_ie[0] != _VENDOR_SPECIFIC_IE_) ||
+	    (_rtw_memcmp(wps_ie + 2, wps_oui , 4) != _TRUE))
+		return attr_ptr;
+
+	/* 6 = 1(Element ID) + 1(Length) + 4(WPS OUI) */
+	attr_ptr = wps_ie + 6; /* goto first attr */
+
+	while (attr_ptr - wps_ie < wps_ielen) {
+		/* 4 = 2(Attribute ID) + 2(Length) */
+		u16 attr_id = RTW_GET_BE16(attr_ptr);
+		u16 attr_data_len = RTW_GET_BE16(attr_ptr + 2);
+		u16 attr_len = attr_data_len + 4;
+
+		/* RTW_INFO("%s attr_ptr:%p, id:%u, length:%u\n", __FUNCTION__, attr_ptr, attr_id, attr_data_len); */
+		if (attr_id == target_attr_id) {
+			target_attr_ptr = attr_ptr;
+
+			if (buf_attr)
+				_rtw_memcpy(buf_attr, attr_ptr, attr_len);
+
+			if (len_attr)
+				*len_attr = attr_len;
+
+			break;
+		} else {
+			attr_ptr += attr_len; /* goto next */
+		}
+
+	}
+
+	return target_attr_ptr;
+}
+
+/**
+ * rtw_get_wps_attr_content - Search a specific WPS attribute content from a given WPS IE
+ * @wps_ie: Address of WPS IE to search
+ * @wps_ielen: Length limit from wps_ie
+ * @target_attr_id: The attribute ID of WPS attribute to search
+ * @buf_content: If not NULL and the WPS attribute is found, WPS attribute content will be copied to the buf starting from buf_content
+ *               If len_content is NULL, only copy one byte.
+ * @len_content: If not NULL and the WPS attribute is found, will set to the length of the WPS attribute content
+ *
+ * Returns: the address of the specific WPS attribute content found, or NULL
+ */
+u8 *rtw_get_wps_attr_content(u8 *wps_ie, uint wps_ielen, u16 target_attr_id , u8 *buf_content, uint *len_content)
+{
+	u8 *attr_ptr;
+	u32 attr_len;
+
+	attr_ptr = rtw_get_wps_attr(wps_ie, wps_ielen, target_attr_id, NULL, &attr_len);
+
+	if (attr_ptr && attr_len) {
+		if (len_content) {
+			if ((buf_content && (*len_content > (attr_len - 4))) || !buf_content)
+				*len_content = attr_len - 4;
+		}
+
+		if (len_content && buf_content) {
+			_rtw_memcpy(buf_content, attr_ptr + 4, *len_content);
+		} else if (buf_content) {
+			_rtw_memcpy(buf_content, attr_ptr + 4, 1);
+		}
+
+		return attr_ptr + 4;
+	}
+
+	if (len_content)
+		*len_content = 0;
+
+	return NULL;
+}
+
+/* OWE */
+
+/**
+ * rtw_get_OWE_ie - Search OWE IE from a series of IEs
+ * @in_ie: Address of IEs to search
+ * @in_len: Length limit from in_ie
+ * @wps_ie: If not NULL and OWE IE is found, OWE IE will be copied to the buf starting from owe_ie
+ * @wps_ielen: If not NULL and OWE IE is found, will set to the length of the entire OWE IE
+ *
+ * Returns: The address of the OWE IE found, or NULL
+ */
+u8 *rtw_get_owe_ie(const u8 *in_ie, uint in_len, u8 *owe_ie, uint *owe_ielen)
+{
+	uint cnt;
+	const u8 *oweie_ptr = NULL;
+	u8 eid;
+
+	if (owe_ielen)
+		*owe_ielen = 0;
+
+	if (!in_ie) {
+		rtw_warn_on(1);
+		return (u8 *)oweie_ptr;
+	}
+
+	if (in_len <= 0)
+		return (u8 *)oweie_ptr;
+
+	cnt = 0;
+
+	while (cnt + 1 + 4 < in_len) {
+		eid = in_ie[cnt];
+
+		if (cnt + 1 + 4 >= MAX_IE_SZ) {
+			rtw_warn_on(1);
+			return NULL;
+		}
+
+		if ((eid == WLAN_EID_EXTENSION) && (in_ie[cnt + 2] == WLAN_EID_EXT_OWE_DH_PARAM)) {
+			oweie_ptr = in_ie + cnt;
+
+			if (owe_ie)
+				_rtw_memcpy(owe_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			if (owe_ielen)
+				*owe_ielen = in_ie[cnt + 1] + 2;
+
+			break;
+		} else
+			cnt += in_ie[cnt + 1] + 2;
+
+	}
+
+	return (u8 *)oweie_ptr;
+}
+
+/* Add extended capabilities element infomation into ext_cap_data of driver */
+void rtw_add_ext_cap_info(u8 *ext_cap_data, u8 *ext_cap_data_len, u8 cap_info)
+{
+	u8 byte_offset = cap_info >> 3;
+	u8 bit_offset = cap_info % 8;
+
+	ext_cap_data[byte_offset] |= BIT(bit_offset);
+
+	/* Enlarge the length of EXT_CAP_IE */
+	if (byte_offset + 1 > *ext_cap_data_len)
+		*ext_cap_data_len = byte_offset + 1;
+
+	#ifdef DBG_EXT_CAP_IE
+	RTW_INFO("%s : cap_info = %u, byte_offset = %u, bit_offset = %u, ext_cap_data_len = %u\n", \
+			__func__, cap_info, byte_offset, bit_offset, *ext_cap_data_len);
+	#endif
+}
+
+/* Remvoe extended capabilities element infomation from ext_cap_data of driver */
+void rtw_remove_ext_cap_info(u8 *ext_cap_data, u8 *ext_cap_data_len, u8 cap_info)
+{
+	u8 byte_offset = cap_info >> 3;
+	u8 bit_offset = cap_info % 8;
+	u8 i, max_len = 0;
+
+	ext_cap_data[byte_offset] &= (~BIT(bit_offset));
+
+	/* Reduce the length of EXT_CAP_IE */
+	for (i = 0; i < WLAN_EID_EXT_CAP_MAX_LEN; i++) {
+		if (ext_cap_data[i] != 0x0)
+			max_len = i + 1;
+	}
+	*ext_cap_data_len = max_len;
+
+	#ifdef DBG_EXT_CAP_IE
+	RTW_INFO("%s : cap_info = %u, byte_offset = %u, bit_offset = %u, ext_cap_data_len = %u\n", \
+			__func__, cap_info, byte_offset, bit_offset, *ext_cap_data_len);
+	#endif
+}
+
+/**
+ * rtw_update_ext_cap_ie - add/update/remove the extended capabilities element of frame
+ *
+ * @ext_cap_data: from &(mlme_priv->ext_capab_ie_data)
+ * @ext_cap_data_len: length of ext_cap_data
+ * @ies: address of ies, e.g. pnetwork->IEs
+ * @ies_len: address of length of ies, e.g. &(pnetwork->IELength)
+ * @ies_offset: offset of ies, e.g. _BEACON_IE_OFFSET_
+ */
+u8 rtw_update_ext_cap_ie(u8 *ext_cap_data, u8 ext_cap_data_len, u8 *ies, u32 *ies_len, u8 ies_offset)
+{
+	u8 *extcap_ie;
+	uint extcap_len_field = 0;
+	uint ie_len = 0;
+
+	if (ext_cap_data_len != 0) {
+		extcap_ie = rtw_get_ie(ies + ies_offset, WLAN_EID_EXT_CAP, &extcap_len_field, *ies_len - ies_offset);
+
+		if (extcap_ie == NULL) {
+			rtw_set_ie(ies + *ies_len, WLAN_EID_EXT_CAP, ext_cap_data_len, ext_cap_data, &ie_len);
+			*ies_len += ie_len;
+		} else {
+			rtw_ies_update_ie(ies, ies_len, ies_offset, WLAN_EID_EXT_CAP, ext_cap_data, ext_cap_data_len);
+		}
+	} else {
+		rtw_ies_remove_ie(ies, ies_len, ies_offset, WLAN_EID_EXT_CAP, NULL, 0);
+	}
+
+	return _SUCCESS;
+}
+
+void rtw_parse_ext_cap_ie(u8 *ext_cap_data, u8 *ext_cap_data_len, u8 *ies, u32 ies_len, u8 ies_offset)
+{
+	u8 *extcap_ie;
+	uint extcap_len_field = 0;
+	u8 i;
+
+	extcap_ie = rtw_get_ie(ies + ies_offset, WLAN_EID_EXT_CAP, &extcap_len_field, ies_len - ies_offset);
+
+	if (extcap_ie != NULL) {
+		extcap_ie = extcap_ie + 2; /* element id and length filed */
+		if (*ext_cap_data_len == 0) {
+			_rtw_memcpy(ext_cap_data, extcap_ie, extcap_len_field);
+			*ext_cap_data_len = extcap_len_field;
+		} else {
+			for (i = 0; i < extcap_len_field; i++)
+				ext_cap_data[i] |= extcap_ie[i];
+		}
+
+		#ifdef DBG_EXT_CAP_IE
+		for (i = 0; i < extcap_len_field; i++)
+			RTW_INFO("%s : Parse extended capabilties[%u] = 0x%x\n", __func__, i, extcap_ie[i]);
+		#endif
+	}
+}
+
+static int rtw_ieee802_11_parse_vendor_specific(u8 *pos, uint elen,
+		struct rtw_ieee802_11_elems *elems,
+		int show_errors)
+{
+	unsigned int oui;
+
+	/* first 3 bytes in vendor specific information element are the IEEE
+	 * OUI of the vendor. The following byte is used a vendor specific
+	 * sub-type. */
+	if (elen < 4) {
+		if (show_errors) {
+			RTW_INFO("short vendor specific "
+				 "information element ignored (len=%lu)\n",
+				 (unsigned long) elen);
+		}
+		return -1;
+	}
+
+	oui = RTW_GET_BE24(pos);
+	switch (oui) {
+	case OUI_MICROSOFT:
+		/* Microsoft/Wi-Fi information elements are further typed and
+		 * subtyped */
+		switch (pos[3]) {
+		case 1:
+			/* Microsoft OUI (00:50:F2) with OUI Type 1:
+			 * real WPA information element */
+			elems->wpa_ie = pos;
+			elems->wpa_ie_len = elen;
+			break;
+		case WME_OUI_TYPE: /* this is a Wi-Fi WME info. element */
+			if (elen < 5) {
+				RTW_DBG("short WME "
+					"information element ignored "
+					"(len=%lu)\n",
+					(unsigned long) elen);
+				return -1;
+			}
+			switch (pos[4]) {
+			case WME_OUI_SUBTYPE_INFORMATION_ELEMENT:
+			case WME_OUI_SUBTYPE_PARAMETER_ELEMENT:
+				elems->wme = pos;
+				elems->wme_len = elen;
+				break;
+			case WME_OUI_SUBTYPE_TSPEC_ELEMENT:
+				elems->wme_tspec = pos;
+				elems->wme_tspec_len = elen;
+				break;
+			default:
+				RTW_DBG("unknown WME "
+					"information element ignored "
+					"(subtype=%d len=%lu)\n",
+					pos[4], (unsigned long) elen);
+				return -1;
+			}
+			break;
+		case 4:
+			/* Wi-Fi Protected Setup (WPS) IE */
+			elems->wps_ie = pos;
+			elems->wps_ie_len = elen;
+			break;
+		default:
+			RTW_DBG("Unknown Microsoft "
+				"information element ignored "
+				"(type=%d len=%lu)\n",
+				pos[3], (unsigned long) elen);
+			return -1;
+		}
+		break;
+
+	case OUI_BROADCOM:
+		switch (pos[3]) {
+		case VENDOR_HT_CAPAB_OUI_TYPE:
+			elems->vendor_ht_cap = pos;
+			elems->vendor_ht_cap_len = elen;
+			break;
+		default:
+			RTW_DBG("Unknown Broadcom "
+				"information element ignored "
+				"(type=%d len=%lu)\n",
+				pos[3], (unsigned long) elen);
+			return -1;
+		}
+		break;
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	case OUI_REALTEK:
+		if (elen == 8) {  // TBTX capable IE length is 8
+			elems->tbtx_cap = pos;
+			elems->tbtx_cap_len = elen;
+		}
+		break;
+#endif
+	default:
+		RTW_DBG("unknown vendor specific information "
+			"element ignored (vendor OUI %02x:%02x:%02x "
+			"len=%lu)\n",
+			pos[0], pos[1], pos[2], (unsigned long) elen);
+		return -1;
+	}
+
+	return 0;
+
+}
+
+static int rtw_ieee802_11_parse_ext_elems(u8 *start, uint elen, struct rtw_ieee802_11_elems *elems)
+{
+	u8 *pos = start;
+	u8 id = *pos;
+
+	switch (id) {
+	case WLAN_EID_EXTENSION_HE_CAPABILITY:
+		elems->he_capabilities = pos;
+		elems->he_capabilities_len = elen;
+		return 0;
+	case WLAN_EID_EXTENSION_HE_OPERATION:
+		elems->he_operation = pos;
+		elems->he_operation_len = elen;
+		return 0;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	case WLAN_EID_EXT_HE_6G_CAP:
+		elems->he_6g_band_cap = pos;
+		elems->he_6g_band_cap_len = elen;
+		return 0;
+#endif
+	default:
+		return -1;
+	}
+}
+
+static ParseRes _rtw_ieee802_11_parse_elems(u8 *start, uint len,
+				    struct rtw_ieee802_11_elems *elems,
+				    int show_errors, bool reset)
+{
+	uint left = len;
+	u8 *pos = start;
+	int unknown = 0;
+
+	if (reset)
+		_rtw_memset(elems, 0, sizeof(*elems));
+
+	while (left >= 2) {
+		u8 id, elen;
+
+		id = *pos++;
+		elen = *pos++;
+		left -= 2;
+
+		if (elen > left) {
+			if (show_errors) {
+				RTW_INFO("IEEE 802.11 element "
+					 "parse failed (id=%d elen=%d "
+					 "left=%lu)\n",
+					 id, elen, (unsigned long) left);
+			}
+			return ParseFailed;
+		}
+
+		switch (id) {
+		case WLAN_EID_SSID:
+			elems->ssid = pos;
+			elems->ssid_len = elen;
+			break;
+		case WLAN_EID_SUPP_RATES:
+			elems->supp_rates = pos;
+			elems->supp_rates_len = elen;
+			break;
+		case WLAN_EID_FH_PARAMS:
+			elems->fh_params = pos;
+			elems->fh_params_len = elen;
+			break;
+		case WLAN_EID_DS_PARAMS:
+			elems->ds_params = pos;
+			elems->ds_params_len = elen;
+			break;
+		case WLAN_EID_CF_PARAMS:
+			elems->cf_params = pos;
+			elems->cf_params_len = elen;
+			break;
+		case WLAN_EID_TIM:
+			elems->tim = pos;
+			elems->tim_len = elen;
+			break;
+		case WLAN_EID_IBSS_PARAMS:
+			elems->ibss_params = pos;
+			elems->ibss_params_len = elen;
+			break;
+		case WLAN_EID_CHALLENGE:
+			elems->challenge = pos;
+			elems->challenge_len = elen;
+			break;
+		case WLAN_EID_ERP_INFO:
+			elems->erp_info = pos;
+			elems->erp_info_len = elen;
+			break;
+		case WLAN_EID_EXT_SUPP_RATES:
+			elems->ext_supp_rates = pos;
+			elems->ext_supp_rates_len = elen;
+			break;
+		case WLAN_EID_VENDOR_SPECIFIC:
+			if (rtw_ieee802_11_parse_vendor_specific(pos, elen,
+					elems,
+					show_errors))
+				unknown++;
+			break;
+		case WLAN_EID_RSN:
+			elems->rsn_ie = pos;
+			elems->rsn_ie_len = elen;
+			break;
+		case WLAN_EID_PWR_CAPABILITY:
+			elems->power_cap = pos;
+			elems->power_cap_len = elen;
+			break;
+		case WLAN_EID_SUPPORTED_CHANNELS:
+			elems->supp_channels = pos;
+			elems->supp_channels_len = elen;
+			break;
+		case WLAN_EID_MOBILITY_DOMAIN:
+			elems->mdie = pos;
+			elems->mdie_len = elen;
+			break;
+		case WLAN_EID_FAST_BSS_TRANSITION:
+			elems->ftie = pos;
+			elems->ftie_len = elen;
+			break;
+		case WLAN_EID_TIMEOUT_INTERVAL:
+			elems->timeout_int = pos;
+			elems->timeout_int_len = elen;
+			break;
+		case WLAN_EID_HT_CAP:
+			elems->ht_capabilities = pos;
+			elems->ht_capabilities_len = elen;
+			break;
+		case WLAN_EID_HT_OPERATION:
+			elems->ht_operation = pos;
+			elems->ht_operation_len = elen;
+			break;
+		case WLAN_EID_VHT_CAPABILITY:
+			elems->vht_capabilities = pos;
+			elems->vht_capabilities_len = elen;
+			break;
+		case WLAN_EID_VHT_OPERATION:
+			elems->vht_operation = pos;
+			elems->vht_operation_len = elen;
+			break;
+		case WLAN_EID_VHT_OP_MODE_NOTIFY:
+			elems->vht_op_mode_notify = pos;
+			elems->vht_op_mode_notify_len = elen;
+			break;
+		case _EID_RRM_EN_CAP_IE_:
+			elems->rm_en_cap = pos;
+			elems->rm_en_cap_len = elen;
+			break;
+		case WLAN_EID_AP_CHANNEL_RPT:
+			elems->ap_channel_rpt= pos;
+			elems->ap_channel_rpt_len = elen;
+			break;
+		case WLAN_EID_COUNTRY:
+			elems->country_info= pos;
+			elems->country_info_len = elen;
+			break;
+#ifdef CONFIG_RTW_MESH
+		case WLAN_EID_PREQ:
+			elems->preq = pos;
+			elems->preq_len = elen;
+			break;
+		case WLAN_EID_PREP:
+			elems->prep = pos;
+			elems->prep_len = elen;
+			break;
+		case WLAN_EID_PERR:
+			elems->perr = pos;
+			elems->perr_len = elen;
+			break;
+		case WLAN_EID_RANN:
+			elems->rann = pos;
+			elems->rann_len = elen;
+			break;
+#endif
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+		case WLAN_EID_MULTIPLE_BSSID:
+			elems->mbssid = pos;
+			elems->mbssid_len = elen;
+			break;
+		case WLAN_EID_NON_TX_BSSID_CAP:
+			elems->non_tx_bssid_cap = pos;
+			elems->non_tx_bssid_cap_len = elen;
+			break;
+#endif
+		case WLAN_EID_EXTENSION:
+			rtw_ieee802_11_parse_ext_elems(pos, elen, elems);
+			break;
+		default:
+			unknown++;
+			if (!show_errors)
+				break;
+			RTW_DBG("IEEE 802.11 element parse "
+				"ignored unknown element (id=%d elen=%d)\n",
+				id, elen);
+			break;
+		}
+
+		left -= elen;
+		pos += elen;
+	}
+
+	if (left)
+		return ParseFailed;
+
+	return unknown ? ParseUnknown : ParseOK;
+
+}
+
+/**
+ * ieee802_11_parse_elems - Parse information elements in management frames
+ * @start: Pointer to the start of IEs
+ * @len: Length of IE buffer in octets
+ * @elems: Data structure for parsed elements
+ * @show_errors: Whether to show parsing errors in debug log
+ * Returns: Parsing result
+ */
+ParseRes rtw_ieee802_11_parse_elems(u8 *start, uint len,
+				    struct rtw_ieee802_11_elems *elems,
+				    int show_errors)
+{
+	return _rtw_ieee802_11_parse_elems(start, len, elems, show_errors, true);
+}
+
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+static bool rtw_mbssid_ntbssid_profile_match_id(u8 *profile, uint len, u8 mbssid_idx)
+{
+	uint left = len;
+	u8 *pos = profile;
+
+	while (left >= 2) {
+		u8 id, elen;
+
+		id = *pos++;
+		elen = *pos++;
+		left -= 2;
+
+		if (elen > left)
+			return false;
+
+		switch (id) {
+		case WLAN_EID_MULTI_BSSID_IDX:
+			if (GET_MULTIPLE_BSSID_IDX_INDEX(pos - 2) == mbssid_idx)
+				return true;
+			break;
+		default:
+			break;
+		}
+
+		left -= elen;
+		pos += elen;
+	}
+
+	return false;
+}
+
+/**
+ * rtw_ieee802_11_override_elems_by_mbssid - override information elements in management frames
+ * @mbssid_ie: Pointer to the start of mbssid IE
+ * @mbssid_ie_len: Length of IE buffer in octets
+ * @mbssid_idx: the specific mbssid index to get for override
+ * @elems: Data structure for parsed elements
+ * @show_errors: Whether to show parsing errors in debug log
+ * Returns: Parsing result
+ */
+ParseRes rtw_ieee802_11_override_elems_by_mbssid(
+	u8 *mbssid_ie, uint mbssid_ie_len, u8 mbssid_idx, struct rtw_ieee802_11_elems *elems
+	, int show_errors)
+{
+	uint left = mbssid_ie_len;
+	u8 *pos = mbssid_ie;
+	u8 max_bssid_indicator;
+	int unknown = 0;
+
+	if (left < 3) {
+		RTW_WARN("%s mbssid_ie_len < 3\n", __func__);
+		return ParseFailed;
+	}
+
+	max_bssid_indicator = GET_MBSSID_MAX_BSSID_INDOCATOR(pos);
+	if (mbssid_idx >= (1 << max_bssid_indicator)) {
+		RTW_WARN("%s mbssid_idx >= max_bssid_indicator(%u)\n"
+			, __func__, 1 << max_bssid_indicator);
+		return ParseFailed;
+	}
+
+	pos += MBSSID_MAX_BSSID_INDICATOR_OFFSET;
+	left -= MBSSID_MAX_BSSID_INDICATOR_OFFSET;
+
+	while (left >= 2) {
+		u8 id, elen;
+
+		id = *pos++;
+		elen = *pos++;
+		left -= 2;
+
+		if (elen > left) {
+			if (show_errors) {
+				RTW_INFO("%s parse failed (id=%d elen=%d left=%lu)\n"
+					, __func__, id, elen, (unsigned long) left);
+			}
+			return ParseFailed;
+		}
+
+		switch (id) {
+		case MBSSID_NONTRANSMITTED_BSSID_PROFILE_ID:
+			if (rtw_mbssid_ntbssid_profile_match_id(pos, elen, mbssid_idx))
+				_rtw_ieee802_11_parse_elems(pos, elen, elems, show_errors, false);
+			break;
+		default:
+			break;
+		}
+		left -= elen;
+		pos += elen;
+	}
+
+	if (left)
+		return ParseFailed;
+
+	return unknown ? ParseUnknown : ParseOK;
+
+}
+#endif /* CONFIG_STA_MULTIPLE_BSSID */
+
+u8 key_char2num(u8 ch);
+u8 key_char2num(u8 ch)
+{
+	if ((ch >= '0') && (ch <= '9'))
+		return ch - '0';
+	else if ((ch >= 'a') && (ch <= 'f'))
+		return ch - 'a' + 10;
+	else if ((ch >= 'A') && (ch <= 'F'))
+		return ch - 'A' + 10;
+	else
+		return 0xff;
+}
+
+u8 str_2char2num(u8 hch, u8 lch);
+u8 str_2char2num(u8 hch, u8 lch)
+{
+	return (key_char2num(hch) * 10) + key_char2num(lch);
+}
+
+u8 key_2char2num(u8 hch, u8 lch);
+u8 key_2char2num(u8 hch, u8 lch)
+{
+	return (key_char2num(hch) << 4) | key_char2num(lch);
+}
+
+void macstr2num(u8 *dst, u8 *src);
+void macstr2num(u8 *dst, u8 *src)
+{
+	int	jj, kk;
+	for (jj = 0, kk = 0; jj < ETH_ALEN; jj++, kk += 3)
+		dst[jj] = key_2char2num(src[kk], src[kk + 1]);
+}
+
+u8 convert_ip_addr(u8 hch, u8 mch, u8 lch)
+{
+	return (key_char2num(hch) * 100) + (key_char2num(mch) * 10) + key_char2num(lch);
+}
+
+
+/*
+ * Description:
+ * rtw_check_invalid_mac_address:
+ * This is only used for checking mac address valid or not.
+ *
+ * Input:
+ * adapter: mac_address pointer.
+ * check_local_bit: check locally bit or not.
+ *
+ * Output:
+ * _TRUE: The mac address is invalid.
+ * _FALSE: The mac address is valid.
+ *
+ * Auther: Isaac.Li
+ */
+u8 rtw_check_invalid_mac_address(u8 *mac_addr, u8 check_local_bit)
+{
+	u8 null_mac_addr[ETH_ALEN] = {0, 0, 0, 0, 0, 0};
+	u8 multi_mac_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 res = _FALSE;
+
+	if (_rtw_memcmp(mac_addr, null_mac_addr, ETH_ALEN)) {
+		res = _TRUE;
+		goto func_exit;
+	}
+
+	if (_rtw_memcmp(mac_addr, multi_mac_addr, ETH_ALEN)) {
+		res = _TRUE;
+		goto func_exit;
+	}
+
+	if (mac_addr[0] & BIT0) {
+		res = _TRUE;
+		goto func_exit;
+	}
+
+	if (check_local_bit == _TRUE) {
+		if (mac_addr[0] & BIT1) {
+			res = _TRUE;
+			goto func_exit;
+		}
+	}
+
+func_exit:
+	return res;
+}
+
+extern char *rtw_initmac;
+/**
+ * rtw_macaddr_cfg - Decide the mac address used
+ * @out: buf to store mac address decided
+ * @hw_mac_addr: mac address from efuse/epprom
+ */
+void rtw_macaddr_cfg(u8 *out, const u8 *hw_mac_addr)
+{
+#define DEFAULT_RANDOM_MACADDR 1
+	u8 mac[ETH_ALEN] = {0};
+
+	if (out == NULL) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	/* Users specify the mac address */
+	if (rtw_initmac) {
+		int jj, kk;
+
+		for (jj = 0, kk = 0; jj < ETH_ALEN; jj++, kk += 3)
+			mac[jj] = key_2char2num(rtw_initmac[kk], rtw_initmac[kk + 1]);
+
+		goto err_chk;
+	}
+
+
+	/* Use the mac address stored in the Efuse */
+	if (hw_mac_addr) {
+		_rtw_memcpy(mac, hw_mac_addr, ETH_ALEN);
+		goto err_chk;
+	}
+
+err_chk:
+	platform_wifi_mac_addr(mac);
+
+	if (rtw_check_invalid_mac_address(mac, _TRUE) == _TRUE) {
+#if DEFAULT_RANDOM_MACADDR
+		RTW_ERR("invalid mac addr:"MAC_FMT", assign random MAC\n", MAC_ARG(mac));
+		*((u32 *)(&mac[2])) = rtw_random32();
+		mac[0] = 0x00;
+		mac[1] = 0xe0;
+		mac[2] = 0x4c;
+#else
+		RTW_ERR("invalid mac addr:"MAC_FMT", assign default one\n", MAC_ARG(mac));
+		mac[0] = 0x00;
+		mac[1] = 0xe0;
+		mac[2] = 0x4c;
+		mac[3] = 0x87;
+		mac[4] = 0x00;
+		mac[5] = 0x00;
+#endif
+	}
+
+	_rtw_memcpy(out, mac, ETH_ALEN);
+	RTW_INFO("%s mac addr:"MAC_FMT"\n", __func__, MAC_ARG(out));
+}
+
+#ifdef CONFIG_80211N_HT
+void dump_ht_cap_ie_content(void *sel, const u8 *buf, u32 buf_len)
+{
+	if (buf_len != HT_CAP_IE_LEN) {
+		RTW_PRINT_SEL(sel, "Invalid HT capability IE len:%d != %d\n", buf_len, HT_CAP_IE_LEN);
+		return;
+	}
+
+	RTW_PRINT_SEL(sel, "cap_info:%02x%02x:%s\n", *(buf), *(buf + 1)
+		, GET_HT_CAP_ELE_CHL_WIDTH(buf) ? " 40MHz" : " 20MHz");
+	RTW_PRINT_SEL(sel, "A-MPDU Parameters:"HT_AMPDU_PARA_FMT"\n"
+		      , HT_AMPDU_PARA_ARG(HT_CAP_ELE_AMPDU_PARA(buf)));
+	RTW_PRINT_SEL(sel, "Supported MCS Set:"HT_SUP_MCS_SET_FMT"\n"
+		      , HT_SUP_MCS_SET_ARG(HT_CAP_ELE_SUP_MCS_SET(buf)));
+}
+
+void dump_ht_cap_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *ht_cap_ie;
+	sint ht_cap_ielen;
+
+	ht_cap_ie = rtw_get_ie(ie, WLAN_EID_HT_CAP, &ht_cap_ielen, ie_len);
+	if (!ie || ht_cap_ie != ie)
+		return;
+
+	dump_ht_cap_ie_content(sel, ht_cap_ie + 2, ht_cap_ielen);
+}
+
+const char *const _ht_sc_offset_str[] = {
+	"SCN",
+	"SCA",
+	"SC-RSVD",
+	"SCB",
+};
+
+void dump_ht_op_ie_content(void *sel, const u8 *buf, u32 buf_len)
+{
+	if (buf_len != HT_OP_IE_LEN) {
+		RTW_PRINT_SEL(sel, "Invalid HT operation IE len:%d != %d\n", buf_len, HT_OP_IE_LEN);
+		return;
+	}
+
+	RTW_PRINT_SEL(sel, "ch:%u%s %s\n"
+		, GET_HT_OP_ELE_PRI_CHL(buf)
+		, GET_HT_OP_ELE_STA_CHL_WIDTH(buf) ? "" : " 20MHz only"
+		, ht_sc_offset_str(GET_HT_OP_ELE_2ND_CHL_OFFSET(buf))
+	);
+}
+
+void dump_ht_op_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *ht_op_ie;
+	sint ht_op_ielen;
+
+	ht_op_ie = rtw_get_ie(ie, WLAN_EID_HT_OPERATION, &ht_op_ielen, ie_len);
+	if (!ie || ht_op_ie != ie)
+		return;
+
+	dump_ht_op_ie_content(sel, ht_op_ie + 2, ht_op_ielen);
+}
+#endif /* CONFIG_80211N_HT */
+
+void dump_ies(void *sel, const u8 *buf, u32 buf_len)
+{
+	const u8 *pos = buf;
+	u8 id, len;
+
+	while (pos - buf + 1 < buf_len) {
+		id = *pos;
+		len = *(pos + 1);
+
+		RTW_PRINT_SEL(sel, "%s ID:%u, LEN:%u\n", __FUNCTION__, id, len);
+#ifdef CONFIG_80211N_HT
+		dump_ht_cap_ie(sel, pos, len + 2);
+		dump_ht_op_ie(sel, pos, len + 2);
+#endif
+#ifdef CONFIG_80211AC_VHT
+		dump_vht_cap_ie(sel, pos, len + 2);
+		dump_vht_op_ie(sel, pos, len + 2);
+#endif
+#ifdef CONFIG_80211AX_HE
+		/* CONFIG_80211AX_HE_TODO */
+#endif
+		dump_wps_ie(sel, pos, len + 2);
+#ifdef CONFIG_P2P
+		dump_p2p_ie(sel, pos, len + 2);
+#ifdef CONFIG_WFD
+		dump_wfd_ie(sel, pos, len + 2);
+#endif
+#endif
+#ifdef CONFIG_RTW_MULTI_AP
+		dump_multi_ap_ie(sel, pos, len + 2);
+#endif
+
+		pos += (2 + len);
+	}
+}
+
+void dump_wps_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *pos = ie;
+	u16 id;
+	u16 len;
+
+	const u8 *wps_ie;
+	uint wps_ielen;
+
+	wps_ie = rtw_get_wps_ie(ie, ie_len, NULL, &wps_ielen);
+	if (wps_ie != ie || wps_ielen == 0)
+		return;
+
+	pos += 6;
+	while (pos - ie + 4 <= ie_len) {
+		id = RTW_GET_BE16(pos);
+		len = RTW_GET_BE16(pos + 2);
+
+		RTW_PRINT_SEL(sel, "%s ID:0x%04x, LEN:%u%s\n", __func__, id, len
+			, ((pos - ie + 4 + len) <= ie_len) ? "" : "(exceed ie_len)");
+
+		pos += (4 + len);
+	}
+}
+
+#if defined(CONFIG_80211AX_HE) && defined(CONFIG_IEEE80211_BAND_6GHZ)
+u8 *rtw_ies_get_he_6g_op_info_ie(u8 *ies, int ies_len)
+{
+	u32 ie_len;
+	u8 he_op_eid_ext = WLAN_EID_EXTENSION_HE_OPERATION;
+	u8 *he_op_ie, *_6g_op_info_ie = NULL;
+
+	he_op_ie = rtw_get_ie_ex(ies, ies_len,
+		WLAN_EID_EXTENSION, &he_op_eid_ext, 1, NULL, &ie_len);
+
+	if (he_op_ie) {
+		if (GET_HE_OP_PARA_6GHZ_OP_INFO_PRESENT(he_op_ie + 3)) {
+			u32 len = HE_OPER_PARAMS_LEN +
+				HE_OPER_BSS_COLOR_INFO_LEN + HE_OPER_BASIC_MCS_LEN;
+
+			if (GET_HE_OP_PARA_VHT_OP_INFO_PRESENT(he_op_ie + 3))
+				len += HE_OPER_VHT_OPER_INFO_LEN;
+			if (GET_HE_OP_PARA_CO_HOSTED_BSS(he_op_ie + 3))
+				len += HE_OPER_MAX_COHOST_BSSID_LEN;
+
+			_6g_op_info_ie = he_op_ie + 3 + len;
+		}
+	}
+	return _6g_op_info_ie;
+}
+#endif /* CONFIG_80211AX_HE && CONFIG_IEEE80211_BAND_6GHZ */
+
+/**
+ * rtw_ies_get_chbw - get operation ch, bw, offset from IEs of BSS.
+ * @ies: pointer of the first tlv IE
+ * @ies_len: length of @ies
+ * @ch: pointer of ch, used as output
+ * @bw: pointer of bw, used as output
+ * @offset: pointer of offset, used as output
+ * @ht: check HT IEs
+ * @vht: check VHT IEs, if true imply ht is true
+ */
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY inline void rtw_ies_get_chbw(u8 *ies, int ies_len, u8 *ch, u8 *bw, u8 *offset, u8 ht, u8 vht)
+{
+	rtw_ies_get_bchbw(ies, ies_len, NULL, ch, bw, offset, NULL, NULL, ht, vht, 0);
+}
+#endif
+
+void rtw_ies_get_bchbw(u8 *ies, int ies_len, enum band_type *band, u8 *chan, u8 *bw,
+	u8 *offset, u8 *freq0, u8 *freq1, u8 ht, u8 vht, u8 he)
+{
+	u8 *p;
+	int	ie_len;
+
+	*chan = 0;
+	*bw = CHANNEL_WIDTH_20;
+	*offset = CHAN_OFFSET_NO_EXT;
+	if (freq0)
+		*freq0 = 0;
+	if (freq1)
+		*freq1 = 0;
+
+	p = rtw_get_ie(ies, _DSSET_IE_, &ie_len, ies_len);
+	if (p && ie_len > 0) {
+		*chan = *(p + 2);
+		if (band)
+			*band = *chan > 14 ? BAND_ON_5G : BAND_ON_24G;
+	}
+
+#ifdef CONFIG_80211N_HT
+	if (ht || vht) {
+		u8 *ht_cap_ie, *ht_op_ie;
+		int ht_cap_ielen, ht_op_ielen;
+
+		ht_cap_ie = rtw_get_ie(ies, EID_HTCapability, &ht_cap_ielen, ies_len);
+		if (ht_cap_ie && ht_cap_ielen) {
+			if (GET_HT_CAP_ELE_CHL_WIDTH(ht_cap_ie + 2))
+				*bw = CHANNEL_WIDTH_40;
+		}
+
+		ht_op_ie = rtw_get_ie(ies, EID_HTInfo, &ht_op_ielen, ies_len);
+		if (ht_op_ie && ht_op_ielen) {
+			if (*chan == 0) {
+				*chan = GET_HT_OP_ELE_PRI_CHL(ht_op_ie + 2);
+				if(band)
+					*band = (*chan > 14) ? BAND_ON_5G : BAND_ON_24G;
+			} else if (*chan != 0 && *chan != GET_HT_OP_ELE_PRI_CHL(ht_op_ie + 2)) {
+				RTW_INFO("%s ch inconsistent, DSSS:%u, HT primary:%u\n"
+					, __func__, *chan, GET_HT_OP_ELE_PRI_CHL(ht_op_ie + 2));
+			}
+
+			if (!GET_HT_OP_ELE_STA_CHL_WIDTH(ht_op_ie + 2))
+				*bw = CHANNEL_WIDTH_20;
+
+			if (*bw == CHANNEL_WIDTH_40) {
+				switch (GET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op_ie + 2)) {
+				case IEEE80211_SCA:
+					*offset = CHAN_OFFSET_UPPER;
+					break;
+				case IEEE80211_SCB:
+					*offset = CHAN_OFFSET_LOWER;
+					break;
+				}
+			}
+		}
+
+#ifdef CONFIG_80211AC_VHT
+		if (vht) {
+			u8 *vht_op_ie;
+			int vht_op_ielen;
+			u8 *vht_cap_ie;
+			int vht_cap_ielen;
+
+			vht_cap_ie = rtw_get_ie(ies, EID_VHTCapability, &vht_cap_ielen, ies_len);
+			if (vht_cap_ie && vht_cap_ielen) {
+
+				vht_op_ie = rtw_get_ie(ies, EID_VHTOperation, &vht_op_ielen, ies_len);
+				if (vht_op_ie) {
+					*bw = rtw_vht_get_oper_bw(ht_op_ie,
+								  vht_cap_ie,
+								  vht_op_ie,
+								  *bw);
+				}
+			}
+		}
+#endif /* CONFIG_80211AC_VHT */
+	}
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_80211AX_HE
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (he) {
+		u8 *_6g_op_info_ie = NULL;
+
+		_6g_op_info_ie = rtw_ies_get_he_6g_op_info_ie(ies, ies_len);
+
+		if (_6g_op_info_ie) {
+			*chan = GET_HE_OP_INFO_PRIMARY_CHAN(_6g_op_info_ie);
+			*bw = GET_HE_OP_INFO_CHAN_WIDTH(_6g_op_info_ie);
+			if (freq0)
+				*freq0 = GET_HE_OP_INFO_CHAN_CTR_FREQ_SEG0(_6g_op_info_ie);
+			if (freq1)
+				*freq1 = GET_HE_OP_INFO_CHAN_CTR_FREQ_SEG1(_6g_op_info_ie);
+			if (band)
+				*band = BAND_ON_6G;
+			if (!rtw_get_offset_by_bchbw(BAND_ON_6G, *chan, *bw, offset)) {
+				RTW_INFO("%s get 6ghz channel offset fail, chan=%u, bw=%u\n",
+					__func__, *chan, *bw);
+			}
+		}
+	}
+#endif
+#endif /* CONFIG_80211AX_HE */
+}
+
+void rtw_bss_get_chbw(WLAN_BSSID_EX *bss, enum band_type *band, u8 *ch, u8 *bw, u8 *offset, u8 ht, u8 vht, u8 he)
+{
+	rtw_ies_get_bchbw(bss->IEs + _FIXED_IE_LENGTH_
+		, bss->IELength - _FIXED_IE_LENGTH_
+		, band, ch, bw, offset, NULL, NULL, ht, vht, he);
+
+	if (*ch == 0) {
+		*ch = bss->Configuration.DSConfig;
+		*band = bss->Configuration.Band;
+	}
+	else if (*ch != bss->Configuration.DSConfig) {
+		RTW_INFO("inconsistent ch - ies:%u bss->Configuration.DSConfig:%u\n"
+			 , *ch, bss->Configuration.DSConfig);
+		*ch = bss->Configuration.DSConfig;
+		*band = bss->Configuration.Band;
+		rtw_warn_on(1);
+	}
+}
+
+/**
+ * rtw_get_p2p_merged_len - Get merged ie length from muitiple p2p ies.
+ * @in_ie: Pointer of the first p2p ie
+ * @in_len: Total len of muiltiple p2p ies
+ * Returns: Length of merged p2p ie length
+ */
+u32 rtw_get_p2p_merged_ies_len(u8 *in_ie, u32 in_len)
+{
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 OUI[4] = { 0x50, 0x6f, 0x9a, 0x09 };
+	int i = 0;
+	int len = 0;
+
+	while (i < in_len) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(in_ie + i);
+
+		if (pIE->ElementID == _VENDOR_SPECIFIC_IE_ && _rtw_memcmp(pIE->data, OUI, 4)) {
+			len += pIE->Length - 4; /* 4 is P2P OUI length, don't count it in this loop */
+		}
+
+		i += (pIE->Length + 2);
+	}
+
+	return len + 4;	/* Append P2P OUI length at last. */
+}
+
+/**
+ * rtw_p2p_merge_ies - Merge muitiple p2p ies into one
+ * @in_ie: Pointer of the first p2p ie
+ * @in_len: Total len of muiltiple p2p ies
+ * @merge_ie: Pointer of merged ie
+ * Returns: Length of merged p2p ie
+ */
+int rtw_p2p_merge_ies(u8 *in_ie, u32 in_len, u8 *merge_ie)
+{
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 len = 0;
+	u8 OUI[4] = { 0x50, 0x6f, 0x9a, 0x09 };
+	u8 ELOUI[6] = { 0xDD, 0x00, 0x50, 0x6f, 0x9a, 0x09 };	/* EID;Len;OUI, Len would copy at the end of function */
+	int i = 0;
+
+	if (merge_ie != NULL) {
+		/* Set first P2P OUI */
+		_rtw_memcpy(merge_ie, ELOUI, 6);
+		merge_ie += 6;
+
+		while (i < in_len) {
+			pIE = (PNDIS_802_11_VARIABLE_IEs)(in_ie + i);
+
+			/* Take out the rest of P2P OUIs */
+			if (pIE->ElementID == _VENDOR_SPECIFIC_IE_ && _rtw_memcmp(pIE->data, OUI, 4)) {
+				_rtw_memcpy(merge_ie, pIE->data + 4, pIE->Length - 4);
+				len += pIE->Length - 4;
+				merge_ie += pIE->Length - 4;
+			}
+
+			i += (pIE->Length + 2);
+		}
+
+		return len + 4;	/* 4 is for P2P OUI */
+
+	}
+
+	return 0;
+}
+
+void dump_p2p_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *pos = ie;
+	u8 id;
+	u16 len;
+
+	const u8 *p2p_ie;
+	uint p2p_ielen;
+
+	p2p_ie = rtw_get_p2p_ie(ie, ie_len, NULL, &p2p_ielen);
+	if (p2p_ie != ie || p2p_ielen == 0)
+		return;
+
+	pos += 6;
+	while (pos - ie + 3 <= ie_len) {
+		id = *pos;
+		len = RTW_GET_LE16(pos + 1);
+
+		RTW_PRINT_SEL(sel, "%s ID:%u, LEN:%u%s\n", __func__, id, len
+			, ((pos - ie + 3 + len) <= ie_len) ? "" : "(exceed ie_len)");
+
+		pos += (3 + len);
+	}
+}
+
+/**
+ * rtw_get_p2p_ie - Search P2P IE from a series of IEs
+ * @in_ie: Address of IEs to search
+ * @in_len: Length limit from in_ie
+ * @p2p_ie: If not NULL and P2P IE is found, P2P IE will be copied to the buf starting from p2p_ie
+ * @p2p_ielen: If not NULL and P2P IE is found, will set to the length of the entire P2P IE
+ *
+ * Returns: The address of the P2P IE found, or NULL
+ */
+u8 *rtw_get_p2p_ie(const u8 *in_ie, int in_len, u8 *p2p_ie, uint *p2p_ielen)
+{
+	uint cnt;
+	const u8 *p2p_ie_ptr = NULL;
+	u8 eid, p2p_oui[4] = {0x50, 0x6F, 0x9A, 0x09};
+
+	if (p2p_ielen)
+		*p2p_ielen = 0;
+
+	if (!in_ie || in_len < 0) {
+		rtw_warn_on(1);
+		return (u8 *)p2p_ie_ptr;
+	}
+
+	if (in_len <= 0)
+		return (u8 *)p2p_ie_ptr;
+
+	cnt = 0;
+
+	while (cnt + 1 + 4 < in_len) {
+		eid = in_ie[cnt];
+
+		if (cnt + 1 + 4 >= MAX_IE_SZ) {
+			rtw_warn_on(1);
+			return NULL;
+		}
+
+		if (eid == WLAN_EID_VENDOR_SPECIFIC && _rtw_memcmp(&in_ie[cnt + 2], p2p_oui, 4) == _TRUE) {
+			p2p_ie_ptr = in_ie + cnt;
+
+			if (p2p_ie)
+				_rtw_memcpy(p2p_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			if (p2p_ielen)
+				*p2p_ielen = in_ie[cnt + 1] + 2;
+
+			break;
+		} else
+			cnt += in_ie[cnt + 1] + 2;
+
+	}
+
+	return (u8 *)p2p_ie_ptr;
+}
+
+u8 *rtw_get_vendor_ie(const u8 *in_ie, int in_len, u8 *vendor_ie, uint *vendor_ielen)
+{
+	uint cnt;
+	const u8 *vendor_ie_ptr = NULL;
+	u8 eid, vendor_oui[4] = {0xC8, 0x3A, 0x6B, 0x01};
+
+	if (vendor_ielen)
+		*vendor_ielen = 0;
+
+	if (!in_ie || in_len < 0) {
+		rtw_warn_on(1);
+		return (u8 *)vendor_ie_ptr;
+	}
+
+	if (in_len <= 0)
+		return (u8 *)vendor_ie_ptr;
+
+	cnt = 0;
+
+	while (cnt + 1 + 4 < in_len) {
+		eid = in_ie[cnt];
+
+		if (cnt + 1 + 4 >= MAX_IE_SZ) {
+			rtw_warn_on(1);
+			return NULL;
+		}
+
+		if (eid == WLAN_EID_VENDOR_SPECIFIC && _rtw_memcmp(&in_ie[cnt + 2], vendor_oui, 4) == _TRUE) {
+			vendor_ie_ptr = in_ie + cnt;
+
+			if (vendor_ie)
+				_rtw_memcpy(vendor_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			if (vendor_ielen)
+				*vendor_ielen = in_ie[cnt + 1] + 2;
+
+			break;
+		} else
+			cnt += in_ie[cnt + 1] + 2;
+
+	}
+
+	return (u8 *)vendor_ie_ptr;
+}
+
+/**
+ * rtw_get_p2p_attr - Search a specific P2P attribute from a given P2P IE
+ * @p2p_ie: Address of P2P IE to search
+ * @p2p_ielen: Length limit from p2p_ie
+ * @target_attr_id: The attribute ID of P2P attribute to search
+ * @buf_attr: If not NULL and the P2P attribute is found, P2P attribute will be copied to the buf starting from buf_attr
+ * @len_attr: If not NULL and the P2P attribute is found, will set to the length of the entire P2P attribute
+ *
+ * Returns: the address of the specific WPS attribute found, or NULL
+ */
+u8 *rtw_get_p2p_attr(u8 *p2p_ie, uint p2p_ielen, u8 target_attr_id , u8 *buf_attr, u32 *len_attr)
+{
+	u8 *attr_ptr = NULL;
+	u8 *target_attr_ptr = NULL;
+	u8 p2p_oui[4] = {0x50, 0x6F, 0x9A, 0x09};
+
+	if (len_attr)
+		*len_attr = 0;
+
+	if (!p2p_ie
+	    || p2p_ielen <= 6
+	    || (p2p_ie[0] != WLAN_EID_VENDOR_SPECIFIC)
+	    || (_rtw_memcmp(p2p_ie + 2, p2p_oui, 4) != _TRUE))
+		return attr_ptr;
+
+	/* 6 = 1(Element ID) + 1(Length) + 3 (OUI) + 1(OUI Type) */
+	attr_ptr = p2p_ie + 6; /* goto first attr */
+
+	while ((attr_ptr - p2p_ie + 3) <= p2p_ielen) {
+		/* 3 = 1(Attribute ID) + 2(Length) */
+		u8 attr_id = *attr_ptr;
+		u16 attr_data_len = RTW_GET_LE16(attr_ptr + 1);
+		u16 attr_len = attr_data_len + 3;
+
+		if (0)
+			RTW_INFO("%s attr_ptr:%p, id:%u, length:%u\n", __func__, attr_ptr, attr_id, attr_data_len);
+
+		if ((attr_ptr - p2p_ie + attr_len) > p2p_ielen)
+			break;
+
+		if (attr_id == target_attr_id) {
+			target_attr_ptr = attr_ptr;
+
+			if (buf_attr)
+				_rtw_memcpy(buf_attr, attr_ptr, attr_len);
+
+			if (len_attr)
+				*len_attr = attr_len;
+
+			break;
+		} else
+			attr_ptr += attr_len;
+	}
+
+	return target_attr_ptr;
+}
+
+/**
+ * rtw_get_p2p_attr_content - Search a specific P2P attribute content from a given P2P IE
+ * @p2p_ie: Address of P2P IE to search
+ * @p2p_ielen: Length limit from p2p_ie
+ * @target_attr_id: The attribute ID of P2P attribute to search
+ * @buf_content: If not NULL and the P2P attribute is found, P2P attribute content will be copied to the buf starting from buf_content
+ *               If len_content is NULL, only copy one byte.
+ * @len_content: If not NULL and the P2P attribute is found, will set to the length of the P2P attribute content
+ *
+ * Returns: the address of the specific P2P attribute content found, or NULL
+ */
+u8 *rtw_get_p2p_attr_content(u8 *p2p_ie, uint p2p_ielen, u8 target_attr_id , u8 *buf_content, uint *len_content)
+{
+	u8 *attr_ptr;
+	u32 attr_len;
+
+	attr_ptr = rtw_get_p2p_attr(p2p_ie, p2p_ielen, target_attr_id, NULL, &attr_len);
+
+	if (attr_ptr && attr_len) {
+		if (len_content) {
+			if ((buf_content && (*len_content > (attr_len - 3))) || !buf_content)
+				*len_content = attr_len - 3;
+		}
+
+		if (len_content && buf_content) {
+			_rtw_memcpy(buf_content, attr_ptr + 3, *len_content);
+		} else if (buf_content) {
+			_rtw_memcpy(buf_content, attr_ptr + 3, 1);
+		}
+
+		return attr_ptr + 3;
+	}
+
+	if (len_content)
+		*len_content = 0;
+
+	return NULL;
+}
+
+u32 rtw_set_p2p_attr_content(u8 *pbuf, u8 attr_id, u16 attr_len, u8 *pdata_attr)
+{
+	u32 a_len;
+
+	*pbuf = attr_id;
+
+	/* *(u16*)(pbuf + 1) = cpu_to_le16(attr_len); */
+	RTW_PUT_LE16(pbuf + 1, attr_len);
+
+	if (pdata_attr)
+		_rtw_memcpy(pbuf + 3, pdata_attr, attr_len);
+
+	a_len = attr_len + 3;
+
+	return a_len;
+}
+
+uint rtw_del_p2p_ie(u8 *ies, uint ies_len_ori, const char *msg)
+{
+#define DBG_DEL_P2P_IE 0
+
+	u8 *target_ie;
+	u32 target_ie_len;
+	uint ies_len = ies_len_ori;
+	int index = 0;
+
+	while (1) {
+		target_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &target_ie_len);
+		if (target_ie && target_ie_len) {
+			u8 *next_ie = target_ie + target_ie_len;
+			uint remain_len = ies_len - (next_ie - ies);
+
+			if (DBG_DEL_P2P_IE && msg) {
+				RTW_INFO("%s %d before\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ies, ies_len);
+
+				RTW_INFO("ies:%p, ies_len:%u\n", ies, ies_len);
+				RTW_INFO("target_ie:%p, target_ie_len:%u\n", target_ie, target_ie_len);
+				RTW_INFO("next_ie:%p, remain_len:%u\n", next_ie, remain_len);
+			}
+
+			_rtw_memmove(target_ie, next_ie, remain_len);
+			_rtw_memset(target_ie + remain_len, 0, target_ie_len);
+			ies_len -= target_ie_len;
+
+			if (DBG_DEL_P2P_IE && msg) {
+				RTW_INFO("%s %d after\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ies, ies_len);
+			}
+
+			index++;
+		} else
+			break;
+	}
+
+	return ies_len;
+}
+
+uint rtw_del_p2p_attr(u8 *ie, uint ielen_ori, u8 attr_id)
+{
+#define DBG_DEL_P2P_ATTR 0
+
+	u8 *target_attr;
+	u32 target_attr_len;
+	uint ielen = ielen_ori;
+	int index = 0;
+
+	while (1) {
+		target_attr = rtw_get_p2p_attr(ie, ielen, attr_id, NULL, &target_attr_len);
+		if (target_attr && target_attr_len) {
+			u8 *next_attr = target_attr + target_attr_len;
+			uint remain_len = ielen - (next_attr - ie);
+
+			if (DBG_DEL_P2P_ATTR) {
+				RTW_INFO("%s %d before\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ie, ielen);
+
+				RTW_INFO("ie:%p, ielen:%u\n", ie, ielen);
+				RTW_INFO("target_attr:%p, target_attr_len:%u\n", target_attr, target_attr_len);
+				RTW_INFO("next_attr:%p, remain_len:%u\n", next_attr, remain_len);
+			}
+
+			_rtw_memmove(target_attr, next_attr, remain_len);
+			_rtw_memset(target_attr + remain_len, 0, target_attr_len);
+			*(ie + 1) -= target_attr_len;
+			ielen -= target_attr_len;
+
+			if (DBG_DEL_P2P_ATTR) {
+				RTW_INFO("%s %d after\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ie, ielen);
+			}
+
+			index++;
+		} else
+			break;
+	}
+
+	return ielen;
+}
+
+inline u8 *rtw_bss_ex_get_p2p_ie(WLAN_BSSID_EX *bss_ex, u8 *p2p_ie, uint *p2p_ielen)
+{
+	return rtw_get_p2p_ie(BSS_EX_TLV_IES(bss_ex), BSS_EX_TLV_IES_LEN(bss_ex), p2p_ie, p2p_ielen);
+}
+
+void rtw_bss_ex_del_p2p_ie(WLAN_BSSID_EX *bss_ex)
+{
+#define DBG_BSS_EX_DEL_P2P_IE 0
+
+	u8 *ies = BSS_EX_TLV_IES(bss_ex);
+	uint ies_len_ori = BSS_EX_TLV_IES_LEN(bss_ex);
+	uint ies_len;
+
+	ies_len = rtw_del_p2p_ie(ies, ies_len_ori, DBG_BSS_EX_DEL_P2P_IE ? __func__ : NULL);
+	bss_ex->IELength -= ies_len_ori - ies_len;
+}
+
+void rtw_bss_ex_del_p2p_attr(WLAN_BSSID_EX *bss_ex, u8 attr_id)
+{
+#define DBG_BSS_EX_DEL_P2P_ATTR 0
+
+	u8 *ies = BSS_EX_TLV_IES(bss_ex);
+	uint ies_len = BSS_EX_TLV_IES_LEN(bss_ex);
+
+	u8 *ie;
+	uint ie_len, ie_len_ori;
+
+	int index = 0;
+
+	while (1) {
+		ie = rtw_get_p2p_ie(ies, ies_len, NULL, &ie_len_ori);
+		if (ie) {
+			u8 *next_ie_ori = ie + ie_len_ori;
+			uint remain_len = bss_ex->IELength - (next_ie_ori - bss_ex->IEs);
+			u8 has_target_attr = 0;
+
+			if (DBG_BSS_EX_DEL_P2P_ATTR) {
+				if (rtw_get_p2p_attr(ie, ie_len_ori, attr_id, NULL, NULL)) {
+					RTW_INFO("%s %d before\n", __func__, index);
+					dump_ies(RTW_DBGDUMP, BSS_EX_TLV_IES(bss_ex), BSS_EX_TLV_IES_LEN(bss_ex));
+
+					RTW_INFO("ies:%p, ies_len:%u\n", ies, ies_len);
+					RTW_INFO("ie:%p, ie_len_ori:%u\n", ie, ie_len_ori);
+					RTW_INFO("next_ie_ori:%p, remain_len:%u\n", next_ie_ori, remain_len);
+					has_target_attr = 1;
+				}
+			}
+
+			ie_len = rtw_del_p2p_attr(ie, ie_len_ori, attr_id);
+			if (ie_len != ie_len_ori) {
+				u8 *next_ie = ie + ie_len;
+
+				_rtw_memmove(next_ie, next_ie_ori, remain_len);
+				_rtw_memset(next_ie + remain_len, 0, ie_len_ori - ie_len);
+				bss_ex->IELength -= ie_len_ori - ie_len;
+
+				ies = next_ie;
+			} else
+				ies = next_ie_ori;
+
+			if (DBG_BSS_EX_DEL_P2P_ATTR) {
+				if (has_target_attr) {
+					RTW_INFO("%s %d after\n", __func__, index);
+					dump_ies(RTW_DBGDUMP, BSS_EX_TLV_IES(bss_ex), BSS_EX_TLV_IES_LEN(bss_ex));
+				}
+			}
+
+			ies_len = remain_len;
+
+			index++;
+		} else
+			break;
+	}
+}
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+int rtw_chk_p2p_wildcard_ssid(WLAN_BSSID_EX *bss_ex)
+{
+	if (_rtw_memcmp(bss_ex->Ssid.Ssid, "DIRECT-", P2P_WILDCARD_SSID_LEN))
+		return _SUCCESS;
+	else
+		return _FAIL;
+}
+
+int rtw_chk_p2p_ie(WLAN_BSSID_EX *bss_ex)
+{
+	u8 *ies = BSS_EX_TLV_IES(bss_ex);
+	uint ies_len = BSS_EX_TLV_IES_LEN(bss_ex);
+
+	u8 *ie;
+	uint ie_len_ori;
+
+	ie = rtw_get_p2p_ie(ies, ies_len, NULL, &ie_len_ori);
+	if(ie)
+		return _SUCCESS;
+	else
+		return _FAIL;
+}
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+
+void dump_wfd_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *pos = ie;
+	u8 id;
+	u16 len;
+
+	const u8 *wfd_ie;
+	uint wfd_ielen;
+
+	wfd_ie = rtw_get_wfd_ie(ie, ie_len, NULL, &wfd_ielen);
+	if (wfd_ie != ie || wfd_ielen == 0)
+		return;
+
+	pos += 6;
+	while (pos - ie + 3 <= ie_len) {
+		id = *pos;
+		len = RTW_GET_BE16(pos + 1);
+
+		RTW_PRINT_SEL(sel, "%s ID:%u, LEN:%u%s\n", __func__, id, len
+			, ((pos - ie + 3 + len) <= ie_len) ? "" : "(exceed ie_len)");
+
+		pos += (3 + len);
+	}
+}
+
+/**
+ * rtw_get_wfd_ie - Search WFD IE from a series of IEs
+ * @in_ie: Address of IEs to search
+ * @in_len: Length limit from in_ie
+ * @wfd_ie: If not NULL and WFD IE is found, WFD IE will be copied to the buf starting from wfd_ie
+ * @wfd_ielen: If not NULL and WFD IE is found, will set to the length of the entire WFD IE
+ *
+ * Returns: The address of the P2P IE found, or NULL
+ */
+u8 *rtw_get_wfd_ie(const u8 *in_ie, int in_len, u8 *wfd_ie, uint *wfd_ielen)
+{
+	uint cnt;
+	const u8 *wfd_ie_ptr = NULL;
+	u8 eid, wfd_oui[4] = {0x50, 0x6F, 0x9A, 0x0A};
+
+	if (wfd_ielen)
+		*wfd_ielen = 0;
+
+	if (!in_ie || in_len < 0) {
+		rtw_warn_on(1);
+		return (u8 *)wfd_ie_ptr;
+	}
+
+	if (in_len <= 0)
+		return (u8 *)wfd_ie_ptr;
+
+	cnt = 0;
+
+	while (cnt + 1 + 4 < in_len) {
+		eid = in_ie[cnt];
+
+		if (cnt + 1 + 4 >= MAX_IE_SZ) {
+			rtw_warn_on(1);
+			return NULL;
+		}
+
+		if (eid == WLAN_EID_VENDOR_SPECIFIC && _rtw_memcmp(&in_ie[cnt + 2], wfd_oui, 4) == _TRUE) {
+			wfd_ie_ptr = in_ie + cnt;
+
+			if (wfd_ie)
+				_rtw_memcpy(wfd_ie, &in_ie[cnt], in_ie[cnt + 1] + 2);
+
+			if (wfd_ielen)
+				*wfd_ielen = in_ie[cnt + 1] + 2;
+
+			break;
+		} else
+			cnt += in_ie[cnt + 1] + 2;
+
+	}
+
+	return (u8 *)wfd_ie_ptr;
+}
+
+/**
+ * rtw_get_wfd_attr - Search a specific WFD attribute from a given WFD IE
+ * @wfd_ie: Address of WFD IE to search
+ * @wfd_ielen: Length limit from wfd_ie
+ * @target_attr_id: The attribute ID of WFD attribute to search
+ * @buf_attr: If not NULL and the WFD attribute is found, WFD attribute will be copied to the buf starting from buf_attr
+ * @len_attr: If not NULL and the WFD attribute is found, will set to the length of the entire WFD attribute
+ *
+ * Returns: the address of the specific WPS attribute found, or NULL
+ */
+u8 *rtw_get_wfd_attr(u8 *wfd_ie, uint wfd_ielen, u8 target_attr_id, u8 *buf_attr, u32 *len_attr)
+{
+	u8 *attr_ptr = NULL;
+	u8 *target_attr_ptr = NULL;
+	u8 wfd_oui[4] = {0x50, 0x6F, 0x9A, 0x0A};
+
+	if (len_attr)
+		*len_attr = 0;
+
+	if (!wfd_ie
+	    || wfd_ielen <= 6
+	    || (wfd_ie[0] != WLAN_EID_VENDOR_SPECIFIC)
+	    || (_rtw_memcmp(wfd_ie + 2, wfd_oui, 4) != _TRUE))
+		return attr_ptr;
+
+	/* 6 = 1(Element ID) + 1(Length) + 3 (OUI) + 1(OUI Type) */
+	attr_ptr = wfd_ie + 6; /* goto first attr */
+
+	while ((attr_ptr - wfd_ie + 3) <= wfd_ielen) {
+		/* 3 = 1(Attribute ID) + 2(Length) */
+		u8 attr_id = *attr_ptr;
+		u16 attr_data_len = RTW_GET_BE16(attr_ptr + 1);
+		u16 attr_len = attr_data_len + 3;
+
+		if (0)
+			RTW_INFO("%s attr_ptr:%p, id:%u, length:%u\n", __func__, attr_ptr, attr_id, attr_data_len);
+
+		if ((attr_ptr - wfd_ie + attr_len) > wfd_ielen)
+			break;
+
+		if (attr_id == target_attr_id) {
+			target_attr_ptr = attr_ptr;
+
+			if (buf_attr)
+				_rtw_memcpy(buf_attr, attr_ptr, attr_len);
+
+			if (len_attr)
+				*len_attr = attr_len;
+
+			break;
+		} else
+			attr_ptr += attr_len;
+	}
+
+	return target_attr_ptr;
+}
+
+/**
+ * rtw_get_wfd_attr_content - Search a specific WFD attribute content from a given WFD IE
+ * @wfd_ie: Address of WFD IE to search
+ * @wfd_ielen: Length limit from wfd_ie
+ * @target_attr_id: The attribute ID of WFD attribute to search
+ * @buf_content: If not NULL and the WFD attribute is found, WFD attribute content will be copied to the buf starting from buf_content
+ * @len_content: If not NULL and the WFD attribute is found, will set to the length of the WFD attribute content
+ *
+ * Returns: the address of the specific WFD attribute content found, or NULL
+ */
+u8 *rtw_get_wfd_attr_content(u8 *wfd_ie, uint wfd_ielen, u8 target_attr_id, u8 *buf_content, uint *len_content)
+{
+	u8 *attr_ptr;
+	u32 attr_len;
+
+	if (len_content)
+		*len_content = 0;
+
+	attr_ptr = rtw_get_wfd_attr(wfd_ie, wfd_ielen, target_attr_id, NULL, &attr_len);
+
+	if (attr_ptr && attr_len) {
+		if (buf_content)
+			_rtw_memcpy(buf_content, attr_ptr + 3, attr_len - 3);
+
+		if (len_content)
+			*len_content = attr_len - 3;
+
+		return attr_ptr + 3;
+	}
+
+	return NULL;
+}
+
+uint rtw_del_wfd_ie(u8 *ies, uint ies_len_ori, const char *msg)
+{
+#define DBG_DEL_WFD_IE 0
+
+	u8 *target_ie;
+	u32 target_ie_len;
+	uint ies_len = ies_len_ori;
+	int index = 0;
+
+	while (1) {
+		target_ie = rtw_get_wfd_ie(ies, ies_len, NULL, &target_ie_len);
+		if (target_ie && target_ie_len) {
+			u8 *next_ie = target_ie + target_ie_len;
+			uint remain_len = ies_len - (next_ie - ies);
+
+			if (DBG_DEL_WFD_IE && msg) {
+				RTW_INFO("%s %d before\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ies, ies_len);
+
+				RTW_INFO("ies:%p, ies_len:%u\n", ies, ies_len);
+				RTW_INFO("target_ie:%p, target_ie_len:%u\n", target_ie, target_ie_len);
+				RTW_INFO("next_ie:%p, remain_len:%u\n", next_ie, remain_len);
+			}
+
+			_rtw_memmove(target_ie, next_ie, remain_len);
+			_rtw_memset(target_ie + remain_len, 0, target_ie_len);
+			ies_len -= target_ie_len;
+
+			if (DBG_DEL_WFD_IE && msg) {
+				RTW_INFO("%s %d after\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ies, ies_len);
+			}
+
+			index++;
+		} else
+			break;
+	}
+
+	return ies_len;
+}
+
+uint rtw_del_wfd_attr(u8 *ie, uint ielen_ori, u8 attr_id)
+{
+#define DBG_DEL_WFD_ATTR 0
+
+	u8 *target_attr;
+	u32 target_attr_len;
+	uint ielen = ielen_ori;
+	int index = 0;
+
+	while (1) {
+		target_attr = rtw_get_wfd_attr(ie, ielen, attr_id, NULL, &target_attr_len);
+		if (target_attr && target_attr_len) {
+			u8 *next_attr = target_attr + target_attr_len;
+			uint remain_len = ielen - (next_attr - ie);
+
+			if (DBG_DEL_WFD_ATTR) {
+				RTW_INFO("%s %d before\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ie, ielen);
+
+				RTW_INFO("ie:%p, ielen:%u\n", ie, ielen);
+				RTW_INFO("target_attr:%p, target_attr_len:%u\n", target_attr, target_attr_len);
+				RTW_INFO("next_attr:%p, remain_len:%u\n", next_attr, remain_len);
+			}
+
+			_rtw_memmove(target_attr, next_attr, remain_len);
+			_rtw_memset(target_attr + remain_len, 0, target_attr_len);
+			*(ie + 1) -= target_attr_len;
+			ielen -= target_attr_len;
+
+			if (DBG_DEL_WFD_ATTR) {
+				RTW_INFO("%s %d after\n", __func__, index);
+				dump_ies(RTW_DBGDUMP, ie, ielen);
+			}
+
+			index++;
+		} else
+			break;
+	}
+
+	return ielen;
+}
+
+inline u8 *rtw_bss_ex_get_wfd_ie(WLAN_BSSID_EX *bss_ex, u8 *wfd_ie, uint *wfd_ielen)
+{
+	return rtw_get_wfd_ie(BSS_EX_TLV_IES(bss_ex), BSS_EX_TLV_IES_LEN(bss_ex), wfd_ie, wfd_ielen);
+}
+
+void rtw_bss_ex_del_wfd_ie(WLAN_BSSID_EX *bss_ex)
+{
+#define DBG_BSS_EX_DEL_WFD_IE 0
+	u8 *ies = BSS_EX_TLV_IES(bss_ex);
+	uint ies_len_ori = BSS_EX_TLV_IES_LEN(bss_ex);
+	uint ies_len;
+
+	ies_len = rtw_del_wfd_ie(ies, ies_len_ori, DBG_BSS_EX_DEL_WFD_IE ? __func__ : NULL);
+	bss_ex->IELength -= ies_len_ori - ies_len;
+}
+
+void rtw_bss_ex_del_wfd_attr(WLAN_BSSID_EX *bss_ex, u8 attr_id)
+{
+#define DBG_BSS_EX_DEL_WFD_ATTR 0
+
+	u8 *ies = BSS_EX_TLV_IES(bss_ex);
+	uint ies_len = BSS_EX_TLV_IES_LEN(bss_ex);
+
+	u8 *ie;
+	uint ie_len, ie_len_ori;
+
+	int index = 0;
+
+	while (1) {
+		ie = rtw_get_wfd_ie(ies, ies_len, NULL, &ie_len_ori);
+		if (ie) {
+			u8 *next_ie_ori = ie + ie_len_ori;
+			uint remain_len = bss_ex->IELength - (next_ie_ori - bss_ex->IEs);
+			u8 has_target_attr = 0;
+
+			if (DBG_BSS_EX_DEL_WFD_ATTR) {
+				if (rtw_get_wfd_attr(ie, ie_len_ori, attr_id, NULL, NULL)) {
+					RTW_INFO("%s %d before\n", __func__, index);
+					dump_ies(RTW_DBGDUMP, BSS_EX_TLV_IES(bss_ex), BSS_EX_TLV_IES_LEN(bss_ex));
+
+					RTW_INFO("ies:%p, ies_len:%u\n", ies, ies_len);
+					RTW_INFO("ie:%p, ie_len_ori:%u\n", ie, ie_len_ori);
+					RTW_INFO("next_ie_ori:%p, remain_len:%u\n", next_ie_ori, remain_len);
+					has_target_attr = 1;
+				}
+			}
+
+			ie_len = rtw_del_wfd_attr(ie, ie_len_ori, attr_id);
+			if (ie_len != ie_len_ori) {
+				u8 *next_ie = ie + ie_len;
+
+				_rtw_memmove(next_ie, next_ie_ori, remain_len);
+				_rtw_memset(next_ie + remain_len, 0, ie_len_ori - ie_len);
+				bss_ex->IELength -= ie_len_ori - ie_len;
+
+				ies = next_ie;
+			} else
+				ies = next_ie_ori;
+
+			if (DBG_BSS_EX_DEL_WFD_ATTR) {
+				if (has_target_attr) {
+					RTW_INFO("%s %d after\n", __func__, index);
+					dump_ies(RTW_DBGDUMP, BSS_EX_TLV_IES(bss_ex), BSS_EX_TLV_IES_LEN(bss_ex));
+				}
+			}
+
+			ies_len = remain_len;
+
+			index++;
+		} else
+			break;
+	}
+}
+
+#ifdef CONFIG_RTW_MULTI_AP
+void dump_multi_ap_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *pos = ie;
+	u8 id;
+	u8 len;
+
+	const u8 *multi_ap_ie;
+	uint multi_ap_ielen;
+
+	multi_ap_ie = rtw_get_ie_ex(ie, ie_len, WLAN_EID_VENDOR_SPECIFIC, MULTI_AP_OUI, 4, NULL, &multi_ap_ielen);
+	if (multi_ap_ie != ie || multi_ap_ielen == 0)
+		return;
+
+	pos += 6;
+	while (pos - ie + 2 <= ie_len) {
+		id = *pos;
+		len = *(pos + 1);
+
+		RTW_PRINT_SEL(sel, "%s ID:%u, LEN:%u%s\n", __func__, id, len
+			, ((pos - ie + 2 + len) <= ie_len) ? "" : "(exceed ie_len)");
+		RTW_DUMP_SEL(sel, pos + 2, len);
+
+		pos += (2 + len);
+	}
+}
+
+/**
+ * rtw_get_multi_ap_ext - Search Multi-AP IE from a series of IEs and return extension subelement value
+ * @ies: Address of IEs to search
+ * @ies_len: Length limit from in_ie
+ *
+ * Returns: The address of the target IE found, or NULL
+ */
+u8 rtw_get_multi_ap_ie_ext(const u8 *ies, int ies_len)
+{
+	u8 *ie;
+	uint ielen;
+	u8 val = 0;
+
+	ie = rtw_get_ie_ex(ies, ies_len, WLAN_EID_VENDOR_SPECIFIC, MULTI_AP_OUI, 4, NULL, &ielen);
+	if (ielen < 9)
+		goto exit;
+
+	if (ie[6] != MULTI_AP_SUB_ELEM_TYPE)
+		goto exit;
+
+	val = ie[8];
+
+exit:
+	return val;
+}
+
+u8 *rtw_set_multi_ap_ie_ext(u8 *pbuf, uint *frlen, u8 val)
+{
+	u8 cont_len = 7;
+
+	*pbuf++ = WLAN_EID_VENDOR_SPECIFIC;
+	*pbuf++ = cont_len;
+	_rtw_memcpy(pbuf, MULTI_AP_OUI, 4);
+	pbuf += 4;
+	*pbuf++ = MULTI_AP_SUB_ELEM_TYPE;
+	*pbuf++ = 1; /* len */
+	*pbuf++ = val;
+
+	if (frlen)
+		*frlen = *frlen + (cont_len + 2);
+
+	return pbuf;
+}
+#endif /* CONFIG_RTW_MULTI_AP */
+
+/* Baron adds to avoid FreeBSD warning */
+int ieee80211_is_empty_essid(const char *essid, int essid_len)
+{
+	/* Single white space is for Linksys APs */
+	if (essid_len == 1 && essid[0] == ' ')
+		return 1;
+
+	/* Otherwise, if the entire essid is 0, we assume it is hidden */
+	while (essid_len) {
+		essid_len--;
+		if (essid[essid_len] != '\0')
+			return 0;
+	}
+
+	return 1;
+}
+
+int ieee80211_get_hdrlen(u16 fc)
+{
+	int hdrlen = 24;
+
+	switch (WLAN_FC_GET_TYPE(fc)) {
+	case RTW_IEEE80211_FTYPE_DATA:
+		if (fc & RTW_IEEE80211_STYPE_QOS_DATA)
+			hdrlen += 2;
+		if ((fc & RTW_IEEE80211_FCTL_FROMDS) && (fc & RTW_IEEE80211_FCTL_TODS))
+			hdrlen += 6; /* Addr4 */
+		break;
+	case RTW_IEEE80211_FTYPE_CTL:
+		switch (WLAN_FC_GET_STYPE(fc)) {
+		case RTW_IEEE80211_STYPE_CTS:
+		case RTW_IEEE80211_STYPE_ACK:
+			hdrlen = 10;
+			break;
+		default:
+			hdrlen = 16;
+			break;
+		}
+		break;
+	}
+
+	return hdrlen;
+}
+
+u8	rtw_ht_mcsset_to_nss(u8 *supp_mcs_set)
+{
+	u8 nss = 1;
+
+	if (supp_mcs_set[3])
+		nss = 4;
+	else if (supp_mcs_set[2])
+		nss = 3;
+	else if (supp_mcs_set[1])
+		nss = 2;
+	else if (supp_mcs_set[0])
+		nss = 1;
+	else
+		RTW_INFO("%s,%d, warning! supp_mcs_set is zero\n", __func__, __LINE__);
+	/* RTW_INFO("%s HT: %dSS\n", __FUNCTION__, nss); */
+	return nss;
+}
+
+u32	rtw_ht_mcs_set_to_bitmap(u8 *mcs_set, u8 nss)
+{
+	u8 i;
+	u32 bitmap = 0;
+
+	for (i = 0; i < nss; i++)
+		bitmap |= mcs_set[i] << (i * 8);
+
+	RTW_INFO("ht_mcs_set=%02x %02x %02x %02x, nss=%u, bitmap=%08x\n"
+		, mcs_set[0], mcs_set[1], mcs_set[2], mcs_set[3], nss, bitmap);
+
+	return bitmap;
+}
+
+/* show MCS rate, unit: 100Kbps */
+u16 rtw_ht_mcs_rate(u8 bw_40MHz, u8 short_GI, unsigned char *MCS_rate)
+{
+	u16 max_rate = 0;
+
+	if (MCS_rate[3]) {
+		if (MCS_rate[3] & BIT(7))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 6000 : 5400) : ((short_GI) ? 2889 : 2600);
+		else if (MCS_rate[3] & BIT(6))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 5400 : 4860) : ((short_GI) ? 2600 : 2340);
+		else if (MCS_rate[3] & BIT(5))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 4800 : 4320) : ((short_GI) ? 2311 : 2080);
+		else if (MCS_rate[3] & BIT(4))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 3600 : 3240) : ((short_GI) ? 1733 : 1560);
+		else if (MCS_rate[3] & BIT(3))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 2400 : 2160) : ((short_GI) ? 1156 : 1040);
+		else if (MCS_rate[3] & BIT(2))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1800 : 1620) : ((short_GI) ? 867 : 780);
+		else if (MCS_rate[3] & BIT(1))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1200 : 1080) : ((short_GI) ? 578 : 520);
+		else if (MCS_rate[3] & BIT(0))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 600 : 540) : ((short_GI) ? 289 : 260);
+	} else if (MCS_rate[2]) {
+		if (MCS_rate[2] & BIT(7))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 4500 : 4050) : ((short_GI) ? 2167 : 1950);
+		else if (MCS_rate[2] & BIT(6))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 4050 : 3645) : ((short_GI) ? 1950 : 1750);
+		else if (MCS_rate[2] & BIT(5))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 3600 : 3240) : ((short_GI) ? 1733 : 1560);
+		else if (MCS_rate[2] & BIT(4))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 2700 : 2430) : ((short_GI) ? 1300 : 1170);
+		else if (MCS_rate[2] & BIT(3))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1800 : 1620) : ((short_GI) ? 867 : 780);
+		else if (MCS_rate[2] & BIT(2))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1350 : 1215) : ((short_GI) ? 650 : 585);
+		else if (MCS_rate[2] & BIT(1))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 900 : 810) : ((short_GI) ? 433 : 390);
+		else if (MCS_rate[2] & BIT(0))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 450 : 405) : ((short_GI) ? 217 : 195);
+	} else if (MCS_rate[1]) {
+		if (MCS_rate[1] & BIT(7))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 3000 : 2700) : ((short_GI) ? 1444 : 1300);
+		else if (MCS_rate[1] & BIT(6))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 2700 : 2430) : ((short_GI) ? 1300 : 1170);
+		else if (MCS_rate[1] & BIT(5))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 2400 : 2160) : ((short_GI) ? 1156 : 1040);
+		else if (MCS_rate[1] & BIT(4))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1800 : 1620) : ((short_GI) ? 867 : 780);
+		else if (MCS_rate[1] & BIT(3))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1200 : 1080) : ((short_GI) ? 578 : 520);
+		else if (MCS_rate[1] & BIT(2))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 900 : 810) : ((short_GI) ? 433 : 390);
+		else if (MCS_rate[1] & BIT(1))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 600 : 540) : ((short_GI) ? 289 : 260);
+		else if (MCS_rate[1] & BIT(0))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 300 : 270) : ((short_GI) ? 144 : 130);
+	} else {
+		if (MCS_rate[0] & BIT(7))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1500 : 1350) : ((short_GI) ? 722 : 650);
+		else if (MCS_rate[0] & BIT(6))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1350 : 1215) : ((short_GI) ? 650 : 585);
+		else if (MCS_rate[0] & BIT(5))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 1200 : 1080) : ((short_GI) ? 578 : 520);
+		else if (MCS_rate[0] & BIT(4))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 900 : 810) : ((short_GI) ? 433 : 390);
+		else if (MCS_rate[0] & BIT(3))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 600 : 540) : ((short_GI) ? 289 : 260);
+		else if (MCS_rate[0] & BIT(2))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 450 : 405) : ((short_GI) ? 217 : 195);
+		else if (MCS_rate[0] & BIT(1))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 300 : 270) : ((short_GI) ? 144 : 130);
+		else if (MCS_rate[0] & BIT(0))
+			max_rate = (bw_40MHz) ? ((short_GI) ? 150 : 135) : ((short_GI) ? 72 : 65);
+	}
+
+	return max_rate;
+}
+
+int rtw_action_frame_parse(const u8 *frame, u32 frame_len, u8 *category, u8 *action)
+{
+	const u8 *frame_body = frame + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u16 fc;
+	u8 c;
+	u8 a = ACT_PUBLIC_MAX;
+
+	fc = le16_to_cpu(((struct rtw_ieee80211_hdr_3addr *)frame)->frame_ctl);
+
+	if ((fc & (RTW_IEEE80211_FCTL_FTYPE | RTW_IEEE80211_FCTL_STYPE))
+	    != (RTW_IEEE80211_FTYPE_MGMT | RTW_IEEE80211_STYPE_ACTION)
+	   )
+		return _FALSE;
+
+	c = frame_body[0];
+
+	switch (c) {
+	case RTW_WLAN_CATEGORY_P2P: /* vendor-specific */
+		break;
+	default:
+		a = frame_body[1];
+	}
+
+	if (category)
+		*category = c;
+	if (action)
+		*action = a;
+
+	return _TRUE;
+}
+
+static const char *_action_public_str[] = {
+	"ACT_PUB_BSSCOEXIST",
+	"ACT_PUB_DSE_ENABLE",
+	"ACT_PUB_DSE_DEENABLE",
+	"ACT_PUB_DSE_REG_LOCATION",
+	"ACT_PUB_EXT_CHL_SWITCH",
+	"ACT_PUB_DSE_MSR_REQ",
+	"ACT_PUB_DSE_MSR_RPRT",
+	"ACT_PUB_MP",
+	"ACT_PUB_DSE_PWR_CONSTRAINT",
+	"ACT_PUB_VENDOR",
+	"ACT_PUB_GAS_INITIAL_REQ",
+	"ACT_PUB_GAS_INITIAL_RSP",
+	"ACT_PUB_GAS_COMEBACK_REQ",
+	"ACT_PUB_GAS_COMEBACK_RSP",
+	"ACT_PUB_TDLS_DISCOVERY_RSP",
+	"ACT_PUB_LOCATION_TRACK",
+	"ACT_PUB_RSVD",
+};
+
+const char *action_public_str(u8 action)
+{
+	action = (action >= ACT_PUBLIC_MAX) ? ACT_PUBLIC_MAX : action;
+	return _action_public_str[action];
+}
+
+#if 0
+/*tmp for sta mode, root cause have to wait supplicant's update.*/
+void rtw_set_spp_amsdu_mode(u8 mode, u8 *rsn_ie, int rsn_ie_len)
+{
+	struct rsne_info info;
+	int i, ret = _SUCCESS;
+	u8 spp_req_cap = 0;
+
+	ret = rtw_rsne_info_parse(rsn_ie, rsn_ie_len, &info);
+	if (ret != _SUCCESS)
+		return;
+
+	if (mode == RTW_AMSDU_MODE_NON_SPP ) {
+		spp_req_cap = 0;
+	} else if (mode == RTW_AMSDU_MODE_SPP) {
+		spp_req_cap = SPP_CAP|SPP_REQ;
+	} else if (mode == RTW_AMSDU_MODE_ALL_DROP) {
+		spp_req_cap = SPP_REQ;
+	} else {
+		RTW_INFO("%s unexpected mode = %d, please check the config\n", __func__, mode);
+		return;
+	}
+
+	SET_RSN_CAP_SPP(info.cap, spp_req_cap);
+	RTW_INFO("%s set spp opt = %d\n", __func__, GET_RSN_CAP_SPP_OPT(info.cap));
+}
+#endif
+
+/*	Returns:
+	_TRUE	-- 	Disable AMSDU
+	_FALSE	--	Enable AMSDU
+*/
+u8 rtw_check_amsdu_disable(u8 mode, u8 spp_opt)
+{
+	u8 ret = _FALSE;
+
+	/* pp amsdu: peer's required has to be 0, or disable */
+	if ((mode == RTW_AMSDU_MODE_NON_SPP) && (spp_opt & SPP_REQ))
+		ret = _TRUE;
+	/* spp amsdu: peer's cap has to be 1, or disable */
+	else if ((mode == RTW_AMSDU_MODE_SPP) && (!(spp_opt & SPP_CAP)))
+		ret = _TRUE;
+	/* mode = all drop */
+	else if (mode == RTW_AMSDU_MODE_ALL_DROP)
+		ret = _TRUE;
+	else
+		ret = _FALSE;
+	return ret;
+}
+
+char *get_macaddr_str(char *str, void *sel, const u8 *addr)
+{
+	if (sel == RTW_DBGDUMP)
+		snprintf(str, MAC_FMT_LEN, MAC_FMT, MAC_ARG(addr));
+	else
+		snprintf(str, MAC_FMT_LEN, MAC_FMT_SEL, MAC_ARG_SEL(addr));
+	return str;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_io.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_io.c
new file mode 100644
index 000000000000..ce0c703bdbba
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_io.c
@@ -0,0 +1,75 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+/*
+
+The purpose of rtw_io.c
+
+a. provides the API
+
+b. provides the protocol engine
+
+c. provides the software interface between caller and the hardware interface
+
+
+Compiler Flag Option:
+
+1. CONFIG_SDIO_HCI:
+    a. USE_SYNC_IRP:  Only sync operations are provided.
+    b. USE_ASYNC_IRP:Both sync/async operations are provided.
+
+2. CONFIG_USB_HCI:
+   a. USE_ASYNC_IRP: Both sync/async operations are provided.
+
+3. CONFIG_CFIO_HCI:
+   b. USE_SYNC_IRP: Only sync operations are provided.
+
+
+Only sync read/rtw_write_mem operations are provided.
+
+jackson@realtek.com.tw
+
+*/
+
+#define _RTW_IO_C_
+
+#include <drv_types.h>
+
+/*
+* Increase and check if the continual_io_error of this @param dvobjprive is larger than MAX_CONTINUAL_IO_ERR
+* @return _TRUE:
+* @return _FALSE:
+*/
+int rtw_inc_and_chk_continual_io_error(struct dvobj_priv *dvobj)
+{
+	int ret = _FALSE;
+	int value;
+
+	value = ATOMIC_INC_RETURN(&dvobj->continual_io_error);
+	if (value > MAX_CONTINUAL_IO_ERR) {
+		RTW_INFO("[dvobj:%p][ERROR] continual_io_error:%d > %d\n", dvobj, value, MAX_CONTINUAL_IO_ERR);
+		ret = _TRUE;
+	} else {
+		/* RTW_INFO("[dvobj:%p] continual_io_error:%d\n", dvobj, value); */
+	}
+	return ret;
+}
+
+/*
+* Set the continual_io_error of this @param dvobjprive to 0
+*/
+void rtw_reset_continual_io_error(struct dvobj_priv *dvobj)
+{
+	ATOMIC_SET(&dvobj->continual_io_error, 0);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_io_records.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_io_records.c
new file mode 100644
index 000000000000..4e01292deb5b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_io_records.c
@@ -0,0 +1,85 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2025 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_IO_RECORDS_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_IO_RECORDS
+#include <rtw_io_records.h>
+
+struct rtw_io_records dbg_rtw_io_records = {
+#if CONFIG_RTW_IO_RECORDS_STATIC
+	.record_num = CONFIG_RTW_IO_RECORDS_NUM,
+	.enable = CONFIG_RTW_IO_RECORDS_ENABLE,
+	.loop = CONFIG_RTW_IO_RECORDS_LOOP,
+#endif
+};
+
+bool rtw_io_records_enabled(void)
+{
+	return dbg_rtw_io_records.enable;
+}
+
+void rtw_io_records_clear(void)
+{
+	int i;
+
+	for (i = 0; i < dbg_rtw_io_records.record_num; i++)
+		dbg_rtw_io_records.record[i].valid = false;
+	dbg_rtw_io_records.pos = 0;
+}
+
+bool rtw_io_record_valid(size_t seq)
+{
+	if (seq < dbg_rtw_io_records.record_num) {
+		size_t oldest_pos = dbg_rtw_io_records.record[dbg_rtw_io_records.pos].valid ? dbg_rtw_io_records.pos : 0;
+		struct rtw_io_record *record = &dbg_rtw_io_records.record[(oldest_pos + seq) % dbg_rtw_io_records.record_num];
+
+		return record->valid;
+	}
+	return false;
+}
+
+int rtw_io_records_init(void)
+{
+#if !CONFIG_RTW_IO_RECORDS_STATIC
+extern uint rtw_io_records_num;
+extern uint rtw_io_records_enable;
+extern uint rtw_io_records_loop;
+
+	size_t record_num = rtw_io_records_num;
+	bool enable = !!rtw_io_records_enable;
+	bool loop = !!rtw_io_records_loop;
+
+	dbg_rtw_io_records.record = rtw_zvmalloc(sizeof(struct rtw_io_record) * record_num);
+	if (!dbg_rtw_io_records.record)
+		return _FAIL;
+
+	dbg_rtw_io_records.record_num = record_num;
+	dbg_rtw_io_records.loop = loop;
+	dbg_rtw_io_records.pos = 0;
+	dbg_rtw_io_records.enable = enable;
+#endif
+	return _SUCCESS;
+}
+
+void rtw_io_records_deinit(void)
+{
+#if !CONFIG_RTW_IO_RECORDS_STATIC
+	if (dbg_rtw_io_records.record)
+		rtw_vmfree(dbg_rtw_io_records.record, sizeof(struct rtw_io_record) * dbg_rtw_io_records.record_num);
+#endif
+}
+#endif /* CONFIG_RTW_IO_RECORDS */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ioctl_query.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ioctl_query.c
new file mode 100644
index 000000000000..7647e807d791
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ioctl_query.c
@@ -0,0 +1,19 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_IOCTL_QUERY_C_
+
+#include <drv_types.h>
+
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ioctl_set.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ioctl_set.c
new file mode 100644
index 000000000000..3e094df16334
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_ioctl_set.c
@@ -0,0 +1,858 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_IOCTL_SET_C_
+
+#include <drv_types.h>
+
+
+extern void indicate_wx_scan_complete_event(_adapter *padapter);
+
+#define IS_MAC_ADDRESS_BROADCAST(addr) \
+	(\
+	 ((addr[0] == 0xff) && (addr[1] == 0xff) && \
+	  (addr[2] == 0xff) && (addr[3] == 0xff) && \
+	  (addr[4] == 0xff) && (addr[5] == 0xff)) ? _TRUE : _FALSE \
+	)
+
+u8 rtw_validate_bssid(const u8 *bssid)
+{
+	u8 ret = _TRUE;
+
+	if (is_zero_mac_addr(bssid)
+	    || is_broadcast_mac_addr(bssid)
+	    || is_multicast_mac_addr(bssid)
+	   )
+		ret = _FALSE;
+
+	return ret;
+}
+
+u8 rtw_validate_ssid(NDIS_802_11_SSID *ssid)
+{
+#ifdef CONFIG_VALIDATE_SSID
+	u8	 i;
+#endif
+	u8	ret = _TRUE;
+
+
+	if (ssid->SsidLength > 32) {
+		ret = _FALSE;
+		goto exit;
+	}
+
+#ifdef CONFIG_VALIDATE_SSID
+	for (i = 0; i < ssid->SsidLength; i++) {
+		/* wifi, printable ascii code must be supported */
+		if (!((ssid->Ssid[i] >= 0x20) && (ssid->Ssid[i] <= 0x7e))) {
+			ret = _FALSE;
+			break;
+		}
+	}
+#endif /* CONFIG_VALIDATE_SSID */
+
+exit:
+
+
+	return ret;
+}
+
+u8 rtw_do_join(_adapter *padapter)
+{
+	_list	*plist, *phead;
+	u8 *pibss = NULL;
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct sitesurvey_parm *parm = NULL;
+	_queue	*queue	= &(pmlmepriv->scanned_queue);
+	u8 ret = _SUCCESS;
+
+	parm = rtw_malloc(sizeof(*parm));
+	if (parm == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+
+	pmlmepriv->dev_cur_network.join_res = -2;
+
+	set_fwstate(pmlmepriv, WIFI_UNDER_LINKING);
+
+	pmlmepriv->pscanned = plist;
+
+	pmlmepriv->to_join = _TRUE;
+
+	rtw_init_sitesurvey_parm(padapter, parm);
+	_rtw_memcpy(&parm->ssid[0], &pmlmepriv->assoc_ssid, sizeof(NDIS_802_11_SSID));
+	parm->ssid_num = 1;
+
+	if (pmlmepriv->assoc_ch) {
+		parm->ch_num = 1;
+		parm->ch[0].hw_value = pmlmepriv->assoc_ch;
+		parm->ch[0].band = pmlmepriv->assoc_band;
+		parm->ch[0].flags = 0;
+	}
+
+	if (_rtw_queue_empty(queue) == _TRUE) {
+		_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+		_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+
+		/* when set_ssid/set_bssid for rtw_do_join(), but scanning queue is empty */
+		/* we try to issue sitesurvey firstly	 */
+
+		if (pmlmepriv->LinkDetectInfo.bBusyTraffic == _FALSE
+		    || rtw_to_roam(padapter) > 0
+		   ) {
+			u8 ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+
+			if ((ssc_chk == SS_ALLOW) || (ssc_chk == SS_DENY_BUSY_TRAFFIC) ){
+				/* submit site_survey_cmd */
+				ret = rtw_sitesurvey_cmd(padapter, parm);
+				if (_SUCCESS != ret)
+					rtw_clear_to_join_status(padapter);
+			} else {
+				/*if (ssc_chk == SS_DENY_BUDDY_UNDER_SURVEY)*/
+				rtw_clear_to_join_status(padapter);
+				ret = _FAIL;
+			}
+		} else {
+			rtw_clear_to_join_status(padapter);
+			ret = _FAIL;
+		}
+
+		goto exit;
+	} else {
+		int select_ret;
+		_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+		select_ret = rtw_select_and_join_from_scanned_queue(pmlmepriv);
+		if (select_ret == _SUCCESS) {
+			pmlmepriv->to_join = _FALSE;
+			set_assoc_timer(pmlmepriv, MAX_JOIN_TIMEOUT); /*_set_timer(&pmlmepriv->assoc_timer, MAX_JOIN_TIMEOUT);*/
+		} else {
+			if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) {
+				/* submit createbss_cmd to change to a ADHOC_MASTER */
+
+				/* pmlmepriv->lock has been acquired by caller... */
+				WLAN_BSSID_EX    *pdev_network = &(padapter->registrypriv.dev_network);
+
+				/*pmlmepriv->fw_state = WIFI_ADHOC_MASTER_STATE;*/
+				init_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+
+				pibss = padapter->registrypriv.dev_network.MacAddress;
+
+				_rtw_memset(&pdev_network->Ssid, 0, sizeof(NDIS_802_11_SSID));
+				_rtw_memcpy(&pdev_network->Ssid, &pmlmepriv->assoc_ssid, sizeof(NDIS_802_11_SSID));
+
+				rtw_update_registrypriv_dev_network(padapter);
+
+				rtw_generate_random_ibss(pibss);
+
+				if (rtw_create_ibss_cmd(padapter, 0) != _SUCCESS) {
+					ret =  _FALSE;
+					goto exit;
+				}
+
+				/* ToDo CONFIG_RTW_MLD: ad-hoc */
+				pmlmepriv->to_join = _FALSE;
+
+
+			} else {
+				/* can't associate ; reset under-linking			 */
+				_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+
+				/* when set_ssid/set_bssid for rtw_do_join(), but there are no desired bss in scanning queue */
+				/* we try to issue sitesurvey firstly			 */
+				if (pmlmepriv->LinkDetectInfo.bBusyTraffic == _FALSE
+				    || rtw_to_roam(padapter) > 0
+				   ) {
+					u8 ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+
+					if ((ssc_chk == SS_ALLOW) || (ssc_chk == SS_DENY_BUSY_TRAFFIC)){
+						/* RTW_INFO(("rtw_do_join() when   no desired bss in scanning queue\n"); */
+						ret = rtw_sitesurvey_cmd(padapter, parm);
+						if (_SUCCESS != ret)
+							rtw_clear_to_join_status(padapter);
+					} else {
+						/*if (ssc_chk == SS_DENY_BUDDY_UNDER_SURVEY) {
+						} else {*/
+						ret = _FAIL;
+						rtw_clear_to_join_status(padapter);
+					}
+				} else {
+					ret = _FAIL;
+					rtw_clear_to_join_status(padapter);
+				}
+			}
+
+		}
+
+	}
+
+exit:
+	if (parm)
+		rtw_mfree(parm, sizeof(*parm));
+
+	return ret;
+}
+
+u8 rtw_set_802_11_bssid(_adapter *padapter, u8 *bssid)
+{
+	u8 status = _SUCCESS;
+
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+
+	RTW_PRINT("set bssid:%pM\n", bssid);
+
+	if ((bssid[0] == 0x00 && bssid[1] == 0x00 && bssid[2] == 0x00 && bssid[3] == 0x00 && bssid[4] == 0x00 && bssid[5] == 0x00) ||
+	    (bssid[0] == 0xFF && bssid[1] == 0xFF && bssid[2] == 0xFF && bssid[3] == 0xFF && bssid[4] == 0xFF && bssid[5] == 0xFF)) {
+		status = _FAIL;
+		goto exit;
+	}
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+
+	RTW_INFO("Set BSSID under fw_state=0x%08x\n", get_fwstate(pmlmepriv));
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE)
+		goto handle_tkip_countermeasure;
+	else if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE)
+		goto release_mlme_lock;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE | WIFI_ADHOC_MASTER_STATE) == _TRUE) {
+
+		if (_rtw_memcmp(&pmlmepriv->dev_cur_network.network.MacAddress, bssid, ETH_ALEN) == _TRUE) {
+			if (!MLME_IS_STA(padapter))
+				goto release_mlme_lock;/* it means driver is in WIFI_ADHOC_MASTER_STATE, we needn't create bss again. */
+		} else {
+
+			rtw_disassoc_cmd(padapter, 0, 0);
+
+			if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+				if (MLME_IS_STA(padapter) == _FALSE)
+					rtw_free_assoc_resources_cmd(padapter, _TRUE, 0);
+				rtw_indicate_disconnect(padapter, 0, _FALSE);
+				pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+				pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND;
+				pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+			}
+
+			if ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+				_clr_fwstate_(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+				set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
+			}
+		}
+	}
+
+handle_tkip_countermeasure:
+	if (rtw_handle_tkip_countermeasure(padapter, __func__) == _FAIL) {
+		status = _FAIL;
+		goto release_mlme_lock;
+	}
+
+	_rtw_memset(&pmlmepriv->assoc_ssid, 0, sizeof(NDIS_802_11_SSID));
+	_rtw_memcpy(&pmlmepriv->assoc_bssid, bssid, ETH_ALEN);
+	pmlmepriv->assoc_ch = 0;
+	pmlmepriv->assoc_band = BAND_MAX;
+	pmlmepriv->assoc_by_bssid = _TRUE;
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE)
+		pmlmepriv->to_join = _TRUE;
+	else
+		status = rtw_do_join(padapter);
+
+release_mlme_lock:
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+exit:
+
+
+	return status;
+}
+
+u8 rtw_set_802_11_ssid(_adapter *padapter, NDIS_802_11_SSID *ssid)
+{
+	u8 status = _SUCCESS;
+
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network *pnetwork = &pmlmepriv->dev_cur_network;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+
+	RTW_PRINT("set ssid [%s] fw_state=0x%08x\n",
+		  ssid->Ssid, get_fwstate(pmlmepriv));
+
+	if (!rtw_hw_is_init_completed(adapter_to_dvobj(padapter))) {
+		status = _FAIL;
+		goto exit;
+	}
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	RTW_INFO("Set SSID under fw_state=0x%08x\n", get_fwstate(pmlmepriv));
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE)
+		goto handle_tkip_countermeasure;
+	else if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE)
+		goto release_mlme_lock;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE | WIFI_ADHOC_MASTER_STATE) == _TRUE) {
+
+		if ((pmlmepriv->assoc_ssid.SsidLength == ssid->SsidLength) &&
+		    (_rtw_memcmp(&pmlmepriv->assoc_ssid.Ssid, ssid->Ssid, ssid->SsidLength) == _TRUE)) {
+			if (!MLME_IS_STA(padapter)) {
+
+				if (rtw_is_same_ibss(padapter, pnetwork) == _FALSE) {
+					/* if in WIFI_ADHOC_MASTER_STATE | WIFI_ADHOC_STATE, create bss or rejoin again */
+					rtw_disassoc_cmd(padapter, 0, 0);
+
+					if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+						rtw_free_assoc_resources_cmd(padapter, _TRUE, 0);
+						rtw_indicate_disconnect(padapter, 0, _FALSE);
+						pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+						pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND;
+						pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+					}
+					if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE) {
+						_clr_fwstate_(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+						set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
+					}
+				} else {
+					goto release_mlme_lock;/* it means driver is in WIFI_ADHOC_MASTER_STATE, we needn't create bss again. */
+				}
+			}
+		} else {
+
+			rtw_disassoc_cmd(padapter, 0, 0);
+
+			if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE){
+				if (MLME_IS_STA(padapter) == _FALSE)
+					rtw_free_assoc_resources_cmd(padapter, _TRUE, 0);
+				rtw_indicate_disconnect(padapter, 0, _FALSE);
+				pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+				pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND;
+				pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+			}
+			if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE) {
+				_clr_fwstate_(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+				set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
+			}
+		}
+	}
+
+handle_tkip_countermeasure:
+	if (rtw_handle_tkip_countermeasure(padapter, __func__) == _FAIL) {
+		status = _FAIL;
+		goto release_mlme_lock;
+	}
+
+	if (rtw_validate_ssid(ssid) == _FALSE) {
+		status = _FAIL;
+		goto release_mlme_lock;
+	}
+
+	_rtw_memcpy(&pmlmepriv->assoc_ssid, ssid, sizeof(NDIS_802_11_SSID));
+	pmlmepriv->assoc_ch = 0;
+	pmlmepriv->assoc_band = BAND_MAX;
+	pmlmepriv->assoc_by_bssid = _FALSE;
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE)
+		pmlmepriv->to_join = _TRUE;
+	else
+		status = rtw_do_join(padapter);
+
+release_mlme_lock:
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+exit:
+
+
+	return status;
+
+}
+
+u8 rtw_set_802_11_connect(_adapter *padapter, const u8 *bssid, NDIS_802_11_SSID *ssid,
+			  u16 ch, enum band_type band)
+{
+	u8 status = _SUCCESS;
+	bool bssid_valid = _TRUE;
+	bool ssid_valid = _TRUE;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+
+	if (!ssid || rtw_validate_ssid(ssid) == _FALSE)
+		ssid_valid = _FALSE;
+
+	if (!bssid || rtw_validate_bssid(bssid) == _FALSE)
+		bssid_valid = _FALSE;
+
+	if (ssid_valid == _FALSE && bssid_valid == _FALSE) {
+		RTW_INFO(FUNC_ADPT_FMT" ssid:%p, ssid_valid:%d, bssid:%p, bssid_valid:%d\n",
+			FUNC_ADPT_ARG(padapter), ssid, ssid_valid, bssid, bssid_valid);
+		status = _FAIL;
+		goto exit;
+	}
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	RTW_PRINT(FUNC_ADPT_FMT"  fw_state=0x%08x\n",
+		  FUNC_ADPT_ARG(padapter), get_fwstate(pmlmepriv));
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE)
+		goto handle_tkip_countermeasure;
+	else if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE)
+		goto release_mlme_lock;
+
+handle_tkip_countermeasure:
+	if (rtw_handle_tkip_countermeasure(padapter, __func__) == _FAIL) {
+		status = _FAIL;
+		goto release_mlme_lock;
+	}
+
+	if (ssid && ssid_valid)
+		_rtw_memcpy(&pmlmepriv->assoc_ssid, ssid, sizeof(NDIS_802_11_SSID));
+	else
+		_rtw_memset(&pmlmepriv->assoc_ssid, 0, sizeof(NDIS_802_11_SSID));
+
+	if (bssid && bssid_valid) {
+		_rtw_memcpy(&pmlmepriv->assoc_bssid, bssid, ETH_ALEN);
+		pmlmepriv->assoc_by_bssid = _TRUE;
+	} else
+		pmlmepriv->assoc_by_bssid = _FALSE;
+
+	pmlmepriv->assoc_ch = ch;
+	pmlmepriv->assoc_band = band;
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE)
+		pmlmepriv->to_join = _TRUE;
+	else
+		status = rtw_do_join(padapter);
+
+release_mlme_lock:
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+exit:
+	return status;
+}
+
+u8 rtw_set_802_11_infrastructure_mode(_adapter *padapter,
+				NDIS_802_11_NETWORK_INFRASTRUCTURE networktype, u8 flags)
+{
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct	wlan_network	*cur_network = &pmlmepriv->dev_cur_network;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	NDIS_802_11_NETWORK_INFRASTRUCTURE *pold_state = &(cur_network->network.InfrastructureMode);
+	u8 ap2sta_mode = _FALSE;
+	u8 ret = _TRUE;
+	u8 is_linked = _FALSE, is_adhoc_master = _FALSE;
+
+	if (*pold_state != networktype) {
+		/* RTW_INFO("change mode, old_mode=%d, new_mode=%d, fw_state=0x%x\n", *pold_state, networktype, get_fwstate(pmlmepriv)); */
+
+		if (*pold_state == Ndis802_11APMode
+			|| *pold_state == Ndis802_11_mesh
+		) {
+			/* change to other mode from Ndis802_11APMode/Ndis802_11_mesh */
+			cur_network->join_res = -1;
+			ap2sta_mode = _TRUE;
+#ifdef CONFIG_NATIVEAP_MLME
+			stop_ap_mode(padapter);
+#endif
+		}
+
+		_rtw_spinlock_bh(&pmlmepriv->lock);
+		is_linked = check_fwstate(pmlmepriv, WIFI_ASOC_STATE);
+		is_adhoc_master = check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+
+		/* flags = 0, means enqueue cmd and no wait */
+		if (flags != 0)
+			_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+		if ((is_linked == _TRUE) || (*pold_state == Ndis802_11IBSS))
+			rtw_disassoc_cmd(padapter, 0, flags);
+
+		if ((is_adhoc_master == _TRUE)
+		    || ((is_linked == _TRUE) && (MLME_IS_STA(padapter) == _FALSE))
+		   )
+			rtw_free_assoc_resources_cmd(padapter, _TRUE, flags);
+
+		if ((*pold_state == Ndis802_11Infrastructure) || (*pold_state == Ndis802_11IBSS)) {
+			if (is_linked == _TRUE) {
+				rtw_indicate_disconnect(padapter, 0, _FALSE); /*will clr Linked_state; before this function, we must have checked whether issue dis-assoc_cmd or not*/
+				pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+				pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND;
+				pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+			}
+		}
+
+		/* flags = 0, means enqueue cmd and no wait */
+		if (flags != 0)
+			_rtw_spinlock_bh(&pmlmepriv->lock);
+
+		*pold_state = networktype;
+
+		_clr_fwstate_(pmlmepriv, ~WIFI_NULL_STATE);
+
+		switch (networktype) {
+		case Ndis802_11IBSS:
+			set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
+			break;
+
+		case Ndis802_11Infrastructure:
+			set_fwstate(pmlmepriv, WIFI_STATION_STATE);
+			break;
+
+		case Ndis802_11APMode:
+			set_fwstate(pmlmepriv, WIFI_AP_STATE);
+#ifdef CONFIG_NATIVEAP_MLME
+			start_ap_mode(padapter);
+			/* rtw_indicate_connect(padapter); */
+#endif
+
+			break;
+
+#ifdef CONFIG_RTW_MESH
+		case Ndis802_11_mesh:
+			set_fwstate(pmlmepriv, WIFI_MESH_STATE);
+			start_ap_mode(padapter);
+			break;
+#endif
+
+		case Ndis802_11AutoUnknown:
+		case Ndis802_11InfrastructureMax:
+			break;
+#ifdef CONFIG_WIFI_MONITOR
+		case Ndis802_11Monitor:
+			set_fwstate(pmlmepriv, WIFI_MONITOR_STATE);
+			break;
+#endif /* CONFIG_WIFI_MONITOR */
+		default:
+			ret = _FALSE;
+			rtw_warn_on(1);
+		}
+
+		/* SecClearAllKeys(adapter); */
+
+
+		_rtw_spinunlock_bh(&pmlmepriv->lock);
+	}
+	return ret;
+}
+
+
+u8 rtw_set_802_11_disassociate(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+		rtw_disassoc_cmd(padapter, 0, 0);
+		/* modify for CONFIG_IEEE80211W, none 11w can use it */
+		if (MLME_IS_STA(padapter) == _FALSE)
+			rtw_free_assoc_resources_cmd(padapter, _TRUE, 0);
+		rtw_indicate_disconnect(padapter, 0, _FALSE);
+		pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+		pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND;
+		pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+	}
+
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+
+	return _TRUE;
+}
+
+u8 rtw_set_802_11_authentication_mode(_adapter *padapter, NDIS_802_11_AUTHENTICATION_MODE authmode)
+{
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	int res;
+	u8 ret;
+
+
+
+	psecuritypriv->ndisauthtype = authmode;
+
+
+	if (psecuritypriv->ndisauthtype > 3)
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (psecuritypriv->ndisauthtype == 6)
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_WAPI;
+#endif
+
+	res = rtw_set_auth(padapter, psecuritypriv);
+
+	if (res == _SUCCESS)
+		ret = _TRUE;
+	else
+		ret = _FALSE;
+
+
+	return ret;
+}
+
+u8 rtw_set_802_11_add_wep(_adapter *padapter, NDIS_802_11_WEP *wep)
+{
+
+	u8		bdefaultkey;
+	u8		btransmitkey;
+	sint		keyid, res;
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	u8		ret = _SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+
+	bdefaultkey = (wep->KeyIndex & 0x40000000) > 0 ? _FALSE : _TRUE; /* for ??? */
+	btransmitkey = (wep->KeyIndex & 0x80000000) > 0 ? _TRUE  : _FALSE;	/* for ??? */
+	keyid = wep->KeyIndex & 0x3fffffff;
+
+	if (keyid >= 4) {
+		ret = _FALSE;
+		goto exit;
+	}
+
+	switch (wep->KeyLength) {
+	case 5:
+		psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+		break;
+	case 13:
+		psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+		break;
+	default:
+		psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		break;
+	}
+
+
+	_rtw_memcpy(&(psecuritypriv->dot11DefKey[keyid].skey[0]), &(wep->KeyMaterial), wep->KeyLength);
+
+	psecuritypriv->dot11DefKeylen[keyid] = wep->KeyLength;
+
+	psecuritypriv->dot11PrivacyKeyIndex = keyid;
+
+
+	res = rtw_set_key(padapter, padapter_link, keyid, 1, _TRUE);
+
+	if (res == _FAIL)
+		ret = _FALSE;
+exit:
+
+
+	return ret;
+
+}
+
+/*
+* rtw_get_cur_max_rate -
+* @adapter: pointer to _adapter structure
+*
+* Return 0 or 100Kbps
+*/
+u16 rtw_get_cur_max_rate(_adapter *adapter)
+{
+	int j;
+	int	i = 0;
+	u16	rate = 0, max_rate = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_priv	*pmlmepriv = &adapter_link->mlmepriv;
+	WLAN_BSSID_EX	*pcur_bss = &pmlmepriv->cur_network.network;
+	int	sta_bssrate_len = 0;
+	unsigned char	sta_bssrate[NumRates];
+	struct sta_info *psta = NULL;
+	u8	short_GI = 0;
+
+#ifdef CONFIG_MP_INCLUDED
+	if (adapter->registrypriv.mp_mode == 1) {
+		if (check_fwstate(&adapter->mlmepriv, WIFI_MP_STATE) == _TRUE)
+			return 0;
+	}
+#endif
+
+	if ((check_fwstate(&adapter->mlmepriv, WIFI_ASOC_STATE) != _TRUE)
+	    && (check_fwstate(&adapter->mlmepriv, WIFI_ADHOC_MASTER_STATE) != _TRUE))
+		return 0;
+
+	psta = rtw_get_stainfo(&adapter->stapriv, get_link_bssid(pmlmepriv));
+	if (psta == NULL)
+		return 0;
+
+	short_GI = query_ra_short_GI(psta, rtw_get_tx_bw_mode(adapter, psta));
+
+#ifdef CONFIG_80211N_HT
+	if (is_supported_ht(psta->phl_sta->wmode)) {
+		max_rate = rtw_ht_mcs_rate((psta->phl_sta->chandef.bw == CHANNEL_WIDTH_40) ? 1 : 0
+			, short_GI
+			, psta->htpriv.ht_cap.supp_mcs_set
+		);
+	}
+#ifdef CONFIG_80211AC_VHT
+	else if (is_supported_vht(psta->phl_sta->wmode))
+		max_rate = ((rtw_vht_mcs_to_data_rate(psta->phl_sta->chandef.bw, short_GI, pmlmepriv->vhtpriv.vht_highest_rate) + 1) >> 1) * 10;
+#ifdef CONFIG_80211AX_HE
+	else if (is_supported_he(psta->phl_sta->wmode))
+		max_rate =
+			((rtw_he_mcs_to_data_rate(psta->phl_sta->chandef.bw,
+						  psta->phl_sta->asoc_cap.ltf_gi,
+						  pmlmepriv->hepriv.he_highest_rate) + 1) >> 1) * 10;
+#endif /* CONFIG_80211AX_HE */
+#endif /* CONFIG_80211AC_VHT */
+	else
+#endif /* CONFIG_80211N_HT */
+	{
+		/*station mode show :station && ap support rate; softap :show ap support rate*/
+		if (MLME_IS_STA(adapter))
+			get_rate_set(adapter, psta->padapter_link, sta_bssrate, &sta_bssrate_len);/*get sta rate and length*/
+
+
+		while ((pcur_bss->SupportedRates[i] != 0) && (pcur_bss->SupportedRates[i] != 0xFF)) {
+			rate = pcur_bss->SupportedRates[i] & 0x7F;/*AP support rates*/
+			/*RTW_INFO("%s rate=%02X \n", __func__, rate);*/
+
+			/*check STA  support rate or not */
+			if (MLME_IS_STA(adapter)) {
+				for (j = 0; j < sta_bssrate_len; j++) {
+					/* Avoid the proprietary data rate (22Mbps) of Handlink WSG-4000 AP */
+					if ((rate | IEEE80211_BASIC_RATE_MASK)
+					    == (sta_bssrate[j] | IEEE80211_BASIC_RATE_MASK)) {
+						if (rate > max_rate) {
+							max_rate = rate;
+						}
+						break;
+					}
+				}
+			} else {
+			
+				if (rate > max_rate)
+					max_rate = rate;
+
+			}
+			i++;
+		}
+
+		max_rate = max_rate * 10 / 2;
+	}
+	return max_rate;
+}
+
+/*
+* rtw_set_scan_mode -
+* @adapter: pointer to _adapter structure
+* @scan_mode:
+*
+* Return _SUCCESS or _FAIL
+*/
+int rtw_set_scan_mode(_adapter *adapter, enum rtw_phl_scan_type scan_mode)
+{
+	if (scan_mode != RTW_PHL_SCAN_ACTIVE &&
+		scan_mode != RTW_PHL_SCAN_PASSIVE)
+		return _FAIL;
+
+	adapter->mlmepriv.scan_mode = scan_mode;
+
+	return _SUCCESS;
+}
+
+/*
+* rtw_set_channel_plan -
+* @adapter: pointer to _adapter structure
+* @channel_plan:
+*
+* Return _SUCCESS or _FAIL
+*/
+int rtw_set_channel_plan(_adapter *adapter, u8 channel_plan, u8 chplan_6g
+	, enum rtw_env_t env, enum rtw_regd_inr inr)
+{
+	struct registry_priv *regsty = adapter_to_regsty(adapter);
+
+	if (!REGSTY_REGD_SRC_FROM_OS(regsty))
+		return rtw_set_chplan_cmd(adapter, RTW_CMDF_WAIT_ACK, channel_plan, chplan_6g, env, inr);
+	RTW_WARN("%s(): not applied\n", __func__);
+	return _SUCCESS;
+}
+
+/*
+* rtw_set_country -
+* @adapter: pointer to _adapter structure
+* @country_code: string of country code
+*
+* Return _SUCCESS or _FAIL
+*/
+int rtw_set_country(_adapter *adapter, const char *country_code
+	, enum rtw_env_t env, enum rtw_regd_inr inr)
+{
+#ifdef CONFIG_RTW_IOCTL_SET_COUNTRY
+#if ((0 - CONFIG_RTW_IOCTL_SET_COUNTRY - 1) == 1) && ((CONFIG_RTW_IOCTL_SET_COUNTRY + 0) != -2) /* defined to empty */
+#undef CONFIG_RTW_IOCTL_SET_COUNTRY
+#define CONFIG_RTW_IOCTL_SET_COUNTRY 1
+#endif
+#else /* not defined */
+#define CONFIG_RTW_IOCTL_SET_COUNTRY 1
+#endif
+
+#if CONFIG_RTW_IOCTL_SET_COUNTRY
+	struct registry_priv *regsty = adapter_to_regsty(adapter);
+
+	if (!REGSTY_REGD_SRC_FROM_OS(regsty))
+		return rtw_set_country_cmd(adapter, RTW_CMDF_WAIT_ACK, country_code, env, inr);
+#endif
+	RTW_WARN("%s(): not applied\n", __func__);
+	return _SUCCESS;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+/*
+* rtw_set_env -
+* @adapter: pointer to _adapter structure
+* @env: environment
+*
+* Return _SUCCESS or _FAIL
+*/
+int rtw_set_env(_adapter *adapter, enum rtw_env_t env, enum rtw_regd_inr inr)
+{
+	struct registry_priv *regsty = adapter_to_regsty(adapter);
+
+	return rtw_set_env_cmd(adapter, RTW_CMDF_WAIT_ACK, env
+		, REGSTY_REGD_SRC_FROM_OS(regsty) ? REGD_SRC_OS : REGD_SRC_RTK_PRIV
+		, inr);
+}
+#endif
+
+/*
+* rtw_set_band -
+* @adapter: pointer to _adapter structure
+* @band: band to set
+*
+* Return _SUCCESS or _FAIL
+*/
+int rtw_set_band(_adapter *adapter, u8 band)
+{
+	if (rtw_band_valid(band)) {
+		RTW_INFO(FUNC_ADPT_FMT" band:%d\n", FUNC_ADPT_ARG(adapter), band);
+		adapter->setband = band;
+		return _SUCCESS;
+	}
+
+	RTW_PRINT(FUNC_ADPT_FMT" band:%d fail\n", FUNC_ADPT_ARG(adapter), band);
+	return _FAIL;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mbo.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mbo.c
new file mode 100644
index 000000000000..c5eabb0c88de
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mbo.c
@@ -0,0 +1,1558 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_MBO_C
+
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_MBO
+
+#ifndef RTW_MBO_DBG
+#define RTW_MBO_DBG	0
+#endif
+#if RTW_MBO_DBG
+	#define RTW_MBO_INFO(fmt, arg...)	\
+		RTW_INFO(fmt, arg)
+	#define RTW_MBO_DUMP(str, data, len)	\
+		RTW_INFO_DUMP(str, data, len)
+#else
+	#define RTW_MBO_INFO(fmt, arg...) do {} while (0)
+	#define RTW_MBO_DUMP(str, data, len) do {} while (0)
+#endif
+
+/* Cellular Data Connectivity field
+ * 1 : Cellular data connection available
+ * 2 : Cellular data connection not available
+ * 3 : Not Cellular data capable
+ * otherwise : Reserved
+*/
+int rtw_mbo_cell_data_conn = 2;
+module_param(rtw_mbo_cell_data_conn, int, 0644);
+
+static u8 wfa_mbo_oui[] = {0x50, 0x6F, 0x9A, 0x16};
+
+#define rtw_mbo_get_oui(p) ((u8 *)(p) + 2)
+
+#define rtw_mbo_get_attr_id(p) ((u8 *)(p))
+
+#define rtw_mbo_get_disallow_res(p) ((u8 *)(p) + 3)
+
+#define rtw_mbo_set_1byte_ie(p, v, l)	\
+	rtw_set_fixed_ie((p), 1, (v), (l))
+
+#define rtw_mbo_set_2byte_ie(p, v, l)	\
+	rtw_set_fixed_ie((p), 2, (v), (l))
+
+#define rtw_mbo_set_4byte_ie(p, v, l)	\
+	rtw_set_fixed_ie((p), 4, (v), (l))
+
+#define rtw_mbo_set_nbyte_ie(p, sz, v, l)	\
+	rtw_set_fixed_ie((p), (sz), (v), (l))
+
+#define rtw_mbo_subfield_set(p, offset, val) (*(p + offset) = val)
+
+#define rtw_mbo_subfields_set(p, offset, buf, len)	\
+	do {	\
+		u32 _offset = 0;	\
+		u8 *_p = p + offset;	\
+		while(_offset < len) {	\
+			*(_p + _offset) = *(buf + _offset);	\
+			_offset++;	\
+		}	\
+	} while(0)
+
+
+void rtw_mbo_ie_init(_adapter *padapter, struct mbo_priv *mbopriv)
+{
+	if(!mbopriv)
+		return;
+	mbopriv->assoc_disallow = 0;
+	mbopriv->cellular_aware = 0;
+	mbopriv->ch_list_num = 0;
+	mbopriv->mbo_oce_element_len = 6;
+	mbopriv->mbo_oce_element[0] = 0xdd;
+	mbopriv->mbo_oce_element[1] = mbopriv->mbo_oce_element_len;
+	mbopriv->mbo_oce_element[2] = 0x50;
+	mbopriv->mbo_oce_element[3] = 0x6f;
+	mbopriv->mbo_oce_element[4] = 0x9a;
+	mbopriv->mbo_oce_element[5] = 0x16;
+}
+
+void rtw_mbo_fill_non_prefer_channel_list(_adapter *padapter, struct mbo_priv *mbopriv,
+										const u8 *pbuf, u8 len)
+{
+	u8 op_class = 0;
+	u8 preference = 0;
+	int i;
+
+	/* invalid length */
+	if(len != 0 && len < 3)
+		return;
+
+	/* reset non-prefer channel list */
+	mbopriv->ch_list_num = 0;
+	op_class = *pbuf;
+	preference = *(pbuf + len - 2);
+
+	if (len == 3 && mbopriv->ch_list_num < MBO_CH_LIST_MAX_NUM) {
+		mbopriv->ch_list[mbopriv->ch_list_num].op_class = op_class;
+		mbopriv->ch_list[mbopriv->ch_list_num].preference = preference;
+		mbopriv->ch_list[mbopriv->ch_list_num].channel = 0;
+		mbopriv->ch_list_num += 1;
+		RTW_INFO("[%s:%d]channel = %d, preference = %d\n", __func__, __LINE__, 0, preference);
+	} else {
+		for (i = 0; i < len - 3; i++) {
+			if(mbopriv->ch_list_num >= MBO_CH_LIST_MAX_NUM)
+				break;
+			mbopriv->ch_list[mbopriv->ch_list_num].op_class = op_class;
+			mbopriv->ch_list[mbopriv->ch_list_num].preference = preference;
+			mbopriv->ch_list[mbopriv->ch_list_num].channel = *(pbuf + 1 + i);
+			mbopriv->ch_list_num += 1;
+			RTW_INFO("[%s:%d]channel = %d, preference = %d\n", __func__, __LINE__,
+									*(pbuf + 1 + i), preference);
+		}
+	}
+}
+
+void rtw_mbo_ie_handler(_adapter *padapter, struct mbo_priv *mbopriv, const u8 *pbuf, uint limit_len)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	uint total_len = 0;
+	u8 attribute_id = 0;
+	u8 attribute_len = 0;
+	u8 mbo_hdr_len = 6;
+	const u8 *p = pbuf;
+
+	if(!mbopriv)
+		return;
+	if (limit_len  > MBO_OCE_ELEMENT_MAX_LEN - mbo_hdr_len) {
+		RTW_ERR("[%s] mbo_oce_element array migh be overrun\n", __func__);
+		return ;
+	}
+
+	rtw_mbo_ie_init(padapter, mbopriv);
+	_rtw_memcpy(mbopriv->mbo_oce_element + mbo_hdr_len, pbuf, limit_len);
+	mbopriv->mbo_oce_element[1] = limit_len + 4;
+
+	while (total_len <= limit_len) {
+		attribute_id = *p;
+		attribute_len = *(p + 1);
+		total_len += attribute_len;
+		if(total_len > limit_len)
+			break;
+
+		switch (attribute_id) {
+			case MBO_AP_CAPABILITY:
+				if(attribute_len == 1){
+					RTW_INFO("[%s]Find attribute MBO_AP_CAPABILITY\n", __func__);
+					if(*(p+2) & 0x40)
+						mbopriv->cellular_aware = 1;
+				}
+				break;
+			case ASSOCIATION_DISALLOW:
+				if(attribute_len == 1){
+					RTW_INFO("[%s]Find attribute ASSOCIATION_DISALLOW\n", __func__);
+					mbopriv->assoc_disallow = *(p+2);
+				}
+				break;
+			case NON_PREFER_CHANNEL_RPT:
+				RTW_INFO("[%s]Find attribute NON_PREFER_CHANNEL_RPT\n", __func__);
+				rtw_mbo_fill_non_prefer_channel_list(padapter, mbopriv, p + 2, attribute_len);
+				break;
+			case CELLULAR_DATA_CAPABILITY:
+			case CELLULAR_DATA_CONNECT_PREFER:
+			case TRANS_REASON_CODE:
+			case TRANS_REJECT_REASON_CODE:
+			case ASSOCIATION_RETRY_DELAY:
+				break;
+			default:
+				RTW_ERR("[%s]Unknown MBO attribute %d\n", __func__, attribute_id);
+		}
+
+		p += (attribute_len + 2);
+	}
+}
+
+void rtw_ap_parse_sta_mbo_element(_adapter *padapter,
+	struct sta_info *psta, u8 *ies_buf, u16 ies_len)
+{
+	uint ie_len = 0;
+	u8 *p;
+	u8 WIFI_ALLIANCE_OUI[] = {0x50, 0x6f, 0x9a};
+
+	ie_len = 0;
+	for (p = ies_buf; ; p += (ie_len + 2)) {
+		p = rtw_get_ie(p, _SSN_IE_1_, &ie_len, (ies_len - (ie_len + 2)));
+		if ((p) && (_rtw_memcmp(p + 2, WIFI_ALLIANCE_OUI, 3)) && (*(p+5) == MBO_OUI_TYPE)) {
+			/* find MBO-OCE information element */
+			psta->mbopriv.enable = _TRUE;
+			rtw_mbo_ie_handler(padapter, &psta->mbopriv, p + 6, ie_len - 4);
+			break;
+		}
+		if ((p == NULL) || (ie_len == 0))
+			break;
+	}
+}
+
+static u8 *rtw_mbo_ie_get(u8 *pie, u32 *plen, u32 limit)
+{
+	const u8 *p = pie;
+	u32 tmp, i;
+
+	if (limit <= 1)
+		return NULL;
+
+	i = 0;
+	*plen = 0;
+	while (1) {
+		if ((*p == _VENDOR_SPECIFIC_IE_) &&
+			(_rtw_memcmp(rtw_mbo_get_oui(p), wfa_mbo_oui, 4))) {
+			*plen = *(p + 1);
+			/* RTW_MBO_DUMP("VENDOR_SPECIFIC_IE MBO: ", p, *(p + 1)); */
+			return (u8 *)p;
+		} else {
+			tmp = *(p + 1);
+			p += (tmp + 2);
+			i += (tmp + 2);
+		}
+
+		if (i >= limit)
+			break;
+	}
+
+	return NULL;
+}
+
+static u8 *rtw_mbo_attrs_get(u8 *pie,
+	u32 limit, u8 attr_id,u32 *attr_len, u8 dbg)
+{
+	u8 *p = NULL;
+	u32 offset, plen = 0;
+
+	if ((pie == NULL) || (limit > MAX_IE_SZ))
+		goto exit;
+
+	if ((p = rtw_mbo_ie_get(pie, &plen, limit)) == NULL)
+		goto exit;
+
+	/* shift 2 + OUI size and move to attributes content */
+	p = p + 2 + sizeof(wfa_mbo_oui);
+	plen = plen - 4;
+
+	if (dbg)
+		RTW_MBO_DUMP("Attributes contents: ", p, plen);
+
+	if ((p = rtw_get_ie(p, attr_id, attr_len, plen)) == NULL)
+		goto exit;
+
+	if (dbg) {
+		RTW_MBO_INFO("%s : id=%u(len=%u)\n",
+			__func__, attr_id, *attr_len);
+		RTW_MBO_DUMP("contents : ", (p + 2), *attr_len);
+	}
+
+exit:
+	return p;
+
+}
+
+static u32 rtw_mbo_attr_sz_get(
+	_adapter *padapter, u8 id)
+{
+	u32 len = 0;
+
+	switch (id) {
+		case RTW_MBO_ATTR_NPREF_CH_RPT_ID:
+			{
+				struct rf_ctl_t *prfctl = \
+					adapter_to_rfctl(padapter);
+				struct npref_ch_rtp *prpt = \
+					&(prfctl->ch_rtp);
+				struct npref_ch* pch;
+				u32 i, attr_len, offset;
+
+				for (i=0; i < prpt->nm_of_rpt; i++) {
+					pch = &prpt->ch_rpt[i];
+					/*attr_len = ch list + op class
+						 + preference + reason */
+					attr_len = pch->nm_of_ch + 3;
+					/* offset = id + len field
+						 + attr_len */
+					offset = attr_len + 2;
+					len += offset;
+				}
+			}
+			break;
+		case RTW_MBO_ATTR_ASSOC_RETRY_DELAY_ID:
+			len = 4;
+			break;
+		case RTW_MBO_ATTR_AP_CAP_ID:
+		case RTW_MBO_ATTR_CELL_DATA_CAP_ID:
+		case RTW_MBO_ATTR_ASSOC_DISABLED_ID:
+		case RTW_MBO_ATTR_TRANS_RES_ID:
+		case RTW_MBO_ATTR_TRANS_REJ_ID:
+			len = 3;
+			break;
+		default:
+			break;
+	}
+
+	return len;
+}
+
+static void rtw_mbo_build_mbo_ie_hdr(
+	u8 **pframe, struct pkt_attrib *pattrib, u8 payload_len)
+{
+	u8 eid = RTW_MBO_EID;
+	u8 len = payload_len + 4;
+
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &eid, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &len, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_4byte_ie(*pframe,
+			wfa_mbo_oui, &(pattrib->pktlen));
+}
+
+void rtw_mbo_build_cell_data_cap_attr(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 attr_id = RTW_MBO_ATTR_CELL_DATA_CAP_ID;
+	u8 attr_len = 1;
+	u8 cell_data_con = rtw_mbo_cell_data_conn;
+
+	/* used Cellular Data Capabilities from supplicant */
+	if (!rtw_mbo_wifi_logo_test(padapter) &&
+		pmlmepriv->pcell_data_cap_ie &&
+		pmlmepriv->cell_data_cap_len == 1) {
+		cell_data_con = *pmlmepriv->pcell_data_cap_ie;
+	}
+
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_id, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_len, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe,
+			&cell_data_con, &(pattrib->pktlen));
+}
+
+
+static void rtw_mbo_build_ap_cap_Indication_attr(
+	_adapter *padapter, u8 **pframe,
+	struct pkt_attrib *pattrib, u8 cap_ind)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 attr_id = RTW_MBO_ATTR_AP_CAP_ID;
+	u8 attr_len = 1;
+	u8 ap_cap_ind = cap_ind;
+
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_id, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_len, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe,
+			&ap_cap_ind, &(pattrib->pktlen));
+}
+
+static void rtw_mbo_build_ap_disallowed_attr(
+	_adapter *padapter, u8 **pframe,
+	struct pkt_attrib *pattrib, u8 code)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 attr_id = RTW_MBO_ATTR_ASSOC_DISABLED_ID;
+	u8 attr_len = 1;
+	u8 reason = 0;
+
+	if (code > 0) {
+		reason = code;
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&attr_id, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&attr_len, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&reason, &(pattrib->pktlen));
+	}
+}
+
+static void rtw_mbo_build_ap_trans_reason_attr(
+	_adapter *padapter, u8 **pframe,
+	struct pkt_attrib *pattrib, u8 code)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 attr_id = RTW_MBO_ATTR_TRANS_RES_ID;
+	u8 attr_len = 1;
+	u8 reason = 0;
+
+	reason = code;
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&attr_id, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&attr_len, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&reason, &(pattrib->pktlen));
+	}
+
+static void rtw_mbo_build_ap_assoc_retry_delay_attr(
+	_adapter *padapter, u8 **pframe,
+	struct pkt_attrib *pattrib, u16 code)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 attr_id = RTW_MBO_ATTR_ASSOC_RETRY_DELAY_ID;
+	u8 attr_len = 2;
+	u16 delay = 0;
+
+	delay = code;
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_id, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_len, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_2byte_ie(*pframe,
+			(u8 *)&delay, &(pattrib->pktlen));
+}
+
+static void rtw_mbo_update_cell_data_cap(
+	_adapter *padapter, u8 *pie, u32 ie_len)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 *mbo_attr;
+	u32	mbo_attrlen;
+
+	if ((pie == NULL) || (ie_len == 0))
+		return;
+
+	mbo_attr = rtw_mbo_attrs_get(pie, ie_len,
+		RTW_MBO_ATTR_CELL_DATA_CAP_ID, &mbo_attrlen, 0);
+
+	if ((mbo_attr == NULL) || (mbo_attrlen == 0) ) {
+		RTW_INFO("MBO : Cellular Data Capabilities not found!\n");
+		return;
+	}
+
+	rtw_buf_update(&pmlmepriv->pcell_data_cap_ie,
+		&pmlmepriv->cell_data_cap_len, (mbo_attr + 2), mbo_attrlen);
+	RTW_MBO_DUMP("rtw_mbo_update_cell_data_cap : ",
+		pmlmepriv->pcell_data_cap_ie, pmlmepriv->cell_data_cap_len);
+}
+
+void rtw_mbo_update_ie_data(
+	_adapter *padapter, u8 *pie, u32 ie_len)
+{
+	rtw_mbo_update_cell_data_cap(padapter, pie, ie_len);
+}
+
+static u8 rtw_mbo_construct_npref_ch_rpt_attr(
+	_adapter *padapter, u8 *pbuf, u32 buf_len, u32 *plen)
+{
+	struct rf_ctl_t *prfctl = adapter_to_rfctl(padapter);
+	struct npref_ch_rtp *prpt = &(prfctl->ch_rtp);
+	struct npref_ch* pch;
+	u32 attr_len, offset;
+	int i;
+	u8 *p = pbuf;
+
+	if (prpt->nm_of_rpt == 0) {
+		*plen = 0;
+		return _FALSE;
+	}
+
+	for (i=0; i < prpt->nm_of_rpt; i++) {
+		pch = &prpt->ch_rpt[i];
+		/* attr_len = ch list + op class + preference + reason */
+		attr_len = pch->nm_of_ch + 3;
+		/* offset = id + len field + attr_len */
+		offset = attr_len + 2;
+		rtw_mbo_subfield_set(p, 0, RTW_MBO_ATTR_NPREF_CH_RPT_ID);
+		rtw_mbo_subfield_set(p, 1, attr_len);
+		rtw_mbo_subfield_set(p, 2, pch->op_class);
+		rtw_mbo_subfields_set(p, 3, pch->chs, pch->nm_of_ch);
+		rtw_mbo_subfield_set(p, (offset - 2), pch->preference);
+		rtw_mbo_subfield_set(p, (offset - 1), pch->reason);
+		p +=  offset;
+		*plen += offset;
+
+		if (*plen >=  buf_len) {
+			RTW_ERR("MBO : construct non-preferred-ch rpt fail!\n");
+			return _FALSE;
+		}
+	}
+
+	return _TRUE;
+}
+
+void rtw_mbo_build_npref_ch_rpt_attr(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	struct rf_ctl_t *prfctl = adapter_to_rfctl(padapter);
+	struct npref_ch_rtp *prpt = &(prfctl->ch_rtp);
+	u32 tmp_sz = 0, body_len = 0;
+	u8 *ptmp;
+
+	tmp_sz = prpt->nm_of_rpt * sizeof(struct npref_ch);
+	ptmp = rtw_zmalloc(tmp_sz);
+	if (ptmp == NULL)
+		return;
+
+	if (rtw_mbo_construct_npref_ch_rpt_attr(
+		padapter, ptmp, tmp_sz, &body_len) == _FALSE) {
+		rtw_mfree(ptmp, tmp_sz);
+		return;
+	}
+
+	RTW_MBO_DUMP("Non-preferred Channel Report :", ptmp, body_len);
+	*pframe = rtw_mbo_set_nbyte_ie(*pframe, body_len,
+			ptmp, &(pattrib->pktlen));
+
+	rtw_mfree(ptmp, tmp_sz);
+}
+
+void rtw_mbo_build_trans_reject_reason_attr(
+	_adapter *padapter, u8 **pframe,
+	struct pkt_attrib *pattrib, u8 *pres)
+{
+	u8 attr_id = RTW_MBO_ATTR_TRANS_REJ_ID;
+	u8 attr_len = 1;
+	u32 len = 0;
+
+	len = rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_TRANS_REJ_ID);
+	if ((len == 0) || (len > 3)) {
+		RTW_ERR("MBO : build Transition Rejection Reason"
+			" attribute fail(len=%u)\n", len);
+		return;
+	}
+
+	rtw_mbo_build_mbo_ie_hdr(pframe, pattrib, len);
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_id, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, &attr_len, &(pattrib->pktlen));
+	*pframe = rtw_mbo_set_1byte_ie(*pframe, pres, &(pattrib->pktlen));
+}
+
+u8 rtw_mbo_disallowed_network(struct wlan_network *pnetwork)
+{
+	u8 *p, *attr_id, *res;
+	u32 attr_len = 0;
+	u8 disallow = _FALSE;
+
+	if ((pnetwork == NULL) || \
+		((p = rtw_mbo_attrs_get(
+			pnetwork->network.IEs,
+		pnetwork->network.IELength,
+		RTW_MBO_ATTR_ASSOC_DISABLED_ID,
+			&attr_len, 0)) == NULL)) {
+		goto exit;
+	}
+
+	RTW_MBO_DUMP("Association Disallowed attribute :",p , attr_len + 2);
+	RTW_INFO("MBO : block "MAC_FMT" assoc disallowed reason %d\n",
+		MAC_ARG(pnetwork->network.MacAddress),
+		*(rtw_mbo_get_disallow_res(p)));
+
+	disallow = _TRUE;
+exit:
+	return disallow;
+}
+
+void rtw_mbo_build_extended_cap(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+
+	rtw_wnm_add_btm_ext_cap(pmlmepriv->ext_capab_ie_data,
+				&(pmlmepriv->ext_capab_ie_len));
+	rtw_mbo_add_internw_ext_cap(pmlmepriv->ext_capab_ie_data,
+				&(pmlmepriv->ext_capab_ie_len));
+	*pframe = rtw_set_ie(*pframe,
+				WLAN_EID_EXT_CAP,
+				pmlmepriv->ext_capab_ie_len,
+				pmlmepriv->ext_capab_ie_data,
+				&(pattrib->pktlen));
+}
+
+static void rtw_mbo_non_pref_chans_dump(struct npref_ch* pch)
+{
+	int i, len = 0;
+	u8 buf[256] = {0}; /* max 60ch x 4byte */
+
+	for (i=0; i < pch->nm_of_ch; i++)
+		len += rtw_sprintf(buf + strlen(buf), sizeof(buf) - len, "%d ", pch->chs[i]);
+
+	RTW_MBO_INFO("%s : op_class=%01x, ch=%s, preference=%d, reason=%d\n",
+		__func__, pch->op_class, buf, pch->preference, pch->reason);
+}
+
+static u8 rtw_mbo_non_pref_chan_exist(struct npref_ch* pch, u8 ch)
+{
+	u32 i;
+	u8 found = _FALSE;
+
+	for (i=0; i < pch->nm_of_ch; i++) {
+		if (pch->chs[i] == ch) {
+			found = _TRUE;
+			break;
+		}
+	}
+
+	return found;
+}
+
+static struct npref_ch* rtw_mbo_non_pref_chan_get(
+	_adapter *padapter, u8 op_class, u8  prefe, u8  res)
+{
+	struct rf_ctl_t *prfctl = adapter_to_rfctl(padapter);
+	struct npref_ch_rtp *prpt = &(prfctl->ch_rtp);
+	struct npref_ch* pch = NULL;
+	int i;
+
+	if (prpt->nm_of_rpt == 0)
+		return pch;
+
+	for (i=0; i < prpt->nm_of_rpt; i++) {
+		if ((prpt->ch_rpt[i].op_class == op_class) &&
+			(prpt->ch_rpt[i].preference == prefe) &&
+			(prpt->ch_rpt[i].reason == res)) {
+			pch = &prpt->ch_rpt[i];
+			break;
+		}
+	}
+
+	return pch;
+}
+
+static void rtw_mbo_non_pref_chan_set(
+	struct npref_ch* pch, u8 op_class,
+	u8 ch, u8  prefe, u8  res, u8 update)
+{
+	u32 offset = pch->nm_of_ch;
+
+	if (update) {
+		if (rtw_mbo_non_pref_chan_exist(pch, ch) == _FALSE) {
+			pch->chs[offset] = ch;
+			pch->nm_of_ch++;
+		}
+	} else {
+		pch->op_class = op_class;
+		pch->chs[0] = ch;
+		pch->preference = prefe;
+		pch->reason = res;
+		pch->nm_of_ch = 1;
+	}
+}
+
+static void  rtw_mbo_non_pref_chans_update(
+	_adapter *padapter, u8 op_class, u8 ch, u8  prefe, u8  res)
+{
+	struct rf_ctl_t *prfctl = adapter_to_rfctl(padapter);
+	struct npref_ch_rtp *pch_rpt = &(prfctl->ch_rtp);
+	struct npref_ch* pch;
+
+	if (pch_rpt->nm_of_rpt >= RTW_MBO_MAX_CH_RPT_NUM) {
+		RTW_ERR("MBO : %d non_pref_chan entries supported!",
+			RTW_MBO_MAX_CH_RPT_NUM);
+		return;
+	}
+
+	if (pch_rpt->nm_of_rpt == 0) {
+		pch = &pch_rpt->ch_rpt[0];
+		rtw_mbo_non_pref_chan_set(pch, op_class,
+				ch, prefe, res, _FALSE);
+		pch_rpt->nm_of_rpt = 1;
+		return;
+	}
+
+	pch = rtw_mbo_non_pref_chan_get(padapter, op_class, prefe, res);
+	if (pch == NULL) {
+		pch = &pch_rpt->ch_rpt[pch_rpt->nm_of_rpt];
+		rtw_mbo_non_pref_chan_set(pch, op_class,
+				ch, prefe, res, _FALSE);
+		pch_rpt->nm_of_rpt++;
+	} else {
+		rtw_mbo_non_pref_chan_set(pch, op_class,
+				ch, prefe, res, _TRUE);
+	}
+
+	rtw_mbo_non_pref_chans_dump(pch);
+}
+
+static void  rtw_mbo_non_pref_chans_set(
+	_adapter *padapter, char *param, ssize_t sz)
+{
+	char *pnext;
+	u32 op_class, ch, prefe, res;
+	int i = 0;
+
+	do {
+		pnext = strsep(&param, " ");
+		if (pnext == NULL)
+			break;
+
+		sscanf(pnext, "%d:%d:%d:%d", &op_class, &ch, &prefe, &res);
+		rtw_mbo_non_pref_chans_update(padapter, op_class,
+					ch, prefe, res);
+
+		if ((i++) > 10) {
+			RTW_ERR("MBO : overflow %d \n", i);
+			break;
+		}
+
+	} while (param != (char*)'\0');
+
+}
+
+static void  rtw_mbo_non_pref_chans_del(
+	_adapter *padapter, char *param, ssize_t sz)
+{
+	struct rf_ctl_t *prfctl = adapter_to_rfctl(padapter);
+	struct npref_ch_rtp *prpt = &(prfctl->ch_rtp);
+
+	RTW_INFO("%s : delete non_pref_chan %s\n", __func__, param);
+	_rtw_memset(prpt, 0, sizeof(struct npref_ch_rtp));
+}
+
+ssize_t rtw_mbo_proc_non_pref_chans_set(
+	struct file *pfile, const char __user *buffer,
+	size_t count, loff_t *pos, void *pdata)
+{
+	struct net_device *dev = pdata;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	u8 tmp[128] = {0};
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		if (strncmp(tmp, "add", 3) == 0) {
+			rtw_mbo_non_pref_chans_set(padapter,
+					&tmp[4], (count - 4));
+		} else if (strncmp(tmp, "delete", 6) == 0) {
+			rtw_mbo_non_pref_chans_del(padapter,
+					&tmp[7], (count - 7));
+		} else {
+			RTW_ERR("MBO : Invalid format : echo [add|delete]"
+			" <oper_class>:<chan>:<preference>:<reason>\n");
+			return -EFAULT;
+		}
+	}
+
+#ifdef CONFIG_RTW_WNM
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) &&
+		check_fwstate(pmlmepriv, WIFI_STATION_STATE))
+		rtw_wnm_issue_action(padapter, &pmlmepriv->nb_info,
+				RTW_WLAN_ACTION_WNM_NOTIF_REQ, 0, 0);
+#endif
+
+	return count;
+}
+
+int rtw_mbo_proc_non_pref_chans_get(
+	struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *prfctl = adapter_to_rfctl(padapter);
+	struct npref_ch_rtp *prpt = &(prfctl->ch_rtp);
+	struct npref_ch* pch;
+	int i,j,len;
+	u8 buf[256] = {0};
+
+	RTW_PRINT_SEL(m, "op_class                     ch    preference    reason \n");
+	RTW_PRINT_SEL(m, "=======================================================\n");
+
+	if (prpt->nm_of_rpt == 0) {
+		RTW_PRINT_SEL(m, " empty table \n");
+		return 0;
+	}
+
+	for (i=0; i < prpt->nm_of_rpt; i++) {
+		len = 0;
+		_rtw_memset(buf, 0, sizeof(buf));
+		pch = &prpt->ch_rpt[i];
+		for (j=0; j < pch->nm_of_ch; j++)
+			len = rtw_sprintf(buf + len, 256 - len, "%02u ", pch->chs[j]);
+
+		RTW_PRINT_SEL(m, "    %04u    %20s           %02u        %02u\n",
+			pch->op_class, buf, pch->preference, pch->reason);
+	}
+
+	return 0;
+}
+
+ssize_t rtw_mbo_proc_cell_data_set(
+	struct file *pfile, const char __user *buffer,
+	size_t count, loff_t *pos, void *pdata)
+{
+	struct net_device *dev = pdata;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	int mbo_cell_data = 0;
+	u8 tmp[8] = {0};
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp))
+		return -EFAULT;
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%d", &mbo_cell_data);
+		if (num == 1) {
+			rtw_mbo_cell_data_conn = mbo_cell_data;
+		#ifdef CONFIG_RTW_WNM
+			if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) &&
+				check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
+				rtw_wnm_issue_action(padapter, &pmlmepriv->nb_info,
+					RTW_WLAN_ACTION_WNM_NOTIF_REQ, 0, 0);
+			}
+		#endif
+		}
+	}
+
+	return count;
+}
+
+int rtw_mbo_proc_cell_data_get(
+	struct seq_file *m, void *v)
+{
+#if 0
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+#endif
+
+	RTW_PRINT_SEL(m, "Cellular Data Connectivity : %d\n",
+			rtw_mbo_cell_data_conn);
+	return 0;
+}
+
+
+static void rtw_mbo_disassoc(_adapter *padapter, u8 *da,
+			u16 reason, u8 wait_ack)
+{
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattrib;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	u8 *pframe;
+	u16 *fctrl;
+	int ret = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info *pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+	pattrib->retry_ctrl = _FALSE;
+        pattrib->key_type = IEEE80211W_RIGHT_KEY;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_DISASSOC);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	reason = cpu_to_le16(reason);
+	pframe = rtw_set_fixed_ie(pframe, _RSON_CODE_ ,
+			(unsigned char *)&reason, &(pattrib->pktlen));
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	if (wait_ack)
+		dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else
+		dump_mgntframe(padapter, pmgntframe);
+	RTW_MBO_INFO("%s : reason %u\n", __func__, reason);
+}
+
+static void rtw_mbo_construct_user_btm_req(
+	_adapter *padapter, struct btm_req_hdr *phdr,
+	u8 *purl, u32 url_len, struct wnm_btm_cant *pbtm_cant)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	struct mbo_user_btm_req_pkt *puser = &(pmbo_attr->user_raw);
+	struct wnm_btm_cant *puser_cant = NULL;
+
+	if (phdr) {
+		puser->hdr.req_mode = phdr->req_mode;
+		puser->hdr.disassoc_timer = phdr->disassoc_timer;
+		puser->hdr.validity_interval = phdr->validity_interval;
+		puser->hdr.term_duration.tsf = phdr->term_duration.tsf;
+		puser->hdr.term_duration.duration = \
+			phdr->term_duration.duration;
+		puser->hdr.term_duration.id = 0x4;
+		puser->hdr.term_duration.len = 0xa;
+		RTW_MBO_INFO("%s : req-mode=0x%x, disassoc_timer=%u, "
+			"validity_interval=%u, tsf=%llu, "
+			"duration=%u\n", __func__,
+			puser->hdr.req_mode, puser->hdr.disassoc_timer,
+			puser->hdr.validity_interval,
+			puser->hdr.term_duration.tsf,
+			puser->hdr.term_duration.duration);
+	}
+
+	if (purl && url_len) {
+		/* TODO */
+	}
+
+	if (pbtm_cant) {
+		struct wnm_btm_cant *pbtm_tb = NULL;
+		u8 i, idx, found = _FALSE;
+
+		for (i = 0; i < RTW_MAX_NB_RPT_NUM; i++) {
+			puser_cant = &puser->btm_cants[i];
+			if (_rtw_memcmp(pbtm_cant->nb_rpt.bssid,
+				puser_cant->nb_rpt.bssid, ETH_ALEN)) {
+				puser_cant->nb_rpt.bss_info = \
+					pbtm_cant->nb_rpt.bss_info;
+				puser_cant->nb_rpt.reg_class = \
+					pbtm_cant->nb_rpt.reg_class;
+				puser_cant->nb_rpt.ch_num = \
+					pbtm_cant->nb_rpt.ch_num;
+				puser_cant->nb_rpt.phy_type = \
+					pbtm_cant->nb_rpt.phy_type;
+				puser_cant->preference = \
+					pbtm_cant->preference;
+				idx = i;
+				found = _TRUE;
+				break;
+			}
+		}
+
+		if (!found) {
+			if (puser->candidate_cnt >= RTW_MAX_NB_RPT_NUM)
+				puser->candidate_cnt = 0;
+			puser_cant = &puser->btm_cants[puser->candidate_cnt];
+			puser_cant->nb_rpt.id = \
+				RTW_WLAN_ACTION_WNM_NB_RPT_ELEM;
+			puser_cant->nb_rpt.len = 0x10;
+			_rtw_memcpy(puser_cant->nb_rpt.bssid,
+				pbtm_cant->nb_rpt.bssid, ETH_ALEN);
+			puser_cant->nb_rpt.bss_info = \
+				pbtm_cant->nb_rpt.bss_info;
+			puser_cant->nb_rpt.reg_class = \
+				pbtm_cant->nb_rpt.reg_class;
+			puser_cant->nb_rpt.ch_num = \
+				pbtm_cant->nb_rpt.ch_num;
+			puser_cant->nb_rpt.phy_type = \
+				pbtm_cant->nb_rpt.phy_type;
+			puser_cant->preference = \
+				pbtm_cant->preference;
+			idx = puser->candidate_cnt;
+			puser->candidate_cnt++;
+		}
+
+		RTW_MBO_INFO("%s:%s idx=%u, bssid("MAC_FMT"),"
+			" bss_info(0x%04X), reg_class(0x%02X),"
+			" ch(%d), phy_type(0x%02X), preference(0x%02X)\n",
+			__func__, (found)?"update":"new", idx,
+			MAC_ARG(puser_cant->nb_rpt.bssid),
+			puser_cant->nb_rpt.bss_info,
+			puser_cant->nb_rpt.reg_class,
+			puser_cant->nb_rpt.ch_num,
+			puser_cant->nb_rpt.phy_type,
+			puser_cant->preference);
+
+	} /* end of if (pbtm_cant) */
+}
+
+static void rtw_mbo_reset_user_btm_req_preference(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	struct mbo_user_btm_req_pkt *puser = &(pmbo_attr->user_raw);
+	struct wnm_btm_cant *puser_cant = NULL;
+	u8 i;
+
+	for (i = 0; i < RTW_MAX_NB_RPT_NUM; i++) {
+		puser_cant = &puser->btm_cants[i];
+		if (_rtw_memcmp(adapter_mac_addr(padapter),
+			puser_cant->nb_rpt.bssid, ETH_ALEN)) {
+			puser_cant->preference = 0;
+			RTW_MBO_INFO("%s : reset "MAC_FMT" BTM preference\n",
+				__func__, MAC_ARG(puser_cant->nb_rpt.bssid));
+			break;
+		}
+	}
+
+}
+
+ssize_t rtw_mbo_proc_attr_set(
+	struct file *pfile, const char __user *buffer,
+	size_t count, loff_t *pos, void *pdata)
+{
+	struct net_device *dev = pdata;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	struct mbo_user_btm_req_pkt *puser = &(pmbo_attr->user_raw);
+	u32 id, val;
+	u8 tmp[64] = {0};
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp))
+		return -EFAULT;
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%d %d", &id, &val);
+		switch (id) {
+			case RTW_MBO_ATTR_AP_CAP_ID:
+				pmbo_attr->ap_cap_ind = val;
+				break;
+			case RTW_MBO_ATTR_CELL_DATA_CAP_ID:
+				pmbo_attr->cell_data_cap = val;
+				break;
+			case RTW_MBO_ATTR_ASSOC_DISABLED_ID:
+				pmbo_attr->assoc_disallow = val;
+				break;
+			case RTW_MBO_ATTR_TRANS_RES_ID:
+				pmbo_attr->reason = val;
+				break;
+			case RTW_MBO_ATTR_ASSOC_RETRY_DELAY_ID:
+				pmbo_attr->delay = val;
+				break;
+			case RTW_MBO_TEST_CMD_REST:
+				RTW_INFO("%s : RTW_MBO_TEST_CMD_REST\n",
+					__func__);
+				_rtw_memset(pmbo_attr, 0,
+					sizeof(struct mbo_attr_info));
+				pmbo_attr->mbo_spec_test = 1;
+				break;
+			case RTW_MBO_TEST_CMD_BTM_REQ_SET:
+				if (count >= 10) {
+					struct btm_req_hdr btm_hdr;
+					u32 disassoc_imnt, term_bit;
+					u32 term_tsf, term_duration;
+
+					_rtw_memset(&btm_hdr, 0,
+							sizeof(btm_hdr));
+
+					num = sscanf(tmp, "%d %u %u %u %u",
+						&id, &disassoc_imnt,
+						&term_bit, &term_tsf,
+						&term_duration);
+
+					if (num < 5)
+						break;
+
+					if (disassoc_imnt > 0)
+						btm_hdr.req_mode |= \
+							DISASSOC_IMMINENT;
+
+					if (term_bit > 0)
+						btm_hdr.req_mode |= \
+						BSS_TERMINATION_INCLUDED;
+
+					btm_hdr.term_duration.tsf = term_tsf;
+					btm_hdr.term_duration.duration = \
+							term_duration;
+
+					rtw_mbo_construct_user_btm_req(
+						padapter, &btm_hdr, NULL, 0,
+						NULL);
+				}
+				break;
+			case RTW_MBO_TEST_CMD_BTM_REQ_SEND:
+				if (count >= 12) {
+					u8 mac_str[18] = {0};
+					u8 dst_mac[ETH_ALEN] = {0};
+					u32 cand_list = 0, disassoc_timer = 0;
+
+					num = sscanf(tmp, "%d %s %u %u",
+						&id, mac_str, &cand_list,
+						&disassoc_timer);
+
+					if (num < 4)
+						break;
+
+					if (sscanf(mac_str, MAC_SFMT,
+						MAC_SARG(
+						dst_mac)) != 6) {
+						break;
+					}
+
+					puser->append_mbo_ie = _TRUE;
+					puser->hdr.dialog_token++;
+					puser->hdr.validity_interval = 0xf;
+					if (cand_list > 0)
+						puser->hdr.req_mode |= \
+						PREFERRED_CANDIDATE_LIST_INCLUDED;
+
+
+					puser->hdr.disassoc_timer = \
+						disassoc_timer;
+
+					if ((puser->hdr.req_mode & \
+						DISASSOC_IMMINENT) == \
+						DISASSOC_IMMINENT) {
+						rtw_mbo_reset_user_btm_req_preference(padapter);
+						if (pmbo_attr->delay == 0)
+							pmbo_attr->delay = 1;
+						if (puser->hdr.disassoc_timer == 0)
+							puser->hdr.disassoc_timer = 1000;
+					}
+
+					if ((puser->hdr.req_mode & \
+						BSS_TERMINATION_INCLUDED) == \
+						BSS_TERMINATION_INCLUDED) {
+						 puser->append_mbo_ie = _FALSE;
+					}
+
+					if (!puser->candidate_cnt) {
+						struct wnm_btm_cant cant;
+						_rtw_memset(&cant, 0,
+								sizeof(cant));
+						_rtw_memcpy(cant.nb_rpt.bssid,
+							adapter_mac_addr(padapter),
+							ETH_ALEN);
+						cant.nb_rpt.reg_class = 115;
+						cant.nb_rpt.ch_num = 36;
+						cant.preference = 0;
+						rtw_mbo_construct_user_btm_req(
+							padapter, NULL,
+							NULL, 0, &cant);
+					}
+
+					rtw_wnm_issue_btm_req(padapter, dst_mac,
+						&puser->hdr, NULL, 0,
+						(u8 *)&puser->btm_cants,
+						puser->candidate_cnt);
+
+					if ((puser->hdr.term_duration.duration)
+						&& (puser->hdr.req_mode & \
+						BSS_TERMINATION_INCLUDED) == \
+						BSS_TERMINATION_INCLUDED) {
+						rtw_mbo_disassoc(padapter,
+							dst_mac, 3, _TRUE);
+					}
+
+					if ((puser->hdr.req_mode & \
+						DISASSOC_IMMINENT) == \
+						DISASSOC_IMMINENT) {
+						rtw_mbo_disassoc(padapter,
+							dst_mac, 3, _TRUE);
+					}
+				} /* end of if (count >= 12) */
+
+				break;
+			case RTW_MBO_TEST_CMD_NB_BSS_ADD:
+				if (count >= 12) {
+					u8 bss_str[18];
+					struct wnm_btm_cant cant;
+					u32 op, op_ch, perf;
+
+					num = sscanf(tmp, "%d %s %u %u %u",
+						&id, bss_str, &op, &op_ch,
+						&perf);
+					if (num < 5)
+						break;
+					_rtw_memset(&cant, 0, sizeof(cant));
+					if (sscanf(bss_str, MAC_SFMT,
+						MAC_SARG(
+						cant.nb_rpt.bssid)) != 6) {
+						break;
+					}
+
+					cant.nb_rpt.reg_class = op;
+					cant.nb_rpt.ch_num = op_ch;
+					cant.preference = perf;
+					rtw_mbo_construct_user_btm_req(
+						padapter, NULL, NULL, 0,
+						&cant);
+				} /* end of if (count >= 12) */
+				break;
+			default:
+				break;
+		}
+	}
+
+	return count;
+}
+
+int rtw_mbo_proc_attr_get(
+	struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+
+	RTW_PRINT_SEL(m, "AP Capability Indication : %d\n",
+				pmbo_attr->ap_cap_ind);
+	RTW_PRINT_SEL(m, "Cellular Data Capabilities : %d\n",
+				pmbo_attr->cell_data_cap);
+	RTW_PRINT_SEL(m, "Association Disallowed : %d\n",
+				pmbo_attr->assoc_disallow);
+	return 0;
+}
+
+static void rtw_mbo_non_pref_chan_subelem_parsing(
+	_adapter *padapter, u8 *subelem, size_t subelem_len)
+{
+	u8 *pnon_pref_chans;
+	u32 non_pref_chan_offset, op_subelem_len;
+	u32 oui_offset = 3;
+	/* wpa_supplicant don't apped OUI Type */
+	u32 oui_type_offset = 0;
+
+	RTW_MBO_DUMP("Non-preferred Channel subelem : ",
+			subelem , subelem_len);
+
+	/* Subelem :
+		Vendor Specific | Length | WFA OUI | OUI Type | MBO Attributes */
+	non_pref_chan_offset = 2 + oui_offset + oui_type_offset;
+	pnon_pref_chans = subelem + non_pref_chan_offset;
+	op_subelem_len = subelem_len - non_pref_chan_offset;
+
+	/* wpa_supplicant don't indicate non_pref_chan length,
+		so we cannot get how many non_pref_chan in a wnm notification */
+	RTW_MBO_DUMP("Non-preferred Channel : ",
+			pnon_pref_chans, op_subelem_len);
+}
+
+void rtw_mbo_wnm_notification_parsing(
+	_adapter *padapter, const u8 *pdata, size_t data_len)
+{
+	u8 *paction;
+	u8 category, action, dialog, type;
+	u32 len;
+
+	if ((pdata == NULL) || (data_len == 0))
+		return;
+
+	RTW_MBO_DUMP("WNM notification data : ", pdata, data_len);
+	paction = (u8 *)pdata + sizeof(struct rtw_ieee80211_hdr_3addr);
+	category = paction[0];
+	action = paction[1];
+	dialog = paction[2];
+	type = paction[3];
+
+	if ((action == RTW_WLAN_ACTION_WNM_NOTIF_REQ) &&
+		(type == WLAN_EID_VENDOR_SPECIFIC)) {
+		rtw_mbo_non_pref_chan_subelem_parsing(padapter, &paction[4],
+			(data_len - sizeof(struct rtw_ieee80211_hdr_3addr)));
+	}
+}
+
+void rtw_mbo_build_wnm_notification(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	struct rf_ctl_t *prfctl = adapter_to_rfctl(padapter);
+	struct npref_ch_rtp *prpt = &(prfctl->ch_rtp);
+	struct npref_ch* pch;
+	u8 subelem_id = WLAN_EID_VENDOR_SPECIFIC;
+	u8 non_pref_ch_oui[] = {0x50, 0x6F, 0x9A, 0x2};
+	u8 cell_data_cap_oui[] = {0x50, 0x6F, 0x9A, 0x3};
+	u8 cell_data_con = rtw_mbo_cell_data_conn;
+	u8 len, cell_data_con_len = 0, *pcont = *pframe;
+	int i;
+
+	if (rtw_mbo_cell_data_conn > 0) {
+		len = 0x5;
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&subelem_id, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&len, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_4byte_ie(*pframe,
+				cell_data_cap_oui, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&cell_data_con, &(pattrib->pktlen));
+		RTW_MBO_INFO("%s : Cellular Data Capabilities subelemen\n",
+				__func__);
+		RTW_MBO_DUMP(":", pcont, len + 2);
+		pcont += len + 2 ;
+	}
+
+	if (prpt->nm_of_rpt == 0) {
+		len = 0x4;
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&subelem_id, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&len, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_4byte_ie(*pframe,
+				non_pref_ch_oui, &(pattrib->pktlen));
+		RTW_MBO_INFO("%s :Non-preferred Channel Report subelement"
+				" without data\n", __func__);
+		return;
+	}
+
+	for (i=0; i < prpt->nm_of_rpt; i++) {
+		pch = &prpt->ch_rpt[i];
+		/* OUI(3B)  + OUT-type(1B) + op-class(1B) + ch list(nB)
+			+ Preference(1B) + reason(1B) */
+		len = pch->nm_of_ch + 7;
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&subelem_id, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&len, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_4byte_ie(*pframe,
+				non_pref_ch_oui, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&pch->op_class, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_nbyte_ie(*pframe,
+				pch->nm_of_ch, pch->chs, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&pch->preference, &(pattrib->pktlen));
+		*pframe = rtw_mbo_set_1byte_ie(*pframe,
+				&pch->reason, &(pattrib->pktlen));
+		RTW_MBO_INFO("%s :Non-preferred Channel Report"
+				" subelement\n", __func__);
+		RTW_MBO_DUMP(":", pcont, len);
+		pcont = *pframe;
+	}
+}
+
+void rtw_mbo_build_probe_req_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	u32 len =0;
+
+	if (padapter->dis_rt_cap & BIT(DIS_MBO))
+		return;
+
+	rtw_mbo_build_extended_cap(padapter, pframe, pattrib);
+
+	len = rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_CELL_DATA_CAP_ID);
+	if ((len == 0) || (len > 3)) {
+		RTW_ERR("MBO : build Cellular Data Capabilities"
+			" attribute fail(len=%u)\n", len);
+		return;
+	}
+
+	rtw_mbo_build_mbo_ie_hdr(pframe, pattrib, len);
+	rtw_mbo_build_cell_data_cap_attr(padapter, pframe, pattrib);
+}
+
+void rtw_mbo_build_assoc_req_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	u32 len = 0;
+
+	len += rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_CELL_DATA_CAP_ID);
+	len += rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_NPREF_CH_RPT_ID);
+	if ((len == 0)|| (len < 3)) {
+		RTW_ERR("MBO : build assoc MBO IE fail(len=%u)\n", len);
+		return;
+	}
+
+	rtw_mbo_build_mbo_ie_hdr(pframe, pattrib, len);
+	rtw_mbo_build_cell_data_cap_attr(padapter, pframe, pattrib);
+	rtw_mbo_build_npref_ch_rpt_attr(padapter, pframe, pattrib);
+}
+
+static void rtw_mbo_build_ap_capability(
+	_adapter *padapter, u8 **pframe,
+	struct pkt_attrib *pattrib)
+{
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *cur_network = &(pmlmeinfo->network);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	u8 *pcap = NULL;
+	u32 cap_len = 0, flen = 0;
+
+	if ((pcap = rtw_mbo_attrs_get(
+		(cur_network->IEs + _FIXED_IE_LENGTH_),
+		(cur_network->IELength - _FIXED_IE_LENGTH_),
+		RTW_MBO_ATTR_AP_CAP_ID, &cap_len, 0)) != NULL)
+		return;
+
+	flen += rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_AP_CAP_ID);
+	if (pmbo_attr->assoc_disallow > 0)
+		flen += rtw_mbo_attr_sz_get(padapter,
+				RTW_MBO_ATTR_ASSOC_DISABLED_ID);
+	if (flen > 0) {
+		rtw_mbo_build_mbo_ie_hdr(pframe, pattrib, flen);
+		rtw_mbo_build_ap_cap_Indication_attr(padapter, pframe,
+				pattrib, pmbo_attr->ap_cap_ind);
+
+		if (pmbo_attr->assoc_disallow > 0) {
+			rtw_mbo_build_ap_disallowed_attr(padapter, pframe,
+				pattrib, pmbo_attr->assoc_disallow);
+		}
+	} /*  end of if (flen > 0) */
+
+}
+
+void rtw_mbo_attr_info_init(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+
+	_rtw_memset(pmbo_attr, 0, sizeof(struct mbo_attr_info));
+}
+
+void rtw_mbo_process_assoc_req(
+	_adapter *padapter, u8 *pie, int ie_len)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	u8 non_pref_ch = 0, ch_op_pref = 1;
+	u8 *pattr = NULL, *ptr = NULL;
+	u32 attr_len = 0, ch_nums = 0;
+	int i, j;
+
+	if (!pie || !ie_len)
+		return;
+
+	if ((pattr = rtw_mbo_attrs_get(pie, ie_len,
+		RTW_MBO_ATTR_NPREF_CH_RPT_ID, &attr_len, 1)) == NULL)
+		return;
+
+	if (attr_len < 3)
+		return;
+
+	ch_nums = (attr_len - 3);
+
+	/* shfit to non-preferred ch rpt field */
+	ptr = pattr + 3;
+	RTW_MBO_DUMP("non-preferred ch rpt :", ptr, ch_nums);
+
+	ch_op_pref = *(ptr + ch_nums);
+	RTW_MBO_INFO("%s : ch_op_pref=0x%02x\n", __func__, ch_op_pref);
+
+	if (ch_op_pref >= 2) {
+		RTW_MBO_INFO("%s : unknow ch operating preference(0x%02x)\n",
+			__func__, ch_op_pref);
+		return;
+	}
+
+	for (i = 0; i < ch_nums; i++) {
+		if (i >= RTW_MBO_MAX_CH_LIST_NUM)
+			break;
+		non_pref_ch = *(ptr + i);
+		RTW_MBO_INFO("%s : non-pref ch %u found in assoc-req\n",
+			__func__, non_pref_ch);
+
+		if (rtw_mbo_wifi_spec_test(padapter)) {
+			struct mbo_user_btm_req_pkt *puser_raw = \
+				&(pmbo_attr->user_raw);
+			struct wnm_btm_cant *pcant = NULL;
+
+			for (j = 0; j < RTW_MAX_NB_RPT_NUM; j++) {
+				pcant = &puser_raw->btm_cants[j];
+				if ((pcant->nb_rpt.ch_num == non_pref_ch) && \
+					(ch_op_pref == 0)) {
+					RTW_MBO_INFO("%s : reset "
+					"preference(%u) for non-preference ch\n",
+					__func__, pcant->preference);
+					pcant->preference = 0;
+				}
+			}
+		}
+
+	} /* end of for-loop ch_nums */
+
+}
+
+void rtw_mbo_build_beacon_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	rtw_mbo_build_ap_capability(padapter, pframe, pattrib);
+}
+
+void rtw_mbo_build_probe_rsp_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	rtw_mbo_build_ap_capability(padapter, pframe, pattrib);
+}
+
+void rtw_mbo_build_assoc_rsp_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *cur_network = &(pmlmeinfo->network);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	u8 *pcap = NULL;
+	u32 len = 0, cap_len = 0 ;
+
+	#if 1
+	if ((pcap = rtw_mbo_attrs_get(
+		(cur_network->IEs + _FIXED_IE_LENGTH_),
+		(cur_network->IELength - _FIXED_IE_LENGTH_),
+		RTW_MBO_ATTR_AP_CAP_ID, &cap_len, 0)) != NULL)
+		return;
+	#endif
+
+	len += rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_AP_CAP_ID);
+	if (pmbo_attr->assoc_disallow > 0)
+		len += rtw_mbo_attr_sz_get(padapter,
+			RTW_MBO_ATTR_ASSOC_DISABLED_ID);
+
+	if (len > 0) {
+		rtw_mbo_build_mbo_ie_hdr(pframe, pattrib, len);
+		rtw_mbo_build_ap_cap_Indication_attr(
+			padapter, pframe, pattrib, pmbo_attr->ap_cap_ind);
+		if (pmbo_attr->assoc_disallow > 0) {
+			rtw_mbo_build_ap_disallowed_attr(padapter, pframe,
+				pattrib, pmbo_attr->assoc_disallow);
+		}
+	}
+
+}
+
+void rtw_mbo_build_wnm_btmreq_reason_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib)
+{
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *cur_network = &(pmlmeinfo->network);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	struct mbo_user_btm_req_pkt *puser = &(pmbo_attr->user_raw);
+	u8 *pcap = NULL;
+	u32 len = 0, cap_len = 0 ;
+
+	if (!puser->append_mbo_ie)
+		return;
+
+	len += rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_AP_CAP_ID);
+	len += rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_TRANS_RES_ID);
+	len += rtw_mbo_attr_sz_get(padapter, RTW_MBO_ATTR_ASSOC_RETRY_DELAY_ID);
+
+	rtw_mbo_build_mbo_ie_hdr(pframe, pattrib, len);
+
+	rtw_mbo_build_ap_cap_Indication_attr(
+			padapter, pframe, pattrib, pmbo_attr->ap_cap_ind);
+
+	rtw_mbo_build_ap_trans_reason_attr(padapter, pframe,
+			pattrib, pmbo_attr->reason);
+
+	rtw_mbo_build_ap_assoc_retry_delay_attr(padapter, pframe,
+			pattrib, pmbo_attr->delay);
+}
+#endif /* CONFIG_RTW_MBO */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mem.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mem.c
new file mode 100644
index 000000000000..1b35bb6b3819
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mem.c
@@ -0,0 +1,40 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <linux/kernel.h>
+#include <drv_types.h>
+#include <rtw_mem.h>
+
+extern int rtkm_prealloc_init(void);
+extern void rtkm_prealloc_destroy(void);
+
+static int __init rtw_mem_init(void)
+{
+	rtkm_prealloc_init();
+	return 0;
+}
+
+static void __exit rtw_mem_exit(void)
+{
+	rtkm_prealloc_destroy();
+}
+
+module_init(rtw_mem_init);
+module_exit(rtw_mem_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Realtek Wireless Lan Driver");
+MODULE_AUTHOR("Realtek Semiconductor Corp.");
+MODULE_VERSION("DRIVERVERSION");
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mi.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mi.c
new file mode 100644
index 000000000000..49b1dd2e1591
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mi.c
@@ -0,0 +1,1483 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_MI_C_
+
+#include <drv_types.h>
+
+/* Find union about ch, bw, ch_offset of all linked/linking interfaces */
+static int _rtw_mi_get_ch_setting_union(struct dvobj_priv *dvobj, _adapter *adapter, u8 band_idx, u8 ifbmp, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	_adapter *iface;
+	struct _ADAPTER_LINK *iface_link;
+	struct link_mlme_ext_priv *mlmeext;
+	int i;
+	enum band_type band_ret = BAND_MAX;
+	u8 ch_ret = 0;
+	u8 bw_ret = CHANNEL_WIDTH_20;
+	u8 offset_ret = CHAN_OFFSET_NO_EXT;
+	int num = 0;
+
+	if (band)
+		*band = 0;
+	if (ch)
+		*ch = 0;
+	if (bw)
+		*bw = CHANNEL_WIDTH_20;
+	if (offset)
+		*offset = CHAN_OFFSET_NO_EXT;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !(ifbmp & BIT(iface->iface_id)))
+			continue;
+
+		if (adapter) {
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+		} else if (band_idx < HW_BAND_MAX){
+			if (!rtw_iface_is_operate_at_hwband(iface, band_idx))
+				continue;
+		}
+
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		iface_link = GET_PRIMARY_LINK(iface);
+		mlmeext = &iface_link->mlmeextpriv;
+
+		if (!check_fwstate(&iface->mlmepriv, WIFI_ASOC_STATE | WIFI_UNDER_LINKING))
+			continue;
+
+		if (check_fwstate(&iface->mlmepriv, WIFI_OP_CH_SWITCHING))
+			continue;
+
+		if (num == 0) {
+			band_ret = mlmeext->chandef.band;
+			ch_ret = mlmeext->chandef.chan;
+			bw_ret = mlmeext->chandef.bw;
+			offset_ret = mlmeext->chandef.offset;
+			num++;
+			continue;
+		}
+
+		if (band_ret != mlmeext->chandef.band || ch_ret != mlmeext->chandef.chan) {
+			num = 0;
+			break;
+		}
+
+		if (bw_ret < mlmeext->chandef.bw) {
+			bw_ret = mlmeext->chandef.bw;
+			offset_ret = mlmeext->chandef.offset;
+		} else if (bw_ret == mlmeext->chandef.bw && offset_ret != mlmeext->chandef.offset) {
+			num = 0;
+			break;
+		}
+
+		num++;
+	}
+
+	if (num) {
+		if (band)
+			*band = band_ret;
+		if (ch)
+			*ch = ch_ret;
+		if (bw)
+			*bw = bw_ret;
+		if (offset)
+			*offset = offset_ret;
+	}
+
+	return num;
+}
+
+int rtw_mi_get_ch_setting_union_by_ifbmp(_adapter *adapter, u8 ifbmp, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	return _rtw_mi_get_ch_setting_union(adapter_to_dvobj(adapter), adapter, HW_BAND_MAX, ifbmp, band, ch, bw, offset);
+}
+
+inline int rtw_mi_get_ch_setting_union(_adapter *adapter, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	return rtw_mi_get_ch_setting_union_by_ifbmp(adapter, 0xFF, band, ch, bw, offset);
+}
+
+inline int rtw_mi_get_ch_setting_union_no_self(_adapter *adapter, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	return rtw_mi_get_ch_setting_union_by_ifbmp(adapter, 0xFF & ~BIT(adapter->iface_id), band, ch, bw, offset);
+}
+
+int rtw_mi_get_bch_setting_union_by_hwband(struct dvobj_priv *dvobj, u8 band_idx
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	return _rtw_mi_get_ch_setting_union(dvobj, NULL, band_idx, 0xFF, band, ch, bw, offset);
+}
+
+int rtw_mi_get_bch_setting_union_by_hwband_ifbmp(struct dvobj_priv *dvobj, u8 band_idx, u8 ifbmp
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	return _rtw_mi_get_ch_setting_union(dvobj, NULL, band_idx, ifbmp, band, ch, bw, offset);
+}
+
+static void _rtw_mi_status(struct dvobj_priv *dvobj, _adapter *adapter, u8 band_idx, u8 ifbmp, struct mi_state *mstate)
+{
+	_adapter *iface;
+	int i;
+
+	_rtw_memset(mstate, 0, sizeof(struct mi_state));
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !(ifbmp & BIT(iface->iface_id)))
+			continue;
+
+		if (adapter) {
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+		} else if (band_idx < HW_BAND_MAX){
+			if (!rtw_iface_is_operate_at_hwband(iface, band_idx))
+				continue;
+		}
+
+		if (MLME_IS_STA(iface)) {
+			MSTATE_STA_NUM(mstate)++;
+			if (check_fwstate(&iface->mlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+				MSTATE_STA_LD_NUM(mstate)++;
+
+				#ifdef CONFIG_TDLS
+				if (iface->tdlsinfo.link_established == _TRUE)
+					MSTATE_TDLS_LD_NUM(mstate)++;
+				#endif
+				#ifdef CONFIG_P2P
+				if (MLME_IS_GC(iface))
+					MSTATE_P2P_GC_NUM(mstate)++;
+				#endif
+			}
+			if (check_fwstate(&iface->mlmepriv, WIFI_UNDER_LINKING) == _TRUE)
+				MSTATE_STA_LG_NUM(mstate)++;
+
+#ifdef CONFIG_AP_MODE
+		} else if (check_fwstate(&iface->mlmepriv, WIFI_AP_STATE) == _TRUE ) {
+			if (check_fwstate(&iface->mlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+				MSTATE_AP_NUM(mstate)++;
+				if (iface->stapriv.asoc_sta_count >= 2)
+					MSTATE_AP_LD_NUM(mstate)++;
+				#ifdef CONFIG_P2P
+				if (MLME_IS_GO(iface))
+					MSTATE_P2P_GO_NUM(mstate)++;
+				#endif
+			} else
+				MSTATE_AP_STARTING_NUM(mstate)++;
+#endif
+
+		} else if (check_fwstate(&iface->mlmepriv, WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE) == _TRUE
+			&& check_fwstate(&iface->mlmepriv, WIFI_ASOC_STATE) == _TRUE
+		) {
+			MSTATE_ADHOC_NUM(mstate)++;
+			if (iface->stapriv.asoc_sta_count > 2)
+				MSTATE_ADHOC_LD_NUM(mstate)++;
+
+#ifdef CONFIG_RTW_MESH
+		} else if (check_fwstate(&iface->mlmepriv, WIFI_MESH_STATE) == _TRUE
+			&& check_fwstate(&iface->mlmepriv, WIFI_ASOC_STATE) == _TRUE
+		) {
+			MSTATE_MESH_NUM(mstate)++;
+			if (iface->stapriv.asoc_sta_count > 2)
+				MSTATE_MESH_LD_NUM(mstate)++;
+#endif
+
+		}
+
+		if (check_fwstate(&iface->mlmepriv, WIFI_UNDER_WPS) == _TRUE)
+			MSTATE_WPS_NUM(mstate)++;
+
+		if (check_fwstate(&iface->mlmepriv, WIFI_UNDER_SURVEY) == _TRUE) {
+			MSTATE_SCAN_NUM(mstate)++;
+
+			if (mlmeext_scan_state(&iface->mlmeextpriv) != SCAN_DISABLE
+				&& mlmeext_scan_state(&iface->mlmeextpriv) != SCAN_BACK_OP)
+				MSTATE_SCAN_ENTER_NUM(mstate)++;
+		}
+
+#ifdef CONFIG_IOCTL_CFG80211
+		if (rtw_cfg80211_get_is_mgmt_tx(iface))
+			MSTATE_MGMT_TX_NUM(mstate)++;
+		#ifdef CONFIG_P2P
+		if (rtw_cfg80211_get_is_roch(iface) == _TRUE)
+			MSTATE_ROCH_NUM(mstate)++;
+		#endif
+#endif /* CONFIG_IOCTL_CFG80211 */
+#ifdef CONFIG_P2P
+		if (MLME_IS_PD(iface))
+			MSTATE_P2P_DV_NUM(mstate)++;
+#endif
+	}
+}
+
+void rtw_mi_status_by_ifbmp(_adapter *adapter, u8 ifbmp, struct mi_state *mstate)
+{
+	_rtw_mi_status(adapter_to_dvobj(adapter), adapter, HW_BAND_MAX, ifbmp, mstate);
+}
+
+inline void rtw_mi_status(_adapter *adapter, struct mi_state *mstate)
+{
+	rtw_mi_status_by_ifbmp(adapter, 0xFF, mstate);
+}
+
+inline void rtw_mi_status_no_self(_adapter *adapter, struct mi_state *mstate)
+{
+	rtw_mi_status_by_ifbmp(adapter, 0xFF & ~BIT(adapter->iface_id), mstate);
+}
+
+inline void rtw_mi_status_no_others(_adapter *adapter, struct mi_state *mstate)
+{
+	rtw_mi_status_by_ifbmp(adapter, BIT(adapter->iface_id), mstate);
+}
+
+void rtw_mi_status_by_hwband(struct dvobj_priv *dvobj, u8 band_idx, struct mi_state *mstate)
+{
+	_rtw_mi_status(dvobj, NULL, band_idx, 0xFF, mstate);
+}
+
+void rtw_mi_status_by_hwband_ifbmp(struct dvobj_priv *dvobj, u8 band_idx, u8 ifbmp, struct mi_state *mstate)
+{
+	_rtw_mi_status(dvobj, NULL, band_idx, ifbmp, mstate);
+}
+
+/* For now, not handle union_ch/bw/offset */
+inline void rtw_mi_status_merge(struct mi_state *d, struct mi_state *a)
+{
+	d->sta_num += a->sta_num;
+	d->ld_sta_num += a->ld_sta_num;
+	d->lg_sta_num += a->lg_sta_num;
+#ifdef CONFIG_TDLS
+	d->ld_tdls_num += a->ld_tdls_num;
+#endif
+#ifdef CONFIG_AP_MODE
+	d->ap_num += a->ap_num;
+	d->ld_ap_num += a->ld_ap_num;
+#endif
+	d->adhoc_num += a->adhoc_num;
+	d->ld_adhoc_num += a->ld_adhoc_num;
+#ifdef CONFIG_RTW_MESH
+	d->mesh_num += a->mesh_num;
+	d->ld_mesh_num += a->ld_mesh_num;
+#endif
+	d->scan_num += a->scan_num;
+	d->scan_enter_num += a->scan_enter_num;
+	d->uwps_num += a->uwps_num;
+#ifdef CONFIG_IOCTL_CFG80211
+	#ifdef CONFIG_P2P
+	d->roch_num += a->roch_num;
+	#endif
+	d->mgmt_tx_num += a->mgmt_tx_num;
+#endif
+}
+
+void dump_mi_status(void *sel, struct dvobj_priv *dvobj)
+{
+	u8 i;
+	struct mi_state *iface_state;
+
+	RTW_PRINT_SEL(sel, "== dvobj-iface_state ==\n");
+
+	for (i = 0; i < MAX_BAND_NUM; i++) {
+		RTW_PRINT_SEL(sel, "== Band-%d iface_state ==\n", i);
+		iface_state = &dvobj->iface_state[i];
+		RTW_PRINT_SEL(sel, "sta_num:%d\n", MSTATE_STA_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "linking_sta_num:%d\n", MSTATE_STA_LG_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "linked_sta_num:%d\n", MSTATE_STA_LD_NUM(iface_state));
+#ifdef CONFIG_TDLS
+		RTW_PRINT_SEL(sel, "linked_tdls_num:%d\n", MSTATE_TDLS_LD_NUM(iface_state));
+#endif
+#ifdef CONFIG_AP_MODE
+		RTW_PRINT_SEL(sel, "ap_num:%d\n", MSTATE_AP_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "starting_ap_num:%d\n", MSTATE_AP_STARTING_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "linked_ap_num:%d\n", MSTATE_AP_LD_NUM(iface_state));
+#endif
+		RTW_PRINT_SEL(sel, "adhoc_num:%d\n", MSTATE_ADHOC_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "linked_adhoc_num:%d\n", MSTATE_ADHOC_LD_NUM(iface_state));
+#ifdef CONFIG_RTW_MESH
+		RTW_PRINT_SEL(sel, "mesh_num:%d\n", MSTATE_MESH_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "linked_mesh_num:%d\n", MSTATE_MESH_LD_NUM(iface_state));
+#endif
+#ifdef CONFIG_P2P
+		RTW_PRINT_SEL(sel, "p2p_device_num:%d\n", MSTATE_P2P_DV_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "p2p_gc_num:%d\n", MSTATE_P2P_GC_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "p2p_go_num:%d\n", MSTATE_P2P_GO_NUM(iface_state));
+#endif
+		RTW_PRINT_SEL(sel, "scan_num:%d\n", MSTATE_SCAN_NUM(iface_state));
+		RTW_PRINT_SEL(sel, "under_wps_num:%d\n", MSTATE_WPS_NUM(iface_state));
+#if defined(CONFIG_IOCTL_CFG80211)
+		#if defined(CONFIG_P2P)
+		RTW_PRINT_SEL(sel, "roch_num:%d\n", MSTATE_ROCH_NUM(iface_state));
+		#endif
+		RTW_PRINT_SEL(sel, "mgmt_tx_num:%d\n", MSTATE_MGMT_TX_NUM(iface_state));
+#endif
+	}
+	RTW_PRINT_SEL(sel, "================\n\n");
+}
+
+void dump_dvobj_mi_status(void *sel, const char *fun_name, _adapter *adapter)
+{
+	RTW_INFO("\n[ %s ] call %s\n", fun_name, __func__);
+	dump_mi_status(sel, adapter_to_dvobj(adapter));
+}
+
+inline void rtw_mi_update_iface_status(struct mlme_priv *pmlmepriv, sint state)
+{
+	_adapter *adapter = container_of(pmlmepriv, _adapter, mlmepriv);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct mi_state *iface_state = NULL;
+	struct mi_state tmp_mstate;
+	u8 hw_band_idx = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	if (state == WIFI_MONITOR_STATE
+		|| state == 0xFFFFFFFF
+	)
+		return;
+
+	if (0)
+		RTW_INFO("%s => will change or clean state to 0x%08x\n", __func__, state);
+
+	if (rtw_is_adapter_up(adapter))
+		hw_band_idx = adapter_link->wrlink->hw_band;
+
+	iface_state = &dvobj->iface_state[hw_band_idx];
+
+	rtw_mi_status(adapter, &tmp_mstate);
+	_rtw_memcpy(iface_state, &tmp_mstate, sizeof(struct mi_state));
+
+#ifdef DBG_IFACE_STATUS
+	DBG_IFACE_STATUS_DUMP(adapter);
+#endif
+}
+u8 rtw_mi_check_status(_adapter *adapter, u8 type)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct mi_state *iface_state;
+	u8 ret = _FALSE;
+	struct mr_query_info info = {0};
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	if (!rtw_is_adapter_up(adapter)) {
+		RTW_ERR("%s-"ADPT_FMT" !rtw_is_adapter_up\n", __func__, ADPT_ARG(adapter));
+		return ret;
+	}
+#ifdef DBG_IFACE_STATUS
+	DBG_IFACE_STATUS_DUMP(adapter);
+	RTW_INFO("%s-"ADPT_FMT" check type:%d\n", __func__, ADPT_ARG(adapter), type);
+#endif
+	iface_state = &dvobj->iface_state[adapter_link->wrlink->hw_band];
+	rtw_phl_mr_query_info(dvobj->phl, adapter->phl_role, adapter_link->wrlink, &info);
+#if 0
+	if (info.cur_info.ld_sta_num != MSTATE_STA_LD_NUM(iface_state))
+		RTW_ERR("ld_sta_num (%d:%d)\n", info.cur_info.ld_sta_num, MSTATE_STA_LD_NUM(iface_state));
+	if (info.cur_info.ap_num != MSTATE_AP_NUM(iface_state))
+		RTW_ERR("ap_num (%d:%d)\n", info.cur_info.ap_num, MSTATE_AP_NUM(iface_state));
+	if (info.cur_info.ld_ap_num != MSTATE_AP_LD_NUM(iface_state))
+		RTW_ERR("ld_ap_num (%d:%d)\n", info.cur_info.ld_ap_num, MSTATE_AP_LD_NUM(iface_state));
+#endif
+	switch (type) {
+	case MI_LINKED:
+		/*if (MSTATE_STA_LD_NUM(iface_state) || MSTATE_AP_NUM(iface_state) || MSTATE_ADHOC_NUM(iface_state) || MSTATE_MESH_NUM(iface_state))*/ /*check_fwstate(&iface->mlmepriv, WIFI_ASOC_STATE)*/
+		if (info.cur_info.ld_sta_num || info.cur_info.ap_num) /*info.cur_info.adhoc_num || info.cur_info.mesh_num*/
+			ret = _TRUE;
+		break;
+	case MI_ASSOC:
+		/*if (MSTATE_STA_LD_NUM(iface_state) || MSTATE_AP_LD_NUM(iface_state) || MSTATE_ADHOC_LD_NUM(iface_state) || MSTATE_MESH_LD_NUM(iface_state))*/
+		if (info.cur_info.ld_sta_num || info.cur_info.ld_ap_num) /*info.cur_info.ld_adhoc_num || info.cur_info.ld_mesh_num*/
+
+		ret = _TRUE;
+		break;
+	case MI_AP_MODE:
+		/*if (MSTATE_AP_NUM(iface_state))*/
+		if (info.cur_info.ap_num)
+			ret = _TRUE;
+		break;
+
+	case MI_AP_ASSOC:
+		/*if (MSTATE_AP_LD_NUM(iface_state))*/
+		if (info.cur_info.ld_ap_num)
+			ret = _TRUE;
+		break;
+	case MI_STA_LINKED:
+		/*if (MSTATE_STA_LD_NUM(iface_state))*/
+		if (info.cur_info.ld_sta_num)
+			ret = _TRUE;
+		break;
+	case MI_STA_LINKING:
+		/*if (MSTATE_STA_LG_NUM(iface_state))*/
+		if (info.cur_info.lg_sta_num)
+			ret = _TRUE;
+		break;
+#if 0
+	case MI_UNDER_WPS:
+		if (MSTATE_WPS_NUM(iface_state))
+			ret = _TRUE;
+		break;
+
+	case MI_ADHOC:
+		if (MSTATE_ADHOC_NUM(iface_state))
+			ret = _TRUE;
+		break;
+	case MI_ADHOC_ASSOC:
+		if (MSTATE_ADHOC_LD_NUM(iface_state))
+			ret = _TRUE;
+		break;
+
+	#ifdef CONFIG_RTW_MESH
+	case MI_MESH:
+		if (MSTATE_MESH_NUM(iface_state))
+			ret = _TRUE;
+		break;
+	case MI_MESH_ASSOC:
+		if (MSTATE_MESH_LD_NUM(iface_state))
+			ret = _TRUE;
+		break;
+	#endif
+
+	case MI_STA_NOLINK: /* this is misleading, but not used now */
+		if (MSTATE_STA_NUM(iface_state) && (!(MSTATE_STA_LD_NUM(iface_state) || MSTATE_STA_LG_NUM(iface_state))))
+			ret = _TRUE;
+		break;
+#endif
+	default:
+		break;
+	}
+	return ret;
+}
+
+/*
+* return value : 0 is failed or have not interface meet condition
+* return value : !0 is success or interface numbers which meet condition
+* return value of ops_func must be _TRUE or _FALSE
+*/
+static u8 _rtw_mi_process(_adapter *padapter, bool exclude_self, bool sb_chk,
+		  void *data, u8(*ops_func)(_adapter *padapter, void *data))
+{
+	int i;
+	_adapter *iface;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	u8 ret = 0;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && rtw_is_adapter_up(iface)) {
+
+			if ((exclude_self) && (iface == padapter))
+				continue;
+
+			if (sb_chk && !rtw_iface_at_same_hwband(padapter, iface))
+				continue;
+
+			if (ops_func)
+				if (_TRUE == ops_func(iface, data))
+					ret++;
+		}
+	}
+	return ret;
+}
+
+static u8 _rtw_mi_process_by_hwband_ifbmp(struct dvobj_priv *dvobj
+	, _adapter *adapter, u8 band_idx, u8 ifbmp
+	, void *data, u8(*ops_func)(_adapter *adapter, void *data))
+{
+	int i;
+	_adapter *iface;
+	u8 ret = 0;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !rtw_is_adapter_up(iface) || !(ifbmp & BIT(iface->iface_id)))
+			continue;
+
+		if (adapter) {
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+		} else if (band_idx < HW_BAND_MAX){
+			if (!rtw_iface_is_operate_at_hwband(iface, band_idx))
+				continue;
+		}
+
+		if (ops_func)
+			if (_TRUE == ops_func(iface, data))
+				ret++;
+	}
+	return ret;
+}
+
+static u8 _rtw_mi_process_without_schk(_adapter *padapter, bool exclude_self,
+		  void *data, u8(*ops_func)(_adapter *padapter, void *data))
+{
+	int i;
+	_adapter *iface;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	u8 ret = 0;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (iface) {
+			if ((exclude_self) && (iface == padapter))
+				continue;
+
+			if (ops_func)
+				if (ops_func(iface, data) == _TRUE)
+					ret++;
+		}
+	}
+	return ret;
+}
+
+static u8 _rtw_mi_netif_caroff_qstop(_adapter *padapter, void *data)
+{
+	struct net_device *pnetdev = padapter->pnetdev;
+
+	rtw_netif_carrier_off(pnetdev);
+	rtw_netif_stop_queue(pnetdev);
+	return _TRUE;
+}
+u8 rtw_mi_netif_caroff_qstop(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_netif_caroff_qstop);
+}
+u8 rtw_mi_buddy_netif_caroff_qstop(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_netif_caroff_qstop);
+}
+
+static u8 _rtw_mi_disconnect(_adapter *adapter, void *data)
+{
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_ext_priv *lmlmeext = &adapter_link->mlmeextpriv;
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *mlmeinfo = &(mlmeext->mlmext_info);
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+
+	if ((MLME_IS_AP(adapter) || MLME_IS_MESH(adapter))
+			&& check_fwstate(mlme, WIFI_ASOC_STATE)) {
+		rtw_phl_cmd_stop_beacon(adapter_to_dvobj(adapter)->phl, adapter_link->wrlink, _TRUE, PHL_CMD_NO_WAIT, 0);
+		rtw_sta_flush(adapter, _TRUE);
+
+		set_fwstate(mlme, WIFI_OP_CH_SWITCHING);
+		rtw_phl_chanctx_del(adapter_to_dvobj(adapter)->phl, adapter->phl_role, adapter_link->wrlink, NULL);
+		rtw_phl_chanctx_add(adapter_to_dvobj(adapter)->phl, adapter->phl_role, adapter_link->wrlink, ((struct disconnect_data *)data)->buddy_chdef, &mr_cc_info);
+
+	} else if (MLME_IS_STA(adapter)
+			&& check_fwstate(mlme, WIFI_ASOC_STATE)) {
+		Disconnect_type disc_code = DISCONNECTION_NOT_YET_OCCUR;
+
+		if (data)
+			disc_code = ((struct disconnect_data *)data)->disc_code;
+
+		rtw_disassoc_cmd(adapter, 500, RTW_CMDF_DIRECTLY);
+		rtw_indicate_disconnect(adapter, 0, _FALSE);
+
+		if (disc_code == DISCONNECTION_BY_DRIVER_DUE_TO_EACH_IFACE_CHBW_NOT_SYNC) {
+			mlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			mlmeinfo->disconnect_code = disc_code;
+			mlmeinfo->wifi_reason_code = WLAN_REASON_DEAUTH_LEAVING;
+		}
+	}
+	return _TRUE;
+}
+
+u8 rtw_mi_disconnect(_adapter *adapter)
+{
+	return _rtw_mi_process(adapter, _FALSE, _TRUE, NULL, _rtw_mi_disconnect);
+}
+u8 rtw_mi_buddy_disconnect(_adapter *adapter, struct disconnect_data *disc_data)
+{
+	return _rtw_mi_process(adapter, _TRUE, _TRUE, (void *)disc_data,
+			       _rtw_mi_disconnect);
+	/*_phl_mr_process_by_band(phl_info, wifi_role, _TRUE, NULL, _rtw_mi_disconnect);*/
+}
+
+#ifdef	PHL_MR_PROC_CMD
+static u8 _rtw_dump_mac_addr(void *drv_priv, u8 wr_idx, void *data)
+{
+	_adapter *adapter;
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+
+	adapter = dvobj->padapters[wr_idx];
+	if (rtw_is_adapter_up(adapter))
+		RTW_INFO("if%d mac_addr : "MAC_FMT"\n", wr_idx, MAC_ARG(adapter_mac_addr(adapter)));
+	return _SUCCESS;
+}
+
+u8 rtw_mi_dump_mac_addr(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	return rtw_phl_mr_process_by_band(dvobj->phl, HW_BAND_0, NULL,
+							_rtw_dump_mac_addr);
+}
+#endif
+
+static u8 _rtw_mi_netif_caron_qstart(_adapter *padapter, void *data)
+{
+	struct net_device *pnetdev = padapter->pnetdev;
+
+	rtw_netif_carrier_on(pnetdev);
+	rtw_netif_start_queue(pnetdev);
+	return _TRUE;
+}
+u8 rtw_mi_netif_caron_qstart(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_netif_caron_qstart);
+}
+u8 rtw_mi_buddy_netif_caron_qstart(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_netif_caron_qstart);
+}
+
+static u8 _rtw_mi_netif_stop_queue(_adapter *padapter, void *data)
+{
+	struct net_device *pnetdev = padapter->pnetdev;
+
+	rtw_netif_stop_queue(pnetdev);
+	return _TRUE;
+}
+u8 rtw_mi_netif_stop_queue(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_netif_stop_queue);
+}
+u8 rtw_mi_buddy_netif_stop_queue(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_netif_stop_queue);
+}
+
+static u8 _rtw_mi_netif_wake_queue(_adapter *padapter, void *data)
+{
+	struct net_device *pnetdev = padapter->pnetdev;
+
+	if (pnetdev)
+		rtw_netif_wake_queue(pnetdev);
+	return _TRUE;
+}
+u8 rtw_mi_netif_wake_queue(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_netif_wake_queue);
+}
+u8 rtw_mi_buddy_netif_wake_queue(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_netif_wake_queue);
+}
+
+static u8 _rtw_mi_netif_carrier_on(_adapter *padapter, void *data)
+{
+	struct net_device *pnetdev = padapter->pnetdev;
+
+	if (pnetdev)
+		rtw_netif_carrier_on(pnetdev);
+	return _TRUE;
+}
+u8 rtw_mi_netif_carrier_on(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_netif_carrier_on);
+}
+u8 rtw_mi_buddy_netif_carrier_on(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_netif_carrier_on);
+}
+
+static u8 _rtw_mi_netif_carrier_off(_adapter *padapter, void *data)
+{
+	struct net_device *pnetdev = padapter->pnetdev;
+
+	if (pnetdev)
+		rtw_netif_carrier_off(pnetdev);
+	return _TRUE;
+}
+u8 rtw_mi_netif_carrier_off(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_netif_carrier_off);
+}
+u8 rtw_mi_buddy_netif_carrier_off(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_netif_carrier_off);
+}
+
+static u8 _rtw_mi_scan_abort(_adapter *adapter, void *data)
+{
+	bool bwait = *(bool *)data;
+
+	if (bwait)
+		rtw_scan_abort(adapter, 0);
+	else
+		rtw_scan_abort_no_wait(adapter);
+
+	return _TRUE;
+}
+void rtw_mi_scan_abort(_adapter *adapter, bool bwait)
+{
+	bool in_data = bwait;
+
+	_rtw_mi_process(adapter, _FALSE, _FALSE, &in_data, _rtw_mi_scan_abort);
+
+}
+void rtw_mi_buddy_scan_abort(_adapter *adapter, bool bwait)
+{
+	bool in_data = bwait;
+
+	_rtw_mi_process(adapter, _TRUE, _FALSE, &in_data, _rtw_mi_scan_abort);
+}
+
+static u8 _rtw_mi_cancel_all_timer(_adapter *adapter, void *data)
+{
+	rtw_cancel_all_timer(adapter);
+	return _TRUE;
+}
+void rtw_mi_cancel_all_timer(_adapter *adapter)
+{
+	_rtw_mi_process(adapter, _FALSE, _FALSE, NULL, _rtw_mi_cancel_all_timer);
+}
+void rtw_mi_buddy_cancel_all_timer(_adapter *adapter)
+{
+	_rtw_mi_process(adapter, _TRUE, _FALSE, NULL, _rtw_mi_cancel_all_timer);
+}
+
+static u8 _rtw_mi_reset_drv_sw(_adapter *adapter, void *data)
+{
+	rtw_reset_drv_sw(adapter);
+	return _TRUE;
+}
+void rtw_mi_reset_drv_sw(_adapter *adapter)
+{
+	_rtw_mi_process_without_schk(adapter, _FALSE, NULL, _rtw_mi_reset_drv_sw);
+}
+void rtw_mi_buddy_reset_drv_sw(_adapter *adapter)
+{
+	_rtw_mi_process_without_schk(adapter, _TRUE, NULL, _rtw_mi_reset_drv_sw);
+}
+
+static u8 _rtw_mi_suspend_free_assoc_resource(_adapter *padapter, void *data)
+{
+	return rtw_suspend_free_assoc_resource(padapter);
+}
+void rtw_mi_suspend_free_assoc_resource(_adapter *adapter)
+{
+	_rtw_mi_process(adapter, _FALSE, _FALSE, NULL, _rtw_mi_suspend_free_assoc_resource);
+}
+void rtw_mi_buddy_suspend_free_assoc_resource(_adapter *adapter)
+{
+	_rtw_mi_process(adapter, _TRUE, _FALSE, NULL, _rtw_mi_suspend_free_assoc_resource);
+}
+
+static u8 _rtw_mi_is_scan_deny(_adapter *adapter, void *data)
+{
+	return rtw_is_scan_deny(adapter);
+}
+
+u8 rtw_mi_is_scan_deny(_adapter *adapter)
+{
+	return _rtw_mi_process(adapter, _FALSE, _FALSE, NULL, _rtw_mi_is_scan_deny);
+
+}
+u8 rtw_mi_buddy_is_scan_deny(_adapter *adapter)
+{
+	return _rtw_mi_process(adapter, _TRUE, _FALSE, NULL, _rtw_mi_is_scan_deny);
+}
+
+#ifdef CONFIG_SET_SCAN_DENY_TIMER
+static u8 _rtw_mi_set_scan_deny(_adapter *adapter, void *data)
+{
+	u32 ms = *(u32 *)data;
+
+	rtw_set_scan_deny(adapter, ms);
+	return _TRUE;
+}
+void rtw_mi_set_scan_deny(_adapter *adapter, u32 ms)
+{
+	u32 in_data = ms;
+
+	_rtw_mi_process(adapter, _FALSE, _FALSE, &in_data, _rtw_mi_set_scan_deny);
+}
+void rtw_mi_buddy_set_scan_deny(_adapter *adapter, u32 ms)
+{
+	u32 in_data = ms;
+
+	_rtw_mi_process(adapter, _TRUE, _FALSE, &in_data, _rtw_mi_set_scan_deny);
+}
+#endif /*CONFIG_SET_SCAN_DENY_TIMER*/
+
+u8 _rtw_mi_busy_traffic_check(_adapter *padapter, void *data)
+{
+	return padapter->mlmepriv.LinkDetectInfo.bBusyTraffic;
+}
+
+u8 rtw_mi_busy_traffic_check(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_busy_traffic_check);
+}
+u8 rtw_mi_buddy_busy_traffic_check(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_busy_traffic_check);
+}
+static u8 _rtw_mi_check_mlmeinfo_state(_adapter *padapter, void *data)
+{
+	u32 state = *(u32 *)data;
+	struct mlme_ext_priv *mlmeext = &padapter->mlmeextpriv;
+
+	/*if (mlmeext_msr(mlmeext) == state)*/
+	if (check_mlmeinfo_state(mlmeext, state))
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+u8 rtw_mi_check_mlmeinfo_state(_adapter *padapter, u32 state)
+{
+	u32 in_data = state;
+
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, &in_data, _rtw_mi_check_mlmeinfo_state);
+}
+
+u8 rtw_mi_buddy_check_mlmeinfo_state(_adapter *padapter, u32 state)
+{
+	u32 in_data = state;
+
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, &in_data, _rtw_mi_check_mlmeinfo_state);
+}
+
+/*#define DBG_DUMP_FW_STATE*/
+#ifdef DBG_DUMP_FW_STATE
+static void rtw_dbg_dump_fwstate(_adapter *padapter, sint state)
+{
+	u8 buf[32] = {0};
+
+	if (state & WIFI_FW_NULL_STATE) {
+		_rtw_memset(buf, 0, 32);
+		sprintf(buf, "WIFI_FW_NULL_STATE");
+		RTW_INFO(FUNC_ADPT_FMT"fwstate-%s\n", FUNC_ADPT_ARG(padapter), buf);
+	}
+
+	if (state & WIFI_ASOC_STATE) {
+		_rtw_memset(buf, 0, 32);
+		sprintf(buf, "WIFI_ASOC_STATE");
+		RTW_INFO(FUNC_ADPT_FMT"fwstate-%s\n", FUNC_ADPT_ARG(padapter), buf);
+	}
+
+	if (state & WIFI_UNDER_LINKING) {
+		_rtw_memset(buf, 0, 32);
+		sprintf(buf, "WIFI_UNDER_LINKING");
+		RTW_INFO(FUNC_ADPT_FMT"fwstate-%s\n", FUNC_ADPT_ARG(padapter), buf);
+	}
+
+	if (state & WIFI_UNDER_SURVEY) {
+		_rtw_memset(buf, 0, 32);
+		sprintf(buf, "WIFI_UNDER_SURVEY");
+		RTW_INFO(FUNC_ADPT_FMT"fwstate-%s\n", FUNC_ADPT_ARG(padapter), buf);
+	}
+}
+#endif
+
+static u8 _rtw_mi_check_fwstate(_adapter *padapter, void *data)
+{
+	u8 ret = _FALSE;
+
+	sint state = *(sint *)data;
+
+	if ((state == WIFI_FW_NULL_STATE) &&
+	    (padapter->mlmepriv.fw_state == WIFI_FW_NULL_STATE))
+		ret = _TRUE;
+	else if (_TRUE == check_fwstate(&padapter->mlmepriv, state))
+		ret = _TRUE;
+#ifdef DBG_DUMP_FW_STATE
+	if (ret)
+		rtw_dbg_dump_fwstate(padapter, state);
+#endif
+	return ret;
+}
+
+u8 rtw_mi_check_fwstate(_adapter *padapter, sint state)
+{
+	sint in_data = state;
+
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, &in_data, _rtw_mi_check_fwstate);
+}
+
+u8 rtw_mi_buddy_check_fwstate(_adapter *padapter, sint state)
+{
+	sint in_data = state;
+
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, &in_data, _rtw_mi_check_fwstate);
+}
+
+u8 rtw_mi_check_fwstate_by_hwband(struct dvobj_priv *dvobj, u8 band_idx, sint state)
+{
+	sint in_data = state;
+
+	return _rtw_mi_process_by_hwband_ifbmp(dvobj, NULL, band_idx, 0xFF, &in_data, _rtw_mi_check_fwstate);
+}
+
+static u8 _rtw_mi_traffic_statistics(_adapter *padapter , void *data)
+{
+	struct dvobj_priv	*pdvobjpriv = adapter_to_dvobj(padapter);
+
+	/* Tx */
+	pdvobjpriv->traffic_stat.tx_bytes += padapter->xmitpriv.tx_bytes;
+	pdvobjpriv->traffic_stat.tx_pkts += padapter->xmitpriv.tx_pkts;
+	pdvobjpriv->traffic_stat.tx_drop += padapter->xmitpriv.tx_drop;
+
+	/* Rx */
+	pdvobjpriv->traffic_stat.rx_bytes += padapter->recvinfo.rx_bytes;
+	pdvobjpriv->traffic_stat.rx_pkts += padapter->recvinfo.rx_pkts;
+	pdvobjpriv->traffic_stat.rx_drop += padapter->recvinfo.rx_drop;
+	return _TRUE;
+}
+u8 rtw_mi_traffic_statistics(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_traffic_statistics);
+}
+
+static u8 _rtw_mi_check_miracast_enabled(_adapter *padapter , void *data)
+{
+	return is_miracast_enabled(padapter);
+}
+u8 rtw_mi_check_miracast_enabled(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_check_miracast_enabled);
+}
+
+static u8 _rtw_mi_dynamic_check_handlder(struct _ADAPTER *a, void *data)
+{
+	rtw_iface_dynamic_check_handlder(a);
+	return _TRUE;
+}
+
+u8 rtw_mi_keep_alive_pre_chk_hdl(struct _ADAPTER *a)
+{
+	u8 ret = _SUCCESS;
+	int i;
+	_adapter *iface;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	u8 if_sta_chk_rx_bmp = 0;
+	u8 if_sta_chk_tx_bmp = 0;
+	u32 if_ap_chk_sta_bmp[MAX_WIFI_ROLE_NUMBER] = {0};
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && rtw_is_adapter_up(iface)) {
+
+			if (!rtw_iface_at_same_hwband(a, iface))
+				continue;
+
+			#ifdef CONFIG_AP_MODE
+			if (MLME_IS_AP(iface) || MLME_IS_MESH(iface)) {
+				expire_timeout_chk(iface, &if_ap_chk_sta_bmp[iface->iface_id]);
+				#ifdef CONFIG_RTW_MESH
+				if (MLME_IS_MESH(iface) && MLME_IS_ASOC(iface))
+					rtw_mesh_peer_status_chk(iface);
+				#endif
+			}
+			#endif
+			linked_status_chk(iface, &if_sta_chk_rx_bmp, &if_sta_chk_tx_bmp);
+		}
+	}
+#ifdef CONFIG_POST_CORE_KEEP_ALIVE
+	rtw_set_post_keep_alive_param(a, if_sta_chk_rx_bmp, if_sta_chk_tx_bmp, if_ap_chk_sta_bmp);
+#endif
+
+	return ret;
+}
+
+u8 rtw_mi_dynamic_check_handlder(struct _ADAPTER *a)
+{
+	return _rtw_mi_process(a, _FALSE, _FALSE, NULL, _rtw_mi_dynamic_check_handlder);
+}
+
+static u8 _rtw_mi_dynamic_chk_wk_hdl(_adapter *adapter, void *data)
+{
+	rtw_iface_dynamic_chk_wk_hdl(adapter);
+	return _TRUE;
+}
+u8 rtw_mi_dynamic_chk_wk_hdl(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_dynamic_chk_wk_hdl);
+}
+u8 rtw_mi_buddy_dynamic_chk_wk_hdl(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_dynamic_chk_wk_hdl);
+}
+
+static u8 _rtw_mi_os_xmit_schedule(_adapter *adapter, void *data)
+{
+	rtw_os_xmit_schedule(adapter);
+	return _TRUE;
+}
+u8 rtw_mi_os_xmit_schedule(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_os_xmit_schedule);
+}
+u8 rtw_mi_buddy_os_xmit_schedule(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_os_xmit_schedule);
+}
+
+static u8 _rtw_mi_report_survey_event(_adapter *adapter, void *data)
+{
+	union recv_frame *precv_frame = (union recv_frame *)data;
+
+	report_survey_event(adapter, precv_frame);
+	return _TRUE;
+}
+u8 rtw_mi_report_survey_event(_adapter *padapter, union recv_frame *precv_frame)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, precv_frame, _rtw_mi_report_survey_event);
+}
+u8 rtw_mi_buddy_report_survey_event(_adapter *padapter, union recv_frame *precv_frame)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, precv_frame, _rtw_mi_report_survey_event);
+}
+
+static u8 _rtw_mi_tx_beacon_hdl(_adapter *adapter, void *data)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	if ((MLME_IS_AP(adapter) || MLME_IS_MESH(adapter))
+		&& check_fwstate(&adapter->mlmepriv, WIFI_ASOC_STATE) == _TRUE
+	) {
+		adapter_link->mlmepriv.update_bcn = _TRUE;
+#ifndef CONFIG_INTERRUPT_BASED_TXBCN
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI) || defined(CONFIG_PCI_BCN_POLLING)
+		tx_beacon_hdl(adapter, NULL);
+#endif
+#endif
+	}
+	return _TRUE;
+}
+u8 rtw_mi_tx_beacon_hdl(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_tx_beacon_hdl);
+}
+u8 rtw_mi_buddy_tx_beacon_hdl(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_tx_beacon_hdl);
+}
+
+static u8 _rtw_mi_set_tx_beacon_cmd(_adapter *adapter, void *data)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_priv *pmlmepriv = &adapter_link->mlmepriv;
+
+	if (MLME_IS_AP(adapter) || MLME_IS_MESH(adapter)) {
+		if (pmlmepriv->update_bcn == _TRUE)
+			set_tx_beacon_cmd(adapter, adapter_link, 0);
+	}
+	return _TRUE;
+}
+u8 rtw_mi_set_tx_beacon_cmd(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _FALSE, _FALSE, NULL, _rtw_mi_set_tx_beacon_cmd);
+}
+u8 rtw_mi_buddy_set_tx_beacon_cmd(_adapter *padapter)
+{
+	return _rtw_mi_process(padapter, _TRUE, _FALSE, NULL, _rtw_mi_set_tx_beacon_cmd);
+}
+
+_adapter *rtw_get_iface_by_macddr(_adapter *padapter, const u8 *mac_addr)
+{
+	int i;
+	_adapter *iface = NULL;
+	u8 bmatch = _FALSE;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	#endif
+	u8 *iface_mac_addr;
+
+#ifdef RTW_PHL_TEST_FPGA
+	return dvobj->padapters[0];
+#endif
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface)
+			continue;
+
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+		if (pwdev_priv->random_mac_enabled) {
+			iface_mac_addr = adapter_pno_mac_addr(iface);
+		} else
+#endif /* CONFIG_RTW_SCAN_RAND */
+			iface_mac_addr = adapter_mac_addr(iface);
+
+		if (rtw_match_self_addr(iface, mac_addr))
+		{
+			bmatch = _TRUE;
+			break;
+		}
+	}
+	if (bmatch)
+		return iface;
+	else
+		return NULL;
+}
+
+
+/*#define CONFIG_SKB_ALLOCATED*/
+#define DBG_SKB_PROCESS
+#ifdef DBG_SKB_PROCESS
+void rtw_dbg_skb_process(_adapter *padapter, union recv_frame *precvframe, union recv_frame *pcloneframe)
+{
+	struct sk_buff *pkt_copy, *pkt_org;
+
+	pkt_org = precvframe->u.hdr.pkt;
+	pkt_copy = pcloneframe->u.hdr.pkt;
+	/*
+		RTW_INFO("%s ===== ORG SKB =====\n", __func__);
+		RTW_INFO(" SKB head(%p)\n", pkt_org->head);
+		RTW_INFO(" SKB data(%p)\n", pkt_org->data);
+		RTW_INFO(" SKB tail(%p)\n", pkt_org->tail);
+		RTW_INFO(" SKB end(%p)\n", pkt_org->end);
+
+		RTW_INFO(" recv frame head(%p)\n", precvframe->u.hdr.rx_head);
+		RTW_INFO(" recv frame data(%p)\n", precvframe->u.hdr.rx_data);
+		RTW_INFO(" recv frame tail(%p)\n", precvframe->u.hdr.rx_tail);
+		RTW_INFO(" recv frame end(%p)\n", precvframe->u.hdr.rx_end);
+
+		RTW_INFO("%s ===== COPY SKB =====\n", __func__);
+		RTW_INFO(" SKB head(%p)\n", pkt_copy->head);
+		RTW_INFO(" SKB data(%p)\n", pkt_copy->data);
+		RTW_INFO(" SKB tail(%p)\n", pkt_copy->tail);
+		RTW_INFO(" SKB end(%p)\n", pkt_copy->end);
+
+		RTW_INFO(" recv frame head(%p)\n", pcloneframe->u.hdr.rx_head);
+		RTW_INFO(" recv frame data(%p)\n", pcloneframe->u.hdr.rx_data);
+		RTW_INFO(" recv frame tail(%p)\n", pcloneframe->u.hdr.rx_tail);
+		RTW_INFO(" recv frame end(%p)\n", pcloneframe->u.hdr.rx_end);
+	*/
+	/*
+		RTW_INFO("%s => recv_frame adapter(%p,%p)\n", __func__, precvframe->u.hdr.adapter, pcloneframe->u.hdr.adapter);
+		RTW_INFO("%s => recv_frame dev(%p,%p)\n", __func__, pkt_org->dev , pkt_copy->dev);
+		RTW_INFO("%s => recv_frame len(%d,%d)\n", __func__, precvframe->u.hdr.len, pcloneframe->u.hdr.len);
+	*/
+	if (precvframe->u.hdr.len != pcloneframe->u.hdr.len)
+		RTW_INFO("%s [WARN]  recv_frame length(%d:%d) compare failed\n", __func__, precvframe->u.hdr.len, pcloneframe->u.hdr.len);
+
+	if (_rtw_memcmp(&precvframe->u.hdr.attrib, &pcloneframe->u.hdr.attrib, sizeof(struct rx_pkt_attrib)) == _FALSE)
+		RTW_INFO("%s [WARN]  recv_frame attrib compare failed\n", __func__);
+
+	if (_rtw_memcmp(precvframe->u.hdr.rx_data, pcloneframe->u.hdr.rx_data, precvframe->u.hdr.len) == _FALSE)
+		RTW_INFO("%s [WARN]  recv_frame rx_data compare failed\n", __func__);
+
+}
+#endif
+
+static s32 _rtw_mi_buddy_clone_bcmc_packet(_adapter *adapter,
+		union recv_frame *precvframe, union recv_frame *pcloneframe)
+{
+	s32 ret = _SUCCESS;
+	s32 process_ret = CORE_RX_CONTINUE;
+#ifdef CONFIG_SKB_ALLOCATED
+	u8 *pbuf = precvframe->u.hdr.rx_data;
+#endif
+	struct rx_pkt_attrib *pattrib = NULL;
+
+	if (pcloneframe) {
+		pcloneframe->u.hdr.adapter = adapter;
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		pcloneframe->u.hdr.adapter_link = GET_PRIMARY_LINK(adapter);
+
+		_rtw_init_listhead(&pcloneframe->u.hdr.list);
+		pcloneframe->u.hdr.precvbuf = NULL;	/*can't access the precvbuf for new arch.*/
+		pcloneframe->u.hdr.len = 0;
+
+		_rtw_memcpy(&pcloneframe->u.hdr.attrib, &precvframe->u.hdr.attrib, sizeof(struct rx_pkt_attrib));
+
+		pattrib = &pcloneframe->u.hdr.attrib;
+#ifdef CONFIG_SKB_ALLOCATED
+		if (rtw_os_alloc_recvframe(adapter, pcloneframe, pbuf, NULL) == _SUCCESS)
+#else
+		if (rtw_os_recvframe_duplicate_skb(adapter, pcloneframe, precvframe->u.hdr.pkt) == _SUCCESS)
+#endif
+		{
+#ifdef CONFIG_SKB_ALLOCATED
+			recvframe_put(pcloneframe, pattrib->pkt_len);
+#endif
+
+#ifdef DBG_SKB_PROCESS
+			rtw_dbg_skb_process(adapter, precvframe, pcloneframe);
+#endif
+			if (validate_recv_frame(adapter, &pcloneframe) != CORE_RX_CONTINUE) {
+				ret = _FAIL;
+				return ret;
+			}
+			process_ret = rtw_core_rx_data_pre_process(adapter, &pcloneframe);
+			if (process_ret == CORE_RX_DEFRAG)
+				return ret;
+			if (process_ret != CORE_RX_CONTINUE) {
+				ret = _FAIL;
+				return ret;
+			}
+			process_ret = rtw_core_rx_data_post_process(adapter, &pcloneframe);
+			if (process_ret == CORE_RX_DONE) {
+				adapter->recvinfo.rx_pkts++;
+			} else if (process_ret == CORE_RX_DROP) {
+				if (pcloneframe == NULL)
+					return ret;
+			} else {
+				ret = _FAIL;
+				return ret;
+			}
+		} else {
+			ret = _FAIL;
+			RTW_ERR("%s()-%d: rtw_os_alloc_recvframe() failed!\n", __func__, __LINE__);
+		}
+	}
+	return ret;
+}
+
+
+void rtw_mi_buddy_clone_bcmc_packet(_adapter *padapter,
+					union recv_frame *precvframe)
+{
+	int i;
+	s32 ret = _SUCCESS;
+	_adapter *iface = NULL;
+	union recv_frame *pcloneframe = NULL;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct recv_priv *precvpriv = &dvobj->recvpriv;
+	_queue *pfree_recv_queue = &precvpriv->free_recv_queue;
+	u8 *fhead = get_recvframe_data(precvframe);
+	u8 type = GetFrameType(fhead);
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || iface == padapter)
+			continue;
+		if (rtw_is_adapter_up(iface) == _FALSE || iface->registered == 0)
+			continue;
+		#if 0
+		if (!rtw_iface_at_same_hwband(padapter, iface))
+			continue;
+		#endif
+
+		if (type == WIFI_DATA_TYPE && !adapter_allow_bmc_data_rx(iface))
+			continue;
+
+		pcloneframe = rtw_alloc_recvframe(pfree_recv_queue);
+		if (pcloneframe) {
+			ret = _rtw_mi_buddy_clone_bcmc_packet(iface, precvframe, pcloneframe);
+			if (ret != _SUCCESS )
+				rtw_free_recvframe(pcloneframe);
+		} else {
+			RTW_ERR("%s rtw_alloc_recvframe failed\n", __func__);
+			rtw_warn_on(1);
+		}
+	}
+}
+
+u8 rtw_mi_get_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	int i;
+	_adapter *iface;
+	u8 ifbmp = 0;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface)
+			continue;
+		if (!rtw_iface_is_operate_at_hwband(iface, band_idx))
+			continue;
+		ifbmp |= BIT(i);
+	}
+
+	return ifbmp;
+}
+
+_adapter *rtw_mi_get_iface_by_hwband(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	u8 ifbmp = rtw_mi_get_ifbmp_by_hwband(dvobj, band_idx);
+
+	if (ifbmp) {
+		int i;
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			if ((ifbmp & BIT(i)) && dvobj->padapters[i])
+				return dvobj->padapters[i];
+		}
+	}
+	return NULL;
+}
+
+static u8 _rtw_mi_get_sta_ifbmp_by_hwband(struct dvobj_priv *dvobj, _adapter *adapter, u8 band_idx, u32 mlme_sbmp)
+{
+	int i;
+	_adapter *iface;
+	u8 ifbmp = 0;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface)
+			continue;
+		if (adapter) {
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+		} else if (band_idx < HW_BAND_MAX){
+			if (!rtw_iface_is_operate_at_hwband(iface, band_idx))
+				continue;
+		}
+
+		if (MLME_IS_STA(iface) && (!mlme_sbmp || CHK_MLME_STATE(iface, mlme_sbmp)))
+			ifbmp |= BIT(i);
+	}
+
+	return ifbmp;
+}
+
+u8 rtw_mi_get_ld_sta_ifbmp(_adapter *adapter)
+{
+	return _rtw_mi_get_sta_ifbmp_by_hwband(adapter_to_dvobj(adapter), adapter, HW_BAND_MAX
+		, WIFI_ASOC_STATE);
+}
+
+u8 rtw_mi_get_ld_sta_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	return _rtw_mi_get_sta_ifbmp_by_hwband(dvobj, NULL, band_idx
+		, WIFI_ASOC_STATE);
+}
+
+u8 rtw_mi_get_lgd_sta_ifbmp(_adapter *adapter)
+{
+	return _rtw_mi_get_sta_ifbmp_by_hwband(adapter_to_dvobj(adapter), adapter, HW_BAND_MAX
+		, WIFI_UNDER_LINKING | WIFI_ASOC_STATE);
+}
+
+u8 rtw_mi_get_lgd_sta_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	return _rtw_mi_get_sta_ifbmp_by_hwband(dvobj, NULL, band_idx
+		, WIFI_UNDER_LINKING | WIFI_ASOC_STATE);
+}
+
+static u8 _rtw_mi_get_ap_mesh_ifbmp_by_hwband(struct dvobj_priv *dvobj, _adapter *adapter, u8 band_idx)
+{
+	int i;
+	_adapter *iface;
+	u8 ifbmp = 0;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface)
+			continue;
+		if (adapter) {
+			if (!rtw_iface_at_same_hwband(adapter, iface))
+				continue;
+		} else if (band_idx < HW_BAND_MAX){
+			if (!rtw_iface_is_operate_at_hwband(iface, band_idx))
+				continue;
+		}
+
+		if (CHK_MLME_STATE(iface, WIFI_AP_STATE | WIFI_MESH_STATE)
+			&& MLME_IS_ASOC(iface))
+			ifbmp |= BIT(i);
+	}
+
+	return ifbmp;
+}
+
+u8 rtw_mi_get_ap_mesh_ifbmp(_adapter *adapter)
+{
+	return _rtw_mi_get_ap_mesh_ifbmp_by_hwband(adapter_to_dvobj(adapter), adapter, HW_BAND_MAX);
+}
+
+u8 rtw_mi_get_ap_mesh_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	return _rtw_mi_get_ap_mesh_ifbmp_by_hwband(dvobj, NULL, band_idx);
+}
+
+_adapter *rtw_mi_get_ap_mesh_iface_by_hwband(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	u8 ifbmp = rtw_mi_get_ap_mesh_ifbmp_by_hwband(dvobj, band_idx);
+
+	if (ifbmp) {
+		int i;
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			if ((ifbmp & BIT(i)) && dvobj->padapters[i])
+				return dvobj->padapters[i];
+		}
+	}
+	return NULL;
+}
+
+bool rtw_iface_is_operate_at_hwband(_adapter *adapter, u8 band_idx)
+{
+	struct _ADAPTER_LINK *alink;
+	u8 lidx;
+
+	if (rtw_is_adapter_up(adapter)) {
+		for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+			alink = GET_LINK(adapter, lidx);
+			if (alink->wrlink->hw_band == band_idx)
+				return true;
+		}
+	}
+
+	return false;
+}
+
+bool rtw_iface_at_same_hwband(_adapter *adapter, _adapter *iface)
+{
+	bool rst = _TRUE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct _ADAPTER_LINK *iface_link = GET_PRIMARY_LINK(iface);
+
+	#ifdef CONFIG_DBCC_SUPPORT
+	if (rtw_is_adapter_up(adapter) && rtw_is_adapter_up(iface))
+		rst = (adapter_link->wrlink->hw_band == iface_link->wrlink->hw_band) ? _TRUE : _FALSE;
+	#endif
+	return rst;
+}
+
+u8 rtw_mi_get_hw_port(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	if (rtw_is_adapter_up(adapter))
+		return adapter_link->wrlink->hw_port;
+	return 0xFF;
+}
+
+u32 ifbmp_to_iflbmp(u8 ifbmp)
+{
+	u8 i, j;
+	u32 iflbmp = 0;
+
+	for (i = 0; i < CONFIG_IFACE_NUMBER; i++) {
+		for (j = 0; j < RTW_RLINK_MAX; j++) {
+			iflbmp |= (ifbmp & BIT(i)) << (j * CONFIG_IFACE_NUMBER);
+		}
+	}
+	return iflbmp;
+}
+
+u8 iflbmp_to_ifbmp(u32 iflbmp)
+{
+	u8 i, j;
+	u8 ifbmp = 0;
+
+	for (i = 0; i < CONFIG_IFACE_NUMBER; i++) {
+		for (j = 0; j < RTW_RLINK_MAX; j++) {
+			ifbmp |= (iflbmp >> (j * CONFIG_IFACE_NUMBER)) & BIT(i);
+		}
+	}
+	return ifbmp;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mlme.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mlme.c
new file mode 100644
index 000000000000..baf2e5f86c91
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mlme.c
@@ -0,0 +1,7195 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_MLME_C_
+#include <drv_types.h>
+
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static void rtw_update_link_he_6ghz_cap(_adapter *padapter,
+				 struct _ADAPTER_LINK *padapter_link,
+				 u8 *pie, uint ie_len, enum band_type band,
+				 u32 ch);
+#endif
+
+extern u8 rtw_do_join(_adapter *padapter);
+
+
+void rtw_init_mlme_timer(_adapter *padapter)
+{
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+	rtw_init_timer(&(pmlmepriv->assoc_timer), rtw_join_timeout_handler, padapter);
+
+#ifdef CONFIG_SET_SCAN_DENY_TIMER
+	rtw_init_timer(&(pmlmepriv->set_scan_deny_timer), rtw_set_scan_deny_timer_hdl, padapter);
+#endif
+
+#ifdef RTK_DMP_PLATFORM
+	_init_workitem(&(pmlmepriv->Linkup_workitem), Linkup_workitem_callback, padapter);
+	_init_workitem(&(pmlmepriv->Linkdown_workitem), Linkdown_workitem_callback, padapter);
+#endif
+}
+
+sint	_rtw_init_mlme_priv(_adapter *padapter)
+{
+	sint	i;
+	u8	*pbuf;
+	struct wlan_network	*pnetwork;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	sint	res = _SUCCESS;
+#ifdef CONFIG_RTW_MULTI_AP
+	struct unassoc_sta_info *unassoc_sta;
+#endif
+	u8	*pmld_buf;
+	struct wlan_mld_network *pmld_network;
+#ifdef CONFIG_LAYER2_ROAMING
+	char str[8];
+#endif
+
+	/* We don't need to memset padapter->XXX to zero, because adapter is allocated by rtw_zvmalloc(). */
+	/* _rtw_memset((u8 *)pmlmepriv, 0, sizeof(struct mlme_priv)); */
+
+
+	/*qos_priv*/
+	/*pmlmepriv->qospriv.qos_option = pregistrypriv->wmm_enable;*/
+
+	/*ht_priv*/
+
+	pmlmepriv->nic_hdl = (u8 *)padapter;
+
+	pmlmepriv->pscanned = NULL;
+	init_fwstate(pmlmepriv, WIFI_STATION_STATE);
+	pmlmepriv->dev_cur_network.network.InfrastructureMode = Ndis802_11AutoUnknown;
+	pmlmepriv->scan_mode = RTW_PHL_SCAN_ACTIVE; /* 1: active, 0: pasive. Maybe someday we should rename this varable to "active_mode" (Jeff) */
+
+	_rtw_spinlock_init(&(pmlmepriv->lock));
+	_rtw_init_queue(&(pmlmepriv->free_bss_pool));
+	_rtw_init_queue(&(pmlmepriv->scanned_queue));
+
+	set_scanned_network_val(pmlmepriv, 0);
+
+	_rtw_memset(&pmlmepriv->assoc_ssid, 0, sizeof(NDIS_802_11_SSID));
+
+	if (padapter->registrypriv.max_bss_cnt != 0)
+		pmlmepriv->max_bss_cnt = padapter->registrypriv.max_bss_cnt;
+	else {
+		pmlmepriv->max_bss_cnt = 0;
+		pmlmepriv->max_bss_cnt += is_supported_24g(padapter->registrypriv.band_type) ? MAX_BSS_CNT : 0;
+		pmlmepriv->max_bss_cnt += is_supported_5g(padapter->registrypriv.band_type) ? MAX_BSS_CNT : 0;
+		pmlmepriv->max_bss_cnt += is_supported_6g(padapter->registrypriv.band_type) ? MAX_BSS_CNT : 0;
+	}
+
+	pbuf = rtw_zvmalloc(pmlmepriv->max_bss_cnt * (sizeof(struct wlan_network)));
+
+	if (pbuf == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pmlmepriv->free_bss_buf = pbuf;
+
+	pnetwork = (struct wlan_network *)pbuf;
+
+	for (i = 0; i < pmlmepriv->max_bss_cnt; i++) {
+		_rtw_init_listhead(&(pnetwork->list));
+
+		rtw_list_insert_tail(&(pnetwork->list), &(pmlmepriv->free_bss_pool.queue));
+
+		pnetwork++;
+	}
+
+	_rtw_init_queue(&(pmlmepriv->free_mld_bss_pool));
+	_rtw_init_queue(&(pmlmepriv->scanned_mld_queue));
+	set_scanned_mld_network_val(pmlmepriv, 0);
+
+	pmld_buf = rtw_zvmalloc(pmlmepriv->max_bss_cnt * (sizeof(struct wlan_mld_network)));
+
+	if (pmld_buf == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pmlmepriv->free_mld_bss_buf = pmld_buf;
+
+	pmld_network = (struct wlan_mld_network *)pmld_buf;
+
+	for (i = 0; i < pmlmepriv->max_bss_cnt; i++) {
+		_rtw_init_listhead(&(pmld_network->list));
+
+		rtw_list_insert_tail(&(pmld_network->list), &(pmlmepriv->free_mld_bss_pool.queue));
+
+		pmld_network++;
+	}
+
+#ifdef CONFIG_RTW_MULTI_AP
+	if (is_primary_adapter(padapter)) {
+		_rtw_init_queue(&(pmlmepriv->free_unassoc_sta_queue));
+		_rtw_init_queue(&(pmlmepriv->unassoc_sta_queue));
+		for (i = 0; i < UNASOC_STA_SRC_NUM; i++)
+			pmlmepriv->unassoc_sta_mode_of_stype[i] = padapter->registrypriv.unassoc_sta_mode_of_stype[i];
+
+		if (padapter->registrypriv.max_unassoc_sta_cnt != 0)
+			pmlmepriv->max_unassoc_sta_cnt = padapter->registrypriv.max_unassoc_sta_cnt;
+		else {
+			pmlmepriv->max_unassoc_sta_cnt = 0;
+			pmlmepriv->max_unassoc_sta_cnt += is_supported_24g(padapter->registrypriv.band_type) ? MAX_UNASSOC_STA_CNT : 0;
+			pmlmepriv->max_unassoc_sta_cnt += is_supported_5g(padapter->registrypriv.band_type) ? MAX_UNASSOC_STA_CNT : 0;
+			pmlmepriv->max_unassoc_sta_cnt += is_supported_6g(padapter->registrypriv.band_type) ? MAX_UNASSOC_STA_CNT : 0;
+		}
+
+		pbuf = rtw_zvmalloc(pmlmepriv->max_unassoc_sta_cnt * (sizeof(struct unassoc_sta_info)));
+		if (pbuf == NULL) {
+			res = _FAIL;
+			goto exit;
+		}
+
+		pmlmepriv->free_unassoc_sta_buf = pbuf;
+		unassoc_sta = (struct unassoc_sta_info *) pbuf;
+		for (i = 0; i < pmlmepriv->max_unassoc_sta_cnt; i++) {
+			_rtw_init_listhead(&(unassoc_sta->list));
+			rtw_list_insert_tail(&(unassoc_sta->list), &(pmlmepriv->free_unassoc_sta_queue.queue));
+			unassoc_sta++;
+		}
+	}
+#endif
+	/* allocate DMA-able/Non-Page memory for cmd_buf and rsp_buf */
+
+	rtw_clear_scan_deny(padapter);
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+	pmlmepriv->bGetGateway = 0;
+	pmlmepriv->GetGatewayTryCnt = 0;
+#endif
+
+#ifdef CONFIG_LAYER2_ROAMING
+#define RTW_ROAM_SCAN_RESULT_EXP_MS (5*1000)
+#define RTW_ROAM_RSSI_DIFF_TH 10
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define RTW_ROAM_SCAN_INTERVAL (10)    /* 10*(2 second)*/
+#else
+#define RTW_ROAM_SCAN_INTERVAL (5)    /* 5*(2 second)*/
+#endif
+#define RTW_ROAM_RSSI_THRESHOLD 70
+
+	pmlmepriv->roam_flags = CONFIG_ROAMING_FLAG;
+	pmlmepriv->roam_scanr_exp_ms = RTW_ROAM_SCAN_RESULT_EXP_MS;
+	pmlmepriv->roam_rssi_diff_th = RTW_ROAM_RSSI_DIFF_TH;
+	pmlmepriv->roam_scan_int 	 = RTW_ROAM_SCAN_INTERVAL;
+	pmlmepriv->roam_rssi_threshold = RTW_ROAM_RSSI_THRESHOLD;
+	pmlmepriv->need_to_roam = _FALSE;
+	pmlmepriv->last_roaming = rtw_get_current_time();
+
+	sprintf(str,"%d", CONFIG_ROAM_SCAN_ORDER);
+	for (i = 0; i < sizeof(pmlmepriv->roam_scan_order); i++)
+		pmlmepriv->roam_scan_order[i] =
+			(str[i]=='2')?BAND_ON_24G:(str[i]=='5')?BAND_ON_5G:(str[i]=='6')?BAND_ON_6G:BAND_MAX;
+
+#endif /* CONFIG_LAYER2_ROAMING */
+
+#ifdef CONFIG_RTW_80211R
+	rtw_ft_info_init(&pmlmepriv->ft_roam);
+#endif
+
+#ifdef CONFIG_RTW_MBO
+#ifdef CONFIG_LAYER2_ROAMING
+#if defined(CONFIG_RTW_WNM) || defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	rtw_roam_nb_info_init(padapter);
+	pmlmepriv->ch_cnt = 0;
+#endif
+#endif
+#endif
+
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_attr_info_init(padapter);
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	rtw_core_twt_init(padapter);
+#endif
+#endif
+
+	pmlmepriv->defs_lmt_sta = 2;
+	pmlmepriv->defs_lmt_time = 5;
+
+	rtw_init_mlme_timer(padapter);
+
+exit:
+
+
+	return res;
+}
+
+void rtw_mfree_mlme_priv_lock(struct mlme_priv *pmlmepriv)
+{
+	_rtw_spinlock_free(&pmlmepriv->lock);
+	_rtw_spinlock_free(&(pmlmepriv->free_bss_pool.lock));
+	_rtw_spinlock_free(&(pmlmepriv->scanned_queue.lock));
+	_rtw_spinlock_free(&(pmlmepriv->free_mld_bss_pool.lock));
+	_rtw_spinlock_free(&(pmlmepriv->scanned_mld_queue.lock));
+#ifdef CONFIG_RTW_MULTI_AP
+	if (is_primary_adapter(mlme_to_adapter(pmlmepriv))) {
+		_rtw_spinlock_free(&(pmlmepriv->unassoc_sta_queue.lock));
+		_rtw_spinlock_free(&(pmlmepriv->free_unassoc_sta_queue.lock));
+	}
+#endif
+}
+
+static void rtw_free_mlme_ie_data(u8 **ppie, u32 *plen)
+{
+	if (*ppie) {
+		rtw_mfree(*ppie, *plen);
+		*plen = 0;
+		*ppie = NULL;
+	}
+}
+
+void rtw_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv)
+{
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	rtw_buf_free(&pmlmepriv->assoc_req, &pmlmepriv->assoc_req_len);
+	rtw_buf_free(&pmlmepriv->assoc_rsp, &pmlmepriv->assoc_rsp_len);
+	rtw_buf_free(&pmlmepriv->probe_rsp, &pmlmepriv->probe_rsp_len);
+	rtw_buf_free(&pmlmepriv->beacon_head_ie, &pmlmepriv->beacon_head_ie_len);
+	rtw_buf_free(&pmlmepriv->beacon_tail_ie, &pmlmepriv->beacon_tail_ie_len);
+
+	rtw_free_mlme_ie_data(&pmlmepriv->wps_beacon_ie, &pmlmepriv->wps_beacon_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wps_probe_req_ie, &pmlmepriv->wps_probe_req_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wps_probe_resp_ie, &pmlmepriv->wps_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wps_assoc_resp_ie, &pmlmepriv->wps_assoc_resp_ie_len);
+
+	rtw_free_mlme_ie_data(&pmlmepriv->p2p_beacon_ie, &pmlmepriv->p2p_beacon_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->p2p_probe_req_ie, &pmlmepriv->p2p_probe_req_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->p2p_probe_resp_ie, &pmlmepriv->p2p_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->p2p_go_probe_resp_ie, &pmlmepriv->p2p_go_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->p2p_assoc_req_ie, &pmlmepriv->p2p_assoc_req_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->p2p_assoc_resp_ie, &pmlmepriv->p2p_assoc_resp_ie_len);
+#endif
+
+#if defined(CONFIG_WFD) && defined(CONFIG_IOCTL_CFG80211)
+	rtw_free_mlme_ie_data(&pmlmepriv->wfd_beacon_ie, &pmlmepriv->wfd_beacon_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wfd_probe_req_ie, &pmlmepriv->wfd_probe_req_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wfd_probe_resp_ie, &pmlmepriv->wfd_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wfd_go_probe_resp_ie, &pmlmepriv->wfd_go_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wfd_assoc_req_ie, &pmlmepriv->wfd_assoc_req_ie_len);
+	rtw_free_mlme_ie_data(&pmlmepriv->wfd_assoc_resp_ie, &pmlmepriv->wfd_assoc_resp_ie_len);
+#endif
+
+#ifdef CONFIG_RTW_80211R
+	rtw_free_mlme_ie_data(&pmlmepriv->auth_rsp, &pmlmepriv->auth_rsp_len);
+#endif
+#ifdef CONFIG_RTW_MBO
+	rtw_buf_free(&pmlmepriv->pcell_data_cap_ie, &pmlmepriv->cell_data_cap_len);
+#endif
+}
+
+#if defined(CONFIG_WFD) && defined(CONFIG_IOCTL_CFG80211)
+int rtw_mlme_update_wfd_ie_data(struct mlme_priv *mlme, u8 type, u8 *ie, u32 ie_len)
+{
+	_adapter *adapter = mlme_to_adapter(mlme);
+	struct wifi_display_info *wfd_info = &adapter->wfd_info;
+	u8 clear = 0;
+	u8 **t_ie = NULL;
+	u32 *t_ie_len = NULL;
+	int ret = _FAIL;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		goto success;
+
+	if (wfd_info->wfd_enable == _TRUE)
+		goto success; /* WFD IE is build by self */
+
+	if (!ie && !ie_len)
+		clear = 1;
+	else if (!ie || !ie_len) {
+		RTW_PRINT(FUNC_ADPT_FMT" type:%u, ie:%p, ie_len:%u"
+			  , FUNC_ADPT_ARG(adapter), type, ie, ie_len);
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	switch (type) {
+	case MLME_BEACON_IE:
+		t_ie = &mlme->wfd_beacon_ie;
+		t_ie_len = &mlme->wfd_beacon_ie_len;
+		break;
+	case MLME_PROBE_REQ_IE:
+		t_ie = &mlme->wfd_probe_req_ie;
+		t_ie_len = &mlme->wfd_probe_req_ie_len;
+		break;
+	case MLME_PROBE_RESP_IE:
+		t_ie = &mlme->wfd_probe_resp_ie;
+		t_ie_len = &mlme->wfd_probe_resp_ie_len;
+		break;
+	case MLME_GO_PROBE_RESP_IE:
+		t_ie = &mlme->wfd_go_probe_resp_ie;
+		t_ie_len = &mlme->wfd_go_probe_resp_ie_len;
+		break;
+	case MLME_ASSOC_REQ_IE:
+		t_ie = &mlme->wfd_assoc_req_ie;
+		t_ie_len = &mlme->wfd_assoc_req_ie_len;
+		break;
+	case MLME_ASSOC_RESP_IE:
+		t_ie = &mlme->wfd_assoc_resp_ie;
+		t_ie_len = &mlme->wfd_assoc_resp_ie_len;
+		break;
+	default:
+		RTW_PRINT(FUNC_ADPT_FMT" unsupported type:%u"
+			  , FUNC_ADPT_ARG(adapter), type);
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (*t_ie) {
+		u32 free_len = *t_ie_len;
+		*t_ie_len = 0;
+		rtw_mfree(*t_ie, free_len);
+		*t_ie = NULL;
+	}
+
+	if (!clear) {
+		*t_ie = rtw_malloc(ie_len);
+		if (*t_ie == NULL) {
+			RTW_ERR(FUNC_ADPT_FMT" type:%u, rtw_malloc() fail\n"
+				, FUNC_ADPT_ARG(adapter), type);
+			goto exit;
+		}
+		_rtw_memcpy(*t_ie, ie, ie_len);
+		*t_ie_len = ie_len;
+	}
+
+	if (*t_ie && *t_ie_len) {
+		u8 *attr_content;
+		u32 attr_contentlen = 0;
+
+		attr_content = rtw_get_wfd_attr_content(*t_ie, *t_ie_len, WFD_ATTR_DEVICE_INFO, NULL, &attr_contentlen);
+		if (attr_content && attr_contentlen) {
+			if (RTW_GET_BE16(attr_content + 2) != wfd_info->rtsp_ctrlport) {
+				wfd_info->rtsp_ctrlport = RTW_GET_BE16(attr_content + 2);
+				RTW_INFO(FUNC_ADPT_FMT" type:%u, RTSP CTRL port = %u\n"
+					, FUNC_ADPT_ARG(adapter), type, wfd_info->rtsp_ctrlport);
+			}
+		}
+	}
+
+success:
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+#endif /* defined(CONFIG_WFD) && defined(CONFIG_IOCTL_CFG80211) */
+
+void _rtw_free_mlme_priv(struct mlme_priv *pmlmepriv)
+{
+	_adapter *adapter = mlme_to_adapter(pmlmepriv);
+	if (NULL == pmlmepriv) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+	rtw_free_mlme_priv_ie_data(pmlmepriv);
+
+	if (pmlmepriv) {
+		rtw_mfree_mlme_priv_lock(pmlmepriv);
+
+		if (pmlmepriv->free_bss_buf)
+			rtw_vmfree(pmlmepriv->free_bss_buf, pmlmepriv->max_bss_cnt * sizeof(struct wlan_network));
+		if (pmlmepriv->free_mld_bss_buf)
+			rtw_vmfree(pmlmepriv->free_mld_bss_buf, pmlmepriv->max_bss_cnt * sizeof(struct wlan_mld_network));
+
+#ifdef CONFIG_RTW_MULTI_AP
+		if (is_primary_adapter(adapter)) {
+			if (pmlmepriv->free_unassoc_sta_buf)
+				rtw_vmfree(pmlmepriv->free_unassoc_sta_buf, pmlmepriv->max_unassoc_sta_cnt * sizeof(struct unassoc_sta_info));
+		}
+#endif
+	}
+exit:
+	return;
+}
+
+int rtw_init_link_mlme_priv(struct _ADAPTER_LINK *padapter_link)
+{
+#ifdef PRIVATE_R
+	sint i = 0;
+#endif
+	struct link_mlme_priv *mlmepriv = &padapter_link->mlmepriv;
+	/* ToDo CONFIG_RTW_MLD: */
+	mlmepriv->to_join = _FALSE;
+	mlmepriv->is_accepted = _FALSE;
+
+#ifdef PRIVATE_R
+	/*init disconnect bcn queue related resoureces.*/
+	_rtw_init_queue(&(mlmepriv->idle_dis_bcn_queue));
+	_rtw_init_queue(&(mlmepriv->busy_dis_bcn_queue));
+
+	_rtw_spinlock_bh(&mlmepriv->idle_dis_bcn_queue.lock);
+	for (i = 0; i < MAX_NUM_DIS_BCN_INFO; i++) {
+		_rtw_init_listhead(&(mlmepriv->dis_bcn_key_info[i].list));
+		_rtw_memset(&(mlmepriv->dis_bcn_key_info[i].bcn_content), 0, sizeof(struct beacon_keys));
+		rtw_list_insert_tail(&(mlmepriv->dis_bcn_key_info[i].list), &(mlmepriv->idle_dis_bcn_queue.queue));
+	}
+	_rtw_spinunlock_bh(&mlmepriv->idle_dis_bcn_queue.lock);
+#endif
+
+	return _SUCCESS;
+}
+
+sint	_rtw_enqueue_network(_queue *queue, struct wlan_network *pnetwork)
+{
+
+	if (pnetwork == NULL)
+		goto exit;
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	rtw_list_insert_tail(&pnetwork->list, &queue->queue);
+
+	_rtw_spinunlock_bh(&queue->lock);
+
+exit:
+
+
+	return _SUCCESS;
+}
+
+/*
+struct	wlan_network *_rtw_dequeue_network(_queue *queue)
+{
+
+	struct wlan_network *pnetwork;
+
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	if (_rtw_queue_empty(queue) == _TRUE)
+
+		pnetwork = NULL;
+
+	else
+	{
+		pnetwork = LIST_CONTAINOR(get_next(&queue->queue), struct wlan_network, list);
+
+		rtw_list_delete(&(pnetwork->list));
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+
+
+	return pnetwork;
+}
+*/
+
+struct	wlan_network *_rtw_alloc_network(struct	mlme_priv *pmlmepriv) /* (_queue *free_queue) */
+{
+	struct	wlan_network	*pnetwork;
+	_queue *free_queue = &pmlmepriv->free_bss_pool;
+	_list *plist = NULL;
+
+
+	_rtw_spinlock_bh(&free_queue->lock);
+
+	if (_rtw_queue_empty(free_queue) == _TRUE) {
+		pnetwork = NULL;
+		goto exit;
+	}
+	plist = get_next(&(free_queue->queue));
+
+	pnetwork = LIST_CONTAINOR(plist , struct wlan_network, list);
+
+	rtw_list_delete(&pnetwork->list);
+
+	pnetwork->network_type = 0;
+	pnetwork->fixed = _FALSE;
+	pnetwork->last_scanned = rtw_get_current_time();
+	pnetwork->last_non_hidden_ssid_ap = pnetwork->last_scanned;
+#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+	pnetwork->acnode_stime = 0;
+	pnetwork->acnode_notify_etime = 0;
+#endif
+
+	pnetwork->aid = 0;
+	pnetwork->join_res = 0;
+#ifdef CONFIG_80211D
+	_rtw_memset(&pnetwork->cisr, 0, sizeof(pnetwork->cisr));
+#endif
+
+	pmlmepriv->num_of_scanned++;
+
+exit:
+	_rtw_spinunlock_bh(&free_queue->lock);
+
+
+	return pnetwork;
+}
+
+void _rtw_free_network(struct	mlme_priv *pmlmepriv , struct wlan_network *pnetwork, u8 isfreeall)
+{
+	u32 delta_time;
+	u32 lifetime = SCANQUEUE_LIFETIME;
+	_queue *free_queue = &(pmlmepriv->free_bss_pool);
+
+
+	if (pnetwork == NULL)
+		goto exit;
+
+	if (pnetwork->fixed == _TRUE)
+		goto exit;
+
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE) ||
+	    (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE))
+		lifetime = 1;
+
+	if (!isfreeall) {
+		delta_time = (u32) rtw_get_passing_time_ms(pnetwork->last_scanned);
+		if (delta_time < lifetime) /* unit:msec */
+			goto exit;
+	}
+
+	_rtw_spinlock_bh(&free_queue->lock);
+
+	rtw_list_delete(&(pnetwork->list));
+
+	rtw_list_insert_tail(&(pnetwork->list), &(free_queue->queue));
+
+	pmlmepriv->num_of_scanned--;
+
+
+	/* RTW_INFO("_rtw_free_network:SSID=%s\n", pnetwork->network.Ssid.Ssid); */
+
+	_rtw_spinunlock_bh(&free_queue->lock);
+
+exit:
+	return;
+}
+
+void _rtw_free_network_nolock(struct	mlme_priv *pmlmepriv, struct wlan_network *pnetwork)
+{
+
+	_queue *free_queue = &(pmlmepriv->free_bss_pool);
+
+
+	if (pnetwork == NULL)
+		goto exit;
+
+	if (pnetwork->fixed == _TRUE)
+		goto exit;
+
+	/* _rtw_spinlock_irq(&free_queue->lock, &sp_flags); */
+
+	rtw_list_delete(&(pnetwork->list));
+
+	rtw_list_insert_tail(&(pnetwork->list), get_list_head(free_queue));
+
+	pmlmepriv->num_of_scanned--;
+
+	/* _rtw_spinunlock_irq(&free_queue->lock, &sp_flags); */
+
+exit:
+	return;
+}
+
+void _rtw_free_network_queue(_adapter *padapter, u8 isfreeall)
+{
+
+	_list *phead, *plist;
+	struct wlan_network *pnetwork;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	_queue *scanned_queue = &pmlmepriv->scanned_queue;
+
+
+
+	_rtw_spinlock_bh(&scanned_queue->lock);
+
+	phead = get_list_head(scanned_queue);
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+		plist = get_next(plist);
+
+		/* If not free all, then the MLD link networks are handled by
+		** rtw_free_mld_network_queue() according to the last scanned time
+		** of their affiliated MLD */
+		if (pnetwork->network.is_mld && !isfreeall)
+			continue;
+		_rtw_free_network(pmlmepriv, pnetwork, isfreeall);
+
+	}
+
+	_rtw_spinunlock_bh(&scanned_queue->lock);
+
+
+}
+
+
+
+
+sint rtw_if_up(_adapter *padapter)
+{
+
+	sint res;
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)) ||
+	    (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _FALSE)) {
+		res = _FALSE;
+	} else
+		res =  _TRUE;
+
+	return res;
+}
+
+
+void rtw_generate_random_ibss(u8 *pibss)
+{
+	*((u32 *)(&pibss[2])) = rtw_random32();
+	pibss[0] = 0x02; /* in ad-hoc mode local bit must set to 1 */
+	pibss[1] = 0x11;
+	pibss[2] = 0x87;
+}
+
+u8 *rtw_get_capability_from_ie(u8 *ie)
+{
+	return ie + 8 + 2;
+}
+
+
+u16 rtw_get_capability(WLAN_BSSID_EX *bss)
+{
+	u16	val;
+
+	_rtw_memcpy((u8 *)&val, rtw_get_capability_from_ie(bss->IEs), 2);
+
+	return le16_to_cpu(val);
+}
+
+u8 *rtw_get_timestampe_from_ie(u8 *ie)
+{
+	return ie + 0;
+}
+
+u8 *rtw_get_beacon_interval_from_ie(u8 *ie)
+{
+	return ie + 8;
+}
+
+
+int	rtw_init_mlme_priv(_adapter *padapter) /* (struct	mlme_priv *pmlmepriv) */
+{
+	int	res;
+	res = _rtw_init_mlme_priv(padapter);/* (pmlmepriv); */
+	return res;
+}
+
+void rtw_free_mlme_priv(struct mlme_priv *pmlmepriv)
+{
+	_rtw_free_mlme_priv(pmlmepriv);
+}
+
+int rtw_enqueue_network(_queue *queue, struct wlan_network *pnetwork)
+{
+	int	res;
+	res = _rtw_enqueue_network(queue, pnetwork);
+	return res;
+}
+
+void rtw_free_network(struct mlme_priv *pmlmepriv, struct	wlan_network *pnetwork, u8 is_freeall)/* (struct	wlan_network *pnetwork, _queue	*free_queue) */
+{
+	_rtw_free_network(pmlmepriv, pnetwork, is_freeall);
+}
+
+void rtw_free_network_nolock(_adapter *padapter, struct wlan_network *pnetwork)
+{
+	_rtw_free_network_nolock(&(padapter->mlmepriv), pnetwork);
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_unlink_bss(padapter, pnetwork);
+#endif /* CONFIG_IOCTL_CFG80211 */
+}
+
+
+void rtw_free_network_queue(_adapter *dev, u8 isfreeall)
+{
+	_rtw_free_network_queue(dev, isfreeall);
+}
+
+#ifdef PRIVATE_R
+struct  dis_bcn_key *_rtw_alloc_dis_bcn_key(_queue *pqueue)
+{
+	_list *plist = NULL, *phead = NULL;
+	struct dis_bcn_key *pdis_bcn_key = NULL;
+
+	if (_rtw_queue_empty(pqueue) == _TRUE)
+		pdis_bcn_key = NULL;
+	else {
+		phead = get_list_head(pqueue);
+		plist = get_next(phead);
+
+		pdis_bcn_key = LIST_CONTAINOR(plist, struct dis_bcn_key, list);
+
+		rtw_list_delete(&pdis_bcn_key->list);
+	}
+	if (pdis_bcn_key != NULL)
+		_rtw_memset(&pdis_bcn_key->bcn_content, 0, sizeof(struct beacon_keys));
+
+	return pdis_bcn_key;
+}
+
+struct dis_bcn_key *rtw_alloc_dis_bcn_key(_queue *pidle_queue, _queue *pbusy_queue)
+{
+	struct  dis_bcn_key *pdis_bcn_key = NULL;
+
+	_rtw_spinlock_bh(&pidle_queue->lock);
+	pdis_bcn_key = _rtw_alloc_dis_bcn_key(pidle_queue);
+	_rtw_spinunlock_bh(&pidle_queue->lock);
+
+	if (pdis_bcn_key == NULL) {
+		/*idle queue is empty, recycle resource from busy queue.*/
+		_rtw_spinlock_bh(&pbusy_queue->lock);
+		pdis_bcn_key = _rtw_alloc_dis_bcn_key(pbusy_queue);
+		_rtw_spinunlock_bh(&pbusy_queue->lock);
+	}
+
+	return pdis_bcn_key;
+}
+
+int rtw_enqueue_dis_bcn_key(struct  dis_bcn_key  *pdis_bcn_key, _queue *pbusy_queue)
+{
+	_rtw_spinlock_bh(&pbusy_queue->lock);
+	rtw_list_insert_tail(&(pdis_bcn_key->list), &(pbusy_queue->queue));
+	_rtw_spinunlock_bh(&pbusy_queue->lock);
+
+	return 0;
+}
+#endif
+
+struct _ADAPTER_LINK *rtw_get_adapter_link_by_linkid(_adapter *padapter, u8 link_id)
+{
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		if (padapter_link->mlmepriv.link_id == link_id)
+			break;
+	}
+	return lidx < padapter->adapter_link_num ? padapter_link : NULL;
+}
+
+struct _ADAPTER_LINK *rtw_get_adapter_link_by_hwband(_adapter *padapter, u8 band_idx)
+{
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		if (padapter_link->wrlink->hw_band == band_idx)
+			break;
+	}
+	return lidx < padapter->adapter_link_num ? padapter_link : NULL;
+}
+
+u8 rtw_adapter_link_get_id(struct _ADAPTER_LINK *alink)
+{
+	_adapter *adapter = alink->adapter;
+	u8 i;
+
+	for (i = 0; i < adapter->adapter_link_num; i++)
+		if (GET_LINK(adapter, i) == alink)
+			return i;
+
+	rtw_warn_on(1);
+	return RTW_RLINK_MAX;
+}
+
+struct wlan_network *rtw_clone_network(_adapter *padapter, WLAN_BSSID_EX *pnetwork)
+{
+	struct wlan_network *cloned_network = NULL;
+
+	cloned_network = (struct wlan_network *)rtw_zmalloc(sizeof(struct wlan_network));
+	if (!cloned_network)
+		goto _exit;
+
+	_rtw_memset(cloned_network, 0, sizeof(struct wlan_network));
+	_rtw_memcpy(&cloned_network->network, pnetwork, get_WLAN_BSSID_EX_sz(pnetwork));
+
+_exit:
+	return cloned_network;
+}
+
+#ifdef CONFIG_80211BE_EHT
+int get_link_idx_by_linkid(struct wlan_mld_network *mld_network, u8 link_id)
+{
+	int lidx;
+	struct wlan_network *link_network = NULL;
+
+	if (!mld_network)
+		goto exit;
+
+	for (lidx = 0; lidx < mld_network->network_num; lidx++) {
+		link_network = GET_LINK_NETWORK(mld_network, lidx);
+		if (!link_network)
+			continue;
+		if (link_network->network.link_id == link_id)
+			return lidx;
+	}
+
+exit:
+	return -1;
+}
+
+struct wlan_network *rtw_get_link_network_by_linkid(struct wlan_mld_network *mld_network, u8 link_id)
+{
+	int lidx;
+
+	lidx = get_link_idx_by_linkid(mld_network, link_id);
+
+	return ((lidx < 0) ? NULL : GET_LINK_NETWORK(mld_network, lidx));
+}
+
+void rtw_unlink_network_by_linkid(struct wlan_mld_network *mld_network, u8 link_id)
+{
+	int lidx;
+	struct wlan_network *link_network = NULL;
+
+	lidx = get_link_idx_by_linkid(mld_network, link_id);
+
+	if (lidx >= 0) {
+		link_network = GET_LINK_NETWORK(mld_network, lidx);
+		link_network->network.mld_network = NULL;
+		mld_network->link_network[lidx] = NULL;
+	}
+}
+
+void rtw_link_mld_network(_adapter *padapter, struct wlan_mld_network *mld_network, struct wlan_network *network)
+{
+	if (!mld_network)
+		goto _exit;
+
+	if (!network)
+		goto _exit;
+
+	if (mld_network->network_num >= RTW_NETWORK_LINK_MAX)
+		goto _exit;
+
+	mld_network->link_network[mld_network->network_num] = network;
+	network->network.mld_network = mld_network;
+	mld_network->network_num++;
+
+_exit:
+	return;
+}
+
+struct wlan_mld_network *rtw_clone_to_join_mld_network(_adapter *padapter, struct wlan_mld_network *pmld_network)
+{
+	struct _ADAPTER_LINK *padapter_link;
+	struct wlan_mld_network *cloned_mld_network = NULL;
+	struct wlan_network *link_network;
+	struct wlan_network *cloned_network = NULL;
+	u8 lidx;
+
+	cloned_mld_network = (struct wlan_mld_network *)rtw_zmalloc(sizeof(struct wlan_mld_network));
+	if (!cloned_mld_network)
+		goto _exit;
+
+	_rtw_memset(cloned_mld_network, 0, sizeof(struct wlan_mld_network));
+	_rtw_memcpy(cloned_mld_network->mac_addr, pmld_network->mac_addr, ETH_ALEN);
+	/* ToDo CONFIG_RTW_MLD: need to store additional infos, such as EML and MLD capability */
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		if (!padapter_link->mlmepriv.to_join)
+			continue;
+
+		link_network = rtw_get_link_network_by_linkid(pmld_network, padapter_link->mlmepriv.link_id);
+		if (!link_network) {
+			RTW_INFO(FUNC_ADPT_FMT" network is NULL for link_id %u, set adapter_link %d to_join to _FALSE\n"
+				, FUNC_ADPT_ARG(padapter), padapter_link->mlmepriv.link_id, padapter_link->wrlink->id);
+			padapter_link->mlmepriv.to_join = _FALSE;
+			continue;
+		}
+
+		cloned_network = rtw_clone_network(padapter, &link_network->network);
+		if (!cloned_network) {
+			RTW_INFO(FUNC_ADPT_FMT" cannot allocate link network for link_id %u, set adapter_link %d to_join to _FALSE\n"
+				, FUNC_ADPT_ARG(padapter), padapter_link->mlmepriv.link_id, padapter_link->wrlink->id);
+			padapter_link->mlmepriv.to_join = _FALSE;
+			continue;
+		}
+		rtw_link_mld_network(padapter, cloned_mld_network, cloned_network);
+	}
+
+_exit:
+	return cloned_mld_network;
+}
+
+void rtw_free_cloned_mld_network(struct wlan_mld_network *mld_network)
+{
+	struct wlan_network *network;
+	u8 lidx;
+
+	if (!mld_network)
+		return;
+
+	for (lidx = 0; lidx < mld_network->network_num; lidx++) {
+		network = GET_LINK_NETWORK(mld_network, lidx);
+		if (network)
+			rtw_mfree((unsigned char *)network, sizeof(struct wlan_network));
+	}
+	rtw_mfree((unsigned char *)mld_network, sizeof(struct wlan_mld_network));
+}
+#endif
+
+struct wlan_mld_network *_rtw_alloc_mld_network(struct mlme_priv *pmlmepriv, const u8 *mac_addr)
+{
+	struct	wlan_mld_network	*pmld_network;
+	_queue *free_queue = &pmlmepriv->free_mld_bss_pool;
+	_list *plist = NULL;
+
+	_rtw_spinlock_bh(&free_queue->lock);
+
+	if (_rtw_queue_empty(free_queue) == _TRUE) {
+		pmld_network = NULL;
+		goto exit;
+	}
+	plist = get_next(&(free_queue->queue));
+
+	pmld_network = LIST_CONTAINOR(plist , struct wlan_mld_network, list);
+	_rtw_memcpy(pmld_network->mac_addr, mac_addr, ETH_ALEN);
+	pmld_network->network_num = 0;
+
+	rtw_list_delete(&pmld_network->list);
+
+	pmlmepriv->num_of_scanned_mld++;
+
+exit:
+	_rtw_spinunlock_bh(&free_queue->lock);
+
+	return pmld_network;
+}
+
+struct wlan_mld_network *rtw_alloc_mld_network(_adapter *adapter, const u8 *mac_addr)
+{
+	struct mlme_priv    *pmlmepriv = &(adapter->mlmepriv);
+	_queue	*queue	= &(pmlmepriv->scanned_mld_queue);
+	struct wlan_mld_network *pmld_network = NULL;
+	_list	*plist, *phead;
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+	while (1) {
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pmld_network = LIST_CONTAINOR(plist, struct wlan_mld_network, list);
+
+		if (_rtw_memcmp(pmld_network->mac_addr, mac_addr, ETH_ALEN))
+			break;
+
+		pmld_network = NULL;
+		plist = get_next(plist);
+
+	}
+
+	if(pmld_network == NULL) {
+		pmld_network = _rtw_alloc_mld_network(&(adapter->mlmepriv), mac_addr);
+		if (pmld_network)
+			rtw_list_insert_tail(&(pmld_network->list), &(queue->queue));
+	}
+
+	return pmld_network;
+}
+
+void _rtw_free_mld_network(struct mlme_priv *pmlmepriv, struct wlan_mld_network *pmld_network, u8 isfreeall)
+{
+	u32 delta_time;
+	u32 lifetime = SCANQUEUE_LIFETIME;
+	_queue *free_queue = &(pmlmepriv->free_mld_bss_pool);
+	struct wlan_network *pnetwork;
+	u8 lidx;
+
+	if (pmld_network == NULL)
+		goto exit;
+
+	if (!isfreeall) {
+		delta_time = (u32) rtw_get_passing_time_ms(pmld_network->last_scanned);
+		if (delta_time < lifetime) /* unit:msec */
+			goto exit;
+
+		for (lidx = 0; lidx < pmld_network->network_num; lidx++) {
+			pnetwork = GET_LINK_NETWORK(pmld_network, lidx);
+			if (pnetwork)
+				_rtw_free_network_nolock(pmlmepriv, pnetwork);
+		}
+	}
+	pmld_network->network_num = 0;
+
+	_rtw_spinlock_bh(&free_queue->lock);
+
+	rtw_list_delete(&(pmld_network->list));
+
+	rtw_list_insert_tail(&(pmld_network->list), &(free_queue->queue));
+
+	pmlmepriv->num_of_scanned_mld--;
+
+
+	/* RTW_INFO("_rtw_free_network:SSID=%s\n", pnetwork->network.Ssid.Ssid); */
+
+	_rtw_spinunlock_bh(&free_queue->lock);
+
+exit:
+	return;
+}
+
+void _rtw_free_mld_network_nolock(struct mlme_priv *pmlmepriv, struct wlan_mld_network *pmld_network)
+{
+
+	_queue *free_queue = &(pmlmepriv->free_mld_bss_pool);
+
+	if (pmld_network == NULL)
+		goto exit;
+
+	/* _rtw_spinlock_irq(&free_queue->lock, &sp_flags); */
+
+	rtw_list_delete(&(pmld_network->list));
+
+	rtw_list_insert_tail(&(pmld_network->list), get_list_head(free_queue));
+
+	pmlmepriv->num_of_scanned_mld--;
+
+	/* _rtw_spinunlock_irq(&free_queue->lock, &sp_flags); */
+
+exit:
+	return;
+}
+
+void _rtw_free_mld_network_queue(_adapter *padapter, u8 isfreeall)
+{
+
+	_list *phead, *plist;
+	struct wlan_mld_network *pmld_network;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	_queue *scanned_mld_queue = &pmlmepriv->scanned_mld_queue;
+	_queue *scanned_queue = &pmlmepriv->scanned_queue;
+
+
+
+	_rtw_spinlock_bh(&scanned_queue->lock);
+	_rtw_spinlock_bh(&scanned_mld_queue->lock);
+
+	phead = get_list_head(scanned_mld_queue);
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+
+		pmld_network = LIST_CONTAINOR(plist, struct wlan_mld_network, list);
+
+		plist = get_next(plist);
+
+		_rtw_free_mld_network(pmlmepriv, pmld_network, isfreeall);
+
+	}
+
+	_rtw_spinunlock_bh(&scanned_mld_queue->lock);
+	_rtw_spinunlock_bh(&scanned_queue->lock);
+
+
+}
+
+void rtw_free_mld_network(struct mlme_priv *pmlmepriv, struct wlan_mld_network *pmld_network, u8 is_freeall)
+{
+	_rtw_free_mld_network(pmlmepriv, pmld_network, is_freeall);
+}
+
+void rtw_free_mld_network_nolock(_adapter *padapter, struct wlan_mld_network *pmld_network)
+{
+	_rtw_free_mld_network_nolock(&(padapter->mlmepriv), pmld_network);
+}
+
+void rtw_free_mld_network_queue(_adapter *dev, u8 isfreeall)
+{
+	_rtw_free_mld_network_queue(dev, isfreeall);
+}
+
+static void rtw_joinbss_update_link_network(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		struct wlan_network *ptarget_wlan, struct wlan_network *pnetwork)
+{
+	struct link_mlme_priv	*pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_security_priv *psecuritypriv = &(padapter_link->securitypriv);
+	struct wlan_network  *cur_network = &(pmlmepriv->cur_network);
+	sint tmp_fw_state = 0x0;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/* why not use ptarget_wlan?? */
+	_rtw_memcpy(&cur_network->network, &pnetwork->network, pnetwork->network.Length);
+	/* some IEs in pnetwork is wrong, so we should use ptarget_wlan IEs */
+	cur_network->network.IELength = ptarget_wlan->network.IELength;
+	_rtw_memcpy(&cur_network->network.IEs[0], &ptarget_wlan->network.IEs[0], MAX_IE_SZ);
+
+	cur_network->aid = padapter->mlmepriv.dev_cur_network.aid;
+
+#ifdef CONFIG_80211N_HT
+	rtw_update_link_ht_cap(padapter, padapter_link, cur_network->network.IEs, cur_network->network.IELength, (u8) cur_network->network.Configuration.DSConfig);
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	rtw_update_link_he_6ghz_cap(padapter, padapter_link,
+			cur_network->network.IEs,
+			cur_network->network.IELength,
+			cur_network->network.Configuration.Band,
+			cur_network->network.Configuration.DSConfig);
+#endif
+}
+
+void rtw_clear_to_join_status(_adapter *padapter) {
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+
+	padapter->mlmepriv.to_join = _FALSE;
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		padapter_link->mlmepriv.to_join = _FALSE;
+	}
+}
+
+void rtw_clear_is_accepted_status(_adapter *padapter) {
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		padapter_link->mlmepriv.is_accepted = _FALSE;
+	}
+}
+
+struct wlan_network *_rtw_find_network(_queue *scanned_queue, const u8 *addr)
+{
+	_list	*phead, *plist;
+	struct	wlan_network *pnetwork = NULL;
+	u8 zero_addr[ETH_ALEN] = {0, 0, 0, 0, 0, 0};
+
+	if (_rtw_memcmp(zero_addr, addr, ETH_ALEN)) {
+		pnetwork = NULL;
+		goto exit;
+	}
+
+	phead = get_list_head(scanned_queue);
+	plist = get_next(phead);
+
+	while (plist != phead) {
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network , list);
+
+		if (_rtw_memcmp(addr, pnetwork->network.MacAddress, ETH_ALEN) == _TRUE)
+			break;
+
+		plist = get_next(plist);
+	}
+
+	if (plist == phead)
+		pnetwork = NULL;
+
+exit:
+	return pnetwork;
+}
+
+struct wlan_network *rtw_find_network(_queue *scanned_queue, const u8 *addr)
+{
+	struct	wlan_network *pnetwork;
+
+	 _rtw_spinlock_bh(&scanned_queue->lock);
+	pnetwork = _rtw_find_network(scanned_queue, addr);
+	_rtw_spinunlock_bh(&scanned_queue->lock);
+
+	return pnetwork;
+}
+
+int rtw_is_same_ibss(_adapter *adapter, struct wlan_network *pnetwork)
+{
+	int ret = _TRUE;
+	struct security_priv *psecuritypriv = &adapter->securitypriv;
+
+	if ((psecuritypriv->dot11PrivacyAlgrthm != _NO_PRIVACY_) &&
+	    (pnetwork->network.Privacy == 0))
+		ret = _FALSE;
+	else if ((psecuritypriv->dot11PrivacyAlgrthm == _NO_PRIVACY_) &&
+		 (pnetwork->network.Privacy == 1))
+		ret = _FALSE;
+	else
+		ret = _TRUE;
+
+	return ret;
+
+}
+
+inline int is_same_ess(WLAN_BSSID_EX *a, WLAN_BSSID_EX *b)
+{
+	return (a->Ssid.SsidLength == b->Ssid.SsidLength)
+	       &&  _rtw_memcmp(a->Ssid.Ssid, b->Ssid.Ssid, a->Ssid.SsidLength) == _TRUE;
+}
+
+int is_same_network(WLAN_BSSID_EX *src, WLAN_BSSID_EX *dst)
+{
+	u16 s_cap, d_cap;
+#ifdef CONFIG_80211BE_EHT
+	u8 s_mld_addr[ETH_ALEN];
+	u8 d_mld_addr[ETH_ALEN];
+	u8 eid_eht_ml = EID_EXT_MULTI_LINK;
+	u32 len;
+	u8 *p;
+#endif
+
+
+	_rtw_memcpy((u8 *)&s_cap, rtw_get_capability_from_ie(src->IEs), 2);
+	_rtw_memcpy((u8 *)&d_cap, rtw_get_capability_from_ie(dst->IEs), 2);
+
+
+	s_cap = le16_to_cpu(s_cap);
+	d_cap = le16_to_cpu(d_cap);
+
+	/* Wi-Fi driver doesn't consider the situation of BCN and ProbRsp sent from the same hidden AP,
+	  * it considers these two packets are sent from different AP.
+	  * Therefore, the scan queue may store two scan results of the same hidden AP, likes below.
+	  *
+	  *  index            bssid              ch    RSSI   SdBm  Noise   age          flag             ssid
+	  *    1    00:e0:4c:55:50:01    153   -73     -73        0     7044   [WPS][ESS]     RTK5G
+	  *    3    00:e0:4c:55:50:01    153   -73     -73        0     7044   [WPS][ESS]
+	  *
+	  * Original rules will compare Ssid, SsidLength, MacAddress, s_cap, d_cap at the same time.
+	  * Wi-Fi driver will assume that the BCN and ProbRsp sent from the same hidden AP are the same network
+	  * after we add an additional rule to compare SsidLength and Ssid.
+	  * It means the scan queue will not store two scan results of the same hidden AP, it only store ProbRsp.
+	  * For customer request.
+	  */
+
+	if (src->is_mld == dst->is_mld) {
+#ifdef CONFIG_80211BE_EHT
+		if (src->is_mld) {
+			p = rtw_get_ie_ex(src->IEs + _FIXED_IE_LENGTH_, src->IELength - _FIXED_IE_LENGTH_,
+				WLAN_EID_EXTENSION, &eid_eht_ml, 1, NULL, &len);
+			_rtw_memcpy(s_mld_addr, p + 3 + EHT_ML_CONTROL_LEN + 1, ETH_ALEN);
+			p = rtw_get_ie_ex(dst->IEs + _FIXED_IE_LENGTH_, dst->IELength - _FIXED_IE_LENGTH_,
+				WLAN_EID_EXTENSION, &eid_eht_ml, 1, NULL, &len);
+			_rtw_memcpy(d_mld_addr, p + 3 + EHT_ML_CONTROL_LEN + 1, ETH_ALEN);
+		}
+#endif
+		if (((_rtw_memcmp(src->MacAddress, dst->MacAddress, ETH_ALEN)) == _TRUE) &&
+			(
+#ifdef CONFIG_80211BE_EHT
+			/* MLD network may haven't update capability
+			** if the collected link info is only from the rediced neighbor report.
+			** Hence compare the mld mac addr instead of the capability */
+			(src->is_mld && (_rtw_memcmp(s_mld_addr, d_mld_addr, ETH_ALEN) == _TRUE)) ||
+#endif
+			(((s_cap & WLAN_CAPABILITY_IBSS) == (d_cap & WLAN_CAPABILITY_IBSS)) &&
+			((s_cap & WLAN_CAPABILITY_BSS) == (d_cap & WLAN_CAPABILITY_BSS)))
+			)) {
+			if ((src->Ssid.SsidLength == dst->Ssid.SsidLength) &&
+				(((_rtw_memcmp(src->Ssid.Ssid, dst->Ssid.Ssid, src->Ssid.SsidLength)) == _TRUE) || //Case of normal AP
+				(is_all_null(src->Ssid.Ssid, src->Ssid.SsidLength) == _TRUE || is_all_null(dst->Ssid.Ssid, dst->Ssid.SsidLength) == _TRUE))) //Case of hidden AP
+				return _TRUE;
+			else if ((src->Ssid.SsidLength == 0 || dst->Ssid.SsidLength == 0)) //Case of hidden AP
+				return _TRUE;
+			else
+				return _FALSE;
+		} else {
+			return _FALSE;
+		}
+	} else {
+		return _FALSE;
+	}
+}
+
+struct wlan_network *_rtw_find_same_network(_queue *scanned_queue, struct wlan_network *network)
+{
+	_list *phead, *plist;
+	struct wlan_network *found = NULL;
+
+	phead = get_list_head(scanned_queue);
+	plist = get_next(phead);
+
+	while (plist != phead) {
+		found = LIST_CONTAINOR(plist, struct wlan_network , list);
+
+		if (is_same_network(&network->network, &found->network))
+			break;
+
+		plist = get_next(plist);
+	}
+
+	if (plist == phead)
+		found = NULL;
+
+	return found;
+}
+
+struct wlan_network *rtw_find_same_network(_queue *scanned_queue, struct wlan_network *network)
+{
+	struct wlan_network *found = NULL;
+
+	if (scanned_queue == NULL || network == NULL)
+		goto exit;
+
+	_rtw_spinlock_bh(&scanned_queue->lock);
+	found = _rtw_find_same_network(scanned_queue, network);
+	_rtw_spinunlock_bh(&scanned_queue->lock);
+
+exit:
+	return found;
+}
+
+struct	wlan_network	*rtw_get_oldest_wlan_network(_queue *scanned_queue)
+{
+	_list	*plist, *phead;
+
+
+	struct	wlan_network	*pwlan = NULL;
+	struct	wlan_network	*oldest = NULL;
+	phead = get_list_head(scanned_queue);
+
+	plist = get_next(phead);
+
+	while (1) {
+
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pwlan = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+		if (pwlan->fixed != _TRUE) {
+			if (oldest == NULL || rtw_time_after(oldest->last_scanned, pwlan->last_scanned))
+				oldest = pwlan;
+		}
+
+		plist = get_next(plist);
+	}
+	return oldest;
+
+}
+
+void rtw_update_adapter_network(WLAN_BSSID_EX *dst, WLAN_BSSID_EX *src, _adapter *padapter, bool update_ie)
+{
+	/* update the primary link network to adapter */
+	rtw_update_network(dst, src, padapter, update_ie);
+#ifdef CONFIG_80211BE_EHT
+	if (src->is_mld && update_ie) {
+		u8 *p;
+		u8 eid_eht_ml = EID_EXT_MULTI_LINK;
+		u32 eht_ml_ie_len;
+
+		p = rtw_get_ie_ex(src->IEs + _BEACON_IE_OFFSET_,
+				src->IELength - _BEACON_IE_OFFSET_,
+				WLAN_EID_EXTENSION, &eid_eht_ml, 1, NULL, &eht_ml_ie_len);
+
+		if ((p != NULL) && (eht_ml_ie_len >= EHT_ML_BASIC_MIN_LEN)) {
+			/* Update MLD Mac address */
+			_rtw_memcpy(dst->MacAddress, p + 3 + EHT_ML_CONTROL_LEN + 1, ETH_ALEN);
+
+			/* ToDo CONFIG_RTW_MLD: record other mld-related capabilities */
+		}
+	}
+#endif
+}
+
+void rtw_update_network(WLAN_BSSID_EX *dst, WLAN_BSSID_EX *src,
+		    _adapter *padapter, bool update_ie)
+{
+	struct _ADAPTER_LINK *padapter_link = NULL;
+#if defined(DBG_RX_SIGNAL_DISPLAY_SSID_MONITORED) && 1
+	u8 ss_ori = dst->PhyInfo.SignalStrength;
+	u8 sq_ori = dst->PhyInfo.SignalQuality;
+	u8 ss_smp = src->PhyInfo.SignalStrength;
+	s8 rssi_smp = src->PhyInfo.rssi;
+#endif
+	s8 rssi_new = src->PhyInfo.rssi;
+	s8 rssi_ori = dst->PhyInfo.rssi;
+	u8 sq_smp = src->PhyInfo.SignalQuality;
+	u8 ss_final = 0;
+	u8 sq_final = 0;
+	s8 rssi_final = 0;
+	u8 dst_rssi_is_valid = true;
+	u8 update_case = 0;
+
+
+#ifdef CONFIG_ANTENNA_DIVERSITY
+	rtw_hal_antdiv_rssi_compared(padapter, dst, src); /* this will update src->PhyInfo.rssi, need consider again */
+#endif
+
+#if defined(DBG_RX_SIGNAL_DISPLAY_SSID_MONITORED) && 1
+	if (strcmp(dst->Ssid.Ssid, DBG_RX_SIGNAL_DISPLAY_SSID_MONITORED) == 0) {
+		RTW_INFO(FUNC_ADPT_FMT" %s("MAC_FMT", ch%u) ss_ori:%3u, sq_ori:%3u, rssi_ori:%d, ss_smp:%3u, sq_smp:%3u, rssi_smp:%d\n"
+			 , FUNC_ADPT_ARG(padapter)
+			, src->Ssid.Ssid, MAC_ARG(src->MacAddress), src->Configuration.DSConfig
+			 , ss_ori, sq_ori, rssi_ori
+			 , ss_smp, sq_smp, rssi_smp
+			);
+	}
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+	if (src->is_mld)
+		padapter_link = rtw_get_adapter_link_by_linkid(padapter, src->link_id);
+#endif
+	if (!padapter_link)
+		padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (src->PhyInfo.isValid && dst->PhyInfo.isValid && sq_smp != 101) {
+		/* Receiving the valid RSSI from correct channel and the RSSI stored in scan queue is also valid */
+		update_case = 1;
+	} else if (src->PhyInfo.isValid && !dst->PhyInfo.isValid) {
+		/* Replace the invalid RSSI stored in scan queue with the received valid RSSI even not from correct channel */
+		update_case = 2;
+	} else {
+		/* Don't update RSSI, use original RSSI */
+		update_case = 3;
+	}
+
+	/* The rule below is 1/5 for sample value, 4/5 for history value */
+	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) && is_same_network(&(padapter_link->mlmepriv.cur_network.network), src)) {
+		/* Take the recvpriv's value for the connected AP*/
+		ss_final = padapter->recvinfo.signal_strength;
+		sq_final = padapter->recvinfo.signal_qual;
+		/* the rssi value here is undecorated, and will be used for antenna diversity */
+		switch (update_case) {
+		case 1:
+			rssi_final = (rssi_new + rssi_ori * 4) / 5;
+			break;
+		case 2:
+			rssi_final = rssi_new;
+			break;
+		case 3:
+			rssi_final = rssi_ori;
+			dst_rssi_is_valid = dst->PhyInfo.isValid; /* backup is_valid of dst */
+			break;
+		}
+	} else {
+		switch (update_case) {
+		case 1:
+			ss_final = ((u32)(src->PhyInfo.SignalStrength) + (u32)(dst->PhyInfo.SignalStrength) * 4) / 5;
+			sq_final = ((u32)(src->PhyInfo.SignalQuality) + (u32)(dst->PhyInfo.SignalQuality) * 4) / 5;
+			rssi_final = (rssi_new + rssi_ori * 4) / 5;
+			break;
+		case 2:
+			ss_final = src->PhyInfo.SignalStrength;
+			sq_final = src->PhyInfo.SignalQuality;
+			rssi_final = rssi_new;
+			break;
+		case 3:
+			ss_final = dst->PhyInfo.SignalStrength;
+			sq_final = dst->PhyInfo.SignalQuality;
+			rssi_final = rssi_ori;
+			dst_rssi_is_valid = dst->PhyInfo.isValid; /* backup is_valid of dst */
+			break;
+		}
+	}
+
+	if (update_ie) {
+		dst->Reserved[0] = src->Reserved[0];
+		dst->Reserved[1] = src->Reserved[1];
+		_rtw_memcpy((u8 *)dst, (u8 *)src, get_WLAN_BSSID_EX_sz(src));
+	}
+
+	dst->PhyInfo.SignalStrength = ss_final;
+	dst->PhyInfo.SignalQuality = sq_final;
+	dst->PhyInfo.rssi = rssi_final;
+	dst->PhyInfo.isValid = dst_rssi_is_valid;
+
+#if defined(DBG_RX_SIGNAL_DISPLAY_SSID_MONITORED) && 1
+	if (strcmp(dst->Ssid.Ssid, DBG_RX_SIGNAL_DISPLAY_SSID_MONITORED) == 0) {
+		RTW_INFO(FUNC_ADPT_FMT" %s("MAC_FMT"), SignalStrength:%u, SignalQuality:%u, RawRSSI:%d\n"
+			 , FUNC_ADPT_ARG(padapter)
+			, dst->Ssid.Ssid, MAC_ARG(dst->MacAddress),
+			dst->PhyInfo.SignalStrength, dst->PhyInfo.SignalQuality, dst->PhyInfo.rssi);
+	}
+#endif
+}
+
+#ifdef CONFIG_RTW_MULTI_AP
+void rtw_map_config_monitor_act_non(_adapter *adapter)
+{
+	rtw_map_config_monitor(adapter, MLME_ACTION_NONE);
+}
+
+void rtw_map_config_monitor(_adapter *adapter, u8 self_act)
+{
+	struct mi_state mstate, mstate_s;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(adapter));
+
+	rtw_mi_status_no_self(adapter, &mstate);
+	rtw_mi_status_no_others(adapter, &mstate_s);
+
+	switch (self_act) {
+	case MLME_AP_STARTED:
+		mstate_s.ap_num = 1;
+		break;
+	case MLME_AP_STOPPED:
+		mstate_s.ap_num = 0;
+		break;
+	case MLME_ACTION_NONE:
+		break;
+	default:
+		break;
+	}
+
+	rtw_mi_status_merge(&mstate, &mstate_s);
+	if (MSTATE_AP_NUM(&mstate) &&
+	    rtw_unassoc_sta_src_chk(adapter, UNASOC_STA_SRC_RX_NMY_UC))
+		rtw_phl_enter_mon_mode(phl, adapter->phl_role);
+	else
+		rtw_phl_leave_mon_mode(phl, adapter->phl_role);
+}
+
+void rtw_unassoc_sta_set_mode(_adapter *adapter, u8 stype, u8 mode)
+{
+	if (stype >= UNASOC_STA_SRC_NUM || mode >= UNASOC_STA_MODE_NUM)
+		return;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	if (adapter->mlmepriv.unassoc_sta_mode_of_stype[stype] == mode)
+		return;
+
+	adapter->mlmepriv.unassoc_sta_mode_of_stype[stype] = mode;
+	rtw_run_in_thread_cmd_wait(adapter, ((void *)(rtw_map_config_monitor_act_non)), adapter, 2000);
+}
+
+bool rtw_unassoc_sta_src_chk(_adapter *adapter, u8 stype)
+{
+	if (stype >= UNASOC_STA_SRC_NUM)
+		return 0;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	return adapter->mlmepriv.unassoc_sta_mode_of_stype[stype] == UNASOC_STA_MODE_ALL
+		|| (adapter->mlmepriv.unassoc_sta_mode_of_stype[stype] == UNASOC_STA_MODE_INTERESTED
+		&& adapter->mlmepriv.interested_unassoc_sta_cnt);
+}
+
+const char *unasoc_sta_src_str[] = {
+	"BMC",
+	"NMY_UC",
+};
+
+const char *unasoc_sta_mode_str[] = {
+	"DISABLED",
+	"INTERESTED",
+	"ALL",
+};
+
+void dump_unassoc_sta(void *sel, _adapter *adapter)
+{
+	struct mlme_priv *mlmepriv;
+	_queue *queue;
+	_list *list, *head;
+	struct unassoc_sta_info **unassoc_sta_arr;
+	struct unassoc_sta_info *unassoc_sta;
+	u16 i, unassoc_sta_cnt = 0;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	mlmepriv = &(adapter->mlmepriv);
+	queue = &(mlmepriv->unassoc_sta_queue);
+	for (i = 0; i < UNASOC_STA_SRC_NUM; i++) {
+		RTW_PRINT_SEL(sel, "[%u]%-6s:%u(%s)\n", i, unasoc_sta_src_str[i],
+			      mlmepriv->unassoc_sta_mode_of_stype[i],
+			      unasoc_sta_mode_str[mlmepriv->unassoc_sta_mode_of_stype[i]]);
+	}
+
+	RTW_PRINT_SEL(sel, "interested_unassoc_sta_cnt:%u\n",
+		      mlmepriv->interested_unassoc_sta_cnt);
+
+	unassoc_sta_arr = rtw_zvmalloc(mlmepriv->max_unassoc_sta_cnt * sizeof(struct unassoc_sta_info *));
+	if (!unassoc_sta_arr)
+		return;
+
+	_rtw_spinlock_bh(&queue->lock);
+	head = get_list_head(queue);
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		unassoc_sta = LIST_CONTAINOR(list, struct unassoc_sta_info, list);
+		list = get_next(list);
+		unassoc_sta_arr[unassoc_sta_cnt++] = unassoc_sta;
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+	RTW_PRINT_SEL(sel, "  %17s %18s %6s\n", "mac_addr", "measure_delta_time", "rssi");
+
+	for (i = 0; i < unassoc_sta_cnt; i++) {
+		u8 rcpi;
+		s8 rx_power;
+		u32 measure_delta_time;
+		char mac_addr_str[MAC_FMT_LEN];
+
+		unassoc_sta = unassoc_sta_arr[i];
+		measure_delta_time = rtw_systime_to_ms(rtw_get_current_time() - unassoc_sta->time);
+
+		RTW_PRINT_SEL(sel, "%c %s %18u %6d\n"
+			      , unassoc_sta->interested ? '*' : ' '
+			      , get_macaddr_str(mac_addr_str, sel, unassoc_sta->addr)
+			      , measure_delta_time, unassoc_sta->recv_signal_power);
+	}
+
+	rtw_vmfree(unassoc_sta_arr, mlmepriv->max_unassoc_sta_cnt * sizeof(struct unassoc_sta_info *));
+}
+
+static void del_unassoc_sta(struct mlme_priv *mlmepriv, struct unassoc_sta_info *unassoc_sta)
+{
+	_queue *free_queue = &(mlmepriv->free_unassoc_sta_queue);
+
+	if (unassoc_sta->interested)
+		mlmepriv->interested_unassoc_sta_cnt--;
+
+	if (mlmepriv->interested_unassoc_sta_cnt == 0) {
+		rtw_run_in_thread_cmd(mlme_to_adapter(mlmepriv)
+			, ((void *)(rtw_map_config_monitor_act_non)), mlme_to_adapter(mlmepriv));
+	}
+
+	_rtw_spinlock_bh(&free_queue->lock);
+	rtw_list_delete(&(unassoc_sta->list));
+	rtw_list_insert_tail(&(unassoc_sta->list), &(free_queue->queue));
+	_rtw_spinunlock_bh(&free_queue->lock);
+}
+
+static u8 del_unassoc_sta_chk(struct mlme_priv *mlmepriv, struct unassoc_sta_info *unassoc_sta)
+{
+	systime cur, lifetime;
+
+	if (unassoc_sta == NULL)
+		return UNASOC_STA_DEL_CHK_SKIP;
+
+	if (unassoc_sta->interested)
+		return UNASOC_STA_DEL_CHK_SKIP;
+
+	cur = rtw_get_current_time();
+	lifetime = unassoc_sta->time + rtw_ms_to_systime(UNASSOC_STA_LIFETIME_MS);
+	if (rtw_time_before(cur, lifetime))
+		return UNASOC_STA_DEL_CHK_ALIVE;
+
+	del_unassoc_sta(mlmepriv, unassoc_sta);
+	return UNASOC_STA_DEL_CHK_DELETED;
+}
+
+static struct unassoc_sta_info *alloc_unassoc_sta(struct mlme_priv *mlmepriv)
+{
+	struct	unassoc_sta_info *unassoc_sta;
+	_queue *free_queue = &mlmepriv->free_unassoc_sta_queue;
+	_list *list = NULL;
+
+	_rtw_spinlock_bh(&free_queue->lock);
+	if (_rtw_queue_empty(free_queue) == _TRUE) {
+		unassoc_sta = NULL;
+		goto exit;
+	}
+
+	list = get_next(&(free_queue->queue));
+	unassoc_sta = LIST_CONTAINOR(list, struct unassoc_sta_info, list);
+	rtw_list_delete(&unassoc_sta->list);
+	_rtw_memset(unassoc_sta->addr, 0, ETH_ALEN);
+	unassoc_sta->recv_signal_power = 0;
+	unassoc_sta->time = 0;
+	unassoc_sta->interested = 0;
+exit:
+	_rtw_spinunlock_bh(&free_queue->lock);
+	return unassoc_sta;
+}
+
+void rtw_del_unassoc_sta_queue(_adapter *adapter)
+{
+	struct unassoc_sta_info *unassoc_sta;
+	struct mlme_priv *mlmepriv;
+	_queue *queue;
+	_list *head, *list;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	mlmepriv = &(adapter->mlmepriv);
+	queue = &(mlmepriv->unassoc_sta_queue);
+
+	_rtw_spinlock_bh(&queue->lock);
+	head = get_list_head(queue);
+	list = get_next(head);
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		unassoc_sta = LIST_CONTAINOR(list , struct unassoc_sta_info, list);
+		list = get_next(list);
+		del_unassoc_sta(mlmepriv, unassoc_sta);
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+}
+
+void rtw_del_unassoc_sta(_adapter *adapter, u8 *addr)
+{
+	struct unassoc_sta_info *unassoc_sta;
+	struct mlme_priv *mlmepriv;
+	_queue *queue;
+	_list *head, *list;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	mlmepriv = &(adapter->mlmepriv);
+	queue = &(mlmepriv->unassoc_sta_queue);
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	head = get_list_head(queue);
+	list = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		unassoc_sta = LIST_CONTAINOR(list , struct unassoc_sta_info, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(addr, unassoc_sta->addr, ETH_ALEN) == _TRUE) {
+			del_unassoc_sta(mlmepriv, unassoc_sta);
+			goto unlock_unassoc_sta_queue;
+		}
+	}
+
+unlock_unassoc_sta_queue:
+	_rtw_spinunlock_bh(&queue->lock);
+}
+
+void rtw_rx_add_unassoc_sta(_adapter *adapter, u8 stype, u8 *addr, s8 recv_signal_power)
+{
+	struct unassoc_sta_info *unassoc_sta;
+	struct unassoc_sta_info *oldest_unassoc_sta = NULL;
+	struct mlme_priv *mlmepriv;
+	_queue *queue;
+	_list *head, *list;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	mlmepriv = &(adapter->mlmepriv);
+	queue = &(mlmepriv->unassoc_sta_queue);
+
+	_rtw_spinlock_bh(&queue->lock);
+	head = get_list_head(queue);
+	list = get_next(head);
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		unassoc_sta = LIST_CONTAINOR(list , struct unassoc_sta_info, list);
+		list = get_next(list);
+		if (_rtw_memcmp(addr, unassoc_sta->addr, ETH_ALEN) == _TRUE) {
+			if (unassoc_sta->interested ||
+			    mlmepriv->unassoc_sta_mode_of_stype[stype] >=
+			    UNASOC_STA_MODE_ALL) {
+				unassoc_sta->recv_signal_power = recv_signal_power;
+				unassoc_sta->time = rtw_get_current_time();
+				goto unlock_unassoc_sta_queue;
+			}
+		}
+
+		if (del_unassoc_sta_chk(mlmepriv, unassoc_sta) == UNASOC_STA_DEL_CHK_ALIVE) {
+			if (oldest_unassoc_sta == NULL)
+				oldest_unassoc_sta = unassoc_sta;
+			else if (rtw_time_before(unassoc_sta->time, oldest_unassoc_sta->time))
+				oldest_unassoc_sta = unassoc_sta;
+		}
+	}
+
+	if (mlmepriv->unassoc_sta_mode_of_stype[stype] <= UNASOC_STA_MODE_INTERESTED)
+		goto unlock_unassoc_sta_queue;
+
+	unassoc_sta = alloc_unassoc_sta(mlmepriv);
+	if (unassoc_sta == NULL) {
+		if (oldest_unassoc_sta) {
+			del_unassoc_sta(mlmepriv, oldest_unassoc_sta);
+			unassoc_sta = alloc_unassoc_sta(mlmepriv);
+		} else
+			goto unlock_unassoc_sta_queue;
+	}
+
+	_rtw_memcpy(unassoc_sta->addr, addr, ETH_ALEN);
+	unassoc_sta->recv_signal_power = recv_signal_power;
+	unassoc_sta->time = rtw_get_current_time();
+	rtw_list_insert_tail(&(unassoc_sta->list), &(queue->queue));
+
+unlock_unassoc_sta_queue:
+	_rtw_spinunlock_bh(&queue->lock);
+}
+
+void rtw_add_interested_unassoc_sta(_adapter *adapter, u8 *addr)
+{
+	struct unassoc_sta_info *unassoc_sta;
+	struct unassoc_sta_info *oldest_unassoc_sta = NULL;
+	struct mlme_priv *mlmepriv;
+	_queue *queue;
+	_list *head, *list;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	mlmepriv = &(adapter->mlmepriv);
+	queue = &(mlmepriv->unassoc_sta_queue);
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	head = get_list_head(queue);
+	list = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		unassoc_sta = LIST_CONTAINOR(list , struct unassoc_sta_info, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(addr, unassoc_sta->addr, ETH_ALEN) == _TRUE) {
+			if (!unassoc_sta->interested) {
+				unassoc_sta->interested = 1;
+				mlmepriv->interested_unassoc_sta_cnt++;
+				if (mlmepriv->interested_unassoc_sta_cnt == 1) {
+					rtw_run_in_thread_cmd(mlme_to_adapter(mlmepriv)
+						, ((void *)(rtw_map_config_monitor_act_non)), mlme_to_adapter(mlmepriv));
+				}
+			}
+			goto unlock_unassoc_sta_queue;
+		}
+
+		if (del_unassoc_sta_chk(mlmepriv, unassoc_sta) == UNASOC_STA_DEL_CHK_ALIVE) {
+			if (oldest_unassoc_sta == NULL)
+				oldest_unassoc_sta = unassoc_sta;
+			else if (rtw_time_after(unassoc_sta->time, oldest_unassoc_sta->time))
+				oldest_unassoc_sta = unassoc_sta;
+		}
+	}
+	unassoc_sta = alloc_unassoc_sta(mlmepriv);
+	if (unassoc_sta == NULL) {
+		RTW_INFO(FUNC_ADPT_FMT": Allocate fail\n", FUNC_ADPT_ARG(adapter));
+		if (oldest_unassoc_sta) {
+			RTW_INFO(FUNC_ADPT_FMT": Delete oldest entry and try again.\n", FUNC_ADPT_ARG(adapter));
+			del_unassoc_sta(mlmepriv, oldest_unassoc_sta);
+			unassoc_sta = alloc_unassoc_sta(mlmepriv);
+		} else
+			goto unlock_unassoc_sta_queue;
+	}
+
+	_rtw_memcpy(unassoc_sta->addr, addr, ETH_ALEN);
+	unassoc_sta->interested = 1;
+	unassoc_sta->recv_signal_power = 0;
+	unassoc_sta->time = rtw_get_current_time() - rtw_ms_to_systime(UNASSOC_STA_LIFETIME_MS);
+	rtw_list_insert_tail(&(unassoc_sta->list), &(queue->queue));
+	mlmepriv->interested_unassoc_sta_cnt++;
+	if (mlmepriv->interested_unassoc_sta_cnt == 1) {
+		rtw_run_in_thread_cmd(mlme_to_adapter(mlmepriv)
+			, ((void *)(rtw_map_config_monitor_act_non)), mlme_to_adapter(mlmepriv));
+	}
+
+unlock_unassoc_sta_queue:
+	_rtw_spinunlock_bh(&queue->lock);
+}
+
+void rtw_undo_interested_unassoc_sta(_adapter *adapter, u8 *addr)
+{
+	struct unassoc_sta_info *unassoc_sta;
+	struct mlme_priv *mlmepriv;
+	_queue *queue;
+	_list *head, *list;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	mlmepriv = &(adapter->mlmepriv);
+	queue = &(mlmepriv->unassoc_sta_queue);
+
+	_rtw_spinlock_bh(&queue->lock);
+	head = get_list_head(queue);
+	list = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		unassoc_sta = LIST_CONTAINOR(list , struct unassoc_sta_info, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(addr, unassoc_sta->addr, ETH_ALEN) == _TRUE) {
+			if (unassoc_sta->interested) {
+				unassoc_sta->interested = 0;
+				mlmepriv->interested_unassoc_sta_cnt--;
+				if (mlmepriv->interested_unassoc_sta_cnt == 0) {
+					rtw_run_in_thread_cmd(mlme_to_adapter(mlmepriv)
+						, ((void *)(rtw_map_config_monitor_act_non)), mlme_to_adapter(mlmepriv));
+				}
+			}
+
+			goto unlock_unassoc_sta_queue;
+		}
+	}
+unlock_unassoc_sta_queue:
+	_rtw_spinunlock_bh(&queue->lock);
+}
+
+void rtw_undo_all_interested_unassoc_sta(_adapter *adapter)
+{
+	struct unassoc_sta_info *unassoc_sta;
+	struct mlme_priv *mlmepriv;
+	_queue *queue;
+	_list *head, *list;
+
+	adapter = GET_PRIMARY_ADAPTER(adapter);
+	mlmepriv = &(adapter->mlmepriv);
+	queue = &(mlmepriv->unassoc_sta_queue);
+
+	_rtw_spinlock_bh(&queue->lock);
+	head = get_list_head(queue);
+	list = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+		unassoc_sta = LIST_CONTAINOR(list , struct unassoc_sta_info, list);
+		list = get_next(list);
+
+		if (unassoc_sta->interested) {
+			unassoc_sta->interested = 0;
+			mlmepriv->interested_unassoc_sta_cnt--;
+			if (mlmepriv->interested_unassoc_sta_cnt == 0) {
+				rtw_run_in_thread_cmd(mlme_to_adapter(mlmepriv)
+					, ((void *)(rtw_map_config_monitor_act_non)), mlme_to_adapter(mlmepriv));
+				goto unlock_unassoc_sta_queue;
+			}
+		}
+	}
+unlock_unassoc_sta_queue:
+	_rtw_spinunlock_bh(&queue->lock);
+}
+
+#endif /* CONFIG_RTW_MULTI_AP */
+
+/* select the desired network based on the capability of the (i)bss.
+ * check items: (1) security
+ *			   (2) network_type
+ *			   (3) WMM
+ *			   (4) HT
+ * (5) others */
+int rtw_is_desired_network(_adapter *adapter, struct wlan_network *pnetwork)
+{
+	struct security_priv *psecuritypriv = &adapter->securitypriv;
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	u32 desired_encmode;
+	u32 privacy;
+
+	/* u8 wps_ie[512]; */
+	uint wps_ielen;
+
+	int bselected = _TRUE;
+
+	desired_encmode = psecuritypriv->ndisencryptstatus;
+	privacy = pnetwork->network.Privacy;
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+		if (rtw_get_wps_ie(pnetwork->network.IEs + _FIXED_IE_LENGTH_, pnetwork->network.IELength - _FIXED_IE_LENGTH_, NULL, &wps_ielen) != NULL)
+			return _TRUE;
+		else
+			return _FALSE;
+	}
+	if (adapter->registrypriv.wifi_spec == 1) { /* for  correct flow of 8021X  to do.... */
+		u8 *p = NULL;
+		uint ie_len = 0;
+
+		if ((desired_encmode == Ndis802_11EncryptionDisabled) && (privacy != 0))
+			bselected = _FALSE;
+
+		if (psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPA2PSK) {
+			p = rtw_get_ie(pnetwork->network.IEs + _BEACON_IE_OFFSET_, _RSN_IE_2_, &ie_len, (pnetwork->network.IELength - _BEACON_IE_OFFSET_));
+			if (p && ie_len > 0)
+				bselected = _TRUE;
+			else
+				bselected = _FALSE;
+		}
+	}
+
+
+	if ((desired_encmode != Ndis802_11EncryptionDisabled) && (privacy == 0)) {
+		RTW_INFO("desired_encmode: %d, privacy: %d\n", desired_encmode, privacy);
+		bselected = _FALSE;
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) {
+		if (pnetwork->network.InfrastructureMode != pmlmepriv->dev_cur_network.network.InfrastructureMode)
+			bselected = _FALSE;
+	}
+
+#ifdef CONFIG_RTW_MBO
+	if (rtw_mbo_disallowed_network(pnetwork) == _TRUE)
+		bselected = _FALSE;
+#endif
+
+	return bselected;
+}
+
+static void free_mld_scanqueue(struct	mlme_priv *pmlmepriv)
+{
+	_queue *free_queue = &pmlmepriv->free_mld_bss_pool;
+	_queue *scan_queue = &pmlmepriv->scanned_mld_queue;
+	_list	*plist, *phead, *ptemp;
+
+
+	_rtw_spinlock_bh(&scan_queue->lock);
+	_rtw_spinlock_bh(&free_queue->lock);
+
+	phead = get_list_head(scan_queue);
+	plist = get_next(phead);
+
+	while (plist != phead) {
+		ptemp = get_next(plist);
+		rtw_list_delete(plist);
+		rtw_list_insert_tail(plist, &free_queue->queue);
+		plist = ptemp;
+		pmlmepriv->num_of_scanned_mld--;
+	}
+
+	_rtw_spinunlock_bh(&free_queue->lock);
+	_rtw_spinunlock_bh(&scan_queue->lock);
+
+}
+
+static void free_scanqueue(struct	mlme_priv *pmlmepriv)
+{
+	_queue *free_queue = &pmlmepriv->free_bss_pool;
+	_queue *scan_queue = &pmlmepriv->scanned_queue;
+	_list	*plist, *phead, *ptemp;
+
+
+	_rtw_spinlock_bh(&scan_queue->lock);
+	_rtw_spinlock_bh(&free_queue->lock);
+
+	phead = get_list_head(scan_queue);
+	plist = get_next(phead);
+
+	while (plist != phead) {
+		ptemp = get_next(plist);
+		rtw_list_delete(plist);
+		rtw_list_insert_tail(plist, &free_queue->queue);
+		plist = ptemp;
+		pmlmepriv->num_of_scanned--;
+	}
+
+	_rtw_spinunlock_bh(&free_queue->lock);
+	_rtw_spinunlock_bh(&scan_queue->lock);
+
+}
+
+void rtw_reset_rx_info(_adapter *adapter)
+{
+	struct recv_info *precvinfo = &adapter->recvinfo;
+
+	precvinfo->dbg_rx_ampdu_drop_count = 0;
+	precvinfo->dbg_rx_ampdu_forced_indicate_count = 0;
+	precvinfo->dbg_rx_ampdu_loss_count = 0;
+	precvinfo->dbg_rx_dup_mgt_frame_drop_count = 0;
+	precvinfo->dbg_rx_ampdu_window_shift_cnt = 0;
+	precvinfo->dbg_rx_drop_count = 0;
+	precvinfo->dbg_rx_conflic_mac_addr_cnt = 0;
+}
+
+/*
+ * Note: There may be I/O in following functions:
+ *	rtw_free_stainfo()
+ *	rtw_init_self_stainfo()
+ */
+void rtw_free_assoc_resources(_adapter *adapter, u8 lock_scanned_queue)
+{
+	struct wlan_network *pwlan = NULL;
+	struct	mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only]
+	** should choose a linked adapter_link */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct wlan_network *tgt_network = &adapter_link->mlmepriv.cur_network;
+#ifdef CONFIG_TDLS
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(adapter));
+	struct rtw_wifi_role_t *wrole = adapter->phl_role;
+	enum role_type rtype = PHL_RTYPE_STATION;
+	enum rtw_phl_status status;
+	struct tdls_info *ptdlsinfo = &adapter->tdlsinfo;
+#endif /* CONFIG_TDLS */
+#ifdef CONFIG_80211BE_EHT
+	struct wlan_mld_network *pmld_network = NULL;
+	struct wlan_network *pnetwork = NULL;
+	int network_num = 0;
+	u8 lidx;
+#endif
+
+
+	RTW_INFO("%s-"ADPT_FMT" tgt_network MacAddress=" MAC_FMT" ssid=%s\n",
+		__func__, ADPT_ARG(adapter), MAC_ARG(tgt_network->network.MacAddress), tgt_network->network.Ssid.Ssid);
+
+	if (MLME_IS_STA(adapter)) {
+		struct sta_info *psta;
+
+		psta = rtw_get_stainfo(&adapter->stapriv, tgt_network->network.MacAddress);
+		if (!psta) {
+			RTW_WARN("[TODO]" FUNC_ADPT_FMT ": fail to find stainfo"
+				 "(" MAC_FMT ")\n",
+				 FUNC_ADPT_ARG(adapter),
+				 MAC_ARG(tgt_network->network.MacAddress));
+			return;
+		}
+
+#ifdef CONFIG_TDLS
+		rtw_free_all_tdls_sta(adapter, _TRUE);
+		rtw_reset_tdls_info(adapter);
+		/* Change wrole type back to PHL_RTYPE_STATION for rtw_free_stainfo later */
+		if(wrole->type == PHL_RTYPE_TDLS) {
+			status = rtw_phl_cmd_wrole_change(phl, wrole, adapter_link->wrlink, WR_CHG_TYPE,
+						(u8*)&rtype, sizeof(enum role_type), PHL_CMD_DIRECTLY, 0);
+			if (status != RTW_PHL_STATUS_SUCCESS) {
+				RTW_ERR("%s - change to phl role type = %d fail with error = %d\n", __func__, rtype, status);
+				rtw_warn_on(1);
+			}
+		}
+#endif /* CONFIG_TDLS */
+
+		/* _rtw_spinlock_bh(&(pstapriv->sta_hash_lock)); */
+		rtw_free_mld_stainfo(adapter, psta->phl_sta->mld);
+		rtw_init_self_stainfo(adapter, PHL_CMD_DIRECTLY);
+		/* _rtw_spinunlock_bh(&(pstapriv->sta_hash_lock)); */
+#ifdef CONFIG_RTW_FSM
+		rtw_fsm_notify_disconnect(&adapter->fsmpriv, psta);
+#endif
+		pmlmeinfo->assoc_AP_vendor = HT_IOT_PEER_UNKNOWN;
+		rtw_update_phl_iot(adapter, HT_IOT_PEER_UNKNOWN);
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)) {
+		struct sta_info *psta;
+
+		rtw_free_all_stainfo(adapter);
+	}
+
+	if (lock_scanned_queue) {
+		_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+		_rtw_spinlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS) || (pmlmepriv->wpa_phase == _TRUE)){
+		RTW_INFO("Dont free disconnecting network of scanned_queue due to uner %s %s phase\n\n",
+			check_fwstate(pmlmepriv, WIFI_UNDER_WPS) ? "WPS" : "",
+			(pmlmepriv->wpa_phase == _TRUE) ? "WPA" : "");
+	} else {
+		pwlan = _rtw_find_same_network(&pmlmepriv->scanned_queue, tgt_network);
+		if (pwlan) {
+			pwlan->fixed = _FALSE;
+
+#ifndef PRIVATE_R
+			RTW_INFO("Free disconnecting network of scanned_queue\n");
+#ifdef CONFIG_80211BE_EHT
+			if (pwlan->network.is_mld) {
+				pmld_network = pwlan->network.mld_network;
+				network_num = pmld_network->network_num;
+				for (lidx = 0; lidx < network_num; lidx++) {
+					pnetwork = GET_LINK_NETWORK(pmld_network, lidx);
+					if (pnetwork) {
+						rtw_free_network_nolock(adapter, pnetwork);
+						pmld_network->network_num--;
+					}
+				}
+				rtw_free_mld_network_nolock(adapter, pmld_network);
+			} else
+#endif
+			{
+				rtw_free_network_nolock(adapter, pwlan);
+			}
+#endif /* !PRIVATE_R */
+#ifdef CONFIG_P2P
+			if (rtw_p2p_chk_role(&adapter->wdinfo, P2P_ROLE_CLIENT) ||
+			    rtw_p2p_chk_role(&adapter->wdinfo, P2P_ROLE_GO)) {
+				rtw_set_scan_deny(adapter, 2000);
+				/* rtw_clear_scan_deny(adapter); */
+			}
+#endif /* CONFIG_P2P */
+		} else
+			RTW_ERR("Free disconnecting network of scanned_queue failed due to pwlan == NULL\n\n");
+	}
+
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) && (adapter->stapriv.asoc_sta_count == 1))
+	    /*||MLME_IS_STA(adapter)*/) {
+		if (pwlan) /* ToDo CONFIG_RTW_MLD: adhoc */
+			rtw_free_network_nolock(adapter, pwlan);
+	}
+
+	if (lock_scanned_queue) {
+		_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+		_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+	}
+
+	adapter->securitypriv.key_mask = 0;
+
+	rtw_reset_rx_info(adapter);
+
+
+}
+
+/*
+*rtw_indicate_connect: the caller has to lock pmlmepriv->lock
+*/
+void rtw_indicate_connect(_adapter *padapter)
+{
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	rtw_clear_to_join_status(padapter);
+
+	if (!check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) {
+
+		set_fwstate(pmlmepriv, WIFI_ASOC_STATE);
+
+		rtw_led_control(padapter, LED_CTL_LINK);
+
+		rtw_os_indicate_connect(padapter);
+
+		#ifdef CONFIG_DFS_MASTER
+		if (CHK_MLME_STATE(padapter, WIFI_AP_STATE | WIFI_MESH_STATE))
+			rtw_indicate_cac_state_on_bss_start(padapter);
+		#endif
+
+		#ifdef CONFIG_RTW_WDS
+		if (MLME_IS_STA(padapter))
+			rtw_wds_gptr_tbl_init(padapter);
+		#endif
+	}
+
+#ifndef CONFIG_RTW_FSM_BTM
+	rtw_set_to_roam(padapter, 0);
+#endif
+	if (!MLME_IS_AP(padapter) && !MLME_IS_MESH(padapter))
+		rtw_mi_set_scan_deny(padapter, 3000);
+
+
+}
+
+
+/*
+*rtw_indicate_disconnect: the caller has to lock pmlmepriv->lock
+*/
+void rtw_indicate_disconnect(_adapter *padapter, u16 reason, u8 locally_generated)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	u8 *wps_ie = NULL;
+	uint wpsie_len = 0;
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS))
+		pmlmepriv->wpa_phase = _TRUE;
+
+	rtw_clear_is_accepted_status(padapter);
+
+	_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING | WIFI_UNDER_WPS | WIFI_OP_CH_SWITCHING | WIFI_UNDER_KEY_HANDSHAKE);
+
+	/* force to clear cur_network_scanned's SELECTED REGISTRAR */
+	if (pmlmepriv->cur_network_scanned) {
+		WLAN_BSSID_EX	*current_joined_bss = &(pmlmepriv->cur_network_scanned->network);
+
+		if (current_joined_bss) {
+			wps_ie = rtw_get_wps_ie(current_joined_bss->IEs + _FIXED_IE_LENGTH_,
+				current_joined_bss->IELength - _FIXED_IE_LENGTH_, NULL, &wpsie_len);
+			if (wps_ie && wpsie_len > 0) {
+				u8 *attr = NULL;
+				u32 attr_len;
+				attr = rtw_get_wps_attr(wps_ie, wpsie_len, WPS_ATTR_SELECTED_REGISTRAR,
+							NULL, &attr_len);
+				if (attr)
+					*(attr + 4) = 0;
+			}
+		}
+	}
+	/* RTW_INFO("clear wps when %s\n", __func__); */
+
+	if (rtw_to_roam(padapter) > 0)
+		_clr_fwstate_(pmlmepriv, WIFI_ASOC_STATE);
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (MLME_IS_STA(padapter))
+		rtw_wapi_return_one_sta_info(padapter, NULL);
+	else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
+		 check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE))
+		rtw_wapi_return_all_sta_info(padapter);
+#endif
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE)
+	    || (rtw_to_roam(padapter) <= 0)
+	   ) {
+		#ifdef CONFIG_RTW_WDS
+		adapter_set_use_wds(padapter, 0);
+		rtw_wds_gptr_tbl_unregister(padapter);
+		#endif
+		#ifdef CONFIG_RTW_MULTI_AP
+		padapter->multi_ap = 0;
+		#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+		if (ATOMIC_READ(&padapter->tbtx_tx_pause) == _TRUE) {
+			ATOMIC_SET(&padapter->tbtx_tx_pause, _FALSE);
+			rtw_tx_control_cmd(padapter);
+		}
+#endif
+
+		#ifdef CONFIG_LAYER2_ROAMING
+		pmlmepriv->roam_network = NULL;
+		#endif
+
+		/* rtw_phl_chanctx_del(adapter_to_dvobj(padapter)->phl, padapter->phl_role, NULL); */
+
+		rtw_os_indicate_disconnect(padapter, reason, locally_generated);
+
+		_clr_fwstate_(pmlmepriv, WIFI_ASOC_STATE);
+
+		rtw_led_control(padapter, LED_CTL_NO_LINK);
+
+		rtw_clear_scan_deny(padapter);
+	}
+
+#ifdef CONFIG_P2P_PS
+	p2p_ps_wk_cmd(padapter, P2P_PS_DISABLE, 1);
+#endif /* CONFIG_P2P_PS */
+}
+
+inline void rtw_indicate_scan_done(_adapter *padapter, bool aborted)
+{
+	RTW_INFO(FUNC_ADPT_FMT" aborted=%d\n", FUNC_ADPT_ARG(padapter), aborted);
+
+	rtw_os_indicate_scan_done(padapter, aborted);
+}
+
+static u32 _rtw_wait_join_done(_adapter *adapter, u8 abort, u32 timeout_ms)
+{
+	systime start;
+	u32 pass_ms;
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+
+	if (!MLME_IS_LINKING(adapter)
+		#ifdef CONFIG_IOCTL_CFG80211
+		&& !rtw_cfg80211_is_connect_requested(adapter)
+		#endif
+	)
+		return 0;
+
+	start = rtw_get_current_time();
+
+	pmlmeext->join_abort = abort;
+	if (abort)
+		set_link_timer(pmlmeext, 1);
+
+	while (rtw_get_passing_time_ms(start) <= timeout_ms
+		&& (MLME_IS_LINKING(adapter)
+			#ifdef CONFIG_IOCTL_CFG80211
+			|| rtw_cfg80211_is_connect_requested(adapter)
+			#endif
+			)
+	) {
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+			break;
+
+		RTW_INFO(FUNC_ADPT_FMT" linking...\n", FUNC_ADPT_ARG(adapter));
+		rtw_msleep_os(20);
+	}
+
+	if (abort) {
+		if (MLME_IS_LINKING(adapter)
+			#ifdef CONFIG_IOCTL_CFG80211
+			|| rtw_cfg80211_is_connect_requested(adapter)
+			#endif
+		) {
+			if (!RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+				RTW_INFO(FUNC_ADPT_FMT" waiting for join_abort time out!\n", FUNC_ADPT_ARG(adapter));
+		}
+	}
+
+	pmlmeext->join_abort = 0;
+	pass_ms = rtw_get_passing_time_ms(start);
+
+	return pass_ms;
+}
+
+u32 rtw_join_abort_timeout(_adapter *adapter, u32 timeout_ms)
+{
+	return _rtw_wait_join_done(adapter, _TRUE, timeout_ms);
+}
+
+static struct sta_info *rtw_joinbss_update_stainfo(_adapter *padapter,
+						struct _ADAPTER_LINK *padapter_link)
+{
+	/* cur_network is already updated from mlmeinfo->network at rtw_joinbss_update_network()
+	** hence use cur_network instead */
+	struct wlan_network *pnetwork = &padapter_link->mlmepriv.cur_network;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct rtw_phl_mld_t *mld = NULL;
+	int i;
+	struct sta_info *psta = NULL;
+	struct recv_reorder_ctrl *preorder_ctrl;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+
+	psta = rtw_get_stainfo(pstapriv, pnetwork->network.MacAddress);
+	if (psta == NULL) {
+		RTW_ERR("%s: can not find "MAC_FMT"\n", __func__, MAC_ARG(pnetwork->network.MacAddress));
+		mld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter->phl_role,
+						padapter->mlmepriv.dev_cur_network.network.MacAddress, DTYPE);
+		/* main_id is don't care for self sta */
+		psta = rtw_alloc_stainfo(pstapriv, pnetwork->network.MacAddress, DTYPE, 0,
+						padapter_link->wrlink->id, PHL_CMD_DIRECTLY);
+	}
+
+	if (psta) { /* update ptarget_sta */
+		RTW_INFO("%s\n", __FUNCTION__);
+
+		/* assign aid */
+		psta->phl_sta->aid = pnetwork->aid;
+
+		update_sta_info(padapter, psta);
+
+		/* update station supportRate */
+		psta->bssratelen = rtw_get_rateset_len(pnetwork->network.SupportedRates);
+		_rtw_memcpy(psta->bssrateset, pnetwork->network.SupportedRates, psta->bssratelen);
+		update_sta_ra_info(padapter, psta);
+
+		psta->phl_sta->wmode = pmlmeext->cur_wireless_mode;
+
+
+		/* security related */
+#ifdef CONFIG_RTW_80211R
+		if ((padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+			&& (psta->ft_pairwise_key_installed == _FALSE))
+#else
+		if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+#endif
+		{
+			u8 *ie;
+			sint ie_len;
+			u8 mfp_opt = MFP_NO;
+			u8 spp_opt = 0;
+
+			padapter_link->securitypriv.binstallGrpkey = _FALSE;
+			padapter_link->securitypriv.bgrpkey_handshake = _FALSE;
+			padapter->securitypriv.busetkipkey = _FALSE;
+
+			ie = rtw_get_ie(pnetwork->network.IEs + _BEACON_IE_OFFSET_, WLAN_EID_RSN
+				, &ie_len, (pnetwork->network.IELength - _BEACON_IE_OFFSET_));
+			if (ie && ie_len > 0
+				&& rtw_parse_wpa2_ie(ie, ie_len + 2, NULL, NULL, NULL, NULL, &mfp_opt, &spp_opt) == _SUCCESS
+			) {
+				if (padapter->securitypriv.mfp_opt >= MFP_OPTIONAL && mfp_opt >= MFP_OPTIONAL)
+					psta->flags |= WLAN_STA_MFP;
+			}
+			if (padapter->securitypriv.dot11PrivacyAlgrthm != _NO_PRIVACY_ ) {
+				/*check if amsdu is allowed */
+				if (rtw_check_amsdu_disable(padapter->registrypriv.amsdu_mode, spp_opt) == _TRUE)
+					psta->flags |= WLAN_STA_AMSDU_DISABLE;
+			}
+			psta->ieee8021x_blocked = _TRUE;
+			psta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+
+			_rtw_memset((u8 *)&psta->dot118021x_UncstKey, 0, sizeof(union Keytype));
+			_rtw_memset((u8 *)&psta->dot11tkiprxmickey, 0, sizeof(union Keytype));
+			_rtw_memset((u8 *)&psta->dot11tkiptxmickey, 0, sizeof(union Keytype));
+		}
+
+		/*	Commented by Albert 2012/07/21 */
+		/*	When doing the WPS, the wps_ie_len won't equal to 0 */
+		/*	And the Wi-Fi driver shouldn't allow the data packet to be tramsmitted. */
+		if (padapter->securitypriv.wps_ie_len != 0) {
+			if (!rtw_sec_chk_auth_type(padapter, MLME_AUTHTYPE_OPEN_SYSTEM))
+				psta->ieee8021x_blocked = _TRUE;
+			padapter->securitypriv.wps_ie_len = 0;
+		}
+
+
+		/* for A-MPDU Rx reordering buffer control for sta_info */
+		/* if A-MPDU Rx is enabled, reseting  rx_ordering_ctrl wstart_b(indicate_seq) to default value=0xffff */
+		/* todo: check if AP can send A-MPDU packets */
+		for (i = 0; i < 16 ; i++) {
+			preorder_ctrl = &psta->recvreorder_ctrl[i];
+			preorder_ctrl->enable = _FALSE;
+			preorder_ctrl->indicate_seq = 0xffff;
+			rtw_clear_bit(RTW_RECV_ACK_OR_TIMEOUT, &preorder_ctrl->rec_abba_rsp_ack);
+			#ifdef DBG_RX_SEQ
+			RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_CLEAR indicate_seq:%u preorder_ctrl->rec_abba_rsp_ack:%lu\n"
+				, FUNC_ADPT_ARG(padapter)
+				, i
+				, preorder_ctrl->indicate_seq
+				, preorder_ctrl->rec_abba_rsp_ack
+				);
+			#endif
+#ifdef CONFIG_RECV_REORDERING_CTRL
+			preorder_ctrl->wsize_b = 64;/* max_ampdu_sz; */ /* ex. 32(kbytes) -> wsize_b=32 */
+#endif
+			preorder_ctrl->ampdu_size = RX_AMPDU_SIZE_INVALID;
+		}
+
+		#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+		_rtw_memcpy(&psta->rm_en_cap, pnetwork->network.PhyInfo.rm_en_cap, 5);
+		#endif
+		#ifdef CONFIG_RTW_MULTI_AP
+		if (padapter->multi_ap & MULTI_AP_BACKHAUL_STA) {
+			u8 multi_ap = rtw_get_multi_ap_ie_ext(padapter->mlmepriv.assoc_rsp + sizeof(struct rtw_ieee80211_hdr_3addr) + 6
+					, padapter->mlmepriv.assoc_rsp_len - sizeof(struct rtw_ieee80211_hdr_3addr) - 6);
+
+			if (multi_ap & MULTI_AP_BACKHAUL_BSS) /* backhaul bss, enable WDS */
+				psta->flags |= WLAN_STA_MULTI_AP | WLAN_STA_WDS;
+			else if (multi_ap & MULTI_AP_FRONTHAUL_BSS) /* fronthaul bss only */
+				psta->flags |= WLAN_STA_MULTI_AP;
+		}
+		#endif
+		#ifdef CONFIG_RTS_FULL_BW
+		rtw_parse_sta_vendor_ie_8812(padapter, psta, BSS_EX_TLV_IES(&pnetwork->network), BSS_EX_TLV_IES_LEN(&pnetwork->network));
+		#endif
+	}
+	return psta;
+
+}
+
+/* pnetwork : returns from rtw_joinbss_event_callback
+ * ptarget_wlan: found from scanned_queue */
+static void rtw_joinbss_update_network(_adapter *padapter, struct wlan_network *ptarget_wlan, struct wlan_network  *pnetwork)
+{
+	struct mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct wlan_network  *cur_network = &(pmlmepriv->dev_cur_network);
+	sint tmp_fw_state = 0x0;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/* why not use ptarget_wlan?? */
+	_rtw_memcpy(&cur_network->network, &pnetwork->network, pnetwork->network.Length);
+	/* some IEs in pnetwork is wrong, so we should use ptarget_wlan IEs */
+	cur_network->network.IELength = ptarget_wlan->network.IELength;
+	_rtw_memcpy(&cur_network->network.IEs[0], &ptarget_wlan->network.IEs[0], MAX_IE_SZ);
+
+	cur_network->aid = pnetwork->join_res;
+	#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	rtw_set_signal_stat_timer(&padapter->recvinfo);
+	#endif
+
+	/*RTW_WKARD_CORE_RSSI_V1 - GEORGIA MUST REFINE*/
+	padapter->recvinfo.signal_strength = ptarget_wlan->network.PhyInfo.SignalStrength;
+	padapter->recvinfo.signal_qual = ptarget_wlan->network.PhyInfo.SignalQuality;
+	/* the ptarget_wlan->network.PhyInfo.rssi is raw data, we use ptarget_wlan->network.PhyInfo.SignalStrength instead (has scaled) */
+	padapter->recvinfo.rssi = ptarget_wlan->network.PhyInfo.rssi;
+#if defined(DBG_RX_SIGNAL_DISPLAY_PROCESSING) && 1
+	RTW_INFO(FUNC_ADPT_FMT" signal_strength:%3u, rssi:%3d, signal_qual:%3u"
+		 "\n"
+		 , FUNC_ADPT_ARG(padapter)
+		 , padapter->recvinfo.signal_strength
+		 , padapter->recvinfo.rssi
+		 , padapter->recvinfo.signal_qual
+		);
+#endif
+	#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	rtw_set_signal_stat_timer(&padapter->recvinfo);
+	#endif
+	/* update fw_state */ /* will clr WIFI_UNDER_LINKING here indirectly */
+
+	switch (pnetwork->network.InfrastructureMode) {
+	case Ndis802_11Infrastructure:
+		/* Check encryption */
+		if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+			tmp_fw_state = tmp_fw_state | WIFI_UNDER_KEY_HANDSHAKE;
+
+		if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS))
+			tmp_fw_state = tmp_fw_state | WIFI_UNDER_WPS;
+
+		init_fwstate(pmlmepriv, WIFI_STATION_STATE | tmp_fw_state);
+
+		break;
+	case Ndis802_11IBSS:
+		/*pmlmepriv->fw_state = WIFI_ADHOC_STATE;*/
+		init_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
+		break;
+	default:
+		/*pmlmepriv->fw_state = WIFI_NULL_STATE;*/
+		init_fwstate(pmlmepriv, WIFI_NULL_STATE);
+		break;
+	}
+
+	rtw_update_protection(padapter, (cur_network->network.IEs) + _FIXED_IE_LENGTH_,
+			      (cur_network->network.IELength));
+}
+
+/* Notes: the fucntion could be > passive_level (the same context as Rx tasklet)
+ * pnetwork : returns from rtw_joinbss_event_callback
+ * ptarget_wlan: found from scanned_queue
+ * if join_res > 0, for (fw_state==WIFI_STATION_STATE), we check if  "ptarget_sta" & "ptarget_wlan" exist.
+ * if join_res > 0, for (fw_state==WIFI_ADHOC_STATE), we only check if "ptarget_wlan" exist.
+ * if join_res > 0, update "cur_network->network" from "pnetwork->network" if (ptarget_wlan !=NULL).
+ */
+/* #define REJOIN */
+void rtw_joinbss_event_prehandle(_adapter *adapter, u8 *pbuf, u16 status)
+{
+	static u8 retry = 0;
+	struct sta_info *ptarget_sta = NULL, *pcur_sta = NULL;
+	struct	sta_priv *pstapriv = &adapter->stapriv;
+	struct	mlme_priv	*pmlmepriv = &(adapter->mlmepriv);
+	struct wlan_network	*pnetwork	= (struct wlan_network *)pbuf;
+	struct wlan_network	*cur_network = NULL;
+	struct wlan_network	*pcur_wlan = NULL, *ptarget_wlan = NULL;
+	unsigned int		the_same_macaddr = _FALSE;
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct wlan_network *ptmp_wlan = NULL;
+	u8 lidx;
+#ifdef CONFIG_80211BE_EHT
+	struct wlan_mld_network *pmld_network = NULL;
+	struct wlan_mld_network *ptarget_wlan_mld = NULL;
+	struct wlan_network *plink_network = NULL;
+#endif
+
+	cur_network = &(adapter_link->mlmepriv.cur_network);
+
+	rtw_get_encrypt_decrypt_from_registrypriv(adapter);
+
+	the_same_macaddr = _rtw_memcmp(pnetwork->network.MacAddress, cur_network->network.MacAddress, ETH_ALEN);
+
+	pnetwork->network.Length = get_WLAN_BSSID_EX_sz(&pnetwork->network);
+	if (pnetwork->network.Length > sizeof(WLAN_BSSID_EX))
+		goto exit;
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 0;
+	pmlmepriv->LinkDetectInfo.LowPowerTransitionCount = 0;
+
+#ifdef CONFIG_RTW_FSM
+	rtw_fsm_notify_connect(&adapter->fsmpriv,
+		rtw_get_stainfo(pstapriv, pnetwork->network.MacAddress), pnetwork->join_res);
+#endif
+	if (pnetwork->join_res > 0) {
+		_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+		_rtw_spinlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+		retry = 0;
+		if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING)) {
+			/* s1. find ptarget_wlan */
+			if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+				if (the_same_macaddr == _TRUE)
+					ptarget_wlan = _rtw_find_network(&pmlmepriv->scanned_queue, cur_network->network.MacAddress);
+				else {
+					pcur_wlan = _rtw_find_network(&pmlmepriv->scanned_queue, cur_network->network.MacAddress);
+					if (pcur_wlan)
+						pcur_wlan->fixed = _FALSE;
+
+					pcur_sta = rtw_get_stainfo(pstapriv, cur_network->network.MacAddress);
+					if (pcur_sta) {
+						/* _rtw_spinlock_bh(&(pstapriv->sta_hash_lock)); */
+						rtw_free_mld_stainfo(adapter, pcur_sta->phl_sta->mld);
+						/* _rtw_spinunlock_bh(&(pstapriv->sta_hash_lock)); */
+					}
+
+					ptarget_wlan = _rtw_find_network(&pmlmepriv->scanned_queue, pnetwork->network.MacAddress);
+					if (MLME_IS_STA(adapter)) {
+						if (ptarget_wlan)
+							ptarget_wlan->fixed = _TRUE;
+					}
+				}
+
+			} else {
+				ptarget_wlan = _rtw_find_same_network(&pmlmepriv->scanned_queue, pnetwork);
+				if (MLME_IS_STA(adapter)) {
+					if (ptarget_wlan)
+						ptarget_wlan->fixed = _TRUE;
+				}
+			}
+
+			/* s2. update cur_network */
+			if (ptarget_wlan) {
+				rtw_joinbss_update_network(adapter, ptarget_wlan, pnetwork);
+				rtw_joinbss_update_link_network(adapter, adapter_link, ptarget_wlan, pnetwork);
+#ifdef CONFIG_80211BE_EHT
+				if (ptarget_wlan->network.is_mld) {
+					ptarget_wlan_mld = ptarget_wlan->network.mld_network;
+					pmld_network = pnetwork->network.mld_network;
+					/* Update mac_addr as mld mac addr for adapter's cur_network */
+					_rtw_memcpy(pmlmepriv->dev_cur_network.network.MacAddress, ptarget_wlan_mld->mac_addr, ETH_ALEN);
+				}
+#endif
+			}
+			else {
+				RTW_PRINT("Can't find ptarget_wlan when joinbss_event callback\n");
+				_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+				_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+				goto ignore_joinbss_callback;
+			}
+#ifdef CONFIG_80211BE_EHT
+			if (ptarget_wlan_mld && pmld_network) {
+				for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+					adapter_link = GET_LINK(adapter, lidx);
+					if (!adapter_link->mlmepriv.is_accepted || lidx == RTW_RLINK_PRIMARY)
+						continue;
+					ptmp_wlan = rtw_get_link_network_by_linkid(ptarget_wlan_mld, adapter_link->mlmepriv.link_id);
+					plink_network = rtw_get_link_network_by_linkid(pmld_network, adapter_link->mlmepriv.link_id);
+					if (ptmp_wlan && plink_network)
+						rtw_joinbss_update_link_network(adapter, adapter_link, ptmp_wlan, plink_network);
+					else {
+						RTW_PRINT("Failed to update cur_network of links \n");
+						_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+						_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+						goto ignore_joinbss_callback;
+					}
+				}
+			}
+#endif
+
+
+			/* s3. find ptarget_sta & update ptarget_sta after update cur_network only for station mode */
+			if (MLME_IS_STA(adapter)) {
+				for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+					adapter_link = GET_LINK(adapter, lidx);
+					if (!adapter_link->mlmepriv.is_accepted)
+						continue;
+					ptarget_sta = rtw_joinbss_update_stainfo(adapter, adapter_link);
+					if (ptarget_sta == NULL) {
+						RTW_ERR("Can't update stainfo when joinbss_event callback\n");
+						_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+						_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+						goto ignore_joinbss_callback;
+					}
+				}
+			}
+
+			/* s4. indicate connect			 */
+			if (MLME_IS_STA(adapter) || MLME_IS_ADHOC(adapter)) {
+				pmlmepriv->cur_network_scanned = ptarget_wlan;
+				rtw_indicate_connect(adapter);
+			}
+
+			/* s5. Cancle assoc_timer					 */
+			/*_cancel_timer_ex(&pmlmepriv->assoc_timer);*/
+			cancel_assoc_timer(pmlmepriv);
+
+
+		} else {
+			_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+			_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+			goto ignore_joinbss_callback;
+		}
+
+		_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+		_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	} else if (pnetwork->join_res == -4) {
+		rtw_reset_securitypriv(adapter);
+		pmlmepriv->join_status = status;
+		/*_set_timer(&pmlmepriv->assoc_timer, 1);*/
+		set_assoc_timer(pmlmepriv, 1);
+
+		/* rtw_free_assoc_resources(adapter, _TRUE); */
+
+		if ((check_fwstate(pmlmepriv, WIFI_UNDER_LINKING)) == _TRUE) {
+			_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+		}
+
+	} else { /* if join_res < 0 (join fails), then try again */
+
+#ifdef REJOIN
+		res = _FAIL;
+		if (retry < 2) {
+			res = rtw_select_and_join_from_scanned_queue(pmlmepriv);
+		}
+
+		if (res == _SUCCESS) {
+			/* extend time of assoc_timer */
+			/*_set_timer(&pmlmepriv->assoc_timer, MAX_JOIN_TIMEOUT);*/
+			set_assoc_timer(pmlmepriv, MAX_JOIN_TIMEOUT);
+			retry++;
+		} else if (res == 2) { /* there is no need to wait for join */
+			_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+			rtw_indicate_connect(adapter);
+		} else {
+#endif
+			pmlmepriv->join_status = status;
+			/*_set_timer(&pmlmepriv->assoc_timer, 1);*/
+			set_assoc_timer(pmlmepriv, 1);
+			/* rtw_free_assoc_resources(adapter, _TRUE); */
+			_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+
+#ifdef REJOIN
+			retry = 0;
+		}
+#endif
+	}
+
+ignore_joinbss_callback:
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+exit:
+	return;
+}
+
+void rtw_joinbss_event_callback(_adapter *adapter, u8 *pbuf)
+{
+#if 0
+	struct wlan_network *pnetwork = (struct wlan_network *)pbuf;
+
+
+	rtw_set_hw_after_join(adapter, pnetwork->join_res);
+	rtw_mi_os_xmit_schedule(adapter);
+#endif
+}
+
+void rtw_stassoc_event_callback(_adapter *adapter, u8 *pbuf)
+{
+	struct sta_info *psta;
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct stassoc_event	*pstassoc	= (struct stassoc_event *)pbuf;
+	struct wlan_network	*cur_network = &(pmlmepriv->dev_cur_network);
+	struct wlan_network	*ptarget_wlan = NULL;
+
+
+#if CONFIG_RTW_MACADDR_ACL
+	if (rtw_access_ctrl(adapter, pstassoc->macaddr) == _FALSE)
+		return;
+#endif
+
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	if (MLME_IS_AP(adapter) || MLME_IS_MESH(adapter)) {
+		psta = rtw_get_stainfo(&adapter->stapriv, pstassoc->macaddr);
+		if (psta) {
+			u8 *passoc_req = NULL;
+			u32 assoc_req_len = 0;
+
+			rtw_alloc_stainfo_hw(&adapter->stapriv, psta);
+
+			rtw_hw_connected_apmode(adapter, psta);
+
+#ifndef CONFIG_AUTO_AP_MODE
+			if (!MLME_IS_MESH(adapter)) {
+				/* report to upper layer */
+				RTW_INFO("indicate_sta_assoc_event to upper layer - hostapd\n");
+				#ifdef CONFIG_IOCTL_CFG80211
+				_rtw_spinlock_bh(&psta->lock);
+				if (psta->passoc_req && psta->assoc_req_len > 0) {
+					passoc_req = rtw_zmalloc(psta->assoc_req_len);
+					if (passoc_req) {
+						assoc_req_len = psta->assoc_req_len;
+						_rtw_memcpy(passoc_req, psta->passoc_req, assoc_req_len);
+					}
+				}
+				_rtw_spinunlock_bh(&psta->lock);
+
+				if (passoc_req && assoc_req_len > 0) {
+					rtw_cfg80211_indicate_sta_assoc(adapter, passoc_req, assoc_req_len);
+					rtw_mfree(passoc_req, assoc_req_len);
+				}
+				#else /* !CONFIG_IOCTL_CFG80211	 */
+				rtw_indicate_sta_assoc_event(adapter, psta);
+				#endif /* !CONFIG_IOCTL_CFG80211 */
+			}
+#endif /* !CONFIG_AUTO_AP_MODE */
+
+			if (is_wep_enc(adapter->securitypriv.dot11PrivacyAlgrthm))
+				rtw_ap_wep_pk_setting(adapter, psta);
+		}
+		goto exit;
+	}
+#endif /* defined (CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME) */
+
+	/* for AD-HOC mode */
+	psta = rtw_get_stainfo(&adapter->stapriv, pstassoc->macaddr);
+	if (psta == NULL) {
+		RTW_ERR(FUNC_ADPT_FMT" get no sta_info with "MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(pstassoc->macaddr));
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	rtw_sta_hal_media_status_rpt_cmd(adapter, psta, 1, RTW_CMDF_DIRECTLY);
+
+	if (adapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+		psta->dot118021XPrivacy = adapter->securitypriv.dot11PrivacyAlgrthm;
+
+
+	psta->ieee8021x_blocked = _FALSE;
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE) ||
+	    (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE)) {
+		if (adapter->stapriv.asoc_sta_count == 2) {
+			_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+			ptarget_wlan = _rtw_find_network(&pmlmepriv->scanned_queue, cur_network->network.MacAddress);
+			pmlmepriv->cur_network_scanned = ptarget_wlan;
+			if (ptarget_wlan)
+				ptarget_wlan->fixed = _TRUE;
+			_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+			/* a sta + bc/mc_stainfo (not Ibss_stainfo) */
+			rtw_indicate_connect(adapter);
+		}
+	}
+
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+
+	mlmeext_sta_add_event_callback(adapter, psta);
+
+#ifdef CONFIG_RTL8711
+	/* submit SetStaKey_cmd to tell fw, fw will allocate an CAM entry for this sta	 */
+	rtw_setstakey_cmd(adapter, psta, GROUP_KEY, _TRUE);
+#endif
+
+exit:
+#ifdef CONFIG_RTS_FULL_BW
+	rtw_set_rts_bw(adapter);
+#endif/*CONFIG_RTS_FULL_BW*/
+	return;
+}
+
+#ifdef CONFIG_IEEE80211W
+void rtw_sta_timeout_event_callback(_adapter *adapter, u8 *pbuf)
+{
+	struct sta_info *psta;
+	struct stadel_event *pstadel = (struct stadel_event *)pbuf;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+
+
+	psta = rtw_get_stainfo(&adapter->stapriv, pstadel->macaddr);
+
+	if (psta) {
+		u8 updated = _FALSE;
+
+		rtw_stapriv_asoc_list_lock(pstapriv);
+		if (rtw_is_list_empty(&psta->asoc_list) == _FALSE) {
+			rtw_stapriv_asoc_list_del(pstapriv, psta);
+			updated = ap_free_sta(adapter, psta, _TRUE, 0, WLAN_REASON_PREV_AUTH_NOT_VALID, _TRUE);
+		}
+		rtw_stapriv_asoc_list_unlock(pstapriv);
+
+		associated_clients_update(adapter, updated, STA_INFO_UPDATE_ALL);
+	}
+
+
+
+}
+#endif /* CONFIG_IEEE80211W */
+
+
+/* Note: the caller should already have lock of a->mlmepriv.lock */
+/* Note2: There may be I/O in rtw_free_assoc_resources() */
+static void _stadel_posthandle_sta(struct _ADAPTER *a,
+				   struct stadel_event *stadel)
+{
+	struct mlme_priv *mlme = &a->mlmepriv;
+	u16 reason = *((unsigned short *)stadel->rsvd);
+	bool roam = _FALSE;
+	struct wlan_network *roam_target = NULL;
+
+#ifdef CONFIG_LAYER2_ROAMING
+	if (a->registrypriv.wifi_spec == 1) {
+		roam = _FALSE;
+	} else if ((reason == WLAN_REASON_EXPIRATION_CHK)
+		   && rtw_chk_roam_flags(a, RTW_ROAM_ON_EXPIRED)) {
+		roam = _TRUE;
+	} else if ((reason == WLAN_REASON_ACTIVE_ROAM)
+		   && rtw_chk_roam_flags(a, RTW_ROAM_ACTIVE)) {
+		roam = _TRUE;
+		roam_target = mlme->roam_network;
+	}
+#ifdef CONFIG_RTW_80211R
+	if ((reason == WLAN_REASON_ACTIVE_ROAM)
+	    && rtw_ft_chk_flags(a, RTW_FT_BTM_ROAM)) {
+		roam = _TRUE;
+		roam_target = mlme->roam_network;
+	}
+#endif
+	if (roam == _TRUE) {
+		if (rtw_to_roam(a) > 0)
+			rtw_dec_to_roam(a); /* this stadel_event is caused by roaming, decrease to_roam */
+		else if (rtw_to_roam(a) == 0)
+			rtw_set_to_roam(a, a->registrypriv.max_roaming_times);
+	} else {
+		rtw_set_to_roam(a, 0);
+	}
+#endif /* CONFIG_LAYER2_ROAMING */
+
+#if 0
+	rtw_free_uc_swdec_pending_queue(adapter_to_dvobj(a));
+#endif
+	if (MLME_IS_ASOC(a))
+		rtw_free_assoc_resources(a, _TRUE);
+	rtw_free_mlme_priv_ie_data(mlme);
+
+	rtw_indicate_disconnect(a, *(u16 *)stadel->rsvd, stadel->locally_generated);
+
+#ifndef CONFIG_RTW_FSM_BTM
+	_rtw_roaming(a, roam_target, RTW_ROAM_ON_RESUME);
+#endif
+}
+
+void rtw_stadel_event_callback(_adapter *adapter, u8 *pbuf)
+{
+	struct sta_info *psta;
+	struct wlan_network *pwlan = NULL;
+	WLAN_BSSID_EX    *pdev_network = NULL;
+	u8 *pibss = NULL;
+	struct	mlme_priv	*pmlmepriv = &(adapter->mlmepriv);
+	struct	stadel_event *pstadel	= (struct stadel_event *)pbuf;
+	struct wlan_network *tgt_network = &(pmlmepriv->dev_cur_network);
+	struct wlan_mld_network *pmld_network = NULL;
+	struct wlan_network *pnetwork = NULL;
+	int network_num = 0;
+	u8 lidx;
+
+	RTW_INFO("%s(mac_id=%d)=" MAC_FMT "\n", __func__, pstadel->mac_id, MAC_ARG(pstadel->macaddr));
+
+	psta = rtw_get_stainfo(&adapter->stapriv, pstadel->macaddr);
+
+	if (psta == NULL) {
+		RTW_INFO("%s(mac_id=%d)=" MAC_FMT " psta == NULL\n", __func__, pstadel->mac_id, MAC_ARG(pstadel->macaddr));
+		/*rtw_warn_on(1);*/
+	}
+
+	if (psta) {
+		rtw_wfd_st_switch(psta, 0);
+		psta->hw_decrypted = _FALSE;
+	}
+
+	if (MLME_IS_MESH(adapter)) {
+		rtw_free_mld_stainfo(adapter, psta->phl_sta->mld);
+		goto exit;
+	}
+
+	if (MLME_IS_AP(adapter)) {
+#ifdef CONFIG_IOCTL_CFG80211
+#ifdef COMPAT_KERNEL_RELEASE
+
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)) || defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER)
+		rtw_cfg80211_indicate_sta_disassoc(adapter, pstadel->macaddr, *(u16 *)pstadel->rsvd);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)) || defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER) */
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+		rtw_hw_disconnect(adapter, psta);
+
+		rtw_free_mld_stainfo(adapter, psta->phl_sta->mld);
+
+		goto exit;
+	}
+
+	mlmeext_sta_del_event_callback(adapter);
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	if (MLME_IS_STA(adapter))
+		_stadel_posthandle_sta(adapter, pstadel);
+
+	if (MLME_IS_ADHOC_MASTER(adapter) ||
+	    MLME_IS_ADHOC(adapter)) {
+
+		/* _rtw_spinlock_bh(&(pstapriv->sta_hash_lock)); */
+		rtw_free_mld_stainfo(adapter, psta->phl_sta->mld);
+		/* _rtw_spinunlock_bh(&(pstapriv->sta_hash_lock)); */
+
+		if (adapter->stapriv.asoc_sta_count == 1) { /* a sta + bc/mc_stainfo (not Ibss_stainfo) */
+			/* rtw_indicate_disconnect(adapter); */ /* removed@20091105 */
+			_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+			_rtw_spinlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+			/* free old ibss network */
+			/* pwlan = _rtw_find_network(&pmlmepriv->scanned_queue, pstadel->macaddr); */
+			pwlan = _rtw_find_network(&pmlmepriv->scanned_queue, tgt_network->network.MacAddress);
+			if (pwlan) {
+				pwlan->fixed = _FALSE;
+#ifdef CONFIG_80211BE_EHT
+				if (pwlan->network.is_mld) {
+					pmld_network = pwlan->network.mld_network;
+					network_num = pmld_network->network_num;
+					for (lidx = 0; lidx < network_num; lidx++) {
+						pnetwork = GET_LINK_NETWORK(pmld_network, lidx);
+						if (pnetwork) {
+							rtw_free_network_nolock(adapter, pnetwork);
+							pmld_network->network_num--;
+						}
+					}
+					rtw_free_mld_network_nolock(adapter, pmld_network);
+				} else
+#endif
+				{
+					rtw_free_network_nolock(adapter, pwlan);
+				}
+			}
+			_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+			_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+			/* re-create ibss */
+			pdev_network = &(adapter->registrypriv.dev_network);
+			pibss = adapter->registrypriv.dev_network.MacAddress;
+
+			_rtw_memcpy(pdev_network, &tgt_network->network, get_WLAN_BSSID_EX_sz(&tgt_network->network));
+
+			_rtw_memset(&pdev_network->Ssid, 0, sizeof(NDIS_802_11_SSID));
+			_rtw_memcpy(&pdev_network->Ssid, &pmlmepriv->assoc_ssid, sizeof(NDIS_802_11_SSID));
+
+			rtw_update_registrypriv_dev_network(adapter);
+
+			rtw_generate_random_ibss(pibss);
+
+			if (MLME_IS_ADHOC(adapter)) {
+				set_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+				_clr_fwstate_(pmlmepriv, WIFI_ADHOC_STATE);
+			}
+
+			if (rtw_create_ibss_cmd(adapter, 0) != _SUCCESS)
+				RTW_ERR("rtw_create_ibss_cmd FAIL\n");
+
+		}
+
+	}
+
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+exit:
+	#ifdef CONFIG_RTS_FULL_BW
+	rtw_set_rts_bw(adapter);
+	#endif/*CONFIG_RTS_FULL_BW*/
+	return;
+}
+
+void rtw_wmm_event_callback(_adapter *padapter, u8 *pbuf)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	WMMOnAssocRsp(padapter, padapter_link);
+}
+
+/*
+* rtw_join_timeout_handler - Timeout/failure handler for CMD JoinBss
+*/
+void rtw_join_timeout_handler(void *ctx)
+{
+	_adapter *adapter = (_adapter *)ctx;
+	struct	mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+
+#if 0
+	if (dev_is_drv_stopped(adapter_to_dvobj(adapter))) {
+		_rtw_up_sema(&pmlmepriv->assoc_terminate);
+		return;
+	}
+#endif
+
+
+
+	RTW_INFO("%s, fw_state=%x\n", __FUNCTION__, get_fwstate(pmlmepriv));
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+		return;
+
+	/* SAE auth timeout, send association request to force trigger disconnected by AP */
+	if ((check_fwstate(pmlmepriv, WIFI_STATION_STATE) == _TRUE) &&
+	    (pmlmeinfo->state & WIFI_FW_AUTH_STATE) &&
+	    rtw_sec_chk_auth_type(adapter, MLME_AUTHTYPE_SAE)) {
+		RTW_INFO("%s: issue_assocreq due to SAE auth timeout.\n", __FUNCTION__);
+		issue_assocreq(adapter);
+	}
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE)
+		rtw_connect_abort(adapter);
+
+#ifdef CONFIG_LAYER2_ROAMING
+	if (rtw_to_roam(adapter) > 0) { /* join timeout caused by roaming */
+		while (1) {
+			rtw_dec_to_roam(adapter);
+			if (rtw_to_roam(adapter) != 0) { /* try another */
+				int do_join_r;
+				RTW_INFO("%s try another roaming\n", __FUNCTION__);
+				do_join_r = rtw_do_join(adapter);
+				if (_SUCCESS != do_join_r) {
+					RTW_INFO("%s roaming do_join return %d\n", __FUNCTION__ , do_join_r);
+					continue;
+				}
+				break;
+			} else {
+				RTW_INFO("%s We've try roaming but fail\n", __FUNCTION__);
+#ifdef CONFIG_RTW_80211R
+				/* rtw_ft_clr_flags(adapter, RTW_FT_PEER_EN|RTW_FT_PEER_OTD_EN); */
+				rtw_ft_reset_status(adapter);
+#endif
+				rtw_indicate_disconnect(adapter, pmlmepriv->join_status, _FALSE);
+				pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+				pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_JOINBSS_TIMEOUT;
+				pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+				break;
+			}
+		}
+
+	} else
+#endif
+	{
+		rtw_indicate_disconnect(adapter, pmlmepriv->join_status, _FALSE);
+		pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+		pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_JOINBSS_TIMEOUT;
+		pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+		free_scanqueue(pmlmepriv);/* ??? */
+		free_mld_scanqueue(pmlmepriv);
+
+#ifdef CONFIG_IOCTL_CFG80211
+		/* indicate disconnect for the case that join_timeout and check_fwstate != FW_LINKED */
+		rtw_cfg80211_indicate_disconnect(adapter, pmlmepriv->join_status, _FALSE);
+#endif /* CONFIG_IOCTL_CFG80211 */
+	}
+
+	pmlmepriv->join_status = 0; /* reset */
+	pmlmeinfo->state = 0; /* clear WIFI_FW_STATE */
+
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+
+#ifdef CONFIG_DRVEXT_MODULE_WSC
+	drvext_assoc_fail_indicate(&adapter->drvextpriv);
+#endif
+
+
+
+}
+
+void rtw_mlme_reset_auto_scan_int(_adapter *adapter, u8 *reason)
+{
+#if defined(CONFIG_RTW_MESH) && defined(CONFIG_DFS_MASTER)
+#if CONFIG_RTW_MESH_OFFCH_CAND
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+#endif
+#endif
+	u32 interval_ms = U32_MAX; /* special value to make min() works well, also means no auto scan */
+
+	*reason = RTW_AUTO_SCAN_REASON_UNSPECIFIED;
+
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(adapter), BW_CAP_40M)
+		&& adapter->registrypriv.wifi_spec
+		&& is_client_associated_to_ap(adapter) == _TRUE
+	) {
+		struct _ADAPTER_LINK *alink;
+		int i;
+
+		for (i = 0; i < ADAPTER_GET_LINK_NUM(adapter); i++) {
+			alink = GET_LINK(adapter, i);
+			if (LINK_MLME_IS_ASOC(alink)
+				&& ALINK_GET_BAND(alink) == BAND_ON_24G
+				/* TODO: AP Connected is 40MHz capability? */
+			) {
+				interval_ms = rtw_min(interval_ms, 60 * 1000);
+				*reason |= RTW_AUTO_SCAN_REASON_2040_BSS;
+				break;
+			}
+		}
+	}
+
+#ifdef CONFIG_80211D
+	if (adapter_to_rfctl(adapter)->cis_enabled
+		&& (adapter_to_rfctl(adapter)->cis_flags & CISF_ENV_BSS)
+		&& adapter_to_rfctl(adapter)->cis_scan_int_ms
+	) {
+		interval_ms = rtw_min(interval_ms, adapter_to_rfctl(adapter)->cis_scan_int_ms);
+		*reason |= RTW_AUTO_SCAN_REASON_CIS_ENV_BSS;
+	}
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	#if CONFIG_RTW_MESH_OFFCH_CAND
+	if (adapter->mesh_cfg.peer_sel_policy.offch_find_int_ms
+		&& rtw_mesh_offch_candidate_accepted(adapter)
+		#ifdef CONFIG_DFS_MASTER
+		&& (!rfctl->radar_detect_enabled || (IS_CH_WAITING(rfctl) && !IS_UNDER_CAC(rfctl)))
+		#endif
+	) {
+		interval_ms = rtw_min(interval_ms, adapter->mesh_cfg.peer_sel_policy.offch_find_int_ms);
+		*reason |= RTW_AUTO_SCAN_REASON_MESH_OFFCH_CAND;
+	}
+	#endif
+#endif /* CONFIG_RTW_MESH */
+
+	if (interval_ms == U32_MAX)
+		interval_ms = 0;
+
+	rtw_mlme_set_auto_scan_int(adapter, interval_ms);
+	return;
+}
+
+
+static void rtw_auto_scan_handler(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	u8 reason = RTW_AUTO_SCAN_REASON_UNSPECIFIED;
+#ifdef CONFIG_LAYER2_ROAMING
+	struct recv_info	*precvinfo = &padapter->recvinfo;
+#ifdef CONFIG_RTW_FSM_BTM
+	struct sta_priv	*pstapriv = &(padapter->stapriv);
+	struct sta_info *psta;
+#endif
+#endif
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	struct registry_priv *regsty = &padapter->registrypriv;
+#endif
+#endif
+
+	rtw_mlme_reset_auto_scan_int(padapter, &reason);
+
+#ifdef CONFIG_P2P
+	if (rtw_p2p_chk_role(&padapter->wdinfo, P2P_ROLE_CLIENT) ||
+	    rtw_p2p_chk_role(&padapter->wdinfo, P2P_ROLE_GO))
+		goto exit;
+#endif
+
+#ifdef CONFIG_TDLS
+	if (padapter->tdlsinfo.link_established == _TRUE)
+		goto exit;
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	if(regsty->wifi_spec && regsty->twt_en)
+		goto exit;
+#endif
+#endif
+
+#ifdef CONFIG_LAYER2_ROAMING
+	if (is_client_associated_to_ap(padapter) &&
+		rtw_chk_roam_flags(padapter, RTW_ROAM_ACTIVE)) {
+		RTW_INFO("avg_val = %d, need_to_roam=%d\n", precvinfo->signal_strength_data.avg_val, pmlmepriv->need_to_roam);
+		if (precvinfo->signal_strength_data.avg_val < pmlmepriv->roam_rssi_threshold) {
+			pmlmepriv->need_to_roam = _TRUE;
+			if (rtw_get_passing_time_ms(pmlmepriv->last_roaming) >= pmlmepriv->roam_scan_int*2000) {
+#if defined(CONFIG_LAYER2_ROAMING) && defined(CONFIG_RTW_80211K)
+				//rtw_roam_nb_discover(padapter, _FALSE);
+#endif
+				reason = RTW_AUTO_SCAN_REASON_ROAM_ACTIVE;
+				pmlmepriv->last_roaming = rtw_get_current_time();
+				goto do_scan;
+			}
+		} else
+			pmlmepriv->need_to_roam = _FALSE;
+	}
+#endif
+	if (pmlmepriv->auto_scan_int_ms == 0
+	    || rtw_get_passing_time_ms(pmlmepriv->scan_start_time) < pmlmepriv->auto_scan_int_ms)
+		goto exit;
+do_scan:
+#ifdef CONFIG_RTW_FSM_BTM
+	if (pmlmepriv->need_to_roam) {
+		psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+		if (psta)
+			rtw_btm_new_obj(padapter, psta, &pmlmepriv->nb_info, RTW_ROAM_ACTIVE);
+	}
+#else
+	rtw_drv_scan_by_self(padapter, reason);
+#endif
+exit:
+	return;
+}
+
+static void rtw_reset_snr_statistics(struct _ADAPTER *adapter)
+{
+	_list	*plist, *phead;
+	struct sta_info *psta = NULL;
+	u8 sta_mac[NUM_STA][ETH_ALEN] = {{0}};
+	uint mac_id[NUM_STA];
+	struct stainfo_stats	*pstats = NULL;
+	struct sta_priv	*pstapriv = &(adapter->stapriv);
+	u32 i, j, macid_rec_idx = 0;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct xmit_priv *pxmitpriv = &(adapter->xmitpriv);
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+
+			if (psta->phl_sta == NULL) {
+				RTW_WARN("%s: phl_sta is null\n", __func__);
+				continue;
+			}
+
+			if ((_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, 6) !=  _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, 6) != _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, adapter_mac_addr(adapter), 6) != _TRUE)) {
+				_rtw_memcpy(&sta_mac[macid_rec_idx][0], psta->phl_sta->mac_addr, ETH_ALEN);
+				mac_id[macid_rec_idx] = psta->phl_sta->macid;
+				macid_rec_idx++;
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < macid_rec_idx; i++) {
+		psta = rtw_get_stainfo(pstapriv, &sta_mac[i][0]);
+		if(psta) {
+			_rtw_spinlock_bh(&psta->lock);
+			psta->snr_num = 0;
+			for (j = 0; j < RTW_PHL_MAX_RF_PATH; j++) {
+				psta->snr_fd_total[j] = 0;
+				psta->snr_td_total[j] = 0;
+				psta->snr_fd_avg[j] = 0;
+				psta->snr_td_avg[j]= 0;
+			}
+			_rtw_spinunlock_bh(&psta->lock);
+		} else {
+			RTW_INFO("STA is gone\n");
+		}
+	}
+}
+void rtw_iface_dynamic_check_handlder(struct _ADAPTER *a)
+{
+	if (!a->netif_up)
+		return;
+	/* reset snr related information */
+	rtw_reset_snr_statistics(a);
+
+	/* auto site survey */
+	rtw_auto_scan_handler(a);
+	dynamic_update_bcn_check(a);
+	traffic_status_watchdog(a);
+
+#ifdef CONFIG_BR_EXT
+if (!adapter_use_wds(a)) {
+	#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 35))
+		rcu_read_lock();
+	#endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 35)) */
+
+	#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35))
+		if (a->pnetdev->br_port
+	#else	/* (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35)) */
+		if (rcu_dereference(a->pnetdev->rx_handler_data)
+	#endif /* (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35)) */
+			&& (MLME_IS_STA(a) || MLME_IS_ADHOC(a))) {
+			/* expire NAT2.5 entry */
+			void nat25_db_expire(_adapter *priv);
+			nat25_db_expire(a);
+
+			if (a->pppoe_connection_in_progress > 0)
+				a->pppoe_connection_in_progress--;
+			if (a->pppoe_connection_in_progress > 0)
+				a->pppoe_connection_in_progress--;
+		}
+
+	#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 35))
+		rcu_read_unlock();
+	#endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 35)) */
+}
+#endif /* CONFIG_BR_EXT */
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+        rtw_cfgvendor_rssi_monitor_evt(a);
+#endif
+}
+
+#if 0
+void rtw_iface_dynamic_check_timer_handlder(_adapter *adapter)
+{
+#ifdef CONFIG_AP_MODE
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+#endif /* CONFIG_AP_MODE */
+
+	if (!adapter->netif_up)
+		return;
+
+#ifdef CONFIG_AP_MODE
+	if (MLME_IS_AP(adapter)|| MLME_IS_MESH(adapter)) {
+		#ifdef CONFIG_BMC_TX_RATE_SELECT
+		rtw_update_bmc_sta_tx_rate(adapter);
+		#endif /*CONFIG_BMC_TX_RATE_SELECT*/
+	}
+#endif /*CONFIG_AP_MODE*/
+
+	rtw_iface_dynamic_check_handlder(adapter);
+}
+#endif
+/*TP_avg(t) = (1/10) * TP_avg(t-1) + (9/10) * TP(t) MBps*/
+static void collect_sta_traffic_statistics(_adapter *adapter)
+{
+/* ToDo */
+#if 0
+	struct macid_ctl_t *macid_ctl = &adapter->dvobj->macid_ctl;
+	struct sta_info *sta;
+	u64 curr_tx_bytes = 0, curr_rx_bytes = 0;
+	u32 curr_tx_mbytes = 0, curr_rx_mbytes = 0;
+	int i;
+
+	for (i = 0; i < MACID_NUM_SW_LIMIT; i++) {
+		sta = macid_ctl->sta[i];
+		if (sta && !is_broadcast_mac_addr(sta->phl_sta->mac_addr)) {
+			if (sta->sta_stats.last_tx_bytes > sta->sta_stats.tx_bytes)
+				sta->sta_stats.last_tx_bytes =  sta->sta_stats.tx_bytes;
+			if (sta->sta_stats.last_rx_bytes > sta->sta_stats.rx_bytes)
+				sta->sta_stats.last_rx_bytes = sta->sta_stats.rx_bytes;
+			if (sta->sta_stats.last_rx_bc_bytes > sta->sta_stats.rx_bc_bytes)
+				sta->sta_stats.last_rx_bc_bytes = sta->sta_stats.rx_bc_bytes;
+			if (sta->sta_stats.last_rx_mc_bytes > sta->sta_stats.rx_mc_bytes)
+				sta->sta_stats.last_rx_mc_bytes = sta->sta_stats.rx_mc_bytes;
+
+			curr_tx_bytes = sta->sta_stats.tx_bytes - sta->sta_stats.last_tx_bytes;
+			curr_rx_bytes = sta->sta_stats.rx_bytes - sta->sta_stats.last_rx_bytes;
+			sta->sta_stats.tx_tp_kbits = (curr_tx_bytes * 8 / 2) >> 10;/*Kbps*/
+			sta->sta_stats.rx_tp_kbits = (curr_rx_bytes * 8 / 2) >> 10;/*Kbps*/
+
+			sta->sta_stats.smooth_tx_tp_kbits = (sta->sta_stats.smooth_tx_tp_kbits * 6 / 10) + (sta->sta_stats.tx_tp_kbits * 4 / 10);/*Kbps*/
+			sta->sta_stats.smooth_rx_tp_kbits = (sta->sta_stats.smooth_rx_tp_kbits * 6 / 10) + (sta->sta_stats.rx_tp_kbits * 4 / 10);/*Kbps*/
+
+			curr_tx_mbytes = (curr_tx_bytes / 2) >> 20;/*MBps*/
+			curr_rx_mbytes = (curr_rx_bytes / 2) >> 20;/*MBps*/
+
+			sta->phl_sta->tx_moving_average_tp =
+				(sta->phl_sta->tx_moving_average_tp / 10) + (curr_tx_mbytes * 9 / 10); /*MBps*/
+
+			sta->phl_sta->rx_moving_average_tp =
+				(sta->phl_sta->rx_moving_average_tp / 10) + (curr_rx_mbytes * 9 /10); /*MBps*/
+
+			rtw_collect_bcn_info(sta->padapter);
+
+			if (adapter->bsta_tp_dump)
+				rtw_hal_dump_sta_traffic(RTW_DBGDUMP, adapter, sta);
+
+			sta->sta_stats.last_tx_bytes = sta->sta_stats.tx_bytes;
+			sta->sta_stats.last_rx_bytes = sta->sta_stats.rx_bytes;
+			sta->sta_stats.last_rx_bc_bytes = sta->sta_stats.rx_bc_bytes;
+			sta->sta_stats.last_rx_mc_bytes = sta->sta_stats.rx_mc_bytes;
+		}
+	}
+#endif
+}
+
+void rtw_sta_traffic_info(void *sel, _adapter *adapter)
+{
+/* ToDo */
+#if 0
+	struct macid_ctl_t *macid_ctl = &adapter->dvobj->macid_ctl;
+	struct sta_info *sta;
+	int i;
+
+	for (i = 0; i < MACID_NUM_SW_LIMIT; i++) {
+		sta = macid_ctl->sta[i];
+		if (sta && !is_broadcast_mac_addr(sta->phl_sta->mac_addr))
+			rtw_hal_dump_sta_traffic(sel, adapter, sta);
+	}
+#endif
+}
+
+/*#define DBG_TRAFFIC_STATISTIC*/
+static void collect_traffic_statistics(_adapter *padapter)
+{
+	struct dvobj_priv	*pdvobjpriv = adapter_to_dvobj(padapter);
+
+	/*_rtw_memset(&pdvobjpriv->traffic_stat, 0, sizeof(struct rtw_traffic_statistics));*/
+
+	/* Tx bytes reset*/
+	pdvobjpriv->traffic_stat.tx_bytes = 0;
+	pdvobjpriv->traffic_stat.tx_pkts = 0;
+	pdvobjpriv->traffic_stat.tx_drop = 0;
+
+	/* Rx bytes reset*/
+	pdvobjpriv->traffic_stat.rx_bytes = 0;
+	pdvobjpriv->traffic_stat.rx_pkts = 0;
+	pdvobjpriv->traffic_stat.rx_drop = 0;
+
+	rtw_mi_traffic_statistics(padapter);
+
+	/* Calculate throughput in last interval */
+	pdvobjpriv->traffic_stat.cur_tx_bytes = pdvobjpriv->traffic_stat.tx_bytes - pdvobjpriv->traffic_stat.last_tx_bytes;
+	pdvobjpriv->traffic_stat.cur_rx_bytes = pdvobjpriv->traffic_stat.rx_bytes - pdvobjpriv->traffic_stat.last_rx_bytes;
+	pdvobjpriv->traffic_stat.last_tx_bytes = pdvobjpriv->traffic_stat.tx_bytes;
+	pdvobjpriv->traffic_stat.last_rx_bytes = pdvobjpriv->traffic_stat.rx_bytes;
+
+	pdvobjpriv->traffic_stat.cur_tx_tp = (u32)(pdvobjpriv->traffic_stat.cur_tx_bytes * 8 / 2 / 1024 / 1024);/*Mbps*/
+	pdvobjpriv->traffic_stat.cur_rx_tp = (u32)(pdvobjpriv->traffic_stat.cur_rx_bytes * 8 / 2 / 1024 / 1024);/*Mbps*/
+
+	#ifdef DBG_TRAFFIC_STATISTIC
+	RTW_INFO("\n========================\n");
+	RTW_INFO("cur_tx_bytes:%lld\n", pdvobjpriv->traffic_stat.cur_tx_bytes);
+	RTW_INFO("cur_rx_bytes:%lld\n", pdvobjpriv->traffic_stat.cur_rx_bytes);
+
+	RTW_INFO("last_tx_bytes:%lld\n", pdvobjpriv->traffic_stat.last_tx_bytes);
+	RTW_INFO("last_rx_bytes:%lld\n", pdvobjpriv->traffic_stat.last_rx_bytes);
+
+	RTW_INFO("cur_tx_tp:%d (Mbps)\n", pdvobjpriv->traffic_stat.cur_tx_tp);
+	RTW_INFO("cur_rx_tp:%d (Mbps)\n", pdvobjpriv->traffic_stat.cur_rx_tp);
+	#endif
+
+#ifdef CONFIG_RTW_NAPI
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+	dynamic_napi_th_chk (padapter);
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+#endif
+
+}
+
+static void rtw_accumulate_fa_count(struct dvobj_priv *dvobj)
+{
+	u32 fa_count;
+	enum phl_band_idx band_idx;
+
+	for (band_idx = HW_BAND_0; band_idx < HW_BAND_MAX; band_idx++) {
+		fa_count = rtw_phl_get_phy_stat_info(GET_PHL_INFO(dvobj),
+						     band_idx,
+						     STAT_INFO_FA_ALL);
+		ATOMIC_ADD((ATOMIC_T *)&dvobj->fa_cnt_acc[band_idx],
+			   (int)fa_count);
+	}
+}
+
+#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+void rtw_dynamic_check_timer_handlder(void *ctx)
+{
+	struct dvobj_priv *pdvobj = (struct dvobj_priv *)ctx;
+	_adapter *adapter = dvobj_get_primary_adapter(pdvobj);
+
+	if (!adapter)
+		goto exit;
+
+#if (MP_DRIVER == 1)
+	if (adapter->registrypriv.mp_mode == 1 && adapter->mppriv.mp_dm == 0) { /* for MP ODM dynamic Tx power tracking */
+		/* RTW_INFO("%s mp_dm =0 return\n", __func__); */
+		goto exit;
+	}
+#endif
+
+	if (!rtw_hw_is_init_completed(pdvobj))
+		goto exit;
+
+	if (RTW_CANNOT_RUN(pdvobj))
+		goto exit;
+
+	collect_traffic_statistics(adapter);
+	collect_sta_traffic_statistics(adapter);
+	rtw_mi_dynamic_check_timer_handlder(adapter);
+
+	rtw_dynamic_chk_wk_cmd(adapter);
+
+exit:
+	_set_timer(&pdvobj->dynamic_chk_timer, 2000);
+}
+#endif
+
+void rtw_core_watchdog_sw_post_hdlr(void *drv_priv)
+{
+	struct dvobj_priv *pdvobj = (struct dvobj_priv *)drv_priv;
+	struct _ADAPTER *adapter = dvobj_get_primary_adapter(pdvobj);
+
+	if (!adapter)
+		goto exit;
+#if (MP_DRIVER == 1)
+	/* for MP ODM dynamic Tx power tracking */
+	if (adapter->registrypriv.mp_mode == 1 && adapter->mppriv.mp_dm == 0) {
+		/* RTW_INFO("%s mp_dm =0 return\n", __func__); */
+		goto exit;
+	}
+#endif
+exit:
+	return;
+}
+
+void rtw_core_watchdog_sw_hdlr(void *drv_priv)
+{
+	struct dvobj_priv *pdvobj = (struct dvobj_priv *)drv_priv;
+	struct _ADAPTER *adapter = dvobj_get_primary_adapter(pdvobj);
+
+	if (!adapter)
+		goto exit;
+#if (MP_DRIVER == 1)
+	/* for MP ODM dynamic Tx power tracking */
+	if (adapter->registrypriv.mp_mode == 1 && adapter->mppriv.mp_dm == 0) {
+		/* RTW_INFO("%s mp_dm =0 return\n", __func__); */
+		goto exit;
+	}
+#endif
+
+	/* SW only dynamic/periodic check even when device is idle */
+	rtw_regd_watchdog_hdl(pdvobj);
+
+	if (!rtw_hw_is_init_completed(pdvobj))
+		goto exit;
+
+	collect_traffic_statistics(adapter);
+	collect_sta_traffic_statistics(adapter);
+	rtw_mi_dynamic_check_handlder(adapter);
+	rtw_mi_keep_alive_pre_chk_hdl(adapter);
+	rtw_dynamic_chk_wk_sw_hdl(adapter);
+	rtw_accumulate_fa_count(pdvobj);
+
+exit:
+	return;
+}
+
+void rtw_core_watchdog_hw_hdlr(void *drv_priv)
+{
+	struct dvobj_priv *pdvobj = (struct dvobj_priv *)drv_priv;
+	struct _ADAPTER *adapter = dvobj_get_primary_adapter(pdvobj);
+
+	if (!adapter)
+		goto exit;
+#if (MP_DRIVER == 1)
+	/* for MP ODM dynamic Tx power tracking */
+	if (adapter->registrypriv.mp_mode == 1 && adapter->mppriv.mp_dm == 0) {
+		/* RTW_INFO("%s mp_dm =0 return\n", __func__); */
+		goto exit;
+	}
+#endif
+	if (!rtw_hw_is_init_completed(pdvobj))
+		goto exit;
+
+	if (RTW_CANNOT_RUN(pdvobj))
+		goto exit;
+
+	rtw_dynamic_chk_wk_hw_hdl(adapter);
+
+exit:
+	return;
+}
+
+#ifdef CONFIG_LAYER2_ROAMING
+/*
+* Select a new roaming candidate from the original @param candidate and @param competitor
+* @return _TRUE: candidate is updated
+* @return _FALSE: candidate is not updated
+*/
+int rtw_check_roaming_candidate(struct mlme_priv *mlme
+	, struct wlan_network **candidate, struct wlan_network *competitor)
+{
+	int updated = _FALSE;
+	_adapter *adapter = container_of(mlme, _adapter, mlmepriv);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct roam_nb_info *pnb = &(mlme->nb_info);
+	u8 *reason = NULL;
+
+	if (!rtw_network_chk_opch_status(rfctl, competitor))
+		goto exit;
+	if (!rtw_network_chk_regu_ies(rfctl, competitor))
+		goto exit;
+
+	if (is_same_ess(&competitor->network, &mlme->dev_cur_network.network) == _FALSE)
+		goto exit;
+	if (rtw_is_desired_network(adapter, competitor) == _FALSE)
+		goto exit;
+#ifdef CONFIG_LAYER2_ROAMING
+	if (mlme->need_to_roam == _FALSE)
+		goto exit;
+#endif
+	/* got specific addr to roam */
+	if (!is_zero_mac_addr(mlme->roam_tgt_addr)) {
+		if (_rtw_memcmp(mlme->roam_tgt_addr, competitor->network.MacAddress, ETH_ALEN) == _TRUE)
+			goto update;
+		reason = "target";
+		goto exit;
+	}
+#ifdef CONFIG_RTW_80211R
+	if (rtw_ft_chk_flags(adapter, RTW_FT_PEER_EN)) {
+		if (rtw_ft_chk_roaming_candidate(adapter, competitor) == _FALSE) {
+			reason = "ft";
+			goto exit;
+		}
+	}
+#ifdef CONFIG_RTW_WNM
+	if (rtw_wnm_btm_diff_bss(adapter, pnb) &&
+		rtw_wnm_btm_roam_candidate(adapter, pnb, competitor)) {
+		goto update;
+	}
+#endif
+#endif
+#ifdef DBG_WNM_UNIT_TEST
+	if (_rtw_memcmp(mlme->cur_network_scanned->network.MacAddress,
+		competitor->network.MacAddress, ETH_ALEN) == _TRUE) {
+		printk("roam: omit Serving AP\n");
+		reason ="ServeAP";
+		goto exit;
+	}
+#else
+	if (rtw_get_passing_time_ms(competitor->last_scanned) >= mlme->roam_scanr_exp_ms) {
+		reason = "age";
+		goto exit;
+	}
+
+	if (competitor->network.PhyInfo.rssi -
+		mlme->cur_network_scanned->network.PhyInfo.rssi < mlme->roam_rssi_diff_th) {
+		reason = "delta";
+		goto exit;
+	}
+
+	if (*candidate != NULL && (*candidate)->network.PhyInfo.rssi >= competitor->network.PhyInfo.rssi) {
+		reason = "rssi";
+		goto exit;
+	}
+#endif
+update:
+	reason = "ok";
+	*candidate = competitor;
+	updated = _TRUE;
+exit:
+	rtw_wnm_candidate_info(competitor, mlme->cur_network_scanned, reason);
+	return updated;
+}
+
+struct wlan_network *rtw_select_roaming_candidate(struct mlme_priv *mlme)
+{
+	//int ret = _FAIL;
+	_list	*phead;
+	_adapter *adapter;
+	_queue	*queue	= &(mlme->scanned_queue);
+	struct	wlan_network	*pnetwork = NULL;
+	struct	wlan_network	*candidate = NULL;
+
+	if (mlme->cur_network_scanned == NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	_rtw_spinlock_bh(&(mlme->scanned_queue.lock));
+	phead = get_list_head(queue);
+	adapter = (_adapter *)mlme->nic_hdl;
+
+	mlme->pscanned = get_next(phead);
+
+	while (!rtw_end_of_queue_search(phead, mlme->pscanned)) {
+
+		pnetwork = LIST_CONTAINOR(mlme->pscanned, struct wlan_network, list);
+		if (pnetwork == NULL)
+			goto exit;
+
+		mlme->pscanned = get_next(mlme->pscanned);
+
+		if (0)
+			RTW_INFO("%s("MAC_FMT", ch%u) rssi:%d\n"
+				 , pnetwork->network.Ssid.Ssid
+				 , MAC_ARG(pnetwork->network.MacAddress)
+				 , pnetwork->network.Configuration.DSConfig
+				 , (int)pnetwork->network.PhyInfo.rssi);
+
+		rtw_check_roaming_candidate(mlme, &candidate, pnetwork);
+
+	}
+
+	if (candidate == NULL) {
+	/*	if parent note lost the path to root and there is no other cadidate, report disconnection	*/
+		RTW_INFO("%s: return _FAIL(candidate == NULL)\n", __FUNCTION__);
+		goto exit;
+	} else {
+		RTW_INFO("%s: candidate: %s("MAC_FMT", ch:%u)\n", __FUNCTION__,
+			candidate->network.Ssid.Ssid, MAC_ARG(candidate->network.MacAddress),
+			 candidate->network.Configuration.DSConfig);
+		mlme->roam_network = candidate;
+
+		if (_rtw_memcmp(candidate->network.MacAddress, mlme->roam_tgt_addr, ETH_ALEN) == _TRUE)
+			_rtw_memset(mlme->roam_tgt_addr, 0, ETH_ALEN);
+	}
+
+exit:
+	_rtw_spinunlock_bh(&(mlme->scanned_queue.lock));
+
+	return candidate;
+}
+#endif /* CONFIG_LAYER2_ROAMING */
+
+/*
+* Select a new join candidate from the original @param candidate and @param competitor
+* @return _TRUE: candidate is updated
+* @return _FALSE: candidate is not updated
+*/
+int rtw_check_join_candidate(struct mlme_priv *mlme
+	    , struct wlan_network **candidate, struct wlan_network *competitor)
+{
+	int updated = _FALSE;
+	_adapter *adapter = container_of(mlme, _adapter, mlmepriv);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	if (!rtw_network_chk_opch_status(rfctl, competitor))
+		goto exit;
+	if (!rtw_network_chk_regu_ies(rfctl, competitor))
+		goto exit;
+
+	/* check bssid, if needed */
+	if (mlme->assoc_by_bssid == _TRUE) {
+		if (_rtw_memcmp(competitor->network.MacAddress, mlme->assoc_bssid, ETH_ALEN) == _FALSE)
+			goto exit;
+	}
+
+	/* check ssid, if needed */
+	if (mlme->assoc_ssid.Ssid[0] && mlme->assoc_ssid.SsidLength) {
+		if (competitor->network.Ssid.SsidLength != mlme->assoc_ssid.SsidLength
+		    || _rtw_memcmp(competitor->network.Ssid.Ssid, mlme->assoc_ssid.Ssid, mlme->assoc_ssid.SsidLength) == _FALSE
+		   )
+			goto exit;
+	}
+
+	if (rtw_is_desired_network(adapter, competitor)  == _FALSE)
+		goto exit;
+
+#ifdef CONFIG_LAYER2_ROAMING
+	if (rtw_to_roam(adapter) > 0) {
+		if (rtw_get_passing_time_ms(competitor->last_scanned) >= mlme->roam_scanr_exp_ms
+		    || is_same_ess(&competitor->network, &mlme->dev_cur_network.network) == _FALSE
+		   )
+			goto exit;
+	}
+#endif
+
+	if (*candidate == NULL || (*candidate)->network.PhyInfo.rssi < competitor->network.PhyInfo.rssi) {
+		*candidate = competitor;
+		updated = _TRUE;
+	}
+
+	if (updated) {
+		RTW_INFO("[by_bssid:%u][assoc_ssid:%s][to_roam:%u] "
+			 "new candidate: %s("MAC_FMT", band %u, ch %u) rssi:%d\n",
+			 mlme->assoc_by_bssid,
+			 mlme->assoc_ssid.Ssid,
+			 rtw_to_roam(adapter),
+			 (*candidate)->network.Ssid.Ssid,
+			 MAC_ARG((*candidate)->network.MacAddress),
+			 (*candidate)->network.Configuration.Band,
+			 (*candidate)->network.Configuration.DSConfig,
+			 (int)(*candidate)->network.PhyInfo.rssi
+			);
+	}
+
+exit:
+	return updated;
+}
+
+/*
+Calling context:
+The caller of the sub-routine will be in critical section...
+
+The caller must hold the following spinlock
+
+pmlmepriv->lock
+
+
+*/
+
+int rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv)
+{
+	int ret;
+	_list	*phead;
+	_adapter *adapter;
+	_queue	*queue	= &(pmlmepriv->scanned_queue);
+	struct	wlan_network	*pnetwork = NULL;
+	struct	wlan_network	*candidate = NULL;
+#ifdef CONFIG_ANTENNA_DIVERSITY
+	u8		bSupportAntDiv = _FALSE;
+#endif
+	struct mlme_ext_priv *pmlmeext;
+	struct mlme_ext_info *pmlmeinfo;
+	struct _ADAPTER_LINK *adapter_link = NULL;
+
+
+	adapter = (_adapter *)pmlmepriv->nic_hdl;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only]
+	** depends on the info provided by the upper layer */
+	adapter_link = GET_PRIMARY_LINK(adapter);
+	/* ToDo CONFIG_RTW_MLD: [Connect]
+	** may need to compare the info of both MLD and links
+	** depends on the info provided by the upper layer */
+	pmlmeext = &adapter->mlmeextpriv;
+	pmlmeinfo = &pmlmeext->mlmext_info;
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+
+#ifdef CONFIG_LAYER2_ROAMING
+	if (pmlmepriv->roam_network) {
+		candidate = pmlmepriv->roam_network;
+		goto candidate_exist;
+	}
+#endif
+
+	phead = get_list_head(queue);
+	pmlmepriv->pscanned = get_next(phead);
+
+	while (!rtw_end_of_queue_search(phead, pmlmepriv->pscanned)) {
+
+		pnetwork = LIST_CONTAINOR(pmlmepriv->pscanned, struct wlan_network, list);
+		if (pnetwork == NULL) {
+			ret = _FAIL;
+			goto exit;
+		}
+
+		pmlmepriv->pscanned = get_next(pmlmepriv->pscanned);
+
+		if (0)
+			RTW_INFO("%s("MAC_FMT", band %u, ch %u) rssi:%d\n"
+				 , pnetwork->network.Ssid.Ssid
+				 , MAC_ARG(pnetwork->network.MacAddress)
+				 , pnetwork->network.Configuration.Band
+				 , pnetwork->network.Configuration.DSConfig
+				 , (int)pnetwork->network.PhyInfo.rssi);
+
+		rtw_check_join_candidate(pmlmepriv, &candidate, pnetwork);
+
+	}
+
+	if (candidate == NULL) {
+		RTW_INFO("%s: return _FAIL(candidate == NULL)\n", __FUNCTION__);
+#ifdef CONFIG_WOWLAN
+		_clr_fwstate_(pmlmepriv, WIFI_ASOC_STATE | WIFI_UNDER_LINKING);
+#endif
+		ret = _FAIL;
+		goto exit;
+	} else {
+		RTW_INFO("%s: candidate: %s("MAC_FMT", band:%u, ch:%u)\n", __FUNCTION__,
+			candidate->network.Ssid.Ssid, MAC_ARG(candidate->network.MacAddress),
+			 candidate->network.Configuration.Band, candidate->network.Configuration.DSConfig);
+		goto candidate_exist;
+	}
+
+candidate_exist:
+
+	/* check for situation of  WIFI_ASOC_STATE */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+		RTW_INFO("%s: WIFI_ASOC_STATE while ask_for_joinbss!!!\n", __FUNCTION__);
+
+#if 0 /* for WPA/WPA2 authentication, wpa_supplicant will expect authentication from AP, it is needed to reconnect AP... */
+		if (is_same_network(&pmlmepriv->cur_network.network, &candidate->network)) {
+			RTW_INFO("%s: WIFI_ASOC_STATE and is same network, it needn't join again\n", __FUNCTION__);
+
+			rtw_indicate_connect(adapter);/* rtw_indicate_connect again */
+
+			ret = 2;
+			goto exit;
+		} else
+#endif
+		{
+			rtw_disassoc_cmd(adapter, 0, 0);
+			if (MLME_IS_STA(adapter) == _FALSE)
+				rtw_free_assoc_resources_cmd(adapter, _TRUE, 0);
+			rtw_indicate_disconnect(adapter, 0, _FALSE);
+
+			pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_CONNECTION_EXIST;
+			pmlmeinfo->wifi_reason_code = WLAN_REASON_DEAUTH_LEAVING;
+		}
+	}
+
+	set_fwstate(pmlmepriv, WIFI_UNDER_LINKING);
+
+	adapter_link->mlmepriv.to_join = _TRUE;
+	/* ToDo CONFIG_RTW_MLD: [Connect]
+	** may need to assign link_id and to_join to pair adapter_link and link network here
+	** depends on the info provided by the upper layer */
+#ifdef CONFIG_80211BE_EHT
+	if (candidate->network.is_mld)
+		adapter_link->mlmepriv.link_id = candidate->network.link_id;
+#endif
+
+	ret = rtw_joinbss_cmd(adapter, candidate);
+
+exit:
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_mld_queue.lock));
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+
+	return ret;
+}
+
+sint rtw_set_auth(_adapter *adapter, struct security_priv *psecuritypriv)
+{
+	struct cmd_obj *pcmd;
+	struct setauth_parm *psetauthparm;
+	struct cmd_priv *pcmdpriv = &(adapter_to_dvobj(adapter)->cmdpriv);
+	sint res = _SUCCESS;
+
+
+	pcmd = (struct	cmd_obj *)rtw_zmalloc(sizeof(struct	cmd_obj));
+	if (pcmd == NULL) {
+		res = _FAIL; /* try again */
+		goto exit;
+	}
+	pcmd->padapter = adapter;
+
+	psetauthparm = (struct setauth_parm *)rtw_zmalloc(sizeof(struct setauth_parm));
+	if (psetauthparm == NULL) {
+		rtw_mfree((unsigned char *)pcmd, sizeof(struct	cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+
+	psetauthparm->mode = (unsigned char)psecuritypriv->dot11AuthAlgrthm;
+
+	init_h2fwcmd_w_parm_no_rsp(pcmd, psetauthparm, CMD_SET_AUTH);
+
+	res = rtw_enqueue_cmd(pcmdpriv, pcmd);
+exit:
+	return res;
+}
+
+sint rtw_set_key(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, sint keyid, u8 set_tx, bool enqueue)
+{
+	u8 keylen;
+	struct setkey_parm setkeyparm;
+	sint res = _SUCCESS;
+	struct security_priv *psecuritypriv = &adapter->securitypriv;
+	struct link_security_priv *lsecuritypriv = &adapter_link->securitypriv;
+
+	_rtw_memset(&setkeyparm, 0, sizeof(struct setkey_parm));
+
+	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) {
+		/* BIP use keyid 4 or 5 */
+		if (keyid >= 4)
+			setkeyparm.algorithm = (unsigned char)psecuritypriv->dot11wCipher;
+		else
+			setkeyparm.algorithm = (unsigned char)psecuritypriv->dot118021XGrpPrivacy;
+	} else {
+		setkeyparm.algorithm = (u8)psecuritypriv->dot11PrivacyAlgrthm;
+
+	}
+	setkeyparm.keyid = (u8)keyid;/* 0~3 */
+	setkeyparm.set_tx = set_tx;
+	if (is_wep_enc(setkeyparm.algorithm))
+		adapter->securitypriv.key_mask |= BIT(setkeyparm.keyid);
+
+	RTW_INFO("==> rtw_set_key algorithm(%x),keyid(%x),key_mask(%x)\n", setkeyparm.algorithm, setkeyparm.keyid, adapter->securitypriv.key_mask);
+
+	switch (setkeyparm.algorithm) {
+
+	case _WEP40_:
+		keylen = 5;
+		_rtw_memcpy(&(setkeyparm.key[0]), &(psecuritypriv->dot11DefKey[keyid].skey[0]), keylen);
+		break;
+	case _WEP104_:
+		keylen = 13;
+		_rtw_memcpy(&(setkeyparm.key[0]), &(psecuritypriv->dot11DefKey[keyid].skey[0]), keylen);
+		break;
+	case _TKIP_:
+		keylen = 16;
+		_rtw_memcpy(&setkeyparm.key, &lsecuritypriv->dot118021XGrpKey[keyid], keylen);
+		break;
+	case _AES_:
+	case _GCMP_:
+		keylen = 16;
+		_rtw_memcpy(&setkeyparm.key, &lsecuritypriv->dot118021XGrpKey[keyid], keylen);
+		break;
+	case _GCMP_256_:
+	case _CCMP_256_:
+		keylen = 32;
+		_rtw_memcpy(&setkeyparm.key, &lsecuritypriv->dot118021XGrpKey[keyid], keylen);
+		break;
+	#ifdef CONFIG_IEEE80211W
+	case _BIP_CMAC_128_:
+		keylen = 16;
+		_rtw_memcpy(&setkeyparm.key, &lsecuritypriv->dot11wBIPKey[keyid], keylen);
+		break;
+	#endif /* CONFIG_IEEE80211W */
+	default:
+		res = _FAIL;
+		goto exit;
+	}
+
+
+	if (enqueue)
+		setkey_hdl(adapter, adapter_link,
+			&setkeyparm, PHL_CMD_NO_WAIT, 0);
+	else
+		setkey_hdl(adapter, adapter_link,
+			&setkeyparm, PHL_CMD_DIRECTLY, 0);
+exit:
+	return res;
+
+}
+
+/* adjust IEs for rtw_joinbss_cmd in WMM */
+int rtw_restruct_wmm_ie(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+					u8 *in_ie, u8 *out_ie, uint in_len,
+					uint initial_out_len)
+{
+	unsigned	int ielength = 0;
+	unsigned int i, j;
+	u8 qos_info = 0;
+
+	i = 12; /* after the fixed IE */
+	while (i < in_len) {
+		ielength = initial_out_len;
+
+		if (in_ie[i] == 0xDD && in_ie[i + 2] == 0x00 && in_ie[i + 3] == 0x50  && in_ie[i + 4] == 0xF2 && in_ie[i + 5] == 0x02 && i + 5 < in_len) { /* WMM element ID and OUI */
+
+			/* Append WMM IE to the last index of out_ie */
+#if 0
+			for (j = i; j < i + (in_ie[i + 1] + 2); j++) {
+				out_ie[ielength] = in_ie[j];
+				ielength++;
+			}
+			out_ie[initial_out_len + 8] = 0x00; /* force the QoS Info Field to be zero */
+#endif
+
+			for (j = i; j < i + 9; j++) {
+				out_ie[ielength] = in_ie[j];
+				ielength++;
+			}
+			out_ie[initial_out_len + 1] = 0x07;
+			out_ie[initial_out_len + 6] = 0x00;
+			out_ie[initial_out_len + 8] = qos_info;
+
+			break;
+		}
+
+		i += (in_ie[i + 1] + 2); /* to the next IE element */
+	}
+
+	return ielength;
+
+}
+
+
+/*
+ * Ported from 8185: IsInPreAuthKeyList(). (Renamed from SecIsInPreAuthKeyList(), 2006-10-13.)
+ * Added by Annie, 2006-05-07.
+ *
+ * Search by BSSID,
+ * Return Value:
+ *		-1		:if there is no pre-auth key in the  table
+ *		>=0		:if there is pre-auth key, and   return the entry id
+ *
+ *   */
+
+static int SecIsInPMKIDList(_adapter *adapter, u8 *bssid)
+{
+	struct security_priv *psecuritypriv = &adapter->securitypriv;
+	int i = 0;
+
+	do {
+		if ((psecuritypriv->PMKIDList[i].bUsed) &&
+		    (_rtw_memcmp(psecuritypriv->PMKIDList[i].Bssid, bssid, ETH_ALEN) == _TRUE))
+			break;
+		else {
+			i++;
+			/* continue; */
+		}
+
+	} while (i < NUM_PMKID_CACHE);
+
+	if (i == NUM_PMKID_CACHE) {
+		i = -1;/* Could not find. */
+	} else {
+		/* There is one Pre-Authentication Key for the specific BSSID. */
+	}
+
+	return i;
+
+}
+
+int rtw_cached_pmkid(_adapter *adapter, u8 *bssid)
+{
+	return SecIsInPMKIDList(adapter, bssid);
+}
+
+int rtw_pmkid_sync_rsn(_adapter *adapter, u8 *ie, uint ie_len, int i_ent)
+{
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct security_priv *sec = &adapter->securitypriv;
+	struct rsne_info info;
+	int i;
+
+	rtw_rsne_info_parse(ie, ie_len, &info);
+
+	if (info.err) {
+		RTW_WARN(FUNC_ADPT_FMT" rtw_rsne_info_parse error\n"
+			, FUNC_ADPT_ARG(adapter));
+		return 0;
+	}
+
+	if (i_ent < 0 && info.pmkid_cnt == 0)
+		goto exit;
+
+	if (i_ent >= 0 && info.pmkid_cnt == 1 && _rtw_memcmp(info.pmkid_list, sec->PMKIDList[i_ent].PMKID, 16)) {
+		RTW_INFO(FUNC_ADPT_FMT" has carried the same PMKID:"KEY_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), KEY_ARG(&sec->PMKIDList[i_ent].PMKID));
+		goto exit;
+	}
+
+	if (info.pmkid_cnt && pmlmepriv->assoc_by_bssid) {
+		RTW_INFO(FUNC_ADPT_FMT " update PMKID list, count:%u\n", FUNC_ADPT_ARG(adapter), info.pmkid_cnt);
+		for (i = 0; i < info.pmkid_cnt; i++)
+			RTW_INFO("    " KEY_FMT "\n", KEY_ARG(info.pmkid_list + i * 16));
+
+		/* add to pmkid catch */
+		_rtw_memcpy(sec->PMKIDList[sec->PMKIDIndex].Bssid, pmlmepriv->assoc_bssid, ETH_ALEN);
+		_rtw_memcpy(sec->PMKIDList[sec->PMKIDIndex].PMKID, info.pmkid_list, WLAN_PMKID_LEN);
+
+		sec->PMKIDList[sec->PMKIDIndex].bUsed = _TRUE;
+		sec->PMKIDIndex++;
+		if (sec->PMKIDIndex == 16)
+			sec->PMKIDIndex = 0;
+	} else if (i_ent >= 0) {
+		RTW_INFO(FUNC_ADPT_FMT " remove PMKID list", FUNC_ADPT_ARG(adapter));
+		/* remove pmkid catch in list */
+		_rtw_memset(sec->PMKIDList[i_ent].Bssid, 0x00, ETH_ALEN);
+		_rtw_memset(sec->PMKIDList[i_ent].PMKID, 0x00, WLAN_PMKID_LEN);
+		sec->PMKIDList[i_ent].bUsed = _FALSE;
+	}
+
+exit:
+	return ie_len;
+}
+
+sint rtw_restruct_sec_ie(_adapter *adapter, u8 *out_ie)
+{
+	u8 authmode = 0x0;
+	uint	ielength = 0;
+	int iEntry;
+
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct security_priv *psecuritypriv = &adapter->securitypriv;
+	uint	ndisauthmode = psecuritypriv->ndisauthtype;
+
+	if ((ndisauthmode == Ndis802_11AuthModeWPA) || (ndisauthmode == Ndis802_11AuthModeWPAPSK))
+		authmode = _WPA_IE_ID_;
+	if ((ndisauthmode == Ndis802_11AuthModeWPA2) || (ndisauthmode == Ndis802_11AuthModeWPA2PSK))
+		authmode = _WPA2_IE_ID_;
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+		_rtw_memcpy(out_ie, psecuritypriv->wps_ie, psecuritypriv->wps_ie_len);
+		ielength = psecuritypriv->wps_ie_len;
+
+	} else if ((authmode == _WPA_IE_ID_) || (authmode == _WPA2_IE_ID_)) {
+		/* copy RSN or SSN		 */
+		_rtw_memcpy(out_ie, psecuritypriv->supplicant_ie, psecuritypriv->supplicant_ie[1] + 2);
+		/* debug for CONFIG_IEEE80211W
+		{
+			int jj;
+			printk("supplicant_ie_length=%d &&&&&&&&&&&&&&&&&&&\n", psecuritypriv->supplicant_ie[1]+2);
+			for(jj=0; jj < psecuritypriv->supplicant_ie[1]+2; jj++)
+				printk(" %02x ", psecuritypriv->supplicant_ie[jj]);
+			printk("\n");
+		}*/
+		ielength = psecuritypriv->supplicant_ie[1] + 2;
+		rtw_report_sec_ie(adapter, authmode, psecuritypriv->supplicant_ie);
+	}
+
+	if (authmode == WLAN_EID_RSN) {
+		iEntry = SecIsInPMKIDList(adapter, pmlmepriv->assoc_bssid);
+		ielength = rtw_pmkid_sync_rsn(adapter, out_ie, ielength, iEntry);
+	}
+
+	if ((psecuritypriv->rsnx_ie_len >= 3)) {
+		u8 *_pos = out_ie + \
+			(psecuritypriv->supplicant_ie[1] + 2);
+		_rtw_memcpy(_pos, psecuritypriv->rsnx_ie,
+			psecuritypriv->rsnx_ie_len);
+		ielength += psecuritypriv->rsnx_ie_len;
+		RTW_INFO_DUMP("update IE for RSNX :",
+			out_ie, ielength);
+	}
+
+
+	return ielength;
+}
+
+void rtw_init_registrypriv_dev_network(_adapter *adapter)
+{
+	struct registry_priv *pregistrypriv = &adapter->registrypriv;
+	WLAN_BSSID_EX    *pdev_network = &pregistrypriv->dev_network;
+	u8 *myhwaddr = adapter_mac_addr(adapter);
+
+
+	_rtw_memcpy(pdev_network->MacAddress, myhwaddr, ETH_ALEN);
+
+	_rtw_memcpy(&pdev_network->Ssid, &pregistrypriv->ssid, sizeof(NDIS_802_11_SSID));
+
+	pdev_network->Configuration.Length = sizeof(NDIS_802_11_CONFIGURATION);
+	pdev_network->Configuration.BeaconPeriod = 100;
+}
+
+void rtw_update_registrypriv_dev_network(_adapter *adapter)
+{
+	int sz = 0;
+	struct registry_priv *pregistrypriv = &adapter->registrypriv;
+	WLAN_BSSID_EX    *pdev_network = &pregistrypriv->dev_network;
+	struct	security_priv	*psecuritypriv = &adapter->securitypriv;
+	struct	wlan_network	*cur_network = &adapter->mlmepriv.dev_cur_network;
+	/* struct	xmit_priv	*pxmitpriv = &adapter->xmitpriv; */
+	struct _ADAPTER_LINK	*adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_ext_priv	*pmlmeext = NULL;
+
+	if (adapter_link)
+		 pmlmeext = &adapter_link->mlmeextpriv;
+#if 0
+	pxmitpriv->vcs_setting = pregistrypriv->vrtl_carrier_sense;
+	pxmitpriv->vcs = pregistrypriv->vcs_type;
+	pxmitpriv->vcs_type = pregistrypriv->vcs_type;
+	/* pxmitpriv->rts_thresh = pregistrypriv->rts_thresh; */
+	pxmitpriv->frag_len = pregistrypriv->frag_thresh;
+
+	adapter->qospriv.qos_option = pregistrypriv->wmm_enable;
+#endif
+
+	pdev_network->Privacy = (psecuritypriv->dot11PrivacyAlgrthm > 0 ? 1 : 0) ; /* adhoc no 802.1x */
+
+	pdev_network->PhyInfo.rssi = 0;
+
+	pdev_network->Configuration.DSConfig = (pregistrypriv->channel);
+
+	if (cur_network->network.InfrastructureMode == Ndis802_11IBSS) {
+		pdev_network->Configuration.ATIMWindow = (0);
+
+		if (pmlmeext && pmlmeext->chandef.chan != 0)
+			pdev_network->Configuration.DSConfig = pmlmeext->chandef.chan;
+		else
+			pdev_network->Configuration.DSConfig = 1;
+	}
+
+	pdev_network->InfrastructureMode = (cur_network->network.InfrastructureMode);
+
+	/* 1. Supported rates */
+	/* 2. IE */
+
+	/* rtw_set_supported_rate(pdev_network->SupportedRates, pregistrypriv->wireless_mode) ; */ /* will be called in rtw_generate_ie */
+	sz = rtw_generate_ie(pregistrypriv);
+
+	pdev_network->IELength = sz;
+
+	pdev_network->Length = get_WLAN_BSSID_EX_sz((WLAN_BSSID_EX *)pdev_network);
+
+	/* notes: translate IELength & Length after assign the Length to cmdsz in createbss_cmd(); */
+	/* pdev_network->IELength = cpu_to_le32(sz); */
+
+
+}
+
+void rtw_get_encrypt_decrypt_from_registrypriv(_adapter *adapter)
+{
+
+
+
+}
+
+/* the fucntion is at passive_level */
+void rtw_joinbss_reset(_adapter *padapter)
+{
+	u8	threshold;
+	struct _ADAPTER_LINK *padapter_link;
+	struct link_mlme_priv *pmlmepriv;
+	struct ht_priv		*phtpriv;
+	struct ampdu_priv	*ampdu_priv;
+	u8 lidx;
+	/* todo: if you want to do something io/reg/hw setting before join_bss, please add code here */
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		pmlmepriv = &padapter_link->mlmepriv;
+#ifdef CONFIG_80211N_HT
+		phtpriv = &pmlmepriv->htpriv;
+		ampdu_priv = &pmlmepriv->ampdu_priv;
+
+		pmlmepriv->num_FortyMHzIntolerant = 0;
+
+		pmlmepriv->num_sta_no_ht = 0;
+		pmlmepriv->sw_to_20mhz = _FALSE;
+
+		ampdu_priv->ampdu_enable = _FALSE;/* reset to disabled */
+
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI)
+		/* TH=1 => means that invalidate usb rx aggregation */
+		/* TH=0 => means that validate usb rx aggregation, use init value. */
+		if (phtpriv->ht_option) {
+			if (padapter->registrypriv.wifi_spec == 1)
+				threshold = 1;
+			else
+				threshold = 0;
+			rtw_hal_set_hwreg(padapter, HW_VAR_RXDMA_AGG_PG_TH, (u8 *)(&threshold));
+		} else {
+			threshold = 1;
+			rtw_hal_set_hwreg(padapter, HW_VAR_RXDMA_AGG_PG_TH, (u8 *)(&threshold));
+		}
+#endif/* #if defined( CONFIG_USB_HCI) || defined (CONFIG_SDIO_HCI) */
+
+#endif/* #ifdef CONFIG_80211N_HT */
+	}
+
+}
+
+
+#ifdef CONFIG_80211N_HT
+void rtw_ht_get_dft_setting(_adapter *padapter,
+				struct protocol_cap_t *dft_proto_cap,
+				struct role_link_cap_t *dft_cap)
+{
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct ht_priv		*phtpriv = &pmlmepriv->dev_htpriv;
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	BOOLEAN		bHwLDPCSupport = _FALSE, bHwSTBCSupport = _FALSE;
+	u8		stbc_rx = 0;
+#ifdef CONFIG_BEAMFORMING
+	BOOLEAN		bHwSupportBeamformer = _FALSE, bHwSupportBeamformee = _FALSE;
+#endif /* CONFIG_BEAMFORMING */
+
+	if (pregistrypriv->wifi_spec)
+		phtpriv->bss_coexist = 1;
+	else
+		phtpriv->bss_coexist = 0;
+
+	/*dft_proto_cap->sgi_40;*/
+	phtpriv->sgi_40m = TEST_FLAG(pregistrypriv->short_gi, BIT1) ? _TRUE : _FALSE;
+	/*dft_proto_cap->sgi_20;*/
+	phtpriv->sgi_20m = TEST_FLAG(pregistrypriv->short_gi, BIT0) ? _TRUE : _FALSE;
+
+	/* LDPC support */
+	CLEAR_FLAGS(phtpriv->ldpc_cap);
+	bHwLDPCSupport = (dft_proto_cap->ht_ldpc) ? _TRUE : _FALSE;
+	if (bHwLDPCSupport) {
+		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT4))
+			SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_RX);
+	}
+
+	bHwLDPCSupport = (dft_cap->tx_ht_ldpc) ? _TRUE : _FALSE;
+	if (bHwLDPCSupport) {
+		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT5))
+			SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_TX);
+	}
+	if (phtpriv->ldpc_cap)
+		RTW_INFO("[HT] HAL Support LDPC = 0x%02X\n", phtpriv->ldpc_cap);
+
+	/* STBC */
+	CLEAR_FLAGS(phtpriv->stbc_cap);
+	if (dft_proto_cap->stbc_ht_tx)
+		SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX);
+
+	if (dft_proto_cap->stbc_ht_rx)
+		SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX);
+
+	if (phtpriv->stbc_cap)
+		RTW_INFO("[HT] HAL Support STBC = 0x%02X\n", phtpriv->stbc_cap);
+
+	/* Beamforming setting */
+	CLEAR_FLAGS(phtpriv->beamform_cap);
+#ifdef CONFIG_BEAMFORMING
+	/* only enable beamforming in STA client mode */
+	if (MLME_IS_STA(padapter) && !MLME_IS_GC(padapter)
+				  && !MLME_IS_ADHOC(padapter)
+				  && !MLME_IS_MESH(padapter))
+	{
+		bHwSupportBeamformer = (dft_proto_cap->ht_su_bfmr) ? _TRUE : _FALSE;
+		bHwSupportBeamformee = (dft_proto_cap->ht_su_bfme) ? _TRUE : _FALSE;
+
+		if (TEST_FLAG(pregistrypriv->beamform_cap, BIT4) && bHwSupportBeamformer) {
+			SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
+			RTW_INFO("[HT] HAL Support Beamformer\n");
+		}
+		if (TEST_FLAG(pregistrypriv->beamform_cap, BIT5) && bHwSupportBeamformee) {
+			SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
+			RTW_INFO("[HT] HAL Support Beamformee\n");
+		}
+	}
+#endif /* CONFIG_BEAMFORMING */
+}
+
+void	rtw_ht_use_default_setting(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	BOOLEAN		bHwLDPCSupport = _FALSE, bHwSTBCSupport = _FALSE;
+	u8		stbc_rx = 0;
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+#ifdef CONFIG_BEAMFORMING
+	BOOLEAN		bHwSupportBeamformer = _FALSE, bHwSupportBeamformee = _FALSE;
+#endif /* CONFIG_BEAMFORMING */
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct protocol_cap_t *protocol_cap = &(padapter_link->wrlink->protocol_cap);
+	struct role_link_cap_t *cap = &(padapter_link->wrlink->cap);
+	struct ht_priv		*phtpriv = &pmlmepriv->htpriv;
+
+	if (pregistrypriv->wifi_spec)
+		phtpriv->bss_coexist = 1;
+	else
+		phtpriv->bss_coexist = 0;
+
+	phtpriv->sgi_40m = TEST_FLAG(pregistrypriv->short_gi, BIT1) ? _TRUE : _FALSE;
+	phtpriv->sgi_20m = TEST_FLAG(pregistrypriv->short_gi, BIT0) ? _TRUE : _FALSE;
+
+	/* LDPC support */
+	if (protocol_cap->ht_ldpc)
+		SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_RX);
+
+	if (cap->tx_ht_ldpc)
+		SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_TX);
+
+	if (phtpriv->ldpc_cap)
+		RTW_DBG("[HT] HAL Support LDPC = 0x%02X\n", phtpriv->ldpc_cap);
+
+	/* STBC */
+	if (protocol_cap->stbc_ht_tx)
+		SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX);
+
+	if (protocol_cap->stbc_ht_rx)
+		SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX);
+
+	if (phtpriv->stbc_cap)
+		RTW_DBG("[HT] HAL Support STBC = 0x%02X\n", phtpriv->stbc_cap);
+
+	/* Beamforming setting */
+	CLEAR_FLAGS(phtpriv->beamform_cap);
+#ifdef CONFIG_BEAMFORMING
+	/* only enable beamforming in STA client mode */
+	if (MLME_IS_STA(padapter) && !MLME_IS_GC(padapter)
+				  && !MLME_IS_ADHOC(padapter)
+				  && !MLME_IS_MESH(padapter))
+	{
+		if (protocol_cap->ht_su_bfmr) {
+			SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
+			RTW_DBG("[HT] HAL Support Beamformer\n");
+		}
+
+		if (protocol_cap->ht_su_bfme) {
+			SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
+			RTW_DBG("[HT] HAL Support Beamformee\n");
+		}
+	}
+#endif /* CONFIG_BEAMFORMING */
+}
+void rtw_build_wmm_ie_ht(_adapter *padapter, u8 *out_ie, uint *pout_len)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	unsigned char WMM_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01, 0x00};
+	int out_len;
+	u8 *pframe;
+
+	if (padapter_link->mlmepriv.qospriv.qos_option == 0) {
+		out_len = *pout_len;
+		pframe = rtw_set_ie(out_ie + out_len, _VENDOR_SPECIFIC_IE_,
+				    _WMM_IE_Length_, WMM_IE, pout_len);
+
+		padapter_link->mlmepriv.qospriv.qos_option = 1;
+	}
+}
+#if defined(CONFIG_80211N_HT)
+/* the fucntion is >= passive_level */
+unsigned int rtw_restructure_ht_ie(_adapter *padapter,
+						struct _ADAPTER_LINK *padapter_link,
+						u8 *in_ie, u8 *out_ie, uint in_len,
+						uint *pout_len, u8 channel)
+{
+	u32 ielen, out_len;
+	HT_CAP_AMPDU_FACTOR max_rx_ampdu_factor = 0;
+	HT_CAP_AMPDU_DENSITY best_ampdu_density = 0;
+	unsigned char *p, *pframe;
+	enum band_type band = channel > 14 ? BAND_ON_5G : BAND_ON_24G;
+	struct rtw_ieee80211_ht_cap ht_capie;
+	u8	cbw40_enable = 0, rf_num = 0, rx_stbc_nss = 0, rx_nss = 0;
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct mlme_ext_info *pmlmeinfo = &(padapter->mlmeextpriv.mlmext_info);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct protocol_cap_t *protocol_cap = &(padapter_link->wrlink->protocol_cap);
+	struct ht_priv		*phtpriv = &pmlmepriv->htpriv;
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv	*pvhtpriv = &pmlmepriv->vhtpriv;
+#endif /* CONFIG_80211AC_VHT */
+
+	phtpriv->ht_option = _FALSE;
+
+	out_len = *pout_len;
+
+	_rtw_memset(&ht_capie, 0, sizeof(struct rtw_ieee80211_ht_cap));
+
+	ht_capie.cap_info = IEEE80211_HT_CAP_DSSSCCK40;
+
+	if (phtpriv->sgi_20m)
+		ht_capie.cap_info |= IEEE80211_HT_CAP_SGI_20;
+
+	/* check if 40MHz is allowed according to hal cap and registry */
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_40M)) {
+		if (band == BAND_ON_5G) {
+			if (REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+				cbw40_enable = 1;
+		} else if (band == BAND_ON_24G) {
+			if (REGSTY_IS_BW_2G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+				cbw40_enable = 1;
+		}
+	}
+
+	if (cbw40_enable) {
+		u8 oper_bw = CHANNEL_WIDTH_20, oper_offset = CHAN_OFFSET_NO_EXT;
+
+		if (in_ie == NULL) {
+			/* TDLS: TODO 20/40 issue */
+			if (MLME_IS_STA(padapter)) {
+				oper_bw = pmlmeext->chandef.bw;
+				if (oper_bw > CHANNEL_WIDTH_40)
+					oper_bw = CHANNEL_WIDTH_40;
+			} else
+				/* TDLS: TODO 40? */
+				oper_bw = CHANNEL_WIDTH_40;
+		} else {
+			p = rtw_get_ie(in_ie, WLAN_EID_HT_OPERATION, &ielen, in_len);
+			if (p && ielen == HT_OP_IE_LEN) {
+				if (GET_HT_OP_ELE_STA_CHL_WIDTH(p + 2)) {
+					switch (GET_HT_OP_ELE_2ND_CHL_OFFSET(p + 2)) {
+					case IEEE80211_SCA:
+						oper_bw = CHANNEL_WIDTH_40;
+						oper_offset = CHAN_OFFSET_UPPER;
+						break;
+					case IEEE80211_SCB:
+						oper_bw = CHANNEL_WIDTH_40;
+						oper_offset = CHAN_OFFSET_LOWER;
+						break;
+					}
+				}
+			}
+			/* IOT issue : AP TP-Link WDR6500 */
+			if(oper_bw == CHANNEL_WIDTH_40){
+				p = rtw_get_ie(in_ie, WLAN_EID_HT_CAP, &ielen, in_len);
+				if (p && ielen == HT_CAP_IE_LEN) {
+					oper_bw = GET_HT_CAP_ELE_CHL_WIDTH(p + 2)  ? CHANNEL_WIDTH_40 : CHANNEL_WIDTH_20;
+					if(oper_bw == CHANNEL_WIDTH_20)
+						oper_offset = CHAN_OFFSET_NO_EXT;
+				}
+			}
+		}
+
+		if (oper_bw == CHANNEL_WIDTH_40
+			&& oper_offset != CHAN_OFFSET_NO_EXT /* check this because TDLS has no info to set offset */
+		) {
+			/* adjust bw to fit in channel plan setting */
+			oper_bw = alink_adjust_linking_bw_by_regd(padapter_link, band, channel, oper_bw, oper_offset);
+			if (oper_bw == CHANNEL_WIDTH_20)
+				oper_offset = CHAN_OFFSET_NO_EXT;
+		}
+
+		if (oper_bw == CHANNEL_WIDTH_40) {
+			ht_capie.cap_info |= IEEE80211_HT_CAP_SUP_WIDTH;
+			if (phtpriv->sgi_40m)
+				ht_capie.cap_info |= IEEE80211_HT_CAP_SGI_40;
+		}
+
+		cbw40_enable = oper_bw == CHANNEL_WIDTH_40 ? 1 : 0;
+	}
+
+	/* todo: disable SM power save mode */
+	ht_capie.cap_info |= IEEE80211_HT_CAP_SM_PS;
+
+	/* RX LDPC */
+	if (TEST_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_RX)) {
+		ht_capie.cap_info |= IEEE80211_HT_CAP_LDPC_CODING;
+		RTW_DBG("[HT] Declare supporting RX LDPC\n");
+	}
+
+	/* TX STBC */
+	if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX)) {
+		ht_capie.cap_info |= IEEE80211_HT_CAP_TX_STBC;
+		RTW_DBG("[HT] Declare supporting TX STBC\n");
+	}
+
+	/* RX STBC */
+	if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX) ||
+			(pregistrypriv->wifi_spec == 1)) {
+		rx_stbc_nss = protocol_cap->stbc_ht_rx;
+		if (rx_stbc_nss > 3)
+			rx_stbc_nss = 3;
+		SET_HT_CAP_ELE_RX_STBC(&ht_capie, rx_stbc_nss);
+		RTW_DBG("[HT] Declare supporting RX STBC = %d\n", rx_stbc_nss);
+	}
+
+	/* fill default supported_mcs_set */
+	_rtw_memcpy(ht_capie.supp_mcs_set, pmlmeext->default_supported_mcs_set, 16);
+
+	/* update default supported_mcs_set */
+	rx_nss = get_phy_rx_nss(padapter, padapter_link);
+
+	switch (rx_nss) {
+	case 1:
+		set_mcs_rate_by_mask(ht_capie.supp_mcs_set, MCS_RATE_1R);
+		break;
+	case 2:
+		#ifdef CONFIG_DISABLE_MCS13TO15
+		if (cbw40_enable && pregistrypriv->wifi_spec != 1)
+			set_mcs_rate_by_mask(ht_capie.supp_mcs_set, MCS_RATE_2R_13TO15_OFF);
+		else
+		#endif
+			set_mcs_rate_by_mask(ht_capie.supp_mcs_set, MCS_RATE_2R);
+		break;
+	case 3:
+		set_mcs_rate_by_mask(ht_capie.supp_mcs_set, MCS_RATE_3R);
+		break;
+	case 4:
+		set_mcs_rate_by_mask(ht_capie.supp_mcs_set, MCS_RATE_4R);
+		break;
+	default:
+		RTW_DBG("rf_type:%d or rx_nss:%u is not expected\n",
+		GET_HAL_RFPATH(adapter_to_dvobj(padapter)), rx_nss);
+	}
+
+	if (protocol_cap->max_amsdu_len > 0) {
+		/*SET_HT_CAP_ELE_MAX_AMSDU_LENGTH(&ht_capie, 1);*/
+		RTW_DBG("%s IEEE80211_HT_CAP_MAX_AMSDU is set\n", __func__);
+		ht_capie.cap_info = ht_capie.cap_info | IEEE80211_HT_CAP_MAX_AMSDU;
+	}
+
+	if (padapter->driver_rx_ampdu_factor != 0xFF)
+		max_rx_ampdu_factor = (HT_CAP_AMPDU_FACTOR)padapter->driver_rx_ampdu_factor;
+	else
+		rtw_hal_get_def_var(padapter, padapter_link, HW_VAR_MAX_RX_AMPDU_FACTOR, &max_rx_ampdu_factor);
+
+	/* rtw_hal_get_def_var(padapter, HW_VAR_MAX_RX_AMPDU_FACTOR, &max_rx_ampdu_factor); */
+	ht_capie.ampdu_params_info = (max_rx_ampdu_factor & 0x03);
+
+	if (padapter->driver_rx_ampdu_spacing != 0xFF)
+		ht_capie.ampdu_params_info |= ((padapter->driver_rx_ampdu_spacing & 0x07) << 2);
+	else {
+		if (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_) {
+			/*
+			*	Todo : Each chip must to ask DD , this chip best ampdu_density setting
+			*	By yiwei.sun
+			*/
+			rtw_hal_get_def_var(padapter, padapter_link, HW_VAR_BEST_AMPDU_DENSITY, &best_ampdu_density);
+
+			ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & (best_ampdu_density << 2));
+
+		} else
+			ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & 0x00);
+	}
+#ifdef CONFIG_BEAMFORMING
+	ht_capie.tx_BF_cap_info = 0;
+
+	/* HT Beamformer*/
+	if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE)) {
+		/* Transmit NDP Capable */
+		SET_HT_CAP_TXBF_TRANSMIT_NDP_CAP(&ht_capie, 1);
+		/* Explicit Compressed Steering Capable */
+		SET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP(&ht_capie, 1);
+		/* Compressed Steering Number Antennas */
+		SET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(&ht_capie, 1);
+		rtw_hal_get_def_var(padapter, padapter_link, HAL_DEF_BEAMFORMER_CAP, (u8 *)&rf_num);
+		if (rf_num > 3)
+			rf_num = 3;
+		SET_HT_CAP_TXBF_CHNL_ESTIMATION_NUM_ANTENNAS(&ht_capie, rf_num);
+	}
+
+	/* HT Beamformee */
+	if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE)) {
+		/* Receive NDP Capable */
+		SET_HT_CAP_TXBF_RECEIVE_NDP_CAP(&ht_capie, 1);
+		/* Explicit Compressed Beamforming Feedback Capable */
+		SET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP(&ht_capie, 2);
+
+		rtw_hal_get_def_var(padapter, padapter_link, HAL_DEF_BEAMFORMEE_CAP, (u8 *)&rf_num);
+		if (rf_num > 3)
+			rf_num = 3;
+#ifdef CONFIG_80211AC_VHT
+		/* IOT action suggested by Yu Chen 2017/3/3 */
+		if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_BROADCOM) &&
+			!pvhtpriv->ap_bf_cap.is_mu_bfer &&
+			pvhtpriv->ap_bf_cap.su_sound_dim == 2)
+			rf_num = (rf_num >= 2 ? 2 : rf_num);
+#endif
+		SET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(&ht_capie, rf_num);
+	}
+#endif/*CONFIG_BEAMFORMING*/
+
+	pframe = rtw_set_ie(out_ie + out_len, _HT_CAPABILITY_IE_,
+		sizeof(struct rtw_ieee80211_ht_cap), (unsigned char *)&ht_capie, pout_len);
+
+	phtpriv->ht_option = _TRUE;
+
+	if (in_ie != NULL) {
+		p = rtw_get_ie(in_ie, _HT_ADD_INFO_IE_, &ielen, in_len);
+		if (p && (ielen == sizeof(struct ieee80211_ht_addt_info))) {
+			out_len = *pout_len;
+			pframe = rtw_set_ie(out_ie + out_len, _HT_ADD_INFO_IE_, ielen, p + 2 , pout_len);
+		}
+	}
+
+	return phtpriv->ht_option;
+
+}
+
+/* the fucntion is > passive_level (in critical_section) */
+void rtw_update_link_ht_cap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *pie, uint ie_len, u8 channel)
+{
+	u8 *p, max_ampdu_sz;
+	int len;
+	/* struct sta_info *bmc_sta, *psta; */
+	struct rtw_ieee80211_ht_cap *pht_capie;
+	struct ieee80211_ht_addt_info *pht_addtinfo;
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct ht_priv		*phtpriv = &pmlmepriv->htpriv;
+	struct ampdu_priv	*ampdu_priv = &pmlmepriv->ampdu_priv;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	/* struct wlan_network *pcur_network = &(pmlmepriv->cur_network);; */
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8 cbw40_enable = 0;
+
+
+	if (!phtpriv->ht_option)
+		return;
+
+	if ((!pmlmeinfo->HT_info_enable) || (!pmlmeinfo->HT_caps_enable))
+		return;
+
+	RTW_INFO("+rtw_update_link_ht_cap()\n");
+
+	/* maybe needs check if ap supports rx ampdu. */
+	if ((ampdu_priv->ampdu_enable == _FALSE) && (pregistrypriv->ampdu_enable == 1)) {
+		if (pregistrypriv->wifi_spec == 1) {
+			/* remove this part because testbed AP should disable RX AMPDU */
+			/* ampdu_priv->ampdu_enable = _FALSE; */
+			ampdu_priv->ampdu_enable = _TRUE;
+		} else
+			ampdu_priv->ampdu_enable = _TRUE;
+	}
+
+
+	/* check Max Rx A-MPDU Size */
+	len = 0;
+	p = rtw_get_ie(pie + _FIXED_IE_LENGTH_, _HT_CAPABILITY_IE_, &len, ie_len - _FIXED_IE_LENGTH_);
+	if (p && len > 0) {
+		pht_capie = (struct rtw_ieee80211_ht_cap *)(p + 2);
+		max_ampdu_sz = (pht_capie->ampdu_params_info & IEEE80211_HT_CAP_AMPDU_FACTOR);
+		max_ampdu_sz = 1 << (max_ampdu_sz + 3); /* max_ampdu_sz (kbytes); */
+
+		/* RTW_INFO("rtw_update_ht_cap(): max_ampdu_sz=%d\n", max_ampdu_sz); */
+		ampdu_priv->rx_ampdu_maxlen = max_ampdu_sz;
+
+	}
+
+
+	len = 0;
+	p = rtw_get_ie(pie + _FIXED_IE_LENGTH_, _HT_ADD_INFO_IE_, &len, ie_len - _FIXED_IE_LENGTH_);
+	if (p && len > 0) {
+		pht_addtinfo = (struct ieee80211_ht_addt_info *)(p + 2);
+		/* todo: */
+	}
+
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_40M)) {
+		if (channel > 14) {
+			if (REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+				cbw40_enable = 1;
+		} else {
+			if (REGSTY_IS_BW_2G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+				cbw40_enable = 1;
+		}
+	}
+
+	/* update cur_bwmode & cur_ch_offset */
+	if ((cbw40_enable) &&
+	    (pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info & BIT(1)) &&
+	    (pmlmeinfo->HT_info.infos[0] & BIT(2))) {
+		int i;
+		u8 rx_nss = 0;
+
+		rx_nss = get_phy_rx_nss(padapter, padapter_link);
+
+
+		/* update the MCS set */
+		for (i = 0; i < 16; i++)
+			pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate[i] &= pmlmeext->default_supported_mcs_set[i];
+
+		/* update the MCS rates */
+		switch (rx_nss) {
+		case 1:
+			set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_1R);
+			break;
+		case 2:
+			#ifdef CONFIG_DISABLE_MCS13TO15
+			if (pmlmeext->chandef.bw == CHANNEL_WIDTH_40 && pregistrypriv->wifi_spec != 1)
+				set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_2R_13TO15_OFF);
+			else
+			#endif
+				set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_2R);
+			break;
+		case 3:
+			set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_3R);
+			break;
+		case 4:
+			set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_4R);
+			break;
+		default:
+			RTW_WARN("rx_nss:%u is not expected\n", rx_nss);
+		}
+
+		/* switch to the 40M Hz mode accoring to the AP */
+		/* pmlmeext->cur_bwmode = CHANNEL_WIDTH_40; */
+		/* Secondary Channel Offset */
+		switch ((pmlmeinfo->HT_info.infos[0] & 0x3)) {
+		case IEEE80211_SCA:
+			pmlmeext->chandef.offset = CHAN_OFFSET_UPPER;
+			break;
+
+		case IEEE80211_SCB:
+			pmlmeext->chandef.offset = CHAN_OFFSET_LOWER;
+			break;
+
+		default:
+			pmlmeext->chandef.bw = CHANNEL_WIDTH_20;
+			pmlmeext->chandef.offset = CHAN_OFFSET_NO_EXT;
+			RTW_INFO("%s : ch offset is not assigned for HT40 mod , update cur_bwmode=%u, cur_ch_offset=%u\n",
+					__func__, pmlmeext->chandef.bw, pmlmeext->chandef.offset);
+			break;
+		}
+	}
+
+	/*  */
+	/* Config SM Power Save setting */
+	/*  */
+	pmlmeinfo->SM_PS = (pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info & 0x0C) >> 2;
+	if (pmlmeinfo->SM_PS == SM_PS_STATIC) {
+#if 0
+		u8 i;
+		/* update the MCS rates */
+		for (i = 0; i < 16; i++)
+			pmlmeinfo->HT_caps.HT_cap_element.MCS_rate[i] &= MCS_rate_1R[i];
+#endif
+		RTW_INFO("%s(): SM_PS_STATIC\n", __FUNCTION__);
+	}
+
+	/*  */
+	/* Config current HT Protection mode. */
+	/*  */
+	pmlmeinfo->HT_protection = pmlmeinfo->HT_info.infos[1] & 0x3;
+}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+void rtw_update_link_he_6ghz_cap(_adapter *padapter,
+				 struct _ADAPTER_LINK *padapter_link,
+				 u8 *pie, uint ie_len, enum band_type band,
+				 u32 ch)
+{
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct ampdu_priv	*ampdu_priv = &pmlmepriv->ampdu_priv;
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+
+	if (band != BAND_ON_6G)
+		return;
+
+	/* maybe needs check if ap supports rx ampdu. */
+	if ((ampdu_priv->ampdu_enable == _FALSE) && (pregistrypriv->ampdu_enable == 1)) {
+		if (pregistrypriv->wifi_spec == 1) {
+			/* remove this part because testbed AP should disable RX AMPDU */
+			/* ampdu_priv->ampdu_enable = _FALSE; */
+			ampdu_priv->ampdu_enable = _TRUE;
+		} else
+			ampdu_priv->ampdu_enable = _TRUE;
+	}
+}
+#endif
+
+void rtw_update_join_priv(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			  WLAN_BSSID_EX *network, WLAN_BSSID_EX *psecnetwork)
+{
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct qos_priv		*pqospriv = &pmlmepriv->qospriv;
+#ifdef CONFIG_80211N_HT
+	struct ht_priv			*phtpriv = &pmlmepriv->htpriv;
+#endif /* CONFIG_80211N_HT */
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv		*pvhtpriv = &pmlmepriv->vhtpriv;
+#endif /* CONFIG_80211AC_VHT */
+#ifdef CONFIG_80211AX_HE
+	struct he_priv		*phepriv = &pmlmepriv->hepriv;
+#endif /* CONFIG_80211AX_HE */
+#ifdef CONFIG_80211BE_EHT
+	struct mlo_priv		*pmlopriv = &padapter->mlmepriv.mlopriv;
+#endif
+	u32 tmp_len;
+	u8 *ptmp = NULL;
+
+	_rtw_memset(psecnetwork, 0, sizeof(WLAN_BSSID_EX));
+	_rtw_memcpy(psecnetwork, network, get_WLAN_BSSID_EX_sz(network));
+
+#ifdef CONFIG_80211D
+	rtw_alink_joinbss_update_regulatory(padapter_link, network);
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	/* save AP beamform_cap info for BCM IOT issue */
+	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_BROADCOM)
+		get_vht_bf_cap(network->IEs,
+				network->IELength,
+				&pvhtpriv->ap_bf_cap);
+#endif
+
+	psecnetwork->IELength = 0;
+
+	/* copy fixed ie */
+	_rtw_memcpy(psecnetwork->IEs, network->IEs, 12);
+	psecnetwork->IELength = 12;
+
+	psecnetwork->IELength += rtw_restruct_sec_ie(padapter, psecnetwork->IEs + psecnetwork->IELength);
+
+
+	pqospriv->qos_option = 0;
+
+	if (padapter->registrypriv.wmm_enable) {
+		tmp_len = rtw_restruct_wmm_ie(padapter, padapter_link,
+					&network->IEs[0], &psecnetwork->IEs[0],
+					network->IELength, psecnetwork->IELength);
+
+		if (psecnetwork->IELength != tmp_len) {
+			psecnetwork->IELength = tmp_len;
+			pqospriv->qos_option = 1; /* There is WMM IE in this corresp. beacon */
+		} else {
+			pqospriv->qos_option = 0;/* There is no WMM IE in this corresp. beacon */
+			RTW_INFO("%s follow wifi logo,no WMM IE will be legacy mode \n", __func__);
+		}
+	}
+
+#ifdef RTW_WKARD_UPDATE_PHL_ROLE_CAP
+	rtw_update_phl_cap_by_rgstry(padapter, padapter_link);
+#endif
+
+#ifdef CONFIG_80211N_HT
+	phtpriv->ht_option = _FALSE;
+	if (pregistrypriv->ht_enable && is_supported_ht(pregistrypriv->wireless_mode)) {
+		ptmp = rtw_get_ie(&network->IEs[12], _HT_CAPABILITY_IE_, &tmp_len, network->IELength - 12);
+		if (ptmp && tmp_len > 0 && (pqospriv->qos_option == 1)) {
+			/*	Added by Albert 2010/06/23 */
+			/*	For the WEP mode, we will use the bg mode to do the connection to avoid some IOT issue. */
+			/*	Especially for Realtek 8192u SoftAP. */
+			if ((padapter->securitypriv.dot11PrivacyAlgrthm != _WEP40_) &&
+				(padapter->securitypriv.dot11PrivacyAlgrthm != _WEP104_) &&
+				(padapter->securitypriv.dot11PrivacyAlgrthm != _TKIP_)) {
+				rtw_ht_use_default_setting(padapter, padapter_link);
+
+				/* rtw_restructure_ht_ie */
+				rtw_restructure_ht_ie(padapter, padapter_link,
+					&network->IEs[12], &psecnetwork->IEs[0],
+					network->IELength - 12, &psecnetwork->IELength,
+					network->Configuration.DSConfig);
+			}
+		}
+	}
+
+#ifdef CONFIG_80211AC_VHT
+	pvhtpriv->vht_option = _FALSE;
+	if (phtpriv->ht_option
+		&& REGSTY_IS_11AC_ENABLE(pregistrypriv)
+		&& (pqospriv->qos_option == 1)
+		&& is_supported_vht(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AC(rfctl)
+	) {
+		u8 vht_enable = 0;
+
+		if (network->Configuration.DSConfig > 14)
+			vht_enable = 1;
+		else if ((REGSTY_IS_11AC_24G_ENABLE(pregistrypriv)) && (padapter->registrypriv.wifi_spec == 0))
+			vht_enable = 1;
+
+		if (vht_enable == 1)
+			rtw_restructure_vht_ie(padapter, padapter_link,
+				&network->IEs[0], &psecnetwork->IEs[0],
+				network->IELength, &psecnetwork->IELength,
+				network->Configuration.Band);
+	}
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+	phepriv->he_option = _FALSE;
+	if (((phtpriv->ht_option && network->Configuration.Band == BAND_ON_24G)
+#ifdef CONFIG_80211AC_VHT
+		|| (pvhtpriv->vht_option && network->Configuration.Band == BAND_ON_5G)
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+		|| (network->Configuration.Band == BAND_ON_6G)
+#endif
+		)
+		&& REGSTY_IS_11AX_ENABLE(pregistrypriv)
+		&& (pqospriv->qos_option == 1)
+		&& is_supported_he(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AX(rfctl)
+	) {
+		rtw_restructure_he_ie(padapter, padapter_link,
+			&network->IEs[0], &psecnetwork->IEs[0],
+			network->IELength, &psecnetwork->IELength,
+			network->Configuration.Band);
+	}
+#endif /* CONFIG_80211AX_HE */
+
+#endif /* CONFIG_80211N_HT */
+
+	rtw_append_extended_cap(padapter, padapter_link,
+				&psecnetwork->IEs[0], &psecnetwork->IELength);
+
+	/* ToDo CONFIG_RTW_MLD: Is it required for each adapter_link? */
+#ifdef CONFIG_RTW_80211R
+	rtw_ft_validate_akm_type(padapter, network);
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+	rtw_restructure_ml_ie(padapter, padapter_link,
+		&network->IEs[0], &psecnetwork->IEs[0],
+		network->IELength, &psecnetwork->IELength);
+#endif
+
+	_rtw_memcpy(network, psecnetwork, get_WLAN_BSSID_EX_sz(psecnetwork));
+
+}
+
+#ifdef CONFIG_TDLS
+void rtw_issue_addbareq_cmd_tdls(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+	struct sta_info *ptdls_sta = NULL;
+	u8 issued;
+	int priority;
+	struct ht_priv	*phtpriv;
+	struct ampdu_priv *ampdu_priv;
+
+	priority = pattrib->priority;
+
+	if (pattrib->direct_link == _TRUE) {
+		ptdls_sta = rtw_get_stainfo(&padapter->stapriv, pattrib->dst);
+		if ((ptdls_sta != NULL) && (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE)) {
+			phtpriv = &ptdls_sta->htpriv;
+			ampdu_priv = &ptdls_sta->ampdu_priv;
+			if ((phtpriv->ht_option == _TRUE
+#if CONFIG_IEEE80211_BAND_6GHZ
+			    || ptdls_sta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+			    )&& ampdu_priv->ampdu_enable == _TRUE) {
+				issued = (ampdu_priv->agg_enable_bitmap >> priority) & 0x1;
+				issued |= (ampdu_priv->candidate_tid_bitmap >> priority) & 0x1;
+
+				if (0 == issued) {
+					RTW_INFO("[%s], p=%d\n", __FUNCTION__, priority);
+					ptdls_sta->ampdu_priv.candidate_tid_bitmap |= BIT((u8)priority);
+					rtw_addbareq_cmd(padapter, (u8)priority, pattrib->dst);
+				}
+			}
+		}
+	}
+}
+#endif /* CONFIG_TDLS */
+
+#ifdef CONFIG_80211N_HT
+static u8 rtw_issue_addbareq_check(_adapter *padapter, struct xmit_frame *pxmitframe, u8 issue_when_busy)
+{
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct registry_priv *pregistry = &padapter->registrypriv;
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+	s32 bmcst = IS_MCAST(pattrib->ra);
+
+	if (bmcst)
+		return _FALSE;
+
+	if ((pregpriv->wifi_spec == 1))
+		return _TRUE;
+
+	if (pregistry->tx_quick_addba_req == 0) {
+		if ((issue_when_busy == _TRUE) && (pmlmepriv->LinkDetectInfo.bBusyTraffic == _FALSE))
+			return _FALSE;
+
+		if (pmlmepriv->LinkDetectInfo.NumTxOkInPeriod < 100)
+			return _FALSE;
+	}
+
+	return _TRUE;
+}
+
+void rtw_issue_addbareq_cmd(_adapter *padapter, struct xmit_frame *pxmitframe, u8 issue_when_busy)
+{
+	u8 issued;
+	int priority;
+	struct sta_info *psta = NULL;
+	struct ht_priv	*phtpriv;
+	struct ampdu_priv *ampdu_priv;
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+
+	if (rtw_issue_addbareq_check(padapter, pxmitframe, issue_when_busy) == _FALSE)
+		return;
+
+	priority = pattrib->priority;
+
+#ifdef CONFIG_TDLS
+	rtw_issue_addbareq_cmd_tdls(padapter, pxmitframe);
+#endif /* CONFIG_TDLS */
+
+	psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+	if (pattrib->psta != psta) {
+		RTW_INFO("%s, pattrib->psta(%p) != psta(%p)\n", __func__, pattrib->psta, psta);
+		return;
+	}
+
+	if (psta == NULL) {
+		RTW_INFO("%s, psta==NUL\n", __func__);
+		return;
+	}
+
+	if (!(psta->state & WIFI_ASOC_STATE)) {
+		RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+		return;
+	}
+
+	phtpriv = &psta->htpriv;
+	ampdu_priv = &psta->ampdu_priv;
+	if ((phtpriv->ht_option == _TRUE
+#if CONFIG_IEEE80211_BAND_6GHZ
+	    || psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+	    ) && ampdu_priv->ampdu_enable == _TRUE) {
+		issued = (ampdu_priv->agg_enable_bitmap >> priority) & 0x1;
+		issued |= (ampdu_priv->candidate_tid_bitmap >> priority) & 0x1;
+
+		if (0 == issued) {
+			RTW_INFO("rtw_issue_addbareq_cmd, p=%d\n", priority);
+			psta->ampdu_priv.candidate_tid_bitmap |= BIT((u8)priority);
+			rtw_addbareq_cmd(padapter, (u8) priority, pattrib->ra);
+		}
+	}
+
+}
+#endif /* CONFIG_80211N_HT */
+void rtw_append_extended_cap(_adapter *padapter,
+					struct _ADAPTER_LINK *padapter_link,
+					u8 *out_ie, uint *pout_len)
+{
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct ht_priv		*phtpriv = &pmlmepriv->htpriv;
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv	*pvhtpriv = &pmlmepriv->vhtpriv;
+#endif /* CONFIG_80211AC_VHT */
+	u8 *ext_cap_data = pmlmepriv->ext_capab_ie_data;
+	u8 *ext_cap_data_len = &(pmlmepriv->ext_capab_ie_len);
+
+	if (phtpriv->bss_coexist)
+		rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, BSS_COEXT);
+
+#ifdef CONFIG_80211AC_VHT
+	if (pvhtpriv->vht_option)
+		rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, OP_MODE_NOTIFICATION);
+#endif /* CONFIG_80211AC_VHT */
+#ifdef CONFIG_RTW_WNM
+	if (padapter->dis_rt_cap & BIT(DIS_MBO))
+		rtw_remove_ext_cap_info(ext_cap_data, ext_cap_data_len, BSS_TRANSITION);
+	else
+		rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, BSS_TRANSITION);
+#endif
+
+#ifdef CONFIG_RTW_MBO
+	if (padapter->dis_rt_cap & BIT(DIS_MBO))
+		rtw_remove_ext_cap_info(ext_cap_data, ext_cap_data_len, INTERWORKING);
+	else
+		rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, INTERWORKING);
+#endif
+
+#ifdef CONFIG_ECSA_PHL
+	if (rtw_is_ecsa_enabled(padapter))
+		rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, EXT_CH_SWITCH);
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	if (TWT_REQ_SUPP(padapter_link->wrlink->protocol_cap.twt))
+		rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, TWT_REQ_SUPPORT);
+	if (TWT_RSP_SUPP(padapter_link->wrlink->protocol_cap.twt))
+		rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, TWT_RSP_SUPPORT);
+#endif
+#endif /* CONFIG_80211AX_HE */
+
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+	rtw_add_ext_cap_info(ext_cap_data, ext_cap_data_len, MULTI_BSSID);
+#endif
+
+	/*
+		From 802.11 specification,if a STA does not support any of capabilities defined
+		in the Extended Capabilities element, then the STA is not required to
+		transmit the Extended Capabilities element.
+	*/
+	rtw_update_ext_cap_ie(ext_cap_data, *ext_cap_data_len, out_ie, pout_len, _BEACON_IE_OFFSET_);
+}
+#endif
+
+#ifdef CONFIG_LAYER2_ROAMING
+inline void rtw_set_to_roam(_adapter *adapter, u8 to_roam)
+{
+	if (to_roam == 0)
+		rtw_clear_to_join_status(adapter);
+	adapter->mlmepriv.to_roam = to_roam;
+}
+
+inline u8 rtw_dec_to_roam(_adapter *adapter)
+{
+	adapter->mlmepriv.to_roam--;
+	return adapter->mlmepriv.to_roam;
+}
+
+inline u8 rtw_to_roam(_adapter *adapter)
+{
+	return adapter->mlmepriv.to_roam;
+}
+
+void rtw_roaming(_adapter *padapter, struct wlan_network *tgt_network, u8 reason)
+{
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	_rtw_roaming(padapter, tgt_network, reason);
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+}
+
+#ifdef CONFIG_RTW_FSM_BTM
+void _rtw_roaming(_adapter *padapter, struct wlan_network *tgt_network, u8 roam_reason)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network *cur_network = &pmlmepriv->dev_cur_network;
+
+	if (0 < rtw_to_roam(padapter)) {
+		RTW_INFO("roaming from %s("MAC_FMT"), length:%d\n",
+			cur_network->network.Ssid.Ssid, MAC_ARG(cur_network->network.MacAddress),
+			 cur_network->network.Ssid.SsidLength);
+		_rtw_memcpy(&pmlmepriv->assoc_ssid, &cur_network->network.Ssid, sizeof(NDIS_802_11_SSID));
+		pmlmepriv->assoc_ch = 0;
+		pmlmepriv->assoc_band = BAND_MAX;
+		pmlmepriv->assoc_by_bssid = _FALSE;
+
+#ifdef CONFIG_WAPI_SUPPORT
+		rtw_wapi_return_all_sta_info(padapter);
+#endif
+		rtw_btm_new_obj(padapter, NULL, &pmlmepriv->nb_info, roam_reason);
+
+	}
+}
+
+#else /* !CONFIG_RTW_FSM_BTM */
+
+void _rtw_roaming(_adapter *padapter, struct wlan_network *tgt_network, u8 roam_reason)
+{
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network *cur_network = &pmlmepriv->dev_cur_network;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	int do_join_r;
+
+	if (0 < rtw_to_roam(padapter)) {
+		RTW_INFO("roaming from %s("MAC_FMT"), length:%d\n",
+			cur_network->network.Ssid.Ssid, MAC_ARG(cur_network->network.MacAddress),
+			 cur_network->network.Ssid.SsidLength);
+		_rtw_memcpy(&pmlmepriv->assoc_ssid, &cur_network->network.Ssid, sizeof(NDIS_802_11_SSID));
+		pmlmepriv->assoc_ch = 0;
+		pmlmepriv->assoc_band = BAND_MAX;
+		pmlmepriv->assoc_by_bssid = _FALSE;
+
+#ifdef CONFIG_WAPI_SUPPORT
+		rtw_wapi_return_all_sta_info(padapter);
+#endif
+
+		while (1) {
+			do_join_r = rtw_do_join(padapter);
+			if (_SUCCESS == do_join_r)
+				break;
+			else {
+				RTW_INFO("roaming do_join return %d\n", do_join_r);
+				rtw_dec_to_roam(padapter);
+
+				if (rtw_to_roam(padapter) > 0)
+					continue;
+				else {
+					RTW_INFO("%s(%d) -to roaming fail, indicate_disconnect\n", __FUNCTION__, __LINE__);
+#ifdef CONFIG_RTW_80211R
+					/* rtw_ft_clr_flags(padapter, RTW_FT_PEER_EN|RTW_FT_PEER_OTD_EN); */
+					rtw_ft_reset_status(padapter);
+#endif
+					rtw_indicate_disconnect(padapter, 0, _FALSE);
+					pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+					pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_LAYER2_ROAMING_TERMINATE;
+					pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+					break;
+				}
+			}
+		}
+	}
+
+}
+#endif /* CONFIG_RTW_FSM_BTM */
+#endif /* CONFIG_LAYER2_ROAMING */
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY inline bool rtw_adjust_chbw(_adapter *adapter, u8 req_ch, u8 *req_bw, u8 *req_offset)
+{
+	return rtw_adjust_bchbw(adapter, (req_ch > 14) ? BAND_ON_5G : BAND_ON_24G, req_ch, req_bw, req_offset);
+}
+#endif
+
+bool rtw_adjust_bchbw(_adapter *adapter, enum band_type req_band, u8 req_ch, u8 *req_bw, u8 *req_offset)
+{
+	struct registry_priv *regsty = &adapter->registrypriv;
+	u8 allowed_bw;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (req_band == BAND_ON_6G)
+		allowed_bw = REGSTY_BW_6G(regsty);
+	else
+#endif
+	if (req_band == BAND_ON_5G)
+		allowed_bw = REGSTY_BW_5G(regsty);
+	else if (req_ch == 14)
+		allowed_bw = CHANNEL_WIDTH_20;
+	else
+		allowed_bw = REGSTY_BW_2G(regsty);
+
+	allowed_bw = rtw_hw_largest_bw(adapter_to_dvobj(adapter), allowed_bw);
+
+	if (allowed_bw == CHANNEL_WIDTH_160 && *req_bw > CHANNEL_WIDTH_160)
+		*req_bw = CHANNEL_WIDTH_160;
+	else if (allowed_bw == CHANNEL_WIDTH_80 && *req_bw > CHANNEL_WIDTH_80)
+		*req_bw = CHANNEL_WIDTH_80;
+	else if (allowed_bw == CHANNEL_WIDTH_40 && *req_bw > CHANNEL_WIDTH_40)
+		*req_bw = CHANNEL_WIDTH_40;
+	else if (allowed_bw == CHANNEL_WIDTH_20 && *req_bw > CHANNEL_WIDTH_20) {
+		*req_bw = CHANNEL_WIDTH_20;
+		*req_offset = CHAN_OFFSET_NO_EXT;
+	} else
+		return _FALSE;
+
+	return _TRUE;
+}
+
+sint rtw_linked_check(_adapter *padapter)
+{
+	if (MLME_IS_AP(padapter) || MLME_IS_MESH(padapter)
+		|| MLME_IS_ADHOC(padapter) || MLME_IS_ADHOC_MASTER(padapter)
+	) {
+		if (padapter->stapriv.asoc_sta_count > 1)
+			return _TRUE;
+	} else {
+		/* Station mode */
+		if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _TRUE)
+			return _TRUE;
+	}
+	return _FALSE;
+}
+/*#define DBG_ADAPTER_STATE_CHK*/
+u8 rtw_is_adapter_up(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj;
+
+	if (padapter == NULL)
+		return _FALSE;
+
+	dvobj = adapter_to_dvobj(padapter);
+
+	if (RTW_CANNOT_RUN(dvobj)) {
+		#ifdef DBG_ADAPTER_STATE_CHK
+		RTW_INFO(FUNC_ADPT_FMT " FALSE -bDriverStopped(%s) bSurpriseRemoved(%s)\n"
+			, FUNC_ADPT_ARG(padapter)
+			, dev_is_drv_stopped(dvobj) ? "True" : "False"
+			, dev_is_surprise_removed(dvobj) ? "True" : "False");
+		#endif
+		return _FALSE;
+	}
+
+	if (padapter->netif_up == _FALSE) {
+		#ifdef DBG_ADAPTER_STATE_CHK
+		RTW_INFO(FUNC_ADPT_FMT " FALSE -(netif_up == _FALSE)\n", FUNC_ADPT_ARG(padapter));
+		#endif
+		return _FALSE;
+	}
+
+	if (padapter->phl_role == NULL) {
+		#ifdef DBG_ADAPTER_STATE_CHK
+		RTW_INFO(FUNC_ADPT_FMT " FALSE -(phl_role == NULL)\n", FUNC_ADPT_ARG(padapter));
+		#endif
+		return _FALSE;
+	}
+
+	return _TRUE;
+}
+
+bool is_miracast_enabled(_adapter *adapter)
+{
+	bool enabled = 0;
+#ifdef CONFIG_WFD
+	struct wifi_display_info *wfdinfo = &adapter->wfd_info;
+
+	enabled = (wfdinfo->stack_wfd_mode & (MIRACAST_SOURCE | MIRACAST_SINK))
+		  || (wfdinfo->op_wfd_mode & (MIRACAST_SOURCE | MIRACAST_SINK));
+#endif
+
+	return enabled;
+}
+
+bool rtw_chk_miracast_mode(_adapter *adapter, u8 mode)
+{
+	bool ret = 0;
+#ifdef CONFIG_WFD
+	struct wifi_display_info *wfdinfo = &adapter->wfd_info;
+
+	ret = (wfdinfo->stack_wfd_mode & mode) || (wfdinfo->op_wfd_mode & mode);
+#endif
+
+	return ret;
+}
+
+const char *get_miracast_mode_str(int mode)
+{
+	if (mode == MIRACAST_SOURCE)
+		return "SOURCE";
+	else if (mode == MIRACAST_SINK)
+		return "SINK";
+	else if (mode == (MIRACAST_SOURCE | MIRACAST_SINK))
+		return "SOURCE&SINK";
+	else if (mode == MIRACAST_DISABLED)
+		return "DISABLED";
+	else
+		return "INVALID";
+}
+
+#ifdef CONFIG_WFD
+static bool wfd_st_match_rule(_adapter *adapter, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port)
+{
+	struct wifi_display_info *wfdinfo = &adapter->wfd_info;
+
+	if (ntohs(*((u16 *)local_port)) == wfdinfo->rtsp_ctrlport
+	    || ntohs(*((u16 *)local_port)) == wfdinfo->tdls_rtsp_ctrlport
+	    || ntohs(*((u16 *)remote_port)) == wfdinfo->peer_rtsp_ctrlport)
+		return _TRUE;
+	return _FALSE;
+}
+
+static struct st_register wfd_st_reg = {
+	.s_proto = 0x06,
+	.rule = wfd_st_match_rule,
+};
+#endif /* CONFIG_WFD */
+
+inline void rtw_wfd_st_switch(struct sta_info *sta, bool on)
+{
+#ifdef CONFIG_WFD
+	if (on)
+		rtw_st_ctl_register(&sta->st_ctl, SESSION_TRACKER_REG_ID_WFD, &wfd_st_reg);
+	else
+		rtw_st_ctl_unregister(&sta->st_ctl, SESSION_TRACKER_REG_ID_WFD);
+#endif
+}
+
+void dump_arp_pkt(void *sel, u8 *da, u8 *sa, u8 *arp, bool tx)
+{
+	char mac_addr_str_sa[MAC_FMT_LEN];
+	char mac_addr_str_da[MAC_FMT_LEN];
+	char mac_addr_str_arp_sa[MAC_FMT_LEN];
+	char mac_addr_str_arp_ta[MAC_FMT_LEN];
+
+	RTW_PRINT_SEL(sel, "%s ARP da=%s, sa=%s\n"
+		, tx ? "send" : "recv"
+		, get_macaddr_str(mac_addr_str_da, sel, da)
+		, get_macaddr_str(mac_addr_str_sa, sel, sa));
+	RTW_PRINT_SEL(sel, "htype=%u, ptype=0x%04x, hlen=%u, plen=%u, oper=%u\n"
+		, GET_ARP_HTYPE(arp), GET_ARP_PTYPE(arp), GET_ARP_HLEN(arp)
+		, GET_ARP_PLEN(arp), GET_ARP_OPER(arp));
+	RTW_PRINT_SEL(sel, "sha=%s, spa="IP_FMT"\n"
+		, get_macaddr_str(mac_addr_str_arp_sa, sel, ARP_SENDER_MAC_ADDR(arp))
+		, IP_ARG(ARP_SENDER_IP_ADDR(arp)));
+	RTW_PRINT_SEL(sel, "tha=%s, tpa="IP_FMT"\n"
+		, get_macaddr_str(mac_addr_str_arp_ta, sel, ARP_TARGET_MAC_ADDR(arp))
+		, IP_ARG(ARP_TARGET_IP_ADDR(arp)));
+}
+
+/* software setting top half for connect abort */
+static void _connect_abort_sw_top_half(struct _ADAPTER *a)
+{
+	cancel_link_timer(&a->mlmeextpriv);
+	cancel_assoc_timer(&a->mlmepriv);
+	a->mlmeextpriv.join_abort = 1;
+}
+
+/* software setting bottom half for connect abort */
+static void _connect_abort_sw_bottom_half(struct _ADAPTER *a)
+{
+	/* ref: rtw_joinbss_event_prehandle(), join_res == -4 */
+	_clr_fwstate_(&a->mlmepriv, WIFI_UNDER_LINKING);
+	rtw_reset_securitypriv(a);
+	a->mlmeextpriv.join_abort = 0;
+}
+
+/*
+ * _connect_disconncet_hw - Handle hardware part of connect abort and fail
+ * @a:		struct _ADAPTER *
+ *
+ * Handle hardware part of connect fail.
+ * Most implement is reference from bottom half of rtw_joinbss_event_callback()
+ * with join_res < 0.
+ *
+ * Reference functions:
+ *	1. rtw_joinbss_event_callback()
+ *	2. rtw_set_hw_after_join(a, -1)
+ *	3. rtw_hw_connect_abort()
+ */
+static void _connect_disconnect_hw(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	u8 *mac;
+	struct _ADAPTER_LINK *alink = NULL;
+	struct rtw_phl_mld_t *mld = NULL;
+	u8 lidx;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	/* ref: rtw_set_hw_after_join(a, -1) */
+	/* bottom half of rtw_hw_connect_abort() - start */
+	for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+		alink = GET_LINK(a, lidx);
+		rtw_phl_chanctx_del(phl, a->phl_role, alink->wrlink, NULL);
+	}
+	/* restore original union ch */
+	rtw_join_done_chk_ch(a, -1);
+	/* free connecting AP sta info */
+	mac = a->mlmeextpriv.mlmext_info.dev_network.MacAddress;
+	mld = rtw_phl_get_mld_by_addr(phl, a->phl_role, mac);
+	if (mld) {
+		rtw_free_mld_stainfo(a, mld);
+	} else {
+		RTW_ERR(FUNC_ADPT_FMT ": Cann't find mld for " MAC_FMT " !\n",
+			FUNC_ADPT_ARG(a), MAC_ARG(mac));
+	}
+	rtw_init_self_stainfo(a, PHL_CMD_DIRECTLY);
+	/* bottom half of rtw_hw_connect_abort() - end */
+
+	/* bottom half of rtw_joinbss_event_callback() */
+	rtw_mi_os_xmit_schedule(a);
+}
+
+static void _connect_abort_notify_cb(void *priv, struct phl_msg *msg)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": connect_st=%u\n",
+		FUNC_ADPT_ARG(a), a->connect_state);
+
+	_connect_disconnect_hw(a);
+	_connect_abort_sw_bottom_half(a);
+
+	_rtw_spinlock_bh(&a->connect_st_lock);
+	if ((a->connect_state != CONNECT_ST_ACQUIRED) || !a->connect_abort) {
+		RTW_ERR(FUNC_ADPT_FMT ": connect_st=%u, abort is %s !\n",
+			FUNC_ADPT_ARG(a), a->connect_state,
+			a->connect_abort?"true":"false");
+	} else {
+		a->connect_state = CONNECT_ST_IDLE;
+		a->connect_abort = false;
+	}
+	_rtw_spinunlock_bh(&a->connect_st_lock);
+}
+
+static enum rtw_phl_status _connect_abort_notify(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	enum rtw_phl_status status;
+
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_CONNECT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_DISCONNECT);
+	msg.band_idx = a->connect_bidx;
+	msg.rsvd[0].ptr = (u8*)a->phl_role;
+
+	attr.opt = MSG_OPT_SEND_IN_ABORT;
+	attr.completion.completion = _connect_abort_notify_cb;
+	attr.completion.priv = a;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d), &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+
+	return status;
+}
+
+static void _connect_swch_done_notify_cb(void *priv, struct phl_msg *msg)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": connect_st=%u\n",
+		FUNC_ADPT_ARG(a), a->connect_state);
+
+	if (msg->inbuf) {
+		rtw_vmfree(msg->inbuf, msg->inlen);
+		msg->inbuf = NULL;
+	}
+}
+
+static enum rtw_phl_status
+_connect_swch_done_notify(struct _ADAPTER *a,
+					struct _ADAPTER_LINK *alink,
+					struct rtw_chan_def *chandef)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	u8 *info = NULL;
+	enum rtw_phl_status status;
+
+
+	info = rtw_vmalloc(sizeof(struct rtw_chan_def));
+	if (!info) {
+		RTW_ERR(FUNC_ADPT_FMT ": Allocate msg hub buffer fail!\n",
+			FUNC_ADPT_ARG(a));
+		return RTW_PHL_STATUS_RESOURCE;
+	}
+	_rtw_memcpy(info, chandef, sizeof(struct rtw_chan_def));
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_CONNECT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_SWCH_DONE);
+	msg.band_idx = a->connect_bidx;
+	msg.inbuf = info;
+	msg.inlen = sizeof(struct rtw_chan_def);
+	msg.rsvd[0].ptr = (void *)alink->wrlink;
+
+	attr.completion.completion = _connect_swch_done_notify_cb;
+	attr.completion.priv = a;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+					       &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		rtw_vmfree(info, sizeof(struct rtw_chan_def));
+		RTW_ERR(FUNC_ADPT_FMT ": send MSG_EVT_SWCH_DONE fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+	}
+
+	return status;
+}
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+static enum rtw_phl_status _connect_end_dbcc_en_notify(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_wifi_role_t *wrole = a->phl_role;
+	enum rtw_phl_status status;
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_CONNECT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_CONNECT_END_DBCC_EN);
+
+	msg.band_idx = a->connect_bidx;
+	msg.inbuf = (u8 *)wrole;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d), &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send MSG_EVT_CONNECT_END_DBCC_EN fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+
+	return status;
+}
+#endif
+
+static enum rtw_phl_status _connect_disconnect_end_notify(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_wifi_role_t *wrole = a->phl_role;
+	enum rtw_phl_status status;
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_CONNECT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_DISCONNECT_END);
+
+	msg.band_idx = a->connect_bidx;
+	msg.inbuf = (u8 *)wrole;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d), &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send MSG_EVT_DISCONNECT_END fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+
+	return status;
+}
+
+static void _connect_cmd_done(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_wifi_role_t *role = a->phl_role;
+	enum rtw_phl_status status;
+
+
+	_rtw_spinlock_bh(&a->connect_st_lock);
+	RTW_DBG(FUNC_ADPT_FMT ": + token=%#010x\n", FUNC_ADPT_ARG(a), a->connect_token);
+	if (!a->connect_token) {
+		_rtw_spinunlock_bh(&a->connect_st_lock);
+		RTW_ERR(FUNC_ADPT_FMT ": token is NULL!\n", FUNC_ADPT_ARG(a));
+		return;
+	}
+
+	status = rtw_phl_free_cmd_token(GET_PHL_INFO(d),
+					a->connect_bidx, &a->connect_token);
+	a->connect_token = 0;
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": free_cmd_token fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+	a->connect_state = CONNECT_ST_IDLE;
+	a->connect_abort = false;
+	_rtw_spinunlock_bh(&a->connect_st_lock);
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+}
+
+static enum phl_mdl_ret_code _connect_acquired(void* dispr, void *priv)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct _WLAN_BSSID_EX *network = &a->mlmeextpriv.mlmext_info.dev_network;
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+
+	_rtw_spinlock_bh(&a->connect_st_lock);
+	if (a->connect_state != CONNECT_ST_REQUESTING)
+		RTW_ERR(FUNC_ADPT_FMT ": connect_st=%u, not requesting?!\n",
+			FUNC_ADPT_ARG(a), a->connect_state);
+	a->connect_state = CONNECT_ST_ACQUIRED;
+	a->connect_bidx = alink->wrlink->hw_band;
+	_rtw_spinunlock_bh(&a->connect_st_lock);
+
+	/*rtw_hw_prepare_connect(a, NULL, network->MacAddress);*/
+	rtw_phl_connect_prepare(GET_PHL_INFO(d), a->connect_bidx, a->phl_role,
+				alink->wrlink, network->MacAddress);
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _connect_abort(void* dispr, void *priv)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	bool inner_abort = false;
+	enum rtw_phl_status phl_status = RTW_PHL_STATUS_FAILURE;
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+
+	_rtw_spinlock_bh(&a->connect_st_lock);
+	RTW_INFO(FUNC_ADPT_FMT ": connect_st=%u, abort is %s\n",
+		 FUNC_ADPT_ARG(a), a->connect_state,
+		 a->connect_abort?"true":"false");
+	if (a->connect_state == CONNECT_ST_IDLE) {
+		_rtw_spinunlock_bh(&a->connect_st_lock);
+		return MDL_RET_SUCCESS;
+	}
+	if (!a->connect_abort) {
+		RTW_INFO(FUNC_ADPT_FMT ": framework asking abort!\n",
+			 FUNC_ADPT_ARG(a));
+		a->connect_abort = true;
+		inner_abort = true;
+	}
+	_rtw_spinunlock_bh(&a->connect_st_lock);
+
+	_connect_abort_sw_top_half(a);
+	if (inner_abort) {
+		/* ref: rtw_join_timeout_handler() */
+		_rtw_spinlock_bh(&a->mlmepriv.lock);
+		a->mlmepriv.join_status = WLAN_STATUS_UNSPECIFIED_FAILURE;
+		rtw_indicate_disconnect(a, a->mlmepriv.join_status, _FALSE);
+#ifdef CONFIG_IOCTL_CFG80211
+		rtw_cfg80211_indicate_disconnect(a, a->mlmepriv.join_status, _FALSE);
+#endif /* CONFIG_IOCTL_CFG80211 */
+		a->mlmepriv.join_status = 0;
+		_rtw_spinunlock_bh(&a->mlmepriv.lock);
+	}
+
+	if (a->connect_state == CONNECT_ST_ACQUIRED)
+		phl_status = _connect_abort_notify(a);
+	a->connect_token = 0; /* framework will free this token later */
+	if (phl_status != RTW_PHL_STATUS_SUCCESS) {
+		/* No callback function, everything should be done here */
+		_connect_abort_sw_bottom_half(a);
+		_rtw_spinlock_bh(&a->connect_st_lock);
+		a->connect_state = CONNECT_ST_IDLE;
+		a->connect_abort = false;
+		_rtw_spinunlock_bh(&a->connect_st_lock);
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _connect_msg_hdlr(void* dispr, void* priv,
+					       struct phl_msg* msg)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_wifi_role_t *role = NULL;
+	struct _WLAN_BSSID_EX *network = &a->mlmeextpriv.mlmext_info.dev_network;
+	struct sta_info *sta = NULL;
+	struct rtw_chan_def chdef = {0};
+	struct phl_msg nextmsg = {0};
+	struct phl_msg_attribute attr = {0};
+	enum rtw_phl_status status;
+	enum phl_mdl_ret_code mdl_err;
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+	int err;
+	u32 res;
+	struct rtw_phl_mld_t *mld = NULL;
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	WLAN_BSSID_EX *link_network = NULL;
+	u8 lidx;
+
+	RTW_DBG(FUNC_ADPT_FMT ": + msg_id=0x%08x\n",
+		FUNC_ADPT_ARG(a), msg->msg_id);
+
+	if (MSG_MDL_ID_FIELD(msg->msg_id) != PHL_FG_MDL_CONNECT &&
+		MSG_MDL_ID_FIELD(msg->msg_id) != PHL_MDL_MRC) {
+		/*RTW_INFO(FUNC_ADPT_FMT ": Message is not from connect module or mrc, "
+			 "skip msg_id=0x%08x - MID(%d), MSG_ID(%d)\n",
+			 FUNC_ADPT_ARG(a), msg->msg_id,
+			 MSG_MDL_ID_FIELD(msg->msg_id),
+			 MSG_EVT_ID_FIELD(msg->msg_id));
+		RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));*/
+		return MDL_RET_IGNORE;
+	}
+
+	if (IS_MSG_FAIL(msg->msg_id)) {
+		RTW_WARN(FUNC_ADPT_FMT ": cmd dispatcher notify cmd failure on "
+			 "msg_id=0x%08x - MID(%d), MSG_ID(%d)\n",
+			 FUNC_ADPT_ARG(a), msg->msg_id,
+			 MSG_MDL_ID_FIELD(msg->msg_id),
+			 MSG_EVT_ID_FIELD(msg->msg_id));
+		switch (MSG_EVT_ID_FIELD(msg->msg_id)) {
+		case MSG_EVT_DISCONNECT_PREPARE:
+		case MSG_EVT_DISCONNECT:
+		case MSG_EVT_DISCONNECT_END:
+			break;
+		default:
+			goto send_disconnect;
+		}
+	}
+
+	role = a->phl_role;
+
+	switch (MSG_EVT_ID_FIELD(msg->msg_id)) {
+	case MSG_EVT_CONNECT_START:
+		RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_CONNECT_START\n",
+			FUNC_ADPT_ARG(a));
+
+		/* Check IOT info (Debug purpose) */
+		if (a->mlmeextpriv.mlmext_info.assoc_AP_vendor == HT_IOT_PEER_UNKNOWN) {
+			RTW_ERR(FUNC_ADPT_FMT ": assoc_AP_vendor gone!\n",
+				FUNC_ADPT_ARG(a));
+		}
+
+		/* ref: top half of rtw_join_cmd_hdl() */
+
+		sta = rtw_get_stainfo(&a->stapriv, a->phl_role->mac_addr);
+		if (sta != NULL)
+			rtw_free_mld_stainfo(a, sta->phl_sta->mld);
+		mld = rtw_phl_alloc_mld(GET_PHL_INFO(d), a->phl_role, network->MacAddress, DTYPE);
+		for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+			alink = GET_LINK(a, lidx);
+			if (!alink->mlmepriv.to_join)
+				continue;
+			link_network = &alink->mlmeextpriv.mlmext_info.network;
+			/* main_id is don't care for self sta */
+			sta = rtw_alloc_stainfo(&a->stapriv, link_network->MacAddress, DTYPE, 0, lidx, PHL_CMD_DIRECTLY);
+			if (sta == NULL) {
+				RTW_ERR(FUNC_ADPT_FMT ": alloc sta " MAC_FMT " fail!\n",
+					FUNC_ADPT_ARG(a), MAC_ARG(network->MacAddress));
+				rtw_free_mld_stainfo(a, mld);
+				rtw_init_self_stainfo(a, PHL_CMD_DIRECTLY);
+				goto send_disconnect;
+			}
+			/* check band, channel, bandwidth, offset and switch */
+			chdef.band = alink->mlmeextpriv.chandef.band;
+			chdef.chan = alink->mlmeextpriv.chandef.chan;
+			chdef.bw = alink->mlmeextpriv.chandef.bw;
+			chdef.offset = alink->mlmeextpriv.chandef.offset;
+			if (!rtw_phl_chanctx_add(GET_PHL_INFO(d), role, alink->wrlink, &chdef, &mr_cc_info)) {
+				RTW_ERR("add chanctx_fail band=%d, chan=%d, bw=%d, offset=%d\n",
+				chdef.band, chdef.chan, chdef.bw, chdef.offset);
+				goto send_disconnect;
+			}
+
+			rtw_hw_update_chan_def(a, alink);
+
+			#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+			rtw_dfs_rd_en_dec_on_mlme_act(a, alink, MLME_STA_CONNECTING, 0);
+			#endif
+		}
+
+#ifdef CONFIG_ANTENNA_DIVERSITY
+		rtw_antenna_select_cmd(a, network->PhyInfo.Optimum_antenna, _FALSE);
+#endif
+
+		SET_MSG_MDL_ID_FIELD(nextmsg.msg_id, PHL_FG_MDL_CONNECT);
+		SET_MSG_EVT_ID_FIELD(nextmsg.msg_id, MSG_EVT_SWCH_START);
+		nextmsg.band_idx = a->connect_bidx;
+
+		status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+						   &nextmsg, &attr, NULL);
+		if (status != RTW_PHL_STATUS_SUCCESS)
+			RTW_ERR(FUNC_ADPT_FMT ": send MSG_EVT_SWCH_START fail(0x%x)!\n",
+					FUNC_ADPT_ARG(a), status);
+		else
+			RTW_DBG(FUNC_ADPT_FMT ": send MSG_EVT_SWCH_START Success!\n",
+					FUNC_ADPT_ARG(a));
+		break;
+
+	case MSG_EVT_SWCH_START:
+		RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_SWCH_START\n",
+			FUNC_ADPT_ARG(a));
+		if (!role) {
+			RTW_ERR(FUNC_ADPT_FMT ": role == NULL\n",
+				FUNC_ADPT_ARG(a));
+			break;
+		}
+
+		for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+			alink = GET_LINK(a, lidx);
+			if (!alink->mlmepriv.to_join)
+				continue;
+			/* ref: bottom half of rtw_join_cmd_hdl() */
+			if (rtw_phl_mr_get_chandef(d->phl, a->phl_role, alink->wrlink, &chdef)
+								!= RTW_PHL_STATUS_SUCCESS) {
+				RTW_ERR("%s get union chandef failed\n", __func__);
+				rtw_warn_on(1);
+				break;
+			}
+			RTW_DBG(FUNC_ADPT_FMT ": Switch to channel before link: "
+				"chan(%d), bw(%d), offset(%d)\n",
+				 FUNC_ADPT_ARG(a),
+				 chdef.chan, chdef.bw, chdef.offset);
+			set_bch_bwmode(a, alink, chdef.band, chdef.chan, chdef.offset, chdef.bw, RFK_TYPE_CONNECT);
+		}
+		alink = GET_PRIMARY_LINK(a);
+		status = _connect_swch_done_notify(a, alink, &alink->wrlink->chandef);
+		break;
+
+	case MSG_EVT_SWCH_DONE:
+		RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_SWCH_DONE\n",
+			FUNC_ADPT_ARG(a));
+
+#ifdef RTK_WKARD_CORE_BTC_STBC_CAP
+		/* In normal mode, reconsider STBC capability setting
+		   before doing client join */
+		if (a->registrypriv.wifi_spec == 0) {
+			for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+				alink = GET_LINK(a, lidx);
+				rtw_phl_cmd_wrole_change(GET_PHL_INFO(d), role, alink->wrlink,
+							 WR_CHG_STBC_CFG, NULL, 0,
+							 PHL_CMD_DIRECTLY, 0);
+			}
+		}
+#endif
+
+		/* ref: last part of rtw_join_cmd_hdl() */
+		cancel_link_timer(&a->mlmeextpriv);
+		start_clnt_join(a);
+
+		break;
+
+	case MSG_EVT_CONNECT_LINKED:
+		RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_CONNECT_LINKED\n",
+			FUNC_ADPT_ARG(a));
+
+		/* ref: top half of rtw_joinbss_event_callback() */
+		/* Remove the unaccepted link sta in this function */
+		err = rtw_set_hw_after_join(a, 0);
+		if (err) {
+			RTW_ERR(FUNC_ADPT_FMT ": set hardware fail(%d) during "
+				"connecting!\n",
+				FUNC_ADPT_ARG(a), err);
+			goto send_disconnect;
+		}
+
+		break;
+
+	case MSG_EVT_CONNECT_END:
+		RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_CONNECT_END\n",
+			FUNC_ADPT_ARG(a));
+
+		/* ref: bottom half of rtw_joinbss_event_callback() */
+		rtw_mi_os_xmit_schedule(a);
+#ifdef CONFIG_DBCC_SUPPORT
+		if (rtw_phl_mr_is_db(adapter_to_dvobj(a)->phl)) {
+			alink = GET_PRIMARY_LINK(a);
+			if (alink->mlmepriv.hepriv.he_option == _TRUE)
+				rtw_he_om_ctrl_trx_ss(a, alink, sta, 1, _FALSE);
+			else if (alink->mlmepriv.vhtpriv.vht_option == _TRUE)
+				rtw_vht_op_mode_ctrl_rx_nss(a, alink, sta, 1, _FALSE);
+		}
+	#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+		else {
+			if (rtw_dbcc_b0_sta_chan_chk(a))
+				rtw_phl_mr_trig_dbcc_enable(GET_PHL_INFO(d));
+		}
+		_connect_end_dbcc_en_notify(a);
+		break;
+	case MSG_EVT_CONNECT_END_DBCC_EN:
+	#endif/*CONFIG_DBCC_P2P_BG_LISTEN*/
+#endif/*CONFIG_DBCC_SUPPORT*/
+		_connect_cmd_done(a);
+#ifdef CONFIG_LAYER2_ROAMING
+		if (a->securitypriv.dot11PrivacyAlgrthm == _NO_PRIVACY_)
+			dequeuq_roam_pkt(a);
+#endif
+		break;
+
+	case MSG_EVT_DISCONNECT_PREPARE:
+		RTW_WARN(FUNC_ADPT_FMT ": MSG_EVT_DISCONNECT_PREPARE\n",
+			 FUNC_ADPT_ARG(a));
+
+		/* STA connect fail case, top half */
+
+		/* top half of rtw_joinbss_event_callback() */
+		err = rtw_set_hw_after_join(a, -1);
+		if (err) {
+			RTW_ERR(FUNC_ADPT_FMT ": set hardware fail(%d) during "
+				"connect abort!\n",
+				FUNC_ADPT_ARG(a), err);
+		}
+
+		rtw_phl_disconnect(GET_PHL_INFO(d), a->connect_bidx, a->phl_role, false);
+
+		break;
+
+	case MSG_EVT_DISCONNECT:
+		RTW_WARN(FUNC_ADPT_FMT ": MSG_EVT_DISCONNECT\n",
+			 FUNC_ADPT_ARG(a));
+
+		/* STA connect fail case, bottom half */
+		_connect_disconnect_hw(a);
+		rtw_clear_to_join_status(a);
+		rtw_clear_is_accepted_status(a);
+
+#ifdef CONFIG_DBCC_SUPPORT
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+		/*B1 GC disconnect*/
+		if (rtw_dbcc_chk_disable_hdl(a, NULL, PHL_CMD_DIRECTLY, DBCC_CHK_STA_DIS))
+#endif
+		{
+			rtw_phl_chanctx_chk(GET_PHL_INFO(adapter_to_dvobj(a)),
+						a->phl_role, alink->wrlink, NULL, &mr_cc_info);
+			if (mr_cc_info.sugg_opmode == MR_OP_DBCC) /*current stay in DBCC*/
+				rtw_phl_mr_trig_dbcc_disable(GET_PHL_INFO(adapter_to_dvobj(a)));
+
+		}
+		status = _connect_disconnect_end_notify(a);
+		if (status == RTW_PHL_STATUS_SUCCESS)
+			break;
+		else
+			if (mr_cc_info.sugg_opmode == MR_OP_DBCC)
+				rtw_phl_mr_cancel_dbcc_action(GET_PHL_INFO(adapter_to_dvobj(a)));
+		fallthrough;
+	case MSG_EVT_DISCONNECT_END:
+#endif
+		_connect_cmd_done(a);
+		break;
+#ifdef CONFIG_DBCC_SUPPORT
+	case MSG_EVT_DBCC_ENABLE:
+		if (MSG_MDL_ID_FIELD(msg->msg_id) == PHL_MDL_MRC) {
+			role = (struct rtw_wifi_role_t *)msg->inbuf;
+			RTW_INFO("[CON-FG] WR(%d) RX MRC- DBCC ENABLE MSG\n", role->id);
+		}
+		break;
+#endif
+	default:
+		break;
+	}
+
+	goto exit;
+
+send_disconnect:
+	/*
+	 * Trigger software handle and notify OS by rtw_joinbss_event_prehandle()
+	 * Trigger hardware handle by sending MSG_EVT_DISCONNECT
+	 */
+	res = report_join_res(a, -4, WLAN_STATUS_UNSPECIFIED_FAILURE);
+	if (res != _SUCCESS) {
+		/*
+		 * Fail to send MSG_EVT_DISCONNECT_PREPARE, do jobs in
+		 * MSG_EVT_DISCONNECT_PREPARE and MSG_EVT_DISCONNECT directly
+		 * here.
+		 */
+
+		/* ref: rtw_set_hw_after_join(a, -1) */
+		a->mlmepriv.wpa_phase = _FALSE;
+		for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+			alink = GET_LINK(a, lidx);
+			link_network = &alink->mlmeextpriv.mlmext_info.network;
+			sta = rtw_get_stainfo(&a->stapriv, link_network->MacAddress);
+			if (sta) {
+				/* rtw_hw_connect_abort(a, sta) */
+				rtw_hw_del_all_key(a, sta, PHL_CMD_DIRECTLY, 0);
+				if (rtw_sta_hal_media_status_rpt_cmd(a, sta, false, RTW_CMDF_DIRECTLY) != _SUCCESS) {
+					RTW_ERR(FUNC_ADPT_FMT ": update media status fail!\n", FUNC_ADPT_ARG(a));
+				}
+			} else {
+				RTW_ERR(FUNC_ADPT_FMT ": stainfo(" MAC_FMT ") not exist!\n",
+					FUNC_ADPT_ARG(a), MAC_ARG(link_network->MacAddress));
+			}
+		}
+		_connect_disconnect_hw(a);
+		rtw_clear_to_join_status(a);
+		rtw_clear_is_accepted_status(a);
+		_connect_cmd_done(a);
+	}
+
+exit:
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _connect_set_info(void* dispr, void* priv,
+					       struct phl_module_op_info* info)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+
+	return MDL_RET_IGNORE;
+}
+
+static enum phl_mdl_ret_code _connect_query_info(void* dispr, void* priv,
+						struct phl_module_op_info* info)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+
+	switch (info->op_code) {
+	case FG_REQ_OP_GET_ROLE:
+		info->outbuf = (u8*)a->phl_role;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	case FG_REQ_OP_GET_ROLE_LINK:
+		info->outbuf = (u8*)alink->wrlink;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	default:
+		break;
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+	return ret;
+}
+
+enum rtw_phl_status rtw_connect_cmd(struct _ADAPTER *a,
+				    struct _WLAN_BSSID_EX *network)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_wifi_role_t *role = a->phl_role;
+	struct rtw_chan_def *chdef;
+	struct mi_state mstate = {0};
+	struct phl_cmd_token_req *cmd_req;
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+	struct _ADAPTER_LINK *alink = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+
+	RTW_DBG(FUNC_ADPT_FMT ": st=%u\n",
+		FUNC_ADPT_ARG(a), a->connect_state);
+
+	/* ref: top half of rtw_join_cmd_hdl(), software only */
+	/* Todo: disconnect before connecting */
+	/*set_hw_before_join(a);*/
+
+	/* Update HT/VHT/HE CAP and chan/bw/offset to a->mlmeextpriv.mlmext_info.network */
+	update_join_info(a, network);
+
+	/* check channel, bandwidth, offset and switch */
+	/* Launch the connection (connect_req) with the primary link */
+	alink = GET_PRIMARY_LINK(a);
+	rlink = alink->wrlink;
+	chdef = &alink->mlmeextpriv.chandef;
+
+#ifdef CONFIG_DBCC_SUPPORT
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+	/*B0,B1 GC Connection*/
+	rtw_dbcc_chk_disable_hdl(a, chdef, PHL_CMD_DIRECTLY, DBCC_CHK_STA_CON);
+#endif
+#endif
+	if (!rtw_phl_chanctx_chk(GET_PHL_INFO(d), role, rlink, chdef, &mr_cc_info))
+	{
+		#ifdef CONFIG_DBCC_SUPPORT
+		if (mr_cc_info.sugg_opmode == MR_OP_DBCC) {
+			rtw_phl_mr_trig_dbcc_enable(GET_PHL_INFO(d));
+		} else
+		#endif
+		{
+			/* ref: not group case in rtw_chk_start_clnt_join() */
+			struct disconnect_data disc_data = {0};
+
+			rtw_mi_status_no_self(a, &mstate);
+			RTW_WARN(FUNC_ADPT_FMT ": channel group fail! ld_sta_num:%u, "
+				 "ap_num:%u, mesh_num:%u\n",
+				 FUNC_ADPT_ARG(a), MSTATE_STA_LD_NUM(&mstate),
+				 MSTATE_AP_NUM(&mstate), MSTATE_MESH_NUM(&mstate));
+			if ((MSTATE_STA_LD_NUM(&mstate) + MSTATE_AP_LD_NUM(&mstate)
+			     + MSTATE_MESH_LD_NUM(&mstate)) >= 4) {
+				status = RTW_PHL_STATUS_RESOURCE;
+				goto exit;
+			}
+			disc_data.disc_code = DISCONNECTION_BY_DRIVER_DUE_TO_EACH_IFACE_CHBW_NOT_SYNC;
+			disc_data.buddy_chdef = chdef;
+
+			rtw_mi_buddy_disconnect(a, &disc_data);
+		}
+	}
+
+	_rtw_spinlock_bh(&a->connect_st_lock);
+
+	if (a->connect_state != CONNECT_ST_IDLE) {
+		status = RTW_PHL_STATUS_SUCCESS;
+		RTW_WARN(FUNC_ADPT_FMT ": connect is on going...\n",
+			 FUNC_ADPT_ARG(a));
+		goto exit;
+	}
+
+	cmd_req = &a->connect_req;
+	cmd_req->role = role;
+	status = rtw_phl_add_cmd_token_req(GET_PHL_INFO(d), rlink->hw_band,
+						cmd_req, &a->connect_token);
+	if ((status != RTW_PHL_STATUS_SUCCESS)
+	    && (status != RTW_PHL_STATUS_PENDING)) {
+		RTW_ERR(FUNC_ADPT_FMT ": add_cmd_token_req fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+		#ifdef CONFIG_DBCC_SUPPORT
+		if (mr_cc_info.sugg_opmode == MR_OP_DBCC)
+			rtw_phl_mr_cancel_dbcc_action(GET_PHL_INFO(d));
+		#endif
+		goto exit;
+	}
+
+	a->connect_state = CONNECT_ST_REQUESTING;
+	status = RTW_PHL_STATUS_SUCCESS;
+
+exit:
+	_rtw_spinunlock_bh(&a->connect_st_lock);
+
+	RTW_DBG(FUNC_ADPT_FMT ": - st=%u ret=%u\n",
+		FUNC_ADPT_ARG(a), a->connect_state, status);
+	return status;
+}
+
+void rtw_connect_abort(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	enum rtw_phl_status status;
+
+
+	_rtw_spinlock_bh(&a->connect_st_lock);
+	RTW_WARN(FUNC_ADPT_FMT ": connect_st=%u, abort=%u\n",
+		 FUNC_ADPT_ARG(a), a->connect_state, a->connect_abort);
+
+	if ((a->connect_state == CONNECT_ST_IDLE) || a->connect_abort) {
+		_rtw_spinunlock_bh(&a->connect_st_lock);
+		return;
+	}
+	a->mlmepriv.wpa_phase = _FALSE;
+	a->connect_abort = true;
+	_rtw_spinunlock_bh(&a->connect_st_lock);
+
+	status = rtw_phl_cancel_cmd_token(GET_PHL_INFO(d),
+					  a->connect_bidx,
+					  &a->connect_token);
+
+	a->connect_token = 0;
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR(FUNC_ADPT_FMT ": cancel_cmd_token fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+		/* Cancel fail, and something needed to be handled by self */
+		/* Release connect resource (software) */
+		_connect_abort_sw_top_half(a);
+		_connect_abort_sw_bottom_half(a);
+
+		_rtw_spinlock_bh(&a->connect_st_lock);
+		a->connect_state = CONNECT_ST_IDLE;
+		a->connect_abort = false;
+		_rtw_spinunlock_bh(&a->connect_st_lock);
+	}
+}
+
+#define _CONNECT_ABORT_TO_		5000	/* unit: ms */
+#define _MSG_INTERVAL_(t, c)		(((t) < c) ? 1 : ((t) / c))
+/**
+ * rtw_connect_abort_wait() - Abort connect FG and wait to finish
+ * @a		pointer of struct _ADAPTER
+ *
+ * Send cancel cmd to framework and wait connect FG to stop.
+ *
+ * Return 0 for connect FG already stopped, -1 for sending cancel command
+ * fail, or -2 for waiting connect FG stop timeout.
+ */
+int rtw_connect_abort_wait(struct _ADAPTER *a)
+{
+	u32 timeout = _CONNECT_ABORT_TO_;
+	systime start_t;
+	u32 pass_t = 0;
+	u32 msg_i;	/* message interval */
+	u32 msg_t = 0;	/* next time point to print message */
+	int err = 0;
+
+
+	/* at most print 10 msg in whole waiting time */
+	msg_i = _MSG_INTERVAL_(timeout, 10);
+
+	_rtw_spinlock_bh(&a->connect_st_lock);
+	start_t = rtw_get_current_time();
+	while ((a->connect_state != CONNECT_ST_IDLE) ||
+	       (check_fwstate(&a->mlmepriv, WIFI_UNDER_LINKING) == _TRUE)) {
+		if (pass_t >= msg_t) {
+			RTW_INFO(FUNC_ADPT_FMT ": connect st=%u, fw_st=%#x, %u ms/%u pass...\n",
+				 FUNC_ADPT_ARG(a), a->connect_state,
+				 get_fwstate(&a->mlmepriv), pass_t, timeout);
+			msg_t += msg_i;
+		}
+		_rtw_spinunlock_bh(&a->connect_st_lock);
+
+		pass_t = rtw_get_passing_time_ms(start_t);
+		if (pass_t > timeout) {
+			RTW_ERR(FUNC_ADPT_FMT ": Timeout, fail to abort connect!"
+				" used:%u > max:%u ms\n",
+				FUNC_ADPT_ARG(a), pass_t, timeout);
+			err = -2; /* Timeout */
+			_rtw_spinlock_bh(&a->connect_st_lock);
+			break;
+		}
+		rtw_connect_abort(a);
+		rtw_msleep_os(1);
+
+		_rtw_spinlock_bh(&a->connect_st_lock);
+	}
+	_rtw_spinunlock_bh(&a->connect_st_lock);
+
+	return err;
+}
+
+void rtw_connect_req_free(struct _ADAPTER *a)
+{
+	rtw_connect_abort_wait(a);
+
+	_rtw_spinlock_free(&a->connect_st_lock);
+}
+
+void rtw_connect_req_init(struct _ADAPTER *a)
+{
+	struct phl_cmd_token_req *req;
+
+
+	_rtw_spinlock_init(&a->connect_st_lock);
+	a->connect_state = CONNECT_ST_IDLE;
+	a->connect_abort = false;
+
+	req = &a->connect_req;
+	req->module_id = PHL_FG_MDL_CONNECT;
+	req->opt = FG_CMD_OPT_EXCLUSIVE;
+	req->priv = a;
+	req->role = NULL; /* a->phl_role, but role will change by time */
+	req->acquired = _connect_acquired;
+	req->abort = _connect_abort;
+	req->msg_hdlr = _connect_msg_hdlr;
+	req->set_info = _connect_set_info;
+	req->query_info = _connect_query_info;
+}
+
+enum rtw_phl_status rtw_connect_disconnect_prepare(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	enum rtw_phl_status status;
+
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_CONNECT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_DISCONNECT_PREPARE);
+	msg.band_idx = a->connect_bidx;
+	msg.rsvd[0].ptr = (u8*)a->phl_role;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d), &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+
+	return status;
+}
+
+static enum rtw_phl_status _disconnect_done_notify(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_wifi_role_t *wrole = a->phl_role;
+	enum rtw_phl_status status;
+
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_DISCONNECT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_DISCONNECT);
+
+	msg.band_idx = a->disconnect_bidx;
+	msg.rsvd[0].ptr = (u8*)wrole;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d), &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+
+	return status;
+}
+
+static enum rtw_phl_status _disconnect_end_notify(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_wifi_role_t *wrole = a->phl_role;
+	enum rtw_phl_status status;
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_FG_MDL_DISCONNECT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_DISCONNECT_END);
+
+	msg.band_idx = a->disconnect_bidx;
+	msg.inbuf = (u8 *)wrole;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d), &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send_msg_to_dispr fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+
+	return status;
+}
+
+/*
+ * _disconnect_free_cmdobj:
+ * Call this function under acquiring disconnect_lock.
+ *
+ * Following items would be done in this function:
+ * 1. trigger complete event if a->discon_cmd->sctx exist.
+ * 2. free a->discon_cmd and set it to NULL.
+ *
+ */
+static void _disconnect_free_cmdobj(struct _ADAPTER *a)
+{
+	struct cmd_obj *cmd;
+
+
+	if (unlikely(!a->discon_cmd)) {
+		RTW_WARN(FUNC_ADPT_FMT ": discon_cmd is NULL! state=%u\n",
+			 FUNC_ADPT_ARG(a), a->disconnect_state);
+		return;
+	}
+	cmd = a->discon_cmd;
+	a->discon_cmd = NULL;
+
+	if (cmd->sctx) {
+		if (cmd->res == H2C_SUCCESS)
+			rtw_sctx_done(&cmd->sctx);
+		else
+			rtw_sctx_done_err(&cmd->sctx, RTW_SCTX_DONE_CMD_ERROR);
+	}
+
+	rtw_free_cmd_obj(cmd);
+}
+
+static void _disconnect_cmd_done(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_wifi_role_t *role = a->phl_role;
+	enum rtw_phl_status status;
+
+
+	_rtw_spinlock_bh(&a->disconnect_lock);
+	/* avoid to race with rtw_disconnect_abort() */
+	RTW_DBG(FUNC_ADPT_FMT ": + token=0x%08x state=%u\n",
+		FUNC_ADPT_ARG(a), a->disconnect_token, a->disconnect_state);
+
+	status = rtw_phl_free_cmd_token(GET_PHL_INFO(d),
+					a->disconnect_bidx, &a->disconnect_token);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": free_cmd_token fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+	a->disconnect_token = 0;
+	a->disconnect_state = DISCON_ST_IDLE;
+
+	_disconnect_free_cmdobj(a);
+
+	RTW_DBG(FUNC_ADPT_FMT ": - state=%u\n",
+		FUNC_ADPT_ARG(a), a->disconnect_state);
+	_rtw_spinunlock_bh(&a->disconnect_lock);
+}
+
+static enum phl_mdl_ret_code _disconnect_acquired(void* dispr, void *priv)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+	a->disconnect_bidx = alink->wrlink->hw_band;
+
+	rtw_phl_disconnect(GET_PHL_INFO(d), a->disconnect_bidx, a->phl_role, true);
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _disconnect_abort(void* dispr, void *priv)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+
+
+	_rtw_spinlock_bh(&a->disconnect_lock);
+	RTW_DBG(FUNC_ADPT_FMT ": + token=0x%08x state=%u\n",
+		FUNC_ADPT_ARG(a), a->disconnect_token, a->disconnect_state);
+
+	/*
+	 * Framework will free disconnect token automatically after abort,
+	 * so do all in _disconnect_cmd_done() besides rtw_phl_free_cmd_token().
+	 * ref: _disconnect_cmd_done()
+	 */
+	if (a->disconnect_state != DISCON_ST_ABORT) {
+		RTW_WARN(FUNC_ADPT_FMT ": framework asking abort! state=%u\n",
+			 FUNC_ADPT_ARG(a), a->disconnect_state);
+	}
+
+	a->disconnect_token = 0;
+	a->disconnect_state = DISCON_ST_IDLE;
+
+	_disconnect_free_cmdobj(a);
+
+	RTW_DBG(FUNC_ADPT_FMT ": - state=%u\n",
+		FUNC_ADPT_ARG(a), a->disconnect_state);
+	_rtw_spinunlock_bh(&a->disconnect_lock);
+
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _disconnect_msg_hdlr(void* dispr, void* priv,
+						  struct phl_msg* msg)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	struct disconnect_parm *discon;
+	struct stadel_event *stadel;
+	struct _WLAN_BSSID_EX *network;
+	u8 is_issue_deauth;
+	u32 retry = 0;
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only]
+	** May use other way to restore the link */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	struct mlme_priv *pmlmepriv = &a->mlmepriv;
+	#ifdef CONFIG_DBCC_SUPPORT
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+	#endif
+
+	RTW_DBG(FUNC_ADPT_FMT ": + msg_id=0x%08x\n",
+		FUNC_ADPT_ARG(a), msg->msg_id);
+
+	if ((MSG_MDL_ID_FIELD(msg->msg_id) != PHL_FG_MDL_DISCONNECT) &&
+	    (MSG_MDL_ID_FIELD(msg->msg_id) != PHL_MDL_MRC)){
+		RTW_INFO(FUNC_ADPT_FMT ": Message is not from disconnect or mrc module, "
+			 "skip msg_id=0x%08x\n", FUNC_ADPT_ARG(a), msg->msg_id);
+		RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+		return MDL_RET_IGNORE;
+	}
+
+	/* Whether msg fail or not */
+	switch (MSG_EVT_ID_FIELD(msg->msg_id)) {
+	case MSG_EVT_DISCONNECT_PREPARE:
+		RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_DISCONNECT_PREPARE\n",
+			FUNC_ADPT_ARG(a));
+
+		/*
+		 * ref: Top half of disconnect_hdl(), before and including
+		 *	rtw_mlmeext_disconnect().
+		 * Note: The bottom half, after rtw_mlmeext_disconnect(),
+		 *	 would be put to case MSG_EVT_DISCONNECT.
+		 */
+
+		if ((a->discon_cmd->cmdcode == CMD_SET_MLME_EVT)
+#if CONFIG_DFS
+			|| IS_RADAR_DETECTED(adapter_to_rfctl(a))
+#ifdef CONFIG_ECSA_PHL
+			|| rtw_mr_is_ecsa_running(a)
+#endif
+#ifdef CONFIG_RTW_80211R
+			|| pmlmepriv->roam_network
+#endif
+#endif
+		   )
+			is_issue_deauth = 0;
+		else
+			is_issue_deauth = 1;
+
+		if (is_issue_deauth) {
+#ifdef CONFIG_PLATFORM_ROCKCHIPS
+			/*
+			 * To avoid connecting to AP fail during resume process,
+			 * change retry count from 5 to 1
+			 */
+			retry = 1;
+#else /* !CONFIG_PLATFORM_ROCKCHIPS */
+			discon = (struct disconnect_parm*)a->discon_cmd->parmbuf;
+			retry = discon->deauth_timeout_ms / 100;
+#endif /* !CONFIG_PLATFORM_ROCKCHIPS */
+
+			network = &alink->mlmeextpriv.mlmext_info.network;
+
+			issue_deauth_ex(a, network->MacAddress,
+					WLAN_REASON_DEAUTH_LEAVING,
+					retry, 100);
+		}
+
+		rtw_mlmeext_disconnect(a);
+
+		status = _disconnect_done_notify(a);
+		if (status == RTW_PHL_STATUS_SUCCESS)
+			break;
+		fallthrough;
+	case MSG_EVT_DISCONNECT:
+		RTW_DBG(FUNC_ADPT_FMT ": MSG_EVT_DISCONNECT\n",
+			FUNC_ADPT_ARG(a));
+
+		rtw_disconnect_ch_switch(a);
+
+		if (a->discon_cmd->cmdcode == CMD_SET_MLME_EVT) {
+			/* EVT_DEL_STA case */
+			/* ref: bottom half of rtw_stadel_event_callback() */
+			stadel = (struct stadel_event*)(a->discon_cmd->parmbuf
+							+ sizeof(struct rtw_evt_header));
+			_stadel_posthandle_sta(a, stadel);
+		} else {
+			rtw_free_assoc_resources(a, _TRUE);
+		}
+
+		#ifdef CONFIG_DBCC_SUPPORT
+		/*After WPS M1~M8 we got a disconnect, this disconnect no need to do DBCC disable*/
+		#ifdef RTW_WKARD_NOT_DO_DBCC_DIS_IN_DRV_STOP
+		if (!check_fwstate(pmlmepriv, WIFI_UNDER_WPS) && (pmlmepriv->wpa_phase != _TRUE && !dev_is_drv_stopped(a->dvobj)))
+		#else
+		if (!check_fwstate(pmlmepriv, WIFI_UNDER_WPS) && (pmlmepriv->wpa_phase != _TRUE))
+		#endif
+		{
+			#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+			/*B1 GC disconnect*/
+			if (rtw_dbcc_chk_disable_hdl(a, NULL, PHL_CMD_DIRECTLY, DBCC_CHK_STA_DIS))
+			#endif
+			{
+				rtw_phl_chanctx_chk(GET_PHL_INFO(adapter_to_dvobj(a)),
+								a->phl_role, alink->wrlink, NULL, &mr_cc_info);
+				if (mr_cc_info.sugg_opmode == MR_OP_DBCC) /*current stay in DBCC*/
+					rtw_phl_mr_trig_dbcc_disable(GET_PHL_INFO(adapter_to_dvobj(a)));
+			}
+		}
+		#endif
+		status = _disconnect_end_notify(a);
+		if (status == RTW_PHL_STATUS_SUCCESS)
+			break;
+		#ifdef CONFIG_DBCC_SUPPORT
+		else
+			if (mr_cc_info.sugg_opmode == MR_OP_DBCC)
+				rtw_phl_mr_cancel_dbcc_action(GET_PHL_INFO(adapter_to_dvobj(a)));
+		#endif
+		fallthrough;
+	case MSG_EVT_DISCONNECT_END:
+		RTW_INFO(FUNC_ADPT_FMT ": MSG_EVT_DISCONNECT_END\n",
+						FUNC_ADPT_ARG(a));
+	#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+		/*B0 GC disconnect*/
+		if (rtw_dbcc_chk_enable_hdl(a, PHL_CMD_DIRECTLY, DBCC_CHK_STA_DIS))
+			rtw_phl_mr_trig_dbcc_enable(GET_PHL_INFO(adapter_to_dvobj(a)));
+
+		rtw_discon_end_dbcc_en_notify(a, PHL_FG_MDL_DISCONNECT);
+		break;
+	case MSG_EVT_DISCONNECT_END_DBCC_EN:
+		RTW_INFO(FUNC_ADPT_FMT ": MSG_EVT_DISCONNECT_END_DBCC_EN\n",
+						FUNC_ADPT_ARG(a));
+	#endif/*CONFIG_DBCC_P2P_BG_LISTEN*/
+		_disconnect_cmd_done(a);
+		break;
+
+#ifdef CONFIG_DBCC_SUPPORT
+	case MSG_EVT_DBCC_DISABLE:
+		if (MSG_MDL_ID_FIELD(msg->msg_id) == PHL_MDL_MRC) {
+			struct rtw_wifi_role_t *role;
+
+			role = (struct rtw_wifi_role_t *)msg->inbuf;
+			RTW_INFO("[DISCON-FG] WR(%d) RX MRC- DBCC DISABLE MSG\n", role->id);
+		}
+		break;
+#endif
+
+	default:
+		break;
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+	return MDL_RET_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _disconnect_set_info(void* dispr, void* priv,
+						struct phl_module_op_info* info)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+	return MDL_RET_IGNORE;
+}
+
+static enum phl_mdl_ret_code _disconnect_query_info(void* dispr, void* priv,
+						struct phl_module_op_info* info)
+{
+	struct _ADAPTER *a = (struct _ADAPTER *)priv;
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+
+	switch (info->op_code) {
+	case FG_REQ_OP_GET_ROLE:
+		info->outbuf = (u8 *)a->phl_role;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	case FG_REQ_OP_GET_ROLE_LINK:
+		info->outbuf = (u8*)&alink->wrlink;
+		ret = MDL_RET_SUCCESS;
+		break;
+
+	default:
+		break;
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": -\n", FUNC_ADPT_ARG(a));
+	return ret;
+}
+
+enum rtw_phl_status rtw_disconnect_cmd(struct _ADAPTER *a, struct cmd_obj *pcmd)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_wifi_role_t *role = a->phl_role;
+	struct rtw_evt_header *hdr;
+	struct stadel_event *stadel;
+	struct sta_info *sta;
+	struct phl_cmd_token_req *cmd_req;
+	enum rtw_phl_status phl_status = RTW_PHL_STATUS_FAILURE;
+	enum rtw_phl_status ret = RTW_PHL_STATUS_SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+
+	_rtw_spinlock_bh(&a->disconnect_lock);
+	RTW_DBG(FUNC_ADPT_FMT ": + token=0x%08x state=%u\n",
+		FUNC_ADPT_ARG(a), a->disconnect_token, a->disconnect_state);
+
+	if (a->disconnect_state != DISCON_ST_IDLE) {
+		RTW_INFO(FUNC_ADPT_FMT ": state=%u disconnect is on going...\n",
+			 FUNC_ADPT_ARG(a), a->disconnect_state);
+		ret = RTW_PHL_STATUS_RESOURCE;
+		goto exit;
+	}
+	a->disconnect_state = DISCON_ST_ONGOING;
+
+	if (pcmd->cmdcode == CMD_SET_MLME_EVT) {
+		/* EVT_DEL_STA case */
+		/* ref: top half of rtw_stadel_event_callback() */
+		hdr = (struct rtw_evt_header*)pcmd->parmbuf;
+		stadel = (struct stadel_event *)(pcmd->parmbuf + sizeof(*hdr));
+		sta = rtw_get_stainfo(&a->stapriv, stadel->macaddr);
+		if (!sta) {
+			RTW_ERR(FUNC_ADPT_FMT ": stainfo(" MAC_FMT ") not exist!\n",
+				FUNC_ADPT_ARG(a), MAC_ARG(stadel->macaddr));
+			a->disconnect_state = DISCON_ST_IDLE;
+			ret = RTW_PHL_STATUS_FAILURE;
+			goto exit;
+		}
+		_rtw_spinunlock_bh(&a->disconnect_lock);
+		rtw_wfd_st_switch(sta, 0);
+		_rtw_spinlock_bh(&a->disconnect_lock);
+		sta->hw_decrypted = _FALSE;
+	}
+
+	a->discon_cmd = pcmd;
+	cmd_req = &a->disconnect_req;
+	cmd_req->role = role;
+
+	phl_status = rtw_phl_add_cmd_token_req(GET_PHL_INFO(d),
+					       alink->wrlink->hw_band,
+					       cmd_req, &a->disconnect_token);
+	if ((phl_status != RTW_PHL_STATUS_SUCCESS) &&
+	    (phl_status != RTW_PHL_STATUS_PENDING)) {
+		RTW_WARN(FUNC_ADPT_FMT ": add_cmd_token_req fail(0x%x)!\n",
+			 FUNC_ADPT_ARG(a), phl_status);
+		a->disconnect_state = DISCON_ST_IDLE;
+		a->discon_cmd = NULL;
+		ret = RTW_PHL_STATUS_FAILURE;
+	}
+
+exit:
+	RTW_DBG(FUNC_ADPT_FMT ": - token=0x%08x state=%u ret=0x%x\n",
+		FUNC_ADPT_ARG(a), a->disconnect_token, a->disconnect_state, ret);
+	_rtw_spinunlock_bh(&a->disconnect_lock);
+
+	return ret;
+}
+
+/* Wait disconnect FG to finish */
+bool rtw_disconnect_wait_complete(struct _ADAPTER *a, u32 timeout)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_wifi_role_t *role = a->phl_role;
+	systime start;
+	u32 pass_t;
+	u32 msg_i;	/* message interval */
+	u32 msg_t = 0;	/* next time point to print message */
+	enum rtw_phl_status status;
+	bool terminated = false;
+
+
+	msg_i = timeout / 10;	/* at most print 10 msg in whole waiting time */
+	if (!msg_i)
+		msg_i = 1;
+	start = rtw_get_current_time();
+
+	do {
+		_rtw_spinlock_bh(&a->disconnect_lock);
+		pass_t = rtw_get_passing_time_ms(start);
+		if (a->disconnect_state == DISCON_ST_IDLE) {
+			_rtw_spinunlock_bh(&a->disconnect_lock);
+			terminated = true;
+			break;
+		}
+		_rtw_spinunlock_bh(&a->disconnect_lock);
+
+		if (pass_t > timeout) {
+			RTW_ERR(FUNC_ADPT_FMT ": Timeout! used:%u > max:%u ms\n",
+				FUNC_ADPT_ARG(a), pass_t, timeout);
+			break;
+		}
+		if (pass_t >= msg_t) {
+			RTW_DBG(FUNC_ADPT_FMT ": Waiting, %u ms/%u pass...\n",
+				FUNC_ADPT_ARG(a), pass_t, timeout);
+			msg_t += msg_i;
+		}
+
+		rtw_usleep_os(_rtw_systime_to_us(1)); /* sleep 1/HZ sec */
+	} while (1);
+
+	RTW_DBG(FUNC_ADPT_FMT ": - %s cost %u ms\n",
+		FUNC_ADPT_ARG(a), terminated?"OK":"Fail!", pass_t);
+
+	return terminated;
+}
+
+/**
+ * rtw_disconnect_abort() - Send cmd to abort disconnect FG
+ * @a		pointer of struct _ADAPTER
+ *
+ * Send cancel cmd to framework to stop disconnect FG.
+ *
+ * Return value could be one of below:
+ *	0	send abort command to CMD-DISP ok
+ *	1	abort is on-going, don't send abort command again
+ *	2	no disconnect FG is on-going, don't send abort command
+ *	negative value
+ *		status code returned from function rtw_phl_cancel_cmd_token()
+ *		and adding minus sign
+ */
+static int rtw_disconnect_abort(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	enum rtw_phl_status status;
+	int err = 0;
+
+
+	_rtw_spinlock_bh(&a->disconnect_lock);
+	/* avoid to race with _disconnect_cmd_done() */
+
+	RTW_DBG(FUNC_ADPT_FMT ": + token=0x%08x state=%u\n",
+		FUNC_ADPT_ARG(a), a->disconnect_token, a->disconnect_state);
+
+	switch (a->disconnect_state) {
+	case DISCON_ST_IDLE:
+		RTW_INFO(FUNC_ADPT_FMT ": no disconnect command.\n",
+			 FUNC_ADPT_ARG(a));
+		err = 2;
+		goto exit;
+	case DISCON_ST_ONGOING:
+		/* normal case*/
+		break;
+	case DISCON_ST_ABORT:
+		RTW_INFO(FUNC_ADPT_FMT ": disconnect abort is on-going...\n",
+			 FUNC_ADPT_ARG(a));
+		err = 1;
+		goto exit;
+	default:
+		RTW_ERR(FUNC_ADPT_FMT ": unexpected case, state=%u !\n",
+			FUNC_ADPT_ARG(a), a->disconnect_state);
+		err = -RTW_PHL_STATUS_UNEXPECTED_ERROR;
+		goto exit;
+	}
+
+	status = rtw_phl_cancel_cmd_token(GET_PHL_INFO(d),
+					  a->disconnect_bidx, &a->disconnect_token);
+	if (status == RTW_PHL_STATUS_SUCCESS) {
+		/* DISCON_ST_ONGOING -> DISCON_ST_ABORT */
+		a->disconnect_state = DISCON_ST_ABORT;
+	} else {
+		err = -status;
+		RTW_ERR(FUNC_ADPT_FMT ": cancel_cmd_token fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+	}
+
+exit:
+	RTW_DBG(FUNC_ADPT_FMT ": - err=%d state=%u\n",
+		FUNC_ADPT_ARG(a), err, a->disconnect_state);
+
+	_rtw_spinunlock_bh(&a->disconnect_lock);
+
+	return err;
+}
+
+#define _DISCONNECT_ABORT_TO_		3000	/* unit: ms */
+/**
+ * rtw_disconnect_abort_wait() - Abort disconnect FG and wait to finish
+ * @a		pointer of struct _ADAPTER
+ *
+ * Send cancel cmd to framework and wait disconnect FG to stop.
+ *
+ * Return 0 for disconnect FG already stopped, -1 for sending cancel command
+ * fail, or -2 for waiting disconnect FG stop timeout.
+ */
+int rtw_disconnect_abort_wait(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	int ret, err = 0;
+
+
+	ret = rtw_disconnect_abort(a);
+	if (ret < 0) {
+		/* Fail to send cancel cmd */
+		err = -1;
+	} else if (ret != 2) {
+		/*
+		 * ret == 0, send abort command success
+		 * ret == 1. abort on-going
+		 * ret == 2, disconnect done
+		 * besides case of ret == 2, wait disconnect to complete
+		 */
+		if (!rtw_disconnect_wait_complete(a, _DISCONNECT_ABORT_TO_))
+			err = -2; /* Timeout */
+	}
+
+	RTW_DBG(FUNC_ADPT_FMT ": - err=%d\n", FUNC_ADPT_ARG(a), err);
+	return err;
+}
+
+void rtw_disconnect_req_free(struct _ADAPTER *a)
+{
+	BUG_ON(rtw_disconnect_abort_wait(a) < 0);
+
+	_rtw_spinlock_free(&a->disconnect_lock);
+}
+
+void rtw_disconnect_req_init(struct _ADAPTER *a)
+{
+	struct phl_cmd_token_req *req;
+
+
+	_rtw_spinlock_init(&a->disconnect_lock);
+	a->disconnect_state = DISCON_ST_IDLE;
+
+	req = &a->disconnect_req;
+	req->module_id = PHL_FG_MDL_DISCONNECT;
+	req->opt = FG_CMD_OPT_EXCLUSIVE;
+	req->priv = a;
+	req->role = NULL;  /* a->phl_role, but role will change by time */
+	req->acquired = _disconnect_acquired;
+	req->abort = _disconnect_abort;
+	req->msg_hdlr = _disconnect_msg_hdlr;
+	req->set_info = _disconnect_set_info;
+	req->query_info = _disconnect_query_info;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mlme_ext.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mlme_ext.c
new file mode 100644
index 000000000000..9d7ef893f41a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mlme_ext.c
@@ -0,0 +1,13567 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_MLME_EXT_C_
+
+#include <drv_types.h>
+
+struct mlme_handler mlme_sta_tbl[] = {
+	{WIFI_ASSOCREQ,		"OnAssocReq",	&OnAssocReq},
+	{WIFI_ASSOCRSP,		"OnAssocRsp",	&OnAssocRsp},
+	{WIFI_REASSOCREQ,	"OnReAssocReq",	&OnAssocReq},
+	{WIFI_REASSOCRSP,	"OnReAssocRsp",	&OnAssocRsp},
+	{WIFI_PROBEREQ,		"OnProbeReq",	&OnProbeReq},
+	{WIFI_PROBERSP,		"OnProbeRsp",		&OnProbeRsp},
+
+	/*----------------------------------------------------------
+					below 2 are reserved
+	-----------------------------------------------------------*/
+	{0,					"DoReserved",		&DoReserved},
+	{0,					"DoReserved",		&DoReserved},
+	{WIFI_BEACON,		"OnBeacon",		&OnBeacon},
+	{WIFI_ATIM,			"OnATIM",		&OnAtim},
+	{WIFI_DISASSOC,		"OnDisassoc",		&OnDisassoc},
+	{WIFI_AUTH,			"OnAuth",		&OnAuthClient},
+	{WIFI_DEAUTH,		"OnDeAuth",		&OnDeAuth},
+	{WIFI_ACTION,		"OnAction",		&OnAction},
+	{WIFI_ACTION_NOACK, "OnActionNoAck",	&OnAction},
+};
+
+#ifdef _CONFIG_NATIVEAP_MLME_
+struct mlme_handler mlme_ap_tbl[] = {
+	{WIFI_ASSOCREQ,		"OnAssocReq",	&OnAssocReq},
+	{WIFI_ASSOCRSP,		"OnAssocRsp",	&OnAssocRsp},
+	{WIFI_REASSOCREQ,	"OnReAssocReq",	&OnAssocReq},
+	{WIFI_REASSOCRSP,	"OnReAssocRsp",	&OnAssocRsp},
+	{WIFI_PROBEREQ,		"OnProbeReq",	&OnProbeReq},
+	{WIFI_PROBERSP,		"OnProbeRsp",		&OnProbeRsp},
+
+	/*----------------------------------------------------------
+					below 2 are reserved
+	-----------------------------------------------------------*/
+	{0,					"DoReserved",		&DoReserved},
+	{0,					"DoReserved",		&DoReserved},
+	{WIFI_BEACON,		"OnBeacon",		&OnBeacon},
+	{WIFI_ATIM,			"OnATIM",		&OnAtim},
+	{WIFI_DISASSOC,		"OnDisassoc",		&OnDisassoc},
+	{WIFI_AUTH,			"OnAuth",		&OnAuth},
+	{WIFI_DEAUTH,		"OnDeAuth",		&OnDeAuth},
+	{WIFI_ACTION,		"OnAction",		&OnAction},
+	{WIFI_ACTION_NOACK, "OnActionNoAck",	&OnAction},
+};
+#endif
+
+struct action_handler OnAction_tbl[] = {
+	{RTW_WLAN_CATEGORY_SPECTRUM_MGMT,	 "ACTION_SPECTRUM_MGMT", on_action_spct},
+	{RTW_WLAN_CATEGORY_QOS, "ACTION_QOS", &OnAction_qos},
+	{RTW_WLAN_CATEGORY_DLS, "ACTION_DLS", &OnAction_dls},
+	{RTW_WLAN_CATEGORY_BACK, "ACTION_BACK", &OnAction_back},
+	{RTW_WLAN_CATEGORY_PUBLIC, "ACTION_PUBLIC", on_action_public},
+	{RTW_WLAN_CATEGORY_RADIO_MEAS, "ACTION_RADIO_MEAS", &on_action_rm},
+	{RTW_WLAN_CATEGORY_FT, "ACTION_FT",	&OnAction_ft},
+	{RTW_WLAN_CATEGORY_HT,	"ACTION_HT",	&OnAction_ht},
+#ifdef CONFIG_IEEE80211W
+	{RTW_WLAN_CATEGORY_SA_QUERY, "ACTION_SA_QUERY", &OnAction_sa_query},
+#else
+	{RTW_WLAN_CATEGORY_SA_QUERY, "ACTION_SA_QUERY", &DoReserved},
+#endif /* CONFIG_IEEE80211W */
+#ifdef CONFIG_RTW_WNM
+	{RTW_WLAN_CATEGORY_WNM, "ACTION_WNM", &on_action_wnm},
+#endif
+	{RTW_WLAN_CATEGORY_UNPROTECTED_WNM, "ACTION_UNPROTECTED_WNM", &DoReserved},
+#ifdef CONFIG_RTW_MESH
+	{RTW_WLAN_CATEGORY_MESH, "ACTION_MESH", &on_action_mesh},
+	{RTW_WLAN_CATEGORY_SELF_PROTECTED, "ACTION_SELF_PROTECTED", &on_action_self_protected},
+#endif
+	{RTW_WLAN_CATEGORY_WMM, "ACTION_WMM", &OnAction_wmm},
+	{RTW_WLAN_CATEGORY_VHT, "ACTION_VHT", &OnAction_vht},
+	{RTW_WLAN_CATEGORY_HE, "ACTION_HE", &OnAction_he},
+	{RTW_WLAN_CATEGORY_PROTECTED_HE, "ACTION_PROTECTED_HE", &OnAction_protected_he},
+	{RTW_WLAN_CATEGORY_P2P, "ACTION_P2P", &OnAction_p2p},
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	{RTW_WLAN_CATEGORY_TBTX, "ACTION_TBTX_TOKEN", &OnAction_tbtx_token},
+#endif
+	{RTW_WLAN_CATEGORY_UNP_S1G, "ACTION_S1G", &on_action_s1g}
+};
+
+
+u8	null_addr[ETH_ALEN] = {0, 0, 0, 0, 0, 0};
+
+/**************************************************
+OUI definitions for the vendor specific IE
+***************************************************/
+unsigned char	RTW_WPA_OUI[] = {0x00, 0x50, 0xf2, 0x01};
+unsigned char WMM_OUI[] = {0x00, 0x50, 0xf2, 0x02};
+unsigned char	WPS_OUI[] = {0x00, 0x50, 0xf2, 0x04};
+unsigned char	P2P_OUI[] = {0x50, 0x6F, 0x9A, 0x09};
+unsigned char	WFD_OUI[] = {0x50, 0x6F, 0x9A, 0x0A};
+unsigned char	DPP_OUI[] = {0x50, 0x6F, 0x9A, 0x1A};
+unsigned char	MULTI_AP_OUI[] = {0x50, 0x6F, 0x9A, 0x1B};
+
+unsigned char	WMM_INFO_OUI[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01};
+unsigned char	WMM_PARA_OUI[] = {0x00, 0x50, 0xf2, 0x02, 0x01, 0x01};
+
+unsigned char WPA_TKIP_CIPHER[4] = {0x00, 0x50, 0xf2, 0x02};
+unsigned char RSN_TKIP_CIPHER[4] = {0x00, 0x0f, 0xac, 0x02};
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+unsigned char REALTEK_TBTX_IE[] = {0x00, 0xe0, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x00};
+#endif
+extern unsigned char REALTEK_96B_IE[];
+
+void init_channel_list(_adapter *padapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct registry_priv *regsty = adapter_to_regsty(padapter);
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	struct p2p_channels *channel_list = &rfctl->channel_list;
+
+	struct p2p_oper_class_map op_class[] = {
+		{ IEEE80211G,  81,   1,  13,  1, BW20 },
+		{ IEEE80211G,  82,  14,  14,  1, BW20 },
+#if 0 /* Do not enable HT40 on 2 GHz */
+		{ IEEE80211G,  83,   1,   9,  1, BW40PLUS },
+		{ IEEE80211G,  84,   5,  13,  1, BW40MINUS },
+#endif
+		{ IEEE80211A, 115,  36,  48,  4, BW20 },
+		{ IEEE80211A, 116,  36,  44,  8, BW40PLUS },
+		{ IEEE80211A, 117,  40,  48,  8, BW40MINUS },
+		{ IEEE80211A, 124, 149, 161,  4, BW20 },
+		{ IEEE80211A, 125, 149, 169,  4, BW20 },
+		{ IEEE80211A, 126, 149, 157,  8, BW40PLUS },
+		{ IEEE80211A, 127, 153, 161,  8, BW40MINUS },
+		{ -1, 0, 0, 0, 0, BW20 }
+	};
+
+	int cla, op;
+
+	cla = 0;
+
+	for (op = 0; op_class[op].op_class; op++) {
+		u8 ch;
+		struct p2p_oper_class_map *o = &op_class[op];
+		struct p2p_reg_class *reg = NULL;
+
+		for (ch = o->min_chan; ch <= o->max_chan; ch += o->inc) {
+			if (rtw_chset_search_ch(chset, ch) == -1)
+				continue;
+#if defined(CONFIG_80211N_HT) || defined(CONFIG_80211AC_VHT) /* CONFIG_80211AX_HE_TODO */
+			if ((padapter->registrypriv.ht_enable == 0) && (o->inc == 8))
+				continue;
+
+			if ((REGSTY_IS_BW_5G_SUPPORT(regsty, CHANNEL_WIDTH_40)) &&
+			    ((o->bw == BW40MINUS) || (o->bw == BW40PLUS)))
+				continue;
+#endif
+			if (reg == NULL) {
+				reg = &channel_list->reg_class[cla];
+				cla++;
+				reg->reg_class = o->op_class;
+				reg->channels = 0;
+			}
+			reg->channel[reg->channels] = ch;
+			reg->channels++;
+		}
+	}
+	channel_list->reg_classes = cla;
+
+}
+
+static void rtw_rfctl_get_band_and_bw_bmp(struct dvobj_priv *dvobj, u8 *band_bmp, u8 bw_bmp_of_band[])
+{
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+
+	if (is_supported_24g(regsty->band_type) && rtw_hw_chk_band_cap(dvobj, BAND_CAP_2G))
+		*band_bmp |= BAND_CAP_2G;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (is_supported_5g(regsty->band_type) && rtw_hw_chk_band_cap(dvobj, BAND_CAP_5G))
+		*band_bmp |= BAND_CAP_5G;
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (is_supported_6g(regsty->band_type) && rtw_hw_chk_band_cap(dvobj, BAND_CAP_6G)
+		&& REGSTY_IS_11AX_ENABLE(regsty) && is_supported_he(regsty->wireless_mode))
+		*band_bmp |= BAND_CAP_6G;
+#endif
+
+	bw_bmp_of_band[BAND_ON_24G] = (ch_width_to_bw_cap(REGSTY_BW_2G(regsty) + 1) - 1) & (dvobj->phl_com->dev_cap.bw_sup);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	bw_bmp_of_band[BAND_ON_5G] = (ch_width_to_bw_cap(REGSTY_BW_5G(regsty) + 1) - 1) & (dvobj->phl_com->dev_cap.bw_sup);
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	bw_bmp_of_band[BAND_ON_6G] = (ch_width_to_bw_cap(REGSTY_BW_6G(regsty) + 1) - 1) & (dvobj->phl_com->dev_cap.bw_sup);
+#endif
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (!REGSTY_IS_11AC_ENABLE(regsty)
+		|| !is_supported_vht(regsty->wireless_mode)
+	)
+		bw_bmp_of_band[BAND_ON_5G] &= ~(BW_CAP_80M | BW_CAP_160M);
+#endif
+
+	if (0) {
+		RTW_INFO("REGSTY_BW_2G(regsty):%u\n", REGSTY_BW_2G(regsty));
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		RTW_INFO("REGSTY_BW_5G(regsty):%u\n", REGSTY_BW_5G(regsty));
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		RTW_INFO("REGSTY_BW_6G(regsty):%u\n", REGSTY_BW_6G(regsty));
+		#endif
+		RTW_INFO("dev_cap(adapter)->bw_sup:0x%x\n", dvobj->phl_com->dev_cap.bw_sup);
+		RTW_INFO("band_bmp:0x%x\n", *band_bmp);
+		RTW_INFO("bw_bmp[2G]:0x%x\n", bw_bmp_of_band[BAND_ON_24G]);
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		RTW_INFO("bw_bmp[5G]:0x%x\n", bw_bmp_of_band[BAND_ON_5G]);
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		RTW_INFO("bw_bmp[6G]:0x%x\n", bw_bmp_of_band[BAND_ON_6G]);
+		#endif
+	}
+}
+
+int rtw_rfctl_init(struct dvobj_priv *dvobj)
+{
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+	u8 band_bmp = 0;
+	u8 bw_bmp_of_band[BAND_MAX] = {0};
+	int ret;
+
+	rtw_rfctl_get_band_and_bw_bmp(dvobj, &band_bmp, bw_bmp_of_band);
+
+	rtw_rfctl_addl_ch_disable_conf_init(rfctl, regsty);
+
+	rtw_regd_req_list_init(rfctl, regsty);
+
+	rfctl->bcn_hint_valid_ms = regsty->bcn_hint_valid_ms;
+
+#ifdef CONFIG_80211D
+	rtw_rfctl_cis_init(rfctl, regsty);
+#endif
+
+	rfctl->edcca_mode_sel = regsty->edcca_mode_sel;
+	rfctl->last_edcca_mode_op_band = BAND_MAX;
+
+	rtw_rfctl_dfs_init(rfctl, regsty);
+
+	rtw_chset_init(&rfctl->chset, band_bmp);
+
+	ret = rtw_rfctl_op_class_pref_init(rfctl, band_bmp, bw_bmp_of_band);
+
+#if defined(PRIVATE_R) && defined(CONFIG_ECSA_PHL)
+	rfctl->p2p_park_ch = 36; /* default set to CH36*/
+#endif
+
+	return ret;
+}
+
+void rtw_rfctl_deinit(struct dvobj_priv *dvobj)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+
+	_rtw_mutex_free(&rfctl->offch_mutex);
+
+	rtw_regd_req_list_free(rfctl);
+
+#ifdef CONFIG_80211D
+	rtw_rfctl_cis_deinit(rfctl);
+#endif
+
+	rtw_rfctl_op_class_pref_deinit(rfctl);
+}
+
+void rtw_rfctl_update_op_mode(struct rf_ctl_t *rfctl, u8 ifbmp_mod, u8 if_op, u8 ifbmp_excl)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+#ifdef CONFIG_MCC_SUPPORT
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+#endif
+	_adapter *iface;
+	u8 op_class = 0;
+	u8 op_ch = 0;
+	s16 op_txpwr_max;
+	u8 if_op_class[CONFIG_IFACE_NUMBER] = {0};
+	u8 if_op_ch[CONFIG_IFACE_NUMBER] = {0};
+	enum band_type band, u_band = BAND_MAX;
+	u8 ch, bw, offset;
+	u8 u_ch = 0;
+	u8 u_bw = CHANNEL_WIDTH_MAX;
+	u8 u_offset = CHAN_OFFSET_NO_DEF;
+	bool notify = 0;
+	int i;
+	struct _ADAPTER_LINK *iface_link;
+	struct link_mlme_ext_priv *mlmeext;
+	bool need_sync_bchbw = true;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface)
+			continue;
+
+		if (ifbmp_excl & BIT(i))
+			continue;
+		if (ifbmp_mod & BIT(i)) {
+			if (!if_op)
+				continue;
+		} else if (!MLME_IS_ASOC(iface) || MLME_IS_OPCH_SW(iface))
+			continue;
+
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		iface_link = GET_PRIMARY_LINK(iface);
+		mlmeext = &iface_link->mlmeextpriv;
+		band = mlmeext->chandef.band;
+		ch = mlmeext->chandef.chan;
+		bw = mlmeext->chandef.bw;
+		offset = mlmeext->chandef.offset;
+		if_op_class[i] = rtw_get_op_class_by_bchbw(band, ch, bw, offset);
+		if_op_ch[i] = if_op_class[i] ? ch : 0;
+
+		if (!u_ch) {
+			u_band = band;
+			u_ch = ch;
+			u_bw = bw;
+			u_offset = offset;
+		} else {
+#ifdef CONFIG_MCC_SUPPORT
+			if (phl_com->dev_sw_cap.mcc_sup == _TRUE) {
+				int chanctx_num = rtw_phl_mr_get_chanctx_num(dvobj->phl, iface->phl_role, iface_link->wrlink);
+
+				if (chanctx_num >= 2)
+					need_sync_bchbw = false;
+			}
+#endif
+#ifdef CONFIG_DBCC_SUPPORT
+			if (need_sync_bchbw) {
+				if (rtw_phl_mr_is_db(dvobj->phl))
+					need_sync_bchbw = false;
+			}
+#endif
+			if (need_sync_bchbw) {
+				rtw_warn_on(!rtw_is_bchbw_grouped(u_band, u_ch, u_bw, u_offset, band, ch, bw, offset));
+				rtw_sync_bchbw(&band, &ch, &bw, &offset, &u_band, &u_ch, &u_bw, &u_offset);
+			}
+		}
+	}
+
+	op_class = rtw_get_op_class_by_bchbw(u_band, u_ch, u_bw, u_offset);
+	op_ch = op_class ? u_ch : 0;
+	op_txpwr_max = rtw_rfctl_get_oper_txpwr_max_mbm(rfctl, u_ch, u_bw, u_offset, ifbmp_mod, if_op, 1);
+
+	if (op_class != rfctl->op_class
+		|| op_ch != rfctl->op_ch
+		|| op_txpwr_max != rfctl->op_txpwr_max
+		|| _rtw_memcmp(if_op_class, rfctl->if_op_class, sizeof(u8) * CONFIG_IFACE_NUMBER) == _FALSE
+		|| _rtw_memcmp(if_op_ch, rfctl->if_op_ch, sizeof(u8) * CONFIG_IFACE_NUMBER) == _FALSE)
+		notify = 1;
+
+	rfctl->op_class = op_class;
+	rfctl->op_ch = op_ch;
+	rfctl->op_txpwr_max = op_txpwr_max;
+	_rtw_memcpy(rfctl->if_op_class, if_op_class, sizeof(u8) * CONFIG_IFACE_NUMBER);
+	_rtw_memcpy(rfctl->if_op_ch, if_op_ch, sizeof(u8) * CONFIG_IFACE_NUMBER);
+
+	if (0)
+		RTW_INFO("radio: %u,%u,%u,%u %d notify:%d\n", u_band, u_ch, u_bw, u_offset, op_txpwr_max, notify);
+
+	if (0) {
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if (!iface)
+				continue;
+			/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+			iface_link = GET_PRIMARY_LINK(iface);
+			mlmeext = &iface_link->mlmeextpriv;
+
+			if (ifbmp_excl & BIT(i))
+				continue;
+			if (ifbmp_mod & BIT(i)) {
+				if (!if_op)
+					continue;
+			} else if (!MLME_IS_ASOC(iface) || MLME_IS_OPCH_SW(iface))
+				continue;
+
+			RTW_INFO(ADPT_FMT": %u,%u,%u,%u\n", ADPT_ARG(iface)
+				, mlmeext->chandef.band, mlmeext->chandef.chan, mlmeext->chandef.bw, mlmeext->chandef.offset);
+		}
+	}
+
+	if (notify)
+		rtw_nlrtw_radio_opmode_notify(rfctl);
+}
+
+/*
+ * Check the @param ch is fit with setband setting of @param adapter
+ * @adapter: the given adapter
+ * @ch: the given channel number
+ *
+ * return _TRUE when check valid, _FALSE not valid
+ */
+bool rtw_mlme_band_check(_adapter *adapter, const u32 ch)
+{
+	if (adapter->setband == WIFI_FREQUENCY_BAND_AUTO /* 2.4G and 5G */
+		|| (adapter->setband == WIFI_FREQUENCY_BAND_2GHZ && ch < 35) /* 2.4G only */
+		|| (adapter->setband == WIFI_FREQUENCY_BAND_5GHZ && ch > 35) /* 5G only */
+	)
+		return _TRUE;
+	return _FALSE;
+}
+inline void RTW_SET_SCAN_BAND_SKIP(_adapter *padapter, int skip_band)
+{
+	int bs = ATOMIC_READ(&padapter->bandskip);
+
+	bs |= skip_band;
+	ATOMIC_SET(&padapter->bandskip, bs);
+}
+
+inline void RTW_CLR_SCAN_BAND_SKIP(_adapter *padapter, int skip_band)
+{
+	int bs = ATOMIC_READ(&padapter->bandskip);
+
+	bs &= ~(skip_band);
+	ATOMIC_SET(&padapter->bandskip, bs);
+}
+inline int RTW_GET_SCAN_BAND_SKIP(_adapter *padapter)
+{
+	return ATOMIC_READ(&padapter->bandskip);
+}
+
+#define RTW_IS_SCAN_BAND_SKIP(padapter, skip_band) (ATOMIC_READ(&padapter->bandskip) & (skip_band))
+
+bool rtw_mlme_ignore_chan(_adapter *adapter, const u32 ch)
+{
+	if (RTW_IS_SCAN_BAND_SKIP(adapter, BAND_24G) && ch < 35) /* SKIP 2.4G Band channel */
+		return _TRUE;
+	if (RTW_IS_SCAN_BAND_SKIP(adapter, BAND_5G)  && ch > 35) /* SKIP 5G Band channel */
+		return _TRUE;
+
+	return _FALSE;
+}
+
+
+/****************************************************************************
+
+Following are the initialization functions for WiFi MLME
+
+*****************************************************************************/
+static void init_mlme_ext_priv_value(_adapter *padapter)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	ATOMIC_SET(&pmlmeext->event_seq, 0);
+	pmlmeext->mgnt_seq = 0;/* reset to zero when disconnect at client mode */
+#ifdef CONFIG_IEEE80211W
+	pmlmeext->sa_query_seq = 0;
+#endif
+	pmlmeext->retry = 0;
+
+	pmlmeext->dev_cur_wireless_mode = padapter->registrypriv.wireless_mode;
+
+	mlmeext_set_scan_state(pmlmeext, SCAN_DISABLE);
+	#if 0
+	pmlmeext->sitesurvey_res.channel_idx = 0;
+	#endif
+	pmlmeext->sitesurvey_res.bss_cnt = 0;
+
+	_rtw_spinlock_init(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+	pmlmeext->sitesurvey_res.scan_ch_ms = SURVEY_TO;
+	pmlmeext->sitesurvey_res.rx_ampdu_accept = RX_AMPDU_ACCEPT_INVALID;
+	pmlmeext->sitesurvey_res.rx_ampdu_size = RX_AMPDU_SIZE_INVALID;
+#ifdef CONFIG_SCAN_BACKOP
+#ifdef CONFIG_LAYER2_ROAMING
+	if (padapter->mlmepriv.roam_flags & RTW_ROAM_ACTIVE)
+		mlmeext_assign_scan_backop_flags_sta(pmlmeext, SS_BACKOP_EN | SS_BACKOP_PS_ANNC | SS_BACKOP_TX_RESUME);
+	else
+#endif
+	mlmeext_assign_scan_backop_flags_sta(pmlmeext, /*SS_BACKOP_EN|*/SS_BACKOP_PS_ANNC | SS_BACKOP_TX_RESUME);
+
+	#ifdef CONFIG_SCAN_BACKOP_STA
+	mlmeext_assign_scan_backop_flags_sta(pmlmeext, SS_BACKOP_EN | SS_BACKOP_PS_ANNC | SS_BACKOP_TX_RESUME);
+	#endif /*CONFIG_SCAN_BACKOP_STA*/
+
+	#ifdef CONFIG_AP_MODE
+	mlmeext_assign_scan_backop_flags_ap(pmlmeext, SS_BACKOP_EN | SS_BACKOP_PS_ANNC | SS_BACKOP_TX_RESUME);
+	#endif
+	#ifdef CONFIG_RTW_MESH
+	mlmeext_assign_scan_backop_flags_mesh(pmlmeext, /*SS_BACKOP_EN | */SS_BACKOP_PS_ANNC | SS_BACKOP_TX_RESUME);
+	#endif
+	#if 0
+	pmlmeext->sitesurvey_res.scan_cnt = 0;
+	#endif
+	pmlmeext->sitesurvey_res.scan_cnt_max = RTW_SCAN_NUM_OF_CH;
+	pmlmeext->sitesurvey_res.backop_ms = RTW_BACK_OP_CH_MS;
+#endif
+#if defined(CONFIG_ANTENNA_DIVERSITY) || defined(DBG_SCAN_SW_ANTDIV_BL)
+	pmlmeext->sitesurvey_res.is_sw_antdiv_bl_scan = 0;
+#endif
+	pmlmeext->scan_abort = _FALSE;
+	pmlmeext->scan_abort_to = _FALSE;
+
+	pmlmeinfo->state = WIFI_FW_NULL_STATE;
+	pmlmeinfo->reauth_count = 0;
+	pmlmeinfo->reassoc_count = 0;
+	pmlmeinfo->link_count = 0;
+	pmlmeinfo->auth_seq = 0;
+	pmlmeinfo->auth_algo = dot11AuthAlgrthm_Open;
+	pmlmeinfo->key_index = 0;
+	pmlmeinfo->iv = 0;
+	pmlmeinfo->enc_algo = _NO_PRIVACY_;
+	pmlmeinfo->authModeToggle = 0;
+
+	_rtw_memset(pmlmeinfo->chg_txt, 0, 128);
+
+
+	pmlmeinfo->dialogToken = 0;
+
+	pmlmeext->action_public_rxseq = 0xffff;
+	pmlmeext->action_public_dialog_token = 0xff;
+}
+
+void init_mlme_ext_timer(_adapter *padapter)
+{
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+
+#if 0
+	rtw_init_timer(&pmlmeext->survey_timer, survey_timer_hdl, padapter);
+#endif
+	rtw_init_timer(&pmlmeext->link_timer, link_timer_hdl, padapter);
+#ifdef CONFIG_RTW_80211R
+	rtw_init_timer(&pmlmeext->ft_link_timer, rtw_ft_link_timer_hdl, padapter);
+	rtw_init_timer(&pmlmeext->ft_roam_timer, rtw_ft_roam_timer_hdl, padapter);
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	rtw_init_timer(&pmlmeext->tbtx_xmit_timer, rtw_tbtx_xmit_timer_hdl, padapter);
+	rtw_init_timer(&pmlmeext->tbtx_token_dispatch_timer, rtw_tbtx_token_dispatch_timer_hdl, padapter);
+#endif
+#ifdef PRIVATE_R
+	rtw_init_timer(&pmlmeext->find_remote_timer, find_remote_timer_hdl, padapter);
+#ifdef CONFIG_P2P
+	rtw_init_timer(&pmlmeext->hide_ssid_timer, hide_ssid_hdl, padapter);
+#endif
+#endif
+#ifdef CONFIG_ECSA_PHL
+	rtw_init_timer(&pmlmeext->csa_wait_bcn_timer, csa_wait_bcn_timer_hdl, padapter);
+#endif
+}
+
+int	init_mlme_ext_priv(_adapter *padapter)
+{
+	int	res = _SUCCESS;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	/* We don't need to memset padapter->XXX to zero, because adapter is allocated by rtw_zvmalloc(). */
+	/* _rtw_memset((u8 *)pmlmeext, 0, sizeof(struct mlme_ext_priv)); */
+
+	pmlmeext->padapter = padapter;
+
+	init_mlme_ext_priv_value(padapter);
+	pmlmeinfo->bAcceptAddbaReq = pregistrypriv->bAcceptAddbaReq;
+
+	init_mlme_ext_timer(padapter);
+
+#ifdef CONFIG_AP_MODE
+	init_mlme_ap_info(padapter);
+#endif
+
+	pmlmeext->last_scan_time = 0;
+
+
+#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+	pmlmeext->active_keep_alive_check = _TRUE;
+#else
+	pmlmeext->active_keep_alive_check = _FALSE;
+#endif
+
+#ifdef DBG_FIXED_CHAN
+	pmlmeext->fixed_chan = 0xFF;
+#endif
+
+#ifdef CONFIG_SUPPORT_STATIC_SMPS
+	pmlmeext->ssmps_en = _FALSE;
+	pmlmeext->ssmps_tx_tp_th = SSMPS_TX_TP_TH;/*Mbps*/
+	pmlmeext->ssmps_rx_tp_th = SSMPS_RX_TP_TH;/*Mbps*/
+	#ifdef DBG_STATIC_SMPS
+	pmlmeext->ssmps_test = _FALSE;
+	#endif
+#endif
+
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	pmlmeext->txss_ctrl_en = _TRUE;
+	pmlmeext->txss_tp_th = TXSS_TP_TH;
+	pmlmeext->txss_tp_chk_cnt = TXSS_TP_CHK_CNT;
+#endif
+
+	pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+	pmlmeinfo->disconnect_code = DISCONNECTION_NOT_YET_OCCUR;
+	pmlmeinfo->illegal_beacon_code = 0;
+	pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+
+	return res;
+
+}
+
+void free_mlme_ext_priv(struct mlme_ext_priv *pmlmeext)
+{
+	_adapter *padapter = pmlmeext->padapter;
+
+	if (!padapter)
+		return;
+
+	_rtw_spinlock_free(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+	if (dev_is_drv_stopped(adapter_to_dvobj(padapter))) {
+		/*cancel_survey_timer(pmlmeext);*/ /*_cancel_timer_ex(&pmlmeext->survey_timer);*/
+		cancel_link_timer(pmlmeext); /*_cancel_timer_ex(&pmlmeext->link_timer);*/
+#ifdef PRIVATE_R
+		_cancel_timer_ex(&pmlmeext->find_remote_timer);
+#ifdef CONFIG_P2P
+		_cancel_timer_ex(&pmlmeext->hide_ssid_timer);
+#endif
+#endif
+#ifdef CONFIG_ECSA_PHL
+		_cancel_timer_ex(&pmlmeext->csa_wait_bcn_timer);
+#endif
+	}
+}
+
+void init_link_mlme_default_rate_set(struct _ADAPTER_LINK *padapter_link)
+{
+	_adapter *padapter = padapter_link->adapter;
+	struct	link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	unsigned	char end_set[1] = {0xff};
+	u8	offset_datarate = 0;
+	u8	offset_basicrate = 0;
+#ifdef CONFIG_80211N_HT
+	unsigned char	supported_mcs_set[16] = {0xff, 0xff, 0xff, 0xff, 0x00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
+#endif
+
+	if (is_supported_tx_cck(padapter->registrypriv.wireless_mode)) {
+
+		unsigned char	datarate_b[B_MODE_RATE_NUM] ={_1M_RATE_, _2M_RATE_, _5M_RATE_, _11M_RATE_};
+		_rtw_memcpy(pmlmeext->datarate, datarate_b, B_MODE_RATE_NUM);
+		_rtw_memcpy(pmlmeext->basicrate, datarate_b, B_MODE_RATE_NUM);
+		offset_datarate += B_MODE_RATE_NUM;
+		offset_basicrate += B_MODE_RATE_NUM;
+		RTW_INFO("%s: support CCK\n", __func__);
+	}
+	if(is_suuported_tx_ofdm(padapter->registrypriv.wireless_mode)) {
+		unsigned char	datarate_g[G_MODE_RATE_NUM] ={_6M_RATE_, _9M_RATE_, _12M_RATE_, _18M_RATE_,_24M_RATE_, _36M_RATE_, _48M_RATE_, _54M_RATE_};
+		unsigned char	basicrate_g[G_MODE_BASIC_RATE_NUM] = {_6M_RATE_, _12M_RATE_, _24M_RATE_};
+		_rtw_memcpy(pmlmeext->datarate + offset_datarate, datarate_g, G_MODE_RATE_NUM);
+		_rtw_memcpy(pmlmeext->basicrate + offset_basicrate,basicrate_g, G_MODE_BASIC_RATE_NUM);
+		offset_datarate += G_MODE_RATE_NUM;
+		offset_basicrate += G_MODE_BASIC_RATE_NUM;
+		RTW_INFO("%s: support OFDM\n", __func__);
+
+	}
+	_rtw_memcpy(pmlmeext->datarate + offset_datarate, end_set, 1);
+	_rtw_memcpy(pmlmeext->basicrate + offset_basicrate, end_set, 1);
+
+#ifdef CONFIG_80211N_HT
+	if (padapter->registrypriv.ht_enable && is_supported_ht(padapter->registrypriv.wireless_mode))
+		_rtw_memcpy(pmlmeext->default_supported_mcs_set, supported_mcs_set, sizeof(pmlmeext->default_supported_mcs_set));
+#endif
+}
+
+int	rtw_init_link_mlme_ext_priv(struct _ADAPTER_LINK *padapter_link)
+{
+	_adapter *padapter = padapter_link->adapter;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	int	res = _SUCCESS;
+
+#ifdef CONFIG_AP_MODE /* moved from init_mlme_ap_info() in rtw_init_mlme_ext_priv() */
+	_rtw_spinlock_init(&pmlmepriv->bcn_update_lock);
+#endif
+
+	/* moved from init_mlme_ext_priv_value() */
+	pmlmeext->chandef.band = pregistrypriv->band;
+	pmlmeext->chandef.chan = pregistrypriv->channel;
+	pmlmeext->chandef.bw = CHANNEL_WIDTH_20;
+	pmlmeext->chandef.offset = CHAN_OFFSET_NO_EXT;
+
+	pmlmeext->cur_wireless_mode = pregistrypriv->wireless_mode;
+	init_link_mlme_default_rate_set(padapter_link);
+
+	if ((pmlmeext->chandef.band != BAND_ON_24G) || ((padapter->registrypriv.wireless_mode & WLAN_MD_11B) == 0))
+		pmlmeext->tx_rate = IEEE80211_OFDM_RATE_6MB;
+	else
+		pmlmeext->tx_rate = IEEE80211_CCK_RATE_1MB;
+
+	pmlmeinfo->slotTime = SHORT_SLOT_TIME;
+	pmlmeinfo->preamble_mode = PREAMBLE_AUTO;
+
+#ifdef PRIVATE_R
+/*infra mode, used to store AP's info*/
+	_rtw_memset(pmlmeinfo->SupportedRates_infra_ap, 0, NDIS_802_11_LENGTH_RATES_EX);
+	pmlmeinfo->ht_vht_received = 0;
+#endif /* PRIVATE_R */
+
+	return res;
+}
+
+int rtw_match_self_addr(_adapter *padapter, const u8 *mac_addr)
+{
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+	/* ToDo CONFIG_RTW_MLD: TBD To see if MAC hdr conversion is enabled */
+	/*if (_rtw_memcmp(mac_addr, adapter_mac_addr(padapter), ETH_ALEN))
+		return _TRUE;
+	*/
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		if (_rtw_memcmp(mac_addr, padapter_link->mac_addr, ETH_ALEN))
+			return _TRUE;
+	}
+	return _FALSE;
+}
+
+int rtw_match_bssid_ext(_adapter *padapter, const u8 *mac_addr) /* mlmext_info->network */
+{
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+	/* ToDo CONFIG_RTW_MLD: TBD To see if MAC hdr conversion is enabled */
+	/*if (_rtw_memcmp(mac_addr, get_my_bssid(&padapter->mlmeextpriv.mlmext_info.dev_network), ETH_ALEN))
+		return _TRUE;*/
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+		if (_rtw_memcmp(mac_addr, get_my_bssid(&padapter_link->mlmeextpriv.mlmext_info.network), ETH_ALEN - 1)) {
+			if (padapter_link->mlmeextpriv.mlmext_info.network.is_mbssid)
+				return _TRUE;
+		}
+#endif
+		if (_rtw_memcmp(mac_addr, get_my_bssid(&padapter_link->mlmeextpriv.mlmext_info.network), ETH_ALEN))
+			return _TRUE;
+
+	}
+	return _FALSE;
+}
+
+int rtw_match_bssid(_adapter *padapter, const u8 *mac_addr) /* mlmepriv->cur_network.network */
+{
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+	/* ToDo CONFIG_RTW_MLD: TBD To see if MAC hdr conversion is enabled */
+	/*if (_rtw_memcmp(mac_addr, get_bssid(&padapter->mlmepriv), ETH_ALEN))
+		return _TRUE;*/
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		/* ToDo CONFIG_RTW_MLD: check wrlink->mstate linked or not */
+		if (_rtw_memcmp(mac_addr, get_link_bssid(&padapter_link->mlmepriv), ETH_ALEN))
+			return _TRUE;
+	}
+	return _FALSE;
+}
+
+static void _mgt_dispatcher(_adapter *padapter, struct mlme_handler *ptable, union recv_frame *precv_frame)
+{
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+
+	if (ptable->func) {
+		/* receive the frames that ra(a1) is my address or ra(a1) is bc address. */
+		if (!rtw_match_self_addr(padapter, GetAddr1Ptr(pframe)) &&
+			!_rtw_memcmp(GetAddr1Ptr(pframe), bc_addr, ETH_ALEN))
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+		{
+			struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+
+			if (!MLME_IS_STA(padapter))
+				return;
+
+			if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _TRUE)
+				return;
+
+			if (pwdev_priv->random_mac_enabled == false)
+				return;
+
+			if (!_rtw_memcmp(GetAddr1Ptr(pframe), adapter_pno_mac_addr(padapter), ETH_ALEN))
+				return;
+		}
+#else
+			return;
+#endif
+		ptable->func(padapter, precv_frame);
+	}
+}
+
+void mgt_dispatcher(_adapter *padapter, union recv_frame *precv_frame)
+{
+	int index;
+	struct mlme_handler *ptable;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	struct sta_info *psta = rtw_get_stainfo(&padapter->stapriv, get_addr2_ptr(pframe));
+
+#if 0
+	{
+		u8 *pbuf;
+		pbuf = GetAddr1Ptr(pframe);
+		RTW_INFO("A1-%x:%x:%x:%x:%x:%x\n", *pbuf, *(pbuf + 1), *(pbuf + 2), *(pbuf + 3), *(pbuf + 4), *(pbuf + 5));
+		pbuf = get_addr2_ptr(pframe);
+		RTW_INFO("A2-%x:%x:%x:%x:%x:%x\n", *pbuf, *(pbuf + 1), *(pbuf + 2), *(pbuf + 3), *(pbuf + 4), *(pbuf + 5));
+		pbuf = GetAddr3Ptr(pframe);
+		RTW_INFO("A3-%x:%x:%x:%x:%x:%x\n", *pbuf, *(pbuf + 1), *(pbuf + 2), *(pbuf + 3), *(pbuf + 4), *(pbuf + 5));
+	}
+#endif
+
+	if (GetFrameType(pframe) != WIFI_MGT_TYPE) {
+		return;
+	}
+
+	/* receive the frames that ra(a1) is my address or ra(a1) is bc address. */
+	if (!rtw_match_self_addr(padapter, GetAddr1Ptr(pframe)) &&
+		!_rtw_memcmp(GetAddr1Ptr(pframe), bc_addr, ETH_ALEN))
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+		{
+			struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+
+			if (!MLME_IS_STA(padapter))
+				return;
+
+			if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _TRUE)
+				return;
+
+			if (pwdev_priv->random_mac_enabled == false)
+				return;
+
+			if (!_rtw_memcmp(GetAddr1Ptr(pframe), adapter_pno_mac_addr(padapter), ETH_ALEN))
+				return;
+		}
+#else
+		return;
+#endif
+
+	ptable = mlme_sta_tbl;
+
+	index = get_frame_sub_type(pframe) >> 4;
+
+#ifdef CONFIG_TDLS
+	if ((index << 4) == WIFI_ACTION) {
+		/* category==public (4), action==TDLS_DISCOVERY_RESPONSE */
+		if (*(pframe + 24) == RTW_WLAN_CATEGORY_PUBLIC && *(pframe + 25) == TDLS_DISCOVERY_RESPONSE) {
+			RTW_INFO("[TDLS] Recv %s from "MAC_FMT"\n", rtw_tdls_action_txt(TDLS_DISCOVERY_RESPONSE), MAC_ARG(get_addr2_ptr(pframe)));
+			On_TDLS_Dis_Rsp(padapter, precv_frame);
+		}
+	}
+#endif /* CONFIG_TDLS */
+
+	if (index >= (sizeof(mlme_sta_tbl) / sizeof(struct mlme_handler))) {
+		return;
+	}
+	ptable += index;
+
+#if 1
+	if (psta != NULL) {
+		if (GetRetry(pframe)) {
+			if (precv_frame->u.hdr.attrib.seq_num == psta->RxMgmtFrameSeqNum) {
+				/* drop the duplicate management frame */
+				padapter->recvinfo.dbg_rx_dup_mgt_frame_drop_count++;
+				RTW_INFO("Drop duplicate management frame with seq_num = %d.\n", precv_frame->u.hdr.attrib.seq_num);
+				return;
+			}
+		}
+		psta->RxMgmtFrameSeqNum = precv_frame->u.hdr.attrib.seq_num;
+	}
+#else
+
+	if (GetRetry(pframe)) {
+		/* return; */
+	}
+#endif
+
+#ifdef CONFIG_AP_MODE
+	switch (get_frame_sub_type(pframe)) {
+	case WIFI_AUTH:
+		if (MLME_IS_AP(padapter) || MLME_IS_MESH(padapter))
+			ptable->func = &OnAuth;
+		else
+			ptable->func = &OnAuthClient;
+		fallthrough;
+	case WIFI_ASSOCREQ:
+	case WIFI_REASSOCREQ:
+		_mgt_dispatcher(padapter, ptable, precv_frame);
+		#ifdef CONFIG_HOSTAPD_MLME
+		if (MLME_IS_AP(padapter))
+			rtw_hostapd_mlme_rx(padapter, precv_frame);
+		#endif
+		break;
+	case WIFI_PROBEREQ:
+		_mgt_dispatcher(padapter, ptable, precv_frame);
+		#ifdef CONFIG_HOSTAPD_MLME
+		if (MLME_IS_AP(padapter))
+			rtw_hostapd_mlme_rx(padapter, precv_frame);
+		#endif
+		break;
+	case WIFI_BEACON:
+		_mgt_dispatcher(padapter, ptable, precv_frame);
+		break;
+	case WIFI_ACTION:
+		_mgt_dispatcher(padapter, ptable, precv_frame);
+		break;
+	default:
+		_mgt_dispatcher(padapter, ptable, precv_frame);
+		#ifdef CONFIG_HOSTAPD_MLME
+		if (MLME_IS_AP(padapter))
+			rtw_hostapd_mlme_rx(padapter, precv_frame);
+		#endif
+		break;
+	}
+#else
+
+	_mgt_dispatcher(padapter, ptable, precv_frame);
+
+#endif
+
+}
+
+#ifdef CONFIG_P2P
+u32 p2p_listen_state_process(_adapter *padapter, unsigned char *da)
+{
+	bool response = _TRUE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (rtw_cfg80211_get_is_roch(padapter) == _FALSE
+		|| rtw_get_oper_ch(padapter, padapter_link) != padapter->wdinfo.listen_channel
+		|| adapter_wdev_data(padapter)->p2p_enabled == _FALSE
+		|| padapter->mlmepriv.wps_probe_resp_ie == NULL
+		|| padapter->mlmepriv.p2p_probe_resp_ie == NULL
+	) {
+#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO(ADPT_FMT" DON'T issue_probersp_p2p: p2p_enabled:%d, wps_probe_resp_ie:%p, p2p_probe_resp_ie:%p\n"
+			, ADPT_ARG(padapter)
+			, adapter_wdev_data(padapter)->p2p_enabled
+			, padapter->mlmepriv.wps_probe_resp_ie
+			, padapter->mlmepriv.p2p_probe_resp_ie);
+		RTW_INFO(ADPT_FMT" DON'T issue_probersp_p2p: is_ro_ch:%d, op_ch:%d, p2p_listen_channel:%d\n"
+			, ADPT_ARG(padapter)
+			, rtw_cfg80211_get_is_roch(padapter)
+			, rtw_get_oper_ch(padapter, padapter_link)
+			, padapter->wdinfo.listen_channel);
+#endif
+		response = _FALSE;
+	}
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	if (response == _TRUE)
+		issue_probersp_p2p(padapter, da);
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_P2P */
+
+#ifdef PRIVATE_R
+static bool check_vendor_ie(_adapter *padapter, u8 *ies, uint ies_len)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	unsigned int ielen;
+	u8 *p;
+	bool ret = _FALSE;
+	unsigned int search_len = 0, remain_len = 0, total_vendor_ie_len = 0;
+	int i;
+
+	p = rtw_get_ie(ies, _VENDOR_SPECIFIC_IE_, (int *)&ielen, ies_len);
+
+	while (p) {
+		for (i = 0; i < MAX_VENDOR_IE_NUM; i++) {
+			if (ielen == pmlmepriv->vendor_ie_len[i] &&
+			    _rtw_memcmp(p, *(pmlmepriv->vendor_ie_filter + i), ielen + 2)) {
+				ret = _TRUE;
+				break;
+			}
+		}
+		total_vendor_ie_len = ielen + 2;
+		search_len = p - ies;
+		remain_len = ies_len - search_len - total_vendor_ie_len;
+
+		p = rtw_get_ie(p + total_vendor_ie_len, _VENDOR_SPECIFIC_IE_,
+			       (int *)&ielen, remain_len);
+	}
+	return ret;
+}
+#ifdef CONFIG_P2P
+static bool is_valid_probereq(_adapter *padapter, u8 *pframe, uint len)
+{
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *cur = &(pmlmeinfo->dev_network);
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	bool ret = _FALSE;
+	unsigned int ielen;
+	u8 *p;
+
+	p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, _SSID_IE_, (int *)&ielen,
+		       len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_);
+
+	if (p) {
+		if ((_rtw_memcmp((void *)(p + 2), "DIRECT-", P2P_WILDCARD_SSID_LEN) &&
+		     (ielen == P2P_WILDCARD_SSID_LEN ||
+		      (ielen == cur->Ssid.SsidLength &&
+		       _rtw_memcmp((void *)(p + 2), (void *)cur->Ssid.Ssid, cur->Ssid.SsidLength)))) ||
+		    (ielen == 0 && pregpriv->go_hidden_ssid_mode != ALL_HIDE_SSID))
+			ret = _TRUE;
+	}
+	return ret;
+}
+
+static void hide_ssid_in_beacon(_adapter *padapter, u8 *pframe, uint len)
+{
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	u8 *wfd_ie;
+	uint wfd_ielen;
+
+	/* check if need to unhide/hide SSID in beacon for miracast */
+	wfd_ie = rtw_get_wfd_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_,
+				len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_,
+				NULL, &wfd_ielen);
+	if (wfd_ie) {
+		if (pregpriv->go_hidden_ssid_mode & BIT(MIRACAST)) {
+			rtw_clear_bit(MIRACAST, &pregpriv->go_hidden_ssid_mode);
+
+			/* check if there is other feature show SSID in beacon */
+			if (pregpriv->go_hidden_ssid_mode == MIRACAST_UNHIDE_SSID)
+				issue_beacon(padapter, 0);
+
+			ATOMIC_SET(&pregpriv->set_hide_ssid_timer, 1);
+			_set_timer(&pmlmeext->hide_ssid_timer, 25000);
+		} else {
+			if (ATOMIC_READ(&pregpriv->set_hide_ssid_timer))
+				_set_timer(&pmlmeext->hide_ssid_timer, 25000);
+		}
+	}
+}
+#endif
+#endif
+
+/****************************************************************************
+
+Following are the callback functions for each subtype of the management frames
+
+*****************************************************************************/
+
+unsigned int OnProbeReq(_adapter *padapter, union recv_frame *precv_frame)
+{
+	unsigned int	ielen;
+	unsigned char	*p;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX	*cur = &(pmlmeinfo->dev_network);
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint len = precv_frame->u.hdr.len;
+	u8 is_valid_p2p_probereq = _FALSE;
+
+#ifdef CONFIG_ATMEL_RC_PATCH
+	u8 *target_ie = NULL, *wps_ie = NULL;
+	u8 *start;
+	uint search_len = 0, wps_ielen = 0, target_ielen = 0;
+	struct sta_info	*psta;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+#endif
+
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	struct rx_pkt_attrib	*pattrib = &precv_frame->u.hdr.attrib;
+	u8 wifi_test_chk_rate = 1;
+#endif
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct rtw_phl_mld_t *mld = NULL;
+	u16 main_id = rtw_phl_get_macid_max_num(phl);
+#ifdef CONFIG_80211BE_EHT
+	u8 eid_eht_ml = EID_EXT_MULTI_LINK;
+	struct rtw_phl_ml_element ml_ele = {0};
+#endif
+
+	if (LINK_MLME_IS_REGU_FORBID(padapter_link))
+		return _SUCCESS;
+
+#ifdef CONFIG_IOCTL_CFG80211
+#ifdef CONFIG_CFG80211_REPORT_PROBE_REQ
+	if (GET_CFG80211_REPORT_MGMT(adapter_wdev_data(padapter),
+		IEEE80211_STYPE_PROBE_REQ) == _TRUE) {
+
+		#ifdef CONFIG_CONCURRENT_MODE
+		if (MLME_IS_AP(padapter) &&
+		    rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_LINKING | WIFI_UNDER_SURVEY)) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("%s : skip probe requst when buddy is linking or scanning\n", __func__);
+			#endif
+			/* don't report probe request */
+			return _SUCCESS;
+		}
+		#endif
+
+#ifdef PRIVATE_R
+		if (pmlmepriv->vendor_ie_filter_enable) {
+			/* check vendor ie for softAP and GO */
+			if (check_fwstate(pmlmepriv, WIFI_AP_STATE) &&
+				!check_vendor_ie(padapter, pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_,
+				len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_))
+				return _FAIL;
+		}
+#ifdef CONFIG_P2P
+		if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+			if (!is_valid_probereq(padapter, pframe, len))
+				return _FAIL;
+			hide_ssid_in_beacon(padapter, pframe, len);
+		}
+#endif
+#endif
+		rtw_cfg80211_rx_probe_request(padapter, precv_frame);
+		return _SUCCESS;
+	} else {
+		return _FAIL;
+	}
+#else /* !CONFIG_CFG80211_REPORT_PROBE_REQ */
+
+#if !defined(RTW_WKARD_DIS_PROBE_REQ_RPT_TO_HOSTAPD)
+	if (MLME_IS_AP(padapter) &&
+		(GET_CFG80211_REPORT_MGMT(adapter_wdev_data(padapter), IEEE80211_STYPE_PROBE_REQ) == _TRUE)) {
+		rtw_cfg80211_rx_probe_request(padapter, precv_frame);
+		return _SUCCESS;
+	}
+#endif
+
+#ifdef CONFIG_P2P
+	if ((adapter_to_dvobj(padapter)->wpas_type == RTW_WPAS_W1FI) &&
+		!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE) &&
+		(GET_CFG80211_REPORT_MGMT(adapter_wdev_data(padapter),
+				IEEE80211_STYPE_PROBE_REQ) == _TRUE)) {
+		rtw_cfg80211_rx_probe_request(padapter, precv_frame);
+		return _SUCCESS;
+	}
+#endif
+#endif /* CONFIG_CFG80211_REPORT_PROBE_REQ */
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+#if !defined(CONFIG_CFG80211_REPORT_PROBE_REQ)
+#ifdef CONFIG_P2P
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DEVICE) ||
+	    rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+		/*	Commented by Albert 2011/03/17 */
+		/*	mcs_rate = 0->CCK 1M rate */
+		/*	mcs_rate = 1->CCK 2M rate */
+		/*	mcs_rate = 2->CCK 5.5M rate */
+		/*	mcs_rate = 3->CCK 11M rate */
+		/*	In the P2P mode, the driver should not support the CCK rate */
+
+		/*	Commented by Kurt 2012/10/16 */
+		/*	IOT issue: Google Nexus7 use 1M rate to send p2p_probe_req after GO nego completed and Nexus7 is client */
+		if (padapter->registrypriv.wifi_spec == 1) {
+			if (pattrib->data_rate <= DESC_RATE11M)
+				wifi_test_chk_rate = 0;
+		}
+
+		if (wifi_test_chk_rate == 1) {
+			is_valid_p2p_probereq = process_probe_req_p2p_ie(pwdinfo, pframe, len);
+			if (is_valid_p2p_probereq == _TRUE) {
+				if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DEVICE)) {
+					/* P2P Device only responds probe response in listen state(ROCH) */
+					if (MLME_IS_ROCH(padapter))
+						p2p_listen_state_process(padapter,  get_sa(pframe));
+					return _SUCCESS;
+				}
+
+				if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO))
+					goto _continue;
+			}
+		}
+	}
+
+_continue:
+#endif /* CONFIG_P2P */
+
+	if (MLME_IS_STA(padapter))
+		return _SUCCESS;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _FALSE &&
+	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE | WIFI_AP_STATE | WIFI_MESH_STATE) == _FALSE)
+		return _SUCCESS;
+
+
+	/* RTW_INFO("+OnProbeReq\n"); */
+
+
+#ifdef CONFIG_ATMEL_RC_PATCH
+	wps_ie = rtw_get_wps_ie(
+			      pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_,
+			      len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_,
+			      NULL, &wps_ielen);
+	if (wps_ie)
+		target_ie = rtw_get_wps_attr_content(wps_ie, wps_ielen, WPS_ATTR_MANUFACTURER, NULL, &target_ielen);
+	if ((target_ie && (target_ielen == 4)) && (_TRUE == _rtw_memcmp((void *)target_ie, "Ozmo", 4))) {
+		/* psta->flag_atmel_rc = 1; */
+		unsigned char *sa_addr = get_sa(pframe);
+
+		printk("%s: Find Ozmo RC -- "MAC_FMT"  \n\n",
+		       __func__, MAC_ARG(sa_addr));
+		_rtw_memcpy(pstapriv->atmel_rc_pattern, get_sa(pframe), ETH_ALEN);
+	}
+#endif
+
+
+#ifdef CONFIG_AUTO_AP_MODE
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE &&
+	    pmlmepriv->dev_cur_network.join_res == _TRUE) {
+		struct sta_info	*psta;
+		u8 *mac_addr, *peer_addr;
+		struct sta_priv *pstapriv = &padapter->stapriv;
+		u8 RC_OUI[4] = {0x00, 0xE0, 0x4C, 0x0A};
+		/* EID[1] + EID_LEN[1] + RC_OUI[4] + MAC[6] + PairingID[2] + ChannelNum[2] */
+
+		p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, _VENDOR_SPECIFIC_IE_, (int *)&ielen,
+			       len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_);
+
+		if (!p || ielen != 14)
+			goto _non_rc_device;
+
+		if (!_rtw_memcmp(p + 2, RC_OUI, sizeof(RC_OUI)))
+			goto _non_rc_device;
+
+		if (!_rtw_memcmp(p + 6, get_sa(pframe), ETH_ALEN)) {
+			RTW_INFO("%s, do rc pairing ("MAC_FMT"), but mac addr mismatch!("MAC_FMT")\n", __FUNCTION__,
+				 MAC_ARG(get_sa(pframe)), MAC_ARG(p + 6));
+
+			goto _non_rc_device;
+		}
+
+		RTW_INFO("%s, got the pairing device("MAC_FMT")\n", __FUNCTION__,  MAC_ARG(get_sa(pframe)));
+
+		/* new a station */
+		psta = rtw_get_stainfo(pstapriv, get_sa(pframe));
+		if (psta == NULL) {
+			/* allocate a new one */
+			RTW_INFO("going to alloc stainfo for rc="MAC_FMT"\n",  MAC_ARG(get_sa(pframe)));
+			mld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter->phl_role, get_sa(pframe), DTYPE);
+			if (mld == NULL) {
+				/* TODO: */
+				RTW_INFO(" Exceed the upper limit of supported MLDs...\n");
+				return _SUCCESS;
+			}
+			/* primary link only */
+			psta = rtw_alloc_stainfo_sw(pstapriv, DTYPE, main_id, padapter_link->wrlink->id, get_sa(pframe));
+			if (psta == NULL) {
+				/* TODO: */
+				RTW_INFO(" Exceed the upper limit of supported clients...\n");
+				rtw_free_mld_stainfo(padapter), mld);
+				return _SUCCESS;
+			}
+			rtw_phl_link_mld_stainfo(mld, psta->phl_sta);
+
+			rtw_stapriv_asoc_list_lock(pstapriv);
+			if (rtw_is_list_empty(&psta->asoc_list)) {
+				psta->expire_to = pstapriv->expire_to;
+				rtw_stapriv_asoc_list_add(pstapriv, psta);
+			}
+			rtw_stapriv_asoc_list_unlock(pstapriv);
+
+			/* generate pairing ID */
+			mac_addr = adapter_mac_addr(padapter);
+			peer_addr = psta->phl_sta->mac_addr;
+			psta->pid = (u16)(((mac_addr[4] << 8) + mac_addr[5]) + ((peer_addr[4] << 8) + peer_addr[5]));
+
+			/* update peer stainfo */
+			psta->isrc = _TRUE;
+
+			/* AID assignment */
+			if (psta->phl_sta->aid > 0)
+				RTW_INFO(FUNC_ADPT_FMT" old AID=%d\n", FUNC_ADPT_ARG(padapter), psta->phl_sta->aid);
+			else {
+				if (!rtw_aid_alloc(padapter, psta)) {
+					RTW_INFO(FUNC_ADPT_FMT" no room for more AIDs\n", FUNC_ADPT_ARG(padapter));
+					return _SUCCESS;
+				}
+				RTW_INFO(FUNC_ADPT_FMT" allocate new AID=%d\n", FUNC_ADPT_ARG(padapter), psta->phl_sta->aid);
+			}
+
+			psta->qos_option = 1;
+			psta->phl_sta->chandef.bw = CHANNEL_WIDTH_20;
+			psta->ieee8021x_blocked = _FALSE;
+#ifdef CONFIG_80211N_HT
+			if(padapter->registrypriv.ht_enable &&
+				is_supported_ht(padapter->registrypriv.wireless_mode))
+			{
+				psta->htpriv.ht_option = _TRUE;
+				psta->ampdu_priv.ampdu_enable = _FALSE;
+				psta->htpriv.sgi_20m = _FALSE;
+				psta->htpriv.sgi_40m = _FALSE;
+				psta->htpriv.ch_offset = CHAN_OFFSET_NO_EXT;
+				psta->ampdu_priv.agg_enable_bitmap = 0x0;/* reset */
+				psta->ampdu_priv.candidate_tid_bitmap = 0x0;/* reset */
+			}
+#endif
+
+			_rtw_memset((void *)&psta->sta_stats, 0, sizeof(struct stainfo_stats));
+
+			_rtw_spinlock_bh(&psta->lock);
+			psta->state |= WIFI_ASOC_STATE;
+			_rtw_spinunlock_bh(&psta->lock);
+
+			report_add_sta_event(padapter, psta->phl_sta->mac_addr);
+
+		}
+
+		issue_probersp(padapter, padapter_link, get_sa(pframe), _FALSE);
+
+		return _SUCCESS;
+
+	}
+
+_non_rc_device:
+
+	return _SUCCESS;
+
+#endif /* CONFIG_AUTO_AP_MODE */
+
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) &&
+	    rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_LINKING | WIFI_UNDER_SURVEY)) {
+		/* don't process probe req */
+		return _SUCCESS;
+	}
+#endif
+
+#ifdef PRIVATE_R
+	if (pmlmepriv->vendor_ie_filter_enable) {
+		/* check vendor ie for softAP and GO */
+		if (check_fwstate(pmlmepriv, WIFI_AP_STATE) &&
+			!check_vendor_ie(padapter, pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_,
+			len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_))
+			goto exit;
+	}
+#ifdef CONFIG_P2P
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+		if (is_valid_probereq(padapter, pframe, len)) {
+			hide_ssid_in_beacon(padapter, pframe, len);
+			goto _issue_probersp;
+		} else {
+			goto exit;
+		}
+	}
+#endif
+#endif
+
+	p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, _SSID_IE_, (int *)&ielen,
+		       len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_);
+	if (!p)
+		goto exit;
+
+	if (is_valid_p2p_probereq == _TRUE)
+		goto _issue_probersp;
+
+	if ((ielen != 0 &&
+	     !_rtw_memcmp((void *)(p + 2), (void *)cur->Ssid.Ssid, cur->Ssid.SsidLength)) ||
+	    (ielen == 0 && pmlmeinfo->hidden_ssid_mode))
+		goto exit;
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, WLAN_EID_MESH_ID, (int *)&ielen,
+				len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_);
+
+		if (!p)
+			goto exit;
+		if (ielen != 0 && _rtw_memcmp((void *)(p + 2), (void *)cur->mesh_id.Ssid, cur->mesh_id.SsidLength) == _FALSE)
+			goto exit;
+	}
+#endif
+
+_issue_probersp:
+	if ((check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE &&
+		padapter_link->mlmepriv.cur_network.join_res == _TRUE) ||
+		check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
+#ifdef CONFIG_80211BE_EHT
+		if (pmlmepriv->mlopriv.mlo_option == _TRUE) {
+			p = rtw_get_ie_ex(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_,
+				len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_,
+				WLAN_EID_EXTENSION, &eid_eht_ml, 1, NULL, (int *)&ielen);
+
+			if (p != NULL && ielen >= EHT_ML_PROBEREQ_MIN_LEN) {
+				rtw_phl_parse_ml_ie(GET_PHL_COM(adapter_to_dvobj(padapter)), p + 3, ielen - 3, &ml_ele);
+				if (ml_ele.type == PROBE_REQUEST_ML) {
+					issue_ml_probersp(padapter, padapter_link, get_sa(pframe), is_valid_p2p_probereq);
+					goto exit;
+				}
+			}
+		}
+#endif
+		/* RTW_INFO("+issue_probersp during ap mode\n"); */
+		issue_probersp(padapter, padapter_link, get_sa(pframe), is_valid_p2p_probereq);
+	}
+
+exit:
+	return _SUCCESS;
+#endif /* !defined(CONFIG_CFG80211_REPORT_PROBE_REQ) */
+}
+
+unsigned int OnProbeRsp(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	u8	*pframe = precv_frame->u.hdr.rx_data;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &padapter->wdinfo;
+#endif
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+
+	if ((mlmeext_chk_scan_state(pmlmeext, SCAN_PROCESS))
+		|| (MLME_IS_MESH(padapter) && check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE))
+	) {
+		struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+		struct link_mlme_ext_info *lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+
+		if (rtw_match_bssid_ext(padapter, GetAddr3Ptr(pframe))
+			&& (pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE && (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
+		{
+			if (!rtw_check_bcn_info(padapter, padapter_link, pframe, precv_frame->u.hdr.len)) {
+				RTW_PRINT(FUNC_ADPT_FMT" ap has changed, disconnect now\n", FUNC_ADPT_ARG(padapter));
+				receive_disconnect(padapter, lmlmeinfo->network.MacAddress , 0, _FALSE);
+				pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+				pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_AP_BEACON_CHANGED;
+				pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+			}
+		}
+
+		rtw_mi_report_survey_event(padapter, precv_frame);
+		return _SUCCESS;
+	}
+
+#if 0 /* move to validate_recv_mgnt_frame */
+	if (_rtw_memcmp(GetAddr3Ptr(pframe), get_my_bssid(&pmlmeinfo->network), ETH_ALEN)) {
+		if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
+			psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+			if (psta != NULL)
+				psta->sta_stats.rx_mgnt_pkts++;
+		}
+	}
+#endif
+
+	return _SUCCESS;
+
+}
+
+/* for 11n Logo 4.2.31/4.2.32 */
+static void rtw_check_legacy_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *pframe, u32 len)
+{
+
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+
+	if (!padapter->registrypriv.wifi_spec)
+		return;
+
+	if(!MLME_IS_AP(padapter))
+		return;
+
+
+	if (pmlmeext->bstart_bss == _TRUE) {
+		int left;
+		unsigned char *pos;
+		struct rtw_ieee802_11_elems elems;
+#ifdef CONFIG_80211N_HT
+		u16 cur_op_mode; 
+#endif
+		/* checking IEs */
+		left = len - sizeof(struct rtw_ieee80211_hdr_3addr) - _BEACON_IE_OFFSET_;
+		pos = pframe + sizeof(struct rtw_ieee80211_hdr_3addr) + _BEACON_IE_OFFSET_;
+		if (rtw_ieee802_11_parse_elems(pos, left, &elems, 1) == ParseFailed) {
+			RTW_INFO("%s: parse fail for "MAC_FMT"\n", __func__, MAC_ARG(GetAddr3Ptr(pframe)));
+			return;
+		}
+#ifdef CONFIG_80211N_HT
+		cur_op_mode = pmlmepriv->ht_op_mode & HT_INFO_OPERATION_MODE_OP_MODE_MASK;
+#endif
+		/* for legacy ap */
+		if (elems.ht_capabilities == NULL && elems.ht_capabilities_len == 0) {
+
+			if (0)
+				RTW_INFO("%s: "MAC_FMT" is legacy ap\n", __func__, MAC_ARG(GetAddr3Ptr(pframe)));
+
+			ATOMIC_SET(&pmlmepriv->olbc, _TRUE);
+			ATOMIC_SET(&pmlmepriv->olbc_ht, _TRUE);
+		}
+			
+	}
+}
+
+static void parsing_country_element_ie(struct link_mlme_ext_priv *pmlmeext, u8* ie, s16 len)
+{
+	int i = 0;
+
+	if (len < 6)
+		return;
+	else
+		ie += 2;
+	/* country info */
+	_rtw_memcpy(pmlmeext->country, ie, 3);
+	ie += 3;
+	len -= 3;
+	/* Do not process the operating triplet field */
+	if(*ie > 200)
+		return;
+	/* channel info triplet to mlmeext */
+	while (len - 3 >= 0) {
+		pmlmeext->ch_info[i].first_ch = *(ie++);
+		pmlmeext->ch_info[i].num_of_ch = *(ie++);
+		pmlmeext->ch_info[i].power_level = *(ie++);
+		len -= 3;
+		i++;
+	}
+}
+
+static void parsing_ap_channel_rpt_ie(struct link_mlme_ext_priv *pmlmeext, u8* ie, s16 len)
+{
+	int i = 0;
+
+	if (len < 2)
+		return;
+	else
+		ie += 2;
+	/* copy op class */
+	pmlmeext->op_class = *ie;
+	ie++;
+	len--;
+	/* copy channel list */
+	while (len > 0) {
+		pmlmeext->channel_list[i] = *(ie++);
+		len--;
+		i++;
+	}
+}
+
+static void rtw_parsing_ap_info(struct _ADAPTER_LINK *padapter_link, u8 *pframe, u32 len)
+{
+	int left;
+	unsigned char *pos;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct rtw_ieee802_11_elems elems;
+
+	if (!padapter_link)
+		return;
+
+	/* checking IEs */
+	left = len - sizeof(struct rtw_ieee80211_hdr_3addr) - _BEACON_IE_OFFSET_;
+	pos = pframe + sizeof(struct rtw_ieee80211_hdr_3addr) + _BEACON_IE_OFFSET_;
+	if (rtw_ieee802_11_parse_elems(pos, left, &elems, 1) == ParseFailed)
+		return;
+
+	/* clear mlmeext country and ch_info content*/
+	if (is_alpha(pmlmeext->country[0]) && is_alpha(pmlmeext->country[1])) {
+		memset(&pmlmeext->country, 0, sizeof(pmlmeext->country));
+		memset(&pmlmeext->ch_info, 0, sizeof(pmlmeext->ch_info));
+	}
+	/* parsing country element */
+	if (elems.country_info)
+		parsing_country_element_ie(pmlmeext, elems.country_info - 2 , elems.country_info_len);
+
+	/* clear mlmeext op_class and channel_list content*/
+	if (pmlmeext->op_class) {
+		pmlmeext->op_class = 0;
+		memset(&pmlmeext->channel_list, 0, sizeof(pmlmeext->channel_list));
+	}
+	/* parsing AP Channel Report */
+	if (elems.ap_channel_rpt)
+		parsing_ap_channel_rpt_ie(pmlmeext, elems.ap_channel_rpt - 2 , elems.ap_channel_rpt_len);
+}
+
+static u8 auth_null_update_cur_network(struct _ADAPTER_LINK *alink, union recv_frame *rframe)
+{
+	_adapter *adapter = alink->adapter;
+	WLAN_BSSID_EX *bss;
+	u8 ret = _FAIL;
+
+	bss = rtw_malloc(sizeof(*bss));
+	if (!bss)
+		goto exit;
+
+	if (collect_bss_info(adapter, rframe, bss) == _SUCCESS) {
+		struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+		struct link_mlme_ext_info *lmlmeinfo = &alink->mlmeextpriv.mlmext_info;
+		u8 *whdr = rframe->u.hdr.rx_data;
+		uint flen = rframe->u.hdr.len;
+		struct beacon_keys recv_bcn_keys;
+		struct wlan_network *scanned;
+
+#ifndef RTW_PHL_TEST_FPGA
+		/* get bcn key from received beacon */
+		if (rtw_get_bcn_keys(adapter, alink, whdr, flen, &recv_bcn_keys) != _TRUE) {
+			RTW_WARN(FUNC_ADPT_FMT" get beacon keys failed\n", FUNC_ADPT_ARG(adapter));
+			goto free_bss;
+		}
+		if (is_hidden_ssid(recv_bcn_keys.ssid, recv_bcn_keys.ssid_len)) {
+			_rtw_memcpy(recv_bcn_keys.ssid, lmlmeinfo->network.Ssid.Ssid, IW_ESSID_MAX_SIZE);
+			recv_bcn_keys.ssid_len = lmlmeinfo->network.Ssid.SsidLength;
+		}
+
+		scanned = rtw_find_network(&pmlmepriv->scanned_queue, get_my_bssid(&lmlmeinfo->network));
+		if (!scanned) {
+			RTW_WARN(FUNC_ADPT_FMT" can't get scanned network\n", FUNC_ADPT_ARG(adapter));
+			ret = RTW_ABORT_LINKING;
+			goto free_bss;
+		}
+		if (!scanned->bcn_keys_valid) {
+			RTW_WARN(FUNC_ADPT_FMT" scanned beacon keys not valid\n", FUNC_ADPT_ARG(adapter));
+			ret = RTW_ABORT_LINKING;
+			goto free_bss;
+		}
+		if (!rtw_bcn_key_compare(&scanned->bcn_keys, &recv_bcn_keys)) {
+			RTW_WARN(FUNC_ADPT_FMT" recv beacon keys conflict with scanned\n", FUNC_ADPT_ARG(adapter));
+			RTW_INFO(FUNC_ADPT_FMT" scanned beacon key:\n", FUNC_ADPT_ARG(adapter));
+			rtw_dump_bcn_keys(RTW_DBGDUMP, &scanned->bcn_keys);
+			RTW_INFO(FUNC_ADPT_FMT" recv beacon key:\n", FUNC_ADPT_ARG(adapter));
+			rtw_dump_bcn_keys(RTW_DBGDUMP, &recv_bcn_keys);
+			ret = RTW_ABORT_LINKING;
+			goto free_bss;
+		}
+#endif
+
+		/* update bcn keys */
+		RTW_INFO(FUNC_ADPT_FMT" beacon keys ready\n", FUNC_ADPT_ARG(adapter));
+		_rtw_memcpy(&alink->mlmepriv.cur_beacon_keys, &recv_bcn_keys, sizeof(recv_bcn_keys));
+
+		/* update cur_network */
+		rtw_update_adapter_network(&(pmlmepriv->dev_cur_network.network), bss, adapter, _TRUE);
+		rtw_update_network(&(alink->mlmepriv.cur_network.network), bss, adapter, _TRUE);
+
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+		alink->mlmepriv.new_beacon_cnts = 0;
+#endif
+
+		ret = _SUCCESS;
+	}
+
+free_bss:
+	rtw_mfree(bss, sizeof(*bss));
+
+exit:
+	return ret;
+}
+
+unsigned int OnBeacon(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct sta_info	*psta;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint len = precv_frame->u.hdr.len;
+	int ret = _SUCCESS;
+#ifdef CONFIG_TDLS
+	struct sta_info *ptdls_sta;
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+#ifdef CONFIG_TDLS_CH_SW
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+#endif
+#endif /* CONFIG_TDLS */
+#if defined(CONFIG_MCC_MODE) || defined(CONFIG_RTW_LPS)
+	struct rtw_bcn_pkt_info bcn_info = {0};
+#endif
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_mlme_ext_info *lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+	struct rtw_phl_mld_t *mld = NULL;
+	u16 main_id = rtw_phl_get_macid_max_num(phl);
+
+	if (pframe == NULL) {
+		RTW_ERR(FUNC_ADPT_FMT" pframe == NULL\n", FUNC_ADPT_ARG(padapter));
+		rtw_warn_on(1);
+		return _SUCCESS;
+	}
+	if (validate_beacon_len(pframe, len) == _FALSE) {
+		return _SUCCESS;
+	}
+
+	if (mlmeext_chk_scan_state(pmlmeext, SCAN_PROCESS)
+		|| (MLME_IS_MESH(padapter) && check_fwstate(pmlmepriv, WIFI_ASOC_STATE))
+	) {
+		if (_rtw_memcmp(GetAddr3Ptr(pframe), get_my_bssid(&lmlmeinfo->network), ETH_ALEN)
+			&& (pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE && (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)
+		) {
+#ifndef RTW_PHL_TEST_FPGA
+			if (!rtw_check_bcn_info(padapter, padapter_link, pframe, len))
+			{
+				RTW_PRINT(FUNC_ADPT_FMT" ap has changed, disconnect now\n", FUNC_ADPT_ARG(padapter));
+				receive_disconnect(padapter, lmlmeinfo->network.MacAddress , 0, _FALSE);
+				pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+				pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_AP_BEACON_CHANGED;
+				pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+			}
+#endif
+		}
+		report_survey_event(padapter, precv_frame);
+		return _SUCCESS;
+	}
+
+	rtw_check_legacy_ap(padapter, padapter_link, pframe, len);
+
+	if (rtw_match_bssid_ext(padapter, GetAddr3Ptr(pframe)))
+	{
+		if ((pmlmeinfo->state & WIFI_FW_AUTH_NULL)
+			&& (rtw_sta_linking_test_wait_done() || pmlmeext->join_abort)
+		) {
+			if (rtw_sta_linking_test_force_fail() || pmlmeext->join_abort) {
+				ret = RTW_ABORT_LINKING;
+				goto end_auth_null;
+			}
+
+			/* we should update current network before auth, or some IE is wrong */
+			ret = auth_null_update_cur_network(padapter_link, precv_frame);
+			if (ret != _SUCCESS)
+				goto end_auth_null;
+
+			/* check the vendor of the assoc AP */
+			pmlmeinfo->assoc_AP_vendor = check_assoc_AP(pframe + sizeof(struct rtw_ieee80211_hdr_3addr), len - sizeof(struct rtw_ieee80211_hdr_3addr));
+			rtw_update_phl_iot(padapter, pmlmeinfo->assoc_AP_vendor);
+
+			/* update TSF Value */
+			update_TSF(&padapter_link->mlmeextpriv, pframe, len);
+			padapter_link->mlmeextpriv.bcn_cnt = 0;
+			padapter_link->mlmeextpriv.last_bcn_cnt = 0;
+
+#ifdef CONFIG_P2P_PS
+			/* Comment by YiWei , in wifi p2p spec the "3.3 P2P Power Management" , "These mechanisms are available in a P2P Group in which only P2P Devices are associated." */
+			/* process_p2p_ps_ie(padapter, (pframe + WLAN_HDR_A3_LEN), (len - WLAN_HDR_A3_LEN)); */
+#endif /* CONFIG_P2P_PS */
+
+			/* start auth */
+			start_clnt_auth(padapter);
+
+end_auth_null:
+			if (ret == RTW_ABORT_LINKING)
+				set_link_timer(pmlmeext, 1);
+
+			return _SUCCESS;
+		}
+
+		if (((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE) && (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)) {
+			psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+			if (psta != NULL) {
+#ifdef CONFIG_RTW_80211R
+				rtw_ft_update_bcn(padapter, precv_frame);
+#endif
+#ifdef RTW_PHL_TEST_FPGA
+				ret = 1;
+#else
+				ret = rtw_check_bcn_info(padapter, padapter_link, pframe, len);
+#endif
+				if (!ret) {
+					RTW_PRINT(FUNC_ADPT_FMT" ap has changed, disconnect now\n", FUNC_ADPT_ARG(padapter));
+					receive_disconnect(padapter, lmlmeinfo->network.MacAddress , 0, _FALSE);
+					pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+					pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_AP_BEACON_CHANGED;
+					pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+					return _SUCCESS;
+				}
+
+				rtw_parsing_ap_info(padapter_link, pframe, len);
+
+				/* update WMM, ERP in the beacon */
+				/* todo: the timer is used instead of the number of the beacon received */
+				if ((sta_rx_pkts(psta) & 0xf) == 0) {
+					/* RTW_INFO("update_bcn_info\n"); */
+					update_beacon_info(padapter, pframe, len, psta);
+				}
+				if (pmlmepriv->cur_network_scanned)
+					pmlmepriv->cur_network_scanned->network.PhyInfo.rssi = precv_frame->u.hdr.attrib.phy_info.recv_signal_power;
+				padapter_link->mlmeextpriv.bcn_cnt++;
+#ifdef CONFIG_BCN_RECV_TIME
+				rtw_rx_bcn_time_update(padapter, padapter_link, len, precv_frame->u.hdr.attrib.data_rate);
+#endif
+#if defined(CONFIG_MCC_MODE) || defined(CONFIG_RTW_LPS)
+				update_TSF(&padapter_link->mlmeextpriv, pframe, len);
+				bcn_info.sta = psta->phl_sta;
+				bcn_info.tsf = padapter_link->mlmeextpriv.TSFValue;
+				rtw_phl_sta_up_rx_bcn(GET_PHL_INFO(adapter_to_dvobj(padapter)), &bcn_info);
+#endif
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+				if (rtw_tdls_is_chsw_allowed(padapter) == _TRUE) {
+					/* Send TDLS Channel Switch Request when receiving Beacon */
+					if ((padapter->tdlsinfo.chsw_info.ch_sw_state & TDLS_CH_SW_INITIATOR_STATE)
+						&& (padapter->tdlsinfo.chsw_info.ch_sw_state & TDLS_WAIT_CH_SW_LAUNCH_STATE)
+						&& (ATOMIC_READ(&pchsw_info->chsw_on) == _TRUE)
+						&& (padapter_link->mlmeextpriv.chandef.chan == rtw_get_oper_ch(padapter, padapter_link))) {
+						ptdls_sta = rtw_get_stainfo(&padapter->stapriv, padapter->tdlsinfo.chsw_info.addr);
+						if (ptdls_sta != NULL) {
+							if (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE) {
+								padapter->tdlsinfo.chsw_info.ch_sw_state &= ~TDLS_WAIT_CH_SW_LAUNCH_STATE;
+								_set_timer(&ptdls_sta->stay_on_base_chnl_timer, TDLS_CH_SW_STAY_ON_BASE_CHNL_TIMEOUT);
+							}
+						}
+					}
+				}
+#endif
+#endif /* CONFIG_TDLS */
+
+				#ifdef CONFIG_ECSA_PHL
+				process_csa_ie(padapter
+					, pframe + WLAN_HDR_A3_LEN + _BEACON_IE_OFFSET_
+					, len - (WLAN_HDR_A3_LEN + _BEACON_IE_OFFSET_));
+				#endif
+
+				#ifdef CONFIG_80211D
+				alink_process_regu_ies(padapter_link
+					, pframe + WLAN_HDR_A3_LEN + _BEACON_IE_OFFSET_
+					, len - (WLAN_HDR_A3_LEN + _BEACON_IE_OFFSET_));
+				#endif
+
+#ifdef CONFIG_P2P_PS
+				process_p2p_ps_ie(padapter, (pframe + WLAN_HDR_A3_LEN), (len - WLAN_HDR_A3_LEN));
+#endif /* CONFIG_P2P_PS */
+
+				if (ATOMIC_READ(&padapter->need_tsf_sync_done) == _TRUE) {
+					rtw_phl_sta_tsf_sync_done(GET_PHL_INFO(adapter_to_dvobj(padapter)), psta->phl_sta);
+					ATOMIC_SET(&padapter->need_tsf_sync_done, _FALSE);
+				}
+#if 0 /* move to validate_recv_mgnt_frame */
+				psta->sta_stats.rx_mgnt_pkts++;
+#endif
+			}
+
+		} else if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
+			u8 rate_set[16];
+			u8 rate_num = 0;
+
+			psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+			if (psta != NULL) {
+				/*
+				* update WMM, ERP in the beacon
+				* todo: the timer is used instead of the number of the beacon received
+				*/
+				if ((sta_rx_pkts(psta) & 0xf) == 0)
+					update_beacon_info(padapter, pframe, len, psta);
+			} else {
+				rtw_ies_get_supported_rate(pframe + WLAN_HDR_A3_LEN + _BEACON_IE_OFFSET_, len - WLAN_HDR_A3_LEN - _BEACON_IE_OFFSET_, rate_set, &rate_num);
+				if (rate_num == 0) {
+					RTW_INFO(FUNC_ADPT_FMT" RX beacon with no supported rate\n", FUNC_ADPT_ARG(padapter));
+					goto _END_ONBEACON_;
+				}
+				/* ToDo CONFIG_RTW_MLD: ad-hoc */
+				mld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter->phl_role, get_addr2_ptr(pframe), DTYPE);
+				if (mld == NULL) {
+					RTW_INFO(FUNC_ADPT_FMT" Exceed the upper limit of supported MLDs\n", FUNC_ADPT_ARG(padapter));
+					goto _END_ONBEACON_;
+				}
+				psta = rtw_alloc_stainfo_sw(pstapriv, DTYPE, main_id, RTW_RLINK_PRIMARY, get_addr2_ptr(pframe));
+				if (psta == NULL) {
+					RTW_INFO(FUNC_ADPT_FMT" Exceed the upper limit of supported clients\n", FUNC_ADPT_ARG(padapter));
+					rtw_free_mld_stainfo(padapter, mld);
+					goto _END_ONBEACON_;
+				}
+				rtw_phl_link_mld_stainfo(mld, psta->phl_sta);
+
+				psta->expire_to = pstapriv->adhoc_expire_to;
+
+				_rtw_memcpy(psta->bssrateset, rate_set, rate_num);
+				psta->bssratelen = rate_num;
+
+				/* update TSF Value */
+				update_TSF(&padapter_link->mlmeextpriv, pframe, len);
+
+				/* report sta add event */
+				report_add_sta_event(padapter, get_addr2_ptr(pframe));
+			}
+		}
+	}
+
+_END_ONBEACON_:
+
+	return _SUCCESS;
+
+}
+
+static u32 rtw_get_sta_num_by_state(_adapter *padapter, u32 state)
+{
+	_list	*plist, *phead;
+	u32	index, sta_num = 0;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &(padapter->stapriv);
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);	
+	for (index = 0; index < NUM_STA; index++) {		
+		phead = &(pstapriv->sta_hash[index]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info , hash_list);
+			if ((psta->state & (state)))
+				sta_num++;
+			plist = get_next(plist);
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	/* RTW_INFO("%s : waiting for %u sta under linking \n", __func__, sta_num); */
+	return sta_num;
+}
+
+static u8  rtw_defs_attack_chk(_adapter *padapter)
+{
+	struct mlme_priv *mlme = &(padapter->mlmepriv);
+	u8 is_reject = _FALSE;
+	u32 sta_limit = 0;
+	u32 stime = rtw_systime_to_ms(rtw_get_current_time());
+	static u32 ptime = 0;
+
+	/* RTW_INFO("%s : ptime=%u, stime=%u, diff=%u\n", __func__, ptime, stime, (stime - ptime)); */
+	if ((ptime > 0) && ((stime - ptime) < mlme->defs_lmt_time)) {
+		sta_limit = rtw_get_sta_num_by_state(padapter, WIFI_FW_LINKING_STATE);
+		if (sta_limit >= mlme->defs_lmt_sta)
+			is_reject = _TRUE;
+	}
+
+	ptime = stime;
+	/* RTW_INFO("%s : current linking num=%u\n", __func__, sta_limit); */
+	return is_reject;	
+}
+
+unsigned int OnAuth(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_AP_MODE
+	unsigned int	auth_mode, seq, ie_len;
+	unsigned char	*sa, *p;
+	u16	algorithm;
+	int	status;
+	static struct sta_info stat;
+	static struct rtw_phl_stainfo_t phl_stat;
+	struct	sta_info	*pstat = NULL;
+	struct	sta_priv *pstapriv = &padapter->stapriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint len = precv_frame->u.hdr.len;
+	u8	offset = 0;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct rtw_wifi_role_t *phl_role = padapter->phl_role;
+	struct rtw_phl_mld_t *mld = NULL;
+	u16 main_id = rtw_phl_get_macid_max_num(phl);
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) &&
+	    rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_LINKING | WIFI_UNDER_SURVEY)) {
+		/* don't process auth request; */
+		return _SUCCESS;
+	}
+#endif /* CONFIG_CONCURRENT_MODE */
+
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
+		return _FAIL;
+
+	if (!MLME_IS_ASOC(padapter))
+		return _SUCCESS;
+
+#if defined(CONFIG_IOCTL_CFG80211) && defined(CONFIG_RTW_MESH)
+	if (MLME_IS_MESH(padapter))
+		return rtw_mesh_on_auth(padapter, precv_frame);
+#endif
+
+	RTW_INFO("+OnAuth\n");
+
+	sa = get_addr2_ptr(pframe);
+
+	auth_mode = psecuritypriv->dot11AuthAlgrthm;
+
+	if (GetPrivacy(pframe)) {
+		u8	*iv;
+		struct rx_pkt_attrib	*prxattrib = &(precv_frame->u.hdr.attrib);
+
+		prxattrib->hdrlen = WLAN_HDR_A3_LEN;
+		prxattrib->encrypt = _WEP40_;
+
+		iv = pframe + prxattrib->hdrlen;
+		prxattrib->key_index = ((iv[3] >> 6) & 0x3);
+
+		prxattrib->iv_len = 4;
+		prxattrib->icv_len = 4;
+
+		rtw_wep_decrypt(padapter, (u8 *)precv_frame);
+
+		offset = 4;
+	}
+
+	algorithm = le16_to_cpu(*(u16 *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset));
+	seq	= le16_to_cpu(*(u16 *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset + 2));
+
+	RTW_INFO("auth alg=%x, seq=%X\n", algorithm, seq);
+
+	if (rtw_check_invalid_mac_address(sa, _FALSE)){
+		RTW_INFO("%s : reject invalid AUTH-req "MAC_FMT"\n", 
+			__func__, MAC_ARG(get_addr2_ptr(pframe)));
+		return _FAIL;
+	}
+
+	if(rtw_defs_attack_chk(padapter))  {
+		struct sta_info *_psta;
+		_psta = rtw_get_stainfo(pstapriv, sa);
+		if ((_psta == NULL) || !(_psta->state & WIFI_FW_ASSOC_SUCCESS)) {
+			status = _STATS_REFUSED_TEMPORARILY_;
+			RTW_ERR("%s : refused temporarily for sa "MAC_FMT" !\n", __func__, MAC_ARG(sa));
+			goto auth_fail;
+		}
+	}
+
+	if (rtw_ap_linking_test_force_auth_fail()) {
+		status = rtw_ap_linking_test_force_auth_fail();
+		RTW_INFO(FUNC_ADPT_FMT" force auth fail with status:%u\n"
+			, FUNC_ADPT_ARG(padapter), status);
+		goto auth_fail;
+	}
+
+	if ((auth_mode == 2) && (algorithm != WLAN_AUTH_SAE) &&
+	    (psecuritypriv->dot11PrivacyAlgrthm != _WEP40_) &&
+	    (psecuritypriv->dot11PrivacyAlgrthm != _WEP104_))
+		auth_mode = 0;
+
+	if (((algorithm > 0 && auth_mode == 0) ||	/* rx a shared-key auth but shared not enabled */
+	    (algorithm == 0 && auth_mode == 1))
+	#ifdef  CONFIG_RTW_80211R_AP
+		&& (algorithm != WLAN_AUTH_FT)
+	#endif
+	    ) {	/* rx a open-system auth but shared-key is enabled */
+		RTW_INFO("auth rejected due to bad alg [alg=%d, auth_mib=%d] %02X%02X%02X%02X%02X%02X\n",
+			algorithm, auth_mode, sa[0], sa[1], sa[2], sa[3], sa[4], sa[5]);
+
+		status = _STATS_NO_SUPP_ALG_;
+
+		goto auth_fail;
+	}
+
+#if CONFIG_RTW_MACADDR_ACL
+	if (rtw_access_ctrl(padapter, sa) == _FALSE) {
+		status = _STATS_UNABLE_HANDLE_STA_;
+		goto auth_fail;
+	}
+#endif
+
+	pstat = rtw_get_stainfo(pstapriv, sa);
+	if (pstat == NULL) {
+
+		mld = rtw_phl_get_mld_by_addr(phl, phl_role, sa);
+		if (mld != NULL) {
+			RTW_INFO("%s mld is not empty, sta is under disconnecting, so reject auth for sa "MAC_FMT"",
+				__func__, MAC_ARG(sa));
+			return _FAIL;
+		}
+
+		/* allocate a new one */
+		RTW_INFO("going to alloc stainfo for sa="MAC_FMT"\n",  MAC_ARG(sa));
+
+		/* ToDo CONFIG_RTW_MLD: replace sa with MLD address */
+		mld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter->phl_role, sa, DTYPE);
+		if (mld == NULL) {
+			RTW_INFO(" Exceed the upper limit of supported MLDs...\n");
+			status = _STATS_UNABLE_HANDLE_STA_;
+			goto auth_fail;
+		}
+		/* Allocate stainfo on the link where AP receives the auth req */
+		pstat = rtw_alloc_stainfo_sw(pstapriv, DTYPE, main_id, padapter_link->wrlink->id, sa);
+		if (pstat == NULL) {
+			RTW_INFO(" Exceed the upper limit of supported clients...\n");
+			rtw_free_mld_stainfo(padapter, mld);
+			status = _STATS_UNABLE_HANDLE_STA_;
+			goto auth_fail;
+		}
+		rtw_phl_link_mld_stainfo(mld, pstat->phl_sta);
+		pstat->state = WIFI_FW_AUTH_NULL;
+		pstat->auth_seq = 0;
+		/* pstat->flags = 0; */
+		/* pstat->capability = 0; */
+	} else {
+		if (ATOMIC_READ(&pstat->deleting) != 0) { /* sta is under disconnecting */
+			status = _STATS_REFUSED_TEMPORARILY_;
+			RTW_ERR("%s : Reject auth refused temporarily for sa "MAC_FMT", sta is deleting=%d\n",
+				__func__, MAC_ARG(sa), ATOMIC_READ(&pstat->deleting));
+			return _FAIL;
+		}
+
+		if (pstat->is_freeing) {
+			RTW_INFO(FUNC_ADPT_FMT" get sta "MAC_FMT" is scheduled to free\n", FUNC_ADPT_ARG(padapter), MAC_ARG(sa));
+			return _SUCCESS;
+		}
+
+#ifdef CONFIG_IEEE80211W
+		if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+			|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+		{
+
+			rtw_stapriv_asoc_list_lock(pstapriv);
+			if (rtw_is_list_empty(&pstat->asoc_list) == _FALSE) {
+				rtw_stapriv_asoc_list_del(pstapriv, pstat);
+				RTW_INFO("%s, asoc_list_cnt=%u\n", __func__, pstapriv->asoc_list_cnt);
+				if (pstat->expire_to > 0)
+					;/* TODO: STA re_auth within expire_to */
+			}
+			rtw_stapriv_asoc_list_unlock(pstapriv);
+
+			if (seq == 1)
+				; /* TODO: STA re_auth and auth timeout */
+
+		}
+	}
+
+#ifdef CONFIG_IEEE80211W
+	if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+		|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+	{
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+		if (pstat->is_freeing || rtw_is_list_empty(&pstat->hash_list)) {
+			_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+			RTW_INFO(FUNC_ADPT_FMT" sta "MAC_FMT" is scheduled to free\n", FUNC_ADPT_ARG(padapter), MAC_ARG(sa));
+			return _SUCCESS;
+		}
+		_rtw_spinlock_bh(&pstapriv->auth_list_lock);
+		if (rtw_is_list_empty(&pstat->auth_list)) {
+
+			rtw_list_insert_tail(&pstat->auth_list, &pstapriv->auth_list);
+			pstapriv->auth_list_cnt++;
+		}
+		_rtw_spinunlock_bh(&pstapriv->auth_list_lock);
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	}
+
+	if (pstat->auth_seq == 0)
+		pstat->expire_to = pstapriv->auth_to;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (GET_CFG80211_REPORT_MGMT(adapter_wdev_data(padapter), IEEE80211_STYPE_AUTH) == _TRUE) {
+		if ((algorithm == WLAN_AUTH_SAE) &&
+			(auth_mode == dot11AuthAlgrthm_8021X)) {
+			pstat->authalg = algorithm;
+
+			rtw_cfg80211_rx_mframe(padapter, precv_frame, NULL);
+			return _SUCCESS;
+		}
+	#ifdef CONFIG_RTW_80211R_AP
+		pstat->authalg = algorithm;
+		if (algorithm == WLAN_AUTH_FT) {
+			rtw_cfg80211_rx_mframe(padapter, precv_frame, "FTTA");
+			return _SUCCESS;
+		}
+	#endif
+	}
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	if ((pstat->auth_seq + 1) != seq) {
+		RTW_INFO("(1)auth rejected because out of seq [rx_seq=%d, exp_seq=%d]!\n",
+			 seq, pstat->auth_seq + 1);
+		status = _STATS_OUT_OF_AUTH_SEQ_;
+		goto auth_fail;
+	}
+
+	if (algorithm == 0 && (auth_mode == 0 || auth_mode == 2 || auth_mode == 3)) {
+		if (seq == 1) {
+#ifdef CONFIG_IEEE80211W
+			if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+				|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+			{
+				pstat->state &= ~WIFI_FW_AUTH_NULL;
+				pstat->state |= WIFI_FW_AUTH_SUCCESS;
+				pstat->expire_to = pstapriv->assoc_to;
+			}
+			pstat->authalg = algorithm;
+		} else {
+			RTW_INFO("(2)auth rejected because out of seq [rx_seq=%d, exp_seq=%d]!\n",
+				 seq, pstat->auth_seq + 1);
+			status = _STATS_OUT_OF_AUTH_SEQ_;
+			goto auth_fail;
+		}
+	} else { /* shared system or auto authentication */
+		if (seq == 1) {
+			/* prepare for the challenging txt... */
+
+			/* get_random_bytes((void *)pstat->chg_txt, 128); */ /* TODO: */
+			_rtw_memset((void *)pstat->chg_txt, 78, 128);
+#ifdef CONFIG_IEEE80211W
+			if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+				|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+			{
+				pstat->state &= ~WIFI_FW_AUTH_NULL;
+				pstat->state |= WIFI_FW_AUTH_STATE;
+			}
+			pstat->authalg = algorithm;
+			pstat->auth_seq = 2;
+		} else if (seq == 3) {
+			/* checking for challenging txt... */
+			RTW_INFO("checking for challenging txt...\n");
+
+			p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + 4 + _AUTH_IE_OFFSET_ , _CHLGETXT_IE_, (int *)&ie_len,
+				len - WLAN_HDR_A3_LEN - _AUTH_IE_OFFSET_ - 4);
+
+			if ((p == NULL) || (ie_len <= 0)) {
+				RTW_INFO("auth rejected because challenge failure!(1)\n");
+				status = _STATS_CHALLENGE_FAIL_;
+				goto auth_fail;
+			}
+
+			if (_rtw_memcmp((void *)(p + 2), pstat->chg_txt, 128)) {
+#ifdef CONFIG_IEEE80211W
+				if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+					|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+				{
+					pstat->state &= (~WIFI_FW_AUTH_STATE);
+					pstat->state |= WIFI_FW_AUTH_SUCCESS;
+					/* challenging txt is correct... */
+					pstat->expire_to =  pstapriv->assoc_to;
+				}
+			} else {
+				RTW_INFO("auth rejected because challenge failure!\n");
+				status = _STATS_CHALLENGE_FAIL_;
+				goto auth_fail;
+			}
+		} else {
+			RTW_INFO("(3)auth rejected because out of seq [rx_seq=%d, exp_seq=%d]!\n",
+				 seq, pstat->auth_seq + 1);
+			status = _STATS_OUT_OF_AUTH_SEQ_;
+			goto auth_fail;
+		}
+	}
+
+
+	/* Now, we are going to issue_auth... */
+	pstat->auth_seq = seq + 1;
+
+#ifdef CONFIG_NATIVEAP_MLME
+	issue_auth(padapter, pstat, (unsigned short)(_STATS_SUCCESSFUL_));
+#endif
+
+	if ((pstat->state & WIFI_FW_AUTH_SUCCESS) || (pstat->state & WIFI_FW_ASSOC_SUCCESS))
+		pstat->auth_seq = 0;
+
+
+	return _SUCCESS;
+
+auth_fail:
+#ifdef CONFIG_NATIVEAP_MLME
+	issue_auth(padapter, pstat, (unsigned short)status);
+#endif
+	/* Free sta_info, phl_sta, mld */
+	if (pstat) {
+		RTW_INFO("%s : fail, free  sta_info, phl_sta, mld, for sa="MAC_FMT"\n",
+			__func__, MAC_ARG(sa));
+		rtw_free_stainfo_sw(padapter , pstat);
+		mld = rtw_phl_get_mld_by_addr(phl, phl_role, sa);
+		rtw_phl_free_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), mld);
+	}
+#endif
+	return _FAIL;
+
+}
+
+unsigned int OnAuthClient(_adapter *padapter, union recv_frame *precv_frame)
+{
+	unsigned int	seq, len, status, algthm, offset;
+	unsigned char	*p;
+	unsigned int	go2asoc = 0;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint pkt_len = precv_frame->u.hdr.len;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_mlme_ext_info	*lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (GET_CFG80211_REPORT_MGMT(adapter_wdev_data(padapter), IEEE80211_STYPE_AUTH) == _TRUE) {
+		if (rtw_sec_chk_auth_type(padapter, MLME_AUTHTYPE_SAE)) {
+			if (rtw_cached_pmkid(padapter, get_my_bssid(&lmlmeinfo->network)) != -1) {
+				RTW_INFO("SAE: PMKSA cache entry found\n");
+				goto normal;
+			}
+			if (!rtw_cfg80211_rx_mframe(padapter, precv_frame, NULL)) {
+				offset = (GetPrivacy(pframe)) ? 4 : 0;
+				pmlmeinfo->auth_status = le16_to_cpu(*(unsigned short *)((SIZE_PTR)pframe + \
+					WLAN_HDR_A3_LEN + offset + 4));
+				if (pmlmeinfo->auth_status == WLAN_STATUS_ASSOC_REJECTED_TEMPORARILY) {
+					pmlmeinfo->auth_status = WLAN_STATUS_AUTH_TIMEOUT;
+					RTW_INFO("%s SAE WKRD auth reject, change status code: %d\n", __func__, pmlmeinfo->auth_status);
+				}
+				set_link_timer(pmlmeext, 1);
+				goto authclnt_fail;
+			}
+			return _SUCCESS;
+		}
+	}
+
+normal:
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	/* check A1 matches or not */
+	if (!rtw_match_self_addr(padapter, get_da(pframe)))
+		return _SUCCESS;
+
+	if (!(pmlmeinfo->state & WIFI_FW_AUTH_STATE) || pmlmeext->join_abort)
+		return _SUCCESS;
+
+	offset = (GetPrivacy(pframe)) ? 4 : 0;
+
+	algthm	= le16_to_cpu(*(unsigned short *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset));
+	seq	= le16_to_cpu(*(unsigned short *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset + 2));
+	status	= le16_to_cpu(*(unsigned short *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset + 4));
+
+	if (status != 0) {
+		RTW_INFO("clnt auth fail, status: %d\n", status);
+		if (status == 13) { /* && pmlmeinfo->auth_algo == dot11AuthAlgrthm_Auto) */
+			if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared)
+				pmlmeinfo->auth_algo = dot11AuthAlgrthm_Open;
+			else
+				pmlmeinfo->auth_algo = dot11AuthAlgrthm_Shared;
+			/* pmlmeinfo->reauth_count = 0; */
+		}
+
+		pmlmeinfo->auth_status = status;
+		set_link_timer(pmlmeext, 1);
+		goto authclnt_fail;
+	}
+
+	if (seq == 2) {
+		if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared) {
+			/* legendary shared system */
+			p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _AUTH_IE_OFFSET_, _CHLGETXT_IE_, (int *)&len,
+				pkt_len - WLAN_HDR_A3_LEN - _AUTH_IE_OFFSET_);
+
+			if (p == NULL) {
+				/* RTW_INFO("marc: no challenge text?\n"); */
+				goto authclnt_fail;
+			}
+
+			if (len > sizeof(pmlmeinfo->chg_txt)) {
+				goto authclnt_fail;
+			}
+
+			_rtw_memcpy((void *)(pmlmeinfo->chg_txt), (void *)(p + 2), len);
+			pmlmeinfo->auth_seq = 3;
+			issue_auth(padapter, NULL, 0);
+			set_link_timer(pmlmeext, REAUTH_TO);
+
+			return _SUCCESS;
+		} else {
+			/* open, or 802.11r FTAA system */
+			go2asoc = 1;
+		}
+	} else if (seq == 4) {
+		if (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared)
+			go2asoc = 1;
+		else
+			goto authclnt_fail;
+	} else {
+		/* this is also illegal */
+		/* RTW_INFO("marc: clnt auth failed due to illegal seq=%x\n", seq); */
+		goto authclnt_fail;
+	}
+
+	if (go2asoc) {
+#ifdef CONFIG_RTW_80211R
+		if (rtw_ft_update_auth_rsp_ies(padapter, pframe, pkt_len))
+			return _SUCCESS;
+#endif
+		RTW_PRINT("auth success, start assoc\n");
+		start_clnt_assoc(padapter);
+		return _SUCCESS;
+	}
+
+authclnt_fail:
+
+	/* pmlmeinfo->state &= ~(WIFI_FW_AUTH_STATE); */
+
+	return _FAIL;
+
+}
+
+static u8 rtw_deny_legacy_sta(_adapter *padapter, struct sta_info *pstat)
+{
+	struct link_mlme_ext_priv *pmlmeext = &pstat->padapter_link->mlmeextpriv;
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	u8 res = _FALSE;
+
+	sta_info_update(padapter, pstat);
+	rtw_ap_set_sta_wmode(padapter, pstat);
+
+	if (pmlmeext->cur_wireless_mode == WLAN_MD_11BGN) {
+		if ((pregpriv->deny_legacy == WLAN_MD_11BG) &&
+			!(pstat->phl_sta->wmode & WLAN_MD_11N))
+			/* 2.4G N only */
+			res = _TRUE;
+		else if ((pregpriv->deny_legacy == WLAN_MD_11B) &&
+				 (pstat->phl_sta->wmode == WLAN_MD_11B))
+			/* 2.4G G+N */
+			res = _TRUE;
+	} else if (pmlmeext->cur_wireless_mode == WLAN_MD_11BG) {
+		if ((pregpriv->deny_legacy == WLAN_MD_11B) &&
+			(pstat->phl_sta->wmode == WLAN_MD_11B))
+			/* 2.4G G only */
+			res = _TRUE;
+	} else if (pmlmeext->cur_wireless_mode == WLAN_MD_11A_AC) {
+		if ((pregpriv->deny_legacy == WLAN_MD_11AN) &&
+			!(pstat->phl_sta->wmode & WLAN_MD_11AC))
+			/* 5G AC only */
+			res = _TRUE;
+		else if ((pregpriv->deny_legacy == WLAN_MD_11A) &&
+				 (pstat->phl_sta->wmode == WLAN_MD_11A))
+			/* 5G N+AC */
+			res = _TRUE;
+	} else if (pmlmeext->cur_wireless_mode == WLAN_MD_11AN) {
+		if ((pregpriv->deny_legacy == WLAN_MD_11A) &&
+			(pstat->phl_sta->wmode == WLAN_MD_11A))
+			/* 5G N only */
+			res = _TRUE;
+	}
+
+	return res;
+}
+
+unsigned int OnAssocReq(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_AP_MODE
+	u16 listen_interval;
+	struct rtw_ieee802_11_elems elems;
+	struct sta_info *pstat;
+	unsigned char		reassoc, *pos;
+	int		left;
+	unsigned short		status = _STATS_SUCCESSFUL_;
+	unsigned short		frame_type, ie_offset = 0;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	WLAN_BSSID_EX	*cur = &(pmlmeinfo->dev_network);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint pkt_len = precv_frame->u.hdr.len;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	u8 p2p_status_code = P2P_STATUS_SUCCESS;
+	u8 *p2pie;
+	u32 p2pielen = 0;
+#endif /* CONFIG_P2P */
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) &&
+	    rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_LINKING | WIFI_UNDER_SURVEY)) {
+		/* don't process assoc request; */
+		return _SUCCESS;
+	}
+#endif /* CONFIG_CONCURRENT_MODE */
+
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
+		return _FAIL;
+
+	if (rtw_check_invalid_mac_address(get_addr2_ptr(pframe), _FALSE)) {
+		RTW_INFO("%s : reject invalid ASSOC-req "MAC_FMT"\n", 
+			__func__, MAC_ARG(get_addr2_ptr(pframe)));
+		return _FAIL;
+	}
+
+	frame_type = get_frame_sub_type(pframe);
+	if (frame_type == WIFI_ASSOCREQ) {
+		reassoc = 0;
+		ie_offset = _ASOCREQ_IE_OFFSET_;
+	} else { /* WIFI_REASSOCREQ */
+		reassoc = 1;
+		ie_offset = _REASOCREQ_IE_OFFSET_;
+	}
+
+
+	if (pkt_len < IEEE80211_3ADDR_LEN + ie_offset) {
+		RTW_INFO("handle_assoc(reassoc=%d) - too short payload (len=%lu)"
+			 "\n", reassoc, (unsigned long)pkt_len);
+		return _FAIL;
+	}
+
+	pstat = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+	if (pstat == (struct sta_info *)NULL) {
+		status = _RSON_CLS2_;
+		goto asoc_class2_error;
+	}
+	if (pstat->is_freeing) {
+		RTW_INFO(FUNC_ADPT_FMT" get sta "MAC_FMT" is scheduled to free\n", FUNC_ADPT_ARG(padapter), MAC_ARG(get_addr2_ptr(pframe)));
+		status = _RSON_AUTH_NO_LONGER_VALID_;
+		goto asoc_class2_error;
+	}
+
+#ifdef RTW_PHL_TEST_FPGA
+	DBGP(" [%s][%d] force assoc \n", __FUNCTION__, __LINE__);
+	pstat->state = WIFI_ASOC_STATE;
+	pstat->state |= WIFI_FW_ASSOC_SUCCESS;
+	issue_asocrsp(padapter, status, pstat, WIFI_ASSOCRSP);
+	return _SUCCESS;
+#endif
+
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	if (pstat->authalg == WLAN_AUTH_SAE) {
+		/* WPA3-SAE */
+		if (((pstat->state) & WIFI_FW_AUTH_NULL)) {
+			/* TODO:
+			   Queue AssocReq and Proccess
+			   by external auth trigger. */
+			RTW_INFO("%s: wait external auth trigger\n", __func__);
+			return _SUCCESS;
+		}
+	}
+
+	/* check if this stat has been successfully authenticated/assocated */
+	if (!((pstat->state) & WIFI_FW_AUTH_SUCCESS)) {
+		if (!((pstat->state) & WIFI_FW_ASSOC_SUCCESS)) {
+			status = _RSON_CLS2_;
+			goto asoc_class2_error;
+		} else {
+			pstat->state &= (~WIFI_FW_ASSOC_SUCCESS);
+			pstat->state |= WIFI_FW_ASSOC_STATE;
+		}
+	} else {
+		pstat->state &= (~WIFI_FW_AUTH_SUCCESS);
+		pstat->state |= WIFI_FW_ASSOC_STATE;
+	}
+
+#if 0/* todo:tkip_countermeasures */
+	if (hapd->tkip_countermeasures) {
+		resp = WLAN_REASON_MICHAEL_MIC_FAILURE;
+		goto fail;
+	}
+#endif
+	/*GEORGIA_TODO_MOVE_CODE_TO_DBG_SYS*/
+	if (rtw_ap_linking_test_force_asoc_fail()) {
+		status = rtw_ap_linking_test_force_asoc_fail();
+		RTW_INFO(FUNC_ADPT_FMT" force asoc fail with status:%u\n"
+			, FUNC_ADPT_ARG(padapter), status);
+		goto OnAssocReqFail;
+	}
+
+	/* now parse all ieee802_11 ie to point to elems */
+	left = pkt_len - (IEEE80211_3ADDR_LEN + ie_offset);
+	pos = pframe + (IEEE80211_3ADDR_LEN + ie_offset);
+	if (rtw_ieee802_11_parse_elems(pos, left, &elems, 1) == ParseFailed) {
+		RTW_INFO("STA " MAC_FMT " sent invalid association request\n",
+			 MAC_ARG(pstat->phl_sta->mac_addr));
+		status = _STATS_FAILURE_;
+		goto OnAssocReqFail;
+	}
+
+	rtw_ap_parse_sta_capability(padapter, pstat, pframe + WLAN_HDR_A3_LEN);
+
+	listen_interval = RTW_GET_LE16(pframe + WLAN_HDR_A3_LEN + 2);
+#if 0/* todo: */
+	/* check listen_interval */
+	if (listen_interval > hapd->conf->max_listen_interval) {
+		hostapd_logger(hapd, mgmt->sa, HOSTAPD_MODULE_IEEE80211,
+			       HOSTAPD_LEVEL_DEBUG,
+			       "Too large Listen Interval (%d)",
+			       listen_interval);
+		resp = WLAN_STATUS_ASSOC_DENIED_LISTEN_INT_TOO_LARGE;
+		goto fail;
+	}
+
+	pstat->listen_interval = listen_interval;
+#endif
+
+	/* now we should check all the fields... */
+	/* checking SSID */
+	if (elems.ssid == NULL
+		|| elems.ssid_len == 0
+		|| elems.ssid_len != cur->Ssid.SsidLength
+		|| _rtw_memcmp(elems.ssid, cur->Ssid.Ssid, cur->Ssid.SsidLength) == _FALSE
+	) {
+		status = _STATS_FAILURE_;
+		goto OnAssocReqFail;
+	}
+
+	/* (Extended) Supported rates */
+	status = rtw_ap_parse_sta_supported_rates(padapter, pstat
+		, pframe + WLAN_HDR_A3_LEN + ie_offset, pkt_len - WLAN_HDR_A3_LEN - ie_offset);
+	if (status != _STATS_SUCCESSFUL_)
+		goto OnAssocReqFail;
+
+	/* check RSN/WPA/WPS */
+	status = rtw_ap_parse_sta_security_ie(padapter, pstat, &elems);
+	if (status != _STATS_SUCCESSFUL_)
+		goto OnAssocReqFail;
+
+	/* check if there is WMM IE & support WWM-PS */
+	rtw_ap_parse_sta_wmm_ie(padapter, pstat
+		, pframe + WLAN_HDR_A3_LEN + ie_offset, pkt_len - WLAN_HDR_A3_LEN - ie_offset);
+
+#ifdef CONFIG_RTS_FULL_BW
+	/*check vendor IE*/
+	rtw_parse_sta_vendor_ie_8812(padapter, pstat
+		, pframe + WLAN_HDR_A3_LEN + ie_offset, pkt_len - WLAN_HDR_A3_LEN - ie_offset);
+#endif/*CONFIG_RTS_FULL_BW*/
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	if (elems.tbtx_cap && elems.tbtx_cap_len != 0) {
+		if(rtw_is_tbtx_capabilty(elems.tbtx_cap, elems.tbtx_cap_len))
+			pstat->tbtx_enable = _TRUE;
+	}
+
+#endif
+
+	rtw_ap_parse_sta_ht_ie(padapter, pstat, &elems);
+	rtw_ap_parse_sta_vht_ie(padapter, pstat, &elems);
+	rtw_ap_parse_sta_he_ie(padapter, pstat, &elems);
+
+	if (pregpriv->deny_legacy && rtw_deny_legacy_sta(padapter, pstat)) {
+		RTW_INFO("Deny legacy STA " MAC_FMT " association\n",
+			 MAC_ARG(pstat->phl_sta->mac_addr));
+		status = WLAN_STATUS_UNSPECIFIED_FAILURE;
+		goto OnAssocReqFail;
+	}
+
+	if (((pstat->flags & WLAN_STA_HT) || (pstat->flags & WLAN_STA_VHT) || (pstat->flags & WLAN_STA_HE)) &&
+	    ((pstat->wpa2_pairwise_cipher & WPA_CIPHER_TKIP) ||
+	     (pstat->wpa_pairwise_cipher & WPA_CIPHER_TKIP))) {
+
+		RTW_INFO("HT/VHT/HE: " MAC_FMT " TKIP association\n", MAC_ARG(pstat->phl_sta->mac_addr));
+
+		pstat->flags &= ~WLAN_STA_HT;
+		pstat->flags &= ~WLAN_STA_VHT;
+		pstat->flags &= ~WLAN_STA_HE;
+	}
+
+#ifdef CONFIG_P2P
+	pstat->is_p2p_device = _FALSE;
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+		p2pie = rtw_get_p2p_ie(pframe + WLAN_HDR_A3_LEN + ie_offset , pkt_len - WLAN_HDR_A3_LEN - ie_offset , NULL, &p2pielen);
+		if (p2pie) {
+			pstat->is_p2p_device = _TRUE;
+			p2p_status_code = (u8)process_assoc_req_p2p_ie(pwdinfo, pframe, pkt_len, pstat);
+			if (p2p_status_code > 0) {
+				pstat->p2p_status_code = p2p_status_code;
+				status = _STATS_CAP_FAIL_;
+				goto OnAssocReqFail;
+			}
+		}
+#ifdef CONFIG_WFD
+		rtw_process_wfd_ies(padapter, pframe + WLAN_HDR_A3_LEN + ie_offset, pkt_len - WLAN_HDR_A3_LEN - ie_offset, __func__);
+#endif
+	}
+	pstat->p2p_status_code = p2p_status_code;
+#endif /* CONFIG_P2P */
+
+	rtw_ap_parse_sta_multi_ap_ie(padapter, pstat, pos, left);
+
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_process_assoc_req(padapter,
+		(pframe + WLAN_HDR_A3_LEN + ie_offset),
+		(pkt_len - WLAN_HDR_A3_LEN - ie_offset));
+#endif
+
+	/* TODO: identify_proprietary_vendor_ie(); */
+	/* Realtek proprietary IE */
+	/* identify if this is Broadcom sta */
+	/* identify if this is ralink sta */
+	/* Customer proprietary IE */
+#ifdef PRIVATE_R
+	if (pmlmepriv->vendor_ie_filter_enable) {
+		if (!check_vendor_ie(padapter, pframe + WLAN_HDR_A3_LEN + _ASOCREQ_IE_OFFSET_,
+			pkt_len - WLAN_HDR_A3_LEN - _ASOCREQ_IE_OFFSET_)) {
+			status = _STATS_UNABLE_HANDLE_STA_;
+			goto OnAssocReqFail;
+		}
+	}
+#endif
+
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	rtw_ap_parse_sta_rm_en_cap(padapter, pstat, &elems);
+#endif
+
+#ifdef CONFIG_RTW_MBO
+	if (pmlmepriv->mbopriv.enable == _TRUE) {
+		if(pmlmepriv->mbopriv.assoc_disallow != 0){
+			RTW_INFO("Reject STA " MAC_FMT ":MBO association disallowed\n",
+			 MAC_ARG(pstat->phl_sta->mac_addr));
+			status = WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA;
+			goto OnAssocReqFail;
+		}
+		rtw_ap_parse_sta_mbo_element(padapter, pstat,
+						pframe + WLAN_HDR_A3_LEN + ie_offset,
+						pkt_len - WLAN_HDR_A3_LEN - ie_offset);
+	}
+#endif /*CONFIG_RTW_MBO*/
+
+	/* AID assignment */
+	if (pstat->phl_sta->aid > 0)
+		RTW_INFO(FUNC_ADPT_FMT" old AID=%d\n", FUNC_ADPT_ARG(padapter), pstat->phl_sta->aid);
+	else {
+		if (!rtw_aid_alloc(padapter, pstat)) {
+			RTW_INFO(FUNC_ADPT_FMT" no room for more AIDs\n", FUNC_ADPT_ARG(padapter));
+			status = WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA;
+			goto OnAssocReqFail;
+		}
+		RTW_INFO(FUNC_ADPT_FMT" allocate new AID=%d\n", FUNC_ADPT_ARG(padapter), pstat->phl_sta->aid);
+	}
+
+	pstat->state &= (~WIFI_FW_ASSOC_STATE);
+	pstat->state |= WIFI_FW_ASSOC_SUCCESS;
+	/* RTW_INFO("==================%s, %d,  (%x), bpairwise_key_installed=%d, MAC:"MAC_FMT"\n"
+	, __func__, __LINE__, pstat->state, pstat->bpairwise_key_installed, MAC_ARG(pstat->phl_sta->mac_addr)); */
+#ifdef CONFIG_IEEE80211W
+	if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+		|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+	{
+		_rtw_spinlock_bh(&pstapriv->auth_list_lock);
+		if (!rtw_is_list_empty(&pstat->auth_list)) {
+			rtw_list_delete(&pstat->auth_list);
+			pstapriv->auth_list_cnt--;
+		}
+		_rtw_spinunlock_bh(&pstapriv->auth_list_lock);
+
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+		if (pstat->is_freeing || rtw_is_list_empty(&pstat->hash_list)) {
+			_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+			RTW_INFO(FUNC_ADPT_FMT" sta "MAC_FMT" is scheduled to free\n", FUNC_ADPT_ARG(padapter), MAC_ARG(get_addr2_ptr(pframe)));
+			status = _RSON_AUTH_NO_LONGER_VALID_;
+			goto asoc_class2_error;
+		}
+		rtw_stapriv_asoc_list_lock(pstapriv);
+		if (rtw_is_list_empty(&pstat->asoc_list)) {
+			pstat->expire_to = pstapriv->expire_to;
+			rtw_stapriv_asoc_list_add(pstapriv, pstat);
+		}
+		rtw_stapriv_asoc_list_unlock(pstapriv);
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	}
+
+	/* now the station is qualified to join our BSS...	 */
+	if (pstat && (pstat->state & WIFI_FW_ASSOC_SUCCESS) && (_STATS_SUCCESSFUL_ == status)) {
+#ifdef CONFIG_NATIVEAP_MLME
+#ifdef CONFIG_IEEE80211W
+		if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+			|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+		{
+			/* .1 bss_cap_update & sta_info_update */
+			bss_cap_update_on_sta_join(padapter, pstat);
+			sta_info_update(padapter, pstat);
+		}
+#ifdef CONFIG_IEEE80211W
+		if (pstat->bpairwise_key_installed == _TRUE && (pstat->flags & WLAN_STA_MFP))
+			status = _STATS_REFUSED_TEMPORARILY_;
+#endif /* CONFIG_IEEE80211W */
+#ifdef CONFIG_RTW_MBO
+		if (rtw_mbo_ap_assoc_disallow(padapter))
+			status = _STATS_UNABLE_HANDLE_STA_;
+#endif
+#ifdef CONFIG_RTW_80211R_AP
+		if ((pstat && (pstat->authalg != WLAN_AUTH_FT)))
+#endif
+		{
+			/* .2 issue assoc rsp before notify station join event. */
+			if (frame_type == WIFI_ASSOCREQ)
+				issue_asocrsp(padapter, status, pstat, WIFI_ASSOCRSP);
+			else
+				issue_asocrsp(padapter, status, pstat, WIFI_REASSOCRSP);
+		}
+
+#ifdef CONFIG_IOCTL_CFG80211
+		_rtw_spinlock_bh(&pstat->lock);
+		if (pstat->passoc_req) {
+			rtw_mfree(pstat->passoc_req, pstat->assoc_req_len);
+			pstat->passoc_req = NULL;
+			pstat->assoc_req_len = 0;
+		}
+
+		pstat->passoc_req =  rtw_zmalloc(pkt_len);
+		if (pstat->passoc_req) {
+			_rtw_memcpy(pstat->passoc_req, pframe, pkt_len);
+			pstat->assoc_req_len = pkt_len;
+		}
+		_rtw_spinunlock_bh(&pstat->lock);
+#endif /* CONFIG_IOCTL_CFG80211 */
+#ifdef CONFIG_IEEE80211W
+		if ((pstat->bpairwise_key_installed != _TRUE && (pstat->flags & WLAN_STA_MFP))
+			|| !(pstat->flags & WLAN_STA_MFP))
+#endif /* CONFIG_IEEE80211W */
+		{
+			/* .3-(1) report sta add event */
+			report_add_sta_event(padapter, pstat->phl_sta->mac_addr);
+		}
+#ifdef CONFIG_IEEE80211W
+		if (pstat->bpairwise_key_installed == _TRUE && (pstat->flags & WLAN_STA_MFP)) {
+			RTW_INFO(MAC_FMT"\n", MAC_ARG(pstat->phl_sta->mac_addr));
+			issue_action_SA_Query(padapter, pstat->padapter_link,
+					pstat->phl_sta->mac_addr, 0, 0, IEEE80211W_RIGHT_KEY);
+		}
+#endif /* CONFIG_IEEE80211W */
+#endif /* CONFIG_NATIVEAP_MLME */
+	}
+
+	return _SUCCESS;
+
+asoc_class2_error:
+
+#ifdef CONFIG_NATIVEAP_MLME
+	issue_deauth(padapter, (void *)get_addr2_ptr(pframe), status);
+#endif
+
+	return _FAIL;
+
+OnAssocReqFail:
+
+
+#ifdef CONFIG_NATIVEAP_MLME
+	/* pstat->phl_sta->aid = 0; */
+	if (frame_type == WIFI_ASSOCREQ)
+		issue_asocrsp(padapter, status, pstat, WIFI_ASSOCRSP);
+	else
+		issue_asocrsp(padapter, status, pstat, WIFI_REASSOCRSP);
+#endif
+
+	/* Free sta_info, phl_sta, mld */
+	if (pstat) {
+		struct rtw_phl_mld_t *mld = NULL;
+		void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+		struct rtw_wifi_role_t *phl_role = padapter->phl_role;
+
+		RTW_INFO("%s : fail, free  sta_info, phl_sta, mld, for sa="MAC_FMT"\n",
+			__func__, MAC_ARG(get_addr2_ptr(pframe)));
+		rtw_free_stainfo_sw(padapter , pstat);
+		mld = rtw_phl_get_mld_by_addr(phl, phl_role, get_addr2_ptr(pframe));
+		rtw_phl_free_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), mld);
+	}
+
+#endif /* CONFIG_AP_MODE */
+
+	return _FAIL;
+
+}
+
+#if defined(CONFIG_LAYER2_ROAMING) && (defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM))
+void rtw_roam_nb_discover(_adapter *padapter, u8 bfroce)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info *pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	u8 nb_req_issue = _FALSE;
+
+	if (!check_fwstate(pmlmepriv, WIFI_ASOC_STATE))
+		return;
+
+	if (!rtw_chk_roam_flags(padapter, RTW_ROAM_ACTIVE))
+		return;
+
+	psta = rtw_get_stainfo(pstapriv, pmlmeinfo->network.MacAddress);
+	if (!psta)
+		return;
+
+	if (bfroce || (!pmlmepriv->nb_info.nb_rpt_is_same))
+		nb_req_issue = _TRUE;
+
+	if (nb_req_issue && (psta->rm_en_cap[0] & RTW_RRM_NB_RPT_EN))
+		rm_add_nb_req(padapter, psta);
+}
+#endif
+
+void rtw_ie_handler(struct _ADAPTER *padapter, struct _ADAPTER_LINK *padapter_link,
+				u8 *ie, u32 ie_len)
+{
+	int i;
+	struct _NDIS_802_11_VARIABLE_IEs *p = NULL;
+#ifdef CONFIG_WAPI_SUPPORT
+	PNDIS_802_11_VARIABLE_IEs pWapiIE = NULL;
+#endif
+
+	if (!ie || (ie_len == 0))
+		return;
+
+	for (i = 0; i < ie_len;) {
+		p = (struct _NDIS_802_11_VARIABLE_IEs *)(ie + i);
+		switch (p->ElementID) {
+#ifdef CONFIG_WAPI_SUPPORT
+		case _WAPI_IE_:
+			pWapiIE = p;
+			break;
+#endif
+
+		case _VENDOR_SPECIFIC_IE_:
+			if (_rtw_memcmp(p->data, WMM_PARA_OUI, 6))	/* WMM */
+				WMM_param_handler(padapter, padapter_link, p);
+#if defined(CONFIG_P2P) && defined(CONFIG_WFD)
+			else if (_rtw_memcmp(p->data, WFD_OUI, 4))		/* WFD */
+				rtw_process_wfd_ie(padapter, (u8 *)p, p->Length, __func__);
+#endif
+			break;
+		case _HT_CAPABILITY_IE_:	/* HT caps */
+			HT_caps_handler(padapter, padapter_link, p);
+#ifdef PRIVATE_R
+			HT_caps_handler_infra_ap(padapter, padapter_link, p);
+#endif /* PRIVATE_R */
+			break;
+
+		case _HT_EXTRA_INFO_IE_:	/* HT info */
+			HT_info_handler(padapter, padapter_link, p);
+			break;
+
+#ifdef CONFIG_80211AC_VHT
+		case EID_VHTCapability:
+			VHT_caps_handler(padapter, padapter_link, p);
+#ifdef PRIVATE_R
+			VHT_caps_handler_infra_ap(padapter, padapter_link, p);
+#endif /* PRIVATE_R */
+			break;
+
+		case EID_VHTOperation:
+			VHT_operation_handler(padapter, padapter_link, p);
+			break;
+#endif
+
+		case WLAN_EID_EXTENSION:
+#ifdef CONFIG_80211AX_HE
+			if (p->data[0] == WLAN_EID_EXTENSION_HE_CAPABILITY)
+				HE_caps_handler(padapter, padapter_link, p);
+			else if (p->data[0] == WLAN_EID_EXTENSION_HE_OPERATION)
+				HE_operation_handler(padapter, padapter_link, p);
+			else if (p->data[0] == WLAN_EID_EXTENSION_HE_MU_EDCA)
+				HE_mu_edca_handler(padapter, padapter_link, p, _TRUE);
+			else if (p->data[0] == WLAN_EID_EXT_HE_6G_CAP)
+				HE_6g_bandcap_handler(padapter, padapter_link, p);
+#endif
+			break;
+		case _ERPINFO_IE_:
+			ERP_IE_handler(padapter, padapter_link, p);
+			break;
+#ifdef CONFIG_TDLS
+		case WLAN_EID_EXT_CAP:
+			if (check_ap_tdls_prohibited(p->data, p->Length) == _TRUE)
+				padapter->tdlsinfo.ap_prohibited = _TRUE;
+			if (check_ap_tdls_ch_switching_prohibited(p->data, p->Length) == _TRUE)
+				padapter->tdlsinfo.ch_switch_prohibited = _TRUE;
+			break;
+#endif /* CONFIG_TDLS */
+
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+		case _EID_RRM_EN_CAP_IE_: /* ToDo CONFIG_RTW_MLD: per link? */
+			RM_IE_handler(padapter, p);
+			break;
+#endif
+
+#ifdef PRIVATE_R
+		/* Infra mode, used to store AP's info , Parse the supported rates from AssocRsp */
+		case _SUPPORTEDRATES_IE_:
+			Supported_rate_infra_ap(padapter, padapter_link, p);
+			break;
+
+		case _EXT_SUPPORTEDRATES_IE_:
+			Extended_Supported_rate_infra_ap(padapter, padapter_link, p);
+			break;
+#endif /* PRIVATE_R */
+		default:
+			break;
+		}
+
+		i += (p->Length + 2);
+	}
+}
+
+unsigned int OnAssocRsp(_adapter *padapter, union recv_frame *precv_frame)
+{
+	uint i;
+	int res;
+	unsigned short status;
+	PNDIS_802_11_VARIABLE_IEs pIE = NULL;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	/* WLAN_BSSID_EX 		*cur_network = &(pmlmeinfo->network); */
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint pkt_len = precv_frame->u.hdr.len;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_mlme_ext_info *lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+#ifdef CONFIG_RTW_FSM
+	struct wlan_network *pnetwork = &padapter_link->mlmepriv.cur_network;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+#endif
+#ifdef CONFIG_80211BE_EHT
+	struct mlo_priv *pmlopriv = &pmlmepriv->mlopriv;
+#endif
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/* check A1 matches or not */
+	if (!rtw_match_self_addr(padapter, get_da(pframe)))
+		return _SUCCESS;
+
+	if (!(pmlmeinfo->state & (WIFI_FW_AUTH_SUCCESS | WIFI_FW_ASSOC_STATE)) || pmlmeext->join_abort)
+		return _SUCCESS;
+
+	if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)
+		return _SUCCESS;
+
+	cancel_link_timer(pmlmeext); /*_cancel_timer_ex(&pmlmeext->link_timer);*/
+
+	/* status */
+	status = le16_to_cpu(*(unsigned short *)(pframe + WLAN_HDR_A3_LEN + 2));
+	if (status > 0) {
+		RTW_INFO("assoc reject, status code: %d\n", status);
+		if (rtw_sec_chk_auth_type(padapter, MLME_AUTHTYPE_SAE) &&
+			(status == WLAN_STATUS_ASSOC_REJECTED_TEMPORARILY)) {
+			status = WLAN_STATUS_AUTH_TIMEOUT;
+			RTW_INFO("%s SAE WKRD assoc reject, change status code: %d\n", __func__, status);
+		}
+		pmlmeinfo->state = WIFI_FW_NULL_STATE;
+		res = -4;
+		goto report_assoc_result;
+	}
+	/* Connection is accepted */
+	padapter_link->mlmepriv.is_accepted = _TRUE;
+
+	/* get capabilities */
+	lmlmeinfo->capability = le16_to_cpu(*(unsigned short *)(pframe + WLAN_HDR_A3_LEN));
+
+	/* set slot time */
+	lmlmeinfo->slotTime = (padapter_link->mlmeextpriv.mlmext_info.capability & BIT(10)) ? 9 : 20;
+
+	/* AID assignment move to rtw_joinbss_update_stainfo */
+	res = (int)(le16_to_cpu(*(unsigned short *)(pframe + WLAN_HDR_A3_LEN + 4)) & 0x3fff);
+
+#ifdef RTW_PHL_TEST_FPGA
+	res = 1;
+#endif
+
+	/* check aid value */
+	if (res < 1 || res > 2007) {
+		RTW_INFO("assoc reject, aid: %d\n", res);
+		pmlmeinfo->state = WIFI_FW_NULL_STATE;
+		res = -4;
+		goto report_assoc_result;
+	}
+
+	/* following are moved to join event callback function */
+	/* to handle HT, WMM, rate adaptive, update MAC reg */
+	/* for not to handle the synchronous IO in the tasklet */
+
+	rtw_ie_handler(padapter, padapter_link,
+				pframe + 6 + WLAN_HDR_A3_LEN,
+				pkt_len - 6 - WLAN_HDR_A3_LEN);
+
+#ifdef CONFIG_80211BE_EHT
+	if (pmlopriv->mlo_option)
+		rtw_eht_ml_ies_handler(padapter, padapter_link,
+				pframe + 6 + WLAN_HDR_A3_LEN,
+				pkt_len - 6 - WLAN_HDR_A3_LEN);
+#endif
+
+#ifdef CONFIG_WAPI_SUPPORT
+	rtw_wapi_on_assoc_ok(padapter, pIE);
+#endif
+
+	pmlmeinfo->state &= (~WIFI_FW_ASSOC_STATE);
+	pmlmeinfo->state |= WIFI_FW_ASSOC_SUCCESS;
+
+	/* Update Basic Rate Table for spec, 2010-12-28 , by thomas */
+	UpdateBrateTbl(padapter, lmlmeinfo->network.SupportedRates);
+
+report_assoc_result:
+	if (res > 0)
+		rtw_buf_update(&pmlmepriv->assoc_rsp, &pmlmepriv->assoc_rsp_len, pframe, pkt_len);
+	else
+		rtw_buf_free(&pmlmepriv->assoc_rsp, &pmlmepriv->assoc_rsp_len);
+
+	report_join_res(padapter, res, status);
+
+#if defined(CONFIG_LAYER2_ROAMING) && (defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM))
+	rtw_roam_nb_discover(padapter, _TRUE);
+#endif
+	return _SUCCESS;
+}
+
+unsigned int OnDeAuth(_adapter *padapter, union recv_frame *precv_frame)
+{
+	unsigned short	reason;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	bool active = _FALSE;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+
+	/* check A3 */
+	if (!(rtw_match_bssid_ext(padapter, GetAddr3Ptr(pframe))))
+		return _SUCCESS;
+
+	RTW_INFO(FUNC_ADPT_FMT" - Start to Disconnect\n", FUNC_ADPT_ARG(padapter));
+
+	reason = le16_to_cpu(*(unsigned short *)(pframe + WLAN_HDR_A3_LEN));
+
+#ifdef CONFIG_WOWLAN
+	if (adapter_to_pwrctl(padapter)->bInSuspend == _TRUE) {
+		pmlmeinfo->wifi_reason_code = reason;
+
+		RTW_INFO("[%s][wow] postpone disconnection after resume procedure finished, reason = %hu.\n",
+			 __func__, reason);
+
+		return _SUCCESS;
+	}
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_AP_MODE
+	if (MLME_IS_AP(padapter)) {
+		struct sta_info *psta;
+		struct sta_priv *pstapriv = &padapter->stapriv;
+
+		RTW_PRINT(FUNC_ADPT_FMT" reason=%u, ta=" MAC_FMT "\n"
+			, FUNC_ADPT_ARG(padapter), reason, MAC_ARG(get_addr2_ptr(pframe)));
+
+		psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+		if (psta) {
+			u8 updated = _FALSE;
+
+			rtw_stapriv_asoc_list_lock(pstapriv);
+			if (rtw_is_list_empty(&psta->asoc_list) == _FALSE) {
+				rtw_stapriv_asoc_list_del(pstapriv, psta);
+				RTW_INFO("%s, asoc_list_cnt=%d\n", __func__, pstapriv->asoc_list_cnt);
+
+#ifdef CONFIG_IEEE80211W
+				/* pmf: 4.3.3.2 */
+				if (psta->flags & WLAN_STA_MFP) {
+					psta->flags &= ~WLAN_STA_MFP;
+					reason = _RSON_CLS2_;
+					active = _TRUE;
+				}
+#endif
+				ATOMIC_INC(&psta->deleting);
+				RTW_INFO("Set sta_info deleting=%u\n", ATOMIC_READ(&psta->deleting));
+				updated = ap_free_sta(padapter, psta, active, 0, reason, _TRUE);
+
+			}
+			rtw_stapriv_asoc_list_unlock(pstapriv);
+
+			associated_clients_update(padapter, updated, STA_INFO_UPDATE_ALL);
+		}
+
+
+		return _SUCCESS;
+	} else
+#endif
+	if (!MLME_IS_MESH(padapter)) {
+		int	ignore_received_deauth = 0;
+
+		/*	Commented by Albert 20130604 */
+		/*	Before sending the auth frame to start the STA/GC mode connection with AP/GO,  */
+		/*	we will send the deauth first. */
+		/*	However, the Win8.1 with BRCM Wi-Fi will send the deauth with reason code 6 to us after receieving our deauth. */
+		/*	Added the following code to avoid this case. */
+		if ((pmlmeinfo->state & WIFI_FW_AUTH_STATE) ||
+		    (pmlmeinfo->state & WIFI_FW_ASSOC_STATE)) {
+			if (reason == WLAN_REASON_CLASS2_FRAME_FROM_NONAUTH_STA)
+				ignore_received_deauth = 1;
+		}
+
+#ifdef CONFIG_RTW_WNM
+		if (!rtw_wnm_try_btm_roam_imnt(padapter))
+			ignore_received_deauth = 1;
+#endif
+		RTW_PRINT(FUNC_ADPT_FMT" reason=%u, ta=" MAC_FMT ", ignore=%d\n"
+			, FUNC_ADPT_ARG(padapter), reason, MAC_ARG(get_addr2_ptr(pframe)), ignore_received_deauth);
+
+		if (0 == ignore_received_deauth) {
+			receive_disconnect(padapter, get_addr2_ptr(pframe), reason, _FALSE);
+			pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DEAUTH;
+			pmlmeinfo->wifi_reason_code = reason;
+		}
+	}
+	pmlmepriv->LinkDetectInfo.bBusyTraffic = _FALSE;
+	return _SUCCESS;
+
+}
+
+unsigned int OnDisassoc(_adapter *padapter, union recv_frame *precv_frame)
+{
+	unsigned short	reason;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	bool active = _FALSE;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+
+	/* check A3 */
+	if (!(rtw_match_bssid_ext(padapter, GetAddr3Ptr(pframe))))
+		return _SUCCESS;
+
+	RTW_INFO(FUNC_ADPT_FMT" - Start to Disconnect\n", FUNC_ADPT_ARG(padapter));
+
+	reason = le16_to_cpu(*(unsigned short *)(pframe + WLAN_HDR_A3_LEN));
+
+#ifdef CONFIG_WOWLAN
+        if (adapter_to_pwrctl(padapter)->bInSuspend == _TRUE) {
+                pmlmeinfo->wifi_reason_code = reason;
+
+                RTW_INFO("[%s][wow] postpone disconnection after resume procedure finished, reason = %hu.\n",
+                         __func__, reason);
+
+                return _SUCCESS;
+        }
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_AP_MODE
+	if (MLME_IS_AP(padapter)) {
+		struct sta_info *psta;
+		struct sta_priv *pstapriv = &padapter->stapriv;
+
+		/* _rtw_spinlock_bh(&(pstapriv->sta_hash_lock));	 */
+		/* rtw_free_stainfo(padapter, psta); */
+		/* _rtw_spinunlock_bh(&(pstapriv->sta_hash_lock));		 */
+
+		RTW_PRINT(FUNC_ADPT_FMT" reason=%u, ta=" MAC_FMT "\n"
+			, FUNC_ADPT_ARG(padapter), reason, MAC_ARG(get_addr2_ptr(pframe)));
+
+		psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+		if (psta) {
+			u8 updated = _FALSE;
+
+			rtw_stapriv_asoc_list_lock(pstapriv);
+			if (rtw_is_list_empty(&psta->asoc_list) == _FALSE) {
+				rtw_stapriv_asoc_list_del(pstapriv, psta);
+				RTW_INFO("%s, asoc_list_cnt=%d\n", __func__, pstapriv->asoc_list_cnt);
+
+#ifdef CONFIG_IEEE80211W
+				/* pmf: 4.3.3.1 */
+				if (psta->flags & WLAN_STA_MFP) {
+					psta->flags &= ~WLAN_STA_MFP;
+					reason = _RSON_CLS2_;
+					active = _TRUE;
+				}
+#endif
+				ATOMIC_INC(&psta->deleting);
+				RTW_INFO("Set sta_info deleting=%u\n", ATOMIC_READ(&psta->deleting));
+				updated = ap_free_sta(padapter, psta, active, 0, reason, _TRUE);
+
+			}
+			rtw_stapriv_asoc_list_unlock(pstapriv);
+
+			associated_clients_update(padapter, updated, STA_INFO_UPDATE_ALL);
+		}
+
+		return _SUCCESS;
+	} else
+#endif
+	if (!MLME_IS_MESH(padapter)) {
+		RTW_PRINT(FUNC_ADPT_FMT" reason=%u, ta=" MAC_FMT "\n"
+			, FUNC_ADPT_ARG(padapter), reason, MAC_ARG(get_addr2_ptr(pframe)));
+
+	#ifdef CONFIG_RTW_WNM
+		if (rtw_wnm_try_btm_roam_imnt(padapter) > 0)
+	#endif
+		{
+			receive_disconnect(padapter, get_addr2_ptr(pframe), reason, _FALSE);
+			pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DISASSOC;
+			pmlmeinfo->wifi_reason_code = reason;
+		}
+	}
+	pmlmepriv->LinkDetectInfo.bBusyTraffic = _FALSE;
+	return _SUCCESS;
+
+}
+
+unsigned int OnAtim(_adapter *padapter, union recv_frame *precv_frame)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+	return _SUCCESS;
+}
+
+unsigned int on_action_spct(_adapter *padapter, union recv_frame *precv_frame)
+{
+	unsigned int ret = _FAIL;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint frame_len = precv_frame->u.hdr.len;
+	u8 *frame_body = (u8 *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 category;
+	u8 action;
+
+	psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+
+	if (!psta)
+		goto exit;
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_SPECTRUM_MGMT)
+		goto exit;
+
+	action = frame_body[1];
+
+	RTW_INFO(FUNC_ADPT_FMT" action:%u\n", FUNC_ADPT_ARG(padapter), action);
+
+	switch (action) {
+	case RTW_WLAN_ACTION_SPCT_MSR_REQ:
+	case RTW_WLAN_ACTION_SPCT_MSR_RPRT:
+	case RTW_WLAN_ACTION_SPCT_TPC_REQ:
+	case RTW_WLAN_ACTION_SPCT_TPC_RPRT:
+		break;
+	case RTW_WLAN_ACTION_SPCT_CHL_SWITCH:
+#if defined(CONFIG_ECSA_PHL)
+		if (MLME_IS_STA(padapter) && MLME_IS_ASOC(padapter)) {
+			process_csa_ie(padapter
+				, frame_body + 2, frame_len - (frame_body - pframe) - 2);
+		}
+#endif
+		break;
+	default:
+		break;
+	}
+
+exit:
+	return ret;
+}
+
+unsigned int OnAction_qos(_adapter *padapter, union recv_frame *precv_frame)
+{
+	return _SUCCESS;
+}
+
+unsigned int OnAction_dls(_adapter *padapter, union recv_frame *precv_frame)
+{
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_RTW_WNM
+unsigned int on_action_wnm(_adapter *adapter, union recv_frame *rframe)
+{
+	unsigned int ret = _FAIL;
+	struct sta_info *sta = NULL;
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct sta_priv *stapriv = &(adapter->stapriv);
+	u8 *frame = rframe->u.hdr.rx_data;
+	u32 frame_len = rframe->u.hdr.len;
+	u8 *frame_body = (u8 *)(frame + sizeof(struct rtw_ieee80211_hdr_3addr));
+	u32 frame_body_len = frame_len - sizeof(struct rtw_ieee80211_hdr_3addr);	
+	u8 category, action;
+	int cnt = 0;
+	char msg[16];
+
+	sta = rtw_get_stainfo(stapriv, get_addr2_ptr(frame));
+	if (!sta)
+		goto exit;
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_WNM)
+		goto exit;
+
+	action = frame_body[1];
+
+	switch (action) {
+#ifdef CONFIG_RTW_80211R
+	case RTW_WLAN_ACTION_WNM_BTM_REQ:
+		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == _TRUE) {
+			RTW_INFO("WNM: BSS Transition Management Request recv.\n");
+			rtw_wnm_process_btm_req(adapter, sta, frame_body, frame_body_len);
+		}
+		ret = _SUCCESS;
+		break;
+#endif		
+	case RTW_WLAN_ACTION_WNM_BTM_RSP:
+		if ((check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE) &&
+			(pmlmepriv->nb_info.features & RTW_WNM_FEATURE_BTM_REQ_EN)) {
+			struct btm_rsp_hdr rsp;
+			u32 sz;
+
+			RTW_INFO("WNM: BSS Transition Management Response recv.\n");
+			sz = rtw_wnm_btm_rsp_candidates_sz_get(adapter,
+					frame_body, frame_body_len);
+			_rtw_memset(&rsp, 0, sizeof(rsp));
+
+			if (sz > 0)
+				rsp.pcandidates = rtw_zmalloc(sz);
+
+			rtw_wnm_process_btm_rsp(adapter, frame_body, frame_body_len, &rsp);
+			/* TODO : handle candidates info in rsp.pcandidates for upper-layer services */
+
+			if (0 && rsp.pcandidates && (rsp.candidates_num > 0))
+				RTW_INFO_DUMP("pcandidates : ", rsp.pcandidates, sz);
+
+			if ((sz > 0) && (rsp.pcandidates != NULL))
+				rtw_mfree(rsp.pcandidates, sz);
+		}
+		fallthrough;
+	default:
+		#ifdef CONFIG_RTW_MBO
+		if (rtw_mbo_wifi_logo_test(adapter) \
+			&& (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE) \
+			&& (action == RTW_WLAN_ACTION_WNM_BTM_QUERY)) {
+			struct mbo_attr_info *pmbo_attr = \
+				&(pmlmepriv->mbo_attr);
+
+			RTW_INFO("%s: BSS Transition Management query"
+				" recv from "MAC_FMT"\n", __func__,
+				MAC_ARG(sta->phl_sta->mac_addr));
+			pmbo_attr->user_raw.hdr.dialog_token = \
+				wnm_btm_dialog_token(frame_body);
+			pmbo_attr->user_raw.hdr.req_mode |= \
+				PREFERRED_CANDIDATE_LIST_INCLUDED;
+			pmbo_attr->user_raw.hdr.validity_interval = 30;
+			rtw_wnm_process_btm_query(
+				adapter, frame_body, frame_body_len);
+			rtw_wnm_issue_btm_req(adapter, sta->phl_sta->mac_addr,
+				&pmbo_attr->user_raw.hdr, NULL, 0,
+				(u8 *)&pmbo_attr->user_raw.btm_cants,
+				pmbo_attr->user_raw.candidate_cnt);
+			break;
+		}
+
+		if (rtw_mbo_wifi_logo_test(adapter) \
+			&& (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE) \
+			&& (action == RTW_WLAN_ACTION_WNM_NOTIF_REQ)) {
+			struct mbo_attr_info *pmbo_attr = \
+				&(pmlmepriv->mbo_attr);
+
+			RTW_INFO("%s: Notification Request"
+				" recv from "MAC_FMT"\n", __func__,
+				MAC_ARG(sta->phl_sta->mac_addr));
+			rtw_wnm_process_notification_req(adapter,
+				frame_body, frame_body_len);
+			break;
+		}
+		#endif /* CONFIG_RTW_MBO */
+		#ifdef CONFIG_IOCTL_CFG80211
+		cnt += sprintf((msg + cnt), "ACT_WNM %u", action);
+		rtw_cfg80211_rx_action(adapter, rframe, msg);
+		#endif
+		ret = _SUCCESS;
+		break;
+	}
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_RTW_WNM */
+
+/**
+ * rtw_rx_ampdu_size - Get the target RX AMPDU buffer size for the specific @adapter
+ * @adapter: the adapter to get target RX AMPDU buffer size
+ *
+ * Returns: the target RX AMPDU buffer size
+ */
+u16 rtw_rx_ampdu_size(_adapter *adapter)
+{
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(adapter_to_dvobj(adapter));
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	u8 hw_band = adapter_link->wrlink->hw_band;
+	HT_CAP_AMPDU_FACTOR max_rx_ampdu_factor = 0;
+	u16 size = MAX_BA_BUFFER_SIZE_HT;
+
+	/* for scan */
+	if (!mlmeext_chk_scan_state(&adapter->mlmeextpriv, SCAN_DISABLE)
+	    && !mlmeext_chk_scan_state(&adapter->mlmeextpriv, SCAN_COMPLETE)
+	    && adapter->mlmeextpriv.sitesurvey_res.rx_ampdu_size != RX_AMPDU_SIZE_INVALID
+	   ) {
+		size = adapter->mlmeextpriv.sitesurvey_res.rx_ampdu_size;
+		goto exit;
+	}
+
+	/* default value based on max_rx_ampdu_factor */
+	if (adapter->driver_rx_ampdu_factor != 0xFF) {
+		max_rx_ampdu_factor = (HT_CAP_AMPDU_FACTOR)adapter->driver_rx_ampdu_factor;
+		/* In Maximum A-MPDU Length Exponent subfield of A-MPDU Parameters field of HT Capabilities element,
+		** the unit of max_rx_ampdu_factor are octets. 8K, 16K, 32K, 64K is right.
+		** But the buffer size subfield of Block Ack Parameter Set field in ADDBA action frame indicates
+		** the number of buffers available for this particular TID. Each buffer is equal to max. size of
+		** MSDU or AMSDU.
+		** The size variable means how many MSDUs or AMSDUs, it's not Kbytes.
+		*/
+		if (MAX_AMPDU_FACTOR_64K == max_rx_ampdu_factor)
+			size = MAX_BA_BUFFER_SIZE_HT;
+		else if (MAX_AMPDU_FACTOR_32K == max_rx_ampdu_factor)
+			size = 32;
+		else if (MAX_AMPDU_FACTOR_16K == max_rx_ampdu_factor)
+			size = 16;
+		else if (MAX_AMPDU_FACTOR_8K == max_rx_ampdu_factor)
+			size = 8;
+	} else {
+
+#ifdef CONFIG_80211AX_HE
+		if (is_supported_he(phl_com->phy_cap[hw_band].proto_sup))
+			size = MAX_BA_BUFFER_SIZE_HE;
+		else
+#endif
+		if (is_supported_ht(phl_com->phy_cap[hw_band].proto_sup))
+			size = MAX_BA_BUFFER_SIZE_HT;
+	}
+
+exit:
+	return size;
+}
+
+/**
+ * rtw_rx_ampdu_is_accept - Get the permission if RX AMPDU should be set up for the specific @adapter
+ * @adapter: the adapter to get the permission if RX AMPDU should be set up
+ *
+ * Returns: accept or not
+ */
+bool rtw_rx_ampdu_is_accept(_adapter *adapter)
+{
+	bool accept;
+
+	if (adapter->fix_rx_ampdu_accept != RX_AMPDU_ACCEPT_INVALID) {
+		accept = adapter->fix_rx_ampdu_accept;
+		goto exit;
+	}
+
+	/* for scan */
+	if (!mlmeext_chk_scan_state(&adapter->mlmeextpriv, SCAN_DISABLE)
+	    && !mlmeext_chk_scan_state(&adapter->mlmeextpriv, SCAN_COMPLETE)
+	    && adapter->mlmeextpriv.sitesurvey_res.rx_ampdu_accept != RX_AMPDU_ACCEPT_INVALID
+	   ) {
+		accept = adapter->mlmeextpriv.sitesurvey_res.rx_ampdu_accept;
+		goto exit;
+	}
+
+	/* default value for other cases */
+	accept = adapter->mlmeextpriv.mlmext_info.bAcceptAddbaReq;
+
+exit:
+	return accept;
+}
+
+/**
+ * rtw_rx_ampdu_set_size - Set the target RX AMPDU buffer size for the specific @adapter and specific @reason
+ * @adapter: the adapter to set target RX AMPDU buffer size
+ * @size: the target RX AMPDU buffer size to set
+ * @reason: reason for the target RX AMPDU buffer size setting
+ *
+ * Returns: whether the target RX AMPDU buffer size is changed
+ */
+bool rtw_rx_ampdu_set_size(_adapter *adapter, u16 size, u8 reason)
+{
+	bool is_adj = _FALSE;
+	struct mlme_ext_priv *mlmeext;
+	struct mlme_ext_info *mlmeinfo;
+
+	mlmeext = &adapter->mlmeextpriv;
+	mlmeinfo = &mlmeext->mlmext_info;
+
+	if (reason == RX_AMPDU_DRV_FIXED) {
+		if (adapter->fix_rx_ampdu_size != size) {
+			adapter->fix_rx_ampdu_size = size;
+			is_adj = _TRUE;
+			RTW_INFO(FUNC_ADPT_FMT" fix_rx_ampdu_size:%u\n", FUNC_ADPT_ARG(adapter), size);
+		}
+	} else if (reason == RX_AMPDU_DRV_SCAN) {
+		struct ss_res *ss = &adapter->mlmeextpriv.sitesurvey_res;
+
+		if (ss->rx_ampdu_size != size) {
+			ss->rx_ampdu_size = size;
+			is_adj = _TRUE;
+			RTW_INFO(FUNC_ADPT_FMT" ss.rx_ampdu_size:%u\n", FUNC_ADPT_ARG(adapter), size);
+		}
+	}
+
+	return is_adj;
+}
+
+/**
+ * rtw_rx_ampdu_set_accept - Set the permission if RX AMPDU should be set up for the specific @adapter and specific @reason
+ * @adapter: the adapter to set if RX AMPDU should be set up
+ * @accept: if RX AMPDU should be set up
+ * @reason: reason for the permission if RX AMPDU should be set up
+ *
+ * Returns: whether the permission if RX AMPDU should be set up is changed
+ */
+bool rtw_rx_ampdu_set_accept(_adapter *adapter, u8 accept, u8 reason)
+{
+	bool is_adj = _FALSE;
+	struct mlme_ext_priv *mlmeext;
+	struct mlme_ext_info *mlmeinfo;
+
+	mlmeext = &adapter->mlmeextpriv;
+	mlmeinfo = &mlmeext->mlmext_info;
+
+	if (reason == RX_AMPDU_DRV_FIXED) {
+		if (adapter->fix_rx_ampdu_accept != accept) {
+			adapter->fix_rx_ampdu_accept = accept;
+			is_adj = _TRUE;
+			RTW_INFO(FUNC_ADPT_FMT" fix_rx_ampdu_accept:%u\n", FUNC_ADPT_ARG(adapter), accept);
+		}
+	} else if (reason == RX_AMPDU_DRV_SCAN) {
+		if (adapter->mlmeextpriv.sitesurvey_res.rx_ampdu_accept != accept) {
+			adapter->mlmeextpriv.sitesurvey_res.rx_ampdu_accept = accept;
+			is_adj = _TRUE;
+			RTW_INFO(FUNC_ADPT_FMT" ss.rx_ampdu_accept:%u\n", FUNC_ADPT_ARG(adapter), accept);
+		}
+	}
+
+	return is_adj;
+}
+
+/**
+ * rx_ampdu_apply_sta_tid - Apply RX AMPDU setting to the specific @sta and @tid
+ * @adapter: the adapter to which @sta belongs
+ * @sta: the sta to be checked
+ * @tid: the tid to be checked
+ * @accept: the target permission if RX AMPDU should be set up
+ * @size: the target RX AMPDU buffer size
+ *
+ * Returns:
+ * 0: no canceled
+ * 1: canceled by no permission
+ * 2: canceled by different buffer size
+ * 3: canceled by potential mismatched status
+ *
+ * Blocking function, may sleep
+ */
+u8 rx_ampdu_apply_sta_tid(_adapter *adapter, struct sta_info *sta, u8 tid, u8 accept, u16 size)
+{
+	u8 ret = 0;
+	struct recv_reorder_ctrl *reorder_ctl = &sta->recvreorder_ctrl[tid];
+
+	if (reorder_ctl->enable == _FALSE) {
+		if (reorder_ctl->ampdu_size != RX_AMPDU_SIZE_INVALID) {
+			send_delba_sta_tid_wait_ack(adapter, 0, sta, tid, 1);
+			ret = 3;
+		}
+		goto exit;
+	}
+
+	if (accept == _FALSE) {
+		send_delba_sta_tid_wait_ack(adapter, 0, sta, tid, 0);
+		ret = 1;
+	} else if (reorder_ctl->ampdu_size != size) {
+		send_delba_sta_tid_wait_ack(adapter, 0, sta, tid, 0);
+		ret = 2;
+	}
+
+exit:
+	return ret;
+}
+
+u16 rx_ampdu_size_sta_limit(_adapter *adapter, struct sta_info *sta)
+{
+	u16 sz_limit = RX_AMPDU_SIZE_INVALID;
+
+#ifdef CONFIG_80211N_HT
+	struct registry_priv *regsty = &adapter->registrypriv;
+	struct _ADAPTER_LINK *adapter_link = sta->padapter_link;
+	struct link_mlme_priv *mlme = &adapter_link->mlmepriv;
+	struct link_mlme_ext_priv *mlmeext = &adapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *mlmeinfo = &mlmeext->mlmext_info;
+	s8 nss = -1;
+	u8 bw = rtw_min(sta->phl_sta->chandef.bw, mlmeext->chandef.bw);
+
+	#ifdef CONFIG_80211AC_VHT
+	#ifdef CONFIG_80211AX_HE
+	if (is_supported_he(sta->phl_sta->wmode)) {
+		nss = sta->phl_sta->asoc_cap.nss_tx;
+	} else
+	#endif
+	if (is_supported_vht(sta->phl_sta->wmode)) {
+		nss = rtw_min(rtw_vht_mcsmap_to_nss(mlme->vhtpriv.vht_mcs_map)
+				, rtw_vht_mcsmap_to_nss(sta->vhtpriv.vht_mcs_map));
+	} else
+	#endif
+	if (is_supported_ht(sta->phl_sta->wmode)) {
+		nss = rtw_min(rtw_ht_mcsset_to_nss(mlmeinfo->HT_caps.u.HT_cap_element.MCS_rate)
+				, rtw_ht_mcsset_to_nss(sta->htpriv.ht_cap.supp_mcs_set));
+	}
+
+	if (nss >= 1)
+		sz_limit = regsty->rx_ampdu_sz_limit_by_nss_bw[nss - 1][bw];
+#endif /* CONFIG_80211N_HT */
+
+	return sz_limit;
+}
+
+/**
+ * rx_ampdu_apply_sta - Apply RX AMPDU setting to the specific @sta
+ * @adapter: the adapter to which @sta belongs
+ * @sta: the sta to be checked
+ * @accept: the target permission if RX AMPDU should be set up
+ * @size: the target RX AMPDU buffer size
+ *
+ * Returns: number of the RX AMPDU assciation canceled for applying current target setting
+ *
+ * Blocking function, may sleep
+ */
+u8 rx_ampdu_apply_sta(_adapter *adapter, struct sta_info *sta, u8 accept, u16 size)
+{
+	u8 change_cnt = 0;
+	int i;
+
+	for (i = 0; i < TID_NUM; i++) {
+		if (rx_ampdu_apply_sta_tid(adapter, sta, i, accept, size) != 0)
+			change_cnt++;
+	}
+
+	return change_cnt;
+}
+
+/**
+ * rtw_rx_ampdu_apply - Apply the current target RX AMPDU setting for the specific @adapter
+ * @adapter: the adapter to be applied
+ *
+ * Returns: number of the RX AMPDU assciation canceled for applying current target setting
+ */
+u16 rtw_rx_ampdu_apply(_adapter *adapter)
+{
+	u16 adj_cnt = 0;
+	struct sta_info *sta;
+	u8 accept = rtw_rx_ampdu_is_accept(adapter);
+	u16 size;
+
+	if (adapter->fix_rx_ampdu_size != RX_AMPDU_SIZE_INVALID)
+		size = adapter->fix_rx_ampdu_size;
+	else
+		size = rtw_rx_ampdu_size(adapter);
+
+	if (MLME_IS_STA(adapter)) {
+		sta = rtw_get_stainfo(&adapter->stapriv, get_bssid(&adapter->mlmepriv));
+		if (sta) {
+			u16 sta_size = size;
+
+			if (adapter->fix_rx_ampdu_size == RX_AMPDU_SIZE_INVALID)
+				sta_size = rtw_min(size, rx_ampdu_size_sta_limit(adapter, sta));
+			adj_cnt += rx_ampdu_apply_sta(adapter, sta, accept, sta_size);
+		}
+		/* TODO: TDLS peer */
+
+	} else if (MLME_IS_AP(adapter) || MLME_IS_MESH(adapter)) {
+		_list *phead, *plist;
+		u8 peer_num = 0;
+		char peers[NUM_STA];
+		struct sta_priv *pstapriv = &adapter->stapriv;
+		int i;
+
+		rtw_stapriv_asoc_list_lock(pstapriv);
+
+		phead = &pstapriv->asoc_list;
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			int stainfo_offset;
+
+			sta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+			plist = get_next(plist);
+
+			stainfo_offset = rtw_stainfo_offset(pstapriv, sta);
+			if (stainfo_offset_valid(stainfo_offset))
+				peers[peer_num++] = stainfo_offset;
+		}
+
+		rtw_stapriv_asoc_list_unlock(pstapriv);
+
+		for (i = 0; i < peer_num; i++) {
+			sta = rtw_get_stainfo_by_offset(pstapriv, peers[i]);
+			if (sta) {
+				u16 sta_size = size;
+
+				if (adapter->fix_rx_ampdu_size == RX_AMPDU_SIZE_INVALID)
+					sta_size = rtw_min(size, rx_ampdu_size_sta_limit(adapter, sta));
+				adj_cnt += rx_ampdu_apply_sta(adapter, sta, accept, sta_size);
+			}
+		}
+	}
+
+	/* TODO: ADHOC */
+
+	return adj_cnt;
+}
+
+unsigned int OnAction_back(_adapter *padapter, union recv_frame *precv_frame)
+{
+	u8 *addr;
+	struct sta_info *psta = NULL;
+	struct recv_reorder_ctrl *preorder_ctrl;
+	unsigned char		*frame_body;
+	unsigned char		category, action;
+	unsigned short	tid, status, reason_code = 0;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	int i;
+
+#ifdef CONFIG_80211N_HT
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/* check RA matches or not	 */
+	if (!rtw_match_self_addr(padapter, GetAddr1Ptr(pframe)))
+		return _SUCCESS;
+
+#if 0
+	/* check A1 matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(padapter), get_da(pframe), ETH_ALEN))
+		return _SUCCESS;
+#endif
+
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
+		if (!(pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
+			return _SUCCESS;
+
+	psta = precv_frame->u.hdr.psta;
+	if (psta == NULL)
+		return _SUCCESS;
+
+	addr = psta->phl_sta->mac_addr;
+	padapter_link = psta->padapter_link;
+
+	frame_body = (unsigned char *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+
+	category = frame_body[0];
+	if (category == RTW_WLAN_CATEGORY_BACK) { /* representing Block Ack */
+#ifdef CONFIG_TDLS
+		if ((psta->tdls_sta_state & TDLS_LINKED_STATE) &&
+		    (psta->htpriv.ht_option == _TRUE
+#if CONFIG_IEEE80211_BAND_6GHZ
+		     || psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+		    ) && (psta->ampdu_priv.ampdu_enable == _TRUE))
+			RTW_INFO("Recv [%s] from direc link\n", __FUNCTION__);
+		else
+#endif /* CONFIG_TDLS */
+			if (!padapter_link->mlmeextpriv.mlmext_info.HT_enable
+#if CONFIG_IEEE80211_BAND_6GHZ
+			    && psta->phl_sta->chandef.band != BAND_ON_6G
+#endif
+			   )
+				return _SUCCESS;
+
+		action = frame_body[1];
+		RTW_INFO("%s, action=%d\n", __FUNCTION__, action);
+		switch (action) {
+		case RTW_WLAN_ACTION_ADDBA_REQ: /* ADDBA request */
+			if (adapter_to_pwrctl(padapter)->bInSuspend == _TRUE) {
+				RTW_INFO("%s addba req  since bInSuspend is TRUE,break!!!!\n", __func__);
+				break;
+			}
+			/* process_addba_req(padapter, (u8*)&(pmlmeinfo->ADDBA_req), GetAddr3Ptr(pframe)); */
+			_rtw_memcpy(&(pmlmeinfo->ADDBA_req), &(frame_body[2]), sizeof(struct ADDBA_request));
+			process_addba_req(padapter, (u8 *)&(pmlmeinfo->ADDBA_req), addr);
+
+			break;
+
+		case RTW_WLAN_ACTION_ADDBA_RESP: /* ADDBA response */
+
+			/* status = frame_body[3] | (frame_body[4] << 8); */ /* endian issue */
+			status = RTW_GET_LE16(&frame_body[3]);
+			tid = ((frame_body[5] >> 2) & 0x7);
+			if (status == 0) {
+				enum rtw_phl_status psts;
+				u16 ba_param = RTW_GET_LE16(&frame_body[5]);
+				u16 buf_num = ba_param >> 6;
+				struct protocol_cap_t *asoc_cap = &psta->phl_sta->asoc_cap;
+
+				/* successful					 */
+				RTW_INFO("[ADDBA_RESP] agg_enable for TID=%d ampdu_num %u\n", tid, buf_num);
+
+				psta->ampdu_priv.agg_enable_bitmap |= 1 << tid;
+				psta->ampdu_priv.candidate_tid_bitmap &= ~BIT(tid);
+				/* amsdu in ampdu */
+				if (pregpriv->tx_ampdu_amsdu == 0)
+					psta->ampdu_priv.tx_amsdu_enable = _FALSE;
+				else if (pregpriv->tx_ampdu_amsdu == 1)
+					psta->ampdu_priv.tx_amsdu_enable = _TRUE;
+				else {
+					if (ba_param & 1)
+						psta->ampdu_priv.tx_amsdu_enable = _TRUE;
+				}
+				asoc_cap->num_ampdu = asoc_cap->num_ampdu > buf_num ?
+					buf_num : asoc_cap->num_ampdu;
+
+				if (pxmitpriv->max_agg_time != 0) {
+					for (i=0; i < 4; i++) {
+						asoc_cap->edca[i].param =
+							(asoc_cap->edca[i].param & 0xFFFF) |
+								(pxmitpriv->max_agg_time << 16);
+						RTW_DBG("edca[%d]: 0x%x\n", i, asoc_cap->edca[i].param);
+					}
+				}
+
+				psts = rtw_phl_cmd_cfg_ampdu(GET_PHL_INFO(adapter_to_dvobj(padapter)),
+						padapter->phl_role, psta->phl_sta, PHL_CMD_DIRECTLY, 0);
+				if (psts != RTW_PHL_STATUS_SUCCESS)
+					RTW_ERR("%s: config ampdu for "MAC_FMT" fail",
+						__func__, MAC_ARG(psta->phl_sta->mac_addr));
+				else
+					RTW_INFO("%s: config ampdu_num to %u for "MAC_FMT" success",
+						__func__, asoc_cap->num_ampdu, MAC_ARG(psta->phl_sta->mac_addr));
+			} else {
+				psta->ampdu_priv.agg_enable_bitmap &= ~BIT(tid);
+			}
+
+			if (psta->state & WIFI_STA_ALIVE_CHK_STATE) {
+				RTW_INFO("%s alive check - rx ADDBA response\n", __func__);
+				psta->ampdu_priv.agg_enable_bitmap &= ~BIT(tid);
+				psta->expire_to = pstapriv->expire_to;
+				psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
+			}
+
+			/* RTW_INFO("marc: ADDBA RSP: %x\n", pmlmeinfo->agg_enable_bitmap); */
+			break;
+
+		case RTW_WLAN_ACTION_DELBA: /* DELBA */
+			if ((frame_body[3] & BIT(3)) == 0) {
+				psta->ampdu_priv.agg_enable_bitmap &= ~(1 << ((frame_body[3] >> 4) & 0xf));
+				psta->ampdu_priv.candidate_tid_bitmap &= ~(1 << ((frame_body[3] >> 4) & 0xf));
+
+				/* reason_code = frame_body[4] | (frame_body[5] << 8); */
+				reason_code = RTW_GET_LE16(&frame_body[4]);
+			} else if ((frame_body[3] & BIT(3)) == BIT(3)) {
+				tid = (frame_body[3] >> 4) & 0x0F;
+
+				preorder_ctrl = &psta->recvreorder_ctrl[tid];
+				preorder_ctrl->enable = _FALSE;
+				preorder_ctrl->ampdu_size = RX_AMPDU_SIZE_INVALID;
+				rtw_delba_cmd(padapter, addr, tid);
+			}
+
+			RTW_INFO("%s(): DELBA: %x(%x)\n", __FUNCTION__, pmlmeinfo->agg_enable_bitmap, reason_code);
+			/* todo: how to notify the host while receiving DELETE BA */
+			break;
+
+		default:
+			break;
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+u32 rtw_build_vendor_ie(_adapter *padapter , unsigned char **pframe , u8 mgmt_frame_tyte)
+{
+	int vendor_ie_num = 0;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u32 len = 0;
+
+	for (vendor_ie_num = 0 ; vendor_ie_num < WLAN_MAX_VENDOR_IE_NUM ; vendor_ie_num++) {
+		if (pmlmepriv->vendor_ielen[vendor_ie_num] > 0 && pmlmepriv->vendor_ie_mask[vendor_ie_num] & mgmt_frame_tyte) {
+			_rtw_memcpy(*pframe , pmlmepriv->vendor_ie[vendor_ie_num] , pmlmepriv->vendor_ielen[vendor_ie_num]);
+			*pframe +=  pmlmepriv->vendor_ielen[vendor_ie_num];
+			len += pmlmepriv->vendor_ielen[vendor_ie_num];
+		}
+	}
+
+	return len;
+}
+#endif
+
+#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+u8 *rtw_set_channel_triplet(struct rtw_chset *chset, u8 *buf_pos, u8 *trip_len, char alpha2[2])
+{
+	u8 i = 0;
+	u8 fch = 0, lch = 0, ch;
+	u8 *cont = buf_pos + 5;
+	RT_CHANNEL_INFO *chinfo;
+
+	while (i < chset->chs_len) {
+		chinfo = &chset->chs[i++];
+
+		if (chinfo->flags & RTW_CHF_DIS)
+			continue;
+
+		ch = chinfo->ChannelNum;
+
+		if (fch == 0) {
+			fch = ch;
+			lch = ch;
+			continue;
+		}
+
+		if (lch + 1 != ch) {
+			*cont = fch;
+			*(cont + 1) = lch - fch + 1;
+			*(cont + 2) = 128;
+			cont += 3;
+			fch = ch;
+		}
+		lch = ch;
+	}
+
+	if (fch) {
+		/* last subband */
+		*cont = fch;
+		*(cont + 1) = lch - fch + 1;
+		*(cont + 2) = 128;
+		cont += 3;
+	}
+
+	if (cont > buf_pos + 5) {
+		*trip_len += (cont - buf_pos - 2);
+		*buf_pos = WLAN_EID_COUNTRY;
+		*(buf_pos + 1) = cont - buf_pos - 2;
+		*(buf_pos + 2) = alpha2[0];
+		*(buf_pos + 3) = alpha2[1];
+		*(buf_pos + 4) = ' ';
+		return cont;
+	}
+	return buf_pos;
+}
+
+u8 rtw_set_ie_country(_adapter *adapter, u8 *pframe, u32 *pkt_len)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	u8 *p = NULL, *dst_ie = NULL, *remainder_ie = NULL, *backup_remainder_ie = NULL;
+	u32 offset = 0, tmp_len = 0;
+	s32 remainder_ielen = 0;
+	u8 trip_len = 0, country_ie_len = 0;
+
+	/* calculate head_len */
+	offset = _FIXED_IE_LENGTH_;
+
+	/* get ssid_ie len */
+	p = rtw_get_ie(pframe + _BEACON_IE_OFFSET_, _SSID_IE_, &tmp_len, (*pkt_len - _BEACON_IE_OFFSET_));
+	if (p != NULL)
+		offset += tmp_len + 2;
+
+	/* get supported rates len */
+	p = rtw_get_ie(pframe + _BEACON_IE_OFFSET_, _SUPPORTEDRATES_IE_, &tmp_len, (*pkt_len - _BEACON_IE_OFFSET_));
+	if (p !=  NULL)
+		offset += tmp_len + 2;
+
+	/* DS Parameter Set IE, len=3 */
+	offset += 3;
+
+	/* get TIM IE len */
+	p = rtw_get_ie(pframe + _BEACON_IE_OFFSET_, _TIM_IE_, &tmp_len, (*pkt_len - _BEACON_IE_OFFSET_));
+	if (p !=  NULL)
+		offset += tmp_len + 2;
+
+	remainder_ie = pframe + offset;
+	remainder_ielen = (*pkt_len - sizeof(struct rtw_ieee80211_hdr_3addr)) - offset;
+
+	dst_ie = pframe + offset;
+
+	if (remainder_ielen > 0) {
+		backup_remainder_ie = rtw_malloc(remainder_ielen);
+		if (backup_remainder_ie && remainder_ie)
+			_rtw_memcpy(backup_remainder_ie, remainder_ie, remainder_ielen);
+	}
+
+	dst_ie = rtw_set_channel_triplet(&rfctl->chset, dst_ie, &trip_len, rfctl->alpha2);
+
+	/* size of country IE */
+	country_ie_len = trip_len + 2;
+
+	if (backup_remainder_ie) {
+		_rtw_memcpy(dst_ie, backup_remainder_ie, remainder_ielen);
+
+		rtw_mfree(backup_remainder_ie, remainder_ielen);
+	}
+
+	*pkt_len += country_ie_len;
+
+	return country_ie_len;
+}
+#endif
+
+#ifdef CONFIG_ECSA_PHL
+u8 *build_supported_op_class_ie(_adapter *padapter, u8 *pbuf, int *pktlen)
+{
+	u8 buf[32];
+	int ie_len = 0;
+
+	ie_len = get_supported_op_class(padapter, buf, sizeof(buf));
+	pbuf = rtw_set_ie(pbuf, WLAN_EID_SUPPORT_OP_CLASS , ie_len,
+			    (unsigned char *)buf, pktlen);
+	return pbuf;
+}
+#endif /* CONFIG_ECSA_PHL */
+
+#ifdef CONFIG_P2P
+void issue_probersp_p2p(_adapter *padapter, unsigned char *da)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	unsigned char					*mac;
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	/* WLAN_BSSID_EX 		*cur_network = &(pmlmeinfo->network); */
+	u16					beacon_interval = 100;
+	u16					capInfo = 0;
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	u8					wpsie[255] = { 0x00 };
+	u32					wpsielen = 0, p2pielen = 0;
+#ifdef CONFIG_WFD
+	u32					wfdielen = 0;
+#endif
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	if (IS_CCK_RATE(pattrib->rate)) {
+		/* force OFDM 6M rate */
+		pattrib->rate = MGN_6M;
+	}
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	mac = padapter_link->mac_addr;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+
+	/*	Use the device address for BSSID field.	 */
+	_rtw_memcpy(pwlanhdr->addr3, mac, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(fctrl, WIFI_PROBERSP);
+
+	pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = pattrib->hdrlen;
+	pframe += pattrib->hdrlen;
+
+	/* timestamp will be inserted by hardware */
+	pframe += 8;
+	pattrib->pktlen += 8;
+
+	/* beacon interval: 2 bytes */
+	_rtw_memcpy(pframe, (unsigned char *) &beacon_interval, 2);
+	pframe += 2;
+	pattrib->pktlen += 2;
+
+	/*	capability info: 2 bytes */
+	/*	ESS and IBSS bits must be 0 (defined in the 3.1.2.1.1 of WiFi Direct Spec) */
+	capInfo |= cap_ShortPremble;
+	capInfo |= cap_ShortSlot;
+
+	_rtw_memcpy(pframe, (unsigned char *) &capInfo, 2);
+	pframe += 2;
+	pattrib->pktlen += 2;
+
+
+	/* SSID */
+	pframe = rtw_set_ie(pframe, _SSID_IE_, 7, pwdinfo->p2p_wildcard_ssid, &pattrib->pktlen);
+
+	/* supported rates... */
+	/*	Use the OFDM rate in the P2P probe response frame. ( 6(B), 9(B), 12, 18, 24, 36, 48, 54 ) */
+	pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, 8, pwdinfo->support_rate, &pattrib->pktlen);
+
+	/* DS parameter set */
+	pframe = rtw_set_ie(pframe, _DSSET_IE_, 1, (unsigned char *)&pwdinfo->listen_channel, &pattrib->pktlen);
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (adapter_wdev_data(padapter)->p2p_enabled) {
+		if (pmlmepriv->wps_probe_resp_ie != NULL && pmlmepriv->p2p_probe_resp_ie != NULL) {
+			/* WPS IE */
+			_rtw_memcpy(pframe, pmlmepriv->wps_probe_resp_ie, pmlmepriv->wps_probe_resp_ie_len);
+			pattrib->pktlen += pmlmepriv->wps_probe_resp_ie_len;
+			pframe += pmlmepriv->wps_probe_resp_ie_len;
+
+			/* P2P IE */
+			_rtw_memcpy(pframe, pmlmepriv->p2p_probe_resp_ie, pmlmepriv->p2p_probe_resp_ie_len);
+			pattrib->pktlen += pmlmepriv->p2p_probe_resp_ie_len;
+			pframe += pmlmepriv->p2p_probe_resp_ie_len;
+		}
+	}
+#endif /* CONFIG_IOCTL_CFG80211		 */
+
+#ifdef CONFIG_WFD
+	wfdielen = rtw_append_probe_resp_wfd_ie(padapter, pframe);
+	pframe += wfdielen;
+	pattrib->pktlen += wfdielen;
+#endif
+
+/* Vendor Specific IE */
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	pattrib->pktlen += rtw_build_vendor_ie(padapter , &pframe , WIFI_P2P_PROBERESP_VENDOR_IE_BIT);
+#endif
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	return;
+
+}
+
+int _issue_probereq_p2p(_adapter *padapter, u8 *da, int wait_ack)
+{
+	int ret = _FAIL;
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	unsigned char			*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short		*fctrl;
+	unsigned char			*mac;
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	u8	bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	u8					wpsie[255] = { 0x00 }, p2pie[255] = { 0x00 };
+	u16					wpsielen = 0, p2pielen = 0;
+#ifdef CONFIG_WFD
+	u32					wfdielen = 0;
+#endif
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	if (IS_CCK_RATE(pattrib->rate)) {
+		/* force OFDM 6M rate */
+		pattrib->rate = MGN_6M;
+	}
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	mac = padapter_link->mac_addr;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	if (da) {
+		_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, da, ETH_ALEN);
+	} else {
+		/*	broadcast probe request frame */
+		_rtw_memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, bc_addr, ETH_ALEN);
+	}
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_PROBEREQ);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_ie(pframe, _SSID_IE_, P2P_WILDCARD_SSID_LEN, pwdinfo->p2p_wildcard_ssid, &(pattrib->pktlen));
+
+	/*	Use the OFDM rate in the P2P probe request frame. ( 6(B), 9(B), 12(B), 24(B), 36, 48, 54 ) */
+	pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, 8, pwdinfo->support_rate, &pattrib->pktlen);
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (adapter_wdev_data(padapter)->p2p_enabled) {
+		if (pmlmepriv->wps_probe_req_ie != NULL && pmlmepriv->p2p_probe_req_ie != NULL) {
+			/* WPS IE */
+			_rtw_memcpy(pframe, pmlmepriv->wps_probe_req_ie, pmlmepriv->wps_probe_req_ie_len);
+			pattrib->pktlen += pmlmepriv->wps_probe_req_ie_len;
+			pframe += pmlmepriv->wps_probe_req_ie_len;
+
+			/* P2P IE */
+			_rtw_memcpy(pframe, pmlmepriv->p2p_probe_req_ie, pmlmepriv->p2p_probe_req_ie_len);
+			pattrib->pktlen += pmlmepriv->p2p_probe_req_ie_len;
+			pframe += pmlmepriv->p2p_probe_req_ie_len;
+		}
+	}
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+#ifdef CONFIG_WFD
+	wfdielen = rtw_append_probe_req_wfd_ie(padapter, pframe);
+	pframe += wfdielen;
+	pattrib->pktlen += wfdielen;
+#endif
+
+/* Vendor Specific IE */
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	pattrib->pktlen += rtw_build_vendor_ie(padapter , &pframe , WIFI_P2P_PROBEREQ_VENDOR_IE_BIT);
+#endif
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+	return ret;
+}
+
+inline void issue_probereq_p2p(_adapter *adapter, u8 *da)
+{
+	_issue_probereq_p2p(adapter, da, _FALSE);
+}
+
+#endif /* CONFIG_P2P */
+
+s32 rtw_action_public_decache(union recv_frame *rframe, u8 token_offset)
+{
+	_adapter *adapter = rframe->u.hdr.adapter;
+	struct mlme_ext_priv *mlmeext = &(adapter->mlmeextpriv);
+	u8 *frame = rframe->u.hdr.rx_data;
+	u16 seq_ctrl = ((rframe->u.hdr.attrib.seq_num & 0xffff) << 4) | (rframe->u.hdr.attrib.frag_num & 0xf);
+	u8 token = *(rframe->u.hdr.rx_data + sizeof(struct rtw_ieee80211_hdr_3addr) + token_offset);
+
+	if (GetRetry(frame)) {
+		if ((seq_ctrl == mlmeext->action_public_rxseq)
+		    && (token == mlmeext->action_public_dialog_token)
+		   ) {
+			RTW_INFO(FUNC_ADPT_FMT" seq_ctrl=0x%x, rxseq=0x%x, token:%d\n",
+				FUNC_ADPT_ARG(adapter), seq_ctrl, mlmeext->action_public_rxseq, token);
+			return _FAIL;
+		}
+	}
+
+	/* TODO: per sta seq & token */
+	mlmeext->action_public_rxseq = seq_ctrl;
+	mlmeext->action_public_dialog_token = token;
+
+	return _SUCCESS;
+}
+
+unsigned int on_action_public_p2p(union recv_frame *precv_frame)
+{
+	_adapter *padapter = precv_frame->u.hdr.adapter;
+
+#ifdef CONFIG_P2P
+#if defined(RTW_DEDICATED_P2P_DEVICE) || !RTW_P2P_GROUP_INTERFACE
+	if (!adapter_wdev_data(padapter)->p2p_enabled) {
+		RTW_INFO(FUNC_ADPT_FMT": failed to receive p2p action frame\n",
+			FUNC_ADPT_ARG(padapter));
+		return _FAIL;
+	} else
+#endif
+	{
+		rtw_cfg80211_rx_p2p_action_public(padapter, precv_frame);
+	}
+#endif /* CONFIG_P2P */
+
+	return _SUCCESS;
+}
+
+unsigned int on_action_public_vendor(union recv_frame *precv_frame)
+{
+	unsigned int ret = _FAIL;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	_adapter *adapter = precv_frame->u.hdr.adapter;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	int cnt = 0;
+	char msg[64];
+
+	if (_rtw_memcmp(frame_body + 2, P2P_OUI, 4) == _TRUE) {
+		if (rtw_action_public_decache(precv_frame, 7) == _FAIL)
+			goto exit;
+
+		if (!rtw_hw_chk_wl_func(dvobj, WL_FUNC_MIRACAST))
+			rtw_rframe_del_wfd_ie(precv_frame, 8);
+
+		ret = on_action_public_p2p(precv_frame);
+	} else if (_rtw_memcmp(frame_body + 2, DPP_OUI, 4) == _TRUE) {
+		u8 dpp_type = frame_body[7];
+
+#ifdef CONFIG_IOCTL_CFG80211
+		cnt += sprintf((msg + cnt), "DPP(type:%u)", dpp_type);
+		rtw_cfg80211_rx_action(adapter, precv_frame, msg);
+#endif
+	}
+
+exit:
+	return ret;
+}
+
+unsigned int on_action_public_default(union recv_frame *precv_frame, u8 action)
+{
+	unsigned int ret = _FAIL;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 token;
+	_adapter *adapter = precv_frame->u.hdr.adapter;
+	int cnt = 0;
+	char msg[64];
+
+	token = frame_body[2];
+
+	if (rtw_action_public_decache(precv_frame, 2) == _FAIL)
+		goto exit;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	cnt += sprintf((msg + cnt), "%s(token:%u)", action_public_str(action), token);
+	rtw_cfg80211_rx_action(adapter, precv_frame, msg);
+#endif
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+unsigned int on_action_public(_adapter *padapter, union recv_frame *precv_frame)
+{
+	unsigned int ret = _FAIL;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint frame_len = precv_frame->u.hdr.len;
+	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 category, action;
+
+	/* check RA matches or broadcast */
+	if (!(_rtw_memcmp(adapter_mac_addr(padapter), GetAddr1Ptr(pframe), ETH_ALEN) ||
+		is_broadcast_mac_addr(GetAddr1Ptr(pframe))))
+		goto exit;
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_PUBLIC)
+		goto exit;
+
+	action = frame_body[1];
+	switch (action) {
+	case ACT_PUBLIC_BSSCOEXIST:
+#ifdef CONFIG_80211N_HT
+#ifdef CONFIG_AP_MODE
+		/*20/40 BSS Coexistence Management frame is a Public Action frame*/
+		if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == _TRUE)
+			rtw_process_public_act_bsscoex(padapter, pframe, frame_len);
+#endif /*CONFIG_AP_MODE*/
+#endif /*CONFIG_80211N_HT*/
+		break;
+	case ACT_PUBLIC_VENDOR:
+		ret = on_action_public_vendor(precv_frame);
+		break;
+	default:
+		ret = on_action_public_default(precv_frame, action);
+		break;
+	}
+
+exit:
+	return ret;
+}
+
+unsigned int OnAction_ft(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_RTW_80211R
+	u32	ret = _FAIL, frame_len = 0;
+	u8	action_code = 0, category = 0;
+	u8	*pframe = NULL, *pframe_body = NULL, *pie = NULL;
+	u8 	tgt_addr[ETH_ALEN], msg[32] = {0};
+	u32	ft_ie_len = 0;
+	u32 status_code = 0;
+	struct mlme_ext_priv *pmlmeext = NULL;
+	struct mlme_ext_info *pmlmeinfo = NULL;
+	struct mlme_priv *pmlmepriv = NULL;
+	struct wlan_network *proam_target = NULL;
+	struct ft_roam_info *pft_roam = NULL;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_mlme_priv *lmlmepriv = &(padapter_link->mlmepriv);
+#ifdef CONFIG_RTW_FSM_BTM
+	struct sta_info		*psta;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+#endif
+
+	pmlmeext = &(padapter->mlmeextpriv);
+	pmlmeinfo = &(pmlmeext->mlmext_info);
+	pmlmepriv = &(padapter->mlmepriv);
+	pft_roam = &(pmlmepriv->ft_roam);
+	pframe = precv_frame->u.hdr.rx_data;
+	frame_len = precv_frame->u.hdr.len;
+	pframe_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	category = pframe_body[0];
+
+	if (category != RTW_WLAN_CATEGORY_FT)
+		goto exit;
+
+	action_code = pframe_body[1];
+	switch (action_code) {
+	case RTW_WLAN_ACTION_FT_RSP:
+		RTW_INFO("FT: RTW_WLAN_ACTION_FT_RSP recv.\n");
+		if (!_rtw_memcmp(adapter_mac_addr(padapter), &pframe_body[2], ETH_ALEN)) {
+			RTW_ERR("FT: Unmatched STA MAC Address "MAC_FMT"\n", MAC_ARG(&pframe_body[2]));
+			goto exit;
+		}
+
+		status_code = le16_to_cpu(*(u16 *)((SIZE_PTR)pframe +  sizeof(struct rtw_ieee80211_hdr_3addr) + 14));
+		if (status_code != 0) {
+			RTW_ERR("FT: WLAN ACTION FT RESPONSE fail, status: %d\n", status_code);
+			goto exit;
+		}
+
+		_rtw_memcpy(tgt_addr, &pframe_body[8], ETH_ALEN);
+		if (is_zero_mac_addr(tgt_addr) || is_broadcast_mac_addr(tgt_addr)) {
+			RTW_ERR("FT: Invalid Target MAC Address "MAC_FMT"\n", MAC_ARG(tgt_addr));
+			goto exit;
+		}
+
+		pie = rtw_get_ie(pframe_body, _MDIE_, &ft_ie_len, frame_len);
+		if (pie) {
+			if (!_rtw_memcmp(&pft_roam->mdid, pie+2, 2)) {
+				RTW_ERR("FT: Invalid MDID\n");
+				goto exit;
+			}
+		}
+
+		_rtw_memcpy(pft_roam->ft_action, pframe, rtw_min(frame_len, RTW_FT_MAX_IE_SZ));
+		pft_roam->ft_action_len = frame_len;
+
+#ifdef CONFIG_RTW_FSM_BTM
+		psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+		if (!psta)
+			goto exit;
+
+		rtw_btm_notify_action_resp(psta->btm);
+#else
+		rtw_ft_set_status(padapter, RTW_FT_REQUESTED_STA);
+		_cancel_timer_ex(&pmlmeext->ft_link_timer);
+
+		/*Disconnect current AP*/
+		receive_disconnect(padapter, lmlmepriv->cur_network.network.MacAddress, WLAN_REASON_ACTIVE_ROAM, _FALSE);
+		pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+		pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_FT;
+		pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+#endif /* CONFIG_RTW_FSM_BTM */
+		ret = _SUCCESS;
+		break;
+	case RTW_WLAN_ACTION_FT_REQ:
+	case RTW_WLAN_ACTION_FT_CONF:
+	case RTW_WLAN_ACTION_FT_ACK:
+	#ifdef CONFIG_IOCTL_CFG80211
+		sprintf(msg, "ACT_FT %u", action_code);
+		rtw_cfg80211_rx_action(padapter, precv_frame, msg);
+	#endif
+		break;
+	default:
+		RTW_ERR("FT: Unsupported FT Action!\n");
+		break;
+	}
+exit:
+	return ret;
+#else
+	return _SUCCESS;
+#endif
+}
+
+unsigned int OnAction_ht(_adapter *padapter, union recv_frame *precv_frame)
+{
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 category, action;
+
+	/* check RA matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(padapter), GetAddr1Ptr(pframe), ETH_ALEN))
+		goto exit;
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_HT)
+		goto exit;
+
+	action = frame_body[1];
+	switch (action) {
+	case RTW_WLAN_ACTION_HT_SM_PS:
+#ifdef CONFIG_80211N_HT
+#ifdef CONFIG_AP_MODE
+		if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == _TRUE)
+			rtw_process_ht_action_smps(padapter, get_addr2_ptr(pframe), frame_body[2]);
+#endif /*CONFIG_AP_MODE*/
+#endif /*CONFIG_80211N_HT*/
+		break;
+	case RTW_WLAN_ACTION_HT_COMPRESS_BEAMFORMING:
+#ifdef CONFIG_BEAMFORMING
+		/*RTW_INFO("RTW_WLAN_ACTION_HT_COMPRESS_BEAMFORMING\n");*/
+		/*rtw_beamforming_get_report_frame(padapter, precv_frame);*/
+#endif /*CONFIG_BEAMFORMING*/
+		break;
+	default:
+		break;
+	}
+
+exit:
+
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_IEEE80211W
+unsigned int OnAction_sa_query(_adapter *padapter, union recv_frame *precv_frame)
+{
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct sta_info		*psta;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u16 tid;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	/* Baron */
+
+	RTW_INFO("OnAction_sa_query\n");
+
+	switch (pframe[WLAN_HDR_A3_LEN + 1]) {
+	case 0: /* SA Query req */
+		_rtw_memcpy(&tid, &pframe[WLAN_HDR_A3_LEN + 2], sizeof(u16));
+		RTW_INFO("OnAction_sa_query request,action=%d, tid=%04x, pframe=%02x-%02x\n"
+			, pframe[WLAN_HDR_A3_LEN + 1], tid, pframe[WLAN_HDR_A3_LEN + 2], pframe[WLAN_HDR_A3_LEN + 3]);
+		issue_action_SA_Query(padapter, padapter_link,
+					get_addr2_ptr(pframe), 1, tid, IEEE80211W_RIGHT_KEY);
+		break;
+
+	case 1: /* SA Query rsp */
+		psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+		if (psta != NULL)
+			_cancel_timer_ex(&psta->dot11w_expire_timer);
+
+		_rtw_memcpy(&tid, &pframe[WLAN_HDR_A3_LEN + 2], sizeof(u16));
+		RTW_INFO("OnAction_sa_query response,action=%d, tid=%04x, cancel timer\n", pframe[WLAN_HDR_A3_LEN + 1], tid);
+		break;
+	default:
+		break;
+	}
+	if (0) {
+		int pp;
+		printk("pattrib->pktlen = %d =>", pattrib->pkt_len);
+		for (pp = 0; pp < pattrib->pkt_len; pp++)
+			printk(" %02x ", pframe[pp]);
+		printk("\n");
+	}
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_IEEE80211W */
+
+unsigned int on_action_rm(_adapter *padapter, union recv_frame *precv_frame)
+{
+#if defined(CONFIG_RTW_80211K)
+	return rm_on_action(padapter, precv_frame);
+#elif defined(CONFIG_RTW_FSM_RRM)
+	return rrm_on_action(padapter, precv_frame);
+#else
+	return _SUCCESS;
+#endif  /* CONFIG_RTW_80211K */
+}
+
+unsigned int OnAction_wmm(_adapter *padapter, union recv_frame *precv_frame)
+{
+	return _SUCCESS;
+}
+
+unsigned int OnAction_vht(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_80211AC_VHT
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	struct rtw_ieee80211_hdr_3addr *whdr = (struct rtw_ieee80211_hdr_3addr *)pframe;
+	u8 *frame_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 category, action;
+	struct sta_info *psta = NULL;
+
+	/* check RA matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(padapter), GetAddr1Ptr(pframe), ETH_ALEN))
+		goto exit;
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_VHT)
+		goto exit;
+
+	action = frame_body[1];
+	switch (action) {
+	case RTW_WLAN_ACTION_VHT_COMPRESSED_BEAMFORMING:
+#ifdef CONFIG_BEAMFORMING
+		/*RTW_INFO("RTW_WLAN_ACTION_VHT_COMPRESSED_BEAMFORMING\n");*/
+		/*rtw_beamforming_get_report_frame(padapter, precv_frame);*/
+#endif /*CONFIG_BEAMFORMING*/
+		break;
+	case RTW_WLAN_ACTION_VHT_OPMODE_NOTIFICATION:
+		/* CategoryCode(1) + ActionCode(1) + OpModeNotification(1) */
+		/* RTW_INFO("RTW_WLAN_ACTION_VHT_OPMODE_NOTIFICATION\n"); */
+		psta = rtw_get_stainfo(&padapter->stapriv, whdr->addr2);
+		if (psta)
+			rtw_process_vht_op_mode_notify(padapter, &frame_body[2], psta);
+		break;
+	case RTW_WLAN_ACTION_VHT_GROUPID_MANAGEMENT:
+#ifdef CONFIG_BEAMFORMING
+		rtw_beamforming_get_vht_gid_mgnt_frame(padapter, precv_frame);
+#endif /* CONFIG_BEAMFORMING */
+		break;
+	default:
+		break;
+	}
+
+exit:
+#endif /* CONFIG_80211AC_VHT */
+
+	return _SUCCESS;
+}
+
+unsigned int OnAction_he(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_80211AX_HE
+	/* CONFIG_80211AX_HE_TODO */
+#endif /* CONFIG_80211AX_HE */
+
+	return _SUCCESS;
+}
+
+unsigned int OnAction_protected_he(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_80211AX_HE
+	/* CONFIG_80211AX_HE_TODO */
+#endif /* CONFIG_80211AX_HE */
+
+	return _SUCCESS;
+}
+
+
+unsigned int OnAction_p2p(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_P2P
+	u8 *frame_body;
+	u8 category, OUI_Subtype, dialogToken = 0;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	uint len = precv_frame->u.hdr.len;
+	struct	wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+
+	/* check RA matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(padapter), GetAddr1Ptr(pframe), ETH_ALEN))
+		return _SUCCESS;
+
+	frame_body = (unsigned char *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+
+	category = frame_body[0];
+	if (category != RTW_WLAN_CATEGORY_P2P)
+		return _SUCCESS;
+
+	if (cpu_to_be32(*((u32 *)(frame_body + 1))) != P2POUI)
+		return _SUCCESS;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (adapter_wdev_data(padapter)->p2p_enabled) {
+		rtw_cfg80211_rx_action_p2p(padapter, precv_frame);
+		return _SUCCESS;
+	}
+#endif /* CONFIG_IOCTL_CFG80211 */
+#endif /* CONFIG_P2P */
+
+	return _SUCCESS;
+
+}
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+unsigned int OnAction_tbtx_token(_adapter *padapter, union recv_frame *precv_frame)
+{
+#define TOKEN_REQ 0x00
+#define TOKEN_REL 0x01
+
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	u32 xmit_time;
+	u8 *src=NULL, *pframe = precv_frame->u.hdr.rx_data;
+	u8 tbtx_action_code;
+	u8 i, nr_send;
+	uint tx_duration = 0;
+
+	if (padapter->tbtx_capability == _FALSE)
+		goto exit;
+
+	tbtx_action_code = *(pframe + WLAN_HDR_A3_LEN + 1);
+
+
+	switch (tbtx_action_code)
+	{
+		case TOKEN_REQ:
+			// parse duration
+			tx_duration = le32_to_cpu(*(uint *)(pframe + WLAN_HDR_A3_LEN + 2));
+			padapter->tbtx_duration = tx_duration/1000; // Mirocsecond to Millisecond
+			ATOMIC_SET(&padapter->tbtx_tx_pause, _FALSE);
+			rtw_tx_control_cmd(padapter);
+			_set_timer(&pmlmeext->tbtx_xmit_timer, padapter->tbtx_duration);
+			ATOMIC_SET(&padapter->tbtx_remove_tx_pause, _FALSE);
+			rtw_tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
+			break;
+#ifdef CONFIG_AP_MODE
+		case TOKEN_REL:
+			src = get_addr2_ptr(pframe);
+			if (!src)
+				goto exit;
+			psta = rtw_get_stainfo(&padapter->stapriv, src);
+			if (!psta)
+				goto exit;
+
+			if (ATOMIC_READ(&pstapriv->nr_token_keeper) < 1)
+				goto exit;
+
+			for (i=0; i< NR_MAXSTA_INSLOT; i++) {
+				if (pstapriv->token_holder[i] == psta) {
+					pstapriv->token_holder[i] = NULL;
+					//RTW_INFO("macaddr1:" MAC_FMT "\n", MAC_ARG(psta->phl_sta->mac_addr));
+					ATOMIC_DEC(&pstapriv->nr_token_keeper);
+					break;
+				}
+			}
+
+			if (ATOMIC_READ(&pstapriv->nr_token_keeper) == 0)
+				_set_timer(&pmlmeext->tbtx_token_dispatch_timer, 1);
+
+			break;
+#endif
+		default:
+			RTW_INFO("Undefined Action Code\n");
+			goto exit;
+			break;
+	}
+
+exit:
+	return _SUCCESS;
+}
+
+void rtw_issue_action_token_rel(_adapter *padapter)
+{
+
+	// Todo: 
+	// gen token
+	/* Token Release Format
+		Category code : 	1 Byte
+		Action code : 		1 Byte */
+	int ret = _FAIL;
+	//u16	*fctrl;
+	u8	val = 0x01;
+	u8	category = RTW_WLAN_CATEGORY_TBTX;
+	u8	*pframe;
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct	mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK 	*padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info	*lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+	//struct sta_info			*psta;
+	//struct sta_priv			*pstapriv = &padapter->stapriv;
+	//struct registry_priv		*pregpriv = &padapter->registrypriv;
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/*update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgnt_tx_rate(padapter, padapter_link, IEEE80211_OFDM_RATE_24MB); // issue action request using OFDM rate? 20190322 Bruce add
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	//fctrl = &(pwlanhdr->frame_ctl);
+	//*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&(lmlmeinfo->network)), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(lmlmeinfo->network)), ETH_ALEN);
+
+	// SetSeqNum??
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(val), &(pattrib->pktlen));
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	//RTW_INFO("%s\n", __func__);
+
+}
+#endif
+
+unsigned int on_action_s1g(_adapter *padapter, union recv_frame *precv_frame)
+{
+	u8 ret = _FAIL;
+	u8 *pframe = NULL;
+	u8 *pframe_body = NULL;
+	u8 action_code = 0;
+	struct rtw_ieee80211_hdr_3addr *whdr;
+	struct sta_info *psta;
+	u32 frame_len = 0;
+
+	pframe = precv_frame->u.hdr.rx_data;
+
+	/* check RA matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(padapter), GetAddr1Ptr(pframe), ETH_ALEN))
+		goto exit;
+
+	whdr = (struct rtw_ieee80211_hdr_3addr *)pframe;
+	pframe_body = (unsigned char *)(pframe +
+	sizeof(struct rtw_ieee80211_hdr_3addr));
+
+	frame_len = precv_frame->u.hdr.len - WLAN_HDR_A3_LEN;
+
+	action_code = pframe_body[1];
+
+	switch (action_code) {
+#ifdef CONFIG_TWT
+		case RTW_WLAN_ACTION_UNP_S1G_TWT_SETUP:
+		case RTW_WLAN_ACTION_UNP_S1G_TWT_TEARDOWN:
+		case RTW_WLAN_ACTION_UNP_S1G_TWT_INFO:
+			ret = rtw_core_twt_on_action(padapter, whdr->addr2,
+				pframe_body, frame_len);
+			break;
+#endif
+		default:
+			RTW_INFO("[%s] unsupported action_code %d\n",__func__, action_code);
+			break;
+	}
+
+exit:
+	return ret;
+}
+
+unsigned int OnAction(_adapter *padapter, union recv_frame *precv_frame)
+{
+	int i;
+	unsigned char	category;
+	struct action_handler *ptable;
+	unsigned char	*frame_body;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+
+	frame_body = (unsigned char *)(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+
+	category = frame_body[0];
+
+	for (i = 0; i < sizeof(OnAction_tbl) / sizeof(struct action_handler); i++) {
+		ptable = &OnAction_tbl[i];
+
+		if (category == ptable->num)
+			ptable->func(padapter, precv_frame);
+
+	}
+
+	return _SUCCESS;
+
+}
+
+unsigned int DoReserved(_adapter *padapter, union recv_frame *precv_frame)
+{
+
+	/* RTW_INFO("rcvd mgt frame(%x, %x)\n", (get_frame_sub_type(pframe) >> 4), *(unsigned int *)GetAddr1Ptr(pframe)); */
+	return _SUCCESS;
+}
+
+struct xmit_frame *_alloc_mgtxmitframe(struct xmit_priv *pxmitpriv, bool once)
+{
+	struct xmit_frame *pmgntframe;
+
+	if (once)
+		pmgntframe = rtw_alloc_xmitframe_once(pxmitpriv);
+	else
+		pmgntframe = rtw_alloc_xmitframe_ext(pxmitpriv);
+
+	if (pmgntframe == NULL) {
+		RTW_INFO(FUNC_ADPT_FMT" alloc xmitframe fail, once:%d\n", FUNC_ADPT_ARG(pxmitpriv->adapter), once);
+		goto exit;
+	}
+#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct xmit_buf *pxmitbuf;
+	pxmitbuf = rtw_alloc_xmitbuf_ext(pxmitpriv);
+	if (pxmitbuf == NULL) {
+		RTW_INFO(FUNC_ADPT_FMT" alloc xmitbuf fail\n", FUNC_ADPT_ARG(pxmitpriv->adapter));
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		pmgntframe = NULL;
+		goto exit;
+	}
+
+	pmgntframe->frame_tag = MGNT_FRAMETAG;
+	pmgntframe->pxmitbuf = pxmitbuf;
+	pmgntframe->buf_addr = pxmitbuf->pbuf;
+	pxmitbuf->priv_data = pmgntframe;
+#else
+	pmgntframe->frame_tag = MGNT_FRAMETAG;
+#endif
+
+exit:
+	return pmgntframe;
+
+}
+
+inline struct xmit_frame *alloc_mgtxmitframe(struct xmit_priv *pxmitpriv)
+{
+	return _alloc_mgtxmitframe(pxmitpriv, _FALSE);
+}
+
+inline struct xmit_frame *alloc_mgtxmitframe_once(struct xmit_priv *pxmitpriv)
+{
+	return _alloc_mgtxmitframe(pxmitpriv, _TRUE);
+}
+
+
+/****************************************************************************
+
+Following are some TX fuctions for WiFi MLME
+
+*****************************************************************************/
+
+void update_mgnt_tx_rate(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 rate)
+{
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+
+	pmlmeext->tx_rate = rate;
+	/* RTW_INFO("%s(): rate = %x\n",__FUNCTION__, rate); */
+}
+
+
+void update_monitor_frame_attrib(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			struct pkt_attrib *pattrib)
+{
+	u8	wireless_mode;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+	struct sta_info		*psta = NULL;
+	struct sta_info		*bmc_sta = NULL;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+
+	psta = rtw_get_stainfo(pstapriv, pattrib->ra);
+	bmc_sta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+
+	if (bmc_sta == NULL) {
+		RTW_ERR("%s bmc_sta=NULL\n", __func__);
+		return;
+	}
+
+	pattrib->hdrlen = 24;
+	pattrib->nr_frags = 1;
+	pattrib->priority = 7;
+	pattrib->mac_id = bmc_sta->phl_sta->macid;
+	pattrib->adapter_link = padapter_link;
+
+	pattrib->pktlen = 0;
+
+	if (padapter_link->mlmeextpriv.tx_rate == IEEE80211_CCK_RATE_1MB)
+		wireless_mode = WLAN_MD_11B;
+	else
+		wireless_mode = WLAN_MD_11G;
+
+
+#ifdef CONFIG_80211AC_VHT
+	pattrib->rate = MGN_VHT1SS_MCS9;
+#else
+	pattrib->rate = MGN_MCS7;
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	/* CONFIG_80211AX_HE_TODO */
+#endif
+
+	pattrib->encrypt = _NO_PRIVACY_;
+	pattrib->bswenc = _FALSE;
+
+	pattrib->qos_en = _FALSE;
+	pattrib->ht_en = 1;
+	pattrib->bwmode = CHANNEL_WIDTH_20;
+	pattrib->ch_offset = CHAN_OFFSET_NO_EXT;
+	pattrib->sgi = _FALSE;
+
+	pattrib->seqnum = pmlmeext->mgnt_seq;
+
+	pattrib->retry_ctrl = _TRUE;
+
+	pattrib->mbssid = 0;
+	pattrib->hw_ssn_sel = pxmitpriv->hw_ssn_seq_no;
+
+}
+
+
+void update_mgntframe_attrib(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				struct pkt_attrib *pattrib)
+{
+	u8	wireless_mode;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+	struct sta_info		*bmc_sta = NULL;
+
+#ifdef CONFIG_P2P_PS_NOA_USE_MACID_SLEEP
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P_PS_NOA_USE_MACID_SLEEP */
+
+	/* _rtw_memset((u8 *)(pattrib), 0, sizeof(struct pkt_attrib)); */
+
+	bmc_sta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+	if (bmc_sta == NULL) {
+		RTW_ERR("%s bmc_sta=NULL\n", __func__);
+		rtw_warn_on(1);
+	}
+	pattrib->type = WIFI_MGT_TYPE;
+	pattrib->hdrlen = 24;
+	pattrib->nr_frags = 1;
+	pattrib->priority = 8;
+	if (bmc_sta != NULL)
+		pattrib->mac_id = bmc_sta->phl_sta->macid;
+	pattrib->adapter_link = padapter_link;
+
+#ifdef CONFIG_P2P_PS_NOA_USE_MACID_SLEEP
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_mi_buddy_check_fwstate(padapter, WIFI_ASOC_STATE))
+#endif /* CONFIG_CONCURRENT_MODE */
+		if (MLME_IS_GC(padapter)) {
+			if (pwdinfo->p2p_ps_mode > P2P_PS_NONE) {
+				struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+				struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+				WLAN_BSSID_EX *cur_network = &(pmlmeinfo->network);
+				struct sta_priv *pstapriv = &padapter->stapriv;
+				struct sta_info *psta;
+
+				psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+				if (psta) {
+					/* use macid sleep during NoA, mgmt frame use ac queue & ap macid */
+					pattrib->mac_id = psta->phl_sta->macid;
+				} else {
+					if (pwdinfo->p2p_ps_state != P2P_PS_DISABLE)
+						RTW_ERR("%s , psta was NULL\n", __func__);
+				}
+			}
+		}
+#endif /* CONFIG_P2P_PS_NOA_USE_MACID_SLEEP */
+
+
+	pattrib->pktlen = 0;
+
+	if (IS_CCK_RATE(padapter_link->mlmeextpriv.tx_rate))
+		wireless_mode = WLAN_MD_11B;
+	else
+		wireless_mode = WLAN_MD_11G;
+	pattrib->rate = padapter_link->mlmeextpriv.tx_rate;
+
+	pattrib->encrypt = _NO_PRIVACY_;
+	pattrib->bswenc = _FALSE;
+
+	pattrib->qos_en = _FALSE;
+	pattrib->ht_en = _FALSE;
+	pattrib->bwmode = CHANNEL_WIDTH_20;
+	pattrib->ch_offset = CHAN_OFFSET_NO_EXT;
+	pattrib->sgi = _FALSE;
+
+	pattrib->seqnum = pmlmeext->mgnt_seq;
+
+	pattrib->retry_ctrl = _TRUE;
+
+	pattrib->mbssid = 0;
+	pattrib->hw_ssn_sel = pxmitpriv->hw_ssn_seq_no;
+}
+
+void update_mgntframe_attrib_addr(_adapter *padapter, struct xmit_frame *pmgntframe)
+{
+	u8	*pframe;
+	struct pkt_attrib	*pattrib = &pmgntframe->attrib;
+#if defined(CONFIG_BEAMFORMING) || defined(CONFIG_ANTENNA_DIVERSITY)
+	struct sta_info		*sta = NULL;
+#endif
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+
+	_rtw_memcpy(pattrib->ra, GetAddr1Ptr(pframe), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, get_addr2_ptr(pframe), ETH_ALEN);
+
+#if defined(CONFIG_BEAMFORMING) || defined(CONFIG_ANTENNA_DIVERSITY)
+	sta = pattrib->psta;
+	if (!sta) {
+		sta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+		pattrib->psta = sta;
+	}
+#endif /* defined(CONFIG_BEAMFORMING) || defined(CONFIG_ANTENNA_DIVERSITY) */
+}
+
+void dump_mgntframe(_adapter *padapter, struct xmit_frame *pmgntframe)
+{
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(&padapter->xmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(&padapter->xmitpriv, pmgntframe);
+		return;
+	}
+
+	/*rtw_hal_mgnt_xmit(padapter, pmgntframe);*/
+	rtw_mgnt_xmit(padapter, pmgntframe);
+}
+
+s32 dump_mgntframe_and_wait(_adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms)
+{
+	s32 ret = _FAIL;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct xmit_buf *pxmitbuf = pmgntframe->pxmitbuf;
+	struct submit_ctx sctx;
+	unsigned long sp_flags;
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		rtw_free_xmitbuf(&padapter->xmitpriv, pmgntframe->pxmitbuf);
+		rtw_free_xmitframe(&padapter->xmitpriv, pmgntframe);
+		return ret;
+	}
+
+	rtw_sctx_init(&sctx, timeout_ms);
+	pxmitbuf->sctx = &sctx;
+
+	/*ret = rtw_hal_mgnt_xmit(padapter, pmgntframe);*/
+	ret = rtw_mgnt_xmit(padapter, pmgntframe);
+
+	if (ret == _SUCCESS)
+		ret = rtw_sctx_wait(&sctx, __func__);
+
+	_rtw_spinlock_irq(&pxmitpriv->lock_sctx, &sp_flags);
+	pxmitbuf->sctx = NULL;
+	_rtw_spinunlock_irq(&pxmitpriv->lock_sctx, &sp_flags);
+	#else
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		rtw_free_xmitframe(&padapter->xmitpriv, pmgntframe);
+		return ret;
+	}
+	#endif
+	return ret;
+}
+
+s32 dump_mgntframe_and_wait_ack_timeout(_adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms)
+{
+#ifdef CONFIG_XMIT_ACK
+	static u8 seq_no = 0;
+	s32 ret = _FAIL;
+	#ifdef CONFIG_XMIT_ACK_BY_REL_RPT
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	#else
+	struct xmit_priv *pxmitpriv = &(GET_PRIMARY_ADAPTER(padapter))->xmitpriv;
+	#endif
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		rtw_free_xmitframe(&padapter->xmitpriv, pmgntframe);
+		return -1;
+	}
+
+	_rtw_mutex_lock_interruptible(&pxmitpriv->ack_tx_mutex);
+	pxmitpriv->ack_tx = _TRUE;
+	pxmitpriv->ack_tx_seq_no = seq_no++;
+	pmgntframe->ack_report = 1;
+	rtw_sctx_init(&(pxmitpriv->ack_tx_ops), timeout_ms);
+	/*if (rtw_hal_mgnt_xmit(padapter, pmgntframe) == _SUCCESS)*/
+	if (rtw_mgnt_xmit(padapter, pmgntframe) == _SUCCESS)
+		ret = rtw_sctx_wait(&(pxmitpriv->ack_tx_ops), __func__);
+
+	pxmitpriv->ack_tx = _FALSE;
+	_rtw_mutex_unlock(&pxmitpriv->ack_tx_mutex);
+
+	return ret;
+#else /* !CONFIG_XMIT_ACK */
+	dump_mgntframe(padapter, pmgntframe);
+	rtw_msleep_os(50);
+	return _SUCCESS;
+#endif /* !CONFIG_XMIT_ACK */
+}
+
+s32 dump_mgntframe_and_wait_ack(_adapter *padapter, struct xmit_frame *pmgntframe)
+{
+	/* In this case, use 500 ms as the default wait_ack timeout */
+	return dump_mgntframe_and_wait_ack_timeout(padapter, pmgntframe, 500);
+}
+
+#ifdef RTW_PHL_BCN //core ops
+s32 rtw_core_issue_beacon(_adapter *padapter, struct xmit_frame *pmgntframe)
+{
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct rtw_bcn_info_cmn *bcn_cmn = NULL;
+	struct pkt_attrib	*pattrib = &pmgntframe->attrib;
+	u8	*pframe = (u8 *)(pmgntframe->buf_addr);
+	void *phl = padapter->dvobj->phl;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+
+	if(!wrole)
+		return _FAIL;
+
+	if(pattrib->pktlen > MAX_BCN_SIZE)
+		return _FAIL;
+
+	bcn_cmn = &padapter_link->wrlink->bcn_cmn;
+	_rtw_memcpy(bcn_cmn->bcn_buf, pframe, pattrib->pktlen);
+	bcn_cmn->bcn_length = pattrib->pktlen;
+	{
+		u8 *pie_start, *pie_tim = NULL;
+		u32 total_ielen, tim_ielen = 0;
+
+		pie_start = pframe + (sizeof(struct rtw_ieee80211_hdr_3addr) + _FIXED_IE_LENGTH_);
+		total_ielen = pattrib->pktlen - (sizeof(struct rtw_ieee80211_hdr_3addr) + _FIXED_IE_LENGTH_);
+
+		pie_tim = rtw_get_ie(pie_start, _TIM_IE_, &tim_ielen, total_ielen);
+
+		if(pie_tim)
+			bcn_cmn->ie_offset_tim = (u32)(pie_tim - pie_start) + _FIXED_IE_LENGTH_;
+	}
+
+	if(bcn_cmn->bcn_added)
+		rtw_phl_cmd_issue_beacon(phl, padapter_link->wrlink, bcn_cmn, PHL_CMD_DIRECTLY, 0);
+	else {
+		struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+		struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+		WLAN_BSSID_EX *cur_network = &(pmlmeinfo->network);
+
+		_rtw_memcpy(bcn_cmn->bssid, get_my_bssid(cur_network), ETH_ALEN);
+		bcn_cmn->bcn_id = padapter->iface_id;
+		bcn_cmn->role_idx = wrole->id;
+		bcn_cmn->lidx = padapter_link->wrlink->id;
+		bcn_cmn->bcn_interval = pmlmeinfo->bcn_interval;
+		bcn_cmn->bcn_offload = (BIT(BCN_HW_SEQ) | BIT(BCN_HW_TIM));
+
+		if (pmlmeext->chandef.band == BAND_ON_24G
+		  #ifdef CONFIG_P2P
+			&& !rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)
+		  #endif
+		   )
+			bcn_cmn->bcn_rate = RTW_DATA_RATE_CCK1;
+		else
+			bcn_cmn->bcn_rate = RTW_DATA_RATE_OFDM6;
+
+		rtw_phl_cmd_issue_beacon(phl, padapter_link->wrlink, bcn_cmn, PHL_CMD_DIRECTLY, 0);
+		bcn_cmn->bcn_added = 1;
+
+		/*
+				[todo]
+				bcn_id: mbssid-ieee
+				bcn_offload: tbd
+				bcn_rate, 0=cck 1=ofdm, 2mac can NOT cck
+		*/
+	}
+	return _SUCCESS;
+}
+#endif
+
+int update_hidden_ssid(u8 *ies, u32 ies_len, u8 hidden_ssid_mode)
+{
+	u8 *ssid_ie;
+	sint ssid_len_ori;
+	int len_diff = 0;
+
+	ssid_ie = rtw_get_ie(ies,  WLAN_EID_SSID, &ssid_len_ori, ies_len);
+
+	/* RTW_INFO("%s hidden_ssid_mode:%u, ssid_ie:%p, ssid_len_ori:%d\n", __FUNCTION__, hidden_ssid_mode, ssid_ie, ssid_len_ori); */
+
+	if (ssid_ie && ssid_len_ori > 0) {
+		switch (hidden_ssid_mode) {
+		case 1: {
+			u8 *next_ie = ssid_ie + 2 + ssid_len_ori;
+			u32 remain_len = 0;
+
+			remain_len = ies_len - (next_ie - ies);
+
+			ssid_ie[1] = 0;
+			_rtw_memcpy(ssid_ie + 2, next_ie, remain_len);
+			len_diff -= ssid_len_ori;
+
+			break;
+		}
+		case 2:
+			_rtw_memset(&ssid_ie[2], 0, ssid_len_ori);
+			break;
+		default:
+			break;
+		}
+	}
+
+	return len_diff;
+}
+
+u8 *rtw_build_reduced_nb_rpt(_adapter *padapter, struct pkt_attrib *pattrib, u8 *pframe)
+{
+	u32 len = 0;
+	len = rtw_phl_build_reduced_nb_rpt(padapter->phl_role, pattrib->adapter_link->wrlink, pframe);
+	pattrib->pktlen += len;
+
+	return pframe + len;
+}
+
+void issue_beacon(_adapter *padapter, int timeout_ms)
+{
+	struct xmit_frame	*pmgntframe;
+	struct pkt_attrib	*pattrib;
+	unsigned char	*pframe;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	unsigned short *fctrl;
+	unsigned int	rate_len;
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+#endif /* #if defined (CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME) */
+	struct mlme_ext_info	*pmlmeinfo = &(padapter->mlmeextpriv.mlmext_info);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*lmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*lmlmeinfo = &(lmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*cur_network = &(lmlmeinfo->network);
+	u8	bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+	u8 *p = NULL;
+	uint ie_len = 0;
+#endif
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+#if 0 /*def CONFIG_BCN_ICF*/
+	pmgntframe = rtw_alloc_bcnxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+#else
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+#endif
+	{
+		RTW_INFO("%s, alloc mgnt frame fail\n", __FUNCTION__);
+		return;
+	}
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	_rtw_spinlock_bh(&padapter_link->mlmepriv.bcn_update_lock);
+#endif /* #if defined (CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME) */
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+#if 0 /*defined(CONFIG_CONCURRENT_MODE)*/
+	if (padapter->hw_port == HW_PORT1)
+		pattrib->mbssid = 1;
+#endif
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(cur_network), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, 0/*pmlmeext->mgnt_seq*/);
+	/* pmlmeext->mgnt_seq++; */
+	set_frame_sub_type(pframe, WIFI_BEACON);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	if (MLME_IS_AP(padapter) || MLME_IS_MESH(padapter)) {
+		/* RTW_INFO("ie len=%d\n", cur_network->IELength); */
+#ifdef CONFIG_P2P
+		/* for P2P : Primary Device Type & Device Name */
+		u32 wpsielen = 0, insert_len = 0;
+		u8 *wpsie = NULL;
+		wpsie = rtw_get_wps_ie(cur_network->IEs + _FIXED_IE_LENGTH_, cur_network->IELength - _FIXED_IE_LENGTH_, NULL, &wpsielen);
+
+		if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO) && wpsie && wpsielen > 0) {
+			uint wps_offset, remainder_ielen;
+			u8 *premainder_ie, *pframe_wscie;
+
+			wps_offset = (uint)(wpsie - cur_network->IEs);
+
+			premainder_ie = wpsie + wpsielen;
+
+			remainder_ielen = cur_network->IELength - wps_offset - wpsielen;
+
+#ifdef CONFIG_IOCTL_CFG80211
+			if (adapter_wdev_data(padapter)->p2p_enabled) {
+				if (pmlmepriv->wps_beacon_ie && pmlmepriv->wps_beacon_ie_len > 0) {
+#ifdef PRIVATE_R
+					/* if no feature need to show ssid, hide GO ssid in beacon */
+					struct registry_priv	*pregpriv = &padapter->registrypriv;
+					int len_diff;
+					u8 hide_ssid = 0;
+
+					if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO) && pregpriv->go_hidden_ssid_mode == ALL_HIDE_SSID)
+						hide_ssid = 1;
+#endif
+					_rtw_memcpy(pframe, cur_network->IEs, wps_offset);
+#ifdef PRIVATE_R
+					len_diff = update_hidden_ssid(
+								pframe + _BEACON_IE_OFFSET_
+								, wps_offset - _BEACON_IE_OFFSET_
+								, hide_ssid);
+					pframe += len_diff;
+					pattrib->pktlen += len_diff;
+#endif
+					pframe += wps_offset;
+					pattrib->pktlen += wps_offset;
+
+					_rtw_memcpy(pframe, pmlmepriv->wps_beacon_ie, pmlmepriv->wps_beacon_ie_len);
+					pframe += pmlmepriv->wps_beacon_ie_len;
+					pattrib->pktlen += pmlmepriv->wps_beacon_ie_len;
+
+					/* copy remainder_ie to pframe */
+					_rtw_memcpy(pframe, premainder_ie, remainder_ielen);
+					pframe += remainder_ielen;
+					pattrib->pktlen += remainder_ielen;
+				} else {
+					_rtw_memcpy(pframe, cur_network->IEs, cur_network->IELength);
+					pframe += cur_network->IELength;
+					pattrib->pktlen += cur_network->IELength;
+				}
+			}
+#endif /* CONFIG_IOCTL_CFG80211 */
+		} else
+#endif /* CONFIG_P2P */
+		{
+			int len_diff;
+			_rtw_memcpy(pframe, cur_network->IEs, cur_network->IELength);
+			len_diff = update_hidden_ssid(
+					   pframe + _BEACON_IE_OFFSET_
+				   , cur_network->IELength - _BEACON_IE_OFFSET_
+					   , pmlmeinfo->hidden_ssid_mode
+				   );
+			pframe += (cur_network->IELength + len_diff);
+			pattrib->pktlen += (cur_network->IELength + len_diff);
+		}
+
+		{
+			u8 *wps_ie;
+			uint wps_ielen;
+			u8 sr = 0;
+			wps_ie = rtw_get_wps_ie(pmgntframe->buf_addr + TXDESC_OFFSET + sizeof(struct rtw_ieee80211_hdr_3addr) + _BEACON_IE_OFFSET_,
+				pattrib->pktlen - sizeof(struct rtw_ieee80211_hdr_3addr) - _BEACON_IE_OFFSET_, NULL, &wps_ielen);
+			if (wps_ie && wps_ielen > 0)
+				rtw_get_wps_attr_content(wps_ie,  wps_ielen, WPS_ATTR_SELECTED_REGISTRAR, (u8 *)(&sr), NULL);
+			if (sr != 0)
+				set_fwstate(pmlmepriv, WIFI_UNDER_WPS);
+			else
+				_clr_fwstate_(pmlmepriv, WIFI_UNDER_WPS);
+		}
+
+#ifdef CONFIG_RTW_MBO
+		rtw_mbo_build_beacon_ies(padapter, &pframe, pattrib);
+#endif
+#ifdef CONFIG_P2P
+		if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+			u32 len = 0;
+#ifdef CONFIG_IOCTL_CFG80211
+			if (adapter_wdev_data(padapter)->p2p_enabled) {
+				len = pmlmepriv->p2p_beacon_ie_len;
+				if (pmlmepriv->p2p_beacon_ie && len > 0) {
+					_rtw_memcpy(pframe, pmlmepriv->p2p_beacon_ie, len);
+					pframe += len;
+					pattrib->pktlen += len;
+				}
+			}
+#endif /* CONFIG_IOCTL_CFG80211 */
+#ifdef CONFIG_P2P_PS
+			pframe = rtw_append_p2p_go_noa_ie(padapter, pframe,
+							  &pattrib->pktlen);
+#endif
+#ifdef CONFIG_WFD
+			len = rtw_append_beacon_wfd_ie(padapter, pframe);
+			pframe += len;
+			pattrib->pktlen += len;
+#endif
+		}
+#endif /* CONFIG_P2P */
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+		pattrib->pktlen += rtw_build_vendor_ie(padapter , &pframe , WIFI_BEACON_VENDOR_IE_BIT);
+#endif
+
+#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+		if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+			/* check if country IE is included */
+			p = rtw_get_ie(BSS_EX_TLV_IES(cur_network), WLAN_EID_COUNTRY, &ie_len, BSS_EX_TLV_IES_LEN(cur_network));
+
+			if (!p)
+				pframe += rtw_set_ie_country(padapter, pmgntframe->buf_addr + TXDESC_OFFSET + sizeof(struct rtw_ieee80211_hdr_3addr), &(pattrib->pktlen));
+		}
+#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+		if (padapter->tbtx_capability == _TRUE)
+			pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 8, REALTEK_TBTX_IE, &pattrib->pktlen);
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+		if (pmlmepriv->mlopriv.mlo_option == _TRUE) {
+			pframe = rtw_build_reduced_nb_rpt(padapter, pattrib, pframe);
+			pframe = rtw_build_ml_ies(padapter, NULL, pframe, pattrib, PACKET_BEACON, NULL, 0, 0);
+		}
+#endif
+		goto _issue_bcn;
+
+	}
+
+	/* below for ad-hoc mode */
+
+	/* timestamp will be inserted by hardware */
+	pframe += 8;
+	pattrib->pktlen += 8;
+
+	/* beacon interval: 2 bytes */
+
+	_rtw_memcpy(pframe, (unsigned char *)(rtw_get_beacon_interval_from_ie(cur_network->IEs)), 2);
+
+	pframe += 2;
+	pattrib->pktlen += 2;
+
+	/* capability info: 2 bytes */
+
+	_rtw_memcpy(pframe, (unsigned char *)(rtw_get_capability_from_ie(cur_network->IEs)), 2);
+
+	pframe += 2;
+	pattrib->pktlen += 2;
+
+	/* SSID */
+	pframe = rtw_set_ie(pframe, _SSID_IE_, cur_network->Ssid.SsidLength, cur_network->Ssid.Ssid, &pattrib->pktlen);
+
+	/* supported rates... */
+	rate_len = rtw_get_rateset_len(cur_network->SupportedRates);
+	pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, ((rate_len > 8) ? 8 : rate_len), cur_network->SupportedRates, &pattrib->pktlen);
+
+	/* DS parameter set */
+	pframe = rtw_set_ie(pframe, _DSSET_IE_, 1, (unsigned char *)&(cur_network->Configuration.DSConfig), &pattrib->pktlen);
+
+	/* if( (pmlmeinfo->state&0x03) == WIFI_FW_ADHOC_STATE) */
+	{
+		u8 erpinfo = 0;
+		u32 ATIMWindow;
+		/* IBSS Parameter Set... */
+		/* ATIMWindow = cur->Configuration.ATIMWindow; */
+		ATIMWindow = 0;
+		pframe = rtw_set_ie(pframe, _IBSS_PARA_IE_, 2, (unsigned char *)(&ATIMWindow), &pattrib->pktlen);
+
+		/* ERP IE */
+		pframe = rtw_set_ie(pframe, _ERPINFO_IE_, 1, &erpinfo, &pattrib->pktlen);
+	}
+
+
+	/* EXTERNDED SUPPORTED RATE */
+	if (rate_len > 8)
+		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_, (rate_len - 8), (cur_network->SupportedRates + 8), &pattrib->pktlen);
+
+
+	/* todo:HT for adhoc */
+
+_issue_bcn:
+
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	padapter_link->mlmepriv.update_bcn = _FALSE;
+
+	_rtw_spinunlock_bh(&padapter_link->mlmepriv.bcn_update_lock);
+#endif /* #if defined (CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME) */
+
+	if ((pattrib->pktlen + TXDESC_SIZE) > MAX_BEACON_LEN) {
+		RTW_ERR("beacon frame too large ,len(%d,%d)\n",
+			(pattrib->pktlen + TXDESC_SIZE), MAX_BEACON_LEN);
+		rtw_warn_on(1);
+		return;
+	}
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	/* RTW_INFO("issue bcn_sz=%d\n", pattrib->last_txcmdsz); */
+	
+#ifdef RTW_PHL_BCN
+	rtw_core_issue_beacon(padapter, pmgntframe);
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	rtw_free_xmitbuf(&padapter->xmitpriv, pmgntframe->pxmitbuf);
+	#endif
+	rtw_free_xmitframe(&padapter->xmitpriv, pmgntframe);
+#else
+	if (timeout_ms > 0)
+		dump_mgntframe_and_wait(padapter, pmgntframe, timeout_ms);
+	else
+		dump_mgntframe(padapter, pmgntframe);
+#endif
+
+}
+
+u8 *rtw_build_probersp_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 *pframe, struct pkt_attrib *pattrib, u8 is_valid_p2p_probereq)
+{
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	u8 *pwps_ie;
+	uint wps_ielen;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+#endif /* #if defined (CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME) */
+	WLAN_BSSID_EX *cur_network = &(padapter_link->mlmeextpriv.mlmext_info.network);
+	unsigned int	rate_len;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+	u8 *pframe_start = pframe;
+#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+	u8 *p = NULL;
+	uint ie_len = 0;
+#endif
+
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) {
+		pwps_ie = rtw_get_wps_ie(cur_network->IEs + _FIXED_IE_LENGTH_, cur_network->IELength - _FIXED_IE_LENGTH_, NULL, &wps_ielen);
+
+		/* inerset & update wps_probe_resp_ie */
+		if ((pmlmepriv->wps_probe_resp_ie != NULL) && pwps_ie && (wps_ielen > 0)) {
+			uint wps_offset, remainder_ielen;
+			u8 *premainder_ie;
+
+			wps_offset = (uint)(pwps_ie - cur_network->IEs);
+
+			premainder_ie = pwps_ie + wps_ielen;
+
+			remainder_ielen = cur_network->IELength - wps_offset - wps_ielen;
+
+			_rtw_memcpy(pframe, cur_network->IEs, wps_offset);
+			pframe += wps_offset;
+			pattrib->pktlen += wps_offset;
+
+			wps_ielen = (uint)pmlmepriv->wps_probe_resp_ie[1];/* to get ie data len */
+			if ((wps_offset + wps_ielen + 2) <= MAX_IE_SZ) {
+				_rtw_memcpy(pframe, pmlmepriv->wps_probe_resp_ie, wps_ielen + 2);
+				pframe += wps_ielen + 2;
+				pattrib->pktlen += wps_ielen + 2;
+			}
+
+			if ((wps_offset + wps_ielen + 2 + remainder_ielen) <= MAX_IE_SZ) {
+				_rtw_memcpy(pframe, premainder_ie, remainder_ielen);
+				pframe += remainder_ielen;
+				pattrib->pktlen += remainder_ielen;
+			}
+		} else {
+			_rtw_memcpy(pframe, cur_network->IEs, cur_network->IELength);
+			pframe += cur_network->IELength;
+			pattrib->pktlen += cur_network->IELength;
+		}
+
+		/* retrieve SSID IE from cur_network->Ssid */
+		{
+			u8 *ssid_ie;
+			sint ssid_ielen;
+			sint ssid_ielen_diff;
+			u8 buf[MAX_IE_SZ];
+			u8 *ies = pframe_start;
+
+			ssid_ie = rtw_get_ie(ies + _FIXED_IE_LENGTH_, _SSID_IE_, &ssid_ielen,
+				     (pframe - ies) - _FIXED_IE_LENGTH_);
+
+			ssid_ielen_diff = cur_network->Ssid.SsidLength - ssid_ielen;
+
+			if (ssid_ie &&  cur_network->Ssid.SsidLength) {
+				uint remainder_ielen;
+				u8 *remainder_ie;
+				remainder_ie = ssid_ie + 2;
+				remainder_ielen = (pframe - remainder_ie);
+
+				if (remainder_ielen > MAX_IE_SZ) {
+					RTW_WARN(FUNC_ADPT_FMT" remainder_ielen > MAX_IE_SZ\n", FUNC_ADPT_ARG(padapter));
+					remainder_ielen = MAX_IE_SZ;
+				}
+
+				_rtw_memcpy(buf, remainder_ie, remainder_ielen);
+				_rtw_memcpy(remainder_ie + ssid_ielen_diff, buf, remainder_ielen);
+				*(ssid_ie + 1) = cur_network->Ssid.SsidLength;
+				_rtw_memcpy(ssid_ie + 2, cur_network->Ssid.Ssid, cur_network->Ssid.SsidLength);
+
+				pframe += ssid_ielen_diff;
+				pattrib->pktlen += ssid_ielen_diff;
+			}
+		}
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+		pattrib->pktlen += rtw_build_vendor_ie(padapter , &pframe , WIFI_PROBERESP_VENDOR_IE_BIT);
+#endif
+#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+		if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+			/* check if country IE is included */
+			p = rtw_get_ie(BSS_EX_TLV_IES(cur_network), WLAN_EID_COUNTRY, &ie_len, BSS_EX_TLV_IES_LEN(cur_network));
+
+			if (!p)
+				pframe += rtw_set_ie_country(padapter, pframe_start, &(pattrib->pktlen));
+		}
+#endif
+	} else
+#endif
+	{
+
+		/* timestamp will be inserted by hardware */
+		pframe += 8;
+		pattrib->pktlen += 8;
+
+		/* beacon interval: 2 bytes */
+
+		_rtw_memcpy(pframe, (unsigned char *)(rtw_get_beacon_interval_from_ie(cur_network->IEs)), 2);
+
+		pframe += 2;
+		pattrib->pktlen += 2;
+
+		/* capability info: 2 bytes */
+
+		_rtw_memcpy(pframe, (unsigned char *)(rtw_get_capability_from_ie(cur_network->IEs)), 2);
+
+		pframe += 2;
+		pattrib->pktlen += 2;
+
+		/* below for ad-hoc mode */
+
+		/* SSID */
+		pframe = rtw_set_ie(pframe, _SSID_IE_, cur_network->Ssid.SsidLength, cur_network->Ssid.Ssid, &pattrib->pktlen);
+
+		/* supported rates... */
+		rate_len = rtw_get_rateset_len(cur_network->SupportedRates);
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, ((rate_len > 8) ? 8 : rate_len), cur_network->SupportedRates, &pattrib->pktlen);
+
+		/* DS parameter set */
+		pframe = rtw_set_ie(pframe, _DSSET_IE_, 1, (unsigned char *)&(cur_network->Configuration.DSConfig), &pattrib->pktlen);
+
+		if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
+			u8 erpinfo = 0;
+			u32 ATIMWindow;
+			/* IBSS Parameter Set... */
+			/* ATIMWindow = cur->Configuration.ATIMWindow; */
+			ATIMWindow = 0;
+			pframe = rtw_set_ie(pframe, _IBSS_PARA_IE_, 2, (unsigned char *)(&ATIMWindow), &pattrib->pktlen);
+
+			/* ERP IE */
+			pframe = rtw_set_ie(pframe, _ERPINFO_IE_, 1, &erpinfo, &pattrib->pktlen);
+		}
+
+
+		/* EXTERNDED SUPPORTED RATE */
+		if (rate_len > 8)
+			pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_, (rate_len - 8), (cur_network->SupportedRates + 8), &pattrib->pktlen);
+
+
+		/* todo:HT for adhoc */
+
+	}
+
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_build_probe_rsp_ies(padapter, &pframe, pattrib);
+#endif
+#ifdef CONFIG_P2P
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)
+	    /* IOT issue, When wifi_spec is not set, send probe_resp with P2P IE even if probe_req has no P2P IE */
+	    && (is_valid_p2p_probereq || !padapter->registrypriv.wifi_spec)) {
+		u32 len = 0;
+#ifdef CONFIG_IOCTL_CFG80211
+		if (adapter_wdev_data(padapter)->p2p_enabled) {
+			/* if pwdinfo->role == P2P_ROLE_DEVICE will call issue_probersp_p2p() */
+			len = pmlmepriv->p2p_go_probe_resp_ie_len;
+			if (pmlmepriv->p2p_go_probe_resp_ie && len > 0) {
+				_rtw_memcpy(pframe, pmlmepriv->p2p_go_probe_resp_ie, len);
+				pframe += len;
+				pattrib->pktlen += len;
+			}
+		}
+#endif /* CONFIG_IOCTL_CFG80211 */
+#ifdef CONFIG_P2P_PS
+		pframe = rtw_append_p2p_go_noa_ie(padapter, pframe,
+						  &pattrib->pktlen);
+#endif
+#ifdef CONFIG_WFD
+		len = rtw_append_probe_resp_wfd_ie(padapter, pframe);
+		pframe += len;
+		pattrib->pktlen += len;
+#endif
+	}
+#endif /* CONFIG_P2P */
+
+
+#ifdef CONFIG_AUTO_AP_MODE
+	{
+		struct sta_info	*psta;
+		struct sta_priv *pstapriv = &padapter->stapriv;
+
+		RTW_INFO("(%s)\n", __FUNCTION__);
+
+		/* check rc station */
+		psta = rtw_get_stainfo(pstapriv, da);
+		if (psta && psta->isrc && psta->pid > 0) {
+			u8 RC_OUI[4] = {0x00, 0xE0, 0x4C, 0x0A};
+			u8 RC_INFO[14] = {0};
+			/* EID[1] + EID_LEN[1] + RC_OUI[4] + MAC[6] + PairingID[2] + ChannelNum[2] */
+			u16 cu_ch = (u16)cur_network->Configuration.DSConfig;
+
+			RTW_INFO("%s, reply rc(pid=0x%x) device "MAC_FMT" in ch=%d\n", __FUNCTION__,
+				 psta->pid, MAC_ARG(psta->phl_sta->mac_addr), cu_ch);
+
+			/* append vendor specific ie */
+			_rtw_memcpy(RC_INFO, RC_OUI, sizeof(RC_OUI));
+			_rtw_memcpy(&RC_INFO[4], mac, ETH_ALEN);
+			_rtw_memcpy(&RC_INFO[10], (u8 *)&psta->pid, 2);
+			_rtw_memcpy(&RC_INFO[12], (u8 *)&cu_ch, 2);
+
+			pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, sizeof(RC_INFO), RC_INFO, &pattrib->pktlen);
+		}
+	}
+#endif /* CONFIG_AUTO_AP_MODE */
+	return pframe;
+}
+
+void issue_probersp(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *da, u8 is_valid_p2p_probereq)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	unsigned char					*mac, *bssid;
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	WLAN_BSSID_EX		*cur_network = &(padapter_link->mlmeextpriv.mlmext_info.network);
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	if (da == NULL)
+		return;
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_INFO("%s, alloc mgnt frame fail\n", __FUNCTION__);
+		return;
+	}
+
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	mac = padapter_link->mac_addr;
+	bssid = cur_network->MacAddress;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, bssid, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(fctrl, WIFI_PROBERSP);
+
+	pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = pattrib->hdrlen;
+	pframe += pattrib->hdrlen;
+
+
+	if (cur_network->IELength > MAX_IE_SZ)
+		return;
+
+	pframe = rtw_build_probersp_ies(padapter, padapter_link, pframe, pattrib, is_valid_p2p_probereq);
+
+#ifdef CONFIG_80211BE_EHT
+	if (padapter->mlmepriv.mlopriv.mlo_option == _TRUE) {
+		pframe = rtw_build_reduced_nb_rpt(padapter, pattrib, pframe);
+		pframe = rtw_build_ml_ies(padapter, NULL, pframe, pattrib, PACKET_PROBE_RESPONSE,
+					NULL, 0, 0);
+	}
+#endif
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	return;
+
+}
+
+#ifdef CONFIG_80211BE_EHT
+void issue_ml_probersp(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, unsigned char *da,
+			u8 is_valid_p2p_probereq)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	unsigned char					*mac, *bssid;
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	WLAN_BSSID_EX		*cur_network = &(padapter_link->mlmeextpriv.mlmext_info.network);
+	u8 *primary_ies;
+	u32 primary_ies_len = 0;
+
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	if (da == NULL)
+		return;
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_INFO("%s, alloc mgnt frame fail\n", __FUNCTION__);
+		return;
+	}
+
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	mac = padapter_link->mac_addr;
+	bssid = cur_network->MacAddress;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, bssid, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(fctrl, WIFI_PROBERSP);
+
+	pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = pattrib->hdrlen;
+	pframe += pattrib->hdrlen;
+
+
+	if (cur_network->IELength > MAX_IE_SZ)
+		return;
+
+	primary_ies = pframe;
+	pframe = rtw_build_probersp_ies(padapter, padapter_link, pframe, pattrib, is_valid_p2p_probereq);
+	primary_ies_len = pattrib->pktlen - sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_build_reduced_nb_rpt(padapter, pattrib, pframe);
+
+	pframe = rtw_build_ml_ies(padapter, NULL, pframe, pattrib, PACKET_PROBE_RESPONSE_ML,
+		primary_ies, primary_ies_len, is_valid_p2p_probereq);
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	return;
+}
+#endif
+
+int _issue_probereq(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		const NDIS_802_11_SSID *pssid, const u8 *da, u8 ch, bool append_wps, int wait_ack)
+{
+	int ret = _FAIL;
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	unsigned char			*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short		*fctrl;
+	unsigned char			*mac;
+	unsigned char			bssrate[NumRates];
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_priv		*pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	int	bssrate_len = 0;
+	u8	bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u32 len = 0;
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+#endif
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+	if ((pwdev_priv->random_mac_enabled)
+	    && (MLME_IS_STA(padapter))
+	    && (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _FALSE))
+		mac = pwdev_priv->pno_mac_addr;
+	else
+#endif
+	mac = padapter_link->mac_addr;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	if (da) {
+		/*	unicast probe request frame */
+		_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, da, ETH_ALEN);
+	} else {
+		/*	broadcast probe request frame */
+		_rtw_memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, bc_addr, ETH_ALEN);
+	}
+
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+	if ((pwdev_priv->random_mac_enabled)
+	    && (MLME_IS_STA(padapter))
+	    && (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _FALSE)) {
+#ifdef CONFIG_RTW_DEBUG
+		RTW_DBG("%s pno_scan_seq_num: %d\n", __func__,
+			 pwdev_priv->pno_scan_seq_num);
+#endif
+		SetSeqNum(pwlanhdr, pwdev_priv->pno_scan_seq_num);
+		pattrib->seqnum = pwdev_priv->pno_scan_seq_num;
+		pattrib->qos_en = 1;
+		pwdev_priv->pno_scan_seq_num++;
+	} else
+#endif
+	{
+		SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+		pmlmeext->mgnt_seq++;
+	}
+	set_frame_sub_type(pframe, WIFI_PROBEREQ);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	if (pssid && !MLME_IS_MESH(padapter))
+		pframe = rtw_set_ie(pframe, _SSID_IE_, pssid->SsidLength, pssid->Ssid, &(pattrib->pktlen));
+	else
+		pframe = rtw_set_ie(pframe, _SSID_IE_, 0, NULL, &(pattrib->pktlen));
+
+	get_rate_set(padapter, padapter_link, bssrate, &bssrate_len);
+
+	if (bssrate_len > 8) {
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , 8, bssrate, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_ , (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
+	} else
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , bssrate_len , bssrate, &(pattrib->pktlen));
+
+	if (ch)
+		pframe = rtw_set_ie(pframe, _DSSET_IE_, 1, &ch, &pattrib->pktlen);
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		if (pssid)
+			pframe = rtw_set_ie_mesh_id(pframe, &pattrib->pktlen, pssid->Ssid, pssid->SsidLength);
+		else
+			pframe = rtw_set_ie_mesh_id(pframe, &pattrib->pktlen, NULL, 0);
+	}
+#endif
+
+#ifdef CONFIG_80211N_HT
+	if (pregistrypriv->ht_enable == _TRUE
+	    && is_supported_ht(pregistrypriv->wireless_mode)) {
+		len = 0;
+		rtw_ht_use_default_setting(padapter, padapter_link);
+		rtw_restructure_ht_ie(padapter, padapter_link, NULL, pframe, 0,
+				      &len, rtw_get_oper_ch(padapter, padapter_link));
+		pattrib->pktlen += len;
+		pframe += len;
+	}
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	if ((pregistrypriv->ht_enable == _TRUE)
+	     && (rtw_get_oper_band(padapter, padapter_link) == BAND_ON_5G)
+	     && REGSTY_IS_11AC_ENABLE(pregistrypriv)
+	     && is_supported_vht(pregistrypriv->wireless_mode)
+	     && RFCTL_REG_EN_11AC(rfctl)
+	) {
+		rtw_vht_get_real_setting(padapter, padapter_link);
+		len = rtw_build_vht_cap_ie(padapter, padapter_link, pframe);
+		pattrib->pktlen += len;
+		pframe += len;
+	}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	if (((pregistrypriv->ht_enable == _TRUE
+		&& (rtw_get_oper_band(padapter, padapter_link) == BAND_ON_24G))
+#ifdef CONFIG_80211AC_VHT
+		|| (REGSTY_IS_11AC_ENABLE(pregistrypriv)
+			&& (rtw_get_oper_band(padapter, padapter_link) == BAND_ON_5G))
+#endif
+		)
+		&& REGSTY_IS_11AX_ENABLE(pregistrypriv)
+		&& is_supported_he(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AX(rfctl)
+		) {
+		len = rtw_build_he_cap_ie(padapter, padapter_link, pframe, rtw_get_oper_band(padapter, padapter_link));
+		pattrib->pktlen += len;
+		pframe += len;
+	}
+#endif
+
+	if (append_wps) {
+		/* add wps_ie for wps2.0 */
+		if (pmlmepriv->wps_probe_req_ie_len > 0 && pmlmepriv->wps_probe_req_ie) {
+			_rtw_memcpy(pframe, pmlmepriv->wps_probe_req_ie, pmlmepriv->wps_probe_req_ie_len);
+			pframe += pmlmepriv->wps_probe_req_ie_len;
+			pattrib->pktlen += pmlmepriv->wps_probe_req_ie_len;
+			/* pmlmepriv->wps_probe_req_ie_len = 0 ; */ /* reset to zero */
+		}
+	}
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	pattrib->pktlen += rtw_build_vendor_ie(padapter , &pframe , WIFI_PROBEREQ_VENDOR_IE_BIT);
+#endif
+
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_build_probe_req_ies(padapter, &pframe, pattrib);
+#endif
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+	return ret;
+}
+
+inline void issue_probereq(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			const NDIS_802_11_SSID *pssid, const u8 *da)
+{
+	_issue_probereq(padapter, padapter_link, pssid, da, 0, 1, _FALSE);
+}
+
+/*
+ * wait_ms == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ * wait_ms > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ * try_cnt means the maximal TX count to try
+ */
+int issue_probereq_ex(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			const NDIS_802_11_SSID *pssid, const u8 *da, u8 ch, bool append_wps,
+			int try_cnt, int wait_ms)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	do {
+		ret = _issue_probereq(padapter, padapter_link,
+					pssid, da, ch, append_wps, wait_ms > 0 ? _TRUE : _FALSE);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		if (da)
+			RTW_INFO(FUNC_ADPT_FMT" to "MAC_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter), MAC_ARG(da),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+		else
+			RTW_INFO(FUNC_ADPT_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+exit:
+	return ret;
+}
+
+/* if psta == NULL, indiate we are station(client) now... */
+void issue_auth(_adapter *padapter, struct sta_info *psta, unsigned short status)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	unsigned int					val32;
+	unsigned short				val16;
+	int use_shared_key = 0;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	struct link_mlme_ext_info *lmlmeinfo;
+
+	if (psta) /* AP Mode */
+		padapter_link = psta->padapter_link;
+	else /* STA Mode */
+		/* Send auth req via the primary link */
+		padapter_link = GET_PRIMARY_LINK(padapter);
+
+ 	lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_AUTH);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+
+	if (psta) { /* for AP mode */
+#ifdef CONFIG_NATIVEAP_MLME
+		_rtw_memcpy(pwlanhdr->addr1, psta->phl_sta->mac_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, padapter_link->mac_addr, ETH_ALEN);
+
+
+		/* setting auth algo number */
+		val16 = (u16)psta->authalg;
+
+		if (status != _STATS_SUCCESSFUL_)
+			val16 = 0;
+
+		if (val16)	{
+			val16 = cpu_to_le16(val16);
+			use_shared_key = 1;
+		}
+
+		pframe = rtw_set_fixed_ie(pframe, _AUTH_ALGM_NUM_, (unsigned char *)&val16, &(pattrib->pktlen));
+
+		/* setting auth seq number */
+		val16 = (u16)psta->auth_seq;
+		val16 = cpu_to_le16(val16);
+		pframe = rtw_set_fixed_ie(pframe, _AUTH_SEQ_NUM_, (unsigned char *)&val16, &(pattrib->pktlen));
+
+		/* setting status code... */
+		val16 = status;
+		val16 = cpu_to_le16(val16);
+		pframe = rtw_set_fixed_ie(pframe, _STATUS_CODE_, (unsigned char *)&val16, &(pattrib->pktlen));
+
+		/* added challenging text... */
+		if ((psta->auth_seq == 2) && (psta->state & WIFI_FW_AUTH_STATE) && (use_shared_key == 1))
+			pframe = rtw_set_ie(pframe, _CHLGETXT_IE_, 128, psta->chg_txt, &(pattrib->pktlen));
+#endif
+	} else {
+		_rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&lmlmeinfo->network), ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&lmlmeinfo->network), ETH_ALEN);
+
+#ifdef CONFIG_RTW_80211R
+		if (rtw_ft_roam(padapter)) {
+			/* 2: 802.11R FTAA */
+			val16 = cpu_to_le16(WLAN_AUTH_FT);
+		} else
+#endif
+		{
+			/* setting auth algo number */
+			val16 = (pmlmeinfo->auth_algo == dot11AuthAlgrthm_Shared) ? 1 : 0;	/* 0:OPEN System, 1:Shared key */
+			if (val16) {
+				val16 = cpu_to_le16(val16);
+				use_shared_key = 1;
+			}
+		}
+
+		/* RTW_INFO("%s auth_algo= %s auth_seq=%d\n",__FUNCTION__,(pmlmeinfo->auth_algo==0)?"OPEN":"SHARED",pmlmeinfo->auth_seq); */
+
+		/* setting IV for auth seq #3 */
+		if ((pmlmeinfo->auth_seq == 3) && (pmlmeinfo->state & WIFI_FW_AUTH_STATE) && (use_shared_key == 1)) {
+			/* RTW_INFO("==> iv(%d),key_index(%d)\n",pmlmeinfo->iv,pmlmeinfo->key_index); */
+			val32 = ((pmlmeinfo->iv++) | (psecuritypriv->dot11PrivacyKeyIndex << 30));
+			val32 = cpu_to_le32(val32);
+			pframe = rtw_set_fixed_ie(pframe, 4, (unsigned char *)&val32, &(pattrib->pktlen));
+
+			pattrib->iv_len = 4;
+		}
+
+		pframe = rtw_set_fixed_ie(pframe, _AUTH_ALGM_NUM_, (unsigned char *)&val16, &(pattrib->pktlen));
+
+		/* setting auth seq number */
+		val16 = pmlmeinfo->auth_seq;
+		val16 = cpu_to_le16(val16);
+		pframe = rtw_set_fixed_ie(pframe, _AUTH_SEQ_NUM_, (unsigned char *)&val16, &(pattrib->pktlen));
+
+
+		/* setting status code... */
+		val16 = status;
+		val16 = cpu_to_le16(val16);
+		pframe = rtw_set_fixed_ie(pframe, _STATUS_CODE_, (unsigned char *)&val16, &(pattrib->pktlen));
+
+#ifdef CONFIG_RTW_80211R
+		rtw_ft_build_auth_req_ies(padapter, pattrib, &pframe);
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+		if (padapter->mlmepriv.mlopriv.mlo_option)
+			pframe = rtw_build_ml_ies(padapter, NULL, pframe, pattrib, PACKET_AUTH, NULL, 0, 0);
+#endif
+
+		/* then checking to see if sending challenging text... */
+		if ((pmlmeinfo->auth_seq == 3) && (pmlmeinfo->state & WIFI_FW_AUTH_STATE) && (use_shared_key == 1)) {
+			pframe = rtw_set_ie(pframe, _CHLGETXT_IE_, 128, pmlmeinfo->chg_txt, &(pattrib->pktlen));
+
+			SetPrivacy(fctrl);
+
+			pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+			pattrib->encrypt = _WEP40_;
+
+			pattrib->icv_len = 4;
+
+			pattrib->pktlen += pattrib->icv_len;
+
+			pattrib->bswenc = _TRUE;
+		}
+
+	}
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	rtw_wep_encrypt(padapter, (u8 *)pmgntframe);
+	RTW_INFO("%s\n", __FUNCTION__);
+	dump_mgntframe(padapter, pmgntframe);
+
+	return;
+}
+
+
+void issue_asocrsp(_adapter *padapter, unsigned short status, struct sta_info *pstat, int pkt_type)
+{
+#ifdef CONFIG_AP_MODE
+	struct xmit_frame	*pmgntframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	struct pkt_attrib *pattrib;
+	unsigned char	*pbuf, *pframe;
+	unsigned short val, ie_status;
+	unsigned short *fctrl;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct _ADAPTER_LINK	*padapter_link = pstat->padapter_link;
+	struct link_mlme_priv	*pmlmepriv = &(padapter_link->mlmepriv);
+	WLAN_BSSID_EX *pnetwork = &padapter_link->mlmeextpriv.mlmext_info.network;
+	u8 *ie = pnetwork->IEs;
+	uint ie_len = 0;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+#ifdef CONFIG_WFD
+	u32					wfdielen = 0;
+#endif
+#endif /* CONFIG_P2P */
+#ifdef CONFIG_RTW_MBO
+	u8 WIFI_ALLIANCE_OUI[] = {0x50, 0x6f, 0x9a};
+#endif /* CONFIG_RTW_MBO */
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy((void *)GetAddr1Ptr(pwlanhdr), pstat->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy((void *)get_addr2_ptr(pwlanhdr), padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy((void *)GetAddr3Ptr(pwlanhdr), get_my_bssid(pnetwork), ETH_ALEN);
+
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP))
+		set_frame_sub_type(pwlanhdr, pkt_type);
+	else
+		return;
+
+	pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen += pattrib->hdrlen;
+	pframe += pattrib->hdrlen;
+
+	/* capability */
+	val = *(unsigned short *)rtw_get_capability_from_ie(ie);
+
+	pframe = rtw_set_fixed_ie(pframe, _CAPABILITY_ , (unsigned char *)&val, &(pattrib->pktlen));
+
+	ie_status = cpu_to_le16(status);
+	pframe = rtw_set_fixed_ie(pframe , _STATUS_CODE_ , (unsigned char *)&ie_status, &(pattrib->pktlen));
+
+	val = cpu_to_le16(pstat->phl_sta->aid | BIT(14) | BIT(15));
+	pframe = rtw_set_fixed_ie(pframe, _ASOC_ID_ , (unsigned char *)&val, &(pattrib->pktlen));
+
+	if (pstat->bssratelen <= 8)
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, pstat->bssratelen, pstat->bssrateset, &(pattrib->pktlen));
+	else {
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, 8, pstat->bssrateset, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_, (pstat->bssratelen - 8), pstat->bssrateset + 8, &(pattrib->pktlen));
+	}
+
+#ifdef CONFIG_IEEE80211W
+	if (status == _STATS_REFUSED_TEMPORARILY_) {
+		u8 timeout_itvl[5];
+		u32 timeout_interval = 3000;
+		/* Association Comeback time */
+		timeout_itvl[0] = 0x03;
+		timeout_interval = cpu_to_le32(timeout_interval);
+		_rtw_memcpy(timeout_itvl + 1, &timeout_interval, 4);
+		pframe = rtw_set_ie(pframe, _TIMEOUT_ITVL_IE_, 5, timeout_itvl, &(pattrib->pktlen));
+	}
+#endif /* CONFIG_IEEE80211W */
+
+#ifdef CONFIG_80211N_HT
+	if ((pstat->flags & WLAN_STA_HT) && (pmlmepriv->htpriv.ht_option)) {
+		uint ie_len = 0;
+
+		/* FILL HT CAP INFO IE */
+		/* p = hostapd_eid_ht_capabilities_info(hapd, p); */
+		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_CAPABILITY_IE_, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
+		}
+
+		/* FILL HT ADD INFO IE */
+		/* p = hostapd_eid_ht_operation(hapd, p); */
+		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_ADD_INFO_IE_, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
+		}
+
+	}
+#endif
+
+	/*adding EXT_CAPAB_IE */
+	if (pmlmepriv->ext_capab_ie_len > 0) {
+		uint ie_len = 0;
+
+		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, WLAN_EID_EXT_CAP, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
+		}
+	}
+
+#ifdef CONFIG_RTW_80211K
+	/* Adding RM capability IE */
+	if (padapter->rmpriv.enable == _TRUE) {
+		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _EID_RRM_EN_CAP_IE_, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
+		}
+	}
+#endif /* CONFIG_RTW_80211K */
+
+#ifdef CONFIG_RTW_FSM_RRM
+	/* Adding RM capability IE */
+	if (padapter->fsmpriv.rmpriv.enable == _TRUE) {
+		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _EID_RRM_EN_CAP_IE_, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
+		}
+	}
+#endif /* CONFIG_RTW_FSM_RRM */
+
+#ifdef CONFIG_RTW_MBO
+	if (padapter->mlmepriv.mbopriv.enable == _TRUE) {
+		ie_len = 0;
+		for (pbuf = ie + _BEACON_IE_OFFSET_; ; pbuf += (ie_len + 2)) {
+			pbuf = rtw_get_ie(pbuf, _SSN_IE_1_, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_ - (ie_len + 2)));
+			if ((pbuf) && (_rtw_memcmp(pbuf + 2, WIFI_ALLIANCE_OUI, 3)) && (*(pbuf+5) == MBO_OUI_TYPE)) {
+				/* find MBO-OCE information element */
+				_rtw_memcpy(pframe, pbuf, ie_len + 2);
+				pframe += (ie_len + 2);
+				pattrib->pktlen += (ie_len + 2);
+				break;
+			}
+			if ((pbuf == NULL) || (ie_len == 0))
+				break;
+		}
+	}
+#endif /* CONFIG_RTW_MBO */
+
+#ifdef CONFIG_80211AC_VHT
+	if ((pstat->flags & WLAN_STA_VHT) && (pmlmepriv->vhtpriv.vht_option)
+	    && (pstat->wpa_pairwise_cipher != WPA_CIPHER_TKIP)
+	    && (pstat->wpa2_pairwise_cipher != WPA_CIPHER_TKIP)) {
+		u32 ie_len = 0;
+
+		/* FILL VHT CAP IE */
+		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, EID_VHTCapability, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
+		}
+
+		/* FILL VHT OPERATION IE */
+		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, EID_VHTOperation, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_));
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
+		}
+	}
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+	if ((pstat->flags & WLAN_STA_HE) && (pmlmepriv->hepriv.he_option)) {
+		u32 ie_len = 0;
+		u8 cap;
+
+		/* FILL HE CAP IE */
+		cap = WLAN_EID_EXTENSION_HE_CAPABILITY;
+		pbuf = rtw_get_ie_ex(ie + _BEACON_IE_OFFSET_, pnetwork->IELength - _BEACON_IE_OFFSET_,
+		WLAN_EID_EXTENSION, &cap, 1, NULL, &ie_len);
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len);
+			pframe += ie_len;
+			pattrib->pktlen += ie_len;
+		}
+
+		/* FILL HE OPERATION IE */
+		cap = WLAN_EID_EXTENSION_HE_OPERATION;
+		pbuf = rtw_get_ie_ex(ie + _BEACON_IE_OFFSET_, pnetwork->IELength - _BEACON_IE_OFFSET_,
+		WLAN_EID_EXTENSION, &cap, 1, NULL, &ie_len);
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len);
+			pframe += ie_len;
+			pattrib->pktlen += ie_len;
+		}
+#if CONFIG_IEEE80211_BAND_6GHZ
+		/* FILL HE 6G BAND CAP IE */
+		cap = WLAN_EID_EXT_HE_6G_CAP;
+		pbuf = rtw_get_ie_ex(ie + _BEACON_IE_OFFSET_, pnetwork->IELength - _BEACON_IE_OFFSET_,
+		WLAN_EID_EXTENSION, &cap, 1, NULL, &ie_len);
+		if (pbuf && ie_len > 0) {
+			_rtw_memcpy(pframe, pbuf, ie_len);
+			pframe += ie_len;
+			pattrib->pktlen += ie_len;
+		}
+#endif
+	}
+#endif /* CONFIG_80211AX_HE */
+
+
+#ifdef CONFIG_RTW_80211R_AP
+	rtw_ft_build_assoc_rsp_ies(padapter, pstat, pattrib, &pframe);
+#endif
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_build_assoc_rsp_ies(padapter, &pframe, pattrib);
+#endif
+
+	/* FILL WMM IE */
+	if ((pstat->flags & WLAN_STA_WME) && (pmlmepriv->qospriv.qos_option)) {
+		uint ie_len = 0;
+		unsigned char WMM_PARA_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x01, 0x01};
+
+		for (pbuf = ie + _BEACON_IE_OFFSET_; ; pbuf += (ie_len + 2)) {
+			pbuf = rtw_get_ie(pbuf, _VENDOR_SPECIFIC_IE_, &ie_len, (pnetwork->IELength - _BEACON_IE_OFFSET_ - (ie_len + 2)));
+			if (pbuf && _rtw_memcmp(pbuf + 2, WMM_PARA_IE, 6)) {
+				_rtw_memcpy(pframe, pbuf, ie_len + 2);
+				pframe += (ie_len + 2);
+				pattrib->pktlen += (ie_len + 2);
+
+				break;
+			}
+
+			if ((pbuf == NULL) || (ie_len == 0))
+				break;
+		}
+
+	}
+
+
+	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK)
+		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 6 , REALTEK_96B_IE, &(pattrib->pktlen));
+
+	/* add WPS IE ie for wps 2.0 */
+	if (padapter->mlmepriv.wps_assoc_resp_ie && padapter->mlmepriv.wps_assoc_resp_ie_len > 0) {
+		_rtw_memcpy(pframe, padapter->mlmepriv.wps_assoc_resp_ie, padapter->mlmepriv.wps_assoc_resp_ie_len);
+
+		pframe += padapter->mlmepriv.wps_assoc_resp_ie_len;
+		pattrib->pktlen += padapter->mlmepriv.wps_assoc_resp_ie_len;
+	}
+
+
+	if ((psecuritypriv->auth_type == MLME_AUTHTYPE_SAE) &&
+		padapter->mlmepriv.assoc_rsp &&
+			padapter->mlmepriv.assoc_rsp_len > 0) {
+		_rtw_memcpy(pframe, padapter->mlmepriv.assoc_rsp,
+			padapter->mlmepriv.assoc_rsp_len);
+		pframe += padapter->mlmepriv.assoc_rsp_len;
+		pattrib->pktlen += padapter->mlmepriv.assoc_rsp_len;
+	}
+
+
+#ifdef CONFIG_P2P
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO) && (pstat->is_p2p_device == _TRUE)) {
+		u32 len = 0;
+
+		if (padapter->mlmepriv.p2p_assoc_resp_ie && padapter->mlmepriv.p2p_assoc_resp_ie_len > 0) {
+			len = padapter->mlmepriv.p2p_assoc_resp_ie_len;
+			_rtw_memcpy(pframe, padapter->mlmepriv.p2p_assoc_resp_ie, len);
+		} else if (padapter->mlmepriv.assoc_rsp && padapter->mlmepriv.assoc_rsp_len > 0) {
+			len = padapter->mlmepriv.assoc_rsp_len;
+			_rtw_memcpy(pframe, padapter->mlmepriv.assoc_rsp, len);
+		}
+
+		pframe += len;
+		pattrib->pktlen += len;
+	}
+
+#ifdef CONFIG_WFD
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+		wfdielen = rtw_append_assoc_resp_wfd_ie(padapter, pframe);
+		pframe += wfdielen;
+		pattrib->pktlen += wfdielen;
+	}
+#endif
+
+#endif /* CONFIG_P2P */
+
+#ifdef CONFIG_RTW_MULTI_AP
+	if (padapter->multi_ap && (pstat->flags & WLAN_STA_MULTI_AP))
+		pframe = rtw_set_multi_ap_ie_ext(pframe, &pattrib->pktlen, padapter->multi_ap);
+#endif
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	pattrib->pktlen += rtw_build_vendor_ie(padapter , &pframe , WIFI_ASSOCRESP_VENDOR_IE_BIT);
+#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	if (padapter->tbtx_capability == _TRUE)
+		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 8, REALTEK_TBTX_IE, &pattrib->pktlen);
+#endif
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+
+#endif
+}
+
+static u32 rtw_append_assoc_req_owe_ie(_adapter *adapter, u8 *pbuf)
+{
+	struct security_priv *sec = &adapter->securitypriv;
+	u32 len = 0;
+
+	if (sec == NULL)
+		goto exit;
+
+	if (sec->owe_ie_len > 0) {
+		len = sec->owe_ie_len;
+		_rtw_memcpy(pbuf, sec->owe_ie, len);
+	}
+
+exit:
+	return len;
+}
+
+int rtw_build_assocreq_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *pframe, struct pkt_attrib *pattrib, u8 is_reassoc)
+{
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct link_mlme_ext_priv *lmlmeext= &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *lmlmeinfo = &lmlmeext->mlmext_info;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	unsigned short				val16;
+#if CONFIG_DFS
+	u16	cap;
+#endif
+	unsigned char	bssrate[NumRates], sta_bssrate[NumRates];
+	int	bssrate_len = 0, sta_bssrate_len = 0;
+	unsigned int	i, j, index = 0;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8	vs_ie_length = 0;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	u8					p2pie[255] = { 0x00 };
+	u16					p2pielen = 0;
+#ifdef CONFIG_WFD
+	u32					wfdielen = 0;
+#endif
+#endif /* CONFIG_P2P */
+	int ret = _SUCCESS;
+
+	/* caps */
+
+#if CONFIG_DFS
+	_rtw_memcpy(&cap, rtw_get_capability_from_ie(lmlmeinfo->network.IEs), 2);
+	cap |= cap_SpecMgmt;
+	_rtw_memcpy(pframe, &cap, 2);
+#else
+	_rtw_memcpy(pframe, rtw_get_capability_from_ie(lmlmeinfo->network.IEs), 2);
+#endif
+
+	pframe += 2;
+	pattrib->pktlen += 2;
+
+	/* listen interval */
+	/* todo: listen interval for power saving */
+	val16 = cpu_to_le16(3);
+	_rtw_memcpy(pframe , (unsigned char *)&val16, 2);
+	pframe += 2;
+	pattrib->pktlen += 2;
+
+	/*Construct Current AP Field for Reassoc-Req only*/
+	if (is_reassoc == _TRUE) {
+		_rtw_memcpy(pframe, get_my_bssid(&(pmlmeinfo->dev_network)), ETH_ALEN);
+		pframe += ETH_ALEN;
+		pattrib->pktlen += ETH_ALEN;
+	}
+
+	/* SSID */
+	pframe = rtw_set_ie(pframe, _SSID_IE_, lmlmeinfo->network.Ssid.SsidLength, lmlmeinfo->network.Ssid.Ssid, &(pattrib->pktlen));
+#if CONFIG_IEEE80211_BAND_5GHZ && CONFIG_DFS
+	/* Dot H */
+	if (lmlmeext->chandef.band == BAND_ON_5G) {
+		struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+		u8 pow_cap_ele[2] = { 0x00 };
+
+		pow_cap_ele[0] = 13;	/* Minimum transmit power capability */
+		pow_cap_ele[1] = 21;	/* Maximum transmit power capability */
+		pframe = rtw_set_ie(pframe, EID_PowerCap, 2, pow_cap_ele, &(pattrib->pktlen));
+
+		pframe = rtw_chset_set_spt_chs_ie(&rfctl->chset, pframe, &(pattrib->pktlen));
+	}
+#endif /* CONFIG_IEEE80211_BAND_5GHZ && CONFIG_DFS */
+
+	/* supported rate & extended supported rate */
+
+#if 1	/* Check if the AP's supported rates are also supported by STA. */
+	get_rate_set(padapter, padapter_link, sta_bssrate, &sta_bssrate_len);
+	/* RTW_INFO("sta_bssrate_len=%d\n", sta_bssrate_len); */
+
+	if (lmlmeext->chandef.band == BAND_ON_24G && lmlmeext->chandef.chan == 14)
+		sta_bssrate_len = 4; /* for JAPAN, channel 14 can only uses B Mode(CCK) */
+
+
+	/* for (i = 0; i < sta_bssrate_len; i++) { */
+	/*	RTW_INFO("sta_bssrate[%d]=%02X\n", i, sta_bssrate[i]); */
+	/* } */
+
+	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
+		if (lmlmeinfo->network.SupportedRates[i] == 0)
+			break;
+		RTW_INFO("network.SupportedRates[%d]=%02X\n", i, lmlmeinfo->network.SupportedRates[i]);
+	}
+
+
+	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
+		if (lmlmeinfo->network.SupportedRates[i] == 0)
+			break;
+
+
+		/* Check if the AP's supported rates are also supported by STA. */
+		for (j = 0; j < sta_bssrate_len; j++) {
+			/* Avoid the proprietary data rate (22Mbps) of Handlink WSG-4000 AP */
+			if ((lmlmeinfo->network.SupportedRates[i] | IEEE80211_BASIC_RATE_MASK)
+			    == (sta_bssrate[j] | IEEE80211_BASIC_RATE_MASK)) {
+				/* RTW_INFO("match i = %d, j=%d\n", i, j); */
+				break;
+			} else {
+				/* RTW_INFO("not match: %02X != %02X\n", (lmlmeinfo->network.SupportedRates[i]|IEEE80211_BASIC_RATE_MASK), (sta_bssrate[j]|IEEE80211_BASIC_RATE_MASK)); */
+			}
+		}
+
+		if (j == sta_bssrate_len) {
+			/* the rate is not supported by STA */
+			RTW_INFO("%s(): the rate[%d]=%02X is not supported by STA!\n", __FUNCTION__, i, lmlmeinfo->network.SupportedRates[i]);
+		} else {
+			/* the rate is supported by STA */
+			bssrate[index++] = lmlmeinfo->network.SupportedRates[i];
+		}
+	}
+
+	bssrate_len = index;
+	RTW_INFO("bssrate_len = %d\n", bssrate_len);
+
+#else	/* Check if the AP's supported rates are also supported by STA. */
+#if 0
+	get_rate_set(padapter, bssrate, &bssrate_len);
+#else
+	for (bssrate_len = 0; bssrate_len < NumRates; bssrate_len++) {
+		if (lmlmeinfo->network.SupportedRates[bssrate_len] == 0)
+			break;
+
+		if (lmlmeinfo->network.SupportedRates[bssrate_len] == 0x2C) /* Avoid the proprietary data rate (22Mbps) of Handlink WSG-4000 AP */
+			break;
+
+		bssrate[bssrate_len] = lmlmeinfo->network.SupportedRates[bssrate_len];
+	}
+#endif
+#endif /* Check if the AP's supported rates are also supported by STA. */
+
+	if ((bssrate_len == 0) && (lmlmeinfo->network.SupportedRates[0] != 0)) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		ret = _FAIL;
+		goto _exit; /* don't connect to AP if no joint supported rate */
+	}
+
+
+	if (bssrate_len > 8) {
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , 8, bssrate, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, _EXT_SUPPORTEDRATES_IE_ , (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
+	} else if (bssrate_len > 0)
+		pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_ , bssrate_len , bssrate, &(pattrib->pktlen));
+	else
+		RTW_INFO("%s: Connect to AP without 11b and 11g data rate!\n", __FUNCTION__);
+
+#ifdef CONFIG_ECSA_PHL
+	if (rtw_is_ecsa_enabled(padapter))
+		pframe = build_supported_op_class_ie(padapter, pframe, &pattrib->pktlen);
+#endif
+
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_build_assoc_req_ies(padapter, &pframe, pattrib);
+#endif
+#ifdef CONFIG_RTW_80211R
+	rtw_ft_build_assoc_req_ies(padapter, is_reassoc, pattrib, &pframe);
+#endif
+#ifdef CONFIG_RTW_80211K
+	if (lmlmeinfo->network.PhyInfo.rm_en_cap[0] /* RM Enabled Capabilities */
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[1]
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[2]
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[3]
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[4])
+		pframe = rtw_set_ie(pframe, _EID_RRM_EN_CAP_IE_, 5,
+				(u8 *)padapter->rmpriv.rm_en_cap_def, &(pattrib->pktlen));
+#endif /* CONFIG_RTW_80211K */
+
+#ifdef CONFIG_RTW_FSM_RRM
+	if (lmlmeinfo->network.PhyInfo.rm_en_cap[0] /* RM Enabled Capabilities */
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[1]
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[2]
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[3]
+		| lmlmeinfo->network.PhyInfo.rm_en_cap[4])
+		pframe = rtw_set_ie(pframe, _EID_RRM_EN_CAP_IE_, 5,
+				(u8 *)padapter->fsmpriv.rmpriv.rm_en_cap_def, &(pattrib->pktlen));
+#endif /* CONFIG_RTW_FSM_RRM */
+
+	/* vendor specific IE, such as WPA, WMM, WPS */
+	for (i = _FIXED_IE_LENGTH_; i < lmlmeinfo->network.IELength;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(lmlmeinfo->network.IEs + i);
+		switch (pIE->ElementID) {
+		case _VENDOR_SPECIFIC_IE_:
+			if ((_rtw_memcmp(pIE->data, RTW_WPA_OUI, 4)) ||
+			    (_rtw_memcmp(pIE->data, WMM_OUI, 4)) ||
+			    (_rtw_memcmp(pIE->data, WPS_OUI, 4))) {
+				vs_ie_length = pIE->Length;
+				if ((!padapter->registrypriv.wifi_spec) && (_rtw_memcmp(pIE->data, WPS_OUI, 4))) {
+					/* Commented by Kurt 20110629 */
+					/* In some older APs, WPS handshake */
+					/* would be fail if we append vender extensions informations to AP */
+
+					vs_ie_length = 14;
+				}
+
+				pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, vs_ie_length, pIE->data, &(pattrib->pktlen));
+			}
+			break;
+
+		case EID_WPA2:
+#ifdef CONFIG_RTW_80211R
+			if ((is_reassoc) && (rtw_ft_roam(padapter))) {
+				rtw_ft_update_rsnie(padapter, _TRUE, pattrib, &pframe);
+			} else
+#endif
+			{
+				pframe = rtw_set_ie(pframe, EID_WPA2, pIE->Length, pIE->data, &(pattrib->pktlen));
+				/* tmp: update rsn's spp related opt. */
+				/*rtw_set_spp_amsdu_mode(padapter->registrypriv.amsdu_mode, pframe - (pIE->Length + 2), pIE->Length +2); */
+
+			}
+			break;
+
+		case WLAN_EID_RSNX:
+			pframe = rtw_set_ie(pframe, WLAN_EID_RSNX, pIE->Length,
+					pIE->data, &(pattrib->pktlen));
+			break;
+
+#ifdef CONFIG_80211N_HT
+		case EID_HTCapability:
+			if (pmlmepriv->htpriv.ht_option == _TRUE) {
+				if (!(is_ap_in_tkip(padapter_link))) {
+					_rtw_memcpy(&(lmlmeinfo->HT_caps), pIE->data, sizeof(struct HT_caps_element));
+
+					lmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info = cpu_to_le16(lmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info);
+
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (rtw_phl_mr_is_db(adapter_to_dvobj(padapter)->phl)) {
+						RTW_INFO("%s, stay in dual-band set SM_PS\n", __func__);
+						SET_HT_CAP_ELE_SM_PS(&(lmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info), SM_PS_STATIC);
+					}
+					#endif
+					pframe = rtw_set_ie(pframe, EID_HTCapability, pIE->Length , (u8 *)(&(lmlmeinfo->HT_caps)), &(pattrib->pktlen));
+				}
+			}
+			break;
+#endif /* CONFIG_80211N_HT */
+
+		case WLAN_EID_EXT_CAP:
+			pframe = rtw_set_ie(pframe, WLAN_EID_EXT_CAP, pIE->Length, pIE->data, &(pattrib->pktlen));
+			break;
+
+#ifdef CONFIG_80211AC_VHT
+		case EID_VHTCapability:
+			if (pmlmepriv->vhtpriv.vht_option == _TRUE)
+				pframe = rtw_set_ie(pframe, EID_VHTCapability, pIE->Length, pIE->data, &(pattrib->pktlen));
+			break;
+
+		case EID_OpModeNotification:
+			if (pmlmepriv->vhtpriv.vht_option == _TRUE) {
+#if 0
+				if (rtw_phl_mr_is_db(adapter_to_dvobj(padapter)->phl)) {
+					u8 orig_ie_rx_nss = GET_VHT_OPERATING_MODE_FIELD_RX_NSS(pIE->data);
+					if (orig_ie_rx_nss > 0) {
+						RTW_INFO("%s, change op_mode_notify_ie(RX NSS:%d->0)\n", __func__, orig_ie_rx_nss);
+						/* change op mode notify IE */
+						SET_VHT_OPERATING_MODE_FIELD_RX_NSS(pIE->data, 0);
+						SET_VHT_OPERATING_MODE_FIELD_RX_NSS_TYPE(pIE->data, 0);
+
+					}
+
+				}
+#endif
+				pframe = rtw_set_ie(pframe, EID_OpModeNotification, pIE->Length, pIE->data, &(pattrib->pktlen));
+			}
+			break;
+#endif /* CONFIG_80211AC_VHT */
+		case WLAN_EID_EXTENSION:
+#ifdef CONFIG_80211AX_HE
+			if ((pIE->data[0] == WLAN_EID_EXTENSION_HE_CAPABILITY
+				|| pIE->data[0] == WLAN_EID_EXT_HE_6G_CAP)
+				&& (pmlmepriv->hepriv.he_option == _TRUE))
+				pframe = rtw_set_ie(pframe, WLAN_EID_EXTENSION, pIE->Length, pIE->data, &(pattrib->pktlen));
+#endif /* CONFIG_80211AX_HE */
+			break;
+		default:
+			break;
+		}
+
+		i += (pIE->Length + 2);
+	}
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	if (padapter->tbtx_capability == _TRUE)
+		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 8 , REALTEK_TBTX_IE, &(pattrib->pktlen));
+#endif
+
+	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK)
+		pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, 6 , REALTEK_96B_IE, &(pattrib->pktlen));
+
+
+#ifdef CONFIG_WAPI_SUPPORT
+	rtw_build_assoc_req_wapi_ie(padapter, pframe, pattrib);
+#endif
+
+
+#ifdef CONFIG_P2P
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (adapter_wdev_data(padapter)->p2p_enabled) {
+		if (padapter->mlmepriv.p2p_assoc_req_ie && padapter->mlmepriv.p2p_assoc_req_ie_len > 0) {
+			_rtw_memcpy(pframe, padapter->mlmepriv.p2p_assoc_req_ie, padapter->mlmepriv.p2p_assoc_req_ie_len);
+			pframe += padapter->mlmepriv.p2p_assoc_req_ie_len;
+			pattrib->pktlen += padapter->mlmepriv.p2p_assoc_req_ie_len;
+		}
+	}
+#ifdef CONFIG_CONCURRENT_MODE
+	else if (padapter->registrypriv.wifi_spec && (pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE) {
+		struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+		_adapter *iface;
+		__le16 *cap_attr;
+		u8 *ptemp;
+		void *pattr;
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+
+			if (!iface || iface == padapter || !rtw_is_adapter_up(iface))
+				continue;
+
+			if (!adapter_wdev_data(iface)->p2p_enabled || !iface->mlmepriv.p2p_beacon_ie
+			    || iface->mlmepriv.p2p_beacon_ie_len == 0)
+				continue;
+
+			ptemp = pframe;
+			/* copy P2P IE */
+			_rtw_memcpy(pframe, iface->mlmepriv.p2p_beacon_ie, iface->mlmepriv.p2p_beacon_ie_len);
+			pframe += iface->mlmepriv.p2p_beacon_ie_len;
+			pattrib->pktlen += iface->mlmepriv.p2p_beacon_ie_len;
+
+			/* get 2 bytes P2P CAPABILITY : P2P DEVICE CAPABILITY(1) + P2P GROUP CAPABILITY(1) */
+			pattr = rtw_get_p2p_attr_content(ptemp, iface->mlmepriv.p2p_beacon_ie_len,
+			                                 P2P_ATTR_CAPABILITY, NULL, NULL);
+
+			/* clear P2P concurrent operation bit of P2P device capability
+			 * for avoid association fail with Cisco AP
+			 */
+			if (pattr) {
+				cap_attr = pattr;
+				*cap_attr &= ~cpu_to_le16(P2P_DEVCAP_CONCURRENT_OPERATION);
+			}
+			break;
+		}
+	}
+#endif /* CONFIG_CONCURRENT_MODE */
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+#ifdef CONFIG_WFD
+	wfdielen = rtw_append_assoc_req_wfd_ie(padapter, pframe);
+	pframe += wfdielen;
+	pattrib->pktlen += wfdielen;
+#endif
+#endif /* CONFIG_P2P */
+
+#ifdef CONFIG_RTW_MULTI_AP
+	if (padapter->multi_ap)
+		pframe = rtw_set_multi_ap_ie_ext(pframe, &pattrib->pktlen, padapter->multi_ap);
+#endif
+
+	/* OWE */
+	{
+	u32 owe_ie_len;
+
+	owe_ie_len = rtw_append_assoc_req_owe_ie(padapter, pframe);
+	pframe += owe_ie_len;
+	pattrib->pktlen += owe_ie_len;
+	}
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	pattrib->pktlen += rtw_build_vendor_ie(padapter, &pframe ,WIFI_ASSOCREQ_VENDOR_IE_BIT);
+#endif
+
+_exit:
+	return ret;
+}
+
+void _issue_assocreq(_adapter *padapter, u8 is_reassoc)
+{
+	int ret = _FAIL;
+	struct xmit_frame				*pmgntframe;
+	struct pkt_attrib				*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr			*pwlanhdr;
+	unsigned short				*fctrl;
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	/* Send assoc req via the primary link */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+#ifdef CONFIG_80211BE_EHT
+	u8 *primary_ies;
+	u32 primary_ies_len;
+#endif
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+	_rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, padapter->mlmeextpriv.mgnt_seq);
+	padapter->mlmeextpriv.mgnt_seq++;
+	if (is_reassoc == _TRUE)
+		set_frame_sub_type(pframe, WIFI_REASSOCREQ);
+	else
+		set_frame_sub_type(pframe, WIFI_ASSOCREQ);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+#ifdef CONFIG_80211BE_EHT
+	primary_ies = pframe;
+#endif
+	ret = rtw_build_assocreq_ies(padapter, padapter_link, pframe, pattrib, is_reassoc);
+	pframe += pattrib->pktlen - sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	if (ret != _SUCCESS) {
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit; /* don't connect to AP if no joint supported rate */
+	}
+
+#ifdef CONFIG_80211BE_EHT
+	primary_ies_len = pframe - primary_ies;
+	if (padapter->mlmepriv.mlopriv.mlo_option)
+		pframe = rtw_build_ml_ies(padapter, NULL, pframe, pattrib, PACKET_ASSOC_REQUEST,
+					primary_ies, primary_ies_len, is_reassoc);
+#endif
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	ret = _SUCCESS;
+
+exit:
+	if (ret == _SUCCESS) {
+		rtw_buf_update(&padapter->mlmepriv.assoc_req, &padapter->mlmepriv.assoc_req_len, (u8 *)pwlanhdr, pattrib->pktlen);
+	#ifdef CONFIG_RTW_WNM
+		if (is_reassoc == _TRUE)
+			rtw_wnm_update_reassoc_req_ie(padapter);
+	#endif
+	} else
+		rtw_buf_free(&padapter->mlmepriv.assoc_req, &padapter->mlmepriv.assoc_req_len);
+
+	return;
+}
+
+void issue_assocreq(_adapter *padapter)
+{
+	_issue_assocreq(padapter, _FALSE);
+}
+
+void issue_reassocreq(_adapter *padapter)
+{
+	_issue_assocreq(padapter, _TRUE);
+}
+
+/* when wait_ack is ture, this function shoule be called at process context */
+static int _issue_nulldata(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *da, unsigned int power_mode, int wait_ack)
+{
+	int ret = _FAIL;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	struct xmit_priv	*pxmitpriv;
+	struct mlme_ext_priv	*pmlmeext;
+	struct link_mlme_ext_info	*pmlmeinfo;
+	u8 a4_shift;
+
+	/* RTW_INFO("%s:%d\n", __FUNCTION__, power_mode); */
+
+	if (!padapter)
+		goto exit;
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	pxmitpriv = &(padapter->xmitpriv);
+	pmlmeext = &(padapter->mlmeextpriv);
+	pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+	pattrib->retry_ctrl = _FALSE;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	if (MLME_IS_AP(padapter))
+		SetFrDs(fctrl);
+	else if (MLME_IS_STA(padapter))
+		SetToDs(fctrl);
+	else if (MLME_IS_MESH(padapter)) {
+		SetToDs(fctrl);
+		SetFrDs(fctrl);
+	}
+
+	if (power_mode)
+		SetPwrMgt(fctrl);
+
+	if (get_tofr_ds(fctrl) == 3) {
+		_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr4, padapter_link->mac_addr, ETH_ALEN);
+		a4_shift = ETH_ALEN;
+		pattrib->hdrlen += ETH_ALEN;
+	} else {
+		_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+		a4_shift = 0;
+	}
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_DATA_NULL);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr) + a4_shift;
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr) + a4_shift;
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+	return ret;
+}
+
+/*
+ * When wait_ms > 0, this function should be called at process context
+ * wait_ms == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ * wait_ms > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ * try_cnt means the maximal TX count to try
+ * da == NULL for station mode
+ */
+int issue_nulldata(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *da, unsigned int power_mode, int try_cnt, int wait_ms)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	/* da == NULL, assum it's null data for sta to ap */
+	if (da == NULL)
+		da = get_my_bssid(&(pmlmeinfo->network));
+
+	do {
+		ret = _issue_nulldata(padapter, padapter_link,
+					da, power_mode, wait_ms > 0 ? _TRUE : _FALSE);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		if (da)
+			RTW_INFO(FUNC_ADPT_FMT" to "MAC_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter), MAC_ARG(da),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+		else
+			RTW_INFO(FUNC_ADPT_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+exit:
+	return ret;
+}
+
+static inline void _ps_announce(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, bool ps)
+{
+	RTW_INFO(FUNC_ADPT_FMT" issue_null(%d)\n", FUNC_ADPT_ARG(adapter), ps);
+	if (MLME_IS_STA(adapter)) {
+		if (is_client_associated_to_ap(adapter) == _TRUE) {
+
+			#ifdef CONFIG_XMIT_ACK
+			if (adapter_link->wrlink->mstate == MLME_LINKED)
+				issue_nulldata(adapter, adapter_link, NULL, ps, 3, PS_ANNC_DRV_RETRY_INT_MS);
+			#else
+			if (adapter_link->wrlink->mstate == MLME_LINKED)
+				issue_nulldata(adapter, adapter_link, NULL, ps, 1, 0);
+			#endif
+		}
+	}
+	#ifdef CONFIG_RTW_MESH
+	else if (MLME_IS_MESH(adapter)) {
+		rtw_mesh_ps_annc(adapter, ps);
+	}
+	#endif
+}
+
+bool rtw_core_issu_null_data(void *priv, u8 ridx, u8 link_idx, bool ps, u8 module_id)
+{
+	struct dvobj_priv *obj = (struct dvobj_priv *)priv;
+	_adapter *iface = NULL;
+	struct _ADAPTER_LINK *iface_link = NULL;
+
+	if (ridx >= CONFIG_IFACE_NUMBER) {
+		RTW_ERR("%s ridx:%d invalid\n", __func__, ridx);
+		rtw_warn_on(1);
+		goto _error;
+	}
+
+	iface = obj->padapters[ridx];
+	if (!rtw_is_adapter_up(iface))
+		goto _error;
+
+	iface_link = GET_LINK(iface, link_idx);
+	_ps_announce(iface, iface_link, ps);
+
+	return _SUCCESS;
+_error:
+	return _FAIL;
+}
+
+
+/* when wait_ack is ture, this function shoule be called at process context */
+static int _issue_qos_nulldata(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				unsigned char *da, u16 tid, u8 ps, int wait_ack, u8 om_info)
+{
+	int ret = _FAIL;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl, *qc;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+	u8 a4_shift;
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	pattrib->hdrlen += 2;
+	pattrib->qos_en = _TRUE;
+	pattrib->eosp = 1;
+	pattrib->ack_policy = 0;
+	pattrib->mdata = 0;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	if (MLME_IS_AP(padapter))
+		SetFrDs(fctrl);
+	else if (MLME_IS_STA(padapter))
+		SetToDs(fctrl);
+	else if (MLME_IS_MESH(padapter)) {
+		SetToDs(fctrl);
+		SetFrDs(fctrl);
+	}
+
+	if (ps)
+		SetPwrMgt(fctrl);
+
+	if (pattrib->mdata)
+		SetMData(fctrl);
+
+	if (get_tofr_ds(fctrl) == 3) {
+		_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr4, padapter_link->mac_addr, ETH_ALEN);
+		a4_shift = ETH_ALEN;
+		pattrib->hdrlen += ETH_ALEN;
+	} else {
+		_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+		a4_shift = 0;
+	}
+
+	qc = (unsigned short *)(pframe + pattrib->hdrlen - 2);
+
+	SetPriority(qc, tid);
+
+	SetEOSP(qc, pattrib->eosp);
+
+	SetAckpolicy(qc, pattrib->ack_policy);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_QOS_DATA_NULL);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr_qos) + a4_shift;
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr_qos) + a4_shift;
+
+#ifdef CONFIG_80211AX_HE
+	if (om_info) {
+		set_htc_order_bit(fctrl);
+		pattrib->rate = MGN_HE1SS_MCS0;
+		rtw_he_fill_htc(padapter, pattrib, (u32 *)pframe);
+		pframe = pframe + 4;
+		pattrib->pktlen = pattrib->pktlen + 4;
+	}
+#endif
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+	return ret;
+}
+
+/*
+ * when wait_ms >0 , this function should be called at process context
+ * wait_ms == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ * wait_ms > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ * try_cnt means the maximal TX count to try
+ * da == NULL for station mode
+ */
+int issue_qos_nulldata(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		unsigned char *da, u16 tid, u8 ps, int try_cnt, int wait_ms, u8 need_om)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	/* da == NULL, assum it's null data for sta to ap*/
+	if (da == NULL)
+		da = get_my_bssid(&(pmlmeinfo->network));
+
+	do {
+		ret = _issue_qos_nulldata(padapter, padapter_link,
+					da, tid, ps, wait_ms > 0 ? _TRUE : _FALSE, need_om);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		if (da)
+			RTW_INFO(FUNC_ADPT_FMT" to "MAC_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter), MAC_ARG(da),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+		else
+			RTW_INFO(FUNC_ADPT_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+exit:
+	return ret;
+}
+
+static int _issue_disasoc_deauth(_adapter *padapter, unsigned char *da
+	, u8 disasoc, unsigned short reason, u8 wait_ack, u8 key_type)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info	*pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+	int ret = _FAIL;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P	 */
+
+	/* RTW_INFO("%s to "MAC_FMT"\n", __func__, MAC_ARG(da)); */
+
+	if (alink_is_tx_blocked_by_ch_waiting(padapter_link))
+		goto exit;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+	pattrib->retry_ctrl = _FALSE;
+	pattrib->key_type = key_type;
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, disasoc ? WIFI_DISASSOC : WIFI_DEAUTH);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	reason = cpu_to_le16(reason);
+	pframe = rtw_set_fixed_ie(pframe, _RSON_CODE_ , (unsigned char *)&reason, &(pattrib->pktlen));
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+	return ret;
+}
+
+static int _issue_disasoc(_adapter *padapter, unsigned char *da, unsigned short reason, u8 wait_ack, u8 key_type)
+{
+	return _issue_disasoc_deauth(padapter, da, 1, reason, wait_ack, key_type);
+}
+
+int issue_disasoc(_adapter *padapter, unsigned char *da, unsigned short reason)
+{
+	RTW_INFO("%s reason(%u) to "MAC_FMT"\n", __func__, reason, MAC_ARG(da));
+	return _issue_disasoc(padapter, da, reason, _FALSE, IEEE80211W_RIGHT_KEY);
+}
+
+static int _issue_deauth(_adapter *padapter, unsigned char *da, unsigned short reason, u8 wait_ack, u8 key_type)
+{
+	return _issue_disasoc_deauth(padapter, da, 0, reason, wait_ack, key_type);
+}
+
+int issue_deauth(_adapter *padapter, unsigned char *da, unsigned short reason)
+{
+	RTW_INFO("%s reason(%u) to "MAC_FMT"\n", __func__, reason, MAC_ARG(da));
+	return _issue_deauth(padapter, da, reason, _FALSE, IEEE80211W_RIGHT_KEY);
+}
+
+#ifdef CONFIG_IEEE80211W
+int issue_deauth_11w(_adapter *padapter, unsigned char *da, unsigned short reason, u8 key_type)
+{
+	RTW_INFO("%s to "MAC_FMT"\n", __func__, MAC_ARG(da));
+	return _issue_deauth(padapter, da, reason, _FALSE, key_type);
+}
+#endif /* CONFIG_IEEE80211W */
+
+/*
+ * wait_ms == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ * wait_ms > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ * try_cnt means the maximal TX count to try
+ */
+int issue_deauth_ex(_adapter *padapter, u8 *da, unsigned short reason, int try_cnt,
+		    int wait_ms)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+	/* ToDo CONFIG_RTW_MLDL [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (alink_is_tx_blocked_by_ch_waiting(padapter_link))
+		goto exit;
+
+	do {
+		ret = _issue_deauth(padapter, da, reason, wait_ms > 0 ? _TRUE : _FALSE, IEEE80211W_RIGHT_KEY);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		if (da)
+			RTW_INFO(FUNC_ADPT_FMT" to "MAC_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter), MAC_ARG(da),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+		else
+			RTW_INFO(FUNC_ADPT_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter),
+				rtw_get_oper_ch(padapter, padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+exit:
+	return ret;
+}
+
+#ifdef CONFIG_IEEE80211W
+void issue_action_SA_Query(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *raddr, unsigned char action, unsigned short tid, u8 key_type)
+{
+	u8	category = RTW_WLAN_CATEGORY_SA_QUERY;
+	u16	reason_code;
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	u8					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	u16					*fctrl;
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct sta_info		*psta;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct registry_priv		*pregpriv = &padapter->registrypriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	RTW_INFO("%s, %04x\n", __FUNCTION__, tid);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_INFO("%s: alloc_mgtxmitframe fail\n", __FUNCTION__);
+		return;
+	}
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+	pattrib->key_type = key_type;
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	if (raddr)
+		_rtw_memcpy(pwlanhdr->addr1, raddr, ETH_ALEN);
+	else
+		_rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &category, &pattrib->pktlen);
+	pframe = rtw_set_fixed_ie(pframe, 1, &action, &pattrib->pktlen);
+
+	switch (action) {
+	case 0: /* SA Query req */
+		pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)&pmlmeext->sa_query_seq, &pattrib->pktlen);
+		pmlmeext->sa_query_seq++;
+		/* send sa query request to AP, AP should reply sa query response in 1 second */
+		if (pattrib->key_type == IEEE80211W_RIGHT_KEY) {
+			psta = rtw_get_stainfo(pstapriv, pwlanhdr->addr1);
+			if (psta != NULL) {
+				/* RTW_INFO("%s, %d, set dot11w_expire_timer\n", __func__, __LINE__); */
+				if (_check_timer_is_active(&psta->dot11w_expire_timer)){
+					/* The timer is active, do not set it again */
+					rtw_free_xmitframe(&padapter->xmitpriv, pmgntframe);
+					return;
+				}
+				_set_timer(&psta->dot11w_expire_timer, 1000);
+			}
+		}
+		break;
+
+	case 1: /* SA Query rsp */
+		/* RTW_INFO("rtw_set_fixed_ie, %04x\n", tid); */
+		pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)&tid, &pattrib->pktlen);
+		break;
+	default:
+		break;
+	}
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+}
+#endif /* CONFIG_IEEE80211W */
+
+/**
+ * issue_action_ba - internal function to TX Block Ack action frame
+ * @padapter: the adapter to TX
+ * @raddr: receiver address
+ * @action: Block Ack Action
+ * @tid: tid
+ * @size: the announced AMPDU buffer size. used by ADDBA_RESP
+ * @paddba_req: used by ADDBA_RESP
+ * @status: status/reason code. used by ADDBA_RESP, DELBA
+ * @initiator: if we are the initiator of AMPDU association. used by DELBA
+ * @wait_ack: used xmit ack
+ *
+ * Returns:
+ * _SUCCESS: No xmit ack is used or acked
+ * _FAIL: not acked when using xmit ack
+ */
+static int issue_action_ba(_adapter *padapter, unsigned char *raddr,
+			   unsigned char action, u8 tid, u16 size,
+			   struct ADDBA_request *paddba_req, u16 status,
+			   u8 initiator, int wait_ack)
+{
+	int ret = _FAIL;
+	u8	category = RTW_WLAN_CATEGORY_BACK;
+	u16	start_seq;
+	u16	BA_para_set;
+	u16	BA_timeout_value;
+	u16	BA_starting_seqctrl;
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	u8					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	u16					*fctrl;
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info	*lmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+	struct sta_info		*psta = NULL;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct registry_priv		*pregpriv = &padapter->registrypriv;
+	u16 num_ampdu = 0;
+
+#ifdef CONFIG_80211N_HT
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	psta = rtw_get_stainfo(pstapriv, raddr);
+	if (psta == NULL) {
+		RTW_WARN(FUNC_ADPT_FMT" sta not found, raddr="MAC_FMT"\n",
+	 		FUNC_ADPT_ARG(padapter), MAC_ARG(raddr));
+		goto exit;
+	}
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	/* _rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN); */
+	_rtw_memcpy(pwlanhdr->addr1, raddr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(lmlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	if (category == 3) {
+		switch (action) {
+		case RTW_WLAN_ACTION_ADDBA_REQ:
+			do {
+				pmlmeinfo->dialogToken++;
+			} while (pmlmeinfo->dialogToken == 0);
+			pframe = rtw_set_fixed_ie(pframe, 1, &(pmlmeinfo->dialogToken), &(pattrib->pktlen));
+
+#ifdef CONFIG_80211AX_HE
+			/* Logo sniffer checks that ampdu length must be <= 64 */
+			if (is_supported_he(psta->phl_sta->wmode) && !padapter->registrypriv.wifi_spec)
+				num_ampdu = MAX_BA_BUFFER_SIZE_HE;
+			else
+#endif
+				num_ampdu = MAX_BA_BUFFER_SIZE_HT;
+
+			BA_para_set = (num_ampdu << 6);
+			/* immediate ack */
+			BA_para_set |= (0x2 | ((tid & 0xf) << 2));
+			RTW_INFO("BA_para_set = 0x%x for agg_num=%d\n", BA_para_set, (BA_para_set >> 6));
+
+			RTW_INFO("[issue ADDBA_REQ] to "MAC_FMT" bw:%d tid:%d ampdu_num:%d\n",
+				MAC_ARG(psta->phl_sta->mac_addr), psta->phl_sta->chandef.bw, tid, (BA_para_set >> 6));
+
+#ifdef CONFIG_TX_AMSDU
+			if (padapter->tx_amsdu >= 1 &&
+				!(psta->flags & WLAN_STA_AMSDU_DISABLE)
+			) /* TX AMSDU enabled */
+				BA_para_set |= BIT(0);
+			else /* TX AMSDU disabled */
+#endif
+				BA_para_set &= ~BIT(0);
+
+			BA_para_set = cpu_to_le16(BA_para_set);
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&(BA_para_set)), &(pattrib->pktlen));
+
+			/* BA_timeout_value = 0xffff; */ /* max: 65535 TUs(~ 65 ms) */
+			BA_timeout_value = 5000;/* ~ 5ms */
+			BA_timeout_value = cpu_to_le16(BA_timeout_value);
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&(BA_timeout_value)), &(pattrib->pktlen));
+
+			/* if ((psta = rtw_get_stainfo(pstapriv, pmlmeinfo->network.MacAddress)) != NULL) */
+
+			start_seq = (psta->sta_xmitpriv.txseq_tid[tid & 0x07] & 0xfff) + 1;
+
+			RTW_INFO("BA_starting_seqctrl = %d for TID=%d\n", start_seq, tid & 0x07);
+
+			psta->BA_starting_seqctrl[tid & 0x07] = start_seq;
+
+			BA_starting_seqctrl = start_seq << 4;
+			BA_starting_seqctrl = cpu_to_le16(BA_starting_seqctrl);
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&(BA_starting_seqctrl)), &(pattrib->pktlen));
+
+			/* ToDo: ADDBA extension */
+			break;
+
+		case RTW_WLAN_ACTION_ADDBA_RESP:
+			pframe = rtw_set_fixed_ie(pframe, 1, &(paddba_req->dialog_token), &(pattrib->pktlen));
+			status = cpu_to_le16(status);
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&status), &(pattrib->pktlen));
+
+			BA_para_set = le16_to_cpu(paddba_req->BA_para_set);
+
+			BA_para_set &= ~IEEE80211_ADDBA_PARAM_TID_MASK;
+			BA_para_set |= (tid << 2) & IEEE80211_ADDBA_PARAM_TID_MASK;
+
+			BA_para_set &= ~RTW_IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK;
+			BA_para_set |= (size << 6) & RTW_IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK;
+
+			if (!padapter->registrypriv.wifi_spec) {
+				if (pregpriv->rx_ampdu_amsdu == 0) /* disabled */
+					BA_para_set &= ~BIT(0);
+				else if (pregpriv->rx_ampdu_amsdu == 1) /* enabled */
+					BA_para_set |= BIT(0);
+			}
+
+			if (psta->flags & WLAN_STA_AMSDU_DISABLE)
+				BA_para_set &= ~BIT(0);
+
+			BA_para_set = cpu_to_le16(BA_para_set);
+
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&(BA_para_set)), &(pattrib->pktlen));
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&(paddba_req->BA_timeout_value)), &(pattrib->pktlen));
+			break;
+
+		case RTW_WLAN_ACTION_DELBA:
+			BA_para_set = 0;
+			BA_para_set |= (tid << 12) & IEEE80211_DELBA_PARAM_TID_MASK;
+			BA_para_set |= (initiator << 11) & IEEE80211_DELBA_PARAM_INITIATOR_MASK;
+
+			BA_para_set = cpu_to_le16(BA_para_set);
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&(BA_para_set)), &(pattrib->pktlen));
+			status = cpu_to_le16(status);
+			pframe = rtw_set_fixed_ie(pframe, 2, (unsigned char *)(&(status)), &(pattrib->pktlen));
+			break;
+		default:
+			break;
+		}
+	}
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+#endif /* CONFIG_80211N_HT */
+	return ret;
+}
+
+/**
+ * issue_addba_req - TX ADDBA_REQ
+ * @adapter: the adapter to TX
+ * @ra: receiver address
+ * @tid: tid
+ */
+inline void issue_addba_req(_adapter *adapter, unsigned char *ra, u8 tid)
+{
+	issue_action_ba(adapter, ra, RTW_WLAN_ACTION_ADDBA_REQ
+			, tid
+			, 0 /* unused */
+			, NULL /* unused */
+			, 0 /* unused */
+			, 0 /* unused */
+			, _FALSE
+		       );
+	RTW_INFO(FUNC_ADPT_FMT" ra="MAC_FMT" tid=%u\n"
+		 , FUNC_ADPT_ARG(adapter), MAC_ARG(ra), tid);
+
+}
+
+/**
+ * issue_addba_rsp - TX ADDBA_RESP
+ * @adapter: the adapter to TX
+ * @ra: receiver address
+ * @tid: tid
+ * @status: status code
+ * @size: the announced AMPDU buffer size
+ * @paddba_req: used by ADDBA_RESP
+ */
+inline void issue_addba_rsp(_adapter *adapter, unsigned char *ra, u8 tid,
+			    u16 status, u16 size,
+			    struct ADDBA_request *paddba_req)
+{
+	issue_action_ba(adapter, ra, RTW_WLAN_ACTION_ADDBA_RESP
+			, tid
+			, size
+			, paddba_req
+			, status
+			, 0 /* unused */
+			, _FALSE
+		       );
+	RTW_INFO(FUNC_ADPT_FMT" ra="MAC_FMT" status=%u, tid=%u, size=%u\n"
+		 , FUNC_ADPT_ARG(adapter), MAC_ARG(ra), status, tid, size);
+}
+
+/**
+ * issue_addba_rsp_wait_ack - TX ADDBA_RESP and wait ack
+ * @adapter: the adapter to TX
+ * @ra: receiver address
+ * @tid: tid
+ * @status: status code
+ * @size: the announced AMPDU buffer size
+ * @paddba_req: used by ADDBA_RESP
+ * @try_cnt: the maximal TX count to try
+ * @wait_ms: == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ *           > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ */
+inline u8 issue_addba_rsp_wait_ack(_adapter *adapter, unsigned char *ra, u8 tid,
+				   u16 status, u16 size,
+				   struct ADDBA_request *paddba_req, int try_cnt,
+				   int wait_ms)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+
+	if (adapter_regu_block_tx(adapter))
+		goto exit;
+
+	do {
+		ret = issue_action_ba(adapter, ra, RTW_WLAN_ACTION_ADDBA_RESP
+				      , tid
+				      , size
+				      , paddba_req
+				      , status
+				      , 0 /* unused */
+				      , _TRUE
+				     );
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		/* goto exit; */
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		RTW_INFO(FUNC_ADPT_FMT" ra="MAC_FMT" status:=%u tid=%u size:%u%s, %d/%d in %u ms\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(ra), status, tid, size
+			, ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+
+exit:
+	return ret;
+}
+
+/**
+ * issue_del_ba - TX DELBA
+ * @adapter: the adapter to TX
+ * @ra: receiver address
+ * @tid: tid
+ * @reason: reason code
+ * @initiator: if we are the initiator of AMPDU association. used by DELBA
+ */
+inline void issue_del_ba(_adapter *adapter, unsigned char *ra, u8 tid, u16 reason, u8 initiator)
+{
+	issue_action_ba(adapter, ra, RTW_WLAN_ACTION_DELBA
+			, tid
+			, 0 /* unused */
+			, NULL /* unused */
+			, reason
+			, initiator
+			, _FALSE
+		       );
+	RTW_INFO(FUNC_ADPT_FMT" ra="MAC_FMT" reason=%u, tid=%u, initiator=%u\n"
+		 , FUNC_ADPT_ARG(adapter), MAC_ARG(ra), reason, tid, initiator);
+}
+
+/**
+ * issue_del_ba_ex - TX DELBA with xmit ack options
+ * @adapter: the adapter to TX
+ * @ra: receiver address
+ * @tid: tid
+ * @reason: reason code
+ * @initiator: if we are the initiator of AMPDU association. used by DELBA
+ * @try_cnt: the maximal TX count to try
+ * @wait_ms: == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ *           > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ */
+int issue_del_ba_ex(_adapter *adapter, unsigned char *ra, u8 tid, u16 reason, u8 initiator
+		    , int try_cnt, int wait_ms)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+
+	if (adapter_regu_block_tx(adapter))
+		goto exit;
+
+	do {
+		ret = issue_action_ba(adapter, ra, RTW_WLAN_ACTION_DELBA
+				      , tid
+				      , 0 /* unused */
+				      , NULL /* unused */
+				      , reason
+				      , initiator
+				      , wait_ms > 0 ? _TRUE : _FALSE
+				     );
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		/* goto exit; */
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		RTW_INFO(FUNC_ADPT_FMT" ra="MAC_FMT" reason=%u, tid=%u, initiator=%u%s, %d/%d in %u ms\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(ra), reason, tid, initiator
+			, ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+exit:
+	return ret;
+}
+
+void issue_action_BSSCoexistPacket(_adapter *padapter)
+{
+	_list		*plist, *phead;
+	unsigned char category, action;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char				*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short			*fctrl;
+	struct	wlan_network	*pnetwork = NULL;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+	_queue		*queue	= &(padapter->mlmepriv.scanned_queue);
+	u8 InfoContent[16] = {0};
+	u8 ICS[8][15];
+#ifdef CONFIG_80211N_HT
+	if ((pmlmepriv->num_FortyMHzIntolerant == 0) && (pmlmepriv->num_sta_no_ht == 0))
+		return;
+
+	if (_TRUE == pmlmeinfo->bwmode_updated)
+		return;
+
+	if (alink_regu_block_tx(padapter_link))
+		return;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+
+	category = RTW_WLAN_CATEGORY_PUBLIC;
+	action = ACT_PUBLIC_BSSCOEXIST;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	/* TODO calculate 40Mhz intolerant via ch and ch offset */
+	/* if (pmlmepriv->num_FortyMHzIntolerant > 0) */
+	{
+		u8 iedata = 0;
+
+		iedata |= BIT(2);/* 20 MHz BSS Width Request */
+		pframe = rtw_set_ie(pframe, EID_BSSCoexistence,  1, &iedata, &(pattrib->pktlen));
+	}
+
+	/*  */
+	_rtw_memset(ICS, 0, sizeof(ICS));
+	if (pmlmepriv->num_sta_no_ht > 0) {
+		int i;
+
+		_rtw_spinlock_bh(&(padapter->mlmepriv.scanned_queue.lock));
+
+		phead = get_list_head(queue);
+		plist = get_next(phead);
+
+		while (1) {
+			int len;
+			u8 *p;
+			WLAN_BSSID_EX *pbss_network;
+
+			if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+				break;
+
+			pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+			plist = get_next(plist);
+
+			pbss_network = (WLAN_BSSID_EX *)&pnetwork->network;
+
+			p = rtw_get_ie(pbss_network->IEs + _FIXED_IE_LENGTH_, _HT_CAPABILITY_IE_, &len, pbss_network->IELength - _FIXED_IE_LENGTH_);
+			if ((p == NULL) || (len == 0)) { /* non-HT */
+				if (BSS_EX_OP_BAND(pbss_network) != BAND_ON_24G)
+					continue;
+
+				ICS[0][pbss_network->Configuration.DSConfig] = 1;
+
+				if (ICS[0][0] == 0)
+					ICS[0][0] = 1;
+			}
+
+		}
+
+		_rtw_spinunlock_bh(&(padapter->mlmepriv.scanned_queue.lock));
+
+
+		for (i = 0; i < 8; i++) {
+			if (ICS[i][0] == 1) {
+				int j, k = 0;
+
+				InfoContent[k] = i;
+				/* SET_BSS_INTOLERANT_ELE_REG_CLASS(InfoContent,i); */
+				k++;
+
+				for (j = 1; j <= 14; j++) {
+					if (ICS[i][j] == 1) {
+						if (k < 16) {
+							InfoContent[k] = j; /* channel number */
+							/* SET_BSS_INTOLERANT_ELE_CHANNEL(InfoContent+k, j); */
+							k++;
+						}
+					}
+				}
+
+				pframe = rtw_set_ie(pframe, EID_BSSIntolerantChlReport, k, InfoContent, &(pattrib->pktlen));
+
+			}
+
+		}
+
+
+	}
+
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+#endif /* CONFIG_80211N_HT */
+}
+
+/* Spatial Multiplexing Powersave (SMPS) action frame */
+int _issue_action_SM_PS(_adapter *padapter, unsigned char *raddr, u8 NewMimoPsMode, u8 wait_ack)
+{
+
+	int ret = _FAIL;
+	unsigned char category = RTW_WLAN_CATEGORY_HT;
+	u8 action = RTW_WLAN_ACTION_HT_SM_PS;
+	u8 sm_power_control = 0;
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattrib;
+	unsigned char *pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] TBD may pass stainfo here to get adapter_link */
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info	*pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+
+
+	if (NewMimoPsMode == SM_PS_DISABLE) {
+		sm_power_control = sm_power_control  & ~(BIT(0)); /* SM Power Save Enable = 0 SM Power Save Disable */
+	} else if (NewMimoPsMode == SM_PS_STATIC) {
+		sm_power_control = sm_power_control | BIT(0);    /* SM Power Save Enable = 1 SM Power Save Enable  */
+		sm_power_control = sm_power_control & ~(BIT(1)); /* SM Mode = 0 Static Mode */
+	} else if (NewMimoPsMode == SM_PS_DYNAMIC) {
+		sm_power_control = sm_power_control | BIT(0); /* SM Power Save Enable = 1 SM Power Save Enable  */
+		sm_power_control = sm_power_control | BIT(1); /* SM Mode = 1 Dynamic Mode */
+	} else
+		return ret;
+
+	if (alink_regu_block_tx(padapter_link))
+		return ret;
+
+	RTW_INFO("%s, sm_power_control=%u, NewMimoPsMode=%u\n", __FUNCTION__ , sm_power_control , NewMimoPsMode);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return ret;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, raddr, ETH_ALEN); /* RA */
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN); /* TA */
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN); /* DA = RA */
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	/* category, action */
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(sm_power_control), &(pattrib->pktlen));
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+	if (ret != _SUCCESS)
+		RTW_INFO("%s, ack to\n", __func__);
+
+	return ret;
+}
+
+/*
+ * wait_ms == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ * wait_ms > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ * try_cnt means the maximal TX count to try
+ */
+int issue_action_SM_PS_wait_ack(_adapter *padapter, unsigned char *raddr, u8 NewMimoPsMode, int try_cnt, int wait_ms)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+
+	if (adapter_regu_block_tx(padapter))
+		goto exit;
+
+	do {
+		ret = _issue_action_SM_PS(padapter, raddr, NewMimoPsMode , wait_ms > 0 ? _TRUE : _FALSE);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		if (raddr)
+			RTW_INFO(FUNC_ADPT_FMT" to "MAC_FMT", %s , %d/%d in %u ms\n",
+				 FUNC_ADPT_ARG(padapter), MAC_ARG(raddr),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+		else
+			RTW_INFO(FUNC_ADPT_FMT", %s , %d/%d in %u ms\n",
+				 FUNC_ADPT_ARG(padapter),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+exit:
+
+	return ret;
+}
+
+int issue_action_SM_PS(_adapter *padapter ,  unsigned char *raddr , u8 NewMimoPsMode)
+{
+	RTW_INFO("%s to "MAC_FMT"\n", __func__, MAC_ARG(raddr));
+	return _issue_action_SM_PS(padapter, raddr, NewMimoPsMode , _FALSE);
+}
+
+/* Spectrum Management Action frame - Channel Switch Announcement */
+void issue_action_spct_ch_switch(_adapter *a, struct _ADAPTER_LINK *alink, u8 *ra)
+{
+#ifdef CONFIG_ECSA_PHL
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattrib;
+	unsigned char *pframe;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	unsigned short *fctrl;
+	struct xmit_priv *pxmitpriv = &(a->xmitpriv);
+	struct mlme_ext_priv *pmlmeext = &(a->mlmeextpriv);
+	struct rtw_phl_ecsa_param *ecsa_param = &(a->ecsa_info.phl_ecsa_param);
+	struct rtw_chan_def n_chdef = ecsa_param->new_chan_def;
+	u8 category, action;
+	u8 is_vht = alink->mlmepriv.vhtpriv.vht_option;
+
+	if (alink_regu_block_tx(alink))
+		return;
+
+	RTW_INFO("CSA : Send CSA action frame to "MAC_FMT"\n", MAC_ARG(ra));
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		return;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(a, alink, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, ra, ETH_ALEN); /* RA */
+	_rtw_memcpy(pwlanhdr->addr2, alink->mac_addr, ETH_ALEN); /* TA */
+	_rtw_memcpy(pwlanhdr->addr3, ra, ETH_ALEN); /* DA = RA */
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	/* Category field and Spectrum Management Action field */
+	category = RTW_WLAN_CATEGORY_SPECTRUM_MGMT;
+	action = RTW_WLAN_ACTION_SPCT_CHL_SWITCH;
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	/* Channel Switch Announcement element */
+	pframe = rtw_set_ie_ch_switch(pframe, &(pattrib->pktlen),
+			ecsa_param->mode, n_chdef.chan, ecsa_param->count);
+
+	/* Secondary Channel Offset element */
+	if (n_chdef.offset != CHAN_OFFSET_NO_EXT)
+		pframe = rtw_set_ie_secondary_ch_offset(pframe, &(pattrib->pktlen), n_chdef.offset);
+
+	/* Mesh Channel Switch Parameters element, doesn't support */
+
+	/* Wide Bandwidth Channel Switch element */
+	if (is_vht && (n_chdef.bw >= CHANNEL_WIDTH_40 && n_chdef.bw <= CHANNEL_WIDTH_80_80)) {
+		u8 ch_width, seg_0, seg_1;
+
+		switch (n_chdef.bw) {
+		case CHANNEL_WIDTH_40:
+			ch_width = 0;
+			break;
+		case CHANNEL_WIDTH_80:
+			ch_width = 1;
+			break;
+		default:
+			ch_width = 1;
+			break;
+		}
+
+		seg_0 = rtw_phl_get_center_ch(&n_chdef);
+		seg_1 = 0;
+		rtw_set_ie_wide_bw_ch_switch(pframe, &(pattrib->pktlen), ch_width, seg_0, seg_1);
+	}
+
+	/* New Transmit Power Envelope element, doesn't support */
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(a, pmgntframe);
+#endif
+}
+
+#ifdef PRIVATE_R
+
+int issue_action_find_remote(_adapter *padapter)
+{
+	int ret = _FAIL;
+	u8		category = RTW_WLAN_CATEGORY_P2P;
+	u16	start_seq;
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	u8					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	u16					*fctrl;
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	u8		bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8		ROKU_OUI[] = {0x00, 0x0d, 0x4b};
+	u8		vendor_spec_subtype = 0x7f;
+	u8		p2p_action_subtype = 0x01;
+	u8		p2p_action_dialog = 0x03;
+	u8		reserved_1 = 0xaa;
+	u8		reserved_2[] = {0xab, 0x01, 0x01};
+	u8		len = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (alink_regu_block_tx(padapter_link))
+		goto exit;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, padapter_link->mac_addr, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &category, &(pattrib->pktlen));
+
+	pframe = rtw_set_fixed_ie(pframe, 3, ROKU_OUI, &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &vendor_spec_subtype, &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &p2p_action_subtype, &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &p2p_action_dialog, &(pattrib->pktlen));
+
+	/* set remote len and MAC address */
+	pframe = rtw_set_fixed_ie(pframe, 1, &reserved_1, &(pattrib->pktlen));
+	len = pwdinfo->num_of_remote * ETH_ALEN;
+	pframe = rtw_set_fixed_ie(pframe, 1, &len, &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, len, pwdinfo->remote_mac_address, &(pattrib->pktlen));
+
+	pframe = rtw_set_fixed_ie(pframe, 3, reserved_2, &(pattrib->pktlen));
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	
+	dump_mgntframe(padapter, pmgntframe);
+	
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+#endif
+
+/**
+ * _send_delba_sta_tid - Cancel the AMPDU association for the specific @sta, @tid
+ * @adapter: the adapter to which @sta belongs
+ * @initiator: if we are the initiator of AMPDU association
+ * @sta: the sta to be checked
+ * @tid: the tid to be checked
+ * @force: cancel and send DELBA even when no AMPDU association is setup
+ * @wait_ack: send delba with xmit ack (valid when initiator == 0)
+ *
+ * Returns:
+ * _FAIL if sta is NULL
+ * when initiator is 1, always _SUCCESS
+ * when initiator is 0, _SUCCESS if DELBA is acked
+ */
+static unsigned int _send_delba_sta_tid(_adapter *adapter, u8 initiator, struct sta_info *sta, u8 tid
+					, u8 force, int wait_ack)
+{
+	int ret = _SUCCESS;
+
+	if (sta == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	if (initiator == 0) {
+		/* recipient */
+		if (force || sta->recvreorder_ctrl[tid].enable == _TRUE) {
+			u16 ampdu_size_bak = sta->recvreorder_ctrl[tid].ampdu_size;
+
+			sta->recvreorder_ctrl[tid].enable = _FALSE;
+			sta->recvreorder_ctrl[tid].ampdu_size = RX_AMPDU_SIZE_INVALID;
+
+			if (rtw_del_rx_ampdu_test_trigger_no_tx_fail())
+				ret = _FAIL;
+			else if (wait_ack)
+				ret = issue_del_ba_ex(adapter, sta->phl_sta->mac_addr, tid, 37, initiator, 3, 1);
+			else
+				issue_del_ba(adapter, sta->phl_sta->mac_addr, tid, 37, initiator);
+
+			if (ret == _FAIL && sta->recvreorder_ctrl[tid].enable == _FALSE)
+				sta->recvreorder_ctrl[tid].ampdu_size = ampdu_size_bak;
+		}
+	} else if (initiator == 1) {
+		/* originator */
+#ifdef CONFIG_80211N_HT
+		if (force || sta->ampdu_priv.agg_enable_bitmap & BIT(tid)) {
+			sta->ampdu_priv.agg_enable_bitmap &= ~BIT(tid);
+			sta->ampdu_priv.candidate_tid_bitmap &= ~BIT(tid);
+			issue_del_ba(adapter, sta->phl_sta->mac_addr, tid, 37, initiator);
+		}
+#endif
+	}
+
+exit:
+	return ret;
+}
+
+inline unsigned int send_delba_sta_tid(_adapter *adapter, u8 initiator, struct sta_info *sta, u8 tid
+				       , u8 force)
+{
+	return _send_delba_sta_tid(adapter, initiator, sta, tid, force, 0);
+}
+
+inline unsigned int send_delba_sta_tid_wait_ack(_adapter *adapter, u8 initiator, struct sta_info *sta, u8 tid
+		, u8 force)
+{
+	return _send_delba_sta_tid(adapter, initiator, sta, tid, force, 1);
+}
+
+unsigned int send_delba(_adapter *padapter, u8 initiator, u8 *addr)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta = NULL;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u16 tid;
+
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
+		if (!(pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
+			return _SUCCESS;
+
+	psta = rtw_get_stainfo(pstapriv, addr);
+	if (psta == NULL)
+		return _SUCCESS;
+
+#if 0
+	RTW_INFO("%s:%s\n", __func__, (initiator == 0) ? "RX_DIR" : "TX_DIR");
+	if (initiator == 1) /* originator */
+		RTW_INFO("tx agg_enable_bitmap(0x%08x)\n", psta->ampdu_priv.agg_enable_bitmap);
+#endif
+
+	for (tid = 0; tid < TID_NUM; tid++)
+		send_delba_sta_tid(padapter, initiator, psta, tid, 0);
+
+	return _SUCCESS;
+}
+
+unsigned int send_beacon(_adapter *padapter)
+{
+#ifdef RTW_PHL_BCN
+	/* bypass TX BCN queue because op ch is switching */
+	if (check_fwstate(&padapter->mlmepriv, WIFI_OP_CH_SWITCHING | WIFI_REGU_FORBID))
+		return _SUCCESS;
+
+	issue_beacon(padapter, 0);
+
+	/* maybe need some mechanism to check bcn ready here */
+
+	return _SUCCESS;
+#else
+#if defined(CONFIG_PCI_HCI) && !defined(CONFIG_PCI_BCN_POLLING)
+
+	/* bypass TX BCN queue because op ch is switching */
+	if (check_fwstate(&padapter->mlmepriv, WIFI_OP_CH_SWITCHING | WIFI_REGU_FORBID))
+		return _SUCCESS;
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	rtw_hal_set_hwreg(padapter, HW_VAR_BCN_VALID, NULL);
+	rtw_hal_set_hwreg(padapter, HW_VAR_DL_BCN_SEL, NULL);
+
+
+	issue_beacon(padapter, 0);
+
+	return _SUCCESS;
+#endif
+
+/* CONFIG_PCI_BCN_POLLING is for pci interface beacon polling mode */
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)|| defined(CONFIG_PCI_BCN_POLLING) 
+	u8 bxmitok = _FALSE;
+	int issue = 0;
+	int poll = 0;
+	systime start = rtw_get_current_time();
+
+	/* bypass TX BCN queue because op ch is switching */
+	if (check_fwstate(&padapter->mlmepriv, WIFI_OP_CH_SWITCHING | WIFI_REGU_FORBID))
+		return _SUCCESS;
+
+	rtw_hal_set_hwreg(padapter, HW_VAR_BCN_VALID, NULL);
+	rtw_hal_set_hwreg(padapter, HW_VAR_DL_BCN_SEL, NULL);
+	do {
+		#if defined(CONFIG_PCI_BCN_POLLING) 
+		issue_beacon(padapter, 0);
+		#else
+		issue_beacon(padapter, 100);
+		#endif
+		issue++;
+		do {
+			#if defined(CONFIG_PCI_BCN_POLLING) 
+			rtw_msleep_os(1);
+			#else
+			rtw_yield_os();
+			#endif
+			rtw_hal_get_hwreg(padapter, HW_VAR_BCN_VALID, (u8 *)(&bxmitok));
+			poll++;
+		} while ((poll % 10) != 0 && _FALSE == bxmitok && !RTW_CANNOT_RUN(adapter_to_dvobj(padapter)));
+		#if defined(CONFIG_PCI_BCN_POLLING) 
+		rtw_hal_unmap_beacon_icf(padapter);
+		#endif
+	} while (bxmitok == _FALSE && (issue < 100) && !RTW_CANNOT_RUN(adapter_to_dvobj(padapter)));
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+		return _FAIL;
+
+
+	if (_FALSE == bxmitok) {
+		RTW_INFO("%s fail! %u ms\n", __FUNCTION__, rtw_get_passing_time_ms(start));
+		return _FAIL;
+	} else {
+		u32 passing_time = rtw_get_passing_time_ms(start);
+
+		if (passing_time > 100 || issue > 3)
+			RTW_INFO("%s success, issue:%d, poll:%d, %u ms\n", __FUNCTION__, issue, poll, rtw_get_passing_time_ms(start));
+		else if (0)
+			RTW_INFO("%s success, issue:%d, poll:%d, %u ms\n", __FUNCTION__, issue, poll, rtw_get_passing_time_ms(start));
+
+		return _SUCCESS;
+	}
+
+#endif /*defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)*/
+#endif /* RTW_PHL_BCN */
+}
+
+/****************************************************************************
+
+Following are some utitity fuctions for WiFi MLME
+
+*****************************************************************************/
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+BOOLEAN Is6GHzPreferScanChannel(u8 channel)
+{
+	if ((channel > 0) && (channel <= 233))
+		if (((channel - 5) % 16) == 0)
+			return _TRUE;
+
+	return _FALSE;
+}
+#endif
+
+BOOLEAN IsLegal5GChannel(
+	_adapter *adapter,
+	u8			channel)
+{
+
+	int i = 0;
+	u8 Channel_5G[45] = {36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58,
+		60, 62, 64, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122,
+		124, 126, 128, 130, 132, 134, 136, 138, 140, 149, 151, 153, 155, 157, 159,
+			     161, 163, 165
+			    };
+	for (i = 0; i < sizeof(Channel_5G); i++)
+		if (channel == Channel_5G[i])
+			return _TRUE;
+	return _FALSE;
+}
+
+/* collect bss info from Beacon and Probe request/response frames. */
+u8 collect_bss_info(_adapter *padapter, union recv_frame *precv_frame, WLAN_BSSID_EX *bssid)
+{
+	int	i;
+	sint len;
+	u8	*p;
+	u8	rf_path;
+	u16	val16, subtype;
+	u8	*pframe = precv_frame->u.hdr.rx_data;
+	u32	packet_len = precv_frame->u.hdr.len;
+	u8 ie_offset;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+#ifdef CONFIG_80211BE_EHT
+	u8 eid_eht_ml = EID_EXT_MULTI_LINK;
+#endif
+
+	len = packet_len - sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	if (len > MAX_IE_SZ) {
+		/*RTW_INFO("IE too long for survey event\n");*/
+		return _FAIL;
+	}
+
+	_rtw_memset(bssid, 0, sizeof(WLAN_BSSID_EX));
+
+	subtype = get_frame_sub_type(pframe);
+
+	if (subtype == WIFI_BEACON) {
+		bssid->Reserved[0] = BSS_TYPE_BCN;
+		ie_offset = _BEACON_IE_OFFSET_;
+	} else {
+		/* FIXME : more type */
+		if (subtype == WIFI_PROBERSP) {
+			ie_offset = _PROBERSP_IE_OFFSET_;
+			bssid->Reserved[0] = BSS_TYPE_PROB_RSP;
+		} else if (subtype == WIFI_PROBEREQ) {
+			ie_offset = _PROBEREQ_IE_OFFSET_;
+			bssid->Reserved[0] = BSS_TYPE_PROB_REQ;
+		} else {
+			bssid->Reserved[0] = BSS_TYPE_UNDEF;
+			ie_offset = _FIXED_IE_LENGTH_;
+		}
+	}
+
+	bssid->Length = sizeof(WLAN_BSSID_EX) - MAX_IE_SZ + len;
+
+	/* below is to copy the information element */
+	bssid->IELength = len;
+	_rtw_memcpy(bssid->IEs, (pframe + sizeof(struct rtw_ieee80211_hdr_3addr)), bssid->IELength);
+
+	/*RTW_WKARD_CORE_RSSI_V1 - GEORGIA MUST REFINE*/
+	/* bssid->PhyInfo.rssi = precv_frame->u.hdr.attrib.SignalStrength; */ /* 0-100 index. */
+	bssid->PhyInfo.SignalQuality = precv_frame->u.hdr.attrib.phy_info.signal_quality;/* in percentage */
+	bssid->PhyInfo.SignalStrength = precv_frame->u.hdr.attrib.phy_info.signal_strength;/* in percentage */
+	bssid->PhyInfo.rssi = precv_frame->u.hdr.attrib.phy_info.recv_signal_power;/*dbm*/
+	bssid->PhyInfo.isValid = precv_frame->u.hdr.attrib.phy_info.is_valid;
+	/* bss info receving the first time and invalid phyinfo */
+	if (!bssid->PhyInfo.isValid) {
+		bssid->PhyInfo.SignalStrength = 30;
+		bssid->PhyInfo.rssi = rtw_phl_rssi_to_dbm(bssid->PhyInfo.SignalStrength);
+	}
+
+	if (precv_frame->u.hdr.attrib.data_rate >= DESC_RATE11M) {
+		bssid->PhyInfo.is_cck_rate = 0;
+		for (rf_path = 0; rf_path < GET_HAL_RFPATH_NUM(dvobj); rf_path++)
+			bssid->PhyInfo.rx_snr[rf_path] =
+				precv_frame->u.hdr.attrib.phy_info.rx_snr[rf_path];
+	} else
+		bssid->PhyInfo.is_cck_rate = 1;
+
+
+	/* checking SSID */
+	p = rtw_get_ie(bssid->IEs + ie_offset, _SSID_IE_, &len, bssid->IELength - ie_offset);
+	if (p == NULL) {
+		RTW_INFO("marc: cannot find SSID for survey event\n");
+		return _FAIL;
+	}
+
+	if (*(p + 1)) {
+		if (len > NDIS_802_11_LENGTH_SSID) {
+			RTW_INFO("%s()-%d: IE too long (%d) for survey event\n", __FUNCTION__, __LINE__, len);
+			return _FAIL;
+		}
+		_rtw_memcpy(bssid->Ssid.Ssid, (p + 2), *(p + 1));
+		bssid->Ssid.SsidLength = *(p + 1);
+	} else
+		bssid->Ssid.SsidLength = 0;
+
+	_rtw_memset(bssid->SupportedRates, 0, NDIS_802_11_LENGTH_RATES_EX);
+
+	/* checking rate info... */
+	i = 0;
+	p = rtw_get_ie(bssid->IEs + ie_offset, _SUPPORTEDRATES_IE_, &len, bssid->IELength - ie_offset);
+	if (p != NULL) {
+		if (len > NDIS_802_11_LENGTH_RATES_EX) {
+			RTW_INFO("%s()-%d: IE too long (%d) for survey event\n", __FUNCTION__, __LINE__, len);
+			return _FAIL;
+		}
+#ifdef CONFIG_COLLECT_BSS_VALIDATE_SUPP_RATE
+		if (rtw_validate_value(_SUPPORTEDRATES_IE_, p+2, len) == _FALSE) {
+			rtw_absorb_ssid_ifneed(padapter, bssid, pframe);
+			RTW_DBG_DUMP("Invalidated Support Rate IE --", p, len+2);
+			return _FAIL;
+		}
+#endif
+		_rtw_memcpy(bssid->SupportedRates, (p + 2), len);
+		i = len;
+	}
+
+	p = rtw_get_ie(bssid->IEs + ie_offset, _EXT_SUPPORTEDRATES_IE_, &len, bssid->IELength - ie_offset);
+	if (p != NULL) {
+		if (len > (NDIS_802_11_LENGTH_RATES_EX - i)) {
+			RTW_INFO("%s()-%d: IE too long (%d) for survey event\n", __FUNCTION__, __LINE__, len);
+			return _FAIL;
+		}
+#ifdef CONFIG_COLLECT_BSS_VALIDATE_SUPP_RATE
+		if ((padapter->registrypriv.wifi_spec == 0) &&
+			(rtw_validate_value(_EXT_SUPPORTEDRATES_IE_, p+2, len) == _FALSE)) {
+			rtw_absorb_ssid_ifneed(padapter, bssid, pframe);
+			RTW_DBG_DUMP("Invalidated EXT Support Rate IE --", p, len+2);
+			return _FAIL;
+		}
+#endif
+		_rtw_memcpy(bssid->SupportedRates + i, (p + 2), len);
+	}
+
+#ifdef CONFIG_P2P
+	if (subtype == WIFI_PROBEREQ) {
+		u8 *p2p_ie;
+		u32	p2p_ielen;
+		/* Set Listion Channel */
+		p2p_ie = rtw_get_p2p_ie(bssid->IEs, bssid->IELength, NULL, &p2p_ielen);
+		if (p2p_ie) {
+			u32	attr_contentlen = 0;
+			u8 listen_ch[5] = { 0x00 };
+
+			attr_contentlen = sizeof(listen_ch);
+			rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_LISTEN_CH, listen_ch, &attr_contentlen);
+			bssid->Configuration.DSConfig = listen_ch[4];
+		} else {
+			/* use current channel */
+			bssid->Configuration.DSConfig = padapter_link->mlmeextpriv.chandef.chan;
+			RTW_INFO("%s()-%d: Cannot get p2p_ie. set DSconfig to op_ch(%d)\n", __FUNCTION__, __LINE__, bssid->Configuration.DSConfig);
+		}
+
+		/* FIXME */
+		bssid->InfrastructureMode = Ndis802_11Infrastructure;
+		_rtw_memcpy(bssid->MacAddress, get_addr2_ptr(pframe), ETH_ALEN);
+		bssid->Privacy = 1;
+		RTW_INFO("%s: returned on line %d ?\n", __func__, __LINE__);
+		return _SUCCESS;
+	}
+#endif /* CONFIG_P2P */
+
+	if (bssid->IELength < 12)
+		return _FAIL;
+
+	/* Checking 6GHz HE capability before checking DSConfig, because 6GHz
+	 * AP could contain DSConfig and 6GHz HE capability.
+	 */
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (IS_ENABLED(CONFIG_IEEE80211_BAND_6GHZ)) {
+		u8 he_op_eid_ext = WLAN_EID_EXTENSION_HE_OPERATION;
+		u32	ielen;
+		u8 *he_op_ie;
+
+		he_op_ie = rtw_get_ie_ex(bssid->IEs + ie_offset, bssid->IELength - ie_offset, WLAN_EID_EXTENSION, &he_op_eid_ext, 1, NULL, &ielen);
+		if(he_op_ie && GET_HE_OP_PARA_6GHZ_OP_INFO_PRESENT(he_op_ie + 3)) {
+			u32 offset = HE_OPER_PARAMS_LEN + HE_OPER_BSS_COLOR_INFO_LEN + HE_OPER_BASIC_MCS_LEN;
+
+			if(GET_HE_OP_PARA_VHT_OP_INFO_PRESENT(he_op_ie + 3))
+				offset += HE_OPER_VHT_OPER_INFO_LEN;
+			if(GET_HE_OP_PARA_CO_HOSTED_BSS(he_op_ie + 3))
+				offset += HE_OPER_MAX_COHOST_BSSID_LEN;
+			bssid->Configuration.Band = BAND_ON_6G;
+			bssid->Configuration.DSConfig = GET_HE_OP_INFO_PRIMARY_CHAN(he_op_ie + 3 + offset);
+			bssid->Configuration.Length = 0;
+
+			goto dsconfig_done;
+		}
+	}
+#endif
+
+	/* Checking for DSConfig */
+	p = rtw_get_ie(bssid->IEs + ie_offset, _DSSET_IE_, &len, bssid->IELength - ie_offset);
+
+	bssid->Configuration.Band = 0;
+	bssid->Configuration.DSConfig = 0;
+	bssid->Configuration.Length = 0;
+
+	if (p) {
+		bssid->Configuration.Band = (*(p + 2) > 14) ? BAND_ON_5G : BAND_ON_24G;
+		bssid->Configuration.DSConfig = *(p + 2);
+	} else {
+		/* In 5G, some ap do not have DSSET IE */
+		/* checking HT info for channel */
+		p = rtw_get_ie(bssid->IEs + ie_offset, _HT_ADD_INFO_IE_, &len, bssid->IELength - ie_offset);
+		if (p) {
+			struct HT_info_element *HT_info = (struct HT_info_element *)(p + 2);
+			bssid->Configuration.Band = (HT_info->primary_channel > 14) ? BAND_ON_5G : BAND_ON_24G;
+			bssid->Configuration.DSConfig = HT_info->primary_channel;
+		} else {
+			/* use current band and channel */
+			bssid->Configuration.Band = rtw_get_oper_band(padapter, padapter_link);
+			bssid->Configuration.DSConfig = rtw_get_oper_ch(padapter, padapter_link);
+		}
+	}
+dsconfig_done:
+
+	_rtw_memcpy(&bssid->Configuration.BeaconPeriod, rtw_get_beacon_interval_from_ie(bssid->IEs), 2);
+	bssid->Configuration.BeaconPeriod = le32_to_cpu(bssid->Configuration.BeaconPeriod);
+
+	val16 = rtw_get_capability((WLAN_BSSID_EX *)bssid);
+
+	if ((val16 & 0x03) == cap_ESS) {
+		bssid->InfrastructureMode = Ndis802_11Infrastructure;
+		_rtw_memcpy(bssid->MacAddress, get_addr2_ptr(pframe), ETH_ALEN);
+	} else if ((val16 & 0x03) == cap_IBSS){
+		bssid->InfrastructureMode = Ndis802_11IBSS;
+		_rtw_memcpy(bssid->MacAddress, GetAddr3Ptr(pframe), ETH_ALEN);
+	} else if ((val16 & 0x03) == 0x00){
+		u8 *mesh_id_ie, *mesh_conf_ie;
+		sint mesh_id_ie_len, mesh_conf_ie_len;
+
+		mesh_id_ie = rtw_get_ie(bssid->IEs + ie_offset, WLAN_EID_MESH_ID, &mesh_id_ie_len, bssid->IELength - ie_offset);
+		mesh_conf_ie = rtw_get_ie(bssid->IEs + ie_offset, WLAN_EID_MESH_CONFIG, &mesh_conf_ie_len, bssid->IELength - ie_offset);
+		if (mesh_id_ie || mesh_conf_ie) {
+			if (!mesh_id_ie) {
+				RTW_INFO("cannot find Mesh ID for survey event\n");
+				return _FAIL;
+			}
+			if (mesh_id_ie_len) {
+				if (mesh_id_ie_len > NDIS_802_11_LENGTH_SSID) {
+					RTW_INFO("Mesh ID too long (%d) for survey event\n", mesh_id_ie_len);
+					return _FAIL;
+				}
+				_rtw_memcpy(bssid->mesh_id.Ssid, (mesh_id_ie + 2), mesh_id_ie_len);
+				bssid->mesh_id.SsidLength = mesh_id_ie_len;
+			} else
+				bssid->mesh_id.SsidLength = 0;
+
+			if (!mesh_conf_ie) {
+				RTW_INFO("cannot find Mesh config for survey event\n");
+				return _FAIL;
+			}
+			if (mesh_conf_ie_len != 7) {
+				RTW_INFO("invalid Mesh conf IE len (%d) for survey event\n", mesh_conf_ie_len);
+				return _FAIL;
+			}
+
+			bssid->InfrastructureMode = Ndis802_11_mesh;
+			_rtw_memcpy(bssid->MacAddress, GetAddr3Ptr(pframe), ETH_ALEN);
+		} else {
+			/* default cases */
+			bssid->InfrastructureMode = Ndis802_11IBSS;
+			_rtw_memcpy(bssid->MacAddress, GetAddr3Ptr(pframe), ETH_ALEN);
+		}
+	}
+
+	if (val16 & BIT(4))
+		bssid->Privacy = 1;
+	else
+		bssid->Privacy = 0;
+
+	bssid->Configuration.ATIMWindow = 0;
+
+	/* 20/40 BSS Coexistence check */
+	if ((pregistrypriv->wifi_spec == 1) && (_FALSE == pmlmeinfo->bwmode_updated)) {
+		struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+#ifdef CONFIG_80211N_HT
+		p = rtw_get_ie(bssid->IEs + ie_offset, _HT_CAPABILITY_IE_, &len, bssid->IELength - ie_offset);
+		if (p && len > 0) {
+			struct HT_caps_element	*pHT_caps;
+			pHT_caps = (struct HT_caps_element *)(p + 2);
+
+			if (pHT_caps->u.HT_cap_element.HT_caps_info & BIT(14))
+				pmlmepriv->num_FortyMHzIntolerant++;
+		} else
+			pmlmepriv->num_sta_no_ht++;
+#endif /* CONFIG_80211N_HT */
+
+	}
+
+#if defined(DBG_RX_SIGNAL_DISPLAY_SSID_MONITORED) & 1
+	if (strcmp(bssid->Ssid.Ssid, DBG_RX_SIGNAL_DISPLAY_SSID_MONITORED) == 0) {
+		RTW_INFO("Receiving %s("MAC_FMT", DSConfig:%u) from ch%u with ss:%3u, sq:%3u, RawRSSI:%d\n"
+			, bssid->Ssid.Ssid, MAC_ARG(bssid->MacAddress), bssid->Configuration.DSConfig
+			, rtw_get_oper_ch(padapter, padapter_link)
+			, bssid->PhyInfo.SignalStrength, bssid->PhyInfo.SignalQuality, bssid->PhyInfo.rssi
+			);
+	}
+#endif
+
+	/* mark bss info receving from nearby channel as SignalQuality 101 */
+	if (bssid->Configuration.DSConfig != rtw_get_oper_ch(padapter, padapter_link))
+		bssid->PhyInfo.SignalQuality = 101;
+
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	p = rtw_get_ie(bssid->IEs + ie_offset, _EID_RRM_EN_CAP_IE_, &len, bssid->IELength - ie_offset);
+	if (p)
+		_rtw_memcpy(bssid->PhyInfo.rm_en_cap, (p + 2), MIN(*(p + 1),
+			    sizeof(bssid->PhyInfo.rm_en_cap)));
+
+	/* save freerun counter */
+	bssid->PhyInfo.free_cnt = precv_frame->u.hdr.attrib.free_cnt;
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+	p = rtw_get_ie_ex(bssid->IEs + ie_offset, bssid->IELength - ie_offset, WLAN_EID_EXTENSION,
+			&eid_eht_ml, 1, NULL, &len);
+	if (p != NULL && len >= EHT_ML_BASIC_MIN_LEN) {
+		bssid->is_mld = _TRUE;
+		if (GET_ML_ELE_ML_CTRL_LINK_ID_INFO_PRESENT(p + 3))
+			bssid->link_id = LE_BITS_TO_1BYTE(p + EHT_ML_BASIC_MIN_LEN, 0, 4);
+	}
+	else
+#endif
+	{
+		bssid->is_mld = _FALSE;
+	}
+
+	return _SUCCESS;
+}
+
+void start_create_ibss(_adapter *padapter)
+{
+	unsigned short	caps;
+	u8	val8;
+	u8	join_type;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*pnetwork = (WLAN_BSSID_EX *)(&(pmlmeinfo->network));
+	pmlmeext->chandef.chan = (u8)pnetwork->Configuration.DSConfig;
+	pmlmeext->chandef.band = (u8)pnetwork->Configuration.Band;
+	pmlmeinfo->bcn_interval = get_beacon_interval(pnetwork);
+
+	/* update wireless mode */
+	update_wireless_mode(padapter, padapter_link);
+
+	/* udpate capability */
+	caps = rtw_get_capability((WLAN_BSSID_EX *)pnetwork);
+	update_capinfo(padapter, padapter_link, caps);
+
+	if (caps & cap_IBSS) { /* adhoc master */
+		/* set_opmode_cmd(padapter, adhoc); */ /* removed */
+
+		rtw_hal_set_hwreg(padapter, HW_VAR_SEC_CFG, NULL);
+
+		/* switch channel */
+		set_bch_bwmode(padapter, padapter_link,
+			       pmlmeext->chandef.band,
+			       pmlmeext->chandef.chan,
+			       CHAN_OFFSET_NO_EXT,
+			       CHANNEL_WIDTH_20,
+			       RFK_TYPE_CONNECT);
+
+		beacon_timing_control(padapter);
+
+		/* issue beacon */
+		if (send_beacon(padapter) == _FAIL) {
+
+			report_join_res(padapter, -1, WLAN_STATUS_UNSPECIFIED_FAILURE);
+			padapter->mlmeextpriv.mlmext_info.state = WIFI_FW_NULL_STATE;
+		} else {
+			rtw_hal_set_hwreg(padapter, HW_VAR_BSSID, padapter->registrypriv.dev_network.MacAddress);
+			join_type = 0;
+			rtw_hal_set_hwreg(padapter, HW_VAR_MLME_JOIN, (u8 *)(&join_type));
+
+			report_join_res(padapter, 1, WLAN_STATUS_SUCCESS);
+			padapter->mlmeextpriv.mlmext_info.state |= WIFI_FW_ASSOC_SUCCESS;
+			rtw_indicate_connect(padapter);
+		}
+	} else {
+		RTW_INFO("start_create_ibss, invalid cap:%x\n", caps);
+		return;
+	}
+}
+
+void start_clnt_join(_adapter *padapter)
+{
+	unsigned short	caps;
+	int beacon_timeout;
+	u8 ASIX_ID[] = {0x00, 0x0E, 0xC6};
+	/* Client join with the primary link */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*pnetwork = (WLAN_BSSID_EX *)(&(pmlmeinfo->network));
+	u8 lidx;
+#if defined(CONFIG_IOCTL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	u8 ht_option = 0;
+#endif
+
+
+	/* update wireless mode and capability of the primary adapter_link*/
+	update_wireless_mode(padapter, padapter_link);
+	caps = rtw_get_capability(pnetwork);
+	update_capinfo(padapter, padapter_link, caps);
+	if (pnetwork->is_mld) {
+		for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+			padapter_link = GET_LINK(padapter, lidx);
+			if (!padapter_link->mlmepriv.to_join || lidx == RTW_RLINK_PRIMARY)
+				continue;
+			update_wireless_mode(padapter, padapter_link);
+			caps = rtw_get_capability(&padapter_link->mlmeextpriv.mlmext_info.network);
+			update_capinfo(padapter, padapter_link, caps);
+		}
+	}
+	/* check if sta is ASIX peer and fix IOT issue if it is. */
+	/*GEORGIA_TODO_FIXIT_HAL_DEP*/
+	if (_rtw_memcmp(get_my_bssid(&pmlmeinfo->network) , ASIX_ID , 3)) {
+		u8 iot_flag = _TRUE;
+		rtw_hal_set_hwreg(padapter, HW_VAR_ASIX_IOT, (u8 *)(&iot_flag));
+	}
+
+#if defined(CONFIG_IOCTL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+#ifdef CONFIG_80211N_HT
+	ht_option = padapter_link->mlmepriv.htpriv.ht_option;
+#endif
+
+	rtw_cfg80211_ch_switch_notify(padapter, GET_PRIMARY_LINK(padapter),
+				      &pmlmeext->chandef, ht_option, _TRUE);
+#endif
+
+	if (caps & cap_ESS) {
+		rtw_hal_set_hwreg(padapter, HW_VAR_SEC_CFG, NULL);
+
+#ifdef CONFIG_DEAUTH_BEFORE_CONNECT
+		/* Because of AP's not receiving deauth before */
+		/* AP may: 1)not response auth or 2)deauth us after link is complete */
+		/* issue deauth before issuing auth to deal with the situation */
+
+		/*	Commented by Albert 2012/07/21 */
+		/*	For the Win8 P2P connection, it will be hard to have a successful connection if this Wi-Fi doesn't connect to it. */
+		{
+#ifdef CONFIG_P2P
+			_queue *queue = &(padapter->mlmepriv.scanned_queue);
+			_list	*head = get_list_head(queue);
+			_list *pos = get_next(head);
+			struct wlan_network *scanned = NULL;
+			u8 ie_offset = 0;
+			bool has_p2p_ie = _FALSE;
+
+			_rtw_spinlock_bh(&(padapter->mlmepriv.scanned_queue.lock));
+
+			for (pos = get_next(head); !rtw_end_of_queue_search(head, pos); pos = get_next(pos)) {
+
+				scanned = LIST_CONTAINOR(pos, struct wlan_network, list);
+
+				if (_rtw_memcmp(&(scanned->network.Ssid), &(pnetwork->Ssid), sizeof(NDIS_802_11_SSID)) == _TRUE
+				    && _rtw_memcmp(scanned->network.MacAddress, pnetwork->MacAddress, sizeof(NDIS_802_11_MAC_ADDRESS)) == _TRUE
+				   ) {
+					ie_offset = (scanned->network.Reserved[0] == BSS_TYPE_PROB_REQ ? 0 : 12);
+					if (rtw_get_p2p_ie(scanned->network.IEs + ie_offset, scanned->network.IELength - ie_offset, NULL, NULL))
+						has_p2p_ie = _TRUE;
+					break;
+				}
+			}
+
+			_rtw_spinunlock_bh(&(padapter->mlmepriv.scanned_queue.lock));
+
+			if (scanned == NULL || rtw_end_of_queue_search(head, pos) || has_p2p_ie == _FALSE)
+#endif /* CONFIG_P2P */
+				/* To avoid connecting to AP fail during resume process, change retry count from 5 to 1 */
+				issue_deauth_ex(padapter, pnetwork->MacAddress, WLAN_REASON_DEAUTH_LEAVING, 1, 100);
+		}
+#endif /* CONFIG_DEAUTH_BEFORE_CONNECT */
+
+		/* here wait for receiving the beacon to start auth */
+		/* and enable a timer */
+		beacon_timeout = decide_wait_for_beacon_timeout(pmlmeinfo->bcn_interval);
+		set_link_timer(&padapter->mlmeextpriv, beacon_timeout);
+		/*_set_timer(&padapter->mlmepriv.assoc_timer,
+			(REAUTH_TO * REAUTH_LIMIT) + (REASSOC_TO * REASSOC_LIMIT) + beacon_timeout);*/
+		set_assoc_timer(&padapter->mlmepriv,
+			(REAUTH_TO * REAUTH_LIMIT) + (REASSOC_TO * REASSOC_LIMIT) + beacon_timeout);
+
+#ifdef CONFIG_RTW_80211R
+		if (rtw_ft_roam(padapter)) {
+			rtw_ft_start_clnt_join(padapter);
+		} else
+#endif
+		{
+			rtw_sta_linking_test_set_start();
+			padapter->mlmeextpriv.mlmext_info.state = WIFI_FW_AUTH_NULL | WIFI_FW_STATION_STATE;
+		}
+
+#ifdef CONFIG_DRV_FAKE_AP
+		_set_timer(&adapter_to_dvobj(padapter)->fakeap.bcn_timer, 5);
+#endif /* CONFIG_DRV_FAKE_AP */
+	} else if (caps & cap_IBSS) { /* adhoc client */
+		rtw_hal_set_hwreg(padapter, HW_VAR_SEC_CFG, NULL);
+
+		beacon_timing_control(padapter);
+
+		padapter->mlmeextpriv.mlmext_info.state = WIFI_FW_ADHOC_STATE;
+
+		report_join_res(padapter, 1, WLAN_STATUS_SUCCESS);
+	} else {
+		/* RTW_INFO("marc: invalid cap:%x\n", caps); */
+		return;
+	}
+
+}
+
+void start_clnt_auth(_adapter *padapter)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	cancel_link_timer(pmlmeext); /*_cancel_timer_ex(&pmlmeext->link_timer);*/
+
+	pmlmeinfo->state &= (~WIFI_FW_AUTH_NULL);
+	pmlmeinfo->state |= WIFI_FW_AUTH_STATE;
+
+	pmlmeinfo->auth_seq = 1;
+	pmlmeinfo->reauth_count = 0;
+	pmlmeinfo->reassoc_count = 0;
+	pmlmeinfo->link_count = 0;
+	pmlmeext->retry = 0;
+
+#ifdef CONFIG_RTW_80211R
+	if (rtw_ft_roam(padapter)) {
+		rtw_ft_set_status(padapter, RTW_FT_AUTHENTICATING_STA);
+		RTW_PRINT("start ft auth\n");
+	} else
+#endif
+		RTW_PRINT("start auth\n");
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (rtw_sec_chk_auth_type(padapter, MLME_AUTHTYPE_SAE)) {
+		if (rtw_cached_pmkid(padapter, get_my_bssid(&pmlmeinfo->dev_network)) != -1) {
+			RTW_INFO("SAE: PMKSA cache entry found\n");
+			padapter->securitypriv.auth_alg = WLAN_AUTH_OPEN;
+			goto no_external_auth;
+		}
+
+		RTW_PRINT("SAE: start external auth\n");
+		set_link_timer(pmlmeext, SAE_AUTH_TIMEOUT);
+		rtw_cfg80211_external_auth_request(padapter, NULL);
+		return;
+	}
+no_external_auth:
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	issue_auth(padapter, NULL, 0);
+
+	set_link_timer(pmlmeext, REAUTH_TO);
+
+}
+
+
+void start_clnt_assoc(_adapter *padapter)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	cancel_link_timer(pmlmeext); /*_cancel_timer_ex(&pmlmeext->link_timer);*/
+
+	pmlmeinfo->state &= (~(WIFI_FW_AUTH_NULL | WIFI_FW_AUTH_STATE));
+	pmlmeinfo->state |= (WIFI_FW_AUTH_SUCCESS | WIFI_FW_ASSOC_STATE);
+
+#ifdef CONFIG_RTW_80211R
+	if (rtw_ft_roam(padapter)
+	#ifdef CONFIG_RTW_WNM
+		|| rtw_wnm_btm_reassoc_req(padapter)
+	#endif
+	)
+		issue_reassocreq(padapter);
+	else
+#endif
+		issue_assocreq(padapter);
+
+	set_link_timer(pmlmeext, REASSOC_TO);
+}
+
+unsigned int receive_disconnect(_adapter *padapter, unsigned char *MacAddr, unsigned short reason, u8 locally_generated)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (!(rtw_match_bssid_ext(padapter, MacAddr)))
+		return _SUCCESS;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE) {
+		if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
+			if (report_del_sta_event(padapter, MacAddr, reason, _TRUE, locally_generated) != _FAIL)
+				pmlmeinfo->state = WIFI_FW_NULL_STATE;
+		} else if (pmlmeinfo->state & WIFI_FW_LINKING_STATE) {
+			if (report_join_res(padapter, -2, reason) != _FAIL)
+				pmlmeinfo->state = WIFI_FW_NULL_STATE;
+		} else
+			RTW_INFO(FUNC_ADPT_FMT" - End to Disconnect\n", FUNC_ADPT_ARG(padapter));
+#ifdef CONFIG_RTW_80211R
+		rtw_ft_roam_status_reset(padapter);
+#endif
+#ifdef CONFIG_RTW_WNM
+		rtw_wnm_reset_btm_state(padapter);
+#endif
+	}
+
+	return _SUCCESS;
+}
+
+/*
+ * Return channel index of struct dvobj_priv.rf_ctl.channel_set[] for channel match
+ * BSS_EX_OP_BAND(bss) & BSS_EX_OP_CH(bss), or -1 if not found.
+ */
+static int rtw_hidden_ssid_bss_count(_adapter *adapter, WLAN_BSSID_EX *bss)
+{
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	int chset_idx;
+
+	if (bss->InfrastructureMode != Ndis802_11Infrastructure)
+		return -1;
+
+	if (!hidden_ssid_ap(bss))
+		return -1;
+
+	chset_idx = rtw_chset_search_bch(chset, BSS_EX_OP_BAND(bss), BSS_EX_OP_CH(bss));
+	if (chset_idx < 0)
+		return chset_idx;
+
+	chset->chs[chset_idx].hidden_bss_cnt++;
+	return chset_idx;
+}
+
+/****************************************************************************
+
+Following are the functions to report events
+
+*****************************************************************************/
+
+void report_survey_event(_adapter *padapter, union recv_frame *precv_frame)
+{
+#ifdef CONFIG_SCAN_REPORT_ENQUEUE
+	struct cmd_obj *pcmd_obj;
+	u8 *pevtcmd;
+	u32 cmdsz;
+	struct rtw_evt_header *evt_hdr;
+	struct cmd_priv *pcmdpriv;
+	struct survey_event *psurvey_evt;
+#else
+	struct survey_event psurvey_evt;
+#endif
+	WLAN_BSSID_EX *bssid;
+	struct mlme_ext_priv *pmlmeext;
+	/* u8 *pframe = precv_frame->u.hdr.rx_data; */
+	/* uint len = precv_frame->u.hdr.len; */
+
+	struct rtw_chset *chset;
+	int ch_set_idx = -1;
+
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = NULL;
+
+
+	if (!padapter)
+		return;
+
+	padapter_link = GET_PRIMARY_LINK(padapter);
+	pmlmeext = &padapter->mlmeextpriv;
+#ifdef CONFIG_SCAN_REPORT_ENQUEUE
+	pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+	pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd_obj == NULL)
+		return;
+	pcmd_obj->padapter = padapter;
+
+	cmdsz = (sizeof(struct survey_event) + sizeof(struct rtw_evt_header));
+	pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+	if (pevtcmd == NULL) {
+		rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+		return;
+	}
+
+	_rtw_init_listhead(&pcmd_obj->list);
+
+	CMD_OBJ_SET_HWBAND(pcmd_obj, HW_BAND_MAX);
+	pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+	pcmd_obj->cmdsz = cmdsz;
+	pcmd_obj->parmbuf = pevtcmd;
+
+	pcmd_obj->rsp = NULL;
+	pcmd_obj->rspsz  = 0;
+
+	evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+	evt_hdr->len = sizeof(struct survey_event);
+	evt_hdr->id = EVT_SURVEY;
+	evt_hdr->seq = ATOMIC_INC_RETURN(&pmlmeext->event_seq);
+
+	psurvey_evt = (struct survey_event *)(pevtcmd + sizeof(struct rtw_evt_header));
+	bssid = (WLAN_BSSID_EX *)&psurvey_evt->bss;
+#else
+	bssid = (WLAN_BSSID_EX *)&psurvey_evt.bss;
+#endif
+
+	if (collect_bss_info(padapter, precv_frame, bssid) == _FAIL) {
+#ifdef CONFIG_SCAN_REPORT_ENQUEUE
+		rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+		rtw_mfree((u8 *)pevtcmd, cmdsz);
+#endif
+		return;
+	}
+
+	ch_set_idx = rtw_hidden_ssid_bss_count(padapter, bssid);
+
+	chset = adapter_to_chset(padapter);
+	_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+	if ((ch_set_idx >= 0) && (chset->chs[ch_set_idx].hidden_bss_cnt == 1)
+	    && pmlmeext->sitesurvey_res.scan_param) {
+		struct rtw_phl_scan_param *phl_scan;
+		struct phl_module_op_info op_info = {0};
+		u8 ch;
+		int i;
+
+		phl_scan = pmlmeext->sitesurvey_res.scan_param;
+		ch = chset->chs[ch_set_idx].ChannelNum;
+		for (i = 0; i < phl_scan->ch_num; i++) {
+			if (phl_scan->ch[i].channel != ch)
+				continue;
+			if (phl_scan->ch[i].type != RTW_PHL_SCAN_PASSIVE)
+				break;
+
+			op_info.op_code = FG_REQ_OP_NOTIFY_BCN_RCV;
+			op_info.inbuf = &ch;
+			op_info.inlen = 1;
+			rtw_phl_set_cur_cmd_info(GET_PHL_INFO(adapter_to_dvobj(padapter)),
+						padapter_link->wrlink->hw_band,
+						&op_info);
+			break;
+		}
+	}
+	_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+#ifdef CONFIG_SCAN_REPORT_ENQUEUE
+	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+#else
+	rtw_survey_event_callback(padapter, (u8 *)bssid);
+#endif
+
+	pmlmeext->sitesurvey_res.bss_cnt++;
+
+	return;
+
+}
+
+/*
+* @acs: aim to trigger channel selection
+*/
+void report_surveydone_event(_adapter *padapter, u8 reason, bool complete, bool acs, u8 flags)
+{
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	/* Currently only use the primary link to scan */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (flags & RTW_CMDF_DIRECTLY) {
+		struct surveydone_event survey_done;
+
+		survey_done.bss_cnt = pmlmeext->sitesurvey_res.bss_cnt;
+		survey_done.activate_ch_cnt = pmlmeext->sitesurvey_res.activate_ch_cnt;
+		survey_done.reason = reason;
+		survey_done.complete = complete;
+		survey_done.acs = acs;
+		RTW_INFO("survey done event(%x) reason:0x%x%s for "ADPT_FMT"-HW_BAND_%d\n",
+			survey_done.bss_cnt,
+			reason,
+			complete ? " complete" : "",
+			ADPT_ARG(padapter),
+			padapter_link->wrlink->hw_band);
+		rtw_surveydone_event_callback(padapter, (u8 *)&survey_done);
+	} else {
+
+		struct cmd_obj *pcmd_obj;
+		u8 *pevtcmd;
+		u32 cmdsz;
+		struct surveydone_event *psurveydone_evt;
+		struct rtw_evt_header *evt_hdr;
+
+		struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+		pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+		if (pcmd_obj == NULL)
+			return;
+		pcmd_obj->padapter = padapter;
+
+		cmdsz = (sizeof(struct surveydone_event) + sizeof(struct rtw_evt_header));
+		pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+		if (pevtcmd == NULL) {
+			rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+			return;
+		}
+
+		_rtw_init_listhead(&pcmd_obj->list);
+
+		CMD_OBJ_SET_HWBAND(pcmd_obj, HW_BAND_MAX);
+		pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+		pcmd_obj->cmdsz = cmdsz;
+		pcmd_obj->parmbuf = pevtcmd;
+
+		pcmd_obj->rsp = NULL;
+		pcmd_obj->rspsz  = 0;
+
+		evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+		evt_hdr->len = sizeof(struct surveydone_event);
+		evt_hdr->id = EVT_SURVEY_DONE;
+		evt_hdr->seq = ATOMIC_INC_RETURN(&pmlmeext->event_seq);
+
+		psurveydone_evt = (struct surveydone_event *)(pevtcmd + sizeof(struct rtw_evt_header));
+		psurveydone_evt->bss_cnt = pmlmeext->sitesurvey_res.bss_cnt;
+		psurveydone_evt->activate_ch_cnt = pmlmeext->sitesurvey_res.activate_ch_cnt;
+		psurveydone_evt->reason = reason;
+		psurveydone_evt->complete = complete;
+		psurveydone_evt->acs = acs;
+
+		RTW_INFO("survey done event(%x) reason:0x%x%s for "ADPT_FMT"- HW_BAND_%d\n",
+			psurveydone_evt->bss_cnt,
+			reason,
+			complete ? " complete" : "",
+			ADPT_ARG(padapter),
+			padapter_link->wrlink->hw_band);
+
+		rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+	}
+
+	return;
+}
+
+u32 report_join_res(_adapter *padapter, int aid_res, u16 status)
+{
+	struct cmd_obj *pcmd_obj;
+	u8	*pevtcmd;
+	u32 cmdsz;
+	struct joinbss_event *pjoinbss_evt;
+	struct rtw_evt_header *evt_hdr;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u32 ret = _FAIL;
+#ifdef CONFIG_80211BE_EHT
+	struct _ADAPTER_LINK *adapter_link;
+	struct wlan_mld_network *pmld_network;
+	struct wlan_network *rec_link_network;
+	WLAN_BSSID_EX *pnetwork;
+	u8 lidx;
+#endif
+
+	pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd_obj == NULL)
+		goto exit;
+	pcmd_obj->padapter = padapter;
+
+	cmdsz = (sizeof(struct joinbss_event) + sizeof(struct rtw_evt_header));
+	pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+	if (pevtcmd == NULL) {
+		rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+		goto exit;
+	}
+
+	_rtw_init_listhead(&pcmd_obj->list);
+
+	CMD_OBJ_SET_HWBAND(pcmd_obj, HW_BAND_MAX);
+	pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+	pcmd_obj->cmdsz = cmdsz;
+	pcmd_obj->parmbuf = pevtcmd;
+
+	pcmd_obj->rsp = NULL;
+	pcmd_obj->rspsz  = 0;
+
+	evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+	evt_hdr->len = sizeof(struct joinbss_event);
+	evt_hdr->id = EVT_JOINBSS;
+	evt_hdr->seq = ATOMIC_INC_RETURN(&padapter->mlmeextpriv.event_seq);
+
+	pjoinbss_evt = (struct joinbss_event *)(pevtcmd + sizeof(struct rtw_evt_header));
+	_rtw_memcpy((unsigned char *)(&(pjoinbss_evt->network.network)), &(pmlmeinfo->network), sizeof(WLAN_BSSID_EX));
+	pjoinbss_evt->network.join_res = pjoinbss_evt->network.aid = aid_res;
+
+	RTW_INFO("report_join_res(%d, %u)\n", aid_res, status);
+#ifdef CONFIG_80211BE_EHT
+	if (pmlmeinfo->network.is_mld) {
+		pmld_network = (struct wlan_mld_network *)rtw_zmalloc(sizeof(struct wlan_mld_network));
+		if (!pmld_network) {
+			rtw_free_cmd_obj(pcmd_obj);
+			RTW_INFO(FUNC_ADPT_FMT" Failed to allocate mld network\n", FUNC_ADPT_ARG(padapter));
+			goto exit;
+		}
+		_rtw_memset(pmld_network, 0, sizeof(struct wlan_mld_network));
+		_rtw_memcpy(pmld_network->mac_addr, padapter->mlmeextpriv.mlmext_info.dev_network.MacAddress, ETH_ALEN);
+
+		/* Link mld network with the primary link */
+		rtw_link_mld_network(padapter, pmld_network, &pjoinbss_evt->network);
+
+		for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+			adapter_link = GET_LINK(padapter, lidx);
+			if (!adapter_link->mlmepriv.to_join || adapter_link == padapter_link)
+				continue;
+
+			pnetwork = &adapter_link->mlmeextpriv.mlmext_info.network;
+			rec_link_network = (struct wlan_network *)rtw_zmalloc(sizeof(struct wlan_network));
+			if (!rec_link_network) {
+				RTW_INFO(FUNC_ADPT_FMT" cannot allocate link network for link_id %u, set adapter_link %d to_join to _FALSE\n"
+					, FUNC_ADPT_ARG(padapter), adapter_link->mlmepriv.link_id, adapter_link->wrlink->id);
+				adapter_link->mlmepriv.to_join = _FALSE;
+				continue;
+			}
+			_rtw_memcpy(&rec_link_network->network, pnetwork, sizeof(WLAN_BSSID_EX));
+			rtw_link_mld_network(padapter, pmld_network, rec_link_network);
+		}
+
+		rtw_joinbss_event_prehandle(padapter, (u8 *)&pjoinbss_evt->network, status);
+
+		rtw_unlink_network_by_linkid(pmld_network, pjoinbss_evt->network.network.link_id);
+		rtw_free_cloned_mld_network(pmld_network);
+	} else
+#endif
+	{
+		rtw_joinbss_event_prehandle(padapter, (u8 *)&pjoinbss_evt->network, status);
+	}
+
+
+	ret = rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+
+exit:
+	return ret;
+}
+
+void report_wmm_edca_update(_adapter *padapter)
+{
+	struct cmd_obj *pcmd_obj;
+	u8 *pevtcmd;
+	u32 cmdsz;
+	struct wmm_event *pwmm_event;
+	struct rtw_evt_header *evt_hdr;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+	pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd_obj == NULL)
+		return;
+	pcmd_obj->padapter = padapter;
+
+	cmdsz = (sizeof(struct wmm_event) + sizeof(struct rtw_evt_header));
+	pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+	if (pevtcmd == NULL) {
+		rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+		return;
+	}
+
+	_rtw_init_listhead(&pcmd_obj->list);
+
+	CMD_OBJ_SET_HWBAND(pcmd_obj, HW_BAND_MAX);
+	pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+	pcmd_obj->cmdsz = cmdsz;
+	pcmd_obj->parmbuf = pevtcmd;
+
+	pcmd_obj->rsp = NULL;
+	pcmd_obj->rspsz  = 0;
+
+	evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+	evt_hdr->len = sizeof(struct wmm_event);
+	evt_hdr->id = EVT_WMM_UPDATE;
+	evt_hdr->seq = ATOMIC_INC_RETURN(&pmlmeext->event_seq);
+
+	pwmm_event = (struct wmm_event *)(pevtcmd + sizeof(struct rtw_evt_header));
+	pwmm_event->wmm = 0;
+
+	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+
+	return;
+
+}
+
+u32 report_del_sta_event(_adapter *padapter, unsigned char *MacAddr, unsigned short reason, bool enqueue, u8 locally_generated)
+{
+	struct cmd_obj *pcmd_obj;
+	u8 *pevtcmd;
+	u32 cmdsz;
+	struct sta_info *psta;
+	int mac_id = -1;
+	struct stadel_event *pdel_sta_evt;
+	struct rtw_evt_header *evt_hdr;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8 res = _SUCCESS;
+
+	if (MLME_IS_AP(padapter)) {
+		RTW_INFO("%s : enqueue=%u\n", __func__, enqueue);
+		if (enqueue) {
+			res = rtw_add_del_sta_cmd_check(padapter, MacAddr, _FALSE);
+		} else {
+			psta = rtw_get_stainfo_to_free(&padapter->stapriv, MacAddr);
+			if (!psta || !psta->phl_sta || !psta->phl_sta->mld) {
+				res = _FAIL;
+			} else {
+				rtw_hw_disconnect(padapter, psta);
+				res = rtw_free_mld_stainfo(padapter, psta->phl_sta->mld);
+			}
+		}
+		goto exit;
+	}
+
+	/* prepare cmd parameter */
+	cmdsz = (sizeof(struct stadel_event) + sizeof(struct rtw_evt_header));
+	pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+	if (pevtcmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+	evt_hdr->len = sizeof(struct stadel_event);
+	evt_hdr->id = EVT_DEL_STA;
+	evt_hdr->seq = ATOMIC_INC_RETURN(&pmlmeext->event_seq);
+
+	pdel_sta_evt = (struct stadel_event *)(pevtcmd + sizeof(struct rtw_evt_header));
+	_rtw_memcpy((unsigned char *)(&(pdel_sta_evt->macaddr)), MacAddr, ETH_ALEN);
+	_rtw_memcpy((unsigned char *)(pdel_sta_evt->rsvd), (unsigned char *)(&reason), 2);
+	psta = rtw_get_stainfo_to_free(&padapter->stapriv, MacAddr);
+	if (psta) {
+		mac_id = (int)psta->phl_sta->macid;
+	} else {
+		mac_id = (-1);
+	}
+	pdel_sta_evt->mac_id = mac_id;
+	pdel_sta_evt->locally_generated = locally_generated;
+
+	if (!enqueue) {
+		/* do directly */
+		rtw_stadel_event_callback(padapter, (u8 *)pdel_sta_evt);
+		rtw_mfree(pevtcmd, cmdsz);
+	} else {
+		pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+		if (pcmd_obj == NULL) {
+			rtw_mfree(pevtcmd, cmdsz);
+			res = _FAIL;
+			goto exit;
+		}
+		pcmd_obj->padapter = padapter;
+
+		_rtw_init_listhead(&pcmd_obj->list);
+		CMD_OBJ_SET_HWBAND(pcmd_obj, HW_BAND_MAX);
+		pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+		pcmd_obj->cmdsz = cmdsz;
+		pcmd_obj->parmbuf = pevtcmd;
+
+		pcmd_obj->rsp = NULL;
+		pcmd_obj->rspsz  = 0;
+
+		res = rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+	}
+
+exit:
+
+	RTW_INFO(FUNC_ADPT_FMT" "MAC_FMT" mac_id=%d, enqueue:%d, res:%u\n"
+		, FUNC_ADPT_ARG(padapter), MAC_ARG(MacAddr), mac_id, enqueue, res);
+
+	return res;
+}
+
+void report_add_sta_event(_adapter *padapter, unsigned char *MacAddr)
+{
+	struct cmd_obj *pcmd_obj;
+	u8 *pevtcmd;
+	u32 cmdsz;
+	struct stassoc_event *padd_sta_evt;
+	struct rtw_evt_header *evt_hdr;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+	if (MLME_IS_AP(padapter)){
+		rtw_add_del_sta_cmd_check(padapter, MacAddr, _TRUE);
+		return;
+	}
+
+	pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd_obj == NULL)
+		return;
+	pcmd_obj->padapter = padapter;
+
+	cmdsz = (sizeof(struct stassoc_event) + sizeof(struct rtw_evt_header));
+	pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+	if (pevtcmd == NULL) {
+		rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+		return;
+	}
+
+	_rtw_init_listhead(&pcmd_obj->list);
+
+	CMD_OBJ_SET_HWBAND(pcmd_obj, HW_BAND_MAX);
+	pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+	pcmd_obj->cmdsz = cmdsz;
+	pcmd_obj->parmbuf = pevtcmd;
+
+	pcmd_obj->rsp = NULL;
+	pcmd_obj->rspsz  = 0;
+
+	evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+	evt_hdr->len = sizeof(struct stassoc_event);
+	evt_hdr->id = EVT_ADD_STA;
+	evt_hdr->seq = ATOMIC_INC_RETURN(&pmlmeext->event_seq);
+
+	padd_sta_evt = (struct stassoc_event *)(pevtcmd + sizeof(struct rtw_evt_header));
+	_rtw_memcpy((unsigned char *)(&(padd_sta_evt->macaddr)), MacAddr, ETH_ALEN);
+
+	RTW_INFO("report_add_sta_event: add STA\n");
+
+	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+
+	return;
+}
+
+/****************************************************************************
+
+Following are the event callback functions
+
+*****************************************************************************/
+
+/* for sta/adhoc mode */
+void update_sta_info(_adapter *padapter, struct sta_info *psta) {
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	/* ERP */
+	VCS_update(padapter, psta);
+
+#ifdef CONFIG_80211N_HT
+	/* HT */
+	if (pmlmepriv->htpriv.ht_option) {
+		psta->htpriv.ht_option = _TRUE;
+
+		psta->ampdu_priv.ampdu_enable = pmlmepriv->ampdu_priv.ampdu_enable;
+
+		psta->ampdu_priv.rx_ampdu_min_spacing = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & IEEE80211_HT_CAP_AMPDU_DENSITY) >> 2;
+
+		if (support_short_GI(padapter, padapter_link, &(pmlmeinfo->HT_caps), CHANNEL_WIDTH_20))
+			psta->htpriv.sgi_20m = _TRUE;
+
+		if (support_short_GI(padapter, padapter_link, &(pmlmeinfo->HT_caps), CHANNEL_WIDTH_40))
+			psta->htpriv.sgi_40m = _TRUE;
+
+		psta->qos_option = _TRUE;
+
+		psta->htpriv.ldpc_cap = pmlmepriv->htpriv.ldpc_cap;
+		psta->htpriv.stbc_cap = pmlmepriv->htpriv.stbc_cap;
+		psta->htpriv.beamform_cap = pmlmepriv->htpriv.beamform_cap;
+
+		_rtw_memcpy(&psta->htpriv.ht_cap, &pmlmeinfo->HT_caps, sizeof(struct rtw_ieee80211_ht_cap));
+		#ifdef CONFIG_BEAMFORMING
+		psta->htpriv.beamform_cap = pmlmepriv->htpriv.beamform_cap;
+		#endif
+	} else
+#endif /* CONFIG_80211N_HT */
+	{
+#ifdef CONFIG_80211N_HT
+		psta->htpriv.ht_option = _FALSE;
+		psta->ampdu_priv.ampdu_enable = _FALSE;
+		psta->ampdu_priv.tx_amsdu_enable = _FALSE;
+		psta->htpriv.sgi_20m = _FALSE;
+		psta->htpriv.sgi_40m = _FALSE;
+#endif /* CONFIG_80211N_HT */
+		psta->qos_option = _FALSE;
+
+	}
+
+#ifdef CONFIG_80211N_HT
+	psta->htpriv.ch_offset = pmlmeext->chandef.offset;
+
+	psta->ampdu_priv.agg_enable_bitmap = 0x0;/* reset */
+	psta->ampdu_priv.candidate_tid_bitmap = 0x0;/* reset */
+#endif /* CONFIG_80211N_HT */
+
+	psta->phl_sta->chandef.bw = pmlmeext->chandef.bw;
+
+	/* QoS */
+	if (pmlmepriv->qospriv.qos_option)
+		psta->qos_option = _TRUE;
+
+#ifdef CONFIG_80211AC_VHT
+	_rtw_memcpy(&psta->vhtpriv, &pmlmepriv->vhtpriv, sizeof(struct vht_priv));
+	if (psta->vhtpriv.vht_option) {
+		/* ToDo: need to API to inform hal_sta->ra_info.is_vht_enable  */
+		/* psta->phl_sta->ra_info.is_vht_enable = _TRUE; */
+		#ifdef CONFIG_BEAMFORMING
+		psta->vhtpriv.beamform_cap = pmlmepriv->vhtpriv.beamform_cap;
+		#endif /*CONFIG_BEAMFORMING*/
+	}
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+	_rtw_memcpy(&psta->hepriv, &pmlmepriv->hepriv, sizeof(struct he_priv));
+#endif /* CONFIG_80211AX_HE */
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (psta->phl_sta->chandef.band == BAND_ON_6G) {
+		psta->ampdu_priv.ampdu_enable = pmlmepriv->ampdu_priv.ampdu_enable;
+		/* update mpdu_min_spacing in HE_6g_bandcap_handler */
+		psta->ampdu_priv.rx_ampdu_min_spacing = psta->hepriv.mpdu_min_spacing;
+	}
+#endif
+
+	/* ToDo: need to API to inform hal_sta->ra_info.is_support_sgi  */
+	/* psta->phl_sta->ra_info.is_support_sgi = query_ra_short_GI(psta, rtw_get_tx_bw_mode(padapter, psta)); */
+
+	_rtw_spinlock_bh(&psta->lock);
+	psta->state = WIFI_ASOC_STATE;
+	_rtw_spinunlock_bh(&psta->lock);
+
+}
+
+void update_sta_trx_nss(_adapter *adapter, struct sta_info *psta)
+{
+	s8 tx_nss, rx_nss;
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	/* get adapter tx nss */
+	tx_nss = rtw_get_sta_tx_nss(adapter, psta);
+	/* get adapter rx nss */
+	rx_nss =  rtw_get_sta_rx_nss(adapter, psta);
+
+	/* peer sta tx should referece adapter rx_nss */
+	psta->phl_sta->asoc_cap.nss_tx = rx_nss;
+	/* peer sta rx should referece adapter tx_nss */
+	psta->phl_sta->asoc_cap.nss_rx = tx_nss;
+
+#ifdef CONFIG_DBCC_SUPPORT
+	if (rtw_phl_mr_is_db(adapter_to_dvobj(adapter)->phl)) {
+		RTW_INFO("%s, stay in dual-band update_sta_trx_nss\n", __func__);
+		rtw_ctrl_and_backup_assoc_cap_rx_nss(adapter, psta, 1);
+	}
+#endif
+
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	rtw_ctrl_txss_update(adapter, psta);
+#endif
+
+	RTW_INFO("STA - MAC_ID:%d, Tx - %d SS, Rx - %d SS\n",
+			psta->phl_sta->macid, tx_nss, rx_nss);
+}
+
+void update_sta_smps_cap(_adapter *adapter, struct sta_info *psta)
+{
+	struct _ADAPTER_LINK	*adapter_link = psta->padapter_link;
+	struct mlme_ext_priv	*pmlmeext = &adapter->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(adapter_link->mlmeextpriv.mlmext_info);
+
+	/*Spatial Multiplexing Power Save*/
+	if (check_fwstate(&adapter->mlmepriv, WIFI_AP_STATE) == _TRUE)
+		psta->phl_sta->asoc_cap.sm_ps = psta->smps_mode;
+	else
+		psta->phl_sta->asoc_cap.sm_ps = pmlmeinfo->SM_PS;
+
+	RTW_INFO("STA - MAC_ID:%d, SM_PS %d\n",
+		psta->phl_sta->macid, psta->phl_sta->asoc_cap.sm_ps);
+
+	if (psta->phl_sta->asoc_cap.sm_ps == SM_PS_STATIC
+		&& psta->phl_sta->asoc_cap.nss_rx != 1) {
+		pmlmeext->txss_bk = psta->phl_sta->asoc_cap.nss_rx;
+		rtw_phl_sta_assoc_cap_process(psta->phl_sta, _TRUE);
+		psta->phl_sta->asoc_cap.nss_rx = 1;
+		psta->phl_sta->asoc_cap.stbc_ht_rx = 0;
+		psta->phl_sta->asoc_cap.stbc_vht_rx = 0;
+		psta->phl_sta->asoc_cap.stbc_he_rx = 0;
+	} else if (psta->phl_sta->asoc_cap.sm_ps == SM_PS_DYNAMIC
+		&& check_fwstate(&adapter->mlmepriv, WIFI_AP_STATE)) {
+		psta->rtsen = 1;
+	}
+	RTW_INFO("STA - MAC_ID:%d, Tx - %d SS, Rx - %d SS\n",
+			psta->phl_sta->macid,
+			psta->phl_sta->asoc_cap.nss_tx,
+			psta->phl_sta->asoc_cap.nss_rx);
+}
+
+void update_sta_rate_mask(_adapter *padapter, struct sta_info *psta)
+{
+	struct registry_priv *registrypriv = &padapter->registrypriv;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 i, tx_nss;
+	u64 tx_ra_bitmap = 0, tmp64=0;
+
+	if (psta == NULL)
+		return;
+
+	padapter_link = psta->padapter_link;
+	/* b/g mode ra_bitmap  */
+	for (i = 0; i < sizeof(psta->bssrateset); i++) {
+		if (psta->bssrateset[i])
+			tx_ra_bitmap |= rtw_get_bit_value_from_ieee_value(psta->bssrateset[i] & 0x7f);
+	}
+
+#ifdef CONFIG_80211N_HT
+	if (registrypriv->ht_enable && is_supported_ht(registrypriv->wireless_mode)) {
+		tx_nss = get_phy_tx_nss(padapter, padapter_link);
+#ifdef CONFIG_80211AC_VHT
+		if (psta->vhtpriv.vht_option) {
+			/* AC mode ra_bitmap */
+			tx_ra_bitmap |= (rtw_vht_mcs_map_to_bitmap(psta->vhtpriv.vht_mcs_map, tx_nss) << 12);
+		} else
+#endif /* CONFIG_80211AC_VHT */
+		{
+			if (psta->htpriv.ht_option) {
+				/* n mode ra_bitmap */
+
+				/* Handling SMPS mode for AP MODE only*/
+				if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == _TRUE) {
+				/*0:static SMPS, 1:dynamic SMPS, 3:SMPS disabled, 2:reserved*/
+					if (psta->smps_mode == SM_PS_STATIC)
+						/*operate with only one active receive chain // 11n-MCS rate <= MSC7*/
+						tx_nss = rtw_min(tx_nss, 1);
+				}
+
+				tmp64 = rtw_ht_mcs_set_to_bitmap(psta->htpriv.ht_cap.supp_mcs_set, tx_nss);
+				tx_ra_bitmap |= (tmp64 << 12);
+			}
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+	/* ToDo: Need API to inform hal_sta->ra_info.ramask */
+	/* psta->phl_sta->ra_info.ramask = tx_ra_bitmap;*/
+	psta->init_rate = get_highest_rate_idx(tx_ra_bitmap) & 0x3f;
+}
+
+void update_sta_ra_info(_adapter *padapter, struct sta_info *psta)
+{
+	update_sta_trx_nss(padapter, psta);
+	update_sta_smps_cap(padapter, psta);
+	update_sta_rate_mask(padapter, psta);
+}
+
+void rtw_mlmeext_disconnect(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct recv_info *precvinfo = &padapter->recvinfo;
+	u8 state_backup = (pmlmeinfo->state & 0x03);
+	u8 ASIX_ID[] = {0x00, 0x0E, 0xC6};
+	u8 *mac;
+	struct sta_info *sta;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+
+	/* check if sta is ASIX peer and fix IOT issue if it is. */
+	if (_rtw_memcmp(get_my_bssid(&pmlmeinfo->dev_network) , ASIX_ID , 3)) {
+		u8 iot_flag = _FALSE;
+		rtw_hal_set_hwreg(padapter, HW_VAR_ASIX_IOT, (u8 *)(&iot_flag));
+	}
+	pmlmeinfo->state = WIFI_FW_NULL_STATE;
+
+	/* switch to the 20M Hz mode after disconnect */
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		padapter_link->mlmeextpriv.chandef.bw = CHANNEL_WIDTH_20;
+		padapter_link->mlmeextpriv.chandef.offset = CHAN_OFFSET_NO_EXT;
+	}
+
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	pmlmeext->txss_1ss = _FALSE;
+#endif
+
+#ifdef CONFIG_LAYER2_ROAMING
+#define RTW_ROAM_DICONNECT_DELAY        20
+	if (pmlmepriv->roam_network)
+		rtw_msleep_os(RTW_ROAM_DICONNECT_DELAY);
+#endif
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		/* before chanctx_del */
+		mac = padapter_link->mlmeextpriv.mlmext_info.network.MacAddress;
+		sta = rtw_get_stainfo(&padapter->stapriv, mac);
+		if (sta)
+			rtw_hw_disconnect(padapter, sta);
+		else
+			RTW_ERR(FUNC_ADPT_FMT ": can't find drv sta info for "
+			MAC_FMT " !\n", FUNC_ADPT_ARG(padapter), MAC_ARG(mac));
+	}
+
+	cancel_link_timer(pmlmeext); /*_cancel_timer_ex(&pmlmeext->link_timer);*/
+
+	/* pmlmepriv->LinkDetectInfo.TrafficBusyState = _FALSE; */
+	pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 0;
+	pmlmepriv->LinkDetectInfo.LowPowerTransitionCount = 0;
+
+#ifdef CONFIG_TDLS
+	padapter->tdlsinfo.ap_prohibited = _FALSE;
+
+	/* For TDLS channel switch, currently we only allow it to work in wifi logo test mode */
+	if (padapter->registrypriv.wifi_spec == 1)
+		padapter->tdlsinfo.ch_switch_prohibited = _FALSE;
+#endif /* CONFIG_TDLS */
+
+#ifdef PRIVATE_R
+	pxmitpriv->tx_bk_drop_cnt = 0;
+	pxmitpriv->tx_be_drop_cnt = 0;
+	pxmitpriv->tx_vi_drop_cnt = 0;
+	pxmitpriv->tx_vo_drop_cnt = 0;
+	_rtw_memset(&(precvinfo->rx_vo_pkt_count_per_data_rate), 0, sizeof(precvinfo->rx_vo_pkt_count_per_data_rate));
+	precvinfo->rx_vo_pkt_retry_count = 0;
+#endif
+
+#ifdef CONFIG_RTS_FULL_BW
+	rtw_set_rts_bw(padapter);
+#endif/*CONFIG_RTS_FULL_BW*/
+}
+
+/* Return 0 for success, otherwise fail. */
+int rtw_set_hw_after_join(struct _ADAPTER *a, int join_res)
+{
+	struct mlme_priv *mlme = &a->mlmepriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	struct sta_info *sta;
+	u8 *mac;
+	int err;
+	struct _ADAPTER_LINK *alink = NULL;
+	u8 lidx;
+	err = SUCCESS;
+
+
+	if (mlme->wpa_phase == _TRUE)
+		mlme->wpa_phase = _FALSE;
+
+	if (join_res < 0) {
+		err = rtw_hw_connect_abort(a);
+		return err;
+	}
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+	mlme->bGetGateway = 1;
+	mlme->GetGatewayTryCnt = 0;
+#endif
+	for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+		alink = GET_LINK(a, lidx);
+		if (alink->mlmepriv.is_accepted) {
+			mac = (u8*)alink->mlmeextpriv.mlmext_info.network.MacAddress;
+			sta = rtw_get_stainfo(&a->stapriv, mac);
+			if (!sta) {
+				RTW_ERR(FUNC_ADPT_FMT ": drv sta_info(" MAC_FMT ") not exist!\n",
+					FUNC_ADPT_ARG(a), MAC_ARG(mac));
+				return -1;
+			}
+			sta->phl_sta->wmode = alink->mlmeextpriv.cur_wireless_mode;
+		} else {
+			mac = (u8*)alink->mlmeextpriv.mlmext_info.network.MacAddress;
+			sta = rtw_get_stainfo(&a->stapriv, mac);
+			if (sta) /* Remove the unaccepted link sta */
+				rtw_hw_connect_remove_sta(a, sta);
+		}
+	}
+	err = rtw_hw_connected(a);
+
+	/*reset counter for accumulated false alarm counter*/
+	ATOMIC_SET((ATOMIC_T *)&dvobj->fa_cnt_acc, 0);
+
+	return err;
+}
+
+/* currently only adhoc mode will go here */
+void mlmeext_sta_add_event_callback(_adapter *padapter, struct sta_info *psta)
+{
+	struct _ADAPTER_LINK	*padapter_link = psta->padapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(padapter->mlmeextpriv.mlmext_info);
+	u8	join_type;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
+		if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) { /* adhoc master or sta_count>1 */
+			/* nothing to do */
+		} else { /* adhoc client */
+
+			/* start beacon */
+			if (send_beacon(padapter) == _FAIL)
+				rtw_warn_on(1);
+
+			pmlmeinfo->state |= WIFI_FW_ASSOC_SUCCESS;
+		}
+
+		join_type = 2;
+		rtw_hal_set_hwreg(padapter, HW_VAR_MLME_JOIN, (u8 *)(&join_type));
+	}
+
+	/* update adhoc sta_info */
+	update_sta_info(padapter, psta);
+
+	update_sta_ra_info(padapter, psta);
+
+	/* ToDo: HT for Ad-hoc */
+	psta->phl_sta->wmode = rtw_check_network_type(psta->bssrateset, psta->bssratelen, pmlmeext->chandef.chan);
+}
+
+void mlmeext_sta_del_event_callback(_adapter *padapter)
+{
+	if (is_client_associated_to_ap(padapter) || is_IBSS_empty(padapter))
+		rtw_mlmeext_disconnect(padapter);
+}
+
+/****************************************************************************
+
+Following are the functions for the timer handlers
+
+*****************************************************************************/
+void _linked_info_dump(_adapter *padapter)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (padapter->bLinkInfoDump) {
+		rtw_hal_get_def_var(padapter, padapter_link, HW_DEF_RA_INFO_DUMP, RTW_DBGDUMP);
+		/*rtw_hal_set_phydm_var(padapter, HAL_PHYDM_RX_INFO_DUMP, RTW_DBGDUMP, _FALSE);*/
+	}
+}
+/********************************************************************
+
+When station does not receive any packet in MAX_CONTINUAL_NORXPACKET_COUNT*2 seconds,
+recipient station will teardown the block ack by issuing DELBA frame.
+
+*********************************************************************/
+void rtw_delba_check(_adapter *padapter, struct sta_info *psta)
+{
+	int	i = 0;
+	int ret = _SUCCESS;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	/*
+		IOT issue,occur Broadcom ap(Buffalo WZR-D1800H,Netgear R6300).
+		AP is originator.AP does not transmit unicast packets when STA response its BAR.
+		This case probably occur ap issue BAR after AP builds BA.
+
+		Follow 802.11 spec, STA shall maintain an inactivity timer for every negotiated Block Ack setup.
+		The inactivity timer is not reset when MPDUs corresponding to other TIDs are received.
+	*/
+	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_BROADCOM) {
+		for (i = 0; i < TID_NUM ; i++) {
+			if ((psta->recvreorder_ctrl[i].enable) &&
+                        (sta_rx_data_qos_pkts(psta, i) == sta_last_rx_data_qos_pkts(psta, i)) ) {
+					if (_TRUE == rtw_inc_and_chk_continual_no_rx_packet(psta, i)) {
+						/* send a DELBA frame to the peer STA with the Reason Code field set to TIMEOUT */
+							ret = issue_del_ba_ex(padapter, psta->phl_sta->mac_addr, i, 39, 0, 3, 1);
+						psta->recvreorder_ctrl[i].enable = _FALSE;
+						if (ret != _FAIL)
+							psta->recvreorder_ctrl[i].ampdu_size = RX_AMPDU_SIZE_INVALID;
+						rtw_reset_continual_no_rx_packet(psta, i);
+					}
+			} else {
+				/* The inactivity timer is reset when MPDUs to the TID is received. */
+				rtw_reset_continual_no_rx_packet(psta, i);
+			}
+		}
+	}
+}
+
+u8 chk_ap_is_alive(_adapter *padapter, struct sta_info *psta)
+{
+	u8 ret = _FALSE;
+#ifdef DBG_EXPIRATION_CHK
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	RTW_INFO(FUNC_ADPT_FMT" rx:"STA_PKTS_FMT", beacon:%llu, probersp_to_self:%llu"
+		/*", probersp_bm:%llu, probersp_uo:%llu, probereq:%llu, BI:%u"*/
+		 ", retry:%u\n"
+		 , FUNC_ADPT_ARG(padapter)
+		 , STA_RX_PKTS_DIFF_ARG(psta)
+		, psta->sta_stats.rx_beacon_pkts - psta->sta_stats.last_rx_beacon_pkts
+		, psta->sta_stats.rx_probersp_pkts - psta->sta_stats.last_rx_probersp_pkts
+		/*, psta->sta_stats.rx_probersp_bm_pkts - psta->sta_stats.last_rx_probersp_bm_pkts
+		, psta->sta_stats.rx_probersp_uo_pkts - psta->sta_stats.last_rx_probersp_uo_pkts
+		, psta->sta_stats.rx_probereq_pkts - psta->sta_stats.last_rx_probereq_pkts
+		 , pmlmeinfo->bcn_interval*/
+		 , pmlmeext->retry
+		);
+
+	RTW_INFO(FUNC_ADPT_FMT" tx_pkts:%llu, link_count:%u\n", FUNC_ADPT_ARG(padapter)
+		 , sta_tx_pkts(psta)
+		 , pmlmeinfo->link_count
+		);
+#endif
+
+	if ((sta_rx_data_pkts(psta) == sta_last_rx_data_pkts(psta))
+	    && sta_rx_beacon_pkts(psta) == sta_last_rx_beacon_pkts(psta)
+	    && sta_rx_probersp_pkts(psta) == sta_last_rx_probersp_pkts(psta)
+	   )
+		ret = _FALSE;
+	else
+		ret = _TRUE;
+
+	sta_update_last_rx_pkts(psta);
+
+	return ret;
+}
+
+u8 chk_adhoc_peer_is_alive(struct sta_info *psta)
+{
+	u8 ret = _TRUE;
+
+#ifdef DBG_EXPIRATION_CHK
+	RTW_INFO("sta:"MAC_FMT", rssi:%d, rx:"STA_PKTS_FMT", beacon:%llu, probersp_to_self:%llu"
+		/*", probersp_bm:%llu, probersp_uo:%llu, probereq:%llu, BI:%u"*/
+		 ", expire_to:%u\n"
+		 , MAC_ARG(psta->phl_sta->mac_addr)
+		 , 0 /* TODO: psta->phl_sta->hal_sta->rssi_stat.rssi */
+		 , STA_RX_PKTS_DIFF_ARG(psta)
+		, psta->sta_stats.rx_beacon_pkts - psta->sta_stats.last_rx_beacon_pkts
+		, psta->sta_stats.rx_probersp_pkts - psta->sta_stats.last_rx_probersp_pkts
+		/*, psta->sta_stats.rx_probersp_bm_pkts - psta->sta_stats.last_rx_probersp_bm_pkts
+		, psta->sta_stats.rx_probersp_uo_pkts - psta->sta_stats.last_rx_probersp_uo_pkts
+		, psta->sta_stats.rx_probereq_pkts - psta->sta_stats.last_rx_probereq_pkts
+		 , pmlmeinfo->bcn_interval*/
+		 , psta->expire_to
+		);
+#endif
+
+	if (sta_rx_data_pkts(psta) == sta_last_rx_data_pkts(psta)
+	    && sta_rx_beacon_pkts(psta) == sta_last_rx_beacon_pkts(psta)
+	    && sta_rx_probersp_pkts(psta) == sta_last_rx_probersp_pkts(psta))
+		ret = _FALSE;
+
+	sta_update_last_rx_pkts(psta);
+
+	return ret;
+}
+
+#ifdef CONFIG_TDLS
+u8 chk_tdls_peer_sta_is_alive(_adapter *padapter, struct sta_info *psta)
+{
+	if ((psta->sta_stats.rx_data_pkts == psta->sta_stats.last_rx_data_pkts)
+	    && (psta->sta_stats.rx_tdls_disc_rsp_pkts == psta->sta_stats.last_rx_tdls_disc_rsp_pkts))
+		return _FALSE;
+
+	return _TRUE;
+}
+
+void linked_status_chk_tdls(_adapter *padapter)
+{
+	struct candidate_pool {
+		struct sta_info *psta;
+		u8 addr[ETH_ALEN];
+	};
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 ack_chk;
+	struct sta_info *psta;
+	int i, num_teardown = 0, num_checkalive = 0;
+	_list	*plist, *phead;
+	struct tdls_txmgmt txmgmt;
+	struct candidate_pool checkalive[MAX_ALLOWED_TDLS_STA_NUM];
+	struct candidate_pool teardown[MAX_ALLOWED_TDLS_STA_NUM];
+	u8 tdls_sta_max = _FALSE;
+
+#define ALIVE_MIN 2
+#define ALIVE_MAX 5
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	_rtw_memset(checkalive, 0x00, sizeof(checkalive));
+	_rtw_memset(teardown, 0x00, sizeof(teardown));
+
+	if ((padapter->tdlsinfo.link_established == _TRUE)) {
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+		for (i = 0; i < NUM_STA; i++) {
+			phead = &(pstapriv->sta_hash[i]);
+			plist = get_next(phead);
+
+			while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+				psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+				plist = get_next(plist);
+
+				if (psta->tdls_sta_state & TDLS_LINKED_STATE) {
+					psta->alive_count++;
+					if (psta->alive_count >= ALIVE_MIN) {
+						if (chk_tdls_peer_sta_is_alive(padapter, psta) == _FALSE) {
+							if (psta->alive_count < ALIVE_MAX) {
+								_rtw_memcpy(checkalive[num_checkalive].addr, psta->phl_sta->mac_addr, ETH_ALEN);
+								checkalive[num_checkalive].psta = psta;
+								num_checkalive++;
+							} else {
+								_rtw_memcpy(teardown[num_teardown].addr, psta->phl_sta->mac_addr, ETH_ALEN);
+								teardown[num_teardown].psta = psta;
+								num_teardown++;
+							}
+						} else
+							psta->alive_count = 0;
+					}
+					psta->sta_stats.last_rx_data_pkts = psta->sta_stats.rx_data_pkts;
+					psta->sta_stats.last_rx_tdls_disc_rsp_pkts = psta->sta_stats.rx_tdls_disc_rsp_pkts;
+
+					if ((num_checkalive >= MAX_ALLOWED_TDLS_STA_NUM) || (num_teardown >= MAX_ALLOWED_TDLS_STA_NUM)) {
+						tdls_sta_max = _TRUE;
+						break;
+					}
+				}
+			}
+
+			if (tdls_sta_max == _TRUE)
+				break;
+		}
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+		if (num_checkalive > 0) {
+			for (i = 0; i < num_checkalive; i++) {
+				_rtw_memcpy(txmgmt.peer, checkalive[i].addr, ETH_ALEN);
+				issue_tdls_dis_req(padapter, &txmgmt);
+				issue_tdls_dis_req(padapter, &txmgmt);
+				issue_tdls_dis_req(padapter, &txmgmt);
+			}
+		}
+
+		if (num_teardown > 0) {
+			for (i = 0; i < num_teardown; i++) {
+				RTW_INFO("[%s %d] Send teardown to "MAC_FMT"\n", __FUNCTION__, __LINE__, MAC_ARG(teardown[i].addr));
+				txmgmt.status_code = _RSON_TDLS_TEAR_TOOFAR_;
+				_rtw_memcpy(txmgmt.peer, teardown[i].addr, ETH_ALEN);
+				issue_tdls_teardown(padapter, &txmgmt, _FALSE);
+			}
+		}
+	}
+
+}
+#endif /* CONFIG_TDLS */
+
+#ifdef CONFIG_POST_CORE_KEEP_ALIVE
+void rtw_core_keep_alive_post_hdlr(void *drv_priv, struct rtw_keep_alive_param *klive)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+	_adapter *iface;
+	int i;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !rtw_is_adapter_up(iface))
+			continue;
+
+#ifdef CONFIG_AP_MODE
+		if (MLME_IS_AP(iface) && klive->ap_chk_sta_bmp[i])
+			expire_timeout_post_chk(iface, klive->ap_chk_sta_bmp[i]);
+		else
+#endif
+		if (is_client_associated_to_ap(iface) &&
+		    (BIT(i) & klive->sta_chk_rx_bmp || BIT(i) & klive->sta_chk_tx_bmp))
+			linked_status_post_chk(iface,
+					       klive->sta_chk_rx_bmp,
+					       klive->sta_chk_tx_bmp);
+	}
+
+	rtw_phl_set_wdog_state_keep_alive(dvobj->phl, false, NULL);
+}
+#endif
+
+inline int rtw_get_rx_chk_limit(_adapter *adapter)
+{
+	return adapter->stapriv.rx_chk_limit;
+}
+
+inline void rtw_set_rx_chk_limit(_adapter *adapter, int limit)
+{
+	adapter->stapriv.rx_chk_limit = limit;
+}
+
+inline int rtw_get_link_count_limit(_adapter *adapter)
+{
+	bool is_p2p_gc = _FALSE;
+	int link_count_limit;
+
+#ifdef CONFIG_P2P
+		is_p2p_gc = rtw_p2p_chk_role(&adapter->wdinfo, P2P_ROLE_CLIENT);
+		if (is_p2p_gc)
+			link_count_limit = 3; /* 8 sec */
+		else
+#endif /* CONFIG_P2P */
+			link_count_limit = 7; /* 16 sec */
+		return link_count_limit;
+}
+
+void linked_status_post_chk(_adapter *padapter, u8 if_sta_chk_rx_bmp, u8 if_sta_chk_tx_bmp)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sta_info		*psta;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	bool is_p2p_gc = _FALSE;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info *lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+	int rx_chk_limit;
+	int link_count_limit;
+	bool rx_chk = _SUCCESS;
+	bool tx_chk = _SUCCESS;
+
+	if (padapter->registrypriv.mp_mode == _TRUE)
+		return;
+
+	if (check_fwstate(pmlmepriv, WIFI_CSA_UPDATE_BEACON))
+		return;
+
+#ifdef CONFIG_P2P
+	is_p2p_gc = rtw_p2p_chk_role(&padapter->wdinfo, P2P_ROLE_CLIENT);
+#endif
+
+	if (if_sta_chk_rx_bmp & BIT(padapter->iface_id))
+		rx_chk = _FAIL;
+	if (if_sta_chk_tx_bmp & BIT(padapter->iface_id))
+		tx_chk = _FAIL;
+
+	rx_chk_limit = rtw_get_rx_chk_limit(padapter);
+	link_count_limit = rtw_get_link_count_limit(padapter);
+	psta = rtw_get_stainfo(pstapriv, lmlmeinfo->network.MacAddress);
+	if (psta == NULL)
+		goto exit;
+#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+	if (pmlmeext->active_keep_alive_check && (rx_chk == _FAIL || tx_chk == _FAIL)) {
+		u8 switch_channel_by_drv = _TRUE;
+		struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+		struct rtw_chan_def hw_chdef = {0};
+		struct rtw_chan_def union_chdef = {0};
+		struct rtw_chan_def bk_chdef = {0};
+
+		if (switch_channel_by_drv) {
+			if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role,
+						padapter_link->wrlink, &union_chdef)
+							!= RTW_PHL_STATUS_SUCCESS) {
+				RTW_ERR("fun:%s line:%d get chandef failed\n", __FUNCTION__, __LINE__);
+				rtw_warn_on(1);
+			}
+			if (padapter_link->mlmeextpriv.chandef.chan != union_chdef.chan)
+				goto bypass_active_keep_alive;
+
+			/* switch to correct channel of current network  before issue keep-alive frames */
+			rtw_get_oper_chdef(padapter, padapter_link, &hw_chdef);
+			if (hw_chdef.chan != padapter_link->mlmeextpriv.chandef.chan) {
+				_rtw_memcpy(&bk_chdef, &hw_chdef, sizeof(struct rtw_chan_def));
+				set_bch_bwmode(padapter, padapter_link, union_chdef.band,
+						 union_chdef.chan, union_chdef.offset,
+						 union_chdef.bw, RFK_TYPE_FORCE_NOT_DO);
+			}
+		}
+
+		if (rx_chk != _SUCCESS)
+			issue_probereq_ex(padapter, padapter_link, &lmlmeinfo->network.Ssid, psta->phl_sta->mac_addr, 0, 0, 3, 1);
+
+		if (tx_chk != _SUCCESS || rx_chk != _SUCCESS) {
+			if (rtw_mi_check_fwstate(padapter, WIFI_UNDER_SURVEY))
+				tx_chk = issue_nulldata(padapter, padapter_link, psta->phl_sta->mac_addr, 1, 3, 1);
+			else
+				tx_chk = issue_nulldata(padapter, padapter_link, psta->phl_sta->mac_addr, 0, 3, 1);
+			/* If tx acked, set rx_chk _SUCCESS to reset the retry count
+			 * unless it is a P2P GC. If the P2P GO is no longer a P2P GO,
+			 * P2P GC tx may still receive acks, but no beacon.
+			 */
+			if (tx_chk == _SUCCESS && !is_p2p_gc)
+				rx_chk = _SUCCESS;
+		}
+
+		/* back to the original operation channel */
+		if (bk_chdef.chan > 0 && switch_channel_by_drv)
+			set_bch_bwmode(padapter, padapter_link,
+					bk_chdef.band, bk_chdef.chan, bk_chdef.offset, bk_chdef.bw, RFK_TYPE_FORCE_NOT_DO);
+
+bypass_active_keep_alive:
+		;
+	} else
+#endif /* CONFIG_ACTIVE_KEEP_ALIVE_CHECK */
+	{
+		if (rx_chk != _SUCCESS) {
+			if (pmlmeext->retry == 1) {
+#ifdef DBG_EXPIRATION_CHK
+				RTW_INFO("issue_probereq to trigger probersp, retry=%d\n", pmlmeext->retry);
+#endif
+				issue_probereq_ex(padapter, padapter_link, &lmlmeinfo->network.Ssid,
+					lmlmeinfo->network.MacAddress, 0, 0, 0, 1);
+				issue_probereq_ex(padapter, padapter_link, &lmlmeinfo->network.Ssid,
+					lmlmeinfo->network.MacAddress, 0, 0, 0, 1);
+				issue_probereq_ex(padapter, padapter_link, &lmlmeinfo->network.Ssid,
+					lmlmeinfo->network.MacAddress, 0, 0, 0, 1);
+			}
+		}
+
+		if (tx_chk != _SUCCESS) {
+			#ifdef DBG_EXPIRATION_CHK
+			RTW_INFO("%s issue_nulldata\n", __FUNCTION__);
+			#endif
+			if (rtw_mi_check_fwstate(padapter, WIFI_UNDER_SURVEY))
+				tx_chk = issue_nulldata(padapter, padapter_link, NULL, 1, 0, 0);
+			else
+				tx_chk = issue_nulldata(padapter, padapter_link, NULL, 0, 1, 1);
+		}
+	}
+
+	if (rx_chk == _FAIL) {
+		if (pmlmeext->retry > rx_chk_limit) {
+			RTW_PRINT(FUNC_ADPT_FMT" disconnect or roaming\n",
+				FUNC_ADPT_ARG(padapter));
+			receive_disconnect(padapter,
+				lmlmeinfo->network.MacAddress,
+				WLAN_REASON_EXPIRATION_CHK, _FALSE);
+			pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_KEEPALIVE_TIMEOUT;
+			pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+			return;
+		}
+	} else {
+		pmlmeext->retry = 0;
+	}
+
+	if (tx_chk == _SUCCESS) {
+		psta->sta_stats.last_tx_pkts = psta->sta_stats.tx_pkts;
+		pmlmeinfo->link_count = 0;
+	}
+
+exit:
+	return;
+}
+
+void linked_status_chk(_adapter *padapter, u8 *if_sta_chk_rx_bmp, u8 *if_sta_chk_tx_bmp)
+{
+	u32	i;
+	struct sta_info		*psta;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info *lmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+	int link_count_limit;
+	/* ToDo CONFIG_RTW_MLD: check retry per link or per MLD? */
+
+#ifdef CONFIG_RTW_WDS
+	rtw_wds_gptr_expire(padapter);
+#endif
+
+	if (padapter->registrypriv.mp_mode == _TRUE)
+		return;
+
+	if (check_fwstate(pmlmepriv, WIFI_CSA_UPDATE_BEACON))
+		return;
+
+	if (is_client_associated_to_ap(padapter)) {
+		/* linked infrastructure client mode */
+
+		bool tx_chk = _SUCCESS, rx_chk = _SUCCESS;
+		int link_count_limit;
+
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+		if (pmlmepriv->bGetGateway == 1 && pmlmepriv->GetGatewayTryCnt < 3) {
+			RTW_INFO("do rtw_gw_addr_query() : %d\n", pmlmepriv->GetGatewayTryCnt);
+			pmlmepriv->GetGatewayTryCnt++;
+			if (rtw_gw_addr_query(padapter) == 0)
+				pmlmepriv->bGetGateway = 0;
+			else {
+				_rtw_memset(pmlmepriv->gw_ip, 0, 4);
+				_rtw_memset(pmlmepriv->gw_mac_addr, 0, ETH_ALEN);
+			}
+		}
+#endif
+
+		link_count_limit = rtw_get_link_count_limit(padapter);
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+		if (ATOMIC_READ(&padapter->tdlsinfo.chsw_info.chsw_on) == _TRUE)
+			return;
+#endif /* CONFIG_TDLS_CH_SW */
+
+#ifdef CONFIG_TDLS_AUTOCHECKALIVE
+		linked_status_chk_tdls(padapter);
+#endif /* CONFIG_TDLS_AUTOCHECKALIVE */
+#endif /* CONFIG_TDLS */
+
+		psta = rtw_get_stainfo(pstapriv, lmlmeinfo->network.MacAddress);
+		if (psta != NULL) {
+#ifdef CONFIG_ISSUE_DELBA_WHEN_NO_TRAFFIC
+			/*issue delba when ap does not tx data packet that is Broadcom ap */
+			rtw_delba_check(padapter, psta);
+#endif
+			if (chk_ap_is_alive(padapter, psta) == _FALSE) {
+				rx_chk = _FAIL;
+				pmlmeext->retry++;
+				*if_sta_chk_rx_bmp |= BIT(padapter->iface_id);
+			}
+
+			if (sta_last_tx_pkts(psta) == sta_tx_pkts(psta)) {
+				tx_chk = _FAIL;
+				if (pmlmeinfo->link_count++ == link_count_limit)
+					*if_sta_chk_tx_bmp |= BIT(padapter->iface_id);
+				pmlmeinfo->link_count %= (link_count_limit + 1);
+			}
+#ifdef CONFIG_POST_CORE_KEEP_ALIVE
+			if (tx_chk != _FAIL && rx_chk != _FAIL) {
+				pmlmeext->retry = 0;
+				psta->sta_stats.last_tx_pkts = psta->sta_stats.tx_pkts;
+				pmlmeinfo->link_count = 0;
+			}
+		}
+#else
+			linked_status_post_chk(padapter, *if_sta_chk_rx_bmp, *if_sta_chk_tx_bmp);
+		}
+#endif
+	} else if (is_client_associated_to_ibss(padapter)) {
+		_list *phead, *plist, dlist;
+
+		_rtw_init_listhead(&dlist);
+
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+		for (i = 0; i < NUM_STA; i++) {
+
+			phead = &(pstapriv->sta_hash[i]);
+			plist = get_next(phead);
+			while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+				psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+				plist = get_next(plist);
+
+				if (is_broadcast_mac_addr(psta->phl_sta->mac_addr))
+					continue;
+
+				if (chk_adhoc_peer_is_alive(psta) || !psta->expire_to)
+					psta->expire_to = pstapriv->adhoc_expire_to;
+				else
+					psta->expire_to--;
+
+				if (psta->expire_to <= 0) {
+					rtw_list_delete(&psta->list);
+					rtw_list_insert_tail(&psta->list, &dlist);
+				}
+			}
+		}
+
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+		plist = get_next(&dlist);
+		while (rtw_end_of_queue_search(&dlist, plist) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, list);
+			plist = get_next(plist);
+			rtw_list_delete(&psta->list);
+			RTW_INFO(FUNC_ADPT_FMT" ibss expire "MAC_FMT"\n"
+				, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr));
+			report_del_sta_event(padapter, psta->phl_sta->mac_addr, WLAN_REASON_EXPIRATION_CHK, _FALSE, _FALSE);
+		}
+	}
+
+}
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+void rtw_tbtx_xmit_timer_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+
+	if (ATOMIC_READ(&padapter->tbtx_remove_tx_pause) == _TRUE){
+		ATOMIC_SET(&padapter->tbtx_tx_pause, _FALSE);
+		rtw_tx_control_cmd(padapter);
+	}else {
+		rtw_issue_action_token_rel(padapter);
+		ATOMIC_SET(&padapter->tbtx_tx_pause, _TRUE);
+		rtw_tx_control_cmd(padapter);
+		_set_timer(&pmlmeext->tbtx_xmit_timer, MAX_TXPAUSE_DURATION);
+		ATOMIC_SET(&padapter->tbtx_remove_tx_pause, _TRUE);
+	}
+}
+
+#ifdef CONFIG_AP_MODE
+void rtw_tbtx_token_dispatch_timer_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	_list *phead, *plist;
+	int i, found = _FALSE;
+	u8 nr_send, th_idx = 0;
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	RTW_DBG("%s:asoc_cnt: %d\n",__func__, pstapriv->tbtx_asoc_list_cnt);
+
+	// check number of TBTX sta
+	if (padapter->stapriv.tbtx_asoc_list_cnt < 2)
+		goto exit;
+
+	// dispatch token
+	
+	nr_send = RTW_DIV_ROUND_UP(pstapriv->tbtx_asoc_list_cnt, NR_TBTX_SLOT);
+
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		/* psta is supporting TBTX */
+		if ((!psta) || (!psta->tbtx_enable))
+			RTW_DBG("sta tbtx_enable is false\n");
+		else {
+			for (i = 0; i < nr_send; i++) {
+				if (pstapriv->last_token_holder == psta) {
+					found = _TRUE;
+					goto outof_loop;
+				}
+			}
+		}
+		plist = get_next(plist);
+	}
+outof_loop:
+	
+	RTW_DBG("rtw_tbtx_token_dispatch_timer_hdl()   th_idx=%d,  nr_send=%d, phead=%p, plist=%p, found=%d\n ", th_idx ,  nr_send, phead, plist, found);
+	if (!found) {
+		plist = get_next(phead);
+		while(rtw_end_of_queue_search(phead, plist) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+			if ((!psta) || (!psta->tbtx_enable))
+				RTW_DBG("sta tbtx_enable is false\n");
+			else {
+					pstapriv->token_holder[th_idx] = psta;
+					rtw_issue_action_token_req(padapter, pstapriv->token_holder[th_idx++]);
+					break;
+			}
+			plist = get_next(plist);
+		}
+	}
+
+	for (i=th_idx; i<nr_send;) {
+		plist = get_next(plist);
+		if (plist == phead)
+			plist = get_next(plist);
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		if ((!psta) || (!psta->tbtx_enable))
+			RTW_DBG("sta tbtx_enable is false\n");
+		else {
+			pstapriv->token_holder[th_idx] = psta;
+			rtw_issue_action_token_req(padapter, pstapriv->token_holder[th_idx++]);
+			i++;
+		}
+	}
+	ATOMIC_SET(&pstapriv->nr_token_keeper, nr_send);
+	
+
+exit:
+	// set_timer
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+	_set_timer(&pmlmeext->tbtx_token_dispatch_timer, TBTX_TX_DURATION); 
+}
+#endif /* CONFIG_AP_MODE */
+#endif /* CONFIG_RTW_TOKEN_BASED_XMIT */
+
+void link_timer_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	/* static unsigned int		rx_pkt = 0; */
+	/* static u64				tx_cnt = 0; */
+	/* struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv); */
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	/* struct sta_priv		*pstapriv = &padapter->stapriv; */
+#ifdef CONFIG_RTW_80211R
+	struct	sta_priv		*pstapriv = &padapter->stapriv;
+	struct	sta_info		*psta = NULL;
+#endif
+
+	if (rtw_sta_linking_test_force_fail())
+		RTW_INFO("rtw_sta_linking_test_force_fail\n");
+
+	if (pmlmeext->join_abort) {
+		RTW_INFO(FUNC_ADPT_FMT" join abort\n", FUNC_ADPT_ARG(padapter));
+		pmlmeinfo->state = WIFI_FW_NULL_STATE;
+		report_join_res(padapter, -4, WLAN_STATUS_UNSPECIFIED_FAILURE);
+		goto exit;
+	}
+
+	if (pmlmeinfo->state & WIFI_FW_AUTH_NULL) {
+		RTW_INFO("link_timer_hdl:no beacon while connecting\n");
+		pmlmeinfo->state = WIFI_FW_NULL_STATE;
+		report_join_res(padapter, -3, WLAN_STATUS_UNSPECIFIED_FAILURE);
+	} else if (pmlmeinfo->state & WIFI_FW_AUTH_STATE) {
+
+#ifdef CONFIG_IOCTL_CFG80211
+		if (rtw_sec_chk_auth_type(padapter, MLME_AUTHTYPE_SAE)) {
+			RTW_INFO("link_timer_hdl: SAE auth timeout.\n");
+			if (pmlmeinfo->auth_status) {
+				report_join_res(padapter, -1, pmlmeinfo->auth_status);
+				pmlmeinfo->auth_status = 0; /* reset */
+			} else {
+				report_join_res(padapter, -1, WLAN_STATUS_AUTH_TIMEOUT);
+			}
+			return;
+		}
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+		/* re-auth timer */
+		if (++pmlmeinfo->reauth_count > REAUTH_LIMIT) {
+			/* if (pmlmeinfo->auth_algo != dot11AuthAlgrthm_Auto) */
+			/* { */
+			pmlmeinfo->state = 0;
+			if (pmlmeinfo->auth_status) {
+				report_join_res(padapter, -1, pmlmeinfo->auth_status);
+				pmlmeinfo->auth_status = 0; /* reset */
+			} else
+				report_join_res(padapter, -1, WLAN_STATUS_UNSPECIFIED_FAILURE);
+			return;
+			/* } */
+			/* else */
+			/* { */
+			/*	pmlmeinfo->auth_algo = dot11AuthAlgrthm_Shared; */
+			/*	pmlmeinfo->reauth_count = 0; */
+			/* } */
+		}
+
+		RTW_INFO("link_timer_hdl: auth timeout and try again\n");
+		pmlmeinfo->auth_seq = 1;
+		issue_auth(padapter, NULL, 0);
+		set_link_timer(pmlmeext, REAUTH_TO);
+	} else if (pmlmeinfo->state & WIFI_FW_ASSOC_STATE) {
+		/* re-assoc timer */
+		if (++pmlmeinfo->reassoc_count > REASSOC_LIMIT) {
+			pmlmeinfo->state = WIFI_FW_NULL_STATE;
+#ifdef CONFIG_RTW_80211R
+			if (rtw_ft_roam(padapter)) {
+				psta = rtw_get_stainfo(pstapriv, pmlmeinfo->dev_network.MacAddress);
+				if (psta)
+					rtw_free_mld_stainfo(padapter, psta->phl_sta->mld);
+			}
+#endif
+			report_join_res(padapter, -2, WLAN_STATUS_UNSPECIFIED_FAILURE);
+			return;
+		}
+
+#ifdef CONFIG_RTW_80211R
+		if (rtw_ft_roam(padapter)) {
+			RTW_INFO("link_timer_hdl: reassoc timeout and try again\n");
+			issue_reassocreq(padapter);
+		} else
+#endif
+		{
+			RTW_INFO("link_timer_hdl: assoc timeout and try again\n");
+			issue_assocreq(padapter);
+		}
+
+		set_link_timer(pmlmeext, REASSOC_TO);
+	}
+
+exit:
+	return;
+}
+
+void addba_timer_hdl(void *ctx)
+{
+	struct sta_info *psta = (struct sta_info *)ctx;
+
+#ifdef CONFIG_80211N_HT
+	struct ht_priv	*phtpriv;
+	struct ampdu_priv *ampdu_priv;
+
+	if (!psta)
+		return;
+
+	phtpriv = &psta->htpriv;
+	ampdu_priv = &psta->ampdu_priv;
+
+	if ((phtpriv->ht_option == _TRUE
+#if CONFIG_IEEE80211_BAND_6GHZ
+	    || psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+	    ) && ampdu_priv->ampdu_enable == _TRUE) {
+		if (ampdu_priv->candidate_tid_bitmap)
+			ampdu_priv->candidate_tid_bitmap = 0x0;
+
+	}
+#endif /* CONFIG_80211N_HT */
+}
+
+#ifdef CONFIG_IEEE80211W
+void report_sta_timeout_event(_adapter *padapter, u8 *MacAddr, unsigned short reason)
+{
+	struct cmd_obj *pcmd_obj;
+	u8 *pevtcmd;
+	u32 cmdsz;
+	struct sta_info *psta;
+	int	mac_id;
+	struct stadel_event *pdel_sta_evt;
+	struct rtw_evt_header *evt_hdr;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+	pcmd_obj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd_obj == NULL)
+		return;
+	pcmd_obj->padapter = padapter;
+
+	cmdsz = (sizeof(struct stadel_event) + sizeof(struct rtw_evt_header));
+	pevtcmd = (u8 *)rtw_zmalloc(cmdsz);
+	if (pevtcmd == NULL) {
+		rtw_mfree((u8 *)pcmd_obj, sizeof(struct cmd_obj));
+		return;
+	}
+
+	_rtw_init_listhead(&pcmd_obj->list);
+
+	CMD_OBJ_SET_HWBAND(pcmd_obj, HW_BAND_MAX);
+	pcmd_obj->cmdcode = CMD_SET_MLME_EVT;
+	pcmd_obj->cmdsz = cmdsz;
+	pcmd_obj->parmbuf = pevtcmd;
+
+	pcmd_obj->rsp = NULL;
+	pcmd_obj->rspsz  = 0;
+
+	evt_hdr = (struct rtw_evt_header *)(pevtcmd);
+	evt_hdr->len = sizeof(struct stadel_event);
+	evt_hdr->id = EVT_TIMEOUT_STA;
+	evt_hdr->seq = ATOMIC_INC_RETURN(&pmlmeext->event_seq);
+
+	pdel_sta_evt = (struct stadel_event *)(pevtcmd + sizeof(struct rtw_evt_header));
+	_rtw_memcpy((unsigned char *)(&(pdel_sta_evt->macaddr)), MacAddr, ETH_ALEN);
+	_rtw_memcpy((unsigned char *)(pdel_sta_evt->rsvd), (unsigned char *)(&reason), 2);
+
+
+	psta = rtw_get_stainfo_to_free(&padapter->stapriv, MacAddr);
+	if (psta)
+		mac_id = (int)psta->phl_sta->macid;
+	else
+		mac_id = (-1);
+
+	pdel_sta_evt->mac_id = mac_id;
+
+	RTW_INFO("report_del_sta_event: delete STA, mac_id=%d\n", mac_id);
+
+	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
+
+	return;
+}
+
+void clnt_sa_query_timeout(_adapter *padapter)
+{
+	struct mlme_ext_priv *mlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *mlmeinfo = &(mlmeext->mlmext_info);
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+	receive_disconnect(padapter, get_my_bssid(&(mlmeinfo->dev_network)), WLAN_REASON_SA_QUERY_TIMEOUT, _FALSE);
+
+	mlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+	mlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_SA_QUERY_TIMEOUT;
+	mlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+}
+
+void sa_query_timer_hdl(void *ctx)
+{
+	struct sta_info *psta = (struct sta_info *)ctx;
+	_adapter *padapter = psta->padapter;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+	if (MLME_IS_STA(padapter) &&
+	    check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		clnt_sa_query_timeout(padapter);
+	else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE)
+		report_sta_timeout_event(padapter, psta->phl_sta->mac_addr, WLAN_REASON_PREV_AUTH_NOT_VALID);
+}
+
+#endif /* CONFIG_IEEE80211W */
+
+#ifdef PRIVATE_R
+void find_remote_timer_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	
+	rtw_find_remote_wk_cmd(padapter);
+	set_find_remote_timer(pmlmeext, 1);
+}
+
+#ifdef CONFIG_P2P
+void hide_ssid_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	rtw_set_bit(MIRACAST, &pregpriv->go_hidden_ssid_mode);
+	ATOMIC_SET(&pregpriv->set_hide_ssid_timer, 0);
+
+	/* check if there is other feature show SSID in beacon */
+	if (pregpriv->go_hidden_ssid_mode == ALL_HIDE_SSID)
+		rtw_hide_ssid_wk_cmd(padapter);
+}
+#endif
+#endif
+
+#ifdef CONFIG_ECSA_PHL
+void csa_wait_bcn_timer_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	_adapter *iface;
+	u8 i;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface)
+			continue;
+		/* Wait beacon timeout, so clear fwstate WIFI_CSA_UPDATE_BEACON */
+		if (check_fwstate(&(iface->mlmepriv), WIFI_CSA_UPDATE_BEACON)) {
+			clr_fwstate(&(iface->mlmepriv), WIFI_CSA_UPDATE_BEACON);
+			RTW_INFO(FUNC_ADPT_FMT" wait beacon timeout, clear state "
+				"WIFI_CSA_UPDATE_BEACON\n", FUNC_ADPT_ARG(iface));
+			return;
+		}
+
+		clr_fwstate(&(iface->mlmepriv), WIFI_CSA_SKIP_CHECK_BEACON);
+	}
+}
+#endif /* CONFIG_ECSA_PHL */
+
+#ifdef CONFIG_AUTO_AP_MODE
+void rtw_auto_ap_rx_msg_dump(_adapter *padapter, union recv_frame *precv_frame, u8 *ehdr_pos)
+{
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_info *psta = precv_frame->u.hdr.psta;
+	struct ethhdr *ehdr = (struct ethhdr *)ehdr_pos;
+
+	RTW_INFO("eth rx: got eth_type=0x%x\n", ntohs(ehdr->h_proto));
+
+	if (psta && psta->isrc && psta->pid > 0) {
+		u16 rx_pid;
+
+		rx_pid = *(u16 *)(ehdr_pos + ETH_HLEN);
+
+		RTW_INFO("eth rx(pid=0x%x): sta("MAC_FMT") pid=0x%x\n",
+			 rx_pid, MAC_ARG(psta->phl_sta->mac_addr), psta->pid);
+
+		if (rx_pid == psta->pid) {
+			int i;
+			u16 len = *(u16 *)(ehdr_pos + ETH_HLEN + 2);
+			/* u16 ctrl_type = *(u16 *)(ehdr_pos + ETH_HLEN + 4); */
+
+			/* RTW_INFO("eth, RC: len=0x%x, ctrl_type=0x%x\n", len, ctrl_type);  */
+			RTW_INFO("eth, RC: len=0x%x\n", len);
+
+			for (i = 0; i < len; i++)
+				RTW_INFO("0x%x\n", *(ehdr_pos + ETH_HLEN + 4 + i));
+			/* RTW_INFO("0x%x\n", *(ehdr_pos + ETH_HLEN + 6 + i)); */
+
+			RTW_INFO("eth, RC-end\n");
+		}
+	}
+
+}
+
+void rtw_start_auto_ap(_adapter *adapter)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	rtw_set_802_11_infrastructure_mode(adapter, Ndis802_11APMode, 0);
+
+	rtw_setopmode_cmd(adapter, Ndis802_11APMode, RTW_CMDF_WAIT_ACK);
+}
+
+static int rtw_auto_ap_start_beacon(_adapter *adapter)
+{
+	int ret = 0;
+	u8 *pbuf = NULL;
+	uint len;
+	u8	supportRate[16];
+	int	sz = 0, rateLen;
+	u8	*ie;
+	u8	wireless_mode, oper_channel;
+	enum band_type band oper_band;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rtw_chan_def u_chdef = {0};
+	u8 ssid[3] = {0}; /* hidden ssid */
+	u32 ssid_len = sizeof(ssid);
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adpater);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+
+	len = 128;
+	pbuf = rtw_zmalloc(len);
+	if (!pbuf)
+		return -ENOMEM;
+
+
+	/* generate beacon */
+	ie = pbuf;
+
+	/* timestamp will be inserted by hardware */
+	sz += 8;
+	ie += sz;
+
+	/* beacon interval : 2bytes */
+	*(u16 *)ie = cpu_to_le16((u16)100); /* BCN_INTERVAL=100; */
+	sz += 2;
+	ie += 2;
+
+	/* capability info */
+	*(u16 *)ie = 0;
+	*(u16 *)ie |= cpu_to_le16(cap_ESS);
+	*(u16 *)ie |= cpu_to_le16(cap_ShortPremble);
+	/* *(u16*)ie |= cpu_to_le16(cap_Privacy); */
+	sz += 2;
+	ie += 2;
+
+	/* SSID */
+	ie = rtw_set_ie(ie, _SSID_IE_, ssid_len, ssid, &sz);
+
+	/* Get OP ch */
+	if (rtw_mi_check_status(adapter, MI_LINKED)) {
+		if (rtw_phl_mr_get_chandef(dvobj->phl, adapter->phl_role,
+							adapter_link->wrlink, &u_chdef)
+								!= RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s get union chandef failed\n", __func__);
+			rtw_warn_on(1);
+		}
+		oper_channel = u_chdef.chan;
+		oper_band = u_chdef.band;
+	}
+	else {
+		oper_channel = adapter_link->mlmeextpriv.chandef.chan;
+		oper_band = adapter_link->mlmeextpriv.chandef.band;
+	}
+
+	/* supported rates */
+	wireless_mode = (WIRELESS_11BG_24N & adapter->registrypriv.wireless_mode);
+	rtw_set_supported_rate(supportRate, wireless_mode, oper_channel, oper_band);
+	rateLen = rtw_get_rateset_len(supportRate);
+	if (rateLen > 8)
+		ie = rtw_set_ie(ie, _SUPPORTEDRATES_IE_, 8, supportRate, &sz);
+	else
+		ie = rtw_set_ie(ie, _SUPPORTEDRATES_IE_, rateLen, supportRate, &sz);
+
+	/* DS parameter set */
+	ie = rtw_set_ie(ie, _DSSET_IE_, 1, &oper_channel, &sz);
+
+	/* ext supported rates */
+	if (rateLen > 8)
+		ie = rtw_set_ie(ie, _EXT_SUPPORTEDRATES_IE_, (rateLen - 8), (supportRate + 8), &sz);
+
+	RTW_INFO("%s, start auto ap beacon sz=%d\n", __FUNCTION__, sz);
+
+	/* lunch ap mode & start to issue beacon */
+	if (rtw_check_beacon_data(adapter, pbuf,  sz) == _SUCCESS) {
+
+	} else
+		ret = -EINVAL;
+
+
+	rtw_mfree(pbuf, len);
+
+	return ret;
+
+}
+#endif/* CONFIG_AUTO_AP_MODE */
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+u8 tx_control_hdl(_adapter *adapter)
+{
+	u8 val;
+
+	if(ATOMIC_READ(&adapter->tbtx_tx_pause))
+		val = 0xff;
+	else
+		val = 0x00;
+
+	rtw_hal_set_hwreg(adapter, HW_VAR_TXPAUSE, &val);
+
+	return H2C_SUCCESS;
+}
+#endif
+void rtw_disconnect_ch_switch(_adapter *adapter)
+{
+	void *phl = adapter_to_dvobj(adapter)->phl;
+	int chanctx_num = 0;
+	struct rtw_chan_def chan_def = {0};
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	if (!MLME_IS_OPCH_SW(adapter)) {
+		/* channel status no need to check here for iface under OPCH_SW */
+
+		chanctx_num = rtw_phl_chanctx_del(phl, adapter->phl_role, adapter_link->wrlink, &chan_def);
+		RTW_INFO("PHL - CH:%d, BW:%d OFF:%d\n", chan_def.chan, chan_def.bw, chan_def.offset);
+
+		if (chanctx_num && chan_def.chan != 0)
+			set_bch_bwmode(adapter, adapter_link,
+					chan_def.band, chan_def.chan, chan_def.offset, chan_def.bw, RFK_TYPE_FORCE_NOT_DO);
+
+		if (MLME_IS_STA(adapter)) {
+			#ifdef CONFIG_80211D
+			rtw_alink_leavebss_update_regulatory(adapter_link);
+			#endif
+
+			#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+			rtw_dfs_rd_en_dec_on_mlme_act(adapter, adapter_link, MLME_STA_DISCONNECTED, 0);
+			#endif
+		}
+
+		rtw_rfctl_update_op_mode(adapter_to_rfctl(adapter), BIT(adapter->iface_id), 0, 0);
+	}
+}
+
+#ifdef CONFIG_AP_MODE
+u8 stop_ap_hdl(_adapter *adapter)
+{
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+#ifdef CONFIG_DFS_MASTER
+	bool hb_to_stop[HW_BAND_MAX];
+	int i;
+#endif
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+
+#ifdef CONFIG_DFS_MASTER
+	for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+		hb_to_stop[i] = rtw_iface_is_operate_at_hwband(adapter, i) ? true : false;
+#endif
+
+	rtw_ap_link_regu_forbid_apply(adapter_link, false, false);
+
+	rtw_set_802_11_infrastructure_mode(adapter, Ndis802_11Infrastructure, RTW_CMDF_DIRECTLY);
+	rtw_setopmode_cmd(adapter, Ndis802_11Infrastructure, RTW_CMDF_DIRECTLY);
+
+#ifdef CONFIG_DFS_MASTER
+	for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+		if (hb_to_stop[i])
+			rtw_dfs_rd_en_dec_update(adapter_to_dvobj(adapter), i);
+#endif
+
+	rtw_free_bcn_entry(adapter);
+
+	rtw_disconnect_ch_switch(adapter);
+	adapter_link->is_ap_chan_ctx_added = _FALSE;
+	return H2C_SUCCESS;
+}
+#endif
+
+u8 setopmode_hdl(_adapter *padapter, u8 *pbuf)
+{
+	u8	type;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct setopmode_parm *psetop = (struct setopmode_parm *)pbuf;
+
+	if (psetop->mode == Ndis802_11APMode
+		|| psetop->mode == Ndis802_11_mesh
+	) {
+		pmlmeinfo->state = WIFI_FW_AP_STATE;
+		type = _HW_STATE_AP_;
+	} else if (psetop->mode == Ndis802_11Infrastructure) {
+		pmlmeinfo->state &= ~(BIT(0) | BIT(1)); /* clear state */
+		pmlmeinfo->state |= WIFI_FW_STATION_STATE;/* set to 	STATION_STATE */
+		type = _HW_STATE_STATION_;
+	} else if (psetop->mode == Ndis802_11IBSS)
+		type = _HW_STATE_ADHOC_;
+	else if (psetop->mode == Ndis802_11Monitor)
+		type = _HW_STATE_MONITOR_;
+	else
+		type = _HW_STATE_NOLINK_;
+
+	if (_FAIL == rtw_hw_iface_type_change(padapter, type)) {
+		RTW_ERR("%s - change iface type fails !\n", __func__);
+		return H2C_CMD_FAIL;
+	}
+
+#ifdef CONFIG_AP_PORT_SWAP
+	rtw_hal_set_hwreg(padapter, HW_VAR_PORT_SWITCH, (u8 *)(&type));
+#endif
+
+	rtw_hal_set_hwreg(padapter, HW_VAR_SET_OPMODE, (u8 *)(&type));
+
+#ifdef CONFIG_AUTO_AP_MODE
+	if (psetop->mode == Ndis802_11APMode)
+		rtw_auto_ap_start_beacon(padapter);
+#endif
+
+	return H2C_SUCCESS;
+
+}
+
+u8 createbss_hdl(_adapter *padapter, u8 *pbuf)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*lmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*lmlmeinfo = &(lmlmeext->mlmext_info);
+	WLAN_BSSID_EX	*lnetwork = (WLAN_BSSID_EX *)(&(lmlmeinfo->network));
+	WLAN_BSSID_EX	*pdev_network = &padapter->registrypriv.dev_network;
+	struct createbss_parm *parm = (struct createbss_parm *)pbuf;
+	u8 ret = H2C_SUCCESS;
+	/* u8	initialgain; */
+
+#ifdef CONFIG_AP_MODE
+	if ((parm->req_ch == 0 && pmlmeinfo->state == WIFI_FW_AP_STATE)
+		|| parm->req_ch != 0
+	) {
+		rtw_core_ap_prepare(padapter, parm);
+		rtw_core_ap_swch_start(padapter, parm);
+		rtw_core_ap_start(padapter, parm);
+		goto exit;
+	}
+#endif
+
+	/* below is for ad-hoc master */
+	if (parm->adhoc) {
+		rtw_warn_on(pdev_network->InfrastructureMode != Ndis802_11IBSS);
+		rtw_joinbss_reset(padapter);
+
+		lmlmeext->chandef.bw = CHANNEL_WIDTH_20;
+		lmlmeext->chandef.offset = CHAN_OFFSET_NO_EXT;
+		lmlmeinfo->ERP_enable = 0;
+		lmlmeinfo->WMM_enable = 0;
+		lmlmeinfo->HT_enable = 0;
+		lmlmeinfo->HT_caps_enable = 0;
+		lmlmeinfo->HT_info_enable = 0;
+		pmlmeinfo->agg_enable_bitmap = 0;
+		pmlmeinfo->candidate_tid_bitmap = 0;
+
+		/* cancel link timer */
+		cancel_link_timer(pmlmeext); /*_cancel_timer_ex(&pmlmeext->link_timer);*/
+
+		/* clear CAM */
+		flush_all_cam_entry(padapter, PHL_CMD_DIRECTLY, 0);
+
+		pdev_network->Length = get_WLAN_BSSID_EX_sz(pdev_network);
+		_rtw_memcpy(lnetwork, pdev_network, FIELD_OFFSET(WLAN_BSSID_EX, IELength));
+		lnetwork->IELength = pdev_network->IELength;
+
+		if (lnetwork->IELength > MAX_IE_SZ) {
+			ret = H2C_PARAMETERS_ERROR;
+			goto ibss_post_hdl;
+		}
+
+		_rtw_memcpy(lnetwork->IEs, pdev_network->IEs, lnetwork->IELength);
+		start_create_ibss(padapter);
+	} else {
+		rtw_warn_on(1);
+		ret = H2C_PARAMETERS_ERROR;
+	}
+
+ibss_post_hdl:
+	rtw_create_ibss_post_hdl(padapter, ret);
+
+exit:
+	return ret;
+}
+
+static void set_hw_before_join(struct _ADAPTER *a)
+{
+	struct mlme_ext_priv *pmlmeext = &a->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct _WLAN_BSSID_EX *pnetwork = &pmlmeinfo->dev_network;
+	struct sta_info *sta;
+
+	/* check already connecting to AP or not */
+	if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
+		if (pmlmeinfo->state & WIFI_FW_STATION_STATE)
+			issue_deauth_ex(a, pnetwork->MacAddress,
+					WLAN_REASON_DEAUTH_LEAVING, 1, 100);
+		pmlmeinfo->state = WIFI_FW_NULL_STATE;
+		cancel_link_timer(pmlmeext); /*_cancel_timer_ex(&pmlmeext->link_timer);*/
+
+		sta = rtw_get_stainfo(&a->stapriv, pnetwork->MacAddress);
+		if (sta)
+			rtw_hw_disconnect(a, sta);
+		else
+			RTW_ERR(FUNC_ADPT_FMT ": can't find drv sta info for "
+				MAC_FMT " !\n",
+				FUNC_ADPT_ARG(a),
+				MAC_ARG(pnetwork->MacAddress));
+	}
+
+	/*reset hw before connection if necessary*/
+
+}
+
+void update_join_link_info(struct _ADAPTER *a, struct _ADAPTER_LINK *alink, struct _WLAN_BSSID_EX *pbuf)
+{
+	struct link_mlme_ext_info *pmlmeinfo = &alink->mlmeextpriv.mlmext_info;
+	struct _WLAN_BSSID_EX *pnetwork = &pmlmeinfo->network;
+	struct _NDIS_802_11_VARIABLE_IEs *pIE;
+	u32 i;
+	u8 join_type;
+
+	alink->mlmepriv.num_FortyMHzIntolerant = 0;
+	alink->mlmepriv.num_sta_no_ht = 0;
+	alink->mlmepriv.ampdu_priv.ampdu_enable = _FALSE;/* reset to disabled */
+
+	pmlmeinfo->ERP_enable = 0;
+	pmlmeinfo->WMM_enable = 0;
+	pmlmeinfo->HT_enable = 0;
+	pmlmeinfo->HT_caps_enable = 0;
+	pmlmeinfo->HT_info_enable = 0;
+	pmlmeinfo->SM_PS = SM_PS_DISABLE;
+	pmlmeinfo->bwmode_updated = _FALSE;
+	/* pmlmeinfo->assoc_AP_vendor = HT_IOT_PEER_MAX; */
+	pmlmeinfo->VHT_enable = 0;
+	pmlmeinfo->HE_enable = 0;
+#ifdef PRIVATE_R
+	pmlmeinfo->ht_vht_received = 0;
+	_rtw_memset(pmlmeinfo->SupportedRates_infra_ap, 0, NDIS_802_11_LENGTH_RATES_EX);
+#endif /* PRIVATE_R */
+	_rtw_memcpy(pnetwork, pbuf, FIELD_OFFSET(WLAN_BSSID_EX, IELength));
+	pnetwork->IELength = pbuf->IELength;
+	_rtw_memcpy(pnetwork->IEs, ((WLAN_BSSID_EX *)pbuf)->IEs, pnetwork->IELength);
+
+	pmlmeinfo->bcn_interval = get_beacon_interval(pnetwork);
+
+	/* _FIXED_IE_LENGTH_ */
+	for (i = _FIXED_IE_LENGTH_; i < (pnetwork->IELength - 2);) {
+		pIE = (struct _NDIS_802_11_VARIABLE_IEs*)(pnetwork->IEs + i);
+
+		switch (pIE->ElementID) {
+		case _VENDOR_SPECIFIC_IE_: /* Get WMM IE. */
+			if (_rtw_memcmp(pIE->data, WMM_OUI, 4))
+				WMM_param_handler(a, alink, pIE);
+			break;
+
+#ifdef CONFIG_80211N_HT
+		case _HT_CAPABILITY_IE_:	/* Get HT Cap IE. */
+			pmlmeinfo->HT_caps_enable = 1;
+			break;
+
+		case _HT_EXTRA_INFO_IE_:	/* Get HT Info IE. */
+			pmlmeinfo->HT_info_enable = 1;
+			break;
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_80211AC_VHT
+		case EID_VHTCapability: /* Get VHT Cap IE. */
+			pmlmeinfo->VHT_enable = 1;
+			break;
+
+		case EID_VHTOperation: /* Get VHT Operation IE. */
+			break;
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+		case WLAN_EID_EXTENSION:
+			if (pIE->data[0] == WLAN_EID_EXTENSION_HE_CAPABILITY)
+				pmlmeinfo->HE_enable = 1;
+			break;
+#endif /* CONFIG_80211AX_HE */
+
+		default:
+			break;
+		}
+
+		i += (pIE->Length + 2);
+	}
+
+	/*get chan/bw/offset info from IEs*/
+	rtw_bss_get_chbw(pnetwork, &alink->mlmeextpriv.chandef.band,
+			 &alink->mlmeextpriv.chandef.chan, (u8 *)&alink->mlmeextpriv.chandef.bw,
+			 (u8 *)&alink->mlmeextpriv.chandef.offset, pmlmeinfo->HT_caps_enable,
+			 pmlmeinfo->VHT_enable, pmlmeinfo->HE_enable);
+
+	/*adjust chan/bw/offset with registary and hw cap*/
+	rtw_adjust_bchbw(a, alink->mlmeextpriv.chandef.band, alink->mlmeextpriv.chandef.chan,
+			(u8 *)&alink->mlmeextpriv.chandef.bw, (u8 *)&alink->mlmeextpriv.chandef.offset);
+
+	RTW_INFO("%s: band=%d, chan=%d, bw=%d, offset=%d\n", __func__,
+			alink->mlmeextpriv.chandef.band, alink->mlmeextpriv.chandef.chan,
+			alink->mlmeextpriv.chandef.bw, alink->mlmeextpriv.chandef.offset);
+}
+
+void update_join_info(struct _ADAPTER *a, struct _WLAN_BSSID_EX *pbuf)
+{
+	struct mlme_ext_priv *pmlmeext = &a->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct _WLAN_BSSID_EX *pnetwork = &pmlmeinfo->dev_network;
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	struct wlan_mld_network *mld_network = NULL;
+	struct wlan_network *link_network = NULL;
+	u8 lidx;
+
+	pmlmeinfo->agg_enable_bitmap = 0;
+	pmlmeinfo->candidate_tid_bitmap = 0;
+	_rtw_memcpy(pnetwork, pbuf, FIELD_OFFSET(WLAN_BSSID_EX, IELength));
+	pnetwork->IELength = pbuf->IELength;
+	_rtw_memcpy(pnetwork->IEs, ((WLAN_BSSID_EX *)pbuf)->IEs, pnetwork->IELength);
+
+	/* Update the join info for the primary link */
+	update_join_link_info(a, alink, pbuf);
+#ifdef CONFIG_80211BE_EHT
+	if (pbuf->is_mld) {
+		mld_network = pbuf->mld_network;
+		/* Replace adapter's mlmeinfo->network->MacAddress with MLD MAC Address */
+		_rtw_memcpy(pnetwork->MacAddress, mld_network->mac_addr, ETH_ALEN);
+		/* Update the join info of the rest desired links, where to_join == _TRUE */
+		for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+			alink = GET_LINK(a, lidx);
+			if (!alink->mlmepriv.to_join || lidx == RTW_RLINK_PRIMARY)
+				continue;
+			/* The link_id is assigned at select_and_join_from_scanned_queue() */
+			link_network = rtw_get_link_network_by_linkid(mld_network, alink->mlmepriv.link_id);
+			if (!link_network) {
+				RTW_INFO(FUNC_ADPT_FMT" network is NULL for link_id %u, set adapter_link %d to_join to _FALSE\n"
+					, FUNC_ADPT_ARG(a), alink->mlmepriv.link_id, alink->mlmepriv.link_id);
+				alink->mlmepriv.to_join = _FALSE;
+				continue;
+			}
+			update_join_link_info(a, alink, &link_network->network);
+		}
+	}
+#endif
+}
+
+#if 0
+static void set_hw_prepare_connect(_adapter *padapter, struct sta_info *sta)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
+	WLAN_BSSID_EX		*pnetwork = &pmlmeinfo->network;
+
+#ifdef CONFIG_ANTENNA_DIVERSITY
+	rtw_antenna_select_cmd(padapter, pnetwork->PhyInfo.Optimum_antenna, _FALSE);
+#endif
+	rtw_hw_prepare_connect(padapter, sta, pnetwork->MacAddress);
+}
+#endif
+
+/*
+join_res = -1 caller is rtw_hw_connect_abort
+join_res = 1 caller is rtw_hw_connected
+*/
+void rtw_join_done_chk_ch(_adapter *adapter, int join_res)
+{
+#define DUMP_ADAPTERS_STATUS 0
+
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	_adapter *iface;
+	struct mlme_priv *mlme;
+	struct mlme_ext_priv *mlmeext;
+	int chanctx_num = 0;
+	struct rtw_chan_def u_chdef = {0};
+	struct rtw_chan_def new_chdef = {0};
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+	int i, ret;
+	bool is_chctx_add = false;
+	struct _ADAPTER_LINK *adapter_link = NULL;
+	struct _ADAPTER_LINK *iface_link = NULL;
+	struct link_mlme_priv *lmlme;
+	struct link_mlme_ext_priv *lmlmeext;
+	u8 lidx, iface_lidx;
+
+	#if DUMP_ADAPTERS_STATUS
+	RTW_INFO(FUNC_ADPT_FMT" enter\n", FUNC_ADPT_ARG(adapter));
+	dump_adapters_status(RTW_DBGDUMP , dvobj);
+	#endif
+
+
+	for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+		adapter_link = GET_LINK(adapter, lidx);
+
+		chanctx_num = rtw_phl_mr_get_chanctx_num(dvobj->phl, adapter->phl_role, adapter_link->wrlink);
+		if (join_res >= 0 && chanctx_num <= 0) {
+			dump_adapters_status(RTW_DBGDUMP , dvobj);
+			RTW_ERR("%s join_res:%d, chanctx_num:%d failed\n", __func__, join_res, chanctx_num);
+			join_res = -1;
+			rtw_warn_on(1);
+		}
+
+		if (rtw_phl_mr_get_chandef(dvobj->phl, adapter->phl_role, adapter_link->wrlink, &u_chdef)
+							!= RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s get union chandef failed\n", __func__);
+			rtw_warn_on(1);
+			return;
+		}
+		if (u_chdef.chan == 0) {
+			RTW_ERR("%s chandef.chan == 0\n", __func__);
+			rtw_warn_on(1);
+			return;
+		}
+
+		_rtw_memcpy(&new_chdef, &u_chdef, sizeof(struct rtw_chan_def));
+		#ifdef DBG_DUMP_CHAN_SET
+		RTW_INFO("[UCH] u_band:%d\n", u_chdef.band);
+		RTW_INFO("[UCH] u_chan:%d\n", u_chdef.chan);
+		RTW_INFO("[UCH] u_bw:%d\n", u_chdef.bw);
+		RTW_INFO("[UCH] u_offset:%d\n", u_chdef.offset);
+		#endif
+		/* Only the accepted adapter_link join success */
+		if (join_res >= 0 && adapter_link->mlmepriv.is_accepted) { /*client join success - restart all ap*/
+			for (i = 0; i < dvobj->iface_nums; i++) {
+				iface = dvobj->padapters[i];
+				mlme = &iface->mlmepriv;
+				mlmeext = &iface->mlmeextpriv;
+
+				if (!iface || iface == adapter || !rtw_is_adapter_up(iface))
+					continue;
+
+				for (iface_lidx = 0; iface_lidx < iface->adapter_link_num; iface_lidx++) {
+					/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+					iface_link = GET_PRIMARY_LINK(iface);
+					lmlme = &iface_link->mlmepriv;
+					lmlmeext = &iface_link->mlmeextpriv;
+
+					/* ToDo CONFIG_RTW_MLD: should compare the band of links */
+					if (!rtw_iface_at_same_hwband(adapter, iface))
+						continue;
+
+					if ((MLME_IS_AP(iface) || MLME_IS_MESH(iface))
+						&& check_fwstate(mlme, WIFI_ASOC_STATE)
+						&& check_fwstate(mlme, WIFI_OP_CH_SWITCHING)
+					) {
+						u8 ori_band, ori_ch, ori_bw, ori_offset;
+
+						rtw_phl_chanctx_del(dvobj->phl, iface->phl_role, iface_link->wrlink, NULL);
+
+						/* handle AP which need to switch ch setting */
+						ori_band = lmlmeext->chandef.band;
+						ori_ch = lmlmeext->chandef.chan;
+						ori_bw = lmlmeext->chandef.bw;
+						ori_offset = lmlmeext->chandef.offset;
+
+						/* restore original bw, adjust bw by registry setting on target ch */
+						lmlmeext->chandef.bw = lmlme->ori_chandef.bw;
+						lmlmeext->chandef.band = new_chdef.band;
+						lmlmeext->chandef.chan = new_chdef.chan;
+
+						/*adjust bw according registery and hw cap*/
+						rtw_adjust_bchbw(iface, lmlmeext->chandef.band, lmlmeext->chandef.chan,
+									(u8 *)&lmlmeext->chandef.bw,
+									(u8 *)&lmlmeext->chandef.offset);
+						#ifdef DBG_DUMP_CHAN_SET
+						RTW_INFO("[IF] if_band:%d\n", lmlmeext->chandef.band);
+						RTW_INFO("[IF] if_chan:%d\n", lmlmeext->chandef.chan);
+						RTW_INFO("[IF] if_bw:%d\n", lmlmeext->chandef.bw);
+						RTW_INFO("[IF] if_offset:%d\n", lmlmeext->chandef.offset);
+						#endif
+						#ifdef CONFIG_RTW_MESH
+						if (MLME_IS_MESH(iface))
+							rtw_mesh_adjust_bchbw(	mlmeext->chandef.band,
+										mlmeext->chandef.chan,
+										&mlmeext->chandef.bw,
+										&mlmeext->chandef.offset);
+						#endif
+
+						/*adjust offset according to union chan and bw*/
+						#if 0
+						rtw_chset_sync_chbw(adapter_to_chset(adapter)
+								, &mlmeext->chandef.chan, &mlmeext->chandef.bw, &mlmeext->chandef.offset
+								, &u_ch, &u_bw, &u_offset, 1, 0);
+						#endif
+
+						new_chdef.bw = lmlmeext->chandef.bw;
+						rtw_phl_adjust_chandef(dvobj->phl, iface_link->wrlink, &new_chdef);
+						lmlmeext->chandef.offset = new_chdef.offset;
+						is_chctx_add = rtw_phl_chanctx_add(dvobj->phl, iface->phl_role,
+											iface_link->wrlink,
+											&new_chdef, &mr_cc_info);
+						rtw_hw_update_chan_def(iface, iface_link);
+
+						RTW_INFO(FUNC_ADPT_FMT" ori:%u %u,%u,%u => cur:%i %u,%u,%u => grp:%i %u,%u,%u - chctx_add:%s\n",
+								FUNC_ADPT_ARG(iface),
+								ori_band, ori_ch, ori_bw, ori_offset,
+								lmlmeext->chandef.band, lmlmeext->chandef.chan, lmlmeext->chandef.bw, lmlmeext->chandef.offset,
+								new_chdef.band, new_chdef.chan, new_chdef.bw, new_chdef.offset,
+								(is_chctx_add) ? "Y" : "N");
+						if (is_chctx_add == false) {
+							RTW_ERR(FUNC_ADPT_FMT" chctx_add failed\n", FUNC_ADPT_ARG(iface));
+							rtw_warn_on(1);
+						}
+
+						rtw_ap_update_bss_bchbw(iface, iface_link, &(lmlmeext->mlmext_info.network), &lmlmeext->chandef);
+
+						_rtw_memcpy(&(lmlme->cur_network.network), &(lmlmeext->mlmext_info.network), sizeof(WLAN_BSSID_EX));
+
+						rtw_start_bss_hdl_after_chbw_decided(iface, iface_link);
+
+						{
+							#if defined(CONFIG_IOCTL_CFG80211) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+							u8 ht_option = 0;
+
+							#ifdef CONFIG_80211N_HT
+							ht_option = lmlme->htpriv.ht_option;
+							#endif
+
+							rtw_cfg80211_ch_switch_notify(iface
+								, GET_PRIMARY_LINK(iface)
+								, &lmlmeext->chandef
+								, ht_option, 0);
+							#endif
+						}
+
+						clr_fwstate(mlme, WIFI_OP_CH_SWITCHING);
+
+						rtw_ap_link_regu_forbid_update(iface_link, false);
+
+						rtw_update_beacon(iface, iface_link, 0xFF, NULL, _TRUE, 0);
+						rtw_phl_cmd_stop_beacon(adapter_to_dvobj(adapter)->phl, iface_link->wrlink, _FALSE, PHL_CMD_DIRECTLY, 0);
+					}
+				}
+			}
+			/* ToDo CONFIG_RTW_MLD: [DFS] */
+			#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+			rtw_dfs_rd_en_dec_on_mlme_act(adapter, adapter_link, MLME_STA_CONNECTED, 0);
+			#endif
+
+			if (u_chdef.chan != new_chdef.chan) {
+				RTW_ERR("[MI] chandef->chan(%d) != u_ch(%d)\n", u_chdef.chan, new_chdef.chan);
+				rtw_warn_on(1);
+			}
+		} else { /*client join failed - restart all ap on ori-chan*/
+			for (i = 0; i < dvobj->iface_nums; i++) {
+				iface = dvobj->padapters[i];
+				mlme = &iface->mlmepriv;
+				mlmeext = &iface->mlmeextpriv;
+
+				if (!iface || iface == adapter || !rtw_is_adapter_up(iface))
+					continue;
+
+				for (iface_lidx = 0; iface_lidx < iface->adapter_link_num; iface_lidx++) {
+					iface_link = GET_LINK(iface, iface_lidx);
+					lmlme = &iface_link->mlmepriv;
+					lmlmeext = &iface_link->mlmeextpriv;
+
+					/* ToDo CONFIG_RTW_MLD: should compare the band of links */
+					if (!rtw_iface_at_same_hwband(adapter, iface))
+						continue;
+
+					if ((MLME_IS_AP(iface) || MLME_IS_MESH(iface))
+						&& check_fwstate(mlme, WIFI_ASOC_STATE)
+						&& check_fwstate(mlme, WIFI_OP_CH_SWITCHING)
+					) {
+						rtw_phl_chanctx_del(dvobj->phl, iface->phl_role, iface_link->wrlink, NULL);
+						_rtw_memcpy(&new_chdef, &lmlmeext->chandef, sizeof(struct rtw_chan_def));
+						is_chctx_add = rtw_phl_chanctx_add(dvobj->phl, iface->phl_role,
+											iface_link->wrlink,
+											&new_chdef, &mr_cc_info);
+						if (is_chctx_add == _FALSE) {
+							RTW_ERR(FUNC_ADPT_FMT "chctx_add failed\n", FUNC_ADPT_ARG(iface));
+							rtw_warn_on(1);
+						}
+						RTW_INFO(FUNC_ADPT_FMT" cur:%u,%u,%u => grp:%u,%u,%u chctx_add:%s\n",
+								FUNC_ADPT_ARG(iface),
+								lmlmeext->chandef.chan, lmlmeext->chandef.bw, lmlmeext->chandef.offset,
+								new_chdef.chan, new_chdef.bw, new_chdef.offset,
+								(is_chctx_add) ? "Y" : "N");
+						clr_fwstate(mlme, WIFI_OP_CH_SWITCHING);
+						rtw_update_beacon(iface,iface_link, 0xFF, NULL, _TRUE, 0);
+						rtw_phl_cmd_stop_beacon(adapter_to_dvobj(adapter)->phl, iface_link->wrlink, _FALSE, PHL_CMD_DIRECTLY, 0);
+					}
+				}
+			}
+
+			#ifdef CONFIG_80211D
+			rtw_alink_leavebss_update_regulatory(adapter_link);
+			#endif
+
+			/* ToDo CONFIG_RTW_MLD: [DFS] */
+			#if CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ
+			rtw_dfs_rd_en_dec_on_mlme_act(adapter, adapter_link, MLME_STA_DISCONNECTED, 0);
+			#endif
+		}
+
+		RTW_INFO("[MI] union:%u,%u,%u,%u\n", new_chdef.band, new_chdef.chan, new_chdef.bw, new_chdef.offset);
+		RTW_INFO("[MI] u chandef:%u,%u,%u,%u\n", u_chdef.band, u_chdef.chan, u_chdef.bw, u_chdef.offset);
+
+		set_bch_bwmode(adapter, adapter_link, new_chdef.band, new_chdef.chan, new_chdef.offset, new_chdef.bw, RFK_TYPE_FORCE_NOT_DO);
+
+		if (join_res >= 0)
+			rtw_rfctl_update_op_mode(adapter_to_rfctl(adapter), BIT(adapter->iface_id), 1, 0);
+	}
+
+	#if DUMP_ADAPTERS_STATUS
+	RTW_INFO(FUNC_ADPT_FMT" exit\n", FUNC_ADPT_ARG(adapter));
+	dump_adapters_status(RTW_DBGDUMP , dvobj);
+	#endif
+}
+
+#if 0
+int rtw_chk_start_clnt_join(_adapter *adapter, u8 *ch, u8 *bw, u8 *offset)
+{
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	u8 u_ch = 0, cur_ch = 0;
+	enum channel_width u_bw = 0, cur_bw = 0;
+	enum chan_offset u_offset = 0, cur_offset = 0;
+	struct rtw_chan_def new_chdef = {0};
+	struct rtw_mr_chctx_info mr_cc_info = {0};
+	bool is_chctx_add = false;
+	struct mi_state mstate;
+	bool connect_allow = _TRUE;
+
+	if (!ch || !bw || !offset) {
+		connect_allow = _FALSE;
+		rtw_warn_on(1);
+		goto _exit;
+	}
+
+	new_chdef.chan = cur_ch = pmlmeext->chandef.chan;
+	new_chdef.bw = cur_bw = pmlmeext->chandef.bw;
+	new_chdef.offset = cur_offset = pmlmeext->chandef.offset;
+	new_chdef.band = rtw_get_band_type(new_chdef.chan);
+
+	RTW_INFO(FUNC_ADPT_FMT" CONT - CH:%d, BW:%d OFF:%d\n",
+			FUNC_ADPT_ARG(adapter),cur_ch, cur_bw, cur_offset);
+
+	is_chctx_add = rtw_phl_chanctx_add(dvobj->phl, adapter->phl_role,
+					&new_chdef, &mr_cc_info);
+
+	if (is_chctx_add == false) {
+		rtw_mi_status_no_self(adapter, &mstate);
+		RTW_INFO(FUNC_ADPT_FMT" others ld_sta_num:%u, ap_num:%u, mesh_num:%u\n"
+				, FUNC_ADPT_ARG(adapter), MSTATE_STA_LD_NUM(&mstate)
+				, MSTATE_AP_NUM(&mstate), MSTATE_MESH_NUM(&mstate));
+
+		if (MSTATE_STA_LD_NUM(&mstate) + MSTATE_AP_LD_NUM(&mstate) + MSTATE_MESH_LD_NUM(&mstate) >= 4)
+			connect_allow = _FALSE;
+
+		#if 0 /*def CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT*/
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			mlme = &iface->mlmepriv;
+
+			if (MLME_IS_STA(iface)
+				&& check_fwstate(mlme, WIFI_ASOC_STATE)
+				#if defined(CONFIG_P2P)
+				&& rtw_p2p_chk_state(&(iface->wdinfo), P2P_STATE_NONE)
+				#endif
+			) {
+				connect_allow = _FALSE;
+				break;
+			}
+		}
+		#endif /* CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT */
+
+		if (connect_allow == _FALSE) {
+			RTW_INFO(FUNC_ADPT_FMT" connect_allow = _FALSE\n", FUNC_ADPT_ARG(adapter));
+			goto _exit;
+		}
+
+		rtw_mi_buddy_disconnect(adapter, DISCONNECTION_BY_DRIVER_DUE_TO_EACH_IFACE_CHBW_NOT_SYNC);
+		rtw_phl_chanctx_del_no_self(dvobj->phl, adapter->phl_role);
+
+		new_chdef.chan = cur_ch;
+		new_chdef.bw = cur_bw;
+		new_chdef.offset = cur_offset;
+		new_chdef.band = rtw_get_band_type(new_chdef.chan);
+		is_chctx_add = rtw_phl_chanctx_add(dvobj->phl, adapter->phl_role,
+					&new_chdef, &mr_cc_info);
+		if (is_chctx_add == false) {
+			RTW_ERR("%s - %d chanctx_add failed\n", __FUNCTION__, __LINE__);
+			rtw_warn_on(1);
+			connect_allow = _FALSE;
+			goto _exit;
+		}
+	}
+
+	#ifdef CONFIG_DFS_MASTER
+	rtw_dfs_rd_en_decision(adapter, MLME_STA_CONNECTING, 0);
+	#endif
+
+	*ch = new_chdef.chan;
+	*bw = new_chdef.bw;
+	*offset = new_chdef.offset;
+_exit:
+	RTW_INFO(FUNC_ADPT_FMT" chctx_add:%s\n", FUNC_ADPT_ARG(adapter), (is_chctx_add) ? "Y" : "N");
+	RTW_INFO("Grouped - CH:%d, BW:%d OFF:%d\n",new_chdef.chan, new_chdef.bw, new_chdef.offset);
+	return connect_allow;
+}
+#endif
+
+static u8 join_cmd_pre_hdl(_adapter *padapter, WLAN_BSSID_EX *network)
+{
+	WLAN_BSSID_EX		*psecnetwork = NULL;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+#ifdef CONFIG_80211BE_EHT
+	struct mlo_priv *pmlopriv = &padapter->mlmepriv.mlopriv;
+	struct _ADAPTER_LINK *adapter_link = NULL;
+	struct wlan_mld_network *pmld_network = NULL;
+	struct wlan_network *plink_network;
+	u8 lidx;
+#endif
+	u8 ret = _FAIL;
+
+	pmlmeinfo->assoc_AP_vendor = check_assoc_AP(network->IEs, network->IELength);
+	rtw_update_phl_iot(padapter, pmlmeinfo->assoc_AP_vendor);
+
+#ifdef CONFIG_80211BE_EHT
+	if (network->is_mld)
+		pmld_network = network->mld_network;
+#endif
+
+	/* Added by Albert 2009/02/18 */
+	/* If the the driver wants to use the bssid to create the connection. */
+	/* If not,  we have to copy the connecting AP's MAC address to it so that */
+	/* the driver just has the bssid information for PMKIDList searching. */
+
+	if (padapter->mlmepriv.assoc_by_bssid == _FALSE) {
+#ifdef CONFIG_80211BE_EHT
+		if (pmld_network)
+			_rtw_memcpy(&padapter->mlmepriv.assoc_bssid[0], &pmld_network->mac_addr[0], ETH_ALEN);
+		else
+#endif
+			_rtw_memcpy(&padapter->mlmepriv.assoc_bssid[0], &network->MacAddress[0], ETH_ALEN);
+	}
+
+	psecnetwork = (WLAN_BSSID_EX *)rtw_zmalloc(sizeof(WLAN_BSSID_EX));
+	if (psecnetwork == NULL) {
+		RTW_ERR(FUNC_ADPT_FMT ": join_cmd_pre_hdl failed to alloc psecnetwork\n",
+			FUNC_ADPT_ARG(padapter));
+		goto exit;
+	}
+
+	rtw_update_join_priv(padapter, padapter_link, network, psecnetwork);
+#ifdef CONFIG_80211BE_EHT
+	pmlopriv->mlo_option = _FALSE;
+	pmlopriv->tid2link_nego = _FALSE;
+
+	if (pmld_network) {
+		for (lidx = 0; lidx < pmld_network->network_num; lidx++) {
+			plink_network = GET_LINK_NETWORK(pmld_network, lidx);
+			if (plink_network->network.link_id == network->link_id)
+				continue;
+
+			adapter_link = rtw_get_adapter_link_by_linkid(padapter, plink_network->network.link_id);
+			if (!adapter_link || (adapter_link == padapter_link))
+				continue;
+
+			rtw_update_join_priv(padapter, adapter_link, &plink_network->network, psecnetwork);
+		}
+	}
+#endif
+	rtw_mfree((u8 *)psecnetwork, sizeof(WLAN_BSSID_EX));
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+u8 rtw_join_cmd_hdl(_adapter *padapter, u8 *pbuf)
+{
+	struct joinbss_parm	*pparm = (struct joinbss_parm *)pbuf;
+	u8 ret = H2C_SUCCESS;
+
+	if ((pparm->network.IELength > MAX_IE_SZ) || (pparm->network.IELength < 2)) {
+		RTW_ERR(FUNC_ADPT_FMT ": invalid IE length(%u)\n",
+			 FUNC_ADPT_ARG(padapter), pparm->network.IELength);
+		ret = H2C_PARAMETERS_ERROR;
+		goto exit;
+	}
+
+	if (join_cmd_pre_hdl(padapter, &pparm->network) != _SUCCESS) {
+		ret = H2C_CMD_FAIL;
+		goto exit;
+	}
+
+	if (rtw_connect_cmd(padapter, &pparm->network) != RTW_PHL_STATUS_SUCCESS)
+		ret = H2C_CMD_FAIL;
+
+exit:
+	if (ret != H2C_SUCCESS)
+		report_join_res(padapter, (-4), WLAN_STATUS_UNSPECIFIED_FAILURE);
+	return ret;
+}
+
+u8 disconnect_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+#if CONFIG_DFS
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+#endif
+	struct disconnect_parm *param = (struct disconnect_parm *)pbuf;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8 val8;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	WLAN_BSSID_EX		*pnetwork = (WLAN_BSSID_EX *)(&(padapter_link->mlmeextpriv.mlmext_info.network));
+
+	/* For STA, disconnect flow only for conneted case */
+	if (MLME_IS_STA(padapter) && !MLME_IS_ASOC(padapter))
+		goto exit;
+
+	if (is_client_associated_to_ap(padapter)
+		#if CONFIG_DFS
+		&& !IS_RADAR_DETECTED(rfctl)
+		#ifdef CONFIG_ECSA_PHL
+		&& !rtw_mr_is_ecsa_running(padapter)
+		#endif
+		#endif
+	) {
+		#ifdef CONFIG_PLATFORM_ROCKCHIPS
+		/* To avoid connecting to AP fail during resume process, change retry count from 5 to 1 */
+		issue_deauth_ex(padapter, pnetwork->MacAddress, WLAN_REASON_DEAUTH_LEAVING, 1, 100);
+		#else
+		issue_deauth_ex(padapter, pnetwork->MacAddress, WLAN_REASON_DEAUTH_LEAVING, param->deauth_timeout_ms / 100, 100);
+		#endif /* CONFIG_PLATFORM_ROCKCHIPS */
+	}
+
+	if (((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) || ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)) {
+		/* Stop BCN */
+		val8 = 0;
+		rtw_hal_set_hwreg(padapter, HW_VAR_BCN_FUNC, (u8 *)(&val8));
+	}
+
+	rtw_mlmeext_disconnect(padapter);
+	#if 0
+	rtw_free_uc_swdec_pending_queue(adapter_to_dvobj(padapter));
+	#endif
+
+exit:
+	return	H2C_SUCCESS;
+}
+
+/**
+ * rtw_ps_annc - check and doing ps announcement for all the adapters
+ * @adapter: the requesting adapter
+ * @ps: power saving or not
+ *
+ * Returns: 0: no ps announcement is doing. 1: ps announcement is doing
+ */
+u8 rtw_ps_annc(_adapter *adapter, bool ps)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	_adapter *iface;
+	int i;
+	u8 ps_anc = 0;
+	struct _ADAPTER_LINK *iface_link;
+	u8 lidx;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface)
+			continue;
+		if (!rtw_iface_at_same_hwband(adapter, iface))
+			continue;
+
+		if (MLME_IS_STA(iface)) {
+			if (is_client_associated_to_ap(iface) == _TRUE) {
+				/* TODO: TDLS peers */
+				for (lidx = 0; lidx < iface->adapter_link_num; lidx++) {
+					iface_link = GET_LINK(iface, lidx);
+					if (iface_link->wrlink->mstate == MLME_LINKED
+						#ifdef CONFIG_MCC_MODE
+						&& !rtw_hw_mcc_chk_inprogress(adapter, iface_link)
+						#endif
+					)
+						issue_nulldata(iface, iface_link, NULL, ps, 3, PS_ANNC_DRV_RETRY_INT_MS);
+				}
+				ps_anc = 1;
+			}
+		#ifdef CONFIG_RTW_MESH
+		} else if (MLME_IS_MESH(iface)) {
+			if (rtw_mesh_ps_annc(iface, ps))
+				ps_anc = 1;
+		#endif
+		}
+	}
+	return ps_anc;
+}
+
+void rtw_back_opch(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	_rtw_mutex_lock_interruptible(&rfctl->offch_mutex);
+
+	if (rfctl->offch_state != OFFCHS_NONE) {
+		rfctl->offch_state = OFFCHS_BACKING_OP;
+		rtw_hal_macid_wakeup_all_used(adapter);
+		rtw_ps_annc(adapter, 0);
+
+		rfctl->offch_state = OFFCHS_NONE;
+		rtw_mi_os_xmit_schedule(adapter);
+	}
+
+	_rtw_mutex_unlock(&rfctl->offch_mutex);
+}
+
+void rtw_leave_opch(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	_rtw_mutex_lock_interruptible(&rfctl->offch_mutex);
+
+	if (rfctl->offch_state == OFFCHS_NONE) {
+		/* prepare to leave operating channel */
+		rfctl->offch_state = OFFCHS_LEAVING_OP;
+
+		/* clear HW TX queue */
+		rtw_hal_set_hwreg(adapter, HW_VAR_CHECK_TXBUF, 0);
+
+		rtw_hal_macid_sleep_all_used(adapter);
+
+		rtw_ps_annc(adapter, 1);
+
+		rfctl->offch_state = OFFCHS_LEAVE_OP;
+	}
+
+	_rtw_mutex_unlock(&rfctl->offch_mutex);
+}
+
+u8 setauth_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	struct setauth_parm		*pparm = (struct setauth_parm *)pbuf;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (pparm->mode < 4)
+		pmlmeinfo->auth_algo = pparm->mode;
+
+	return	H2C_SUCCESS;
+}
+
+static u8 amsdu_spp_enable(_adapter *pdapter, enum security_type type)
+{
+	u8 ret = _FALSE;
+	if (pdapter->registrypriv.amsdu_mode == RTW_AMSDU_MODE_SPP) {
+		if ( type == _AES_ || type == _CCMP_256_
+			|| type == _GCMP_ || type == _GCMP_256_ )
+			ret = _SUCCESS;
+	}
+	return ret;
+}
+
+/*Set WEP key or Group Key*/
+u8 setkey_hdl(struct _ADAPTER *a,struct _ADAPTER_LINK *alink, struct setkey_parm *key,
+		enum phl_cmd_type cmd_type,  u32 cmd_timeout)
+{
+	struct mlme_ext_info *info = &a->mlmeextpriv.mlmext_info;
+	u8 keytype = 1; /* 1 for Group Key */
+	u8 *mac;
+	struct sta_info *sta;
+	u8 spp = 0;
+
+
+	if (((info->state & 0x03) == WIFI_FW_AP_STATE)
+	    || ((info->state & 0x03) == WIFI_FW_ADHOC_STATE))
+		mac = alink->mac_addr;
+	else
+		mac = get_link_bssid(&alink->mlmepriv);
+
+	sta = rtw_get_stainfo(&a->stapriv, mac);
+	if (!sta) {
+		RTW_ERR("%s: sta %pM not found\n", __func__, mac);
+		goto exit;
+	}
+
+	/* main tx key for wep. */
+	if (key->set_tx)
+		a->mlmeextpriv.mlmext_info.key_index = key->keyid;
+
+	/* Change to Unicast for WEP */
+	if (is_wep_enc(key->algorithm))
+		keytype = 0;
+
+	#ifdef CONFIG_IEEE80211W
+	if (key->algorithm == _BIP_CMAC_128_)
+		keytype = 2;
+	#endif
+
+	if (amsdu_spp_enable(a, key->algorithm) == _SUCCESS)
+		spp = 1;
+
+	RTW_PRINT("%s: set %s key for " MAC_FMT ", kid:%d type:%u algo:%s\n",
+	  __func__, is_wep_enc(key->algorithm)?"WEP":"group", MAC_ARG(mac),
+	  key->keyid, keytype, security_type_str(key->algorithm));
+
+	/* ToDo CONFIG_RTW_MLD: HALMAC API */
+	rtw_hw_add_key(a, sta, key->keyid, key->algorithm, keytype, key->key, spp, cmd_type, cmd_timeout);
+
+#ifdef CONFIG_LAYER2_ROAMING
+	dequeuq_roam_pkt(a);
+#endif
+
+exit:
+	return H2C_SUCCESS;
+}
+
+void rtw_ap_wep_pk_setting(_adapter *adapter, struct sta_info *psta)
+{
+	struct security_priv *psecuritypriv = &(adapter->securitypriv);
+	struct set_stakey_parm	sta_pparm;
+	sint keyid;
+
+	if (!is_wep_enc(psecuritypriv->dot11PrivacyAlgrthm))
+		return;
+
+	for (keyid = 0; keyid < 4; keyid++) {
+		if ((psecuritypriv->key_mask & BIT(keyid)) && (keyid == psecuritypriv->dot11PrivacyKeyIndex)) {
+			sta_pparm.algorithm = psecuritypriv->dot11PrivacyAlgrthm;
+			sta_pparm.keyid = keyid;
+			sta_pparm.gk = 0;
+			_rtw_memcpy(sta_pparm.key, &(psecuritypriv->dot11DefKey[keyid].skey[0]), 16);
+			_rtw_memcpy(sta_pparm.addr, psta->phl_sta->mac_addr, ETH_ALEN);
+
+			RTW_PRINT(FUNC_ADPT_FMT"set WEP - PK with "MAC_FMT" keyid:%u\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->phl_sta->mac_addr), keyid);
+
+			set_stakey_hdl(adapter, &sta_pparm, PHL_CMD_DIRECTLY, 0);
+		}
+	}
+}
+
+u8 set_stakey_hdl(struct _ADAPTER *a, struct set_stakey_parm *key,
+					enum phl_cmd_type cmd_type,  u32 cmd_timeout)
+{
+	struct sta_priv *pstapriv = &a->stapriv;
+	struct sta_info *sta;
+	int err = 0;
+	u8 ret = H2C_SUCCESS;
+	u8 spp = 0;
+
+
+	sta = rtw_get_stainfo(pstapriv, key->addr);
+	if (!sta) {
+		RTW_ERR("%s: sta %pM not found\n", __func__, key->addr);
+		ret = H2C_REJECTED;
+		goto exit;
+	}
+
+	a->mlmeextpriv.mlmext_info.enc_algo = key->algorithm;
+
+	if (key->algorithm == _NO_PRIVACY_) {
+		RTW_INFO("%s: del all key for " MAC_FMT " for _NO_PRIVACY_\n",
+				 __func__, MAC_ARG(key->addr));
+		err = rtw_hw_del_all_key(a, sta, cmd_type, cmd_timeout);
+	} else {
+		RTW_INFO("%s: set %s key for " MAC_FMT ", kid:%d algo:%s\n",
+				 __func__, key->gk?"group":"pairwise",
+				 MAC_ARG(key->addr), key->keyid,
+				 security_type_str(key->algorithm));
+		if (amsdu_spp_enable(a, key->algorithm) == _SUCCESS)
+			spp = 1;
+#ifdef CONFIG_WAPI_SUPPORT
+		if(key->algorithm == _SMS4_)
+			sta->phl_sta->wapi = true;
+#endif
+
+		/* ToDo CONFIG_RTW_MLD: HALMAC API
+		** pairwise key uses mld addr */
+		err = rtw_hw_add_key(a, sta, key->keyid, key->algorithm,
+				     key->gk, key->key, spp, cmd_type, cmd_timeout);
+	}
+	if (!(key->gk))
+		ATOMIC_INC(&sta->keytrack);	/*CVE-2020-24587*/
+	if (err)
+		RTW_ERR("%s: FAIL to set %s key for %pM, kid:%d algo:%s !\n",
+			__func__, key->gk?"group":"pairwise",
+			key->addr, key->keyid,
+			security_type_str(key->algorithm));
+
+	ret = H2C_SUCCESS_RSP;
+
+exit:
+	return ret;
+}
+
+u8 add_ba_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	struct addBaReq_parm	*pparm = (struct addBaReq_parm *)pbuf;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct sta_info *psta = rtw_get_stainfo(&padapter->stapriv, pparm->addr);
+	struct _ADAPTER_LINK *padapter_link;
+
+	if (!psta)
+		return	H2C_SUCCESS;
+
+	padapter_link = psta->padapter_link;
+
+#ifdef CONFIG_80211N_HT
+	if (((pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)
+		&& (padapter_link->mlmeextpriv.mlmext_info.HT_enable
+#if CONFIG_IEEE80211_BAND_6GHZ
+		|| psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+		)) || ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)) {
+		/* pmlmeinfo->ADDBA_retry_count = 0; */
+		/* pmlmeinfo->candidate_tid_bitmap |= (0x1 << pparm->tid);		 */
+		/* psta->ampdu_priv.candidate_tid_bitmap |= BIT(pparm->tid); */
+		issue_addba_req(padapter, pparm->addr, (u8)pparm->tid);
+		_set_timer(&psta->addba_retry_timer, ADDBA_TO);
+	}
+#ifdef CONFIG_TDLS
+	else if ((psta->tdls_sta_state & TDLS_LINKED_STATE) &&
+		 (psta->htpriv.ht_option == _TRUE
+#if CONFIG_IEEE80211_BAND_6GHZ
+		  || psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+		 ) && (psta->ampdu_priv.ampdu_enable == _TRUE)) {
+		issue_addba_req(padapter, pparm->addr, (u8)pparm->tid);
+		_set_timer(&psta->addba_retry_timer, ADDBA_TO);
+	}
+#endif /* CONFIG */
+	else
+		psta->ampdu_priv.candidate_tid_bitmap &= ~BIT(pparm->tid);
+#endif /* CONFIG_80211N_HT */
+	return	H2C_SUCCESS;
+}
+
+
+u8 add_ba_rsp_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	struct addBaRsp_parm *pparm = (struct addBaRsp_parm *)pbuf;
+	struct recv_reorder_ctrl *preorder_ctrl;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	u8 ret = _TRUE;
+
+	psta = rtw_get_stainfo(pstapriv, pparm->addr);
+	if (!psta)
+		goto exit;
+
+	if (pparm->status == 0) {
+		rtw_phl_start_rx_ba_session(padapter->dvobj->phl, psta->phl_sta,
+					    pparm->preq.dialog_token, 3,
+					    pparm->start_seq, 0, pparm->tid,
+					    pparm->size);
+	}
+
+	preorder_ctrl = &psta->recvreorder_ctrl[pparm->tid];
+	ret = issue_addba_rsp_wait_ack(padapter, pparm->addr, pparm->tid,
+				       pparm->status, pparm->size,
+				       &(pparm->preq), 3, 50);
+
+#ifdef CONFIG_UPDATE_INDICATE_SEQ_WHILE_PROCESS_ADDBA_REQ
+	/* status = 0 means accept this addba req, so update indicate seq = start_seq under this compile flag */
+	if (pparm->status == 0) {
+		preorder_ctrl->indicate_seq = pparm->start_seq;
+		#ifdef DBG_RX_SEQ
+		RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_UPDATE indicate_seq:%d, start_seq:%d\n"
+			, FUNC_ADPT_ARG(padapter), preorder_ctrl->tid, preorder_ctrl->indicate_seq, pparm->start_seq);
+		#endif
+	}
+#else
+	rtw_set_bit(RTW_RECV_ACK_OR_TIMEOUT, &preorder_ctrl->rec_abba_rsp_ack);
+	#ifdef DBG_RX_SEQ
+	RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_CLEAR indicate_seq:%d, "
+                 "start_seq:%d preorder_ctrl->rec_abba_rsp_ack =%lu "
+                 "pparm->preq.dialog_token=%d pparm->preq.ba_p_set=%d\n"
+		, FUNC_ADPT_ARG(padapter)
+		, preorder_ctrl->tid
+		, preorder_ctrl->indicate_seq
+		, pparm->start_seq
+		, preorder_ctrl->rec_abba_rsp_ack
+		, pparm->preq.dialog_token
+		, pparm->preq.ba_p_set
+		);
+	#endif
+#endif
+
+	/*
+	  * status = 0 means accept this addba req
+	  * status = 37 means reject this addba req
+	  */
+	if (pparm->status == 0) {
+		preorder_ctrl->enable = _TRUE;
+		preorder_ctrl->ampdu_size = pparm->size;
+	} else if (pparm->status == 37)
+		preorder_ctrl->enable = _FALSE;
+
+exit:
+	return H2C_SUCCESS;
+}
+
+u8 delba_hdl(struct _ADAPTER *a, unsigned char *pbuf)
+{
+	struct addBaReq_parm *parm = (struct addBaReq_parm *)pbuf;
+	struct sta_info *sta;
+
+
+	sta = rtw_get_stainfo(&a->stapriv, parm->addr);
+	if (!sta) {
+		RTW_WARN(FUNC_ADPT_FMT ": No STA(" MAC_FMT ") for DELBA!\n",
+			 FUNC_ADPT_ARG(a), MAC_ARG(parm->addr));
+		return	H2C_SUCCESS;
+	}
+
+	rtw_phl_stop_rx_ba_session(a->dvobj->phl, sta->phl_sta, parm->tid);
+
+	return	H2C_SUCCESS;
+}
+
+u8 chk_bmc_sleepq_cmd(_adapter *padapter)
+{
+	struct cmd_obj *cmd;
+	struct cmd_priv *pcmdpriv = &(adapter_to_dvobj(padapter)->cmdpriv);
+	u8 res = _SUCCESS;
+
+
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	init_h2fwcmd_w_parm_no_parm_rsp(cmd, CMD_CHK_BMCSLEEPQ);
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+exit:
+	return res;
+}
+
+u8 set_tx_beacon_cmd(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 flags)
+{
+	struct cmd_obj	*cmd;
+	struct Tx_Beacon_param	*ptxBeacon_parm;
+	struct cmd_priv	*pcmdpriv = &(adapter_to_dvobj(padapter)->cmdpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(padapter->mlmeextpriv.mlmext_info);
+	struct link_mlme_ext_priv	*lmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*lmlmeinfo = &(lmlmeext->mlmext_info);
+	struct submit_ctx sctx;
+	u8	res = _SUCCESS;
+	int len_diff = 0;
+
+	/*prepare cmd parameter*/
+	ptxBeacon_parm = (struct Tx_Beacon_param *)rtw_zmalloc(sizeof(struct Tx_Beacon_param));
+	if (ptxBeacon_parm == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	_rtw_memcpy(&(ptxBeacon_parm->network), &(lmlmeinfo->network), sizeof(WLAN_BSSID_EX));
+
+	len_diff = update_hidden_ssid(
+			   ptxBeacon_parm->network.IEs + _BEACON_IE_OFFSET_
+		   , ptxBeacon_parm->network.IELength - _BEACON_IE_OFFSET_
+			   , pmlmeinfo->hidden_ssid_mode
+		   );
+	ptxBeacon_parm->network.IELength += len_diff;
+
+
+	/* need enqueue, prepare cmd_obj and enqueue */
+	cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (cmd == NULL) {
+		res = _FAIL;
+		rtw_mfree((u8 *)ptxBeacon_parm, sizeof(*ptxBeacon_parm));
+		goto exit;
+	}
+	cmd->padapter = padapter;
+
+	init_h2fwcmd_w_parm_no_rsp(cmd, ptxBeacon_parm, CMD_TX_BEACON);
+
+	if (flags & RTW_CMDF_WAIT_ACK) {
+		cmd->sctx = &sctx;
+		rtw_sctx_init(&sctx, 10 * 1000);
+	}
+
+	res = rtw_enqueue_cmd(pcmdpriv, cmd);
+
+	if (res == _SUCCESS && (flags & RTW_CMDF_WAIT_ACK)) {
+		rtw_sctx_wait(&sctx, __func__);
+		_rtw_mutex_lock_interruptible(&pcmdpriv->sctx_mutex);
+		if (sctx.status == RTW_SCTX_SUBMITTED)
+			cmd->sctx = NULL;
+		_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+	}
+exit:
+	return res;
+}
+
+u8 mlme_evt_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	struct rtw_evt_header *evt_hdr;
+	u8 *peventbuf;
+	void (*event_callback)(_adapter *dev, u8 *pbuf);
+	struct cmd_priv *pcmd_priv = &adapter_to_dvobj(padapter)->cmdpriv;
+
+	if (pbuf == NULL)
+		goto _abort_event_;
+
+	evt_hdr = (struct rtw_evt_header *)pbuf;
+	peventbuf = pbuf + sizeof(struct rtw_evt_header);
+
+#ifdef CHECK_EVENT_SEQ
+	/* checking event sequence...		 */
+	if (evt_hdr->seq != (ATOMIC_READ(&pcmd_priv->event_seq) & 0x7f)) {
+		pcmd_priv->event_seq = (evt_hdr->seq + 1) & 0x7f;
+		goto _abort_event_;
+	}
+#endif
+
+	/* checking if event code is valid */
+	if (evt_hdr->id >= EVT_ID_MAX) {
+		goto _abort_event_;
+	}
+
+	/* checking if event size match the event parm size	 */
+	if ((wlanevents[evt_hdr->id].parmsize != 0) &&
+	    (wlanevents[evt_hdr->id].parmsize != evt_hdr->len)) {
+
+		goto _abort_event_;
+
+	}
+
+	ATOMIC_INC(&pcmd_priv->event_seq);
+
+	if (peventbuf) {
+		event_callback = wlanevents[evt_hdr->id].event_callback;
+		event_callback(padapter, (u8 *)peventbuf);
+		pcmd_priv->evt_done_cnt++;
+	}
+
+_abort_event_:
+	return H2C_SUCCESS;
+
+}
+
+u8 chk_bmc_sleepq_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+#ifdef CONFIG_AP_MODE
+	struct sta_info *psta_bmc;
+	_list	*xmitframe_plist, *xmitframe_phead;
+	struct xmit_frame *pxmitframe = NULL;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct sta_priv  *pstapriv = &padapter->stapriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	/* for BC/MC Frames */
+	psta_bmc = rtw_get_bcmc_stainfo(padapter, padapter_link);
+	if (!psta_bmc)
+		return H2C_SUCCESS;
+
+	/* _rtw_spinlock_bh(&psta_bmc->sleep_q.lock); */
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	if ((rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, 0)) && (psta_bmc->sleepq_len > 0)) {
+#ifndef CONFIG_PCI_HCI
+		rtw_mdelay_os(10);/* 10ms, ATIM(HIQ) Windows */
+#endif
+
+		xmitframe_phead = get_list_head(&psta_bmc->sleep_q);
+		xmitframe_plist = get_next(xmitframe_phead);
+
+		while ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE) {
+			pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+			xmitframe_plist = get_next(xmitframe_plist);
+
+			rtw_list_delete(&pxmitframe->list);
+
+			psta_bmc->sleepq_len--;
+			if (psta_bmc->sleepq_len > 0)
+				pxmitframe->attrib.mdata = 1;
+			else
+				pxmitframe->attrib.mdata = 0;
+
+			pxmitframe->attrib.triggered = 1;
+
+#if 0
+
+			if (xmitframe_hiq_filter(pxmitframe) == _TRUE)
+				pxmitframe->attrib.qsel = rtw_hal_get_qsel(padapter,QSLT_HIGH_ID);/* HIQ */
+
+			_rtw_spinunlock_bh(&psta_bmc->sleep_q.lock);
+			if (rtw_hal_xmit(padapter, pxmitframe) == _TRUE)
+				rtw_os_xmit_complete(padapter, pxmitframe);
+			_rtw_spinlock_bh(&psta_bmc->sleep_q.lock);
+#endif
+			rtw_intf_xmitframe_enqueue(padapter, pxmitframe);
+		}
+
+		if (rtw_get_intf_type(padapter) != RTW_HCI_PCIE) {
+			/* check hi queue and bmc_sleepq */
+			rtw_chk_hi_queue_cmd(padapter);
+		}
+	}
+
+	/* _rtw_spinunlock_bh(&psta_bmc->sleep_q.lock); */
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+#endif
+
+	return H2C_SUCCESS;
+}
+
+u8 tx_beacon_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	/*RTW_INFO(FUNC_ADPT_FMT, FUNC_ADPT_ARG(padapter));*/
+	if (send_beacon(padapter) == _FAIL) {
+		RTW_INFO("issue_beacon, fail!\n");
+		return H2C_PARAMETERS_ERROR;
+	}
+
+	/* tx bc/mc frames after update TIM */
+	chk_bmc_sleepq_hdl(padapter, NULL);
+	return H2C_SUCCESS;
+}
+
+/*
+* according to channel
+* add/remove WLAN_BSSID_EX.IEs's ERP ie
+* set WLAN_BSSID_EX.SupportedRates
+* update WLAN_BSSID_EX.IEs's Supported Rate and Extended Supported Rate ie
+*/
+void change_band_update_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			WLAN_BSSID_EX *pnetwork, struct rtw_chan_def *chandef)
+{
+	u8	network_type, rate_len, total_rate_len, remainder_rate_len;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	u8	erpinfo = 0x4;
+
+	if (chandef->band == BAND_ON_5G ||
+	    chandef->band == BAND_ON_6G) {
+		network_type = WLAN_MD_11A;
+		total_rate_len = IEEE80211_NUM_OFDM_RATESLEN;
+		rtw_remove_bcn_ie(padapter, pnetwork, _ERPINFO_IE_);
+		#ifdef CONFIG_80211AC_VHT
+		/* if channel in 5G band, then add vht ie . */
+		if ((pmlmepriv->htpriv.ht_option == _TRUE)
+			&& REGSTY_IS_11AC_ENABLE(&padapter->registrypriv)
+			&& is_supported_vht(padapter->registrypriv.wireless_mode)
+			&& RFCTL_REG_EN_11AC(rfctl)
+		) {
+			if (pmlmepriv->upper_layer_setting)
+				rtw_reattach_vht_ies(padapter, padapter_link, pnetwork);
+			else if (REGSTY_IS_11AC_AUTO(&padapter->registrypriv))
+				rtw_vht_ies_attach(padapter, padapter_link, pnetwork);
+		}
+		#endif
+		#ifdef CONFIG_80211AX_HE
+		/* CONFIG_80211AX_HE_TODO */
+		#endif
+	} else { /* BAND_ON_24G */
+		network_type = 0;
+		total_rate_len = 0;
+		if (padapter->registrypriv.wireless_mode & WLAN_MD_11B) {
+			network_type |= WLAN_MD_11B;
+			total_rate_len += IEEE80211_CCK_RATE_LEN;
+		}
+		if (padapter->registrypriv.wireless_mode & WLAN_MD_11G) {
+			network_type |= WLAN_MD_11G;
+			total_rate_len += IEEE80211_NUM_OFDM_RATESLEN;
+		}
+		rtw_add_bcn_ie(padapter, pnetwork, _ERPINFO_IE_, &erpinfo, 1);
+		#ifdef CONFIG_80211AC_VHT
+		rtw_vht_ies_detach(padapter, padapter_link, pnetwork);
+		#endif
+		#ifdef CONFIG_80211AX_HE
+		/* CONFIG_80211AX_HE_TODO */
+		#endif
+	}
+
+	rtw_set_supported_rate(pnetwork->SupportedRates, network_type,
+			       chandef->chan, chandef->band);
+
+	UpdateBrateTbl(padapter, pnetwork->SupportedRates);
+
+	if (total_rate_len > 8) {
+		rate_len = 8;
+		remainder_rate_len = total_rate_len - 8;
+	} else {
+		rate_len = total_rate_len;
+		remainder_rate_len = 0;
+	}
+
+	rtw_add_bcn_ie(padapter, pnetwork, _SUPPORTEDRATES_IE_, pnetwork->SupportedRates, rate_len);
+
+	if (remainder_rate_len)
+		rtw_add_bcn_ie(padapter, pnetwork, _EXT_SUPPORTEDRATES_IE_, (pnetwork->SupportedRates + 8), remainder_rate_len);
+	else
+		rtw_remove_bcn_ie(padapter, pnetwork, _EXT_SUPPORTEDRATES_IE_);
+
+	pnetwork->Length = get_WLAN_BSSID_EX_sz(pnetwork);
+}
+
+void rtw_set_external_auth_status(_adapter *padapter,
+	const void *data, int len)
+{
+#ifdef CONFIG_IOCTL_CFG80211
+	struct net_device *dev = padapter->pnetdev;
+	struct wiphy *wiphy = adapter_to_wiphy(padapter);
+	struct rtw_external_auth_params params;
+
+	/* convert data to external_auth_params */
+	params.action = RTW_GET_BE32((u8 *)data);
+	_rtw_memcpy(&params.bssid, (u8 *)data + 4, ETH_ALEN);
+	_rtw_memcpy(&params.ssid.ssid, (u8 *)data + 10, WLAN_SSID_MAXLEN);
+	params.ssid.ssid_len = RTW_GET_BE64((u8 *)data + 42);
+	params.key_mgmt_suite = RTW_GET_BE32((u8 *)data + 58);
+	params.status = RTW_GET_BE16((u8 *)data + 62);
+	_rtw_memcpy(&params.pmkid, (u8 *)data + 64, PMKID_LEN);
+
+	rtw_cfg80211_external_auth_status(wiphy, dev, &params);
+#endif /* CONFIG_IOCTL_CFG80211 */
+}
+
+u8 rtw_set_chbw_hdl(_adapter *padapter, u8 *pbuf)
+{
+	struct set_ch_parm *set_ch_parm;
+	struct _ADAPTER_LINK *padapter_link;
+	struct link_mlme_ext_priv	*pmlmeext;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	if (!pbuf)
+		return H2C_PARAMETERS_ERROR;
+
+	set_ch_parm = (struct set_ch_parm *)pbuf;
+
+	padapter_link = GET_LINK(padapter, set_ch_parm->link_idx);
+	pmlmeext = &padapter_link->mlmeextpriv;
+
+	RTW_INFO(FUNC_NDEV_FMT" ch:%u, bw:%u, ch_offset:%u\n",
+		 FUNC_NDEV_ARG(padapter->pnetdev),
+		 set_ch_parm->ch, set_ch_parm->bw, set_ch_parm->ch_offset);
+
+	pmlmeext->chandef.band = set_ch_parm->band;
+	pmlmeext->chandef.chan = set_ch_parm->ch;
+	pmlmeext->chandef.offset = set_ch_parm->ch_offset;
+	pmlmeext->chandef.bw = set_ch_parm->bw;
+
+	set_bch_bwmode(padapter, padapter_link,
+				set_ch_parm->band,
+				set_ch_parm->ch,
+				set_ch_parm->ch_offset,
+				set_ch_parm->bw,
+				set_ch_parm->rt_type);
+	rtw_rfctl_update_op_mode(dvobj_to_rfctl(dvobj), 0, 0, 0);
+
+	return	H2C_SUCCESS;
+}
+
+u8 led_blink_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	struct LedBlink_param *ledBlink_param;
+
+	if (!pbuf)
+		return H2C_PARAMETERS_ERROR;
+
+	ledBlink_param = (struct LedBlink_param *)pbuf;
+
+#ifdef CONFIG_RTW_LED_HANDLED_BY_CMD_THREAD
+	BlinkHandler((PLED_DATA)ledBlink_param->pLed);
+#endif
+
+	return	H2C_SUCCESS;
+}
+
+u8 tdls_hdl(_adapter *padapter, unsigned char *pbuf)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+#ifdef CONFIG_TDLS
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+#ifdef CONFIG_TDLS_CH_SW
+	struct tdls_ch_switch *pchsw_info = &ptdlsinfo->chsw_info;
+#endif
+	struct TDLSoption_param *TDLSoption;
+	struct sta_info *ptdls_sta = NULL;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct sta_info *ap_sta = rtw_get_stainfo(&padapter->stapriv, get_my_bssid(&(pmlmeinfo->network)));
+	u8 survey_channel, i, min, option;
+	struct tdls_txmgmt txmgmt;
+	u32 setchtime, resp_sleep = 0, wait_time;
+	u8 zaddr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	u8 ret;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	enum role_type rtype = PHL_RTYPE_TDLS;
+	enum rtw_phl_status status;
+	u8 sta_band = 0;
+
+	if (!pbuf)
+		return H2C_PARAMETERS_ERROR;
+
+	TDLSoption = (struct TDLSoption_param *)pbuf;
+	option = TDLSoption->option;
+
+	if (wrole->type != PHL_RTYPE_TDLS) {
+		status = rtw_phl_cmd_wrole_change(phl, wrole, padapter_link->wrlink, WR_CHG_TYPE,
+					(u8*)&rtype, sizeof(enum role_type), PHL_CMD_DIRECTLY, 0);
+		if (status != RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s - change to phl role type = %d fail with error = %d\n", __func__, rtype, status);
+			rtw_warn_on(1);
+		}
+	}
+
+	if (!_rtw_memcmp(TDLSoption->addr, zaddr, ETH_ALEN)) {
+		ptdls_sta = rtw_get_stainfo(&(padapter->stapriv), TDLSoption->addr);
+		if (ptdls_sta == NULL)
+			return H2C_REJECTED;
+	}
+
+	/* _rtw_spinlock_bh(&(ptdlsinfo->hdl_lock)); */
+	/* RTW_INFO("[%s] option:%d\n", __FUNCTION__, option); */
+
+	switch (option) {
+	case TDLS_ESTABLISHED:
+
+		if (_FAIL == rtw_alloc_stainfo_hw(&(padapter->stapriv), ptdls_sta)) {
+			RTW_ERR("%s - rtw_alloc_stainfo_hw for option %d fail\n", __func__, option);
+			return H2C_REJECTED;
+		}
+
+		update_sta_ra_info(padapter, ptdls_sta);
+
+#ifdef CONFIG_80211N_HT
+#ifdef CONFIG_80211AC_VHT
+#ifdef CONFIG_80211AX_HE
+		if (ptdls_sta->hepriv.he_option == _TRUE)
+			sta_band = WLAN_MD_11AX;
+		else
+#endif
+		if (ptdls_sta->vhtpriv.vht_option == _TRUE)
+			sta_band = WLAN_MD_11AC;
+		else
+#endif
+		if (ptdls_sta->htpriv.ht_option == _TRUE)
+			sta_band = WLAN_MD_11N;
+#endif
+		if (pmlmeext->chandef.band != BAND_ON_24G)
+			sta_band |= WLAN_MD_11A;
+		else {
+			if ((cckratesonly_included(ptdls_sta->bssrateset, ptdls_sta->bssratelen)) == _TRUE)
+				sta_band |= WLAN_MD_11B;
+			else if ((cckrates_included(ptdls_sta->bssrateset, ptdls_sta->bssratelen)) == _TRUE)
+				sta_band |= WLAN_MD_11BG;
+			else
+				sta_band |= WLAN_MD_11G;
+		}
+		ptdls_sta->phl_sta->wmode = sta_band & padapter->registrypriv.wireless_mode;
+
+		/* ref: top-half of rtw_hw_connected() */
+		rtw_update_phl_sta_cap(padapter, ptdls_sta, &ptdls_sta->phl_sta->asoc_cap);
+		rtw_sta_hal_media_status_rpt_cmd(padapter, ptdls_sta, true, RTW_CMDF_DIRECTLY);
+
+		if (padapter->securitypriv.dot11PrivacyAlgrthm != _NO_PRIVACY_)
+			rtw_tdls_set_key(padapter, ptdls_sta);
+
+		/* Inform PHL for phl_mr_info_upt */
+		rtw_send_tdls_sync_msg(padapter);
+		break;
+	case TDLS_ISSUE_PTI:
+		ptdls_sta->tdls_sta_state |= TDLS_WAIT_PTR_STATE;
+		issue_tdls_peer_traffic_indication(padapter, ptdls_sta);
+		_set_timer(&ptdls_sta->pti_timer, TDLS_PTI_TIME);
+		break;
+#ifdef CONFIG_TDLS_CH_SW
+	case TDLS_CH_SW_RESP:
+		_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+		txmgmt.status_code = 0;
+		_rtw_memcpy(txmgmt.peer, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+
+		if (ap_sta)
+			rtw_phl_cmd_set_macid_pause_ac(wrole, ap_sta->phl_sta, _TRUE, PHL_CMD_DIRECTLY, 0);
+		issue_nulldata(padapter, padapter_link, NULL, 1, 1, 3);
+
+		RTW_INFO("[TDLS ] issue tdls channel switch response\n");
+		ret = issue_tdls_ch_switch_rsp(padapter, &txmgmt, _TRUE);
+
+		/* If we receive TDLS_CH_SW_REQ at off channel which it's target is AP's channel */
+		/* then we just switch to AP's channel*/
+		if (padapter_link->mlmeextpriv.chandef.chan == pchsw_info->off_ch_num) {
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_END_TO_BASE_CHNL, 0);
+			break;
+		}
+
+		if (ret == _SUCCESS)
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_TO_OFF_CHNL, 0);
+		else
+			RTW_INFO("[TDLS] issue_tdls_ch_switch_rsp wait ack fail !!!!!!!!!!\n");
+
+		break;
+	case TDLS_CH_SW_PREPARE:
+		/* to collect IQK info of off-chnl */
+		set_channel_bwmode(padapter,
+				ptdls_sta->padapter_link,
+				pchsw_info->off_ch_num,
+				pchsw_info->ch_offset,
+				(pchsw_info->ch_offset) ? CHANNEL_WIDTH_40 : CHANNEL_WIDTH_20,
+				RFK_TYPE_FORCE_DO);
+
+		/* switch back to base-chnl */
+		set_channel_bwmode(padapter,
+				ptdls_sta->padapter_link,
+				pmlmeext->chandef.chan,
+				pmlmeext->chandef.offset,
+				pmlmeext->chandef.bw,
+				RFK_TYPE_FORCE_NOT_DO);
+
+		pchsw_info->ch_sw_state |= TDLS_CH_SWITCH_PREPARE_STATE; /* Already do prepare */
+
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_START, 0);
+
+		break;
+	case TDLS_CH_SW_START:
+		rtw_tdls_set_ch_sw_oper_control(padapter, _TRUE);
+		break;
+	case TDLS_CH_SW_TO_OFF_CHNL:
+		if (pchsw_info->ch_sw_state & TDLS_CH_SW_INITIATOR_STATE) {
+			if(ap_sta)
+				rtw_phl_cmd_set_macid_pause_ac(wrole, ap_sta->phl_sta, _TRUE, PHL_CMD_DIRECTLY, 0);
+			issue_nulldata(padapter, padapter_link, NULL, 1, 1, 3);
+		}
+
+		if (padapter->registrypriv.wifi_spec == 0) {
+			if (!(pchsw_info->ch_sw_state & TDLS_CH_SW_INITIATOR_STATE))
+				_set_timer(&ptdls_sta->ch_sw_timer, (u32)(ptdls_sta->ch_switch_timeout) / 1000);
+		}
+
+		if (rtw_tdls_do_ch_sw(padapter, ptdls_sta, TDLS_CH_SW_OFF_CHNL, pchsw_info->off_ch_num,
+			pchsw_info->ch_offset, (pchsw_info->ch_offset) ? CHANNEL_WIDTH_40 : CHANNEL_WIDTH_20, ptdls_sta->ch_switch_time) == _SUCCESS) {
+			pchsw_info->ch_sw_state &= ~(TDLS_PEER_AT_OFF_STATE);
+			if (pchsw_info->ch_sw_state & TDLS_CH_SW_INITIATOR_STATE) {
+				if (issue_nulldata_to_TDLS_peer_STA(ptdls_sta->padapter, ptdls_sta, 0, 1,
+						(padapter->registrypriv.wifi_spec == 0) ? 3 : 0) == _FAIL)
+					rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_TO_BASE_CHNL, 0);
+			}
+		} else {
+			if (!(pchsw_info->ch_sw_state & TDLS_CH_SW_INITIATOR_STATE))
+				_cancel_timer_ex(&ptdls_sta->ch_sw_timer);
+		}
+
+
+		break;
+	case TDLS_CH_SW_END:
+	case TDLS_CH_SW_END_TO_BASE_CHNL:
+		rtw_tdls_set_ch_sw_oper_control(padapter, _FALSE);
+		pchsw_info->ch_sw_state &= ~TDLS_CH_SWITCH_PREPARE_STATE;
+		_cancel_timer_ex(&ptdls_sta->ch_sw_timer);
+		_cancel_timer_ex(&ptdls_sta->stay_on_base_chnl_timer);
+		_cancel_timer_ex(&ptdls_sta->ch_sw_monitor_timer);
+#if 0
+		_rtw_memset(pHalData->tdls_ch_sw_iqk_info_base_chnl, 0x00, sizeof(pHalData->tdls_ch_sw_iqk_info_base_chnl));
+		_rtw_memset(pHalData->tdls_ch_sw_iqk_info_off_chnl, 0x00, sizeof(pHalData->tdls_ch_sw_iqk_info_off_chnl));
+#endif
+
+		pchsw_info->ch_sw_state &= ~TDLS_CH_SWITCH_ON_STATE;
+		if (option == TDLS_CH_SW_END_TO_BASE_CHNL) {
+			if (pmlmeext->chandef.chan != rtw_get_oper_ch(padapter, padapter_link)) {
+				pchsw_info->off_ch_num = pmlmeext->chandef.chan;
+				pchsw_info->ch_offset = pmlmeext->chandef.offset;
+				issue_tdls_ch_switch_req(padapter, ptdls_sta);
+			}
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_TO_BASE_CHNL, 0);
+		}
+
+		break;
+	case TDLS_CH_SW_TO_BASE_CHNL_UNSOLICITED:
+	case TDLS_CH_SW_TO_BASE_CHNL:
+		pchsw_info->ch_sw_state &= ~(TDLS_PEER_AT_OFF_STATE | TDLS_WAIT_CH_RSP_STATE);
+
+		if (padapter_link->mlmeextpriv.chandef.chan != rtw_get_oper_ch(padapter, padapter_link)) {
+			if (option == TDLS_CH_SW_TO_BASE_CHNL_UNSOLICITED) {
+				if (ptdls_sta != NULL) {
+					/* Send unsolicited channel switch rsp. to peer */
+					_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+					txmgmt.status_code = 0;
+					_rtw_memcpy(txmgmt.peer, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+					issue_tdls_ch_switch_rsp(padapter, &txmgmt, _FALSE);
+				}
+			}
+
+			if (rtw_tdls_do_ch_sw(padapter, ptdls_sta, TDLS_CH_SW_BASE_CHNL, pmlmeext->chandef.chan,
+				pmlmeext->chandef.offset, pmlmeext->chandef.bw, ptdls_sta->ch_switch_time) == _SUCCESS) {
+				if (ap_sta)
+					rtw_phl_cmd_set_macid_pause_ac(wrole, ap_sta->phl_sta, _FALSE, PHL_CMD_DIRECTLY, 0);
+				issue_nulldata(padapter, padapter_link, NULL, 0, 3, 3);
+				/* set ch sw monitor timer for responder */
+				if ((pchsw_info->ch_sw_state & TDLS_CH_SWITCH_ON_STATE) && !(pchsw_info->ch_sw_state & TDLS_CH_SW_INITIATOR_STATE))
+					_set_timer(&ptdls_sta->ch_sw_monitor_timer, TDLS_CH_SW_MONITOR_TIMEOUT);
+			}
+		}
+		break;
+#endif
+	case TDLS_TEARDOWN_STA:
+	case TDLS_TEARDOWN_STA_TOOFAR:
+	case TDLS_TEARDOWN_STA_NO_WAIT:
+		_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+		if (option == TDLS_TEARDOWN_STA_TOOFAR)
+			txmgmt.status_code = _RSON_TDLS_TEAR_TOOFAR_;
+		else
+			txmgmt.status_code = _RSON_TDLS_TEAR_UN_RSN_;
+		_rtw_memcpy(txmgmt.peer, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+
+		issue_tdls_teardown(padapter, &txmgmt, (option == TDLS_TEARDOWN_STA) ? _TRUE : _FALSE);
+
+		break;
+	case TDLS_TEARDOWN_STA_LOCALLY:
+	case TDLS_TEARDOWN_STA_LOCALLY_POST:
+#ifdef CONFIG_TDLS_CH_SW
+		if (_rtw_memcmp(TDLSoption->addr, pchsw_info->addr, ETH_ALEN) == _TRUE) {
+			pchsw_info->ch_sw_state &= ~(TDLS_CH_SWITCH_PREPARE_STATE|
+						   TDLS_CH_SW_INITIATOR_STATE |
+						   TDLS_CH_SWITCH_ON_STATE |
+						   TDLS_PEER_AT_OFF_STATE |
+						   TDLS_WAIT_CH_SW_LAUNCH_STATE);
+			rtw_tdls_set_ch_sw_oper_control(padapter, _FALSE);
+			_rtw_memset(pchsw_info->addr, 0x00, ETH_ALEN);
+		}
+#endif
+
+		if (option == TDLS_TEARDOWN_STA_LOCALLY)
+			rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+
+		rtw_tdls_teardown_post_hdl(padapter, ptdls_sta, _FALSE);
+
+		if (ptdlsinfo->tdls_sctx != NULL)
+			rtw_sctx_done(&(ptdlsinfo->tdls_sctx));
+
+		break;
+	}
+
+	/* _rtw_spinunlock_bh(&(ptdlsinfo->hdl_lock)); */
+
+	return H2C_SUCCESS;
+#else
+	return H2C_REJECTED;
+#endif /* CONFIG_TDLS */
+
+}
+
+u8 run_in_thread_hdl(_adapter *padapter, u8 *pbuf)
+{
+	struct RunInThread_param *p;
+
+
+	if (NULL == pbuf)
+		return H2C_PARAMETERS_ERROR;
+	p = (struct RunInThread_param *)pbuf;
+
+	if (p->func)
+		p->func(p->context);
+
+	return H2C_SUCCESS;
+}
+
+int rtw_sae_preprocess(_adapter *adapter, const u8 *buf, u32 len, u8 tx)
+{
+#ifdef CONFIG_IOCTL_CFG80211
+	const u8 *frame_body = buf + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u16 alg;
+	u16 seq;
+	u16 status;
+	int ret = _FAIL;
+
+	alg = RTW_GET_LE16(frame_body);
+	if (alg != WLAN_AUTH_SAE)
+		goto exit;
+
+	seq = RTW_GET_LE16(frame_body + 2);
+	status = RTW_GET_LE16(frame_body + 4);
+
+	RTW_INFO("RTW_%s:AUTH alg:0x%04x, seq:0x%04x, status:0x%04x, mesg:%s\n",
+		(tx == _TRUE) ? "Tx" : "Rx", alg, seq, status,
+		(seq == 1) ? "Commit" : "Confirm");
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		rtw_mesh_sae_check_frames(adapter, buf, len, tx, alg, seq, status);
+		goto exit;
+	}
+#endif
+
+	/* STA mode connect WPA3 AP, but receive auth with status = 1 */
+	if (tx == false && status == WLAN_REASON_UNSPECIFIED && MLME_IS_STA(adapter)) {
+		RTW_INFO("WPA3 auth fail, so trigger join timeout to notify disconnection\n");
+		set_assoc_timer(&(adapter->mlmepriv), 1);
+	}
+
+	if (tx && (seq == 2) && (status == 0) && MLME_IS_AP(adapter)) {
+		/* queue confirm frame until external auth status update */
+		struct sta_priv *pstapriv = &adapter->stapriv;
+		struct sta_info	*psta = NULL;
+
+		psta = rtw_get_stainfo(pstapriv, GetAddr1Ptr(buf));
+		if (psta) {
+			_rtw_spinlock_bh(&psta->lock);
+			if (psta->pauth_frame) {
+				rtw_mfree(psta->pauth_frame, psta->auth_len);
+				psta->pauth_frame = NULL;
+				psta->auth_len = 0;
+			}
+
+			psta->pauth_frame =  rtw_zmalloc(len);
+			if (psta->pauth_frame) {
+				_rtw_memcpy(psta->pauth_frame, buf, len);
+				psta->auth_len = len;
+			}
+			_rtw_spinunlock_bh(&psta->lock);
+
+			ret = 2;
+		}
+	}
+
+	/* let supplicant to handle SAE_HASH_TO_ELEMENT(error)
+	 * 0x7e(126) : SAE_HASH_TO_ELEMENT
+	 * SAE authentication uses direct hashing,
+	 * instead of looping, to obtain the PWE.
+	 * 0x4c(76) : ANTI_CLOGGING_TOKEN_REQ
+	 * 0x4d(77) : FINITE_CYCLIC_GROUP_NOT_SUPPORTED
+	 * WPAS needs to resend AUTH req. with the token,
+	 * driver must pass 0x4c to WPAS to handle it. */
+	if (status == 0 || status == 0x7e || status == 0x4c || status == 0x4d)
+		return _SUCCESS;
+exit:
+	return ret;
+#else
+	return _SUCCESS;
+#endif /* CONFIG_IOCTL_CFG80211 */
+}
+
+char UNKNOWN_EVT[16] = "UNKNOWN_EVT";
+char *rtw_evt_name(struct rtw_evt_header *pev)
+{
+	if (pev->id >= EVT_ID_MAX)
+		return UNKNOWN_EVT;
+
+	return wlanevents[pev->id].name;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mp.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mp.c
new file mode 100644
index 000000000000..cdbebf25e775
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_mp.c
@@ -0,0 +1,4084 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_MP_C_
+#include <drv_types.h>
+#ifdef PLATFORM_FREEBSD
+	#include <sys/unistd.h>		/* for RFHIGHPID */
+#endif
+
+
+#ifdef CONFIG_MP_VHT_HW_TX_MODE
+#define CEILING_POS(X) ((X - (int)(X)) > 0 ? (int)(X + 1) : (int)(X))
+#define CEILING_NEG(X) ((X - (int)(X)) < 0 ? (int)(X - 1) : (int)(X))
+#define ceil(X) (((X) > 0) ? CEILING_POS(X) : CEILING_NEG(X))
+
+int rtfloor(float x)
+{
+	int i = x - 2;
+	while
+	(++i <= x - 1)
+		;
+	return i;
+}
+#endif
+
+#ifdef CONFIG_MP_INCLUDED
+bool rtw_mp_is_cck_rate(u16 rate)
+{
+	return ((rate & 0x1ff) <= _11M_RATE_) ? true : false;
+}
+
+static void _init_mp_priv_(struct mp_priv *pmp_priv)
+{
+	WLAN_BSSID_EX *pnetwork;
+	NDIS_802_11_MAC_ADDRESS network_macaddr;
+	BOOLEAN bSetRxBssid = _FALSE;
+
+	if (pmp_priv->bSetRxBssid) {
+		bSetRxBssid = pmp_priv->bSetRxBssid;
+		_rtw_memcpy(network_macaddr, pmp_priv->network_macaddr, ETH_ALEN);
+	}
+	
+	_rtw_memset(pmp_priv, 0, sizeof(struct mp_priv));
+
+	pmp_priv->mode = MP_OFF;
+
+	pmp_priv->band = 0;
+	pmp_priv->channel = 1;
+	pmp_priv->bandwidth = CHANNEL_WIDTH_20;
+	pmp_priv->prime_channel_offset = CHAN_OFFSET_NO_EXT;
+	//pmp_priv->rateidx = RATE_1M;
+	pmp_priv->txpoweridx = 0;
+	pmp_priv->antenna_tx = MP_ANTENNA_A;
+	pmp_priv->antenna_rx = MP_ANTENNA_A;
+	pmp_priv->antenna_trx = MP_ANTENNA_A;
+
+	pmp_priv->check_mp_pkt = 0;
+
+	pmp_priv->tx_pktcount = 0;
+
+	pmp_priv->rx_bssidpktcount = 0;
+	pmp_priv->rx_pktcount = 0;
+	pmp_priv->rx_crcerrpktcount = 0;
+
+	pmp_priv->network_macaddr[0] = 0x00;
+	pmp_priv->network_macaddr[1] = 0xE0;
+	pmp_priv->network_macaddr[2] = 0x4C;
+	pmp_priv->network_macaddr[3] = 0x87;
+	pmp_priv->network_macaddr[4] = 0x66;
+	pmp_priv->network_macaddr[5] = 0x55;
+
+	pmp_priv->bSetRxBssid = _FALSE;
+	pmp_priv->bRTWSmbCfg = _FALSE;
+	pmp_priv->bloopback = _FALSE;
+
+	pmp_priv->bloadefusemap = _FALSE;
+	pmp_priv->brx_filter_beacon = _FALSE;
+	pmp_priv->mplink_brx = _FALSE;
+	pmp_priv->mp_keep_btc_mode = BTC_MODE_MAX;
+
+	if (bSetRxBssid){
+		pmp_priv->bSetRxBssid = bSetRxBssid;
+		_rtw_memcpy(pmp_priv->network_macaddr, network_macaddr, ETH_ALEN);
+	}
+
+	pnetwork = &pmp_priv->mp_network.network;
+	_rtw_memcpy(pnetwork->MacAddress, pmp_priv->network_macaddr, ETH_ALEN);
+
+	pnetwork->Ssid.SsidLength = 8;
+	_rtw_memcpy(pnetwork->Ssid.Ssid, "mp_871x", pnetwork->Ssid.SsidLength);
+
+	pmp_priv->tx.payload = MP_TX_Payload_default_random;
+#ifdef CONFIG_80211N_HT
+	pmp_priv->tx.attrib.ht_en = 1;
+#endif
+
+	pmp_priv->mpt_ctx.mpt_rate_index = 1;
+	pmp_priv->tx_shape_idx = 255;
+	pmp_priv->rx_cal_process = 0;
+}
+
+
+static void mp_init_xmit_attrib(struct mp_tx *pmptx, _adapter *padapter)
+{
+	struct pkt_attrib *pattrib;
+
+	/* init xmitframe attribute */
+	pattrib = &pmptx->attrib;
+	_rtw_memset(pattrib, 0, sizeof(struct pkt_attrib));
+	_rtw_memset(pmptx->desc, 0, TXDESC_SIZE);
+
+	pattrib->ether_type = 0x8712;
+#if 0
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+#endif
+	_rtw_memset(pattrib->dst, 0xFF, ETH_ALEN);
+
+	/*	pattrib->dhcp_pkt = 0;
+	 *	pattrib->pktlen = 0; */
+	pattrib->ack_policy = 0;
+	/*	pattrib->pkt_hdrlen = ETH_HLEN; */
+	pattrib->hdrlen = WLAN_HDR_A3_LEN;
+	pattrib->subtype = WIFI_DATA;
+	pattrib->priority = 0;
+	/*pattrib->qsel = pattrib->priority;*/
+	/*	do_queue_select(padapter, pattrib); */
+	pattrib->nr_frags = 1;
+	pattrib->encrypt = 0;
+	pattrib->bswenc = _FALSE;
+	pattrib->qos_en = _FALSE;
+
+	pattrib->pktlen = 1500;
+}
+
+s32 init_mp_priv(_adapter *padapter)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+
+	_init_mp_priv_(pmppriv);
+	pmppriv->papdater = padapter;
+	if (0) //(IS_HARDWARE_TYPE_8822C(padapter))
+		pmppriv->mp_dm = 1;/* default enable dpk tracking */
+	else
+		pmppriv->mp_dm = 0;
+
+	pmppriv->preamble = 0;
+	pmppriv->tx.stop = 1;
+	pmppriv->bSetTxPower = 0;		/*for  manually set tx power*/
+	pmppriv->bTxBufCkFail = _FALSE;
+	pmppriv->pktInterval = 100;
+	pmppriv->pktLength = 1000;
+	pmppriv->bprocess_mp_mode = _FALSE;
+	pmppriv->rtw_mp_tx_method = RTW_MP_PMACT_TX;
+	pmppriv->rtw_mp_tx_state = 0;
+	pmppriv->rtw_mp_cur_phy = 0;
+	pmppriv->rtw_mp_pmact_patt_idx = 0;
+	pmppriv->rtw_mp_pmact_ppdu_type = 0;
+	pmppriv->rtw_mp_dbcc = 0;
+	pmppriv->pre_refcw_cck_pwridxa = 0;
+	pmppriv->pre_refcw_cck_pwridxb = 0;
+	pmppriv->pre_refcw_ofdm_pwridxa = 0;
+	pmppriv->pre_refcw_ofdm_pwridxb = 0;
+	pmppriv->rtw_mp_data_bandwidth = CHANNEL_WIDTH_20;
+	pmppriv->rtw_mp_tx_time = 0;
+	pmppriv->rtw_mp_trxsc = 0;
+	pmppriv->rtw_mp_stbc = 0;
+	pmppriv->rtw_mp_he_sigb = 0;
+	pmppriv->rtw_mp_he_sigb_dcm = 0;
+	pmppriv->rtw_mp_plcp_gi = 1;
+	pmppriv->rtw_mp_plcp_ltf = 0;
+	pmppriv->rtw_mp_plcp_tx_time = 0;
+	pmppriv->rtw_mp_plcp_tx_mode = 0;
+	pmppriv->rtw_mp_plcp_tx_user = 1;
+	pmppriv->rtw_mp_he_er_su_ru_106_en = 0;
+	pmppriv->rtw_mp_ru_tone = MP_RU_TONE_26;
+	pmppriv->rtw_coding = 1;  /* 1 LDPC, 0 BCC */
+	pmppriv->mp_plcp_useridx = 0;
+	pmppriv->mp_plcp_user[0].plcp_mcs = 0;
+	pmppriv->mp_plcp_user[0].aid = 0;
+	pmppriv->mp_plcp_user[0].coding = 12;/*(Global_Var.RU_user_coding[Global_Var.RU_current_user] == (int)Coding.BCC) ? 10 : 12;*/
+	pmppriv->mp_plcp_user[0].dcm = 0;
+	pmppriv->mp_plcp_user[0].plcp_txlen = 1000;
+	pmppriv->mp_plcp_user[0].ru_alloc = 0;
+	pmppriv->mp_plcp_user[0].txbf = 0;
+	pmppriv->mp_plcp_user[0].pwr_boost_db = 0;
+
+	pmppriv->btc_path = BTC_MODE_WL;
+	pmppriv->bspecif_tssi_de = false;
+
+	_rtw_memset(pmppriv->st_giltf, 0, sizeof(struct rtw_mp_giltf_data)*5);
+
+	mp_init_xmit_attrib(&pmppriv->tx, padapter);
+#if 0
+	switch (GET_HAL_RFPATH(padapter)) {
+	case RF_1T1R:
+		pmppriv->antenna_tx = ANTENNA_A;
+		pmppriv->antenna_rx = ANTENNA_A;
+		break;
+	case RF_1T2R:
+	default:
+		pmppriv->antenna_tx = ANTENNA_A;
+		pmppriv->antenna_rx = ANTENNA_AB;
+		break;
+	case RF_2T2R:
+		pmppriv->antenna_tx = ANTENNA_AB;
+		pmppriv->antenna_rx = ANTENNA_AB;
+		break;
+	case RF_2T4R:
+		pmppriv->antenna_tx = ANTENNA_BC;
+		pmppriv->antenna_rx = ANTENNA_ABCD;
+		break;
+	}
+#endif
+
+	return _SUCCESS;
+}
+
+#if 0
+void free_mp_priv(struct mp_priv *pmp_priv)
+{
+	if (pmp_priv->pallocated_mp_xmitframe_buf) {
+		rtw_mfree(pmp_priv->pallocated_mp_xmitframe_buf, 0);
+		pmp_priv->pallocated_mp_xmitframe_buf = NULL;
+	}
+	pmp_priv->pmp_xmtframe_buf = NULL;
+}
+#endif
+#if 0
+static void PHY_IQCalibrate_default(
+		_adapter *adapter,
+		BOOLEAN	bReCovery
+)
+{
+	RTW_INFO("%s\n", __func__);
+}
+
+static void PHY_LCCalibrate_default(
+		_adapter *adapter
+)
+{
+	RTW_INFO("%s\n", __func__);
+}
+
+static void PHY_SetRFPathSwitch_default(
+		_adapter *adapter,
+		BOOLEAN		bMain
+)
+{
+	RTW_INFO("%s\n", __func__);
+}
+#endif
+
+void mpt_InitHWConfig(_adapter *adapter)
+{
+#ifdef CONFIG_RTL8822B
+	if (IS_HARDWARE_TYPE_8822B(adapter)) {
+		u32 tmp_reg = 0;
+
+		rtw_write16(adapter, REG_RXFLTMAP1_8822B, 0x2000);
+		/* fixed wifi can't 2.4g tx suggest by Szuyitasi 20160504 */
+		phy_set_bb_reg(adapter, 0x70, bMaskByte3, 0x0e);
+		RTW_INFO(" 0x73 = 0x%x\n", phy_query_bb_reg(adapter, 0x70, bMaskByte3));
+		phy_set_bb_reg(adapter, 0x1704, bMaskDWord, 0x0000ff00);
+		RTW_INFO(" 0x1704 = 0x%x\n", phy_query_bb_reg(adapter, 0x1704, bMaskDWord));
+		phy_set_bb_reg(adapter, 0x1700, bMaskDWord, 0xc00f0038);
+		RTW_INFO(" 0x1700 = 0x%x\n", phy_query_bb_reg(adapter, 0x1700, bMaskDWord));
+	}
+#endif /* CONFIG_RTL8822B */
+#ifdef CONFIG_RTL8821C
+	if (IS_HARDWARE_TYPE_8821C(adapter))
+		rtw_write16(adapter, REG_RXFLTMAP1_8821C, 0x2000);
+#endif /* CONFIG_RTL8821C */
+#if defined(CONFIG_RTL8822C)
+	if( IS_HARDWARE_TYPE_8822C(adapter)) {
+		rtw_write16(adapter, REG_RXFLTMAP1_8822C, 0x2000);
+		/* 0x7D8[31] : time out enable when cca is not assert
+			0x60D[7:0] : time out value (Unit : us)*/
+		rtw_write8(adapter, 0x7db, 0xc0);
+		RTW_INFO(" 0x7d8 = 0x%x\n", rtw_read8(adapter, 0x7d8));
+		rtw_write8(adapter, 0x60d, 0x0c);
+		RTW_INFO(" 0x60d = 0x%x\n", rtw_read8(adapter, 0x60d));
+		phy_set_bb_reg(adapter, 0x1c44, BIT10, 0x1);
+		RTW_INFO(" 0x1c44 = 0x%x\n", phy_query_bb_reg(adapter, 0x1c44, bMaskDWord));
+	}
+#endif
+#if defined(CONFIG_RTL8814B)
+	if(IS_HARDWARE_TYPE_8814B(adapter))
+		rtw_write16(adapter, REG_RXFLTMAP1_8814B, 0x2000);
+#endif
+
+}
+
+static void PHY_IQCalibrate(_adapter *padapter, u8 bReCovery)
+{
+	//halrf_iqk_trigger(adapter_to_phydm(padapter), bReCovery);
+}
+
+static void PHY_LCCalibrate(_adapter *padapter)
+{
+	//halrf_lck_trigger(adapter_to_phydm(padapter));
+}
+
+#if 0
+static u8 PHY_QueryRFPathSwitch(_adapter *padapter)
+{
+	u8 bmain = 0;
+
+
+	return bmain;
+}
+#endif
+
+static void  PHY_SetRFPathSwitch(_adapter *padapter , BOOLEAN bMain) {
+
+}
+
+#if 0
+static void phy_switch_rf_path_set(_adapter *padapter , u8 *prf_set_State) {
+#ifdef CONFIG_RTL8821C
+	struct dm_struct *phydm = adapter_to_phydm(padapter);
+
+	if (IS_HARDWARE_TYPE_8821C(padapter)) {
+		config_phydm_set_ant_path(phydm, *prf_set_State, phydm->current_ant_num_8821c);
+		/* Do IQK when switching to BTG/WLG, requested by RF Binson */
+		if (*prf_set_State == SWITCH_TO_BTG || *prf_set_State == SWITCH_TO_WLG)
+			PHY_IQCalibrate(padapter, FALSE);
+	}
+#endif
+
+}
+#endif
+
+s32
+MPT_InitializeAdapter(
+		_adapter *adapter,
+		u8				Channel
+)
+{
+	s32		rtStatus = _SUCCESS;
+	PMPT_CONTEXT	pMptCtx = &adapter->mppriv.mpt_ctx;
+	/*u32		ledsetting;*/
+
+	pMptCtx->bMptDrvUnload = _FALSE;
+	pMptCtx->bMassProdTest = _FALSE;
+	pMptCtx->bMptIndexEven = _TRUE;	/* default gain index is -6.0db */
+	pMptCtx->h2cReqNum = 0x0;
+	/* init for BT MP */
+	mpt_InitHWConfig(adapter);
+
+	pMptCtx->bMptWorkItemInProgress = _FALSE;
+	pMptCtx->CurrMptAct = NULL;
+	pMptCtx->mpt_rf_path = RF_PATH_A;
+	/* ------------------------------------------------------------------------- */
+	/* Don't accept any packets */
+	//rtw_write32(adapter, REG_RCR, 0);
+
+	/* ledsetting = rtw_read32(adapter, REG_LEDCFG0); */
+	/* rtw_write32(adapter, REG_LEDCFG0, ledsetting & ~LED0DIS); */
+
+	/* rtw_write32(adapter, REG_LEDCFG0, 0x08080); */
+	/*ledsetting = rtw_read32(adapter, REG_LEDCFG0);*/
+
+
+	PHY_LCCalibrate(adapter);
+	PHY_IQCalibrate(adapter, _FALSE);
+	/* dm_check_txpowertracking(adapter_to_phydm(adapter));	*/ /* trigger thermal meter */
+
+	PHY_SetRFPathSwitch(adapter, 1/*pHalData->bDefaultAntenna*/); /* default use Main */
+#if 0
+	pMptCtx->backup0xc50 = (u8)phy_query_bb_reg(adapter, rOFDM0_XAAGCCore1, bMaskByte0);
+	pMptCtx->backup0xc58 = (u8)phy_query_bb_reg(adapter, rOFDM0_XBAGCCore1, bMaskByte0);
+	pMptCtx->backup0xc30 = (u8)phy_query_bb_reg(adapter, rOFDM0_RxDetector1, bMaskByte0);
+	pMptCtx->backup0x52_RF_A = (u8)phy_query_rf_reg(adapter, RF_PATH_A, RF_0x52, 0x000F0);
+	pMptCtx->backup0x52_RF_B = (u8)phy_query_rf_reg(adapter, RF_PATH_B, RF_0x52, 0x000F0);
+#endif	
+	return	rtStatus;
+}
+
+/*-----------------------------------------------------------------------------
+ * Function:	MPT_DeInitAdapter()
+ *
+ * Overview:	Extra DeInitialization for Mass Production Test.
+ *
+ * Input:		_adapter *adapter
+ *
+ * Output:		NONE
+ *
+ * Return:		NONE
+ *
+ * Revised History:
+ *	When		Who		Remark
+ *	05/08/2007	MHC		Create Version 0.
+ *	05/18/2007	MHC		Add normal driver MPHalt code.
+ *
+ *---------------------------------------------------------------------------*/
+void
+MPT_DeInitAdapter(
+		_adapter *adapter
+)
+{
+	struct mp_priv *pmppriv = &adapter->mppriv;
+	struct mp_tx	*pmptx = &pmppriv->tx;
+
+	PMPT_CONTEXT		pMptCtx = &adapter->mppriv.mpt_ctx;
+	pMptCtx->bMptDrvUnload = _TRUE;
+
+	if(pmptx->pallocated_buf) {
+		rtw_mfree(pmptx->pallocated_buf, pmptx->buf_size);
+		pmptx->pallocated_buf = NULL;
+	}
+}
+
+static u8 mpt_ProStartTest(_adapter *padapter)
+{
+	PMPT_CONTEXT pMptCtx = &padapter->mppriv.mpt_ctx;
+
+	pMptCtx->bMassProdTest = _TRUE;
+	pMptCtx->is_start_cont_tx = _FALSE;
+	pMptCtx->bCckContTx = _FALSE;
+	pMptCtx->bOfdmContTx = _FALSE;
+	pMptCtx->bSingleCarrier = _FALSE;
+	pMptCtx->is_carrier_suppression = _FALSE;
+	pMptCtx->is_single_tone = _FALSE;
+	pMptCtx->HWTxmode = PACKETS_TX;
+
+	return _SUCCESS;
+}
+
+void rtw_mp_cal_trigger(_adapter *padapter, u8 cal_tye)
+{
+	struct rtw_mp_cal_arg	*mp_cal_arg = NULL;
+
+	mp_cal_arg = _rtw_malloc(sizeof(struct rtw_mp_cal_arg));
+	if (mp_cal_arg)
+		_rtw_memset((void *)mp_cal_arg, 0, sizeof(struct rtw_mp_cal_arg));
+	else {
+		RTW_ERR("%s,malloc fail ! ", __func__);
+		return;
+	}
+	mp_cal_arg->cal_type = cal_tye;
+
+	if (cal_tye == RTW_MP_CAL_DPK_TRACK) {
+		if (rtw_mp_phl_calibration(padapter,
+					mp_cal_arg,
+					RTW_MP_CAL_CMD_TRIGGER_DPK_TRACKING))
+			RTW_INFO("%s ok\n", __func__);
+		else
+			RTW_INFO("%s fail\n", __func__);
+	} else {
+		if (rtw_mp_phl_calibration(padapter,
+					mp_cal_arg,
+					RTW_MP_CAL_CMD_TRIGGER_CAL))
+			RTW_INFO("%s ok\n", __func__);
+		else
+			RTW_INFO("%s fail\n", __func__);
+	}
+	if (mp_cal_arg)
+		_rtw_mfree(mp_cal_arg, sizeof(struct rtw_mp_cal_arg));
+
+	return;
+}
+
+void rtw_mp_cal_capab(_adapter *padapter, u8 cal_tye, u8 benable)
+{
+	struct rtw_mp_cal_arg	*mp_cal_arg = NULL;
+
+	mp_cal_arg = _rtw_malloc(sizeof(struct rtw_mp_cal_arg));
+	if (mp_cal_arg)
+		_rtw_memset((void *)mp_cal_arg, 0, sizeof(struct rtw_mp_cal_arg));
+	else {
+		RTW_ERR("%s,malloc fail ! ", __func__);
+		return;
+	}
+
+	mp_cal_arg->cal_type = cal_tye;
+	mp_cal_arg->enable = benable;
+
+	if (rtw_mp_phl_calibration(padapter, mp_cal_arg, RTW_MP_CAL_CMD_SET_CAPABILITY_CAL))
+		RTW_INFO("%s ok\n", __func__);
+	else
+		RTW_INFO("%s fail\n", __func__);
+
+	if (mp_cal_arg)
+		_rtw_mfree(mp_cal_arg, sizeof(struct rtw_mp_cal_arg));
+
+	return;
+}
+
+/*
+ * General use
+ */
+#if 0
+s32 SetPowerTracking(_adapter *padapter, u8 enable)
+{
+
+	//hal_mpt_SetPowerTracking(padapter, enable);
+	return 0;
+}
+
+void GetPowerTracking(_adapter *padapter, u8 *enable)
+{
+	//hal_mpt_GetPowerTracking(padapter, enable);
+	return;
+}
+#endif
+
+void rtw_mp_trigger_iqk(_adapter *padapter)
+{
+	rtw_mp_cal_trigger(padapter, RTW_MP_CAL_IQK);
+}
+
+void rtw_mp_trigger_lck(_adapter *padapter)
+{
+	rtw_mp_cal_trigger(padapter, RTW_MP_CAL_LCK);
+}
+
+void rtw_mp_trigger_dpk(_adapter *padapter)
+{
+	rtw_mp_cal_trigger(padapter, RTW_MP_CAL_DPK);
+}
+
+#if 0
+void rtw_mp_trigger_tssi(_adapter *padapter)
+{
+	rtw_mp_cal_trigger(padapter, RTW_MP_CAL_TSSI);
+}
+
+void rtw_mp_trigger_ch_rfk(_adapter *padapter)
+{
+	rtw_mp_cal_trigger(padapter, RTW_MP_CAL_CHL_RFK);
+}
+
+void rtw_mp_trigger_dack(_adapter *padapter)
+{
+	rtw_mp_cal_trigger(padapter, RTW_MP_CAL_DACK);
+}
+#endif
+
+static void init_mp_data(_adapter *padapter)
+{
+#if 0	
+	struct dm_struct *phydm = adapter_to_phydm(padapter);
+
+	/*disable BCN*/
+	v8 = rtw_read8(padapter, REG_BCN_CTRL);
+	v8 &= ~EN_BCN_FUNCTION;
+	rtw_write8(padapter, REG_BCN_CTRL, v8);
+
+	phydm->rf_calibrate_info.txpowertrack_control = _FALSE;
+#endif	
+}
+
+u32 mp_join(_adapter *padapter, u8 mode)
+{
+	WLAN_BSSID_EX bssid;
+	struct sta_info *psta;
+	u32 length;
+	s32 res = _SUCCESS;
+	u8 i = 0;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network *tgt_network = &pmlmepriv->dev_cur_network;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	WLAN_BSSID_EX		*pnetwork = (WLAN_BSSID_EX *)(&(pmlmeinfo->dev_network));
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct rtw_phl_mld_t *pmld = NULL;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+
+	/* 1. initialize a new WLAN_BSSID_EX */
+	_rtw_memset(&bssid, 0, sizeof(WLAN_BSSID_EX));
+	RTW_INFO("%s ,pmppriv->network_macaddr=%x %x %x %x %x %x\n", __func__,
+		pmppriv->network_macaddr[0], pmppriv->network_macaddr[1], pmppriv->network_macaddr[2], pmppriv->network_macaddr[3], pmppriv->network_macaddr[4],
+		 pmppriv->network_macaddr[5]);
+	_rtw_memcpy(bssid.MacAddress, pmppriv->network_macaddr, ETH_ALEN);
+
+	if (mode == WIFI_FW_ADHOC_STATE) {
+		bssid.Ssid.SsidLength = strlen("mp_pseudo_adhoc");
+		_rtw_memcpy(bssid.Ssid.Ssid, (u8 *)"mp_pseudo_adhoc", bssid.Ssid.SsidLength);
+		bssid.InfrastructureMode = Ndis802_11IBSS;
+		bssid.IELength = 0;
+		bssid.Configuration.DSConfig = pmppriv->channel;
+
+	} else if (mode == WIFI_FW_STATION_STATE) {
+		bssid.Ssid.SsidLength = strlen("mp_pseudo_STATION");
+		_rtw_memcpy(bssid.Ssid.Ssid, (u8 *)"mp_pseudo_STATION", bssid.Ssid.SsidLength);
+		bssid.InfrastructureMode = Ndis802_11Infrastructure;
+		bssid.IELength = 0;
+	}
+
+	length = get_WLAN_BSSID_EX_sz(&bssid);
+	if (length % 4)
+		bssid.Length = ((length >> 2) + 1) << 2; /* round up to multiple of 4 bytes. */
+	else
+		bssid.Length = length;
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	if (check_fwstate(pmlmepriv, WIFI_MP_STATE) == _TRUE)
+		goto end_of_mp_start_test;
+
+	/* init mp_start_test status */
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		_adapter *iface = NULL;
+		struct mlme_priv *buddy_mlmepriv;
+
+		iface = dvobj->padapters[i];
+		buddy_mlmepriv = &iface->mlmepriv;
+
+		if (iface == NULL)
+			continue;
+		if (rtw_is_adapter_up(iface) == _FALSE)
+			continue;
+		if (check_fwstate(buddy_mlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+			rtw_disassoc_cmd(iface, 500, RTW_CMDF_WAIT_ACK);
+			rtw_indicate_disconnect(iface, 0, _FALSE);
+		}
+	}
+
+	pmppriv->prev_fw_state = get_fwstate(pmlmepriv);
+	/*pmlmepriv->fw_state = WIFI_MP_STATE;*/
+	init_fwstate(pmlmepriv, WIFI_MP_STATE);
+	set_fwstate(pmlmepriv, WIFI_UNDER_LINKING);
+
+	/* 3 2. create a new psta for mp driver */
+	/* clear psta in the cur_network, if any */
+	psta = rtw_get_stainfo(&padapter->stapriv, tgt_network->network.MacAddress);
+	if (psta)
+		rtw_free_mld_stainfo(padapter, psta->phl_sta->mld);
+	/* ToDo CONFIG_RTW_MLD: MLD MAC Address */
+	pmld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter->phl_role, bssid.MacAddress, DTYPE);
+	if (pmld == NULL) {
+		init_fwstate(pmlmepriv, pmppriv->prev_fw_state);
+		res = _FAIL;
+		goto end_of_mp_start_test;
+	}
+	/* main_id is don't care for self */
+	psta = rtw_alloc_stainfo(&padapter->stapriv, bssid.MacAddress, DTYPE, 0, padapter_link->wrlink->id, PHL_CMD_DIRECTLY);
+	if (psta == NULL) {
+		/*pmlmepriv->fw_state = pmppriv->prev_fw_state;*/
+		init_fwstate(pmlmepriv, pmppriv->prev_fw_state);
+		res = _FAIL;
+		goto end_of_mp_start_test;
+	}
+	if (mode == WIFI_FW_ADHOC_STATE)
+		set_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+	else
+		set_fwstate(pmlmepriv, WIFI_STATION_STATE);
+	/* 3 3. join psudo AdHoc */
+	tgt_network->join_res = 1;
+	tgt_network->aid = psta->phl_sta->aid = 1;
+
+	_rtw_memcpy(&padapter->registrypriv.dev_network, &bssid, length);
+	rtw_update_registrypriv_dev_network(padapter);
+	_rtw_memcpy(&tgt_network->network, &padapter->registrypriv.dev_network, padapter->registrypriv.dev_network.Length);
+	_rtw_memcpy(pnetwork, &padapter->registrypriv.dev_network, padapter->registrypriv.dev_network.Length);
+
+	rtw_indicate_connect(padapter);
+	_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+	set_fwstate(pmlmepriv, WIFI_ASOC_STATE);
+
+end_of_mp_start_test:
+
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+	if (1) { /* (res == _SUCCESS) */
+		/* set MSR to WIFI_FW_ADHOC_STATE */
+		if (mode == WIFI_FW_ADHOC_STATE) {
+			/* set msr to WIFI_FW_ADHOC_STATE */
+			pmlmeinfo->state = WIFI_FW_ADHOC_STATE;
+			rtw_hal_set_hwreg(padapter, HW_VAR_BSSID, padapter->registrypriv.dev_network.MacAddress);
+			pmlmeinfo->state |= WIFI_FW_ASSOC_SUCCESS;
+		} else {
+			RTW_INFO("%s , pmppriv->network_macaddr =%x %x %x %x %x %x\n", __func__,
+				pmppriv->network_macaddr[0], pmppriv->network_macaddr[1], pmppriv->network_macaddr[2], pmppriv->network_macaddr[3], pmppriv->network_macaddr[4],
+				pmppriv->network_macaddr[5]);
+
+			rtw_hal_set_hwreg(padapter, HW_VAR_BSSID, pmppriv->network_macaddr);
+		}
+	}
+
+	return res;
+}
+/* This function initializes the DUT to the MP test mode */
+s32 mp_start_test(_adapter *padapter)
+{
+#if 0
+	struct mp_priv *pmppriv = &padapter->mppriv;
+#endif
+	s32 res = _SUCCESS;
+
+	padapter->registrypriv.mp_mode = 1;
+	init_mp_priv(padapter);
+
+	init_mp_data(padapter);
+#if 0
+#ifdef CONFIG_PCI_HCI
+	hal = GET_PHL_COM(adapter_to_dvobj(padapter));
+	hal->pci_backdoor_ctrl = 0;
+	rtw_pci_aspm_config(padapter);
+#endif
+
+
+	/* 3 0. update mp_priv */
+	switch (GET_HAL_RFPATH(padapter)) {
+		case RF_1T1R:
+			pmppriv->antenna_tx = ANTENNA_A;
+			pmppriv->antenna_rx = ANTENNA_A;
+			break;
+		case RF_1T2R:
+		default:
+			pmppriv->antenna_tx = ANTENNA_A;
+			pmppriv->antenna_rx = ANTENNA_AB;
+			break;
+		case RF_2T2R:
+			pmppriv->antenna_tx = ANTENNA_AB;
+			pmppriv->antenna_rx = ANTENNA_AB;
+			break;
+		case RF_2T4R:
+			pmppriv->antenna_tx = ANTENNA_AB;
+			pmppriv->antenna_rx = ANTENNA_ABCD;
+			break;
+	}
+#endif
+	mpt_ProStartTest(padapter);
+
+	mp_join(padapter, WIFI_FW_ADHOC_STATE);
+
+	return res;
+}
+/* ------------------------------------------------------------------------------
+ * This function change the DUT from the MP test mode into normal mode */
+void mp_stop_test(_adapter *padapter)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network *tgt_network = &pmlmepriv->dev_cur_network;
+	struct sta_info *psta;
+#ifdef CONFIG_PCI_HCI
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+#endif
+
+	if (pmppriv->mode == MP_ON) {
+		pmppriv->bSetTxPower = 0;
+		_rtw_spinlock_bh(&pmlmepriv->lock);
+		if (check_fwstate(pmlmepriv, WIFI_MP_STATE) == _FALSE)
+			goto end_of_mp_stop_test;
+
+		/* 3 1. disconnect psudo AdHoc */
+		rtw_indicate_disconnect(padapter, 0, _FALSE);
+
+		/* 3 2. clear psta used in mp test mode.
+		*	rtw_free_assoc_resources(padapter, _TRUE); */
+		psta = rtw_get_stainfo(&padapter->stapriv, tgt_network->network.MacAddress);
+		if (psta)
+			rtw_free_mld_stainfo(padapter, psta->phl_sta->mld);
+
+		/* 3 3. return to normal state (default:station mode) */
+		/*pmlmepriv->fw_state = pmppriv->prev_fw_state; */ /* WIFI_STATION_STATE;*/
+		init_fwstate(pmlmepriv, pmppriv->prev_fw_state);
+
+		/* flush the cur_network */
+		_rtw_memset(tgt_network, 0, sizeof(struct wlan_network));
+
+		_clr_fwstate_(pmlmepriv, WIFI_MP_STATE);
+
+end_of_mp_stop_test:
+
+		_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+#if 0//def CONFIG_PCI_HCI
+		hal = GET_PHL_COM(adapter_to_dvobj(padapter));
+		hal->pci_backdoor_ctrl = registry_par->pci_aspm_config;
+		rtw_pci_aspm_config(padapter);
+#endif
+	}
+}
+
+void SetChannel(_adapter *padapter)
+{
+#ifdef CONFIG_MP_INCLUDED
+	rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_CH_BW);
+#endif
+}
+
+void SetGpio(_adapter *padapter)
+{
+	rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_GPIO);
+}
+
+void SetBandwidth(_adapter *padapter)
+{
+#ifdef CONFIG_MP_INCLUDED
+	rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_CH_BW);
+#endif
+}
+
+u8 rtw_mp_rfpath2txnss(_adapter *padapter, enum rf_path path)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u8 tx_nss = MP_NSS1;
+
+	if (path < RF_PATH_AB)
+		tx_nss = MP_NSS1;
+	else if (path > RF_PATH_D && path < RF_PATH_ABC)
+		tx_nss = MP_NSS2;
+	else if (path > RF_PATH_CD && path < RF_PATH_ABCD)
+		tx_nss = MP_NSS3;
+	else if (path == RF_PATH_ABCD)
+		tx_nss = MP_NSS4;
+	else
+		RTW_ERR("not support path %d\n", path);
+
+	return tx_nss;
+}
+
+void SetAntenna(_adapter *padapter)
+{
+	mp_ant_path mp_trx_path = padapter->mppriv.antenna_trx;
+	u8 cfg_rf_path = RF_PATH_A;
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	void *phl = GET_PHL_INFO(dvobj);
+
+	switch (mp_trx_path) {
+	case MP_ANTENNA_A:
+		cfg_rf_path = RF_PATH_A;
+		break;
+	case MP_ANTENNA_B:
+		cfg_rf_path = RF_PATH_B;
+		break;
+	case MP_ANTENNA_C:
+		cfg_rf_path = RF_PATH_C;
+		break;
+	case MP_ANTENNA_D:
+		cfg_rf_path = RF_PATH_D;
+		break;
+	case MP_ANTENNA_AB:
+		cfg_rf_path = RF_PATH_AB;
+		break;
+	case MP_ANTENNA_AC:
+		cfg_rf_path = RF_PATH_AC;
+		break;
+	case MP_ANTENNA_AD:
+		cfg_rf_path = RF_PATH_AD;
+		break;
+	case MP_ANTENNA_BC:
+		cfg_rf_path = RF_PATH_BC;
+		break;
+	case MP_ANTENNA_BD:
+		cfg_rf_path = RF_PATH_BD;
+		break;
+	case MP_ANTENNA_CD:
+		cfg_rf_path = RF_PATH_CD;
+		break;
+	case MP_ANTENNA_ABC:
+		cfg_rf_path = RF_PATH_ABC;
+		break;
+	case MP_ANTENNA_BCD:
+		cfg_rf_path = RF_PATH_BCD;
+		break;
+	case MP_ANTENNA_ABD:
+		cfg_rf_path = RF_PATH_ABD;
+		break;
+	case MP_ANTENNA_ACD:
+		cfg_rf_path = RF_PATH_ACD;
+		break;
+	case MP_ANTENNA_ABCD:
+		cfg_rf_path = RF_PATH_ABCD;
+		break;
+	default:
+		RTW_INFO("%s ,default Antenna Path A\n", __func__);
+		break;
+	}
+	padapter->mppriv.curr_rfpath = cfg_rf_path;
+
+	if(registry_par->mp_mode == 1) {
+		rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_RF_TXRX_PATH);
+	}
+	else {
+		rtw_phl_cfg_trx_path(phl,cfg_rf_path,registry_par->tx_nss,cfg_rf_path,registry_par->rx_nss);
+	}
+}
+
+s16 rtw_mp_get_pwr_refcw(_adapter *adapter, u8 rfpath, u8 is_cck)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	s16 txpwr_refcw_idx;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+	ptxpwr_arg.rfpath = rfpath;
+	ptxpwr_arg.is_cck = is_cck;
+
+	rtw_mp_phl_txpower(adapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_GET_TXPWR_REF_CW);
+	txpwr_refcw_idx = ptxpwr_arg.txpwr_index / TX_POWER_CODE_WORD_BASE;
+
+	RTW_INFO("%s ,pwr ref cw: %d\n", __func__, txpwr_refcw_idx);
+	return txpwr_refcw_idx;
+}
+
+u16 rtw_mp_get_pwr_ref(_adapter *adapter, u8 rfpath)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	struct mp_priv *pmppriv = &adapter->mppriv;
+	s16 txpwr_ref;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+	ptxpwr_arg.rfpath = rfpath;
+	ptxpwr_arg.is_cck = rtw_mp_is_cck_rate(pmppriv->rateidx);
+
+	rtw_mp_phl_txpower(adapter,  &ptxpwr_arg, RTW_MP_TXPWR_CMD_GET_TXPWR_REF);
+	txpwr_ref = ptxpwr_arg.txpwr_ref / TX_POWER_BASE;
+
+	RTW_INFO("%s ,pwr ref: %d\n", __func__, txpwr_ref);
+	return txpwr_ref;
+}
+
+u16 rtw_mp_txpower_dbm(_adapter *adapter, u8 rf_path)
+{
+	struct mp_priv *pmppriv = &adapter->mppriv;
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	u16 tmp_power_ref = 0;
+	u16 agc_cw_val = 0;
+	s16 pre_pwr_refcw_idx = 0;
+	u8 rfpath_i = 0;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(adapter);
+	u8 tx_nss = get_phy_tx_nss(adapter, padapter_link);
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.txpwr = pmppriv->txpowerdbm;
+	ptxpwr_arg.is_cck = rtw_mp_is_cck_rate(pmppriv->rateidx);
+	ptxpwr_arg.rfpath = rf_path;
+
+	if (pmppriv->tssi_mode >= RTW_MP_TSSI_ON && ptxpwr_arg.txpwr > 17 * TX_POWER_BASE) {
+		pmppriv->txpowerdbm = 16 * TX_POWER_BASE;
+
+		for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+			rtw_mp_txpower_dbm(adapter, rfpath_i);
+		rtw_msleep_os(20);
+		pmppriv->txpowerdbm = ptxpwr_arg.txpwr;
+	}
+
+	rtw_mp_phl_txpower(adapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_SET_TXPWR);
+
+	if (pmppriv->pre_refcw_cck_pwridxa == 0 || pmppriv->pre_refcw_cck_pwridxb == 0) {
+		pmppriv->pre_refcw_cck_pwridxa = rtw_mp_get_pwr_refcw(adapter, RF_PATH_A, 1);
+		pmppriv->pre_refcw_cck_pwridxb = rtw_mp_get_pwr_refcw(adapter, RF_PATH_B, 1);
+	}
+	if (pmppriv->pre_refcw_ofdm_pwridxa == 0 || pmppriv->pre_refcw_ofdm_pwridxb == 0) {
+		pmppriv->pre_refcw_ofdm_pwridxa = rtw_mp_get_pwr_refcw(adapter, RF_PATH_A, 0);
+		pmppriv->pre_refcw_ofdm_pwridxb = rtw_mp_get_pwr_refcw(adapter, RF_PATH_B, 0);
+	}
+
+	if (rf_path == RF_PATH_A && ptxpwr_arg.is_cck == true)
+		pre_pwr_refcw_idx = pmppriv->pre_refcw_cck_pwridxa;
+	else if (rf_path == RF_PATH_B && ptxpwr_arg.is_cck == true)
+		pre_pwr_refcw_idx = pmppriv->pre_refcw_cck_pwridxb;
+	else if (rf_path == RF_PATH_A && ptxpwr_arg.is_cck == false)
+		pre_pwr_refcw_idx = pmppriv->pre_refcw_ofdm_pwridxa;
+	else if (rf_path == RF_PATH_B && ptxpwr_arg.is_cck == false)
+		pre_pwr_refcw_idx = pmppriv->pre_refcw_ofdm_pwridxb;
+
+	tmp_power_ref = rtw_mp_get_pwr_ref(adapter, rf_path);
+	RTW_INFO("%s () tmp_power_ref: %d !! pre_pwr_refcw_idx:%d, pwr offset %d\n",
+							__func__, tmp_power_ref, pre_pwr_refcw_idx , pmppriv->txpoweridx);
+	agc_cw_val = pmppriv->txpowerdbm - tmp_power_ref + pre_pwr_refcw_idx + pmppriv->txpoweridx;
+
+	ptxpwr_arg.txpwr_index = (pre_pwr_refcw_idx + pmppriv->txpoweridx) * TX_POWER_CODE_WORD_BASE;
+	rtw_mp_phl_txpower(adapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_SET_TXPWR_INDEX);
+
+	return agc_cw_val;
+}
+
+void SetDataRate(_adapter *padapter)
+{
+
+	rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_RATE_IDX);
+
+	return;
+}
+
+void rtw_mp_set_rfpath_switch(_adapter *adapter)
+{
+	rtw_mp_phl_config_arg(adapter, RTW_MP_CONFIG_CMD_SWITCH_ANTENNA);
+	return;
+}
+
+
+void GetUuid(_adapter *adapter, u32 *uuid)
+{
+	struct rtw_mp_config_arg pmp_arg;
+	u16 i = 0;
+
+	_rtw_memset((void *)&pmp_arg, 0, sizeof(struct rtw_mp_config_arg));
+
+	pmp_arg.mp_class = RTW_MP_CLASS_CONFIG;
+	pmp_arg.cmd = RTW_MP_CONFIG_CMD_GET_UUID;
+	RTW_INFO("%s, id: %d !!!\n", __func__, pmp_arg.cmd);
+
+	rtw_mp_set_phl_cmd(adapter, (void*)&pmp_arg, sizeof(struct rtw_mp_config_arg));
+
+	while (i != 100) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(adapter, (void*)&pmp_arg, sizeof(struct rtw_mp_config_arg));
+
+		if (pmp_arg.cmd_ok && pmp_arg.status == RTW_PHL_STATUS_SUCCESS) {
+			*uuid = pmp_arg.uuid;
+			RTW_INFO("%s, SET CMD OK, uuid = %d\n", __func__, pmp_arg.uuid);
+			break;
+		} else {
+			if (i > 100) {
+				RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__, pmp_arg.status);
+				break;
+			}
+			i++;
+			rtw_msleep_os(10);
+		}
+	}
+}
+
+void GetThermalMeter(_adapter *adapter, u8 rfpath ,u8 *value)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	u16 i = 0;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.mp_class = RTW_MP_CLASS_TXPWR;
+	ptxpwr_arg.cmd = RTW_MP_TXPWR_CMD_GET_THERMAL;
+	ptxpwr_arg.rfpath = rfpath;
+
+	rtw_mp_set_phl_cmd(adapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+
+	while (i != 100) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(adapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+
+		if (ptxpwr_arg.cmd_ok && ptxpwr_arg.status == RTW_PHL_STATUS_SUCCESS) {
+			*value = ptxpwr_arg.thermal;
+			RTW_INFO("%s, SET CMD OK, thermal = %d\n", __func__, ptxpwr_arg.thermal);
+			break;
+		} else {
+			if (i > 100) {
+				RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__, ptxpwr_arg.status);
+					break;
+				}
+			i++;
+			rtw_msleep_os(10);
+		}
+	}
+}
+
+void rtw_mp_singlecarrier_tx(_adapter *adapter, u8 bstart)
+{
+	RTW_INFO("Not Supported SetSingleCarrierTx \n");
+}
+
+void rtw_mp_singletone_tx(_adapter *adapter, u8 bstart)
+{
+	struct mp_priv *pmp_priv = &adapter->mppriv;
+
+	rtw_mp_txpwr_level(adapter);
+	rtw_phl_mp_tx_cmd(adapter, RTW_MP_TX_SINGLE_TONE, pmp_priv->rtw_mp_tx_method, bstart);
+}
+
+void rtw_mp_carriersuppr_tx(_adapter *adapter, u8 bstart)
+{
+	struct mp_priv *pmp_priv = &adapter->mppriv;
+
+	rtw_mp_txpwr_level(adapter);
+	rtw_phl_mp_tx_cmd(adapter, RTW_MP_TX_CCK_Carrier_Suppression, pmp_priv->rtw_mp_tx_method, bstart);
+}
+
+void rtw_mp_continuous_tx(_adapter *adapter, u8 bstart)
+{
+	struct mp_priv *pmp_priv = &adapter->mppriv;
+
+	rtw_mp_txpwr_level(adapter);
+	rtw_phl_mp_tx_cmd(adapter, RTW_MP_TX_CONTINUOUS, pmp_priv->rtw_mp_tx_method, bstart);
+}
+
+void rtw_mp_txpwr_level(_adapter *adapter)
+{
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	u8 rfpath_i = 0;
+	u8 tx_nss = get_phy_tx_nss(adapter, adapter_link);
+
+	for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+		rtw_mp_txpower_dbm(adapter, rfpath_i);
+
+	return;
+}
+
+/* ------------------------------------------------------------------------------ */
+static void dump_mpframe(_adapter *padapter, struct xmit_frame *pmpframe)
+{
+	/*rtw_hal_mgnt_xmit(padapter, pmpframe);*/
+	rtw_mgnt_xmit(padapter, pmpframe);
+}
+
+static struct xmit_frame *alloc_mp_xmitframe(struct xmit_priv *pxmitpriv)
+{
+	struct xmit_frame *pmpframe;
+
+	pmpframe = rtw_alloc_xmitframe(pxmitpriv, 0);
+	if (pmpframe == NULL)
+		return NULL;
+
+	pmpframe->frame_tag = MP_FRAMETAG;
+
+	return pmpframe;
+
+}
+
+static thread_return mp_xmit_packet_thread(thread_context context)
+{
+	struct xmit_frame	*pxmitframe;
+	struct mp_tx		*pmptx;
+	struct mp_priv	*pmp_priv;
+	struct xmit_priv	*pxmitpriv;
+	_adapter *padapter;
+
+	pmp_priv = (struct mp_priv *)context;
+	pmptx = &pmp_priv->tx;
+	padapter = pmp_priv->papdater;
+	pxmitpriv = &(padapter->xmitpriv);
+
+	rtw_thread_enter("RTW_MP_THREAD");
+
+	RTW_INFO("%s:pkTx Start\n", __func__);
+	while (1) {
+		pxmitframe = alloc_mp_xmitframe(pxmitpriv);
+
+		if (pxmitframe == NULL) {
+			if (pmptx->stop ||
+			    RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+				goto exit;
+			else {
+				rtw_usleep_os(10);
+				continue;
+			}
+		}
+		_rtw_memcpy((u8 *)(pxmitframe->buf_addr + TXDESC_OFFSET), pmptx->buf, pmptx->write_size);
+		_rtw_memcpy(&(pxmitframe->attrib), &(pmptx->attrib), sizeof(struct pkt_attrib));
+
+
+		rtw_usleep_os(padapter->mppriv.pktInterval);
+		dump_mpframe(padapter, pxmitframe);
+
+		pmptx->sended++;
+		pmp_priv->tx_pktcount++;
+
+		if (pmptx->stop ||
+		    RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			goto exit;
+		if ((pmptx->count != 0) &&
+		    (pmptx->count == pmptx->sended))
+			goto exit;
+
+		flush_signals_thread();
+	}
+
+exit:
+	/* RTW_INFO("%s:pkTx Exit\n", __func__); */
+	rtw_mfree(pmptx->pallocated_buf, pmptx->buf_size);
+	pmptx->pallocated_buf = NULL;
+	pmptx->stop = 1;
+
+	rtw_thread_exit(NULL);
+	return 0;
+}
+
+
+u8 rtw_phl_mp_tx_cmd(_adapter *padapter, enum rtw_mp_tx_cmd cmdid,
+							enum rtw_mp_tx_method tx_method, boolean bstart)
+{
+	struct mp_priv	*pmppriv = &padapter->mppriv;
+	struct pkt_attrib *pattrib = &pmppriv->tx.attrib;
+	struct rtw_mp_tx_arg tx_arg;
+	u16 i = 0;
+	u8 user_idx = pmppriv->mp_plcp_useridx;
+
+	_rtw_memset((void *)&tx_arg, 0, sizeof(struct rtw_mp_tx_arg));
+	tx_arg.mp_class = RTW_MP_CLASS_TX;
+	tx_arg.tx_method = tx_method;
+	tx_arg.tx_mode = (pmppriv->is_tmac_mode == 1)? 0:1;
+	tx_arg.cmd = cmdid;
+	tx_arg.cmd_ok = 0;
+	tx_arg.tx_ok = 0;
+	tx_arg.data_rate = pmppriv->rateidx;
+	tx_arg.is_cck = rtw_mp_is_cck_rate(pmppriv->rateidx);
+	tx_arg.start_tx = bstart;
+	tx_arg.tx_path = pmppriv->curr_rfpath;
+	tx_arg.gi = pmppriv->rtw_mp_plcp_gi;
+	tx_arg.period = pmppriv->pktInterval;
+	tx_arg.plcp_usr_idx = user_idx;
+	tx_arg.stbc = pmppriv->rtw_mp_stbc;
+	tx_arg.tx_state = 0;
+	tx_arg.apep = pattrib->pktlen;
+	pmppriv->mp_plcp_user[user_idx].coding = pmppriv->rtw_coding == 0 ? 10 : 12;  /* 1 LDPC, 0 BCC */
+
+	switch (cmdid) {
+		case RTW_MP_TX_PACKETS:
+			RTW_INFO("%s,SET MP_TX_PACKETS tx_method %d\n", __func__, tx_arg.tx_method);
+			if (tx_method == RTW_MP_PMACT_TX) {
+				tx_arg.tx_mode = RTW_MP_PMAC_PKTS_TX;
+				tx_arg.tx_cnt = pmppriv->tx.count;
+				tx_arg.tx_time = pmppriv->rtw_mp_tx_time;
+
+			} else if (tx_method == RTW_MP_FW_PMACT_TX) {
+				tx_arg.tx_mode = RTW_MP_PMAC_FW_TRIG_TX;
+				tx_arg.tx_cnt = 0;
+				tx_arg.tx_time = 60;
+			}
+
+			break;
+		case RTW_MP_TX_CONTINUOUS:
+			RTW_INFO("%s,SET MP_TX_CONTINUOUS\n", __func__);
+			tx_arg.tx_mode = RTW_MP_PMAC_CONT_TX;
+			break;
+		case RTW_MP_TX_SINGLE_TONE:
+			RTW_INFO("%s,SET TX_OFDM_Single_Tone\n", __func__);
+			tx_arg.tx_mode = RTW_MP_PMAC_OFDM_SINGLE_TONE_TX;
+			break;
+		case RTW_MP_TX_CCK_Carrier_Suppression:
+			RTW_INFO("%s,SET TX_CCK_Carrier_Suppression\n", __func__);
+			tx_arg.tx_mode = RTW_MP_PMAC_CCK_CARRIER_SIPPRESSION_TX;
+			break;
+		case RTW_MP_TX_CONFIG_PLCP_PATTERN:
+			RTW_INFO("%s,SET MP_TX_CONFIG_PLCP_PATTERN\n", __func__);
+			tx_arg.plcp_case_id = pmppriv->rtw_mp_pmact_patt_idx;
+			tx_arg.plcp_ppdu_type = pmppriv->rtw_mp_pmact_ppdu_type;
+			break;
+		case RTW_MP_TX_CONFIG_PLCP_COMMON_INFO:
+			tx_arg.dbw = pmppriv->rtw_mp_data_bandwidth;
+			tx_arg.long_preamble_en = pmppriv->preamble;
+			tx_arg.stbc = pmppriv->rtw_mp_stbc;
+			tx_arg.gi = pmppriv->rtw_mp_plcp_gi;
+			tx_arg.he_ltf_type = pmppriv->rtw_mp_plcp_ltf;
+			tx_arg.he_mcs_sigb = pmppriv->rtw_mp_he_sigb;
+			tx_arg.he_dcm_sigb = pmppriv->rtw_mp_he_sigb_dcm;
+			tx_arg.bss_color = 0;
+			tx_arg.he_er_u106ru_en = pmppriv->rtw_mp_he_er_su_ru_106_en;
+			if (pmppriv->rtw_mp_tx_method == RTW_MP_FW_PMACT_TX) {
+				pmppriv->rtw_mp_plcp_tx_time = 5484;
+				pmppriv->rtw_mp_plcp_tx_mode = 1;
+			}
+			tx_arg.max_tx_time_0p4us = pmppriv->rtw_mp_plcp_tx_time;
+			tx_arg.mode = pmppriv->rtw_mp_plcp_tx_mode;
+			tx_arg.ppdu_type = pmppriv->rtw_mp_pmact_ppdu_type;
+			tx_arg.cbw = pmppriv->bandwidth;
+			tx_arg.txsc = pmppriv->rtw_mp_trxsc;
+			tx_arg.n_user = pmppriv->rtw_mp_plcp_tx_user; 
+			tx_arg.ru_alloc = pmppriv->mp_plcp_user[user_idx].ru_alloc *2;
+
+			RTW_INFO("%s,SET TX_CONFIG_PLCP_COMMON_INFO\n", __func__);
+			RTW_INFO("%s=============================\n", __func__);
+			RTW_INFO("%s dbw = %d\n", __func__, tx_arg.dbw);
+			RTW_INFO("%s long_preamble_en = %d\n", __func__, tx_arg.long_preamble_en);
+			RTW_INFO("%s stbc = %d\n", __func__, tx_arg.stbc);
+			RTW_INFO("%s gi = %d\n", __func__, tx_arg.gi);
+			RTW_INFO("%s tb_l_len = %d\n", __func__, tx_arg.tb_l_len);
+			RTW_INFO("%s he_ltf_type = %d\n", __func__, tx_arg.he_ltf_type);
+			RTW_INFO("%s he_mcs_sigb = %d\n", __func__, tx_arg.he_mcs_sigb);
+			RTW_INFO("%s he_dcm_sigb = %d\n", __func__, tx_arg.he_dcm_sigb);
+			RTW_INFO("%s max_tx_time_0p4us = %d\n", __func__, tx_arg.max_tx_time_0p4us);
+			RTW_INFO("%s bss_color = %d\n", __func__, tx_arg.bss_color);
+			RTW_INFO("%s he_er_u106ru_en = %d\n", __func__, tx_arg.he_er_u106ru_en);
+			RTW_INFO("%s mode = %d\n", __func__, tx_arg.mode);
+			RTW_INFO("%s ppdu_type = %d\n", __func__, tx_arg.ppdu_type);
+			RTW_INFO("%s cbw = %d\n", __func__, tx_arg.cbw);
+			RTW_INFO("%s txsc = %d\n", __func__, tx_arg.txsc);
+			RTW_INFO("%s n_user = %d\n", __func__, tx_arg.n_user);
+			RTW_INFO("%s=============================\n", __func__);
+			break;
+		case RTW_MP_TX_CMD_PHY_OK:
+			/*TBD. API not ready*/
+			RTW_INFO("%s,SET MP_TX_CMD_PHY_OK\n", __func__);
+			break;
+		case RTW_MP_TX_CONFIG_PLCP_USER_INFO:
+			tx_arg.plcp_usr_idx = user_idx;
+			tx_arg.mcs = pmppriv->mp_plcp_user[user_idx].plcp_mcs;
+			tx_arg.fec = pmppriv->rtw_coding;
+			tx_arg.dcm = pmppriv->mp_plcp_user[user_idx].dcm;
+			tx_arg.aid = pmppriv->mp_plcp_user[user_idx].aid;
+			tx_arg.scrambler_seed = (rtw_random32() % 127) + 1;
+			tx_arg.random_init_seed = (rtw_random32() % 127) + 1;
+			tx_arg.apep = pmppriv->mp_plcp_user[user_idx].plcp_txlen;
+			tx_arg.ru_alloc = pmppriv->mp_plcp_user[user_idx].ru_alloc *2;
+			tx_arg.nss = pmppriv->mp_plcp_user[user_idx].plcp_nss + 1;
+			tx_arg.pwr_boost_db = pmppriv->mp_plcp_user[user_idx].pwr_boost_db;
+
+			RTW_INFO("%s,SET MP_TX_CONFIG_PLCP_USER_INFO\n", __func__);
+			RTW_INFO("%s plcp_usr_idx = %d\n", __func__, tx_arg.plcp_usr_idx);
+			RTW_INFO("%s mcs = %d\n", __func__, tx_arg.mcs);
+			RTW_INFO("%s fec = %d\n", __func__, tx_arg.fec);
+			RTW_INFO("%s dcm = %d\n", __func__, tx_arg.dcm);
+			RTW_INFO("%s ascrambler_seed = %d\n", __func__, tx_arg.scrambler_seed);
+			RTW_INFO("%s random_init_seed = %d\n", __func__, tx_arg.random_init_seed);
+			RTW_INFO("%s apep = %d\n", __func__, tx_arg.apep);
+			RTW_INFO("%s ru_alloc = %d\n", __func__, tx_arg.ru_alloc);
+			RTW_INFO("%s nss = %d\n", __func__, tx_arg.nss);
+			break;
+		case RTW_MP_TX_CHECK_TX_IDLE:
+			RTW_INFO("%s,GET RTW_MP_TX_CHECK_TX_IDLE !\n", __func__);
+			break;
+		case RTW_MP_TX_DPD_BYPASS:
+			RTW_INFO("%s,SET RTW_MP_TX_DPD_BYPASS !\n", __func__);
+			break;
+		default:
+				RTW_INFO("%s,SET MP_TX_MODE None\n", __func__);
+			return 0;
+	}
+
+	rtw_mp_set_phl_cmd(padapter, (void*)&tx_arg, sizeof(struct rtw_mp_tx_arg));
+	while (i != 100) {
+		rtw_mp_get_phl_cmd(padapter, (void*)&tx_arg, sizeof(struct rtw_mp_tx_arg));
+		if (tx_arg.cmd_ok && tx_arg.status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,SET CMD OK\n", __func__);
+			if (cmdid == RTW_MP_TX_CMD_PHY_OK) {
+				RTW_INFO("%s, Get Tx Rpt OK CNT:%d\n", __func__, tx_arg.tx_ok);
+				padapter->mppriv.tx.sended = tx_arg.tx_ok;
+			} else if (cmdid == RTW_MP_TX_CHECK_TX_IDLE){
+				RTW_INFO("%s, Get Tx state:%d\n", __func__, tx_arg.tx_state);
+				pmppriv->rtw_mp_tx_state = tx_arg.tx_state;
+			}
+			break;
+		} else {
+			if (i > 100) {
+				RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__, tx_arg.status);
+				break;
+			}
+			i++;
+			rtw_msleep_os(10);
+		}
+	}
+	return (tx_arg.cmd_ok);
+}
+
+thread_return mp_xmit_phl_packet_thread(thread_context context)
+{
+
+	struct mp_priv	*pmppriv = (struct mp_priv *)context;
+	struct mp_tx	*pmptx = &pmppriv->tx;
+	_adapter *padapter = pmppriv->papdater;
+	struct pkt_attrib *pattrib = &pmppriv->tx.attrib;
+	struct dvobj_priv	*dvobj = adapter_to_dvobj(padapter);
+	struct rtw_trx_test_param test_param = {0};
+	u8 payload;
+	u32 pkt_size = pattrib->pktlen;
+	u32 idle_cnt = 0, busy_cnt = 0, total_cnt = 0;
+	u16 i = 0;
+	u8 *ptr, *pkt_start, *pkt_end;
+
+
+	RTW_INFO("%s: Start !!\n", __func__);
+	if (pmptx->pallocated_buf)
+		rtw_mfree(pmptx->pallocated_buf, pmptx->buf_size);
+
+	pmptx->buf_size =  pattrib->pktlen;
+	pmptx->pallocated_buf = rtw_zmalloc(pmptx->buf_size);
+	if (pmptx->pallocated_buf == NULL) {
+		RTW_INFO("%s: malloc(%d) fail!!\n", __func__, pmppriv->tx.buf_size);
+		return 0;
+	}
+
+	ptr = pmptx->pallocated_buf;
+	pkt_start = ptr;
+	pkt_end = pkt_start + pkt_size;
+	switch (pmppriv->tx.payload) {
+		case MP_TX_Payload_00:
+			RTW_INFO("MP packet tx 0x00 payload!\n");
+			payload = 0x00;
+			_rtw_memset(ptr, 0x00, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_5a:
+			RTW_INFO("MP packet tx 0x5a payload!\n");
+			payload = 0x5a;
+			_rtw_memset(ptr, 0x5a, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_a5:
+			RTW_INFO("MP packet tx 0xa5 payload!\n");
+			payload = 0xa5;
+			_rtw_memset(ptr, 0xa5, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_ff:
+			RTW_INFO("MP packet tx 0xff payload!\n");
+			payload = 0xff;
+			_rtw_memset(ptr, 0xff, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_default_random:
+			RTW_INFO("MP packet tx default random payload!\n");
+			for (i = 0; i < pkt_end - ptr; i++)
+				ptr[i] = rtw_random32() % 0xFF;
+			break;
+		default:
+			RTW_INFO("Config payload type default use 0x%x\n!", pmppriv->tx.payload);
+			_rtw_memset(ptr, pmppriv->tx.payload, pkt_end - ptr);
+			break;
+	}
+
+	rtw_phl_trx_default_param(GET_PHL_INFO(dvobj), &test_param);
+
+	test_param.tx_cap.f_rate = (u16)pmppriv->rateidx;
+	test_param.tx_payload_size = pattrib->pktlen;
+	test_param.tx_cap.f_gi_ltf = 0;
+	test_param.tx_cap.f_stbc = 0;
+	test_param.tx_cap.f_ldpc = 0;
+//	test_param.tx_payload_buf = pmptx->pallocated_buf;
+
+	RTW_INFO("%s: Start !! tx method %d\n", __func__, pmppriv->rtw_mp_tx_method);
+	while (1) {
+		if (pmppriv->rtw_mp_tx_method == RTW_MP_TMACT_TX) {
+
+			/*/if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, pmppriv->rtw_mp_tx_method, _TRUE)) {*/
+			if (rtw_phl_trx_testsuite(GET_PHL_INFO(dvobj), &test_param) == 0)
+				RTW_DBG("%s: rtw_phl_trx_testsuite!!\n", __func__);
+			else
+				rtw_usleep_os(10);
+		}
+
+				pmptx->sended++;
+				pmppriv->tx_pktcount++;
+
+		rtw_usleep_os(padapter->mppriv.pktInterval);
+
+		if (pmptx->stop || RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			goto exit;
+		if ((pmptx->count != 0) && (pmptx->count == pmptx->sended))
+			goto exit;
+		flush_signals_thread();
+	}
+exit:
+	pmptx->stop = 1;
+
+	RTW_INFO("%s: Exit !!\n", __func__);
+	rtw_thread_exit(NULL);
+
+//	rtw_phl_get_tx_req_stats(GET_PHL_INFO(dvobj),
+//				&idle_cnt, &busy_cnt, &total_cnt);
+	RTW_INFO("%s: tx req idle cnt=%d ,busy=%d, total=%d!! i =%d\n",
+				__func__, idle_cnt, busy_cnt, total_cnt, i);
+
+	return 0;
+}
+
+static void rtw_get_tx_idle(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	struct rtw_mp_ax_tx_cnt tx_cnt = {0};
+	struct rtw_mp_ax_tx_cnt tx_cnt_new = {0};
+	u8 i, j;
+
+	pmp_priv->rtw_mp_tx_state = 0;
+	for (j = 0; j < 100 ; j++) {
+		int t = 0, cnt = 0;
+
+		rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_CHECK_TX_IDLE,
+							pmp_priv->rtw_mp_tx_method, _FALSE);
+
+		RTW_INFO("%s CHK CNT : %d, tx state %d\n", __func__, cnt, pmp_priv->rtw_mp_tx_state);
+		if (pmp_priv->rtw_mp_tx_state == 1) {
+			rtw_phl_get_mac_sel_tx_status(dvobj->phl, 0, (void*)&tx_cnt);
+			RTW_INFO("[MP 1] TX CNT - LCCK : %d\n", tx_cnt.txcnt[0]);
+			RTW_INFO("[MP 1] TX CNT - SCCK : %d\n", tx_cnt.txcnt[1]);
+			RTW_INFO("[MP 1] TX CNT - OFDM : %d\n", tx_cnt.txcnt[2]);
+			RTW_INFO("[MP 1] TX CNT - HT : %d\n", tx_cnt.txcnt[3]);
+			RTW_INFO("[MP 1] TX CNT - HTGF : %d\n", tx_cnt.txcnt[4]);
+			RTW_INFO("[MP 1] TX CNT - VHTSU : %d\n", tx_cnt.txcnt[5]);
+			RTW_INFO("[MP 1] TX CNT - VHTMU : %d\n", tx_cnt.txcnt[6]);
+			RTW_INFO("[MP 1] TX CNT - HESU : %d\n", tx_cnt.txcnt[7]);
+			RTW_INFO("[MP 1] TX CNT - HEERSU : %d\n", tx_cnt.txcnt[8]);
+			RTW_INFO("[MP 1] TX CNT - HEMU : %d\n", tx_cnt.txcnt[9]);
+			RTW_INFO("[MP 1] TX CNT - HETB : %d\n", tx_cnt.txcnt[10]);
+			rtw_phl_get_mac_sel_tx_status(dvobj->phl, 0, (void*)&tx_cnt_new);
+			RTW_INFO("[MP 2] TX CNT - LCCK : %d\n", tx_cnt_new.txcnt[0]);
+			RTW_INFO("[MP 2] TX CNT - SCCK : %d\n", tx_cnt_new.txcnt[1]);
+			RTW_INFO("[MP 2] TX CNT - OFDM : %d\n", tx_cnt_new.txcnt[2]);
+			RTW_INFO("[MP 2] TX CNT - HT : %d\n", tx_cnt_new.txcnt[3]);
+			RTW_INFO("[MP 2] TX CNT - HTGF : %d\n", tx_cnt_new.txcnt[4]);
+			RTW_INFO("[MP 2] TX CNT - VHTSU : %d\n", tx_cnt_new.txcnt[5]);
+			RTW_INFO("[MP 2] TX CNT - VHTMU : %d\n", tx_cnt_new.txcnt[6]);
+			RTW_INFO("[MP 2] TX CNT - HESU : %d\n", tx_cnt_new.txcnt[7]);
+			RTW_INFO("[MP 2] TX CNT - HEERSU : %d\n", tx_cnt_new.txcnt[8]);
+			RTW_INFO("[MP 2] TX CNT - HEMU : %d\n", tx_cnt_new.txcnt[9]);
+			RTW_INFO("[MP 2] TX CNT - HETB : %d\n", tx_cnt_new.txcnt[10]);
+			for (i = 0; i < 10 ; i++) {
+				if (tx_cnt.txcnt[i] == 0) {
+					t = 1;
+					continue;
+				}
+				if (tx_cnt_new.txcnt[i] > tx_cnt.txcnt[i]) {
+					t = 1;
+					break;
+				} else if (tx_cnt_new.txcnt[i] == tx_cnt.txcnt[i]) {
+					cnt++;
+				}
+			}
+			rtw_usleep_os(500);
+			RTW_INFO("[MP 2] TX CNT : %d\n", cnt);
+			}
+		if (t == 1 || cnt ==10)
+			break;
+		rtw_msleep_os(1);
+	}
+	return;
+}
+
+
+static enum rf_path mp_get_path_from_ant_num(u8 antnum)
+{
+	enum rf_path ret = RF_PATH_B;
+
+	switch (antnum) {
+		default:
+			break;
+		case 1:
+			ret = RF_PATH_B;
+			break;
+		case 2:
+			ret = RF_PATH_AB;
+			break;
+		case 3:
+			ret = RF_PATH_ABC;
+			break;
+	}
+	return ret;
+}
+
+static void rtw_dpd_bypass(_adapter *padapter, u8 phy_idx)
+{
+	u32 data32 = 0;
+	u32 dpd_bypass_val = 0;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_com_t *phl_com = dvobj->phl_com;
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	u8 bk_rfpath = pmp_priv->curr_rfpath;
+
+	if (pmp_priv->rtw_mp_tx_method != RTW_MP_TMACT_TX) {
+		pmp_priv->is_tmac_mode = 1;
+		pmp_priv->curr_rfpath = mp_get_path_from_ant_num(phl_com->phy_cap[0].tx_path_num);
+		rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_TXRX_MODE);
+	}
+
+	if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, RTW_MP_SW_TX, _TRUE))
+		RTW_INFO("%s: RTW_MP_SW_TX!!\n", __func__);
+	else
+		RTW_INFO("%s: RTW_MP_SW_TX fail!!\n", __func__);
+
+	if (phy_idx == 0) {
+		data32 = rtw_phl_read32(dvobj->phl, 0xD6F0);
+		RTW_INFO("%s:Phy0 read 0xD6F0 = 0x%x !!\n", __func__, data32);
+		dpd_bypass_val = ((data32 & 0x4000000) >> 26);
+		RTW_INFO("%s: dpd_bypass_val = 0x%x !!\n", __func__, dpd_bypass_val);
+	} else if (phy_idx == 1) {
+		data32 = rtw_phl_read32(dvobj->phl, 0xF6F0);
+		RTW_INFO("%s:Phy1 read 0xF6F0 = 0x%x !!\n", __func__, data32);
+		dpd_bypass_val = ((data32 & 0x4000000) >> 26);
+		RTW_INFO("%s: dpd_bypass_val = 0x%x !!\n", __func__, dpd_bypass_val);
+	}
+	if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_DPD_BYPASS,
+							pmp_priv->rtw_mp_tx_method, _FALSE))
+		RTW_INFO("%s: RTW_MP_TX_DPD_BYPASS!!\n", __func__);
+
+	rtw_get_tx_idle(padapter);
+
+	if (pmp_priv->rtw_mp_tx_method != RTW_MP_TMACT_TX) {
+		pmp_priv->is_tmac_mode = 0;
+		pmp_priv->curr_rfpath = bk_rfpath;
+		rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_TXRX_MODE);
+	}
+}
+
+static void rtw_pretx_trkdpk(_adapter *padapter)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	u8 tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	u8 rfpath_i = 0;
+	u8 tssi_mode = pmp_priv->tssi_mode;
+	u32 bk_tx_stop = pmp_priv->tx.stop;
+	u32 bk_tx_count = pmp_priv->tx.count;
+	s16 bk_txpwr = mpt_get_tx_power_finalabs_val(padapter, tx_nss -1);
+
+	rtw_get_tx_idle(padapter);
+
+	if (pmp_priv->bSetTxPower == 1)
+		bk_txpwr = pmp_priv->txpowerdbm;
+
+	pmp_priv->txpowerdbm = 17 * TX_POWER_BASE ;
+	for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+		rtw_mp_txpower_dbm(padapter, rfpath_i);
+
+	pmp_priv->tx.stop = 0;
+	pmp_priv->tx.count = 0;
+	pmp_priv->rtw_mp_tx_time = 5484;
+
+	if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, RTW_MP_PMACT_TX, _TRUE))
+		RTW_INFO("%s: RTW_MP_TX_PACKETS!!\n", __func__);
+
+	rtw_msleep_os(100);
+
+	if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, RTW_MP_PMACT_TX, _FALSE))
+		RTW_INFO("%s: RTW_MP_TX_PACKETS Done!!\n", __func__);
+
+	rtw_mp_cal_trigger(padapter, RTW_MP_CAL_DPK_TRACK);
+
+	/*End Workaround to trackiing DPK */
+	pmp_priv->tx.stop = bk_tx_stop;
+	pmp_priv->tx.count = bk_tx_count;
+	pmp_priv->rtw_mp_tx_time = 0;
+
+	pmp_priv->txpowerdbm = bk_txpwr;
+
+	for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+		rtw_mp_txpower_dbm(padapter, rfpath_i);
+
+
+}
+
+void rtw_set_phl_packet_tx(_adapter *padapter, u8 bStart)
+{
+	struct mp_priv *pmp_priv;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u8 rfpath_i = 0;
+	u32 val32 = 0;
+	u8 tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	pmp_priv = &padapter->mppriv;
+
+	RTW_INFO("%s: PACKET TX tx method %d!!\n", __func__, pmp_priv->rtw_mp_tx_method);
+
+	if (bStart) {
+		if (pmp_priv->tssi_mode >= RTW_MP_TSSI_ON && pmp_priv->bspecif_tssi_de) {
+			for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+				rtw_mp_set_tsside2verify(padapter, pmp_priv->specif_tsside_val, rfpath_i);
+			}
+			val32 = rtw_phl_read32(dvobj->phl, 0xC340);
+			val32 &= 0xFFFFF800;
+			rtw_phl_write32(dvobj->phl, 0xC340, val32);
+
+			val32 = rtw_phl_read32(dvobj->phl, 0x12344);/*B mode CCA off*/
+			val32 |= BIT31;
+			rtw_phl_write32(dvobj->phl, 0x12344, val32);
+
+			val32 = rtw_phl_read32(dvobj->phl, 0x10c3c);/*OFDM CCA off*/
+			val32 |= BIT9;
+			rtw_phl_write32(dvobj->phl, 0x10c3c, val32);
+			RTW_INFO("%s: !! tx method %d\n", __func__, pmp_priv->rtw_mp_tx_method);
+			pmp_priv->tx.sended = 0;
+			pmp_priv->tx.stop = 0;
+			pmp_priv->tx_pktcount = 0;
+			if (pmp_priv->rtw_mp_tx_method == RTW_MP_PMACT_TX ||
+					pmp_priv->rtw_mp_tx_method == RTW_MP_FW_PMACT_TX) {
+					RTW_INFO("%s: PLCP_USER_INFO & PLCP_COMMON_INFO!!\n", __func__);
+					if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_CONFIG_PLCP_USER_INFO,
+											pmp_priv->rtw_mp_tx_method, _TRUE) == true)
+						RTW_INFO("%s: PLCP_USER_INFO done!!\n", __func__);
+					if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_CONFIG_PLCP_COMMON_INFO,
+												pmp_priv->rtw_mp_tx_method, _TRUE) == true)
+						RTW_INFO("%s: RTW_MP_TX_CONFIG_PLCP_COMMON_INFO done!!\n", __func__);
+
+					rtw_pretx_trkdpk(padapter);
+			}
+			if (pmp_priv->bloopback != _TRUE)
+				rtw_dpd_bypass(padapter, pmp_priv->rtw_mp_cur_phy);
+
+			if (pmp_priv->rtw_mp_tx_method == RTW_MP_FW_PMACT_TX) {
+				if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, pmp_priv->rtw_mp_tx_method, _TRUE))
+					RTW_INFO("%s: RTW_MP_FW_PMACT_TX!!\n", __func__);
+				else
+					RTW_INFO("%s: RTW_MP_FW_PMACT_TX fail!!\n", __func__);
+			} else if (pmp_priv->rtw_mp_tx_method == RTW_MP_PMACT_TX) {
+				if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, pmp_priv->rtw_mp_tx_method, _TRUE))
+					RTW_DBG("%s: RTW_MP_TX_PACKETS!!\n", __func__);
+			} else {
+				pmp_priv->tx.PktTxThread = rtw_thread_start(
+										mp_xmit_phl_packet_thread, pmp_priv, "RTW_MP_Tx_THREAD");
+				if (pmp_priv->tx.PktTxThread == NULL)
+					RTW_ERR("Create PktTx Thread Fail !!!!!\n");
+			}
+
+		for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+			rtw_mp_txpower_dbm(padapter, rfpath_i);
+
+		pmp_priv->bspecif_tssi_de = false;
+		pmp_priv->specif_tsside_val = 0;
+		pmp_priv->bSetTxPower = 0;
+	}
+	if (!bStart) {
+			val32 = rtw_phl_read32(dvobj->phl, 0xC340);
+			val32 |= 0x000007FF;
+			rtw_phl_write32(dvobj->phl, 0xC340, val32);
+			val32 = rtw_phl_read32(dvobj->phl, 0x12344);/*B mode CCA on*/
+			val32 &= 0xEFFFFFFF;
+			rtw_phl_write32(dvobj->phl, 0x12344, val32);
+
+			val32 = rtw_phl_read32(dvobj->phl, 0x10c3c);/*OFDM CCA on*/
+			val32 &= 0xFFFFFDFF;
+			rtw_phl_write32(dvobj->phl, 0x10c3c, val32);
+			if (pmp_priv->rtw_mp_tx_method == RTW_MP_FW_PMACT_TX) {
+				if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, pmp_priv->rtw_mp_tx_method, false)) {
+					RTW_INFO("%s: RTW_MP_FW_TMACT_TX Stop TX!!\n", __func__);
+					return;
+				}
+			} else if (pmp_priv->rtw_mp_tx_method == RTW_MP_PMACT_TX) {
+				if (rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_PACKETS, pmp_priv->rtw_mp_tx_method, false)) {
+					RTW_INFO("%s: RTW_MP_PMACT_TX Stop TX!!\n", __func__);
+					return;
+				}
+			}
+		}
+
+}
+
+void rtw_pre_phl_packet_tx(_adapter *padapter, u8 bStart)
+{
+	struct mp_priv *pmp_priv;
+	pmp_priv = &padapter->mppriv;
+
+	if (bStart) {
+		RTW_INFO("%s: !! tx method %d\n", __func__, pmp_priv->rtw_mp_tx_method);
+		pmp_priv->tx.sended = 0;
+		pmp_priv->tx.stop = 0;
+		pmp_priv->tx_pktcount = 0;
+
+		rtw_dpd_bypass(padapter, pmp_priv->rtw_mp_cur_phy);
+		rtw_get_tx_idle(padapter);
+
+		pmp_priv->tx.PktTxThread = rtw_thread_start(
+				mp_xmit_phl_packet_thread, pmp_priv, "RTW_MP_Tx_THREAD");
+		if (pmp_priv->tx.PktTxThread == NULL)
+			RTW_ERR("Create PktTx Thread Fail !!!!!\n");
+	}
+}
+
+
+void rtw_mp_set_packet_tx(_adapter *padapter)
+{
+	u8 *ptr, *pkt_start, *pkt_end;
+	u32 pkt_size = 0, i = 0, idx = 0, tmp_idx = 0;
+	struct rtw_ieee80211_hdr *hdr;
+	u8 payload;
+	s32 bmcast;
+	struct pkt_attrib *pattrib;
+	struct mp_priv *pmp_priv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	pmp_priv = &padapter->mppriv;
+
+	if (pmp_priv->tx.stop)
+		return;
+	pmp_priv->tx.sended = 0;
+	pmp_priv->tx.stop = 0;
+	pmp_priv->tx_pktcount = 0;
+
+	/* 3 1. update_attrib() */
+	pattrib = &pmp_priv->tx.attrib;
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
+	bmcast = IS_MCAST(pattrib->ra);
+	if (bmcast)
+		pattrib->psta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+	else
+		pattrib->psta = rtw_get_stainfo(&padapter->stapriv, get_bssid(&padapter->mlmepriv));
+
+	if (pattrib->psta == NULL) {
+		RTW_INFO("%s:psta = NULL !!\n", __func__);
+		return;
+	}
+
+	pattrib->mac_id = pattrib->psta->phl_sta->macid;
+	pattrib->mbssid = 0;
+
+	pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->pktlen;
+
+	/* 3 2. allocate xmit buffer */
+	pkt_size = pattrib->last_txcmdsz;
+
+	if (pmp_priv->tx.pallocated_buf)
+		rtw_mfree(pmp_priv->tx.pallocated_buf, pmp_priv->tx.buf_size);
+	pmp_priv->tx.write_size = pkt_size;
+	pmp_priv->tx.buf_size = pkt_size + SZ_ALIGN_XMITFRAME_EXT;
+	pmp_priv->tx.pallocated_buf = rtw_zmalloc(pmp_priv->tx.buf_size);
+	if (pmp_priv->tx.pallocated_buf == NULL) {
+		RTW_INFO("%s: malloc(%d) fail!!\n", __func__, pmp_priv->tx.buf_size);
+		return;
+	}
+	pmp_priv->tx.buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pmp_priv->tx.pallocated_buf), SZ_ALIGN_XMITFRAME_EXT);
+	ptr = pmp_priv->tx.buf;
+
+	_rtw_memset(pmp_priv->tx.desc, 0, TXDESC_SIZE);
+	pkt_start = ptr;
+	pkt_end = pkt_start + pkt_size;
+
+	/* 3 3. init TX descriptor */
+#if defined(CONFIG_RTL8822B)
+	if (IS_HARDWARE_TYPE_8822B(padapter))
+		rtl8822b_prepare_mp_txdesc(padapter, pmp_priv);
+#endif /* CONFIG_RTL8822B */
+
+#if defined(CONFIG_RTL8822C)
+	if (IS_HARDWARE_TYPE_8822C(padapter))
+		rtl8822c_prepare_mp_txdesc(padapter, pmp_priv);
+#endif /* CONFIG_RTL8822C */
+
+#if defined(CONFIG_RTL8821C)
+	if (IS_HARDWARE_TYPE_8821C(padapter))
+		rtl8821c_prepare_mp_txdesc(padapter, pmp_priv);
+#endif /* CONFIG_RTL8821C */
+
+#if defined(CONFIG_RTL8814B)
+	if (IS_HARDWARE_TYPE_8814B(padapter))
+		rtl8814b_prepare_mp_txdesc(padapter, pmp_priv);
+#endif /* CONFIG_RTL8814B */
+
+	/* 3 4. make wlan header, make_wlanhdr() */
+	hdr = (struct rtw_ieee80211_hdr *)pkt_start;
+	set_frame_sub_type(&hdr->frame_ctl, pattrib->subtype);
+
+	_rtw_memcpy(hdr->addr1, pattrib->dst, ETH_ALEN); /* DA */
+	_rtw_memcpy(hdr->addr2, pattrib->src, ETH_ALEN); /* SA */
+	_rtw_memcpy(hdr->addr3, get_bssid(&padapter->mlmepriv), ETH_ALEN); /* RA, BSSID */
+
+	/* 3 5. make payload */
+	ptr = pkt_start + pattrib->hdrlen;
+
+	if (pmp_priv->mplink_btx == _TRUE) {
+		_rtw_memcpy(ptr, pmp_priv->mplink_buf, pkt_end - ptr);
+	} else {
+		switch (pmp_priv->tx.payload) {
+		case MP_TX_Payload_00:
+			RTW_INFO("MP packet tx 0x00 payload!\n");
+			payload = 0x00;
+			_rtw_memset(ptr, 0x00, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_5a:
+			RTW_INFO("MP packet tx 0x5a payload!\n");
+			payload = 0x5a;
+			_rtw_memset(ptr, 0x5a, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_a5:
+			RTW_INFO("MP packet tx 0xa5 payload!\n");
+			payload = 0xa5;
+			_rtw_memset(ptr, 0xa5, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_ff:
+			RTW_INFO("MP packet tx 0xff payload!\n");
+			payload = 0xff;
+			_rtw_memset(ptr, 0xff, pkt_end - ptr);
+			break;
+		case MP_TX_Payload_prbs9:
+			RTW_INFO("MP packet tx PRBS9 payload!\n");
+			while (idx <= pkt_end - ptr) {
+				int start = 0x02;
+				int a = start;
+
+				for (i = 0;; i++) {
+						int newbit = (((a >> 8) ^ (a >> 4)) & 1);
+						a = ((a << 1) | newbit) & 0x1ff;
+						RTW_DBG("%x ", a);
+						ptr[idx + i] = a;
+
+						if (a == start) {
+							RTW_INFO("payload repetition period is %d , end %d\n", i , idx);
+							tmp_idx += i;
+							break;
+						}
+						if (idx + i >= (pkt_end - ptr)) {
+							tmp_idx += (idx + i);
+							RTW_INFO(" repetition period payload end curr ptr %d\n", idx + i);
+							break;
+						}
+				}
+				idx = tmp_idx;
+			}
+			break;
+		case MP_TX_Payload_default_random:
+			RTW_INFO("MP packet tx default random payload!\n");
+			for (i = 0; i < pkt_end - ptr; i++)
+				ptr[i] = rtw_random32() % 0xFF;
+			break;
+		default:
+			RTW_INFO("Config payload type default use 0x%x\n!", pmp_priv->tx.payload);
+			_rtw_memset(ptr, pmp_priv->tx.payload, pkt_end - ptr);
+			break;
+		}
+	}
+	/* 3 6. start thread */
+#ifdef PLATFORM_LINUX
+	pmp_priv->tx.PktTxThread = rtw_thread_start(mp_xmit_packet_thread, pmp_priv, "RTW_MP_THREAD");
+	if (pmp_priv->tx.PktTxThread == NULL)
+		RTW_ERR("Create PktTx Thread Fail !!!!!\n");
+#endif
+#ifdef PLATFORM_FREEBSD
+	{
+		struct proc *p;
+		struct thread *td;
+		pmp_priv->tx.PktTxThread = kproc_kthread_add(mp_xmit_packet_thread, pmp_priv,
+			&p, &td, RFHIGHPID, 0, "MPXmitThread", "MPXmitThread");
+
+		if (pmp_priv->tx.PktTxThread < 0)
+			RTW_INFO("Create PktTx Thread Fail !!!!!\n");
+	}
+#endif
+	return;
+}
+
+void rtw_mp_reset_phy_count(_adapter *adapter)
+{
+	RTW_INFO("%s()!!!\n", __func__);
+	rtw_mp_phl_config_arg(adapter, RTW_MP_CONFIG_CMD_SET_RESET_PHY_COUNT);
+	rtw_mp_phl_config_arg(adapter, RTW_MP_CONFIG_CMD_SET_RESET_MAC_COUNT);
+	rtw_mp_phl_config_arg(adapter, RTW_MP_CONFIG_CMD_SET_RESET_DRV_COUNT);
+}
+
+struct psd_init_regs {
+	/* 3 wire */
+	int reg_88c;
+	int reg_c00;
+	int reg_e00;
+	int reg_1800;
+	int reg_1a00;
+	/* cck */
+	int reg_800;
+	int reg_808;
+};
+
+#if 0
+static int rtw_mp_psd_init(_adapter *padapter, struct psd_init_regs *regs)
+{
+	u8 rf_type = GET_HAL_RFPATH(adapter_to_dvobj(padapter));
+#if 0
+	switch (rf_type) {
+	/* 1R */
+	case RF_1T1R:
+		if (rtw_hw_chk_proto_cap(padapter, PROTO_CAP_11AC)) {
+			/* 11AC 1R PSD Setting 3wire & cck off */
+			regs->reg_c00 = rtw_read32(padapter, 0xC00);
+			phy_set_bb_reg(padapter, 0xC00, 0x3, 0x00);
+			regs->reg_808 = rtw_read32(padapter, 0x808);
+			phy_set_bb_reg(padapter, 0x808, 0x10000000, 0x0);
+		} else {
+			/* 11N 3-wire off 1 */
+			regs->reg_88c = rtw_read32(padapter, 0x88C);
+			phy_set_bb_reg(padapter, 0x88C, 0x300000, 0x3);
+			/* 11N CCK off */
+			regs->reg_800 = rtw_read32(padapter, 0x800);
+			phy_set_bb_reg(padapter, 0x800, 0x1000000, 0x0);
+		}
+	break;
+
+	/* 2R */
+	case RF_1T2R:
+	case RF_2T2R:
+		if (rtw_hw_chk_proto_cap(padapter, PROTO_CAP_11AC)) {
+			/* 11AC 2R PSD Setting 3wire & cck off */
+			regs->reg_c00 = rtw_read32(padapter, 0xC00);
+			regs->reg_e00 = rtw_read32(padapter, 0xE00);
+			phy_set_bb_reg(padapter, 0xC00, 0x3, 0x00);
+			phy_set_bb_reg(padapter, 0xE00, 0x3, 0x00);
+			regs->reg_808 = rtw_read32(padapter, 0x808);
+			phy_set_bb_reg(padapter, 0x808, 0x10000000, 0x0);
+		} else {
+			/* 11N 3-wire off 2 */
+			regs->reg_88c = rtw_read32(padapter, 0x88C);
+			phy_set_bb_reg(padapter, 0x88C, 0xF00000, 0xF);
+			/* 11N CCK off */
+			regs->reg_800 = rtw_read32(padapter, 0x800);
+			phy_set_bb_reg(padapter, 0x800, 0x1000000, 0x0);
+		}
+	break;
+
+	/* 3R */
+	case RF_2T3R:
+	case RF_3T3R:
+		if (rtw_hw_chk_proto_cap(padapter, PROTO_CAP_11AC)) {
+			/* 11AC 3R PSD Setting 3wire & cck off */
+			regs->reg_c00 = rtw_read32(padapter, 0xC00);
+			regs->reg_e00 = rtw_read32(padapter, 0xE00);
+			regs->reg_1800 = rtw_read32(padapter, 0x1800);
+			phy_set_bb_reg(padapter, 0xC00, 0x3, 0x00);
+			phy_set_bb_reg(padapter, 0xE00, 0x3, 0x00);
+			phy_set_bb_reg(padapter, 0x1800, 0x3, 0x00);
+			regs->reg_808 = rtw_read32(padapter, 0x808);
+			phy_set_bb_reg(padapter, 0x808, 0x10000000, 0x0);
+		} else {
+			RTW_ERR("%s: 11n don't support 3R\n", __func__);
+			return -1;
+		}
+		break;
+
+	/* 4R */
+	case RF_2T4R:
+	case RF_3T4R:
+	case RF_4T4R:
+		if (rtw_hw_chk_proto_cap(padapter, PROTO_CAP_11AC)) {
+			/* 11AC 4R PSD Setting 3wire & cck off */
+			regs->reg_c00 = rtw_read32(padapter, 0xC00);
+			regs->reg_e00 = rtw_read32(padapter, 0xE00);
+			regs->reg_1800 = rtw_read32(padapter, 0x1800);
+			regs->reg_1a00 = rtw_read32(padapter, 0x1A00);
+			phy_set_bb_reg(padapter, 0xC00, 0x3, 0x00);
+			phy_set_bb_reg(padapter, 0xE00, 0x3, 0x00);
+			phy_set_bb_reg(padapter, 0x1800, 0x3, 0x00);
+			phy_set_bb_reg(padapter, 0x1A00, 0x3, 0x00);
+			regs->reg_808 = rtw_read32(padapter, 0x808);
+			phy_set_bb_reg(padapter, 0x808, 0x10000000, 0x0);
+		} else {
+			RTW_ERR("%s: 11n don't support 4R\n", __func__);
+			return -1;
+		}
+		break;
+
+	default:
+		RTW_ERR("%s: unknown %d rf type\n", __func__,
+			GET_HAL_RFPATH(adapter_to_dvobj(padapter)));
+		return -1;
+	}
+
+	/* Set PSD points, 0=128, 1=256, 2=512, 3=1024 */
+	if (rtw_hw_chk_proto_cap(padapter, PROTO_CAP_11AC))
+		phy_set_bb_reg(padapter, 0x910, 0xC000, 3);
+	else
+		phy_set_bb_reg(padapter, 0x808, 0xC000, 3);
+#endif
+	RTW_INFO("%s: set %d rf type done\n", __func__,
+		GET_HAL_RFPATH(adapter_to_dvobj(padapter)));
+	return 0;
+}
+
+static int rtw_mp_psd_close(_adapter *padapter, struct psd_init_regs *regs)
+{
+	u8 rf_type = GET_HAL_RFPATH(adapter_to_dvobj(padapter));
+
+#if 0
+	if (!rtw_hw_chk_proto_cap(padapter, PROTO_CAP_11AC)) {
+		/* 11n 3wire restore */
+		rtw_write32(padapter, 0x88C, regs->reg_88c);
+		/* 11n cck restore */
+		rtw_write32(padapter, 0x800, regs->reg_800);
+		RTW_INFO("%s: restore %d rf type\n", __func__, rf_type);
+		return 0;
+	}
+
+	/* 11ac 3wire restore */
+	switch (rf_type) {
+	case RF_1T1R:
+		rtw_write32(padapter, 0xC00, regs->reg_c00);
+		break;
+	case RF_1T2R:
+	case RF_2T2R:
+		rtw_write32(padapter, 0xC00, regs->reg_c00);
+		rtw_write32(padapter, 0xE00, regs->reg_e00);
+		break;
+	case RF_2T3R:
+	case RF_3T3R:
+		rtw_write32(padapter, 0xC00, regs->reg_c00);
+		rtw_write32(padapter, 0xE00, regs->reg_e00);
+		rtw_write32(padapter, 0x1800, regs->reg_1800);
+		break;
+	case RF_2T4R:
+	case RF_3T4R:
+	case RF_4T4R:
+		rtw_write32(padapter, 0xC00, regs->reg_c00);
+		rtw_write32(padapter, 0xE00, regs->reg_e00);
+		rtw_write32(padapter, 0x1800, regs->reg_1800);
+		rtw_write32(padapter, 0x1A00, regs->reg_1a00);
+		break;
+	default:
+		RTW_WARN("%s: unknown %d rf type\n", __func__, rf_type);
+		break;
+	}
+
+	/* 11ac cck restore */
+	rtw_write32(padapter, 0x808, regs->reg_808);
+#endif	
+	RTW_INFO("%s: restore %d rf type done\n", __func__, rf_type);
+	return 0;
+}
+
+/* reg 0x808[9:0]: FFT data x
+ * reg 0x808[22]:  0  -->  1  to get 1 FFT data y
+ * reg 0x8B4[15:0]: FFT data y report */
+static u32 rtw_GetPSDData(_adapter *adapter, u32 point)
+{
+	u32 psd_val = 0;
+#if 0
+#if defined(CONFIG_RTL8822B) || defined(CONFIG_RTL8821C) || defined(CONFIG_RTL8822C)
+	u16 psd_reg = 0x910;
+	u16 psd_regL = 0xF44;
+#else
+	u16 psd_reg = 0x808;
+	u16 psd_regL = 0x8B4;
+#endif
+
+	psd_val = rtw_read32(adapter, psd_reg);
+
+	psd_val &= 0xFFBFFC00;
+	psd_val |= point;
+
+	rtw_write32(adapter, psd_reg, psd_val);
+	rtw_mdelay_os(1);
+	psd_val |= 0x00400000;
+
+	rtw_write32(adapter, psd_reg, psd_val);
+	rtw_mdelay_os(1);
+
+	psd_val = rtw_read32(adapter, psd_regL);
+#if defined(CONFIG_RTL8821C)
+	psd_val = (psd_val & 0x00FFFFFF) / 32;
+#else
+	psd_val &= 0x0000FFFF;
+#endif
+#endif
+	return psd_val;
+}
+#endif
+
+u8 rtw_mp_phl_psd_cmd(_adapter *padapter, struct rtw_mp_cal_arg	*psd_arg, u8 cmdid)
+{
+	struct mp_priv	*pmppriv = &padapter->mppriv;
+	u16 i = 0;
+	u32 cmd_size = sizeof(struct rtw_mp_cal_arg);
+
+	psd_arg->mp_class = RTW_MP_CLASS_CAL;
+	psd_arg->cmd = cmdid;
+
+	rtw_mp_set_phl_cmd(padapter, (void*)psd_arg, cmd_size);
+	while (i <= 100) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(padapter, (void*)psd_arg, cmd_size);
+		if (psd_arg->cmd_ok && psd_arg->status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,SET CMD OK\n", __func__);
+			return psd_arg->cmd_ok;
+		} else {
+			if (i > 100) {
+				RTW_DBG("%s,GET CMD FAIL !!! status %d\n", __func__, psd_arg->status);
+				break;
+			}
+			i++;
+		}
+	}
+	return 0;
+}
+
+/*
+ * pts	start_point_min		stop_point_max
+ * 128	64			64 + 128 = 192
+ * 256	128			128 + 256 = 384
+ * 512	256			256 + 512 = 768
+ * 1024	512			512 + 1024 = 1536
+ *
+ */
+u32 mp_query_psd(_adapter *adapter, u8 *data)
+{
+	struct mp_priv *pmppriv = &adapter->mppriv;
+	struct rtw_mp_cal_arg	*my_psd_arg = NULL;
+
+	u32 i, psd_pts = 0, psd_start = 0, psd_stop = 0;
+	u32 fft = 0, avg = 0, iq_path = 0;
+	u32 psd_data = 0;
+	char *pdata = NULL;
+
+	my_psd_arg = _rtw_malloc(sizeof(struct rtw_mp_cal_arg));
+
+	if (my_psd_arg)
+		_rtw_memset((void *)my_psd_arg, 0, sizeof(struct rtw_mp_cal_arg));
+	else {
+		RTW_ERR("%s() psd_arg malloc fail !\n", __func__);
+		return 0;
+	}
+
+	if (sscanf(data, "fft=%d,avg=%d,iq=%d", &fft, &avg, &iq_path) == 3) {
+
+		my_psd_arg->path = pmppriv->curr_rfpath;
+		my_psd_arg->iq_path = iq_path;
+		my_psd_arg->avg = avg;
+		my_psd_arg->fft = fft;
+
+		RTW_INFO("%s: PSD RF_Path:%d fft=%d,avg=%d,iq_path=%d !\n",
+			__func__, pmppriv->antenna_trx, fft, avg, iq_path);
+
+		if (rtw_mp_phl_psd_cmd(adapter, my_psd_arg, RTW_MP_CAL_CMD_PSD_INIT))
+			RTW_INFO("PSD_INIT CMD OK!\n");
+		else
+			RTW_INFO("PSD_INIT CMD FAIL!\n");
+	} else if (strncmp(data, "restore", 7) == 0) {
+
+		my_psd_arg->path = pmppriv->curr_rfpath;
+
+		if (rtw_mp_phl_psd_cmd(adapter, my_psd_arg, RTW_MP_CAL_CMD_PSD_RESTORE))
+			RTW_INFO("PSD_RESTORE CMD OK!\n");
+		else
+			RTW_INFO("PSD_RESTORE CMD FAIL!\n");
+	} else {
+		if (sscanf(data, "pts=%d,start=%d,stop=%d",
+			&psd_pts, &psd_start, &psd_stop) == 3) {
+			my_psd_arg->upoint = psd_pts;
+			my_psd_arg->start_point = psd_start;
+			my_psd_arg->stop_point = psd_stop;
+		} else {
+			my_psd_arg->upoint = 1280;
+			my_psd_arg->start_point = 640;
+			my_psd_arg->stop_point = 960;
+			RTW_INFO("Default PSD setting!\n");
+		}
+
+		if (rtw_mp_phl_psd_cmd(adapter, my_psd_arg, RTW_MP_CAL_CMD_PSD_QUERY))
+			RTW_INFO("PSD_QUERY CMD OK!\n");
+		else
+			RTW_INFO("PSD_QUERY CMD FAIL!\n");
+
+		data[0] = '\0';
+		i = 0;
+	
+		while (i < 320) {
+			pdata = data + strlen(data);
+			pdata += sprintf(pdata, "%x ", (my_psd_arg->outbuf[i]));
+			i++;
+		}
+	}
+
+	if (my_psd_arg)
+		_rtw_mfree(my_psd_arg, sizeof(struct rtw_mp_cal_arg));
+
+	return strlen(data) + 1;
+}
+
+u8
+mpt_to_mgnt_rate(
+		u32	MptRateIdx
+)
+{
+	/* Mapped to MGN_XXX defined in MgntGen.h */
+	switch (MptRateIdx) {
+	/* CCK rate. */
+	case	MPT_RATE_1M:
+		return MGN_1M;
+	case	MPT_RATE_2M:
+		return MGN_2M;
+	case	MPT_RATE_55M:
+		return MGN_5_5M;
+	case	MPT_RATE_11M:
+		return MGN_11M;
+
+	/* OFDM rate. */
+	case	MPT_RATE_6M:
+		return MGN_6M;
+	case	MPT_RATE_9M:
+		return MGN_9M;
+	case	MPT_RATE_12M:
+		return MGN_12M;
+	case	MPT_RATE_18M:
+		return MGN_18M;
+	case	MPT_RATE_24M:
+		return MGN_24M;
+	case	MPT_RATE_36M:
+		return MGN_36M;
+	case	MPT_RATE_48M:
+		return MGN_48M;
+	case	MPT_RATE_54M:
+		return MGN_54M;
+
+	/* HT rate. */
+	case	MPT_RATE_MCS0:
+		return MGN_MCS0;
+	case	MPT_RATE_MCS1:
+		return MGN_MCS1;
+	case	MPT_RATE_MCS2:
+		return MGN_MCS2;
+	case	MPT_RATE_MCS3:
+		return MGN_MCS3;
+	case	MPT_RATE_MCS4:
+		return MGN_MCS4;
+	case	MPT_RATE_MCS5:
+		return MGN_MCS5;
+	case	MPT_RATE_MCS6:
+		return MGN_MCS6;
+	case	MPT_RATE_MCS7:
+		return MGN_MCS7;
+	case	MPT_RATE_MCS8:
+		return MGN_MCS8;
+	case	MPT_RATE_MCS9:
+		return MGN_MCS9;
+	case	MPT_RATE_MCS10:
+		return MGN_MCS10;
+	case	MPT_RATE_MCS11:
+		return MGN_MCS11;
+	case	MPT_RATE_MCS12:
+		return MGN_MCS12;
+	case	MPT_RATE_MCS13:
+		return MGN_MCS13;
+	case	MPT_RATE_MCS14:
+		return MGN_MCS14;
+	case	MPT_RATE_MCS15:
+		return MGN_MCS15;
+	case	MPT_RATE_MCS16:
+		return MGN_MCS16;
+	case	MPT_RATE_MCS17:
+		return MGN_MCS17;
+	case	MPT_RATE_MCS18:
+		return MGN_MCS18;
+	case	MPT_RATE_MCS19:
+		return MGN_MCS19;
+	case	MPT_RATE_MCS20:
+		return MGN_MCS20;
+	case	MPT_RATE_MCS21:
+		return MGN_MCS21;
+	case	MPT_RATE_MCS22:
+		return MGN_MCS22;
+	case	MPT_RATE_MCS23:
+		return MGN_MCS23;
+	case	MPT_RATE_MCS24:
+		return MGN_MCS24;
+	case	MPT_RATE_MCS25:
+		return MGN_MCS25;
+	case	MPT_RATE_MCS26:
+		return MGN_MCS26;
+	case	MPT_RATE_MCS27:
+		return MGN_MCS27;
+	case	MPT_RATE_MCS28:
+		return MGN_MCS28;
+	case	MPT_RATE_MCS29:
+		return MGN_MCS29;
+	case	MPT_RATE_MCS30:
+		return MGN_MCS30;
+	case	MPT_RATE_MCS31:
+		return MGN_MCS31;
+
+	/* VHT rate. */
+	case	MPT_RATE_VHT1SS_MCS0:
+		return MGN_VHT1SS_MCS0;
+	case	MPT_RATE_VHT1SS_MCS1:
+		return MGN_VHT1SS_MCS1;
+	case	MPT_RATE_VHT1SS_MCS2:
+		return MGN_VHT1SS_MCS2;
+	case	MPT_RATE_VHT1SS_MCS3:
+		return MGN_VHT1SS_MCS3;
+	case	MPT_RATE_VHT1SS_MCS4:
+		return MGN_VHT1SS_MCS4;
+	case	MPT_RATE_VHT1SS_MCS5:
+		return MGN_VHT1SS_MCS5;
+	case	MPT_RATE_VHT1SS_MCS6:
+		return MGN_VHT1SS_MCS6;
+	case	MPT_RATE_VHT1SS_MCS7:
+		return MGN_VHT1SS_MCS7;
+	case	MPT_RATE_VHT1SS_MCS8:
+		return MGN_VHT1SS_MCS8;
+	case	MPT_RATE_VHT1SS_MCS9:
+		return MGN_VHT1SS_MCS9;
+	case	MPT_RATE_VHT2SS_MCS0:
+		return MGN_VHT2SS_MCS0;
+	case	MPT_RATE_VHT2SS_MCS1:
+		return MGN_VHT2SS_MCS1;
+	case	MPT_RATE_VHT2SS_MCS2:
+		return MGN_VHT2SS_MCS2;
+	case	MPT_RATE_VHT2SS_MCS3:
+		return MGN_VHT2SS_MCS3;
+	case	MPT_RATE_VHT2SS_MCS4:
+		return MGN_VHT2SS_MCS4;
+	case	MPT_RATE_VHT2SS_MCS5:
+		return MGN_VHT2SS_MCS5;
+	case	MPT_RATE_VHT2SS_MCS6:
+		return MGN_VHT2SS_MCS6;
+	case	MPT_RATE_VHT2SS_MCS7:
+		return MGN_VHT2SS_MCS7;
+	case	MPT_RATE_VHT2SS_MCS8:
+		return MGN_VHT2SS_MCS8;
+	case	MPT_RATE_VHT2SS_MCS9:
+		return MGN_VHT2SS_MCS9;
+	case	MPT_RATE_VHT3SS_MCS0:
+		return MGN_VHT3SS_MCS0;
+	case	MPT_RATE_VHT3SS_MCS1:
+		return MGN_VHT3SS_MCS1;
+	case	MPT_RATE_VHT3SS_MCS2:
+		return MGN_VHT3SS_MCS2;
+	case	MPT_RATE_VHT3SS_MCS3:
+		return MGN_VHT3SS_MCS3;
+	case	MPT_RATE_VHT3SS_MCS4:
+		return MGN_VHT3SS_MCS4;
+	case	MPT_RATE_VHT3SS_MCS5:
+		return MGN_VHT3SS_MCS5;
+	case	MPT_RATE_VHT3SS_MCS6:
+		return MGN_VHT3SS_MCS6;
+	case	MPT_RATE_VHT3SS_MCS7:
+		return MGN_VHT3SS_MCS7;
+	case	MPT_RATE_VHT3SS_MCS8:
+		return MGN_VHT3SS_MCS8;
+	case	MPT_RATE_VHT3SS_MCS9:
+		return MGN_VHT3SS_MCS9;
+	case	MPT_RATE_VHT4SS_MCS0:
+		return MGN_VHT4SS_MCS0;
+	case	MPT_RATE_VHT4SS_MCS1:
+		return MGN_VHT4SS_MCS1;
+	case	MPT_RATE_VHT4SS_MCS2:
+		return MGN_VHT4SS_MCS2;
+	case	MPT_RATE_VHT4SS_MCS3:
+		return MGN_VHT4SS_MCS3;
+	case	MPT_RATE_VHT4SS_MCS4:
+		return MGN_VHT4SS_MCS4;
+	case	MPT_RATE_VHT4SS_MCS5:
+		return MGN_VHT4SS_MCS5;
+	case	MPT_RATE_VHT4SS_MCS6:
+		return MGN_VHT4SS_MCS6;
+	case	MPT_RATE_VHT4SS_MCS7:
+		return MGN_VHT4SS_MCS7;
+	case	MPT_RATE_VHT4SS_MCS8:
+		return MGN_VHT4SS_MCS8;
+	case	MPT_RATE_VHT4SS_MCS9:
+		return MGN_VHT4SS_MCS9;
+
+	case	MPT_RATE_LAST:	/* fully automatiMGN_VHT2SS_MCS1;	 */
+	default:
+		RTW_INFO("<===mpt_to_mgnt_rate(), Invalid Rate: %d!!\n", MptRateIdx);
+		return 0x0;
+	}
+}
+
+
+u8 rtw_mp_hwrate2mptrate(u8 rate)
+{
+	u8	ret_rate = MGN_1M;
+
+	switch (rate) {
+	case DESC_RATE1M:
+		ret_rate = MPT_RATE_1M;
+		break;
+	case DESC_RATE2M:
+		ret_rate = MPT_RATE_2M;
+		break;
+	case DESC_RATE5_5M:
+		ret_rate = MPT_RATE_55M;
+		break;
+	case DESC_RATE11M:
+		ret_rate = MPT_RATE_11M;
+		break;
+	case DESC_RATE6M:
+		ret_rate = MPT_RATE_6M;
+		break;
+	case DESC_RATE9M:
+		ret_rate = MPT_RATE_9M;
+		break;
+	case DESC_RATE12M:
+		ret_rate = MPT_RATE_12M;
+		break;
+	case DESC_RATE18M:
+		ret_rate = MPT_RATE_18M;
+		break;
+	case DESC_RATE24M:
+		ret_rate = MPT_RATE_24M;
+		break;
+	case DESC_RATE36M:
+		ret_rate = MPT_RATE_36M;
+		break;
+	case DESC_RATE48M:
+		ret_rate = MPT_RATE_48M;
+		break;
+	case DESC_RATE54M:
+		ret_rate = MPT_RATE_54M;
+		break;
+	case DESC_RATEMCS0:
+		ret_rate = MPT_RATE_MCS0;
+		break;
+	case DESC_RATEMCS1:
+		ret_rate = MPT_RATE_MCS1;
+		break;
+	case DESC_RATEMCS2:
+		ret_rate = MPT_RATE_MCS2;
+		break;
+	case DESC_RATEMCS3:
+		ret_rate = MPT_RATE_MCS3;
+		break;
+	case DESC_RATEMCS4:
+		ret_rate = MPT_RATE_MCS4;
+		break;
+	case DESC_RATEMCS5:
+		ret_rate = MPT_RATE_MCS5;
+		break;
+	case DESC_RATEMCS6:
+		ret_rate = MPT_RATE_MCS6;
+		break;
+	case DESC_RATEMCS7:
+		ret_rate = MPT_RATE_MCS7;
+		break;
+	case DESC_RATEMCS8:
+		ret_rate = MPT_RATE_MCS8;
+		break;
+	case DESC_RATEMCS9:
+		ret_rate = MPT_RATE_MCS9;
+		break;
+	case DESC_RATEMCS10:
+		ret_rate = MPT_RATE_MCS10;
+		break;
+	case DESC_RATEMCS11:
+		ret_rate = MPT_RATE_MCS11;
+		break;
+	case DESC_RATEMCS12:
+		ret_rate = MPT_RATE_MCS12;
+		break;
+	case DESC_RATEMCS13:
+		ret_rate = MPT_RATE_MCS13;
+		break;
+	case DESC_RATEMCS14:
+		ret_rate = MPT_RATE_MCS14;
+		break;
+	case DESC_RATEMCS15:
+		ret_rate = MPT_RATE_MCS15;
+		break;
+	case DESC_RATEMCS16:
+		ret_rate = MPT_RATE_MCS16;
+		break;
+	case DESC_RATEMCS17:
+		ret_rate = MPT_RATE_MCS17;
+		break;
+	case DESC_RATEMCS18:
+		ret_rate = MPT_RATE_MCS18;
+		break;
+	case DESC_RATEMCS19:
+		ret_rate = MPT_RATE_MCS19;
+		break;
+	case DESC_RATEMCS20:
+		ret_rate = MPT_RATE_MCS20;
+		break;
+	case DESC_RATEMCS21:
+		ret_rate = MPT_RATE_MCS21;
+		break;
+	case DESC_RATEMCS22:
+		ret_rate = MPT_RATE_MCS22;
+		break;
+	case DESC_RATEMCS23:
+		ret_rate = MPT_RATE_MCS23;
+		break;
+	case DESC_RATEMCS24:
+		ret_rate = MPT_RATE_MCS24;
+		break;
+	case DESC_RATEMCS25:
+		ret_rate = MPT_RATE_MCS25;
+		break;
+	case DESC_RATEMCS26:
+		ret_rate = MPT_RATE_MCS26;
+		break;
+	case DESC_RATEMCS27:
+		ret_rate = MPT_RATE_MCS27;
+		break;
+	case DESC_RATEMCS28:
+		ret_rate = MPT_RATE_MCS28;
+		break;
+	case DESC_RATEMCS29:
+		ret_rate = MPT_RATE_MCS29;
+		break;
+	case DESC_RATEMCS30:
+		ret_rate = MPT_RATE_MCS30;
+		break;
+	case DESC_RATEMCS31:
+		ret_rate = MPT_RATE_MCS31;
+		break;
+	case DESC_RATEVHTSS1MCS0:
+		ret_rate = MPT_RATE_VHT1SS_MCS0;
+		break;
+	case DESC_RATEVHTSS1MCS1:
+		ret_rate = MPT_RATE_VHT1SS_MCS1;
+		break;
+	case DESC_RATEVHTSS1MCS2:
+		ret_rate = MPT_RATE_VHT1SS_MCS2;
+		break;
+	case DESC_RATEVHTSS1MCS3:
+		ret_rate = MPT_RATE_VHT1SS_MCS3;
+		break;
+	case DESC_RATEVHTSS1MCS4:
+		ret_rate = MPT_RATE_VHT1SS_MCS4;
+		break;
+	case DESC_RATEVHTSS1MCS5:
+		ret_rate = MPT_RATE_VHT1SS_MCS5;
+		break;
+	case DESC_RATEVHTSS1MCS6:
+		ret_rate = MPT_RATE_VHT1SS_MCS6;
+		break;
+	case DESC_RATEVHTSS1MCS7:
+		ret_rate = MPT_RATE_VHT1SS_MCS7;
+		break;
+	case DESC_RATEVHTSS1MCS8:
+		ret_rate = MPT_RATE_VHT1SS_MCS8;
+		break;
+	case DESC_RATEVHTSS1MCS9:
+		ret_rate = MPT_RATE_VHT1SS_MCS9;
+		break;
+	case DESC_RATEVHTSS2MCS0:
+		ret_rate = MPT_RATE_VHT2SS_MCS0;
+		break;
+	case DESC_RATEVHTSS2MCS1:
+		ret_rate = MPT_RATE_VHT2SS_MCS1;
+		break;
+	case DESC_RATEVHTSS2MCS2:
+		ret_rate = MPT_RATE_VHT2SS_MCS2;
+		break;
+	case DESC_RATEVHTSS2MCS3:
+		ret_rate = MPT_RATE_VHT2SS_MCS3;
+		break;
+	case DESC_RATEVHTSS2MCS4:
+		ret_rate = MPT_RATE_VHT2SS_MCS4;
+		break;
+	case DESC_RATEVHTSS2MCS5:
+		ret_rate = MPT_RATE_VHT2SS_MCS5;
+		break;
+	case DESC_RATEVHTSS2MCS6:
+		ret_rate = MPT_RATE_VHT2SS_MCS6;
+		break;
+	case DESC_RATEVHTSS2MCS7:
+		ret_rate = MPT_RATE_VHT2SS_MCS7;
+		break;
+	case DESC_RATEVHTSS2MCS8:
+		ret_rate = MPT_RATE_VHT2SS_MCS8;
+		break;
+	case DESC_RATEVHTSS2MCS9:
+		ret_rate = MPT_RATE_VHT2SS_MCS9;
+		break;
+	case DESC_RATEVHTSS3MCS0:
+		ret_rate = MPT_RATE_VHT3SS_MCS0;
+		break;
+	case DESC_RATEVHTSS3MCS1:
+		ret_rate = MPT_RATE_VHT3SS_MCS1;
+		break;
+	case DESC_RATEVHTSS3MCS2:
+		ret_rate = MPT_RATE_VHT3SS_MCS2;
+		break;
+	case DESC_RATEVHTSS3MCS3:
+		ret_rate = MPT_RATE_VHT3SS_MCS3;
+		break;
+	case DESC_RATEVHTSS3MCS4:
+		ret_rate = MPT_RATE_VHT3SS_MCS4;
+		break;
+	case DESC_RATEVHTSS3MCS5:
+		ret_rate = MPT_RATE_VHT3SS_MCS5;
+		break;
+	case DESC_RATEVHTSS3MCS6:
+		ret_rate = MPT_RATE_VHT3SS_MCS6;
+		break;
+	case DESC_RATEVHTSS3MCS7:
+		ret_rate = MPT_RATE_VHT3SS_MCS7;
+		break;
+	case DESC_RATEVHTSS3MCS8:
+		ret_rate = MPT_RATE_VHT3SS_MCS8;
+		break;
+	case DESC_RATEVHTSS3MCS9:
+		ret_rate = MPT_RATE_VHT3SS_MCS9;
+		break;
+	case DESC_RATEVHTSS4MCS0:
+		ret_rate = MPT_RATE_VHT4SS_MCS0;
+		break;
+	case DESC_RATEVHTSS4MCS1:
+		ret_rate = MPT_RATE_VHT4SS_MCS1;
+		break;
+	case DESC_RATEVHTSS4MCS2:
+		ret_rate = MPT_RATE_VHT4SS_MCS2;
+		break;
+	case DESC_RATEVHTSS4MCS3:
+		ret_rate = MPT_RATE_VHT4SS_MCS3;
+		break;
+	case DESC_RATEVHTSS4MCS4:
+		ret_rate = MPT_RATE_VHT4SS_MCS4;
+		break;
+	case DESC_RATEVHTSS4MCS5:
+		ret_rate = MPT_RATE_VHT4SS_MCS5;
+		break;
+	case DESC_RATEVHTSS4MCS6:
+		ret_rate = MPT_RATE_VHT4SS_MCS6;
+		break;
+	case DESC_RATEVHTSS4MCS7:
+		ret_rate = MPT_RATE_VHT4SS_MCS7;
+		break;
+	case DESC_RATEVHTSS4MCS8:
+		ret_rate = MPT_RATE_VHT4SS_MCS8;
+		break;
+	case DESC_RATEVHTSS4MCS9:
+		ret_rate = MPT_RATE_VHT4SS_MCS9;
+		break;
+
+	default:
+		RTW_INFO("hwrate_to_mrate(): Non supported Rate [%x]!!!\n", rate);
+		break;
+	}
+	return ret_rate;
+}
+
+static u16 srate2herate(u8 *str)
+{
+
+	RTW_INFO("%s(): %s!!!\n", __func__, str);
+	if (!strcmp(str, "1M")) return HRATE_CCK1;
+	else if (!strcmp(str, "2M")) return HRATE_CCK2;
+	else if (!strcmp(str, "5.5M")) return HRATE_CCK5_5;
+	else if (!strcmp(str, "11M")) return HRATE_CCK11;
+	else if (!strcmp(str, "6M")) return HRATE_OFDM6;
+	else if (!strcmp(str, "9M")) return HRATE_OFDM9;
+	else if (!strcmp(str, "12M")) return HRATE_OFDM12;
+	else if (!strcmp(str, "18M")) return HRATE_OFDM18;
+	else if (!strcmp(str, "24M")) return HRATE_OFDM24;
+	else if (!strcmp(str, "36M")) return HRATE_OFDM36;
+	else if (!strcmp(str, "48M")) return HRATE_OFDM48;
+	else if (!strcmp(str, "54M")) return HRATE_OFDM54;
+	else if (!strcmp(str, "HTMCS0")) return HRATE_MCS0;
+	else if (!strcmp(str, "HTMCS1")) return HRATE_MCS1;
+	else if (!strcmp(str, "HTMCS2")) return HRATE_MCS2;
+	else if (!strcmp(str, "HTMCS3")) return HRATE_MCS3;
+	else if (!strcmp(str, "HTMCS4")) return HRATE_MCS4;
+	else if (!strcmp(str, "HTMCS5")) return HRATE_MCS5;
+	else if (!strcmp(str, "HTMCS6")) return HRATE_MCS6;
+	else if (!strcmp(str, "HTMCS7")) return HRATE_MCS7;
+	else if (!strcmp(str, "HTMCS8")) return HRATE_MCS8;
+	else if (!strcmp(str, "HTMCS9")) return HRATE_MCS9;
+	else if (!strcmp(str, "HTMCS10")) return HRATE_MCS10;
+	else if (!strcmp(str, "HTMCS11")) return HRATE_MCS11;
+	else if (!strcmp(str, "HTMCS12")) return HRATE_MCS12;
+	else if (!strcmp(str, "HTMCS13")) return HRATE_MCS13;
+	else if (!strcmp(str, "HTMCS14")) return HRATE_MCS14;
+	else if (!strcmp(str, "HTMCS15")) return HRATE_MCS15;
+	else if (!strcmp(str, "HTMCS16")) return HRATE_MCS16;
+	else if (!strcmp(str, "HTMCS17")) return HRATE_MCS17;
+	else if (!strcmp(str, "HTMCS18")) return HRATE_MCS18;
+	else if (!strcmp(str, "HTMCS19")) return HRATE_MCS19;
+	else if (!strcmp(str, "HTMCS20")) return HRATE_MCS20;
+	else if (!strcmp(str, "HTMCS21")) return HRATE_MCS21;
+	else if (!strcmp(str, "HTMCS22")) return HRATE_MCS22;
+	else if (!strcmp(str, "HTMCS23")) return HRATE_MCS23;
+	else if (!strcmp(str, "HTMCS24")) return HRATE_MCS24;
+	else if (!strcmp(str, "HTMCS25")) return HRATE_MCS25;
+	else if (!strcmp(str, "HTMCS26")) return HRATE_MCS26;
+	else if (!strcmp(str, "HTMCS27")) return HRATE_MCS27;
+	else if (!strcmp(str, "HTMCS28")) return HRATE_MCS28;
+	else if (!strcmp(str, "HTMCS29")) return HRATE_MCS29;
+	else if (!strcmp(str, "HTMCS30")) return HRATE_MCS30;
+	else if (!strcmp(str, "HTMCS31")) return HRATE_MCS31;
+	else if (!strcmp(str, "VHT1MCS0")) return HRATE_VHT_NSS1_MCS0;
+	else if (!strcmp(str, "VHT1MCS1")) return HRATE_VHT_NSS1_MCS1;
+	else if (!strcmp(str, "VHT1MCS2")) return HRATE_VHT_NSS1_MCS2;
+	else if (!strcmp(str, "VHT1MCS3")) return HRATE_VHT_NSS1_MCS3;
+	else if (!strcmp(str, "VHT1MCS4")) return HRATE_VHT_NSS1_MCS4;
+	else if (!strcmp(str, "VHT1MCS5")) return HRATE_VHT_NSS1_MCS5;
+	else if (!strcmp(str, "VHT1MCS6")) return HRATE_VHT_NSS1_MCS6;
+	else if (!strcmp(str, "VHT1MCS7")) return HRATE_VHT_NSS1_MCS7;
+	else if (!strcmp(str, "VHT1MCS8")) return HRATE_VHT_NSS1_MCS8;
+	else if (!strcmp(str, "VHT1MCS9")) return HRATE_VHT_NSS1_MCS9;
+	else if (!strcmp(str, "VHT2MCS0")) return HRATE_VHT_NSS2_MCS0;
+	else if (!strcmp(str, "VHT2MCS1")) return HRATE_VHT_NSS2_MCS1;
+	else if (!strcmp(str, "VHT2MCS2")) return HRATE_VHT_NSS2_MCS2;
+	else if (!strcmp(str, "VHT2MCS3")) return HRATE_VHT_NSS2_MCS3;
+	else if (!strcmp(str, "VHT2MCS4")) return HRATE_VHT_NSS2_MCS4;
+	else if (!strcmp(str, "VHT2MCS5")) return HRATE_VHT_NSS2_MCS5;
+	else if (!strcmp(str, "VHT2MCS6")) return HRATE_VHT_NSS2_MCS6;
+	else if (!strcmp(str, "VHT2MCS7")) return HRATE_VHT_NSS2_MCS7;
+	else if (!strcmp(str, "VHT2MCS8")) return HRATE_VHT_NSS2_MCS8;
+	else if (!strcmp(str, "VHT2MCS9")) return HRATE_VHT_NSS2_MCS9;
+	else if (!strcmp(str, "VHT3MCS0")) return HRATE_VHT_NSS3_MCS0;
+	else if (!strcmp(str, "VHT3MCS1")) return HRATE_VHT_NSS3_MCS1;
+	else if (!strcmp(str, "VHT3MCS2")) return HRATE_VHT_NSS3_MCS2;
+	else if (!strcmp(str, "VHT3MCS3")) return HRATE_VHT_NSS3_MCS3;
+	else if (!strcmp(str, "VHT3MCS4")) return HRATE_VHT_NSS3_MCS4;
+	else if (!strcmp(str, "VHT3MCS5")) return HRATE_VHT_NSS3_MCS5;
+	else if (!strcmp(str, "VHT3MCS6")) return HRATE_VHT_NSS3_MCS6;
+	else if (!strcmp(str, "VHT3MCS7")) return HRATE_VHT_NSS3_MCS7;
+	else if (!strcmp(str, "VHT3MCS8")) return HRATE_VHT_NSS3_MCS8;
+	else if (!strcmp(str, "VHT3MCS9")) return HRATE_VHT_NSS3_MCS9;
+	else if (!strcmp(str, "VHT4MCS0")) return HRATE_VHT_NSS4_MCS0;
+	else if (!strcmp(str, "VHT4MCS1")) return HRATE_VHT_NSS4_MCS1;
+	else if (!strcmp(str, "VHT4MCS2")) return HRATE_VHT_NSS4_MCS2;
+	else if (!strcmp(str, "VHT4MCS3")) return HRATE_VHT_NSS4_MCS3;
+	else if (!strcmp(str, "VHT4MCS4")) return HRATE_VHT_NSS4_MCS4;
+	else if (!strcmp(str, "VHT4MCS5")) return HRATE_VHT_NSS4_MCS5;
+	else if (!strcmp(str, "VHT4MCS6")) return HRATE_VHT_NSS4_MCS6;
+	else if (!strcmp(str, "VHT4MCS7")) return HRATE_VHT_NSS4_MCS7;
+	else if (!strcmp(str, "VHT4MCS8")) return HRATE_VHT_NSS4_MCS8;
+	else if (!strcmp(str, "VHT4MCS9")) return HRATE_VHT_NSS4_MCS9;
+	else if (!strcmp(str, "HE1MCS0")) return HRATE_HE_NSS1_MCS0;
+	else if (!strcmp(str, "HE1MCS1")) return HRATE_HE_NSS1_MCS1;
+	else if (!strcmp(str, "HE1MCS2")) return HRATE_HE_NSS1_MCS2;
+	else if (!strcmp(str, "HE1MCS3")) return HRATE_HE_NSS1_MCS3;
+	else if (!strcmp(str, "HE1MCS4")) return HRATE_HE_NSS1_MCS4;
+	else if (!strcmp(str, "HE1MCS5")) return HRATE_HE_NSS1_MCS5;
+	else if (!strcmp(str, "HE1MCS6")) return HRATE_HE_NSS1_MCS6;
+	else if (!strcmp(str, "HE1MCS7")) return HRATE_HE_NSS1_MCS7;
+	else if (!strcmp(str, "HE1MCS8")) return HRATE_HE_NSS1_MCS8;
+	else if (!strcmp(str, "HE1MCS9")) return HRATE_HE_NSS1_MCS9;
+	else if (!strcmp(str, "HE1MCS10")) return HRATE_HE_NSS1_MCS10;
+	else if (!strcmp(str, "HE1MCS11")) return HRATE_HE_NSS1_MCS11;
+	else if (!strcmp(str, "HE2MCS0")) return HRATE_HE_NSS2_MCS0;
+	else if (!strcmp(str, "HE2MCS1")) return HRATE_HE_NSS2_MCS1;
+	else if (!strcmp(str, "HE2MCS2")) return HRATE_HE_NSS2_MCS2;
+	else if (!strcmp(str, "HE2MCS3")) return HRATE_HE_NSS2_MCS3;
+	else if (!strcmp(str, "HE2MCS4")) return HRATE_HE_NSS2_MCS4;
+	else if (!strcmp(str, "HE2MCS5")) return HRATE_HE_NSS2_MCS5;
+	else if (!strcmp(str, "HE2MCS6")) return HRATE_HE_NSS2_MCS6;
+	else if (!strcmp(str, "HE2MCS7")) return HRATE_HE_NSS2_MCS7;
+	else if (!strcmp(str, "HE2MCS8")) return HRATE_HE_NSS2_MCS8;
+	else if (!strcmp(str, "HE2MCS9")) return HRATE_HE_NSS2_MCS9;
+	else if (!strcmp(str, "HE2MCS10")) return HRATE_HE_NSS2_MCS10;
+	else if (!strcmp(str, "HE2MCS11")) return HRATE_HE_NSS2_MCS11;
+	else if (!strcmp(str, "HE3MCS0")) return HRATE_HE_NSS3_MCS0;
+	else if (!strcmp(str, "HE3MCS1")) return HRATE_HE_NSS3_MCS1;
+	else if (!strcmp(str, "HE3MCS2")) return HRATE_HE_NSS3_MCS2;
+	else if (!strcmp(str, "HE3MCS3")) return HRATE_HE_NSS3_MCS3;
+	else if (!strcmp(str, "HE3MCS4")) return HRATE_HE_NSS3_MCS4;
+	else if (!strcmp(str, "HE3MCS5")) return HRATE_HE_NSS3_MCS5;
+	else if (!strcmp(str, "HE3MCS6")) return HRATE_HE_NSS3_MCS6;
+	else if (!strcmp(str, "HE3MCS7")) return HRATE_HE_NSS3_MCS7;
+	else if (!strcmp(str, "HE3MCS8")) return HRATE_HE_NSS3_MCS8;
+	else if (!strcmp(str, "HE3MCS9")) return HRATE_HE_NSS3_MCS9;
+	else if (!strcmp(str, "HE3MCS10")) return HRATE_HE_NSS3_MCS10;
+	else if (!strcmp(str, "HE3MCS11")) return HRATE_HE_NSS3_MCS11;
+	else if (!strcmp(str, "HE4MCS0")) return HRATE_HE_NSS4_MCS0;
+	else if (!strcmp(str, "HE4MCS1")) return HRATE_HE_NSS4_MCS1;
+	else if (!strcmp(str, "HE4MCS2")) return HRATE_HE_NSS4_MCS2;
+	else if (!strcmp(str, "HE4MCS3")) return HRATE_HE_NSS4_MCS3;
+	else if (!strcmp(str, "HE4MCS4")) return HRATE_HE_NSS4_MCS4;
+	else if (!strcmp(str, "HE4MCS5")) return HRATE_HE_NSS4_MCS5;
+	else if (!strcmp(str, "HE4MCS6")) return HRATE_HE_NSS4_MCS6;
+	else if (!strcmp(str, "HE4MCS7")) return HRATE_HE_NSS4_MCS7;
+	else if (!strcmp(str, "HE4MCS8")) return HRATE_HE_NSS4_MCS8;
+	else if (!strcmp(str, "HE4MCS9")) return HRATE_HE_NSS4_MCS9;
+	else if (!strcmp(str, "HE4MCS10")) return HRATE_HE_NSS4_MCS10;
+	else if (!strcmp(str, "HE4MCS11")) return HRATE_HE_NSS4_MCS11;
+	return 0;
+}
+
+#define MP_RATE_STR_NUM 132
+#define MP_RATE_STR_LEN 12
+const char rtw_mp_rateidx_arrdef[MP_RATE_STR_NUM][MP_RATE_STR_LEN] = {
+	"1M", "2M", "5.5M", "11M", "6M", "9M", "12M", "18M", "24M", "36M", "48M", "54M",
+	"HTMCS0", "HTMCS1", "HTMCS2", "HTMCS3", "HTMCS4", "HTMCS5", "HTMCS6", "HTMCS7",
+	"HTMCS8", "HTMCS9", "HTMCS10", "HTMCS11", "HTMCS12", "HTMCS13", "HTMCS14", "HTMCS15",
+	"HTMCS16", "HTMCS17", "HTMCS18", "HTMCS19", "HTMCS20", "HTMCS21", "HTMCS22", "HTMCS23",
+	"HTMCS24", "HTMCS25", "HTMCS26", "HTMCS27", "HTMCS28", "HTMCS29", "HTMCS30", "HTMCS31",
+	"VHT1MCS0", "VHT1MCS1", "VHT1MCS2", "VHT1MCS3", "VHT1MCS4",
+	"VHT1MCS5", "VHT1MCS6", "VHT1MCS7", "VHT1MCS8", "VHT1MCS9",
+	"VHT2MCS0", "VHT2MCS1", "VHT2MCS2", "VHT2MCS3", "VHT2MCS4",
+	"VHT2MCS5", "VHT2MCS6", "VHT2MCS7", "VHT2MCS8", "VHT2MCS9",
+	"VHT3MCS0", "VHT3MCS1", "VHT3MCS2", "VHT3MCS3", "VHT3MCS4",
+	"VHT3MCS5", "VHT3MCS6", "VHT3MCS7", "VHT3MCS8", "VHT3MCS9",
+	"VHT4MCS0", "VHT4MCS1", "VHT4MCS2", "VHT4MCS3", "VHT4MCS4",
+	"VHT4MCS5", "VHT4MCS6", "VHT4MCS7", "VHT4MCS8", "VHT4MCS9",
+	"HE1MCS0", "HE1MCS1", "HE1MCS2", "HE1MCS3", "HE1MCS4", "HE1MCS5",
+	"HE1MCS6", "HE1MCS7", "HE1MCS8", "HE1MCS9", "HE1MCS10", "HE1MCS11",
+	"HE2MCS0", "HE2MCS1", "HE2MCS2", "HE2MCS3", "HE2MCS4", "HE2MCS5",
+	"HE2MCS6", "HE2MCS7", "HE2MCS8", "HE2MCS9", "HE2MCS10", "HE2MCS11",
+	"HE3MCS0", "HE3MCS1", "HE3MCS2", "HE3MCS3", "HE3MCS4", "HE3MCS5",
+	"HE3MCS6", "HE3MCS7", "HE3MCS8", "HE3MCS9", "HE3MCS10", "HE3MCS11",
+	"HE4MCS0", "HE4MCS1", "HE4MCS2", "HE4MCS3", "HE4MCS4", "HE4MCS5",
+	"HE4MCS6", "HE4MCS7", "HE4MCS8", "HE4MCS9", "HE4MCS10", "HE4MCS11"};
+
+u16 rtw_mp_rate_parse(_adapter *adapter, u8 *target_str)
+{
+	struct mp_priv	*pmppriv = &adapter->mppriv;
+
+	u16 i = 0, mcs_idx_base = 0;
+	u16 hrateidx = 0;
+	u8 plcp_nss = 0, ppdu_type = 0;
+	u8 user_idx = pmppriv->mp_plcp_useridx;
+	u8 giltf_num = 0;
+
+#if 1/*def CONFIG_80211AX_HE*/
+	hrateidx = srate2herate(target_str);
+
+	if (MP_IS_1T_HRATE(hrateidx))
+		plcp_nss = MP_NSS1;
+	else if (MP_IS_2T_HRATE(hrateidx))
+		plcp_nss = MP_NSS2;
+	else if (MP_IS_3T_HRATE(hrateidx))
+		plcp_nss = MP_NSS3;
+	else if (MP_IS_4T_HRATE(hrateidx))
+		plcp_nss = MP_NSS4;
+	RTW_INFO("%s , PLCP NSS idx = %d\n", __func__ , plcp_nss);
+
+	if (MP_IS_CCK_HRATE(hrateidx))
+		ppdu_type = RTW_MP_TYPE_CCK;
+	else if (MP_IS_OFDM_HRATE(hrateidx)) {
+		ppdu_type = RTW_MP_TYPE_LEGACY;
+		mcs_idx_base = 4;
+	} else if (MP_IS_HT_HRATE(hrateidx)) {
+		ppdu_type = RTW_MP_TYPE_HT_MF;
+		mcs_idx_base = 12;
+	} else if (MP_IS_VHT_HRATE(hrateidx)) {
+		if (plcp_nss == MP_NSS1)
+			mcs_idx_base = 44;
+		else if (plcp_nss == MP_NSS2)
+			mcs_idx_base = 44 + 10;
+		else if (plcp_nss == MP_NSS3)
+			mcs_idx_base = 44 + 20;
+		else if (plcp_nss == MP_NSS4)
+			mcs_idx_base = 44 + 30;
+
+		ppdu_type = RTW_MP_TYPE_VHT;
+	} else if (MP_IS_HE_HRATE(hrateidx)) {
+		if (plcp_nss == MP_NSS1)
+			mcs_idx_base = 84;
+		else if (plcp_nss == MP_NSS2)
+			mcs_idx_base = 84 + 12;
+		else if (plcp_nss == MP_NSS3)
+			mcs_idx_base = 84 + 24;
+		else if (plcp_nss == MP_NSS4)
+			mcs_idx_base = 84 + 36;
+
+		ppdu_type = RTW_MP_TYPE_HE_SU;
+	}
+
+	pmppriv->mp_plcp_user[user_idx].plcp_nss = plcp_nss;
+	pmppriv->rtw_mp_pmact_ppdu_type = ppdu_type;
+
+	giltf_num = rtw_update_giltf(adapter);
+	if (giltf_num != 0) {
+		RTW_INFO("Config default GI+LTF to %s , \
+			CMD to config [mp_plcp_data giltf=num] \n", pmppriv->st_giltf[0].type_str);
+		pmppriv->rtw_mp_plcp_gi = pmppriv->st_giltf[0].gi;;
+		pmppriv->rtw_mp_plcp_ltf = pmppriv->st_giltf[0].ltf;
+		RTW_INFO("%s:default gi=%d ltf=%d\n", __func__, pmppriv->rtw_mp_plcp_gi, pmppriv->rtw_mp_plcp_ltf);
+	}
+	rtw_mp_update_coding(adapter);
+
+		for (i = 0; i <= MP_RATE_STR_LEN; i++) {
+		if (strcmp(target_str, rtw_mp_rateidx_arrdef[i + mcs_idx_base]) == 0) {
+				pmppriv->mp_plcp_user[user_idx].plcp_mcs = i;
+				RTW_INFO("%s , PLCP MCS = %d\n", __func__ , pmppriv->mp_plcp_user[user_idx].plcp_mcs);
+					break;
+				}
+		}
+
+	if (1) {
+		printk("%s ,please input a Data RATE String as:", __func__);
+		for (i = 0; i <= sizeof(rtw_mp_rateidx_arrdef) / sizeof(rtw_mp_rateidx_arrdef[0]); i++) {
+			printk("%s ", rtw_mp_rateidx_arrdef[i]);
+			if (i % 10 == 0)
+				printk("\n");
+			}
+	}
+	return hrateidx;
+#else
+	for (i = 0; i <= MP_RATE_STR_NUM; i++) {
+		if (strcmp(targetStr, rtw_mp_rateidx_arrdef[i]) == 0) {
+			RTW_INFO("%s , index = %d\n", __func__ , i);
+			return i;
+		}
+	}
+
+	printk("%s ,please input a Data RATE String as:", __func__);
+	for (i = 0; i <= MP_RATE_STR_NUM; i++) {
+		printk("%s ", rtw_mp_rateidx_arrdef[i]);
+		if (i % 10 == 0)
+			printk("\n");
+	}
+	return _FAIL;
+#endif
+}
+
+u8 rtw_mp_mode_check(_adapter *adapter)
+{
+	_adapter *primary_adapter = GET_PRIMARY_ADAPTER(adapter);
+
+	if (primary_adapter->registrypriv.mp_mode == 1 || primary_adapter->mppriv.bprocess_mp_mode == _TRUE)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+#if 0
+bool rtw_is_mp_tssitrk_on(_adapter *adapter)
+{
+	_adapter *primary_adapter = GET_PRIMARY_ADAPTER(adapter);
+
+	return primary_adapter->mppriv.tssitrk_on;
+}
+#endif
+
+u32 mpt_ProQueryCalTxPower(
+	_adapter *adapter,
+	u8		RfPath
+)
+{
+	PMPT_CONTEXT		pMptCtx = &(adapter->mppriv.mpt_ctx);
+#if 0
+	u32			TxPower = 1;
+	struct txpwr_idx_comp tic;
+	u8 mgn_rate = mpt_to_mgnt_rate(pMptCtx->mpt_rate_index);
+	RATE_SECTION rs = mgn_rate_to_rs(mgn_rate);
+
+	TxPower = rtw_hal_get_tx_power_index(adapter, RfPath, rs, mgn_rate
+		, pHalData->current_channel_bw, pHalData->current_band_type, pHalData->current_channel, &tic);
+
+	dump_tx_power_index_inline(RTW_DBGDUMP, RfPath
+		, pHalData->current_channel_bw, pHalData->current_channel
+		, mgn_rate, TxPower, &tic);
+
+	adapter->mppriv.txpoweridx = (u8)TxPower;
+	if (RfPath == RF_PATH_A)
+		pMptCtx->TxPwrLevel[RF_PATH_A] = (u8)TxPower;
+	else if (RfPath == RF_PATH_B)
+		pMptCtx->TxPwrLevel[RF_PATH_B] = (u8)TxPower;
+	else if (RfPath == RF_PATH_C)
+		pMptCtx->TxPwrLevel[RF_PATH_C] = (u8)TxPower;
+	else if (RfPath == RF_PATH_D)
+		pMptCtx->TxPwrLevel[RF_PATH_D] = (u8)TxPower;
+	hal_mpt_SetTxPower(adapter);
+		return TxPower;
+#endif
+	return 0;
+}
+
+u32 rtw_mpt_raw2dec_dbm(u32 val)
+{
+	u32 actdbm = 0;
+	u32 dec = val % TX_POWER_BASE;
+
+	actdbm = ((dec * 100) / TX_POWER_BASE);
+
+	return actdbm;
+}
+
+u32 mpt_get_tx_power_finalabs_val(_adapter *padapter, u8 rf_path)
+{
+	PMPT_CONTEXT		pMptCtx = &(padapter->mppriv.mpt_ctx);
+
+	u8 mgn_rate = mpt_to_mgnt_rate(pMptCtx->mpt_rate_index);
+	u32 powerdbm = 0;
+
+	struct mp_priv	*pmppriv = &padapter->mppriv;
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	u16 i = 0;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.mp_class = RTW_MP_CLASS_TXPWR;
+	ptxpwr_arg.cmd = RTW_MP_TXPWR_CMD_READ_PWR_TABLE;
+
+	ptxpwr_arg.channel = pmppriv->channel;
+	ptxpwr_arg.bandwidth = pmppriv->bandwidth;
+	ptxpwr_arg.rate = pmppriv->rateidx;
+	ptxpwr_arg.rfpath = rf_path;
+	ptxpwr_arg.beamforming = 0;
+	ptxpwr_arg.dcm = 0;
+	ptxpwr_arg.offset = 0;
+
+	rtw_mp_set_phl_cmd(padapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+
+	while (i != 1000) {
+		rtw_msleep_os(1);
+		rtw_mp_get_phl_cmd(padapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+		if (ptxpwr_arg.cmd_ok && ptxpwr_arg.status == RTW_PHL_STATUS_SUCCESS) {
+			powerdbm = ptxpwr_arg.table_item;
+			RTW_INFO("%s,SET CMD OK\n", __func__);
+			break;
+		} else {
+			if (i > 1000) {
+				RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__, ptxpwr_arg.status);
+					break;
+				}
+			i++;
+		}
+	}
+
+	return powerdbm;
+}
+
+u8 rtw_mpt_set_power_limit_en(_adapter *padapter, bool en_val)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	u16 i = 0;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.mp_class = RTW_MP_CLASS_TXPWR;
+	ptxpwr_arg.cmd = RTW_MP_TXPWR_CMD_SET_PWR_LMT_EN;
+
+	ptxpwr_arg.pwr_lmt_en = en_val;
+
+	rtw_mp_set_phl_cmd(padapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+
+	while (i != 10) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(padapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+		if (ptxpwr_arg.cmd_ok && ptxpwr_arg.status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,SET CMD OK\n", __func__);
+			return ptxpwr_arg.cmd_ok;
+		} else {
+			if (i > 10) {
+				RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__, ptxpwr_arg.status);
+				break;
+			}
+			i++;
+		}
+	}
+
+	return 0;
+}
+
+bool rtw_mpt_get_power_limit_en(_adapter *padapter)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	u16 i = 0;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.mp_class = RTW_MP_CLASS_TXPWR;
+	ptxpwr_arg.cmd = RTW_MP_TXPWR_CMD_GET_PWR_LMT_EN;
+	ptxpwr_arg.pwr_lmt_en = _FALSE;
+
+	rtw_mp_set_phl_cmd(padapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+
+	while (i != 10) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(padapter, (void*)&ptxpwr_arg, sizeof(struct rtw_mp_txpwr_arg));
+		if (ptxpwr_arg.cmd_ok && ptxpwr_arg.status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s, CMD OK\n", __func__);
+			return ptxpwr_arg.pwr_lmt_en;
+		} else {
+			if (i > 10) {
+				RTW_INFO("%s,GET CMD FAIL !!! status %d\n",
+					__func__, ptxpwr_arg.status);
+				break;
+			}
+			i++;
+		}
+	}
+
+	return ptxpwr_arg.pwr_lmt_en;
+}
+
+void rtw_mp_get_phl_cmd(_adapter *padapter, void* buf, u32 buflen)
+{
+	struct dvobj_priv	*dvobj = adapter_to_dvobj(padapter);
+	struct rtw_mp_test_cmdbuf *cmdbuf = NULL;
+
+	if (buflen > RTW_MAX_TEST_CMD_BUF) {
+		RTW_ERR("%s, buflen %d over CMD BUF Size !\n", __func__, buflen);
+		return;
+	}
+
+	cmdbuf = _rtw_malloc(sizeof(struct rtw_mp_test_cmdbuf));
+
+	if (cmdbuf) {
+		_rtw_memset((void *)cmdbuf, 0, sizeof(struct rtw_mp_test_cmdbuf));
+		cmdbuf->type = 0;
+		cmdbuf->len = buflen;
+		_rtw_memcpy((void *)cmdbuf->buf, buf, buflen);
+		rtw_phl_test_submodule_get_rpt(dvobj->phl_com, (void *)cmdbuf, sizeof(struct rtw_mp_test_cmdbuf));
+		_rtw_memcpy((void *)buf, (void *)cmdbuf->buf, buflen);
+	}
+	if (cmdbuf)
+		_rtw_mfree(cmdbuf, sizeof(struct rtw_mp_test_cmdbuf));
+}
+
+void rtw_mp_set_phl_cmd(_adapter *padapter, void* buf, u32 buflen)
+{
+	struct dvobj_priv	*dvobj = adapter_to_dvobj(padapter);
+	struct rtw_mp_test_cmdbuf *cmdbuf = NULL;
+
+	if (buflen > RTW_MAX_TEST_CMD_BUF) {
+		RTW_ERR("%s, buflen %d over CMD BUF Size !\n", __func__, buflen);
+		return;
+	}
+
+	cmdbuf = _rtw_malloc(sizeof(struct rtw_mp_test_cmdbuf));
+
+	if (cmdbuf) {
+		_rtw_memset((void *)cmdbuf, 0, sizeof(struct rtw_mp_test_cmdbuf));
+
+		cmdbuf->type = 0;
+		cmdbuf->len = buflen;
+		_rtw_memcpy((void *)cmdbuf->buf, buf, buflen);
+		rtw_phl_test_submodule_cmd_process(dvobj->phl_com, (void*)cmdbuf, sizeof(struct rtw_mp_test_cmdbuf));
+	}
+	if (cmdbuf)
+		_rtw_mfree(cmdbuf, sizeof(struct rtw_mp_test_cmdbuf));
+}
+
+u8 rtw_mp_phl_txpower(_adapter *padapter, struct rtw_mp_txpwr_arg	*ptxpwr_arg, u8 cmdid)
+{
+	struct mp_priv	*pmppriv = &padapter->mppriv;
+	u16 i = 0;
+
+	ptxpwr_arg->mp_class = RTW_MP_CLASS_TXPWR;
+	ptxpwr_arg->cmd = cmdid;
+
+	rtw_mp_set_phl_cmd(padapter, (void*)ptxpwr_arg,  sizeof(struct rtw_mp_txpwr_arg));
+
+	while (1) {
+		rtw_msleep_os(1);
+		rtw_mp_get_phl_cmd(padapter, (void*)ptxpwr_arg,  sizeof(struct rtw_mp_txpwr_arg));
+		if (ptxpwr_arg->cmd_ok && ptxpwr_arg->status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,SET CMD OK\n", __func__);
+			break;
+		} else {
+			if (i > 500) {
+				RTW_DBG("%s,GET CMD FAIL !!! status %d\n", __func__, ptxpwr_arg->status);
+				break;
+			}
+			i++;
+		}
+	}
+
+	return ptxpwr_arg->cmd_ok;
+}
+
+bool rtw_mp_phl_config_arg(_adapter *padapter, enum rtw_mp_config_cmdid cmdid)
+{
+	struct mp_priv	*pmppriv = &padapter->mppriv;
+	struct rtw_mp_config_arg pmp_arg;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_com_t *phl_com = dvobj->phl_com;
+	enum rf_path tx, rx;
+	u16 i = 0;
+
+	_rtw_memset((void *)&pmp_arg, 0, sizeof(struct rtw_mp_config_arg));
+
+	pmp_arg.mp_class = RTW_MP_CLASS_CONFIG;
+	pmp_arg.cmd = cmdid;
+	RTW_INFO("%s, id: %d !!!\n", __func__, cmdid);
+
+	switch (cmdid) {
+	case RTW_MP_CONFIG_CMD_SET_CH_BW:
+		if (pmppriv->channel > 6000) {
+			pmp_arg.channel = pmppriv->channel - 6000;
+			pmp_arg.band = 2;
+		} else {
+			pmp_arg.channel = pmppriv->channel;
+			if (pmppriv->band == 2)
+				pmp_arg.band = 2;
+			else {
+				if (pmppriv->channel >= 36)
+				pmp_arg.band = 1;
+				else if (pmppriv->channel <= 14)
+				pmp_arg.band = 0;
+			}
+		}
+		RTW_INFO("%s,channel %d band: %d !!!\n", __func__, pmppriv->channel, pmp_arg.band);
+		pmp_arg.bandwidth = pmppriv->bandwidth;
+		pmp_arg.sc_idx = pmppriv->prime_channel_offset;
+		break;
+	case RTW_MP_CONFIG_CMD_SET_TXRX_MODE:
+		pmp_arg.is_tmac_mode = pmppriv->is_tmac_mode;
+		pmp_arg.tx_rfpath = pmppriv->curr_rfpath;
+		pmp_arg.rx_rfpath = pmppriv->curr_rfpath;
+		pmp_arg.ant_tx = pmppriv->mp_plcp_user[pmppriv->mp_plcp_useridx].plcp_nss;
+		pmp_arg.ant_rx = pmppriv->mp_plcp_user[pmppriv->mp_plcp_useridx].plcp_nss;
+		break;
+	case RTW_MP_CONFIG_CMD_SET_RATE_IDX:
+		pmp_arg.rate_idx= pmppriv->rateidx;
+		break;
+	case RTW_MP_CONFIG_CMD_SET_RF_TXRX_PATH:
+		pmp_arg.ant_tx = pmppriv->curr_rfpath;
+		pmp_arg.ant_rx = pmppriv->curr_rfpath;
+		break;
+	case RTW_MP_CONFIG_CMD_STOP_DUT:
+	case RTW_MP_CONFIG_CMD_START_DUT:
+	case RTW_MP_CONFIG_CMD_SET_PHY_INDEX:
+		pmp_arg.cur_phy = pmppriv->rtw_mp_cur_phy;
+		break;
+	case RTW_MP_CONFIG_CMD_SET_RESET_PHY_COUNT:
+	case RTW_MP_CONFIG_CMD_SET_RESET_MAC_COUNT:
+	case RTW_MP_CONFIG_CMD_SET_RESET_DRV_COUNT:
+		pmp_arg.cur_phy = pmppriv->rtw_mp_cur_phy;
+		break;
+	case RTW_MP_CONFIG_CMD_SWITCH_BT_PATH:
+		pmp_arg.btc_mode = pmppriv->btc_path;
+		break;
+	case RTW_MP_CONFIG_CMD_SET_MAC_LOOPBK_ENTER:
+	case RTW_MP_CONFIG_CMD_SET_MAC_LOOPBK_SPEED:
+		/*pmp_arg.loopbk_speed = pmppriv->loopbk_speed;*/
+		break;
+	case RTW_MP_CONFIG_CMD_SET_MAC_GENERNAL_IO_TEST:
+		pmppriv->mac_iotest_res = 0;
+		RTW_INFO("%s, MAC_GENERNAL_IO_TEST !!!\n", __func__);
+		break;
+	case RTW_MP_CONFIG_CMD_SET_GPIO:
+		pmp_arg.gpio_cfg.gpio_id = pmppriv->gpio_id;
+		pmp_arg.gpio_cfg.gpio_enable = pmppriv->gpio_enable;
+		break;
+	case RTW_MP_CONFIG_CMD_SWITCH_ANTENNA:
+		pmp_arg.ant_sw = pmppriv->ant_sw;
+		RTW_INFO("%s, MAC_GENERNAL_IO_TEST !!!\n", __func__);
+		break;
+	default:
+		RTW_INFO("%s, No CMD case match !!!\n", __func__);
+		return _FALSE;
+	}
+
+	rtw_mp_set_phl_cmd(padapter, (void*)&pmp_arg,  sizeof(struct rtw_mp_config_arg));
+	while (1) {
+		rtw_mp_get_phl_cmd(padapter, (void*)&pmp_arg,  sizeof(struct rtw_mp_config_arg));
+		if (pmp_arg.cmd_ok && pmp_arg.status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s, %d GET CMD OK !!!\n", __func__, cmdid);
+			if (cmdid == RTW_MP_CONFIG_CMD_SET_MAC_GENERNAL_IO_TEST)
+				pmppriv->mac_iotest_res = 1;
+			break;
+		} else {
+			i++;
+			rtw_msleep_os(1);
+			if (i == 1000) {
+				RTW_INFO("%s, %d GET CMD FAIL !!!\n", __func__, cmdid);
+				break;
+			}
+		}
+	}
+	return pmp_arg.cmd_ok;
+}
+
+void rtw_mp_phl_rx_physts(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg, bool bstart)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u8 i = 0;
+
+	rx_arg->mp_class = RTW_MP_CLASS_RX;
+	rx_arg->cmd = RTW_MP_RX_CMD_GET_PHYSTS;
+	rx_arg->enable = bstart;
+
+	rx_arg->cmd_ok = _FALSE;
+	rtw_mp_set_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+
+	while (i != 10) {
+			rtw_msleep_os(100);
+			rtw_mp_get_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+			if (rx_arg->cmd_ok && rx_arg->status == RTW_PHL_STATUS_SUCCESS) {
+				RTW_INFO("%s,GET CMD RX OK\n", __func__);
+				break;
+			} else {
+				if (i > 10) {
+					RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__,rx_arg->status);
+					break;
+				}
+				i++;
+			}
+	}
+}
+
+static u8 rtw_get_rx_cal_process_status(_adapter *padapter)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u32 i = 0;
+	u8 ret = _FALSE;
+
+	while (i <= 20) {
+		if (pmppriv->rx_cal_process == 0) {
+			ret = _TRUE;
+			RTW_INFO("%s,rx_cal_process false\n", __func__);
+			break;
+		} else
+			rtw_msleep_os(50);
+		i++;
+	}
+
+	return ret;
+}
+
+void rtw_mp_phl_rx_rssi(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u8 i = 0;
+
+	rx_arg->mp_class = RTW_MP_CLASS_RX;
+	rx_arg->cmd = RTW_MP_RX_CMD_GET_RSSI;
+	rx_arg->cmd_ok = _FALSE;
+
+	if (rtw_get_rx_cal_process_status(padapter)) {
+		pmppriv->rx_cal_process = 1;
+	rtw_mp_set_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+	}
+
+	while (i != 10) {
+			rtw_msleep_os(30);
+			rtw_mp_get_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+			if (rx_arg->cmd_ok && rx_arg->status == RTW_PHL_STATUS_SUCCESS) {
+				RTW_INFO("%s,Path%d rssi:%d GET CMD RX OK\n", __func__, rx_arg->rf_path, rx_arg->rssi);
+				break;
+			} else {
+				if (i > 50) {
+					RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__,rx_arg->status);
+					break;
+				}
+				i++;
+			}
+	}
+	pmppriv->rx_cal_process = 0;
+}
+
+void rtw_mp_phl_rx_gain_offset(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg, u8 path_num)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u8 all_path_num = path_num;
+	u8 rf_path = 0;
+
+	rx_arg->mp_class = RTW_MP_CLASS_RX;
+	rx_arg->cmd = RTW_MP_RX_CMD_SET_GAIN_OFFSET;
+
+	if (path_num > 1) {
+		path_num = 0;
+		all_path_num -= 1;
+	}
+	if (rtw_get_rx_cal_process_status(padapter)) {
+		pmppriv->rx_cal_process = 1;
+
+	for (rf_path = path_num; rf_path <= all_path_num; rf_path++) {
+		u16 i = 0;
+
+		rx_arg->rf_path = rf_path;
+		rx_arg->cmd_ok = _FALSE;
+
+		rtw_mp_set_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+
+		while (i != 10) {
+					rtw_msleep_os(30);
+				rtw_mp_get_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+				if (rx_arg->cmd_ok && rx_arg->status == RTW_PHL_STATUS_SUCCESS) {
+					RTW_INFO("%s,rf path %d GET CMD RX OK\n", __func__, rx_arg->rf_path);
+					break;
+				} else {
+						if (i > 50) {
+						RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__,rx_arg->status);
+						break;
+					}
+					i++;
+				}
+		}
+	}
+	}
+	pmppriv->rx_cal_process = 0;
+}
+
+
+void rtw_mp_phl_query_rx(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg ,u8 rx_qurey_type)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u8 rxcmd[2] = {0, 0};
+	u8 cmd_idx = 0;
+
+	rx_arg->mp_class = RTW_MP_CLASS_RX;
+	rx_arg->rx_ok = 0;
+	rx_arg->rx_err = 0;
+
+	if (rx_qurey_type == 0) {
+		rxcmd[0] = RTW_MP_RX_CMD_PHY_CRC_OK;
+		rxcmd[1] = RTW_MP_RX_CMD_PHY_CRC_ERR;
+	} else if (rx_qurey_type == 1) {
+		rxcmd[0] = RTW_MP_RX_CMD_MAC_CRC_OK;
+		rxcmd[1] = RTW_MP_RX_CMD_MAC_CRC_ERR;
+	} else if (rx_qurey_type == 2) {
+		rxcmd[0] = RTW_MP_RX_CMD_DRV_CRC_OK;
+		rxcmd[1] = RTW_MP_RX_CMD_DRV_CRC_ERR;
+	}
+
+	if (rtw_get_rx_cal_process_status(padapter)) {
+		pmppriv->rx_cal_process = 1;
+
+	for (cmd_idx = 0; cmd_idx < 2 ; cmd_idx++) {
+		u16 i = 0;
+
+		rx_arg->cmd = rxcmd[cmd_idx];
+		rx_arg->cmd_ok = _FALSE;
+		rtw_mp_set_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+
+		while (i != 10) {
+					rtw_msleep_os(30);
+				rtw_mp_get_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+				if (rx_arg->cmd_ok && rx_arg->status == RTW_PHL_STATUS_SUCCESS) {
+					RTW_INFO("%s,GET CMD RX OK:%d ,RX ERR:%d\n", __func__, rx_arg->rx_ok, rx_arg->rx_err);
+					break;
+				} else {
+						if (i > 50) {
+						RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__,rx_arg->status);
+						break;
+					}
+					i++;
+				}
+		}
+	}
+	}
+
+	pmppriv->rx_cal_process = 0;
+}
+
+u8 rtw_mp_phl_rx_cal_cmd(_adapter *padapter)
+{
+	struct rtw_mp_cal_arg	*cal_arg = NULL;
+	u8 rxcmd = RTW_MP_CAL_CMD_TRIGGER_WATCHDOG_CAL;
+
+	cal_arg = _rtw_malloc(sizeof(struct rtw_mp_cal_arg));
+	if (cal_arg==NULL) {
+		RTW_WARN("%s: _rtw_malloc null to exit\n", __func__);
+		return 0;
+	}
+
+	cal_arg->mp_class = RTW_MP_CLASS_CAL;
+	cal_arg->cmd = rxcmd;
+	cal_arg->cmd_ok = _FALSE;
+
+	if (rtw_get_rx_cal_process_status(padapter)) {
+		rtw_mp_set_phl_cmd(padapter, (void*)cal_arg, sizeof(struct rtw_mp_cal_arg));
+		rtw_msleep_os(50);
+		rtw_mp_get_phl_cmd(padapter, (void*)cal_arg, sizeof(struct rtw_mp_cal_arg));
+		if (cal_arg->cmd_ok && cal_arg->status == RTW_PHL_STATUS_SUCCESS)
+		RTW_DBG("%s,cal ok \n", __func__);
+		else {
+			RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__, cal_arg->status);
+		}
+	}
+	if (cal_arg)
+		_rtw_mfree(cal_arg, sizeof(struct rtw_mp_cal_arg));
+
+	return cal_arg->cmd_ok;
+}
+
+
+
+static thread_return mp_rx_phl_cal_thread(thread_context context)
+{
+	struct mp_priv	*pmp_priv;
+	_adapter *padapter;
+	pmp_priv = (struct mp_priv *)context;
+	padapter = pmp_priv->papdater;
+
+	rtw_thread_enter("RTW_MP_RX_Cal_THREAD");
+	while (1) {
+			pmp_priv->rx_cal_process = 1;
+			if (rtw_mp_phl_rx_cal_cmd(padapter)) {
+				RTW_DBG("%s,GET CMD cal ok \n", __func__);
+				
+			} else {
+				RTW_INFO("%s,GET CMD status FAIL\n", __func__);
+			}
+			pmp_priv->rx_cal_process = 0;
+
+			if (pmp_priv->rx_cal_stop ||
+				RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			goto exit;
+
+			flush_signals_thread();
+			rtw_msleep_os(2000);
+	}
+	exit:
+		RTW_INFO("%s: Exit\n", __func__);
+		pmp_priv->rx_cal_stop = 1;
+
+		rtw_thread_exit(NULL);
+		return 0;
+
+}
+
+void rtw_mp_rx_phl_cal_timer(_adapter *padapter)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+
+	pmppriv->rx_cal_thread = rtw_thread_start(
+					mp_rx_phl_cal_thread, pmppriv, "RTW_MP_Rx_CAL_THREAD");
+			if (pmppriv->rx_cal_thread == NULL)
+				RTW_ERR("Create Rx cal Thread Fail !!!!!\n");
+}
+
+void rtw_mp_set_crystal_cap(_adapter *padapter, u32 xcapvalue)
+{
+	struct rtw_mp_reg_arg cfg_arg;
+	u8 sc_xo_idx = 1;
+	u8 i = 0;
+
+	_rtw_memset((void *)&cfg_arg, 0, sizeof(struct rtw_mp_reg_arg));
+
+	cfg_arg.mp_class = RTW_MP_CLASS_REG;
+	cfg_arg.cmd = RTW_MP_REG_CMD_SET_XCAP;
+	cfg_arg.status = RTW_PHL_STATUS_FAILURE;
+
+	for (i <=0 ; i <= sc_xo_idx; i++) {
+		cfg_arg.cmd_ok = 0;
+		cfg_arg.sc_xo = i;
+		cfg_arg.io_value = xcapvalue;
+		rtw_mp_set_phl_cmd(padapter, (void*)&cfg_arg,  sizeof(struct rtw_mp_reg_arg));
+
+		while (i != 10) {
+				rtw_msleep_os(10);
+				rtw_mp_get_phl_cmd(padapter, (void*)&cfg_arg,  sizeof(struct rtw_mp_reg_arg));
+				if (cfg_arg.cmd_ok && cfg_arg.status == RTW_PHL_STATUS_SUCCESS) {
+					RTW_INFO("%s,SET CMD TX OK\n", __func__);
+					break;
+				} else {
+					if (i > 10) {
+						RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__, cfg_arg.status);
+						break;
+				}
+				i++;
+			}
+		}
+	}
+
+}
+
+u8 rtw_mp_phl_calibration(_adapter *padapter, struct rtw_mp_cal_arg	*pcal_arg, u8 cmdid)
+{
+	struct mp_priv	*pmppriv = &padapter->mppriv;
+	u16 i = 0;
+	u32 cmd_size = sizeof(struct rtw_mp_cal_arg);
+
+	pcal_arg->mp_class = RTW_MP_CLASS_CAL;
+	pcal_arg->cmd = cmdid;
+
+	rtw_mp_set_phl_cmd(padapter, (void*)pcal_arg, cmd_size);
+
+	while (i <= 10) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(padapter, (void*)pcal_arg, cmd_size);
+		if (pcal_arg->cmd_ok && pcal_arg->status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,SET CMD OK\n", __func__);
+			break;
+		} else {
+			if (i > 10) {
+				RTW_DBG("%s,GET CMD FAIL !!! status %d\n", __func__, pcal_arg->status);
+				break;
+			}
+			i++;
+		}
+	}
+
+	return pcal_arg->cmd_ok;
+}
+
+u8 rtw_mp_phl_reg(_adapter *padapter, struct rtw_mp_reg_arg	*reg_arg, u8 cmdid)
+{
+	struct mp_priv	*pmppriv = &padapter->mppriv;
+	u16 i = 0;
+	u32 cmd_size = sizeof(struct rtw_mp_reg_arg);
+
+	reg_arg->mp_class = RTW_MP_CLASS_REG;
+	reg_arg->cmd = cmdid;
+
+	rtw_mp_set_phl_cmd(padapter, (void*)reg_arg, cmd_size);
+
+	while (i <= 10) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(padapter, (void*)reg_arg, cmd_size);
+		if (reg_arg->cmd_ok && reg_arg->status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,SET CMD OK\n", __func__);
+			break;
+		} else {
+			if (i > 10) {
+				RTW_DBG("%s,GET CMD FAIL !!! status %d\n", __func__, reg_arg->status);
+				break;
+			}
+			i++;
+		}
+	}
+
+	return reg_arg->cmd_ok;
+}
+
+
+u8 rtw_update_giltf(_adapter *padapter)
+{
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 idx = 0;
+	u8 ppdu_type = pmp_priv->rtw_mp_pmact_ppdu_type;
+	u8 user_idx = pmp_priv->mp_plcp_useridx;
+
+	RTW_INFO("%s: ppdu_type=%s\n", __func__ , PPDU_TYPE_STR(pmp_priv->rtw_mp_pmact_ppdu_type));
+	if (ppdu_type == RTW_MP_TYPE_HT_MF || ppdu_type == RTW_MP_TYPE_HT_GF
+										|| ppdu_type == RTW_MP_TYPE_VHT) {
+		pmp_priv->st_giltf[0].gi = 0;
+		pmp_priv->st_giltf[0].ltf = 0;
+		strcpy(pmp_priv->st_giltf[0].type_str, "0.4");
+		pmp_priv->st_giltf[1].gi = 1;
+		pmp_priv->st_giltf[1].ltf = 0;
+		strcpy(pmp_priv->st_giltf[1].type_str, "0.8");
+		idx = 1;
+	} else if (ppdu_type > RTW_MP_TYPE_VHT) {
+		if (ppdu_type == RTW_MP_TYPE_HE_SU || ppdu_type == RTW_MP_TYPE_HE_ER_SU) {
+
+			if (!pmp_priv->mp_plcp_user[user_idx].dcm && !pmp_priv->rtw_mp_stbc) {
+				pmp_priv->st_giltf[0].gi = 1;
+				pmp_priv->st_giltf[0].ltf = 0;
+				strcpy(pmp_priv->st_giltf[0].type_str, "1x 0.8");
+				pmp_priv->st_giltf[1].gi = 1;
+				pmp_priv->st_giltf[1].ltf = 1;
+				strcpy(pmp_priv->st_giltf[1].type_str, "2x 0.8");
+				pmp_priv->st_giltf[2].gi = 2;
+				pmp_priv->st_giltf[2].ltf = 1;
+				strcpy(pmp_priv->st_giltf[2].type_str, "2x 1.6");
+				pmp_priv->st_giltf[3].gi = 3;
+				pmp_priv->st_giltf[3].ltf = 2;
+				strcpy(pmp_priv->st_giltf[3].type_str, "4x 3.2");
+				pmp_priv->st_giltf[4].gi = 1;
+				pmp_priv->st_giltf[4].ltf = 2;
+				strcpy(pmp_priv->st_giltf[4].type_str, "4x 0.8");
+				idx = 4;
+			} else {
+				pmp_priv->st_giltf[0].gi = 1;
+				pmp_priv->st_giltf[0].ltf = 0;
+				strcpy(pmp_priv->st_giltf[0].type_str, "1x 0.8");
+				pmp_priv->st_giltf[1].gi = 1;
+				pmp_priv->st_giltf[1].ltf = 1;
+				strcpy(pmp_priv->st_giltf[1].type_str, "2x 0.8");
+				pmp_priv->st_giltf[2].gi = 2;
+				pmp_priv->st_giltf[2].ltf = 1;
+				strcpy(pmp_priv->st_giltf[2].type_str, "2x 1.6");
+				pmp_priv->st_giltf[3].gi = 3;
+				pmp_priv->st_giltf[3].ltf = 2;
+				strcpy(pmp_priv->st_giltf[3].type_str, "4x 3.2");
+				idx = 3;
+			}
+		}else if (ppdu_type == RTW_MP_TYPE_HE_MU_OFDMA) {
+			pmp_priv->st_giltf[0].gi = 1;
+			pmp_priv->st_giltf[0].ltf = 1;
+			strcpy(pmp_priv->st_giltf[0].type_str, "2x 0.8");
+			pmp_priv->st_giltf[1].gi = 2;
+			pmp_priv->st_giltf[1].ltf = 1;
+			strcpy(pmp_priv->st_giltf[1].type_str, "2x 1.6");
+			pmp_priv->st_giltf[2].gi = 1;
+			pmp_priv->st_giltf[2].ltf = 2;
+			strcpy(pmp_priv->st_giltf[2].type_str, "4x 0.8");
+			pmp_priv->st_giltf[3].gi = 3;
+			pmp_priv->st_giltf[3].ltf = 2;
+			strcpy(pmp_priv->st_giltf[3].type_str, "4x 3.2");
+			idx = 3;
+		} else if (ppdu_type == RTW_MP_TYPE_HE_TB) {
+				pmp_priv->st_giltf[0].gi = 2;
+				pmp_priv->st_giltf[0].ltf = 0;
+				strcpy(pmp_priv->st_giltf[0].type_str, "1x 1.6");
+				pmp_priv->st_giltf[1].gi = 2;
+				pmp_priv->st_giltf[1].ltf = 1;
+				strcpy(pmp_priv->st_giltf[1].type_str, "2x 1.6");
+				pmp_priv->st_giltf[2].gi = 3;
+				pmp_priv->st_giltf[2].ltf = 2;
+				strcpy(pmp_priv->st_giltf[2].type_str, "4x 3.2");
+				idx = 2;
+			}
+		}
+	return idx;
+}
+
+void rtw_mp_update_coding(_adapter *padapter)
+{
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 ppdu_type = pmp_priv->rtw_mp_pmact_ppdu_type;
+	u8 user_idx = pmp_priv->mp_plcp_useridx;
+
+	if (ppdu_type == RTW_MP_TYPE_HE_SU || (ppdu_type == RTW_MP_TYPE_HE_TB && pmp_priv->bandwidth >= CHANNEL_WIDTH_40) ||
+		pmp_priv->rtw_mp_ru_tone >= MP_RU_TONE_484) {
+
+		RTW_INFO("%s, PPDU HE SU , over 40M, RU Tone over 484\n", __func__);
+		pmp_priv->rtw_coding = 1;/* 1 LDPC, 0 BCC */
+	} else if (ppdu_type == RTW_MP_TYPE_HE_MU_OFDMA) {
+		RTW_INFO("%s, PPDU HE MU not support seeting\n", __func__);
+	}
+
+	RTW_INFO("%s, coding: %s\n", __func__, (pmp_priv->mp_plcp_user[user_idx].coding?"LDPC":"BCC"));
+}
+
+u8 rtw_mp_update_ru_tone(_adapter *padapter)
+{
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 user_idx = pmp_priv->mp_plcp_useridx;
+	u8 cur_coding = pmp_priv->mp_plcp_user[user_idx].coding;
+	u8 ruidx = MP_RU_TONE_26;
+	u8 i = 0 , j = 0;
+
+	if (cur_coding == 0) {/* 1 LDPC, 0 BCC */
+		ruidx = MP_RU_TONE_242;
+		if (pmp_priv->rtw_mp_ru_tone < MP_RU_TONE_242)
+			pmp_priv->rtw_mp_ru_tone = MP_RU_TONE_242;
+	} else {
+		ruidx = MP_RU_TONE_26;
+	}
+
+	for (i = 0; i <= 5; i++) {
+		pmp_priv->ru_tone_sel_list[i] = ruidx++;
+		if (ruidx > MP_RU_TONE_996)
+			break;
+	}
+	return i;
+}
+
+u8 rtw_mp_update_ru_alloc(_adapter *padapter)
+{
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 cur_tone = pmp_priv->rtw_mp_ru_tone;
+	u8 i = 0;
+	u8 alloc_start = 0, alloc_end = 0;
+	u8 user_idx = pmp_priv->mp_plcp_useridx;
+
+	switch (cur_tone) {
+	case MP_RU_TONE_26:
+					alloc_start = 0;
+					alloc_end = 9;
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_40)
+						alloc_end = 18;
+
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_80)
+						alloc_end = 37;
+					break;
+	case MP_RU_TONE_52:
+					alloc_start = 37;
+					alloc_end = 41;
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_40)
+						alloc_end = 45;
+
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_80)
+						alloc_end = 53;
+
+					break;
+	case MP_RU_TONE_106:
+					alloc_start = 53;
+					alloc_end = 55;
+
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_40)
+						alloc_end = 57;
+
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_80)
+						alloc_end = 61;
+
+					break;
+	case MP_RU_TONE_242:
+					alloc_start = 61;
+					alloc_end = 61;
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_40)
+						alloc_end = 62;
+
+					if (pmp_priv->bandwidth == CHANNEL_WIDTH_80)
+						alloc_end = 64;
+
+					break;
+	case MP_RU_TONE_484:
+					alloc_start = 65;
+					alloc_end = 66;
+					break;
+	case MP_RU_TONE_996:
+					alloc_start = 67;
+					alloc_end = 67;
+					break;
+	};
+
+	if (pmp_priv->mp_plcp_user[user_idx].ru_alloc < alloc_start ||
+			pmp_priv->mp_plcp_user[user_idx].ru_alloc > alloc_end)
+		pmp_priv->mp_plcp_user[user_idx].ru_alloc = alloc_start;
+
+	for (i = 0; i < 68; i++) {
+		if (alloc_start + i > alloc_end)
+			break;
+		pmp_priv->ru_alloc_list[i] = alloc_start + i;
+		RTW_INFO("%s, Ru alloc:%d\n", __func__, pmp_priv->ru_alloc_list[i]);
+
+	}
+	return i;
+}
+
+u32 rtw_mp_get_tssi_de(_adapter *padapter, u8 rf_path)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.rfpath = rf_path;
+
+	if (rtw_mp_phl_txpower(padapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_GET_TSSI))
+		return ptxpwr_arg.tssi;
+	else
+		return 0;
+}
+
+s32 rtw_mp_get_online_tssi_de(_adapter *padapter, s32 out_pwr, s32 tgdbm, u8 rf_path)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.dbm = tgdbm;
+	ptxpwr_arg.pout = out_pwr;
+	ptxpwr_arg.rfpath = rf_path;
+
+	if (rtw_mp_phl_txpower(padapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_GET_ONLINE_TSSI_DE))
+		return ptxpwr_arg.online_tssi_de;
+	else
+		return 0;
+}
+
+u8 rtw_mp_set_tsside2verify(_adapter *padapter, u32 tssi_de, u8 rf_path)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u8 tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	u8 i = 0;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.tssi = tssi_de;
+	ptxpwr_arg.rfpath = rf_path;
+
+	if (rtw_mp_phl_txpower(padapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_SET_TSSI))
+		return true;
+	else
+		return false;
+}
+
+u8 rtw_mp_set_tssi_pwrtrk(_adapter *padapter, u8 tssi_state)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.txpwr_track_status = tssi_state;
+
+	if (rtw_mp_phl_txpower(padapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_SET_PWR_TRACK_STATUS))
+		return true;
+	else
+		return false;
+}
+
+u8 rtw_mp_get_tssi_pwrtrk(_adapter *padapter)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	u8	tssi_state = RTW_MP_TSSI_OFF;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	if (rtw_mp_phl_txpower(padapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_GET_PWR_TRACK_STATUS)) {
+		tssi_state = ptxpwr_arg.txpwr_track_status;
+		RTW_INFO("%s, get tssi_state = %d\n", __func__, tssi_state);
+	} else
+		RTW_INFO("%s, rtw_mp_phl_txpower fail\n", __func__);
+
+	return tssi_state;
+}
+
+u8 rtw_mp_set_tx_shape_idx(_adapter *padapter)
+{
+	struct rtw_mp_txpwr_arg	ptxpwr_arg;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+
+	_rtw_memset((void *)&ptxpwr_arg, 0, sizeof(struct rtw_mp_txpwr_arg));
+
+	ptxpwr_arg.is_cck = rtw_mp_is_cck_rate(pmppriv->rateidx);
+	ptxpwr_arg.channel = pmppriv->channel;
+	ptxpwr_arg.sharp_id = pmppriv->tx_shape_idx;
+	RTW_INFO("%s(), tx_shape id=%d\n", __func__, ptxpwr_arg.sharp_id);
+
+	if (rtw_mp_phl_txpower(padapter, &ptxpwr_arg, RTW_MP_TXPWR_CMD_SET_TX_POW_PATTERN_SHARP))
+		return true;
+	else
+		return false;
+}
+
+#if 0
+void rtw_mp_halt(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	uint status = _FALSE;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	struct rtw_test_module_info test_module_info;
+	struct rtw_mp_test_rpt my_mp_test_rpt;
+	struct mp_tx	*pmptx = &pmppriv->tx;
+
+	if (pmppriv->mode == MP_ON) {
+		status = rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_STOP_DUT);
+		RTW_INFO("RTW_MP_CONFIG_CMD_STOP_DUT %s\n", status == _TRUE ? "ok" : "fail");
+		rtw_phl_test_get_rpt(dvobj->phl_com, (u8*)&my_mp_test_rpt, sizeof(my_mp_test_rpt));
+
+		if (my_mp_test_rpt.status == _TRUE)
+			RTW_INFO("TM Sub finished OK!!!\n");
+#ifdef CONFIG_BTC
+		if (pmppriv->mp_keep_btc_mode != BTC_MODE_MAX) {
+			pmppriv->btc_path = pmppriv->mp_keep_btc_mode;
+			rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SWITCH_BT_PATH);
+		}
+#endif
+		test_module_info.tm_mode = RTW_DRV_MODE_NORMAL;
+		test_module_info.tm_type = RTW_TEST_SUB_MODULE_MP;
+		dvobj->phl_com->dev_sw_cap.ps_cap.ips_en = pmppriv->keep_ips_status;
+		dvobj->phl_com->dev_sw_cap.ps_cap.lps_en = pmppriv->keep_lps_status;
+
+		rtw_phl_test_submodule_deinit(dvobj->phl_com, &test_module_info);
+
+		MPT_DeInitAdapter(padapter);
+		padapter->registrypriv.mp_mode = 0;
+		RTW_INFO("%s() done!!!\n", __func__);
+	}
+	pmppriv->mode = MP_OFF;
+
+	if (pmptx->pallocated_buf)
+		rtw_mfree(pmptx->pallocated_buf, pmptx->buf_size);
+}
+#endif
+
+void rtw_mp_phl_set_mac_loopbk(_adapter *padapter)
+{
+
+	rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_MAC_LOOPBK_ENTER);
+	RTW_INFO("%s() done!!!\n", __func__);
+}
+
+void rtw_mp_phl_set_mac_loopbk_speed(_adapter *padapter)
+{
+
+	rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_MAC_LOOPBK_SPEED);
+	RTW_INFO("%s() done!!!\n", __func__);
+}
+
+void rtw_mp_phl_set_mac_io_test(_adapter *padapter)
+{
+
+	rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_MAC_GENERNAL_IO_TEST);
+	RTW_INFO("%s() done!!!\n", __func__);
+}
+
+#if 0
+u8 rtw_mp_get_tx_req_recycle(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	uint status = _FALSE;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+
+	u32 idle_cnt = 0, busy_cnt = 0, total_cnt = 0;
+	u8 tx_report = 0;
+	u32 i = 0;
+#if 0
+	while(1) {
+		rtw_msleep_os(10);
+		rtw_phl_get_tx_req_stats(GET_PHL_INFO(dvobj),
+			&idle_cnt, &busy_cnt, &total_cnt);
+
+		if ((total_cnt - idle_cnt) == 0) {
+			tx_report = 1;
+			RTW_INFO("Tx report recycle success !\n");
+			break;
+		}
+		if (i >= 100)
+			break;
+		i++;
+	}
+#endif
+	RTW_INFO("%s: tx req idle cnt=%d ,busy=%d, total=%d!!\n",
+		__func__, idle_cnt, busy_cnt, total_cnt);
+	return tx_report;
+}
+#endif
+
+void rtw_mp_phl_rx_reset_fltr(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg, bool bstart)
+{
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	u8 i = 0;
+
+	rx_arg->mp_class = RTW_MP_CLASS_RX;
+	rx_arg->cmd = RTW_MP_RX_CMD_SET_RX_FLTR;
+	rx_arg->rx_fltr_enable = bstart;
+	_rtw_memcpy(rx_arg->rx_fltr_addr, pmppriv->network_macaddr, ETH_ALEN);
+
+	rtw_mp_set_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+
+	while (i != 10) {
+		rtw_msleep_os(100);
+		rtw_mp_get_phl_cmd(padapter, (void*)rx_arg,  sizeof(struct rtw_mp_rx_arg));
+		if (rx_arg->cmd_ok && rx_arg->status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,GET CMD RX OK\n", __func__);
+			break;
+		} else {
+			if (i > 10) {
+				RTW_INFO("%s,GET CMD FAIL !!! status %d\n", __func__,rx_arg->status);
+				break;
+			}
+			i++;
+		}
+	}
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_p2p.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_p2p.c
new file mode 100644
index 000000000000..23b315af09ca
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_p2p.c
@@ -0,0 +1,3345 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_P2P_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_P2P
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37))
+	#error "Linux kernel version is too old to enable CONFIG_P2P"
+#endif
+#ifndef CONFIG_IOCTL_CFG80211
+	#error "Enable CONFIG_P2P without CONFIG_IOCTL_CFG80211"
+#endif
+
+int is_any_client_associated(_adapter *padapter)
+{
+	return padapter->stapriv.asoc_list_cnt ? _TRUE : _FALSE;
+}
+
+#ifdef CONFIG_WFD
+u32 build_beacon_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u16 val16 = 0;
+	u32 len = 0, wfdielen = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110812 */
+	/*	According to the WFD Specification, the beacon frame should contain 4 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID */
+	/*	3. Coupled Sink Information */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+
+	if (P2P_ROLE_GO == pwdinfo->role) {
+		if (is_any_client_associated(pwdinfo->padapter)) {
+			/*	WFD primary sink + WiFi Direct mode + WSD (WFD Service Discovery) */
+			val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_WSD;
+			RTW_PUT_BE16(wfdie + wfdielen, val16);
+		} else {
+			/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD (WFD Service Discovery) */
+			val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+			RTW_PUT_BE16(wfdie + wfdielen, val16);
+		}
+
+	} else {
+		/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+		val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+		RTW_PUT_BE16(wfdie + wfdielen, val16);
+	}
+
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_probe_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u16 val16 = 0;
+	u32 len = 0, wfdielen = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110812 */
+	/*	According to the WFD Specification, the probe request frame should contain 4 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID */
+	/*	3. Coupled Sink Information */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+
+	if (1 == pwdinfo->wfd_tdls_enable) {
+		/*	WFD primary sink + available for WFD session + WiFi TDLS mode + WSC ( WFD Service Discovery )	 */
+		val16 = pwfd_info->wfd_device_type |
+			WFD_DEVINFO_SESSION_AVAIL |
+			WFD_DEVINFO_WSD |
+			WFD_DEVINFO_PC_TDLS;
+		RTW_PUT_BE16(wfdie + wfdielen, val16);
+	} else {
+		/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSC ( WFD Service Discovery )	 */
+		val16 = pwfd_info->wfd_device_type |
+			WFD_DEVINFO_SESSION_AVAIL |
+			WFD_DEVINFO_WSD;
+		RTW_PUT_BE16(wfdie + wfdielen, val16);
+	}
+
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_probe_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf, u8 tunneled)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+	u16 v16 = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110812 */
+	/*	According to the WFD Specification, the probe response frame should contain 4 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID */
+	/*	3. Coupled Sink Information */
+	/*	4. WFD Session Information */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + available for WFD session + WiFi Direct mode */
+
+	if (P2P_ROLE_GO == pwdinfo->role) {
+		if (is_any_client_associated(pwdinfo->padapter)) {
+			if (pwdinfo->wfd_tdls_enable) {
+				/*	TDLS mode + WSD ( WFD Service Discovery ) */
+				v16 = pwfd_info->wfd_device_type | WFD_DEVINFO_WSD | WFD_DEVINFO_PC_TDLS | WFD_DEVINFO_HDCP_SUPPORT;
+				RTW_PUT_BE16(wfdie + wfdielen, v16);
+			} else {
+				/*	WiFi Direct mode + WSD ( WFD Service Discovery ) */
+				v16 =  pwfd_info->wfd_device_type | WFD_DEVINFO_WSD | WFD_DEVINFO_HDCP_SUPPORT;
+				RTW_PUT_BE16(wfdie + wfdielen, v16);
+			}
+		} else {
+			if (pwdinfo->wfd_tdls_enable) {
+				/*	available for WFD session + TDLS mode + WSD ( WFD Service Discovery ) */
+				v16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD | WFD_DEVINFO_PC_TDLS | WFD_DEVINFO_HDCP_SUPPORT;
+				RTW_PUT_BE16(wfdie + wfdielen, v16);
+			} else {
+				/*	available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+				v16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD | WFD_DEVINFO_HDCP_SUPPORT;
+				RTW_PUT_BE16(wfdie + wfdielen, v16);
+			}
+		}
+	} else {
+		if (pwdinfo->wfd_tdls_enable) {
+			/*	available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+			v16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD | WFD_DEVINFO_PC_TDLS | WFD_DEVINFO_HDCP_SUPPORT;
+			RTW_PUT_BE16(wfdie + wfdielen, v16);
+		} else {
+			/*	available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+			v16 =  pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD | WFD_DEVINFO_HDCP_SUPPORT;
+			RTW_PUT_BE16(wfdie + wfdielen, v16);
+		}
+	}
+
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+		/*	WFD Session Information ATTR */
+		/*	Type: */
+		wfdie[wfdielen++] = WFD_ATTR_SESSION_INFO;
+
+		/*	Length: */
+		/*	Note: In the WFD specification, the size of length field is 2. */
+		RTW_PUT_BE16(wfdie + wfdielen, 0x0000);
+		wfdielen += 2;
+
+		/*	Todo: to add the list of WFD device info descriptor in WFD group. */
+
+	}
+#ifdef CONFIG_CONCURRENT_MODE
+#ifdef CONFIG_TDLS
+	{
+		int i;
+		_adapter *iface = NULL;
+		struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if ((iface) && rtw_is_adapter_up(iface)) {
+				if (iface == padapter)
+					continue;
+				if (!rtw_iface_at_same_hwband(padapter, iface))
+					continue;
+				if ((tunneled == 0) && (iface->wdinfo.wfd_tdls_enable == 1)) {
+					/*	Alternative MAC Address ATTR
+						Type:					*/
+					wfdie[wfdielen++] = WFD_ATTR_ALTER_MAC;
+
+					/*	Length:
+						Note: In the WFD specification, the size of length field is 2.*/
+					RTW_PUT_BE16(wfdie + wfdielen,  ETH_ALEN);
+					wfdielen += 2;
+
+					/*	Value:
+						Alternative MAC Address*/
+					_rtw_memcpy(wfdie + wfdielen, adapter_mac_addr(iface), ETH_ALEN);
+					wfdielen += ETH_ALEN;
+				}
+			}
+		}
+	}
+
+#endif /* CONFIG_TDLS*/
+#endif /* CONFIG_CONCURRENT_MODE */
+
+	pbuf = rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_assoc_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u16 val16 = 0;
+	u32 len = 0, wfdielen = 0;
+	_adapter			*padapter = NULL;
+	struct mlme_priv			*pmlmepriv = NULL;
+	struct wifi_display_info		*pwfd_info = NULL;
+
+	padapter = pwdinfo->padapter;
+	pmlmepriv = &padapter->mlmepriv;
+	pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/* Only P2P GC need to include the WFD IE in the association request */
+	if (!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_CLIENT))
+		goto exit;
+
+	/* WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110812 */
+	/*	According to the WFD Specification, the probe request frame should contain 4 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID */
+	/*	3. Coupled Sink Information */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_assoc_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	u16 val16 = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110812 */
+	/*	According to the WFD Specification, the probe request frame should contain 4 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID */
+	/*	3. Coupled Sink Information */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_nego_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	u16 val16 = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110825 */
+	/*	According to the WFD Specification, the negotiation request frame should contain 3 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID ( Optional ) */
+	/*	3. Local IP Adress ( Optional ) */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + WiFi Direct mode + WSD ( WFD Service Discovery ) + WFD Session Available */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_WSD | WFD_DEVINFO_SESSION_AVAIL;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_nego_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	u16 val16 = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110825 */
+	/*	According to the WFD Specification, the negotiation request frame should contain 3 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID ( Optional ) */
+	/*	3. Local IP Adress ( Optional ) */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + WiFi Direct mode + WSD ( WFD Service Discovery ) + WFD Session Available */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_WSD | WFD_DEVINFO_SESSION_AVAIL;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_nego_confirm_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	u16 val16 = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110825 */
+	/*	According to the WFD Specification, the negotiation request frame should contain 3 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID ( Optional ) */
+	/*	3. Local IP Adress ( Optional ) */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + WiFi Direct mode + WSD ( WFD Service Discovery ) + WFD Session Available */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_WSD | WFD_DEVINFO_SESSION_AVAIL;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+
+	pbuf = rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_invitation_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	u16 val16 = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110825 */
+	/*	According to the WFD Specification, the provision discovery request frame should contain 3 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID ( Optional ) */
+	/*	3. Local IP Adress ( Optional ) */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	if (P2P_ROLE_GO == pwdinfo->role) {
+		/*	WFD Session Information ATTR */
+		/*	Type: */
+		wfdie[wfdielen++] = WFD_ATTR_SESSION_INFO;
+
+		/*	Length: */
+		/*	Note: In the WFD specification, the size of length field is 2. */
+		RTW_PUT_BE16(wfdie + wfdielen, 0x0000);
+		wfdielen += 2;
+
+		/*	Todo: to add the list of WFD device info descriptor in WFD group. */
+
+	}
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_invitation_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u16 val16 = 0;
+	u32 len = 0, wfdielen = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110825 */
+	/*	According to the WFD Specification, the provision discovery request frame should contain 3 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID ( Optional ) */
+	/*	3. Local IP Adress ( Optional ) */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	if (P2P_ROLE_GO == pwdinfo->role) {
+		/*	WFD Session Information ATTR */
+		/*	Type: */
+		wfdie[wfdielen++] = WFD_ATTR_SESSION_INFO;
+
+		/*	Length: */
+		/*	Note: In the WFD specification, the size of length field is 2. */
+		RTW_PUT_BE16(wfdie + wfdielen, 0x0000);
+		wfdielen += 2;
+
+		/*	Todo: to add the list of WFD device info descriptor in WFD group. */
+
+	}
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_provdisc_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	u16 val16 = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110825 */
+	/*	According to the WFD Specification, the provision discovery request frame should contain 3 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID ( Optional ) */
+	/*	3. Local IP Adress ( Optional ) */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+
+u32 build_provdisc_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf)
+{
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 len = 0, wfdielen = 0;
+	u16 val16 = 0;
+	_adapter *padapter = pwdinfo->padapter;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->wdinfo.wfd_info;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+	/*	WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/*	WFA WFD v1.0 */
+
+	/*	Commented by Albert 20110825 */
+	/*	According to the WFD Specification, the provision discovery response frame should contain 3 WFD attributes */
+	/*	1. WFD Device Information */
+	/*	2. Associated BSSID ( Optional ) */
+	/*	3. Local IP Adress ( Optional ) */
+
+
+	/*	WFD Device Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value1: */
+	/*	WFD device information */
+	/*	WFD primary sink + available for WFD session + WiFi Direct mode + WSD ( WFD Service Discovery ) */
+	val16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL | WFD_DEVINFO_WSD;
+	RTW_PUT_BE16(wfdie + wfdielen, val16);
+	wfdielen += 2;
+
+	/*	Value2: */
+	/*	Session Management Control Port */
+	/*	Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->rtsp_ctrlport);
+	wfdielen += 2;
+
+	/*	Value3: */
+	/*	WFD Device Maximum Throughput */
+	/*	300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/*	Associated BSSID ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	wfdielen += ETH_ALEN;
+
+	/*	Coupled Sink Information ATTR */
+	/*	Type: */
+	wfdie[wfdielen++] = WFD_ATTR_COUPLED_SINK_INFO;
+
+	/*	Length: */
+	/*	Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0007);
+	wfdielen += 2;
+
+	/*	Value: */
+	/*	Coupled Sink Status bitmap */
+	/*	Not coupled/available for Coupling */
+	wfdie[wfdielen++] = 0;
+	/* MAC Addr. */
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+	wfdie[wfdielen++] = 0;
+
+	rtw_set_ie(pbuf, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, &len);
+
+exit:
+	return len;
+}
+#endif /* CONFIG_WFD */
+
+u32 process_probe_req_p2p_ie(struct wifidirect_info *pwdinfo, u8 *pframe, uint len)
+{
+	u8 *p;
+	u32 ret = _FALSE;
+	u8 *p2pie;
+	u32	p2pielen = 0;
+	int ssid_len = 0, rate_cnt = 0;
+
+	p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, _SUPPORTEDRATES_IE_, (int *)&rate_cnt,
+		       len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_);
+
+	if (rate_cnt <= 4) {
+		int i, g_rate = 0;
+
+		for (i = 0; i < rate_cnt; i++) {
+			if (((*(p + 2 + i) & 0xff) != 0x02) &&
+			    ((*(p + 2 + i) & 0xff) != 0x04) &&
+			    ((*(p + 2 + i) & 0xff) != 0x0B) &&
+			    ((*(p + 2 + i) & 0xff) != 0x16))
+				g_rate = 1;
+		}
+
+		if (g_rate == 0) {
+			/*	There is no OFDM rate included in SupportedRates IE of this probe request frame */
+			/*	The driver should response this probe request. */
+			return ret;
+		}
+	} else {
+		/*	rate_cnt > 4 means the SupportRates IE contains the OFDM rate because the count of CCK rates are 4. */
+		/*	We should proceed the following check for this probe request. */
+	}
+
+	/*	Added comments by Albert 20100906 */
+	/*	There are several items we should check here. */
+	/*	1. This probe request frame must contain the P2P IE. (Done) */
+	/*	2. This probe request frame must contain the wildcard SSID. (Done) */
+	/*	3. Wildcard BSSID. (Todo) */
+	/*	4. Destination Address. ( Done in mgt_dispatcher function ) */
+	/*	5. Requested Device Type in WSC IE. (Todo) */
+	/*	6. Device ID attribute in P2P IE. (Todo) */
+
+	p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, _SSID_IE_, (int *)&ssid_len,
+		       len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_);
+
+	ssid_len &= 0xff;	/*	Just last 1 byte is valid for ssid len of the probe request */
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DEVICE) || rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+		p2pie = rtw_get_p2p_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_ , len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_ , NULL, &p2pielen);
+		if (p2pie) {
+			if ((p != NULL) && _rtw_memcmp((void *)(p + 2), (void *) pwdinfo->p2p_wildcard_ssid , 7)) {
+				/* todo: */
+				/* Check Requested Device Type attributes in WSC IE. */
+				/* Check Device ID attribute in P2P IE */
+
+				ret = _TRUE;
+			} else if ((p != NULL) && (ssid_len == 0))
+				ret = _TRUE;
+		} else {
+			/* non -p2p device */
+		}
+
+	}
+
+
+	return ret;
+
+}
+
+u32 process_assoc_req_p2p_ie(struct wifidirect_info *pwdinfo, u8 *pframe, uint len, struct sta_info *psta)
+{
+	u8 status_code = P2P_STATUS_SUCCESS;
+	u8 *pbuf, *pattr_content = NULL;
+	u32 attr_contentlen = 0;
+	u16 cap_attr = 0;
+	unsigned short	frame_type, ie_offset = 0;
+	u8 *ies;
+	u32 ies_len;
+	u8 *p2p_ie;
+	u32	p2p_ielen = 0;
+
+	if (!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO))
+		return P2P_STATUS_FAIL_REQUEST_UNABLE;
+
+	frame_type = get_frame_sub_type(pframe);
+	if (frame_type == WIFI_ASSOCREQ)
+		ie_offset = _ASOCREQ_IE_OFFSET_;
+	else /* WIFI_REASSOCREQ */
+		ie_offset = _REASOCREQ_IE_OFFSET_;
+
+	ies = pframe + WLAN_HDR_A3_LEN + ie_offset;
+	ies_len = len - WLAN_HDR_A3_LEN - ie_offset;
+
+	p2p_ie = rtw_get_p2p_ie(ies , ies_len , NULL, &p2p_ielen);
+
+	if (!p2p_ie) {
+		RTW_INFO("[%s] P2P IE not Found!!\n", __FUNCTION__);
+		status_code =  P2P_STATUS_FAIL_INVALID_PARAM;
+	} else
+		RTW_INFO("[%s] P2P IE Found!!\n", __FUNCTION__);
+
+	while (p2p_ie) {
+		/* Check P2P Capability ATTR */
+		attr_contentlen = sizeof(cap_attr);
+		if (rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_CAPABILITY, (u8 *)&cap_attr, (uint *) &attr_contentlen)) {
+			RTW_INFO("[%s] Got P2P Capability Attr!!\n", __FUNCTION__);
+			cap_attr = le16_to_cpu(cap_attr);
+			psta->dev_cap = cap_attr & 0xff;
+		}
+
+		/* Check Extended Listen Timing ATTR */
+
+
+		/* Check P2P Device Info ATTR */
+		if (rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_DEVICE_INFO, NULL, (uint *)&attr_contentlen)) {
+			RTW_INFO("[%s] Got P2P DEVICE INFO Attr!!\n", __FUNCTION__);
+			pattr_content = pbuf = rtw_zmalloc(attr_contentlen);
+			if (pattr_content) {
+				u8 num_of_secdev_type;
+				u16 dev_name_len;
+
+
+				rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_DEVICE_INFO , pattr_content, (uint *)&attr_contentlen);
+
+				_rtw_memcpy(psta->dev_addr, 	pattr_content, ETH_ALEN);/* P2P Device Address */
+
+				pattr_content += ETH_ALEN;
+
+				_rtw_memcpy(&psta->config_methods, pattr_content, 2);/* Config Methods */
+				psta->config_methods = be16_to_cpu(psta->config_methods);
+
+				pattr_content += 2;
+
+				_rtw_memcpy(psta->primary_dev_type, pattr_content, 8);
+
+				pattr_content += 8;
+
+				num_of_secdev_type = *pattr_content;
+				pattr_content += 1;
+
+				if (num_of_secdev_type == 0)
+					psta->num_of_secdev_type = 0;
+				else {
+					u32 len;
+
+					psta->num_of_secdev_type = num_of_secdev_type;
+
+					len = (sizeof(psta->secdev_types_list) < (num_of_secdev_type * 8)) ? (sizeof(psta->secdev_types_list)) : (num_of_secdev_type * 8);
+
+					_rtw_memcpy(psta->secdev_types_list, pattr_content, len);
+
+					pattr_content += (num_of_secdev_type * 8);
+				}
+
+
+				/* dev_name_len = attr_contentlen - ETH_ALEN - 2 - 8 - 1 - (num_of_secdev_type*8); */
+				psta->dev_name_len = 0;
+				if (WPS_ATTR_DEVICE_NAME == be16_to_cpu(*(u16 *)pattr_content)) {
+					dev_name_len = be16_to_cpu(*(u16 *)(pattr_content + 2));
+
+					psta->dev_name_len = (sizeof(psta->dev_name) < dev_name_len) ? sizeof(psta->dev_name) : dev_name_len;
+
+					_rtw_memcpy(psta->dev_name, pattr_content + 4, psta->dev_name_len);
+				}
+
+				rtw_mfree(pbuf, attr_contentlen);
+
+			}
+
+		}
+
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+
+	}
+
+	return status_code;
+
+}
+
+#ifdef CONFIG_IOCTL_CFG80211
+#if 0
+static void rtw_change_p2pie_op_ch(_adapter *padapter, const u8 *frame_body, u32 len, u8 ch)
+{
+	u8 *ies, *p2p_ie;
+	u32 ies_len, p2p_ielen;
+
+	ies = (u8 *)(frame_body + _PUBLIC_ACTION_IE_OFFSET_);
+	ies_len = len - _PUBLIC_ACTION_IE_OFFSET_;
+
+	p2p_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &p2p_ielen);
+
+	while (p2p_ie) {
+		u32	attr_contentlen = 0;
+		u8 *pattr = NULL;
+
+		/* Check P2P_ATTR_OPERATING_CH */
+		attr_contentlen = 0;
+		pattr = NULL;
+		pattr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, (uint *)&attr_contentlen);
+		if (pattr != NULL)
+			*(pattr + 4) = ch;
+
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+	}
+}
+#endif
+
+#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+static void rtw_change_p2pie_ch_list(_adapter *padapter, const u8 *frame_body, u32 len, u8 ch)
+{
+	u8 *ies, *p2p_ie;
+	u32 ies_len, p2p_ielen;
+
+#ifdef CONFIG_MCC_MODE
+	if (GET_PHL_COM(adapter_to_dvobj(padapter))->dev_cap.mcc_sup == true)
+		return;
+#endif /* CONFIG_MCC_MODE */
+
+	ies = (u8 *)(frame_body + _PUBLIC_ACTION_IE_OFFSET_);
+	ies_len = len - _PUBLIC_ACTION_IE_OFFSET_;
+
+	p2p_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &p2p_ielen);
+
+	while (p2p_ie) {
+		u32	attr_contentlen = 0;
+		u8 *pattr = NULL;
+
+		/* Check P2P_ATTR_CH_LIST */
+		pattr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_CH_LIST, NULL, (uint *)&attr_contentlen);
+		if (pattr != NULL) {
+			int i;
+			u32 num_of_ch;
+			u8 *pattr_temp = pattr + 3 ;
+
+			attr_contentlen -= 3;
+
+			while (attr_contentlen > 0) {
+				num_of_ch = *(pattr_temp + 1);
+
+				for (i = 0; i < num_of_ch; i++)
+					*(pattr_temp + 2 + i) = ch;
+
+				pattr_temp += (2 + num_of_ch);
+				attr_contentlen -= (2 + num_of_ch);
+			}
+		}
+
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+	}
+}
+#endif
+
+#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+static bool rtw_chk_p2pie_ch_list_with_buddy(_adapter *padapter, const u8 *frame_body, u32 len)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	bool fit = _FALSE;
+	u8 *ies, *p2p_ie;
+	u32 ies_len, p2p_ielen;
+	struct rtw_chan_def u_chdef = {0};
+	u8 union_ch = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+#ifdef CONFIG_MCC_MODE
+	if (GET_PHL_COM(adapter_to_dvobj(padapter))->dev_cap.mcc_sup == true)
+		return fit;
+#endif /* CONFIG_MCC_MODE */
+
+	if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role,
+				padapter_link->wrlink, &u_chdef)
+			!= RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s get union chandef failed\n", __func__);
+		rtw_warn_on(1);
+	}
+	union_ch = u_chdef.chan;
+
+	ies = (u8 *)(frame_body + _PUBLIC_ACTION_IE_OFFSET_);
+	ies_len = len - _PUBLIC_ACTION_IE_OFFSET_;
+
+	p2p_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &p2p_ielen);
+
+	while (p2p_ie) {
+		u32	attr_contentlen = 0;
+		u8 *pattr = NULL;
+
+		/* Check P2P_ATTR_CH_LIST */
+		pattr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_CH_LIST, NULL, (uint *)&attr_contentlen);
+		if (pattr != NULL) {
+			int i;
+			u32 num_of_ch;
+			u8 *pattr_temp = pattr + 3 ;
+
+			attr_contentlen -= 3;
+
+			while (attr_contentlen > 0) {
+				num_of_ch = *(pattr_temp + 1);
+
+				for (i = 0; i < num_of_ch; i++) {
+					if (*(pattr_temp + 2 + i) == union_ch) {
+						RTW_INFO(FUNC_ADPT_FMT" ch_list fit buddy_ch:%u\n", FUNC_ADPT_ARG(padapter), union_ch);
+						fit = _TRUE;
+						break;
+					}
+				}
+
+				pattr_temp += (2 + num_of_ch);
+				attr_contentlen -= (2 + num_of_ch);
+			}
+		}
+
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+	}
+
+	return fit;
+}
+
+#if defined(CONFIG_P2P_INVITE_IOT)
+static bool rtw_chk_p2pie_op_ch_with_buddy(_adapter *padapter, const u8 *frame_body, u32 len)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	bool fit = _FALSE;
+	u8 *ies, *p2p_ie;
+	u32 ies_len, p2p_ielen;
+	struct rtw_chan_def u_chdef = {0};
+	u8 union_ch = 0;
+
+#ifdef CONFIG_MCC_MODE
+	if (GET_PHL_COM(adapter_to_dvobj(padapter))->dev_cap.mcc_sup == true)
+		return;
+#endif /* CONFIG_MCC_MODE */
+
+	if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role, &u_chdef)
+							!= RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s get union chandef failed\n", __func__);
+		rtw_warn_on(1);
+	}
+	union_ch = u_chdef.chan;
+	ies = (u8 *)(frame_body + _PUBLIC_ACTION_IE_OFFSET_);
+	ies_len = len - _PUBLIC_ACTION_IE_OFFSET_;
+
+	p2p_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &p2p_ielen);
+
+	while (p2p_ie) {
+		u32	attr_contentlen = 0;
+		u8 *pattr = NULL;
+
+		/* Check P2P_ATTR_OPERATING_CH */
+		attr_contentlen = 0;
+		pattr = NULL;
+		pattr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, (uint *)&attr_contentlen);
+		if (pattr != NULL) {
+			if (*(pattr + 4) == union_ch) {
+				RTW_INFO(FUNC_ADPT_FMT" op_ch fit buddy_ch:%u\n", FUNC_ADPT_ARG(padapter), union_ch);
+				fit = _TRUE;
+				break;
+			}
+		}
+
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+	}
+
+	return fit;
+}
+#endif
+
+static void rtw_cfg80211_adjust_p2pie_channel(_adapter *padapter, const u8 *frame_body, u32 len)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u8 *ies, *p2p_ie;
+	u32 ies_len, p2p_ielen;
+	struct rtw_chan_def u_chdef = {0};
+	u8 union_ch = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+#ifdef CONFIG_MCC_MODE
+	if (GET_PHL_COM(adapter_to_dvobj(padapter))->dev_cap.mcc_sup == true)
+		return;
+#endif /* CONFIG_MCC_MODE */
+
+	if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role,
+					padapter_link->wrlink, &u_chdef)
+						!= RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s get union chandef failed\n", __func__);
+		rtw_warn_on(1);
+	}
+	union_ch = u_chdef.chan;
+
+	ies = (u8 *)(frame_body + _PUBLIC_ACTION_IE_OFFSET_);
+	ies_len = len - _PUBLIC_ACTION_IE_OFFSET_;
+
+	p2p_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &p2p_ielen);
+
+	while (p2p_ie) {
+		u32	attr_contentlen = 0;
+		u8 *pattr = NULL;
+
+		/* Check P2P_ATTR_CH_LIST */
+		pattr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_CH_LIST, NULL, (uint *)&attr_contentlen);
+		if (pattr != NULL) {
+			int i;
+			u32 num_of_ch;
+			u8 *pattr_temp = pattr + 3 ;
+
+			attr_contentlen -= 3;
+
+			while (attr_contentlen > 0) {
+				num_of_ch = *(pattr_temp + 1);
+
+				for (i = 0; i < num_of_ch; i++) {
+					if (*(pattr_temp + 2 + i) && *(pattr_temp + 2 + i) != union_ch) {
+						RTW_ERR("replace ch_list:%u with:%u\n", *(pattr_temp + 2 + i), union_ch);
+						*(pattr_temp + 2 + i) = union_ch; /*forcing to the same channel*/
+					}
+				}
+
+				pattr_temp += (2 + num_of_ch);
+				attr_contentlen -= (2 + num_of_ch);
+			}
+		}
+
+		/* Check P2P_ATTR_OPERATING_CH */
+		attr_contentlen = 0;
+		pattr = NULL;
+		pattr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, (uint *)&attr_contentlen);
+		if (pattr != NULL) {
+			if (*(pattr + 4) && *(pattr + 4) != union_ch) {
+				RTW_ERR("replace op_ch:%u with:%u\n", *(pattr + 4), union_ch);
+				*(pattr + 4) = union_ch; /*forcing to the same channel	*/
+			}
+		}
+
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+
+	}
+
+}
+#endif
+
+#ifdef CONFIG_WFD
+u32 rtw_xframe_build_wfd_ie(struct xmit_frame *xframe)
+{
+	_adapter *adapter = xframe->padapter;
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	u8 *frame = xframe->buf_addr + TXDESC_OFFSET;
+	u8 *frame_body = frame + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *frame_tail = frame + xframe->attrib.pktlen;
+	u8 category, action, OUI_Subtype, dialogToken = 0;
+	u32	wfdielen = 0;
+
+	category = frame_body[0];
+	if (category == RTW_WLAN_CATEGORY_PUBLIC) {
+		action = frame_body[1];
+		if (action == ACT_PUBLIC_VENDOR
+		    && _rtw_memcmp(frame_body + 2, P2P_OUI, 4) == _TRUE
+		   ) {
+			OUI_Subtype = frame_body[6];
+			dialogToken = frame_body[7];
+
+			switch (OUI_Subtype) {
+			case P2P_GO_NEGO_REQ:
+				wfdielen = build_nego_req_wfd_ie(wdinfo, frame_tail);
+				break;
+			case P2P_GO_NEGO_RESP:
+				wfdielen = build_nego_resp_wfd_ie(wdinfo, frame_tail);
+				break;
+			case P2P_GO_NEGO_CONF:
+				wfdielen = build_nego_confirm_wfd_ie(wdinfo, frame_tail);
+				break;
+			case P2P_INVIT_REQ:
+				wfdielen = build_invitation_req_wfd_ie(wdinfo, frame_tail);
+				break;
+			case P2P_INVIT_RESP:
+				wfdielen = build_invitation_resp_wfd_ie(wdinfo, frame_tail);
+				break;
+			case P2P_PROVISION_DISC_REQ:
+				wfdielen = build_provdisc_req_wfd_ie(wdinfo, frame_tail);
+				break;
+			case P2P_PROVISION_DISC_RESP:
+				wfdielen = build_provdisc_resp_wfd_ie(wdinfo, frame_tail);
+				break;
+			case P2P_DEVDISC_REQ:
+			case P2P_DEVDISC_RESP:
+			default:
+				break;
+			}
+
+		}
+	} else if (category == RTW_WLAN_CATEGORY_P2P) {
+		OUI_Subtype = frame_body[5];
+		dialogToken = frame_body[6];
+
+#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("ACTION_CATEGORY_P2P: OUI=0x%x, OUI_Subtype=%d, dialogToken=%d\n"
+			, cpu_to_be32(*((u32 *)(frame_body + 1))), OUI_Subtype, dialogToken);
+#endif
+
+		switch (OUI_Subtype) {
+		case P2P_NOTICE_OF_ABSENCE:
+			break;
+		case P2P_PRESENCE_REQUEST:
+			break;
+		case P2P_PRESENCE_RESPONSE:
+			break;
+		case P2P_GO_DISC_REQUEST:
+			break;
+		default:
+			break;
+		}
+	} else
+		RTW_INFO("%s, action frame category=%d\n", __func__, category);
+
+	xframe->attrib.pktlen += wfdielen;
+
+	return wfdielen;
+}
+#endif /* CONFIG_WFD */
+
+bool rtw_xframe_del_wfd_ie(struct xmit_frame *xframe)
+{
+#define DBG_XFRAME_DEL_WFD_IE 0
+	u8 *frame = xframe->buf_addr + TXDESC_OFFSET;
+	u8 *frame_body = frame + sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *frame_tail = frame + xframe->attrib.pktlen;
+	u8 category, action, OUI_Subtype;
+	u8 *ies = NULL;
+	uint ies_len_ori = 0;
+	uint ies_len = 0;
+
+	category = frame_body[0];
+	if (category == RTW_WLAN_CATEGORY_PUBLIC) {
+		action = frame_body[1];
+		if (action == ACT_PUBLIC_VENDOR
+		    && _rtw_memcmp(frame_body + 2, P2P_OUI, 4) == _TRUE
+		   ) {
+			OUI_Subtype = frame_body[6];
+
+			switch (OUI_Subtype) {
+			case P2P_GO_NEGO_REQ:
+			case P2P_GO_NEGO_RESP:
+			case P2P_GO_NEGO_CONF:
+			case P2P_INVIT_REQ:
+			case P2P_INVIT_RESP:
+			case P2P_PROVISION_DISC_REQ:
+			case P2P_PROVISION_DISC_RESP:
+				ies = frame_body + 8;
+				ies_len_ori = frame_tail - (frame_body + 8);
+				break;
+			}
+		}
+	}
+
+	if (ies && ies_len_ori) {
+		ies_len = rtw_del_wfd_ie(ies, ies_len_ori, DBG_XFRAME_DEL_WFD_IE ? __func__ : NULL);
+		xframe->attrib.pktlen -= (ies_len_ori - ies_len);
+	}
+
+	return ies_len_ori != ies_len;
+}
+
+/*
+* rtw_xframe_chk_wfd_ie -
+*
+*/
+void rtw_xframe_chk_wfd_ie(struct xmit_frame *xframe)
+{
+	_adapter *adapter = xframe->padapter;
+#ifdef CONFIG_IOCTL_CFG80211
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+#endif
+	u8 build = 0;
+	u8 del = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		del = 1;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (wdinfo->wfd_info->wfd_enable == _TRUE)
+#endif
+		del = build = 1;
+
+	if (del)
+		rtw_xframe_del_wfd_ie(xframe);
+
+#ifdef CONFIG_WFD
+	if (build)
+		rtw_xframe_build_wfd_ie(xframe);
+#endif
+}
+
+u8 *dump_p2p_attr_ch_list(u8 *p2p_ie, uint p2p_ielen, u8 *buf, u32 buf_len)
+{
+	uint attr_contentlen = 0;
+	u8 *pattr = NULL;
+	int w_sz = 0;
+	u8 ch_cnt = 0;
+	u8 ch_list[MAX_CHANNEL_NUM];
+
+	pattr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_CH_LIST, NULL, &attr_contentlen);
+	if (pattr != NULL) {
+		int i, j;
+		u32 num_of_ch;
+		u8 *pattr_temp = pattr + 3 ;
+
+		attr_contentlen -= 3;
+
+		_rtw_memset(ch_list, 0, 40);
+
+		while (attr_contentlen > 0) {
+			num_of_ch = *(pattr_temp + 1);
+
+			for (i = 0; i < num_of_ch; i++) {
+				for (j = 0; j < ch_cnt; j++) {
+					if (ch_list[j] == *(pattr_temp + 2 + i))
+						break;
+				}
+				if (j >= ch_cnt)
+					ch_list[ch_cnt++] = *(pattr_temp + 2 + i);
+				if (ch_cnt == MAX_CHANNEL_NUM) {
+					RTW_INFO("channel list array is used up, may need to increase array size\n");
+					goto make_str;
+				}
+			}
+			pattr_temp += (2 + num_of_ch);
+			attr_contentlen -= (2 + num_of_ch);
+		}
+make_str:
+		for (j = 0; j < ch_cnt; j++) {
+			if (j == 0)
+				w_sz += snprintf(buf + w_sz, buf_len - w_sz, "%u", ch_list[j]);
+			else if (ch_list[j] - ch_list[j - 1] != 1)
+				w_sz += snprintf(buf + w_sz, buf_len - w_sz, ", %u", ch_list[j]);
+			else if (j != ch_cnt - 1 && ch_list[j + 1] - ch_list[j] == 1) {
+				/* empty */
+			} else
+				w_sz += snprintf(buf + w_sz, buf_len - w_sz, "-%u", ch_list[j]);
+		}
+	}
+	return buf;
+}
+
+/*
+ * return _TRUE if requester is GO, _FALSE if responder is GO
+ */
+bool rtw_p2p_nego_intent_compare(u8 req, u8 resp)
+{
+	if (req >> 1 == resp >> 1)
+		return  req & 0x01 ? _TRUE : _FALSE;
+	else if (req >> 1 > resp >> 1)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+int rtw_p2p_check_frames(_adapter *padapter, const u8 *buf, u32 len, u8 tx)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	int is_p2p_frame = (-1);
+	unsigned char	*frame_body;
+	u8 category, action, OUI_Subtype, dialogToken = 0;
+	u8 *p2p_ie = NULL;
+	uint p2p_ielen = 0;
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	int status = -1;
+	u8 ch_list_buf[128] = {'\0'};
+	int op_ch = -1;
+	int listen_ch = -1;
+	u8 intent = 0;
+	u8 *iaddr = NULL;
+	u8 *gbssid = NULL;
+
+	frame_body = (unsigned char *)(buf + sizeof(struct rtw_ieee80211_hdr_3addr));
+	category = frame_body[0];
+	/* just for check */
+	if (category == RTW_WLAN_CATEGORY_PUBLIC) {
+		action = frame_body[1];
+		if (action == ACT_PUBLIC_VENDOR
+			&& _rtw_memcmp(frame_body + 2, P2P_OUI, 4) == _TRUE
+		) {
+			OUI_Subtype = frame_body[6];
+			dialogToken = frame_body[7];
+			is_p2p_frame = OUI_Subtype;
+
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("ACTION_CATEGORY_PUBLIC: ACT_PUBLIC_VENDOR, OUI=0x%x, OUI_Subtype=%d, dialogToken=%d\n",
+				cpu_to_be32(*((u32 *)(frame_body + 2))), OUI_Subtype, dialogToken);
+			#endif
+
+			p2p_ie = rtw_get_p2p_ie(
+				(u8 *)buf + sizeof(struct rtw_ieee80211_hdr_3addr) + _PUBLIC_ACTION_IE_OFFSET_
+				, len - sizeof(struct rtw_ieee80211_hdr_3addr) - _PUBLIC_ACTION_IE_OFFSET_
+				, NULL, &p2p_ielen);
+
+			switch (OUI_Subtype) { /* OUI Subtype */
+				u8 *cont;
+				uint cont_len;
+			case P2P_GO_NEGO_REQ: {
+				struct rtw_wdev_nego_info *nego_info = &pwdev_priv->nego_info;
+
+				if (tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+					if (rtw_mi_check_status(padapter, MI_LINKED) && padapter->registrypriv.full_ch_in_p2p_handshake == 0)
+						rtw_cfg80211_adjust_p2pie_channel(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr));
+					#endif
+				}
+
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, &cont_len);
+				if (cont)
+					op_ch = *(cont + 4);
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_LISTEN_CH, NULL, &cont_len);
+				if (cont)
+					listen_ch = *(cont + 4);
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_GO_INTENT, NULL, &cont_len);
+				if (cont)
+					intent = *cont;
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_INTENDED_IF_ADDR, NULL, &cont_len);
+				if (cont && cont_len == 6)
+					iaddr = cont;
+
+				if (nego_info->token != dialogToken)
+					rtw_wdev_nego_info_init(nego_info);
+
+				_rtw_memcpy(nego_info->peer_mac, tx ? GetAddr1Ptr(buf) : get_addr2_ptr(buf), ETH_ALEN);
+				if (iaddr)
+					_rtw_memcpy(tx ? nego_info->iface_addr : nego_info->peer_iface_addr, iaddr, ETH_ALEN);
+				nego_info->active = tx ? 1 : 0;
+				nego_info->token = dialogToken;
+				nego_info->req_op_ch = op_ch;
+				nego_info->req_listen_ch = listen_ch;
+				nego_info->req_intent = intent;
+				nego_info->state = 0;
+
+				dump_p2p_attr_ch_list(p2p_ie, p2p_ielen, ch_list_buf, 128);
+				RTW_INFO("RTW_%s:P2P_GO_NEGO_REQ, dialogToken=%d, intent:%u%s, listen_ch:%d, op_ch:%d, ch_list:%s"
+					, (tx == _TRUE) ? "Tx" : "Rx" , dialogToken , (intent >> 1) , intent & 0x1 ? "+" : "-" , listen_ch , op_ch , ch_list_buf);
+				if (iaddr)
+					_RTW_INFO(", iaddr:"MAC_FMT, MAC_ARG(iaddr));
+				_RTW_INFO("\n");
+
+				if (!tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+					{
+						if (rtw_mi_check_status(padapter, MI_LINKED)
+						    && rtw_chk_p2pie_ch_list_with_buddy(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr)) == _FALSE
+						    && padapter->registrypriv.full_ch_in_p2p_handshake == 0) {
+							RTW_INFO(FUNC_ADPT_FMT" ch_list has no intersect with buddy\n", FUNC_ADPT_ARG(padapter));
+							rtw_change_p2pie_ch_list(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr), 0);
+						}
+					}
+					#endif
+				}
+
+				break;
+			}
+			case P2P_GO_NEGO_RESP: {
+				struct rtw_wdev_nego_info *nego_info = &pwdev_priv->nego_info;
+
+				if (tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+						if (rtw_mi_check_status(padapter, MI_LINKED) && padapter->registrypriv.full_ch_in_p2p_handshake == 0)
+							rtw_cfg80211_adjust_p2pie_channel(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr));
+					#endif
+				}
+
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, &cont_len);
+				if (cont)
+					op_ch = *(cont + 4);
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_GO_INTENT, NULL, &cont_len);
+				if (cont)
+					intent = *cont;
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_STATUS, NULL, &cont_len);
+				if (cont)
+					status = *cont;
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_INTENDED_IF_ADDR, NULL, &cont_len);
+				if (cont && cont_len == 6)
+					iaddr = cont;
+
+				if (nego_info->token == dialogToken && nego_info->state == 0
+					&& _rtw_memcmp(nego_info->peer_mac, tx ? GetAddr1Ptr(buf) : get_addr2_ptr(buf), ETH_ALEN) == _TRUE
+				) {
+					if (iaddr)
+						_rtw_memcpy(tx ? nego_info->iface_addr : nego_info->peer_iface_addr, iaddr, ETH_ALEN);
+					nego_info->status = (status == -1) ? 0xff : status;
+					nego_info->rsp_op_ch = op_ch;
+					nego_info->rsp_intent = intent;
+					nego_info->state = 1;
+					if (status != 0)
+						nego_info->token = 0; /* init */
+				}
+
+				dump_p2p_attr_ch_list(p2p_ie, p2p_ielen, ch_list_buf, 128);
+				RTW_INFO("RTW_%s:P2P_GO_NEGO_RESP, dialogToken=%d, intent:%u%s, status:%d, op_ch:%d, ch_list:%s"
+					, (tx == _TRUE) ? "Tx" : "Rx", dialogToken, (intent >> 1), intent & 0x1 ? "+" : "-", status, op_ch, ch_list_buf);
+				if (iaddr)
+					_RTW_INFO(", iaddr:"MAC_FMT, MAC_ARG(iaddr));
+				_RTW_INFO("\n");
+
+				if (!tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+					{
+						if (rtw_mi_check_status(padapter, MI_LINKED)
+						    && rtw_chk_p2pie_ch_list_with_buddy(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr)) == _FALSE
+						    && padapter->registrypriv.full_ch_in_p2p_handshake == 0) {
+							RTW_INFO(FUNC_ADPT_FMT" ch_list has no intersect with buddy\n", FUNC_ADPT_ARG(padapter));
+							rtw_change_p2pie_ch_list(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr), 0);
+						}
+					}
+					#endif
+				}
+
+				break;
+			}
+			case P2P_GO_NEGO_CONF: {
+				struct rtw_wdev_nego_info *nego_info = &pwdev_priv->nego_info;
+				bool is_go = _FALSE;
+
+				if (tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+						if (rtw_mi_check_status(padapter, MI_LINKED) && padapter->registrypriv.full_ch_in_p2p_handshake == 0)
+							rtw_cfg80211_adjust_p2pie_channel(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr));
+					#endif
+				}
+
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, &cont_len);
+				if (cont)
+					op_ch = *(cont + 4);
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_STATUS, NULL, &cont_len);
+				if (cont)
+					status = *cont;
+
+				if (nego_info->token == dialogToken && nego_info->state == 1
+				    && _rtw_memcmp(nego_info->peer_mac, tx ? GetAddr1Ptr(buf) : get_addr2_ptr(buf), ETH_ALEN) == _TRUE
+				   ) {
+					nego_info->status = (status == -1) ? 0xff : status;
+					nego_info->conf_op_ch = (op_ch == -1) ? 0 : op_ch;
+					nego_info->state = 2;
+
+					if (status == 0) {
+						if (rtw_p2p_nego_intent_compare(nego_info->req_intent, nego_info->rsp_intent) ^ !tx)
+							is_go = _TRUE;
+					}
+
+					nego_info->token = 0; /* init */
+				}
+
+				dump_p2p_attr_ch_list(p2p_ie, p2p_ielen, ch_list_buf, 128);
+				RTW_INFO("RTW_%s:P2P_GO_NEGO_CONF, dialogToken=%d, status:%d, op_ch:%d, ch_list:%s\n"
+					, (tx == _TRUE) ? "Tx" : "Rx", dialogToken, status, op_ch, ch_list_buf);
+
+				if (!tx) {
+				}
+
+				break;
+			}
+			case P2P_INVIT_REQ: {
+				struct rtw_wdev_invit_info *invit_info = &pwdev_priv->invit_info;
+				int flags = -1;
+
+				if (tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+						if (rtw_mi_check_status(padapter, MI_LINKED)
+						    && padapter->registrypriv.full_ch_in_p2p_handshake == 0)
+							rtw_cfg80211_adjust_p2pie_channel(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr));
+					#endif
+				}
+
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_INVITATION_FLAGS, NULL, &cont_len);
+				if (cont)
+					flags = *cont;
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, &cont_len);
+				if (cont)
+					op_ch = *(cont + 4);
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_GROUP_BSSID, NULL, &cont_len);
+				if (cont && cont_len == 6)
+					gbssid = cont;
+
+				if (invit_info->token != dialogToken)
+					rtw_wdev_invit_info_init(invit_info);
+
+				_rtw_memcpy(invit_info->peer_mac, tx ? GetAddr1Ptr(buf) : get_addr2_ptr(buf), ETH_ALEN);
+				if (gbssid)
+					_rtw_memcpy(invit_info->group_bssid, gbssid, ETH_ALEN);
+				invit_info->active = tx ? 1 : 0;
+				invit_info->token = dialogToken;
+				invit_info->flags = (flags == -1) ? 0x0 : flags;
+				invit_info->req_op_ch = op_ch;
+				invit_info->state = 0;
+
+				dump_p2p_attr_ch_list(p2p_ie, p2p_ielen, ch_list_buf, 128);
+				RTW_INFO("RTW_%s:P2P_INVIT_REQ, dialogToken=%d, flags:0x%02x, op_ch:%d, ch_list:%s"
+					, (tx == _TRUE) ? "Tx" : "Rx", dialogToken, flags, op_ch, ch_list_buf);
+				if (gbssid)
+					_RTW_INFO(", gbssid:"MAC_FMT, MAC_ARG(gbssid));
+				_RTW_INFO("\n");
+
+				if (!tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+					{
+						if (rtw_mi_check_status(padapter, MI_LINKED) && padapter->registrypriv.full_ch_in_p2p_handshake == 0) {
+							#if defined(CONFIG_P2P_INVITE_IOT)
+							if (op_ch != -1 && rtw_chk_p2pie_op_ch_with_buddy(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr)) == _FALSE) {
+								RTW_INFO(FUNC_ADPT_FMT" op_ch:%u has no intersect with buddy\n", FUNC_ADPT_ARG(padapter), op_ch);
+								rtw_change_p2pie_ch_list(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr), 0);
+							} else
+							#endif
+							if (rtw_chk_p2pie_ch_list_with_buddy(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr)) == _FALSE) {
+								RTW_INFO(FUNC_ADPT_FMT" ch_list has no intersect with buddy\n", FUNC_ADPT_ARG(padapter));
+								rtw_change_p2pie_ch_list(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr), 0);
+							}
+						}
+					}
+					#endif
+				}
+
+				break;
+			}
+			case P2P_INVIT_RESP: {
+				struct rtw_wdev_invit_info *invit_info = &pwdev_priv->invit_info;
+
+				if (tx) {
+					#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT)
+					#ifdef CONFIG_DBCC_SUPPORT
+					if (GET_PHL_COM(dvobj)->dev_cap.dbcc_sup == false)
+					#endif /*CONFIG_DBCC_SUPPORT*/
+						if (rtw_mi_check_status(padapter, MI_LINKED) && padapter->registrypriv.full_ch_in_p2p_handshake == 0)
+							rtw_cfg80211_adjust_p2pie_channel(padapter, frame_body, len - sizeof(struct rtw_ieee80211_hdr_3addr));
+					#endif
+				}
+
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_STATUS, NULL, &cont_len);
+				if (cont) {
+					#ifdef CONFIG_P2P_INVITE_IOT
+					if (tx && *cont == 7) {
+						RTW_INFO("TX_P2P_INVITE_RESP, status is no common channel, change to unknown group\n");
+						*cont = 8; /* unknow group status */
+					}
+					#endif /* CONFIG_P2P_INVITE_IOT */
+					status = *cont;
+				}
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_OPERATING_CH, NULL, &cont_len);
+				if (cont)
+					op_ch = *(cont + 4);
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_GROUP_BSSID, NULL, &cont_len);
+				if (cont && cont_len == 6)
+					gbssid = cont;
+
+				if (invit_info->token == dialogToken && invit_info->state == 0
+				    && _rtw_memcmp(invit_info->peer_mac, tx ? GetAddr1Ptr(buf) : get_addr2_ptr(buf), ETH_ALEN) == _TRUE
+				   ) {
+					invit_info->status = (status == -1) ? 0xff : status;
+					invit_info->rsp_op_ch = op_ch;
+					invit_info->state = 1;
+					invit_info->token = 0; /* init */
+				}
+
+				dump_p2p_attr_ch_list(p2p_ie, p2p_ielen, ch_list_buf, 128);
+				RTW_INFO("RTW_%s:P2P_INVIT_RESP, dialogToken=%d, status:%d, op_ch:%d, ch_list:%s"
+					, (tx == _TRUE) ? "Tx" : "Rx", dialogToken, status, op_ch, ch_list_buf);
+				if (gbssid)
+					_RTW_INFO(", gbssid:"MAC_FMT, MAC_ARG(gbssid));
+				_RTW_INFO("\n");
+
+				if (!tx) {
+				}
+
+				break;
+			}
+			case P2P_DEVDISC_REQ:
+				RTW_INFO("RTW_%s:P2P_DEVDISC_REQ, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken);
+				break;
+			case P2P_DEVDISC_RESP:
+				cont = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_STATUS, NULL, &cont_len);
+				RTW_INFO("RTW_%s:P2P_DEVDISC_RESP, dialogToken=%d, status:%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken, cont ? *cont : -1);
+				break;
+			case P2P_PROVISION_DISC_REQ:
+				RTW_INFO("RTW_%s:P2P_PROVISION_DISC_REQ, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken);
+				break;
+			case P2P_PROVISION_DISC_RESP:
+				RTW_INFO("RTW_%s:P2P_PROVISION_DISC_RESP, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken);
+				break;
+			default:
+				RTW_INFO("RTW_%s:OUI_Subtype=%d, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", OUI_Subtype, dialogToken);
+				break;
+			}
+
+		}
+
+	} else if (category == RTW_WLAN_CATEGORY_P2P) {
+		OUI_Subtype = frame_body[5];
+		dialogToken = frame_body[6];
+
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("ACTION_CATEGORY_P2P: OUI=0x%x, OUI_Subtype=%d, dialogToken=%d\n",
+			cpu_to_be32(*((u32 *)(frame_body + 1))), OUI_Subtype, dialogToken);
+		#endif
+
+		is_p2p_frame = OUI_Subtype;
+
+		switch (OUI_Subtype) {
+		case P2P_NOTICE_OF_ABSENCE:
+			RTW_INFO("RTW_%s:P2P_NOTICE_OF_ABSENCE, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken);
+			break;
+		case P2P_PRESENCE_REQUEST:
+			RTW_INFO("RTW_%s:P2P_PRESENCE_REQUEST, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken);
+			break;
+		case P2P_PRESENCE_RESPONSE:
+			RTW_INFO("RTW_%s:P2P_PRESENCE_RESPONSE, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken);
+			break;
+		case P2P_GO_DISC_REQUEST:
+			RTW_INFO("RTW_%s:P2P_GO_DISC_REQUEST, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", dialogToken);
+			break;
+		default:
+			RTW_INFO("RTW_%s:OUI_Subtype=%d, dialogToken=%d\n", (tx == _TRUE) ? "Tx" : "Rx", OUI_Subtype, dialogToken);
+			break;
+		}
+
+	}
+
+	return is_p2p_frame;
+}
+#endif /* CONFIG_IOCTL_CFG80211	 */
+
+int process_p2p_cross_connect_ie(_adapter *padapter, u8 *IEs, u32 IELength)
+{
+	int ret = _TRUE;
+	u8 *ies;
+	u32 ies_len;
+	u8 *p2p_ie;
+	u32	p2p_ielen = 0;
+	u8	p2p_attr[MAX_P2P_IE_LEN] = { 0x00 };/* NoA length should be n*(13) + 2 */
+	u32	attr_contentlen = 0;
+
+
+
+	if (IELength <= _BEACON_IE_OFFSET_)
+		return ret;
+
+	ies = IEs + _BEACON_IE_OFFSET_;
+	ies_len = IELength - _BEACON_IE_OFFSET_;
+
+	p2p_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &p2p_ielen);
+
+	while (p2p_ie) {
+		/* Get P2P Manageability IE. */
+		attr_contentlen = sizeof(p2p_attr);
+		if (rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_MANAGEABILITY, p2p_attr, &attr_contentlen)) {
+			if ((p2p_attr[0] & (BIT(0) | BIT(1))) == 0x01)
+				ret = _FALSE;
+			break;
+		}
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+	}
+
+	return ret;
+}
+
+#ifdef CONFIG_P2P_PS
+
+static void _build_p2p_go_noa_ie(struct wifidirect_info *wdinfo,
+				 u8 *ie, u16 *ie_len)
+{
+	u8 *p2p_noa_attr_ie;
+	u8 p2p_noa_attr_pos = 0;
+	u8 noa_desc_num = 1;
+	u8 opp_ps = 0; /* Disable OppPS */
+	u8 noa_count = 0;
+	u32 noa_duration = 0;
+	u32 noa_interval = 0;
+	u32 noa_start_time = 0;
+	u8 noa_index = 0;
+	/* support 1 NoA desc */
+	u8 num_noa_desc = 1;
+	u8 noa_attr_len = 0;
+	u16 noa_ie_len = 0;
+
+	noa_index = wdinfo->noa_index;
+	noa_count = wdinfo->noa_count[0];
+	noa_duration = wdinfo->noa_duration[0];
+	noa_interval = wdinfo->noa_interval[0];
+	noa_start_time = wdinfo->noa_start_time[0];
+	/* Noa Attrute Body Len */
+	noa_attr_len = num_noa_desc * 13 + 2;
+	/* P2P NoA IE Len(included P2P_OUI, attrute ID, attrute Len) */
+	noa_ie_len = noa_attr_len + 4 + 1 + 2;
+	/* Totol IE Len(included Element ID & NoA IE Len) */
+	*ie_len = noa_ie_len + 2;
+	if (*ie_len > MAX_P2P_IE_LEN) {
+		RTW_ERR("%s: ie len = %d\n", __func__, *ie_len);
+		*ie_len = 0;
+		return;
+	}
+
+	/* Element ID:0xDD  */
+	*ie = _VENDOR_SPECIFIC_IE_;
+	/* NoA IE Len */
+	*(ie + 1) = noa_ie_len;
+	p2p_noa_attr_ie = (ie + 2);
+
+	/* P2P OUI(4 bytes) */
+	_rtw_memcpy(p2p_noa_attr_ie, P2P_OUI, 4);
+	p2p_noa_attr_pos = p2p_noa_attr_pos + 4;
+
+	/* attrute ID(1 byte) */
+	p2p_noa_attr_ie[p2p_noa_attr_pos] = P2P_ATTR_NOA;
+	p2p_noa_attr_pos = p2p_noa_attr_pos + 1;
+
+	/* attrute length(2 bytes) length = noa_desc_num*13 + 2 */
+	RTW_PUT_LE16(p2p_noa_attr_ie + p2p_noa_attr_pos,
+		     (noa_desc_num * 13 + 2));
+	p2p_noa_attr_pos = p2p_noa_attr_pos + 2;
+
+	/* Index (1 byte) */
+	p2p_noa_attr_ie[p2p_noa_attr_pos] = noa_index;
+	p2p_noa_attr_pos = p2p_noa_attr_pos + 1;
+
+	/* CTWindow and OppPS Parameters (1 byte) */
+	p2p_noa_attr_ie[p2p_noa_attr_pos] = opp_ps;
+	p2p_noa_attr_pos = p2p_noa_attr_pos+ 1;
+
+	/* NoA Count (1 byte) */
+	p2p_noa_attr_ie[p2p_noa_attr_pos] = noa_count;
+	p2p_noa_attr_pos = p2p_noa_attr_pos + 1;
+
+	/* NoA Duration (4 bytes) unit: microseconds */
+	RTW_PUT_LE32(p2p_noa_attr_ie + p2p_noa_attr_pos, noa_duration);
+	p2p_noa_attr_pos = p2p_noa_attr_pos + 4;
+
+	/* NoA Interval (4 bytes) unit: microseconds */
+	RTW_PUT_LE32(p2p_noa_attr_ie + p2p_noa_attr_pos, noa_interval);
+	p2p_noa_attr_pos = p2p_noa_attr_pos + 4;
+
+	/* NoA Start Time (4 bytes) unit: microseconds */
+	RTW_PUT_LE32(p2p_noa_attr_ie + p2p_noa_attr_pos, noa_start_time);
+}
+
+u8 *rtw_append_p2p_go_noa_ie(struct _ADAPTER *adapter, u8 *frame, u32 *len)
+{
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	u16 p2p_go_noa_ie_len = 0;
+
+	if (!MLME_IS_GO(adapter))
+		return frame;
+
+	if (wdinfo->noa_num == 0)
+		return frame;
+
+	_build_p2p_go_noa_ie(wdinfo, frame, &p2p_go_noa_ie_len);
+	*len = *len + p2p_go_noa_ie_len;
+
+	return frame + p2p_go_noa_ie_len;
+}
+
+static void _tsf32_tog_update_single_noa(void *priv,
+					 struct rtw_wifi_role_t *w_role,
+					 struct rtw_phl_noa_desc *desc)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *) priv;
+	struct _ADAPTER *a = NULL;
+	u8 id = 0;
+	u8 ret = _SUCCESS;
+	struct wifidirect_info *wdinfo;
+	struct _ADAPTER_LINK *alink;
+
+	if (w_role == NULL) {
+		RTW_ERR("%s wrole is NULL\n", __func__);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	id = w_role->id;
+	if (id >= CONFIG_IFACE_NUMBER) {
+		RTW_ERR("%s error id (%d)\n", __func__, id);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	a = dvobj->padapters[id];
+	if (a == NULL) {
+		RTW_ERR("%s adapter(%d) is NULL\n", __func__, id);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	/* by pass non-GO case */
+	if (!MLME_IS_GO(a))
+		goto exit;
+
+	RTW_INFO(FUNC_ADPT_FMT":(%d)\n", FUNC_ADPT_ARG(a), id);
+	RTW_INFO("start_t_h=0x%02x,start_t_l=0x%02x\n",
+		 desc->start_t_h, desc->start_t_l);
+	RTW_INFO("dur=0x%d,cnt=0x%d,interval=0x%d,noa id=%d\n",
+		 desc->duration, desc->count, desc->interval, desc->noa_id);
+
+	wdinfo = &a->wdinfo;
+	/* update NoA IE */
+	wdinfo->noa_index = wdinfo->noa_index + 1;
+	wdinfo->noa_num = 1;
+	wdinfo->noa_count[0] = desc->count;
+	wdinfo->noa_duration[0] =desc->duration;
+	wdinfo->noa_interval[0] = desc->interval;
+	wdinfo->noa_start_time[0] = desc->start_t_l;
+
+	alink = GET_PRIMARY_LINK(a);
+	rtw_update_beacon(a, alink, _VENDOR_SPECIFIC_IE_, P2P_OUI, _TRUE,
+			  RTW_CMDF_DIRECTLY);
+
+exit:
+	return;
+}
+
+struct rtw_phl_p2pps_ops rtw_p2pps_ops = {
+	.priv = NULL,
+	.tsf32_tog_update_noa = NULL,
+	.tsf32_tog_update_single_noa = &_tsf32_tog_update_single_noa,
+};
+
+void rtw_core_register_p2pps_ops(struct dvobj_priv *dvobj)
+{
+	struct rtw_phl_p2pps_ops ops = {0};
+	ops.priv = (void *)dvobj;
+	rtw_phl_p2pps_init_ops(dvobj->phl, &ops);
+}
+
+void process_p2p_ps_ie(_adapter *padapter, u8 *IEs, u32 IELength)
+{
+	u8 *ies;
+	u32 ies_len;
+	u8 *p2p_ie;
+	u32	p2p_ielen = 0;
+	u8 *noa_attr; /* NoA length should be n*(13) + 2 */
+	u32	attr_contentlen = 0;
+
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	u8	find_p2p = _FALSE, find_p2p_ps = _FALSE;
+	u8	noa_offset, noa_num, noa_index;
+
+
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE) ||
+	    rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DEVICE))
+		return;
+
+	if (IELength <= _BEACON_IE_OFFSET_)
+		return;
+
+	ies = IEs + _BEACON_IE_OFFSET_;
+	ies_len = IELength - _BEACON_IE_OFFSET_;
+
+	p2p_ie = rtw_get_p2p_ie(ies, ies_len, NULL, &p2p_ielen);
+
+	while (p2p_ie) {
+		find_p2p = _TRUE;
+		/* Get Notice of Absence IE. */
+		noa_attr = rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_NOA, NULL, &attr_contentlen);
+		if (noa_attr) {
+			find_p2p_ps = _TRUE;
+			noa_index = noa_attr[0];
+
+			if ((pwdinfo->p2p_ps_mode == P2P_PS_NONE) ||
+			    (noa_index != pwdinfo->noa_index)) { /* if index change, driver should reconfigure related setting. */
+				pwdinfo->noa_index = noa_index;
+				pwdinfo->opp_ps = noa_attr[1] >> 7;
+				if (pwdinfo->opp_ps != 1)
+					pwdinfo->ctwindow = 0;
+				else
+					pwdinfo->ctwindow = noa_attr[1] & 0x7F;
+				noa_offset = 2;
+				noa_num = 0;
+				/* NoA length should be n*(13) + 2 */
+				if (attr_contentlen > 2 && (attr_contentlen - 2) % 13 == 0) {
+					while (noa_offset < attr_contentlen && noa_num < P2P_MAX_NOA_NUM) {
+						/* _rtw_memcpy(&wifidirect_info->noa_count[noa_num], &noa_attr[noa_offset], 1); */
+						pwdinfo->noa_count[noa_num] = noa_attr[noa_offset];
+						noa_offset += 1;
+
+						_rtw_memcpy(&pwdinfo->noa_duration[noa_num], &noa_attr[noa_offset], 4);
+						noa_offset += 4;
+
+						_rtw_memcpy(&pwdinfo->noa_interval[noa_num], &noa_attr[noa_offset], 4);
+						noa_offset += 4;
+
+						_rtw_memcpy(&pwdinfo->noa_start_time[noa_num], &noa_attr[noa_offset], 4);
+						noa_offset += 4;
+
+						noa_num++;
+					}
+				}
+				pwdinfo->noa_num = noa_num;
+
+				if (pwdinfo->opp_ps == 1) {
+					pwdinfo->p2p_ps_mode = P2P_PS_CTWINDOW;
+					/* driver should wait LPS for entering CTWindow */
+					/* if (adapter_to_pwrctl(padapter)->bFwCurrentInPSMode == _TRUE)
+						p2p_ps_wk_cmd(padapter, P2P_PS_ENABLE, 1); */
+				} else if (pwdinfo->noa_num > 0) {
+					pwdinfo->p2p_ps_mode = P2P_PS_NOA;
+					p2p_ps_wk_cmd(padapter, P2P_PS_ENABLE, 1);
+				} else if (pwdinfo->p2p_ps_mode > P2P_PS_NONE)
+					p2p_ps_wk_cmd(padapter, P2P_PS_DISABLE, 1);
+			}
+
+			break; /* find target, just break. */
+		}
+
+		/* Get the next P2P IE */
+		p2p_ie = rtw_get_p2p_ie(p2p_ie + p2p_ielen, ies_len - (p2p_ie - ies + p2p_ielen), NULL, &p2p_ielen);
+
+	}
+
+	if (find_p2p == _TRUE) {
+		if ((pwdinfo->p2p_ps_mode > P2P_PS_NONE) && (find_p2p_ps == _FALSE))
+			p2p_ps_wk_cmd(padapter, P2P_PS_DISABLE, 1);
+	}
+
+}
+
+void p2p_ps_wk_hdl(_adapter *padapter, u8 p2p_ps_state)
+{
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	int i;
+
+	/* Pre action for p2p state */
+	switch (p2p_ps_state) {
+	case P2P_PS_DISABLE:
+		RTW_INFO("%s, P2P PS disabled\n", __func__);
+
+		rtw_phl_p2pps_noa_disable_all(GET_PHL_INFO(dvobj),
+					      padapter->phl_role);
+		pwdinfo->p2p_ps_mode = P2P_PS_NONE;
+		pwdinfo->p2p_ps_state = p2p_ps_state;
+		pwdinfo->noa_index = 0;
+		pwdinfo->ctwindow = 0;
+		pwdinfo->opp_ps = 0;
+		pwdinfo->noa_num = 0;
+		for (i = 0; i < P2P_MAX_NOA_NUM; i++) {
+			pwdinfo->noa_count[i] = 0;
+			pwdinfo->noa_duration[i] = 0;
+			pwdinfo->noa_interval[i] = 0;
+			pwdinfo->noa_start_time[i] = 0;
+		}
+
+		break;
+	case P2P_PS_ENABLE:
+		if (pwdinfo->p2p_ps_mode <= P2P_PS_NONE)
+			return;
+
+		RTW_INFO("%s, P2P PS enabled\n", __func__);
+
+		/* Disable all NoA desc before update NoA desc */
+		rtw_phl_p2pps_noa_disable_all(GET_PHL_INFO(dvobj),
+					      padapter->phl_role);
+
+		for (i = 0; i < pwdinfo->noa_num; i++) {
+			struct rtw_phl_noa_desc desc = {0};
+			enum rtw_phl_status phl_sts;
+
+			desc.enable = _TRUE;
+			/* config NOA start time */
+			desc.start_t_h = (u32)(pmlmeext->TSFValue >> 32);
+			desc.start_t_l = pwdinfo->noa_start_time[i];
+			/* config NOA duration */
+			desc.duration = pwdinfo->noa_duration[i];
+			/* config NOA interval */
+			desc.interval = pwdinfo->noa_interval[i];
+			/* config NOA count */
+			desc.count = pwdinfo->noa_count[i];
+			desc.w_role = padapter->phl_role;
+			desc.rlink = padapter_link->wrlink;
+			if (pwdinfo->noa_count[i] == 255)
+				desc.tag = P2PPS_TRIG_GC_255;
+			else
+				desc.tag = P2PPS_TRIG_GC;
+
+			RTW_INFO("[NoA Desc]- entry: %d\n", i);
+			RTW_INFO("[NoA Desc]- duration: %u\n", desc.duration);
+			RTW_INFO("[NoA Desc]- interval: %u\n", desc.interval);
+			RTW_INFO("[NoA Desc]- high_start_time: %u\n",
+				 desc.start_t_h);
+			RTW_INFO("[NoA Desc]- low_start_time: %u\n",
+				 desc.start_t_l);
+			RTW_INFO("[NoA Desc]- count: %u\n",desc.count);
+
+			phl_sts = rtw_phl_p2pps_noa_update(GET_PHL_INFO(dvobj),
+							   &desc);
+			if (phl_sts != RTW_PHL_STATUS_SUCCESS)
+				RTW_ERR("rtw_phl_p2pps_noa_update failed\n");
+		}
+
+		break;
+	default:
+		break;
+	}
+}
+
+u8 p2p_ps_wk_cmd(_adapter *padapter, u8 p2p_ps_state, u8 enqueue)
+{
+	struct cmd_obj	*cmd;
+	struct drvextra_cmd_parm *pdrvextra_cmd_parm;
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+	struct cmd_priv	*pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8 res = _SUCCESS;
+
+
+	if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE) ||
+	    rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DEVICE))
+		return res;
+
+	if (enqueue) {
+		cmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+		if (cmd == NULL) {
+			res = _FAIL;
+			goto exit;
+		}
+		cmd->padapter = padapter;
+
+		pdrvextra_cmd_parm = (struct drvextra_cmd_parm *)rtw_zmalloc(sizeof(struct drvextra_cmd_parm));
+		if (pdrvextra_cmd_parm == NULL) {
+			rtw_mfree((unsigned char *)cmd, sizeof(struct cmd_obj));
+			res = _FAIL;
+			goto exit;
+		}
+
+		pdrvextra_cmd_parm->ec_id = P2P_PS_WK_CID;
+		pdrvextra_cmd_parm->type = p2p_ps_state;
+		pdrvextra_cmd_parm->size = 0;
+		pdrvextra_cmd_parm->pbuf = NULL;
+
+		init_h2fwcmd_w_parm_no_rsp(cmd, pdrvextra_cmd_parm, CMD_SET_DRV_EXTRA);
+
+		res = rtw_enqueue_cmd(pcmdpriv, cmd);
+	} else
+		p2p_ps_wk_hdl(padapter, p2p_ps_state);
+exit:
+	return res;
+}
+
+void rtw_append_probe_resp_p2p_go_noa(struct xmit_frame *xframe)
+{
+	_adapter *adapter = xframe->padapter;
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	u8 *frame = xframe->buf_addr + TXDESC_OFFSET;
+	u8 frame_ie_offset = WLAN_HDR_A3_LEN + _PROBERSP_IE_OFFSET_;
+	u8 *p2p_ie = NULL;
+	u16 p2p_go_noa_ie_len = 0;
+
+	if (!MLME_IS_GO(adapter))
+		return;
+
+	if (wdinfo->noa_num == 0)
+		return;
+
+	p2p_ie = rtw_get_p2p_ie(frame + frame_ie_offset,
+				xframe->attrib.pktlen - frame_ie_offset,
+				NULL, NULL);
+	if (p2p_ie) {
+		_build_p2p_go_noa_ie(wdinfo, frame + xframe->attrib.pktlen,
+				     &p2p_go_noa_ie_len);
+		xframe->attrib.pktlen += p2p_go_noa_ie_len;
+	}
+}
+#endif /* CONFIG_P2P_PS */
+
+void rtw_append_probe_resp_vendor_ie(struct xmit_frame *xframe)
+{
+	_adapter *adapter = xframe->padapter;
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	u8 *frame = xframe->buf_addr + TXDESC_OFFSET;
+	u8 *frame_tail = frame + xframe->attrib.pktlen;
+	u8 frame_ie_offset = WLAN_HDR_A3_LEN + _PROBERSP_IE_OFFSET_;
+	u8 *vendor_ie = NULL;
+	u32 len = 0;
+
+	if (!MLME_IS_GO(adapter))
+		return;
+
+	vendor_ie = rtw_get_vendor_ie(frame + frame_ie_offset,
+				xframe->attrib.pktlen - frame_ie_offset,
+				NULL, NULL);
+	if (vendor_ie)
+		return;
+
+	if (pmlmepriv->probe_rsp) {
+		len = pmlmepriv->probe_rsp_len;
+		if (len > 0) {
+			_rtw_memcpy(frame_tail, pmlmepriv->probe_rsp, len);
+			xframe->attrib.pktlen += len;
+			frame_tail = frame + xframe->attrib.pktlen;
+		}
+	}
+}
+
+void rtw_append_probe_resp_p2p_ie(struct xmit_frame *xframe)
+{
+	_adapter *adapter = xframe->padapter;
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	u8 *frame = xframe->buf_addr + TXDESC_OFFSET;
+	u8 *frame_tail = frame + xframe->attrib.pktlen;
+	u8 frame_ie_offset = WLAN_HDR_A3_LEN + _PROBERSP_IE_OFFSET_;
+	u8 *p2p_ie = NULL;
+	u32 len = 0;
+
+	if (!MLME_IS_GO(adapter))
+		return;
+
+	p2p_ie = rtw_get_p2p_ie(frame + frame_ie_offset,
+				xframe->attrib.pktlen - frame_ie_offset,
+				NULL, NULL);
+	if (p2p_ie)
+		return;
+
+	if (pmlmepriv->p2p_go_probe_resp_ie) {
+		len = pmlmepriv->p2p_go_probe_resp_ie_len;
+		if (len > 0) {
+			_rtw_memcpy(frame_tail, pmlmepriv->p2p_go_probe_resp_ie, len);
+			xframe->attrib.pktlen += len;
+			frame_tail = frame + xframe->attrib.pktlen;
+		}
+	}
+
+	len = build_probe_resp_wfd_ie(wdinfo, frame_tail, 0);
+	xframe->attrib.pktlen += len;
+}
+
+void reset_global_wifidirect_info(_adapter *padapter)
+{
+	struct wifidirect_info	*pwdinfo;
+
+	pwdinfo = &padapter->wdinfo;
+	rtw_tdls_wfd_enable(padapter, 0);
+	pwdinfo->wfd_tdls_weaksec = _TRUE;
+}
+
+#ifdef CONFIG_WFD
+int rtw_init_wifi_display_info(_adapter *padapter)
+{
+	int	res = _SUCCESS;
+	struct wifi_display_info *pwfd_info = &padapter->wfd_info;
+
+	/* Used in P2P and TDLS */
+	pwfd_info->init_rtsp_ctrlport = 554;
+#ifdef CONFIG_IOCTL_CFG80211
+	pwfd_info->rtsp_ctrlport = 0;
+#else
+	pwfd_info->rtsp_ctrlport = pwfd_info->init_rtsp_ctrlport; /* set non-zero value for legacy wfd */
+#endif
+	pwfd_info->tdls_rtsp_ctrlport = 0;
+	pwfd_info->peer_rtsp_ctrlport = 0;	/*	Reset to 0 */
+	pwfd_info->wfd_enable = _FALSE;
+	pwfd_info->wfd_device_type = WFD_DEVINFO_PSINK;
+	pwfd_info->scan_result_type = SCAN_RESULT_P2P_ONLY;
+
+	/* Used in P2P */
+	pwfd_info->peer_session_avail = _TRUE;
+	pwfd_info->wfd_pc = _FALSE;
+
+	/* Used in TDLS */
+	_rtw_memset(pwfd_info->ip_address, 0x00, 4);
+	_rtw_memset(pwfd_info->peer_ip_address, 0x00, 4);
+	return res;
+
+}
+
+inline void rtw_wfd_enable(_adapter *adapter, bool on)
+{
+	struct wifi_display_info *wfdinfo = &adapter->wfd_info;
+
+	if (on) {
+		wfdinfo->rtsp_ctrlport = wfdinfo->init_rtsp_ctrlport;
+		wfdinfo->wfd_enable = _TRUE;
+
+	} else {
+		wfdinfo->wfd_enable = _FALSE;
+		wfdinfo->rtsp_ctrlport = 0;
+	}
+}
+
+inline void rtw_wfd_set_ctrl_port(_adapter *adapter, u16 port)
+{
+	struct wifi_display_info *wfdinfo = &adapter->wfd_info;
+
+	wfdinfo->init_rtsp_ctrlport = port;
+	if (wfdinfo->wfd_enable == _TRUE)
+		wfdinfo->rtsp_ctrlport = port;
+	if (adapter->wdinfo.wfd_tdls_enable == 1)
+		wfdinfo->tdls_rtsp_ctrlport = port;
+}
+
+inline void rtw_tdls_wfd_enable(_adapter *adapter, bool on)
+{
+	struct wifi_display_info *wfdinfo = &adapter->wfd_info;
+
+	if (on) {
+		wfdinfo->tdls_rtsp_ctrlport = wfdinfo->init_rtsp_ctrlport;
+		adapter->wdinfo.wfd_tdls_enable = 1;
+
+	} else {
+		adapter->wdinfo.wfd_tdls_enable = 0;
+		wfdinfo->tdls_rtsp_ctrlport = 0;
+	}
+}
+
+u32 rtw_append_beacon_wfd_ie(_adapter *adapter, u8 *pbuf)
+{
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	u8 build_ie_by_self = 0;
+	u32 len = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (_TRUE == wdinfo->wfd_info->wfd_enable)
+#endif
+		build_ie_by_self = 1;
+
+	if (build_ie_by_self)
+		len = build_beacon_wfd_ie(wdinfo, pbuf);
+#ifdef CONFIG_IOCTL_CFG80211
+	else if (mlme->wfd_beacon_ie && mlme->wfd_beacon_ie_len > 0) {
+		len = mlme->wfd_beacon_ie_len;
+		_rtw_memcpy(pbuf, mlme->wfd_beacon_ie, len);
+	}
+#endif
+
+exit:
+	return len;
+}
+
+u32 rtw_append_probe_req_wfd_ie(_adapter *adapter, u8 *pbuf)
+{
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	u8 build_ie_by_self = 0;
+	u32 len = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (_TRUE == wdinfo->wfd_info->wfd_enable)
+#endif
+		build_ie_by_self = 1;
+
+	if (build_ie_by_self)
+		len = build_probe_req_wfd_ie(wdinfo, pbuf);
+#ifdef CONFIG_IOCTL_CFG80211
+	else if (mlme->wfd_probe_req_ie && mlme->wfd_probe_req_ie_len > 0) {
+		len = mlme->wfd_probe_req_ie_len;
+		_rtw_memcpy(pbuf, mlme->wfd_probe_req_ie, len);
+	}
+#endif
+
+exit:
+	return len;
+}
+
+u32 rtw_append_probe_resp_wfd_ie(_adapter *adapter, u8 *pbuf)
+{
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	u8 build_ie_by_self = 0;
+	u32 len = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (_TRUE == wdinfo->wfd_info->wfd_enable)
+#endif
+		build_ie_by_self = 1;
+
+	if (build_ie_by_self)
+		len = build_probe_resp_wfd_ie(wdinfo, pbuf, 0);
+#ifdef CONFIG_IOCTL_CFG80211
+	else if (mlme->wfd_probe_resp_ie && mlme->wfd_probe_resp_ie_len > 0) {
+		len = mlme->wfd_probe_resp_ie_len;
+		_rtw_memcpy(pbuf, mlme->wfd_probe_resp_ie, len);
+	}
+#endif
+
+exit:
+	return len;
+}
+
+u32 rtw_append_assoc_req_wfd_ie(_adapter *adapter, u8 *pbuf)
+{
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	u8 build_ie_by_self = 0;
+	u32 len = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (_TRUE == wdinfo->wfd_info->wfd_enable)
+#endif
+		build_ie_by_self = 1;
+
+	if (build_ie_by_self)
+		len = build_assoc_req_wfd_ie(wdinfo, pbuf);
+#ifdef CONFIG_IOCTL_CFG80211
+	else if (mlme->wfd_assoc_req_ie && mlme->wfd_assoc_req_ie_len > 0) {
+		len = mlme->wfd_assoc_req_ie_len;
+		_rtw_memcpy(pbuf, mlme->wfd_assoc_req_ie, len);
+	}
+#endif
+
+exit:
+	return len;
+}
+
+u32 rtw_append_assoc_resp_wfd_ie(_adapter *adapter, u8 *pbuf)
+{
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	u8 build_ie_by_self = 0;
+	u32 len = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		goto exit;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (_TRUE == wdinfo->wfd_info->wfd_enable)
+#endif
+		build_ie_by_self = 1;
+
+	if (build_ie_by_self)
+		len = build_assoc_resp_wfd_ie(wdinfo, pbuf);
+#ifdef CONFIG_IOCTL_CFG80211
+	else if (mlme->wfd_assoc_resp_ie && mlme->wfd_assoc_resp_ie_len > 0) {
+		len = mlme->wfd_assoc_resp_ie_len;
+		_rtw_memcpy(pbuf, mlme->wfd_assoc_resp_ie, len);
+	}
+#endif
+
+exit:
+	return len;
+}
+
+#endif /* CONFIG_WFD */
+
+void init_wifidirect_info(_adapter *padapter, enum P2P_ROLE role)
+{
+	struct wifidirect_info	*pwdinfo;
+#ifdef CONFIG_WFD
+	struct wifi_display_info	*pwfd_info = &padapter->wfd_info;
+#endif
+	pwdinfo = &padapter->wdinfo;
+
+	pwdinfo->padapter = padapter;
+
+	if (role == P2P_ROLE_DEVICE)
+		rtw_p2p_set_role(pwdinfo, P2P_ROLE_DEVICE);
+	else if (role == P2P_ROLE_CLIENT)
+		rtw_p2p_set_role(pwdinfo, P2P_ROLE_CLIENT);
+	else if (role == P2P_ROLE_GO)
+		rtw_p2p_set_role(pwdinfo, P2P_ROLE_GO);
+
+	/*	Use the OFDM rate in the P2P probe response frame. ( 6(B), 9(B), 12, 18, 24, 36, 48, 54 )	 */
+	pwdinfo->support_rate[0] = 0x8c;	/*	6(B) */
+	pwdinfo->support_rate[1] = 0x92;	/*	9(B) */
+	pwdinfo->support_rate[2] = 0x18;	/*	12 */
+	pwdinfo->support_rate[3] = 0x24;	/*	18 */
+	pwdinfo->support_rate[4] = 0x30;	/*	24 */
+	pwdinfo->support_rate[5] = 0x48;	/*	36 */
+	pwdinfo->support_rate[6] = 0x60;	/*	48 */
+	pwdinfo->support_rate[7] = 0x6c;	/*	54 */
+
+	_rtw_memcpy((void *) pwdinfo->p2p_wildcard_ssid, "DIRECT-", 7);
+
+#ifdef CONFIG_WFD
+	pwdinfo->wfd_info = pwfd_info;
+#endif /* CONFIG_WFD */
+
+	pwdinfo->wfd_tdls_enable = 0;
+}
+
+void _rtw_p2p_set_role(struct wifidirect_info *wdinfo, enum P2P_ROLE role)
+{
+	if (wdinfo->role != role) {
+		wdinfo->role = role;
+		rtw_mi_update_iface_status(&(wdinfo->padapter->mlmepriv), 0);
+	}
+}
+
+#ifdef CONFIG_DBG_P2P
+
+/**
+ * rtw_p2p_role_txt - Get the p2p role name as a text string
+ * @role: P2P role
+ * Returns: The state name as a printable text string
+ */
+const char *rtw_p2p_role_txt(enum P2P_ROLE role)
+{
+	switch (role) {
+	case P2P_ROLE_DISABLE:
+		return "P2P_ROLE_DISABLE";
+	case P2P_ROLE_DEVICE:
+		return "P2P_ROLE_DEVICE";
+	case P2P_ROLE_CLIENT:
+		return "P2P_ROLE_CLIENT";
+	case P2P_ROLE_GO:
+		return "P2P_ROLE_GO";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+/**
+ * rtw_p2p_state_txt - Get the p2p state name as a text string
+ * @state: P2P state
+ * Returns: The state name as a printable text string
+ */
+const char *rtw_p2p_state_txt(enum P2P_STATE state)
+{
+	switch (state) {
+	case P2P_STATE_NONE:
+		return "P2P_STATE_NONE";
+	case P2P_STATE_IDLE:
+		return "P2P_STATE_IDLE";
+	case P2P_STATE_LISTEN:
+		return "P2P_STATE_LISTEN";
+	case P2P_STATE_FIND_PHASE_SEARCH:
+		return "P2P_STATE_FIND_PHASE_SEARCH";
+	case P2P_STATE_GONEGO_OK:
+		return "P2P_STATE_GONEGO_OK";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+#if 0
+void dbg_rtw_p2p_restore_state(struct wifidirect_info *wdinfo, const char *caller, int line)
+{
+	if (wdinfo->pre_p2p_state != -1) {
+		RTW_INFO("[CONFIG_DBG_P2P]%s:%d restore from %s to %s\n", caller, line
+			, p2p_state_str[wdinfo->p2p_state], p2p_state_str[wdinfo->pre_p2p_state]
+			);
+		_rtw_p2p_restore_state(wdinfo);
+	} else {
+		RTW_INFO("[CONFIG_DBG_P2P]%s:%d restore no pre state, cur state %s\n", caller, line
+			 , p2p_state_str[wdinfo->p2p_state]
+			);
+	}
+}
+#endif
+void dbg_rtw_p2p_set_role(struct wifidirect_info *wdinfo, enum P2P_ROLE role, const char *caller, int line)
+{
+	if (wdinfo->role != role) {
+		enum P2P_ROLE old_role = wdinfo->role;
+		_rtw_p2p_set_role(wdinfo, role);
+		RTW_INFO("[CONFIG_DBG_P2P]%s:%d set_role from %s to %s\n", caller, line
+			, rtw_p2p_role_txt(old_role), rtw_p2p_role_txt(wdinfo->role)
+			);
+	} else {
+		RTW_INFO("[CONFIG_DBG_P2P]%s:%d set_role to same role %s\n", caller, line
+			 , rtw_p2p_role_txt(wdinfo->role)
+			);
+	}
+}
+#endif /* CONFIG_DBG_P2P */
+
+
+int rtw_p2p_enable(_adapter *padapter, enum P2P_ROLE role)
+{
+	int ret = _SUCCESS;
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (role == P2P_ROLE_DEVICE || role == P2P_ROLE_CLIENT || role == P2P_ROLE_GO) {
+#ifndef CONFIG_CFG80211_REPORT_PROBE_REQ
+#if defined(CONFIG_CONCURRENT_MODE) && !RTW_P2P_GROUP_INTERFACE
+		/*	Commented by Albert 2011/12/30 */
+		/*	The driver just supports 1 P2P group operation. */
+		/*	So, this function will do nothing if the buddy adapter had enabled the P2P function. */
+		/*if(!rtw_p2p_chk_state(pbuddy_wdinfo, P2P_STATE_NONE))
+			return ret;*/
+		/* Only selected interface can be P2P interface */
+		if (padapter->iface_id != padapter->registrypriv.sel_p2p_iface) {
+			RTW_ERR("%s, iface_id:%d is not P2P interface!\n", __func__, padapter->iface_id);
+			ret = _FAIL;
+			return ret;
+		}
+#endif
+#endif
+
+		/* The P2P device does not need to remove the CCK rate of the
+		 * adapter as it can act as an STA.
+		 */
+		if (role != P2P_ROLE_DEVICE)
+			update_tx_basic_rate(padapter, padapter_link, WLAN_MD_11AGN);
+
+		/* Enable P2P function */
+		init_wifidirect_info(padapter, role);
+
+		#ifdef CONFIG_IOCTL_CFG80211
+		adapter_wdev_data(padapter)->p2p_enabled = _TRUE;
+		#endif
+
+#if 0
+		rtw_hal_set_phydm_var(padapter, HAL_PHYDM_P2P_STATE, NULL, _TRUE);
+#ifdef CONFIG_WFD
+		if (rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+			rtw_hal_set_phydm_var(padapter, HAL_PHYDM_WIFI_DISPLAY_STATE, NULL, _TRUE);
+#endif
+#endif
+	} else if (role == P2P_ROLE_DISABLE) {
+
+		#ifdef CONFIG_IOCTL_CFG80211
+		adapter_wdev_data(padapter)->p2p_enabled = _FALSE;
+		#endif
+
+		pwdinfo->listen_channel = 0;
+
+		/* Disable P2P function */
+		if (!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE))
+			rtw_p2p_set_role(pwdinfo, P2P_ROLE_DISABLE);
+		else
+			RTW_ERR("%s, disable P2P when p2p role is P2P_ROLE_DISABLE!\n", __func__);
+
+#if 0
+		rtw_hal_set_phydm_var(padapter, HAL_PHYDM_P2P_STATE, NULL, _FALSE);
+#ifdef CONFIG_WFD
+		if (rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+			rtw_hal_set_phydm_var(padapter, HAL_PHYDM_WIFI_DISPLAY_STATE, NULL, _FALSE);
+#endif
+#endif
+
+		/* Restore to initial setting. */
+		update_tx_basic_rate(padapter, padapter_link, padapter->registrypriv.wireless_mode);
+	}
+
+exit:
+	return ret;
+}
+
+#endif /* CONFIG_P2P */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_phl.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_phl.c
new file mode 100644
index 000000000000..1fb5b1178aed
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_phl.c
@@ -0,0 +1,4732 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_PHL_C_
+#include <drv_types.h>
+
+static enum phl_cmd_type rtw_cmdf_to_phl_cmd_type(u8 flag)
+{
+	if (flag == RTW_CMDF_DIRECTLY)
+		return PHL_CMD_DIRECTLY;
+	if (flag == RTW_CMDF_WAIT_ACK)
+		return PHL_CMD_WAIT;
+	return PHL_CMD_NO_WAIT;
+}
+
+/***************** export API to osdep/core*****************/
+
+static const char *const _band_cap_str[] = {
+	/* BIT0 */"2G",
+	/* BIT1 */"5G",
+	/* BIT2 */"6G",
+};
+
+static const char *const _bw_cap_str[] = {
+	/* BIT0 */"20M",
+	/* BIT1 */"40M",
+	/* BIT2 */"80M",
+	/* BIT3 */"160M",
+	/* BIT4 */"320M",
+	/* BIT5 */"80_80M",
+	/* BIT6 */"5M",
+	/* BIT7 */"10M",
+};
+
+static const char *const _proto_cap_str[] = {
+	/* BIT0 */"b",
+	/* BIT1 */"a",
+	/* BIT2 */"g",
+	/* BIT3 */"n",
+	/* BIT4 */"ac",
+	/* BIT5 */"ax",
+};
+
+static const char *const _wl_func_str[] = {
+	/* BIT0 */"P2P",
+	/* BIT1 */"MIRACAST",
+	/* BIT2 */"TDLS",
+	/* BIT3 */"FTM",
+};
+
+static const char *const hw_cap_str = "[HW-CAP]";
+void rtw_hw_dump_hal_spec(void *sel, struct dvobj_priv *dvobj)
+{
+	struct hal_spec_t *hal_spec = GET_HAL_SPEC(dvobj);
+	_adapter *padapter = dvobj_get_primary_adapter(dvobj);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	int num_elements = 0;
+	int i;
+
+	RTW_PRINT_SEL(sel, "%s ic_name:%s\n", hw_cap_str, hal_spec->ic_name);
+	RTW_PRINT_SEL(sel, "%s macid_num:%u\n", hw_cap_str, hal_spec->macid_num);
+	RTW_PRINT_SEL(sel, "%s sec_cap:0x%02x\n", hw_cap_str, hal_spec->sec_cap);
+	RTW_PRINT_SEL(sel, "%s sec_cam_ent_num:%u\n", hw_cap_str, hal_spec->sec_cam_ent_num);
+
+	RTW_PRINT_SEL(sel, "%s rfpath_num_2g:%u\n", hw_cap_str, hal_spec->rfpath_num_2g);
+	RTW_PRINT_SEL(sel, "%s rfpath_num_5g:%u\n", hw_cap_str, hal_spec->rfpath_num_5g);
+	RTW_PRINT_SEL(sel, "%s rf_reg_path_num:%u\n", hw_cap_str, hal_spec->rf_reg_path_num);
+	RTW_PRINT_SEL(sel, "%s max_tx_cnt:%u\n", hw_cap_str, hal_spec->max_tx_cnt);
+	if (padapter_link && padapter_link->wrlink) {
+		RTW_PRINT_SEL(sel, "%s tx_nss_num:%u\n", hw_cap_str, get_phy_tx_nss(padapter, padapter_link));
+		RTW_PRINT_SEL(sel, "%s rx_nss_num:%u\n", hw_cap_str, get_phy_rx_nss(padapter, padapter_link));
+	}
+
+	RTW_PRINT_SEL(sel, "%s band_cap:", hw_cap_str);
+	for (i = 0; i < BAND_CAP_BIT_NUM; i++) {
+		if (((dvobj->phl_com->dev_cap.band_sup) >> i) & BIT0 && _band_cap_str[i])
+			_RTW_PRINT_SEL(sel, "%s ", _band_cap_str[i]);
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+
+	RTW_PRINT_SEL(sel, "%s bw_cap:", hw_cap_str);
+	for (i = 0; i < BW_CAP_BIT_NUM; i++) {
+		if (((dvobj->phl_com->dev_cap.bw_sup) >> i) & BIT0 && _bw_cap_str[i])
+			_RTW_PRINT_SEL(sel, "%s ", _bw_cap_str[i]);
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+
+	RTW_PRINT_SEL(sel, "%s proto_cap:", hw_cap_str);
+	num_elements = sizeof(_proto_cap_str) / sizeof(_proto_cap_str[0]);
+	for (i = 0; i < num_elements; i++) {
+		if ((dvobj->phl_com->phy_cap[0].proto_sup >> i) & BIT0 && _proto_cap_str[i])
+			_RTW_PRINT_SEL(sel, "%s ", _proto_cap_str[i]);
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+
+#if 0 /*GEORGIA_TODO_FIXIT*/
+	RTW_PRINT_SEL(sel, "%s txgi_max:%u\n", hw_cap_str, hal_spec->txgi_max);
+	RTW_PRINT_SEL(sel, "%s txgi_pdbm:%u\n", hw_cap_str, hal_spec->txgi_pdbm);
+#endif
+	RTW_PRINT_SEL(sel, "%s wl_func:", hw_cap_str);
+	for (i = 0; i < WL_FUNC_BIT_NUM; i++) {
+		if (((hal_spec->wl_func) >> i) & BIT0 && _wl_func_str[i])
+			_RTW_PRINT_SEL(sel, "%s ", _wl_func_str[i]);
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+
+#if 0 /*GEORGIA_TODO_FIXIT*/
+
+	RTW_PRINT_SEL(sel, "%s pg_txpwr_saddr:0x%X\n", hw_cap_str, hal_spec->pg_txpwr_saddr);
+	RTW_PRINT_SEL(sel, "%s pg_txgi_diff_factor:%u\n", hw_cap_str, hal_spec->pg_txgi_diff_factor);
+#endif
+}
+
+void rtw_dump_phl_sta_info(void *sel, struct sta_info *sta)
+{
+	struct rtw_phl_stainfo_t *phl_sta = sta->phl_sta;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	RTW_PRINT_SEL(sel, "[PHL STA]- role-idx: %d\n", phl_sta->wrole->id);
+
+	RTW_PRINT_SEL(sel, "[PHL STA]- mac_addr:%s\n", get_macaddr_str(mac_addr_str, sel, phl_sta->mac_addr));
+	RTW_PRINT_SEL(sel, "[PHL STA]- aid: %d\n", phl_sta->aid);
+	RTW_PRINT_SEL(sel, "[PHL STA]- macid: %d\n", phl_sta->macid);
+
+	RTW_PRINT_SEL(sel, "[PHL STA]- wifi_band: %d\n", phl_sta->chandef.band);
+	RTW_PRINT_SEL(sel, "[PHL STA]- bw: %d\n", phl_sta->chandef.bw);
+	RTW_PRINT_SEL(sel, "[PHL STA]- chan: %d\n", phl_sta->chandef.chan);
+	RTW_PRINT_SEL(sel, "[PHL STA]- offset: %d\n", phl_sta->chandef.offset);
+}
+
+inline bool rtw_hw_chk_band_cap(struct dvobj_priv *dvobj, u8 cap)
+{
+	return dvobj->phl_com->dev_cap.band_sup & cap;
+}
+
+inline bool rtw_hw_chk_bw_cap(struct dvobj_priv *dvobj, u8 cap)
+{
+	return dvobj->phl_com->dev_cap.bw_sup & cap;
+}
+
+inline bool rtw_hw_chk_proto_cap(struct dvobj_priv *dvobj, u8 cap)
+{
+	return dvobj->phl_com->phy_cap[0].proto_sup & cap;
+}
+
+inline bool rtw_hw_chk_wl_func(struct dvobj_priv *dvobj, u8 func)
+{
+	return GET_HAL_SPEC(dvobj)->wl_func & func;
+}
+
+inline bool rtw_hw_is_band_support(struct dvobj_priv *dvobj, u8 band)
+{
+	return (dvobj->phl_com->dev_cap.band_sup & band_to_band_cap(band));
+}
+
+inline bool rtw_hw_is_bw_support(struct dvobj_priv *dvobj, u8 bw)
+{
+	return (dvobj->phl_com->dev_cap.bw_sup & ch_width_to_bw_cap(bw));
+}
+
+inline bool rtw_hw_is_wireless_mode_support(struct dvobj_priv *dvobj, u8 mode)
+{
+	u8 proto_sup = dvobj->phl_com->phy_cap[0].proto_sup;
+
+	if (mode == WLAN_MD_11B)
+		if ((proto_sup & WLAN_MD_11B) && rtw_hw_chk_band_cap(dvobj, BAND_CAP_2G))
+			return 1;
+
+	if (mode == WLAN_MD_11G)
+		if ((proto_sup & WLAN_MD_11G) && rtw_hw_chk_band_cap(dvobj, BAND_CAP_2G))
+			return 1;
+
+	if (mode == WLAN_MD_11A)
+		if ((proto_sup & WLAN_MD_11A) && rtw_hw_chk_band_cap(dvobj, BAND_CAP_5G))
+			return 1;
+
+	#ifdef CONFIG_80211N_HT
+	if (mode == WLAN_MD_11N)
+		if (proto_sup & WLAN_MD_11N)
+			return 1;
+	#endif
+
+	#ifdef CONFIG_80211AC_VHT
+	if (mode == WLAN_MD_11AC)
+		if ((proto_sup & WLAN_MD_11AC) && rtw_hw_chk_band_cap(dvobj, BAND_CAP_5G))
+			return 1;
+	#endif
+
+	#ifdef CONFIG_80211AX_HE
+	if (mode == WLAN_MD_11AX)
+		if (proto_sup & WLAN_MD_11AX)
+			return 1;
+	#endif
+	return 0;
+}
+
+
+inline u8 rtw_hw_get_wireless_mode(struct dvobj_priv *dvobj)
+{
+	u8 proto_sup = dvobj->phl_com->phy_cap[0].proto_sup;
+	u8 wireless_mode = 0;
+
+	if(proto_sup & WLAN_MD_11B)
+		wireless_mode |= WLAN_MD_11B;
+
+	if(proto_sup & WLAN_MD_11G)
+		wireless_mode |= WLAN_MD_11G;
+
+	if(proto_sup & WLAN_MD_11A)
+		wireless_mode |= WLAN_MD_11A;
+
+	#ifdef CONFIG_80211N_HT
+	if(proto_sup & WLAN_MD_11N)
+		wireless_mode |= WLAN_MD_11N;
+	#endif
+
+	#ifdef CONFIG_80211AC_VHT
+	if(proto_sup & WLAN_MD_11AC)
+		wireless_mode |= WLAN_MD_11AC;
+	#endif
+
+	#ifdef CONFIG_80211AX_HE
+	if(proto_sup & WLAN_MD_11AX) {
+		wireless_mode |= WLAN_MD_11AX;
+	}
+	#endif
+
+	return wireless_mode;
+}
+
+inline u8 rtw_hw_get_band_cap(struct dvobj_priv *dvobj)
+{
+	return dvobj->phl_com->dev_cap.band_sup;
+}
+
+inline bool rtw_hw_is_mimo_support(_adapter *adapter)
+{
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	if ((get_phy_tx_nss(adapter, adapter_link) == 1) &&
+		(get_phy_rx_nss(adapter, adapter_link) == 1))
+		return 0;
+	return 1;
+}
+
+/*
+* rtw_hw_largest_bw - starting from in_bw, get largest bw supported by HAL
+* @adapter:
+* @in_bw: starting bw, value of enum channel_width
+*
+* Returns: value of enum channel_width
+*/
+u8 rtw_hw_largest_bw(struct dvobj_priv *dvobj, u8 in_bw)
+{
+	for (; in_bw > CHANNEL_WIDTH_20; in_bw--) {
+		if (rtw_hw_is_bw_support(dvobj, in_bw))
+			break;
+	}
+
+	if (!rtw_hw_is_bw_support(dvobj, in_bw))
+		rtw_warn_on(1);
+
+	return in_bw;
+}
+
+u8 rtw_hw_get_mac_addr(struct dvobj_priv *dvobj, u8 *hw_mac_addr)
+{
+	if (rtw_phl_get_mac_addr_efuse(dvobj->phl, hw_mac_addr) != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s failed\n", __func__);
+		return _FAIL;
+	}
+	return _SUCCESS;
+}
+
+u8 rtw_core_deregister_phl_msg(struct dvobj_priv *dvobj)
+{
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+
+	psts = rtw_phl_msg_hub_deregister_recver(dvobj->phl, MSG_RECV_CORE);
+	if(psts	== RTW_PHL_STATUS_FAILURE) {
+		RTW_ERR("%s failed\n", __func__);
+		return _FAIL;
+	}
+	return _SUCCESS;
+}
+
+void rtw_hw_deinit(struct dvobj_priv *dvobj)
+{
+	if (dvobj->phl) {
+		rtw_phl_trx_free(dvobj->phl);
+		rtw_core_deregister_phl_msg(dvobj);
+		rtw_phl_watchdog_deinit(dvobj->phl);
+		rtw_phl_deinit(dvobj->phl);
+	}
+
+	#ifdef DBG_PHL_MEM_ALLOC
+	RTW_INFO("[PHL-MEM] %s PHL memory :%d\n", __func__,
+					ATOMIC_READ(&(dvobj->phl_mem)));
+	#endif
+}
+
+
+#if 0 /*GEORGIA_TODO_FIXIT*/
+void rtw_hw_intf_cfg(struct dvobj_priv *dvobj, struct hal_com_t *hal_com)
+{
+	struct hci_info_st hci_info;
+
+	#ifdef CONFIG_PCI_HCI
+	if (dvobj->interface_type == RTW_HCI_PCIE) {
+		PPCI_DATA pci = dvobj_to_pci(dvobj);
+		//hci_info.
+	}
+	#endif
+
+	#ifdef CONFIG_USB_HCI
+	if (dvobj->interface_type == RTW_HCI_USB) {
+		PUSB_DATA usb = dvobj_to_usb(dvobj);
+		#if 0
+		u8 usb_speed; /* 1.1, 2.0 or 3.0 */
+		u16 usb_bulkout_size;
+		u8 nr_endpoint; /*MAX_ENDPOINT_NUM*/
+
+		/* Bulk In , Out Pipe information */
+		int RtInPipe[MAX_BULKIN_NUM];
+		u8 RtNumInPipes;
+		int RtOutPipe[MAX_BULKOUT_NUM];
+		u8 RtNumOutPipes;
+		#endif
+		//hci_info
+	}
+	#endif
+
+	#ifdef CONFIG_SDIO_HCI
+	if (dvobj->interface_type == RTW_HCI_SDIO) {
+		struct sdio_data *sdio = dvobj_to_sdio(dvobj);
+
+		hci_info.clock = sdio->clock;
+		hci_info.timing = sdio->timing;
+		hci_info.sd3_bus_mode = sdio->sd3_bus_mode;
+		hci_info.block_sz = sdio->block_transfer_len;
+		hci_info.align_sz = sdio->block_transfer_len;
+	}
+	#endif
+
+	rtw_hal_intf_config(hal_com, &hci_info);
+}
+#endif
+
+static void _hw_ic_info_cfg(struct dvobj_priv *dvobj, struct rtw_ic_info *ic_info)
+{
+	_rtw_memset(ic_info, 0,sizeof(struct rtw_ic_info));
+
+	ic_info->ic_id = dvobj->ic_id;
+	ic_info->hci_type = dvobj->interface_type;
+
+	#ifdef CONFIG_PCI_HCI
+	if (dvobj->interface_type == RTW_HCI_PCIE) {
+		PPCI_DATA pci = dvobj_to_pci(dvobj);
+
+	}
+	#endif
+
+	#ifdef CONFIG_USB_HCI
+	if (dvobj->interface_type == RTW_HCI_USB) {
+		PUSB_DATA usb = dvobj_to_usb(dvobj);
+
+		ic_info->usb_info.usb_speed = usb->usb_speed;
+		ic_info->usb_info.usb_bulkout_size = usb->usb_bulkout_size;
+		ic_info->usb_info.inep_num = usb->RtNumInPipes;
+		ic_info->usb_info.outep_num = usb->RtNumOutPipes;
+	}
+	#endif
+
+	#ifdef CONFIG_SDIO_HCI
+	if (dvobj->interface_type == RTW_HCI_SDIO) {
+		struct sdio_data *sdio = dvobj_to_sdio(dvobj);
+
+		ic_info->sdio_info.clock = sdio->clock;
+		ic_info->sdio_info.timing = sdio->timing;
+		ic_info->sdio_info.sd3_bus_mode = sdio->sd3_bus_mode;
+		ic_info->sdio_info.io_align_sz = 4;
+		ic_info->sdio_info.block_sz = sdio->block_transfer_len;
+		ic_info->sdio_info.tx_align_sz = sdio->block_transfer_len;
+		ic_info->sdio_info.tx_512_by_byte_mode =
+				(sdio->max_byte_size >= 512) ? true : false;
+	}
+	#endif
+}
+
+u8 rtw_backup_and_get_final_ss(_adapter *adapter, struct sta_info *sta, u8 chg_ss)
+{
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	u8 final_ss = 0;
+
+	if (chg_ss) {
+		final_ss = chg_ss;
+		rtw_ctrl_and_backup_assoc_cap_rx_nss(adapter, sta, final_ss);
+	} else {
+		sta->phl_sta->asoc_cap.nss_rx = pmlmeext->txss_bk;
+		rtw_phl_sta_assoc_cap_process(sta->phl_sta, _FALSE);
+		final_ss = sta->phl_sta->asoc_cap.nss_rx;
+	}
+	//RTW_INFO("%s, adapter=0x%08x, sta=0x%08x\n", __func__, adapter, sta);
+	RTW_INFO("%s, final_ss=%d, txss_bk=%d, nss_rx=%d", __func__, final_ss, pmlmeext->txss_bk, sta->phl_sta->asoc_cap.nss_rx);
+	return final_ss;
+}
+
+void rtw_ctrl_and_backup_assoc_cap_rx_nss(_adapter *adapter, struct sta_info *sta, u8 rx_nss)
+{
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+
+	pmlmeext->txss_bk = sta->phl_sta->asoc_cap.nss_rx;
+	rtw_phl_sta_assoc_cap_process(sta->phl_sta, _TRUE);
+
+	if (rx_nss == 1) {
+		sta->phl_sta->asoc_cap.nss_rx = 1;
+		sta->phl_sta->asoc_cap.stbc_ht_rx = 0;
+		sta->phl_sta->asoc_cap.stbc_vht_rx = 0;
+		sta->phl_sta->asoc_cap.stbc_he_rx = 0;
+	} else {
+		sta->phl_sta->asoc_cap.nss_rx = rx_nss;
+	}
+}
+
+#ifdef CONFIG_DBCC_SUPPORT
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+bool rtw_dbcc_b0_sta_chan_chk(struct _ADAPTER *a)
+{
+	bool rst = false;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	struct rtw_wifi_role_link_t *rlink = alink->wrlink;
+	struct rtw_chan_def *chdef = &rlink->chandef;
+	enum mr_op_mode cur_op_mode = MR_OP_MAX;
+
+	if (!rtw_is_adapter_up(a)) {
+		RTW_ERR(FUNC_ADPT_FMT " is inavailable ......\n",
+					FUNC_ADPT_ARG(a));
+		rtw_warn_on(1);
+		return rst;
+	}
+
+	if (!is_client_associated_to_ap(a))
+		return rst;
+
+	if (MLME_IS_GC(a))
+		return rst;
+
+	cur_op_mode = rtw_phl_mr_get_opmode(dvobj->phl, a->phl_role, rlink);
+	if (cur_op_mode != MR_OP_SWR)
+		return rst;
+
+	if (BAND_5GHZ(chdef->band)) {
+		if (CH_5GHZ_BAND2(chdef->chan) || CH_5GHZ_BAND3(chdef->chan))
+			rst = true;
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+		if (adapter_to_regsty(a)->dbcc_lg_sim && a->iface_id == 0)
+			rst = true;
+#endif
+
+	} else if (BAND_6GHZ(chdef->band)) {
+		/*do something*/
+		rst = true;
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+		if (adapter_to_regsty(a)->dbcc_lg_sim && a->iface_id == 0)
+			rst = true;
+#endif
+	}
+
+	return rst;
+}
+
+#define DBG_DBCC_LG_SCENARIO
+enum dbcc_rtype {
+	DBCC_RTYPE_STATION,
+	DBCC_RTYPE_P2P /*P2P_DEVICE, P2P_GO, P2P_GC*/
+};
+
+static struct _ADAPTER *
+_search_adapter_by_rtype(struct _ADAPTER *a, u8 band_idx, enum dbcc_rtype rtype)
+{
+	int i;
+	_adapter *iface = NULL;
+	_adapter *tg_apt = NULL;
+	struct _ADAPTER_LINK *tg_alink = NULL;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+
+	if (0 && (!rtw_is_adapter_up(a))) {
+		RTW_ERR(FUNC_ADPT_FMT " is inavailable ......\n",
+					FUNC_ADPT_ARG(a));
+		rtw_warn_on(1);
+		goto _exit;
+	}
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !rtw_is_adapter_up(iface))
+			continue;
+
+		tg_alink = GET_PRIMARY_LINK(iface);
+
+		if (rtype == DBCC_RTYPE_P2P) {
+			if((tg_alink->wrlink->hw_band == band_idx) &&
+			   (MLME_IS_PD(iface) || MLME_IS_GO(iface) ||   MLME_IS_GC(iface))) {
+			   tg_apt = iface;
+			   break;
+			   }
+			#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+			if (adapter_to_regsty(a)->dbcc_lg_sim) {
+				if ((iface != a) && (tg_alink->wrlink->hw_band == band_idx) && (iface->iface_id == 1)) {
+				       tg_apt = iface;
+				       break;
+			       }
+		    	}
+		    	#endif
+		}
+		else if (rtype == DBCC_RTYPE_STATION) {
+			if ((tg_alink->wrlink->hw_band == band_idx) && (MLME_IS_STA(iface))) {
+				tg_apt = iface;
+				break;
+			  }
+		}
+	}
+
+#ifdef DBG_DBCC_LG_SCENARIO
+	RTW_INFO("[%s]--$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\n", __func__);
+	if (tg_apt) {
+		if (rtype == DBCC_RTYPE_P2P) {
+			RTW_INFO(FUNC_ADPT_FMT " is P2P-Device at Band-%d......\n",
+				FUNC_ADPT_ARG(tg_apt), tg_alink->wrlink->hw_band);
+		} else if (rtype == DBCC_RTYPE_STATION) {
+			RTW_INFO(FUNC_ADPT_FMT " is STA at Band-%d......\n",
+				FUNC_ADPT_ARG(tg_apt), tg_alink->wrlink->hw_band);
+		}
+	} else {
+		if (rtype == DBCC_RTYPE_P2P)
+			RTW_ERR("%s cannot find P2P on HW_B(%d)\n", __func__, band_idx);
+		else if (rtype == DBCC_RTYPE_STATION)
+			RTW_ERR("%s cannot find STA on HW_B(%d)\n", __func__, band_idx);
+		rtw_warn_on(1);
+	}
+	RTW_INFO("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$--[%s]\n", __func__);
+#endif
+_exit:
+	return tg_apt;
+}
+
+bool rtw_dbcc_chk_enable_hdl(_adapter *adapter,
+					enum phl_cmd_type cmd_type,
+					enum dbcc_chk_pcd chk_pcd)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(adapter);
+	struct rtw_wifi_role_link_t *rlink = alink->wrlink;
+	bool enable_dbcc = false;
+
+	if (rtw_phl_mr_is_db(dvobj->phl))
+		return enable_dbcc;
+
+#ifdef DBG_DBCC_LG_SCENARIO
+RTW_INFO("[%s]-- $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\n", __func__);
+#endif
+	switch (chk_pcd) {
+	case DBCC_CHK_STA_CON:
+	case DBCC_CHK_AP_START:
+	{
+
+		#ifdef DBG_DBCC_LG_SCENARIO
+		if (chk_pcd == DBCC_CHK_STA_CON) {
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- STA_CONNCTION\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		} else {
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- START AP\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		}
+		#endif
+	}
+	break;
+
+	case DBCC_CHK_STA_DIS:
+	case DBCC_CHK_AP_STOP:
+	{
+		_adapter *sta_adp = NULL;
+
+		#ifdef DBG_DBCC_LG_SCENARIO
+		if (chk_pcd == DBCC_CHK_STA_DIS)
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- EVT: STA_DISCON\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		else
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- EVT: STOP AP\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		#endif
+		/*B0 GC,GO*/
+		if (((dvobj->iface_nums == 2) || (dvobj->iface_nums == 3)) &&
+		    (rlink->hw_band == HW_BAND_0) &&
+		    #ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+		    (adapter_to_regsty(adapter)->dbcc_lg_sim)
+		    #else
+		    (MLME_IS_PD(adapter) || MLME_IS_GO(adapter) || MLME_IS_GC(adapter))
+		    #endif
+		) {
+			sta_adp = _search_adapter_by_rtype(adapter, HW_BAND_0, DBCC_RTYPE_STATION);
+			if (sta_adp) {
+				if (rtw_dbcc_b0_sta_chan_chk(sta_adp)) {
+					enable_dbcc = true;
+					RTW_INFO("B0 STA is stady in DFS/6G CH\n");
+				}
+				else {
+					RTW_INFO("B0 STA is not stady in DFS/6G CH\n");
+				}
+			} else {
+				RTW_ERR("F-%s L-%d cannot find B0 STA\n", __FUNCTION__, __LINE__);
+			}
+		}
+	}
+	break;
+	}
+
+#ifdef DBG_DBCC_LG_SCENARIO
+	RTW_INFO("enable_dbcc:%d\n", enable_dbcc);
+	RTW_INFO("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ --[%s]\n", __func__);
+#endif
+	return enable_dbcc;
+}
+
+bool rtw_dbcc_chk_disable_hdl(_adapter *adapter,
+					struct rtw_chan_def *new_chdef,
+					enum phl_cmd_type cmd_type,
+					enum dbcc_chk_pcd chk_pcd)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(adapter);
+	struct rtw_wifi_role_link_t *rlink = alink->wrlink;
+
+	bool disable_dbcc = false;
+
+	if (!rtw_phl_mr_is_db(dvobj->phl))
+		return disable_dbcc;
+
+#ifdef DBG_DBCC_LG_SCENARIO
+RTW_INFO("[%s] -- $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\n", __func__);
+#endif
+	switch (chk_pcd) {
+	case DBCC_CHK_STA_CON:
+	case DBCC_CHK_AP_START:
+	{
+		struct rtw_mr_chctx_info mr_cc_info = {0};
+
+		disable_dbcc = false;
+#ifdef DBG_DBCC_LG_SCENARIO
+		if (chk_pcd == DBCC_CHK_STA_CON) {
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- STA_CONNCTION\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		} else {
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- START AP\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		}
+#endif
+
+		if ((dvobj->iface_nums == 2) && (rlink->hw_band == HW_BAND_1) &&
+			(MLME_IS_GO(adapter) || MLME_IS_GC(adapter))) {/*B1 GC,GO*/
+			if (rtw_phl_chanctx_chk_by_band(dvobj->phl, HW_BAND_0, new_chdef, &mr_cc_info)) {
+				/*SCC or MCC*/
+				rtw_phl_cmd_dbcc_disable(adapter->phl_role, rlink->hw_band, cmd_type, 0);
+				disable_dbcc = true;
+			}
+		}
+		else if ((dvobj->iface_nums == 3) && (rlink->hw_band == HW_BAND_0) &&
+			(MLME_IS_GO(adapter) || MLME_IS_GC(adapter)
+			#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+			|| (adapter_to_regsty(adapter)->dbcc_lg_sim)
+			#endif
+			)) {/*B0 GC,GO*/
+			if (rtw_phl_chanctx_chk_by_band(dvobj->phl, HW_BAND_0, new_chdef, &mr_cc_info)) {
+				_adapter *pd_adp = NULL;
+				struct _ADAPTER_LINK *pd_alink;
+				struct rtw_wifi_role_link_t *pd_rlink;
+
+				#ifdef DBG_DBCC_LG_SCENARIO
+				RTW_INFO("execute rtw_phl_cmd_dbcc_disable\n");
+				#endif
+				pd_adp = _search_adapter_by_rtype(adapter, HW_BAND_1, DBCC_RTYPE_P2P);
+				if (pd_adp) {
+					#ifdef DBG_DBCC_LG_SCENARIO
+					RTW_INFO("F-%s L-%d : Find B1 P2P-PD\n", __FUNCTION__, __LINE__);
+					#endif
+					pd_alink = GET_PRIMARY_LINK(pd_adp);
+					pd_rlink = pd_alink->wrlink;
+					rtw_phl_cmd_dbcc_disable(pd_adp->phl_role, pd_rlink->hw_band, cmd_type, 0);
+					disable_dbcc = true;
+				} else {
+					#ifdef DBG_DBCC_LG_SCENARIO
+					RTW_ERR("F-%s L-%d : Cannot find B1 P2P-PD\n", __FUNCTION__, __LINE__);
+					rtw_warn_on(1);
+					#endif
+				}
+			} else {
+				//pause  B0 GC,GO - trx
+				//reallocate B0 GC,GO to B1
+				#ifdef DBG_DBCC_LG_SCENARIO
+				RTW_INFO(FUNC_ADPT_FMT" on HW_B(%d)\n",
+					FUNC_ADPT_ARG(adapter),
+					rlink->hw_band);
+				#endif
+				rtw_phl_wifi_role_realloc_band(dvobj->phl, adapter->phl_role, rlink);
+				#ifdef DBG_DBCC_LG_SCENARIO
+				RTW_INFO("execute rtw_phl_wifi_role_realloc_band\n");
+				RTW_INFO(FUNC_ADPT_FMT" on HW_B(%d)\n",
+					FUNC_ADPT_ARG(adapter),
+					rlink->hw_band);
+				#endif
+			}
+		}
+	}
+		break;
+
+	case DBCC_CHK_STA_DIS:
+	case DBCC_CHK_AP_STOP:
+	{
+		_adapter *sta_adp = NULL;
+
+		disable_dbcc = true;
+
+		#ifdef DBG_DBCC_LG_SCENARIO
+		if (chk_pcd == DBCC_CHK_STA_DIS)
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- EVT: STA_DISCON\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		else
+			RTW_INFO(FUNC_ADPT_FMT" HW_B(%d)- EVT: STOP AP\n",
+				FUNC_ADPT_ARG(adapter),
+				rlink->hw_band);
+		#endif
+		/*B1 GC,GO*/
+		if (((dvobj->iface_nums == 2) || (dvobj->iface_nums == 3)) &&
+		    (rlink->hw_band == HW_BAND_1) &&
+		    (MLME_IS_PD(adapter) || MLME_IS_GO(adapter) || MLME_IS_GC(adapter)
+			#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+			|| (adapter_to_regsty(adapter)->dbcc_lg_sim)
+			#endif
+		    )
+		) {
+			sta_adp = _search_adapter_by_rtype(adapter, HW_BAND_0, DBCC_RTYPE_STATION);
+			if (sta_adp) {
+				if (rtw_dbcc_b0_sta_chan_chk(sta_adp)) {
+					disable_dbcc = false;
+					RTW_INFO("B0 STA is stady in DFS/6G CH\n");
+				}
+				else {
+					RTW_INFO("B0 STA is not stady in DFS/6G CH\n");
+				}
+			} else {
+				RTW_ERR("F-%s L-%d cannot find B0 STA\n", __FUNCTION__, __LINE__);
+			}
+
+		}
+	}
+		break;
+	}
+
+#ifdef DBG_DBCC_LG_SCENARIO
+	RTW_INFO("disable_dbcc:%d\n", disable_dbcc);
+	RTW_INFO("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$--[%s]\n", __func__);
+#endif
+
+	return disable_dbcc;
+}
+
+enum rtw_phl_status
+rtw_discon_end_dbcc_en_notify(struct _ADAPTER *a, enum phl_module_id mdl_id)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_wifi_role_t *wrole = a->phl_role;
+	enum rtw_phl_status status;
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, mdl_id);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_DISCONNECT_END_DBCC_EN);
+
+	if (mdl_id == PHL_FG_MDL_DISCONNECT)
+		msg.band_idx = a->disconnect_bidx;
+	else if (mdl_id == PHL_FG_MDL_AP_STOP)
+		msg.band_idx = a->ap_stop_cmd_bidx;
+
+	msg.inbuf = (u8 *)wrole;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d), &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR(FUNC_ADPT_FMT ": send MSG_EVT_DISCONNECT_END_DBCC_EN fail(0x%x)!\n",
+			FUNC_ADPT_ARG(a), status);
+
+	return status;
+}
+#endif /*CONFIG_DBCC_P2P_BG_LISTEN*/
+static void _dbcc_proto_go(_adapter *adapter, u8 dbcc_en)
+{
+	struct sta_priv *stapriv = NULL;
+	struct sta_info *sta = NULL;
+	int i, stainfo_offset;
+	_list	*plist, *phead;
+	u8 chk_num = 0;
+	u8 chk_list[NUM_STA];
+	struct _ADAPTER_LINK *a_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_priv *mlmepriv = &(a_link->mlmepriv);
+	struct link_mlme_ext_priv *mlmeext = &(a_link->mlmeextpriv);
+	struct link_mlme_ext_info *mlmeinfo = &(mlmeext->mlmext_info);
+	WLAN_BSSID_EX *network = &(mlmeinfo->network);
+	struct HT_caps_element *ht_caps;
+	u8 *p, *ie = network->IEs;
+	u32 len = 0;
+
+	stapriv = &adapter->stapriv;
+
+	if (!MLME_IS_GO(adapter))
+		return;
+
+	if (!mlmepriv->htpriv.ht_option)
+			return;
+
+	if (!mlmeinfo->HT_caps_enable)
+			return;
+
+	rtw_stapriv_asoc_list_lock(stapriv);
+
+	phead = &stapriv->asoc_list;
+	plist = get_next(phead);
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		sta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+
+		stainfo_offset = rtw_stainfo_offset(stapriv, sta);
+		if (stainfo_offset_valid(stainfo_offset))
+			chk_list[chk_num++] = stainfo_offset;
+		continue;
+	}
+
+	rtw_stapriv_asoc_list_unlock(stapriv);
+
+
+	/* issue SMPS */
+	if (chk_num) {
+		for (i = 0; i < chk_num; i++) {
+			sta = rtw_get_stainfo_by_offset(stapriv, chk_list[i]);
+			if (dbcc_en)
+				rtw_ssmps_enter(adapter, sta);
+			else
+				rtw_ssmps_leave(adapter, sta);
+		}
+	}
+
+	/* update BCN */
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_CAPABILITY_IE_, &len, (network->IELength - _BEACON_IE_OFFSET_));
+	if (p && len > 0) {
+		ht_caps = (struct HT_caps_element *)(p + 2);
+		RTW_INFO("[DBCC] orig ht caps info = 0x%02x\n", ht_caps->u.HT_cap_element.HT_caps_info);
+
+		if (dbcc_en)
+			SET_HT_CAP_ELE_SM_PS(&(ht_caps->u.HT_cap_element.HT_caps_info), SM_PS_STATIC);
+		else
+			SET_HT_CAP_ELE_SM_PS(&(ht_caps->u.HT_cap_element.HT_caps_info), SM_PS_DISABLE);
+
+		RTW_INFO("[DBCC] modify ht caps info = 0x%02x\n", ht_caps->u.HT_cap_element.HT_caps_info);
+		rtw_update_beacon(adapter, a_link, _HT_CAPABILITY_IE_, NULL, _TRUE, RTW_CMDF_DIRECTLY);
+	}
+}
+
+static u8 _rtw_dbcc_proto_hdl(void *drv_priv, u8 wr_idx, void *data)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+	_adapter *adapter;
+	struct mlme_priv *pmlmepriv = NULL;
+	struct sta_priv *pstapriv = NULL;
+	struct sta_info *psta = NULL;
+	struct dbcc_proto_msg *dbcc_proto = NULL;
+	struct _ADAPTER_LINK *alink = NULL;
+	u8 rst = _FAIL;
+	u8 final_ss = 0;
+
+	adapter = dvobj->padapters[wr_idx];
+	if (!rtw_is_adapter_up(adapter)) {
+		RTW_ERR("if%d mac_addr : "MAC_FMT" not active\n", wr_idx, MAC_ARG(adapter_mac_addr(adapter)));
+		goto _exit;
+	}
+
+	if (!rtw_hw_is_mimo_support(adapter)) {
+		RTW_ERR("if%d mac_addr : "MAC_FMT" not support mimo\n", wr_idx, MAC_ARG(adapter_mac_addr(adapter)));
+		goto _exit;
+	}
+
+	alink = GET_PRIMARY_LINK(adapter);
+	pmlmepriv = &adapter->mlmepriv;
+	dbcc_proto = (struct dbcc_proto_msg *)data;
+
+	if (dbcc_proto->dbcc_en) {
+		if (dbcc_proto->wr->id == wr_idx)
+			goto _exit;
+	}
+
+	if (is_client_associated_to_ap(adapter)) {
+	/*if ((MLME_IS_STA(adapter) || MLME_IS_GC(adapter)) && MLME_IS_ASOC(adapter)) {*/
+		RTW_INFO(ADPT_FMT" associated to AP\n", ADPT_ARG(adapter));
+		/*if (!pmlmeext->ssmps_en)*/
+		pstapriv = &adapter->stapriv;
+		psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+		if (psta == NULL) {
+			RTW_ERR(ADPT_FMT" sta == NULL\n", ADPT_ARG(adapter));
+			rtw_warn_on(1);
+			goto _exit;
+		}
+		/* N mode - smps*/
+
+		if (psta->phl_sta->wmode & WLAN_MD_11AX) {
+			if (dbcc_proto->dbcc_en)
+				final_ss = rtw_backup_and_get_final_ss(adapter, psta, 1);
+			else
+				final_ss = rtw_backup_and_get_final_ss(adapter, psta, 0);
+			rtw_he_om_ctrl_trx_ss(adapter, alink, psta, final_ss, _TRUE);
+		} else if (psta->phl_sta->wmode & WLAN_MD_11AC) {
+			if (dbcc_proto->dbcc_en)
+				final_ss = rtw_backup_and_get_final_ss(adapter, psta, 1);
+			else
+				final_ss = rtw_backup_and_get_final_ss(adapter, psta, 0);
+			rtw_vht_op_mode_ctrl_rx_nss(adapter, alink, psta, final_ss, _TRUE);
+		} else if (psta->phl_sta->wmode & WLAN_MD_11N) {
+			if (dbcc_proto->dbcc_en)
+				rtw_ssmps_enter(adapter, psta);
+			else
+				rtw_ssmps_leave(adapter, psta);
+		}
+	}
+	else if (MLME_IS_AP(adapter) || MLME_IS_GO(adapter) || MLME_IS_ADHOC_MASTER(adapter)) {
+		RTW_INFO(ADPT_FMT" is AP\n", ADPT_ARG(adapter));
+		if (MLME_IS_GO(adapter)) {
+			_dbcc_proto_go(adapter, dbcc_proto->dbcc_en);
+		}
+		else if (MLME_IS_AP(adapter)) {
+			/* Rx cap chang - Protocol*/
+
+			/* re-send beacon*/
+			if (dbcc_proto->dbcc_en == _FALSE) {
+				RTW_INFO(ADPT_FMT" send_beacon\n", ADPT_ARG(adapter));
+				if(send_beacon(adapter) == _FAIL)
+					RTW_ERR(ADPT_FMT" send_beacon, fail!\n", ADPT_ARG(adapter));
+			}
+		}
+		else {
+			RTW_INFO(ADPT_FMT" not support dbcc yet\n", ADPT_ARG(adapter));
+		}
+	}
+	rst = _SUCCESS;
+
+_exit:
+	return rst;
+}
+
+u8 core_dbcc_protocol_hdl(void *drv, enum phl_band_idx band_idx,
+					struct dbcc_proto_msg *dbcc_proto)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv;
+	struct rtw_wifi_role_t *wrole = NULL;
+	_adapter *adapter = NULL;
+
+	wrole = dbcc_proto->wr;
+	if (wrole == NULL) {
+		rtw_warn_on(1);
+		RTW_ERR("%s role == NULL\n", __func__);
+		return _FAIL;
+	}
+	adapter = dvobj->padapters[wrole->id];
+
+
+	if (dbcc_proto->dbcc_en) {
+		#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+		_adapter *sta_adp = NULL;
+		#endif
+
+		RTW_INFO(FUNC_ADPT_FMT " DBCC enable ......\n",
+				FUNC_ADPT_ARG(adapter));
+		/* loop adapter
+		* 1. change Rx cap
+		* issue_smps ,issue_action_omi , issue_action_om - issue(wait_ack)
+		* monitor rx-rate if change to 1ss (wait_ack)
+		* 2. change Tx cap
+		* Reregister RA to 1x1 - general mdl
+		*/
+		rtw_phl_mr_process_by_band(dvobj->phl, HW_BAND_0, dbcc_proto,
+							_rtw_dbcc_proto_hdl);
+		#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+		sta_adp = _search_adapter_by_rtype(adapter, HW_BAND_0, DBCC_RTYPE_STATION);
+
+		if (sta_adp && rtw_dbcc_b0_sta_chan_chk(sta_adp)) {
+			_adapter *pd_adp = _search_adapter_by_rtype(adapter, HW_BAND_0, DBCC_RTYPE_P2P);
+
+			if (pd_adp) {
+				wrole = pd_adp->phl_role;
+				#ifdef DBG_DBCC_LG_SCENARIO
+				RTW_INFO("F-%s L-%d : Find B0 P2P\n", __FUNCTION__, __LINE__);
+				#endif
+			} else {
+				#ifdef DBG_DBCC_LG_SCENARIO
+				RTW_ERR("F-%s L-%d : Cannot find B0 P2P\n", __FUNCTION__, __LINE__);
+				rtw_warn_on(1);
+				#endif
+			}
+		}
+
+		if (!dbcc_proto->dbcc_cmd_direct)
+		#endif
+		rtw_phl_mr_dbcc_enable(dvobj->phl, band_idx, wrole);
+	}
+	else { /*dbcc disable*/
+		RTW_INFO(FUNC_ADPT_FMT " DBCC disable ......\n",
+				FUNC_ADPT_ARG(adapter));
+		/* loop adapter
+		* 1. change Rx cap
+		* issue_smps ,issue_action_omi , issue_action_om - issue(wait_ack)
+		* monitor rx-rate if change to 2ss (wait_ack)
+		* 2. change Tx cap
+		* Reregister RA to 2x2 - general mdl
+		*/
+		rtw_phl_mr_process_by_band(dvobj->phl, HW_BAND_0, dbcc_proto,
+							_rtw_dbcc_proto_hdl);
+		#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+		if (!dbcc_proto->dbcc_cmd_direct)
+		#endif
+		rtw_phl_mr_dbcc_disable(dvobj->phl, band_idx, wrole);
+	}
+	return _SUCCESS;
+}
+#endif /*CONFIG_DBCC_SUPPORT*/
+
+#ifdef CONFIG_DFS_MASTER
+static void phl_radar_detect_msg_hdl(struct dvobj_priv *dvobj, struct phl_msg *msg)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+
+	if (rfctl->radar_detect_enabled) {
+		/* TODO: PHL info for specific band and range */
+		enum phl_band_idx band_idx = HW_BAND_0;
+		u8 radar_cch = 0;
+		enum channel_width radar_bw = CHANNEL_WIDTH_MAX;
+
+		band_idx = rfctl->radar_detect_hwband;
+		radar_cch = rfctl->radar_detect_cch[band_idx];
+		radar_bw = rfctl->radar_detect_bw[band_idx];
+		rtw_dfs_rd_hdl(dvobj, band_idx, radar_cch, radar_bw);
+	}
+}
+#endif /* CONFIG_DFS_MASTER */
+
+/* handling msg hub event with PHL_MDL_POWER_MGNT as module id */
+static void rtw_msg_hub_power_mgnt_evt_hdlr(struct dvobj_priv *dvobj, u16 evt_id, u8 *buf, u32 len, enum rtw_rf_state *ori_rf_state)
+{
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+	struct wiphy *wiphy = adapter_to_wiphy(adapter);
+	enum rtw_rf_state *new_rf_state = NULL;
+
+	if (buf == NULL) {
+		RTW_WARN("%s : Msg info buffer is NULL!\n", __func__);
+		return;
+	}
+
+	if (len != sizeof(*new_rf_state)) {
+		RTW_WARN("%s : Msg info len does not match!\n", __func__);
+		return;
+	}
+
+	new_rf_state = (enum rtw_rf_state *)buf;
+
+	switch (evt_id) {
+	case MSG_EVT_HW_RF_CHG:
+		if (*ori_rf_state != *new_rf_state) {
+			rtw_wiphy_rfkill_set_hw_state(wiphy, (*new_rf_state == RTW_RF_OFF ? 1 : 0));
+			RTW_INFO("%s : [%s] evt_id %d.\n", __func__, (*new_rf_state == RTW_RF_OFF ? "RTW_RF_OFF" : "RTW_RF_ON"), evt_id);
+			*ori_rf_state = *new_rf_state;
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+void core_handler_phl_msg(void *drv_priv, struct phl_msg *msg)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+	u8 mdl_id = MSG_MDL_ID_FIELD(msg->msg_id);
+	u16 evt_id = MSG_EVT_ID_FIELD(msg->msg_id);
+
+	switch(mdl_id) {
+	case PHL_MDL_RX:
+	{
+		_adapter *iface;
+		u8 i = 0;
+
+		if (evt_id == MSG_EVT_BCN_RESEND) {
+			for (i = 0; i < dvobj->iface_nums; i++) {
+				iface = dvobj->padapters[i];
+				if(!rtw_is_adapter_up(iface))
+					continue;
+				/*
+				if (!rtw_iface_at_same_hwband(padapter, iface))
+					continue;
+				*/
+				if(MLME_IS_MESH(iface)
+					|| MLME_IS_AP(iface)
+					|| MLME_IS_ADHOC_MASTER(iface)) {
+					if (send_beacon(iface) == _FAIL)
+						RTW_ERR(ADPT_FMT" issue_beacon, fail!\n",
+									ADPT_ARG(iface));
+				}
+			}
+		}
+		#ifdef CONFIG_XMIT_ACK_BY_CCX_RPT
+		else if (evt_id == MSG_EVT_CCX_REPORT_TX_OK || evt_id == MSG_EVT_CCX_REPORT_TX_FAIL) {
+			struct rtw_tx_pkt_rpt *tx_rpt = NULL;
+			struct xmit_priv *pxmitpriv;
+
+			tx_rpt = (struct rtw_tx_pkt_rpt *)msg->inbuf;
+			if (tx_rpt == NULL) {
+				RTW_ERR("%s get tx_rpt failed\n", __func__);
+				rtw_warn_on(1);
+				break;
+			}
+
+			iface = dvobj_get_primary_adapter(dvobj);
+			pxmitpriv = &iface->xmitpriv;
+
+			if ((pxmitpriv->ack_tx_seq_no & 0x0F) != tx_rpt->seq_no) {
+				RTW_ERR("%s seq_no-(%d)%d:%d missmatch\n", __func__,
+					pxmitpriv->ack_tx_seq_no,
+					(pxmitpriv->ack_tx_seq_no & 0x0F),
+					tx_rpt->seq_no);
+				RTW_INFO("[XMIT_ACK] ack_tx_seq_no:%d\n", pxmitpriv->ack_tx_seq_no);
+				RTW_INFO("[XMIT_ACK] mac_id:%d seq_no:%d\n", tx_rpt->macid, tx_rpt->seq_no);
+			}
+
+			if (evt_id == MSG_EVT_CCX_REPORT_TX_OK)
+				rtw_ack_tx_done(&iface->xmitpriv, RTW_SCTX_DONE_SUCCESS);
+			else if (evt_id == MSG_EVT_CCX_REPORT_TX_FAIL)
+				rtw_ack_tx_done(&iface->xmitpriv, RTW_SCTX_DONE_CCX_PKT_FAIL);
+		}
+		#endif
+		#ifdef CONFIG_DFS_MASTER
+		else if (evt_id == MSG_EVT_DFS_RD_IS_DETECTING)
+			phl_radar_detect_msg_hdl(dvobj, msg);
+		#endif
+		#ifdef CONFIG_TDLS
+		#ifdef CONFIG_TDLS_CH_SW
+		else if (evt_id == MSG_EVT_BCN_EARLY_REPORT) {
+			struct rtw_wifi_role_t *wrole = NULL;
+			struct rtw_bcn_early_rpt *bcn_rpt = NULL;
+			bcn_rpt = (struct rtw_bcn_early_rpt *)msg->inbuf;
+			if (bcn_rpt == NULL) {
+				RTW_ERR("%s get bcn_rpt failed\n", __func__);
+				rtw_warn_on(1);
+				break;
+			}
+			wrole = rtw_phl_get_role_by_band_port(GET_PHL_INFO(dvobj), bcn_rpt->band, bcn_rpt->port);
+			if (wrole) {
+				iface = dvobj->padapters[wrole->id];
+				if (ATOMIC_READ(&iface->tdlsinfo.chsw_info.chsw_on) == _TRUE) {
+					iface->tdlsinfo.chsw_info.ch_sw_state |= TDLS_WAIT_CH_SW_LAUNCH_STATE;
+					rtw_tdls_ch_sw_back_to_base_chnl(iface);
+				}
+			}
+		}
+		#endif
+		#endif
+	}
+	break;
+	case PHL_MDL_SER:
+	{
+		if (evt_id == MSG_EVT_SER_L2)
+			RTW_ERR("RECV PHL MSG_EVT_SER_L2\n");
+	}
+	break;
+	case PHL_MDL_WOW:
+	{
+		/*if (evt_id == MSG_EVT_WOW_WAKE_RSN)
+			RTW_INFO("[wow] wake rsn : %d\n", msg->inbuf[0]);*/
+	}
+	break;
+	case PHL_MDL_MRC:
+	{
+
+	}
+	break;
+	case PHL_MDL_POWER_MGNT:
+	{
+		_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+		struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(adapter);
+		rtw_msg_hub_power_mgnt_evt_hdlr(dvobj, evt_id, msg->inbuf, msg->inlen, &pwrpriv->rfkill_state);
+	}
+		break;
+
+case PHL_MDL_GENERAL:
+#ifdef CONFIG_BTC
+#ifdef CONFIG_BTC_TRXSS_CHG
+		if ((evt_id == MSG_EVT_ANN_RX1SS) || (evt_id == MSG_EVT_ANN_RX_MAXSS))
+			rtw_btc_trxss_chg_hdl(dvobj, msg, evt_id);
+#endif
+#endif
+		break;
+
+	default:
+		RTW_ERR("%s mdl_id :%d not support\n", __func__, mdl_id);
+		break;
+	}
+}
+
+u8 rtw_core_register_phl_msg(struct dvobj_priv *dvobj)
+{
+	struct phl_msg_receiver ctx = {0};
+	u8 imr[] = {PHL_MDL_RX, PHL_MDL_SER, PHL_MDL_WOW, PHL_MDL_MRC, PHL_MDL_POWER_MGNT, PHL_MDL_GENERAL};
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+
+	ctx.incoming_evt_notify = core_handler_phl_msg;
+	ctx.priv = (void*)dvobj;
+
+	psts = rtw_phl_msg_hub_register_recver(dvobj->phl, &ctx, MSG_RECV_CORE);
+	if(psts	== RTW_PHL_STATUS_FAILURE) {
+		RTW_ERR("phl_msg_hub_register failed\n");
+		return _FAIL;
+	}
+
+	psts = rtw_phl_msg_hub_update_recver_mask(dvobj->phl,
+					MSG_RECV_CORE, imr, sizeof(imr), false);
+	if(psts	== RTW_PHL_STATUS_FAILURE) {
+		RTW_ERR("phl_msg_hub_update_recver_mask failed\n");
+		return _FAIL;
+	}
+	return _SUCCESS;
+}
+
+/*RTW_WKARD_CORE_RSSI_V1*/
+s8 rtw_phl_rssi_to_dbm(u8 rssi)
+{
+	return rssi - PHL_MAX_RSSI;
+}
+
+
+#ifdef CONFIG_MCC_MODE
+u8 rtw_hw_mcc_chk_inprogress(struct _ADAPTER *a, struct _ADAPTER_LINK *adapter_link)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	u8 ret = _FALSE;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		goto exit;
+
+	ret = rtw_phl_mr_coex_query_inprogress(phl, adapter_link->wrlink->hw_band, RTW_MR_COEX_CHK_INPROGRESS);
+
+exit:
+	return ret;
+}
+
+#ifdef CONFIG_P2P_PS
+static int _mcc_update_noa(void *priv, struct rtw_phl_mcc_noa *param)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *) priv;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct _ADAPTER *a = NULL;
+	struct wifidirect_info *wdinfo;
+	u8 id = 0;
+	u8 ret = _SUCCESS;
+	u8 noa_en = 0;
+#ifdef CONFIG_PHL_P2PPS
+	struct rtw_phl_noa_desc desc= {0};
+#endif
+	struct _ADAPTER_LINK *alink;
+
+	wrole = param->wrole;
+	if (wrole == NULL) {
+		RTW_ERR("%s wrole is NULL\n", __func__);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	id = wrole->id;
+	if (id >= CONFIG_IFACE_NUMBER) {
+		RTW_ERR("%s error id (%d)\n", __func__, id);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	a = dvobj->padapters[id];
+	if (a == NULL) {
+		RTW_ERR("%s adapter(%d) is NULL\n", __func__, id);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	/* by pass non-GO case */
+	if (!MLME_IS_GO(a))
+		goto exit;
+
+	wdinfo = &a->wdinfo;
+	RTW_INFO(FUNC_ADPT_FMT":(%d)\n", FUNC_ADPT_ARG(a), id);
+	RTW_INFO("start_t_h=0x%02x,start_t_l=0x%02x\n", param->start_t_h, param->start_t_l);
+	RTW_INFO("dur=0x%d,cnt=0x%d,interval=0x%d\n", param->dur, param->cnt, param->interval);
+	/* cnt will be 0, if disable NoA */
+	if (param->cnt == 0)
+		noa_en = _FALSE;
+	else
+		noa_en = _TRUE;
+
+#ifdef CONFIG_PHL_P2PPS
+	/* enable TSF32 toggle */
+	desc.tag = P2PPS_TRIG_MCC;
+	if (noa_en)
+		desc.enable = _TRUE;
+	else
+		desc.enable = _FALSE;
+	desc.duration = param->dur * NET80211_TU_TO_US;
+	desc.interval = param->interval * NET80211_TU_TO_US;
+	desc.start_t_h = param->start_t_h;
+	desc.start_t_l = param->start_t_l;
+	desc.count = param->cnt;
+	desc.w_role = param->wrole;
+	desc.rlink = param->rlink;
+	if (rtw_phl_p2pps_noa_update(dvobj->phl, &desc) != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s rtw_phl_p2pps_noa_update fail\n", __func__);
+		ret = _FAIL;
+		goto exit;
+	}
+#endif
+
+	/* update NoA IE */
+	wdinfo->noa_index = wdinfo->noa_index + 1;
+	if (noa_en)
+		wdinfo->noa_num = 1;
+	else
+		wdinfo->noa_num = 0;
+	wdinfo->noa_count[0] = param->cnt;
+	wdinfo->noa_duration[0] =param->dur * NET80211_TU_TO_US;
+	wdinfo->noa_interval[0] = param->interval * NET80211_TU_TO_US;
+	wdinfo->noa_start_time[0] = param->start_t_l;
+
+	alink = GET_PRIMARY_LINK(a);
+	rtw_update_beacon(a, alink, _VENDOR_SPECIFIC_IE_, P2P_OUI, _TRUE, RTW_CMDF_DIRECTLY);
+exit:
+	return ret;
+}
+#endif
+
+/* default setting  */
+static int _mcc_get_setting(void *priv, struct rtw_phl_mcc_setting_info *param)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *) priv;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct _ADAPTER *a = NULL;
+	struct wifidirect_info *wdinfo;
+	u8 id = 0;
+	u8 ret = _SUCCESS;
+
+	wrole = param->wrole;
+	if (wrole == NULL) {
+		RTW_ERR("%s wrole is NULL\n", __func__);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	id = wrole->id;
+	if (id >= CONFIG_IFACE_NUMBER) {
+		RTW_ERR("%s error id (%d)\n", __func__, id);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	a = dvobj->padapters[id];
+	if (a == NULL) {
+		RTW_ERR("%s adapter(%d) is NULL\n", __func__, id);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	if (MLME_IS_GO(a) || MLME_IS_GC(a))
+		param->dur = 50;
+	else
+		param->dur = 50;
+
+	if (MLME_IS_STA(a) || MLME_IS_GC(a))
+		param->tx_null_early = 5;
+	else
+		param->tx_null_early = NONSPECIFIC_SETTING;
+
+	RTW_INFO("%s: adapter(%d) dur=%d, tx_null_early=%d\n", __func__, id, param->dur, param->tx_null_early);
+
+exit:
+	return ret;
+}
+
+struct rtw_phl_mcc_ops rtw_mcc_ops = {
+	.priv = NULL,
+	.mcc_update_noa = _mcc_update_noa,
+	.mcc_get_setting = _mcc_get_setting,
+};
+#endif
+
+struct rtw_phl_mr_ops rtw_mr_ops = {
+/*
+#ifdef CONFIG_PHL_P2PPS
+	.phl_mr_update_noa = &dummy_p2p_noa;
+#endif
+*/
+#ifdef CONFIG_MCC_MODE
+	.mcc_ops = &rtw_mcc_ops,
+#endif
+
+#ifdef CONFIG_DBCC_SUPPORT
+	.dbcc_protocol_hdl = core_dbcc_protocol_hdl,
+#endif
+};
+
+void rtw_core_register_mr_config(struct dvobj_priv *dvobj)
+{
+#ifdef CONFIG_MCC_MODE
+	rtw_mr_ops.mcc_ops->priv = (void *)dvobj;
+#endif
+	rtw_phl_mr_ops_init(dvobj->phl, &rtw_mr_ops);
+}
+
+#ifdef CONFIG_ECSA_PHL
+static void rtw_core_set_ecsa_ops(struct dvobj_priv *d)
+{
+	struct rtw_phl_ecsa_ops ops = {0};
+
+	ops.priv = (void *)d;
+	ops.update_beacon = rtw_ecsa_update_beacon;
+	ops.update_chan_info = rtw_ecsa_mr_update_chan_info_by_role;
+	ops.check_ecsa_allow = rtw_ap_check_ecsa_allow;
+	ops.ecsa_complete = rtw_ecsa_complete;
+	ops.check_tx_resume_allow = rtw_ecsa_check_tx_resume_allow;
+	rtw_phl_ecsa_init_ops(GET_PHL_INFO(d), &ops);
+}
+#endif
+
+static void rtw_core_tx_power_tbl_loaded(void *drv_priv, bool target_loaded, bool limit_loaded)
+{
+	struct dvobj_priv * dvobj = drv_priv;
+
+	RTW_INFO("%s target=%d limit=%d\n", __func__, target_loaded, limit_loaded);
+
+	if (target_loaded)
+		;
+#if CONFIG_TXPWR_LIMIT
+	if (limit_loaded)
+		rtw_txpwr_update_cur_lmt_regs(dvobj, true);
+#endif
+}
+
+#ifdef CONFIG_TDLS
+static void rtw_core_set_tdls_ops(struct dvobj_priv *d)
+{
+	struct rtw_phl_tdls_ops ops = {0};
+
+	ops.priv = (void *)d;
+	ops.check_tdls_frame = rtw_check_tdls_frame;
+	rtw_phl_tdls_init_ops(GET_PHL_INFO(d), &ops);
+}
+#endif
+
+static void rtw_core_set_phl_ops(struct dvobj_priv *dvobj)
+{
+	struct rtw_phl_evt_ops *evt_ops;
+
+	evt_ops = &(dvobj->phl_com->evt_ops);
+
+	evt_ops->rx_process = rtw_core_rx_process;
+	evt_ops->tx_recycle = rtw_core_tx_recycle;
+#ifdef CONFIG_WOWLAN
+	evt_ops->indicate_wake_rsn = rtw_core_wow_handle_wake_up_rsn;
+#ifdef CONFIG_GTK_OL
+	evt_ops->wow_handle_sec_info_update = rtw_update_gtk_ofld_info;
+#endif
+#endif /* CONFIG_WOWLAN */
+	#ifdef CONFIG_RTW_IPS
+	evt_ops->set_rf_state = rtw_core_set_ips_state;
+	#endif
+	evt_ops->issue_null_data = rtw_core_issu_null_data;
+	#ifdef CONFIG_PCI_HCI
+	evt_ops->os_query_local_buf = rtw_os_query_local_buf;
+	evt_ops->os_return_local_buf = rtw_os_return_local_buf;
+	#endif
+
+	evt_ops->tx_power_tbl_loaded = rtw_core_tx_power_tbl_loaded;
+
+	#if 0 /*TBD*/
+	rtw_phl_snd_init_ops_send_ndpa(dvobj->phl,
+				       rtw_core_beamform_send_ndpa);
+	evt_ops->ap_ps_sta_ps_change = rtw_core_ap_sta_ps_chg;
+	#endif
+}
+
+u8 rtw_hw_init(struct dvobj_priv *dvobj)
+{
+	u8 rst = _FAIL;
+	enum rtw_phl_status phl_status;
+	struct rtw_ic_info ic_info;
+
+#ifdef DBG_PHL_MEM_ALLOC
+	ATOMIC_SET(&dvobj->phl_mem, 0);
+#endif
+
+	_hw_ic_info_cfg(dvobj, &ic_info);
+	phl_status = rtw_phl_init(dvobj, &(dvobj->phl), &ic_info);
+
+	if ((phl_status != RTW_PHL_STATUS_SUCCESS) || (dvobj->phl == NULL)) {
+		RTW_ERR("%s - rtw_phl_init failed status(%d), dvobj->phl(%p)\n",
+			__func__, phl_status, dvobj->phl);
+		goto _free_hal;
+	}
+
+	dvobj->phl_com = rtw_phl_get_com(dvobj->phl);
+
+	/*init sw cap from registary*/
+	rtw_core_update_default_setting(dvobj);
+
+	/* sw & hw cap*/
+	rtw_phl_cap_pre_config(dvobj->phl);
+
+	#ifdef CONFIG_RX_PSTS_PER_PKT
+	rtw_phl_init_ppdu_sts_para(dvobj->phl_com,
+		_TRUE, _FALSE,
+		RTW_PHL_PSTS_FLTR_MGNT | RTW_PHL_PSTS_FLTR_DATA /*| RTW_PHL_PSTS_FLTR_CTRL*/
+		);
+	#endif
+	/*init datapath section*/
+	rtw_phl_trx_alloc(dvobj->phl);
+	rtw_core_set_phl_ops(dvobj);
+
+	rtw_core_register_phl_msg(dvobj);
+
+	/* load wifi feature or capability from efuse*/
+	phl_status = rtw_phl_preload(dvobj->phl);
+	if (phl_status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s - rtw_phl_preload from efuse failed\n", __func__);
+		goto _free_hal;
+	}
+
+	rtw_phl_final_cap_decision(dvobj->phl);
+
+	/* after final cap decision */
+	rtw_core_register_mr_config(dvobj);
+#ifdef CONFIG_P2P_PS
+	rtw_core_register_p2pps_ops(dvobj);
+#endif
+	#ifdef CONFIG_ECSA_PHL
+	rtw_core_set_ecsa_ops(dvobj);
+	#endif
+	#ifdef CONFIG_TDLS
+	rtw_core_set_tdls_ops(dvobj);
+	#endif
+
+	rtw_dump_rfe_type(dvobj);
+
+	rtw_phl_watchdog_init(dvobj->phl,
+				0,
+				rtw_core_watchdog_sw_hdlr,
+				#ifdef CONFIG_POST_CORE_KEEP_ALIVE
+				rtw_core_keep_alive_post_hdlr,
+				#endif
+				rtw_core_watchdog_hw_hdlr,
+				rtw_core_watchdog_sw_post_hdlr);
+
+	rst = _SUCCESS;
+	return rst;
+
+_free_hal :
+	rtw_hw_deinit(dvobj);
+	return rst;
+}
+
+u8 rtw_hw_start(struct dvobj_priv *dvobj)
+{
+	if (dev_is_hw_start(dvobj))
+		return _FAIL;
+
+	if (rtw_phl_start(GET_PHL_INFO(dvobj)) != RTW_PHL_STATUS_SUCCESS)
+		return _FAIL;
+
+	rtw_phl_watchdog_start(dvobj->phl);
+
+	dev_set_hw_start(dvobj);
+
+	return _SUCCESS;
+}
+void rtw_hw_stop(struct dvobj_priv *dvobj)
+{
+	if (!dev_is_hw_start(dvobj))
+		return;
+
+	rtw_phl_watchdog_stop(dvobj->phl);
+
+	rtw_phl_stop(GET_PHL_INFO(dvobj));
+
+	dev_clr_hw_start(dvobj);
+}
+
+bool rtw_hw_get_init_completed(struct dvobj_priv *dvobj)
+{
+	return rtw_phl_is_init_completed(GET_PHL_INFO(dvobj));
+}
+
+bool rtw_hw_is_init_completed(struct dvobj_priv *dvobj)
+{
+	return (rtw_phl_is_init_completed(GET_PHL_INFO(dvobj))) ? _TRUE : _FALSE;
+}
+
+#define NSS_VALID(nss) (nss > 0)
+void rtw_hw_cap_init(struct dvobj_priv *dvobj)
+{
+	struct hal_spec_t *hal_spec = GET_HAL_SPEC(dvobj);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	struct phy_cap_t *phy_cap = phl_com->phy_cap;
+	struct registry_priv  *regpriv =
+		&(dvobj_get_primary_adapter(dvobj)->registrypriv);
+
+#ifdef DIRTY_FOR_WORK
+	dvobj->phl_com->rf_path_num = hal_spec->rf_reg_path_num; /*GET_HAL_RFPATH_NUM*/
+	dvobj->phl_com->rf_type = RF_2T2R; /*GET_HAL_RFPATH*/
+#endif
+}
+
+
+/*
+ * _ch_offset_drv2phl() - Convert driver channel offset to PHL type
+ * @ch_offset:	channel offset, ref: HAL_PRIME_CHNL_OFFSET_*
+ *
+ * Return PHL channel offset type "enum chan_offset"
+ */
+static enum chan_offset _ch_offset_drv2phl(u8 ch_offset)
+{
+	if (ch_offset == CHAN_OFFSET_UPPER)
+		return CHAN_OFFSET_UPPER;
+	if (ch_offset == CHAN_OFFSET_LOWER)
+		return CHAN_OFFSET_LOWER;
+
+	return CHAN_OFFSET_NO_EXT;
+}
+
+/*
+ * rtw_hw_set_ch_bw() - Set channel, bandwidth and channel offset
+ * @a:		pointer of struct _ADAPTER
+ * @band:	wifi band
+ * @ch:		channel
+ * @bw:		bandwidth
+ * @offset:	channel offset, ref: HAL_PRIME_CHNL_OFFSET_*
+ *
+ * Set channel, bandwidth and channel offset.
+ *
+ * Return 0 for success, otherwise fail
+ */
+int rtw_hw_set_ch_bw(struct _ADAPTER *a, struct _ADAPTER_LINK *alink,
+			enum band_type band, u8 ch, enum channel_width bw, u8 offset,
+			enum rfk_tri_type rt_type)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	int err = 0;
+	struct rtw_chan_def chdef = {0};
+	enum phl_cmd_type cmd_type = PHL_CMD_DIRECTLY;
+	u32 cmd_timeout = 0;
+
+#ifdef CONFIG_MCC_MODE
+	if (rtw_hw_mcc_chk_inprogress(a, alink)) {
+		RTW_WARN("under mcc, skip ch setting\n");
+		return err;
+	}
+#endif
+
+	chdef.band = band;
+	chdef.chan = ch;
+	chdef.bw = bw;
+	chdef.offset = offset;
+
+	_rtw_mutex_lock_interruptible(&dvobj->setch_mutex);
+#ifdef DBG_CONFIG_CMD_DISP
+	if (a->cmd_type == 0xFF) {
+		cmd_type = PHL_CMD_DIRECTLY;
+		cmd_timeout = 0;
+	}
+	else {
+		cmd_type = a->cmd_type;
+		cmd_timeout = a->cmd_timeout;
+	}
+#endif
+
+	if(a->phl_role == NULL) {
+		err = -1;
+		RTW_INFO("%s: a->phl_role null!!! set band(%u) ch(%u) bw(%u) offset(%u) FAIL!\n",
+			__func__, band, ch, bw, offset);
+		_rtw_mutex_unlock(&dvobj->setch_mutex);
+
+		return err;
+	}
+
+	status = rtw_phl_cmd_set_ch_bw(a->phl_role, alink->wrlink,
+					&chdef, rt_type, cmd_type, cmd_timeout);
+
+	if (status == RTW_PHL_STATUS_SUCCESS) {
+		if (a->bNotifyChannelChange)
+			RTW_INFO("[%s] band = %d, ch = %d, offset = %d, bwmode = %d, success\n",
+				__FUNCTION__, band, ch, offset, bw);
+
+	} else {
+		err = -1;
+		RTW_ERR("%s: set band(%u) ch(%u) bw(%u) offset(%u) FAIL!\n",
+			__func__, band, ch, bw, offset);
+	}
+
+	_rtw_mutex_unlock(&dvobj->setch_mutex);
+
+	return err;
+}
+
+void rtw_update_roch_chan_def(struct _ADAPTER_LINK *adapter_link,
+				u8 remain_ch, enum channel_width remain_bw,
+				enum chan_offset offset,
+				enum band_type band)
+{
+	struct link_mlme_ext_priv *mlmeext = &(adapter_link->mlmeextpriv);
+	/*update chan_def*/
+	mlmeext->chandef.chan = remain_ch;
+	mlmeext->chandef.bw = remain_bw;
+	mlmeext->chandef.band = band;
+	mlmeext->chandef.offset = offset;
+	RTW_DBG("%s: ch(%u) bw(%u) band(%u)\n",
+			__func__,
+			mlmeext->chandef.chan,
+			mlmeext->chandef.bw,
+			mlmeext->chandef.band);
+}
+
+
+void rtw_hw_update_chan_def(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct rtw_chset *chset = &rfctl->chset;
+	struct link_mlme_ext_priv *mlmeext = &(adapter_link->mlmeextpriv);
+	struct rtw_wifi_role_link_t *rlink = adapter_link->wrlink;
+	struct rtw_phl_stainfo_t *phl_sta_self = NULL;
+
+	/*update chan_def*/
+	rlink->chandef.band = mlmeext->chandef.band;
+	rlink->chandef.chan = mlmeext->chandef.chan;
+	rlink->chandef.bw = mlmeext->chandef.bw;
+	rlink->chandef.offset = mlmeext->chandef.offset;
+	rlink->chandef.center_ch = rtw_phl_get_center_ch(&mlmeext->chandef);
+	/* ToDo: 80+80 BW & 160 BW */
+
+	rlink->chandef.is_dfs = rtw_chset_is_dfs_bchbw(chset
+		, rlink->chandef.band
+		, rlink->chandef.chan
+		, rlink->chandef.bw
+		, rlink->chandef.offset);
+
+	phl_sta_self = rtw_phl_get_stainfo_self(adapter_to_dvobj(adapter)->phl, rlink);
+	_rtw_memcpy(&phl_sta_self->chandef, &rlink->chandef, sizeof(struct rtw_chan_def));
+}
+
+static void _dump_phl_role_info(struct rtw_wifi_role_t *wrole)
+{
+	u8 lidx;
+	RTW_INFO("[WROLE]- role-idx: %d\n", wrole->id);
+
+	RTW_INFO("[WROLE]- type: %d\n", wrole->type);
+	RTW_INFO("[WROLE]- mstate: %d\n", wrole->mstate);
+	RTW_INFO("[WROLE]- mac_addr:"MAC_FMT"\n", MAC_ARG(wrole->mac_addr));
+	for (lidx = 0; lidx < wrole->rlink_num; lidx++) {
+		RTW_INFO("- [WRLINK]- link-idx: %d\n", wrole->rlink[lidx].id);
+		RTW_INFO("- [WRLINK]- hw_band: %d\n", wrole->rlink[lidx].hw_band);
+		RTW_INFO("- [WRLINK]- hw_port: %d\n", wrole->rlink[lidx].hw_port);
+		RTW_INFO("- [WRLINK]- hw_wmm: %d\n", wrole->rlink[lidx].hw_wmm);
+		RTW_INFO("- [WRLINK]- band: %d\n", wrole->rlink[lidx].chandef.band);
+		RTW_INFO("- [WRLINK]- chan: %d\n", wrole->rlink[lidx].chandef.chan);
+		RTW_INFO("- [WRLINK]- bw: %d\n", wrole->rlink[lidx].chandef.bw);
+		RTW_INFO("- [WRLINK]- offset: %d\n",wrole->rlink[lidx].chandef.offset);
+	}
+	// Freddie ToDo: MBSSID
+}
+
+void rtw_collect_adapter_link_mac_addr(_adapter *adapter, u8 **mac)
+{
+	struct _ADAPTER_LINK *adapter_link;
+	u8 lidx;
+
+	for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+		adapter_link = GET_LINK(adapter, lidx);
+		mac[lidx] = adapter_link->mac_addr;
+	}
+}
+
+u8 rtw_hw_iface_init(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	u8 phl_role_idx = INVALID_WIFI_ROLE_IDX;
+	u8 rst = _FAIL;
+	int chctx_num = 0;
+	struct _ADAPTER_LINK *adapter_link = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	u8 *adapter_link_mac_addr[RTW_RLINK_MAX] = {NULL};
+	int lidx;
+#if defined(CONFIG_RTW_IPS) || defined(CONFIG_RTW_LPS)
+	bool ps_allow = _FALSE;
+
+	rtw_phl_ps_set_rt_cap(GET_PHL_INFO(dvobj), HW_BAND_0, ps_allow, PS_RT_CORE_INIT);
+#endif
+	// Freddie ToDo: For AP mode, net type should be set to net device already.
+
+	rtw_collect_adapter_link_mac_addr(adapter, adapter_link_mac_addr);
+
+
+	/* will allocate phl self sta info */
+	phl_role_idx = rtw_phl_wifi_role_alloc(GET_PHL_INFO(dvobj),
+			adapter_mac_addr(adapter),
+			adapter_link_mac_addr,
+			PHL_RTYPE_STATION,
+			adapter->iface_id, &(adapter->phl_role),
+			DTYPE,
+			_FALSE);
+
+	if ((phl_role_idx == INVALID_WIFI_ROLE_IDX) ||
+		(adapter->phl_role == NULL)) {
+		RTW_ERR("rtw_phl_wifi_role_alloc failed\n");
+		rtw_warn_on(1);
+		goto _error;
+	}
+	for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+		adapter_link = GET_LINK(adapter, lidx);
+		rlink = &(adapter->phl_role->rlink[lidx]);
+		if (rlink == NULL) {
+			RTW_ERR("%s rlink == NULL\n", __func__);
+			goto _error;
+		}
+		adapter_link->wrlink = rlink;
+	}
+
+	/*init default value*/
+	#ifdef DBG_CONFIG_CMD_DISP
+	adapter->cmd_type = 0xFF;
+	adapter->cmd_timeout = 0;
+	#endif
+	for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+		adapter_link = GET_LINK(adapter, lidx);
+		rtw_hw_update_chan_def(adapter, adapter_link);
+		chctx_num = rtw_phl_mr_get_chanctx_num(GET_PHL_INFO(dvobj), adapter->phl_role, adapter_link->wrlink);
+		if (chctx_num == 0) {
+			if (rtw_phl_cmd_set_ch_bw(adapter->phl_role, adapter_link->wrlink,
+							&(adapter_link->wrlink->chandef),
+							_FALSE, PHL_CMD_WAIT, 0)
+							!= RTW_PHL_STATUS_SUCCESS) {
+				RTW_ERR("%s init ch failed\n", __func__);
+			}
+		}
+	}
+
+	_dump_phl_role_info(adapter->phl_role);
+
+	/* init self staion info after wifi role alloc */
+	rst = rtw_init_self_stainfo(adapter, PHL_CMD_WAIT);
+
+#if defined (CONFIG_PCI_HCI) && defined (CONFIG_PCIE_TRX_MIT_FIX)
+	rtw_pcie_trx_mit_cmd(adapter, 0, 0,
+			     PCIE_RX_INT_MIT_TIMER, 0, 1);
+#endif
+#if defined(CONFIG_RTW_IPS) || defined(CONFIG_RTW_LPS)
+	ps_allow = _TRUE;
+	rtw_phl_ps_set_rt_cap(GET_PHL_INFO(dvobj), HW_BAND_0, ps_allow, PS_RT_CORE_INIT);
+#endif
+
+#ifdef CONFIG_HW_RTS
+	/* Disable HW CTS2self */
+	rtw_phl_hw_cts2self_cfg(GET_PHL_INFO(dvobj), HW_BAND_0, 0, 0, 0);
+#endif
+
+	return rst;
+
+_error:
+	return rst;
+}
+
+u8 rtw_hw_iface_type_change(_adapter *adapter, u8 iface_type)
+{
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(adapter));
+#ifdef CONFIG_WIFI_MONITOR
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(adapter_to_dvobj(adapter));
+#endif
+	struct rtw_wifi_role_t *wrole = adapter->phl_role;
+	enum role_type rtype = PHL_RTYPE_NONE;
+	enum rtw_phl_status status;
+	struct sta_info *sta = NULL;
+	struct _ADAPTER_LINK *adapter_link = NULL;
+	struct rtw_phl_mld_t *mld = NULL;
+	u8 lidx;
+
+	if (wrole == NULL) {
+		RTW_ERR("%s - wrole = NULL\n", __func__);
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+
+	switch (iface_type) {
+	case _HW_STATE_ADHOC_:
+		rtype = PHL_RTYPE_ADHOC;
+		break;
+	case _HW_STATE_STATION_:
+		rtype = PHL_RTYPE_STATION;
+		break;
+	case _HW_STATE_AP_:
+		rtype = PHL_RTYPE_AP;
+		break;
+	case _HW_STATE_MONITOR_:
+		rtype = PHL_RTYPE_MONITOR;
+		break;
+	case _HW_STATE_NOLINK_:
+	default:
+		/* TBD */
+		break;
+	}
+
+	status = rtw_phl_cmd_wrole_change(phl, wrole, NULL,
+				WR_CHG_TYPE, (u8*)&rtype, sizeof(enum role_type),
+				PHL_CMD_DIRECTLY, 0);
+
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s - change to phl role type = %d fail with error = %d\n",
+			__func__, rtype, status);
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+
+#ifdef CONFIG_WIFI_MONITOR
+	if (rtype == PHL_RTYPE_MONITOR) {
+		phl_com->append_fcs = false; /* This need to check again by yiwei*/
+		rtw_phl_enter_mon_mode(phl, wrole);
+	} else {
+		phl_com->append_fcs = true; /* This need to check again by yiwei*/
+		rtw_phl_leave_mon_mode(phl, wrole);
+	}
+#endif
+
+	/* AP allocates self-station and changes broadcast-station before hostapd adds key */
+	if (rtype == PHL_RTYPE_AP) {
+		mld = rtw_phl_get_mld_by_addr(GET_PHL_INFO(adapter_to_dvobj(adapter)), wrole, adapter_mac_addr(adapter));
+		if (mld == NULL) {
+			mld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(adapter)), wrole, adapter_mac_addr(adapter), DTYPE);
+			if (mld == NULL) {
+				RTW_ERR("%s - allocate AP self-mld failed\n", __func__);
+				rtw_warn_on(1);
+				return _FAIL;
+			}
+		}
+		for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+			adapter_link = GET_LINK(adapter, lidx);
+			sta = rtw_get_stainfo(&adapter->stapriv, adapter_link->mac_addr);
+			if (sta == NULL) {
+				/* main_id is don't care for self sta */
+				sta = rtw_alloc_stainfo(&adapter->stapriv, adapter_link->mac_addr, DTYPE, 0, lidx, PHL_CMD_DIRECTLY);
+				if (sta == NULL) {
+					RTW_ERR("%s - allocate AP self-stations failed\n", __func__);
+					rtw_warn_on(1);
+					return _FAIL;
+				}
+			}
+		}
+	}
+
+	RTW_INFO("%s - change to type = %d success !\n", __func__, iface_type);
+
+	return _SUCCESS;
+}
+
+/*
+ * Terminate every phl(wifi) role related works before phl role gone.
+ */
+static void _phl_role_free_prepare(struct _ADAPTER *adapter)
+{
+	rtw_connect_abort_wait(adapter);
+	BUG_ON(rtw_disconnect_abort_wait(adapter) < 0);
+}
+
+void rtw_hw_iface_deinit(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+#if defined(CONFIG_RTW_IPS) || defined(CONFIG_RTW_LPS)
+	bool ps_allow = _FALSE;
+
+	rtw_phl_ps_set_rt_cap(GET_PHL_INFO(dvobj), HW_BAND_0, ps_allow, PS_RT_CORE_INIT);
+#endif
+	if (adapter->phl_role) {
+		_phl_role_free_prepare(adapter);
+		rtw_free_self_stainfo(adapter);
+		rtw_phl_wifi_role_free(GET_PHL_INFO(dvobj), adapter->phl_role->id);
+		adapter->phl_role = NULL;
+	}
+#if defined(CONFIG_RTW_IPS) || defined(CONFIG_RTW_LPS)
+	ps_allow = _TRUE;
+	rtw_phl_ps_set_rt_cap(GET_PHL_INFO(dvobj), HW_BAND_0, ps_allow, PS_RT_CORE_INIT);
+#endif
+}
+
+/*
+ * _sec_algo_drv2phl() - Convert security algorithm to PHL's definition
+ * @drv_algo:		security algorithm
+ * @phl_algo:		security algorithm for PHL, ref to enum rtw_enc_algo
+ * @phl_key_len:	key length
+ *
+ * Convert driver's security algorithm defintion to PHL's type.
+ *
+ */
+static void _sec_algo_drv2phl(enum security_type drv_algo,
+			      u8 *algo, u8 *key_len)
+{
+	u8 phl_algo = RTW_ENC_NONE;
+	u8 phl_key_len = 0;
+
+	switch(drv_algo) {
+	case _NO_PRIVACY_:
+		phl_algo = RTW_ENC_NONE;
+		phl_key_len = 0;
+		break;
+	case _WEP40_:
+		phl_algo = RTW_ENC_WEP40;
+		phl_key_len = 5;
+		break;
+	case _TKIP_:
+	case _TKIP_WTMIC_:
+		phl_algo = RTW_ENC_TKIP;
+		phl_key_len = 16;
+		break;
+	case _AES_:
+		phl_algo = RTW_ENC_CCMP;
+		phl_key_len = 16;
+		break;
+	case _WEP104_:
+		phl_algo = RTW_ENC_WEP104;
+		phl_key_len = 13;
+		break;
+	case _SMS4_:
+		phl_algo = RTW_ENC_WAPI;
+		phl_key_len = 32;
+		break;
+	case _GCMP_:
+		phl_algo = RTW_ENC_GCMP;
+		phl_key_len = 16;
+		break;
+	case _CCMP_256_:
+		phl_algo = RTW_ENC_CCMP256;
+		phl_key_len = 32;
+		break;
+	case _GCMP_256_:
+		phl_algo = RTW_ENC_GCMP256;
+		phl_key_len = 32;
+		break;
+#ifdef CONFIG_IEEE80211W
+	case _BIP_CMAC_128_:
+		phl_algo = RTW_ENC_BIP_CCMP128;
+		phl_key_len = 16;
+		break;
+#endif /* CONFIG_IEEE80211W */
+	default:
+		RTW_ERR("%s: No rule to covert drv algo(0x%x) to phl!!\n",
+			__func__, drv_algo);
+		phl_algo = RTW_ENC_MAX;
+		phl_key_len = 0;
+		break;
+	}
+
+	if(algo)
+		*algo = phl_algo;
+	if(key_len)
+		*key_len = phl_key_len;
+}
+
+/*
+ * _sec_algo_phl2drv() - Convert security algorithm to core layer definition
+ * @drv_algo:		security algorithm for core layer, ref to enum security_type
+ * @phl_algo:		security algorithm for PHL, ref to enum rtw_enc_algo
+ * @drv_key_len:	key length
+ *
+ * Convert PHL's security algorithm defintion to core layer definition.
+ *
+ */
+static void _sec_algo_phl2drv(enum rtw_enc_algo phl_algo,
+			      u8 *algo, u8 *key_len)
+{
+	u8 drv_algo = RTW_ENC_NONE;
+	u8 drv_key_len = 0;
+
+	switch(phl_algo) {
+	case RTW_ENC_NONE:
+		drv_algo = _NO_PRIVACY_;
+		drv_key_len = 0;
+		break;
+	case RTW_ENC_WEP40:
+		drv_algo = _WEP40_;
+		drv_key_len = 5;
+		break;
+	case RTW_ENC_TKIP:
+		/* drv_algo = _TKIP_WTMIC_ */
+		drv_algo = _TKIP_;
+		drv_key_len = 16;
+		break;
+	case RTW_ENC_CCMP:
+		drv_algo = _AES_;
+		drv_key_len = 16;
+		break;
+	case RTW_ENC_WEP104:
+		drv_algo = _WEP104_;
+		drv_key_len = 13;
+		break;
+	case RTW_ENC_WAPI:
+		drv_algo = _SMS4_;
+		drv_key_len = 32;
+		break;
+	case RTW_ENC_GCMP:
+		drv_algo = _GCMP_;
+		drv_key_len = 16;
+		break;
+	case RTW_ENC_CCMP256:
+		drv_algo = _CCMP_256_;
+		drv_key_len = 32;
+		break;
+	case RTW_ENC_GCMP256:
+		drv_algo = _GCMP_256_;
+		drv_key_len = 32;
+		break;
+#ifdef CONFIG_IEEE80211W
+	case RTW_ENC_BIP_CCMP128:
+		drv_algo = _BIP_CMAC_128_;
+		drv_key_len = 16;
+		break;
+#endif /* CONFIG_IEEE80211W */
+	default:
+		RTW_ERR("%s: No rule to covert phl algo(0x%x) to drv!!\n",
+			__func__, phl_algo);
+		drv_algo = _SEC_TYPE_MAX_;
+		drv_key_len = 0;
+		break;
+	}
+
+	if(algo)
+		*algo = drv_algo;
+	if(key_len)
+		*key_len = drv_key_len;
+}
+
+u8 rtw_sec_algo_drv2phl(enum security_type drv_algo)
+{
+	u8 algo = 0;
+
+	_sec_algo_drv2phl(drv_algo, &algo, NULL);
+	return algo;
+}
+
+u8 rtw_sec_algo_phl2drv(enum rtw_enc_algo phl_algo)
+{
+	u8 algo = 0;
+
+	_sec_algo_phl2drv(phl_algo, &algo, NULL);
+	return algo;
+}
+
+static int rtw_hw_chk_sec_mode(struct _ADAPTER *a, struct sta_info *sta,
+			enum phl_cmd_type cmd_type,  u32 cmd_timeout)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	u8 sec_mode = 0;
+	struct security_priv *psecuritypriv = &a->securitypriv;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	if (!phl)
+		return _FAIL;
+
+	sec_mode = rtw_phl_trans_sec_mode(
+		rtw_sec_algo_drv2phl(psecuritypriv->dot11PrivacyAlgrthm),
+		rtw_sec_algo_drv2phl(psecuritypriv->dot118021XGrpPrivacy));
+
+	RTW_INFO("After phl trans_sec_mode = %d\n", sec_mode);
+
+	if (sec_mode != sta->phl_sta->sec_mode) {
+		RTW_INFO("%s: original sec_mode =%d update sec mode to %d.\n",
+			__func__, sta->phl_sta->sec_mode, sec_mode);
+		status = rtw_phl_cmd_change_stainfo(phl, sta->phl_sta, STA_CHG_SEC_MODE,
+				&sec_mode, sizeof(u8), cmd_type, cmd_timeout);
+	/* To Do: check the return status */
+	} else {
+		RTW_INFO("%s: sec mode remains the same. skip update.\n", __func__);
+	}
+	return _SUCCESS;
+}
+
+/*
+ * rtw_hw_add_key() - Add security key
+ * @a:		pointer of struct _ADAPTER
+ * @sta:	pointer of struct sta_info
+ * @keyid:	key index
+ * @keyalgo:	key algorithm
+ * @keytype:	0: unicast / 1: multicast / 2: bip (ref: enum SEC_CAM_KEY_TYPE)
+ * @key:	key content
+ * @spp:	spp mode
+ *
+ * Add security key.
+ *
+ * Return 0 for success, otherwise fail.
+ */
+int rtw_hw_add_key(struct _ADAPTER *a, struct sta_info *sta,
+		u8 keyid, enum security_type keyalgo, u8 keytype, u8 *key,
+		u8 spp, enum phl_cmd_type cmd_type,  u32 cmd_timeout)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	struct phl_sec_param_h crypt = {0};
+	enum rtw_phl_status status;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	if (rtw_hw_chk_sec_mode(a, sta, cmd_type, cmd_timeout) == _FAIL)
+		return -1;
+
+	crypt.keyid = keyid;
+	crypt.key_type= keytype;
+	crypt.spp = spp;
+	_sec_algo_drv2phl(keyalgo, &crypt.enc_type, &crypt.key_len);
+
+	if (sta->phl_sta == NULL) {
+		RTW_ERR("%s: Fail sta->phl_sta = NULL\n", __func__);
+		return -1;
+	}
+
+	/* delete key before adding key */
+	rtw_phl_cmd_del_key(phl, sta->phl_sta, &crypt, cmd_type, cmd_timeout);
+	status = rtw_phl_cmd_add_key(phl, sta->phl_sta, &crypt, key, cmd_type, cmd_timeout);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+/*
+ * rtw_hw_del_key() - Delete security key
+ * @a:		pointer of struct _ADAPTER
+ * @sta:	pointer of struct sta_info
+ * @keyid:	key index
+ * @keytype:	0: unicast / 1: multicast / 2: bip (ref: enum SEC_CAM_KEY_TYPE)
+ *
+ * Delete security key by macid, keyid and keytype.
+ *
+ * Return 0 for success, otherwise fail.
+ */
+int rtw_hw_del_key(struct _ADAPTER *a, struct sta_info *sta,
+		u8 keyid, u8 keytype, enum phl_cmd_type cmd_type, u32 cmd_timeout)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	struct phl_sec_param_h crypt = {0};
+	enum rtw_phl_status status;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	crypt.keyid = keyid;
+	crypt.key_type= keytype;
+
+	status = rtw_phl_cmd_del_key(phl, sta->phl_sta, &crypt, cmd_type, cmd_timeout);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+/*
+ * rtw_hw_del_all_key() - Delete all security key for this STA
+ * @a:		pointer of struct _ADAPTER
+ * @sta:	pointer of struct sta_info
+ *
+ * Delete all security keys belong to this STA.
+ *
+ * Return 0 for success, otherwise fail.
+ */
+int rtw_hw_del_all_key(struct _ADAPTER *a, struct sta_info *sta,
+			enum phl_cmd_type cmd_type, u32 cmd_timeout)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	u8 keyid;
+	u8 keytype;
+	struct phl_sec_param_h crypt = {0};
+	enum rtw_phl_status status;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	/* Delete Group and Pairwise key */
+	for (keytype = 0; keytype < 2; keytype++) {
+		for (keyid = 0; keyid < 4; keyid++) {
+			crypt.keyid = keyid;
+			crypt.key_type = keytype;
+			rtw_phl_cmd_del_key(phl, sta->phl_sta, &crypt, cmd_type, cmd_timeout);
+		}
+	}
+
+	/* Delete BIP key */
+	crypt.key_type = 2;
+	for (keyid = 4; keyid <= BIP_MAX_KEYID; keyid++) {
+		crypt.keyid = keyid;
+		rtw_phl_cmd_del_key(phl, sta->phl_sta, &crypt, cmd_type, cmd_timeout);
+	}
+
+	return 0;
+}
+
+#if 0
+/* connect */
+int rtw_hw_prepare_connect(struct _ADAPTER *a, struct sta_info *sta, u8 *target_addr)
+{
+	/*adapter->phl_role.mac_addr*/
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	status = rtw_phl_connect_prepare(phl, a->phl_role, target_addr);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s: Fail to setup hardware for connecting!(%d)\n",
+			__func__, status);
+		return -1;
+	}
+	/* Todo: Enable TSF update */
+	/* Todo: Set support short preamble or not by beacon capability */
+	/* Todo: Set slot time */
+
+	return 0;
+}
+#endif
+
+int rtw_hw_connect_remove_sta(struct _ADAPTER *a, struct sta_info *sta)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	rtw_hw_del_all_key(a, sta, PHL_CMD_DIRECTLY, 0);
+
+	status = rtw_phl_cmd_update_media_status(phl, sta->phl_sta, NULL, false,
+							PHL_CMD_DIRECTLY, 0);
+	if (status != RTW_PHL_STATUS_SUCCESS)
+		return -1;
+
+	rtw_phl_chanctx_del(adapter_to_dvobj(a)->phl, a->phl_role, sta->padapter_link->wrlink, NULL);
+
+	/* free connecting AP sta info */
+	rtw_free_stainfo(a, sta);
+
+	return 0;
+}
+
+
+/* Handle connect fail case */
+int rtw_hw_connect_abort(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	struct _ADAPTER_LINK *alink = NULL;
+	struct sta_info *sta = NULL;
+	u8 lidx;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+		alink = GET_LINK(a, lidx);
+		if (!alink->mlmepriv.to_join)
+			continue;
+		sta = rtw_get_stainfo(&(a->stapriv), alink->mlmeextpriv.mlmext_info.network.MacAddress);
+		if (!sta) {
+			RTW_ERR(FUNC_ADPT_FMT ": drv sta_info not exist!\n", FUNC_ADPT_ARG(a));
+			return -1;
+		}
+		rtw_hw_del_all_key(a, sta, PHL_CMD_DIRECTLY, 0);
+		status = rtw_phl_cmd_update_media_status(phl, sta->phl_sta, NULL, false,
+								PHL_CMD_DIRECTLY, 0);
+		if (status != RTW_PHL_STATUS_SUCCESS)
+			return -1;
+	}
+
+	return 0;
+}
+
+#ifdef RTW_WKARD_UPDATE_PHL_ROLE_CAP
+/**
+ * rtw_update_phl_cap_by_rgstry() - Update cap & protocol_cap of phl_role
+ * @a:		struct _ADAPTER*
+ *
+ * Update cap & protocol_cap of a->phl_role by registry/driver parameters.
+ *
+ */
+void rtw_update_phl_cap_by_rgstry(struct _ADAPTER *a, struct _ADAPTER_LINK *alink)
+{
+	struct registry_priv *rgstry = &a->registrypriv;
+	struct role_link_cap_t *cap = &alink->wrlink->cap;
+	struct protocol_cap_t *prtcl = &alink->wrlink->protocol_cap;
+
+	/* LDPC */
+	prtcl->ht_ldpc &= (TEST_FLAG(rgstry->ldpc_cap, BIT4) ? 1 : 0);
+	cap->tx_ht_ldpc &= (TEST_FLAG(rgstry->ldpc_cap, BIT5) ? 1 : 0);
+	prtcl->vht_ldpc &= (TEST_FLAG(rgstry->ldpc_cap, BIT0) ? 1 : 0);
+	cap->tx_vht_ldpc &= (TEST_FLAG(rgstry->ldpc_cap, BIT1) ? 1 : 0);
+	prtcl->he_ldpc &= (TEST_FLAG(rgstry->ldpc_cap, BIT2) ? 1 : 0);
+	cap->tx_he_ldpc &= (TEST_FLAG(rgstry->ldpc_cap, BIT3) ? 1 : 0);
+}
+#endif /* RTW_WKARD_UPDATE_PHL_ROLE_CAP */
+
+static void _dump_phl_sta_asoc_cap(struct sta_info *sta)
+{
+	struct rtw_phl_stainfo_t *phl_sta = sta->phl_sta;
+	struct protocol_cap_t *asoc_cap = &phl_sta->asoc_cap;
+#define _loc_dbg_func	RTW_DBG
+#define _loc_dbg(f)     _loc_dbg_func(#f ": %u\n", asoc_cap->f)
+
+
+	_loc_dbg_func("[PHL STA ASOC CAP]- mac_addr: " MAC_FMT "\n",
+		      MAC_ARG(phl_sta->mac_addr));
+	_loc_dbg(ht_ldpc);
+	_loc_dbg(vht_ldpc);
+	_loc_dbg(he_ldpc);
+	_loc_dbg(stbc_ht_rx);
+	_loc_dbg(stbc_vht_rx);
+	_loc_dbg(stbc_he_rx);
+	_loc_dbg(vht_su_bfmr);
+	_loc_dbg(vht_su_bfme);
+	_loc_dbg(vht_mu_bfmr);
+	_loc_dbg(vht_mu_bfme);
+	_loc_dbg(bfme_sts);
+	_loc_dbg(num_snd_dim);
+	_loc_dbg_func("[PHL STA ASOC CAP]- end\n");
+}
+
+#ifdef CONFIG_80211N_HT
+#ifdef CONFIG_80211AC_VHT
+static void update_phl_sta_cap_vht(struct _ADAPTER *a, struct sta_info *sta,
+			           struct protocol_cap_t *cap)
+{
+	struct vht_priv *vht;
+
+
+	vht = &sta->vhtpriv;
+
+	if (cap->ampdu_len_exp < vht->ampdu_len)
+		cap->ampdu_len_exp = vht->ampdu_len;
+	if (cap->max_amsdu_len < vht->max_mpdu_len)
+		cap->max_amsdu_len = vht->max_mpdu_len;
+
+	cap->sgi_80 = (vht->sgi_80m == _TRUE) ? 1 : 0;
+	cap->sgi_160 = (vht->sgi_160m == _TRUE) ? 1 : 0;
+
+	_rtw_memcpy(cap->vht_rx_mcs, vht->vht_mcs_map, 2);
+	/* Todo: cap->vht_tx_mcs[2]; */
+	if (vht->op_present)
+		_rtw_memcpy(cap->vht_basic_mcs, &vht->vht_op[3], 2);
+}
+#endif /* CONFIG_80211AC_VHT */
+static void update_phl_sta_cap_ht(struct _ADAPTER *a, struct sta_info *sta,
+			          struct protocol_cap_t *cap)
+{
+	struct link_mlme_ext_info *info;
+	struct ht_priv *ht;
+	struct ampdu_priv *ampdu_priv;
+
+	info = &sta->padapter_link->mlmeextpriv.mlmext_info;
+	ht = &sta->htpriv;
+	ampdu_priv = &sta->ampdu_priv;
+
+
+	cap->ampdu_density = ampdu_priv->rx_ampdu_min_spacing;
+	cap->ampdu_len_exp = GET_HT_CAP_ELE_MAX_AMPDU_LEN_EXP(&ht->ht_cap);
+	cap->amsdu_in_ampdu = 1;
+	cap->max_amsdu_len = GET_HT_CAP_ELE_MAX_AMSDU_LENGTH(&ht->ht_cap);
+
+	/* move to update_sta_smps_cap  */
+	/*GET_HT_CAP_ELE_SM_PS(&info->HT_caps.u.HT_cap_element.HT_caps_info);*/
+	/* cap->sm_ps = info->SM_PS; */
+
+	cap->sgi_20 = (ht->sgi_20m == _TRUE) ? 1 : 0;
+	cap->sgi_40 = (ht->sgi_40m == _TRUE) ? 1 : 0;
+
+	_rtw_memcpy(cap->ht_rx_mcs, ht->ht_cap.supp_mcs_set, 4);
+	/* Todo: cap->ht_tx_mcs[4]; */
+	if (info->HT_info_enable)
+		_rtw_memcpy(cap->ht_basic_mcs, info->HT_info.MCS_rate, 4);
+}
+#endif /* CONFIG_80211N_HT */
+
+static u16 _get_default_num_ampdu(struct _ADAPTER *a, struct sta_info *sta,
+		struct protocol_cap_t *cap)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	struct rtw_wifi_role_link_t *rlink = sta->phl_sta->rlink;
+	struct protocol_cap_t       *rlink_cap = &(rlink->protocol_cap);
+	u8 band_idx = rlink->hw_band;
+
+	return rlink_cap->num_ampdu > phl_com->phy_cap[band_idx].txagg_num ?
+		phl_com->phy_cap[band_idx].txagg_num : rlink_cap->num_ampdu;
+
+}
+
+void rtw_update_phl_sta_cap(struct _ADAPTER *a, struct sta_info *sta,
+			    struct protocol_cap_t *cap)
+{
+	struct link_mlme_ext_info *info;
+
+	info = &sta->padapter_link->mlmeextpriv.mlmext_info;
+
+	/* MAC related */
+	/* update beacon interval */
+	cap->bcn_interval = info->bcn_interval;
+#if 0
+	cap->num_ampdu;		/* HT, VHT, HE */
+	cap->ampdu_density:3;	/* HT, VHT, HE */
+	cap->ampdu_len_exp;	/* HT, VHT, HE */
+	cap->amsdu_in_ampdu:1;	/* HT, VHT, HE */
+	cap->max_amsdu_len:2;	/* HT, VHT, HE */
+	cap->htc_rx:1;
+	cap->sm_ps:2;		/* HT */
+	cap->trig_padding:2;
+	cap->twt:6;
+	cap->all_ack:1;
+	cap->a_ctrl:3;
+	cap->ops:1;
+	cap->ht_vht_trig_rx:1;
+#endif
+	cap->short_slot = (info->slotTime == SHORT_SLOT_TIME) ? 1 : 0;
+	cap->preamble = (info->preamble_mode == PREAMBLE_SHORT) ? 1 : 0;
+#if 0
+	cap->sgi_20:1;		/* HT */
+	cap->sgi_40:1;		/* HT */
+	cap->sgi_80:1;		/* VHT */
+	cap->sgi_160:1		/* VHT, HE */
+	/* update in WMMOnAssocRsp() @ core/rtw_wlan_util.c */
+	struct rtw_edca_param edca[4]; 	/* Access Category, 0:BE, 1:BK, 2:VI, 3:VO */
+	u8 mu_qos_info;
+	/* update in HE_mu_edca_handler() @ core/rtw_he.c */
+	struct rtw_mu_edca_param mu_edca[4];	/* HE */
+
+	/* BB related */
+	cap->ht_ldpc:1;		/* HT, HT_caps_handler() */
+	cap->vht_ldpc:1;	/* VHT, VHT_caps_handler() */
+	cap->he_ldpc:1;		/* HE, HE_phy_caps_handler() */
+	cap->sgi:1;
+	cap->su_bfmr:1;
+	cap->su_bfme:1;
+	cap->mu_bfmr:1;
+	cap->mu_bfme:1;
+	cap->bfme_sts:3;
+	cap->num_snd_dim:3;
+#endif
+	_rtw_memset(cap->supported_rates, 0, 12);
+	_rtw_memcpy(cap->supported_rates, sta->bssrateset,
+		    sta->bssratelen < 12 ? sta->bssratelen : 12);
+#if 0
+	cap->ht_rx_mcs[4];	/* HT */
+	cap->ht_tx_mcs[4];	/* HT */
+	cap->ht_basic_mcs[4];	/* Basic rate of HT */
+	cap->vht_rx_mcs[2];	/* VHT */
+	cap->vht_tx_mcs[2];	/* VHT */
+	cap->vht_basic_mcs[2];	/* Basic rate of VHT */
+#endif
+#if 0
+	/* HE done */
+	cap->he_rx_mcs[2];
+	cap->he_tx_mcs[2];
+	cap->he_basic_mcs[2];	/* Basic rate of HE */
+	cap->stbc_ht_rx:2;	/* HT_caps_handler() */
+	cap->stbc_vht_rx:3;	/* VHT_caps_handler() */
+	cap->stbc_he_rx:1;	/* HE_phy_caps_handler() */
+	cap->stbc_tx:1;
+	cap->ltf_gi;
+	cap->doppler_tx:1;
+	cap->doppler_rx:1;
+	cap->dcm_max_const_tx:2;
+	cap->dcm_max_nss_tx:1;
+	cap->dcm_max_const_rx:2;
+	cap->dcm_max_nss_rx:1;
+	cap->partial_bw_su_in_mu:1;
+	cap->bfme_sts_greater_80mhz:3;
+	cap->num_snd_dim_greater_80mhz:3;
+	cap->stbc_tx_greater_80mhz:1;
+	cap->stbc_rx_greater_80mhz:1;
+	cap->ng_16_su_fb:1;
+	cap->ng_16_mu_fb:1;
+	cap->cb_sz_su_fb:1;
+	cap->cb_sz_mu_fb:1;
+	cap->trig_su_bfm_fb:1;
+	cap->trig_mu_bfm_fb:1;
+	cap->trig_cqi_fb:1;
+	cap->partial_bw_su_er:1;
+	cap->pkt_padding:2;
+	cap->ppe_th[24];
+	cap->pwr_bst_factor:1;
+	cap->max_nc:3;
+	cap->dcm_max_ru:2;
+	cap->long_sigb_symbol:1;
+	cap->non_trig_cqi_fb:1;
+	cap->tx_1024q_ru:1;
+	cap->rx_1024q_ru:1;
+	cap->fbw_su_using_mu_cmprs_sigb:1;
+	cap->fbw_su_using_mu_non_cmprs_sigb:1;
+	cap->er_su:1;
+	cap->tb_pe:3;
+	cap->txop_du_rts_th;
+#endif
+
+#ifdef CONFIG_80211N_HT
+	if (sta->htpriv.ht_option) {
+		update_phl_sta_cap_ht(a, sta, cap);
+#ifdef CONFIG_80211AC_VHT
+		if (sta->vhtpriv.vht_option)
+			update_phl_sta_cap_vht(a, sta, cap);;
+#endif /* CONFIG_80211AC_VHT */
+	}
+	/* Set A-MPDU number to default cap as it is resolved
+	   by ADDBA req/resp handshaking */
+	cap->num_ampdu = _get_default_num_ampdu(a, sta, cap);
+	RTW_PRINT("Updated A-MPDU number to %u as default for "MAC_FMT"\n",
+	          cap->num_ampdu, MAC_ARG(sta->phl_sta->mac_addr));
+#endif /* CONFIG_80211N_HT */
+}
+
+/**
+ * rtw_update_phl_sta_edca() - Update PHL STA EDCA parameter
+ * @a:		struct _ADAPTER *
+ * @ac:		Access Category, ref: enum rtw_ac
+ * @param:	AIFS:BIT[7:0], CWMIN:BIT[11:8], CWMAX:BIT[15:12],
+ *		TXOP:BIT[31:16]
+ *		ref: struct rtw_edca_param
+ *
+ * Update WMM EDCA parameter set to PHL STA protocol capability.
+ */
+void rtw_update_phl_sta_edca(struct _ADAPTER *a, struct _ADAPTER_LINK *alink, enum rtw_ac ac, u32 param)
+{
+	struct sta_info *sta;
+	struct protocol_cap_t *cap;
+	u8 *mac;
+
+
+	mac = (u8*)alink->mlmeextpriv.mlmext_info.network.MacAddress;
+	sta = rtw_get_stainfo(&a->stapriv, mac);
+	if (!sta) {
+		RTW_ERR(FUNC_ADPT_FMT ": drv sta_info(" MAC_FMT ") not exist!\n",
+			FUNC_ADPT_ARG(a), MAC_ARG(mac));
+		return;
+	}
+	cap = &sta->phl_sta->asoc_cap;
+
+	cap->edca[ac].param = param;
+}
+
+/**
+ * rtw_hw_set_edca() - setup WMM EDCA parameter
+ * @a:		struct _ADAPTER *
+ * @ac:		Access Category, 0:BE, 1:BK, 2:VI, 3:VO
+ * @param:	AIFS:BIT[7:0], CWMIN:BIT[11:8], CWMAX:BIT[15:12],
+ *		TXOP:BIT[31:16]
+ *
+ * Setup WMM EDCA parameter set.
+ *
+ * Return 0 for SUCCESS, otherwise fail.
+ */
+int rtw_hw_set_edca(struct _ADAPTER *a, struct _ADAPTER_LINK *alink, u8 ac, u32 param)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	struct rtw_edca_param edca = {0};
+	enum rtw_phl_status status;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	edca.ac = ac;
+	edca.param = param;
+
+	status = rtw_phl_cmd_wrole_change(phl, a->phl_role, alink->wrlink,
+				WR_CHG_EDCA_PARAM, (u8*)&edca, sizeof(struct rtw_edca_param),
+				PHL_CMD_DIRECTLY, 0);
+
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s: fail to set edca parameter, ac(%u), "
+			"param(0x%08x)\n",
+			__func__, ac, param);
+		return -1;
+	}
+
+	return 0;
+}
+
+int rtw_hw_connected(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	struct security_priv *psecuritypriv = &a->securitypriv;
+	struct _ADAPTER_LINK *alink = NULL;
+	struct sta_info *sta = NULL;
+	u8 lidx;
+
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+		alink = GET_LINK(a, lidx);
+		if (!alink->mlmepriv.is_accepted)
+			continue;
+		sta = rtw_get_stainfo(&a->stapriv, alink->mlmeextpriv.mlmext_info.network.MacAddress);
+		if (!sta)
+			return -1;
+		rtw_update_phl_sta_cap(a, sta, &sta->phl_sta->asoc_cap);
+		_dump_phl_sta_asoc_cap(sta);
+
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+		if (alink->mlmeextpriv.mlmext_info.network.is_mbssid) {
+			sta->phl_sta->is_nontx = true;
+			/*transmitted BSSID*/
+			_rtw_memcpy(sta->phl_sta->ref_mac,
+				alink->mlmeextpriv.mlmext_info.network.tx_bssid, ETH_ALEN);
+		}
+#endif
+
+		if (rtw_sta_hal_media_status_rpt_cmd(a, sta, true, RTW_CMDF_DIRECTLY) != _SUCCESS)
+			return -1;
+		rtw_dump_phl_sta_info(RTW_DBGDUMP, sta);
+
+		/* Todo: udpate capability: short preamble, slot time */
+		update_capinfo(a, alink, alink->mlmeextpriv.mlmext_info.capability);
+
+		WMMOnAssocRsp(a, alink);
+	}
+
+	/* Todo: update IOT-releated issue */
+#if 0
+	update_IOT_info(a);
+#endif
+	/* Todo: RTS full bandwidth setting */
+#if 0
+#ifdef CONFIG_RTS_FULL_BW
+	rtw_set_rts_bw(a);
+#endif /* CONFIG_RTS_FULL_BW */
+#endif
+	/* Todo: Basic rate setting */
+#if 0
+	rtw_hal_set_hwreg(a, HW_VAR_BASIC_RATE, cur_network->SupportedRates);
+#endif
+
+	/* Todo: HT: AMPDU factor, min space, max time and related parameters */
+#if 0
+#ifdef CONFIG_80211N_HT
+	HTOnAssocRsp(a);
+#endif /* CONFIG_80211N_HT */
+#endif
+	/* Todo: VHT */
+#if 0
+#ifdef CONFIG_80211AC_VHT
+	VHTOnAssocRsp(a);
+#endif
+#endif
+	/* Todo: Set Data rate and RA */
+#if 0
+	set_sta_rate(a, psta);
+#endif
+	/* Todo: IC specific hardware setting */
+#if 0
+	join_type = 2;
+	rtw_hal_set_hwreg(a, HW_VAR_MLME_JOIN, (u8 *)(&join_type));
+#endif
+	if ((a->mlmeextpriv.mlmext_info.state & 0x03) == WIFI_FW_STATION_STATE) {
+		/* Todo: Correct TSF */
+#if 0
+		correct_TSF(a, MLME_STA_CONNECTED);
+#endif
+	}
+
+	/* Todo: btcoex connect event notify */
+#if 0
+	rtw_btcoex_connect_notify(a, join_type);
+#endif
+	/* Todo: Beamforming setting */
+#if 0
+	beamforming_wk_cmd(a, BEAMFORMING_CTRL_ENTER, (u8 *)psta, sizeof(struct sta_info), 0);
+#endif
+
+	rtw_join_done_chk_ch(a, 1);
+	status = rtw_phl_connected(phl, a->connect_bidx, a->phl_role, sta->phl_sta);
+
+	if (status != RTW_PHL_STATUS_SUCCESS)
+			return -1;
+
+#ifdef CONFIG_80211AX_HE
+	for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+		alink = GET_LINK(a, lidx);
+		if (!alink->mlmepriv.is_accepted)
+			continue;
+		rtw_he_init_om_info(a, alink);
+	}
+#endif
+	ATOMIC_SET(&a->need_tsf_sync_done, _TRUE);
+	return 0;
+}
+
+int rtw_hw_disconnect(struct _ADAPTER *a, struct sta_info *sta)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	int tid;
+	u8 is_ap_self = _FALSE;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	if (MLME_IS_AP(a) &&
+		_rtw_memcmp(a->phl_role->mac_addr, sta->phl_sta->mac_addr, ETH_ALEN))
+		is_ap_self = _TRUE;
+
+	/* Check and reset setting related to rx ampdu resources of PHL. */
+	for (tid = 0; tid < TID_NUM; tid++) {
+		if(sta->recvreorder_ctrl[tid].enable == _TRUE) {
+			sta->recvreorder_ctrl[tid].enable =_FALSE;
+			rtw_phl_stop_rx_ba_session(phl, sta->phl_sta, tid);
+			RTW_INFO(FUNC_ADPT_FMT"stop process tid %d \n",
+				FUNC_ADPT_ARG(a), tid);
+		}
+	}
+
+	/*reset sec setting and clean all connection setting*/
+	rtw_hw_del_all_key(a, sta, PHL_CMD_DIRECTLY, 0);
+
+	if (is_ap_self == _FALSE) {
+		status = rtw_phl_cmd_update_media_status(phl, sta->phl_sta, NULL, false,
+						PHL_CMD_DIRECTLY, 0);
+		if (status != RTW_PHL_STATUS_SUCCESS)
+			return -1;
+
+		rtw_dump_phl_sta_info(RTW_DBGDUMP, sta);
+	}
+
+	return 0;
+}
+
+int rtw_hw_connected_apmode(struct _ADAPTER *a, struct sta_info *sta)
+{
+	struct dvobj_priv *d;
+	void *phl;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	if (!phl)
+		return -1;
+
+	rtw_ap_set_sta_wmode(a, sta);
+	update_sta_ra_info(a, sta);
+	rtw_update_phl_sta_cap(a, sta, &sta->phl_sta->asoc_cap);
+
+	if (_SUCCESS != rtw_sta_hal_media_status_rpt_cmd(a, sta, true, RTW_CMDF_DIRECTLY))
+		return -1;
+
+	rtw_dump_phl_sta_info(RTW_DBGDUMP, sta);
+
+	return 0;
+}
+
+#ifdef PHL_USE_RA_BW_MODE
+#define RTW_SET_PHL_STA_TX_BW_MODE(phl_sta, bw_mode) (phl_sta)->tx_bw_mode = (bw_mode)
+#else
+#define RTW_SET_PHL_STA_TX_BW_MODE(phl_sta, bw_mode) do {} while (0)
+#endif
+
+u8 rtw_sta_hal_media_status_rpt_cmd(_adapter *a, struct sta_info *sta, bool connected, u8 flag)
+{
+	enum phl_cmd_type cmd_type = rtw_cmdf_to_phl_cmd_type(flag);
+	u8 *addr = NULL;
+	enum rtw_phl_status status;
+
+	if (connected) {
+		addr = sta->phl_sta->mac_addr;
+		RTW_SET_PHL_STA_TX_BW_MODE(sta->phl_sta, rtw_get_tx_bw_mode(a, sta));
+	}
+
+	status = rtw_phl_cmd_update_media_status(GET_PHL_INFO(adapter_to_dvobj(a))
+		, sta->phl_sta, connected ? addr : NULL, connected, cmd_type, 0);
+
+	return status == RTW_PHL_STATUS_SUCCESS ? _SUCCESS : _FAIL;
+}
+
+u8 rtw_sta_hal_ra_mask_update_cmd(_adapter *a, struct sta_info *sta, u8 flag)
+{
+	enum phl_cmd_type cmd_type = rtw_cmdf_to_phl_cmd_type(flag);
+	enum rtw_phl_status status;
+
+	RTW_SET_PHL_STA_TX_BW_MODE(sta->phl_sta, rtw_get_tx_bw_mode(a, sta));
+
+	status = rtw_phl_cmd_change_stainfo(GET_PHL_INFO(adapter_to_dvobj(a))
+		, sta->phl_sta, STA_CHG_RAMASK, NULL, 0, cmd_type, 0);
+
+	return status == RTW_PHL_STATUS_SUCCESS ? _SUCCESS : _FAIL;
+}
+
+u8 rtw_link_hal_core_stop_beacon(struct _ADAPTER_LINK *alink, bool stop, u8 flag)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(alink->adapter);
+	enum phl_cmd_type cmd_type = rtw_cmdf_to_phl_cmd_type(flag);
+	enum rtw_phl_status status;
+
+	status = rtw_phl_cmd_core_stop_beacon(GET_PHL_INFO(dvobj), alink->wrlink
+		, stop, cmd_type, 0);
+
+	return status == RTW_PHL_STATUS_SUCCESS ? _SUCCESS : _FAIL;
+}
+
+u8 rtw_hal_get_def_var(struct _ADAPTER *a, struct _ADAPTER_LINK *alink,
+				enum _HAL_DEF_VARIABLE def_var, void *val)
+{
+	struct rtw_wifi_role_t *wrole = a->phl_role;
+	struct protocol_cap_t *proto_cap = &(alink->wrlink->protocol_cap);
+
+	switch (def_var) {
+	case HAL_DEF_IS_SUPPORT_ANT_DIV:
+		*(u8*)val = _FALSE;
+		break;
+	case HAL_DEF_DBG_DUMP_RXPKT:
+		*(u8*)val = 0;
+		break;
+	case HAL_DEF_BEAMFORMER_CAP:
+		*(u8*)val = proto_cap->num_snd_dim;
+		break;
+	case HAL_DEF_BEAMFORMEE_CAP:
+		*(u8*)val = proto_cap->bfme_sts;
+		break;
+	case HW_VAR_MAX_RX_AMPDU_FACTOR:
+		/* HT only */
+		*(enum _HT_CAP_AMPDU_FACTOR*)val = MAX_AMPDU_FACTOR_64K;
+		break;
+	case HW_DEF_RA_INFO_DUMP:
+		/* do nothing */
+		break;
+	case HAL_DEF_DBG_DUMP_TXPKT:
+		*(u8*)val = 0;
+		break;
+	case HAL_DEF_TX_PAGE_SIZE:
+		/* would be removed later */
+		break;
+	case HW_VAR_BEST_AMPDU_DENSITY:
+		*(u8*)val = 0;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+#ifdef RTW_DETECT_HANG
+#define HANG_DETECT_THR 3
+#define MAC_RX_FULL_DROP_CHK_MAX_NUM 100
+void rtw_is_rxff_hang(_adapter *padapter, struct rxff_hang_info *prxff_hang_info)
+{
+	struct dvobj_priv *pdvobjpriv = padapter->dvobj;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(padapter);
+	struct rtw_wifi_role_link_t *rlink = adapter_link->wrlink;
+	void *phl = GET_PHL_INFO(pdvobjpriv);
+	u16 mac_rx_full_drop_cnt = 0;
+	u8 chk_cnt = 0;
+	enum rtw_phl_status psts;
+
+	if (prxff_hang_info->rx_cnt != prxff_hang_info->last_rx_cnt) {
+		prxff_hang_info->last_rx_cnt = prxff_hang_info->rx_cnt;
+		prxff_hang_info->rx_ff_hang_cnt = 0;
+		prxff_hang_info->dbg_is_rxff_hang = _FALSE;
+		return;
+	}
+
+	psts = rtw_phl_cmd_set_reset_rx_cnt(phl, rlink->hw_band,
+					    PHL_CMD_DIRECTLY, 0);
+	if (psts != RTW_PHL_STATUS_SUCCESS) {
+		RTW_WARN("%s: reset rx dbg cnt failed\n", __func__);
+		return;
+	}
+
+	do {
+		psts = rtw_phl_cmd_get_rx_cnt_by_idx(phl,rlink->hw_band,
+						     RXCNT_FULLDRP_PKT,
+						     &mac_rx_full_drop_cnt,
+						     PHL_CMD_DIRECTLY, 0);
+		if (psts != RTW_PHL_STATUS_SUCCESS) {
+			RTW_WARN("%s: get rx full drop cnt failed\n", __func__);
+			continue;
+		}
+
+		if (mac_rx_full_drop_cnt) {
+			RTW_INFO("%s: mac rx full drop w/o HCI rx\n", __func__);
+			prxff_hang_info->rx_ff_hang_cnt++;
+			break;
+		}
+	} while (++chk_cnt < MAC_RX_FULL_DROP_CHK_MAX_NUM);
+
+	if (prxff_hang_info->rx_ff_hang_cnt >= HANG_DETECT_THR)
+		prxff_hang_info->dbg_is_rxff_hang = _TRUE;
+}
+
+void rtw_is_fw_hang(_adapter *padapter, struct fw_hang_info *pfw_hang_info)
+{
+	struct dvobj_priv *pdvobjpriv = padapter->dvobj;
+	void *phl = GET_PHL_INFO(pdvobjpriv);
+	enum rtw_fw_status fw_sts;
+
+	fw_sts = rtw_phl_get_fw_status(phl);
+
+	if (fw_sts == RTW_FW_STATUS_NOFW) {
+		pfw_hang_info->dbg_is_fw_gone = _TRUE;
+		pfw_hang_info->dbg_is_fw_hang = _FALSE;
+	} else {
+		pfw_hang_info->dbg_is_fw_gone = _FALSE;
+
+		if (fw_sts == RTW_FW_STATUS_ASSERT ||
+		    fw_sts == RTW_FW_STATUS_EXCEP ||
+		    fw_sts == RTW_FW_STATUS_RXI300 ||
+		    fw_sts == RTW_FW_STATUS_HANG)
+			pfw_hang_info->dbg_is_fw_hang = _TRUE;
+		else
+			pfw_hang_info->dbg_is_fw_hang = _FALSE;
+	}
+}
+
+void rtw_is_hang_check(_adapter *padapter)
+{
+	u32 start_time = rtw_get_current_time();
+	struct dvobj_priv *pdvobjpriv = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &pdvobjpriv->drv_dbg;
+	struct hang_info *phang_info = &pdbgpriv->dbg_hang_info;
+	/* struct fw_hang_info *pfw_hang_info = &phang_info->dbg_fw_hang_info; */
+	struct rxff_hang_info *prxff_hang_info = &phang_info->dbg_rxff_hang_info;
+	struct fw_hang_info *pfw_hang_info = &phang_info->dbg_fw_hang_info;
+	u8 is_fw_in_ps_mode = _FALSE;
+	u8 is_fw_ps_awake = _TRUE;
+
+	if (phang_info->is_stop)
+		return;
+
+	if (rtw_hw_get_init_completed(pdvobjpriv) && (!is_fw_in_ps_mode) &&
+	    is_fw_ps_awake) {
+		phang_info->enter_cnt++;
+
+		rtw_is_rxff_hang(padapter, prxff_hang_info);
+		rtw_is_fw_hang(padapter, pfw_hang_info);
+	}
+}
+#endif /* RTW_DETECT_HANG */
+
+static u8 target_drv2phl(enum _HT_IOT_PEER drv)
+{
+	switch (drv) {
+	case HT_IOT_PEER_UNKNOWN:
+	default:
+		return 0;
+	}
+
+	return 0;
+}
+
+void rtw_update_phl_iot(struct _ADAPTER *a, enum _HT_IOT_PEER peer)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	u32 iot;
+
+
+	iot = IOT_ID(target_drv2phl(peer));
+	phl_com->id.iot_id[a->phl_role->id] = iot;
+	RTW_INFO(FUNC_ADPT_FMT ": iot = 0x%08x\n", FUNC_ADPT_ARG(a), iot);
+}
+
+#ifdef CONFIG_RTW_ACS
+u8 rtw_acs_get_clm_ratio(struct _ADAPTER *a, enum band_type band, u8 ch)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_acs_info_parm parm = {0};
+	enum rtw_phl_status rtn;
+
+	parm.idx = rtw_phl_get_acs_chnl_tbl_idx(d->phl, band, ch);
+	rtn = rtw_phl_get_acs_info(d->phl, &parm);
+
+	if (rtn == RTW_PHL_STATUS_SUCCESS)
+		return parm.rpt.clm_ratio;
+
+	return 0;
+}
+
+u8 rtw_acs_get_nhm_ratio(struct _ADAPTER *a, enum band_type band, u8 ch)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_acs_info_parm parm = {0};
+	enum rtw_phl_status rtn;
+
+	parm.idx = rtw_phl_get_acs_chnl_tbl_idx(d->phl, band, ch);
+	rtn = rtw_phl_get_acs_info(d->phl, &parm);
+
+	if (rtn == RTW_PHL_STATUS_SUCCESS)
+		return parm.rpt.nhm_ratio;
+
+	return 0;
+}
+
+s8 rtw_acs_get_noise_dbm(struct _ADAPTER *a, enum band_type band, u8 ch)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_acs_info_parm parm = {0};
+	enum rtw_phl_status rtn;
+
+	parm.idx = rtw_phl_get_acs_chnl_tbl_idx(d->phl, band, ch);
+	rtn = rtw_phl_get_acs_info(d->phl, &parm);
+
+	if (rtn == RTW_PHL_STATUS_SUCCESS) {
+		parm.rpt.nhm_pwr -= 110; /* to dBm */
+		return parm.rpt.nhm_pwr;
+	}
+
+	return -110;
+}
+
+int rtw_acs_get_report(struct _ADAPTER *a, enum band_type band, u8 ch, struct rtw_acs_info_parm *rpt)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	enum rtw_phl_status rtn;
+
+	rpt->idx = rtw_phl_get_acs_chnl_tbl_idx(d->phl, band, ch);
+	rtn = rtw_phl_get_acs_info(d->phl, rpt);
+
+	if (rtn != RTW_PHL_STATUS_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+#endif /* CONFIG_RTW_ACS */
+
+void rtw_dump_env_rpt(struct _ADAPTER *a, void *sel)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(d);
+	void *phl = GET_PHL_INFO(d);
+	struct rtw_env_report rpt;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	rtw_phl_get_env_rpt(phl, &rpt, alink->wrlink->hw_band);
+
+	RTW_PRINT_SEL(sel, "tx_ratio:%d (%%)\n", rpt.nhm_tx_ratio);
+	RTW_PRINT_SEL(sel, "clm_ratio:%d (%%)\n", rpt.nhm_cca_ratio);
+	RTW_PRINT_SEL(sel, "nhm_ratio:%d (%%)\n", rpt.nhm_ratio);
+	RTW_PRINT_SEL(sel, "nhm:%d (dBm)\n", (rpt.nhm_pwr - 110));
+}
+
+#ifdef DBG_RX_DFRAME_RAW_DATA
+void rtw_dump_rx_dframe_info(struct _ADAPTER *padapter, void *sel)
+{
+#define DBG_RX_DFRAME_RAW_DATA_UC	0
+#define DBG_RX_DFRAME_RAW_DATA_BMC	1
+#define DBG_RX_DFRAME_RAW_DATA_TYPES	2
+
+	struct dvobj_priv *devob = adapter_to_dvobj(padapter);
+	struct recv_info *recvinfo = &padapter->recvinfo;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	struct sta_recv_dframe_info *psta_dframe_info;
+	_list *plist, *phead;
+	u8 isCCKrate, rf_path;
+	int i, j;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	char mac_addr_str[MAC_FMT_LEN];
+
+	if (recvinfo->store_law_data_flag) {
+
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+		for (i = 0; i < NUM_STA; i++) {
+			phead = &(pstapriv->sta_hash[i]);
+			plist = get_next(phead);
+
+			while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+				psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+				plist = get_next(plist);
+
+				if (psta && psta->phl_sta) {
+					if ((_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, ETH_ALEN)  !=   _TRUE)
+					    && (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, ETH_ALEN)  !=  _TRUE)
+					    && (_rtw_memcmp(psta->phl_sta->mac_addr, adapter_mac_addr(padapter), ETH_ALEN)  !=  _TRUE)) {
+
+						RTW_PRINT_SEL(sel, "==============================\n");
+						RTW_PRINT_SEL(sel, "macaddr = %s\n", get_macaddr_str(mac_addr_str, sel, psta->phl_sta->mac_addr));
+
+						for (j = 0; j < DBG_RX_DFRAME_RAW_DATA_TYPES; j++) {
+							if (j == DBG_RX_DFRAME_RAW_DATA_UC) {
+								psta_dframe_info = &psta->sta_dframe_info;
+								RTW_PRINT_SEL(sel, "\n");
+								RTW_PRINT_SEL(sel, "Unicast:\n");
+							} else if (j == DBG_RX_DFRAME_RAW_DATA_BMC) {
+								psta_dframe_info = &psta->sta_dframe_info_bmc;
+								RTW_PRINT_SEL(sel, "\n");
+								RTW_PRINT_SEL(sel, "Broadcast/Multicast:\n");
+							}
+
+							isCCKrate = (psta_dframe_info->sta_data_rate <= DESC_RATE11M) ? _TRUE : _FALSE;
+
+							RTW_PRINT_SEL(sel, "BW=%s, sgi =%d\n", ch_width_str(psta_dframe_info->sta_bw_mode), psta_dframe_info->sta_sgi);
+							RTW_PRINT_SEL(sel, "Rx_Data_Rate = %s\n", HDATA_RATE(psta_dframe_info->sta_data_rate));
+
+							for (rf_path = 0; rf_path < GET_HAL_RFPATH_NUM(devob); rf_path++) {
+								if (!isCCKrate) {
+									RTW_PRINT_SEL(sel , "RF_PATH_%d RSSI:%d(dBm)", rf_path, psta_dframe_info->sta_RxPwr[rf_path]);
+									RTW_PRINT_SEL(sel, "\nrx_ofdm_snr:%d(dB)\n", psta_dframe_info->sta_ofdm_snr[rf_path]);
+								} else
+									RTW_PRINT_SEL(sel , "RF_PATH_%d RSSI:%d(dBm)\n", rf_path, (psta_dframe_info->sta_mimo_signal_strength[rf_path]));
+							}
+						}
+					}
+				}
+			}
+		}
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	}
+}
+#endif
+
+void rtw_dump_rfe_type(struct dvobj_priv *d)
+{
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(d);
+
+	RTW_INFO("RFE Type: %d\n", phl_com->dev_cap.rfe_type);
+}
+
+#ifdef CONFIG_WOWLAN
+static u8 _cfg_keep_alive_info(struct _ADAPTER *a, u8 enable)
+{
+	struct rtw_keep_alive_info info;
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	u8 check_period = 5;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	_rtw_memset(&info, 0, sizeof(struct rtw_keep_alive_info));
+
+	if (enable) {
+		info.keep_alive_en = enable;
+		info.keep_alive_period = check_period;
+#ifdef CONFIG_ARP_KEEP_ALIVE
+		/* Require arp_en = 1 in _cfg_arp_ofld_info() */
+		info.keep_alive_pkt_type = PKT_TYPE_ARP_RSP;
+#else
+		info.keep_alive_pkt_type = PKT_TYPE_NULL_DATA;
+#endif
+
+		RTW_INFO("%s: keep_alive_en=%d, keep_alive_period=%d, keep_alive_pkt_type=%d\n",
+			 __func__, info.keep_alive_en, info.keep_alive_period,
+			 info.keep_alive_pkt_type);
+
+		status = rtw_phl_cfg_keep_alive_info(phl, &info);
+		if (status != RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s fail(%d)\n", __func__, status);
+			return _FAIL;
+		}
+	}
+
+	return _SUCCESS;
+}
+
+static u8 _cfg_disc_det_info(struct _ADAPTER *a, u8 enable)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(a);
+	struct rtw_disc_det_info *wow_disc = &wowpriv->wow_disc;
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	struct registry_priv *registry_par;
+	u8 check_period = 100, trypkt_num = 5;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	registry_par = &a->registrypriv;
+
+	wow_disc->disc_det_en = enable;
+
+	/* wake up event includes deauth wake up */
+	if (registry_par->wakeup_event & BIT(2))
+		wow_disc->disc_wake_en = _TRUE;
+	else
+		wow_disc->disc_wake_en = _FALSE;
+	wow_disc->try_pkt_count = trypkt_num;
+	wow_disc->check_period = check_period;
+
+	wow_disc->cnt_bcn_lost_en = 0;
+	wow_disc->cnt_bcn_lost_limit = 0;
+
+	status = rtw_phl_cfg_disc_det_info(phl, wow_disc);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("%s fail(%d)\n", __func__, status);
+		return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+static u8 _cfg_arp_ofld_info(struct _ADAPTER *a)
+{
+	struct rtw_arp_ofld_info info;
+	struct dvobj_priv *d;
+	void *phl;
+	struct mlme_ext_priv *pmlmeext = &(a->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct mlme_priv *pmlmepriv = &(a->mlmepriv);
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	_rtw_memset(&info, 0, sizeof(struct rtw_arp_ofld_info));
+
+	info.arp_en = 1;
+
+	if (info.arp_en) {
+		/* Sender IP address */
+		_rtw_memcpy(info.arp_ofld_content.host_ipv4_addr,
+			    pmlmeinfo->ip_addr, IPV4_ADDRESS_LENGTH);
+
+#ifdef CONFIG_ARP_KEEP_ALIVE
+		/*
+		 * Driver only needs to fill in the Target IP address and Target
+		 * MAC address when ARP keepalive is enabled. In general, when
+		 * the FW receives an ARP request, it will use the Sender IP
+		 * address and Sender MAC address of the ARP request as the
+		 * destination IP address and destination MAC address of the ARP
+		 * response.
+		 */
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+		if (!is_zero_mac_addr(pmlmepriv->gw_mac_addr)) {
+			RTW_INFO("%s: gw arp keepalive enabled\n", __func__);
+
+			/* Gateway MAC address of A3 */
+			_rtw_memcpy(info.arp_ofld_content.a3,
+				    pmlmepriv->gw_mac_addr, MAC_ADDRESS_LENGTH);
+
+			/* Gateway Target IP address */
+			_rtw_memcpy(info.arp_ofld_content.remote_ipv4_addr,
+				    pmlmepriv->gw_ip, IPV4_ADDRESS_LENGTH);
+
+			/* Gateway Target MAC address */
+			_rtw_memcpy(info.arp_ofld_content.remote_mac_addr,
+				    pmlmepriv->gw_mac_addr, MAC_ADDRESS_LENGTH);
+		} else
+#endif /* CONFIG_ARP_KEEP_ALIVE_GW */
+		{
+			/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+			struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+			struct rtw_phl_stainfo_t *psta = NULL;
+
+			RTW_INFO("%s: ap arp keepalive enabled\n", __func__);
+
+			psta = rtw_phl_get_stainfo_self(phl, alink->wrlink);
+
+			/* AP MAC address of A3 */
+			_rtw_memcpy(info.arp_ofld_content.a3,
+				    psta->mac_addr, MAC_ADDRESS_LENGTH);
+
+			/* STA Target IP address */
+			_rtw_memcpy(info.arp_ofld_content.remote_ipv4_addr,
+				    pmlmeinfo->ip_addr, IPV4_ADDRESS_LENGTH);
+
+			/* STA Target MAC address */
+			_rtw_memcpy(info.arp_ofld_content.remote_mac_addr,
+				    alink->mac_addr, MAC_ADDRESS_LENGTH);
+		}
+#endif /* CONFIG_ARP_KEEP_ALIVE */
+	}
+
+	rtw_phl_cfg_arp_ofld_info(phl, &info);
+
+	return _SUCCESS;
+}
+
+static u8 _cfg_ndp_ofld_info(struct _ADAPTER *a)
+{
+	struct rtw_ndp_ofld_info info;
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	_rtw_memset(&info, 0, sizeof(struct rtw_ndp_ofld_info));
+
+	rtw_phl_cfg_ndp_ofld_info(phl, &info);
+
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_GTK_OL
+static u8 _cfg_gtk_ofld_info(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	struct rtw_gtk_ofld_info gtk_ofld_info = {0};
+	struct rtw_gtk_ofld_content *gtk_ofld_content = NULL;
+	struct security_priv *securitypriv = &a->securitypriv;
+	struct sta_info *sta = NULL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	sta = rtw_get_stainfo(&a->stapriv, get_bssid(&a->mlmepriv));
+	gtk_ofld_content = &gtk_ofld_info.gtk_ofld_content;
+
+	if (securitypriv->binstallKCK_KEK) {
+		gtk_ofld_info.gtk_en = _TRUE;
+
+		gtk_ofld_info.akmtype_byte3 = securitypriv->rsn_akm_suite_type;
+
+		gtk_ofld_content->kck_len = RTW_KCK_LEN;
+		_rtw_memcpy(gtk_ofld_content->kck, sta->kck, RTW_KCK_LEN);
+
+		gtk_ofld_content->kek_len = RTW_KEK_LEN;
+		_rtw_memcpy(gtk_ofld_content->kek, sta->kek, RTW_KEK_LEN);
+
+		if (securitypriv->dot11PrivacyAlgrthm == _TKIP_) {
+			gtk_ofld_info.tkip_en = _TRUE;
+			/* The driver offloads the Tx MIC key here, which is
+			 * actually the Rx MIC key, but the driver definition is
+			 * the opposite of the correct definition.
+			 */
+			_rtw_memcpy(gtk_ofld_content->rxmickey,
+				    sta->dot11tkiptxmickey.skey, RTW_TKIP_MIC_LEN);
+		}
+
+		_rtw_memcpy(gtk_ofld_content->replay_cnt, sta->replay_ctr,
+			    RTW_REPLAY_CTR_LEN);
+	}
+
+#ifdef CONFIG_IEEE80211W
+	if (SEC_IS_BIP_KEY_INSTALLED(&alink->securitypriv)) {
+		gtk_ofld_info.ieee80211w_en = 1;
+		RTW_PUT_LE32(gtk_ofld_content->igtk_keyid,
+			     alink->securitypriv.dot11wBIPKeyid);
+		RTW_PUT_LE64(gtk_ofld_content->ipn,
+			     alink->securitypriv.dot11wBIPrxpn.val);
+		_rtw_memcpy(gtk_ofld_content->igtk[0],
+			    alink->securitypriv.dot11wBIPKey[4].skey, RTW_IGTK_LEN);
+		_rtw_memcpy(gtk_ofld_content->igtk[1],
+			    alink->securitypriv.dot11wBIPKey[5].skey, RTW_IGTK_LEN);
+		gtk_ofld_content->igtk_len = RTW_IGTK_LEN;
+
+		_rtw_memcpy(gtk_ofld_content->psk,
+			    sta->dot118021x_UncstKey.skey, RTW_PTK_LEN);
+		gtk_ofld_content->psk_len = RTW_PTK_LEN;
+	}
+#endif
+
+	rtw_phl_cfg_gtk_ofld_info(phl, &gtk_ofld_info);
+
+	return _SUCCESS;
+}
+#endif
+
+#ifdef CONFIG_MDNS_OFFLOAD
+static u8 _cfg_mdns_ofld_info(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	struct mlme_ext_priv *pmlmeext = &(a->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct wow_priv *wowpriv = adapter_to_wowlan(a);
+	struct rtw_mdns_ofld_info *info = &wowpriv->mdns_ofld_info;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	info->mdns_en = 1;
+
+	if (info->mdns_en) {
+		/* Sender IP address */
+		if (!is_all_null(pmlmeinfo->ip_addr, RTW_IP_ADDR_LEN))
+			_rtw_memcpy(info->mdns_ipv4_header.src_ipv4_addr,
+			            pmlmeinfo->ip_addr, IPV4_ADDRESS_LENGTH);
+#ifdef CONFIG_IPV6
+		if (!is_all_null(pmlmeinfo->ip6_addr, RTW_IPv6_ADDR_LEN))
+			_rtw_memcpy(info->mdns_ipv6_header.src_ipv6_addr,
+			            pmlmeinfo->ip6_addr, IPV6_ADDRESS_LENGTH);
+#endif
+	}
+	rtw_phl_cfg_mdns_ofld_info(phl, info);
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_MDNS_OFFLOAD */
+
+#ifdef CONFIG_WOW_APF
+static u8 _cfg_apf_info(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	struct mlme_ext_priv *pmlmeext = &(a->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct wow_priv *wowpriv = adapter_to_wowlan(a);
+	struct rtw_apf_info *info = &wowpriv->apf_info;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	rtw_phl_cfg_apf_info(phl, info);
+	return _SUCCESS;
+}
+#endif /* CONFIG_WOW_APF */
+
+static u8 _cfg_realwow_info(struct _ADAPTER *a)
+{
+	struct rtw_realwow_info info;
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	/* default disable */
+	_rtw_memset(&info, 0, sizeof(struct rtw_realwow_info));
+	status = rtw_phl_cfg_realwow_info(phl, &info);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("%s fail(%d)\n", __func__, status);
+		return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+static u8 _cfg_wow_wake(struct _ADAPTER *a, u8 wow_en)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	struct rtw_wow_wake_info wow_wake_event = {0};
+	struct security_priv *securitypriv;
+	struct registry_priv  *registry_par = &a->registrypriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	securitypriv = &a->securitypriv;
+
+	wow_wake_event.wow_en = _TRUE;
+	/* wake up by magic packet */
+	if (registry_par->wakeup_event & BIT(0))
+		wow_wake_event.magic_pkt_en = _TRUE;
+	else
+		wow_wake_event.magic_pkt_en = _FALSE;
+	/* wake up by deauth packet */
+	if (registry_par->wakeup_event & BIT(2))
+		wow_wake_event.deauth_wakeup = _TRUE;
+	else
+		wow_wake_event.deauth_wakeup = _FALSE;
+	/* wake up by pattern match packet */
+	if ((registry_par->wakeup_event & (BIT(1) | BIT(3)))
+#ifdef CONFIG_GOOGLE_CAST_WAKEUP
+		|| _TRUE
+#endif
+	) {
+		wow_wake_event.pattern_match_en = _TRUE;
+
+		rtw_wow_pattern_clean(a, RTW_DEFAULT_PATTERN);
+#ifdef CONFIG_GOOGLE_CAST_WAKEUP
+		rtw_set_google_cast_mdns_wow_pattern(a);
+#endif
+		if (registry_par->wakeup_event & BIT(1))
+			rtw_set_default_pattern(a);
+
+		if (!(registry_par->wakeup_event & BIT(3)))
+			rtw_wow_pattern_clean(a, RTW_CUSTOMIZED_PATTERN);
+	} else {
+		wow_wake_event.pattern_match_en = _FALSE;
+	}
+	/* wake up by ptk rekey */
+	if (registry_par->wakeup_event & BIT(4))
+		wow_wake_event.rekey_wakeup = _TRUE;
+	else
+		wow_wake_event.rekey_wakeup = _FALSE;
+
+	wow_wake_event.pairwise_sec_algo = rtw_sec_algo_drv2phl(securitypriv->dot11PrivacyAlgrthm);
+	wow_wake_event.group_sec_algo = rtw_sec_algo_drv2phl(securitypriv->dot118021XGrpPrivacy);
+#ifdef CONFIG_IEEE80211W
+	if (SEC_IS_BIP_KEY_INSTALLED(&alink->securitypriv))
+		wow_wake_event.bip_sec_algo = rtw_sec_algo_drv2phl(securitypriv->dot11wCipher);
+#endif
+
+	rtw_construct_remote_control_info(a, &wow_wake_event.remote_wake_ctrl_info);
+
+	status = rtw_phl_cfg_wow_wake(phl, &wow_wake_event);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("%s fail(%d)\n", __func__, status);
+		return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_PNO_SUPPORT
+static u8 _cfg_nlo_info(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	struct wow_priv *wowpriv;
+	void *phl;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	wowpriv = adapter_to_wowlan(a);
+
+	rtw_phl_cfg_nlo_info(phl, &wowpriv->wow_nlo);
+
+	return _SUCCESS;
+}
+#endif
+
+static u8 _cfg_wow_wake_no_link(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	struct rtw_wow_wake_info wow_wake_event = {0};
+	struct registry_priv  *registry_par = &a->registrypriv;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	wow_wake_event.wow_en = _TRUE;
+#ifdef CONFIG_WRC_WOW_MAGIC
+	/* wake up by magic packet */
+	if (registry_par->wakeup_event & BIT(0))
+		wow_wake_event.magic_pkt_en = _TRUE;
+	else
+		wow_wake_event.magic_pkt_en = _FALSE;
+#endif
+
+	status = rtw_phl_cfg_wow_wake(phl, &wow_wake_event);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("%s fail(%d)\n", __func__, status);
+		return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+static u8 _cfg_wow_gpio(struct _ADAPTER *a)
+{
+	struct rtw_wow_wake_info info = {0};
+	struct dvobj_priv *d;
+	void *phl;
+	enum rtw_phl_status status;
+	struct wow_priv *wowpriv = adapter_to_wowlan(a);
+	struct rtw_wow_gpio_info *wow_gpio = &wowpriv->wow_gpio;
+	struct registry_priv  *registry_par = &a->registrypriv;
+	struct rtw_dev2hst_gpio_info *d2h_gpio_info = &wow_gpio->d2h_gpio_info;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+#ifdef CONFIG_GPIO_WAKEUP
+	d2h_gpio_info->dev2hst_gpio_en = _TRUE;
+
+	/* ToDo: fw/halmac do not support so far
+	pwrctrlpriv->hst2dev_high_active = HIGH_ACTIVE_HST2DEV;
+	*/
+#ifdef CONFIG_RTW_ONE_PIN_GPIO
+	wow_gpio->dev2hst_gpio_mode = RTW_AX_SW_IO_MODE_INPUT;
+	status = rtw_phl_cfg_wow_set_sw_gpio_mode(phl, gpio);
+#else
+	#ifdef CONFIG_WAKEUP_GPIO_INPUT_MODE
+	wow_gpio->dev2hst_gpio_mode = RTW_AX_SW_IO_MODE_OUTPUT_OD;
+	d2h_gpio_info->gpio_output_input = _TRUE;
+	#else
+	wow_gpio->dev2hst_gpio_mode = RTW_AX_SW_IO_MODE_OUTPUT_PP;
+	d2h_gpio_info->gpio_output_input = _FALSE;
+	#endif /*CONFIG_WAKEUP_GPIO_INPUT_MODE*/
+	/* switch GPIO to open-drain or push-pull */
+	status = rtw_phl_cfg_wow_set_sw_gpio_mode(phl, wow_gpio);
+	/*default low active, gpio_active and dev2hst_high is the same thing
+	, but two halmac implementation. FW and halmac need to refine */
+	status = rtw_phl_cfg_wow_sw_gpio_ctrl(phl, wow_gpio);
+	RTW_INFO("%s: set GPIO_%d %d as default. status=%d\n",
+		 __func__, WAKEUP_GPIO_IDX, wow_gpio->dev2hst_high, status);
+#endif /* CONFIG_RTW_ONE_PIN_GPIO */
+
+	/* SDIO inband wake sdio_wakeup_enable
+	d2h_gpio_info->data_pin_wakeup = info->data_pin_wakeup;
+	*/
+	/* two halmac implementation. FW and halmac need to refine */
+	wow_gpio->dev2hst_gpio = WAKEUP_GPIO_IDX;
+	d2h_gpio_info->gpio_num = WAKEUP_GPIO_IDX;
+#if defined(CONFIG_USB_HCI) && !defined(CONFIG_USB_INBAND)
+	d2h_gpio_info->disable_inband = _TRUE;
+#endif
+
+	status = rtw_phl_cfg_gpio_wake_pulse(phl, wow_gpio);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("%s fail(%d)\n", __func__, status);
+		return _FAIL;
+	}
+#endif /* CONFIG_GPIO_WAKEUP */
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+static u8 _cfg_periodic_wake(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d;
+	struct wow_priv *wowpriv;
+	void *phl;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+	wowpriv = adapter_to_wowlan(a);
+
+	/* wake_period and wake_duration are initialized in rtw_init_wow() */
+	wowpriv->wow_periodic_wake.periodic_wake_en = _TRUE;
+
+	rtw_phl_cfg_periodic_wake_info(phl, &wowpriv->wow_periodic_wake);
+
+	return _SUCCESS;
+}
+#endif
+
+static u8 _wow_cfg(struct _ADAPTER *a, u8 wow_en, u8 no_link_mode)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	struct rtw_phl_stainfo_t *phl_sta;
+	enum rtw_phl_status status;
+
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	if (!no_link_mode) {
+		if (!_cfg_keep_alive_info(a, wow_en))
+			return _FAIL;
+
+		if(!_cfg_disc_det_info(a, wow_en))
+			return _FAIL;
+
+		if (!_cfg_arp_ofld_info(a))
+			return _FAIL;
+
+		if (!_cfg_ndp_ofld_info(a))
+			return _FAIL;
+
+#ifdef CONFIG_GTK_OL
+		if (!_cfg_gtk_ofld_info(a))
+			return _FAIL;
+#endif
+
+#ifdef CONFIG_MDNS_OFFLOAD
+		if (!_cfg_mdns_ofld_info(a))
+			return _FAIL;
+#endif
+
+#ifdef CONFIG_WOW_APF
+		if (!_cfg_apf_info(a))
+			return _FAIL;
+#endif
+
+		if (!_cfg_realwow_info(a))
+			return _FAIL;
+
+		if (!_cfg_wow_wake(a, wow_en))
+			return _FAIL;
+	} else {
+#ifdef CONFIG_PNO_SUPPORT
+		if (!_cfg_nlo_info(a))
+			return _FAIL;
+#endif
+		if (!_cfg_wow_wake_no_link(a))
+			return _FAIL;
+	}
+
+	if(!_cfg_wow_gpio(a))
+		return _FAIL;
+
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+	if (!_cfg_periodic_wake(a))
+		return _FAIL;
+#endif
+
+	return _SUCCESS;
+}
+
+u8 rtw_hw_wow(struct _ADAPTER *a, u8 wow_en)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	struct pwrctrl_priv *pwrpriv;
+	struct rtw_phl_stainfo_t *phl_sta;
+	enum rtw_phl_status status;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	pwrpriv = adapter_to_pwrctl(a);
+	d = adapter_to_dvobj(a);
+	phl = GET_PHL_INFO(d);
+
+	if (!_wow_cfg(a, wow_en, pwrpriv->wowlan_no_link_mode))
+		return _FAIL;
+
+	rtw_phl_wow_set_no_link_mode(phl, pwrpriv->wowlan_no_link_mode);
+
+	phl_sta = rtw_phl_get_stainfo_self(phl, alink->wrlink);
+
+	if (wow_en) {
+#ifdef CONFIG_WRC_WOW_MAGIC
+		rtw_cfg_wrc_wol_magic(a, _TRUE);
+#endif
+		status = rtw_phl_suspend(phl, phl_sta, wow_en);
+	} else {
+		status = rtw_phl_resume(phl, phl_sta, &wow_en);
+#ifdef CONFIG_WRC_WOW_MAGIC
+		rtw_cfg_wrc_wol_magic(a, _FALSE);
+#endif
+	}
+
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s wow %s fail(status: %d)\n", __func__, wow_en ? "enable" : "disable", status);
+		return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_WOWLAN */
+
+static u32 rtw_tx_sts_total(u32 *tx_sts, u8 num)
+{
+	u32 ret = 0;
+	int i = 0;
+
+	for (i = 0; i < num; i++)
+		ret += tx_sts[i];
+	return ret;
+}
+
+int rtw_get_sta_tx_stat(_adapter *adapter, struct sta_info *psta)
+{
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_PCI_HCI)
+	struct stainfo_stats	*pstats = NULL;
+
+	u32 tx_retry_cnt[PHL_AC_QUEUE_TOTAL] = {0};
+	u32 tx_fail_cnt[PHL_AC_QUEUE_TOTAL] = {0};
+	u32 tx_ok_cnt[PHL_AC_QUEUE_TOTAL] = {0};
+
+	rtw_phl_get_tx_ra_retry_rpt(GET_PHL_INFO(adapter_to_dvobj(adapter)),
+				    psta->phl_sta, tx_retry_cnt,
+				    PHL_AC_QUEUE_TOTAL, 1);
+	rtw_phl_get_tx_fail_rpt(GET_PHL_INFO(adapter_to_dvobj(adapter)), psta->phl_sta,
+		tx_fail_cnt, PHL_AC_QUEUE_TOTAL);
+	rtw_phl_get_tx_ok_rpt(GET_PHL_INFO(adapter_to_dvobj(adapter)), psta->phl_sta,
+		tx_ok_cnt, PHL_AC_QUEUE_TOTAL);
+	pstats = &psta->sta_stats;
+	pstats->tx_retry_cnt = rtw_tx_sts_total(tx_retry_cnt, PHL_AC_QUEUE_TOTAL);
+	pstats->tx_fail_cnt = rtw_tx_sts_total(tx_fail_cnt, PHL_AC_QUEUE_TOTAL);
+	pstats->tx_ok_cnt =  rtw_tx_sts_total(tx_ok_cnt, PHL_AC_QUEUE_TOTAL);
+	pstats->total_tx_retry_cnt += pstats->tx_retry_cnt;
+
+	pstats->tx_fail_cnt_sum += pstats->tx_fail_cnt;
+	pstats->tx_retry_cnt_sum += pstats->tx_retry_cnt;
+
+	return _SUCCESS;
+#else
+	return RTW_NOT_SUPPORT;
+#endif
+}
+
+static enum rtw_edcca_mode rtw_edcca_mode_to_phl(enum rtw_edcca_mode_t mode)
+{
+	switch (mode) {
+	case RTW_EDCCA_NORM:
+		return RTW_EDCCA_NORMAL;
+	case RTW_EDCCA_ADAPT:
+		return RTW_EDCCA_ETSI;
+	case RTW_EDCCA_CS:
+		return RTW_EDCCA_JP;
+	case RTW_EDCCA_CBP:
+		return RTW_EDCCA_FCC;
+	default:
+		return RTW_EDCCA_MAX;
+	}
+}
+
+static bool rtw_edcca_hal_mode_supported(struct dvobj_priv* dvobj, enum rtw_edcca_mode_t mode)
+{
+	return rtw_edcca_mode_to_phl(mode) != RTW_EDCCA_MAX;
+}
+
+void rtw_edcca_hal_update(struct dvobj_priv *dvobj)
+{
+	void *phl = GET_PHL_INFO(dvobj);
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	enum phl_band_idx band_idx = HW_BAND_0; /* TODO: DBCC */
+	struct rtw_chan_def chdef;
+	int chctx_num;
+	enum band_type band;
+	u8 mode = RTW_EDCCA_NORM;
+	enum rtw_edcca_mode phl_mode = rtw_edcca_mode_to_phl(mode);
+
+	chctx_num = rtw_phl_mr_get_chandef_by_hwband(phl, band_idx, &chdef);
+	if (chctx_num > 1) {
+		RTW_WARN("%s can't handle MCC case\n", __func__);
+		goto exit;
+	}
+
+	if (chdef.chan != 0) {
+		band = chdef.band;
+		rfctl->last_edcca_mode_op_band = band;
+	} else if (rfctl->last_edcca_mode_op_band != BAND_MAX)
+		band = rfctl->last_edcca_mode_op_band;
+	else {
+		rtw_phl_get_cur_hal_chdef_by_hwband(phl, band_idx, &chdef);
+		band = chdef.band;
+	}
+
+	mode = rtw_get_edcca_mode(dvobj, band);
+	/*
+	* may get band not existing in current channel plan
+	* then edcca mode RTW_EDCCA_MODE_NUM is got
+	* this is not a real problem because this band is not used for TX
+	* change to RTW_EDCCA_NORM to avoid warning calltrace below
+	*/
+	if (mode == RTW_EDCCA_MODE_NUM)
+		mode = RTW_EDCCA_NORM;
+
+	if (!rtw_edcca_hal_mode_supported(dvobj, mode)) {
+		RTW_WARN("%s edcca mode %s is not supported by HAL, set to %s\n", __func__
+			, rtw_edcca_mode_str(mode), rtw_edcca_mode_str(RTW_EDCCA_NORM));
+		mode = RTW_EDCCA_NORM;
+	}
+
+	phl_mode = rtw_edcca_mode_to_phl(mode);
+
+exit:
+	if (rtw_phl_get_edcca_mode(phl) != phl_mode)
+		rtw_phl_set_edcca_mode(phl, phl_mode);
+}
+
+#if CONFIG_TXPWR_LIMIT
+enum txpwr_lmt_reg_exc_match _rtw_txpwr_hal_lmt_reg_exc_search(struct dvobj_priv* dvobj
+	, enum band_type band, const char *country, u8 domain, const char **reg_name)
+{
+	u8 hal_match;
+
+	hal_match = rtw_phl_ext_reg_codemap_of_band_search(GET_PHL_INFO(dvobj)
+		, band, domain, country, reg_name);
+
+	if (hal_match & RTW_PHL_EXT_REG_MATCH_COUNTRY)
+		return TXPWR_LMT_REG_EXC_MATCH_COUNTRY;
+	if (hal_match & RTW_PHL_EXT_REG_MATCH_DOMAIN)
+		return TXPWR_LMT_REG_EXC_MATCH_DOMAIN;
+	return TXPWR_LMT_REG_EXC_MATCH_NONE;
+}
+
+enum txpwr_lmt_reg_exc_match rtw_txpwr_hal_lmt_reg_exc_search(struct dvobj_priv* dvobj
+	, const char *country, u8 domain, const char **reg_name)
+{
+	return _rtw_txpwr_hal_lmt_reg_exc_search(dvobj, BAND_ON_24G, country, domain, reg_name);
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum txpwr_lmt_reg_exc_match rtw_txpwr_hal_lmt_reg_exc_6g_search(struct dvobj_priv* dvobj
+	, const char *country, u8 domain, const char **reg_name)
+{
+	return _rtw_txpwr_hal_lmt_reg_exc_search(dvobj, BAND_ON_6G, country, domain, reg_name);
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+bool rtw_txpwr_hal_lmt_reg_search(struct dvobj_priv* dvobj, enum band_type band, const char *name)
+{
+	int hal_regu = rtw_phl_get_pw_lmt_regu_type_of_band_from_str(GET_PHL_INFO(dvobj), band, name);
+
+	if (hal_regu == -1)
+		return false;
+
+	if (strcmp(name, txpwr_lmt_str(TXPWR_LMT_NONE)) == 0
+		|| strcmp(name, txpwr_lmt_str(TXPWR_LMT_WW)) == 0)
+		return true;
+
+	return rtw_phl_pw_lmt_regu_tbl_exist(GET_PHL_INFO(dvobj), band, hal_regu);
+}
+
+void rtw_txpwr_hal_set_current_lmt_regs_by_name(struct dvobj_priv* dvobj, char *names_of_band[], int names_len_of_band[])
+{
+	struct txpwr_regu_info_t hal_conf;
+	enum band_type band;
+	int hal_regu, hal_none_regu;
+	char *name;
+	u8 *regu;
+	u8 regu_num;
+
+	_rtw_memset(&hal_conf, 0, sizeof(hal_conf));
+
+	hal_conf.force = true;
+	for (band = 0; band < BAND_MAX; band++) {
+		if (!names_of_band[band] || !names_len_of_band[band])
+			continue;
+
+		hal_none_regu = rtw_phl_get_pw_lmt_regu_type_of_band_from_str(GET_PHL_INFO(dvobj), band, "NONE");
+		regu_num = 0;
+		ustrs_for_each_str(names_of_band[band], names_len_of_band[band], name)
+				regu_num++;
+		if (!regu_num)
+			continue;
+
+		regu = rtw_malloc(regu_num);
+		if (!regu)
+			break;
+
+		if (band == BAND_ON_24G) {
+			hal_conf.regu_2g = regu;
+			hal_conf.regu_2g_len = regu_num;
+		} else if (band == BAND_ON_5G) {
+			hal_conf.regu_5g = regu;
+			hal_conf.regu_5g_len = regu_num;
+		} else if (band == BAND_ON_6G) {
+			hal_conf.regu_6g = regu;
+			hal_conf.regu_6g_len = regu_num;
+		} else {
+			rtw_mfree(regu, regu_num);
+			continue;
+		}
+
+		regu_num = 0;
+		ustrs_for_each_str(names_of_band[band], names_len_of_band[band], name) {
+			if (rtw_txpwr_hal_lmt_reg_search(dvobj, band, name)) {
+				hal_regu = rtw_phl_get_pw_lmt_regu_type_of_band_from_str(GET_PHL_INFO(dvobj), band, name);
+				regu[regu_num++] = hal_regu >= 0 ? hal_regu : hal_none_regu;
+			} else
+				regu[regu_num++] = hal_none_regu;
+		}
+	}
+
+	rtw_phl_cmd_set_pw_lmt_regu(GET_PHL_INFO(dvobj), &hal_conf, true, PHL_CMD_DIRECTLY, 0);
+
+	if (hal_conf.regu_2g)
+		rtw_mfree(hal_conf.regu_2g, hal_conf.regu_2g_len);
+	if (hal_conf.regu_5g)
+		rtw_mfree(hal_conf.regu_5g, hal_conf.regu_5g_len);
+	if (hal_conf.regu_6g)
+		rtw_mfree(hal_conf.regu_6g, hal_conf.regu_6g_len);
+}
+
+void rtw_txpwr_hal_get_current_lmt_regs_name(struct dvobj_priv* dvobj, char *names_of_band[], int names_len_of_band[])
+{
+	struct txpwr_regu_info_t *hal_info;
+	enum band_type band;
+	u8 *reg;
+	u8 reg_len;
+	int i;
+	const char *hal_name;
+
+	for (band = 0; band < BAND_MAX; band++) {
+		names_of_band[band] = NULL;
+		names_len_of_band[band] = 0;
+	}
+
+	hal_info = rtw_phl_get_pw_lmt_regu_info(GET_PHL_INFO(dvobj));
+	if (!hal_info) {
+		RTW_ERR("%s rtw_phl_get_pw_lmt_regu_info return NULL\n", __func__);
+		return;
+	}
+
+	for (band = 0; band < BAND_MAX; band++) {
+		reg_len = 0;
+		if (band == BAND_ON_24G) {
+			reg = hal_info->regu_2g;
+			reg_len = hal_info->regu_2g_len;
+		} else if (band == BAND_ON_5G) {
+			reg = hal_info->regu_5g;
+			reg_len = hal_info->regu_5g_len;
+		} else if (band == BAND_ON_6G) {
+			reg = hal_info->regu_6g;
+			reg_len = hal_info->regu_6g_len;
+		}
+
+		for (i = 0; i < reg_len; i++) {
+			hal_name = rtw_phl_get_pw_lmt_regu_str_from_type_of_band(GET_PHL_INFO(dvobj), band, reg[i]);
+			if (hal_name)
+				ustrs_add(&names_of_band[band], &names_len_of_band[band], hal_name);
+		}
+	}
+
+	rtw_phl_free_pw_lmt_regu_info(GET_PHL_INFO(dvobj), hal_info);
+}
+
+#define TXPWR_LMT_RS_CCK	0
+#define TXPWR_LMT_RS_OFDM	1
+#define TXPWR_LMT_RS_HT		2
+#define TXPWR_LMT_RS_VHT	3
+#define TXPWR_LMT_RS_HE		4
+#define TXPWR_LMT_RS_NUM	5
+
+#define TXPWR_LMT_MAX_BANDWIDTH_NUM	4 /* 20MHz ~ 160MHz */
+
+/* TXBF Capabilities */
+#define TXPWR_LMT_NO_TXBF	0
+#define TXPWR_LMT_TXBF		1
+#define TXPWR_LMT_TXBF_NUM	2
+
+static const char *const _txpwr_lmt_rs_str[] = {
+	[TXPWR_LMT_RS_CCK]	= "CCK",
+	[TXPWR_LMT_RS_OFDM]	= "OFDM",
+	[TXPWR_LMT_RS_HT]	= "HT",
+	[TXPWR_LMT_RS_VHT]	= "VHT",
+	[TXPWR_LMT_RS_HE]	= "HE",
+	[TXPWR_LMT_RS_NUM]	= "UNKNOWN",
+};
+
+#define txpwr_lmt_rs_str(rs) (((rs) >= TXPWR_LMT_RS_NUM) ? _txpwr_lmt_rs_str[TXPWR_LMT_RS_NUM] : _txpwr_lmt_rs_str[(rs)])
+
+static u16 rtw_txpwr_lmt_rs_to_data_rate(int txpwr_lmt_rs)
+{
+	u16 data_rate;
+	switch (txpwr_lmt_rs) {
+		case TXPWR_LMT_RS_CCK:
+			data_rate = RTW_DATA_RATE_CCK11;
+			break;
+		case TXPWR_LMT_RS_OFDM:
+			data_rate = RTW_DATA_RATE_OFDM54;
+			break;
+		case TXPWR_LMT_RS_HT:
+		case TXPWR_LMT_RS_VHT:
+		case TXPWR_LMT_RS_HE:
+			data_rate = RTW_DATA_RATE_HE_NSS1_MCS11;
+			break;
+		default:
+			data_rate = RTW_DATA_RATE_OFDM6;
+			break;
+	}
+
+	return data_rate;
+}
+
+void dump_txpwr_lmt(void *sel, _adapter *adapter)
+{
+#define TMP_STR_LEN 32
+	struct dvobj_priv *devob = adapter_to_dvobj(adapter);
+	int band, bw, ch_num, tlrs, ntx_idx, bf, regu;
+	const char *str;
+	char fmt[16];
+	char tmp_str[TMP_STR_LEN];
+	u8 ch, i;
+	u8 max_regd_num;
+	u8 txgi_pdbm, txgi_max;
+	s8 txgi_ww;
+
+	txgi_pdbm = rtw_phl_get_tx_tbl_to_tx_pwr_times(GET_PHL_INFO(devob));
+	txgi_max = rtw_phl_get_power_limit_value_na(GET_PHL_INFO(devob));
+	txgi_ww = rtw_phl_get_power_limit_value_ww(GET_PHL_INFO(devob));
+
+	for (band = BAND_ON_24G; band <= BAND_ON_6G; band++) {
+		if (!rtw_hw_is_band_support(devob, band))
+			continue;
+		max_regd_num = rtw_phl_get_regulation_max_num(GET_PHL_INFO(devob), band);
+
+		for (bw = 0; bw < TXPWR_LMT_MAX_BANDWIDTH_NUM; bw++) {
+			u8 (*center_chs)(u8, u8);
+
+			if (!rtw_hw_is_bw_support(devob, bw))
+				break;
+			if (band == BAND_ON_24G && bw >= CHANNEL_WIDTH_80)
+				break;
+
+			ch_num = center_chs_num_of_band[band](bw);
+			if (ch_num == 0) {
+				rtw_warn_on(1);
+				break;
+			}
+			center_chs = center_chs_of_band[band];
+
+			for (tlrs = TXPWR_LMT_RS_CCK; tlrs < TXPWR_LMT_RS_NUM; tlrs++) {
+
+				if (band != BAND_ON_24G && tlrs == TXPWR_LMT_RS_CCK)
+					continue;
+				if (bw > CHANNEL_WIDTH_20 && tlrs == TXPWR_LMT_RS_OFDM)
+					continue;
+				if (tlrs == TXPWR_LMT_RS_HT || tlrs == TXPWR_LMT_RS_VHT)
+					continue;
+
+				for (ntx_idx = RF_1TX; ntx_idx < RF_PATH_MAX; ntx_idx++) {
+
+					if (ntx_idx + 1 > GET_TX_PATH_NUM(devob, HW_BAND_0))
+						continue;
+
+					for (bf = TXPWR_LMT_NO_TXBF; bf < TXPWR_LMT_TXBF_NUM; bf++) {
+						if (bf == TXPWR_LMT_TXBF && (ntx_idx == RF_1TX ||
+							(tlrs == TXPWR_LMT_RS_CCK || tlrs == TXPWR_LMT_RS_OFDM)))
+							continue;
+
+						RTW_PRINT_SEL(sel, "[%s][%s][%s][%uT][%s]\n"
+							, band_str(band)
+							, ch_width_str(bw)
+							, txpwr_lmt_rs_str(tlrs)
+							, ntx_idx + 1
+							, bf? "BF": "NA"
+						);
+
+						/* header for limit in dBm  */
+						RTW_PRINT_SEL(sel, "%3s ", "ch");
+
+						for (regu = 0; regu < max_regd_num; regu++) {
+							if (rtw_phl_pw_lmt_regu_tbl_exist(GET_PHL_INFO(devob), band, regu)) {
+								str = rtw_phl_get_pw_lmt_regu_str_from_type_of_band(GET_PHL_INFO(devob), band, regu);
+								if (!str || strcmp(str, "INTERSECT") == 0 || strcmp(str, "EXT") == 0)
+									continue;
+								sprintf(fmt, "%%%zus%%s ", strlen(str) >= 6 ? 1 : 6 - strlen(str));
+								snprintf(tmp_str, TMP_STR_LEN, fmt
+									, rtw_phl_is_current_pwr_lmt_regu(GET_PHL_INFO(devob), band, regu) ? "*" : ""
+									, str);
+								_RTW_PRINT_SEL(sel, "%s", tmp_str);
+							}
+						}
+						_RTW_PRINT_SEL(sel, "\n");
+
+						for (i = 0; i < ch_num; i++) {
+							ch = center_chs(bw, i);
+							if (ch == 0) {
+								rtw_warn_on(1);
+								break;
+							}
+
+							/* dump limit in dBm */
+							RTW_PRINT_SEL(sel, "%3u ", ch);
+							for (regu = 0 ; regu < max_regd_num; regu++) {
+								if(rtw_phl_pw_lmt_regu_tbl_exist(GET_PHL_INFO(devob), band, regu)) {
+									str = rtw_phl_get_pw_lmt_regu_str_from_type_of_band(GET_PHL_INFO(devob), band, regu);
+									if (!str || strcmp(str, "INTERSECT") == 0 || strcmp(str, "EXT") == 0)
+										continue;
+									txpwr_idx_get_dbm_str(
+										rtw_phl_get_power_limit_option(GET_PHL_INFO(devob), HW_BAND_0,
+											RF_PATH_A, rtw_txpwr_lmt_rs_to_data_rate(tlrs), bw, bf, ntx_idx, ch, band, regu),
+										txgi_max, txgi_ww, txgi_pdbm, strlen(str), tmp_str, TMP_STR_LEN);
+									_RTW_PRINT_SEL(sel, "%s ", tmp_str);
+								}
+							}
+							_RTW_PRINT_SEL(sel, "\n");
+						}
+						RTW_PRINT_SEL(sel, "\n");
+					}/* loop for beamforming */
+				}/* loop for tx_num */
+			} /* loop for rate sections */
+		} /* loop for bandwidths */
+	} /* loop for bands */
+}
+
+#ifdef CONFIG_80211AX_HE
+#define TXPWR_LMT_RU26		0
+#define TXPWR_LMT_RU52		1
+#define TXPWR_LMT_RU106		2
+#define TXPWR_LMT_RU_NUM	3
+
+static const char *const _txpwr_lmt_ru_str[] = {
+	[TXPWR_LMT_RU26]	= "RU26",
+	[TXPWR_LMT_RU52]	= "RU52",
+	[TXPWR_LMT_RU106]	= "RU106",
+	[TXPWR_LMT_RU_NUM]	= "RU_NUM",
+};
+
+#define txpwr_lmt_ru_str(ru) (((ru) >= TXPWR_LMT_RU_NUM) ? _txpwr_lmt_ru_str[TXPWR_LMT_RU_NUM] : _txpwr_lmt_ru_str[(ru)])
+
+void dump_txpwr_lmt_ru(void *sel, _adapter *adapter)
+{
+#define TMP_STR_LEN 32
+	struct dvobj_priv *devob = adapter_to_dvobj(adapter);
+	int band, bw, ch_num, tlrs, ntx_idx, regu;
+	const char *str;
+	char fmt[16];
+	char tmp_str[TMP_STR_LEN];
+	u8 ch, i;
+	u8 max_regd_num;
+	u8 txgi_pdbm, txgi_max;
+	s8 txgi_ww;
+
+	txgi_pdbm = rtw_phl_get_tx_tbl_to_tx_pwr_times(GET_PHL_INFO(devob));
+	txgi_max = rtw_phl_get_power_limit_value_na(GET_PHL_INFO(devob));
+	txgi_ww = rtw_phl_get_power_limit_value_ww(GET_PHL_INFO(devob));
+
+	for (band = BAND_ON_24G; band <= BAND_ON_6G; band++) {
+		if (!rtw_hw_is_band_support(devob, band))
+			continue;
+		max_regd_num = rtw_phl_get_regulation_max_num(GET_PHL_INFO(devob), band);
+
+		for (bw = 0; bw < TXPWR_LMT_RU_NUM; bw++) {
+			u8 (*center_chs)(u8, u8);
+
+			ch_num = center_chs_num_of_band[band](CHANNEL_WIDTH_20);
+			if (ch_num == 0) {
+				rtw_warn_on(1);
+				break;
+			}
+			center_chs = center_chs_of_band[band];
+
+			for (tlrs = TXPWR_LMT_RS_HE; tlrs < TXPWR_LMT_RS_NUM; tlrs++) {
+
+				for (ntx_idx = RF_1TX; ntx_idx < RF_PATH_MAX; ntx_idx++) {
+
+					if (ntx_idx + 1 > GET_TX_PATH_NUM(devob, HW_BAND_0))
+						continue;
+
+					RTW_PRINT_SEL(sel, "[%s][%s][%s][%uT]\n"
+						, band_str(band)
+						, txpwr_lmt_ru_str(bw)
+						, txpwr_lmt_rs_str(tlrs)
+						, ntx_idx + 1
+					);
+
+					/* header for limit in dBm  */
+					RTW_PRINT_SEL(sel, "%3s ", "ch");
+
+					for (regu = 0; regu < max_regd_num; regu++) {
+						if (rtw_phl_pw_lmt_regu_tbl_exist(GET_PHL_INFO(devob), band, regu)) {
+							str = rtw_phl_get_pw_lmt_regu_str_from_type_of_band(GET_PHL_INFO(devob), band, regu);
+							if (!str || strcmp(str, "INTERSECT") == 0 || strcmp(str, "EXT") == 0)
+								continue;
+							sprintf(fmt, "%%%zus%%s ", strlen(str) >= 6 ? 1 : 6 - strlen(str));
+							snprintf(tmp_str, TMP_STR_LEN, fmt
+								, rtw_phl_is_current_pwr_lmt_regu(GET_PHL_INFO(devob), band, regu) ? "*" : ""
+								, str);
+							_RTW_PRINT_SEL(sel, "%s", tmp_str);
+						}
+					}
+					_RTW_PRINT_SEL(sel, "\n");
+
+					for (i = 0; i < ch_num; i++) {
+						ch = center_chs(CHANNEL_WIDTH_20, i);
+						if (ch == 0) {
+							rtw_warn_on(1);
+							break;
+						}
+
+						/* dump limit in dBm */
+						RTW_PRINT_SEL(sel, "%3u ", ch);
+						for (regu = 0 ; regu < max_regd_num; regu++) {
+							if(rtw_phl_pw_lmt_regu_tbl_exist(GET_PHL_INFO(devob), band, regu)) {
+								str = rtw_phl_get_pw_lmt_regu_str_from_type_of_band(GET_PHL_INFO(devob), band, regu);
+								if (!str || strcmp(str, "INTERSECT") == 0 || strcmp(str, "EXT") == 0)
+									continue;
+								txpwr_idx_get_dbm_str(
+									rtw_phl_get_power_limit_ru_option(GET_PHL_INFO(devob), HW_BAND_0, RF_PATH_A,
+										rtw_txpwr_lmt_rs_to_data_rate(tlrs), bw, ntx_idx, ch, band, regu),
+									txgi_max, txgi_ww, txgi_pdbm, strlen(str), tmp_str, TMP_STR_LEN);
+								_RTW_PRINT_SEL(sel, "%s ", tmp_str);
+							}
+						}
+						_RTW_PRINT_SEL(sel, "\n");
+					}
+					RTW_PRINT_SEL(sel, "\n");
+				}/* loop for tx_num */
+			} /* loop for rate sections */
+		} /* loop for rus */
+	} /* loop for bands */
+}
+#endif /* CONFIG_80211AX_HE */
+#endif /* CONFIG_TXPWR_LIMIT */
+
+extern enum rtw_data_rate _rate_mrate2phl(enum MGN_RATE mrate);
+void dump_txpwr_by_rate(void *sel, _adapter *adapter)
+{
+	struct dvobj_priv *devob = adapter_to_dvobj(adapter);
+	int band, rs, tx_num, n, i;
+	u8 rate_num, ratio, value;
+	ratio = rtw_phl_get_tx_tbl_to_tx_pwr_times(GET_PHL_INFO(devob));
+
+	for (band = BAND_ON_24G; band <= BAND_ON_6G; band++) {
+		if (!rtw_hw_is_band_support(devob, band))
+			continue;
+
+		RTW_PRINT_SEL(sel, "[%s]\n", band_str(band));
+
+		for (rs = 0; rs < RATE_SECTION_NUM; rs++) {
+			tx_num = rate_section_to_tx_num(rs);
+			if (tx_num >= GET_PHY_TX_NSS_BY_BAND(adapter_to_dvobj(adapter), HW_BAND_0))
+				continue;
+
+			if (band != BAND_ON_24G && IS_CCK_RATE_SECTION(rs))
+				continue;
+
+			rate_num = rate_section_rate_num(rs);
+
+			RTW_PRINT_SEL(sel, "%7s: ", rate_section_str(rs));
+
+			/* dump power by rate in db */
+			for (n = rate_num - 1; n >= 0; n--) {
+				value = rtw_phl_get_power_by_rate_band(GET_PHL_INFO(devob), HW_BAND_0,
+								       _rate_mrate2phl(rates_by_sections[rs].rates[n]),
+								       IS_DCM_RATE_SECTION(rs) ? 1 : 0, 0, band);
+
+				if (value % ratio) {
+					_RTW_PRINT_SEL(sel, "%2d.%d ", value / ratio
+							, (value % ratio) * 100 / ratio);
+				} else
+					_RTW_PRINT_SEL(sel, "%5d ", value / ratio);
+			}
+			RTW_PRINT_SEL(sel, "\n");
+		}
+
+		/* dump offset */
+		RTW_PRINT_SEL(sel, "%7s: ", "Offset");
+		for (i = 0; i < 5; i++) {
+			if (band != BAND_ON_24G && IS_CCK_RATE(mgn_rates_offset[i]))
+				continue;
+
+			value = rtw_phl_get_power_by_rate_band(GET_PHL_INFO(devob), HW_BAND_0,
+							       _rate_mrate2phl(mgn_rates_offset[i]),
+							       0, 1, band);
+			if (value % ratio) {
+				_RTW_PRINT_SEL(sel, "%2d.%d ", value / ratio, (value % ratio) * 100 / ratio);
+			} else
+				_RTW_PRINT_SEL(sel, "%5d ", value / ratio);
+		}
+		RTW_PRINT_SEL(sel, "\n");
+	}
+}
+
+#ifdef CONFIG_DFS_MASTER
+void rtw_dfs_hal_radar_detect_disable(struct dvobj_priv *dvobj, u8 band_idx)
+{
+	rtw_phl_cmd_dfs_rd_disable(GET_PHL_INFO(dvobj), band_idx, PHL_CMD_DIRECTLY, 0);
+}
+
+void rtw_dfs_hal_radar_detect_enable(struct dvobj_priv *dvobj, u8 band_idx, bool cac, u32 rd_freq_hi, u32 rd_freq_lo)
+{
+	rtw_phl_cmd_dfs_rd_enable_with_sp_freq_range(GET_PHL_INFO(dvobj), band_idx, cac, rd_freq_hi, rd_freq_lo, PHL_CMD_DIRECTLY, 0);
+}
+
+void rtw_dfs_hal_set_cac_status(struct dvobj_priv *dvobj, u8 band_idx, bool cac)
+{
+	rtw_phl_cmd_dfs_rd_set_cac_status(GET_PHL_INFO(dvobj), band_idx, cac, PHL_CMD_DIRECTLY, 0);
+}
+
+void rtw_dfs_hal_csa_mg_tx_pause(struct dvobj_priv *dvobj, u8 band_idx, bool pause)
+{
+	rtw_phl_cmd_dfs_csa_mg_tx_pause(GET_PHL_INFO(dvobj), band_idx, pause, PHL_CMD_DIRECTLY, 0);
+}
+
+static const enum dfs_regd_t _rtw_dfs_regd_to_phl[RTW_DFS_REGD_NUM] = {
+	/* elements not listed here will get DFS_REGD_UNKNOWN(0) */
+	[RTW_DFS_REGD_NONE]	= DFS_REGD_UNKNOWN,
+	[RTW_DFS_REGD_FCC]	= DFS_REGD_FCC,
+	[RTW_DFS_REGD_MKK]	= DFS_REGD_JAP,
+	[RTW_DFS_REGD_ETSI]	= DFS_REGD_ETSI,
+	[RTW_DFS_REGD_KCC]	= DFS_REGD_KCC,
+};
+
+#define rtw_dfs_regd_to_phl(region) (((region) >= RTW_DFS_REGD_NUM) ? _rtw_dfs_regd_to_phl[RTW_DFS_REGD_NONE] : _rtw_dfs_regd_to_phl[(region)])
+
+bool rtw_dfs_hal_region_supported(struct dvobj_priv* dvobj, enum rtw_dfs_regd domain)
+{
+	return domain == RTW_DFS_REGD_NONE
+		|| rtw_dfs_regd_to_phl(domain) != DFS_REGD_UNKNOWN;
+}
+
+void rtw_dfs_hal_update_region(struct dvobj_priv *dvobj, u8 band_idx, enum rtw_dfs_regd domain)
+{
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+
+	rtw_phl_cmd_dfs_change_domain(GET_PHL_INFO(dvobj), band_idx, rtw_dfs_regd_to_phl(domain), PHL_CMD_DIRECTLY, 0);
+}
+
+u8 rtw_dfs_hal_radar_detect_polling_int_ms(struct dvobj_priv *dvobj)
+{
+	return 100;
+}
+#endif /* CONFIG_DFS_MASTER */
+
+bool rtw_txpwr_hal_is_txpwr_limit_needed(struct dvobj_priv *dvobj)
+{
+#if CONFIG_TXPWR_LIMIT
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	enum rtw_pwr_limit_type type = phl_com->dev_cap.pwrlmt_type;
+	u8 efuse_regulatory = phl_com->dev_cap.rf_board_opt & 0x3;
+
+	if (type == RTW_PWBYRATE_AND_PWLMT
+		|| (type == RTW_PWLMT_BY_EFUSE && efuse_regulatory == 1))
+		return true;
+#endif
+	return false;
+}
+
+bool rtw_txpwr_hal_get_pwr_lmt_en(struct dvobj_priv *dvobj)
+{
+	return rtw_phl_get_pwr_lmt_en(GET_PHL_INFO(dvobj), HW_BAND_0);
+}
+
+bool rtw_txpwr_hal_get_ext_info(struct dvobj_priv *dvobj, struct tx_power_ext_info *info)
+{
+	void *phl_info = GET_PHL_INFO(dvobj);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	u8 band_idx = HW_BAND_0;
+	struct phy_sw_cap_t *sw_cap = &phl_com->phy_sw_cap[band_idx];
+
+	if (!rtw_hw_is_init_completed(dvobj))
+		return false;
+
+	SET_TXPWR_PARAM_STATUS(&info->by_rate
+		, phl_com->dev_cap.pwrbyrate_off == RTW_PW_BY_RATE_ON
+		, phl_com->dev_cap.pwrbyrate_off == RTW_PW_BY_RATE_ON
+		, sw_cap->rf_txpwr_byrate_info.para_src == RTW_PARA_SRC_EXTNAL);
+
+	SET_TXPWR_PARAM_STATUS(&info->lmt
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, sw_cap->rf_txpwrlmt_info.para_src == RTW_PARA_SRC_EXTNAL);
+
+#ifdef CONFIG_80211AX_HE
+	SET_TXPWR_PARAM_STATUS(&info->lmt_ru
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, sw_cap->rf_txpwrlmt_ru_info.para_src == RTW_PARA_SRC_EXTNAL);
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	SET_TXPWR_PARAM_STATUS(&info->lmt_6g
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, sw_cap->rf_txpwrlmt_6g_info.para_src == RTW_PARA_SRC_EXTNAL);
+
+	SET_TXPWR_PARAM_STATUS(&info->lmt_ru_6g
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, rtw_phl_get_pwr_lmt_en(phl_info, band_idx)
+		, sw_cap->rf_txpwrlmt_ru_6g_info.para_src == RTW_PARA_SRC_EXTNAL);
+#endif
+
+	return true;
+}
+
+/*
+* check if user specified mbm is valid
+*/
+bool phy_is_txpwr_user_mbm_valid(_adapter *adapter, s16 mbm)
+{
+	return 1;
+}
+
+bool phy_is_txpwr_user_lmt_specified(struct dvobj_priv *dvobj)
+{
+	s16 total_mbm = UNSPECIFIED_MBM;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	total_mbm = rtw_cfg80211_dev_get_total_txpwr_lmt_mbm(dvobj);
+#endif
+
+	return total_mbm != UNSPECIFIED_MBM;
+}
+
+/*
+* Return value in unit of TX Gain Index
+* hal_spec.txgi_max means unspecified
+*/
+static s8 phy_get_txpwr_user_lmt(struct dvobj_priv *dvobj, u8 ntx_idx)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	void *phl_info = GET_PHL_INFO(dvobj);
+	s16 total_mbm = UNSPECIFIED_MBM;
+	s8 lmt;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	total_mbm = rtw_cfg80211_dev_get_total_txpwr_lmt_mbm(dvobj);
+#endif
+	if (total_mbm != UNSPECIFIED_MBM) {
+		/* TODO: consider antenna_gain */
+		lmt = txpwr_mbm_to_txgi_s8_with_max(total_mbm - mb_of_ntx(ntx_idx + 1)
+			, rtw_phl_get_power_limit_value_na(phl_info)
+			, rtw_phl_get_tx_tbl_to_tx_pwr_times(phl_info));
+	} else
+		lmt = rtw_phl_get_power_limit_value_na(phl_info);
+
+	return lmt;
+}
+
+static void rtw_set_ext_pwr_lmt_into(struct dvobj_priv *dvobj, struct rtw_phl_ext_pwr_lmt_info *ext_pwr_lmt_info)
+{
+	int i = 0;
+	s8 pwr_lmt;
+
+	for (i = 0; i <= RF_PATH_B; i++){
+		pwr_lmt = phy_get_txpwr_user_lmt(dvobj, i);
+		ext_pwr_lmt_info->ext_pwr_lmt_2_4g[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_5g_band1[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_5g_band2[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_5g_band3[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_5g_band4[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_6g_unii_5_1[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_6g_unii_5_2[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_6g_unii_6[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_6g_unii_7_1[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_6g_unii_7_2[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_6g_unii_8[i] = pwr_lmt;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_2_4g[i]= 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_5g_band1[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_5g_band2[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_5g_band3[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_5g_band4[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_6g_unii_5_1[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_6g_unii_5_2[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_6g_unii_6[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_6g_unii_7_1[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_6g_unii_7_2[i] = 0;
+		ext_pwr_lmt_info->ext_pwr_lmt_ant_6g_unii_8[i] = 0;
+	}
+}
+
+void rtw_txpwr_hal_update_pwr(struct dvobj_priv *dvobj, enum phl_band_idx band_idx)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	void *phl_info = GET_PHL_INFO(dvobj);
+	struct txpwr_ctl_param args;
+	struct rtw_phl_ext_pwr_lmt_info ext_pwr_lmt_info;
+	int i;
+
+	/* ext_pwr_lmt is for all hwband here */
+	rtw_set_ext_pwr_lmt_into(dvobj, &ext_pwr_lmt_info);
+	if (phy_is_txpwr_user_lmt_specified(dvobj)) {
+		rtw_phl_set_ext_pwr_lmt_en(phl_info, true);
+		for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+			rtw_phl_enable_ext_pwr_lmt(phl_info, i, &ext_pwr_lmt_info);
+	} else
+		rtw_phl_set_ext_pwr_lmt_en(phl_info, false);
+
+	txpwr_ctl_param_init(&args);
+	args.force_write_txpwr = true;
+	args.constraint_mb = rfctl->tpc_mode == TPC_MODE_MANUAL ? rfctl->tpc_manual_constraint : 0;
+
+	for (i = HW_BAND_0; i < HW_BAND_MAX; i++) {
+		if (band_idx < HW_BAND_MAX && band_idx != i)
+			continue;
+		args.band_idx = i;
+		rtw_phl_cmd_txpwr_ctl(phl_info, &args, PHL_CMD_DIRECTLY, 0);
+	}
+}
+
+inline u8 get_phy_tx_nss(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	u8 txss = 0;
+
+	if (adapter_link->wrlink)
+		txss = GET_PHY_TX_NSS_BY_BAND(adapter_to_dvobj(adapter), adapter_link->wrlink->hw_band);
+	else
+		rtw_warn_on(1);
+
+	return txss;
+}
+
+inline u8 get_phy_rx_nss(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	u8 rxss = 0;
+
+	if (adapter_link->wrlink)
+		rxss = GET_PHY_RX_NSS_BY_BAND(adapter_to_dvobj(adapter), adapter_link->wrlink->hw_band);
+	else
+		rtw_warn_on(1);
+
+	return rxss;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_phl_cmd.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_phl_cmd.c
new file mode 100644
index 000000000000..4343649e3b28
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_phl_cmd.c
@@ -0,0 +1,335 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+
+static void
+phl_run_core_cmd(void *drv_priv, u8 *cmd, u32 cmd_len, enum rtw_phl_status status)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+	struct cmd_obj *pcmd = (struct cmd_obj *)cmd;
+	_adapter *padapter = pcmd->padapter;
+	struct cmd_priv *pcmdpriv = &dvobj->cmdpriv;
+
+	if (status == RTW_PHL_STATUS_CANNOT_IO ||
+		status == RTW_PHL_STATUS_CMD_ERROR ||
+		RTW_CANNOT_RUN(dvobj)) {
+		RTW_INFO(FUNC_ADPT_FMT "%s FALSE -bDriverStopped(%s) bSurpriseRemoved(%s)\n"
+			, FUNC_ADPT_ARG(padapter)
+			, rtw_cmd_name(pcmd)
+			, dev_is_drv_stopped(dvobj) ? "True" : "False"
+			, dev_is_surprise_removed(dvobj) ? "True" : "False");
+
+		_rtw_mutex_lock(&pcmdpriv->sctx_mutex);
+		if (pcmd->sctx) {
+			if (0)
+				RTW_PRINT(FUNC_ADPT_FMT" pcmd->sctx\n", FUNC_ADPT_ARG(pcmd->padapter));
+			rtw_sctx_done_err(&pcmd->sctx, RTW_SCTX_DONE_CMD_DROP);
+		}
+		_rtw_mutex_unlock(&pcmdpriv->sctx_mutex);
+		rtw_free_cmd_obj(pcmd);
+		return;
+	}
+
+	if (status == RTW_PHL_STATUS_CMD_TIMEOUT)
+		RTW_ERR("%s %s cmd timeout\n", __func__, rtw_cmd_name(pcmd));
+	else if (status == RTW_PHL_STATUS_FAILURE) /*PHL fail due to 1. CMD_DROP:cmd abort or cancel 2.CMD_FAIL*/
+		RTW_ERR("%s %s cmd failure\n", __func__, rtw_cmd_name(pcmd));
+
+	rtw_run_cmd(padapter, pcmd, false);
+}
+
+static u32 _evt_joinbss_hdl(struct _ADAPTER *a, struct wlan_network *network)
+{
+	struct dvobj_priv *d;
+	u8 *mac;
+	struct sta_info *sta;
+	enum rtw_phl_status status;
+	u32 res = _FAIL;
+
+
+	d = adapter_to_dvobj(a);
+	mac = network->network.MacAddress;
+	sta = rtw_get_stainfo(&a->stapriv, mac);
+	if (!sta) {
+		RTW_ERR(FUNC_ADPT_FMT ": sta(" MAC_FMT ") not found!\n",
+			FUNC_ADPT_ARG(a), MAC_ARG(mac));
+		goto disconnect;
+	}
+
+	if (network->join_res < 0)
+		goto disconnect;
+
+	status = rtw_phl_connect_linked(d->phl, a->connect_bidx, a->phl_role, sta->phl_sta, mac);
+	if (status == RTW_PHL_STATUS_SUCCESS)
+		return _SUCCESS;
+
+	RTW_ERR(FUNC_ADPT_FMT ": rtw_phl_connect_linked FAIL(%u)!\n",
+		FUNC_ADPT_ARG(a), status);
+
+disconnect:
+	RTW_INFO(FUNC_ADPT_FMT ": something may go wrong, run disconnect! "
+		 "join_res=%d for " MAC_FMT "\n",
+		 FUNC_ADPT_ARG(a), network->join_res, MAC_ARG(mac));
+
+	status = rtw_connect_disconnect_prepare(a);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR(FUNC_ADPT_FMT ": rtw_connect_disconnect_prepare FAIL(%u)!\n",
+			FUNC_ADPT_ARG(a), status);
+		goto exit;
+	}
+	res = _SUCCESS;
+
+exit:
+	return res;
+}
+
+#ifdef CONFIG_PCIE_TRX_MIT
+static void rtw_pcie_trx_mit_cb(void *drv_priv, u8 *cmd, u32 cmd_len, enum rtw_phl_status status)
+{
+	/* cmd point to mit_info. */
+	struct rtw_pcie_trx_mit_info_t *mit_info = (struct rtw_pcie_trx_mit_info_t *)cmd;
+
+	RTW_DBG("%s: mit_info pointer=%p\n", __func__, mit_info);
+	rtw_mfree(mit_info, sizeof(struct rtw_pcie_trx_mit_info_t));
+}
+
+u8 rtw_pcie_trx_mit_cmd(_adapter *padapter, u32 tx_timer, u8 tx_counter,
+			u32 rx_timer, u8 rx_counter, u8 fixed_mit)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_pcie_trx_mit_info_t *mit_info;
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+	u8 res = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	mit_info = (struct rtw_pcie_trx_mit_info_t *)rtw_zmalloc(sizeof(struct rtw_pcie_trx_mit_info_t));
+	if (mit_info == NULL)
+		goto exit;
+
+	mit_info->tx_timer = tx_timer;
+	mit_info->tx_counter = tx_counter;
+	mit_info->rx_timer = rx_timer;
+	mit_info->rx_counter = rx_counter;
+	mit_info->fixed_mitigation = fixed_mit;
+
+	RTW_DBG("%s: mit_info pointer=%p\n", __func__, mit_info);
+
+	psts = rtw_phl_cmd_enqueue(dvobj->phl,
+				padapter_link->wrlink->hw_band,
+				MSG_EVT_PCIE_TRX_MIT,
+				(u8 *)mit_info, sizeof(mit_info),
+				rtw_pcie_trx_mit_cb,
+				PHL_CMD_NO_WAIT, 0);
+
+	/* Send cmd fail */
+	if (psts != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s: cmd failed!\n", __func__);
+		rtw_mfree(mit_info, sizeof(struct rtw_pcie_trx_mit_info_t));
+		goto exit;
+	}
+
+	res = _SUCCESS;
+
+exit:
+	return res;
+}
+#endif /* CONFIG_PCIE_TRX_MIT */
+
+#ifdef CONFIG_POST_CORE_KEEP_ALIVE
+u8 rtw_set_post_keep_alive_param(_adapter *padapter, u8 if_sta_chk_rx_bmp,
+			   u8 if_sta_chk_tx_bmp, u32 *if_ap_chk_sta_bmp)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_keep_alive_param klive_param = {0};
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+	u8 res = _FAIL;
+	bool need_post_chk = _FALSE;
+	u8 i;
+
+#ifdef CONFIG_AP_MODE
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (if_ap_chk_sta_bmp[i] != 0) {
+			need_post_chk = _TRUE;
+			break;
+		}
+	}
+#endif
+	/* skip post keep alive chk */
+	if (!need_post_chk && if_sta_chk_rx_bmp == 0 && if_sta_chk_tx_bmp == 0) {
+		res = _SUCCESS;
+		goto exit;
+	}
+
+	klive_param.sta_chk_rx_bmp = if_sta_chk_rx_bmp;
+	klive_param.sta_chk_tx_bmp = if_sta_chk_tx_bmp;
+	_rtw_memcpy(klive_param.ap_chk_sta_bmp, if_ap_chk_sta_bmp, sizeof(klive_param.ap_chk_sta_bmp));
+
+	psts = rtw_phl_set_wdog_state_keep_alive(dvobj->phl, true, &klive_param);
+	if (psts == RTW_PHL_STATUS_SUCCESS)
+		res = _SUCCESS;
+
+exit:
+	return res;
+}
+#endif
+
+#ifdef CONFIG_TDLS
+enum rtw_phl_status rtw_send_tdls_sync_msg(struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_wifi_role_t *role = a->phl_role;
+	enum rtw_phl_status status;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+
+	RTW_DBG(FUNC_ADPT_FMT ": +\n", FUNC_ADPT_ARG(a));
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_MRC);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_TDLS_SYNC);
+	msg.inbuf = (u8 *)role;
+	msg.rsvd[0].ptr = (u8 *)alink->wrlink;
+	msg.band_idx = alink->wrlink->hw_band;
+
+	status = rtw_phl_send_msg_to_dispr(GET_PHL_INFO(d),
+					   &msg, &attr, NULL);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR(FUNC_ADPT_FMT ": Send tdls sync event fail(0x%02x)!\n",
+			FUNC_ADPT_ARG(a), status);
+	}
+
+	return status;
+}
+#endif /* CONFIG_TDLS */
+
+u32 rtw_enqueue_phl_cmd(struct cmd_obj *pcmd)
+{
+	u32 res = _FAIL;
+	_adapter *padapter = pcmd->padapter;
+	enum phl_band_idx band_idx = pcmd->band_idx;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+
+	if (band_idx >= HW_BAND_MAX) {
+		/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+		struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+		if (rtw_is_adapter_up(padapter) == _FALSE) {
+			RTW_ERR("%s Drop CMD: %s\n", __func__, rtw_cmd_name(pcmd));
+			goto free_cmd;
+		}
+
+		band_idx = ALINK_GET_HWBAND(padapter_link);
+	}
+
+	switch (pcmd->cmdcode) {
+	case CMD_JOINBSS:
+		/*
+		* Call rtw_connect_cmd() in rtw_join_cmd_hdl()
+		*/
+		break;
+
+	case CMD_DISCONNECT:
+		/*
+		 * Disconnect for STA wouldn't come to here,
+		 * because it would be handled in rtw_disassoc_cmd() directly.
+		 */
+		break;
+
+	case CMD_SET_MLME_EVT:
+		{
+		struct rtw_evt_header *hdr;
+		struct wlan_network *network;
+
+		hdr = (struct rtw_evt_header*)pcmd->parmbuf;
+		switch (hdr->id) {
+		case EVT_JOINBSS:
+			network = (struct wlan_network *)(pcmd->parmbuf + sizeof(*hdr));
+			res = _evt_joinbss_hdl(padapter, network);
+			goto free_cmd;
+
+		case EVT_DEL_STA:
+			if (MLME_IS_STA(padapter)) {
+				psts = rtw_disconnect_cmd(padapter, pcmd);
+				if (psts != RTW_PHL_STATUS_SUCCESS) {
+					if (psts == RTW_PHL_STATUS_RESOURCE)
+						res = _SUCCESS;
+					goto free_cmd;
+				}
+				/* pcmd & pcmd->parmbuf would be freed in framework */
+				res = _SUCCESS;
+				goto exit;
+			}
+			break;
+
+		default:
+			break;
+		}
+		/* not handled event would be processed later */
+		break;
+		}
+
+	case CMD_CREATE_BSS:
+		{
+		psts = rtw_ap_start_cmd(pcmd);
+		if (psts != RTW_PHL_STATUS_SUCCESS)
+			goto free_cmd;
+		/* pcmd & pcmd->parmbuf should be freed here */
+		res = _SUCCESS;
+		goto exit;
+		}
+
+	case CMD_SET_DRV_EXTRA:
+		{
+		struct drvextra_cmd_parm *parm;
+
+		parm = (struct drvextra_cmd_parm *)pcmd->parmbuf;
+		if (parm->ec_id == STOP_AP_WK_CID) {
+			psts = rtw_ap_stop_cmd(pcmd);
+			if (psts != RTW_PHL_STATUS_SUCCESS)
+				goto free_cmd;
+			res = _SUCCESS;
+			goto exit;
+		}
+		break;
+		}
+
+	default:
+		break;
+	}
+
+	psts = rtw_phl_cmd_enqueue(dvobj->phl,
+			band_idx,
+			MSG_EVT_LINUX_CMD_WRK,
+			(u8 *)pcmd, sizeof(struct cmd_obj),
+			phl_run_core_cmd,
+			PHL_CMD_NO_WAIT, 0);
+
+	/* Send cmd fail */
+	if (psts != RTW_PHL_STATUS_SUCCESS)
+		goto free_cmd; /* keep res == _FAIL */
+
+	res = _SUCCESS;
+	goto exit;
+
+free_cmd:
+	rtw_free_cmd_obj(pcmd);
+
+exit:
+	return res;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_prealloc.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_prealloc.c
new file mode 100644
index 000000000000..edafb66498f0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_prealloc.c
@@ -0,0 +1,480 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/gfp.h>
+#include <drv_types.h>
+#include <rtw_mem.h>
+
+#if defined(CONFIG_STACKTRACE) &&                                              \
+	(LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0))
+#define STACKTRACE 1
+#define MAX_STACK_TRACE 4
+#endif /* CONFIG_STACKTRACE */
+
+#define TRACE_ORDER(a, b) ((a) && (((a)&BIT(b)) > 0))
+
+#if (defined(CONFIG_RTKM) && defined(CONFIG_RTKM_STANDALONE))
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#undef RTW_PRINT
+#define RTW_PRINT pr_info
+#else /* !CONFIG_RTKM */
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+#define pr_fmt(fmt) "RTKM: " fmt
+#endif /* CONFIG_RTKM */
+
+#ifndef MAX_ORDER
+#define MAX_ORDER MAX_PAGE_ORDER
+#endif
+
+struct phy_mem_list {
+	_lock lock;
+	_list list;
+	struct rb_root rb_tree;
+	unsigned short entries;
+	unsigned short used;
+	unsigned short peak;
+	unsigned char order;
+} rtkm_phy_list[MAX_ORDER];
+
+struct mem_entry {
+	_list list;
+	struct rb_node rbn;
+	void *data;
+	size_t size;
+#ifdef STACKTRACE
+#ifndef CONFIG_ARCH_STACKWALK
+	struct stack_trace trace;
+#endif /* CONFIG_ARCH_STACKWALK */
+	unsigned long stack_entries[MAX_STACK_TRACE];
+#endif /* STACKTRACE */
+	unsigned char order;
+	unsigned char is_use;
+};
+
+#ifdef STACKTRACE
+#ifdef CONFIG_ARCH_STACKWALK
+static inline void stacktrace_print(const struct mem_entry *entries,
+				    unsigned int nr_entries, int spaces)
+{
+	stack_trace_print(entries->stack_entries, nr_entries, spaces);
+}
+
+static inline int stacktrace_save(struct mem_entry *store)
+{
+	return stack_trace_save(store->stack_entries,
+				ARRAY_SIZE(store->stack_entries), 1);
+}
+#else /* !CONFIG_ARCH_STACKWALK */
+static inline void stacktrace_print(const struct mem_entry *entries,
+				    unsigned int nr_entries, int spaces)
+{
+	stack_trace_print(entries->trace.entries, nr_entries, spaces);
+}
+
+static inline void stacktrace_save(struct mem_entry *store)
+{
+	store->trace.skip = 0;
+	store->trace.nr_entries = 0;
+	store->trace.entries = store->stack_entries;
+	store->trace.max_entries = MAX_STACK_TRACE;
+	save_stack_trace(&store->trace);
+}
+#endif /* CONFIG_ARCH_STACKWALK */
+#else /* !STACKTRACE */
+#define stacktrace_print(a, b, c)
+#define stacktrace_save(a)
+#endif /* STACKTRACE */
+
+/* Trace mpool */
+static unsigned int rtkm_trace = 0;
+module_param(rtkm_trace, uint, 0644);
+MODULE_PARM_DESC(rtkm_trace, "Trace memory pool");
+
+/* Preallocated memory expansion */
+static bool rtkm_mem_exp = 1;
+module_param(rtkm_mem_exp, bool, 0644);
+MODULE_PARM_DESC(rtkm_mem_exp, "Preallocated memory expansion");
+
+#ifndef RTKM_MPOOL_0
+#define RTKM_MPOOL_0 0
+#endif
+#ifndef RTKM_MPOOL_1
+#define RTKM_MPOOL_1 0
+#endif
+#ifndef RTKM_MPOOL_2
+#define RTKM_MPOOL_2 0
+#endif
+#ifndef RTKM_MPOOL_3
+#define RTKM_MPOOL_3 0
+#endif
+#ifndef RTKM_MPOOL_4
+#define RTKM_MPOOL_4 0
+#endif
+#ifndef RTKM_MPOOL_5
+#define RTKM_MPOOL_5 0
+#endif
+#ifndef RTKM_MPOOL_6
+#define RTKM_MPOOL_6 0
+#endif
+#ifndef RTKM_MPOOL_7
+#define RTKM_MPOOL_7 0
+#endif
+#ifndef RTKM_MPOOL_8
+#define RTKM_MPOOL_8 0
+#endif
+
+/* Preallocated memory pool  */
+static int mpool[MAX_ORDER] = {
+	[0] = RTKM_MPOOL_0, [1] = RTKM_MPOOL_1, [2] = RTKM_MPOOL_2,
+	[3] = RTKM_MPOOL_3, [4] = RTKM_MPOOL_4, [5] = RTKM_MPOOL_5,
+	[6] = RTKM_MPOOL_6, [7] = RTKM_MPOOL_7, [8] = RTKM_MPOOL_8
+};
+static int n_mpool = 1;
+module_param_array(mpool, int, &n_mpool, 0644);
+MODULE_PARM_DESC(mpool, "Preallocated memory pool");
+
+static inline void dump_mstatus(void *sel)
+{
+	int i;
+	unsigned int a, b, c;
+	long unsigned int musage = 0;
+
+	a = b = c = 0;
+
+	RTW_PRINT_SEL(sel,
+		      "====================== RTKM ======================\n");
+	RTW_PRINT_SEL(sel, "%6s %10s %10s %10s %10s\n", "order", "use", "peak",
+		      rtkm_mem_exp ? "alloc+" : "alloc", "size");
+	RTW_PRINT_SEL(sel,
+		      "--------------------------------------------------\n");
+	for (i = 0; i < MAX_ORDER; i++) {
+		if (rtkm_phy_list[i].entries) {
+			RTW_PRINT_SEL(
+				sel, "%6d %10d %10d %10d %10lu\n", i,
+				rtkm_phy_list[i].used, rtkm_phy_list[i].peak,
+				rtkm_phy_list[i].entries,
+				(rtkm_phy_list[i].entries) * (PAGE_SIZE << i));
+			a += rtkm_phy_list[i].used;
+			b += rtkm_phy_list[i].peak;
+			c += rtkm_phy_list[i].entries;
+			musage += (rtkm_phy_list[i].entries) * (PAGE_SIZE << i);
+		}
+	}
+	RTW_PRINT_SEL(sel, "%6s %10d %10d %10d %10lu\n", "sum", a, b, c,
+		      musage);
+}
+
+void rtkm_dump_mstatus(void *sel)
+{
+	dump_mstatus(sel);
+}
+EXPORT_SYMBOL(rtkm_dump_mstatus);
+
+void rtkm_set_trace(unsigned int mask)
+{
+	rtkm_trace = mask;
+}
+EXPORT_SYMBOL(rtkm_set_trace);
+
+static void rb_insert_mem(struct phy_mem_list *mlist, struct mem_entry *entry)
+{
+	struct rb_node **p = &mlist->rb_tree.rb_node;
+	struct rb_node *parent = NULL;
+	struct mem_entry *tmp = NULL;
+
+	while (*p) {
+		parent = *p;
+		tmp = rb_entry(parent, struct mem_entry, rbn);
+
+		if (tmp->data < entry->data)
+			p = &(*p)->rb_left;
+		else
+			p = &(*p)->rb_right;
+	}
+
+	rb_link_node(&entry->rbn, parent, p);
+	rb_insert_color(&entry->rbn, &mlist->rb_tree);
+}
+
+static struct mem_entry *rb_find_mem(struct phy_mem_list *mlist,
+				     const void *objp)
+{
+	struct rb_node *n = mlist->rb_tree.rb_node;
+	struct mem_entry *entry = NULL;
+
+	while (n) {
+		entry = rb_entry(n, struct mem_entry, rbn);
+
+		if (entry->data == objp)
+			return entry;
+		else if (entry->data < objp)
+			n = n->rb_left;
+		else
+			n = n->rb_right;
+	}
+
+	return NULL;
+}
+
+static inline void *create_mem_entry(int order)
+{
+	struct mem_entry *entry;
+
+	entry = _rtw_malloc(sizeof(struct mem_entry));
+	if (entry == NULL) {
+		pr_warn("%s: alloc memory entry fail!\n", __func__);
+		return NULL;
+	}
+	entry->order = order;
+	entry->is_use = _FALSE;
+	/* get memory by pages */
+	entry->data = (void *)__get_free_pages(
+		in_interrupt() ? GFP_ATOMIC : GFP_KERNEL, entry->order);
+	if (entry->data == NULL) {
+		_rtw_mfree(entry, sizeof(struct mem_entry));
+		pr_warn("%s: alloc memory oreder-%d fail!\n", __func__, order);
+		return NULL;
+	}
+
+	return entry;
+}
+
+static inline void *_kmalloc(size_t size, gfp_t flags, int clear)
+{
+	int order = 0;
+	int warn = _FALSE;
+	struct mem_entry *entry = NULL;
+
+	order = get_order(size);
+
+	if (rtkm_phy_list[order].entries == rtkm_phy_list[order].used) {
+		if (rtkm_mem_exp) {
+			warn = _TRUE;
+			pr_warn("%s: No enough order-%d pool\n", __func__,
+				order);
+			entry = create_mem_entry(order);
+			if (entry) {
+				_rtw_spinlock_bh(&rtkm_phy_list[order].lock);
+				list_add_tail(
+					&entry->list,
+					&rtkm_phy_list[entry->order].list);
+				rtkm_phy_list[entry->order].entries++;
+				_rtw_spinunlock_bh(&rtkm_phy_list[order].lock);
+			}
+		}
+
+		if (entry == NULL) {
+			pr_warn("%s: No more memory for size %ld\n", __func__,
+				size);
+			WARN_ON(1);
+			return NULL;
+		}
+	}
+
+	_rtw_spinlock_bh(&rtkm_phy_list[order].lock);
+	list_for_each_entry (entry, &rtkm_phy_list[order].list, list) {
+		if (entry->is_use == _FALSE) {
+			list_del_init(&entry->list);
+			entry->is_use = _TRUE;
+			entry->size = size;
+			if (clear == _TRUE)
+				memset(entry->data, 0, size);
+			stacktrace_save(entry);
+			rtkm_phy_list[order].used++;
+			list_add_tail(&entry->list, &rtkm_phy_list[order].list);
+			rb_insert_mem(&rtkm_phy_list[order], entry);
+			break;
+		}
+	}
+	if (rtkm_phy_list[order].peak < rtkm_phy_list[order].used)
+		rtkm_phy_list[order].peak = rtkm_phy_list[order].used;
+	_rtw_spinunlock_bh(&rtkm_phy_list[order].lock);
+
+	if ((warn) || TRACE_ORDER(rtkm_trace, order)) {
+		pr_info("%s: require(%p, %lu) usage(%d %u/%u)\n", __func__,
+			entry->data, entry->size, order,
+			rtkm_phy_list[order].used,
+			rtkm_phy_list[order].entries);
+		stacktrace_print(entry, MAX_STACK_TRACE, 0);
+	}
+
+	return entry->data;
+}
+
+static inline void _kfree(const void *objp, size_t size)
+{
+	int found = _FALSE, order = 0;
+	struct mem_entry *entry;
+
+	order = get_order(size);
+
+	if (!list_empty(&rtkm_phy_list[order].list)) {
+		_rtw_spinlock_bh(&rtkm_phy_list[order].lock);
+
+		entry = rb_find_mem(&rtkm_phy_list[order], objp);
+		if (entry && (entry->is_use == _TRUE) &&
+		    (entry->data == objp)) {
+			if (TRACE_ORDER(rtkm_trace, order)) {
+				pr_info("%s: release(%p, %lu)\n", __func__,
+					objp, size);
+			}
+			rtw_list_delete(&entry->list);
+			rtkm_phy_list[order].used--;
+			entry->is_use = _FALSE;
+			entry->size = 0;
+			rb_erase(&entry->rbn, &rtkm_phy_list[order].rb_tree);
+			list_add(&entry->list, &rtkm_phy_list[order].list);
+			found = _TRUE;
+		}
+
+		_rtw_spinunlock_bh(&rtkm_phy_list[order].lock);
+	}
+	if (found == _FALSE) {
+		pr_warn("%s: not found (%p, %lu)\n", __func__, objp, size);
+	}
+}
+
+void *rtkm_kmalloc(size_t size, gfp_t flags)
+{
+	if (size > RTKM_MGMT_SIZE)
+		return _kmalloc(size, flags, _FALSE);
+	else
+		return kmalloc(size, flags);
+}
+EXPORT_SYMBOL(rtkm_kmalloc);
+
+void *rtkm_kzalloc(size_t size, gfp_t flags)
+{
+	if (size > RTKM_MGMT_SIZE)
+		return _kmalloc(size, flags, _TRUE);
+	else
+		return kzalloc(size, flags);
+}
+EXPORT_SYMBOL(rtkm_kzalloc);
+
+void rtkm_kfree(const void *objp, size_t size)
+{
+	if (size > RTKM_MGMT_SIZE)
+		return _kfree(objp, size);
+	else
+		return kfree(objp);
+}
+EXPORT_SYMBOL(rtkm_kfree);
+
+static inline int rtkm_init_phy(void)
+{
+	int ret = 0, i, j;
+	struct mem_entry *entry;
+
+	pr_info("%s", __func__);
+	pr_info("%s: memory expansion:%d\n", __func__, rtkm_mem_exp);
+
+	for (i = (MAX_ORDER - 1); i > -1; i--) {
+		INIT_LIST_HEAD(&rtkm_phy_list[i].list);
+		_rtw_spinlock_init(&rtkm_phy_list[i].lock);
+		rtkm_phy_list[i].rb_tree = RB_ROOT;
+
+		for (j = 0; (ret == 0) && (j < mpool[i]); j++) {
+			entry = create_mem_entry(i);
+			if (entry == NULL) {
+				ret = -ENOMEM;
+				break;
+			}
+
+			list_add_tail(&entry->list,
+				      &rtkm_phy_list[entry->order].list);
+			rtkm_phy_list[entry->order].entries++;
+		}
+	}
+	if (ret == 0)
+		dump_mstatus(RTW_DBGDUMP);
+
+	return ret;
+}
+
+static inline void rtkm_destroy_phy(void)
+{
+	int i = 0;
+	struct mem_entry *entry;
+
+	pr_info("%s", __func__);
+	dump_mstatus(RTW_DBGDUMP);
+
+	for (i = 0; i < MAX_ORDER; i++) {
+		if (rtkm_phy_list[i].used)
+			pr_err("%s: memory leak! order=%d num=%d\n", __func__,
+			       i, rtkm_phy_list[i].used);
+
+		if (rtkm_phy_list[i].rb_tree.rb_node != NULL)
+			pr_err("%s: rb tree leak! order=%d\n", __func__, i);
+
+		while (!list_empty(&rtkm_phy_list[i].list)) {
+			entry = list_entry(rtkm_phy_list[i].list.next,
+					   struct mem_entry, list);
+			list_del_init(&entry->list);
+			if (entry->is_use == _TRUE) {
+				rb_erase(&entry->rbn,
+					 &rtkm_phy_list[i].rb_tree);
+				pr_err("%s: memory leak! (%p, %lu)\n", __func__,
+				       entry->data, entry->size);
+				stacktrace_print(entry, MAX_STACK_TRACE, 0);
+			}
+			if (entry->data)
+				free_pages((unsigned long)(entry->data),
+					   entry->order);
+			entry->data = NULL;
+			entry->size = 0;
+			entry->is_use = _FALSE;
+			_rtw_mfree(entry, sizeof(struct mem_entry));
+			entry = NULL;
+			rtkm_phy_list[i].entries--;
+		}
+		_rtw_spinlock_free(&rtkm_phy_list[i].lock);
+	}
+}
+
+int rtkm_prealloc_init(void)
+{
+	int ret = 0;
+
+	pr_info("%s\n", __func__);
+
+	ret = rtkm_init_phy();
+	if (ret == -ENOMEM) {
+		pr_err("No enough memory for phiscal.");
+		rtkm_destroy_phy();
+	}
+
+	pr_info("%s: done ret=%d\n", __func__, ret);
+	return ret;
+}
+EXPORT_SYMBOL(rtkm_prealloc_init);
+
+void rtkm_prealloc_destroy(void)
+{
+	pr_info("%s\n", __func__);
+
+	rtkm_destroy_phy();
+
+	pr_info("%s: done\n", __func__);
+}
+EXPORT_SYMBOL(rtkm_prealloc_destroy);
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_pwrctrl.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_pwrctrl.c
new file mode 100644
index 000000000000..5af345f5fd0f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_pwrctrl.c
@@ -0,0 +1,355 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_PWRCTRL_C_
+
+#include <drv_types.h>
+#ifdef CONFIG_RTW_IPS
+bool rtw_core_set_ips_state(void *drv_priv, enum rtw_rf_state state)
+{
+        struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+        enum rtw_phl_status pstatus = RTW_PHL_STATUS_FAILURE;
+
+        if (state == RTW_RF_ON) {
+                pstatus = rtw_phl_rf_on(dvobj->phl);
+        } else if (state == RTW_RF_OFF) {
+                pstatus = rtw_phl_rf_off(dvobj->phl);
+        }
+
+        if (RTW_PHL_STATUS_SUCCESS == pstatus)
+                return true;
+        else
+                return false;
+}
+#endif
+
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+	static void resume_workitem_callback(struct work_struct *work);
+#endif /* CONFIG_RESUME_IN_WORKQUEUE */
+
+void rtw_init_pwrctrl_priv(_adapter *padapter)
+{
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+
+#if defined(CONFIG_CONCURRENT_MODE)
+	if (!is_primary_adapter(padapter))
+		return;
+#endif
+	pwrctrlpriv->bInSuspend = _FALSE;
+	pwrctrlpriv->bkeepfwalive = _FALSE;
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+	_init_workitem(&pwrctrlpriv->resume_work, resume_workitem_callback, NULL);
+	pwrctrlpriv->rtw_workqueue = create_singlethread_workqueue("rtw_workqueue");
+#endif /* CONFIG_RESUME_IN_WORKQUEUE */
+}
+
+
+void rtw_free_pwrctrl_priv(_adapter *adapter)
+{
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(adapter);
+
+#if defined(CONFIG_CONCURRENT_MODE)
+	if (!is_primary_adapter(adapter))
+		return;
+#endif
+
+
+	/* _rtw_memset((unsigned char *)pwrctrlpriv, 0, sizeof(struct pwrctrl_priv)); */
+
+
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+	if (pwrctrlpriv->rtw_workqueue) {
+		flush_workqueue(pwrctrlpriv->rtw_workqueue);
+		destroy_workqueue(pwrctrlpriv->rtw_workqueue);
+	}
+#endif
+}
+
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+extern int rtw_resume_process(_adapter *padapter);
+
+static void resume_workitem_callback(struct work_struct *work)
+{
+	struct pwrctrl_priv *pwrpriv = container_of(work, struct pwrctrl_priv, resume_work);
+	struct dvobj_priv *dvobj = pwrctl_to_dvobj(pwrpriv);
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	rtw_resume_process(adapter);
+
+	rtw_resume_unlock_suspend();
+}
+
+void rtw_resume_in_workqueue(struct pwrctrl_priv *pwrpriv)
+{
+	/* accquire system's suspend lock preventing from falliing asleep while resume in workqueue */
+	/* rtw_lock_suspend(); */
+
+	rtw_resume_lock_suspend();
+
+#if 1
+	queue_work(pwrpriv->rtw_workqueue, &pwrpriv->resume_work);
+#else
+	_set_workitem(&pwrpriv->resume_work);
+#endif
+}
+#endif /* CONFIG_RESUME_IN_WORKQUEUE */
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) || defined(CONFIG_ANDROID_POWER)
+inline bool rtw_is_earlysuspend_registered(struct pwrctrl_priv *pwrpriv)
+{
+	return (pwrpriv->early_suspend.suspend) ? _TRUE : _FALSE;
+}
+
+inline bool rtw_is_do_late_resume(struct pwrctrl_priv *pwrpriv)
+{
+	return (pwrpriv->do_late_resume) ? _TRUE : _FALSE;
+}
+
+inline void rtw_set_do_late_resume(struct pwrctrl_priv *pwrpriv, bool enable)
+{
+	pwrpriv->do_late_resume = enable;
+}
+#endif
+
+#ifdef CONFIG_HAS_EARLYSUSPEND
+extern int rtw_resume_process(_adapter *padapter);
+static void rtw_early_suspend(struct early_suspend *h)
+{
+	struct pwrctrl_priv *pwrpriv = container_of(h, struct pwrctrl_priv, early_suspend);
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	rtw_set_do_late_resume(pwrpriv, _FALSE);
+}
+
+static void rtw_late_resume(struct early_suspend *h)
+{
+	struct pwrctrl_priv *pwrpriv = container_of(h, struct pwrctrl_priv, early_suspend);
+	struct dvobj_priv *dvobj = pwrctl_to_dvobj(pwrpriv);
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	if (pwrpriv->do_late_resume) {
+		rtw_set_do_late_resume(pwrpriv, _FALSE);
+		rtw_resume_process(adapter);
+	}
+}
+
+void rtw_register_early_suspend(struct pwrctrl_priv *pwrpriv)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/* jeff: set the early suspend level before blank screen, so we wll do late resume after scree is lit */
+	pwrpriv->early_suspend.level = EARLY_SUSPEND_LEVEL_BLANK_SCREEN - 20;
+	pwrpriv->early_suspend.suspend = rtw_early_suspend;
+	pwrpriv->early_suspend.resume = rtw_late_resume;
+	register_early_suspend(&pwrpriv->early_suspend);
+
+
+}
+
+void rtw_unregister_early_suspend(struct pwrctrl_priv *pwrpriv)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	rtw_set_do_late_resume(pwrpriv, _FALSE);
+
+	if (pwrpriv->early_suspend.suspend)
+		unregister_early_suspend(&pwrpriv->early_suspend);
+
+	pwrpriv->early_suspend.suspend = NULL;
+	pwrpriv->early_suspend.resume = NULL;
+}
+#endif /* CONFIG_HAS_EARLYSUSPEND */
+
+#ifdef CONFIG_ANDROID_POWER
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+	extern int rtw_resume_process(_adapter *padapter);
+#endif
+static void rtw_early_suspend(android_early_suspend_t *h)
+{
+	struct pwrctrl_priv *pwrpriv = container_of(h, struct pwrctrl_priv, early_suspend);
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	rtw_set_do_late_resume(pwrpriv, _FALSE);
+}
+
+static void rtw_late_resume(android_early_suspend_t *h)
+{
+	struct pwrctrl_priv *pwrpriv = container_of(h, struct pwrctrl_priv, early_suspend);
+	struct dvobj_priv *dvobj = pwrctl_to_dvobj(pwrpriv);
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+
+	RTW_INFO("%s\n", __FUNCTION__);
+	if (pwrpriv->do_late_resume) {
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+		rtw_set_do_late_resume(pwrpriv, _FALSE);
+		rtw_resume_process(adapter);
+#endif
+	}
+}
+
+void rtw_register_early_suspend(struct pwrctrl_priv *pwrpriv)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	/* jeff: set the early suspend level before blank screen, so we wll do late resume after scree is lit */
+	pwrpriv->early_suspend.level = ANDROID_EARLY_SUSPEND_LEVEL_BLANK_SCREEN - 20;
+	pwrpriv->early_suspend.suspend = rtw_early_suspend;
+	pwrpriv->early_suspend.resume = rtw_late_resume;
+	android_register_early_suspend(&pwrpriv->early_suspend);
+}
+
+void rtw_unregister_early_suspend(struct pwrctrl_priv *pwrpriv)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	rtw_set_do_late_resume(pwrpriv, _FALSE);
+
+	if (pwrpriv->early_suspend.suspend)
+		android_unregister_early_suspend(&pwrpriv->early_suspend);
+
+	pwrpriv->early_suspend.suspend = NULL;
+	pwrpriv->early_suspend.resume = NULL;
+}
+#endif /* CONFIG_ANDROID_POWER */
+
+static void _rtw_ssmps(_adapter *adapter, struct sta_info *sta)
+{
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	issue_action_SM_PS_wait_ack(adapter , sta->phl_sta->mac_addr,
+			sta->phl_sta->asoc_cap.sm_ps, 3, 1);
+
+	if (sta->phl_sta->asoc_cap.sm_ps == SM_PS_STATIC) {
+		pmlmeext->txss_bk = sta->phl_sta->asoc_cap.nss_rx;
+		rtw_phl_sta_assoc_cap_process(sta->phl_sta, _TRUE);
+		sta->phl_sta->asoc_cap.nss_rx = 1;
+		sta->phl_sta->asoc_cap.stbc_ht_rx = 0;
+		sta->phl_sta->asoc_cap.stbc_vht_rx = 0;
+		sta->phl_sta->asoc_cap.stbc_he_rx = 0;
+	} else {
+		sta->phl_sta->asoc_cap.nss_rx = pmlmeext->txss_bk;
+		rtw_phl_sta_assoc_cap_process(sta->phl_sta, _FALSE);
+	}
+
+	rtw_sta_hal_ra_mask_update_cmd(adapter, sta, RTW_CMDF_DIRECTLY);
+}
+
+void rtw_ssmps_enter(_adapter *adapter, struct sta_info *sta)
+{
+	if (sta->phl_sta->asoc_cap.sm_ps == SM_PS_STATIC)
+		return;
+
+	RTW_INFO(ADPT_FMT" STA [" MAC_FMT "]\n", ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+
+	sta->phl_sta->asoc_cap.sm_ps = SM_PS_STATIC;
+	_rtw_ssmps(adapter, sta);
+}
+
+void rtw_ssmps_leave(_adapter *adapter, struct sta_info *sta)
+{
+	if (sta->phl_sta->asoc_cap.sm_ps == SM_PS_DISABLE)
+		return;
+
+	RTW_INFO(ADPT_FMT" STA [" MAC_FMT "] \n", ADPT_ARG(adapter), MAC_ARG(sta->phl_sta->mac_addr));
+	sta->phl_sta->asoc_cap.sm_ps = SM_PS_DISABLE;
+	_rtw_ssmps(adapter, sta);
+}
+
+void rtw_update_ips_setting(int make_level, int ins_level, u8 *mode, u8 *cap, bool is_wow)
+{
+	if (ins_level != PS_IPS_MAX)
+		make_level = ins_level;
+
+	switch (make_level) {
+	case PS_IPS_NONE:
+		*mode = PS_OP_MODE_DISABLED;
+		*cap = PS_CAP_PWRON;
+		break;
+	case PS_IPS_RF_OFF:
+		*mode = is_wow? PS_OP_MODE_FORCE_ENABLED: PS_OP_MODE_AUTO;
+		*cap = PS_CAP_PWRON | PS_CAP_RF_OFF;
+		break;
+	case PS_IPS_CLK_GATED:
+		*mode = is_wow? PS_OP_MODE_FORCE_ENABLED: PS_OP_MODE_AUTO;
+		*cap = PS_CAP_PWRON | PS_CAP_RF_OFF | PS_CAP_CLK_GATED;
+		break;
+	case PS_IPS_PWR_GATED:
+		*mode = is_wow? PS_OP_MODE_FORCE_ENABLED: PS_OP_MODE_AUTO;
+		*cap = PS_CAP_PWRON | PS_CAP_RF_OFF | PS_CAP_CLK_GATED | PS_CAP_PWR_GATED;
+		break;
+	case PS_PWR_OFF:
+		*mode = is_wow? PS_OP_MODE_FORCE_ENABLED: PS_OP_MODE_AUTO;
+		*cap = PS_CAP_PWR_OFF;
+		break;
+	default:
+		RTW_ERR("%s ips mode level (%d) invalid\n", __func__, ins_level);
+	}
+}
+
+void rtw_update_lps_setting(int make_level, int ins_level, u8 *mode, u8 *cap, bool is_wow)
+{
+	if (ins_level != PS_LPS_MAX)
+		make_level = ins_level;
+
+	switch (make_level) {
+	case PS_LPS_NONE:
+		*mode = PS_OP_MODE_DISABLED;
+		*cap = PS_CAP_PWRON;
+		break;
+	case PS_LPS_RF_OFF:
+		*mode = is_wow? PS_OP_MODE_FORCE_ENABLED: PS_OP_MODE_AUTO;
+		*cap = PS_CAP_PWRON | PS_CAP_RF_OFF;
+		break;
+	case PS_LPS_CLK_GATED:
+		*mode = is_wow? PS_OP_MODE_FORCE_ENABLED: PS_OP_MODE_AUTO;
+		*cap = PS_CAP_PWRON | PS_CAP_RF_OFF | PS_CAP_CLK_GATED;
+		break;
+	case PS_LPS_PWR_GATED:
+		*mode = is_wow? PS_OP_MODE_FORCE_ENABLED: PS_OP_MODE_AUTO;
+		*cap = PS_CAP_PWRON | PS_CAP_RF_OFF | PS_CAP_CLK_GATED | PS_CAP_PWR_GATED;
+		break;
+	default:
+		RTW_ERR("%s lps mode level (%d) invalid\n", __func__, ins_level);
+	}
+}
+
+#if defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW)
+void rtw_update_lps_listen_beacon_mode(
+	u8 bcn_mode, u8 awake_interval,
+	enum rtw_lps_listen_bcn_mode *lps_bcn_mode, u8 *lps_awake_interval)
+{
+	if (bcn_mode == RTW_LPS_LISTEN_BCN_MAX) {
+		*lps_bcn_mode = RTW_LPS_RLBM_MAX;
+		*lps_awake_interval = 0;
+	} else if (bcn_mode == RTW_LPS_RLBM_MIN ||
+		   bcn_mode == RTW_LPS_RLBM_MAX) {
+		*lps_bcn_mode = bcn_mode;
+		*lps_awake_interval = 0;
+		RTW_INFO(
+			"%s: awake interval will not be applied, since beacon mode is not RTW_LPS_RLBM_USERDEFINE\n",
+			__func__);
+	} else if (bcn_mode == RTW_LPS_RLBM_USERDEFINE) {
+		*lps_bcn_mode = bcn_mode;
+		*lps_awake_interval = awake_interval;
+	} else {
+		RTW_ERR("%s lps listen beacon mode (%d) invalid\n", __func__,
+			bcn_mode);
+	}
+}
+#endif /* defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW) */
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_recv.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_recv.c
new file mode 100644
index 000000000000..2c80511c4aeb
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_recv.c
@@ -0,0 +1,5864 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_RECV_C_
+
+#include <drv_types.h>
+
+static void rtw_signal_stat_timer_hdl(void *ctx);
+
+enum {
+	SIGNAL_STAT_CALC_PROFILE_0 = 0,
+	SIGNAL_STAT_CALC_PROFILE_1,
+#ifdef CONFIG_MP_INCLUDED
+	SIGNAL_STAT_CALC_PROFILE_2,
+#endif
+	SIGNAL_STAT_CALC_PROFILE_MAX
+};
+
+u8 signal_stat_calc_profile[SIGNAL_STAT_CALC_PROFILE_MAX][2] = {
+	{4, 1},	/* Profile 0 => pre_stat : curr_stat = 4 : 1 */
+	{3, 7}	/* Profile 1 => pre_stat : curr_stat = 3 : 7 */
+#ifdef CONFIG_MP_INCLUDED
+	,{0, 10}
+#endif
+};
+
+#ifndef RTW_SIGNAL_STATE_CALC_PROFILE
+	#define RTW_SIGNAL_STATE_CALC_PROFILE SIGNAL_STAT_CALC_PROFILE_1
+#endif
+
+u8 rtw_bridge_tunnel_header[] = { 0xaa, 0xaa, 0x03, 0x00, 0x00, 0xf8 };
+u8 rtw_rfc1042_header[] = { 0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00 };
+static u8 SNAP_ETH_TYPE_IPX[2] = {0x81, 0x37};
+static u8 SNAP_ETH_TYPE_APPLETALK_AARP[2] = {0x80, 0xf3};
+#ifdef CONFIG_TDLS
+static u8 SNAP_ETH_TYPE_TDLS[2] = {0x89, 0x0d};
+#endif
+
+void _rtw_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv)
+{
+
+	_rtw_memset((u8 *)psta_recvpriv, 0, sizeof(struct sta_recv_priv));
+
+	_rtw_spinlock_init(&psta_recvpriv->lock);
+
+	/* for(i=0; i<MAX_RX_NUMBLKS; i++) */
+	/*	_rtw_init_queue(&psta_recvpriv->blk_strms[i]); */
+
+	_rtw_init_queue(&psta_recvpriv->defrag_q);
+
+
+}
+
+u8 rtw_init_recv_info(_adapter *adapter)
+{
+	u8 ret = _SUCCESS;
+	struct recv_info *recvinfo = &adapter->recvinfo;
+
+	recvinfo->sink_udpport = 0;
+	recvinfo->pre_rtp_rxseq = 0;
+	recvinfo->cur_rtp_rxseq = 0;
+#ifdef DBG_RX_SIGNAL_DISPLAY_RAW_DATA
+	recvinfo->store_law_data_flag = _TRUE;
+#else
+	recvinfo->store_law_data_flag = _FALSE;
+#endif
+#ifdef PRIVATE_R
+	_rtw_memset(&(recvinfo->rx_vo_pkt_count_per_data_rate), 0, sizeof(recvinfo->rx_vo_pkt_count_per_data_rate));
+	recvinfo->rx_vo_pkt_retry_count = 0;
+	recvinfo->last_rx_uc_data = 0;
+	recvinfo->last_rx_uc_data_rate_avg = 0;
+#endif
+	#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	rtw_init_timer(&recvinfo->signal_stat_timer, rtw_signal_stat_timer_hdl, adapter);
+
+	recvinfo->signal_stat_sampling_interval = 2000; /* ms */
+	/* recvinfo->signal_stat_converging_constant = 5000; */ /* ms */
+	#endif
+	return ret;
+}
+
+/*#define DBG_RECV_FRAME*/
+#ifdef DBG_RECV_FRAME
+void _dump_recv_priv(struct dvobj_priv *dvobj, _queue *pfree_recv_queue)
+{
+	struct recv_priv *precvpriv = &dvobj->recvpriv;
+
+	RTW_INFO("%s free_recvframe_cnt:%d\n", __func__, precvpriv->free_recvframe_cnt);
+	RTW_INFO("%s dvobj:%p pfree_recv_queue:%p : %p\n",
+		__func__, dvobj, &(precvpriv->free_recv_queue), pfree_recv_queue);
+}
+
+#endif
+
+sint rtw_init_recv_priv(struct dvobj_priv *dvobj)
+{
+	sint i;
+	union recv_frame *precvframe;
+	sint	res = _SUCCESS;
+	struct recv_priv *precvpriv = &dvobj->recvpriv;
+
+	#ifdef CONFIG_RECV_THREAD_MODE
+	_rtw_init_sema(&precvpriv->recv_sema, 0);
+	#endif
+
+	_rtw_init_queue(&precvpriv->free_recv_queue);
+	#if 0
+	_rtw_init_queue(&precvpriv->uc_swdec_pending_queue);
+	#endif
+
+	precvpriv->dvobj = dvobj;
+
+	precvpriv->free_recvframe_cnt = NR_RECVFRAME;
+
+	rtw_os_recv_resource_init(precvpriv);
+
+	precvpriv->pallocated_frame_buf = rtw_zvmalloc(NR_RECVFRAME * sizeof(union recv_frame) + RXFRAME_ALIGN_SZ);
+
+	if (precvpriv->pallocated_frame_buf == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	/* _rtw_memset(precvpriv->pallocated_frame_buf, 0, NR_RECVFRAME * sizeof(union recv_frame) + RXFRAME_ALIGN_SZ); */
+
+	precvpriv->precv_frame_buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(precvpriv->pallocated_frame_buf), RXFRAME_ALIGN_SZ);
+	/* precvpriv->precv_frame_buf = precvpriv->pallocated_frame_buf + RXFRAME_ALIGN_SZ - */
+	/*						((SIZE_PTR) (precvpriv->pallocated_frame_buf) &(RXFRAME_ALIGN_SZ-1)); */
+
+	precvframe = (union recv_frame *) precvpriv->precv_frame_buf;
+
+
+	for (i = 0; i < NR_RECVFRAME ; i++) {
+		_rtw_init_listhead(&(precvframe->u.list));
+
+		rtw_list_insert_tail(&(precvframe->u.list), &(precvpriv->free_recv_queue.queue));
+
+		rtw_os_recv_resource_alloc(precvframe);
+
+		precvframe->u.hdr.len = 0;
+
+		precvframe->u.hdr.dvobj = dvobj;
+		precvframe->u.hdr.adapter = NULL;
+		precvframe->u.hdr.rx_req = NULL;
+
+		precvframe++;
+	}
+	#ifdef DBG_RECV_FRAME
+	RTW_INFO("%s =>precvpriv->free_recvframe_cnt:%d\n", __func__, precvpriv->free_recvframe_cnt);
+	#endif
+
+	res = rtw_intf_init_recv_priv(dvobj);
+	#ifdef DBG_RECV_FRAME
+	_dump_recv_priv(dvobj, &dvobj->recvpriv.free_recv_queue);
+	#endif
+exit:
+	return res;
+
+}
+
+void rtw_free_recv_priv(struct dvobj_priv *dvobj)
+{
+	struct recv_priv *precvpriv = &dvobj->recvpriv;
+
+	#if 0
+	rtw_free_uc_swdec_pending_queue(dvobj);
+	#endif
+
+#ifdef CONFIG_RECV_THREAD_MODE
+	_rtw_free_sema(&precvpriv->recv_sema);
+#endif
+
+	rtw_os_recv_resource_free(precvpriv);
+
+	if (precvpriv->pallocated_frame_buf)
+		rtw_vmfree(precvpriv->pallocated_frame_buf, NR_RECVFRAME * sizeof(union recv_frame) + RXFRAME_ALIGN_SZ);
+
+	_rtw_deinit_queue(&precvpriv->free_recv_queue);
+
+	rtw_intf_free_recv_priv(dvobj);
+}
+
+union recv_frame *_rtw_alloc_recvframe(_queue *pfree_recv_queue)
+{
+	union recv_frame  *precvframe;
+	_list	*plist, *phead;
+	struct recv_priv *precvpriv;
+	struct dvobj_priv *dvobj;
+
+#ifdef DBG_RECV_FRAME
+	RTW_INFO("%s =>pfree_recv_queue:%p\n", __func__, pfree_recv_queue);
+#endif
+
+	if (_rtw_queue_empty(pfree_recv_queue) == _TRUE) {
+		precvframe = NULL;
+	}
+	else {
+		phead = get_list_head(pfree_recv_queue);
+
+		plist = get_next(phead);
+
+		precvframe = LIST_CONTAINOR(plist, union recv_frame, u);
+		rtw_list_delete(&precvframe->u.hdr.list);
+		dvobj = precvframe->u.hdr.dvobj;
+		precvpriv = &dvobj->recvpriv;
+		precvpriv->free_recvframe_cnt--;
+	}
+
+	return precvframe;
+
+}
+union recv_frame *rtw_alloc_recvframe(_queue *pfree_recv_queue)
+{
+	union recv_frame *precvframe = NULL;
+	#ifdef DBG_RECV_FRAME
+	struct recv_priv *precvpriv;
+	struct dvobj_priv *dvobj;
+
+	RTW_INFO("%s =>pfree_recv_queue:%p\n", __func__, pfree_recv_queue);
+	#endif
+	_rtw_spinlock_bh(&pfree_recv_queue->lock);
+
+	precvframe = _rtw_alloc_recvframe(pfree_recv_queue);
+
+	_rtw_spinunlock_bh(&pfree_recv_queue->lock);
+
+	if(precvframe) {
+		precvframe->u.hdr.rx_req = NULL;
+		#ifdef DBG_RECV_FRAME
+		dvobj = precvframe->u.hdr.dvobj;
+		precvpriv = &dvobj->recvpriv;
+		RTW_INFO("%s =>dvobj:%p precvpriv->free_recvframe_cnt:%d\n",
+			__func__,
+			dvobj,
+			precvpriv->free_recvframe_cnt);
+		#endif
+	}
+	return precvframe;
+}
+
+void rtw_init_recvframe(union recv_frame *precvframe)
+{
+	/* Perry: This can be removed */
+	_rtw_init_listhead(&precvframe->u.hdr.list);
+
+	precvframe->u.hdr.len = 0;
+}
+
+int rtw_free_recvframe(union recv_frame *precvframe)
+{
+	struct dvobj_priv *dvobj;
+	struct recv_priv *precvpriv;
+	_queue	*pfree_recv_queue;
+
+	if(!precvframe) {
+		RTW_ERR("%s precvframe is NULL\n", __func__);
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+
+	dvobj = precvframe->u.hdr.dvobj;
+	precvpriv = &dvobj->recvpriv;
+	pfree_recv_queue = &(precvpriv->free_recv_queue);
+#ifdef DBG_RECV_FRAME
+	RTW_INFO("%s dvobj:%p, phl:%p\n", __func__,dvobj, dvobj->phl);
+#endif
+
+#ifdef RTW_PHL_RX
+	if(precvframe->u.hdr.rx_req)
+		rtw_phl_return_rxbuf(GET_PHL_INFO(dvobj), (u8*)precvframe->u.hdr.rx_req);
+#endif
+
+	rtw_os_free_recvframe(precvframe);
+
+	_rtw_spinlock_bh(&pfree_recv_queue->lock);
+
+	rtw_list_delete(&(precvframe->u.hdr.list));
+
+	precvframe->u.hdr.len = 0;
+
+	rtw_list_insert_tail(&(precvframe->u.hdr.list), get_list_head(pfree_recv_queue));
+	precvpriv->free_recvframe_cnt++;
+
+	_rtw_spinunlock_bh(&pfree_recv_queue->lock);
+#ifdef DBG_RECV_FRAME
+	RTW_INFO("%s =>precvpriv->free_recvframe_cnt:%d\n", __func__, precvpriv->free_recvframe_cnt);
+#endif
+
+	return _SUCCESS;
+
+}
+
+
+bool rtw_rframe_del_wfd_ie(union recv_frame *rframe, u8 ies_offset)
+{
+#define DBG_RFRAME_DEL_WFD_IE 0
+	u8 *ies = rframe->u.hdr.rx_data + sizeof(struct rtw_ieee80211_hdr_3addr) + ies_offset;
+	uint ies_len_ori = rframe->u.hdr.len - (ies - rframe->u.hdr.rx_data);
+	uint ies_len;
+
+	ies_len = rtw_del_wfd_ie(ies, ies_len_ori, DBG_RFRAME_DEL_WFD_IE ? __func__ : NULL);
+	rframe->u.hdr.len -= ies_len_ori - ies_len;
+
+	return ies_len_ori != ies_len;
+}
+
+#if 0
+sint _rtw_enqueue_recvframe(union recv_frame *precvframe, _queue *queue)
+{
+
+	_adapter *padapter = precvframe->u.hdr.adapter;
+	struct recv_priv *precvpriv = &padapter->recvpriv;
+
+
+	/* _rtw_init_listhead(&(precvframe->u.hdr.list)); */
+	rtw_list_delete(&(precvframe->u.hdr.list));
+
+
+	rtw_list_insert_tail(&(precvframe->u.hdr.list), get_list_head(queue));
+
+	if (padapter != NULL) {
+		if (queue == &precvpriv->free_recv_queue)
+			precvpriv->free_recvframe_cnt++;
+	}
+
+
+	return _SUCCESS;
+}
+
+sint rtw_enqueue_recvframe(union recv_frame *precvframe, _queue *queue)
+{
+	sint ret;
+
+	/* _spinlock(&pfree_recv_queue->lock); */
+	_rtw_spinlock_bh(&queue->lock);
+	ret = _rtw_enqueue_recvframe(precvframe, queue);
+	/* _rtw_spinunlock(&pfree_recv_queue->lock); */
+	_rtw_spinunlock_bh(&queue->lock);
+
+	return ret;
+}
+#endif
+
+/*
+caller : defrag ; recvframe_chk_defrag in recv_thread  (passive)
+pframequeue: defrag_queue : will be accessed in recv_thread  (passive)
+
+using spinlock to protect
+
+*/
+
+void rtw_free_recvframe_queue(_queue *pframequeue, u8 need_lock)
+{
+	union	recv_frame	*precvframe;
+	_list	*plist, *phead;
+
+	if (need_lock)
+		_rtw_spinlock_bh(&pframequeue->lock);
+
+	phead = get_list_head(pframequeue);
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		precvframe = LIST_CONTAINOR(plist, union recv_frame, u);
+
+		plist = get_next(plist);
+
+		/* rtw_list_delete(&precvframe->u.hdr.list); */ /* will do this in rtw_free_recvframe() */
+
+		rtw_free_recvframe(precvframe);
+	}
+
+	if (need_lock)
+		_rtw_spinunlock_bh(&pframequeue->lock);
+
+
+}
+
+#if 0
+u32 rtw_free_uc_swdec_pending_queue(struct dvobj_priv *dvobj)
+{
+	u32 cnt = 0;
+	union recv_frame *pending_frame;
+	while ((pending_frame = rtw_alloc_recvframe(&dvobj->recvpriv.uc_swdec_pending_queue))) {
+		rtw_free_recvframe(pending_frame);
+		cnt++;
+	}
+
+	if (cnt)
+		RTW_INFO("dequeue %d\n", cnt);
+
+	return cnt;
+}
+#endif
+
+sint recvframe_chkmic(_adapter *adapter,  union recv_frame *precvframe)
+{
+
+	sint	i, res = _SUCCESS;
+	u32	datalen;
+	u8	miccode[8];
+	u8	bmic_err = _FALSE, brpt_micerror = _TRUE;
+	u8	*pframe, *payload, *pframemic;
+	u8	*mickey;
+	/* u8	*iv,rxdata_key_idx=0; */
+	struct _ADAPTER_LINK *adapter_link;
+	struct	link_security_priv	*lsecuritypriv;
+	struct	sta_info		*stainfo;
+	struct	rx_pkt_attrib	*prxattrib = &precvframe->u.hdr.attrib;
+	struct	security_priv	*psecuritypriv = &adapter->securitypriv;
+	struct	mlme_ext_priv	*pmlmeext = &(adapter->mlmeextpriv);
+	struct	mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	stainfo = precvframe->u.hdr.psta;
+
+	if (prxattrib->encrypt == _TKIP_) {
+
+		/* calculate mic code */
+		if (stainfo != NULL) {
+			adapter_link = stainfo->padapter_link;
+			lsecuritypriv = &adapter_link->securitypriv;
+			if (IS_MCAST(prxattrib->ra)) {
+				/* mickey=&psecuritypriv->dot118021XGrprxmickey.skey[0]; */
+				/* iv = precvframe->u.hdr.rx_data+prxattrib->hdrlen; */
+				/* rxdata_key_idx =( ((iv[3])>>6)&0x3) ; */
+				mickey = &lsecuritypriv->dot118021XGrprxmickey[prxattrib->key_index].skey[0];
+
+				/* RTW_INFO("\n recvframe_chkmic: bcmc key psecuritypriv->dot118021XGrpKeyid(%d),pmlmeinfo->key_index(%d) ,recv key_id(%d)\n", */
+				/*								psecuritypriv->dot118021XGrpKeyid,pmlmeinfo->key_index,rxdata_key_idx); */
+
+				if (lsecuritypriv->binstallGrpkey == _FALSE)
+				{
+					res = _FAIL;
+					RTW_INFO("\n recvframe_chkmic:didn't install group key!!!!!!!!!!\n");
+					goto exit;
+				}
+			} else {
+				mickey = &stainfo->dot11tkiprxmickey.skey[0];
+			}
+
+			datalen = precvframe->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len - prxattrib->icv_len - 8; /* icv_len included the mic code */
+			pframe = precvframe->u.hdr.rx_data;
+			payload = pframe + prxattrib->hdrlen + prxattrib->iv_len;
+
+
+			/* rtw_seccalctkipmic(&stainfo->dot11tkiprxmickey.skey[0],pframe,payload, datalen ,&miccode[0],(unsigned char)prxattrib->priority); */ /* care the length of the data */
+
+			rtw_seccalctkipmic(mickey, pframe, payload, datalen , &miccode[0], (unsigned char)prxattrib->priority); /* care the length of the data */
+
+			pframemic = payload + datalen;
+
+			bmic_err = _FALSE;
+
+			for (i = 0; i < 8; i++) {
+				if (miccode[i] != *(pframemic + i)) {
+					bmic_err = _TRUE;
+				}
+			}
+
+
+			if (bmic_err == _TRUE) {
+
+
+
+				/* double check key_index for some timing issue , */
+				/* cannot compare with psecuritypriv->dot118021XGrpKeyid also cause timing issue */
+				if ((IS_MCAST(prxattrib->ra) == _TRUE)  && (prxattrib->key_index != pmlmeinfo->key_index))
+					brpt_micerror = _FALSE;
+
+				if ((prxattrib->bdecrypted == _TRUE) && (brpt_micerror == _TRUE)) {
+					rtw_handle_tkip_mic_err(adapter, stainfo, (u8)IS_MCAST(prxattrib->ra));
+					RTW_INFO(" mic error :prxattrib->bdecrypted=%d\n", prxattrib->bdecrypted);
+				} else {
+					RTW_INFO(" mic error :prxattrib->bdecrypted=%d\n", prxattrib->bdecrypted);
+				}
+
+				res = _FAIL;
+
+			} else {
+				/* mic checked ok */
+				if ((lsecuritypriv->bcheck_grpkey == _FALSE) && (IS_MCAST(prxattrib->ra) == _TRUE))
+				{
+					lsecuritypriv->bcheck_grpkey = _TRUE;
+				}
+			}
+
+		}
+
+		recvframe_pull_tail(precvframe, 8);
+
+	}
+
+exit:
+
+
+	return res;
+
+}
+
+/*#define DBG_RX_SW_DECRYPTOR*/
+
+/* decrypt and set the ivlen,icvlen of the recv_frame */
+union recv_frame *decryptor(_adapter *padapter, union recv_frame *precv_frame)
+{
+
+	struct rx_pkt_attrib *prxattrib = &precv_frame->u.hdr.attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	union recv_frame *return_packet = precv_frame;
+	struct sta_info *psta = precv_frame->u.hdr.psta;
+	u32  res = _SUCCESS;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+
+	DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt);
+
+
+	if (prxattrib->encrypt > 0) {
+		u8 *iv = precv_frame->u.hdr.rx_data + prxattrib->hdrlen;
+		prxattrib->key_index = (((iv[3]) >> 6) & 0x3) ;
+
+		if (prxattrib->key_index > WEP_KEYS) {
+			RTW_INFO("prxattrib->key_index(%d) > WEP_KEYS\n", prxattrib->key_index);
+
+			switch (prxattrib->encrypt) {
+			case _WEP40_:
+			case _WEP104_:
+				prxattrib->key_index = psecuritypriv->dot11PrivacyKeyIndex;
+				break;
+			case _TKIP_:
+			case _AES_:
+			case _GCMP_:
+			case _GCMP_256_:
+			case _CCMP_256_:
+			default:
+				prxattrib->key_index = lsecuritypriv->dot118021XGrpKeyid;
+				break;
+			}
+		}
+	}
+
+	if (prxattrib->encrypt && !prxattrib->bdecrypted) {
+		if (GetFrameType(get_recvframe_data(precv_frame)) == WIFI_DATA
+			#ifdef CONFIG_CONCURRENT_MODE
+			&& !IS_MCAST(prxattrib->ra) /* bc/mc packets may use sw decryption for concurrent mode */
+			#endif
+		) {
+			if (IS_MCAST(prxattrib->ra))
+				psecuritypriv->hw_decrypted = _FALSE;
+			else
+				psta->hw_decrypted = _FALSE;
+		}
+
+#ifdef DBG_RX_SW_DECRYPTOR
+		RTW_INFO(ADPT_FMT" - sec_type:%s DO SW decryption\n",
+			ADPT_ARG(padapter), security_type_str(prxattrib->encrypt));
+#endif
+
+#ifdef DBG_RX_DECRYPTOR
+		RTW_INFO("[%s] %d: PKT decrypted(%d), PKT encrypt(%d), Set %pM hw_decrypted(%d)\n",
+			 __FUNCTION__,
+			 __LINE__,
+			 prxattrib->bdecrypted,
+			 prxattrib->encrypt,
+			 psta->phl_sta->mac_addr,
+			 psta->hw_decrypted);
+#endif
+
+		switch (prxattrib->encrypt) {
+		case _WEP40_:
+		case _WEP104_:
+			DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_wep);
+			rtw_wep_decrypt(padapter, (u8 *)precv_frame);
+			break;
+		case _TKIP_:
+			DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_tkip);
+			res = rtw_tkip_decrypt(padapter, (u8 *)precv_frame);
+			break;
+		case _AES_:
+		case _CCMP_256_:
+			DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_aes);
+			res = rtw_aes_decrypt(padapter, (u8 *)precv_frame);
+			break;
+		case _GCMP_:
+		case _GCMP_256_:
+			DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_gcmp);
+			res = rtw_gcmp_decrypt(padapter, (u8 *)precv_frame);
+			break;
+#ifdef CONFIG_WAPI_SUPPORT
+		case _SMS4_:
+			DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_wapi);
+			rtw_sms4_decrypt(padapter, (u8 *)precv_frame);
+			break;
+#endif
+		default:
+			break;
+		}
+	} else if (prxattrib->bdecrypted == 1
+		   && prxattrib->encrypt > 0
+		&& (psecuritypriv->busetkipkey == 1 || prxattrib->encrypt != _TKIP_)
+		  ) {
+#if 0
+		if ((prxstat->icv == 1) && (prxattrib->encrypt != _AES_)) {
+			psecuritypriv->hw_decrypted = _FALSE;
+
+
+			rtw_free_recvframe(precv_frame);
+
+			return_packet = NULL;
+
+		} else
+#endif
+		{
+			DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_hw);
+
+			psecuritypriv->hw_decrypted = _TRUE;
+			psta->hw_decrypted = _TRUE;
+#ifdef DBG_RX_DECRYPTOR
+			RTW_INFO("[%s] %d: PKT decrypted(%d), PKT encrypt(%d), Set %pM hw_decrypted(%d)\n",
+			 __FUNCTION__,
+			 __LINE__,
+			 prxattrib->bdecrypted,
+			 prxattrib->encrypt,
+			 psta->phl_sta->mac_addr,
+			 psta->hw_decrypted);
+#endif
+		}
+	} else {
+		DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_unknown);
+#ifdef DBG_RX_DECRYPTOR
+		RTW_INFO("[%s] %d: PKT decrypted(%d), PKT encrypt(%d), Set %pM hw_decrypted(%d)\n",
+			 __FUNCTION__,
+			 __LINE__,
+			 prxattrib->bdecrypted,
+			 prxattrib->encrypt,
+			 psta->phl_sta->mac_addr,
+			 psta->hw_decrypted);
+#endif
+	}
+
+	#ifdef CONFIG_RTW_MESH
+	if (res != _FAIL
+		&& !prxattrib->amsdu
+		&& prxattrib->mesh_ctrl_present)
+		res = rtw_mesh_rx_validate_mctrl_non_amsdu(padapter, precv_frame);
+	#endif
+
+	if (res == _FAIL) {
+		/* Let rtw_core_rx_process or rtw_mi_buddy_clone_bcmc_packet */
+		/* to handle it.*/
+		/* rtw_free_recvframe(return_packet); */
+		return_packet = NULL;
+	} else
+		prxattrib->bdecrypted = _TRUE;
+	/* recvframe_chkmic(adapter, precv_frame);   */ /* move to recvframme_defrag function */
+
+
+	return return_packet;
+
+}
+
+/* ###set the security information in the recv_frame */
+union recv_frame *portctrl(_adapter *adapter, union recv_frame *precv_frame)
+{
+	u8 *psta_addr = NULL;
+	u8 *ptr;
+	uint  auth_alg;
+	struct recv_frame_hdr *pfhdr;
+	struct sta_info *psta;
+	struct sta_priv *pstapriv ;
+	union recv_frame *prtnframe;
+	u16	ether_type = 0;
+	u16  eapol_type = 0x888e;/* for Funia BD's WPA issue  */
+	struct rx_pkt_attrib *pattrib;
+
+
+	pstapriv = &adapter->stapriv;
+
+	auth_alg = adapter->securitypriv.dot11AuthAlgrthm;
+
+	ptr = get_recvframe_data(precv_frame);
+	pfhdr = &precv_frame->u.hdr;
+	pattrib = &pfhdr->attrib;
+	psta_addr = pattrib->ta;
+
+	prtnframe = NULL;
+
+	psta = precv_frame->u.hdr.psta;
+
+
+	if (auth_alg == dot11AuthAlgrthm_8021X) {
+		if ((psta != NULL) && (psta->ieee8021x_blocked)) {
+			/* blocked */
+			/* only accept EAPOL frame */
+
+			prtnframe = precv_frame;
+
+			/* get ether_type */
+			ptr = ptr + pfhdr->attrib.hdrlen + pfhdr->attrib.iv_len + LLC_HEADER_SIZE;
+			_rtw_memcpy(&ether_type, ptr, 2);
+			ether_type = ntohs((unsigned short)ether_type);
+
+			if (ether_type == eapol_type)
+				prtnframe = precv_frame;
+			else {
+				/* Let rtw_core_rx_process to handle it */
+				/* rtw_free_recvframe(precv_frame); */
+				prtnframe = NULL;
+			}
+		} else {
+			/* allowed */
+			/* check decryption status, and decrypt the frame if needed */
+
+
+			prtnframe = precv_frame;
+			/* check is the EAPOL frame or not (Rekey) */
+			/* if(ether_type == eapol_type){ */
+			/* check Rekey */
+
+			/*	prtnframe=precv_frame; */
+			/* } */
+		}
+	} else
+		prtnframe = precv_frame;
+
+
+	return prtnframe;
+
+}
+
+/* VALID_PN_CHK
+ * Return true when PN is legal, otherwise false.
+ * Legal PN:
+ *	1. If old PN is 0, any PN is legal
+ *	2. PN > old PN
+ */
+#define PN_LESS_CHK(a, b)	(((a-b) & 0x800000000000) != 0)
+#define VALID_PN_CHK(new, old)	(((old) == 0) || PN_LESS_CHK(old, new))
+
+sint recv_ucast_pn_decache(union recv_frame *precv_frame)
+{
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_info *sta = precv_frame->u.hdr.psta;
+	struct stainfo_rxcache *prxcache = &sta->sta_recvpriv.rxcache;
+	u8 *pdata = precv_frame->u.hdr.rx_data;
+	sint tid = precv_frame->u.hdr.attrib.priority;
+	u64 curr_pn = 0, pkt_pn = 0;
+	u8 pn[8] = {0};
+
+	if (tid > 15)
+		return _FAIL;
+
+	if (pattrib->encrypt == _TKIP_ || pattrib->encrypt == _AES_ ||
+	    pattrib->encrypt == _GCMP_ || pattrib->encrypt == _CCMP_256_ ||
+	    pattrib->encrypt == _GCMP_256_) {
+		rtw_iv_to_pn((pdata + pattrib->hdrlen), pn, NULL, pattrib->encrypt);
+		pkt_pn = RTW_GET_LE64(pn);
+
+		rtw_iv_to_pn(prxcache->iv[tid], pn, NULL, pattrib->encrypt);
+		curr_pn = RTW_GET_LE64(pn);
+
+		if (!VALID_PN_CHK(pkt_pn, curr_pn)) {
+			/* return _FAIL; */
+		} else {
+			prxcache->last_tid = tid;
+			_rtw_memcpy(prxcache->iv[tid],
+				    (pdata + pattrib->hdrlen),
+				    sizeof(prxcache->iv[tid]));
+		}
+	}
+
+	return _SUCCESS;
+}
+
+sint recv_bcast_pn_decache(union recv_frame *precv_frame)
+{
+	_adapter *padapter = precv_frame->u.hdr.adapter;
+	struct _ADAPTER_LINK *adapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_security_priv *lsecuritypriv = &adapter_link->securitypriv;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	u8 *pdata = precv_frame->u.hdr.rx_data;
+	u64 curr_pn = 0, pkt_pn = 0;
+	u8 pn[8] = {0};
+	u8 key_id;
+
+	if ((pattrib->encrypt == _TKIP_ || pattrib->encrypt == _AES_ ||
+	     pattrib->encrypt == _GCMP_ || pattrib->encrypt == _CCMP_256_ ||
+	     pattrib->encrypt == _GCMP_256_) &&
+	    (MLME_IS_STA(padapter))) {
+		rtw_iv_to_pn((pdata + pattrib->hdrlen), pn, &key_id,
+			     pattrib->encrypt);
+		pkt_pn = RTW_GET_LE64(pn);
+		curr_pn = le64_to_cpu(*(u64*)lsecuritypriv->iv_seq[key_id]);
+
+		if (!VALID_PN_CHK(pkt_pn, curr_pn))
+			return _FAIL;
+
+		*(u64*)lsecuritypriv->iv_seq[key_id] = cpu_to_le64(pkt_pn);
+	}
+
+	return _SUCCESS;
+}
+
+sint recv_decache(union recv_frame *precv_frame)
+{
+	struct sta_info *psta = precv_frame->u.hdr.psta;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	_adapter *adapter = psta->padapter;
+	sint tid = pattrib->priority;
+	u16 seq_ctrl = ((precv_frame->u.hdr.attrib.seq_num & 0xffff) << 4) |
+		       (precv_frame->u.hdr.attrib.frag_num & 0xf);
+	u16 *prxseq;
+
+	if (tid > 15)
+		return _FAIL;
+
+	if (pattrib->qos) {
+		if (IS_MCAST(pattrib->ra))
+			prxseq = &psta->sta_recvpriv.bmc_tid_rxseq[tid];
+		else
+			prxseq = &psta->sta_recvpriv.rxcache.tid_rxseq[tid];
+	} else {
+		if (IS_MCAST(pattrib->ra)) {
+			prxseq = &psta->sta_recvpriv.nonqos_bmc_rxseq;
+			#ifdef DBG_RX_SEQ
+			RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" nonqos bmc seq_num:%d\n"
+				, FUNC_ADPT_ARG(adapter), pattrib->seq_num);
+			#endif
+
+		} else {
+			prxseq = &psta->sta_recvpriv.nonqos_rxseq;
+			#ifdef DBG_RX_SEQ
+			RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" nonqos seq_num:%d\n"
+				, FUNC_ADPT_ARG(adapter), pattrib->seq_num);
+			#endif
+		}
+	}
+
+	if (seq_ctrl == *prxseq) {
+		/* for non-AMPDU case	*/
+		psta->sta_stats.duplicate_cnt++;
+
+		if (psta->sta_stats.duplicate_cnt % 100 == 0)
+			RTW_INFO("%s: tid=%u seq=%d frag=%d\n", __func__
+				, tid, precv_frame->u.hdr.attrib.seq_num
+				, precv_frame->u.hdr.attrib.frag_num);
+
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recv_decache _FAIL for sta="MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->phl_sta->mac_addr));
+		#endif
+		return _FAIL;
+	}
+	*prxseq = seq_ctrl;
+
+	return _SUCCESS;
+}
+
+void process_pwrbit_data(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *psta)
+{
+#ifdef CONFIG_AP_MODE
+	unsigned char pwrbit;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+
+	pwrbit = GetPwrMgt(ptr);
+
+	if (pwrbit) {
+		if (!(psta->state & WIFI_SLEEP_STATE)) {
+			/* psta->state |= WIFI_SLEEP_STATE; */
+			/* rtw_tim_map_set(padapter, pstapriv->sta_dz_bitmap, BIT(psta->phl_sta->aid)); */
+
+			stop_sta_xmit(padapter, psta);
+			/* RTW_INFO_DUMP("to sleep, sta_dz_bitmap=", pstapriv->sta_dz_bitmap, pstapriv->aid_bmp_len); */
+		}
+	} else {
+		if (psta->state & WIFI_SLEEP_STATE) {
+			/* psta->state ^= WIFI_SLEEP_STATE; */
+			/* rtw_tim_map_clear(padapter, pstapriv->sta_dz_bitmap, BIT(psta->phl_sta->aid)); */
+
+			wakeup_sta_to_xmit(padapter, psta);
+			/* RTW_INFO_DUMP("to wakeup, sta_dz_bitmap=", pstapriv->sta_dz_bitmap, pstapriv->aid_bmp_len); */
+		}
+	}
+#endif
+}
+
+void process_wmmps_data(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *psta)
+{
+#ifdef CONFIG_AP_MODE
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+
+#ifdef CONFIG_TDLS
+	if (!(psta->tdls_sta_state & TDLS_LINKED_STATE)) {
+#endif /* CONFIG_TDLS */
+
+		if (!psta->qos_option)
+			return;
+
+		if (!(psta->qos_info & 0xf))
+			return;
+
+#ifdef CONFIG_TDLS
+	}
+#endif /* CONFIG_TDLS		 */
+
+	if (psta->state & WIFI_SLEEP_STATE) {
+		u8 wmmps_ac = 0;
+
+		switch (pattrib->priority) {
+		case 1:
+		case 2:
+			wmmps_ac = psta->uapsd_bk & BIT(1);
+			break;
+		case 4:
+		case 5:
+			wmmps_ac = psta->uapsd_vi & BIT(1);
+			break;
+		case 6:
+		case 7:
+			wmmps_ac = psta->uapsd_vo & BIT(1);
+			break;
+		case 0:
+		case 3:
+		default:
+			wmmps_ac = psta->uapsd_be & BIT(1);
+			break;
+		}
+
+		if (wmmps_ac) {
+			if (psta->sleepq_ac_len > 0) {
+				/* process received triggered frame */
+				xmit_delivery_enabled_frames(padapter, psta);
+			} else {
+				/* issue one qos null frame with More data bit = 0 and the EOSP bit set (=1) */
+				issue_qos_nulldata(padapter, precv_frame->u.hdr.adapter_link,
+						psta->phl_sta->mac_addr, (u16)pattrib->priority, 0, 0, 0, _FALSE);
+			}
+		}
+
+	}
+
+
+#endif
+
+}
+
+#ifdef CONFIG_TDLS
+sint OnTDLS(_adapter *adapter, union recv_frame *precv_frame)
+{
+	struct rx_pkt_attrib	*pattrib = &precv_frame->u.hdr.attrib;
+	sint ret = _SUCCESS;
+	u8 *paction = get_recvframe_data(precv_frame);
+	u8 category_field = 1;
+#ifdef CONFIG_WFD
+	u8 WFA_OUI[3] = { 0x50, 0x6f, 0x9a };
+#endif /* CONFIG_WFD */
+	struct tdls_info *ptdlsinfo = &(adapter->tdlsinfo);
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct sta_priv *pstapriv = &(adapter->stapriv);
+	struct sta_info *ptdls_sta = NULL;
+
+	/* point to action field */
+	paction += pattrib->hdrlen
+		   + pattrib->iv_len
+		   + SNAP_SIZE
+		   + ETH_TYPE_LEN
+		   + PAYLOAD_TYPE_LEN
+		   + category_field;
+
+	RTW_INFO("[TDLS] Recv %s from "MAC_FMT" with SeqNum = %d\n", rtw_tdls_action_txt(*paction), MAC_ARG(pattrib->src), GetSequence(get_recvframe_data(precv_frame)));
+
+	if (rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_TDLS) == _FALSE) {
+		RTW_INFO("Ignore tdls frame since hal doesn't support tdls\n");
+		ret = _FAIL;
+		return ret;
+	}
+
+	if (rtw_is_tdls_enabled(adapter) == _FALSE) {
+		RTW_INFO("recv tdls frame, "
+			 "but tdls haven't enabled\n");
+		ret = _FAIL;
+		return ret;
+	}
+
+	ptdls_sta = rtw_get_stainfo(pstapriv, get_sa(ptr));
+	if (ptdls_sta == NULL) {
+		switch (*paction) {
+		case TDLS_SETUP_REQUEST:
+		case TDLS_DISCOVERY_REQUEST:
+			break;
+		default:
+			RTW_INFO("[TDLS] %s - Direct Link Peer = "MAC_FMT" not found for action = %d\n", __func__, MAC_ARG(get_sa(ptr)), *paction);
+			ret = _FAIL;
+			goto exit;
+		}
+	}
+
+	switch (*paction) {
+	case TDLS_SETUP_REQUEST:
+		ret = On_TDLS_Setup_Req(adapter, precv_frame, ptdls_sta);
+		break;
+	case TDLS_SETUP_RESPONSE:
+		ret = On_TDLS_Setup_Rsp(adapter, precv_frame, ptdls_sta);
+		break;
+	case TDLS_SETUP_CONFIRM:
+		ret = On_TDLS_Setup_Cfm(adapter, precv_frame, ptdls_sta);
+		break;
+	case TDLS_TEARDOWN:
+		ret = On_TDLS_Teardown(adapter, precv_frame, ptdls_sta);
+		break;
+	case TDLS_DISCOVERY_REQUEST:
+		ret = On_TDLS_Dis_Req(adapter, precv_frame);
+		break;
+	case TDLS_PEER_TRAFFIC_INDICATION:
+		ret = On_TDLS_Peer_Traffic_Indication(adapter, precv_frame, ptdls_sta);
+		break;
+	case TDLS_PEER_TRAFFIC_RESPONSE:
+		ret = On_TDLS_Peer_Traffic_Rsp(adapter, precv_frame, ptdls_sta);
+		break;
+#ifdef CONFIG_TDLS_CH_SW
+	case TDLS_CHANNEL_SWITCH_REQUEST:
+		ret = On_TDLS_Ch_Switch_Req(adapter, precv_frame, ptdls_sta);
+		break;
+	case TDLS_CHANNEL_SWITCH_RESPONSE:
+		ret = On_TDLS_Ch_Switch_Rsp(adapter, precv_frame, ptdls_sta);
+		break;
+#endif
+#ifdef CONFIG_WFD
+	/* First byte of WFA OUI */
+	case 0x50:
+		if (_rtw_memcmp(WFA_OUI, paction, 3)) {
+			/* Probe request frame */
+			if (*(paction + 3) == 0x04) {
+				/* WFDTDLS: for sigma test, do not setup direct link automatically */
+				ptdlsinfo->dev_discovered = _TRUE;
+				RTW_INFO("recv tunneled probe request frame\n");
+				issue_tunneled_probe_rsp(adapter, precv_frame);
+			}
+			/* Probe response frame */
+			if (*(paction + 3) == 0x05) {
+				/* WFDTDLS: for sigma test, do not setup direct link automatically */
+				ptdlsinfo->dev_discovered = _TRUE;
+				RTW_INFO("recv tunneled probe response frame\n");
+			}
+		}
+		break;
+#endif /* CONFIG_WFD */
+	default:
+		RTW_INFO("receive TDLS frame %d but not support\n", *paction);
+		ret = _FAIL;
+		break;
+	}
+
+exit:
+	return ret;
+
+}
+
+sint rtw_tdls_rx_data_validate_hdr(
+	_adapter *adapter,
+	union recv_frame *precv_frame,
+	struct sta_info **psta
+)
+{
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	sint ret = _SUCCESS;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	struct _ADAPTER_LINK *adapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_mlme_priv *pmlmepriv = &adapter_link->mlmepriv;
+	u8 *mybssid  = get_link_bssid(pmlmepriv);
+	u8 *myhwaddr = adapter_link->mac_addr;
+	u8 *sta_addr = pattrib->ta;
+	sint bmcast = IS_MCAST(pattrib->dst);
+	struct tdls_info *ptdlsinfo = &adapter->tdlsinfo;
+#ifdef CONFIG_TDLS_CH_SW
+	struct tdls_ch_switch *pchsw_info = &ptdlsinfo->chsw_info;
+#endif
+	struct sta_info *ptdls_sta = NULL;
+	u8 *psnap_type = ptr + pattrib->hdrlen + pattrib->iv_len + SNAP_SIZE;
+	/* frame body located after [+2]: ether-type, [+1]: payload type */
+	u8 *pframe_body = psnap_type + 2 + 1;
+	*psta = ptdls_sta = rtw_get_stainfo(pstapriv, pattrib->ta);
+	if (ptdls_sta == NULL) {
+		ret = _FAIL;
+		goto exit;
+	} else if (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE) {
+		/* filter packets that SA is myself or multicast or broadcast */
+		if (_rtw_memcmp(myhwaddr, pattrib->src, ETH_ALEN)) {
+			ret = _FAIL;
+			goto exit;
+		}
+		/* da should be for me */
+		if ((!_rtw_memcmp(myhwaddr, pattrib->dst, ETH_ALEN)) && (!bmcast)) {
+			ret = _FAIL;
+			goto exit;
+		}
+		/* check BSSID */
+		if (_rtw_memcmp(pattrib->bssid, "\x0\x0\x0\x0\x0\x0", ETH_ALEN) ||
+		    _rtw_memcmp(mybssid, "\x0\x0\x0\x0\x0\x0", ETH_ALEN) ||
+		    (!_rtw_memcmp(pattrib->bssid, mybssid, ETH_ALEN))) {
+			ret = _FAIL;
+			goto exit;
+		}
+
+#ifdef CONFIG_TDLS_CH_SW
+		if (ATOMIC_READ(&pchsw_info->chsw_on) == _TRUE) {
+			if (adapter_link->mlmeextpriv.chandef.chan != rtw_get_oper_ch(adapter, adapter_link)) {
+				pchsw_info->ch_sw_state |= TDLS_PEER_AT_OFF_STATE;
+				if (!(pchsw_info->ch_sw_state & TDLS_CH_SW_INITIATOR_STATE))
+					_cancel_timer_ex(&ptdls_sta->ch_sw_timer);
+				/* On_TDLS_Peer_Traffic_Rsp(adapter, precv_frame); */
+			}
+		}
+#endif
+
+		/* process UAPSD tdls sta */
+		process_pwrbit_data(adapter, precv_frame, ptdls_sta);
+
+		/* if NULL-frame, check pwrbit */
+		if ((get_frame_sub_type(ptr) & WIFI_DATA_NULL) == WIFI_DATA_NULL) {
+			/* NULL-frame with pwrbit=1, buffer_STA should buffer frames for sleep_STA */
+			if (GetPwrMgt(ptr)) {
+				/* it would be triggered when we are off channel and receiving NULL DATA */
+				/* we can confirm that peer STA is at off channel */
+				RTW_INFO("TDLS: recv peer null frame with pwr bit 1\n");
+				/* ptdls_sta->tdls_sta_state|=TDLS_PEER_SLEEP_STATE; */
+			}
+			/* TODO: Updated BSSID's seq. */
+			/* RTW_INFO("drop Null Data\n"); */
+			ptdls_sta->tdls_sta_state &= ~(TDLS_WAIT_PTR_STATE);
+			ret = _FAIL;
+			goto exit;
+		}
+
+		#if 0
+		/* receive some of all TDLS management frames, process it at ON_TDLS */
+		if (_rtw_memcmp(psnap_type, SNAP_ETH_TYPE_TDLS, 2)) {
+			ret = OnTDLS(adapter, precv_frame);
+			goto exit;
+		}
+		#endif
+
+		if ((get_frame_sub_type(ptr) & WIFI_QOS_DATA_TYPE) == WIFI_QOS_DATA_TYPE)
+			process_wmmps_data(adapter, precv_frame, ptdls_sta);
+
+		ptdls_sta->tdls_sta_state &= ~(TDLS_WAIT_PTR_STATE);
+
+	}
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_TDLS */
+
+#ifdef PRIVATE_R
+#define RTW_RATE_SS_MASK (BIT(4) | BIT(5) | BIT(6))
+#define RTW_RATE_SS_SHIFT 4
+#define RTW_GET_SS(_rtw_data_rate) (u8) ((_rtw_data_rate & RTW_RATE_SS_MASK) >> RTW_RATE_SS_SHIFT)
+
+u8 rtw_get_idx_by_rx_rate(u16 data_rate)
+{
+	u8 idx = 0;
+	u8 ss = 0;
+
+ 	switch (RTW_GET_RATE_MODE(data_rate)) {
+	case RTW_LEGACY_MODE: /* 0x0 ~ 0xB */
+		idx = data_rate & 0Xf;
+		break;
+	case RTW_HT_MODE: /* 0xC ~ 0x2B */
+		idx = (data_rate & 0x1f) + 0xc;
+		break;
+	case RTW_VHT_MODE: /* 0x2C ~ 0x53 (MCS0 ~ MCS9 => 10) */
+		ss = RTW_GET_SS(data_rate);
+		idx = (data_rate & 0xf) + (ss * 10) + 0x2c;
+		break;
+	case RTW_HE_MODE: /* 0x54 ~ 0x83 (MCS0 ~ MCS11 => 12) */
+		ss = RTW_GET_SS(data_rate);
+		idx = (data_rate & 0xf) + (ss * 12) + 0x54;
+	}
+
+	return idx;
+}
+#endif /* PRIVATE_R */
+
+void count_rx_stats(_adapter *padapter, union recv_frame *prframe, struct sta_info *sta)
+{
+	int	sz;
+	struct sta_info		*psta = NULL;
+	struct stainfo_stats	*pstats = NULL;
+	struct rx_pkt_attrib	*pattrib = &prframe->u.hdr.attrib;
+	struct recv_info *precvinfo = &padapter->recvinfo;
+#ifdef PRIVATE_R
+	struct sta_priv *pstapriv_primary = &(GET_PRIMARY_ADAPTER(padapter))->stapriv;
+#endif
+
+#ifdef PRIVATE_R
+	u8 idx = 0;
+
+	idx = rtw_get_idx_by_rx_rate(pattrib->data_rate);
+
+	if ((pattrib->priority == 6 || pattrib->priority == 7) && (idx < 132)) {
+		precvinfo->rx_vo_pkt_count_per_data_rate[idx]++;
+	}
+#endif
+
+	sz = get_recvframe_len(prframe);
+	precvinfo->rx_bytes += sz;
+
+	padapter->mlmepriv.LinkDetectInfo.NumRxOkInPeriod++;
+
+	if ((!MacAddr_isBcst(pattrib->dst)) && (!IS_MCAST(pattrib->dst)))
+		padapter->mlmepriv.LinkDetectInfo.NumRxUnicastOkInPeriod++;
+
+	if (sta)
+		psta = sta;
+	else
+		psta = prframe->u.hdr.psta;
+
+	if (psta) {
+		u8 is_ra_bmc = IS_MCAST(pattrib->ra);
+
+		pstats = &psta->sta_stats;
+
+		pstats->last_rx_time = rtw_get_current_time();
+		pstats->rx_data_pkts++;
+		pstats->rx_bytes += sz;
+		if (is_broadcast_mac_addr(pattrib->ra)) {
+			pstats->rx_data_bc_pkts++;
+			pstats->rx_bc_bytes += sz;
+		} else if (is_ra_bmc) {
+			pstats->rx_data_mc_pkts++;
+			pstats->rx_mc_bytes += sz;
+		}
+
+		if (!is_ra_bmc) {
+			/*pstats->rxratecnt[pattrib->data_rate]++;*/ /* FIXME overflow */
+			/*record rx packets for every tid*/
+			pstats->rx_data_qos_pkts[pattrib->priority]++;
+		}
+#ifdef ROKU_PRIVATE
+		/* Record unicast data packets infomation */
+		if (!is_broadcast_mac_addr(pattrib->ra) && !is_ra_bmc) {
+			u16 rate_val;
+			rate_val = rtw_desc_rate_to_bitrate(psta->phl_sta->chandef.bw, psta->init_rate,
+					psta->phl_sta->chandef.bw ? psta->htpriv.sgi_40m : psta->htpriv.sgi_20m);
+			_rtw_spinlock_bh(&pstapriv_primary->last_rx_uc_data_lock);
+			precvinfo->last_rx_uc_data++;
+			precvinfo->last_rx_uc_data_rate_avg += rate_val;
+			_rtw_spinunlock_bh(&pstapriv_primary->last_rx_uc_data_lock);
+		}
+#endif
+	}
+}
+
+sint sta2sta_data_frame(
+	_adapter *adapter,
+	union recv_frame *precv_frame,
+	struct sta_info **psta
+)
+{
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	sint ret = _SUCCESS;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct	sta_priv		*pstapriv = &adapter->stapriv;
+	struct	mlme_priv	*pmlmepriv = &adapter->mlmepriv;
+	u8 *mybssid  = get_bssid(pmlmepriv);
+	u8 *myhwaddr = adapter_mac_addr(adapter);
+	u8 *sta_addr = pattrib->ta;
+	sint bmcast = IS_MCAST(pattrib->dst);
+
+	/* RTW_INFO("[%s] %d, seqnum:%d\n", __FUNCTION__, __LINE__, pattrib->seq_num); */
+
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) ||
+	    (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+
+		/* filter packets that SA is myself or multicast or broadcast */
+		if (rtw_match_self_addr(adapter, pattrib->src))
+		{
+			ret = _FAIL;
+			goto exit;
+		}
+
+		if ((!rtw_match_self_addr(adapter, pattrib->dst)) && (!bmcast))
+		{
+			ret = _FAIL;
+			goto exit;
+		}
+
+		if (_rtw_memcmp(pattrib->bssid, "\x0\x0\x0\x0\x0\x0", ETH_ALEN) ||
+			rtw_match_bssid(adapter, "\x0\x0\x0\x0\x0\x0") ||
+			(!rtw_match_bssid(adapter, pattrib->bssid)))
+		{
+			ret = _FAIL;
+			goto exit;
+		}
+
+	} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE) == _TRUE) {
+		_rtw_memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->src, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
+		_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+		sta_addr = mybssid;
+	} else
+		ret  = _FAIL;
+
+	*psta = precv_frame->u.hdr.psta;
+	if (*psta == NULL) {
+#ifdef CONFIG_MP_INCLUDED
+		if (adapter->registrypriv.mp_mode == 1) {
+			if (check_fwstate(pmlmepriv, WIFI_MP_STATE) == _TRUE)
+				adapter->mppriv.rx_pktloss++;
+		}
+#endif
+		ret = _FAIL;
+		goto exit;
+	}
+
+exit:
+	return ret;
+
+}
+
+sint ap2sta_data_frame(
+	_adapter *adapter,
+	union recv_frame *precv_frame,
+	struct sta_info **psta)
+{
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	sint ret = _SUCCESS;
+	struct	sta_priv		*pstapriv = &adapter->stapriv;
+	struct	mlme_priv	*pmlmepriv = &adapter->mlmepriv;
+	u8 *myhwaddr = adapter_mac_addr(adapter);
+	sint bmcast = IS_MCAST(pattrib->dst);
+
+	if ((check_fwstate(pmlmepriv, WIFI_MP_STATE) == _TRUE) &&
+		   (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)) {
+		_rtw_memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->src, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
+		_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+
+		*psta = precv_frame->u.hdr.psta; /* get sta_info */
+		if (*psta == NULL) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" can't get psta under WIFI_MP_STATE ; drop pkt\n"
+				, FUNC_ADPT_ARG(adapter));
+			#endif
+			ret = _FAIL;
+			goto exit;
+		}
+
+	} else {
+		if (rtw_match_self_addr(adapter, pattrib->dst) && (!bmcast))
+		{
+			*psta = precv_frame->u.hdr.psta;
+			if (*psta == NULL) {
+
+				/* for AP multicast issue , modify by yiwei */
+				static systime send_issue_deauth_time = 0;
+
+				/* RTW_INFO("After send deauth , %u ms has elapsed.\n", rtw_get_passing_time_ms(send_issue_deauth_time)); */
+
+				if (rtw_get_passing_time_ms(send_issue_deauth_time) > 10000 || send_issue_deauth_time == 0) {
+					send_issue_deauth_time = rtw_get_current_time();
+
+					RTW_INFO("issue_deauth to the ap=" MAC_FMT " for the reason(7)\n", MAC_ARG(pattrib->bssid));
+
+					issue_deauth(adapter, pattrib->bssid, WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
+				}
+			}
+		}
+
+		ret = _FAIL;
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" fw_state:0x%x\n"
+			, FUNC_ADPT_ARG(adapter), get_fwstate(pmlmepriv));
+		#endif
+	}
+
+exit:
+	return ret;
+
+}
+
+sint sta2ap_data_frame(
+	_adapter *adapter,
+	union recv_frame *precv_frame,
+	struct sta_info **psta)
+{
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct	sta_priv		*pstapriv = &adapter->stapriv;
+	struct	mlme_priv	*pmlmepriv = &adapter->mlmepriv;
+	sint ret = _SUCCESS;
+
+	if ((check_fwstate(pmlmepriv, WIFI_MP_STATE) == _TRUE) &&
+		   (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)) {
+		/* RTW_INFO("%s ,in WIFI_MP_STATE\n",__func__); */
+		_rtw_memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->src, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
+		_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+
+		*psta = precv_frame->u.hdr.psta; /* get sta_info */
+		if (*psta == NULL) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" can't get psta under WIFI_MP_STATE ; drop pkt\n"
+				, FUNC_ADPT_ARG(adapter));
+			#endif
+			ret = _FAIL;
+			goto exit;
+		}
+
+	} else {
+		u8 *myhwaddr = adapter_mac_addr(adapter);
+		if (!rtw_match_self_addr(adapter, pattrib->ra))
+		{
+			ret = RTW_RX_HANDLED;
+			goto exit;
+		}
+#ifndef CONFIG_CUSTOMER_ALIBABA_GENERAL
+		RTW_INFO("issue_deauth to sta=" MAC_FMT " for the reason(7)\n", MAC_ARG(pattrib->src));
+		issue_deauth(adapter, pattrib->src, WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
+#endif
+		ret = RTW_RX_HANDLED;
+		goto exit;
+	}
+
+exit:
+
+
+	return ret;
+
+}
+
+int rtw_sta_rx_data_validate_hdr(_adapter *adapter, union recv_frame *rframe, struct sta_info **sta)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	u8 *myhwaddr = adapter_mac_addr(adapter);
+	struct rx_pkt_attrib *rattrib = &rframe->u.hdr.attrib;
+	u8 *whdr = get_recvframe_data(rframe);
+	u8 is_ra_bmc = IS_MCAST(GetAddr1Ptr(whdr)) ? 1 : 0;
+	sint ret = _FAIL;
+
+	if (rattrib->to_fr_ds == 0) {
+		_rtw_memcpy(rattrib->ra, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->ta, get_addr2_ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->dst, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->src, get_addr2_ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->bssid, GetAddr3Ptr(whdr), ETH_ALEN);
+
+		#ifdef CONFIG_TDLS
+		if (adapter->tdlsinfo.link_established == _TRUE)
+			ret = rtw_tdls_rx_data_validate_hdr(adapter, rframe, sta);
+		else
+		#endif
+		{
+			/* For Station mode, sa and bssid should always be BSSID, and DA is my mac-address */
+			if (!_rtw_memcmp(rattrib->bssid, rattrib->src, ETH_ALEN))
+				goto exit;
+
+			if (rattrib->addr_cam_vld)
+				*sta = rtw_get_stainfo_by_macid(stapriv, rattrib->macid);
+			else
+				*sta = rtw_get_stainfo(stapriv, get_addr2_ptr(whdr));
+			if (*sta)
+				ret = _SUCCESS;
+		}
+		goto exit;
+	}
+
+	if (!(MLME_STATE(adapter) & (WIFI_ASOC_STATE | WIFI_UNDER_LINKING))) {
+		if (!is_ra_bmc) {
+			/* for AP multicast issue , modify by yiwei */
+			static systime send_issue_deauth_time = 0;
+
+			/* RTW_INFO("After send deauth , %u ms has elapsed.\n", rtw_get_passing_time_ms(send_issue_deauth_time)); */
+			if (rtw_get_passing_time_ms(send_issue_deauth_time) > 10000 || send_issue_deauth_time == 0) {
+				send_issue_deauth_time = rtw_get_current_time();
+				RTW_INFO(FUNC_ADPT_FMT" issue_deauth to "MAC_FMT" with reason(7), mlme_state:0x%x\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(get_addr2_ptr(whdr)), MLME_STATE(adapter));
+				issue_deauth(adapter, get_addr2_ptr(whdr), WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
+			}
+		}
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" fw_state:0x%x\n"
+			, FUNC_ADPT_ARG(adapter), MLME_STATE(adapter));
+		#endif
+		goto exit;
+	}
+
+	_rtw_memcpy(rattrib->ra, GetAddr1Ptr(whdr), ETH_ALEN);
+	_rtw_memcpy(rattrib->ta, get_addr2_ptr(whdr), ETH_ALEN);
+
+	switch (rattrib->to_fr_ds) {
+	case 1:
+		_rtw_memcpy(rattrib->dst, GetAddr1Ptr(whdr), ETH_ALEN);
+		_rtw_memcpy(rattrib->src, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->bssid, get_addr2_ptr(whdr), ETH_ALEN);
+		break;
+	case 3:
+		_rtw_memcpy(rattrib->dst, GetAddr3Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->src, GetAddr4Ptr(whdr), ETH_ALEN); /* may change after checking AMSDU subframe header */
+		_rtw_memcpy(rattrib->bssid, get_addr2_ptr(whdr), ETH_ALEN);
+		break;
+	default:
+		ret = RTW_RX_HANDLED; /* don't count for drop */
+		goto exit;
+	}
+
+	/* filter packets that SA is myself */
+	if (!rattrib->amsdu && rtw_match_self_addr(adapter, rattrib->src))
+	{
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" SA="MAC_FMT", myhwaddr="MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(rattrib->src), MAC_ARG(myhwaddr));
+		#endif
+		goto exit;
+	}
+
+	if (rattrib->addr_cam_vld)
+		*sta = rtw_get_stainfo_by_macid(stapriv, rattrib->macid);
+	else
+		*sta = rtw_get_stainfo(stapriv, rattrib->ta);
+	if (*sta == NULL) {
+		#ifndef CONFIG_CUSTOMER_ALIBABA_GENERAL
+		if (!is_ra_bmc && !IS_RADAR_DETECTED(adapter_to_rfctl(adapter))) {
+			RTW_INFO(FUNC_ADPT_FMT" issue_deauth to "MAC_FMT" with reason(7), unknown TA\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(rattrib->ta));
+			issue_deauth(adapter, rattrib->ta, WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
+		}
+		#endif
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" can't get psta under STATION_MODE ; drop pkt\n"
+			, FUNC_ADPT_ARG(adapter));
+		#endif
+		goto exit;
+	}
+
+#ifdef CONFIG_RTW_WDS_AUTO_EN
+	if (rattrib->to_fr_ds == 3 && !(sta->flags & WLAN_STA_WDS))
+		sta->flags |= WLAN_STA_WDS;
+#endif
+
+	/*if ((get_frame_sub_type(whdr) & WIFI_QOS_DATA_TYPE) == WIFI_QOS_DATA_TYPE) {
+	}
+	*/
+
+	if (get_frame_sub_type(whdr) & BIT(6)) {
+		/* No data, will not indicate to upper layer, temporily count it here */
+		count_rx_stats(adapter, rframe, *sta);
+		ret = RTW_RX_HANDLED;
+		goto exit;
+	}
+
+#ifdef CONFIG_RTW_WDS
+	if (adapter_use_wds(adapter)
+		&& !rattrib->amsdu && IS_MCAST(rattrib->dst)
+		&& rtw_rx_wds_gptr_check(adapter, rattrib->src)
+	) {
+		/* will not indicate to upper layer, temporily count it here */
+		count_rx_stats(adapter, rframe, *sta);
+		ret = RTW_RX_HANDLED;
+		goto exit;
+	}
+#endif
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+int rtw_sta_rx_amsdu_act_check(union recv_frame *rframe
+	, const u8 *da, const u8 *sa)
+{
+	int act = RTW_RX_MSDU_ACT_INDICATE;
+
+#ifdef CONFIG_RTW_WDS
+	_adapter *adapter = rframe->u.hdr.adapter;
+
+	if (adapter_use_wds(adapter)
+		&& IS_MCAST(da)
+		&& rtw_rx_wds_gptr_check(adapter, sa)
+	) {
+		act = 0;
+	}
+#endif
+
+	return act;
+}
+
+#ifdef CONFIG_AP_MODE
+sint rtw_proccess_pspoll(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *psta)
+{
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	u16 aid;
+	u8 wmmps_ac = 0;
+
+	aid = GetAid(pframe);
+	if (psta->phl_sta->aid != aid)
+		return _FAIL;
+
+	switch (pattrib->priority) {
+	case 1:
+	case 2:
+		wmmps_ac = psta->uapsd_bk & BIT(0);
+		break;
+	case 4:
+	case 5:
+		wmmps_ac = psta->uapsd_vi & BIT(0);
+		break;
+	case 6:
+	case 7:
+		wmmps_ac = psta->uapsd_vo & BIT(0);
+		break;
+	case 0:
+	case 3:
+	default:
+		wmmps_ac = psta->uapsd_be & BIT(0);
+		break;
+	}
+
+	if (wmmps_ac)
+		return _FAIL;
+
+	if (psta->state & WIFI_STA_ALIVE_CHK_STATE) {
+		RTW_INFO("%s alive check-rx ps-poll\n", __func__);
+		psta->expire_to = pstapriv->expire_to;
+		psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
+	}
+
+	if ((psta->state & WIFI_SLEEP_STATE) && (rtw_tim_map_is_set(adapter, pstapriv->sta_dz_bitmap, psta->phl_sta->aid))) {
+		_list	*xmitframe_plist, *xmitframe_phead;
+		struct xmit_frame *pxmitframe = NULL;
+		struct xmit_priv *pxmitpriv = &adapter->xmitpriv;
+		/* _rtw_spinlock_bh(&psta->sleep_q.lock); */
+		_rtw_spinlock_bh(&pxmitpriv->lock);
+
+		xmitframe_phead = get_list_head(&psta->sleep_q);
+		xmitframe_plist = get_next(xmitframe_phead);
+
+		if ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE) {
+			pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+			xmitframe_plist = get_next(xmitframe_plist);
+			rtw_list_delete(&pxmitframe->list);
+
+			psta->sleepq_len--;
+
+			if (psta->sleepq_len > 0)
+				pxmitframe->attrib.mdata = 1;
+			else
+				pxmitframe->attrib.mdata = 0;
+
+			pxmitframe->attrib.triggered = 1;
+
+			/* RTW_INFO("handling ps-poll, q_len=%d\n", psta->sleepq_len); */
+			/* RTW_INFO_DUMP("handling, tim=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len); */
+#if 0
+			_rtw_spinunlock_bh(&psta->sleep_q.lock);
+			if (rtw_hal_xmit(padapter, pxmitframe) == _TRUE)
+				rtw_os_xmit_complete(padapter, pxmitframe);
+			_rtw_spinlock_bh(&psta->sleep_q.lock);
+#endif
+			rtw_intf_xmitframe_enqueue(adapter, pxmitframe);
+
+			if (psta->sleepq_len == 0) {
+				rtw_tim_map_clear(adapter, pstapriv->tim_bitmap, psta->phl_sta->aid);
+				/* RTW_INFO("after handling ps-poll\n"); */
+				/* RTW_INFO_DUMP("after handling, tim=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len); */
+
+				/* upate BCN for TIM IE */
+				/* update_BCNTIM(padapter);		 */
+				rtw_update_beacon(adapter, psta->padapter_link,
+						_TIM_IE_, NULL, _TRUE, 0);
+			}
+			/* _rtw_spinunlock_bh(&psta->sleep_q.lock); */
+			_rtw_spinunlock_bh(&pxmitpriv->lock);
+		} else {
+			/* _rtw_spinunlock_bh(&psta->sleep_q.lock); */
+			_rtw_spinunlock_bh(&pxmitpriv->lock);
+			/* RTW_INFO("no buffered packets to xmit\n"); */
+			if (rtw_tim_map_is_set(adapter, pstapriv->tim_bitmap, psta->phl_sta->aid)) {
+				if (psta->sleepq_len == 0) {
+					RTW_INFO("no buffered packets to xmit\n");
+					/* issue nulldata with More data bit = 0 to indicate we have no buffered packets */
+					issue_nulldata(adapter, psta->padapter_link,
+							psta->phl_sta->mac_addr, 0, 0, 0);
+				} else {
+					RTW_INFO("error!psta->sleepq_len=%d\n", psta->sleepq_len);
+					psta->sleepq_len = 0;
+				}
+				rtw_tim_map_clear(adapter, pstapriv->tim_bitmap, psta->phl_sta->aid);
+
+				/* upate BCN for TIM IE */
+				/* update_BCNTIM(padapter); */
+				rtw_update_beacon(adapter, psta->padapter_link,
+						_TIM_IE_, NULL, _TRUE, 0);
+			}
+		}
+	}
+	return _SUCCESS;
+}
+#endif /*CONFIG_AP_MODE*/
+sint validate_recv_ctrl_frame(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	struct sta_info *psta = NULL;
+	u8 subtype = 0;
+	bool ra_is_self = _FALSE;
+	bool ra_is_bc = _FALSE;
+
+	/* uint len = precv_frame->u.hdr.len; */
+
+	/* RTW_INFO("+validate_recv_ctrl_frame\n"); */
+
+	if (GetFrameType(pframe) != WIFI_CTRL_TYPE)
+		return _FAIL;
+
+	subtype = get_frame_sub_type(pframe);
+	/* receive the frames that ra(a1) is my address */
+	ra_is_self = rtw_match_self_addr(padapter, GetAddr1Ptr(pframe));
+	ra_is_bc = is_broadcast_mac_addr(GetAddr1Ptr(pframe));
+
+	/*only keep to self or BC(trigger frame)*/
+	if(ra_is_self == _FALSE) {
+		if(ra_is_bc == _TRUE) {
+			if(subtype != WIFI_TRIGGER)
+				return _FAIL;
+		} else {
+			return _FAIL;
+		}
+	}
+
+	if (pattrib->addr_cam_vld)
+		psta = rtw_get_stainfo_by_macid(pstapriv, pattrib->macid);
+	else
+		psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+	if (psta == NULL)
+		return _FAIL;
+
+	precv_frame->u.hdr.psta = psta;
+
+	/* for rx pkt statistics */
+	psta->sta_stats.last_rx_time = rtw_get_current_time();
+	psta->sta_stats.rx_ctrl_pkts++;
+
+	switch (subtype) {
+	#ifdef CONFIG_AP_MODE
+	case WIFI_PSPOLL :
+		{
+			sint rst;
+			/*only ra(a1) is my address*/
+			rst = rtw_proccess_pspoll(padapter, precv_frame, psta);
+			/*RTW_INFO(FUNC_ADPT_FMT" pspoll handle %d\n", FUNC_ADPT_ARG(padapter), rst);*/
+		}
+		break;
+	#endif
+	case WIFI_BAR :
+		/*only ra(a1) is my address*/
+		rtw_process_bar_frame(padapter, precv_frame);
+		break;
+	case WIFI_TRIGGER :
+		#ifdef CONFIG_80211AX_HE
+		rtw_process_he_triggerframe(padapter, precv_frame);
+		#endif
+		break;
+	default :
+		break;
+	}
+	return _FAIL;
+
+}
+
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+static sint validate_mgmt_protect(_adapter *adapter, union recv_frame *precv_frame)
+{
+#define DBG_VALIDATE_MGMT_PROTECT 0
+#define DBG_VALIDATE_MGMT_DEC 0
+
+	struct security_priv *sec = &adapter->securitypriv;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_info	*psta = precv_frame->u.hdr.psta;
+	u8 *ptr;
+	u8 type;
+	u8 subtype;
+	u8 is_bmc;
+	u8 category = 0xFF;
+	struct _ADAPTER_LINK *adapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_security_priv *lsec = &adapter_link->securitypriv;
+
+#ifdef CONFIG_IEEE80211W
+	const u8 *igtk;
+	u16 igtk_id;
+	u64* ipn;
+	enum security_type bip_cipher;
+#endif
+
+	u8 *mgmt_DATA;
+	u32 data_len = 0;
+
+	sint ret;
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		if (!adapter->mesh_info.mesh_auth_id)
+			return pattrib->privacy ? _FAIL : _SUCCESS;
+	} else
+#endif
+	if (SEC_IS_BIP_KEY_INSTALLED(lsec) == _FALSE)
+		return _SUCCESS;
+
+	ptr = precv_frame->u.hdr.rx_data;
+	type = GetFrameType(ptr);
+	subtype = get_frame_sub_type(ptr); /* bit(7)~bit(2) */
+	is_bmc = IS_MCAST(GetAddr1Ptr(ptr));
+
+#if DBG_VALIDATE_MGMT_PROTECT
+	if (subtype == WIFI_DEAUTH) {
+		RTW_INFO(FUNC_ADPT_FMT" bmc:%u, deauth, privacy:%u, encrypt:%u, bdecrypted:%u\n"
+			, FUNC_ADPT_ARG(adapter)
+			, is_bmc, pattrib->privacy, pattrib->encrypt, pattrib->bdecrypted);
+	} else if (subtype == WIFI_DISASSOC) {
+		RTW_INFO(FUNC_ADPT_FMT" bmc:%u, disassoc, privacy:%u, encrypt:%u, bdecrypted:%u\n"
+			, FUNC_ADPT_ARG(adapter)
+			, is_bmc, pattrib->privacy, pattrib->encrypt, pattrib->bdecrypted);
+	} if (subtype == WIFI_ACTION) {
+		if (pattrib->privacy) {
+			RTW_INFO(FUNC_ADPT_FMT" bmc:%u, action(?), privacy:%u, encrypt:%u, bdecrypted:%u\n"
+				, FUNC_ADPT_ARG(adapter)
+				, is_bmc, pattrib->privacy, pattrib->encrypt, pattrib->bdecrypted);
+		} else {
+			RTW_INFO(FUNC_ADPT_FMT" bmc:%u, action(%u), privacy:%u, encrypt:%u, bdecrypted:%u\n"
+				, FUNC_ADPT_ARG(adapter), is_bmc
+				, *(ptr + sizeof(struct rtw_ieee80211_hdr_3addr))
+				, pattrib->privacy, pattrib->encrypt, pattrib->bdecrypted);
+		}
+	}
+#endif
+
+	if (!pattrib->privacy) {
+		if (!psta || !(psta->flags & WLAN_STA_MFP)) {
+			/* peer is not MFP capable, no need to check */
+			goto exit;
+		}
+
+		if (subtype == WIFI_ACTION)
+			category = *(ptr + sizeof(struct rtw_ieee80211_hdr_3addr));
+
+		if (is_bmc) {
+			/* broadcast cases */
+			if (subtype == WIFI_ACTION) {
+				if (CATEGORY_IS_GROUP_PRIVACY(category)) {
+					/* drop broadcast group privacy action frame without encryption */
+					#if DBG_VALIDATE_MGMT_PROTECT
+					RTW_INFO(FUNC_ADPT_FMT" broadcast gp action(%u) w/o encrypt\n"
+						, FUNC_ADPT_ARG(adapter), category);
+					#endif
+					goto fail;
+				}
+				if (CATEGORY_IS_ROBUST(category)) {
+					/* broadcast robust action frame need BIP check */
+					goto bip_verify;
+				}
+			}
+			if (subtype == WIFI_DEAUTH || subtype == WIFI_DISASSOC) {
+				/* broadcast deauth or disassoc frame need BIP check */
+				goto bip_verify;
+			}
+			goto exit;
+
+		} else {
+			/* unicast cases */
+			#ifdef CONFIG_IEEE80211W
+			if (subtype == WIFI_DEAUTH || subtype == WIFI_DISASSOC) {
+				if (!MLME_IS_MESH(adapter)
+				#ifdef CONFIG_RTW_WNM
+				&& (rtw_wnm_try_btm_roam_imnt(adapter) > 0)
+				#endif
+				) {
+					unsigned short reason = le16_to_cpu(*(unsigned short *)(ptr + WLAN_HDR_A3_LEN));
+
+					#if DBG_VALIDATE_MGMT_PROTECT
+					RTW_INFO(FUNC_ADPT_FMT" unicast %s, reason=%d w/o encrypt\n"
+						, FUNC_ADPT_ARG(adapter), subtype == WIFI_DEAUTH ? "deauth" : "disassoc", reason);
+					#endif
+					if (reason == 6 || reason == 7) {
+						/* issue sa query request */
+						issue_action_SA_Query(adapter, psta->padapter_link,
+								psta->phl_sta->mac_addr, 0, 0, IEEE80211W_RIGHT_KEY);
+					}
+				}
+				goto fail;
+			}
+			#endif
+
+			if (subtype == WIFI_ACTION && CATEGORY_IS_ROBUST(category)) {
+				if (psta->bpairwise_key_installed == _TRUE) {
+					#if DBG_VALIDATE_MGMT_PROTECT
+					RTW_INFO(FUNC_ADPT_FMT" unicast robust action(%d) w/o encrypt\n"
+						, FUNC_ADPT_ARG(adapter), category);
+					#endif
+					goto fail;
+				}
+			}
+			goto exit;
+		}
+
+bip_verify:
+#ifdef CONFIG_IEEE80211W
+		#ifdef CONFIG_RTW_MESH
+		if (MLME_IS_MESH(adapter)) {
+			if (psta->igtk_bmp) {
+				bip_cipher = psta->dot11wCipher;
+				igtk = psta->igtk.skey;
+				igtk_id = psta->igtk_id;
+				ipn = &psta->igtk_pn.val;
+			} else {
+				/* mesh MFP without IGTK */
+				goto exit;
+			}
+		} else
+		#endif
+		{
+			bip_cipher = sec->dot11wCipher;
+			igtk = lsec->dot11wBIPKey[lsec->dot11wBIPKeyid].skey;
+			igtk_id = lsec->dot11wBIPKeyid;
+			ipn = &lsec->dot11wBIPrxpn.val;
+		}
+
+		/* verify BIP MME IE */
+		ret = rtw_bip_verify(bip_cipher, pattrib->pkt_len,
+			get_recvframe_data(precv_frame),
+			get_recvframe_len(precv_frame),
+			igtk, igtk_id, ipn, (u8 *)precv_frame);
+
+		if (ret == _FAIL) {
+			/* RTW_INFO("802.11w BIP verify fail\n"); */
+			goto fail;
+
+		} else if (ret == RTW_RX_HANDLED) {
+			#if DBG_VALIDATE_MGMT_PROTECT
+			RTW_INFO(FUNC_ADPT_FMT" none protected packet\n", FUNC_ADPT_ARG(adapter));
+			#endif
+			goto fail;
+		}
+#endif /* CONFIG_IEEE80211W */
+		goto exit;
+	}
+
+	if (!psta || !(psta->flags & WLAN_STA_MFP)) {
+		/* not peer or peer is not MFP capable, drop it */
+		goto fail;
+	}
+
+	/* cases to decrypt mgmt frame */
+#if 0
+	pattrib->bdecrypted = 0;
+#endif
+#ifdef CONFIG_RTW_MESH
+	if (is_bmc)
+		pattrib->encrypt = psta->group_privacy;
+	else
+#endif
+	pattrib->encrypt = psta->dot118021XPrivacy;
+	pattrib->hdrlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	/* set iv and icv length */
+	SET_ICE_IV_LEN(pattrib->iv_len, pattrib->icv_len, pattrib->encrypt);
+	_rtw_memcpy(pattrib->ra, GetAddr1Ptr(ptr), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, get_addr2_ptr(ptr), ETH_ALEN);
+
+	/* actual management data frame body */
+	data_len = pattrib->pkt_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
+	mgmt_DATA = rtw_zmalloc(data_len);
+	if (mgmt_DATA == NULL) {
+		RTW_INFO(FUNC_ADPT_FMT" mgmt allocate fail  !!!!!!!!!\n", FUNC_ADPT_ARG(adapter));
+		goto fail;
+	}
+
+#if DBG_VALIDATE_MGMT_DEC
+	/* dump the packet content before decrypt */
+	{
+		int pp;
+
+		printk("pattrib->pktlen = %d =>", pattrib->pkt_len);
+		for (pp = 0; pp < pattrib->pkt_len; pp++)
+		printk(" %02x ", ptr[pp]);
+		printk("\n");
+	}
+#endif
+
+	precv_frame = decryptor(adapter, precv_frame);
+	/* save actual management data frame body */
+	_rtw_memcpy(mgmt_DATA, ptr + pattrib->hdrlen + pattrib->iv_len, data_len);
+	/* overwrite the iv field */
+	_rtw_memcpy(ptr + pattrib->hdrlen, mgmt_DATA, data_len);
+	/* remove the iv and icv length */
+	pattrib->pkt_len = pattrib->pkt_len - pattrib->iv_len - pattrib->icv_len;
+	rtw_mfree(mgmt_DATA, data_len);
+
+#if DBG_VALIDATE_MGMT_DEC
+	/* print packet content after decryption */
+	{
+		int pp;
+
+		printk("after decryption pattrib->pktlen = %d @@=>", pattrib->pkt_len);
+		for (pp = 0; pp < pattrib->pkt_len; pp++)
+		printk(" %02x ", ptr[pp]);
+		printk("\n");
+	}
+#endif
+
+	if (!precv_frame) {
+		#if DBG_VALIDATE_MGMT_PROTECT
+		RTW_INFO(FUNC_ADPT_FMT" mgmt descrypt fail  !!!!!!!!!\n", FUNC_ADPT_ARG(adapter));
+		#endif
+		goto fail;
+	}
+
+exit:
+	return _SUCCESS;
+
+fail:
+	return _FAIL;
+
+}
+#endif /* defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH) */
+
+s32 recvframe_chk_defrag(_adapter *padapter, union recv_frame **pprecv_frame);
+
+sint validate_recv_mgnt_frame(_adapter *padapter, union recv_frame **pprecv_frame)
+{
+	union recv_frame *precv_frame = *pprecv_frame;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_info *psta;
+	if (pattrib->addr_cam_vld)
+		psta = rtw_get_stainfo_by_macid(&padapter->stapriv, pattrib->macid);
+	else
+		psta = rtw_get_stainfo(&padapter->stapriv, get_addr2_ptr(precv_frame->u.hdr.rx_data));
+
+	precv_frame->u.hdr.psta = psta;
+
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+	if (validate_mgmt_protect(padapter, precv_frame) == _FAIL) {
+		DBG_COUNTER(padapter->rx_logs.core_rx_pre_mgmt_err_80211w);
+		goto exit;
+	}
+#endif
+
+	if (recvframe_chk_defrag(padapter, pprecv_frame) != CORE_RX_CONTINUE)
+		return _SUCCESS;
+	precv_frame = *pprecv_frame;
+
+	/* for rx pkt statistics */
+	if (psta) {
+		psta->sta_stats.last_rx_time = rtw_get_current_time();
+		psta->sta_stats.rx_mgnt_pkts++;
+		if (get_frame_sub_type(precv_frame->u.hdr.rx_data) == WIFI_BEACON)
+			psta->sta_stats.rx_beacon_pkts++;
+		else if (get_frame_sub_type(precv_frame->u.hdr.rx_data) == WIFI_PROBEREQ)
+			psta->sta_stats.rx_probereq_pkts++;
+		else if (get_frame_sub_type(precv_frame->u.hdr.rx_data) == WIFI_PROBERSP) {
+			if (rtw_match_self_addr(padapter, GetAddr1Ptr(precv_frame->u.hdr.rx_data)) == _TRUE)
+				psta->sta_stats.rx_probersp_pkts++;
+			else if (is_broadcast_mac_addr(GetAddr1Ptr(precv_frame->u.hdr.rx_data))
+				|| is_multicast_mac_addr(GetAddr1Ptr(precv_frame->u.hdr.rx_data)))
+				psta->sta_stats.rx_probersp_bm_pkts++;
+			else
+				psta->sta_stats.rx_probersp_uo_pkts++;
+		}
+	}
+
+	mgt_dispatcher(padapter, precv_frame);
+
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+exit:
+#endif
+	return _SUCCESS;
+
+}
+
+sint validate_recv_data_frame(_adapter *adapter, union recv_frame *precv_frame)
+{
+	u8 bretry, a4_shift;
+	struct sta_info *psta = NULL;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct security_priv *psecuritypriv = &adapter->securitypriv;
+	struct recv_info *precvinfo = &adapter->recvinfo;
+	sint ret = _SUCCESS;
+#ifdef PRIVATE_R
+	struct stainfo_stats *pstats = NULL;
+#endif
+
+	bretry = GetRetry(ptr);
+	a4_shift = (pattrib->to_fr_ds == 3) ? ETH_ALEN : 0;
+
+	/* some address fields are different when using AMSDU */
+	if (pattrib->qos)
+		pattrib->amsdu = GetAMsdu(ptr + WLAN_HDR_A3_LEN + a4_shift);
+	else
+		pattrib->amsdu = 0;
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		ret = rtw_mesh_rx_data_validate_hdr(adapter, precv_frame, &psta);
+		goto pre_validate_status_chk;
+	} else
+#endif
+#ifdef CONFIG_AP_MODE
+	if (MLME_IS_AP(adapter)) {
+		ret = rtw_ap_rx_data_validate_hdr(adapter, precv_frame, &psta);
+		goto pre_validate_status_chk;
+	} else
+#endif
+	if (MLME_IS_STA(adapter)) {
+		ret = rtw_sta_rx_data_validate_hdr(adapter, precv_frame, &psta);
+		goto pre_validate_status_chk;
+	}
+
+	switch (pattrib->to_fr_ds) {
+	case 0:
+		_rtw_memcpy(pattrib->ra, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->ta, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->src, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
+		ret = sta2sta_data_frame(adapter, precv_frame, &psta);
+		break;
+
+	case 1:
+		_rtw_memcpy(pattrib->ra, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->ta, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->src, GetAddr3Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->bssid, get_addr2_ptr(ptr), ETH_ALEN);
+		ret = ap2sta_data_frame(adapter, precv_frame, &psta);
+		break;
+
+	case 2:
+		_rtw_memcpy(pattrib->ra, GetAddr1Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->ta, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->dst, GetAddr3Ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->src, get_addr2_ptr(ptr), ETH_ALEN);
+		_rtw_memcpy(pattrib->bssid, GetAddr1Ptr(ptr), ETH_ALEN);
+		ret = sta2ap_data_frame(adapter, precv_frame, &psta);
+		break;
+
+	case 3:
+	default:
+		/* WDS is not supported */
+		ret = _FAIL;
+		break;
+	}
+
+pre_validate_status_chk:
+
+#ifdef PRIVATE_R
+	if(psta!=NULL && bretry){
+		pstats = &psta->sta_stats;
+		ATOMIC_ADD((ATOMIC_T *)&pstats->rx_retry_cnt, 1);
+	}
+#endif
+
+	if (ret == _FAIL) {
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" case:%d, res:%d, ra="MAC_FMT", ta="MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), pattrib->to_fr_ds, ret, MAC_ARG(GetAddr1Ptr(ptr)), MAC_ARG(get_addr2_ptr(ptr)));
+		#endif
+		goto exit;
+	} else if (ret == RTW_RX_HANDLED)
+		goto exit;
+
+
+	if (psta == NULL) {
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" psta == NULL, ra="MAC_FMT", ta="MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(GetAddr1Ptr(ptr)), MAC_ARG(get_addr2_ptr(ptr)));
+		#endif
+		ret = _FAIL;
+		goto exit;
+	}
+
+	if ((psta->flags & WLAN_STA_AMSDU_DISABLE) && pattrib->amsdu) {
+	#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" amsdu not allowed"MAC_FMT"\n"
+			, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->cmn.mac_addr));
+	#endif
+		ret = _FAIL;
+		goto exit;
+
+	}
+
+	precv_frame->u.hdr.psta = psta;
+#ifdef CONFIG_RECV_REORDERING_CTRL
+	precv_frame->u.hdr.preorder_ctrl = NULL;
+#endif
+	pattrib->ack_policy = 0;
+
+	/* parsing QC field */
+	if (pattrib->qos == 1) {
+		pattrib->priority = GetPriority((ptr + WLAN_HDR_A3_LEN + a4_shift)); /* point to Qos field*/
+		pattrib->ack_policy = GetAckpolicy((ptr + WLAN_HDR_A3_LEN + a4_shift));
+		pattrib->hdrlen = WLAN_HDR_A3_QOS_LEN + a4_shift;
+	} else {
+		pattrib->priority = 0;
+		pattrib->hdrlen = WLAN_HDR_A3_LEN + a4_shift;
+	}
+
+	if (pattrib->order) /* HT-CTRL 11n */
+		pattrib->hdrlen += 4;
+
+	/* decache, drop duplicate recv packets */
+	ret = recv_decache(precv_frame);
+	if (ret  == _FAIL)
+		goto exit;
+
+#ifdef PRIVATE_R
+	if (bretry && (pattrib->priority == 6 || pattrib->priority == 7))
+		precvinfo->rx_vo_pkt_retry_count++;
+#endif
+	if (!IS_MCAST(pattrib->ra)) {
+#ifdef CONFIG_RECV_REORDERING_CTRL
+		if (pattrib->qos)
+			precv_frame->u.hdr.preorder_ctrl = &psta->recvreorder_ctrl[pattrib->priority];
+#endif
+		if (recv_ucast_pn_decache(precv_frame) == _FAIL) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recv_ucast_pn_decache return _FAIL for sta="MAC_FMT"\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->phl_sta->mac_addr));
+			#endif
+			ret = _FAIL;
+			goto exit;
+		}
+	} else {
+		if (recv_bcast_pn_decache(precv_frame) == _FAIL) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recv_bcast_pn_decache return _FAIL for sta="MAC_FMT"\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->phl_sta->mac_addr));
+			#endif
+			ret = _FAIL;
+			goto exit;
+		}
+	}
+
+	if (pattrib->privacy) {
+#ifdef CONFIG_TDLS
+		if ((psta->tdls_sta_state & TDLS_LINKED_STATE) && (psta->dot118021XPrivacy == _AES_))
+			pattrib->encrypt = psta->dot118021XPrivacy;
+		else
+#endif /* CONFIG_TDLS */
+			GET_ENCRY_ALGO(psecuritypriv, psta, pattrib->encrypt, IS_MCAST(pattrib->ra));
+
+
+		SET_ICE_IV_LEN(pattrib->iv_len, pattrib->icv_len, pattrib->encrypt);
+	} else {
+		pattrib->encrypt = 0;
+		pattrib->iv_len = pattrib->icv_len = 0;
+	}
+
+	/* drop unprotected frame in protected network. */
+	if (psecuritypriv->dot11PrivacyAlgrthm != _NO_PRIVACY_ ) {
+		if (IS_MCAST(pattrib->ra)) {
+			if (!pattrib->privacy) {
+				#ifdef DBG_RX_DROP_FRAME
+				RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT"recv plaintext bmc packet for sta="MAC_FMT"\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->cmn.mac_addr));
+				#endif
+				ret = _FAIL;
+				goto exit;
+			}
+		} else {
+			/* unicast */
+			u16 ether_type;
+			u8* ether_ptr = NULL;
+			u16  eapol_type = 0x888e;
+			ether_ptr = ptr + pattrib->hdrlen + pattrib->iv_len + RATTRIB_GET_MCTRL_LEN(pattrib) + LLC_HEADER_SIZE;
+			_rtw_memcpy(&ether_type, ether_ptr, 2);
+			ether_type = ntohs((unsigned short)ether_type);
+
+			if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) {
+			/* CVE-2020-26140, CVE-2020-26143, CVE-2020-26147,  let eapol packet go through*/
+				if (!pattrib->privacy && ether_type != eapol_type ) {
+					#ifdef DBG_RX_DROP_FRAME
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT"recv plaintext unicast packet for sta="MAC_FMT"\n"
+						, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->cmn.mac_addr));
+					#endif
+					ret = _FAIL;
+					goto exit;
+				}
+				/* CVE-2020-26144, pevernt plaintext A-MSDU */
+				/* This can prevent plantext A-MSDU cloacked as an EAPOL frame */
+				if (!pattrib->privacy && pattrib->amsdu) {
+					#ifdef DBG_RX_DROP_FRAME
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT"recv plaintext A-MSDU for sta="MAC_FMT"\n"
+						, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->cmn.mac_addr));
+					#endif
+					ret = _FAIL;
+					goto exit;
+				}
+				/* CVE-2020-26139,  Drop any forwarding eapol packet until 4-way has done.	*/
+				if ((ether_type == eapol_type)
+					&& (MLME_IS_AP(adapter) || MLME_IS_MESH(adapter))
+					&& (psta->dot118021XPrivacy == _NO_PRIVACY_)
+					&& (!rtw_match_self_addr(adapter, pattrib->dst))
+				) {
+					#ifdef DBG_RX_DROP_FRAME
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recv eapol packet forwarding(dst:"MAC_FMT") before 4-way finish.\n"
+						, FUNC_ADPT_ARG(adapter), MAC_ARG(pattrib->dst));
+					#endif
+					ret = _FAIL;
+					goto exit;
+				}
+			} else {
+				/* CVE-2020-26140, CVE-2020-26143, CVE-2020-26147 */
+				if (!pattrib->privacy) {
+					#ifdef DBG_RX_DROP_FRAME
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT"recv plaintext packet for sta="MAC_FMT"\n"
+						, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->cmn.mac_addr));
+					#endif
+					ret = _FAIL;
+					goto exit;
+				}
+		 	}
+		}
+	}
+
+#ifdef CONFIG_RTW_MESH
+	if (!pattrib->amsdu
+		&& pattrib->mesh_ctrl_present
+		&& (!pattrib->encrypt || pattrib->bdecrypted))
+		ret = rtw_mesh_rx_validate_mctrl_non_amsdu(adapter, precv_frame);
+#endif
+
+#ifdef CONFIG_TDLS
+	if (pattrib->is_tdls_frame) {
+		ret = OnTDLS(adapter, precv_frame);
+		if (ret == _FAIL)
+			goto exit;
+	}
+#endif /* CONFIG_TDLS */
+
+exit:
+
+#ifdef RTW_PHL_TEST_FPGA
+	return _SUCCESS;
+#endif
+	return ret;
+}
+
+static inline void dump_rx_packet(u8 *ptr)
+{
+	int i;
+
+	RTW_INFO("#############################\n");
+	for (i = 0; i < 64; i = i + 8)
+		RTW_INFO("%02X:%02X:%02X:%02X:%02X:%02X:%02X:%02X:\n", *(ptr + i),
+			*(ptr + i + 1), *(ptr + i + 2) , *(ptr + i + 3) , *(ptr + i + 4), *(ptr + i + 5), *(ptr + i + 6), *(ptr + i + 7));
+	RTW_INFO("#############################\n");
+}
+
+#ifdef DBG_RX_DFRAME_RAW_DATA
+static void rx_process_dframe_raw_data(union recv_frame *precvframe)
+{
+	_adapter *padapter = precvframe->u.hdr.adapter;
+	struct dvobj_priv *devob = adapter_to_dvobj(padapter);
+	struct sta_recv_dframe_info *psta_dframe_info;
+	struct sta_info *psta = precvframe->u.hdr.psta;
+	struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct recv_info *recvinfo = &padapter->recvinfo;
+	u8 isCCKrate, rf_path, dframe_type;
+	u8 *ptr;
+	u8 bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	ptr = precvframe->u.hdr.rx_data;
+	dframe_type = GetFrameType(ptr);
+
+	if (recvinfo->store_law_data_flag) {
+		if ((dframe_type == WIFI_DATA_TYPE) || (dframe_type == WIFI_QOS_DATA_TYPE) || (padapter->registrypriv.mp_mode == 1)) {
+			if (psta) {
+				if (IS_MCAST(get_ra(get_recvframe_data(precvframe)))) {
+					psta_dframe_info = &psta->sta_dframe_info_bmc;
+				} else {
+					psta_dframe_info = &psta->sta_dframe_info;
+				}
+
+				if ((_rtw_memcmp(get_my_bssid(&pmlmeinfo->dev_network), bc_addr, ETH_ALEN) != _TRUE) || (padapter->registrypriv.mp_mode == 1)) {
+					psta_dframe_info->sta_data_rate = pattrib->data_rate;
+					psta_dframe_info->sta_sgi = pattrib->sgi;
+					psta_dframe_info->sta_bw_mode = pattrib->bw;
+					isCCKrate = (pattrib->data_rate <= DESC_RATE11M) ? _TRUE : _FALSE;
+					for (rf_path = 0; rf_path < GET_HAL_RFPATH_NUM(devob); rf_path++) {
+						psta_dframe_info->sta_mimo_signal_strength[rf_path] = pattrib->phy_info.signal_strength;
+						if (!isCCKrate) {
+							psta_dframe_info->sta_ofdm_snr[rf_path] = pattrib->phy_info.snr_fd[rf_path];
+							psta_dframe_info->sta_RxPwr[rf_path] = rtw_phl_rssi_to_dbm(pattrib->phy_info.rx_pwr[rf_path]);;
+						}
+					}
+				}
+			}
+		}
+	}
+}
+#endif /* DBG_RX_DFRAME_RAW_DATA */
+
+#ifdef CONFIG_SNR_RPT
+static void rx_process_snr_info(union recv_frame *precvframe)
+{
+	_adapter *padapter = precvframe->u.hdr.adapter;
+	struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;
+	struct phydm_phyinfo_struct *phy_info = &pattrib->phy_info;
+	u8 *wlanhdr = NULL;
+	u8 *ta, *ra;
+	u8 is_ra_bmc;
+	struct sta_priv *pstapriv;
+	struct sta_info *psta = NULL;
+	int i;
+
+	wlanhdr = precvframe->u.hdr.rx_data;
+	ta = get_ta(wlanhdr);
+	pstapriv = &padapter->stapriv;
+	psta = precvframe->u.hdr.psta;
+
+	if (psta) {
+		_rtw_spinlock_bh(&psta->lock);
+		if (phy_info->is_valid) {
+			psta->snr_num++;
+			for ( i = 0; i < RTW_PHL_MAX_RF_PATH; i++) {
+				psta->snr_fd_total[i] += pattrib->phy_info.snr_fd[i];
+				psta->snr_td_total[i] += pattrib->phy_info.snr_td[i];
+				psta->snr_fd_avg[i] = psta->snr_fd_total[i]/psta->snr_num;
+				psta->snr_td_avg[i] = psta->snr_td_total[i]/psta->snr_num;
+				#if 0
+				RTW_INFO("path = %d, AVG_SNR_FD = %d, AVG_SNR_TD = %d\n",
+				i, psta->snr_fd_avg[i], psta->snr_td_avg[i]);
+				#endif
+			}
+
+		}
+		_rtw_spinunlock_bh(&psta->lock);
+	}
+
+}
+#endif /* CONFIG_SNR_RPT */
+sint validate_recv_frame(_adapter *adapter, union recv_frame **pprecv_frame)
+{
+	/* shall check frame subtype, to / from ds, da, bssid */
+
+	/* then call check if rx seq/frag. duplicated. */
+
+	u8 type;
+	u8 subtype;
+	sint retval = _SUCCESS;
+	union recv_frame *precv_frame = *pprecv_frame;
+
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct recv_info *precvinfo = &adapter->recvinfo;
+
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	u8  ver = (unsigned char)(*ptr) & 0x3 ;
+#ifdef CONFIG_FIND_BEST_CHANNEL
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+#endif
+
+#ifdef CONFIG_TDLS
+	struct tdls_info *ptdlsinfo = &adapter->tdlsinfo;
+#endif /* CONFIG_TDLS */
+#ifdef CONFIG_WAPI_SUPPORT
+	PRT_WAPI_T	pWapiInfo = &adapter->wapiInfo;
+	struct recv_frame_hdr *phdr = &precv_frame->u.hdr;
+	u8 wai_pkt = 0;
+	u16 sc;
+	u8 external_len = 0;
+#endif
+	struct _ADAPTER_LINK *adapter_link = precv_frame->u.hdr.adapter_link;
+
+
+#ifdef CONFIG_FIND_BEST_CHANNEL
+	if (pmlmeext->sitesurvey_res.state == SCAN_PROCESS) {
+		int ch_set_idx = rtw_chset_search_ch(chset, rtw_get_oper_ch(adapter, adapter_link));
+		if (ch_set_idx >= 0)
+			chset->chs[ch_set_idx].rx_count++;
+	}
+#endif
+
+#ifdef CONFIG_TDLS
+	if (ptdlsinfo->ch_sensing == 1 && ptdlsinfo->cur_channel != 0)
+		ptdlsinfo->collect_pkt_num[ptdlsinfo->cur_channel - 1]++;
+#endif /* CONFIG_TDLS */
+
+#ifdef RTK_DMP_PLATFORM
+	if (0) {
+		RTW_INFO("++\n");
+		{
+			int i;
+			for (i = 0; i < 64; i = i + 8)
+				RTW_INFO("%02X:%02X:%02X:%02X:%02X:%02X:%02X:%02X:", *(ptr + i),
+					*(ptr + i + 1), *(ptr + i + 2) , *(ptr + i + 3) , *(ptr + i + 4), *(ptr + i + 5), *(ptr + i + 6), *(ptr + i + 7));
+
+		}
+		RTW_INFO("--\n");
+	}
+#endif /* RTK_DMP_PLATFORM */
+
+	/* add version chk */
+	if (ver != 0) {
+		retval = _FAIL;
+		DBG_COUNTER(adapter->rx_logs.core_rx_pre_ver_err);
+		goto exit;
+	}
+
+	type =  GetFrameType(ptr);
+	subtype = get_frame_sub_type(ptr); /* bit(7)~bit(2) */
+
+	pattrib->to_fr_ds = get_tofr_ds(ptr);
+
+	pattrib->frag_num = GetFragNum(ptr);
+	pattrib->seq_num = GetSequence(ptr);
+
+	pattrib->pw_save = GetPwrMgt(ptr);
+	pattrib->mfrag = GetMFrag(ptr);
+	pattrib->mdata = GetMData(ptr);
+	pattrib->privacy = GetPrivacy(ptr);
+	pattrib->order = GetOrder(ptr);
+#ifdef CONFIG_WAPI_SUPPORT
+	sc = (pattrib->seq_num << 4) | pattrib->frag_num;
+#endif
+
+#ifdef RTW_PHL_DBG_CMD
+	pattrib->wl_type = type;
+	pattrib->wl_subtype = subtype;
+
+	if(type == WIFI_DATA_TYPE)
+		core_add_record(adapter, REC_RX_DATA, precv_frame);
+	else
+		core_add_record(adapter, REC_RX_MGMT, precv_frame);
+#endif
+
+
+#if 1 /* Dump rx packets */
+	{
+		u8 bDumpRxPkt = 0;
+
+		rtw_hal_get_def_var(adapter, adapter_link, HAL_DEF_DBG_DUMP_RXPKT, &(bDumpRxPkt));
+		if (bDumpRxPkt == 1) /* dump all rx packets */
+			dump_rx_packet(ptr);
+		else if ((bDumpRxPkt == 2) && (type == WIFI_MGT_TYPE))
+			dump_rx_packet(ptr);
+		else if ((bDumpRxPkt == 3) && (type == WIFI_DATA_TYPE))
+			dump_rx_packet(ptr);
+	}
+#endif
+#ifdef RTW_WKARD_CORE_RSSI_V1
+	rx_process_phy_info(precv_frame);
+#endif
+
+#ifdef CONFIG_SNR_RPT
+	rx_process_snr_info(precv_frame);
+#endif /* CONFIG_SNR_RPT */
+	switch (type) {
+	case WIFI_MGT_TYPE: /* mgnt */
+		DBG_COUNTER(adapter->rx_logs.core_rx_pre_mgmt);
+		retval = validate_recv_mgnt_frame(adapter, pprecv_frame);
+		if (retval == _FAIL) {
+			DBG_COUNTER(adapter->rx_logs.core_rx_pre_mgmt_err);
+		}
+		retval = _FAIL; /* only data frame return _SUCCESS */
+		break;
+	case WIFI_CTRL_TYPE: /* ctrl */
+		DBG_COUNTER(adapter->rx_logs.core_rx_pre_ctrl);
+		retval = validate_recv_ctrl_frame(adapter, precv_frame);
+		if (retval == _FAIL) {
+			DBG_COUNTER(adapter->rx_logs.core_rx_pre_ctrl_err);
+		}
+		retval = _FAIL; /* only data frame return _SUCCESS */
+		break;
+	case WIFI_DATA_TYPE: /* data */
+		DBG_COUNTER(adapter->rx_logs.core_rx_pre_data);
+#ifdef CONFIG_WAPI_SUPPORT
+		if (pattrib->qos)
+			external_len = 2;
+		else
+			external_len = 0;
+
+		wai_pkt = rtw_wapi_is_wai_packet(adapter, ptr);
+
+		phdr->bIsWaiPacket = wai_pkt;
+
+		if (wai_pkt != 0) {
+			if (sc != adapter->wapiInfo.wapiSeqnumAndFragNum)
+				adapter->wapiInfo.wapiSeqnumAndFragNum = sc;
+			else {
+				retval = _FAIL;
+				DBG_COUNTER(adapter->rx_logs.core_rx_pre_data_wapi_seq_err);
+				break;
+			}
+		} else {
+
+			if (rtw_wapi_drop_for_key_absent(adapter, get_addr2_ptr(ptr))) {
+				retval = _FAIL;
+				WAPI_TRACE(WAPI_RX, "drop for key absent for rx\n");
+				DBG_COUNTER(adapter->rx_logs.core_rx_pre_data_wapi_key_err);
+				break;
+			}
+		}
+
+#endif
+
+		pattrib->qos = (subtype & BIT(7)) ? 1 : 0;
+		retval = validate_recv_data_frame(adapter, precv_frame);
+		if (retval == _FAIL) {
+			precvinfo->dbg_rx_drop_count++;
+			DBG_COUNTER(adapter->rx_logs.core_rx_pre_data_err);
+		} else if (retval == _SUCCESS) {
+			#ifdef DBG_RX_DUMP_EAP
+			if (!pattrib->encrypt || pattrib->bdecrypted) {
+				u8 bDumpRxPkt;
+				u16 eth_type;
+
+				/* dump eapol */
+				rtw_hal_get_def_var(adapter, adapter_link, HAL_DEF_DBG_DUMP_RXPKT, &(bDumpRxPkt));
+				/* get ether_type */
+				_rtw_memcpy(&eth_type, ptr + pattrib->hdrlen + pattrib->iv_len + RATTRIB_GET_MCTRL_LEN(pattrib) + LLC_HEADER_SIZE, 2);
+				eth_type = ntohs((unsigned short) eth_type);
+				if ((bDumpRxPkt == 4) && (eth_type == 0x888e))
+					dump_rx_packet(ptr);
+			}
+			#endif
+		} else
+			DBG_COUNTER(adapter->rx_logs.core_rx_pre_data_handled);
+		break;
+	default:
+		DBG_COUNTER(adapter->rx_logs.core_rx_pre_unknown);
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" fail! type=0x%x\n"
+			, FUNC_ADPT_ARG(adapter), type);
+		#endif
+		retval = _FAIL;
+		break;
+	}
+
+exit:
+
+
+	return retval;
+}
+
+#ifdef CONFIG_MP_INCLUDED
+int validate_mp_recv_frame(_adapter *adapter, union recv_frame *precv_frame)
+{
+	int ret = _SUCCESS;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	u8 type, subtype;
+	struct mp_priv *pmppriv = &adapter->mppriv;
+	struct mp_tx		*pmptx;
+	unsigned char	*sa , *da, *bs;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	u32 i = 0;
+	u8 rtk_prefix[]={0x52, 0x65, 0x61, 0x6C, 0x4C, 0x6F, 0x76, 0x65, 0x54, 0x65, 0x6B};
+	u8 *prx_data;
+	pmptx = &pmppriv->tx;
+
+	if (pmppriv->bloopback) {
+		RTW_DBG_DUMP("mp tx buf", pmptx->pallocated_buf, pmptx->buf_size);
+		RTW_DBG_DUMP("precv_frame", ptr, precv_frame->u.hdr.len);
+		if (_rtw_memcmp(ptr + 26, pmptx->pallocated_buf, precv_frame->u.hdr.len -26) == _FALSE) {
+			RTW_INFO("Compare payload content Fail !!!\n");
+			ret = _FAIL;
+		}
+	}
+	if (pmppriv->bSetRxBssid == _TRUE) {
+
+		sa = get_addr2_ptr(ptr);
+		da = GetAddr1Ptr(ptr);
+		bs = GetAddr3Ptr(ptr);
+		type =	GetFrameType(ptr);
+		subtype = get_frame_sub_type(ptr); /* bit(7)~bit(2)  */
+
+		if (_rtw_memcmp(bs, adapter->mppriv.network_macaddr, ETH_ALEN) == _FALSE) {
+			ret = _FAIL;
+		}
+		else {
+#ifdef RTW_WKARD_CORE_RSSI_V1
+			rx_process_phy_info(precv_frame);
+#endif
+		}
+
+		RTW_DBG("############ type:0x%02x subtype:0x%02x #################\n", type, subtype);
+		RTW_DBG("A2 sa %02X:%02X:%02X:%02X:%02X:%02X \n", *(sa) , *(sa + 1), *(sa+ 2), *(sa + 3), *(sa + 4), *(sa + 5));
+		RTW_DBG("A1 da %02X:%02X:%02X:%02X:%02X:%02X \n", *(da) , *(da + 1), *(da+ 2), *(da + 3), *(da + 4), *(da + 5));
+		RTW_DBG("A3 bs %02X:%02X:%02X:%02X:%02X:%02X \n --------------------------\n", *(bs) , *(bs + 1), *(bs+ 2), *(bs + 3), *(bs + 4), *(bs + 5));
+	}
+
+	if (!adapter->mppriv.bmac_filter)
+		return ret;
+
+	if (_rtw_memcmp(get_addr2_ptr(ptr), adapter->mppriv.mac_filter, ETH_ALEN) == _FALSE)
+		ret = _FAIL;
+
+	return ret;
+}
+
+sint mp_recv_frame(_adapter *adapter, union recv_frame *precv_frame)
+{
+	sint ret = _SUCCESS;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	struct recv_info *precvinfo = &adapter->recvinfo;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	u8  ver = (unsigned char)(*ptr) & 0x3 ;
+
+	struct mp_priv *pmppriv = &adapter->mppriv;
+
+	u8 type;
+	u8 subtype;
+	u8 *psa, *pda, *pbssid;
+	struct sta_info *psta = NULL;
+
+	if (pattrib->crc_err == 1)
+		pmppriv->rx_crcerrpktcount++;
+	else {
+		if (_SUCCESS == validate_mp_recv_frame(adapter, precv_frame))
+			pmppriv->rx_pktcount++;
+		else
+			pmppriv->rx_pktcount_filter_out++;
+	}
+
+exit:
+	ret = _SUCCESS;
+	return ret;
+
+}
+
+#endif
+/* Reture expected handling for LLC */
+enum rtw_rx_llc_hdl rtw_recv_llc_parse(u8 *msdu, u16 msdu_len)
+{
+	u16	eth_type;
+
+	if (msdu_len < 8)
+		return RTW_RX_LLC_KEEP;
+
+	eth_type = RTW_GET_BE16(msdu + SNAP_SIZE);
+
+	if ((_rtw_memcmp(msdu, rtw_rfc1042_header, SNAP_SIZE)
+			&& eth_type != ETH_P_AARP && eth_type != ETH_P_IPX)
+		|| _rtw_memcmp(msdu, rtw_bridge_tunnel_header, SNAP_SIZE)) {
+		/* remove RFC1042 or Bridge-Tunnel encapsulation and replace EtherType */
+		return RTW_RX_LLC_REMOVE;
+	} else {
+		/* Leave Ethernet header part of hdr and full payload */
+		return RTW_RX_LLC_KEEP;
+	}
+
+	/* TODO: VLAN tagged */
+}
+
+
+/* remove the wlanhdr and add the eth_hdr */
+sint wlanhdr_to_ethhdr(union recv_frame *precvframe, enum rtw_rx_llc_hdl llc_hdl)
+{
+	u8 *ptr = get_recvframe_data(precvframe) ; /* point to frame_ctrl field */
+	struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;
+	sint rmv_len;
+	u16 len;
+	sint ret = _SUCCESS;
+#ifdef CONFIG_RTW_CORE_RXSC
+	_adapter *adapter = precvframe->u.hdr.adapter;
+	struct core_rxsc_entry *rxsc_entry = NULL;
+	u16 eth_type;
+#endif
+
+	if (pattrib->encrypt)
+		recvframe_pull_tail(precvframe, pattrib->icv_len);
+
+	rmv_len = pattrib->hdrlen + pattrib->iv_len + RATTRIB_GET_MCTRL_LEN(pattrib) + (llc_hdl ? SNAP_SIZE : 0);
+	len = precvframe->u.hdr.len - rmv_len;
+
+#ifdef CONFIG_RTW_CORE_RXSC
+	_rtw_memcpy(&eth_type, ptr + rmv_len, 2);
+	eth_type = ntohs((unsigned short)eth_type); /* pattrib->ether_type */
+	pattrib->eth_type = eth_type;
+	if (llc_hdl == RTW_RX_LLC_REMOVE)
+		pattrib->bsnaphdr = _TRUE;
+	else if (llc_hdl == RTW_RX_LLC_KEEP)
+		pattrib->bsnaphdr = _FALSE;
+
+	rxsc_entry = core_rxsc_alloc_entry(adapter, precvframe);
+
+	if (rxsc_entry) {
+		/* cache offset of payload */
+		rxsc_entry->rxsc_payload_offset = (rmv_len - sizeof(struct ethhdr) + (pattrib->bsnaphdr ? 2 : 0));
+
+		/* cache padding size of tail */
+		if (pattrib->encrypt)
+			rxsc_entry->rxsc_trim_pad = pattrib->icv_len;
+		else
+			rxsc_entry->rxsc_trim_pad = 0;
+
+		/* cache WLAN header */
+		_rtw_memcpy((void *)&rxsc_entry->rxsc_wlanhdr, ptr, pattrib->hdrlen);
+	}
+#endif
+	ptr = recvframe_pull(precvframe, (rmv_len - sizeof(struct ethhdr) + (llc_hdl ? 2 : 0)));
+	if (!ptr) {
+		ret = _FAIL;
+		goto exiting;
+	}
+
+
+	_rtw_memcpy(ptr, pattrib->dst, ETH_ALEN);
+	_rtw_memcpy(ptr + ETH_ALEN, pattrib->src, ETH_ALEN);
+
+	if (!llc_hdl) {
+		len = htons(len);
+		_rtw_memcpy(ptr + 12, &len, 2);
+	}
+
+	rtw_rframe_set_os_pkt(precvframe);
+
+#ifdef CONFIG_RTW_CORE_RXSC
+	if (rxsc_entry) {
+		/* Cache ETH header */
+		memcpy((void *)&rxsc_entry->rxsc_ethhdr, ptr, sizeof(rxsc_entry->rxsc_ethhdr));
+
+		/* Cache Rx Attribute */
+		memcpy((void *)&rxsc_entry->rxsc_attrib, pattrib, sizeof(rxsc_entry->rxsc_attrib));
+	}
+#ifdef RTW_PHL_DBG_CMD
+	adapter->core_logs.rxCnt_data_orig++;
+#endif
+#endif
+
+exiting:
+	return ret;
+
+}
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+#ifndef CONFIG_SDIO_RX_COPY
+#ifdef PLATFORM_LINUX
+static void recvframe_expand_pkt(
+	_adapter *padapter,
+	union recv_frame *prframe)
+{
+	struct recv_frame_hdr *pfhdr;
+	struct sk_buff *ppkt;
+	u8 shift_sz;
+	u32 alloc_sz;
+	u8 *ptr;
+
+
+	pfhdr = &prframe->u.hdr;
+
+	/*	6 is for IP header 8 bytes alignment in QoS packet case. */
+	if (pfhdr->attrib.qos)
+		shift_sz = 6;
+	else
+		shift_sz = 0;
+
+	/* for first fragment packet, need to allocate */
+	/* (1536 + RXDESC_SIZE + drvinfo_sz) to reassemble packet */
+	/*	8 is for skb->data 8 bytes alignment.
+	*	alloc_sz = _RND(1536 + RXDESC_SIZE + pfhdr->attrib.drvinfosize + shift_sz + 8, 128); */
+	alloc_sz = 1664; /* round (1536 + 24 + 32 + shift_sz + 8) to 128 bytes alignment */
+
+	/* 3 1. alloc new skb */
+	/* prepare extra space for 4 bytes alignment */
+	ppkt = rtw_skb_alloc(alloc_sz);
+
+	if (!ppkt)
+		return; /* no way to expand */
+
+	/* 3 2. Prepare new skb to replace & release old skb */
+	/* force ppkt->data at 8-byte alignment address */
+	skb_reserve(ppkt, 8 - ((SIZE_PTR)ppkt->data & 7));
+	/* force ip_hdr at 8-byte alignment address according to shift_sz */
+	skb_reserve(ppkt, shift_sz);
+
+	/* copy data to new pkt */
+	ptr = skb_put(ppkt, pfhdr->len);
+	if (ptr)
+		_rtw_memcpy(ptr, pfhdr->rx_data, pfhdr->len);
+
+	rtw_skb_free(pfhdr->pkt);
+
+	/* attach new pkt to recvframe */
+	pfhdr->pkt = ppkt;
+	pfhdr->rx_head = ppkt->head;
+	pfhdr->rx_data = ppkt->data;
+	pfhdr->rx_tail = skb_tail_pointer(ppkt);
+	pfhdr->rx_end = skb_end_pointer(ppkt);
+}
+#else /*!= PLATFORM_LINUX*/
+#warning "recvframe_expand_pkt not implement, defrag may crash system"
+#endif
+#endif /*#ifndef CONFIG_SDIO_RX_COPY*/
+#endif
+
+/* perform defrag */
+union recv_frame *recvframe_defrag(_adapter *adapter, _queue *defrag_q)
+{
+	_list	*plist, *phead;
+	u8	*data, wlanhdr_offset;
+	u8	curfragnum;
+	struct recv_frame_hdr *pfhdr, *pnfhdr;
+	union recv_frame *prframe, *pnextrframe;
+	u8 *pdata = NULL;
+	u64 tmp_iv_hdr = 0;
+	u64 pkt_pn = 0, cur_pn = 0;
+	struct rx_pkt_attrib *pattrib = NULL;
+
+	curfragnum = 0;
+	phead = get_list_head(defrag_q);
+	plist = get_next(phead);
+	prframe = LIST_CONTAINOR(plist, union recv_frame, u);
+	/* CVE-2020-26146 */
+	pattrib = &prframe->u.hdr.attrib;
+	if (pattrib->encrypt == _AES_  || pattrib->encrypt == _CCMP_256_
+		|| pattrib->encrypt == _GCMP_ || pattrib->encrypt == _GCMP_256_ ) {
+		pdata = prframe->u.hdr.rx_data;
+		tmp_iv_hdr = le64_to_cpu(*(u64*)(pdata + pattrib->hdrlen));
+		/* get the first frame's PN. */
+		cur_pn = CCMPH_2_PN(tmp_iv_hdr);
+	}
+
+	pfhdr = &prframe->u.hdr;
+	rtw_list_delete(&(prframe->u.list));
+
+	if (curfragnum != pfhdr->attrib.frag_num) {
+		/* the first fragment number must be 0 */
+		/* free the whole queue */
+		rtw_free_recvframe(prframe);
+		rtw_free_recvframe_queue(defrag_q, _FALSE);
+
+		return NULL;
+	}
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+#ifndef CONFIG_SDIO_RX_COPY
+	recvframe_expand_pkt(adapter, prframe);
+#endif
+#endif
+
+	curfragnum++;
+
+	plist = get_list_head(defrag_q);
+
+	plist = get_next(plist);
+
+	data = get_recvframe_data(prframe);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		pnextrframe = LIST_CONTAINOR(plist, union recv_frame , u);
+		pnfhdr = &pnextrframe->u.hdr;
+		/* CVE-2020-26146, check whether the PN is consecutive. */
+		pattrib = &pnextrframe->u.hdr.attrib;
+		if (pattrib->encrypt == _AES_  || pattrib->encrypt == _CCMP_256_
+			|| pattrib->encrypt == _GCMP_ || pattrib->encrypt == _GCMP_256_ ) {
+			pdata = pnextrframe->u.hdr.rx_data;
+			tmp_iv_hdr = le64_to_cpu(*(u64*)(pdata + pattrib->hdrlen));
+			pkt_pn = CCMPH_2_PN(tmp_iv_hdr);
+			if (pkt_pn != cur_pn + 1) {
+				RTW_INFO("%s non-consective PN! old:%llu, new:%llu\n",
+					__func__, cur_pn, pkt_pn);
+				/* PN must be consecutive */
+				/* release the defrag_q & prframe */
+				rtw_free_recvframe(prframe);
+				rtw_free_recvframe_queue(defrag_q, _FALSE);
+				return NULL;
+			} else {
+				cur_pn = pkt_pn;
+			}
+		}
+
+		/* CVE-2020-24587, The keytrack of the fragment is supposed to be the same with other's	*/
+		if (pfhdr->keytrack != pnfhdr->keytrack) {
+			RTW_INFO("Inconsistent key track, drop fragmented frame!\n");
+			rtw_free_recvframe(prframe);
+			rtw_free_recvframe_queue(defrag_q, _FALSE);
+			return NULL;
+		}
+
+		/* check the fragment sequence  (2nd ~n fragment frame) */
+
+		if (curfragnum != pnfhdr->attrib.frag_num) {
+			/* the fragment number must be increasing  (after decache) */
+			/* release the defrag_q & prframe */
+			rtw_free_recvframe(prframe);
+			rtw_free_recvframe_queue(defrag_q, _FALSE);
+			return NULL;
+		}
+
+		/* copy the 2nd~n fragment frame's payload to the first fragment */
+		/* get the 2nd~last fragment frame's payload */
+
+		wlanhdr_offset = pnfhdr->attrib.hdrlen + pnfhdr->attrib.iv_len;
+
+		recvframe_pull(pnextrframe, wlanhdr_offset);
+
+		if ((pfhdr->rx_end - pfhdr->rx_tail) < pnfhdr->len) {
+			RTW_INFO("Not enough buffer space, drop fragmented frame!\n");
+			rtw_free_recvframe(prframe);
+			rtw_free_recvframe_queue(defrag_q, _FALSE);
+			return NULL;
+		}
+
+		curfragnum++;
+
+		/* append  to first fragment frame's tail (if privacy frame, pull the ICV) */
+		recvframe_pull_tail(prframe, pfhdr->attrib.icv_len);
+
+		/* _rtw_memcpy */
+		_rtw_memcpy(pfhdr->rx_tail, pnfhdr->rx_data, pnfhdr->len);
+
+		recvframe_put(prframe, pnfhdr->len);
+
+		pfhdr->attrib.icv_len = pnfhdr->attrib.icv_len;
+		plist = get_next(plist);
+
+	};
+
+	/* free the defrag_q queue and return the prframe */
+	rtw_free_recvframe_queue(defrag_q, _FALSE);
+
+
+
+	return prframe;
+}
+
+/* check if need to defrag, if needed queue the frame to defrag_q */
+s32 recvframe_chk_defrag(_adapter *padapter, union recv_frame **pprecv_frame)
+{
+	u8	ismfrag;
+	u8	fragnum;
+	u8	*psta_addr;
+	struct recv_frame_hdr *pfhdr;
+	struct sta_info *psta;
+	struct sta_priv *pstapriv;
+	_list *phead;
+	union recv_frame *prtnframe = NULL;
+	_queue *pdefrag_q = NULL;
+	union recv_frame *precv_frame = *pprecv_frame;
+	s32 ret = CORE_RX_CONTINUE;
+
+
+	pstapriv = &padapter->stapriv;
+
+	pfhdr = &precv_frame->u.hdr;
+
+	/* need to define struct of wlan header frame ctrl */
+	ismfrag = pfhdr->attrib.mfrag;
+	fragnum = pfhdr->attrib.frag_num;
+
+	psta_addr = pfhdr->attrib.ta;
+	psta = precv_frame->u.hdr.psta;
+	if (psta == NULL) {
+		u8 type = GetFrameType(pfhdr->rx_data);
+		if (type != WIFI_DATA_TYPE) {
+			psta = rtw_get_bcmc_stainfo(padapter, precv_frame->u.hdr.adapter_link);
+			if (psta)
+				pdefrag_q = &psta->sta_recvpriv.defrag_q;
+		} else
+			pdefrag_q = NULL;
+	} else
+		pdefrag_q = &psta->sta_recvpriv.defrag_q;
+
+	if ((ismfrag == 0) && (fragnum == 0)) {
+		ret = CORE_RX_CONTINUE;
+	} else {
+		/* CVE-2020-26145, group addressed frame cannot use fragmentation!! */
+		if (IS_MCAST(pfhdr->attrib.ra)) {
+			RTW_INFO("DROP group addressed fragment!\n");
+			ret = CORE_RX_DROP;
+		}
+		/* CVE-2020-24587 */
+		if ((psta) && (pdefrag_q))
+			precv_frame->u.hdr.keytrack = ATOMIC_READ(&psta->keytrack);
+	}
+
+	if (ismfrag == 1) {
+		/* 0~(n-1) fragment frame */
+		/* enqueue to defraf_g */
+		if (pdefrag_q != NULL) {
+			if (fragnum == 0) {
+				/* the first fragment */
+				if (_rtw_queue_empty(pdefrag_q) == _FALSE) {
+					/* free current defrag_q */
+					rtw_free_recvframe_queue(pdefrag_q, _TRUE);
+				}
+			}
+
+
+			/* Then enqueue the 0~(n-1) fragment into the defrag_q */
+
+			_rtw_spinlock_bh(&pdefrag_q->lock);
+			phead = get_list_head(pdefrag_q);
+			rtw_list_insert_tail(&pfhdr->list, phead);
+			_rtw_spinunlock_bh(&pdefrag_q->lock);
+
+
+			ret = CORE_RX_DEFRAG;
+		} else {
+			/* can't find this ta's defrag_queue, so free this recv_frame */
+			ret = CORE_RX_DROP;
+		}
+
+	}
+
+	if ((ismfrag == 0) && (fragnum != 0)) {
+		/* the last fragment frame */
+		/* enqueue the last fragment */
+		if (pdefrag_q != NULL) {
+			_rtw_spinlock_bh(&pdefrag_q->lock);
+			phead = get_list_head(pdefrag_q);
+			rtw_list_insert_tail(&pfhdr->list, phead);
+
+			/* call recvframe_defrag to defrag */
+			 *pprecv_frame = recvframe_defrag(padapter, pdefrag_q);
+			_rtw_spinunlock_bh(&pdefrag_q->lock);
+			if (*pprecv_frame == NULL)
+				ret = CORE_RX_DROP;
+			else {
+				ret = CORE_RX_CONTINUE;
+			}
+		} else {
+			/* can't find this ta's defrag_queue, so free this recv_frame */
+			ret = CORE_RX_DROP;
+		}
+
+	}
+
+	return ret;
+
+}
+
+static int rtw_recv_indicatepkt_check(union recv_frame *rframe, u8 *ehdr_pos, u32 pkt_len)
+{
+	_adapter *adapter = rframe->u.hdr.adapter;
+	struct recv_info *recvinfo = &adapter->recvinfo;
+	struct ethhdr *ehdr = (struct ethhdr *)ehdr_pos;
+	struct rx_pkt_attrib *pattrib = &rframe->u.hdr.attrib;
+#ifdef DBG_IP_R_MONITOR
+	int i;
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_priv	*pmlmepriv = &adapter->mlmepriv;
+	struct wlan_network *cur_network = &(pmlmepriv->dev_cur_network);
+#endif/*DBG_IP_R_MONITOR*/
+	enum eap_type eapol_type;
+	int ret = _FAIL;
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (rtw_wapi_check_for_drop(adapter, rframe, ehdr_pos)) {
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" rtw_wapi_check_for_drop\n"
+			, FUNC_ADPT_ARG(adapter));
+		#endif
+		goto exit;
+	}
+#endif
+
+	if (rframe->u.hdr.psta)
+		rtw_st_ctl_rx(rframe->u.hdr.psta, ehdr_pos);
+
+	if (ntohs(ehdr->h_proto) == 0x888e) {
+		eapol_type = parsing_eapol_packet(adapter, ehdr_pos + ETH_HLEN, rframe->u.hdr.psta, 0);
+		if ((eapol_type == EAPOL_1_4 || eapol_type == EAPOL_3_4) && pattrib->encrypt == 0) {
+			rframe->u.hdr.psta->resp_nonenc_eapol_key_starttime = rtw_get_current_time();
+			RTW_INFO("Receive unencrypted eapol key\n");
+		}
+	}
+#ifdef DBG_ARP_DUMP
+	else if (ntohs(ehdr->h_proto) == ETH_P_ARP)
+		dump_arp_pkt(RTW_DBGDUMP, ehdr->h_dest, ehdr->h_source, ehdr_pos + ETH_HLEN, 0);
+#endif
+
+	if (recvinfo->sink_udpport > 0)
+		rtw_sink_rtp_seq_dbg(adapter, ehdr_pos);
+
+#ifdef DBG_UDP_PKT_LOSE_11AC
+	#define PAYLOAD_LEN_LOC_OF_IP_HDR 0x10 /*ethernet payload length location of ip header (DA + SA+eth_type+(version&hdr_len)) */
+
+	if (ntohs(ehdr->h_proto) == ETH_P_ARP) {
+		/* ARP Payload length will be 42bytes or 42+18(tailer)=60bytes*/
+		if (pkt_len != 42 && pkt_len != 60)
+			RTW_INFO("Error !!%s,ARP Payload length %u not correct\n" , __func__ , pkt_len);
+	} else if (ntohs(ehdr->h_proto) == ETH_P_IP) {
+		if (be16_to_cpu(*((u16 *)(ehdr_pos + PAYLOAD_LEN_LOC_OF_IP_HDR))) != (pkt_len) - ETH_HLEN) {
+			RTW_INFO("Error !!%s,Payload length not correct\n" , __func__);
+			RTW_INFO("%s, IP header describe Total length=%u\n" , __func__ , be16_to_cpu(*((u16 *)(ehdr_pos + PAYLOAD_LEN_LOC_OF_IP_HDR))));
+			RTW_INFO("%s, Pkt real length=%u\n" , __func__ , (pkt_len) - ETH_HLEN);
+		}
+	}
+#endif
+
+#ifdef DBG_IP_R_MONITOR
+	#define LEN_ARP_OP_HDR 7 /*ARP OERATION */
+	if (ntohs(ehdr->h_proto) == ETH_P_ARP) {
+
+		if(check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE){
+			if(ehdr_pos[ETHERNET_HEADER_SIZE+LEN_ARP_OP_HDR] == 2) {
+
+				RTW_INFO("%s,[DBG_ARP] Rx ARP RSP Packet;SeqNum = %d !\n",
+					__FUNCTION__, pattrib->seq_num);
+
+				dump_arp_pkt(RTW_DBGDUMP, ehdr->h_dest, ehdr->h_source, ehdr_pos + ETH_HLEN, 0);
+
+			}
+		}
+	}
+#endif/*DBG_IP_R_MONITOR*/
+
+#ifdef CONFIG_AUTO_AP_MODE
+	if (ntohs(ehdr->h_proto) == 0x8899)
+		rtw_auto_ap_rx_msg_dump(adapter, rframe, ehdr_pos);
+#endif
+
+	ret = _SUCCESS;
+
+#ifdef CONFIG_WAPI_SUPPORT
+exit:
+#endif
+	return ret;
+}
+
+#if defined(CONFIG_AP_MODE) || defined(CONFIG_RTW_MESH)
+static void recv_free_fwd_resource(_adapter *adapter, struct xmit_frame *fwd_frame, _list *f_list)
+{
+	struct xmit_priv *xmitpriv = &adapter->xmitpriv;
+
+	if (fwd_frame)
+		core_tx_free_xmitframe(adapter, fwd_frame);
+
+#if defined(CONFIG_RTW_WDS) || CONFIG_RTW_DATA_BMC_TO_UC
+	if (!rtw_is_list_empty(f_list)) {
+		struct xmit_frame *fframe;
+		_list *list;
+
+		list = get_next(f_list);
+		while (rtw_end_of_queue_search(f_list, list) == _FALSE) {
+			fframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+			list = get_next(list);
+			rtw_list_delete(&fframe->list);
+			core_tx_free_xmitframe(adapter, fframe);
+		}
+	}
+#endif
+}
+
+static void recv_fwd_pkt_hdl(_adapter *adapter, struct sk_buff *pkt
+	, u8 act, struct xmit_frame *fwd_frame, _list *f_list)
+{
+	struct xmit_priv *xmitpriv = &adapter->xmitpriv;
+	struct sk_buff *fwd_pkt = pkt;
+
+	if (act & RTW_RX_MSDU_ACT_INDICATE) {
+		fwd_pkt = rtw_skb_copy(pkt);
+		if (!fwd_pkt) {
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s rtw_skb_copy fail\n", __func__);
+			#endif
+			recv_free_fwd_resource(adapter, fwd_frame, f_list);
+			goto exit;
+		}
+	}
+
+#if defined(CONFIG_RTW_WDS) || CONFIG_RTW_DATA_BMC_TO_UC
+	if (!rtw_is_list_empty(f_list)) {
+		_list *list = get_next(f_list);
+		struct xmit_frame *fframe;
+
+		while (rtw_end_of_queue_search(f_list, list) == _FALSE) {
+			fframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+			list = get_next(list);
+			rtw_list_delete(&fframe->list);
+
+			if (!fwd_frame && rtw_is_list_empty(f_list)) /* the last fwd_pkt */
+				fframe->pkt = fwd_pkt;
+			else
+				fframe->pkt = rtw_skb_copy(fwd_pkt);
+			if (!fframe->pkt) {
+				core_tx_free_xmitframe(adapter, fframe);
+				continue;
+			}
+
+			core_tx_per_packet(adapter, fframe, &fframe->pkt, NULL);
+		}
+	}
+#endif
+
+	if (fwd_frame) {
+		fwd_frame->pkt = fwd_pkt;
+		if (core_tx_per_packet(adapter, fwd_frame, &fwd_pkt, NULL) < 0) {
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s rtw_xmit_posthandle fail\n", __func__);
+			#endif
+			xmitpriv->tx_drop++;
+		}
+	}
+
+exit:
+	return;
+}
+#endif /* defined(CONFIG_AP_MODE) || defined(CONFIG_RTW_MESH) */
+
+/*
+ * From WFA suggestion:						*
+ * If first subframe meets one of the following condition,	*
+ * the whole received AMSDU should drop.			*
+ * 1. subframe's DA is not the same as RA in From DS case.	*
+ * 2. subframe's SA is not the same as TA in To DS case.	*
+ * 3. subframe's DA is AA:AA:03:00:00:00			*
+								*/
+static u8 validate_amsdu_content(_adapter *padapter, union recv_frame *prframe,
+	const u8 *da, const u8 *sa)
+{
+	struct rx_pkt_attrib	*pattrib = &prframe->u.hdr.attrib;
+	u8 ret = _SUCCESS;
+
+	/* Use the recommendation method form Wi-Fi alliance to check subframe */
+	/* in protected network */
+	if (padapter->registrypriv.amsdu_mode == RTW_AMSDU_MODE_NON_SPP &&
+		padapter->securitypriv.dot11PrivacyAlgrthm != _NO_PRIVACY_) {
+
+		/* 1.check From DS */
+		if (pattrib->to_fr_ds == 1) {
+			if (_rtw_memcmp(da, pattrib->ra, ETH_ALEN) == _FALSE)
+			ret = _FAIL;
+		}
+
+		/* 2.check To DS */
+		if (pattrib->to_fr_ds == 2) {
+			if (_rtw_memcmp(sa, pattrib->ta, ETH_ALEN) == _FALSE)
+			ret = _FAIL;
+		}
+
+		/* 3.Check whether DA is AA:AA:03:00:00:00 */
+		if (_rtw_memcmp(da, rtw_rfc1042_header, ETH_ALEN) == _TRUE)
+			ret = _FAIL;
+
+	}
+
+	return ret;
+
+}
+
+int amsdu_to_msdu(_adapter *padapter, union recv_frame **pprframe)
+{
+	union recv_frame *prframe = *pprframe;
+	struct rx_pkt_attrib *rattrib = &prframe->u.hdr.attrib;
+	int	a_len, padding_len;
+	u16	nSubframe_Length;
+	u8	nr_subframes, i;
+	u8	*pdata;
+	struct sk_buff *sub_pkt, *subframes[MAX_SUBFRAME_COUNT];
+	const u8 *da, *sa;
+	int act;
+#if defined(CONFIG_AP_MODE) || defined(CONFIG_RTW_MESH)
+	struct xmit_frame *fwd_frame;
+	_list f_list;
+#endif
+	enum rtw_rx_llc_hdl llc_hdl;
+	u8 mctrl_len = 0;
+	int	ret = _SUCCESS;
+
+	nr_subframes = 0;
+
+	recvframe_pull(prframe, rattrib->hdrlen);
+
+	if (rattrib->iv_len > 0)
+		recvframe_pull(prframe, rattrib->iv_len);
+
+	if (rattrib->encrypt)
+		recvframe_pull_tail(prframe, rattrib->icv_len);
+
+	a_len = prframe->u.hdr.len;
+	pdata = prframe->u.hdr.rx_data;
+
+	while (a_len > ETH_HLEN) {
+		/* Offset 12 denote 2 mac address */
+		nSubframe_Length = RTW_GET_BE16(pdata + 12);
+		if (a_len < (ETHERNET_HEADER_SIZE + nSubframe_Length)) {
+			RTW_INFO("nRemain_Length is %d and nSubframe_Length is : %d\n", a_len, nSubframe_Length);
+			break;
+		}
+
+		act = RTW_RX_MSDU_ACT_INDICATE;
+
+		#if defined(CONFIG_AP_MODE) || defined(CONFIG_RTW_MESH)
+		fwd_frame = NULL;
+		#endif
+
+		#ifdef CONFIG_RTW_MESH
+		if (MLME_IS_MESH(padapter)) {
+			u8 *mda = pdata, *msa = pdata + ETH_ALEN;
+			struct rtw_ieee80211s_hdr *mctrl = (struct rtw_ieee80211s_hdr *)(pdata + ETH_HLEN);
+			int v_ret;
+
+			v_ret = rtw_mesh_rx_data_validate_mctrl(padapter, prframe
+				, mctrl, mda, msa, &mctrl_len, &da, &sa);
+			if (v_ret != _SUCCESS)
+				goto move_to_next;
+
+			if (validate_amsdu_content(padapter, prframe, da, sa) == _FAIL) {
+				RTW_INFO("%s check subframe content fail!\n", __func__);
+				break;
+			}
+
+			llc_hdl = rtw_recv_llc_parse(pdata + ETH_HLEN + mctrl_len, nSubframe_Length - mctrl_len);
+			act = rtw_mesh_rx_msdu_act_check(prframe
+				, mda, msa, da, sa, mctrl
+				, pdata + ETH_HLEN + mctrl_len, llc_hdl
+				, &fwd_frame, &f_list);
+		} else
+		#endif
+		{
+			da = pdata;
+			sa = pdata + ETH_ALEN;
+
+			if (validate_amsdu_content(padapter, prframe, da, sa) == _FAIL) {
+				RTW_INFO("%s check subframe content fail!\n", __func__);
+				break;
+			}
+
+			llc_hdl = rtw_recv_llc_parse(pdata + ETH_HLEN, nSubframe_Length);
+			#ifdef CONFIG_AP_MODE
+			if (MLME_IS_AP(padapter)) {
+				act = rtw_ap_rx_msdu_act_check(prframe, da, sa
+					, pdata + ETH_HLEN, llc_hdl, &fwd_frame, &f_list);
+			} else
+			#endif
+			if (MLME_IS_STA(padapter))
+				act = rtw_sta_rx_amsdu_act_check(prframe, da, sa);
+		}
+
+		if (!act)
+			goto move_to_next;
+
+		rtw_led_rx_control(padapter, da);
+
+		sub_pkt = rtw_os_alloc_msdu_pkt(prframe, da, sa
+			, pdata + ETH_HLEN + mctrl_len, nSubframe_Length - mctrl_len, llc_hdl);
+		if (sub_pkt == NULL) {
+			if (act & RTW_RX_MSDU_ACT_INDICATE) {
+				#ifdef DBG_RX_DROP_FRAME
+				RTW_INFO("DBG_RX_DROP_FRAME %s rtw_os_alloc_msdu_pkt fail\n", __func__);
+				#endif
+			}
+			#if defined(CONFIG_AP_MODE) || defined(CONFIG_RTW_MESH)
+			if (act & RTW_RX_MSDU_ACT_FORWARD) {
+				#ifdef DBG_TX_DROP_FRAME
+				RTW_INFO("DBG_TX_DROP_FRAME %s rtw_os_alloc_msdu_pkt fail\n", __func__);
+				#endif
+				recv_free_fwd_resource(padapter, fwd_frame, &f_list);
+			}
+			#endif
+			break;
+		}
+
+		#if defined(CONFIG_AP_MODE) || defined(CONFIG_RTW_MESH)
+		if (act & RTW_RX_MSDU_ACT_FORWARD) {
+			recv_fwd_pkt_hdl(padapter, sub_pkt, act, fwd_frame, &f_list);
+			if (!(act & RTW_RX_MSDU_ACT_INDICATE))
+				goto move_to_next;
+		}
+		#endif
+
+		if (rtw_recv_indicatepkt_check(prframe, rtw_skb_data(sub_pkt), rtw_skb_len(sub_pkt)) == _SUCCESS)
+			subframes[nr_subframes++] = sub_pkt;
+		else
+			rtw_skb_free(sub_pkt);
+
+#if defined(CONFIG_AP_MODE) || defined(CONFIG_RTW_MESH)
+move_to_next:
+#endif
+		/* move the data point to data content */
+		pdata += ETH_HLEN;
+		a_len -= ETH_HLEN;
+
+		if (nr_subframes >= MAX_SUBFRAME_COUNT) {
+			RTW_WARN("ParseSubframe(): Too many Subframes! Packets dropped!\n");
+			break;
+		}
+
+		pdata += nSubframe_Length;
+		a_len -= nSubframe_Length;
+		if (a_len != 0) {
+			padding_len = 4 - ((nSubframe_Length + ETH_HLEN) & (4 - 1));
+			if (padding_len == 4)
+				padding_len = 0;
+
+			if (a_len < padding_len) {
+				RTW_INFO("ParseSubframe(): a_len < padding_len !\n");
+				break;
+			}
+			pdata += padding_len;
+			a_len -= padding_len;
+		}
+	}
+
+	for (i = 0; i < nr_subframes; i++) {
+		sub_pkt = subframes[i];
+
+		/* Indicat the packets to upper layer */
+		if (sub_pkt)
+			rtw_os_recv_indicate_pkt(padapter, sub_pkt, prframe);
+	}
+
+	prframe->u.hdr.len = 0;
+	rtw_free_recvframe(prframe);/* free this recv_frame */
+	*pprframe = NULL;
+
+	return ret;
+}
+
+static int recv_process_mpdu(_adapter *padapter, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	int ret;
+
+	if (pattrib->amsdu) {
+		ret = amsdu_to_msdu(padapter, &prframe);
+		if (ret != _SUCCESS) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" amsdu_to_msdu fail\n"
+				, FUNC_ADPT_ARG(padapter));
+			#endif
+			rtw_free_recvframe(prframe);
+			goto exit;
+		}
+	} else {
+		int act = RTW_RX_MSDU_ACT_INDICATE;
+
+		#ifdef CONFIG_RTW_MESH /* TODO: move AP mode forward & b2u logic here */
+		struct xmit_frame *fwd_frame = NULL;
+		_list f_list;
+
+		if (MLME_IS_MESH(padapter) && pattrib->mesh_ctrl_present) {
+			act = rtw_mesh_rx_msdu_act_check(prframe
+				, pattrib->mda, pattrib->msa
+				, pattrib->dst, pattrib->src
+				, (struct rtw_ieee80211s_hdr *)(get_recvframe_data(prframe) + pattrib->hdrlen + pattrib->iv_len)
+				, &fwd_frame, &f_list);
+		}
+		#endif
+
+		#ifdef CONFIG_RTW_MESH
+		if (!act) {
+			rtw_free_recvframe(prframe);
+			ret = _FAIL;
+			goto exit;
+		}
+		#endif
+
+		rtw_led_rx_control(padapter, pattrib->dst);
+
+		ret = wlanhdr_to_ethhdr(prframe, 0);
+		if (ret != _SUCCESS) {
+			if (act & RTW_RX_MSDU_ACT_INDICATE) {
+				#ifdef DBG_RX_DROP_FRAME
+				RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" wlanhdr_to_ethhdr: drop pkt\n"
+					, FUNC_ADPT_ARG(padapter));
+				#endif
+			}
+			#ifdef CONFIG_RTW_MESH
+			if (act & RTW_RX_MSDU_ACT_FORWARD) {
+				#ifdef DBG_TX_DROP_FRAME
+				RTW_INFO("DBG_TX_DROP_FRAME %s wlanhdr_to_ethhdr fail\n", __func__);
+				#endif
+				recv_free_fwd_resource(padapter, fwd_frame, &f_list);
+			}
+			#endif
+			rtw_free_recvframe(prframe);
+			goto exit;
+		}
+
+		#ifdef CONFIG_RTW_MESH
+		if (act & RTW_RX_MSDU_ACT_FORWARD) {
+			recv_fwd_pkt_hdl(padapter, prframe->u.hdr.pkt, act, fwd_frame, &f_list);
+			if (!(act & RTW_RX_MSDU_ACT_INDICATE)) {
+				prframe->u.hdr.pkt = NULL;
+				rtw_free_recvframe(prframe);
+				goto exit;
+			}
+		}
+		#endif
+
+		if (!RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+			ret = rtw_recv_indicatepkt_check(prframe
+				, get_recvframe_data(prframe), get_recvframe_len(prframe));
+			if (ret != _SUCCESS) {
+				rtw_free_recvframe(prframe);
+				goto exit;
+			}
+
+			/* indicate this recv_frame */
+			ret = rtw_recv_indicatepkt(padapter, prframe);
+			if (ret != _SUCCESS) {
+				#ifdef DBG_RX_DROP_FRAME
+				RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" rtw_recv_indicatepkt fail!\n"
+					, FUNC_ADPT_ARG(padapter));
+				#endif
+				goto exit;
+			}
+		} else {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" DS:%u SR:%u\n"
+				, FUNC_ADPT_ARG(padapter)
+				, dev_is_drv_stopped(adapter_to_dvobj(padapter))
+				, dev_is_surprise_removed(adapter_to_dvobj(padapter)));
+			#endif
+			ret = _SUCCESS; /* don't count as packet drop */
+			rtw_free_recvframe(prframe);
+		}
+	}
+
+exit:
+	return ret;
+}
+
+#if defined(CONFIG_80211N_HT) && defined(CONFIG_RECV_REORDERING_CTRL)
+static int check_indicate_seq(struct recv_reorder_ctrl *preorder_ctrl, u16 seq_num)
+{
+	_adapter *padapter = preorder_ctrl->padapter;
+	struct recv_info *precvinfo = &padapter->recvinfo;
+	u8	wsize = preorder_ctrl->wsize_b;
+	u16	wend;
+
+	/* Rx Reorder initialize condition. */
+	if (preorder_ctrl->indicate_seq == 0xFFFF) {
+		preorder_ctrl->indicate_seq = seq_num;
+		#ifdef DBG_RX_SEQ
+		RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_INIT indicate_seq:%d, seq_num:%d\n"
+			, FUNC_ADPT_ARG(padapter), preorder_ctrl->tid, preorder_ctrl->indicate_seq, seq_num);
+		#endif
+	}
+	wend = (preorder_ctrl->indicate_seq + wsize - 1) & 0xFFF; /* % 4096; */
+
+	/* Drop out the packet which SeqNum is smaller than WinStart */
+	if (SN_LESS(seq_num, preorder_ctrl->indicate_seq)) {
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO(FUNC_ADPT_FMT" tid:%u indicate_seq:%d > seq_num:%d\n"
+			, FUNC_ADPT_ARG(padapter), preorder_ctrl->tid, preorder_ctrl->indicate_seq, seq_num);
+		#endif
+		return _FALSE;
+	}
+
+	/*
+	* Sliding window manipulation. Conditions includes:
+	* 1. Incoming SeqNum is equal to WinStart =>Window shift 1
+	* 2. Incoming SeqNum is larger than the WinEnd => Window shift N
+	*/
+	if (SN_EQUAL(seq_num, preorder_ctrl->indicate_seq)) {
+		preorder_ctrl->indicate_seq = (preorder_ctrl->indicate_seq + 1) & 0xFFF;
+		#ifdef DBG_RX_SEQ
+		RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_EQUAL indicate_seq:%d, seq_num:%d\n"
+			, FUNC_ADPT_ARG(padapter), preorder_ctrl->tid, preorder_ctrl->indicate_seq, seq_num);
+		#endif
+
+	} else if (SN_LESS(wend, seq_num)) {
+		/* boundary situation, when seq_num cross 0xFFF */
+		if (seq_num >= (wsize - 1))
+			preorder_ctrl->indicate_seq = seq_num + 1 - wsize;
+		else
+			preorder_ctrl->indicate_seq = 0xFFF - (wsize - (seq_num + 1)) + 1;
+
+		precvinfo->dbg_rx_ampdu_window_shift_cnt++;
+		#ifdef DBG_RX_SEQ
+		RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_LESS(wend, seq_num) indicate_seq:%d, seq_num:%d\n"
+			, FUNC_ADPT_ARG(padapter), preorder_ctrl->tid, preorder_ctrl->indicate_seq, seq_num);
+		#endif
+	}
+
+	return _TRUE;
+}
+
+static int enqueue_reorder_recvframe(struct recv_reorder_ctrl *preorder_ctrl, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	_queue *ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
+	_list	*phead, *plist;
+	union recv_frame *pnextrframe;
+	struct rx_pkt_attrib *pnextattrib;
+
+	/* DbgPrint("+enqueue_reorder_recvframe()\n"); */
+
+	/* _rtw_spinlock_irq(&ppending_recvframe_queue->lock, &sp_flags); */
+
+	phead = get_list_head(ppending_recvframe_queue);
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		pnextrframe = LIST_CONTAINOR(plist, union recv_frame, u);
+		pnextattrib = &pnextrframe->u.hdr.attrib;
+
+		if (SN_LESS(pnextattrib->seq_num, pattrib->seq_num))
+			plist = get_next(plist);
+		else if (SN_EQUAL(pnextattrib->seq_num, pattrib->seq_num)) {
+			/* Duplicate entry is found!! Do not insert current entry. */
+
+			/* _rtw_spinunlock_irq(&ppending_recvframe_queue->lock, &sp_flags); */
+
+			return _FALSE;
+		} else
+			break;
+
+		/* DbgPrint("enqueue_reorder_recvframe():while\n"); */
+
+	}
+
+
+	/* _rtw_spinlock_irq(&ppending_recvframe_queue->lock, &sp_flags); */
+
+	rtw_list_delete(&(prframe->u.hdr.list));
+
+	rtw_list_insert_tail(&(prframe->u.hdr.list), plist);
+
+	/* _rtw_spinunlock_irq(&ppending_recvframe_queue->lock, &sp_flags); */
+
+
+	return _TRUE;
+
+}
+
+static void recv_indicatepkts_pkt_loss_cnt(_adapter *padapter, u64 prev_seq, u64 current_seq)
+{
+	struct recv_info *precvinfo = &padapter->recvinfo;
+
+	if (current_seq < prev_seq) {
+		precvinfo->dbg_rx_ampdu_loss_count += (4096 + current_seq - prev_seq);
+		precvinfo->rx_drop += (4096 + current_seq - prev_seq);
+	} else {
+		precvinfo->dbg_rx_ampdu_loss_count += (current_seq - prev_seq);
+		precvinfo->rx_drop += (current_seq - prev_seq);
+	}
+}
+
+static int recv_indicatepkts_in_order(_adapter *padapter, struct recv_reorder_ctrl *preorder_ctrl, int bforced)
+{
+	_list	*phead, *plist;
+	union recv_frame *prframe;
+	struct rx_pkt_attrib *pattrib;
+	/* u8 index = 0; */
+	int bPktInBuf = _FALSE;
+	struct recv_info *precvinfo = &padapter->recvinfo;
+	_queue *ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
+
+	DBG_COUNTER(padapter->rx_logs.core_rx_post_indicate_in_oder);
+
+	/* DbgPrint("+recv_indicatepkts_in_order\n"); */
+
+	/* _rtw_spinlock_irq(&ppending_recvframe_queue->lock, &sp_flags); */
+
+	phead =	get_list_head(ppending_recvframe_queue);
+	plist = get_next(phead);
+
+#if 0
+	/* Check if there is any other indication thread running. */
+	if (pTS->RxIndicateState == RXTS_INDICATE_PROCESSING)
+		return;
+#endif
+
+	/* Handling some condition for forced indicate case. */
+	if (bforced == _TRUE) {
+		precvinfo->dbg_rx_ampdu_forced_indicate_count++;
+		if (rtw_is_list_empty(phead)) {
+			/* _rtw_spinunlock_irq(&ppending_recvframe_queue->lock, &sp_flags); */
+			return _TRUE;
+		}
+
+		prframe = LIST_CONTAINOR(plist, union recv_frame, u);
+		pattrib = &prframe->u.hdr.attrib;
+
+		#ifdef DBG_RX_SEQ
+		RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u FORCE indicate_seq:%d, seq_num:%d\n"
+			, FUNC_ADPT_ARG(padapter), preorder_ctrl->tid, preorder_ctrl->indicate_seq, pattrib->seq_num);
+		#endif
+		recv_indicatepkts_pkt_loss_cnt(padapter, preorder_ctrl->indicate_seq, pattrib->seq_num);
+		preorder_ctrl->indicate_seq = pattrib->seq_num;
+	}
+
+	/* Prepare indication list and indication. */
+	/* Check if there is any packet need indicate. */
+	while (!rtw_is_list_empty(phead)) {
+
+		prframe = LIST_CONTAINOR(plist, union recv_frame, u);
+		pattrib = &prframe->u.hdr.attrib;
+
+		if (!SN_LESS(preorder_ctrl->indicate_seq, pattrib->seq_num)) {
+
+#if 0
+			/* This protect buffer from overflow. */
+			if (index >= REORDER_WIN_SIZE) {
+				RT_ASSERT(FALSE, ("IndicateRxReorderList(): Buffer overflow!!\n"));
+				bPktInBuf = TRUE;
+				break;
+			}
+#endif
+
+			plist = get_next(plist);
+			rtw_list_delete(&(prframe->u.hdr.list));
+
+			if (SN_EQUAL(preorder_ctrl->indicate_seq, pattrib->seq_num)) {
+				preorder_ctrl->indicate_seq = (preorder_ctrl->indicate_seq + 1) & 0xFFF;
+				#ifdef DBG_RX_SEQ
+				RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_EQUAL indicate_seq:%d, seq_num:%d\n"
+					, FUNC_ADPT_ARG(padapter), preorder_ctrl->tid, preorder_ctrl->indicate_seq, pattrib->seq_num);
+				#endif
+			}
+
+#if 0
+			index++;
+			if (index == 1) {
+				/* Cancel previous pending timer. */
+				/* PlatformCancelTimer(adapter, &pTS->RxPktPendingTimer); */
+				if (bforced != _TRUE) {
+					/* RTW_INFO("_cancel_timer_ex(&preorder_ctrl->reordering_ctrl_timer);\n"); */
+					_cancel_timer_ex(&preorder_ctrl->reordering_ctrl_timer);
+				}
+			}
+#endif
+
+			/* Set this as a lock to make sure that only one thread is indicating packet. */
+			/* pTS->RxIndicateState = RXTS_INDICATE_PROCESSING; */
+
+			/* Indicate packets */
+			/* RT_ASSERT((index<=REORDER_WIN_SIZE), ("RxReorderIndicatePacket(): Rx Reorder buffer full!!\n")); */
+
+
+			/* indicate this recv_frame */
+			/* DbgPrint("recv_indicatepkts_in_order, indicate_seq=%d, seq_num=%d\n", precvpriv->indicate_seq, pattrib->seq_num); */
+			if (recv_process_mpdu(padapter, prframe) != _SUCCESS)
+				precvinfo->dbg_rx_drop_count++;
+
+			/* Update local variables. */
+			bPktInBuf = _FALSE;
+
+		} else {
+			bPktInBuf = _TRUE;
+			break;
+		}
+
+		/* DbgPrint("recv_indicatepkts_in_order():while\n"); */
+
+	}
+
+	/* _rtw_spinunlock_irq(&ppending_recvframe_queue->lock, sp_flags); */
+
+#if 0
+	/* Release the indication lock and set to new indication step. */
+	if (bPktInBuf) {
+		/*  Set new pending timer. */
+		/* pTS->RxIndicateState = RXTS_INDICATE_REORDER; */
+		/* PlatformSetTimer(adapter, &pTS->RxPktPendingTimer, pHTInfo->RxReorderPendingTime); */
+
+		_set_timer(&preorder_ctrl->reordering_ctrl_timer, REORDER_WAIT_TIME);
+	} else {
+		/* pTS->RxIndicateState = RXTS_INDICATE_IDLE; */
+	}
+#endif
+	/* _rtw_spinunlock_irq(&ppending_recvframe_queue->lock, sp_flags); */
+
+	/* return _TRUE; */
+	return bPktInBuf;
+
+}
+
+static int recv_indicatepkt_reorder(_adapter *padapter, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	struct recv_reorder_ctrl *preorder_ctrl = prframe->u.hdr.preorder_ctrl;
+	_queue *ppending_recvframe_queue = preorder_ctrl ? &preorder_ctrl->pending_recvframe_queue : NULL;
+	struct recv_info  *precvinfo = &padapter->recvinfo;
+
+	if (!pattrib->qos || !preorder_ctrl || preorder_ctrl->enable == _FALSE)
+		goto _success_exit;
+
+	DBG_COUNTER(padapter->rx_logs.core_rx_post_indicate_reoder);
+
+	_rtw_spinlock_bh(&ppending_recvframe_queue->lock);
+
+	if(rtw_test_and_clear_bit(RTW_RECV_ACK_OR_TIMEOUT, &preorder_ctrl->rec_abba_rsp_ack))
+		preorder_ctrl->indicate_seq = 0xFFFF;
+	#ifdef DBG_RX_SEQ
+	RTW_INFO("DBG_RX_SEQ %s:preorder_ctrl->rec_abba_rsp_ack = %u,indicate_seq = %d\n"
+		, __func__
+		, preorder_ctrl->rec_abba_rsp_ack
+		, preorder_ctrl->indicate_seq);
+	#endif
+
+	/* s2. check if winstart_b(indicate_seq) needs to been updated */
+	if (!check_indicate_seq(preorder_ctrl, pattrib->seq_num)) {
+		precvinfo->dbg_rx_ampdu_drop_count++;
+		/* pHTInfo->RxReorderDropCounter++; */
+		/* ReturnRFDList(adapter, pRfd); */
+		/* _rtw_spinunlock_irq(&ppending_recvframe_queue->lock, sp_flags); */
+		/* return _FAIL; */
+
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" check_indicate_seq fail\n"
+			, FUNC_ADPT_ARG(padapter));
+		#endif
+#if 0
+		rtw_recv_indicatepkt(padapter, prframe);
+
+		_rtw_spinunlock_bh(&ppending_recvframe_queue->lock);
+
+		goto _success_exit;
+#else
+		goto _err_exit;
+#endif
+	}
+
+
+	/* s3. Insert all packet into Reorder Queue to maintain its ordering. */
+	if (!enqueue_reorder_recvframe(preorder_ctrl, prframe)) {
+		/* DbgPrint("recv_indicatepkt_reorder, enqueue_reorder_recvframe fail!\n"); */
+		/* _rtw_spinunlock_irq(&ppending_recvframe_queue->lock, sp_flags); */
+		/* return _FAIL; */
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" enqueue_reorder_recvframe fail\n"
+			, FUNC_ADPT_ARG(padapter));
+		#endif
+		goto _err_exit;
+	}
+
+
+	/* s4. */
+	/* Indication process. */
+	/* After Packet dropping and Sliding Window shifting as above, we can now just indicate the packets */
+	/* with the SeqNum smaller than latest WinStart and buffer other packets. */
+	/*  */
+	/* For Rx Reorder condition: */
+	/* 1. All packets with SeqNum smaller than WinStart => Indicate */
+	/* 2. All packets with SeqNum larger than or equal to WinStart => Buffer it. */
+	/*  */
+
+	/* recv_indicatepkts_in_order(padapter, preorder_ctrl, _TRUE); */
+	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, _FALSE) == _TRUE) {
+		if (!preorder_ctrl->bReorderWaiting) {
+			preorder_ctrl->bReorderWaiting = _TRUE;
+			_set_timer(&preorder_ctrl->reordering_ctrl_timer, REORDER_WAIT_TIME);
+		}
+		_rtw_spinunlock_bh(&ppending_recvframe_queue->lock);
+	} else {
+		preorder_ctrl->bReorderWaiting = _FALSE;
+		_rtw_spinunlock_bh(&ppending_recvframe_queue->lock);
+		_cancel_timer_ex(&preorder_ctrl->reordering_ctrl_timer);
+	}
+
+	return RTW_RX_HANDLED;
+
+_success_exit:
+
+	return _SUCCESS;
+
+_err_exit:
+
+	_rtw_spinunlock_bh(&ppending_recvframe_queue->lock);
+
+	return _FAIL;
+}
+
+
+void rtw_reordering_ctrl_timeout_handler(void *pcontext)
+{
+	struct recv_reorder_ctrl *preorder_ctrl = (struct recv_reorder_ctrl *)pcontext;
+	_adapter *padapter = NULL;
+	_queue *ppending_recvframe_queue = NULL;
+
+
+	if (preorder_ctrl == NULL)
+		return;
+
+	padapter = preorder_ctrl->padapter;
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+		return;
+
+	ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
+
+	/* RTW_INFO("+rtw_reordering_ctrl_timeout_handler()=>\n"); */
+
+	_rtw_spinlock_bh(&ppending_recvframe_queue->lock);
+
+	preorder_ctrl->bReorderWaiting = _FALSE;
+
+	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, _TRUE) == _TRUE)
+		_set_timer(&preorder_ctrl->reordering_ctrl_timer, REORDER_WAIT_TIME);
+
+	_rtw_spinunlock_bh(&ppending_recvframe_queue->lock);
+
+}
+#endif /* defined(CONFIG_80211N_HT) && defined(CONFIG_RECV_REORDERING_CTRL) */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+int recv_frame_monitor(_adapter *padapter, union recv_frame *rframe, struct rtw_recv_pkt *rx_req)
+{
+	int ret = _SUCCESS;
+
+#ifdef CONFIG_WIFI_MONITOR
+	struct net_device *ndev = padapter->pnetdev;
+	struct sk_buff *pskb = NULL;
+
+	if (rframe == NULL)
+		goto exit;
+
+	/* read skb information from recv frame */
+	pskb = rframe->u.hdr.pkt;
+	pskb->len = rframe->u.hdr.len;
+	pskb->data = rframe->u.hdr.rx_data;
+	skb_set_tail_pointer(pskb, rframe->u.hdr.len);
+
+	if (ndev->type == ARPHRD_IEEE80211_RADIOTAP) {
+		/* fill radiotap header */
+		if (rtw_fill_radiotap_hdr(padapter, rframe->u.hdr.adapter_link,
+					&rframe->u.hdr.attrib, rx_req, (u8 *)pskb) == _FAIL) {
+			ret = _FAIL;
+			goto exit;
+		}
+	}
+
+	/* write skb information to recv frame */
+	skb_reset_mac_header(pskb);
+	rframe->u.hdr.len = pskb->len;
+	rframe->u.hdr.rx_data = pskb->data;
+	rframe->u.hdr.rx_head = pskb->head;
+	rframe->u.hdr.rx_tail = skb_tail_pointer(pskb);
+	rframe->u.hdr.rx_end = skb_end_pointer(pskb);
+
+	if (!RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		/* indicate this recv_frame */
+		ret = rtw_recv_monitor(padapter, rframe);
+	} else
+		ret = _FAIL;
+
+exit:
+#endif /* CONFIG_WIFI_MONITOR */
+
+	return ret;
+}
+#endif
+
+#if 0
+static void recv_set_iseq_before_mpdu_process(union recv_frame *rframe, u16 seq_num, const char *caller)
+{
+#if defined(CONFIG_80211N_HT) && defined(CONFIG_RECV_REORDERING_CTRL)
+	struct recv_reorder_ctrl *reorder_ctrl = rframe->u.hdr.preorder_ctrl;
+
+	if (reorder_ctrl) {
+		reorder_ctrl->indicate_seq = seq_num;
+		#ifdef DBG_RX_SEQ
+		RTW_INFO("DBG_RX_SEQ %s("ADPT_FMT")-B tid:%u indicate_seq:%d, seq_num:%d\n"
+			, caller, ADPT_ARG(reorder_ctrl->padapter)
+			, reorder_ctrl->tid, reorder_ctrl->indicate_seq, seq_num);
+		#endif
+	}
+#endif
+}
+
+static void recv_set_iseq_after_mpdu_process(union recv_frame *rframe, u16 seq_num, const char *caller)
+{
+#if defined(CONFIG_80211N_HT) && defined(CONFIG_RECV_REORDERING_CTRL)
+	struct recv_reorder_ctrl *reorder_ctrl = rframe->u.hdr.preorder_ctrl;
+
+	if (reorder_ctrl) {
+		reorder_ctrl->indicate_seq = (reorder_ctrl->indicate_seq + 1) % 4096;
+		#ifdef DBG_RX_SEQ
+		RTW_INFO("DBG_RX_SEQ %s("ADPT_FMT")-A tid:%u indicate_seq:%d, seq_num:%d\n"
+			, caller, ADPT_ARG(reorder_ctrl->padapter)
+			, reorder_ctrl->tid, reorder_ctrl->indicate_seq, seq_num);
+		#endif
+	}
+#endif
+}
+
+
+#ifdef CONFIG_MP_INCLUDED
+int validate_mp_recv_frame(_adapter *adapter, union recv_frame *precv_frame)
+{
+	int ret = _SUCCESS;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	u8 type, subtype;
+	struct mp_priv *pmppriv = &adapter->mppriv;
+	struct mp_tx		*pmptx;
+	unsigned char	*sa , *da, *bs;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
+	u32 i = 0;
+	u8 rtk_prefix[]={0x52, 0x65, 0x61, 0x6C, 0x4C, 0x6F, 0x76, 0x65, 0x54, 0x65, 0x6B};
+	u8 *prx_data;
+	pmptx = &pmppriv->tx;
+
+
+	if (pmppriv->mplink_brx == _FALSE) {
+
+		u8 bDumpRxPkt = 0;
+		type =  GetFrameType(ptr);
+		subtype = get_frame_sub_type(ptr); /* bit(7)~bit(2)	 */
+
+		RTW_INFO("hdr len = %d iv_len=%d \n", pattrib->hdrlen , pattrib->iv_len);
+		prx_data = ptr + pattrib->hdrlen + pattrib->iv_len;
+
+		for (i = 0; i < precv_frame->u.hdr.len; i++) {
+			if (precv_frame->u.hdr.len < (11 + i))
+				break;
+
+			if (_rtw_memcmp(prx_data + i, (void *)&rtk_prefix, 11) == _FALSE) {
+				bDumpRxPkt = 0;
+				RTW_DBG("prx_data = %02X != rtk_prefix[%d] = %02X \n", *(prx_data + i), i , rtk_prefix[i]);
+				} else {
+				bDumpRxPkt = 1;
+				RTW_DBG("prx_data = %02X = rtk_prefix[%d] = %02X \n", *(prx_data + i), i , rtk_prefix[i]);
+				break;
+				}
+		}
+
+		if (bDumpRxPkt == 1) { /* dump all rx packets */
+			int i;
+			RTW_INFO("############ type:0x%02x subtype:0x%02x #################\n", type, subtype);
+
+			for (i = 0; i < precv_frame->u.hdr.len; i = i + 8)
+				RTW_INFO("%02X:%02X:%02X:%02X:%02X:%02X:%02X:%02X:\n", *(ptr + i),
+					*(ptr + i + 1), *(ptr + i + 2) , *(ptr + i + 3) , *(ptr + i + 4), *(ptr + i + 5), *(ptr + i + 6), *(ptr + i + 7));
+				RTW_INFO("#############################\n");
+				_rtw_memset(pmppriv->mplink_buf, '\0' , sizeof(pmppriv->mplink_buf));
+				_rtw_memcpy(pmppriv->mplink_buf, ptr, precv_frame->u.hdr.len);
+				pmppriv->mplink_rx_len = precv_frame->u.hdr.len;
+				pmppriv->mplink_brx =_TRUE;
+		}
+	}
+	if (pmppriv->bloopback) {
+		if (_rtw_memcmp(ptr + 24, pmptx->buf + 24, precv_frame->u.hdr.len - 24) == _FALSE) {
+			RTW_INFO("Compare payload content Fail !!!\n");
+			ret = _FAIL;
+		}
+	}
+ 	if (pmppriv->bSetRxBssid == _TRUE) {
+
+		sa = get_addr2_ptr(ptr);
+		da = GetAddr1Ptr(ptr);
+		bs = GetAddr3Ptr(ptr);
+		type =	GetFrameType(ptr);
+		subtype = get_frame_sub_type(ptr); /* bit(7)~bit(2)  */
+
+		if (_rtw_memcmp(bs, adapter->mppriv.network_macaddr, ETH_ALEN) == _FALSE)
+			ret = _FAIL;
+
+		RTW_DBG("############ type:0x%02x subtype:0x%02x #################\n", type, subtype);
+		RTW_DBG("A2 sa %02X:%02X:%02X:%02X:%02X:%02X \n", *(sa) , *(sa + 1), *(sa+ 2), *(sa + 3), *(sa + 4), *(sa + 5));
+		RTW_DBG("A1 da %02X:%02X:%02X:%02X:%02X:%02X \n", *(da) , *(da + 1), *(da+ 2), *(da + 3), *(da + 4), *(da + 5));
+		RTW_DBG("A3 bs %02X:%02X:%02X:%02X:%02X:%02X \n --------------------------\n", *(bs) , *(bs + 1), *(bs+ 2), *(bs + 3), *(bs + 4), *(bs + 5));
+	}
+
+	if (!adapter->mppriv.bmac_filter)
+		return ret;
+
+	if (_rtw_memcmp(get_addr2_ptr(ptr), adapter->mppriv.mac_filter, ETH_ALEN) == _FALSE)
+		ret = _FAIL;
+
+	return ret;
+}
+
+static sint MPwlanhdr_to_ethhdr(union recv_frame *precvframe)
+{
+	sint	rmv_len;
+	u16 eth_type, len;
+	u8	bsnaphdr;
+	u8	*psnap_type;
+	u8 mcastheadermac[] = {0x01, 0x00, 0x5e};
+
+	struct ieee80211_snap_hdr	*psnap;
+
+	sint ret = _SUCCESS;
+	_adapter	*adapter = precvframe->u.hdr.adapter;
+
+	u8	*ptr = get_recvframe_data(precvframe) ; /* point to frame_ctrl field */
+	struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;
+
+
+	if (pattrib->encrypt)
+		recvframe_pull_tail(precvframe, pattrib->icv_len);
+
+	psnap = (struct ieee80211_snap_hdr *)(ptr + pattrib->hdrlen + pattrib->iv_len);
+	psnap_type = ptr + pattrib->hdrlen + pattrib->iv_len + SNAP_SIZE;
+	/* convert hdr + possible LLC headers into Ethernet header */
+	/* eth_type = (psnap_type[0] << 8) | psnap_type[1]; */
+	if ((_rtw_memcmp(psnap, rtw_rfc1042_header, SNAP_SIZE) &&
+	     (_rtw_memcmp(psnap_type, SNAP_ETH_TYPE_IPX, 2) == _FALSE) &&
+	     (_rtw_memcmp(psnap_type, SNAP_ETH_TYPE_APPLETALK_AARP, 2) == _FALSE)) ||
+	    /* eth_type != ETH_P_AARP && eth_type != ETH_P_IPX) || */
+	    _rtw_memcmp(psnap, rtw_bridge_tunnel_header, SNAP_SIZE)) {
+		/* remove RFC1042 or Bridge-Tunnel encapsulation and replace EtherType */
+		bsnaphdr = _TRUE;
+	} else {
+		/* Leave Ethernet header part of hdr and full payload */
+		bsnaphdr = _FALSE;
+	}
+
+	rmv_len = pattrib->hdrlen + pattrib->iv_len + (bsnaphdr ? SNAP_SIZE : 0);
+	len = precvframe->u.hdr.len - rmv_len;
+
+
+	_rtw_memcpy(&eth_type, ptr + rmv_len, 2);
+	eth_type = ntohs((unsigned short)eth_type); /* pattrib->ether_type */
+	pattrib->eth_type = eth_type;
+
+	{
+		ptr = recvframe_pull(precvframe, (rmv_len - sizeof(struct ethhdr) + (bsnaphdr ? 2 : 0)));
+	}
+
+	_rtw_memcpy(ptr, pattrib->dst, ETH_ALEN);
+	_rtw_memcpy(ptr + ETH_ALEN, pattrib->src, ETH_ALEN);
+
+	if (!bsnaphdr) {
+		len = htons(len);
+		_rtw_memcpy(ptr + 12, &len, 2);
+	}
+
+
+	len = htons(pattrib->seq_num);
+	/* RTW_INFO("wlan seq = %d ,seq_num =%x\n",len,pattrib->seq_num); */
+	_rtw_memcpy(ptr + 12, &len, 2);
+	if (adapter->mppriv.bRTWSmbCfg == _TRUE) {
+		/* if(_rtw_memcmp(mcastheadermac, pattrib->dst, 3) == _TRUE) */ /* SimpleConfig Dest. */
+		/*			_rtw_memcpy(ptr+ETH_ALEN, pattrib->bssid, ETH_ALEN); */
+
+		if (_rtw_memcmp(mcastheadermac, pattrib->bssid, 3) == _TRUE) /* SimpleConfig Dest. */
+			_rtw_memcpy(ptr, pattrib->bssid, ETH_ALEN);
+
+	}
+
+
+	return ret;
+
+}
+
+int mp_recv_frame(_adapter *padapter, union recv_frame *rframe)
+{
+	int ret = _SUCCESS;
+	struct rx_pkt_attrib *pattrib = &rframe->u.hdr.attrib;
+#ifdef CONFIG_MP_INCLUDED
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mp_priv *pmppriv = &padapter->mppriv;
+#endif /* CONFIG_MP_INCLUDED */
+	u8 type;
+	u8 *ptr = rframe->u.hdr.rx_data;
+	u8 *psa, *pda, *pbssid;
+	struct sta_info *psta = NULL;
+	DBG_COUNTER(padapter->rx_logs.core_rx_pre);
+
+	if ((check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)) { /* &&(padapter->mppriv.check_mp_pkt == 0)) */
+		if (pattrib->crc_err == 1)
+			padapter->mppriv.rx_crcerrpktcount++;
+		else {
+			if (_SUCCESS == validate_mp_recv_frame(padapter, rframe))
+				padapter->mppriv.rx_pktcount++;
+			else
+				padapter->mppriv.rx_pktcount_filter_out++;
+		}
+
+		if (pmppriv->rx_bindicatePkt == _FALSE) {
+			ret = _FAIL;
+			rtw_free_recvframe(rframe);/* free this recv_frame */
+			goto exit;
+		} else {
+			type =	GetFrameType(ptr);
+			pattrib->to_fr_ds = get_tofr_ds(ptr);
+			pattrib->frag_num = GetFragNum(ptr);
+			pattrib->seq_num = GetSequence(ptr);
+			pattrib->pw_save = GetPwrMgt(ptr);
+			pattrib->mfrag = GetMFrag(ptr);
+			pattrib->mdata = GetMData(ptr);
+			pattrib->privacy = GetPrivacy(ptr);
+			pattrib->order = GetOrder(ptr);
+
+			if (type == WIFI_DATA_TYPE) {
+				pda = get_da(ptr);
+				psa = get_sa(ptr);
+				pbssid = get_hdr_bssid(ptr);
+
+				_rtw_memcpy(pattrib->dst, pda, ETH_ALEN);
+				_rtw_memcpy(pattrib->src, psa, ETH_ALEN);
+				_rtw_memcpy(pattrib->bssid, pbssid, ETH_ALEN);
+
+				switch (pattrib->to_fr_ds) {
+				case 0:
+					_rtw_memcpy(pattrib->ra, pda, ETH_ALEN);
+					_rtw_memcpy(pattrib->ta, psa, ETH_ALEN);
+					ret = sta2sta_data_frame(padapter, rframe, &psta);
+					break;
+
+				case 1:
+					_rtw_memcpy(pattrib->ra, pda, ETH_ALEN);
+					_rtw_memcpy(pattrib->ta, pbssid, ETH_ALEN);
+					ret = ap2sta_data_frame(padapter, rframe, &psta);
+					break;
+
+				case 2:
+					_rtw_memcpy(pattrib->ra, pbssid, ETH_ALEN);
+					_rtw_memcpy(pattrib->ta, psa, ETH_ALEN);
+					ret = sta2ap_data_frame(padapter, rframe, &psta);
+					break;
+				case 3:
+					_rtw_memcpy(pattrib->ra, GetAddr1Ptr(ptr), ETH_ALEN);
+					_rtw_memcpy(pattrib->ta, get_addr2_ptr(ptr), ETH_ALEN);
+					ret = _FAIL;
+					break;
+				default:
+					ret = _FAIL;
+					break;
+				}
+
+				if (ret != _SUCCESS) {
+#ifdef DBG_RX_DROP_FRAME
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" 2_data_frame fail: drop pkt\n"
+															, FUNC_ADPT_ARG(padapter));
+#endif
+					ret = _FAIL;
+					goto exit;
+				}
+
+				ret = MPwlanhdr_to_ethhdr(rframe);
+
+				if (ret != _SUCCESS) {
+					#ifdef DBG_RX_DROP_FRAME
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" wlanhdr_to_ethhdr: drop pkt\n"
+						, FUNC_ADPT_ARG(padapter));
+					#endif
+					ret = _FAIL;
+					goto exit;
+				}
+				if (!RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+					/* indicate this recv_frame */
+					ret = rtw_recv_indicatepkt(padapter, rframe);
+					if (ret != _SUCCESS) {
+						#ifdef DBG_RX_DROP_FRAME
+						RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" rtw_recv_indicatepkt fail!\n"
+							, FUNC_ADPT_ARG(padapter));
+						#endif
+						ret = _FAIL;
+						goto exit;
+					}
+				} else {
+					#ifdef DBG_RX_DROP_FRAME
+					RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" bDriverStopped(%s) OR bSurpriseRemoved(%s)\n"
+						, FUNC_ADPT_ARG(padapter)
+						, dev_is_drv_stopped(adapter_to_dvobj(padapter)) ? "True" : "False"
+						, dev_is_surprise_removed(adapter_to_dvobj(padapter)) ? "True" : "False");
+					#endif
+					ret = _FAIL;
+					goto exit;
+				}
+
+			}
+		}
+	}
+exit:
+	rtw_free_recvframe(rframee);/* free this recv_frame */
+	ret = _FAIL;
+	return ret;
+
+}
+#endif
+
+int recv_func_prehandle(_adapter *padapter, union recv_frame *rframe)
+{
+	int ret = _SUCCESS;
+#ifdef DBG_RX_COUNTER_DUMP
+	struct rx_pkt_attrib *pattrib = &rframe->u.hdr.attrib;
+#endif
+	_queue *pfree_recv_queue = &padapter->recvpriv.free_recv_queue;
+
+#ifdef DBG_RX_COUNTER_DUMP
+	if (padapter->dump_rx_cnt_mode & DUMP_DRV_RX_COUNTER) {
+		if (pattrib->crc_err == 1)
+			padapter->drv_rx_cnt_crcerror++;
+		else
+			padapter->drv_rx_cnt_ok++;
+	}
+#endif
+
+#ifdef CONFIG_MP_INCLUDED
+	if (padapter->registrypriv.mp_mode == 1 || padapter->mppriv.bRTWSmbCfg == _TRUE) {
+		mp_recv_frame(padapter, rframe);
+		ret = _FAIL;
+		goto exit;
+	} else
+#endif
+	{
+		/* check the frame crtl field and decache */
+		ret = validate_recv_frame(padapter, &rframe);
+		if (ret != _SUCCESS) {
+			rtw_free_recvframe(rframee);/* free this recv_frame */
+			goto exit;
+		}
+	}
+exit:
+	return ret;
+}
+
+/*#define DBG_RX_BMC_FRAME*/
+int recv_func_posthandle(_adapter *padapter, union recv_frame *prframe)
+{
+	int ret = _SUCCESS;
+	union recv_frame *orig_prframe = prframe;
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	struct recv_info *precvinfo = &padapter->recvinfo;
+	_queue *pfree_recv_queue = &padapter->recvpriv.free_recv_queue;
+#ifdef CONFIG_TDLS
+	u8 *psnap_type, *pcategory;
+#endif /* CONFIG_TDLS */
+
+	DBG_COUNTER(padapter->rx_logs.core_rx_post);
+
+	prframe = decryptor(padapter, prframe);
+	if (prframe == NULL) {
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" decryptor: drop pkt\n"
+			, FUNC_ADPT_ARG(padapter));
+		#endif
+		ret = _FAIL;
+		DBG_COUNTER(padapter->rx_logs.core_rx_post_decrypt_err);
+		goto _recv_data_drop;
+	}
+
+#ifdef DBG_RX_BMC_FRAME
+	if (IS_MCAST(pattrib->ra))
+		RTW_INFO("%s =>"ADPT_FMT" Rx BC/MC from "MAC_FMT"\n", __func__, ADPT_ARG(padapter), MAC_ARG(pattrib->ta));
+#endif
+
+#if 0
+	if (is_primary_adapter(padapter)) {
+		RTW_INFO("+++\n");
+		{
+			int i;
+			u8	*ptr = get_recvframe_data(prframe);
+			for (i = 0; i < 140; i = i + 8)
+				RTW_INFO("%02X:%02X:%02X:%02X:%02X:%02X:%02X:%02X:", *(ptr + i),
+					*(ptr + i + 1), *(ptr + i + 2) , *(ptr + i + 3) , *(ptr + i + 4), *(ptr + i + 5), *(ptr + i + 6), *(ptr + i + 7));
+
+		}
+		RTW_INFO("---\n");
+	}
+#endif
+
+#ifdef CONFIG_TDLS
+	/* check TDLS frame */
+	psnap_type = get_recvframe_data(orig_prframe) + pattrib->hdrlen + pattrib->iv_len + SNAP_SIZE;
+	pcategory = psnap_type + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+
+	if ((_rtw_memcmp(psnap_type, SNAP_ETH_TYPE_TDLS, ETH_TYPE_LEN)) &&
+	    ((*pcategory == RTW_WLAN_CATEGORY_TDLS) || (*pcategory == RTW_WLAN_CATEGORY_P2P))) {
+		ret = OnTDLS(padapter, prframe);
+		if (ret == _FAIL)
+			goto _exit_recv_func;
+	}
+#endif /* CONFIG_TDLS */
+
+	prframe = recvframe_chk_defrag(padapter, prframe);
+	if (prframe == NULL)	{
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recvframe_chk_defrag: drop pkt\n"
+			, FUNC_ADPT_ARG(padapter));
+		#endif
+		DBG_COUNTER(padapter->rx_logs.core_rx_post_defrag_err);
+		goto _recv_data_drop;
+	}
+
+	prframe = portctrl(padapter, prframe);
+	if (prframe == NULL) {
+		#ifdef DBG_RX_DROP_FRAME
+		RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" portctrl: drop pkt\n"
+			, FUNC_ADPT_ARG(padapter));
+		#endif
+		ret = _FAIL;
+		DBG_COUNTER(padapter->rx_logs.core_rx_post_portctrl_err);
+		goto _recv_data_drop;
+	}
+
+	count_rx_stats(padapter, prframe, NULL);
+
+#ifdef CONFIG_WAPI_SUPPORT
+	rtw_wapi_update_info(padapter, prframe);
+#endif
+
+#if defined(CONFIG_80211N_HT) && defined(CONFIG_RECV_REORDERING_CTRL)
+	/* including perform A-MPDU Rx Ordering Buffer Control */
+	ret = recv_indicatepkt_reorder(padapter, prframe);
+	if (ret == _FAIL) {
+		rtw_free_recvframe(orig_prframe);
+		goto _recv_data_drop;
+	} else if (ret == RTW_RX_HANDLED) /* queued OR indicated in order */
+		goto _exit_recv_func;
+#endif
+
+	recv_set_iseq_before_mpdu_process(prframe, pattrib->seq_num, __func__);
+	ret = recv_process_mpdu(padapter, prframe);
+	recv_set_iseq_after_mpdu_process(prframe, pattrib->seq_num, __func__);
+	if (ret == _FAIL)
+		goto _recv_data_drop;
+
+_exit_recv_func:
+	return ret;
+
+_recv_data_drop:
+	precvinfo->dbg_rx_drop_count++;
+	return ret;
+}
+
+int recv_func(_adapter *padapter, union recv_frame *rframe)
+{
+	int ret;
+	struct rx_pkt_attrib *prxattrib = &rframe->u.hdr.attrib;
+	struct recv_priv *recvpriv = &padapter->recvpriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct mlme_priv *mlmepriv = &padapter->mlmepriv;
+#ifdef CONFIG_CUSTOMER_ALIBABA_GENERAL
+	u8 type;
+	u8 *ptr = rframe->u.hdr.rx_data;
+#endif
+
+	if (check_fwstate(mlmepriv, WIFI_MONITOR_STATE)) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+		recv_frame_monitor(padapter, rframe);
+#endif
+		ret = _SUCCESS;
+		goto exit;
+	}
+
+#ifdef CONFIG_CUSTOMER_ALIBABA_GENERAL
+	type = GetFrameType(ptr);
+	if ((type == WIFI_DATA_TYPE)&& MLME_IS_STA(padapter)) {
+		struct wlan_network *cur_network = &(mlmepriv->cur_network);
+		if ( _rtw_memcmp(get_addr2_ptr(ptr), cur_network->network.MacAddress, ETH_ALEN)==0) {
+			recv_frame_monitor(padapter, rframe);
+			ret = _SUCCESS;
+			goto exit;
+		}
+	}
+#endif
+	/* check if need to handle uc_swdec_pending_queue*/
+	if (MLME_IS_STA(padapter) && psecuritypriv->busetkipkey) {
+		union recv_frame *pending_frame;
+		int cnt = 0;
+
+		while ((pending_frame = rtw_alloc_recvframe(&recvpriv->uc_swdec_pending_queue))) {
+			cnt++;
+			DBG_COUNTER(padapter->rx_logs.core_rx_dequeue);
+			recv_func_posthandle(padapter, pending_frame);
+		}
+
+		if (cnt)
+			RTW_INFO(FUNC_ADPT_FMT" dequeue %d from uc_swdec_pending_queue\n",
+				 FUNC_ADPT_ARG(padapter), cnt);
+	}
+
+	DBG_COUNTER(padapter->rx_logs.core_rx);
+	ret = recv_func_prehandle(padapter, rframe);
+
+	if (ret == _SUCCESS) {
+
+		/* check if need to enqueue into uc_swdec_pending_queue*/
+		if (MLME_IS_STA(padapter) &&
+		    !IS_MCAST(prxattrib->ra) && prxattrib->encrypt > 0 &&
+		    (prxattrib->bdecrypted == 0 || psecuritypriv->sw_decrypt == _TRUE) &&
+		    psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPAPSK &&
+		    !psecuritypriv->busetkipkey) {
+			DBG_COUNTER(padapter->rx_logs.core_rx_enqueue);
+			rtw_enqueue_recvframe(rframe, &recvpriv->uc_swdec_pending_queue);
+			/* RTW_INFO("%s: no key, enqueue uc_swdec_pending_queue\n", __func__); */
+
+			if (recvpriv->free_recvframe_cnt < NR_RECVFRAME / 4) {
+				/* to prevent from recvframe starvation, get recvframe from uc_swdec_pending_queue to free_recvframe_cnt */
+				rframe = rtw_alloc_recvframe(&padapter->recvpriv.uc_swdec_pending_queue);
+				if (rframe)
+					goto do_posthandle;
+			}
+			goto exit;
+		}
+
+do_posthandle:
+		ret = recv_func_posthandle(padapter, rframe);
+	}
+
+exit:
+	return ret;
+}
+
+s32 rtw_recv_entry(union recv_frame *precvframe)
+{
+	_adapter *padapter;
+	struct recv_info *precvinfo;
+	s32 ret = _SUCCESS;
+
+
+
+	padapter = precvframe->u.hdr.adapter;
+
+	precvinfo = &padapter->recvinfo;
+
+
+	ret = recv_func(padapter, precvframe);
+	if (ret == _FAIL) {
+		goto _recv_entry_drop;
+	}
+
+
+	precvinfo->rx_pkts++;
+
+
+	return ret;
+
+_recv_entry_drop:
+
+#ifdef CONFIG_MP_INCLUDED
+	if (padapter->registrypriv.mp_mode == 1)
+		padapter->mppriv.rx_pktloss = precvinfo->rx_drop;
+#endif
+
+
+
+	return ret;
+}
+#endif
+
+#ifdef CONFIG_SIGNAL_STAT_PROCESS
+static void rtw_signal_stat_timer_hdl(void *ctx)
+{
+	_adapter *adapter = (_adapter *)ctx;
+	struct recv_info *recvinfo = &adapter->recvinfo;
+	/* ToDo CONFIG_RTW_MLD: currently primary link only */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	u32 tmp_s, tmp_q;
+	u8 avg_signal_strength = 0;
+	u8 avg_signal_qual = 0;
+	u32 num_signal_strength = 0;
+	u32 num_signal_qual = 0;
+	u8 ratio_pre_stat = 0, ratio_curr_stat = 0, ratio_total = 0, ratio_profile = SIGNAL_STAT_CALC_PROFILE_0;
+
+	if (recvinfo->is_signal_dbg) {
+		/* update the user specific value, signal_strength_dbg, to signal_strength, rssi */
+		recvinfo->signal_strength = recvinfo->signal_strength_dbg;
+		recvinfo->rssi = (s8)rtw_phl_rssi_to_dbm((u8)recvinfo->signal_strength_dbg);
+	} else {
+
+		if (recvinfo->signal_strength_data.update_req == 0) { /* update_req is clear, means we got rx */
+			avg_signal_strength = recvinfo->signal_strength_data.avg_val;
+			num_signal_strength = recvinfo->signal_strength_data.total_num;
+			/* after avg_vals are accquired, we can re-stat the signal values */
+			recvinfo->signal_strength_data.update_req = 1;
+		}
+
+		if (recvinfo->signal_qual_data.update_req == 0) { /* update_req is clear, means we got rx */
+			avg_signal_qual = recvinfo->signal_qual_data.avg_val;
+			num_signal_qual = recvinfo->signal_qual_data.total_num;
+			/* after avg_vals are accquired, we can re-stat the signal values */
+			recvinfo->signal_qual_data.update_req = 1;
+		}
+
+		if (num_signal_strength == 0) {
+			if (rtw_get_on_cur_ch_time(adapter) == 0
+			    || rtw_get_passing_time_ms(rtw_get_on_cur_ch_time(adapter)) < 2 * adapter_link->mlmeextpriv.mlmext_info.bcn_interval
+			   )
+				goto set_timer;
+		}
+
+		if (check_fwstate(&adapter->mlmepriv, WIFI_UNDER_SURVEY) == _TRUE
+		    || check_fwstate(&adapter->mlmepriv, WIFI_ASOC_STATE) == _FALSE
+		   )
+			goto set_timer;
+
+#ifdef CONFIG_CONCURRENT_MODE
+		if (rtw_mi_buddy_check_fwstate(adapter, WIFI_UNDER_SURVEY) == _TRUE)
+			goto set_timer;
+#endif
+
+#ifdef CONFIG_MP_INCLUDED
+		if (adapter->registrypriv.mp_mode == 1) {
+			ratio_profile = SIGNAL_STAT_CALC_PROFILE_2;
+		}
+		else
+#endif
+		if (RTW_SIGNAL_STATE_CALC_PROFILE < SIGNAL_STAT_CALC_PROFILE_MAX)
+			ratio_profile = RTW_SIGNAL_STATE_CALC_PROFILE;
+
+		ratio_pre_stat = signal_stat_calc_profile[ratio_profile][0];
+		ratio_curr_stat = signal_stat_calc_profile[ratio_profile][1];
+		ratio_total = ratio_pre_stat + ratio_curr_stat;
+
+		/* update value of signal_strength, rssi, signal_qual */
+		tmp_s = (ratio_curr_stat * avg_signal_strength + ratio_pre_stat * recvinfo->signal_strength);
+		if (tmp_s % ratio_total)
+			tmp_s = tmp_s / ratio_total + 1;
+		else
+			tmp_s = tmp_s / ratio_total;
+		if (tmp_s > PHL_MAX_RSSI)
+			tmp_s = PHL_MAX_RSSI;
+
+		tmp_q = (ratio_curr_stat * avg_signal_qual + ratio_pre_stat * recvinfo->signal_qual);
+		if (tmp_q % ratio_total)
+			tmp_q = tmp_q / ratio_total + 1;
+		else
+			tmp_q = tmp_q / ratio_total;
+		if (tmp_q > PHL_MAX_RSSI)
+			tmp_q = PHL_MAX_RSSI;
+
+		recvinfo->signal_strength = tmp_s;
+		recvinfo->rssi = (s8)rtw_phl_rssi_to_dbm(tmp_s);
+		recvinfo->signal_qual = tmp_q;
+
+#if defined(DBG_RX_SIGNAL_DISPLAY_PROCESSING) && 1
+		RTW_INFO(FUNC_ADPT_FMT" signal_strength:%3u, rssi:%3d, signal_qual:%3u"
+			 ", num_signal_strength:%u, num_signal_qual:%u"
+			 ", on_cur_ch_ms:%d"
+			 "\n"
+			 , FUNC_ADPT_ARG(adapter)
+			 , recvinfo->signal_strength
+			 , recvinfo->rssi
+			 , recvinfo->signal_qual
+			 , num_signal_strength, num_signal_qual
+			, rtw_get_on_cur_ch_time(adapter) ? rtw_get_passing_time_ms(rtw_get_on_cur_ch_time(adapter)) : 0
+			);
+#endif
+	}
+
+set_timer:
+	rtw_set_signal_stat_timer(recvinfo);
+
+}
+#endif/*CONFIG_SIGNAL_STAT_PROCESS*/
+
+/*
+* Increase and check if the continual_no_rx_packet of this @param pmlmepriv is larger than MAX_CONTINUAL_NORXPACKET_COUNT
+* @return _TRUE:
+* @return _FALSE:
+*/
+int rtw_inc_and_chk_continual_no_rx_packet(struct sta_info *sta, int tid_index)
+{
+
+	int ret = _FALSE;
+	int value = ATOMIC_INC_RETURN(&sta->continual_no_rx_packet[tid_index]);
+
+	if (value >= MAX_CONTINUAL_NORXPACKET_COUNT)
+		ret = _TRUE;
+
+	return ret;
+}
+
+/*
+* Set the continual_no_rx_packet of this @param pmlmepriv to 0
+*/
+void rtw_reset_continual_no_rx_packet(struct sta_info *sta, int tid_index)
+{
+	ATOMIC_SET(&sta->continual_no_rx_packet[tid_index], 0);
+}
+
+u8 adapter_allow_bmc_data_rx(_adapter *adapter)
+{
+	if (check_fwstate(&adapter->mlmepriv, WIFI_MONITOR_STATE | WIFI_MP_STATE) == _TRUE)
+		return 1;
+
+	if (MLME_IS_AP(adapter))
+		return 0;
+
+	if (rtw_linked_check(adapter) == _FALSE)
+		return 0;
+
+	return 1;
+}
+#if 0
+s32 pre_recv_entry(union recv_frame *precvframe, u8 *pphy_status)
+{
+	s32 ret = _SUCCESS;
+	u8 *pbuf = precvframe->u.hdr.rx_data;
+	u8 *pda = get_ra(pbuf);
+	u8 ra_is_bmc = IS_MCAST(pda);
+	_adapter *primary_padapter = precvframe->u.hdr.adapter;
+#ifdef CONFIG_CONCURRENT_MODE
+	_adapter *iface = NULL;
+
+	#ifdef CONFIG_MP_INCLUDED
+	if (rtw_mp_mode_check(primary_padapter))
+		goto bypass_concurrent_hdl;
+	#endif
+
+	if (ra_is_bmc == _FALSE) { /*unicast packets*/
+		iface = rtw_get_iface_by_macddr(primary_padapter , pda);
+		if (NULL == iface) {
+			#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+			if (_rtw_memcmp(pda, adapter_pno_mac_addr(primary_padapter),
+					ETH_ALEN) != _TRUE)
+			#endif
+			RTW_INFO("%s [WARN] Cannot find appropriate adapter - mac_addr : "MAC_FMT"\n", __func__, MAC_ARG(pda));
+			/*rtw_warn_on(1);*/
+		} else {
+			precvframe->u.hdr.adapter = iface;
+		}
+	} else { /* Handle BC/MC Packets*/
+		rtw_mi_buddy_clone_bcmc_packet(primary_padapter, precvframe, pphy_status);
+	}
+#ifdef CONFIG_MP_INCLUDED
+bypass_concurrent_hdl:
+#endif
+#endif /* CONFIG_CONCURRENT_MODE */
+	if (primary_padapter->registrypriv.mp_mode != 1) {
+		/* skip unnecessary bmc data frame for primary adapter */
+		if (ra_is_bmc == _TRUE && GetFrameType(pbuf) == WIFI_DATA_TYPE
+			&& !adapter_allow_bmc_data_rx(precvframe->u.hdr.adapter)
+		) {
+			rtw_free_recvframe(precvframe);
+			goto exit;
+		}
+	}
+
+	if (pphy_status) {
+		rx_query_phy_status(precvframe, pphy_status);
+
+#ifdef CONFIG_WIFI_MONITOR
+		rx_query_moinfo(&precvframe->u.hdr.attrib, pphy_status);
+#endif
+	}
+	ret = rtw_recv_entry(precvframe);
+
+exit:
+	return ret;
+}
+#endif
+
+
+#ifdef CONFIG_RECV_THREAD_MODE
+thread_return rtw_recv_thread(thread_context context)
+{
+	_adapter *adapter = (_adapter *)context;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct recv_priv *recvpriv = &dvobj->recvpriv;
+	s32 err = _SUCCESS;
+#ifdef RTW_RECV_THREAD_HIGH_PRIORITY
+#ifdef PLATFORM_LINUX
+	struct sched_param param = { .sched_priority = 1 };
+
+	sched_setscheduler(current, SCHED_FIFO, &param);
+#endif /* PLATFORM_LINUX */
+#endif /*RTW_RECV_THREAD_HIGH_PRIORITY*/
+	rtw_thread_enter("RTW_RECV_THREAD");
+
+	RTW_INFO(FUNC_ADPT_FMT" enter\n", FUNC_ADPT_ARG(adapter));
+
+	do {
+		err = _rtw_down_sema(&recvpriv->recv_sema);
+		if (_FAIL == err) {
+			RTW_ERR(FUNC_ADPT_FMT" down recv_sema fail!\n", FUNC_ADPT_ARG(adapter));
+			goto exit;
+		}
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(adapter))) {
+			RTW_DBG(FUNC_ADPT_FMT "- bDriverStopped(%s) bSurpriseRemoved(%s)\n",
+				FUNC_ADPT_ARG(adapter),
+				dev_is_drv_stopped(adapter_to_dvobj(adapter)) ? "True" : "False",
+				dev_is_surprise_removed(adapter_to_dvobj(adapter)) ? "True" : "False");
+			goto exit;
+		}
+
+		err = rtw_intf_recv_hdl(adapter);
+
+		if (err == RTW_RFRAME_UNAVAIL
+			|| err == RTW_RFRAME_PKT_UNAVAIL
+		) {
+			rtw_msleep_os(1);
+			_rtw_up_sema(&recvpriv->recv_sema);
+		}
+
+		flush_signals_thread();
+
+	} while (err != _FAIL);
+
+exit:
+
+	RTW_INFO(FUNC_ADPT_FMT " Exit\n", FUNC_ADPT_ARG(adapter));
+
+	rtw_thread_wait_stop();
+
+	return 0;
+}
+#endif /* CONFIG_RECV_THREAD_MODE */
+
+#ifdef CONFIG_USB_HCI
+u8 rtw_init_lite_recv_resource(struct dvobj_priv *dvobj)
+{
+	u8 ret = _SUCCESS;
+	u32 literecvbuf_nr = RTW_LITERECVBUF_NR;
+	struct lite_data_buf *literecvbuf;
+	struct trx_data_buf_q  *literecvbuf_q = &dvobj->literecvbuf_q;
+	int i;
+	struct data_urb *recvurb;
+	struct trx_urb_buf_q *recv_urb_q = &dvobj->recv_urb_q;
+	u32 recvurb_nr = RTW_RECVURB_NR;
+#ifdef CONFIG_USB_INTERRUPT_IN_PIPE
+	struct lite_data_buf *intinbuf;
+	struct trx_data_buf_q  *intin_buf_q = &dvobj->intin_buf_q;
+	u32 intin_buf_nr = RTW_INTINBUF_NR;
+	struct data_urb *intin_urb;
+	struct trx_urb_buf_q *intin_urb_q = &dvobj->intin_urb_q;
+	u32 intin_urb_nr = RTW_INTINURB_NR;
+#endif 
+
+	/* init lite_recv_buf */
+	_rtw_init_queue(&literecvbuf_q->free_data_buf_queue);
+
+	literecvbuf_q->alloc_data_buf =
+		rtw_zvmalloc(literecvbuf_nr * sizeof(struct lite_data_buf) + 4);
+
+	if (literecvbuf_q->alloc_data_buf  == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	literecvbuf_q->data_buf=
+	(u8 *)N_BYTE_ALIGNMENT((SIZE_PTR)(literecvbuf_q->alloc_data_buf), 4);
+
+	literecvbuf = (struct lite_data_buf *)literecvbuf_q->data_buf;
+
+	for (i = 0; i < literecvbuf_nr; i++) {
+		_rtw_init_listhead(&literecvbuf->list);
+		rtw_list_insert_tail(&literecvbuf->list,
+			&(literecvbuf_q->free_data_buf_queue.queue));
+		literecvbuf++;
+	}
+	literecvbuf_q->free_data_buf_cnt = literecvbuf_nr;
+
+	/* init recv_urb */
+	_rtw_init_queue(&recv_urb_q->free_urb_buf_queue);
+	recv_urb_q->alloc_urb_buf=
+		rtw_zvmalloc(recvurb_nr * sizeof(struct data_urb) + 4);
+	if (recv_urb_q->alloc_urb_buf== NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	recv_urb_q->urb_buf =
+		(u8 *)N_BYTE_ALIGNMENT((SIZE_PTR)(recv_urb_q->alloc_urb_buf), 4);
+
+	recvurb = (struct data_urb *)recv_urb_q->urb_buf;
+	for (i = 0; i < recvurb_nr; i++) {
+		_rtw_init_listhead(&recvurb->list);
+		ret = rtw_os_urb_resource_alloc(recvurb);
+		rtw_list_insert_tail(&recvurb->list,
+			&(recv_urb_q->free_urb_buf_queue.queue));
+		recvurb++;
+	}
+	recv_urb_q->free_urb_buf_cnt = recvurb_nr;
+	ATOMIC_SET(&(dvobj->rx_pending_cnt), 0);
+
+#ifdef CONFIG_USB_INTERRUPT_IN_PIPE
+
+	/* init int_in_buf */
+	_rtw_init_queue(&intin_buf_q->free_data_buf_queue);
+
+	intin_buf_q->alloc_data_buf =
+		rtw_zvmalloc(intin_buf_nr * sizeof(struct lite_data_buf) + 4);
+
+	if (intin_buf_q->alloc_data_buf  == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	intin_buf_q->data_buf=
+	(u8 *)N_BYTE_ALIGNMENT((SIZE_PTR)(intin_buf_q->alloc_data_buf), 4);
+
+	intinbuf = (struct lite_data_buf *)intin_buf_q->data_buf;
+
+	for (i = 0; i < intin_buf_nr; i++) {
+		_rtw_init_listhead(&intinbuf->list);
+		rtw_list_insert_tail(&intinbuf->list,
+			&(intin_buf_q->free_data_buf_queue.queue));
+		intinbuf++;
+	}
+	intin_buf_q->free_data_buf_cnt = intin_buf_nr;
+
+	/* init int_in_urb */
+	_rtw_init_queue(&intin_urb_q->free_urb_buf_queue);
+	intin_urb_q->alloc_urb_buf=
+		rtw_zvmalloc(intin_urb_nr * sizeof(struct data_urb) + 4);
+	if (intin_urb_q->alloc_urb_buf== NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	intin_urb_q->urb_buf =
+		(u8 *)N_BYTE_ALIGNMENT((SIZE_PTR)(intin_urb_q->alloc_urb_buf), 4);
+
+	intin_urb = (struct data_urb *)intin_urb_q->urb_buf;
+	for (i = 0; i < intin_urb_nr; i++) {
+		_rtw_init_listhead(&intin_urb->list);
+		ret = rtw_os_urb_resource_alloc(intin_urb);
+		rtw_list_insert_tail(&intin_urb->list,
+			&(intin_urb_q->free_urb_buf_queue.queue));
+		intin_urb++;
+	}
+	intin_urb_q->free_urb_buf_cnt = intin_urb_nr;
+#endif
+
+exit:
+	return ret;
+}
+
+void rtw_free_lite_recv_resource(struct dvobj_priv *dvobj)
+{
+	u8 ret = _SUCCESS;
+	u32 literecvbuf_nr = RTW_LITERECVBUF_NR;
+	struct lite_data_buf *literecvbuf;
+	struct trx_data_buf_q  *literecvbuf_q = &dvobj->literecvbuf_q;
+	int i;
+	struct data_urb *recvurb;
+	struct trx_urb_buf_q *recv_urb_q = &dvobj->recv_urb_q;
+	u32 recvurb_nr = RTW_RECVURB_NR;
+#ifdef CONFIG_USB_INTERRUPT_IN_PIPE
+	struct lite_data_buf *intinbuf;
+	struct trx_data_buf_q *intin_buf_q = &dvobj->intin_buf_q;
+	u32 intin_buf_nr = RTW_INTINBUF_NR;
+	struct data_urb *intin_urb;
+	struct trx_urb_buf_q *intin_urb_q = &dvobj->intin_urb_q;
+	u32 intin_urb_nr = RTW_INTINURB_NR;
+#endif 
+
+	if (literecvbuf_q->alloc_data_buf)
+		rtw_vmfree(literecvbuf_q->alloc_data_buf,
+			literecvbuf_nr * sizeof(struct lite_data_buf) + 4);
+
+	recvurb = (struct data_urb *)recv_urb_q->urb_buf;
+	for (i = 0; i < recvurb_nr; i++) {
+		rtw_os_urb_resource_free(recvurb);
+		recvurb++;
+	}
+
+	if (recv_urb_q->alloc_urb_buf)
+		rtw_vmfree(recv_urb_q->alloc_urb_buf,
+			recvurb_nr * sizeof(struct data_urb) + 4);
+
+#ifdef CONFIG_USB_INTERRUPT_IN_PIPE
+	if (intin_buf_q->alloc_data_buf)
+		rtw_vmfree(intin_buf_q->alloc_data_buf,
+			intin_buf_nr * sizeof(struct lite_data_buf) + 4);
+
+	intin_urb = (struct data_urb *)intin_urb_q->urb_buf;
+	for (i = 0; i < intin_urb_nr; i++) {
+		rtw_os_urb_resource_free(intin_urb);
+		intin_urb++;
+	}
+
+	if (intin_urb_q->alloc_urb_buf)
+		rtw_vmfree(intin_urb_q->alloc_urb_buf,
+			intin_urb_nr * sizeof(struct data_urb) + 4);
+#endif
+}
+#endif
+
+#ifdef RTW_PHL_RX
+void rx_dump_skb(struct sk_buff *skb)
+{
+	int idx=0;
+	u8 *tmp=skb->data;
+	printk("===");
+	printk("[%s]skb=%p len=%d\n", __FUNCTION__, skb, skb->len);
+	
+#if 0	
+	printk("data-tail=0x%x-0x%x(%d)\n", 
+		skb->data, skb->tail, (skb->tail - skb->data));
+	printk("head-end=0x%x-0x%x(%d)\n", 
+		skb->head, skb->end, (skb->end - skb->head));
+#endif
+	
+	for(idx=0; idx<skb->len; idx++){
+		printk("%02x ", tmp[idx]);
+		if(idx%20==19)
+			printk("\n");
+	}
+	printk("\n===\n");
+}
+
+void dump_rxreq(_adapter *adapter, union recv_frame *prframe)
+{
+
+
+}
+
+void dump_recv_frame(_adapter *adapter, union recv_frame *prframe)
+{
+	struct recv_frame_hdr *hdr = &(prframe->u.hdr);
+	struct rx_pkt_attrib *rxattr = &(prframe->u.hdr.attrib);
+	
+	printk("[%s]prframe=0x%p len=%d\n", __FUNCTION__, prframe, hdr->len);
+
+	printk("head-tail=0x%p-0x%p\n", hdr->rx_head, hdr->rx_tail);
+	printk("data-end=0x%p-0x%p\n", hdr->rx_data, hdr->rx_end);
+
+	printk("dst=%pM\n", rxattr->dst);
+	printk("src=%pM\n", rxattr->src);
+	printk("ra=%pM\n", rxattr->ra);
+	printk("ta=%pM\n", rxattr->ta);
+	printk("bssid=%pM\n", rxattr->bssid);
+}
+
+void core_update_recvframe_pkt( union recv_frame *prframe, struct rtw_recv_pkt *rx_req)
+{
+	struct rtw_pkt_buf_list *pkt = rx_req->pkt_list;
+	struct sk_buff *skb = prframe->u.hdr.pkt;
+
+	skb_reserve(skb, pkt->vir_addr - skb->data);
+	skb_put(skb, pkt->length);
+
+	prframe->u.hdr.rx_data = skb->data;
+	prframe->u.hdr.rx_tail = skb_tail_pointer(skb);
+	prframe->u.hdr.len = skb->len;
+	prframe->u.hdr.rx_head = skb->head;
+	prframe->u.hdr.rx_end = skb_end_pointer(skb);
+
+	return;
+}
+
+static int core_alloc_recvframe_pkt(union recv_frame *prframe,
+				    struct rtw_recv_pkt *phlrx)
+{
+	struct rtw_pkt_buf_list *pktbuf;
+	u8 shift_sz;
+	u32 alloc_sz;
+	struct sk_buff *pkt = NULL;
+	u8 *pbuf = NULL;
+
+
+	rtw_warn_on(phlrx->pkt_cnt > 1);
+	pktbuf = phlrx->pkt_list; /* &phlrx->pkt_list[0] */
+
+	/* For 8 bytes IP header alignment. */
+	if (phlrx->mdata.qos)
+		/* Qos data, wireless lan header length is 26 */
+		shift_sz = 6;
+	else
+		shift_sz = 0;
+
+	/*
+	 * For first fragment packet, driver need allocate 1536 to
+	 * defrag packet.
+	 * And need 8 is for skb->data 8 bytes alignment.
+	 * Round (1536 + shift_sz + 8) to 128 bytes alignment,
+	 * and finally get 1664.
+	 */
+	alloc_sz = pktbuf->length;
+	if ((phlrx->mdata.more_frag == 1) && (phlrx->mdata.frag_num == 0)) {
+		if (alloc_sz <= 1650)
+			alloc_sz = 1664;
+		else
+			alloc_sz += 14;
+	} else {
+		/*
+		 * 6 is for IP header 8 bytes alignment in QoS packet case.
+		 * 8 is for skb->data 4 bytes alignment.
+		 */
+		alloc_sz += 14;
+	}
+
+	pkt = rtw_skb_alloc(alloc_sz);
+	if (!pkt) {
+		RTW_ERR("%s: alloc skb fail! sz=%u (mfrag=%u, frag_num=%u)\n",
+			__FUNCTION__, alloc_sz, phlrx->mdata.more_frag,
+			phlrx->mdata.frag_num);
+		return -1;
+	}
+
+	/* force pkt->data at 8-byte alignment address */
+	skb_reserve(pkt, 8 - ((SIZE_PTR)pkt->data & 7));
+	/* force ip_hdr at 8-byte alignment address according to shift_sz. */
+	skb_reserve(pkt, shift_sz);
+	pbuf = skb_put(pkt, pktbuf->length);
+	_rtw_memcpy(pbuf, pktbuf->vir_addr, pktbuf->length);
+
+	prframe->u.hdr.pkt = pkt;
+	prframe->u.hdr.rx_data = pkt->data;
+	prframe->u.hdr.rx_tail = skb_tail_pointer(pkt);
+	prframe->u.hdr.len = pkt->len;
+	prframe->u.hdr.rx_head = pkt->head;
+	prframe->u.hdr.rx_end = skb_end_pointer(pkt);
+
+	return 0;
+}
+
+void core_update_recvframe_mdata(union recv_frame *prframe, struct rtw_recv_pkt *rx_req)
+{
+	struct rx_pkt_attrib *prxattrib = &prframe->u.hdr.attrib;
+	struct rtw_r_meta_data *mdata = &rx_req->mdata;
+
+	prxattrib->bdecrypted = !(mdata->sw_dec);
+	prxattrib->pkt_len = mdata->pktlen;
+	prxattrib->icv_err = mdata->icverr;
+	prxattrib->crc_err = mdata->crc32;
+	prxattrib->data_rate = mdata->rx_rate; /* enum rtw_data_rate */
+	prxattrib->gi_ltf = mdata->rx_gi_ltf;
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_RX
+	prxattrib->csum_valid = mdata->chksum_ofld_en;
+	prxattrib->csum_err = mdata->chksum_status;
+#endif
+	/* when 0 indicate no encrypt. when non-zero, indicate the encrypt algorith */
+	prxattrib->encrypt = rtw_sec_algo_phl2drv((enum rtw_enc_algo)mdata->sec_type);
+
+#if 0 //todo
+//Security (sw-decrypt & calculate payload offset)
+	u8	bdecrypted;
+	u8	encrypt; /* when 0 indicate no encrypt. when non-zero, indicate the encrypt algorith */
+	u8	iv_len;
+	u8	icv_len;
+	u8	crc_err;
+	u8	icv_err;
+#endif
+	prxattrib->addr_cam_vld = mdata->addr_cam_vld;
+	prxattrib->macid = mdata->macid;
+#ifdef CONFIG_TDLS
+	prxattrib->is_tdls_frame = mdata->is_tdls_frame;
+#endif
+
+
+	return;
+}
+
+#ifdef RTW_WKARD_CORE_RSSI_V1
+static inline void _rx_process_ss_sq(_adapter *padapter, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	struct signal_stat *ss = &padapter->recvinfo.signal_strength_data;
+	struct signal_stat *sq = &padapter->recvinfo.signal_qual_data;
+
+	if (ss->update_req) {
+		ss->total_num = 0;
+		ss->total_val = 0;
+		ss->update_req = 0;
+	}
+
+	ss->total_num++;
+	ss->total_val += pattrib->phy_info.signal_strength;
+	ss->avg_val = ss->total_val / ss->total_num;
+
+	if (sq->update_req) {
+		sq->total_num = 0;
+		sq->total_val = 0;
+		sq->update_req = 0;
+	}
+
+	sq->total_num++;
+	sq->total_val += pattrib->phy_info.signal_quality;
+	sq->avg_val = sq->total_val / sq->total_num;
+}
+
+/*#define DBG_RECV_INFO*/
+void rx_process_phy_info(union recv_frame *precvframe)
+{
+	_adapter *padapter = precvframe->u.hdr.adapter;
+	struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;
+	struct phydm_phyinfo_struct *phy_info = &pattrib->phy_info;
+	u8 *wlanhdr = NULL;
+	u8 *ta, *ra;
+	u8 is_ra_bmc;
+	struct sta_priv *pstapriv;
+	struct sta_info *psta = NULL;
+	struct recv_info  *precvinfo = &padapter->recvinfo;
+
+	bool is_packet_match_bssid = _FALSE;
+	bool is_packet_to_self = _FALSE;
+	bool is_packet_beacon = _FALSE;
+
+	wlanhdr = precvframe->u.hdr.rx_data;
+	ta = get_ta(wlanhdr);
+	ra = get_ra(wlanhdr);
+	is_ra_bmc = IS_MCAST(ra);
+
+	if (_rtw_memcmp(adapter_mac_addr(padapter), ta, ETH_ALEN) == _TRUE) {
+		static systime start_time = 0;
+
+		#if 0 /*For debug */
+		if (IsFrameTypeCtrl(wlanhdr)) {
+			RTW_INFO("-->Control frame: Y\n");
+			RTW_INFO("-->pkt_len: %d\n", pattrib->pkt_len);
+			RTW_INFO("-->Sub Type = 0x%X\n", get_frame_sub_type(wlanhdr));
+		}
+
+		/* Dump first 40 bytes of header */
+		int i = 0;
+
+		for (i = 0; i < 40; i++)
+			RTW_INFO("%d: %X\n", i, *((u8 *)wlanhdr + i));
+
+		RTW_INFO("\n");
+		#endif
+
+		if ((start_time == 0) || (rtw_get_passing_time_ms(start_time) > 5000)) {
+			RTW_ERR("Warning!!! %s: Confilc mac addr!!\n", __func__);
+			start_time = rtw_get_current_time();
+		}
+		precvinfo->dbg_rx_conflic_mac_addr_cnt++;
+		return;
+	}
+
+	pstapriv = &padapter->stapriv;
+	psta = precvframe->u.hdr.psta;
+
+	is_packet_match_bssid = (!IsFrameTypeCtrl(wlanhdr))
+			&& (!pattrib->icv_err) && (!pattrib->crc_err)
+			&& ((!MLME_IS_MESH(padapter) && _rtw_memcmp(get_hdr_bssid(wlanhdr), get_bssid(&padapter->mlmepriv), ETH_ALEN))
+				|| (MLME_IS_MESH(padapter) && psta));
+
+	/*is_to_self = (!pattrib->icv_err) && (!pattrib->crc_err)
+			&& _rtw_memcmp(ra, adapter_mac_addr(padapter), ETH_ALEN);*/
+
+	is_packet_to_self = is_packet_match_bssid
+			&& _rtw_memcmp(ra, adapter_mac_addr(padapter), ETH_ALEN);
+
+	is_packet_beacon = is_packet_match_bssid
+			 && (get_frame_sub_type(wlanhdr) == WIFI_BEACON);
+
+
+	if (psta && IsFrameTypeData(wlanhdr)) {
+
+
+		if (is_ra_bmc)
+			psta->curr_rx_rate_bmc = pattrib->data_rate;
+		else {
+			psta->curr_rx_rate = pattrib->data_rate;
+			psta->curr_rx_gi_ltf = pattrib->gi_ltf;
+		}
+	}
+
+	#if 0
+	/* If bw is initial value, get from phy status */
+	if (pattrib->bw == CHANNEL_WIDTH_MAX)
+		pattrib->bw = p_phy_info->band_width;
+	#endif
+#ifdef DBG_RECV_INFO
+	RTW_INFO("%s match_bssid:%d, to_self:%d, is_beacon:%d",
+		__func__, is_packet_match_bssid, is_packet_to_self, is_packet_beacon);
+
+	RTW_INFO("hdr_bssid:"MAC_FMT" my_bssid:"MAC_FMT"\n", 
+				MAC_ARG(get_hdr_bssid(wlanhdr)),
+				MAC_ARG(get_bssid(&padapter->mlmepriv)));
+
+	RTW_INFO("ra:"MAC_FMT" my_addr:"MAC_FMT"\n", 
+				MAC_ARG(ra),
+				MAC_ARG(adapter_mac_addr(padapter)));
+
+#endif
+	precvframe->u.hdr.psta = NULL;
+	if (padapter->registrypriv.mp_mode != 1) {
+		if ((!MLME_IS_MESH(padapter) && is_packet_match_bssid)
+			|| (MLME_IS_MESH(padapter) && psta)) {
+			if (psta) {
+				precvframe->u.hdr.psta = psta;
+#ifdef DBG_RX_DFRAME_RAW_DATA
+			rx_process_dframe_raw_data(precvframe);
+#endif
+			}
+			if (phy_info->is_valid)
+				_rx_process_ss_sq(padapter, precvframe);/*signal_strength & signal_quality*/
+		} else if (is_packet_to_self || is_packet_beacon) {
+			if (psta) {
+				precvframe->u.hdr.psta = psta;
+#ifdef DBG_RX_DFRAME_RAW_DATA
+			rx_process_dframe_raw_data(precvframe);
+#endif
+			}
+			if (phy_info->is_valid)
+				_rx_process_ss_sq(padapter, precvframe);/*signal_strength & signal_quality*/
+		}
+	}
+#ifdef CONFIG_MP_INCLUDED
+	else {
+		padapter->mppriv.rx_rate = pattrib->data_rate;
+		if (padapter->mppriv.brx_filter_beacon == _TRUE) {
+			if (is_packet_beacon) {
+				RTW_INFO("in MP Rx is_packet_beacon\n");
+				if (psta)
+					precvframe->u.hdr.psta = psta;
+				_rx_process_ss_sq(padapter, precvframe);
+			}
+		} else	{
+			if (psta)
+				precvframe->u.hdr.psta = psta;
+			_rx_process_ss_sq(padapter, precvframe);
+		}
+	}
+#endif
+}
+
+
+/*#define DBG_PHY_INFO*/
+void core_update_recvframe_phyinfo(union recv_frame *prframe, struct rtw_recv_pkt *rx_req)
+{
+	struct rx_pkt_attrib *attrib = &prframe->u.hdr.attrib;
+	struct rtw_phl_ppdu_phy_info *phy_info = &rx_req->phy_info;
+	u8 ptype, pstype;
+	int i;
+
+	_rtw_memset(&attrib->phy_info, 0, sizeof(struct phydm_phyinfo_struct));
+
+	ptype = phy_info->frame_type & 0x03;
+	pstype = phy_info->frame_type >> 2;
+
+	#ifdef DBG_PHY_INFO
+	{
+		u8 type, subtype;
+		u8 *ptr = prframe->u.hdr.rx_data;
+
+		type =	GetFrameType(ptr) >> 2;
+		subtype = get_frame_sub_type(ptr) >> 4; /* bit(7)~bit(2) */
+		if ((type != WIFI_CTRL_TYPE) && (ptype != type || pstype != subtype)) {
+			RTW_INFO("[DBG-PHY-INFO]- FT:0x%02x, FST:0x%02x\t", type, subtype);
+			RTW_INFO("A1:"MAC_FMT" A2:"MAC_FMT" A3:"MAC_FMT"\n", 
+					MAC_ARG(GetAddr1Ptr(ptr)),
+					MAC_ARG(get_addr2_ptr(ptr)),
+					MAC_ARG(GetAddr3Ptr(ptr)));
+		}
+	}
+	#endif
+
+	if (phy_info->is_valid) {
+		attrib->phy_info.is_valid = true;
+		attrib->phy_info.signal_strength = phy_info->rssi;
+		attrib->phy_info.signal_quality = phy_info->rssi;
+		attrib->phy_info.recv_signal_power = rtw_phl_rssi_to_dbm(phy_info->rssi);
+		attrib->ch = phy_info->ch_idx;
+		/* snr info */
+		attrib->phy_info.snr_fd_avg = phy_info->snr_fd_avg;
+		attrib->phy_info.snr_td_avg = phy_info->snr_td_avg;
+		for ( i = 0; i < RTW_PHL_MAX_RF_PATH; i++) {
+			attrib->phy_info.rx_pwr[i] = phy_info->rssi_path[i];
+			attrib->phy_info.snr_fd[i] = phy_info->snr_fd[i];
+			attrib->phy_info.snr_td[i] = phy_info->snr_td[i];
+		}
+
+		#ifdef DBG_PHY_INFO
+		RTW_INFO("[PHY-INFO] ft:0x%02x-0x%02x rssi:%d, ch_idx:%d, tx_bf:%d\n",
+			ptype, pstype, phy_info->rssi, phy_info->ch_idx, phy_info->tx_bf);
+
+		RTW_INFO("[PHY-INFO] ss:%d sq:%d rssi:%d\n",
+				attrib->phy_info.signal_strength,
+				attrib->phy_info.signal_quality,
+				attrib->phy_info.recv_signal_power);
+		#endif
+
+	} else {
+		#ifdef DBG_PHY_INFO
+		RTW_INFO("[PHY-INFO-INVALID] ftype:0x%02x-0x%02x rssi:%d, ch_idx:%d, tx_bf:%d\n",
+			ptype, pstype, phy_info->rssi, phy_info->ch_idx, phy_info->tx_bf);
+		#endif
+	}
+}
+#endif /*RTW_WKARD_CORE_RSSI_V1*/
+
+s32 core_rx_process_amsdu(_adapter *adapter, union recv_frame **pprframe)
+{
+	if (amsdu_to_msdu(adapter, pprframe) != _SUCCESS)
+		return CORE_RX_DROP;
+
+	return CORE_RX_DONE;
+}
+
+s32 core_rx_process_msdu(_adapter *adapter, union recv_frame **pprframe)
+{
+	union recv_frame *prframe = *pprframe;
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	u8 *msdu = get_recvframe_data(prframe)
+		+ pattrib->hdrlen + pattrib->iv_len + RATTRIB_GET_MCTRL_LEN(pattrib);
+	u16 msdu_len = prframe->u.hdr.len
+		- pattrib->hdrlen - pattrib->iv_len - RATTRIB_GET_MCTRL_LEN(pattrib)
+		- (pattrib->encrypt ? pattrib->icv_len : 0);
+	enum rtw_rx_llc_hdl llc_hdl = rtw_recv_llc_parse(msdu, msdu_len);
+	int act = RTW_RX_MSDU_ACT_INDICATE;
+
+#if defined(CONFIG_AP_MODE)
+	struct xmit_frame *fwd_frame = NULL;
+	_list f_list;
+
+	if (MLME_IS_AP(adapter))
+		act = rtw_ap_rx_msdu_act_check(prframe, pattrib->dst, pattrib->src
+				, msdu, llc_hdl, &fwd_frame, &f_list);
+	if (!act)
+		return CORE_RX_DROP;
+#endif
+
+	if(wlanhdr_to_ethhdr(prframe, llc_hdl) != _SUCCESS) {
+		if (act & RTW_RX_MSDU_ACT_INDICATE) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" wlanhdr_to_ethhdr: drop pkt\n"
+				, FUNC_ADPT_ARG(adapter));
+			#endif
+		}
+
+		#if defined(CONFIG_AP_MODE)
+		if (act & RTW_RX_MSDU_ACT_FORWARD) {
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s wlanhdr_to_ethhdr fail\n", __func__);
+			#endif
+			recv_free_fwd_resource(adapter, fwd_frame, &f_list);
+		}
+		#endif
+		return CORE_RX_DROP;
+	}
+
+	#if defined(CONFIG_AP_MODE)
+	if (act & RTW_RX_MSDU_ACT_FORWARD) {
+		recv_fwd_pkt_hdl(adapter, prframe->u.hdr.pkt, act, fwd_frame, &f_list);
+		if (!(act & RTW_RX_MSDU_ACT_INDICATE)) {
+			prframe->u.hdr.pkt = NULL;
+			rtw_free_recvframe(prframe);
+			*pprframe = NULL;
+			return CORE_RX_DONE;
+		}
+	}
+	#endif
+
+	if (rtw_recv_indicatepkt_check(prframe,
+		get_recvframe_data(prframe), get_recvframe_len(prframe)) != _SUCCESS)
+		return CORE_RX_DROP;
+
+	if (rtw_recv_indicatepkt(adapter, prframe) != _SUCCESS) {
+		*pprframe = NULL;
+		return CORE_RX_DROP;
+	}
+
+	return CORE_RX_DONE;
+}
+
+
+s32 rtw_core_rx_data_post_process(_adapter *adapter, union recv_frame **pprframe)
+{
+	union recv_frame *prframe = *pprframe;
+	//amsdu
+	//make eth hdr
+	//forward
+
+	//recv_process_mpdu
+	  //amsdu_to_msdu
+	  //wlanhdr_to_ethhdr
+	  //rtw_recv_indicatepkt_check
+	  //rtw_recv_indicatepkt
+
+
+//todo hw amsdu
+	if (prframe->u.hdr.attrib.amsdu) 
+		return core_rx_process_amsdu(adapter, pprframe);
+	else
+		return core_rx_process_msdu(adapter, pprframe);
+	
+}
+
+
+s32 rtw_core_rx_data_pre_process(_adapter *adapter, union recv_frame **prframe)
+{
+	//recv_func_posthandle
+	  //decryptor
+	  //portctrl
+	  //count_rx_stats
+#ifdef CONFIG_TDLS
+#endif
+#ifdef DBG_RX_BMC_FRAME
+#endif
+#ifdef CONFIG_WAPI_SUPPORT
+#endif
+
+	union recv_frame * ret_frame = NULL;
+
+	s32 ret = CORE_RX_CONTINUE;
+
+	ret_frame = decryptor(adapter, *prframe);
+	if (ret_frame == NULL)
+		return CORE_RX_DROP;
+	else
+		*prframe = ret_frame;
+
+	ret = recvframe_chk_defrag(adapter, prframe);
+	if (ret != CORE_RX_CONTINUE) {
+		if (ret == CORE_RX_DROP) {
+			#ifdef DBG_RX_DROP_FRAME
+			RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recvframe_chk_defrag: drop pkt\n"
+				, FUNC_ADPT_ARG(adapter));
+			#endif
+			DBG_COUNTER(adapter->rx_logs.core_rx_post_defrag_err);
+		}
+		return ret;
+	}
+
+	/* Rx TKIP MIC */
+	if ((*prframe)->u.hdr.attrib.privacy) {
+		if (recvframe_chkmic(adapter,  *prframe) == _FAIL) {
+			return CORE_RX_DROP;
+		}
+	}
+
+	ret_frame = portctrl(adapter, *prframe);
+	if (ret_frame == NULL)
+		return CORE_RX_DROP;
+	else
+		*prframe = ret_frame;
+
+	count_rx_stats(adapter, *prframe, NULL);
+
+#ifdef CONFIG_WAPI_SUPPORT
+	rtw_wapi_update_info(adapter, *prframe);
+#endif
+
+	return CORE_RX_CONTINUE;
+}
+
+s32 rtw_core_update_recvframe(struct dvobj_priv *dvobj,
+	union recv_frame *prframe, struct rtw_recv_pkt *rx_req)
+{
+	u8 *pbuf = NULL;
+	u8 *pda = NULL;
+	_adapter *iface = NULL;
+	u8 is_bmc = _FALSE;
+	enum rtw_core_rx_state rx_state = CORE_RX_CONTINUE;
+	_adapter *primary_padapter = dvobj_get_primary_adapter(dvobj);
+	int err;
+	struct mlme_priv	*pmlmepriv = NULL;
+
+	if (rx_req->mdata.bc || rx_req->mdata.mc)
+		is_bmc = _TRUE;
+
+	//pre_recv_entry
+	//rtw_get_iface_by_macddr
+	if (rx_req->os_priv) {
+		prframe->u.hdr.pkt = rx_req->os_priv; /*skb*/
+		core_update_recvframe_pkt(prframe, rx_req);
+	} else {
+		err = core_alloc_recvframe_pkt(prframe, rx_req);
+		if (err) {
+			rx_state = CORE_RX_FAIL;
+			goto exit;
+		}
+	}
+
+	core_update_recvframe_mdata(prframe, rx_req);
+	#ifdef RTW_WKARD_CORE_RSSI_V1
+	core_update_recvframe_phyinfo(prframe, rx_req);
+	#endif
+
+	prframe->u.hdr.adapter = primary_padapter;
+	prframe->u.hdr.pkt->dev = primary_padapter->pnetdev;
+
+	if (!is_bmc) {
+		pbuf = prframe->u.hdr.rx_data;
+		pda = get_ra(pbuf);
+		iface = rtw_get_iface_by_macddr(primary_padapter, pda);
+		if(iface) {
+			prframe->u.hdr.adapter = iface;
+			prframe->u.hdr.pkt->dev = iface->pnetdev;
+			/* ToDo CONFIG_RTW_MLD: TBD to see if there is direct info from mdata */
+			if (rx_req->rx_rlink)
+				prframe->u.hdr.adapter_link = GET_LINK(iface, rx_req->rx_rlink->id);
+			else
+				prframe->u.hdr.adapter_link = GET_PRIMARY_LINK(iface);
+		}
+		else {
+			#if 0 /*#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI - TODO*/
+			if (_rtw_memcmp(ra, adapter_pno_mac_addr(primary_padapter), ETH_ALEN))
+					goto query_phy_status;
+			#endif
+
+			#ifdef CONFIG_RTW_MULTI_AP
+			/* unasoc STA RCPI */
+			if (rtw_unassoc_sta_src_chk(primary_padapter, UNASOC_STA_SRC_RX_NMY_UC) &&
+				prframe->u.hdr.attrib.phy_info.recv_signal_power != 0) {
+				rtw_rx_add_unassoc_sta(primary_padapter, UNASOC_STA_SRC_RX_NMY_UC,
+					get_ta(pbuf), prframe->u.hdr.attrib.phy_info.recv_signal_power);
+				/* ToDo CONFIG_RTW_MLD: TBD to see if there is direct info from mdata */
+				if (rx_req->rx_rlink)
+					prframe->u.hdr.adapter_link = GET_LINK(primary_padapter, rx_req->rx_rlink->id);
+				else
+					prframe->u.hdr.adapter_link = GET_PRIMARY_LINK(primary_padapter);
+			} else
+			#endif
+				RTW_DBG("%s [WARN] Cannot find appropriate adapter - mac_addr : "MAC_FMT"\n"
+						, __func__, MAC_ARG(pda));
+			rx_state = CORE_RX_FAIL;
+		}
+	}
+	else {
+		/* ToDo CONFIG_RTW_MLD: TBD to see if there is direct info from mdata */
+		if (rx_req->rx_rlink)
+			prframe->u.hdr.adapter_link = GET_LINK(primary_padapter, rx_req->rx_rlink->id);
+		else
+			prframe->u.hdr.adapter_link = GET_PRIMARY_LINK(primary_padapter);
+		/*clone bcmc frame for all active adapter*/
+		rtw_mi_buddy_clone_bcmc_packet(primary_padapter, prframe);
+
+		#ifdef CONFIG_RTW_MULTI_AP
+		pbuf = prframe->u.hdr.rx_data;
+		/* unasoc STA RCPI */
+		if (rtw_unassoc_sta_src_chk(primary_padapter, UNASOC_STA_SRC_RX_BMC) &&
+			prframe->u.hdr.attrib.phy_info.recv_signal_power != 0) {
+			rtw_rx_add_unassoc_sta(primary_padapter, UNASOC_STA_SRC_RX_BMC,
+				get_ta(pbuf), prframe->u.hdr.attrib.phy_info.recv_signal_power);
+		}
+		#endif
+	}
+
+exit:
+	prframe->u.hdr.rx_req = rx_req;
+
+	pmlmepriv = &(prframe->u.hdr.adapter)->mlmepriv;
+	if (check_fwstate(pmlmepriv, WIFI_MONITOR_STATE)) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+			recv_frame_monitor(prframe->u.hdr.adapter, prframe, rx_req);
+#endif
+			rx_state = CORE_RX_FAIL;
+	}
+
+	return rx_state;
+}
+
+#ifdef CONFIG_RTW_NAPI
+#ifdef CONFIG_RTW_NAPI_V2
+static void rtw_core_rx_napi_v2(struct dvobj_priv *dvobj)
+{
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+	struct recv_priv *recvpriv = &dvobj->recvpriv;
+
+	if (adapter->registrypriv.en_napi) {
+		_adapter *a;
+		u8 i;
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			a = dvobj->padapters[i];
+			if ((rtw_if_up(a) == _TRUE)
+				&& skb_queue_len(&recvpriv->rx_napi_skb_queue))
+				napi_schedule(&a->napi);
+		}
+	}
+}
+#endif
+#endif
+
+enum rtw_phl_status rtw_core_rx_process(void *drv_priv)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+	_adapter *adapter = NULL;
+	struct rtw_recv_pkt *rx_req = NULL;
+	struct rtw_pkt_buf_list *pkt = NULL;
+	union recv_frame *prframe = NULL;
+	struct rx_pkt_attrib *prxattrib = NULL;
+	u16 rx_pkt_num = 0;
+	struct recv_priv *precvpriv = &dvobj->recvpriv;
+	s32 pre_process_ret = CORE_RX_CONTINUE;
+	s32 process_ret = CORE_RX_CONTINUE;
+
+	rx_pkt_num = rtw_phl_query_new_rx_num(GET_PHL_INFO(dvobj));
+
+#ifdef DBG_RECV_FRAME
+	RTW_INFO("%s dvobj:%p, phl:%p rx_pkt_num:%d, free_recv_queue:%p\n",
+	__func__, dvobj, dvobj->phl, rx_pkt_num, &precvpriv->free_recv_queue);
+#endif
+
+	while (rx_pkt_num--) {
+		prframe = rtw_alloc_recvframe(&precvpriv->free_recv_queue);
+		if (prframe == NULL) {
+			RTW_ERR("F-%s L-%d rtw_alloc_recvframe failed\n", __FUNCTION__, __LINE__);
+			goto rx_error;
+		}
+
+		//_rtw_init_listhead
+
+		rx_req = rtw_phl_query_rx_pkt(GET_PHL_INFO(dvobj));
+		if(rx_req == NULL)
+			goto rx_stop;
+
+		if(rtw_core_update_recvframe(dvobj, prframe, rx_req) != CORE_RX_CONTINUE)
+			goto rx_next;
+
+		prxattrib = &prframe->u.hdr.attrib;
+		if (prxattrib->icv_err || prxattrib->crc_err)
+			goto rx_next;
+
+		adapter = prframe->u.hdr.adapter;
+
+		if (!rtw_is_adapter_up(adapter))
+			goto rx_next;
+
+#ifdef CONFIG_RTW_CORE_RXSC
+		if (core_rxsc_apply_check(adapter, prframe) == CORE_RX_GO_SHORTCUT &&
+			core_rxsc_apply_shortcut(adapter, prframe) == CORE_RX_DONE)
+			continue;
+#endif
+
+		//recv_func_prehandle
+		//mgt_dispatcher exist here && sw decrypt mgmt 
+		//?? todo power save
+#ifdef CONFIG_MP_INCLUDED
+		if (rtw_mp_mode_check(adapter)) {
+			if(mp_recv_frame(adapter, prframe))
+				goto rx_next;
+		} else
+#endif
+		{
+		if (validate_recv_frame(adapter, &prframe) != CORE_RX_CONTINUE)
+			goto rx_next;
+		}
+		pre_process_ret = rtw_core_rx_data_pre_process(adapter, &prframe);
+		if (pre_process_ret == CORE_RX_DEFRAG)
+			continue;
+		if (pre_process_ret != CORE_RX_CONTINUE)
+			goto rx_next;
+
+		process_ret = rtw_core_rx_data_post_process(adapter, &prframe);
+		if (process_ret == CORE_RX_DONE) {
+			adapter->recvinfo.rx_pkts++;
+			continue;
+		} else if (process_ret == CORE_RX_DROP) {
+			/*
+			alread free prframe in rtw_core_rx_data_post_process
+			no need free again
+			*/
+			if (prframe == NULL)
+				continue;
+		}
+
+rx_next:
+		rtw_free_recvframe(prframe);
+		continue;
+rx_stop:
+		rtw_free_recvframe(prframe);
+		break;
+rx_error:
+		break;
+	}
+
+#ifdef CONFIG_RTW_NAPI
+#ifdef CONFIG_RTW_NAPI_V2
+	rtw_core_rx_napi_v2(dvobj);
+#endif
+#endif
+
+	return RTW_PHL_STATUS_SUCCESS;
+}
+#endif /*RTW_PHL_RX*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_recv_shortcut.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_recv_shortcut.c
new file mode 100644
index 000000000000..d8b4d1b8b7b2
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_recv_shortcut.c
@@ -0,0 +1,247 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+
+#ifdef CONFIG_RTW_CORE_RXSC
+#define is_equal_mac_addr(lhs,rhs) ((((unsigned long)(lhs)|(unsigned long)(rhs)) & 0x01) ? \
+				    ((*(unsigned char*)(lhs) ^ (*(unsigned char*)(rhs))) | (*(unsigned char*)(lhs + 1) ^ (*(unsigned char*)(rhs + 1))) | (*(unsigned char*)(lhs + 2) ^ (*(unsigned char*)(rhs + 2)))|\
+				    (*(unsigned char*)(lhs + 3) ^ (*(unsigned char*)(rhs + 3))) | (*(unsigned char*)(lhs + 4) ^ (*(unsigned char*)(rhs + 4))) | (*(unsigned char*)(lhs + 5) ^ (*(unsigned char*)(rhs + 5))))==0 :\
+				    ((*(unsigned short*)(lhs) ^ (*(unsigned short*)(rhs))) | (*(unsigned short*)(lhs + 2) ^ (*(unsigned short*)(rhs + 2))) | (*(unsigned short*)(lhs + 4) ^ (*(unsigned short*)(rhs + 4))))==0  \
+				   )
+
+u8 core_rxsc_check_alloc(_adapter *adapter, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+
+	if (!adapter->enable_rxsc)
+		return _FAIL;
+
+	if (pattrib->amsdu ||
+		!pattrib->bdecrypted ||
+		pattrib->mfrag ||
+		pattrib->to_fr_ds == 3 ||
+		!pattrib->bsnaphdr)
+		return _FAIL;
+
+	if (pattrib->eth_type == ETH_P_ARP ||
+		pattrib->eth_type == 0x888e ||
+		pattrib->eth_type == 0x8899)
+		return _FAIL;
+
+	if (!prframe->u.hdr.psta)
+		return _FAIL;
+
+	if (IS_MCAST(pattrib->ra) || IS_MCAST(pattrib->dst))
+		return _FAIL;
+
+#ifdef CONFIG_RTW_MESH
+	if (pattrib->mesh_ctrl_present)
+		return _FAIL;
+#endif
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (prframe->u.hdr.bIsWaiPacket)
+		return _FAIL;
+#endif
+
+	return _SUCCESS;
+}
+
+struct core_rxsc_entry *core_rxsc_alloc_entry(_adapter *adapter, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	struct sta_info *psta = prframe->u.hdr.psta;
+	struct core_rxsc_entry *rxsc_entry = NULL;
+	u8 rxsc_idx, cnt = 0;
+
+	if (core_rxsc_check_alloc(adapter, prframe)!=_SUCCESS)
+		return NULL;
+
+	rxsc_idx = psta->rxsc_idx_new;
+
+	for (cnt=0; cnt<NUM_RXSC_ENTRY; cnt++) {
+		rxsc_idx = rxsc_idx%NUM_RXSC_ENTRY;
+		if (psta->rxsc_entry[rxsc_idx].status != RXSC_ENTRY_APPLYING)
+			break;
+		rxsc_idx++;
+	}
+
+	rxsc_entry = &psta->rxsc_entry[rxsc_idx];
+
+	if (rxsc_entry->status == RXSC_ENTRY_APPLYING)
+		return NULL;
+
+	rxsc_entry->status = RXSC_ENTRY_VALID;
+	psta->rxsc_idx_new = (rxsc_idx + 1) % NUM_RXSC_ENTRY;
+
+	return rxsc_entry;
+}
+
+sint core_rxsc_get_entry(_adapter *adapter, union recv_frame *prframe)
+{
+	struct sta_info *psta = NULL;
+	u8 *pframe = prframe->u.hdr.rx_data;
+	u8 *lhs, *rhs = NULL;
+	u8 idx = 0;
+	struct core_rxsc_entry *rxsc_entry = NULL;
+
+	if (GetFrameType(pframe)!=WIFI_DATA_TYPE)
+		return _FAIL;
+
+	psta = rtw_get_stainfo(&adapter->stapriv, get_addr2_ptr(pframe));
+
+	if (!psta)
+		return _FAIL;
+
+	lhs = GetAddr1Ptr(pframe);
+
+	for (idx = 0; idx < NUM_RXSC_ENTRY; idx++) {
+		rxsc_entry = &psta->rxsc_entry[(psta->rxsc_idx_cached + idx) % NUM_RXSC_ENTRY];
+
+		if (rxsc_entry->status == RXSC_ENTRY_INVALID)
+			continue;
+
+		rhs = (u8*)rxsc_entry->rxsc_wlanhdr.addr1;
+#if 0
+		DBGP("(A1)lhs=%pM rhs=%pM isEqual=%d %d\n", lhs, rhs, is_equal_mac_addr(lhs,rhs), is_equal_mac_addr(lhs+1,rhs));
+		DBGP("(A2)lhs=%pM rhs=%pM\n", lhs+6, rhs+6);
+		DBGP("(A3)lhs=%pM rhs=%pM\n", lhs+12, rhs+12);
+		DBGP("rxsc_payload_offset=%d \n", rxsc_entry->rxsc_payload_offset);
+		DBGP("(LLC)lhs=%pM rhs=%pM\n",
+			&pframe[rxsc_entry->rxsc_payload_offset+(sizeof(struct ethhdr))-SNAP_SIZE-2],  &rtw_rfc1042_header);
+		DBGP("(eth_type)lhs=0x%x rhs=0x%x\n",
+			*(unsigned short *)(&pframe[rxsc_entry->rxsc_payload_offset+(ETH_ALEN*2)]), *(unsigned short *)(&rxsc_entry->rxsc_ethhdr.h_proto));
+
+		DBGP("%d %d %d %d %d \n",
+			is_equal_mac_addr(lhs,rhs),
+			is_equal_mac_addr(lhs+6,rhs+6),
+			is_equal_mac_addr(lhs+12,rhs+12),
+			!memcmp(&pframe[rxsc_entry->rxsc_payload_offset+(sizeof(struct ethhdr))-SNAP_SIZE-2], &rtw_rfc1042_header, ETH_ALEN),
+			(*(unsigned short *)(&pframe[rxsc_entry->rxsc_payload_offset+(ETH_ALEN*2)]) == *(unsigned short *)(&rxsc_entry->rxsc_ethhdr.h_proto))
+			);
+#endif
+		if (is_equal_mac_addr(lhs,rhs) &&
+			is_equal_mac_addr(lhs + 6, rhs + 6) &&
+			is_equal_mac_addr(lhs + 12, rhs + 12) &&
+			!memcmp(&pframe[rxsc_entry->rxsc_payload_offset+(sizeof(struct ethhdr)) - SNAP_SIZE - 2], &rtw_rfc1042_header, ETH_ALEN) &&
+			(*(unsigned short *)(&pframe[rxsc_entry->rxsc_payload_offset+(ETH_ALEN * 2)]) == *(unsigned short *)(&rxsc_entry->rxsc_ethhdr.h_proto))) {
+			prframe->u.hdr.psta = psta;
+			prframe->u.hdr.rxsc_entry = rxsc_entry;
+			psta->rxsc_idx_cached = idx;
+			return _SUCCESS;
+		}
+	}
+
+	return _FAIL;
+}
+
+s32 core_rxsc_apply_check(_adapter *adapter, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	struct core_rxsc_entry *rxsc_entry = NULL;
+
+	if (!adapter->enable_rxsc)
+		return CORE_RX_CONTINUE;
+
+	if (core_rxsc_get_entry(adapter, prframe) != _SUCCESS)
+		return CORE_RX_CONTINUE;
+
+	rxsc_entry = prframe->u.hdr.rxsc_entry;
+
+	rxsc_entry->status = RXSC_ENTRY_APPLYING;
+	wmb();
+	_rtw_memcpy(pattrib, &rxsc_entry->rxsc_attrib, sizeof(rxsc_entry->rxsc_attrib));
+	wmb();
+	rxsc_entry->status = RXSC_ENTRY_VALID;
+
+	return CORE_RX_GO_SHORTCUT;
+}
+
+s32 core_rxsc_apply_shortcut(_adapter *adapter, union recv_frame *prframe)
+{
+	struct rx_pkt_attrib *pattrib = &prframe->u.hdr.attrib;
+	struct core_rxsc_entry *rxsc_entry = prframe->u.hdr.rxsc_entry;
+	struct sta_info *psta = prframe->u.hdr.psta;
+	u8 *pframe = prframe->u.hdr.rx_data;
+	u8 *ptr = NULL;
+
+	/* Shortcut of validate_recv_frame */
+	if (pattrib->encrypt == _AES_) {
+		if (pattrib->qos)
+			pattrib->priority = GetPriority((pframe + WLAN_HDR_A3_LEN));
+		else
+			pattrib->priority = 0;
+
+		if (!IS_MCAST(pattrib->ra)) {
+
+#ifdef CONFIG_RECV_REORDERING_CTRL
+			if (pattrib->qos)
+				prframe->u.hdr.preorder_ctrl = &psta->recvreorder_ctrl[pattrib->priority];
+#endif
+
+			if (recv_ucast_pn_decache(prframe) == _FAIL) {
+			#ifdef DBG_RX_DROP_FRAME
+				RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recv_ucast_pn_decache return _FAIL for sta="MAC_FMT"\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->cmn.mac_addr));
+			#endif
+				return CORE_RX_DROP;
+			}
+		} else {
+			if (recv_bcast_pn_decache(prframe) == _FAIL) {
+			#ifdef DBG_RX_DROP_FRAME
+				RTW_INFO("DBG_RX_DROP_FRAME "FUNC_ADPT_FMT" recv_bcast_pn_decache return _FAIL for sta="MAC_FMT"\n"
+					, FUNC_ADPT_ARG(adapter), MAC_ARG(psta->cmn.mac_addr));
+			#endif
+				return CORE_RX_DROP;
+			}
+		}
+	}
+
+	process_pwrbit_data(adapter, prframe, psta);
+
+	if ((get_frame_sub_type(pframe) & WIFI_QOS_DATA_TYPE) == WIFI_QOS_DATA_TYPE)
+		process_wmmps_data(adapter, prframe, psta);
+
+	/* Shortcut of rtw_core_rx_data_pre_process */
+	count_rx_stats(adapter, prframe, NULL);
+
+	/* Shortcut of rtw_core_rx_data_post_process */
+	/* 802.11 -> 802.3 */
+	if (rxsc_entry->rxsc_trim_pad)
+		recvframe_pull_tail(prframe, rxsc_entry->rxsc_trim_pad);
+
+	ptr = recvframe_pull(prframe, rxsc_entry->rxsc_payload_offset);
+	_rtw_memcpy(ptr, (u8 *)(&rxsc_entry->rxsc_ethhdr), sizeof(rxsc_entry->rxsc_ethhdr));
+	rtw_rframe_set_os_pkt(prframe);
+
+	/* Packet forward to OS */
+	rtw_os_recv_indicate_pkt(adapter, prframe->u.hdr.pkt, prframe);
+	prframe->u.hdr.pkt = NULL;
+	rtw_free_recvframe(prframe);
+#ifdef RTW_PHL_DBG_CMD
+	adapter->core_logs.rxCnt_data_shortcut++;
+#endif
+	return CORE_RX_DONE;
+}
+
+void core_rxsc_clear_entry(_adapter *adapter, struct sta_info *psta)
+{
+	u32 idx = 0;
+	for (idx=0; idx<NUM_RXSC_ENTRY; idx++)
+		psta->rxsc_entry[idx].status = RXSC_ENTRY_INVALID;
+}
+#endif /* CONFIG_RTW_CORE_RXSC */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rf.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rf.c
new file mode 100644
index 000000000000..15def99581b1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rf.c
@@ -0,0 +1,2201 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_RF_C_
+
+#include <drv_types.h>
+
+u8 center_ch_2g[CENTER_CH_2G_NUM] = {
+/* G00 */1, 2,
+/* G01 */3, 4, 5,
+/* G02 */6, 7, 8,
+/* G03 */9, 10, 11,
+/* G04 */12, 13,
+/* G05 */14
+};
+
+#define ch_to_cch_2g_idx(ch) ((ch) - 1)
+
+u8 center_ch_2g_40m[CENTER_CH_2G_40M_NUM] = {
+	3,
+	4,
+	5,
+	6,
+	7,
+	8,
+	9,
+	10,
+	11,
+};
+
+u8 op_chs_of_cch_2g_40m[CENTER_CH_2G_40M_NUM][2] = {
+	{1, 5}, /* 3 */
+	{2, 6}, /* 4 */
+	{3, 7}, /* 5 */
+	{4, 8}, /* 6 */
+	{5, 9}, /* 7 */
+	{6, 10}, /* 8 */
+	{7, 11}, /* 9 */
+	{8, 12}, /* 10 */
+	{9, 13}, /* 11 */
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+u8 center_ch_5g_all[CENTER_CH_5G_ALL_NUM] = {
+/* G00 */36, 38, 40,
+	42,
+/* G01 */44, 46, 48,
+	/* 50, */
+/* G02 */52, 54, 56,
+	58,
+/* G03 */60, 62, 64,
+/* G04 */100, 102, 104,
+	106,
+/* G05 */108, 110, 112,
+	/* 114, */
+/* G06 */116, 118, 120,
+	122,
+/* G07 */124, 126, 128,
+/* G08 */132, 134, 136,
+	138,
+/* G09 */140, 142, 144,
+/* G10 */149, 151, 153,
+	155,
+/* G11 */157, 159, 161,
+	/* 163, */
+/* G12 */165, 167, 169,
+	171,
+/* G13 */173, 175, 177
+};
+
+u8 center_ch_5g_20m[CENTER_CH_5G_20M_NUM] = {
+/* G00 */36, 40,
+/* G01 */44, 48,
+/* G02 */52, 56,
+/* G03 */60, 64,
+/* G04 */100, 104,
+/* G05 */108, 112,
+/* G06 */116, 120,
+/* G07 */124, 128,
+/* G08 */132, 136,
+/* G09 */140, 144,
+/* G10 */149, 153,
+/* G11 */157, 161,
+/* G12 */165, 169,
+/* G13 */173, 177
+};
+
+#define ch_to_cch_5g_20m_idx(ch) \
+	( \
+		((ch) >= 36 && (ch) <= 64) ? (((ch) - 36) >> 2) : \
+		((ch) >= 100 && (ch) <= 144) ? 8 + (((ch) - 100) >> 2) : \
+		((ch) >= 149 && (ch) <= 177) ? 20 + (((ch) - 149) >> 2) : 255 \
+	)
+
+u8 center_ch_5g_40m[CENTER_CH_5G_40M_NUM] = {
+/* G00 */38,
+/* G01 */46,
+/* G02 */54,
+/* G03 */62,
+/* G04 */102,
+/* G05 */110,
+/* G06 */118,
+/* G07 */126,
+/* G08 */134,
+/* G09 */142,
+/* G10 */151,
+/* G11 */159,
+/* G12 */167,
+/* G13 */175
+};
+
+u8 op_chs_of_cch_5g_40m[CENTER_CH_5G_40M_NUM][2] = {
+	{36, 40}, /* 38 */
+	{44, 48}, /* 46 */
+	{52, 56}, /* 54 */
+	{60, 64}, /* 62 */
+	{100, 104}, /* 102 */
+	{108, 112}, /* 110 */
+	{116, 120}, /* 118 */
+	{124, 128}, /* 126 */
+	{132, 136}, /* 134 */
+	{140, 144}, /* 142 */
+	{149, 153}, /* 151 */
+	{157, 161}, /* 159 */
+	{165, 169}, /* 167 */
+	{173, 177}, /* 175 */
+};
+
+u8 center_ch_5g_80m[CENTER_CH_5G_80M_NUM] = {
+/* G00 ~ G01*/42,
+/* G02 ~ G03*/58,
+/* G04 ~ G05*/106,
+/* G06 ~ G07*/122,
+/* G08 ~ G09*/138,
+/* G10 ~ G11*/155,
+/* G12 ~ G13*/171
+};
+
+u8 op_chs_of_cch_5g_80m[CENTER_CH_5G_80M_NUM][4] = {
+	{36, 40, 44, 48}, /* 42 */
+	{52, 56, 60, 64}, /* 58 */
+	{100, 104, 108, 112}, /* 106 */
+	{116, 120, 124, 128}, /* 122 */
+	{132, 136, 140, 144}, /* 138 */
+	{149, 153, 157, 161}, /* 155 */
+	{165, 169, 173, 177}, /* 171 */
+};
+
+u8 center_ch_5g_160m[CENTER_CH_5G_160M_NUM] = {
+/* G00 ~ G03*/50,
+/* G04 ~ G07*/114,
+/* G10 ~ G13*/163
+};
+
+u8 op_chs_of_cch_5g_160m[CENTER_CH_5G_160M_NUM][8] = {
+	{36, 40, 44, 48, 52, 56, 60, 64}, /* 50 */
+	{100, 104, 108, 112, 116, 120, 124, 128}, /* 114 */
+	{149, 153, 157, 161, 165, 169, 173, 177}, /* 163 */
+};
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+u8 center_ch_6g_20m[CENTER_CH_6G_20M_NUM] = {
+	1, 5, 9, 13, 17, 21, 25, 29,
+	33, 37, 41, 45, 49, 53, 57, 61,
+	65, 69, 73, 77, 81, 85, 89, 93,
+	97, 101, 105, 109, 113, 117, 121, 125,
+	129, 133, 137, 141, 145, 149, 153, 157,
+	161, 165, 169, 173, 177, 181, 185, 189,
+	193, 197, 201, 205, 209, 213, 217, 221,
+	225, 229, 233, 237, 241, 245, 249, 253,
+};
+
+u8 center_ch_6g_40m[CENTER_CH_6G_40M_NUM] = {
+	3, 11, 19, 27,
+	35, 43, 51, 59,
+	67, 75, 83, 91,
+	99, 107, 115, 123,
+	131, 139, 147, 155,
+	163, 171, 179, 187,
+	195, 203, 211, 219,
+	227, 235, 243, 251,
+};
+
+u8 op_chs_of_cch_6g_40m[CENTER_CH_6G_40M_NUM][2] = {
+	{1, 5}, /* 3 */
+	{9, 13}, /* 11 */
+	{17, 21}, /* 19 */
+	{25, 29}, /* 27 */
+	{33, 37}, /* 35 */
+	{41, 45}, /* 43 */
+	{49, 53}, /* 51 */
+	{57, 61}, /* 59 */
+	{65, 69}, /* 67 */
+	{73, 77}, /* 75 */
+	{81, 85}, /* 83 */
+	{89, 93}, /* 91 */
+	{97, 101}, /* 99 */
+	{105, 109}, /* 107 */
+	{113, 117}, /* 115 */
+	{121, 125}, /* 123 */
+	{129, 133}, /* 131 */
+	{137, 141}, /* 139 */
+	{145, 149}, /* 147 */
+	{153, 157}, /* 155 */
+	{161, 165}, /* 163 */
+	{169, 173}, /* 171 */
+	{177, 181}, /* 179 */
+	{185, 189}, /* 187 */
+	{193, 197}, /* 195 */
+	{201, 205}, /* 203 */
+	{209, 213}, /* 211 */
+	{217, 221}, /* 219 */
+	{225, 229}, /* 227 */
+	{233, 237}, /* 235 */
+	{241, 245}, /* 243 */
+	{249, 253}, /* 251 */
+};
+
+u8 center_ch_6g_80m[CENTER_CH_6G_80M_NUM] = {
+	7, 23,
+	39, 55,
+	71, 87,
+	103, 119,
+	135, 151,
+	167, 183,
+	199, 215,
+	231, 247,
+};
+
+u8 op_chs_of_cch_6g_80m[CENTER_CH_6G_80M_NUM][4] = {
+	{1, 5, 9, 13}, /* 7 */
+	{17, 21, 25, 29}, /* 23 */
+	{33, 37, 41, 45}, /* 39 */
+	{49, 53, 57, 61}, /* 55 */
+	{65, 69, 73, 77}, /* 71 */
+	{81, 85, 89, 93}, /* 87 */
+	{97, 101, 105, 109}, /* 103 */
+	{113, 117, 121, 125}, /* 119 */
+	{129, 133, 137, 141}, /* 135 */
+	{145, 149, 153, 157}, /* 151 */
+	{161, 165, 169, 173}, /* 167 */
+	{177, 181, 185, 189}, /* 183 */
+	{193, 197, 201, 205}, /* 199 */
+	{209, 213, 217, 221}, /* 215 */
+	{225, 229, 233, 237}, /* 231 */
+	{241, 245, 249, 253}, /* 247 */
+};
+
+u8 center_ch_6g_160m[CENTER_CH_6G_160M_NUM] = {
+	15,
+	47,
+	79,
+	111,
+	143,
+	175,
+	207,
+	239,
+};
+
+u8 op_chs_of_cch_6g_160m[CENTER_CH_6G_160M_NUM][8] = {
+	{1, 5, 9, 13, 17, 21, 25, 29}, /* 15 */
+	{33, 37, 41, 45, 49, 53, 57, 61}, /* 47 */
+	{65, 69, 73, 77, 81, 85, 89, 93}, /* 79 */
+	{97, 101, 105, 109, 113, 117, 121, 125}, /* 111 */
+	{129, 133, 137, 141, 145, 149, 153, 157}, /* 143 */
+	{161, 165, 169, 173, 177, 181, 185, 189}, /* 175 */
+	{193, 197, 201, 205, 209, 213, 217, 221}, /* 207 */
+	{225, 229, 233, 237, 241, 245, 249, 253}, /* 239 */
+};
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+struct center_chs_ent_t {
+	u8 ch_num;
+	u8 *chs;
+};
+
+struct center_chs_ent_t center_chs_2g_by_bw[] = {
+	{CENTER_CH_2G_NUM, center_ch_2g},
+	{CENTER_CH_2G_40M_NUM, center_ch_2g_40m},
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+struct center_chs_ent_t center_chs_5g_by_bw[] = {
+	{CENTER_CH_5G_20M_NUM, center_ch_5g_20m},
+	{CENTER_CH_5G_40M_NUM, center_ch_5g_40m},
+	{CENTER_CH_5G_80M_NUM, center_ch_5g_80m},
+	{CENTER_CH_5G_160M_NUM, center_ch_5g_160m},
+};
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+struct center_chs_ent_t center_chs_6g_by_bw[] = {
+	{CENTER_CH_6G_20M_NUM, center_ch_6g_20m},
+	{CENTER_CH_6G_40M_NUM, center_ch_6g_40m},
+	{CENTER_CH_6G_80M_NUM, center_ch_6g_80m},
+	{CENTER_CH_6G_160M_NUM, center_ch_6g_160m},
+};
+#endif
+
+/*
+ * Get center channel of smaller bandwidth by @param band, @param cch, @param bw, @param offset
+ * @band: the given band
+ * @cch: the given center channel
+ * @bw: the given bandwidth
+ * @offset: the given primary SC offset of the given bandwidth
+ *
+ * return center channel of smaller bandiwdth if valid, or 0
+ */
+static u8 _rtw_get_scch_by_bcch_offset(enum band_type band, u8 cch, u8 bw, u8 offset)
+{
+	u8 t_cch = 0;
+
+	if (bw == CHANNEL_WIDTH_20) {
+		t_cch = cch;
+		goto exit;
+	}
+
+	if (offset == CHAN_OFFSET_NO_EXT) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (band == BAND_ON_24G) {
+		/* 2.4G, 40MHz */
+		if (cch >= 3 && cch <= 11 && bw == CHANNEL_WIDTH_40) {
+			t_cch = (offset == CHAN_OFFSET_LOWER) ? cch + 2 : cch - 2;
+			goto exit;
+		}
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G) {
+		/* 5G, 160MHz */
+		if (cch >= 50 && cch <= 163 && bw == CHANNEL_WIDTH_160) {
+			t_cch = (offset == CHAN_OFFSET_LOWER) ? cch + 8 : cch - 8;
+			goto exit;
+
+		/* 5G, 80MHz */
+		} else if (cch >= 42 && cch <= 171 && bw == CHANNEL_WIDTH_80) {
+			t_cch = (offset == CHAN_OFFSET_LOWER) ? cch + 4 : cch - 4;
+			goto exit;
+
+		/* 5G, 40MHz */
+		} else if (cch >= 38 && cch <= 175 && bw == CHANNEL_WIDTH_40) {
+			t_cch = (offset == CHAN_OFFSET_LOWER) ? cch + 2 : cch - 2;
+			goto exit;
+
+		}
+	}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G) {
+		/* 6G, 160MHz */
+		if (cch >= 15 && cch <= 239 && bw == CHANNEL_WIDTH_160) {
+			t_cch = (offset == CHAN_OFFSET_LOWER) ? cch + 8 : cch - 8;
+			goto exit;
+
+		/* 6G, 80MHz */
+		} else if (cch >= 7 && cch <= 247 && bw == CHANNEL_WIDTH_80) {
+			t_cch = (offset == CHAN_OFFSET_LOWER) ? cch + 4 : cch - 4;
+			goto exit;
+
+		/* 6G, 40MHz */
+		} else if (cch >= 3 && cch <= 251 && bw == CHANNEL_WIDTH_40) {
+			t_cch = (offset == CHAN_OFFSET_LOWER) ? cch + 2 : cch - 2;
+			goto exit;
+
+		}
+	}
+#endif
+
+	rtw_warn_on(1);
+
+exit:
+	return t_cch;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_scch_by_cch_offset(u8 cch, u8 bw, u8 offset)
+{
+	return _rtw_get_scch_by_bcch_offset(rtw_is_2g_ch(cch) ? BAND_ON_24G : BAND_ON_5G, cch, bw, offset);
+}
+
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_scch_by_cch_opch(u8 cch, u8 bw, u8 opch)
+{
+	u8 offset = CHAN_OFFSET_NO_EXT;
+
+	if (opch > cch)
+		offset = CHAN_OFFSET_LOWER;
+	else if (opch < cch)
+		offset = CHAN_OFFSET_UPPER;
+
+	return rtw_get_scch_by_cch_offset(cch, bw, offset);
+}
+#endif
+
+u8 rtw_get_scch_by_bcch_offset(enum band_type band, u8 cch, u8 bw, u8 offset)
+{
+	return _rtw_get_scch_by_bcch_offset(band, cch, bw, offset);
+}
+
+/*
+ * Get center channel of smaller bandwidth by @param band, @param cch, @param bw, @param opch
+ * @band: the given band
+ * @cch: the given center channel
+ * @bw: the given bandwidth
+ * @opch: the given operating channel
+ *
+ * return center channel of smaller bandiwdth if valid, or 0
+ */
+u8 rtw_get_scch_by_bcch_opch(enum band_type band, u8 cch, u8 bw, u8 opch)
+{
+	u8 offset = CHAN_OFFSET_NO_EXT;
+
+	if (opch > cch)
+		offset = CHAN_OFFSET_LOWER;
+	else if (opch < cch)
+		offset = CHAN_OFFSET_UPPER;
+
+	return rtw_get_scch_by_bcch_offset(band, cch, bw, offset);
+}
+
+struct op_chs_ent_t {
+	u8 ch_num;
+	u8 *chs;
+};
+
+struct op_chs_ent_t op_chs_of_cch_2g_by_bw[] = {
+	{1, center_ch_2g},
+	{2, (u8 *)op_chs_of_cch_2g_40m},
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+struct op_chs_ent_t op_chs_of_cch_5g_by_bw[] = {
+	{1, center_ch_5g_20m},
+	{2, (u8 *)op_chs_of_cch_5g_40m},
+	{4, (u8 *)op_chs_of_cch_5g_80m},
+	{8, (u8 *)op_chs_of_cch_5g_160m},
+};
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+struct op_chs_ent_t op_chs_of_cch_6g_by_bw[] = {
+	{1, center_ch_6g_20m},
+	{2, (u8 *)op_chs_of_cch_6g_40m},
+	{4, (u8 *)op_chs_of_cch_6g_80m},
+	{8, (u8 *)op_chs_of_cch_6g_160m},
+};
+#endif
+
+u8 center_chs_2g_num(u8 bw)
+{
+	if (bw > CHANNEL_WIDTH_40)
+		return 0;
+
+	return center_chs_2g_by_bw[bw].ch_num;
+}
+
+u8 center_chs_2g(u8 bw, u8 id)
+{
+	if (bw > CHANNEL_WIDTH_40)
+		return 0;
+
+	if (id >= center_chs_2g_num(bw))
+		return 0;
+
+	return center_chs_2g_by_bw[bw].chs[id];
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+u8 center_chs_5g_num(u8 bw)
+{
+	if (bw > CHANNEL_WIDTH_160)
+		return 0;
+
+	return center_chs_5g_by_bw[bw].ch_num;
+}
+
+u8 center_chs_5g(u8 bw, u8 id)
+{
+	if (bw > CHANNEL_WIDTH_160)
+		return 0;
+
+	if (id >= center_chs_5g_num(bw))
+		return 0;
+
+	return center_chs_5g_by_bw[bw].chs[id];
+}
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+u8 center_chs_6g_num(u8 bw)
+{
+	if (bw > CHANNEL_WIDTH_160)
+		return 0;
+
+	return center_chs_6g_by_bw[bw].ch_num;
+}
+
+u8 center_chs_6g(u8 bw, u8 id)
+{
+	if (bw > CHANNEL_WIDTH_160)
+		return 0;
+
+	if (id >= center_chs_6g_num(bw))
+		return 0;
+
+	return center_chs_6g_by_bw[bw].chs[id];
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+u8 (*center_chs_num_of_band[BAND_MAX])(u8 bw) = {
+	[BAND_ON_24G] = center_chs_2g_num,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	[BAND_ON_5G] = center_chs_5g_num,
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	[BAND_ON_6G] = center_chs_6g_num,
+#endif
+};
+
+u8 (*center_chs_of_band[BAND_MAX])(u8 bw, u8 id) = {
+	[BAND_ON_24G] = center_chs_2g,
+#if CONFIG_IEEE80211_BAND_5GHZ
+	[BAND_ON_5G] = center_chs_5g,
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	[BAND_ON_6G] = center_chs_6g,
+#endif
+};
+
+/*
+ * Get available op channels by @param cch, @param bw
+ * @cch: the given center channel
+ * @bw: the given bandwidth
+ * @op_chs: the pointer to return pointer of op channel array
+ * @op_ch_num: the pointer to return pointer of op channel number
+ *
+ * return valid (1) or not (0)
+ */
+static u8 _rtw_get_op_chs_by_bcch_bw(enum band_type band, u8 cch, u8 bw, u8 **op_chs, u8 *op_ch_num)
+{
+	int i;
+	struct center_chs_ent_t *c_chs_ent = NULL;
+	struct op_chs_ent_t *op_chs_ent = NULL;
+	u8 valid = 1;
+
+	if (band == BAND_ON_24G
+		&& bw <= CHANNEL_WIDTH_40
+	) {
+		c_chs_ent = &center_chs_2g_by_bw[bw];
+		op_chs_ent = &op_chs_of_cch_2g_by_bw[bw];
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	} else if (band == BAND_ON_5G
+		&& bw <= CHANNEL_WIDTH_160
+	) {
+		c_chs_ent = &center_chs_5g_by_bw[bw];
+		op_chs_ent = &op_chs_of_cch_5g_by_bw[bw];
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	} else if (band == BAND_ON_6G
+		&& bw <= CHANNEL_WIDTH_160
+	) {
+		c_chs_ent = &center_chs_6g_by_bw[bw];
+		op_chs_ent = &op_chs_of_cch_6g_by_bw[bw];
+#endif
+
+	} else {
+		valid = 0;
+		goto exit;
+	}
+
+	for (i = 0; i < c_chs_ent->ch_num; i++)
+		if (cch == *(c_chs_ent->chs + i))
+			break;
+
+	if (i == c_chs_ent->ch_num) {
+		valid = 0;
+		goto exit;
+	}
+
+	*op_chs = op_chs_ent->chs + op_chs_ent->ch_num * i;
+	*op_ch_num = op_chs_ent->ch_num;
+
+exit:
+	return valid;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_op_chs_by_cch_bw(u8 cch, u8 bw, u8 **op_chs, u8 *op_ch_num)
+{
+	return _rtw_get_op_chs_by_bcch_bw(rtw_is_2g_ch(cch) ? BAND_ON_24G : BAND_ON_5G, cch, bw, op_chs, op_ch_num);
+}
+#endif
+
+u8 rtw_get_op_chs_by_bcch_bw(enum band_type band, u8 cch, u8 bw, u8 **op_chs, u8 *op_ch_num)
+{
+	return _rtw_get_op_chs_by_bcch_bw(band, cch, bw, op_chs, op_ch_num);
+}
+
+static u8 rtw_get_offsets_by_chbw_2g(u8 ch, u8 bw, u8 *r_offset, u8 *r_offset_num)
+{
+	u8 valid = 1;
+	u8 offset[2] = {CHAN_OFFSET_NO_EXT, CHAN_OFFSET_NO_EXT};
+	u8 offset_num = 1;
+
+	if (bw >= CHANNEL_WIDTH_80
+		|| ch < 1 || ch > 14
+	) {
+		valid = 0;
+		goto exit;
+	}
+
+	if (bw == CHANNEL_WIDTH_20)
+		goto exit;
+
+	if (ch >= 1 && ch <= 4)
+		offset[0] = CHAN_OFFSET_UPPER;
+	else if (ch >= 5 && ch <= 9) {
+		if (!r_offset_num) {
+			/* return single offset */
+			if (r_offset && (*r_offset == CHAN_OFFSET_UPPER || *r_offset == CHAN_OFFSET_LOWER))
+				offset[0] = *r_offset; /* both lower and upper is valid, obey input value */
+			else
+				offset[0] = CHAN_OFFSET_LOWER; /* default use primary upper */
+		} else {
+			offset_num = 2;
+			offset[0] = CHAN_OFFSET_LOWER;
+			offset[1] = CHAN_OFFSET_UPPER;
+		}
+	} else if (ch >= 10 && ch <= 13)
+		offset[0] = CHAN_OFFSET_LOWER;
+	else {
+		valid = 0; /* ch14 doesn't support 40MHz bandwidth */
+		goto exit;
+	}
+
+exit:
+	if (valid) {
+		if (r_offset) {
+			*r_offset = offset[0];
+			if (offset_num == 2)
+				*(r_offset + 1) = offset[1];
+		}
+		if (r_offset_num)
+			*r_offset_num = offset_num;
+	}
+	return valid;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtw_get_offset_by_chbw_5g(u8 ch, u8 bw, u8 *r_offset)
+{
+	u8 valid = 1;
+	u8 offset = CHAN_OFFSET_NO_EXT;
+
+	if (ch < 36 || ch > 177) {
+		valid = 0;
+		goto exit;
+	}
+
+	switch (ch) {
+	case 36:
+	case 44:
+	case 52:
+	case 60:
+	case 100:
+	case 108:
+	case 116:
+	case 124:
+	case 132:
+	case 140:
+	case 149:
+	case 157:
+	case 165:
+	case 173:
+		if (bw >= CHANNEL_WIDTH_40 && bw <= CHANNEL_WIDTH_160)
+			offset = CHAN_OFFSET_UPPER;
+		break;
+	case 40:
+	case 48:
+	case 56:
+	case 64:
+	case 104:
+	case 112:
+	case 120:
+	case 128:
+	case 136:
+	case 144:
+	case 153:
+	case 161:
+	case 169:
+	case 177:
+		if (bw >= CHANNEL_WIDTH_40 && bw <= CHANNEL_WIDTH_160)
+			offset = CHAN_OFFSET_LOWER;
+		break;
+	default:
+		valid = 0;
+		break;
+	}
+
+exit:
+	if (valid && r_offset)
+		*r_offset = offset;
+	return valid;
+}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static u8 rtw_get_offset_by_chbw_6g(u8 ch, u8 bw, u8 *r_offset)
+{
+	if (ch >= 1 && ch <= 253) {
+		u8 mod8 = ch % 8;
+
+		if (mod8 == 1) {
+			if (bw == CHANNEL_WIDTH_20)
+				*r_offset = CHAN_OFFSET_NO_EXT;
+			else
+				*r_offset = CHAN_OFFSET_UPPER;
+			return 1;
+		}
+		if (mod8 == 5) {
+			if (bw == CHANNEL_WIDTH_20)
+				*r_offset = CHAN_OFFSET_NO_EXT;
+			else
+				*r_offset = CHAN_OFFSET_LOWER;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+#endif
+
+u8 rtw_get_offset_by_bchbw(enum band_type band, u8 ch, u8 bw, u8 *r_offset)
+{
+	if (band == BAND_ON_24G) {
+		u8 offset[1] = {r_offset ? *r_offset : CHAN_OFFSET_NO_EXT};
+		u8 ret = rtw_get_offsets_by_chbw_2g(ch, bw, offset, NULL);
+
+		if (ret && r_offset)
+			*r_offset = offset[0];
+		return ret;
+	}
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G)
+		return rtw_get_offset_by_chbw_5g(ch, bw, r_offset);
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G)
+		return rtw_get_offset_by_chbw_6g(ch, bw, r_offset);
+#endif
+	return 0;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_offset_by_chbw(u8 ch, u8 bw, u8 *r_offset)
+{
+	return rtw_get_offset_by_bchbw(rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch, bw, r_offset);
+}
+#endif
+
+u8 rtw_get_offsets_by_bchbw(enum band_type band, u8 ch, u8 bw, u8 *r_offset, u8 *r_offset_num)
+{
+	u8 ret = 0;
+
+	if (band == BAND_ON_24G)
+		ret = rtw_get_offsets_by_chbw_2g(ch, bw, r_offset, r_offset_num);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G) {
+		ret = rtw_get_offset_by_chbw_5g(ch, bw, r_offset);
+		if (ret && r_offset_num)
+			*r_offset_num = 1;
+	}
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G) {
+		ret = rtw_get_offset_by_chbw_6g(ch, bw, r_offset);
+		if (ret && r_offset_num)
+			*r_offset_num = 1;
+	}
+#endif
+	return ret;
+}
+
+static u8 rtw_get_center_ch_2g(u8 ch, u8 bw, u8 offset)
+{
+	u8 cch = ch;
+
+	if (bw == CHANNEL_WIDTH_80) {
+		/* special case for 2.4G */
+		cch = 7;
+	} else if (bw == CHANNEL_WIDTH_40) {
+		if (offset == CHAN_OFFSET_UPPER)
+			cch = ch + 2;
+		else if (offset == CHAN_OFFSET_LOWER)
+			cch = ch - 2;
+
+	} else if (bw == CHANNEL_WIDTH_20
+		|| bw == CHANNEL_WIDTH_10
+		|| bw == CHANNEL_WIDTH_5
+	)
+		; /* same as ch */
+	else
+		rtw_warn_on(1);
+
+	return cch;
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static u8 rtw_get_center_ch_5g(u8 ch, u8 bw, u8 offset)
+{
+	u8 cch = ch;
+
+	if (bw == CHANNEL_WIDTH_160) {
+		if (ch % 4 == 0) {
+			if (ch >= 36 && ch <= 64)
+				cch = 50;
+			else if (ch >= 100 && ch <= 128)
+				cch = 114;
+		} else if (ch % 4 == 1) {
+			if (ch >= 149 && ch <= 177)
+				cch = 163;
+		}
+
+	} else if (bw == CHANNEL_WIDTH_80) {
+		if (ch % 4 == 0) {
+			if (ch >= 36 && ch <= 48)
+				cch = 42;
+			else if (ch >= 52 && ch <= 64)
+				cch = 58;
+			else if (ch >= 100 && ch <= 112)
+				cch = 106;
+			else if (ch >= 116 && ch <= 128)
+				cch = 122;
+			else if (ch >= 132 && ch <= 144)
+				cch = 138;
+		} else if (ch % 4 == 1) {
+			if (ch >= 149 && ch <= 161)
+				cch = 155;
+			else if (ch >= 165 && ch <= 177)
+				cch = 171;
+		}
+
+	} else if (bw == CHANNEL_WIDTH_40) {
+		if (offset == CHAN_OFFSET_UPPER)
+			cch = ch + 2;
+		else if (offset == CHAN_OFFSET_LOWER)
+			cch = ch - 2;
+
+	} else if (bw == CHANNEL_WIDTH_20
+		|| bw == CHANNEL_WIDTH_10
+		|| bw == CHANNEL_WIDTH_5
+	)
+		; /* same as ch */
+	else
+		rtw_warn_on(1);
+
+	return cch;
+}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static u8 rtw_get_center_ch_6g(u8 ch, u8 bw, u8 offset)
+{
+	static const u8 start[CHANNEL_WIDTH_MAX] = {
+		[CHANNEL_WIDTH_40] = 3,
+		[CHANNEL_WIDTH_80] = 7,
+		[CHANNEL_WIDTH_160] = 15,
+	};
+	static const u8 shift[CHANNEL_WIDTH_MAX] = {
+		[CHANNEL_WIDTH_40] = 3,
+		[CHANNEL_WIDTH_80] = 4,
+		[CHANNEL_WIDTH_160] = 5,
+	};
+	u8 cch = ch;
+
+	if (bw < CHANNEL_WIDTH_20 || bw > CHANNEL_WIDTH_160) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (bw == CHANNEL_WIDTH_20)
+		goto exit;
+
+	cch = (((ch - 1) >> shift[bw]) << shift[bw]) + start[bw];
+
+exit:
+	return cch;
+}
+#endif
+
+u8 rtw_get_center_ch_by_band(enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	if (band == BAND_ON_24G)
+		return rtw_get_center_ch_2g(ch, bw, offset);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G)
+		return rtw_get_center_ch_5g(ch, bw, offset);
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G)
+		return rtw_get_center_ch_6g(ch, bw, offset);
+#endif
+	return 0;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_center_ch(u8 ch, u8 bw, u8 offset)
+{
+	return rtw_get_center_ch_by_band(rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch, bw, offset);
+}
+#endif
+
+/**
+ * rtw_is_bchbw_grouped - test if the two ch settings can be grouped together
+ * @band_a: band of set a
+ * @ch_a: ch of set a
+ * @bw_a: bw of set a
+ * @offset_a: offset of set a
+ * @band_b: band of set b
+ * @ch_b: ch of set b
+ * @bw_b: bw of set b
+ * @offset_b: offset of set b
+ */
+bool _rtw_is_bchbw_grouped(enum band_type band_a, u8 ch_a, u8 bw_a, u8 offset_a
+	, enum band_type band_b, u8 ch_b, u8 bw_b, u8 offset_b)
+{
+	bool is_grouped = _FALSE;
+
+	if (band_a != band_b || ch_a != ch_b) {
+		/* band/ch is different */
+		goto exit;
+	}
+
+	if ((bw_a == CHANNEL_WIDTH_40 || bw_a == CHANNEL_WIDTH_80 || bw_a == CHANNEL_WIDTH_160)
+		   && (bw_b == CHANNEL_WIDTH_40 || bw_b == CHANNEL_WIDTH_80 || bw_b == CHANNEL_WIDTH_160)
+	) {
+		if (offset_a != offset_b)
+			goto exit;
+	}
+
+	is_grouped = _TRUE;
+
+exit:
+	return is_grouped;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY bool rtw_is_chbw_grouped(u8 ch_a, u8 bw_a, u8 offset_a, u8 ch_b, u8 bw_b, u8 offset_b)
+{
+	enum band_type band = BAND_ON_24G; /* unknown band, use same band for a & b */
+
+	return _rtw_is_bchbw_grouped(band, ch_a, bw_a, offset_a, band, ch_b, bw_b, offset_b);
+}
+#endif
+
+bool rtw_is_bchbw_grouped(enum band_type band_a, u8 ch_a, u8 bw_a, u8 offset_a
+	, enum band_type band_b, u8 ch_b, u8 bw_b, u8 offset_b)
+{
+	return _rtw_is_bchbw_grouped(band_a, ch_a, bw_a, offset_a, band_b, ch_b, bw_b, offset_b);
+}
+
+/**
+ * rtw_sync_bchbw - obey g_band, g_ch, adjust g_bw, g_offset, bw, offset
+ * @req_band: pointer of the request band, may be modified further
+ * @req_ch: pointer of the request ch, may be modified further
+ * @req_bw: pointer of the request bw, may be modified further
+ * @req_offset: pointer of the request offset, may be modified further
+ * @g_band: pointer of the ongoing group band
+ * @g_ch: pointer of the ongoing group ch
+ * @g_bw: pointer of the ongoing group bw, may be modified further
+ * @g_offset: pointer of the ongoing group offset, may be modified further
+ */
+void _rtw_sync_bchbw(enum band_type *req_band, u8 *req_ch, u8 *req_bw, u8 *req_offset
+	, enum band_type *g_band, u8 *g_ch, u8 *g_bw, u8 *g_offset)
+{
+	*req_band = *g_band;
+	*req_ch = *g_ch;
+
+	if (*req_bw == CHANNEL_WIDTH_80 && *g_band == BAND_ON_24G) {
+		/*2.4G ch, downgrade to 40Mhz */
+		*req_bw = CHANNEL_WIDTH_40;
+	}
+
+	switch (*req_bw) {
+	case CHANNEL_WIDTH_160:
+	case CHANNEL_WIDTH_80:
+	case CHANNEL_WIDTH_40:
+		if (*g_bw == CHANNEL_WIDTH_40 || *g_bw == CHANNEL_WIDTH_80 || *g_bw == CHANNEL_WIDTH_160)
+			*req_offset = *g_offset;
+		else if (*g_bw == CHANNEL_WIDTH_20)
+			rtw_get_offset_by_bchbw(*req_band, *req_ch, *req_bw, req_offset);
+
+		if (*req_offset == CHAN_OFFSET_NO_EXT) {
+			RTW_ERR("%s req %s BW without offset, down to 20MHz\n", __func__, ch_width_str(*req_bw));
+			rtw_warn_on(1);
+			*req_bw = CHANNEL_WIDTH_20;
+		}
+		break;
+	case CHANNEL_WIDTH_20:
+		*req_offset = CHAN_OFFSET_NO_EXT;
+		break;
+	default:
+		RTW_ERR("%s req unsupported BW:%u\n", __func__, *req_bw);
+		rtw_warn_on(1);
+	}
+
+	if (*req_bw > *g_bw) {
+		*g_bw = *req_bw;
+		*g_offset = *req_offset;
+	}
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY void rtw_sync_chbw(u8 *req_ch, u8 *req_bw, u8 *req_offset, u8 *g_ch, u8 *g_bw, u8 *g_offset)
+{
+	enum band_type band = rtw_is_2g_ch(*g_ch) ? BAND_ON_24G : BAND_ON_5G; /* follow g_ch's band */
+
+	_rtw_sync_bchbw(&band, req_ch, req_bw, req_offset, &band, g_ch, g_bw, g_offset);
+}
+#endif
+
+void rtw_sync_bchbw(enum band_type *req_band, u8 *req_ch, u8 *req_bw, u8 *req_offset
+	, enum band_type *g_band, u8 *g_ch, u8 *g_bw, u8 *g_offset)
+{
+	_rtw_sync_bchbw(req_band, req_ch, req_bw, req_offset, g_band, g_ch, g_bw, g_offset);
+}
+
+static int rtw_ch2freq_2g(int ch)
+{
+	if (ch >= 1 && ch <= 14) {
+		if (ch == 14)
+			return 2484;
+		else if (ch < 14)
+			return 2407 + ch * 5;
+	}
+
+	return 0; /* not supported */
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static int rtw_ch2freq_5g(int ch)
+{
+	if (ch >= 36 && ch <= 177)
+		return 5000 + ch * 5;
+
+	return 0; /* not supported */
+}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static int rtw_ch2freq_6g(int ch)
+{
+	if (ch >= 1 && ch <= 253)
+		return 5950 + ch * 5;
+
+	return 0; /* not supported */
+}
+#endif
+
+int rtw_bch2freq(enum band_type band, int ch)
+{
+	if (band == BAND_ON_24G)
+		return rtw_ch2freq_2g(ch);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G)
+		return rtw_ch2freq_5g(ch);
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G)
+		return rtw_ch2freq_6g(ch);
+#endif
+	return 0;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY int rtw_ch2freq(int ch)
+{
+	return rtw_bch2freq(rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch);
+}
+#endif
+
+int rtw_freq2ch(int freq)
+{
+	/* see 802.11 17.3.8.3.2 and Annex J */
+	if (freq == 2484)
+		return 14;
+	else if (freq < 2484)
+		return (freq - 2407) / 5;
+	else if (freq >= 4910 && freq <= 4980)
+		return (freq - 4000) / 5;
+	else if (freq >= 5000 && freq < 5950)
+		return (freq - 5000) / 5;
+	else if (freq >= 5950 && freq <= 7215)
+		return (freq - 5950) / 5;
+	else if (freq >= 58320 && freq <= 64800)
+		return (freq - 56160) / 2160;
+	else
+		return 0;
+}
+
+enum band_type rtw_freq2band(int freq)
+{
+	if (freq <= 2484)
+		return BAND_ON_24G;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (freq >= 5000 && freq < 5950)
+		return BAND_ON_5G;
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (freq >= 5950 && freq <= 7215)
+		return BAND_ON_6G;
+#endif
+	else
+		return BAND_MAX;
+}
+
+enum channel_width rtw_frange_to_bw(u32 hi, u32 lo)
+{
+	u32 width = hi - lo;
+
+	switch (width) {
+	case 160:
+		return CHANNEL_WIDTH_160;
+	case 80:
+		return CHANNEL_WIDTH_80;
+	case 40:
+		return CHANNEL_WIDTH_40;
+	case 20:
+		return CHANNEL_WIDTH_20;
+	case 10:
+		return CHANNEL_WIDTH_10;
+	case 5:
+		return CHANNEL_WIDTH_5;
+	default:
+		return CHANNEL_WIDTH_MAX;
+	}
+}
+
+bool rtw_freq_consecutive(int a, int b)
+{
+	enum band_type band_a, band_b;
+
+	band_a = rtw_freq2band(a);
+	if (band_a == BAND_MAX)
+		return 0;
+	band_b = rtw_freq2band(b);
+	if (band_b == BAND_MAX || band_a != band_b)
+		return 0;
+
+	switch (band_a) {
+	case BAND_ON_24G:
+		return rtw_abs(a - b) == 5;
+	case BAND_ON_5G:
+#if CONFIG_IEEE80211_BAND_6GHZ
+	case BAND_ON_6G:
+#endif
+		return rtw_abs(a - b) == 20;
+	default:
+		return 0;
+	}
+}
+
+bool rtw_bcchbw_to_freq_range(enum band_type band, u8 c_ch, u8 bw, u32 *hi, u32 *lo)
+{
+	u32 freq;
+	u32 hi_ret = 0, lo_ret = 0;
+	bool valid = _FALSE;
+
+	if (hi)
+		*hi = 0;
+	if (lo)
+		*lo = 0;
+
+	freq = rtw_bch2freq(band, c_ch);
+
+	if (!freq) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (bw == CHANNEL_WIDTH_160) {
+		hi_ret = freq + 80;
+		lo_ret = freq - 80;
+	} else if (bw == CHANNEL_WIDTH_80) {
+		hi_ret = freq + 40;
+		lo_ret = freq - 40;
+	} else if (bw == CHANNEL_WIDTH_40) {
+		hi_ret = freq + 20;
+		lo_ret = freq - 20;
+	} else if (bw == CHANNEL_WIDTH_20) {
+		hi_ret = freq + 10;
+		lo_ret = freq - 10;
+	} else
+		rtw_warn_on(1);
+
+	if (hi)
+		*hi = hi_ret;
+	if (lo)
+		*lo = lo_ret;
+
+	valid = _TRUE;
+
+exit:
+	return valid;
+}
+
+bool rtw_bchbw_to_freq_range(enum band_type band, u8 ch, u8 bw, u8 offset, u32 *hi, u32 *lo)
+{
+	return rtw_bcchbw_to_freq_range(band
+		, rtw_get_center_ch_by_band(band, ch, bw, offset)
+		, bw, hi, lo);
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY bool rtw_chbw_to_freq_range(u8 ch, u8 bw, u8 offset, u32 *hi, u32 *lo)
+{
+	return rtw_bchbw_to_freq_range(rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch, bw, offset, hi, lo);
+}
+#endif
+
+const char *const _ch_width_str[CHANNEL_WIDTH_MAX] = {
+	[CHANNEL_WIDTH_20]		= "20MHz",
+	[CHANNEL_WIDTH_40]		= "40MHz",
+	[CHANNEL_WIDTH_80]		= "80MHz",
+	[CHANNEL_WIDTH_160]		= "160MHz",
+	[CHANNEL_WIDTH_80_80]	= "80_80MHz",
+	[CHANNEL_WIDTH_5]		= "5MHz",
+	[CHANNEL_WIDTH_10]		= "10MHz",
+};
+
+const u8 _ch_width_to_bw_cap[CHANNEL_WIDTH_MAX] = {
+	[CHANNEL_WIDTH_20]		= BW_CAP_20M,
+	[CHANNEL_WIDTH_40]		= BW_CAP_40M,
+	[CHANNEL_WIDTH_80]		= BW_CAP_80M,
+	[CHANNEL_WIDTH_160]		= BW_CAP_160M,
+	[CHANNEL_WIDTH_80_80]	= BW_CAP_80_80M,
+	[CHANNEL_WIDTH_5]		= BW_CAP_5M,
+	[CHANNEL_WIDTH_10]		= BW_CAP_10M,
+};
+
+const char *const _rtw_band_str[] = {
+	[BAND_ON_24G]	= "2.4G",
+	[BAND_ON_5G]	= "5G",
+	[BAND_ON_6G]	= "6G",
+	[BAND_MAX]		= "BAND_MAX",
+};
+
+const u8 _band_to_band_cap[] = {
+	[BAND_ON_24G]	= BAND_CAP_2G,
+	[BAND_ON_5G]	= BAND_CAP_5G,
+	[BAND_ON_6G]	= BAND_CAP_6G,
+	[BAND_MAX]		= 0,
+};
+
+const char *const _opc_bw_str[OPC_BW_NUM] = {
+	[OPC_BW20]		= "20M ",
+	[OPC_BW40PLUS]	= "40M+",
+	[OPC_BW40MINUS]	= "40M-",
+	[OPC_BW40]		= "40M ",
+	[OPC_BW80]		= "80M ",
+	[OPC_BW160]		= "160M ",
+	[OPC_BW80P80]	= "80+80M ",
+};
+
+const u8 _opc_bw_to_ch_width[OPC_BW_NUM] = {
+	[OPC_BW20]		= CHANNEL_WIDTH_20,
+	[OPC_BW40PLUS]	= CHANNEL_WIDTH_40,
+	[OPC_BW40MINUS]	= CHANNEL_WIDTH_40,
+	[OPC_BW40]		= CHANNEL_WIDTH_40,
+	[OPC_BW80]		= CHANNEL_WIDTH_80,
+	[OPC_BW160]		= CHANNEL_WIDTH_160,
+	[OPC_BW80P80]	= CHANNEL_WIDTH_80_80,
+};
+
+/* global operating class database */
+#define OP_CLASS_ENT(_class, _band, _bw, _len, arg...) \
+	{.class_id = _class, .band = _band, .bw = _bw, .len_ch_attr = (uint8_t[_len + 1]) {_len, ##arg},}
+
+/* 802.11-2020, 802.11ax-2021 Table E-4, partial */
+const struct op_class_t global_op_class[] = {
+	/* 2G ch1~13, 20M */
+	OP_CLASS_ENT(81,	BAND_ON_24G,	OPC_BW20,		13,	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13),
+	/* 2G ch14, 20M */
+	OP_CLASS_ENT(82,	BAND_ON_24G,	OPC_BW20,		1,	14),
+	/* 2G, 40M */
+	OP_CLASS_ENT(83,	BAND_ON_24G,	OPC_BW40PLUS,	9,	1, 2, 3, 4, 5, 6, 7, 8, 9),
+	OP_CLASS_ENT(84,	BAND_ON_24G,	OPC_BW40MINUS,	9,	5, 6, 7, 8, 9, 10, 11, 12, 13),
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	/* 5G band 1, 20M & 40M */
+	OP_CLASS_ENT(115,	BAND_ON_5G,		OPC_BW20,		4,	36, 40, 44, 48),
+	OP_CLASS_ENT(116,	BAND_ON_5G,		OPC_BW40PLUS,	2,	36, 44),
+	OP_CLASS_ENT(117,	BAND_ON_5G,		OPC_BW40MINUS,	2,	40, 48),
+	/* 5G band 2, 20M & 40M */
+	OP_CLASS_ENT(118,	BAND_ON_5G,		OPC_BW20,		4,	52, 56, 60, 64),
+	OP_CLASS_ENT(119,	BAND_ON_5G,		OPC_BW40PLUS,	2,	52, 60),
+	OP_CLASS_ENT(120,	BAND_ON_5G,		OPC_BW40MINUS,	2,	56, 64),
+	/* 5G band 3, 20M & 40M */
+	OP_CLASS_ENT(121,	BAND_ON_5G,		OPC_BW20,		12,	100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144),
+	OP_CLASS_ENT(122,	BAND_ON_5G,		OPC_BW40PLUS,	6,	100, 108, 116, 124, 132, 140),
+	OP_CLASS_ENT(123,	BAND_ON_5G,		OPC_BW40MINUS,	6,	104, 112, 120, 128, 136, 144),
+	/* 5G band 4, 20M & 40M */
+	OP_CLASS_ENT(124,	BAND_ON_5G,		OPC_BW20,		4,	149, 153, 157, 161),
+	OP_CLASS_ENT(125,	BAND_ON_5G,		OPC_BW20,		8,	149, 153, 157, 161, 165, 169, 173, 177),
+	OP_CLASS_ENT(126,	BAND_ON_5G,		OPC_BW40PLUS,	4,	149, 157, 165, 173),
+	OP_CLASS_ENT(127,	BAND_ON_5G,		OPC_BW40MINUS,	4,	153, 161, 169, 177),
+	/* 5G, 80M & 160M */
+	OP_CLASS_ENT(128,	BAND_ON_5G,		OPC_BW80,		28,	36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177),
+	OP_CLASS_ENT(129,	BAND_ON_5G,		OPC_BW160,		24,	36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 149, 153, 157, 161, 165, 169, 173, 177),
+	#if 0 /* TODO */
+	/* 5G, 80+80M */
+	OP_CLASS_ENT(130,	BAND_ON_5G,		OPC_BW80P80,	28,	36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, 169, 173, 177),
+	#endif
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* 6G, 20M */
+	OP_CLASS_ENT(131,	BAND_ON_6G,		OPC_BW20,		59,	1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93
+															, 97, 101, 105, 109, 113, 117
+															, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189
+															, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233),
+	/* 6G, 40M */
+	OP_CLASS_ENT(132,	BAND_ON_6G,		OPC_BW40,		58,	1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93
+															, 97, 101, 105, 109, 113, 117
+															, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189
+															, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229),
+	/* 6G, 80M */
+	OP_CLASS_ENT(133,	BAND_ON_6G,		OPC_BW80,		56,	1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93
+															, 97, 101, 105, 109, 113, 117
+															, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189
+															, 193, 197, 201, 205, 209, 213, 217, 221),
+	/* 6G, 160M */
+	OP_CLASS_ENT(134,	BAND_ON_6G,		OPC_BW160,		56,	1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93
+															, 97, 101, 105, 109, 113, 117
+															, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189
+															, 193, 197, 201, 205, 209, 213, 217, 221),
+	#if 0 /* TODO */
+	/* 6G, 80+80M */
+	OP_CLASS_ENT(135,	BAND_ON_6G,		OPC_BW80P80,	56,	1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69, 73, 77, 81, 85, 89, 93
+															, 97, 101, 105, 109, 113, 117
+															, 121, 125, 129, 133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185, 189
+															, 193, 197, 201, 205, 209, 213, 217, 221),
+	/* 6G, 20M, ch2 */
+	OP_CLASS_ENT(136,	BAND_ON_6G,		OPC_BW20,		1,	2),
+	#endif
+#endif
+};
+
+const int global_op_class_num = sizeof(global_op_class) / sizeof(struct op_class_t);
+
+const struct op_class_t *get_global_op_class_by_id(u8 gid)
+{
+	int i;
+
+	for (i = 0; i < global_op_class_num; i++)
+		if (global_op_class[i].class_id == gid)
+			break;
+
+	return i < global_op_class_num ? &global_op_class[i] : NULL;
+}
+
+bool is_valid_global_op_class_id(u8 gid)
+{
+	return get_global_op_class_by_id(gid) ? 1 : 0;
+}
+
+bool is_valid_global_op_class_ch(const struct op_class_t *opc, u8 ch)
+{
+	int array_idx;
+	int i;
+
+	if (opc < global_op_class
+		|| (((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t)
+	) {
+		RTW_ERR("Invalid opc pointer:%p (global_op_class:%p, sizeof(struct op_class_t):%zu, %zu)\n"
+			, opc, global_op_class, sizeof(struct op_class_t), (((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t));
+		return 0;
+	}
+
+	array_idx = (((u8 *)opc) - ((u8 *)global_op_class)) / sizeof(struct op_class_t);
+
+	for (i = 0; i < OPC_CH_LIST_LEN(global_op_class[array_idx]); i++)
+		if (OPC_CH_LIST_CH(global_op_class[array_idx], i) == ch)
+			break;
+
+	return i < OPC_CH_LIST_LEN(global_op_class[array_idx]);
+}
+
+/* -2: logic error, -1: error, 0: is already BW20 */
+s16 get_sub_op_class(u8 gid, u8 ch)
+{
+	const struct op_class_t *opc = get_global_op_class_by_id(gid);
+	int i;
+	enum channel_width bw;
+
+	if (!opc)
+		return -1;
+
+	if (!is_valid_global_op_class_ch(opc, ch)) {
+		return -1;
+	}
+
+	if (opc->bw == OPC_BW20)
+		return 0;
+
+	bw = opc_bw_to_ch_width(opc->bw);
+
+	for (i = 0; i < global_op_class_num; i++) {
+		if (bw != opc_bw_to_ch_width(global_op_class[i].bw) + 1)
+			continue;
+		if (is_valid_global_op_class_ch(&global_op_class[i], ch))
+			break;
+	}
+
+	return i < global_op_class_num ? global_op_class[i].class_id : -2;
+}
+
+static void dump_op_class_ch_title(void *sel)
+{
+	RTW_PRINT_SEL(sel, "%-5s %-4s %-7s ch_list\n"
+		, "class", "band", "bw");
+}
+
+static void dump_global_op_class_ch_single(void *sel, u8 gid)
+{
+	u8 i;
+
+	RTW_PRINT_SEL(sel, "%5u %4s %7s"
+		, global_op_class[gid].class_id
+		, band_str(global_op_class[gid].band)
+		, opc_bw_str(global_op_class[gid].bw));
+
+	for (i = 0; i < OPC_CH_LIST_LEN(global_op_class[gid]); i++)
+		_RTW_PRINT_SEL(sel, " %u", OPC_CH_LIST_CH(global_op_class[gid], i));
+
+	_RTW_PRINT_SEL(sel, "\n");
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static bool dbg_global_op_class_validate(u8 gid)
+{
+	u8 i;
+	u8 ch, bw, offset, cch;
+	bool ret = 1;
+
+	switch (global_op_class[gid].bw) {
+	case OPC_BW20:
+		bw = CHANNEL_WIDTH_20;
+		offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case OPC_BW40PLUS:
+		bw = CHANNEL_WIDTH_40;
+		offset = CHAN_OFFSET_UPPER;
+		break;
+	case OPC_BW40MINUS:
+		bw = CHANNEL_WIDTH_40;
+		offset = CHAN_OFFSET_LOWER;
+		break;
+	case OPC_BW40:
+		bw = CHANNEL_WIDTH_40;
+		offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case OPC_BW80:
+		bw = CHANNEL_WIDTH_80;
+		offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case OPC_BW160:
+		bw = CHANNEL_WIDTH_160;
+		offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case OPC_BW80P80: /* TODO */
+	default:
+		RTW_ERR("%s class:%u unsupported opc_bw:%u\n"
+			, __func__, global_op_class[gid].class_id, global_op_class[gid].bw);
+		ret = 0;
+		goto exit;
+	}
+
+	for (i = 0; i < OPC_CH_LIST_LEN(global_op_class[gid]); i++) {
+		u8 *op_chs;
+		u8 op_ch_num;
+		u8 k;
+
+		ch = OPC_CH_LIST_CH(global_op_class[gid], i);
+		cch = rtw_get_center_ch_by_band(global_op_class[gid].band, ch ,bw, offset);
+		if (!cch) {
+			RTW_ERR("%s can't get cch from class:%u ch:%u\n"
+				, __func__, global_op_class[gid].class_id, ch);
+			ret = 0;
+			continue;
+		}
+
+		if (!rtw_get_op_chs_by_bcch_bw(global_op_class[gid].band, cch, bw, &op_chs, &op_ch_num)) {
+			RTW_ERR("%s can't get op chs from class:%u cch:%u\n"
+				, __func__, global_op_class[gid].class_id, cch);
+			ret = 0;
+			continue;
+		}
+
+		for (k = 0; k < op_ch_num; k++) {
+			if (*(op_chs + k) == ch)
+				break;
+		}
+		if (k >= op_ch_num) {
+			RTW_ERR("%s can't get ch:%u from op_chs class:%u cch:%u\n"
+				, __func__, ch, global_op_class[i].class_id, cch);
+			ret = 0;
+		}
+	}
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+void dump_global_op_class(void *sel)
+{
+	u8 i;
+
+	dump_op_class_ch_title(sel);
+
+	for (i = 0; i < global_op_class_num; i++)
+		dump_global_op_class_ch_single(sel, i);
+}
+
+u8 _rtw_get_op_class_by_bchbw(enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	int i;
+	u8 gid = 0; /* invalid */
+
+	switch (bw) {
+	case CHANNEL_WIDTH_20:
+	case CHANNEL_WIDTH_40:
+	case CHANNEL_WIDTH_80:
+	case CHANNEL_WIDTH_160:
+	#if 0 /* TODO */
+	case CHANNEL_WIDTH_80_80:
+	#endif
+		break;
+	default:
+		goto exit;
+	}
+
+	for (i = 0; i < global_op_class_num; i++) {
+		if (band != global_op_class[i].band)
+			continue;
+
+		if (opc_bw_to_ch_width(global_op_class[i].bw) != bw)
+			continue;
+
+		if ((global_op_class[i].bw == OPC_BW40PLUS
+				&& offset != CHAN_OFFSET_UPPER)
+			|| (global_op_class[i].bw == OPC_BW40MINUS
+				&& offset != CHAN_OFFSET_LOWER)
+		)
+			continue;
+
+		if (is_valid_global_op_class_ch(&global_op_class[i], ch))
+			goto get;
+	}
+
+get:
+	if (i < global_op_class_num) {
+		#if 0 /* TODO */
+		if (bw == CHANNEL_WIDTH_80_80) {
+			/* search another ch */
+			if (!is_valid_global_op_class_ch(&global_op_class[i], ch2))
+				goto exit;
+		}
+		#endif
+
+		gid = global_op_class[i].class_id;
+	}
+
+exit:
+	return gid;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_op_class_by_chbw(u8 ch, u8 bw, u8 offset)
+{
+	enum band_type band = BAND_MAX;
+
+	if (rtw_is_2g_ch(ch))
+		band = BAND_ON_24G;
+	else if (rtw_is_5g_ch(ch))
+		band = BAND_ON_5G;
+	else
+		return 0; /* invalid */
+
+	return _rtw_get_op_class_by_bchbw(band, ch, bw, offset);
+}
+#endif
+
+u8 rtw_get_op_class_by_bchbw(enum band_type band, u8 ch, u8 bw, u8 offset)
+{
+	return _rtw_get_op_class_by_bchbw(band, ch, bw, offset);
+}
+
+u8 rtw_get_bw_offset_by_op_class_ch(u8 gid, u8 ch, u8 *bw, u8 *offset)
+{
+	const struct op_class_t *opc;
+	u8 valid = 0;
+	int i;
+
+	opc = get_global_op_class_by_id(gid);
+	if (!opc)
+		goto exit;
+
+	*bw = opc_bw_to_ch_width(opc->bw);
+
+	if (opc->bw == OPC_BW40PLUS)
+		*offset = CHAN_OFFSET_UPPER;
+	else if (opc->bw == OPC_BW40MINUS)
+		*offset = CHAN_OFFSET_LOWER;
+
+	if (rtw_get_offset_by_bchbw(opc->band, ch, *bw, offset))
+		valid = 1;
+
+exit:
+	return valid;
+}
+
+#ifdef CONFIG_ECSA_PHL
+int alink_get_supported_op_class(struct _ADAPTER_LINK *padapter_link, u8 *op_set, int len)
+{
+	_adapter *padapter = padapter_link->adapter;
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	int match = 0, i = 0, j, k = 0;
+	const struct op_class_t *cl;
+	u8 cur_op_class;
+	u8 *ch;
+
+	cur_op_class = rtw_get_op_class_by_bchbw(pmlmeext->chandef.band,
+						pmlmeext->chandef.chan,
+						pmlmeext->chandef.bw,
+						pmlmeext->chandef.offset);
+
+	if (cur_op_class && k < len) {
+		/* current op class SHALL be the 1st supported op class */
+		*op_set = cur_op_class;
+		k++;
+	}
+
+	for (i = 0; i < global_op_class_num; i++) {
+		cl = &global_op_class[i];
+		ch = cl->len_ch_attr;
+
+		for (j = 0; j < ch[0]; j++) {
+			if ((match = rtw_chset_search_bch(chset, cl->band, ch[j+1])) == -1) //
+				break;
+		}
+
+		if (match != -1 && k < len)
+			op_set[k++] = cl->class_id;
+	}
+	return (k > len ? len : k);
+}
+
+int get_supported_op_class(_adapter *padapter, u8 *op_set, int len)
+{
+	return alink_get_supported_op_class(GET_PRIMARY_LINK(padapter), op_set, len);
+}
+#endif /* CONFIG_ECSA_PHL */
+
+static struct op_class_pref_t *opc_pref_alloc(u8 class_id)
+{
+	int i, j;
+	struct op_class_pref_t *opc_pref = NULL;
+	u8 ch_num;
+
+	for (i = 0; i < global_op_class_num; i++)
+		if (global_op_class[i].class_id == class_id)
+			break;
+
+	if (i >= global_op_class_num)
+		goto exit;
+
+	ch_num = OPC_CH_LIST_LEN(global_op_class[i]);
+	opc_pref = rtw_zmalloc(sizeof(*opc_pref) + (sizeof(struct op_ch_t) * ch_num));
+	if (!opc_pref)
+		goto exit;
+
+	opc_pref->class_id = global_op_class[i].class_id;
+	opc_pref->band = global_op_class[i].band;
+	opc_pref->bw = global_op_class[i].bw;
+
+	for (j = 0; j < OPC_CH_LIST_LEN(global_op_class[i]); j++) {
+		opc_pref->chs[j].ch = OPC_CH_LIST_CH(global_op_class[i], j);
+		opc_pref->chs[j].static_non_op = 1;
+		opc_pref->chs[j].no_ir = 1;
+		opc_pref->chs[j].max_txpwr = UNSPECIFIED_MBM;
+	}
+	opc_pref->ch_num = ch_num;
+
+exit:
+	return opc_pref;
+}
+
+static void opc_pref_free(struct op_class_pref_t *opc_pref)
+{
+	rtw_mfree(opc_pref, sizeof(*opc_pref) + (sizeof(struct op_ch_t) * opc_pref->ch_num));
+}
+
+enum band_type rtw_get_band_by_op_class(u8 op_class)
+{
+	int array_idx;
+	const struct op_class_t *opc = get_global_op_class_by_id(op_class);
+
+	if (opc < global_op_class
+		|| (((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t)
+	) {
+		RTW_ERR("Invalid opc pointer:%p (global_op_class:%p, sizeof(struct op_class_t):%zu, %zu)\n"
+			, opc, global_op_class, sizeof(struct op_class_t),
+			(((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t));
+		return BAND_MAX;
+	}
+
+	array_idx = (((u8 *)opc) - ((u8 *)global_op_class)) / sizeof(struct op_class_t);
+
+	return global_op_class[array_idx].band;
+}
+
+int rtw_rfctl_op_class_pref_init(struct rf_ctl_t *rfctl, u8 band_bmp, u8 bw_bmp[])
+{
+	u8 bw;
+	struct op_class_pref_t *opc_pref;
+	int i;
+	u8 op_class_num = 0;
+	int ret = _FAIL;
+
+	rfctl->spt_op_class_ch = rtw_zmalloc(sizeof(struct op_class_pref_t *) * global_op_class_num);
+	if (!rfctl->spt_op_class_ch) {
+		RTW_ERR("%s alloc rfctl->spt_op_class_ch fail\n", __func__);
+		goto exit;
+	}
+
+	for (i = 0; i < global_op_class_num; i++) {
+		#ifdef CONFIG_RTW_DEBUG
+		rtw_warn_on(!dbg_global_op_class_validate(i));
+		#endif
+
+		if (!(band_bmp & band_to_band_cap(global_op_class[i].band)))
+			continue;
+
+		bw = opc_bw_to_ch_width(global_op_class[i].bw);
+		if (bw == CHANNEL_WIDTH_MAX
+			|| bw == CHANNEL_WIDTH_80_80 /* TODO */
+		)
+			continue;
+
+		if (!(bw_bmp[global_op_class[i].band] & ch_width_to_bw_cap(bw)))
+			continue;
+
+		opc_pref = opc_pref_alloc(global_op_class[i].class_id);
+		if (!opc_pref) {
+			RTW_ERR("%s opc_pref_alloc(%u) fail\n", __func__, global_op_class[i].class_id);
+			goto exit;
+		}
+
+		if (opc_pref->ch_num) {
+			rfctl->spt_op_class_ch[i] = opc_pref;
+			op_class_num++;
+		} else
+			opc_pref_free(opc_pref);
+	}
+
+	rfctl->cap_spt_op_class_num = op_class_num;
+	ret = _SUCCESS;
+
+exit:
+	if (ret != _SUCCESS)
+		rtw_rfctl_op_class_pref_deinit(rfctl);
+
+	return ret;
+}
+
+void rtw_rfctl_op_class_pref_deinit(struct rf_ctl_t *rfctl)
+{
+	int i;
+
+	if (!rfctl->spt_op_class_ch)
+		return;
+
+	for (i = 0; i < global_op_class_num; i++) {
+		if (rfctl->spt_op_class_ch[i]) {
+			opc_pref_free(rfctl->spt_op_class_ch[i]);
+			rfctl->spt_op_class_ch[i] = NULL;
+		}
+	}
+
+	rtw_mfree(rfctl->spt_op_class_ch, sizeof(struct op_class_pref_t *) * global_op_class_num);
+	rfctl->spt_op_class_ch = NULL;
+}
+
+void op_class_pref_apply_regulatory(struct rf_ctl_t *rfctl, u8 reason)
+{
+	struct rtw_chset *chset = &rfctl->chset;
+	u8 ch, bw, offset, cch;
+	struct op_class_pref_t *opc_pref;
+	int i, j;
+	u8 reg_op_class_num = 0;
+	u8 op_class_num = 0;
+
+	for (i = 0; i < global_op_class_num; i++) {
+		if (!rfctl->spt_op_class_ch[i])
+			continue;
+		opc_pref = rfctl->spt_op_class_ch[i];
+
+		/* reset all channel */
+		for (j = 0; j < opc_pref->ch_num; j++) {
+			if (reason >= REG_CHANGE)
+				opc_pref->chs[j].static_non_op = 1;
+			if (reason != REG_TXPWR_CHANGE)
+				opc_pref->chs[j].no_ir = 1;
+			if (reason >= REG_TXPWR_CHANGE)
+				opc_pref->chs[j].max_txpwr = UNSPECIFIED_MBM;
+		}
+		if (reason >= REG_CHANGE)
+			opc_pref->op_ch_num = 0;
+		if (reason != REG_TXPWR_CHANGE)
+			opc_pref->ir_ch_num = 0;
+
+		switch (opc_pref->bw) {
+		case OPC_BW20:
+			bw = CHANNEL_WIDTH_20;
+			offset = CHAN_OFFSET_NO_EXT;
+			break;
+		case OPC_BW40PLUS:
+			bw = CHANNEL_WIDTH_40;
+			offset = CHAN_OFFSET_UPPER;
+			break;
+		case OPC_BW40MINUS:
+			bw = CHANNEL_WIDTH_40;
+			offset = CHAN_OFFSET_LOWER;
+			break;
+		case OPC_BW40:
+			bw = CHANNEL_WIDTH_40;
+			offset = CHAN_OFFSET_NO_EXT;
+			break;
+		case OPC_BW80:
+			bw = CHANNEL_WIDTH_80;
+			offset = CHAN_OFFSET_NO_EXT;
+			break;
+		case OPC_BW160:
+			bw = CHANNEL_WIDTH_160;
+			offset = CHAN_OFFSET_NO_EXT;
+			break;
+		case OPC_BW80P80: /* TODO */
+		default:
+			continue;
+		}
+
+		if (!RFCTL_REG_EN_11AC(rfctl)
+			&& (bw == CHANNEL_WIDTH_80 || bw == CHANNEL_WIDTH_160))
+			continue;
+
+		for (j = 0; j < opc_pref->ch_num; j++) {
+			u8 *op_chs;
+			u8 op_ch_num;
+			u8 k, l;
+			int chset_idx;
+
+			ch = opc_pref->chs[j].ch;
+
+			if (reason >= REG_TXPWR_CHANGE)
+				opc_pref->chs[j].max_txpwr = rtw_rfctl_get_reg_max_txpwr_mbm(rfctl, opc_pref->band, ch, bw, offset, 1);
+
+			if (reason == REG_TXPWR_CHANGE)
+				continue;
+
+			cch = rtw_get_center_ch_by_band(opc_pref->band, ch ,bw, offset);
+			if (!cch)
+				continue;
+
+			if (!rtw_get_op_chs_by_bcch_bw(opc_pref->band, cch, bw, &op_chs, &op_ch_num))
+				continue;
+
+			for (k = 0, l = 0; k < op_ch_num; k++) {
+				chset_idx = rtw_chset_search_bch(chset, opc_pref->band, *(op_chs + k));
+				if (chset_idx == -1)
+					break;
+				if (bw >= CHANNEL_WIDTH_40) {
+					if ((chset->chs[chset_idx].flags & RTW_CHF_NO_HT40U) && k % 2 == 0)
+						break;
+					if ((chset->chs[chset_idx].flags & RTW_CHF_NO_HT40L) && k % 2 == 1)
+						break;
+				}
+				if (bw >= CHANNEL_WIDTH_80 && (chset->chs[chset_idx].flags & RTW_CHF_NO_80MHZ))
+					break;
+				if (bw >= CHANNEL_WIDTH_160 && (chset->chs[chset_idx].flags & RTW_CHF_NO_160MHZ))
+					break;
+				if ((chset->chs[chset_idx].flags & RTW_CHF_DFS) && !rtw_rfctl_radar_detect_supported(rfctl))
+					continue;
+				if (chset->chs[chset_idx].flags & RTW_CHF_NO_IR)
+					continue;
+				l++;
+			}
+			if (k < op_ch_num)
+				continue;
+
+			if (reason >= REG_CHANGE) {
+				opc_pref->chs[j].static_non_op = 0;
+				opc_pref->op_ch_num++;
+			}
+
+			if (l >= op_ch_num) {
+				opc_pref->chs[j].no_ir = 0;
+				opc_pref->ir_ch_num++;
+			}
+		}
+
+		if (opc_pref->op_ch_num)
+			reg_op_class_num++;
+		if (opc_pref->ir_ch_num)
+			op_class_num++;
+	}
+
+	rfctl->reg_spt_op_class_num = reg_op_class_num;
+	rfctl->cur_spt_op_class_num = op_class_num;
+}
+
+static void dump_opc_pref_single(void *sel, struct op_class_pref_t *opc_pref, bool show_snon_ocp, bool show_no_ir, bool detail)
+{
+	u8 i;
+	u8 ch_num = 0;
+
+	if (!show_snon_ocp && !opc_pref->op_ch_num)
+		return;
+	if (!show_no_ir && !opc_pref->ir_ch_num)
+		return;
+
+	RTW_PRINT_SEL(sel, "%5u %4s %7s"
+		, opc_pref->class_id
+		, band_str(opc_pref->band)
+		, opc_bw_str(opc_pref->bw));
+	for (i = 0; i < opc_pref->ch_num; i++) {
+		if ((show_snon_ocp || !opc_pref->chs[i].static_non_op)
+			&& (show_no_ir || !opc_pref->chs[i].no_ir)
+		) {
+			if (detail)
+				_RTW_PRINT_SEL(sel, " %4u", opc_pref->chs[i].ch);
+			else
+				_RTW_PRINT_SEL(sel, " %u", opc_pref->chs[i].ch);
+		}
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+
+	if (!detail)
+		return;
+
+	RTW_PRINT_SEL(sel, "                  ");
+	for (i = 0; i < opc_pref->ch_num; i++) {
+		if ((show_snon_ocp || !opc_pref->chs[i].static_non_op)
+			&& (show_no_ir || !opc_pref->chs[i].no_ir)
+		) {
+			_RTW_PRINT_SEL(sel, "   %c%c"
+				, opc_pref->chs[i].no_ir ? ' ' : 'I'
+				, opc_pref->chs[i].static_non_op ? ' ' : 'E'
+			);
+		}
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+
+	RTW_PRINT_SEL(sel, "                  ");
+	for (i = 0; i < opc_pref->ch_num; i++) {
+		if ((show_snon_ocp || !opc_pref->chs[i].static_non_op)
+			&& (show_no_ir || !opc_pref->chs[i].no_ir)
+		) {
+			if (opc_pref->chs[i].max_txpwr == UNSPECIFIED_MBM)
+				_RTW_PRINT_SEL(sel, "     ");
+			else
+				_RTW_PRINT_SEL(sel, " %4d", opc_pref->chs[i].max_txpwr);
+		}
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+}
+
+void dump_cap_spt_op_class_ch(void *sel, struct rf_ctl_t *rfctl, bool detail)
+{
+	u8 i;
+
+	dump_op_class_ch_title(sel);
+
+	for (i = 0; i < global_op_class_num; i++) {
+		if (!rfctl->spt_op_class_ch[i])
+			continue;
+		dump_opc_pref_single(sel, rfctl->spt_op_class_ch[i], 1, 1, detail);
+	}
+
+	RTW_PRINT_SEL(sel, "op_class number:%d\n", rfctl->cap_spt_op_class_num);
+}
+
+void dump_reg_spt_op_class_ch(void *sel, struct rf_ctl_t *rfctl, bool detail)
+{
+	u8 i;
+
+	dump_op_class_ch_title(sel);
+
+	for (i = 0; i < global_op_class_num; i++) {
+		if (!rfctl->spt_op_class_ch[i])
+			continue;
+		dump_opc_pref_single(sel, rfctl->spt_op_class_ch[i], 0, 1, detail);
+	}
+
+	RTW_PRINT_SEL(sel, "op_class number:%d\n", rfctl->reg_spt_op_class_num);
+}
+
+void dump_cur_spt_op_class_ch(void *sel, struct rf_ctl_t *rfctl, bool detail)
+{
+	u8 i;
+
+	dump_op_class_ch_title(sel);
+
+	for (i = 0; i < global_op_class_num; i++) {
+		if (!rfctl->spt_op_class_ch[i])
+			continue;
+		dump_opc_pref_single(sel, rfctl->spt_op_class_ch[i], 0, 0, detail);
+	}
+
+	RTW_PRINT_SEL(sel, "op_class number:%d\n", rfctl->cur_spt_op_class_num);
+}
+
+const u8 _rf_type_to_rf_tx_cnt[] = {
+	1, /*RF_1T1R*/
+	1, /*RF_1T2R*/
+	2, /*RF_2T2R*/
+	2, /*RF_2T3R*/
+	2, /*RF_2T4R*/
+	3, /*RF_3T3R*/
+	3, /*RF_3T4R*/
+	4, /*RF_4T4R*/
+	1, /*RF_TYPE_MAX*/
+};
+
+const u8 _rf_type_to_rf_rx_cnt[] = {
+	1, /*RF_1T1R*/
+	2, /*RF_1T2R*/
+	2, /*RF_2T2R*/
+	3, /*RF_2T3R*/
+	4, /*RF_2T4R*/
+	3, /*RF_3T3R*/
+	4, /*RF_3T4R*/
+	4, /*RF_4T4R*/
+	1, /*RF_TYPE_MAX*/
+};
+
+const char *const _rf_type_to_rfpath_str[] = {
+	"RF_1T1R",
+	"RF_1T2R",
+	"RF_2T2R",
+	"RF_2T3R",
+	"RF_2T4R",
+	"RF_3T3R",
+	"RF_3T4R",
+	"RF_4T4R",
+	"RF_TYPE_MAX"
+};
+
+static const u8 _trx_num_to_rf_type[RF_PATH_MAX][RF_PATH_MAX] = {
+	{RF_1T1R,		RF_1T2R,		RF_TYPE_MAX,	RF_TYPE_MAX},
+	{RF_TYPE_MAX,	RF_2T2R,		RF_2T3R,		RF_2T4R},
+	{RF_TYPE_MAX,	RF_TYPE_MAX,	RF_3T3R,		RF_3T4R},
+	{RF_TYPE_MAX,	RF_TYPE_MAX,	RF_TYPE_MAX,	RF_4T4R},
+};
+
+enum rf_type trx_num_to_rf_type(u8 tx_num, u8 rx_num)
+{
+	if (tx_num > 0 && tx_num <= RF_PATH_MAX && rx_num > 0 && rx_num <= RF_PATH_MAX)
+		return _trx_num_to_rf_type[tx_num - 1][rx_num - 1];
+	return RF_TYPE_MAX;
+}
+
+enum rf_type trx_bmp_to_rf_type(u8 tx_bmp, u8 rx_bmp)
+{
+	u8 tx_num = 0;
+	u8 rx_num = 0;
+	int i;
+
+	for (i = 0; i < RF_PATH_MAX; i++) {
+		if (tx_bmp >> i & BIT0)
+			tx_num++;
+		if (rx_bmp >> i & BIT0)
+			rx_num++;
+	}
+
+	return trx_num_to_rf_type(tx_num, rx_num);
+}
+
+bool rf_type_is_a_in_b(enum rf_type a, enum rf_type b)
+{
+	return rf_type_to_rf_tx_cnt(a) <= rf_type_to_rf_tx_cnt(b)
+		&& rf_type_to_rf_rx_cnt(a) <= rf_type_to_rf_rx_cnt(b);
+}
+
+static void rtw_path_bmp_limit_from_higher(u8 *bmp, u8 *bmp_bit_cnt, u8 bit_cnt_lmt)
+{
+	int i;
+
+	for (i = RF_PATH_MAX - 1; *bmp_bit_cnt > bit_cnt_lmt && i >= 0; i--) {
+		if (*bmp & BIT(i)) {
+			*bmp &= ~BIT(i);
+			(*bmp_bit_cnt)--;
+		}
+	}
+}
+
+u8 rtw_restrict_trx_path_bmp_by_rftype(u8 trx_path_bmp, enum rf_type type, u8 *tx_num, u8 *rx_num)
+{
+	u8 bmp_tx = (trx_path_bmp & 0xF0) >> 4;
+	u8 bmp_rx = trx_path_bmp & 0x0F;
+	u8 bmp_tx_num = 0, bmp_rx_num = 0;
+	u8 tx_num_lmt, rx_num_lmt;
+	enum rf_type ret_type = RF_TYPE_MAX;
+	int i, j;
+
+	for (i = 0; i < RF_PATH_MAX; i++) {
+		if (bmp_tx & BIT(i))
+			bmp_tx_num++;
+		if (bmp_rx & BIT(i))
+			bmp_rx_num++;
+	}
+
+	/* limit higher bit first according to input type */
+	tx_num_lmt = rf_type_to_rf_tx_cnt(type);
+	rx_num_lmt = rf_type_to_rf_rx_cnt(type);
+	rtw_path_bmp_limit_from_higher(&bmp_tx, &bmp_tx_num, tx_num_lmt);
+	rtw_path_bmp_limit_from_higher(&bmp_rx, &bmp_rx_num, rx_num_lmt);
+
+	/* search for valid rf_type (larger RX prefer) */
+	for (j = bmp_rx_num; j > 0; j--) {
+		for (i = bmp_tx_num; i > 0; i--) {
+			ret_type = trx_num_to_rf_type(i, j);
+			if (RF_TYPE_VALID(ret_type)) {
+				rtw_path_bmp_limit_from_higher(&bmp_tx, &bmp_tx_num, i);
+				rtw_path_bmp_limit_from_higher(&bmp_rx, &bmp_rx_num, j);
+				if (tx_num)
+					*tx_num = bmp_tx_num;
+				if (rx_num)
+					*rx_num = bmp_rx_num;
+				goto exit;
+			}
+		}
+	}
+
+exit:
+	return RF_TYPE_VALID(ret_type) ? ((bmp_tx << 4) | bmp_rx) : 0x00;
+}
+
+#if CONFIG_TXPWR_LIMIT
+void dump_regd_exc_list(void *sel, struct rf_ctl_t *rfctl)
+{
+	/* TODO: get from phl */
+}
+#endif /* CONFIG_TXPWR_LIMIT */
+
+bool rtw_is_long_cac_range(u32 hi, u32 lo, u8 dfs_region)
+{
+	return (dfs_region == RTW_DFS_REGD_ETSI && rtw_is_range_overlap(hi, lo, 5650, 5600)) ? _TRUE : _FALSE;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY bool rtw_is_long_cac_ch(u8 ch, u8 bw, u8 offset, u8 dfs_region)
+{
+	u32 hi, lo;
+
+	if (rtw_chbw_to_freq_range(ch, bw, offset, &hi, &lo) == _FALSE)
+		return _FALSE;
+
+	return rtw_is_long_cac_range(hi, lo, dfs_region) ? _TRUE : _FALSE;
+}
+#endif
+
+bool rtw_is_long_cac_bch(enum band_type band, u8 ch, u8 bw, u8 offset, u8 dfs_region)
+{
+	u32 hi, lo;
+
+	if (rtw_bchbw_to_freq_range(band, ch, bw, offset, &hi, &lo) == _FALSE)
+		return _FALSE;
+
+	return rtw_is_long_cac_range(hi, lo, dfs_region) ? _TRUE : _FALSE;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm.c
new file mode 100644
index 000000000000..ac69f6348f94
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm.c
@@ -0,0 +1,2986 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+#ifdef CONFIG_RTW_80211K
+#include "rtw_rm_fsm.h"
+#include "rtw_rm_util.h"
+#endif
+
+#define pstr(s) s+strlen(s)
+#ifndef MIN
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+#endif
+
+u8 rm_post_event_hdl(_adapter *padapter, u8 *pbuf)
+{
+#ifdef CONFIG_RTW_80211K
+	struct rm_event *pev = (struct rm_event *)pbuf;
+
+	_rm_post_event(padapter, pev->rmid, pev->evid);
+	rm_handler(padapter, pev);
+#endif
+	return H2C_SUCCESS;
+}
+
+#ifdef CONFIG_RTW_80211K
+void rm_update_cap(u8 *frame_head, _adapter *pa, u32 pktlen, int offset)
+{
+	u8 *res;
+	sint len;
+
+	res = rtw_get_ie(frame_head + offset, _EID_RRM_EN_CAP_IE_, &len,
+			 pktlen - offset);
+	if (res != NULL)
+		_rtw_memcpy((void *)pa->rmpriv.rm_en_cap_def, (res + 2),
+			    MIN(len, sizeof(pa->rmpriv.rm_en_cap_def)));
+}
+#endif
+
+#ifdef CONFIG_RTW_80211K
+struct cmd_meas_type_ {
+	u8 id;
+	char *name;
+};
+
+char *rm_type_req_name(u8 meas_type) {
+
+	switch (meas_type) {
+	case basic_req:
+		return "basic_req";
+	case cca_req:
+		return "cca_req";
+	case rpi_histo_req:
+		return "rpi_histo_req";
+	case ch_load_req:
+		return "ch_load_req";
+	case noise_histo_req:
+		return "noise_histo_req";
+	case bcn_req:
+		return "bcn_req";
+	case frame_req:
+		return "frame_req";
+	case sta_statis_req:
+		return "sta_statis_req";
+	}
+	return "unknown_req";
+};
+
+char *rm_type_rep_name(u8 meas_type) {
+
+	switch (meas_type) {
+	case basic_rep:
+		return "basic_rep";
+	case cca_rep:
+		return "cca_rep";
+	case rpi_histo_rep:
+		return "rpi_histo_rep";
+	case ch_load_rep:
+		return "ch_load_rep";
+	case noise_histo_rep:
+		return "noise_histo_rep";
+	case bcn_rep:
+		return "bcn_rep";
+	case frame_rep:
+		return "frame_rep";
+	case sta_statis_rep:
+		return "sta_statis_rep";
+	}
+	return "unknown_rep";
+};
+
+char *rm_en_cap_name(enum rm_cap_en en)
+{
+	switch (en) {
+	case RM_LINK_MEAS_CAP_EN:
+		return "RM_LINK_MEAS_CAP_EN";
+	case RM_NB_REP_CAP_EN:
+		return "RM_NB_REP_CAP_EN";
+	case RM_PARAL_MEAS_CAP_EN:
+		return "RM_PARAL_MEAS_CAP_EN";
+	case RM_REPEAT_MEAS_CAP_EN:
+		return "RM_REPEAT_MEAS_CAP_EN";
+	case RM_BCN_PASSIVE_MEAS_CAP_EN:
+		return "RM_BCN_PASSIVE_MEAS_CAP_EN";
+	case RM_BCN_ACTIVE_MEAS_CAP_EN:
+		return "RM_BCN_ACTIVE_MEAS_CAP_EN";
+	case RM_BCN_TABLE_MEAS_CAP_EN:
+		return "RM_BCN_TABLE_MEAS_CAP_EN";
+	case RM_BCN_MEAS_REP_COND_CAP_EN:
+		return "RM_BCN_MEAS_REP_COND_CAP_EN";
+
+	case RM_FRAME_MEAS_CAP_EN:
+		return "RM_FRAME_MEAS_CAP_EN";
+	case RM_CH_LOAD_CAP_EN:
+		return "RM_CH_LOAD_CAP_EN";
+	case RM_NOISE_HISTO_CAP_EN:
+		return "RM_NOISE_HISTO_CAP_EN";
+	case RM_STATIS_MEAS_CAP_EN:
+		return "RM_STATIS_MEAS_CAP_EN";
+	case RM_LCI_MEAS_CAP_EN:
+		return "RM_LCI_MEAS_CAP_EN";
+	case RM_LCI_AMIMUTH_CAP_EN:
+		return "RM_LCI_AMIMUTH_CAP_EN";
+	case RM_TRANS_STREAM_CAT_MEAS_CAP_EN:
+		return "RM_TRANS_STREAM_CAT_MEAS_CAP_EN";
+	case RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN:
+		return "RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN";
+
+	case RM_AP_CH_REP_CAP_EN:
+		return "RM_AP_CH_REP_CAP_EN";
+	case RM_RM_MIB_CAP_EN:
+		return "RM_RM_MIB_CAP_EN";
+	case RM_OP_CH_MAX_MEAS_DUR0:
+		return "RM_OP_CH_MAX_MEAS_DUR0";
+	case RM_OP_CH_MAX_MEAS_DUR1:
+		return "RM_OP_CH_MAX_MEAS_DUR1";
+	case RM_OP_CH_MAX_MEAS_DUR2:
+		return "RM_OP_CH_MAX_MEAS_DUR2";
+	case RM_NONOP_CH_MAX_MEAS_DUR0:
+		return "RM_NONOP_CH_MAX_MEAS_DUR0";
+	case RM_NONOP_CH_MAX_MEAS_DUR1:
+		return "RM_NONOP_CH_MAX_MEAS_DUR1";
+	case RM_NONOP_CH_MAX_MEAS_DUR2:
+		return "RM_NONOP_CH_MAX_MEAS_DUR2";
+
+	case RM_MEAS_PILOT_CAP0:
+		return "RM_MEAS_PILOT_CAP0";		/* 24-26 */
+	case RM_MEAS_PILOT_CAP1:
+		return "RM_MEAS_PILOT_CAP1";
+	case RM_MEAS_PILOT_CAP2:
+		return "RM_MEAS_PILOT_CAP2";
+	case RM_MEAS_PILOT_TRANS_INFO_CAP_EN:
+		return "RM_MEAS_PILOT_TRANS_INFO_CAP_EN";
+	case RM_NB_REP_TSF_OFFSET_CAP_EN:
+		return "RM_NB_REP_TSF_OFFSET_CAP_EN";
+	case RM_RCPI_MEAS_CAP_EN:
+		return "RM_RCPI_MEAS_CAP_EN";		/* 29 */
+	case RM_RSNI_MEAS_CAP_EN:
+		return "RM_RSNI_MEAS_CAP_EN";
+	case RM_BSS_AVG_ACCESS_DELAY_CAP_EN:
+		return "RM_BSS_AVG_ACCESS_DELAY_CAP_EN";
+
+	case RM_AVALB_ADMIS_CAPACITY_CAP_EN:
+		return "RM_AVALB_ADMIS_CAPACITY_CAP_EN";
+	case RM_ANT_CAP_EN:
+		return "RM_ANT_CAP_EN";
+	case RM_RSVD:
+	case RM_MAX:
+	default:
+		break;
+	}
+	return "unknown";
+}
+
+void _dump_buf(u8 *ptr, u32 len, const char *caller)
+{
+#if (RM_MORE_DBG_MSG)
+	u8 str[128];
+	int i;
+
+	printk("%s()len=%d\n",caller ,len);
+	memset(str, 0, sizeof(str));
+	for (i=0;i<len;i++) {
+		sprintf(str+strlen(str),"%02x ",*ptr++);
+		if ((i+1)%16==0) {
+			printk("%s\n",str);
+			memset(str, 0, sizeof(str));
+		}
+	}
+	if ((i+1)%16!=0)
+		printk("%s\n",str);
+#endif
+}
+#define dump_buf(a, b) _dump_buf(a, b, __func__)
+
+int rm_en_cap_chk_and_set(struct rm_obj *prm, enum rm_cap_en en)
+{
+	int idx;
+	u8 cap;
+
+
+	if (en >= RM_MAX)
+		return _FALSE;
+
+	idx = en / 8;
+	cap = prm->psta->padapter->rmpriv.rm_en_cap_def[idx];
+
+	if (!(cap & BIT(en - (idx*8)))) {
+		RTW_INFO("RM: %s incapable\n",rm_en_cap_name(en));
+		rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+		return _FALSE;
+	}
+	return _SUCCESS;
+}
+
+/* for caller outside rm */
+u8 rm_add_nb_req(_adapter *padapter, struct sta_info *psta)
+{
+	struct rm_obj *prm;
+
+	prm = rm_alloc_rmobj(padapter);
+
+	if (prm == NULL) {
+		RTW_ERR("RM: unable to alloc rm obj for requeset\n");
+		return _FALSE;
+	}
+
+	prm->psta = psta;
+	prm->q.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+	prm->q.diag_token = rm_gen_dialog_token(padapter);
+	prm->q.m_token = rm_gen_meas_token(padapter);
+	prm->rmid = rm_gen_rmid(padapter, prm, RM_MASTER);
+	prm->q.action_code = RM_ACT_NB_REP_REQ;
+
+	#if 0
+	if (pmac) { /* find sta_info according to bssid */
+		pmac += 4; /* skip mac= */
+		if (hwaddr_parse(pmac, bssid) == NULL) {
+			sprintf(pstr(s), "Err: \nincorrect mac format\n");
+			return _FAIL;
+		}
+		psta = rm_get_sta(padapter, 0xff, bssid);
+	}
+	#endif
+
+	/* enquee rmobj */
+	rm_enqueue_rmobj(padapter, prm, _FALSE);
+
+	RTW_INFO("RM: rmid=%x add req to " MAC_FMT "\n",
+		prm->rmid, MAC_ARG(psta->phl_sta->mac_addr));
+
+	return _SUCCESS;
+}
+
+static u8 *build_wlan_hdr(_adapter *padapter, struct xmit_frame *pmgntframe,
+	struct sta_info *psta, u16 frame_type)
+{
+	u8 *pframe;
+	u16 *fctrl;
+	struct pkt_attrib *pattr;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_ext_info *pmlmeinfo = &padapter_link->mlmeextpriv.mlmext_info;
+
+
+	/* update attribute */
+	pattr = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattr);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, psta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
+
+	RTW_INFO("RM: dst = " MAC_FMT "\n", MAC_ARG(pwlanhdr->addr1));
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	SetFragNum(pframe, 0);
+
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattr->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	return pframe;
+}
+
+void rm_set_rep_mode(struct rm_obj *prm, u8 mode)
+{
+
+	RTW_INFO("RM: rmid=%x set %s\n",
+		prm->rmid,
+		mode|MEAS_REP_MOD_INCAP?"INCAP":
+		mode|MEAS_REP_MOD_REFUSE?"REFUSE":
+		mode|MEAS_REP_MOD_LATE?"LATE":"");
+
+	prm->p.m_mode |= mode;
+}
+
+int issue_null_reply(struct rm_obj *prm)
+{
+	int len=0, my_len;
+	u8 *pframe, m_mode;
+	_adapter *padapter = prm->psta->padapter;
+	struct pkt_attrib *pattr;
+	struct xmit_frame *pmgntframe;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+
+
+	m_mode = prm->q.m_mode;
+	if (m_mode || prm->q.rpt == 0) {
+		RTW_INFO("RM: rmid=%x reply (%s repeat=%d)\n",
+			prm->rmid,
+			m_mode&MEAS_REP_MOD_INCAP?"INCAP":
+			m_mode&MEAS_REP_MOD_REFUSE?"REFUSE":
+			m_mode&MEAS_REP_MOD_LATE?"LATE":"no content",
+			prm->q.rpt);
+	}
+
+	switch (prm->q.action_code) {
+	case RM_ACT_RADIO_MEAS_REQ:
+		len = 8;
+		break;
+	case RM_ACT_NB_REP_REQ:
+		len = 3;
+		break;
+	case RM_ACT_LINK_MEAS_REQ:
+		len = 3;
+		break;
+	default:
+		break;
+	}
+
+	if (len==0)
+		return _FALSE;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_ERR("RM: %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(padapter, pmgntframe, prm->psta, WIFI_ACTION);
+	pframe = rtw_set_fixed_ie(pframe, 3, &prm->p.category, &pattr->pktlen);
+
+	my_len = 0;
+	if (len>5) {
+		prm->p.len = len - 3 - 2;
+		pframe = rtw_set_fixed_ie(pframe, len - 3,
+			&prm->p.e_id, &my_len);
+	}
+
+	pattr->pktlen += my_len;
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+int ready_for_scan(struct rm_obj *prm)
+{
+	_adapter *padapter = prm->psta->padapter;
+	u8 ssc_chk;
+
+	if (!rtw_is_adapter_up(padapter))
+		return _FALSE;
+
+	ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+
+	if (ssc_chk == SS_ALLOW)
+		return _SUCCESS;
+
+	return _FALSE;
+}
+
+int rm_get_chset(struct rm_obj *prm)
+{
+	int i,meas_ch_amount=0;
+	u8 op_class=0, val8;
+	struct rtw_ieee80211_channel *pch_set;
+
+	pch_set = &prm->q.ch_set[0];
+
+	_rtw_memset(pch_set, 0,
+		sizeof(struct rtw_ieee80211_channel) * RTW_CHANNEL_SCAN_AMOUNT);
+
+	op_class = prm->q.op_class;
+	if (prm->q.ch_num == 0) {
+		/* ch_num=0   : scan all ch in operating class */
+		meas_ch_amount = rm_get_ch_set(op_class, pch_set, RTW_CHANNEL_SCAN_AMOUNT);
+
+	} else if (prm->q.ch_num == 255) {
+		/* 802.11 p.1066 */
+		/* ch_num=255 : If the Channel Number is 255 and includes
+		 * AP Channel Report subelements
+		 */
+		meas_ch_amount = rm_get_ch_set_from_bcn_req_opt(&prm->q.opt.bcn, pch_set, RTW_CHANNEL_SCAN_AMOUNT);
+	} else {
+		pch_set[0].hw_value = prm->q.ch_num;
+		pch_set[0].band = prm->q.band;
+		meas_ch_amount = 1;
+		RTW_INFO("RM: meas_ch->hw_value = %u\n", pch_set->hw_value);
+	}
+
+	/* get means channel */
+	prm->q.ch_set_ch_amount = meas_ch_amount;
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("survey (%d) channels\n", meas_ch_amount);
+#endif
+	return 0;
+}
+
+int rm_sitesurvey(struct rm_obj *prm)
+{
+	int meas_ch_amount=0;
+	struct rtw_ieee80211_channel *pch_set;
+	struct sitesurvey_parm *parm;
+
+
+	RTW_INFO("RM: rmid=%x %s\n",prm->rmid, __func__);
+
+	parm = rtw_malloc(sizeof(*parm));
+	if (parm == NULL)
+		return _FAIL;
+
+	rm_get_chset(prm);
+	pch_set = &prm->q.ch_set[0];
+
+	meas_ch_amount = MIN(prm->q.ch_set_ch_amount, RTW_CHANNEL_SCAN_AMOUNT);
+	_rtw_memset(parm, 0, sizeof(struct sitesurvey_parm));
+	_rtw_memcpy(parm->ch, pch_set, sizeof(struct rtw_ieee80211_channel) * meas_ch_amount);
+
+	_rtw_memcpy(&parm->ssid[0], &prm->q.opt.bcn.ssid, IW_ESSID_MAX_SIZE);
+
+	parm->ssid_num = 1;
+	parm->scan_mode = prm->q.scan_mode;
+	parm->ch_num = meas_ch_amount;
+	parm->rrm_token = prm->rmid;
+	parm->duration = prm->q.meas_dur;
+	parm->scan_type = RTW_SCAN_RRM;
+	/* parm.bw = BW_20M; */
+
+	rtw_sitesurvey_cmd(prm->psta->padapter, parm);
+	rtw_mfree(parm, sizeof(*parm));
+
+	return _SUCCESS;
+}
+
+static int rm_parse_ch_load_s_elem(struct rm_obj *prm, u8 *pbody, int req_len)
+{
+	u8 *popt_id;
+	int i, p=0; /* position */
+	int len = req_len;
+
+
+	prm->q.opt_s_elem_len = len;
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: opt_s_elem_len=%d\n", len);
+#endif
+	while (len) {
+
+		switch (pbody[p]) {
+		case ch_load_rep_info:
+			/* check RM_EN */
+			rm_en_cap_chk_and_set(prm, RM_CH_LOAD_CAP_EN);
+
+			_rtw_memcpy(&(prm->q.opt.clm.rep_cond),
+				&pbody[p+2], sizeof(prm->q.opt.clm.rep_cond));
+
+			RTW_INFO("RM: ch_load_rep_info=%u:%u\n",
+				prm->q.opt.clm.rep_cond.cond,
+				prm->q.opt.clm.rep_cond.threshold);
+			break;
+		default:
+			break;
+
+		}
+		len = len - (int)pbody[p+1] - 2;
+		p = p + (int)pbody[p+1] + 2;
+#if (RM_MORE_DBG_MSG)
+		RTW_INFO("RM: opt_s_elem_len=%d\n",len);
+#endif
+	}
+	return _SUCCESS;
+}
+
+static int rm_parse_noise_histo_s_elem(struct rm_obj *prm,
+	u8 *pbody, int req_len)
+{
+	u8 *popt_id;
+	int i, p=0; /* position */
+	int len = req_len;
+
+
+	prm->q.opt_s_elem_len = len;
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: opt_s_elem_len=%d\n", len);
+#endif
+
+	while (len) {
+
+		switch (pbody[p]) {
+		case noise_histo_rep_info:
+			/* check RM_EN */
+			rm_en_cap_chk_and_set(prm, RM_NOISE_HISTO_CAP_EN);
+
+			_rtw_memcpy(&(prm->q.opt.nhm.rep_cond),
+				&pbody[p+2], sizeof(prm->q.opt.nhm.rep_cond));
+
+			RTW_INFO("RM: noise_histo_rep_info=%u:%u\n",
+				prm->q.opt.nhm.rep_cond.cond,
+				prm->q.opt.nhm.rep_cond.threshold);
+			break;
+		default:
+			break;
+
+       		}
+		len = len - (int)pbody[p+1] - 2;
+		p = p + (int)pbody[p+1] + 2;
+#if (RM_MORE_DBG_MSG)
+		RTW_INFO("RM: opt_s_elem_len=%d\n",len);
+#endif
+	}
+	return _SUCCESS;
+}
+
+static int rm_parse_bcn_req_s_elem(struct rm_obj *prm, u8 *pbody, int req_len)
+{
+	u8 *popt_id;
+	int i, p=0; /* position */
+	int len = req_len;
+	int ap_ch_rpt_idx = 0;
+	struct _RT_OPERATING_CLASS *op;
+
+
+	/* opt length,2:pbody[0]+ pbody[1] */
+	/* first opt id : pbody[18] */
+
+	prm->q.opt_s_elem_len = len;
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: opt_s_elem_len=%d\n", len);
+#endif
+
+	popt_id = prm->q.opt.bcn.opt_id;
+	while (len && prm->q.opt.bcn.opt_id_num < BCN_REQ_OPT_MAX_NUM) {
+
+		switch (pbody[p]) {
+		case bcn_req_ssid:
+			RTW_INFO("bcn_req_ssid\n");
+
+#if (DBG_BCN_REQ_WILDCARD)
+			RTW_INFO("DBG set ssid to WILDCARD\n");
+#else
+#if (DBG_BCN_REQ_SSID)
+			RTW_INFO("DBG set ssid to %s\n",DBG_BCN_REQ_SSID_NAME);
+			i = strlen(DBG_BCN_REQ_SSID_NAME);
+			prm->q.opt.bcn.ssid.SsidLength = i;
+			_rtw_memcpy(&(prm->q.opt.bcn.ssid.Ssid), DBG_BCN_REQ_SSID_NAME,
+				MIN(i, sizeof(prm->q.opt.bcn.ssid.Ssid)-1));
+
+#else /* original */
+			prm->q.opt.bcn.ssid.SsidLength = pbody[p+1];
+			_rtw_memcpy(&(prm->q.opt.bcn.ssid.Ssid), &pbody[p+2],
+				MIN(pbody[p+1], sizeof(prm->q.opt.bcn.ssid.Ssid)-1));
+#endif
+#endif
+
+			RTW_INFO("RM: bcn_req_ssid=%s\n",
+				prm->q.opt.bcn.ssid.Ssid);
+
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+			break;
+
+		case bcn_req_rep_info:
+			/* check RM_EN */
+			rm_en_cap_chk_and_set(prm, RM_BCN_MEAS_REP_COND_CAP_EN);
+
+			_rtw_memcpy(&(prm->q.opt.bcn.rep_cond),
+				&pbody[p+2], sizeof(prm->q.opt.bcn.rep_cond));
+
+			RTW_INFO("bcn_req_rep_info=%u:%u\n",
+				prm->q.opt.bcn.rep_cond.cond,
+				prm->q.opt.bcn.rep_cond.threshold);
+
+			/*popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];*/
+			break;
+
+		case bcn_req_rep_detail:
+#if DBG_BCN_REQ_DETAIL
+			prm->q.opt.bcn.rep_detail = 2; /* all IE in beacon */
+#else
+			prm->q.opt.bcn.rep_detail = pbody[p+2];
+#endif
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+
+#if (RM_MORE_DBG_MSG)
+			RTW_INFO("RM: report_detail=%d\n",
+				prm->q.opt.bcn.rep_detail);
+#endif
+			break;
+
+		case bcn_req_last_bcn_rpt_ind:
+			prm->q.opt.bcn.rep_last_bcn_ind = pbody[p+2];
+#if (RM_MORE_DBG_MSG)
+			RTW_INFO("RM: bcn_req_last_bcn_rpt_ind=%d\n",
+				prm->q.opt.bcn.rep_last_bcn_ind);
+#endif
+			break;
+
+		case bcn_req_req:
+			RTW_INFO("RM: bcn_req_req\n");
+
+			prm->q.opt.bcn.req_start = rtw_malloc(pbody[p+1]);
+
+			if (prm->q.opt.bcn.req_start == NULL) {
+				RTW_ERR("RM: req_start malloc fail!!\n");
+				break;
+			}
+
+			for (i = 0; i < pbody[p+1]; i++)
+				*((prm->q.opt.bcn.req_start)+i) =
+					pbody[p+2+i];
+
+			prm->q.opt.bcn.req_len = pbody[p+1];
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+			break;
+
+		case bcn_req_ap_ch_rep:
+#if (RM_MORE_DBG_MSG)
+			RTW_INFO("RM: bcn_req_ap_ch_rep\n");
+#endif
+			if (ap_ch_rpt_idx > BCN_REQ_OPT_AP_CH_RPT_MAX_NUM) {
+				RTW_ERR("RM: bcn_req_ap_ch_rep over size\n");
+				break;
+			}
+			popt_id[prm->q.opt.bcn.opt_id_num++] = pbody[p];
+
+			/* get channel list
+			 * EID:len:op-class:ch-list
+			 */
+			op = rtw_malloc(sizeof (*op));
+			op->global_op_class = pbody[p + 2];
+			i = pbody[p + 1] - 1; /* ch list len; (-1) is op class */
+
+#if (RM_MORE_DBG_MSG)
+			RTW_INFO("%d op class %d has %d ch\n",
+				ap_ch_rpt_idx,op->global_op_class,i);
+#endif
+			op->Len = i;
+			memcpy(op->Channel, &pbody[p + 3],
+				MIN(i, MAX_CH_NUM_IN_OP_CLASS));
+			prm->q.opt.bcn.ap_ch_rpt[ap_ch_rpt_idx++] = op;
+			prm->q.opt.bcn.ap_ch_rpt_num = ap_ch_rpt_idx;
+			break;
+
+		default:
+			break;
+
+       		}
+		len = len - (int)pbody[p+1] - 2;
+		p = p + (int)pbody[p+1] + 2;
+#if (RM_MORE_DBG_MSG)
+		RTW_INFO("RM: opt_s_elem_len=%d\n",len);
+#endif
+	}
+
+	return _SUCCESS;
+}
+
+static int rm_parse_meas_req(struct rm_obj *prm, u8 *pbody)
+{
+	int p; /* position */
+	int req_len;
+
+
+	req_len = (int)pbody[1];
+	p = 5;
+
+	prm->q.op_class = pbody[p++];
+	prm->q.ch_num = pbody[p++];
+	prm->q.band = rtw_get_band_by_op_class(prm->q.op_class);
+	/* error handling */
+	if (prm->q.band == BAND_MAX)
+		prm->q.band = BAND_ON_24G;
+	prm->q.rand_intvl = le16_to_cpu(*(u16*)(&pbody[p]));
+	p+=2;
+	prm->q.meas_dur = le16_to_cpu(*(u16*)(&pbody[p]));
+	p+=2;
+
+	if (prm->q.m_type == bcn_req) {
+		/*
+		 * 0: passive
+		 * 1: active
+		 * 2: bcn_table
+		 */
+		prm->q.scan_mode = pbody[p++];
+
+		/* BSSID */
+		_rtw_memcpy(&(prm->q.bssid), &pbody[p], 6);
+		p+=6;
+
+		/*
+		 * default, used when Reporting detail subelement
+		 * is not included in Beacon Request
+		 */
+		prm->q.opt.bcn.rep_detail = 2;
+	}
+
+	if (req_len-(p-2) <= 0) /* without sub-element */
+		return _SUCCESS;
+
+	switch (prm->q.m_type) {
+	case bcn_req:
+		rm_parse_bcn_req_s_elem(prm, &pbody[p], req_len-(p-2));
+		break;
+	case ch_load_req:
+		rm_parse_ch_load_s_elem(prm, &pbody[p], req_len-(p-2));
+		break;
+	case noise_histo_req:
+		rm_parse_noise_histo_s_elem(prm, &pbody[p], req_len-(p-2));
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+/* receive measurement request */
+int rm_recv_radio_mens_req(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	struct rm_obj *prm;
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = &pdiag_body[5];
+	u8 rmid, update = 0;
+
+
+#if 0
+	/* search existing rm_obj */
+	rmid = psta->phl_sta->aid << 16
+		| pdiag_body[2] << 8
+		| RM_SLAVE;
+
+	prm = rm_get_rmobj(padapter, rmid);
+	if (prm) {
+		RTW_INFO("RM: Found an exist meas rmid=%u\n", rmid);
+		update = 1;
+	} else
+#endif
+	prm = rm_alloc_rmobj(padapter);
+
+	if (prm == NULL) {
+		RTW_ERR("RM: unable to alloc rm obj for requeset\n");
+		return _FALSE;
+	}
+
+	prm->psta = psta;
+	prm->q.diag_token = pdiag_body[2];
+	prm->q.rpt = le16_to_cpu(*(u16*)(&pdiag_body[3]));
+
+	/* Figure 8-104 Measurement Requested format */
+	prm->q.e_id = pmeas_body[0];
+	prm->q.m_token = pmeas_body[2];
+	prm->q.m_mode = pmeas_body[3];
+	prm->q.m_type = pmeas_body[4];
+	prm->rmid = rm_gen_rmid(padapter, prm, RM_SLAVE);
+
+	RTW_INFO("RM: rmid=%x, bssid " MAC_FMT "\n", prm->rmid,
+		MAC_ARG(prm->psta->phl_sta->mac_addr));
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: element_id = %d\n", prm->q.e_id);
+	RTW_INFO("RM: length = %d\n", (int)pmeas_body[1]);
+	RTW_INFO("RM: meas_token = %d\n", prm->q.m_token);
+	RTW_INFO("RM: meas_mode = %d\n", prm->q.m_mode);
+	RTW_INFO("RM: meas_type = %d\n", prm->q.m_type);
+#endif
+
+	if (prm->q.e_id != _MEAS_REQ_IE_) /* 38 */
+		return _FALSE;
+
+	switch (prm->q.m_type) {
+	case bcn_req:
+		RTW_INFO("RM: recv beacon_request\n");
+		switch (prm->q.scan_mode) {
+		case bcn_req_passive:
+			rm_en_cap_chk_and_set(prm, RM_BCN_PASSIVE_MEAS_CAP_EN);
+			break;
+		case bcn_req_active:
+			rm_en_cap_chk_and_set(prm, RM_BCN_ACTIVE_MEAS_CAP_EN);
+			break;
+		case bcn_req_bcn_table:
+			rm_en_cap_chk_and_set(prm, RM_BCN_TABLE_MEAS_CAP_EN);
+			break;
+		default:
+			rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+			break;
+		}
+		break;
+	case ch_load_req:
+		RTW_INFO("RM: recv ch_load_request\n");
+		rm_en_cap_chk_and_set(prm, RM_CH_LOAD_CAP_EN);
+		break;
+	case noise_histo_req:
+		RTW_INFO("RM: recv noise_histogram_request\n");
+		rm_en_cap_chk_and_set(prm, RM_NOISE_HISTO_CAP_EN);
+		break;
+	default:
+		RTW_INFO("RM: recv unknown request type 0x%02x\n",
+			prm->q.m_type);
+		rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+		goto done;
+       }
+	rm_parse_meas_req(prm, pmeas_body);
+done:
+	if (!update)
+		rm_enqueue_rmobj(padapter, prm, _FALSE);
+
+	return _SUCCESS;
+}
+
+/* receive measurement report */
+int rm_recv_radio_mens_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	int ret = _FALSE;
+	struct rm_obj *prm;
+	u32 rmid, len;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = &pdiag_body[3];
+
+
+	rmid = psta->phl_sta->aid << 16
+		| pdiag_body[2] << 8
+		| RM_MASTER;
+
+	prm = rm_get_rmobj(padapter, rmid);
+	if (prm == NULL) {
+		/* not belong to us, report to upper */
+		rtw_cfg80211_rx_rrm_action(psta->padapter, precv_frame);
+		return _TRUE;
+	}
+
+	prm->p.action_code = pdiag_body[1];
+	prm->p.diag_token = pdiag_body[2];
+
+	/* Figure 8-140 Measuremnt Report format */
+	prm->p.e_id = pmeas_body[0];
+	prm->p.m_token = pmeas_body[2];
+	prm->p.m_mode = pmeas_body[3];
+	prm->p.m_type = pmeas_body[4];
+
+	RTW_INFO("RM: rmid=%x, bssid " MAC_FMT "\n", prm->rmid,
+		MAC_ARG(prm->psta->phl_sta->mac_addr));
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: element_id = %d\n", prm->p.e_id);
+	RTW_INFO("RM: length = %d\n", (int)pmeas_body[1]);
+	RTW_INFO("RM: meas_token = %d\n", prm->p.m_token);
+	RTW_INFO("RM: meas_mode = %d\n", prm->p.m_mode);
+	RTW_INFO("RM: meas_type = %d\n", prm->p.m_type);
+#endif
+	if (prm->p.e_id != _MEAS_RSP_IE_) /* 39 */
+		return _FALSE;
+
+	RTW_INFO("RM: recv %s\n", rm_type_rep_name(prm->p.m_type));
+	rm_post_event(padapter, prm->rmid, RM_EV_recv_rep);
+
+	/* report to upper via ioctl */
+	if ((prm->from_ioctl == true) &&
+		prm->q.m_type == bcn_req) {
+		len = pmeas_body[1] + 2; /* 2 : EID(1B)  length(1B) */
+		indicate_beacon_report(prm->psta->phl_sta->mac_addr,
+			1, len, pmeas_body);
+	}
+
+	return ret;
+}
+
+/* receive link measurement request */
+int rm_recv_link_mens_req(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	struct rm_obj *prm;
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = &pdiag_body[3];
+	u8 rmid, update = 0;
+	int i;
+
+
+	prm = rm_alloc_rmobj(padapter);
+
+	if (prm == NULL) {
+		RTW_ERR("RM: unable to alloc rm obj for requeset\n");
+		return _FALSE;
+	}
+
+	prm->psta = psta;
+	prm->q.action_code = pdiag_body[1];
+	prm->q.diag_token = pdiag_body[2];
+
+	prm->q.tx_pwr_used = pmeas_body[0];
+	prm->q.tx_pwr_max = pmeas_body[1];
+	prm->q.rx_pwr = precv_frame->u.hdr.attrib.phy_info.rx_power;
+	prm->q.rx_rate = precv_frame->u.hdr.attrib.data_rate;
+	prm->q.rx_bw = precv_frame->u.hdr.attrib.bw;
+	prm->q.rx_rsni = rm_get_frame_rsni(prm, precv_frame);
+	prm->rmid = rm_gen_rmid(padapter, prm, RM_SLAVE);
+
+	RTW_INFO("RM: rmid=%x, bssid" MAC_FMT " rx_pwr=%ddBm, rate=%s\n",
+		prm->rmid, MAC_ARG(prm->psta->phl_sta->mac_addr), prm->q.rx_pwr,
+		MGN_RATE_STR(prm->q.rx_rate));
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: tx_pwr_used =%d dBm\n", prm->q.tx_pwr_used);
+	RTW_INFO("RM: tx_pwr_max  =%d dBm\n", prm->q.tx_pwr_max);
+#endif
+
+	if (!update)
+		rm_enqueue_rmobj(padapter, prm, _FALSE);
+
+	return _SUCCESS;
+}
+
+/* receive link measurement report */
+int rm_recv_link_mens_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	int ret = _FALSE;
+	struct rm_obj *prm;
+	u32 rmid;
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = pdiag_body + 3;
+	s8 val;
+
+
+	rmid = psta->phl_sta->aid << 16
+		| pdiag_body[2] << 8
+		| RM_MASTER;
+
+	prm = rm_get_rmobj(padapter, rmid);
+	if (prm == NULL) {
+		/* not belong to us, report to upper */
+		rtw_cfg80211_rx_rrm_action(psta->padapter, precv_frame);
+		return _TRUE;
+	}
+
+	RTW_INFO("RM: rmid=%x, bssid " MAC_FMT "\n", prm->rmid,
+		MAC_ARG(prm->psta->phl_sta->mac_addr));
+
+	prm->p.action_code = pdiag_body[1];
+	prm->p.diag_token = pdiag_body[2];
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: action_code = %d\n", prm->p.action_code);
+	RTW_INFO("RM: diag_token  = %d\n", prm->p.diag_token);
+	RTW_INFO("RM: xmit_power  = %d dBm\n", pmeas_body[2]);
+	RTW_INFO("RM: link_margin = %d dBm\n", pmeas_body[3]);
+	RTW_INFO("RM: xmit_ant    = %d\n", pmeas_body[4]);
+	RTW_INFO("RM: recv_ant    = %d\n", pmeas_body[5]);
+	RTW_INFO("RM: RCPI        = %d\n", pmeas_body[6]);
+	RTW_INFO("RM: RSNI        = %d\n", pmeas_body[7]);
+#endif
+	RTW_INFO("RM: recv link meas report ...\n");
+	ret = rm_post_event(padapter, prm->rmid, RM_EV_recv_rep);
+
+	return ret;
+}
+
+
+int rm_radio_mens_nb_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta)
+{
+	u8 *pdiag_body = (u8 *)(precv_frame->u.hdr.rx_data +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+	u8 *pmeas_body = &pdiag_body[3];
+	u32 len = precv_frame->u.hdr.len;
+	u32 rmid;
+	struct rm_obj *prm;
+
+
+	rmid = psta->phl_sta->aid << 16
+		| pdiag_body[2] << 8
+		| RM_MASTER;
+
+	prm = rm_get_rmobj(padapter, rmid);
+
+	if (prm == NULL) {
+		/* not belong to us, report to upper */
+		rtw_cfg80211_rx_rrm_action(psta->padapter, precv_frame);
+		return _TRUE;
+	}
+
+	prm->p.action_code = pdiag_body[1];
+	prm->p.diag_token = pdiag_body[2];
+	prm->p.e_id = pmeas_body[0];
+
+	RTW_INFO("RM: rmid=%x, bssid " MAC_FMT "\n", prm->rmid,
+		MAC_ARG(prm->psta->phl_sta->mac_addr));
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: element_id = %d\n", prm->p.e_id);
+	RTW_INFO("RM: length = %d\n", (int)pmeas_body[1]);
+#endif
+	rm_post_event(padapter, prm->rmid, RM_EV_recv_rep);
+
+#ifdef CONFIG_RTW_MBO
+#ifdef CONFIG_LAYER2_ROAMING
+	if (rtw_wnm_btm_candidates_survey(padapter
+			,(pdiag_body + 3)
+			,(len - sizeof(struct rtw_ieee80211_hdr_3addr))
+			,_FALSE) == _FAIL)
+		return _FALSE;
+#endif
+#endif
+	rtw_cfg80211_rx_rrm_action(padapter, precv_frame);
+
+	return _TRUE;
+}
+
+unsigned int rm_on_action(_adapter *padapter, union recv_frame *precv_frame)
+{
+	u32 ret = _FAIL;
+	u8 *pframe = NULL;
+	u8 *pframe_body = NULL;
+	u8 action_code = 0;
+	u8 diag_token = 0;
+	struct rtw_ieee80211_hdr_3addr *whdr;
+	struct sta_info *psta;
+
+
+	pframe = precv_frame->u.hdr.rx_data;
+
+	/* check RA matches or not */
+	if (!_rtw_memcmp(adapter_mac_addr(padapter),
+		GetAddr1Ptr(pframe), ETH_ALEN))
+		goto exit;
+
+	whdr = (struct rtw_ieee80211_hdr_3addr *)pframe;
+	RTW_INFO("RM: %s bssid = " MAC_FMT "\n",
+		__func__, MAC_ARG(whdr->addr2));
+
+	psta = rtw_get_stainfo(&padapter->stapriv, whdr->addr2);
+
+        if (!psta) {
+		RTW_ERR("RM: psta not found\n");
+                goto exit;
+        }
+
+	pframe_body = (unsigned char *)(pframe +
+		sizeof(struct rtw_ieee80211_hdr_3addr));
+
+	/* Figure 8-438 radio measurement request frame Action field format */
+	/* Category = pframe_body[0] = 5 (Radio Measurement) */
+	action_code = pframe_body[1];
+	diag_token = pframe_body[2];
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: %s radio_action=%x, diag_token=%x\n", __func__,
+		action_code, diag_token);
+#endif
+
+	switch (action_code) {
+
+	case RM_ACT_RADIO_MEAS_REQ:
+		RTW_INFO("RM: RM_ACT_RADIO_MEAS_REQ\n");
+		ret = rm_recv_radio_mens_req(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_RADIO_MEAS_REP:
+		RTW_INFO("RM: RM_ACT_RADIO_MEAS_REP\n");
+		ret = rm_recv_radio_mens_rep(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_LINK_MEAS_REQ:
+		RTW_INFO("RM: RM_ACT_LINK_MEAS_REQ\n");
+		ret = rm_recv_link_mens_req(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_LINK_MEAS_REP:
+		RTW_INFO("RM: RM_ACT_LINK_MEAS_REP\n");
+		ret = rm_recv_link_mens_rep(padapter, precv_frame, psta);
+		break;
+
+	case RM_ACT_NB_REP_REQ:
+		RTW_INFO("RM: RM_ACT_NB_REP_REQ\n");
+		break;
+
+	case RM_ACT_NB_REP_RESP:
+		RTW_INFO("RM: RM_ACT_NB_REP_RESP\n");
+		ret = rm_radio_mens_nb_rep(padapter, precv_frame, psta);
+		break;
+
+	default:
+		/* TODO reply incabable */
+		RTW_ERR("RM: unknown specturm management action %2x\n",
+			action_code);
+		break;
+	}
+exit:
+	return ret;
+}
+
+static u8 *rm_gen_bcn_detail_2_elem(_adapter *padapter, u8 *pframe, u8 frag_id,
+	struct rm_obj *prm, u8 **pie, u32 *ie_len, unsigned int *fr_len)
+{
+	u32 frag_max_len = 255 - *fr_len - 4; /* 4: frag_id */
+	int len = *ie_len;
+	u8 slen = 0, frag_len = 0, more = 0;
+	u8 *ies = *pie;
+	u8 id, val8;
+
+	if (prm->q.opt.bcn.rep_last_bcn_ind)
+		frag_max_len -= 3;
+
+	/* report_detail = 2 */
+	if (frag_id == 0) {
+		frag_len += _FIXED_IE_LENGTH_;
+		len -= _FIXED_IE_LENGTH_;
+	}
+
+	while (len >= 0) {
+
+		id = ies[frag_len];
+		slen = (unsigned int)ies[frag_len + 1] + 2;
+
+		if ((frag_len + slen) >= frag_max_len ||
+		    frag_len == *ie_len) {
+			/* ID */
+			val8 = 1; /* reported frame body */
+			pframe = rtw_set_fixed_ie(pframe, 1, &val8, fr_len);
+
+			/* LEN */
+			pframe = rtw_set_fixed_ie(pframe, 1, &frag_len, fr_len);
+
+			/* frame body */
+			pframe = rtw_set_fixed_ie(pframe, frag_len, ies, fr_len);
+
+			goto done;
+		}
+
+		len -= slen;
+		frag_len += slen;
+	}
+done:
+	*pie = ies + frag_len;
+
+	if (len > 0)
+		*ie_len = len; /* remain len */
+	else
+		*ie_len = 0;
+
+	return pframe;
+}
+
+static u8 *rm_gen_bcn_detail_1_elem(_adapter *padapter, u8 *pframe, u8 frag_id,
+	struct rm_obj *prm, u8 **ies, u32 *ie_len, unsigned int *fr_len)
+{
+	unsigned int my_len = 0;
+	int j = 0, k, len;
+	u8 *plen;
+	u8 *ptr;
+	u8 val8, eid;
+
+	/* report_detail = 1 */
+
+	/* ID */
+	val8 = 1; /* 1:reported frame body */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	plen = pframe;
+	val8 = 0;
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* report_detail = 1 */
+	/* all fixed lenght fields */
+	pframe = rtw_set_fixed_ie(pframe,
+		_FIXED_IE_LENGTH_, *ies, &my_len);
+
+	for (j = 0; j < prm->q.opt.bcn.opt_id_num; j++) {
+		switch (prm->q.opt.bcn.opt_id[j]) {
+		case bcn_req_ssid:
+			/* SSID */
+#if (RM_MORE_DBG_MSG)
+			RTW_INFO("RM: bcn_req_ssid\n");
+#endif
+			ptr = rtw_get_ie(*ies + _FIXED_IE_LENGTH_, _SSID_IE_,
+				&len, *ie_len - _FIXED_IE_LENGTH_);
+
+			if (ptr)
+				pframe = rtw_set_ie(pframe, _SSID_IE_, len, ptr + 2, &my_len);
+			break;
+		case bcn_req_req:
+			if (prm->q.opt.bcn.req_start == NULL)
+				break;
+#if (RM_MORE_DBG_MSG)
+			RTW_INFO("RM: bcn_req_req");
+#endif
+			for (k=0; k<prm->q.opt.bcn.req_len; k++) {
+				eid = prm->q.opt.bcn.req_start[k];
+
+				val8 = *ie_len - _FIXED_IE_LENGTH_;
+				ptr = rtw_get_ie(*ies + _FIXED_IE_LENGTH_,
+					eid, &len, val8);
+
+				if (!ptr)
+					continue;
+#if (RM_MORE_DBG_MSG)
+				switch (eid) {
+				case EID_SsId:
+					RTW_INFO("RM: EID_SSID\n");
+					break;
+				case EID_QBSSLoad:
+					RTW_INFO("RM: EID_QBSSLoad\n");
+					break;
+				case EID_HTCapability:
+					RTW_INFO("RM: EID_HTCapability\n");
+					break;
+				case _MDIE_:
+					RTW_INFO("RM: EID_MobilityDomain\n");
+					break;
+				case EID_Vendor:
+					RTW_INFO("RM: EID_Vendor\n");
+					break;
+				default:
+					RTW_INFO("RM: EID %d todo\n",eid);
+					break;
+				}
+#endif
+				pframe = rtw_set_ie(pframe, eid,
+					len, ptr + 2, &my_len);
+			} /* for() */
+			break;
+		case bcn_req_rep_detail:
+			RTW_INFO("RM: bcn_req_rep_detail\n");
+			break;
+		case bcn_req_ap_ch_rep:
+			RTW_INFO("RM: bcn_req_ap_ch_rep\n");
+			break;
+		default:
+			RTW_INFO("RM: OPT %d TODO\n",prm->q.opt.bcn.opt_id[j]);
+			break;
+		}
+	}
+	/*
+	 * update my length
+	 * content length does NOT include ID and LEN
+	 */
+	val8 = my_len - 2;
+	rtw_set_fixed_ie(plen, 1, &val8, &j);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	return pframe;
+}
+
+u8 rm_bcn_req_cond_mach(struct rm_obj *prm, struct wlan_network *pnetwork)
+{
+	u8 val8;
+
+
+	switch(prm->q.opt.bcn.rep_cond.cond) {
+	case bcn_rep_cond_immediately:
+		return _SUCCESS;
+	case bcn_req_cond_rcpi_greater:
+		val8 = rm_get_bcn_rcpi(prm, pnetwork);
+		if (val8 > prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	case bcn_req_cond_rcpi_less:
+		val8 = rm_get_bcn_rcpi(prm, pnetwork);
+		if (val8 < prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	case bcn_req_cond_rsni_greater:
+		val8 = rm_get_bcn_rsni(prm, pnetwork);
+		if (val8 != 255 && val8 > prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	case bcn_req_cond_rsni_less:
+		val8 = rm_get_bcn_rsni(prm, pnetwork);
+		if (val8 != 255 && val8 < prm->q.opt.bcn.rep_cond.threshold)
+			return _SUCCESS;
+		break;
+	default:
+		RTW_ERR("RM: bcn_req cond %u not support\n",
+			prm->q.opt.bcn.rep_cond.cond);
+		break;
+	}
+	return _FALSE;
+}
+
+static u8 *rm_gen_bcn_rep_ie (struct rm_obj *prm, u8 last,
+	u8 *pframe, struct wlan_network *pnetwork, unsigned int *fr_len)
+{
+	int snr, i = 0;
+	u8 val8, *plen, *ies, frag_id = 0, more;
+	u16 val16;
+	u32 val32, ie_len;
+	u64 val64;
+	unsigned int my_len;
+	_adapter *padapter = prm->psta->padapter;
+	u8 frag_bcn_idx[4] = {0x02, 0x2, 0x1, 0x0};
+	u8 last_bcn_ind[3] = {0xa4, 0x1, 0x0};
+
+	ies = pnetwork->network.IEs;
+
+	ie_len = pnetwork->network.IELength;
+more:
+	my_len = 0;
+	plen = pframe + 1;
+	pframe = rtw_set_fixed_ie(pframe, 7, &prm->p.e_id, &my_len);
+
+	/* Actual Measurement StartTime */
+	val64 = cpu_to_le64(prm->meas_start_time);
+	pframe = rtw_set_fixed_ie(pframe, 8, (u8 *)&val64, &my_len);
+
+	/* Measurement Duration */
+	val16 = prm->meas_end_time - prm->meas_start_time;
+	val16 = cpu_to_le16(val16);
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8*)&val16, &my_len);
+
+	/* TODO
+	* ReportedFrameInformation:
+	* 0 :beacon or probe rsp
+	* 1 :pilot frame
+	*/
+	val8 = 0; /* report frame info */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RCPI */
+	val8 = rm_get_bcn_rcpi(prm, pnetwork);
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RSNI */
+	val8 = rm_get_bcn_rsni(prm, pnetwork);
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* BSSID */
+	pframe = rtw_set_fixed_ie(pframe, 6,
+		(u8 *)&pnetwork->network.MacAddress, &my_len);
+
+	/*
+	 * AntennaID
+	 * 0: unknown
+	 * 255: multiple antenna (Diversity)
+	 */
+	val8 = 0;
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* ParentTSF */
+	val32 = prm->meas_start_time + pnetwork->network.PhyInfo.free_cnt;
+	pframe = rtw_set_fixed_ie(pframe, 4, (u8 *)&val32, &my_len);
+
+	/* Generate Beacon detail */
+
+	/* Reporting Detail values
+	 * 0: No fixed length fields or elements
+	 * 1: All fixed length fields and any requested elements
+	 *    in the Request info element if present
+	 * 2: All fixed length fields and elements
+	 * 3-255: Reserved
+	 */
+	if (prm->q.opt.bcn.rep_detail == 1) {
+		pframe = rm_gen_bcn_detail_1_elem(padapter, pframe, frag_id,
+			prm, &ies, &ie_len, &my_len);
+
+	} else if (prm->q.opt.bcn.rep_detail == 2) {
+		pframe = rm_gen_bcn_detail_2_elem(padapter, pframe, frag_id,
+			prm, &ies, &ie_len, &my_len);
+
+		/* fragment id num */
+		more = (ie_len > 0);
+		val8 = more << 7 | frag_id;
+		frag_bcn_idx[3] = val8;
+
+		pframe = rtw_set_fixed_ie(pframe, sizeof(frag_bcn_idx), frag_bcn_idx, &my_len);
+
+		if (prm->q.opt.bcn.rep_last_bcn_ind) {
+			last_bcn_ind[2] = last;
+			pframe = rtw_set_fixed_ie(pframe,
+				sizeof(last_bcn_ind), last_bcn_ind, &my_len);
+		}
+	}
+
+	/*
+	* update my length
+	* content length does NOT include ID and LEN
+	*/
+	val8 = my_len - 2;
+	rtw_set_fixed_ie(plen, 1, &val8, &i);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	if (ie_len && prm->q.opt.bcn.rep_detail == 2 && frag_id < 3) {
+		frag_id++;
+		goto more;
+	}
+
+	return pframe;
+}
+
+static int rm_match_sub_elem(_adapter *padapter,
+	struct rm_obj *prm, struct wlan_network *pnetwork)
+{
+	WLAN_BSSID_EX *pbss = &pnetwork->network;
+	unsigned int my_len;
+	int i, j, k, len;
+	u8 *plen;
+	u8 *ptr;
+	u8 eid;
+
+
+	my_len = 0;
+	/* Reporting Detail values
+	 * 0: No fixed length fields or elements
+	 * 1: All fixed length fields and any requested elements
+	 *    in the Request info element if present
+	 * 2: All fixed length fields and elements
+	 * 3-255: Reserved
+	 */
+
+	/* report_detail != 1  */
+	if (prm->q.opt.bcn.rep_detail != 1)
+		return _TRUE;
+
+	/* report_detail = 1 */
+
+	for (j = 0; j < prm->q.opt.bcn.opt_id_num; j++) {
+		switch (prm->q.opt.bcn.opt_id[j]) {
+		case bcn_req_ssid:
+			/* SSID */
+#ifdef RM_MORE_DBG_MSG
+			RTW_INFO("RM: bcn_req_ssid\n");
+#endif
+			if (pbss->Ssid.SsidLength == 0)
+				return _FALSE;
+			break;
+		case bcn_req_req:
+			if (prm->q.opt.bcn.req_start == NULL)
+				break;
+#ifdef RM_MORE_DBG_MSG
+			RTW_INFO("RM: bcn_req_req");
+#endif
+			for (k=0; k<prm->q.opt.bcn.req_len; k++) {
+				eid = prm->q.opt.bcn.req_start[k];
+
+				i = pbss->IELength - _FIXED_IE_LENGTH_;
+				ptr = rtw_get_ie(pbss->IEs + _FIXED_IE_LENGTH_,
+					eid, &len, i);
+
+#ifdef RM_MORE_DBG_MSG
+				switch (eid) {
+				case EID_SsId:
+					RTW_INFO("RM: EID_SSID\n");
+					break;
+				case EID_QBSSLoad:
+					RTW_INFO("RM: EID_QBSSLoad\n");
+					break;
+				case EID_HTCapability:
+					RTW_INFO("RM: EID_HTCapability\n");
+					break;
+				case _MDIE_:
+					RTW_INFO("RM: EID_MobilityDomain\n");
+					break;
+				case EID_Vendor:
+					RTW_INFO("RM: EID_Vendor\n");
+					break;
+				default:
+					RTW_INFO("RM: EID %d todo\n",eid);
+					break;
+				}
+#endif
+				if (!ptr) {
+					RTW_INFO("RM: EID %d not found\n",eid);
+					return _FALSE;
+				}
+			} /* for() */
+			break;
+		case bcn_req_rep_detail:
+			RTW_INFO("RM: bcn_req_rep_detail\n");
+			break;
+		case bcn_req_ap_ch_rep:
+			RTW_INFO("RM: bcn_req_ap_ch_rep\n");
+			break;
+		default:
+			RTW_INFO("RM: OPT %d TODO\n",prm->q.opt.bcn.opt_id[j]);
+			break;
+		}
+	}
+	return _TRUE;
+}
+
+static int retrieve_scan_result(struct rm_obj *prm)
+{
+	_list *plist, *phead;
+	_queue *queue;
+	_adapter *padapter = prm->psta->padapter;
+	struct rtw_ieee80211_channel *pch_set;
+	struct wlan_network *pnetwork = NULL;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	int i, j;
+	PWLAN_BSSID_EX pbss;
+	unsigned int matched_network;
+	int len, my_len;
+	u8 last = 0;
+	u8 buf_idx, *pbuf = NULL, *tmp_buf = NULL;
+	u16 xframe_ext_sz = SZ_XMITFRAME_EXT;
+
+
+	tmp_buf = rtw_malloc(xframe_ext_sz);
+	if (tmp_buf == NULL)
+		return 0;
+
+	matched_network = 0;
+	queue = &(pmlmepriv->scanned_queue);
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	/* get requested measurement channel set */
+	pch_set = prm->q.ch_set;
+
+	/* search scan queue to find requested SSID */
+	while (1) {
+
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+		pbss = &pnetwork->network;
+
+#if 0
+		RTW_INFO("RM: ch %u ssid %s bssid "MAC_FMT"\n",
+			pbss->Configuration.DSConfig, pbss->Ssid.Ssid,
+			MAC_ARG(pbss->MacAddress));
+		/*
+		* report network if requested channel set contains
+		* the channel matchs selected network
+		*/
+		if (rtw_chset_search_ch(adapter_to_chset(padapter),
+			pbss->Configuration.DSConfig) < 0) /* not match */
+			goto next;
+
+		if (rtw_mlme_band_check(padapter, pbss->Configuration.DSConfig)
+			== _FALSE)
+			goto next;
+#endif
+		if (rtw_validate_ssid(&(pbss->Ssid)) == _FALSE)
+			goto next;
+
+		/* match bssid */
+		if (is_wildcard_bssid(prm->q.bssid) == _FALSE)
+			if (_rtw_memcmp(prm->q.bssid,
+				pbss->MacAddress, 6) == _FALSE)
+				goto next;
+		/*
+		 * default wildcard SSID. wildcard SSID:
+		 * A SSID value (null) used to represent all SSIDs
+		 */
+
+		/* match ssid */
+		if ((prm->q.opt.bcn.ssid.SsidLength > 0) &&
+			_rtw_memcmp(prm->q.opt.bcn.ssid.Ssid,
+			pbss->Ssid.Ssid,
+			prm->q.opt.bcn.ssid.SsidLength) == _FALSE)
+			goto next;
+
+		/* go through measurement requested channels */
+		for (i = 0; i < prm->q.ch_set_ch_amount; i++) {
+			if ((pch_set[i].hw_value) ==
+				(pbss->Configuration.DSConfig)) /* match ch */
+				break;
+		}
+		if (i >= prm->q.ch_set_ch_amount) /* channel mismatch */
+			goto next;
+
+		/* match condition */
+		if (rm_bcn_req_cond_mach(prm, pnetwork) == _FALSE) {
+			RTW_INFO("RM: condition mismatch ch %u ssid %s bssid "MAC_FMT"\n",
+				pbss->Configuration.DSConfig, pbss->Ssid.Ssid,
+				MAC_ARG(pbss->MacAddress));
+			RTW_INFO("RM: condition %u:%u\n",
+				prm->q.opt.bcn.rep_cond.cond,
+				prm->q.opt.bcn.rep_cond.threshold);
+			goto next;
+		}
+
+		/* match subelement */
+		if (rm_match_sub_elem(padapter, prm, pnetwork) == _FALSE)
+			goto next;
+
+		/* Found a matched SSID */
+		matched_network++;
+
+		RTW_INFO("RM: ch %u Found %s bssid "MAC_FMT"\n",
+			pbss->Configuration.DSConfig, pbss->Ssid.Ssid,
+			MAC_ARG(pbss->MacAddress));
+
+		prm->ap[prm->ap_num++] = pnetwork;
+		if (prm->ap_num >= MAX_RM_AP_NUM)
+			break;
+next:
+		plist = get_next(plist);
+	} /* while() */
+
+	/* generate packet */
+	my_len = 0;
+	buf_idx = 0;
+	for (i = 0; i < prm->ap_num; i++) {
+		len = 0;
+		last = (i == (prm->ap_num - 1));
+		_rtw_memset(tmp_buf, 0, xframe_ext_sz);
+		pnetwork = prm->ap[i];
+		rm_gen_bcn_rep_ie(prm, last, tmp_buf, pnetwork, &len);
+		prm->ap[i] = NULL;
+new_packet:
+		if (my_len == 0) {
+			pbuf = rtw_malloc(xframe_ext_sz);
+			if (pbuf == NULL)
+				goto fail;
+			prm->buf[buf_idx].pbuf = pbuf;
+		}
+
+		if ((xframe_ext_sz - (my_len+len+24+4)) > 0) {
+			pbuf = rtw_set_fixed_ie(pbuf,
+				len, tmp_buf, &my_len);
+			prm->buf[buf_idx].len = my_len;
+		} else {
+			if (my_len == 0) /* not enough space */
+				goto fail;
+
+			my_len = 0;
+			buf_idx++;
+
+			if (buf_idx >= MAX_RM_PKT_NUM) {
+				for (j = i; j < prm->ap_num; j++)
+					prm->ap[i] = NULL;
+				break;
+			}
+
+			goto new_packet;
+		}
+	}
+
+fail:
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	if (tmp_buf)
+		rtw_mfree(tmp_buf, xframe_ext_sz);
+
+	RTW_INFO("RM: Found %d matched %s (%d)\n", matched_network,
+		prm->q.opt.bcn.ssid.Ssid, buf_idx+1);
+
+	if (prm->buf[buf_idx].pbuf)
+		return buf_idx + 1;
+
+	return 0;
+}
+
+int issue_beacon_rep(struct rm_obj *prm)
+{
+	int i, my_len;
+	u8 *pframe;
+	_adapter *padapter = prm->psta->padapter;
+	struct pkt_attrib *pattr;
+	struct xmit_frame *pmgntframe;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	int pkt_num;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u16 xframe_ext_sz = SZ_XMITFRAME_EXT;
+
+	pkt_num = retrieve_scan_result(prm);
+
+	if (pkt_num == 0) {
+		issue_null_reply(prm);
+		return _SUCCESS;
+	}
+
+	for (i=0;i<pkt_num;i++) {
+
+		pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+		if (pmgntframe == NULL) {
+			RTW_ERR("RM: %s alloc xmit_frame fail\n",__func__);
+			goto fail;
+		}
+		pattr = &pmgntframe->attrib;
+		pframe = build_wlan_hdr(padapter,
+			pmgntframe, prm->psta, WIFI_ACTION);
+		pframe = rtw_set_fixed_ie(pframe,
+			3, &prm->p.category, &pattr->pktlen);
+
+		my_len = 0;
+		pframe = rtw_set_fixed_ie(pframe,
+			prm->buf[i].len, prm->buf[i].pbuf, &my_len);
+
+		pattr->pktlen += my_len;
+		pattr->last_txcmdsz = pattr->pktlen;
+		dump_mgntframe(padapter, pmgntframe);
+	}
+fail:
+	/*GEORGIA_TODO_FIXIT*/
+	for (i = 0; i < pkt_num; i++) {
+		if (prm->buf[i].pbuf) {
+			rtw_mfree(prm->buf[i].pbuf, xframe_ext_sz);
+			prm->buf[i].pbuf = NULL;
+			prm->buf[i].len = 0;
+		}
+	}
+	return _SUCCESS;
+}
+
+/* neighbor request */
+int issue_nb_req(struct rm_obj *prm)
+{
+	_adapter *padapter = prm->psta->padapter;
+	struct sta_info *psta = prm->psta;
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pmgntframe = NULL;
+	struct pkt_attrib *pattr = NULL;
+	u8 val8;
+	u8 *pframe = NULL;
+
+
+	RTW_INFO("RM: %s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_ERR("RM: %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(padapter, pmgntframe, psta, WIFI_ACTION);
+	pframe = rtw_set_fixed_ie(pframe,
+		3, &prm->q.category, &pattr->pktlen);
+
+	if (prm->q.pssid) {
+
+		u8 sub_ie[64] = {0};
+		u8 *pie = &sub_ie[2];
+
+		RTW_INFO("RM: Send NB Req to "MAC_FMT" for(SSID) %s searching\n",
+			MAC_ARG(pmlmepriv->cur_network.network.MacAddress),
+			pmlmepriv->cur_network.network.Ssid.Ssid);
+
+		val8 = strlen(prm->q.pssid);
+		sub_ie[0] = 0; /*SSID*/
+		sub_ie[1] = val8;
+
+		_rtw_memcpy(pie, prm->q.pssid, val8);
+
+		pframe = rtw_set_fixed_ie(pframe, val8 + 2,
+			sub_ie, &pattr->pktlen);
+	} else {
+
+		if (!pmlmepriv->cur_network.network.Ssid.SsidLength)
+			RTW_INFO("RM: Send NB Req to "MAC_FMT"\n",
+				MAC_ARG(pmlmepriv->cur_network.network.MacAddress));
+		else {
+			u8 sub_ie[64] = {0};
+			u8 *pie = &sub_ie[2];
+
+			RTW_INFO("RM: Send NB Req to "MAC_FMT" for(SSID) %s searching\n",
+				MAC_ARG(pmlmepriv->cur_network.network.MacAddress),
+				pmlmepriv->cur_network.network.Ssid.Ssid);
+
+			sub_ie[0] = 0; /*SSID*/
+			sub_ie[1] = pmlmepriv->cur_network.network.Ssid.SsidLength;
+
+			_rtw_memcpy(pie, pmlmepriv->cur_network.network.Ssid.Ssid,
+				pmlmepriv->cur_network.network.Ssid.SsidLength);
+
+			pframe = rtw_set_fixed_ie(pframe,
+				pmlmepriv->cur_network.network.Ssid.SsidLength + 2,
+				sub_ie, &pattr->pktlen);
+		}
+	}
+
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+/* issue link measurement request */
+int issue_link_meas_req(struct rm_obj *prm)
+{
+	_adapter *padapter = prm->psta->padapter;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct sta_info *psta = prm->psta;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pmgntframe = NULL;
+	struct pkt_attrib *pattr = NULL;
+	u8 *pframe = NULL;
+	s8 pwr_used, path_a_pwr;
+
+
+	RTW_INFO("RM: %s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_ERR("RM: %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(padapter, pmgntframe, psta, WIFI_ACTION);
+
+	/* Category, Action code, Dialog token */
+	pframe = rtw_set_fixed_ie(pframe,
+		3, &prm->q.category, &pattr->pktlen);
+
+	/* xmit power used */
+	/* we don't know actual TX power due to RA may change TX rate;
+	 * But if we fix TX rate then we can get specific tx power
+	 */
+	pattr->rate = MGN_6M;
+	rm_get_tx_power(padapter, padapter_link->mlmeextpriv.chandef.band, MGN_6M, &pwr_used);
+	pframe = rtw_set_fixed_ie(pframe,
+		1, &pwr_used, &pattr->pktlen);
+
+	/* Max xmit power */
+	rm_get_path_a_max_tx_power(padapter, &path_a_pwr);
+	pframe = rtw_set_fixed_ie(pframe,
+		1, &path_a_pwr, &pattr->pktlen);
+
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+/* issue link measurement report */
+int issue_link_meas_rep(struct rm_obj *prm)
+{
+	u8 val8;
+	u8 *pframe;
+	unsigned int my_len;
+	_adapter *padapter = prm->psta->padapter;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattr;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sta_info *psta = prm->psta;
+	int i;
+	u8 tpc[4];
+	s8 pwr_used;
+
+
+	RTW_INFO("RM: %s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_ERR("RM: ERR %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(padapter, pmgntframe, psta, WIFI_ACTION);
+	/* Category, action code, Dialog token */
+	pframe = rtw_set_fixed_ie(pframe, 3,
+		&prm->p.category, &pattr->pktlen);
+
+	my_len = 0;
+
+	/* TPC report */
+	rm_get_tx_power(padapter, padapter_link->mlmeextpriv.chandef.band, MGN_6M, &pwr_used);
+	tpc[0] = EID_TPC;
+	tpc[1] = 2; /* length */
+
+	/* TX power */
+	tpc[2] = pwr_used;
+
+	/* link margin */
+	rm_get_rx_sensitivity(padapter, prm->q.rx_bw, prm->q.rx_rate, &pwr_used);
+	tpc[3] = prm->q.rx_pwr - pwr_used; /* RX sensitivity */
+	pattr->rate = MGN_6M; /* use fix rate to get fixed RX sensitivity */
+
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: rx_pwr=%ddBm - rx_sensitivity=%ddBm = link_margin=%ddB\n",
+		prm->q.rx_pwr, pwr_used, tpc[3]);
+#endif
+	pframe = rtw_set_fixed_ie(pframe, 4, tpc, &my_len);
+
+	/* RECV antenna ID */
+	val8 = 0; /* unknown antenna */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* XMIT antenna ID */
+	/* Fix rate 6M(1T) always use main antenna to TX */
+	val8 = 1; /* main antenna */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RCPI */
+	val8 = translate_dbm_to_rcpi(prm->q.rx_pwr);
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* RSNI */
+	val8 = prm->q.rx_rsni;
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* length */
+	//val8 = (u8)my_len-2;
+	//rtw_set_fixed_ie(plen, 1, &val8, &i); /* use variable i to ignore it */
+
+	pattr->pktlen += my_len;
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+static u8 *rm_gen_bcn_req_s_elem(_adapter *padapter,
+	struct rm_obj *prm, u8 *pframe, unsigned int *fr_len)
+{
+	u8 val8, l;
+	int i;
+	unsigned int my_len = 0;
+	struct _RT_OPERATING_CLASS *op;
+
+
+	val8 = bcn_req_active; /* measurement mode T8-64 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* bssid */
+	pframe = rtw_set_fixed_ie(pframe, 6, prm->q.bssid, &my_len);
+
+	/*
+	 * opt ssid (0)
+	 */
+	l = MIN(32, (int)prm->q.opt.bcn.ssid.SsidLength);
+
+	l = (int)prm->q.opt.bcn.ssid.SsidLength;
+
+	if (l > 32)
+		RTW_ERR("RM: %s SSID len over size %d! skip it!\n",__func__, l);
+
+	if (l > 0 && l <= 32) {
+		/* Type */
+		val8 = bcn_req_ssid;
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&val8, &my_len);
+		/* Len */
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&l, &my_len);
+		/* Value */
+		pframe = rtw_set_fixed_ie(pframe, l,
+			prm->q.opt.bcn.ssid.Ssid, &my_len);
+	}
+
+	/*
+	 * opt reporting detail (2)
+	 */
+	/* Type */
+	val8 = bcn_req_rep_detail;
+	pframe = rtw_set_fixed_ie(pframe, 1,
+		&val8, &my_len);
+	/* Len */
+	l = 1;
+	pframe = rtw_set_fixed_ie(pframe, 1,
+		&l, &my_len);
+	/* Value */
+	pframe = rtw_set_fixed_ie(pframe, l,
+		&prm->q.opt.bcn.rep_detail, &my_len);
+
+	/*
+	 * opt request (10)
+	 */
+
+	if (prm->q.opt.bcn.req_id_num > 0) {
+		/* Type */
+		val8 = bcn_req_req;
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&val8, &my_len);
+		/* Len */
+		l = prm->q.opt.bcn.req_id_num;
+		pframe = rtw_set_fixed_ie(pframe, 1,
+			&l, &my_len);
+		/* Value */
+		pframe = rtw_set_fixed_ie(pframe, l,
+			prm->q.opt.bcn.req_id, &my_len);
+	}
+
+	/*
+	 * opt ap channel report (51)
+	 */
+	for (i = 0; i < prm->q.opt.bcn.ap_ch_rpt_num; i++) {
+		op = prm->q.opt.bcn.ap_ch_rpt[i];
+		if (op == NULL)
+			break;
+		/* Type */
+		val8 = bcn_req_ap_ch_rep;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		l = (u8)op->Len + 1;
+		/* length */
+		pframe = rtw_set_fixed_ie(pframe, 1, &l, &my_len);
+
+		/* op class */
+		val8 = op->global_op_class;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		/* channel */
+		pframe = rtw_set_fixed_ie(pframe, op->Len, op->Channel, &my_len);
+	}
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	/* optional subelements */
+	return pframe;
+}
+
+static u8 *rm_gen_ch_load_req_s_elem(_adapter *padapter,
+	u8 *pframe, unsigned int *fr_len)
+{
+	u8 val8;
+	unsigned int my_len = 0;
+
+
+	val8 = 1; /* 1: channel load T8-60 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 2; /* channel load length = 2 (extensible)  */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* channel load condition : 0 (issue when meas done) T8-61 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* channel load reference value : 0 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	return pframe;
+}
+
+static u8 *rm_gen_noise_histo_req_s_elem(_adapter *padapter,
+	u8 *pframe, unsigned int *fr_len)
+{
+	u8 val8;
+	unsigned int my_len = 0;
+
+
+	val8 = 1; /* 1: noise histogram T8-62 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 2; /* noise histogram length = 2 (extensible)  */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* noise histogram condition : 0 (issue when meas done) T8-63 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	val8 = 0; /* noise histogram reference value : 0 */
+	pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+
+	/* update length to caller */
+	*fr_len += my_len;
+
+	return pframe;
+}
+
+int issue_radio_meas_req(struct rm_obj *prm)
+{
+	u8 val8;
+	u8 *pframe;
+	u8 *plen;
+	u16 val16;
+	int my_len, i = 0;
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattr;
+	_adapter *padapter = prm->psta->padapter;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+
+	RTW_INFO("RM: %s - %s\n", __func__, rm_type_req_name(prm->q.m_type));
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_ERR("RM: %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(padapter, pmgntframe, prm->psta, WIFI_ACTION);
+
+	/* Category, Action code, Dialog token */
+	pframe = rtw_set_fixed_ie(pframe, 3, &prm->q.category, &pattr->pktlen);
+
+	/* repeat */
+	val16 = cpu_to_le16(prm->q.rpt);
+	pframe = rtw_set_fixed_ie(pframe, 2,
+		(unsigned char *)&(val16), &pattr->pktlen);
+
+	my_len = 0;
+	plen = pframe + 1;
+
+	/* Element ID, Length, Meas token, Meas Mode, Meas type, op class, ch */
+	pframe = rtw_set_fixed_ie(pframe, 7, &prm->q.e_id, &my_len);
+
+	/* random interval */
+	val16 = cpu_to_le16(prm->q.rand_intvl); /* TU */
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8 *)&val16, &my_len);
+
+	/* measurement duration */
+	val16 = cpu_to_le16(prm->q.meas_dur);
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8 *)&val16, &my_len);
+
+	/* optional subelement */
+	switch (prm->q.m_type) {
+	case bcn_req:
+		pframe = rm_gen_bcn_req_s_elem(padapter, prm,  pframe, &my_len);
+		break;
+	case ch_load_req:
+		pframe = rm_gen_ch_load_req_s_elem(padapter, pframe, &my_len);
+		break;
+	case noise_histo_req:
+		pframe = rm_gen_noise_histo_req_s_elem(padapter,
+			pframe, &my_len);
+		break;
+	case basic_req:
+	default:
+		break;
+	}
+
+	/* length */
+	val8 = (u8)my_len - 2;
+	rtw_set_fixed_ie(plen, 1, &val8, &i);
+
+	pattr->pktlen += my_len;
+
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+int rm_radio_meas_report_cond(struct rm_obj *prm)
+{
+	u8 val8;
+	int i, ret = _FAIL;
+
+
+	switch (prm->q.m_type) {
+	case ch_load_req:
+		val8 = prm->p.ch_load;
+		switch (prm->q.opt.clm.rep_cond.cond) {
+		case ch_load_cond_immediately:
+			ret = _SUCCESS;
+			break;
+		case ch_load_cond_anpi_equal_greater:
+			if (val8 >= prm->q.opt.clm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		case ch_load_cond_anpi_equal_less:
+			if (val8 <= prm->q.opt.clm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		default:
+			break;
+		}
+		break;
+	case noise_histo_req:
+		val8 = prm->p.anpi;
+		switch (prm->q.opt.nhm.rep_cond.cond) {
+		case noise_histo_cond_immediately:
+			ret = _SUCCESS;
+			break;
+		case noise_histo_cond_anpi_equal_greater:
+			if (val8 >= prm->q.opt.nhm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		case noise_histo_cond_anpi_equal_less:
+			if (val8 <= prm->q.opt.nhm.rep_cond.threshold)
+				ret = _SUCCESS;
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+	return ret;
+}
+
+int retrieve_radio_meas_result(struct rm_obj *prm)
+{
+#ifdef CONFIG_RTW_ACS
+	struct dvobj_priv *dvobj = adapter_to_dvobj(prm->psta->padapter);
+#endif
+	int i, ch = -1;
+	u8 val8;
+
+
+	ch = rtw_chset_search_bch(adapter_to_chset(prm->psta->padapter),
+		prm->q.band, prm->q.ch_num);
+
+	if ((ch == -1) || (ch >= MAX_CHANNEL_NUM)) {
+		RTW_ERR("RM: get ch(CH:%d) fail\n", prm->q.ch_num);
+		ch = 0;
+	}
+
+	switch (prm->q.m_type) {
+	case ch_load_req:
+#if 0 /* def CONFIG_RTW_ACS */
+		val8 = hal_data->acs.clm_ratio[ch];
+#else
+		val8 = 0;
+#endif
+		prm->p.ch_load = val8;
+		break;
+	case noise_histo_req:
+#if 0 /* def CONFIG_RTW_ACS */
+		/* ANPI */
+		prm->p.anpi = hal_data->acs.nhm_ratio[ch];
+
+		/* IPI 0~10 */
+		for (i=0;i<11;i++)
+			prm->p.ipi[i] = hal_data->acs.nhm[ch][i];
+#else
+		val8 = 0;
+		prm->p.anpi = val8;
+		for (i=0;i<11;i++)
+			prm->p.ipi[i] = val8;
+#endif
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+int issue_radio_meas_rep(struct rm_obj *prm)
+{
+	u8 val8 = 0;
+	u8 *pframe = NULL;
+	u8 *plen = NULL;
+	u16 val16 = 0;
+	u64 val64 = 0;
+	unsigned int my_len = 0;
+	_adapter *padapter = prm->psta->padapter;
+	struct xmit_frame *pmgntframe = NULL;
+	struct pkt_attrib *pattr = NULL;
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct sta_info *psta = prm->psta;
+	int i = 0;
+
+
+	RTW_INFO("RM: %s\n", __func__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_ERR("RM: ERR %s alloc xmit_frame fail\n",__func__);
+		return _FALSE;
+	}
+	pattr = &pmgntframe->attrib;
+	pframe = build_wlan_hdr(padapter, pmgntframe, psta, WIFI_ACTION);
+	pframe = rtw_set_fixed_ie(pframe, 3,
+		&prm->p.category, &pattr->pktlen);
+
+	my_len = 0;
+	plen = pframe + 1;
+	pframe = rtw_set_fixed_ie(pframe, 7, &prm->p.e_id, &my_len);
+
+	/* Actual Meas start time - 8 bytes */
+	val64 = cpu_to_le64(prm->meas_start_time);
+	pframe = rtw_set_fixed_ie(pframe, 8, (u8 *)&val64, &my_len);
+
+	/* measurement duration */
+	val16 = prm->meas_end_time - prm->meas_start_time;
+	val16 = cpu_to_le16(val16);
+	pframe = rtw_set_fixed_ie(pframe, 2, (u8 *)&val16, &my_len);
+
+	/* optional subelement */
+	switch (prm->q.m_type) {
+	case ch_load_req:
+		val8 = prm->p.ch_load;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		break;
+	case noise_histo_req:
+		/*
+		 * AntennaID
+		 * 0: unknown
+		 * 255: multiple antenna (Diversity)
+		 */
+		val8 = 0;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		/* ANPI */
+		val8 = prm->p.anpi;
+		pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		/* IPI 0~10 */
+		for (i=0;i<11;i++) {
+			val8 = prm->p.ipi[i];
+			pframe = rtw_set_fixed_ie(pframe, 1, &val8, &my_len);
+		}
+		break;
+	default:
+		break;
+	}
+	/* length */
+	val8 = (u8)my_len-2;
+	rtw_set_fixed_ie(plen, 1, &val8, &i); /* use variable i to ignore it */
+
+	pattr->pktlen += my_len;
+	pattr->last_txcmdsz = pattr->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+void rtw_ap_parse_sta_rm_en_cap(_adapter *padapter,
+	struct sta_info *psta, struct rtw_ieee802_11_elems *elem)
+{
+	if (elem->rm_en_cap) {
+		RTW_INFO("assoc.rm_en_cap="RM_CAP_FMT"\n",
+			RM_CAP_ARG(elem->rm_en_cap));
+
+		_rtw_memcpy(psta->rm_en_cap, (elem->rm_en_cap),
+			MIN(elem->rm_en_cap_len, sizeof(psta->rm_en_cap)));
+	}
+}
+
+void RM_IE_handler(_adapter *padapter, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	int i;
+
+	_rtw_memcpy(&padapter->rmpriv.rm_en_cap_assoc, pIE->data,
+		    MIN(pIE->Length, sizeof(padapter->rmpriv.rm_en_cap_assoc)));
+	RTW_INFO("assoc.rm_en_cap="RM_CAP_FMT"\n", RM_CAP_ARG(pIE->data));
+}
+
+/* Debug command */
+
+#if (RM_SUPPORT_IWPRIV_DBG)
+static int hex2num(char c)
+{
+	if (c >= '0' && c <= '9')
+		return c - '0';
+	if (c >= 'a' && c <= 'f')
+		return c - 'a' + 10;
+	if (c >= 'A' && c <= 'F')
+		return c - 'A' + 10;
+	return -1;
+}
+
+int hex2byte(const char *hex)
+{
+	int a, b;
+	a = hex2num(*hex++);
+	if (a < 0)
+		return -1;
+	b = hex2num(*hex++);
+	if (b < 0)
+		return -1;
+	return (a << 4) | b;
+}
+
+static char * hwaddr_parse(char *txt, u8 *addr)
+{
+	size_t i;
+
+	for (i = 0; i < ETH_ALEN; i++) {
+		int a;
+
+		a = hex2byte(txt);
+		if (a < 0)
+			return NULL;
+		txt += 2;
+		addr[i] = a;
+		if (i < ETH_ALEN - 1 && *txt++ != ':')
+			return NULL;
+	}
+	return txt;
+}
+
+void rm_dbg_list_sta(_adapter *padapter, char *s)
+{
+	int i;
+	struct sta_info *psta;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	_list *plist, *phead;
+
+
+	sprintf(pstr(s), "\n");
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist,
+				struct sta_info, hash_list);
+
+			plist = get_next(plist);
+
+			sprintf(pstr(s), "=========================================\n");
+			sprintf(pstr(s), "mac=" MAC_FMT "\n",
+				MAC_ARG(psta->phl_sta->mac_addr));
+			sprintf(pstr(s), "state=0x%x, aid=%d, macid=%d\n",
+				psta->state, psta->phl_sta->aid, psta->phl_sta->macid);
+			sprintf(pstr(s), "rm_cap="RM_CAP_FMT"\n",
+				RM_CAP_ARG(psta->rm_en_cap));
+		}
+
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	sprintf(pstr(s), "=========================================\n");
+}
+
+void rm_dbg_help(_adapter *padapter, char *s)
+{
+	int i;
+
+
+	sprintf(pstr(s), "\n");
+	sprintf(pstr(s), "rrm list_sta\n");
+	sprintf(pstr(s), "rrm list_meas\n");
+
+	sprintf(pstr(s), "rrm add_meas <aid=1|mac=>,m=<bcn|clm|nhm|nb|link>,rpt=\n");
+	sprintf(pstr(s), "rrm run_meas <aid=1|evid=>\n");
+	sprintf(pstr(s), "rrm del_meas\n");
+
+	sprintf(pstr(s), "rrm run_meas rmid=xxxx,ev=xx\n");
+	sprintf(pstr(s), "rrm activate\n");
+
+	for (i=0;i<RM_EV_max;i++)
+		sprintf(pstr(s), "\t%2d %s\n",i, rm_event_name(i) );
+	sprintf(pstr(s), "\n");
+}
+
+struct sta_info *rm_get_sta(_adapter *padapter, u16 aid, u8* pbssid)
+{
+	int i;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	_list *plist, *phead;
+
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist,
+				struct sta_info, hash_list);
+
+			plist = get_next(plist);
+
+			if (psta->phl_sta->aid == aid)
+				goto done;
+
+			if (pbssid && _rtw_memcmp(psta->phl_sta->mac_addr,
+				pbssid, 6))
+				goto done;
+		}
+
+	}
+	psta = NULL;
+done:
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	return psta;
+}
+
+static int rm_dbg_modify_meas(_adapter *padapter, char *s)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct mlme_ext_info *pmlmeinfo = &padapter->mlmeextpriv.mlmext_info;
+	struct rm_obj *prm;
+	struct sta_info *psta;
+	char *pmac, *ptr, *paid, *prpt, *pnbp, *pclm, *pnhm, *pbcn, *plnk;
+	unsigned val;
+	u8 bssid[ETH_ALEN];
+	int i;
+
+	/* example :
+	* rrm add_meas <aid=1|mac=>,m=<nb|clm|nhm|bcn|link>,<rept=>
+	* rrm run_meas <aid=1|evid=>
+	*/
+	paid = strstr(s, "aid=");
+	pmac = strstr(s, "mac=");
+	pbcn = strstr(s, "m=bcn");
+	pclm = strstr(s, "m=clm");
+	pnhm = strstr(s, "m=nhm");
+	pnbp = strstr(s, "m=nb");
+	plnk = strstr(s, "m=link");
+	prpt = strstr(s, "rpt=");
+
+	/* set all ',' to NULL (end of line) */
+	ptr = s;
+	while (ptr) {
+		ptr = strchr(ptr, ',');
+		if (ptr) {
+			*(ptr) = 0x0;
+			ptr++;
+		}
+	}
+	prm = (struct rm_obj *)prmpriv->prm_sel;
+	prm->q.m_token = rm_gen_meas_token(padapter);
+	psta = prm->psta;
+
+	for (i=0;i<6;i++)
+		prm->q.bssid[i] = 0xff; /* wildcard bssid */
+	if (paid) { /* find sta_info according to aid */
+		paid += 4; /* skip aid= */
+		sscanf(paid, "%u", &val); /* aid=x */
+		psta = rm_get_sta(padapter, val, NULL);
+
+	} else if (pmac) { /* find sta_info according to bssid */
+		pmac += 4; /* skip mac= */
+		if (hwaddr_parse(pmac, bssid) == NULL) {
+			sprintf(pstr(s), "Err: \nincorrect mac format\n");
+			return _FAIL;
+		}
+		psta = rm_get_sta(padapter, 0xff, bssid);
+	}
+
+	if (psta) {
+		prm->psta = psta;
+
+		prm->q.diag_token = rm_gen_dialog_token(padapter);
+		prm->rmid = rm_gen_rmid(padapter, prm, RM_MASTER);
+	} else
+		return _FAIL;
+
+	prm->q.action_code = RM_ACT_RADIO_MEAS_REQ;
+	if (pbcn) {
+		prm->q.m_type = bcn_req;
+		prm->q.rand_intvl = le16_to_cpu(100);
+		prm->q.meas_dur = le16_to_cpu(100);
+	} else if (pnhm) {
+		prm->q.m_type = noise_histo_req;
+	} else if (pclm) {
+		prm->q.m_type = ch_load_req;
+	} else if (pnbp) {
+		prm->q.action_code = RM_ACT_NB_REP_REQ;
+	} else if (plnk) {
+		prm->q.action_code = RM_ACT_LINK_MEAS_REQ;
+	} else
+		return _FAIL;
+
+	if (prpt) {
+		prpt += 4; /* skip rpt= */
+		sscanf(prpt, "%u", &val);
+		prm->q.rpt = (u8)val;
+	}
+
+	return _SUCCESS;
+}
+
+static void rm_dbg_activate_meas(_adapter *padapter, char *s)
+{
+	struct rm_priv *prmpriv = &(padapter->rmpriv);
+	struct rm_obj *prm;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+
+	if (prmpriv->prm_sel == NULL) {
+		sprintf(pstr(s), "\nErr: No inActivate measurement\n");
+		return;
+	}
+	prm = (struct rm_obj *)prmpriv->prm_sel;
+
+	/* verify attributes */
+	if (prm->psta == NULL) {
+		sprintf(pstr(s), "\nErr: inActivate meas has no psta\n");
+		return;
+	}
+
+	/* measure current channel */
+	prm->q.ch_num = padapter_link->mlmeextpriv.chandef.chan;
+	prm->q.band = padapter_link->mlmeextpriv.chandef.band;
+	prm->q.op_class = rtw_get_op_class_by_bchbw(prm->q.band,
+		prm->q.ch_num, CHANNEL_WIDTH_20, CHAN_OFFSET_NO_EXT);
+
+	/* enquee rmobj */
+	rm_enqueue_rmobj(padapter, prm, _FALSE);
+
+	sprintf(pstr(s), "\nActivate rmid=%x, state=%s, meas_type=%s\n",
+		prm->rmid, rm_state_name(prm->state),
+		rm_type_req_name(prm->q.m_type));
+
+	sprintf(pstr(s), "aid=%d, mac=" MAC_FMT "\n",
+		prm->psta->phl_sta->aid, MAC_ARG(prm->psta->phl_sta->mac_addr));
+
+	/* clearn inActivate prm info */
+	prmpriv->prm_sel = NULL;
+}
+
+/* for ioctl */
+int rm_send_bcn_reqs(_adapter *padapter, u8 *sta_addr, u8 op_class, u8 ch,
+	u16 measure_duration, u8 measure_mode, u8 *bssid, u8 *ssid,
+	u8 reporting_detail,
+	u8 n_ap_ch_rpt, struct _RT_OPERATING_CLASS *rpt,
+	u8 n_elem_id, u8 *elem_id_list)
+{
+	struct rm_obj *prm;
+	char *pact;
+	struct sta_info *psta;
+	struct _RT_OPERATING_CLASS *prpt;
+	void *ptr;
+	int i,j,sz;
+	u8 bcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+
+	if (n_ap_ch_rpt > BCN_REQ_OPT_AP_CH_RPT_MAX_NUM) {
+		RTW_ERR("RM: chset num %d > %d\n",
+			n_ap_ch_rpt, BCN_REQ_OPT_AP_CH_RPT_MAX_NUM);
+		return -1;
+	}
+	/* dest sta */
+	psta = rtw_get_stainfo(&padapter->stapriv, sta_addr);
+        if (!psta) {
+		RTW_ERR("RM: psta not found\n");
+		return -2;
+        }
+	prm = rm_alloc_rmobj(padapter);
+	if (prm == NULL) {
+		RTW_ERR("RM: unable to alloc rm obj for requeset\n");
+		return -3;
+	}
+
+	prm->psta = psta;
+	prm->q.meas_dur = measure_duration;
+
+	/* Figure 8-104 Measurement Requested format */
+	prm->q.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+	prm->q.action_code = RM_ACT_RADIO_MEAS_REQ;
+	prm->q.m_mode = measure_mode;
+	prm->q.m_type = bcn_req;
+	prm->q.diag_token = rm_gen_dialog_token(padapter);
+	prm->q.m_token = rm_gen_meas_token(padapter);
+	prm->rmid = rm_gen_rmid(padapter, prm, RM_MASTER);
+
+	prm->q.e_id = _MEAS_REQ_IE_; /* 38 */
+	prm->q.ch_num = ch;
+	prm->q.op_class = op_class;
+	prm->q.band = rtw_get_band_by_op_class(op_class);
+	/* error handling */
+	if (prm->q.band == BAND_MAX)
+		prm->q.band = BAND_ON_24G;
+	prm->from_ioctl = true;
+
+	if (bssid != NULL)
+		memcpy(prm->q.bssid, bssid, ETH_ALEN);
+	else
+		memcpy(prm->q.bssid, bcast, ETH_ALEN);
+
+	if (ssid != NULL) {
+		i = MIN(32, strlen(ssid));
+		prm->q.opt.bcn.ssid.SsidLength = i;
+		memcpy(prm->q.opt.bcn.ssid.Ssid, ssid, i);
+	}
+
+	if (n_ap_ch_rpt > 0) {
+		prm->q.opt.bcn.ap_ch_rpt_num = n_ap_ch_rpt;
+		j = 0;
+		for (i = 0; i < n_ap_ch_rpt; i++) {
+			prpt = rpt++;
+			if (prpt == NULL)
+				break;
+
+			sz = sizeof(struct _RT_OPERATING_CLASS) * prpt->Len;
+			ptr = rtw_malloc(sz);
+			_rtw_memcpy(ptr, prpt, sz);
+			prm->q.opt.bcn.ap_ch_rpt[i] = (struct _RT_OPERATING_CLASS *)ptr;
+		}
+	}
+	prm->q.opt.bcn.rep_detail = reporting_detail;
+
+	if ((n_elem_id > 0) && (n_elem_id < BCN_REQ_REQ_OPT_MAX_NUM)) {
+		prm->q.opt.bcn.req_id_num = n_elem_id;
+		_rtw_memcpy(prm->q.opt.bcn.req_id, elem_id_list, n_elem_id);
+	}
+
+	/* enquee rmobj */
+	rm_enqueue_rmobj(padapter, prm, _FALSE);
+
+	RTW_INFO("\nAdd rmid=%x, meas_type=%s ok\n",
+		prm->rmid, rm_type_req_name(prm->q.m_type));
+
+	if (prm->psta)
+		RTW_INFO("mac="MAC_FMT"\n", MAC_ARG(prm->psta->phl_sta->mac_addr));
+	return 0;
+}
+
+void indicate_beacon_report(u8 *sta_addr,
+	u8 n_measure_rpt, u32 elem_len, u8 *elem)
+{
+	RTW_INFO("RM: recv bcn reprot from mac="MAC_FMT"\n", MAC_ARG(sta_addr));
+}
+
+static void rm_dbg_add_meas(_adapter *padapter, char *s)
+{
+	struct rm_priv *prmpriv = &(padapter->rmpriv);
+	struct rm_obj *prm;
+	char *pact;
+
+
+	/* example :
+	* rrm add_meas <aid=1|mac=>,m=<nb|clm|nhm|link>
+	* rrm run_meas <aid=1|evid=>
+	*/
+	prm = (struct rm_obj *)prmpriv->prm_sel;
+	if (prm == NULL)
+		prm = rm_alloc_rmobj(padapter);
+
+	if (prm == NULL) {
+		sprintf(pstr(s), "\nErr: alloc meas fail\n");
+		return;
+	}
+
+        prmpriv->prm_sel = prm;
+
+	pact = strstr(s, "act");
+	if (rm_dbg_modify_meas(padapter, s) == _FAIL) {
+
+		sprintf(pstr(s), "\nErr: add meas fail\n");
+		rm_free_rmobj(prm);
+		prmpriv->prm_sel = NULL;
+		return;
+	}
+	prm->q.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+	prm->q.e_id = _MEAS_REQ_IE_; /* 38 */
+
+	sprintf(pstr(s), "\nAdd rmid=%x, meas_type=%s ok\n",
+		prm->rmid, rm_type_req_name(prm->q.m_type));
+
+	if (prm->psta)
+		sprintf(pstr(s), "mac="MAC_FMT"\n",
+			MAC_ARG(prm->psta->phl_sta->mac_addr));
+
+	if (pact)
+		rm_dbg_activate_meas(padapter, pstr(s));
+}
+
+static void rm_dbg_del_meas(_adapter *padapter, char *s)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct rm_obj *prm = (struct rm_obj *)prmpriv->prm_sel;
+
+
+	if (prm) {
+		sprintf(pstr(s), "\ndelete rmid=%x\n",prm->rmid);
+
+		/* free inActivate meas - enqueue yet  */
+		prmpriv->prm_sel = NULL;
+		rtw_mfree(prmpriv->prm_sel, sizeof(struct rm_obj));
+	} else
+		sprintf(pstr(s), "Err: no inActivate measurement\n");
+}
+
+static void rm_dbg_run_meas(_adapter *padapter, char *s)
+{
+	struct rm_obj *prm;
+	char *pevid, *prmid;
+	u32 rmid, evid;
+
+
+	prmid = strstr(s, "rmid="); /* hex */
+	pevid = strstr(s, "evid="); /* dec */
+
+	if (prmid && pevid) {
+		prmid += 5; /* rmid= */
+		sscanf(prmid, "%x", &rmid);
+
+		pevid += 5; /* evid= */
+		sscanf(pevid, "%u", &evid);
+	} else {
+		sprintf(pstr(s), "\nErr: incorrect attribute\n");
+		return;
+	}
+
+	prm = rm_get_rmobj(padapter, rmid);
+
+	if (!prm) {
+		sprintf(pstr(s), "\nErr: measurement not found\n");
+		return;
+	}
+
+	if (evid >= RM_EV_max) {
+		sprintf(pstr(s), "\nErr: wrong event id\n");
+		return;
+	}
+
+	rm_post_event(padapter, prm->rmid, evid);
+	sprintf(pstr(s), "\npost %s to rmid=%x\n",rm_event_name(evid), rmid);
+}
+
+static void rm_dbg_show_meas(struct rm_obj *prm, char *s)
+{
+	struct sta_info *psta;
+
+	psta = prm->psta;
+
+	if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+
+		sprintf(pstr(s), "\nrmid=%x, meas_type=%s\n",
+			prm->rmid, rm_type_req_name(prm->q.m_type));
+
+	} else  if (prm->q.action_code == RM_ACT_NB_REP_REQ) {
+
+		sprintf(pstr(s), "\nrmid=%x, action=neighbor_req\n",
+			prm->rmid);
+	} else
+		sprintf(pstr(s), "\nrmid=%x, action=unknown\n",
+			prm->rmid);
+
+	if (psta)
+		sprintf(pstr(s), "aid=%d, mac="MAC_FMT"\n",
+			psta->phl_sta->aid, MAC_ARG(psta->phl_sta->mac_addr));
+
+	sprintf(pstr(s), "clock=%d, state=%s, rpt=%u/%u\n",
+		(int)ATOMIC_READ(&prm->pclock->counter),
+		rm_state_name(prm->state), prm->p.rpt, prm->q.rpt);
+}
+
+static void rm_dbg_list_meas(_adapter *padapter, char *s)
+{
+	int meas_amount;
+	struct rm_obj *prm;
+	struct sta_info *psta;
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	_queue *queue = &prmpriv->rm_queue;
+	_list *plist, *phead;
+	unsigned long sp_flags;
+
+	sprintf(pstr(s), "\n");
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+	meas_amount = 0;
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		prm = LIST_CONTAINOR(plist, struct rm_obj, list);
+		meas_amount++;
+		plist = get_next(plist);
+		psta = prm->psta;
+		sprintf(pstr(s), "=========================================\n");
+
+		rm_dbg_show_meas(prm, s);
+	}
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+	sprintf(pstr(s), "=========================================\n");
+
+	if (meas_amount==0) {
+		sprintf(pstr(s), "No Activate measurement\n");
+		sprintf(pstr(s), "=========================================\n");
+	}
+
+	if (prmpriv->prm_sel == NULL)
+		sprintf(pstr(s), "\nNo inActivate measurement\n");
+	else {
+		sprintf(pstr(s), "\ninActivate measurement\n");
+		rm_dbg_show_meas((struct rm_obj *)prmpriv->prm_sel, s);
+	}
+}
+#endif /* RM_SUPPORT_IWPRIV_DBG */
+
+int verify_bcn_req(_adapter *padapter, struct sta_info *psta)
+{
+	char *bssid =  NULL;
+	char ssid[] = "RealKungFu";
+	u8 op_class = 0;
+	u8 ch = 255;
+	u16 measure_duration = 100;
+	u8 reporting_detaial = 0;
+	u8 n_ap_ch_rpt = 6;
+	u8 measure_mode = bcn_req_active;
+	u8 req[] = {1,2,3};
+	u8 req_len = sizeof(req);
+
+
+	static RT_OPERATING_CLASS US[] = {
+	/* 0, OP_CLASS_NULL */	//{  0,  0, {}},
+	/* 1, OP_CLASS_1 */	{115,  4, {36, 40, 44, 48}},
+	/* 2, OP_CLASS_2 */	{118,  4, {52, 56, 60, 64}},
+	/* 3, OP_CLASS_3 */	{124,  4, {149, 153, 157, 161}},
+	/* 4, OP_CLASS_4 */	{121, 11, {100, 104, 108, 112, 116, 120, 124,
+						128, 132, 136, 140}},
+	/* 5, OP_CLASS_5 */	{125,  5, {149, 153, 157, 161, 165}},
+	/* 6, OP_CLASS_12 */	{ 81, 11, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}}
+	};
+
+	rm_send_bcn_reqs(padapter, psta->phl_sta->mac_addr, op_class, ch,
+		measure_duration, measure_mode, bssid, ssid,
+		reporting_detaial, n_ap_ch_rpt, US, req_len, req);
+	return 0;
+}
+
+void rm_dbg_cmd(_adapter *padapter, char *s)
+{
+	unsigned val;
+	char *paid;
+	struct sta_info *psta=NULL;
+
+#if (RM_SUPPORT_IWPRIV_DBG)
+	if (_rtw_memcmp(s, "help", 4)) {
+		rm_dbg_help(padapter, s);
+
+	} else if (_rtw_memcmp(s, "send_bcn_req", 12)) {
+
+		/* rtwpriv wls1 rrm send_bcn_req aid=1 */
+		paid = strstr(s, "aid=");
+		if (paid) { /* find sta_info according to aid */
+			paid += 4; /* skip aid= */
+			sscanf(paid, "%u", &val); /* aid=x */
+			psta = rm_get_sta(padapter, val, NULL);
+
+			if (psta)
+				verify_bcn_req(padapter, psta);
+		}
+
+	} else if (_rtw_memcmp(s, "list_sta", 8)) {
+		rm_dbg_list_sta(padapter, s);
+
+	} else if (_rtw_memcmp(s, "list_meas", 9)) {
+		rm_dbg_list_meas(padapter, s);
+
+	} else if (_rtw_memcmp(s, "add_meas", 8)) {
+		rm_dbg_add_meas(padapter, s);
+
+	} else if (_rtw_memcmp(s, "del_meas", 8)) {
+		rm_dbg_del_meas(padapter, s);
+
+	} else if (_rtw_memcmp(s, "activate", 8)) {
+		rm_dbg_activate_meas(padapter, s);
+
+	} else if (_rtw_memcmp(s, "run_meas", 8)) {
+		rm_dbg_run_meas(padapter, s);
+
+	} else if (_rtw_memcmp(s, "nb", 2)) {
+
+		paid = strstr(s, "aid=");
+
+		if (paid) { /* find sta_info according to aid */
+			paid += 4; /* skip aid= */
+			sscanf(paid, "%u", &val); /* aid=x */
+			psta = rm_get_sta(padapter, val, NULL);
+
+			if (psta)
+				rm_add_nb_req(padapter, psta);
+		}
+	}
+#else
+	sprintf(pstr(s), "\n");
+	sprintf(pstr(s), "rrm debug command was disabled\n");
+#endif
+}
+#endif /* CONFIG_RTW_80211K */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm_fsm.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm_fsm.c
new file mode 100644
index 000000000000..54efe1533a6e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm_fsm.c
@@ -0,0 +1,1016 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+#ifdef CONFIG_RTW_80211K
+#include "rtw_rm_fsm.h"
+#include "rtw_rm_util.h"
+
+struct fsm_state {
+	u8 *name;
+	int(*fsm_func)(struct rm_obj *prm, enum RM_EV_ID evid);
+};
+
+static void rm_state_initial(struct rm_obj *prm);
+static void rm_state_goto(struct rm_obj *prm, enum RM_STATE rm_state);
+static void rm_state_run(struct rm_obj *prm, enum RM_EV_ID evid);
+static struct rm_event *rm_dequeue_ev(_queue *queue);
+static struct rm_obj *rm_dequeue_rm(_queue *queue);
+
+void rm_timer_callback(void *data)
+{
+	int i;
+	_adapter *padapter = (_adapter *)data;
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct rm_clock *pclock;
+
+
+	/* deal with clock */
+	for (i=0;i<RM_TIMER_NUM;i++) {
+		pclock = &prmpriv->clock[i];
+		if (pclock->prm == NULL
+			||(ATOMIC_READ(&(pclock->counter)) == 0))
+			continue;
+
+		ATOMIC_DEC(&(pclock->counter));
+
+		if (ATOMIC_READ(&(pclock->counter)) == 0)
+			rm_post_event(pclock->prm->psta->padapter,
+				pclock->prm->rmid, prmpriv->clock[i].evid);
+	}
+	_set_timer(&prmpriv->rm_timer, CLOCK_UNIT);
+}
+
+int rtw_init_rm(_adapter *padapter)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+
+
+	RTW_INFO("RM: %s\n",__func__);
+	_rtw_init_queue(&(prmpriv->rm_queue));
+	_rtw_init_queue(&(prmpriv->ev_queue));
+
+	/* bit 0-7 */
+	prmpriv->rm_en_cap_def[0] = 0
+		| BIT(RM_LINK_MEAS_CAP_EN)
+		| BIT(RM_NB_REP_CAP_EN)
+		/*| BIT(RM_PARAL_MEAS_CAP_EN)*/
+		| BIT(RM_REPEAT_MEAS_CAP_EN)
+		| BIT(RM_BCN_PASSIVE_MEAS_CAP_EN)
+		| BIT(RM_BCN_ACTIVE_MEAS_CAP_EN)
+		| BIT(RM_BCN_TABLE_MEAS_CAP_EN)
+		| BIT(RM_BCN_MEAS_REP_COND_CAP_EN);
+
+	/* bit  8-15 */
+	prmpriv->rm_en_cap_def[1] = 0
+		/*| BIT(RM_FRAME_MEAS_CAP_EN - 8)*/
+#ifdef CONFIG_RTW_ACS
+		| BIT(RM_CH_LOAD_CAP_EN - 8)
+		| BIT(RM_NOISE_HISTO_CAP_EN - 8)
+#endif
+		/*| BIT(RM_STATIS_MEAS_CAP_EN - 8)*/
+		/*| BIT(RM_LCI_MEAS_CAP_EN - 8)*/
+		/*| BIT(RM_LCI_AMIMUTH_CAP_EN - 8)*/
+		/*| BIT(RM_TRANS_STREAM_CAT_MEAS_CAP_EN - 8)*/
+		/*| BIT(RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN - 8)*/;
+
+	/* bit 16-23 */
+	prmpriv->rm_en_cap_def[2] = 0
+		/*| BIT(RM_AP_CH_REP_CAP_EN - 16)*/
+		/*| BIT(RM_RM_MIB_CAP_EN - 16)*/
+		/*| BIT(RM_OP_CH_MAX_MEAS_DUR0 - 16)*/
+		/*| BIT(RM_OP_CH_MAX_MEAS_DUR1 - 16)*/
+		/*| BIT(RM_OP_CH_MAX_MEAS_DUR2 - 16)*/
+		/*| BIT(RM_NONOP_CH_MAX_MEAS_DUR0 - 16)*/
+		/*| BIT(RM_NONOP_CH_MAX_MEAS_DUR1 - 16)*/
+		/*| BIT(RM_NONOP_CH_MAX_MEAS_DUR2 - 16)*/;
+
+	/* bit 24-31 */
+	prmpriv->rm_en_cap_def[3] = 0
+		/*| BIT(RM_MEAS_PILOT_CAP0 - 24)*/
+		/*| BIT(RM_MEAS_PILOT_CAP1 - 24)*/
+		/*| BIT(RM_MEAS_PILOT_CAP2 - 24)*/
+		/*| BIT(RM_MEAS_PILOT_TRANS_INFO_CAP_EN - 24)*/
+		/*| BIT(RM_NB_REP_TSF_OFFSET_CAP_EN - 24)*/
+		| BIT(RM_RCPI_MEAS_CAP_EN - 24)
+		| BIT(RM_RSNI_MEAS_CAP_EN - 24)
+		/*| BIT(RM_BSS_AVG_ACCESS_DELAY_CAP_EN - 24)*/;
+
+	/* bit 32-39 */
+	prmpriv->rm_en_cap_def[4] = 0
+		/*| BIT(RM_BSS_AVG_ACCESS_DELAY_CAP_EN - 32)*/
+		/*| BIT(RM_AVALB_ADMIS_CAPACITY_CAP_EN - 32)*/
+		/*| BIT(RM_ANT_CAP_EN - 32)*/;
+
+	prmpriv->enable = _TRUE;
+	prmpriv->meas_token = 1;
+
+	/* clock timer */
+	rtw_init_timer(&prmpriv->rm_timer,
+		rm_timer_callback, padapter);
+	_set_timer(&prmpriv->rm_timer, CLOCK_UNIT);
+
+	return _SUCCESS;
+}
+
+int rtw_deinit_rm(_adapter *padapter)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct rm_obj *prm;
+	struct rm_event *pev;
+
+
+	RTW_INFO("RM: %s\n",__func__);
+	prmpriv->enable = _FALSE;
+	_cancel_timer_ex(&prmpriv->rm_timer);
+
+	/* free all events and measurements */
+	while((pev = rm_dequeue_ev(&prmpriv->ev_queue)) != NULL)
+		rtw_mfree((void *)pev, sizeof(struct rm_event));
+
+	while((prm = rm_dequeue_rm(&prmpriv->rm_queue)) != NULL)
+		rm_state_run(prm, RM_EV_cancel);
+
+	_rtw_deinit_queue(&(prmpriv->rm_queue));
+	_rtw_deinit_queue(&(prmpriv->ev_queue));
+
+	return _SUCCESS;
+}
+
+int rtw_free_rm_priv(_adapter *padapter)
+{
+	return rtw_deinit_rm(padapter);
+}
+
+static int rm_enqueue_ev(_queue *queue, struct rm_event *obj, bool to_head)
+{
+	unsigned long sp_flags;
+
+	if (obj == NULL)
+		return _FAIL;
+
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+
+	if (to_head)
+		rtw_list_insert_head(&obj->list, &queue->queue);
+	else
+		rtw_list_insert_tail(&obj->list, &queue->queue);
+
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+	return _SUCCESS;
+}
+
+static void rm_set_clock(struct rm_obj *prm, u32 ms, enum RM_EV_ID evid)
+{
+	ATOMIC_SET(&(prm->pclock->counter), (ms/CLOCK_UNIT));
+	prm->pclock->evid = evid;
+}
+
+static struct rm_clock *rm_alloc_clock(_adapter *padapter, struct rm_obj *prm)
+{
+	int i;
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct rm_clock *pclock = NULL;
+
+
+	for (i=0;i<RM_TIMER_NUM;i++) {
+		pclock = &prmpriv->clock[i];
+
+		if (pclock->prm == NULL) {
+			pclock->prm = prm;
+			ATOMIC_SET(&(pclock->counter), 0);
+			pclock->evid = RM_EV_max;
+			break;
+		}
+	}
+	return pclock;
+}
+
+static void rm_cancel_clock(struct rm_obj *prm)
+{
+	ATOMIC_SET(&(prm->pclock->counter), 0);
+	prm->pclock->evid = RM_EV_max;
+}
+
+static void rm_free_clock(struct rm_clock *pclock)
+{
+	pclock->prm = NULL;
+	ATOMIC_SET(&(pclock->counter), 0);
+	pclock->evid = RM_EV_max;
+}
+
+static int is_list_linked(const struct list_head *head)
+{
+	return head->prev != NULL;
+}
+
+void rm_free_rmobj(struct rm_obj *prm)
+{
+	if (is_list_linked(&prm->list))
+		rtw_list_delete(&prm->list);
+
+	if (prm->q.pssid)
+		rtw_mfree(prm->q.pssid, strlen(prm->q.pssid)+1);
+
+	if (prm->q.opt.bcn.req_start)
+		rtw_mfree(prm->q.opt.bcn.req_start,
+			prm->q.opt.bcn.req_len);
+
+	if (prm->pclock)
+		rm_free_clock(prm->pclock);
+
+	rtw_mfree((void *)prm, sizeof(struct rm_obj));
+}
+
+struct rm_obj *rm_alloc_rmobj(_adapter *padapter)
+{
+	struct rm_obj *prm;
+
+
+	prm = (struct rm_obj *)rtw_malloc(sizeof(struct rm_obj));
+	if (prm == NULL)
+		return NULL;
+
+	_rtw_memset(prm, 0, sizeof(struct rm_obj));
+
+	/* alloc timer */
+	if ((prm->pclock = rm_alloc_clock(padapter, prm)) == NULL) {
+		rm_free_rmobj(prm);
+		return NULL;
+	}
+	return prm;
+}
+
+int rm_enqueue_rmobj(_adapter *padapter, struct rm_obj *prm, bool to_head)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	_queue *queue = &prmpriv->rm_queue;
+	unsigned long sp_flags;
+
+	if (prm == NULL)
+		return _FAIL;
+
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+	if (to_head)
+		rtw_list_insert_head(&prm->list, &queue->queue);
+	else
+		rtw_list_insert_tail(&prm->list, &queue->queue);
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+	rm_state_initial(prm);
+
+	return _SUCCESS;
+}
+
+static struct rm_obj *rm_dequeue_rm(_queue *queue)
+{
+	struct rm_obj *prm;
+	unsigned long sp_flags;
+
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+	if (rtw_is_list_empty(&(queue->queue)))
+		prm = NULL;
+	else {
+		prm = LIST_CONTAINOR(get_next(&(queue->queue)),
+			struct rm_obj, list);
+		/* rtw_list_delete(&prm->list); */
+	}
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+	return prm;
+}
+
+static struct rm_event *rm_dequeue_ev(_queue *queue)
+{
+	struct rm_event *ev;
+	unsigned long sp_flags;
+
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+	if (rtw_is_list_empty(&(queue->queue)))
+		ev = NULL;
+	else {
+		ev = LIST_CONTAINOR(get_next(&(queue->queue)),
+			struct rm_event, list);
+		rtw_list_delete(&ev->list);
+	}
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+	return ev;
+}
+
+static struct rm_obj *_rm_get_rmobj(_queue *queue, u32 rmid)
+{
+	_list *phead, *plist;
+	struct rm_obj *prm = NULL;
+	unsigned long sp_flags;
+
+	if (rmid == 0)
+		return NULL;
+
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+
+		prm = LIST_CONTAINOR(plist, struct rm_obj, list);
+		if (rmid == (prm->rmid)) {
+			_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+			return prm;
+		}
+		plist = get_next(plist);
+	}
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+
+	return NULL;
+}
+
+struct sta_info *rm_get_psta(_adapter *padapter, u32 rmid)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct rm_obj *prm;
+
+
+	prm = _rm_get_rmobj(&prmpriv->rm_queue, rmid);
+
+	if (prm)
+		return prm->psta;
+
+	return NULL;
+}
+
+struct rm_obj *rm_get_rmobj(_adapter *padapter, u32 rmid)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+
+	return _rm_get_rmobj(&prmpriv->rm_queue, rmid);
+}
+
+u8 rtw_rm_post_envent_cmd(_adapter *padapter, u32 rmid, u8 evid)
+{
+	struct cmd_obj *pcmd;
+	struct rm_event *pev;
+	struct cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8 res = _SUCCESS;
+
+
+	pcmd = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmd == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pcmd->padapter = padapter;
+
+	pev = (struct rm_event*)rtw_zmalloc(sizeof(struct rm_event));
+
+	if (pev == NULL) {
+		rtw_mfree((u8 *) pcmd, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+	pev->rmid = rmid;
+	pev->evid = evid;
+
+	init_h2fwcmd_w_parm_no_rsp(pcmd, pev, CMD_RM_POST_EVENT);
+	res = rtw_enqueue_cmd(pcmdpriv, pcmd);
+exit:
+	return res;
+}
+
+int rm_post_event(_adapter *padapter, u32 rmid, enum RM_EV_ID evid)
+{
+	if (padapter->rmpriv.enable == _FALSE)
+		return _FALSE;
+
+	RTW_INFO("RM: post asyn %s to rmid=%x\n", rm_event_name(evid), rmid);
+	rtw_rm_post_envent_cmd(padapter, rmid, evid);
+	return _SUCCESS;
+}
+
+int _rm_post_event(_adapter *padapter, u32 rmid, enum RM_EV_ID evid)
+{
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct rm_event *pev;
+
+	if (evid >= RM_EV_max || rmid == 0)
+		return _FALSE;
+
+	pev = (struct rm_event *)rtw_malloc(sizeof(struct rm_event));
+	if (pev == NULL)
+		return _FALSE;
+
+	pev->rmid = rmid;
+	pev->evid = evid;
+
+	RTW_INFO("RM: post sync %s to rmid=%x\n", rm_event_name(evid), rmid);
+	rm_enqueue_ev(&prmpriv->ev_queue, pev, _FALSE);
+
+	return _SUCCESS;
+}
+
+static void rm_bcast_aid_handler(_adapter *padapter, struct rm_event *pev)
+{
+	_list *phead, *plist;
+	_queue *queue = &padapter->rmpriv.rm_queue;
+	struct rm_obj *prm;
+	unsigned long sp_flags;
+
+	_rtw_spinlock_irq(&queue->lock, &sp_flags);
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+
+		prm = LIST_CONTAINOR(plist, struct rm_obj, list);
+		plist = get_next(plist);
+		if (RM_GET_AID(pev->rmid) == RM_GET_AID(prm->rmid)) {
+			_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+			rm_state_run(prm, pev->evid);
+			_rtw_spinlock_irq(&queue->lock, &sp_flags);
+		}
+	}
+	_rtw_spinunlock_irq(&queue->lock, &sp_flags);
+	return;
+}
+
+/* main handler of RM (Resource Management) */
+void rm_handler(_adapter *padapter, struct rm_event *pe)
+{
+	int i;
+	struct rm_priv *prmpriv = &padapter->rmpriv;
+	struct rm_obj *prm;
+	struct rm_event *pev;
+
+
+	/* dequeue event */
+	while((pev = rm_dequeue_ev(&prmpriv->ev_queue)) != NULL)
+	{
+		if (RM_IS_ID_FOR_ALL(pev->rmid)) {
+			/* apply to all aid mateched measurement */
+			rm_bcast_aid_handler(padapter, pev);
+			rtw_mfree((void *)pev, sizeof(struct rm_event));
+			continue;
+		}
+
+		/* retrieve rmobj */
+		prm = _rm_get_rmobj(&prmpriv->rm_queue, pev->rmid);
+		if (prm == NULL) {
+			RTW_ERR("RM: rmid=%x event=%s doesn't find rm obj\n",
+				pev->rmid, rm_event_name(pev->evid));
+			rtw_mfree((void *)pev, sizeof(struct rm_event));
+			return;
+		}
+		/* run state machine */
+		rm_state_run(prm, pev->evid);
+		rtw_mfree((void *)pev, sizeof(struct rm_event));
+	}
+}
+
+static int rm_issue_meas_req(struct rm_obj *prm)
+{
+	switch (prm->q.action_code) {
+	case RM_ACT_RADIO_MEAS_REQ:
+		switch (prm->q.m_type) {
+		case bcn_req:
+		case ch_load_req:
+		case noise_histo_req:
+			issue_radio_meas_req(prm);
+			break;
+		default:
+			break;
+		} /* meas_type */
+		break;
+	case RM_ACT_NB_REP_REQ:
+		/* issue neighbor request */
+		issue_nb_req(prm);
+		break;
+	case RM_ACT_LINK_MEAS_REQ:
+		issue_link_meas_req(prm);
+		break;
+	default:
+		return _FALSE;
+	} /* action_code */
+
+	return _SUCCESS;
+}
+
+/*
+* RM state machine
+*/
+
+static int rm_state_idle(struct rm_obj *prm, enum RM_EV_ID evid)
+{
+	_adapter *padapter = prm->psta->padapter;
+	struct _ADAPTER_LINK *padapter_link = prm->psta->padapter_link;
+	u8 val8;
+	u32 val32;
+
+
+	prm->p.category = RTW_WLAN_CATEGORY_RADIO_MEAS;
+
+	switch (evid) {
+	case RM_EV_state_in:
+		switch (prm->q.action_code) {
+		case RM_ACT_RADIO_MEAS_REQ:
+			/* copy attrib from meas_req to meas_rep */
+			prm->p.action_code = RM_ACT_RADIO_MEAS_REP;
+			prm->p.diag_token = prm->q.diag_token;
+			prm->p.e_id = _MEAS_RSP_IE_;
+			prm->p.m_token = prm->q.m_token;
+			prm->p.m_type = prm->q.m_type;
+			prm->p.rpt = prm->q.rpt;
+			prm->p.ch_num = prm->q.ch_num;
+			prm->p.op_class = prm->q.op_class;
+			prm->p.band = prm->q.band;
+
+			if (prm->q.m_type == ch_load_req
+				|| prm->q.m_type == noise_histo_req) {
+				/*
+				 * phydm measure current ch periodically
+				 * scan current ch is not necessary
+				 */
+				val8 = padapter_link->mlmeextpriv.chandef.chan;
+				if (prm->q.ch_num == val8)
+					prm->poll_mode = 1;
+			}
+			RTW_INFO("RM: rmid=%x %s switch in repeat=%u\n",
+				prm->rmid, rm_type_req_name(prm->q.m_type),
+				prm->q.rpt);
+			break;
+		case RM_ACT_NB_REP_REQ:
+			prm->p.action_code = RM_ACT_NB_REP_RESP;
+			RTW_INFO("RM: rmid=%x Neighbor request switch in\n",
+				prm->rmid);
+			break;
+		case RM_ACT_LINK_MEAS_REQ:
+			prm->p.diag_token = prm->q.diag_token;
+			prm->p.action_code = RM_ACT_LINK_MEAS_REP;
+			RTW_INFO("RM: rmid=%x Link meas switch in\n",
+				prm->rmid);
+			break;
+		default:
+			prm->p.action_code = prm->q.action_code;
+			rm_set_rep_mode(prm, MEAS_REP_MOD_INCAP);
+			RTW_INFO("RM: rmid=%x recv unknown action %d\n",
+				prm->rmid,prm->p.action_code);
+			break;
+		} /* switch() */
+
+		if (prm->rmid & RM_MASTER) {
+			if (rm_issue_meas_req(prm) == _SUCCESS)
+				rm_state_goto(prm, RM_ST_WAIT_MEAS);
+			else
+				rm_state_goto(prm, RM_ST_END);
+			return _SUCCESS;
+		} else {
+			rm_state_goto(prm, RM_ST_DO_MEAS);
+			return _SUCCESS;
+		}
+
+		if (prm->p.m_mode) {
+			issue_null_reply(prm);
+			rm_state_goto(prm, RM_ST_END);
+			return _SUCCESS;
+		}
+		if (prm->q.rand_intvl) {
+		#if 0 /*GEORGIA_TODO_REDEFINE_IO*/
+			/* get low tsf to generate random interval */
+			val32 = rtw_read32(padapter, REG_TSFTR);
+		#endif
+			val32 = rtw_hal_get_ltsf(padapter);
+			val32 = val32 % prm->q.rand_intvl;
+			RTW_INFO("RM: rmid=%x rand_intval=%d, rand=%d\n",
+				prm->rmid, (int)prm->q.rand_intvl,val32);
+			rm_set_clock(prm, prm->q.rand_intvl,
+				RM_EV_delay_timer_expire);
+			return _SUCCESS;
+		}
+		break;
+	case RM_EV_delay_timer_expire:
+		rm_state_goto(prm, RM_ST_DO_MEAS);
+		break;
+	case RM_EV_cancel:
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_state_out:
+		rm_cancel_clock(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+/* we do the measuring */
+static int rm_state_do_meas(struct rm_obj *prm, enum RM_EV_ID evid)
+{
+	_adapter *padapter = prm->psta->padapter;
+	u8 val8;
+	u64 val64;
+
+
+	switch (evid) {
+	case RM_EV_state_in:
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			switch (prm->q.m_type) {
+			case bcn_req:
+				if (prm->q.m_mode == bcn_req_bcn_table) {
+					RTW_INFO("RM: rmid=%x Beacon table\n",
+						prm->rmid);
+					rm_get_chset(prm);
+					_rm_post_event(padapter, prm->rmid,
+						RM_EV_survey_done);
+					return _SUCCESS;
+				}
+				break;
+			case ch_load_req:
+			case noise_histo_req:
+				if (prm->poll_mode)
+					_rm_post_event(padapter, prm->rmid,
+						RM_EV_survey_done);
+				return _SUCCESS;
+			default:
+				rm_state_goto(prm, RM_ST_END);
+				return _SUCCESS;
+			}
+
+			if (!ready_for_scan(prm)) {
+				prm->wait_busy = RM_BUSY_TRAFFIC_TIMES;
+				RTW_INFO("RM: wait busy traffic - %d\n",
+					prm->wait_busy);
+				rm_set_clock(prm, RM_WAIT_BUSY_TIMEOUT,
+					RM_EV_busy_timer_expire);
+				return _SUCCESS;
+			}
+		} else if (prm->q.action_code == RM_ACT_LINK_MEAS_REQ) {
+			; /* do nothing */
+			rm_state_goto(prm, RM_ST_SEND_REPORT);
+			return _SUCCESS;
+		}
+		_rm_post_event(padapter, prm->rmid, RM_EV_start_meas);
+		break;
+	case RM_EV_start_meas:
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			/* resotre measurement start time */
+			prm->meas_start_time = rtw_hal_get_tsftr_by_port(padapter
+									, rtw_hal_get_port(padapter));
+
+			switch (prm->q.m_type) {
+			case bcn_req:
+				val8 = 1; /* Enable free run counter */
+				rtw_hal_set_hwreg(padapter,
+					HW_VAR_FREECNT, &val8);
+				rm_sitesurvey(prm);
+				break;
+			case ch_load_req:
+			case noise_histo_req:
+				rm_sitesurvey(prm);
+				break;
+			default:
+				rm_state_goto(prm, RM_ST_END);
+				return _SUCCESS;
+				break;
+			}
+		}
+		/* handle measurement timeout */
+		rm_set_clock(prm, RM_MEAS_TIMEOUT, RM_EV_meas_timer_expire);
+		break;
+	case RM_EV_survey_done:
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			switch (prm->q.m_type) {
+			case bcn_req:
+				rm_cancel_clock(prm);
+				rm_state_goto(prm, RM_ST_SEND_REPORT);
+				return _SUCCESS;
+			case ch_load_req:
+			case noise_histo_req:
+				retrieve_radio_meas_result(prm);
+
+				if (rm_radio_meas_report_cond(prm) == _SUCCESS)
+					rm_state_goto(prm, RM_ST_SEND_REPORT);
+				else
+					rm_set_clock(prm, RM_COND_INTVL,
+						RM_EV_retry_timer_expire);
+				break;
+			default:
+				rm_state_goto(prm, RM_ST_END);
+				return _SUCCESS;
+			}
+		}
+		break;
+	case RM_EV_meas_timer_expire:
+		RTW_INFO("RM: rmid=%x measurement timeount\n",prm->rmid);
+		rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+		issue_null_reply(prm);
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_busy_timer_expire:
+		if (!ready_for_scan(prm) && prm->wait_busy--) {
+			RTW_INFO("RM: wait busy - %d\n",prm->wait_busy);
+			rm_set_clock(prm, RM_WAIT_BUSY_TIMEOUT,
+				RM_EV_busy_timer_expire);
+			break;
+		}
+		else if (prm->wait_busy <= 0) {
+			RTW_INFO("RM: wait busy timeout\n");
+			rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+			issue_null_reply(prm);
+			rm_state_goto(prm, RM_ST_END);
+			return _SUCCESS;
+		}
+		_rm_post_event(padapter, prm->rmid, RM_EV_start_meas);
+		break;
+	case RM_EV_request_timer_expire:
+		rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+		issue_null_reply(prm);
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_retry_timer_expire:
+		/* expired due to meas condition mismatch, meas again */
+		_rm_post_event(padapter, prm->rmid, RM_EV_start_meas);
+		break;
+	case RM_EV_cancel:
+		rm_set_rep_mode(prm, MEAS_REP_MOD_REFUSE);
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_state_out:
+		rm_cancel_clock(prm);
+		/* resotre measurement end time */
+		prm->meas_end_time = rtw_hal_get_tsftr_by_port(padapter
+								, rtw_hal_get_port(padapter));
+
+		val8 = 0; /* Disable free run counter */
+		rtw_hal_set_hwreg(padapter, HW_VAR_FREECNT, &val8);
+		break;
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static int rm_state_wait_meas(struct rm_obj *prm, enum RM_EV_ID evid)
+{
+	u8 val8;
+	u64 val64;
+
+
+	switch (evid) {
+	case RM_EV_state_in:
+		/* we create meas_req, waiting for peer report */
+		rm_set_clock(prm, RM_REQ_TIMEOUT,
+			RM_EV_request_timer_expire);
+		break;
+	case RM_EV_recv_rep:
+		rm_state_goto(prm, RM_ST_RECV_REPORT);
+		break;
+	case RM_EV_request_timer_expire:
+	case RM_EV_cancel:
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_state_out:
+		rm_cancel_clock(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static int rm_state_send_report(struct rm_obj *prm, enum RM_EV_ID evid)
+{
+	u8 val8;
+
+
+	switch (evid) {
+	case RM_EV_state_in:
+		/* we have to issue report */
+		if (prm->q.action_code == RM_ACT_RADIO_MEAS_REQ) {
+			switch (prm->q.m_type) {
+			case bcn_req:
+				issue_beacon_rep(prm);
+				break;
+			case ch_load_req:
+			case noise_histo_req:
+				issue_radio_meas_rep(prm);
+				break;
+			default:
+				rm_state_goto(prm, RM_ST_END);
+				return _SUCCESS;
+			}
+
+		} else if (prm->q.action_code == RM_ACT_LINK_MEAS_REQ) {
+			issue_link_meas_rep(prm);
+			rm_state_goto(prm, RM_ST_END);
+			return _SUCCESS;
+
+		} else {
+			rm_state_goto(prm, RM_ST_END);
+			return _SUCCESS;
+		}
+
+		/* check repeat */
+		if (prm->p.rpt) {
+			RTW_INFO("RM: rmid=%x repeat=%u/%u\n",
+				prm->rmid, prm->p.rpt,
+				prm->q.rpt);
+			prm->p.rpt--;
+			/*
+			* we recv meas_req,
+			* delay for a wihile and than meas again
+			*/
+			if (prm->poll_mode)
+				rm_set_clock(prm, RM_REPT_POLL_INTVL,
+					RM_EV_repeat_delay_expire);
+			else
+				rm_set_clock(prm, RM_REPT_SCAN_INTVL,
+					RM_EV_repeat_delay_expire);
+			return _SUCCESS;
+		}
+		/* we are done */
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_repeat_delay_expire:
+		rm_state_goto(prm, RM_ST_DO_MEAS);
+		break;
+	case RM_EV_cancel:
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_state_out:
+		rm_cancel_clock(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static int rm_state_recv_report(struct rm_obj *prm, enum RM_EV_ID evid)
+{
+	u8 val8;
+
+
+	switch (evid) {
+	case RM_EV_state_in:
+		/* we issue meas_req, got peer's meas report */
+		switch (prm->p.action_code) {
+		case RM_ACT_RADIO_MEAS_REP:
+			/* check refuse, incapable and repeat */
+			val8 = prm->p.m_mode;
+			if (val8) {
+				RTW_INFO("RM: rmid=%x peer reject (%s repeat=%d)\n",
+					prm->rmid,
+					val8|MEAS_REP_MOD_INCAP?"INCAP":
+					val8|MEAS_REP_MOD_REFUSE?"REFUSE":
+					val8|MEAS_REP_MOD_LATE?"LATE":"",
+					prm->p.rpt);
+				rm_state_goto(prm, RM_ST_END);
+				return _SUCCESS;
+			}
+			break;
+		case RM_ACT_NB_REP_RESP:
+			/* report to upper layer if needing */
+			rm_state_goto(prm, RM_ST_END);
+			return _SUCCESS;
+		default:
+			rm_state_goto(prm, RM_ST_END);
+			return _SUCCESS;
+		}
+		/* check repeat */
+		if (prm->p.rpt) {
+			RTW_INFO("RM: rmid=%x repeat=%u/%u\n",
+				prm->rmid, prm->p.rpt,
+				prm->q.rpt);
+			prm->p.rpt--;
+			/* waitting more report */
+			rm_state_goto(prm, RM_ST_WAIT_MEAS);
+			break;
+		}
+		/* we are done */
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_cancel:
+		rm_state_goto(prm, RM_ST_END);
+		break;
+	case RM_EV_state_out:
+		rm_cancel_clock(prm);
+		break;
+	default:
+		break;
+	}
+	return _SUCCESS;
+}
+
+static int rm_state_end(struct rm_obj *prm, enum RM_EV_ID evid)
+{
+	switch (evid) {
+	case RM_EV_state_in:
+		_rm_post_event(prm->psta->padapter, prm->rmid, RM_EV_state_out);
+		break;
+
+	case RM_EV_cancel:
+	case RM_EV_state_out:
+	default:
+		rm_free_rmobj(prm);
+		break;
+	}
+	return _SUCCESS;
+}
+
+struct fsm_state rm_fsm[] = {
+	{"RM_ST_IDLE",		rm_state_idle},
+	{"RM_ST_DO_MEAS",	rm_state_do_meas},
+	{"RM_ST_WAIT_MEAS", 	rm_state_wait_meas},
+	{"RM_ST_SEND_REPORT", 	rm_state_send_report},
+	{"RM_ST_RECV_REPORT", 	rm_state_recv_report},
+	{"RM_ST_END", 		rm_state_end}
+};
+
+char *rm_state_name(enum RM_STATE state)
+{
+	return rm_fsm[state].name;
+}
+
+char *rm_event_name(enum RM_EV_ID evid)
+{
+	switch(evid) {
+	case RM_EV_state_in:
+		return "RM_EV_state_in";
+	case RM_EV_busy_timer_expire:
+		return "RM_EV_busy_timer_expire";
+	case RM_EV_delay_timer_expire:
+		return "RM_EV_delay_timer_expire";
+	case RM_EV_meas_timer_expire:
+		return "RM_EV_meas_timer_expire";
+	case RM_EV_repeat_delay_expire:
+		return "RM_EV_repeat_delay_expire";
+	case RM_EV_retry_timer_expire:
+		return "RM_EV_retry_timer_expire";
+	case RM_EV_request_timer_expire:
+		return "RM_EV_request_timer_expire";
+	case RM_EV_wait_report:
+		return "RM_EV_wait_report";
+	case RM_EV_start_meas:
+		return "RM_EV_start_meas";
+	case RM_EV_survey_done:
+		return "RM_EV_survey_done";
+	case RM_EV_recv_rep:
+		return "RM_EV_recv_report";
+	case RM_EV_cancel:
+		return "RM_EV_cancel";
+	case RM_EV_state_out:
+		return "RM_EV_state_out";
+	case RM_EV_max:
+		return "RM_EV_max";
+	default:
+		return "RM_EV_unknown";
+	}
+	return "UNKNOWN";
+}
+
+static void rm_state_initial(struct rm_obj *prm)
+{
+	prm->state = RM_ST_IDLE;
+
+	RTW_INFO("\n");
+	RTW_INFO("RM: rmid=%x %-18s -> %s\n",prm->rmid,
+		"new measurement", rm_fsm[prm->state].name);
+
+	rm_post_event(prm->psta->padapter, prm->rmid, RM_EV_state_in);
+}
+
+static void rm_state_run(struct rm_obj *prm, enum RM_EV_ID evid)
+{
+	RTW_INFO("RM: rmid=%x %-18s    %s\n",prm->rmid,
+		rm_fsm[prm->state].name,rm_event_name(evid));
+
+	rm_fsm[prm->state].fsm_func(prm, evid);
+}
+
+static void rm_state_goto(struct rm_obj *prm, enum RM_STATE rm_state)
+{
+	if (prm->state == rm_state)
+		return;
+
+	rm_state_run(prm, RM_EV_state_out);
+
+	RTW_INFO("\n");
+	RTW_INFO("RM: rmid=%x %-18s -> %s\n",prm->rmid,
+		rm_fsm[prm->state].name, rm_fsm[rm_state].name);
+
+	prm->state = rm_state;
+	rm_state_run(prm, RM_EV_state_in);
+}
+#endif /* CONFIG_RTW_80211K */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm_util.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm_util.c
new file mode 100644
index 000000000000..f39ec82bd009
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_rm_util.c
@@ -0,0 +1,460 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+#ifdef CONFIG_RTW_80211K
+#include "rtw_rm_fsm.h"
+#include "rtw_rm_util.h"
+
+u8 rm_get_ch_set(u8 op_class, struct rtw_ieee80211_channel *pch_set, u8 pch_num)
+{
+	int i, array_idx;
+	const struct op_class_t *opc = get_global_op_class_by_id(op_class);
+
+	if (opc < global_op_class
+		|| (((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t)
+	) {
+		RTW_ERR("Invalid opc pointer:%p (global_op_class:%p, sizeof(struct op_class_t):%zu, %zu)\n"
+			, opc, global_op_class, sizeof(struct op_class_t),
+			(((u8 *)opc) - ((u8 *)global_op_class)) % sizeof(struct op_class_t));
+		return 0;
+	}
+
+	array_idx = (((u8 *)opc) - ((u8 *)global_op_class)) / sizeof(struct op_class_t);
+	if (pch_num < OPC_CH_LIST_LEN(global_op_class[array_idx])) {
+		RTW_ERR("Invalid pch len %d < %d\n",pch_num,
+			OPC_CH_LIST_LEN(global_op_class[array_idx]));
+		return 0;
+	}
+
+	for (i = 0; i < OPC_CH_LIST_LEN(global_op_class[array_idx]); i++) {
+		pch_set[i].hw_value = OPC_CH_LIST_CH(global_op_class[array_idx], i);
+		pch_set[i].band = global_op_class[array_idx].band;
+	}
+
+	return i;
+}
+
+
+u8 rm_get_ch_set_from_bcn_req_opt(struct bcn_req_opt *opt,
+	struct rtw_ieee80211_channel *pch_set, u8 pch_num)
+{
+	int i,j,k,sz;
+	struct _RT_OPERATING_CLASS *ap_ch_rpt;
+	enum band_type band;
+	u8 ch_amount = 0;
+
+	k = 0;
+	for (i = 0; i < opt->ap_ch_rpt_num; i++) {
+		if (opt->ap_ch_rpt[i] == NULL)
+			break;
+
+		ap_ch_rpt = opt->ap_ch_rpt[i];
+		band = rtw_get_band_by_op_class(ap_ch_rpt->global_op_class);
+		/* error handling */
+		if (band == BAND_MAX)
+			band = BAND_ON_24G;
+
+		if ((k + ap_ch_rpt->Len) > pch_num) {
+			RTW_ERR("RM: ch num exceed %d > %d\n", (k + ap_ch_rpt->Len), pch_num);
+			return k;
+		}
+
+		for (j = 0; j < ap_ch_rpt->Len; j++) {
+			pch_set[k].hw_value =
+				ap_ch_rpt->Channel[j];
+			pch_set[k].band = band;
+			RTW_INFO("RM: meas_ch[%d].hw_value = %u\n",
+				j, pch_set[k].hw_value);
+			k++;
+		}
+	}
+	return k;
+}
+
+int is_wildcard_bssid(u8 *bssid)
+{
+	int i;
+	u8 val8 = 0xff;
+
+
+	for (i=0;i<6;i++)
+		val8 &= bssid[i];
+
+	if (val8 == 0xff)
+		return _SUCCESS;
+	return _FALSE;
+}
+
+u8 translate_dbm_to_rcpi(s8 SignalPower)
+{
+	/* RCPI = Int{(Power in dBm + 110)*2} for 0dBm > Power > -110dBm
+	 *    0	: power <= -110.0 dBm
+	 *    1	: power =  -109.5 dBm
+	 *    2	: power =  -109.0 dBm
+	 */
+	return (SignalPower + 110)*2;
+}
+
+u8 translate_percentage_to_rcpi(u32 SignalStrengthIndex)
+{
+	/* Translate to dBm (x=y-100) */
+	return translate_dbm_to_rcpi(SignalStrengthIndex - 100);
+}
+
+u8 rm_get_bcn_rcpi(struct rm_obj *prm, struct wlan_network *pnetwork)
+{
+	return translate_percentage_to_rcpi(
+		pnetwork->network.PhyInfo.SignalStrength);
+}
+
+u8 rm_get_frame_rsni(struct rm_obj *prm, union recv_frame *pframe)
+{
+	int i;
+	u8 val8 = 0, snr;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(prm->psta->padapter);
+	u8 rf_path = GET_HAL_RFPATH_NUM(dvobj);
+#if 0
+	if (IS_CCK_RATE((hw_rate_to_m_rate(pframe->u.hdr.attrib.data_rate))))
+		val8 = 255;
+	else {
+		snr = 0;
+		for (i = 0; i < rf_path; i++)
+			snr += pframe->u.hdr.attrib.phy_info.rx_snr[i];
+		snr = snr / rf_path;
+		val8 = (u8)(snr + 10)*2;
+	}
+#endif
+	return val8;
+}
+
+u8 rm_get_bcn_rsni(struct rm_obj *prm, struct wlan_network *pnetwork)
+{
+	int i;
+	u8 val8, snr;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(prm->psta->padapter);
+	u8 rf_path = GET_HAL_RFPATH_NUM(dvobj);
+
+
+	if (pnetwork->network.PhyInfo.is_cck_rate) {
+		/* current HW doesn't have CCK RSNI */
+		/* 255 indicates RSNI is unavailable */
+		val8 = 255;
+	} else {
+		snr = 0;
+		for (i = 0; i < rf_path; i++) {
+			snr += pnetwork->network.PhyInfo.rx_snr[i];
+		}
+		snr = snr / rf_path;
+		val8 = (u8)(snr + 10)*2;
+	}
+	return val8;
+}
+
+/* output: pwr (unit dBm) */
+int rm_get_tx_power(_adapter *adapter, enum band_type band, enum MGN_RATE rate, s8 *pwr)
+{
+	struct dvobj_priv *devob = adapter_to_dvobj(adapter);
+	u8 rs = mgn_rate_to_rs(rate);
+
+	*pwr = rtw_phl_get_power_by_rate_band(GET_PHL_INFO(devob), HW_BAND_0, _rate_mrate2phl(rate),
+					       IS_DCM_RATE_SECTION(rs) ? 1 : 0, 0, band);
+
+	return 0;
+}
+
+u8 rm_gen_dialog_token(_adapter *padapter)
+{
+	struct rm_priv *prmpriv = &(padapter->rmpriv);
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	do {
+		pmlmeinfo->dialogToken++;
+	} while (pmlmeinfo->dialogToken == 0);
+
+	return pmlmeinfo->dialogToken;
+}
+
+u8 rm_gen_meas_token(_adapter *padapter)
+{
+	struct rm_priv *prmpriv = &(padapter->rmpriv);
+
+	do {
+		prmpriv->meas_token++;
+	} while (prmpriv->meas_token == 0);
+
+	return prmpriv->meas_token;
+}
+
+u32 rm_gen_rmid(_adapter *padapter, struct rm_obj *prm, u8 role)
+{
+	u32 rmid;
+
+	if (prm->psta == NULL)
+		goto err;
+
+	if (prm->q.diag_token == 0)
+		goto err;
+
+	rmid = prm->psta->phl_sta->aid << 16
+		| prm->q.diag_token << 8
+		| role;
+
+	return rmid;
+err:
+	RTW_ERR("RM: unable to gen rmid\n");
+	return 0;
+}
+
+int rm_get_rx_sensitivity(_adapter *adapter, enum channel_width bw, enum MGN_RATE rate, s8 *pwr)
+{
+	s8 rx_sensitivity = -110;
+
+	switch(rate) {
+	case MGN_1M:
+		rx_sensitivity= -101;
+		break;
+	case MGN_2M:
+		rx_sensitivity= -98;
+		break;
+	case MGN_5_5M:
+		rx_sensitivity= -92;
+		break;
+	case MGN_11M:
+		rx_sensitivity= -89;
+		break;
+	case MGN_6M:
+	case MGN_9M:
+	case MGN_12M:
+		rx_sensitivity = -92;
+		break;
+	case MGN_18M:
+		rx_sensitivity = -90;
+		break;
+	case MGN_24M:
+		rx_sensitivity = -88;
+		break;
+	case MGN_36M:
+		rx_sensitivity = -84;
+		break;
+	case MGN_48M:
+		rx_sensitivity = -79;
+		break;
+	case MGN_54M:
+		rx_sensitivity = -78;
+		break;
+
+	case MGN_MCS0:
+	case MGN_MCS8:
+	case MGN_MCS16:
+	case MGN_MCS24:
+	case MGN_VHT1SS_MCS0:
+	case MGN_VHT2SS_MCS0:
+	case MGN_VHT3SS_MCS0:
+	case MGN_VHT4SS_MCS0:
+		/* BW20 BPSK 1/2 */
+		rx_sensitivity = -82;
+		break;
+
+	case MGN_MCS1:
+	case MGN_MCS9:
+	case MGN_MCS17:
+	case MGN_MCS25:
+	case MGN_VHT1SS_MCS1:
+	case MGN_VHT2SS_MCS1:
+	case MGN_VHT3SS_MCS1:
+	case MGN_VHT4SS_MCS1:
+		/* BW20 QPSK 1/2 */
+		rx_sensitivity = -79;
+		break;
+
+	case MGN_MCS2:
+	case MGN_MCS10:
+	case MGN_MCS18:
+	case MGN_MCS26:
+	case MGN_VHT1SS_MCS2:
+	case MGN_VHT2SS_MCS2:
+	case MGN_VHT3SS_MCS2:
+	case MGN_VHT4SS_MCS2:
+		/* BW20 QPSK 3/4 */
+		rx_sensitivity = -77;
+		break;
+
+	case MGN_MCS3:
+	case MGN_MCS11:
+	case MGN_MCS19:
+	case MGN_MCS27:
+	case MGN_VHT1SS_MCS3:
+	case MGN_VHT2SS_MCS3:
+	case MGN_VHT3SS_MCS3:
+	case MGN_VHT4SS_MCS3:
+		/* BW20 16-QAM 1/2 */
+		rx_sensitivity = -74;
+		break;
+
+	case MGN_MCS4:
+	case MGN_MCS12:
+	case MGN_MCS20:
+	case MGN_MCS28:
+	case MGN_VHT1SS_MCS4:
+	case MGN_VHT2SS_MCS4:
+	case MGN_VHT3SS_MCS4:
+	case MGN_VHT4SS_MCS4:
+		/* BW20 16-QAM 3/4 */
+		rx_sensitivity = -70;
+		break;
+
+	case MGN_MCS5:
+	case MGN_MCS13:
+	case MGN_MCS21:
+	case MGN_MCS29:
+	case MGN_VHT1SS_MCS5:
+	case MGN_VHT2SS_MCS5:
+	case MGN_VHT3SS_MCS5:
+	case MGN_VHT4SS_MCS5:
+		/* BW20 64-QAM 2/3 */
+		rx_sensitivity = -66;
+		break;
+
+	case MGN_MCS6:
+	case MGN_MCS14:
+	case MGN_MCS22:
+	case MGN_MCS30:
+	case MGN_VHT1SS_MCS6:
+	case MGN_VHT2SS_MCS6:
+	case MGN_VHT3SS_MCS6:
+	case MGN_VHT4SS_MCS6:
+		/* BW20 64-QAM 3/4 */
+		rx_sensitivity = -65;
+		break;
+
+	case MGN_MCS7:
+	case MGN_MCS15:
+	case MGN_MCS23:
+	case MGN_MCS31:
+	case MGN_VHT1SS_MCS7:
+	case MGN_VHT2SS_MCS7:
+	case MGN_VHT3SS_MCS7:
+	case MGN_VHT4SS_MCS7:
+		/* BW20 64-QAM 5/6 */
+		rx_sensitivity = -64;
+		break;
+
+	case MGN_VHT1SS_MCS8:
+	case MGN_VHT2SS_MCS8:
+	case MGN_VHT3SS_MCS8:
+	case MGN_VHT4SS_MCS8:
+		/* BW20 256-QAM 3/4 */
+		rx_sensitivity = -59;
+		break;
+
+	case MGN_VHT1SS_MCS9:
+	case MGN_VHT2SS_MCS9:
+	case MGN_VHT3SS_MCS9:
+	case MGN_VHT4SS_MCS9:
+		/* BW20 256-QAM 5/6 */
+		rx_sensitivity = -57;
+		break;
+
+	default:
+		return -1;
+		break;
+
+	}
+
+	switch(bw) {
+	case CHANNEL_WIDTH_20:
+		break;
+	case CHANNEL_WIDTH_40:
+		rx_sensitivity -= 3;
+		break;
+	case CHANNEL_WIDTH_80:
+		rx_sensitivity -= 6;
+		break;
+	case CHANNEL_WIDTH_160:
+		rx_sensitivity -= 9;
+		break;
+	case CHANNEL_WIDTH_5:
+	case CHANNEL_WIDTH_10:
+	case CHANNEL_WIDTH_80_80:
+	default:
+		return -1;
+		break;
+	}
+	*pwr = rx_sensitivity;
+
+	return 0;
+}
+
+/* output: path_a max tx power in dBm */
+int rm_get_path_a_max_tx_power(_adapter *adapter, s8 *path_a)
+{
+#if 0 /*GEORGIA_TODO_FIXIT*/
+
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct hal_spec_t *hal_spec = GET_HAL_SPEC(dvobj);
+	HAL_DATA_TYPE *hal_data = GET_PHL_COM(dvobj);
+	int path, tx_num, band, bw, ch, n, rs;
+	u8 rate_num;
+	s8 max_pwr[RF_PATH_MAX], pwr;
+
+
+	band = hal_data->current_band_type;
+	bw = hal_data->current_channel_bw;
+	ch = hal_data->current_channel;
+
+	for (path = 0; path < RF_PATH_MAX; path++) {
+		if (!HAL_SPEC_CHK_RF_PATH(hal_spec, band, path))
+			break;
+
+		max_pwr[path] = -127; /* min value of s8 */
+#if (RM_MORE_DBG_MSG)
+		RTW_INFO("RM: [%s][%c]\n", band_str(band), rf_path_char(path));
+#endif
+		for (rs = 0; rs < RATE_SECTION_NUM; rs++) {
+			tx_num = rate_section_to_tx_num(rs);
+
+			if (tx_num >= hal_spec->tx_nss_num)
+				continue;
+
+			if (band == BAND_ON_5G && IS_CCK_RATE_SECTION(rs))
+				continue;
+
+			if (IS_VHT_RATE_SECTION(rs) && !IS_HARDWARE_TYPE_JAGUAR_ALL(adapter))
+				continue;
+
+			rate_num = rate_section_rate_num(rs);
+
+			/* get power by rate in db */
+			for (n = rate_num - 1; n >= 0; n--) {
+				pwr = phy_get_tx_power_final_absolute_value(adapter, path, rates_by_sections[rs].rates[n], bw, ch);
+				max_pwr[path] = MAX(max_pwr[path], pwr);
+#if (RM_MORE_DBG_MSG)
+				RTW_INFO("RM: %9s = %2d\n",
+					MGN_RATE_STR(rates_by_sections[rs].rates[n]), pwr);
+#endif
+			}
+		}
+	}
+#if (RM_MORE_DBG_MSG)
+	RTW_INFO("RM: path_a max_pwr=%ddBm\n", max_pwr[0]);
+#endif
+	*path_a = max_pwr[0];
+#endif
+	return 0;
+}
+
+#endif /* CONFIG_RTW_80211K */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_scan.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_scan.c
new file mode 100644
index 000000000000..5058a65acfa4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_scan.c
@@ -0,0 +1,3114 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+
+/*rtw_mlme.c*/
+void rtw_init_sitesurvey_parm(_adapter *padapter, struct sitesurvey_parm *pparm)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+	_rtw_memset(pparm, 0, sizeof(struct sitesurvey_parm));
+	pparm->scan_mode = pmlmepriv->scan_mode;
+}
+
+#ifdef CONFIG_SET_SCAN_DENY_TIMER
+inline bool rtw_is_scan_deny(_adapter *adapter)
+{
+	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
+	return (ATOMIC_READ(&mlmepriv->set_scan_deny) != 0) ? _TRUE : _FALSE;
+}
+inline void rtw_clear_scan_deny(_adapter *adapter)
+{
+	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
+	ATOMIC_SET(&mlmepriv->set_scan_deny, 0);
+	if (0)
+		RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+}
+
+void rtw_set_scan_deny_timer_hdl(void *ctx)
+{
+	_adapter *adapter = (_adapter *)ctx;
+
+	rtw_clear_scan_deny(adapter);
+}
+void rtw_set_scan_deny(_adapter *adapter, u32 ms)
+{
+	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
+	if (0)
+		RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+	ATOMIC_SET(&mlmepriv->set_scan_deny, 1);
+	_set_timer(&mlmepriv->set_scan_deny_timer, ms);
+}
+#endif
+
+void rtw_drv_scan_by_self(_adapter *padapter, u8 reason)
+{
+	struct sitesurvey_parm *parm = NULL;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	int i;
+	u8 ssc_chk;
+
+	if (!rtw_is_adapter_up(padapter))
+		goto exit;
+
+	ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+	if (ssc_chk == SS_DENY_BUSY_TRAFFIC) {
+		#ifdef CONFIG_LAYER2_ROAMING
+		if (rtw_chk_roam_flags(padapter, RTW_ROAM_ACTIVE) && pmlmepriv->need_to_roam == _TRUE) {
+			RTW_INFO(FUNC_ADPT_FMT" need to roam, don't care BusyTraffic\n", FUNC_ADPT_ARG(padapter));
+		 } else
+		#endif
+		{
+			RTW_INFO(FUNC_ADPT_FMT" exit BusyTraffic\n", FUNC_ADPT_ARG(padapter));
+			goto exit;
+		}
+	} else if (ssc_chk != SS_ALLOW) {
+		goto exit;
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT" reason:0x%02x\n", FUNC_ADPT_ARG(padapter), reason);
+
+	parm = rtw_malloc(sizeof(*parm));
+	if (parm == NULL)
+		goto exit;
+
+	rtw_init_sitesurvey_parm(padapter, parm);
+	parm->reason = reason;
+
+	/* only for 20/40 BSS */
+	if (reason == RTW_AUTO_SCAN_REASON_2040_BSS) {
+		for (i=0;i<14;i++) {
+			parm->ch[i].band = BAND_ON_24G;
+			parm->ch[i].hw_value = i + 1;
+			parm->ch[i].flags = RTW_IEEE80211_CHAN_PASSIVE_SCAN;
+		}
+		parm->ch_num = 14;
+		rtw_sitesurvey_cmd(padapter, parm);
+		goto exit;
+	}
+
+#ifdef CONFIG_RTW_MBO
+#if defined(CONFIG_RTW_WNM) || defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	if ((reason == RTW_AUTO_SCAN_REASON_ROAM)
+		&& (rtw_roam_nb_scan_list_set(padapter, parm))) {
+		rtw_sitesurvey_cmd(padapter, parm);
+		goto exit;
+	}
+#endif
+#endif
+
+	rtw_sitesurvey_cmd(padapter, NULL);
+exit:
+	if (parm)
+		rtw_mfree(parm, sizeof(*parm));
+
+	return;
+}
+
+#ifdef CONFIG_RTW_ACS
+u8 rtw_set_acs_sitesurvey(_adapter *adapter)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(adapter);
+	void *phl = adapter_to_dvobj(adapter)->phl;
+	struct sitesurvey_parm parm;
+	u8 uch;
+	struct rtw_chan_def union_chdef = {0};
+	u8 ch_num = 0;
+	int i;
+	enum band_type band;
+	u8 (*center_chs_num)(u8) = NULL;
+	u8 (*center_chs)(u8, u8) = NULL;
+	u8 ret = _FAIL;
+
+	if (rtw_phl_mr_get_chandef(phl, adapter->phl_role, padapter_link->wrlink,
+		&union_chdef) != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("fun:%s line:%d get chandef failed\n", __FUNCTION__, __LINE__);
+		rtw_warn_on(1);
+		goto exit;
+	}
+	uch = union_chdef.chan;
+
+	_rtw_memset(&parm, 0, sizeof(struct sitesurvey_parm));
+	parm.scan_mode = RTW_PHL_SCAN_PASSIVE;
+	parm.bw = CHANNEL_WIDTH_20;
+	parm.acs = 1;
+
+	for (band = BAND_ON_24G; band < BAND_MAX; band++) {
+		if (band == BAND_ON_24G) {
+			center_chs_num = center_chs_2g_num;
+			center_chs = center_chs_2g;
+		} else
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		if (band == BAND_ON_5G) {
+			center_chs_num = center_chs_5g_num;
+			center_chs = center_chs_5g;
+		} else
+		#endif
+		{
+			center_chs_num = NULL;
+			center_chs = NULL;
+		}
+
+		if (!center_chs_num || !center_chs)
+			continue;
+
+		if (rfctl->ch_sel_within_same_band) {
+			if (rtw_is_2g_ch(uch) && band != BAND_ON_24G)
+				continue;
+			#if CONFIG_IEEE80211_BAND_5GHZ
+			if (rtw_is_5g_ch(uch) && band != BAND_ON_5G)
+				continue;
+			#endif
+		}
+
+		ch_num = center_chs_num(CHANNEL_WIDTH_20);
+		for (i = 0; i < ch_num && parm.ch_num < RTW_CHANNEL_SCAN_AMOUNT; i++) {
+			parm.ch[parm.ch_num].hw_value = center_chs(CHANNEL_WIDTH_20, i);
+			parm.ch[parm.ch_num].flags = RTW_IEEE80211_CHAN_PASSIVE_SCAN;
+			parm.ch_num++;
+		}
+	}
+
+	ret = rtw_sitesurvey_cmd(adapter, &parm);
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_RTW_ACS */
+
+static u32 _rtw_wait_scan_done(_adapter *adapter, u32 timeout_ms)
+{
+	systime start;
+	u32 pass_ms;
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct mlme_ext_priv *pmlmeext = &(adapter->mlmeextpriv);
+	u8 abort_timeout = false;
+
+	start = rtw_get_current_time();
+
+	while ((rtw_cfg80211_get_is_roch(adapter) == _TRUE || check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY))
+	       && rtw_get_passing_time_ms(start) <= timeout_ms) {
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+			break;
+
+		RTW_INFO(FUNC_NDEV_FMT"fw_state=WIFI_UNDER_SURVEY!\n", FUNC_NDEV_ARG(adapter->pnetdev));
+		rtw_msleep_os(20);
+		abort_timeout = true;
+	}
+
+	if (_TRUE == abort_timeout && check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY)) {
+		if (!RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+			RTW_ERR(FUNC_NDEV_FMT"waiting for scan_abort time out!\n",
+					FUNC_NDEV_ARG(adapter->pnetdev));
+		pmlmeext->scan_abort_to = _TRUE;
+		rtw_indicate_scan_done(adapter, _TRUE);
+	}
+
+	pmlmeext->scan_abort = _FALSE;
+	RTW_INFO(FUNC_ADPT_FMT "- %s....scan_abort:%d\n",
+		FUNC_ADPT_ARG(adapter), __func__, pmlmeext->scan_abort);
+	pass_ms = rtw_get_passing_time_ms(start);
+
+	RTW_INFO("%s scan timeout value:%d ms, total take:%d ms\n",
+				__func__, timeout_ms, pass_ms);
+	return pass_ms;
+}
+
+/*
+* timeout_ms > 0:rtw_scan_abort_timeout , = 0:rtw_scan_wait_completed
+*/
+u32 rtw_scan_abort(_adapter *adapter, u32 timeout_ms)
+{
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+	u32 pass_ms = 0;
+
+	if (rtw_cfg80211_get_is_roch(adapter) == _TRUE || check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY)) {
+		pmlmeext->scan_abort = _TRUE;
+		RTW_INFO(FUNC_ADPT_FMT "- %s....scan_abort:%d\n",
+			FUNC_ADPT_ARG(adapter), __func__, pmlmeext->scan_abort);
+		rtw_sctx_init(&pmlmeext->sitesurvey_res.sctx, timeout_ms);
+
+		_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+		if (pmlmeext->sitesurvey_res.scan_param)
+			psts = rtw_phl_cmd_scan_cancel(adapter_to_dvobj(adapter)->phl,
+					pmlmeext->sitesurvey_res.scan_param);
+		_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+		if (psts == RTW_PHL_STATUS_SUCCESS)
+			rtw_sctx_wait(&pmlmeext->sitesurvey_res.sctx, __func__);
+		pass_ms = _rtw_wait_scan_done(adapter, timeout_ms);
+	}
+	return pass_ms;
+}
+
+void rtw_scan_abort_no_wait(_adapter *adapter)
+{
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+
+	_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+	if (pmlmeext->sitesurvey_res.scan_param)
+		rtw_phl_cmd_scan_cancel(adapter_to_dvobj(adapter)->phl,
+			pmlmeext->sitesurvey_res.scan_param);
+	_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+}
+
+static inline bool _rtw_scan_abort_check(_adapter *adapter, const char *caller)
+{
+	struct	mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct submit_ctx *sctx = &pmlmeext->sitesurvey_res.sctx;
+
+	RTW_INFO(FUNC_ADPT_FMT "- %s....scan_abort:%d\n",
+			FUNC_ADPT_ARG(adapter), __func__, pmlmeext->scan_abort);
+
+	if (pmlmeext->scan_abort == _FALSE)
+		return _FALSE;
+
+	if (pmlmeext->scan_abort_to) {
+		RTW_ERR("%s scan abort timeout\n", caller);
+		rtw_warn_on(1);
+	}
+
+	pmlmeext->scan_abort = _FALSE;
+	pmlmeext->scan_abort_to = _FALSE;
+	if (sctx) {
+		RTW_INFO("%s scan abort .....(%d ms)\n", caller, rtw_get_passing_time_ms(sctx->submit_time));
+		rtw_sctx_done(&sctx);
+	}
+	return _TRUE;
+}
+static struct wlan_network *alloc_network(struct	mlme_priv *pmlmepriv) /* (_queue	*free_queue) */
+{
+	struct	wlan_network	*pnetwork;
+	pnetwork = _rtw_alloc_network(pmlmepriv);
+	return pnetwork;
+}
+
+static void update_current_network(_adapter *adapter, WLAN_BSSID_EX *pnetwork)
+{
+	struct	mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct	_ADAPTER_LINK *adapter_link = NULL;
+	struct	link_mlme_priv *lmlmepriv;
+
+	if ((check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)) {
+#ifdef CONFIG_80211BE_EHT
+		if (pnetwork->is_mld)
+			adapter_link = rtw_get_adapter_link_by_linkid(adapter, pnetwork->link_id);
+#endif
+		if(!adapter_link)
+			adapter_link = GET_PRIMARY_LINK(adapter);
+
+		lmlmepriv = &(adapter_link->mlmepriv);
+
+		if (is_same_network(&(lmlmepriv->cur_network.network), pnetwork)) {
+			rtw_update_network(&(lmlmepriv->cur_network.network), pnetwork, adapter, _TRUE);
+			rtw_update_protection(adapter, (lmlmepriv->cur_network.network.IEs) + _FIXED_IE_LENGTH_,
+			      	lmlmepriv->cur_network.network.IELength);
+		}
+	}
+}
+
+u32 rtw_parse_reduced_nb_rpt(_adapter *adapter, u8 *ies, u32 ies_len,
+		struct rtw_phl_rnb_rpt_element *rnb_ele)
+{
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(adapter_to_dvobj(adapter));
+	u32 rnb_rpt_ie_len;
+	u8 *p;
+
+	p = rtw_get_ie(ies, EID_REDUCED_NEIGHBOR_REPORT,
+			&rnb_rpt_ie_len, ies_len);
+	if (p == NULL || rnb_rpt_ie_len == 0)
+		return 0;
+
+	rtw_phl_parse_reduced_nb_rpt(phl_com, p + 2, rnb_rpt_ie_len, rnb_ele);
+	return rnb_rpt_ie_len;
+}
+
+#ifdef CONFIG_80211BE_EHT
+void update_mld_link_from_ml_link_info(_adapter *adapter, struct wlan_network *pnetwork,
+		struct rtw_phl_per_sta_profile_element *per_sta_profile)
+{
+	u8 *p;
+	u8 eid_eht_ml = EID_EXT_MULTI_LINK;
+	u32 update_ie_len = 0;
+	u32 val;
+
+	if (per_sta_profile == NULL)
+		return;
+
+	/* STA Control field */
+	if (per_sta_profile->complete_profile)
+		pnetwork->network.is_complete_profile = _TRUE;
+	if (per_sta_profile->mac_addr_present)
+		_rtw_memcpy(pnetwork->network.MacAddress, per_sta_profile->mac_addr, ETH_ALEN);
+	if (per_sta_profile->bcn_interval_present) {
+		pnetwork->network.Configuration.BeaconPeriod = per_sta_profile->bcn_interval;
+		val = cpu_to_le32(per_sta_profile->bcn_interval);
+		p = rtw_get_beacon_interval_from_ie(pnetwork->network.IEs);
+		_rtw_memcpy(p , (unsigned char *)&val, _BEACON_ITERVAL_);
+	}
+	if (per_sta_profile->dtim_info_present) {
+		/* ToDo CONFIG_RTW_MLD: update dtim in TIM element
+		** which may also be related to tid-to-link mapping */
+	}
+
+	/* ToDo CONFIG_RTW_MLD: STA Info field */
+
+	/* STA Profile field */
+	/* Replace capability */
+	p = rtw_get_capability_from_ie(pnetwork->network.IEs);
+	_rtw_memcpy(p, per_sta_profile->sta_profile, _CAPABILITY_);
+
+	update_ie_len = pnetwork->network.IELength - _FIXED_IE_LENGTH_;
+
+	/* Update IEs with inheritance */
+	rtw_update_ies_with_inheritance(pnetwork->network.IEs + _FIXED_IE_LENGTH_,
+				&update_ie_len, per_sta_profile->sta_profile + _CAPABILITY_,
+				per_sta_profile->sta_profile_len - _CAPABILITY_);
+
+	pnetwork->network.IELength = update_ie_len +  _FIXED_IE_LENGTH_;
+
+}
+
+struct rtw_phl_tbtt_info *rtw_get_next_mld_tbtt_info(_adapter *adapter,
+		struct rtw_phl_rnb_rpt_element *rnb_ele, u8 *start_idx)
+{
+	struct rtw_phl_neighbor_ap *nb_ap;
+	struct tbtt_info_header *hdr;
+	struct rtw_phl_tbtt_info *tbtt_info = NULL;
+	u8 j;
+
+	while (*start_idx < rnb_ele->nb_ap_num) {
+		nb_ap = &rnb_ele->nb_aps[*start_idx];
+		hdr = &nb_ap->tbtt_info_hdr;
+		if (!hdr->is_legal || !hdr->mld_param_is_present)
+			continue;
+		for (j = 0; j < hdr->cnt + 1; j++) {
+			tbtt_info = &nb_ap->tbtt_infos[j];
+			if (tbtt_info->mld_param.mld_id == 0)
+				return tbtt_info;
+		}
+		*start_idx++;
+	}
+	return NULL;
+}
+
+void update_mld_link_from_rnb_info(_adapter *adapter, struct wlan_network *pnetwork,
+		struct rtw_phl_neighbor_ap *nb_ap, struct rtw_phl_tbtt_info *tbtt_info)
+{
+	struct tbtt_info_header *hdr = &nb_ap->tbtt_info_hdr;
+	pnetwork->network.Configuration.DSConfig = nb_ap->ch;
+	/* ToDo CONFIG_RTW_MLD: offset, short_ssid, bss_param, max_tx_pwr, mld_param.bss_params_chg_cnt */
+	if (hdr->bssid_is_present)
+		_rtw_memcpy(&pnetwork->network.MacAddress, tbtt_info->bssid, ETH_ALEN);
+	if (hdr->mld_param_is_present)
+		pnetwork->network.link_id = tbtt_info->mld_param.link_id;
+}
+
+struct wlan_network *update_scanned_mld_network(_adapter *adapter, WLAN_BSSID_EX *target,
+		struct wlan_network *choice, struct wlan_network *found_network, int target_find)
+{
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	bool update_ie = _FALSE;
+	u8 network_num = 0;
+	struct wlan_mld_network *pmld_network = NULL;
+	struct wlan_network *pnetwork = NULL;
+	struct rtw_phl_ml_element *ml_ele = NULL;
+	struct basic_ml *basic_ml;
+	struct rtw_phl_rnb_rpt_element *rnb_ele = NULL;
+	struct rtw_phl_tbtt_info *tbtt_info;
+	u8 lidx = 0;
+	u8 i = 0;
+	u8 lnw_idx;
+
+	ml_ele = (struct rtw_phl_ml_element *)rtw_zmalloc(sizeof(struct rtw_phl_ml_element));
+	rnb_ele = (struct rtw_phl_rnb_rpt_element *)rtw_zmalloc(sizeof(struct rtw_phl_rnb_rpt_element));
+
+	if (!ml_ele || !rnb_ele)
+		goto _exit;
+
+	/* Reduced Neighbor Report element */
+	if (!rtw_parse_reduced_nb_rpt(adapter, target->IEs + _FIXED_IE_LENGTH_,
+			target->IELength - _FIXED_IE_LENGTH_, rnb_ele))
+		goto _exit;
+
+	/* Multilink element */
+	if (!rtw_parse_ml_ie(adapter, target->IEs + _FIXED_IE_LENGTH_,
+				target->IELength - _FIXED_IE_LENGTH_, ml_ele))
+		goto _exit;
+
+	basic_ml =  &ml_ele->common_info.basic_ml;
+
+	if (basic_ml->link_id_info_present)
+		target->link_id = basic_ml->link_id;
+
+	if (!target_find) {
+		if (_rtw_queue_empty(&(pmlmepriv->free_bss_pool)) == _TRUE) {
+			/* If there are no more slots, expire the choice */
+			/* list_del_init(&choice->list); */
+			pnetwork = choice;
+			if (pnetwork == NULL)
+				goto _exit;
+
+			pmld_network = pnetwork->network.mld_network;
+			_rtw_memcpy(pmld_network->mac_addr, basic_ml->mld_address, ETH_ALEN);
+			network_num = pmld_network->network_num;
+			do {
+				pnetwork = GET_LINK_NETWORK(pmld_network, lidx);
+
+				if (lidx == 0) {
+					pnetwork->network.is_complete_profile = _TRUE;
+					/* Replace the link network */
+					_rtw_memcpy(&(pnetwork->network), target,  get_WLAN_BSSID_EX_sz(target));
+				} else {
+					tbtt_info = rtw_get_next_mld_tbtt_info(adapter, rnb_ele, &i);
+					if (i >= rnb_ele->nb_ap_num || !tbtt_info)
+						break;
+
+					pnetwork->network.is_complete_profile = _FALSE;
+					/* Replace the link network */
+					_rtw_memcpy(&(pnetwork->network), target, get_WLAN_BSSID_EX_sz(target));
+					update_mld_link_from_rnb_info(adapter, pnetwork, &rnb_ele->nb_aps[i], tbtt_info);
+					i++;
+					for (lnw_idx = 0; lnw_idx < ml_ele->profile_num; lnw_idx++) {
+						if (pnetwork->network.link_id == ml_ele->profile[lnw_idx].link_id) {
+							update_mld_link_from_ml_link_info(adapter, pnetwork, &ml_ele->profile[lnw_idx]);
+							break;
+						}
+					}
+				}
+
+				pnetwork->bcn_keys_valid = 0;
+				/* pnetwork is complete profile if lidx==0 or ml_ele->profile is found */
+				if (pnetwork->network.is_complete_profile) {
+					if (target->Reserved[0] == BSS_TYPE_BCN || target->Reserved[0] == BSS_TYPE_PROB_RSP)
+						rtw_update_bcn_keys_of_network(pnetwork);
+				}
+				/* variable initialize */
+				pnetwork->fixed = _FALSE;
+				pnetwork->last_scanned = rtw_get_current_time();
+				pnetwork->last_non_hidden_ssid_ap = pnetwork->last_scanned;
+				#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+				pnetwork->acnode_stime = 0;
+				pnetwork->acnode_notify_etime = 0;
+				#endif
+
+				pnetwork->network_type = 0;
+				pnetwork->aid = 0;
+				pnetwork->join_res = 0;
+
+				/* bss info not receving from the right channel */
+				if (pnetwork->network.PhyInfo.SignalQuality == 101)
+					pnetwork->network.PhyInfo.SignalQuality = 0;
+
+				/* bss info receving the first time and invalid phyinfo */
+				if (!pnetwork->network.PhyInfo.isValid) {
+					pnetwork->network.PhyInfo.SignalStrength = 30;
+					pnetwork->network.PhyInfo.rssi = rtw_phl_rssi_to_dbm(pnetwork->network.PhyInfo.SignalStrength);
+				}
+
+				/* Link mld_network and network */
+				pnetwork->network.mld_network = pmld_network;
+
+				lidx++;
+
+			} while (i < rnb_ele->nb_ap_num && lidx < network_num);
+			/* Free the rest networks */
+			while (lidx < network_num) {
+				pnetwork = GET_LINK_NETWORK(pmld_network, lidx);
+				_rtw_free_network_nolock(pmlmepriv, pnetwork);
+				pmld_network->link_network[lidx] = NULL;
+				pmld_network->network_num--;
+				lidx++;
+			}
+		}
+		else {
+			/* MLD Mac Address */
+			pmld_network = rtw_alloc_mld_network(adapter, basic_ml->mld_address);
+			if (!pmld_network)
+				goto _exit;
+			target->Length = get_WLAN_BSSID_EX_sz(target);
+			do {
+				if (pmld_network->network_num >= RTW_NETWORK_LINK_MAX)
+					break;
+
+				pnetwork = alloc_network(pmlmepriv); /* will update scan_time */
+
+				if (!pnetwork)
+					goto _exit;
+
+				if (lidx == 0) {
+					pnetwork->network.is_complete_profile = _TRUE;
+					_rtw_memcpy(&(pnetwork->network), target, target->Length);
+				} else {
+					tbtt_info = rtw_get_next_mld_tbtt_info(adapter, rnb_ele, &i);
+					if (i >= rnb_ele->nb_ap_num || !tbtt_info)
+						break;
+
+					pnetwork->network.is_complete_profile = _FALSE;
+					_rtw_memcpy(&(pnetwork->network), target, target->Length);
+					update_mld_link_from_rnb_info(adapter, pnetwork, &rnb_ele->nb_aps[i], tbtt_info);
+					i++;
+					for (lnw_idx = 0; lnw_idx < ml_ele->profile_num; lnw_idx++) {
+						if (pnetwork->network.link_id == ml_ele->profile[lnw_idx].link_id) {
+							update_mld_link_from_ml_link_info(adapter, pnetwork, &ml_ele->profile[lnw_idx]);
+							break;
+						}
+					}
+				}
+
+				pnetwork->bcn_keys_valid = 0;
+				/* pnetwork is complete profile if lidx==0 or ml_ele->profile is found */
+				if (pnetwork->network.is_complete_profile) {
+					if (target->Reserved[0] == BSS_TYPE_BCN || target->Reserved[0] == BSS_TYPE_PROB_RSP)
+						rtw_update_bcn_keys_of_network(pnetwork);
+				}
+
+				pmld_network->link_network[pmld_network->network_num] = pnetwork;
+				pmld_network->network_num++;
+
+				pnetwork->last_scanned = rtw_get_current_time();
+
+				/* bss info not receving from the right channel */
+				if (pnetwork->network.PhyInfo.SignalQuality == 101)
+					pnetwork->network.PhyInfo.SignalQuality = 0;
+
+				/* bss info receving the first time and invalid phyinfo */
+				if (!pnetwork->network.PhyInfo.isValid) {
+					pnetwork->network.PhyInfo.SignalStrength = 30;
+					pnetwork->network.PhyInfo.rssi = rtw_phl_rssi_to_dbm(pnetwork->network.PhyInfo.SignalStrength);
+				}
+
+				rtw_list_insert_tail(&(pnetwork->list), &(pmlmepriv->scanned_queue.queue));
+
+				/* Link mld_network and network */
+				pnetwork->network.mld_network = pmld_network;
+
+				lidx++;
+
+			} while (i < rnb_ele->nb_ap_num);
+		}
+	}
+	else {
+		#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+		systime last_scanned;
+		#endif
+		struct beacon_keys bcn_keys;
+		bool bcn_keys_valid;
+		bool is_hidden_ssid_ap;
+		struct wlan_network *link_network;
+		WLAN_BSSID_EX *link_wlan = NULL;
+		pmld_network = found_network->network.mld_network;
+		link_network = (struct wlan_network *)rtw_zmalloc(sizeof(struct wlan_network));
+		if (!link_network)
+			goto _exit;
+
+		/* complete when the primary network has been updated and there is no more per-sta profile */
+		do {
+			update_ie = _FALSE;
+			_rtw_memset(link_network, 0, sizeof(struct wlan_network));
+			_rtw_memcpy(&link_network->network, &found_network->network, get_WLAN_BSSID_EX_sz(&found_network->network));
+			if (lidx == 0) {
+				pnetwork = found_network;
+				link_network->network.is_complete_profile = _TRUE;
+			} else {
+				tbtt_info = rtw_get_next_mld_tbtt_info(adapter, rnb_ele, &i);
+				if (i >= rnb_ele->nb_ap_num || !tbtt_info)
+					break;
+
+				pnetwork = rtw_get_link_network_by_linkid(pmld_network, tbtt_info->mld_param.link_id);
+				if (!pnetwork) {
+					i++;
+					lidx++;
+					continue;
+				}
+				link_network->network.is_complete_profile = _FALSE;
+				update_mld_link_from_rnb_info(adapter, link_network, &rnb_ele->nb_aps[i], tbtt_info);
+				i++;
+				for (lnw_idx = 0; lnw_idx < ml_ele->profile_num; lnw_idx++) {
+					if (link_network->network.link_id == ml_ele->profile[lnw_idx].link_id) {
+						update_mld_link_from_ml_link_info(adapter, link_network, &ml_ele->profile[lnw_idx]);
+						break;
+					}
+				}
+			}
+
+			link_wlan = &link_network->network;
+
+			#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+			last_scanned = pnetwork->last_scanned;
+			#endif
+
+			bcn_keys_valid = 0;
+			is_hidden_ssid_ap = 0;
+			pnetwork->last_scanned = rtw_get_current_time();
+
+			if (link_wlan->Reserved[0] == BSS_TYPE_BCN || link_wlan->Reserved[0] == BSS_TYPE_PROB_RSP) {
+				if (link_wlan->InfrastructureMode == Ndis802_11Infrastructure) {
+					is_hidden_ssid_ap = hidden_ssid_ap(link_wlan);
+					if (!is_hidden_ssid_ap) /* update last time it's non hidden ssid AP */
+						pnetwork->last_non_hidden_ssid_ap = rtw_get_current_time();
+				}
+				/* pnetwork is complete profile if lidx==0 or ml_ele->profile is found */
+				if (link_wlan->is_complete_profile)
+					bcn_keys_valid = rtw_get_bcn_keys_from_bss(link_wlan, &bcn_keys);
+			}
+
+			if (link_wlan->is_complete_profile) {
+				if (link_wlan->InfrastructureMode == Ndis802_11_mesh
+					|| link_wlan->Reserved[0] >= pnetwork->network.Reserved[0])
+					update_ie = _TRUE;
+				else if (link_wlan->InfrastructureMode == Ndis802_11Infrastructure && !pnetwork->fixed
+					&& rtw_get_passing_time_ms(pnetwork->last_non_hidden_ssid_ap) > SCANQUEUE_LIFETIME)
+					update_ie = _TRUE;
+				else if (bcn_keys_valid) {
+					if (is_hidden_ssid(bcn_keys.ssid, bcn_keys.ssid_len)) {
+						/* hidden ssid, replace with current beacon ssid directly */
+						_rtw_memcpy(bcn_keys.ssid, pnetwork->bcn_keys.ssid, pnetwork->bcn_keys.ssid_len);
+						bcn_keys.ssid_len = pnetwork->bcn_keys.ssid_len;
+					}
+					if (rtw_bcn_key_compare(&pnetwork->bcn_keys, &bcn_keys) == _FALSE)
+						update_ie = _TRUE;
+				}
+			}
+
+			#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+			if (!MLME_IS_MESH(adapter) || !MLME_IS_ASOC(adapter)
+				|| pnetwork->network.Configuration.DSConfig != link_wlan->Configuration.DSConfig
+				|| rtw_get_passing_time_ms(last_scanned) > adapter->mesh_cfg.peer_sel_policy.scanr_exp_ms
+				|| !rtw_bss_is_same_mbss(&pnetwork->network, link_wlan)
+			) {
+				pnetwork->acnode_stime = 0;
+				pnetwork->acnode_notify_etime = 0;
+			}
+			#endif
+
+			if (bcn_keys_valid) {
+				_rtw_memcpy(&pnetwork->bcn_keys, &bcn_keys, sizeof(bcn_keys));
+				pnetwork->bcn_keys_valid = 1;
+			} else if (update_ie)
+				pnetwork->bcn_keys_valid = 0;
+
+			rtw_update_network(&(pnetwork->network), link_wlan, adapter, update_ie);
+
+			/* Link mld_network and network */
+			pnetwork->network.mld_network = pmld_network;
+
+			lidx++;
+
+		} while (i < rnb_ele->nb_ap_num);
+
+		if (link_network)
+			rtw_mfree((unsigned char *)link_network, sizeof(struct wlan_network));
+	}
+
+	pmld_network->last_scanned = rtw_get_current_time();
+
+	for (i = 0; i < pmld_network->network_num; i++) {
+		#ifdef CONFIG_80211D
+		rtw_update_scanned_network_cisr(adapter_to_rfctl(adapter), pmld_network->link_network[i]);
+		#endif
+
+		#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+		if (MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter))
+			rtw_mesh_update_scanned_acnode_status(adapter, pmld_network->link_network[i]);
+		#endif
+	}
+
+_exit:
+	if (ml_ele)
+		rtw_mfree((unsigned char *)ml_ele, sizeof(struct rtw_phl_ml_element));
+	if (rnb_ele)
+		rtw_mfree((unsigned char *)rnb_ele, sizeof(struct rtw_phl_rnb_rpt_element));
+
+	if (pmld_network)
+		pnetwork = GET_LINK_NETWORK(pmld_network, RTW_RLINK_PRIMARY);
+	else
+		pnetwork = NULL;
+	return pnetwork;
+}
+#endif
+
+/*Caller must hold pmlmepriv->lock first.*/
+static bool update_scanned_network(_adapter *adapter, WLAN_BSSID_EX *target,
+	struct wlan_network **ppnetwork)
+{
+	_list	*plist, *phead;
+	u32	bssid_ex_sz;
+	struct mlme_priv	*pmlmepriv = &(adapter->mlmepriv);
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(adapter->wdinfo);
+#endif /* CONFIG_P2P */
+	_queue	*queue	= &(pmlmepriv->scanned_queue);
+	struct wlan_network	*pnetwork = NULL;
+	struct wlan_network	*choice = NULL;
+	int target_find = 0;
+	bool update_ie = _FALSE;
+	_queue *mld_queue = &(pmlmepriv->scanned_mld_queue);
+
+	_rtw_spinlock_bh(&queue->lock);
+	_rtw_spinlock_bh(&mld_queue->lock);
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+#if 0
+	RTW_INFO("%s => ssid:%s , rssi:%ld , ss:%d\n",
+		__func__, target->Ssid.Ssid, target->PhyInfo.rssi, target->PhyInfo.SignalStrength);
+#endif
+
+	while (1) {
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+#ifdef CONFIG_P2P
+		if (_rtw_memcmp(pnetwork->network.MacAddress, target->MacAddress, ETH_ALEN) &&
+		    _rtw_memcmp(pnetwork->network.Ssid.Ssid, "DIRECT-", 7) &&
+		    rtw_get_p2p_ie(pnetwork->network.IEs + _FIXED_IE_LENGTH_,
+				   pnetwork->network.IELength - _FIXED_IE_LENGTH_,
+				   NULL, NULL)) {
+			target_find = 1;
+			break;
+		}
+#endif
+
+		if (is_same_network(&(pnetwork->network), target)) {
+			target_find = 1;
+			break;
+		}
+
+		if (rtw_roam_flags(adapter)) {
+			/* TODO: don't  select netowrk in the same ess as choice if it's new enough*/
+		}
+		if (pnetwork->fixed) {
+			plist = get_next(plist);
+			continue;
+		}
+
+		/* ToDo CONFIG_RTW_MLD:
+		** if the MLD network in the scanned_mld_queue has fewer links than that of target MLD,
+		** may need to get a set of choice to free */
+#ifdef CONFIG_RSSI_PRIORITY
+		if ((choice == NULL) || (pnetwork->network.PhyInfo.SignalStrength < choice->network.PhyInfo.SignalStrength)) {
+			#ifdef CONFIG_RTW_MESH
+			if (!MLME_IS_MESH(adapter) || !MLME_IS_ASOC(adapter)
+				|| !rtw_bss_is_same_mbss(&pmlmepriv->dev_cur_network.network, &pnetwork->network))
+			#endif
+			if (target->is_mld == pnetwork->network.is_mld)
+				choice = pnetwork;
+		}
+#else
+		if (choice == NULL || rtw_time_after(choice->last_scanned, pnetwork->last_scanned)) {
+			#ifdef CONFIG_RTW_MESH
+			if (!MLME_IS_MESH(adapter) || !MLME_IS_ASOC(adapter)
+				|| !rtw_bss_is_same_mbss(&pmlmepriv->dev_cur_network.network, &pnetwork->network))
+			#endif
+			if (target->is_mld == pnetwork->network.is_mld)
+				choice = pnetwork;
+		}
+#endif
+		plist = get_next(plist);
+
+	}
+
+
+	/* If we didn't find a match, then get a new network slot to initialize
+	 * with this beacon's information */
+	/* if (rtw_end_of_queue_search(phead,plist)== _TRUE) { */
+#ifdef CONFIG_80211BE_EHT
+	if (target->is_mld) {
+		pnetwork = update_scanned_mld_network(adapter, target, choice, pnetwork, target_find);
+	} else /* !target->is_mld == legacy path*/
+#endif /* CONFIG_80211BE_EHT */
+	{
+		if (!target_find) {
+			if (_rtw_queue_empty(&(pmlmepriv->free_bss_pool)) == _TRUE) {
+				/* If there are no more slots, expire the choice */
+				/* list_del_init(&choice->list); */
+				pnetwork = choice;
+				if (pnetwork == NULL)
+					goto unlock_scan_queue;
+
+#ifdef CONFIG_RSSI_PRIORITY
+				RTW_DBG("%s => ssid:%s ,bssid:"MAC_FMT"  will be deleted from scanned_queue (rssi:%d , ss:%d)\n",
+				__func__, pnetwork->network.Ssid.Ssid, MAC_ARG(pnetwork->network.MacAddress),
+				pnetwork->network.PhyInfo.rssi, pnetwork->network.PhyInfo.SignalStrength);
+#else
+				RTW_DBG("%s => ssid:%s ,bssid:"MAC_FMT" will be deleted from scanned_queue\n",
+				__func__, pnetwork->network.Ssid.Ssid, MAC_ARG(pnetwork->network.MacAddress));
+#endif
+
+				_rtw_memcpy(&(pnetwork->network), target,  get_WLAN_BSSID_EX_sz(target));
+				pnetwork->bcn_keys_valid = 0;
+				if (target->Reserved[0] == BSS_TYPE_BCN || target->Reserved[0] == BSS_TYPE_PROB_RSP)
+					rtw_update_bcn_keys_of_network(pnetwork);
+				/* variable initialize */
+				pnetwork->fixed = _FALSE;
+				pnetwork->last_scanned = rtw_get_current_time();
+				pnetwork->last_non_hidden_ssid_ap = pnetwork->last_scanned;
+				#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+				pnetwork->acnode_stime = 0;
+				pnetwork->acnode_notify_etime = 0;
+				#endif
+
+				pnetwork->network_type = 0;
+				pnetwork->aid = 0;
+				pnetwork->join_res = 0;
+
+				/* bss info not receving from the right channel */
+				if (pnetwork->network.PhyInfo.SignalQuality == 101)
+					pnetwork->network.PhyInfo.SignalQuality = 0;
+
+			} else {
+				/* Otherwise just pull from the free list */
+
+				pnetwork = alloc_network(pmlmepriv); /* will update scan_time */
+				if (pnetwork == NULL)
+					goto unlock_scan_queue;
+
+				bssid_ex_sz = get_WLAN_BSSID_EX_sz(target);
+				target->Length = bssid_ex_sz;
+
+				_rtw_memcpy(&(pnetwork->network), target, bssid_ex_sz);
+				pnetwork->bcn_keys_valid = 0;
+				if (target->Reserved[0] == BSS_TYPE_BCN || target->Reserved[0] == BSS_TYPE_PROB_RSP)
+					rtw_update_bcn_keys_of_network(pnetwork);
+
+				/* bss info not receving from the right channel */
+				if (pnetwork->network.PhyInfo.SignalQuality == 101)
+					pnetwork->network.PhyInfo.SignalQuality = 0;
+
+				rtw_list_insert_tail(&(pnetwork->list), &(queue->queue));
+			}
+		}
+		else {
+			/* we have an entry and we are going to update it. But this entry may
+			 * be already expired. In this case we do the same as we found a new
+		 	* net and call the new_net handler
+		 	*/
+			#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+			systime last_scanned = pnetwork->last_scanned;
+			#endif
+			struct beacon_keys bcn_keys;
+			bool bcn_keys_valid = 0;
+			bool is_hidden_ssid_ap = 0;
+
+			pnetwork->last_scanned = rtw_get_current_time();
+
+			if (target->Reserved[0] == BSS_TYPE_BCN || target->Reserved[0] == BSS_TYPE_PROB_RSP) {
+				if (target->InfrastructureMode == Ndis802_11Infrastructure) {
+					is_hidden_ssid_ap = hidden_ssid_ap(target);
+					if (!is_hidden_ssid_ap) /* update last time it's non hidden ssid AP */
+						pnetwork->last_non_hidden_ssid_ap = rtw_get_current_time();
+				}
+				bcn_keys_valid = rtw_get_bcn_keys_from_bss(target, &bcn_keys);
+			}
+
+			if (target->InfrastructureMode == Ndis802_11_mesh
+				|| target->Reserved[0] >= pnetwork->network.Reserved[0])
+				update_ie = _TRUE;
+			else if (target->InfrastructureMode == Ndis802_11Infrastructure && !pnetwork->fixed
+				&& rtw_get_passing_time_ms(pnetwork->last_non_hidden_ssid_ap) > SCANQUEUE_LIFETIME)
+				update_ie = _TRUE;
+			else if (bcn_keys_valid) {
+				if (is_hidden_ssid(bcn_keys.ssid, bcn_keys.ssid_len)) {
+					/* hidden ssid, replace with current beacon ssid directly */
+					_rtw_memcpy(bcn_keys.ssid, pnetwork->bcn_keys.ssid, pnetwork->bcn_keys.ssid_len);
+					bcn_keys.ssid_len = pnetwork->bcn_keys.ssid_len;
+				}
+				if (rtw_bcn_key_compare(&pnetwork->bcn_keys, &bcn_keys) == _FALSE)
+					update_ie = _TRUE;
+			}
+
+			#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+			if (!MLME_IS_MESH(adapter) || !MLME_IS_ASOC(adapter)
+				|| pnetwork->network.Configuration.DSConfig != target->Configuration.DSConfig
+				|| rtw_get_passing_time_ms(last_scanned) > adapter->mesh_cfg.peer_sel_policy.scanr_exp_ms
+				|| !rtw_bss_is_same_mbss(&pnetwork->network, target)
+			) {
+				pnetwork->acnode_stime = 0;
+				pnetwork->acnode_notify_etime = 0;
+			}
+			#endif
+
+			if (bcn_keys_valid) {
+				_rtw_memcpy(&pnetwork->bcn_keys, &bcn_keys, sizeof(bcn_keys));
+				pnetwork->bcn_keys_valid = 1;
+			} else if (update_ie)
+				pnetwork->bcn_keys_valid = 0;
+
+			rtw_update_network(&(pnetwork->network), target, adapter, update_ie);
+		}
+
+		#ifdef CONFIG_80211D
+		rtw_update_scanned_network_cisr(adapter_to_rfctl(adapter), pnetwork);
+		#endif
+
+		#if defined(CONFIG_RTW_MESH) && CONFIG_RTW_MESH_ACNODE_PREVENT
+		if (MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter))
+			rtw_mesh_update_scanned_acnode_status(adapter, pnetwork);
+		#endif
+	}
+
+unlock_scan_queue:
+	_rtw_spinunlock_bh(&mld_queue->lock);
+	_rtw_spinunlock_bh(&queue->lock);
+
+#ifdef CONFIG_RTW_MESH
+	if (pnetwork && MLME_IS_MESH(adapter)
+		&& check_fwstate(pmlmepriv, WIFI_ASOC_STATE)
+		&& !check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY)
+	)
+		rtw_chk_candidate_peer_notify(adapter, pnetwork);
+#endif
+	*ppnetwork = pnetwork;
+
+	return update_ie;
+}
+
+static struct wlan_network *add_network(_adapter *adapter, WLAN_BSSID_EX *bss)
+{
+	struct wlan_network *pnetwork = NULL;
+	bool update_ie;
+	/* _queue *queue = &(pmlmepriv->scanned_queue); */
+	/* _rtw_spinlock_bh(&queue->lock); */
+
+#if defined(CONFIG_P2P) && defined(CONFIG_P2P_REMOVE_GROUP_INFO)
+	if (adapter->registrypriv.wifi_spec == 0)
+		rtw_bss_ex_del_p2p_attr(bss, P2P_ATTR_GROUP_INFO);
+#endif
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+	if (adapter->registrypriv.ignore_go_in_scan) {
+		if(rtw_chk_p2p_wildcard_ssid(bss) == _SUCCESS ||
+			rtw_chk_p2p_ie(bss) == _SUCCESS)
+			return NULL;
+	}
+	/*100 was follow n & ac IC setting SignalStrength rang was 0~100*/
+	if(adapter->registrypriv->ignore_low_rssi_in_scan != 0xff &&
+		bss->PhyInfo.rssi < (adapter->registrypriv->ignore_low_rssi_in_scan - 100))
+		return NULL;
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		rtw_bss_ex_del_wfd_ie(bss);
+
+	/* Wi-Fi driver will update the current network if the scan result of the connected AP be updated by scan. */
+	update_ie = update_scanned_network(adapter, bss, &pnetwork);
+
+	if (update_ie)
+		update_current_network(adapter, bss);
+
+	/* _rtw_spinunlock_bh(&queue->lock); */
+	return pnetwork;
+
+}
+
+void dump_scanned_queue(void *sel, _adapter *adapter)
+{
+#ifdef CONFIG_80211D
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define SURVEY_INFO_TITLE_FMT_80211D_6G "  "
+#define SURVEY_INFO_TITLE_ARG_80211D_6G
+#define SURVEY_INFO_VALUE_FMT_80211D_6G " %c %c"
+#define SURVEY_INFO_VALUE_ARG_80211D_6G	, show_cisr ? rtw_env_char(pnetwork->cisr.env) : ' ' \
+	, (show_cisr && pnetwork->cisr.reg_info < CIS_6G_REG_RSVD) ? '0' + pnetwork->cisr.reg_info : '-'
+#else
+#define SURVEY_INFO_TITLE_FMT_80211D_6G ""
+#define SURVEY_INFO_TITLE_ARG_80211D_6G
+#define SURVEY_INFO_VALUE_FMT_80211D_6G "  "
+#define SURVEY_INFO_VALUE_ARG_80211D_6G
+#endif
+
+#define SURVEY_INFO_TITLE_FMT_80211D " %-7s"SURVEY_INFO_TITLE_FMT_80211D_6G
+#define SURVEY_INFO_TITLE_ARG_80211D , "rg_info"SURVEY_INFO_TITLE_ARG_80211D_6G
+#define SURVEY_INFO_VALUE_FMT_80211D " "ALPHA2_FMT""SURVEY_INFO_VALUE_FMT_80211D_6G" %u%c"
+#define SURVEY_INFO_VALUE_ARG_80211D \
+	, show_cisr && is_alpha(pnetwork->cisr.alpha2[0]) ? pnetwork->cisr.alpha2[0] : '-' \
+	, show_cisr && is_alpha(pnetwork->cisr.alpha2[1]) ? pnetwork->cisr.alpha2[1] : '-' \
+	SURVEY_INFO_VALUE_ARG_80211D_6G \
+	, show_cisr ? pnetwork->cisr.status : COUNTRY_IE_SLAVE_NOCOUNTRY \
+	, rtw_network_chk_regu_ies(rfctl, pnetwork) ? ' ' : 'x'
+#else
+#define SURVEY_INFO_TITLE_FMT_80211D ""
+#define SURVEY_INFO_VALUE_FMT_80211D ""
+#define SURVEY_INFO_TITLE_ARG_80211D
+#define SURVEY_INFO_VALUE_ARG_80211D
+#endif /* CONFIG_80211D */
+
+#ifdef CONFIG_80211D
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	bool show_cisr = rfctl->collect_network_cisr;
+#endif
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	_queue *queue = &mlme->scanned_queue;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_priv *lmlme = &(alink->mlmepriv);
+	struct wlan_network *pnetwork = NULL;
+	_list *plist, *phead;
+	s32 notify_signal;
+	s16 notify_noise = 0;
+	u16 index = 0, ie_cap = 0;
+	unsigned char *ie_wpa = NULL, *ie_wpa2 = NULL, *ie_wps = NULL;
+	unsigned char *ie_wpa3 = NULL;
+	unsigned char *ie_p2p = NULL;
+	char *flag_str = NULL;
+	int ielen = 0;
+	u32 wpsielen = 0;
+	u32 akm;
+#ifdef CONFIG_RTW_MESH
+	const char *ssid_title_str = "ssid/mesh_id";
+#else
+	const char *ssid_title_str = "ssid";
+#endif
+
+	flag_str = rtw_zmalloc(64);
+	if (!flag_str)
+		return;
+
+	RTW_PRINT_SEL(sel, "%-5s %-17s %-4s %-3s %-4s %-4s %-5s"
+		SURVEY_INFO_TITLE_FMT_80211D
+		" %-5s %-32s %s\n"
+		, "index", "bssid", "band", "ch", "RSSI", "SdBm", "Noise"
+		SURVEY_INFO_TITLE_ARG_80211D
+		, "age", "flag", ssid_title_str);
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	while (!rtw_end_of_queue_search(phead, plist)) {
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+		plist = get_next(plist);
+
+		if (check_fwstate(mlme, WIFI_ASOC_STATE) == _TRUE &&
+		    is_same_network(&lmlme->cur_network.network, &pnetwork->network)) {
+			notify_signal = rtw_phl_rssi_to_dbm(adapter->recvinfo.signal_strength);/* dbm */
+		} else {
+			notify_signal = rtw_phl_rssi_to_dbm(pnetwork->network.PhyInfo.SignalStrength);/* dbm */
+		}
+
+#if 0 /*def CONFIG_BACKGROUND_NOISE_MONITOR*/
+		if (IS_NM_ENABLE(adapter))
+			notify_noise = rtw_noise_query_by_chan_num(adapter, pnetwork->network.Configuration.DSConfig);
+#endif
+
+		ie_wpa = rtw_get_wpa_ie(&pnetwork->network.IEs[12], &ielen, pnetwork->network.IELength - 12);
+		ie_wpa2 = rtw_get_wpa2_ie(&pnetwork->network.IEs[12], &ielen, pnetwork->network.IELength - 12);
+		ie_wpa3 = NULL;
+		if (ie_wpa2 &&
+		    rtw_parse_wpa2_ie(ie_wpa2, ielen+2, NULL, NULL, NULL, &akm,
+				      NULL, NULL) == _SUCCESS) {
+			if (akm & WLAN_AKM_TYPE_SAE) {
+				ie_wpa3 = ie_wpa2;
+				if (akm == WLAN_AKM_TYPE_SAE)
+					ie_wpa2 = NULL;
+			}
+		}
+		ie_cap = rtw_get_capability(&pnetwork->network);
+		ie_wps = rtw_get_wps_ie(&pnetwork->network.IEs[12], pnetwork->network.IELength - 12, NULL, &wpsielen);
+		ie_p2p = rtw_get_p2p_ie(&pnetwork->network.IEs[12], pnetwork->network.IELength - 12, NULL, &ielen);
+		sprintf(flag_str, "%s%s%s%s%s%s%s%s%s",
+			(ie_wpa) ? "[WPA]" : "",
+			(ie_wpa2) ? "[WPA2]" : "",
+			(ie_wpa3) ? "[WPA3]" : "",
+			(!ie_wpa && !ie_wpa && ie_cap & BIT(4)) ? "[WEP]" : "",
+			(ie_wps) ? "[WPS]" : "",
+			(pnetwork->network.InfrastructureMode == Ndis802_11IBSS) ? "[IBSS]" :
+				(pnetwork->network.InfrastructureMode == Ndis802_11_mesh) ? "[MESH]" : "",
+			(ie_cap & BIT(0)) ? "[ESS]" : "",
+			(pnetwork->network.Reserved[0] == BSS_TYPE_BCN) ? "[B]" : \
+			(pnetwork->network.Reserved[0] == BSS_TYPE_PROB_RSP) ? "[P]" : "[U]",
+			(ie_p2p) ? "[P2P]" : "");
+
+		RTW_PRINT_SEL(sel, "%5d "MAC_FMT" %4s %3d %4d %4d %5d"
+			SURVEY_INFO_VALUE_FMT_80211D
+			" %5d %-32s %s\n"
+			, ++index
+			, MAC_ARG(pnetwork->network.MacAddress)
+			, band_str(BSS_EX_OP_BAND(&pnetwork->network))
+			, BSS_EX_OP_CH(&pnetwork->network)
+			, pnetwork->network.PhyInfo.rssi
+			, notify_signal
+			, notify_noise
+			SURVEY_INFO_VALUE_ARG_80211D
+			, rtw_get_passing_time_ms(pnetwork->last_scanned)
+			, flag_str
+			, pnetwork->network.InfrastructureMode == Ndis802_11_mesh ? pnetwork->network.mesh_id.Ssid : pnetwork->network.Ssid.Ssid
+		);
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+
+	rtw_mfree(flag_str, 64);
+}
+
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+static inline void rtw_gen_new_bssid(const u8 *bssid, u8 max_bssid_ind,
+					  u8 mbssid_index, u8 *new_bssid)
+{
+	u8 i = 0;
+	u8 max_num = 1;
+	u8 B;
+	u8 new_a5;
+
+	for (i = 0; i < max_bssid_ind; i++)
+		max_num = max_num * 2;
+	/*RTW_INFO("%s, max_num=%d\n", __func__, max_num);*/
+	/*RTW_INFO("%s, %02x,%02x,%02x,%02x,%02x,%02x \n", __func__, bssid[0], bssid[1], bssid[2], bssid[3], bssid[4], bssid[5]);*/
+
+	B = bssid[5] % max_num;
+
+	new_a5 = bssid[5] - B + ((B + mbssid_index) % max_num);
+
+	new_bssid[0] = bssid[0];
+	new_bssid[1] = bssid[1];
+	new_bssid[2] = bssid[2];
+	new_bssid[3] = bssid[3];
+	new_bssid[4] = bssid[4];
+	new_bssid[5] = new_a5;
+
+	/*RTW_INFO("%s, %02x,%02x,%02x,%02x,%02x,%02x \n", __func__, new_bssid[0], new_bssid[1], new_bssid[2], new_bssid[3], new_bssid[4], new_bssid[5]);*/
+}
+
+void add_mbssid_network(_adapter *padapter, WLAN_BSSID_EX *ref_bss)
+{
+	WLAN_BSSID_EX *pbss;
+	u32 sub_ies_len;
+	u8 *mbssid_ie_ptr = NULL;
+	PNDIS_802_11_VARIABLE_IEs pIE, sub_pie;
+	u8 max_bssid_indicator;
+	int i,j;
+	u8* mbssid_ie;
+	sint mbssid_len;
+	u8 mbssid_index;
+	u8 copy_ie_offset;
+	u32 copy_ie_len = 0;
+
+	mbssid_ie = rtw_get_ie(ref_bss->IEs + _BEACON_IE_OFFSET_
+		                              , WLAN_EID_MULTIPLE_BSSID
+		                              , &mbssid_len
+		                              , (ref_bss->IELength- _BEACON_IE_OFFSET_));
+	if (!mbssid_ie)
+		return;
+#if 0
+	else
+		RTW_PRINT_DUMP("mbssid_ie: ", (const u8 *)mbssid_ie, mbssid_len);
+#endif
+
+	mbssid_ie_ptr = mbssid_ie;
+	max_bssid_indicator = GET_MBSSID_MAX_BSSID_INDOCATOR(mbssid_ie_ptr);
+	/*RTW_INFO("%s, max_bssid_indicator=%d\n", __func__, max_bssid_indicator);*/
+	mbssid_ie_ptr = mbssid_ie_ptr + MBSSID_MAX_BSSID_INDICATOR_OFFSET;
+
+	for (i = 0; i + 1 < mbssid_len;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(mbssid_ie_ptr + i);
+
+		switch (pIE->ElementID) {
+		case MBSSID_NONTRANSMITTED_BSSID_PROFILE_ID:
+			sub_ies_len = pIE->Length;
+			pbss = (WLAN_BSSID_EX *)rtw_zmalloc(sizeof(WLAN_BSSID_EX));
+			if (pbss) {
+				_rtw_memcpy(pbss, ref_bss, sizeof(WLAN_BSSID_EX));
+				_rtw_memset(pbss->IEs, 0, MAX_IE_SZ);
+				copy_ie_len =  _TIMESTAMP_ + _BEACON_ITERVAL_;
+				_rtw_memcpy(pbss->IEs, ref_bss->IEs, copy_ie_len);
+			} else {
+				return;
+			}
+
+			for (j = 0; j + 1 < sub_ies_len;) {
+				sub_pie = (PNDIS_802_11_VARIABLE_IEs)(pIE->data + j);
+				switch (sub_pie->ElementID) {
+				case WLAN_EID_NON_TX_BSSID_CAP:
+					/*RTW_INFO("%s, sub_pie->Length=%d\n", __func__, sub_pie->Length);*/
+					/*RTW_PRINT_DUMP("WLAN_EID_NON_TX_BSSID_CAP: ", (const u8 *)sub_pie->data, sub_pie->Length);*/
+					copy_ie_offset =  _TIMESTAMP_ + _BEACON_ITERVAL_;
+					_rtw_memcpy(pbss->IEs + copy_ie_offset, sub_pie->data, sub_pie->Length);
+					break;
+				case WLAN_EID_SSID:
+					/*RTW_PRINT_DUMP("WLAN_EID_SSID: ", (const u8 *)sub_pie->data, sub_pie->Length);*/
+					/*RTW_INFO("%s, ref_bss->IELength=%d\n", __func__, ref_bss->IELength);*/
+					/*RTW_PRINT_DUMP("A ref_bss->IEs: ", (const u8 *)ref_bss->IEs, ref_bss->IELength);*/
+					copy_ie_offset =  _TIMESTAMP_ + _BEACON_ITERVAL_ + _CAPABILITY_;
+					copy_ie_len =  WLAN_IE_ID_LEN + WLAN_IE_LEN_LEN;
+					_rtw_memcpy(pbss->IEs + copy_ie_offset, sub_pie, copy_ie_len);
+
+					copy_ie_offset = copy_ie_offset + WLAN_IE_ID_LEN + WLAN_IE_LEN_LEN;
+					_rtw_memcpy(pbss->IEs + copy_ie_offset, sub_pie->data, sub_pie->Length);
+					_rtw_memcpy(pbss->IEs + copy_ie_offset + sub_pie->Length
+						                , ref_bss->IEs + copy_ie_offset + ref_bss->Ssid.SsidLength
+						                , ref_bss->IELength - (copy_ie_offset + ref_bss->Ssid.SsidLength));
+
+					pbss->IELength = ref_bss->IELength + (sub_pie->Length - ref_bss->Ssid.SsidLength);
+					/*RTW_INFO("%s, ref_bss->Ssid.SsidLength=%d\n", __func__, ref_bss->Ssid.SsidLength);*/
+					/*RTW_INFO("%s, sub_pie->Length=%d\n", __func__, sub_pie->Length);*/
+					/*RTW_INFO("%s, pbss->IELength=%d\n", __func__, pbss->IELength);*/
+					/*RTW_PRINT_DUMP("B pbss->IEs: ", (const u8 *)pbss->IEs, pbss->IELength);*/
+
+					_rtw_memset(pbss->Ssid.Ssid, 0, pbss->Ssid.SsidLength);
+					_rtw_memcpy(pbss->Ssid.Ssid, sub_pie->data, sub_pie->Length);
+					pbss->Ssid.SsidLength = sub_pie->Length;
+					break;
+				case WLAN_EID_MULTI_BSSID_IDX:
+					/*RTW_INFO("%s, sub_pie->Length=%d\n", __func__, sub_pie->Length);*/
+					/*RTW_PRINT_DUMP("WLAN_EID_MULTI_BSSID_IDX: ", (const u8 *)sub_pie->data, sub_pie->Length);*/
+					mbssid_index = GET_MULTIPLE_BSSID_IDX_INDEX((u8 *)sub_pie);
+					/*RTW_INFO("%s,mbssid_index=%d\n", __func__, mbssid_index);*/
+					/*record  transmitted BSSID*/
+					_rtw_memcpy(pbss->tx_bssid, ref_bss->MacAddress, ETH_ALEN);
+					rtw_gen_new_bssid(ref_bss->MacAddress, max_bssid_indicator
+						                          , mbssid_index, pbss->MacAddress);
+					pbss->mbssid_index = mbssid_index;
+					break;
+				default:
+					break;
+				}
+
+				j += (sub_pie->Length + WLAN_IE_ID_LEN + WLAN_IE_LEN_LEN);
+				/*RTW_INFO("%s, j=%d\n", __func__, j);*/
+			}
+			pbss->is_mbssid = _TRUE;
+			add_network(padapter, pbss);
+			rtw_mfree((u8 *)pbss, sizeof(WLAN_BSSID_EX));
+			break;
+		case MBSSID_VENDOR_SPECIFIC_ID:
+			break;
+		default:
+			break;
+		}
+
+		i += (pIE->Length + WLAN_IE_ID_LEN + WLAN_IE_LEN_LEN);
+		/*RTW_INFO("%s, i=%d\n", __func__, i);*/
+	}
+}
+#endif
+
+void rtw_survey_event_callback(_adapter	*adapter, u8 *pbuf)
+{
+	u32 len;
+	u8 val8;
+	WLAN_BSSID_EX *bss;
+	struct	mlme_priv	*pmlmepriv = &(adapter->mlmepriv);
+
+	bss = (WLAN_BSSID_EX *)pbuf;
+
+	len = get_WLAN_BSSID_EX_sz(bss);
+	if (len > (sizeof(WLAN_BSSID_EX))) {
+		return;
+	}
+
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+    	val8 = 0;
+	rtw_hal_get_hwreg(adapter, HW_VAR_FREECNT, &val8);
+
+	/* use TSF if no free run counter */
+	if (val8==0)
+		bss->PhyInfo.free_cnt = (u32)rtw_hal_get_tsftr_by_port(
+			adapter, rtw_hal_get_port(adapter));
+#endif
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	/* update IBSS_network 's timestamp */
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) == _TRUE) {
+		if (_rtw_memcmp(&(pmlmepriv->dev_cur_network.network.MacAddress), bss->MacAddress, ETH_ALEN)) {
+			struct wlan_network *ibss_wlan = NULL;
+
+			_rtw_memcpy(pmlmepriv->dev_cur_network.network.IEs, bss->IEs, 8);
+			_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+			ibss_wlan = _rtw_find_network(&pmlmepriv->scanned_queue,  bss->MacAddress);
+			if (ibss_wlan) {
+				_rtw_memcpy(ibss_wlan->network.IEs , bss->IEs, 8);
+				_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+				goto exit;
+			}
+			_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+		}
+	}
+
+	/* lock pmlmepriv->lock when you accessing network_q */
+	if ((check_fwstate(pmlmepriv, WIFI_UNDER_LINKING)) == _FALSE) {
+		struct wlan_network *pnetwork;
+
+		if (bss->Ssid.Ssid[0] == 0)
+			bss->Ssid.SsidLength = 0;
+
+		pnetwork = add_network(adapter, bss);
+		if (pnetwork && bss->InfrastructureMode == Ndis802_11Infrastructure) {
+			if (MLME_IS_SCAN(adapter)) {
+				adapter->mlmeextpriv.sitesurvey_res.activate_ch_cnt
+					+= rtw_process_beacon_hint(adapter_to_rfctl(adapter), pnetwork);
+			}
+		}
+
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+		add_mbssid_network(adapter, bss);
+#endif
+#ifdef CONFIG_RTW_FSM_BTM
+		if (adapter->fsmpriv.btmpriv.btm) {
+			/* check SSID */
+			if ((bss->Ssid.SsidLength == pmlmepriv->dev_cur_network.network.Ssid.SsidLength) &&
+				_rtw_memcmp(bss->Ssid.Ssid, pmlmepriv->dev_cur_network.network.Ssid.Ssid,
+				bss->Ssid.SsidLength))
+				rtw_btm_notify_scan_found_candidate(adapter->fsmpriv.btmpriv.btm, pnetwork);
+		}
+#endif
+	}
+
+exit:
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+
+	return;
+}
+
+void rtw_surveydone_event_callback(_adapter *adapter, u8 *pbuf)
+{
+	struct surveydone_event *parm = (struct surveydone_event *)pbuf;
+	struct	mlme_priv	*pmlmepriv = &adapter->mlmepriv;
+	struct mlme_ext_priv	*pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	if (pmlmepriv->wps_probe_req_ie) {
+		u32 free_len = pmlmepriv->wps_probe_req_ie_len;
+		pmlmepriv->wps_probe_req_ie_len = 0;
+		rtw_mfree(pmlmepriv->wps_probe_req_ie, free_len);
+		pmlmepriv->wps_probe_req_ie = NULL;
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _FALSE) {
+		RTW_INFO(FUNC_ADPT_FMT" fw_state:0x%x\n", FUNC_ADPT_ARG(adapter), get_fwstate(pmlmepriv));
+		/* rtw_warn_on(1); */
+	}
+
+	_clr_fwstate_(pmlmepriv, WIFI_UNDER_SURVEY);
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	if (adapter_to_pwrctl(adapter)->bInSuspend == _TRUE)
+		goto done;
+
+	#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	rtw_set_signal_stat_timer(&adapter->recvinfo);
+	#endif
+	if (pmlmepriv->to_join == _TRUE) {
+		if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE)) {
+			if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _FALSE) {
+				set_fwstate(pmlmepriv, WIFI_UNDER_LINKING);
+
+				if (rtw_select_and_join_from_scanned_queue(pmlmepriv) == _SUCCESS) {
+					/*_set_timer(&pmlmepriv->assoc_timer, MAX_JOIN_TIMEOUT);*/
+					set_assoc_timer(pmlmepriv, MAX_JOIN_TIMEOUT);
+				}
+				else {
+					WLAN_BSSID_EX    *pdev_network = &(adapter->registrypriv.dev_network);
+					u8 *pibss = adapter->registrypriv.dev_network.MacAddress;
+
+					/* pmlmepriv->fw_state ^= WIFI_UNDER_SURVEY; */ /* because don't set assoc_timer */
+					_clr_fwstate_(pmlmepriv, WIFI_UNDER_SURVEY);
+
+
+					_rtw_memset(&pdev_network->Ssid, 0, sizeof(NDIS_802_11_SSID));
+					_rtw_memcpy(&pdev_network->Ssid, &pmlmepriv->assoc_ssid, sizeof(NDIS_802_11_SSID));
+
+					rtw_update_registrypriv_dev_network(adapter);
+					rtw_generate_random_ibss(pibss);
+
+					/*pmlmepriv->fw_state = WIFI_ADHOC_MASTER_STATE;*/
+					init_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+
+					if (rtw_create_ibss_cmd(adapter, 0) != _SUCCESS)
+						RTW_ERR("rtw_create_ibss_cmd FAIL\n");
+
+					rtw_clear_to_join_status(adapter);
+				}
+			}
+		} else {
+			int s_ret;
+			set_fwstate(pmlmepriv, WIFI_UNDER_LINKING);
+			rtw_clear_to_join_status(adapter);
+			s_ret = rtw_select_and_join_from_scanned_queue(pmlmepriv);
+			if (_SUCCESS == s_ret) {
+				/*_set_timer(&pmlmepriv->assoc_timer, MAX_JOIN_TIMEOUT);*/
+				set_assoc_timer(pmlmepriv, MAX_JOIN_TIMEOUT);
+			} else if (s_ret == 2) { /* there is no need to wait for join */
+				_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+				rtw_indicate_connect(adapter);
+			} else {
+				RTW_INFO("try_to_join, but select scanning queue fail, to_roam:%d\n", rtw_to_roam(adapter));
+
+#ifdef CONFIG_RTW_FSM_BTM
+				if (rtw_to_roam(adapter) == 0)
+					rtw_indicate_disconnect(adapter, 0, _FALSE);
+				_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+#else
+				if (rtw_to_roam(adapter) != 0) {
+					struct sitesurvey_parm *scan_parm = NULL;
+					u8 ssc_chk = rtw_sitesurvey_condition_check(adapter, _FALSE);
+
+					scan_parm = rtw_malloc(sizeof(*scan_parm));
+					if (scan_parm) {
+						rtw_init_sitesurvey_parm(adapter, scan_parm);
+						_rtw_memcpy(&scan_parm->ssid[0], &pmlmepriv->assoc_ssid, sizeof(NDIS_802_11_SSID));
+						scan_parm->ssid_num = 1;
+					}
+
+					if (scan_parm == NULL || rtw_dec_to_roam(adapter) == 0
+						|| (ssc_chk != SS_ALLOW && ssc_chk != SS_DENY_BUSY_TRAFFIC)
+						|| _SUCCESS != rtw_sitesurvey_cmd(adapter, scan_parm)
+					   ) {
+						rtw_set_to_roam(adapter, 0);
+						if (MLME_IS_ASOC(adapter) == _TRUE)
+							rtw_free_assoc_resources(adapter, _TRUE);
+						rtw_indicate_disconnect(adapter, 0, _FALSE);
+					} else
+						pmlmepriv->to_join = _TRUE;
+
+					if (scan_parm)
+						rtw_mfree(scan_parm, sizeof(*scan_parm));
+				} else
+					rtw_indicate_disconnect(adapter, 0, _FALSE);
+				_clr_fwstate_(pmlmepriv, WIFI_UNDER_LINKING);
+#endif /* CONFIG_RTW_FSM_BTM */
+			}
+		}
+	}
+#ifndef CONFIG_RTW_FSM_BTM
+	else {
+		if (rtw_chk_roam_flags(adapter, RTW_ROAM_ACTIVE)
+                #if (defined(CONFIG_RTW_WNM) && defined(CONFIG_RTW_80211R))
+                        || adapter->mlmepriv.need_to_roam == _TRUE
+                #endif
+		) {
+			if (MLME_IS_STA(adapter)
+			    && check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+				if (rtw_select_roaming_candidate(pmlmepriv)) {
+#ifdef CONFIG_RTW_80211R
+					rtw_ft_start_roam(adapter,
+						(u8 *)pmlmepriv->roam_network->network.MacAddress);
+#else
+					receive_disconnect(adapter, pmlmepriv->dev_cur_network.network.MacAddress
+						, WLAN_REASON_ACTIVE_ROAM, _FALSE);
+					pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+					pmlmeinfo->disconnect_code = DISCONNECTION_BY_DRIVER_DUE_TO_ROAMING;
+					pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+#endif
+				}
+			}
+		}
+	}
+#endif /* CONFIG_RTW_FSM_BTM */
+
+done:
+	RTW_INFO("scan complete in %dms\n",rtw_get_passing_time_ms(pmlmepriv->scan_start_time));
+
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+#ifdef CONFIG_P2P_PS
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		p2p_ps_wk_cmd(adapter, P2P_PS_SCAN_DONE, 0);
+#endif /* CONFIG_P2P_PS */
+
+	rtw_mi_os_xmit_schedule(adapter);
+#ifdef CONFIG_DRVEXT_MODULE_WSC
+	drvext_surveydone_callback(&adapter->drvextpriv);
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_surveydone_event_callback(adapter);
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	rtw_indicate_scan_done(adapter, pmlmeext->scan_abort);
+
+#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_IOCTL_CFG80211)
+	rtw_cfg80211_indicate_scan_done_for_buddy(adapter, _FALSE);
+#endif
+
+	if (parm->activate_ch_cnt)
+		rtw_beacon_hint_ch_change_notifier(adapter_to_rfctl(adapter));
+
+#ifdef CONFIG_80211D
+	if (adapter_to_rfctl(adapter)->cis_enabled
+		&& (adapter_to_rfctl(adapter)->cis_flags & CISF_ENV_BSS))
+		rtw_apply_scan_network_country_ie_cmd(adapter, RTW_CMDF_DIRECTLY);
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	#if CONFIG_RTW_MESH_OFFCH_CAND
+	if (rtw_mesh_offch_candidate_accepted(adapter)) {
+		u8 ch;
+
+		ch = rtw_mesh_select_operating_ch(adapter);
+		if (ch && pmlmepriv->dev_cur_network.network.Configuration.DSConfig != ch) {
+			u8 ifbmp = rtw_mi_get_ap_mesh_ifbmp(adapter);
+			s8 band = REQ_BAND_NONE; /* TODO */
+
+			if (ifbmp) {
+				/* switch to selected channel */
+				rtw_change_bss_bchbw_cmd(adapter, RTW_CMDF_DIRECTLY, ifbmp, 0, band, ch, REQ_BW_ORI, REQ_OFFSET_NONE);
+				issue_probereq_ex(adapter, &pmlmepriv->dev_cur_network.network.mesh_id, NULL, 0, 0, 0, 0);
+			} else
+				rtw_warn_on(1);
+		}
+	}
+	#endif
+#endif /* CONFIG_RTW_MESH */
+
+#ifdef CONFIG_RTW_ACS
+	if (parm->acs) {
+		u8 ifbmp = rtw_mi_get_ap_mesh_ifbmp(adapter);
+
+		if (ifbmp)
+			rtw_change_bss_bchbw_cmd(adapter, RTW_CMDF_DIRECTLY, ifbmp, 0, REQ_BAND_NONE, REQ_CH_INT_INFO, REQ_BW_ORI, REQ_OFFSET_NONE);
+	}
+#endif
+}
+
+u8 _rtw_sitesurvey_condition_check(const char *caller, _adapter *adapter, bool check_sc_interval)
+{
+	u8 ss_condition = SS_ALLOW;
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct registry_priv *registry_par = &adapter->registrypriv;
+
+
+#ifdef CONFIG_MP_INCLUDED
+	if (rtw_mp_mode_check(adapter)) {
+		RTW_INFO("%s ("ADPT_FMT") MP mode block Scan request\n", caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_MP_MODE;
+		goto _exit;
+	}
+#endif
+
+#ifdef DBG_LA_MODE
+	if(registry_par->la_mode_en == 1 && MLME_IS_ASOC(adapter)) {
+		RTW_INFO("%s ("ADPT_FMT") LA debug mode block Scan request\n", caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_LA_MODE;
+		goto _exit;
+	}
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (adapter_wdev_data(adapter)->block_scan == _TRUE) {
+		RTW_INFO("%s ("ADPT_FMT") wdev_priv.block_scan is set\n", caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_BLOCK_SCAN;
+		goto _exit;
+	}
+#endif
+
+	if (adapter_to_dvobj(adapter)->scan_deny == _TRUE) {
+		RTW_INFO("%s ("ADPT_FMT") tpt mode, scan deny!\n", caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_BLOCK_SCAN;
+		goto _exit;
+	}
+
+	if (rtw_is_scan_deny(adapter)) {
+		RTW_INFO("%s ("ADPT_FMT") : scan deny\n", caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_BY_DRV;
+		goto _exit;
+	}
+
+	#if 0 /*GEORGIA_TODO_FIXIT*/
+	if (adapter_to_rfctl(adapter)->adaptivity_en
+	    && rtw_hal_get_phy_edcca_flag(adapter)
+	    && rtw_is_2g_ch(GET_PHL_COM(adapter_to_dvobj(adapter))->current_channel)) {
+		RTW_WARN(FUNC_ADPT_FMT": Adaptivity block scan! (ch=%u)\n",
+			 FUNC_ADPT_ARG(adapter),
+			 GET_PHL_COM(adapter_to_dvobj(adapter))->current_channel);
+		ss_condition = SS_DENY_ADAPTIVITY;
+		goto _exit;
+	}
+	#endif
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)){
+		if(check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+			RTW_INFO("%s ("ADPT_FMT") : scan abort!! AP mode process WPS\n", caller, ADPT_ARG(adapter));
+			ss_condition = SS_DENY_SELF_AP_UNDER_WPS;
+			goto _exit;
+		} else if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE) {
+			RTW_INFO("%s ("ADPT_FMT") : scan abort!!AP mode under linking (fwstate=0x%x)\n",
+				caller, ADPT_ARG(adapter), pmlmepriv->fw_state);
+			ss_condition = SS_DENY_SELF_AP_UNDER_LINKING;
+			goto _exit;
+		} else if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE) {
+			RTW_INFO("%s ("ADPT_FMT") : scan abort!!AP mode under survey (fwstate=0x%x)\n",
+				caller, ADPT_ARG(adapter), pmlmepriv->fw_state);
+			ss_condition = SS_DENY_SELF_AP_UNDER_SURVEY;
+			goto _exit;
+		}
+	} else {
+		if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE) {
+			RTW_INFO("%s ("ADPT_FMT") : scan abort!!STA mode under linking (fwstate=0x%x)\n",
+				caller, ADPT_ARG(adapter), pmlmepriv->fw_state);
+			ss_condition = SS_DENY_SELF_STA_UNDER_LINKING;
+			goto _exit;
+		} else if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE) {
+			RTW_INFO("%s ("ADPT_FMT") : scan abort!!STA mode under survey (fwstate=0x%x)\n",
+				caller, ADPT_ARG(adapter), pmlmepriv->fw_state);
+			ss_condition = SS_DENY_SELF_STA_UNDER_SURVEY;
+			goto _exit;
+		}
+	}
+
+#ifdef CONFIG_CONCURRENT_MODE
+#ifdef PRIVATE_R
+	if (rtw_mi_buddy_check_fwstate(adapter, WIFI_UNDER_LINKING)) {
+#else
+	if (rtw_mi_buddy_check_fwstate(adapter, WIFI_UNDER_LINKING | WIFI_UNDER_WPS)) {
+#endif /* PRIVATE_R */
+		RTW_INFO("%s ("ADPT_FMT") : scan abort!! buddy_intf under linking or wps\n", caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_BUDDY_UNDER_LINK_WPS;
+		goto _exit;
+
+	} else if (rtw_mi_buddy_check_fwstate(adapter, WIFI_UNDER_SURVEY)) {
+		RTW_INFO("%s ("ADPT_FMT") : scan abort!! buddy_intf under survey\n", caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_BUDDY_UNDER_SURVEY;
+		goto _exit;
+	}
+#endif /* CONFIG_CONCURRENT_MODE */
+
+#ifdef RTW_BUSY_DENY_SCAN
+	/*
+	 * busy traffic check
+	 * Rules:
+	 * 1. If (scan interval <= BUSY_TRAFFIC_SCAN_DENY_PERIOD) always allow
+	 *    scan, otherwise goto rule 2.
+	 * 2. Deny scan if any interface is busy, otherwise allow scan.
+	 */
+	if (pmlmepriv->lastscantime
+	    && (rtw_get_passing_time_ms(pmlmepriv->lastscantime) >
+		registry_par->scan_interval_thr)
+	    && rtw_mi_busy_traffic_check(adapter)) {
+		RTW_WARN("%s ("ADPT_FMT") : scan abort!! BusyTraffic\n",
+			 caller, ADPT_ARG(adapter));
+		ss_condition = SS_DENY_BUSY_TRAFFIC;
+		goto _exit;
+	}
+#endif /* RTW_BUSY_DENY_SCAN */
+
+_exit:
+	return ss_condition;
+}
+
+
+/*rtw_mlme_ext.c*/
+void sitesurvey_set_offch_state(_adapter *adapter, u8 scan_state)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	_rtw_mutex_lock_interruptible(&rfctl->offch_mutex);
+
+	switch (scan_state) {
+	case SCAN_DISABLE:
+	case SCAN_BACK_OP:
+		rfctl->offch_state = OFFCHS_NONE;
+		break;
+	case SCAN_START:
+	case SCAN_LEAVING_OP:
+		rfctl->offch_state = OFFCHS_LEAVING_OP;
+		break;
+	case SCAN_ENTER:
+	case SCAN_LEAVE_OP:
+		rfctl->offch_state = OFFCHS_LEAVE_OP;
+		break;
+	case SCAN_COMPLETE:
+	case SCAN_BACKING_OP:
+		rfctl->offch_state = OFFCHS_BACKING_OP;
+		break;
+	default:
+		break;
+	}
+
+	_rtw_mutex_unlock(&rfctl->offch_mutex);
+}
+static u8 rtw_scan_sparse(_adapter *adapter, struct rtw_ieee80211_channel *ch, u8 ch_num)
+{
+	/* interval larger than this is treated as backgroud scan */
+#ifndef RTW_SCAN_SPARSE_BG_INTERVAL_MS
+#define RTW_SCAN_SPARSE_BG_INTERVAL_MS 12000
+#endif
+
+#ifndef RTW_SCAN_SPARSE_CH_NUM_MIRACAST
+#define RTW_SCAN_SPARSE_CH_NUM_MIRACAST 1
+#endif
+#ifndef RTW_SCAN_SPARSE_CH_NUM_BG
+#define RTW_SCAN_SPARSE_CH_NUM_BG 4
+#endif
+
+#define SCAN_SPARSE_CH_NUM_INVALID 255
+
+	static u8 token = 255;
+	u32 interval;
+	bool busy_traffic = _FALSE;
+	bool miracast_enabled = _FALSE;
+	bool bg_scan = _FALSE;
+	u8 max_allow_ch = SCAN_SPARSE_CH_NUM_INVALID;
+	u8 scan_division_num;
+	u8 ret_num = ch_num;
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	if (mlmeext->last_scan_time == 0)
+		mlmeext->last_scan_time = rtw_get_current_time();
+
+	interval = rtw_get_passing_time_ms(mlmeext->last_scan_time);
+
+
+	if (rtw_mi_busy_traffic_check(adapter))
+		busy_traffic = _TRUE;
+
+	if (rtw_mi_check_miracast_enabled(adapter))
+		miracast_enabled = _TRUE;
+
+	if (interval > RTW_SCAN_SPARSE_BG_INTERVAL_MS)
+		bg_scan = _TRUE;
+
+	/* max_allow_ch by conditions*/
+
+#if RTW_SCAN_SPARSE_MIRACAST
+	if (miracast_enabled == _TRUE && busy_traffic == _TRUE)
+		max_allow_ch = rtw_min(max_allow_ch, RTW_SCAN_SPARSE_CH_NUM_MIRACAST);
+#endif
+
+#if RTW_SCAN_SPARSE_BG
+	if (bg_scan == _TRUE)
+		max_allow_ch = rtw_min(max_allow_ch, RTW_SCAN_SPARSE_CH_NUM_BG);
+#endif
+
+
+	if (max_allow_ch != SCAN_SPARSE_CH_NUM_INVALID) {
+		int i;
+		int k = 0;
+
+		scan_division_num = (ch_num / max_allow_ch) + ((ch_num % max_allow_ch) ? 1 : 0);
+		token = (token + 1) % scan_division_num;
+
+		if (0)
+			RTW_INFO("scan_division_num:%u, token:%u\n", scan_division_num, token);
+
+		for (i = 0; i < ch_num; i++) {
+			if (ch[i].hw_value && (i % scan_division_num) == token
+			   ) {
+				if (i != k)
+					_rtw_memcpy(&ch[k], &ch[i], sizeof(struct rtw_ieee80211_channel));
+				k++;
+			}
+		}
+
+		_rtw_memset(&ch[k], 0, sizeof(struct rtw_ieee80211_channel));
+
+		ret_num = k;
+		mlmeext->last_scan_time = rtw_get_current_time();
+	}
+
+	return ret_num;
+}
+
+int rtw_scan_ch_decision(_adapter *padapter, struct rtw_ieee80211_channel *out,
+		u32 out_num, struct rtw_ieee80211_channel *in, u32 in_num, bool no_sparse)
+{
+	int i, j;
+	int set_idx;
+	u8 chan;
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	struct registry_priv *regsty = dvobj_to_regsty(adapter_to_dvobj(padapter));
+
+	/* clear first */
+	_rtw_memset(out, 0, sizeof(struct rtw_ieee80211_channel) * out_num);
+
+	/* acquire channels from in */
+	j = 0;
+	for (i = 0; i < in_num; i++) {
+
+		if (0)
+			RTW_INFO(FUNC_ADPT_FMT" "CHAN_FMT"\n", FUNC_ADPT_ARG(padapter), CHAN_ARG(&in[i]));
+
+		if (!in[i].hw_value || (in[i].flags & RTW_IEEE80211_CHAN_DISABLED))
+			continue;
+		if (rtw_mlme_band_check(padapter, in[i].hw_value) == _FALSE)
+			continue;
+
+		set_idx = rtw_chset_search_bch(chset, in[i].band, in[i].hw_value);
+
+		if (set_idx >= 0) {
+			if (j >= out_num) {
+				RTW_PRINT(FUNC_ADPT_FMT" out_num:%u not enough\n",
+					  FUNC_ADPT_ARG(padapter), out_num);
+				break;
+			}
+
+			_rtw_memcpy(&out[j], &in[i], sizeof(struct rtw_ieee80211_channel));
+
+			if (chset->chs[set_idx].flags & (RTW_CHF_NO_IR | RTW_CHF_DFS))
+				out[j].flags |= RTW_IEEE80211_CHAN_PASSIVE_SCAN;
+
+			j++;
+		}
+		if (j >= out_num)
+			break;
+	}
+
+	/* if out is empty, use chset as default */
+	if (j == 0) {
+		for (i = 0; i < chset->chs_len; i++) {
+			if (chset->chs[i].flags & RTW_CHF_DIS)
+				continue;
+			chan = chset->chs[i].ChannelNum;
+			if (rtw_mlme_band_check(padapter, chan) == _TRUE) {
+				if (rtw_mlme_ignore_chan(padapter, chan) == _TRUE)
+					continue;
+
+				if (j >= out_num) {
+					RTW_PRINT(FUNC_ADPT_FMT" out_num:%u not enough\n",
+						FUNC_ADPT_ARG(padapter), out_num);
+					break;
+				}
+
+				out[j].hw_value = chan;
+				out[j].band = chset->chs[i].band;
+				if (0)
+					RTW_INFO(FUNC_ADPT_FMT"band:%d ch:%u\n",
+						FUNC_ADPT_ARG(padapter), out[j].band, chan);
+
+				if (chset->chs[i].flags & (RTW_CHF_NO_IR | RTW_CHF_DFS))
+					out[j].flags |= RTW_IEEE80211_CHAN_PASSIVE_SCAN;
+
+				j++;
+			}
+		}
+	}
+
+	if (!no_sparse
+		&& !regsty->wifi_spec
+		&& j > 6 /* assume ch_num > 6 is normal scan */
+	) {
+		/* scan_sparse */
+		j = rtw_scan_sparse(padapter, out, j);
+	}
+
+	return j;
+}
+#ifdef CONFIG_SCAN_BACKOP
+u8 rtw_scan_backop_decision(_adapter *adapter)
+{
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct mi_state mstate;
+	struct mr_query_info info = {0};
+	u8 backop_flags = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	rtw_mi_status(adapter, &mstate);
+	rtw_phl_mr_query_info(dvobj->phl, adapter->phl_role,
+			adapter_link->wrlink, &info);
+
+	if ((info.cur_info.ld_sta_num && mlmeext_chk_scan_backop_flags_sta(mlmeext, SS_BACKOP_EN))
+		|| (info.cur_info.sta_num && mlmeext_chk_scan_backop_flags_sta(mlmeext, SS_BACKOP_EN_NL)))
+		backop_flags |= mlmeext_scan_backop_flags_sta(mlmeext);
+
+#ifdef CONFIG_AP_MODE
+	if ((info.cur_info.ld_ap_num && mlmeext_chk_scan_backop_flags_ap(mlmeext, SS_BACKOP_EN))
+		|| (info.cur_info.ap_num && mlmeext_chk_scan_backop_flags_ap(mlmeext, SS_BACKOP_EN_NL)))
+		backop_flags |= mlmeext_scan_backop_flags_ap(mlmeext);
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	if ((MSTATE_MESH_LD_NUM(&mstate) && mlmeext_chk_scan_backop_flags_mesh(mlmeext, SS_BACKOP_EN))
+		|| (MSTATE_MESH_NUM(&mstate) && mlmeext_chk_scan_backop_flags_mesh(mlmeext, SS_BACKOP_EN_NL)))
+		backop_flags |= mlmeext_scan_backop_flags_mesh(mlmeext);
+#endif
+
+	return backop_flags;
+}
+#endif
+
+/* remain on channel priv */
+#define ROCH_CH_READY	0x1
+
+struct scan_priv {
+	_adapter *padapter;
+
+	/* for remain on channel callback */
+	struct wireless_dev *wdev;
+	struct ieee80211_channel channel;
+	u8 channel_type;
+	unsigned int duration;
+	u64 cookie;
+
+	u8 restore_ch;
+
+	u8 roch_step;
+#if defined(CONFIG_RTW_80211K)
+	u32 rrm_token;	/* 80211k use it to identify caller */
+#endif
+#ifdef CONFIG_RTW_FSM
+	struct sta_info *psta;
+#if defined(CONFIG_RTW_FSM_RRM) || defined(CONFIG_RTW_FSM_BTM)
+	u8 nb_num;
+	struct nb_bssid nb[RTW_MAX_NB_RPT_NUM];
+#endif
+#endif /* CONFIG_RTW_FSMCONFIG_RTW_FSM */
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	bool pending_6ghz_scan; /* indicate scan 6GHz channels later */
+	bool split_scan_6ghz; /* indicate if current scan is second scan for 6GHz */
+#endif
+#endif
+	u8 reason;
+};
+
+static struct rtw_phl_scan_param *_alloc_phl_param(_adapter *adapter, u8 scan_ch_num)
+{
+	struct rtw_phl_scan_param *phl_param = NULL;
+	struct scan_priv *scan_priv = NULL;
+
+	if (adapter->phl_role == NULL) {
+		RTW_ERR(FUNC_ADPT_FMT" phl_role == NULL\n", FUNC_ADPT_ARG(adapter));
+		goto _err_exit;
+	}
+
+	if (scan_ch_num == 0) {
+		RTW_ERR("%s scan_ch_num = 0\n", __func__);
+		goto _err_exit;
+	}
+	/*create mem of PHL Scan parameter*/
+	phl_param = rtw_zmalloc(sizeof(*phl_param));
+	if (phl_param == NULL) {
+		RTW_ERR("%s alloc phl_param fail\n", __func__);
+		goto _err_exit;
+	}
+
+	scan_priv = rtw_zmalloc(sizeof(*scan_priv));
+	if (scan_priv == NULL) {
+		RTW_ERR("%s alloc scan_priv fail\n", __func__);
+		goto _err_scanpriv;
+	}
+	scan_priv->padapter = adapter;
+	phl_param->priv = scan_priv;
+	phl_param->wifi_role = adapter->phl_role;
+	phl_param->back_op.mode = SCAN_BKOP_NONE;
+
+	phl_param->ch_sz = sizeof(struct phl_scan_channel) * (scan_ch_num + 1);
+	phl_param->ch = rtw_zmalloc(phl_param->ch_sz);
+
+	if (phl_param->ch == NULL) {
+		RTW_ERR("%s: alloc phl scan ch fail\n", __func__);
+		goto _err_param_ch;
+	}
+
+	return phl_param;
+
+_err_param_ch:
+	if (scan_priv)
+		rtw_mfree(scan_priv, sizeof(*scan_priv));
+_err_scanpriv:
+	if (phl_param) {
+		rtw_mfree(phl_param, sizeof(*phl_param));
+		phl_param = NULL;
+	}
+_err_exit:
+	rtw_warn_on(1);
+	return NULL;
+}
+
+static u8 _free_phl_param(_adapter *adapter, struct rtw_phl_scan_param *phl_param)
+{
+	u8 res = _FAIL;
+
+	if (!phl_param)
+		return res;
+
+	if (phl_param->ch)
+		rtw_mfree(phl_param->ch, phl_param->ch_sz);
+	if (phl_param->priv)
+		rtw_mfree(phl_param->priv, sizeof(struct scan_priv));
+	rtw_mfree(phl_param, sizeof(struct rtw_phl_scan_param));
+
+	res = _SUCCESS;
+	return res;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#define SCAN_6G_TIME_UNIT 20 /* 20ms */
+static int scan_probe_6g_cb(void *priv, struct rtw_phl_scan_param *param, u8 sctrl_idx)
+{
+	struct cmd_scan_ctrl *sctrl = &param->sctrl[sctrl_idx];
+	struct phl_scan_channel *scan_ch = sctrl->scan_ch;
+	struct rtw_wifi_role_t *wrole = sctrl->wrole;
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	NDIS_802_11_SSID ssid;
+	int i;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+
+	if (scan_ch->band != BAND_ON_6G)
+		return 0;
+
+	if (!Is6GHzPreferScanChannel(scan_ch->channel))
+		return 0;
+
+	/* send probe request after first 20ms interval */
+	if ((scan_ch->pass_t / SCAN_6G_TIME_UNIT) != 1)
+		return 0;
+
+	if (wrole == NULL) {
+		rtw_warn_on(1);
+		return 0;
+	}
+
+	/*RTW_INFO("%s: start_t=%d, remain_t=%d, pass_t=%d, last_t=%d\n",
+			__func__, scan_ch->start_t, scan_ch->remain_t, scan_ch->pass_t, scan_ch->last_t);*/
+
+	padapter = dvobj->padapters[wrole->id];
+	if (!rtw_is_adapter_up(padapter))
+		return 0;
+	padapter_link = GET_PRIMARY_LINK(padapter);
+
+	for (i = 0; i < param->ssid_num; i++) {
+		if (param->ssid[i].ssid_len == 0)
+			continue;
+
+		ssid.SsidLength = param->ssid[i].ssid_len;
+		_rtw_memcpy(ssid.Ssid, &param->ssid[i].ssid, ssid.SsidLength);
+		if (padapter->registrypriv.wifi_spec)
+			issue_probereq(padapter, padapter_link, &ssid, NULL);
+		else
+			issue_probereq_ex(padapter, padapter_link,&ssid, NULL, 0, 0, 0, 0);
+	}
+
+	return 0;
+}
+#endif
+
+static int scan_issue_pbreq_cb(void *priv, struct rtw_phl_scan_param *param, u8 sctrl_idx)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *wrole = param->sctrl[sctrl_idx].wrole;
+	NDIS_802_11_SSID ssid;
+	int i;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = NULL;
+
+	if (wrole == NULL) {
+		rtw_warn_on(1);
+		return 0;
+	}
+
+	padapter = dvobj->padapters[wrole->id];
+	if (!rtw_is_adapter_up(padapter))
+		return 0;
+	padapter_link = GET_PRIMARY_LINK(padapter);
+
+	/* active scan behavior */
+	if (padapter->registrypriv.wifi_spec)
+		issue_probereq(padapter, padapter_link, NULL, NULL);
+	else
+		issue_probereq_ex(padapter, padapter_link, NULL, NULL, 0, 0, 0, 0);
+
+	issue_probereq(padapter, padapter_link, NULL, NULL);
+
+	for (i = 0; i < param->ssid_num; i++) {
+		if (param->ssid[i].ssid_len == 0)
+			continue;
+
+		ssid.SsidLength = param->ssid[i].ssid_len;
+		_rtw_memcpy(ssid.Ssid, &param->ssid[i].ssid, ssid.SsidLength);
+		/* IOT issue,
+		 * Send one probe req without WPS IE,
+		 * when not wifi_spec
+		 */
+		if (padapter->registrypriv.wifi_spec)
+			issue_probereq(padapter, padapter_link, &ssid, NULL);
+		else
+			issue_probereq_ex(padapter, padapter_link,&ssid, NULL, 0, 0, 0, 0);
+
+		issue_probereq(padapter, padapter_link, &ssid, NULL);
+	}
+
+	return 0;
+}
+
+static int scan_complete_cb(void *priv, struct rtw_phl_scan_param *param)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct _ADAPTER_LINK *alink;
+	int i;
+	bool acs = _FALSE;
+	int ret = _FAIL;
+
+	if (!rtw_is_adapter_up(padapter))
+		goto _exit;
+
+	#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+	{
+		struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+		const u8 *mac_addr = wdev_to_ndev(pwdev_priv->rtw_wdev)->dev_addr;
+
+		if (pwdev_priv->random_mac_enabled
+		    && (MLME_IS_STA(padapter))
+		    && (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _FALSE))
+			rtw_set_mac_addr(padapter, mac_addr, true, true);
+
+		pwdev_priv->random_mac_enabled = false;
+	}
+	#endif /* CONFIG_RTW_SCAN_RAND */
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	if ((scan_priv->pending_6ghz_scan == true) &&
+	    (_rtw_scan_abort_check(padapter, __func__) == _FALSE)) {
+		ret = rtw_cfg80211_split_scan_6ghz(padapter);
+		if(ret == _SUCCESS) {
+			RTW_INFO(FUNC_ADPT_FMT" takes %d ms to scan %d channels\n",
+				FUNC_ADPT_ARG(padapter), param->total_scan_time, param->ch_num);
+			RTW_INFO("%s: perform pended scan for 6GHz channels\n", __func__);
+			_free_phl_param(padapter, param);
+			return _SUCCESS;
+		} else {
+			RTW_INFO("%s: 6GHz pended scan FAILED, ret=%d\n", __func__, ret);
+		}
+	}
+	#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+	#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+	mlmeext_set_scan_state(pmlmeext, SCAN_DISABLE);
+	report_surveydone_event(padapter,
+		scan_priv->reason,
+		param->result == SCAN_REQ_COMPLETE,
+		acs, RTW_CMDF_DIRECTLY);
+
+	for (i = 0; i < ADAPTER_GET_LINK_NUM(padapter); i++) {
+		alink = GET_LINK(padapter, i);
+		if (LINK_MLME_IS_ASOC(alink) && ALINK_GET_BAND(alink) == BAND_ON_24G
+			/* TODO: AP Connected is 40MHz capability? */
+		) {
+			issue_action_BSSCoexistPacket(padapter);
+			issue_action_BSSCoexistPacket(padapter);
+			issue_action_BSSCoexistPacket(padapter);
+		}
+	}
+
+	ret = _SUCCESS;
+
+_exit:
+	RTW_INFO(FUNC_ADPT_FMT" takes %d ms to scan %d channels\n",
+			FUNC_ADPT_ARG(padapter), param->total_scan_time,
+			param->ch_num);
+	_rtw_scan_abort_check(padapter, __func__);
+
+	_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+	if (param != pmlmeext->sitesurvey_res.scan_param) {
+		RTW_WARN(FUNC_ADPT_FMT ": inconsistent scan param! (%px, %px)\n",
+			 FUNC_ADPT_ARG(padapter), param,
+			 pmlmeext->sitesurvey_res.scan_param);
+	}
+	_free_phl_param(padapter, param);
+	pmlmeext->sitesurvey_res.scan_param = NULL;
+	_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+	return ret;
+}
+
+static int scan_start_cb(void *priv, struct rtw_phl_scan_param *param)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+
+	#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+	{
+		struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+		const u8 *pno_mac_addr = pwdev_priv->pno_mac_addr;
+
+		if (pwdev_priv->random_mac_enabled
+		    && (MLME_IS_STA(padapter))
+		    && (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _FALSE)) {
+			u16 seq_num;
+
+			rtw_set_mac_addr(padapter, pno_mac_addr, true, true);
+			get_random_bytes(&seq_num, 2);
+			pwdev_priv->pno_scan_seq_num = seq_num & 0xFFF;
+		} else {
+			pwdev_priv->random_mac_enabled = false;
+		}
+	}
+	#endif /* CONFIG_RTW_SCAN_RAND */
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	if (scan_priv->split_scan_6ghz == false)
+	#endif
+	#endif
+	{
+		pmlmeext->sitesurvey_res.bss_cnt = 0;
+		pmlmeext->sitesurvey_res.activate_ch_cnt = 0;
+	}
+
+	//TODO remove
+	mlmeext_set_scan_state(pmlmeext, SCAN_PROCESS);
+
+	_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+	if (pmlmeext->sitesurvey_res.scan_param &&
+	    (param != pmlmeext->sitesurvey_res.scan_param)) {
+		RTW_ERR(FUNC_ADPT_FMT ": unexpected scan param! (%px, %px)\n",
+			FUNC_ADPT_ARG(padapter), param,
+			pmlmeext->sitesurvey_res.scan_param);
+	}
+	pmlmeext->sitesurvey_res.scan_param = param;
+	_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+	return 0;
+}
+
+#ifdef CONFIG_P2P
+static int scan_issue_p2p_pbreq_cb(void *priv, struct rtw_phl_scan_param *param, u8 sctrl_idx)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *wrole = param->sctrl[sctrl_idx].wrole;
+
+	if (wrole == NULL) {
+		rtw_warn_on(1);
+		return 0;
+	}
+
+	padapter = dvobj->padapters[wrole->id];
+
+	if (!rtw_is_adapter_up(padapter))
+		return 0;
+
+	issue_probereq_p2p(padapter, NULL);
+	issue_probereq_p2p(padapter, NULL);
+	issue_probereq_p2p(padapter, NULL);
+	return 0;
+}
+#endif
+
+static int scan_ch_ready_cb(void *priv, struct rtw_phl_scan_param *param, u8 sctrl_idx)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct phl_scan_channel *scan_ch = param->sctrl[sctrl_idx].scan_ch;
+
+	RTW_INFO("%s: band=%d, ch=%d\n", __func__, scan_ch->band, scan_ch->channel);
+	return 0;
+}
+
+
+static struct rtw_phl_scan_ops scan_ops_cb = {
+	.scan_start = scan_start_cb,
+	.scan_ch_ready = scan_ch_ready_cb,
+	.scan_complete = scan_complete_cb,
+	.scan_issue_pbreq = scan_issue_pbreq_cb,
+	/*.scan_issue_null_data = scan_issu_null_data_cb*/
+};
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static struct rtw_phl_scan_ops scan_ops_6g_cb = {
+	.scan_start = scan_start_cb,
+	.scan_ch_ready = scan_ch_ready_cb,
+	.scan_complete = scan_complete_cb,
+	.scan_issue_pbreq = scan_issue_pbreq_cb,
+	.scan_probe = scan_probe_6g_cb,
+};
+#endif
+
+#ifdef CONFIG_P2P
+static struct rtw_phl_scan_ops scan_ops_p2p_cb = {
+	.scan_start = scan_start_cb,
+	.scan_ch_ready = scan_ch_ready_cb,
+	.scan_complete = scan_complete_cb,
+	.scan_issue_pbreq = scan_issue_p2p_pbreq_cb,
+	/*.scan_issue_null_data = scan_issu_null_data_cb*/
+};
+#endif
+
+#ifdef CONFIG_RTW_FSM
+static int scan_start_fsm_cb(void *priv, struct rtw_phl_scan_param *param)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+
+	scan_start_cb(priv, param);
+	rtw_fsm_notify_scan_start(&padapter->fsmpriv, scan_priv->psta);
+
+	return 0;
+}
+#endif
+
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+static int scan_complete_rrm_cb(void *priv, struct rtw_phl_scan_param *param)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	int ret = _FAIL;
+
+	if (!rtw_is_adapter_up(padapter))
+		goto _exit;
+
+	mlmeext_set_scan_state(pmlmeext, SCAN_DISABLE);
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	_clr_fwstate_(pmlmepriv, WIFI_UNDER_SURVEY);
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+	/* inform RRM scan complete */
+#ifdef CONFIG_RTW_80211K
+	rm_post_event(padapter, scan_priv->rrm_token, RM_EV_survey_done);
+#endif
+	ret = _SUCCESS;
+
+_exit:
+	RTW_INFO(FUNC_ADPT_FMT" %d ms to scan %d channels (%x)\n",
+		FUNC_ADPT_ARG(padapter), param->total_scan_time,
+		param->ch_num, pmlmeext->sitesurvey_res.bss_cnt);
+	_rtw_scan_abort_check(padapter, __func__);
+
+#ifdef CONFIG_RTW_FSM
+	rtw_fsm_notify_scan_done(&padapter->fsmpriv, scan_priv->psta);
+#endif
+
+	_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+	if (param != pmlmeext->sitesurvey_res.scan_param) {
+		RTW_WARN(FUNC_ADPT_FMT ": inconsistent scan param! (%px, %px)\n",
+			 FUNC_ADPT_ARG(padapter), param,
+			 pmlmeext->sitesurvey_res.scan_param);
+	}
+	_free_phl_param(padapter, param);
+	pmlmeext->sitesurvey_res.scan_param = NULL;
+	_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+	return ret;
+}
+
+static struct rtw_phl_scan_ops scan_ops_rrm_cb = {
+#ifdef CONFIG_RTW_FSM
+	.scan_start = scan_start_fsm_cb,
+#else
+	.scan_start = scan_start_cb,
+#endif
+	.scan_ch_ready = scan_ch_ready_cb,
+	.scan_complete = scan_complete_rrm_cb,
+	.scan_issue_pbreq = scan_issue_pbreq_cb,
+};
+#endif /* CONFIG_RTW_80211K */
+
+#ifdef CONFIG_RTW_FSM_BTM
+static int scan_complete_wnm_cb(void *priv, struct rtw_phl_scan_param *param)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	int ret = _FAIL;
+
+	if (!rtw_is_adapter_up(padapter))
+		goto _exit;
+
+	mlmeext_set_scan_state(pmlmeext, SCAN_DISABLE);
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	_clr_fwstate_(pmlmepriv, WIFI_UNDER_SURVEY);
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+	ret = _SUCCESS;
+_exit:
+	RTW_INFO(FUNC_ADPT_FMT" %d ms to scan %d channels (%x)\n",
+		FUNC_ADPT_ARG(padapter), param->total_scan_time,
+		param->ch_num, pmlmeext->sitesurvey_res.bss_cnt);
+	_rtw_scan_abort_check(padapter, __func__);
+
+	rtw_fsm_notify_scan_done(&padapter->fsmpriv, scan_priv->psta);
+
+	_free_phl_param(padapter, param);
+	pmlmeext->sitesurvey_res.scan_param = NULL;
+
+	return ret;
+}
+
+static int scan_issue_pbreq_wnm_cb(void *priv, struct rtw_phl_scan_param *param, u8 sctrl_idx)
+{
+	struct scan_priv *spriv = (struct scan_priv *)priv;
+	_adapter *padapter = spriv->padapter;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_wifi_role_t *wrole = param->sctrl[sctrl_idx].wrole;
+	NDIS_802_11_SSID ssid;
+	int i, j;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = NULL;
+
+	if (wrole == NULL) {
+		rtw_warn_on(1);
+		return 0;
+	}
+
+	padapter = dvobj->padapters[wrole->id];
+	if (!rtw_is_adapter_up(padapter))
+		return 0;
+
+	padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (!param->ssid_num) {
+		issue_probereq(padapter, padapter_link, NULL, NULL);
+		return 0;
+	}
+
+	for (i = 0; i < param->ssid_num; i++) {
+		if (param->ssid[i].ssid_len == 0)
+			continue;
+
+		ssid.SsidLength = param->ssid[i].ssid_len;
+		_rtw_memcpy(ssid.Ssid, &param->ssid[i].ssid, ssid.SsidLength);
+
+		for (j = 0; j < spriv->nb_num; j++) {
+			if (param->ch->channel == spriv->nb[j].ch)
+				issue_probereq(padapter, padapter_link, &ssid, spriv->nb[j].bssid);
+		}
+		issue_probereq(padapter, padapter_link, &ssid, NULL);
+	}
+
+	return 0;
+}
+
+static struct rtw_phl_scan_ops scan_ops_wnm_cb = {
+	.scan_start = scan_start_fsm_cb,
+	.scan_ch_ready = scan_ch_ready_cb,
+	.scan_complete = scan_complete_wnm_cb,
+	.scan_issue_pbreq = scan_issue_pbreq_wnm_cb,
+	/*.scan_issue_null_data = scan_issu_null_data_cb*/
+};
+#endif /* CONFIG_RTW_FSM_BTM */
+
+#ifndef SCAN_PER_CH_EX_TIME
+#define SCAN_PER_CH_EX_TIME	40 /*8852bs sw ch overhead*/
+#endif /*SCAN_PER_CH_EX_TIME*/
+
+static u32 rtw_scan_timeout_decision(_adapter *padapter,
+	struct rtw_phl_scan_param *phl_param, u16 scan_ch_ms)
+{
+	u32 scan_timeout_ms = 0;
+	u8 max_chan_num;
+	u32 back_op_times = 0;
+	u16 backop_ms = 0;
+	u8 scan_cnt_max;
+	u16 ext_act_ms = 0;
+	int i;
+	u8 issue_null_time;
+	u16 p_ch_ex_time;
+	u32 non_op_buf;
+	#ifdef CONFIG_SCAN_BACKOP
+	u8 backop_cout = 0;
+	#endif /*CONFIG_SCAN_BACKOP*/
+	max_chan_num = phl_param->ch_num;
+
+
+	if (padapter->registrypriv.scan_pch_ex_time != 0)
+		p_ch_ex_time = padapter->registrypriv.scan_pch_ex_time;
+	else
+		p_ch_ex_time = SCAN_PER_CH_EX_TIME;
+
+	/*issue null time,null(0)+null(1),undefine flag phl sleep 50ms*/
+	issue_null_time = 10;
+	#ifndef RTW_WKARD_TX_NULL_WD_RP
+	issue_null_time += 50;
+	#endif /*RTW_WKARD_TX_NULL_WD_RP*/
+
+	#ifdef CONFIG_SCAN_BACKOP
+	if (phl_param->back_op.mode == SCAN_BKOP_CNT) {
+		backop_ms = phl_param->back_op.ch_dur_ms;
+		scan_cnt_max = phl_param->back_op.ch_intv;
+		if (scan_cnt_max == 0) {
+			scan_cnt_max = 1;
+			RTW_ERR("%s scan_cnt_max=0\n", __func__);
+			rtw_warn_on(1);
+		}
+		backop_cout = max_chan_num / scan_cnt_max;
+		back_op_times = backop_cout * (backop_ms + p_ch_ex_time + issue_null_time);
+	}
+	#endif
+
+	/* Extended active scan time */
+	for (i = 0; i < max_chan_num; i++) {
+		if (phl_param->ch[i].ext_act_scan == EXT_ACT_SCAN_ENABLE)
+			ext_act_ms += phl_param->ext_act_scan_period;
+	}
+
+	/*non op channel buffer time + scan start/done issue null*/
+	non_op_buf = max_chan_num * p_ch_ex_time + issue_null_time;
+
+	scan_timeout_ms = (scan_ch_ms * max_chan_num) + back_op_times
+			  + ext_act_ms + non_op_buf;
+	#ifdef DBG_SITESURVEY
+	RTW_INFO("%s , max_chan_num = %d , scan_cnt_max=%d,p_ch_ex_time=%d, "
+			"issue_null_time=%d\n",
+			__func__,max_chan_num, scan_cnt_max,
+			p_ch_ex_time, issue_null_time);
+
+	RTW_INFO(FUNC_ADPT_FMT ": scan_timeout_ms=%u, ch_num:%u scan_ch_ms=%u, "
+		 "back_op_times=%u, ext_act_ms=%u\n",
+		 FUNC_ADPT_ARG(padapter), scan_timeout_ms,
+		 max_chan_num, scan_ch_ms, back_op_times, ext_act_ms);
+	#endif /*DBG_SITESURVEY*/
+
+	return scan_timeout_ms;
+}
+
+/*
+rtw_sitesurvey_cmd(~)
+	### NOTE:#### (!!!!)
+	MUST TAKE CARE THAT BEFORE CALLING THIS FUNC, YOU SHOULD HAVE LOCKED pmlmepriv->lock
+*/
+static void scan_channel_list_filled(_adapter *padapter,
+	struct rtw_phl_scan_param *phl_param, struct sitesurvey_parm *param)
+{
+	struct phl_scan_channel *phl_ch = phl_param->ch;
+	u8 i = 0;
+	#ifdef CONFIG_RTW_ACS
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	phl_param->acs = _TRUE;
+	phl_param->nhm_include_cca = _FALSE;
+	#endif /*CONFIG_RTW_ACS*/
+
+	for (i = 0; i < param->ch_num; i++) {
+		phl_ch[i].channel = param->ch[i].hw_value;
+		phl_ch[i].band = param->ch[i].band;
+		phl_ch[i].scan_mode = NORMAL_SCAN_MODE;
+		phl_ch[i].bw = param->bw;
+		phl_ch[i].duration = param->duration;
+
+		#ifdef CONFIG_RTW_ACS
+		phl_ch[i].acs_idx = rtw_phl_get_acs_chnl_tbl_idx(dvobj->phl, phl_ch[i].band ,phl_ch[i].channel);
+		#endif /*CONFIG_RTW_ACS*/
+
+		if (param->ch[i].flags & RTW_IEEE80211_CHAN_PASSIVE_SCAN) {
+			phl_ch[i].type = RTW_PHL_SCAN_PASSIVE;
+		} else {
+			phl_ch[i].type = RTW_PHL_SCAN_ACTIVE;
+			phl_ch[i].duration = ACTIVE_CH_SURVEY_TO;
+		}
+	}
+	phl_param->ch_num = param->ch_num;
+}
+
+u8 rtw_sitesurvey_cmd(_adapter *padapter, struct sitesurvey_parm *pparm)
+{
+	u8 res = _FAIL;
+	u8 i, has_ssid = 0;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct rtw_phl_scan_param *phl_param = NULL;
+	struct rtw_ieee80211_channel ch[RTW_CHANNEL_SCAN_AMOUNT] = {0};
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct sitesurvey_parm *tmp_parm = NULL;
+	struct mlme_ext_priv *mlmeext = &padapter->mlmeextpriv;
+	struct ss_res *ss = &mlmeext->sitesurvey_res;
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM) || defined(CONFIG_RTW_FSM_BTM) || (CONFIG_IEEE80211_BAND_6GHZ && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)))
+	struct scan_priv *scan_priv = NULL;
+#endif
+
+	if (pparm == NULL) {
+		tmp_parm = rtw_zmalloc(sizeof(struct sitesurvey_parm));
+		if (tmp_parm == NULL) {
+			RTW_ERR("%s alloc tmp_parm fail\n", __func__);
+			goto _err_exit;
+		}
+		rtw_init_sitesurvey_parm(padapter, tmp_parm);
+		pparm = tmp_parm;
+	}
+
+	/* backup original ch list */
+	_rtw_memcpy(ch, pparm->ch,
+		sizeof(struct rtw_ieee80211_channel) * pparm->ch_num);
+
+	/* modify ch list according to chanel plan */
+	pparm->ch_num = rtw_scan_ch_decision(padapter,
+					pparm->ch, RTW_CHANNEL_SCAN_AMOUNT,
+					ch, pparm->ch_num, pparm->acs);
+
+	if (pparm->duration == 0)
+		pparm->duration = ss->scan_ch_ms; /* ms */
+
+	/*create mem of PHL Scan parameter*/
+	phl_param = _alloc_phl_param(padapter, pparm->ch_num);
+	if (phl_param == NULL) {
+		RTW_ERR("%s alloc phl_param fail\n", __func__);
+		goto _err_param;
+	}
+
+	scan_priv = (struct scan_priv *)phl_param->priv;
+
+	/* STEP_1 transfer to rtw channel list to phl channel list */
+	scan_channel_list_filled(padapter, phl_param, pparm);
+
+	/* STEP_2 copy the ssid info to phl param */
+	phl_param->ssid_num = rtw_min(pparm->ssid_num, SCAN_SSID_AMOUNT);
+	for (i = 0; i < phl_param->ssid_num; ++i) {
+		phl_param->ssid[i].ssid_len = pparm->ssid[i].SsidLength;
+		_rtw_memcpy(&phl_param->ssid[i].ssid, &pparm->ssid[i].Ssid, phl_param->ssid[i].ssid_len);
+	}
+
+	/* STEP_2.1 set EXT_ACT_SCAN_ENABLE for hidden AP scan */
+	for (i = 0; i < phl_param->ssid_num; i++) {
+		if (phl_param->ssid[i].ssid_len)
+			has_ssid++;
+	}
+
+	if (has_ssid) {
+		phl_param->ext_act_scan_period = RTW_EXTEND_ACTIVE_SCAN_PERIOD;
+		for (i = 0; i < phl_param->ch_num; i++) {
+			int chset_idx;
+			chset_idx = rtw_chset_search_bch(chset,
+							phl_param->ch[i].band, phl_param->ch[i].channel);
+			if (chset_idx < 0) {
+				RTW_ERR(FUNC_ADPT_FMT ": cann't find ch %u in chset!\n",
+					FUNC_ADPT_ARG(padapter), phl_param->ch[i].channel);
+				continue;
+			}
+			if ((phl_param->ch[i].type == RTW_PHL_SCAN_PASSIVE)
+			    && !CH_IS_NON_OCP(&chset->chs[chset_idx]))
+				phl_param->ch[i].ext_act_scan = EXT_ACT_SCAN_ENABLE;
+		}
+	}
+	#ifdef CONFIG_DBCC_SUPPORT
+	if (!adapter_to_dvobj(padapter)->dis_dbcc_scan)
+		phl_param->opt = SCAN_CMD_OPT_DB;
+	#endif
+
+	scan_priv->reason = pparm->reason;
+
+	/* STEP_3 set ops according to scan_type */
+	switch (pparm->scan_type) {
+	#ifdef CONFIG_P2P
+	case RTW_SCAN_P2P:
+		phl_param->ops = &scan_ops_p2p_cb;
+	break;
+	#endif
+
+	#ifdef CONFIG_RTW_FSM_BTM
+	case RTW_SCAN_ROAM:
+
+		phl_param->ops = &scan_ops_wnm_cb;
+		scan_priv->psta = pparm->psta;
+		memcpy(scan_priv->nb, pparm->nb, sizeof(struct nb_bssid) * RTW_MAX_NB_RPT_NUM);
+		scan_priv->nb_num = pparm->nb_num;
+
+		#ifdef CONFIG_SCAN_BACKOP
+		if (rtw_scan_backop_decision(padapter)) {
+			phl_param->back_op.ch_dur_ms = ss->backop_ms;
+			phl_param->back_op.mode = SCAN_BKOP_CNT;
+			phl_param->back_op.ch_intv = ss->scan_cnt_max;
+		} else
+		#endif /* CONFIG_SCAN_BACKOP */
+		{
+			phl_param->back_op.mode = SCAN_BKOP_NONE;
+		}
+	break;
+	#endif
+	#ifdef CONFIG_RTW_80211K
+	case RTW_SCAN_RRM:
+		phl_param->ops = &scan_ops_rrm_cb;
+		scan_priv->rrm_token = pparm->rrm_token;
+		ss->token = pparm->rrm_token;
+	break;
+	#endif
+	#ifdef CONFIG_RTW_FSM_RRM
+	case RTW_SCAN_RRM:
+		phl_param->ops = &scan_ops_rrm_cb;
+		scan_priv->psta = pparm->psta;
+	break;
+	#endif
+
+	case RTW_SCAN_NORMAL:
+	default:
+		phl_param->ops = &scan_ops_cb;
+
+		#ifdef CONFIG_SCAN_BACKOP
+		if (rtw_scan_backop_decision(padapter)) {
+			phl_param->back_op.ch_dur_ms = ss->backop_ms;
+			phl_param->back_op.mode = SCAN_BKOP_CNT;
+			phl_param->back_op.ch_intv = ss->scan_cnt_max;
+		} else
+		#endif /* CONFIG_SCAN_BACKOP */
+		{
+			phl_param->back_op.mode = SCAN_BKOP_NONE;
+		}
+
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+		scan_priv->pending_6ghz_scan = pparm->pending_6ghz_scan;
+		scan_priv->split_scan_6ghz = pparm->split_scan_6ghz;
+		#endif
+
+		if (pparm->scan_6ghz_only) {
+			phl_param->ops = &scan_ops_6g_cb;
+			phl_param->probe_t = SCAN_6G_TIME_UNIT;
+		}
+		#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+	break;
+	}
+#ifdef CONFIG_SCANOFLD
+	if (rtw_phl_scanofld_support(GET_PHL_INFO(dvobj))) {
+		phl_param->mode = SCAN_MD_FW;
+		phl_param->chkpt_time = 0;
+	}
+#endif
+	phl_param->max_scan_time = rtw_scan_timeout_decision(padapter, phl_param, pparm->duration);
+	#ifdef PRIVATE_R
+	phl_param->max_scan_time += 10000;
+	#endif
+
+	/* STEP_4 reset variables for each scan */
+	for (i = 0; i < chset->chs_len; i++)
+		chset->chs[i].hidden_bss_cnt = 0;
+
+	set_fwstate(pmlmepriv, WIFI_UNDER_SURVEY);
+	_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+	if (pmlmeext->sitesurvey_res.scan_param) {
+		RTW_WARN(FUNC_ADPT_FMT ": unexpected scan param! (%px, %px)\n",
+			 FUNC_ADPT_ARG(padapter), phl_param,
+			 pmlmeext->sitesurvey_res.scan_param);
+	}
+	if(rtw_phl_cmd_scan_request(dvobj->phl, phl_param) != RTW_PHL_STATUS_SUCCESS) {
+		_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+		RTW_ERR("%s request scam_cmd failed\n", __func__);
+		_clr_fwstate_(pmlmepriv, WIFI_UNDER_SURVEY);
+		goto _err_req_param;
+	}
+
+	pmlmeext->sitesurvey_res.scan_param = phl_param;
+	_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	if (pparm->split_scan_6ghz == false)
+	#endif
+	#endif
+	{
+		/* Do not free scan queue in split scan */
+		rtw_free_network_queue(padapter, _FALSE);
+		rtw_free_mld_network_queue(padapter, _FALSE);
+
+		/* Do not reset scan start time in split scan */
+		pmlmepriv->scan_start_time = rtw_get_current_time();
+	}
+
+	rtw_led_control(padapter, LED_CTL_SITE_SURVEY);
+	if (tmp_parm)
+		rtw_mfree(tmp_parm, sizeof(*tmp_parm));
+	res = _SUCCESS;
+	return res;
+
+_err_req_param:
+	_free_phl_param(padapter, phl_param);
+_err_param:
+	if (tmp_parm)
+		rtw_mfree(tmp_parm, sizeof(*tmp_parm));
+_err_exit:
+	rtw_warn_on(1);
+	return res;
+}
+
+/* inform caller phl_scan are ready on remain channel */
+static int roch_ready_cb(void *priv, struct rtw_phl_scan_param *param, u8 sctrl_idx)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct cfg80211_roch_info *pcfg80211_rochinfo =
+		&padapter->cfg80211_rochinfo;
+
+	RTW_INFO("%s cookie:0x%llx\n", __func__,
+		pcfg80211_rochinfo->remain_on_ch_cookie);
+
+	if ((scan_priv->roch_step & ROCH_CH_READY))
+		return 0;
+
+	scan_priv->roch_step |= ROCH_CH_READY;
+
+	rtw_cfg80211_ready_on_channel(
+		scan_priv->wdev,
+		scan_priv->cookie,
+		&scan_priv->channel,
+		scan_priv->channel_type,
+		scan_priv->duration,
+		GFP_KERNEL);
+	return 0;
+}
+
+static int roch_off_ch_tx_cb(void *priv,
+	struct rtw_phl_scan_param *param, void *data)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(scan_priv->padapter);
+
+	RTW_ERR("CMD_SCAN call %s\n", __func__);
+	rtw_warn_on(1);
+
+	return 0;
+}
+
+static int roch_start_cb(void *priv, struct rtw_phl_scan_param *param)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct cfg80211_roch_info *pcfg80211_rochinfo;
+
+	pcfg80211_rochinfo = &padapter->cfg80211_rochinfo;
+
+	//TODO remove
+	mlmeext_set_scan_state(&padapter->mlmeextpriv, SCAN_PROCESS);
+
+	rtw_cfg80211_set_is_roch(padapter, _TRUE);
+	pcfg80211_rochinfo->ro_ch_wdev = scan_priv->wdev;
+	pcfg80211_rochinfo->remain_on_ch_cookie = scan_priv->cookie;
+	pcfg80211_rochinfo->duration = scan_priv->duration;
+	rtw_cfg80211_set_last_ro_ch_time(padapter);
+	_rtw_memcpy(&pcfg80211_rochinfo->remain_on_ch_channel,
+		&scan_priv->channel, sizeof(struct ieee80211_channel));
+	#if (KERNEL_VERSION(3, 8, 0) > LINUX_VERSION_CODE)
+	pcfg80211_rochinfo->remain_on_ch_type = scan_priv->channel_type;
+	#endif
+	pcfg80211_rochinfo->restore_channel = scan_priv->restore_ch;
+
+	_rtw_spinlock_bh(&padapter->mlmeextpriv.sitesurvey_res.scan_param_lock);
+	if (padapter->mlmeextpriv.sitesurvey_res.scan_param) {
+		RTW_ERR(FUNC_ADPT_FMT ": unexpected scan param! (%px, %px)\n",
+			FUNC_ADPT_ARG(padapter), param,
+			padapter->mlmeextpriv.sitesurvey_res.scan_param);
+	}
+	padapter->mlmeextpriv.sitesurvey_res.scan_param = param;
+	_rtw_spinunlock_bh(&padapter->mlmeextpriv.sitesurvey_res.scan_param_lock);
+
+	return 0;
+}
+
+static int roch_complete_cb(void *priv, struct rtw_phl_scan_param *param)
+{
+	struct scan_priv *scan_priv = (struct scan_priv *)priv;
+	_adapter *padapter = scan_priv->padapter;
+	struct cfg80211_roch_info *pcfg80211_rochinfo =
+		&padapter->cfg80211_rochinfo;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+#endif
+	int ret = _FAIL;
+
+	if (!rtw_is_adapter_up(padapter))
+		goto _exit;
+
+	mlmeext_set_scan_state(pmlmeext, SCAN_DISABLE);
+
+	/* roch_ready() and roch_complete() MUST be a PAIR
+	 * otherwise will caurse wpa_supplicant hang!!!
+	 * This case may happen when someone cancel remain on ch
+	 * before it really start. (called roch_ready()).
+	 */
+	if (!(scan_priv->roch_step & ROCH_CH_READY))
+		roch_ready_cb(priv, param, 0);
+
+#ifdef CONFIG_P2P
+	if (adapter_wdev_data(padapter)->p2p_enabled)
+		RTW_INFO("%s, p2p role=%d\n", __func__, rtw_p2p_role(pwdinfo));
+#endif
+
+	rtw_cfg80211_set_is_roch(padapter, _FALSE);
+	pcfg80211_rochinfo->ro_ch_wdev = NULL;
+	rtw_cfg80211_set_last_ro_ch_time(padapter);
+
+	ret = _SUCCESS;
+
+_exit:
+	/* callback to cfg80211 */
+	rtw_cfg80211_remain_on_channel_expired(scan_priv->wdev
+		, scan_priv->cookie
+		, &scan_priv->channel
+		, scan_priv->channel_type, GFP_KERNEL);
+
+	RTW_INFO("cfg80211_remain_on_channel_expired cookie:0x%llx\n"
+		, pcfg80211_rochinfo->remain_on_ch_cookie);
+
+	RTW_INFO(FUNC_ADPT_FMT" takes %d ms to scan %d channels\n",
+			FUNC_ADPT_ARG(padapter), param->total_scan_time,
+			param->ch_num);
+	_rtw_scan_abort_check(padapter, __func__);
+
+	_rtw_spinlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+	if (param != pmlmeext->sitesurvey_res.scan_param) {
+		RTW_ERR(FUNC_ADPT_FMT ": inconsistent scan param! (%px, %px)\n",
+			FUNC_ADPT_ARG(padapter), param,
+			pmlmeext->sitesurvey_res.scan_param);
+	}
+	_free_phl_param(padapter, param);
+	pmlmeext->sitesurvey_res.scan_param = NULL;
+	_rtw_spinunlock_bh(&pmlmeext->sitesurvey_res.scan_param_lock);
+
+	return ret;
+}
+
+/* remain on channel callback */
+static struct rtw_phl_scan_ops remain_ops_cb = {
+	.scan_start = roch_start_cb,
+	.scan_ch_ready = roch_ready_cb,
+	.scan_off_ch_tx = roch_off_ch_tx_cb,
+	.scan_complete = roch_complete_cb,
+	/*.scan_issue_null_data = scan_issu_null_data_cb*/
+};
+
+#ifdef CONFIG_IOCTL_CFG80211
+static u8 roch_stay_in_cur_chan(_adapter *padapter)
+{
+	int i;
+	_adapter *iface;
+	struct mlme_priv *pmlmepriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u8 rst = _FALSE;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (!dvobj->padapters[i])
+			continue;
+
+		iface = dvobj->padapters[i];
+		if (!rtw_iface_at_same_hwband(padapter, iface))
+			continue;
+
+		pmlmepriv = &iface->mlmepriv;
+
+		if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING | WIFI_UNDER_WPS | WIFI_UNDER_KEY_HANDSHAKE) == _TRUE) {
+			RTW_INFO(ADPT_FMT"- WIFI_UNDER_LINKING |WIFI_UNDER_WPS | WIFI_UNDER_KEY_HANDSHAKE (mlme state:0x%x)\n",
+					ADPT_ARG(iface), get_fwstate(&iface->mlmepriv));
+			rst = _TRUE;
+			break;
+		}
+		#ifdef CONFIG_AP_MODE
+		if (MLME_IS_AP(iface) || MLME_IS_MESH(iface)) {
+			if (rtw_ap_sta_states_check(iface) == _TRUE) {
+				rst = _TRUE;
+			break;
+			}
+		}
+		#endif
+	}
+
+	return rst;
+}
+
+u8 rtw_phl_remain_on_ch_cmd(_adapter *padapter,
+	u64 cookie, struct wireless_dev *wdev,
+	struct ieee80211_channel *ch, u8 ch_type,
+	unsigned int duration, struct back_op_param *bkop_parm)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_scan_param *phl_param = NULL;
+	struct scan_priv *scan_priv = NULL;
+	u16 remain_ch;
+	enum band_type remain_band;
+	u8 chan_num;
+	u8 res = _FAIL;
+	struct rtw_chan_def u_chdef = {0};
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	/* prepare remain channel - check channel */
+	remain_ch = (u16)rtw_freq2ch(ch->center_freq);
+	remain_band = nl80211_band_to_rtw_band(ch->band);
+	if (roch_stay_in_cur_chan(padapter) == _TRUE) {
+		if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role,
+					padapter_link->wrlink, &u_chdef)
+						!= RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s get union chandef failed\n", __func__);
+			rtw_warn_on(1);
+		}
+		remain_ch = u_chdef.chan;
+		remain_band = u_chdef.band;
+		RTW_INFO(FUNC_ADPT_FMT" stay in union ch:%d, band:%d\n",
+			FUNC_ADPT_ARG(padapter), remain_ch, remain_band);
+	}
+	chan_num = 1;
+
+	phl_param = _alloc_phl_param(padapter, chan_num);
+	if (phl_param == NULL) {
+		RTW_ERR("%s alloc phl_param fail\n", __func__);
+		goto _err_exit;
+	}
+
+	/*** fill phl parameter - scan_priv ***/
+	scan_priv = (struct scan_priv *)phl_param->priv;
+	scan_priv->padapter = padapter;
+	scan_priv->wdev = wdev;
+	_rtw_memcpy(&scan_priv->channel, ch, sizeof(*ch));
+	scan_priv->channel_type = ch_type;
+	scan_priv->cookie = cookie;
+	scan_priv->duration = duration;
+	scan_priv->restore_ch = rtw_get_oper_ch(padapter, padapter_link);
+
+	/* fill phl param - chan */
+	phl_param->ch->channel = remain_ch;
+	phl_param->ch->bw = CHANNEL_WIDTH_20;
+	phl_param->ch->band = remain_band;
+	phl_param->ch->duration = duration;
+	phl_param->ch->scan_mode = P2P_LISTEN_MODE;
+	phl_param->ch_num = chan_num;
+
+	/* fill back op param */
+	phl_param->back_op.mode = SCAN_BKOP_TIMER;
+	phl_param->back_op.ch_intv = 1;
+	phl_param->back_op.ch_dur_ms = bkop_parm->on_ch_dur;/*op_ch time*/
+	phl_param->back_op.off_ch_dur_ms = bkop_parm->off_ch_dur;/*ro_ch time*/
+	phl_param->back_op.off_ch_ext_dur_ms = bkop_parm->off_ch_ext_dur;
+
+	/*phl_param->opt = SCAN_CMD_OPT_DB;*/
+	phl_param->ops = &remain_ops_cb;
+
+	phl_param->max_scan_time = duration;
+	_rtw_spinlock_bh(&padapter->mlmeextpriv.sitesurvey_res.scan_param_lock);
+	if(rtw_phl_cmd_scan_request(dvobj->phl, phl_param) == RTW_PHL_STATUS_FAILURE) {
+		_rtw_spinunlock_bh(&padapter->mlmeextpriv.sitesurvey_res.scan_param_lock);
+		RTW_ERR("%s request scam_cmd failed\n", __func__);
+		goto _err_req_param;
+	}
+	_rtw_spinunlock_bh(&padapter->mlmeextpriv.sitesurvey_res.scan_param_lock);
+
+	RTW_INFO(FUNC_ADPT_FMT" ch:%u duration:%d, cookie:0x%llx\n"
+			, FUNC_ADPT_ARG(padapter), remain_ch,	duration, cookie);
+	res = _SUCCESS;
+	return res;
+
+_err_req_param:
+	_free_phl_param(padapter, phl_param);
+_err_exit:
+	rtw_warn_on(1);
+	return res;
+}
+#endif /*CONFIG_IOCTL_CFG80211*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sdio.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sdio.c
new file mode 100644
index 000000000000..e049a86375f7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sdio.c
@@ -0,0 +1,203 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_SDIO_C_
+
+/*#include "drv_types_sdio.h"*/	/* RTW_SDIO_ADDR_CMD52_GEN */
+#include "drv_types.h"		/* drv_types_sdio.h, struct dvobj_priv and etc. */
+#include "sdio_ops.h"		/* rtw_sdio_raw_read(), rtw_sdio_raw_write() */
+
+/*
+ * Description:
+ *	Use SDIO cmd52 or cmd53 to read/write data
+ *
+ * Parameters:
+ *	d	pointer of device object(struct dvobj_priv)
+ *	addr	SDIO address, 17 bits
+ *	buf	buffer for I/O
+ *	len	length
+ *	write	0:read, 1:write
+ *	cmd52	0:cmd52, 1:cmd53
+ *
+ * Return:
+ *	_SUCCESS	I/O ok.
+ *	_FAIL		I/O fail.
+ */
+static u8 sdio_io(struct dvobj_priv *d, u32 addr, void *buf, size_t len, u8 write, u8 cmd52)
+{
+#ifdef DBG_SDIO
+#if (DBG_SDIO >= 3)
+	struct sdio_data *sdio = dvobj_to_sdio(d);
+#endif /* DBG_SDIO >= 3 */
+#endif /* DBG_SDIO */
+	u32 addr_drv;	/* address with driver defined bit */
+	int err;
+	u8 retry = 0;
+	u8 stop_retry = _FALSE;	/* flag for stopping retry or not */
+
+	if (dev_is_surprise_removed(d)) {
+		RTW_ERR("%s: bSurpriseRemoved, skip %s 0x%05x, %zu bytes\n",
+			__FUNCTION__, write?"write":"read", addr, len);
+		return _FAIL;
+	}
+
+	addr_drv = addr;
+	if (cmd52)
+		addr_drv = RTW_SDIO_ADDR_CMD52_GEN(addr_drv);
+
+	do {
+		if (write)
+			err = rtw_sdio_raw_write(d, addr_drv, buf, len, 0);
+		else
+			err = rtw_sdio_raw_read(d, addr_drv, buf, len, 0);
+		if (!err) {
+			if (retry) {
+				RTW_INFO("%s: Retry %s OK! addr=0x%05x %zu bytes, retry=%u,%u\n",
+					 __FUNCTION__, write?"write":"read",
+					 addr, len, retry, ATOMIC_READ(&d->continual_io_error));
+				RTW_INFO_DUMP("Data: ", buf, len);
+			}
+			rtw_reset_continual_io_error(d);
+			break;
+		}
+		RTW_ERR("%s: %s FAIL! error(%d) addr=0x%05x %zu bytes, retry=%u,%u\n",
+			__FUNCTION__, write?"write":"read", err, addr, len,
+			retry, ATOMIC_READ(&d->continual_io_error));
+
+#ifdef DBG_SDIO
+#if (DBG_SDIO >= 3)
+		if (sdio->dbg_enable) {
+			if (sdio->err_test && sdio->err_test_triggered)
+				sdio->err_test = 0;
+
+			if (sdio->err_stop) {
+				RTW_ERR("%s: I/O error! Set surprise remove flag ON!\n",
+					__FUNCTION__);
+				dev_set_surprise_removed(d);
+				return _FAIL;
+			}
+		}
+#endif /* DBG_SDIO >= 3 */
+#endif /* DBG_SDIO */
+
+		retry++;
+		stop_retry = rtw_inc_and_chk_continual_io_error(d);
+		if ((err == -1) || (stop_retry == _TRUE) || (retry > SD_IO_TRY_CNT)) {
+			/* critical error, unrecoverable */
+			RTW_ERR("%s: Fatal error! Set surprise remove flag ON! (retry=%u,%u)\n",
+				__FUNCTION__, retry, ATOMIC_READ(&d->continual_io_error));
+			dev_set_surprise_removed(d);
+			return _FAIL;
+		}
+
+		/* WLAN IOREG or SDIO Local */
+		if ((addr & 0x10000) || !(addr & 0xE000)) {
+			RTW_WARN("%s: Retry %s addr=0x%05x %zu bytes, retry=%u,%u\n",
+				 __FUNCTION__, write?"write":"read", addr, len,
+				 retry, ATOMIC_READ(&d->continual_io_error));
+			continue;
+		}
+		return _FAIL;
+	} while (1);
+
+	return _SUCCESS;
+}
+
+u8 rtw_sdio_read_cmd52(struct dvobj_priv *d, u32 addr, void *buf, size_t len)
+{
+	return sdio_io(d, addr, buf, len, 0, 1);
+}
+
+u8 rtw_sdio_read_cmd53(struct dvobj_priv *d, u32 addr, void *buf, size_t len)
+{
+	return sdio_io(d, addr, buf, len, 0, 0);
+}
+
+u8 rtw_sdio_write_cmd52(struct dvobj_priv *d, u32 addr, void *buf, size_t len)
+{
+	return sdio_io(d, addr, buf, len, 1, 1);
+}
+
+u8 rtw_sdio_write_cmd53(struct dvobj_priv *d, u32 addr, void *buf, size_t len)
+{
+	return sdio_io(d, addr, buf, len, 1, 0);
+}
+
+u8 rtw_sdio_f0_read(struct dvobj_priv *d, u32 addr, void *buf, size_t len)
+{
+	int err;
+	u8 ret;
+
+
+	ret = _SUCCESS;
+	addr = RTW_SDIO_ADDR_F0_GEN(addr);
+
+	err = rtw_sdio_raw_read(d, addr, buf, len, 0);
+	if (err)
+		ret = _FAIL;
+
+	return ret;
+}
+
+/**
+ * rtw_sdio_cmd53_align_size() - Align size to one CMD53 could complete
+ * @d		struct dvobj_priv*
+ * @len		length to align
+ *
+ * Adjust len to align block size, and the new size could be transfered by one
+ * CMD53.
+ * If len < block size, it would keep original value, otherwise the value
+ * would be rounded up by block size.
+ *
+ * Return adjusted length.
+ */
+size_t rtw_sdio_cmd53_align_size(struct dvobj_priv *d, size_t len)
+{
+	u32 blk_sz;
+
+
+	blk_sz = rtw_sdio_get_block_size(d);
+	if (len <= blk_sz)
+		return len;
+
+	return _RND(len, blk_sz);
+}
+
+static s32 sdio_init_xmit_priv(struct _ADAPTER *a)
+{
+	return _SUCCESS;
+}
+
+static void sdio_free_xmit_priv(_adapter *adapter)
+{
+}
+
+static s32 sdio_init_recv_priv(struct dvobj_priv *dvobj)
+{
+	return _SUCCESS;
+}
+
+static void sdio_free_recv_priv(struct dvobj_priv *dvobj)
+{
+}
+
+struct rtw_intf_ops sdio_ops = {
+	/****************** xmit *********************/
+	.init_xmit_priv = sdio_init_xmit_priv,
+	.free_xmit_priv = sdio_free_xmit_priv,
+
+	/******************  recv *********************/
+	.init_recv_priv = sdio_init_recv_priv,
+	.free_recv_priv = sdio_free_recv_priv,
+};
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sec_cam.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sec_cam.c
new file mode 100644
index 000000000000..a1f716001641
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sec_cam.c
@@ -0,0 +1,1129 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_SEC_CAM_C_
+
+#include <drv_types.h>
+#include <rtw_sec_cam.h>
+
+#if 0
+
+#define SEC_CAM_ENT_ID_TITLE_FMT "%-2s"
+#define SEC_CAM_ENT_ID_TITLE_ARG "id"
+#define SEC_CAM_ENT_ID_VALUE_FMT "%2u"
+#define SEC_CAM_ENT_ID_VALUE_ARG(id) (id)
+
+#define SEC_CAM_ENT_TITLE_FMT "%-6s %-17s %-32s %-3s %-8s %-2s %-2s %-5s"
+#define SEC_CAM_ENT_TITLE_ARG "ctrl", "addr", "key", "kid", "type", "MK", "GK", "valid"
+#define SEC_CAM_ENT_VALUE_FMT "0x%04x "MAC_FMT" "KEY_FMT" %3u %-8s %2u %2u %5u"
+#define SEC_CAM_ENT_VALUE_ARG(ent) \
+	(ent)->ctrl \
+	, MAC_ARG((ent)->mac) \
+	, KEY_ARG((ent)->key) \
+	, ((ent)->ctrl) & 0x03 \
+	, (((ent)->ctrl) & 0x200) ? \
+	security_type_str((((ent)->ctrl) >> 2 & 0x7) | _SEC_TYPE_256_) : \
+	security_type_str(((ent)->ctrl) >> 2 & 0x7) \
+	, (((ent)->ctrl) >> 5) & 0x01 \
+	, (((ent)->ctrl) >> 6) & 0x01 \
+	, (((ent)->ctrl) >> 15) & 0x01
+
+void dump_sec_cam_ent(void *sel, struct sec_cam_ent *ent, int id)
+{
+	if (id >= 0) {
+		RTW_PRINT_SEL(sel, SEC_CAM_ENT_ID_VALUE_FMT " " SEC_CAM_ENT_VALUE_FMT"\n"
+			, SEC_CAM_ENT_ID_VALUE_ARG(id), SEC_CAM_ENT_VALUE_ARG(ent));
+	} else
+		RTW_PRINT_SEL(sel, SEC_CAM_ENT_VALUE_FMT"\n", SEC_CAM_ENT_VALUE_ARG(ent));
+}
+
+void dump_sec_cam_ent_title(void *sel, u8 has_id)
+{
+	if (has_id) {
+		RTW_PRINT_SEL(sel, SEC_CAM_ENT_ID_TITLE_FMT " " SEC_CAM_ENT_TITLE_FMT"\n"
+			, SEC_CAM_ENT_ID_TITLE_ARG, SEC_CAM_ENT_TITLE_ARG);
+	} else
+		RTW_PRINT_SEL(sel, SEC_CAM_ENT_TITLE_FMT"\n", SEC_CAM_ENT_TITLE_ARG);
+}
+
+void dump_sec_cam(void *sel, _adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	struct sec_cam_ent ent;
+	int i;
+
+	RTW_PRINT_SEL(sel, "HW sec cam:\n");
+	dump_sec_cam_ent_title(sel, 1);
+	for (i = 0; i < cam_ctl->num; i++) {
+		rtw_hal_sec_read_cam_ent(adapter, i, (u8 *)(&ent.ctrl), ent.mac, ent.key);
+		dump_sec_cam_ent(sel , &ent, i);
+	}
+}
+
+void dump_sec_cam_cache(void *sel, _adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	int i;
+
+	RTW_PRINT_SEL(sel, "SW sec cam cache:\n");
+	dump_sec_cam_ent_title(sel, 1);
+	for (i = 0; i < cam_ctl->num; i++) {
+		if (dvobj->cam_cache[i].ctrl != 0)
+			dump_sec_cam_ent(sel, &dvobj->cam_cache[i], i);
+	}
+
+}
+
+void invalidate_cam_all(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	u8 val8 = 0;
+
+	rtw_hal_set_hwreg(padapter, HW_VAR_CAM_INVALID_ALL, &val8);
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	rtw_sec_cam_map_clr_all(&cam_ctl->used);
+	_rtw_memset(dvobj->cam_cache, 0, sizeof(struct sec_cam_ent) * SEC_CAM_ENT_NUM_SW_LIMIT);
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+}
+
+void _clear_cam_entry(_adapter *padapter, u8 entry)
+{
+	unsigned char null_sta[6] = {0};
+	unsigned char null_key[32] = {0};
+
+	rtw_hal_sec_write_cam_ent(padapter, entry, 0, null_sta, null_key);
+}
+
+inline void _write_cam(_adapter *adapter, u8 id, u16 ctrl, u8 *mac, u8 *key)
+{
+#ifdef CONFIG_WRITE_CACHE_ONLY
+	write_cam_cache(adapter, id , ctrl, mac, key);
+#else
+	rtw_hal_sec_write_cam_ent(adapter, id, ctrl, mac, key);
+	write_cam_cache(adapter, id , ctrl, mac, key);
+#endif
+}
+
+inline void write_cam(_adapter *adapter, u8 id, u16 ctrl, u8 *mac, u8 *key)
+{
+	if (ctrl & BIT(9)) {
+		_write_cam(adapter, id, ctrl, mac, key);
+		_write_cam(adapter, (id + 1), ctrl | BIT(5), mac, (key + 16));
+		RTW_INFO_DUMP("key-0: ", key, 16);
+		RTW_INFO_DUMP("key-1: ", (key + 16), 16);
+	} else
+		_write_cam(adapter, id, ctrl, mac, key);
+}
+
+inline void clear_cam_entry(_adapter *adapter, u8 id)
+{
+	_clear_cam_entry(adapter, id);
+	clear_cam_cache(adapter, id);
+}
+
+inline void write_cam_from_cache(_adapter *adapter, u8 id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	struct sec_cam_ent cache;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	_rtw_memcpy(&cache, &dvobj->cam_cache[id], sizeof(struct sec_cam_ent));
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	rtw_hal_sec_write_cam_ent(adapter, id, cache.ctrl, cache.mac, cache.key);
+}
+void write_cam_cache(_adapter *adapter, u8 id, u16 ctrl, u8 *mac, u8 *key)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+
+	dvobj->cam_cache[id].ctrl = ctrl;
+	_rtw_memcpy(dvobj->cam_cache[id].mac, mac, ETH_ALEN);
+	_rtw_memcpy(dvobj->cam_cache[id].key, key, 16);
+
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+}
+
+void clear_cam_cache(_adapter *adapter, u8 id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+
+	_rtw_memset(&(dvobj->cam_cache[id]), 0, sizeof(struct sec_cam_ent));
+
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+}
+
+inline bool _rtw_camctl_chk_cap(_adapter *adapter, u8 cap)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	if (cam_ctl->sec_cap & cap)
+		return _TRUE;
+	return _FALSE;
+}
+
+inline void _rtw_camctl_set_flags(_adapter *adapter, u32 flags)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	cam_ctl->flags |= flags;
+}
+
+inline void rtw_camctl_set_flags(_adapter *adapter, u32 flags)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	_rtw_camctl_set_flags(adapter, flags);
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+}
+
+inline void _rtw_camctl_clr_flags(_adapter *adapter, u32 flags)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	cam_ctl->flags &= ~flags;
+}
+
+inline void rtw_camctl_clr_flags(_adapter *adapter, u32 flags)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	_rtw_camctl_clr_flags(adapter, flags);
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+}
+
+inline bool _rtw_camctl_chk_flags(_adapter *adapter, u32 flags)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	if (cam_ctl->flags & flags)
+		return _TRUE;
+	return _FALSE;
+}
+
+void dump_sec_cam_map(void *sel, struct sec_cam_bmp *map, u8 max_num)
+{
+	RTW_PRINT_SEL(sel, "0x%08x\n", map->m0);
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	if (max_num && max_num > 32)
+		RTW_PRINT_SEL(sel, "0x%08x\n", map->m1);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	if (max_num && max_num > 64)
+		RTW_PRINT_SEL(sel, "0x%08x\n", map->m2);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	if (max_num && max_num > 96)
+		RTW_PRINT_SEL(sel, "0x%08x\n", map->m3);
+#endif
+}
+
+inline bool rtw_sec_camid_is_set(struct sec_cam_bmp *map, u8 id)
+{
+	if (id < 32)
+		return map->m0 & BIT(id);
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	else if (id < 64)
+		return map->m1 & BIT(id - 32);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	else if (id < 96)
+		return map->m2 & BIT(id - 64);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	else if (id < 128)
+		return map->m3 & BIT(id - 96);
+#endif
+	else
+		rtw_warn_on(1);
+
+	return 0;
+}
+
+inline void rtw_sec_cam_map_set(struct sec_cam_bmp *map, u8 id)
+{
+	if (id < 32)
+		map->m0 |= BIT(id);
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	else if (id < 64)
+		map->m1 |= BIT(id - 32);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	else if (id < 96)
+		map->m2 |= BIT(id - 64);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	else if (id < 128)
+		map->m3 |= BIT(id - 96);
+#endif
+	else
+		rtw_warn_on(1);
+}
+
+inline void rtw_sec_cam_map_clr(struct sec_cam_bmp *map, u8 id)
+{
+	if (id < 32)
+		map->m0 &= ~BIT(id);
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	else if (id < 64)
+		map->m1 &= ~BIT(id - 32);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	else if (id < 96)
+		map->m2 &= ~BIT(id - 64);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	else if (id < 128)
+		map->m3 &= ~BIT(id - 96);
+#endif
+	else
+		rtw_warn_on(1);
+}
+
+inline void rtw_sec_cam_map_clr_all(struct sec_cam_bmp *map)
+{
+	map->m0 = 0;
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	map->m1 = 0;
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	map->m2 = 0;
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	map->m3 = 0;
+#endif
+}
+
+inline bool rtw_sec_camid_is_drv_forbid(struct cam_ctl_t *cam_ctl, u8 id)
+{
+	struct sec_cam_bmp forbid_map;
+
+	forbid_map.m0 = 0x00000ff0;
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	forbid_map.m1 = 0x00000000;
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	forbid_map.m2 = 0x00000000;
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	forbid_map.m3 = 0x00000000;
+#endif
+
+	if (id < 32)
+		return forbid_map.m0 & BIT(id);
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	else if (id < 64)
+		return forbid_map.m1 & BIT(id - 32);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	else if (id < 96)
+		return forbid_map.m2 & BIT(id - 64);
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	else if (id < 128)
+		return forbid_map.m3 & BIT(id - 96);
+#endif
+	else
+		rtw_warn_on(1);
+
+	return 1;
+}
+
+bool _rtw_sec_camid_is_used(struct cam_ctl_t *cam_ctl, u8 id)
+{
+	bool ret = _FALSE;
+
+	if (id >= cam_ctl->num) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+#if 0 /* for testing */
+	if (rtw_sec_camid_is_drv_forbid(cam_ctl, id)) {
+		ret = _TRUE;
+		goto exit;
+	}
+#endif
+
+	ret = rtw_sec_camid_is_set(&cam_ctl->used, id);
+
+exit:
+	return ret;
+}
+
+inline bool rtw_sec_camid_is_used(struct cam_ctl_t *cam_ctl, u8 id)
+{
+	bool ret;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	ret = _rtw_sec_camid_is_used(cam_ctl, id);
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	return ret;
+}
+u8 rtw_get_sec_camid(_adapter *adapter, u8 max_bk_key_num, u8 *sec_key_id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	int i;
+
+	u8 sec_cam_num = 0;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	for (i = 0; i < cam_ctl->num; i++) {
+		if (_rtw_sec_camid_is_used(cam_ctl, i)) {
+			sec_key_id[sec_cam_num++] = i;
+			if (sec_cam_num == max_bk_key_num)
+				break;
+		}
+	}
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	return sec_cam_num;
+}
+
+inline bool _rtw_camid_is_gk(_adapter *adapter, u8 cam_id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	bool ret = _FALSE;
+
+	if (cam_id >= cam_ctl->num) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (_rtw_sec_camid_is_used(cam_ctl, cam_id) == _FALSE)
+		goto exit;
+
+	ret = (dvobj->cam_cache[cam_id].ctrl & BIT6) ? _TRUE : _FALSE;
+
+exit:
+	return ret;
+}
+
+inline bool rtw_camid_is_gk(_adapter *adapter, u8 cam_id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	bool ret;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	ret = _rtw_camid_is_gk(adapter, cam_id);
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	return ret;
+}
+
+bool cam_cache_chk(_adapter *adapter, u8 id, u8 *addr, s16 kid, s8 gk)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	bool ret = _FALSE;
+
+	if (addr && _rtw_memcmp(dvobj->cam_cache[id].mac, addr, ETH_ALEN) == _FALSE)
+		goto exit;
+	if (kid >= 0 && kid != (dvobj->cam_cache[id].ctrl & 0x03))
+		goto exit;
+	if (gk != -1 && (gk ? _TRUE : _FALSE) != _rtw_camid_is_gk(adapter, id))
+		goto exit;
+
+	ret = _TRUE;
+
+exit:
+	return ret;
+}
+
+s16 _rtw_camid_search(_adapter *adapter, u8 *addr, s16 kid, s8 gk)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	int i;
+	s16 cam_id = -1;
+
+	for (i = 0; i < cam_ctl->num; i++) {
+		if (cam_cache_chk(adapter, i, addr, kid, gk)) {
+			cam_id = i;
+			break;
+		}
+	}
+
+	if (0) {
+		if (addr)
+			RTW_INFO(FUNC_ADPT_FMT" addr:"MAC_FMT" kid:%d, gk:%d, return cam_id:%d\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(addr), kid, gk, cam_id);
+		else
+			RTW_INFO(FUNC_ADPT_FMT" addr:%p kid:%d, gk:%d, return cam_id:%d\n"
+				, FUNC_ADPT_ARG(adapter), addr, kid, gk, cam_id);
+	}
+
+	return cam_id;
+}
+
+s16 rtw_camid_search(_adapter *adapter, u8 *addr, s16 kid, s8 gk)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	s16 cam_id = -1;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	cam_id = _rtw_camid_search(adapter, addr, kid, gk);
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	return cam_id;
+}
+
+s16 rtw_get_camid(_adapter *adapter, u8 *addr, s16 kid, u8 gk, bool ext_sec)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	int i;
+#if 0 /* for testing */
+	static u8 start_id = 0;
+#else
+	u8 start_id = 0;
+#endif
+	s16 cam_id = -1;
+
+	if (addr == NULL) {
+		RTW_PRINT(FUNC_ADPT_FMT" mac_address is NULL\n"
+			  , FUNC_ADPT_ARG(adapter));
+		rtw_warn_on(1);
+		goto _exit;
+	}
+
+	/* find cam entry which has the same addr, kid (, gk bit) */
+	if (_rtw_camctl_chk_cap(adapter, SEC_CAP_CHK_BMC) == _TRUE)
+		i = _rtw_camid_search(adapter, addr, kid, gk);
+	else
+		i = _rtw_camid_search(adapter, addr, kid, -1);
+
+	if (i >= 0) {
+		cam_id = i;
+		goto _exit;
+	}
+
+	for (i = 0; i < cam_ctl->num; i++) {
+		/* bypass default key which is allocated statically */
+#if 0//ndef CONFIG_CONCURRENT_MODE
+		if (((i + start_id) % cam_ctl->num) < 4)
+			continue;
+#endif
+		if (_rtw_sec_camid_is_used(cam_ctl, ((i + start_id) % cam_ctl->num)) == _FALSE) {
+			if (ext_sec) {
+				/* look out continue slot */
+				if (((i + 1) < cam_ctl->num) &&
+					(_rtw_sec_camid_is_used(cam_ctl, (((i + 1) + start_id) % cam_ctl->num)) == _FALSE))
+			break;
+				else
+					continue;
+			} else
+				break;
+		}
+	}
+
+	if (i == cam_ctl->num) {
+		RTW_PRINT(FUNC_ADPT_FMT" %s key with "MAC_FMT" id:%u no room\n"
+			, FUNC_ADPT_ARG(adapter), gk ? "group" : "pairwise", MAC_ARG(addr), kid);
+		rtw_warn_on(1);
+		goto _exit;
+	}
+
+	cam_id = ((i + start_id) % cam_ctl->num);
+	start_id = ((i + start_id + 1) % cam_ctl->num);
+
+_exit:
+	return cam_id;
+}
+
+s16 rtw_camid_alloc(_adapter *adapter, struct sta_info *sta, u8 kid, u8 gk, bool ext_sec, bool *used)
+{
+	struct mlme_ext_info *mlmeinfo = &adapter->mlmeextpriv.mlmext_info;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	s16 cam_id = -1;
+
+	*used = _FALSE;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+
+	if ((((mlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) || ((mlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE))
+	    && !sta) {
+		/*
+		* 1. non-STA mode WEP key
+		* 2. group TX key
+		*/
+#if 0//ndef CONFIG_CONCURRENT_MODE
+		/* static alloction to default key by key ID when concurrent is not defined */
+		if (kid > 3) {
+			RTW_PRINT(FUNC_ADPT_FMT" group key with invalid key id:%u\n"
+				  , FUNC_ADPT_ARG(adapter), kid);
+			rtw_warn_on(1);
+			goto bitmap_handle;
+		}
+		cam_id = kid;
+#else
+		u8 *addr = adapter_mac_addr(adapter);
+
+		cam_id = rtw_get_camid(adapter, addr, kid, gk, ext_sec);
+		if (1)
+			RTW_PRINT(FUNC_ADPT_FMT" group key with "MAC_FMT" assigned cam_id:%u\n"
+				, FUNC_ADPT_ARG(adapter), MAC_ARG(addr), cam_id);
+#endif
+	} else {
+		/*
+		* 1. STA mode WEP key
+		* 2. STA mode group RX key
+		* 3. sta key (pairwise, group RX)
+		*/
+		u8 *addr = sta ? sta->phl_sta->mac_addr : NULL;
+
+		if (!sta) {
+			if (!(mlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)) {
+				/* bypass STA mode group key setting before connected(ex:WEP) because bssid is not ready */
+				goto bitmap_handle;
+			}
+			addr = get_bssid(&adapter->mlmepriv);/*A2*/
+		}
+		cam_id = rtw_get_camid(adapter, addr, kid, gk, ext_sec);
+	}
+
+
+bitmap_handle:
+	if (cam_id >= 0) {
+		*used = _rtw_sec_camid_is_used(cam_ctl, cam_id);
+		rtw_sec_cam_map_set(&cam_ctl->used, cam_id);
+		if (ext_sec)
+			rtw_sec_cam_map_set(&cam_ctl->used, cam_id + 1);
+	}
+
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	return cam_id;
+}
+
+void rtw_camid_set(_adapter *adapter, u8 cam_id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+
+	if (cam_id < cam_ctl->num)
+		rtw_sec_cam_map_set(&cam_ctl->used, cam_id);
+
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+}
+
+void rtw_camid_free(_adapter *adapter, u8 cam_id)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+
+	if (cam_id < cam_ctl->num)
+		rtw_sec_cam_map_clr(&cam_ctl->used, cam_id);
+
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+}
+
+/*Must pause TX/RX before use this API*/
+inline void rtw_sec_cam_swap(_adapter *adapter, u8 cam_id_a, u8 cam_id_b)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	struct sec_cam_ent cache_a, cache_b;
+	bool cam_a_used, cam_b_used;
+
+	if (1)
+		RTW_INFO(ADPT_FMT" - sec_cam %d,%d swap\n", ADPT_ARG(adapter), cam_id_a, cam_id_b);
+
+	if (cam_id_a == cam_id_b)
+		return;
+
+	/*setp-1. backup org cam_info*/
+	_rtw_spinlock_bh(&cam_ctl->lock);
+
+	cam_a_used = _rtw_sec_camid_is_used(cam_ctl, cam_id_a);
+	cam_b_used = _rtw_sec_camid_is_used(cam_ctl, cam_id_b);
+
+	if (cam_a_used)
+		_rtw_memcpy(&cache_a, &dvobj->cam_cache[cam_id_a], sizeof(struct sec_cam_ent));
+
+	if (cam_b_used)
+		_rtw_memcpy(&cache_b, &dvobj->cam_cache[cam_id_b], sizeof(struct sec_cam_ent));
+
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	/*setp-2. clean cam_info*/
+	if (cam_a_used) {
+		rtw_camid_free(adapter, cam_id_a);
+		clear_cam_entry(adapter, cam_id_a);
+	}
+	if (cam_b_used) {
+		rtw_camid_free(adapter, cam_id_b);
+		clear_cam_entry(adapter, cam_id_b);
+	}
+
+	/*setp-3. set cam_info*/
+	if (cam_a_used) {
+		write_cam(adapter, cam_id_b, cache_a.ctrl, cache_a.mac, cache_a.key);
+		rtw_camid_set(adapter, cam_id_b);
+	}
+
+	if (cam_b_used) {
+		write_cam(adapter, cam_id_a, cache_b.ctrl, cache_b.mac, cache_b.key);
+		rtw_camid_set(adapter, cam_id_a);
+	}
+}
+
+s16 rtw_get_empty_cam_entry(_adapter *adapter, u8 start_camid)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	int i;
+	s16 cam_id = -1;
+
+	_rtw_spinlock_bh(&cam_ctl->lock);
+	for (i = start_camid; i < cam_ctl->num; i++) {
+		if (_FALSE == _rtw_sec_camid_is_used(cam_ctl, i)) {
+			cam_id = i;
+			break;
+		}
+	}
+	_rtw_spinunlock_bh(&cam_ctl->lock);
+
+	return cam_id;
+}
+void rtw_clean_dk_section(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = dvobj_to_sec_camctl(dvobj);
+	s16 ept_cam_id;
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		if (rtw_sec_camid_is_used(cam_ctl, i)) {
+			ept_cam_id = rtw_get_empty_cam_entry(adapter, 4);
+			if (ept_cam_id > 0)
+				rtw_sec_cam_swap(adapter, i, ept_cam_id);
+		}
+	}
+}
+void rtw_clean_hw_dk_cam(_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < 4; i++)
+		rtw_hal_sec_clr_cam_ent(adapter, i);
+		/*_clear_cam_entry(adapter, i);*/
+}
+#endif
+void flush_all_cam_entry(struct _ADAPTER *a, enum phl_cmd_type cmd_type, u32 cmd_timeout)
+{
+	struct sta_priv	*stapriv = &a->stapriv;
+	u8 *mac;
+	struct sta_info	*sta;
+	struct _ADAPTER_LINK *alink;
+	u8 lidx;
+
+
+	if (MLME_IS_STA(a)) {
+		for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+			alink = GET_LINK(a, lidx);
+			mac = alink->mlmeextpriv.mlmext_info.network.MacAddress;
+			sta = rtw_get_stainfo(stapriv, mac);
+			if (sta) {
+				if (sta->state & WIFI_AP_STATE)
+					/*clear cam when ap free per sta_info*/
+					RTW_INFO("%s: sta->state(0x%x) is AP, "
+					 	"do nothing\n",
+						 __func__, sta->state);
+				else
+					rtw_hw_del_all_key(a, sta, cmd_type, cmd_timeout);
+			} else {
+				RTW_WARN("%s: cann't find sta for " MAC_FMT "\n", __func__, MAC_ARG(mac));
+				rtw_warn_on(1);
+			}
+		}
+	} else if (MLME_IS_AP(a) || MLME_IS_MESH(a)) {
+		for (lidx = 0; lidx < a->adapter_link_num; lidx++) {
+			alink = GET_LINK(a, lidx);
+			mac = alink->mac_addr;
+			sta = rtw_get_stainfo(stapriv, mac);
+			if (sta)
+				rtw_hw_del_all_key(a, sta, cmd_type, cmd_timeout);
+		}
+	}
+}
+
+#ifdef  CONFIG_DBG_AX_CAM
+/*
+1. Mode 0 : no key
+2. Mode 1 : unicast	0 - 6
+3. Mode 2 : unicast	0 - 1	Group: 2 - 4	BIP: 5 - 6
+4. Mode 3 : unicast	0 - 1	Group: 2 - 5	BIP: 6
+*/
+static enum CAM_KEY_TYPE search_keytype_by_mode(u8 mode, u8 entry_num)
+{
+	enum CAM_KEY_TYPE key_type = KEY_TYPE_NONE;
+
+	switch (mode) {
+	case 0:
+		key_type = KEY_TYPE_NONE;
+		break;
+	case 1:
+		key_type = KEY_TYPE_UNI;
+		break;
+	case 2:
+		if (entry_num < 2)
+			key_type = KEY_TYPE_UNI;
+		else if (entry_num < 5)
+			key_type = KEY_TYPE_GROUP;
+		else if (entry_num < 7)
+			key_type = KEY_TYPE_BIP;
+		else
+			key_type = KEY_TYPE_NONE;
+		break;
+	case 3:
+		if (entry_num < 2)
+			key_type = KEY_TYPE_UNI;
+		else if (entry_num < 6)
+			key_type = KEY_TYPE_GROUP;
+		else if (entry_num < 7)
+			key_type = KEY_TYPE_BIP;
+		else
+			key_type = KEY_TYPE_NONE;
+		break;
+	default:
+		key_type = KEY_TYPE_NONE;
+		break;
+	}
+	return key_type;
+}
+
+static const char* enc_algo_to_string[] = {
+	"RTW_ENC_NONE",
+	"RTW_ENC_WEP40",
+	"RTW_ENC_WEP104",
+	"RTW_ENC_TKIP",
+	"RTW_ENC_WAPI",
+	"RTW_ENC_GCMSMS4",
+	"RTW_ENC_CCMP",
+	"RTW_ENC_CCMP256",
+	"RTW_ENC_GCMP",
+	"RTW_ENC_GCMP256",
+	"RTW_ENC_BIP_CCMP128",
+	"RTW_ENC_MAX"
+};
+
+static const char* type_to_string[] = {
+	"unicast",
+	"multicast",
+	"BIP",
+	"NONE"
+};
+
+static const char* nettype_to_string[] = {
+	"NoLink",
+	"Ad-hoc",
+	"Infra",
+	"AP"
+};
+
+static u8 get_keyid(u8 num, u8* addr_cam)
+{
+	u8 keyid = 0;
+
+	switch (num) {
+	case 0:
+		keyid = GET_AX_ADDR_CAM_SEC_ENT0_KEYID(addr_cam);
+		break;
+	case 1:
+		keyid = GET_AX_ADDR_CAM_SEC_ENT1_KEYID(addr_cam);
+		break;
+	case 2:
+		keyid = GET_AX_ADDR_CAM_SEC_ENT2_KEYID(addr_cam);
+		break;
+	case 3:
+		keyid = GET_AX_ADDR_CAM_SEC_ENT3_KEYID(addr_cam);
+		break;
+	case 4:
+		keyid = GET_AX_ADDR_CAM_SEC_ENT4_KEYID(addr_cam);
+		break;
+	case 5:
+		keyid = GET_AX_ADDR_CAM_SEC_ENT5_KEYID(addr_cam);
+		break;
+	case 6:
+		keyid = GET_AX_ADDR_CAM_SEC_ENT6_KEYID(addr_cam);
+		break;
+	default:
+		keyid = 0;
+		break;
+	}
+	return keyid;
+}
+
+static u8 get_sec_entry(u8 num, u8* addr_cam)
+{
+	u8 entry = 0;
+
+	switch (num) {
+	case 0:
+		entry = GET_AX_ADDR_CAM_SEC_ENT0(addr_cam);
+		break;
+	case 1:
+		entry = GET_AX_ADDR_CAM_SEC_ENT1(addr_cam);
+		break;
+	case 2:
+		entry = GET_AX_ADDR_CAM_SEC_ENT2(addr_cam);
+		break;
+	case 3:
+		entry = GET_AX_ADDR_CAM_SEC_ENT3(addr_cam);
+		break;
+	case 4:
+		entry = GET_AX_ADDR_CAM_SEC_ENT4(addr_cam);
+		break;
+	case 5:
+		entry = GET_AX_ADDR_CAM_SEC_ENT5(addr_cam);
+		break;
+	case 6:
+		entry = GET_AX_ADDR_CAM_SEC_ENT6(addr_cam);
+		break;
+	default:
+		entry = 0;
+		break;
+	}
+	return entry;
+}
+
+static void dump_cam_info(void* sel, u8 *buf, u32 start, u32 end)
+{
+	int i, j = 1;
+
+	for (i = start; i < end; i += 1) {
+		if (j % 4 == 1)
+			RTW_PRINT_SEL(sel,"0x%04x", i);
+		RTW_PRINT_SEL(sel," 0x%08x ", *(u32 *)(buf+ (i*4)));
+		if ((j++) % 4 == 0)
+			RTW_PRINT_SEL(sel,"\n");
+	}
+}
+
+static void search_sec_cam_by_entry(struct _ADAPTER *a, u8 num, u8* target_entry)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	void *phl;
+	u32 sec_tmp_buf = 0;
+	int i;
+
+	phl = GET_PHL_INFO(dvobj);
+	rtw_phl_write32(phl, INDIRECT_ACCESS_ADDR, (SEC_CAM_BASE_ADDR + num*32));
+
+	for (i = 0; i < 5; i++) {
+		sec_tmp_buf = rtw_phl_read32(phl,INDIRECT_ACCESS_VALUE + i*4);
+		_rtw_memcpy((target_entry + i*4), &sec_tmp_buf, 4);
+	}
+}
+
+static void search_bssid_cam_by_entry(struct _ADAPTER *a, u8 num, u8* target_entry)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	void *phl;
+	u32 BSSID_tmp_buf = 0;
+	int i;
+
+	phl = GET_PHL_INFO(dvobj);
+	rtw_phl_write32(phl, INDIRECT_ACCESS_ADDR, (BSSID_CAM_BASE_ADDR+ num*8));
+
+	for (i = 0 ; i < 2; i++) {
+		BSSID_tmp_buf = rtw_phl_read32(phl,INDIRECT_ACCESS_VALUE + i*4);
+		_rtw_memcpy((target_entry + i*4), &BSSID_tmp_buf, 4);
+	}
+}
+
+static u8 varify_256_len(u8 enc)
+{
+	u8 hit = _FALSE;
+
+	if (enc == 4|| enc == 7|| enc == 9)
+		hit = _TRUE;
+	return  hit;
+}
+
+int get_ax_address_cam(void* sel, struct _ADAPTER *a)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	void *phl;
+	/*addr cam: 128 entries*/
+	int loop_num = 128;
+	/*addr cam: 11 DW but the last one all rsvd*/
+	u32 addr_tmp_buf = 0;
+	u8  addr_map[40] = {0};
+	int i, j;
+
+	phl = GET_PHL_INFO(dvobj);
+
+	for (i = 0; i < loop_num; i++) {
+		_rtw_memset(addr_map,0,sizeof(addr_map));
+		rtw_phl_write32(phl, INDIRECT_ACCESS_ADDR, (ADDR_CAM_BASE_ADDR + i*64));
+
+		for (j = 0; j < 10; j++) {
+			addr_tmp_buf = rtw_phl_read32(phl,INDIRECT_ACCESS_VALUE + j*4);
+			_rtw_memcpy(&addr_map[j*4], &addr_tmp_buf, 4);
+		}
+		RTW_PRINT_SEL(sel, "======= ADDR CAM (%d)DUMP  =======\n", i);
+		dump_cam_info(sel,addr_map,0,10);
+		RTW_PRINT_SEL(sel, "\n");
+	}
+	return 0;
+}
+
+int get_ax_sec_cam(void* sel, struct _ADAPTER *a)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	void *phl;
+	/*security cam: 128 entries*/
+	int loop_num = 128;
+	/*security cam: 5 DW*/
+	u32 sec_tmp_buf = 0;
+	u8  sec_map[20] = {0};
+	int i, j;
+
+	phl = GET_PHL_INFO(dvobj);
+
+	for (i = 0; i < loop_num; i++) {
+		_rtw_memset(sec_map,0,sizeof(sec_map));
+		rtw_phl_write32(phl, INDIRECT_ACCESS_ADDR, (SEC_CAM_BASE_ADDR + i*32));
+
+		for (j = 0; j < 5; j++) {
+			sec_tmp_buf = rtw_phl_read32(phl,INDIRECT_ACCESS_VALUE + j*4);
+			_rtw_memcpy(&sec_map[j*4], &sec_tmp_buf, 4);
+		}
+
+		RTW_PRINT_SEL(sel, "======= sec CAM (%d)DUMP  =======\n", i);
+		dump_cam_info(sel, sec_map, 0, 5);
+		RTW_PRINT_SEL(sel, "\n");
+	}
+	return 0;
+}
+
+static void dump_valid_key(void* sel, u8* addr_map, u8* sec_map, u8* bssid_map
+		, u8 sec_entry, u8 key_type, u8 keyid)
+{
+	u8 SMA[ETH_ALEN] = {0}, TMA[ETH_ALEN] = {0};
+	u8 macid = 0, nettype = 0;
+	char mac_addr_str_SMA[MAC_FMT_LEN];
+	char mac_addr_str_TMA[MAC_FMT_LEN];
+	char mac_addr_str[MAC_FMT_LEN];
+
+	macid = GET_AX_ADDR_CAM_MACID(addr_map);
+	nettype = GET_AX_ADDR_CAM_NET_TYPE(addr_map);
+
+	_rtw_memcpy(SMA, &addr_map[8], ETH_ALEN);
+	_rtw_memcpy(TMA,&addr_map[14], ETH_ALEN);
+
+	RTW_PRINT_SEL(sel, "%-5u %s %s  %s %s %-5u %-5u %s %-7u %-3u %s "KEY_FMT"\n"
+		, macid, nettype_to_string[nettype]
+		, get_macaddr_str(mac_addr_str_SMA, sel, SMA)
+		, get_macaddr_str(mac_addr_str_TMA, sel, TMA)
+		, get_macaddr_str(mac_addr_str, sel, &bssid_map[2])
+		, keyid, sec_entry
+		, enc_algo_to_string[GET_AX_SEC_CAM_TYPE(sec_map)]
+		, GET_AX_SEC_CAM_EXT_KEY(sec_map), GET_AX_SEC_SPP_MODE_(sec_map)
+		, type_to_string[key_type], KEY_ARG(&sec_map[4]));
+}
+
+static void prepare_to_dump_valid_key(void* sel, struct _ADAPTER *a, u8* addr_map, u8 ent)
+{
+	u8 sec_map[20] = {0};
+	u8 keyid = 0, sec_entry = 0, key_type = KEY_TYPE_NONE;
+	u8 bssid_entry;
+	u8 bssid_map[8] = {0};
+	u8 enc_algor = 0;
+
+	key_type = search_keytype_by_mode(GET_AX_ADDR_CAM_SEC_ENT_MODE(addr_map) ,ent);
+	sec_entry = get_sec_entry(ent, addr_map);
+	bssid_entry = GET_AX_ADDR_CAM_BSSID_CAM_IDX(addr_map);
+	keyid = get_keyid(ent,addr_map);
+	search_sec_cam_by_entry(a, sec_entry, sec_map);
+	search_bssid_cam_by_entry(a, bssid_entry, bssid_map);
+	enc_algor = GET_AX_SEC_CAM_TYPE(sec_map);
+
+	if (varify_256_len(enc_algor) == _TRUE) {
+		dump_valid_key(sel, addr_map, sec_map, bssid_map, sec_entry, key_type, keyid);
+		search_sec_cam_by_entry(a, sec_entry + 1, sec_map);
+		dump_valid_key(sel, addr_map, sec_map, bssid_map, sec_entry + 1, key_type, keyid);
+	} else {
+		dump_valid_key(sel, addr_map, sec_map, bssid_map, sec_entry, key_type, keyid);
+	}
+}
+
+int get_ax_valid_key(void* sel, struct _ADAPTER *a)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(a);
+	void *phl;
+	/*addr cam: 128 entries*/
+	int loop_num = 128;
+	/*addr cam: 11 DW but the last one all rsvd*/
+	u32 addr_tmp_buf = 0;
+	u8 addr_map[40]= {0};
+	u8 sec_ent_valid = 0;
+	u8 macid = 0;
+	int i, j;
+
+	phl = GET_PHL_INFO(dvobj);
+
+	RTW_PRINT_SEL(sel, "Dump valid KEY\n");
+	RTW_PRINT_SEL(sel, "%-5s %-7s %-17s %-17s %-17s %-5s %-5s %-8s %-3s"\
+		"  %-3s  %s  %s\n", "macid", "NETTYPE", "SMA", "TMA", "BSSID", "keyid"\
+		, "sec_entry", "enc_algor", "ext_key", "ssp", "key_type", "key");
+
+	for (i = 0; i < loop_num; i++) {
+		_rtw_memset(addr_map,0,sizeof(addr_map));
+		rtw_phl_write32(phl, INDIRECT_ACCESS_ADDR, (ADDR_CAM_BASE_ADDR + i*64));
+
+		for (j = 0; j < 10; j++) {
+			addr_tmp_buf = rtw_phl_read32(phl,INDIRECT_ACCESS_VALUE + j*4);
+			_rtw_memcpy(&addr_map[j*4], &addr_tmp_buf, 4);
+		}
+		sec_ent_valid = GET_AX_ADDR_CAM_SEC_ENT_VALID(addr_map);
+
+		for (j = 0 ;j< 7; j++) {
+			if (sec_ent_valid & BIT(j))
+				prepare_to_dump_valid_key(sel, a, addr_map, j);
+		}
+	}
+	return 0;
+}
+#endif /* CONFIG_DBG_AX_CAM */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_security.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_security.c
new file mode 100644
index 000000000000..cea3208b32f1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_security.c
@@ -0,0 +1,2787 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define  _RTW_SECURITY_C_
+
+#include <drv_types.h>
+#include <rtw_swcrypto.h>
+
+static const char *_security_type_str[] = {
+	"N/A",
+	"WEP40",
+	"TKIP",
+	"TKIP_WM",
+	"AES",
+	"WEP104",
+	"SMS4",
+	"GCMP",
+};
+
+static const char *_security_type_bip_str[] = {
+	"BIP_CMAC_128",
+	"BIP_GMAC_128",
+	"BIP_GMAC_256",
+	"BIP_CMAC_256",
+};
+
+const char *security_type_str(u8 value)
+{
+#ifdef CONFIG_IEEE80211W
+	if ((_BIP_MAX_ > value) && (value >= _BIP_CMAC_128_))
+		return _security_type_bip_str[value & ~_SEC_TYPE_BIT_];
+#endif
+
+	if (_CCMP_256_ == value)
+		return "CCMP_256";
+	if (_GCMP_256_ == value)
+		return "GCMP_256";
+
+	if (_SEC_TYPE_MAX_ > value)
+		return _security_type_str[value];
+
+	return NULL;
+}
+
+#ifdef CONFIG_IEEE80211W
+u32 security_type_bip_to_gmcs(enum security_type type)
+{
+	switch (type) {
+	case _BIP_CMAC_128_:
+		return WPA_CIPHER_BIP_CMAC_128;
+	case _BIP_GMAC_128_:
+		return WPA_CIPHER_BIP_GMAC_128;
+	case _BIP_GMAC_256_:
+		return WPA_CIPHER_BIP_GMAC_256;
+	case _BIP_CMAC_256_:
+		return WPA_CIPHER_BIP_CMAC_256;
+	default:
+		return 0;
+	}
+}
+#endif
+
+#ifdef DBG_SW_SEC_CNT
+#define WEP_SW_ENC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->wep_sw_enc_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->wep_sw_enc_cnt_mc++; \
+	else \
+		sec->wep_sw_enc_cnt_uc++; \
+	} while (0)
+
+#define WEP_SW_DEC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->wep_sw_dec_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->wep_sw_dec_cnt_mc++; \
+	else \
+		sec->wep_sw_dec_cnt_uc++; \
+	} while (0)
+
+#define TKIP_SW_ENC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->tkip_sw_enc_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->tkip_sw_enc_cnt_mc++; \
+	else \
+		sec->tkip_sw_enc_cnt_uc++; \
+	} while (0)
+
+#define TKIP_SW_DEC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->tkip_sw_dec_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->tkip_sw_dec_cnt_mc++; \
+	else \
+		sec->tkip_sw_dec_cnt_uc++; \
+	} while (0)
+
+#define AES_SW_ENC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->aes_sw_enc_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->aes_sw_enc_cnt_mc++; \
+	else \
+		sec->aes_sw_enc_cnt_uc++; \
+	} while (0)
+
+#define AES_SW_DEC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->aes_sw_dec_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->aes_sw_dec_cnt_mc++; \
+	else \
+		sec->aes_sw_dec_cnt_uc++; \
+	} while (0)
+
+#define GCMP_SW_ENC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->gcmp_sw_enc_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->gcmp_sw_enc_cnt_mc++; \
+	else \
+		sec->gcmp_sw_enc_cnt_uc++; \
+	} while (0)
+
+#define GCMP_SW_DEC_CNT_INC(sec, ra) do {\
+	if (is_broadcast_mac_addr(ra)) \
+		sec->gcmp_sw_dec_cnt_bc++; \
+	else if (is_multicast_mac_addr(ra)) \
+		sec->gcmp_sw_dec_cnt_mc++; \
+	else \
+		sec->gcmp_sw_dec_cnt_uc++; \
+	} while (0)
+#else
+#define WEP_SW_ENC_CNT_INC(sec, ra)
+#define WEP_SW_DEC_CNT_INC(sec, ra)
+#define TKIP_SW_ENC_CNT_INC(sec, ra)
+#define TKIP_SW_DEC_CNT_INC(sec, ra)
+#define AES_SW_ENC_CNT_INC(sec, ra)
+#define AES_SW_DEC_CNT_INC(sec, ra)
+#define GCMP_SW_ENC_CNT_INC(sec, ra)
+#define GCMP_SW_DEC_CNT_INC(sec, ra)
+#endif /* DBG_SW_SEC_CNT */
+
+/* *****WEP related***** */
+
+#define CRC32_POLY 0x04c11db7
+
+struct arc4context {
+	u32 x;
+	u32 y;
+	u8 state[256];
+};
+
+
+static void arcfour_init(struct arc4context	*parc4ctx, u8 *key, u32	key_len)
+{
+	u32	t, u;
+	u32	keyindex;
+	u32	stateindex;
+	u8 *state;
+	u32	counter;
+	state = parc4ctx->state;
+	parc4ctx->x = 0;
+	parc4ctx->y = 0;
+	for (counter = 0; counter < 256; counter++)
+		state[counter] = (u8)counter;
+	keyindex = 0;
+	stateindex = 0;
+	for (counter = 0; counter < 256; counter++) {
+		t = state[counter];
+		stateindex = (stateindex + key[keyindex] + t) & 0xff;
+		u = state[stateindex];
+		state[stateindex] = (u8)t;
+		state[counter] = (u8)u;
+		if (++keyindex >= key_len)
+			keyindex = 0;
+	}
+}
+static u32 arcfour_byte(struct arc4context	*parc4ctx)
+{
+	u32 x;
+	u32 y;
+	u32 sx, sy;
+	u8 *state;
+	state = parc4ctx->state;
+	x = (parc4ctx->x + 1) & 0xff;
+	sx = state[x];
+	y = (sx + parc4ctx->y) & 0xff;
+	sy = state[y];
+	parc4ctx->x = x;
+	parc4ctx->y = y;
+	state[y] = (u8)sx;
+	state[x] = (u8)sy;
+	return state[(sx + sy) & 0xff];
+}
+
+
+static void arcfour_encrypt(struct arc4context	*parc4ctx,
+			    u8 *dest,
+			    u8 *src,
+			    u32 len)
+{
+	u32	i;
+	for (i = 0; i < len; i++)
+		dest[i] = src[i] ^ (unsigned char)arcfour_byte(parc4ctx);
+}
+
+static sint bcrc32initialized = 0;
+static u32 crc32_table[256];
+
+
+static u8 crc32_reverseBit(u8 data)
+{
+	return (u8)((data << 7) & 0x80) | ((data << 5) & 0x40) | ((data << 3) & 0x20) | ((data << 1) & 0x10) | ((data >> 1) & 0x08) | ((data >> 3) & 0x04) | ((data >> 5) & 0x02) | ((
+				data >> 7) & 0x01) ;
+}
+
+static void crc32_init(void)
+{
+	if (bcrc32initialized == 1)
+		goto exit;
+	else {
+		sint i, j;
+		u32 c;
+		u8 *p = (u8 *)&c, *p1;
+		u8 k;
+
+		c = 0x12340000;
+
+		for (i = 0; i < 256; ++i) {
+			k = crc32_reverseBit((u8)i);
+			for (c = ((u32)k) << 24, j = 8; j > 0; --j)
+				c = c & 0x80000000 ? (c << 1) ^ CRC32_POLY : (c << 1);
+			p1 = (u8 *)&crc32_table[i];
+
+			p1[0] = crc32_reverseBit(p[3]);
+			p1[1] = crc32_reverseBit(p[2]);
+			p1[2] = crc32_reverseBit(p[1]);
+			p1[3] = crc32_reverseBit(p[0]);
+		}
+		bcrc32initialized = 1;
+	}
+exit:
+	return;
+}
+
+static u32 getcrc32(u8 *buf, sint len)
+{
+	u8 *p;
+	u32  crc;
+	if (bcrc32initialized == 0)
+		crc32_init();
+
+	crc = 0xffffffff;       /* preload shift register, per CRC-32 spec */
+
+	for (p = buf; len > 0; ++p, --len)
+		crc = crc32_table[(crc ^ *p) & 0xff] ^ (crc >> 8);
+	return ~crc;    /* transmit complement, per CRC-32 spec */
+}
+
+/*
+	Need to consider the fragment  situation
+*/
+void rtw_wep_encrypt(_adapter *padapter, u8 *pxmitframe)
+{
+	struct xmit_frame *xf = (struct xmit_frame *)pxmitframe;
+	struct pkt_attrib *pattrib = &xf->attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct rtw_xmit_req *txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	unsigned char crc[4];
+	struct arc4context mycontext;
+	sint curfragnum, length;
+	u32 keylength;
+	u8 *pframe, *payload, *iv, *key;
+	u8 wepkey[16];
+
+
+	if ((pattrib->encrypt != _WEP40_) && (pattrib->encrypt != _WEP104_))
+		return;
+
+	/* start to encrypt each fragment */
+	keylength = psecuritypriv->dot11DefKeylen[psecuritypriv->dot11PrivacyKeyIndex];
+	key = psecuritypriv->dot11DefKey[psecuritypriv->dot11PrivacyKeyIndex].skey;
+
+	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
+		if (!(pmlmeinfo->state & WIFI_FW_AUTH_STATE)) {
+			txreq = &xf->phl_txreq[curfragnum];
+			rtw_warn_on(txreq->pkt_cnt != 1);
+			pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+			pframe = pkt_list->vir_addr;
+			length = pkt_list->length - pattrib->hdrlen
+				 - pattrib->iv_len - pattrib->icv_len;
+		} else {
+			/* special case for WEP auth */
+			/* only valid when curfragnum==0 */
+			rtw_warn_on(pattrib->nr_frags > 1);
+			pframe = xf->buf_addr + TXDESC_OFFSET;
+			length = pattrib->last_txcmdsz - pattrib->hdrlen
+				 - pattrib->iv_len - pattrib->icv_len;
+		}
+		iv = pframe + pattrib->hdrlen;
+		_rtw_memcpy(&wepkey[0], iv, 3);
+		_rtw_memcpy(&wepkey[3], key, keylength);
+		payload = pframe + pattrib->iv_len + pattrib->hdrlen;
+
+		*((u32 *)crc) = cpu_to_le32(getcrc32(payload, length));
+		arcfour_init(&mycontext, wepkey, 3 + keylength);
+		arcfour_encrypt(&mycontext, payload, payload, length);
+		arcfour_encrypt(&mycontext, payload + length, crc, 4);
+	}
+
+	WEP_SW_ENC_CNT_INC(psecuritypriv, pattrib->ra);
+}
+
+void rtw_wep_decrypt(_adapter  *padapter, u8 *precvframe)
+{
+	/* exclude ICV */
+	u8	crc[4];
+	struct arc4context	 mycontext;
+	sint	length;
+	u32	keylength;
+	u8	*pframe, *payload, *iv, wepkey[16];
+	u8	 keyindex;
+	struct	rx_pkt_attrib	*prxattrib = &(((union recv_frame *)precvframe)->u.hdr.attrib);
+	struct	security_priv	*psecuritypriv = &padapter->securitypriv;
+
+
+	pframe = (unsigned char *)((union recv_frame *)precvframe)->u.hdr.rx_data;
+
+	/* start to decrypt recvframe */
+	if ((prxattrib->encrypt == _WEP40_) || (prxattrib->encrypt == _WEP104_)) {
+		iv = pframe + prxattrib->hdrlen;
+		/* keyindex=(iv[3]&0x3); */
+		keyindex = prxattrib->key_index;
+		keylength = psecuritypriv->dot11DefKeylen[keyindex];
+		_rtw_memcpy(&wepkey[0], iv, 3);
+		/* _rtw_memcpy(&wepkey[3], &psecuritypriv->dot11DefKey[psecuritypriv->dot11PrivacyKeyIndex].skey[0],keylength); */
+		_rtw_memcpy(&wepkey[3], &psecuritypriv->dot11DefKey[keyindex].skey[0], keylength);
+		length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len;
+
+		payload = pframe + prxattrib->iv_len + prxattrib->hdrlen;
+
+		/* decrypt payload include icv */
+		arcfour_init(&mycontext, wepkey, 3 + keylength);
+		arcfour_encrypt(&mycontext, payload, payload,  length);
+
+		/* calculate icv and compare the icv */
+		*((u32 *)crc) = le32_to_cpu(getcrc32(payload, length - 4));
+
+
+		WEP_SW_DEC_CNT_INC(psecuritypriv, prxattrib->ra);
+	}
+
+
+	return;
+
+}
+
+/* 3		=====TKIP related===== */
+
+static u32 secmicgetuint32(u8 *p)
+/* Convert from Byte[] to Us4Byte32 in a portable way */
+{
+	s32 i;
+	u32 res = 0;
+	for (i = 0; i < 4; i++)
+		res |= ((u32)(*p++)) << (8 * i);
+	return res;
+}
+
+static void secmicputuint32(u8 *p, u32 val)
+/* Convert from Us4Byte32 to Byte[] in a portable way */
+{
+	long i;
+	for (i = 0; i < 4; i++) {
+		*p++ = (u8)(val & 0xff);
+		val >>= 8;
+	}
+}
+
+static void secmicclear(struct mic_data *pmicdata)
+{
+	/* Reset the state to the empty message. */
+	pmicdata->L = pmicdata->K0;
+	pmicdata->R = pmicdata->K1;
+	pmicdata->nBytesInM = 0;
+	pmicdata->M = 0;
+}
+
+void rtw_secmicsetkey(struct mic_data *pmicdata, u8 *key)
+{
+	/* Set the key */
+	pmicdata->K0 = secmicgetuint32(key);
+	pmicdata->K1 = secmicgetuint32(key + 4);
+	/* and reset the message */
+	secmicclear(pmicdata);
+}
+
+void rtw_secmicappendbyte(struct mic_data *pmicdata, u8 b)
+{
+	/* Append the byte to our word-sized buffer */
+	pmicdata->M |= ((unsigned long)b) << (8 * pmicdata->nBytesInM);
+	pmicdata->nBytesInM++;
+	/* Process the word if it is full. */
+	if (pmicdata->nBytesInM >= 4) {
+		pmicdata->L ^= pmicdata->M;
+		pmicdata->R ^= ROL32(pmicdata->L, 17);
+		pmicdata->L += pmicdata->R;
+		pmicdata->R ^= ((pmicdata->L & 0xff00ff00) >> 8) | ((pmicdata->L & 0x00ff00ff) << 8);
+		pmicdata->L += pmicdata->R;
+		pmicdata->R ^= ROL32(pmicdata->L, 3);
+		pmicdata->L += pmicdata->R;
+		pmicdata->R ^= ROR32(pmicdata->L, 2);
+		pmicdata->L += pmicdata->R;
+		/* Clear the buffer */
+		pmicdata->M = 0;
+		pmicdata->nBytesInM = 0;
+	}
+}
+
+void rtw_secmicappend(struct mic_data *pmicdata, u8 *src, u32 nbytes)
+{
+	/* This is simple */
+	while (nbytes > 0) {
+		rtw_secmicappendbyte(pmicdata, *src++);
+		nbytes--;
+	}
+}
+
+void rtw_secgetmic(struct mic_data *pmicdata, u8 *dst)
+{
+	/* Append the minimum padding */
+	rtw_secmicappendbyte(pmicdata, 0x5a);
+	rtw_secmicappendbyte(pmicdata, 0);
+	rtw_secmicappendbyte(pmicdata, 0);
+	rtw_secmicappendbyte(pmicdata, 0);
+	rtw_secmicappendbyte(pmicdata, 0);
+	/* and then zeroes until the length is a multiple of 4 */
+	while (pmicdata->nBytesInM != 0)
+		rtw_secmicappendbyte(pmicdata, 0);
+	/* The appendByte function has already computed the result. */
+	secmicputuint32(dst, pmicdata->L);
+	secmicputuint32(dst + 4, pmicdata->R);
+	/* Reset to the empty message. */
+	secmicclear(pmicdata);
+}
+
+
+void rtw_seccalctkipmic(u8 *key, u8 *header, u8 *data, u32 data_len, u8 *mic_code, u8 pri)
+{
+
+	struct mic_data	micdata;
+	u8 priority[4] = {0x0, 0x0, 0x0, 0x0};
+	rtw_secmicsetkey(&micdata, key);
+	priority[0] = pri;
+
+	/* Michael MIC pseudo header: DA, SA, 3 x 0, Priority */
+	if (header[1] & 1) { /* ToDS==1 */
+		rtw_secmicappend(&micdata, &header[16], 6);  /* DA */
+		if (header[1] & 2) /* From Ds==1 */
+			rtw_secmicappend(&micdata, &header[24], 6);
+		else
+			rtw_secmicappend(&micdata, &header[10], 6);
+	} else {	/* ToDS==0 */
+		rtw_secmicappend(&micdata, &header[4], 6);   /* DA */
+		if (header[1] & 2) /* From Ds==1 */
+			rtw_secmicappend(&micdata, &header[16], 6);
+		else
+			rtw_secmicappend(&micdata, &header[10], 6);
+
+	}
+	rtw_secmicappend(&micdata, &priority[0], 4);
+
+
+	rtw_secmicappend(&micdata, data, data_len);
+
+	rtw_secgetmic(&micdata, mic_code);
+}
+
+
+
+
+/* macros for extraction/creation of unsigned char/unsigned short values */
+#define RotR1(v16)   ((((v16) >> 1) & 0x7FFF) ^ (((v16) & 1) << 15))
+#define   Lo8(v16)   ((u8)((v16)       & 0x00FF))
+#define   Hi8(v16)   ((u8)(((v16) >> 8) & 0x00FF))
+#define  Lo16(v32)   ((u16)((v32)       & 0xFFFF))
+#define  Hi16(v32)   ((u16)(((v32) >> 16) & 0xFFFF))
+#define  Mk16(hi, lo) ((lo) ^ (((u16)(hi)) << 8))
+
+/* select the Nth 16-bit word of the temporal key unsigned char array TK[]  */
+#define  TK16(N)     Mk16(tk[2*(N)+1], tk[2*(N)])
+
+/* S-box lookup: 16 bits --> 16 bits */
+#define _S_(v16)     (Sbox1[0][Lo8(v16)] ^ Sbox1[1][Hi8(v16)])
+
+/* fixed algorithm "parameters" */
+#define PHASE1_LOOP_CNT   8    /* this needs to be "big enough"     */
+#define TA_SIZE           6    /*  48-bit transmitter address      */
+#define TK_SIZE          16    /* 128-bit temporal key             */
+#define P1K_SIZE         10    /*  80-bit Phase1 key               */
+#define RC4_KEY_SIZE     16    /* 128-bit RC4KEY (104 bits unknown) */
+
+
+/* 2-unsigned char by 2-unsigned char subset of the full AES S-box table */
+static const unsigned short Sbox1[2][256] =      /* Sbox for hash (can be in ROM)    */
+{ {
+		0xC6A5, 0xF884, 0xEE99, 0xF68D, 0xFF0D, 0xD6BD, 0xDEB1, 0x9154,
+		0x6050, 0x0203, 0xCEA9, 0x567D, 0xE719, 0xB562, 0x4DE6, 0xEC9A,
+		0x8F45, 0x1F9D, 0x8940, 0xFA87, 0xEF15, 0xB2EB, 0x8EC9, 0xFB0B,
+		0x41EC, 0xB367, 0x5FFD, 0x45EA, 0x23BF, 0x53F7, 0xE496, 0x9B5B,
+		0x75C2, 0xE11C, 0x3DAE, 0x4C6A, 0x6C5A, 0x7E41, 0xF502, 0x834F,
+		0x685C, 0x51F4, 0xD134, 0xF908, 0xE293, 0xAB73, 0x6253, 0x2A3F,
+		0x080C, 0x9552, 0x4665, 0x9D5E, 0x3028, 0x37A1, 0x0A0F, 0x2FB5,
+		0x0E09, 0x2436, 0x1B9B, 0xDF3D, 0xCD26, 0x4E69, 0x7FCD, 0xEA9F,
+		0x121B, 0x1D9E, 0x5874, 0x342E, 0x362D, 0xDCB2, 0xB4EE, 0x5BFB,
+		0xA4F6, 0x764D, 0xB761, 0x7DCE, 0x527B, 0xDD3E, 0x5E71, 0x1397,
+		0xA6F5, 0xB968, 0x0000, 0xC12C, 0x4060, 0xE31F, 0x79C8, 0xB6ED,
+		0xD4BE, 0x8D46, 0x67D9, 0x724B, 0x94DE, 0x98D4, 0xB0E8, 0x854A,
+		0xBB6B, 0xC52A, 0x4FE5, 0xED16, 0x86C5, 0x9AD7, 0x6655, 0x1194,
+		0x8ACF, 0xE910, 0x0406, 0xFE81, 0xA0F0, 0x7844, 0x25BA, 0x4BE3,
+		0xA2F3, 0x5DFE, 0x80C0, 0x058A, 0x3FAD, 0x21BC, 0x7048, 0xF104,
+		0x63DF, 0x77C1, 0xAF75, 0x4263, 0x2030, 0xE51A, 0xFD0E, 0xBF6D,
+		0x814C, 0x1814, 0x2635, 0xC32F, 0xBEE1, 0x35A2, 0x88CC, 0x2E39,
+		0x9357, 0x55F2, 0xFC82, 0x7A47, 0xC8AC, 0xBAE7, 0x322B, 0xE695,
+		0xC0A0, 0x1998, 0x9ED1, 0xA37F, 0x4466, 0x547E, 0x3BAB, 0x0B83,
+		0x8CCA, 0xC729, 0x6BD3, 0x283C, 0xA779, 0xBCE2, 0x161D, 0xAD76,
+		0xDB3B, 0x6456, 0x744E, 0x141E, 0x92DB, 0x0C0A, 0x486C, 0xB8E4,
+		0x9F5D, 0xBD6E, 0x43EF, 0xC4A6, 0x39A8, 0x31A4, 0xD337, 0xF28B,
+		0xD532, 0x8B43, 0x6E59, 0xDAB7, 0x018C, 0xB164, 0x9CD2, 0x49E0,
+		0xD8B4, 0xACFA, 0xF307, 0xCF25, 0xCAAF, 0xF48E, 0x47E9, 0x1018,
+		0x6FD5, 0xF088, 0x4A6F, 0x5C72, 0x3824, 0x57F1, 0x73C7, 0x9751,
+		0xCB23, 0xA17C, 0xE89C, 0x3E21, 0x96DD, 0x61DC, 0x0D86, 0x0F85,
+		0xE090, 0x7C42, 0x71C4, 0xCCAA, 0x90D8, 0x0605, 0xF701, 0x1C12,
+		0xC2A3, 0x6A5F, 0xAEF9, 0x69D0, 0x1791, 0x9958, 0x3A27, 0x27B9,
+		0xD938, 0xEB13, 0x2BB3, 0x2233, 0xD2BB, 0xA970, 0x0789, 0x33A7,
+		0x2DB6, 0x3C22, 0x1592, 0xC920, 0x8749, 0xAAFF, 0x5078, 0xA57A,
+		0x038F, 0x59F8, 0x0980, 0x1A17, 0x65DA, 0xD731, 0x84C6, 0xD0B8,
+		0x82C3, 0x29B0, 0x5A77, 0x1E11, 0x7BCB, 0xA8FC, 0x6DD6, 0x2C3A,
+	},
+
+
+	{  /* second half of table is unsigned char-reversed version of first! */
+		0xA5C6, 0x84F8, 0x99EE, 0x8DF6, 0x0DFF, 0xBDD6, 0xB1DE, 0x5491,
+		0x5060, 0x0302, 0xA9CE, 0x7D56, 0x19E7, 0x62B5, 0xE64D, 0x9AEC,
+		0x458F, 0x9D1F, 0x4089, 0x87FA, 0x15EF, 0xEBB2, 0xC98E, 0x0BFB,
+		0xEC41, 0x67B3, 0xFD5F, 0xEA45, 0xBF23, 0xF753, 0x96E4, 0x5B9B,
+		0xC275, 0x1CE1, 0xAE3D, 0x6A4C, 0x5A6C, 0x417E, 0x02F5, 0x4F83,
+		0x5C68, 0xF451, 0x34D1, 0x08F9, 0x93E2, 0x73AB, 0x5362, 0x3F2A,
+		0x0C08, 0x5295, 0x6546, 0x5E9D, 0x2830, 0xA137, 0x0F0A, 0xB52F,
+		0x090E, 0x3624, 0x9B1B, 0x3DDF, 0x26CD, 0x694E, 0xCD7F, 0x9FEA,
+		0x1B12, 0x9E1D, 0x7458, 0x2E34, 0x2D36, 0xB2DC, 0xEEB4, 0xFB5B,
+		0xF6A4, 0x4D76, 0x61B7, 0xCE7D, 0x7B52, 0x3EDD, 0x715E, 0x9713,
+		0xF5A6, 0x68B9, 0x0000, 0x2CC1, 0x6040, 0x1FE3, 0xC879, 0xEDB6,
+		0xBED4, 0x468D, 0xD967, 0x4B72, 0xDE94, 0xD498, 0xE8B0, 0x4A85,
+		0x6BBB, 0x2AC5, 0xE54F, 0x16ED, 0xC586, 0xD79A, 0x5566, 0x9411,
+		0xCF8A, 0x10E9, 0x0604, 0x81FE, 0xF0A0, 0x4478, 0xBA25, 0xE34B,
+		0xF3A2, 0xFE5D, 0xC080, 0x8A05, 0xAD3F, 0xBC21, 0x4870, 0x04F1,
+		0xDF63, 0xC177, 0x75AF, 0x6342, 0x3020, 0x1AE5, 0x0EFD, 0x6DBF,
+		0x4C81, 0x1418, 0x3526, 0x2FC3, 0xE1BE, 0xA235, 0xCC88, 0x392E,
+		0x5793, 0xF255, 0x82FC, 0x477A, 0xACC8, 0xE7BA, 0x2B32, 0x95E6,
+		0xA0C0, 0x9819, 0xD19E, 0x7FA3, 0x6644, 0x7E54, 0xAB3B, 0x830B,
+		0xCA8C, 0x29C7, 0xD36B, 0x3C28, 0x79A7, 0xE2BC, 0x1D16, 0x76AD,
+		0x3BDB, 0x5664, 0x4E74, 0x1E14, 0xDB92, 0x0A0C, 0x6C48, 0xE4B8,
+		0x5D9F, 0x6EBD, 0xEF43, 0xA6C4, 0xA839, 0xA431, 0x37D3, 0x8BF2,
+		0x32D5, 0x438B, 0x596E, 0xB7DA, 0x8C01, 0x64B1, 0xD29C, 0xE049,
+		0xB4D8, 0xFAAC, 0x07F3, 0x25CF, 0xAFCA, 0x8EF4, 0xE947, 0x1810,
+		0xD56F, 0x88F0, 0x6F4A, 0x725C, 0x2438, 0xF157, 0xC773, 0x5197,
+		0x23CB, 0x7CA1, 0x9CE8, 0x213E, 0xDD96, 0xDC61, 0x860D, 0x850F,
+		0x90E0, 0x427C, 0xC471, 0xAACC, 0xD890, 0x0506, 0x01F7, 0x121C,
+		0xA3C2, 0x5F6A, 0xF9AE, 0xD069, 0x9117, 0x5899, 0x273A, 0xB927,
+		0x38D9, 0x13EB, 0xB32B, 0x3322, 0xBBD2, 0x70A9, 0x8907, 0xA733,
+		0xB62D, 0x223C, 0x9215, 0x20C9, 0x4987, 0xFFAA, 0x7850, 0x7AA5,
+		0x8F03, 0xF859, 0x8009, 0x171A, 0xDA65, 0x31D7, 0xC684, 0xB8D0,
+		0xC382, 0xB029, 0x775A, 0x111E, 0xCB7B, 0xFCA8, 0xD66D, 0x3A2C,
+	}
+};
+
+/*
+**********************************************************************
+* Routine: Phase 1 -- generate P1K, given TA, TK, IV32
+*
+* Inputs:
+*     tk[]      = temporal key                         [128 bits]
+*     ta[]      = transmitter's MAC address            [ 48 bits]
+*     iv32      = upper 32 bits of IV                  [ 32 bits]
+* Output:
+*     p1k[]     = Phase 1 key                          [ 80 bits]
+*
+* Note:
+*     This function only needs to be called every 2**16 packets,
+*     although in theory it could be called every packet.
+*
+**********************************************************************
+*/
+static void phase1(u16 *p1k, const u8 *tk, const u8 *ta, u32 iv32)
+{
+	sint  i;
+	/* Initialize the 80 bits of P1K[] from IV32 and TA[0..5]    */
+	p1k[0]      = Lo16(iv32);
+	p1k[1]      = Hi16(iv32);
+	p1k[2]      = Mk16(ta[1], ta[0]); /* use TA[] as little-endian */
+	p1k[3]      = Mk16(ta[3], ta[2]);
+	p1k[4]      = Mk16(ta[5], ta[4]);
+
+	/* Now compute an unbalanced Feistel cipher with 80-bit block */
+	/* size on the 80-bit block P1K[], using the 128-bit key TK[] */
+	for (i = 0; i < PHASE1_LOOP_CNT ; i++) {
+		/* Each add operation here is mod 2**16 */
+		p1k[0] += _S_(p1k[4] ^ TK16((i & 1) + 0));
+		p1k[1] += _S_(p1k[0] ^ TK16((i & 1) + 2));
+		p1k[2] += _S_(p1k[1] ^ TK16((i & 1) + 4));
+		p1k[3] += _S_(p1k[2] ^ TK16((i & 1) + 6));
+		p1k[4] += _S_(p1k[3] ^ TK16((i & 1) + 0));
+		p1k[4] += (unsigned short)i;                     /* avoid "slide attacks" */
+	}
+}
+
+
+/*
+**********************************************************************
+* Routine: Phase 2 -- generate RC4KEY, given TK, P1K, IV16
+*
+* Inputs:
+*     tk[]      = Temporal key                         [128 bits]
+*     p1k[]     = Phase 1 output key                   [ 80 bits]
+*     iv16      = low 16 bits of IV counter            [ 16 bits]
+* Output:
+*     rc4key[]  = the key used to encrypt the packet   [128 bits]
+*
+* Note:
+*     The value {TA,IV32,IV16} for Phase1/Phase2 must be unique
+*     across all packets using the same key TK value. Then, for a
+*     given value of TK[], this TKIP48 construction guarantees that
+*     the final RC4KEY value is unique across all packets.
+*
+* Suggested implementation optimization: if PPK[] is "overlaid"
+*     appropriately on RC4KEY[], there is no need for the final
+*     for loop below that copies the PPK[] result into RC4KEY[].
+*
+**********************************************************************
+*/
+static void phase2(u8 *rc4key, const u8 *tk, const u16 *p1k, u16 iv16)
+{
+	sint  i;
+	u16 PPK[6];                          /* temporary key for mixing   */
+	/* Note: all adds in the PPK[] equations below are mod 2**16        */
+	for (i = 0; i < 5; i++)
+		PPK[i] = p1k[i];    /* first, copy P1K to PPK     */
+	PPK[5]  =  p1k[4] + iv16;            /* next,  add in IV16         */
+
+	/* Bijective non-linear mixing of the 96 bits of PPK[0..5]          */
+	PPK[0] +=    _S_(PPK[5] ^ TK16(0));   /* Mix key in each "round"     */
+	PPK[1] +=    _S_(PPK[0] ^ TK16(1));
+	PPK[2] +=    _S_(PPK[1] ^ TK16(2));
+	PPK[3] +=    _S_(PPK[2] ^ TK16(3));
+	PPK[4] +=    _S_(PPK[3] ^ TK16(4));
+	PPK[5] +=    _S_(PPK[4] ^ TK16(5));   /* Total # S-box lookups == 6 */
+
+	/* Final sweep: bijective, "linear". Rotates kill LSB correlations   */
+	PPK[0] +=  RotR1(PPK[5] ^ TK16(6));
+	PPK[1] +=  RotR1(PPK[0] ^ TK16(7));   /* Use all of TK[] in Phase2  */
+	PPK[2] +=  RotR1(PPK[1]);
+	PPK[3] +=  RotR1(PPK[2]);
+	PPK[4] +=  RotR1(PPK[3]);
+	PPK[5] +=  RotR1(PPK[4]);
+	/* Note: At this point, for a given key TK[0..15], the 96-bit output */
+	/*       value PPK[0..5] is guaranteed to be unique, as a function  */
+	/*       of the 96-bit "input" value   {TA,IV32,IV16}. That is, P1K  */
+	/*       is now a keyed permutation of {TA,IV32,IV16}.              */
+
+	/* Set RC4KEY[0..3], which includes "cleartext" portion of RC4 key   */
+	rc4key[0] = Hi8(iv16);                /* RC4KEY[0..2] is the WEP IV */
+	rc4key[1] = (Hi8(iv16) | 0x20) & 0x7F; /* Help avoid weak (FMS) keys */
+	rc4key[2] = Lo8(iv16);
+	rc4key[3] = Lo8((PPK[5] ^ TK16(0)) >> 1);
+
+
+	/* Copy 96 bits of PPK[0..5] to RC4KEY[4..15]  (little-endian)      */
+	for (i = 0; i < 6; i++) {
+		rc4key[4 + 2 * i] = Lo8(PPK[i]);
+		rc4key[5 + 2 * i] = Hi8(PPK[i]);
+	}
+}
+
+
+/* The hlen isn't include the IV */
+u32 rtw_tkip_encrypt(_adapter *padapter, u8 *pxmitframe)
+{
+	struct xmit_frame *xf = (struct xmit_frame *)pxmitframe;
+	struct pkt_attrib *pattrib = &xf->attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct rtw_xmit_req *txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	u16 pnl;
+	u32 pnh;
+	u8 rc4key[16];
+	u8 ttkey[16];
+	u8 crc[4];
+	struct arc4context mycontext;
+	sint curfragnum, length;
+	u32 prwskeylen;
+	u8 *pframe, *payload, *iv, *prwskey;
+	union pn48 dot11txpn;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+
+	if (pattrib->encrypt != _TKIP_)
+		return _SUCCESS;
+
+	/* start to encrypt each fragment */
+	if (IS_MCAST(pattrib->ra))
+		prwskey = lsecuritypriv->dot118021XGrpKey[lsecuritypriv->dot118021XGrpKeyid].skey;
+	else
+		prwskey = pattrib->dot118021x_UncstKey.skey;
+	prwskeylen = 16;
+
+	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
+		txreq = &xf->phl_txreq[curfragnum];
+		rtw_warn_on(txreq->pkt_cnt != 1);
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		pframe = pkt_list->vir_addr;
+		length = pkt_list->length - pattrib->hdrlen
+			 - pattrib->iv_len - pattrib->icv_len;
+
+		iv = pframe + pattrib->hdrlen;
+		payload = pframe + pattrib->iv_len + pattrib->hdrlen;
+
+		GET_TKIP_PN(iv, dot11txpn);
+
+		pnl = (u16)(dot11txpn.val);
+		pnh = (u32)(dot11txpn.val >> 16);
+
+		phase1((u16 *)&ttkey[0], prwskey, &pattrib->ta[0], pnh);
+		phase2(&rc4key[0], prwskey, (u16 *)&ttkey[0], pnl);
+
+		*((u32 *)crc) = cpu_to_le32(getcrc32(payload, length));
+
+		arcfour_init(&mycontext, rc4key, 16);
+		arcfour_encrypt(&mycontext, payload, payload, length);
+		arcfour_encrypt(&mycontext, payload + length, crc, 4);
+	}
+
+	TKIP_SW_ENC_CNT_INC(psecuritypriv, pattrib->ra);
+
+	return _SUCCESS;
+}
+
+/* The hlen isn't include the IV */
+u32 rtw_tkip_decrypt(_adapter *padapter, u8 *precvframe)
+{
+	/* exclude ICV */
+	u16 pnl;
+	u32 pnh;
+	u8   rc4key[16];
+	u8   ttkey[16];
+	u8	crc[4];
+	struct arc4context mycontext;
+	sint			length;
+	u32	prwskeylen;
+
+	u8	*pframe, *payload, *iv, *prwskey;
+	union pn48 dot11txpn;
+	struct	sta_info		*stainfo;
+	struct	rx_pkt_attrib	*prxattrib = &((union recv_frame *)precvframe)->u.hdr.attrib;
+	struct	security_priv	*psecuritypriv = &padapter->securitypriv;
+	u32		res = _SUCCESS;
+	struct _ADAPTER_LINK		*padapter_link = ((union recv_frame *)precvframe)->u.hdr.adapter_link;
+	struct	link_security_priv	*lsecuritypriv = &padapter_link->securitypriv;
+
+
+	pframe = (unsigned char *)((union recv_frame *)precvframe)->u.hdr.rx_data;
+
+	/* 4 start to decrypt recvframe */
+	if (prxattrib->encrypt == _TKIP_) {
+
+		stainfo = ((union recv_frame *)precvframe)->u.hdr.psta;
+		if (stainfo != NULL) {
+
+			if (IS_MCAST(prxattrib->ra)) {
+				static systime start = 0;
+				static u32 no_gkey_bc_cnt = 0;
+				static u32 no_gkey_mc_cnt = 0;
+
+				if (lsecuritypriv->binstallGrpkey == _FALSE) {
+					res = _FAIL;
+
+					if (start == 0)
+						start = rtw_get_current_time();
+
+					if (is_broadcast_mac_addr(prxattrib->ra))
+						no_gkey_bc_cnt++;
+					else
+						no_gkey_mc_cnt++;
+
+					if (rtw_get_passing_time_ms(start) > 1000) {
+						if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+							RTW_PRINT(FUNC_ADPT_FMT" no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+								FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+						}
+						start = rtw_get_current_time();
+						no_gkey_bc_cnt = 0;
+						no_gkey_mc_cnt = 0;
+					}
+					goto exit;
+				}
+
+				if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+					RTW_PRINT(FUNC_ADPT_FMT" gkey installed. no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+						FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+				}
+				start = 0;
+				no_gkey_bc_cnt = 0;
+				no_gkey_mc_cnt = 0;
+
+				/* RTW_INFO("rx bc/mc packets, to perform sw rtw_tkip_decrypt\n"); */
+				/* prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey; */
+				prwskey = lsecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
+				prwskeylen = 16;
+			} else {
+				prwskey = &stainfo->dot118021x_UncstKey.skey[0];
+				prwskeylen = 16;
+			}
+
+			iv = pframe + prxattrib->hdrlen;
+			payload = pframe + prxattrib->iv_len + prxattrib->hdrlen;
+			length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len;
+
+			GET_TKIP_PN(iv, dot11txpn);
+
+			pnl = (u16)(dot11txpn.val);
+			pnh = (u32)(dot11txpn.val >> 16);
+
+			/* ToDo CONFIG_RTW_MLD: MLD addr for unicast, link addr for multicast */
+			phase1((u16 *)&ttkey[0], prwskey, &prxattrib->ta[0], pnh);
+			phase2(&rc4key[0], prwskey, (unsigned short *)&ttkey[0], pnl);
+
+			/* 4 decrypt payload include icv */
+
+			arcfour_init(&mycontext, rc4key, 16);
+			arcfour_encrypt(&mycontext, payload, payload, length);
+
+			*((u32 *)crc) = le32_to_cpu(getcrc32(payload, length - 4));
+
+			if (crc[3] != payload[length - 1] || crc[2] != payload[length - 2] || crc[1] != payload[length - 3] || crc[0] != payload[length - 4]) {
+				res = _FAIL;
+			}
+
+			TKIP_SW_DEC_CNT_INC(psecuritypriv, prxattrib->ra);
+		} else {
+			res = _FAIL;
+		}
+
+	}
+exit:
+	return res;
+
+}
+
+
+/* 3			=====AES related===== */
+#if (NEW_CRYPTO == 0)
+
+#define MAX_MSG_SIZE	2048
+/*****************************/
+/******** SBOX Table *********/
+/*****************************/
+
+static  u8 sbox_table[256] = {
+	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
+	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
+	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
+	0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
+	0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc,
+	0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
+	0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a,
+	0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
+	0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
+	0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
+	0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b,
+	0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+	0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85,
+	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
+	0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
+	0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
+	0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17,
+	0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
+	0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88,
+	0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
+	0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
+	0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
+	0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9,
+	0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+	0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6,
+	0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
+	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
+	0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
+	0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94,
+	0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
+	0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68,
+	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
+};
+
+/*****************************/
+/**** Function Prototypes ****/
+/*****************************/
+
+static void bitwise_xor(u8 *ina, u8 *inb, u8 *out);
+static void construct_mic_iv(
+	u8 *mic_header1,
+	sint qc_exists,
+	sint a4_exists,
+	u8 *mpdu,
+	uint payload_length,
+	u8 *pn_vector,
+	uint frtype);/* add for CONFIG_IEEE80211W, none 11w also can use */
+static void construct_mic_header1(
+	u8 *mic_header1,
+	sint header_length,
+	u8 *mpdu,
+	uint frtype);/* add for CONFIG_IEEE80211W, none 11w also can use */
+static void construct_mic_header2(
+	u8 *mic_header2,
+	u8 *mpdu,
+	sint a4_exists,
+	sint qc_exists);
+static void construct_ctr_preload(
+	u8 *ctr_preload,
+	sint a4_exists,
+	sint qc_exists,
+	u8 *mpdu,
+	u8 *pn_vector,
+	sint c,
+	uint frtype);/* add for CONFIG_IEEE80211W, none 11w also can use */
+static void xor_128(u8 *a, u8 *b, u8 *out);
+static void xor_32(u8 *a, u8 *b, u8 *out);
+static u8 sbox(u8 a);
+static void next_key(u8 *key, sint round);
+static void byte_sub(u8 *in, u8 *out);
+static void shift_row(u8 *in, u8 *out);
+static void mix_column(u8 *in, u8 *out);
+static void aes128k128d(u8 *key, u8 *data, u8 *ciphertext);
+
+
+/****************************************/
+/* aes128k128d()                       */
+/* Performs a 128 bit AES encrypt with */
+/* 128 bit data.                       */
+/****************************************/
+static void xor_128(u8 *a, u8 *b, u8 *out)
+{
+	sint i;
+	for (i = 0; i < 16; i++)
+		out[i] = a[i] ^ b[i];
+}
+
+
+static void xor_32(u8 *a, u8 *b, u8 *out)
+{
+	sint i;
+	for (i = 0; i < 4; i++)
+		out[i] = a[i] ^ b[i];
+}
+
+
+static u8 sbox(u8 a)
+{
+	return sbox_table[(sint)a];
+}
+
+
+static void next_key(u8 *key, sint round)
+{
+	u8 rcon;
+	u8 sbox_key[4];
+	u8 rcon_table[12] = {
+		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
+		0x1b, 0x36, 0x36, 0x36
+	};
+	sbox_key[0] = sbox(key[13]);
+	sbox_key[1] = sbox(key[14]);
+	sbox_key[2] = sbox(key[15]);
+	sbox_key[3] = sbox(key[12]);
+
+	rcon = rcon_table[round];
+
+	xor_32(&key[0], sbox_key, &key[0]);
+	key[0] = key[0] ^ rcon;
+
+	xor_32(&key[4], &key[0], &key[4]);
+	xor_32(&key[8], &key[4], &key[8]);
+	xor_32(&key[12], &key[8], &key[12]);
+}
+
+
+static void byte_sub(u8 *in, u8 *out)
+{
+	sint i;
+	for (i = 0; i < 16; i++)
+		out[i] = sbox(in[i]);
+}
+
+
+static void shift_row(u8 *in, u8 *out)
+{
+	out[0] =  in[0];
+	out[1] =  in[5];
+	out[2] =  in[10];
+	out[3] =  in[15];
+	out[4] =  in[4];
+	out[5] =  in[9];
+	out[6] =  in[14];
+	out[7] =  in[3];
+	out[8] =  in[8];
+	out[9] =  in[13];
+	out[10] = in[2];
+	out[11] = in[7];
+	out[12] = in[12];
+	out[13] = in[1];
+	out[14] = in[6];
+	out[15] = in[11];
+}
+
+
+static void mix_column(u8 *in, u8 *out)
+{
+	sint i;
+	u8 add1b[4];
+	u8 add1bf7[4];
+	u8 rotl[4];
+	u8 swap_halfs[4];
+	u8 andf7[4];
+	u8 rotr[4];
+	u8 temp[4];
+	u8 tempb[4];
+	for (i = 0 ; i < 4; i++) {
+		if ((in[i] & 0x80) == 0x80)
+			add1b[i] = 0x1b;
+		else
+			add1b[i] = 0x00;
+	}
+
+	swap_halfs[0] = in[2];    /* Swap halfs */
+	swap_halfs[1] = in[3];
+	swap_halfs[2] = in[0];
+	swap_halfs[3] = in[1];
+
+	rotl[0] = in[3];        /* Rotate left 8 bits */
+	rotl[1] = in[0];
+	rotl[2] = in[1];
+	rotl[3] = in[2];
+
+	andf7[0] = in[0] & 0x7f;
+	andf7[1] = in[1] & 0x7f;
+	andf7[2] = in[2] & 0x7f;
+	andf7[3] = in[3] & 0x7f;
+
+	for (i = 3; i > 0; i--) { /* logical shift left 1 bit */
+		andf7[i] = andf7[i] << 1;
+		if ((andf7[i - 1] & 0x80) == 0x80)
+			andf7[i] = (andf7[i] | 0x01);
+	}
+	andf7[0] = andf7[0] << 1;
+	andf7[0] = andf7[0] & 0xfe;
+
+	xor_32(add1b, andf7, add1bf7);
+
+	xor_32(in, add1bf7, rotr);
+
+	temp[0] = rotr[0];         /* Rotate right 8 bits */
+	rotr[0] = rotr[1];
+	rotr[1] = rotr[2];
+	rotr[2] = rotr[3];
+	rotr[3] = temp[0];
+
+	xor_32(add1bf7, rotr, temp);
+	xor_32(swap_halfs, rotl, tempb);
+	xor_32(temp, tempb, out);
+}
+
+
+static void aes128k128d(u8 *key, u8 *data, u8 *ciphertext)
+{
+	sint round;
+	sint i;
+	u8 intermediatea[16];
+	u8 intermediateb[16];
+	u8 round_key[16];
+	for (i = 0; i < 16; i++)
+		round_key[i] = key[i];
+
+	for (round = 0; round < 11; round++) {
+		if (round == 0) {
+			xor_128(round_key, data, ciphertext);
+			next_key(round_key, round);
+		} else if (round == 10) {
+			byte_sub(ciphertext, intermediatea);
+			shift_row(intermediatea, intermediateb);
+			xor_128(intermediateb, round_key, ciphertext);
+		} else { /* 1 - 9 */
+			byte_sub(ciphertext, intermediatea);
+			shift_row(intermediatea, intermediateb);
+			mix_column(&intermediateb[0], &intermediatea[0]);
+			mix_column(&intermediateb[4], &intermediatea[4]);
+			mix_column(&intermediateb[8], &intermediatea[8]);
+			mix_column(&intermediateb[12], &intermediatea[12]);
+			xor_128(intermediatea, round_key, ciphertext);
+			next_key(round_key, round);
+		}
+	}
+}
+
+
+/************************************************/
+/* construct_mic_iv()                          */
+/* Builds the MIC IV from header fields and PN */
+/* Baron think the function is construct CCM   */
+/* nonce                                       */
+/************************************************/
+static void construct_mic_iv(
+	u8 *mic_iv,
+	sint qc_exists,
+	sint a4_exists,
+	u8 *mpdu,
+	uint payload_length,
+	u8 *pn_vector,
+	uint frtype/* add for CONFIG_IEEE80211W, none 11w also can use */
+)
+{
+	sint i;
+	mic_iv[0] = 0x59;
+	if (qc_exists && a4_exists)
+		mic_iv[1] = mpdu[30] & 0x0f;    /* QoS_TC          */
+	if (qc_exists && !a4_exists)
+		mic_iv[1] = mpdu[24] & 0x0f;   /* mute bits 7-4   */
+	if (!qc_exists)
+		mic_iv[1] = 0x00;
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+	/* 802.11w management frame should set management bit(4) */
+	if (frtype == WIFI_MGT_TYPE)
+		mic_iv[1] |= BIT(4);
+#endif
+	for (i = 2; i < 8; i++)
+		mic_iv[i] = mpdu[i + 8];                    /* mic_iv[2:7] = A2[0:5] = mpdu[10:15] */
+#ifdef CONSISTENT_PN_ORDER
+	for (i = 8; i < 14; i++)
+		mic_iv[i] = pn_vector[i - 8];           /* mic_iv[8:13] = PN[0:5] */
+#else
+	for (i = 8; i < 14; i++)
+		mic_iv[i] = pn_vector[13 - i];          /* mic_iv[8:13] = PN[5:0] */
+#endif
+	mic_iv[14] = (unsigned char)(payload_length / 256);
+	mic_iv[15] = (unsigned char)(payload_length % 256);
+}
+
+
+/************************************************/
+/* construct_mic_header1()                     */
+/* Builds the first MIC header block from      */
+/* header fields.                              */
+/* Build AAD SC,A1,A2                          */
+/************************************************/
+static void construct_mic_header1(
+	u8 *mic_header1,
+	sint header_length,
+	u8 *mpdu,
+	uint frtype/* add for CONFIG_IEEE80211W, none 11w also can use */
+)
+{
+	mic_header1[0] = (u8)((header_length - 2) / 256);
+	mic_header1[1] = (u8)((header_length - 2) % 256);
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+	/* 802.11w management frame don't AND subtype bits 4,5,6 of frame control field */
+	if (frtype == WIFI_MGT_TYPE)
+		mic_header1[2] = mpdu[0];
+	else
+#endif
+		mic_header1[2] = mpdu[0] & 0xcf;    /* Mute CF poll & CF ack bits */
+
+	mic_header1[3] = mpdu[1] & 0xc7;    /* Mute retry, more data and pwr mgt bits */
+	mic_header1[4] = mpdu[4];       /* A1 */
+	mic_header1[5] = mpdu[5];
+	mic_header1[6] = mpdu[6];
+	mic_header1[7] = mpdu[7];
+	mic_header1[8] = mpdu[8];
+	mic_header1[9] = mpdu[9];
+	mic_header1[10] = mpdu[10];     /* A2 */
+	mic_header1[11] = mpdu[11];
+	mic_header1[12] = mpdu[12];
+	mic_header1[13] = mpdu[13];
+	mic_header1[14] = mpdu[14];
+	mic_header1[15] = mpdu[15];
+}
+
+
+/************************************************/
+/* construct_mic_header2()                     */
+/* Builds the last MIC header block from       */
+/* header fields.                              */
+/************************************************/
+static void construct_mic_header2(
+	u8 *mic_header2,
+	u8 *mpdu,
+	sint a4_exists,
+	sint qc_exists
+)
+{
+	sint i;
+	for (i = 0; i < 16; i++)
+		mic_header2[i] = 0x00;
+
+	mic_header2[0] = mpdu[16];    /* A3 */
+	mic_header2[1] = mpdu[17];
+	mic_header2[2] = mpdu[18];
+	mic_header2[3] = mpdu[19];
+	mic_header2[4] = mpdu[20];
+	mic_header2[5] = mpdu[21];
+
+	/* mic_header2[6] = mpdu[22] & 0xf0;    SC */
+	mic_header2[6] = 0x00;
+	mic_header2[7] = 0x00; /* mpdu[23]; */
+
+
+	if (!qc_exists && a4_exists) {
+		for (i = 0; i < 6; i++)
+			mic_header2[8 + i] = mpdu[24 + i]; /* A4 */
+
+	}
+
+	if (qc_exists && !a4_exists) {
+		mic_header2[8] = mpdu[24] & 0x0f; /* mute bits 15 - 4 */
+		mic_header2[9] = mpdu[25] & 0x00;
+	}
+
+	if (qc_exists && a4_exists) {
+		for (i = 0; i < 6; i++)
+			mic_header2[8 + i] = mpdu[24 + i]; /* A4 */
+
+		mic_header2[14] = mpdu[30] & 0x0f;
+		mic_header2[15] = mpdu[31] & 0x00;
+	}
+
+}
+
+
+/************************************************/
+/* construct_mic_header2()                     */
+/* Builds the last MIC header block from       */
+/* header fields.                              */
+/* Baron think the function is construct CCM   */
+/* nonce                                       */
+/************************************************/
+static void construct_ctr_preload(
+	u8 *ctr_preload,
+	sint a4_exists,
+	sint qc_exists,
+	u8 *mpdu,
+	u8 *pn_vector,
+	sint c,
+	uint frtype /* add for CONFIG_IEEE80211W, none 11w also can use */
+)
+{
+	sint i = 0;
+	for (i = 0; i < 16; i++)
+		ctr_preload[i] = 0x00;
+	i = 0;
+
+	ctr_preload[0] = 0x01;                                  /* flag */
+	if (qc_exists && a4_exists)
+		ctr_preload[1] = mpdu[30] & 0x0f;   /* QoC_Control */
+	if (qc_exists && !a4_exists)
+		ctr_preload[1] = mpdu[24] & 0x0f;
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+	/* 802.11w management frame should set management bit(4) */
+	if (frtype == WIFI_MGT_TYPE)
+		ctr_preload[1] |= BIT(4);
+#endif
+	for (i = 2; i < 8; i++)
+		ctr_preload[i] = mpdu[i + 8];                       /* ctr_preload[2:7] = A2[0:5] = mpdu[10:15] */
+#ifdef CONSISTENT_PN_ORDER
+	for (i = 8; i < 14; i++)
+		ctr_preload[i] =    pn_vector[i - 8];           /* ctr_preload[8:13] = PN[0:5] */
+#else
+	for (i = 8; i < 14; i++)
+		ctr_preload[i] =    pn_vector[13 - i];          /* ctr_preload[8:13] = PN[5:0] */
+#endif
+	ctr_preload[14] = (unsigned char)(c / 256);   /* Ctr */
+	ctr_preload[15] = (unsigned char)(c % 256);
+}
+
+
+/************************************/
+/* bitwise_xor()                   */
+/* A 128 bit, bitwise exclusive or */
+/************************************/
+static void bitwise_xor(u8 *ina, u8 *inb, u8 *out)
+{
+	sint i;
+	for (i = 0; i < 16; i++)
+		out[i] = ina[i] ^ inb[i];
+}
+
+
+static sint aes_cipher(u8 *key, uint	hdrlen,
+		       u8 *pframe, uint plen)
+{
+	/*	static unsigned char	message[MAX_MSG_SIZE]; */
+	uint	qc_exists, a4_exists, i, j, payload_remainder,
+		num_blocks, payload_index;
+
+	u8 pn_vector[6];
+	u8 mic_iv[16];
+	u8 mic_header1[16];
+	u8 mic_header2[16];
+	u8 ctr_preload[16];
+
+	/* Intermediate Buffers */
+	u8 chain_buffer[16];
+	u8 aes_out[16];
+	u8 padded_buffer[16];
+	u8 mic[8];
+	/*	uint	offset = 0; */
+	uint	frtype  = GetFrameType(pframe);
+	uint	frsubtype  = get_frame_sub_type(pframe);
+
+	frsubtype = frsubtype >> 4;
+
+
+	_rtw_memset((void *)mic_iv, 0, 16);
+	_rtw_memset((void *)mic_header1, 0, 16);
+	_rtw_memset((void *)mic_header2, 0, 16);
+	_rtw_memset((void *)ctr_preload, 0, 16);
+	_rtw_memset((void *)chain_buffer, 0, 16);
+	_rtw_memset((void *)aes_out, 0, 16);
+	_rtw_memset((void *)padded_buffer, 0, 16);
+
+	if ((hdrlen == WLAN_HDR_A3_LEN) || (hdrlen ==  WLAN_HDR_A3_QOS_LEN))
+		a4_exists = 0;
+	else
+		a4_exists = 1;
+
+	if (
+		((frtype | frsubtype) == WIFI_DATA_CFACK) ||
+		((frtype | frsubtype) == WIFI_DATA_CFPOLL) ||
+		((frtype | frsubtype) == WIFI_DATA_CFACKPOLL)) {
+		qc_exists = 1;
+		if (hdrlen != WLAN_HDR_A3_QOS_LEN && hdrlen != WLAN_HDR_A4_QOS_LEN)
+			hdrlen += 2;
+	}
+	/* add for CONFIG_IEEE80211W, none 11w also can use */
+	else if ((frtype == WIFI_DATA) &&
+		 ((frsubtype == 0x08) ||
+		  (frsubtype == 0x09) ||
+		  (frsubtype == 0x0a) ||
+		  (frsubtype == 0x0b))) {
+		if (hdrlen != WLAN_HDR_A3_QOS_LEN && hdrlen != WLAN_HDR_A4_QOS_LEN)
+			hdrlen += 2;
+		qc_exists = 1;
+	} else
+		qc_exists = 0;
+
+	pn_vector[0] = pframe[hdrlen];
+	pn_vector[1] = pframe[hdrlen + 1];
+	pn_vector[2] = pframe[hdrlen + 4];
+	pn_vector[3] = pframe[hdrlen + 5];
+	pn_vector[4] = pframe[hdrlen + 6];
+	pn_vector[5] = pframe[hdrlen + 7];
+
+	construct_mic_iv(
+		mic_iv,
+		qc_exists,
+		a4_exists,
+		pframe,	 /* message, */
+		plen,
+		pn_vector,
+		frtype /* add for CONFIG_IEEE80211W, none 11w also can use */
+	);
+
+	construct_mic_header1(
+		mic_header1,
+		hdrlen,
+		pframe,	/* message */
+		frtype /* add for CONFIG_IEEE80211W, none 11w also can use */
+	);
+	construct_mic_header2(
+		mic_header2,
+		pframe,	/* message, */
+		a4_exists,
+		qc_exists
+	);
+
+
+	payload_remainder = plen % 16;
+	num_blocks = plen / 16;
+
+	/* Find start of payload */
+	payload_index = (hdrlen + 8);
+
+	/* Calculate MIC */
+	aes128k128d(key, mic_iv, aes_out);
+	bitwise_xor(aes_out, mic_header1, chain_buffer);
+	aes128k128d(key, chain_buffer, aes_out);
+	bitwise_xor(aes_out, mic_header2, chain_buffer);
+	aes128k128d(key, chain_buffer, aes_out);
+
+	for (i = 0; i < num_blocks; i++) {
+		bitwise_xor(aes_out, &pframe[payload_index], chain_buffer);/* bitwise_xor(aes_out, &message[payload_index], chain_buffer); */
+
+		payload_index += 16;
+		aes128k128d(key, chain_buffer, aes_out);
+	}
+
+	/* Add on the final payload block if it needs padding */
+	if (payload_remainder > 0) {
+		for (j = 0; j < 16; j++)
+			padded_buffer[j] = 0x00;
+		for (j = 0; j < payload_remainder; j++) {
+			padded_buffer[j] = pframe[payload_index++];/* padded_buffer[j] = message[payload_index++]; */
+		}
+		bitwise_xor(aes_out, padded_buffer, chain_buffer);
+		aes128k128d(key, chain_buffer, aes_out);
+
+	}
+
+	for (j = 0 ; j < 8; j++)
+		mic[j] = aes_out[j];
+
+	/* Insert MIC into payload */
+	for (j = 0; j < 8; j++)
+		pframe[payload_index + j] = mic[j];	/* message[payload_index+j] = mic[j]; */
+
+	payload_index = hdrlen + 8;
+	for (i = 0; i < num_blocks; i++) {
+		construct_ctr_preload(
+			ctr_preload,
+			a4_exists,
+			qc_exists,
+			pframe,	/* message, */
+			pn_vector,
+			i + 1,
+			frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */
+		aes128k128d(key, ctr_preload, aes_out);
+		bitwise_xor(aes_out, &pframe[payload_index], chain_buffer);/* bitwise_xor(aes_out, &message[payload_index], chain_buffer); */
+		for (j = 0; j < 16; j++)
+			pframe[payload_index++] = chain_buffer[j];/* for (j=0; j<16;j++) message[payload_index++] = chain_buffer[j]; */
+	}
+
+	if (payload_remainder > 0) {        /* If there is a short final block, then pad it,*/
+		/* encrypt it and copy the unpadded part back  */
+		construct_ctr_preload(
+			ctr_preload,
+			a4_exists,
+			qc_exists,
+			pframe,	/* message, */
+			pn_vector,
+			num_blocks + 1,
+			frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */
+
+		for (j = 0; j < 16; j++)
+			padded_buffer[j] = 0x00;
+		for (j = 0; j < payload_remainder; j++) {
+			padded_buffer[j] = pframe[payload_index + j]; /* padded_buffer[j] = message[payload_index+j]; */
+		}
+		aes128k128d(key, ctr_preload, aes_out);
+		bitwise_xor(aes_out, padded_buffer, chain_buffer);
+		for (j = 0; j < payload_remainder; j++)
+			pframe[payload_index++] = chain_buffer[j];/* for (j=0; j<payload_remainder;j++) message[payload_index++] = chain_buffer[j]; */
+	}
+
+	/* Encrypt the MIC */
+	construct_ctr_preload(
+		ctr_preload,
+		a4_exists,
+		qc_exists,
+		pframe,	/* message, */
+		pn_vector,
+		0,
+		frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */
+
+	for (j = 0; j < 16; j++)
+		padded_buffer[j] = 0x00;
+	for (j = 0; j < 8; j++) {
+		padded_buffer[j] = pframe[j + hdrlen + 8 + plen]; /* padded_buffer[j] = message[j+hdrlen+8+plen]; */
+	}
+
+	aes128k128d(key, ctr_preload, aes_out);
+	bitwise_xor(aes_out, padded_buffer, chain_buffer);
+	for (j = 0; j < 8; j++)
+		pframe[payload_index++] = chain_buffer[j];/* for (j=0; j<8;j++) message[payload_index++] = chain_buffer[j]; */
+	return _SUCCESS;
+}
+#endif /* (NEW_CRYPTO == 0) */
+
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+u32 rtw_core_aes_encrypt(_adapter *padapter, u8 *pxmitframe)
+{
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pxframe = (struct xmit_frame *)pxmitframe;
+	sint curfragnum;
+	u32 prwskeylen;
+	u8 *pwlanhdr, *payload;
+	u8 *prwskey;
+	u8 hw_hdr_offset = 0;
+
+	u32 res = _SUCCESS;
+
+	if (((struct xmit_frame *)pxmitframe)->buf_addr == NULL)
+		return _FAIL;
+
+	pwlanhdr = pxframe->wlhdr[0];
+	payload = pxframe->pkt->data + pxframe->attrib.pkt_hdrlen - RTW_SZ_LLC;
+
+	/* start to encrypt each fragment */
+	if ((pxframe->attrib.encrypt == _AES_) ||
+	    (pxframe->attrib.encrypt == _CCMP_256_)) {
+
+		if (IS_MCAST(pxframe->attrib.ra))
+			prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey;
+		else {
+			prwskey = pxframe->attrib.psta->dot118021x_UncstKey.skey;
+		}
+
+#ifdef CONFIG_TDLS
+		{
+			/* Swencryption */
+			struct	sta_info		*ptdls_sta;
+			ptdls_sta = rtw_get_stainfo(&padapter->stapriv, pxframe->attrib.dst);
+			if ((ptdls_sta != NULL) && (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE)) {
+				RTW_INFO("[%s] for tdls link\n", __FUNCTION__);
+				prwskey = &ptdls_sta->tpk.tk[0];
+			}
+		}
+#endif /* CONFIG_TDLS */
+
+		prwskeylen = (pxframe->attrib.encrypt == _CCMP_256_) ? 32 : 16;
+
+		for (curfragnum = 0; curfragnum < pxframe->attrib.nr_frags; curfragnum++) {
+			u8 *pdata = pxframe->pkt->data + pxframe->attrib.pkt_hdrlen - RTW_SZ_LLC + curfragnum*pxframe->attrib.frag_datalen;
+			
+			if ((curfragnum + 1) == pxframe->attrib.nr_frags) {    /* the last fragment */
+				u32 ls_datelen = pxframe->pkt->len - curfragnum*pxframe->attrib.frag_datalen;
+				_rtw_core_ccmp_encrypt(prwskey, prwskeylen, 
+					pxframe->attrib.hdrlen, pxframe->wlhdr[curfragnum], ls_datelen, pdata);
+			} else {
+				_rtw_core_ccmp_encrypt(prwskey, prwskeylen, 
+					pxframe->attrib.hdrlen, pxframe->wlhdr[curfragnum], pxframe->attrib.frag_datalen, pdata);
+			}
+		}
+
+		AES_SW_ENC_CNT_INC(psecuritypriv, pxframe->attrib.ra);
+
+	}
+
+	return res;
+}
+#endif
+
+u32 rtw_aes_encrypt(_adapter *padapter, u8 *pxmitframe)
+{
+	struct xmit_frame *xf = (struct xmit_frame *)pxmitframe;
+	struct pkt_attrib *pattrib = &xf->attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct rtw_xmit_req *txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	sint curfragnum, plen;
+	u32 prwskeylen;
+	u8 *pframe;
+	u8 *prwskey;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+
+	if ((pattrib->encrypt != _AES_) && (pattrib->encrypt != _CCMP_256_))
+		return _SUCCESS;
+
+	/* start to encrypt each fragment */
+	if (IS_MCAST(pattrib->ra))
+		prwskey = lsecuritypriv->dot118021XGrpKey[lsecuritypriv->dot118021XGrpKeyid].skey;
+	else
+		prwskey = pattrib->dot118021x_UncstKey.skey;
+
+#ifdef CONFIG_TDLS
+	{
+		/* Swencryption */
+		struct	sta_info		*ptdls_sta;
+		ptdls_sta = rtw_get_stainfo(&padapter->stapriv, &pattrib->dst[0]);
+		if ((ptdls_sta != NULL) && (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE)) {
+			RTW_INFO("[%s] for tdls link\n", __FUNCTION__);
+			prwskey = &ptdls_sta->tpk.tk[0];
+		}
+	}
+#endif /* CONFIG_TDLS */
+
+	prwskeylen = (pattrib->encrypt == _CCMP_256_) ? 32 : 16;
+
+	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
+		txreq = &xf->phl_txreq[curfragnum];
+		rtw_warn_on(txreq->pkt_cnt != 1);
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		pframe = pkt_list->vir_addr;
+		plen = pkt_list->length - pattrib->hdrlen
+		       - pattrib->iv_len - pattrib->icv_len;
+		_rtw_ccmp_encrypt(padapter, prwskey, prwskeylen, pattrib->hdrlen,
+				  pframe, plen);
+	}
+
+	AES_SW_ENC_CNT_INC(psecuritypriv, pattrib->ra);
+
+	return _SUCCESS;
+}
+
+#if (NEW_CRYPTO == 0)
+static sint aes_decipher(u8 *key, uint	hdrlen,
+			 u8 *pframe, uint plen)
+{
+	static u8	message[MAX_MSG_SIZE];
+	uint	qc_exists, a4_exists, i, j, payload_remainder,
+		num_blocks, payload_index;
+	sint res = _SUCCESS;
+	u8 pn_vector[6];
+	u8 mic_iv[16];
+	u8 mic_header1[16];
+	u8 mic_header2[16];
+	u8 ctr_preload[16];
+
+	/* Intermediate Buffers */
+	u8 chain_buffer[16];
+	u8 aes_out[16];
+	u8 padded_buffer[16];
+	u8 mic[8];
+
+
+	/*	uint	offset = 0; */
+	uint	frtype  = GetFrameType(pframe);
+	uint	frsubtype  = get_frame_sub_type(pframe);
+	frsubtype = frsubtype >> 4;
+
+
+	_rtw_memset((void *)mic_iv, 0, 16);
+	_rtw_memset((void *)mic_header1, 0, 16);
+	_rtw_memset((void *)mic_header2, 0, 16);
+	_rtw_memset((void *)ctr_preload, 0, 16);
+	_rtw_memset((void *)chain_buffer, 0, 16);
+	_rtw_memset((void *)aes_out, 0, 16);
+	_rtw_memset((void *)padded_buffer, 0, 16);
+
+	/* start to decrypt the payload */
+
+	num_blocks = (plen - 8) / 16; /* (plen including LLC, payload_length and mic ) */
+
+	payload_remainder = (plen - 8) % 16;
+
+	pn_vector[0]  = pframe[hdrlen];
+	pn_vector[1]  = pframe[hdrlen + 1];
+	pn_vector[2]  = pframe[hdrlen + 4];
+	pn_vector[3]  = pframe[hdrlen + 5];
+	pn_vector[4]  = pframe[hdrlen + 6];
+	pn_vector[5]  = pframe[hdrlen + 7];
+
+	if ((hdrlen == WLAN_HDR_A3_LEN) || (hdrlen ==  WLAN_HDR_A3_QOS_LEN))
+		a4_exists = 0;
+	else
+		a4_exists = 1;
+
+	if (
+		((frtype | frsubtype) == WIFI_DATA_CFACK) ||
+		((frtype | frsubtype) == WIFI_DATA_CFPOLL) ||
+		((frtype | frsubtype) == WIFI_DATA_CFACKPOLL)) {
+		qc_exists = 1;
+		if (hdrlen != WLAN_HDR_A3_QOS_LEN && hdrlen != WLAN_HDR_A4_QOS_LEN)
+			hdrlen += 2;
+	} /* only for data packet . add for CONFIG_IEEE80211W, none 11w also can use */
+	else if ((frtype == WIFI_DATA) &&
+		 ((frsubtype == 0x08) ||
+		  (frsubtype == 0x09) ||
+		  (frsubtype == 0x0a) ||
+		  (frsubtype == 0x0b))) {
+		if (hdrlen != WLAN_HDR_A3_QOS_LEN && hdrlen != WLAN_HDR_A4_QOS_LEN)
+			hdrlen += 2;
+		qc_exists = 1;
+	} else
+		qc_exists = 0;
+
+
+	/* now, decrypt pframe with hdrlen offset and plen long */
+
+	payload_index = hdrlen + 8; /* 8 is for extiv */
+
+	for (i = 0; i < num_blocks; i++) {
+		construct_ctr_preload(
+			ctr_preload,
+			a4_exists,
+			qc_exists,
+			pframe,
+			pn_vector,
+			i + 1,
+			frtype /* add for CONFIG_IEEE80211W, none 11w also can use */
+		);
+
+		aes128k128d(key, ctr_preload, aes_out);
+		bitwise_xor(aes_out, &pframe[payload_index], chain_buffer);
+
+		for (j = 0; j < 16; j++)
+			pframe[payload_index++] = chain_buffer[j];
+	}
+
+	if (payload_remainder > 0) {        /* If there is a short final block, then pad it,*/
+		/* encrypt it and copy the unpadded part back  */
+		construct_ctr_preload(
+			ctr_preload,
+			a4_exists,
+			qc_exists,
+			pframe,
+			pn_vector,
+			num_blocks + 1,
+			frtype /* add for CONFIG_IEEE80211W, none 11w also can use */
+		);
+
+		for (j = 0; j < 16; j++)
+			padded_buffer[j] = 0x00;
+		for (j = 0; j < payload_remainder; j++)
+			padded_buffer[j] = pframe[payload_index + j];
+		aes128k128d(key, ctr_preload, aes_out);
+		bitwise_xor(aes_out, padded_buffer, chain_buffer);
+		for (j = 0; j < payload_remainder; j++)
+			pframe[payload_index++] = chain_buffer[j];
+	}
+
+	/* start to calculate the mic	 */
+	if ((hdrlen + plen + 8) <= MAX_MSG_SIZE)
+		_rtw_memcpy((void *)message, pframe, (hdrlen + plen + 8)); /* 8 is for ext iv len */
+
+
+	pn_vector[0] = pframe[hdrlen];
+	pn_vector[1] = pframe[hdrlen + 1];
+	pn_vector[2] = pframe[hdrlen + 4];
+	pn_vector[3] = pframe[hdrlen + 5];
+	pn_vector[4] = pframe[hdrlen + 6];
+	pn_vector[5] = pframe[hdrlen + 7];
+
+
+
+	construct_mic_iv(
+		mic_iv,
+		qc_exists,
+		a4_exists,
+		message,
+		plen - 8,
+		pn_vector,
+		frtype /* add for CONFIG_IEEE80211W, none 11w also can use */
+	);
+
+	construct_mic_header1(
+		mic_header1,
+		hdrlen,
+		message,
+		frtype /* add for CONFIG_IEEE80211W, none 11w also can use */
+	);
+	construct_mic_header2(
+		mic_header2,
+		message,
+		a4_exists,
+		qc_exists
+	);
+
+
+	payload_remainder = (plen - 8) % 16;
+	num_blocks = (plen - 8) / 16;
+
+	/* Find start of payload */
+	payload_index = (hdrlen + 8);
+
+	/* Calculate MIC */
+	aes128k128d(key, mic_iv, aes_out);
+	bitwise_xor(aes_out, mic_header1, chain_buffer);
+	aes128k128d(key, chain_buffer, aes_out);
+	bitwise_xor(aes_out, mic_header2, chain_buffer);
+	aes128k128d(key, chain_buffer, aes_out);
+
+	for (i = 0; i < num_blocks; i++) {
+		bitwise_xor(aes_out, &message[payload_index], chain_buffer);
+
+		payload_index += 16;
+		aes128k128d(key, chain_buffer, aes_out);
+	}
+
+	/* Add on the final payload block if it needs padding */
+	if (payload_remainder > 0) {
+		for (j = 0; j < 16; j++)
+			padded_buffer[j] = 0x00;
+		for (j = 0; j < payload_remainder; j++)
+			padded_buffer[j] = message[payload_index++];
+		bitwise_xor(aes_out, padded_buffer, chain_buffer);
+		aes128k128d(key, chain_buffer, aes_out);
+
+	}
+
+	for (j = 0 ; j < 8; j++)
+		mic[j] = aes_out[j];
+
+	/* Insert MIC into payload */
+	for (j = 0; j < 8; j++)
+		message[payload_index + j] = mic[j];
+
+	payload_index = hdrlen + 8;
+	for (i = 0; i < num_blocks; i++) {
+		construct_ctr_preload(
+			ctr_preload,
+			a4_exists,
+			qc_exists,
+			message,
+			pn_vector,
+			i + 1,
+			frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */
+		aes128k128d(key, ctr_preload, aes_out);
+		bitwise_xor(aes_out, &message[payload_index], chain_buffer);
+		for (j = 0; j < 16; j++)
+			message[payload_index++] = chain_buffer[j];
+	}
+
+	if (payload_remainder > 0) {        /* If there is a short final block, then pad it,*/
+		/* encrypt it and copy the unpadded part back  */
+		construct_ctr_preload(
+			ctr_preload,
+			a4_exists,
+			qc_exists,
+			message,
+			pn_vector,
+			num_blocks + 1,
+			frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */
+
+		for (j = 0; j < 16; j++)
+			padded_buffer[j] = 0x00;
+		for (j = 0; j < payload_remainder; j++)
+			padded_buffer[j] = message[payload_index + j];
+		aes128k128d(key, ctr_preload, aes_out);
+		bitwise_xor(aes_out, padded_buffer, chain_buffer);
+		for (j = 0; j < payload_remainder; j++)
+			message[payload_index++] = chain_buffer[j];
+	}
+
+	/* Encrypt the MIC */
+	construct_ctr_preload(
+		ctr_preload,
+		a4_exists,
+		qc_exists,
+		message,
+		pn_vector,
+		0,
+		frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */
+
+	for (j = 0; j < 16; j++)
+		padded_buffer[j] = 0x00;
+	for (j = 0; j < 8; j++)
+		padded_buffer[j] = message[j + hdrlen + 8 + plen - 8];
+
+	aes128k128d(key, ctr_preload, aes_out);
+	bitwise_xor(aes_out, padded_buffer, chain_buffer);
+	for (j = 0; j < 8; j++)
+		message[payload_index++] = chain_buffer[j];
+
+	/* compare the mic */
+	for (i = 0; i < 8; i++) {
+		if (pframe[hdrlen + 8 + plen - 8 + i] != message[hdrlen + 8 + plen - 8 + i]) {
+			RTW_INFO("aes_decipher:mic check error mic[%d]: pframe(%x) != message(%x)\n",
+				i, pframe[hdrlen + 8 + plen - 8 + i], message[hdrlen + 8 + plen - 8 + i]);
+			res = _FAIL;
+		}
+	}
+	return res;
+}
+#endif /* (NEW_CRYPTO == 0) */
+
+#if NEW_CRYPTO
+u32 rtw_aes_decrypt(_adapter *padapter, u8 *precvframe)
+{
+	struct sta_info *stainfo;
+	struct rx_pkt_attrib *prxattrib = &((union recv_frame *)precvframe)->u.hdr.attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	u8 *pframe;
+	u8 *prwskey;
+	u32 res = _SUCCESS;
+	struct _ADAPTER_LINK *padapter_link = ((union recv_frame *)precvframe)->u.hdr.adapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+	pframe = (unsigned char *)((union recv_frame *)precvframe)->u.hdr.rx_data;
+	/* start to encrypt each fragment */
+	if ((prxattrib->encrypt == _AES_) ||
+	    (prxattrib->encrypt == _CCMP_256_)) {
+
+		stainfo = ((union recv_frame *)precvframe)->u.hdr.psta;
+		if (stainfo != NULL) {
+
+			if (IS_MCAST(prxattrib->ra)) {
+				static systime start = 0;
+				static u32 no_gkey_bc_cnt = 0;
+				static u32 no_gkey_mc_cnt = 0;
+
+				if ((!MLME_IS_MESH(padapter)
+					&& lsecuritypriv->binstallGrpkey == _FALSE)
+					#ifdef CONFIG_RTW_MESH
+					|| !(stainfo->gtk_bmp | BIT(prxattrib->key_index))
+					#endif
+				) {
+					res = _FAIL;
+
+					if (start == 0)
+						start = rtw_get_current_time();
+
+					if (is_broadcast_mac_addr(prxattrib->ra))
+						no_gkey_bc_cnt++;
+					else
+						no_gkey_mc_cnt++;
+
+					if (rtw_get_passing_time_ms(start) > 1000) {
+						if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+							RTW_PRINT(FUNC_ADPT_FMT" no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+								FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+						}
+						start = rtw_get_current_time();
+						no_gkey_bc_cnt = 0;
+						no_gkey_mc_cnt = 0;
+					}
+
+					goto exit;
+				}
+
+				if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+					RTW_PRINT(FUNC_ADPT_FMT" gkey installed. no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+						FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+				}
+				start = 0;
+				no_gkey_bc_cnt = 0;
+				no_gkey_mc_cnt = 0;
+
+				#ifdef CONFIG_RTW_MESH
+				if (MLME_IS_MESH(padapter)) {
+					/* TODO: multiple GK? */
+					prwskey = &stainfo->gtk.skey[0];
+				} else
+				#endif
+				{
+					prwskey = lsecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
+					if (lsecuritypriv->dot118021XGrpKeyid != prxattrib->key_index)
+					{
+						RTW_DBG("not match packet_index=%d, install_index=%d\n"
+							, prxattrib->key_index, lsecuritypriv->dot118021XGrpKeyid);
+						res = _FAIL;
+						goto exit;
+					}
+				}
+			} else
+				prwskey = &stainfo->dot118021x_UncstKey.skey[0];
+
+			res = _rtw_ccmp_decrypt(padapter, prwskey,
+				prxattrib->encrypt == _CCMP_256_ ? 32 : 16,
+				prxattrib->hdrlen, pframe,
+				((union recv_frame *)precvframe)->u.hdr.len);
+
+			AES_SW_DEC_CNT_INC(psecuritypriv, prxattrib->ra);
+		} else {
+			res = _FAIL;
+		}
+
+	}
+exit:
+	return res;
+}
+#else
+u32	rtw_aes_decrypt(_adapter *padapter, u8 *precvframe)
+{
+	/* exclude ICV */
+
+
+	/*static*/
+	/*	unsigned char	message[MAX_MSG_SIZE]; */
+
+
+	/* Intermediate Buffers */
+
+
+	sint		length;
+	u8	*pframe, *prwskey;	/* , *payload,*iv */
+	struct	sta_info		*stainfo;
+	struct	rx_pkt_attrib	*prxattrib = &((union recv_frame *)precvframe)->u.hdr.attrib;
+	struct	security_priv	*psecuritypriv = &padapter->securitypriv;
+	u32	res = _SUCCESS;
+	pframe = (unsigned char *)((union recv_frame *)precvframe)->u.hdr.rx_data;
+	struct _ADAPTER_LINK *padapter_link = ((union recv_frame *)precvframe)->u.hdr.adapter_link;
+
+	/* 4 start to encrypt each fragment */
+	if ((prxattrib->encrypt == _AES_)) {
+
+		stainfo = ((union recv_frame *)precvframe)->u.hdr.psta;
+		if (stainfo != NULL) {
+
+			if (IS_MCAST(prxattrib->ra)) {
+				static systime start = 0;
+				static u32 no_gkey_bc_cnt = 0;
+				static u32 no_gkey_mc_cnt = 0;
+
+				/* RTW_INFO("rx bc/mc packets, to perform sw rtw_aes_decrypt\n"); */
+				/* prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey; */
+				if ((!MLME_IS_MESH(padapter)
+					&& lsecuritypriv->binstallGrpkey == _FALSE)
+					#ifdef CONFIG_RTW_MESH
+					|| !(stainfo->gtk_bmp | BIT(prxattrib->key_index))
+					#endif
+				) {
+					res = _FAIL;
+
+					if (start == 0)
+						start = rtw_get_current_time();
+
+					if (is_broadcast_mac_addr(prxattrib->ra))
+						no_gkey_bc_cnt++;
+					else
+						no_gkey_mc_cnt++;
+
+					if (rtw_get_passing_time_ms(start) > 1000) {
+						if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+							RTW_PRINT(FUNC_ADPT_FMT" no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+								FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+						}
+						start = rtw_get_current_time();
+						no_gkey_bc_cnt = 0;
+						no_gkey_mc_cnt = 0;
+					}
+
+					goto exit;
+				}
+
+				if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+					RTW_PRINT(FUNC_ADPT_FMT" gkey installed. no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+						FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+				}
+				start = 0;
+				no_gkey_bc_cnt = 0;
+				no_gkey_mc_cnt = 0;
+
+				#ifdef CONFIG_RTW_MESH
+				if (MLME_IS_MESH(padapter)) {
+					/* TODO: multiple GK? */
+					prwskey = &stainfo->gtk.skey[0];
+				} else
+				#endif
+				{
+					prwskey = lsecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
+					if (lsecuritypriv->dot118021XGrpKeyid != prxattrib->key_index) {
+						RTW_DBG("not match packet_index=%d, install_index=%d\n"
+							, prxattrib->key_index, lsecuritypriv->dot118021XGrpKeyid);
+						res = _FAIL;
+						goto exit;
+					}
+				}
+			} else
+				prwskey = &stainfo->dot118021x_UncstKey.skey[0];
+
+			length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len;
+#if 0
+			/*  add for CONFIG_IEEE80211W, debug */
+			if (0)
+				printk("@@@@@@@@@@@@@@@@@@ length=%d, prxattrib->hdrlen=%d, prxattrib->pkt_len=%d\n"
+				       , length, prxattrib->hdrlen, prxattrib->pkt_len);
+			if (0) {
+				int no;
+				/* test print PSK */
+				printk("PSK key below:\n");
+				for (no = 0; no < 16; no++)
+					printk(" %02x ", prwskey[no]);
+				printk("\n");
+			}
+			if (0) {
+				int no;
+				/* test print PSK */
+				printk("frame:\n");
+				for (no = 0; no < prxattrib->pkt_len; no++)
+					printk(" %02x ", pframe[no]);
+				printk("\n");
+			}
+#endif
+
+			res = aes_decipher(prwskey, prxattrib->hdrlen, pframe, length);
+
+			AES_SW_DEC_CNT_INC(psecuritypriv, prxattrib->ra);
+		} else {
+			res = _FAIL;
+		}
+
+	}
+exit:
+	return res;
+}
+#endif
+
+#ifdef CONFIG_RTW_MESH_AEK
+/* for AES-SIV, wrapper to ase_siv_encrypt and aes_siv_decrypt */
+int rtw_aes_siv_encrypt(const u8 *key, size_t key_len, const u8 *pw,
+	size_t pwlen, size_t num_elem,
+	const u8 *addr[], const size_t *len, u8 *out)
+{
+	return _aes_siv_encrypt(key, key_len, pw, pwlen,
+		num_elem, addr, len, out);
+}
+
+int rtw_aes_siv_decrypt(const u8 *key, size_t key_len, const u8 *iv_crypt, size_t iv_c_len,
+	size_t num_elem, const u8 *addr[], const size_t *len, u8 *out)
+{
+	return _aes_siv_decrypt(key, key_len, iv_crypt,
+		iv_c_len, num_elem, addr, len, out);
+}
+#endif /* CONFIG_RTW_MESH_AEK */
+
+#ifdef CONFIG_TDLS
+void wpa_tdls_generate_tpk(_adapter *padapter, void *sta)
+{
+	struct sta_info *psta = (struct sta_info *)sta;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+	_tdls_generate_tpk(psta, adapter_mac_addr(padapter), get_bssid(pmlmepriv));
+}
+
+/**
+ * wpa_tdls_ftie_mic - Calculate TDLS FTIE MIC
+ * @kck: TPK-KCK
+ * @lnkid: Pointer to the beginning of Link Identifier IE
+ * @rsnie: Pointer to the beginning of RSN IE used for handshake
+ * @timeoutie: Pointer to the beginning of Timeout IE used for handshake
+ * @ftie: Pointer to the beginning of FT IE
+ * @mic: Pointer for writing MIC
+ *
+ * Calculate MIC for TDLS frame.
+ */
+int wpa_tdls_ftie_mic(u8 *kck, u8 trans_seq,
+		      u8 *lnkid, u8 *rsnie, u8 *timeoutie, u8 *ftie,
+		      u8 *mic)
+{
+	u8 *buf, *pos;
+	struct wpa_tdls_ftie *_ftie;
+	struct wpa_tdls_lnkid *_lnkid;
+	int ret;
+	int len = 2 * ETH_ALEN + 1 + 2 + lnkid[1] + 2 + rsnie[1] +
+		  2 + timeoutie[1] + 2 + ftie[1];
+	buf = rtw_zmalloc(len);
+	if (!buf) {
+		RTW_INFO("TDLS: No memory for MIC calculation\n");
+		return -1;
+	}
+
+	pos = buf;
+	_lnkid = (struct wpa_tdls_lnkid *) lnkid;
+	/* 1) TDLS initiator STA MAC address */
+	_rtw_memcpy(pos, _lnkid->init_sta, ETH_ALEN);
+	pos += ETH_ALEN;
+	/* 2) TDLS responder STA MAC address */
+	_rtw_memcpy(pos, _lnkid->resp_sta, ETH_ALEN);
+	pos += ETH_ALEN;
+	/* 3) Transaction Sequence number */
+	*pos++ = trans_seq;
+	/* 4) Link Identifier IE */
+	_rtw_memcpy(pos, lnkid, 2 + lnkid[1]);
+	pos += 2 + lnkid[1];
+	/* 5) RSN IE */
+	_rtw_memcpy(pos, rsnie, 2 + rsnie[1]);
+	pos += 2 + rsnie[1];
+	/* 6) Timeout Interval IE */
+	_rtw_memcpy(pos, timeoutie, 2 + timeoutie[1]);
+	pos += 2 + timeoutie[1];
+	/* 7) FTIE, with the MIC field of the FTIE set to 0 */
+	_rtw_memcpy(pos, ftie, 2 + ftie[1]);
+	_ftie = (struct wpa_tdls_ftie *) pos;
+	_rtw_memset(_ftie->mic, 0, TDLS_MIC_LEN);
+	pos += 2 + ftie[1];
+
+	/* ret = omac1_aes_128(kck, buf, pos - buf, mic); */
+	ret = _bip_ccmp_protect(kck, 16, buf, pos - buf, mic);
+	rtw_mfree(buf, len);
+	return ret;
+
+}
+
+/**
+ * wpa_tdls_teardown_ftie_mic - Calculate TDLS TEARDOWN FTIE MIC
+ * @kck: TPK-KCK
+ * @lnkid: Pointer to the beginning of Link Identifier IE
+ * @reason: Reason code of TDLS Teardown
+ * @dialog_token: Dialog token that was used in the MIC calculation for TPK Handshake Message 3
+ * @trans_seq: Transaction Sequence number (1 octet) which shall be set to the value 4
+ * @ftie: Pointer to the beginning of FT IE
+ * @mic: Pointer for writing MIC
+ *
+ * Calculate MIC for TDLS TEARDOWN frame according to Section 10.22.5 in IEEE 802.11 - 2012.
+ */
+int wpa_tdls_teardown_ftie_mic(u8 *kck, u8 *lnkid, u16 reason,
+			       u8 dialog_token, u8 trans_seq, u8 *ftie, u8 *mic)
+{
+	u8 *buf, *pos;
+	struct wpa_tdls_ftie *_ftie;
+	int ret;
+	int len = 2 + lnkid[1] + 2 + 1 + 1 + 2 + ftie[1];
+
+	buf = rtw_zmalloc(len);
+	if (!buf) {
+		RTW_INFO("TDLS: No memory for MIC calculation\n");
+		return -1;
+	}
+
+	pos = buf;
+	/* 1) Link Identifier IE */
+	_rtw_memcpy(pos, lnkid, 2 + lnkid[1]);
+	pos += 2 + lnkid[1];
+	/* 2) Reason Code */
+	_rtw_memcpy(pos, (u8 *)&reason, 2);
+	pos += 2;
+	/* 3) Dialog Token */
+	*pos++ = dialog_token;
+	/* 4) Transaction Sequence number */
+	*pos++ = trans_seq;
+	/* 5) FTIE, with the MIC field of the FTIE set to 0 */
+	_rtw_memcpy(pos, ftie, 2 + ftie[1]);
+	_ftie = (struct wpa_tdls_ftie *) pos;
+	_rtw_memset(_ftie->mic, 0, TDLS_MIC_LEN);
+	pos += 2 + ftie[1];
+
+	/* ret = omac1_aes_128(kck, buf, pos - buf, mic); */
+	ret = _bip_ccmp_protect(kck, 16, buf, pos - buf, mic);
+	rtw_mfree(buf, len);
+	return ret;
+
+}
+
+int tdls_verify_mic(u8 *kck, u8 trans_seq,
+		    u8 *lnkid, u8 *rsnie, u8 *timeoutie, u8 *ftie)
+{
+	u8 *buf, *pos;
+	int len;
+	u8 mic[16];
+	int ret;
+	u8 *rx_ftie, *tmp_ftie;
+
+	if (lnkid == NULL || rsnie == NULL ||
+	    timeoutie == NULL || ftie == NULL)
+		return _FAIL;
+
+	len = 2 * ETH_ALEN + 1 + 2 + 18 + 2 + *(rsnie + 1) + 2 + *(timeoutie + 1) + 2 + *(ftie + 1);
+
+	buf = rtw_zmalloc(len);
+	if (buf == NULL)
+		return _FAIL;
+
+	pos = buf;
+	/* 1) TDLS initiator STA MAC address */
+	_rtw_memcpy(pos, lnkid + ETH_ALEN + 2, ETH_ALEN);
+	pos += ETH_ALEN;
+	/* 2) TDLS responder STA MAC address */
+	_rtw_memcpy(pos, lnkid + 2 * ETH_ALEN + 2, ETH_ALEN);
+	pos += ETH_ALEN;
+	/* 3) Transaction Sequence number */
+	*pos++ = trans_seq;
+	/* 4) Link Identifier IE */
+	_rtw_memcpy(pos, lnkid, 2 + 18);
+	pos += 2 + 18;
+	/* 5) RSN IE */
+	_rtw_memcpy(pos, rsnie, 2 + *(rsnie + 1));
+	pos += 2 + *(rsnie + 1);
+	/* 6) Timeout Interval IE */
+	_rtw_memcpy(pos, timeoutie, 2 + *(timeoutie + 1));
+	pos += 2 + *(timeoutie + 1);
+	/* 7) FTIE, with the MIC field of the FTIE set to 0 */
+	_rtw_memcpy(pos, ftie, 2 + *(ftie + 1));
+	pos += 2;
+	tmp_ftie = (u8 *)(pos + 2);
+	_rtw_memset(tmp_ftie, 0, 16);
+	pos += *(ftie + 1);
+
+	/* ret = omac1_aes_128(kck, buf, pos - buf, mic); */
+	ret = _bip_ccmp_protect(kck, 16, buf, pos - buf, mic);
+	rtw_mfree(buf, len);
+	if (ret == _FAIL)
+		return _FAIL;
+	rx_ftie = ftie + 4;
+
+	if (_rtw_memcmp2(mic, rx_ftie, 16) == 0) {
+		/* Valid MIC */
+		return _SUCCESS;
+	}
+
+	/* Invalid MIC */
+	RTW_INFO("[%s] Invalid MIC\n", __FUNCTION__);
+	return _FAIL;
+
+}
+#endif /* CONFIG_TDLS */
+
+/* Restore HW wep key setting according to key_mask */
+void rtw_sec_restore_wep_key(_adapter *adapter)
+{
+	struct security_priv *securitypriv = &(adapter->securitypriv);
+	sint keyid;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only]
+	** but currently no one call this function */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	if ((_WEP40_ == securitypriv->dot11PrivacyAlgrthm) || (_WEP104_ == securitypriv->dot11PrivacyAlgrthm)) {
+		for (keyid = 0; keyid < 4; keyid++) {
+			if (securitypriv->key_mask & BIT(keyid)) {
+				if (keyid == securitypriv->dot11PrivacyKeyIndex)
+					rtw_set_key(adapter, adapter_link, keyid, 1, _FALSE);
+				else
+					rtw_set_key(adapter, adapter_link, keyid, 0, _FALSE);
+			}
+		}
+	}
+}
+
+u8 rtw_handle_tkip_countermeasure(_adapter *adapter, const char *caller)
+{
+	struct security_priv *securitypriv = &(adapter->securitypriv);
+	u8 status = _SUCCESS;
+
+	if (securitypriv->btkip_countermeasure == _TRUE) {
+		u32 passing_ms = rtw_get_passing_time_ms(securitypriv->btkip_countermeasure_time);
+		if (passing_ms > 60 * 1000) {
+			RTW_PRINT("%s("ADPT_FMT") countermeasure time:%ds > 60s\n",
+				  caller, ADPT_ARG(adapter), passing_ms / 1000);
+			securitypriv->btkip_countermeasure = _FALSE;
+			securitypriv->btkip_countermeasure_time = 0;
+		} else {
+			RTW_PRINT("%s("ADPT_FMT") countermeasure time:%ds < 60s\n",
+				  caller, ADPT_ARG(adapter), passing_ms / 1000);
+			status = _FAIL;
+		}
+	}
+
+	return status;
+}
+
+#ifdef CONFIG_WOWLAN
+u16 rtw_cal_crc16(u8 data, u16 crc)
+{
+	u8 shift_in, data_bit;
+	u8 crc_bit4, crc_bit11, crc_bit15;
+	u16 crc_result;
+	int index;
+
+	for (index = 0; index < 8; index++) {
+		crc_bit15 = ((crc & BIT15) ? 1 : 0);
+		data_bit = (data & (BIT0 << index) ? 1 : 0);
+		shift_in = crc_bit15 ^ data_bit;
+		/*printf("crc_bit15=%d, DataBit=%d, shift_in=%d\n",
+		 * crc_bit15, data_bit, shift_in);*/
+
+		crc_result = crc << 1;
+
+		if (shift_in == 0)
+			crc_result &= (~BIT0);
+		else
+			crc_result |= BIT0;
+		/*printf("CRC =%x\n",CRC_Result);*/
+
+		crc_bit11 = ((crc & BIT11) ? 1 : 0) ^ shift_in;
+
+		if (crc_bit11 == 0)
+			crc_result &= (~BIT12);
+		else
+			crc_result |= BIT12;
+
+		/*printf("bit12 CRC =%x\n",CRC_Result);*/
+
+		crc_bit4 = ((crc & BIT4) ? 1 : 0) ^ shift_in;
+
+		if (crc_bit4 == 0)
+			crc_result &= (~BIT5);
+		else
+			crc_result |= BIT5;
+
+		/* printf("bit5 CRC =%x\n",CRC_Result); */
+		/* repeat using the last result*/
+		crc = crc_result;
+	}
+	return crc;
+}
+
+/*
+ * function name :rtw_calc_crc
+ *
+ * input: char* pattern , pattern size
+ *
+ */
+u16 rtw_calc_crc(u8  *pdata, int length)
+{
+	u16 crc = 0xffff;
+	int i;
+
+	for (i = 0; i < length; i++)
+		crc = rtw_cal_crc16(pdata[i], crc);
+	/* get 1' complement */
+	crc = ~crc;
+
+	return crc;
+}
+#endif /*CONFIG_WOWLAN*/
+
+u32 rtw_calc_crc32(u8 *data, size_t len)
+{
+	size_t i;
+	u32 crc = 0xFFFFFFFF;
+
+	if (bcrc32initialized == 0)
+		crc32_init();
+
+	for (i = 0; i < len; i++)
+		crc = crc32_table[(crc ^ data[i]) & 0xff] ^ (crc >> 8);
+
+	/* return 1' complement */
+	return ~crc;
+}
+
+
+/**
+ * rtw_gcmp_encrypt - 
+ * @padapter:
+ * @pxmitframe:
+ *
+ */
+u32 rtw_gcmp_encrypt(_adapter *padapter, u8 *pxmitframe)
+{
+	struct xmit_frame *xf = (struct xmit_frame *)pxmitframe;
+	struct pkt_attrib *pattrib = &xf->attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct rtw_xmit_req *txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	sint curfragnum, plen;
+	u32 prwskeylen;
+	u8 *pframe = NULL;
+	u8 *prwskey = NULL;
+	u8 hw_hdr_offset = 0;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+
+	if ((pattrib->encrypt != _GCMP_) && (pattrib->encrypt != _GCMP_256_))
+		return _SUCCESS;
+
+	/* start to encrypt each fragment */
+	if (IS_MCAST(pattrib->ra))
+		prwskey = lsecuritypriv->dot118021XGrpKey[lsecuritypriv->dot118021XGrpKeyid].skey;
+	else
+		prwskey = pattrib->dot118021x_UncstKey.skey;
+	prwskeylen = (pattrib->encrypt == _GCMP_256_) ? 32 : 16;
+
+	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
+		txreq = &xf->phl_txreq[curfragnum];
+		rtw_warn_on(txreq->pkt_cnt != 1);
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		pframe = pkt_list->vir_addr;
+		plen = pkt_list->length - pattrib->hdrlen
+		       - pattrib->iv_len - pattrib->icv_len;
+		_rtw_gcmp_encrypt(padapter, prwskey, prwskeylen, pattrib->hdrlen,
+				  pframe, plen);
+	}
+
+	GCMP_SW_ENC_CNT_INC(psecuritypriv, pattrib->ra);
+
+	return _SUCCESS;
+}
+
+u32 rtw_gcmp_decrypt(_adapter *padapter, u8 *precvframe)
+{
+	u32 prwskeylen;
+	u8 * pframe,*prwskey;
+	struct sta_info *stainfo;
+	struct rx_pkt_attrib *prxattrib = &((union recv_frame *)precvframe)->u.hdr.attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	u32 res = _SUCCESS;
+	struct _ADAPTER_LINK *padapter_link = ((union recv_frame *)precvframe)->u.hdr.adapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+	pframe = (unsigned char *)((union recv_frame *)precvframe)->u.hdr.rx_data;
+
+	if ((prxattrib->encrypt == _GCMP_) ||
+		(prxattrib->encrypt == _GCMP_256_)) {
+		stainfo = ((union recv_frame *)precvframe)->u.hdr.psta;
+		if (stainfo != NULL) {
+			if (IS_MCAST(prxattrib->ra)) {
+				static systime start = 0;
+				static u32 no_gkey_bc_cnt = 0;
+				static u32 no_gkey_mc_cnt = 0;
+
+				if ((!MLME_IS_MESH(padapter)
+					&& lsecuritypriv->binstallGrpkey == _FALSE)
+					#ifdef CONFIG_RTW_MESH
+					|| !(stainfo->gtk_bmp | BIT(prxattrib->key_index))
+					#endif
+				) {
+					res = _FAIL;
+
+					if (start == 0)
+						start = rtw_get_current_time();
+
+					if (is_broadcast_mac_addr(prxattrib->ra))
+						no_gkey_bc_cnt++;
+					else
+						no_gkey_mc_cnt++;
+
+					if (rtw_get_passing_time_ms(start) > 1000) {
+						if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+							RTW_PRINT(FUNC_ADPT_FMT" no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+								FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+						}
+						start = rtw_get_current_time();
+						no_gkey_bc_cnt = 0;
+						no_gkey_mc_cnt = 0;
+					}
+
+					goto exit;
+				}
+
+				if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+					RTW_PRINT(FUNC_ADPT_FMT" gkey installed. no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+						FUNC_ADPT_ARG(padapter), no_gkey_bc_cnt, no_gkey_mc_cnt);
+				}
+				start = 0;
+				no_gkey_bc_cnt = 0;
+				no_gkey_mc_cnt = 0;
+
+				#ifdef CONFIG_RTW_MESH
+				if (MLME_IS_MESH(padapter)) {
+					/* TODO: multiple GK? */
+					prwskey = &stainfo->gtk.skey[0];
+				} else
+				#endif
+				{
+					prwskey = lsecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
+					if (lsecuritypriv->dot118021XGrpKeyid != prxattrib->key_index)
+					{
+						RTW_DBG("not match packet_index=%d, install_index=%d\n"
+							, prxattrib->key_index, lsecuritypriv->dot118021XGrpKeyid);
+						res = _FAIL;
+						goto exit;
+					}
+				}
+			} else
+				prwskey = &stainfo->dot118021x_UncstKey.skey[0];
+
+			res = _rtw_gcmp_decrypt(padapter, prwskey,
+				prxattrib->encrypt == _GCMP_256_ ? 32 : 16,
+				prxattrib->hdrlen, pframe,
+				((union recv_frame *)precvframe)->u.hdr.len);
+
+			GCMP_SW_DEC_CNT_INC(psecuritypriv, prxattrib->ra);
+		} else {
+			res = _FAIL;
+		}
+
+	}
+exit:
+	return res;
+}
+
+
+#ifdef CONFIG_IEEE80211W
+u8 rtw_calculate_bip_mic(enum security_type gmcs, u8 *whdr_pos, s32 len,
+	const u8 *key, const u8 *data, size_t data_len, u8 *mic)
+{
+	u8 res = _SUCCESS;
+
+	if (gmcs == _BIP_CMAC_128_) {
+		if (_bip_ccmp_protect(key, 16, data, data_len, mic) == _FALSE) {
+			res = _FAIL;
+			RTW_ERR("%s : _bip_ccmp_protect(128) fail!", __func__);
+		}
+	} else if (gmcs == _BIP_CMAC_256_) {
+		if (_bip_ccmp_protect(key, 32, data, data_len, mic) == _FALSE) {
+			res = _FAIL;
+			RTW_ERR("%s : _bip_ccmp_protect(256) fail!", __func__);
+		}
+	} else if (gmcs == _BIP_GMAC_128_) {
+		if (_bip_gcmp_protect(whdr_pos, len, key, 16,
+				data, data_len, mic) == _FALSE) {
+			res = _FAIL;
+			RTW_ERR("%s : _bip_gcmp_protect(128) fail!", __func__);
+		}
+	} else if (gmcs == _BIP_GMAC_256_) {
+		if (_bip_gcmp_protect(whdr_pos, len, key, 32,
+				data, data_len, mic) == _FALSE) {
+			res = _FAIL;
+			RTW_ERR("%s : _bip_gcmp_protect(256) fail!", __func__);
+		}
+	} else {
+		res = _FAIL;
+		RTW_ERR("%s : unsupport dot11wCipher !\n", __func__);
+	}
+
+	return res;
+}
+
+
+u32 rtw_bip_verify(enum security_type gmcs, u16 pkt_len,
+	u8 *whdr_pos, sint flen, const u8 *key, u16 keyid, u64 *ipn,
+	u8 *precvframe)
+{
+	u8 * BIP_AAD,*mme;
+	u32 res = _FAIL;
+	uint len, ori_len;
+	u16 pkt_keyid = 0;
+	u64 pkt_ipn = 0;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	u8 mic[16];
+	u8 mic_len, mme_offset;
+	struct rx_pkt_attrib *prxattrib = &((union recv_frame *)precvframe)->u.hdr.attrib;
+
+	mic_len = (gmcs == _BIP_CMAC_128_) ? 8 : 16;
+
+	if (flen < WLAN_HDR_A3_LEN || flen - WLAN_HDR_A3_LEN < mic_len)
+		return RTW_RX_HANDLED;
+
+	mme_offset = (mic_len == 8) ? 18 : 26;
+	mme = whdr_pos + flen - mme_offset;
+	if (*mme != _MME_IE_)
+		return RTW_RX_HANDLED;
+
+	/* copy key index */
+	_rtw_memcpy(&pkt_keyid, mme + 2, 2);
+	pkt_keyid = le16_to_cpu(pkt_keyid);
+	if (pkt_keyid != keyid) {
+		RTW_INFO("BIP key index error!\n");
+		return _FAIL;
+	}
+
+	/* save packet number */
+	_rtw_memcpy(&pkt_ipn, mme + 4, 6);
+	pkt_ipn = le64_to_cpu(pkt_ipn);
+	/* BIP packet number should bigger than previous BIP packet */
+	if (pkt_ipn <= *ipn) { /* wrap around? */
+		RTW_INFO("replay BIP packet\n");
+		return _FAIL;
+	}
+
+	/*HW decrtped case*/
+	if (prxattrib->bdecrypted) {
+		/*Need to record IPN value */
+		*ipn = pkt_ipn;
+		return _SUCCESS;
+	}
+
+	ori_len = flen - WLAN_HDR_A3_LEN + BIP_AAD_SIZE;
+	BIP_AAD = rtw_zmalloc(ori_len);
+	if (BIP_AAD == NULL) {
+		RTW_INFO("BIP AAD allocate fail\n");
+		return _FAIL;
+	}
+
+	/* mapping to wlan header */
+	pwlanhdr = (struct rtw_ieee80211_hdr *)whdr_pos;
+
+	/* save the frame body + MME (w/o mic) */
+	_rtw_memcpy(BIP_AAD + BIP_AAD_SIZE,
+		whdr_pos + WLAN_HDR_A3_LEN,
+		flen - WLAN_HDR_A3_LEN - mic_len);
+
+	/* conscruct AAD, copy frame control field */
+	_rtw_memcpy(BIP_AAD, &pwlanhdr->frame_ctl, 2);
+	ClearRetry(BIP_AAD);
+	ClearPwrMgt(BIP_AAD);
+	ClearMData(BIP_AAD);
+	/* conscruct AAD, copy address 1 to address 3 */
+	_rtw_memcpy(BIP_AAD + 2, pwlanhdr->addr1, 6);
+	_rtw_memcpy(BIP_AAD + 2 + 6, pwlanhdr->addr2, 6);
+	_rtw_memcpy(BIP_AAD + 2 + 6 + 6, pwlanhdr->addr3, 6);
+
+	if (rtw_calculate_bip_mic(gmcs, whdr_pos,
+			pkt_len, key, BIP_AAD, ori_len, mic) == _FAIL)
+		goto BIP_exit;
+
+	/* MIC field should be last 8 bytes of packet (packet without FCS) */
+	if (_rtw_memcmp(mic, whdr_pos + flen - mic_len, mic_len)) {
+		*ipn = pkt_ipn;
+		res = _SUCCESS;
+	} else
+		RTW_INFO("BIP MIC error!\n");
+
+#if 0
+	/* management packet content */
+	{
+		int pp;
+		RTW_INFO("pkt: ");
+		RTW_INFO_DUMP("", whdr_pos, flen);
+		RTW_INFO("\n");
+		/* BIP AAD + management frame body + MME(MIC is zero) */
+		RTW_INFO("AAD+PKT: ");
+		RTW_INFO_DUMP("", BIP_AAD, ori_len);
+		RTW_INFO("\n");
+		/* show the MIC result */
+		RTW_INFO("mic: ");
+		RTW_INFO_DUMP("", mic, mic_len);
+		RTW_INFO("\n");
+	}
+#endif
+
+BIP_exit:
+
+	rtw_mfree(BIP_AAD, ori_len);
+	return res;
+}
+
+#endif /* CONFIG_IEEE80211W */
+
+u8 rtw_iv_to_pn(u8 *iv, u8 *pn, u8 *key_id, u32 enc_algo)
+{
+	/* iv and pn must be Little Endian format */
+	switch (enc_algo) {
+	case _TKIP_:
+		*(pn)     = *(iv + 2);
+		*(pn + 1) = *(iv);
+		break;
+	case _AES_:
+	case _GCMP_:
+	case _CCMP_256_:
+	case _GCMP_256_:
+		*(pn)     = *(iv);
+		*(pn + 1) = *(iv + 1);
+		break;
+	default:
+		return _FAIL;
+	}
+
+	*(pn + 2) = *(iv + 4);
+	*(pn + 3) = *(iv + 5);
+	*(pn + 4) = *(iv + 6);
+	*(pn + 5) = *(iv + 7);
+
+	if (key_id)
+		*key_id = *(iv + 3) >> 6;
+
+	return _SUCCESS;
+}
+
+u8 rtw_pn_to_iv(u8 *pn, u8 *iv, u8 key_id, u32 enc_algo)
+{
+	/* iv and pn must be Little Endian format */
+	switch (enc_algo) {
+	case _TKIP_:
+		*(iv)     = *(pn + 1);
+		*(iv + 1) = (*(pn + 1) | 0x20) & 0x7F;
+		*(iv + 2) = *(pn);
+		break;
+	case _AES_:
+	case _GCMP_:
+	case _CCMP_256_:
+	case _GCMP_256_:
+		*(iv)     = *(pn);
+		*(iv + 1) = *(pn + 1);
+		*(iv + 2) = 0;
+		break;
+	default:
+		return _FAIL;
+	}
+
+	*(iv + 3) = BIT(5) | ((key_id & 0x3) << 6);
+	*(iv + 4) = *(pn + 2);
+	*(iv + 5) = *(pn + 3);
+	*(iv + 6) = *(pn + 4);
+	*(iv + 7) = *(pn + 5);
+
+	return _SUCCESS;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sta_mgt.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sta_mgt.c
new file mode 100644
index 000000000000..8b1ff216e854
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_sta_mgt.c
@@ -0,0 +1,1879 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_STA_MGT_C_
+
+#include <drv_types.h>
+
+bool test_st_match_rule(_adapter *adapter, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port)
+{
+	if (ntohs(*((u16 *)local_port)) == 5001 || ntohs(*((u16 *)remote_port)) == 5001)
+		return _TRUE;
+	return _FALSE;
+}
+
+struct st_register test_st_reg = {
+	.s_proto = 0x06,
+	.rule = test_st_match_rule,
+};
+
+inline void rtw_st_ctl_init(struct st_ctl_t *st_ctl)
+{
+	_rtw_memset(st_ctl->reg, 0 , sizeof(struct st_register) * SESSION_TRACKER_REG_ID_NUM);
+	_rtw_init_queue(&st_ctl->tracker_q);
+}
+
+inline void rtw_st_ctl_clear_tracker_q(struct st_ctl_t *st_ctl)
+{
+	_list *plist, *phead;
+	struct session_tracker *st;
+
+	_rtw_spinlock_bh(&st_ctl->tracker_q.lock);
+	phead = &st_ctl->tracker_q.queue;
+	plist = get_next(phead);
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		st = LIST_CONTAINOR(plist, struct session_tracker, list);
+		plist = get_next(plist);
+		rtw_list_delete(&st->list);
+		rtw_mfree((u8 *)st, sizeof(struct session_tracker));
+	}
+	_rtw_spinunlock_bh(&st_ctl->tracker_q.lock);
+}
+
+inline void rtw_st_ctl_deinit(struct st_ctl_t *st_ctl)
+{
+	rtw_st_ctl_clear_tracker_q(st_ctl);
+	_rtw_deinit_queue(&st_ctl->tracker_q);
+}
+
+inline void rtw_st_ctl_register(struct st_ctl_t *st_ctl, u8 st_reg_id, struct st_register *reg)
+{
+	if (st_reg_id >= SESSION_TRACKER_REG_ID_NUM) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	st_ctl->reg[st_reg_id].s_proto = reg->s_proto;
+	st_ctl->reg[st_reg_id].rule = reg->rule;
+}
+
+inline void rtw_st_ctl_unregister(struct st_ctl_t *st_ctl, u8 st_reg_id)
+{
+	int i;
+
+	if (st_reg_id >= SESSION_TRACKER_REG_ID_NUM) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	st_ctl->reg[st_reg_id].s_proto = 0;
+	st_ctl->reg[st_reg_id].rule = NULL;
+
+	/* clear tracker queue if no session trecker registered */
+	for (i = 0; i < SESSION_TRACKER_REG_ID_NUM; i++)
+		if (st_ctl->reg[i].s_proto != 0)
+			break;
+	if (i >= SESSION_TRACKER_REG_ID_NUM)
+		rtw_st_ctl_clear_tracker_q(st_ctl);
+}
+
+inline bool rtw_st_ctl_chk_reg_s_proto(struct st_ctl_t *st_ctl, u8 s_proto)
+{
+	bool ret = _FALSE;
+	int i;
+
+	for (i = 0; i < SESSION_TRACKER_REG_ID_NUM; i++) {
+		if (st_ctl->reg[i].s_proto == s_proto) {
+			ret = _TRUE;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+inline bool rtw_st_ctl_chk_reg_rule(struct st_ctl_t *st_ctl, _adapter *adapter, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port)
+{
+	bool ret = _FALSE;
+	int i;
+	st_match_rule rule;
+
+	for (i = 0; i < SESSION_TRACKER_REG_ID_NUM; i++) {
+		rule = st_ctl->reg[i].rule;
+		if (rule && rule(adapter, local_naddr, local_port, remote_naddr, remote_port) == _TRUE) {
+			ret = _TRUE;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+void rtw_st_ctl_rx(struct sta_info *sta, u8 *ehdr_pos)
+{
+	_adapter *adapter = sta->padapter;
+	struct ethhdr *etherhdr = (struct ethhdr *)ehdr_pos;
+
+	if (ntohs(etherhdr->h_proto) == ETH_P_IP) {
+		u8 *ip = ehdr_pos + ETH_HLEN;
+
+		if (GET_IPV4_PROTOCOL(ip) == 0x06  /* TCP */
+			&& rtw_st_ctl_chk_reg_s_proto(&sta->st_ctl, 0x06) == _TRUE
+		) {
+			u8 *tcp = ip + GET_IPV4_IHL(ip) * 4;
+
+			if (rtw_st_ctl_chk_reg_rule(&sta->st_ctl, adapter, IPV4_DST(ip), TCP_DST(tcp), IPV4_SRC(ip), TCP_SRC(tcp)) == _TRUE) {
+				if (GET_TCP_SYN(tcp) && GET_TCP_ACK(tcp)) {
+					session_tracker_add_cmd(adapter, sta
+						, IPV4_DST(ip), TCP_DST(tcp)
+						, IPV4_SRC(ip), TCP_SRC(tcp));
+					if (DBG_SESSION_TRACKER)
+						RTW_INFO(FUNC_ADPT_FMT" local:"IP_FMT":"PORT_FMT", remote:"IP_FMT":"PORT_FMT" SYN-ACK\n"
+							, FUNC_ADPT_ARG(adapter)
+							, IP_ARG(IPV4_DST(ip)), PORT_ARG(TCP_DST(tcp))
+							, IP_ARG(IPV4_SRC(ip)), PORT_ARG(TCP_SRC(tcp)));
+				}
+				if (GET_TCP_FIN(tcp)) {
+					session_tracker_del_cmd(adapter, sta
+						, IPV4_DST(ip), TCP_DST(tcp)
+						, IPV4_SRC(ip), TCP_SRC(tcp));
+					if (DBG_SESSION_TRACKER)
+						RTW_INFO(FUNC_ADPT_FMT" local:"IP_FMT":"PORT_FMT", remote:"IP_FMT":"PORT_FMT" FIN\n"
+							, FUNC_ADPT_ARG(adapter)
+							, IP_ARG(IPV4_DST(ip)), PORT_ARG(TCP_DST(tcp))
+							, IP_ARG(IPV4_SRC(ip)), PORT_ARG(TCP_SRC(tcp)));
+				}
+			}
+
+		}
+	}
+}
+
+#define SESSION_TRACKER_FMT IP_FMT":"PORT_FMT" "IP_FMT":"PORT_FMT" %u %d"
+#define SESSION_TRACKER_ARG(st) IP_ARG(&(st)->local_naddr), PORT_ARG(&(st)->local_port), IP_ARG(&(st)->remote_naddr), PORT_ARG(&(st)->remote_port), (st)->status, rtw_get_passing_time_ms((st)->set_time)
+
+void dump_st_ctl(void *sel, struct st_ctl_t *st_ctl)
+{
+	int i;
+	_list *plist, *phead;
+	struct session_tracker *st;
+
+	if (!DBG_SESSION_TRACKER)
+		return;
+
+	for (i = 0; i < SESSION_TRACKER_REG_ID_NUM; i++)
+		RTW_PRINT_SEL(sel, "reg%d: %u %p\n", i, st_ctl->reg[i].s_proto, st_ctl->reg[i].rule);
+
+	_rtw_spinlock_bh(&st_ctl->tracker_q.lock);
+	phead = &st_ctl->tracker_q.queue;
+	plist = get_next(phead);
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		st = LIST_CONTAINOR(plist, struct session_tracker, list);
+		plist = get_next(plist);
+
+		RTW_PRINT_SEL(sel, SESSION_TRACKER_FMT"\n", SESSION_TRACKER_ARG(st));
+	}
+	_rtw_spinunlock_bh(&st_ctl->tracker_q.lock);
+
+}
+
+static void _rtw_init_stainfo(struct sta_priv *stapriv, struct sta_info *psta)
+{
+	bool pool_init = psta->padapter ? false : true; /* called by _rtw_init_sta_priv() */
+
+#ifdef CONFIG_AP_MODE
+	if (!pool_init) {
+		rtw_warn_on(!rtw_is_list_empty(&psta->asoc_list));
+		rtw_warn_on(!rtw_is_list_empty(&psta->auth_list));
+	}
+#endif
+
+	_rtw_memset((u8 *)psta, 0, sizeof(struct sta_info));
+
+	psta->padapter = stapriv->padapter;
+
+	_rtw_spinlock_init(&psta->lock);
+	_rtw_init_listhead(&psta->list);
+	_rtw_init_listhead(&psta->hash_list);
+	/* _rtw_init_listhead(&psta->sleep_list); */
+	/* _rtw_init_listhead(&psta->wakeup_list);	 */
+
+	_rtw_init_queue(&psta->sleep_q);
+
+	_rtw_init_sta_xmit_priv(&psta->sta_xmitpriv);
+	_rtw_init_sta_recv_priv(&psta->sta_recvpriv);
+
+#ifdef CONFIG_AP_MODE
+	_rtw_init_listhead(&psta->asoc_list);
+	_rtw_init_listhead(&psta->auth_list);
+	psta->bpairwise_key_installed = _FALSE;
+	psta->start_active = _FALSE;
+
+#ifdef CONFIG_RTW_80211R_AP
+	rtw_ft_peer_info_init(psta);
+#endif
+#ifdef CONFIG_RTW_80211R
+	psta->ft_pairwise_key_installed = _FALSE;
+#endif
+#endif /* CONFIG_AP_MODE	 */
+	rtw_st_ctl_init(&psta->st_ctl);
+	psta->smps_mode = SM_PS_DISABLE;
+
+	ATOMIC_SET(&psta->deleting, 0);
+}
+
+u32	_rtw_init_sta_priv(struct	sta_priv *pstapriv)
+{
+	_adapter *adapter = container_of(pstapriv, _adapter, stapriv);
+	/* struct macid_ctl_t *macid_ctl = adapter_to_macidctl(adapter); */
+#ifdef CONFIG_AP_MODE
+	struct registry_priv *regsty = adapter_to_regsty(adapter);
+#endif
+	struct sta_info *psta;
+	s32 i;
+	u32 ret = _FAIL;
+
+	pstapriv->padapter = adapter;
+
+	pstapriv->pallocated_stainfo_buf = rtw_zvmalloc(
+		sizeof(struct sta_info) * NUM_STA + MEM_ALIGNMENT_OFFSET);
+	if (!pstapriv->pallocated_stainfo_buf)
+		goto exit;
+
+	pstapriv->pstainfo_buf = pstapriv->pallocated_stainfo_buf;
+	if ((SIZE_PTR)pstapriv->pstainfo_buf & MEM_ALIGNMENT_PADDING)
+		pstapriv->pstainfo_buf += MEM_ALIGNMENT_OFFSET -
+			((SIZE_PTR)pstapriv->pstainfo_buf & MEM_ALIGNMENT_PADDING);
+
+	_rtw_init_queue(&pstapriv->free_sta_queue);
+
+	_rtw_spinlock_init(&pstapriv->sta_hash_lock);
+	_rtw_spinlock_init(&pstapriv->active_time_lock);
+#ifdef PRIVATE_R
+	_rtw_spinlock_init(&pstapriv->last_rx_uc_data_lock);
+#endif
+
+	/* _rtw_init_queue(&pstapriv->asoc_q); */
+	pstapriv->asoc_sta_count = 0;
+	_rtw_init_queue(&pstapriv->sleep_q);
+	_rtw_init_queue(&pstapriv->wakeup_q);
+
+	psta = (struct sta_info *)(pstapriv->pstainfo_buf);
+
+
+	for (i = 0; i < NUM_STA; i++) {
+		_rtw_init_stainfo(pstapriv, psta);
+
+		_rtw_init_listhead(&(pstapriv->sta_hash[i]));
+
+		rtw_list_insert_tail(&psta->list, get_list_head(&pstapriv->free_sta_queue));
+
+		psta++;
+	}
+
+	pstapriv->adhoc_expire_to = 4; /* 4 * 2 = 8 sec */
+
+#ifdef CONFIG_AP_MODE
+	pstapriv->max_aid = rtw_phl_get_macid_max_num(
+				GET_PHL_INFO(adapter_to_dvobj(pstapriv->padapter)));
+	if (regsty->max_ap_assoc_sta)
+		pstapriv->max_aid = rtw_min(pstapriv->max_aid, regsty->max_ap_assoc_sta);
+	pstapriv->rr_aid = 0;
+	pstapriv->started_aid = 1;
+	pstapriv->sta_aid = rtw_zmalloc(pstapriv->max_aid * sizeof(struct sta_info *));
+	if (!pstapriv->sta_aid)
+		goto exit;
+	pstapriv->aid_bmp_len = AID_BMP_LEN(pstapriv->max_aid);
+	pstapriv->sta_dz_bitmap = rtw_zmalloc(pstapriv->aid_bmp_len);
+	if (!pstapriv->sta_dz_bitmap)
+		goto exit;
+	pstapriv->tim_bitmap = rtw_zmalloc(pstapriv->aid_bmp_len);
+	if (!pstapriv->tim_bitmap)
+		goto exit;
+
+	_rtw_init_listhead(&pstapriv->asoc_list);
+	_rtw_init_listhead(&pstapriv->auth_list);
+	_rtw_spinlock_init(&pstapriv->asoc_list_lock);
+	_rtw_spinlock_init(&pstapriv->auth_list_lock);
+	pstapriv->asoc_list_cnt = 0;
+	pstapriv->auth_list_cnt = 0;
+
+	_rtw_init_listhead(&pstapriv->add_sta_list);
+	pstapriv->add_sta_list_cnt = 0;
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	pstapriv->tbtx_asoc_list_cnt = 0;
+#endif
+
+	pstapriv->auth_to = 3; /* 3*2 = 6 sec */
+	pstapriv->assoc_to = 3;
+	/* pstapriv->expire_to = 900; */ /* 900*2 = 1800 sec = 30 min, expire after no any traffic. */
+	/* pstapriv->expire_to = 30; */ /* 30*2 = 60 sec = 1 min, expire after no any traffic. */
+#ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+	pstapriv->expire_to = 3; /* 3*2 = 6 sec */
+#else
+	pstapriv->expire_to = 60;/* 60*2 = 120 sec = 2 min, expire after no any traffic. */
+#endif
+#ifdef CONFIG_ATMEL_RC_PATCH
+	_rtw_memset(pstapriv->atmel_rc_pattern, 0, ETH_ALEN);
+#endif
+	pstapriv->max_num_sta = NUM_STA;
+	if (regsty->max_ap_assoc_sta)
+		pstapriv->max_num_sta = rtw_min(pstapriv->max_num_sta, regsty->max_ap_assoc_sta);
+
+#endif
+
+#if CONFIG_RTW_MACADDR_ACL
+	for (i = 0; i < RTW_ACL_PERIOD_NUM; i++)
+		rtw_macaddr_acl_init(adapter, i);
+#endif
+
+#if CONFIG_RTW_PRE_LINK_STA
+	rtw_pre_link_sta_ctl_init(pstapriv);
+#endif
+
+#if defined(DBG_ROAMING_TEST)
+	rtw_set_rx_chk_limit(adapter,1);
+#elif defined(CONFIG_ACTIVE_KEEP_ALIVE_CHECK)
+	rtw_set_rx_chk_limit(adapter,4);
+#else
+	rtw_set_rx_chk_limit(adapter,8);
+#endif
+
+	ret = _SUCCESS;
+
+exit:
+	if (ret != _SUCCESS) {
+		if (pstapriv->pallocated_stainfo_buf)
+			rtw_vmfree(pstapriv->pallocated_stainfo_buf,
+				sizeof(struct sta_info) * NUM_STA + MEM_ALIGNMENT_OFFSET);
+		#ifdef CONFIG_AP_MODE
+		if (pstapriv->sta_aid)
+			rtw_mfree(pstapriv->sta_aid, pstapriv->max_aid * sizeof(struct sta_info *));
+		if (pstapriv->sta_dz_bitmap)
+			rtw_mfree(pstapriv->sta_dz_bitmap, pstapriv->aid_bmp_len);
+		#endif
+	}
+
+	return ret;
+}
+
+inline int rtw_stainfo_offset(struct sta_priv *stapriv, struct sta_info *sta)
+{
+	int offset = (((u8 *)sta) - stapriv->pstainfo_buf) / sizeof(struct sta_info);
+
+	if (!stainfo_offset_valid(offset))
+		RTW_INFO("%s invalid offset(%d), out of range!!!", __func__, offset);
+
+	return offset;
+}
+
+inline struct sta_info *rtw_get_stainfo_by_offset(struct sta_priv *stapriv, int offset)
+{
+	if (!stainfo_offset_valid(offset))
+		RTW_INFO("%s invalid offset(%d), out of range!!!", __func__, offset);
+
+	return (struct sta_info *)(stapriv->pstainfo_buf + offset * sizeof(struct sta_info));
+}
+
+void	_rtw_free_sta_xmit_priv_lock(struct sta_xmit_priv *psta_xmitpriv);
+void	_rtw_free_sta_xmit_priv_lock(struct sta_xmit_priv *psta_xmitpriv)
+{
+
+	_rtw_spinlock_free(&psta_xmitpriv->lock);
+
+	_rtw_spinlock_free(&(psta_xmitpriv->be_q.sta_pending.lock));
+	_rtw_spinlock_free(&(psta_xmitpriv->bk_q.sta_pending.lock));
+	_rtw_spinlock_free(&(psta_xmitpriv->vi_q.sta_pending.lock));
+	_rtw_spinlock_free(&(psta_xmitpriv->vo_q.sta_pending.lock));
+}
+
+static void	_rtw_free_sta_recv_priv_lock(struct sta_recv_priv *psta_recvpriv)
+{
+
+	_rtw_spinlock_free(&psta_recvpriv->lock);
+
+	_rtw_spinlock_free(&(psta_recvpriv->defrag_q.lock));
+
+
+}
+
+void rtw_mfree_stainfo(struct sta_info *psta);
+void rtw_mfree_stainfo(struct sta_info *psta)
+{
+	_rtw_spinlock_free(&psta->lock);
+
+	_rtw_free_sta_xmit_priv_lock(&psta->sta_xmitpriv);
+	_rtw_free_sta_recv_priv_lock(&psta->sta_recvpriv);
+}
+
+
+/* this function is used to free the memory of lock || sema for all stainfos */
+void rtw_mfree_all_stainfo(struct sta_priv *pstapriv);
+void rtw_mfree_all_stainfo(struct sta_priv *pstapriv)
+{
+	_list	*plist, *phead;
+	struct sta_info *psta = NULL;
+
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	phead = get_list_head(&pstapriv->free_sta_queue);
+	plist = get_next(phead);
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		psta = LIST_CONTAINOR(plist, struct sta_info , list);
+		plist = get_next(plist);
+
+		rtw_mfree_stainfo(psta);
+	}
+
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+
+}
+
+void rtw_mfree_sta_priv_lock(struct	sta_priv *pstapriv);
+void rtw_mfree_sta_priv_lock(struct	sta_priv *pstapriv)
+{
+	rtw_mfree_all_stainfo(pstapriv); /* be done before free sta_hash_lock */
+
+	_rtw_spinlock_free(&pstapriv->free_sta_queue.lock);
+
+	_rtw_spinlock_free(&pstapriv->sta_hash_lock);
+	_rtw_spinlock_free(&pstapriv->active_time_lock);
+	_rtw_spinlock_free(&pstapriv->wakeup_q.lock);
+	_rtw_spinlock_free(&pstapriv->sleep_q.lock);
+#ifdef PRIVATE_R
+	_rtw_spinlock_free(&pstapriv->last_rx_uc_data_lock);
+#endif
+
+#ifdef CONFIG_AP_MODE
+	_rtw_spinlock_free(&pstapriv->asoc_list_lock);
+	_rtw_spinlock_free(&pstapriv->auth_list_lock);
+#endif
+
+}
+
+u32	_rtw_free_sta_priv(struct	sta_priv *pstapriv)
+{
+	int	index;
+
+	if (pstapriv) {
+		#ifdef CONFIG_RECV_REORDERING_CTRL
+		_list	*phead, *plist;
+		struct sta_info *psta = NULL;
+		struct recv_reorder_ctrl *preorder_ctrl;
+
+		/*	delete all reordering_ctrl_timer		*/
+		_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+		for (index = 0; index < NUM_STA; index++) {
+			phead = &(pstapriv->sta_hash[index]);
+			plist = get_next(phead);
+
+			while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+				int i;
+				psta = LIST_CONTAINOR(plist, struct sta_info , hash_list);
+#ifdef CONFIG_IEEE80211W
+				_cancel_timer_ex(&psta->dot11w_expire_timer);
+#endif
+				_cancel_timer_ex(&psta->addba_retry_timer);
+				plist = get_next(plist);
+
+				for (i = 0; i < 16 ; i++) {
+					preorder_ctrl = &psta->recvreorder_ctrl[i];
+					_cancel_timer_ex(&preorder_ctrl->reordering_ctrl_timer);
+				}
+			}
+		}
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+		/*===============================*/
+		#endif
+
+		rtw_mfree_sta_priv_lock(pstapriv);
+
+#if CONFIG_RTW_MACADDR_ACL
+		for (index = 0; index < RTW_ACL_PERIOD_NUM; index++)
+			rtw_macaddr_acl_deinit(pstapriv->padapter, index);
+#endif
+
+#if CONFIG_RTW_PRE_LINK_STA
+		rtw_pre_link_sta_ctl_deinit(pstapriv);
+#endif
+
+		if (pstapriv->pallocated_stainfo_buf)
+			rtw_vmfree(pstapriv->pallocated_stainfo_buf,
+				sizeof(struct sta_info) * NUM_STA + MEM_ALIGNMENT_OFFSET);
+		#ifdef CONFIG_AP_MODE
+		if (pstapriv->sta_aid)
+			rtw_mfree(pstapriv->sta_aid, pstapriv->max_aid * sizeof(struct sta_info *));
+		if (pstapriv->sta_dz_bitmap)
+			rtw_mfree(pstapriv->sta_dz_bitmap, pstapriv->aid_bmp_len);
+		if (pstapriv->tim_bitmap)
+			rtw_mfree(pstapriv->tim_bitmap, pstapriv->aid_bmp_len);
+		#endif
+	}
+
+	return _SUCCESS;
+}
+
+#if defined(CONFIG_80211N_HT) && defined(CONFIG_RECV_REORDERING_CTRL)
+static void rtw_init_recv_timer(struct recv_reorder_ctrl *preorder_ctrl)
+{
+	_adapter *padapter = preorder_ctrl->padapter;
+
+	rtw_init_timer(&(preorder_ctrl->reordering_ctrl_timer), rtw_reordering_ctrl_timeout_handler, preorder_ctrl);
+}
+#endif
+
+/* struct	sta_info *rtw_alloc_stainfo(_queue *pfree_sta_queue, unsigned char *hwaddr) */
+static struct sta_info *_rtw_alloc_core_stainfo(struct sta_priv *pstapriv,
+													const u8 *hwaddr)
+{
+	s32	index;
+	_list	*phash_list;
+	struct sta_info *psta;
+	_queue *pfree_sta_queue;
+	struct recv_reorder_ctrl *preorder_ctrl;
+	struct rtw_wifi_role_t *phl_role = pstapriv->padapter->phl_role;
+	int i = 0;
+	u16  wRxSeqInitialValue = 0xffff;
+
+
+	pfree_sta_queue = &pstapriv->free_sta_queue;
+
+	/* _rtw_spinlock_bh(&(pfree_sta_queue->lock)); */
+	_rtw_spinlock_bh(&(pstapriv->sta_hash_lock));
+	if (_rtw_queue_empty(pfree_sta_queue) == _TRUE) {
+		/* _rtw_spinunlock_bh(&(pstapriv->sta_hash_lock));*/
+
+		psta = NULL;
+	} else {
+		psta = LIST_CONTAINOR(get_next(&pfree_sta_queue->queue), struct sta_info, list);
+
+		rtw_list_delete(&(psta->list));
+
+		/* _rtw_spinunlock_bh(&(pfree_sta_queue->lock)); */
+		_rtw_init_stainfo(pstapriv, psta);
+
+		/*
+		for sta mode, due to self sta info & AP sta info are the same sta info
+		using self hash index for sta mode, or bmc sta will not found self sta
+		*/
+		switch (phl_role->type) {
+		case PHL_RTYPE_NONE:
+		case PHL_RTYPE_STATION:
+		case PHL_RTYPE_ADHOC:
+		case PHL_RTYPE_P2P_DEVICE:
+		case PHL_RTYPE_P2P_GC:
+			index = wifi_mac_hash(phl_role->mac_addr);
+			break;
+		default:
+			index = wifi_mac_hash(hwaddr);
+			break;
+		}
+
+		if (index >= NUM_STA) {
+			psta = NULL;
+			goto exit;
+		}
+		phash_list = &(pstapriv->sta_hash[index]);
+
+		/* _rtw_spinlock_bh(&(pstapriv->sta_hash_lock)); */
+
+		rtw_list_insert_tail(&psta->hash_list, phash_list);
+
+		pstapriv->asoc_sta_count++;
+
+		rtw_mi_update_iface_status(&(pstapriv->padapter->mlmepriv), 0);
+
+		/* _rtw_spinunlock_bh(&(pstapriv->sta_hash_lock)); */
+
+		/* Commented by Albert 2009/08/13
+		 * For the SMC router, the sequence number of first packet of WPS handshake will be 0.
+		 * In this case, this packet will be dropped by recv_decache function if we use the 0x00 as the default value for tid_rxseq variable.
+		 * So, we initialize the tid_rxseq variable as the 0xffff. */
+
+		for (i = 0; i < 16; i++) {
+			_rtw_memcpy(&psta->sta_recvpriv.rxcache.tid_rxseq[i], &wRxSeqInitialValue, 2);
+			_rtw_memcpy(&psta->sta_recvpriv.bmc_tid_rxseq[i], &wRxSeqInitialValue, 2);
+			_rtw_memset(&psta->sta_recvpriv.rxcache.iv[i], 0, sizeof(psta->sta_recvpriv.rxcache.iv[i]));
+		}
+
+		rtw_init_timer(&psta->addba_retry_timer, addba_timer_hdl, psta);
+#ifdef CONFIG_IEEE80211W
+		rtw_init_timer(&psta->dot11w_expire_timer, sa_query_timer_hdl, psta);
+#endif /* CONFIG_IEEE80211W */
+#ifdef CONFIG_TDLS
+		rtw_init_tdls_timer(pstapriv->padapter, psta);
+#endif /* CONFIG_TDLS */
+
+		/* for A-MPDU Rx reordering buffer control */
+		for (i = 0; i < 16 ; i++) {
+			preorder_ctrl = &psta->recvreorder_ctrl[i];
+			preorder_ctrl->padapter = pstapriv->padapter;
+			preorder_ctrl->tid = i;
+			preorder_ctrl->enable = _FALSE;
+			preorder_ctrl->indicate_seq = 0xffff;
+			#ifdef DBG_RX_SEQ
+			RTW_INFO("DBG_RX_SEQ "FUNC_ADPT_FMT" tid:%u SN_CLEAR indicate_seq:%d\n"
+				, FUNC_ADPT_ARG(pstapriv->padapter), i, preorder_ctrl->indicate_seq);
+			#endif
+
+			preorder_ctrl->ampdu_size = RX_AMPDU_SIZE_INVALID;
+
+			#ifdef CONFIG_RECV_REORDERING_CTRL
+			/* preorder_ctrl->wsize_b = (NR_RECVBUFF-2); */
+			preorder_ctrl->wsize_b = 64;/* 64; */
+			_rtw_init_queue(&preorder_ctrl->pending_recvframe_queue);
+
+			rtw_init_recv_timer(preorder_ctrl);
+			#endif
+			rtw_clear_bit(RTW_RECV_ACK_OR_TIMEOUT, &preorder_ctrl->rec_abba_rsp_ack);
+
+		}
+		ATOMIC_SET(&psta->keytrack, 0);
+		/* ToDo: need to API to init hal_sta->ra_info->rssi_stat.rssi  */
+		#if 0
+		psta->phl_sta->rssi_stat.rssi = (-1);
+		psta->phl_sta->rssi_stat.rssi_cck = (-1);
+		psta->phl_sta->rssi_stat.rssi_ofdm = (-1);
+		#endif
+#ifdef CONFIG_ATMEL_RC_PATCH
+		psta->flag_atmel_rc = 0;
+#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+		psta->tbtx_enable = _FALSE;
+#endif
+		/* init for the sequence number of received management frame */
+		psta->RxMgmtFrameSeqNum = 0xffff;
+		_rtw_memset(&psta->sta_stats, 0, sizeof(struct stainfo_stats));
+	}
+
+exit:
+
+	_rtw_spinunlock_bh(&(pstapriv->sta_hash_lock));
+
+
+	if (psta)
+		rtw_mi_update_iface_status(&(pstapriv->padapter->mlmepriv), 0);
+
+	return psta;
+}
+
+static void _rtw_alloc_phl_stainfo(struct sta_info *sta, struct	sta_priv *stapriv,
+				const u8 *hwaddr, enum rtw_device_type dtype, u16 main_id,
+				u8 link_idx, enum phl_cmd_type cmd_type)
+{
+	enum rtw_phl_status pstatus = RTW_PHL_STATUS_SUCCESS;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(stapriv->padapter));
+	struct rtw_wifi_role_t *wrole = stapriv->padapter->phl_role;
+	struct rtw_wifi_role_link_t *rlink = sta->padapter_link->wrlink;
+	bool alloc = _TRUE, only_hw = _FALSE;
+
+	/* Do not use this function in interrupt context  */
+	pstatus = rtw_phl_cmd_alloc_stainfo(phl, &sta->phl_sta, (u8 *)hwaddr,
+					    wrole, dtype, main_id, rlink, alloc,
+					    only_hw, cmd_type, 0);
+
+	if (sta->phl_sta) {
+		rtw_dump_phl_sta_info(RTW_DBGDUMP, sta);
+	} else {
+		RTW_ERR(FUNC_ADPT_FMT ": fail to alloc PHL sta "
+			"for " MAC_FMT " (status=%d)!\n",
+			FUNC_ADPT_ARG(stapriv->padapter),
+			MAC_ARG(hwaddr),
+			pstatus);
+	}
+}
+
+struct sta_info *rtw_alloc_stainfo(struct sta_priv *stapriv, const u8 *hwaddr,
+				enum rtw_device_type dtype, u16 main_id, u8 link_idx, enum phl_cmd_type cmd_type)
+{
+	struct sta_info *sta;
+	/* can use in interrupt context */
+	sta = _rtw_alloc_core_stainfo(stapriv, hwaddr);
+	sta->padapter_link = GET_LINK(sta->padapter, link_idx);
+
+	/* can not use in interrupt context */
+	_rtw_alloc_phl_stainfo(sta, stapriv, hwaddr, dtype,
+			main_id, link_idx, cmd_type);
+
+	return sta;
+}
+
+u32 rtw_alloc_stainfo_hw(struct	sta_priv *stapriv, struct sta_info *psta)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(stapriv->padapter));
+	struct rtw_wifi_role_t *wrole = stapriv->padapter->phl_role;
+	bool alloc = _TRUE, only_hw = _TRUE;
+
+	if ((psta != NULL) && (psta->phl_sta != NULL) && (psta->phl_sta->active == _FALSE))
+		status = rtw_phl_cmd_alloc_stainfo(phl, &psta->phl_sta,
+						(u8 *)psta->phl_sta->mac_addr,
+						wrole,
+						psta->phl_sta->mld->type,
+						psta->phl_sta->macid,
+						psta->phl_sta->rlink,
+						alloc, only_hw,
+						PHL_CMD_DIRECTLY, 0);
+
+	return (status == RTW_PHL_STATUS_SUCCESS) ? _SUCCESS : _FAIL;
+}
+
+/* using pstapriv->sta_hash_lock to protect */
+u32 static _rtw_free_core_stainfo(_adapter *padapter , struct sta_info *psta, u8 aid, const u8 *hwaddr)
+{
+	int i;
+	_queue *pfree_sta_queue;
+	struct recv_reorder_ctrl *preorder_ctrl;
+	struct	sta_xmit_priv	*pstaxmitpriv;
+	struct	xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+	struct	sta_priv *pstapriv = &padapter->stapriv;
+	struct hw_xmit *phwxmit;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	int pending_qcnt[4];
+	u8 is_pre_link_sta = _FALSE;
+
+	if (psta == NULL)
+		goto exit;
+
+#if defined(CONFIG_CORE_TXSC) && defined(USE_ONE_WLHDR)
+	/* free shortcut entry wlhdr buffer */
+	for (i = 0; i < CORE_TXSC_ENTRY_NUM; i++) {
+		if (psta->txsc_entry_cache[i].txsc_wlhdr) {
+			rtw_mfree(psta->txsc_entry_cache[i].txsc_wlhdr, CORE_TXSC_WLHDR_SIZE);
+			psta->txsc_entry_cache[i].txsc_wlhdr = NULL;
+		}
+	}
+#endif
+
+#ifdef CONFIG_RTW_80211K
+	if (aid)
+		rm_post_event(padapter, RM_ID_FOR_ALL(aid), RM_EV_cancel);
+#endif
+	#if CONFIG_RTW_PRE_LINK_STA
+	if (hwaddr)
+		is_pre_link_sta = rtw_is_pre_link_sta(pstapriv, hwaddr);
+
+	if (is_pre_link_sta == _FALSE) {
+		_rtw_spinlock_bh(&(pstapriv->sta_hash_lock));
+		rtw_list_delete(&psta->hash_list);
+		pstapriv->asoc_sta_count--;
+		_rtw_spinunlock_bh(&(pstapriv->sta_hash_lock));
+		rtw_mi_update_iface_status(&(padapter->mlmepriv), 0);
+	} else {
+		_rtw_spinlock_bh(&psta->lock);
+		psta->state = WIFI_FW_PRE_LINK;
+		_rtw_spinunlock_bh(&psta->lock);
+	}
+	#else
+	_rtw_spinlock_bh(&(pstapriv->sta_hash_lock));
+	rtw_list_delete(&psta->hash_list);
+	pstapriv->asoc_sta_count--;
+	_rtw_spinunlock_bh(&(pstapriv->sta_hash_lock));
+	rtw_mi_update_iface_status(&(padapter->mlmepriv), 0);
+
+	#endif
+
+	_rtw_spinlock_bh(&psta->lock);
+	psta->state &= ~WIFI_ASOC_STATE;
+	_rtw_spinunlock_bh(&psta->lock);
+
+	pfree_sta_queue = &pstapriv->free_sta_queue;
+
+
+	pstaxmitpriv = &psta->sta_xmitpriv;
+
+	/* rtw_list_delete(&psta->sleep_list); */
+
+	/* rtw_list_delete(&psta->wakeup_list); */
+
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	rtw_free_xmitframe_queue(pxmitpriv, &psta->sleep_q);
+	psta->sleepq_len = 0;
+
+	/* vo */
+	/* _rtw_spinlock_bh(&(pxmitpriv->vo_pending.lock)); */
+	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->vo_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->vo_q.tx_pending));
+	phwxmit = pxmitpriv->hwxmits;
+	phwxmit->accnt -= pstaxmitpriv->vo_q.qcnt;
+	pending_qcnt[0] = pstaxmitpriv->vo_q.qcnt;
+	pstaxmitpriv->vo_q.qcnt = 0;
+	/* _rtw_spinunlock_bh(&(pxmitpriv->vo_pending.lock)); */
+
+	/* vi */
+	/* _rtw_spinlock_bh(&(pxmitpriv->vi_pending.lock)); */
+	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->vi_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->vi_q.tx_pending));
+	phwxmit = pxmitpriv->hwxmits + 1;
+	phwxmit->accnt -= pstaxmitpriv->vi_q.qcnt;
+	pending_qcnt[1] = pstaxmitpriv->vi_q.qcnt;
+	pstaxmitpriv->vi_q.qcnt = 0;
+	/* _rtw_spinunlock_bh(&(pxmitpriv->vi_pending.lock)); */
+
+	/* be */
+	/* _rtw_spinlock_bh(&(pxmitpriv->be_pending.lock)); */
+	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->be_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->be_q.tx_pending));
+	phwxmit = pxmitpriv->hwxmits + 2;
+	phwxmit->accnt -= pstaxmitpriv->be_q.qcnt;
+	pending_qcnt[2] = pstaxmitpriv->be_q.qcnt;
+	pstaxmitpriv->be_q.qcnt = 0;
+	/* _rtw_spinunlock_bh(&(pxmitpriv->be_pending.lock)); */
+
+	/* bk */
+	/* _rtw_spinlock_bh(&(pxmitpriv->bk_pending.lock)); */
+	rtw_free_xmitframe_queue(pxmitpriv, &pstaxmitpriv->bk_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->bk_q.tx_pending));
+	phwxmit = pxmitpriv->hwxmits + 3;
+	phwxmit->accnt -= pstaxmitpriv->bk_q.qcnt;
+	pending_qcnt[3] = pstaxmitpriv->bk_q.qcnt;
+	pstaxmitpriv->bk_q.qcnt = 0;
+	/* _rtw_spinunlock_bh(&(pxmitpriv->bk_pending.lock)); */
+
+	rtw_os_wake_queue_at_free_stainfo(padapter, pending_qcnt);
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+
+	/* re-init sta_info; 20061114 */ /* will be init in alloc_stainfo */
+	/* _rtw_init_sta_xmit_priv(&psta->sta_xmitpriv); */
+	/* _rtw_init_sta_recv_priv(&psta->sta_recvpriv); */
+#ifdef CONFIG_IEEE80211W
+	_cancel_timer_ex(&psta->dot11w_expire_timer);
+#endif /* CONFIG_IEEE80211W */
+	_cancel_timer_ex(&psta->addba_retry_timer);
+
+#ifdef CONFIG_TDLS
+	psta->tdls_sta_state = TDLS_STATE_NONE;
+#endif /* CONFIG_TDLS */
+
+	/* for A-MPDU Rx reordering buffer control, cancel reordering_ctrl_timer */
+	for (i = 0; i < 16 ; i++) {
+		preorder_ctrl = &psta->recvreorder_ctrl[i];
+		rtw_clear_bit(RTW_RECV_ACK_OR_TIMEOUT, &preorder_ctrl->rec_abba_rsp_ack);
+
+#ifdef CONFIG_RECV_REORDERING_CTRL
+		_list	*phead, *plist;
+		union recv_frame *prframe;
+		_queue *ppending_recvframe_queue;
+
+		_cancel_timer_ex(&preorder_ctrl->reordering_ctrl_timer);
+
+		ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
+
+		_rtw_spinlock_bh(&ppending_recvframe_queue->lock);
+
+		phead =	get_list_head(ppending_recvframe_queue);
+		plist = get_next(phead);
+
+		while (!rtw_is_list_empty(phead)) {
+			prframe = LIST_CONTAINOR(plist, union recv_frame, u);
+
+			plist = get_next(plist);
+
+			rtw_list_delete(&(prframe->u.hdr.list));
+
+			rtw_free_recvframe(prframe);
+		}
+
+		_rtw_spinunlock_bh(&ppending_recvframe_queue->lock);
+#endif /*CONFIG_RECV_REORDERING_CTRL*/
+	}
+
+	/* CVE-2020-24586, clear defrag queue */
+	{
+		_list	*phead, *plist;
+		_queue *pfree_sta_queue, *pdefrag_q = &psta->sta_recvpriv.defrag_q;
+		union recv_frame *prframe;
+
+		_rtw_spinlock_bh(&pdefrag_q->lock);
+		phead = get_list_head(pdefrag_q);
+		plist = get_next(phead);
+		while (!rtw_is_list_empty(phead)) {
+			prframe = LIST_CONTAINOR(plist, union recv_frame, u);
+			plist = get_next(plist);
+			rtw_list_delete(&(prframe->u.hdr.list));
+			rtw_free_recvframe(prframe);
+		}
+		_rtw_spinunlock_bh(&pdefrag_q->lock);
+	}
+
+#ifdef CONFIG_AP_MODE
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	if (!rtw_is_list_empty(&psta->asoc_list))
+		rtw_stapriv_asoc_list_del(pstapriv, psta);
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	_rtw_spinlock_bh(&pstapriv->auth_list_lock);
+	if (!rtw_is_list_empty(&psta->auth_list)) {
+		rtw_list_delete(&psta->auth_list);
+		pstapriv->auth_list_cnt--;
+	}
+	_rtw_spinunlock_bh(&pstapriv->auth_list_lock);
+
+	psta->expire_to = 0;
+#ifdef CONFIG_ATMEL_RC_PATCH
+	psta->flag_atmel_rc = 0;
+#endif
+	psta->sleepq_ac_len = 0;
+	psta->qos_info = 0;
+
+	psta->max_sp_len = 0;
+	psta->uapsd_bk = 0;
+	psta->uapsd_be = 0;
+	psta->uapsd_vi = 0;
+	psta->uapsd_vo = 0;
+
+	psta->has_legacy_ac = 0;
+	psta->start_active = _FALSE;
+	psta->smps_mode = SM_PS_DISABLE;
+
+#ifdef CONFIG_NATIVEAP_MLME
+
+	if (aid && pmlmeinfo->state == _HW_STATE_AP_) {
+		rtw_tim_map_clear(padapter, pstapriv->sta_dz_bitmap, aid);
+		rtw_tim_map_clear(padapter, pstapriv->tim_bitmap, aid);
+
+		/* rtw_indicate_sta_disassoc_event(padapter, psta); */
+
+		if (pstapriv->sta_aid[aid - 1] == psta) {
+			pstapriv->sta_aid[aid - 1] = NULL;
+			if (psta->phl_sta)
+				psta->phl_sta->aid = 0;
+		}
+	}
+
+#endif /* CONFIG_NATIVEAP_MLME	 */
+
+#if !defined(CONFIG_ACTIVE_KEEP_ALIVE_CHECK) && defined(CONFIG_80211N_HT)
+	psta->under_exist_checking = 0;
+#endif
+
+#ifdef CONFIG_RTW_80211R_AP
+	rtw_ft_peer_info_free(psta);
+#endif
+#endif /* CONFIG_AP_MODE	 */
+
+	rtw_st_ctl_deinit(&psta->st_ctl);
+
+	psta->is_freeing = false;
+	if (is_pre_link_sta == _FALSE) {
+		_rtw_spinlock_free(&psta->lock);
+
+		/* _rtw_spinlock_bh(&(pfree_sta_queue->lock)); */
+		_rtw_spinlock_bh(&(pstapriv->sta_hash_lock));
+		rtw_list_insert_tail(&psta->list, get_list_head(pfree_sta_queue));
+		_rtw_spinunlock_bh(&(pstapriv->sta_hash_lock));
+		/* _rtw_spinunlock_bh(&(pfree_sta_queue->lock)); */
+	}
+
+exit:
+	return _SUCCESS;
+}
+
+static void _rtw_free_phl_stainfo(_adapter *adapter, struct sta_info *sta, u8 only_free_sw)
+{
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(adapter));
+	enum rtw_phl_status pstaus = RTW_PHL_STATUS_SUCCESS;
+	u8 hwaddr[ETH_ALEN] = {0};
+	bool alloc = _FALSE, only_hw = _FALSE;
+
+	if (sta != NULL) {
+		if (sta->phl_sta)
+			_rtw_memcpy(hwaddr, sta->phl_sta->mac_addr, ETH_ALEN);
+		if (only_free_sw)
+			pstaus = rtw_phl_free_stainfo_sw(phl, sta->phl_sta);
+		else
+			pstaus = rtw_phl_cmd_alloc_stainfo(phl, &sta->phl_sta,
+							sta->phl_sta->mac_addr,
+							adapter->phl_role,
+							sta->phl_sta->mld->type,
+							sta->phl_sta->macid,
+							sta->phl_sta->rlink,
+							alloc, only_hw,
+							PHL_CMD_DIRECTLY, 0);
+
+		if (pstaus != RTW_PHL_STATUS_SUCCESS)
+			RTW_ERR(FUNC_ADPT_FMT ": fail to free PHL sta "
+				"for " MAC_FMT " (status=%d)!\n",
+				FUNC_ADPT_ARG(adapter),
+				MAC_ARG(hwaddr),
+				pstaus);
+		else
+			/* free stainfo success, set pointer to NULL */
+			sta->phl_sta = NULL;
+	}
+}
+
+u32	rtw_free_stainfo(_adapter *padapter, struct sta_info *psta)
+{
+	u16 aid = 0;
+	u8 *hw_addr = NULL;
+
+	if (!psta) {
+		RTW_WARN("%s(): psta is NULL\n", __func__);
+		return _FAIL;
+	}
+
+	if (psta->phl_sta) {
+		aid = psta->phl_sta->aid;
+		hw_addr = psta->phl_sta->mac_addr;
+	} else {
+		RTW_WARN("%s(): phl_sta is NULL\n", __func__);
+	}
+	_rtw_free_core_stainfo(padapter, psta, aid, hw_addr);
+	_rtw_free_phl_stainfo(padapter, psta, _FALSE);
+	psta->phl_sta = NULL;
+	return _SUCCESS;
+}
+
+u32	rtw_free_stainfo_sw(_adapter *padapter, struct sta_info *psta)
+{
+	u16 aid = 0;
+	u8 *hw_addr = NULL;
+
+	if (!psta) {
+		RTW_WARN("%s(): psta is NULL\n", __func__);
+		return _FAIL;
+	}
+
+	if (psta->phl_sta) {
+		aid = psta->phl_sta->aid;
+		hw_addr = psta->phl_sta->mac_addr;
+	} else {
+		RTW_WARN("%s(): phl_sta is NULL\n", __func__);
+	}
+	_rtw_free_core_stainfo(padapter, psta, aid, hw_addr);
+	_rtw_free_phl_stainfo(padapter, psta, _TRUE);
+	return _SUCCESS;
+}
+
+/* free all stainfo which in sta_hash[all] */
+void rtw_free_all_stainfo(_adapter *padapter)
+{
+	_list	*plist, *phead;
+	s32	index;
+	struct sta_info *psta = NULL;
+	struct	sta_priv *pstapriv = &padapter->stapriv;
+	u8 free_sta_num = 0;
+	char free_sta_list[NUM_STA];
+	int stainfo_offset;
+	struct rtw_phl_mld_t *pmld = NULL;
+
+	if (pstapriv->asoc_sta_count == 1)
+		goto exit;
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	for (index = 0; index < NUM_STA; index++) {
+		phead = &(pstapriv->sta_hash[index]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info , hash_list);
+
+			plist = get_next(plist);
+
+			/* pbcmc_stainfo is self stainfo */
+			if (!rtw_is_self_stainfo(padapter, psta)) {
+				#if CONFIG_RTW_PRE_LINK_STA
+				if (rtw_is_pre_link_sta(pstapriv, psta->phl_sta->mac_addr) == _FALSE)
+				#endif
+					rtw_list_delete(&psta->hash_list);
+
+				stainfo_offset = rtw_stainfo_offset(pstapriv, psta);
+				if (stainfo_offset_valid(stainfo_offset))
+					free_sta_list[free_sta_num++] = stainfo_offset;
+			}
+
+		}
+	}
+
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+
+	for (index = 0; index < free_sta_num; index++) {
+		psta = rtw_get_stainfo_by_offset(pstapriv, free_sta_list[index]);
+		pmld = psta->phl_sta->mld;
+		if (pmld)
+			rtw_phl_free_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), pmld);
+		rtw_free_stainfo(padapter , psta);
+	}
+
+exit:
+	return;
+}
+
+bool rtw_is_self_addr_stainfo(_adapter *adapter, struct sta_info *sta)
+{
+	struct _ADAPTER_LINK *alink;
+	u8 lidx;
+
+	for (lidx = 0; lidx < adapter->adapter_link_num; lidx++) {
+		alink = GET_LINK(adapter, lidx);
+		if (_rtw_memcmp(alink->wrlink->mac_addr, sta->phl_sta->mac_addr, 6))
+			return true;
+	}
+	return false;
+}
+
+bool rtw_is_self_stainfo(_adapter *padapter, struct sta_info *sta)
+{
+	struct _ADAPTER_LINK *padapter_link;
+	u8 lidx;
+	struct rtw_phl_stainfo_t *self_sta = NULL;
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		self_sta = rtw_phl_get_stainfo_self(adapter_to_dvobj(padapter)->phl, padapter_link->wrlink);
+		if (sta->phl_sta == self_sta)
+			return _TRUE;
+	}
+	return _FALSE;
+}
+
+u32 rtw_free_mld_stainfo(_adapter *padapter, struct rtw_phl_mld_t *mld)
+{
+	u8 lidx;
+	struct sta_info *psta = NULL;
+
+	if (!mld)
+		return _SUCCESS;
+
+	for (lidx = 0; lidx < mld->sta_num; lidx++) {
+		psta = rtw_get_link_stainfo(&(padapter->stapriv), mld, lidx);
+		if (psta == NULL)
+			continue;
+		_rtw_free_core_stainfo(padapter, psta, psta->phl_sta->aid, psta->phl_sta->mac_addr);
+		_rtw_free_phl_stainfo(padapter, psta, _FALSE);
+	}
+	rtw_phl_free_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)), mld);
+	return _SUCCESS;
+}
+
+struct sta_info *rtw_get_stainfo_by_macid(struct sta_priv *pstapriv, u16 macid)
+{
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(pstapriv->padapter));
+	struct sta_info *sta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+
+	phl_sta = rtw_phl_get_stainfo_by_macid(phl, macid);
+	if (phl_sta)
+		sta = rtw_get_stainfo(pstapriv, phl_sta->mac_addr);
+	return sta;
+}
+
+/* any station allocated can be searched by hash list */
+static struct sta_info *_rtw_get_stainfo(struct sta_priv *pstapriv, const u8 *hwaddr, bool to_free)
+{
+	_list	*plist, *phead;
+	struct sta_info *psta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta_self = NULL;
+	struct _ADAPTER *padapter = pstapriv->padapter;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct rtw_wifi_role_t *phl_role;
+	u8 link_addr[ETH_ALEN] = {0};
+	struct rtw_phl_mld_t *mld = NULL;
+	u8 is_link_addr = _FALSE;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8 lidx;
+	u32 index;
+
+	if (hwaddr == NULL)
+		return NULL;
+
+	if (pstapriv->padapter->phl_role == NULL) {
+		RTW_ERR(FUNC_ADPT_FMT" phl_role == NULL\n", FUNC_ADPT_ARG(pstapriv->padapter));
+		rtw_warn_on(1);
+		return NULL;
+	}
+	phl_role = pstapriv->padapter->phl_role;
+
+	_rtw_memcpy(link_addr, hwaddr, ETH_ALEN);
+
+	/* core layer only maintain stainfo with link address
+	** so first check if hwaddr is mld address or link address
+	** if hwaddr is mld address, replace it with primary link address */
+	if (padapter->adapter_link_num != RTW_ONE_LINK) {
+		mld = rtw_phl_get_mld_by_addr(phl, phl_role, link_addr);
+		if (mld) {
+			for (lidx = 0; lidx < mld->sta_num; lidx++) {
+				phl_sta = rtw_phl_get_stainfo_by_mld(mld, lidx);
+				if (phl_sta == NULL)
+					continue;
+				if (_rtw_memcmp(link_addr, phl_sta->mac_addr, ETH_ALEN) == _TRUE) {
+					is_link_addr = _TRUE;
+					break;
+				}
+			}
+			if (!is_link_addr && mld->phl_sta[RTW_RLINK_PRIMARY] != NULL)
+				_rtw_memcpy(link_addr, mld->phl_sta[RTW_RLINK_PRIMARY]->mac_addr, ETH_ALEN);
+		}
+	}
+
+	/* if addr is bcmc addr, return self phl_sta */
+	if (IS_MCAST(link_addr)) {
+		/* ToDo CONFIG_RTW_MLD: bcmc self stainfo? */
+		phl_sta_self = rtw_phl_get_stainfo_self(phl, &(phl_role->rlink[RTW_RLINK_PRIMARY]));
+
+		if (phl_sta_self == NULL) {
+			RTW_INFO("%s: get phl sta self fail", __func__);
+			return NULL;
+		}
+
+		/*
+		due to using self sta to replace bcmc sta
+		change index to self hash index
+		*/
+
+		index = wifi_mac_hash(phl_role->mac_addr);
+	} else {
+
+		/*
+		for sta mode due to self sta info & AP sta info are the same sta info
+		using self hash index for sta mode, or bmc sta will not found self sta
+		*/
+		switch (phl_role->type) {
+		case PHL_RTYPE_NONE:
+		case PHL_RTYPE_STATION:
+		case PHL_RTYPE_ADHOC:
+		case PHL_RTYPE_P2P_DEVICE:
+		case PHL_RTYPE_P2P_GC:
+			index = wifi_mac_hash(phl_role->mac_addr);
+		break;
+		case PHL_RTYPE_TDLS:
+			if (_rtw_memcmp(link_addr, pstapriv->padapter->mlmepriv.dev_cur_network.network.MacAddress, ETH_ALEN) == _TRUE)
+				index = wifi_mac_hash(phl_role->mac_addr);
+			else
+				index = wifi_mac_hash(link_addr);
+		break;
+		default:
+			index = wifi_mac_hash(link_addr);
+		break;
+		}
+	}
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	phead = &(pstapriv->sta_hash[index]);
+	plist = get_next(phead);
+
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+
+		psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+
+		if (psta->phl_sta == NULL) {
+			psta = NULL;
+			RTW_DBG("phl_sta of sta is NULL\n");
+			plist = get_next(plist);
+			continue;
+		}
+
+		if (IS_MCAST(link_addr) && psta->phl_sta == phl_sta_self)
+			break;
+
+		/* if add is not bcmc addr, compare mac_addr  */
+		if ((_rtw_memcmp(psta->phl_sta->mac_addr, link_addr, ETH_ALEN)) == _TRUE) {
+			/* if found the matched address */
+			if (to_free)
+				psta->is_freeing = true;
+			break;
+		}
+		psta = NULL;
+		plist = get_next(plist);
+	}
+
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	return psta;
+
+}
+
+struct sta_info *rtw_get_stainfo(struct sta_priv *stapriv, const u8 *hwaddr)
+{
+	return _rtw_get_stainfo(stapriv, hwaddr, false);
+}
+
+struct sta_info *rtw_get_stainfo_to_free(struct sta_priv *stapriv, const u8 *hwaddr)
+{
+	return _rtw_get_stainfo(stapriv, hwaddr, true);
+}
+
+void rtw_stainfo_claim_to_free_no_lock(struct sta_info *sta)
+{
+	if (!rtw_is_list_empty(&sta->hash_list))
+		sta->is_freeing = true;
+}
+
+void rtw_stainfo_claim_to_free(struct sta_info *sta)
+{
+	_rtw_spinlock_bh(&sta->padapter->stapriv.sta_hash_lock);
+	rtw_stainfo_claim_to_free_no_lock(sta);
+	_rtw_spinunlock_bh(&sta->padapter->stapriv.sta_hash_lock);
+}
+
+struct sta_info *rtw_get_link_stainfo(struct sta_priv *stapriv, struct rtw_phl_mld_t *pmld, u8 lidx)
+{
+	struct sta_info *sta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+
+	if (pmld) {
+		phl_sta = rtw_phl_get_stainfo_by_mld(pmld, lidx);
+		if (phl_sta)
+			sta = rtw_get_stainfo(stapriv, phl_sta->mac_addr);
+	}
+
+	return sta;
+}
+
+struct sta_info *rtw_get_primary_stainfo(struct sta_priv *stapriv, struct sta_info *psta)
+{
+	if (psta)
+		return rtw_get_link_stainfo(stapriv, psta->phl_sta->mld, RTW_RLINK_PRIMARY);
+	else
+		return NULL;
+}
+
+struct sta_info *rtw_get_primary_stainfo_by_addr(struct sta_priv *stapriv, const u8 *hwaddr)
+{
+	struct sta_info *psta = rtw_get_stainfo(stapriv, hwaddr);
+	struct sta_info *primary_sta = psta;
+	struct rtw_phl_mld_t *mld = NULL;
+
+	if (psta && psta->phl_sta) {
+		mld = psta->phl_sta->mld;
+
+		if (mld && mld->sta_num > 1)
+			primary_sta = rtw_get_primary_stainfo(stapriv, psta);
+	}
+
+	return primary_sta;
+}
+u32	rtw_free_self_stainfo(_adapter *adapter)
+{
+	struct sta_info *sta = NULL;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct rtw_phl_mld_t *mld = NULL;
+	u8 lidx;
+	u16 aid = 0;
+
+	mld = rtw_phl_get_mld_self(GET_PHL_INFO(adapter_to_dvobj(adapter)), adapter->phl_role);
+	for (lidx = 0; lidx < mld->sta_num; lidx++) {
+		sta = rtw_get_link_stainfo(stapriv, mld, lidx);
+		if (sta != NULL) {
+			aid = sta->phl_sta->aid;
+			_rtw_free_core_stainfo(adapter, sta, aid, sta->phl_sta->mac_addr);
+			_rtw_free_phl_stainfo(adapter, sta, _FALSE);
+		}
+	}
+	return _SUCCESS;
+}
+
+u32 rtw_init_self_stainfo(_adapter *padapter, enum phl_cmd_type cmd_type)
+{
+
+	struct sta_info *psta;
+	struct tx_servq *ptxservq;
+	u32 res = _SUCCESS;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	u8 lidx;
+	struct rtw_phl_mld_t *mld = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	u16 main_id = rtw_phl_get_macid_max_num(phl);
+
+	mld = rtw_phl_get_mld_by_addr(GET_PHL_INFO(adapter_to_dvobj(padapter)),
+				padapter->phl_role,
+				padapter->phl_role->mac_addr);
+	if (mld == NULL)
+		mld = rtw_phl_alloc_mld(GET_PHL_INFO(adapter_to_dvobj(padapter)),
+						padapter->phl_role,
+						padapter->phl_role->mac_addr,
+						DTYPE);
+	for (lidx = 0; lidx < mld->sta_num; lidx++) {
+		rlink = &(padapter->phl_role->rlink[lidx]);
+		psta = rtw_get_stainfo(pstapriv, rlink->mac_addr);
+		if (psta == NULL) {
+			psta = rtw_alloc_stainfo(pstapriv, rlink->mac_addr, DTYPE, main_id, lidx, cmd_type);
+			if (psta == NULL) {
+				RTW_ERR("%s alloc self sta fail\n", __func__);
+				res = _FAIL;
+				goto exit;
+			}
+		}
+		main_id = psta->phl_sta->macid;
+	}
+	/* self mld and self stainfo, no need to link again */
+exit:
+	return res;
+
+}
+
+
+struct sta_info *rtw_get_bcmc_stainfo(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct sta_info	*psta;
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	/* return self sta */
+	_list	*plist, *phead;
+	struct rtw_phl_stainfo_t *phl_sta_self = NULL;
+	u32 index;
+
+	phl_sta_self = rtw_phl_get_stainfo_self(GET_PHL_INFO(adapter_to_dvobj(padapter)), padapter_link->wrlink);
+
+	if (phl_sta_self == NULL) {
+		RTW_INFO("%s: get phl sta self fail", __func__);
+		return NULL;
+	}
+	index = wifi_mac_hash(padapter->phl_role->mac_addr);
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	phead = &(pstapriv->sta_hash[index]);
+	plist = get_next(phead);
+
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+
+		psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+
+		if (psta->phl_sta == NULL) {
+			psta = NULL;
+			RTW_ERR("phl_sta of sta is NULL\n");
+			plist = get_next(plist);
+			continue;
+		}
+
+		if (psta->phl_sta == phl_sta_self)
+			break;
+
+		psta = NULL;
+		plist = get_next(plist);
+	}
+
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+	return psta;
+
+}
+
+#ifdef CONFIG_AP_MODE
+u16 rtw_aid_alloc(_adapter *adapter, struct sta_info *sta)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	u16 aid, i, used_cnt = 0;
+
+	for (i = 0; i < stapriv->max_aid; i++) {
+		aid = ((i + stapriv->started_aid - 1) % stapriv->max_aid) + 1;
+		if (stapriv->sta_aid[aid - 1] == NULL)
+			break;
+		if (++used_cnt >= stapriv->max_num_sta)
+			break;
+	}
+
+	/* check for aid limit and assoc limit  */
+	if (i >= stapriv->max_aid || used_cnt >= stapriv->max_num_sta)
+		aid = 0;
+
+	sta->phl_sta->aid = aid;
+	if (aid) {
+		stapriv->sta_aid[aid - 1] = sta;
+		if (stapriv->rr_aid)
+			stapriv->started_aid = (aid % stapriv->max_aid) + 1;
+	}
+
+	return aid;
+}
+
+void dump_aid_status(void *sel, _adapter *adapter)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	u8 *aid_bmp;
+	u16 i, used_cnt = 0;
+
+	aid_bmp = rtw_zmalloc(stapriv->aid_bmp_len);
+	if (!aid_bmp)
+		return;
+
+	for (i = 1; i <= stapriv->max_aid; i++) {
+		if (stapriv->sta_aid[i - 1]) {
+			aid_bmp[i / 8] |= BIT(i % 8);
+			++used_cnt;
+		}
+	}
+
+	RTW_PRINT_SEL(sel, "used_cnt:%u/%u\n", used_cnt, stapriv->max_aid);
+	RTW_MAP_DUMP_SEL(sel, "aid_map:", aid_bmp, stapriv->aid_bmp_len);
+	RTW_PRINT_SEL(sel, "\n");
+
+	RTW_PRINT_SEL(sel, "%-2s %-11s\n", "rr", "started_aid");
+	RTW_PRINT_SEL(sel, "%2d %11d\n", stapriv->rr_aid, stapriv->started_aid);
+
+	rtw_mfree(aid_bmp, stapriv->aid_bmp_len);
+}
+
+void rtw_stapriv_asoc_list_lock(struct sta_priv *stapriv)
+{
+	_rtw_spinlock_bh(&stapriv->asoc_list_lock);
+}
+
+void rtw_stapriv_asoc_list_unlock(struct sta_priv *stapriv)
+{
+	_rtw_spinunlock_bh(&stapriv->asoc_list_lock);
+}
+
+void rtw_stapriv_asoc_list_add(struct sta_priv *stapriv, struct sta_info *sta)
+{
+	rtw_warn_on(!_rtw_spin_is_locked(&stapriv->asoc_list_lock));
+	rtw_list_insert_tail(&sta->asoc_list, &stapriv->asoc_list);
+	stapriv->asoc_list_cnt++;
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	if (sta->tbtx_enable)
+		stapriv->tbtx_asoc_list_cnt++;
+#endif
+}
+
+void rtw_stapriv_asoc_list_del(struct sta_priv *stapriv, struct sta_info *sta)
+{
+	rtw_warn_on(!_rtw_spin_is_locked(&stapriv->asoc_list_lock));
+	rtw_list_delete(&sta->asoc_list);
+	stapriv->asoc_list_cnt--;
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	if (sta->tbtx_enable)
+		stapriv->tbtx_asoc_list_cnt--;
+#endif
+	STA_SET_MESH_PLINK(sta, NULL);
+}
+#endif /* CONFIG_AP_MODE */
+
+#if CONFIG_RTW_MACADDR_ACL
+const char *const _acl_period_str[RTW_ACL_PERIOD_NUM] = {
+	"DEV",
+	"BSS",
+};
+
+const char *const _acl_mode_str[RTW_ACL_MODE_MAX] = {
+	"DISABLED",
+	"ACCEPT_UNLESS_LISTED",
+	"DENY_UNLESS_LISTED",
+};
+
+u8 _rtw_access_ctrl(_adapter *adapter, u8 period, const u8 *mac_addr)
+{
+	u8 res = _TRUE;
+	_list *list, *head;
+	struct rtw_wlan_acl_node *acl_node;
+	u8 match = _FALSE;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct wlan_acl_pool *acl;
+	_queue	*acl_node_q;
+
+	if (period >= RTW_ACL_PERIOD_NUM) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	acl = &stapriv->acl_list[period];
+	acl_node_q = &acl->acl_node_q;
+
+	if (acl->mode != RTW_ACL_MODE_ACCEPT_UNLESS_LISTED
+		&& acl->mode != RTW_ACL_MODE_DENY_UNLESS_LISTED)
+		goto exit;
+
+	_rtw_spinlock_bh(&(acl_node_q->lock));
+	head = get_list_head(acl_node_q);
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		acl_node = LIST_CONTAINOR(list, struct rtw_wlan_acl_node, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(acl_node->addr, mac_addr, ETH_ALEN)) {
+			if (acl_node->valid == _TRUE) {
+				match = _TRUE;
+				break;
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&(acl_node_q->lock));
+
+	if (acl->mode == RTW_ACL_MODE_ACCEPT_UNLESS_LISTED)
+		res = (match == _TRUE) ?  _FALSE : _TRUE;
+	else /* RTW_ACL_MODE_DENY_UNLESS_LISTED */
+		res = (match == _TRUE) ?  _TRUE : _FALSE;
+
+exit:
+	return res;
+}
+
+struct sta_info *rtw_alloc_stainfo_sw(struct sta_priv *stapriv, enum rtw_device_type dtype,
+				u16 main_id, u8 link_idx, const u8 *hwaddr)
+{
+	struct sta_info *sta;
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(adapter_to_dvobj(stapriv->padapter));
+	struct rtw_wifi_role_link_t *rlink = NULL;
+
+	/* can use in interrupt context */
+	sta = _rtw_alloc_core_stainfo(stapriv, hwaddr);
+	if (sta == NULL)
+		return NULL;
+	sta->padapter_link = GET_LINK(sta->padapter, link_idx);
+	rlink = sta->padapter_link->wrlink;
+
+	if (sta != NULL) {
+		sta->phl_sta = rtw_phl_alloc_stainfo_sw(
+			GET_PHL_INFO(adapter_to_dvobj(stapriv->padapter)),
+			(u8 *)hwaddr, stapriv->padapter->phl_role
+			, dtype
+			, main_id
+			, rlink
+			);
+
+		if (sta->phl_sta) {
+			rtw_dump_phl_sta_info(RTW_DBGDUMP, sta);
+		} else {
+			RTW_ERR(FUNC_ADPT_FMT ": fail to alloc PHL sta "
+				"for " MAC_FMT " !\n",
+				FUNC_ADPT_ARG(stapriv->padapter),
+				MAC_ARG(hwaddr));
+			_rtw_free_core_stainfo(stapriv->padapter, sta, 0, hwaddr);
+			sta = NULL;
+		}
+	}
+
+	return sta;
+}
+
+u8 rtw_access_ctrl(_adapter *adapter, const u8 *mac_addr)
+{
+	int i;
+
+	for (i = 0; i < RTW_ACL_PERIOD_NUM; i++)
+		if (_rtw_access_ctrl(adapter, i, mac_addr) == _FALSE)
+			return _FALSE;
+
+	return _TRUE;
+}
+
+void dump_macaddr_acl(void *sel, _adapter *adapter)
+{
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct wlan_acl_pool *acl;
+	int i, j;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	for (j = 0; j < RTW_ACL_PERIOD_NUM; j++) {
+		RTW_PRINT_SEL(sel, "period:%s(%d)\n", acl_period_str(j), j);
+
+		acl = &stapriv->acl_list[j];
+		RTW_PRINT_SEL(sel, "mode:%s(%d)\n", acl_mode_str(acl->mode), acl->mode);
+		RTW_PRINT_SEL(sel, "num:%d/%d\n", acl->num, NUM_ACL);
+		for (i = 0; i < NUM_ACL; i++) {
+			if (acl->aclnode[i].valid == _FALSE)
+				continue;
+			RTW_PRINT_SEL(sel, "%s\n", get_macaddr_str(mac_addr_str, sel, acl->aclnode[i].addr));
+		}
+		RTW_PRINT_SEL(sel, "\n");
+	}
+}
+#endif /* CONFIG_RTW_MACADDR_ACL */
+
+#if CONFIG_RTW_PRE_LINK_STA
+bool rtw_is_pre_link_sta(struct sta_priv *stapriv, u8 *addr)
+{
+	struct pre_link_sta_ctl_t *pre_link_sta_ctl = &stapriv->pre_link_sta_ctl;
+	struct sta_info *sta = NULL;
+	u8 exist = _FALSE;
+	int i;
+
+	_rtw_spinlock_bh(&(pre_link_sta_ctl->lock));
+	for (i = 0; i < RTW_PRE_LINK_STA_NUM; i++) {
+		if (pre_link_sta_ctl->node[i].valid == _TRUE
+			&& _rtw_memcmp(pre_link_sta_ctl->node[i].addr, addr, ETH_ALEN) == _TRUE
+		) {
+			exist = _TRUE;
+			break;
+		}
+	}
+	_rtw_spinunlock_bh(&(pre_link_sta_ctl->lock));
+
+	return exist;
+}
+#endif
+
+#if CONFIG_RTW_PRE_LINK_STA
+struct sta_info *rtw_pre_link_sta_add(struct sta_priv *stapriv, u8 *hwaddr)
+{
+	struct pre_link_sta_ctl_t *pre_link_sta_ctl = &stapriv->pre_link_sta_ctl;
+	struct pre_link_sta_node_t *node = NULL;
+	struct sta_info *sta = NULL;
+	u8 exist = _FALSE;
+	int i;
+
+	if (rtw_check_invalid_mac_address(hwaddr, _FALSE) == _TRUE)
+		goto exit;
+
+	_rtw_spinlock_bh(&(pre_link_sta_ctl->lock));
+	for (i = 0; i < RTW_PRE_LINK_STA_NUM; i++) {
+		if (pre_link_sta_ctl->node[i].valid == _TRUE
+			&& _rtw_memcmp(pre_link_sta_ctl->node[i].addr, hwaddr, ETH_ALEN) == _TRUE
+		) {
+			node = &pre_link_sta_ctl->node[i];
+			exist = _TRUE;
+			break;
+		}
+
+		if (node == NULL && pre_link_sta_ctl->node[i].valid == _FALSE)
+			node = &pre_link_sta_ctl->node[i];
+	}
+
+	if (exist == _FALSE && node) {
+		_rtw_memcpy(node->addr, hwaddr, ETH_ALEN);
+		node->valid = _TRUE;
+		pre_link_sta_ctl->num++;
+	}
+	_rtw_spinunlock_bh(&(pre_link_sta_ctl->lock));
+
+	if (node == NULL)
+		goto exit;
+
+	sta = rtw_alloc_stainfo(stapriv, hwaddr, PHL_CMD_WAIT);
+	if (!sta)
+		goto exit;
+
+	sta->state = WIFI_FW_PRE_LINK;
+
+exit:
+	return sta;
+}
+
+void rtw_pre_link_sta_del(struct sta_priv *stapriv, u8 *hwaddr)
+{
+	struct pre_link_sta_ctl_t *pre_link_sta_ctl = &stapriv->pre_link_sta_ctl;
+	struct pre_link_sta_node_t *node = NULL;
+	struct sta_info *sta = NULL;
+	u8 exist = _FALSE;
+	int i;
+	/* ToDo CONFIG_RTW_MLD: free per MLD or per sta? */
+
+	if (rtw_check_invalid_mac_address(hwaddr, _FALSE) == _TRUE)
+		goto exit;
+
+	_rtw_spinlock_bh(&(pre_link_sta_ctl->lock));
+	for (i = 0; i < RTW_PRE_LINK_STA_NUM; i++) {
+		if (pre_link_sta_ctl->node[i].valid == _TRUE
+			&& _rtw_memcmp(pre_link_sta_ctl->node[i].addr, hwaddr, ETH_ALEN) == _TRUE
+		) {
+			node = &pre_link_sta_ctl->node[i];
+			exist = _TRUE;
+			break;
+		}
+	}
+
+	if (exist == _TRUE && node) {
+		node->valid = _FALSE;
+		pre_link_sta_ctl->num--;
+	}
+	_rtw_spinunlock_bh(&(pre_link_sta_ctl->lock));
+
+	if (exist == _FALSE)
+		goto exit;
+
+	sta = rtw_get_stainfo(stapriv, hwaddr);
+	if (!sta)
+		goto exit;
+
+	if (sta->state == WIFI_FW_PRE_LINK)
+		rtw_free_stainfo(stapriv->padapter, sta);
+
+exit:
+	return;
+}
+
+void rtw_pre_link_sta_ctl_reset(struct sta_priv *stapriv)
+{
+	struct pre_link_sta_ctl_t *pre_link_sta_ctl = &stapriv->pre_link_sta_ctl;
+	struct pre_link_sta_node_t *node = NULL;
+	struct sta_info *sta = NULL;
+	int i, j = 0;
+	/* ToDo CONFIG_RTW_MLD: reset per MLD or per sta? */
+
+	u8 addrs[RTW_PRE_LINK_STA_NUM][ETH_ALEN];
+
+	_rtw_memset(addrs, 0, RTW_PRE_LINK_STA_NUM * ETH_ALEN);
+
+	_rtw_spinlock_bh(&(pre_link_sta_ctl->lock));
+	for (i = 0; i < RTW_PRE_LINK_STA_NUM; i++) {
+		if (pre_link_sta_ctl->node[i].valid == _FALSE)
+			continue;
+		_rtw_memcpy(&(addrs[j][0]), pre_link_sta_ctl->node[i].addr, ETH_ALEN);
+		pre_link_sta_ctl->node[i].valid = _FALSE;
+		pre_link_sta_ctl->num--;
+		j++;
+	}
+	_rtw_spinunlock_bh(&(pre_link_sta_ctl->lock));
+
+	for (i = 0; i < j; i++) {
+		sta = rtw_get_stainfo(stapriv, &(addrs[i][0]));
+		if (!sta)
+			continue;
+
+		if (sta->state == WIFI_FW_PRE_LINK)
+			rtw_free_stainfo(stapriv->padapter, sta);
+	}
+}
+
+void rtw_pre_link_sta_ctl_init(struct sta_priv *stapriv)
+{
+	struct pre_link_sta_ctl_t *pre_link_sta_ctl = &stapriv->pre_link_sta_ctl;
+	int i;
+
+	_rtw_spinlock_init(&pre_link_sta_ctl->lock);
+	pre_link_sta_ctl->num = 0;
+	for (i = 0; i < RTW_PRE_LINK_STA_NUM; i++)
+		pre_link_sta_ctl->node[i].valid = _FALSE;
+}
+
+void rtw_pre_link_sta_ctl_deinit(struct sta_priv *stapriv)
+{
+	struct pre_link_sta_ctl_t *pre_link_sta_ctl = &stapriv->pre_link_sta_ctl;
+	int i;
+
+	rtw_pre_link_sta_ctl_reset(stapriv);
+
+	_rtw_spinlock_free(&pre_link_sta_ctl->lock);
+}
+
+void dump_pre_link_sta_ctl(void *sel, struct sta_priv *stapriv)
+{
+	struct pre_link_sta_ctl_t *pre_link_sta_ctl = &stapriv->pre_link_sta_ctl;
+	int i;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	RTW_PRINT_SEL(sel, "num:%d/%d\n", pre_link_sta_ctl->num, RTW_PRE_LINK_STA_NUM);
+
+	for (i = 0; i < RTW_PRE_LINK_STA_NUM; i++) {
+		if (pre_link_sta_ctl->node[i].valid == _FALSE)
+			continue;
+		RTW_PRINT_SEL(sel, "%s\n", get_macaddr_str(mac_addr_str, sel, pre_link_sta_ctl->node[i].addr));
+	}
+}
+#endif /* CONFIG_RTW_PRE_LINK_STA */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_swcrypto.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_swcrypto.c
new file mode 100644
index 000000000000..1285aaebda07
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_swcrypto.c
@@ -0,0 +1,319 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include <drv_types.h>
+#include <aes.h>
+#include <aes_siv.h>
+#include <aes_wrap.h>
+#include <sha256.h>
+#include <wlancrypto_wrap.h>
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+int _rtw_core_ccmp_encrypt(u8 *key, u32 key_len, uint hdrlen, u8 *phdr, uint datalen, u8 *pdata)
+{
+	u8 *enc = NULL;
+	size_t enc_len = 0;
+
+	if (key_len == 16) { /* 128 bits */
+		core_ccmp_encrypt(key,
+			hdrlen, phdr,
+			datalen, pdata,
+			(hdrlen == 26) ? (phdr + hdrlen - 2) : NULL,
+			NULL, 0, &enc_len);
+	} else if (key_len == 32) { /* 256 bits */
+		core_ccmp_256_encrypt(key,
+			hdrlen, phdr,
+			datalen, pdata,
+			(hdrlen == 26) ? (phdr + hdrlen - 2) : NULL,
+			NULL, 0, &enc_len);
+	}
+
+	return _SUCCESS;
+}
+#endif
+
+/**
+ * rtw_ccmp_encrypt - 
+ * @key: the temporal key 
+ * @hdrlen: mac header length
+ * @frame: the frame including the mac header, pn and payload
+ * @plen: payload length, i.e., length of the plain text, without PN and MIC
+ */
+int _rtw_ccmp_encrypt(_adapter *padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame, uint plen)
+{
+	u8 *enc = NULL;
+	size_t enc_len = 0;
+
+	if (key_len == 16) { /* 128 bits */
+		enc = ccmp_encrypt(padapter, key,
+			frame,
+			hdrlen + plen,
+			hdrlen,
+			(hdrlen == 26) ? (frame + hdrlen - 2) : NULL,
+			NULL, 0, &enc_len);
+	} else if (key_len == 32) { /* 256 bits */
+		enc = ccmp_256_encrypt(padapter, key,
+			frame,
+			hdrlen + plen,
+			hdrlen,
+			(hdrlen == 26) ? (frame + hdrlen - 2) : NULL,
+			NULL, 0, &enc_len);
+	}
+
+	if (enc == NULL) {
+		RTW_INFO("Failed to encrypt CCMP(%u) frame", key_len);
+		return _FAIL;
+	}
+
+	/* Copy @enc back to @frame and free @enc */
+	_rtw_memcpy(frame, enc, enc_len);
+	rtw_mfree(enc, enc_len + AES_BLOCK_SIZE);
+
+	return _SUCCESS;
+}
+
+
+/**
+ * rtw_ccmp_decrypt -
+ * @key: the temporal key
+ * @hdrlen: length of the mac header
+ * @frame: the raw frame (@hdrlen + PN + enc_data + MIC)
+ * @plen: length of the frame (@hdrlen + PN + enc_data + MIC)
+ */
+int _rtw_ccmp_decrypt(_adapter * padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame,
+	uint plen)
+{
+	u8 *plain = NULL;
+	size_t plain_len = 0;
+	const struct ieee80211_hdr *hdr;
+
+	hdr = (const struct ieee80211_hdr *)frame;
+
+	if (key_len == 16) { /* 128 bits */
+		plain = ccmp_decrypt(padapter, key,
+			hdr,
+			frame + hdrlen, /* PN + enc_data + MIC */
+			plen - hdrlen, /* PN + enc_data + MIC */
+			&plain_len);
+	} else if (key_len == 32) { /* 256 bits */
+		plain = ccmp_256_decrypt(padapter, key,
+			hdr,
+			frame + hdrlen, /* PN + enc_data + MIC */
+			plen - hdrlen, /* PN + enc_data + MIC */
+			&plain_len);
+	}
+
+	if (plain == NULL) {
+		RTW_INFO("Failed to decrypt CCMP(%u) frame", key_len);
+		return _FAIL;
+	}
+
+	/* Copy @plain back to @frame and free @plain */
+	_rtw_memcpy(frame + hdrlen + 8, plain, plain_len);
+	rtw_mfree(plain, plen - hdrlen + AES_BLOCK_SIZE);
+
+	RTW_DBG_DUMP("ccmp_decrypt(): decrypted frame\n",
+		frame, hdrlen + 8 + plen);
+
+	return _SUCCESS;
+}
+
+
+#ifdef CONFIG_RTW_MESH_AEK
+/* wrapper to ase_siv_encrypt and aes_siv_decrypt */
+int _aes_siv_encrypt(const u8 *key, size_t key_len,
+	const u8 *pw, size_t pwlen,
+	size_t num_elem, const u8 *addr[], const size_t *len, u8 *out)
+{
+	return aes_siv_encrypt(key, key_len, pw, pwlen, num_elem, addr, len, out);
+}
+int _aes_siv_decrypt(const u8 *key, size_t key_len,
+	const u8 *iv_crypt, size_t iv_c_len,
+	size_t num_elem, const u8 *addr[], const size_t *len, u8 *out)
+{
+	return aes_siv_decrypt(key, key_len, iv_crypt, iv_c_len, num_elem, addr, len, out);
+}
+#endif
+
+
+/**
+ * _rtw_gcmp_encrypt - 
+ * @key: the temporal key 
+ * @hdrlen: mac header length
+ * @frame: the frame including the mac header, pn and payload
+ * @plen: payload length, i.e., length of the plain text, without PN and MIC
+ */
+int _rtw_gcmp_encrypt(_adapter * padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame, uint plen)
+{
+	u8 *enc = NULL;
+	size_t enc_len = 0;
+
+	enc = gcmp_encrypt(padapter, key, key_len,
+		frame,
+		hdrlen + plen,
+		hdrlen,
+		(hdrlen == 26) ? (frame + hdrlen - 2) : NULL,
+		NULL, 0, &enc_len);
+	if (enc == NULL) {
+		RTW_INFO("Failed to encrypt GCMP frame");
+		return _FAIL;
+	}
+
+	/* Copy @enc back to @frame and free @enc */
+	_rtw_memcpy(frame, enc, enc_len);
+	rtw_mfree(enc, enc_len + AES_BLOCK_SIZE);
+
+	return _SUCCESS;
+}
+
+
+/**
+ * _rtw_gcmp_decrypt -
+ * @key: the temporal key
+ * @hdrlen: length of the mac header
+ * @frame: the raw frame (@hdrlen + PN + enc_data + MIC)
+ * @plen: length of the frame (@hdrlen + PN + enc_data + MIC)
+ */
+int _rtw_gcmp_decrypt(_adapter *padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame, uint plen)
+{
+	u8 *plain = NULL;
+	size_t plain_len = 0;
+	const struct ieee80211_hdr *hdr;
+
+	hdr = (const struct ieee80211_hdr *)frame;
+
+	plain = gcmp_decrypt(padapter, key, key_len,
+		hdr,
+		frame + hdrlen, /* PN + enc_data + MIC */
+		plen - hdrlen, /* PN + enc_data + MIC */
+		&plain_len);
+
+	if (plain == NULL) {
+		RTW_INFO("Failed to decrypt GCMP(%u) frame", key_len);
+		return _FAIL;
+	}
+
+	/* Copy @plain back to @frame and free @plain */
+	_rtw_memcpy(frame + hdrlen + 8, plain, plain_len);
+	rtw_mfree(plain, plen - hdrlen + AES_BLOCK_SIZE);
+
+	RTW_DBG_DUMP("gcmp_decipher(): decrypted frame\n",
+		frame, hdrlen + 8 + plen);
+
+	return _SUCCESS;
+}
+
+
+#if  defined(CONFIG_IEEE80211W) | defined(CONFIG_TDLS)
+u8 _bip_ccmp_protect(const u8 *key, size_t key_len,
+	const u8 *data, size_t data_len, u8 *mic)
+{
+	u8 res = _SUCCESS;
+
+	if (key_len == 16) {
+		if (omac1_aes_128(key, data, data_len, mic)) {
+			res = _FAIL;
+			RTW_ERR("%s : omac1_aes_128 fail!", __func__);
+		}
+	} else if (key_len == 32) {
+		if (omac1_aes_256(key, data, data_len, mic)) {
+			res = _FAIL;
+			RTW_ERR("%s : omac1_aes_256 fail!", __func__);
+		}
+	} else {
+		RTW_ERR("%s : key_len not match!", __func__);
+		res = _FAIL;
+	}
+
+	return  res;
+}
+
+
+u8 _bip_gcmp_protect(u8 *whdr_pos, size_t len,
+	const u8 *key, size_t key_len,
+	const u8 *data, size_t data_len, u8 *mic)
+{
+	u8 res = _SUCCESS;
+	u32 mic_len = 16;
+	u8 nonce[12], *npos;
+	const u8 *gcmp_ipn;
+
+	gcmp_ipn = whdr_pos + len - mic_len - 6;
+
+	/* Nonce: A2 | IPN */
+	_rtw_memcpy(nonce, get_addr2_ptr(whdr_pos), ETH_ALEN);
+	npos = nonce + ETH_ALEN;
+	*npos++ = gcmp_ipn[5];
+	*npos++ = gcmp_ipn[4];
+	*npos++ = gcmp_ipn[3];
+	*npos++ = gcmp_ipn[2];
+	*npos++ = gcmp_ipn[1];
+	*npos++ = gcmp_ipn[0];
+
+	if (aes_gmac(key, key_len, nonce, sizeof(nonce),
+			data, data_len, mic)) {
+		res = _FAIL;
+		RTW_ERR("%s : aes_gmac fail!", __func__);
+	}
+
+	return res;
+}
+#endif /* CONFIG_IEEE80211W */
+
+
+#ifdef CONFIG_TDLS
+void _tdls_generate_tpk(void *sta, const u8 *own_addr, const u8 *bssid)
+{
+	struct sta_info *psta = (struct sta_info *)sta;
+	u8 *SNonce = psta->SNonce;
+	u8 *ANonce = psta->ANonce;
+
+	u8 key_input[SHA256_MAC_LEN];
+	const u8 *nonce[2];
+	size_t len[2];
+	u8 data[3 * ETH_ALEN];
+
+	/* IEEE Std 802.11z-2010 8.5.9.1:
+	 * TPK-Key-Input = SHA-256(min(SNonce, ANonce) || max(SNonce, ANonce))
+	 */
+	len[0] = 32;
+	len[1] = 32;
+	if (_rtw_memcmp2(SNonce, ANonce, 32) < 0) {
+		nonce[0] = SNonce;
+		nonce[1] = ANonce;
+	} else {
+		nonce[0] = ANonce;
+		nonce[1] = SNonce;
+	}
+
+	sha256_vector(2, nonce, len, key_input);
+
+	/*
+	 * TPK = KDF-Hash-Length(TPK-Key-Input, "TDLS PMK",
+	 *	min(MAC_I, MAC_R) || max(MAC_I, MAC_R) || BSSID)
+	 */
+
+	if (_rtw_memcmp2(own_addr, psta->phl_sta->mac_addr, ETH_ALEN) < 0) {
+		_rtw_memcpy(data, own_addr, ETH_ALEN);
+		_rtw_memcpy(data + ETH_ALEN, psta->phl_sta->mac_addr, ETH_ALEN);
+	} else {
+		_rtw_memcpy(data, psta->phl_sta->mac_addr, ETH_ALEN);
+		_rtw_memcpy(data + ETH_ALEN, own_addr, ETH_ALEN);
+	}
+
+	_rtw_memcpy(data + 2 * ETH_ALEN, bssid, ETH_ALEN);
+
+	sha256_prf(key_input, SHA256_MAC_LEN, "TDLS PMK", data, sizeof(data), (u8 *)&psta->tpk, sizeof(psta->tpk));
+}
+#endif /* CONFIG_TDLS */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_tdls.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_tdls.c
new file mode 100644
index 000000000000..6950b9a7310a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_tdls.c
@@ -0,0 +1,3901 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_TDLS_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_TDLS
+#define ONE_SEC 	1000 /* 1000 ms */
+
+extern unsigned char MCS_rate_2R[16];
+extern unsigned char MCS_rate_1R[16];
+
+bool rtw_check_tdls_frame(void *priv, struct rtw_phl_stainfo_t *phl_sta,
+		struct rtw_recv_pkt *phlrx)
+{
+	struct dvobj_priv *d = (struct dvobj_priv *)priv;
+	struct rtw_wifi_role_t *wrole = phl_sta->wrole;
+	struct _ADAPTER *a = d->padapters[wrole->id];
+	struct security_priv *psecuritypriv = &a->securitypriv;
+	struct sta_info *psta = rtw_get_stainfo(&a->stapriv, phl_sta->mac_addr);
+	struct rtw_r_meta_data *mdata = &phlrx->mdata;
+	struct rtw_pkt_buf_list *pktbuf;
+	bool is_tdls = _FALSE;
+	u8 a4_shift, privacy, iv_len, icv_len, hdrlen, encrypt;
+	u8 *psnap_type, *pcategory;
+	static u8 SNAP_ETH_TYPE_TDLS[2] = {0x89, 0x0d};
+
+	if (wrole->type != PHL_RTYPE_STATION && wrole->type != PHL_RTYPE_TDLS)
+		goto _exit;
+
+	/* check TDLS frame */
+	if (mdata->frame_type != RTW_FRAME_TYPE_DATA)
+		goto _exit;
+
+	rtw_warn_on(phlrx->pkt_cnt > 1);
+	pktbuf = phlrx->pkt_list; /* &phlrx->pkt_list[0] */
+
+	a4_shift = mdata->a4_frame ? ETH_ALEN : 0;
+	privacy = GetPrivacy(pktbuf->vir_addr);
+
+	if (mdata->qos)
+		hdrlen = WLAN_HDR_A3_QOS_LEN + a4_shift;
+	else
+		hdrlen = WLAN_HDR_A3_LEN + a4_shift;
+
+	if (privacy) {
+		if ((psta->tdls_sta_state & TDLS_LINKED_STATE) && (psta->dot118021XPrivacy == _AES_))
+			encrypt = psta->dot118021XPrivacy;
+		else
+			GET_ENCRY_ALGO(psecuritypriv, psta, encrypt, IS_MCAST(GetAddr1Ptr(pktbuf->vir_addr)));
+		SET_ICE_IV_LEN(iv_len, icv_len, encrypt);
+	} else {
+		iv_len = icv_len = 0;
+	}
+
+	psnap_type = pktbuf->vir_addr + hdrlen + iv_len + SNAP_SIZE;
+	pcategory = psnap_type + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	if ((_rtw_memcmp(psnap_type, SNAP_ETH_TYPE_TDLS, ETH_TYPE_LEN)) &&
+		(*pcategory == RTW_WLAN_CATEGORY_TDLS))
+		is_tdls = _TRUE;
+
+_exit:
+
+	return is_tdls;
+}
+
+inline void rtw_tdls_set_link_established(_adapter *adapter, bool en)
+{
+	adapter->tdlsinfo.link_established = en;
+	rtw_mi_update_iface_status(&(adapter->mlmepriv), 0);
+}
+
+void rtw_reset_tdls_info(_adapter *padapter)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	ptdlsinfo->ap_prohibited = _FALSE;
+
+	/* For TDLS channel switch, currently we only allow it to work in wifi logo test mode */
+	if (padapter->registrypriv.wifi_spec == 1)
+		ptdlsinfo->ch_switch_prohibited = _FALSE;
+	else
+		ptdlsinfo->ch_switch_prohibited = _TRUE;
+
+	rtw_tdls_set_link_established(padapter, _FALSE);
+	ptdlsinfo->sta_cnt = 0;
+	ptdlsinfo->sta_maximum = _FALSE;
+
+#ifdef CONFIG_TDLS_CH_SW
+	ptdlsinfo->chsw_info.ch_sw_state = TDLS_STATE_NONE;
+	ATOMIC_SET(&ptdlsinfo->chsw_info.chsw_on, _FALSE);
+	ptdlsinfo->chsw_info.off_ch_num = 0;
+	ptdlsinfo->chsw_info.ch_offset = CHAN_OFFSET_NO_EXT;
+	ptdlsinfo->chsw_info.cur_time = 0;
+	ptdlsinfo->chsw_info.delay_switch_back = _FALSE;
+	ptdlsinfo->chsw_info.dump_stack = _FALSE;
+#endif
+
+	ptdlsinfo->ch_sensing = 0;
+	ptdlsinfo->watchdog_count = 0;
+	ptdlsinfo->dev_discovered = _FALSE;
+
+#ifdef CONFIG_WFD
+	ptdlsinfo->wfd_info = &padapter->wfd_info;
+#endif
+
+	ptdlsinfo->tdls_sctx = NULL;
+}
+
+int rtw_init_tdls_info(_adapter *padapter)
+{
+	int	res = _SUCCESS;
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	rtw_reset_tdls_info(padapter);
+
+#ifdef CONFIG_TDLS_DRIVER_SETUP
+	ptdlsinfo->driver_setup = _TRUE;
+#else
+	ptdlsinfo->driver_setup = _FALSE;
+#endif /* CONFIG_TDLS_DRIVER_SETUP */
+
+	_rtw_spinlock_init(&ptdlsinfo->cmd_lock);
+	_rtw_spinlock_init(&ptdlsinfo->hdl_lock);
+
+	return res;
+
+}
+
+void rtw_free_tdls_info(struct tdls_info *ptdlsinfo)
+{
+	_rtw_spinlock_free(&ptdlsinfo->cmd_lock);
+	_rtw_spinlock_free(&ptdlsinfo->hdl_lock);
+
+	_rtw_memset(ptdlsinfo, 0, sizeof(struct tdls_info));
+
+}
+
+void rtw_free_all_tdls_sta(_adapter *padapter, u8 enqueue_cmd)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	_list	*plist, *phead;
+	s32	index;
+	struct sta_info *psta = NULL;
+	struct sta_info *ptdls_sta[NUM_STA];
+	u8 empty_hwaddr[ETH_ALEN] = { 0x00 };
+
+	_rtw_memset(ptdls_sta, 0x00, sizeof(ptdls_sta));
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (index = 0; index < NUM_STA; index++) {
+		phead = &(pstapriv->sta_hash[index]);
+		plist = get_next(phead);
+
+		while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+
+			plist = get_next(plist);
+
+			if (psta->tdls_sta_state != TDLS_STATE_NONE)
+				ptdls_sta[index] = psta;
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	for (index = 0; index < NUM_STA; index++) {
+		if (ptdls_sta[index]) {
+			struct TDLSoption_param tdls_param;
+
+			psta = ptdls_sta[index];
+
+			RTW_INFO("Do tear down to "MAC_FMT" by enqueue_cmd = %d\n", MAC_ARG(psta->phl_sta->mac_addr), enqueue_cmd);
+
+			_rtw_memcpy(&(tdls_param.addr), psta->phl_sta->mac_addr, ETH_ALEN);
+			tdls_param.option = TDLS_TEARDOWN_STA_NO_WAIT;
+			tdls_hdl(padapter, (unsigned char *)&(tdls_param));
+		}
+	}
+}
+
+int check_ap_tdls_prohibited(u8 *pframe, u8 pkt_len)
+{
+	u8 tdls_prohibited_bit = 0x40; /* bit(38); TDLS_prohibited */
+
+	if (pkt_len < 5)
+		return _FALSE;
+
+	pframe += 4;
+	if ((*pframe) & tdls_prohibited_bit)
+		return _TRUE;
+
+	return _FALSE;
+}
+
+int check_ap_tdls_ch_switching_prohibited(u8 *pframe, u8 pkt_len)
+{
+	u8 tdls_ch_swithcing_prohibited_bit = 0x80; /* bit(39); TDLS_channel_switching prohibited */
+
+	if (pkt_len < 5)
+		return _FALSE;
+
+	pframe += 4;
+	if ((*pframe) & tdls_ch_swithcing_prohibited_bit)
+		return _TRUE;
+
+	return _FALSE;
+}
+
+u8 rtw_is_tdls_enabled(_adapter *padapter)
+{
+	return padapter->registrypriv.en_tdls;
+}
+
+void rtw_set_tdls_enable(_adapter *padapter, u8 enable)
+{
+	padapter->registrypriv.en_tdls = enable;
+	RTW_INFO("%s: en_tdls = %d\n", __func__, rtw_is_tdls_enabled(padapter));
+}
+
+void rtw_enable_tdls_func(_adapter *padapter)
+{
+	if (rtw_is_tdls_enabled(padapter) == _TRUE)
+		return;
+	rtw_set_tdls_enable(padapter, _TRUE);
+}
+
+void rtw_disable_tdls_func(_adapter *padapter, u8 enqueue_cmd)
+{
+	if (rtw_is_tdls_enabled(padapter) == _FALSE)
+		return;
+
+	rtw_free_all_tdls_sta(padapter, enqueue_cmd);
+	rtw_reset_tdls_info(padapter);
+
+	rtw_set_tdls_enable(padapter, _FALSE);
+}
+
+u8 rtw_is_tdls_sta_existed(_adapter *padapter)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	int i = 0;
+	_list	*plist, *phead;
+	u8 ret = _FALSE;
+
+	if (rtw_is_tdls_enabled(padapter) == _FALSE)
+		return _FALSE;
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+			if (psta->tdls_sta_state != TDLS_STATE_NONE) {
+				ret = _TRUE;
+				goto Exit;
+			}
+		}
+	}
+
+Exit:
+
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	return ret;
+}
+
+u8 rtw_tdls_is_setup_allowed(_adapter *padapter)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	if (is_client_associated_to_ap(padapter) == _FALSE)
+		return _FALSE;
+
+	if (ptdlsinfo->ap_prohibited == _TRUE)
+		return _FALSE;
+
+	return _TRUE;
+}
+
+#ifdef CONFIG_TDLS_CH_SW
+u8 rtw_tdls_is_chsw_allowed(_adapter *padapter)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	if (ptdlsinfo->ch_switch_prohibited == _TRUE)
+		return _FALSE;
+
+	if (padapter->registrypriv.wifi_spec == 0)
+		return _FALSE;
+
+	return _TRUE;
+}
+#endif
+
+void _update_tdls_mgntframe_attrib(_adapter *padapter, struct sta_info *sta,
+				struct pkt_attrib *pattrib)
+{
+	u8	wireless_mode;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+	struct sta_info		*bmc_sta = NULL;
+	struct _ADAPTER_LINK *padapter_link = sta->padapter_link;
+
+	pattrib->type = WIFI_MGT_TYPE;
+	pattrib->hdrlen = 24;
+	pattrib->nr_frags = 1;
+	pattrib->priority = 8;
+	pattrib->mac_id = sta->phl_sta->macid;
+	pattrib->adapter_link = padapter_link;
+	pattrib->pktlen = 0;
+
+	if (IS_CCK_RATE(padapter_link->mlmeextpriv.tx_rate))
+		wireless_mode = WLAN_MD_11B;
+	else
+		wireless_mode = WLAN_MD_11G;
+	pattrib->rate = padapter_link->mlmeextpriv.tx_rate;
+
+	pattrib->encrypt = _NO_PRIVACY_;
+	pattrib->bswenc = _FALSE;
+
+	pattrib->qos_en = _FALSE;
+	pattrib->ht_en = _FALSE;
+	pattrib->bwmode = CHANNEL_WIDTH_20;
+	pattrib->ch_offset = CHAN_OFFSET_NO_EXT;
+	pattrib->sgi = _FALSE;
+
+	pattrib->seqnum = pmlmeext->mgnt_seq;
+
+	pattrib->retry_ctrl = _TRUE;
+
+	pattrib->mbssid = 0;
+	pattrib->hw_ssn_sel = pxmitpriv->hw_ssn_seq_no;
+}
+
+int _issue_nulldata_to_TDLS_peer_STA(_adapter *padapter, struct sta_info *sta,
+		unsigned int power_mode, int wait_ms)
+{
+	int ret = _FAIL;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	unsigned char					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl, *qc;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct _ADAPTER_LINK *padapter_link = sta->padapter_link;
+	struct link_mlme_ext_info	*lmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+	_update_tdls_mgntframe_attrib(padapter, sta, pattrib);
+
+	pattrib->hdrlen += 2;
+	pattrib->qos_en = _TRUE;
+	pattrib->eosp = 1;
+	pattrib->ack_policy = 0;
+	pattrib->mdata = 0;
+	pattrib->retry_ctrl = _FALSE;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	if (power_mode)
+		SetPwrMgt(fctrl);
+
+	qc = (unsigned short *)(pframe + pattrib->hdrlen - 2);
+
+	SetPriority(qc, 7);	/* Set priority to VO */
+
+	SetEOSP(qc, pattrib->eosp);
+
+	SetAckpolicy(qc, pattrib->ack_policy);
+
+	_rtw_memcpy(pwlanhdr->addr1, sta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(lmlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_QOS_DATA_NULL);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr_qos);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr_qos);
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	if (wait_ms)
+		ret = dump_mgntframe_and_wait_ack_timeout(padapter, pmgntframe, wait_ms);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+	return ret;
+
+}
+
+/*
+ *wait_ms == 0 means that there is no need to wait ack through C2H_CCX_TX_RPT
+ *wait_ms > 0 means you want to wait ack through C2H_CCX_TX_RPT, and the value of wait_ms means the interval between each TX
+ *try_cnt means the maximal TX count to try
+ */
+int issue_nulldata_to_TDLS_peer_STA(_adapter *padapter, struct sta_info *sta,
+		unsigned int power_mode, int try_cnt, int wait_ms)
+{
+	int ret;
+	int i = 0;
+	systime start = rtw_get_current_time();
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+#if 0
+	psta = rtw_get_stainfo(&padapter->stapriv, da);
+	if (psta) {
+		if (power_mode)
+			rtw_hal_macid_sleep(padapter, psta->phl_sta->macid);
+		else
+			rtw_hal_macid_wakeup(padapter, psta->phl_sta->macid);
+	} else {
+		RTW_INFO(FUNC_ADPT_FMT ": Can't find sta info for " MAC_FMT ", skip macid %s!!\n",
+			FUNC_ADPT_ARG(padapter), MAC_ARG(da), power_mode ? "sleep" : "wakeup");
+		rtw_warn_on(1);
+	}
+#endif
+
+	do {
+		ret = _issue_nulldata_to_TDLS_peer_STA(padapter, sta, power_mode, wait_ms);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && (ret == _FAIL || wait_ms == 0));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		if (sta)
+			RTW_INFO(FUNC_ADPT_FMT" to "MAC_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter), MAC_ARG(sta->phl_sta->mac_addr), rtw_get_oper_ch(padapter, sta->padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+		else
+			RTW_INFO(FUNC_ADPT_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(padapter), rtw_get_oper_ch(padapter, sta->padapter_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+exit:
+	return ret;
+}
+
+/* TDLS encryption(if needed) will always be CCMP */
+void rtw_tdls_set_key(_adapter *padapter, struct sta_info *ptdls_sta)
+{
+	ptdls_sta->dot118021XPrivacy = _AES_;
+	rtw_setstakey_cmd(padapter, ptdls_sta, TDLS_KEY, _TRUE);
+}
+
+#ifdef CONFIG_80211N_HT
+void rtw_tdls_process_ht_cap(_adapter *padapter, struct sta_info *ptdls_sta, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct _ADAPTER_LINK	*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct ht_priv			*phtpriv = &pmlmepriv->htpriv;
+	u8	max_AMPDU_len, min_MPDU_spacing;
+	u8	cur_ldpc_cap = 0, cur_stbc_cap = 0, cur_beamform_cap = 0;
+
+	/* Save HT capabilities in the sta object */
+	_rtw_memset(&ptdls_sta->htpriv.ht_cap, 0, sizeof(struct rtw_ieee80211_ht_cap));
+	if (pIE->data && pIE->Length >= sizeof(struct rtw_ieee80211_ht_cap)) {
+		ptdls_sta->flags |= WLAN_STA_HT;
+		ptdls_sta->flags |= WLAN_STA_WME;
+
+		_rtw_memcpy(&ptdls_sta->htpriv.ht_cap, pIE->data, sizeof(struct rtw_ieee80211_ht_cap));
+	} else {
+		ptdls_sta->flags &= ~WLAN_STA_HT;
+		return;
+	}
+
+	if (ptdls_sta->flags & WLAN_STA_HT) {
+		if (padapter->registrypriv.ht_enable == _TRUE && is_supported_ht(padapter->registrypriv.wireless_mode) ) {
+			ptdls_sta->htpriv.ht_option = _TRUE;
+			ptdls_sta->qos_option = _TRUE;
+		} else {
+			ptdls_sta->htpriv.ht_option = _FALSE;
+			ptdls_sta->qos_option = _FALSE;
+		}
+	}
+
+	/* HT related cap */
+	if (ptdls_sta->htpriv.ht_option) {
+		/* Check if sta supports rx ampdu */
+		if (padapter->registrypriv.ampdu_enable == 1)
+			ptdls_sta->ampdu_priv.ampdu_enable = _TRUE;
+
+		/* AMPDU Parameters field */
+
+		/* Get MIN of MAX AMPDU Length Exp */
+		if ((pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x3) > (*(pIE->data + 2) & 0x3))
+			max_AMPDU_len = (*(pIE->data + 2) & 0x3);
+		else
+			max_AMPDU_len = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x3);
+		/* Get MAX of MIN MPDU Start Spacing */
+		if ((pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c) > (*(pIE->data + 2) & 0x1c))
+			min_MPDU_spacing = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c);
+		else
+			min_MPDU_spacing = (*(pIE->data + 2) & 0x1c);
+		ptdls_sta->ampdu_priv.rx_ampdu_min_spacing = max_AMPDU_len | min_MPDU_spacing;
+
+
+		/* Check if sta support s Short GI 20M */
+		if ((phtpriv->sgi_20m == _TRUE) && (ptdls_sta->htpriv.ht_cap.cap_info & cpu_to_le16(IEEE80211_HT_CAP_SGI_20)))
+			ptdls_sta->htpriv.sgi_20m = _TRUE;
+
+		/* Check if sta support s Short GI 40M */
+		if ((phtpriv->sgi_40m == _TRUE) && (ptdls_sta->htpriv.ht_cap.cap_info & cpu_to_le16(IEEE80211_HT_CAP_SGI_40)))
+			ptdls_sta->htpriv.sgi_40m = _TRUE;
+
+		/* Bwmode would still followed AP's setting */
+		if (ptdls_sta->htpriv.ht_cap.cap_info & cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH)) {
+			if (pmlmeext->chandef.bw >= CHANNEL_WIDTH_40)
+				ptdls_sta->phl_sta->chandef.bw = CHANNEL_WIDTH_40;
+			ptdls_sta->htpriv.ch_offset = pmlmeext->chandef.offset;
+		}
+
+		/* Config LDPC Coding Capability */
+		if (TEST_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_TX) && GET_HT_CAP_ELE_LDPC_CAP(pIE->data)) {
+			SET_FLAG(cur_ldpc_cap, (LDPC_HT_ENABLE_TX | LDPC_HT_CAP_TX));
+			RTW_INFO("Enable HT Tx LDPC!\n");
+		}
+		ptdls_sta->htpriv.ldpc_cap = cur_ldpc_cap;
+
+		/* Config STBC setting */
+		if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX) && GET_HT_CAP_ELE_RX_STBC(pIE->data)) {
+			SET_FLAG(cur_stbc_cap, (STBC_HT_ENABLE_TX | STBC_HT_CAP_TX));
+			RTW_INFO("Enable HT Tx STBC!\n");
+		}
+		ptdls_sta->htpriv.stbc_cap = cur_stbc_cap;
+
+#ifdef CONFIG_BEAMFORMING
+		/* Config Tx beamforming setting */
+		if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE) &&
+			GET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP(pIE->data))
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
+
+		if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE) &&
+			GET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP(pIE->data))
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
+		ptdls_sta->htpriv.beamform_cap = cur_beamform_cap;
+		if (cur_beamform_cap)
+			RTW_INFO("Client HT Beamforming Cap = 0x%02X\n", cur_beamform_cap);
+#endif /* CONFIG_BEAMFORMING */
+	}
+
+}
+
+u8 *rtw_tdls_set_ht_cap(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	rtw_ht_use_default_setting(padapter, padapter_link);
+
+	if (padapter->registrypriv.wifi_spec == 1) {
+		padapter_link->mlmepriv.htpriv.sgi_20m = _FALSE;
+		padapter_link->mlmepriv.htpriv.sgi_40m = _FALSE;
+	}
+
+	rtw_restructure_ht_ie(padapter, padapter_link, NULL, pframe, 0, &(pattrib->pktlen), padapter_link->mlmeextpriv.chandef.chan);
+
+	return pframe + pattrib->pktlen;
+}
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+void rtw_tdls_process_vht_cap(_adapter *padapter, struct sta_info *ptdls_sta, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct _ADAPTER_LINK	*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv			*pvhtpriv = &pmlmepriv->vhtpriv;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8	cur_ldpc_cap = 0, cur_stbc_cap = 0, tx_nss = 0;
+	u16 cur_beamform_cap = 0;
+	u8	*pcap_mcs;
+
+	if (pIE == NULL)
+		return;
+
+	_rtw_memset(&ptdls_sta->vhtpriv, 0, sizeof(struct vht_priv));
+	if (pIE->data && pIE->Length == 12) {
+		ptdls_sta->flags |= WLAN_STA_VHT;
+
+		_rtw_memcpy(ptdls_sta->vhtpriv.vht_cap, pIE->data, 12);
+
+#if 0
+		if (elems.vht_op_mode_notify && elems.vht_op_mode_notify_len == 1)
+			_rtw_memcpy(&pstat->vhtpriv.vht_op_mode_notify, elems.vht_op_mode_notify, 1);
+		else /* for Frame without Operating Mode notify ie; default: 80M */
+			pstat->vhtpriv.vht_op_mode_notify = CHANNEL_WIDTH_80;
+#else
+		ptdls_sta->vhtpriv.vht_op_mode_notify = CHANNEL_WIDTH_80;
+#endif
+	} else {
+		ptdls_sta->flags &= ~WLAN_STA_VHT;
+		return;
+	}
+
+	if (ptdls_sta->flags & WLAN_STA_VHT) {
+		if (REGSTY_IS_11AC_ENABLE(&padapter->registrypriv)
+		    && is_supported_vht(padapter->registrypriv.wireless_mode)
+		    && RFCTL_REG_EN_11AC(rfctl)
+		) {
+			ptdls_sta->vhtpriv.vht_option = _TRUE;
+			/* ToDo: need to API to inform hal_sta->ra_info.is_vht_enable  */
+			#if 0
+			ptdls_sta->phl_sta->ra_info.is_vht_enable = _TRUE;
+			#endif
+		}
+		else
+			ptdls_sta->vhtpriv.vht_option = _FALSE;
+	}
+
+	/* Follows VHT_caps_handler */
+	if (ptdls_sta->vhtpriv.vht_option == _FALSE)
+		return;
+
+	if (ptdls_sta->phl_sta == NULL)
+		return;
+
+	phl_sta = ptdls_sta->phl_sta;
+
+	/* B4 Rx LDPC */
+	if (TEST_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_TX) &&
+	    GET_VHT_CAPABILITY_ELE_RX_LDPC(pIE->data)) {
+		SET_FLAG(cur_ldpc_cap, (LDPC_VHT_ENABLE_TX | LDPC_VHT_CAP_TX));
+		RTW_INFO("Current VHT LDPC Setting = %02X\n", cur_ldpc_cap);
+	}
+	ptdls_sta->vhtpriv.ldpc_cap = cur_ldpc_cap;
+
+	/* B5 Short GI for 80 MHz */
+	ptdls_sta->vhtpriv.sgi_80m = (GET_VHT_CAPABILITY_ELE_SHORT_GI80M(pIE->data) & pvhtpriv->sgi_80m) ? _TRUE : _FALSE;
+	/*RTW_INFO("Current ShortGI80MHz = %d\n", pvhtpriv->sgi_80m);*/
+
+	/* B8 B9 B10 Rx STBC */
+	if (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_TX) &&
+		GET_VHT_CAPABILITY_ELE_RX_STBC(pIE->data)) {
+		SET_FLAG(cur_stbc_cap, (STBC_VHT_ENABLE_TX | STBC_VHT_CAP_TX));
+		RTW_INFO("Current VHT STBC Setting = %02X\n", cur_stbc_cap);
+	}
+	ptdls_sta->vhtpriv.stbc_cap = cur_stbc_cap;
+
+	phl_sta->asoc_cap.vht_su_bfmr = GET_VHT_CAPABILITY_ELE_SU_BFER(pIE->data);
+	phl_sta->asoc_cap.vht_su_bfme = GET_VHT_CAPABILITY_ELE_SU_BFEE(pIE->data);
+	phl_sta->asoc_cap.vht_mu_bfmr = GET_VHT_CAPABILITY_ELE_MU_BFER(pIE->data);
+	phl_sta->asoc_cap.bfme_sts = GET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(pIE->data);
+	phl_sta->asoc_cap.num_snd_dim = GET_VHT_CAPABILITY_ELE_SU_BFER_SOUND_DIM_NUM(pIE->data);
+
+	RTW_INFO("%s: VHT STA assoc_cap:\n", __func__);
+	RTW_INFO("- SU BFer: %d\n", phl_sta->asoc_cap.vht_su_bfmr);
+	RTW_INFO("- SU BFee: %d\n", phl_sta->asoc_cap.vht_su_bfme);
+	RTW_INFO("- MU BFer: %d\n", phl_sta->asoc_cap.vht_mu_bfmr);
+	RTW_INFO("- BFee STS: %d\n", phl_sta->asoc_cap.bfme_sts);
+	RTW_INFO("- BFer SND DIM number: %d\n", phl_sta->asoc_cap.num_snd_dim);
+
+#ifdef CONFIG_BEAMFORMING
+	/*
+	 * B11 SU Beamformer Capable,
+	 * the target supports Beamformer and we are Beamformee
+	 */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE)
+		&& GET_VHT_CAPABILITY_ELE_SU_BFER(pIE->data)) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE);
+
+		/* Shift to BEAMFORMING_VHT_BEAMFORMEE_STS_CAP */
+		SET_FLAG(cur_beamform_cap, GET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(pIE->data) << 8);
+
+		/*
+		 * B19 MU Beamformer Capable,
+		 * the target supports Beamformer and we are Beamformee
+		 */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE)
+			&& GET_VHT_CAPABILITY_ELE_MU_BFER(pIE->data))
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE);
+	}
+
+	/*
+	 * B12 SU Beamformee Capable,
+	 * the target supports Beamformee and we are Beamformer
+	 */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE)
+		&& GET_VHT_CAPABILITY_ELE_SU_BFEE(pIE->data)) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+
+		/* Shit to BEAMFORMING_VHT_BEAMFORMER_SOUND_DIM */
+		SET_FLAG(cur_beamform_cap, GET_VHT_CAPABILITY_ELE_SU_BFER_SOUND_DIM_NUM(pIE->data) << 12);
+
+		/*
+		 * B20 MU Beamformee Capable,
+		 * the target supports Beamformee and we are Beamformer
+		 */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE)
+			&& GET_VHT_CAPABILITY_ELE_MU_BFEE(pIE->data))
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+	}
+
+	ptdls_sta->vhtpriv.beamform_cap = cur_beamform_cap;
+	RTW_INFO("Current VHT Beamforming Setting=0x%04X\n", cur_beamform_cap);
+	/* ToDo: need to API to inform hal_sta->bf_info.vht_beamform_cap  */
+	#if 0
+	ptdls_sta->phl_sta->bf_info.vht_beamform_cap = cur_beamform_cap;
+	#endif
+#endif /* CONFIG_BEAMFORMING */
+
+	/* B0 B1 Maximum MPDU Length */
+	ptdls_sta->vhtpriv.max_mpdu_len = GET_VHT_CAPABILITY_ELE_MAX_MPDU_LENGTH(pIE->data);
+	/* B23 B24 B25 Maximum A-MPDU Length Exponent */
+	ptdls_sta->vhtpriv.ampdu_len = GET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(pIE->data); 
+
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_RX_MCS(pIE->data);
+	tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	rtw_vht_nss_to_mcsmap(tx_nss, ptdls_sta->vhtpriv.vht_mcs_map, pcap_mcs);
+	ptdls_sta->vhtpriv.vht_highest_rate = rtw_get_vht_highest_rate(ptdls_sta->vhtpriv.vht_mcs_map);
+}
+
+void rtw_tdls_process_vht_operation(_adapter *padapter, struct sta_info *ptdls_sta, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct registry_priv *regsty = &padapter->registrypriv;
+	struct _ADAPTER_LINK	*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	u8 operation_bw = 0;
+
+	if (GET_VHT_OPERATION_ELE_CHL_WIDTH(pIE->data) >= CH_WIDTH_80_160M) {
+
+		operation_bw = CHANNEL_WIDTH_80;
+
+		if (rtw_hw_is_bw_support(adapter_to_dvobj(padapter), operation_bw) && REGSTY_IS_BW_5G_SUPPORT(regsty, operation_bw)
+			&& (operation_bw <= pmlmeext->chandef.bw))
+			ptdls_sta->phl_sta->chandef.bw = operation_bw;
+		else
+			ptdls_sta->phl_sta->chandef.bw = pmlmeext->chandef.bw;
+	} else
+		ptdls_sta->phl_sta->chandef.bw = pmlmeext->chandef.bw;
+	_rtw_memcpy(ptdls_sta->vhtpriv.vht_op, pIE->data,
+			pIE->Length > VHT_OP_IE_LEN ? VHT_OP_IE_LEN : pIE->Length);
+}
+
+void rtw_tdls_process_vht_op_mode_notify(_adapter *padapter, struct sta_info *ptdls_sta, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct registry_priv *regsty = &padapter->registrypriv;
+	struct _ADAPTER_LINK	*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv		*pvhtpriv = &pmlmepriv->vhtpriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	u8	target_bw;
+	u8	target_rxss, current_rxss;
+
+	if (pvhtpriv->vht_option == _FALSE)
+		return;
+
+	target_bw = GET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(pIE->data);
+	target_rxss = (GET_VHT_OPERATING_MODE_FIELD_RX_NSS(pIE->data) + 1);
+
+	if (rtw_hw_is_bw_support(adapter_to_dvobj(padapter), target_bw) && REGSTY_IS_BW_5G_SUPPORT(regsty, target_bw)
+		&& (target_bw <= pmlmeext->chandef.bw))
+		ptdls_sta->phl_sta->chandef.bw = target_bw;
+	else
+		ptdls_sta->phl_sta->chandef.bw = pmlmeext->chandef.bw;
+
+	current_rxss = rtw_vht_mcsmap_to_nss(ptdls_sta->vhtpriv.vht_mcs_map);
+	if (target_rxss != current_rxss) {
+		u8	vht_mcs_map[2] = {};
+
+		rtw_vht_nss_to_mcsmap(target_rxss, vht_mcs_map, ptdls_sta->vhtpriv.vht_mcs_map);
+		_rtw_memcpy(ptdls_sta->vhtpriv.vht_mcs_map, vht_mcs_map, 2);
+	}
+}
+
+u8 *rtw_tdls_set_aid(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	return rtw_set_ie(pframe, EID_AID, 2, (u8 *)&(padapter->mlmepriv.dev_cur_network.aid), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_vht_cap(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	u32 ie_len = 0;
+
+	rtw_vht_get_real_setting(padapter, pattrib->adapter_link);
+
+	ie_len = rtw_build_vht_cap_ie(padapter, pattrib->adapter_link, pframe);
+	pattrib->pktlen += ie_len;
+
+	return pframe + ie_len;
+}
+
+u8 *rtw_tdls_set_vht_operation(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib, enum band_type band, u8 channel)
+{
+	u32 ie_len = 0;
+
+	ie_len = rtw_build_vht_operation_ie(padapter, pattrib->adapter_link, pframe, band, channel);
+	pattrib->pktlen += ie_len;
+
+	return pframe + ie_len;
+}
+
+u8 *rtw_tdls_set_vht_op_mode_notify(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib, u8 bw)
+{
+	u32 ie_len = 0;
+
+	ie_len = rtw_build_vht_op_mode_notify_ie(padapter, pattrib->adapter_link, pframe, bw);
+	pattrib->pktlen += ie_len;
+
+	return pframe + ie_len;
+}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+u8 *rtw_tdls_set_he_cap(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	u32 ie_len = 0;
+	rtw_he_use_default_setting(padapter, pattrib->adapter_link);
+	ie_len = rtw_build_he_cap_ie(padapter, pattrib->adapter_link, pframe,
+				     pattrib->adapter_link->mlmeextpriv.chandef.band);
+	pattrib->pktlen += ie_len;
+
+	return pframe + pattrib->pktlen;
+}
+
+u8 *rtw_tdls_set_he_operation(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	u32 ie_len = 0;
+
+	ie_len = rtw_build_he_operation_ie(padapter, pattrib->adapter_link,
+			pframe, &pattrib->adapter_link->mlmeextpriv.chandef);
+	pattrib->pktlen += ie_len;
+
+	return pframe + ie_len;
+}
+
+/* Follows HE_caps_handler */
+void rtw_tdls_process_he_cap(_adapter *padapter, struct sta_info *ptdls_sta, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct _ADAPTER_LINK *padapter_link = ptdls_sta->padapter_link;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8 supp_mcs_len = 4;
+	u8 *ele_start = (&(pIE->data[0]) + 1);
+
+	if (pIE == NULL)
+		return;
+
+	_rtw_memset(&ptdls_sta->hepriv, 0, sizeof(struct he_priv));
+	if (pIE->data && pIE->Length >= 22 && pIE->Length <= HE_CAP_ELE_MAX_LEN) {
+		ptdls_sta->flags |= WLAN_STA_HE;
+		_rtw_memcpy(ptdls_sta->hepriv.he_cap, pIE->data, pIE->Length);
+
+	} else {
+		ptdls_sta->flags &= ~WLAN_STA_HE;
+		return;
+	}
+
+	if (ptdls_sta->flags & WLAN_STA_HE) {
+		if (REGSTY_IS_11AX_ENABLE(&padapter->registrypriv)
+			&& is_supported_he(padapter->registrypriv.wireless_mode)
+			/* CONFIG_80211AX_HE_TODO
+			&& (!rfctl->country_ent || COUNTRY_CHPLAN_EN_11AX(rfctl->country_ent)) */
+			) {
+			ptdls_sta->hepriv.he_option = _TRUE;
+			/* ToDo: need to API to inform hal_sta->ra_info.is_he_enable  */
+			#if 0
+			ptdls_sta->phl_sta->ra_info.is_he_enable = _TRUE;
+			#endif
+		}
+		else
+			ptdls_sta->hepriv.he_option = _FALSE;
+	}
+
+	if (ptdls_sta->hepriv.he_option == _FALSE)
+		return;
+
+	if (ptdls_sta->phl_sta == NULL)
+		return;
+
+	phl_sta = ptdls_sta->phl_sta;
+
+	/* HE MAC Caps */
+	HE_mac_caps_handler(padapter, phl_sta, ele_start);
+	ele_start += HE_CAP_ELE_MAC_CAP_LEN;
+
+	/* HE PHY Caps */
+	HE_phy_caps_handler(padapter, phl_sta, ele_start, &supp_mcs_len);
+	ele_start += HE_CAP_ELE_PHY_CAP_LEN;
+
+	/* HE Supp MCS Set */
+	rtw_he_set_asoc_cap_supp_mcs(padapter, phl_sta, ele_start, supp_mcs_len);
+	ptdls_sta->hepriv.he_highest_rate = rtw_he_get_highest_rate(phl_sta->asoc_cap.he_rx_mcs);
+	/* HE_supp_mcs_handler(padapter, phl_sta, ele_start, supp_mcs_len); */
+	ele_start += supp_mcs_len;
+
+	/* HE PPE Thresholds */
+	HE_ppe_thre_handler(padapter, phl_sta, ele_start);
+
+}
+
+/* Follows HE_operation_handler */
+void rtw_tdls_process_he_operation(_adapter *padapter, struct sta_info *ptdls_sta, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct _ADAPTER_LINK	*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct he_priv		*phepriv = &pmlmepriv->hepriv;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8 *ele_start = (&(pIE->data[0]) + 1);
+
+	if (phepriv->he_option == _FALSE)
+		return;
+
+	if (ptdls_sta->phl_sta == NULL)
+		return;
+
+	if (pIE->data && pIE->Length >= 7 && pIE->Length <= HE_OPER_ELE_MAX_LEN)
+		_rtw_memcpy(ptdls_sta->hepriv.he_op, pIE->data, pIE->Length);
+
+	phl_sta = ptdls_sta->phl_sta;
+
+	phl_sta->asoc_cap.er_su = !GET_HE_OP_PARA_ER_SU_DISABLE(ele_start);
+	phl_sta->asoc_cap.bsscolor = GET_HE_OP_BSS_COLOR_INFO_BSS_COLOR(ele_start);
+	phl_sta->tf_trs = _TRUE;
+}
+#endif
+
+u8 *rtw_tdls_set_sup_ch(_adapter *adapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	return rtw_chset_set_spt_chs_ie(adapter_to_chset(adapter), pframe, &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_rsnie(struct tdls_txmgmt *ptxmgmt, u8 *pframe, struct pkt_attrib *pattrib,  int init, struct sta_info *ptdls_sta)
+{
+	u8 *p = NULL;
+	int len = 0;
+
+	if (ptxmgmt->len > 0)
+		p = rtw_get_ie(ptxmgmt->buf, _RSN_IE_2_, &len, ptxmgmt->len);
+
+	if (p != NULL)
+		return rtw_set_ie(pframe, _RSN_IE_2_, len, p + 2, &(pattrib->pktlen));
+	else if (init == _TRUE)
+		return rtw_set_ie(pframe, _RSN_IE_2_, sizeof(TDLS_RSNIE), TDLS_RSNIE, &(pattrib->pktlen));
+	else
+		return rtw_set_ie(pframe, _RSN_IE_2_, sizeof(ptdls_sta->TDLS_RSNIE), ptdls_sta->TDLS_RSNIE, &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_ext_cap(u8 *pframe, struct pkt_attrib *pattrib)
+{
+	return rtw_set_ie(pframe, WLAN_EID_EXT_CAP , sizeof(TDLS_EXT_CAPIE), TDLS_EXT_CAPIE, &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_qos_cap(u8 *pframe, struct pkt_attrib *pattrib)
+{
+	return rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, sizeof(TDLS_WMMIE), TDLS_WMMIE,  &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_ftie(struct tdls_txmgmt *ptxmgmt, u8 *pframe, struct pkt_attrib *pattrib, u8 *ANonce, u8 *SNonce)
+{
+	struct wpa_tdls_ftie FTIE = {0};
+	u8 *p = NULL;
+	int len = 0;
+
+	if (ptxmgmt->len > 0)
+		p = rtw_get_ie(ptxmgmt->buf, _FTIE_, &len, ptxmgmt->len);
+
+	if (p != NULL)
+		return rtw_set_ie(pframe, _FTIE_, len, p + 2, &(pattrib->pktlen));
+	else {
+		if (ANonce != NULL)
+			_rtw_memcpy(FTIE.Anonce, ANonce, WPA_NONCE_LEN);
+		if (SNonce != NULL)
+			_rtw_memcpy(FTIE.Snonce, SNonce, WPA_NONCE_LEN);
+
+		return rtw_set_ie(pframe, _FTIE_, TDLS_FTIE_DATA_LEN,
+						  (u8 *)FTIE.data, &(pattrib->pktlen));
+	}
+}
+
+u8 *rtw_tdls_set_timeout_interval(struct tdls_txmgmt *ptxmgmt, u8 *pframe, struct pkt_attrib *pattrib, int init, struct sta_info *ptdls_sta)
+{
+	u8 timeout_itvl[5];	/* set timeout interval to maximum value */
+	u32 timeout_interval = TDLS_TPK_RESEND_COUNT;
+	u8 *p = NULL;
+	int len = 0;
+
+	if (ptxmgmt->len > 0)
+		p = rtw_get_ie(ptxmgmt->buf, _TIMEOUT_ITVL_IE_, &len, ptxmgmt->len);
+
+	if (p != NULL)
+		return rtw_set_ie(pframe, _TIMEOUT_ITVL_IE_, len, p + 2, &(pattrib->pktlen));
+	else {
+		/* Timeout interval */
+		timeout_itvl[0] = 0x02;
+		if (init == _TRUE)
+			_rtw_memcpy(timeout_itvl + 1, &timeout_interval, 4);
+		else
+			_rtw_memcpy(timeout_itvl + 1, (u8 *)(&ptdls_sta->TDLS_PeerKey_Lifetime), 4);
+
+		return rtw_set_ie(pframe, _TIMEOUT_ITVL_IE_, 5, timeout_itvl, &(pattrib->pktlen));
+	}
+}
+
+u8 *rtw_tdls_set_bss_coexist(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	u8 iedata = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (padapter_link->mlmepriv.num_FortyMHzIntolerant > 0)
+		iedata |= BIT(2);	/* 20 MHz BSS Width Request */
+
+	/* Information Bit should be set by TDLS test plan 5.9 */
+	iedata |= BIT(0);
+	return rtw_set_ie(pframe, WLAN_EID_20_40_BSS_COEXISTENCE, 1, &iedata, &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_payload_type(u8 *pframe, struct pkt_attrib *pattrib)
+{
+	u8 payload_type = 0x02;
+	return rtw_set_fixed_ie(pframe, 1, &(payload_type), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_category(u8 *pframe, struct pkt_attrib *pattrib, u8 category)
+{
+	return rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_action(u8 *pframe, struct pkt_attrib *pattrib, struct tdls_txmgmt *ptxmgmt)
+{
+	return rtw_set_fixed_ie(pframe, 1, &(ptxmgmt->action_code), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_status_code(u8 *pframe, struct pkt_attrib *pattrib, struct tdls_txmgmt *ptxmgmt)
+{
+	return rtw_set_fixed_ie(pframe, 2, (u8 *)&(ptxmgmt->status_code), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_dialog(u8 *pframe, struct pkt_attrib *pattrib, struct tdls_txmgmt *ptxmgmt)
+{
+	u8 dialogtoken = 1;
+	if (ptxmgmt->dialog_token)
+		return rtw_set_fixed_ie(pframe, 1, &(ptxmgmt->dialog_token), &(pattrib->pktlen));
+	else
+		return rtw_set_fixed_ie(pframe, 1, &(dialogtoken), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_reg_class(u8 *pframe, struct pkt_attrib *pattrib, struct sta_info *ptdls_sta)
+{
+	u8 reg_class = 22;
+	return rtw_set_fixed_ie(pframe, 1, &(reg_class), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_second_channel_offset(u8 *pframe, struct pkt_attrib *pattrib, u8 ch_offset)
+{
+	return rtw_set_ie(pframe, WLAN_EID_SECONDARY_CHANNEL_OFFSET, 1, &ch_offset, &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_capability(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	struct _ADAPTER_LINK	*padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
+	u8 cap_from_ie[2] = {0};
+
+	_rtw_memcpy(cap_from_ie, rtw_get_capability_from_ie(pmlmeinfo->network.IEs), 2);
+
+	return rtw_set_fixed_ie(pframe, 2, cap_from_ie, &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_supported_rate(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	u8 bssrate[NDIS_802_11_LENGTH_RATES_EX];
+	int bssrate_len = 0;
+	u8 more_supportedrates = 0;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+
+	rtw_set_supported_rate(bssrate,
+		(padapter->registrypriv.wireless_mode == WLAN_MD_MAX) ? padapter_link->mlmeextpriv.cur_wireless_mode : padapter->registrypriv.wireless_mode,
+			GET_WIFI_ROLE_LINK_CURRENT_CH(padapter_link),
+			GET_WIFI_ROLE_LINK_CURRENT_BAND(padapter_link));
+	bssrate_len = rtw_get_rateset_len(bssrate);
+
+	if (bssrate_len > 8) {
+		pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES , 8, bssrate, &(pattrib->pktlen));
+		more_supportedrates = 1;
+	} else
+		pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES , bssrate_len , bssrate, &(pattrib->pktlen));
+
+	/* extended supported rates */
+	if (more_supportedrates == 1)
+		pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES , (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
+
+	return pframe;
+}
+
+u8 *rtw_tdls_set_sup_reg_class(u8 *pframe, struct pkt_attrib *pattrib)
+{
+	return rtw_set_ie(pframe, _SRC_IE_ , sizeof(TDLS_SRC), TDLS_SRC, &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_linkid(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib, u8 init)
+{
+	struct _ADAPTER_LINK	*padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	u8 link_id_addr[18] = {0};
+
+	_rtw_memcpy(link_id_addr, get_my_bssid(&(pmlmeinfo->network)), 6);
+
+	if (init == _TRUE) {
+		_rtw_memcpy((link_id_addr + 6), pattrib->src, 6);
+		_rtw_memcpy((link_id_addr + 12), pattrib->dst, 6);
+	} else {
+		_rtw_memcpy((link_id_addr + 6), pattrib->dst, 6);
+		_rtw_memcpy((link_id_addr + 12), pattrib->src, 6);
+	}
+	return rtw_set_ie(pframe, _LINK_ID_IE_, 18, link_id_addr, &(pattrib->pktlen));
+}
+
+#ifdef CONFIG_TDLS_CH_SW
+u8 *rtw_tdls_set_target_ch(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	u8 target_ch = 1;
+	if (padapter->tdlsinfo.chsw_info.off_ch_num)
+		return rtw_set_fixed_ie(pframe, 1, &(padapter->tdlsinfo.chsw_info.off_ch_num), &(pattrib->pktlen));
+	else
+		return rtw_set_fixed_ie(pframe, 1, &(target_ch), &(pattrib->pktlen));
+}
+
+u8 *rtw_tdls_set_ch_sw(u8 *pframe, struct pkt_attrib *pattrib, struct sta_info *ptdls_sta)
+{
+	u8 ch_switch_timing[4] = {0};
+	u16 switch_time = (ptdls_sta->ch_switch_time >= TDLS_CH_SWITCH_TIME * 1000) ?
+			  ptdls_sta->ch_switch_time : TDLS_CH_SWITCH_TIME;
+	u16 switch_timeout = (ptdls_sta->ch_switch_timeout >= TDLS_CH_SWITCH_TIMEOUT * 1000) ?
+		     ptdls_sta->ch_switch_timeout : TDLS_CH_SWITCH_TIMEOUT;
+
+	_rtw_memcpy(ch_switch_timing, &switch_time, 2);
+	_rtw_memcpy(ch_switch_timing + 2, &switch_timeout, 2);
+
+	return rtw_set_ie(pframe, _CH_SWITCH_TIMING_,  4, ch_switch_timing, &(pattrib->pktlen));
+}
+
+void rtw_tdls_set_ch_sw_oper_control(_adapter *padapter, u8 enable)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct phl_info_t *phl_info = GET_PHL_INFO(dvobj);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	enum rtw_phl_status status;
+
+	if (enable == _TRUE)
+		rtw_phl_set_chsw_ofld_info(phl_com, _TRUE, _TRUE, _TRUE);
+	else
+		rtw_phl_set_chsw_ofld_info(phl_com, _FALSE, _FALSE, _FALSE);
+
+
+	if (ATOMIC_READ(&padapter->tdlsinfo.chsw_info.chsw_on) != enable)
+		ATOMIC_SET(&padapter->tdlsinfo.chsw_info.chsw_on, enable);
+
+	/* Enable bcn erly report */
+	RTW_INFO("[TDLS] %s Bcn Early C2H Report\n", (enable == _TRUE) ? "Start" : "Stop");
+	status = rtw_phl_cmd_wrole_change(phl_info, padapter->phl_role, padapter_link->wrlink,
+		WR_CHG_BCN_EARLY_RPT_CFG, (u8 *)&enable, sizeof(enable),
+		PHL_CMD_DIRECTLY, 0);
+}
+
+void rtw_tdls_ch_sw_back_to_base_chnl(_adapter *padapter)
+{
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if ((ATOMIC_READ(&pchsw_info->chsw_on) == _TRUE) &&
+	    (padapter_link->mlmeextpriv.chandef.chan != rtw_get_oper_ch(padapter, padapter_link)))
+		rtw_tdls_cmd(padapter, pchsw_info->addr, TDLS_CH_SW_TO_BASE_CHNL_UNSOLICITED, 0);
+}
+
+static void rtw_tdls_chsw_oper_init(_adapter *padapter, u32 timeout_ms)
+{
+	struct submit_ctx	*chsw_sctx = &padapter->tdlsinfo.chsw_info.chsw_sctx;
+
+	rtw_sctx_init(chsw_sctx, timeout_ms);
+}
+
+static int rtw_tdls_chsw_oper_wait(_adapter *padapter)
+{
+	struct submit_ctx	*chsw_sctx = &padapter->tdlsinfo.chsw_info.chsw_sctx;
+
+	return rtw_sctx_wait(chsw_sctx, __func__);
+}
+
+void rtw_tdls_chsw_oper_done(_adapter *padapter)
+{
+	struct submit_ctx	*chsw_sctx = &padapter->tdlsinfo.chsw_info.chsw_sctx;
+
+	rtw_sctx_done(&chsw_sctx);
+}
+
+s32 rtw_tdls_do_ch_sw(_adapter *padapter, struct sta_info *ptdls_sta, u8 chnl_type, u8 channel, u8 channel_offset, u16 bwmode, u16 ch_switch_time)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u32 ch_sw_time_start, ch_sw_time_spent, wait_time;
+	s32 ret = _SUCCESS;
+
+	ch_sw_time_start = rtw_systime_to_ms(rtw_get_current_time());
+
+	set_channel_bwmode(padapter, ptdls_sta->padapter_link, channel, channel_offset, bwmode, RFK_TYPE_FORCE_NOT_DO);
+	ch_sw_time_spent = rtw_systime_to_ms(rtw_get_current_time()) - ch_sw_time_start;
+	if (chnl_type == TDLS_CH_SW_OFF_CHNL) {
+		if ((u32)ch_switch_time / 1000 > ch_sw_time_spent)
+			wait_time = (u32)ch_switch_time / 1000 - ch_sw_time_spent;
+		else
+			wait_time = 0;
+		if (wait_time > 0)
+			rtw_msleep_os(wait_time);
+	}
+
+	return ret;
+}
+#endif
+
+u8 *rtw_tdls_set_wmm_params(_adapter *padapter, u8 *pframe, struct pkt_attrib *pattrib)
+{
+	struct _ADAPTER_LINK	*padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8 wmm_param_ele[24] = {0};
+
+	if (&pmlmeinfo->WMM_param) {
+		_rtw_memcpy(wmm_param_ele, WMM_PARA_OUI, 6);
+		if (_rtw_memcmp(&pmlmeinfo->WMM_param, &wmm_param_ele[6], 18) == _TRUE)
+			/* Use default WMM Param */
+			_rtw_memcpy(wmm_param_ele + 6, (u8 *)&TDLS_WMM_PARAM_IE, sizeof(TDLS_WMM_PARAM_IE));
+		else
+			_rtw_memcpy(wmm_param_ele + 6, (u8 *)&pmlmeinfo->WMM_param, sizeof(pmlmeinfo->WMM_param));
+		return rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_,  24, wmm_param_ele, &(pattrib->pktlen));
+	} else
+		return pframe;
+}
+
+#ifdef CONFIG_WFD
+void rtw_tdls_process_wfd_ie(struct tdls_info *ptdlsinfo, u8 *ptr, u8 length)
+{
+	u8 *wfd_ie;
+	u32 wfd_ielen = 0;
+	_adapter *adapter = tdls_info_to_adapter(ptdlsinfo);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	if (!rtw_hw_chk_wl_func(dvobj, WL_FUNC_MIRACAST))
+		return;
+
+	/* Try to get the TCP port information when receiving the negotiation response. */
+
+	wfd_ie = rtw_get_wfd_ie(ptr, length, NULL, &wfd_ielen);
+	while (wfd_ie) {
+		u8 *attr_content;
+		u32	attr_contentlen = 0;
+		int	i;
+
+		RTW_INFO("[%s] WFD IE Found!!\n", __FUNCTION__);
+		attr_content = rtw_get_wfd_attr_content(wfd_ie, wfd_ielen, WFD_ATTR_DEVICE_INFO, NULL, &attr_contentlen);
+		if (attr_content && attr_contentlen) {
+			ptdlsinfo->wfd_info->peer_rtsp_ctrlport = RTW_GET_BE16(attr_content + 2);
+			RTW_INFO("[%s] Peer PORT NUM = %d\n", __FUNCTION__, ptdlsinfo->wfd_info->peer_rtsp_ctrlport);
+		}
+
+		attr_content = rtw_get_wfd_attr_content(wfd_ie, wfd_ielen, WFD_ATTR_LOCAL_IP_ADDR, NULL, &attr_contentlen);
+		if (attr_content && attr_contentlen) {
+			_rtw_memcpy(ptdlsinfo->wfd_info->peer_ip_address, (attr_content + 1), 4);
+			RTW_INFO("[%s] Peer IP = %02u.%02u.%02u.%02u\n", __FUNCTION__,
+				ptdlsinfo->wfd_info->peer_ip_address[0], ptdlsinfo->wfd_info->peer_ip_address[1],
+				ptdlsinfo->wfd_info->peer_ip_address[2], ptdlsinfo->wfd_info->peer_ip_address[3]);
+		}
+
+		wfd_ie = rtw_get_wfd_ie(wfd_ie + wfd_ielen, (ptr + length) - (wfd_ie + wfd_ielen), NULL, &wfd_ielen);
+	}
+}
+
+int issue_tunneled_probe_req(_adapter *padapter)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	u8 baddr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	struct tdls_txmgmt txmgmt;
+	int ret = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("[%s]\n", __FUNCTION__);
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	txmgmt.action_code = TUNNELED_PROBE_REQ;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, baddr, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, get_bssid(pmlmepriv), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, &txmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+	dump_mgntframe(padapter, pmgntframe);
+	ret = _SUCCESS;
+exit:
+
+	return ret;
+}
+
+int issue_tunneled_probe_rsp(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct tdls_txmgmt txmgmt;
+	int ret = _FAIL;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+
+	RTW_INFO("[%s]\n", __FUNCTION__);
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	txmgmt.action_code = TUNNELED_PROBE_RSP;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, precv_frame->u.hdr.attrib.src, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, get_bssid(pmlmepriv), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, &txmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+	dump_mgntframe(padapter, pmgntframe);
+	ret = _SUCCESS;
+exit:
+
+	return ret;
+}
+#endif /* CONFIG_WFD */
+
+int issue_tdls_setup_req(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, int wait_ack)
+{
+	struct tdls_info	*ptdlsinfo = &padapter->tdlsinfo;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *ptdls_sta = NULL;
+	int ret = _FAIL;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	enum role_type rtype = PHL_RTYPE_TDLS;
+	enum rtw_phl_status status;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	u16 main_id = rtw_phl_get_macid_max_num(phl);
+	struct rtw_phl_mld_t *pmld = NULL;
+
+	/* Retry timer should be set at least 301 sec, using TPK_count counting 301 times. */
+	u32 timeout_interval = TDLS_TPK_RESEND_COUNT;
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	if (rtw_tdls_is_setup_allowed(padapter) == _FALSE)
+		goto exit;
+
+	if (IS_MCAST(ptxmgmt->peer))
+		goto exit;
+
+	ptdls_sta = rtw_get_stainfo(pstapriv, ptxmgmt->peer);
+	if (ptdlsinfo->sta_maximum == _TRUE) {
+		if (ptdls_sta == NULL)
+			goto exit;
+		else if (!(ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE))
+			goto exit;
+	}
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	if (wrole->type != PHL_RTYPE_TDLS) {
+		status = rtw_phl_cmd_wrole_change(phl, wrole, NULL,
+			WR_CHG_TYPE, (u8*)&rtype, sizeof(enum role_type), PHL_CMD_DIRECTLY, 0);
+		if (status != RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s - change to phl role type = %d fail with error = %d\n", __func__, rtype, status);
+			rtw_warn_on(1);
+		}
+	}
+
+	if (ptdls_sta == NULL) {
+		pmld = rtw_phl_alloc_mld(phl, padapter->phl_role, ptxmgmt->peer, DTYPE);
+		if (pmld == NULL) {
+				RTW_INFO("[%s] rtw_phl_alloc_mld fail\n", __FUNCTION__);
+				goto exit;
+		}
+		ptdls_sta = rtw_alloc_stainfo_sw(pstapriv, DTYPE, main_id, padapter_link->wrlink->id, ptxmgmt->peer);
+		if (ptdls_sta == NULL) {
+			RTW_INFO("[%s] rtw_alloc_stainfo fail\n", __FUNCTION__);
+			#if 0 /*CONFIG_CORE_XMITBUF*/
+			rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+			#endif
+			rtw_free_xmitframe(pxmitpriv, pmgntframe);
+			goto exit;
+		}
+		rtw_phl_link_mld_stainfo(pmld, ptdls_sta->phl_sta);
+		ptdlsinfo->sta_cnt++;
+	}
+
+	ptxmgmt->action_code = TDLS_SETUP_REQUEST;
+
+	pattrib = &pmgntframe->attrib;
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, get_bssid(pmlmepriv), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, padapter_link, pattrib);
+
+	if (ptdlsinfo->sta_cnt == MAX_ALLOWED_TDLS_STA_NUM)
+		ptdlsinfo->sta_maximum  = _TRUE;
+
+	ptdls_sta->tdls_sta_state |= TDLS_RESPONDER_STATE;
+
+	if (rtw_tdls_is_driver_setup(padapter) == _TRUE) {
+		ptdls_sta->TDLS_PeerKey_Lifetime = timeout_interval;
+		_set_timer(&ptdls_sta->handshake_timer, TDLS_HANDSHAKE_TIME);
+	}
+
+	/*pattrib->qsel = pattrib->priority;*/
+
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, ptxmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+
+	return ret;
+}
+
+int _issue_tdls_teardown(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta, u8 wait_ack)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	int ret = _FAIL;
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	ptxmgmt->action_code = TDLS_TEARDOWN;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	rtw_mi_set_scan_deny(padapter, 550);
+	rtw_mi_scan_abort(padapter, _TRUE);
+
+	pattrib = &pmgntframe->attrib;
+
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+
+	if (ptxmgmt->status_code == _RSON_TDLS_TEAR_UN_RSN_)
+		_rtw_memcpy(pattrib->ra, ptxmgmt->peer, ETH_ALEN);
+	else
+		_rtw_memcpy(pattrib->ra, get_bssid(pmlmepriv), ETH_ALEN);
+
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, ptdls_sta->padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, ptxmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	if (rtw_tdls_is_driver_setup(padapter) == _TRUE)
+		if (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE)
+			if (pattrib->encrypt)
+				_cancel_timer_ex(&ptdls_sta->TPK_timer);
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+
+	return ret;
+}
+
+int issue_tdls_teardown(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, u8 wait_ack)
+{
+	struct sta_info *ptdls_sta = NULL;
+	int ret = _FAIL;
+
+	ptdls_sta = rtw_get_stainfo(&(padapter->stapriv), ptxmgmt->peer);
+	if (ptdls_sta == NULL) {
+		RTW_INFO("No tdls_sta for tearing down\n");
+		goto exit;
+	}
+
+	ret = _issue_tdls_teardown(padapter, ptxmgmt, ptdls_sta, wait_ack);
+	if ((ptxmgmt->status_code == _RSON_TDLS_TEAR_UN_RSN_) && (ret == _FAIL)) {
+		/* Change status code and send teardown again via AP */
+		ptxmgmt->status_code = _RSON_TDLS_TEAR_TOOFAR_;
+		ret = _issue_tdls_teardown(padapter, ptxmgmt, ptdls_sta, wait_ack);
+	}
+
+	if (rtw_tdls_is_driver_setup(padapter)) {
+		rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+		rtw_tdls_cmd(padapter, ptxmgmt->peer, TDLS_TEARDOWN_STA_LOCALLY_POST, 0);
+	}
+
+exit:
+	return ret;
+}
+
+int issue_tdls_dis_req(_adapter *padapter, struct tdls_txmgmt *ptxmgmt)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	int ret = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	ptxmgmt->action_code = TDLS_DISCOVERY_REQUEST;
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, get_bssid(pmlmepriv), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, ptxmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+	dump_mgntframe(padapter, pmgntframe);
+	RTW_INFO("issue tdls dis req\n");
+
+	ret = _SUCCESS;
+exit:
+
+	return ret;
+}
+
+int issue_tdls_setup_rsp(_adapter *padapter, struct tdls_txmgmt *ptxmgmt)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	int ret = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	ptxmgmt->action_code = TDLS_SETUP_RESPONSE;
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, get_bssid(&(padapter->mlmepriv)), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, ptxmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	ret = _SUCCESS;
+exit:
+
+	return ret;
+
+}
+
+int issue_tdls_setup_cfm(_adapter *padapter, struct tdls_txmgmt *ptxmgmt)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	int ret = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	ptxmgmt->action_code = TDLS_SETUP_CONFIRM;
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, get_bssid(&padapter->mlmepriv), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, ptxmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	ret = _SUCCESS;
+exit:
+
+	return ret;
+
+}
+
+/* TDLS Discovery Response frame is a management action frame */
+int issue_tdls_dis_rsp(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, u8 privacy)
+{
+	struct xmit_frame		*pmgntframe;
+	struct pkt_attrib		*pattrib;
+	unsigned char			*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short		*fctrl;
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	int ret = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	/* unicast probe request frame */
+	_rtw_memcpy(pwlanhdr->addr1, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->dst, pwlanhdr->addr1, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->src, pwlanhdr->addr2, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_bssid(&padapter->mlmepriv), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, pwlanhdr->addr3, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	rtw_build_tdls_dis_rsp_ies(padapter, pmgntframe, pframe, ptxmgmt, privacy);
+
+	pattrib->nr_frags = 1;
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+int issue_tdls_peer_traffic_rsp(_adapter *padapter, struct sta_info *ptdls_sta, struct tdls_txmgmt *ptxmgmt)
+{
+	struct xmit_frame	*pmgntframe;
+	struct pkt_attrib	*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	int ret = _FAIL;
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	ptxmgmt->action_code = TDLS_PEER_TRAFFIC_RESPONSE;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, ptdls_sta->padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, ptxmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	dump_mgntframe(padapter, pmgntframe);
+	ret = _SUCCESS;
+
+exit:
+
+	return ret;
+}
+
+int issue_tdls_peer_traffic_indication(_adapter *padapter, struct sta_info *ptdls_sta)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv			*pxmitpriv = &(padapter->xmitpriv);
+	struct tdls_txmgmt txmgmt;
+	int ret = _FAIL;
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	txmgmt.action_code = TDLS_PEER_TRAFFIC_INDICATION;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, get_bssid(pmlmepriv), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	/* PTI frame's priority should be AC_VO */
+	pattrib->priority = 7;
+
+	update_tdls_attrib(padapter, ptdls_sta->padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, &txmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	dump_mgntframe(padapter, pmgntframe);
+	ret = _SUCCESS;
+
+exit:
+
+	return ret;
+}
+
+#ifdef CONFIG_TDLS_CH_SW
+int issue_tdls_ch_switch_req(_adapter *padapter, struct sta_info *ptdls_sta)
+{
+	struct xmit_frame	*pmgntframe;
+	struct pkt_attrib	*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	struct tdls_txmgmt txmgmt;
+	int ret = _FAIL;
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	if (rtw_tdls_is_chsw_allowed(padapter) == _FALSE) {
+		RTW_INFO("[TDLS] Ignore %s since channel switch is not allowed\n", __func__);
+		goto exit;
+	}
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	txmgmt.action_code = TDLS_CHANNEL_SWITCH_REQUEST;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, ptdls_sta->padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, &txmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	dump_mgntframe(padapter, pmgntframe);
+	ret = _SUCCESS;
+exit:
+
+	return ret;
+}
+
+int issue_tdls_ch_switch_rsp(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, int wait_ack)
+{
+	struct xmit_frame	*pmgntframe;
+	struct pkt_attrib	*pattrib;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	int ret = _FAIL;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only]*/
+	struct _ADAPTER_LINK	*padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("[TDLS] %s\n", __FUNCTION__);
+
+	if (rtw_tdls_is_chsw_allowed(padapter) == _FALSE) {
+		RTW_INFO("[TDLS] Ignore %s since channel switch is not allowed\n", __func__);
+		goto exit;
+	}
+
+	ptxmgmt->action_code = TDLS_CHANNEL_SWITCH_RESPONSE;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL)
+		goto exit;
+
+	pattrib = &pmgntframe->attrib;
+
+	pmgntframe->frame_tag = DATA_FRAMETAG;
+	pattrib->ether_type = 0x890d;
+
+	_rtw_memcpy(pattrib->dst, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, ptxmgmt->peer, ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+
+	update_tdls_attrib(padapter, padapter_link, pattrib);
+	/*pattrib->qsel = pattrib->priority;*/
+	/*
+		_rtw_spinlock_bh(&pxmitpriv->lock);
+		if(xmitframe_enqueue_for_tdls_sleeping_sta(padapter, pmgntframe)==_TRUE){
+			_rtw_spinunlock_bh(&pxmitpriv->lock);
+			return _FALSE;
+		}
+	*/
+	if (rtw_xmit_tdls_coalesce(padapter, pmgntframe, ptxmgmt) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pmgntframe);
+		goto exit;
+	}
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack_timeout(padapter, pmgntframe, 10);
+	else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+exit:
+
+	return ret;
+}
+#endif
+
+int On_TDLS_Dis_Rsp(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct sta_info *ptdls_sta = NULL, *psta = rtw_get_stainfo(&(padapter->stapriv), get_bssid(&(padapter->mlmepriv)));
+	u8 *ptr = precv_frame->u.hdr.rx_data, *psa;
+	struct rx_pkt_attrib *pattrib = &(precv_frame->u.hdr.attrib);
+	struct tdls_info *ptdlsinfo = &(padapter->tdlsinfo);
+	u8 empty_addr[ETH_ALEN] = { 0x00 };
+	int rssi = 0;
+	struct tdls_txmgmt txmgmt;
+	int ret = _SUCCESS;
+
+	/* ToDo: need API to query hal_sta->rssi_stat.rssi */
+	/*
+	if (psta)
+		rssi = psta->phl_sta->hal_sta->rssi_stat.rssi;
+	*/
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	/* WFDTDLS: for sigma test, not to setup direct link automatically */
+	ptdlsinfo->dev_discovered = _TRUE;
+
+	psa = get_sa(ptr);
+	ptdls_sta = rtw_get_stainfo(&(padapter->stapriv), psa);
+	if (ptdls_sta != NULL)
+		ptdls_sta->sta_stats.rx_tdls_disc_rsp_pkts++;
+
+#ifdef CONFIG_TDLS_AUTOSETUP
+	if (ptdls_sta != NULL) {
+		/* Record the tdls sta with lowest signal strength */
+		if (ptdlsinfo->sta_maximum == _TRUE && ptdls_sta->alive_count >= 1) {
+			if (_rtw_memcmp(ptdlsinfo->ss_record.macaddr, empty_addr, ETH_ALEN)) {
+				_rtw_memcpy(ptdlsinfo->ss_record.macaddr, psa, ETH_ALEN);
+				ptdlsinfo->ss_record.RxPWDBAll = pattrib->phy_info.rx_pwdb_all;
+			} else {
+				if (ptdlsinfo->ss_record.RxPWDBAll < pattrib->phy_info.rx_pwdb_all) {
+					_rtw_memcpy(ptdlsinfo->ss_record.macaddr, psa, ETH_ALEN);
+					ptdlsinfo->ss_record.RxPWDBAll = pattrib->phy_info.rx_pwdb_all;
+				}
+			}
+		}
+	} else {
+		if (ptdlsinfo->sta_maximum == _TRUE) {
+			if (_rtw_memcmp(ptdlsinfo->ss_record.macaddr, empty_addr, ETH_ALEN)) {
+				/* All traffics are busy, do not set up another direct link. */
+				ret = _FAIL;
+				goto exit;
+			} else {
+				if (pattrib->phy_info.rx_pwdb_all > ptdlsinfo->ss_record.RxPWDBAll) {
+					_rtw_memcpy(txmgmt.peer, ptdlsinfo->ss_record.macaddr, ETH_ALEN);
+					/* issue_tdls_teardown(padapter, ptdlsinfo->ss_record.macaddr, _FALSE); */
+				} else {
+					ret = _FAIL;
+					goto exit;
+				}
+			}
+		}
+
+
+		if (pattrib->phy_info.rx_pwdb_all + TDLS_SIGNAL_THRESH >= rssi) {
+			RTW_INFO("pattrib->RxPWDBAll=%d, pdmpriv->undecorated_smoothed_pwdb=%d\n", pattrib->phy_info.rx_pwdb_all, rssi);
+			_rtw_memcpy(txmgmt.peer, psa, ETH_ALEN);
+			issue_tdls_setup_req(padapter, &txmgmt, _FALSE);
+		}
+	}
+exit:
+#endif /* CONFIG_TDLS_AUTOSETUP */
+
+	return ret;
+
+}
+
+sint On_TDLS_Setup_Req(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	u8 *psa, *pmyid;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct rx_pkt_attrib	*prx_pkt_attrib = &precv_frame->u.hdr.attrib;
+	u8 *prsnie, *ppairwise_cipher;
+	u8 i, k;
+	u8 ccmp_included = 0, rsnie_included = 0;
+	u16 j, pairwise_count;
+	u8 SNonce[32];
+	u32 timeout_interval = TDLS_TPK_RESEND_COUNT;
+	sint parsing_length;	/* Frame body length, without icv_len */
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 FIXED_IE = 5;
+	unsigned char		supportRate[16];
+	int				supportRateNum = 0;
+	struct tdls_txmgmt txmgmt;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	enum role_type rtype = PHL_RTYPE_TDLS;
+	enum rtw_phl_status status;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	u16 main_id = rtw_phl_get_macid_max_num(phl);
+	struct rtw_phl_mld_t *pmld = NULL;
+
+	if (rtw_tdls_is_setup_allowed(padapter) == _FALSE)
+		goto exit;
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	psa = get_sa(ptr);
+
+	if (ptdlsinfo->sta_maximum == _TRUE) {
+		if (ptdls_sta == NULL)
+			goto exit;
+		else if (!(ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE))
+			goto exit;
+	}
+
+	pmyid = adapter_mac_addr(padapter);
+	ptr += prx_pkt_attrib->hdrlen + prx_pkt_attrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	parsing_length = ((union recv_frame *)precv_frame)->u.hdr.len
+			 - prx_pkt_attrib->hdrlen
+			 - prx_pkt_attrib->iv_len
+			 - prx_pkt_attrib->icv_len
+			 - LLC_HEADER_SIZE
+			 - ETH_TYPE_LEN
+			 - PAYLOAD_TYPE_LEN;
+
+	if (wrole->type != PHL_RTYPE_TDLS) {
+		status = rtw_phl_cmd_wrole_change(phl, wrole, NULL,
+			WR_CHG_TYPE, (u8*)&rtype, sizeof(enum role_type), PHL_CMD_DIRECTLY, 0);
+		if (status != RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s - change to phl role type = %d fail with error = %d\n", __func__, rtype, status);
+			rtw_warn_on(1);
+		}
+        }
+
+	if (ptdls_sta == NULL) {
+		pmld = rtw_phl_alloc_mld(phl, padapter->phl_role, psa, DTYPE);
+		if (pmld == NULL) {
+				RTW_INFO("[%s] rtw_phl_alloc_mld fail\n", __FUNCTION__);
+				goto exit;
+		}
+		ptdls_sta = rtw_alloc_stainfo_sw(pstapriv, DTYPE, main_id, padapter_link->wrlink->id, psa);
+		if (ptdls_sta == NULL) {
+			RTW_INFO("[%s] rtw_alloc_stainfo fail\n", __FUNCTION__);
+			goto exit;
+		}
+		rtw_phl_link_mld_stainfo(pmld, ptdls_sta->phl_sta);
+		ptdlsinfo->sta_cnt++;
+	}
+	else {
+		if (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE) {
+			/* If the direct link is already set up */
+			/* Process as re-setup after tear down */
+			RTW_INFO("re-setup a direct link\n");
+			if (rtw_tdls_is_driver_setup(padapter) == _FALSE)
+				ptdls_sta->tdls_sta_state |= TDLS_RESETUP_STATE;
+		}
+		/* Already receiving TDLS setup request */
+		else if (ptdls_sta->tdls_sta_state & TDLS_INITIATOR_STATE) {
+			RTW_INFO("receive duplicated TDLS setup request frame in handshaking\n");
+			goto exit;
+		}
+		/* When receiving and sending setup_req to the same link at the same time */
+		/* STA with higher MAC_addr would be initiator */
+		else if (ptdls_sta->tdls_sta_state & TDLS_RESPONDER_STATE) {
+			RTW_INFO("receive setup_req after sending setup_req\n");
+			for (i = 0; i < 6; i++) {
+				if (*(pmyid + i) == *(psa + i)) {
+				} else if (*(pmyid + i) > *(psa + i)) {
+					ptdls_sta->tdls_sta_state = TDLS_INITIATOR_STATE;
+					break;
+				} else if (*(pmyid + i) < *(psa + i))
+					goto exit;
+			}
+		}
+	}
+
+	if (ptdls_sta) {
+		txmgmt.dialog_token = *(ptr + 2);	/* Copy dialog token */
+		txmgmt.status_code = _STATS_SUCCESSFUL_;
+
+		/* Parsing information element */
+		for (j = FIXED_IE; j < parsing_length;) {
+
+			pIE = (PNDIS_802_11_VARIABLE_IEs)(ptr + j);
+
+			switch (pIE->ElementID) {
+			case WLAN_EID_SUPP_RATES:
+				if (pIE->Length <= sizeof(supportRate)) {
+					_rtw_memcpy(supportRate, pIE->data, pIE->Length);
+					supportRateNum = pIE->Length;
+				}
+				break;
+			case WLAN_EID_COUNTRY:
+				break;
+			case WLAN_EID_EXT_SUPP_RATES:
+				if ((supportRateNum + pIE->Length) <= sizeof(supportRate)) {
+					_rtw_memcpy(supportRate + supportRateNum, pIE->data, pIE->Length);
+					supportRateNum += pIE->Length;
+				}
+				break;
+			case WLAN_EID_SUPPORTED_CHANNELS:
+				break;
+			case WLAN_EID_RSN:
+				rsnie_included = 1;
+				if (prx_pkt_attrib->encrypt) {
+					prsnie = (u8 *)pIE;
+					/* Check CCMP pairwise_cipher presence. */
+					ppairwise_cipher = prsnie + 10;
+					_rtw_memcpy(ptdls_sta->TDLS_RSNIE, pIE->data,
+						(pIE->Length <= sizeof(ptdls_sta->TDLS_RSNIE) ? pIE->Length : sizeof(ptdls_sta->TDLS_RSNIE)));
+					pairwise_count = *(u16 *)(ppairwise_cipher - 2);
+					for (k = 0; k < pairwise_count; k++) {
+						if (_rtw_memcmp(ppairwise_cipher + 4 * k, RSN_CIPHER_SUITE_CCMP, 4) == _TRUE)
+							ccmp_included = 1;
+					}
+
+					if (ccmp_included == 0)
+						txmgmt.status_code = _STATS_INVALID_RSNIE_;
+				}
+				break;
+			case WLAN_EID_EXT_CAP:
+				break;
+			case WLAN_EID_VENDOR_SPECIFIC:
+				break;
+			case WLAN_EID_FAST_BSS_TRANSITION:
+				if (prx_pkt_attrib->encrypt)
+					_rtw_memcpy(SNonce, (ptr + j + 52), 32);
+				break;
+			case WLAN_EID_TIMEOUT_INTERVAL:
+				if (prx_pkt_attrib->encrypt)
+					timeout_interval = cpu_to_le32(*(u32 *)(ptr + j + 3));
+				break;
+			case _RIC_Descriptor_IE_:
+				break;
+#ifdef CONFIG_80211N_HT
+			case WLAN_EID_HT_CAP:
+				rtw_tdls_process_ht_cap(padapter, ptdls_sta, pIE);
+				break;
+#endif
+#ifdef CONFIG_80211AC_VHT
+			case EID_AID:
+				break;
+			case WLAN_EID_VHT_CAPABILITY:
+				rtw_tdls_process_vht_cap(padapter, ptdls_sta, pIE);
+				break;
+#endif
+#ifdef CONFIG_80211AX_HE
+			case WLAN_EID_EXTENSION:
+				if (pIE->data[0] == WLAN_EID_EXTENSION_HE_CAPABILITY)
+					rtw_tdls_process_he_cap(padapter, ptdls_sta, pIE);
+				break;
+#endif
+			case WLAN_EID_20_40_BSS_COEXISTENCE:
+				break;
+			case _LINK_ID_IE_:
+				if (_rtw_memcmp(get_bssid(pmlmepriv), pIE->data, 6) == _FALSE)
+					txmgmt.status_code = _STATS_NOT_IN_SAME_BSS_;
+				break;
+			default:
+				break;
+			}
+
+			j += (pIE->Length + 2);
+
+		}
+
+		/* Check status code */
+		/* If responder STA has/hasn't security on AP, but request hasn't/has RSNIE, it should reject */
+		if (txmgmt.status_code == _STATS_SUCCESSFUL_) {
+			if (rsnie_included && prx_pkt_attrib->encrypt == 0)
+				txmgmt.status_code = _STATS_SEC_DISABLED_;
+			else if (rsnie_included == 0 && prx_pkt_attrib->encrypt)
+				txmgmt.status_code = _STATS_INVALID_PARAMETERS_;
+
+#ifdef CONFIG_WFD
+			/* WFD test plan version 0.18.2 test item 5.1.5 */
+			/* SoUT does not use TDLS if AP uses weak security */
+			if (padapter->wdinfo.wfd_tdls_enable && (rsnie_included && prx_pkt_attrib->encrypt != _AES_))
+				txmgmt.status_code = _STATS_SEC_DISABLED_;
+#endif /* CONFIG_WFD */
+		}
+
+		ptdls_sta->tdls_sta_state |= TDLS_INITIATOR_STATE;
+		if (prx_pkt_attrib->encrypt) {
+			_rtw_memcpy(ptdls_sta->SNonce, SNonce, 32);
+
+			if (timeout_interval <= 300)
+				ptdls_sta->TDLS_PeerKey_Lifetime = TDLS_TPK_RESEND_COUNT;
+			else
+				ptdls_sta->TDLS_PeerKey_Lifetime = timeout_interval;
+		}
+
+		/* Update station supportRate */
+		ptdls_sta->bssratelen = supportRateNum;
+		_rtw_memcpy(ptdls_sta->bssrateset, supportRate, supportRateNum);
+
+		/* -2: AP + BC/MC sta, -4: default key */
+		if (ptdlsinfo->sta_cnt == MAX_ALLOWED_TDLS_STA_NUM)
+			ptdlsinfo->sta_maximum = _TRUE;
+
+#ifdef CONFIG_WFD
+		rtw_tdls_process_wfd_ie(ptdlsinfo, ptr + FIXED_IE, parsing_length);
+#endif
+
+	} else
+		goto exit;
+
+	_rtw_memcpy(txmgmt.peer, prx_pkt_attrib->src, ETH_ALEN);
+
+	if (rtw_tdls_is_driver_setup(padapter)) {
+		issue_tdls_setup_rsp(padapter, &txmgmt);
+
+		if (txmgmt.status_code == _STATS_SUCCESSFUL_)
+			_set_timer(&ptdls_sta->handshake_timer, TDLS_HANDSHAKE_TIME);
+		else {
+			rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_LOCALLY_POST, 0);
+		}
+	}
+
+exit:
+
+	return _SUCCESS;
+}
+
+int On_TDLS_Setup_Rsp(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib	*prx_pkt_attrib = &precv_frame->u.hdr.attrib;
+	u8 *psa;
+	u16 status_code = 0;
+	sint parsing_length;	/* Frame body length, without icv_len */
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 FIXED_IE = 7;
+	u8 ANonce[32];
+	u8  *pftie = NULL, *ptimeout_ie = NULL, *plinkid_ie = NULL, *prsnie = NULL, *pftie_mic = NULL, *ppairwise_cipher = NULL;
+	u16 pairwise_count, j, k;
+	u8 verify_ccmp = 0;
+	unsigned char		supportRate[16];
+	int				supportRateNum = 0;
+	struct tdls_txmgmt txmgmt;
+	int ret = _SUCCESS;
+	u32 timeout_interval = TDLS_TPK_RESEND_COUNT;
+	struct _ADAPTER_LINK *padapter_link = ptdls_sta->padapter_link;
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	psa = get_sa(ptr);
+
+	ptr += prx_pkt_attrib->hdrlen + prx_pkt_attrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	parsing_length = ((union recv_frame *)precv_frame)->u.hdr.len
+			 - prx_pkt_attrib->hdrlen
+			 - prx_pkt_attrib->iv_len
+			 - prx_pkt_attrib->icv_len
+			 - LLC_HEADER_SIZE
+			 - ETH_TYPE_LEN
+			 - PAYLOAD_TYPE_LEN;
+
+	_rtw_memcpy(&status_code, ptr + 2, 2);
+
+	if (status_code != 0) {
+		RTW_INFO("[TDLS] %s status_code = %d, free_tdls_sta\n", __FUNCTION__, status_code);
+		rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_LOCALLY_POST, 0);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	status_code = 0;
+
+	/* parsing information element */
+	for (j = FIXED_IE; j < parsing_length;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(ptr + j);
+
+		switch (pIE->ElementID) {
+		case WLAN_EID_SUPP_RATES:
+			if (pIE->Length <= sizeof(supportRate)) {
+				_rtw_memcpy(supportRate, pIE->data, pIE->Length);
+				supportRateNum = pIE->Length;
+			}
+			break;
+		case WLAN_EID_COUNTRY:
+			break;
+		case WLAN_EID_EXT_SUPP_RATES:
+			if ((supportRateNum + pIE->Length) <= sizeof(supportRate)) {
+				_rtw_memcpy(supportRate + supportRateNum, pIE->data, pIE->Length);
+				supportRateNum += pIE->Length;
+			}
+			break;
+		case WLAN_EID_SUPPORTED_CHANNELS:
+			break;
+		case WLAN_EID_RSN:
+			prsnie = (u8 *)pIE;
+			/* Check CCMP pairwise_cipher presence. */
+			ppairwise_cipher = prsnie + 10;
+			_rtw_memcpy(&pairwise_count, (u16 *)(ppairwise_cipher - 2), 2);
+			for (k = 0; k < pairwise_count; k++) {
+				if (_rtw_memcmp(ppairwise_cipher + 4 * k, RSN_CIPHER_SUITE_CCMP, 4) == _TRUE)
+					verify_ccmp = 1;
+			}
+		case WLAN_EID_EXT_CAP:
+			break;
+		case WLAN_EID_VENDOR_SPECIFIC:
+			if (_rtw_memcmp((u8 *)pIE + 2, WMM_INFO_OUI, 6) == _TRUE) {
+				/* WMM Info ID and OUI */
+				if ((pregistrypriv->wmm_enable == _TRUE) || (padapter_link->mlmepriv.htpriv.ht_option == _TRUE))
+					ptdls_sta->qos_option = _TRUE;
+			}
+			break;
+		case WLAN_EID_FAST_BSS_TRANSITION:
+			pftie = (u8 *)pIE;
+			_rtw_memcpy(ANonce, (ptr + j + 20), 32);
+			break;
+		case WLAN_EID_TIMEOUT_INTERVAL:
+			ptimeout_ie = (u8 *)pIE;
+			timeout_interval = cpu_to_le32(*(u32 *)(ptimeout_ie + 3));
+			break;
+		case _RIC_Descriptor_IE_:
+			break;
+#ifdef CONFIG_80211N_HT
+		case WLAN_EID_HT_CAP:
+			rtw_tdls_process_ht_cap(padapter, ptdls_sta, pIE);
+			break;
+#endif
+#ifdef CONFIG_80211AC_VHT
+		case EID_AID:
+			/* todo in the future if necessary */
+			break;
+		case WLAN_EID_VHT_CAPABILITY:
+			rtw_tdls_process_vht_cap(padapter, ptdls_sta, pIE);
+			break;
+		case WLAN_EID_VHT_OP_MODE_NOTIFY:
+			rtw_tdls_process_vht_op_mode_notify(padapter, ptdls_sta, pIE);
+			break;
+#endif
+#ifdef CONFIG_80211AX_HE
+		case WLAN_EID_EXTENSION:
+			if (pIE->data[0] == WLAN_EID_EXTENSION_HE_CAPABILITY)
+				rtw_tdls_process_he_cap(padapter, ptdls_sta, pIE);
+			break;
+#endif
+		case WLAN_EID_20_40_BSS_COEXISTENCE:
+			break;
+		case _LINK_ID_IE_:
+			plinkid_ie = (u8 *)pIE;
+			break;
+		default:
+			break;
+		}
+
+		j += (pIE->Length + 2);
+
+	}
+
+	ptdls_sta->bssratelen = supportRateNum;
+	_rtw_memcpy(ptdls_sta->bssrateset, supportRate, supportRateNum);
+	_rtw_memcpy(ptdls_sta->ANonce, ANonce, 32);
+
+#ifdef CONFIG_WFD
+	rtw_tdls_process_wfd_ie(ptdlsinfo, ptr + FIXED_IE, parsing_length);
+#endif
+
+	if (prx_pkt_attrib->encrypt) {
+		if (verify_ccmp == 1) {
+			txmgmt.status_code = _STATS_SUCCESSFUL_;
+			if (rtw_tdls_is_driver_setup(padapter) == _TRUE) {
+				wpa_tdls_generate_tpk(padapter, ptdls_sta);
+				if (tdls_verify_mic(ptdls_sta->tpk.kck, 2, plinkid_ie, prsnie, ptimeout_ie, pftie) == _FAIL) {
+					RTW_INFO("[TDLS] %s tdls_verify_mic fail, free_tdls_sta\n", __FUNCTION__);
+					rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+					rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_LOCALLY_POST, 0);
+					ret = _FAIL;
+					goto exit;
+				}
+				ptdls_sta->TDLS_PeerKey_Lifetime = timeout_interval;
+			}
+		} else
+			txmgmt.status_code = _STATS_INVALID_RSNIE_;
+	} else
+		txmgmt.status_code = _STATS_SUCCESSFUL_;
+
+	if (rtw_tdls_is_driver_setup(padapter) == _TRUE) {
+		_rtw_memcpy(txmgmt.peer, prx_pkt_attrib->src, ETH_ALEN);
+		issue_tdls_setup_cfm(padapter, &txmgmt);
+
+		if (txmgmt.status_code == _STATS_SUCCESSFUL_) {
+			rtw_tdls_set_link_established(padapter, _TRUE);
+
+			if (ptdls_sta->tdls_sta_state & TDLS_RESPONDER_STATE) {
+				ptdls_sta->tdls_sta_state |= TDLS_LINKED_STATE;
+				ptdls_sta->state |= WIFI_ASOC_STATE;
+				_cancel_timer_ex(&ptdls_sta->handshake_timer);
+			}
+
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_ESTABLISHED, 0);
+
+		}
+	}
+
+exit:
+	if (rtw_tdls_is_driver_setup(padapter) == _TRUE)
+		return ret;
+	else
+		return _SUCCESS;
+
+}
+
+int On_TDLS_Setup_Cfm(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib	*prx_pkt_attrib = &precv_frame->u.hdr.attrib;
+	u8 *psa;
+	u16 status_code = 0;
+	sint parsing_length;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 FIXED_IE = 5;
+	u8  *pftie = NULL, *ptimeout_ie = NULL, *plinkid_ie = NULL, *prsnie = NULL, *pftie_mic = NULL, *ppairwise_cipher = NULL;
+	u16 j, pairwise_count;
+	int ret = _SUCCESS;
+
+	psa = get_sa(ptr);
+
+	ptr += prx_pkt_attrib->hdrlen + prx_pkt_attrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	parsing_length = ((union recv_frame *)precv_frame)->u.hdr.len
+			 - prx_pkt_attrib->hdrlen
+			 - prx_pkt_attrib->iv_len
+			 - prx_pkt_attrib->icv_len
+			 - LLC_HEADER_SIZE
+			 - ETH_TYPE_LEN
+			 - PAYLOAD_TYPE_LEN;
+
+	_rtw_memcpy(&status_code, ptr + 2, 2);
+
+	if (status_code != 0) {
+		RTW_INFO("[%s] status_code = %d\n, free_tdls_sta", __FUNCTION__, status_code);
+		rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_LOCALLY_POST, 0);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	/* Parsing information element */
+	for (j = FIXED_IE; j < parsing_length;) {
+
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(ptr + j);
+
+		switch (pIE->ElementID) {
+		case WLAN_EID_RSN:
+			prsnie = (u8 *)pIE;
+			break;
+		case WLAN_EID_VENDOR_SPECIFIC:
+			if (_rtw_memcmp((u8 *)pIE + 2, WMM_PARA_OUI, 6) == _TRUE) {
+				/* WMM Parameter ID and OUI */
+				ptdls_sta->qos_option = _TRUE;
+			}
+			break;
+		case WLAN_EID_FAST_BSS_TRANSITION:
+			pftie = (u8 *)pIE;
+			break;
+		case WLAN_EID_TIMEOUT_INTERVAL:
+			ptimeout_ie = (u8 *)pIE;
+			break;
+#ifdef CONFIG_80211N_HT
+		case WLAN_EID_HT_OPERATION:
+			break;
+#endif
+#ifdef CONFIG_80211AC_VHT
+		case WLAN_EID_VHT_OPERATION:
+			rtw_tdls_process_vht_operation(padapter, ptdls_sta, pIE);
+			break;
+		case WLAN_EID_VHT_OP_MODE_NOTIFY:
+			rtw_tdls_process_vht_op_mode_notify(padapter, ptdls_sta, pIE);
+			break;
+#endif
+#ifdef CONFIG_80211AX_HE
+		case WLAN_EID_EXTENSION:
+			if (pIE->data[0] == WLAN_EID_EXTENSION_HE_OPERATION)
+				rtw_tdls_process_he_operation(padapter, ptdls_sta, pIE);
+			break;
+#endif
+		case _LINK_ID_IE_:
+			plinkid_ie = (u8 *)pIE;
+			break;
+		default:
+			break;
+		}
+
+		j += (pIE->Length + 2);
+
+	}
+
+	if (prx_pkt_attrib->encrypt) {
+		/* Verify mic in FTIE MIC field */
+		if (rtw_tdls_is_driver_setup(padapter) &&
+		    (tdls_verify_mic(ptdls_sta->tpk.kck, 3, plinkid_ie, prsnie, ptimeout_ie, pftie) == _FAIL)) {
+			rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_LOCALLY_POST, 0);
+			ret = _FAIL;
+			goto exit;
+		}
+	}
+
+	if (rtw_tdls_is_driver_setup(padapter)) {
+		rtw_tdls_set_link_established(padapter, _TRUE);
+
+		if (ptdls_sta->tdls_sta_state & TDLS_INITIATOR_STATE) {
+			ptdls_sta->tdls_sta_state |= TDLS_LINKED_STATE;
+			ptdls_sta->state |= WIFI_ASOC_STATE;
+			_cancel_timer_ex(&ptdls_sta->handshake_timer);
+		}
+
+		if (prx_pkt_attrib->encrypt) {
+			/* Start  TPK timer */
+			ptdls_sta->TPK_count = 0;
+			_set_timer(&ptdls_sta->TPK_timer, ONE_SEC);
+		}
+
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_ESTABLISHED, 0);
+	}
+
+exit:
+	return ret;
+
+}
+
+int On_TDLS_Dis_Req(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct rx_pkt_attrib	*prx_pkt_attrib = &precv_frame->u.hdr.attrib;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta_ap;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	sint parsing_length;	/* Frame body length, without icv_len */
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 FIXED_IE = 3, *dst;
+	u16 j;
+	struct tdls_txmgmt txmgmt;
+	int ret = _SUCCESS;
+
+	if (rtw_tdls_is_driver_setup(padapter) == _FALSE)
+		goto exit;
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	ptr += prx_pkt_attrib->hdrlen + prx_pkt_attrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	txmgmt.dialog_token = *(ptr + 2);
+	_rtw_memcpy(&txmgmt.peer, precv_frame->u.hdr.attrib.src, ETH_ALEN);
+	txmgmt.action_code = TDLS_DISCOVERY_RESPONSE;
+	parsing_length = ((union recv_frame *)precv_frame)->u.hdr.len
+			 - prx_pkt_attrib->hdrlen
+			 - prx_pkt_attrib->iv_len
+			 - prx_pkt_attrib->icv_len
+			 - LLC_HEADER_SIZE
+			 - ETH_TYPE_LEN
+			 - PAYLOAD_TYPE_LEN;
+
+	/* Parsing information element */
+	for (j = FIXED_IE; j < parsing_length;) {
+
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(ptr + j);
+
+		switch (pIE->ElementID) {
+		case _LINK_ID_IE_:
+			psta_ap = rtw_get_stainfo(pstapriv, pIE->data);
+			if (psta_ap == NULL)
+				goto exit;
+			dst = pIE->data + 12;
+			if (MacAddr_isBcst(dst) == _FALSE && (_rtw_memcmp(adapter_mac_addr(padapter), dst, ETH_ALEN) == _FALSE))
+				goto exit;
+			break;
+		default:
+			break;
+		}
+
+		j += (pIE->Length + 2);
+
+	}
+
+	issue_tdls_dis_rsp(padapter, &txmgmt, prx_pkt_attrib->privacy);
+
+exit:
+	return ret;
+
+}
+
+int On_TDLS_Teardown(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib	*prx_pkt_attrib = &precv_frame->u.hdr.attrib;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	u8 reason;
+
+	reason = *(ptr + prx_pkt_attrib->hdrlen + prx_pkt_attrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN + 2);
+	RTW_INFO("[TDLS] %s Reason code(%d)\n", __FUNCTION__, reason);
+
+	if (rtw_tdls_is_driver_setup(padapter)) {
+		rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_LOCALLY_POST, 0);
+	}
+
+	return _SUCCESS;
+
+}
+
+#if 0
+u8 TDLS_check_ch_state(uint state)
+{
+	if (state & TDLS_CH_SWITCH_ON_STATE &&
+	    state & TDLS_PEER_AT_OFF_STATE) {
+		if (state & TDLS_PEER_SLEEP_STATE)
+			return 2;	/* U-APSD + ch. switch */
+		else
+			return 1;	/* ch. switch */
+	} else
+		return 0;
+}
+#endif
+
+int On_TDLS_Peer_Traffic_Indication(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	struct rx_pkt_attrib	*pattrib = &precv_frame->u.hdr.attrib;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct tdls_txmgmt txmgmt;
+
+	ptr += pattrib->hdrlen + pattrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+
+		txmgmt.dialog_token = *(ptr + 2);
+		issue_tdls_peer_traffic_rsp(padapter, ptdls_sta, &txmgmt);
+		/* issue_nulldata_to_TDLS_peer_STA(padapter, ptdls_sta->phl_sta->mac_addr, 0, 0, 0); */
+
+	return _SUCCESS;
+}
+
+/* We process buffered data for 1. U-APSD, 2. ch. switch, 3. U-APSD + ch. switch here */
+int On_TDLS_Peer_Traffic_Rsp(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct rx_pkt_attrib	*pattrib = &precv_frame->u.hdr.attrib;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 wmmps_ac = 0;
+	/* u8 state=TDLS_check_ch_state(ptdls_sta->tdls_sta_state); */
+	int i;
+
+	ptdls_sta->sta_stats.rx_data_pkts++;
+
+	ptdls_sta->tdls_sta_state &= ~(TDLS_WAIT_PTR_STATE);
+
+	/* Check 4-AC queue bit */
+	if (ptdls_sta->uapsd_vo || ptdls_sta->uapsd_vi || ptdls_sta->uapsd_be || ptdls_sta->uapsd_bk)
+		wmmps_ac = 1;
+
+	/* If it's a direct link and have buffered frame */
+	if (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE) {
+		if (wmmps_ac) {
+			_list	*xmitframe_plist, *xmitframe_phead;
+			struct xmit_frame *pxmitframe = NULL;
+
+			_rtw_spinlock_bh(&ptdls_sta->sleep_q.lock);
+
+			xmitframe_phead = get_list_head(&ptdls_sta->sleep_q);
+			xmitframe_plist = get_next(xmitframe_phead);
+
+			/* transmit buffered frames */
+			while (rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist) == _FALSE) {
+				pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+				xmitframe_plist = get_next(xmitframe_plist);
+				rtw_list_delete(&pxmitframe->list);
+
+				ptdls_sta->sleepq_len--;
+				ptdls_sta->sleepq_ac_len--;
+				if (ptdls_sta->sleepq_len > 0) {
+					pxmitframe->attrib.mdata = 1;
+					pxmitframe->attrib.eosp = 0;
+				} else {
+					pxmitframe->attrib.mdata = 0;
+					pxmitframe->attrib.eosp = 1;
+				}
+				pxmitframe->attrib.triggered = 1;
+
+				rtw_intf_xmitframe_enqueue(padapter, pxmitframe);
+			}
+
+			if (ptdls_sta->sleepq_len == 0)
+				RTW_INFO("no buffered packets for tdls to xmit\n");
+			else {
+				RTW_INFO("error!psta->sleepq_len=%d\n", ptdls_sta->sleepq_len);
+				ptdls_sta->sleepq_len = 0;
+			}
+
+			_rtw_spinunlock_bh(&ptdls_sta->sleep_q.lock);
+
+		}
+
+	}
+
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_TDLS_CH_SW
+sint On_TDLS_Ch_Switch_Req(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib	*prx_pkt_attrib = &precv_frame->u.hdr.attrib;
+	sint parsing_length;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 FIXED_IE = 4;
+	u16 j;
+	u8 zaddr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	u16 switch_time = TDLS_CH_SWITCH_TIME * 1000, switch_timeout = TDLS_CH_SWITCH_TIMEOUT * 1000;
+	struct _ADAPTER_LINK *padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct rtw_chan_def chdef = {0};
+
+	if (rtw_tdls_is_chsw_allowed(padapter) == _FALSE) {
+		RTW_INFO("[TDLS] Ignore %s since channel switch is not allowed\n", __func__);
+		return _FAIL;
+	}
+
+	ptdls_sta->ch_switch_time = switch_time;
+	ptdls_sta->ch_switch_timeout = switch_timeout;
+
+	ptr += prx_pkt_attrib->hdrlen + prx_pkt_attrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	parsing_length = ((union recv_frame *)precv_frame)->u.hdr.len
+			 - prx_pkt_attrib->hdrlen
+			 - prx_pkt_attrib->iv_len
+			 - prx_pkt_attrib->icv_len
+			 - LLC_HEADER_SIZE
+			 - ETH_TYPE_LEN
+			 - PAYLOAD_TYPE_LEN;
+
+	pchsw_info->off_ch_num = *(ptr + 2);
+
+	if ((*(ptr + 2) == 2) && (rtw_hw_is_band_support(adapter_to_dvobj(padapter), BAND_ON_5G)))
+		pchsw_info->off_ch_num = 44;
+
+	if (pchsw_info->off_ch_num != pmlmeext->chandef.chan)
+		pchsw_info->delay_switch_back = _FALSE;
+
+	/* Parsing information element */
+	for (j = FIXED_IE; j < parsing_length;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(ptr + j);
+
+		switch (pIE->ElementID) {
+		case WLAN_EID_SECONDARY_CHANNEL_OFFSET:
+			switch (*(pIE->data)) {
+			case IEEE80211_SCA:
+				pchsw_info->ch_offset = CHAN_OFFSET_UPPER;
+				break;
+
+			case IEEE80211_SCB:
+				pchsw_info->ch_offset = CHAN_OFFSET_LOWER;
+				break;
+
+			default:/*IEEE80211_SCN*/
+				pchsw_info->ch_offset = CHAN_OFFSET_NO_EXT;
+				break;
+			}
+			break;
+		case _LINK_ID_IE_:
+			break;
+		case _CH_SWITCH_TIMING_:
+			ptdls_sta->ch_switch_time = (RTW_GET_LE16(pIE->data) >= TDLS_CH_SWITCH_TIME * 1000) ?
+				RTW_GET_LE16(pIE->data) : TDLS_CH_SWITCH_TIME * 1000;
+			ptdls_sta->ch_switch_timeout = (RTW_GET_LE16(pIE->data + 2) >= TDLS_CH_SWITCH_TIMEOUT * 1000) ?
+				RTW_GET_LE16(pIE->data + 2) : TDLS_CH_SWITCH_TIMEOUT * 1000;
+			RTW_INFO("[TDLS] %s ch_switch_time:%d, ch_switch_timeout:%d\n"
+				, __FUNCTION__, RTW_GET_LE16(pIE->data), RTW_GET_LE16(pIE->data + 2));
+			break;
+		default:
+			break;
+		}
+
+		j += (pIE->Length + 2);
+	}
+
+	chdef.chan = pchsw_info->off_ch_num;
+	chdef.bw = (pchsw_info->ch_offset) ? CHANNEL_WIDTH_40 : CHANNEL_WIDTH_20;
+	chdef.offset = pchsw_info->ch_offset;
+	chdef.band = rtw_get_band_type(chdef.chan);
+
+	if (!(pchsw_info->ch_sw_state & TDLS_CH_SWITCH_PREPARE_STATE)) {
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_PREPARE, 0);
+		return _FAIL;
+	}
+
+	/* cancel ch sw monitor timer for responder */
+	if (!(pchsw_info->ch_sw_state & TDLS_CH_SW_INITIATOR_STATE))
+		_cancel_timer_ex(&ptdls_sta->ch_sw_monitor_timer);
+
+	if (_rtw_memcmp(pchsw_info->addr, zaddr, ETH_ALEN) == _TRUE)
+		_rtw_memcpy(pchsw_info->addr, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+
+	if (ATOMIC_READ(&pchsw_info->chsw_on) == _FALSE)
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_START, 0);
+
+	rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_RESP, 0);
+
+	return _SUCCESS;
+}
+
+sint On_TDLS_Ch_Switch_Rsp(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta)
+{
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *ptr = precv_frame->u.hdr.rx_data;
+	struct rx_pkt_attrib	*prx_pkt_attrib = &precv_frame->u.hdr.attrib;
+	sint parsing_length;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	u8 FIXED_IE = 4;
+	u16 status_code, j, switch_time, switch_timeout;
+	struct _ADAPTER_LINK *padapter_link = precv_frame->u.hdr.adapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	int ret = _SUCCESS;
+
+	if (rtw_tdls_is_chsw_allowed(padapter) == _FALSE) {
+		RTW_INFO("[TDLS] Ignore %s since channel switch is not allowed\n", __func__);
+		return _SUCCESS;
+	}
+
+	/* If we receive Unsolicited TDLS Channel Switch Response when channel switch is running, */
+	/* we will go back to base channel and terminate this channel switch procedure */
+	if (ATOMIC_READ(&pchsw_info->chsw_on) == _TRUE) {
+		if (pmlmeext->chandef.chan != rtw_get_oper_ch(padapter, padapter_link)) {
+			RTW_INFO("[TDLS] Rx unsolicited channel switch response\n");
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_TO_BASE_CHNL, 0);
+			goto exit;
+		}
+	}
+
+	ptr += prx_pkt_attrib->hdrlen + prx_pkt_attrib->iv_len + LLC_HEADER_SIZE + ETH_TYPE_LEN + PAYLOAD_TYPE_LEN;
+	parsing_length = ((union recv_frame *)precv_frame)->u.hdr.len
+			 - prx_pkt_attrib->hdrlen
+			 - prx_pkt_attrib->iv_len
+			 - prx_pkt_attrib->icv_len
+			 - LLC_HEADER_SIZE
+			 - ETH_TYPE_LEN
+			 - PAYLOAD_TYPE_LEN;
+
+	_rtw_memcpy(&status_code, ptr + 2, 2);
+
+	if (status_code != 0) {
+		RTW_INFO("[TDLS] %s status_code:%d\n", __func__, status_code);
+		pchsw_info->ch_sw_state &= ~(TDLS_CH_SW_INITIATOR_STATE);
+		rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_END, 0);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	/* Parsing information element */
+	for (j = FIXED_IE; j < parsing_length;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(ptr + j);
+
+		switch (pIE->ElementID) {
+		case _LINK_ID_IE_:
+			break;
+		case _CH_SWITCH_TIMING_:
+			_rtw_memcpy(&switch_time, pIE->data, 2);
+			if (switch_time > ptdls_sta->ch_switch_time)
+				_rtw_memcpy(&ptdls_sta->ch_switch_time, &switch_time, 2);
+
+			_rtw_memcpy(&switch_timeout, pIE->data + 2, 2);
+			if (switch_timeout > ptdls_sta->ch_switch_timeout)
+				_rtw_memcpy(&ptdls_sta->ch_switch_timeout, &switch_timeout, 2);
+			break;
+		default:
+			break;
+		}
+
+		j += (pIE->Length + 2);
+	}
+
+	if ((pmlmeext->chandef.chan == rtw_get_oper_ch(padapter, padapter_link)) &&
+	    (pchsw_info->ch_sw_state & TDLS_WAIT_CH_RSP_STATE)) {
+		if (ATOMIC_READ(&pchsw_info->chsw_on) == _TRUE)
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_TO_OFF_CHNL, 0);
+	}
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_TDLS_CH_SW */
+
+#ifdef CONFIG_WFD
+void wfd_ie_tdls(_adapter *padapter, u8 *pframe, u32 *pktlen)
+{
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct wifi_display_info	*pwfd_info = padapter->tdlsinfo.wfd_info;
+	u8 wfdie[MAX_WFD_IE_LEN] = { 0x00 };
+	u32 wfdielen = 0;
+	u16 v16 = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST))
+		return;
+
+	/* WFD OUI */
+	wfdielen = 0;
+	wfdie[wfdielen++] = 0x50;
+	wfdie[wfdielen++] = 0x6F;
+	wfdie[wfdielen++] = 0x9A;
+	wfdie[wfdielen++] = 0x0A;	/* WFA WFD v1.0 */
+
+	/*
+	 *	Commented by Albert 20110825
+	 *	According to the WFD Specification, the negotiation request frame should contain 3 WFD attributes
+	 *	1. WFD Device Information
+	 *	2. Associated BSSID ( Optional )
+	 *	3. Local IP Adress ( Optional )
+	 */
+
+	/* WFD Device Information ATTR */
+	/* Type: */
+	wfdie[wfdielen++] = WFD_ATTR_DEVICE_INFO;
+
+	/* Length: */
+	/* Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/* Value1: */
+	/* WFD device information */
+	/* available for WFD session + Preferred TDLS + WSD ( WFD Service Discovery ) */
+	v16 = pwfd_info->wfd_device_type | WFD_DEVINFO_SESSION_AVAIL
+		| WFD_DEVINFO_PC_TDLS | WFD_DEVINFO_WSD;
+	RTW_PUT_BE16(wfdie + wfdielen, v16);
+	wfdielen += 2;
+
+	/* Value2: */
+	/* Session Management Control Port */
+	/* Default TCP port for RTSP messages is 554 */
+	RTW_PUT_BE16(wfdie + wfdielen, pwfd_info->tdls_rtsp_ctrlport);
+	wfdielen += 2;
+
+	/* Value3: */
+	/* WFD Device Maximum Throughput */
+	/* 300Mbps is the maximum throughput */
+	RTW_PUT_BE16(wfdie + wfdielen, 300);
+	wfdielen += 2;
+
+	/* Associated BSSID ATTR */
+	/* Type: */
+	wfdie[wfdielen++] = WFD_ATTR_ASSOC_BSSID;
+
+	/* Length: */
+	/* Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0006);
+	wfdielen += 2;
+
+	/* Value: */
+	/* Associated BSSID */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		_rtw_memcpy(wfdie + wfdielen, &pmlmepriv->assoc_bssid[0], ETH_ALEN);
+	else
+		_rtw_memset(wfdie + wfdielen, 0x00, ETH_ALEN);
+
+	/* Local IP Address ATTR */
+	wfdie[wfdielen++] = WFD_ATTR_LOCAL_IP_ADDR;
+
+	/* Length: */
+	/* Note: In the WFD specification, the size of length field is 2. */
+	RTW_PUT_BE16(wfdie + wfdielen, 0x0005);
+	wfdielen += 2;
+
+	/* Version: */
+	/* 0x01: Version1;IPv4 */
+	wfdie[wfdielen++] = 0x01;
+
+	/* IPv4 Address */
+	_rtw_memcpy(wfdie + wfdielen, pwfd_info->ip_address, 4);
+	wfdielen += 4;
+
+	pframe = rtw_set_ie(pframe, _VENDOR_SPECIFIC_IE_, wfdielen, (unsigned char *) wfdie, pktlen);
+
+}
+#endif /* CONFIG_WFD */
+
+void rtw_build_tdls_setup_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct _ADAPTER_LINK 	*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	int i = 0 ;
+	u32 time;
+	u8 *pframe_head;
+
+	/* SNonce */
+	if (pattrib->encrypt) {
+		for (i = 0; i < 8; i++) {
+			time = rtw_get_current_time();
+			_rtw_memcpy(&ptdls_sta->SNonce[4 * i], (u8 *)&time, 4);
+		}
+	}
+
+	pframe_head = pframe;	/* For rtw_tdls_set_ht_cap() */
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_dialog(pframe, pattrib, ptxmgmt);
+
+	pframe = rtw_tdls_set_capability(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_supported_rate(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_sup_ch(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_sup_reg_class(pframe, pattrib);
+
+	if (pattrib->encrypt)
+		pframe = rtw_tdls_set_rsnie(ptxmgmt, pframe, pattrib,  _TRUE, ptdls_sta);
+
+	pframe = rtw_tdls_set_ext_cap(pframe, pattrib);
+
+	if (pattrib->encrypt) {
+		pframe = rtw_tdls_set_ftie(ptxmgmt
+					   , pframe
+					   , pattrib
+					   , NULL
+					   , ptdls_sta->SNonce);
+
+		pframe = rtw_tdls_set_timeout_interval(ptxmgmt, pframe, pattrib, _TRUE, ptdls_sta);
+	}
+
+#ifdef CONFIG_80211N_HT
+	/* Sup_reg_classes(optional) */
+	if (pregistrypriv->ht_enable == _TRUE)
+		pframe = rtw_tdls_set_ht_cap(padapter, pframe_head, pattrib);
+#endif
+
+	pframe = rtw_tdls_set_bss_coexist(padapter, pframe, pattrib);
+
+	pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+
+	if ((padapter->registrypriv.wmm_enable == _TRUE) || (padapter_link->mlmepriv.htpriv.ht_option == _TRUE))
+		pframe = rtw_tdls_set_qos_cap(pframe, pattrib);
+
+#ifdef CONFIG_80211AC_VHT
+	if ((pregistrypriv->ht_enable == _TRUE) && (pmlmeext->chandef.chan > 14)
+		&& REGSTY_IS_11AC_ENABLE(pregistrypriv)
+		&& is_supported_vht(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AC(rfctl)
+		) {
+		pframe = rtw_tdls_set_aid(padapter, pframe, pattrib);
+		pframe = rtw_tdls_set_vht_cap(padapter, pframe, pattrib);
+	}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	if ((((pregistrypriv->ht_enable == _TRUE) && (pmlmeext->chandef.band == BAND_ON_24G))
+#ifdef CONFIG_80211AC_VHT
+		|| (REGSTY_IS_11AC_ENABLE(pregistrypriv) && (pmlmeext->chandef.band == BAND_ON_5G))
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+		|| (pmlmeext->chandef.band == BAND_ON_6G)
+#endif
+		)
+		&& REGSTY_IS_11AX_ENABLE(pregistrypriv)
+		&& is_supported_he(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AX(rfctl)
+	) {
+		pframe = rtw_tdls_set_he_cap(padapter, pframe, pattrib);
+	}
+#endif
+
+#ifdef CONFIG_WFD
+	if (padapter->wdinfo.wfd_tdls_enable == 1)
+		wfd_ie_tdls(padapter, pframe, &(pattrib->pktlen));
+#endif
+
+}
+
+void rtw_build_tdls_setup_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct _ADAPTER_LINK		*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	u8 k; /* for random ANonce */
+	u8  *pftie = NULL, *ptimeout_ie = NULL, *plinkid_ie = NULL, *prsnie = NULL, *pftie_mic = NULL;
+	u32 time;
+	u8 *pframe_head;
+
+	if (pattrib->encrypt) {
+		for (k = 0; k < 8; k++) {
+			time = rtw_get_current_time();
+			_rtw_memcpy(&ptdls_sta->ANonce[4 * k], (u8 *)&time, 4);
+		}
+	}
+
+	pframe_head = pframe;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_status_code(pframe, pattrib, ptxmgmt);
+
+	if (ptxmgmt->status_code != 0) {
+		RTW_INFO("[%s] status_code:%04x\n", __FUNCTION__, ptxmgmt->status_code);
+		return;
+	}
+
+	pframe = rtw_tdls_set_dialog(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_capability(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_supported_rate(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_sup_ch(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_sup_reg_class(pframe, pattrib);
+
+	if (pattrib->encrypt) {
+		prsnie = pframe;
+		pframe = rtw_tdls_set_rsnie(ptxmgmt, pframe, pattrib,  _FALSE, ptdls_sta);
+	}
+
+	pframe = rtw_tdls_set_ext_cap(pframe, pattrib);
+
+	if (pattrib->encrypt) {
+		if (rtw_tdls_is_driver_setup(padapter) == _TRUE)
+			wpa_tdls_generate_tpk(padapter, ptdls_sta);
+
+		pftie = pframe;
+		pftie_mic = pframe + 4;
+		pframe = rtw_tdls_set_ftie(ptxmgmt
+					   , pframe
+					   , pattrib
+					   , ptdls_sta->ANonce
+					   , ptdls_sta->SNonce);
+
+		ptimeout_ie = pframe;
+		pframe = rtw_tdls_set_timeout_interval(ptxmgmt, pframe, pattrib, _FALSE, ptdls_sta);
+	}
+
+#ifdef CONFIG_80211N_HT
+	/* Sup_reg_classes(optional) */
+	if (pregistrypriv->ht_enable == _TRUE)
+		pframe = rtw_tdls_set_ht_cap(padapter, pframe_head, pattrib);
+#endif
+
+	pframe = rtw_tdls_set_bss_coexist(padapter, pframe, pattrib);
+
+	plinkid_ie = pframe;
+	pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _FALSE);
+
+	/* Fill FTIE mic */
+	if (pattrib->encrypt && rtw_tdls_is_driver_setup(padapter) == _TRUE)
+		wpa_tdls_ftie_mic(ptdls_sta->tpk.kck, 2, plinkid_ie, prsnie, ptimeout_ie, pftie, pftie_mic);
+
+	if ((padapter->registrypriv.wmm_enable == _TRUE) || (padapter_link->mlmepriv.htpriv.ht_option == _TRUE))
+		pframe = rtw_tdls_set_qos_cap(pframe, pattrib);
+
+#ifdef CONFIG_80211AC_VHT
+	if ((pregistrypriv->ht_enable == _TRUE) && (pmlmeext->chandef.chan > 14)
+		&& REGSTY_IS_11AC_ENABLE(pregistrypriv)
+		&& is_supported_vht(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AC(rfctl)
+		) {
+		pframe = rtw_tdls_set_aid(padapter, pframe, pattrib);
+		pframe = rtw_tdls_set_vht_cap(padapter, pframe, pattrib);
+		pframe = rtw_tdls_set_vht_op_mode_notify(padapter, pframe, pattrib, pmlmeext->chandef.bw);
+	}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	if ((((pregistrypriv->ht_enable == _TRUE) && (pmlmeext->chandef.band == BAND_ON_24G))
+#ifdef CONFIG_80211AC_VHT
+		|| (REGSTY_IS_11AC_ENABLE(pregistrypriv) && (pmlmeext->chandef.band == BAND_ON_5G))
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+		|| (pmlmeext->chandef.band == BAND_ON_6G)
+#endif
+		)
+		&& REGSTY_IS_11AX_ENABLE(pregistrypriv)
+		&& is_supported_he(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AX(rfctl)
+	) {
+		pframe = rtw_tdls_set_he_cap(padapter, pframe, pattrib);
+	}
+#endif
+
+#ifdef CONFIG_WFD
+	if (padapter->wdinfo.wfd_tdls_enable)
+		wfd_ie_tdls(padapter, pframe, &(pattrib->pktlen));
+#endif
+
+}
+
+void rtw_build_tdls_setup_cfm_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct _ADAPTER_LINK	*padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+
+	unsigned int ie_len;
+	unsigned char *p;
+	u8 wmm_param_ele[24] = {0};
+	u8  *pftie = NULL, *ptimeout_ie = NULL, *plinkid_ie = NULL, *prsnie = NULL, *pftie_mic = NULL;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_status_code(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_dialog(pframe, pattrib, ptxmgmt);
+
+	if (ptxmgmt->status_code != 0)
+		return;
+
+	if (pattrib->encrypt) {
+		prsnie = pframe;
+		pframe = rtw_tdls_set_rsnie(ptxmgmt, pframe, pattrib, _TRUE, ptdls_sta);
+	}
+
+	if (pattrib->encrypt) {
+		pftie = pframe;
+		pftie_mic = pframe + 4;
+		pframe = rtw_tdls_set_ftie(ptxmgmt
+					   , pframe
+					   , pattrib
+					   , ptdls_sta->ANonce
+					   , ptdls_sta->SNonce);
+
+		ptimeout_ie = pframe;
+		pframe = rtw_tdls_set_timeout_interval(ptxmgmt, pframe, pattrib, _TRUE, ptdls_sta);
+
+		if (rtw_tdls_is_driver_setup(padapter) == _TRUE) {
+			/* Start TPK timer */
+			ptdls_sta->TPK_count = 0;
+			_set_timer(&ptdls_sta->TPK_timer, ONE_SEC);
+		}
+	}
+
+	/* HT operation; todo */
+
+	plinkid_ie = pframe;
+	pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+
+	if (pattrib->encrypt && (rtw_tdls_is_driver_setup(padapter) == _TRUE))
+		wpa_tdls_ftie_mic(ptdls_sta->tpk.kck, 3, plinkid_ie, prsnie, ptimeout_ie, pftie, pftie_mic);
+
+	if (ptdls_sta->qos_option == _TRUE)
+		pframe = rtw_tdls_set_wmm_params(padapter, pframe, pattrib);
+
+#ifdef CONFIG_80211AC_VHT
+	if ((pregistrypriv->ht_enable == _TRUE)
+		&& (ptdls_sta->vhtpriv.vht_option == _TRUE) && (pmlmeext->chandef.chan > 14)
+		&& REGSTY_IS_11AC_ENABLE(pregistrypriv)
+		&& is_supported_vht(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AC(rfctl)
+		) {
+		pframe = rtw_tdls_set_vht_operation(padapter, pframe, pattrib, pmlmeext->chandef.band, pmlmeext->chandef.chan);
+		pframe = rtw_tdls_set_vht_op_mode_notify(padapter, pframe, pattrib, pmlmeext->chandef.bw);
+	}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	if ((((pregistrypriv->ht_enable == _TRUE) && (pmlmeext->chandef.band == BAND_ON_24G))
+#ifdef CONFIG_80211AC_VHT
+		|| (REGSTY_IS_11AC_ENABLE(pregistrypriv) && (pmlmeext->chandef.band == BAND_ON_5G))
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+		|| (pmlmeext->chandef.band == BAND_ON_6G)
+#endif
+		)
+		&& (ptdls_sta->hepriv.he_option == _TRUE)
+		&& REGSTY_IS_11AX_ENABLE(pregistrypriv)
+		&& is_supported_he(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AX(rfctl)
+		) {
+		pframe = rtw_tdls_set_he_operation(padapter, pframe, pattrib);
+	}
+#endif
+}
+
+void rtw_build_tdls_teardown_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	u8  *pftie = NULL, *pftie_mic = NULL, *plinkid_ie = NULL;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_status_code(pframe, pattrib, ptxmgmt);
+
+	if (pattrib->encrypt) {
+		pftie = pframe;
+		pftie_mic = pframe + 4;
+		pframe = rtw_tdls_set_ftie(ptxmgmt
+					   , pframe
+					   , pattrib
+					   , ptdls_sta->ANonce
+					   , ptdls_sta->SNonce);
+	}
+
+	plinkid_ie = pframe;
+	if (ptdls_sta->tdls_sta_state & TDLS_INITIATOR_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _FALSE);
+	else if (ptdls_sta->tdls_sta_state & TDLS_RESPONDER_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+
+	if (pattrib->encrypt && (rtw_tdls_is_driver_setup(padapter) == _TRUE))
+		wpa_tdls_teardown_ftie_mic(ptdls_sta->tpk.kck, plinkid_ie, ptxmgmt->status_code, 1, 4, pftie, pftie_mic);
+}
+
+void rtw_build_tdls_dis_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt)
+{
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_dialog(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+
+}
+
+void rtw_build_tdls_dis_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, u8 privacy)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	u8 *pframe_head, pktlen_index;
+	struct _ADAPTER_LINK	*padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+
+	pktlen_index = pattrib->pktlen;
+	pframe_head = pframe;
+
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_PUBLIC);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_dialog(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_capability(padapter, pframe, pattrib);
+
+	pframe = rtw_tdls_set_supported_rate(padapter, pframe, pattrib);
+
+	pframe = rtw_tdls_set_sup_ch(padapter, pframe, pattrib);
+
+	if (privacy)
+		pframe = rtw_tdls_set_rsnie(ptxmgmt, pframe, pattrib, _TRUE, NULL);
+
+	pframe = rtw_tdls_set_ext_cap(pframe, pattrib);
+
+	if (privacy) {
+		pframe = rtw_tdls_set_ftie(ptxmgmt, pframe, pattrib, NULL, NULL);
+		pframe = rtw_tdls_set_timeout_interval(ptxmgmt, pframe, pattrib,  _TRUE, NULL);
+	}
+
+#ifdef CONFIG_80211N_HT
+	if (pregistrypriv->ht_enable == _TRUE)
+		pframe = rtw_tdls_set_ht_cap(padapter, pframe_head - pktlen_index, pattrib);
+#endif
+
+	pframe = rtw_tdls_set_bss_coexist(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _FALSE);
+
+#ifdef CONFIG_80211AC_VHT
+	if ((pregistrypriv->ht_enable == _TRUE) && (pmlmeext->chandef.chan > 14)
+		&& REGSTY_IS_11AC_ENABLE(pregistrypriv)
+		&& is_supported_vht(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AC(rfctl)
+		) {
+		pframe = rtw_tdls_set_vht_cap(padapter, pframe, pattrib);
+	}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	if ((((pregistrypriv->ht_enable == _TRUE) && (pmlmeext->chandef.band == BAND_ON_24G))
+#ifdef CONFIG_80211AC_VHT
+		|| (REGSTY_IS_11AC_ENABLE(pregistrypriv) && (pmlmeext->chandef.band == BAND_ON_5G))
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+		|| (pmlmeext->chandef.band == BAND_ON_6G)
+#endif
+		)
+		&& REGSTY_IS_11AX_ENABLE(pregistrypriv)
+		&& is_supported_he(pregistrypriv->wireless_mode)
+		&& RFCTL_REG_EN_11AX(rfctl)
+		) {
+		pframe = rtw_tdls_set_he_cap(padapter, pframe, pattrib);
+	}
+#endif
+
+}
+
+
+void rtw_build_tdls_peer_traffic_indication_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	u8 AC_queue = 0;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_dialog(pframe, pattrib, ptxmgmt);
+
+	if (ptdls_sta->tdls_sta_state & TDLS_INITIATOR_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _FALSE);
+	else if (ptdls_sta->tdls_sta_state & TDLS_RESPONDER_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+
+	/* PTI control */
+	/* PU buffer status */
+	if (ptdls_sta->uapsd_bk & BIT(1))
+		AC_queue = BIT(0);
+	if (ptdls_sta->uapsd_be & BIT(1))
+		AC_queue = BIT(1);
+	if (ptdls_sta->uapsd_vi & BIT(1))
+		AC_queue = BIT(2);
+	if (ptdls_sta->uapsd_vo & BIT(1))
+		AC_queue = BIT(3);
+	pframe = rtw_set_ie(pframe, _PTI_BUFFER_STATUS_, 1, &AC_queue, &(pattrib->pktlen));
+
+}
+
+void rtw_build_tdls_peer_traffic_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_dialog(pframe, pattrib, ptxmgmt);
+
+	if (ptdls_sta->tdls_sta_state & TDLS_INITIATOR_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _FALSE);
+	else if (ptdls_sta->tdls_sta_state & TDLS_RESPONDER_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+}
+
+#ifdef CONFIG_TDLS_CH_SW
+void rtw_build_tdls_ch_switch_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	u16 switch_time = TDLS_CH_SWITCH_TIME * 1000, switch_timeout = TDLS_CH_SWITCH_TIMEOUT * 1000;
+
+	ptdls_sta->ch_switch_time = switch_time;
+	ptdls_sta->ch_switch_timeout = switch_timeout;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_target_ch(padapter, pframe, pattrib);
+	pframe = rtw_tdls_set_reg_class(pframe, pattrib, ptdls_sta);
+
+	if (ptdlsinfo->chsw_info.ch_offset != CHAN_OFFSET_NO_EXT) {
+		switch (ptdlsinfo->chsw_info.ch_offset) {
+		case CHAN_OFFSET_UPPER:
+			pframe = rtw_tdls_set_second_channel_offset(pframe, pattrib, IEEE80211_SCA);
+			break;
+		case CHAN_OFFSET_LOWER:
+			pframe = rtw_tdls_set_second_channel_offset(pframe, pattrib, IEEE80211_SCB);
+			break;
+		}
+	}
+
+	if (ptdls_sta->tdls_sta_state & TDLS_INITIATOR_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _FALSE);
+	else if (ptdls_sta->tdls_sta_state & TDLS_RESPONDER_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+
+	pframe = rtw_tdls_set_ch_sw(pframe, pattrib, ptdls_sta);
+
+}
+
+void rtw_build_tdls_ch_switch_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta)
+{
+
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_tdls_set_category(pframe, pattrib, RTW_WLAN_CATEGORY_TDLS);
+	pframe = rtw_tdls_set_action(pframe, pattrib, ptxmgmt);
+	pframe = rtw_tdls_set_status_code(pframe, pattrib, ptxmgmt);
+
+	if (ptdls_sta->tdls_sta_state & TDLS_INITIATOR_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _FALSE);
+	else if (ptdls_sta->tdls_sta_state & TDLS_RESPONDER_STATE)
+		pframe = rtw_tdls_set_linkid(padapter, pframe, pattrib, _TRUE);
+
+	pframe = rtw_tdls_set_ch_sw(pframe, pattrib, ptdls_sta);
+}
+#endif
+
+#ifdef CONFIG_WFD
+void rtw_build_tunneled_probe_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe)
+{
+	u8 i;
+	_adapter *iface = NULL;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+	struct wifidirect_info *pwdinfo;
+
+	u8 category = RTW_WLAN_CATEGORY_P2P;
+	u8 WFA_OUI[3] = { 0x50, 0x6f, 0x9a};
+	u8 probe_req = 4;
+	u8 wfdielen = 0;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 3, WFA_OUI, &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(probe_req), &(pattrib->pktlen));
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if ((iface) && rtw_is_adapter_up(iface) &&
+			rtw_iface_at_same_hwband(padapter, iface)) {
+
+			pwdinfo = &iface->wdinfo;
+			if (pwdinfo->wfd_tdls_enable) {
+				wfdielen = build_probe_req_wfd_ie(pwdinfo, pframe);
+				pframe += wfdielen;
+				pattrib->pktlen += wfdielen;
+			}
+		}
+	}
+}
+
+void rtw_build_tunneled_probe_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe)
+{
+	u8 i;
+	_adapter *iface = NULL;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	struct wifidirect_info *pwdinfo;
+	u8 category = RTW_WLAN_CATEGORY_P2P;
+	u8 WFA_OUI[3] = { 0x50, 0x6f, 0x9a};
+	u8 probe_rsp = 5;
+	u8 wfdielen = 0;
+
+	pframe = rtw_tdls_set_payload_type(pframe, pattrib);
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 3, WFA_OUI, &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(probe_rsp), &(pattrib->pktlen));
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && rtw_is_adapter_up(iface) &&
+			rtw_iface_at_same_hwband(padapter, iface)) {
+
+			pwdinfo = &iface->wdinfo;
+			if (pwdinfo->wfd_tdls_enable) {
+				wfdielen = build_probe_resp_wfd_ie(pwdinfo, pframe, 1);
+				pframe += wfdielen;
+				pattrib->pktlen += wfdielen;
+			}
+		}
+	}
+}
+#endif /* CONFIG_WFD */
+
+void _tdls_tpk_timer_hdl(void *FunctionContext)
+{
+	struct sta_info *ptdls_sta = (struct sta_info *)FunctionContext;
+	struct tdls_txmgmt txmgmt;
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	ptdls_sta->TPK_count++;
+	/* TPK_timer expired in a second */
+	/* Retry timer should set at least 301 sec. */
+	if (ptdls_sta->TPK_count >= (ptdls_sta->TDLS_PeerKey_Lifetime - 3)) {
+		RTW_INFO("[TDLS] %s, Re-Setup TDLS link with "MAC_FMT" since TPK lifetime expires!\n",
+			__FUNCTION__, MAC_ARG(ptdls_sta->phl_sta->mac_addr));
+		ptdls_sta->TPK_count = 0;
+		_rtw_memcpy(txmgmt.peer, ptdls_sta->phl_sta->mac_addr, ETH_ALEN);
+		issue_tdls_setup_req(ptdls_sta->padapter, &txmgmt, _FALSE);
+	}
+
+	_set_timer(&ptdls_sta->TPK_timer, ONE_SEC);
+}
+
+#ifdef CONFIG_TDLS_CH_SW
+void _tdls_ch_switch_timer_hdl(void *FunctionContext)
+{
+	struct sta_info *ptdls_sta = (struct sta_info *)FunctionContext;
+	_adapter *padapter = ptdls_sta->padapter;
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+
+	rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_END_TO_BASE_CHNL, 0);
+	RTW_INFO("[TDLS] %s, can't get traffic from op_ch:%d\n", __func__,
+		 rtw_get_oper_ch(padapter, ptdls_sta->padapter_link));
+}
+
+void _tdls_delay_timer_hdl(void *FunctionContext)
+{
+	struct sta_info *ptdls_sta = (struct sta_info *)FunctionContext;
+	_adapter *padapter = ptdls_sta->padapter;
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+
+	RTW_INFO("[TDLS] %s, op_ch:%d, tdls_state:0x%08x\n", __func__,
+		 rtw_get_oper_ch(padapter, ptdls_sta->padapter_link), ptdls_sta->tdls_sta_state);
+	pchsw_info->delay_switch_back = _TRUE;
+}
+
+void _tdls_stay_on_base_chnl_timer_hdl(void *FunctionContext)
+{
+	struct sta_info *ptdls_sta = (struct sta_info *)FunctionContext;
+	_adapter *padapter = ptdls_sta->padapter;
+	struct _ADAPTER_LINK *padapter_link = ptdls_sta->padapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+
+	if (ptdls_sta != NULL) {
+		if (pmlmeext->chandef.chan == rtw_get_oper_ch(padapter, padapter_link)) {
+			issue_tdls_ch_switch_req(padapter, ptdls_sta);
+			pchsw_info->ch_sw_state |= TDLS_WAIT_CH_RSP_STATE;
+		}
+	}
+}
+
+void _tdls_ch_switch_monitor_timer_hdl(void *FunctionContext)
+{
+	struct sta_info *ptdls_sta = (struct sta_info *)FunctionContext;
+	_adapter *padapter = ptdls_sta->padapter;
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+
+	rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_END, 0);
+	RTW_INFO("[TDLS] %s, does not receive ch sw req\n", __func__);
+}
+
+#endif
+
+void _tdls_handshake_timer_hdl(void *FunctionContext)
+{
+	struct sta_info *ptdls_sta = (struct sta_info *)FunctionContext;
+	_adapter *padapter = NULL;
+
+	if (ptdls_sta != NULL) {
+		padapter = ptdls_sta->padapter;
+
+		RTW_INFO("[TDLS] Handshake time out\n");
+		if (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE)
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA, 0);
+		else
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_LOCALLY, 0);
+	}
+}
+
+void _tdls_pti_timer_hdl(void *FunctionContext)
+{
+	struct sta_info *ptdls_sta = (struct sta_info *)FunctionContext;
+	_adapter *padapter = NULL;
+
+	if (ptdls_sta != NULL) {
+		padapter = ptdls_sta->padapter;
+
+		if (ptdls_sta->tdls_sta_state & TDLS_WAIT_PTR_STATE) {
+			RTW_INFO("[TDLS] Doesn't receive PTR from peer dev:"MAC_FMT"; "
+				"Send TDLS Tear Down\n", MAC_ARG(ptdls_sta->phl_sta->mac_addr));
+			rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_TEARDOWN_STA_TOOFAR, 0);
+		}
+	}
+}
+
+void rtw_init_tdls_timer(_adapter *padapter, struct sta_info *psta)
+{
+	psta->padapter = padapter;
+	rtw_init_timer(&psta->TPK_timer, _tdls_tpk_timer_hdl, psta);
+#ifdef CONFIG_TDLS_CH_SW
+	rtw_init_timer(&psta->ch_sw_timer, _tdls_ch_switch_timer_hdl, psta);
+	rtw_init_timer(&psta->delay_timer, _tdls_delay_timer_hdl, psta);
+	rtw_init_timer(&psta->stay_on_base_chnl_timer, _tdls_stay_on_base_chnl_timer_hdl, psta);
+	rtw_init_timer(&psta->ch_sw_monitor_timer, _tdls_ch_switch_monitor_timer_hdl, psta);
+#endif
+	rtw_init_timer(&psta->handshake_timer, _tdls_handshake_timer_hdl, psta);
+	rtw_init_timer(&psta->pti_timer, _tdls_pti_timer_hdl, psta);
+}
+
+void rtw_cancel_tdls_timer(struct sta_info *psta)
+{
+	_cancel_timer_ex(&psta->TPK_timer);
+#ifdef CONFIG_TDLS_CH_SW
+	_cancel_timer_ex(&psta->ch_sw_timer);
+	_cancel_timer_ex(&psta->delay_timer);
+	_cancel_timer_ex(&psta->stay_on_base_chnl_timer);
+	_cancel_timer_ex(&psta->ch_sw_monitor_timer);
+#endif
+	_cancel_timer_ex(&psta->handshake_timer);
+	_cancel_timer_ex(&psta->pti_timer);
+}
+
+void rtw_tdls_teardown_pre_hdl(_adapter *padapter, struct sta_info *psta)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+
+	rtw_cancel_tdls_timer(psta);
+
+	_rtw_spinlock_bh(&(pstapriv->sta_hash_lock));
+	if (ptdlsinfo->sta_cnt != 0)
+		ptdlsinfo->sta_cnt--;
+	_rtw_spinunlock_bh(&(pstapriv->sta_hash_lock));
+
+	if (ptdlsinfo->sta_cnt < MAX_ALLOWED_TDLS_STA_NUM) {
+		ptdlsinfo->sta_maximum = _FALSE;
+		_rtw_memset(&ptdlsinfo->ss_record, 0x00, sizeof(struct tdls_ss_record));
+	}
+
+	if (ptdlsinfo->sta_cnt == 0)
+		rtw_tdls_set_link_established(padapter, _FALSE);
+	else
+		RTW_INFO("Remain tdls sta:%02x\n", ptdlsinfo->sta_cnt);
+}
+
+void rtw_tdls_teardown_post_hdl(_adapter *padapter, struct sta_info *psta, u8 enqueue_cmd)
+{
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	enum role_type rtype = PHL_RTYPE_STATION;
+	enum rtw_phl_status status;
+	int tid;
+
+	/* Clear cam */
+	rtw_clearstakey_cmd(padapter, psta, enqueue_cmd);
+
+	for (tid = 0; tid < TID_NUM; tid++) {
+		if (psta->recvreorder_ctrl[tid].enable == _TRUE) {
+			psta->recvreorder_ctrl[tid].enable =_FALSE;
+			rtw_phl_stop_rx_ba_session(phl, psta->phl_sta, tid);
+			RTW_INFO(FUNC_ADPT_FMT"stop process tid %d \n",
+				FUNC_ADPT_ARG(padapter), tid);
+		}
+	}
+
+	rtw_sta_hal_media_status_rpt_cmd(padapter, psta, false, RTW_CMDF_DIRECTLY);
+
+	/* Free tdls sta info */
+	rtw_free_mld_stainfo(padapter, psta->phl_sta->mld);
+
+	if (ptdlsinfo->sta_cnt == 0) {
+		status = rtw_phl_cmd_wrole_change(phl, wrole, NULL,
+			WR_CHG_TYPE, (u8*)&rtype, sizeof(enum role_type), PHL_CMD_DIRECTLY, 0);
+		if (status != RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s - change to phl role type = %d fail with error = %d\n", __func__, rtype, status);
+			rtw_warn_on(1);
+		}
+		/* Inform PHL for phl_mr_info_upt */
+		rtw_send_tdls_sync_msg(padapter);
+	}
+}
+
+int rtw_tdls_is_driver_setup(_adapter *padapter)
+{
+	return padapter->tdlsinfo.driver_setup;
+}
+
+const char *rtw_tdls_action_txt(enum TDLS_ACTION_FIELD action)
+{
+	switch (action) {
+	case TDLS_SETUP_REQUEST:
+		return "TDLS_SETUP_REQUEST";
+	case TDLS_SETUP_RESPONSE:
+		return "TDLS_SETUP_RESPONSE";
+	case TDLS_SETUP_CONFIRM:
+		return "TDLS_SETUP_CONFIRM";
+	case TDLS_TEARDOWN:
+		return "TDLS_TEARDOWN";
+	case TDLS_PEER_TRAFFIC_INDICATION:
+		return "TDLS_PEER_TRAFFIC_INDICATION";
+	case TDLS_CHANNEL_SWITCH_REQUEST:
+		return "TDLS_CHANNEL_SWITCH_REQUEST";
+	case TDLS_CHANNEL_SWITCH_RESPONSE:
+		return "TDLS_CHANNEL_SWITCH_RESPONSE";
+	case TDLS_PEER_PSM_REQUEST:
+		return "TDLS_PEER_PSM_REQUEST";
+	case TDLS_PEER_PSM_RESPONSE:
+		return "TDLS_PEER_PSM_RESPONSE";
+	case TDLS_PEER_TRAFFIC_RESPONSE:
+		return "TDLS_PEER_TRAFFIC_RESPONSE";
+	case TDLS_DISCOVERY_REQUEST:
+		return "TDLS_DISCOVERY_REQUEST";
+	case TDLS_DISCOVERY_RESPONSE:
+		return "TDLS_DISCOVERY_RESPONSE";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+#endif /* CONFIG_TDLS */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx.c
new file mode 100644
index 000000000000..eb0184242f7b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx.c
@@ -0,0 +1,158 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_TRX_C_
+#include <drv_types.h>		/* struct dvobj_priv and etc. */
+
+
+#ifdef RTW_PHL_TX
+s32 rtw_core_tx_mgmt(_adapter *padapter, struct xmit_frame *pxframe)
+{
+
+	pxframe->xftype = RTW_TX_DRV_MGMT;
+
+#ifdef RTW_PHL_DBG_CMD
+	core_add_record(padapter, REC_TX_MGMT, pxframe);
+#endif
+
+	if(core_tx_prepare_phl(padapter, pxframe) == FAIL)
+		return _FAIL;
+
+	if (core_tx_call_phl(padapter, pxframe, NULL) == FAIL)
+		return _FAIL;
+
+	return _SUCCESS;
+}
+#endif
+
+#ifdef CONFIG_DRV_FAKE_AP
+int rtw_fakeap_tx(struct _ADAPTER*, struct xmit_frame*);
+#endif /* CONFIG_DRV_FAKE_AP */
+/*rtw_hal_mgnt_xmit*/
+s32 rtw_mgnt_xmit(_adapter *adapter, struct xmit_frame *pmgntframe)
+{
+	s32 ret = _FAIL;
+	struct pkt_attrib *attrib = &pmgntframe->attrib;
+	struct _ADAPTER_LINK *adapter_link = attrib->adapter_link;
+
+	update_mgntframe_attrib_addr(adapter, pmgntframe);
+
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+	if ((!MLME_IS_MESH(adapter)
+		&& SEC_IS_BIP_KEY_INSTALLED(&adapter_link->securitypriv) == _TRUE)
+		#ifdef CONFIG_RTW_MESH
+		|| (MLME_IS_MESH(adapter) && adapter->mesh_info.mesh_auth_id)
+		#endif
+	)
+		rtw_mgmt_xmitframe_coalesce(adapter, pmgntframe->pkt, pmgntframe);
+#endif
+
+#ifdef CONFIG_DRV_FAKE_AP
+#ifndef RTW_PHL_TEST_FPGA
+	if (rtw_fakeap_tx(adapter, pmgntframe) == _SUCCESS)
+		return _SUCCESS;
+#endif
+#endif /* CONFIG_DRV_FAKE_AP */
+	if (ret == _FAIL)
+		core_tx_free_xmitframe(adapter, pmgntframe);
+
+	ret = rtw_core_tx_mgmt(adapter, pmgntframe);
+	return ret;
+}
+
+
+struct lite_data_buf *rtw_alloc_litedatabuf(struct trx_data_buf_q *data_buf_q)
+{
+	struct lite_data_buf *litedatabuf =  NULL;
+	_list *list, *head;
+	_queue *free_litedatabuf_q = &data_buf_q->free_data_buf_queue;
+	unsigned long sp_flags;
+
+	/* RTW_INFO("+rtw_alloc_litexmitbuf\n"); */
+
+	_rtw_spinlock_irq(&free_litedatabuf_q->lock, &sp_flags);
+
+	if (_rtw_queue_empty(free_litedatabuf_q) == _TRUE)
+		litedatabuf = NULL;
+	else {
+
+		head = get_list_head(free_litedatabuf_q);
+
+		list = get_next(head);
+
+		litedatabuf = LIST_CONTAINOR(list,
+			struct lite_data_buf, list);
+
+		rtw_list_delete(&(litedatabuf->list));
+	}
+
+	if (litedatabuf !=  NULL) {
+		data_buf_q->free_data_buf_cnt--;
+
+
+		if (litedatabuf->sctx) {
+			RTW_INFO("%s plitexmitbuf->sctx is not NULL\n",
+				__func__);
+			rtw_sctx_done_err(&litedatabuf->sctx,
+				RTW_SCTX_DONE_BUF_ALLOC);
+		}
+	}
+
+	_rtw_spinunlock_irq(&free_litedatabuf_q->lock, &sp_flags);
+
+
+	return litedatabuf;
+}
+
+s32 rtw_free_litedatabuf(struct trx_data_buf_q *data_buf_q,
+		struct lite_data_buf *lite_data_buf)
+{
+	_queue *free_litedatabuf_q = &data_buf_q->free_data_buf_queue;
+	unsigned long sp_flags;
+
+	/* RTW_INFO("+rtw_free_litexmitbuf\n"); */
+
+	if (data_buf_q == NULL)
+		return _FAIL;
+
+	if (lite_data_buf == NULL)
+		return _FAIL;
+
+	lite_data_buf->pbuf = NULL;
+	lite_data_buf->phl_buf_ptr = NULL;
+#ifdef CONFIG_USB_HCI
+	lite_data_buf->dataurb = NULL;
+#endif
+
+	if (lite_data_buf->sctx) {
+		RTW_INFO("%s lite_data_buf->sctx is not NULL\n", __func__);
+		rtw_sctx_done_err(&lite_data_buf->sctx, RTW_SCTX_DONE_BUF_FREE);
+		return _FAIL;
+	}
+
+	_rtw_spinlock_irq(&free_litedatabuf_q->lock, &sp_flags);
+
+	rtw_list_delete(&lite_data_buf->list);
+
+	rtw_list_insert_tail(&(lite_data_buf->list),
+		get_list_head(free_litedatabuf_q));
+
+	data_buf_q->free_data_buf_cnt++;
+
+	_rtw_spinunlock_irq(&free_litedatabuf_q->lock, &sp_flags);
+
+
+	return _SUCCESS;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx_pci.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx_pci.c
new file mode 100644
index 000000000000..b7ea14ac3f58
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx_pci.c
@@ -0,0 +1,188 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_TRX_PCI_C_
+#include <drv_types.h>		/* struct dvobj_priv and etc. */
+
+static void rtw_mi_pci_tasklets_kill(_adapter *padapter)
+{
+	int i;
+	_adapter *iface;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && rtw_is_adapter_up(iface)) {
+			rtw_tasklet_kill(&(padapter->xmitpriv.xmit_tasklet));
+		}
+	}
+}
+
+/********************************xmit section*****************************/
+s32 pci_init_xmit_priv(_adapter *adapter)
+{
+	s32 ret = _SUCCESS;
+	struct xmit_priv *pxmitpriv = &adapter->xmitpriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	_rtw_spinlock_init(&dvobj_to_pci(dvobj)->irq_th_lock);
+
+	return ret;
+}
+
+void pci_free_xmit_priv(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	_rtw_spinlock_free(&dvobj_to_pci(dvobj)->irq_th_lock);
+}
+
+static s32 pci_xmit_direct(_adapter *adapter, struct xmit_frame *pxmitframe)
+{
+	s32 res = _SUCCESS;
+#if 0
+	#ifdef CONFIG_XMIT_THREAD_MODE
+	struct xmit_priv *pxmitpriv = &adapter->xmitpriv;
+	#endif
+
+
+	res = rtw_xmitframe_coalesce(adapter, pxmitframe->pkt, pxmitframe);
+	if (res == _SUCCESS) {
+	#if 0 /*def CONFIG_XMIT_THREAD_MODE*/
+		enqueue_pending_xmitbuf(pxmitpriv, pxmitframe->pxmitbuf);
+	#else
+		pci_dump_xframe(adapter, pxmitframe);
+	#endif
+	}
+#endif
+	return res;
+}
+
+static s32 pci_data_xmit(_adapter *adapter, struct xmit_frame *pxmitframe)
+{
+	s32 res;
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct xmit_buf *pxmitbuf = NULL;
+	#endif
+	struct xmit_priv *pxmitpriv = &adapter->xmitpriv;
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+#ifdef CONFIG_TX_AMSDU
+	int tx_amsdu = adapter->tx_amsdu;
+	u8 amsdu_timeout = 0;
+#endif
+
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	if (rtw_txframes_sta_ac_pending(adapter, pattrib) > 0)
+		goto enqueue;
+
+#ifndef CONFIG_XMIT_THREAD_MODE
+	/*
+	if (rtw_hal_pci_check_enough_txdesc(GET_PHL_COM(dvobj), pattrib->qsel) == _FALSE)
+		goto enqueue;
+	*/
+	if (rtw_xmit_ac_blocked(adapter) == _TRUE)
+		goto enqueue;
+#endif
+
+
+	if (rtw_mi_check_status(adapter, MI_STA_LINKING))
+		goto enqueue;
+
+#ifdef CONFIG_TX_AMSDU
+	if (MLME_IS_STA(adapter) &&
+		check_amsdu_tx_support(adapter, pattrib)) {
+
+		if (IS_AMSDU_AMPDU_VALID(pattrib))
+			goto enqueue;
+	}
+#endif
+#if 0 /*CONFIG_CORE_XMITBUF*/
+	pxmitbuf = rtw_alloc_xmitbuf(pxmitpriv);
+	if (pxmitbuf == NULL)
+		goto enqueue;
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+	pxmitframe->pxmitbuf = pxmitbuf;
+	pxmitframe->buf_addr = pxmitbuf->pbuf;
+	pxmitbuf->priv_data = pxmitframe;
+#else
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+#endif
+	if (pci_xmit_direct(adapter, pxmitframe) != _SUCCESS) {
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		rtw_free_xmitbuf(pxmitpriv, pxmitbuf);
+		#endif
+		rtw_free_xmitframe(pxmitpriv, pxmitframe);
+	}
+
+	return _TRUE;
+
+enqueue:
+	res = rtw_xmitframe_enqueue(adapter, pxmitframe);
+
+#ifdef CONFIG_TX_AMSDU
+	if(res == _SUCCESS && tx_amsdu == 2)
+	{
+		amsdu_timeout = rtw_amsdu_get_timer_status(adapter, pattrib->priority);
+		if(amsdu_timeout == RTW_AMSDU_TIMER_SETTING)
+		{
+			rtw_amsdu_cancel_timer(adapter, pattrib->priority);
+			rtw_amsdu_set_timer_status(adapter, pattrib->priority,
+				RTW_AMSDU_TIMER_UNSET);
+		}
+	}
+#endif
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+	if (res != _SUCCESS) {
+		rtw_free_xmitframe(pxmitpriv, pxmitframe);
+
+		pxmitpriv->tx_drop++;
+		return _TRUE;
+	}
+
+#ifdef CONFIG_TX_AMSDU
+	rtw_tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
+#endif
+	return _FALSE;
+}
+
+/******************************** recv section*******************************/
+s32 pci_init_recv_priv(struct dvobj_priv *dvobj)
+{
+	s32 ret = _SUCCESS;
+
+	return ret;
+}
+
+void pci_free_recv_priv(struct dvobj_priv *dvobj)
+{
+
+}
+
+
+struct rtw_intf_ops pci_ops = {
+	.init_xmit_priv = pci_init_xmit_priv,
+	.free_xmit_priv = pci_free_xmit_priv,
+	.data_xmit	= pci_data_xmit,
+
+	.init_recv_priv = pci_init_recv_priv,
+	.free_recv_priv = pci_free_recv_priv,
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx_usb.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx_usb.c
new file mode 100644
index 000000000000..122c67789061
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_trx_usb.c
@@ -0,0 +1,50 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_TRX_USB_C_
+#include <drv_types.h>		/* struct dvobj_priv and etc. */
+
+
+/********************************xmit section*******************************/
+s32 usb_init_xmit_priv(_adapter *adapter)
+{
+	return _SUCCESS;
+}
+
+void usb_free_xmit_priv(_adapter *adapter)
+{
+}
+
+/******************************** recv section*******************************/
+int usb_init_recv_priv(struct dvobj_priv *dvobj)
+{
+	int res = _SUCCESS;
+
+	return res;
+}
+
+void usb_free_recv_priv(struct dvobj_priv *dvobj)
+{
+
+}
+
+struct rtw_intf_ops usb_ops = {
+	.init_xmit_priv = usb_init_xmit_priv,
+	.free_xmit_priv = usb_free_xmit_priv,
+
+	.init_recv_priv = usb_init_recv_priv,
+	.free_recv_priv = usb_free_recv_priv,
+
+};
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_twt.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_twt.c
new file mode 100644
index 000000000000..da0fb946bb40
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_twt.c
@@ -0,0 +1,1114 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_TWT_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+
+#if 0
+void _twt_debug_print_info(_adapter *padapter, struct rtw_phl_twt_element *twt_element, const char* func_name)
+{
+	struct rtw_phl_indiv_twt_para_set *twt_para = &twt_element->info.i_twt_para_set;
+
+	RTW_INFO("[TWT] %s - nego_type             = %d\n", func_name, TWT_NEGO_TYPE(twt_element));
+	if (TWT_NEGO_TYPE(twt_element) == RTW_PHL_INDIV_TWT ||
+		TWT_NEGO_TYPE(twt_element) == RTW_PHL_WAKE_TBTT_INR) {
+		RTW_INFO("[TWT] %s - request               = %d\n", func_name, twt_para->req_type.twt_request);
+		RTW_INFO("[TWT] %s - twt_setup_cmd         = %d\n", func_name, TWT_I_SETUP_CMD(twt_element));
+
+		RTW_INFO("[TWT] %s - twt_protection        = %d\n", func_name, twt_para->req_type.twt_protection);
+		RTW_INFO("[TWT] %s - trigger               = %d\n", func_name, twt_para->req_type.trigger);
+		RTW_INFO("[TWT] %s - implicit              = %d\n", func_name, twt_para->req_type.implicit);
+		RTW_INFO("[TWT] %s - flow_type             = %d\n", func_name, twt_para->req_type.flow_type);
+		RTW_INFO("[TWT] %s - twt_flow_id           = %d\n", func_name, TWT_I_FLOW_ID(twt_element));
+		RTW_INFO("[TWT] %s - twt_wake_int_exp      = %d\n", func_name, twt_para->req_type.twt_wake_int_exp);
+		RTW_INFO("[TWT] %s - target_wake_t_h       = %u\n", func_name, twt_para->target_wake_t_h);
+		RTW_INFO("[TWT] %s - target_wake_t_l       = %u\n", func_name, twt_para->target_wake_t_l);
+		RTW_INFO("[TWT] %s - nom_min_twt_wake_dur  = %d\n", func_name, twt_para->nom_min_twt_wake_dur);
+		RTW_INFO("[TWT] %s - twt_wake_int_mantissa = %d\n", func_name, twt_para->twt_wake_int_mantissa);
+	}
+}
+#endif
+
+static void _twt_teardown_done(void *priv, struct rtw_phl_stainfo_t *sta,
+			       enum rtw_phl_status sts)
+{
+	if (RTW_PHL_STATUS_SUCCESS == sts || RTW_PHL_STATUS_CMD_SUCCESS == sts )
+		RTW_INFO("[TWT] %s: success\n", __func__);
+	else
+		RTW_INFO("[TWT] %s: fail (%d)\n", __func__, sts);
+}
+
+static void _twt_accept_done(void *priv, struct rtw_phl_stainfo_t *sta,
+			     struct rtw_phl_twt_setup_info *setup_info,
+			     enum rtw_phl_status sts)
+{
+	struct dvobj_priv * dvobj = priv;
+	_adapter *padapter = dvobj_get_primary_adapter(dvobj);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct rtw_phl_twt_element *twt_ele = &setup_info->twt_element;
+
+	if (RTW_PHL_STATUS_SUCCESS == sts || RTW_PHL_STATUS_CMD_SUCCESS == sts ) {
+		RTW_INFO("[TWT] %s: success\n", __func__);
+		return;
+	}
+
+	rtw_issue_twt_teardown_cmn(padapter, padapter_link, sta->mac_addr,
+				   twt_ele, 3, 500, TWT_NEGO_TYPE_I(twt_ele));
+}
+
+static void _rtw_core_twt_ele_init(struct rtw_phl_twt_element *twt_ele)
+{
+	_rtw_memset(twt_ele, 0x0, sizeof(struct rtw_phl_twt_element));
+}
+
+u8 rtw_core_twt_init(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct he_priv *phepriv = &pmlmepriv->dev_hepriv;
+
+	_rtw_core_twt_ele_init(&phepriv->twt_ele);
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_reset(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	struct he_priv *phepriv = &padapter_link->mlmepriv.hepriv;
+
+	_rtw_core_twt_ele_init(&phepriv->twt_ele);
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_isetup_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			   struct sta_info *psta, struct rtw_phl_twt_element *twt_ele)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_indiv_twt_para_set *isetup = &twt_ele->info.i_twt_para_set;
+	struct rtw_phl_twt_get_twt_i get_twt_i = {0};
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct rtw_phl_twt_setup_info *nego_info = &phl_sta->wrole->twt_setup_info;
+
+	if (isetup->req_type.twt_setup_cmd == RTW_PHL_SUGGEST_TWT) {
+		get_twt_i.wrole = padapter->phl_role;
+		get_twt_i.id = IGNORE_CFG_ID;
+		get_twt_i.offset = 3 * 100;
+		get_twt_i.tsf_h = &isetup->target_wake_t_h;
+		get_twt_i.tsf_l = &isetup->target_wake_t_l;
+		if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_get_target_wake_time(GET_PHL_INFO(dvobj),
+			&get_twt_i, PHL_CMD_DIRECTLY, 0)) {
+			RTW_ERR("[TWT] %s: rtw_phl_twt_get_target_wake_time fail!\n", __func__);
+			goto exit;
+		}
+	} else {
+		isetup->target_wake_t_h = 0;
+		isetup->target_wake_t_l = 0;
+	}
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_get_new_flow_id(GET_PHL_INFO(dvobj),
+					phl_sta, &isetup->req_type.twt_flow_id)) {
+		RTW_ERR("[TWT] %s: rtw_phl_twt_get_new_flow_id fail!\n", __func__);
+		goto exit;
+	}
+
+	_rtw_memcpy(&nego_info->twt_element, twt_ele, sizeof(struct rtw_phl_twt_element));
+	nego_info->dialog_token++;
+
+	RTW_INFO("[TWT] %s: twt_flow_id:%d, dialog_token:%d\n", __func__,
+		 isetup->req_type.twt_flow_id, nego_info->dialog_token);
+
+	rtw_issue_twt_setup(padapter, padapter_link, phl_sta->mac_addr, nego_info);
+
+	return _SUCCESS;
+
+exit:
+	rtw_core_twt_reset(padapter, padapter_link);
+
+	return _FAIL;
+}
+
+u8 rtw_core_twt_iteardown_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      struct sta_info *psta,
+			      struct rtw_phl_twt_sta_teardown_i *twt_trdwn_cfg)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	u8 ret = _SUCCESS;
+
+	if (_SUCCESS != rtw_issue_twt_teardown(padapter, padapter_link, phl_sta->mac_addr,
+		&twt_trdwn_cfg->twt_flow, 3, 500)) {
+		RTW_ERR("[TWT] %s: rtw_issue_twt_teardown fail!\n", __func__);
+		ret = _FAIL;
+	}
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_teardown_for_sta_mode(GET_PHL_INFO(dvobj),
+		twt_trdwn_cfg)) {
+		RTW_ERR("[TWT] %s: rtw_phl_twt_teardown_for_sta_mode fail!\n", __func__);
+		ret = _FAIL;
+	}
+
+	rtw_core_twt_reset(padapter, padapter_link);
+	return ret;
+}
+
+u8 rtw_core_twt_bsetup_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			   struct sta_info *psta, struct rtw_phl_twt_element *twt_ele)
+{
+	struct rtw_phl_bcast_twt_para_set *bsetup = &twt_ele->info.b_twt_para_set[0];
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct rtw_phl_twt_setup_info *nego_info = &phl_sta->wrole->twt_setup_info;
+
+	_rtw_memcpy(&nego_info->twt_element, twt_ele, sizeof(struct rtw_phl_twt_element));
+	nego_info->dialog_token++;
+
+	RTW_INFO("[TWT] %s: btwt_id %d, dialog_token %d\n", __func__,
+		 bsetup->btwt_i.btwt_id, nego_info->dialog_token);
+
+	rtw_issue_twt_setup(padapter, padapter_link, phl_sta->mac_addr, nego_info);
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_bteardown_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      struct sta_info *psta,
+			      struct rtw_phl_twt_sta_teardown_i *twt_trdwn_cfg)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	u8 ret = _SUCCESS;
+
+	if (_SUCCESS != rtw_issue_twt_teardown(padapter, padapter_link, phl_sta->mac_addr,
+		&twt_trdwn_cfg->twt_flow, 3, 500)) {
+		RTW_ERR("[TWT] %s: rtw_issue_twt_teardown fail!\n", __func__);
+		ret = _FAIL;
+	}
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_teardown_for_sta_mode(GET_PHL_INFO(dvobj),
+		twt_trdwn_cfg)) {
+		RTW_ERR("[TWT] %s: rtw_phl_twt_teardown_for_sta_mode fail!\n", __func__);
+		ret = _FAIL;
+	}
+
+	rtw_core_twt_reset(padapter, padapter_link);
+	return ret;
+}
+
+u8 rtw_core_twt_set_info_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			     struct sta_info *psta,
+			     struct rtw_core_twt_set_info *twt_info)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct rtw_phl_twt_info_f_hdr_i info_hdr = {0};
+	struct rtw_phl_twt_info_f *info_f = &info_hdr.info_f;
+	u8 used_map = 0, i = 0, id = 0;
+
+	rtw_phl_twt_get_all_id(GET_PHL_INFO(dvobj), phl_sta, RTW_PHL_INDIV_TWT, &used_map);
+	while ((used_map >> i) > 0) {
+		if ((used_map >> i) & BIT0) {
+			id = i;
+			break;
+		}
+		i++;
+	}
+
+	info_f->twt_flow_id = id;
+
+	if (twt_info->suspend == _FALSE) {
+		struct rtw_phl_twt_get_twt_i get_twt_i = {0};
+		u8 cfg_id = 0;
+		u32 tsf_h = 0, tsf_l = 0;
+
+		if (!rtw_phl_twt_get_cfg_id(GET_PHL_INFO(dvobj), phl_sta,
+			RTW_PHL_INDIV_TWT, info_hdr.info_f.twt_flow_id, &cfg_id)) {
+			RTW_ERR("[TWT] %s: fail to get cfg_id by macid %d and twt_flow_id %d!\n",
+				__func__, phl_sta->macid, info_hdr.info_f.twt_flow_id);
+			return _FAIL;
+		}
+		get_twt_i.offset = twt_info->dur;
+		get_twt_i.wrole = padapter->phl_role;
+		get_twt_i.id = cfg_id;
+		get_twt_i.tsf_h = &tsf_h;
+		get_twt_i.tsf_l = &tsf_l;
+		if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_get_target_wake_time(GET_PHL_INFO(dvobj),
+			&get_twt_i, PHL_CMD_DIRECTLY, 0)) {
+			RTW_ERR("[TWT] %s: rtw_phl_twt_get_target_wake_time fail!\n", __func__);
+			return _FAIL;
+		}
+		info_f->next_twt = tsf_h;
+		info_f->next_twt = (info_f->next_twt << 32);
+		info_f->next_twt |= tsf_l;
+		info_f->next_twt_size = 3;
+	}
+
+	info_hdr.sta = phl_sta;
+
+	RTW_INFO("[TWT] %s: twt_flow_id %d, rsp_req %d, next_twt_req %d, next_twt_size %d, all_twt %d, next_twt 0x%08X %08X\n",
+		__func__, info_f->twt_flow_id, info_f->rsp_req, info_f->next_twt_req,
+		info_f->next_twt_size, info_f->all_twt, (u32)(info_f->next_twt >> 32),
+		(u32)(info_f->next_twt));
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_info_f_hrl(GET_PHL_INFO(dvobj), &info_hdr)) {
+		RTW_ERR("[TWT] %s: rtw_phl_twt_info_f_hrl fail!\n", __func__);
+		return _FAIL;
+	}
+
+	return rtw_issue_twt_info(padapter, padapter_link, phl_sta->mac_addr, info_f);
+}
+
+u8 rtw_core_twt_on_isetup_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			     struct sta_info *psta, struct rtw_phl_twt_setup_info *twt_setup_info)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct rtw_phl_twt_element *twt_ele = &twt_setup_info->twt_element;
+	struct rtw_phl_twt_setup_info *nego_info = &phl_sta->wrole->twt_setup_info;
+	struct rtw_phl_twt_sta_accept_i acpt_i = {0};
+
+	if (TWT_REQER_I(twt_ele)) {
+		RTW_ERR("[TWT] %s: not support to be TWT responder in STA mode!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	if (!twt_setup_info->dialog_token) {
+		RTW_ERR("[TWT] %s: invalid dialog token 0!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	if ((twt_setup_info->dialog_token != nego_info->dialog_token)) {
+		RTW_ERR("[TWT] %s: dialog token mismatch (tx %d, rx %d)!\n", __func__,
+			nego_info->dialog_token, twt_setup_info->dialog_token);
+		goto twt_setup_fail;
+	}
+
+	if (TWT_SETUP_CMD_I(twt_ele) != RTW_PHL_ACCEPT_TWT) {
+		RTW_ERR("[TWT] %s: can't support setup cmd %d currently!\n", __func__,
+			TWT_SETUP_CMD_I(twt_ele));
+		goto twt_setup_fail;
+	}
+
+	_rtw_memcpy(&acpt_i.setup_info, twt_setup_info, sizeof(struct rtw_phl_twt_setup_info));
+	acpt_i.sta = phl_sta;
+	acpt_i.accept_done = _twt_accept_done;
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_accept_for_sta_mode(GET_PHL_INFO(dvobj),
+		&acpt_i)) {
+		RTW_ERR("[TWT] %s: rtw_phl_twt_accept_for_sta_mode fail!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	return _SUCCESS;
+
+twt_setup_fail:
+	rtw_issue_twt_teardown_cmn(padapter, padapter_link, phl_sta->mac_addr,
+				   twt_ele, 3, 500, TWT_NEGO_TYPE_I(twt_ele));
+
+	return _FAIL;
+}
+
+u8 rtw_core_twt_on_bsetup_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			     struct sta_info *psta, struct rtw_phl_twt_setup_info *twt_setup_info)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct rtw_phl_twt_element *twt_ele = &twt_setup_info->twt_element;
+	struct rtw_phl_twt_setup_info *nego_info = &phl_sta->wrole->twt_setup_info;
+	struct rtw_phl_bcast_twt_para_set *para_src = NULL, *para_dst = NULL;
+	struct rtw_phl_twt_sta_accept_i acpt_i = {0};
+	u8 idx = 0, acpt_num = 0;
+
+	if (!twt_setup_info->dialog_token) {
+		RTW_ERR("[TWT] %s: invalid dialog token 0!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	if ((twt_setup_info->dialog_token != nego_info->dialog_token)) {
+		RTW_ERR("[TWT] %s: dialog token mismatch (tx %d, rx %d)!\n", __func__,
+			nego_info->dialog_token, twt_setup_info->dialog_token);
+		goto twt_setup_fail;
+	}
+
+	for (idx = 0; idx < twt_ele->num_btwt_para; idx++) {
+		if (TWT_REQER_B(twt_ele, idx))
+			continue;
+
+		if (TWT_SETUP_CMD_B(twt_ele, idx) != RTW_PHL_ACCEPT_TWT)
+			continue;
+
+		para_src = &twt_ele->info.b_twt_para_set[idx];
+		para_dst = &acpt_i.setup_info.twt_element.info.b_twt_para_set[acpt_num];
+		_rtw_memcpy(para_dst, para_src, sizeof(struct rtw_phl_bcast_twt_para_set));
+		acpt_num++;
+	}
+
+	if (!acpt_num) {
+		RTW_ERR("[TWT] %s: no btwt para set is accepted!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	_rtw_memcpy(&acpt_i.setup_info.twt_element.twt_ctrl, &twt_ele->twt_ctrl,
+		    sizeof(struct rtw_phl_twt_control));
+	acpt_i.setup_info.dialog_token = twt_setup_info->dialog_token;
+	acpt_i.sta = phl_sta;
+	acpt_i.accept_done = _twt_accept_done;
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_accept_for_sta_mode(GET_PHL_INFO(dvobj),
+		&acpt_i)) {
+		RTW_ERR("[TWT] %s: rtw_phl_twt_accept_for_sta_mode fail!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	return _SUCCESS;
+
+twt_setup_fail:
+	rtw_issue_twt_teardown_cmn(padapter, padapter_link, phl_sta->mac_addr,
+				   twt_ele, 3, 500, TWT_NEGO_TYPE_I(twt_ele));
+
+	return _FAIL;
+}
+
+u8 rtw_core_twt_on_teardown_hdl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			     struct sta_info *psta, struct rtw_phl_twt_flow_field *twt_flow_info)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_twt_sta_teardown_i teardown_cfg = {0};
+
+	teardown_cfg.sta = psta->phl_sta;
+	_rtw_memcpy(&teardown_cfg.twt_flow, &twt_flow_info, sizeof(struct rtw_phl_twt_flow_field));
+	teardown_cfg.teardown_done = _twt_teardown_done;
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_teardown_for_sta_mode(GET_PHL_INFO(dvobj),
+		&teardown_cfg)) {
+		RTW_ERR("[TWT] %s: rtw_phl_twt_teardown_for_sta_mode fail!\n", __func__);
+		return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_hdl(_adapter *padapter, u8 *pbuf)
+{
+	struct rtw_core_twt_cmd_obj *ptwtcmd = NULL;
+	u8 ret = _FAIL;
+
+	if (!pbuf)
+		return _FALSE;
+
+	ptwtcmd = (struct rtw_core_twt_cmd_obj*)pbuf;
+
+	switch (ptwtcmd->cmd_id) {
+	case CORE_TWT_CMD_ISETUP:
+		ret = rtw_core_twt_isetup_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_phl_twt_element*)ptwtcmd->pparm);
+		break;
+	case CORE_TWT_CMD_BSETUP:
+		ret = rtw_core_twt_bsetup_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_phl_twt_element*)ptwtcmd->pparm);
+		break;
+	case CORE_TWT_CMD_ITEARDOWN:
+		ret = rtw_core_twt_iteardown_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_phl_twt_sta_teardown_i*)ptwtcmd->pparm);
+		break;
+	case CORE_TWT_CMD_BTEARDOWN:
+		ret = rtw_core_twt_bteardown_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_phl_twt_sta_teardown_i*)ptwtcmd->pparm);
+		break;
+	case CORE_TWT_CMD_SET_INFO:
+		ret = rtw_core_twt_set_info_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_core_twt_set_info*)ptwtcmd->pparm);
+		break;
+	case CORE_TWT_CMD_ON_ISETUP:
+		ret = rtw_core_twt_on_isetup_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_phl_twt_setup_info*)ptwtcmd->pparm);
+		break;
+	case CORE_TWT_CMD_ON_BSETUP:
+		ret = rtw_core_twt_on_bsetup_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_phl_twt_setup_info*)ptwtcmd->pparm);
+		break;
+	case CORE_TWT_CMD_ON_TEARDOWN:
+		ret = rtw_core_twt_on_teardown_hdl(ptwtcmd->padapter, ptwtcmd->padapter_link,
+			ptwtcmd->psta, (struct rtw_phl_twt_flow_field*)ptwtcmd->pparm);
+		break;
+	}
+
+	if (ptwtcmd->pparm_need_free == _TRUE)
+		rtw_mfree(ptwtcmd->pparm, ptwtcmd->parm_sz);
+
+	return ret;
+}
+
+u8 rtw_core_twt_cmd(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		    struct sta_info *psta, enum rtw_core_twt_cmd_id cmd_id,
+		    void *pparm, u32 pparm_sz, bool parm_need_free)
+{
+	struct	cmd_obj	*pcmdobj;
+	struct	rtw_core_twt_cmd_obj *ptwtcmd;
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct	cmd_priv *pcmdpriv = &adapter_to_dvobj(padapter)->cmdpriv;
+	u8 res = _SUCCESS;
+
+	pcmdobj = (struct cmd_obj *)rtw_zmalloc(sizeof(struct cmd_obj));
+	if (pcmdobj == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+	pcmdobj->padapter = padapter;
+
+	ptwtcmd = (struct rtw_core_twt_cmd_obj*)rtw_zmalloc(sizeof(struct rtw_core_twt_cmd_obj));
+	if (ptwtcmd == NULL) {
+		rtw_mfree((u8 *)pcmdobj, sizeof(struct cmd_obj));
+		res = _FAIL;
+		goto exit;
+	}
+	_rtw_memset(ptwtcmd, 0x0, sizeof(struct	rtw_core_twt_cmd_obj));
+
+	ptwtcmd->padapter = padapter;
+	ptwtcmd->padapter_link = padapter_link;
+	ptwtcmd->psta = psta;
+	ptwtcmd->cmd_id = cmd_id;
+	ptwtcmd->pparm = pparm;
+	ptwtcmd->parm_sz = pparm_sz;
+	ptwtcmd->pparm_need_free = parm_need_free;
+
+	init_h2fwcmd_w_parm_no_rsp(pcmdobj, ptwtcmd, CMD_TWT);
+	res = rtw_enqueue_cmd(pcmdpriv, pcmdobj);
+
+exit:
+	return res;
+}
+
+u8 rtw_core_twt_set_ctrl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 u32 ndp_pag_ind, u32 rsp_pm_mode, u32 nego_type,
+			 u32 twt_info_dis, u32 wk_dur_unit)
+{
+	struct rtw_phl_twt_element *twt_ele = &padapter_link->mlmepriv.hepriv.twt_ele;
+
+	twt_ele->twt_ctrl.ndp_paging_indic = ndp_pag_ind;
+	twt_ele->twt_ctrl.responder_pm_mode = rsp_pm_mode;
+	twt_ele->twt_ctrl.nego_type = nego_type;
+	twt_ele->twt_ctrl.twt_info_frame_disable = twt_info_dis;
+	twt_ele->twt_ctrl.wake_dur_unit = wk_dur_unit;
+
+	RTW_INFO("[TWT] %s: ndp_pag_ind %d, rsp_pm_md %d, ng_type %d, twt_info_dis %d, wk_dur_unt %d\n",
+		 __func__, ndp_pag_ind, rsp_pm_mode, nego_type, twt_info_dis, wk_dur_unit);
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_set_ireq(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 u32 request, u32 setup_cmd, u32 trigger, u32 implicit,
+			 u32 flow_type, u32 wk_intvl_exp, u32 protection)
+{
+	struct rtw_phl_twt_element *twt_ele = &padapter_link->mlmepriv.hepriv.twt_ele;
+	struct rtw_phl_req_type_indiv *ireq = &twt_ele->info.i_twt_para_set.req_type;
+
+	ireq->twt_request = request;
+	ireq->twt_setup_cmd = setup_cmd;
+	ireq->trigger = trigger;
+	ireq->implicit = implicit;
+	ireq->flow_type = flow_type;
+	ireq->twt_wake_int_exp = wk_intvl_exp;
+	ireq->twt_protection = protection;
+
+	RTW_INFO("[TWT] %s: req %d, stp_cmd %d, trig %d, implct %d, flow_type %d, wk_intvl_exp %d protct %d\n",
+		 __func__, request, setup_cmd, trigger, implicit, flow_type, wk_intvl_exp,
+		 protection);
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_set_isetup(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			   struct sta_info *psta, u32 target_wk_time,
+			   u32 nom_min_wk_dur, u32 wk_int_mantissa, u32 twt_chnl)
+{
+	struct rtw_phl_twt_element *twt_ele = &padapter_link->mlmepriv.hepriv.twt_ele;
+	struct rtw_phl_indiv_twt_para_set *isetup = &twt_ele->info.i_twt_para_set;
+
+	isetup->nom_min_twt_wake_dur = nom_min_wk_dur;
+	isetup->twt_wake_int_mantissa = wk_int_mantissa;
+	isetup->twt_channel = twt_chnl;
+
+	RTW_INFO("[TWT] %s: trgt_wk_t %d, nom_min_twt_wk_dur %d, twt_wk_int_mantissa %d, twt_chnl %d\n",
+		 __func__, target_wk_time, nom_min_wk_dur, wk_int_mantissa, twt_chnl);
+
+	return rtw_core_twt_cmd(padapter, padapter_link, psta, CORE_TWT_CMD_ISETUP,
+				(void*)twt_ele, sizeof(struct rtw_phl_twt_element),
+				_FALSE);
+}
+
+u8 rtw_core_twt_set_iteardown(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      struct sta_info *psta, u32 nego_type,
+			      u32 flow_id, u32 trdwn_all)
+{
+	struct rtw_phl_twt_sta_teardown_i *twt_trdwn_cfg = NULL;
+
+	twt_trdwn_cfg = (struct rtw_phl_twt_sta_teardown_i*)rtw_zmalloc(
+		sizeof(struct rtw_phl_twt_sta_teardown_i));
+	if (twt_trdwn_cfg == NULL)
+		return _FAIL;
+
+	twt_trdwn_cfg->twt_flow.nego_type = nego_type;
+	twt_trdwn_cfg->twt_flow.info.twt_flow01.twt_flow_id = flow_id;
+	twt_trdwn_cfg->twt_flow.info.twt_flow01.teardown_all = trdwn_all;
+	twt_trdwn_cfg->sta = psta->phl_sta;
+	twt_trdwn_cfg->teardown_done = _twt_teardown_done;
+
+	RTW_INFO("[TWT] %s: nego_type %d, flow_id %d, trdwn_all %d\n", __func__,
+		 nego_type, flow_id, trdwn_all);
+
+	return rtw_core_twt_cmd(padapter, padapter_link, psta, CORE_TWT_CMD_ITEARDOWN,
+				(void*)twt_trdwn_cfg, sizeof(struct rtw_phl_twt_sta_teardown_i),
+				_TRUE);
+}
+
+u8 rtw_core_twt_set_breq(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 u32 request, u32 setup_cmd, u32 trigger, u32 lst_bcparm_set,
+			 u32 flow_type, u32 btwt_rcmd, u32 wk_intvl_exp)
+{
+	struct rtw_phl_twt_element *twt_ele = &padapter_link->mlmepriv.hepriv.twt_ele;
+	struct rtw_phl_btwt_req_type *breq = &twt_ele->info.b_twt_para_set[0].req_type;
+
+	breq->twt_request = request;
+	breq->twt_setup_cmd = setup_cmd;
+	breq->trigger = trigger;
+	breq->lst_bc_para_set = lst_bcparm_set;
+	breq->flow_type = flow_type;
+	breq->btwt_rcmd = btwt_rcmd;
+	breq->twt_wake_int_exp = wk_intvl_exp;
+
+	RTW_INFO("[TWT] %s: req %d, stp_cmd %d, trig %d, lst_bcparm_set %d, flow_type %d, btwt_rcmd %d, wk_intvl_exp %d\n",
+		 __func__, request, setup_cmd, trigger, lst_bcparm_set, flow_type, btwt_rcmd, wk_intvl_exp);
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_set_bsetup(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			   struct sta_info *psta, u32 target_wk_time,
+			   u32 nom_min_wk_dur, u32 wk_int_mantissa, u32 btwt_id,
+			   u32 btwt_prstnc)
+{
+	struct rtw_phl_twt_element *twt_ele = &padapter_link->mlmepriv.hepriv.twt_ele;
+	struct rtw_phl_bcast_twt_para_set *bsetup = &twt_ele->info.b_twt_para_set[0];
+
+	twt_ele->num_btwt_para = 1;
+	bsetup->target_wake_t_l = target_wk_time;
+	bsetup->nom_min_twt_wake_dur = nom_min_wk_dur;
+	bsetup->twt_wake_int_mantissa = wk_int_mantissa;
+	bsetup->btwt_i.btwt_id = btwt_id;
+	bsetup->btwt_i.btwt_prstnc = btwt_prstnc;
+
+	RTW_INFO("[TWT] %s: trgt_wk_t %d, nom_min_twt_wk_dur %d, twt_wk_int_mantissa %d, btwt_id %d btwt_prstnc %d\n",
+		 __func__, target_wk_time, nom_min_wk_dur, wk_int_mantissa, btwt_id, btwt_prstnc);
+
+	return rtw_core_twt_cmd(padapter, padapter_link, psta, CORE_TWT_CMD_BSETUP,
+				(void*)twt_ele, sizeof(struct rtw_phl_twt_element),
+				_FALSE);
+}
+
+u8 rtw_core_twt_set_bteardown(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      struct sta_info *psta, u32 nego_type,
+			      u32 btwt_id, u32 trdwn_all)
+{
+	struct rtw_phl_twt_sta_teardown_i *twt_trdwn_cfg = NULL;
+
+	twt_trdwn_cfg = (struct rtw_phl_twt_sta_teardown_i*)rtw_zmalloc(
+		sizeof(struct rtw_phl_twt_sta_teardown_i));
+	if (twt_trdwn_cfg == NULL)
+		return _FAIL;
+
+	twt_trdwn_cfg->twt_flow.nego_type = nego_type;
+	twt_trdwn_cfg->twt_flow.info.twt_flow3.bcast_twt_id = btwt_id;
+	twt_trdwn_cfg->twt_flow.info.twt_flow3.teardown_all = trdwn_all;
+	twt_trdwn_cfg->sta = psta->phl_sta;
+	twt_trdwn_cfg->teardown_done = _twt_teardown_done;
+
+	RTW_INFO("[TWT] %s: nego_type %d, btwt_id %d, trdwn_all %d\n", __func__,
+		 nego_type, btwt_id, trdwn_all);
+
+	return rtw_core_twt_cmd(padapter, padapter_link, psta, CORE_TWT_CMD_BTEARDOWN,
+				(void*)twt_trdwn_cfg, sizeof(struct rtw_phl_twt_sta_teardown_i),
+				_TRUE);
+}
+
+u8 rtw_core_twt_set_info(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 struct sta_info *psta, bool suspend, u32 dur)
+{
+	struct rtw_core_twt_set_info *twt_info = NULL;
+
+	twt_info = (struct rtw_core_twt_set_info*)rtw_zmalloc(
+		sizeof(struct rtw_core_twt_set_info));
+	if (twt_info == NULL)
+		return _FAIL;
+
+	twt_info->suspend = suspend;
+	twt_info->dur = dur;
+
+	RTW_INFO("[TWT] %s: suspend %d, dur %d\n", __func__, suspend, dur);
+
+	return rtw_core_twt_cmd(padapter, padapter_link, psta, CORE_TWT_CMD_SET_INFO,
+				(void*)twt_info, sizeof(struct rtw_core_twt_set_info),
+				_TRUE);
+}
+
+u8 rtw_issue_twt_setup(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		       u8 *da, struct rtw_phl_twt_setup_info *twt_setup_info)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	u8					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	u8					*mac, *bssid;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*cur_network = &(pmlmeinfo->network);
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	u8 len = 0;
+	u8 category = RTW_WLAN_CATEGORY_UNP_S1G;
+	u8 action = RTW_WLAN_ACTION_UNP_S1G_TWT_SETUP;
+	u8 twt_element_id = WLAN_EID_TWT;
+
+	if (da == NULL)
+		return _FAIL;
+
+	if (alink_regu_block_tx(padapter_link))
+		return _FAIL;
+
+	RTW_INFO("[TWT] %s: issue TWT Setup action frame to "MAC_FMT"\n",
+	      __func__, MAC_ARG(da));
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_INFO("[TWT] %s: alloc_mgtxmitframe fail\n", __func__);
+		return _FAIL;
+	}
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	mac = adapter_mac_addr(padapter);
+	bssid = cur_network->MacAddress;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, bssid, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, padapter->mlmeextpriv.mgnt_seq);
+	padapter->mlmeextpriv.mgnt_seq++;
+	set_frame_sub_type(fctrl, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(twt_setup_info->dialog_token), &(pattrib->pktlen));
+
+	/* fill TWT element id */
+	pframe = rtw_set_fixed_ie(pframe, 1, &(twt_element_id), &(pattrib->pktlen));
+
+	/* fill TWT element body */
+	status = rtw_phl_twt_fill_twt_element(&twt_setup_info->twt_element, pframe + 1, &len);
+
+	/* fill TWT element length */
+	*pframe = len;
+
+	pframe += len + 1;
+	pattrib->pktlen += len + 1;
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+static u8 _issue_twt_teardown(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      u8 *da, struct rtw_phl_twt_flow_field *twt_flow_info,
+			      int wait_ms)
+{
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	u8					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	u8					*mac, *bssid;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*cur_network = &(pmlmeinfo->network);
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	u16 len = 0;
+	u8 category = RTW_WLAN_CATEGORY_UNP_S1G;
+	u8 action = RTW_WLAN_ACTION_UNP_S1G_TWT_TEARDOWN;
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	u8 ret = _FAIL;
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_INFO("[TWT] %s - alloc_mgtxmitframe fail\n", __func__);
+		return _FAIL;
+	}
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	mac = adapter_mac_addr(padapter);
+	bssid = cur_network->MacAddress;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, bssid, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, padapter->mlmeextpriv.mgnt_seq);
+	padapter->mlmeextpriv.mgnt_seq++;
+	set_frame_sub_type(fctrl, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	/* fill TWT flow body */
+	status = rtw_phl_twt_fill_flow_field(twt_flow_info, pframe, &len);
+
+	pframe += len;
+	pattrib->pktlen += len;
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	if (wait_ms) {
+		ret = dump_mgntframe_and_wait_ack_timeout(padapter, pmgntframe, wait_ms);
+	} else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+	return ret;
+}
+
+u8 rtw_issue_twt_teardown(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			  u8 *da, struct rtw_phl_twt_flow_field *twt_flow_info,
+			  int try_cnt, int wait_ms)
+{
+	systime start = rtw_get_current_time();
+	int i = 0;
+	u8 ret = _FAIL;
+
+	if (da == NULL)
+		return _FAIL;
+
+	if (alink_regu_block_tx(padapter_link))
+		return _FAIL;
+
+	RTW_INFO("[TWT] %s: issue TWT teardown frame to "MAC_FMT"\n",
+		 __func__, MAC_ARG(da));
+
+	do {
+		ret = _issue_twt_teardown(padapter, padapter_link, da, twt_flow_info,
+					  wait_ms);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(10);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#if 0
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		RTW_INFO("[TWT] "FUNC_ADPT_FMT" to "MAC_FMT", %s, %d/%d in %u ms\n",
+			 FUNC_ADPT_ARG(padapter), MAC_ARG(da), ret == _SUCCESS ?
+			 ", acked" : ", not acked", i, try_cnt,
+			 rtw_get_passing_time_ms(start));
+	}
+exit:
+	return ret;
+}
+
+u8 rtw_issue_twt_teardown_cmn(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      u8 *da, struct rtw_phl_twt_element *twt_ele,
+			      int try_cnt, int wait_ms, bool individual)
+{
+	struct rtw_phl_twt_flow_field trdwn_info = {0};
+
+	trdwn_info.nego_type = twt_ele->twt_ctrl.nego_type;
+
+	if (individual) {
+		trdwn_info.info.twt_flow01.twt_flow_id =
+			twt_ele->info.i_twt_para_set.req_type.twt_flow_id;
+	} else {
+		trdwn_info.info.twt_flow3.bcast_twt_id = 0;
+		trdwn_info.info.twt_flow3.teardown_all = 1;
+	}
+
+	return rtw_issue_twt_teardown(padapter, padapter_link, da, &trdwn_info,
+				      try_cnt, wait_ms);
+}
+
+u8 rtw_issue_twt_info(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		       u8 *da, struct rtw_phl_twt_info_f *twt_info)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	struct xmit_frame			*pmgntframe;
+	struct pkt_attrib			*pattrib;
+	u8					*pframe;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	unsigned short				*fctrl;
+	u8					*mac, *bssid;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*cur_network = &(pmlmeinfo->network);
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	u16 len = 0;
+	u8 category = RTW_WLAN_CATEGORY_UNP_S1G;
+	u8 action = RTW_WLAN_ACTION_UNP_S1G_TWT_INFO;
+
+	if (da == NULL)
+		return _FAIL;
+
+	if (alink_regu_block_tx(padapter_link))
+		return _FAIL;
+
+	RTW_INFO("[TWT] %s: issue TWT Info action frame to "MAC_FMT"\n",
+	      __func__, MAC_ARG(da));
+
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		RTW_INFO("[TWT] %s: alloc_mgtxmitframe fail\n", __func__);
+		return _FAIL;
+	}
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	mac = adapter_mac_addr(padapter);
+	bssid = cur_network->MacAddress;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+	_rtw_memcpy(pwlanhdr->addr1, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, mac, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, bssid, ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, padapter->mlmeextpriv.mgnt_seq);
+	padapter->mlmeextpriv.mgnt_seq++;
+	set_frame_sub_type(fctrl, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	/* fill TWT Info body */
+	status = rtw_phl_twt_fill_info_field(twt_info, pframe, &len);
+
+	pframe += len;
+	pattrib->pktlen += len;
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+
+	return _SUCCESS;
+}
+
+static u8 _twt_on_setup_sta(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			    u8 *raddr, u8 *ptwt_ie, u32 frame_len)
+{
+	struct sta_info *psta;
+	struct rtw_phl_twt_setup_info *twt_setup_info = NULL;
+	struct rtw_phl_twt_element *twt_ele = NULL;
+	u16 len = frame_len;
+
+	RTW_INFO("[TWT] %s: Recv TWT setup frame: mac_addr = " MAC_FMT ", diag_token = %d, len = %d\n",
+		 __func__, MAC_ARG(raddr), ptwt_ie[2], len);
+
+	psta = rtw_get_stainfo(&padapter->stapriv, raddr);
+	if (psta == NULL || psta->phl_sta == NULL) {
+		RTW_ERR("[TWT] %s: STA not exist!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	twt_setup_info = (struct rtw_phl_twt_setup_info*)rtw_zmalloc(
+		sizeof(struct rtw_phl_twt_setup_info));
+	if (twt_setup_info == NULL) {
+		RTW_ERR("[TWT] %s: malloc fail!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	twt_ele = &twt_setup_info->twt_element;
+
+	/* parse TWT setup frame content */
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_parse_setup_info(ptwt_ie, len,
+		twt_setup_info)) {
+		RTW_ERR("[TWT] %s: rtw_phl_parse_twt_setup_info fail!\n", __func__);
+		goto twt_setup_fail;
+	}
+
+	/* handle itwt case and btwt case seperately */
+	if (_SUCCESS != rtw_core_twt_cmd(padapter, padapter_link, psta,
+			(TWT_NEGO_TYPE_I(twt_ele)) ? CORE_TWT_CMD_ON_ISETUP :
+			CORE_TWT_CMD_ON_BSETUP, (void*)twt_setup_info,
+			sizeof(struct rtw_phl_twt_setup_info), _TRUE)) {
+			RTW_ERR("[TWT] %s: rtw_core_twt_cmd fail!\n", __func__);
+			goto twt_setup_fail;
+		}
+
+	return _SUCCESS;
+
+twt_setup_fail:
+	rtw_issue_twt_teardown_cmn(padapter, padapter_link, psta->phl_sta->mac_addr,
+				   twt_ele, 0, 0, TWT_NEGO_TYPE_I(twt_ele));
+
+	return _FAIL;
+}
+
+static u8 _twt_on_teardown_sta(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			       u8 *raddr, u8 *pframe, u32 frame_len)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_twt_flow_field *twt_flow_info = NULL;
+	struct sta_info *psta;
+	u16 len = frame_len;
+
+	RTW_INFO("[TWT] %s: Recv TWT flow frame, mac_addr = " MAC_FMT "\n",
+		 __func__, MAC_ARG(raddr));
+
+	psta = rtw_get_stainfo(&padapter->stapriv, raddr);
+	if (psta == NULL || psta->phl_sta == NULL) {
+		RTW_INFO("[TWT] %s: STA not exist!\n", __func__);
+		return _FAIL;
+	}
+
+	twt_flow_info = (struct rtw_phl_twt_flow_field*)rtw_zmalloc(
+		sizeof(struct rtw_phl_twt_flow_field));
+	if (twt_flow_info == NULL) {
+		RTW_ERR("[TWT] %s: malloc fail!\n", __func__);
+		return _FAIL;
+	}
+
+	if (RTW_PHL_STATUS_SUCCESS != rtw_phl_twt_parse_flow_field(pframe + 2, len,
+		twt_flow_info)) {
+		RTW_ERR("[TWT] %s: rtw_phl_parse_twt_flow_field fail!\n", __func__);
+		return _FAIL;
+	}
+
+	if (_SUCCESS != rtw_core_twt_cmd(padapter, padapter_link, psta,
+			CORE_TWT_CMD_ON_TEARDOWN, (void*)twt_flow_info,
+			sizeof(struct rtw_phl_twt_flow_field), _TRUE)) {
+		RTW_ERR("[TWT] %s: rtw_core_twt_cmd fail!\n", __func__);
+			return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+u8 rtw_core_twt_on_action(_adapter *padapter, u8 *raddr, u8 *pframe_body, u32 frame_len)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	u8 ret = _FAIL;
+	u8 action_code = 0;
+
+	action_code = pframe_body[1];
+
+	/* check dev cap support for TWT */
+	if ((!TWT_REQ_SUPP(padapter_link->wrlink->protocol_cap.twt)) &&
+	    (!TWT_RSP_SUPP(padapter_link->wrlink->protocol_cap.twt))) {
+		RTW_INFO("[TWT] %s: Dev cap does not support TWT\n", __func__);
+		return _SUCCESS;
+	}
+
+	/* only handle twt action frame if in wifi test mode or TWT is enabled currently */
+	if (!padapter->registrypriv.wifi_spec && !padapter->registrypriv.twt_en) {
+		RTW_INFO("[TWT] %s: Only handle action code %d in wifi test mode\n",
+			 __func__, action_code);
+		return _SUCCESS;
+	}
+
+	/* limit to use in STA mode associated to 11AX AP currently */
+	if (!MLME_IS_STA(padapter)) {
+		RTW_INFO("[TWT] %s: Only support to use in STA mode\n", __func__);
+		return _SUCCESS;
+	}
+	if (!MLME_IS_ASOC(padapter)) {
+		RTW_INFO("[TWT] %s: Only support to use when assoc. to AP\n", __func__);
+		return _SUCCESS;
+	}
+	if (padapter_link->mlmepriv.hepriv.he_option == _FALSE) {
+		RTW_INFO("[TWT] %s: Only support to use in 802.11AX mode\n", __func__);
+		return _SUCCESS;
+	}
+
+	switch (action_code) {
+		case RTW_WLAN_ACTION_UNP_S1G_TWT_SETUP:
+			ret = _twt_on_setup_sta(padapter, padapter_link, raddr,
+						pframe_body, frame_len);
+			break;
+		case RTW_WLAN_ACTION_UNP_S1G_TWT_TEARDOWN:
+			ret = _twt_on_teardown_sta(padapter, padapter_link, raddr,
+						   pframe_body, frame_len);
+			break;
+		case RTW_WLAN_ACTION_UNP_S1G_TWT_INFO:
+			RTW_INFO("[TWT] %s: Not support TWT information action frame\n", __func__);
+			ret = _SUCCESS;
+			break;
+		default:
+			break;
+	}
+
+	return ret;
+}
+
+#endif /* CONFIG_TWT */
+#endif /* CONFIG_80211AX_HE */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_txpwr.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_txpwr.c
new file mode 100644
index 000000000000..b199e709d227
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_txpwr.c
@@ -0,0 +1,163 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_TXPWR_C_
+
+#include <drv_types.h>
+
+static const s16 _mb_of_ntx[] = {
+	0,		/* 1TX */
+	301,	/* 2TX */
+	477,	/* 3TX */
+	602,	/* 4TX */
+	699,	/* 5TX */
+	778,	/* 6TX */
+	845,	/* 7TX */
+	903,	/* 8TX */
+};
+
+/* get mB(100 *dB) for specifc TX count relative to 1TX */
+s16 mb_of_ntx(u8 ntx)
+{
+	if (ntx == 0 || ntx > 8) {
+		RTW_ERR("ntx=%u, out of range\n", ntx);
+		rtw_warn_on(1);
+		return 0;
+	}
+
+	return _mb_of_ntx[ntx - 1];
+}
+
+/*
+* input with txpwr value in unit of mBm
+* return txpwr in unit of TX Gain Index
+*/
+s8 txpwr_mbm_to_txgi_s8_with_max(s16 mbm, u8 txgi_max, u8 txgi_pdbm)
+{
+	s16 max_mbm = (txgi_max > S8_MAX ? S8_MAX : txgi_max) * MBM_PDBM / txgi_pdbm;
+	s16 min_mbm = S8_MIN * MBM_PDBM  / txgi_pdbm;
+
+	mbm = rtw_min(max_mbm, mbm);
+	mbm = rtw_max(min_mbm, mbm);
+
+	return mbm * txgi_pdbm / MBM_PDBM;
+}
+
+/*
+* input with txpwr value in unit of txpwr index
+* return string in length 6 at least (for -xx.xx)
+*/
+void txpwr_idx_get_dbm_str(s8 idx, u8 txgi_max, s8 txgi_ww, u8 txgi_pdbm, SIZE_T cwidth, char dbm_str[], u8 dbm_str_len)
+{
+	char fmt[16];
+
+	if (idx == txgi_max) {
+		snprintf(fmt, 16, "%%%zus", cwidth >= 6 ? cwidth + 1 : 6);
+		snprintf(dbm_str, dbm_str_len, fmt, "NA");
+	} else if (idx == txgi_ww) {
+		snprintf(fmt, 16, "%%%zus", cwidth >= 6 ? cwidth + 1 : 6);
+		snprintf(dbm_str, dbm_str_len, fmt, "WW");
+	} else if (idx > -txgi_pdbm && idx < 0) { /* -0.xx */
+		snprintf(fmt, 16, "%%%zus-0.%%02d", cwidth >= 6 ? cwidth - 4 : 1);
+		snprintf(dbm_str, dbm_str_len, fmt, "", (rtw_abs(idx) % txgi_pdbm) * 100 / txgi_pdbm);
+	} else if (idx % txgi_pdbm) { /* d.xx */
+		snprintf(fmt, 16, "%%%zud.%%02d", cwidth >= 6 ? cwidth - 2 : 3);
+		snprintf(dbm_str, dbm_str_len, fmt, idx / txgi_pdbm, (rtw_abs(idx) % txgi_pdbm) * 100 / txgi_pdbm);
+	} else { /* d */
+		snprintf(fmt, 16, "%%%zud", cwidth >= 6 ? cwidth + 1 : 6);
+		snprintf(dbm_str, dbm_str_len, fmt, idx / txgi_pdbm);
+	}
+}
+
+/*
+* input with txpwr value in unit of mbm
+* return string in length 6 at least (for -xx.xx)
+*/
+void txpwr_mbm_get_dbm_str(s16 mbm, SIZE_T cwidth, char dbm_str[], u8 dbm_str_len)
+{
+	char fmt[16];
+
+	if (mbm == UNSPECIFIED_MBM) {
+		snprintf(fmt, 16, "%%%zus", cwidth >= 6 ? cwidth + 1 : 6);
+		snprintf(dbm_str, dbm_str_len, fmt, "NA");
+	} else if (mbm > -MBM_PDBM && mbm < 0) { /* -0.xx */
+		snprintf(fmt, 16, "%%%zus-0.%%02d", cwidth >= 6 ? cwidth - 4 : 1);
+		snprintf(dbm_str, dbm_str_len, fmt, "", (rtw_abs(mbm) % MBM_PDBM) * 100 / MBM_PDBM);
+	} else if (mbm % MBM_PDBM) { /* d.xx */
+		snprintf(fmt, 16, "%%%zud.%%02d", cwidth >= 6 ? cwidth - 2 : 3);
+		snprintf(dbm_str, dbm_str_len, fmt, mbm / MBM_PDBM, (rtw_abs(mbm) % MBM_PDBM) * 100 / MBM_PDBM);
+	} else { /* d */
+		snprintf(fmt, 16, "%%%zud", cwidth >= 6 ? cwidth + 1 : 6);
+		snprintf(dbm_str, dbm_str_len, fmt, mbm / MBM_PDBM);
+	}
+}
+
+void rtw_update_txpwr_level(struct dvobj_priv *dvobj, enum phl_band_idx band_idx)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+
+	rtw_txpwr_hal_update_pwr(dvobj, band_idx);
+
+	rtw_rfctl_update_op_mode(rfctl, 0, 0, 0);
+}
+
+void rtw_update_txpwr_level_all_hwband(struct dvobj_priv *dvobj)
+{
+	rtw_update_txpwr_level(dvobj, HW_BAND_MAX);
+}
+
+void dump_tx_power_ext_info(void *sel, struct dvobj_priv *dvobj)
+{
+	struct tx_power_ext_info info;
+	struct {
+		const char *str;
+		struct txpwr_param_status *status;
+	} params[] = {
+		{"tx_power_by_rate", &info.by_rate},
+		{"tx_power_limit", &info.lmt},
+		#ifdef CONFIG_80211AX_HE
+		{"tx_power_limit_ru", &info.lmt_ru},
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		{"tx_power_limit_6g", &info.lmt_6g},
+		{"tx_power_limit_ru_6g", &info.lmt_ru_6g},
+		#endif
+	};
+	u8 num_of_param = sizeof(params) / sizeof(params[0]);
+	u8 i;
+
+	if (rtw_txpwr_hal_get_ext_info(dvobj, &info)) {
+		for (i = 0; i < num_of_param; i++) {
+			RTW_PRINT_SEL(sel, "%s: %s, %s, %s\n", params[i].str
+				, params[i].status->enable ? "enabled" : "disabled"
+				, params[i].status->loaded ? "loaded" : "unloaded"
+				, params[i].status->external_src ? "file" : "default"
+			);
+		}
+	} else
+		RTW_PRINT_SEL(sel, "not ready\n");
+}
+
+void dump_txpwr_tpc_settings(void *sel, struct dvobj_priv *dvobj)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+
+	if (rfctl->tpc_mode == TPC_MODE_DISABLE)
+		RTW_PRINT_SEL(sel, "mode:DISABLE(%d)\n", rfctl->tpc_mode);
+	else if (rfctl->tpc_mode == TPC_MODE_MANUAL) {
+		RTW_PRINT_SEL(sel, "mode:MANUAL(%d)\n", rfctl->tpc_mode);
+		RTW_PRINT_SEL(sel, "constraint:%d (mB)\n", rfctl->tpc_manual_constraint);
+	}
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_txpwr.h b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_txpwr.h
new file mode 100644
index 000000000000..03290608e03c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_txpwr.h
@@ -0,0 +1,64 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_TXPWR_H__
+#define __RTW_TXPWR_H__
+
+#define MBM_PDBM 100
+#define UNSPECIFIED_MBM 32767 /* maximum of s16 */
+
+#define TPC_MODE_DISABLE	0
+#define TPC_MODE_MANUAL		1
+#define TPC_MODE_INVALID	2	/* keep last */
+
+#define TPC_MANUAL_CONSTRAINT_MAX 600 /* mB */
+
+struct txpwr_param_status {
+	bool enable;
+	bool loaded;
+	bool external_src;
+};
+
+#define SET_TXPWR_PARAM_STATUS(_status, _enable, _loaded, _ext_src) \
+	do { \
+		(_status)->enable = _enable; \
+		(_status)->loaded = _loaded; \
+		(_status)->external_src = _ext_src; \
+	} while (0)
+
+struct tx_power_ext_info {
+	struct txpwr_param_status by_rate;
+	struct txpwr_param_status lmt;
+#ifdef CONFIG_80211AX_HE
+	struct txpwr_param_status lmt_ru;
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	struct txpwr_param_status lmt_6g;
+	struct txpwr_param_status lmt_ru_6g;
+#endif
+};
+
+s16 mb_of_ntx(u8 ntx);
+s8 txpwr_mbm_to_txgi_s8_with_max(s16 mbm, u8 txgi_max, u8 txgi_pdbm);
+
+void txpwr_idx_get_dbm_str(s8 idx, u8 txgi_max, s8 txgi_ww, u8 txgi_pdbm, SIZE_T cwidth, char dbm_str[], u8 dbm_str_len);
+void txpwr_mbm_get_dbm_str(s16 mbm, SIZE_T cwidth, char dbm_str[], u8 dbm_str_len);
+
+void rtw_update_txpwr_level(struct dvobj_priv *dvobj, enum phl_band_idx band_idx);
+void rtw_update_txpwr_level_all_hwband(struct dvobj_priv *dvobj);
+
+void dump_tx_power_ext_info(void *sel, struct dvobj_priv *dvobj);
+void dump_txpwr_tpc_settings(void *sel, struct dvobj_priv *dvobj);
+
+#endif /* __RTW_TXPWR_H__ */
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_vht.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_vht.c
new file mode 100644
index 000000000000..b661e0bf750d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_vht.c
@@ -0,0 +1,1840 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_VHT_C
+
+#include <drv_types.h>
+
+#ifdef CONFIG_80211AC_VHT
+const u16 _vht_max_mpdu_len[] = {
+	3895,
+	7991,
+	11454,
+	0,
+};
+
+const u8 _vht_sup_ch_width_set_to_bw_cap[] = {
+	BW_CAP_80M,
+	BW_CAP_80M | BW_CAP_160M,
+	BW_CAP_80M | BW_CAP_160M | BW_CAP_80_80M,
+	0,
+};
+
+const char *const _vht_sup_ch_width_set_str[] = {
+	"80MHz",
+	"160MHz",
+	"160MHz & 80+80MHz",
+	"BW-RSVD",
+};
+
+void dump_vht_cap_ie_content(void *sel, const u8 *buf, u32 buf_len)
+{
+	if (buf_len != VHT_CAP_IE_LEN) {
+		RTW_PRINT_SEL(sel, "Invalid VHT capability IE len:%d != %d\n", buf_len, VHT_CAP_IE_LEN);
+		return;
+	}
+
+	RTW_PRINT_SEL(sel, "cap_info:%02x %02x %02x %02x: MAX_MPDU_LEN:%u %s%s%s%s%s RX-STBC:%u MAX_AMPDU_LEN:%u\n"
+		, *(buf), *(buf + 1), *(buf + 2), *(buf + 3)
+		, vht_max_mpdu_len(GET_VHT_CAPABILITY_ELE_MAX_MPDU_LENGTH(buf))
+		, vht_sup_ch_width_set_str(GET_VHT_CAPABILITY_ELE_CHL_WIDTH(buf))
+		, GET_VHT_CAPABILITY_ELE_RX_LDPC(buf) ? " RX-LDPC" : ""
+		, GET_VHT_CAPABILITY_ELE_SHORT_GI80M(buf) ? " SGI-80" : ""
+		, GET_VHT_CAPABILITY_ELE_SHORT_GI160M(buf) ? " SGI-160" : ""
+		, GET_VHT_CAPABILITY_ELE_TX_STBC(buf) ? " TX-STBC" : ""
+		, GET_VHT_CAPABILITY_ELE_RX_STBC(buf)
+		, VHT_MAX_AMPDU_LEN(GET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(buf))
+	);
+}
+
+void dump_vht_cap_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *vht_cap_ie;
+	sint vht_cap_ielen;
+
+	vht_cap_ie = rtw_get_ie(ie, WLAN_EID_VHT_CAPABILITY, &vht_cap_ielen, ie_len);
+	if (!ie || vht_cap_ie != ie)
+		return;
+
+	dump_vht_cap_ie_content(sel, vht_cap_ie + 2, vht_cap_ielen);
+}
+
+const char *const _vht_op_ch_width_str[] = {
+	"20 or 40MHz",
+	"80MHz",
+	"160MHz",
+	"80+80MHz",
+	"BW-RSVD",
+};
+
+void dump_vht_op_ie_content(void *sel, const u8 *buf, u32 buf_len)
+{
+	if (buf_len != VHT_OP_IE_LEN) {
+		RTW_PRINT_SEL(sel, "Invalid VHT operation IE len:%d != %d\n", buf_len, VHT_OP_IE_LEN);
+		return;
+	}
+
+	RTW_PRINT_SEL(sel, "\tVHT Operation Info:\n");
+	RTW_PRINT_SEL(sel, "\tChannel Width: %u\n"
+					, GET_VHT_OPERATION_ELE_CHL_WIDTH(buf));
+	RTW_PRINT_SEL(sel, "\tCenter Frequency Channel for 80 and 160 MHz Operation: %u\n"
+					, GET_VHT_OPERATION_ELE_CENTER_FREQ1(buf));
+	RTW_PRINT_SEL(sel, "\tCenter Frequency Channel for 80+80 MHz Operation: %u\n"
+					, GET_VHT_OPERATION_ELE_CENTER_FREQ2(buf));
+	RTW_PRINT_SEL(sel, "\tVHT Basic MCS Set: 0x%04x\n"
+					, GET_VHT_OPERATION_ELE_BASIC_MCS_SET(buf));
+}
+
+void dump_vht_op_ie(void *sel, const u8 *ie, u32 ie_len)
+{
+	const u8 *vht_op_ie;
+	sint vht_op_ielen;
+
+	vht_op_ie = rtw_get_ie(ie, WLAN_EID_VHT_OPERATION, &vht_op_ielen, ie_len);
+	if (!ie || vht_op_ie != ie)
+		return;
+
+	dump_vht_op_ie_content(sel, vht_op_ie + 2, vht_op_ielen);
+}
+
+/*				20/40/80/160,	ShortGI,	MCS Rate  */
+const u16 VHT_MCS_DATA_RATE[CHANNEL_WIDTH_MAX][2][30] = {
+	{	{
+			13, 26, 39, 52, 78, 104, 117, 130, 156, 156,
+			26, 52, 78, 104, 156, 208, 234, 260, 312, 312,
+			39, 78, 117, 156, 234, 312, 351, 390, 468, 520
+		},			/* Long GI, 20MHz */
+		{
+			14, 29, 43, 58, 87, 116, 130, 144, 173, 173,
+			29, 58, 87, 116, 173, 231, 260, 289, 347, 347,
+			43,	87, 130, 173, 260, 347, 390,	433,	520, 578
+		}
+	},		/* Short GI, 20MHz */
+	{	{
+			27, 54, 81, 108, 162, 216, 243, 270, 324, 360,
+			54, 108, 162, 216, 324, 432, 486, 540, 648, 720,
+			81, 162, 243, 324, 486, 648, 729, 810, 972, 1080
+		}, 		/* Long GI, 40MHz */
+		{
+			30, 60, 90, 120, 180, 240, 270, 300, 360, 400,
+			60, 120, 180, 240, 360, 480, 540, 600, 720, 800,
+			90, 180, 270, 360, 540, 720, 810, 900, 1080, 1200
+		}
+	},		/* Short GI, 40MHz */
+	{	{
+			59, 117,  176, 234, 351, 468, 527, 585, 702, 780,
+			117, 234, 351, 468, 702, 936, 1053, 1170, 1404, 1560,
+			176, 351, 527, 702, 1053, 1404, 1580, 1755, 2106, 2340
+		},	/* Long GI, 80MHz */
+		{
+			65, 130, 195, 260, 390, 520, 585, 650, 780, 867,
+			130, 260, 390, 520, 780, 1040, 1170, 1300, 1560, 1734,
+			195, 390, 585, 780, 1170, 1560, 1755, 1950, 2340, 2600
+		}
+	},	/* Short GI, 80MHz */
+	{	{
+			117, 234, 351, 468, 702, 936, 1053, 1170, 1404, 1560,
+			234, 468, 702, 936, 1404, 1872, 2106, 2340, 2808, 3120,
+			351, 702, 1053, 1404, 2106, 2808, 3159, 3510, 4212, 4212
+		},	/* Long GI, 160MHz */
+		{
+			130, 260, 390, 520, 780, 1040, 1170, 1300, 1560, 1734,
+			260, 520, 780, 1040, 1560, 2080, 2340, 2600, 3120, 3466,
+			390, 780, 1170, 1560, 2340, 3120, 3510, 3900, 4680, 4680
+		}
+	}	/* Short GI, 160MHz */
+};
+
+u8	rtw_get_vht_highest_rate(u8 *pvht_mcs_map)
+{
+	u8	i, j;
+	u8	bit_map;
+	u8	vht_mcs_rate = 0;
+
+	for (i = 0; i < 2; i++) {
+		if (pvht_mcs_map[i] != 0xff) {
+			for (j = 0; j < 8; j += 2) {
+				bit_map = (pvht_mcs_map[i] >> j) & 3;
+
+				if (bit_map != 3)
+					vht_mcs_rate = MGN_VHT1SS_MCS7 + 10 * j / 2 + i * 40 + bit_map; /* VHT rate indications begin from 0x90 */
+			}
+		}
+	}
+
+	/* RTW_INFO("HighestVHTMCSRate is %x\n", vht_mcs_rate); */
+	return vht_mcs_rate;
+}
+
+u8	rtw_vht_mcsmap_to_nss(u8 *pvht_mcs_map)
+{
+	u8	i, j;
+	u8	bit_map;
+	u8	nss = 0;
+
+	for (i = 0; i < 2; i++) {
+		if (pvht_mcs_map[i] != 0xff) {
+			for (j = 0; j < 8; j += 2) {
+				bit_map = (pvht_mcs_map[i] >> j) & 3;
+
+				if (bit_map != 3)
+					nss++;
+			}
+		}
+	}
+
+	/* RTW_INFO("%s : %dSS\n", __FUNCTION__, nss); */
+	return nss;
+}
+
+void rtw_vht_nss_to_mcsmap(u8 nss, u8 *target_mcs_map, u8 *cur_mcs_map)
+{
+	u8	i, j;
+	u8	cur_rate, target_rate;
+
+	for (i = 0; i < 2; i++) {
+		target_mcs_map[i] = 0;
+		for (j = 0; j < 8; j += 2) {
+			cur_rate = (cur_mcs_map[i] >> j) & 3;
+			if (cur_rate == 3) /* 0x3 indicates not supported that num of SS */
+				target_rate = 3;
+			else if (nss <= ((j / 2) + i * 4))
+				target_rate = 3;
+			else
+				target_rate = cur_rate;
+
+			target_mcs_map[i] |= (target_rate << j);
+		}
+	}
+
+	/* RTW_INFO("%s : %dSS\n", __FUNCTION__, nss); */
+}
+
+u16	rtw_vht_mcs_to_data_rate(u8 bw, u8 short_GI, u8 vht_mcs_rate)
+{
+	if (vht_mcs_rate > MGN_VHT3SS_MCS9)
+		vht_mcs_rate = MGN_VHT3SS_MCS9;
+	/* RTW_INFO("bw=%d, short_GI=%d, ((vht_mcs_rate - MGN_VHT1SS_MCS0)&0x3f)=%d\n", bw, short_GI, ((vht_mcs_rate - MGN_VHT1SS_MCS0)&0x3f)); */
+	return VHT_MCS_DATA_RATE[bw][short_GI][((vht_mcs_rate - MGN_VHT1SS_MCS0) & 0x3f)];
+}
+
+u32	rtw_get_dft_vht_cap_ie(_adapter *padapter, u8 *pbuf)
+{
+	u8 bw;
+	u16 HighestRate;
+	u8 *pcap, *pcap_mcs;
+	u32 len = 0;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct vht_priv *pvhtpriv = &pmlmepriv->dev_vhtpriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	pcap = pvhtpriv->vht_cap;
+	_rtw_memset(pcap, 0, 32);
+
+	/*
+	* VHT Capabilities Information field : B0 to B31
+	*/
+
+	/* B0 B1 Maximum MPDU Length */
+	SET_VHT_CAPABILITY_ELE_MAX_MPDU_LENGTH(pcap, pvhtpriv->max_mpdu_len);
+
+	/* B2 B3 Supported Channel Width Set */
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) {
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80_80M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_80_80)) {
+			SET_VHT_CAPABILITY_ELE_CHL_WIDTH(pcap, 2);
+			RTW_INFO("[VHT] Declare supporting 160MHz and 80+80MHz\n");
+		} else {
+			SET_VHT_CAPABILITY_ELE_CHL_WIDTH(pcap, 1);
+			RTW_INFO("[VHT] Declare supporting 160MHz\n");
+		}
+	} else
+		SET_VHT_CAPABILITY_ELE_CHL_WIDTH(pcap, 0);
+
+	/* B4 Rx LDPC */
+	if (TEST_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_RX)) {
+		SET_VHT_CAPABILITY_ELE_RX_LDPC(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting RX LDPC\n");
+	}
+
+	/* B5 ShortGI for 80MHz */
+	SET_VHT_CAPABILITY_ELE_SHORT_GI80M(pcap, pvhtpriv->sgi_80m ? 1 : 0);
+	if (pvhtpriv->sgi_80m)
+		RTW_INFO("[VHT] Declare supporting SGI 80MHz\n");
+
+	/* B6 Short GI for 160 and 80+80 MHz */
+	SET_VHT_CAPABILITY_ELE_SHORT_GI160M(pcap, pvhtpriv->sgi_160m ? 1 : 0);
+	if (pvhtpriv->sgi_160m) {
+		RTW_INFO("[VHT] Declare supporting SGI 160MHz and 80+80MHz\n");
+	}
+
+	/* B7 Tx STBC */
+	if (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_TX)) {
+		SET_VHT_CAPABILITY_ELE_TX_STBC(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting TX STBC\n");
+	}
+
+	/* B8 B9 B10 Rx STBC */
+	if (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_RX)) {
+		SET_VHT_CAPABILITY_ELE_RX_STBC(pcap, pvhtpriv->rx_stbc_nss);
+		RTW_INFO("[VHT] Declare supporting RX STBC = %d\n", pvhtpriv->rx_stbc_nss);
+	}
+
+	#ifdef CONFIG_BEAMFORMING
+	/* B11 SU Beamformer Capable */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE)) {
+		SET_VHT_CAPABILITY_ELE_SU_BFER(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting SU Beamformer\n");
+		/* B16 17 18 Number of Sounding Dimensions */
+		SET_VHT_CAPABILITY_ELE_SOUNDING_DIMENSIONS(pcap, pvhtpriv->num_snd_dim);
+		/* B19 MU Beamformer Capable */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE)) {
+			SET_VHT_CAPABILITY_ELE_MU_BFER(pcap, 1);
+			RTW_INFO("[VHT] Declare supporting MU Beamformer\n");
+		}
+	}
+
+	/* B12 SU Beamformee Capable */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE)) {
+		u8 bfme_sts = pvhtpriv->bfme_sts;
+
+		SET_VHT_CAPABILITY_ELE_SU_BFEE(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting SU Beamformee\n");
+
+		/* IOT action suggested by Yu Chen 2017/3/3 */
+		if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_BROADCOM) &&
+			!pvhtpriv->ap_bf_cap.is_mu_bfer &&
+			pvhtpriv->ap_bf_cap.su_sound_dim == 2)
+
+		/* B13 14 15 Beamformee STS Capability */
+		SET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(pcap, bfme_sts);
+
+		/* B20 MU Beamformee Capable */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE)) {
+			SET_VHT_CAPABILITY_ELE_MU_BFEE(pcap, 1);
+			RTW_INFO("[VHT] Declare supporting MU Beamformee\n");
+		}
+	}
+	#endif/*CONFIG_BEAMFORMING*/
+
+	/* B21 VHT TXOP PS */
+	if (pvhtpriv->txop_ps) {
+		SET_VHT_CAPABILITY_ELE_TXOP_PS(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting VHT TXOP power save\n");
+	}
+
+	/* B22 +HTC-VHT Capable */
+	if (pvhtpriv->htc_vht) {
+		SET_VHT_CAPABILITY_ELE_HTC_VHT(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting VHT variant HT Control\n");
+	}
+
+	/* B23 24 25 Maximum A-MPDU Length Exponent */
+	SET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(pcap, pvhtpriv->ampdu_len);
+	RTW_INFO("[VHT] Declare supporting RX A-MPDU Length Exponent = %u\n", pvhtpriv->ampdu_len);
+
+	/* B26 27 VHT Link Adaptation Capable */
+	if (pvhtpriv->link_adap_cap) {
+		SET_VHT_CAPABILITY_ELE_LINK_ADAPTION(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting link adaptation using VHT variant HT Control\n");
+	}
+
+	/* B28 Rx Antenna Pattern Consistency */
+	if (pvhtpriv->rx_ant_pattern) {
+		SET_VHT_CAPABILITY_ELE_RX_ANT_PATTERN(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting RX Antenna Pattern Consistency\n");
+	}
+
+	/* B29 Tx Antenna Pattern Consistency */
+	if (pvhtpriv->tx_ant_pattern) {
+		SET_VHT_CAPABILITY_ELE_TX_ANT_PATTERN(pcap, 1);
+		RTW_INFO("[VHT] Declare supporting TX Antenna Pattern Consistency\n");
+	}
+
+	/* B30 B31 Extended NSS BW Support */
+	SET_VHT_CAPABILITY_ELE_EXT_NSS_BW(pcap, pvhtpriv->ext_nss_bw);
+
+	/*
+	* Supported VHT-MCS and NSS Set : 8 bytes
+	*/
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_RX_MCS(pcap);
+	_rtw_memcpy(pcap_mcs, pvhtpriv->vht_mcs_map, 2);
+
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_TX_MCS(pcap);
+	_rtw_memcpy(pcap_mcs, pvhtpriv->vht_mcs_map, 2);
+
+	/* find the largest bw supported by both registry and hal */
+	bw = rtw_hw_largest_bw(adapter_to_dvobj(padapter), REGSTY_BW_5G(pregistrypriv));
+
+	HighestRate = VHT_MCS_DATA_RATE[bw][pvhtpriv->sgi_80m][((pvhtpriv->vht_highest_rate - MGN_VHT1SS_MCS0) & 0x3f)];
+	HighestRate = (HighestRate + 1) >> 1;
+
+	SET_VHT_CAPABILITY_ELE_MCS_RX_HIGHEST_RATE(pcap, HighestRate); /* indicate we support highest rx rate is 600Mbps. */
+	SET_VHT_CAPABILITY_ELE_MCS_TX_HIGHEST_RATE(pcap, HighestRate); /* indicate we support highest tx rate is 600Mbps. */
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160))
+		SET_VHT_CAPABILITY_ELE_MCS_TX_EXT_NSS_BW_CAP(pcap, 1); /*160M*/
+
+	pbuf = rtw_set_ie(pbuf, EID_VHTCapability, 12, pcap, &len);
+
+	return len;
+}
+
+/* Initialized vhtpriv by PHL default setting */
+void rtw_vht_get_dft_setting(_adapter *padapter,
+			struct protocol_cap_t *dft_proto_cap,
+			struct role_link_cap_t *dft_cap)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct vht_priv *pvhtpriv = &pmlmepriv->dev_vhtpriv;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	BOOLEAN bHwLDPCSupport = _FALSE, bHwSTBCSupport = _FALSE;
+	u8 stbc_rx = 0;
+#ifdef CONFIG_BEAMFORMING
+	BOOLEAN	bHwSupportBeamformer = _FALSE, bHwSupportBeamformee = _FALSE;
+	u8 mu_bfer, mu_bfee;
+#endif /* CONFIG_BEAMFORMING */
+	u8 tx_nss, rx_nss;
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	/* Short GI */
+	pvhtpriv->sgi_80m = ((dft_proto_cap->sgi_80)
+		&& rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80M)
+		&& TEST_FLAG(pregistrypriv->short_gi, BIT2));
+
+	pvhtpriv->sgi_160m = ((dft_proto_cap->sgi_160)
+		&& rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M)
+		&& TEST_FLAG(pregistrypriv->short_gi, BIT3));
+
+	/* LDPC */
+	bHwLDPCSupport = (dft_proto_cap->vht_ldpc) ? _TRUE : _FALSE;
+	if (bHwLDPCSupport) {
+		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT0))
+			SET_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_RX);
+	}
+
+	bHwLDPCSupport = (dft_cap->tx_vht_ldpc) ? _TRUE : _FALSE;
+	if (bHwLDPCSupport) {
+		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT1))
+			SET_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_TX);
+	}
+
+	if (pvhtpriv->ldpc_cap)
+		RTW_INFO("[VHT] Support LDPC = 0x%02X\n", pvhtpriv->ldpc_cap);
+
+	/* STBC */
+	if (dft_proto_cap->stbc_vht_tx)
+		SET_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_TX);
+
+	if (dft_proto_cap->stbc_vht_rx) {
+		SET_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_RX);
+		pvhtpriv->rx_stbc_nss = dft_proto_cap->stbc_vht_rx;
+	}
+
+	if (pvhtpriv->stbc_cap)
+		RTW_INFO("[VHT] Support STBC = 0x%02X\n", pvhtpriv->stbc_cap);
+
+	/* Beamforming setting */
+#ifdef CONFIG_BEAMFORMING
+	/* only enable beamforming in STA client mode */
+	if (MLME_IS_STA(padapter) && !MLME_IS_GC(padapter)
+				  && !MLME_IS_ADHOC(padapter)
+				  && !MLME_IS_MESH(padapter))
+	{
+
+		bHwSupportBeamformer = (dft_proto_cap->vht_su_bfmr) ? _TRUE : _FALSE;
+		bHwSupportBeamformee = (dft_proto_cap->vht_su_bfme) ? _TRUE : _FALSE;
+
+		mu_bfer = (dft_proto_cap->vht_mu_bfmr) ? _TRUE : _FALSE;
+		mu_bfee = (dft_proto_cap->vht_mu_bfme) ? _TRUE : _FALSE;
+
+		if (TEST_FLAG(pregistrypriv->beamform_cap, BIT0) && bHwSupportBeamformer) {
+#ifdef CONFIG_CONCURRENT_MODE
+			if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) {
+				SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+				RTW_INFO("[VHT] CONCURRENT AP Support Beamformer\n");
+				if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(2))
+				    && (_TRUE == mu_bfer)) {
+					SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+					RTW_INFO("[VHT] Support MU-MIMO AP\n");
+				}
+				pvhtpriv->num_snd_dim = dft_proto_cap->num_snd_dim;
+			} else
+				RTW_INFO("[VHT] CONCURRENT not AP ;not allow  Support Beamformer\n");
+#else
+			SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+			RTW_INFO("[VHT] Support Beamformer\n");
+			if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(2))
+			    && (_TRUE == mu_bfer)
+			    && ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)) {
+				SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+				RTW_INFO("[VHT] Support MU-MIMO AP\n");
+			}
+#endif
+		}
+		if (TEST_FLAG(pregistrypriv->beamform_cap, BIT1) && bHwSupportBeamformee) {
+			SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE);
+			RTW_INFO("[VHT] Support Beamformee\n");
+			pvhtpriv->bfme_sts = dft_proto_cap->bfme_sts;
+			if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(3))
+			    && (_TRUE == mu_bfee)
+			    && ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)) {
+				SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE);
+				RTW_INFO("[VHT] Support MU-MIMO STA\n");
+			}
+		}
+	}
+#endif /* CONFIG_BEAMFORMING */
+
+	/* We don't support these features, and PHL doesn't define */
+	pvhtpriv->txop_ps = 0;
+	pvhtpriv->htc_vht = 1;
+	pvhtpriv->link_adap_cap = 0;
+	pvhtpriv->tx_ant_pattern= 0;
+	pvhtpriv->rx_ant_pattern= 0;
+	pvhtpriv->ext_nss_bw = 0;
+
+	pvhtpriv->ampdu_len = pregistrypriv->ampdu_factor;
+	pvhtpriv->max_mpdu_len = dft_proto_cap->max_amsdu_len;
+
+	tx_nss = GET_PHY_TX_NSS_BY_BAND(adapter_to_dvobj(padapter), HW_BAND_0);
+	rx_nss = GET_PHY_RX_NSS_BY_BAND(adapter_to_dvobj(padapter), HW_BAND_0);
+
+	/* for now, vhtpriv.vht_mcs_map comes from RX NSS */
+	rtw_vht_nss_to_mcsmap(rx_nss, pvhtpriv->vht_mcs_map, pregistrypriv->vht_rx_mcs_map);
+	pvhtpriv->vht_highest_rate = rtw_get_vht_highest_rate(pvhtpriv->vht_mcs_map);
+}
+
+/* Initialized vhtpriv by adapter real setting */
+void rtw_vht_get_real_setting(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+#ifdef CONFIG_BEAMFORMING
+	BOOLEAN bHwSupportBeamformer = _FALSE, bHwSupportBeamformee = _FALSE;
+	u8 mu_bfer, mu_bfee;
+#endif /* CONFIG_BEAMFORMING */
+	u8 tx_nss, rx_nss;
+	u8 rf_type = 0;
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct registry_priv *pregistrypriv = &(padapter->registrypriv);
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct protocol_cap_t *proto_cap = &(padapter_link->wrlink->protocol_cap);
+	struct role_link_cap_t *cap = &(padapter_link->wrlink->cap);
+	struct vht_priv *pvhtpriv = &pmlmepriv->vhtpriv;
+
+	/* Short GI */
+	pvhtpriv->sgi_80m = ((proto_cap->sgi_80)
+		&& rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80M)
+		&& TEST_FLAG(pregistrypriv->short_gi, BIT2));
+
+	pvhtpriv->sgi_160m = ((proto_cap->sgi_160)
+		&& rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M)
+		&& TEST_FLAG(pregistrypriv->short_gi, BIT3));
+
+	/* LDPC support */
+	if (proto_cap->vht_ldpc)
+		SET_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_RX);
+
+	if (cap->tx_vht_ldpc)
+		SET_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_TX);
+
+	if (pvhtpriv->ldpc_cap)
+		RTW_DBG("[VHT] Support LDPC = 0x%02X\n", pvhtpriv->ldpc_cap);
+
+	/* STBC */
+	if (proto_cap->stbc_vht_tx)
+		SET_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_TX);
+
+	if (proto_cap->stbc_vht_rx) {
+		SET_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_RX);
+		pvhtpriv->rx_stbc_nss = proto_cap->stbc_vht_rx;
+	}
+
+	if (pvhtpriv->stbc_cap)
+		RTW_DBG("[VHT] Support STBC = 0x%02X\n", pvhtpriv->stbc_cap);
+
+	/* Beamforming setting */
+	CLEAR_FLAGS(pvhtpriv->beamform_cap);
+#ifdef CONFIG_BEAMFORMING
+	if (proto_cap->vht_su_bfmr) {
+		SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+		RTW_DBG("[VHT] HAL Support Beamformer\n");
+		if (proto_cap->vht_mu_bfmr) {
+			SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+			RTW_DBG("[VHT] Support MU-MIMO AP\n");
+		}
+	}
+
+	if (proto_cap->vht_su_bfme) {
+		u8 bfme_sts = 0;
+
+		rtw_hal_get_def_var(padapter, padapter_link, HAL_DEF_BEAMFORMEE_CAP, (u8 *)&bfme_sts);
+		pvhtpriv->bfme_sts = bfme_sts;
+		SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE);
+		RTW_DBG("[VHT] HAL Support Beamformee\n");
+		if (proto_cap->vht_mu_bfme) {
+			SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE);
+			RTW_DBG("[VHT] Support MU-MIMO STA\n");
+		}
+	}
+#endif
+
+/* ToDo: check bfee flow will move change role type and modify wifi_role->protocol_cap */
+#if 0
+#ifdef CONFIG_BEAMFORMING
+	/* only enable beamforming in STA client mode */
+	if (MLME_IS_STA(padapter) && !MLME_IS_GC(padapter)
+				  && !MLME_IS_ADHOC(padapter)
+				  && !MLME_IS_MESH(padapter))
+	{
+		rtw_hal_get_def_var(padapter, HAL_DEF_EXPLICIT_BEAMFORMER,
+			(u8 *)&bHwSupportBeamformer);
+		rtw_hal_get_def_var(padapter, HAL_DEF_EXPLICIT_BEAMFORMEE,
+			(u8 *)&bHwSupportBeamformee);
+		mu_bfer = _FALSE;
+		mu_bfee = _FALSE;
+		rtw_hal_get_def_var(padapter, HAL_DEF_VHT_MU_BEAMFORMER, &mu_bfer);
+		rtw_hal_get_def_var(padapter, HAL_DEF_VHT_MU_BEAMFORMEE, &mu_bfee);
+		if (TEST_FLAG(pregistrypriv->beamform_cap, BIT0) && bHwSupportBeamformer) {
+#ifdef CONFIG_CONCURRENT_MODE
+			if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) {
+				u8 num_snd_dim = 0;
+
+				SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+				RTW_INFO("[VHT] CONCURRENT AP Support Beamformer\n");
+				if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(2))
+				    && (_TRUE == mu_bfer)) {
+					SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+					RTW_INFO("[VHT] Support MU-MIMO AP\n");
+				}
+				rtw_hal_get_def_var(padapter, HAL_DEF_BEAMFORMER_CAP, (u8 *)&num_snd_dim);
+				pvhtpriv->num_snd_dim = num_snd_dim;
+			} else
+				RTW_INFO("[VHT] CONCURRENT not AP ;not allow  Support Beamformer\n");
+#else
+			SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+			RTW_INFO("[VHT] Support Beamformer\n");
+			if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(2))
+			    && (_TRUE == mu_bfer)
+			    && ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)) {
+				SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+				RTW_INFO("[VHT] Support MU-MIMO AP\n");
+			}
+#endif
+		}
+		if (TEST_FLAG(pregistrypriv->beamform_cap, BIT1) && bHwSupportBeamformee) {
+			u8 bfme_sts = 0;
+
+			SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE);
+			RTW_INFO("[VHT] Support Beamformee\n");
+			rtw_hal_get_def_var(padapter, HAL_DEF_BEAMFORMEE_CAP, (u8 *)&bfme_sts);
+			pvhtpriv->bfme_sts = bfme_sts;
+			if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(3))
+			    && (_TRUE == mu_bfee)
+			    && ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)) {
+				SET_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE);
+				RTW_INFO("[VHT] Support MU-MIMO STA\n");
+			}
+		}
+	}
+#endif /* CONFIG_BEAMFORMING */
+#endif
+
+	/* We don't support these features, and PHL doesn't define */
+	pvhtpriv->txop_ps = 0;
+	pvhtpriv->htc_vht = 1;
+	pvhtpriv->link_adap_cap = 0;
+	pvhtpriv->tx_ant_pattern= 0;
+	pvhtpriv->rx_ant_pattern= 0;
+	pvhtpriv->ext_nss_bw = 0;
+
+	pvhtpriv->ampdu_len = pregistrypriv->ampdu_factor;
+	pvhtpriv->max_mpdu_len = proto_cap->max_amsdu_len;
+
+	tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	rx_nss = get_phy_rx_nss(padapter, padapter_link);
+
+	/* for now, vhtpriv.vht_mcs_map comes from RX NSS */
+	rtw_vht_nss_to_mcsmap(rx_nss, pvhtpriv->vht_mcs_map, pregistrypriv->vht_rx_mcs_map);
+	pvhtpriv->vht_highest_rate = rtw_get_vht_highest_rate(pvhtpriv->vht_mcs_map);
+}
+
+u64	rtw_vht_mcs_map_to_bitmap(u8 *mcs_map, u8 nss)
+{
+	u8 i, j, tmp;
+	u64 bitmap = 0;
+	u8 bits_nss = nss * 2;
+
+	for (i = j = 0; i < bits_nss; i += 2, j += 10) {
+		/* every two bits means single sptial stream */
+		tmp = (mcs_map[i / 8] >> i) & 3;
+
+		switch (tmp) {
+		case 2:
+			bitmap = bitmap | (0x03ff << j);
+			break;
+		case 1:
+			bitmap = bitmap | (0x01ff << j);
+			break;
+		case 0:
+			bitmap = bitmap | (0x00ff << j);
+			break;
+		default:
+			break;
+		}
+	}
+
+	RTW_INFO("vht_mcs_map=%02x %02x, nss=%u => bitmap=%016llx\n"
+		, mcs_map[0], mcs_map[1], nss, bitmap);
+
+	return bitmap;
+}
+
+#ifdef CONFIG_BEAMFORMING
+void update_sta_vht_info_apmode_bf_cap(_adapter *padapter, struct sta_info *psta)
+{
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_priv	*pmlmepriv = &(padapter_link->mlmepriv);
+	struct vht_priv	*pvhtpriv_ap = &pmlmepriv->vhtpriv;
+	struct vht_priv	*pvhtpriv_sta = &psta->vhtpriv;
+	u16	cur_beamform_cap = 0;
+
+	/* B11 SU Beamformer Capable, the target supports Beamformer and we are Beamformee */
+	if (TEST_FLAG(pvhtpriv_ap->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE) &&
+	    GET_VHT_CAPABILITY_ELE_SU_BFEE(pvhtpriv_sta->vht_cap)) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE);
+		/*Shift to BEAMFORMING_VHT_BEAMFORMER_STS_CAP*/
+		SET_FLAG(cur_beamform_cap, GET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(pvhtpriv_sta->vht_cap) << 8);
+	}
+
+	/* B12 SU Beamformee Capable, the target supports Beamformee and we are Beamformer */
+	if (TEST_FLAG(pvhtpriv_ap->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE) &&
+	    GET_VHT_CAPABILITY_ELE_SU_BFER(pvhtpriv_sta->vht_cap)) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+		/*Shit to BEAMFORMING_VHT_BEAMFORMEE_SOUND_DIM*/
+		SET_FLAG(cur_beamform_cap, GET_VHT_CAPABILITY_ELE_SU_BFER_SOUND_DIM_NUM(pvhtpriv_sta->vht_cap) << 12);
+	}
+
+	if (cur_beamform_cap)
+		RTW_INFO("Current STA(%d) VHT Beamforming Setting = %02X\n", psta->phl_sta->aid, cur_beamform_cap);
+
+	pvhtpriv_sta->beamform_cap = cur_beamform_cap;
+
+	/* ToDo: need to API to inform hal_sta->bf_info.vht_beamform_cap  */
+	#if 0
+	psta->phl_sta->bf_info.vht_beamform_cap = cur_beamform_cap;
+	#endif
+}
+#endif
+
+void update_sta_vht_info_apmode(_adapter *padapter, void *sta)
+{
+	struct sta_info	*psta = (struct sta_info *)sta;
+	struct rtw_phl_stainfo_t *phl_sta = psta->phl_sta;
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct protocol_cap_t *protocol_cap = &(padapter_link->wrlink->protocol_cap);
+	struct vht_priv *pvhtpriv_ap = &pmlmepriv->vhtpriv;
+	struct vht_priv *pvhtpriv_sta = &psta->vhtpriv;
+
+	u8	cur_ldpc_cap = 0, cur_stbc_cap = 0;
+	s8 bw_mode = -1;
+	u8	*pcap_mcs;
+
+	if (pvhtpriv_sta->vht_option == _FALSE)
+		return;
+
+	if (pvhtpriv_sta->op_present) {
+		if (GET_VHT_OPERATION_ELE_CHL_WIDTH(pvhtpriv_sta->vht_op) >= CH_WIDTH_80_160M)
+			bw_mode = CHANNEL_WIDTH_80;
+	}
+
+	if (pvhtpriv_sta->notify_present)
+		bw_mode = GET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(&pvhtpriv_sta->vht_op_mode_notify);
+	else if (MLME_IS_AP(padapter)) {
+		/* for VHT client without Operating Mode Notify IE; minimal 80MHz */
+		if (bw_mode < CHANNEL_WIDTH_80)
+			bw_mode = CHANNEL_WIDTH_80;
+	}
+
+	if (bw_mode != -1)
+		psta->phl_sta->chandef.bw = bw_mode; /* update bw_mode only if get value from VHT IEs */
+
+	/* ToDo: need to API to inform hal_sta->ra_info.is_vht_enable  */
+	/* psta->phl_sta->ra_info.is_vht_enable = _TRUE; */
+
+	/* B4 Rx LDPC */
+	if (TEST_FLAG(pvhtpriv_ap->ldpc_cap, LDPC_VHT_ENABLE_TX) &&
+	    GET_VHT_CAPABILITY_ELE_RX_LDPC(pvhtpriv_sta->vht_cap)) {
+		SET_FLAG(cur_ldpc_cap, (LDPC_VHT_ENABLE_TX | LDPC_VHT_CAP_TX));
+		RTW_INFO("Current STA(%d) VHT LDPC = %02X\n", psta->phl_sta->aid, cur_ldpc_cap);
+	}
+	pvhtpriv_sta->ldpc_cap = cur_ldpc_cap;
+
+	if (psta->phl_sta->chandef.bw > pmlmeext->chandef.bw)
+		psta->phl_sta->chandef.bw = pmlmeext->chandef.bw;
+
+	if (psta->phl_sta->chandef.bw == CHANNEL_WIDTH_80) {
+		/* B5 Short GI for 80 MHz */
+		pvhtpriv_sta->sgi_80m = (GET_VHT_CAPABILITY_ELE_SHORT_GI80M(pvhtpriv_sta->vht_cap) & pvhtpriv_ap->sgi_80m) ? _TRUE : _FALSE;
+		/* RTW_INFO("Current STA ShortGI80MHz = %d\n", pvhtpriv_sta->sgi_80m); */
+	} else if (psta->phl_sta->chandef.bw >= CHANNEL_WIDTH_160) {
+		/* B6 Short GI for 160 MHz */
+		pvhtpriv_sta->sgi_160m = (GET_VHT_CAPABILITY_ELE_SHORT_GI160M(pvhtpriv_sta->vht_cap) & pvhtpriv_ap->sgi_160m) ? _TRUE : _FALSE;
+		/* RTW_INFO("Current STA ShortGI160MHz = %d\n", pvhtpriv_sta->sgi_160m); */
+	}
+
+	/* B8 B9 B10 Rx STBC */
+	if (TEST_FLAG(pvhtpriv_ap->stbc_cap, STBC_VHT_ENABLE_TX) &&
+	    GET_VHT_CAPABILITY_ELE_RX_STBC(pvhtpriv_sta->vht_cap)) {
+		SET_FLAG(cur_stbc_cap, (STBC_VHT_ENABLE_TX | STBC_VHT_CAP_TX));
+		RTW_INFO("Current STA(%d) VHT STBC = %02X\n", psta->phl_sta->aid, cur_stbc_cap);
+		phl_sta->asoc_cap.stbc_vht_rx =
+			protocol_cap->stbc_vht_tx ?
+			GET_VHT_CAPABILITY_ELE_RX_STBC(pvhtpriv_sta->vht_cap) : 0;
+	}
+	pvhtpriv_sta->stbc_cap = cur_stbc_cap;
+	phl_sta->asoc_cap.stbc_vht_tx =
+		GET_VHT_CAPABILITY_ELE_TX_STBC(pvhtpriv_sta->vht_cap);
+
+#ifdef CONFIG_BEAMFORMING
+	update_sta_vht_info_apmode_bf_cap(padapter, psta);
+#endif
+
+	/* B23 B24 B25 Maximum A-MPDU Length Exponent */
+	pvhtpriv_sta->ampdu_len = GET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(pvhtpriv_sta->vht_cap);
+
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_RX_MCS(pvhtpriv_sta->vht_cap);
+	_rtw_memcpy(pvhtpriv_sta->vht_mcs_map, pcap_mcs, 2);
+	pvhtpriv_sta->vht_highest_rate = rtw_get_vht_highest_rate(pvhtpriv_sta->vht_mcs_map);
+}
+
+#ifdef PRIVATE_R
+u8 VHT_get_ss_from_map(u8 *vht_mcs_map)
+{
+	u8 i, j;
+	u8 ss = 0;
+
+	for (i = 0; i < 2; i++) {
+		if (vht_mcs_map[i] != 0xff) {
+			for (j = 0; j < 8; j += 2) {
+				if (((vht_mcs_map[i] >> j) & 0x03) == 0x03)
+					break;
+				ss++;
+			}
+		}
+
+	}
+
+return ss;
+}
+
+void VHT_caps_handler_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv_infra_ap	*pvhtpriv = &pmlmepriv->vhtpriv_infra_ap;
+	u8      cur_stbc_cap_infra_ap = 0;
+	u16	cur_beamform_cap_infra_ap = 0;
+	u8	*pcap_mcs;
+	u8	*pcap_mcs_tx;
+	u8	Rx_ss = 0, Tx_ss = 0;
+
+	struct link_mlme_ext_priv		*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info		*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (pIE == NULL)
+		return;
+
+	pmlmeinfo->ht_vht_received |= BIT(1);
+
+	pvhtpriv->ldpc_cap_infra_ap = GET_VHT_CAPABILITY_ELE_RX_LDPC(pIE->data);
+
+	if (GET_VHT_CAPABILITY_ELE_RX_STBC(pIE->data))
+		SET_FLAG(cur_stbc_cap_infra_ap, STBC_VHT_ENABLE_RX);
+	if (GET_VHT_CAPABILITY_ELE_TX_STBC(pIE->data))
+		SET_FLAG(cur_stbc_cap_infra_ap, STBC_VHT_ENABLE_TX);
+	pvhtpriv->stbc_cap_infra_ap = cur_stbc_cap_infra_ap;
+
+	/*store ap info for channel bandwidth*/
+	pvhtpriv->channel_width_infra_ap = GET_VHT_CAPABILITY_ELE_CHL_WIDTH(pIE->data);
+
+	/*check B11: SU Beamformer Capable and B12: SU Beamformee B19: MU Beamformer B20:MU Beamformee*/
+	if (GET_VHT_CAPABILITY_ELE_SU_BFER(pIE->data))
+		SET_FLAG(cur_beamform_cap_infra_ap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+	if (GET_VHT_CAPABILITY_ELE_SU_BFEE(pIE->data))
+		SET_FLAG(cur_beamform_cap_infra_ap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE);
+	if (GET_VHT_CAPABILITY_ELE_MU_BFER(pIE->data))
+		SET_FLAG(cur_beamform_cap_infra_ap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+	if (GET_VHT_CAPABILITY_ELE_MU_BFEE(pIE->data))
+		SET_FLAG(cur_beamform_cap_infra_ap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE);
+	pvhtpriv->beamform_cap_infra_ap = cur_beamform_cap_infra_ap;
+
+	/*store information about vht_mcs_set*/
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_RX_MCS(pIE->data);
+	pcap_mcs_tx = GET_VHT_CAPABILITY_ELE_TX_MCS(pIE->data);
+	_rtw_memcpy(pvhtpriv->vht_mcs_map_infra_ap, pcap_mcs, 2);
+	_rtw_memcpy(pvhtpriv->vht_mcs_map_tx_infra_ap, pcap_mcs_tx, 2);
+
+	Rx_ss = VHT_get_ss_from_map(pvhtpriv->vht_mcs_map_infra_ap);
+	Tx_ss = VHT_get_ss_from_map(pvhtpriv->vht_mcs_map_tx_infra_ap);
+	if (Rx_ss >= Tx_ss) {
+		pvhtpriv->number_of_streams_infra_ap = Rx_ss;
+	} else{
+		pvhtpriv->number_of_streams_infra_ap = Tx_ss;
+	}
+
+}
+#endif /* PRIVATE_R */
+
+void VHT_caps_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct rtw_wifi_role_t 	*wrole = padapter->phl_role;
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct protocol_cap_t *protocol_cap = &padapter_link->wrlink->protocol_cap;
+	struct vht_priv		*pvhtpriv = &pmlmepriv->vhtpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX           *cur_network = &(pmlmeinfo->network);
+	struct sta_priv 	*pstapriv = &padapter->stapriv;
+	struct sta_info		*psta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	u8	cur_ldpc_cap = 0, cur_stbc_cap = 0, rx_nss = 0;
+	u16	cur_beamform_cap = 0;
+	u8	*pcap_mcs;
+
+	if (pIE == NULL)
+		return;
+
+	if (pvhtpriv->vht_option == _FALSE)
+		return;
+
+	psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+	if (psta == NULL)
+		return;
+	phl_sta = psta->phl_sta;
+
+	pmlmeinfo->VHT_enable = 1;
+
+	/* B4 Rx LDPC */
+	if (TEST_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_TX) &&
+	    GET_VHT_CAPABILITY_ELE_RX_LDPC(pIE->data)) {
+		SET_FLAG(cur_ldpc_cap, (LDPC_VHT_ENABLE_TX | LDPC_VHT_CAP_TX));
+		RTW_INFO("Current VHT LDPC Setting = %02X\n", cur_ldpc_cap);
+		phl_sta->asoc_cap.vht_ldpc = 1;
+	}
+	pvhtpriv->ldpc_cap = cur_ldpc_cap;
+
+	/* B5 Short GI for 80 MHz */
+	pvhtpriv->sgi_80m = (GET_VHT_CAPABILITY_ELE_SHORT_GI80M(pIE->data) & pvhtpriv->sgi_80m) ? _TRUE : _FALSE;
+	/* RTW_INFO("Current ShortGI80MHz = %d\n", pvhtpriv->sgi_80m); */
+
+	/* B6 Short GI for 160 MHz */
+	pvhtpriv->sgi_160m = (GET_VHT_CAPABILITY_ELE_SHORT_GI160M(pIE->data) & pvhtpriv->sgi_160m) ? _TRUE : _FALSE;
+	/* RTW_INFO("Current ShortGI160MHz = %d\n", pvhtpriv->sgi_160m); */
+
+	/* B8 B9 B10 Rx STBC */
+	if (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_TX) &&
+	    GET_VHT_CAPABILITY_ELE_RX_STBC(pIE->data)) {
+		SET_FLAG(cur_stbc_cap, (STBC_VHT_ENABLE_TX | STBC_VHT_CAP_TX));
+		RTW_INFO("Current VHT STBC Setting = %02X\n", cur_stbc_cap);
+		phl_sta->asoc_cap.stbc_vht_rx =
+			protocol_cap->stbc_vht_tx ?
+			GET_VHT_CAPABILITY_ELE_RX_STBC(pIE->data) : 0;
+	}
+	pvhtpriv->stbc_cap = cur_stbc_cap;
+	phl_sta->asoc_cap.stbc_vht_tx = GET_VHT_CAPABILITY_ELE_TX_STBC(pIE->data);
+
+	phl_sta->asoc_cap.vht_su_bfmr = GET_VHT_CAPABILITY_ELE_SU_BFER(pIE->data);
+	phl_sta->asoc_cap.vht_su_bfme = GET_VHT_CAPABILITY_ELE_SU_BFEE(pIE->data);
+	phl_sta->asoc_cap.vht_mu_bfmr = GET_VHT_CAPABILITY_ELE_MU_BFER(pIE->data);
+	phl_sta->asoc_cap.vht_mu_bfme = GET_VHT_CAPABILITY_ELE_MU_BFEE(pIE->data);
+	phl_sta->asoc_cap.bfme_sts = GET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(pIE->data);
+	phl_sta->asoc_cap.num_snd_dim = GET_VHT_CAPABILITY_ELE_SU_BFER_SOUND_DIM_NUM(pIE->data);
+
+	RTW_INFO("%s: VHT STA assoc_cap:\n", __func__);
+	RTW_INFO("- SU BFer: %d\n", phl_sta->asoc_cap.vht_su_bfmr);
+	RTW_INFO("- SU BFee: %d\n", phl_sta->asoc_cap.vht_su_bfme);
+	RTW_INFO("- MU BFer: %d\n", phl_sta->asoc_cap.vht_mu_bfmr);
+	RTW_INFO("- MU BFee: %d\n", phl_sta->asoc_cap.vht_mu_bfme);
+	RTW_INFO("- BFee STS: %d\n", phl_sta->asoc_cap.bfme_sts);
+	RTW_INFO("- BFer SND DIM number: %d\n", phl_sta->asoc_cap.num_snd_dim);
+
+#ifdef CONFIG_BEAMFORMING
+	/*
+	 * B11 SU Beamformer Capable,
+	 * the target supports Beamformer and we are Beamformee
+	 */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE)
+	    && GET_VHT_CAPABILITY_ELE_SU_BFER(pIE->data)) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE);
+
+		/* Shift to BEAMFORMING_VHT_BEAMFORMEE_STS_CAP */
+		SET_FLAG(cur_beamform_cap, GET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(pIE->data) << 8);
+
+		/*
+		 * B19 MU Beamformer Capable,
+		 * the target supports Beamformer and we are Beamformee
+		 */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE)
+		    && GET_VHT_CAPABILITY_ELE_MU_BFER(pIE->data))
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE);
+	}
+
+	/*
+	 * B12 SU Beamformee Capable,
+	 * the target supports Beamformee and we are Beamformer
+	 */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE)
+	    && GET_VHT_CAPABILITY_ELE_SU_BFEE(pIE->data)) {
+		SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE);
+
+		/* Shit to BEAMFORMING_VHT_BEAMFORMER_SOUND_DIM */
+		SET_FLAG(cur_beamform_cap, GET_VHT_CAPABILITY_ELE_SU_BFER_SOUND_DIM_NUM(pIE->data) << 12);
+
+		/*
+		 * B20 MU Beamformee Capable,
+		 * the target supports Beamformee and we are Beamformer
+		 */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE)
+		    && GET_VHT_CAPABILITY_ELE_MU_BFEE(pIE->data))
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE);
+	}
+
+	pvhtpriv->beamform_cap = cur_beamform_cap;
+	RTW_INFO("Current VHT Beamforming Setting=0x%04X\n", cur_beamform_cap);
+#endif /* CONFIG_BEAMFORMING */
+
+	/* B0 B1 Maximum MPDU Length */
+	pvhtpriv->max_mpdu_len = GET_VHT_CAPABILITY_ELE_MAX_MPDU_LENGTH(pIE->data);
+	/* B23 B24 B25 Maximum A-MPDU Length Exponent */
+	pvhtpriv->ampdu_len = GET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(pIE->data);
+
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_RX_MCS(pIE->data);
+	rx_nss = get_phy_rx_nss(padapter, padapter_link);
+	rtw_vht_nss_to_mcsmap(rx_nss, pvhtpriv->vht_mcs_map, pcap_mcs);
+	pvhtpriv->vht_highest_rate = rtw_get_vht_highest_rate(pvhtpriv->vht_mcs_map);
+}
+
+void VHT_operation_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv		*pvhtpriv = &pmlmepriv->vhtpriv;
+
+	if (pIE == NULL)
+		return;
+
+	if (pvhtpriv->vht_option == _FALSE)
+		return;
+
+	if (pIE->Length != VHT_OP_IE_LEN) {
+		RTW_WARN("%s: unexpected IE length(%u)!\n",
+			 __func__, pIE->Length);
+	}
+
+	pvhtpriv->op_present = 1;
+	_rtw_memcpy(pvhtpriv->vht_op, pIE->data,
+		    pIE->Length > VHT_OP_IE_LEN ? VHT_OP_IE_LEN : pIE->Length);
+}
+
+void rtw_process_vht_op_mode_notify(_adapter *padapter, u8 *pframe, void *sta)
+{
+	struct registry_priv	*regsty = &padapter->registrypriv;
+	struct sta_info		*psta = (struct sta_info *)sta;
+	struct _ADAPTER_LINK	*padapter_link = psta->padapter_link;
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv		*pvhtpriv = &pmlmepriv->vhtpriv;
+	u8	target_bw;
+	u8	target_rxss, current_rxss;
+	u8	update_ra = _FALSE;
+	u8 tx_nss = 0;
+
+	if (pvhtpriv->vht_option == _FALSE)
+		return;
+
+	target_bw = GET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(pframe);
+	tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	target_rxss = rtw_min(tx_nss, (GET_VHT_OPERATING_MODE_FIELD_RX_NSS(pframe) + 1));
+
+	if (target_bw != psta->phl_sta->chandef.bw) {
+		if (rtw_hw_is_bw_support(adapter_to_dvobj(padapter), target_bw)
+		    && REGSTY_IS_BW_5G_SUPPORT(regsty, target_bw)
+		   ) {
+			update_ra = _TRUE;
+			psta->phl_sta->chandef.bw = target_bw;
+		}
+	}
+
+	current_rxss = rtw_vht_mcsmap_to_nss(psta->vhtpriv.vht_mcs_map);
+	if (target_rxss != current_rxss) {
+		u8	vht_mcs_map[2] = {};
+
+		update_ra = _TRUE;
+
+		rtw_vht_nss_to_mcsmap(target_rxss, vht_mcs_map, psta->vhtpriv.vht_mcs_map);
+		_rtw_memcpy(psta->vhtpriv.vht_mcs_map, vht_mcs_map, 2);
+
+		update_sta_ra_info(padapter, psta);
+	}
+
+	if (update_ra)
+		rtw_sta_hal_ra_mask_update_cmd(padapter, psta, 0);
+}
+
+u32	rtw_build_vht_operation_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *pbuf, enum band_type band, u8 channel)
+{
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv			*pvhtpriv = &pmlmepriv->vhtpriv;
+	/* struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv; */
+
+	u8	ChnlWidth, center_freq, bw_mode = CHANNEL_WIDTH_20;
+	u32	len = 0;
+	u8	operation[5];
+
+	_rtw_memset(operation, 0, 5);
+
+	/* TODO: control op bw with other info */
+	if (band == BAND_ON_24G)
+		bw_mode = REGSTY_BW_2G(pregistrypriv);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G)
+		bw_mode = REGSTY_BW_5G(pregistrypriv);
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G)
+		bw_mode = REGSTY_BW_6G(pregistrypriv);
+#endif
+	else
+		rtw_warn_on(1);
+
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80M | BW_CAP_160M)
+		&& bw_mode >= CHANNEL_WIDTH_80
+	) {
+		center_freq = rtw_get_center_ch_by_band(band, channel, bw_mode, CHAN_OFFSET_NO_EXT);
+		ChnlWidth = CH_WIDTH_80_160M;
+	} else {
+		center_freq = 0;
+		ChnlWidth = CH_WIDTH_20_40M;
+	}
+
+	SET_VHT_OPERATION_ELE_CHL_WIDTH(operation, ChnlWidth);
+	/* center frequency */
+	SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(operation, center_freq);/* Todo: need to set correct center channel */
+	SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(operation, 0);
+
+	_rtw_memcpy(operation + 3, pvhtpriv->vht_mcs_map, 2);
+
+	rtw_set_ie(pbuf, EID_VHTOperation, 5, operation, &len);
+
+	return len;
+}
+
+u32	rtw_build_vht_op_mode_notify_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+						u8 *pbuf, u8 bw)
+{
+	/* struct registry_priv *pregistrypriv = &padapter->registrypriv; */
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv	*pvhtpriv = &pmlmepriv->vhtpriv;
+	u32	len = 0;
+	u8	opmode = 0;
+	u8	chnl_width, rx_nss;
+
+	chnl_width = bw;
+	rx_nss = rtw_vht_mcsmap_to_nss(pvhtpriv->vht_mcs_map);
+
+
+	if (chnl_width == CHANNEL_WIDTH_160) {
+		SET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(&opmode, 2);
+		SET_VHT_OPERATING_MODE_FIELD_80_80_160_BW(&opmode, 1);
+	} else
+		SET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(&opmode, chnl_width);
+	SET_VHT_OPERATING_MODE_FIELD_RX_NSS(&opmode, (rx_nss - 1));
+	SET_VHT_OPERATING_MODE_FIELD_RX_NSS_TYPE(&opmode, 0); /* Todo */
+
+	pvhtpriv->vht_op_mode_notify = opmode;
+
+	pbuf = rtw_set_ie(pbuf, EID_OpModeNotification, 1, &opmode, &len);
+
+	return len;
+}
+
+u32	rtw_build_vht_cap_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *pbuf)
+{
+	u8 bw;
+	u16 HighestRate;
+	u8 *pcap, *pcap_mcs;
+	u32 len = 0;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv *pvhtpriv = &pmlmepriv->vhtpriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	pcap = pvhtpriv->vht_cap;
+	_rtw_memset(pcap, 0, 32);
+
+	/*
+	* VHT Capabilities Information field : B0 to B31
+	*/
+
+	/* B0 B1 Maximum MPDU Length */
+	SET_VHT_CAPABILITY_ELE_MAX_MPDU_LENGTH(pcap, pvhtpriv->max_mpdu_len);
+
+	/* B2 B3 Supported Channel Width Set */
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160)) {
+		if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80_80M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_80_80)) {
+			SET_VHT_CAPABILITY_ELE_CHL_WIDTH(pcap, 2);
+			/* RTW_INFO("[VHT] Declare supporting 160MHz and 80+80MHz\n"); */
+		} else {
+			SET_VHT_CAPABILITY_ELE_CHL_WIDTH(pcap, 1);
+			/* RTW_INFO("[VHT] Declare supporting 160MHz\n"); */
+		}
+	} else
+		SET_VHT_CAPABILITY_ELE_CHL_WIDTH(pcap, 0);
+
+	/* B4 Rx LDPC */
+	if (TEST_FLAG(pvhtpriv->ldpc_cap, LDPC_VHT_ENABLE_RX)) {
+		SET_VHT_CAPABILITY_ELE_RX_LDPC(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting RX LDPC\n"); */
+	}
+
+	/* B5 ShortGI for 80MHz */
+	SET_VHT_CAPABILITY_ELE_SHORT_GI80M(pcap, pvhtpriv->sgi_80m ? 1 : 0);
+	/*
+	if (pvhtpriv->sgi_80m)
+		RTW_INFO("[VHT] Declare supporting SGI 80MHz\n");
+	*/
+
+	/* B6 Short GI for 160 and 80+80 MHz */
+	SET_VHT_CAPABILITY_ELE_SHORT_GI160M(pcap, pvhtpriv->sgi_160m ? 1 : 0);
+	/*
+	if (pvhtpriv->sgi_160m) {
+		RTW_INFO("[VHT] Declare supporting SGI 160MHz and 80+80MHz\n");
+	}
+	*/
+
+	/* B7 Tx STBC */
+	if (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_TX)) {
+		SET_VHT_CAPABILITY_ELE_TX_STBC(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting TX STBC\n"); */
+	}
+
+	/* B8 B9 B10 Rx STBC */
+	if (TEST_FLAG(pvhtpriv->stbc_cap, STBC_VHT_ENABLE_RX)) {
+		SET_VHT_CAPABILITY_ELE_RX_STBC(pcap, pvhtpriv->rx_stbc_nss);
+		/* RTW_INFO("[VHT] Declare supporting RX STBC = %d\n", pvhtpriv->rx_stbc_nss); */
+	}
+
+	#ifdef CONFIG_BEAMFORMING
+	/* B11 SU Beamformer Capable */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMER_ENABLE)) {
+		SET_VHT_CAPABILITY_ELE_SU_BFER(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting SU Beamformer\n"); */
+
+		/* B16 17 18 Number of Sounding Dimensions */
+		SET_VHT_CAPABILITY_ELE_SOUNDING_DIMENSIONS(pcap, pvhtpriv->num_snd_dim);
+
+		/* B19 MU Beamformer Capable */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_AP_ENABLE)) {
+			SET_VHT_CAPABILITY_ELE_MU_BFER(pcap, 1);
+			/* RTW_INFO("[VHT] Declare supporting MU Beamformer\n"); */
+		}
+	}
+
+	/* B12 SU Beamformee Capable */
+	if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_BEAMFORMEE_ENABLE)) {
+		u8 bfme_sts = pvhtpriv->bfme_sts;
+
+		SET_VHT_CAPABILITY_ELE_SU_BFEE(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting SU Beamformee\n"); */
+
+		/* IOT action suggested by Yu Chen 2017/3/3 */
+		if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_BROADCOM) &&
+			!pvhtpriv->ap_bf_cap.is_mu_bfer &&
+			pvhtpriv->ap_bf_cap.su_sound_dim == 2)
+			bfme_sts = (bfme_sts >= 2 ? 2 : bfme_sts);
+
+		/* B13 14 15 Beamformee STS Capability */
+		SET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(pcap, bfme_sts);
+
+		/* B20 MU Beamformee Capable */
+		if (TEST_FLAG(pvhtpriv->beamform_cap, BEAMFORMING_VHT_MU_MIMO_STA_ENABLE)) {
+			SET_VHT_CAPABILITY_ELE_MU_BFEE(pcap, 1);
+			/* RTW_INFO("[VHT] Declare supporting MU Beamformee\n"); */
+		}
+	}
+	#endif/*CONFIG_BEAMFORMING*/
+
+	/* B21 VHT TXOP PS */
+	if (pvhtpriv->txop_ps) {
+		SET_VHT_CAPABILITY_ELE_TXOP_PS(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting VHT TXOP power save\n"); */
+	}
+
+	/* B22 +HTC-VHT Capable */
+	if (pvhtpriv->htc_vht) {
+		SET_VHT_CAPABILITY_ELE_HTC_VHT(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting VHT variant HT Control\n"); */
+	}
+
+	/* B23 24 25 Maximum A-MPDU Length Exponent */
+	SET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(pcap, pvhtpriv->ampdu_len);
+	/* RTW_INFO("[VHT] Declare supporting RX A-MPDU Length Exponent = %u\n", pvhtpriv->ampdu_len); */
+
+	/* B26 27 VHT Link Adaptation Capable */
+	if (pvhtpriv->link_adap_cap) {
+		SET_VHT_CAPABILITY_ELE_LINK_ADAPTION(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting link adaptation using VHT variant HT Control\n"); */
+	}
+
+	/* B28 Rx Antenna Pattern Consistency */
+	if (pvhtpriv->rx_ant_pattern) {
+		SET_VHT_CAPABILITY_ELE_RX_ANT_PATTERN(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting RX Antenna Pattern Consistency\n"); */
+	}
+
+	/* B29 Tx Antenna Pattern Consistency */
+	if (pvhtpriv->tx_ant_pattern) {
+		SET_VHT_CAPABILITY_ELE_TX_ANT_PATTERN(pcap, 1);
+		/* RTW_INFO("[VHT] Declare supporting TX Antenna Pattern Consistency\n"); */
+	}
+
+	/* B30 B31 Extended NSS BW Support */
+	SET_VHT_CAPABILITY_ELE_EXT_NSS_BW(pcap, pvhtpriv->ext_nss_bw);
+
+	/*
+	* Supported VHT-MCS and NSS Set : 8 bytes
+	*/
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_RX_MCS(pcap);
+	_rtw_memcpy(pcap_mcs, pvhtpriv->vht_mcs_map, 2);
+
+	pcap_mcs = GET_VHT_CAPABILITY_ELE_TX_MCS(pcap);
+	_rtw_memcpy(pcap_mcs, pvhtpriv->vht_mcs_map, 2);
+
+	/* find the largest bw supported by both registry and hal */
+	bw = rtw_hw_largest_bw(adapter_to_dvobj(padapter), REGSTY_BW_5G(pregistrypriv));
+
+	if(bw >= ARRAY_SIZE(VHT_MCS_DATA_RATE)){
+		RTW_WARN("BW parameter value is out of range:%u\n", bw);
+		bw = ARRAY_SIZE(VHT_MCS_DATA_RATE) - 1;
+	}
+	HighestRate = VHT_MCS_DATA_RATE[bw][0][((pvhtpriv->vht_highest_rate - MGN_VHT1SS_MCS0) & 0x3f)];
+	HighestRate = (HighestRate + 1) >> 1;
+
+	SET_VHT_CAPABILITY_ELE_MCS_RX_HIGHEST_RATE(pcap, HighestRate); /* indicate we support highest rx rate is 600Mbps. */
+	SET_VHT_CAPABILITY_ELE_MCS_TX_HIGHEST_RATE(pcap, HighestRate); /* indicate we support highest tx rate is 600Mbps. */
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_160M) && REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_160))
+		SET_VHT_CAPABILITY_ELE_MCS_TX_EXT_NSS_BW_CAP(pcap, 1); /*160M*/
+
+	pbuf = rtw_set_ie(pbuf, EID_VHTCapability, 12, pcap, &len);
+
+	return len;
+}
+
+u8 rtw_vht_get_oper_bw(u8 *ht_op_ie, u8 *vht_cap_ie, u8 *vht_op_ie, u8 oper_bw)
+{
+	u8 ccfs0 = 0;
+	u8 ccfs1 = 0;
+	u8 ccfs2 = 0;
+
+	if (GET_VHT_OPERATION_ELE_CHL_WIDTH(vht_op_ie + 2) >= CH_WIDTH_80_160M)
+		oper_bw = CHANNEL_WIDTH_80;
+
+	/*Ref 802.11-2020.pdf
+	  * Table 9-272 Setting of the Supported Channel Width Set subfield and Extended NSS BW
+	  *	Support subfield at a STA transmitting the VHT Capabilities Information field
+	  * Table 11-23 VHT BSS bandwidth
+	  * Table 11-25 Extended NSS channel width
+	  */
+	if (GET_VHT_CAPABILITY_ELE_CHL_WIDTH(vht_cap_ie + 2) == 1 &&  GET_VHT_CAPABILITY_ELE_EXT_NSS_BW(vht_cap_ie + 2) == 0) {
+		if (oper_bw == CHANNEL_WIDTH_80) {
+			ccfs0 = GET_VHT_OPERATION_ELE_CENTER_FREQ1(vht_op_ie + 2);
+			ccfs1 = GET_VHT_OPERATION_ELE_CENTER_FREQ2(vht_op_ie + 2);
+			if (ccfs1 > 0 && (ccfs1 - ccfs0 == 8 || ccfs0 - ccfs1 == 8) )
+				oper_bw = CHANNEL_WIDTH_160;
+		}
+	} else if (GET_VHT_CAPABILITY_ELE_CHL_WIDTH(vht_cap_ie + 2) == 0 &&  GET_VHT_CAPABILITY_ELE_EXT_NSS_BW(vht_cap_ie + 2) == 1) {
+		if (oper_bw == CHANNEL_WIDTH_80) {
+			ccfs0 = GET_VHT_OPERATION_ELE_CENTER_FREQ1(vht_op_ie + 2);
+			ccfs1 = GET_VHT_OPERATION_ELE_CENTER_FREQ2(vht_op_ie + 2);
+			if (ht_op_ie != NULL)
+				ccfs2 = GET_HT_OP_ELE_CHL_CENTER_FREQ_SEGMENT2(ht_op_ie + 2);
+			else
+				return oper_bw;
+			if (ccfs1 == 0 &&  (ccfs2 - ccfs0 == 8 || ccfs0 - ccfs2 == 8))
+				oper_bw = CHANNEL_WIDTH_160;
+		}
+	}
+
+	return oper_bw;
+}
+
+
+u32 rtw_restructure_vht_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+					u8 *in_ie, u8 *out_ie, uint in_len, uint *pout_len, enum band_type band)
+{
+	u32	ielen;
+	u8 max_bw = CHANNEL_WIDTH_20;
+	u8 oper_ch, oper_bw = CHANNEL_WIDTH_20, oper_offset = CHAN_OFFSET_NO_EXT;
+	u8 *out_vht_op_ie, *ht_op_ie, *vht_cap_ie, *vht_op_ie;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv	*pvhtpriv = &pmlmepriv->vhtpriv;
+
+	rtw_vht_get_real_setting(padapter, padapter_link);
+
+	ht_op_ie = rtw_get_ie(in_ie + 12, WLAN_EID_HT_OPERATION, &ielen, in_len - 12);
+	if (!ht_op_ie || ielen != HT_OP_IE_LEN)
+		goto exit;
+	vht_cap_ie = rtw_get_ie(in_ie + 12, EID_VHTCapability, &ielen, in_len - 12);
+	if (!vht_cap_ie || ielen != VHT_CAP_IE_LEN)
+		goto exit;
+	vht_op_ie = rtw_get_ie(in_ie + 12, EID_VHTOperation, &ielen, in_len - 12);
+	if (!vht_op_ie || ielen != VHT_OP_IE_LEN)
+		goto exit;
+
+	/* VHT Capabilities element */
+	*pout_len += rtw_build_vht_cap_ie(padapter, padapter_link, out_ie + *pout_len);
+
+
+	/* VHT Operation element */
+	out_vht_op_ie = out_ie + *pout_len;
+	rtw_set_ie(out_vht_op_ie, EID_VHTOperation, VHT_OP_IE_LEN, vht_op_ie + 2 , pout_len);
+
+	/* get primary channel from HT_OP_IE */
+	oper_ch = GET_HT_OP_ELE_PRI_CHL(ht_op_ie + 2);
+
+	/* find the largest bw supported by both registry and hal */
+	if (band == BAND_ON_24G)
+		max_bw = REGSTY_BW_2G(pregistrypriv);
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G)
+		max_bw = REGSTY_BW_5G(pregistrypriv);
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G)
+		max_bw = REGSTY_BW_6G(pregistrypriv);
+#endif
+	else
+		rtw_warn_on(1);
+	max_bw = rtw_hw_largest_bw(adapter_to_dvobj(padapter), max_bw);
+
+	if (max_bw >= CHANNEL_WIDTH_40) {
+		/* get bw offset form HT_OP_IE */
+		if (GET_HT_OP_ELE_STA_CHL_WIDTH(ht_op_ie + 2)) {
+			switch (GET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op_ie + 2)) {
+			case IEEE80211_SCA:
+				oper_bw = CHANNEL_WIDTH_40;
+				oper_offset = CHAN_OFFSET_UPPER;
+				break;
+			case IEEE80211_SCB:
+				oper_bw = CHANNEL_WIDTH_40;
+				oper_offset = CHAN_OFFSET_LOWER;
+				break;
+			}
+		}
+
+		if (oper_bw == CHANNEL_WIDTH_40) {
+			oper_bw = rtw_vht_get_oper_bw(ht_op_ie, vht_cap_ie,
+						      vht_op_ie, oper_bw);
+			oper_bw = rtw_min(oper_bw, max_bw);
+		}
+	}
+
+	/* try downgrage bw to fit in channel plan setting */
+	oper_bw = alink_adjust_linking_bw_by_regd(padapter_link, band, oper_ch, oper_bw, oper_offset);
+	if (oper_bw == CHANNEL_WIDTH_20)
+		oper_offset = CHAN_OFFSET_NO_EXT;
+
+	/* update VHT_OP_IE */
+	if (oper_bw < CHANNEL_WIDTH_80) {
+		SET_VHT_OPERATION_ELE_CHL_WIDTH(out_vht_op_ie + 2, CH_WIDTH_20_40M);
+		SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(out_vht_op_ie + 2, 0);
+		SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(out_vht_op_ie + 2, 0);
+	} else if (oper_bw == CHANNEL_WIDTH_80) {
+		u8 cch = rtw_get_center_ch_by_band(band, oper_ch, oper_bw, oper_offset);
+
+		SET_VHT_OPERATION_ELE_CHL_WIDTH(out_vht_op_ie + 2, CH_WIDTH_80_160M);
+		SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(out_vht_op_ie + 2, cch);
+		SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(out_vht_op_ie + 2, 0);
+
+	} else if (oper_bw == CHANNEL_WIDTH_160) {
+		/* Need add VHT operation setting for 160M ? */
+	} else {
+		RTW_ERR(FUNC_ADPT_FMT" unsupported BW:%u\n", FUNC_ADPT_ARG(padapter), oper_bw);
+		rtw_warn_on(1);
+	}
+
+	/* Operating Mode Notification element */
+	*pout_len += rtw_build_vht_op_mode_notify_ie(padapter, padapter_link, out_ie + *pout_len, oper_bw);
+
+	pvhtpriv->vht_option = _TRUE;
+
+exit:
+	return pvhtpriv->vht_option;
+
+}
+
+void VHTOnAssocRsp(_adapter *padapter)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct vht_priv		*pvhtpriv = &pmlmepriv->vhtpriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	u8	ht_AMPDU_len;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	if (!pmlmeinfo->HT_enable)
+		return;
+
+	if (!pmlmeinfo->VHT_enable)
+		return;
+
+	ht_AMPDU_len = pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x03;
+
+	if (pvhtpriv->ampdu_len > ht_AMPDU_len)
+		rtw_hal_set_hwreg(padapter, HW_VAR_AMPDU_FACTOR, (u8 *)(&pvhtpriv->ampdu_len));
+
+	rtw_hal_set_hwreg(padapter, HW_VAR_AMPDU_MAX_TIME, (u8 *)(&pvhtpriv->vht_highest_rate));
+}
+
+void rtw_vht_ies_attach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			WLAN_BSSID_EX *pnetwork)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	u8 cap_len, operation_len;
+	uint len = 0;
+	sint ie_len = 0;
+	u8 *p = NULL;
+
+	p = rtw_get_ie(pnetwork->IEs + _BEACON_IE_OFFSET_, EID_VHTCapability, &ie_len,
+			(pnetwork->IELength - _BEACON_IE_OFFSET_));
+	if (p && ie_len > 0)
+		return;
+
+	rtw_vht_get_real_setting(padapter, padapter_link);
+
+	/* VHT Operation mode notifiy bit in Extended IE (127) */
+	rtw_add_ext_cap_info(pmlmepriv->ext_capab_ie_data, &(pmlmepriv->ext_capab_ie_len), OP_MODE_NOTIFICATION);
+	rtw_update_ext_cap_ie(pmlmepriv->ext_capab_ie_data, pmlmepriv->ext_capab_ie_len, pnetwork->IEs \
+		, &(pnetwork->IELength), _BEACON_IE_OFFSET_);
+
+	/* VHT Capabilities element */
+	cap_len = rtw_build_vht_cap_ie(padapter, padapter_link, pnetwork->IEs + pnetwork->IELength);
+	pnetwork->IELength += cap_len;
+
+	/* VHT Operation element */
+	operation_len = rtw_build_vht_operation_ie(padapter, padapter_link, pnetwork->IEs + pnetwork->IELength,
+						pnetwork->Configuration.Band, pnetwork->Configuration.DSConfig);
+	pnetwork->IELength += operation_len;
+
+	rtw_check_for_vht20(padapter, pnetwork->IEs + _BEACON_IE_OFFSET_, pnetwork->IELength - _BEACON_IE_OFFSET_);
+
+	pmlmepriv->vhtpriv.vht_option = _TRUE;
+}
+
+void rtw_vht_ies_detach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			WLAN_BSSID_EX *pnetwork)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+
+	rtw_remove_ext_cap_info(pmlmepriv->ext_capab_ie_data, &(pmlmepriv->ext_capab_ie_len), OP_MODE_NOTIFICATION);
+	rtw_update_ext_cap_ie(pmlmepriv->ext_capab_ie_data, pmlmepriv->ext_capab_ie_len, pnetwork->IEs \
+		, &(pnetwork->IELength), _BEACON_IE_OFFSET_);
+
+	rtw_remove_bcn_ie(padapter, pnetwork, EID_VHTCapability);
+	rtw_remove_bcn_ie(padapter, pnetwork, EID_VHTOperation);
+	rtw_remove_bcn_ie(padapter, pnetwork, EID_VHTTransmitPower);
+
+	pmlmepriv->vhtpriv.vht_option = _FALSE;
+}
+
+void rtw_check_for_vht20(_adapter *adapter, u8 *ies, int ies_len)
+{
+	u8 ht_ch, ht_bw, ht_offset;
+	u8 vht_ch, vht_bw, vht_offset;
+
+	rtw_ies_get_bchbw(ies, ies_len, NULL, &ht_ch, &ht_bw, &ht_offset, NULL, NULL, 1, 0, 0);
+	rtw_ies_get_bchbw(ies, ies_len, NULL, &vht_ch, &vht_bw, &vht_offset, NULL, NULL, 1, 1, 0);
+
+	if (ht_bw == CHANNEL_WIDTH_20 && vht_bw >= CHANNEL_WIDTH_80) {
+		u8 *vht_op_ie;
+		int vht_op_ielen;
+
+		RTW_INFO(FUNC_ADPT_FMT" vht80 is not allowed without ht40\n", FUNC_ADPT_ARG(adapter));
+		vht_op_ie = rtw_get_ie(ies, EID_VHTOperation, &vht_op_ielen, ies_len);
+		if (vht_op_ie && vht_op_ielen) {
+			RTW_INFO(FUNC_ADPT_FMT" switch to vht20\n", FUNC_ADPT_ARG(adapter));
+			SET_VHT_OPERATION_ELE_CHL_WIDTH(vht_op_ie + 2, CH_WIDTH_20_40M);
+			SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(vht_op_ie + 2, 0);
+			SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(vht_op_ie + 2, 0);
+		}
+	}
+}
+
+/* We need to update the (mlmepriv->vhtpriv) */
+void rtw_update_drv_vht_cap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *vht_cap_ie)
+{
+	/* Initialize VHT capability element */
+	rtw_vht_get_real_setting(padapter, padapter_link);
+
+	RTW_INFO("Don't setting VHT capability IE from hostap, builded by driver temporarily\n");
+	rtw_build_vht_cap_ie(padapter, padapter_link, vht_cap_ie);
+}
+
+void rtw_check_vht_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			WLAN_BSSID_EX *pnetwork)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct vht_priv *pvhtpriv = &(pmlmepriv->vhtpriv);
+	u32 ie_len;
+	u32 ies_len = pnetwork->IELength - _BEACON_IE_OFFSET_;
+	u8 *ies = pnetwork->IEs + _BEACON_IE_OFFSET_;
+	u8 *vht_cap_ie, *vht_op_ie;
+
+	vht_cap_ie = rtw_get_ie(ies, EID_VHTCapability, &ie_len, ies_len);
+
+	vht_op_ie = rtw_get_ie(ies, EID_VHTOperation, &ie_len, ies_len);
+
+	rtw_update_drv_vht_cap(padapter, padapter_link, vht_cap_ie);
+
+	rtw_add_ext_cap_info(pmlmepriv->ext_capab_ie_data, &(pmlmepriv->ext_capab_ie_len), OP_MODE_NOTIFICATION);
+	rtw_update_ext_cap_ie(pmlmepriv->ext_capab_ie_data, pmlmepriv->ext_capab_ie_len, pnetwork->IEs \
+		, &(pnetwork->IELength), _BEACON_IE_OFFSET_);
+
+	/* Backup these two VHT IEs from hostapd/wpa_supplicant for restore usage */
+	if (vht_cap_ie != NULL)
+		_rtw_memcpy(pvhtpriv->vht_cap_ie_backup, vht_cap_ie + 2, VHT_CAP_IE_LEN);
+
+	if (vht_op_ie != NULL)
+		_rtw_memcpy(pvhtpriv->vht_op_ie_backup, vht_op_ie + 2, VHT_OP_IE_LEN);
+
+	/* TODO : We don't handle this IE like before, so remove it */
+	rtw_remove_bcn_ie(padapter, pnetwork, EID_VHTTransmitPower);
+}
+
+void rtw_update_probe_rsp_vht_cap(struct _ADAPTER *a, u8 *ies, sint ies_len)
+{
+	struct _ADAPTER_LINK *a_link = GET_PRIMARY_LINK(a);
+	u8 *vht_cap_ie;
+	sint ie_len;
+
+	vht_cap_ie = rtw_get_ie(ies, WLAN_EID_VHT_CAPABILITY, &ie_len, ies_len);
+	if (vht_cap_ie) {
+		rtw_vht_get_real_setting(a, a_link);
+		rtw_build_vht_cap_ie(a, a_link, vht_cap_ie);
+	}
+}
+
+void rtw_reattach_vht_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork)
+{
+	struct link_mlme_priv *pmlmepriv = &(padapter_link->mlmepriv);
+	struct vht_priv *pvhtpriv = &(pmlmepriv->vhtpriv);
+	u8 *vht_cap_ie = pnetwork->IEs + pnetwork->IELength;
+	u8 *vht_op_ie;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+	vht_op_ie = rtw_set_ie(vht_cap_ie, EID_VHTCapability, VHT_CAP_IE_LEN,
+		pvhtpriv->vht_cap_ie_backup, &(pnetwork->IELength));
+
+	rtw_set_ie(vht_op_ie, EID_VHTOperation, VHT_OP_IE_LEN,
+		pvhtpriv->vht_op_ie_backup, &(pnetwork->IELength));
+
+	rtw_add_ext_cap_info(pmlmepriv->ext_capab_ie_data, &(pmlmepriv->ext_capab_ie_len), OP_MODE_NOTIFICATION);
+	rtw_update_ext_cap_ie(pmlmepriv->ext_capab_ie_data, pmlmepriv->ext_capab_ie_len, pnetwork->IEs \
+	, &(pnetwork->IELength), _BEACON_IE_OFFSET_);
+
+	pmlmepriv->vhtpriv.vht_option = _TRUE;
+}
+
+/* rx_nss: number of rx spatial stream to be used  */
+u8 _issue_op_mode_notify_frame(_adapter *a, struct _ADAPTER_LINK *a_link,
+		u8 *ra, u8 rx_nss, enum channel_width bw, u8 wait_ack)
+{
+	int ret = _FAIL;
+	u8 category = RTW_WLAN_CATEGORY_VHT;
+	u8 action = RTW_WLAN_ACTION_VHT_OPMODE_NOTIFICATION;
+	struct xmit_frame *mgntframe;
+	struct pkt_attrib *attrib;
+	u8 *frame;
+	struct rtw_ieee80211_hdr *wlanhdr;
+	u16 *fctrl;
+	struct link_mlme_ext_info *mlmeinfo = &(a_link->mlmeextpriv.mlmext_info);
+	struct xmit_priv *xmitpriv = &(a->xmitpriv);
+	struct mlme_ext_priv *mlmeext = &(a->mlmeextpriv);
+	u8 opmode = 0, op_bw, op_80_80_bw;
+	u32 ie_len;
+
+	if (alink_regu_block_tx(a_link)) {
+		ret = _FALSE;
+		goto exit;
+	}
+
+	mgntframe = alloc_mgtxmitframe(xmitpriv);
+	if (mgntframe == NULL) {
+		ret = _FALSE;
+		goto exit;
+	}
+
+	/* update attribute */
+	attrib = &mgntframe->attrib;
+	update_mgntframe_attrib(a, a_link, attrib);
+
+	_rtw_memset(mgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	frame = (u8 *)(mgntframe->buf_addr) + TXDESC_OFFSET;
+	wlanhdr = (struct rtw_ieee80211_hdr *)frame;
+
+	fctrl = &(wlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	/*RA*/
+	_rtw_memcpy(wlanhdr->addr1, ra, ETH_ALEN);
+	/*TA*/
+	_rtw_memcpy(wlanhdr->addr2, a_link->mac_addr, ETH_ALEN);
+	/*BSSID*/
+	_rtw_memcpy(wlanhdr->addr3, get_my_bssid(&(mlmeinfo->network)), ETH_ALEN);
+
+	SetSeqNum(wlanhdr, mlmeext->mgnt_seq);
+	mlmeext->mgnt_seq++;
+	set_frame_sub_type(frame, WIFI_ACTION);
+
+	frame += sizeof(struct rtw_ieee80211_hdr_3addr);
+	attrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	/* category, action */
+	frame = rtw_set_fixed_ie(frame, 1, &category, &attrib->pktlen);
+	frame = rtw_set_fixed_ie(frame, 1, &action, &attrib->pktlen);
+
+	/* Operating Mode field */
+	switch (bw) {
+	case CHANNEL_WIDTH_20:
+		op_bw = 0;
+		op_80_80_bw = 0;
+		break;
+	case CHANNEL_WIDTH_40:
+		op_bw = 1;
+		op_80_80_bw = 0;
+		break;
+	case CHANNEL_WIDTH_80:
+		op_bw = 2;
+		op_80_80_bw = 0;
+		break;
+	case CHANNEL_WIDTH_160:
+	case CHANNEL_WIDTH_80_80:
+		op_bw = 2;
+		op_80_80_bw = 1;
+		break;
+	default:
+		RTW_INFO("%s bw(%d) not support\n", __func__, bw);
+		ret = _FAIL;
+		goto exit;
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT ": op_bw:%d, op_80_80_bw=%d, rx_nss=%d\n",
+		FUNC_ADPT_ARG(a), op_bw, op_80_80_bw, rx_nss);
+	SET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(&opmode, op_bw);
+	SET_VHT_OPERATING_MODE_FIELD_80_80_160_BW(&opmode, op_80_80_bw);
+	SET_VHT_OPERATING_MODE_FIELD_RX_NSS(&opmode, (rx_nss - 1));
+	SET_VHT_OPERATING_MODE_FIELD_RX_NSS_TYPE(&opmode, 0);
+	frame = rtw_set_fixed_ie(frame, 1, &opmode, &attrib->pktlen);
+
+	attrib->last_txcmdsz = attrib->pktlen;
+
+	if (wait_ack)
+		ret = dump_mgntframe_and_wait_ack(a, mgntframe);
+	else {
+		dump_mgntframe(a, mgntframe);
+		ret = _SUCCESS;
+	}
+
+	if (ret != _SUCCESS)
+		RTW_INFO("%s, ack to\n", __func__);
+
+exit:
+	return ret;
+}
+
+/* rx_nss: number of rx spatial stream to be used  */
+u8 rtw_issue_op_mode_notify_frame(_adapter *a, struct _ADAPTER_LINK *a_link,
+		u8 *ra, u8 rx_nss, enum channel_width bw, u8 try_cnt, u8 wait_ms)
+{
+	int ret = _FAIL;
+	int i = 0;
+	systime start = rtw_get_current_time();
+
+	if (alink_regu_block_tx(a_link))
+		goto exit;
+
+	do {
+		ret = _issue_op_mode_notify_frame(a, a_link, ra, rx_nss, bw, wait_ms > 0 ? _TRUE : _FALSE);
+
+		i++;
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(a)))
+			break;
+
+		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
+			rtw_msleep_os(wait_ms);
+
+	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
+
+	if (ret != _FAIL) {
+		ret = _SUCCESS;
+#ifndef DBG_XMIT_ACK
+		goto exit;
+#endif
+	}
+
+	if (try_cnt && wait_ms) {
+		if (ra)
+			RTW_INFO(FUNC_ADPT_FMT" to "MAC_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(a), MAC_ARG(ra),
+				rtw_get_oper_ch(a, a_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+		else
+			RTW_INFO(FUNC_ADPT_FMT", ch:%u%s, %d/%d in %u ms\n",
+				FUNC_ADPT_ARG(a),
+				rtw_get_oper_ch(a, a_link),
+				ret == _SUCCESS ? ", acked" : "", i, try_cnt, rtw_get_passing_time_ms(start));
+	}
+
+exit:
+	return ret;
+}
+
+void rtw_vht_op_mode_ctrl_rx_nss(_adapter *adapter, struct _ADAPTER_LINK *a_link,
+									struct sta_info *sta, u8 final_rx_nss, bool need_update_ra)
+{
+	struct link_mlme_ext_info *a_mlmeinfo = &(a_link->mlmeextpriv.mlmext_info);
+	struct rtw_chan_def *a_ch_def = &a_link->mlmeextpriv.chandef;
+
+	rtw_issue_op_mode_notify_frame(adapter, a_link, get_my_bssid(&(a_mlmeinfo->network)),
+		                                          final_rx_nss, a_ch_def->bw, 3, 10);
+	if (need_update_ra)
+		rtw_sta_hal_ra_mask_update_cmd(adapter, sta, RTW_CMDF_DIRECTLY);
+}
+
+#endif /* CONFIG_80211AC_VHT */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wapi.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wapi.c
new file mode 100644
index 000000000000..f9fbc43dff84
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wapi.c
@@ -0,0 +1,1317 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifdef CONFIG_WAPI_SUPPORT
+
+#include <linux/unistd.h>
+#include <linux/etherdevice.h>
+#include <drv_types.h>
+#include <rtw_wapi.h>
+
+
+u32 wapi_debug_component =
+	/*				WAPI_INIT	|
+	 *				WAPI_API	|
+	 *				WAPI_TX	|
+	 *				WAPI_RX	| */
+	WAPI_ERR ; /* always open err flags on */
+
+void WapiFreeAllStaInfo(_adapter *padapter)
+{
+	PRT_WAPI_T				pWapiInfo;
+	PRT_WAPI_STA_INFO		pWapiStaInfo;
+	PRT_WAPI_BKID			pWapiBkid;
+
+	WAPI_TRACE(WAPI_INIT, "===========> %s\n", __FUNCTION__);
+	pWapiInfo = &padapter->wapiInfo;
+
+	/* Pust to Idle List */
+	rtw_wapi_return_all_sta_info(padapter);
+
+	/* Sta Info List */
+	while (!list_empty(&(pWapiInfo->wapiSTAIdleList))) {
+		pWapiStaInfo = (PRT_WAPI_STA_INFO)list_entry(pWapiInfo->wapiSTAIdleList.next, RT_WAPI_STA_INFO, list);
+		list_del_init(&pWapiStaInfo->list);
+	}
+
+	/* BKID List */
+	while (!list_empty(&(pWapiInfo->wapiBKIDIdleList))) {
+		pWapiBkid = (PRT_WAPI_BKID)list_entry(pWapiInfo->wapiBKIDIdleList.next, RT_WAPI_BKID, list);
+		list_del_init(&pWapiBkid->list);
+	}
+	WAPI_TRACE(WAPI_INIT, "<=========== %s\n", __FUNCTION__);
+	return;
+}
+
+void WapiSetIE(_adapter *padapter)
+{
+	PRT_WAPI_T		pWapiInfo = &(padapter->wapiInfo);
+	/* PRT_WAPI_BKID	pWapiBkid; */
+	u16		protocolVer = 1;
+	u16		akmCnt = 1;
+	u16		suiteCnt = 1;
+	u16		capability = 0;
+	u8		OUI[3];
+
+	OUI[0] = 0x00;
+	OUI[1] = 0x14;
+	OUI[2] = 0x72;
+
+	pWapiInfo->wapiIELength = 0;
+	/* protocol version */
+	_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, &protocolVer, 2);
+	pWapiInfo->wapiIELength += 2;
+	/* akm */
+	_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, &akmCnt, 2);
+	pWapiInfo->wapiIELength += 2;
+
+	if (pWapiInfo->bWapiPSK) {
+		_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, OUI, 3);
+		pWapiInfo->wapiIELength += 3;
+		pWapiInfo->wapiIE[pWapiInfo->wapiIELength] = 0x2;
+		pWapiInfo->wapiIELength += 1;
+	} else {
+		_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, OUI, 3);
+		pWapiInfo->wapiIELength += 3;
+		pWapiInfo->wapiIE[pWapiInfo->wapiIELength] = 0x1;
+		pWapiInfo->wapiIELength += 1;
+	}
+
+	/* usk */
+	_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, &suiteCnt, 2);
+	pWapiInfo->wapiIELength += 2;
+	_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, OUI, 3);
+	pWapiInfo->wapiIELength += 3;
+	pWapiInfo->wapiIE[pWapiInfo->wapiIELength] = 0x1;
+	pWapiInfo->wapiIELength += 1;
+
+	/* msk */
+	_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, OUI, 3);
+	pWapiInfo->wapiIELength += 3;
+	pWapiInfo->wapiIE[pWapiInfo->wapiIELength] = 0x1;
+	pWapiInfo->wapiIELength += 1;
+
+	/* Capbility */
+	_rtw_memcpy(pWapiInfo->wapiIE + pWapiInfo->wapiIELength, &capability, 2);
+	pWapiInfo->wapiIELength += 2;
+}
+
+
+/*  PN1 > PN2, return 1,
+ *  else return 0.
+ */
+u32 WapiComparePN(u8 *PN1, u8 *PN2)
+{
+	char i;
+
+	if ((NULL == PN1) || (NULL == PN2))
+		return 1;
+
+	/* overflow case */
+	if ((PN2[15] - PN1[15]) & 0x80)
+		return 1;
+
+	for (i = 16; i > 0; i--) {
+		if (PN1[i - 1] == PN2[i - 1])
+			continue;
+		else if (PN1[i - 1] > PN2[i - 1])
+			return 1;
+		else
+			return 0;
+	}
+
+	return 0;
+}
+
+u8
+WapiGetEntryForCamWrite(_adapter *padapter, u8 *pMacAddr, u8 KID, BOOLEAN IsMsk)
+{
+	PRT_WAPI_T		pWapiInfo = NULL;
+	/* PRT_WAPI_CAM_ENTRY	pEntry=NULL; */
+	u8 i = 0;
+	u8 ret = 0xff;
+
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	pWapiInfo =  &padapter->wapiInfo;
+
+	/* exist? */
+	for (i = 0; i < WAPI_CAM_ENTRY_NUM; i++) {
+		if (pWapiInfo->wapiCamEntry[i].IsUsed
+		    && (_rtw_memcmp(pMacAddr, pWapiInfo->wapiCamEntry[i].PeerMacAddr, ETH_ALEN) == _TRUE)
+		    && pWapiInfo->wapiCamEntry[i].keyidx == KID
+		    && pWapiInfo->wapiCamEntry[i].type == IsMsk) {
+			ret = pWapiInfo->wapiCamEntry[i].entry_idx; /* cover it */
+			break;
+		}
+	}
+
+	if (i == WAPI_CAM_ENTRY_NUM) { /* not found */
+		for (i = 0; i < WAPI_CAM_ENTRY_NUM; i++) {
+			if (pWapiInfo->wapiCamEntry[i].IsUsed == 0) {
+				pWapiInfo->wapiCamEntry[i].IsUsed = 1;
+				pWapiInfo->wapiCamEntry[i].type = IsMsk;
+				pWapiInfo->wapiCamEntry[i].keyidx = KID;
+				_rtw_memcpy(pWapiInfo->wapiCamEntry[i].PeerMacAddr, pMacAddr, ETH_ALEN);
+				ret = pWapiInfo->wapiCamEntry[i].entry_idx;
+				break;
+			}
+		}
+	}
+
+	WAPI_TRACE(WAPI_API, "<========== %s\n", __FUNCTION__);
+	return ret;
+
+	/*
+		if(RTIsListEmpty(&pWapiInfo->wapiCamIdleList)) {
+			return 0;
+		}
+
+		pEntry = (PRT_WAPI_CAM_ENTRY)RTRemoveHeadList(&pWapiInfo->wapiCamIdleList);
+		RTInsertTailList(&pWapiInfo->wapiCamUsedList, &pEntry->list);
+
+
+		return pEntry->entry_idx;*/
+}
+
+u8 WapiGetEntryForCamClear(_adapter *padapter, u8 *pPeerMac, u8 keyid, u8 IsMsk)
+{
+	PRT_WAPI_T		pWapiInfo = NULL;
+	u8		i = 0;
+
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	pWapiInfo =  &padapter->wapiInfo;
+
+	for (i = 0; i < WAPI_CAM_ENTRY_NUM; i++) {
+		if (pWapiInfo->wapiCamEntry[i].IsUsed
+		    && (_rtw_memcmp(pPeerMac, pWapiInfo->wapiCamEntry[i].PeerMacAddr, ETH_ALEN) == _TRUE)
+		    && pWapiInfo->wapiCamEntry[i].keyidx == keyid
+		    && pWapiInfo->wapiCamEntry[i].type == IsMsk) {
+			pWapiInfo->wapiCamEntry[i].IsUsed = 0;
+			pWapiInfo->wapiCamEntry[i].keyidx = 2;
+			_rtw_memset(pWapiInfo->wapiCamEntry[i].PeerMacAddr, 0, ETH_ALEN);
+
+			WAPI_TRACE(WAPI_API, "<========== %s\n", __FUNCTION__);
+			return pWapiInfo->wapiCamEntry[i].entry_idx;
+		}
+	}
+
+	WAPI_TRACE(WAPI_API, "<====WapiGetReturnCamEntry(), No this cam entry.\n");
+	return 0xff;
+	/*
+		if(RTIsListEmpty(&pWapiInfo->wapiCamUsedList)) {
+			return FALSE;
+		}
+
+		pList = &pWapiInfo->wapiCamUsedList;
+		while(pList->Flink != &pWapiInfo->wapiCamUsedList)
+		{
+			pEntry = (PRT_WAPI_CAM_ENTRY)pList->Flink;
+			if(PlatformCompareMemory(pPeerMac,pEntry->PeerMacAddr, ETHER_ADDRLEN)== 0
+				&& keyid == pEntry->keyidx)
+			{
+				RTRemoveEntryList(pList);
+				RTInsertHeadList(&pWapiInfo->wapiCamIdleList, pList);
+				return pEntry->entry_idx;
+			}
+			pList = pList->Flink;
+		}
+
+		return 0;
+	*/
+}
+
+void
+WapiResetAllCamEntry(_adapter *padapter)
+{
+	PRT_WAPI_T		pWapiInfo;
+	int				i;
+
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	pWapiInfo =  &padapter->wapiInfo;
+
+	for (i = 0; i < WAPI_CAM_ENTRY_NUM; i++) {
+		_rtw_memset(pWapiInfo->wapiCamEntry[i].PeerMacAddr, 0, ETH_ALEN);
+		pWapiInfo->wapiCamEntry[i].IsUsed = 0;
+		pWapiInfo->wapiCamEntry[i].keyidx = 2; /* invalid */
+		pWapiInfo->wapiCamEntry[i].entry_idx = 4 + i * 2;
+	}
+
+	WAPI_TRACE(WAPI_API, "<========== %s\n", __FUNCTION__);
+
+	return;
+}
+
+u8 WapiWriteOneCamEntry(
+	_adapter	*padapter,
+	u8			*pMacAddr,
+	u8			KeyId,
+	u8			EntryId,
+	u8			EncAlg,
+	u8			bGroupKey,
+	u8			*pKey
+)
+{
+	u8 retVal = 0;
+	u16 usConfig = 0;
+
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	if (EntryId >= 32) {
+		WAPI_TRACE(WAPI_ERR, "<=== CamAddOneEntry(): ulKeyId exceed!\n");
+		return retVal;
+	}
+
+	usConfig = usConfig | (0x01 << 15) | ((u16)(EncAlg) << 2) | (KeyId);
+
+	if (EncAlg == _SMS4_) {
+		if (bGroupKey == 1)
+			usConfig |= (0x01 << 6);
+		if ((EntryId % 2) == 1) /* ==0 sec key; == 1mic key */
+			usConfig |= (0x01 << 5);
+	}
+
+	write_cam(padapter, EntryId, usConfig, pMacAddr, pKey);
+
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+	return 1;
+}
+
+void rtw_wapi_init(_adapter *padapter)
+{
+	PRT_WAPI_T		pWapiInfo;
+	int				i;
+
+	WAPI_TRACE(WAPI_INIT, "===========> %s\n", __FUNCTION__);
+	RT_ASSERT_RET(padapter);
+
+	if (!padapter->WapiSupport) {
+		WAPI_TRACE(WAPI_INIT, "<========== %s, WAPI not supported!\n", __FUNCTION__);
+		return;
+	}
+
+	pWapiInfo =  &padapter->wapiInfo;
+	pWapiInfo->bWapiEnable = false;
+
+	/* Init BKID List */
+	INIT_LIST_HEAD(&pWapiInfo->wapiBKIDIdleList);
+	INIT_LIST_HEAD(&pWapiInfo->wapiBKIDStoreList);
+	for (i = 0; i < WAPI_MAX_BKID_NUM; i++)
+		list_add_tail(&pWapiInfo->wapiBKID[i].list, &pWapiInfo->wapiBKIDIdleList);
+
+	/* Init STA List */
+	INIT_LIST_HEAD(&pWapiInfo->wapiSTAIdleList);
+	INIT_LIST_HEAD(&pWapiInfo->wapiSTAUsedList);
+	for (i = 0; i < WAPI_MAX_STAINFO_NUM; i++)
+		list_add_tail(&pWapiInfo->wapiSta[i].list, &pWapiInfo->wapiSTAIdleList);
+
+	for (i = 0; i < WAPI_CAM_ENTRY_NUM; i++) {
+		pWapiInfo->wapiCamEntry[i].IsUsed = 0;
+		pWapiInfo->wapiCamEntry[i].keyidx = 2; /* invalid */
+		pWapiInfo->wapiCamEntry[i].entry_idx = 4 + i * 2;
+	}
+
+	WAPI_TRACE(WAPI_INIT, "<========== %s\n", __FUNCTION__);
+}
+
+void rtw_wapi_free(_adapter *padapter)
+{
+	WAPI_TRACE(WAPI_INIT, "===========> %s\n", __FUNCTION__);
+	RT_ASSERT_RET(padapter);
+
+	if (!padapter->WapiSupport) {
+		WAPI_TRACE(WAPI_INIT, "<========== %s, WAPI not supported!\n", __FUNCTION__);
+		return;
+	}
+
+	WapiFreeAllStaInfo(padapter);
+
+	WAPI_TRACE(WAPI_INIT, "<========== %s\n", __FUNCTION__);
+}
+
+void rtw_wapi_disable_tx(_adapter *padapter)
+{
+	WAPI_TRACE(WAPI_INIT, "===========> %s\n", __FUNCTION__);
+	RT_ASSERT_RET(padapter);
+
+	if (!padapter->WapiSupport) {
+		WAPI_TRACE(WAPI_INIT, "<========== %s, WAPI not supported!\n", __FUNCTION__);
+		return;
+	}
+
+	padapter->wapiInfo.wapiTxMsk.bTxEnable = false;
+	padapter->wapiInfo.wapiTxMsk.bSet = false;
+
+	WAPI_TRACE(WAPI_INIT, "<========== %s\n", __FUNCTION__);
+}
+
+u8 rtw_wapi_is_wai_packet(_adapter *padapter, u8 *pkt_data)
+{
+	PRT_WAPI_T pWapiInfo = &(padapter->wapiInfo);
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct security_priv   *psecuritypriv = &padapter->securitypriv;
+	PRT_WAPI_STA_INFO pWapiSta = NULL;
+	u8 WaiPkt = 0, *pTaddr, bFind = false;
+	u8 Offset_TypeWAI = 0 ;	/* (mac header len + llc length) */
+
+	WAPI_TRACE(WAPI_TX | WAPI_RX, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return 0;
+	}
+
+	Offset_TypeWAI = 24 + 6 ;
+
+	/* YJ,add,091103. Data frame may also have skb->data[30]=0x88 and skb->data[31]=0xb4. */
+	if ((pkt_data[1] & 0x40) != 0) {
+		/* RTW_INFO("data is privacy\n"); */
+		return 0;
+	}
+
+	pTaddr = get_addr2_ptr(pkt_data);
+	if (list_empty(&pWapiInfo->wapiSTAUsedList))
+		bFind = false;
+	else {
+		list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+			if (_rtw_memcmp(pTaddr, pWapiSta->PeerMacAddr, 6) == _TRUE) {
+				bFind = true;
+				break;
+			}
+		}
+	}
+
+	WAPI_TRACE(WAPI_TX | WAPI_RX, "%s: bFind=%d pTaddr="MAC_FMT"\n", __FUNCTION__, bFind, MAC_ARG(pTaddr));
+
+	if (pkt_data[0] == WIFI_QOS_DATA_TYPE)
+		Offset_TypeWAI += 2;
+
+	/* 88b4? */
+	if ((pkt_data[Offset_TypeWAI] == 0x88) && (pkt_data[Offset_TypeWAI + 1] == 0xb4)) {
+		WaiPkt = pkt_data[Offset_TypeWAI + 5];
+
+		psecuritypriv->hw_decrypted = _TRUE;
+	} else
+		WAPI_TRACE(WAPI_TX | WAPI_RX, "%s(): non wai packet\n", __FUNCTION__);
+
+	WAPI_TRACE(WAPI_TX | WAPI_RX, "%s(): Recvd WAI frame. IsWAIPkt(%d)\n", __FUNCTION__, WaiPkt);
+
+	return	WaiPkt;
+}
+
+
+void rtw_wapi_update_info(_adapter *padapter, union recv_frame *precv_frame)
+{
+	PRT_WAPI_T     pWapiInfo = &(padapter->wapiInfo);
+	struct recv_frame_hdr *precv_hdr;
+	u8	*ptr;
+	u8	*pTA;
+	u8	*pRecvPN;
+
+
+	WAPI_TRACE(WAPI_RX, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_RX, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return;
+	}
+
+	precv_hdr = &precv_frame->u.hdr;
+	ptr = precv_hdr->rx_data;
+
+	if (precv_hdr->attrib.qos == 1)
+		precv_hdr->UserPriority = GetTid(ptr);
+	else
+		precv_hdr->UserPriority = 0;
+
+	pTA = get_addr2_ptr(ptr);
+	_rtw_memcpy((u8 *)precv_hdr->WapiSrcAddr, pTA, 6);
+	pRecvPN = ptr + precv_hdr->attrib.hdrlen + 2;
+	_rtw_memcpy((u8 *)precv_hdr->WapiTempPN, pRecvPN, 16);
+
+	WAPI_TRACE(WAPI_RX, "<========== %s\n", __FUNCTION__);
+}
+
+/****************************************************************************
+TRUE-----------------Drop
+FALSE---------------- handle
+add to support WAPI to N-mode
+*****************************************************************************/
+u8 rtw_wapi_check_for_drop(
+	_adapter *padapter,
+	union recv_frame *precv_frame,
+	u8 *ehdr_ops
+)
+{
+	PRT_WAPI_T     pWapiInfo = &(padapter->wapiInfo);
+	u8			*pLastRecvPN = NULL;
+	u8			bFind = false;
+	PRT_WAPI_STA_INFO	pWapiSta = NULL;
+	u8			bDrop = false;
+	struct recv_frame_hdr *precv_hdr = &precv_frame->u.hdr;
+	u8					WapiAEPNInitialValueSrc[16] = {0x37, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+	u8					WapiAEMultiCastPNInitialValueSrc[16] = {0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+	u8					*ptr = ehdr_ops;
+	int					i;
+
+	WAPI_TRACE(WAPI_RX, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_RX, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return false;
+	}
+
+	if (precv_hdr->bIsWaiPacket != 0) {
+		if (precv_hdr->bIsWaiPacket == 0x8) {
+
+			RTW_INFO("rtw_wapi_check_for_drop: dump packet\n");
+			for (i = 0; i < 50; i++) {
+				RTW_INFO("%02X  ", ptr[i]);
+				if ((i + 1) % 8 == 0)
+					RTW_INFO("\n");
+			}
+			RTW_INFO("\n rtw_wapi_check_for_drop: dump packet\n");
+
+			for (i = 0; i < 16; i++) {
+				if (ptr[i + 27] != 0)
+					break;
+			}
+
+			if (i == 16) {
+				WAPI_TRACE(WAPI_RX, "rtw_wapi_check_for_drop: drop with zero BKID\n");
+				return true;
+			} else
+				return false;
+		} else
+			return false;
+	}
+
+	if (list_empty(&pWapiInfo->wapiSTAUsedList))
+		bFind = false;
+	else {
+		list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+			if (_rtw_memcmp(precv_hdr->WapiSrcAddr, pWapiSta->PeerMacAddr, ETH_ALEN) == _TRUE) {
+				bFind = true;
+				break;
+			}
+		}
+	}
+	WAPI_TRACE(WAPI_RX, "%s: bFind=%d prxb->WapiSrcAddr="MAC_FMT"\n", __FUNCTION__, bFind, MAC_ARG(precv_hdr->WapiSrcAddr));
+
+	if (bFind) {
+		if (IS_MCAST(precv_hdr->attrib.ra)) {
+			WAPI_TRACE(WAPI_RX, "rtw_wapi_check_for_drop: multicast case\n");
+			pLastRecvPN = pWapiSta->lastRxMulticastPN;
+		} else {
+			WAPI_TRACE(WAPI_RX, "rtw_wapi_check_for_drop: unicast case\n");
+			switch (precv_hdr->UserPriority) {
+			case 0:
+			case 3:
+				pLastRecvPN = pWapiSta->lastRxUnicastPNBEQueue;
+				break;
+			case 1:
+			case 2:
+				pLastRecvPN = pWapiSta->lastRxUnicastPNBKQueue;
+				break;
+			case 4:
+			case 5:
+				pLastRecvPN = pWapiSta->lastRxUnicastPNVIQueue;
+				break;
+			case 6:
+			case 7:
+				pLastRecvPN = pWapiSta->lastRxUnicastPNVOQueue;
+				break;
+			default:
+				WAPI_TRACE(WAPI_ERR, "%s: Unknown TID\n", __FUNCTION__);
+				break;
+			}
+		}
+
+		if (!WapiComparePN(precv_hdr->WapiTempPN, pLastRecvPN)) {
+			WAPI_TRACE(WAPI_RX, "%s: Equal PN!!\n", __FUNCTION__);
+			if (IS_MCAST(precv_hdr->attrib.ra))
+				_rtw_memcpy(pLastRecvPN, WapiAEMultiCastPNInitialValueSrc, 16);
+			else
+				_rtw_memcpy(pLastRecvPN, WapiAEPNInitialValueSrc, 16);
+			bDrop = true;
+		} else
+			_rtw_memcpy(pLastRecvPN, precv_hdr->WapiTempPN, 16);
+	}
+
+	WAPI_TRACE(WAPI_RX, "<========== %s\n", __FUNCTION__);
+	return bDrop;
+}
+
+void rtw_build_probe_resp_wapi_ie(_adapter *padapter, unsigned char *pframe, struct pkt_attrib *pattrib)
+{
+	PRT_WAPI_T pWapiInfo = &(padapter->wapiInfo);
+	u8 WapiIELength = 0;
+
+	WAPI_TRACE(WAPI_MLME, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport)  || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported!\n", __FUNCTION__);
+		return;
+	}
+
+	WapiSetIE(padapter);
+	WapiIELength = pWapiInfo->wapiIELength;
+	pframe[0] = _WAPI_IE_;
+	pframe[1] = WapiIELength;
+	_rtw_memcpy(pframe + 2, pWapiInfo->wapiIE, WapiIELength);
+	pframe += WapiIELength + 2;
+	pattrib->pktlen += WapiIELength + 2;
+
+	WAPI_TRACE(WAPI_MLME, "<========== %s\n", __FUNCTION__);
+}
+
+void rtw_build_beacon_wapi_ie(_adapter *padapter, unsigned char *pframe, struct pkt_attrib *pattrib)
+{
+	PRT_WAPI_T pWapiInfo = &(padapter->wapiInfo);
+	u8 WapiIELength = 0;
+	WAPI_TRACE(WAPI_MLME, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport)  || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported!\n", __FUNCTION__);
+		return;
+	}
+
+	WapiSetIE(padapter);
+	WapiIELength = pWapiInfo->wapiIELength;
+	pframe[0] = _WAPI_IE_;
+	pframe[1] = WapiIELength;
+	_rtw_memcpy(pframe + 2, pWapiInfo->wapiIE, WapiIELength);
+	pframe += WapiIELength + 2;
+	pattrib->pktlen += WapiIELength + 2;
+
+	WAPI_TRACE(WAPI_MLME, "<========== %s\n", __FUNCTION__);
+}
+
+void rtw_build_assoc_req_wapi_ie(_adapter *padapter, unsigned char *pframe, struct pkt_attrib *pattrib)
+{
+	PRT_WAPI_BKID		pWapiBKID;
+	u16					bkidNum;
+	PRT_WAPI_T			pWapiInfo = &(padapter->wapiInfo);
+	u8					WapiIELength = 0;
+
+	WAPI_TRACE(WAPI_MLME, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported!\n", __FUNCTION__);
+		return;
+	}
+
+	WapiSetIE(padapter);
+	WapiIELength = pWapiInfo->wapiIELength;
+	bkidNum = 0;
+	if (!list_empty(&(pWapiInfo->wapiBKIDStoreList))) {
+		list_for_each_entry(pWapiBKID, &pWapiInfo->wapiBKIDStoreList, list) {
+			bkidNum++;
+			_rtw_memcpy(pWapiInfo->wapiIE + WapiIELength + 2, pWapiBKID->bkid, 16);
+			WapiIELength += 16;
+		}
+	}
+	_rtw_memcpy(pWapiInfo->wapiIE + WapiIELength, &bkidNum, 2);
+	WapiIELength += 2;
+
+	pframe[0] = _WAPI_IE_;
+	pframe[1] = WapiIELength;
+	_rtw_memcpy(pframe + 2, pWapiInfo->wapiIE, WapiIELength);
+	pframe += WapiIELength + 2;
+	pattrib->pktlen += WapiIELength + 2;
+	WAPI_TRACE(WAPI_MLME, "<========== %s\n", __FUNCTION__);
+}
+
+void rtw_wapi_on_assoc_ok(_adapter *padapter, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	PRT_WAPI_T pWapiInfo = &(padapter->wapiInfo);
+	PRT_WAPI_STA_INFO pWapiSta;
+	u8 WapiAEPNInitialValueSrc[16] = {0x37, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+	/* u8 WapiASUEPNInitialValueSrc[16] = {0x36,0x5C,0x36,0x5C,0x36,0x5C,0x36,0x5C,0x36,0x5C,0x36,0x5C,0x36,0x5C,0x36,0x5C} ; */
+	u8 WapiAEMultiCastPNInitialValueSrc[16] = {0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+
+	WAPI_TRACE(WAPI_MLME, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return;
+	}
+
+	pWapiSta = (PRT_WAPI_STA_INFO)list_entry(pWapiInfo->wapiSTAIdleList.next, RT_WAPI_STA_INFO, list);
+	list_del_init(&pWapiSta->list);
+	list_add_tail(&pWapiSta->list, &pWapiInfo->wapiSTAUsedList);
+	_rtw_memcpy(pWapiSta->PeerMacAddr, padapter->mlmeextpriv.mlmext_info.dev_network.MacAddress, 6);
+	_rtw_memcpy(pWapiSta->lastRxMulticastPN, WapiAEMultiCastPNInitialValueSrc, 16);
+	_rtw_memcpy(pWapiSta->lastRxUnicastPN, WapiAEPNInitialValueSrc, 16);
+
+	/* For chenk PN error with Qos Data after s3: add by ylb 20111114 */
+	_rtw_memcpy(pWapiSta->lastRxUnicastPNBEQueue, WapiAEPNInitialValueSrc, 16);
+	_rtw_memcpy(pWapiSta->lastRxUnicastPNBKQueue, WapiAEPNInitialValueSrc, 16);
+	_rtw_memcpy(pWapiSta->lastRxUnicastPNVIQueue, WapiAEPNInitialValueSrc, 16);
+	_rtw_memcpy(pWapiSta->lastRxUnicastPNVOQueue, WapiAEPNInitialValueSrc, 16);
+
+	WAPI_TRACE(WAPI_MLME, "<========== %s\n", __FUNCTION__);
+}
+
+
+void rtw_wapi_return_one_sta_info(_adapter *padapter, u8 *MacAddr)
+{
+	PRT_WAPI_T				pWapiInfo;
+	PRT_WAPI_STA_INFO		pWapiStaInfo = NULL;
+	PRT_WAPI_BKID			pWapiBkid = NULL;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+
+	pWapiInfo = &padapter->wapiInfo;
+
+	WAPI_TRACE(WAPI_API, "==========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return;
+	}
+
+	if (MLME_IS_STA(padapter)) {
+		while (!list_empty(&(pWapiInfo->wapiBKIDStoreList))) {
+			pWapiBkid = (PRT_WAPI_BKID)list_entry(pWapiInfo->wapiBKIDStoreList.next, RT_WAPI_BKID, list);
+			list_del_init(&pWapiBkid->list);
+			_rtw_memset(pWapiBkid->bkid, 0, 16);
+			list_add_tail(&pWapiBkid->list, &pWapiInfo->wapiBKIDIdleList);
+		}
+	}
+
+
+	WAPI_TRACE(WAPI_API, " %s: after clear bkid\n", __FUNCTION__);
+
+
+	/* Remove STA info */
+	if (list_empty(&(pWapiInfo->wapiSTAUsedList))) {
+		WAPI_TRACE(WAPI_API, " %s: wapiSTAUsedList is null\n", __FUNCTION__);
+		return;
+	} else {
+
+		WAPI_TRACE(WAPI_API, " %s: wapiSTAUsedList is not null\n", __FUNCTION__);
+#if 0
+		pWapiStaInfo = (PRT_WAPI_STA_INFO)list_entry((pWapiInfo->wapiSTAUsedList.next), RT_WAPI_STA_INFO, list);
+
+		list_for_each_entry(pWapiStaInfo, &(pWapiInfo->wapiSTAUsedList), list) {
+
+			RTW_INFO("MAC Addr %02x-%02x-%02x-%02x-%02x-%02x\n", MacAddr[0], MacAddr[1], MacAddr[2], MacAddr[3], MacAddr[4], MacAddr[5]);
+
+
+			RTW_INFO("peer Addr %02x-%02x-%02x-%02x-%02x-%02x\n", pWapiStaInfo->PeerMacAddr[0], pWapiStaInfo->PeerMacAddr[1], pWapiStaInfo->PeerMacAddr[2], pWapiStaInfo->PeerMacAddr[3],
+				pWapiStaInfo->PeerMacAddr[4], pWapiStaInfo->PeerMacAddr[5]);
+
+			if (pWapiStaInfo == NULL) {
+				WAPI_TRACE(WAPI_API, " %s: pWapiStaInfo == NULL Case\n", __FUNCTION__);
+				return;
+			}
+
+			if (pWapiStaInfo->PeerMacAddr == NULL) {
+				WAPI_TRACE(WAPI_API, " %s: pWapiStaInfo->PeerMacAddr == NULL Case\n", __FUNCTION__);
+				return;
+			}
+
+			if (MacAddr == NULL) {
+				WAPI_TRACE(WAPI_API, " %s: MacAddr == NULL Case\n", __FUNCTION__);
+				return;
+			}
+
+			if (_rtw_memcmp(pWapiStaInfo->PeerMacAddr, MacAddr, ETH_ALEN) == _TRUE) {
+				pWapiStaInfo->bAuthenticateInProgress = false;
+				pWapiStaInfo->bSetkeyOk = false;
+				_rtw_memset(pWapiStaInfo->PeerMacAddr, 0, ETH_ALEN);
+				list_del_init(&pWapiStaInfo->list);
+				list_add_tail(&pWapiStaInfo->list, &pWapiInfo->wapiSTAIdleList);
+				break;
+			}
+
+		}
+#endif
+
+		while (!list_empty(&(pWapiInfo->wapiSTAUsedList))) {
+			pWapiStaInfo = (PRT_WAPI_STA_INFO)list_entry(pWapiInfo->wapiSTAUsedList.next, RT_WAPI_STA_INFO, list);
+
+			RTW_INFO("peer Addr %02x-%02x-%02x-%02x-%02x-%02x\n", pWapiStaInfo->PeerMacAddr[0], pWapiStaInfo->PeerMacAddr[1], pWapiStaInfo->PeerMacAddr[2], pWapiStaInfo->PeerMacAddr[3],
+				pWapiStaInfo->PeerMacAddr[4], pWapiStaInfo->PeerMacAddr[5]);
+
+			list_del_init(&pWapiStaInfo->list);
+			_rtw_memset(pWapiStaInfo->PeerMacAddr, 0, ETH_ALEN);
+			pWapiStaInfo->bSetkeyOk = 0;
+			list_add_tail(&pWapiStaInfo->list, &pWapiInfo->wapiSTAIdleList);
+		}
+
+	}
+
+	WAPI_TRACE(WAPI_API, "<========== %s\n", __FUNCTION__);
+	return;
+}
+
+void rtw_wapi_return_all_sta_info(_adapter *padapter)
+{
+	PRT_WAPI_T				pWapiInfo;
+	PRT_WAPI_STA_INFO		pWapiStaInfo;
+	PRT_WAPI_BKID			pWapiBkid;
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	pWapiInfo = &padapter->wapiInfo;
+
+	if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return;
+	}
+
+	/* Sta Info List */
+	while (!list_empty(&(pWapiInfo->wapiSTAUsedList))) {
+		pWapiStaInfo = (PRT_WAPI_STA_INFO)list_entry(pWapiInfo->wapiSTAUsedList.next, RT_WAPI_STA_INFO, list);
+		list_del_init(&pWapiStaInfo->list);
+		_rtw_memset(pWapiStaInfo->PeerMacAddr, 0, ETH_ALEN);
+		pWapiStaInfo->bSetkeyOk = 0;
+		list_add_tail(&pWapiStaInfo->list, &pWapiInfo->wapiSTAIdleList);
+	}
+
+	/* BKID List */
+	while (!list_empty(&(pWapiInfo->wapiBKIDStoreList))) {
+		pWapiBkid = (PRT_WAPI_BKID)list_entry(pWapiInfo->wapiBKIDStoreList.next, RT_WAPI_BKID, list);
+		list_del_init(&pWapiBkid->list);
+		_rtw_memset(pWapiBkid->bkid, 0, 16);
+		list_add_tail(&pWapiBkid->list, &pWapiInfo->wapiBKIDIdleList);
+	}
+	WAPI_TRACE(WAPI_API, "<========== %s\n", __FUNCTION__);
+}
+
+void CAM_empty_entry(
+	_adapter *adapter,
+	u8			ucIndex
+)
+{
+	rtw_hal_set_hwreg(adapter, HW_VAR_CAM_EMPTY_ENTRY, (u8 *)(&ucIndex));
+}
+
+void rtw_wapi_clear_cam_entry(_adapter *padapter, u8 *pMacAddr)
+{
+	u8 UcIndex = 0;
+
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!padapter->wapiInfo.bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return;
+	}
+
+	UcIndex = WapiGetEntryForCamClear(padapter, pMacAddr, 0, 0);
+	if (UcIndex != 0xff) {
+		/* CAM_mark_invalid(adapter, UcIndex); */
+		CAM_empty_entry(padapter, UcIndex);
+	}
+
+	UcIndex = WapiGetEntryForCamClear(padapter, pMacAddr, 1, 0);
+	if (UcIndex != 0xff) {
+		/* CAM_mark_invalid(adapter, UcIndex); */
+		CAM_empty_entry(padapter, UcIndex);
+	}
+
+	UcIndex = WapiGetEntryForCamClear(padapter, pMacAddr, 0, 1);
+	if (UcIndex != 0xff) {
+		/* CAM_mark_invalid(adapter, UcIndex); */
+		CAM_empty_entry(padapter, UcIndex);
+	}
+
+	UcIndex = WapiGetEntryForCamClear(padapter, pMacAddr, 1, 1);
+	if (UcIndex != 0xff) {
+		/* CAM_mark_invalid(padapter, UcIndex); */
+		CAM_empty_entry(padapter, UcIndex);
+	}
+
+	WAPI_TRACE(WAPI_API, "<========== %s\n", __FUNCTION__);
+}
+
+void rtw_wapi_clear_all_cam_entry(_adapter *padapter)
+{
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!padapter->wapiInfo.bWapiEnable)) {
+		WAPI_TRACE(WAPI_MLME, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return;
+	}
+
+	invalidate_cam_all(padapter); /* is this ok? */
+	WapiResetAllCamEntry(padapter);
+
+	WAPI_TRACE(WAPI_API, "<=========== %s\n", __FUNCTION__);
+}
+
+void rtw_wapi_set_key(_adapter *padapter, RT_WAPI_KEY *pWapiKey, RT_WAPI_STA_INFO *pWapiSta, u8 bGroupKey)
+{
+	struct setkey_parm *psetkeyparm;
+	struct set_stakey_parm *psetstakeyparm;
+	PRT_WAPI_T		pWapiInfo =  &padapter->wapiInfo;
+	u8			*pMacAddr = pWapiSta->PeerMacAddr;
+	u32 EntryId = 0;
+	BOOLEAN IsPairWise = false ;
+	u8 keylen;
+
+	WAPI_TRACE(WAPI_API, "===========> %s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!padapter->wapiInfo.bWapiEnable)) {
+		WAPI_TRACE(WAPI_API, "<========== %s, WAPI not supported or not enabled!\n", __FUNCTION__);
+		return;
+	}
+
+	psetstakeyparm = (struct set_stakey_parm *)rtw_zmalloc(sizeof(struct set_stakey_parm));
+	if (NULL == psetstakeyparm) {
+		WAPI_TRACE(WAPI_API, "<========== new %s, could not zmalloc psetstakeyparm at line %d\n", __FUNCTION__, __LINE__);
+		return;
+	}
+	psetstakeyparm->algorithm = _SMS4_;
+	keylen = 16;
+
+	_rtw_memcpy(psetstakeyparm->addr, pMacAddr, ETH_ALEN);
+	_rtw_memcpy(&(psetstakeyparm->key[0]), &pWapiKey->dataKey, keylen);
+	_rtw_memcpy(&(psetstakeyparm->key[16]), &pWapiKey->micKey, keylen);
+	psetstakeyparm->keyid = pWapiKey->keyId;
+
+	if(bGroupKey) {
+		/* Group Key */
+		psetstakeyparm->gk = 1;
+		WAPI_TRACE(WAPI_API, "new %s: group keyid = %d, gk = %d, algorithm = %d\n", __FUNCTION__, psetstakeyparm->keyid, psetstakeyparm->gk, psetstakeyparm->algorithm);
+	} else {
+		/* Pairwise Key */
+		psetstakeyparm->gk = 0;
+		WAPI_TRACE(WAPI_API, "new %s: pairwise keyid = %d, gk = %d, algorithm = %d\n", __FUNCTION__, psetstakeyparm->keyid, psetstakeyparm->gk, psetstakeyparm->algorithm);
+	}
+
+	WAPI_DATA(WAPI_API, "new %s: ", psetstakeyparm->key, 32);
+
+	set_stakey_hdl(padapter, psetstakeyparm, PHL_CMD_DIRECTLY, 0);
+
+	rtw_mfree((u8 *) psetstakeyparm, sizeof(struct setkey_parm));
+	WAPI_TRACE(WAPI_API, "Set Wapi Key :KeyId:%d,EntryId:%d,PairwiseKey:%d.\n", pWapiKey->keyId, EntryId, !bGroupKey);
+	WAPI_TRACE(WAPI_API, "<=========== %s\n", __FUNCTION__);
+}
+
+#if 0
+/* YJ,test,091013 */
+void wapi_test_set_key(_adapter *padapter, u8 *buf)
+{
+	/*Data: keyType(1) + bTxEnable(1) + bAuthenticator(1) + bUpdate(1) + PeerAddr(6) + DataKey(16) + MicKey(16) + KeyId(1)*/
+	PRT_WAPI_T			pWapiInfo = &padapter->wapiInfo;
+	PRT_WAPI_BKID		pWapiBkid;
+	PRT_WAPI_STA_INFO	pWapiSta;
+	u8					data[43];
+	bool					bTxEnable;
+	bool					bUpdate;
+	bool					bAuthenticator;
+	u8					PeerAddr[6];
+	u8					WapiAEPNInitialValueSrc[16] = {0x37, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+	u8					WapiASUEPNInitialValueSrc[16] = {0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+	u8					WapiAEMultiCastPNInitialValueSrc[16] = {0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+
+	WAPI_TRACE(WAPI_INIT, "===========>%s\n", __FUNCTION__);
+
+	if (!padapter->WapiSupport)
+		return;
+
+	copy_from_user(data, buf, 43);
+	bTxEnable = data[1];
+	bAuthenticator = data[2];
+	bUpdate = data[3];
+	_rtw_memcpy(PeerAddr, data + 4, 6);
+
+	if (data[0] == 0x3) {
+		if (!list_empty(&(pWapiInfo->wapiBKIDIdleList))) {
+			pWapiBkid = (PRT_WAPI_BKID)list_entry(pWapiInfo->wapiBKIDIdleList.next, RT_WAPI_BKID, list);
+			list_del_init(&pWapiBkid->list);
+			_rtw_memcpy(pWapiBkid->bkid, data + 10, 16);
+			WAPI_DATA(WAPI_INIT, "SetKey - BKID", pWapiBkid->bkid, 16);
+			list_add_tail(&pWapiBkid->list, &pWapiInfo->wapiBKIDStoreList);
+		}
+	} else {
+		list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+			if (_rtw_memcmp(pWapiSta->PeerMacAddr, PeerAddr, 6)) {
+				pWapiSta->bAuthenticatorInUpdata = false;
+				switch (data[0]) {
+				case 1:              /* usk */
+					if (bAuthenticator) {       /* authenticator */
+						_rtw_memcpy(pWapiSta->lastTxUnicastPN, WapiAEPNInitialValueSrc, 16);
+						if (!bUpdate) {    /* first */
+							WAPI_TRACE(WAPI_INIT, "AE fisrt set usk\n");
+							pWapiSta->wapiUsk.bSet = true;
+							_rtw_memcpy(pWapiSta->wapiUsk.dataKey, data + 10, 16);
+							_rtw_memcpy(pWapiSta->wapiUsk.micKey, data + 26, 16);
+							pWapiSta->wapiUsk.keyId = *(data + 42);
+							pWapiSta->wapiUsk.bTxEnable = true;
+							WAPI_DATA(WAPI_INIT, "SetKey - AE USK Data Key", pWapiSta->wapiUsk.dataKey, 16);
+							WAPI_DATA(WAPI_INIT, "SetKey - AE USK Mic Key", pWapiSta->wapiUsk.micKey, 16);
+						} else {           /* update */
+							WAPI_TRACE(WAPI_INIT, "AE update usk\n");
+							pWapiSta->wapiUskUpdate.bSet = true;
+							pWapiSta->bAuthenticatorInUpdata = true;
+							_rtw_memcpy(pWapiSta->wapiUskUpdate.dataKey, data + 10, 16);
+							_rtw_memcpy(pWapiSta->wapiUskUpdate.micKey, data + 26, 16);
+							_rtw_memcpy(pWapiSta->lastRxUnicastPNBEQueue, WapiASUEPNInitialValueSrc, 16);
+							_rtw_memcpy(pWapiSta->lastRxUnicastPNBKQueue, WapiASUEPNInitialValueSrc, 16);
+							_rtw_memcpy(pWapiSta->lastRxUnicastPNVIQueue, WapiASUEPNInitialValueSrc, 16);
+							_rtw_memcpy(pWapiSta->lastRxUnicastPNVOQueue, WapiASUEPNInitialValueSrc, 16);
+							_rtw_memcpy(pWapiSta->lastRxUnicastPN, WapiASUEPNInitialValueSrc, 16);
+							pWapiSta->wapiUskUpdate.keyId = *(data + 42);
+							pWapiSta->wapiUskUpdate.bTxEnable = true;
+						}
+					} else {
+						if (!bUpdate) {
+							WAPI_TRACE(WAPI_INIT, "ASUE fisrt set usk\n");
+							if (bTxEnable) {
+								pWapiSta->wapiUsk.bTxEnable = true;
+								_rtw_memcpy(pWapiSta->lastTxUnicastPN, WapiASUEPNInitialValueSrc, 16);
+							} else {
+								pWapiSta->wapiUsk.bSet = true;
+								_rtw_memcpy(pWapiSta->wapiUsk.dataKey, data + 10, 16);
+								_rtw_memcpy(pWapiSta->wapiUsk.micKey, data + 26, 16);
+								pWapiSta->wapiUsk.keyId = *(data + 42);
+								pWapiSta->wapiUsk.bTxEnable = false;
+							}
+						} else {
+							WAPI_TRACE(WAPI_INIT, "ASUE update usk\n");
+							if (bTxEnable) {
+								pWapiSta->wapiUskUpdate.bTxEnable = true;
+								if (pWapiSta->wapiUskUpdate.bSet) {
+									_rtw_memcpy(pWapiSta->wapiUsk.dataKey, pWapiSta->wapiUskUpdate.dataKey, 16);
+									_rtw_memcpy(pWapiSta->wapiUsk.micKey, pWapiSta->wapiUskUpdate.micKey, 16);
+									pWapiSta->wapiUsk.keyId = pWapiSta->wapiUskUpdate.keyId;
+									_rtw_memcpy(pWapiSta->lastRxUnicastPNBEQueue, WapiASUEPNInitialValueSrc, 16);
+									_rtw_memcpy(pWapiSta->lastRxUnicastPNBKQueue, WapiASUEPNInitialValueSrc, 16);
+									_rtw_memcpy(pWapiSta->lastRxUnicastPNVIQueue, WapiASUEPNInitialValueSrc, 16);
+									_rtw_memcpy(pWapiSta->lastRxUnicastPNVOQueue, WapiASUEPNInitialValueSrc, 16);
+									_rtw_memcpy(pWapiSta->lastRxUnicastPN, WapiASUEPNInitialValueSrc, 16);
+									pWapiSta->wapiUskUpdate.bTxEnable = false;
+									pWapiSta->wapiUskUpdate.bSet = false;
+								}
+								_rtw_memcpy(pWapiSta->lastTxUnicastPN, WapiASUEPNInitialValueSrc, 16);
+							} else {
+								pWapiSta->wapiUskUpdate.bSet = true;
+								_rtw_memcpy(pWapiSta->wapiUskUpdate.dataKey, data + 10, 16);
+								_rtw_memcpy(pWapiSta->wapiUskUpdate.micKey, data + 26, 16);
+								pWapiSta->wapiUskUpdate.keyId = *(data + 42);
+								pWapiSta->wapiUskUpdate.bTxEnable = false;
+							}
+						}
+					}
+					break;
+				case 2:		/* msk */
+					if (bAuthenticator) {        /* authenticator */
+						pWapiInfo->wapiTxMsk.bSet = true;
+						_rtw_memcpy(pWapiInfo->wapiTxMsk.dataKey, data + 10, 16);
+						_rtw_memcpy(pWapiInfo->wapiTxMsk.micKey, data + 26, 16);
+						pWapiInfo->wapiTxMsk.keyId = *(data + 42);
+						pWapiInfo->wapiTxMsk.bTxEnable = true;
+						_rtw_memcpy(pWapiInfo->lastTxMulticastPN, WapiAEMultiCastPNInitialValueSrc, 16);
+
+						if (!bUpdate) {    /* first */
+							WAPI_TRACE(WAPI_INIT, "AE fisrt set msk\n");
+							if (!pWapiSta->bSetkeyOk)
+								pWapiSta->bSetkeyOk = true;
+							pWapiInfo->bFirstAuthentiateInProgress = false;
+						} else                /* update */
+							WAPI_TRACE(WAPI_INIT, "AE update msk\n");
+
+						WAPI_DATA(WAPI_INIT, "SetKey - AE MSK Data Key", pWapiInfo->wapiTxMsk.dataKey, 16);
+						WAPI_DATA(WAPI_INIT, "SetKey - AE MSK Mic Key", pWapiInfo->wapiTxMsk.micKey, 16);
+					} else {
+						if (!bUpdate) {
+							WAPI_TRACE(WAPI_INIT, "ASUE fisrt set msk\n");
+							pWapiSta->wapiMsk.bSet = true;
+							_rtw_memcpy(pWapiSta->wapiMsk.dataKey, data + 10, 16);
+							_rtw_memcpy(pWapiSta->wapiMsk.micKey, data + 26, 16);
+							pWapiSta->wapiMsk.keyId = *(data + 42);
+							pWapiSta->wapiMsk.bTxEnable = false;
+							if (!pWapiSta->bSetkeyOk)
+								pWapiSta->bSetkeyOk = true;
+							pWapiInfo->bFirstAuthentiateInProgress = false;
+							WAPI_DATA(WAPI_INIT, "SetKey - ASUE MSK Data Key", pWapiSta->wapiMsk.dataKey, 16);
+							WAPI_DATA(WAPI_INIT, "SetKey - ASUE MSK Mic Key", pWapiSta->wapiMsk.micKey, 16);
+						} else {
+							WAPI_TRACE(WAPI_INIT, "ASUE update msk\n");
+							pWapiSta->wapiMskUpdate.bSet = true;
+							_rtw_memcpy(pWapiSta->wapiMskUpdate.dataKey, data + 10, 16);
+							_rtw_memcpy(pWapiSta->wapiMskUpdate.micKey, data + 26, 16);
+							pWapiSta->wapiMskUpdate.keyId = *(data + 42);
+							pWapiSta->wapiMskUpdate.bTxEnable = false;
+						}
+					}
+					break;
+				default:
+					WAPI_TRACE(WAPI_ERR, "Unknown Flag\n");
+					break;
+				}
+			}
+		}
+	}
+	WAPI_TRACE(WAPI_INIT, "<===========%s\n", __FUNCTION__);
+}
+
+
+void wapi_test_init(_adapter *padapter)
+{
+	u8 keybuf[100];
+	u8 mac_addr[ETH_ALEN] = {0x00, 0xe0, 0x4c, 0x72, 0x04, 0x70};
+	u8 UskDataKey[16] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f};
+	u8 UskMicKey[16] = {0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f};
+	u8 UskId = 0;
+	u8 MskDataKey[16] = {0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f};
+	u8 MskMicKey[16] = {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f};
+	u8 MskId = 0;
+
+	WAPI_TRACE(WAPI_INIT, "===========>%s\n", __FUNCTION__);
+
+	/* Enable Wapi */
+	WAPI_TRACE(WAPI_INIT, "%s: Enable wapi!!!!\n", __FUNCTION__);
+	padapter->wapiInfo.bWapiEnable = true;
+	padapter->pairwise_key_type = KEY_TYPE_SMS4;
+	ieee->group_key_type = KEY_TYPE_SMS4;
+	padapter->wapiInfo.extra_prefix_len = WAPI_EXT_LEN;
+	padapter->wapiInfo.extra_postfix_len = SMS4_MIC_LEN;
+
+	/* set usk */
+	WAPI_TRACE(WAPI_INIT, "%s: Set USK!!!!\n", __FUNCTION__);
+	_rtw_memset(keybuf, 0, 100);
+	keybuf[0] = 1;                           /* set usk */
+	keybuf[1] = 1; 				/* enable tx */
+	keybuf[2] = 1; 				/* AE */
+	keybuf[3] = 0; 				/* not update */
+
+	_rtw_memcpy(keybuf + 4, mac_addr, ETH_ALEN);
+	_rtw_memcpy(keybuf + 10, UskDataKey, 16);
+	_rtw_memcpy(keybuf + 26, UskMicKey, 16);
+	keybuf[42] = UskId;
+	wapi_test_set_key(padapter, keybuf);
+
+	_rtw_memset(keybuf, 0, 100);
+	keybuf[0] = 1;                           /* set usk */
+	keybuf[1] = 1; 				/* enable tx */
+	keybuf[2] = 0; 				/* AE */
+	keybuf[3] = 0; 				/* not update */
+
+	_rtw_memcpy(keybuf + 4, mac_addr, ETH_ALEN);
+	_rtw_memcpy(keybuf + 10, UskDataKey, 16);
+	_rtw_memcpy(keybuf + 26, UskMicKey, 16);
+	keybuf[42] = UskId;
+	wapi_test_set_key(padapter, keybuf);
+
+	/* set msk */
+	WAPI_TRACE(WAPI_INIT, "%s: Set MSK!!!!\n", __FUNCTION__);
+	_rtw_memset(keybuf, 0, 100);
+	keybuf[0] = 2;                                /* set msk */
+	keybuf[1] = 1;                               /* Enable TX */
+	keybuf[2] = 1; 				/* AE */
+	keybuf[3] = 0;                              /* not update */
+	_rtw_memcpy(keybuf + 4, mac_addr, ETH_ALEN);
+	_rtw_memcpy(keybuf + 10, MskDataKey, 16);
+	_rtw_memcpy(keybuf + 26, MskMicKey, 16);
+	keybuf[42] = MskId;
+	wapi_test_set_key(padapter, keybuf);
+
+	_rtw_memset(keybuf, 0, 100);
+	keybuf[0] = 2;                                /* set msk */
+	keybuf[1] = 1;                               /* Enable TX */
+	keybuf[2] = 0; 				/* AE */
+	keybuf[3] = 0;                              /* not update */
+	_rtw_memcpy(keybuf + 4, mac_addr, ETH_ALEN);
+	_rtw_memcpy(keybuf + 10, MskDataKey, 16);
+	_rtw_memcpy(keybuf + 26, MskMicKey, 16);
+	keybuf[42] = MskId;
+	wapi_test_set_key(padapter, keybuf);
+	WAPI_TRACE(WAPI_INIT, "<===========%s\n", __FUNCTION__);
+}
+#endif
+
+void rtw_wapi_get_iv(_adapter *padapter, u8 *pRA, u8 *IV)
+{
+	PWLAN_HEADER_WAPI_EXTENSION pWapiExt = NULL;
+	PRT_WAPI_T         pWapiInfo = &padapter->wapiInfo;
+	bool	bPNOverflow = false;
+	bool	bFindMatchPeer = false;
+	PRT_WAPI_STA_INFO  pWapiSta = NULL;
+
+	pWapiExt = (PWLAN_HEADER_WAPI_EXTENSION)IV;
+
+	WAPI_DATA(WAPI_RX, "wapi_get_iv: pra", pRA, 6);
+
+	if (IS_MCAST(pRA)) {
+		if (!pWapiInfo->wapiTxMsk.bTxEnable) {
+			WAPI_TRACE(WAPI_ERR, "%s: bTxEnable = 0!!\n", __FUNCTION__);
+			return;
+		}
+
+		if (pWapiInfo->wapiTxMsk.keyId <= 1) {
+			pWapiExt->KeyIdx = pWapiInfo->wapiTxMsk.keyId;
+			pWapiExt->Reserved = 0;
+			bPNOverflow = WapiIncreasePN(pWapiInfo->lastTxMulticastPN, 1);
+			_rtw_memcpy(pWapiExt->PN, pWapiInfo->lastTxMulticastPN, 16);
+		}
+	} else {
+		if (list_empty(&pWapiInfo->wapiSTAUsedList)) {
+			WAPI_TRACE(WAPI_RX, "rtw_wapi_get_iv: list is empty\n");
+			_rtw_memset(IV, 10, 18);
+			return;
+		} else {
+			list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+				WAPI_DATA(WAPI_RX, "rtw_wapi_get_iv: peermacaddr ", pWapiSta->PeerMacAddr, 6);
+				if (_rtw_memcmp((u8 *)pWapiSta->PeerMacAddr, pRA, 6) == _TRUE) {
+					bFindMatchPeer = true;
+					break;
+				}
+			}
+
+			WAPI_TRACE(WAPI_RX, "bFindMatchPeer: %d\n", bFindMatchPeer);
+			WAPI_DATA(WAPI_RX, "Addr", pRA, 6);
+
+			if (bFindMatchPeer) {
+				if ((!pWapiSta->wapiUskUpdate.bTxEnable) && (!pWapiSta->wapiUsk.bTxEnable))
+					return;
+
+				if (pWapiSta->wapiUsk.keyId <= 1) {
+					if (pWapiSta->wapiUskUpdate.bTxEnable)
+						pWapiExt->KeyIdx = pWapiSta->wapiUskUpdate.keyId;
+					else
+						pWapiExt->KeyIdx = pWapiSta->wapiUsk.keyId;
+
+					pWapiExt->Reserved = 0;
+					bPNOverflow = WapiIncreasePN(pWapiSta->lastTxUnicastPN, 2);
+					_rtw_memcpy(pWapiExt->PN, pWapiSta->lastTxUnicastPN, 16);
+
+				}
+			}
+		}
+
+	}
+
+}
+
+bool rtw_wapi_drop_for_key_absent(_adapter *padapter, u8 *pRA)
+{
+	PRT_WAPI_T         pWapiInfo = &padapter->wapiInfo;
+	bool				bFindMatchPeer = false;
+	bool				bDrop = false;
+	PRT_WAPI_STA_INFO  pWapiSta = NULL;
+	struct security_priv		*psecuritypriv = &padapter->securitypriv;
+
+	WAPI_DATA(WAPI_RX, "rtw_wapi_drop_for_key_absent: ra ", pRA, 6);
+
+	if (psecuritypriv->dot11PrivacyAlgrthm == _SMS4_) {
+		if ((!padapter->WapiSupport) || (!pWapiInfo->bWapiEnable))
+			return true;
+
+		if (IS_MCAST(pRA)) {
+			if (!pWapiInfo->wapiTxMsk.bTxEnable) {
+				bDrop = true;
+				WAPI_TRACE(WAPI_RX, "rtw_wapi_drop_for_key_absent: multicast key is absent\n");
+				return bDrop;
+			}
+		} else {
+			if (!list_empty(&pWapiInfo->wapiSTAUsedList)) {
+				list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+					WAPI_DATA(WAPI_RX, "rtw_wapi_drop_for_key_absent: pWapiSta->PeerMacAddr ", pWapiSta->PeerMacAddr, 6);
+					if (_rtw_memcmp(pRA, pWapiSta->PeerMacAddr, 6) == _TRUE) {
+						bFindMatchPeer = true;
+						break;
+					}
+				}
+				if (bFindMatchPeer)	{
+					if (!pWapiSta->wapiUsk.bTxEnable) {
+						bDrop = true;
+						WAPI_TRACE(WAPI_RX, "rtw_wapi_drop_for_key_absent: unicast key is absent\n");
+						return bDrop;
+					}
+				} else {
+					bDrop = true;
+					WAPI_TRACE(WAPI_RX, "rtw_wapi_drop_for_key_absent: no peer find\n");
+					return bDrop;
+				}
+
+			} else {
+				bDrop = true;
+				WAPI_TRACE(WAPI_RX, "rtw_wapi_drop_for_key_absent: no sta  exist\n");
+				return bDrop;
+			}
+		}
+	} else
+		return bDrop;
+
+	return bDrop;
+}
+
+void rtw_wapi_set_set_encryption(_adapter *padapter, struct ieee_param *param)
+{
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	PRT_WAPI_T			pWapiInfo = &padapter->wapiInfo;
+	PRT_WAPI_STA_INFO	pWapiSta;
+	u8					WapiASUEPNInitialValueSrc[16] = {0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+	u8					WapiAEPNInitialValueSrc[16] = {0x37, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+	u8					WapiAEMultiCastPNInitialValueSrc[16] = {0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C, 0x36, 0x5C} ;
+
+	if (param->u.crypt.set_tx == 1) {
+		list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+			if (_rtw_memcmp(pWapiSta->PeerMacAddr, param->sta_addr, 6)) {
+				_rtw_memcpy(pWapiSta->lastTxUnicastPN, WapiASUEPNInitialValueSrc, 16);
+
+				pWapiSta->wapiUsk.bSet = true;
+				_rtw_memcpy(pWapiSta->wapiUsk.dataKey, param->u.crypt.key, 16);
+				_rtw_memcpy(pWapiSta->wapiUsk.micKey, param->u.crypt.key + 16, 16);
+				pWapiSta->wapiUsk.keyId = param->u.crypt.idx;
+				psecuritypriv->dot11PrivacyKeyIndex = param->u.crypt.idx;
+				pWapiSta->wapiUsk.bTxEnable = true;
+
+				_rtw_memcpy(pWapiSta->lastRxUnicastPNBEQueue, WapiAEPNInitialValueSrc, 16);
+				_rtw_memcpy(pWapiSta->lastRxUnicastPNBKQueue, WapiAEPNInitialValueSrc, 16);
+				_rtw_memcpy(pWapiSta->lastRxUnicastPNVIQueue, WapiAEPNInitialValueSrc, 16);
+				_rtw_memcpy(pWapiSta->lastRxUnicastPNVOQueue, WapiAEPNInitialValueSrc, 16);
+				_rtw_memcpy(pWapiSta->lastRxUnicastPN, WapiAEPNInitialValueSrc, 16);
+				pWapiSta->wapiUskUpdate.bTxEnable = false;
+				pWapiSta->wapiUskUpdate.bSet = false;
+
+				if (psecuritypriv->sw_encrypt == false || psecuritypriv->sw_decrypt == false) {
+					/* set unicast key for ASUE */
+					rtw_wapi_set_key(padapter, &pWapiSta->wapiUsk, pWapiSta, false);
+				}
+			}
+		}
+	} else {
+		list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+			if (_rtw_memcmp(pWapiSta->PeerMacAddr, get_bssid(pmlmepriv), 6)) {
+				pWapiSta->wapiMsk.bSet = true;
+				_rtw_memcpy(pWapiSta->wapiMsk.dataKey, param->u.crypt.key, 16);
+				_rtw_memcpy(pWapiSta->wapiMsk.micKey, param->u.crypt.key + 16, 16);
+				pWapiSta->wapiMsk.keyId = param->u.crypt.idx;
+				lsecuritypriv->dot118021XGrpKeyid = param->u.crypt.idx;
+				pWapiSta->wapiMsk.bTxEnable = false;
+				if (!pWapiSta->bSetkeyOk)
+					pWapiSta->bSetkeyOk = true;
+				pWapiSta->bAuthenticateInProgress = false;
+
+				_rtw_memcpy(pWapiSta->lastRxMulticastPN, WapiAEMultiCastPNInitialValueSrc, 16);
+
+				if (psecuritypriv->sw_decrypt == false) {
+					/* set rx broadcast key for ASUE */
+					rtw_wapi_set_key(padapter, &pWapiSta->wapiMsk, pWapiSta, true);
+				}
+			}
+		}
+	}
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wapi_sms4.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wapi_sms4.c
new file mode 100644
index 000000000000..4d3a1205118d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wapi_sms4.c
@@ -0,0 +1,922 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifdef CONFIG_WAPI_SUPPORT
+
+#include <linux/unistd.h>
+#include <linux/etherdevice.h>
+#include <drv_types.h>
+#include <rtw_wapi.h>
+
+
+#ifdef CONFIG_WAPI_SW_SMS4
+
+#define WAPI_LITTLE_ENDIAN
+/* #define BIG_ENDIAN */
+#define ENCRYPT  0
+#define DECRYPT  1
+
+
+/**********************************************************
+ **********************************************************/
+const u8 Sbox[256] = {
+	0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05,
+	0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99,
+	0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62,
+	0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6,
+	0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8,
+	0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35,
+	0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87,
+	0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e,
+	0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1,
+	0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3,
+	0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f,
+	0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51,
+	0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8,
+	0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0,
+	0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84,
+	0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48
+};
+
+const u32 CK[32] = {
+	0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269,
+	0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9,
+	0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249,
+	0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9,
+	0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229,
+	0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299,
+	0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209,
+	0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279
+};
+
+#define Rotl(_x, _y) (((_x) << (_y)) | ((_x) >> (32 - (_y))))
+
+#define ByteSub(_A) (Sbox[(_A) >> 24 & 0xFF] << 24 | \
+		     Sbox[(_A) >> 16 & 0xFF] << 16 | \
+		     Sbox[(_A) >>  8 & 0xFF] <<  8 | \
+		     Sbox[(_A) & 0xFF])
+
+#define L1(_B) ((_B) ^ Rotl(_B, 2) ^ Rotl(_B, 10) ^ Rotl(_B, 18) ^ Rotl(_B, 24))
+#define L2(_B) ((_B) ^ Rotl(_B, 13) ^ Rotl(_B, 23))
+
+static void
+xor_block(void *dst, void *src1, void *src2)
+/* 128-bit xor: *dst = *src1 xor *src2. Pointers must be 32-bit aligned */
+{
+	((u32 *)dst)[0] = ((u32 *)src1)[0] ^ ((u32 *)src2)[0];
+	((u32 *)dst)[1] = ((u32 *)src1)[1] ^ ((u32 *)src2)[1];
+	((u32 *)dst)[2] = ((u32 *)src1)[2] ^ ((u32 *)src2)[2];
+	((u32 *)dst)[3] = ((u32 *)src1)[3] ^ ((u32 *)src2)[3];
+}
+
+
+void SMS4Crypt(u8 *Input, u8 *Output, u32 *rk)
+{
+	u32 r, mid, x0, x1, x2, x3, *p;
+	p = (u32 *)Input;
+	x0 = p[0];
+	x1 = p[1];
+	x2 = p[2];
+	x3 = p[3];
+#ifdef WAPI_LITTLE_ENDIAN
+	x0 = Rotl(x0, 16);
+	x0 = ((x0 & 0x00FF00FF) << 8) | ((x0 & 0xFF00FF00) >> 8);
+	x1 = Rotl(x1, 16);
+	x1 = ((x1 & 0x00FF00FF) << 8) | ((x1 & 0xFF00FF00) >> 8);
+	x2 = Rotl(x2, 16);
+	x2 = ((x2 & 0x00FF00FF) << 8) | ((x2 & 0xFF00FF00) >> 8);
+	x3 = Rotl(x3, 16);
+	x3 = ((x3 & 0x00FF00FF) << 8) | ((x3 & 0xFF00FF00) >> 8);
+#endif
+	for (r = 0; r < 32; r += 4) {
+		mid = x1 ^ x2 ^ x3 ^ rk[r + 0];
+		mid = ByteSub(mid);
+		x0 ^= L1(mid);
+		mid = x2 ^ x3 ^ x0 ^ rk[r + 1];
+		mid = ByteSub(mid);
+		x1 ^= L1(mid);
+		mid = x3 ^ x0 ^ x1 ^ rk[r + 2];
+		mid = ByteSub(mid);
+		x2 ^= L1(mid);
+		mid = x0 ^ x1 ^ x2 ^ rk[r + 3];
+		mid = ByteSub(mid);
+		x3 ^= L1(mid);
+	}
+#ifdef WAPI_LITTLE_ENDIAN
+	x0 = Rotl(x0, 16);
+	x0 = ((x0 & 0x00FF00FF) << 8) | ((x0 & 0xFF00FF00) >> 8);
+	x1 = Rotl(x1, 16);
+	x1 = ((x1 & 0x00FF00FF) << 8) | ((x1 & 0xFF00FF00) >> 8);
+	x2 = Rotl(x2, 16);
+	x2 = ((x2 & 0x00FF00FF) << 8) | ((x2 & 0xFF00FF00) >> 8);
+	x3 = Rotl(x3, 16);
+	x3 = ((x3 & 0x00FF00FF) << 8) | ((x3 & 0xFF00FF00) >> 8);
+#endif
+	p = (u32 *)Output;
+	p[0] = x3;
+	p[1] = x2;
+	p[2] = x1;
+	p[3] = x0;
+}
+
+
+
+void SMS4KeyExt(u8 *Key, u32 *rk, u32 CryptFlag)
+{
+	u32 r, mid, x0, x1, x2, x3, *p;
+
+	p = (u32 *)Key;
+	x0 = p[0];
+	x1 = p[1];
+	x2 = p[2];
+	x3 = p[3];
+#ifdef WAPI_LITTLE_ENDIAN
+	x0 = Rotl(x0, 16);
+	x0 = ((x0 & 0xFF00FF) << 8) | ((x0 & 0xFF00FF00) >> 8);
+	x1 = Rotl(x1, 16);
+	x1 = ((x1 & 0xFF00FF) << 8) | ((x1 & 0xFF00FF00) >> 8);
+	x2 = Rotl(x2, 16);
+	x2 = ((x2 & 0xFF00FF) << 8) | ((x2 & 0xFF00FF00) >> 8);
+	x3 = Rotl(x3, 16);
+	x3 = ((x3 & 0xFF00FF) << 8) | ((x3 & 0xFF00FF00) >> 8);
+#endif
+
+	x0 ^= 0xa3b1bac6;
+	x1 ^= 0x56aa3350;
+	x2 ^= 0x677d9197;
+	x3 ^= 0xb27022dc;
+	for (r = 0; r < 32; r += 4) {
+		mid = x1 ^ x2 ^ x3 ^ CK[r + 0];
+		mid = ByteSub(mid);
+		rk[r + 0] = x0 ^= L2(mid);
+		mid = x2 ^ x3 ^ x0 ^ CK[r + 1];
+		mid = ByteSub(mid);
+		rk[r + 1] = x1 ^= L2(mid);
+		mid = x3 ^ x0 ^ x1 ^ CK[r + 2];
+		mid = ByteSub(mid);
+		rk[r + 2] = x2 ^= L2(mid);
+		mid = x0 ^ x1 ^ x2 ^ CK[r + 3];
+		mid = ByteSub(mid);
+		rk[r + 3] = x3 ^= L2(mid);
+	}
+	if (CryptFlag == DECRYPT) {
+		for (r = 0; r < 16; r++)
+			mid = rk[r], rk[r] = rk[31 - r], rk[31 - r] = mid;
+	}
+}
+
+
+void WapiSMS4Cryption(u8 *Key, u8 *IV, u8 *Input, u16 InputLength,
+		      u8 *Output, u16 *OutputLength, u32 CryptFlag)
+{
+	u32 blockNum, i, j, rk[32];
+	u16 remainder;
+	u8 blockIn[16], blockOut[16], tempIV[16], k;
+
+	*OutputLength = 0;
+	remainder = InputLength & 0x0F;
+	blockNum = InputLength >> 4;
+	if (remainder != 0)
+		blockNum++;
+	else
+		remainder = 16;
+
+	for (k = 0; k < 16; k++)
+		tempIV[k] = IV[15 - k];
+
+	_rtw_memcpy(blockIn, tempIV, 16);
+
+	SMS4KeyExt((u8 *)Key, rk, CryptFlag);
+
+	for (i = 0; i < blockNum - 1; i++) {
+		SMS4Crypt((u8 *)blockIn, blockOut, rk);
+		xor_block(&Output[i * 16], &Input[i * 16], blockOut);
+		_rtw_memcpy(blockIn, blockOut, 16);
+	}
+
+	*OutputLength = i * 16;
+
+	SMS4Crypt((u8 *)blockIn, blockOut, rk);
+
+	for (j = 0; j < remainder; j++)
+		Output[i * 16 + j] = Input[i * 16 + j] ^ blockOut[j];
+	*OutputLength += remainder;
+
+}
+
+void WapiSMS4Encryption(u8 *Key, u8 *IV, u8 *Input, u16 InputLength,
+			u8 *Output, u16 *OutputLength)
+{
+
+	WapiSMS4Cryption(Key, IV, Input, InputLength, Output, OutputLength, ENCRYPT);
+}
+
+void WapiSMS4Decryption(u8 *Key, u8 *IV, u8 *Input, u16 InputLength,
+			u8 *Output, u16 *OutputLength)
+{
+	/* OFB mode: is also ENCRYPT flag */
+	WapiSMS4Cryption(Key, IV, Input, InputLength, Output, OutputLength, ENCRYPT);
+}
+
+void WapiSMS4CalculateMic(u8 *Key, u8 *IV, u8 *Input1, u8 Input1Length,
+		  u8 *Input2, u16 Input2Length, u8 *Output, u8 *OutputLength)
+{
+	u32 blockNum, i, remainder, rk[32];
+	u8 BlockIn[16], BlockOut[16], TempBlock[16], tempIV[16], k;
+
+	*OutputLength = 0;
+	remainder = Input1Length & 0x0F;
+	blockNum = Input1Length >> 4;
+
+	for (k = 0; k < 16; k++)
+		tempIV[k] = IV[15 - k];
+
+	_rtw_memcpy(BlockIn, tempIV, 16);
+
+	SMS4KeyExt((u8 *)Key, rk, ENCRYPT);
+
+	SMS4Crypt((u8 *)BlockIn, BlockOut, rk);
+
+	for (i = 0; i < blockNum; i++) {
+		xor_block(BlockIn, (Input1 + i * 16), BlockOut);
+		SMS4Crypt((u8 *)BlockIn, BlockOut, rk);
+	}
+
+	if (remainder != 0) {
+		_rtw_memset(TempBlock, 0, 16);
+		_rtw_memcpy(TempBlock, (Input1 + blockNum * 16), remainder);
+
+		xor_block(BlockIn, TempBlock, BlockOut);
+		SMS4Crypt((u8 *)BlockIn, BlockOut, rk);
+	}
+
+	remainder = Input2Length & 0x0F;
+	blockNum = Input2Length >> 4;
+
+	for (i = 0; i < blockNum; i++) {
+		xor_block(BlockIn, (Input2 + i * 16), BlockOut);
+		SMS4Crypt((u8 *)BlockIn, BlockOut, rk);
+	}
+
+	if (remainder != 0) {
+		_rtw_memset(TempBlock, 0, 16);
+		_rtw_memcpy(TempBlock, (Input2 + blockNum * 16), remainder);
+
+		xor_block(BlockIn, TempBlock, BlockOut);
+		SMS4Crypt((u8 *)BlockIn, BlockOut, rk);
+	}
+
+	_rtw_memcpy(Output, BlockOut, 16);
+	*OutputLength = 16;
+}
+
+void SecCalculateMicSMS4(
+	u8		KeyIdx,
+	u8        *MicKey,
+	u8        *pHeader,
+	u8        *pData,
+	u16       DataLen,
+	u8        *MicBuffer
+)
+{
+#if 0
+	struct ieee80211_hdr_3addr_qos *header;
+	u8 TempBuf[34], TempLen = 32, MicLen, QosOffset, *IV;
+	u16 *pTemp, fc;
+
+	WAPI_TRACE(WAPI_TX | WAPI_RX, "=========>%s\n", __FUNCTION__);
+
+	header = (struct ieee80211_hdr_3addr_qos *)pHeader;
+	_rtw_memset(TempBuf, 0, 34);
+	_rtw_memcpy(TempBuf, pHeader, 2); /* FrameCtrl */
+	pTemp = (u16 *)TempBuf;
+	*pTemp &= 0xc78f;       /* bit4,5,6,11,12,13 */
+
+	_rtw_memcpy((TempBuf + 2), (pHeader + 4), 12); /* Addr1, Addr2 */
+	_rtw_memcpy((TempBuf + 14), (pHeader + 22), 2); /* SeqCtrl */
+	pTemp = (u16 *)(TempBuf + 14);
+	*pTemp &= 0x000f;
+
+	_rtw_memcpy((TempBuf + 16), (pHeader + 16), 6); /* Addr3 */
+
+	fc = le16_to_cpu(header->frame_ctl);
+
+
+
+	if (GetFrDs((u16 *)&fc) && GetToDs((u16 *)&fc)) {
+		_rtw_memcpy((TempBuf + 22), (pHeader + 24), 6);
+		QosOffset = 30;
+	} else {
+		_rtw_memset((TempBuf + 22), 0, 6);
+		QosOffset = 24;
+	}
+
+	if ((fc & 0x0088) == 0x0088) {
+		_rtw_memcpy((TempBuf + 28), (pHeader + QosOffset), 2);
+		TempLen += 2;
+		/* IV = pHeader + QosOffset + 2 + SNAP_SIZE + sizeof(u16) + 2; */
+		IV = pHeader + QosOffset + 2 + 2;
+	} else {
+		IV = pHeader + QosOffset + 2;
+		/* IV = pHeader + QosOffset + SNAP_SIZE + sizeof(u16) + 2; */
+	}
+
+	TempBuf[TempLen - 1] = (u8)(DataLen & 0xff);
+	TempBuf[TempLen - 2] = (u8)((DataLen & 0xff00) >> 8);
+	TempBuf[TempLen - 4] = KeyIdx;
+
+	WAPI_DATA(WAPI_TX, "CalculateMic - KEY", MicKey, 16);
+	WAPI_DATA(WAPI_TX, "CalculateMic - IV", IV, 16);
+	WAPI_DATA(WAPI_TX, "CalculateMic - TempBuf", TempBuf, TempLen);
+	WAPI_DATA(WAPI_TX, "CalculateMic - pData", pData, DataLen);
+
+	WapiSMS4CalculateMic(MicKey, IV, TempBuf, TempLen,
+			     pData, DataLen, MicBuffer, &MicLen);
+
+	if (MicLen != 16)
+		WAPI_TRACE(WAPI_ERR, "%s: MIC Length Error!!\n", __FUNCTION__);
+
+	WAPI_TRACE(WAPI_TX | WAPI_RX, "<=========%s\n", __FUNCTION__);
+#endif
+}
+
+/* AddCount: 1 or 2.
+ *  If overflow, return 1,
+ *  else return 0.
+ */
+u8 WapiIncreasePN(u8 *PN, u8 AddCount)
+{
+	u8  i;
+
+	if (NULL == PN)
+		return 1;
+	/* YJ,test,091102 */
+	/*
+	if(AddCount == 2){
+		RTW_INFO("############################%s(): PN[0]=0x%x\n", __FUNCTION__, PN[0]);
+		if(PN[0] == 0x48){
+			PN[0] += AddCount;
+			return 1;
+		}else{
+			PN[0] += AddCount;
+			return 0;
+		}
+	}
+	*/
+	/* YJ,test,091102,end */
+
+	for (i = 0; i < 16; i++) {
+		if (PN[i] + AddCount <= 0xff) {
+			PN[i] += AddCount;
+			return 0;
+		} else {
+			PN[i] += AddCount;
+			AddCount = 1;
+		}
+	}
+	return 1;
+}
+
+
+void WapiGetLastRxUnicastPNForQoSData(
+	u8			UserPriority,
+	PRT_WAPI_STA_INFO    pWapiStaInfo,
+	u8 *PNOut
+)
+{
+	WAPI_TRACE(WAPI_RX, "===========> %s\n", __FUNCTION__);
+	switch (UserPriority) {
+	case 0:
+	case 3:
+		_rtw_memcpy(PNOut, pWapiStaInfo->lastRxUnicastPNBEQueue, 16);
+		break;
+	case 1:
+	case 2:
+		_rtw_memcpy(PNOut, pWapiStaInfo->lastRxUnicastPNBKQueue, 16);
+		break;
+	case 4:
+	case 5:
+		_rtw_memcpy(PNOut, pWapiStaInfo->lastRxUnicastPNVIQueue, 16);
+		break;
+	case 6:
+	case 7:
+		_rtw_memcpy(PNOut, pWapiStaInfo->lastRxUnicastPNVOQueue, 16);
+		break;
+	default:
+		WAPI_TRACE(WAPI_ERR, "%s: Unknown TID\n", __FUNCTION__);
+		break;
+	}
+	WAPI_TRACE(WAPI_RX, "<=========== %s\n", __FUNCTION__);
+}
+
+
+void WapiSetLastRxUnicastPNForQoSData(
+	u8		UserPriority,
+	u8           *PNIn,
+	PRT_WAPI_STA_INFO    pWapiStaInfo
+)
+{
+	WAPI_TRACE(WAPI_RX, "===========> %s\n", __FUNCTION__);
+	switch (UserPriority) {
+	case 0:
+	case 3:
+		_rtw_memcpy(pWapiStaInfo->lastRxUnicastPNBEQueue, PNIn, 16);
+		break;
+	case 1:
+	case 2:
+		_rtw_memcpy(pWapiStaInfo->lastRxUnicastPNBKQueue, PNIn, 16);
+		break;
+	case 4:
+	case 5:
+		_rtw_memcpy(pWapiStaInfo->lastRxUnicastPNVIQueue, PNIn, 16);
+		break;
+	case 6:
+	case 7:
+		_rtw_memcpy(pWapiStaInfo->lastRxUnicastPNVOQueue, PNIn, 16);
+		break;
+	default:
+		WAPI_TRACE(WAPI_ERR, "%s: Unknown TID\n", __FUNCTION__);
+		break;
+	}
+	WAPI_TRACE(WAPI_RX, "<=========== %s\n", __FUNCTION__);
+}
+
+
+/****************************************************************************
+ FALSE not RX-Reorder
+ TRUE do RX Reorder
+add to support WAPI to N-mode
+*****************************************************************************/
+u8 WapiCheckPnInSwDecrypt(
+	_adapter *padapter,
+	struct sk_buff *pskb
+)
+{
+	u8				ret = false;
+
+#if 0
+	struct ieee80211_hdr_3addr_qos *header;
+	u16				fc;
+	u8				*pDaddr, *pTaddr, *pRaddr;
+
+	header = (struct ieee80211_hdr_3addr_qos *)pskb->data;
+	pTaddr = header->addr2;
+	pRaddr = header->addr1;
+	fc = le16_to_cpu(header->frame_ctl);
+
+	if (GetToDs(&fc))
+		pDaddr = header->addr3;
+	else
+		pDaddr = header->addr1;
+
+	if ((_rtw_memcmp(pRaddr, padapter->pnetdev->dev_addr, ETH_ALEN) == 0)
+	    &&	!(pDaddr)
+	    && (GetFrameType(&fc) == WIFI_QOS_DATA_TYPE))
+		/* && ieee->pHTInfo->bCurrentHTSupport && */
+		/* ieee->pHTInfo->bCurRxReorderEnable) */
+		ret = false;
+	else
+		ret = true;
+#endif
+	WAPI_TRACE(WAPI_RX, "%s: return %d\n", __FUNCTION__, ret);
+	return ret;
+}
+
+int SecSMS4HeaderFillIV(_adapter *padapter, u8 *pxmitframe)
+{
+	struct pkt_attrib *pattrib = &((struct xmit_frame *)pxmitframe)->attrib;
+	u8 *frame = ((struct xmit_frame *)pxmitframe)->buf_addr + TXDESC_OFFSET;
+	u8 *pSecHeader = NULL, *pos = NULL, *pRA = NULL;
+	u8 bPNOverflow = false, bFindMatchPeer = false, hdr_len = 0;
+	PWLAN_HEADER_WAPI_EXTENSION pWapiExt = NULL;
+	PRT_WAPI_T         pWapiInfo = &padapter->wapiInfo;
+	PRT_WAPI_STA_INFO  pWapiSta = NULL;
+	int ret = 0;
+
+	WAPI_TRACE(WAPI_TX, "=========>%s\n", __FUNCTION__);
+
+	return ret;
+#if 0
+	hdr_len = sMacHdrLng;
+	if (GetFrameType(pskb->data) == WIFI_QOS_DATA_TYPE)
+		hdr_len += 2;
+	/* hdr_len += SNAP_SIZE + sizeof(u16); */
+
+	pos = skb_push(pskb, padapter->wapiInfo.extra_prefix_len);
+	memmove(pos, pos + padapter->wapiInfo.extra_prefix_len, hdr_len);
+
+	pSecHeader = pskb->data + hdr_len;
+	pWapiExt = (PWLAN_HEADER_WAPI_EXTENSION)pSecHeader;
+	pRA = pskb->data + 4;
+
+	WAPI_DATA(WAPI_TX, "FillIV - Before Fill IV", pskb->data, pskb->len);
+
+	/* Address 1 is always receiver's address */
+	if (IS_MCAST(pRA)) {
+		if (!pWapiInfo->wapiTxMsk.bTxEnable) {
+			WAPI_TRACE(WAPI_ERR, "%s: bTxEnable = 0!!\n", __FUNCTION__);
+			return -2;
+		}
+		if (pWapiInfo->wapiTxMsk.keyId <= 1) {
+			pWapiExt->KeyIdx = pWapiInfo->wapiTxMsk.keyId;
+			pWapiExt->Reserved = 0;
+			bPNOverflow = WapiIncreasePN(pWapiInfo->lastTxMulticastPN, 1);
+			_rtw_memcpy(pWapiExt->PN, pWapiInfo->lastTxMulticastPN, 16);
+			if (bPNOverflow) {
+				/* Update MSK Notification. */
+				WAPI_TRACE(WAPI_ERR, "===============>%s():multicast PN overflow\n", __FUNCTION__);
+				rtw_wapi_app_event_handler(padapter, NULL, 0, pRA, false, false, true, 0, false);
+			}
+		} else {
+			WAPI_TRACE(WAPI_ERR, "%s: Invalid Wapi Multicast KeyIdx!!\n", __FUNCTION__);
+			ret = -3;
+		}
+	} else {
+		list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+			if (_rtw_memcmp(pWapiSta->PeerMacAddr, pRA, 6) == _TRUE) {
+				bFindMatchPeer = true;
+				break;
+			}
+		}
+		if (bFindMatchPeer) {
+			if ((!pWapiSta->wapiUskUpdate.bTxEnable) && (!pWapiSta->wapiUsk.bTxEnable)) {
+				WAPI_TRACE(WAPI_ERR, "%s: bTxEnable = 0!!\n", __FUNCTION__);
+				return -4;
+			}
+			if (pWapiSta->wapiUsk.keyId <= 1) {
+				if (pWapiSta->wapiUskUpdate.bTxEnable)
+					pWapiExt->KeyIdx = pWapiSta->wapiUskUpdate.keyId;
+				else
+					pWapiExt->KeyIdx = pWapiSta->wapiUsk.keyId;
+
+				pWapiExt->Reserved = 0;
+				bPNOverflow = WapiIncreasePN(pWapiSta->lastTxUnicastPN, 2);
+				_rtw_memcpy(pWapiExt->PN, pWapiSta->lastTxUnicastPN, 16);
+				if (bPNOverflow) {
+					/* Update USK Notification. */
+					WAPI_TRACE(WAPI_ERR, "===============>%s():unicast PN overflow\n", __FUNCTION__);
+					rtw_wapi_app_event_handler(padapter, NULL, 0, pWapiSta->PeerMacAddr, false, true, false, 0, false);
+				}
+			} else {
+				WAPI_TRACE(WAPI_ERR, "%s: Invalid Wapi Unicast KeyIdx!!\n", __FUNCTION__);
+				ret = -5;
+			}
+		} else {
+			WAPI_TRACE(WAPI_ERR, "%s: Can not find Peer Sta "MAC_FMT"!!\n", __FUNCTION__, MAC_ARG(pRA));
+			ret = -6;
+		}
+	}
+
+	WAPI_DATA(WAPI_TX, "FillIV - After Fill IV", pskb->data, pskb->len);
+	WAPI_TRACE(WAPI_TX, "<=========%s\n", __FUNCTION__);
+	return ret;
+#endif
+}
+
+/* WAPI SW Enc: must have done Coalesce! */
+void SecSWSMS4Encryption(
+	_adapter *padapter,
+	u8 *pxmitframe
+)
+{
+	PRT_WAPI_T		pWapiInfo = &padapter->wapiInfo;
+	PRT_WAPI_STA_INFO   pWapiSta = NULL;
+	u8 *pframe = ((struct xmit_frame *)pxmitframe)->buf_addr + TXDESC_SIZE;
+	struct pkt_attrib *pattrib = &((struct xmit_frame *)pxmitframe)->attrib;
+
+	u8 *SecPtr = NULL, *pRA, *pMicKey = NULL, *pDataKey = NULL, *pIV = NULL;
+	u8 IVOffset, DataOffset, bFindMatchPeer = false, KeyIdx = 0, MicBuffer[16];
+	u16 OutputLength;
+
+	WAPI_TRACE(WAPI_TX, "=========>%s\n", __FUNCTION__);
+
+	WAPI_TRACE(WAPI_TX, "hdrlen: %d\n", pattrib->hdrlen);
+
+	return;
+
+	DataOffset = pattrib->hdrlen + pattrib->iv_len;
+
+	pRA = pframe + 4;
+
+
+	if (IS_MCAST(pRA)) {
+		KeyIdx = pWapiInfo->wapiTxMsk.keyId;
+		pIV = pWapiInfo->lastTxMulticastPN;
+		pMicKey = pWapiInfo->wapiTxMsk.micKey;
+		pDataKey = pWapiInfo->wapiTxMsk.dataKey;
+	} else {
+		if (!list_empty(&(pWapiInfo->wapiSTAUsedList))) {
+			list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+				if (_TRUE == _rtw_memcmp(pWapiSta->PeerMacAddr, pRA, 6)) {
+					bFindMatchPeer = true;
+					break;
+				}
+			}
+
+			if (bFindMatchPeer) {
+				if (pWapiSta->wapiUskUpdate.bTxEnable) {
+					KeyIdx = pWapiSta->wapiUskUpdate.keyId;
+					WAPI_TRACE(WAPI_TX, "%s(): Use update USK!! KeyIdx=%d\n", __FUNCTION__, KeyIdx);
+					pIV = pWapiSta->lastTxUnicastPN;
+					pMicKey = pWapiSta->wapiUskUpdate.micKey;
+					pDataKey = pWapiSta->wapiUskUpdate.dataKey;
+				} else {
+					KeyIdx = pWapiSta->wapiUsk.keyId;
+					WAPI_TRACE(WAPI_TX, "%s(): Use USK!! KeyIdx=%d\n", __FUNCTION__, KeyIdx);
+					pIV = pWapiSta->lastTxUnicastPN;
+					pMicKey = pWapiSta->wapiUsk.micKey;
+					pDataKey = pWapiSta->wapiUsk.dataKey;
+				}
+			} else {
+				WAPI_TRACE(WAPI_ERR, "%s: Can not find Peer Sta!!\n", __FUNCTION__);
+				return;
+			}
+		} else {
+			WAPI_TRACE(WAPI_ERR, "%s: wapiSTAUsedList is empty!!\n", __FUNCTION__);
+			return;
+		}
+	}
+
+	SecPtr = pframe;
+	SecCalculateMicSMS4(KeyIdx, pMicKey, SecPtr, (SecPtr + DataOffset), pattrib->pktlen, MicBuffer);
+
+	WAPI_DATA(WAPI_TX, "Encryption - MIC", MicBuffer, padapter->wapiInfo.extra_postfix_len);
+
+	_rtw_memcpy(pframe + pattrib->hdrlen + pattrib->iv_len + pattrib->pktlen - pattrib->icv_len,
+	       (u8 *)MicBuffer,
+	       padapter->wapiInfo.extra_postfix_len
+	      );
+
+
+	WapiSMS4Encryption(pDataKey, pIV, (SecPtr + DataOffset), pattrib->pktlen + pattrib->icv_len, (SecPtr + DataOffset), &OutputLength);
+
+	WAPI_DATA(WAPI_TX, "Encryption - After SMS4 encryption", pframe, pattrib->hdrlen + pattrib->iv_len + pattrib->pktlen);
+
+	WAPI_TRACE(WAPI_TX, "<=========%s\n", __FUNCTION__);
+}
+
+u8 SecSWSMS4Decryption(
+	_adapter *padapter,
+	u8		*precv_frame,
+	struct recv_priv *precv_priv
+)
+{
+	PRT_WAPI_T pWapiInfo = &padapter->wapiInfo;
+	struct recv_frame_hdr *precv_hdr;
+	PRT_WAPI_STA_INFO   pWapiSta = NULL;
+	u8 IVOffset, DataOffset, bFindMatchPeer = false, bUseUpdatedKey = false;
+	u8 KeyIdx, MicBuffer[16], lastRxPNforQoS[16];
+	u8 *pRA, *pTA, *pMicKey, *pDataKey, *pLastRxPN, *pRecvPN, *pSecData, *pRecvMic, *pos;
+	u8 TID = 0;
+	u16 OutputLength, DataLen;
+	u8   bQosData;
+	struct sk_buff	*pskb;
+
+	WAPI_TRACE(WAPI_RX, "=========>%s\n", __FUNCTION__);
+
+	return 0;
+
+	precv_hdr = &((union recv_frame *)precv_frame)->u.hdr;
+	pskb = (struct sk_buff *)(precv_hdr->rx_data);
+	precv_hdr->bWapiCheckPNInDecrypt = WapiCheckPnInSwDecrypt(padapter, pskb);
+	WAPI_TRACE(WAPI_RX, "=========>%s: check PN  %d\n", __FUNCTION__, precv_hdr->bWapiCheckPNInDecrypt);
+	WAPI_DATA(WAPI_RX, "Decryption - Before decryption", pskb->data, pskb->len);
+
+	IVOffset = sMacHdrLng;
+	bQosData = GetFrameType(pskb->data) == WIFI_QOS_DATA_TYPE;
+	if (bQosData)
+		IVOffset += 2;
+
+	/* if(GetHTC()) */
+	/*	IVOffset += 4; */
+
+	/* IVOffset += SNAP_SIZE + sizeof(u16); */
+
+	DataOffset = IVOffset + padapter->wapiInfo.extra_prefix_len;
+
+	pRA = pskb->data + 4;
+	pTA = pskb->data + 10;
+	KeyIdx = *(pskb->data + IVOffset);
+	pRecvPN = pskb->data + IVOffset + 2;
+	pSecData = pskb->data + DataOffset;
+	DataLen = pskb->len - DataOffset;
+	pRecvMic = pskb->data + pskb->len - padapter->wapiInfo.extra_postfix_len;
+	TID = GetTid(pskb->data);
+
+	if (!list_empty(&(pWapiInfo->wapiSTAUsedList))) {
+		list_for_each_entry(pWapiSta, &pWapiInfo->wapiSTAUsedList, list) {
+			if (_TRUE == _rtw_memcmp(pWapiSta->PeerMacAddr, pTA, 6)) {
+				bFindMatchPeer = true;
+				break;
+			}
+		}
+	}
+
+	if (!bFindMatchPeer) {
+		WAPI_TRACE(WAPI_ERR, "%s: Can not find Peer Sta "MAC_FMT" for Key Info!!!\n", __FUNCTION__, MAC_ARG(pTA));
+		return false;
+	}
+
+	if (IS_MCAST(pRA)) {
+		WAPI_TRACE(WAPI_RX, "%s: Multicast decryption !!!\n", __FUNCTION__);
+		if (pWapiSta->wapiMsk.keyId == KeyIdx && pWapiSta->wapiMsk.bSet) {
+			pLastRxPN = pWapiSta->lastRxMulticastPN;
+			if (!WapiComparePN(pRecvPN, pLastRxPN)) {
+				WAPI_TRACE(WAPI_ERR, "%s: MSK PN is not larger than last, Dropped!!!\n", __FUNCTION__);
+				WAPI_DATA(WAPI_ERR, "pRecvPN:", pRecvPN, 16);
+				WAPI_DATA(WAPI_ERR, "pLastRxPN:", pLastRxPN, 16);
+				return false;
+			}
+
+			_rtw_memcpy(pLastRxPN, pRecvPN, 16);
+			pMicKey = pWapiSta->wapiMsk.micKey;
+			pDataKey = pWapiSta->wapiMsk.dataKey;
+		} else if (pWapiSta->wapiMskUpdate.keyId == KeyIdx && pWapiSta->wapiMskUpdate.bSet) {
+			WAPI_TRACE(WAPI_RX, "%s: Use Updated MSK for Decryption !!!\n", __FUNCTION__);
+			bUseUpdatedKey = true;
+			_rtw_memcpy(pWapiSta->lastRxMulticastPN, pRecvPN, 16);
+			pMicKey = pWapiSta->wapiMskUpdate.micKey;
+			pDataKey = pWapiSta->wapiMskUpdate.dataKey;
+		} else {
+			WAPI_TRACE(WAPI_ERR, "%s: Can not find MSK with matched KeyIdx(%d), Dropped !!!\n", __FUNCTION__, KeyIdx);
+			return false;
+		}
+	} else {
+		WAPI_TRACE(WAPI_RX, "%s: Unicast decryption !!!\n", __FUNCTION__);
+		if (pWapiSta->wapiUsk.keyId == KeyIdx && pWapiSta->wapiUsk.bSet) {
+			WAPI_TRACE(WAPI_RX, "%s: Use USK for Decryption!!!\n", __FUNCTION__);
+			if (precv_hdr->bWapiCheckPNInDecrypt) {
+				if (GetFrameType(pskb->data) == WIFI_QOS_DATA_TYPE) {
+					WapiGetLastRxUnicastPNForQoSData(TID, pWapiSta, lastRxPNforQoS);
+					pLastRxPN = lastRxPNforQoS;
+				} else
+					pLastRxPN = pWapiSta->lastRxUnicastPN;
+				if (!WapiComparePN(pRecvPN, pLastRxPN))
+					return false;
+				if (bQosData)
+					WapiSetLastRxUnicastPNForQoSData(TID, pRecvPN, pWapiSta);
+				else
+					_rtw_memcpy(pWapiSta->lastRxUnicastPN, pRecvPN, 16);
+			} else
+				_rtw_memcpy(precv_hdr->WapiTempPN, pRecvPN, 16);
+
+			if (MLME_IS_STA(padapter)) {
+				if ((pRecvPN[0] & 0x1) == 0) {
+					WAPI_TRACE(WAPI_ERR, "%s: Rx USK PN is not odd when Infra STA mode, Dropped !!!\n", __FUNCTION__);
+					return false;
+				}
+			}
+
+			pMicKey = pWapiSta->wapiUsk.micKey;
+			pDataKey = pWapiSta->wapiUsk.dataKey;
+		} else if (pWapiSta->wapiUskUpdate.keyId == KeyIdx && pWapiSta->wapiUskUpdate.bSet) {
+			WAPI_TRACE(WAPI_RX, "%s: Use Updated USK for Decryption!!!\n", __FUNCTION__);
+			if (pWapiSta->bAuthenticatorInUpdata)
+				bUseUpdatedKey = true;
+			else
+				bUseUpdatedKey = false;
+
+			if (bQosData)
+				WapiSetLastRxUnicastPNForQoSData(TID, pRecvPN, pWapiSta);
+			else
+				_rtw_memcpy(pWapiSta->lastRxUnicastPN, pRecvPN, 16);
+			pMicKey = pWapiSta->wapiUskUpdate.micKey;
+			pDataKey = pWapiSta->wapiUskUpdate.dataKey;
+		} else {
+			WAPI_TRACE(WAPI_ERR, "%s: No valid USK!!!KeyIdx=%d pWapiSta->wapiUsk.keyId=%d pWapiSta->wapiUskUpdate.keyId=%d\n", __FUNCTION__, KeyIdx, pWapiSta->wapiUsk.keyId,
+				   pWapiSta->wapiUskUpdate.keyId);
+			/* dump_buf(pskb->data,pskb->len); */
+			return false;
+		}
+	}
+
+	WAPI_DATA(WAPI_RX, "Decryption - DataKey", pDataKey, 16);
+	WAPI_DATA(WAPI_RX, "Decryption - IV", pRecvPN, 16);
+	WapiSMS4Decryption(pDataKey, pRecvPN, pSecData, DataLen, pSecData, &OutputLength);
+
+	if (OutputLength != DataLen)
+		WAPI_TRACE(WAPI_ERR, "%s:  Output Length Error!!!!\n", __FUNCTION__);
+
+	WAPI_DATA(WAPI_RX, "Decryption - After decryption", pskb->data, pskb->len);
+
+	DataLen -= padapter->wapiInfo.extra_postfix_len;
+
+	SecCalculateMicSMS4(KeyIdx, pMicKey, pskb->data, pSecData, DataLen, MicBuffer);
+
+	WAPI_DATA(WAPI_RX, "Decryption - MIC received", pRecvMic, SMS4_MIC_LEN);
+	WAPI_DATA(WAPI_RX, "Decryption - MIC calculated", MicBuffer, SMS4_MIC_LEN);
+
+	if (_TRUE == _rtw_memcmp(MicBuffer, pRecvMic, padapter->wapiInfo.extra_postfix_len)) {
+		WAPI_TRACE(WAPI_RX, "%s: Check MIC OK!!\n", __FUNCTION__);
+		if (bUseUpdatedKey) {
+			/* delete the old key */
+			if (IS_MCAST(pRA)) {
+				WAPI_TRACE(WAPI_API, "%s(): AE use new update MSK!!\n", __FUNCTION__);
+				pWapiSta->wapiMsk.keyId = pWapiSta->wapiMskUpdate.keyId;
+				_rtw_memcpy(pWapiSta->wapiMsk.dataKey, pWapiSta->wapiMskUpdate.dataKey, 16);
+				_rtw_memcpy(pWapiSta->wapiMsk.micKey, pWapiSta->wapiMskUpdate.micKey, 16);
+				pWapiSta->wapiMskUpdate.bTxEnable = pWapiSta->wapiMskUpdate.bSet = false;
+			} else {
+				WAPI_TRACE(WAPI_API, "%s(): AE use new update USK!!\n", __FUNCTION__);
+				pWapiSta->wapiUsk.keyId = pWapiSta->wapiUskUpdate.keyId;
+				_rtw_memcpy(pWapiSta->wapiUsk.dataKey, pWapiSta->wapiUskUpdate.dataKey, 16);
+				_rtw_memcpy(pWapiSta->wapiUsk.micKey, pWapiSta->wapiUskUpdate.micKey, 16);
+				pWapiSta->wapiUskUpdate.bTxEnable = pWapiSta->wapiUskUpdate.bSet = false;
+			}
+		}
+	} else {
+		WAPI_TRACE(WAPI_ERR, "%s:  Check MIC Error, Dropped !!!!\n", __FUNCTION__);
+		return false;
+	}
+
+	pos = pskb->data;
+	memmove(pos + padapter->wapiInfo.extra_prefix_len, pos, IVOffset);
+	skb_pull(pskb, padapter->wapiInfo.extra_prefix_len);
+
+	WAPI_TRACE(WAPI_RX, "<=========%s\n", __FUNCTION__);
+
+	return true;
+}
+
+u32	rtw_sms4_encrypt(_adapter *padapter, u8 *pxmitframe)
+{
+
+	u8	*pframe;
+	u32 res = _SUCCESS;
+
+	WAPI_TRACE(WAPI_TX, "=========>%s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!padapter->wapiInfo.bWapiEnable)) {
+		WAPI_TRACE(WAPI_TX, "<========== %s, WAPI not supported or enabled!\n", __FUNCTION__);
+		return _FAIL;
+	}
+
+	if (((struct xmit_frame *)pxmitframe)->buf_addr == NULL)
+		return _FAIL;
+
+	pframe = ((struct xmit_frame *)pxmitframe)->buf_addr + TXDESC_OFFSET;
+
+	SecSWSMS4Encryption(padapter, pxmitframe);
+
+	WAPI_TRACE(WAPI_TX, "<=========%s\n", __FUNCTION__);
+	return res;
+}
+
+u32	rtw_sms4_decrypt(_adapter *padapter, u8 *precvframe)
+{
+	u8	*pframe;
+	u32 res = _SUCCESS;
+
+	WAPI_TRACE(WAPI_RX, "=========>%s\n", __FUNCTION__);
+
+	if ((!padapter->WapiSupport) || (!padapter->wapiInfo.bWapiEnable)) {
+		WAPI_TRACE(WAPI_RX, "<========== %s, WAPI not supported or enabled!\n", __FUNCTION__);
+		return _FAIL;
+	}
+
+
+	/* drop packet when hw decrypt fail
+	* return tempraily */
+	return _FAIL;
+
+	/* pframe=(unsigned char *)((union recv_frame*)precvframe)->u.hdr.rx_data; */
+
+	if (false == SecSWSMS4Decryption(padapter, precvframe, &adapter_to_dvobj(padapter)->recvpriv)) {
+		WAPI_TRACE(WAPI_ERR, "%s():SMS4 decrypt frame error\n", __FUNCTION__);
+		return _FAIL;
+	}
+
+	WAPI_TRACE(WAPI_RX, "<=========%s\n", __FUNCTION__);
+	return res;
+}
+
+#else
+
+u32	rtw_sms4_encrypt(_adapter *padapter, u8 *pxmitframe)
+{
+	WAPI_TRACE(WAPI_TX, "=========>Dummy %s\n", __FUNCTION__);
+	WAPI_TRACE(WAPI_TX, "<=========Dummy %s\n", __FUNCTION__);
+	return _SUCCESS;
+}
+
+u32	rtw_sms4_decrypt(_adapter *padapter, u8 *precvframe)
+{
+	WAPI_TRACE(WAPI_RX, "=========>Dummy %s\n", __FUNCTION__);
+	WAPI_TRACE(WAPI_RX, "<=========Dummy %s\n", __FUNCTION__);
+	return _SUCCESS;
+}
+
+#endif
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wlan_util.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wlan_util.c
new file mode 100644
index 000000000000..e523c16d4c42
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wlan_util.c
@@ -0,0 +1,3838 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_WLAN_UTIL_C_
+
+#include <drv_types.h>
+
+unsigned char ARTHEROS_OUI1[] = {0x00, 0x03, 0x7f};
+unsigned char ARTHEROS_OUI2[] = {0x00, 0x13, 0x74};
+
+unsigned char BROADCOM_OUI1[] = {0x00, 0x10, 0x18};
+unsigned char BROADCOM_OUI2[] = {0x00, 0x0a, 0xf7};
+unsigned char BROADCOM_OUI3[] = {0x00, 0x05, 0xb5};
+
+
+unsigned char CISCO_OUI[] = {0x00, 0x40, 0x96};
+unsigned char MARVELL_OUI[] = {0x00, 0x50, 0x43};
+unsigned char RALINK_OUI[] = {0x00, 0x0c, 0x43};
+unsigned char REALTEK_OUI[] = {0x00, 0xe0, 0x4c};
+unsigned char AIRGOCAP_OUI[] = {0x00, 0x0a, 0xf5};
+
+unsigned char REALTEK_96B_IE[] = {0x00, 0xe0, 0x4c, 0x02, 0x01, 0x20};
+
+extern unsigned char RTW_WPA_OUI[];
+extern unsigned char WPA_TKIP_CIPHER[4];
+extern unsigned char RSN_TKIP_CIPHER[4];
+
+#define R2T_PHY_DELAY	(0)
+
+/* #define WAIT_FOR_BCN_TO_MIN	(3000) */
+#define WAIT_FOR_BCN_TO_MIN	(6000)
+#define WAIT_FOR_BCN_TO_MAX	(20000)
+
+static u8 rtw_basic_rate_cck[4] = {
+	IEEE80211_CCK_RATE_1MB | IEEE80211_BASIC_RATE_MASK, IEEE80211_CCK_RATE_2MB | IEEE80211_BASIC_RATE_MASK,
+	IEEE80211_CCK_RATE_5MB | IEEE80211_BASIC_RATE_MASK, IEEE80211_CCK_RATE_11MB | IEEE80211_BASIC_RATE_MASK
+};
+
+static u8 rtw_basic_rate_ofdm[3] = {
+	IEEE80211_OFDM_RATE_6MB | IEEE80211_BASIC_RATE_MASK, IEEE80211_OFDM_RATE_12MB | IEEE80211_BASIC_RATE_MASK,
+	IEEE80211_OFDM_RATE_24MB | IEEE80211_BASIC_RATE_MASK
+};
+
+static u8 rtw_basic_rate_mix[7] = {
+	IEEE80211_CCK_RATE_1MB | IEEE80211_BASIC_RATE_MASK, IEEE80211_CCK_RATE_2MB | IEEE80211_BASIC_RATE_MASK,
+	IEEE80211_CCK_RATE_5MB | IEEE80211_BASIC_RATE_MASK, IEEE80211_CCK_RATE_11MB | IEEE80211_BASIC_RATE_MASK,
+	IEEE80211_OFDM_RATE_6MB | IEEE80211_BASIC_RATE_MASK, IEEE80211_OFDM_RATE_12MB | IEEE80211_BASIC_RATE_MASK,
+	IEEE80211_OFDM_RATE_24MB | IEEE80211_BASIC_RATE_MASK
+};
+
+extern u8	WIFI_CCKRATES[];
+bool rtw_is_cck_rate(u8 rate)
+{
+	int i;
+
+	for (i = 0; i < 4; i++)
+		if ((WIFI_CCKRATES[i] & 0x7F) == (rate & 0x7F))
+			return 1;
+	return 0;
+}
+
+extern u8	WIFI_OFDMRATES[];
+bool rtw_is_ofdm_rate(u8 rate)
+{
+	int i;
+
+	for (i = 0; i < 8; i++)
+		if ((WIFI_OFDMRATES[i] & 0x7F) == (rate & 0x7F))
+			return 1;
+	return 0;
+}
+
+/* test if rate is defined in rtw_basic_rate_cck */
+bool rtw_is_basic_rate_cck(u8 rate)
+{
+	int i;
+
+	for (i = 0; i < 4; i++)
+		if ((rtw_basic_rate_cck[i] & 0x7F) == (rate & 0x7F))
+			return 1;
+	return 0;
+}
+
+/* test if rate is defined in rtw_basic_rate_ofdm */
+bool rtw_is_basic_rate_ofdm(u8 rate)
+{
+	int i;
+
+	for (i = 0; i < 3; i++)
+		if ((rtw_basic_rate_ofdm[i] & 0x7F) == (rate & 0x7F))
+			return 1;
+	return 0;
+}
+
+/* test if rate is defined in rtw_basic_rate_mix */
+bool rtw_is_basic_rate_mix(u8 rate)
+{
+	int i;
+
+	for (i = 0; i < 7; i++)
+		if ((rtw_basic_rate_mix[i] & 0x7F) == (rate & 0x7F))
+			return 1;
+	return 0;
+}
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+int new_bcn_max = RTW_BCN_NEW_MAX_CNT;
+#endif
+int cckrates_included(unsigned char *rate, int ratelen)
+{
+	int	i;
+
+	for (i = 0; i < ratelen; i++) {
+		if ((((rate[i]) & 0x7f) == 2)	|| (((rate[i]) & 0x7f) == 4) ||
+		    (((rate[i]) & 0x7f) == 11)  || (((rate[i]) & 0x7f) == 22))
+			return _TRUE;
+	}
+
+	return _FALSE;
+
+}
+
+int cckratesonly_included(unsigned char *rate, int ratelen)
+{
+	int	i;
+
+	for (i = 0; i < ratelen; i++) {
+		if ((((rate[i]) & 0x7f) != 2) && (((rate[i]) & 0x7f) != 4) &&
+		    (((rate[i]) & 0x7f) != 11)  && (((rate[i]) & 0x7f) != 22))
+			return _FALSE;
+	}
+
+	return _TRUE;
+}
+
+s8 rtw_get_sta_rx_nss(_adapter *adapter, struct sta_info *psta)
+{
+	s8 nss = 1;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct _ADAPTER_LINK *padapter_link = NULL;
+
+	if (!psta)
+		return nss;
+
+	padapter_link = psta->padapter_link;
+	nss = get_phy_rx_nss(adapter, padapter_link);
+
+#ifdef CONFIG_80211N_HT
+	#ifdef CONFIG_80211AC_VHT
+	#ifdef CONFIG_80211AX_HE
+	if (psta->hepriv.he_option)
+		nss = psta->phl_sta->asoc_cap.nss_tx;
+	else
+	#endif /* CONFIG_80211AX_HE */
+	if (psta->vhtpriv.vht_option)
+		nss = rtw_min(nss, rtw_vht_mcsmap_to_nss(psta->vhtpriv.vht_mcs_map));
+	else
+	#endif /* CONFIG_80211AC_VHT */
+	if (psta->htpriv.ht_option)
+		nss = rtw_min(nss, rtw_ht_mcsset_to_nss(psta->htpriv.ht_cap.supp_mcs_set));
+#endif /*CONFIG_80211N_HT*/
+	RTW_INFO("%s: %d ss\n", __func__, nss);
+	return nss;
+}
+
+s8 rtw_get_sta_tx_nss(_adapter *adapter, struct sta_info *psta)
+{
+	s8 nss = 1;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+
+	if (!psta)
+		return nss;
+
+	padapter_link = psta->padapter_link;
+	nss = get_phy_tx_nss(adapter, padapter_link);
+
+#ifdef CONFIG_80211N_HT
+	#ifdef CONFIG_80211AC_VHT
+	#ifdef CONFIG_80211AX_HE
+	if (psta->hepriv.he_option)
+		nss = psta->phl_sta->asoc_cap.nss_rx;
+	else
+	#endif /* CONFIG_80211AX_HE */
+	if (psta->vhtpriv.vht_option)
+		nss = rtw_min(nss, rtw_vht_mcsmap_to_nss(psta->vhtpriv.vht_mcs_map));
+	else
+	#endif /* CONFIG_80211AC_VHT */
+	if (psta->htpriv.ht_option)
+		nss = rtw_min(nss, rtw_ht_mcsset_to_nss(psta->htpriv.ht_cap.supp_mcs_set));
+#endif /*CONFIG_80211N_HT*/
+	RTW_INFO("%s: %d SS\n", __func__, nss);
+	return nss;
+}
+
+unsigned char ratetbl_val_2wifirate(unsigned char rate)
+{
+	unsigned char val = 0;
+
+	switch (rate & 0x7f) {
+	case 0:
+		val = IEEE80211_CCK_RATE_1MB;
+		break;
+
+	case 1:
+		val = IEEE80211_CCK_RATE_2MB;
+		break;
+
+	case 2:
+		val = IEEE80211_CCK_RATE_5MB;
+		break;
+
+	case 3:
+		val = IEEE80211_CCK_RATE_11MB;
+		break;
+
+	case 4:
+		val = IEEE80211_OFDM_RATE_6MB;
+		break;
+
+	case 5:
+		val = IEEE80211_OFDM_RATE_9MB;
+		break;
+
+	case 6:
+		val = IEEE80211_OFDM_RATE_12MB;
+		break;
+
+	case 7:
+		val = IEEE80211_OFDM_RATE_18MB;
+		break;
+
+	case 8:
+		val = IEEE80211_OFDM_RATE_24MB;
+		break;
+
+	case 9:
+		val = IEEE80211_OFDM_RATE_36MB;
+		break;
+
+	case 10:
+		val = IEEE80211_OFDM_RATE_48MB;
+		break;
+
+	case 11:
+		val = IEEE80211_OFDM_RATE_54MB;
+		break;
+
+	}
+
+	return val;
+
+}
+
+int is_basicrate(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, unsigned char rate)
+{
+	int i;
+	unsigned char val;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+
+	for (i = 0; i < NumRates; i++) {
+		val = pmlmeext->basicrate[i];
+
+		if ((val != 0xff) && (val != 0xfe)) {
+			if (rate == ratetbl_val_2wifirate(val))
+				return _TRUE;
+		}
+	}
+
+	return _FALSE;
+}
+
+unsigned int ratetbl2rateset(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				unsigned char *rateset)
+{
+	int i;
+	unsigned char rate;
+	unsigned int	len = 0;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+
+	for (i = 0; i < NumRates; i++) {
+		rate = pmlmeext->datarate[i];
+
+		if (rtw_get_oper_band(padapter, padapter_link) != BAND_ON_24G && rate < _6M_RATE_) /* non 2G no support CCK rate*/
+			continue;
+
+		switch (rate) {
+		case 0xff:
+			return len;
+
+		case 0xfe:
+			continue;
+
+		default:
+			rate = ratetbl_val_2wifirate(rate);
+
+			if (is_basicrate(padapter, padapter_link, rate) == _TRUE)
+				rate |= IEEE80211_BASIC_RATE_MASK;
+
+			rateset[len] = rate;
+			len++;
+			break;
+		}
+	}
+	return len;
+}
+
+void get_rate_set(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *pbssrate, int *bssrate_len)
+{
+	unsigned char supportedrates[NumRates];
+
+	_rtw_memset(supportedrates, 0, NumRates);
+	*bssrate_len = ratetbl2rateset(padapter, padapter_link, supportedrates);
+	_rtw_memcpy(pbssrate, supportedrates, *bssrate_len);
+}
+
+void set_mcs_rate_by_mask(u8 *mcs_set, u32 mask)
+{
+	u8 mcs_rate_1r = (u8)(mask & 0xff);
+	u8 mcs_rate_2r = (u8)((mask >> 8) & 0xff);
+	u8 mcs_rate_3r = (u8)((mask >> 16) & 0xff);
+	u8 mcs_rate_4r = (u8)((mask >> 24) & 0xff);
+
+	mcs_set[0] &= mcs_rate_1r;
+	mcs_set[1] &= mcs_rate_2r;
+	mcs_set[2] &= mcs_rate_3r;
+	mcs_set[3] &= mcs_rate_4r;
+}
+
+void UpdateBrateTbl(
+	_adapter *adapter,
+	u8			*mBratesOS
+)
+{
+	u8	i;
+	u8	rate;
+
+	/* 1M, 2M, 5.5M, 11M, 6M, 12M, 24M are mandatory. */
+	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
+		rate = mBratesOS[i] & 0x7f;
+		switch (rate) {
+		case IEEE80211_CCK_RATE_1MB:
+		case IEEE80211_CCK_RATE_2MB:
+		case IEEE80211_CCK_RATE_5MB:
+		case IEEE80211_CCK_RATE_11MB:
+		case IEEE80211_OFDM_RATE_6MB:
+		case IEEE80211_OFDM_RATE_12MB:
+		case IEEE80211_OFDM_RATE_24MB:
+			mBratesOS[i] |= IEEE80211_BASIC_RATE_MASK;
+			break;
+		}
+	}
+
+}
+
+void UpdateBrateTblForSoftAP(u8 *bssrateset, u32 bssratelen)
+{
+	u8	i;
+	u8	rate;
+
+	for (i = 0; i < bssratelen; i++) {
+		rate = bssrateset[i] & 0x7f;
+		switch (rate) {
+		case IEEE80211_CCK_RATE_1MB:
+		case IEEE80211_CCK_RATE_2MB:
+		case IEEE80211_CCK_RATE_5MB:
+		case IEEE80211_CCK_RATE_11MB:
+			bssrateset[i] |= IEEE80211_BASIC_RATE_MASK;
+			break;
+		}
+	}
+
+}
+/*rtw_phl_mr_get_chandef(dvobj->phl, adapter->phl_role, &chandef); => mr union chan*/
+/*rtw_phl_get_cur_hal_chdef(adapter->phl_role, &chandef) => hal chan*/
+
+void rtw_get_oper_chdef(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, struct rtw_chan_def *chandef)
+{
+	if (!adapter->phl_role)
+		return;
+
+	if (rtw_phl_get_cur_hal_chdef(adapter->phl_role, adapter_link->wrlink, chandef) != RTW_PHL_STATUS_SUCCESS)
+		RTW_ERR("%s failed\n", __func__);
+}
+
+u8 rtw_get_oper_band(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	struct rtw_chan_def cur_chandef = {0};
+
+	rtw_get_oper_chdef(adapter, adapter_link, &cur_chandef);
+	return cur_chandef.band;
+}
+
+u8 rtw_get_oper_ch(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	struct rtw_chan_def cur_chandef = {0};
+
+	rtw_get_oper_chdef(adapter, adapter_link, &cur_chandef);
+	return cur_chandef.chan;
+}
+
+u8 rtw_get_oper_bw(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	struct rtw_chan_def cur_chandef = {0};
+
+	rtw_get_oper_chdef(adapter, adapter_link, &cur_chandef);
+	return cur_chandef.bw;
+}
+
+u8 rtw_get_oper_choffset(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	struct rtw_chan_def cur_chandef = {0};
+
+	rtw_get_oper_chdef(adapter, adapter_link, &cur_chandef);
+	return cur_chandef.offset;
+}
+
+int rtw_get_oper_chdef_by_hwband(struct dvobj_priv *dvobj, enum phl_band_idx band_idx
+	, struct rtw_chan_def *chandef)
+{
+	if (rtw_phl_get_cur_hal_chdef_by_hwband(GET_PHL_INFO(dvobj), band_idx, chandef) != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s failed\n", __func__);
+		return _FAIL;
+	}
+	return _SUCCESS;
+}
+
+int rtw_get_oper_bchbw_by_hwband(struct dvobj_priv *dvobj, enum phl_band_idx band_idx
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	struct rtw_chan_def chdef;
+
+	if (rtw_phl_get_cur_hal_chdef_by_hwband(GET_PHL_INFO(dvobj), band_idx, &chdef) != RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("%s failed\n", __func__);
+		*ch = 0;
+		return _FAIL;
+	}
+
+	*band = chdef.band;
+	*ch = chdef.chan;
+	*bw = chdef.bw;
+	*offset = chdef.offset;
+	return _SUCCESS;
+}
+
+inline systime rtw_get_on_oper_ch_time(_adapter *adapter)
+{
+	return adapter_to_dvobj(adapter)->on_oper_ch_time;
+}
+
+inline systime rtw_get_on_cur_ch_time(_adapter *adapter)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	if (adapter_link && adapter_link->mlmeextpriv.chandef.chan == rtw_get_oper_ch(adapter, adapter_link))
+		return adapter_to_dvobj(adapter)->on_oper_ch_time;
+	else
+		return 0;
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY void set_channel_bwmode(_adapter *padapter,
+				struct _ADAPTER_LINK *padapter_link,
+				unsigned char channel,
+				unsigned char channel_offset,
+				unsigned short bwmode,
+				enum rfk_tri_type rt_type)
+{
+	rtw_hw_set_ch_bw(padapter, padapter_link, (channel > 14) ? BAND_ON_5G : BAND_ON_24G,
+		channel, (enum channel_width)bwmode, channel_offset, rt_type);
+}
+#endif
+
+void set_bch_bwmode(_adapter *padapter,
+				struct _ADAPTER_LINK *padapter_link,
+				enum band_type band,
+				unsigned char channel,
+				unsigned char channel_offset,
+				unsigned short bwmode,
+				enum rfk_tri_type rt_type)
+{
+	rtw_hw_set_ch_bw(padapter, padapter_link, band,
+		channel, (enum channel_width)bwmode, channel_offset, rt_type);
+}
+
+__inline u8 *get_my_bssid(WLAN_BSSID_EX *pnetwork)
+{
+	return pnetwork->MacAddress;
+}
+
+u16 get_beacon_interval(WLAN_BSSID_EX *bss)
+{
+	unsigned short val;
+	_rtw_memcpy((unsigned char *)&val, rtw_get_beacon_interval_from_ie(bss->IEs), 2);
+
+	return le16_to_cpu(val);
+
+}
+
+int is_client_associated_to_ap(_adapter *padapter)
+{
+	struct mlme_ext_priv	*pmlmeext;
+	struct mlme_ext_info	*pmlmeinfo;
+
+	if (!padapter)
+		return _FAIL;
+
+	pmlmeext = &padapter->mlmeextpriv;
+	pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if ((pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) && ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE))
+		return _TRUE;
+	else
+		return _FAIL;
+}
+
+int is_client_associated_to_ibss(_adapter *padapter)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if ((pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) && ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE))
+		return _TRUE;
+	else
+		return _FAIL;
+}
+
+/*GEORGIA_TODO_FIXIT*/
+#define GET_H2CCMD_MSRRPT_PARM_OPMODE(__pH2CCmd) 	LE_BITS_TO_1BYTE(((u8 *)(__pH2CCmd)), 0, 1)
+#define GET_H2CCMD_MSRRPT_PARM_ROLE(__pH2CCmd)	LE_BITS_TO_1BYTE(((u8 *)(__pH2CCmd)), 4, 4)
+
+int is_IBSS_empty(_adapter *padapter)
+{
+/* ToDo */
+#if 0
+	int i;
+	struct macid_ctl_t *macid_ctl = &padapter->dvobj->macid_ctl;
+
+	for (i = 0; i < macid_ctl->num; i++) {
+		if (!rtw_macid_is_used(macid_ctl, i))
+			continue;
+		if (!rtw_macid_is_iface_specific(macid_ctl, i, padapter))
+			continue;
+		if (!GET_H2CCMD_MSRRPT_PARM_OPMODE(&macid_ctl->h2c_msr[i]))
+			continue;
+		if (GET_H2CCMD_MSRRPT_PARM_ROLE(&macid_ctl->h2c_msr[i]) == H2C_MSR_ROLE_ADHOC)
+			return _FAIL;
+	}
+#endif
+	return _TRUE;
+}
+
+unsigned int decide_wait_for_beacon_timeout(unsigned int bcn_interval)
+{
+	if ((bcn_interval << 2) < WAIT_FOR_BCN_TO_MIN)
+		return WAIT_FOR_BCN_TO_MIN;
+	else if ((bcn_interval << 2) > WAIT_FOR_BCN_TO_MAX)
+		return WAIT_FOR_BCN_TO_MAX;
+	else
+		return bcn_interval << 2;
+}
+
+#if defined(CONFIG_P2P) && defined(CONFIG_WFD)
+void rtw_process_wfd_ie(_adapter *adapter, u8 *wfd_ie, u8 wfd_ielen, const char *tag)
+{
+	struct wifidirect_info *wdinfo = &adapter->wdinfo;
+
+	u8 *attr_content;
+	u32 attr_contentlen = 0;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		return;
+
+	RTW_INFO("[%s] Found WFD IE\n", tag);
+	attr_content = rtw_get_wfd_attr_content(wfd_ie, wfd_ielen, WFD_ATTR_DEVICE_INFO, NULL, &attr_contentlen);
+	if (attr_content && attr_contentlen) {
+		wdinfo->wfd_info->peer_rtsp_ctrlport = RTW_GET_BE16(attr_content + 2);
+		RTW_INFO("[%s] Peer PORT NUM = %d\n", tag, wdinfo->wfd_info->peer_rtsp_ctrlport);
+	}
+}
+
+void rtw_process_wfd_ies(_adapter *adapter, u8 *ies, u8 ies_len, const char *tag)
+{
+	u8 *wfd_ie;
+	u32	wfd_ielen;
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(adapter), WL_FUNC_MIRACAST))
+		return;
+
+	wfd_ie = rtw_get_wfd_ie(ies, ies_len, NULL, &wfd_ielen);
+	while (wfd_ie) {
+		rtw_process_wfd_ie(adapter, wfd_ie, wfd_ielen, tag);
+		wfd_ie = rtw_get_wfd_ie(wfd_ie + wfd_ielen, (ies + ies_len) - (wfd_ie + wfd_ielen), NULL, &wfd_ielen);
+	}
+}
+#endif /* defined(CONFIG_P2P) && defined(CONFIG_WFD) */
+
+int WMM_param_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs	pIE)
+{
+	/* struct registry_priv	*pregpriv = &padapter->registrypriv; */
+	struct link_mlme_priv	*pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (pmlmepriv->qospriv.qos_option == 0) {
+		pmlmeinfo->WMM_enable = 0;
+		return _FALSE;
+	}
+
+	if (_rtw_memcmp(&(pmlmeinfo->WMM_param), (pIE->data + 6), sizeof(struct WMM_para_element)))
+		return _FALSE;
+	else
+		_rtw_memcpy(&(pmlmeinfo->WMM_param), (pIE->data + 6), sizeof(struct WMM_para_element));
+	pmlmeinfo->WMM_enable = 1;
+	return _TRUE;
+
+#if 0
+	if (pregpriv->wifi_spec == 1) {
+		if (pmlmeinfo->WMM_enable == 1) {
+			/* todo: compare the parameter set count & decide wheher to update or not */
+			return _FAIL;
+		} else {
+			pmlmeinfo->WMM_enable = 1;
+			_rtw_rtw_memcpy(&(pmlmeinfo->WMM_param), (pIE->data + 6), sizeof(struct WMM_para_element));
+			return _TRUE;
+		}
+	} else {
+		pmlmeinfo->WMM_enable = 0;
+		return _FAIL;
+	}
+#endif
+
+}
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+u8 rtw_is_tbtx_capabilty(u8 *p, u8 len){
+	int i;
+	u8 tbtx_cap_ie[8] = {0x00, 0xe0, 0x4c, 0x01, 0x00, 0x00, 0x00, 0x00};
+
+	for (i = 0; i < len; i++) {
+		if (*(p + i) != tbtx_cap_ie[i])
+			return _FALSE;
+		else
+			continue;
+	}
+	return _TRUE;
+}
+#endif
+
+void WMMOnAssocRsp(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	u8	ACI, ACM, AIFS, ECWMin, ECWMax, aSifsTime;
+	u8	acm_mask;
+	u16	TXOP;
+	u32	acParm, i;
+	u32	edca[4], inx[4];
+	u8 ac_be = 0;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	acm_mask = 0;
+	if (WIFI_ROLE_LINK_IS_ON_5G(padapter_link) ||
+	    WIFI_ROLE_LINK_IS_ON_6G(padapter_link) ||
+	    (pmlmeext->cur_wireless_mode & WLAN_MD_11N))
+		aSifsTime = 16;
+	else
+		aSifsTime = 10;
+
+	if (pmlmeinfo->WMM_enable == 0) {
+		padapter->mlmepriv.acm_mask = 0;
+
+		AIFS = aSifsTime + (2 * pmlmeinfo->slotTime);
+
+		if (pmlmeext->cur_wireless_mode & (WLAN_MD_11G | WLAN_MD_11A)) {
+			ECWMin = 4;
+			ECWMax = 10;
+		} else if (pmlmeext->cur_wireless_mode & WLAN_MD_11B) {
+			ECWMin = 5;
+			ECWMax = 10;
+		} else {
+			ECWMin = 4;
+			ECWMax = 10;
+		}
+
+		TXOP = 0;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_hw_set_edca(padapter, padapter_link, 0, acParm);
+		rtw_update_phl_sta_edca(padapter, padapter_link, RTW_AC_BE, acParm);
+		rtw_hw_set_edca(padapter, padapter_link, 1, acParm);
+		rtw_update_phl_sta_edca(padapter, padapter_link, RTW_AC_BK, acParm);
+		rtw_hw_set_edca(padapter, padapter_link, 2, acParm);
+		rtw_update_phl_sta_edca(padapter, padapter_link, RTW_AC_VI, acParm);
+
+		ECWMin = 2;
+		ECWMax = 3;
+		TXOP = 0x2f;
+		acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+		rtw_hw_set_edca(padapter, padapter_link, 3, acParm);
+		rtw_update_phl_sta_edca(padapter, padapter_link, RTW_AC_VO, acParm);
+	} else {
+		edca[0] = edca[1] = edca[2] = edca[3] = 0;
+
+		for (i = 0; i < 4; i++) {
+			ACI = (pmlmeinfo->WMM_param.ac_param[i].ACI_AIFSN >> 5) & 0x03;
+			ACM = (pmlmeinfo->WMM_param.ac_param[i].ACI_AIFSN >> 4) & 0x01;
+
+			/* AIFS = AIFSN * slot time + SIFS - r2t phy delay */
+			AIFS = (pmlmeinfo->WMM_param.ac_param[i].ACI_AIFSN & 0x0f) * pmlmeinfo->slotTime + aSifsTime;
+
+			ECWMin = (pmlmeinfo->WMM_param.ac_param[i].CW & 0x0f);
+			ECWMax = (pmlmeinfo->WMM_param.ac_param[i].CW & 0xf0) >> 4;
+			TXOP = le16_to_cpu(pmlmeinfo->WMM_param.ac_param[i].TXOP_limit);
+
+			acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+			rtw_hw_set_edca(padapter, padapter_link, ACI, acParm);
+			rtw_update_phl_sta_edca(padapter, padapter_link, ACI, acParm);
+
+			switch (ACI) {
+			case 0x0:
+				acm_mask |= (ACM ? BIT(1) : 0);
+				edca[XMIT_BE_QUEUE] = acParm;
+				break;
+
+			case 0x1:
+				/* acm_mask |= (ACM? BIT(0):0); */
+				edca[XMIT_BK_QUEUE] = acParm;
+				break;
+
+			case 0x2:
+				acm_mask |= (ACM ? BIT(2) : 0);
+				edca[XMIT_VI_QUEUE] = acParm;
+				break;
+
+			case 0x3:
+				acm_mask |= (ACM ? BIT(3) : 0);
+				edca[XMIT_VO_QUEUE] = acParm;
+				break;
+			}
+
+			RTW_INFO("WMM(%x): %x, %x\n", ACI, ACM, acParm);
+
+			if (i == ac_be) {
+				padapter->last_edca = acParm;
+				acParm = rtw_get_turbo_edca(padapter, AIFS, ECWMin, ECWMax, TXOP);
+				if (acParm) {
+					rtw_hw_set_edca(padapter, padapter_link, ACI, acParm);
+					padapter->last_edca = acParm;
+				}
+			}
+
+		}
+
+		if (padapter->registrypriv.acm_method == 1)
+			rtw_hal_set_hwreg(padapter, HW_VAR_ACM_CTRL, (u8 *)(&acm_mask));
+		else
+			padapter->mlmepriv.acm_mask = acm_mask;
+
+		inx[0] = 0;
+		inx[1] = 1;
+		inx[2] = 2;
+		inx[3] = 3;
+
+		if (pregpriv->wifi_spec == 1) {
+			u32	j, tmp, change_inx = _FALSE;
+
+			/* entry indx: 0->vo, 1->vi, 2->be, 3->bk. */
+			for (i = 0; i < 4; i++) {
+				for (j = i + 1; j < 4; j++) {
+					/* compare CW and AIFS */
+					if ((edca[j] & 0xFFFF) < (edca[i] & 0xFFFF))
+						change_inx = _TRUE;
+					else if ((edca[j] & 0xFFFF) == (edca[i] & 0xFFFF)) {
+						/* compare TXOP */
+						if ((edca[j] >> 16) > (edca[i] >> 16))
+							change_inx = _TRUE;
+					}
+
+					if (change_inx) {
+						tmp = edca[i];
+						edca[i] = edca[j];
+						edca[j] = tmp;
+
+						tmp = inx[i];
+						inx[i] = inx[j];
+						inx[j] = tmp;
+
+						change_inx = _FALSE;
+					}
+				}
+			}
+		}
+
+		for (i = 0; i < 4; i++) {
+			pxmitpriv->wmm_para_seq[i] = inx[i];
+			RTW_INFO("wmm_para_seq(%d): %d\n", i, pxmitpriv->wmm_para_seq[i]);
+		}
+	}
+}
+
+static void bwmode_update_check(_adapter *padapter,struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE)
+{
+#ifdef CONFIG_80211N_HT
+	unsigned char	 new_bwmode;
+	unsigned char  new_ch_offset;
+	struct HT_info_element	*pHT_info;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct link_mlme_priv	*pmlmepriv = &(padapter_link->mlmepriv);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct ht_priv			*phtpriv = &pmlmepriv->htpriv;
+	u8	cbw40_enable = 0;
+
+	if (!pIE)
+		return;
+
+	if (phtpriv->ht_option == _FALSE)
+		return;
+
+	if (pmlmeext->chandef.bw >= CHANNEL_WIDTH_80)
+		return;
+
+	if (pmlmeext->chandef.bw != CHANNEL_WIDTH_40) /* not FC STA */
+		return;
+
+	if (pIE->Length > sizeof(struct HT_info_element))
+		return;
+
+	pHT_info = (struct HT_info_element *)pIE->data;
+
+	if (rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_40M)) {
+		if (pmlmeext->chandef.chan > 14) {
+			if (REGSTY_IS_BW_5G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+				cbw40_enable = 1;
+		} else {
+			if (REGSTY_IS_BW_2G_SUPPORT(pregistrypriv, CHANNEL_WIDTH_40))
+				cbw40_enable = 1;
+		}
+	}
+
+	if ((pHT_info->infos[0] & BIT(2)) && cbw40_enable) {
+		new_bwmode = CHANNEL_WIDTH_40;
+
+		switch (pHT_info->infos[0] & 0x3) {
+		case 1:
+			new_ch_offset = CHAN_OFFSET_UPPER;
+			break;
+
+		case 3:
+			new_ch_offset = CHAN_OFFSET_LOWER;
+			break;
+
+		default:
+			new_bwmode = CHANNEL_WIDTH_20;
+			new_ch_offset = CHAN_OFFSET_NO_EXT;
+			break;
+		}
+	} else {
+		new_bwmode = CHANNEL_WIDTH_20;
+		new_ch_offset = CHAN_OFFSET_NO_EXT;
+	}
+
+
+	if ((pmlmepriv->sw_to_20mhz == 0 && new_bwmode == CHANNEL_WIDTH_20)
+		|| (pmlmepriv->sw_to_20mhz == 1 && new_bwmode == CHANNEL_WIDTH_40)
+	) {
+		pmlmeinfo->bwmode_updated = _TRUE;
+		pmlmepriv->sw_to_20mhz = new_bwmode == CHANNEL_WIDTH_20 ? 1 : 0;
+		RTW_INFO(FUNC_ADPT_FMT" switching to %uMHz\n", FUNC_ADPT_ARG(padapter)
+			, pmlmepriv->sw_to_20mhz ? 20 : 40);
+
+		/* update HT info also */
+		HT_info_handler(padapter, padapter_link, pIE);
+	} else
+		pmlmeinfo->bwmode_updated = _FALSE;
+
+	if (_TRUE == pmlmeinfo->bwmode_updated) {
+		struct sta_info *psta;
+		WLAN_BSSID_EX	*cur_network = &(pmlmeinfo->network);
+		struct sta_priv	*pstapriv = &padapter->stapriv;
+
+		/* update ap's stainfo */
+		psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+		if (psta) {
+			struct ht_priv	*phtpriv_sta = &psta->htpriv;
+
+			psta->phl_sta->chandef.bw = new_bwmode;
+			phtpriv_sta->ch_offset = new_ch_offset;
+
+			rtw_sta_hal_ra_mask_update_cmd(padapter, psta, 0);
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+}
+
+#ifdef PRIVATE_R
+void Supported_rate_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	unsigned int	i;
+	struct link_mlme_ext_priv		*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info		*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (pIE == NULL)
+		return;
+
+	for (i = 0 ; i < pIE->Length; i++)
+		pmlmeinfo->SupportedRates_infra_ap[i] = (pIE->data[i]);
+
+}
+
+void Extended_Supported_rate_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	unsigned int i, j;
+	struct link_mlme_ext_priv		*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info		*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (pIE == NULL)
+		return;
+
+	if (pIE->Length > 0) {
+		for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
+			if (pmlmeinfo->SupportedRates_infra_ap[i] == 0)
+				break;
+		}
+		for (j = 0; j < pIE->Length; j++)
+			pmlmeinfo->SupportedRates_infra_ap[i+j] = (pIE->data[j]);
+	}
+
+}
+
+void HT_get_ss_from_mcs_set(u8 *mcs_set, u8 *Rx_ss)
+{
+	u8 i, j;
+	u8 r_ss = 0, t_ss = 0;
+
+	for (i = 0; i < 4; i++) {
+		if ((mcs_set[3-i] & 0xff) != 0x00) {
+			r_ss = 4-i;
+			break;
+		}
+	}
+
+	*Rx_ss = r_ss;
+}
+
+void HT_caps_handler_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	unsigned int	i;
+	u8	cur_stbc_cap_infra_ap = 0;
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct ht_priv_infra_ap		*phtpriv = &pmlmepriv->htpriv_infra_ap;
+
+	struct link_mlme_ext_priv		*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info		*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (pIE == NULL)
+		return;
+
+	pmlmeinfo->ht_vht_received |= BIT(0);
+
+	/*copy MCS_SET*/
+	for (i = 3; i < 19; i++)
+		phtpriv->MCS_set_infra_ap[i-3] = (pIE->data[i]);
+
+	/*get number of stream from mcs set*/
+	HT_get_ss_from_mcs_set(phtpriv->MCS_set_infra_ap, &phtpriv->Rx_ss_infra_ap);
+
+	phtpriv->rx_highest_data_rate_infra_ap = le16_to_cpu(GET_HT_CAP_ELE_RX_HIGHEST_DATA_RATE(pIE->data));
+
+	phtpriv->ldpc_cap_infra_ap = GET_HT_CAP_ELE_LDPC_CAP(pIE->data);
+
+	if (GET_HT_CAP_ELE_RX_STBC(pIE->data))
+		SET_FLAG(cur_stbc_cap_infra_ap, STBC_HT_ENABLE_RX);
+	if (GET_HT_CAP_ELE_TX_STBC(pIE->data))
+		SET_FLAG(cur_stbc_cap_infra_ap, STBC_HT_ENABLE_TX);
+	phtpriv->stbc_cap_infra_ap = cur_stbc_cap_infra_ap;
+
+	/*store ap info SGI 20m 40m*/
+	phtpriv->sgi_20m_infra_ap = GET_HT_CAP_ELE_SHORT_GI20M(pIE->data);
+	phtpriv->sgi_40m_infra_ap = GET_HT_CAP_ELE_SHORT_GI40M(pIE->data);
+
+	/*store ap info for supported channel bandwidth*/
+	phtpriv->channel_width_infra_ap = GET_HT_CAP_ELE_CHL_WIDTH(pIE->data);
+}
+#endif /* PRIVATE_R */
+
+void HT_caps_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE)
+{
+#ifdef CONFIG_80211N_HT
+	unsigned int i;
+	u8 max_AMPDU_len, min_MPDU_spacing;
+	u8 cur_ldpc_cap = 0, cur_stbc_cap = 0, cur_beamform_cap = 0, rx_nss = 0;
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	struct protocol_cap_t *protocol_cap = &padapter_link->wrlink->protocol_cap;
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
+	struct sta_info *sta = NULL;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+#ifdef CONFIG_DISABLE_MCS13TO15
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+#endif
+
+	if (pIE == NULL)
+		return;
+
+	if (phtpriv->ht_option == _FALSE)
+		return;
+
+	if (pIE->Length > HT_CAPS_IE_LEN)
+		return;
+
+	pmlmeinfo->HT_caps_enable = 1;
+
+	for (i = 0; i < (pIE->Length); i++) {
+		if (i != 2) {
+			/*	Commented by Albert 2010/07/12 */
+			/*	Got the endian issue here. */
+			pmlmeinfo->HT_caps.u.HT_cap[i] &= *(pIE->data + i);
+		} else {
+			/* AMPDU Parameters field */
+
+			/* Get MIN of MAX AMPDU Length Exp */
+			if ((pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x3) > (*(pIE->data + i) & 0x3))
+				max_AMPDU_len = (*(pIE->data + i) & 0x3);
+			else
+				max_AMPDU_len = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x3);
+
+			/* Get MAX of MIN MPDU Start Spacing */
+			if ((pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c) > (*(pIE->data + i) & 0x1c))
+				min_MPDU_spacing = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c);
+			else
+				min_MPDU_spacing = (*(pIE->data + i) & 0x1c);
+
+			pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para = max_AMPDU_len | min_MPDU_spacing;
+		}
+	}
+
+	/*	Commented by Albert 2010/07/12 */
+	/*	Have to handle the endian issue after copying. */
+	/*	HT_ext_caps didn't be used yet.	 */
+	pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info = le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info);
+	pmlmeinfo->HT_caps.u.HT_cap_element.HT_ext_caps = le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_ext_caps);
+
+	/* update the MCS set */
+	for (i = 0; i < 16; i++)
+		pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate[i] &= pmlmeext->default_supported_mcs_set[i];
+
+	rx_nss = get_phy_rx_nss(padapter, padapter_link);
+
+	switch (rx_nss) {
+	case 1:
+		set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_1R);
+		break;
+	case 2:
+		#ifdef CONFIG_DISABLE_MCS13TO15
+		if (pmlmeext->chandef.bw == CHANNEL_WIDTH_40 && pregistrypriv->wifi_spec != 1)
+			set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_2R_13TO15_OFF);
+		else
+		#endif
+			set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_2R);
+		break;
+	case 3:
+		set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_3R);
+		break;
+	case 4:
+		set_mcs_rate_by_mask(pmlmeinfo->HT_caps.u.HT_cap_element.MCS_rate, MCS_RATE_4R);
+		break;
+	default:
+		RTW_WARN("rf_type:%d or tx_nss:%u is not expected\n", GET_HAL_RFPATH(adapter_to_dvobj(padapter)), rx_nss);
+	}
+
+	if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE)) {
+		/* Config STBC setting */
+		if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX) && GET_HT_CAP_ELE_RX_STBC(pIE->data)) {
+			SET_FLAG(cur_stbc_cap, STBC_HT_ENABLE_TX);
+			RTW_INFO("Enable HT Tx STBC !\n");
+		}
+		phtpriv->stbc_cap = cur_stbc_cap;
+
+#ifdef CONFIG_BEAMFORMING
+		/* Config Tx beamforming setting */
+		if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE) &&
+		    GET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP(pIE->data)) {
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
+			/* Shift to BEAMFORMING_HT_BEAMFORMEE_CHNL_EST_CAP*/
+			SET_FLAG(cur_beamform_cap, GET_HT_CAP_TXBF_CHNL_ESTIMATION_NUM_ANTENNAS(pIE->data) << 6);
+		}
+
+		if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE) &&
+		    GET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP(pIE->data)) {
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
+			/* Shift to BEAMFORMING_HT_BEAMFORMER_STEER_NUM*/
+			SET_FLAG(cur_beamform_cap, GET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(pIE->data) << 4);
+		}
+		phtpriv->beamform_cap = cur_beamform_cap;
+		if (cur_beamform_cap)
+			RTW_INFO("AP HT Beamforming Cap = 0x%02X\n", cur_beamform_cap);
+#endif /*CONFIG_BEAMFORMING*/
+	} else {
+		/*WIFI_STATION_STATEorI_ADHOC_STATE or WIFI_ADHOC_MASTER_STATE*/
+		sta = rtw_get_stainfo(&padapter->stapriv, pmlmeinfo->network.MacAddress);
+		if (!sta) {
+			RTW_ERR(FUNC_ADPT_FMT ": STA(" MAC_FMT ") not found!\n",
+				FUNC_ADPT_ARG(padapter), MAC_ARG(pmlmeinfo->network.MacAddress));
+			return;
+		}
+		if (!sta->phl_sta) {
+			RTW_ERR(FUNC_ADPT_FMT ": PHL STA(" MAC_FMT ") not exist!\n",
+				FUNC_ADPT_ARG(padapter), MAC_ARG(pmlmeinfo->network.MacAddress));
+			return;
+		}
+		phl_sta = sta->phl_sta;
+
+		/* Config LDPC Coding Capability */
+		if (TEST_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_TX) && GET_HT_CAP_ELE_LDPC_CAP(pIE->data)) {
+			SET_FLAG(cur_ldpc_cap, (LDPC_HT_ENABLE_TX | LDPC_HT_CAP_TX));
+			RTW_INFO("Enable HT Tx LDPC!\n");
+			phl_sta->asoc_cap.ht_ldpc = 1;
+		}
+		phtpriv->ldpc_cap = cur_ldpc_cap;
+
+		/* Config STBC setting */
+		if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX) && GET_HT_CAP_ELE_RX_STBC(pIE->data)) {
+			SET_FLAG(cur_stbc_cap, (STBC_HT_ENABLE_TX | STBC_HT_CAP_TX));
+			RTW_INFO("Enable HT Tx STBC!\n");
+			phl_sta->asoc_cap.stbc_ht_rx =
+				protocol_cap->stbc_ht_tx ? GET_HT_CAP_ELE_RX_STBC(pIE->data) : 0;
+		}
+		phtpriv->stbc_cap = cur_stbc_cap;
+		phl_sta->asoc_cap.stbc_ht_tx = GET_HT_CAP_ELE_TX_STBC(pIE->data);
+
+#ifdef CONFIG_BEAMFORMING
+		/* Config beamforming setting */
+		if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE) &&
+		    GET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP(pIE->data)) {
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
+			/* Shift to BEAMFORMING_HT_BEAMFORMEE_CHNL_EST_CAP*/
+			SET_FLAG(cur_beamform_cap, GET_HT_CAP_TXBF_CHNL_ESTIMATION_NUM_ANTENNAS(pIE->data) << 6);
+		}
+
+		if (TEST_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE) &&
+		    GET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP(pIE->data)) {
+			SET_FLAG(cur_beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
+			/* Shift to BEAMFORMING_HT_BEAMFORMER_STEER_NUM*/
+			SET_FLAG(cur_beamform_cap, GET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(pIE->data) << 4);
+		}
+		phtpriv->beamform_cap = cur_beamform_cap;
+		if (cur_beamform_cap)
+			RTW_INFO("Client HT Beamforming Cap = 0x%02X\n", cur_beamform_cap);
+#endif /*CONFIG_BEAMFORMING*/
+	}
+
+#endif /* CONFIG_80211N_HT */
+}
+
+void HT_info_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE)
+{
+#ifdef CONFIG_80211N_HT
+	struct link_mlme_priv		*pmlmepriv = &padapter_link->mlmepriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	struct ht_priv			*phtpriv = &pmlmepriv->htpriv;
+
+	if (pIE == NULL)
+		return;
+
+	if (phtpriv->ht_option == _FALSE)
+		return;
+
+
+	if (pIE->Length > sizeof(struct HT_info_element))
+		return;
+
+	pmlmeinfo->HT_info_enable = 1;
+	_rtw_memcpy(&(pmlmeinfo->HT_info), pIE->data, pIE->Length);
+#endif /* CONFIG_80211N_HT */
+	return;
+}
+
+void HTOnAssocRsp(_adapter *padapter)
+{
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	unsigned char		max_AMPDU_len;
+	unsigned char		min_MPDU_spacing;
+	/* struct registry_priv	 *pregpriv = &padapter->registrypriv; */
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	if ((pmlmeinfo->HT_info_enable) && (pmlmeinfo->HT_caps_enable))
+		pmlmeinfo->HT_enable = 1;
+	else {
+		pmlmeinfo->HT_enable = 0;
+		/* set_channel_bwmode(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode); */
+		return;
+	}
+
+	/* handle A-MPDU parameter field */
+	/*
+		AMPDU_para [1:0]:Max AMPDU Len => 0:8k , 1:16k, 2:32k, 3:64k
+		AMPDU_para [4:2]:Min MPDU Start Spacing
+	*/
+	max_AMPDU_len = pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x03;
+
+	min_MPDU_spacing = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para & 0x1c) >> 2;
+
+	rtw_hal_set_hwreg(padapter, HW_VAR_AMPDU_MIN_SPACE, (u8 *)(&min_MPDU_spacing));
+#ifdef CONFIG_80211N_HT
+	rtw_hal_set_hwreg(padapter, HW_VAR_AMPDU_FACTOR, (u8 *)(&max_AMPDU_len));
+#endif /* CONFIG_80211N_HT */
+#if 0 /* move to rtw_update_ht_cap() */
+	if ((pregpriv->bw_mode > 0) &&
+	    (pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info & BIT(1)) &&
+	    (pmlmeinfo->HT_info.infos[0] & BIT(2))) {
+		/* switch to the 40M Hz mode accoring to the AP */
+		pmlmeext->cur_bwmode = CHANNEL_WIDTH_40;
+		switch ((pmlmeinfo->HT_info.infos[0] & 0x3)) {
+		case IEEE80211_SCA:
+			pmlmeext->cur_ch_offset = CHAN_OFFSET_UPPER;
+			break;
+
+		case IEEE80211_SCB:
+			pmlmeext->cur_ch_offset = CHAN_OFFSET_LOWER;
+			break;
+
+		default:
+			pmlmeext->cur_ch_offset = CHAN_OFFSET_NO_EXT;
+			break;
+		}
+	}
+#endif
+
+	/* set_channel_bwmode(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode); */
+
+#if 0 /* move to rtw_update_ht_cap() */
+	/*  */
+	/* Config SM Power Save setting */
+	/*  */
+	pmlmeinfo->SM_PS = (pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info & 0x0C) >> 2;
+	if (pmlmeinfo->SM_PS == SM_PS_STATIC) {
+#if 0
+		u8 i;
+		/* update the MCS rates */
+		for (i = 0; i < 16; i++)
+			pmlmeinfo->HT_caps.HT_cap_element.MCS_rate[i] &= MCS_rate_1R[i];
+#endif
+		RTW_INFO("%s(): SM_PS_STATIC\n", __FUNCTION__);
+	}
+
+	/*  */
+	/* Config current HT Protection mode. */
+	/*  */
+	pmlmeinfo->HT_protection = pmlmeinfo->HT_info.infos[1] & 0x3;
+#endif
+
+}
+
+void ERP_IE_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		PNDIS_802_11_VARIABLE_IEs pIE)
+{
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (pIE->Length > 1)
+		return;
+
+	pmlmeinfo->ERP_enable = 1;
+	_rtw_memcpy(&(pmlmeinfo->ERP_IE), pIE->data, pIE->Length);
+}
+
+void VCS_update(_adapter *padapter, struct sta_info *psta)
+{
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	switch (pregpriv->vrtl_carrier_sense) { /* 0:off 1:on 2:auto */
+	case 0: /* off */
+		psta->rtsen = 0;
+		psta->cts2self = 0;
+		break;
+
+	case 1: /* on */
+		if (pregpriv->vcs_type == 1) { /* 1:RTS/CTS 2:CTS to self */
+			psta->rtsen = 1;
+			psta->cts2self = 0;
+		} else {
+			psta->rtsen = 0;
+			psta->cts2self = 1;
+		}
+		break;
+
+	case 2: /* auto */
+	default:
+		if (((pmlmeinfo->ERP_enable) && (pmlmeinfo->ERP_IE & BIT(1)))
+			/*||(pmlmepriv->ht_op_mode & HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT)*/
+		) {
+			if (pregpriv->vcs_type == 1) {
+				psta->rtsen = 1;
+				psta->cts2self = 0;
+			} else {
+				psta->rtsen = 0;
+				psta->cts2self = 1;
+			}
+		} else {
+			psta->rtsen = 0;
+			psta->cts2self = 0;
+		}
+		break;
+	}
+
+	switch (pregpriv->hw_rts_en) { /* 0:disable 1:enable */
+	case 0:
+		psta->hw_rts_en = 0;
+		break;
+	case 1:
+		psta->hw_rts_en = 1;
+		break;
+	default:
+		RTW_WARN("%s: unexpected value(%d) for hw_rts_en.\n", __func__, pregpriv->hw_rts_en);
+		break;
+	}
+
+}
+
+void	update_ldpc_stbc_cap(struct sta_info *psta)
+{
+#ifdef CONFIG_80211N_HT
+
+#ifdef CONFIG_80211AC_VHT
+#ifdef CONFIG_80211AX_HE
+	/* CONFIG_80211AX_HE_TODO */
+#endif /* CONFIG_80211AX_HE */
+	if (psta->vhtpriv.vht_option) {
+		if (TEST_FLAG(psta->vhtpriv.ldpc_cap, LDPC_VHT_ENABLE_TX))
+			psta->phl_sta->asoc_cap.vht_ldpc = 1;
+		else
+			psta->phl_sta->asoc_cap.vht_ldpc = 0;
+	} else
+#endif /* CONFIG_80211AC_VHT */
+		if (psta->htpriv.ht_option) {
+			if (TEST_FLAG(psta->htpriv.ldpc_cap, LDPC_HT_ENABLE_TX))
+				psta->phl_sta->asoc_cap.ht_ldpc = 1;
+			else
+				psta->phl_sta->asoc_cap.ht_ldpc = 0;
+		} else {
+			psta->phl_sta->asoc_cap.vht_ldpc = 0;
+			psta->phl_sta->asoc_cap.ht_ldpc = 0;
+		}
+
+#endif /* CONFIG_80211N_HT */
+}
+
+int check_ielen(u8 *start, uint len)
+{
+	int left = len;
+	u8 *pos = start;
+	u8 id, elen;
+
+	while (left >= 2) {
+		id = *pos++;
+		elen = *pos++;
+		left -= 2;
+
+		if (elen > left) {
+			RTW_ERR("IEEE 802.11 element parse failed (id=%d elen=%d left=%lu)\n",
+					id, elen, (unsigned long) left);
+			return _FALSE;
+		}
+		if ((id == WLAN_EID_VENDOR_SPECIFIC) && (elen < 3))
+				return _FALSE;
+
+		left -= elen;
+		pos += elen;
+	}
+	if (left)
+		return _FALSE;
+
+	return _TRUE;
+}
+
+int validate_beacon_len(u8 *pframe, u32 len)
+{
+	u8 ie_offset = _BEACON_IE_OFFSET_ + sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	if (len < ie_offset) {
+		RTW_ERR("%s: incorrect beacon length(%d)\n", __func__, len);
+		rtw_warn_on(1);
+		return _FALSE;
+	}
+
+	if (check_ielen(pframe + ie_offset, len - ie_offset) == _FALSE)
+		return _FALSE;
+
+	return _TRUE;
+}
+
+
+u8 support_rate_ranges[] = {
+	IEEE80211_CCK_RATE_1MB,
+	IEEE80211_CCK_RATE_2MB,
+	IEEE80211_CCK_RATE_5MB,
+	IEEE80211_CCK_RATE_11MB,
+	IEEE80211_OFDM_RATE_6MB,
+	IEEE80211_OFDM_RATE_9MB,
+	IEEE80211_OFDM_RATE_12MB,
+	IEEE80211_OFDM_RATE_18MB,
+	IEEE80211_PBCC_RATE_22MB,
+	IEEE80211_FREAK_RATE_22_5MB,
+	IEEE80211_OFDM_RATE_24MB,
+	IEEE80211_OFDM_RATE_36MB,
+	IEEE80211_OFDM_RATE_48MB,
+	IEEE80211_OFDM_RATE_54MB,
+};
+
+inline bool match_ranges(u16 EID, u32 value)
+{
+	int i;
+	int nr_range;
+
+	switch (EID) {
+	case _EXT_SUPPORTEDRATES_IE_:
+	case _SUPPORTEDRATES_IE_:
+		nr_range = sizeof(support_rate_ranges)/sizeof(u8);
+		for (i = 0; i < nr_range; i++) {
+			/*	clear bit7 before searching.	*/
+			value &= ~BIT(7);
+			if (value == support_rate_ranges[i])
+				return _TRUE;
+		}
+		break;
+	default:
+		break;
+	};
+	return _FALSE;
+}
+
+/*
+ * rtw_validate_value: validate the IE contain.
+ *
+ *	Input :
+ *		EID : Element ID
+ *		p	: IE buffer (without EID & length)
+ *		len	: IE length
+ *	return:
+ * 		_TRUE	: All Values are validated.
+ *		_FALSE	: At least one value is NOT validated.
+ */
+bool rtw_validate_value(u16 EID, u8 *p, u16 len)
+{
+	u8 rate;
+	u32 i, nr_val;
+
+	switch (EID) {
+	case _EXT_SUPPORTEDRATES_IE_:
+	case _SUPPORTEDRATES_IE_:
+		nr_val = len;
+		for (i=0; i<nr_val; i++) {
+			rate = *(p+i);
+			/*
+			 * Skip HT, VHT, HE and SAE H2E only BSS membership selectors
+			 * since they're not rates.
+			 */
+			if (rate == (0x80 | BSS_MEMBERSHIP_SELECTOR_HT_PHY) ||
+			    rate == (0x80 | BSS_MEMBERSHIP_SELECTOR_VHT_PHY) ||
+			    rate == (0x80 | BSS_MEMBERSHIP_SELECTOR_HE_PHY) ||
+			    rate == (0x80 | BSS_MEMBERSHIP_SELECTOR_SAE_H2E))
+				continue;
+			if (match_ranges(EID, rate) == _FALSE)
+				return _FALSE;
+		}
+		break;
+	default:
+		break;
+	};
+	return _TRUE;
+}
+
+bool is_hidden_ssid(char *ssid, int len)
+{
+	return len == 0 || is_all_null(ssid, len) == _TRUE;
+}
+
+inline bool hidden_ssid_ap(WLAN_BSSID_EX *snetwork)
+{
+	return ((snetwork->Ssid.SsidLength == 0) ||
+		is_all_null(snetwork->Ssid.Ssid, snetwork->Ssid.SsidLength) == _TRUE);
+}
+
+/*
+	Get SSID if this ilegal frame(probe resp) comes from a hidden SSID AP.
+	Update the SSID to the corresponding pnetwork in scan queue.
+*/
+void rtw_absorb_ssid_ifneed(_adapter *padapter, WLAN_BSSID_EX *bssid, u8 *pframe)
+{
+	struct wlan_network *scanned = NULL;
+	WLAN_BSSID_EX	*snetwork;
+	u8 ie_offset, *p=NULL, *next_ie=NULL;
+	u8 *mac;
+	sint ssid_len_ori;
+	u32 remain_len = 0;
+	u8 backupIE[MAX_IE_SZ];
+	u16 subtype;
+
+	mac = get_addr2_ptr(pframe);
+	subtype = get_frame_sub_type(pframe);
+
+	if (subtype == WIFI_BEACON) {
+		bssid->Reserved[0] = BSS_TYPE_BCN;
+		ie_offset = _BEACON_IE_OFFSET_;
+	} else {
+		/* FIXME : more type */
+		if (subtype == WIFI_PROBERSP) {
+			ie_offset = _PROBERSP_IE_OFFSET_;
+			bssid->Reserved[0] = BSS_TYPE_PROB_RSP;
+		} else if (subtype == WIFI_PROBEREQ) {
+			ie_offset = _PROBEREQ_IE_OFFSET_;
+			bssid->Reserved[0] = BSS_TYPE_PROB_REQ;
+		} else {
+			bssid->Reserved[0] = BSS_TYPE_UNDEF;
+			ie_offset = _FIXED_IE_LENGTH_;
+		}
+	}
+
+	_rtw_spinlock_bh(&padapter->mlmepriv.scanned_queue.lock);
+	scanned = _rtw_find_network(&padapter->mlmepriv.scanned_queue, mac);
+	if (!scanned) {
+		_rtw_spinunlock_bh(&padapter->mlmepriv.scanned_queue.lock);
+		return;
+	}
+
+	snetwork = &(scanned->network);
+	/* scan queue records as Hidden SSID && Input frame is NOT Hidden SSID	*/
+	if (hidden_ssid_ap(snetwork) && !hidden_ssid_ap(bssid)) {
+		p = rtw_get_ie(snetwork->IEs+ie_offset, _SSID_IE_, &ssid_len_ori, snetwork->IELength-ie_offset);
+		if (!p) {
+			_rtw_spinunlock_bh(&padapter->mlmepriv.scanned_queue.lock);
+			return;
+		}
+		next_ie = p + 2 + ssid_len_ori;
+		remain_len = snetwork->IELength - (next_ie - snetwork->IEs);
+		scanned->network.Ssid.SsidLength = bssid->Ssid.SsidLength;
+		_rtw_memcpy(scanned->network.Ssid.Ssid, bssid->Ssid.Ssid, bssid->Ssid.SsidLength);
+
+		//update pnetwork->ssid, pnetwork->ssidlen
+		_rtw_memcpy(backupIE, next_ie, remain_len);
+		*(p+1) = bssid->Ssid.SsidLength;
+		_rtw_memcpy(p+2, bssid->Ssid.Ssid, bssid->Ssid.SsidLength);
+		_rtw_memcpy(p+2+bssid->Ssid.SsidLength, backupIE, remain_len);
+		snetwork->IELength += bssid->Ssid.SsidLength;
+	}
+	_rtw_spinunlock_bh(&padapter->mlmepriv.scanned_queue.lock);
+}
+
+#ifdef DBG_RX_BCN
+void rtw_debug_rx_bcn(_adapter *adapter, u8 *pframe, u32 packet_len)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_ext_priv *pmlmeext = &adapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *mlmeinfo = &(pmlmeext->mlmext_info);
+	u16 sn = ((struct rtw_ieee80211_hdr_3addr *)pframe)->seq_ctl >> 4;
+	u64 tsf, tsf_offset;
+	u8 dtim_cnt, dtim_period, tim_bmap, tim_pvbit;
+
+	update_TSF(pmlmeext, pframe, packet_len);
+	tsf = pmlmeext->TSFValue;
+	tsf_offset = rtw_modular64(pmlmeext->TSFValue, (mlmeinfo->bcn_interval * 1024));
+
+	/*get TIM IE*/
+	/*DTIM Count*/
+	dtim_cnt = pmlmeext->tim[0];
+	/*DTIM Period*/
+	dtim_period = pmlmeext->tim[1];
+	/*Bitmap*/
+	tim_bmap = pmlmeext->tim[2];
+	/*Partial VBitmap AID 0 ~ 7*/
+	tim_pvbit = pmlmeext->tim[3];
+
+	RTW_INFO("[BCN] SN-%d, TSF-%lld(us), offset-%lld, bcn_interval-%d DTIM-%d[%d] bitmap-0x%02x-0x%02x\n",
+		sn, tsf, tsf_offset, mlmeinfo->bcn_interval, dtim_period, dtim_cnt, tim_bmap, tim_pvbit);
+}
+#endif
+
+/*
+ * rtw_get_bcn_keys: get beacon keys from recv frame
+ *
+ * TODO:
+ *	WLAN_EID_COUNTRY
+ *	WLAN_EID_ERP_INFO
+ *	WLAN_EID_CHANNEL_SWITCH
+ *	WLAN_EID_PWR_CONSTRAINT
+ */
+static int _rtw_get_bcn_keys(u8 *cap_info, u32 buf_len, u8 def_ch
+	, struct _ADAPTER_LINK *adapter_link, u8 mbssid_idx
+	, struct beacon_keys *recv_beacon)
+{
+	int left;
+	u16 capability;
+	unsigned char *pos;
+	struct rtw_ieee802_11_elems elems;
+
+	_rtw_memset(recv_beacon, 0, sizeof(*recv_beacon));
+
+	/* checking capabilities */
+	capability = le16_to_cpu(*(unsigned short *)(cap_info));
+
+	/* checking IEs */
+	left = buf_len - 2;
+	pos = cap_info + 2;
+	if (rtw_ieee802_11_parse_elems(pos, left, &elems, 1) == ParseFailed)
+		return _FALSE;
+
+	if (elems.ht_capabilities) {
+		if (elems.ht_capabilities_len != 26)
+			return _FALSE;
+	}
+
+	if (elems.ht_operation) {
+		if (elems.ht_operation_len != 22)
+			return _FALSE;
+	}
+
+	if (elems.vht_capabilities) {
+		if (elems.vht_capabilities_len != 12)
+			return _FALSE;
+	}
+
+	if (elems.vht_operation) {
+		if (elems.vht_operation_len != 5)
+			return _FALSE;
+	}
+
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+	if (elems.mbssid && mbssid_idx) {
+		if (rtw_ieee802_11_override_elems_by_mbssid(elems.mbssid - 2, elems.mbssid_len + 2, mbssid_idx, &elems, 1) == ParseFailed)
+			return _FALSE;
+		if (elems.non_tx_bssid_cap) {
+			if (elems.non_tx_bssid_cap_len != 2)
+				return _FALSE;
+			capability = le16_to_cpu(*(unsigned short *)(elems.non_tx_bssid_cap));
+		}
+	}
+#endif
+
+	if (rtw_elems_get_supported_rate(&elems, recv_beacon->rate_set, &recv_beacon->rate_num) == _FAIL)
+		return _FALSE;
+
+	if (recv_beacon->rate_num) {
+		if (cckratesonly_included(recv_beacon->rate_set, recv_beacon->rate_num) == _TRUE)
+			recv_beacon->proto_cap |= PROTO_CAP_11B;
+		else if (cckrates_included(recv_beacon->rate_set, recv_beacon->rate_num) == _TRUE)
+			recv_beacon->proto_cap |= PROTO_CAP_11B | PROTO_CAP_11G;
+		else
+			recv_beacon->proto_cap |= PROTO_CAP_11G;
+	}
+
+	if (elems.ht_capabilities && elems.ht_operation)
+		recv_beacon->proto_cap |= PROTO_CAP_11N;
+
+	if (elems.vht_capabilities && elems.vht_operation)
+		recv_beacon->proto_cap |= PROTO_CAP_11AC;
+
+	if (elems.he_capabilities && elems.he_operation)
+		recv_beacon->proto_cap |= PROTO_CAP_11AX;
+
+	/* check bw and channel offset */
+	rtw_ies_get_bchbw(pos, left, &recv_beacon->band, &recv_beacon->ch, &recv_beacon->bw, &recv_beacon->offset, NULL, NULL, 1, 1, 1);
+	if (!recv_beacon->ch)
+		recv_beacon->ch = def_ch;
+
+	/* checking SSID */
+	if (elems.ssid) {
+		if (elems.ssid_len > sizeof(recv_beacon->ssid))
+			return _FALSE;
+
+		_rtw_memcpy(recv_beacon->ssid, elems.ssid, elems.ssid_len);
+		recv_beacon->ssid_len = elems.ssid_len;
+	}
+
+	/* checking RSN first */
+	if (elems.rsn_ie && elems.rsn_ie_len) {
+		recv_beacon->encryp_protocol = ENCRYP_PROTOCOL_RSN;
+		rtw_parse_wpa2_ie(elems.rsn_ie - 2, elems.rsn_ie_len + 2,
+			&recv_beacon->group_cipher, &recv_beacon->pairwise_cipher,
+			NULL, &recv_beacon->akm, NULL, NULL);
+	}
+	/* checking WPA secon */
+	else if (elems.wpa_ie && elems.wpa_ie_len) {
+		recv_beacon->encryp_protocol = ENCRYP_PROTOCOL_WPA;
+		rtw_parse_wpa_ie(elems.wpa_ie - 2, elems.wpa_ie_len + 2,
+			&recv_beacon->group_cipher, &recv_beacon->pairwise_cipher,
+				 &recv_beacon->akm);
+	} else if (capability & BIT(4))
+		recv_beacon->encryp_protocol = ENCRYP_PROTOCOL_WEP;
+
+	if (adapter_link) {
+		struct link_mlme_ext_priv *pmlmeext = &adapter_link->mlmeextpriv;
+
+		if (elems.tim && elems.tim_len) {
+			#ifdef DBG_RX_BCN
+			_rtw_memcpy(pmlmeext->tim, elems.tim, 4);
+			#endif
+			pmlmeext->dtim = elems.tim[1];
+		}
+
+		/* checking RTW TBTX */
+		#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+		if (elems.tbtx_cap && elems.tbtx_cap_len) {
+			struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+			if (rtw_is_tbtx_capabilty(elems.tbtx_cap, elems.tbtx_cap_len))
+				RTW_DBG("AP support TBTX\n");
+		}
+		#endif
+	}
+
+	return _TRUE;
+}
+
+int rtw_get_bcn_keys(_adapter *adapter, struct _ADAPTER_LINK *adapter_link
+	, u8 *whdr, u32 flen, struct beacon_keys *bcn_keys)
+{
+	return _rtw_get_bcn_keys(
+		whdr + WLAN_HDR_A3_LEN + 10
+		, flen - WLAN_HDR_A3_LEN - 10
+		, adapter_link->mlmeextpriv.chandef.chan, adapter_link
+		, BSS_EX_MBSSID_IDX(&adapter_link->mlmeextpriv.mlmext_info.network)
+		, bcn_keys);
+}
+
+int rtw_get_bcn_keys_from_bss(WLAN_BSSID_EX *bss, struct beacon_keys *bcn_keys)
+{
+	return _rtw_get_bcn_keys(
+		bss->IEs + 10
+		, bss->IELength - 10
+		, bss->Configuration.DSConfig, NULL
+		, 0
+		, bcn_keys);
+}
+
+int rtw_update_bcn_keys_of_network(struct wlan_network *network)
+{
+	network->bcn_keys_valid = rtw_get_bcn_keys_from_bss(&network->network, &network->bcn_keys);
+	return network->bcn_keys_valid;
+}
+
+#define CIPHER_STR(c, type)	c & WPA_CIPHER_##type ? "["#type"]" : ""
+#define CIPHER_FMT		"%s%s%s%s%s%s%s%s%s%s%s%s"
+#define CIPHER_ARG(c)		CIPHER_STR(c, NONE), \
+				CIPHER_STR(c, WEP40), \
+				CIPHER_STR(c, WEP104), \
+				CIPHER_STR(c, TKIP), \
+				CIPHER_STR(c, CCMP), \
+				CIPHER_STR(c, GCMP), \
+				CIPHER_STR(c, GCMP_256), \
+				CIPHER_STR(c, CCMP_256), \
+				CIPHER_STR(c, BIP_CMAC_128), \
+				CIPHER_STR(c, BIP_GMAC_128), \
+				CIPHER_STR(c, BIP_GMAC_256), \
+				CIPHER_STR(c, BIP_CMAC_256)
+#define AKM_STR(akm, type)	akm & WLAN_AKM_TYPE_##type ? "["#type"]" : ""
+
+void rtw_dump_bcn_keys(void *sel, struct beacon_keys *recv_beacon)
+{
+	u8 ssid[IW_ESSID_MAX_SIZE + 1];
+
+	_rtw_memcpy(ssid, recv_beacon->ssid, recv_beacon->ssid_len);
+	ssid[recv_beacon->ssid_len] = '\0';
+
+	RTW_PRINT_SEL(sel, "ssid = %s (len = %u)\n", ssid, recv_beacon->ssid_len);
+	RTW_PRINT_SEL(sel, "ch = %u,%u,%u,%u\n"
+		, recv_beacon->band, recv_beacon->ch, recv_beacon->bw, recv_beacon->offset);
+	RTW_PRINT_SEL(sel, "proto_cap = 0x%02x\n", recv_beacon->proto_cap);
+	RTW_MAP_DUMP_SEL(sel, "rate_set = "
+		, recv_beacon->rate_set, recv_beacon->rate_num);
+	RTW_PRINT_SEL(sel, "sec = %d\n", recv_beacon->encryp_protocol);
+	RTW_PRINT_SEL(sel, "GTK = 0x%x " CIPHER_FMT "\n"
+		, recv_beacon->group_cipher
+		, CIPHER_ARG(recv_beacon->group_cipher));
+	RTW_PRINT_SEL(sel, "PTK = 0x%x " CIPHER_FMT "\n"
+		, recv_beacon->pairwise_cipher
+		, CIPHER_ARG(recv_beacon->pairwise_cipher));
+	RTW_PRINT_SEL(sel, "AKM = 0x%08x %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n"
+		, recv_beacon->akm
+		, AKM_STR(recv_beacon->akm, 8021X)
+		, AKM_STR(recv_beacon->akm, PSK)
+		, AKM_STR(recv_beacon->akm, FT_8021X)
+		, AKM_STR(recv_beacon->akm, FT_PSK)
+		, AKM_STR(recv_beacon->akm, 8021X_SHA256)
+		, AKM_STR(recv_beacon->akm, PSK_SHA256)
+		, AKM_STR(recv_beacon->akm, TDLS)
+		, AKM_STR(recv_beacon->akm, SAE)
+		, AKM_STR(recv_beacon->akm, FT_OVER_SAE)
+		, AKM_STR(recv_beacon->akm, 8021X_SUITE_B)
+		, AKM_STR(recv_beacon->akm, 8021X_SUITE_B_192)
+		, AKM_STR(recv_beacon->akm, FILS_SHA256)
+		, AKM_STR(recv_beacon->akm, FILS_SHA384)
+		, AKM_STR(recv_beacon->akm, FT_FILS_SHA256)
+		, AKM_STR(recv_beacon->akm, FT_FILS_SHA384));
+}
+
+bool rtw_bcn_key_compare(struct beacon_keys *cur, struct beacon_keys *recv)
+{
+#define BCNKEY_VERIFY_PROTO_CAP 0
+#define BCNKEY_VERIFY_WHOLE_RATE_SET 0
+
+	struct beacon_keys tmp;
+	bool ret = _FALSE;
+
+	if (!rtw_is_bchbw_grouped(cur->band, cur->ch, cur->bw, cur->offset
+			, recv->band, recv->ch, recv->bw, recv->offset))
+		goto exit;
+
+	_rtw_memcpy(&tmp, cur, sizeof(tmp));
+
+	/* check fields excluding below */
+	tmp.band = recv->band;
+	tmp.ch = recv->ch;
+	tmp.bw = recv->bw;
+	tmp.offset = recv->offset;
+#ifdef PRIVATE_R
+	/* SSID don't check */
+	tmp.ssid_len = recv->ssid_len;
+	_rtw_memcpy(tmp.ssid, recv->ssid, IW_ESSID_MAX_SIZE);
+#endif
+	if (!BCNKEY_VERIFY_PROTO_CAP)
+		tmp.proto_cap = recv->proto_cap;
+	if (!BCNKEY_VERIFY_WHOLE_RATE_SET) {
+		tmp.rate_num = recv->rate_num;
+		_rtw_memcpy(tmp.rate_set, recv->rate_set, 12);
+	}
+
+	if (_rtw_memcmp(&tmp, recv, sizeof(*recv)) == _FALSE)
+		goto exit;
+
+	ret = _TRUE;
+
+exit:
+	return ret;
+}
+
+int rtw_check_bcn_info(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+		u8 *pframe, u32 packet_len)
+{
+	u8 *pbssid = GetAddr3Ptr(pframe);
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct link_mlme_priv *lmlmepriv = 	&adapter_link->mlmepriv;
+	struct beacon_keys *cur_beacon = &adapter_link->mlmepriv.cur_beacon_keys;
+	struct beacon_keys recv_beacon;
+	int ret = 0;
+	u8 ifbmp_m = rtw_mi_get_ap_mesh_ifbmp(adapter);
+	u8 ifbmp_s = rtw_mi_get_ld_sta_ifbmp(adapter);
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	u8 ssid_is_hidden = _FALSE;
+	struct link_mlme_ext_priv *plmlmeext = &(adapter_link->mlmeextpriv);
+	struct wlan_network *cur_network = &(adapter_link->mlmepriv.cur_network);
+
+	if (is_client_associated_to_ap(adapter) == _FALSE)
+		goto exit_success;
+
+	if (rtw_get_bcn_keys(adapter, adapter_link, pframe, packet_len, &recv_beacon) == _FALSE)
+		goto exit_success; /* parsing failed => broken IE */
+
+#ifdef DBG_RX_BCN
+	rtw_debug_rx_bcn(adapter, pframe, packet_len);
+#endif
+	ssid_is_hidden = is_hidden_ssid(recv_beacon.ssid, recv_beacon.ssid_len);
+
+	if (recv_beacon.ssid_len != cur_beacon->ssid_len) {
+		pmlmeinfo->illegal_beacon_code |= SSID_LENGTH_CHANGED;
+		pmlmeinfo->illegal_beacon_code |= SSID_CHANGED;
+		if (!ssid_is_hidden) {
+			RTW_WARN("%s: Ignore ssid len change new %d old %d\n",
+				 __func__, recv_beacon.ssid_len,
+				 cur_beacon->ssid_len);
+		}
+	} else if ((_rtw_memcmp(recv_beacon.ssid,
+				cur_beacon->ssid,
+				cur_beacon->ssid_len) == _FALSE)) {
+		pmlmeinfo->illegal_beacon_code |= SSID_CHANGED;
+		if (!ssid_is_hidden) {
+			RTW_INFO_DUMP("[old ssid]: ",
+				      cur_beacon->ssid, cur_beacon->ssid_len);
+			RTW_INFO_DUMP("[new ssid]: ",
+				      recv_beacon.ssid, cur_beacon->ssid_len);
+		}
+	}
+
+	/* hidden ssid, replace with current beacon ssid directly */
+	if (ssid_is_hidden) {
+		_rtw_memcpy(recv_beacon.ssid, cur_beacon->ssid, cur_beacon->ssid_len);
+		recv_beacon.ssid_len = cur_beacon->ssid_len;
+	}
+
+#ifdef CONFIG_ECSA_PHL
+	if (check_fwstate(pmlmepriv, WIFI_CSA_UPDATE_BEACON)) {
+		u8 c_ch, c_offset, c_bw;
+		u8 bcn_ch, bcn_bw, bcn_offset;
+		struct sta_info *psta = NULL;
+		struct rtw_chan_def ori_u_chdef = {0};
+		struct rtw_chan_def new_u_chdef = {0};
+		bool bw_offset_changed = false;
+
+		c_ch = plmlmeext->chandef.chan;
+		c_bw = plmlmeext->chandef.bw;
+		c_offset = plmlmeext->chandef.offset;
+
+		#ifdef DBG_CSA
+		RTW_INFO("CSA : Wait AP for updating its beacon, wait %u beacon, current.ch = %u, recv_beacon.ch = %u\n",
+			pmlmepriv->bcn_cnts_after_csa + 1, c_ch, recv_beacon.ch);
+		#endif
+
+		/*
+		* Case 1 : channel is different
+		* 	AP doesn't udpate its beacon.
+		* Case 2 : channel is same
+		*	AP may not update its beacon yet, so we wait 5 beacon.
+		* Case 3 : we already wait 5 beacon
+		*	We assume that AP update its beacon.
+		*/
+		if (c_ch != recv_beacon.ch) {
+			goto exit_success;
+		} else if (pmlmepriv->bcn_cnts_after_csa < WAIT_BCN_TIMES) {
+			pmlmepriv->bcn_cnts_after_csa += 1;
+			goto exit_success;
+		} else {
+			pmlmepriv->bcn_cnts_after_csa = 0;
+		}
+
+		_rtw_memcpy(cur_beacon, &recv_beacon, sizeof(recv_beacon));
+		clr_fwstate(pmlmepriv, WIFI_CSA_UPDATE_BEACON);
+		/* the radar detect flow may need this */
+		clr_fwstate(pmlmepriv, WIFI_CSA_SKIP_CHECK_BEACON);
+		_cancel_timer_nowait(&adapter->mlmeextpriv.csa_wait_bcn_timer);
+
+		rtw_ecsa_update_sta_mlme(adapter_link, pframe, packet_len);
+
+		bcn_ch = recv_beacon.ch;
+		bcn_bw = recv_beacon.bw;
+		bcn_offset = recv_beacon.offset;
+
+		#ifdef DBG_CSA
+		RTW_INFO("CSA : copy new beacon, recv_beacon ch/bw/offset = %u,%u,%u\n",
+			bcn_ch, bcn_bw, bcn_offset);
+		RTW_INFO("CSA : before update STA mode bw/offset, current bw/offset = %u,%u\n",
+			c_bw, c_offset);
+		#endif
+
+		/* STA role bw/offset is different from associated AP */
+		if (c_bw != bcn_bw || c_offset != bcn_offset) {
+			RTW_INFO("CSA : STA needs to sync bw/offset with AP\n");
+			bw_offset_changed = true;
+			plmlmeext->chandef.bw = bcn_bw;
+			plmlmeext->chandef.offset = bcn_offset;
+		}
+
+		if (rtw_adjust_bchbw(adapter, plmlmeext->chandef.band,
+		    plmlmeext->chandef.chan, (u8*)&(plmlmeext->chandef.bw),
+		    (u8*)&(plmlmeext->chandef.offset))) {
+			RTW_INFO("csa : limit bandwith by SW capability\n");
+			bw_offset_changed = true;
+		}
+
+		if (bw_offset_changed) {
+			rtw_phl_mr_get_chandef(d->phl, adapter->phl_role, adapter_link->wrlink, &ori_u_chdef);
+
+			/* update wifi role chandef */
+			rtw_hw_update_chan_def(adapter, adapter_link);
+
+			/* update chanctx */
+			if (rtw_phl_mr_upt_chandef(d->phl, adapter_link->wrlink) == RTW_PHL_STATUS_FAILURE)
+				rtw_warn_on(1);
+
+			rtw_phl_mr_get_chandef(d->phl, adapter->phl_role, adapter_link->wrlink, &new_u_chdef);
+
+			#if (CONFIG_DFS && CONFIG_IEEE80211_BAND_5GHZ)
+			rtw_dfs_rd_en_dec_on_mlme_act(adapter, adapter_link, MLME_OPCH_SWITCH, ifbmp_s);
+			#endif
+
+			if (ori_u_chdef.bw != new_u_chdef.bw || ori_u_chdef.offset != new_u_chdef.offset)
+				rtw_set_chbw_cmd(adapter, adapter_link, &new_u_chdef, 0, RFK_TYPE_FORCE_DO);
+
+			RTW_INFO("CSA : after update bw/offset, STA mode new bw/offset = %u,%u\n", \
+				(u8)new_u_chdef.bw, (u8)new_u_chdef.offset);
+		} else {
+			RTW_INFO("CSA : STA mode bw/offset is same as AP\n");
+		}
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+		{
+			u8 ht_option = adapter_link->mlmepriv.htpriv.ht_option;
+
+			rtw_cfg80211_ch_switch_notify(adapter, GET_PRIMARY_LINK(adapter), &plmlmeext->chandef, ht_option, 0);
+		}
+		#endif
+
+		/* update RA mask */
+		psta = rtw_get_stainfo(&adapter->stapriv, get_link_bssid(&adapter_link->mlmepriv));
+		if (psta) {
+			psta->phl_sta->chandef.bw = plmlmeext->chandef.bw;
+			rtw_sta_hal_ra_mask_update_cmd(adapter, psta, 0);
+		}
+
+		RTW_INFO("CSA : update beacon done, WIFI_CSA_UPDATE_BEACON is clear\n");
+	}
+#endif /* CONFIG_ECSA_PHL */
+
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+	if (_rtw_memcmp(&recv_beacon, cur_beacon, sizeof(recv_beacon)) == _TRUE)
+		lmlmepriv->new_beacon_cnts = 0;
+	else if ((lmlmepriv->new_beacon_cnts == 0) ||
+		_rtw_memcmp(&recv_beacon, &lmlmepriv->new_beacon_keys, sizeof(recv_beacon)) == _FALSE) {
+		RTW_DBG("%s: start new beacon (seq=%d)\n", __func__, GetSequence(pframe));
+
+		if (lmlmepriv->new_beacon_cnts == 0) {
+			RTW_ERR("%s: cur beacon key\n", __func__);
+			RTW_DBG_EXPR(rtw_dump_bcn_keys(RTW_DBGDUMP, cur_beacon));
+		}
+
+		RTW_DBG("%s: new beacon key\n", __func__);
+		RTW_DBG_EXPR(rtw_dump_bcn_keys(RTW_DBGDUMP, &recv_beacon));
+
+		_rtw_memcpy(&lmlmepriv->new_beacon_keys, &recv_beacon, sizeof(recv_beacon));
+		lmlmepriv->new_beacon_cnts = 1;
+	} else {
+		RTW_DBG("%s: new beacon again (seq=%d)\n", __func__, GetSequence(pframe));
+		lmlmepriv->new_beacon_cnts++;
+	}
+
+	/* if counter >= max, it means beacon is changed really */
+	if ((new_bcn_max != 0) && (lmlmepriv->new_beacon_cnts >= new_bcn_max))
+#else
+	if (_rtw_memcmp(&recv_beacon, cur_beacon, sizeof(recv_beacon)) == _FALSE)
+#endif
+	{
+		if (check_fwstate(pmlmepriv, WIFI_CSA_SKIP_CHECK_BEACON)) {
+			RTW_INFO(FUNC_ADPT_FMT" CSA : skip new beacon key before switching channel\n",
+				 FUNC_ADPT_ARG(adapter));
+			goto exit_success;
+		}
+
+		RTW_INFO(FUNC_ADPT_FMT" new beacon occur!!\n", FUNC_ADPT_ARG(adapter));
+		RTW_INFO(FUNC_ADPT_FMT" cur beacon key:\n", FUNC_ADPT_ARG(adapter));
+		rtw_dump_bcn_keys(RTW_DBGDUMP, cur_beacon);
+		RTW_INFO(FUNC_ADPT_FMT" new beacon key:\n", FUNC_ADPT_ARG(adapter));
+		rtw_dump_bcn_keys(RTW_DBGDUMP, &recv_beacon);
+
+		if (recv_beacon.ch != cur_beacon->ch)
+			pmlmeinfo->illegal_beacon_code |= BEACON_CHANNEL_CHANGED;
+		if (recv_beacon.encryp_protocol != cur_beacon->encryp_protocol)
+			pmlmeinfo->illegal_beacon_code |= ENCRYPT_PROTOCOL_CHANGED;
+		if (recv_beacon.pairwise_cipher != cur_beacon->pairwise_cipher)
+			pmlmeinfo->illegal_beacon_code |= PAIRWISE_CIPHER_CHANGED;
+		if (recv_beacon.group_cipher != cur_beacon->group_cipher)
+			pmlmeinfo->illegal_beacon_code |= GROUP_CIPHER_CHANGED;
+		if (recv_beacon.akm != cur_beacon->akm)
+			pmlmeinfo->illegal_beacon_code |= IS_8021X_CHANGED;
+#ifdef PRIVATE_R
+		if ((pmlmeinfo->illegal_beacon_code & ~(SSID_CHANGED|SSID_LENGTH_CHANGED)) != 0) {
+			struct dis_bcn_key *dis_bcn_key_info;
+			dis_bcn_key_info = rtw_alloc_dis_bcn_key(&(adapter_link->mlmepriv.idle_dis_bcn_queue), &(adapter_link->mlmepriv.busy_dis_bcn_queue));
+			if (dis_bcn_key_info) {
+				_rtw_memcpy(&(dis_bcn_key_info->bcn_content), &recv_beacon, sizeof(recv_beacon));
+				rtw_enqueue_dis_bcn_key(dis_bcn_key_info, &(adapter_link->mlmepriv.busy_dis_bcn_queue));
+			} else {
+				RTW_INFO("alloc dis bcn info fail\n");
+			}
+			goto exit;
+		}
+#endif
+
+		if (rtw_bcn_key_compare(cur_beacon, &recv_beacon) == _FALSE)
+			goto exit;
+
+		_rtw_memcpy(cur_beacon, &recv_beacon, sizeof(recv_beacon));
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+		lmlmepriv->new_beacon_cnts = 0;
+#endif
+	}
+
+exit_success:
+	ret = 1;
+
+exit:
+	return ret;
+}
+
+void update_beacon_info(_adapter *padapter, u8 *pframe, uint pkt_len, struct sta_info *psta)
+{
+	unsigned int i;
+	unsigned int len;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+
+#ifdef CONFIG_TDLS
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	u8 tdls_prohibited[] = { 0x00, 0x00, 0x00, 0x00, 0x10 }; /* bit(38): TDLS_prohibited */
+#endif /* CONFIG_TDLS */
+
+	len = pkt_len - (_BEACON_IE_OFFSET_ + WLAN_HDR_A3_LEN);
+
+	for (i = 0; i < len;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(pframe + (_BEACON_IE_OFFSET_ + WLAN_HDR_A3_LEN) + i);
+
+		switch (pIE->ElementID) {
+		case _VENDOR_SPECIFIC_IE_:
+			/* to update WMM paramter set while receiving beacon */
+			if (_rtw_memcmp(pIE->data, WMM_PARA_OUI, 6) && pIE->Length == WLAN_WMM_LEN)	/* WMM */
+				(WMM_param_handler(padapter, padapter_link, pIE)) ? report_wmm_edca_update(padapter) : 0;
+
+			break;
+
+		case _HT_EXTRA_INFO_IE_:	/* HT info */
+			/* HT_info_handler(padapter, pIE); */
+			bwmode_update_check(padapter, padapter_link, pIE);
+			break;
+#ifdef CONFIG_80211AC_VHT
+		case EID_OpModeNotification:
+			rtw_process_vht_op_mode_notify(padapter, pIE->data, psta);
+			break;
+#endif /* CONFIG_80211AC_VHT */
+		case _ERPINFO_IE_:
+			ERP_IE_handler(padapter, padapter_link, pIE);
+			VCS_update(padapter, psta);
+			break;
+
+#ifdef CONFIG_TDLS
+		case WLAN_EID_EXT_CAP:
+			if (check_ap_tdls_prohibited(pIE->data, pIE->Length) == _TRUE)
+				ptdlsinfo->ap_prohibited = _TRUE;
+			if (check_ap_tdls_ch_switching_prohibited(pIE->data, pIE->Length) == _TRUE)
+				ptdlsinfo->ch_switch_prohibited = _TRUE;
+			break;
+#endif /* CONFIG_TDLS */
+#ifdef CONFIG_80211AX_HE
+		case WLAN_EID_EXTENSION:
+			if (pIE->data[0] == WLAN_EID_EXTENSION_HE_OPERATION)
+				HE_operation_handler(padapter, padapter_link, pIE);
+			else if (pIE->data[0] == WLAN_EID_EXTENSION_HE_MU_EDCA)
+				HE_mu_edca_handler(padapter, padapter_link, pIE, _FALSE);
+			break;
+#endif
+		default:
+			break;
+		}
+
+		i += (pIE->Length + 2);
+	}
+}
+
+#ifdef CONFIG_ECSA_PHL
+void process_csa_ie(_adapter *padapter, u8 *ies, uint ies_len)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct ieee80211_info_element *ie;
+	u8 *wide_bw_ie;
+	uint wide_bw_ie_len;
+	u8 csa_ch = 0, csa_offset = 0, csa_ch_width = 0xff, seg_0 = 0, seg_1 = 0;
+	u8 csa_switch_cnt = 0, csa_mode = 0;
+	u8 ecsa_mode = 0, ecsa_op_class = 0, ecsa_ch = 0, ecsa_switch_cnt = 0;
+
+#ifdef DBG_CSA
+	u8 *p;
+	u32 ie_len = 0;
+	p = rtw_get_ie(ies, WLAN_EID_CHANNEL_SWITCH, &ie_len, ies_len);
+	if (p && ie_len > 0)
+		RTW_INFO("CSA : %s, CH = %u, count = %u\n",__func__, *(p+2+1), *(p+2+2));
+#endif
+
+	/* compare with scheduling CSA to block incoming CSA IE */
+	if (rtw_mr_is_ecsa_running(padapter)
+		|| check_fwstate(pmlmepriv, WIFI_CSA_UPDATE_BEACON))
+		return;
+
+	for_each_ie(ie, ies, ies_len) {
+		switch (ie->id) {
+		case WLAN_EID_CHANNEL_SWITCH:
+			csa_mode = *(ie->data);
+			csa_ch = *(ie->data + 1);
+			csa_switch_cnt = *(ie->data + 2);
+
+			RTW_INFO("CSA : CHANNEL_SWITCH IE, mode = %u, ch = %u, switch count = %u\n",
+				csa_mode, csa_ch, csa_switch_cnt);
+			break;
+		/* This element is not necessary for channel switching */
+		case WLAN_EID_SECONDARY_CHANNEL_OFFSET:
+			csa_offset = *(ie->data);
+
+			RTW_INFO("CSA : SECONDARY_CHANNEL_OFFSET IE, secondary_offset = %u\n",
+				csa_offset);
+			break;
+		case WLAN_EID_VHT_WIDE_BW_CHSWITCH:
+			csa_ch_width = *(ie->data);
+			seg_0 = *(ie->data + 1);
+			seg_1 = *(ie->data + 2);
+
+			RTW_INFO("CSA : WIDE_BW_CHSWITCH IE, channel width = %u, segment_0 = %u, segment_1 = %u\n",
+				csa_ch_width, seg_0, seg_1);
+			break;
+		case WLAN_EID_CHANNEL_SWITCH_WRAPPER:
+			wide_bw_ie = rtw_get_ie(ie->data, WLAN_EID_VHT_WIDE_BW_CHSWITCH, &wide_bw_ie_len, ie->len);
+			if (wide_bw_ie) {
+				if ((ie->data+ie->len-1) < (wide_bw_ie+1+wide_bw_ie_len)) {
+					RTW_INFO("CSA : %s,out of range \n",__func__);
+					break;
+				}
+				csa_ch_width = *(wide_bw_ie + 2);
+				seg_0 = *(wide_bw_ie + 3);
+				seg_1 = *(wide_bw_ie + 4);
+
+				RTW_INFO("CSA : WIDE_BW_CHSWITCH IE, channel width = %u, segment_0 = %u, segment_1 = %u\n",
+					csa_ch_width, seg_0, seg_1);
+			}
+			break;
+		case WLAN_EID_ECSA:
+			ecsa_mode = *(ie->data);
+			ecsa_op_class = *(ie->data + 1);
+			ecsa_ch = *(ie->data + 2);
+			ecsa_switch_cnt = *(ie->data + 3);
+
+			RTW_INFO("CSA : EXT_CHANNEL_SWITCH IE, mode = %u, op class = %u, ch = %u, switch count = %u\n",
+				ecsa_mode, ecsa_op_class, ecsa_ch, ecsa_switch_cnt);
+			break;
+		default:
+			break;
+		}
+	}
+
+	/* Check conflict of CSA IE and ECSA IE */
+	if (rtw_is_ecsa_enabled(padapter) && ecsa_ch != 0) {
+		if (csa_ch == 0) { /* has ECSA IE, no CSA IE */
+			csa_mode = ecsa_mode;
+			csa_switch_cnt = ecsa_switch_cnt;
+			csa_ch = ecsa_ch;
+		} else if (csa_mode != ecsa_mode ||
+				csa_ch != ecsa_ch ||
+				csa_switch_cnt != ecsa_switch_cnt) { /*has both IE, but conflict */
+			RTW_ERR("CSA : There has a conflict of CSA IE and ECSA IE\n");
+			return;
+		}
+	}
+
+	if (csa_ch != 0) {
+		u8 csa_band = BAND_MAX;
+		u8 csa_bw = 0; /* handle at ECSA function, fill 0 here */
+
+		if (ecsa_op_class)
+			csa_band = rtw_get_band_by_op_class(ecsa_op_class);
+
+		/* legacy CSA, backward compatiable; NOT happen on 6G channel */
+		if (csa_band == BAND_MAX)
+			csa_band = (csa_ch > 14) ? BAND_ON_5G : BAND_ON_24G;
+
+		rtw_hal_trigger_csa_start(padapter, CSA_STA_RX_CSA_IE,
+			csa_mode, ecsa_op_class, csa_switch_cnt,
+			csa_band, csa_ch, csa_bw, csa_offset,
+			csa_ch_width, seg_0, seg_1);
+	}
+}
+#endif /* CONFIG_ECSA_PHL */
+
+enum eap_type parsing_eapol_packet(_adapter *padapter, u8 *key_payload, struct sta_info *psta, u8 trx_type)
+{
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	struct ieee802_1x_hdr *hdr;
+	struct wpa_eapol_key *key;
+	u16 key_info;
+	u8 key_nonce[WPA_NONCE_LEN] = {0};	/* key_nonce is only empty in EAPOL 4/4 */
+	char *trx_msg = trx_type ? "send" : "recv";
+	enum eap_type eapol_type;
+
+	hdr = (struct ieee802_1x_hdr *) key_payload;
+
+	 /* WPS - eapol start packet */
+	if (hdr->type == 1 && hdr->length == 0) {
+		RTW_INFO("%s eapol start packet\n", trx_msg);
+		return EAPOL_START;
+	}
+
+	if (hdr->type == 0) { /* WPS - eapol packet */
+		RTW_INFO("%s eapol packet\n", trx_msg);
+		return EAPOL_PACKET;
+	}
+
+	key = (struct wpa_eapol_key *) (hdr + 1);
+	key_info = be16_to_cpu(*((u16 *)(key->key_info)));
+
+	if (!(key_info & WPA_KEY_INFO_KEY_TYPE)) { /* WPA group key handshake */
+		if (key_info & WPA_KEY_INFO_ACK) {
+			RTW_PRINT("%s eapol packet - WPA Group Key 1/2\n", trx_msg);
+			eapol_type = EAPOL_WPA_GROUP_KEY_1_2;
+		} else {
+			RTW_PRINT("%s eapol packet - WPA Group Key 2/2\n", trx_msg);
+			eapol_type = EAPOL_WPA_GROUP_KEY_2_2;
+
+			/* WPA key-handshake has completed */
+			if (psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPAPSK)
+				psta->state &= (~WIFI_UNDER_KEY_HANDSHAKE);
+		}
+	} else if (key_info & WPA_KEY_INFO_MIC) {
+		if (key_info & WPA_KEY_INFO_ACK) {
+			RTW_PRINT("%s eapol packet 3/4\n", trx_msg);
+			eapol_type = EAPOL_3_4;
+		} else if (_rtw_memcmp(key_nonce, key->key_nonce, WPA_NONCE_LEN)) {
+			RTW_PRINT("%s eapol packet 4/4\n", trx_msg);
+			eapol_type = EAPOL_4_4;
+		} else {
+			RTW_PRINT("%s eapol packet 2/4\n", trx_msg);
+			eapol_type = EAPOL_2_4;
+		}
+	} else {
+		RTW_PRINT("%s eapol packet 1/4\n", trx_msg);
+		eapol_type = EAPOL_1_4;
+	}
+
+	return eapol_type;
+}
+
+unsigned int is_ap_in_tkip(struct _ADAPTER_LINK *padapter_link)
+{
+	u32 i;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*cur_network = &(pmlmeinfo->network);
+
+	if (rtw_get_capability((WLAN_BSSID_EX *)cur_network) & WLAN_CAPABILITY_PRIVACY) {
+		for (i = _FIXED_IE_LENGTH_; i < pmlmeinfo->network.IELength;) {
+			pIE = (PNDIS_802_11_VARIABLE_IEs)(pmlmeinfo->network.IEs + i);
+
+			switch (pIE->ElementID) {
+			case _VENDOR_SPECIFIC_IE_:
+				if ((_rtw_memcmp(pIE->data, RTW_WPA_OUI, 4)) && (_rtw_memcmp((pIE->data + 12), WPA_TKIP_CIPHER, 4)))
+					return _TRUE;
+				break;
+
+			case _RSN_IE_2_:
+				if (_rtw_memcmp((pIE->data + 8), RSN_TKIP_CIPHER, 4))
+					return _TRUE;
+				break;
+
+			default:
+				break;
+			}
+
+			i += (pIE->Length + 2);
+		}
+
+		return _FALSE;
+	} else
+		return _FALSE;
+
+}
+
+unsigned int should_forbid_n_rate(_adapter *padapter)
+{
+	u32 i;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	WLAN_BSSID_EX  *cur_network = &pmlmepriv->dev_cur_network.network;
+
+	if (rtw_get_capability((WLAN_BSSID_EX *)cur_network) & WLAN_CAPABILITY_PRIVACY) {
+		for (i = _FIXED_IE_LENGTH_; i < cur_network->IELength;) {
+			pIE = (PNDIS_802_11_VARIABLE_IEs)(cur_network->IEs + i);
+
+			switch (pIE->ElementID) {
+			case _VENDOR_SPECIFIC_IE_:
+				if (_rtw_memcmp(pIE->data, RTW_WPA_OUI, 4) &&
+				    ((_rtw_memcmp((pIE->data + 12), WPA_CIPHER_SUITE_CCMP, 4)) ||
+				     (_rtw_memcmp((pIE->data + 16), WPA_CIPHER_SUITE_CCMP, 4))))
+					return _FALSE;
+				break;
+
+			case _RSN_IE_2_:
+				if ((_rtw_memcmp((pIE->data + 8), RSN_CIPHER_SUITE_CCMP, 4))  ||
+				    (_rtw_memcmp((pIE->data + 12), RSN_CIPHER_SUITE_CCMP, 4)))
+					return _FALSE;
+				break;
+
+			default:
+				break;
+			}
+
+			i += (pIE->Length + 2);
+		}
+
+		return _TRUE;
+	} else
+		return _FALSE;
+
+}
+
+
+unsigned int is_ap_in_wep(struct _ADAPTER_LINK *padapter_link)
+{
+	u32 i;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*cur_network = &(pmlmeinfo->network);
+
+	if (rtw_get_capability((WLAN_BSSID_EX *)cur_network) & WLAN_CAPABILITY_PRIVACY) {
+		for (i = _FIXED_IE_LENGTH_; i < pmlmeinfo->network.IELength;) {
+			pIE = (PNDIS_802_11_VARIABLE_IEs)(pmlmeinfo->network.IEs + i);
+
+			switch (pIE->ElementID) {
+			case _VENDOR_SPECIFIC_IE_:
+				if (_rtw_memcmp(pIE->data, RTW_WPA_OUI, 4))
+					return _FALSE;
+				break;
+
+			case _RSN_IE_2_:
+				return _FALSE;
+
+			default:
+				break;
+			}
+
+			i += (pIE->Length + 2);
+		}
+
+		return _TRUE;
+	} else
+		return _FALSE;
+
+}
+
+int wifirate2_ratetbl_inx(unsigned char rate);
+int wifirate2_ratetbl_inx(unsigned char rate)
+{
+	int	inx = 0;
+	rate = rate & 0x7f;
+
+	switch (rate) {
+	case 54*2:
+		inx = 11;
+		break;
+
+	case 48*2:
+		inx = 10;
+		break;
+
+	case 36*2:
+		inx = 9;
+		break;
+
+	case 24*2:
+		inx = 8;
+		break;
+
+	case 18*2:
+		inx = 7;
+		break;
+
+	case 12*2:
+		inx = 6;
+		break;
+
+	case 9*2:
+		inx = 5;
+		break;
+
+	case 6*2:
+		inx = 4;
+		break;
+
+	case 11*2:
+		inx = 3;
+		break;
+	case 11:
+		inx = 2;
+		break;
+
+	case 2*2:
+		inx = 1;
+		break;
+
+	case 1*2:
+		inx = 0;
+		break;
+
+	}
+	return inx;
+}
+
+unsigned int update_basic_rate(unsigned char *ptn, unsigned int ptn_sz)
+{
+	unsigned int i, num_of_rate;
+	unsigned int mask = 0;
+
+	num_of_rate = (ptn_sz > NumRates) ? NumRates : ptn_sz;
+
+	for (i = 0; i < num_of_rate; i++) {
+		if ((*(ptn + i)) & 0x80)
+			mask |= 0x1 << wifirate2_ratetbl_inx(*(ptn + i));
+	}
+	return mask;
+}
+
+unsigned int update_supported_rate(unsigned char *ptn, unsigned int ptn_sz)
+{
+	unsigned int i, num_of_rate;
+	unsigned int mask = 0;
+
+	num_of_rate = (ptn_sz > NumRates) ? NumRates : ptn_sz;
+
+	for (i = 0; i < num_of_rate; i++)
+		mask |= 0x1 << wifirate2_ratetbl_inx(*(ptn + i));
+
+	return mask;
+}
+
+int support_short_GI(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			struct HT_caps_element *pHT_caps, u8 bwmode)
+{
+	unsigned char					bit_offset;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	if (!(pmlmeinfo->HT_enable))
+		return _FAIL;
+
+	bit_offset = (bwmode & CHANNEL_WIDTH_40) ? 6 : 5;
+
+	if (pHT_caps->u.HT_cap_element.HT_caps_info & (0x1 << bit_offset))
+		return _SUCCESS;
+	else
+		return _FAIL;
+}
+
+unsigned char get_highest_rate_idx(u64 mask)
+{
+	int i;
+	unsigned char rate_idx = 0;
+
+	for (i = 63; i >= 0; i--) {
+		if ((mask >> i) & 0x01) {
+			rate_idx = i;
+			break;
+		}
+	}
+
+	return rate_idx;
+}
+
+#if 0
+unsigned char get_lowest_rate_idx_ex(u64 mask, int start_bit)
+{
+	int i;
+	unsigned char rate_idx = 0;
+
+	for (i = start_bit; i < 64; i++) {
+		if ((mask >> i) & 0x01) {
+			rate_idx = i;
+			break;
+		}
+	}
+
+	return rate_idx;
+}
+
+u8 get_highest_bw_cap(u8 bwmode)
+{
+	u8 hbw = CHANNEL_WIDTH_20;
+
+	if (bwmode & BW_CAP_80_80M)
+		hbw = CHANNEL_WIDTH_80_80;
+	else if (bwmode & BW_CAP_160M)
+		hbw = CHANNEL_WIDTH_160;
+	else if (bwmode & BW_CAP_80M)
+		hbw = CHANNEL_WIDTH_80;
+	else if (bwmode & BW_CAP_40M)
+		hbw = CHANNEL_WIDTH_40;
+	else if (bwmode & BW_CAP_20M)
+		hbw = CHANNEL_WIDTH_20;
+	else if (bwmode & BW_CAP_10M)
+		hbw = CHANNEL_WIDTH_10;
+	else if (bwmode & BW_CAP_5M)
+		hbw = CHANNEL_WIDTH_5;
+
+	return hbw;
+}
+#endif
+
+/* Update RRSR and Rate for USERATE */
+void update_tx_basic_rate(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 wirelessmode)
+{
+	NDIS_802_11_RATES_EX	supported_rates;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+
+	_rtw_memset(supported_rates, 0, NDIS_802_11_LENGTH_RATES_EX);
+
+	/* clear B mod if current channel is in non 2G band, avoid tx cck rate in non 2G band. */
+	if (pmlmeext->chandef.band != BAND_ON_24G)
+		wirelessmode &= ~(WLAN_MD_11B);
+
+	if ((wirelessmode & WLAN_MD_11B) && (wirelessmode == WLAN_MD_11B))
+		_rtw_memcpy(supported_rates, rtw_basic_rate_cck, 4);
+	else if (wirelessmode & WLAN_MD_11B)
+		_rtw_memcpy(supported_rates, rtw_basic_rate_mix, 7);
+	else
+		_rtw_memcpy(supported_rates, rtw_basic_rate_ofdm, 3);
+
+	if (wirelessmode & WLAN_MD_11B)
+		update_mgnt_tx_rate(padapter, padapter_link, IEEE80211_CCK_RATE_1MB);
+	else
+		update_mgnt_tx_rate(padapter, padapter_link, IEEE80211_OFDM_RATE_6MB);
+
+	rtw_hal_set_hwreg(padapter, HW_VAR_BASIC_RATE, supported_rates);
+}
+
+unsigned char check_assoc_AP(u8 *pframe, uint len)
+{
+	unsigned int	i;
+	PNDIS_802_11_VARIABLE_IEs	pIE;
+
+	for (i = _FIXED_IE_LENGTH_; i < len;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(pframe + i);
+
+		switch (pIE->ElementID) {
+		case _VENDOR_SPECIFIC_IE_:
+			if ((_rtw_memcmp(pIE->data, ARTHEROS_OUI1, 3)) || (_rtw_memcmp(pIE->data, ARTHEROS_OUI2, 3))) {
+				RTW_INFO("link to Artheros AP\n");
+				return HT_IOT_PEER_ATHEROS;
+			} else if ((_rtw_memcmp(pIE->data, BROADCOM_OUI1, 3))
+				   || (_rtw_memcmp(pIE->data, BROADCOM_OUI2, 3))
+				|| (_rtw_memcmp(pIE->data, BROADCOM_OUI3, 3))) {
+				RTW_INFO("link to Broadcom AP\n");
+				return HT_IOT_PEER_BROADCOM;
+			} else if (_rtw_memcmp(pIE->data, MARVELL_OUI, 3)) {
+				RTW_INFO("link to Marvell AP\n");
+				return HT_IOT_PEER_MARVELL;
+			} else if (_rtw_memcmp(pIE->data, RALINK_OUI, 3)) {
+				RTW_INFO("link to Ralink AP\n");
+				return HT_IOT_PEER_RALINK;
+			} else if (_rtw_memcmp(pIE->data, CISCO_OUI, 3)) {
+				RTW_INFO("link to Cisco AP\n");
+				return HT_IOT_PEER_CISCO;
+			} else if (_rtw_memcmp(pIE->data, REALTEK_OUI, 3)) {
+				u32	Vender = HT_IOT_PEER_REALTEK;
+
+				if (pIE->Length >= 5) {
+					if (pIE->data[4] == 1) {
+						/* if(pIE->data[5] & RT_HT_CAP_USE_LONG_PREAMBLE) */
+						/*	bssDesc->BssHT.RT2RT_HT_Mode |= RT_HT_CAP_USE_LONG_PREAMBLE; */
+
+						if (pIE->data[5] & RT_HT_CAP_USE_92SE) {
+							/* bssDesc->BssHT.RT2RT_HT_Mode |= RT_HT_CAP_USE_92SE; */
+							Vender = HT_IOT_PEER_REALTEK_92SE;
+						}
+					}
+
+					if (pIE->data[5] & RT_HT_CAP_USE_SOFTAP)
+						Vender = HT_IOT_PEER_REALTEK_SOFTAP;
+
+					if (pIE->data[4] == 2) {
+						if (pIE->data[6] & RT_HT_CAP_USE_JAGUAR_CBV) {
+							Vender = HT_IOT_PEER_REALTEK_JAGUAR_CBVAP;
+							RTW_INFO("link to Realtek JAGUAR_CBVAP\n");
+						}
+						if (pIE->data[6] & RT_HT_CAP_USE_JAGUAR_CCV) {
+							Vender = HT_IOT_PEER_REALTEK_JAGUAR_CCVAP;
+							RTW_INFO("link to Realtek JAGUAR_CCVAP\n");
+						}
+					}
+				}
+
+				RTW_INFO("link to Realtek AP\n");
+				return Vender;
+			} else if (_rtw_memcmp(pIE->data, AIRGOCAP_OUI, 3)) {
+				RTW_INFO("link to Airgo Cap\n");
+				return HT_IOT_PEER_AIRGO;
+			} else
+				break;
+
+		default:
+			break;
+		}
+
+		i += (pIE->Length + 2);
+	}
+
+	RTW_INFO("link to new AP\n");
+	return HT_IOT_PEER_UNKNOWN;
+}
+
+void get_assoc_AP_Vendor(char *vendor, u8 assoc_AP_vendor)
+{
+	switch (assoc_AP_vendor) {
+
+	case HT_IOT_PEER_UNKNOWN:
+	sprintf(vendor, "%s", "unknown");
+	break;
+
+	case HT_IOT_PEER_REALTEK:
+	case HT_IOT_PEER_REALTEK_92SE:
+	case HT_IOT_PEER_REALTEK_SOFTAP:
+	case HT_IOT_PEER_REALTEK_JAGUAR_CBVAP:
+	case HT_IOT_PEER_REALTEK_JAGUAR_CCVAP:
+
+	sprintf(vendor, "%s", "Realtek");
+	break;
+
+	case HT_IOT_PEER_BROADCOM:
+	sprintf(vendor, "%s", "Broadcom");
+	break;
+
+	case HT_IOT_PEER_MARVELL:
+	sprintf(vendor, "%s", "Marvell");
+	break;
+
+	case HT_IOT_PEER_RALINK:
+	sprintf(vendor, "%s", "Ralink");
+	break;
+
+	case HT_IOT_PEER_CISCO:
+	sprintf(vendor, "%s", "Cisco");
+	break;
+
+	case HT_IOT_PEER_AIRGO:
+	sprintf(vendor, "%s", "Airgo");
+	break;
+
+	case HT_IOT_PEER_ATHEROS:
+	sprintf(vendor, "%s", "Atheros");
+	break;
+
+	default:
+	sprintf(vendor, "%s", "unkown");
+	break;
+	}
+
+}
+#ifdef CONFIG_RTS_FULL_BW
+void rtw_parse_sta_vendor_ie_8812(_adapter *adapter, struct sta_info *sta, u8 *tlv_ies, u16 tlv_ies_len)
+{
+	unsigned char REALTEK_OUI[] = {0x00,0xe0, 0x4c};
+	u8 *p;
+
+	p = rtw_get_ie_ex(tlv_ies, tlv_ies_len, WLAN_EID_VENDOR_SPECIFIC, REALTEK_OUI, 3, NULL, NULL);
+	if (!p)
+		goto exit;
+	else {
+		if(*(p+1) > 6 ) {
+
+			if(*(p+6) != 2)
+				goto exit;
+
+			if(*(p+8) == RT_HT_CAP_USE_JAGUAR_CBV)
+				sta->vendor_8812 = TRUE;
+			else if (*(p+8) == RT_HT_CAP_USE_JAGUAR_CCV)
+				sta->vendor_8812 = TRUE;
+		}
+	}
+exit:
+	return;
+}
+#endif/*CONFIG_RTS_FULL_BW*/
+
+#ifdef CONFIG_80211AC_VHT
+void get_vht_bf_cap(u8 *pframe, uint len, struct vht_bf_cap *bf_cap)
+{
+	unsigned int i;
+	PNDIS_802_11_VARIABLE_IEs pIE;
+
+	for (i = _FIXED_IE_LENGTH_; i < len;) {
+		pIE = (PNDIS_802_11_VARIABLE_IEs)(pframe + i);
+
+		switch (pIE->ElementID) {
+
+		case EID_VHTCapability:
+			bf_cap->is_mu_bfer = GET_VHT_CAPABILITY_ELE_MU_BFER(pIE->data);
+			bf_cap->su_sound_dim = GET_VHT_CAPABILITY_ELE_SU_BFER_SOUND_DIM_NUM(pIE->data);
+			break;
+		default:
+			break;
+		}
+		i += (pIE->Length + 2);
+	}
+}
+#endif
+
+void update_capinfo(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, u16 updateCap)
+{
+	struct link_mlme_ext_priv	*pmlmeext = &adapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	BOOLEAN		ShortPreamble;
+
+	/* Check preamble mode, 2005.01.06, by rcnjko. */
+	/* Mark to update preamble value forever, 2008.03.18 by lanhsin */
+	/* if( pMgntInfo->RegPreambleMode == PREAMBLE_AUTO ) */
+	{
+
+		if (updateCap & cShortPreamble) {
+			/* Short Preamble */
+			if (pmlmeinfo->preamble_mode != PREAMBLE_SHORT) { /* PREAMBLE_LONG or PREAMBLE_AUTO */
+				ShortPreamble = _TRUE;
+				pmlmeinfo->preamble_mode = PREAMBLE_SHORT;
+				rtw_hal_set_hwreg(adapter, HW_VAR_ACK_PREAMBLE, (u8 *)&ShortPreamble);
+			}
+		} else {
+			/* Long Preamble */
+			if (pmlmeinfo->preamble_mode != PREAMBLE_LONG) { /* PREAMBLE_SHORT or PREAMBLE_AUTO */
+				ShortPreamble = _FALSE;
+				pmlmeinfo->preamble_mode = PREAMBLE_LONG;
+				rtw_hal_set_hwreg(adapter, HW_VAR_ACK_PREAMBLE, (u8 *)&ShortPreamble);
+			}
+		}
+	}
+
+	if (updateCap & cIBSS) {
+		/* Filen: See 802.11-2007 p.91 */
+		pmlmeinfo->slotTime = NON_SHORT_SLOT_TIME;
+	} else {
+		/* Filen: See 802.11-2007 p.90 */
+		if (pmlmeext->cur_wireless_mode & (WLAN_MD_11N | WLAN_MD_11A | WLAN_MD_11AC))
+			pmlmeinfo->slotTime = SHORT_SLOT_TIME;
+		else if (pmlmeext->cur_wireless_mode & (WLAN_MD_11G)) {
+			if ((updateCap & cShortSlotTime) /* && (!(pMgntInfo->pHTInfo->RT2RT_HT_Mode & RT_HT_CAP_USE_LONG_PREAMBLE)) */) {
+				/* Short Slot Time */
+				pmlmeinfo->slotTime = SHORT_SLOT_TIME;
+			} else {
+				/* Long Slot Time */
+				pmlmeinfo->slotTime = NON_SHORT_SLOT_TIME;
+			}
+		} else {
+			/* B Mode */
+			pmlmeinfo->slotTime = NON_SHORT_SLOT_TIME;
+		}
+	}
+
+	rtw_hal_set_hwreg(adapter, HW_VAR_SLOT_TIME, &pmlmeinfo->slotTime);
+
+}
+
+/*
+* set adapter.mlmeextpriv.mlmext_info.HT_enable
+* set adapter.mlmeextpriv.cur_wireless_mode
+* set SIFS register
+* set mgmt tx rate
+*/
+void update_wireless_mode(_adapter *padapter, struct _ADAPTER_LINK *padapter_link)
+{
+	int ratelen, network_type = 0;
+	u32 SIFS_Timer;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*cur_network = &(pmlmeinfo->network);
+	unsigned char			*rate = cur_network->SupportedRates;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+
+	ratelen = rtw_get_rateset_len(cur_network->SupportedRates);
+
+	if ((pmlmeinfo->HT_info_enable) && (pmlmeinfo->HT_caps_enable))
+		pmlmeinfo->HT_enable = 1;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (pmlmeext->chandef.band == BAND_ON_6G)
+		network_type = WLAN_MD_6G_MIX;
+	else
+#endif
+	if (pmlmeext->chandef.band == BAND_ON_5G) {
+		if (pmlmeinfo->HE_enable)
+			network_type = WLAN_MD_11AX;
+		else if (pmlmeinfo->VHT_enable)
+			network_type = WLAN_MD_11AC;
+		else if (pmlmeinfo->HT_enable)
+			network_type = WLAN_MD_11N;
+
+		network_type |= WLAN_MD_11A;
+	} else {
+		if (pmlmeinfo->HE_enable)
+			network_type = WLAN_MD_11AX;
+		else if (pmlmeinfo->VHT_enable)
+			network_type = WLAN_MD_11AC;
+		else if (pmlmeinfo->HT_enable)
+			network_type = WLAN_MD_11N;
+
+		if ((cckratesonly_included(rate, ratelen)) == _TRUE)
+			network_type |= WLAN_MD_11B;
+		else if ((cckrates_included(rate, ratelen)) == _TRUE)
+			network_type |= WLAN_MD_11BG;
+		else
+			network_type |= WLAN_MD_11G;
+	}
+
+	pmlmeext->cur_wireless_mode = network_type & padapter->registrypriv.wireless_mode;
+	/* RTW_INFO("network_type=%02x, padapter->registrypriv.wireless_mode=%02x\n", network_type, padapter->registrypriv.wireless_mode); */
+
+	if ((pmlmeext->cur_wireless_mode & WLAN_MD_11B) &&
+	    #ifdef CONFIG_P2P
+	    !rtw_p2p_chk_role(pwdinfo, P2P_ROLE_CLIENT) &&
+	    !rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO) &&
+	    #endif
+	    1)
+		update_mgnt_tx_rate(padapter, padapter_link, IEEE80211_CCK_RATE_1MB);
+	else
+		update_mgnt_tx_rate(padapter, padapter_link, IEEE80211_OFDM_RATE_6MB);
+}
+
+#if 0
+void update_sta_basic_rate(struct sta_info *psta, u8 wireless_mode)
+{
+	if (is_supported_tx_cck(wireless_mode)) {
+		/* Only B, B/G, and B/G/N AP could use CCK rate */
+		_rtw_memcpy(psta->bssrateset, rtw_basic_rate_cck, 4);
+		psta->bssratelen = 4;
+	} else {
+		_rtw_memcpy(psta->bssrateset, rtw_basic_rate_ofdm, 3);
+		psta->bssratelen = 3;
+	}
+}
+#endif
+
+static int _rtw_get_supported_rate(u8 *sup_r_ie, sint sup_r_ie_len
+	, u8 *ext_sup_r_ie, sint ext_sup_r_ie_len
+	, u8 *rate_set, u8 *rate_num)
+{
+	u8 *p;
+	int i, j;
+
+	struct support_rate_handler support_rate_tbl[] = {
+		{IEEE80211_CCK_RATE_1MB, 		_FALSE,		_FALSE},
+		{IEEE80211_CCK_RATE_2MB, 		_FALSE,		_FALSE},
+		{IEEE80211_CCK_RATE_5MB, 		_FALSE,		_FALSE},
+		{IEEE80211_CCK_RATE_11MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_6MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_9MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_12MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_18MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_24MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_36MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_48MB,		_FALSE,		_FALSE},
+		{IEEE80211_OFDM_RATE_54MB,		_FALSE,		_FALSE},
+	};
+
+	*rate_num = 0;
+	if (sup_r_ie == NULL)
+		goto ext_rate;
+
+	/* get valid supported rates */
+	for (i = 0; i < 12; i++) {
+		p = sup_r_ie + 2;
+		for (j = 0; j < sup_r_ie_len; j++) {
+			if ((*p & ~BIT(7)) == support_rate_tbl[i].rate){
+				support_rate_tbl[i].existence = _TRUE;
+				if ((*p) & BIT(7))
+					support_rate_tbl[i].basic = _TRUE;
+			}
+			p++;
+		}
+	}
+
+ext_rate:
+	if (ext_sup_r_ie) {
+		/* get valid extended supported rates */
+		for (i = 0; i < 12; i++) {
+			p = ext_sup_r_ie + 2;
+			for (j = 0; j < ext_sup_r_ie_len; j++) {
+				if ((*p & ~BIT(7)) == support_rate_tbl[i].rate){
+					support_rate_tbl[i].existence = _TRUE;
+					if ((*p) & BIT(7))
+						support_rate_tbl[i].basic = _TRUE;
+				}
+				p++;
+			}
+		}
+	}
+
+	for (i = 0; i < 12; i++){
+		if (support_rate_tbl[i].existence){
+			if (support_rate_tbl[i].basic)
+				rate_set[*rate_num] = support_rate_tbl[i].rate | IEEE80211_BASIC_RATE_MASK;
+			else
+				rate_set[*rate_num] = support_rate_tbl[i].rate;
+			*rate_num += 1;
+		}
+	}
+
+	if (0) {
+		int k;
+
+		for (k = 0; k < *rate_num; k++)
+			RTW_INFO("rate:0x%02x\n", *(rate_set + k));
+	}
+
+	return _SUCCESS;
+}
+
+int rtw_ies_get_supported_rate(u8 *ies, uint ies_len, u8 *rate_set, u8 *rate_num)
+{
+	u8 *sup_r_ie;
+	sint sup_r_ie_len;
+	u8 *ext_sup_r_ie;
+	sint ext_sup_r_ie_len;
+
+	if (!rate_set || !rate_num)
+		return _FALSE;
+
+	sup_r_ie = rtw_get_ie(ies, _SUPPORTEDRATES_IE_, &sup_r_ie_len, ies_len);
+	ext_sup_r_ie = rtw_get_ie(ies, _EXT_SUPPORTEDRATES_IE_, &ext_sup_r_ie_len, ies_len);
+
+	return _rtw_get_supported_rate(sup_r_ie, sup_r_ie_len
+		, ext_sup_r_ie, ext_sup_r_ie_len, rate_set, rate_num);
+}
+
+int rtw_elems_get_supported_rate(struct rtw_ieee802_11_elems *elems, u8 *rate_set, u8 *rate_num)
+{
+	u8 *sup_r_ie;
+	sint sup_r_ie_len;
+	u8 *ext_sup_r_ie;
+	sint ext_sup_r_ie_len;
+
+	if (!rate_set || !rate_num)
+		return _FALSE;
+
+	sup_r_ie = elems->supp_rates ? elems->supp_rates - 2 : NULL;
+	sup_r_ie_len = elems->supp_rates_len;
+	ext_sup_r_ie = elems->ext_supp_rates ? elems->ext_supp_rates - 2 : NULL;
+	ext_sup_r_ie_len = elems->ext_supp_rates_len;
+
+	return _rtw_get_supported_rate(sup_r_ie, sup_r_ie_len
+		, ext_sup_r_ie, ext_sup_r_ie_len, rate_set, rate_num);
+}
+
+void process_addba_req(_adapter *padapter, u8 *paddba_req, u8 *addr)
+{
+	struct sta_info *psta;
+	u16 tid, start_seq, param;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct ADDBA_request	*preq = (struct ADDBA_request *)paddba_req;
+	u8 accept = _FALSE;
+	u16 size;
+
+	psta = rtw_get_stainfo(pstapriv, addr);
+	if (!psta)
+		goto exit;
+
+	/* ToDo: ADDBA extension */
+	start_seq = le16_to_cpu(preq->BA_starting_seqctrl) >> 4;
+	param = le16_to_cpu(preq->BA_para_set);
+	tid = (param >> 2) & 0x0f;
+
+	accept = rtw_rx_ampdu_is_accept(padapter);
+	if (padapter->fix_rx_ampdu_size != RX_AMPDU_SIZE_INVALID)
+		size = padapter->fix_rx_ampdu_size;
+	else {
+		size = (param >> 6) & 0x3ff;
+		if (size > 0)
+			size = rtw_min(rtw_rx_ampdu_size(padapter), size);
+		else
+			size = rtw_rx_ampdu_size(padapter);
+		size = rtw_min(size, rx_ampdu_size_sta_limit(padapter, psta));
+	}
+
+	if (accept == _TRUE)
+		rtw_addbarsp_cmd(padapter, addr, tid, preq, 0, size, start_seq);
+	else
+		rtw_addbarsp_cmd(padapter, addr, tid, preq, 37, size, start_seq);/* reject ADDBA Req */
+
+exit:
+	return;
+}
+
+void rtw_process_bar_frame(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 *pframe = precv_frame->u.hdr.rx_data;
+	struct sta_info *psta = NULL;
+	struct recv_reorder_ctrl *preorder_ctrl = NULL;
+	u8 tid = 0;
+	u16 start_seq=0;
+
+	psta = rtw_get_stainfo(pstapriv, get_addr2_ptr(pframe));
+	if (psta == NULL)
+		goto exit;
+
+	tid = ((cpu_to_le16((*(u16 *)(pframe + 16))) & 0xf000) >> 12);
+	preorder_ctrl = &psta->recvreorder_ctrl[tid];
+	start_seq = ((cpu_to_le16(*(u16 *)(pframe + 18))) >> 4);
+	preorder_ctrl->indicate_seq = start_seq;
+
+	rtw_phl_rx_bar(padapter->dvobj->phl, psta->phl_sta, tid, start_seq);
+	/* for Debug use */
+	if (0)
+		RTW_INFO(FUNC_ADPT_FMT" tid=%d, start_seq=%d\n", FUNC_ADPT_ARG(padapter),  tid, start_seq);
+
+exit:
+	return;
+}
+
+void update_TSF(struct link_mlme_ext_priv *pmlmeext, u8 *pframe, uint len)
+{
+	u8 *pIE;
+	u32 *pbuf;
+
+	pIE = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+	pbuf = (u32 *)pIE;
+
+	pmlmeext->TSFValue = le32_to_cpu(*(pbuf + 1));
+
+	pmlmeext->TSFValue = pmlmeext->TSFValue << 32;
+
+	pmlmeext->TSFValue |= le32_to_cpu(*pbuf);
+}
+
+#ifdef CONFIG_BCN_RECV_TIME
+/*	calculate beacon receiving time
+	1.RxBCNTime(CCK_1M) = [192us(preamble)] + [length of beacon(byte)*8us] + [10us]
+	2.RxBCNTime(OFDM_6M) = [8us(S) + 8us(L) + 4us(L-SIG)] + [(length of beacon(byte)/3 + 1] *4us] + [10us]
+*/
+inline u16 _rx_bcn_time_calculate(uint bcn_len, u8 data_rate)
+{
+	u16 rx_bcn_time = 0;/*us*/
+
+	if (data_rate == DESC_RATE1M)
+		rx_bcn_time = 192 + bcn_len * 8 + 10;
+	else if(data_rate == DESC_RATE6M)
+		rx_bcn_time = 8 + 8 + 4 + (bcn_len /3 + 1) * 4 + 10;
+/*
+	else
+		RTW_ERR("%s invalid data rate(0x%02x)\n", __func__, data_rate);
+*/
+	return rx_bcn_time;
+}
+void rtw_rx_bcn_time_update(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, uint bcn_len, u8 data_rate)
+{
+	struct link_mlme_ext_priv *pmlmeext = &adapter_link->mlmeextpriv;
+
+	pmlmeext->bcn_rx_time = _rx_bcn_time_calculate(bcn_len, data_rate);
+}
+#endif
+
+void beacon_timing_control(_adapter *padapter)
+{
+	rtw_hal_bcn_param_setting(padapter);
+}
+
+inline void rtw_collect_bcn_info(_adapter *adapter, struct _ADAPTER_LINK *adapter_link)
+{
+	struct link_mlme_ext_priv *pmlmeext = &adapter_link->mlmeextpriv;
+
+	if (!is_client_associated_to_ap(adapter))
+		return;
+
+	pmlmeext->cur_bcn_cnt = pmlmeext->bcn_cnt - pmlmeext->last_bcn_cnt;
+	pmlmeext->last_bcn_cnt = pmlmeext->bcn_cnt;
+	/*TODO get offset of bcn's timestamp*/
+	/*pmlmeext->bcn_timestamp;*/
+}
+
+inline bool rtw_bmp_is_set(const u8 *bmp, u8 bmp_len, u8 id)
+{
+	if (id / 8 >= bmp_len)
+		return 0;
+
+	return bmp[id / 8] & BIT(id % 8);
+}
+
+inline void rtw_bmp_set(u8 *bmp, u8 bmp_len, u8 id)
+{
+	if (id / 8 < bmp_len)
+		bmp[id / 8] |= BIT(id % 8);
+}
+
+inline void rtw_bmp_clear(u8 *bmp, u8 bmp_len, u8 id)
+{
+	if (id / 8 < bmp_len)
+		bmp[id / 8] &= ~BIT(id % 8);
+}
+
+inline bool rtw_bmp_not_empty(const u8 *bmp, u8 bmp_len)
+{
+	int i;
+
+	for (i = 0; i < bmp_len; i++) {
+		if (bmp[i])
+			return 1;
+	}
+
+	return 0;
+}
+
+inline bool rtw_bmp_not_empty_exclude_bit0(const u8 *bmp, u8 bmp_len)
+{
+	int i;
+
+	for (i = 0; i < bmp_len; i++) {
+		if (i == 0) {
+			if (bmp[i] & 0xFE)
+				return 1;
+		} else {
+			if (bmp[i])
+				return 1;
+		}
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_AP_MODE
+/* Check the id be set or not in map , if yes , return a none zero value*/
+bool rtw_tim_map_is_set(_adapter *padapter, const u8 *map, u8 id)
+{
+	return rtw_bmp_is_set(map, padapter->stapriv.aid_bmp_len, id);
+}
+
+/* Set the id into map array*/
+void rtw_tim_map_set(_adapter *padapter, u8 *map, u8 id)
+{
+	rtw_bmp_set(map, padapter->stapriv.aid_bmp_len, id);
+}
+
+/* Clear the id from map array*/
+void rtw_tim_map_clear(_adapter *padapter, u8 *map, u8 id)
+{
+	rtw_bmp_clear(map, padapter->stapriv.aid_bmp_len, id);
+}
+
+/* Check have anyone bit be set , if yes return true*/
+bool rtw_tim_map_anyone_be_set(_adapter *padapter, const u8 *map)
+{
+	return rtw_bmp_not_empty(map, padapter->stapriv.aid_bmp_len);
+}
+
+/* Check have anyone bit be set exclude bit0 , if yes return true*/
+bool rtw_tim_map_anyone_be_set_exclude_aid0(_adapter *padapter, const u8 *map)
+{
+	return rtw_bmp_not_empty_exclude_bit0(map, padapter->stapriv.aid_bmp_len);
+}
+#endif /* CONFIG_AP_MODE */
+
+_adapter *dvobj_get_unregisterd_adapter(struct dvobj_priv *dvobj)
+{
+	_adapter *adapter = NULL;
+	int i;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (dvobj->padapters[i]->registered == 0)
+			break;
+	}
+
+	if (i < dvobj->iface_nums)
+		adapter = dvobj->padapters[i];
+
+	return adapter;
+}
+
+_adapter *dvobj_get_adapter_by_addr(struct dvobj_priv *dvobj, u8 *addr)
+{
+	_adapter *adapter = NULL;
+	int i;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (_rtw_memcmp(dvobj->padapters[i]->mac_addr, addr, ETH_ALEN) == _TRUE)
+			break;
+	}
+
+	if (i < dvobj->iface_nums)
+		adapter = dvobj->padapters[i];
+
+	return adapter;
+}
+
+static u32 rtw_get_he_bitrate(u8 mcs, u8 bw, u8 nss, u8 sgi)
+{
+	static const u32 base[4][3][12] = { /*[bw][GI][MCS] */
+		/* BW20 */
+		{{   8600000,  17200000,  25800000,  34400000, /* GI 0.8u */
+		    51600000,  68800000,  77400000,  86000000,
+		   103200000, 114700000, 129000000, 143400000,
+		},
+		{    8100000,  16300000,  24400000,  32500000, /* GI 1.6u */
+		    48800000,  65000000,  73100000,  81300000,
+		    97500000, 108300000, 121900000, 135400000,
+		},
+		{    7300000,  14600000,  21900000,  29300000, /* GI 3.2u */
+		    43900000,  58500000,  65800000,  73100000,
+		    87800000,  97500000, 109700000, 121900000,
+		}},
+		/* BW40 */
+		{{  17200000,  34400000,  51600000,  68800000, /* GI 0.8u */
+		   103200000, 137600000, 154900000, 172100000,
+		   206500000, 229400000, 258100000, 286800000,
+		},
+		{   16300000,  32500000,  48800000,  65000000, /* GI 1.6u */
+		    97500000, 130000000, 146300000, 162500000,
+		   195000000, 216700000, 243800000, 270800000,
+		},
+		{   14600000,  29300000,  43900000,  58500000, /* GI 3.2u */
+		    87800000, 117000000, 131600000, 146300000,
+		   175500000, 195000000, 219400000, 243800000,
+		}},
+		/* BW80 */
+		{{  36000000,  72100000, 108100000, 144100000, /* GI 0.8u */
+		   216200000, 288200000, 324300000, 360300000,
+		   432400000, 480400000, 540400000, 600500000,
+		},
+		{   34000000,  68100000, 102100000, 136100000, /* GI 1.6u */
+		   204200000, 272200000, 306300000, 340300000,
+		   408300000, 453700000, 510400000, 567100000,
+		},
+		{   30600000,  61300000,  91900000, 122500000, /* GI 3.2u */
+		   183800000, 245000000, 275600000, 306300000,
+		   367500000, 408300000, 459400000, 510400000,
+		}},
+		/* BW160 and BW80+80 */
+		{{  72100000, 144100000, 216200000, 288200000, /* GI 0.8u */
+		   432400000, 576500000, 648500000, 720600000,
+		   864700000, 960800000,1080900000,1201000000,
+		},
+		{   68100000, 136100000, 204200000, 272200000, /* GI 1.6u */
+		   408300000, 544400000, 612500000, 680600000,
+		   816700000, 907400000,1020800000,1134300000,
+		},
+		{   61300000, 122500000, 183800000, 245000000, /* GI 3.2u */
+		   367500000, 490000000, 551300000, 612500000,
+		   735000000, 816700000, 918800000,1020800000,
+		}}
+	};
+	u32 bitrate;
+	int bw_idx, sgi_idx;
+
+	if (mcs > 11) {
+		RTW_INFO("Invalid mcs = %d\n", mcs);
+		return 0;
+	}
+
+	if (nss > 4 || nss < 1) {
+		RTW_INFO("Now only support nss = 1, 2, 3, 4\n");
+		return 0;
+	}
+
+	switch (bw) {
+	case CHANNEL_WIDTH_80_80:
+	case CHANNEL_WIDTH_160:
+		bw_idx = 3;
+		break;
+	case CHANNEL_WIDTH_80:
+		bw_idx = 2;
+		break;
+	case CHANNEL_WIDTH_40:
+		bw_idx = 1;
+		break;
+	case CHANNEL_WIDTH_20:
+		bw_idx = 0;
+		break;
+	default:
+		RTW_INFO("bw = %d currently not supported\n", bw);
+		return 0;
+	}
+
+	/* refer to mdata.rx_gi_ltf */
+	switch (sgi) {
+	case RTW_GILTF_LGI_4XHE32:
+		sgi_idx = 2; /* 3.2 GI */
+		break;
+	case RTW_GILTF_2XHE16:
+	case RTW_GILTF_1XHE16:
+		sgi_idx = 1; /* 1.6 GI */
+		break;
+	case RTW_GILTF_SGI_4XHE08:
+	case RTW_GILTF_2XHE08:
+	case RTW_GILTF_1XHE08:
+		sgi_idx = 0; /* 0.8 GI */
+		break;
+	default:
+		RTW_INFO("gi_ltf = %d currently not supported\n", sgi);
+		return 0;
+	}
+	bitrate = base[bw_idx][sgi_idx][mcs];
+	bitrate *= nss;
+	return (bitrate/100000);
+}
+
+static u32 rtw_get_vht_bitrate(u8 mcs, u8 bw, u8 nss, u8 sgi)
+{
+	static const u32 base[4][10] = {
+		{   6500000,
+		   13000000,
+		   19500000,
+		   26000000,
+		   39000000,
+		   52000000,
+		   58500000,
+		   65000000,
+		   78000000,
+		/* not in the spec, but some devices use this: */
+		   86500000,
+		},
+		{  13500000,
+		   27000000,
+		   40500000,
+		   54000000,
+		   81000000,
+		  108000000,
+		  121500000,
+		  135000000,
+		  162000000,
+		  180000000,
+		},
+		{  29300000,
+		   58500000,
+		   87800000,
+		  117000000,
+		  175500000,
+		  234000000,
+		  263300000,
+		  292500000,
+		  351000000,
+		  390000000,
+		},
+		{  58500000,
+		  117000000,
+		  175500000,
+		  234000000,
+		  351000000,
+		  468000000,
+		  526500000,
+		  585000000,
+		  702000000,
+		  780000000,
+		},
+	};
+	u32 bitrate;
+	int bw_idx;
+
+	if (mcs > 9) {
+		RTW_INFO("Invalid mcs = %d\n", mcs);
+		return 0;
+	}
+
+	if (nss > 4 || nss < 1) {
+		RTW_INFO("Now only support nss = 1, 2, 3, 4\n");
+	}
+
+	switch (bw) {
+	case CHANNEL_WIDTH_160:
+		bw_idx = 3;
+		break;
+	case CHANNEL_WIDTH_80:
+		bw_idx = 2;
+		break;
+	case CHANNEL_WIDTH_40:
+		bw_idx = 1;
+		break;
+	case CHANNEL_WIDTH_20:
+		bw_idx = 0;
+		break;
+	default:
+		RTW_INFO("bw = %d currently not supported\n", bw);
+		return 0;
+	}
+
+	bitrate = base[bw_idx][mcs];
+	bitrate *= nss;
+
+	if (sgi)
+		bitrate = (bitrate / 9) * 10;
+
+	/* do NOT round down here */
+	return (bitrate + 50000) / 100000;
+}
+
+static u32 rtw_get_ht_bitrate(u8 mcs, u8 bw, u8 sgi)
+{
+	int modulation, streams, bitrate;
+
+	/* the formula below does only work for MCS values smaller than 32 */
+	if (mcs >= 32) {
+		RTW_INFO("Invalid mcs = %d\n", mcs);
+		return 0;
+	}
+
+	if (bw > 1) {
+		RTW_INFO("Now HT only support bw = 0(20Mhz), 1(40Mhz)\n");
+		return 0;
+	}
+
+	modulation = mcs & 7;
+	streams = (mcs >> 3) + 1;
+
+	bitrate = (bw == 1) ? 13500000 : 6500000;
+
+	if (modulation < 4)
+		bitrate *= (modulation + 1);
+	else if (modulation == 4)
+		bitrate *= (modulation + 2);
+	else
+		bitrate *= (modulation + 3);
+
+	bitrate *= streams;
+
+	if (sgi)
+		bitrate = (bitrate / 9) * 10;
+
+	/* do NOT round down here */
+	return (bitrate + 50000) / 100000;
+}
+
+/**
+ * @bw: 0(20Mhz), 1(40Mhz), 2(80Mhz), 3(160Mhz)
+ * @data_rate: enum rtw_data_rate;
+ * @sgi: enum rtw_gi_ltf
+ * Returns: bitrate in 100kbps
+ */
+u32 rtw_desc_rate_to_bitrate(u8 bw, u16 data_rate, u8 sgi)
+{
+	u32 bitrate = DESC_RATE1M;
+
+	if (data_rate <= DESC_RATE54M){
+		u16 ofdm_rate[12] = {10, 20, 55, 110,
+			60, 90, 120, 180, 240, 360, 480, 540};
+		bitrate = ofdm_rate[data_rate];
+	} else if ((RTW_DATA_RATE_MCS0 <= data_rate) &&
+		   (data_rate <= RTW_DATA_RATE_MCS31)) {
+		u8 mcs = data_rate - RTW_DATA_RATE_MCS0;
+		bitrate = rtw_get_ht_bitrate(mcs, bw, sgi);
+	} else if ((RTW_DATA_RATE_VHT_NSS1_MCS0 <= data_rate) &&
+		   (data_rate <= RTW_DATA_RATE_VHT_NSS4_MCS9)) {
+		u8 mcs = data_rate & 0xF;
+		u8 nss  = ((data_rate - RTW_DATA_RATE_VHT_NSS1_MCS0) >> 4) + 1;
+		bitrate = rtw_get_vht_bitrate(mcs, bw, nss, sgi);
+	} else if ((RTW_DATA_RATE_HE_NSS1_MCS0 <= data_rate) &&
+		   (data_rate <= RTW_DATA_RATE_HE_NSS4_MCS11)) {
+		u8 mcs = data_rate & 0xF;
+		u8 nss  = ((data_rate - RTW_DATA_RATE_HE_NSS1_MCS0) >> 4) + 1;
+		bitrate = rtw_get_he_bitrate(mcs, bw, nss, sgi);
+	} else {
+		/* 60Ghz ??? */
+		bitrate = 1;
+	}
+
+	return bitrate;
+}
+
+u16 rtw_get_current_tx_rate(_adapter *adapter, struct sta_info *psta)
+{
+	u16 rate_id = 0;
+	struct rtw_phl_rainfo ra_info;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	if (!psta)
+		return rate_id;
+
+	if (adapter->fix_rate != NO_FIX_RATE)
+		rate_id = GET_FIX_RATE(adapter->fix_rate);
+	else {
+		rtw_phl_query_rainfo(dvobj->phl, psta->phl_sta, &ra_info);
+		rate_id = ra_info.rate; /* enum rtw_data_rate */
+	}
+
+	return rate_id;
+}
+
+u8 rtw_get_current_tx_sgi(_adapter *adapter, struct sta_info *psta)
+{
+	u8 curr_tx_sgi = 0;
+	struct rtw_phl_rainfo ra_info;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	if (!psta)
+		return curr_tx_sgi;
+
+	if (adapter->fix_rate != NO_FIX_RATE)
+		/* fix rate */
+		curr_tx_sgi = GET_FIX_RATE_SGI(adapter->fix_rate);
+	else {
+		rtw_phl_query_rainfo(dvobj->phl, psta->phl_sta, &ra_info);
+		curr_tx_sgi = ra_info.gi_ltf;
+	}
+
+	return curr_tx_sgi;
+}
+
+bool rtw_chk_phy_can_append_actrl(_adapter *padapter, struct sta_info *psta)
+{
+	bool is_actrl = _FALSE;
+	struct rtw_phl_rainfo ra_info;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	if (!psta)
+		return is_actrl;
+
+	if (padapter->fix_rate != NO_FIX_RATE)
+		if (rtw_get_current_tx_rate(padapter, psta) < RTW_DATA_RATE_MCS0)
+			return is_actrl;
+
+	rtw_phl_query_rainfo(dvobj->phl, psta->phl_sta, &ra_info);
+	is_actrl = ra_info.is_actrl;
+
+	return is_actrl;
+}
+
+void rtw_get_current_rx_info(_adapter *adapter, struct sta_info *psta,
+	u16 *rate, u8 *bw, u8 *gi_ltf)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	if (!psta)
+		return;
+
+	rtw_phl_get_rx_stat(dvobj->phl, psta->phl_sta, rate, bw, gi_ltf);
+}
+
+#ifdef CONFIG_RTW_MULTI_AP
+u8 rtw_get_ch_utilization(_adapter *adapter)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(d);
+	void *phl = GET_PHL_INFO(d);
+	struct rtw_env_report rpt;
+	u16 clm, nhm, ch_util;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	rtw_phl_get_env_rpt(phl, &rpt, adapter_link->wrlink->hw_band);
+
+	clm = rpt.nhm_cca_ratio;
+	nhm = rpt.nhm_ratio;
+
+	ch_util = clm / 3 + (2 * (nhm / 3));
+	/* For Multi-AP, scaling 0-100 to 0-255 */
+	ch_util = 255 * ch_util / 100;
+
+	return (u8)ch_util;
+}
+
+void rtw_ch_util_rpt(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	_adapter *iface;
+	int i, j;
+	u8 i_rpts = 0;
+	u8 *ch_util;
+	u8 **bssid;
+	u8 threshold = GET_PRIMARY_ADAPTER(adapter)->ch_util_threshold;
+	u8 need_rpt = 0;
+
+	if (threshold == 0)
+		return;
+
+	ch_util = rtw_zmalloc(sizeof(u8) * dvobj->iface_nums);
+	if (!ch_util)
+		goto err_out;
+
+	bssid = (u8 **)rtw_zmalloc(sizeof(u8 *) * dvobj->iface_nums);
+	if (!bssid)
+		goto err_out1;
+
+	for (j = 0; j < dvobj->iface_nums; j++) {
+		*(bssid + j) = (u8 *)rtw_zmalloc(sizeof(u8) * ETH_ALEN);
+		if (!(*(bssid + j)))
+			goto err_out2;
+	}
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && MLME_IS_AP(iface) &&
+			rtw_iface_at_same_hwband(adapter, iface)) {
+
+			*(ch_util + i_rpts) = rtw_get_ch_utilization(iface);
+			_rtw_memcpy(*(bssid + i_rpts), iface->mac_addr, ETH_ALEN);
+			if (*(ch_util + i_rpts) > threshold)
+				need_rpt = 1;
+
+			i_rpts++;
+		}
+	}
+
+	if (need_rpt)
+		rtw_nlrtw_ch_util_rpt(adapter, i_rpts, ch_util, bssid);
+
+	rtw_mfree(ch_util, sizeof(u8) * dvobj->iface_nums);
+	for (i = 0; i < dvobj->iface_nums; i++)
+		rtw_mfree(*(bssid + i), ETH_ALEN);
+
+	rtw_mfree(bssid, sizeof(u8 *) * dvobj->iface_nums);
+
+	return;
+
+err_out2:
+	for (i = 0; i < j; i++)
+		rtw_mfree(*(bssid + i), sizeof(u8) * ETH_ALEN);
+	rtw_mfree(bssid, sizeof(sizeof(u8 *) * dvobj->iface_nums));
+err_out1:
+	rtw_mfree(ch_util, sizeof(u8) * dvobj->iface_nums);
+err_out:
+	RTW_INFO("[%s] rtw_zmalloc fail\n", __func__);
+}
+#endif
+
+
+static inline void _set_mac_addr_sw(struct _ADAPTER *padapter, const u8 *mac_addr)
+{
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta;
+	struct _ADAPTER_LINK *adapter_link;
+	u32 index, new_index;
+	u8 lidx;
+	_list *plist, *phead;
+	struct rtw_wifi_role_t *phl_role = padapter->phl_role;
+
+	/*
+	 * Dequeue sta_info related to the original mac_addr,
+	 * then enqueue it to new mac_addr hash index
+	 */
+
+	/* Workaround - phl member might be exist due to incorrect netdev_open in cfg80211_rtw_change_iface() */
+	if (phl_role) {
+		index = wifi_mac_hash(phl_role->mac_addr);
+		new_index = wifi_mac_hash(mac_addr);
+
+		_rtw_spinlock_bh(&(pstapriv->sta_hash_lock));
+		phead = &(pstapriv->sta_hash[index]);
+		plist = get_next(phead);
+
+		/* find out the one with old mac address, should be only one */
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			if (_rtw_memcmp(psta->phl_sta->mac_addr, phl_role->mac_addr, ETH_ALEN) == _TRUE)
+				break;
+
+			psta = NULL;
+			plist = get_next(plist);
+		}
+
+		if (psta) {
+			rtw_list_delete(&psta->hash_list);
+			rtw_list_insert_tail(&psta->hash_list, &(pstapriv->sta_hash[new_index]));
+		} else {
+			RTW_ERR("%s(): can't find sta_info with mac_addr "MAC_FMT"\n" , __func__, MAC_ARG(phl_role->mac_addr));
+		}
+		_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+		_rtw_memcpy(phl_role->mac_addr, mac_addr, MAC_ALEN);
+	}
+
+	_rtw_memcpy(adapter_mac_addr(padapter), mac_addr, ETH_ALEN);
+	 /* sync mac addr to adapter link */
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		adapter_link = GET_LINK(padapter, lidx);
+		_rtw_memcpy(adapter_link->mac_addr, mac_addr, ETH_ALEN);
+
+		/* offset for adapter_link mac-addr */
+		adapter_link->mac_addr[ETH_ALEN - 1] += lidx;
+
+		RTW_INFO(FUNC_ADPT_FMT": Set adapter link(id=%d) Mac Addr to "MAC_FMT" Successfully\n"
+		 , FUNC_ADPT_ARG(padapter), lidx, MAC_ARG(adapter_link->mac_addr));
+	}
+}
+
+#define SET_MAC_ADDR_HW_CMD_TIMEOUT 1000
+/**
+ * rtw_set_mac_addr - Set SW/HW MAC address
+ * @padapter: Pointer to the adapter structure
+ * @mac_addr: Pointer to the 6-byte MAC address
+ * @cfg_hw: Boolean flag to configure hardware or not
+ * @directly: Boolean flag to execute role change command directly or not
+ *
+ * This function sets the MAC address in the software structure and optionally
+ * configures the hardware with the new MAC address. If hardware configuration
+ * is requested, it attempts to change the MAC address using the PHL command
+ * interface. In case of failure, it restores the original MAC address.
+ *
+ * Return: _SUCCESS or _FAIL
+ */
+u8 rtw_set_mac_addr(_adapter *padapter, const u8 *mac_addr, bool cfg_hw, bool directly)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(padapter);
+	struct _ADAPTER_LINK *adapter_link;
+	u8 lidx;
+	_list *plist, *phead;
+	struct rtw_wifi_role_t *phl_role = padapter->phl_role;
+	void *phl = GET_PHL_INFO(d);
+	u8 orig_mac_addr[ETH_ALEN];
+	int cmd_type = (directly ? PHL_CMD_DIRECTLY : PHL_CMD_WAIT);
+	int timeout = (cmd_type == PHL_CMD_DIRECTLY ? 0 : SET_MAC_ADDR_HW_CMD_TIMEOUT);
+	enum rtw_phl_status pstatus = RTW_PHL_STATUS_SUCCESS;
+
+	/* Workaround - padapter->netif_up might be true due to incorrect netdev_open in cfg80211_rtw_change_iface() */
+	if (padapter->netif_up == _TRUE)
+		cfg_hw = true;
+
+	_rtw_memcpy(orig_mac_addr, adapter_mac_addr(padapter), ETH_ALEN);
+	_set_mac_addr_sw(padapter, mac_addr);
+
+	if (cfg_hw) {
+		for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+			adapter_link = GET_LINK(padapter, lidx);
+			pstatus = rtw_phl_cmd_wrole_change(phl,
+							   padapter->phl_role,
+							   adapter_link->wrlink,
+							   WR_CHG_MADDR,
+							   mac_addr,
+							   ETH_ALEN,
+							   cmd_type,
+							   timeout);
+
+			if (pstatus != RTW_PHL_STATUS_SUCCESS) {
+				RTW_ERR("%s() set "MAC_FMT" failed(%d), restore to origin"MAC_FMT"\n",
+					__func__,  pstatus, MAC_ARG(mac_addr), MAC_ARG(orig_mac_addr));
+				_set_mac_addr_sw(padapter, orig_mac_addr);
+				break;
+			}
+		}
+
+	}
+
+	return (pstatus ==  RTW_PHL_STATUS_SUCCESS ? _SUCCESS : _FAIL);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wnm.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wnm.c
new file mode 100644
index 000000000000..2ed72cd22874
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wnm.c
@@ -0,0 +1,1507 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+
+#ifndef RTW_WNM_DBG
+	#define RTW_WNM_DBG	0
+#endif
+#if RTW_WNM_DBG
+	#define RTW_WNM_INFO(fmt, arg...)	\
+		RTW_INFO(fmt, arg)
+	#define RTW_WNM_DUMP(str, data, len)	\
+		RTW_INFO_DUMP(str, data, len)
+#else
+	#define RTW_WNM_INFO(fmt, arg...) do {} while (0)
+	#define RTW_WNM_DUMP(str, data, len) do {} while (0)
+#endif
+
+#ifdef CONFIG_RTW_WNM
+
+static u32 wnm_defualt_validity_time = 6000;
+static u32 wnm_default_disassoc_time = 5000;
+static u32 wnm_disassoc_wait_time = 500;
+
+/* for wifi test, need more validity time to wait scan done */
+static u32 wnm_ext_validity_time = 4000;
+
+static u8 wnm_non_pref_ch_oui[] = {0x50, 0x6F, 0x9A, 0x2};
+
+#define rtw_wnm_get_non_pref_ch_oui(p) ((u8 *)(p) + 2)
+
+void rtw_wnm_candidate_info(struct wlan_network *pnetwork, struct wlan_network *cnetwork, u8 *reason)
+{
+	if (reason)
+		RTW_INFO("candidate:%s %s("MAC_FMT", ch%3u) rssi:%d dBm, age:%5d [%7s]\n",
+			 (pnetwork == cnetwork) ? "*" : " " ,
+			 pnetwork->network.Ssid.Ssid,
+			 MAC_ARG(pnetwork->network.MacAddress),
+			 pnetwork->network.Configuration.DSConfig,
+			 (int)pnetwork->network.PhyInfo.rssi,
+			 rtw_get_passing_time_ms(pnetwork->last_scanned),
+			 reason
+			);
+}
+
+static void rtw_wmn_btm_cache_update(
+	_adapter *padapter, struct btm_req_hdr *phdr)
+{
+	struct btm_rpt_cache *pcache = \
+		&(padapter->mlmepriv.nb_info.btm_cache);
+
+	pcache->dialog_token = phdr->dialog_token;
+	pcache->req_mode = phdr->req_mode;
+	pcache->disassoc_timer = le16_to_cpu(phdr->disassoc_timer);
+
+	if (phdr->validity_interval  > 0)
+		pcache->validity_interval = phdr->validity_interval;
+
+	pcache->term_duration.id = phdr->term_duration.id;
+	pcache->term_duration.len = phdr->term_duration.len;
+	pcache->term_duration.tsf = le64_to_cpu(phdr->term_duration.tsf);
+	pcache->term_duration.duration =  le16_to_cpu(phdr->term_duration.duration);
+
+	RTW_WNM_INFO("%s: req_mode(0x%02x), disassoc_timer(0x%04x), "
+		"validity_interval(0x%02x %s), tsf(0x%llx), duration(0x%02x)\n",
+		__func__, pcache->req_mode, pcache->disassoc_timer,
+		pcache->validity_interval, (!phdr->validity_interval)?
+		"default":"", pcache->term_duration.tsf,
+		pcache->term_duration.duration);
+#ifdef RTW_MBO_VALID_TIME
+	pcache->validity_time = RTW_MBO_VALID_TIME;
+#else
+
+	if (pcache->validity_interval > 0) {
+		pcache->validity_time = pcache->validity_interval * 100;
+	#ifdef CONFIG_RTW_MBO
+		if (rtw_mbo_wifi_logo_test(padapter))
+			pcache->validity_time += wnm_ext_validity_time;
+	#endif
+	}
+#endif /*RTW_MBO_VALID_TIME*/
+
+	if (pcache->disassoc_timer > 0) {
+		pcache->disassoc_time= pcache->disassoc_timer * 100;
+	#ifdef CONFIG_RTW_MBO
+		if (rtw_mbo_wifi_logo_test(padapter))
+			pcache->disassoc_time += wnm_ext_validity_time;
+	#endif
+	}
+
+	pcache->req_stime = rtw_get_current_time();
+
+	RTW_WNM_INFO("%s: validity_time=%u, disassoc_time=%u\n",
+		__func__, pcache->validity_time, pcache->disassoc_time);
+}
+
+static u8 rtw_wnm_btm_candidate_validity(
+		struct btm_rpt_cache *pcache, u8 flag)
+{
+	u8 is_validity =_TRUE;
+	u32 req_validity_time = rtw_get_passing_time_ms(pcache->req_stime);
+
+	if ((flag & BIT(0)) && (req_validity_time > pcache->validity_time))
+		is_validity = _FALSE;
+
+	if ((flag & BIT(1)) && (req_validity_time > pcache->disassoc_time))
+		is_validity = _FALSE;
+
+	RTW_WNM_INFO("%s : validity=%u, rtime=%u, vtime=%u. dtime=%u\n",
+			__func__, is_validity, req_validity_time,
+			pcache->validity_time, pcache->disassoc_time);
+	return is_validity;
+}
+
+u8 rtw_wmn_btm_rsp_reason_decision(_adapter *padapter, struct roam_nb_info *pnb)
+{
+	struct recv_info *recvinfo = &(padapter->recvinfo);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 reason = 0;
+
+	if (!rtw_wnm_btm_diff_bss(padapter, pnb)) {
+		/* Reject - No suitable BSS transition candidates */
+		reason = 7;
+		RTW_INFO("WNM : Reject - No suitable BSS transition candidates,"
+			"roam_target_addr:" MAC_FMT ", cur_network_mac:" MAC_FMT "\n",
+			MAC_ARG(padapter->mlmepriv.nb_info.roam_target_addr),
+			MAC_ARG(padapter->mlmepriv.dev_cur_network.network.MacAddress));
+		goto candidate_remove;
+	}
+
+#ifdef CONFIG_RTW_80211R
+	if (rtw_ft_chk_flags(padapter, RTW_FT_BTM_ROAM)) {
+		/* Accept */
+		reason = 0;
+		goto under_survey;
+	}
+#endif
+	if ((pnb->btm_cache.req_mode & DISASSOC_IMMINENT) == 0) {
+		/* Reject - Unspecified reject reason */
+		reason = 1;
+		goto candidate_remove;
+	}
+
+	if (recvinfo->signal_strength_data.avg_val >=
+		pmlmepriv->roam_rssi_threshold) {
+		reason = 1;
+		RTW_INFO("WNM : Reject - under high roam rssi(%u, %u) \n",
+			recvinfo->signal_strength_data.avg_val,			
+			pmlmepriv->roam_rssi_threshold);
+		goto candidate_remove;
+	}
+
+#ifdef CONFIG_RTW_80211R
+under_survey:
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY)) {
+		RTW_WNM_INFO("%s reject due to _FW_UNDER_SURVEY\n", __func__);
+		reason = 1;
+	}
+#endif
+
+candidate_remove:
+	if (reason !=0)
+		rtw_wnm_reset_btm_candidate(&pmlmepriv->nb_info);
+
+	return reason;
+}
+
+static u32 rtw_wnm_btm_candidates_offset_get(u8* pframe)
+{
+	u32 offset = 0;
+
+	if (!pframe)
+		return 0;
+
+	offset += 7;
+
+	/* BSS Termination Duration check */
+	if (wnm_btm_bss_term_inc(pframe))
+		offset += 12;
+
+	/* Session Information URL check*/
+	if (wnm_btm_ess_disassoc_im(pframe)) {
+		/*URL length field + URL variable length*/
+		offset = 1 + *(pframe + offset);
+	}
+
+	RTW_WNM_INFO("%s : hdr offset=%u\n", __func__, offset);
+	return offset;
+}
+
+static void rtw_wnm_btm_req_hdr_parsing(u8* pframe, struct btm_req_hdr *phdr)
+{
+	u8 *pos;
+	u32 offset = 0;
+
+	if (!pframe || !phdr)
+		return;
+
+	_rtw_memset(phdr, 0, sizeof(struct btm_req_hdr));
+	phdr->dialog_token = wnm_btm_dialog_token(pframe);
+	phdr->req_mode  = wnm_btm_req_mode(pframe);
+	phdr->disassoc_timer = wnm_btm_disassoc_timer(pframe);
+	phdr->validity_interval = wnm_btm_valid_interval(pframe);
+	if (wnm_btm_bss_term_inc(pframe)) {
+		pos = wnm_btm_term_duration_offset(pframe);
+		if (*pos == WNM_BTM_TERM_DUR_SUBEID) {
+			phdr->term_duration.id = *pos;
+			phdr->term_duration.len = *(pos + 1);
+			phdr->term_duration.tsf = *((u64*)(pos + 2));
+			phdr->term_duration.duration= *((u16*)(pos + 10));
+		} else
+			RTW_WNM_INFO("%s : invaild BSS Termination Duration"
+				" content!\n", __func__);
+	}
+
+	RTW_WNM_INFO("WNM: req_mode(0x%02x), disassoc_timer(0x%04x),"
+			" validity_interval(0x%02x)\n",
+			phdr->req_mode, phdr->disassoc_timer,
+			phdr->validity_interval);
+
+	if (wnm_btm_bss_term_inc(pframe)) {
+		RTW_WNM_INFO("WNM: tsf(0x%llx), duration(0x%4x)\n",
+				phdr->term_duration.tsf,
+				phdr->term_duration.duration);
+	}
+}
+
+u8 rtw_wnm_btm_reassoc_req(_adapter *padapter)
+{
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct roam_nb_info *pnb = &(pmlmepriv->nb_info);
+	u8 breassoc = _FALSE;
+
+	if (_rtw_memcmp(get_my_bssid(&(pmlmeinfo->dev_network)),
+		pnb->roam_target_addr, ETH_ALEN)) {
+		RTW_WNM_INFO("%s : bss "MAC_FMT" found in roam_target "
+			MAC_FMT"\n", __func__,
+			MAC_ARG(get_my_bssid(&(pmlmeinfo->dev_network))),
+			MAC_ARG(pnb->roam_target_addr));
+		breassoc = _TRUE;
+	}
+
+	return breassoc;
+}
+
+void rtw_wnm_roam_scan_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	if (rtw_is_scan_deny(padapter)) {
+		RTW_WNM_INFO("%s: roam scan would abort by scan_deny!\n",
+				__func__);
+	}
+
+#ifdef CONFIG_RTW_80211R
+	if (rtw_ft_chk_flags(padapter, RTW_FT_BTM_ROAM)) {
+		pmlmepriv->need_to_roam = _TRUE;
+		rtw_set_to_roam(padapter,
+			padapter->registrypriv.max_roaming_times);
+		RTW_WNM_INFO("%s : enable roaming\n", __func__);
+	}
+
+	rtw_drv_scan_by_self(padapter, RTW_AUTO_SCAN_REASON_ROAM);
+#endif
+}
+
+static void rtw_wnm_roam_scan(_adapter *padapter)
+{
+	struct roam_nb_info *pnb = &(padapter->mlmepriv.nb_info);
+
+	if (rtw_is_scan_deny(padapter)) {
+		_cancel_timer_ex(&pnb->roam_scan_timer);
+		_set_timer(&pnb->roam_scan_timer, 1000);
+	} else
+		rtw_wnm_roam_scan_hdl((void *)padapter);
+}
+
+void rtw_wnm_disassoc_chk_hdl(void *ctx)
+{
+	_adapter *padapter = (_adapter *)ctx;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct roam_nb_info *pnb = &(pmlmepriv->nb_info);
+
+	RTW_WNM_INFO("%s : expired\n", __func__);
+	if (pnb->disassoc_waiting <= 0 ) {
+		RTW_WNM_INFO("%s : btm roam is interrupted by disassoc\n",
+				__func__);
+		return;
+	}
+
+	pnb->disassoc_waiting = _FALSE;
+	rtw_wnm_roam_scan(padapter);
+}
+
+u8 rtw_wnm_try_btm_roam_imnt(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct roam_nb_info *pnb = &(pmlmepriv->nb_info);
+	struct btm_rpt_cache *pcache = &(pnb->btm_cache);
+	u8 reason = 0, flag = 0;
+
+	flag = BIT(0) | BIT(1);
+	if (!rtw_wnm_btm_candidate_validity(pcache, flag))
+		return 1;
+
+#ifdef CONFIG_RTW_MBO
+	if (!rtw_mbo_wifi_logo_test(padapter)
+		&& !(pcache->req_mode & DISASSOC_IMMINENT)) {
+		RTW_WNM_INFO("%s : non-disassoc imminet req\n",  __func__);
+		return 1;
+	}
+#endif
+
+	RTW_WNM_INFO("%s : disassoc_waiting(%d)\n",
+			__func__, pnb->disassoc_waiting);
+	if (pnb->disassoc_waiting) {
+		_cancel_timer_ex(&pnb->disassoc_chk_timer);
+		pnb->disassoc_waiting = _FALSE;
+		rtw_wnm_roam_scan_hdl((void *)padapter);
+	} else
+		RTW_WNM_INFO("%s : waiting for btm roaming start/finish\n",
+				__func__);
+
+	return reason;
+}
+
+void rtw_wnm_process_btm_req(_adapter *padapter,
+	struct sta_info *psta, u8* pframe, u32 frame_len)
+{
+	struct roam_nb_info *pnb = &(padapter->mlmepriv.nb_info);
+	struct btm_req_hdr req_hdr;
+	u8 *ptr, reason = 7;
+	u32 elem_len, offset;
+
+	rtw_wnm_btm_req_hdr_parsing(pframe, &req_hdr);
+	offset = rtw_wnm_btm_candidates_offset_get(pframe);
+	if (offset == 0)
+		return;
+
+	if (req_hdr.req_mode & PREFERRED_CANDIDATE_LIST_INCLUDED && (frame_len - offset) < 15) {
+		RTW_INFO("WNM : Reject - Candidate list included bit is set,"
+			"but no BSS transition candidates found in BTM req!\n");
+		rtw_wnm_issue_action(padapter, pnb,
+			RTW_WLAN_ACTION_WNM_BTM_RSP, 7, req_hdr.dialog_token);
+		return;
+	}
+
+	rtw_wmn_btm_cache_update(padapter, &req_hdr);
+
+	ptr = (pframe + offset);
+	elem_len = (frame_len - offset);
+
+	rtw_wnm_btm_candidates_survey(padapter, ptr, elem_len, _TRUE);
+
+#ifndef PRIVATE_R
+	/* MBO must respond to 11v BTM Requests from the AP with a Reject. */
+	if (pnb->nb_rpt_ch_list_num == 0)
+#endif /* PRIVATE_R */
+		goto done;
+
+#ifdef CONFIG_RTW_FSM_BTM
+	if (!rtw_btm_new_obj(padapter, psta, pnb, RTW_ROAM_BTM)) {
+		reason = 5; /* Reject—BSS termination delay requested */
+		goto done;
+	}
+	return;
+#else
+	rtw_wnm_btm_candidate_select(padapter, pnb);
+	reason = rtw_wmn_btm_rsp_reason_decision(padapter, pnb);
+#endif /* CONFIG_RTW_FSM_BTM */
+
+done:
+#ifdef CONFIG_RTW_MBO
+	/* for wifi-test; AP2 could power-off when BTM-req received */
+	if (((reason > 0) && (rtw_mbo_wifi_logo_test(padapter))) ||
+			(reason == 7)) {
+		_rtw_memcpy(pnb->roam_target_addr,
+			pnb->nb_rpt[0].bssid, ETH_ALEN);
+		RTW_WNM_INFO("%s : used report 0 as roam_target_addr"
+			"(reason=%u)\n", __func__, reason);
+		reason = 0;
+		pnb->preference_en = _TRUE;
+	}
+	rtw_wnm_issue_action(padapter, pnb,
+		RTW_WLAN_ACTION_WNM_BTM_RSP, reason, req_hdr.dialog_token);
+
+	if (reason == 0) {
+		pnb->disassoc_waiting = _TRUE;
+		_set_timer(&pnb->disassoc_chk_timer, wnm_disassoc_wait_time);
+	}
+#endif
+}
+
+void rtw_wnm_reset_btm_candidate(struct roam_nb_info *pnb)
+{
+	pnb->preference_en = _FALSE;
+	_rtw_memset(pnb->roam_target_addr, 0, ETH_ALEN);
+}
+
+void rtw_wnm_reset_btm_cache(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct roam_nb_info *pnb = &(pmlmepriv->nb_info);
+	struct btm_rpt_cache *pcache = &(pnb->btm_cache);
+	u8 flag = 0;
+
+	flag |= BIT(0);
+	if (rtw_wnm_btm_candidate_validity(pcache, flag))
+		return;
+
+	rtw_wnm_reset_btm_candidate(pnb);
+	_rtw_memset(pcache, 0, sizeof(struct btm_rpt_cache));
+	pcache->validity_time = wnm_defualt_validity_time;
+	pcache->disassoc_time= wnm_default_disassoc_time;
+
+#ifdef CONFIG_RTW_80211R
+	if (rtw_ft_chk_flags(padapter, RTW_FT_BTM_ROAM)) {
+		pmlmepriv->need_to_roam = _FALSE;
+		rtw_set_to_roam(padapter, 0);
+		RTW_WNM_INFO("%s : disabled roaming\n", __func__);
+	}
+#endif
+}
+
+void rtw_wnm_reset_btm_state(_adapter *padapter)
+{
+	struct roam_nb_info *pnb = &(padapter->mlmepriv.nb_info);
+
+	pnb->last_nb_rpt_entries = 0;
+	pnb->nb_rpt_is_same = _TRUE;
+	pnb->nb_rpt_valid = _FALSE;
+	pnb->nb_rpt_ch_list_num = 0;
+	pnb->disassoc_waiting = -1;
+	_rtw_memset(&pnb->nb_rpt, 0, sizeof(pnb->nb_rpt));
+	_rtw_memset(&pnb->nb_rpt_ch_list, 0, sizeof(pnb->nb_rpt_ch_list));
+	rtw_wnm_reset_btm_cache(padapter);
+}
+
+u32 rtw_wnm_btm_rsp_candidates_sz_get(
+	_adapter *padapter, u8* pframe, u32 frame_len)
+{
+	u32 num = 0, sz = 0;
+	u8 status;
+	u8 *ptr;
+
+	if (!pframe || (frame_len <= 5))
+		goto exit;
+
+	status = wnm_btm_rsp_status(pframe);
+	if (((status != 0) && (status != 6)) || (frame_len < 23))
+		goto exit;
+
+	if (status == 0)
+		num = (frame_len - 5 - ETH_ALEN)/18;
+	else
+		num = (frame_len - 5)/18;
+	sz = sizeof(struct wnm_btm_cant) * num;
+exit:
+	RTW_WNM_INFO("WNM: %u candidates(sz=%u) in BTM rsp\n", num, sz);
+	return sz;
+}
+
+void rtw_wnm_process_btm_rsp(_adapter *padapter,
+	u8* pframe, u32 frame_len, struct btm_rsp_hdr *prsp)
+{
+	if ((pframe == NULL) || (frame_len == 0))
+		return;
+
+	prsp->dialog_token = wnm_btm_dialog_token(pframe);
+	prsp->status = wnm_btm_rsp_status(pframe);
+	prsp->termination_delay = wnm_btm_rsp_term_delay(pframe);
+	prsp->status = *(pframe + 3);
+	prsp->termination_delay = *(pframe + 4);
+
+	/* no Target BSSID & Candidate in frame */
+	if (frame_len <= 5)
+		return;
+
+	/* accept */
+	if ((prsp->status == 0) && (frame_len >= 11))
+		_rtw_memcpy(prsp->bssid, (pframe + 5), ETH_ALEN);
+
+	/* STA BSS Transition Candidate List provided,
+		and at least one NB report exist */
+	if (((prsp->status == 0) || (prsp->status == 6)) \
+		&& (frame_len >= 23)) {
+		struct wnm_btm_cant cant;
+		u8 *ptr, *pend;
+		u32 idx = 0;
+
+		ptr = pframe + 5;
+		if (prsp->status == 0)
+			ptr += ETH_ALEN;
+
+		pend = ptr + frame_len;
+		prsp->candidates_num = 0;
+		while (ptr < pend) {
+			if (*ptr != RTW_WLAN_ACTION_WNM_NB_RPT_ELEM)
+				break;
+			_rtw_memset(&cant, 0, sizeof(cant));
+			cant.nb_rpt.id = *ptr;
+			cant.nb_rpt.len = *(ptr + 1);
+			_rtw_memcpy(cant.nb_rpt.bssid, (ptr + 2), ETH_ALEN);
+			cant.nb_rpt.bss_info = *((u32 *)(ptr + 8));
+			cant.nb_rpt.reg_class = *(ptr + 12);
+			cant.nb_rpt.ch_num = *(ptr + 13);
+			cant.nb_rpt.phy_type= *(ptr + 14);
+
+			if (*(ptr + 15) == WNM_BTM_CAND_PREF_SUBEID)
+				cant.preference = *(ptr + 17);
+			ptr = ptr + cant.nb_rpt.len + 2;
+			if (prsp->pcandidates) {
+				prsp->candidates_num++;
+				_rtw_memcpy((prsp->pcandidates + \
+				sizeof(cant) * idx), &cant, sizeof(cant));
+			}
+
+			idx++;
+			RTW_WNM_INFO("WNM: btm rsp candidate bssid("MAC_FMT
+				") ,bss_info(0x%04X), reg_class(0x%02X), ch(%d),"
+				" phy_type(0x%02X), preference(0x%02X)\n",
+				MAC_ARG(cant.nb_rpt.bssid), cant.nb_rpt.bss_info,
+				cant.nb_rpt.reg_class, cant.nb_rpt.ch_num,
+				cant.nb_rpt.phy_type, cant.preference);
+			if ((prsp->pcandidates) && \
+				(prsp->candidates_num > 0)) {
+				RTW_WNM_DUMP("WNM candidates: ", prsp->pcandidates,
+						(sizeof(struct wnm_btm_cant) * prsp->candidates_num));
+			}
+		}
+	}
+
+}
+
+void rtw_wnm_hdr_init(_adapter *padapter,
+	struct xmit_frame *pactionframe, u8 *pmac,
+	u8 action, u8 **pcontent)
+{
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct pkt_attrib *pattrib;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	u16 *pfctrl;
+	u8 category;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info *pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+
+	pattrib = &(pactionframe->attrib);
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+	_rtw_memset(pactionframe->buf_addr, 0,
+		(WLANHDR_OFFSET + TXDESC_OFFSET));
+
+	*pcontent = (u8 *)(pactionframe->buf_addr + TXDESC_OFFSET);
+	pwlanhdr = (struct rtw_ieee80211_hdr *)(*pcontent);
+	pfctrl = &(pwlanhdr->frame_ctl);
+	*(pfctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, pmac, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(*pcontent, WIFI_ACTION);
+
+	*pcontent += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	category = RTW_WLAN_CATEGORY_WNM;
+	*pcontent = rtw_set_fixed_ie(*pcontent, 1,
+			&(category), &(pattrib->pktlen));
+	*pcontent = rtw_set_fixed_ie(*pcontent, 1,
+			&(action), &(pattrib->pktlen));
+}
+
+void rtw_wnm_build_btm_req_ies(_adapter *padapter,
+	u8 **pframe, struct pkt_attrib *pattrib,
+	struct btm_req_hdr *phdr, u8 *purl, u32 url_len,
+	u8 *pcandidates, u8 candidate_cnt)
+{
+	int i;
+
+	*pframe = rtw_set_fixed_ie(*pframe, 1,
+				&phdr->dialog_token, &(pattrib->pktlen));
+	*pframe = rtw_set_fixed_ie(*pframe, 1,
+				&phdr->req_mode, &(pattrib->pktlen));
+	*pframe = rtw_set_fixed_ie(*pframe, 2,
+				(u8 *)&phdr->disassoc_timer, &(pattrib->pktlen));
+	*pframe = rtw_set_fixed_ie(*pframe, 1,
+				&phdr->validity_interval, &(pattrib->pktlen));
+
+	if (phdr->req_mode & BSS_TERMINATION_INCLUDED) {
+		*pframe = rtw_set_fixed_ie(*pframe, 1,
+					&phdr->term_duration.id, &(pattrib->pktlen));
+		*pframe = rtw_set_fixed_ie(*pframe, 1,
+					&phdr->term_duration.len, &(pattrib->pktlen));
+		*pframe = rtw_set_fixed_ie(*pframe, 8,
+				(u8 *)&phdr->term_duration.tsf,
+				&(pattrib->pktlen));
+		*pframe = rtw_set_fixed_ie(*pframe, 2,
+				(u8 *)&phdr->term_duration.duration,
+				&(pattrib->pktlen));
+	}
+
+	if ((purl != NULL) && (url_len > 0) &&
+		(phdr->req_mode & ESS_DISASSOC_IMMINENT)) {
+		*pframe = rtw_set_fixed_ie(*pframe, 1,
+					(u8 *)&url_len, &(pattrib->pktlen));
+		*pframe = rtw_set_fixed_ie(*pframe,
+					url_len, purl, &(pattrib->pktlen));
+	}
+
+	if ((pcandidates != NULL) && (candidate_cnt > 0)) {
+		for (i=0; i<candidate_cnt; i++) {
+			struct wnm_btm_cant *pcandidate = \
+				((struct wnm_btm_cant *)pcandidates) + i;
+			struct nb_rpt_hdr *prpt = &(pcandidate->nb_rpt);
+
+			*pframe = rtw_set_fixed_ie(*pframe, 1,
+					&pcandidate->nb_rpt.id,
+					&(pattrib->pktlen));
+			*pframe = rtw_set_fixed_ie(*pframe, 1,
+					&pcandidate->nb_rpt.len,
+					&(pattrib->pktlen));
+			*pframe = rtw_set_fixed_ie(*pframe, ETH_ALEN,
+					pcandidate->nb_rpt.bssid,
+					&(pattrib->pktlen));
+			*pframe = rtw_set_fixed_ie(*pframe, 4,
+					(u8 *)&pcandidate->nb_rpt.bss_info,
+					&(pattrib->pktlen));
+			*pframe = rtw_set_fixed_ie(*pframe, 1,
+					&pcandidate->nb_rpt.reg_class,
+					&(pattrib->pktlen));
+			*pframe = rtw_set_fixed_ie(*pframe, 1,
+					&pcandidate->nb_rpt.ch_num,
+					&(pattrib->pktlen));
+			*pframe = rtw_set_fixed_ie(*pframe, 1,
+					&pcandidate->nb_rpt.phy_type,
+					&(pattrib->pktlen));
+			*pframe = rtw_set_ie(*pframe, WNM_BTM_CAND_PREF_SUBEID,
+					1, (u8 *)&pcandidate->preference,
+					&(pattrib->pktlen));
+		}
+	}
+
+
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_build_wnm_btmreq_reason_ies(padapter, pframe, pattrib);
+#endif
+}
+
+void rtw_wnm_issue_btm_req(_adapter *padapter,
+	u8 *pmac, struct btm_req_hdr *phdr,
+	u8 *purl, u32 url_len,
+	u8 *pcandidates, u8 candidate_cnt)
+{
+	struct roam_nb_info *pnb = &(padapter->mlmepriv.nb_info);
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct xmit_frame *pmgntframe;
+	struct pkt_attrib *pattrib;
+	u8 action, *pframe, dialog_token = 0;
+
+	if (!pmac || is_zero_mac_addr(pmac)
+		|| is_broadcast_mac_addr(pmac))
+		return ;
+
+	if ((pmgntframe = alloc_mgtxmitframe(pxmitpriv)) == NULL)
+		return ;
+
+	rtw_wnm_hdr_init(padapter, pmgntframe, pmac,
+		RTW_WLAN_ACTION_WNM_BTM_REQ, &pframe);
+
+	pattrib = &(pmgntframe->attrib);
+	rtw_wnm_build_btm_req_ies(padapter, &pframe, pattrib,
+		phdr, purl, url_len, pcandidates, candidate_cnt);
+
+	if (0) {
+		u8 *__p =  (u8 *)(pmgntframe->buf_addr + TXDESC_OFFSET);
+		RTW_WNM_DUMP("WNM BTM REQ :", __p, pattrib->pktlen);
+	}
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+	/* dump_mgntframe_and_wait_ack(padapter, pmgntframe); */
+	RTW_INFO("WNM: BSS Transition Management Request sent\n");
+}
+
+void rtw_wnm_issue_action(_adapter *padapter, struct roam_nb_info *pnb,
+	u8 action, u8 reason, u8 dialog)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct xmit_frame *pmgntframe;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct pkt_attrib *pattrib;
+	u8 category, termination_delay, *pframe, dialog_token = 0;
+#ifdef CONFIG_RTW_MBO
+	u8 mbo_trans_rej_res = 1;  /* Unspecified reason */
+	u8 mbo_notif_req_type ;
+#endif
+	u16 *fctrl;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_info *pmlmeinfo = &(padapter_link->mlmeextpriv.mlmext_info);
+
+	if ((pmgntframe = alloc_mgtxmitframe(pxmitpriv)) == NULL)
+		return ;
+
+	pattrib = &(pmgntframe->attrib);
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+	_rtw_memset(pmgntframe->buf_addr, 0, (WLANHDR_OFFSET + TXDESC_OFFSET));
+
+	pframe = (u8 *)(pmgntframe->buf_addr + TXDESC_OFFSET);
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pframe, WIFI_ACTION);
+
+	pframe += sizeof(struct rtw_ieee80211_hdr_3addr);
+	pattrib->pktlen = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	category = RTW_WLAN_CATEGORY_WNM;
+	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
+	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
+
+	switch (action) {
+		case RTW_WLAN_ACTION_WNM_BTM_QUERY:
+			dialog_token++;
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(dialog_token), &(pattrib->pktlen));
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(reason), &(pattrib->pktlen));
+			RTW_INFO("WNM: BSS Transition Management Query sent\n");
+			break;
+		case RTW_WLAN_ACTION_WNM_BTM_RSP:
+			dialog_token = dialog;
+			termination_delay = 0;
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(dialog_token), &(pattrib->pktlen));
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(reason), &(pattrib->pktlen));
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(termination_delay),
+					&(pattrib->pktlen));
+			if (!reason && !is_zero_mac_addr(
+				pnb->roam_target_addr)) {
+				pframe = rtw_set_fixed_ie(pframe, 6,
+					pnb->roam_target_addr,
+					&(pattrib->pktlen));
+			}
+
+#ifdef CONFIG_RTW_MBO
+#ifdef PRIVATE_R
+
+#else
+			rtw_mbo_build_trans_reject_reason_attr(padapter,
+				&pframe, pattrib, &mbo_trans_rej_res);
+#endif /* PRIVATE_R */
+#endif
+
+			RTW_INFO("WNM: BSS Transition Management Response"
+				" sent(reason:%d)\n", reason);
+			break;
+		case RTW_WLAN_ACTION_WNM_NOTIF_REQ:
+#ifdef CONFIG_RTW_MBO
+			dialog_token++;
+			mbo_notif_req_type = WLAN_EID_VENDOR_SPECIFIC;
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(dialog_token),
+					&(pattrib->pktlen));
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(mbo_notif_req_type),
+					&(pattrib->pktlen));
+			rtw_mbo_build_wnm_notification(padapter,
+					&pframe, pattrib);
+			RTW_INFO("WNM: Notification request sent\n");
+#endif
+			break;
+		case RTW_WLAN_ACTION_WNM_NOTIF_RSP:
+#ifdef CONFIG_RTW_MBO
+			dialog_token = dialog;
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(dialog_token), &(pattrib->pktlen));
+			pframe = rtw_set_fixed_ie(pframe, 1,
+					&(reason), &(pattrib->pktlen));
+			RTW_INFO("WNM: Notification Response sent\n");
+#endif
+			break;
+		default:
+			goto exit;
+	}
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+	dump_mgntframe(padapter, pmgntframe);
+	/* dump_mgntframe_and_wait_ack(padapter, pmgntframe); */
+
+exit:
+	return;
+}
+
+/* argument req_ie@cfg80211_roamed()/cfg80211_connect_result()
+	is association request IEs format. if driver used reassoc-req format,
+	RSN IE could not be parsed @supplicant process */
+void rtw_wnm_update_reassoc_req_ie(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u32 dup_len, offset;
+	u8 *pdup;
+
+	if (!pmlmepriv->assoc_req || !pmlmepriv->assoc_req_len)
+		return;
+
+	/* total len is assoc req len without Current AP Field*/
+	dup_len = pmlmepriv->assoc_req_len - ETH_ALEN;
+
+	/* offset is a len of 80211 header +
+		capability(2B) + listen interval(2B) */
+	offset =  sizeof(struct rtw_ieee80211_hdr_3addr) + 4;
+
+	pdup = rtw_zmalloc(dup_len);
+	if (pdup) {
+		/* remove Current AP Field @reassoc req IE */
+		_rtw_memcpy(pdup, pmlmepriv->assoc_req, offset);
+		_rtw_memcpy(pdup + offset,
+				pmlmepriv->assoc_req + offset + ETH_ALEN,
+				pmlmepriv->assoc_req_len - offset - ETH_ALEN);
+		rtw_buf_update(&pmlmepriv->assoc_req,
+			&pmlmepriv->assoc_req_len, pdup, dup_len);
+		rtw_mfree(pdup, dup_len);
+	}
+}
+#endif /* CONFIG_RTW_WNM */
+
+#if defined(CONFIG_RTW_WNM) || defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+void rtw_roam_nb_info_init(_adapter *padapter)
+{
+	struct roam_nb_info *pnb = &(padapter->mlmepriv.nb_info);
+	struct btm_rpt_cache *pcache = &(pnb->btm_cache);
+
+	_rtw_memset(&pnb->nb_rpt, 0, sizeof(pnb->nb_rpt));
+	_rtw_memset(&pnb->nb_rpt_ch_list, 0, sizeof(pnb->nb_rpt_ch_list));
+	_rtw_memset(&pnb->roam_target_addr, 0, ETH_ALEN);
+	pnb->nb_rpt_valid = _FALSE;
+	pnb->nb_rpt_ch_list_num = 0;
+	pnb->preference_en = _FALSE;
+	pnb->nb_rpt_is_same = _TRUE;
+	pnb->last_nb_rpt_entries = 0;
+	pnb->disassoc_waiting = -1;
+#ifdef CONFIG_RTW_WNM
+	pnb->features = 0;
+	/* pnb->features |= RTW_WNM_FEATURE_BTM_REQ_EN; */
+
+	rtw_init_timer(&pnb->roam_scan_timer,
+		rtw_wnm_roam_scan_hdl,
+		padapter);
+	rtw_init_timer(&pnb->disassoc_chk_timer,
+		rtw_wnm_disassoc_chk_hdl,
+		padapter);
+
+	_rtw_memset(pcache, 0, sizeof(struct btm_rpt_cache));
+	pcache->validity_time = wnm_defualt_validity_time;
+	pcache->disassoc_time= wnm_default_disassoc_time ;
+#endif
+}
+
+u8 rtw_roam_nb_scan_list_set(
+	_adapter *padapter, struct sitesurvey_parm *pparm)
+{
+	u8 ret = _FALSE;
+	u32 i;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct roam_nb_info *pnb = &(pmlmepriv->nb_info);
+
+#ifdef CONFIG_RTW_80211R
+	if (!rtw_chk_roam_flags(padapter, RTW_ROAM_ACTIVE)
+		&& !rtw_ft_chk_flags(padapter, RTW_FT_BTM_ROAM))
+		return ret;
+#endif
+
+	if (!pmlmepriv->need_to_roam)
+		return ret;
+
+	if ((!pmlmepriv->nb_info.nb_rpt_valid) || (!pnb->nb_rpt_ch_list_num))
+		return ret;
+
+	if (!pparm)
+		return ret;
+
+	if (rtw_roam_busy_scan(padapter, pnb)) {
+		pparm->ch_num = 1;
+		pparm->ch[pmlmepriv->ch_cnt].hw_value =
+			pnb->nb_rpt_ch_list[pmlmepriv->ch_cnt].hw_value;
+		pparm->ch[pmlmepriv->ch_cnt].band =
+			pnb->nb_rpt_ch_list[pmlmepriv->ch_cnt].band;
+		ret = _TRUE;
+
+		RTW_WNM_INFO("%s: ch_cnt=%u, (%u)hw_value=%u\n",
+			__func__, pparm->ch_num, pmlmepriv->ch_cnt,
+			pparm->ch[pmlmepriv->ch_cnt].hw_value);
+
+		pmlmepriv->ch_cnt++;
+
+		if (pmlmepriv->ch_cnt == pnb->nb_rpt_ch_list_num) {
+			pmlmepriv->nb_info.nb_rpt_valid = _FALSE;
+			pmlmepriv->ch_cnt = 0;
+		}
+		goto set_bssid_list;
+	}
+
+	pparm->ch_num = (pnb->nb_rpt_ch_list_num > RTW_MAX_NB_RPT_NUM)?
+		(RTW_MAX_NB_RPT_NUM):(pnb->nb_rpt_ch_list_num);
+	for (i=0; i<pparm->ch_num; i++) {
+		pparm->ch[i].hw_value = pnb->nb_rpt_ch_list[i].hw_value;
+		pparm->ch[i].flags = RTW_IEEE80211_CHAN_PASSIVE_SCAN;
+		pparm->ch[i].band = pnb->nb_rpt_ch_list[i].band;
+	}
+
+	pmlmepriv->nb_info.nb_rpt_valid = _FALSE;
+	pmlmepriv->ch_cnt = 0;
+	ret = _TRUE;
+
+set_bssid_list:
+	return ret;
+}
+
+static u8 rtw_wnm_nb_elem_parsing(
+	u8* pdata, u32 data_len, u8 from_btm,
+	u32 *nb_rpt_num, u8 *nb_rpt_is_same,
+	struct roam_nb_info *pnb, struct wnm_btm_cant *pcandidates)
+{
+	u8 bfound = _FALSE, ret = _SUCCESS;
+	u8 *ptr, *pend, *op;
+	u32 elem_len, subelem_len, op_len;
+	u32 i, nb_rpt_entries = 0;
+	struct nb_rpt_hdr *pie;
+	struct wnm_btm_cant *pcandidate;
+
+	if ((!pdata) || (!pnb))
+		return _FAIL;
+
+	if ((from_btm) && (!pcandidates))
+		return _FAIL;
+
+	ptr = pdata;
+	pend = ptr + data_len;
+	elem_len = data_len;
+	subelem_len = (u32)*(pdata+1);
+
+	for (i=0; i < RTW_MAX_NB_RPT_NUM; i++) {
+		if (((ptr + 7) > pend) || (elem_len < subelem_len))
+			break;
+
+		if (*ptr != RTW_WLAN_ACTION_WNM_NB_RPT_ELEM) {
+			RTW_WNM_INFO("WNM: end of data(0x%2x)!\n", *ptr);
+			break;
+		}
+
+		pie = (struct nb_rpt_hdr *)ptr;
+		if (from_btm) {
+			op = rtw_get_ie((u8 *)(ptr+15),
+				WNM_BTM_CAND_PREF_SUBEID,
+				&op_len, (subelem_len - 15));
+		}
+
+		ptr = (u8 *)(ptr + subelem_len + 2);
+		elem_len -= (subelem_len +2);
+		subelem_len = *(ptr+1);
+		if (from_btm) {
+			pcandidate = (pcandidates + i);
+			_rtw_memcpy(&pcandidate->nb_rpt, pie,
+				sizeof(struct nb_rpt_hdr));
+			if (op && (op_len !=0)) {
+				pcandidate->preference = *(op + 2);
+				bfound = _TRUE;
+			} else
+				pcandidate->preference = 0;
+
+			RTW_WNM_INFO("WNM: preference check bssid("MAC_FMT
+				") ,bss_info(0x%04X), reg_class(0x%02X), ch(%d),"
+				" phy_type(0x%02X), preference(0x%02X)\n",
+				MAC_ARG(pcandidate->nb_rpt.bssid),
+				pcandidate->nb_rpt.bss_info,
+				pcandidate->nb_rpt.reg_class,
+				pcandidate->nb_rpt.ch_num,
+				pcandidate->nb_rpt.phy_type,
+				pcandidate->preference);
+		} else {
+			if (_rtw_memcmp(&pnb->nb_rpt[i], pie,
+				sizeof(struct nb_rpt_hdr)) == _FALSE)
+				*nb_rpt_is_same = _FALSE;
+			_rtw_memcpy(&pnb->nb_rpt[i], pie,
+				sizeof(struct nb_rpt_hdr));
+		}
+		nb_rpt_entries++;
+	}
+
+	if (from_btm)
+		pnb->preference_en = (bfound)?_TRUE:_FALSE;
+
+	*nb_rpt_num = nb_rpt_entries;
+	return ret;
+}
+
+/* selection sorting based on preference value
+ * IN : 		nb_rpt_entries - candidate num
+ * IN/OUT :	pcandidates	- candidate list
+ * return : TRUE - means pcandidates is updated.
+ */
+static u8 rtw_wnm_candidates_sorting(
+	u32 nb_rpt_entries, struct wnm_btm_cant *pcandidates)
+{
+	u8 updated = _FALSE;
+	u32 i, j, pos;
+	struct wnm_btm_cant swap;
+	struct wnm_btm_cant *pcant_1, *pcant_2;
+
+	if ((!nb_rpt_entries) || (!pcandidates))
+		return updated;
+
+	for (i=0; i < (nb_rpt_entries - 1); i++) {
+		pos = i;
+		for (j=(i + 1); j < nb_rpt_entries; j++) {
+			pcant_1 = pcandidates+pos;
+			pcant_2 = pcandidates+j;
+			if ((pcant_1->preference) < (pcant_2->preference))
+				pos = j;
+		}
+
+		if (pos != i) {
+			updated = _TRUE;
+			_rtw_memcpy(&swap, (pcandidates+i),
+					sizeof(struct wnm_btm_cant));
+			_rtw_memcpy((pcandidates+i), (pcandidates+pos),
+					sizeof(struct wnm_btm_cant));
+			_rtw_memcpy((pcandidates+pos), &swap,
+					sizeof(struct wnm_btm_cant));
+		}
+	}
+	return updated;
+}
+
+static void rtw_wnm_nb_info_update(
+	u32 nb_rpt_entries, u8 from_btm,
+	struct roam_nb_info *pnb,
+	struct wnm_btm_cant *pcandidates,
+	u8 *nb_rpt_is_same)
+{
+	u8 is_found;
+	u32 i, j;
+	struct wnm_btm_cant *pcand;
+
+	if (!pnb)
+		return;
+
+	pnb->nb_rpt_ch_list_num = 0;
+	for (i=0; i<nb_rpt_entries; i++) {
+		is_found = _FALSE;
+		if (from_btm) {
+			pcand = (pcandidates+i);
+			if (_rtw_memcmp(&pnb->nb_rpt[i], &pcand->nb_rpt,
+					sizeof(struct nb_rpt_hdr)) == _FALSE)
+				*nb_rpt_is_same = _FALSE;
+			_rtw_memcpy(&pnb->nb_rpt[i], &pcand->nb_rpt,
+					sizeof(struct nb_rpt_hdr));
+		}
+
+		RTW_WNM_INFO("WNM: bssid(" MAC_FMT
+			") , bss_info(0x%04X), reg_class(0x%02X),"
+			" ch_num(%d), phy_type(0x%02X)\n",
+			MAC_ARG(pnb->nb_rpt[i].bssid), pnb->nb_rpt[i].bss_info,
+			pnb->nb_rpt[i].reg_class, pnb->nb_rpt[i].ch_num,
+			pnb->nb_rpt[i].phy_type);
+
+		if (pnb->nb_rpt[i].ch_num == 0)
+			continue;
+
+		for (j=0; j<nb_rpt_entries; j++) {
+			if (pnb->nb_rpt[i].ch_num ==
+				pnb->nb_rpt_ch_list[j].hw_value) {
+				is_found = _TRUE;
+				break;
+			}
+		}
+
+		if (!is_found) {
+			pnb->nb_rpt_ch_list[pnb->nb_rpt_ch_list_num].hw_value =\
+				pnb->nb_rpt[i].ch_num;
+			pnb->nb_rpt_ch_list[pnb->nb_rpt_ch_list_num].band =
+				rtw_get_band_by_op_class(pnb->nb_rpt[i].reg_class);
+			/* error handling */
+			if (pnb->nb_rpt_ch_list[pnb->nb_rpt_ch_list_num].band == BAND_MAX)
+				pnb->nb_rpt_ch_list[pnb->nb_rpt_ch_list_num].band = BAND_ON_24G;
+
+			pnb->nb_rpt_ch_list_num++;
+		}
+	}
+}
+
+struct wlan_network * rtw_wnm_btm_candidate_check(_adapter *padapter,
+	struct roam_nb_info *pnb, struct wlan_network *pnetwork)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network *cnetwork = pmlmepriv->cur_network_scanned;
+	int i, preference = 0;
+	s8 rssi;
+
+	for (i = 0; i < pnb->last_nb_rpt_entries; i++) {
+		if ((_rtw_memcmp(pnb->nb_rpt[i].bssid,\
+			padapter->mlmepriv.dev_cur_network.network.MacAddress, ETH_ALEN))) {
+			/* pnb has self entry */
+			preference = 1;
+			continue;
+		}
+
+		if (pnetwork == cnetwork) {
+			rtw_wnm_candidate_info(pnetwork, cnetwork, "self");
+			continue;
+		}
+
+#ifndef DBG_WNM_UNIT_TEST
+		/* check RSSI */
+		if (!rtw_mbo_wifi_logo_test(padapter)) {
+			rssi = pnetwork->network.PhyInfo.rssi - cnetwork->network.PhyInfo.rssi;
+			if (rssi < pmlmepriv->roam_rssi_diff_th) {
+				rtw_wnm_candidate_info(pnetwork, cnetwork, "delta");
+				continue;
+			}
+		}
+#endif
+
+		if (_rtw_memcmp(pnb->nb_rpt[i].bssid,
+			pnetwork->network.MacAddress, ETH_ALEN) == _TRUE) {
+			_rtw_memcpy(pnb->roam_target_addr, pnb->nb_rpt[i].bssid, ETH_ALEN);
+
+			/* Only accept 1'st entry(highest preference) */
+			if (pnb->preference_en && i != preference) {
+				rtw_wnm_candidate_info(pnetwork, cnetwork, "!prefer");
+				break;
+			}
+			rtw_wnm_candidate_info(pnetwork, cnetwork, "ok");
+			return pnetwork;
+		}
+	}
+	return NULL;
+}
+
+struct wlan_network * rtw_wnm_btm_candidate_select(_adapter *padapter, struct roam_nb_info *pnb)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct wlan_network *pnetwork = NULL;
+	u8 bfound = _FALSE;
+	u8 ignore_currrent = _FALSE;
+	u32 i;
+
+#ifdef CONFIG_RTW_80211R
+	if (rtw_ft_chk_flags(padapter, RTW_FT_BTM_ROAM))
+		ignore_currrent = _TRUE;
+#endif
+
+	for (i = 0; i < pnb->last_nb_rpt_entries; i++) {
+		if (ignore_currrent &&
+			(_rtw_memcmp(pnb->nb_rpt[i].bssid,\
+			padapter->mlmepriv.dev_cur_network.network.MacAddress,
+			ETH_ALEN))) {
+			RTW_WNM_INFO("WNM: ignore candidate "MAC_FMT
+				" for it's connected(%u)!\n",
+					MAC_ARG(pnb->nb_rpt[i].bssid), i);
+			continue;
+		}
+
+		pnetwork = rtw_find_network(
+				&(pmlmepriv->scanned_queue),
+				pnb->nb_rpt[i].bssid);
+
+		if (pnetwork) {
+			bfound = _TRUE;
+			break;
+		}
+	}
+
+	if (bfound) {
+		_rtw_memcpy(pnb->roam_target_addr,
+			pnb->nb_rpt[i].bssid, ETH_ALEN);
+		RTW_INFO("WNM: select btm entry(%d) -"
+			" %s("MAC_FMT", ch:%u) rssi:%d\n"
+			, i
+			, pnetwork->network.Ssid.Ssid
+			, MAC_ARG(pnetwork->network.MacAddress)
+			, pnetwork->network.Configuration.DSConfig
+			, (int)pnetwork->network.PhyInfo.rssi);
+	} else
+		_rtw_memset(pnb->roam_target_addr,0, ETH_ALEN);
+
+	return pnetwork;
+}
+
+u32 rtw_wnm_btm_candidates_survey(
+	_adapter *padapter, u8* pframe, u32 elem_len, u8 from_btm)
+{
+	struct roam_nb_info *pnb = &(padapter->mlmepriv.nb_info);
+	struct wnm_btm_cant *pcandidate_list = NULL;
+	u8 nb_rpt_is_same = _TRUE;
+	u32	ret = _FAIL;
+	u32 nb_rpt_entries = 0;
+
+	if (from_btm) {
+		u32 mlen = sizeof(struct wnm_btm_cant) * RTW_MAX_NB_RPT_NUM;
+		pcandidate_list = (struct wnm_btm_cant *)rtw_malloc(mlen);
+		if (pcandidate_list == NULL)
+			goto exit;
+	}
+
+	/*clean the status set last time*/
+	_rtw_memset(&pnb->nb_rpt_ch_list, 0, sizeof(pnb->nb_rpt_ch_list));
+	pnb->nb_rpt_valid = _FALSE;
+	if (!rtw_wnm_nb_elem_parsing(
+			pframe, elem_len, from_btm,
+			&nb_rpt_entries, &nb_rpt_is_same,
+			pnb, pcandidate_list))
+		goto exit;
+
+	if (nb_rpt_entries != 0) {
+		if ((from_btm) && (rtw_wnm_btm_preference_cap(pnb))) {
+			rtw_wnm_candidates_sorting(
+				nb_rpt_entries, pcandidate_list);
+		}
+
+		rtw_wnm_nb_info_update(
+			nb_rpt_entries, from_btm,
+			pnb, pcandidate_list, &nb_rpt_is_same);
+	}
+
+	RTW_WNM_INFO("nb_rpt_is_same = %d, nb_rpt_entries = %d,"
+		" last_nb_rpt_entries = %d\n",
+		nb_rpt_is_same, nb_rpt_entries,
+		pnb->last_nb_rpt_entries);
+	if ((nb_rpt_is_same == _TRUE) &&
+		(nb_rpt_entries == pnb->last_nb_rpt_entries)) {
+		pnb->nb_rpt_is_same = _TRUE;
+	} else {
+		pnb->nb_rpt_is_same = _FALSE;
+		pnb->last_nb_rpt_entries = nb_rpt_entries;
+	}
+	pnb->nb_rpt_valid = _TRUE;
+	ret = _SUCCESS;
+
+exit:
+	if (from_btm && pcandidate_list) {
+		rtw_mfree((u8 *)pcandidate_list,
+			sizeof(struct wnm_btm_cant) * RTW_MAX_NB_RPT_NUM);
+	}
+
+	return ret;
+}
+
+void rtw_wnm_process_btm_query(_adapter *padapter, u8* pframe, u32 frame_len)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct nb_rpt_hdr *pie;
+	u8 *ptr, *pend, *op;
+	u32 elem_len, subelem_len, op_len;
+	u32 i, nb_rpt_entries = 0;
+	if (!pframe || !frame_len)
+		return;
+
+	/* no preferred bss transition candidate list include */
+	if (wnm_btm_query_reason(pframe) != 0x13)
+		return;
+
+	ptr = pframe + 4;
+	elem_len = frame_len - 4;
+	pend = ptr + elem_len;
+	subelem_len = (u32)*(ptr + 1);
+
+	RTW_WNM_DUMP("BTM QUERY :", ptr, elem_len);
+	for (i = 0; i < RTW_MAX_NB_RPT_NUM; i++) {
+		if (((ptr + 7) > pend) || (elem_len < subelem_len))
+			break;
+		if (*ptr != RTW_WLAN_ACTION_WNM_NB_RPT_ELEM)
+			break;
+		pie = (struct nb_rpt_hdr *)ptr;
+		op = rtw_get_ie((u8 *)(ptr+15), WNM_BTM_CAND_PREF_SUBEID,
+				&op_len, (subelem_len - 15));
+
+#ifdef CONFIG_RTW_MBO
+/* for debug only */
+#if 0
+		if (rtw_mbo_wifi_logo_test(padapter) &&
+			check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE) {
+			struct mbo_attr_info *pmbo_attr = \
+					&(pmlmepriv->mbo_attr);
+			 struct mbo_user_btm_req_pkt *puser = \
+					&(pmbo_attr->user_raw);
+			struct wnm_btm_cant *puser_cant;
+			u8 j = 0, idx = 0, found = _FALSE;
+
+			for (j = 0; j < RTW_MAX_NB_RPT_NUM; j++) {
+				puser_cant = &puser->btm_cants[j];
+				if (_rtw_memcmp(pie->bssid,
+					puser_cant->nb_rpt.bssid, ETH_ALEN)) {
+					puser_cant->nb_rpt.bss_info = \
+						pie->bss_info;
+					puser_cant->nb_rpt.reg_class = \
+						pie->reg_class;
+					puser_cant->nb_rpt.ch_num = \
+						pie->ch_num;
+					puser_cant->nb_rpt.phy_type = \
+						pie->phy_type;
+					puser_cant->preference = \
+						*(op + 2);
+					idx = j;
+					found = _TRUE;
+					break;
+				}
+			}
+
+			if (!found) {
+				if (puser->candidate_cnt >= RTW_MAX_NB_RPT_NUM)
+					puser->candidate_cnt = 0;
+				puser_cant = \
+					&puser->btm_cants[puser->candidate_cnt];
+				puser_cant->nb_rpt.id = \
+					RTW_WLAN_ACTION_WNM_NB_RPT_ELEM;
+				puser_cant->nb_rpt.len = 0x10;
+				 _rtw_memcpy(puser_cant->nb_rpt.bssid,
+					pie->bssid, ETH_ALEN);
+				puser_cant->nb_rpt.bss_info = \
+						pie->bss_info;
+				puser_cant->nb_rpt.reg_class = \
+						pie->reg_class;
+				puser_cant->nb_rpt.ch_num = \
+						pie->ch_num;
+				puser_cant->nb_rpt.phy_type = \
+						pie->phy_type;
+				puser_cant->preference = \
+						*(op + 2);
+				puser->candidate_cnt++;
+			}
+
+			RTW_WNM_INFO("%s: idx=%u, bssid("MAC_FMT"),"
+				" bss_info(0x%04X), reg_class(0x%02X),"
+				" ch(%d), phy_type(0x%02X), preference(0x%02X)\n",
+				__func__, (found)?idx:puser->candidate_cnt,
+				MAC_ARG(puser_cant->nb_rpt.bssid),
+				puser_cant->nb_rpt.bss_info,
+				puser_cant->nb_rpt.reg_class,
+				puser_cant->nb_rpt.ch_num,
+				puser_cant->nb_rpt.phy_type,
+				puser_cant->preference);
+		}
+#endif
+#endif /* end of CONFIG_RTW_MBO */
+
+		ptr = (u8 *)(ptr + subelem_len + 2);
+		elem_len -= (subelem_len +2);
+		subelem_len = *(ptr+1);
+	} /* end of for-loop RTW_MAX_NB_RPT_NUM */
+
+}
+
+static u8 *rtw_wnm_non_pref_ch_attr_get(u8 *pie, u32 *plen, u32 limit)
+{
+	const u8 *p = pie;
+	u32 tmp, i;
+
+	if (limit <= 1)
+		return NULL;
+
+	i = 0;
+	*plen = 0;
+	while (1) {
+		if ((*p == _VENDOR_SPECIFIC_IE_) &&
+			(_rtw_memcmp(rtw_wnm_get_non_pref_ch_oui(p),
+			wnm_non_pref_ch_oui, 4))) {
+			*plen = *(p + 1);
+			return (u8 *)p;
+		} else {
+			tmp = *(p + 1);
+			p += (tmp + 2);
+			i += (tmp + 2);
+		}
+
+		if (i >= limit)
+			break;
+	}
+
+	return NULL;
+}
+
+void rtw_wnm_process_notification_req(
+	_adapter *padapter, u8 *pframe, u32 frame_len)
+{
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+#ifdef CONFIG_RTW_MBO
+	struct mbo_attr_info *pmbo_attr = &(pmlmepriv->mbo_attr);
+	struct mbo_user_btm_req_pkt *puser_raw = &(pmbo_attr->user_raw);
+#endif
+	struct wnm_btm_cant *pcant = NULL;
+	u8 dialog, type;
+	u8 *psubie;
+	u8 *ptr = NULL;
+	u32 left_len, subie_len;
+
+	if (!pframe || (frame_len < 5))
+		return;
+
+	/* RTW_WNM_DUMP("notify-req :", pframe, frame_len); */
+	dialog = *(pframe + 2);
+	type = *(pframe + 3);
+
+	if (type != WLAN_EID_VENDOR_SPECIFIC)
+		return;
+
+	ptr = (pframe + 4);
+	left_len = (frame_len - 4);
+
+	while((psubie = rtw_wnm_non_pref_ch_attr_get(
+		ptr, &subie_len, left_len)) != NULL) {
+
+		/* RTW_WNM_DUMP("notify-req subie :", (psubie + 2), subie_len); */
+
+		ptr += (subie_len + 2);
+		left_len -= (subie_len + 2);
+
+#ifdef CONFIG_RTW_MBO
+		if (rtw_mbo_wifi_spec_test(padapter) && \
+			(subie_len > 7)) {
+
+			u8 *pload = NULL, *pclass = NULL;
+			u8 ch_nums = 0;
+			u8 non_pref_ch = 0;
+			u8 ch_op_pref = 1, reason = 0;
+			u32 i, j;
+
+			/* nums = sublen - oui_len -
+				(class_len - op_len - reason_len) */
+			ch_nums = subie_len - 4 - 3;
+			if (ch_nums >= RTW_MBO_MAX_CH_LIST_NUM)
+				ch_nums = RTW_MBO_MAX_CH_LIST_NUM;
+
+			/* skip sudid, len, oui */
+			pclass = (psubie + 6);
+			ch_op_pref = *(pclass + ch_nums + 1);
+			reason = *(pclass + ch_nums + 2);
+			RTW_WNM_INFO("%s : class=0x%02x, operating=0x%02x, "
+				"reason=0x%02x\n", __func__, *pclass,
+				ch_op_pref, reason);
+
+			for (i = 0; i < ch_nums; i++) {
+				non_pref_ch = *(pclass + 1 + i);
+				RTW_WNM_INFO("%s : non-pref ch %u "
+					"found in notify-req operating 0x%02x\n",
+					__func__, non_pref_ch, ch_op_pref);
+
+				for (j = 0; j < RTW_MAX_NB_RPT_NUM; j++) {
+					pcant = &puser_raw->btm_cants[j];
+					if (pcant->nb_rpt.ch_num == non_pref_ch) {
+						if (ch_op_pref == 0) {
+							RTW_WNM_INFO("%s : reset "
+							"preference(%u) for non-preference ch\n",
+							__func__, pcant->preference);
+							pcant->preference = 0;
+						} else {
+							if (!pcant->preference)
+								pcant->preference = 64;
+						}
+					}
+
+				} /* end of for loop RTW_MAX_NB_RPT_NUM */
+
+			} /* end of for loop ch_nums*/
+
+		}
+#endif /* end of CONFIG_RTW_MB*/
+
+	} /* end of while loop */
+
+	rtw_wnm_issue_action(padapter, &pmlmepriv->nb_info,
+		RTW_WLAN_ACTION_WNM_NOTIF_RSP, 0, dialog);
+
+}
+#endif /*defined(CONFIG_RTW_WNM) || defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wow.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wow.c
new file mode 100644
index 000000000000..6a4db703b76f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_wow.c
@@ -0,0 +1,1300 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_WOW_C_
+
+#include <drv_types.h>
+
+#if defined(CONFIG_WOWLAN) || defined(CONFIG_AP_WOWLAN)
+	#include <linux/inetdevice.h>
+	#define ETH_TYPE_OFFSET	12
+	#define PROTOCOL_OFFSET	23
+	#define IP_OFFSET	30
+	#define IPv6_OFFSET	38
+	#define IPv6_PROTOCOL_OFFSET	20
+#endif
+
+#ifdef CONFIG_WOWLAN
+
+void rtw_init_wow(_adapter *padapter)
+{
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+
+#ifdef CONFIG_GPIO_WAKEUP
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	void *phl = GET_PHL_INFO(dvobj);
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	struct rtw_wow_gpio_info *wow_gpio = &wowpriv->wow_gpio;
+	struct rtw_dev2hst_gpio_info *d2h_gpio_info = &wow_gpio->d2h_gpio_info;
+	u8 toggle_pulse = CONFIG_TOGGLE_PULSE, gpio_time_unit = 1, gpio_pulse_count = CONFIG_PULSE_COUNT;
+	u8 gpio_pulse_period = 20, gpio_pulse_dura = 10;
+	u8 rsn_a_en = 0, rsn_a = 0, rsn_a_time_unit = 0, rsn_a_toggle_pulse = DEV2HST_TOGGLE;
+	u8 rsn_a_pulse_count = 0, rsn_a_pulse_period = 0, rsn_a_pulse_duration = 0;
+#endif
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+	struct rtw_periodic_wake_info *wow_periodic_wake = &wowpriv->wow_periodic_wake;
+#endif
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) || defined(CONFIG_ANDROID_POWER)
+	pwrctrlpriv->early_suspend.suspend = NULL;
+	rtw_register_early_suspend(pwrctrlpriv);
+#endif /* CONFIG_HAS_EARLYSUSPEND || CONFIG_ANDROID_POWER */
+	pwrctrlpriv->wowlan_mode = _FALSE;
+	pwrctrlpriv->wowlan_ap_mode = _FALSE;
+	pwrctrlpriv->wowlan_p2p_mode = _FALSE;
+	pwrctrlpriv->wowlan_in_resume = _FALSE;
+
+	wowpriv->wow_wake_reason = 0;
+
+#ifdef CONFIG_GPIO_WAKEUP
+#ifdef PRIVATE_R
+	rsn_a_en = 1;
+	toggle_pulse = DEV2HST_PULSE;
+	gpio_pulse_count = 1;
+	gpio_pulse_dura = 20;
+	gpio_pulse_period = 40;
+	rsn_a_toggle_pulse = DEV2HST_PULSE;
+	rsn_a_time_unit = 1;
+	rsn_a_pulse_count = 1;
+	rsn_a_pulse_duration = 60;
+	rsn_a_pulse_period = 120;
+	rsn_a = 0x21;
+#endif /* PRIVATE_R */
+	pwrctrlpriv->hst2dev_high_active = HIGH_ACTIVE_HST2DEV;
+	/*default low active*/
+	d2h_gpio_info->gpio_active = HIGH_ACTIVE_DEV2HST;
+
+	d2h_gpio_info->toggle_pulse = toggle_pulse;
+	d2h_gpio_info->gpio_time_unit = gpio_time_unit;
+	d2h_gpio_info->gpio_pulse_dura = gpio_pulse_dura;
+	d2h_gpio_info->gpio_pulse_period = gpio_pulse_period;
+	d2h_gpio_info->gpio_pulse_count = gpio_pulse_count;
+
+	wow_gpio->dev2hst_gpio = WAKEUP_GPIO_IDX;
+	d2h_gpio_info->rsn_a_en = rsn_a_en;
+	d2h_gpio_info->rsn_a_toggle_pulse = rsn_a_toggle_pulse;
+	d2h_gpio_info->rsn_a_time_unit = rsn_a_time_unit;
+	d2h_gpio_info->rsn_a = rsn_a;
+	d2h_gpio_info->rsn_a_pulse_duration = rsn_a_pulse_duration;
+	d2h_gpio_info->rsn_a_pulse_period = rsn_a_pulse_period;
+	d2h_gpio_info->rsn_a_pulse_count = rsn_a_pulse_count;
+
+#ifdef CONFIG_RTW_ONE_PIN_GPIO
+	wow_gpio->dev2hst_gpio_mode = RTW_AX_SW_IO_MODE_INPUT;
+	status = rtw_phl_cfg_wow_set_sw_gpio_mode(phl, wow_gpio);
+#else
+	#ifdef CONFIG_WAKEUP_GPIO_INPUT_MODE
+	wow_gpio->dev2hst_gpio_mode = RTW_AX_SW_IO_MODE_OUTPUT_OD;
+	#else
+	wow_gpio->dev2hst_gpio_mode = RTW_AX_SW_IO_MODE_OUTPUT_PP;
+	#endif /*CONFIG_WAKEUP_GPIO_INPUT_MODE*/
+	/* switch GPIO to open-drain or push-pull */
+	status = rtw_phl_cfg_wow_set_sw_gpio_mode(phl, wow_gpio);
+	wow_gpio->dev2hst_high = d2h_gpio_info->gpio_active == 0 ? 1 : 0;
+	status = rtw_phl_cfg_wow_sw_gpio_ctrl(phl, wow_gpio);
+	RTW_INFO("%s: set GPIO_%d %d as default. status=%d\n",
+		 __func__, WAKEUP_GPIO_IDX, wow_gpio->dev2hst_high, status);
+#endif /* CONFIG_RTW_ONE_PIN_GPIO */
+#endif /* CONFIG_GPIO_WAKEUP */
+
+#ifdef CONFIG_WOWLAN
+	if (!(registry_par->wakeup_event & BIT(3)))
+		rtw_wow_pattern_clean(padapter, RTW_CUSTOMIZED_PATTERN);
+
+	rtw_wow_pattern_clean(padapter, RTW_DEFAULT_PATTERN);
+
+	_rtw_mutex_init(&pwrctrlpriv->wowlan_pattern_cam_mutex);
+
+	pwrctrlpriv->wowlan_aoac_rpt_loc = 0;
+#ifdef CONFIG_MDNS_OFFLOAD
+	_rtw_memset(&wowpriv->mdns_ofld_info, 0,
+			sizeof(struct rtw_mdns_ofld_info));
+#endif
+#ifdef CONFIG_WOW_APF
+	wowpriv->apf_info.apf_en = 0;
+	wowpriv->apf_info.apf_prog_num = 0;
+	wowpriv->apf_info.apf_prog_total_size = 0;
+	_rtw_memset(wowpriv->apf_info.apf_prog, 0, MAX_APF_PROG_SIZE);
+#endif
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+	wow_periodic_wake->wake_period = WOW_DEFAULT_WAKE_PERIOD;
+	wow_periodic_wake->wake_duration = WOW_DEFAULT_WAKE_DURATION;
+#endif
+}
+
+void rtw_free_wow(_adapter *adapter)
+{
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(adapter);
+#ifdef CONFIG_WOW_APF
+	struct wow_priv *wowpriv = adapter_to_wowlan(adapter);
+	struct rtw_apf_info *apf_info = &wowpriv->apf_info;
+#endif
+
+	_rtw_mutex_free(&pwrctrlpriv->wowlan_pattern_cam_mutex);
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) || defined(CONFIG_ANDROID_POWER)
+	rtw_unregister_early_suspend(pwrctrlpriv);
+#endif /* CONFIG_HAS_EARLYSUSPEND || CONFIG_ANDROID_POWER */
+#ifdef CONFIG_WOW_APF
+	RTW_INFO("%s() clear apf prog related resource\n", __func__);
+	RTW_INFO("apf prog num(%u)\n", apf_info->apf_prog_num);
+
+	apf_info->apf_prog_len = 0;
+	apf_info->apf_data_len = 0;
+	apf_info->apf_en = 0;
+	apf_info->apf_prog_num = 0;
+	apf_info->apf_prog_total_size = 0;
+#endif /* CONFIG_WOW_APF */
+}
+
+void rtw_wowlan_set_pattern_cast_type(_adapter *adapter, struct rtw_wowcam_upd_info *wowcam_info)
+{
+	if (is_broadcast_mac_addr(wowcam_info->ptrn))
+		wowcam_info->bc = 1;
+	else if (is_multicast_mac_addr(wowcam_info->ptrn))
+		wowcam_info->mc = 1;
+	else if (!memcmp(wowcam_info->ptrn, adapter_mac_addr(adapter), ETH_ALEN))
+		wowcam_info->uc = 1;
+}
+
+bool rtw_wowlan_parser_pattern_cmd(u8 *input, char *pattern,
+				   int *pattern_len, char *bit_mask)
+{
+	char *cp = NULL;
+	size_t len = 0;
+	int pos = 0, mask_pos = 0, res = 0;
+	u8 member[2] = {0};
+
+	/* To get the pattern string after "=", when we use :
+	 * iwpriv wlanX pattern=XX:XX:..:XX
+	 */
+	cp = strchr(input, '=');
+	if (cp) {
+		*cp = 0;
+		cp++;
+		input = cp;
+	}
+
+	/* To take off the newline character '\n'(0x0a) at the end of pattern string,
+	 * when we use echo xxxx > /proc/xxxx
+	 */
+	cp = strchr(input, '\n');
+	if (cp)
+		*cp = 0;
+
+	while (input) {
+		cp = strsep((char **)(&input), ":");
+
+		if (bit_mask && (strcmp(cp, "-") == 0 ||
+				 strcmp(cp, "xx") == 0 ||
+				 strcmp(cp, "--") == 0)) {
+			/* skip this byte and leave mask bit unset */
+		} else {
+			u8 hex;
+
+			if (strlen(cp) != 2) {
+				RTW_ERR("%s:[ERROR] hex len != 2, input=[%s]\n",
+					__func__, cp);
+				goto error;
+			}
+
+			if (hexstr2bin(cp, &hex, 1) < 0) {
+				RTW_ERR("%s:[ERROR] pattern is invalid, input=[%s]\n",
+					__func__, cp);
+				goto error;
+			}
+
+			pattern[pos] = hex;
+			mask_pos = pos / 8;
+			if (bit_mask)
+				bit_mask[mask_pos] |= 1 << (pos % 8);
+		}
+
+		pos++;
+	}
+
+	(*pattern_len) = pos;
+
+	return _TRUE;
+error:
+	return _FALSE;
+}
+
+u8 rtw_wow_pattern_set(_adapter *adapter, struct rtw_wowcam_upd_info *wowcam_info,
+		       enum pattern_type set_type)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(adapter);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	void *phl = GET_PHL_INFO(dvobj);
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+	u8 cam_idx = MAX_WKFM_CAM_NUM;
+
+	rtw_wowlan_set_pattern_cast_type(adapter, wowcam_info);
+	status = rtw_phl_add_wow_ptrn_info(phl, wowcam_info, &cam_idx);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		RTW_INFO("Add wow pattern fail(%d)\n", status);
+		return _FAIL;
+	}
+
+	wowpriv->wow_ptrn_valid[cam_idx] = set_type;
+
+	RTW_INFO("wowcam_id: %d, type: %d\n", cam_idx, set_type);
+
+	return _SUCCESS;
+}
+
+void rtw_wow_pattern_clean(_adapter *adapter, enum pattern_type clean_type)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(adapter);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	void *phl = GET_PHL_INFO(dvobj);
+	u8 cam_idx;
+
+	for (cam_idx = 0; cam_idx < MAX_WKFM_CAM_NUM; cam_idx++) {
+		if (wowpriv->wow_ptrn_valid[cam_idx] == clean_type) {
+			rtw_phl_remove_wow_ptrn_info(phl, cam_idx);
+			wowpriv->wow_ptrn_valid[cam_idx] = RTW_INVALID_PATTERN;
+		}
+	}
+}
+
+#ifdef CONFIG_GOOGLE_CAST_WAKEUP
+#define rtw_user_wow_patten_elem(len, array, args...)	\
+	{.conts_len = len,  .pconts = (u8[len]){array, ##args}}
+
+void rtw_set_google_cast_mdns_wow_pattern(_adapter *adapter)
+{
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(adapter);
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct rtw_wowcam_upd_info wowcam_info = {0};
+	u8 index = 0;
+	u8 eth_dest_mac[ETH_ALEN] = {0x01, 0x00, 0x5e, 0x00, 0x00, 0xfb};
+	u8 eth_protocol[2] = {0x08, 0x00};
+	u8 ip_ver = 0x45;
+	u8 ip_protocol = 0x11;
+	u8 dest_port[2] = {0x14, 0xe9};
+	u8 *ptr;
+	struct pattern_cont_t {
+		u8 conts_len;
+		u8 *pconts;
+	};
+	struct pattern_cont_t conts[] = {
+		rtw_user_wow_patten_elem(0x51, /*_%9E5E7C8F47989526C9BCD95D24084F6F0B27C5ED._sub._googlecast._tcp.local*/
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, 0x5F, 0x25, 0x39, 0x45, 0x35,
+				0x45, 0x37, 0x43, 0x38, 0x46, 0x34, 0x37, 0x39, 0x38, 0x39, 0x35, 0x32, 0x36, 0x43, 0x39, 0x42,
+				0x43, 0x44, 0x39, 0x35, 0x44, 0x32, 0x34, 0x30, 0x38, 0x34, 0x46, 0x36, 0x46, 0x30, 0x42, 0x32,
+				0x37, 0x43, 0x35, 0x45, 0x44, 0x04, 0x5F, 0x73, 0x75, 0x62, 0x0B, 0x5F, 0x67, 0x6F, 0x6F, 0x67,
+				0x6C, 0x65, 0x63, 0x61, 0x73, 0x74, 0x04, 0x5F, 0x74, 0x63, 0x70, 0x05, 0x6C, 0x6F, 0x63, 0x61,
+				0x6C),
+		rtw_user_wow_patten_elem(0x21, /*_googlecast._tcp.local*/
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0B, 0x5F, 0x67, 0x6F, 0x6F, 0x67,
+				0x6C, 0x65, 0x63, 0x61, 0x73, 0x74, 0x04, 0x5F, 0x74, 0x63, 0x70, 0x05, 0x6C, 0x6F, 0x63, 0x61,
+				0x6C),
+		rtw_user_wow_patten_elem(0x30, /*_233637DE._sub._googlecast._tcp.local*/
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x5F, 0x32, 0x33, 0x33, 0x36,
+				0x33, 0x37, 0x44, 0x45, 0x04, 0x5F, 0x73, 0x75, 0x62, 0x0B, 0x5F, 0x67, 0x6F, 0x6F, 0x67, 0x6C,
+				0x65, 0x63, 0x61, 0x73, 0x74, 0x04, 0x5F, 0x74, 0x63, 0x70, 0x05, 0x6C, 0x6F, 0x63, 0x61, 0x6C),
+	};
+	struct pattern_cont_t masks[] = {
+		rtw_user_wow_patten_elem(MAX_WKFM_SIZE,
+				0x3f, 0x70, 0x80, 0x00, 0x30, 0x30, 0xc0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f),
+		rtw_user_wow_patten_elem(10,
+				0x3f, 0x70, 0x80, 0x00, 0x30, 0x30, 0xc0, 0xff, 0xff, 0x1f),
+		rtw_user_wow_patten_elem(12,
+				0x3f, 0x70, 0x80, 0x00, 0x30, 0x30, 0xc0, 0xff, 0xff, 0xff, 0xff, 0x0f),
+	};
+
+	u8 *target = NULL;
+
+	for (index = 0 ; index < GOOGLE_CAST_PATTERN_NUM ; index++){
+		_rtw_memset((void *)&wowcam_info, 0, sizeof(wowcam_info));
+
+		target = wowcam_info.ptrn;
+
+		_rtw_memcpy(target, eth_dest_mac, ETH_ALEN);
+		target += ETH_TYPE_OFFSET;
+		wowcam_info.ptrn_len += ETH_TYPE_OFFSET;
+
+		_rtw_memcpy(target, eth_protocol, 2);
+		target += 2;
+		wowcam_info.ptrn_len += 2;
+
+		*target = ip_ver;
+		target += 1;
+		wowcam_info.ptrn_len += 1;
+
+		/*padding*/
+		target += 8;
+		wowcam_info.ptrn_len += 8;
+
+		*target = ip_protocol;
+		target += 1;
+		wowcam_info.ptrn_len += 1;
+
+		/*padding*/
+		target += 2;
+		wowcam_info.ptrn_len += 2;
+
+		/*skip src & des ip*/
+		target += (RTW_IP_ADDR_LEN * 2);
+		wowcam_info.ptrn_len += (RTW_IP_ADDR_LEN * 2);
+
+		/*skip src port*/
+		target += 2;
+		wowcam_info.ptrn_len += 2;
+
+		_rtw_memcpy(target, dest_port, 2);
+		target += 2;
+		wowcam_info.ptrn_len += 2;
+
+		/*padding*/
+		target += 6;
+		wowcam_info.ptrn_len += 6;
+
+		_rtw_memcpy(target, conts[index].pconts, conts[index].conts_len);
+		wowcam_info.ptrn_len += conts[index].conts_len;
+
+		_rtw_memcpy(wowcam_info.mask, masks[index].pconts, masks[index].conts_len);
+
+		rtw_wow_pattern_set(adapter, &wowcam_info, RTW_DEFAULT_PATTERN);
+	}
+}
+#endif /*CONFIG_GOOGLE_CAST_WAKEUP*/
+
+void rtw_set_default_pattern(_adapter *adapter)
+{
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(adapter);
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct rtw_wowcam_upd_info wowcam_info = {0};
+	u8 index = 0;
+	u8 multicast_addr[3] = {0x01, 0x00, 0x5e};
+	u8 multicast_ip[4] = {0xe0, 0x28, 0x28, 0x2a};
+
+	u8 unicast_mask[5] = {0x3f, 0x70, 0x80, 0xc0, 0x03};
+	u8 icmpv6_mask[7] = {0x00, 0x70, 0x10, 0x00, 0xc0, 0xc0, 0x3f};
+	u8 multicast_mask[5] = {0x07, 0x70, 0x80, 0xc0, 0x03};
+
+	u8 ip_protocol[3] = {0x08, 0x00, 0x45};
+	u8 ipv6_protocol[3] = {0x86, 0xdd, 0x60};
+
+	u8 *target = NULL;
+
+	/*TCP/ICMP unicast*/
+	for (index = 0 ; index < DEFAULT_PATTERN_NUM ; index++) {
+		_rtw_memset((void *)&wowcam_info, 0, sizeof(wowcam_info));
+
+		switch (index) {
+		case 0:
+			target = wowcam_info.ptrn;
+			_rtw_memcpy(target, adapter_mac_addr(adapter), ETH_ALEN);
+
+			target += ETH_TYPE_OFFSET;
+			_rtw_memcpy(target, &ip_protocol, sizeof(ip_protocol));
+
+			/* TCP */
+			target += (PROTOCOL_OFFSET - ETH_TYPE_OFFSET);
+			_rtw_memset(target, 0x06, 1);
+
+			target += (IP_OFFSET - PROTOCOL_OFFSET);
+
+			_rtw_memcpy(target, pmlmeinfo->ip_addr, RTW_IP_ADDR_LEN);
+
+			_rtw_memcpy(wowcam_info.mask,
+				    &unicast_mask, sizeof(unicast_mask));
+
+			wowcam_info.ptrn_len = IP_OFFSET + RTW_IP_ADDR_LEN;
+
+			rtw_wow_pattern_set(adapter, &wowcam_info, RTW_DEFAULT_PATTERN);
+
+			break;
+		case 1:
+			target = wowcam_info.ptrn;
+			_rtw_memcpy(target, adapter_mac_addr(adapter), ETH_ALEN);
+
+			target += ETH_TYPE_OFFSET;
+			_rtw_memcpy(target, &ip_protocol, sizeof(ip_protocol));
+
+			/* ICMP */
+			target += (PROTOCOL_OFFSET - ETH_TYPE_OFFSET);
+			_rtw_memset(target, 0x01, 1);
+
+			target += (IP_OFFSET - PROTOCOL_OFFSET);
+			_rtw_memcpy(target, pmlmeinfo->ip_addr, RTW_IP_ADDR_LEN);
+
+			_rtw_memcpy(wowcam_info.mask,
+				    &unicast_mask, sizeof(unicast_mask));
+
+			wowcam_info.ptrn_len = IP_OFFSET + RTW_IP_ADDR_LEN;
+
+			rtw_wow_pattern_set(adapter, &wowcam_info, RTW_DEFAULT_PATTERN);
+
+			break;
+#ifdef CONFIG_IPV6
+		case 2:
+			if (pwrpriv->wowlan_ns_offload_en == _TRUE) {
+				target = wowcam_info.ptrn;
+				target += ETH_TYPE_OFFSET;
+
+				_rtw_memcpy(target, &ipv6_protocol,
+					    sizeof(ipv6_protocol));
+
+				/* ICMPv6 */
+				target += (IPv6_PROTOCOL_OFFSET -
+					   ETH_TYPE_OFFSET);
+				_rtw_memset(target, 0x3a, 1);
+
+				target += (IPv6_OFFSET - IPv6_PROTOCOL_OFFSET);
+				_rtw_memcpy(target, pmlmeinfo->ip6_addr,
+					    RTW_IPv6_ADDR_LEN);
+
+				_rtw_memcpy(wowcam_info.mask,
+					    &icmpv6_mask, sizeof(icmpv6_mask));
+
+				wowcam_info.ptrn_len = IPv6_OFFSET + RTW_IPv6_ADDR_LEN;
+
+				rtw_wow_pattern_set(adapter, &wowcam_info,
+						    RTW_DEFAULT_PATTERN);
+			}
+			break;
+#endif /*CONFIG_IPV6*/
+		case 3:
+			target = wowcam_info.ptrn;
+			_rtw_memcpy(target, &multicast_addr, sizeof(multicast_addr));
+
+			target += ETH_TYPE_OFFSET;
+			_rtw_memcpy(target, &ip_protocol, sizeof(ip_protocol));
+
+			/* UDP */
+			target += (PROTOCOL_OFFSET - ETH_TYPE_OFFSET);
+			_rtw_memset(target, 0x11, 1);
+
+			target += (IP_OFFSET - PROTOCOL_OFFSET);
+			_rtw_memcpy(target, &multicast_ip, sizeof(multicast_ip));
+
+			_rtw_memcpy(wowcam_info.mask,
+				    &multicast_mask, sizeof(multicast_mask));
+
+			wowcam_info.ptrn_len = IP_OFFSET + sizeof(multicast_ip);
+
+			rtw_wow_pattern_set(adapter, &wowcam_info, RTW_DEFAULT_PATTERN);
+
+			break;
+		default:
+			break;
+		}
+	}
+
+	return;
+}
+
+void rtw_dump_priv_pattern(_adapter *adapter, u8 idx)
+{
+	struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(adapter);
+	char str_1[128];
+	char *p_str;
+	u8 val8 = 0;
+	int i = 0, j = 0, len = 0, max_len = 0;
+
+	RTW_INFO("=========[%d]========\n", idx);
+
+	RTW_INFO(">>>priv_pattern_content:\n");
+	p_str = str_1;
+	max_len = sizeof(str_1);
+	for (i = 0 ; i < MAX_WKFM_PATTERN_SIZE / 8 ; i++) {
+		_rtw_memset(p_str, 0, max_len);
+		len = 0;
+		for (j = 0 ; j < 8 ; j++) {
+			val8 = pwrctl->patterns[idx].content[i * 8 + j];
+			len += snprintf(p_str + len, max_len - len,
+					"%02x ", val8);
+		}
+		RTW_INFO("%s\n", p_str);
+	}
+
+	RTW_INFO(">>>priv_pattern_mask:\n");
+	for (i = 0 ; i < MAX_WKFM_SIZE / 8 ; i++) {
+		_rtw_memset(p_str, 0, max_len);
+		len = 0;
+		for (j = 0 ; j < 8 ; j++) {
+			val8 = pwrctl->patterns[idx].mask[i * 8 + j];
+			len += snprintf(p_str + len, max_len - len,
+					"%02x ", val8);
+		}
+		RTW_INFO("%s\n", p_str);
+	}
+
+	RTW_INFO(">>>priv_pattern_len:\n");
+	RTW_INFO("%s: len: %d\n", __func__, pwrctl->patterns[idx].len);
+}
+
+void rtw_wow_pattern_sw_dump(_adapter *adapter)
+{
+	int i;
+
+	RTW_INFO("********[RTK priv-patterns]*********\n");
+	for (i = 0 ; i < MAX_WKFM_CAM_NUM; i++)
+		rtw_dump_priv_pattern(adapter, i);
+}
+
+void rtw_get_sec_iv(_adapter *padapter, u8 *pcur_dot11txpn, u8 *StaAddr)
+{
+	struct sta_info		*psta;
+	struct security_priv *psecpriv = &padapter->securitypriv;
+
+	_rtw_memset(pcur_dot11txpn, 0, 8);
+	if (NULL == StaAddr)
+		return;
+	psta = rtw_get_stainfo(&padapter->stapriv, StaAddr);
+	RTW_INFO("%s(): StaAddr: %02x %02x %02x %02x %02x %02x\n",
+		 __func__, StaAddr[0], StaAddr[1], StaAddr[2],
+		 StaAddr[3], StaAddr[4], StaAddr[5]);
+
+	if (psta) {
+		if ((psecpriv->dot11PrivacyAlgrthm == _AES_) ||
+		    (psecpriv->dot11PrivacyAlgrthm == _CCMP_256_))
+			AES_IV(pcur_dot11txpn, psta->dot11txpn, 0);
+		else if (psecpriv->dot11PrivacyAlgrthm == _TKIP_)
+			TKIP_IV(pcur_dot11txpn, psta->dot11txpn, 0);
+		else if ((psecpriv->dot11PrivacyAlgrthm == _GCMP_) ||
+			 (psecpriv->dot11PrivacyAlgrthm == _GCMP_256_))
+			GCMP_IV(pcur_dot11txpn, psta->dot11txpn, 0);
+	}
+}
+
+void rtw_construct_remote_control_info(_adapter *adapter,
+				       struct rtw_remote_wake_ctrl_info *ctrl_info)
+{
+	struct security_priv *securitypriv = &adapter->securitypriv;
+	struct stainfo_rxcache *rxcache = NULL;
+	struct sta_info *sta = NULL;
+	u8 gtk_rx_iv[4][IV_LENGTH] = {0};
+	u8 tid_id = 0;
+	u8 i = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_security_priv *lsecuritypriv = &adapter_link->securitypriv;
+
+	sta = rtw_get_stainfo(&adapter->stapriv, get_link_bssid(&adapter_link->mlmepriv));
+
+	if (!sta) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	rxcache = &sta->sta_recvpriv.rxcache;
+
+	rtw_get_sec_iv(adapter, ctrl_info->ptk_tx_iv, get_link_bssid(&adapter_link->mlmepriv));
+	RTW_INFO("[wow] ptk_tx_iv = " IV_FMT "\n", IV_ARG(ctrl_info->ptk_tx_iv));
+
+	ctrl_info->valid_check = REMOTECTRL_INFO_VALID_CHECK;
+	ctrl_info->symbol_check_en |= REMOTECTRL_INFO_SYMBOL_CHK_PTK |
+				      REMOTECTRL_INFO_SYMBOL_CHK_GTK;
+
+	ctrl_info->gtk_key_idx = lsecuritypriv->dot118021XGrpKeyid;
+	RTW_INFO("[wow] gtk_key_idx = %d\n", ctrl_info->gtk_key_idx);
+
+	tid_id = rxcache->last_tid;
+	_rtw_memcpy(ctrl_info->ptk_rx_iv, rxcache->iv[tid_id], IV_LENGTH);
+	RTW_INFO("[wow] ptk_rx_iv = " IV_FMT "\n", IV_ARG(ctrl_info->ptk_rx_iv));
+
+	for (i = 0; i < 4; i++) {
+		rtw_pn_to_iv(lsecuritypriv->iv_seq[i], gtk_rx_iv[i], i,
+			     securitypriv->dot118021XGrpPrivacy);
+		RTW_INFO("[wow] gtk_rx_iv[%u] = " IV_FMT "\n", i, IV_ARG(gtk_rx_iv[i]));
+	}
+	_rtw_memcpy(ctrl_info->gtk_rx_iv_idx0, gtk_rx_iv[0], IV_LENGTH);
+	_rtw_memcpy(ctrl_info->gtk_rx_iv_idx1, gtk_rx_iv[1], IV_LENGTH);
+	_rtw_memcpy(ctrl_info->gtk_rx_iv_idx2, gtk_rx_iv[2], IV_LENGTH);
+	_rtw_memcpy(ctrl_info->gtk_rx_iv_idx3, gtk_rx_iv[3], IV_LENGTH);
+}
+
+void rtw_core_wow_handle_wake_up_rsn(void *drv_priv, u8 rsn)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+	struct wow_priv *wowpriv = dvobj_to_wowlan(dvobj);
+
+	wowpriv->wow_wake_reason = rsn;
+}
+
+#ifdef CONFIG_MDNS_OFFLOAD
+static struct rtw_mdns_resp_entry *
+_rtw_get_mdns_resp_entry(_adapter *padapter, u8 index)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_mdns_ofld_info *ofld_info = &wowpriv->mdns_ofld_info;
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+
+	if (index < MAX_MDNS_RESP_NUM)
+		resp_entry = &ofld_info->resp_entry[index];
+
+	return resp_entry;
+}
+
+int rtw_wow_add_mdns_resp(_adapter *padapter, u8 index, u8 *resp_content, u16 content_len)
+{
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+
+	resp_entry = _rtw_get_mdns_resp_entry(padapter, index);
+	if (resp_entry == NULL)
+		return _FAIL;
+
+	_rtw_memcpy(resp_entry->content, resp_content, content_len);
+	resp_entry->content_len = content_len;
+
+	return _SUCCESS;
+}
+
+int rtw_wow_del_mdns_resp(_adapter *padapter, u8 index)
+{
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+
+	resp_entry = _rtw_get_mdns_resp_entry(padapter, index);
+	if (resp_entry == NULL)
+		return _FAIL;
+
+	_rtw_memset(resp_entry, 0, sizeof(struct rtw_mdns_resp_entry));
+
+	return _SUCCESS;
+}
+
+int rtw_wow_get_mdns_resp_ent(_adapter *padapter, u8 index, struct rtw_mdns_resp_entry **resp_entry)
+{
+	*resp_entry = _rtw_get_mdns_resp_entry(padapter, index);
+	if (*resp_entry == NULL)
+		return _FAIL;
+
+	return _SUCCESS;
+}
+
+int rtw_wow_add_mdns_match_crit(_adapter *padapter, u8 index, u16 match_type, u16 name_offset, u16 name_len)
+{
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+	struct rtw_mdns_match_criteria *match_ct = NULL;
+
+	resp_entry = _rtw_get_mdns_resp_entry(padapter, index);
+	if (resp_entry == NULL)
+		return _FAIL;
+
+	if (resp_entry->match_ct_num == MAX_MDNS_MATCH_CRITERIA_NUM)
+		return _FAIL;
+
+	match_ct = &resp_entry->match_ct[resp_entry->match_ct_num];
+
+	match_ct->name_offset = name_offset;
+	match_ct->type = match_type;
+	match_ct->name_len = name_len;
+
+	resp_entry->match_ct_num += 1;
+
+	return _SUCCESS;
+}
+
+int rtw_wow_del_mdns_match_crit(_adapter *padapter, u8 index)
+{
+	struct rtw_mdns_resp_entry *resp_entry = NULL;
+	struct rtw_mdns_match_criteria *match_ct = NULL;
+
+	resp_entry = _rtw_get_mdns_resp_entry(padapter, index);
+	if (resp_entry == NULL)
+		return _FAIL;
+
+	resp_entry->match_ct_num = 0;
+	_rtw_memset(resp_entry->match_ct, 0, sizeof(resp_entry->match_ct));
+
+	return _SUCCESS;
+}
+
+int rtw_wow_add_mdns_passthru_name(_adapter *padapter, u8 *name, u8 name_len)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_mdns_ofld_info *ofld_info = &wowpriv->mdns_ofld_info;
+	struct rtw_mdns_passthru_list *passthru_list = &ofld_info->passthru_list;
+	struct rtw_mdns_passthru_name *passthru_name = NULL;
+
+	if (passthru_list->passthru_name_num == MAX_MDNS_PASSTHRU_NAME_NUM)
+		return _FAIL;
+
+	passthru_name = &passthru_list->passthru_name[passthru_list->passthru_name_num];
+	_rtw_memcpy(passthru_name->name, name, name_len);
+	passthru_name->name_len = name_len;
+
+	passthru_list->passthru_name_num += 1;
+
+	return _SUCCESS;
+}
+
+void rtw_wow_clr_mdns_passthru_name(_adapter *padapter)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_mdns_ofld_info *ofld_info = &wowpriv->mdns_ofld_info;
+	struct rtw_mdns_passthru_list *passthru_list = &ofld_info->passthru_list;
+
+	passthru_list->passthru_name_num = 0;
+	_rtw_memset(passthru_list->passthru_name, 0,
+		    sizeof(struct rtw_mdns_passthru_name) * MAX_MDNS_PASSTHRU_NAME_NUM);
+}
+
+void rtw_wow_get_mdns_passthru_list(_adapter *padapter, struct rtw_mdns_passthru_list **passthru_list)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_mdns_ofld_info *ofld_info = &wowpriv->mdns_ofld_info;
+
+	*passthru_list = &ofld_info->passthru_list;
+}
+#endif /* CONFIG_MDNS_OFFLOAD */
+
+#ifdef CONFIG_WOW_APF
+u8 rtw_wow_set_apf(struct _ADAPTER *padapter, u8 *apf_prog, u16 apf_prog_len)
+{
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_apf_info *info = &wowpriv->apf_info;
+	u16 apf_ram_len;
+	u16 apf_data_len;
+
+	RTW_INFO("%s()\n", __func__);
+	/*data len = apf_ram_len() - apf_prog_len*/
+	apf_data_len = MAX_APF_PROG_SIZE - apf_prog_len;
+	RTW_INFO("apf_prog_len(%u) apf_data_len(%u)\n", apf_prog_len, apf_data_len);
+	apf_ram_len = apf_prog_len + apf_data_len;
+	/* Check apf prog limitation */
+	if (apf_ram_len > MAX_APF_PROG_SIZE) {
+		RTW_ERR("%s() apf prog size (%u) exceed prog limit(%u)\n",
+				__func__, apf_ram_len, MAX_APF_PROG_SIZE);
+	}
+
+	/* Dont clear apf program ram, this apf ram is controlled by android */
+	info->apf_prog_len = apf_prog_len;
+	info->apf_data_len = apf_data_len;
+	_rtw_memcpy(info->apf_prog, apf_prog, apf_prog_len);
+
+	info->apf_prog_num = 1;
+	info->apf_prog_total_size = apf_ram_len;
+
+	if (info->apf_prog_num > 0) {
+		RTW_INFO("%s apf_en is 1\n", __func__);
+		info->apf_en = 1;
+	}
+
+#ifdef CONFIG_WOW_APF_DBG
+	RTW_INFO("[APFDBG]%s() apf prog num(%u) total_size(%u)\n", __func__, info->apf_prog_num, info->apf_prog_total_size);
+	RTW_INFO_DUMP(NULL, info->apf_prog, info->apf_prog_len);
+#endif
+	return _SUCCESS;
+}
+
+u8 rtw_wow_get_apf(struct _ADAPTER *padapter, u8 *apf_prog, u16 *apf_prog_len) {
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_apf_info *info = &wowpriv->apf_info;
+	u16 apf_ram_len;
+	u8 apf_data_len;
+
+	RTW_INFO("%s()\n", __func__);
+	if (apf_prog == NULL || apf_prog_len == NULL || info->apf_prog == NULL) {
+		RTW_ERR("%s() buffer or apf_prog[%u] is NULL\n", __func__, 0);
+#ifdef CONFIG_WOW_APF_DBG
+		RTW_INFO("[APFDBG]====apf prog_len(%u) data_len(%u)====\n"
+					 , info->apf_prog_len, info->apf_data_len);
+		RTW_INFO_DUMP(NULL, info->apf_prog, info->apf_prog_len + info->apf_data_len);
+#endif
+		return _FAIL;
+	}
+
+	*apf_prog_len = info->apf_prog_len;
+	info->apf_data_len = MAX_APF_PROG_SIZE - info->apf_prog_len;
+	_rtw_memcpy(apf_prog, info->apf_prog, info->apf_prog_len + info->apf_data_len);
+#ifdef CONFIG_WOW_APF_DBG
+	RTW_INFO("[APFDBG]%s() Get apf_prog, prog_len(%u) data_len(%u)\n"
+			 , __func__,  *apf_prog_len, info->apf_data_len);
+	RTW_INFO_DUMP(NULL, apf_prog, *apf_prog_len + info->apf_data_len);
+#endif
+	return _SUCCESS;
+}
+
+u8 rtw_wow_clear_apf(struct _ADAPTER *padapter) {
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_apf_info *info = &wowpriv->apf_info;
+	u8 apf_ram_len;
+
+	RTW_INFO("%s()\n", __func__);
+	_rtw_memset(info->apf_prog, 0, MAX_APF_PROG_SIZE);
+	info->apf_prog_len = 0;
+	info->apf_data_len = 0;
+	info->apf_en = 0;
+	info->apf_prog_num = 0;
+	info->apf_prog_total_size = 0;
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_WOW_APF */
+
+#ifdef CONFIG_GTK_OL
+void _update_aoac_rpt_phase_0(_adapter *adapter, struct rtw_aoac_report *aoac_info)
+{
+	struct security_priv *securitypriv = &adapter->securitypriv;
+	struct stainfo_rxcache *rxcache = NULL;
+	struct sta_info *sta = NULL;
+	u8 pn[8] = {0};
+	u8 gtk_key_idx = 0;
+	u8 i = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_security_priv *lsecuritypriv = &adapter_link->securitypriv;
+
+	/* handle ptk rx iv */
+	/* This Rx IV has no effect, the driver does not drop unicast packets
+	 * due to different PNs. (Refer to recv_ucast_pn_decache)
+	 */
+	sta = rtw_get_stainfo(&adapter->stapriv, get_bssid(&adapter->mlmepriv));
+	if (sta) {
+		if (rtw_iv_to_pn(aoac_info->ptk_rx_iv, pn, NULL,
+				 securitypriv->dot11PrivacyAlgrthm)) {
+			rxcache = &sta->sta_recvpriv.rxcache;
+			for (i = 0; i < TID_NUM; i++)
+				_rtw_memcpy(rxcache->iv[i], aoac_info->ptk_rx_iv,
+					    IV_LENGTH);
+			sta->dot11rxpn.val = RTW_GET_LE64(pn);
+			RTW_INFO("[wow] ptk_rx_pn = " PN_FMT "\n", PN_ARG(pn));
+		}
+	}
+
+	/* handle gtk rx iv */
+	gtk_key_idx = aoac_info->key_idx;
+	if (rtw_iv_to_pn(aoac_info->gtk_rx_iv[gtk_key_idx], pn, NULL,
+			 securitypriv->dot118021XGrpPrivacy)) {
+		_rtw_memcpy(lsecuritypriv->iv_seq[gtk_key_idx], pn, 8);
+		RTW_INFO("[wow] gtk_rx_pn[%u] = " PN_FMT "\n", gtk_key_idx, PN_ARG(pn));
+	}
+
+#ifdef CONFIG_IEEE80211W
+	/* handle igtk rx ipn */
+	if (SEC_IS_BIP_KEY_INSTALLED(lsecuritypriv)) {
+		lsecuritypriv->dot11wBIPrxpn.val = RTW_GET_LE64(aoac_info->igtk_ipn);
+		RTW_INFO("[wow] igtk_rx_pn = " PN_FMT "\n", PN_ARG(aoac_info->igtk_ipn));
+	}
+#endif
+}
+
+void _update_aoac_rpt_phase_1(_adapter *adapter, struct rtw_aoac_report *aoac_info)
+{
+	struct security_priv *securitypriv = &adapter->securitypriv;
+	struct sta_info *sta = NULL;
+	u8 pn[8] = {0};
+	u8 gtk_key_idx = 0;
+	u8 key_len = 0;
+	u8 i = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_security_priv *lsecuritypriv = &adapter_link->securitypriv;
+
+	/* handle ptk tx iv */
+	sta = rtw_get_stainfo(&adapter->stapriv, get_link_bssid(&adapter_link->mlmepriv));
+	if (sta) {
+		if (rtw_iv_to_pn(aoac_info->ptk_tx_iv, pn, NULL,
+				 securitypriv->dot11PrivacyAlgrthm)) {
+			sta->dot11txpn.val = RTW_GET_LE64(pn);
+			RTW_INFO("[wow] ptk_tx_pn = " PN_FMT "\n", PN_ARG(pn));
+		}
+	}
+
+	if (aoac_info->rekey_ok) {
+		/* update gtk key */
+		gtk_key_idx = aoac_info->key_idx;
+		lsecuritypriv->dot118021XGrpKeyid = gtk_key_idx;
+
+		switch (securitypriv->dot118021XGrpPrivacy) {
+		case _TKIP_:
+		case _AES_:
+		case _GCMP_:
+			key_len = 16;
+			break;
+		case _GCMP_256_:
+		case _CCMP_256_:
+			key_len = 32;
+			break;
+		default:
+			key_len = 0;
+		}
+
+		if (key_len)
+			_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[gtk_key_idx].skey,
+					aoac_info->gtk, key_len);
+		/* update tkip dot118021XGrptxmickey dot118021XGrprxmickey */
+		if (securitypriv->dot118021XGrpPrivacy == _TKIP_) {
+			/* The order of the GTK Tx/Rx mic keys in the AOAC report is
+			 * reversed compared to the GTK Tx/Rx mic keys provided by
+			 * wpa_supplicant.
+			 */
+			_rtw_memcpy(lsecuritypriv->dot118021XGrptxmickey[gtk_key_idx].skey,
+					&aoac_info->gtk[24], 8);
+			_rtw_memcpy(lsecuritypriv->dot118021XGrprxmickey[gtk_key_idx].skey,
+					&aoac_info->gtk[16], 8);
+		}
+		rtw_set_key(adapter, adapter_link, gtk_key_idx, 1, _TRUE);
+
+		/* update eapol replay_counter */
+		_rtw_memcpy(sta->replay_ctr, aoac_info->eapol_key_replay_count,
+			    RTW_REPLAY_CTR_LEN);
+#ifdef CONFIG_IEEE80211W
+		if (SEC_IS_BIP_KEY_INSTALLED(lsecuritypriv)) {
+			switch (securitypriv->dot11wCipher) {
+			case _BIP_CMAC_128_:
+			case _BIP_GMAC_128_:
+				key_len = 16;
+				break;
+			case _BIP_CMAC_256_:
+			case _BIP_GMAC_256_:
+				key_len = 32;
+				break;
+			default:
+				key_len = 0;
+			}
+
+			/*
+			 * The Rx IPN has already been updated in
+			 * _update_aoac_rpt_phase_0(), so no update is done here
+			 * unless WoWLAN FW updates the Rx IPN before sending
+			 * out the phase1 AOAC report.
+			 */
+			lsecuritypriv->dot11wBIPKeyid = RTW_GET_LE32(aoac_info->igtk_key_id);
+			_rtw_memcpy(lsecuritypriv->dot11wBIPKey[lsecuritypriv->dot11wBIPKeyid].skey,
+				    &aoac_info->igtk, key_len);
+		}
+#endif
+	} else {
+		RTW_INFO("[wow] no rekey event\n");
+	}
+
+	for (i = 0; i < 4; i++) {
+		if (rtw_iv_to_pn(aoac_info->gtk_rx_iv[i], pn, NULL,
+				 securitypriv->dot118021XGrpPrivacy)) {
+			_rtw_memcpy(lsecuritypriv->iv_seq[i], pn, 8);
+			RTW_INFO("[wow] gtk_rx_pn[%u] = " PN_FMT "\n", i, PN_ARG(pn));
+		}
+	}
+}
+
+void rtw_update_gtk_ofld_info(void *drv_priv, struct rtw_aoac_report *aoac_info,
+			      u8 aoac_report_get_ok, u8 rx_ready)
+{
+	struct dvobj_priv *dvobj = (struct dvobj_priv *)drv_priv;
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+
+	if (!rx_ready)
+		_update_aoac_rpt_phase_0(adapter, aoac_info);
+	else
+		_update_aoac_rpt_phase_1(adapter, aoac_info);
+}
+#endif /* CONFIG_GTK_OL */
+
+#ifdef CONFIG_WRC_WOW_MAGIC
+u8 rtw_cfg_wrc_wol_magic(_adapter *padapter, u8 enable)
+{
+	struct registry_priv *registry_par = &padapter->registrypriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u8 iface_num = GET_IFACE_NUMS(padapter);
+	_adapter *padapter_wrc = NULL;
+	struct _ADAPTER_LINK *padapter_wrc_link = NULL;
+	enum rtw_phl_status pstatus;
+	u8 ret = _FAIL;
+	u8 i;
+
+	if (!registry_par->wakeup_event & BIT(0)) {
+		RTW_INFO("[wow][wrc] no magic packet wake up\n");
+		goto exit;
+	}
+
+	for (i = 0; i < iface_num; i++) {
+		if (i != padapter->iface_id)
+			padapter_wrc = GET_ADAPTER(padapter, i);
+	}
+
+	if (padapter_wrc == NULL) {
+		RTW_ERR("[wow][wrc] no avaliable interface\n");
+		goto exit;
+	}
+
+	/* ToDo CONFIG_RTW_MLD: currently primary link only */
+	padapter_wrc_link = GET_PRIMARY_LINK(padapter_wrc);
+
+	if (enable) {
+		struct rtw_phl_stainfo_t *phl_sta;
+		enum role_type rtype = PHL_RTYPE_AP;
+
+		pstatus = rtw_phl_cmd_wrole_change(GET_PHL_INFO(dvobj),
+						   padapter_wrc->phl_role,
+						   padapter_wrc_link->wrlink,
+						   WR_CHG_MADDR,
+						   padapter->mac_addr,
+						   MACADDRLEN,
+						   PHL_CMD_DIRECTLY, 0);
+		if (pstatus != RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("[wow][wrc] change role mac failed\n");
+			goto exit;
+		}
+
+		phl_sta = rtw_phl_get_stainfo_self(GET_PHL_INFO(dvobj),
+						   padapter_wrc_link->wrlink);
+		if (!phl_sta) {
+			RTW_ERR("[wow][wrc] change role type failed\n");
+			goto exit;
+		}
+
+		phl_sta->wol_magic = _TRUE;
+		phl_sta->hit_rule = _TRUE;
+
+		pstatus = rtw_phl_cmd_wrole_change(GET_PHL_INFO(dvobj),
+						   padapter_wrc->phl_role,
+						   padapter_wrc_link->wrlink,
+						   WR_CHG_TYPE, (u8*)&rtype,
+						   sizeof(enum role_type),
+						   PHL_CMD_DIRECTLY, 0);
+		if (pstatus != RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("[wow][wrc] change role type failed\n");
+			goto exit;
+		}
+	} else {
+		u8 role_id;
+		u8 *padapter_wrc_link_mac_addr[RTW_RLINK_MAX] = {NULL};
+
+		rtw_phl_wifi_role_free(GET_PHL_INFO(dvobj),
+				       padapter_wrc->phl_role->id);
+		padapter_wrc->phl_role = NULL;
+
+		rtw_collect_adapter_link_mac_addr(padapter_wrc,
+						  padapter_wrc_link_mac_addr);
+
+		role_id = rtw_phl_wifi_role_alloc(GET_PHL_INFO(dvobj),
+						  padapter_wrc->mac_addr,
+						  padapter_wrc_link_mac_addr,
+						  PHL_RTYPE_STATION,
+						  padapter_wrc->iface_id,
+						  &padapter_wrc->phl_role,
+						  DTYPE,
+						  _FALSE);
+		if (role_id == INVALID_WIFI_ROLE_IDX ||
+		    padapter_wrc->phl_role == NULL) {
+			RTW_ERR("[wow][wrc] rtw_phl_wifi_role_alloc failed\n");
+			rtw_warn_on(1);
+			goto exit;
+		}
+	}
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_WRC_WOW_MAGIC */
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_PNO_SUPPORT
+#ifdef CONFIG_PNO_SECURITY_OFFLOAD
+static void nlo_parse_cipher_list(struct rtw_nlo_info *wow_nlo, char *list_str)
+{
+	char *pch;
+	char *pnext;
+	char *pend;
+	u8 index = 0;
+
+	pch = list_str;
+	while (strlen(pch) != 0) {
+		pnext = strstr(pch, "key_mgmt=");
+		if (pnext == NULL)
+			break;
+
+		pch = pnext + strlen(CIPHER_IE);
+		pend = strstr(pch, "}");
+		if (strncmp(pch, CIPHER_NONE, strlen(CIPHER_NONE)) == 0) {
+			wow_nlo->chipertype[index] = NLO_CIPHER_OPEN;
+		} else if (strncmp(pch, CIPHER_WPA_PSK, strlen(CIPHER_WPA_PSK)) == 0) {
+			wow_nlo->chipertype[index] = NLO_CIPHER_WPA_TKIP |
+						     NLO_CIPHER_WPA_AES |
+						     NLO_CIPHER_WPA2_TKIP |
+						     NLO_CIPHER_WPA2_AES;
+		} else if (strncmp(pch, CIPHER_WPA_EAP, strlen(CIPHER_WPA_EAP)) == 0) {
+			wow_nlo->chipertype[index] = NLO_CIPHER_WEP;
+		}
+
+		index++;
+		pch = pend + 1;
+	}
+}
+
+static void nlo_security_init(struct rtw_nlo_info *wow_nlo)
+{
+	struct file *fp;
+	mm_segment_t fs;
+	loff_t pos = 0;
+	u8 *source = NULL;
+	long len = 0;
+
+	fp = filp_open("/data/misc/wifi/wpa_supplicant.conf", O_RDONLY,  0644);
+	if (IS_ERR(fp)) {
+		RTW_INFO("Error, wpa_supplicant.conf doesn't exist.\n");
+		RTW_INFO("Error, cipher array using default value.\n");
+		return;
+	} else {
+		RTW_INFO("Open wpa_supplicant.conf successfully.\n");
+	}
+
+	len = i_size_read(fp->f_path.dentry->d_inode);
+	if (len < 0 || len > 2048) {
+		RTW_INFO("Error, file size is bigger than 2048.\n");
+		RTW_INFO("Error, cipher array using default value.\n");
+		return;
+	}
+
+	fs = get_fs();
+	set_fs(KERNEL_DS);
+
+	source = rtw_zmalloc(2048);
+
+	if (source != NULL) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+		len = kernel_read(fp, source, len, &pos);
+#else
+		len = vfs_read(fp, source, len, &pos);
+#endif
+		nlo_parse_cipher_list(wow_nlo, source);
+		rtw_mfree(source, 2048);
+	}
+
+	wow_nlo->compare_cipher_type = _TRUE;
+
+	set_fs(fs);
+	filp_close(fp, NULL);
+}
+#endif
+
+static void nlo_scan_ch_init(struct rtw_nlo_info *wow_nlo,
+			     struct ieee80211_channel **channels,
+			     u32 n_channels)
+{
+	u8 i = 0;
+
+	while (i < MAX_NLO_CHANNEL && i < n_channels) {
+		wow_nlo->channel_list[i].bw = CHANNEL_WIDTH_20;
+		wow_nlo->channel_list[i].center_chan = channels[i]->hw_value;
+		wow_nlo->channel_list[i].chan = channels[i]->hw_value;
+		wow_nlo->channel_list[i].period = 100;
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0))
+		if (channels[i]->flags & IEEE80211_CHAN_PASSIVE_SCAN)
+#else
+		if (channels[i]->flags & IEEE80211_CHAN_NO_IR)
+#endif
+			wow_nlo->channel_list[i].tx_pkt = 0;
+		else
+			wow_nlo->channel_list[i].tx_pkt = 1;
+		wow_nlo->channel_list[i].tx_data_pause = 1;
+		i++;
+	}
+	wow_nlo->channel_num = i;
+}
+
+static void nlo_ssid_init(struct rtw_nlo_info *wow_nlo,
+			  struct cfg80211_ssid *ssids, int n_ssids)
+{
+	u8 i = 0;
+
+	while (i < MAX_NLO_NUM && i < n_ssids) {
+		_rtw_memcpy((void *)wow_nlo->ssid[i], (void *)ssids[i].ssid,
+			    ssids[i].ssid_len);
+		wow_nlo->ssidlen[i] = ssids[i].ssid_len;
+		i++;
+	}
+
+	wow_nlo->num_of_networks = i;
+	wow_nlo->num_of_hidden_ap = i;
+}
+
+int rtw_nlo_enable(struct net_device *net, struct cfg80211_ssid *ssids,
+		   int n_ssids, struct ieee80211_channel **channels,
+		   u32 n_channels, u32 delay, u32 interval, u32 iterations,
+		   u32 slow_interval)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_nlo_info *wow_nlo = &wowpriv->wow_nlo;
+
+	_rtw_memset((void *)wow_nlo, 0, sizeof(struct rtw_nlo_info));
+
+	wow_nlo->nlo_en = _TRUE;
+
+	nlo_scan_ch_init(wow_nlo, channels, n_channels);
+	nlo_ssid_init(wow_nlo, ssids, n_ssids);
+#ifdef CONFIG_PNO_SECURITY_OFFLOAD
+	nlo_security_init(wow_nlo);
+#endif
+
+	wow_nlo->delay = delay * 1000;
+	wow_nlo->period = interval * 1000;
+	wow_nlo->cycle = iterations;
+	wow_nlo->slow_period = slow_interval * 1000;
+	wow_nlo->construct_pbreq = NULL;;
+
+	rtw_nlo_debug(net);
+
+	return 0;
+}
+
+int rtw_nlo_disable(struct net_device *net)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_nlo_info *wow_nlo = &wowpriv->wow_nlo;
+
+	wow_nlo->nlo_en = _FALSE;
+
+	return 0;
+}
+
+void rtw_nlo_debug(struct net_device *net)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_nlo_info *wow_nlo = &wowpriv->wow_nlo;
+	struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(padapter);
+	int i;
+
+	RTW_INFO("********NLO_INFO********\n");
+	RTW_INFO("compare_cipher_type: %d\n", wow_nlo->compare_cipher_type);
+	RTW_INFO("ssid_num: %d\n", wow_nlo->num_of_networks);
+	for (i = 0; i < wow_nlo->num_of_networks; i++) {
+		RTW_INFO("%d SSID (%s) length (%d) cipher (%#x)\n",
+			 i, wow_nlo->ssid[i], wow_nlo->ssidlen[i],
+			 wow_nlo->chipertype[i]);
+	}
+	RTW_INFO("delay: %d\n", wow_nlo->delay);
+	RTW_INFO("fast_scan_iterations: %d\n", wow_nlo->cycle);
+	RTW_INFO("fast_scan_period: %d\n", wow_nlo->period);
+	RTW_INFO("slow_scan_period: %d\n", wow_nlo->slow_period);
+
+	RTW_INFO("********SCAN_INFO*******\n");
+	RTW_INFO("ch_num: %d\n", wow_nlo->channel_num);
+	for (i = 0 ;i < wow_nlo->channel_num; i++) {
+		RTW_INFO("[%02d] avtive:%d, timeout:%d, ch:%02d\n",
+			 i, wow_nlo->channel_list[i].tx_pkt,
+			 wow_nlo->channel_list[i].period,
+			 wow_nlo->channel_list[i].chan);
+	}
+	RTW_INFO("************************\n");
+}
+#endif /* CONFIG_PNO_SUPPORT */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_xmit.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_xmit.c
new file mode 100644
index 000000000000..5c626e6b3c41
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_xmit.c
@@ -0,0 +1,9669 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_XMIT_C_
+
+#include <drv_types.h>
+
+static u8 P802_1H_OUI[P80211_OUI_LEN] = { 0x00, 0x00, 0xf8 };
+static u8 RFC1042_OUI[P80211_OUI_LEN] = { 0x00, 0x00, 0x00 };
+
+static void _init_txservq(struct tx_servq *ptxservq)
+{
+	_rtw_init_listhead(&ptxservq->tx_pending);
+	_rtw_init_queue(&ptxservq->sta_pending);
+	ptxservq->qcnt = 0;
+}
+
+
+void	_rtw_init_sta_xmit_priv(struct sta_xmit_priv *psta_xmitpriv)
+{
+
+
+	_rtw_memset((unsigned char *)psta_xmitpriv, 0, sizeof(struct sta_xmit_priv));
+
+	_rtw_spinlock_init(&psta_xmitpriv->lock);
+
+	/* for(i = 0 ; i < MAX_NUMBLKS; i++) */
+	/*	_init_txservq(&(psta_xmitpriv->blk_q[i])); */
+
+	_init_txservq(&psta_xmitpriv->be_q);
+	_init_txservq(&psta_xmitpriv->bk_q);
+	_init_txservq(&psta_xmitpriv->vi_q);
+	_init_txservq(&psta_xmitpriv->vo_q);
+	_rtw_init_listhead(&psta_xmitpriv->legacy_dz);
+	_rtw_init_listhead(&psta_xmitpriv->apsd);
+
+
+}
+
+void rtw_init_xmit_block(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	_rtw_spinlock_init(&dvobj->xmit_block_lock);
+	dvobj->xmit_block = XMIT_BLOCK_NONE;
+
+}
+void rtw_free_xmit_block(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	_rtw_spinlock_free(&dvobj->xmit_block_lock);
+}
+
+#ifdef RTW_PHL_TX
+u8 alloc_txring(_adapter *padapter)
+{
+	struct xmit_txreq_buf *ptxreq_buf = NULL;
+	u32 idx, alloc_sz = 0, alloc_sz_txreq = 0;
+	u8 res = _SUCCESS;
+
+	u32 offset_head = (sizeof(struct rtw_xmit_req) * RTW_MAX_FRAG_NUM);
+	u32 offset_tail = offset_head + (SZ_HEAD_BUF * RTW_MAX_FRAG_NUM);
+	u32 offset_list = offset_tail + (SZ_TAIL_BUF * RTW_MAX_FRAG_NUM);
+
+	PHLTX_ENTER;
+
+	alloc_sz = (SZ_TX_RING * RTW_MAX_FRAG_NUM);
+	alloc_sz_txreq = MAX_TX_RING_NUM * (sizeof(struct xmit_txreq_buf));
+
+	RTW_INFO("eric-tx [%s] alloc_sz = %d, alloc_sz_txreq = %d\n", __FUNCTION__, alloc_sz, alloc_sz_txreq);
+
+	padapter->pxmit_txreq_buf = rtw_vmalloc(alloc_sz_txreq);
+	ptxreq_buf = (struct xmit_txreq_buf *)padapter->pxmit_txreq_buf;
+
+	_rtw_init_queue(&padapter->free_txreq_queue);
+
+	for (idx = 0; idx < MAX_TX_RING_NUM; idx++) {
+
+		padapter->tx_pool_ring[idx] = rtw_zmalloc(alloc_sz);
+		if (!padapter->tx_pool_ring[idx]) {
+			RTW_ERR("[core] alloc txring fail, plz check.\n");
+			res = _FAIL;
+			break;
+		}
+		_rtw_init_listhead(&ptxreq_buf->list);
+		ptxreq_buf->txreq = padapter->tx_pool_ring[idx];
+		ptxreq_buf->head = padapter->tx_pool_ring[idx] + offset_head;
+		ptxreq_buf->tail = padapter->tx_pool_ring[idx] + offset_tail;
+		ptxreq_buf->pkt_list = padapter->tx_pool_ring[idx] + offset_list;
+
+		#ifdef USE_PREV_WLHDR_BUF /* CONFIG_CORE_TXSC */
+		ptxreq_buf->macid = 0xff;
+		ptxreq_buf->txsc_id = 0xff;
+		#endif
+
+		rtw_list_insert_tail(&(ptxreq_buf->list), &(padapter->free_txreq_queue.queue));
+
+		ptxreq_buf++;
+	}
+
+	padapter->free_txreq_cnt = MAX_TX_RING_NUM;
+
+	return res;
+}
+
+void free_txring(_adapter *padapter)
+{
+	u32 idx, alloc_sz = 0, alloc_sz_txreq = 0;
+#ifdef CONFIG_CORE_TXSC
+	struct rtw_xmit_req *txreq = NULL;
+	struct xmit_txreq_buf *txreq_buf = NULL;
+	u8 j;
+#endif
+
+	PHLTX_ENTER;
+
+	alloc_sz = (SZ_TX_RING * RTW_MAX_FRAG_NUM);
+	alloc_sz_txreq = MAX_TX_RING_NUM * (sizeof(struct xmit_txreq_buf));
+
+	RTW_INFO("eric-tx [%s] alloc_sz = %d,  alloc_sz_txreq = %d\n", __func__, alloc_sz, alloc_sz_txreq);
+
+	for (idx = 0; idx < MAX_TX_RING_NUM; idx++) {
+		if (padapter->tx_pool_ring[idx]) {
+#ifdef CONFIG_CORE_TXSC
+			txreq = (struct rtw_xmit_req *)padapter->tx_pool_ring[idx];
+			if (txreq->treq_type == RTW_PHL_TREQ_TYPE_CORE_TXSC) {
+				txreq_buf = (struct xmit_txreq_buf *)txreq->os_priv;
+				if (txreq_buf) {
+					/* CONFGI_TXSC_AMSDU */
+					for (j = 0; j < txreq_buf->pkt_cnt; j++) {
+						if (txreq_buf->pkt[j])
+							rtw_os_pkt_complete(padapter, (void *)txreq_buf->pkt[j]);
+					}
+				}
+			}
+#endif
+			rtw_mfree(padapter->tx_pool_ring[idx], alloc_sz);
+		}
+	}
+
+	_rtw_spinlock_free(&padapter->free_txreq_queue.lock);
+	rtw_vmfree(padapter->pxmit_txreq_buf, alloc_sz_txreq);
+}
+
+#endif
+
+
+s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, _adapter *padapter)
+{
+	int i;
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct xmit_buf *pxmitbuf;
+	#endif
+	struct xmit_frame *pxframe;
+	sint res = _SUCCESS;
+	/* MGT_TXREQ_MGT */
+	u8 *txreq = NULL, *pkt_list = NULL;
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	u8 xmitbuf_nr = GET_HAL_XMITBUF_NR(dvobj);
+	u16 xmitbuf_sz = GET_HAL_XMITBUF_SZ(dvobj);
+
+	u8 xmitbuf_ext_nr = GET_HAL_XMITBUF_EXT_NR(dvobj);
+	u16 xmitbuf_ext_sz = GET_HAL_XMITBUF_EXT_SZ(dvobj);
+	#endif
+
+	/* We don't need to memset padapter->XXX to zero, because adapter is allocated by rtw_zvmalloc(). */
+	/* _rtw_memset((unsigned char *)pxmitpriv, 0, sizeof(struct xmit_priv)); */
+
+	_rtw_spinlock_init(&pxmitpriv->lock);
+	_rtw_spinlock_init(&pxmitpriv->lock_sctx);
+	#if 0 /*def CONFIG_XMIT_THREAD_MODE*/
+	_rtw_init_sema(&pxmitpriv->xmit_sema, 0);
+	#endif
+
+	/*
+	Please insert all the queue initializaiton using _rtw_init_queue below
+	*/
+
+	pxmitpriv->adapter = padapter;
+
+	/* for(i = 0 ; i < MAX_NUMBLKS; i++) */
+	/*	_rtw_init_queue(&pxmitpriv->blk_strms[i]); */
+
+	_rtw_init_queue(&pxmitpriv->be_pending);
+	_rtw_init_queue(&pxmitpriv->bk_pending);
+	_rtw_init_queue(&pxmitpriv->vi_pending);
+	_rtw_init_queue(&pxmitpriv->vo_pending);
+	_rtw_init_queue(&pxmitpriv->bm_pending);
+
+	/* _rtw_init_queue(&pxmitpriv->legacy_dz_queue); */
+	/* _rtw_init_queue(&pxmitpriv->apsd_queue); */
+
+	_rtw_init_queue(&pxmitpriv->free_xmit_queue);
+
+	/*
+	Please allocate memory with the sz = (struct xmit_frame) * NR_XMITFRAME,
+	and initialize free_xmit_frame below.
+	Please also apply  free_txobj to link_up all the xmit_frames...
+	*/
+
+	pxmitpriv->pallocated_frame_buf = rtw_zvmalloc(NR_XMITFRAME * sizeof(struct xmit_frame) + 4);
+
+	if (pxmitpriv->pallocated_frame_buf  == NULL) {
+		pxmitpriv->pxmit_frame_buf = NULL;
+		res = _FAIL;
+		goto exit;
+	}
+	pxmitpriv->pxmit_frame_buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_frame_buf), 4);
+	/* pxmitpriv->pxmit_frame_buf = pxmitpriv->pallocated_frame_buf + 4 - */
+	/*						((SIZE_PTR) (pxmitpriv->pallocated_frame_buf) &3); */
+
+	pxframe = (struct xmit_frame *) pxmitpriv->pxmit_frame_buf;
+
+	for (i = 0; i < NR_XMITFRAME; i++) {
+		_rtw_init_listhead(&(pxframe->list));
+
+		pxframe->padapter = padapter;
+		pxframe->frame_tag = NULL_FRAMETAG;
+
+		pxframe->pkt = NULL;
+
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		pxframe->buf_addr = NULL;
+		pxframe->pxmitbuf = NULL;
+		#else
+		/*alloc buf_addr*/
+		/*rtw_os_xmit_resource_alloc(padapter, pxframe);*/
+		#endif
+
+		rtw_list_insert_tail(&(pxframe->list), &(pxmitpriv->free_xmit_queue.queue));
+
+		pxframe++;
+	}
+
+	pxmitpriv->free_xmitframe_cnt = NR_XMITFRAME;
+
+	pxmitpriv->frag_len = MAX_FRAG_THRESHOLD;
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	/* init xmit_buf */
+	_rtw_init_queue(&pxmitpriv->free_xmitbuf_queue);
+	_rtw_init_queue(&pxmitpriv->pending_xmitbuf_queue);
+
+	pxmitpriv->pallocated_xmitbuf = rtw_zvmalloc(xmitbuf_nr * sizeof(struct xmit_buf) + 4);
+
+	if (pxmitpriv->pallocated_xmitbuf  == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	pxmitpriv->pxmitbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_xmitbuf), 4);
+	/* pxmitpriv->pxmitbuf = pxmitpriv->pallocated_xmitbuf + 4 - */
+	/*						((SIZE_PTR) (pxmitpriv->pallocated_xmitbuf) &3); */
+
+	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf;
+
+	for (i = 0; i < xmitbuf_nr; i++) {
+		_rtw_init_listhead(&pxmitbuf->list);
+
+		pxmitbuf->priv_data = NULL;
+		pxmitbuf->padapter = padapter;
+		pxmitbuf->buf_tag = XMITBUF_DATA;
+
+		/* Tx buf allocation may fail sometimes, so sleep and retry. */
+		res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf,
+					(xmitbuf_sz + SZ_ALIGN_XMITFRAME_EXT), _TRUE);
+		if (res == _FAIL) {
+			rtw_msleep_os(10);
+			res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf,
+					(xmitbuf_sz + SZ_ALIGN_XMITFRAME_EXT), _TRUE);
+			if (res == _FAIL)
+				goto exit;
+		}
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+		pxmitbuf->phead = pxmitbuf->pbuf;
+		pxmitbuf->pend = pxmitbuf->pbuf + xmitbuf_sz;
+		pxmitbuf->len = 0;
+		pxmitbuf->pdata = pxmitbuf->ptail = pxmitbuf->phead;
+#endif
+
+		pxmitbuf->flags = XMIT_VO_QUEUE;
+
+		rtw_list_insert_tail(&pxmitbuf->list, &(pxmitpriv->free_xmitbuf_queue.queue));
+#ifdef DBG_XMIT_BUF
+		pxmitbuf->no = i;
+#endif
+
+		pxmitbuf++;
+
+	}
+
+	pxmitpriv->free_xmitbuf_cnt = xmitbuf_nr;
+	#endif
+	/* init xframe_ext queue,  the same count as extbuf */
+	_rtw_init_queue(&pxmitpriv->free_xframe_ext_queue);
+#ifdef CONFIG_LAYER2_ROAMING
+	_rtw_init_queue(&pxmitpriv->rpkt_queue);
+#endif
+
+	pxmitpriv->xframe_ext_alloc_addr = rtw_zvmalloc(NR_XMITFRAME_EXT * sizeof(struct xmit_frame) + 4);
+
+	if (pxmitpriv->xframe_ext_alloc_addr  == NULL) {
+		pxmitpriv->xframe_ext = NULL;
+		res = _FAIL;
+		goto exit;
+	}
+	pxmitpriv->xframe_ext = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->xframe_ext_alloc_addr), 4);
+	pxframe = (struct xmit_frame *)pxmitpriv->xframe_ext;
+
+	/* MGT_TXREQ_QMGT */
+	pxmitpriv->xframe_ext_txreq_alloc_addr = rtw_zmalloc(NR_XMITFRAME_EXT * SZ_MGT_RING);
+	if (pxmitpriv->xframe_ext_txreq_alloc_addr  == NULL) {
+		pxmitpriv->xframe_ext_txreq = NULL;
+		res = _FAIL;
+		goto exit;
+	}
+	pxmitpriv->xframe_ext_txreq = pxmitpriv->xframe_ext_txreq_alloc_addr;
+	txreq = pxmitpriv->xframe_ext_txreq;
+	pkt_list = pxmitpriv->xframe_ext_txreq + sizeof(struct rtw_xmit_req);
+
+	for (i = 0; i < NR_XMITFRAME_EXT; i++) {
+		_rtw_init_listhead(&(pxframe->list));
+
+		pxframe->padapter = padapter;
+		pxframe->frame_tag = NULL_FRAMETAG;
+
+		pxframe->pkt = NULL;
+
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		pxframe->buf_addr = NULL;
+		pxframe->pxmitbuf = NULL;
+		#else
+		/*alloc buf_addr*/
+		rtw_os_xmit_resource_alloc(padapter, pxframe);
+		#endif
+
+		pxframe->ext_tag = 1;
+
+		/* MGT_TXREQ_QMGT */
+		pxframe->phl_txreq = (struct rtw_xmit_req *)txreq;
+		pxframe->phl_txreq->pkt_list = pkt_list;
+
+		rtw_list_insert_tail(&(pxframe->list), &(pxmitpriv->free_xframe_ext_queue.queue));
+
+		pxframe++;
+		/* MGT_TXREQ_QMGT */
+		txreq += SZ_MGT_RING;
+		pkt_list += SZ_MGT_RING;
+	}
+	pxmitpriv->free_xframe_ext_cnt = NR_XMITFRAME_EXT;
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+	/* Init xmit extension buff */
+	_rtw_init_queue(&pxmitpriv->free_xmit_extbuf_queue);
+
+	pxmitpriv->pallocated_xmit_extbuf = rtw_zvmalloc(xmitbuf_ext_nr * sizeof(struct xmit_buf) + 4);
+
+	if (pxmitpriv->pallocated_xmit_extbuf  == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	pxmitpriv->pxmit_extbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_xmit_extbuf), 4);
+
+	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmit_extbuf;
+
+	for (i = 0; i < xmitbuf_ext_nr; i++) {
+		_rtw_init_listhead(&pxmitbuf->list);
+
+		pxmitbuf->priv_data = NULL;
+		pxmitbuf->padapter = padapter;
+		pxmitbuf->buf_tag = XMITBUF_MGNT;
+
+		res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf,
+					xmitbuf_ext_sz + SZ_ALIGN_XMITFRAME_EXT, _TRUE);
+		if (res == _FAIL) {
+			res = _FAIL;
+			goto exit;
+		}
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+		pxmitbuf->phead = pxmitbuf->pbuf;
+		pxmitbuf->pend = pxmitbuf->pbuf + xmitbuf_ext_sz;
+		pxmitbuf->len = 0;
+		pxmitbuf->pdata = pxmitbuf->ptail = pxmitbuf->phead;
+#endif
+
+		rtw_list_insert_tail(&pxmitbuf->list, &(pxmitpriv->free_xmit_extbuf_queue.queue));
+#ifdef DBG_XMIT_BUF_EXT
+		pxmitbuf->no = i;
+#endif
+		pxmitbuf++;
+
+	}
+
+	pxmitpriv->free_xmit_extbuf_cnt = xmitbuf_ext_nr;
+
+	/*GEORGIA_TODO_FIXIT_IC_GEN_DEPENDENCE*/
+	for (i = 0; i < CMDBUF_MAX; i++) {
+		pxmitbuf = &pxmitpriv->pcmd_xmitbuf[i];
+		if (pxmitbuf) {
+			_rtw_init_listhead(&pxmitbuf->list);
+
+			pxmitbuf->priv_data = NULL;
+			pxmitbuf->padapter = padapter;
+			pxmitbuf->buf_tag = XMITBUF_CMD;
+
+			res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf,
+					MAX_CMDBUF_SZ + SZ_ALIGN_XMITFRAME_EXT, _TRUE);
+			if (res == _FAIL) {
+				res = _FAIL;
+				goto exit;
+			}
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+			pxmitbuf->phead = pxmitbuf->pbuf;
+			pxmitbuf->pend = pxmitbuf->pbuf + MAX_CMDBUF_SZ;
+			pxmitbuf->len = 0;
+			pxmitbuf->pdata = pxmitbuf->ptail = pxmitbuf->phead;
+#endif
+			pxmitbuf->alloc_sz = MAX_CMDBUF_SZ + SZ_ALIGN_XMITFRAME_EXT;
+		}
+	}
+#endif
+	rtw_alloc_hwxmits(padapter);
+	rtw_init_hwxmits(pxmitpriv->hwxmits, pxmitpriv->hwxmit_entry);
+
+	for (i = 0; i < 4; i++)
+		pxmitpriv->wmm_para_seq[i] = i;
+
+#ifdef CONFIG_USB_HCI
+	pxmitpriv->txirp_cnt = 1;
+
+	_rtw_init_sema(&(pxmitpriv->tx_retevt), 0);
+
+	/* per AC pending irp */
+	pxmitpriv->beq_cnt = 0;
+	pxmitpriv->bkq_cnt = 0;
+	pxmitpriv->viq_cnt = 0;
+	pxmitpriv->voq_cnt = 0;
+#ifdef PRIVATE_R
+	pxmitpriv->tx_bk_drop_cnt = 0;
+	pxmitpriv->tx_vi_drop_cnt = 0;
+	pxmitpriv->tx_vo_drop_cnt = 0;
+	pxmitpriv->tx_be_drop_cnt = 0;
+#endif
+#endif
+
+
+#ifdef CONFIG_XMIT_ACK
+	pxmitpriv->ack_tx = _FALSE;
+	_rtw_mutex_init(&pxmitpriv->ack_tx_mutex);
+	rtw_sctx_init(&pxmitpriv->ack_tx_ops, 0);
+	#ifdef CONFIG_XMIT_ACK_BY_REL_RPT
+	rtw_ack_txfb_init(padapter, &(pxmitpriv->ack_txfb));
+	#endif
+#endif
+
+#ifdef CONFIG_TX_AMSDU
+	rtw_init_timer(&(pxmitpriv->amsdu_vo_timer),
+		rtw_amsdu_vo_timeout_handler, padapter);
+	pxmitpriv->amsdu_vo_timeout = RTW_AMSDU_TIMER_UNSET;
+
+	rtw_init_timer(&(pxmitpriv->amsdu_vi_timer),
+		rtw_amsdu_vi_timeout_handler, padapter);
+	pxmitpriv->amsdu_vi_timeout = RTW_AMSDU_TIMER_UNSET;
+
+	rtw_init_timer(&(pxmitpriv->amsdu_be_timer),
+		rtw_amsdu_be_timeout_handler, padapter);
+	pxmitpriv->amsdu_be_timeout = RTW_AMSDU_TIMER_UNSET;
+
+	rtw_init_timer(&(pxmitpriv->amsdu_bk_timer),
+		rtw_amsdu_bk_timeout_handler, padapter);
+	pxmitpriv->amsdu_bk_timeout = RTW_AMSDU_TIMER_UNSET;
+
+	pxmitpriv->amsdu_debug_set_timer = 0;
+	pxmitpriv->amsdu_debug_timeout = 0;
+	pxmitpriv->amsdu_debug_tasklet = 0;
+	pxmitpriv->amsdu_debug_enqueue = 0;
+	pxmitpriv->amsdu_debug_dequeue = 0;
+	for (i = 0; i < AMSDU_DEBUG_MAX_COUNT; i++)
+		pxmitpriv->amsdu_debug_coalesce[i] = 0;
+#endif
+#ifdef DBG_TXBD_DESC_DUMP
+	pxmitpriv->dump_txbd_desc = 0;
+#endif
+	rtw_init_xmit_block(padapter);
+#ifdef CONFIG_TX_AMSDU_SW_MODE
+	rtw_tasklet_init(&padapter->xmitpriv.xmit_tasklet,
+		         core_tx_amsdu_tasklet, (unsigned long)padapter);
+#endif
+	rtw_intf_init_xmit_priv(padapter);
+
+#ifdef RTW_PHL_TX //alloc xmit resource
+	printk("eric-tx CALL alloc_txring !!!!\n");
+	if (alloc_txring(padapter) == _FAIL) {
+		RTW_ERR("[core] alloc_txring fail !!!\n");
+		res = _FAIL;
+		goto exit;
+	}
+#endif
+
+#if defined(CONFIG_CORE_TXSC)
+	_rtw_spinlock_init(&pxmitpriv->txsc_lock);
+#endif
+
+exit:
+
+	return res;
+}
+
+void  rtw_mfree_xmit_priv_lock(struct xmit_priv *pxmitpriv)
+{
+	_rtw_spinlock_free(&pxmitpriv->lock);
+	#if 0 /*def CONFIG_XMIT_THREAD_MODE*/
+	_rtw_free_sema(&pxmitpriv->xmit_sema);
+	#endif
+
+	_rtw_spinlock_free(&pxmitpriv->be_pending.lock);
+	_rtw_spinlock_free(&pxmitpriv->bk_pending.lock);
+	_rtw_spinlock_free(&pxmitpriv->vi_pending.lock);
+	_rtw_spinlock_free(&pxmitpriv->vo_pending.lock);
+	_rtw_spinlock_free(&pxmitpriv->bm_pending.lock);
+
+	/* _rtw_spinlock_free(&pxmitpriv->legacy_dz_queue.lock); */
+	/* _rtw_spinlock_free(&pxmitpriv->apsd_queue.lock); */
+
+	_rtw_spinlock_free(&pxmitpriv->free_xmit_queue.lock);
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	_rtw_spinlock_free(&pxmitpriv->free_xmitbuf_queue.lock);
+	_rtw_spinlock_free(&pxmitpriv->pending_xmitbuf_queue.lock);
+	#endif
+}
+
+
+void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv)
+{
+	int i;
+	_adapter *padapter = pxmitpriv->adapter;
+	struct xmit_frame *pxmitframe;
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct xmit_buf *pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf;
+	#endif
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u8 xmitbuf_nr = GET_HAL_XMITBUF_NR(dvobj);
+	u16 xmitbuf_sz = GET_HAL_XMITBUF_SZ(dvobj);
+
+	u8 xmitbuf_ext_nr = GET_HAL_XMITBUF_EXT_NR(dvobj);
+	u16 xmitbuf_ext_sz = GET_HAL_XMITBUF_EXT_SZ(dvobj);
+	#endif
+
+	rtw_intf_free_xmit_priv(padapter);
+
+	rtw_mfree_xmit_priv_lock(pxmitpriv);
+
+	if (pxmitpriv->pxmit_frame_buf == NULL)
+		goto out;
+
+	pxmitframe = (struct xmit_frame *) pxmitpriv->pxmit_frame_buf;
+
+	for (i = 0; i < NR_XMITFRAME; i++) {
+		rtw_os_xmit_complete(padapter, pxmitframe);
+		/*alloc buf_addr*/
+		/*rtw_os_xmit_resource_free(padapter, pxmitframe);*/
+		pxmitframe++;
+	}
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	for (i = 0; i < xmitbuf_nr; i++) {
+		rtw_os_xmit_resource_free(padapter, pxmitbuf,
+			(xmitbuf_sz + SZ_ALIGN_XMITFRAME_EXT), _TRUE);
+
+		pxmitbuf++;
+	}
+	#endif
+	if (pxmitpriv->pallocated_frame_buf)
+		rtw_vmfree(pxmitpriv->pallocated_frame_buf,
+			NR_XMITFRAME * sizeof(struct xmit_frame) + 4);
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	if (pxmitpriv->pallocated_xmitbuf)
+		rtw_vmfree(pxmitpriv->pallocated_xmitbuf,
+			xmitbuf_nr * sizeof(struct xmit_buf) + 4);
+	#endif
+
+	/* free xframe_ext queue,  the same count as extbuf */
+	if (pxmitpriv->xframe_ext == NULL)
+		goto out;
+
+	pxmitframe = (struct xmit_frame *)pxmitpriv->xframe_ext;
+	for (i = 0; i < NR_XMITFRAME_EXT; i++) {
+		rtw_os_xmit_complete(padapter, pxmitframe);
+		/*free buf_addr*/
+		rtw_os_xmit_resource_free(padapter, pxmitframe);
+		pxmitframe++;
+	}
+
+	if (pxmitpriv->xframe_ext_alloc_addr)
+		rtw_vmfree(pxmitpriv->xframe_ext_alloc_addr,
+			NR_XMITFRAME_EXT * sizeof(struct xmit_frame) + 4);
+	_rtw_spinlock_free(&pxmitpriv->free_xframe_ext_queue.lock);
+
+	if (pxmitpriv->xframe_ext_txreq_alloc_addr)
+		rtw_mfree(pxmitpriv->xframe_ext_txreq_alloc_addr, NR_XMITFRAME_EXT * SZ_MGT_RING);
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+
+	/* free xmit extension buff */
+	_rtw_spinlock_free(&pxmitpriv->free_xmit_extbuf_queue.lock);
+
+	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmit_extbuf;
+	for (i = 0; i < xmitbuf_ext_nr; i++) {
+		rtw_os_xmit_resource_free(padapter, pxmitbuf,
+			(xmitbuf_ext_sz + SZ_ALIGN_XMITFRAME_EXT), _TRUE);
+
+		pxmitbuf++;
+	}
+
+	if (pxmitpriv->pallocated_xmit_extbuf)
+		rtw_vmfree(pxmitpriv->pallocated_xmit_extbuf,
+			xmitbuf_ext_nr * sizeof(struct xmit_buf) + 4);
+
+	for (i = 0; i < CMDBUF_MAX; i++) {
+		pxmitbuf = &pxmitpriv->pcmd_xmitbuf[i];
+		if (pxmitbuf != NULL)
+			rtw_os_xmit_resource_free(padapter, pxmitbuf, MAX_CMDBUF_SZ + SZ_ALIGN_XMITFRAME_EXT, _TRUE);
+	}
+	#endif
+	rtw_free_hwxmits(padapter);
+
+#ifdef CONFIG_LAYER2_ROAMING
+	_rtw_spinlock_free(&pxmitpriv->rpkt_queue.lock);
+#endif
+
+#ifdef RTW_PHL_TX
+	free_txring(padapter);
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+	txsc_clear(padapter);
+	_rtw_spinlock_free(&pxmitpriv->txsc_lock);
+#endif
+
+#ifdef CONFIG_XMIT_ACK
+	_rtw_mutex_free(&pxmitpriv->ack_tx_mutex);
+#endif
+	rtw_free_xmit_block(padapter);
+out:
+	return;
+}
+
+#ifdef CONFIG_USB_HCI
+u8 rtw_init_lite_xmit_resource(struct dvobj_priv *dvobj)
+{
+
+	u8 ret = _SUCCESS;
+/*YiWei_todo need use correct litexmitbuf_nr urb_nr*/
+	u32 litexmitbuf_nr = RTW_LITEXMITBUF_NR;
+	u32 litexmitbuf_ext_nr = RTW_LITEXMITBUF_NR;
+	struct lite_data_buf *litexmitbuf;
+	struct trx_data_buf_q  *litexmitbuf_q = &dvobj->litexmitbuf_q;
+	struct trx_data_buf_q  *litexmit_extbuf_q = &dvobj->litexmit_extbuf_q;
+	int i;
+	struct trx_urb_buf_q *xmit_urb_q = &dvobj->xmit_urb_q;
+	struct data_urb *xmiturb;
+	u32 urb_nr = RTW_XMITURB_NR;
+
+	/* init lite_xmit_buf */
+	_rtw_init_queue(&litexmitbuf_q->free_data_buf_queue);
+
+	litexmitbuf_q->alloc_data_buf =
+		rtw_zvmalloc(litexmitbuf_nr * sizeof(struct lite_data_buf) + 4);
+
+	if (litexmitbuf_q->alloc_data_buf  == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	litexmitbuf_q->data_buf =
+		(u8 *)N_BYTE_ALIGNMENT((SIZE_PTR)(litexmitbuf_q->alloc_data_buf), 4);
+
+	litexmitbuf = (struct lite_data_buf *)litexmitbuf_q->data_buf;
+
+	for (i = 0; i < litexmitbuf_nr; i++) {
+		_rtw_init_listhead(&litexmitbuf->list);
+		rtw_list_insert_tail(&litexmitbuf->list,
+			&(litexmitbuf_q->free_data_buf_queue.queue));
+		litexmitbuf++;
+	}
+	litexmitbuf_q->free_data_buf_cnt = litexmitbuf_nr;
+
+
+	/* Init lite xmit extension buff */
+	_rtw_init_queue(&litexmit_extbuf_q->free_data_buf_queue);
+
+	litexmit_extbuf_q->alloc_data_buf =
+		rtw_zvmalloc(litexmitbuf_ext_nr * sizeof(struct lite_data_buf) + 4);
+
+	if (litexmit_extbuf_q->alloc_data_buf  == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	litexmit_extbuf_q->data_buf =
+		(u8 *)N_BYTE_ALIGNMENT((SIZE_PTR)(litexmit_extbuf_q->alloc_data_buf), 4);
+
+	litexmitbuf = (struct lite_data_buf *)litexmit_extbuf_q->data_buf;
+
+	for (i = 0; i < litexmitbuf_ext_nr; i++) {
+		_rtw_init_listhead(&litexmitbuf->list);
+		rtw_list_insert_tail(&litexmitbuf->list,
+			&(litexmit_extbuf_q->free_data_buf_queue.queue));
+		litexmitbuf++;
+	}
+	litexmit_extbuf_q->free_data_buf_cnt = litexmitbuf_ext_nr;
+
+	/* init xmit_urb */
+	_rtw_init_queue(&xmit_urb_q->free_urb_buf_queue);
+	xmit_urb_q->alloc_urb_buf =
+		rtw_zvmalloc(urb_nr * sizeof(struct data_urb) + 4);
+	if (xmit_urb_q->alloc_urb_buf == NULL) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	xmit_urb_q->urb_buf =
+		(u8 *)N_BYTE_ALIGNMENT((SIZE_PTR)(xmit_urb_q->alloc_urb_buf), 4);
+
+	xmiturb = (struct data_urb *)xmit_urb_q->urb_buf;
+	for (i = 0; i < urb_nr; i++) {
+		_rtw_init_listhead(&xmiturb->list);
+		ret = rtw_os_urb_resource_alloc(xmiturb);
+		rtw_list_insert_tail(&xmiturb->list,
+			&(xmit_urb_q->free_urb_buf_queue.queue));
+		xmiturb++;
+	}
+	xmit_urb_q->free_urb_buf_cnt = urb_nr;
+
+exit:
+	return ret;
+}
+
+void rtw_free_lite_xmit_resource(struct dvobj_priv *dvobj)
+{
+	u8 ret = _SUCCESS;
+/*YiWei_todo need use correct litexmitbuf_nr urb_nr*/
+	u32 litexmitbuf_nr = RTW_LITEXMITBUF_NR;
+	u32 litexmitbuf_ext_nr = RTW_LITEXMITBUF_NR;
+	struct trx_data_buf_q  *litexmitbuf_q = &dvobj->litexmitbuf_q;
+	struct trx_data_buf_q  *litexmit_extbuf_q = &dvobj->litexmit_extbuf_q;
+	struct data_urb *xmiturb;
+	struct trx_urb_buf_q *xmit_urb_q = &dvobj->xmit_urb_q;
+	u32 urb_nr = RTW_XMITURB_NR;
+	int i;
+
+	if (litexmitbuf_q->alloc_data_buf)
+		rtw_vmfree(litexmitbuf_q->alloc_data_buf,
+			litexmitbuf_nr * sizeof(struct lite_data_buf) + 4);
+
+	if (litexmit_extbuf_q->alloc_data_buf)
+		rtw_vmfree(litexmit_extbuf_q->alloc_data_buf,
+			litexmitbuf_ext_nr * sizeof(struct lite_data_buf) + 4);
+
+	xmiturb = (struct data_urb *)xmit_urb_q->urb_buf;
+	for (i = 0; i < urb_nr; i++) {
+		rtw_os_urb_resource_free(xmiturb);
+		xmiturb++;
+	}
+
+	if (xmit_urb_q->alloc_urb_buf)
+		rtw_vmfree(xmit_urb_q->alloc_urb_buf,
+			urb_nr * sizeof(struct data_urb) + 4);
+}
+#endif
+
+u8 rtw_get_tx_bw_mode(_adapter *adapter, struct sta_info *sta)
+{
+	u8 bw;
+	struct _ADAPTER_LINK *adapter_link = sta->padapter_link;
+
+	bw = sta->phl_sta->chandef.bw;
+	if (MLME_STATE(adapter) & WIFI_ASOC_STATE) {
+		switch (adapter_link->mlmeextpriv.chandef.band) {
+		case BAND_ON_6G:
+			bw = rtw_min(bw, ADAPTER_TX_BW_6G(adapter));
+			break;
+		case BAND_ON_5G:
+			bw = rtw_min(bw, ADAPTER_TX_BW_5G(adapter));
+			break;
+		case BAND_ON_24G:
+		default:
+			bw = rtw_min(bw, ADAPTER_TX_BW_2G(adapter));
+			break;
+		}
+	}
+
+	return bw;
+}
+
+void rtw_get_adapter_tx_rate_bmp_by_bw(_adapter *adapter, u8 bw, u16 *r_bmp_cck_ofdm, u32 *r_bmp_ht, u64 *r_bmp_vht)
+{
+/* ToDo */
+#if 0
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj);
+	u8 fix_bw = 0xFF;
+	u16 bmp_cck_ofdm = 0;
+	u32 bmp_ht = 0;
+	u64 bmp_vht = 0;
+	int i;
+
+	if (adapter->fix_rate != NO_FIX_RATE && adapter->fix_bw != NO_FIX_BW)
+		fix_bw = adapter->fix_bw;
+
+	/* TODO: adapter->fix_rate */
+
+	for (i = 0; i < macid_ctl->num; i++) {
+		if (!rtw_macid_is_used(macid_ctl, i))
+			continue;
+		if (!rtw_macid_is_iface_specific(macid_ctl, i, adapter))
+			continue;
+
+		if (bw == CHANNEL_WIDTH_20) /* CCK, OFDM always 20MHz */
+			bmp_cck_ofdm |= macid_ctl->rate_bmp0[i] & 0x00000FFF;
+
+		/* bypass mismatch bandwidth for HT, VHT */
+		if ((fix_bw != 0xFF && fix_bw != bw) || (fix_bw == 0xFF && macid_ctl->bw[i] != bw))
+			continue;
+
+		if (macid_ctl->vht_en[i])
+			bmp_vht |= (macid_ctl->rate_bmp0[i] >> 12) | (macid_ctl->rate_bmp1[i] << 20);
+		else
+			bmp_ht |= (macid_ctl->rate_bmp0[i] >> 12) | (macid_ctl->rate_bmp1[i] << 20);
+	}
+
+	/* TODO: mlmeext->tx_rate*/
+
+	if (r_bmp_cck_ofdm)
+		*r_bmp_cck_ofdm = bmp_cck_ofdm;
+	if (r_bmp_ht)
+		*r_bmp_ht = bmp_ht;
+	if (r_bmp_vht)
+		*r_bmp_vht = bmp_vht;
+#endif
+}
+
+void rtw_get_shared_macid_tx_rate_bmp_by_bw(struct dvobj_priv *dvobj, u8 bw, u16 *r_bmp_cck_ofdm, u32 *r_bmp_ht, u64 *r_bmp_vht)
+{
+/* ToDo */
+#if 0
+	struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj);
+	u16 bmp_cck_ofdm = 0;
+	u32 bmp_ht = 0;
+	u64 bmp_vht = 0;
+	int i;
+
+	for (i = 0; i < macid_ctl->num; i++) {
+		if (!rtw_macid_is_used(macid_ctl, i))
+			continue;
+		if (!rtw_macid_is_iface_shared(macid_ctl, i))
+			continue;
+
+		if (bw == CHANNEL_WIDTH_20) /* CCK, OFDM always 20MHz */
+			bmp_cck_ofdm |= macid_ctl->rate_bmp0[i] & 0x00000FFF;
+
+		/* bypass mismatch bandwidth for HT, VHT */
+		if (macid_ctl->bw[i] != bw)
+			continue;
+
+		if (macid_ctl->vht_en[i])
+			bmp_vht |= (macid_ctl->rate_bmp0[i] >> 12) | (macid_ctl->rate_bmp1[i] << 20);
+		else
+			bmp_ht |= (macid_ctl->rate_bmp0[i] >> 12) | (macid_ctl->rate_bmp1[i] << 20);
+	}
+
+	if (r_bmp_cck_ofdm)
+		*r_bmp_cck_ofdm = bmp_cck_ofdm;
+	if (r_bmp_ht)
+		*r_bmp_ht = bmp_ht;
+	if (r_bmp_vht)
+		*r_bmp_vht = bmp_vht;
+#endif
+}
+
+void rtw_update_tx_rate_bmp(struct dvobj_priv *dvobj)
+{
+#if 0 /*GEORGIA_TODO_FIXIT*/
+
+	struct rf_ctl_t *rf_ctl = dvobj_to_rfctl(dvobj);
+	_adapter *adapter = dvobj_get_primary_adapter(dvobj);
+	HAL_DATA_TYPE *hal_data = GET_PHL_COM(dvobj);
+	u8 bw;
+	u16 bmp_cck_ofdm, tmp_cck_ofdm;
+	u32 bmp_ht, tmp_ht, ori_bmp_ht[2];
+	u64 bmp_vht, tmp_vht, ori_bmp_vht[4];
+	int i;
+
+	for (bw = CHANNEL_WIDTH_20; bw <= CHANNEL_WIDTH_160; bw++) {
+		/* backup the original ht & vht bmp */
+		if (bw <= CHANNEL_WIDTH_40)
+			ori_bmp_ht[bw] = rf_ctl->rate_bmp_ht_by_bw[bw];
+		if (bw <= CHANNEL_WIDTH_160)
+			ori_bmp_vht[bw] = rf_ctl->rate_bmp_vht_by_bw[bw];
+
+		bmp_cck_ofdm = bmp_ht = bmp_vht = 0;
+		if (rtw_hw_is_bw_support(dvobj, bw)) {
+			for (i = 0; i < dvobj->iface_nums; i++) {
+				if (!dvobj->padapters[i])
+					continue;
+				rtw_get_adapter_tx_rate_bmp_by_bw(dvobj->padapters[i], bw, &tmp_cck_ofdm, &tmp_ht, &tmp_vht);
+				bmp_cck_ofdm |= tmp_cck_ofdm;
+				bmp_ht |= tmp_ht;
+				bmp_vht |= tmp_vht;
+			}
+			rtw_get_shared_macid_tx_rate_bmp_by_bw(dvobj, bw, &tmp_cck_ofdm, &tmp_ht, &tmp_vht);
+			bmp_cck_ofdm |= tmp_cck_ofdm;
+			bmp_ht |= tmp_ht;
+			bmp_vht |= tmp_vht;
+		}
+		if (bw == CHANNEL_WIDTH_20)
+			rf_ctl->rate_bmp_cck_ofdm = bmp_cck_ofdm;
+		if (bw <= CHANNEL_WIDTH_40)
+			rf_ctl->rate_bmp_ht_by_bw[bw] = bmp_ht;
+		if (bw <= CHANNEL_WIDTH_160)
+			rf_ctl->rate_bmp_vht_by_bw[bw] = bmp_vht;
+	}
+
+#if CONFIG_TXPWR_LIMIT
+#ifndef DBG_HIGHEST_RATE_BMP_BW_CHANGE
+#define DBG_HIGHEST_RATE_BMP_BW_CHANGE 0
+#endif
+
+	if (hal_data->txpwr_limit_loaded) {
+		u8 ori_highest_ht_rate_bw_bmp;
+		u8 ori_highest_vht_rate_bw_bmp;
+		u8 highest_rate_bw;
+		u8 highest_rate_bw_bmp;
+		u8 update_ht_rs = _FALSE;
+		u8 update_vht_rs = _FALSE;
+
+		/* backup the original ht & vht highest bw bmp */
+		ori_highest_ht_rate_bw_bmp = rf_ctl->highest_ht_rate_bw_bmp;
+		ori_highest_vht_rate_bw_bmp = rf_ctl->highest_vht_rate_bw_bmp;
+
+		highest_rate_bw_bmp = BW_CAP_20M;
+		highest_rate_bw = CHANNEL_WIDTH_20;
+		for (bw = CHANNEL_WIDTH_20; bw <= CHANNEL_WIDTH_40; bw++) {
+			if (rf_ctl->rate_bmp_ht_by_bw[highest_rate_bw] < rf_ctl->rate_bmp_ht_by_bw[bw]) {
+				highest_rate_bw_bmp = ch_width_to_bw_cap(bw);
+				highest_rate_bw = bw;
+			} else if (rf_ctl->rate_bmp_ht_by_bw[highest_rate_bw] == rf_ctl->rate_bmp_ht_by_bw[bw])
+				highest_rate_bw_bmp |= ch_width_to_bw_cap(bw);
+		}
+		rf_ctl->highest_ht_rate_bw_bmp = highest_rate_bw_bmp;
+
+		if (ori_highest_ht_rate_bw_bmp != rf_ctl->highest_ht_rate_bw_bmp
+			|| largest_bit(ori_bmp_ht[highest_rate_bw]) != largest_bit(rf_ctl->rate_bmp_ht_by_bw[highest_rate_bw])
+		) {
+			if (DBG_HIGHEST_RATE_BMP_BW_CHANGE) {
+				RTW_INFO("highest_ht_rate_bw_bmp:0x%02x=>0x%02x\n", ori_highest_ht_rate_bw_bmp, rf_ctl->highest_ht_rate_bw_bmp);
+				RTW_INFO("rate_bmp_ht_by_bw[%u]:0x%08x=>0x%08x\n", highest_rate_bw, ori_bmp_ht[highest_rate_bw], rf_ctl->rate_bmp_ht_by_bw[highest_rate_bw]);
+			}
+			if (rf_ctl->rate_bmp_ht_by_bw[highest_rate_bw])
+				update_ht_rs = _TRUE;
+		}
+
+		highest_rate_bw_bmp = BW_CAP_20M;
+		highest_rate_bw = CHANNEL_WIDTH_20;
+		for (bw = CHANNEL_WIDTH_20; bw <= CHANNEL_WIDTH_160; bw++) {
+			if (rf_ctl->rate_bmp_vht_by_bw[highest_rate_bw] < rf_ctl->rate_bmp_vht_by_bw[bw]) {
+				highest_rate_bw_bmp = ch_width_to_bw_cap(bw);
+				highest_rate_bw = bw;
+			} else if (rf_ctl->rate_bmp_vht_by_bw[highest_rate_bw] == rf_ctl->rate_bmp_vht_by_bw[bw])
+				highest_rate_bw_bmp |= ch_width_to_bw_cap(bw);
+		}
+		rf_ctl->highest_vht_rate_bw_bmp = highest_rate_bw_bmp;
+
+		if (ori_highest_vht_rate_bw_bmp != rf_ctl->highest_vht_rate_bw_bmp
+			|| largest_bit_64(ori_bmp_vht[highest_rate_bw]) != largest_bit_64(rf_ctl->rate_bmp_vht_by_bw[highest_rate_bw])
+		) {
+			if (DBG_HIGHEST_RATE_BMP_BW_CHANGE) {
+				RTW_INFO("highest_vht_rate_bw_bmp:0x%02x=>0x%02x\n", ori_highest_vht_rate_bw_bmp, rf_ctl->highest_vht_rate_bw_bmp);
+				RTW_INFO("rate_bmp_vht_by_bw[%u]:0x%016llx=>0x%016llx\n", highest_rate_bw, ori_bmp_vht[highest_rate_bw], rf_ctl->rate_bmp_vht_by_bw[highest_rate_bw]);
+			}
+			if (rf_ctl->rate_bmp_vht_by_bw[highest_rate_bw])
+				update_vht_rs = _TRUE;
+		}
+
+		/* TODO: per rfpath and rate section handling? */
+		if (update_ht_rs == _TRUE || update_vht_rs == _TRUE)
+			rtw_hal_set_tx_power_level(dvobj_get_primary_adapter(dvobj), hal_data->current_channel);
+	}
+#endif /* CONFIG_TXPWR_LIMIT */
+#endif
+}
+
+u8 rtw_get_tx_bw_bmp_of_ht_rate(struct dvobj_priv *dvobj, u8 rate, u8 max_bw)
+{
+	struct rf_ctl_t *rf_ctl = dvobj_to_rfctl(dvobj);
+	u8 bw;
+	u8 bw_bmp = 0;
+	u32 rate_bmp;
+
+	if (!IS_HT_RATE(rate)) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	rate_bmp = 1 << (rate - MGN_MCS0);
+
+	if (max_bw > CHANNEL_WIDTH_40)
+		max_bw = CHANNEL_WIDTH_40;
+
+	for (bw = CHANNEL_WIDTH_20; bw <= max_bw; bw++) {
+		/* RA may use lower rate for retry */
+		if (rf_ctl->rate_bmp_ht_by_bw[bw] >= rate_bmp)
+			bw_bmp |= ch_width_to_bw_cap(bw);
+	}
+
+exit:
+	return bw_bmp;
+}
+
+u8 rtw_get_tx_bw_bmp_of_vht_rate(struct dvobj_priv *dvobj, u8 rate, u8 max_bw)
+{
+	struct rf_ctl_t *rf_ctl = dvobj_to_rfctl(dvobj);
+	u8 bw;
+	u8 bw_bmp = 0;
+	u64 rate_bmp;
+
+	if (!IS_VHT_RATE(rate)) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	rate_bmp = BIT_ULL(rate - MGN_VHT1SS_MCS0);
+
+	if (max_bw > CHANNEL_WIDTH_160)
+		max_bw = CHANNEL_WIDTH_160;
+
+	for (bw = CHANNEL_WIDTH_20; bw <= max_bw; bw++) {
+		/* RA may use lower rate for retry */
+		if (rf_ctl->rate_bmp_vht_by_bw[bw] >= rate_bmp)
+			bw_bmp |= ch_width_to_bw_cap(bw);
+	}
+
+exit:
+	return bw_bmp;
+}
+
+s16 rtw_rfctl_get_oper_txpwr_max_mbm(struct rf_ctl_t *rfctl, u8 ch, u8 bw, u8 offset, u8 ifbmp_mod, u8 if_op, bool eirp)
+{
+	/* TODO: get maximum txpower of current operating class & channel belongs to this radio */
+	s16 mbm = 2000;
+	return mbm;
+}
+
+s16 rtw_rfctl_get_reg_max_txpwr_mbm(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset, bool eirp)
+{
+	/* TODO: get maximum txpower of current operating class & channel belongs to this radio allowed by regulatory */
+	s16 mbm = 1300;
+	return mbm;
+}
+
+u8 query_ra_short_GI(struct sta_info *psta, u8 bw)
+{
+	u8	sgi = _FALSE, sgi_20m = _FALSE, sgi_40m = _FALSE;
+	u8	sgi_80m = _FALSE, sgi_160m = _FALSE;
+
+#ifdef CONFIG_80211N_HT
+#ifdef CONFIG_80211AC_VHT
+#ifdef CONFIG_80211AX_HE
+	/* CONFIG_80211AX_HE_TODO */
+#endif /* CONFIG_80211AX_HE */
+	if (psta->vhtpriv.vht_option) {
+		sgi_80m = psta->vhtpriv.sgi_80m;
+		sgi_160m = psta->vhtpriv.sgi_160m;
+	}
+#endif
+	sgi_20m = psta->htpriv.sgi_20m;
+	sgi_40m = psta->htpriv.sgi_40m;
+#endif
+
+	switch (bw) {
+	case CHANNEL_WIDTH_160:
+		sgi = sgi_160m;
+		break;
+	case CHANNEL_WIDTH_80:
+		sgi = sgi_80m;
+		break;
+	case CHANNEL_WIDTH_40:
+		sgi = sgi_40m;
+		break;
+	case CHANNEL_WIDTH_20:
+	default:
+		sgi = sgi_20m;
+		break;
+	}
+
+	return sgi;
+}
+
+static void update_attrib_vcs_info(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	u32	sz;
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	/* struct sta_info	*psta = pattrib->psta; */
+	struct _ADAPTER_LINK		*padapter_link = pattrib->adapter_link;
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	/*
+		if(pattrib->psta)
+		{
+			psta = pattrib->psta;
+		}
+		else
+		{
+			RTW_INFO("%s, call rtw_get_stainfo()\n", __func__);
+			psta=rtw_get_stainfo(&padapter->stapriv ,&pattrib->ra[0] );
+		}
+
+		if(psta==NULL)
+		{
+			RTW_INFO("%s, psta==NUL\n", __func__);
+			return;
+		}
+
+		if(!(psta->state &WIFI_ASOC_STATE))
+		{
+			RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+			return;
+		}
+	*/
+
+	if (pattrib->nr_frags != 1)
+		sz = padapter->xmitpriv.frag_len;
+	else /* no frag */
+		sz = pattrib->last_txcmdsz;
+
+	/* (1) RTS_Threshold is compared to the MPDU, not MSDU. */
+	/* (2) If there are more than one frag in  this MSDU, only the first frag uses protection frame. */
+	/*		Other fragments are protected by previous fragment. */
+	/*		So we only need to check the length of first fragment. */
+	if (pmlmeext->cur_wireless_mode < WLAN_MD_11N  || padapter->registrypriv.wifi_spec) {
+		if (sz > padapter->registrypriv.rts_thresh)
+			pattrib->vcs_mode = RTS_CTS;
+		else {
+			if (pattrib->rtsen)
+				pattrib->vcs_mode = RTS_CTS;
+			else if (pattrib->cts2self)
+				pattrib->vcs_mode = CTS_TO_SELF;
+			else
+				pattrib->vcs_mode = NONE_VCS;
+		}
+	} else {
+		while (_TRUE) {
+#if 0 /* Todo */
+			/* check IOT action */
+			if (pHTInfo->IOTAction & HT_IOT_ACT_FORCED_CTS2SELF) {
+				pattrib->vcs_mode = CTS_TO_SELF;
+				pattrib->rts_rate = MGN_24M;
+				break;
+			} else if (pHTInfo->IOTAction & (HT_IOT_ACT_FORCED_RTS | HT_IOT_ACT_PURE_N_MODE)) {
+				pattrib->vcs_mode = RTS_CTS;
+				pattrib->rts_rate = MGN_24M;
+				break;
+			}
+#endif
+
+			/* IOT action */
+			if ((padapter->mlmeextpriv.mlmext_info.assoc_AP_vendor == HT_IOT_PEER_ATHEROS) && (pattrib->ampdu_en == _TRUE) &&
+				(padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)) {
+				pattrib->vcs_mode = CTS_TO_SELF;
+				break;
+			}
+
+
+			/* check ERP protection */
+			if (pattrib->rtsen || pattrib->cts2self) {
+				if (pattrib->rtsen)
+					pattrib->vcs_mode = RTS_CTS;
+				else if (pattrib->cts2self)
+					pattrib->vcs_mode = CTS_TO_SELF;
+
+				break;
+			}
+
+			/* check HT op mode */
+			if (pattrib->ht_en) {
+				u8 HTOpMode = pmlmeinfo->HT_protection;
+				if ((pmlmeext->chandef.bw && (HTOpMode == 2 || HTOpMode == 3)) ||
+				    (!pmlmeext->chandef.bw && HTOpMode == 3)) {
+					pattrib->vcs_mode = RTS_CTS;
+					break;
+				}
+			}
+
+			/* check rts */
+			if (sz > padapter->registrypriv.rts_thresh) {
+				pattrib->vcs_mode = RTS_CTS;
+				break;
+			}
+
+			/* to do list: check MIMO power save condition. */
+
+			/* check AMPDU aggregation for TXOP */
+			if (pattrib->ampdu_en == _TRUE) {
+				pattrib->vcs_mode = RTS_CTS;
+				break;
+			}
+
+			pattrib->vcs_mode = NONE_VCS;
+			break;
+		}
+	}
+
+	/* for debug : force driver control vrtl_carrier_sense. */
+	if (padapter->driver_vcs_en == 1) {
+		/* u8 driver_vcs_en; */ /* Enable=1, Disable=0 driver control vrtl_carrier_sense. */
+		/* u8 driver_vcs_type; */ /* force 0:disable VCS, 1:RTS-CTS, 2:CTS-to-self when vcs_en=1. */
+		pattrib->vcs_mode = padapter->driver_vcs_type;
+	}
+
+}
+
+static void update_attrib_phy_info(_adapter *padapter, struct pkt_attrib *pattrib, struct sta_info *psta)
+{
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct link_mlme_ext_priv *mlmeext = &pattrib->adapter_link->mlmeextpriv;
+	u8 bw;
+
+	pattrib->rtsen = psta->rtsen;
+	pattrib->cts2self = psta->cts2self;
+	pattrib->hw_rts_en = psta->hw_rts_en;
+
+	pattrib->mdata = 0;
+	pattrib->eosp = 0;
+#ifdef CONFIG_80211AX_HE
+	if (psta->hepriv.he_option == _TRUE)
+		pattrib->eosp = 1;
+#endif
+	pattrib->triggered = 0;
+	pattrib->ampdu_spacing = 0;
+
+	/* ht_en, init rate, ,bw, ch_offset, sgi */
+
+	/* ToDo: Need API to inform hal_sta->ra_info.rate_id */
+	/* pattrib->raid = psta->phl_sta->ra_info.rate_id; */
+
+	bw = rtw_get_tx_bw_mode(padapter, psta);
+	pattrib->bwmode = rtw_min(bw, mlmeext->chandef.bw);
+	pattrib->sgi = query_ra_short_GI(psta, pattrib->bwmode);
+
+	if (psta->phl_sta->wmode & WLAN_MD_11AX) {
+		pattrib->ldpc = psta->phl_sta->asoc_cap.he_ldpc;
+		pattrib->stbc = (psta->phl_sta->asoc_cap.stbc_he_rx > 0) ? 1:0;
+	} else if (psta->phl_sta->wmode & WLAN_MD_11AC) {
+		pattrib->ldpc = psta->phl_sta->asoc_cap.vht_ldpc;
+		pattrib->stbc = (psta->phl_sta->asoc_cap.stbc_vht_rx > 0) ? 1:0;
+	} else if (psta->phl_sta->wmode & WLAN_MD_11N) {
+		pattrib->ldpc = psta->phl_sta->asoc_cap.ht_ldpc;
+		pattrib->stbc = (psta->phl_sta->asoc_cap.stbc_ht_rx > 0) ? 1:0;
+	} else {
+		pattrib->ldpc = 0;
+		pattrib->stbc = 0;
+	}
+
+#ifdef CONFIG_80211N_HT
+	if (pregistrypriv->ht_enable &&
+		is_supported_ht(pregistrypriv->wireless_mode)) {
+		pattrib->ht_en = psta->htpriv.ht_option;
+		pattrib->ch_offset = psta->htpriv.ch_offset;
+		pattrib->ampdu_en = _FALSE;
+		if (padapter->driver_ampdu_spacing != 0xFF) /* driver control AMPDU Density for peer sta's rx */
+			pattrib->ampdu_spacing = padapter->driver_ampdu_spacing;
+		else
+			pattrib->ampdu_spacing = psta->ampdu_priv.rx_ampdu_min_spacing;
+
+		/* check if enable ampdu */
+		if ((pattrib->ht_en
+#if CONFIG_IEEE80211_BAND_6GHZ
+		    || psta->phl_sta->chandef.band == BAND_ON_6G
+#endif
+		    ) && psta->ampdu_priv.ampdu_enable) {
+			if (psta->ampdu_priv.agg_enable_bitmap & BIT(pattrib->priority)) {
+				pattrib->ampdu_en = _TRUE;
+				if (psta->ampdu_priv.tx_amsdu_enable == _TRUE)
+					pattrib->amsdu_ampdu_en = _TRUE;
+				else
+					pattrib->amsdu_ampdu_en = _FALSE;
+			}
+		}
+	}
+#endif /* CONFIG_80211N_HT */
+	/* if(pattrib->ht_en && psta->ampdu_priv.ampdu_enable) */
+	/* { */
+	/*	if(psta->ampdu_priv.agg_enable_bitmap & BIT(pattrib->priority)) */
+	/*		pattrib->ampdu_en = _TRUE; */
+	/* }	 */
+
+#ifdef CONFIG_TDLS
+	if (pattrib->direct_link == _TRUE) {
+		psta = pattrib->ptdls_sta;
+
+		/* pattrib->raid = psta->phl_sta->ra_info.rate_id; */
+#ifdef CONFIG_80211N_HT
+		if (pregistrypriv->ht_enable &&
+			is_supported_ht(pregistrypriv->wireless_mode)) {
+				pattrib->bwmode = rtw_get_tx_bw_mode(padapter, psta);
+				pattrib->ht_en = psta->htpriv.ht_option;
+				pattrib->ch_offset = psta->htpriv.ch_offset;
+				pattrib->sgi = query_ra_short_GI(psta, pattrib->bwmode);
+		}
+#endif /* CONFIG_80211N_HT */
+	}
+#endif /* CONFIG_TDLS */
+
+	pattrib->retry_ctrl = _FALSE;
+}
+
+static s32 update_attrib_sec_iv_info(_adapter *padapter, struct pkt_attrib *pattrib)
+{
+	struct sta_info *psta = pattrib->psta;
+	sint bmcast = IS_MCAST(pattrib->ra);
+
+	if (!psta)
+		return _FAIL;
+
+	switch (pattrib->encrypt) {
+	case _WEP40_:
+	case _WEP104_:
+		WEP_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+		break;
+
+	case _TKIP_:
+		if (bmcast)
+			TKIP_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+		else
+			TKIP_IV(pattrib->iv, psta->dot11txpn, 0);
+		break;
+
+	case _AES_:
+		if (bmcast)
+			AES_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+		else
+			AES_IV(pattrib->iv, psta->dot11txpn, 0);
+
+		break;
+
+	case _GCMP_:
+	case _GCMP_256_:
+		if (bmcast)
+			GCMP_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+		else
+			GCMP_IV(pattrib->iv, psta->dot11txpn, 0);
+
+		break;
+
+	case _CCMP_256_:
+		if (bmcast)
+			GCMP_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+		else
+			GCMP_IV(pattrib->iv, psta->dot11txpn, 0);
+
+		break;
+
+#ifdef CONFIG_WAPI_SUPPORT
+	case _SMS4_:
+		rtw_wapi_get_iv(padapter, pattrib->ra, pattrib->iv);
+		break;
+#endif
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+static s32 update_attrib_sec_info(_adapter *padapter, struct pkt_attrib *pattrib, struct sta_info *psta, enum eap_type eapol_type)
+{
+	sint res = _SUCCESS;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	sint bmcast = IS_MCAST(pattrib->ra);
+	s8 hw_decrypted = _FALSE;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	struct link_security_priv *lsecuritypriv = NULL;
+	u8 hw_security_zero_hdrlen = _FALSE;
+
+	_rtw_memset(pattrib->dot118021x_UncstKey.skey, 0, 16);
+	_rtw_memset(pattrib->dot11tkiptxmickey.skey, 0, 16);
+
+	if (psta == NULL || psta->phl_sta == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	padapter_link = psta->padapter_link;
+	lsecuritypriv = &padapter_link->securitypriv;
+	pattrib->mac_id = psta->phl_sta->macid;
+
+	/* Comment by Owen at 2020/05/19
+	 * Issue: RTK STA sends encrypted 4-way 4/4 when AP thinks the 4-way incomplete
+	 * In TCL pressure test, AP may resend 4-way 3/4 with new replay counter in 2 ms.
+	 * In this situation, STA sends unencrypted 4-way 4/4 with old replay counter after more
+	 * than 2 ms, followed by the encrypted 4-way 4/4 with new replay counter. Because the
+	 * AP only accepts unencrypted 4-way 4/4 with a new play counter, and the STA encrypts
+	 * each 4-way 4/4 at this time, the 4-way handshake cannot be completed.
+	 * So we modified that after STA receives unencrypted 4-way 1/4 and 4-way 3/4,
+	 * 4-way 2/4 and 4-way 4/4 sent by STA in the next 100 ms are not encrypted.
+	 */
+	if (psta->ieee8021x_blocked == _TRUE ||
+		((eapol_type == EAPOL_2_4 || eapol_type == EAPOL_4_4) &&
+		rtw_get_passing_time_ms(psta->resp_nonenc_eapol_key_starttime) <= 100)) {
+
+		if (eapol_type == EAPOL_2_4 || eapol_type == EAPOL_4_4)
+			RTW_INFO("Respond unencrypted eapol key\n");
+
+		pattrib->encrypt = 0;
+
+		if ((pattrib->ether_type != 0x888e) && (check_fwstate(pmlmepriv, WIFI_MP_STATE) == _FALSE)) {
+#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s psta->ieee8021x_blocked == _TRUE,  pattrib->ether_type(%04x) != 0x888e\n", __FUNCTION__, pattrib->ether_type);
+#endif
+			res = _FAIL;
+			goto exit;
+		}
+	} else {
+		GET_ENCRY_ALGO(psecuritypriv, psta, pattrib->encrypt, bmcast);
+
+#ifdef CONFIG_WAPI_SUPPORT
+		if (pattrib->ether_type == 0x88B4)
+			pattrib->encrypt = _NO_PRIVACY_;
+#endif
+
+		switch (psecuritypriv->dot11AuthAlgrthm) {
+		case dot11AuthAlgrthm_Open:
+		case dot11AuthAlgrthm_Shared:
+		case dot11AuthAlgrthm_Auto:
+			pattrib->key_idx = (u8)psecuritypriv->dot11PrivacyKeyIndex;
+			break;
+		case dot11AuthAlgrthm_8021X:
+			if (bmcast)
+				pattrib->key_idx = (u8)lsecuritypriv->dot118021XGrpKeyid;
+			else
+				pattrib->key_idx = 0;
+			break;
+		default:
+			pattrib->key_idx = 0;
+			break;
+		}
+
+		/* For WPS 1.0 WEP, driver should not encrypt EAPOL Packet for WPS handshake. */
+		if (((pattrib->encrypt == _WEP40_) || (pattrib->encrypt == _WEP104_)) && (pattrib->ether_type == 0x888e))
+			pattrib->encrypt = _NO_PRIVACY_;
+
+	}
+
+#ifdef CONFIG_TDLS
+	if (pattrib->direct_link == _TRUE) {
+		if (pattrib->encrypt > 0)
+			pattrib->encrypt = _AES_;
+	}
+#endif
+
+	switch (pattrib->encrypt) {
+	case _WEP40_:
+	case _WEP104_:
+		pattrib->iv_len = 4;
+		pattrib->icv_len = 4;
+		hw_security_zero_hdrlen = _TRUE;
+		break;
+
+	case _TKIP_:
+		pattrib->iv_len = 8;
+		pattrib->icv_len = 4;
+		hw_security_zero_hdrlen = _TRUE;
+
+		if (psecuritypriv->busetkipkey == _FAIL) {
+#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s psecuritypriv->busetkipkey(%d)==_FAIL drop packet\n", __FUNCTION__, psecuritypriv->busetkipkey);
+#endif
+			res = _FAIL;
+			goto exit;
+		}
+
+		_rtw_memcpy(pattrib->dot11tkiptxmickey.skey, psta->dot11tkiptxmickey.skey, 16);
+
+		break;
+
+	case _AES_:
+
+		pattrib->iv_len = 8;
+		pattrib->icv_len = 8;
+		hw_security_zero_hdrlen = _TRUE;
+
+		break;
+
+	case _GCMP_:
+	case _GCMP_256_:
+
+		pattrib->iv_len = 8;
+		pattrib->icv_len = 16;
+		hw_security_zero_hdrlen = _TRUE;
+
+		break;
+
+	case _CCMP_256_:
+
+		pattrib->iv_len = 8;
+		pattrib->icv_len = 16;
+		hw_security_zero_hdrlen = _TRUE;
+
+		break;
+
+#ifdef CONFIG_WAPI_SUPPORT
+	case _SMS4_:
+		pattrib->iv_len = 18;
+		pattrib->icv_len = 16;
+		break;
+#endif
+	default:
+		pattrib->iv_len = 0;
+		pattrib->icv_len = 0;
+		break;
+	}
+
+	if (pattrib->encrypt > 0) {
+		_rtw_memcpy(pattrib->dot118021x_UncstKey.skey
+			, psta->dot118021x_UncstKey.skey
+			, (pattrib->encrypt & _SEC_TYPE_256_) ? 32 : 16);
+	}
+
+	if (!bmcast)
+		hw_decrypted = psta->hw_decrypted;
+	else
+		hw_decrypted = psecuritypriv->hw_decrypted;
+
+	if (pattrib->encrypt &&
+		(padapter->securitypriv.sw_encrypt == _TRUE || hw_decrypted == _FALSE || pattrib->encrypt == _TKIP_)) {
+		pattrib->bswenc = _TRUE;
+	} else {
+		pattrib->bswenc = _FALSE;
+	}
+
+#if defined(CONFIG_CONCURRENT_MODE)
+	pattrib->bmc_camid = padapter->securitypriv.dot118021x_bmc_cam_id;
+#endif
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (pattrib->encrypt == _SMS4_)
+		pattrib->bswenc = _FALSE;
+#endif
+
+	if ((pattrib->encrypt) && (eapol_type == EAPOL_4_4))
+		pattrib->bswenc = _TRUE;
+
+	if ((pattrib->bswenc == _FALSE) &&
+	    (hw_security_zero_hdrlen == _TRUE) &&
+	    (padapter->dvobj->phl_com->dev_cap.sec_cap.hw_form_hdr)) {
+		pattrib->iv_len = 0;
+	}
+exit:
+
+	return res;
+
+}
+
+u8	qos_acm(u8 acm_mask, u8 priority)
+{
+	u8	change_priority = priority;
+
+	switch (priority) {
+	case 0:
+	case 3:
+		if (acm_mask & BIT(1))
+			change_priority = 1;
+		break;
+	case 1:
+	case 2:
+		break;
+	case 4:
+	case 5:
+		if (acm_mask & BIT(2))
+			change_priority = 0;
+		break;
+	case 6:
+	case 7:
+		if (acm_mask & BIT(3))
+			change_priority = 5;
+		break;
+	default:
+		RTW_INFO("qos_acm(): invalid pattrib->priority: %d!!!\n", priority);
+		break;
+	}
+
+	return change_priority;
+}
+
+/* refer to IEEE802.11-2016 Table R-3; Comply with IETF RFC4594 */
+u8 tos_to_up(u8 tos)
+{
+	u8 up = 0;
+	u8 dscp;
+	u8 mode = CONFIG_RTW_UP_MAPPING_RULE;
+
+
+	/* tos precedence mapping */
+	if (mode == 0) {
+		up = tos >> 5;
+		return up;
+	}
+
+	/* refer to IEEE802.11-2016 Table R-3;
+	 * DCSP 32(CS4) comply with IETF RFC4594
+	 */
+	dscp = (tos >> 2);
+
+	if (dscp == 0)
+		up = 0;
+	else if (dscp >= 1 && dscp <= 9)
+		up = 1;
+	else if (dscp >= 10 && dscp <= 16)
+		up = 2;
+	else if (dscp >= 17 && dscp <= 23)
+		up = 3;
+	else if (dscp >= 24 && dscp <= 31)
+		up = 4;
+	else if (dscp >= 33 && dscp <= 40)
+		up = 5;
+	else if ((dscp >= 41 && dscp <= 47) || (dscp == 32))
+		up = 6;
+	else if (dscp >= 48 && dscp <= 63)
+		up = 7;
+
+	return up;
+}
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+static void set_qos_core(struct xmit_frame *pxframe)
+{
+	s32 UserPriority = 0;
+
+	if (!pxframe->pkt)
+		goto null_pkt;
+
+	/* get UserPriority from IP hdr */
+	if (pxframe->attrib.ether_type == 0x0800) {
+		struct pkt_file ppktfile;
+		struct ethhdr etherhdr;
+		struct iphdr ip_hdr;
+
+		_rtw_open_pktfile(pxframe->pkt, &ppktfile);
+		_rtw_pktfile_read(&ppktfile, (unsigned char *)&etherhdr, ETH_HLEN);
+		_rtw_pktfile_read(&ppktfile, (u8 *)&ip_hdr, sizeof(ip_hdr));
+		UserPriority = tos_to_up(ip_hdr.tos);
+	}
+
+
+	#ifdef CONFIG_ICMP_VOQ
+	if (pxframe->attrib.icmp_pkt == 1)/*use VO queue to send icmp packet*/
+		UserPriority = 7;
+	#endif
+	#ifdef CONFIG_IP_R_MONITOR
+	if (pxframe->attrib.ether_type == ETH_P_ARP)
+		UserPriority = 7;
+	#endif/*CONFIG_IP_R_MONITOR*/
+
+null_pkt:
+	pxframe->attrib.priority = UserPriority;
+	pxframe->attrib.hdrlen = WLAN_HDR_A3_QOS_LEN;
+	pxframe->attrib.subtype = WIFI_QOS_DATA_TYPE;
+}
+#endif
+
+static void set_qos(struct sk_buff *pkt, struct pkt_attrib *pattrib)
+{
+	s32 UserPriority = 0;
+
+	if (!pkt)
+		goto null_pkt;
+
+	/* get UserPriority from IP hdr */
+	if (pattrib->ether_type == 0x0800) {
+		struct pkt_file ppktfile;
+		struct ethhdr etherhdr;
+		struct iphdr ip_hdr;
+
+		_rtw_open_pktfile(pkt, &ppktfile);
+		_rtw_pktfile_read(&ppktfile, (unsigned char *)&etherhdr, ETH_HLEN);
+		_rtw_pktfile_read(&ppktfile, (u8 *)&ip_hdr, sizeof(ip_hdr));
+		/*		UserPriority = (ntohs(ip_hdr.tos) >> 5) & 0x3; */
+		UserPriority = tos_to_up(ip_hdr.tos);
+	}
+	/*
+		else if (pattrib->ether_type == 0x888e) {
+
+
+			UserPriority = 7;
+		}
+	*/
+
+	#ifdef CONFIG_ICMP_VOQ
+	if (pattrib->icmp_pkt == 1)/*use VO queue to send icmp packet*/
+		UserPriority = 7;
+	#endif
+	#ifdef CONFIG_IP_R_MONITOR
+	if (pattrib->ether_type == ETH_P_ARP)
+		UserPriority = 7;
+	#endif/*CONFIG_IP_R_MONITOR*/
+
+null_pkt:
+	pattrib->priority = UserPriority;
+	pattrib->hdrlen = XATTRIB_GET_WDS(pattrib) ? WLAN_HDR_A4_QOS_LEN : WLAN_HDR_A3_QOS_LEN;
+	pattrib->a4_hdr = XATTRIB_GET_WDS(pattrib) ? 1 : 0;
+	pattrib->subtype = WIFI_QOS_DATA_TYPE;
+}
+
+#ifdef CONFIG_TDLS
+u8 rtw_check_tdls_established(_adapter *padapter, struct pkt_attrib *pattrib)
+{
+	pattrib->ptdls_sta = NULL;
+
+	pattrib->direct_link = _FALSE;
+	if (padapter->tdlsinfo.link_established == _TRUE) {
+		pattrib->ptdls_sta = rtw_get_stainfo(&padapter->stapriv, pattrib->dst);
+#if 1
+		if ((pattrib->ptdls_sta != NULL) &&
+		    (pattrib->ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE) &&
+		    (pattrib->ether_type != 0x0806)) {
+			pattrib->direct_link = _TRUE;
+			/* RTW_INFO("send ptk to "MAC_FMT" using direct link\n", MAC_ARG(pattrib->dst)); */
+		}
+#else
+		if (pattrib->ptdls_sta != NULL &&
+		    pattrib->ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE) {
+			pattrib->direct_link = _TRUE;
+#if 0
+			RTW_INFO("send ptk to "MAC_FMT" using direct link\n", MAC_ARG(pattrib->dst));
+#endif
+		}
+
+		/* ARP frame may be helped by AP*/
+		if (pattrib->ether_type != 0x0806)
+			pattrib->direct_link = _FALSE;
+#endif
+	}
+
+	return pattrib->direct_link;
+}
+
+s32 update_tdls_attrib(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, struct pkt_attrib *pattrib)
+{
+
+	struct sta_info *psta = NULL;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	struct qos_priv		*pqospriv = &pmlmepriv->qospriv;
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+
+	s32 res = _SUCCESS;
+
+	psta = rtw_get_stainfo(pstapriv, pattrib->ra);
+	if (psta == NULL)	{
+		res = _FAIL;
+		goto exit;
+	}
+
+	pattrib->mac_id = psta->phl_sta->macid;
+	pattrib->psta = psta;
+	pattrib->adapter_link = padapter_link;
+	pattrib->ack_policy = 0;
+	/* get ether_hdr_len */
+	pattrib->pkt_hdrlen = ETH_HLEN;
+
+	pattrib->qos_en = psta->qos_option;
+
+	/* [TDLS] TODO: setup req/rsp should be AC_BK */
+	if (pqospriv->qos_option &&  psta->qos_option) {
+		pattrib->priority = 4;	/* tdls management frame should be AC_VI */
+		pattrib->hdrlen = WLAN_HDR_A3_QOS_LEN;
+		pattrib->subtype = WIFI_QOS_DATA_TYPE;
+	} else {
+		pattrib->priority = 0;
+		pattrib->hdrlen = WLAN_HDR_A3_LEN;
+		pattrib->subtype = WIFI_DATA_TYPE;
+	}
+
+	pattrib->rate = pmlmeext->tx_rate;
+
+	/* TODO:_lock */
+	if (update_attrib_sec_info(padapter, pattrib, psta, NON_EAPOL) == _FAIL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	update_attrib_phy_info(padapter, pattrib, psta);
+
+
+exit:
+
+	return res;
+}
+
+#endif /* CONFIG_TDLS */
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+static s32 update_xmitframe_from_hdr(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	uint i;
+	struct pkt_file pktfile;
+	struct sta_info *psta = NULL;
+	struct ethhdr etherhdr;
+	struct sk_buff *pkt = NULL;
+	sint bmcast;
+
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct qos_priv		*pqospriv = &pmlmepriv->qospriv;
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+
+	PHLTX_LOG;
+
+	if (pxframe->pkt)
+		pkt = pxframe->pkt;
+	else
+		return FAIL;
+
+	PHLTX_LOG;
+
+	_rtw_open_pktfile(pkt, &pktfile);
+	i = _rtw_pktfile_read(&pktfile, (u8 *)&etherhdr, ETH_HLEN);
+
+	pxframe->attrib.ether_type = ntohs(etherhdr.h_proto);
+
+	if (MLME_IS_MESH(padapter)) /* address resolve is done for mesh */
+		goto get_sta_info;
+
+	_rtw_memcpy(pxframe->attrib.dst, &etherhdr.h_dest, ETH_ALEN);
+	_rtw_memcpy(pxframe->attrib.src, &etherhdr.h_source, ETH_ALEN);
+
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) ||
+	    (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+		_rtw_memcpy(pxframe->attrib.ra, pxframe->attrib.dst, ETH_ALEN);
+		_rtw_memcpy(pxframe->attrib.ta, adapter_mac_addr(padapter), ETH_ALEN);
+	} else if (MLME_IS_STA(padapter)) {
+
+#if 0//def CONFIG_TDLS  //rtw_phl_tx
+		if (rtw_check_tdls_established(padapter, pattrib) == _TRUE)
+			_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);	/* For TDLS direct link Tx, set ra to be same to dst */
+		else
+#endif
+		_rtw_memcpy(pxframe->attrib.ra, get_bssid(pmlmepriv), ETH_ALEN);
+		_rtw_memcpy(pxframe->attrib.ta, adapter_mac_addr(padapter), ETH_ALEN);
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_sta);
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+		_rtw_memcpy(pxframe->attrib.ra, pxframe->attrib.dst, ETH_ALEN);
+		_rtw_memcpy(pxframe->attrib.ta, get_bssid(pmlmepriv), ETH_ALEN);
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_ap);
+	} else
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_unknown);
+
+	PHLTX_LOG;
+
+get_sta_info:
+	bmcast = IS_MCAST(pxframe->attrib.ra);
+	if (bmcast) {
+		PHLTX_LOG;
+		psta = rtw_get_bcmc_stainfo(padapter);
+		if (psta == NULL) { /* if we cannot get psta => drop the pkt */
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_sta);
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s get sta_info fail, ra:" MAC_FMT"\n", __func__, MAC_ARG(pxframe->attrib.ra));
+			#endif
+			return FAIL;
+		}
+	} else {
+		PHLTX_LOG;
+		psta = rtw_get_stainfo(pstapriv, pxframe->attrib.ra);
+		if (psta == NULL) { /* if we cannot get psta => drop the pkt */
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_ucast_sta);
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s get sta_info fail, ra:" MAC_FMT"\n", __func__, MAC_ARG(pxframe->attrib.ra));
+			#endif
+			return FAIL;
+		} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE && !(psta->state & WIFI_ASOC_STATE)) {
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_ucast_ap_link);
+			return FAIL;
+		}
+	}
+
+	PHLTX_LOG;
+
+	if (!(psta->state & WIFI_ASOC_STATE)) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_link);
+		RTW_INFO("%s-"ADPT_FMT" psta("MAC_FMT")->state(0x%x) != WIFI_ASOC_STATE\n",
+			__func__, ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr), psta->state);
+		return FAIL;
+	}
+
+	pxframe->attrib.psta = psta;
+
+	PHLTX_LOG;
+
+	pxframe->attrib.pktlen = pktfile.pkt_len;
+
+	/* TODO: 802.1Q VLAN header */
+	/* TODO: IPV6 */
+
+	if (ETH_P_IP == pxframe->attrib.ether_type) {
+		u8 ip[20];
+
+		_rtw_pktfile_read(&pktfile, ip, 20);
+
+		if (GET_IPV4_IHL(ip) * 4 > 20)
+			_rtw_pktfile_read(&pktfile, NULL, GET_IPV4_IHL(ip) - 20);
+
+		pxframe->attrib.icmp_pkt = 0;
+		pxframe->attrib.dhcp_pkt = 0;
+		pxframe->attrib.hipriority_pkt = 0;
+
+		if (GET_IPV4_PROTOCOL(ip) == 0x01) { /* ICMP */
+			pxframe->attrib.icmp_pkt = 1;
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_icmp);
+
+		} else if (GET_IPV4_PROTOCOL(ip) == 0x11) { /* UDP */
+			u8 udp[24];
+
+			_rtw_pktfile_read(&pktfile, udp, 24);
+
+			if ((GET_UDP_SRC(udp) == 68 && GET_UDP_DST(udp) == 67)
+				|| (GET_UDP_SRC(udp) == 67 && GET_UDP_DST(udp) == 68)
+			) {
+				/* 67 : UDP BOOTP server, 68 : UDP BOOTP client */
+				if (pxframe->attrib.pktlen > 282) { /* MINIMUM_DHCP_PACKET_SIZE */
+					pxframe->attrib.dhcp_pkt = 1;
+					DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_dhcp);
+					if (0)
+						RTW_INFO("send DHCP packet\n");
+				}
+			}
+
+			/* WaveAgent packet, increase priority so that the system can read data in time */
+			if (((GET_UDP_SIG1(udp) == 0xcc) || (GET_UDP_SIG1(udp) == 0xdd)) &&
+				(GET_UDP_SIG2(udp) == 0xe2)) {
+				pxframe->attrib.hipriority_pkt = 1;
+			}
+
+		} else if (GET_IPV4_PROTOCOL(ip) == 0x06 /* TCP */
+			&& rtw_st_ctl_chk_reg_s_proto(&psta->st_ctl, 0x06) == _TRUE
+		) {
+			u8 tcp[20];
+
+			_rtw_pktfile_read(&pktfile, tcp, 20);
+
+			if (rtw_st_ctl_chk_reg_rule(&psta->st_ctl, padapter, IPV4_SRC(ip), TCP_SRC(tcp), IPV4_DST(ip), TCP_DST(tcp)) == _TRUE) {
+				if (GET_TCP_SYN(tcp) && GET_TCP_ACK(tcp)) {
+					session_tracker_add_cmd(padapter, psta
+						, IPV4_SRC(ip), TCP_SRC(tcp)
+						, IPV4_SRC(ip), TCP_DST(tcp));
+					if (DBG_SESSION_TRACKER)
+						RTW_INFO(FUNC_ADPT_FMT" local:"IP_FMT":"PORT_FMT", remote:"IP_FMT":"PORT_FMT" SYN-ACK\n"
+							, FUNC_ADPT_ARG(padapter)
+							, IP_ARG(IPV4_SRC(ip)), PORT_ARG(TCP_SRC(tcp))
+							, IP_ARG(IPV4_DST(ip)), PORT_ARG(TCP_DST(tcp)));
+				}
+				if (GET_TCP_FIN(tcp)) {
+					session_tracker_del_cmd(padapter, psta
+						, IPV4_SRC(ip), TCP_SRC(tcp)
+						, IPV4_SRC(ip), TCP_DST(tcp));
+					if (DBG_SESSION_TRACKER)
+						RTW_INFO(FUNC_ADPT_FMT" local:"IP_FMT":"PORT_FMT", remote:"IP_FMT":"PORT_FMT" FIN\n"
+							, FUNC_ADPT_ARG(padapter)
+							, IP_ARG(IPV4_SRC(ip)), PORT_ARG(TCP_SRC(tcp))
+							, IP_ARG(IPV4_DST(ip)), PORT_ARG(TCP_DST(tcp)));
+				}
+			}
+		}
+
+	} else if (0x888e == pxframe->attrib.ether_type)
+		parsing_eapol_packet(padapter, pktfile.cur_addr, psta, 1);
+#if defined (DBG_ARP_DUMP) || defined (DBG_IP_R_MONITOR)
+	else if (pxframe->attrib.ether_type == ETH_P_ARP) {
+		u8 arp[28] = {0};
+
+		_rtw_pktfile_read(&pktfile, arp, 28);
+		dump_arp_pkt(RTW_DBGDUMP, etherhdr.h_dest, etherhdr.h_source, arp, 1);
+	}
+#endif
+
+	PHLTX_LOG;
+
+	if ((pxframe->attrib.ether_type == 0x888e) || (pxframe->attrib.dhcp_pkt == 1))
+		rtw_mi_set_scan_deny(padapter, 3000);
+
+	if (MLME_IS_STA(padapter) &&
+		pxframe->attrib.ether_type == ETH_P_ARP &&
+		!IS_MCAST(pxframe->attrib.dst)) {
+		rtw_mi_set_scan_deny(padapter, 1000);
+		rtw_mi_scan_abort(padapter, _FALSE); /*rtw_scan_abort_no_wait*/
+	}
+
+
+	PHLTX_LOG;
+
+	/* get ether_hdr_len */
+	pxframe->attrib.pkt_hdrlen = ETH_HLEN;/* (pattrib->ether_type == 0x8100) ? (14 + 4 ): 14; */ /* vlan tag */
+
+	pxframe->attrib.hdrlen = WLAN_HDR_A3_LEN;
+	pxframe->attrib.type = WIFI_DATA_TYPE;
+	pxframe->attrib.subtype = WIFI_DATA_TYPE;
+	pxframe->attrib.qos_en = pxframe->attrib.psta->qos_option;
+	pxframe->attrib.priority = 0;
+
+	pxframe->attrib.frag_len = pxmitpriv->frag_len;
+
+
+
+	PHLTX_LOG;
+
+	return SUCCESS;
+}
+
+
+static s32 update_xmitframe_qos(_adapter *padapter, struct xmit_frame *pxframe)
+{
+
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct qos_priv		*pqospriv = &pmlmepriv->qospriv;
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+
+	if (!pxframe->attrib.qos_en)
+		return SUCCESS;
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE | WIFI_MESH_STATE
+			| WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)
+		) {
+				set_qos_core(pxframe);
+			#if 0//rtw_phl_tx def CONFIG_RTW_MESH
+				if (MLME_IS_MESH(padapter))
+					rtw_mesh_tx_set_whdr_mctrl_len(pattrib->mesh_frame_mode, pattrib);
+			#endif
+		} else {
+#if 0// def CONFIG_TDLS
+			if (pattrib->direct_link == _TRUE) {
+				if (pattrib->qos_en)
+					set_qos(pkt, pattrib);
+			} else
+#endif
+			{
+				if (pqospriv->qos_option) {
+					set_qos_core(pxframe);
+
+					if (pmlmepriv->acm_mask != 0)
+						pxframe->attrib.priority = qos_acm(pmlmepriv->acm_mask, pxframe->attrib.priority);
+				}
+			}
+		}
+
+	return SUCCESS;
+}
+
+static s32 update_xmitframe_security(_adapter *padapter, struct xmit_frame *pxframe) //rtw_phl_tx todo
+{
+	sint res = _SUCCESS;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	sint bmcast = IS_MCAST(pxframe->attrib.ra);
+
+#if 0
+	_rtw_memset(pattrib->dot118021x_UncstKey.skey,  0, 16);
+	_rtw_memset(pattrib->dot11tkiptxmickey.skey,  0, 16);
+	pattrib->mac_id = psta->phl_sta->macid;
+#endif
+
+	if (pxframe->attrib.psta->ieee8021x_blocked == _TRUE) {
+
+		pxframe->attrib.encrypt = 0;
+
+		if ((pxframe->attrib.ether_type != 0x888e) && (check_fwstate(pmlmepriv, WIFI_MP_STATE) == _FALSE)) {
+#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s psta->ieee8021x_blocked == _TRUE,  pattrib->ether_type(%04x) != 0x888e\n", __FUNCTION__, pxframe->attrib.ether_type);
+#endif
+			res = _FAIL;
+			goto exit;
+		}
+	} else {
+		GET_ENCRY_ALGO(psecuritypriv, pxframe->attrib.psta, pxframe->attrib.encrypt, bmcast);
+
+#ifdef CONFIG_WAPI_SUPPORT
+		if (pxframe->attrib.ether_type == 0x88B4)
+			pxframe->attrib.encrypt = _NO_PRIVACY_;
+#endif
+
+		switch (psecuritypriv->dot11AuthAlgrthm) {
+		case dot11AuthAlgrthm_Open:
+		case dot11AuthAlgrthm_Shared:
+		case dot11AuthAlgrthm_Auto:
+			pxframe->attrib.key_idx = (u8)psecuritypriv->dot11PrivacyKeyIndex;
+			break;
+		case dot11AuthAlgrthm_8021X:
+			if (bmcast)
+				pxframe->attrib.key_idx = (u8)psecuritypriv->dot118021XGrpKeyid;
+			else
+				pxframe->attrib.key_idx = 0;
+			break;
+		default:
+			pxframe->attrib.key_idx = 0;
+			break;
+		}
+
+		/* For WPS 1.0 WEP, driver should not encrypt EAPOL Packet for WPS handshake. */
+		if (((pxframe->attrib.encrypt == _WEP40_) || (pxframe->attrib.encrypt == _WEP104_)) && (pxframe->attrib.ether_type == 0x888e))
+			pxframe->attrib.encrypt = _NO_PRIVACY_;
+
+	}
+
+#if 0 //def CONFIG_TDLS
+	if (pattrib->direct_link == _TRUE) {
+		if (pxframe->attrib.encrypt > 0)
+			pxframe->attrib.encrypt = _AES_;
+	}
+#endif
+
+	switch (pxframe->attrib.encrypt) {
+	case _WEP40_:
+	case _WEP104_:
+		pxframe->attrib.iv_len = 4;
+		pxframe->attrib.icv_len = 4;
+		WEP_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, pxframe->attrib.key_idx);
+		break;
+
+	case _TKIP_:
+		pxframe->attrib.iv_len = 8;
+		pxframe->attrib.icv_len = 4;
+
+		if (psecuritypriv->busetkipkey == _FAIL) {
+#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s psecuritypriv->busetkipkey(%d)==_FAIL drop packet\n", __FUNCTION__, psecuritypriv->busetkipkey);
+#endif
+			res = _FAIL;
+			goto exit;
+		}
+
+		if (bmcast)
+			TKIP_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, pxframe->attrib.key_idx);
+		else
+			TKIP_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, 0);
+
+
+		//_rtw_memcpy(pattrib->dot11tkiptxmickey.skey, pxframe->attrib.psta->dot11tkiptxmickey.skey, 16);
+
+		break;
+
+	case _AES_:
+
+		pxframe->attrib.iv_len = 8;
+		pxframe->attrib.icv_len = 8;
+
+		if (bmcast)
+			AES_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, pxframe->attrib.key_idx);
+		else
+			AES_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, 0);
+
+		break;
+
+	case _GCMP_:
+	case _GCMP_256_:
+
+		pxframe->attrib.iv_len = 8;
+		pxframe->attrib.icv_len = 16;
+
+		if (bmcast)
+			GCMP_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, pxframe->attrib.key_idx);
+		else
+			GCMP_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, 0);
+
+		break;
+
+	case _CCMP_256_:
+
+		pxframe->attrib.iv_len = 8;
+		pxframe->attrib.icv_len = 16;
+
+		if (bmcast)
+			GCMP_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, pxframe->attrib.key_idx);
+		else
+			GCMP_IV(pxframe->attrib.iv, pxframe->attrib.psta->dot11txpn, 0);
+
+		break;
+
+#ifdef CONFIG_WAPI_SUPPORT
+	case _SMS4_:
+		pxframe->attrib.iv_len = 18;
+		pxframe->attrib.icv_len = 16;
+		rtw_wapi_get_iv(padapter, pxframe->attrib.ra, pxframe->attrib.iv);
+		break;
+#endif
+	default:
+		pxframe->attrib.iv_len = 0;
+		pxframe->attrib.icv_len = 0;
+		break;
+	}
+
+#if 0
+	if (pxframe->attrib.encrypt > 0) {
+		_rtw_memcpy(pattrib->dot118021x_UncstKey.skey
+			, pxframe->attrib.psta->dot118021x_UncstKey.skey
+			, (pxframe->attrib.encrypt & _SEC_TYPE_256_) ? 32 : 16);
+	}
+#endif
+
+	if (pxframe->attrib.encrypt &&
+	    ((padapter->securitypriv.sw_encrypt == _TRUE) || (psecuritypriv->hw_decrypted == _FALSE))) {
+		pxframe->attrib.bswenc = _TRUE;
+	} else {
+		pxframe->attrib.bswenc = _FALSE;
+	}
+
+#if defined(CONFIG_CONCURRENT_MODE)
+	//pattrib->bmc_camid = padapter->securitypriv.dot118021x_bmc_cam_id;
+#endif
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (pxframe->attrib.encrypt == _SMS4_)
+		pxframe->attrib.bswenc = _FALSE;
+#endif
+
+exit:
+	return res;
+
+}
+
+static s32 update_xmitframe_hw(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	pxframe->phl_txreq->mdata.rts_en = pxframe->attrib.psta->rtsen;
+	pxframe->phl_txreq->mdata.cts2self = pxframe->attrib.psta->cts2self;
+	pxframe->phl_txreq->mdata.ampdu_density = 0;
+	return 0;
+}
+
+#if 0
+static s32 rtw_core_update_txattrib(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	uint i;
+	struct pkt_file pktfile;
+	struct sta_info *psta = NULL;
+	struct ethhdr etherhdr;
+	struct sk_buff *pkt = NULL;
+
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct qos_priv		*pqospriv = &pmlmepriv->qospriv;
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+	sint res = _SUCCESS;
+
+#if 0//rtw_phl_tx todo def CONFIG_BEAMFORMING
+	update_attrib_txbf_info(padapter, pattrib, psta);
+#endif
+
+#if 0
+	/* TODO:_lock */
+	if (update_attrib_sec_info(padapter, pattrib, psta, NON_EAPOL) == _FAIL) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_sec);
+		res = _FAIL;
+		goto exit;
+	}
+#endif
+
+	update_attrib_phy_info(padapter, pattrib, psta);
+
+	/* RTW_INFO("%s ==> mac_id(%d)\n",__FUNCTION__,pattrib->mac_id ); */
+	/* TODO:_unlock */
+
+#ifdef CONFIG_AUTO_AP_MODE
+	if (psta->isrc && psta->pid > 0)
+		pattrib->pctrl = _TRUE;
+	else
+#endif
+		pattrib->pctrl = 0;
+
+	pattrib->ack_policy = 0;
+
+	if (bmcast)
+		pattrib->rate = psta->init_rate;
+
+	/* pattrib->priority = 5; */ /* force to used VI queue, for testing */
+	pattrib->hw_ssn_sel = pxmitpriv->hw_ssn_seq_no;
+
+	pattrib->wdinfo_en = 1;/*FPGA_test YiWei need modify*/
+
+	rtw_set_tx_chksum_offload(pkt, pattrib);
+
+exit:
+	return res;
+}
+#endif
+#endif
+
+static u8 rtw_chk_htc_en(_adapter *padapter, struct sta_info *psta, struct pkt_attrib *pattrib)
+{
+
+#ifdef CONFIG_80211AX_HE
+	if (psta->hepriv.he_option == _TRUE) {
+		/*By test, some HE AP eapol & arp & dhcp pkt can not append ht control*/
+		if ((0x888e == pattrib->ether_type) || (0x0806 == pattrib->ether_type) || (pattrib->dhcp_pkt == 1))
+			return 0;
+		else if (!rtw_chk_phy_can_append_actrl(padapter, psta))
+			return 0;
+		else
+			return rtw_he_htc_en(padapter, psta);
+	}
+#endif
+
+	return 0;
+}
+
+#ifdef CONFIG_TX_AMSDU
+static void update_attrib_tx_amsdu(_adapter *padapter, struct pkt_attrib *pattrib, struct sta_info *psta)
+{
+	enum rtw_data_rate ra_rate;
+
+	ra_rate = rtw_get_current_tx_rate(padapter, psta);
+
+	switch (ra_rate) {
+		case RTW_DATA_RATE_MCS0:
+		case RTW_DATA_RATE_MCS1:
+		case RTW_DATA_RATE_MCS2:
+		case RTW_DATA_RATE_MCS8:
+		case RTW_DATA_RATE_MCS9:
+		case RTW_DATA_RATE_MCS10:
+		case RTW_DATA_RATE_MCS16:
+		case RTW_DATA_RATE_MCS17:
+		case RTW_DATA_RATE_MCS18:
+		case RTW_DATA_RATE_MCS24:
+		case RTW_DATA_RATE_MCS25:
+		case RTW_DATA_RATE_MCS26:
+		case RTW_DATA_RATE_VHT_NSS1_MCS0:
+		case RTW_DATA_RATE_VHT_NSS1_MCS1:
+		case RTW_DATA_RATE_VHT_NSS1_MCS2:
+		case RTW_DATA_RATE_VHT_NSS2_MCS0:
+		case RTW_DATA_RATE_VHT_NSS2_MCS1:
+		case RTW_DATA_RATE_VHT_NSS2_MCS2:
+		case RTW_DATA_RATE_VHT_NSS3_MCS0:
+		case RTW_DATA_RATE_VHT_NSS3_MCS1:
+		case RTW_DATA_RATE_VHT_NSS3_MCS2:
+		case RTW_DATA_RATE_VHT_NSS4_MCS0:
+		case RTW_DATA_RATE_VHT_NSS4_MCS1:
+		case RTW_DATA_RATE_VHT_NSS4_MCS2:
+		case RTW_DATA_RATE_HE_NSS1_MCS0:
+		case RTW_DATA_RATE_HE_NSS1_MCS1:
+		case RTW_DATA_RATE_HE_NSS1_MCS2:
+		case RTW_DATA_RATE_HE_NSS2_MCS0:
+		case RTW_DATA_RATE_HE_NSS2_MCS1:
+		case RTW_DATA_RATE_HE_NSS2_MCS2:
+		case RTW_DATA_RATE_HE_NSS3_MCS0:
+		case RTW_DATA_RATE_HE_NSS3_MCS1:
+		case RTW_DATA_RATE_HE_NSS3_MCS2:
+		case RTW_DATA_RATE_HE_NSS4_MCS0:
+		case RTW_DATA_RATE_HE_NSS4_MCS1:
+		case RTW_DATA_RATE_HE_NSS4_MCS2:
+			pattrib->tx_amsdu_en = _FALSE;
+			break;
+		default:
+			if (ra_rate <= RTW_DATA_RATE_OFDM54)
+				pattrib->tx_amsdu_en = _FALSE;
+			else
+				pattrib->tx_amsdu_en = _TRUE;
+			break;
+	}
+}
+#endif
+
+static s32 update_attrib(_adapter *padapter, struct sk_buff *pkt, struct pkt_attrib *pattrib)
+{
+	uint i;
+	struct pkt_file pktfile;
+	struct sta_info *psta = NULL;
+	struct ethhdr etherhdr;
+
+	sint bmcast;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct mlme_priv		*pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+	sint res = _SUCCESS;
+	enum eap_type eapol_type = NON_EAPOL;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	struct qos_priv		*pqospriv = NULL;
+
+	DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib);
+
+	_rtw_open_pktfile(pkt, &pktfile);
+	i = _rtw_pktfile_read(&pktfile, (u8 *)&etherhdr, ETH_HLEN);
+
+	pattrib->ether_type = ntohs(etherhdr.h_proto);
+
+	if (MLME_STATE(padapter) & (WIFI_AP_STATE | WIFI_MESH_STATE)) /* address resolve is done for ap/mesh */
+		goto get_sta_info;
+
+	_rtw_memcpy(pattrib->dst, &etherhdr.h_dest, ETH_ALEN);
+	_rtw_memcpy(pattrib->src, &etherhdr.h_source, ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, adapter_mac_addr(padapter), ETH_ALEN);
+
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) ||
+	    (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+		_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_adhoc);
+	} else if (MLME_IS_STA(padapter)) {
+		#ifdef CONFIG_TDLS
+		if (rtw_check_tdls_established(padapter, pattrib) == _TRUE)
+			_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);	/* For TDLS direct link Tx, set ra to be same to dst */
+		else
+		#endif
+		{
+			_rtw_memcpy(pattrib->ra, get_bssid(pmlmepriv), ETH_ALEN);
+			#ifdef CONFIG_RTW_WDS
+			if (adapter_use_wds(padapter)
+				&& _rtw_memcmp(pattrib->src, pattrib->ta, ETH_ALEN) == _FALSE
+			) {
+				pattrib->wds = 1;
+				if (IS_MCAST(pattrib->dst))
+					rtw_tx_wds_gptr_update(padapter, pattrib->src);
+			}
+			#endif
+		}
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_sta);
+	} else
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_unknown);
+
+get_sta_info:
+	bmcast = IS_MCAST(pattrib->ra);
+	if (bmcast) {
+		/* ToDo CONFIG_RTW_MLD: [current primary link only]*/
+		padapter_link = GET_PRIMARY_LINK(padapter);
+		/* Update ta as link address for bcmc sta */
+		_rtw_memcpy(pattrib->ta, padapter_link->mac_addr, ETH_ALEN);
+		psta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+		if (psta == NULL) { /* if we cannot get psta => drop the pkt */
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_sta);
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s get sta_info fail, ra:" MAC_FMT"\n", __func__, MAC_ARG(pattrib->ra));
+			#endif
+			res = _FAIL;
+			goto exit;
+		}
+	} else {
+		/* Find the primary sta with the minimum macid */
+		psta = rtw_get_primary_stainfo_by_addr(pstapriv, pattrib->ra);
+		if (psta == NULL) { /* if we cannot get psta => drop the pkt */
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_ucast_sta);
+			#ifdef DBG_TX_DROP_FRAME
+			RTW_INFO("DBG_TX_DROP_FRAME %s get sta_info fail, ra:" MAC_FMT"\n", __func__, MAC_ARG(pattrib->ra));
+			#endif
+			res = _FAIL;
+			goto exit;
+		} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE && !(psta->state & WIFI_ASOC_STATE)) {
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_ucast_ap_link);
+			res = _FAIL;
+			goto exit;
+		}
+
+		#ifdef CONFIG_RTW_WDS
+		if (XATTRIB_GET_WDS(pattrib) && !(psta->flags & WLAN_STA_WDS))
+			pattrib->wds = 0;
+		#endif
+	}
+
+	padapter_link = psta->padapter_link;
+	pattrib->adapter_link = padapter_link;
+	pqospriv = &padapter_link->mlmepriv.qospriv;
+
+	if (!(psta->state & WIFI_ASOC_STATE)) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_link);
+		RTW_INFO("%s-"ADPT_FMT" psta("MAC_FMT")->state(0x%x) != WIFI_ASOC_STATE\n",
+			__func__, ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr), psta->state);
+		res = _FAIL;
+		goto exit;
+	}
+
+	pattrib->pktlen = pktfile.pkt_len;
+#ifdef CONFIG_CORE_TXSC
+	pattrib->frag_len = pxmitpriv->frag_len;
+#endif
+
+	/* TODO: 802.1Q VLAN header */
+	/* TODO: IPV6 */
+
+	if (ETH_P_IP == pattrib->ether_type) {
+		u8 ip[20];
+
+		_rtw_pktfile_read(&pktfile, ip, 20);
+
+		if (GET_IPV4_IHL(ip) * 4 > 20)
+			_rtw_pktfile_read(&pktfile, NULL, GET_IPV4_IHL(ip) - 20);
+
+		pattrib->icmp_pkt = 0;
+		pattrib->dhcp_pkt = 0;
+		pattrib->hipriority_pkt = 0;
+
+		if (GET_IPV4_PROTOCOL(ip) == 0x01) { /* ICMP */
+			pattrib->icmp_pkt = 1;
+			DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_icmp);
+
+		} else if (GET_IPV4_PROTOCOL(ip) == 0x11) { /* UDP */
+			u8 udp[24];
+
+			_rtw_pktfile_read(&pktfile, udp, 24);
+
+			if ((GET_UDP_SRC(udp) == 68 && GET_UDP_DST(udp) == 67)
+				|| (GET_UDP_SRC(udp) == 67 && GET_UDP_DST(udp) == 68)
+			) {
+				/* 67 : UDP BOOTP server, 68 : UDP BOOTP client */
+				if (pattrib->pktlen > 282) { /* MINIMUM_DHCP_PACKET_SIZE */
+					pattrib->dhcp_pkt = 1;
+					DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_dhcp);
+					if (0)
+						RTW_INFO("send DHCP packet\n");
+				}
+			}
+
+			/* WaveAgent packet, increase priority so that the system can read data in time */
+			if (((GET_UDP_SIG1(udp) == 0xcc) || (GET_UDP_SIG1(udp) == 0xdd)) &&
+				(GET_UDP_SIG2(udp) == 0xe2)) {
+				pattrib->hipriority_pkt = 1;
+			}
+
+		} else if (GET_IPV4_PROTOCOL(ip) == 0x06 /* TCP */
+			&& rtw_st_ctl_chk_reg_s_proto(&psta->st_ctl, 0x06) == _TRUE
+		) {
+			u8 tcp[20];
+
+			_rtw_pktfile_read(&pktfile, tcp, 20);
+
+			if (rtw_st_ctl_chk_reg_rule(&psta->st_ctl, padapter, IPV4_SRC(ip), TCP_SRC(tcp), IPV4_DST(ip), TCP_DST(tcp)) == _TRUE) {
+				if (GET_TCP_SYN(tcp) && GET_TCP_ACK(tcp)) {
+					session_tracker_add_cmd(padapter, psta
+						, IPV4_SRC(ip), TCP_SRC(tcp)
+						, IPV4_SRC(ip), TCP_DST(tcp));
+					if (DBG_SESSION_TRACKER)
+						RTW_INFO(FUNC_ADPT_FMT" local:"IP_FMT":"PORT_FMT", remote:"IP_FMT":"PORT_FMT" SYN-ACK\n"
+							, FUNC_ADPT_ARG(padapter)
+							, IP_ARG(IPV4_SRC(ip)), PORT_ARG(TCP_SRC(tcp))
+							, IP_ARG(IPV4_DST(ip)), PORT_ARG(TCP_DST(tcp)));
+				}
+				if (GET_TCP_FIN(tcp)) {
+					session_tracker_del_cmd(padapter, psta
+						, IPV4_SRC(ip), TCP_SRC(tcp)
+						, IPV4_SRC(ip), TCP_DST(tcp));
+					if (DBG_SESSION_TRACKER)
+						RTW_INFO(FUNC_ADPT_FMT" local:"IP_FMT":"PORT_FMT", remote:"IP_FMT":"PORT_FMT" FIN\n"
+							, FUNC_ADPT_ARG(padapter)
+							, IP_ARG(IPV4_SRC(ip)), PORT_ARG(TCP_SRC(tcp))
+							, IP_ARG(IPV4_DST(ip)), PORT_ARG(TCP_DST(tcp)));
+				}
+			}
+		}
+
+	} else if (0x888e == pattrib->ether_type)
+		eapol_type = parsing_eapol_packet(padapter, pktfile.cur_addr, psta, 1);
+#if defined (DBG_ARP_DUMP) || defined (DBG_IP_R_MONITOR)
+	else if (pattrib->ether_type == ETH_P_ARP) {
+		u8 arp[28] = {0};
+
+		_rtw_pktfile_read(&pktfile, arp, 28);
+		dump_arp_pkt(RTW_DBGDUMP, etherhdr.h_dest, etherhdr.h_source, arp, 1);
+	}
+#endif
+
+	if ((pattrib->ether_type == 0x888e) || (pattrib->dhcp_pkt == 1))
+		rtw_mi_set_scan_deny(padapter, 3000);
+
+#ifndef PRIVATE_R
+	if (MLME_IS_STA(padapter) &&
+		pattrib->ether_type == ETH_P_ARP &&
+		!IS_MCAST(pattrib->dst)) {
+		rtw_mi_set_scan_deny(padapter, 1000);
+		rtw_mi_scan_abort(padapter, _FALSE); /*rtw_scan_abort_no_wait*/
+	}
+#endif
+
+	/* TODO:_lock */
+	if (update_attrib_sec_info(padapter, pattrib, psta, eapol_type) == _FAIL) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_upd_attrib_err_sec);
+		res = _FAIL;
+		goto exit;
+	}
+
+	/* get ether_hdr_len */
+	pattrib->pkt_hdrlen = ETH_HLEN;/* (pattrib->ether_type == 0x8100) ? (14 + 4 ): 14; */ /* vlan tag */
+
+	pattrib->hdrlen = XATTRIB_GET_WDS(pattrib) ? WLAN_HDR_A4_LEN : WLAN_HDR_A3_LEN;
+	pattrib->type = WIFI_DATA_TYPE;
+	pattrib->subtype = WIFI_DATA_TYPE;
+	pattrib->qos_en = psta->qos_option;
+	pattrib->priority = 0;
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE | WIFI_MESH_STATE
+		| WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)
+	) {
+		if (pattrib->qos_en) {
+			set_qos(pkt, pattrib);
+			#ifdef CONFIG_RTW_MESH
+			if (MLME_IS_MESH(padapter))
+				rtw_mesh_tx_set_whdr_mctrl_len(pattrib->mesh_frame_mode, pattrib);
+			#endif
+		}
+	} else {
+#ifdef CONFIG_TDLS
+		if (pattrib->direct_link == _TRUE) {
+			if (pattrib->qos_en)
+				set_qos(pkt, pattrib);
+		} else
+#endif
+		{
+			if (pqospriv->qos_option) {
+				set_qos(pkt, pattrib);
+
+				if (pmlmepriv->acm_mask != 0)
+					pattrib->priority = qos_acm(pmlmepriv->acm_mask, pattrib->priority);
+			}
+		}
+	}
+
+	pattrib->order = rtw_chk_htc_en(padapter, psta, pattrib);
+	if (pattrib->order) {
+		if (pattrib->qos_en)
+			pattrib->hdrlen = XATTRIB_GET_WDS(pattrib) ? WLAN_HDR_A4_QOS_HTC_LEN : WLAN_HDR_A3_QOS_HTC_LEN;
+		else
+			pattrib->hdrlen = XATTRIB_GET_WDS(pattrib) ? WLAN_HDR_A4_HTC_LEN : WLAN_HDR_A3_HTC_LEN;
+	}
+	pattrib->a4_hdr = XATTRIB_GET_WDS(pattrib) ? 1 : 0;
+#ifdef CONFIG_TX_AMSDU
+	update_attrib_tx_amsdu(padapter, pattrib, psta);
+#endif
+	update_attrib_phy_info(padapter, pattrib, psta);
+
+	/* RTW_INFO("%s ==> mac_id(%d)\n",__FUNCTION__,pattrib->mac_id ); */
+
+	pattrib->psta = psta;
+	/* TODO:_unlock */
+
+#ifdef CONFIG_AUTO_AP_MODE
+	if (psta->isrc && psta->pid > 0)
+		pattrib->pctrl = _TRUE;
+	else
+#endif
+		pattrib->pctrl = 0;
+
+	pattrib->ack_policy = 0;
+
+	if (bmcast)
+		pattrib->rate = psta->init_rate;
+
+	/* pattrib->priority = 5; */ /* force to used VI queue, for testing */
+	pattrib->hw_ssn_sel = pxmitpriv->hw_ssn_seq_no;
+
+	pattrib->wdinfo_en = 1;/*FPGA_test YiWei need modify*/
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	rtw_set_tx_chksum_offload(pkt, pattrib);
+#endif
+
+exit:
+	return res;
+}
+
+static s32 xmitframe_addmic(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	sint			curfragnum, length;
+	u8	*pframe, *payload, mic[8];
+	struct	mic_data		micdata;
+	/* struct	sta_info		*stainfo; */
+	struct	pkt_attrib	*pattrib = &pxmitframe->attrib;
+	struct	security_priv	*psecuritypriv = &padapter->securitypriv;
+	struct	xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+	u8 priority[4] = {0x0, 0x0, 0x0, 0x0};
+	u8 hw_hdr_offset = 0;
+	sint bmcst = IS_MCAST(pattrib->ra);
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct	link_security_priv	*lsecuritypriv = &padapter_link->securitypriv;
+
+	/*
+		if(pattrib->psta)
+		{
+			stainfo = pattrib->psta;
+		}
+		else
+		{
+			RTW_INFO("%s, call rtw_get_stainfo()\n", __func__);
+			stainfo=rtw_get_stainfo(&padapter->stapriv ,&pattrib->ra[0]);
+		}
+
+		if(stainfo==NULL)
+		{
+			RTW_INFO("%s, psta==NUL\n", __func__);
+			return _FAIL;
+		}
+
+		if(!(stainfo->state &WIFI_ASOC_STATE))
+		{
+			RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, stainfo->state);
+			return _FAIL;
+		}
+	*/
+
+
+#ifdef CONFIG_USB_TX_AGGREGATION
+	hw_hdr_offset = TXDESC_SIZE + (pxmitframe->pkt_offset * PACKET_OFFSET_SZ);;
+#else
+#ifdef CONFIG_TX_EARLY_MODE
+	hw_hdr_offset = TXDESC_OFFSET + EARLY_MODE_INFO_SIZE;
+#else
+	hw_hdr_offset = TXDESC_OFFSET;
+#endif
+#endif
+
+	if (pattrib->encrypt == _TKIP_) { /* if(psecuritypriv->dot11PrivacyAlgrthm==_TKIP_PRIVACY_) */
+		/* encode mic code */
+		/* if(stainfo!= NULL) */
+		{
+			u8 null_key[16] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
+
+			pframe = pxmitframe->buf_addr + hw_hdr_offset;
+
+			if (bmcst) {
+				if (_rtw_memcmp(lsecuritypriv->dot118021XGrptxmickey[lsecuritypriv->dot118021XGrpKeyid].skey, null_key, 16) == _TRUE) {
+					/* DbgPrint("\nxmitframe_addmic:stainfo->dot11tkiptxmickey==0\n"); */
+					/* rtw_msleep_os(10); */
+					return _FAIL;
+				}
+				/* start to calculate the mic code */
+				rtw_secmicsetkey(&micdata, lsecuritypriv->dot118021XGrptxmickey[lsecuritypriv->dot118021XGrpKeyid].skey);
+			} else {
+				if (_rtw_memcmp(&pattrib->dot11tkiptxmickey.skey[0], null_key, 16) == _TRUE) {
+					/* DbgPrint("\nxmitframe_addmic:stainfo->dot11tkiptxmickey==0\n"); */
+					/* rtw_msleep_os(10); */
+					return _FAIL;
+				}
+				/* start to calculate the mic code */
+				rtw_secmicsetkey(&micdata, &pattrib->dot11tkiptxmickey.skey[0]);
+			}
+
+			if (pframe[1] & 1) { /* ToDS==1 */
+				rtw_secmicappend(&micdata, &pframe[16], 6);  /* DA */
+				if (pframe[1] & 2) /* From Ds==1 */
+					rtw_secmicappend(&micdata, &pframe[24], 6);
+				else
+					rtw_secmicappend(&micdata, &pframe[10], 6);
+			} else {	/* ToDS==0 */
+				rtw_secmicappend(&micdata, &pframe[4], 6);   /* DA */
+				if (pframe[1] & 2) /* From Ds==1 */
+					rtw_secmicappend(&micdata, &pframe[16], 6);
+				else
+					rtw_secmicappend(&micdata, &pframe[10], 6);
+
+			}
+
+			if (pattrib->qos_en)
+				priority[0] = (u8)pxmitframe->attrib.priority;
+
+
+			rtw_secmicappend(&micdata, &priority[0], 4);
+
+			payload = pframe;
+
+			for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
+				payload = (u8 *)RND4((SIZE_PTR)(payload));
+
+				payload = payload + pattrib->hdrlen + pattrib->iv_len;
+				if ((curfragnum + 1) == pattrib->nr_frags) {
+					length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - ((pattrib->bswenc) ? pattrib->icv_len : 0);
+					rtw_secmicappend(&micdata, payload, length);
+					payload = payload + length;
+				} else {
+					length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - ((pattrib->bswenc) ? pattrib->icv_len : 0);
+					rtw_secmicappend(&micdata, payload, length);
+					payload = payload + length + pattrib->icv_len;
+				}
+			}
+			rtw_secgetmic(&micdata, &(mic[0]));
+			/* add mic code  and add the mic code length in last_txcmdsz */
+
+			_rtw_memcpy(payload, &(mic[0]), 8);
+			pattrib->last_txcmdsz += 8;
+
+			payload = payload - pattrib->last_txcmdsz + 8;
+		}
+	}
+
+
+	return _SUCCESS;
+}
+
+/*#define DBG_TX_SW_ENCRYPTOR*/
+
+static s32 xmitframe_swencrypt(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+
+
+	if (!pattrib->bswenc)
+		return _SUCCESS;
+
+#ifdef DBG_TX_SW_ENCRYPTOR
+	RTW_INFO(ADPT_FMT" - sec_type:%s DO SW encryption\n",
+		ADPT_ARG(padapter), security_type_str(pattrib->encrypt));
+#endif
+
+	switch (pattrib->encrypt) {
+	case _WEP40_:
+	case _WEP104_:
+		rtw_wep_encrypt(padapter, (u8 *)pxmitframe);
+		break;
+	case _TKIP_:
+		rtw_tkip_encrypt(padapter, (u8 *)pxmitframe);
+		break;
+	case _AES_:
+	case _CCMP_256_:
+		rtw_aes_encrypt(padapter, (u8 *)pxmitframe);
+		break;
+	case _GCMP_:
+	case _GCMP_256_:
+		rtw_gcmp_encrypt(padapter, (u8 *)pxmitframe);
+		break;
+#ifdef CONFIG_WAPI_SUPPORT
+	case _SMS4_:
+		rtw_sms4_encrypt(padapter, (u8 *)pxmitframe);
+#endif
+	default:
+		break;
+	}
+
+	return _SUCCESS;
+}
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+static s32 rtw_core_xmitframe_addmic(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	sint curfragnum, payload_length;
+	u8	*pwlhdr, *payload, mic[8];
+	struct	mic_data		micdata;
+	/* struct	sta_info		*stainfo; */
+	struct	security_priv	*psecuritypriv = &padapter->securitypriv;
+	struct	xmit_priv		*pxmitpriv = &padapter->xmitpriv;
+	u8 priority[4] = {0x0, 0x0, 0x0, 0x0};
+	u8 hw_hdr_offset = 0;
+	sint bmcst = IS_MCAST(pxframe->attrib.ra);
+
+	if (pxframe->attrib.encrypt == _TKIP_) { /* if(psecuritypriv->dot11PrivacyAlgrthm==_TKIP_PRIVACY_) */
+		/* encode mic code */
+		/* if(stainfo!= NULL) */
+		{
+			u8 null_key[16] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
+
+			pwlhdr = pxframe->wlhdr[0];
+			payload = pxframe->pkt->data + pxframe->attrib.pkt_hdrlen;
+			payload_length = pxframe->pkt->len - pxframe->attrib.pkt_hdrlen;
+
+			if (bmcst) {
+				if (_rtw_memcmp(psecuritypriv->dot118021XGrptxmickey[psecuritypriv->dot118021XGrpKeyid].skey, null_key, 16) == _TRUE) {
+					/* DbgPrint("\nxmitframe_addmic:stainfo->dot11tkiptxmickey==0\n"); */
+					/* rtw_msleep_os(10); */
+					return _FAIL;
+				}
+				/* start to calculate the mic code */
+				rtw_secmicsetkey(&micdata, psecuritypriv->dot118021XGrptxmickey[psecuritypriv->dot118021XGrpKeyid].skey);
+			} else {
+				if (_rtw_memcmp(&pxframe->attrib.psta->dot11tkiptxmickey.skey[0], null_key, 16) == _TRUE) {
+					/* DbgPrint("\nxmitframe_addmic:stainfo->dot11tkiptxmickey==0\n"); */
+					/* rtw_msleep_os(10); */
+					return _FAIL;
+				}
+				/* start to calculate the mic code */
+				rtw_secmicsetkey(&micdata, &pxframe->attrib.psta->dot11tkiptxmickey.skey[0]);
+			}
+
+			if (pwlhdr[1] & 1) { /* ToDS==1 */
+				rtw_secmicappend(&micdata, &pwlhdr[16], 6);  /* DA */
+				if (pwlhdr[1] & 2) /* From Ds==1 */
+					rtw_secmicappend(&micdata, &pwlhdr[24], 6);
+				else
+					rtw_secmicappend(&micdata, &pwlhdr[10], 6);
+			} else {	/* ToDS==0 */
+				rtw_secmicappend(&micdata, &pwlhdr[4], 6);   /* DA */
+				if (pwlhdr[1] & 2) /* From Ds==1 */
+					rtw_secmicappend(&micdata, &pwlhdr[16], 6);
+				else
+					rtw_secmicappend(&micdata, &pwlhdr[10], 6);
+
+			}
+
+			if (pxframe->attrib.qos_en)
+				priority[0] = (u8)pxframe->attrib.qos_en;
+
+			rtw_secmicappend(&micdata, &priority[0], 4);
+
+			payload = (u8 *)RND4((SIZE_PTR)(payload));
+			rtw_secmicappend(&micdata, payload, payload_length);
+
+			rtw_secgetmic(&micdata, &(mic[0]));
+			/* add mic code  and add the mic code length in last_txcmdsz */
+
+			_rtw_memcpy(pxframe->wltail[0]+pxframe->attrib.icv_len, &(mic[0]), 8);
+		}
+	}
+
+	return _SUCCESS;
+}
+
+/*#define DBG_TX_SW_ENCRYPTOR*/
+
+static s32 rtw_core_xmitframe_swencrypt(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	if (pxframe->attrib.bswenc) {
+#ifdef DBG_TX_SW_ENCRYPTOR
+		RTW_INFO(ADPT_FMT" - sec_type:%s DO SW encryption\n",
+			ADPT_ARG(padapter), security_type_str(pxframe->attrib.encrypt));
+#endif
+
+		switch (pxframe->attrib.encrypt) {
+		case _WEP40_:
+		case _WEP104_:
+			//rtw_wep_encrypt(padapter, (u8 *)pxmitframe);
+			break;
+		case _TKIP_:
+			//rtw_tkip_encrypt(padapter, (u8 *)pxmitframe);
+			break;
+		case _AES_:
+		case _CCMP_256_:
+			rtw_core_aes_encrypt(padapter, (u8 *)pxframe);
+			break;
+		case _GCMP_:
+		case _GCMP_256_:
+			//rtw_gcmp_encrypt(padapter, (u8 *)pxmitframe);
+			break;
+#ifdef CONFIG_WAPI_SUPPORT
+		case _SMS4_:
+			//rtw_sms4_encrypt(padapter, (u8 *)pxmitframe);
+#endif
+		default:
+			break;
+		}
+
+	}
+	return _SUCCESS;
+}
+
+
+s32 rtw_core_make_wlanhdr(_adapter *padapter, u8 *hdr, struct xmit_frame *pxframe)
+{
+	u16 *qc;
+
+	struct rtw_ieee80211_hdr *pwlanhdr = (struct rtw_ieee80211_hdr *)hdr;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct qos_priv *pqospriv = &pmlmepriv->qospriv;
+	u8 qos_option = _FALSE;
+	sint res = _SUCCESS;
+	u16 *fctrl = &pwlanhdr->frame_ctl;
+
+	_rtw_memset(hdr, 0, WLANHDR_OFFSET);
+
+	set_frame_sub_type(fctrl, pxframe->attrib.subtype);
+
+	if (pxframe->attrib.subtype & WIFI_DATA_TYPE) {
+		if (MLME_IS_STA(padapter)) {
+#ifdef CONFIG_TDLS
+			if (pattrib->direct_link == _TRUE) {
+				/* TDLS data transfer, ToDS=0, FrDs=0 */
+				_rtw_memcpy(pwlanhdr->addr1, pxframe->attrib.dst, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr2, pxframe->attrib.src, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv), ETH_ALEN);
+
+				if (pxframe->attrib.qos_en)
+					qos_option = _TRUE;
+			} else
+#endif /* CONFIG_TDLS */
+			{
+				/* to_ds = 1, fr_ds = 0; */
+				/* 1.Data transfer to AP */
+				/* 2.Arp pkt will relayed by AP */
+				SetToDs(fctrl);
+				_rtw_memcpy(pwlanhdr->addr1, get_bssid(pmlmepriv), ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr2, pxframe->attrib.ta, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr3, pxframe->attrib.dst, ETH_ALEN);
+
+				if (pqospriv->qos_option)
+					qos_option = _TRUE;
+			}
+		} else if ((check_fwstate(pmlmepriv,  WIFI_AP_STATE) == _TRUE)) {
+			/* to_ds = 0, fr_ds = 1; */
+			SetFrDs(fctrl);
+			_rtw_memcpy(pwlanhdr->addr1, pxframe->attrib.dst, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr2, get_bssid(pmlmepriv), ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr3, pxframe->attrib.src, ETH_ALEN);
+
+			if (pxframe->attrib.qos_en)
+				qos_option = _TRUE;
+		} else if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) ||
+			(check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+			_rtw_memcpy(pwlanhdr->addr1, pxframe->attrib.dst, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr2, pxframe->attrib.ta, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv), ETH_ALEN);
+
+			if (pxframe->attrib.qos_en)
+				qos_option = _TRUE;
+#ifdef CONFIG_RTW_MESH
+		} else if (MLME_IS_STA(padapter)) {
+			rtw_mesh_tx_build_whdr(padapter, pattrib, fctrl, pwlanhdr);
+			if (pxframe->attrib.qos_en)
+				qos_option = _TRUE;
+			else {
+				RTW_WARN("[%s] !qos_en in Mesh\n", __FUNCTION__);
+				res = _FAIL;
+				goto exit;
+			}
+#endif
+		} else {
+			res = _FAIL;
+			goto exit;
+		}
+
+		if (pxframe->attrib.mdata)
+			SetMData(fctrl);
+
+		if (pxframe->attrib.encrypt)
+			SetPrivacy(fctrl);
+
+		if (qos_option) {
+			qc = (unsigned short *)(hdr + pxframe->attrib.hdrlen - 2);
+
+			if (pxframe->attrib.priority)
+				SetPriority(qc, pxframe->attrib.priority);
+
+			SetEOSP(qc, pxframe->attrib.eosp);
+
+			SetAckpolicy(qc, pxframe->attrib.ack_policy);
+
+			if (pxframe->attrib.amsdu)
+				SetAMsdu(qc, pxframe->attrib.amsdu);
+#ifdef CONFIG_RTW_MESH
+			if (MLME_IS_MESH(padapter)) {
+				/* active: don't care, light sleep: 0, deep sleep: 1*/
+				set_mps_lv(qc, 0); //TBD
+
+				/* TBD: temporary set (rspi, eosp) = (0, 1) which means End MPSP */
+				set_rspi(qc, 0);
+				SetEOSP(qc, 1);
+
+				set_mctrl_present(qc, 1);
+			}
+#endif
+		}
+
+		/* TODO: fill HT Control Field */
+
+		/* Update Seq Num will be handled by f/w */
+		{
+			struct sta_info *psta;
+			psta = pxframe->attrib.psta;
+
+			if (psta == NULL) {
+				RTW_INFO("%s, psta==NUL\n", __func__);
+				return _FAIL;
+			}
+
+			if (!(psta->state & WIFI_ASOC_STATE)) {
+				RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+				return _FAIL;
+			}
+
+			if (psta) {
+				psta->sta_xmitpriv.txseq_tid[pxframe->attrib.priority]++;
+				psta->sta_xmitpriv.txseq_tid[pxframe->attrib.priority] &= 0xFFF;
+				pxframe->attrib.seqnum = psta->sta_xmitpriv.txseq_tid[pxframe->attrib.priority];
+
+				SetSeqNum(hdr, pxframe->attrib.seqnum);
+
+#ifdef CONFIG_80211N_HT
+#if 0 /* move into update_attrib_phy_info(). */
+				/* check if enable ampdu */
+				if (pattrib->ht_en && psta->ampdu_priv.ampdu_enable) {
+					if (psta->ampdu_priv.agg_enable_bitmap & BIT(pattrib->priority))
+						pattrib->ampdu_en = _TRUE;
+				}
+#endif
+				/* re-check if enable ampdu by BA_starting_seqctrl */
+				if (pxframe->attrib.ampdu_en == _TRUE) {
+					u16 tx_seq;
+
+					tx_seq = psta->BA_starting_seqctrl[pxframe->attrib.priority & 0x0f];
+
+					/* check BA_starting_seqctrl */
+					if (SN_LESS(pxframe->attrib.seqnum, tx_seq)) {
+						/* RTW_INFO("tx ampdu seqnum(%d) < tx_seq(%d)\n", pattrib->seqnum, tx_seq); */
+						pxframe->attrib.ampdu_en = _FALSE;/* AGG BK */
+					} else if (SN_EQUAL(pxframe->attrib.seqnum, tx_seq)) {
+						psta->BA_starting_seqctrl[pxframe->attrib.priority & 0x0f] = (tx_seq + 1) & 0xfff;
+
+						pxframe->attrib.ampdu_en = _TRUE;/* AGG EN */
+					} else {
+						/* RTW_INFO("tx ampdu over run\n"); */
+						psta->BA_starting_seqctrl[pxframe->attrib.priority & 0x0f] = (pxframe->attrib.seqnum + 1) & 0xfff;
+						pxframe->attrib.ampdu_en = _TRUE;/* AGG EN */
+					}
+
+				}
+#endif /* CONFIG_80211N_HT */
+			}
+		}
+
+	} else {
+
+	}
+
+exit:
+
+
+	return res;
+}
+
+
+
+
+
+#endif
+
+static void rtw_fill_htc_in_wlanhdr(_adapter *padapter, struct pkt_attrib *pattrib, u32 *phtc_buf)
+{
+#ifdef CONFIG_80211AX_HE
+	rtw_he_fill_htc(padapter, pattrib, phtc_buf);
+#endif
+}
+
+s32 rtw_make_wlanhdr(_adapter *padapter, u8 *hdr, struct pkt_attrib *pattrib)
+{
+	u16 *qc;
+	u32 *htc = NULL;
+
+	struct rtw_ieee80211_hdr *pwlanhdr = (struct rtw_ieee80211_hdr *)hdr;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct qos_priv *pqospriv = &padapter_link->mlmepriv.qospriv;
+	u8 qos_option = _FALSE, htc_option = _FALSE;
+	sint res = _SUCCESS;
+	u16 *fctrl = &pwlanhdr->frame_ctl;
+
+	/* struct sta_info *psta; */
+
+	/* sint bmcst = IS_MCAST(pattrib->ra); */
+
+
+	/*
+		psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+		if(pattrib->psta != psta)
+		{
+			RTW_INFO("%s, pattrib->psta(%p) != psta(%p)\n", __func__, pattrib->psta, psta);
+			return;
+		}
+
+		if(psta==NULL)
+		{
+			RTW_INFO("%s, psta==NUL\n", __func__);
+			return _FAIL;
+		}
+
+		if(!(psta->state &WIFI_ASOC_STATE))
+		{
+			RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+			return _FAIL;
+		}
+	*/
+
+#ifdef RTW_PHL_TX
+	_rtw_memset(hdr, 0, pattrib->hdrlen);
+#else
+	_rtw_memset(hdr, 0, WLANHDR_OFFSET);
+#endif
+
+	set_frame_sub_type(fctrl, pattrib->subtype);
+
+	if (pattrib->subtype & WIFI_DATA_TYPE) {
+		if (MLME_IS_STA(padapter)) {
+#ifdef CONFIG_TDLS
+			if (pattrib->direct_link == _TRUE) {
+				/* TDLS data transfer, ToDS=0, FrDs=0 */
+				_rtw_memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv), ETH_ALEN);
+
+				if (pattrib->qos_en)
+					qos_option = _TRUE;
+			} else
+#endif /* CONFIG_TDLS */
+			{
+				#ifdef CONFIG_RTW_WDS
+				if (pattrib->wds) {
+					SetToDs(fctrl);
+					SetFrDs(fctrl);
+					_rtw_memcpy(pwlanhdr->addr1, pattrib->ra, ETH_ALEN);
+					_rtw_memcpy(pwlanhdr->addr2, pattrib->ta, ETH_ALEN);
+					_rtw_memcpy(pwlanhdr->addr3, pattrib->dst, ETH_ALEN);
+					_rtw_memcpy(pwlanhdr->addr4, pattrib->src, ETH_ALEN);
+				} else
+				#endif
+				{
+					/* to_ds = 1, fr_ds = 0; */
+					/* 1.Data transfer to AP */
+					/* 2.Arp pkt will relayed by AP */
+					SetToDs(fctrl);
+					_rtw_memcpy(pwlanhdr->addr1, get_bssid(pmlmepriv), ETH_ALEN);
+					_rtw_memcpy(pwlanhdr->addr2, pattrib->ta, ETH_ALEN);
+					_rtw_memcpy(pwlanhdr->addr3, pattrib->dst, ETH_ALEN);
+				}
+
+				if (pqospriv->qos_option)
+					qos_option = _TRUE;
+			}
+		} else if ((check_fwstate(pmlmepriv,  WIFI_AP_STATE) == _TRUE)) {
+			#ifdef CONFIG_RTW_WDS
+			if (pattrib->wds) {
+				SetToDs(fctrl);
+				SetFrDs(fctrl);
+				_rtw_memcpy(pwlanhdr->addr1, pattrib->ra, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr2, pattrib->ta, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr3, pattrib->dst, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr4, pattrib->src, ETH_ALEN);
+			} else
+			#endif
+			{
+				/* to_ds = 0, fr_ds = 1; */
+				SetFrDs(fctrl);
+				_rtw_memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr2, get_bssid(pmlmepriv), ETH_ALEN);
+				_rtw_memcpy(pwlanhdr->addr3, pattrib->src, ETH_ALEN);
+			}
+
+			if (pattrib->qos_en)
+				qos_option = _TRUE;
+		} else if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) ||
+			(check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+			_rtw_memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr2, pattrib->ta, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv), ETH_ALEN);
+
+			if (pattrib->qos_en)
+				qos_option = _TRUE;
+#ifdef CONFIG_RTW_MESH
+		} else if (check_fwstate(pmlmepriv, WIFI_MESH_STATE) == _TRUE) {
+			rtw_mesh_tx_build_whdr(padapter, pattrib, fctrl, pwlanhdr);
+			if (pattrib->qos_en)
+				qos_option = _TRUE;
+			else {
+				RTW_WARN("[%s] !qos_en in Mesh\n", __FUNCTION__);
+				res = _FAIL;
+				goto exit;
+			}
+#endif
+		} else {
+			res = _FAIL;
+			goto exit;
+		}
+
+		if (pattrib->mdata)
+			SetMData(fctrl);
+
+		if (pattrib->encrypt)
+			SetPrivacy(fctrl);
+
+		if (pattrib->order)
+			htc_option = _TRUE;
+
+		if (qos_option) {
+			qc = (unsigned short *)(hdr + (XATTRIB_GET_WDS(pattrib) ? WLAN_HDR_A4_LEN : WLAN_HDR_A3_LEN));
+
+			if (pattrib->priority)
+				SetPriority(qc, pattrib->priority);
+
+			SetEOSP(qc, pattrib->eosp);
+
+			SetAckpolicy(qc, pattrib->ack_policy);
+
+			if (pattrib->amsdu)
+				SetAMsdu(qc, pattrib->amsdu);
+#ifdef CONFIG_RTW_MESH
+			if (MLME_IS_MESH(padapter)) {
+				/* active: don't care, light sleep: 0, deep sleep: 1*/
+				set_mps_lv(qc, 0); //TBD
+
+				/* TBD: temporary set (rspi, eosp) = (0, 1) which means End MPSP */
+				set_rspi(qc, 0);
+				SetEOSP(qc, 1);
+
+				set_mctrl_present(qc, 1);
+			}
+#endif
+		}
+
+		/* TODO: fill HT Control Field */
+		if (htc_option == _TRUE) {
+			set_htc_order_bit(fctrl);
+
+			htc = (u32 *)(hdr + pattrib->hdrlen - 4);
+			rtw_fill_htc_in_wlanhdr(padapter, pattrib, htc);
+		}
+
+		/* Update Seq Num will be handled by f/w */
+		{
+			struct sta_info *psta;
+			psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+			if (pattrib->psta != psta) {
+				RTW_INFO("%s, pattrib->psta(%p) != psta(%p)\n", __func__, pattrib->psta, psta);
+				return _FAIL;
+			}
+
+			if (psta == NULL) {
+				RTW_INFO("%s, psta==NUL\n", __func__);
+				return _FAIL;
+			}
+
+			if (!(psta->state & WIFI_ASOC_STATE)) {
+				RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+				return _FAIL;
+			}
+
+
+			if (psta) {
+				psta->sta_xmitpriv.txseq_tid[pattrib->priority]++;
+				psta->sta_xmitpriv.txseq_tid[pattrib->priority] &= 0xFFF;
+				pattrib->seqnum = psta->sta_xmitpriv.txseq_tid[pattrib->priority];
+
+				SetSeqNum(hdr, pattrib->seqnum);
+
+#ifdef CONFIG_80211N_HT
+#if 0 /* move into update_attrib_phy_info(). */
+				/* check if enable ampdu */
+				if (pattrib->ht_en && psta->ampdu_priv.ampdu_enable) {
+					if (psta->ampdu_priv.agg_enable_bitmap & BIT(pattrib->priority))
+						pattrib->ampdu_en = _TRUE;
+				}
+#endif
+				/* re-check if enable ampdu by BA_starting_seqctrl */
+				if (pattrib->ampdu_en == _TRUE) {
+					u16 tx_seq;
+
+					tx_seq = psta->BA_starting_seqctrl[pattrib->priority & 0x0f];
+
+					/* check BA_starting_seqctrl */
+					if (SN_LESS(pattrib->seqnum, tx_seq)) {
+						/* RTW_INFO("tx ampdu seqnum(%d) < tx_seq(%d)\n", pattrib->seqnum, tx_seq); */
+						pattrib->ampdu_en = _FALSE;/* AGG BK */
+					} else if (SN_EQUAL(pattrib->seqnum, tx_seq)) {
+						psta->BA_starting_seqctrl[pattrib->priority & 0x0f] = (tx_seq + 1) & 0xfff;
+
+						pattrib->ampdu_en = _TRUE;/* AGG EN */
+					} else {
+						/* RTW_INFO("tx ampdu over run\n"); */
+						psta->BA_starting_seqctrl[pattrib->priority & 0x0f] = (pattrib->seqnum + 1) & 0xfff;
+						pattrib->ampdu_en = _TRUE;/* AGG EN */
+					}
+
+				}
+#endif /* CONFIG_80211N_HT */
+			}
+		}
+
+	} else {
+
+	}
+
+exit:
+
+
+	return res;
+}
+
+s32 rtw_txframes_pending(_adapter *padapter)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	return ((_rtw_queue_empty(&pxmitpriv->be_pending) == _FALSE) ||
+		(_rtw_queue_empty(&pxmitpriv->bk_pending) == _FALSE) ||
+		(_rtw_queue_empty(&pxmitpriv->vi_pending) == _FALSE) ||
+		(_rtw_queue_empty(&pxmitpriv->vo_pending) == _FALSE));
+}
+
+s32 rtw_txframes_sta_ac_pending(_adapter *padapter, struct pkt_attrib *pattrib)
+{
+	struct sta_info *psta;
+	struct tx_servq *ptxservq;
+	int priority = pattrib->priority;
+	/*
+		if(pattrib->psta)
+		{
+			psta = pattrib->psta;
+		}
+		else
+		{
+			RTW_INFO("%s, call rtw_get_stainfo()\n", __func__);
+			psta=rtw_get_stainfo(&padapter->stapriv ,&pattrib->ra[0]);
+		}
+	*/
+	psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+	if (pattrib->psta != psta) {
+		RTW_INFO("%s, pattrib->psta(%p) != psta(%p)\n", __func__, pattrib->psta, psta);
+		return 0;
+	}
+
+	if (psta == NULL) {
+		RTW_INFO("%s, psta==NUL\n", __func__);
+		return 0;
+	}
+
+	if (!(psta->state & WIFI_ASOC_STATE)) {
+		RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+		return 0;
+	}
+
+	switch (priority) {
+	case 1:
+	case 2:
+		ptxservq = &(psta->sta_xmitpriv.bk_q);
+		break;
+	case 4:
+	case 5:
+		ptxservq = &(psta->sta_xmitpriv.vi_q);
+		break;
+	case 6:
+	case 7:
+		ptxservq = &(psta->sta_xmitpriv.vo_q);
+		break;
+	case 0:
+	case 3:
+	default:
+		ptxservq = &(psta->sta_xmitpriv.be_q);
+		break;
+
+	}
+
+	return ptxservq->qcnt;
+}
+
+#ifdef CONFIG_TDLS
+
+int rtw_build_tdls_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt)
+{
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+	struct sta_info *ptdls_sta = NULL;
+	int res = _SUCCESS;
+
+	ptdls_sta = rtw_get_stainfo((&padapter->stapriv), pattrib->dst);
+	if (ptdls_sta == NULL) {
+		switch (ptxmgmt->action_code) {
+		case TDLS_DISCOVERY_REQUEST:
+		case TUNNELED_PROBE_REQ:
+		case TUNNELED_PROBE_RSP:
+			break;
+		default:
+			RTW_INFO("[TDLS] %s - Direct Link Peer = "MAC_FMT" not found for action = %d\n", __func__, MAC_ARG(pattrib->dst), ptxmgmt->action_code);
+			res = _FAIL;
+			goto exit;
+		}
+	}
+
+	switch (ptxmgmt->action_code) {
+	case TDLS_SETUP_REQUEST:
+		rtw_build_tdls_setup_req_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+	case TDLS_SETUP_RESPONSE:
+		rtw_build_tdls_setup_rsp_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+	case TDLS_SETUP_CONFIRM:
+		rtw_build_tdls_setup_cfm_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+	case TDLS_TEARDOWN:
+		rtw_build_tdls_teardown_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+	case TDLS_DISCOVERY_REQUEST:
+		rtw_build_tdls_dis_req_ies(padapter, pxmitframe, pframe, ptxmgmt);
+		break;
+	case TDLS_PEER_TRAFFIC_INDICATION:
+		rtw_build_tdls_peer_traffic_indication_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+#ifdef CONFIG_TDLS_CH_SW
+	case TDLS_CHANNEL_SWITCH_REQUEST:
+		rtw_build_tdls_ch_switch_req_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+	case TDLS_CHANNEL_SWITCH_RESPONSE:
+		rtw_build_tdls_ch_switch_rsp_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+#endif
+	case TDLS_PEER_TRAFFIC_RESPONSE:
+		rtw_build_tdls_peer_traffic_rsp_ies(padapter, pxmitframe, pframe, ptxmgmt, ptdls_sta);
+		break;
+#ifdef CONFIG_WFD
+	case TUNNELED_PROBE_REQ:
+		rtw_build_tunneled_probe_req_ies(padapter, pxmitframe, pframe);
+		break;
+	case TUNNELED_PROBE_RSP:
+		rtw_build_tunneled_probe_rsp_ies(padapter, pxmitframe, pframe);
+		break;
+#endif /* CONFIG_WFD */
+	default:
+		res = _FAIL;
+		break;
+	}
+
+exit:
+	return res;
+}
+
+s32 rtw_make_tdls_wlanhdr(_adapter *padapter, u8 *hdr, struct pkt_attrib *pattrib, struct tdls_txmgmt *ptxmgmt)
+{
+	u16 *qc;
+	struct rtw_ieee80211_hdr *pwlanhdr = (struct rtw_ieee80211_hdr *)hdr;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct qos_priv *pqospriv = &padapter_link->mlmepriv.qospriv;
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	struct sta_info *psta = NULL, *ptdls_sta = NULL;
+	u8 tdls_seq = 0, baddr[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+
+	sint res = _SUCCESS;
+	u16 *fctrl = &pwlanhdr->frame_ctl;
+
+
+	_rtw_memset(hdr, 0, WLANHDR_OFFSET);
+
+	set_frame_sub_type(fctrl, pattrib->subtype);
+
+	switch (ptxmgmt->action_code) {
+	case TDLS_SETUP_REQUEST:
+	case TDLS_SETUP_RESPONSE:
+	case TDLS_SETUP_CONFIRM:
+	case TDLS_PEER_TRAFFIC_INDICATION:
+	case TDLS_PEER_PSM_REQUEST:
+	case TUNNELED_PROBE_REQ:
+	case TUNNELED_PROBE_RSP:
+	case TDLS_DISCOVERY_REQUEST:
+		SetToDs(fctrl);
+		_rtw_memcpy(pwlanhdr->addr1, get_bssid(pmlmepriv), ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, pattrib->dst, ETH_ALEN);
+		break;
+	case TDLS_CHANNEL_SWITCH_REQUEST:
+	case TDLS_CHANNEL_SWITCH_RESPONSE:
+	case TDLS_PEER_PSM_RESPONSE:
+	case TDLS_PEER_TRAFFIC_RESPONSE:
+		_rtw_memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
+		_rtw_memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv), ETH_ALEN);
+		tdls_seq = 1;
+		break;
+	case TDLS_TEARDOWN:
+		if (ptxmgmt->status_code == _RSON_TDLS_TEAR_UN_RSN_) {
+			_rtw_memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv), ETH_ALEN);
+			tdls_seq = 1;
+		} else {
+			SetToDs(fctrl);
+			_rtw_memcpy(pwlanhdr->addr1, get_bssid(pmlmepriv), ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
+			_rtw_memcpy(pwlanhdr->addr3, pattrib->dst, ETH_ALEN);
+		}
+		break;
+	}
+
+	if (pattrib->encrypt)
+		SetPrivacy(fctrl);
+
+	if (ptxmgmt->action_code == TDLS_PEER_TRAFFIC_RESPONSE)
+		SetPwrMgt(fctrl);
+
+	if (pqospriv->qos_option) {
+		qc = (unsigned short *)(hdr + pattrib->hdrlen - 2);
+		if (pattrib->priority)
+			SetPriority(qc, pattrib->priority);
+		SetAckpolicy(qc, pattrib->ack_policy);
+	}
+
+	psta = pattrib->psta;
+
+	/* 1. update seq_num per link by sta_info */
+	/* 2. rewrite encrypt to _AES_, also rewrite iv_len, icv_len */
+	if (tdls_seq == 1) {
+		ptdls_sta = rtw_get_stainfo(pstapriv, pattrib->dst);
+		if (ptdls_sta) {
+			ptdls_sta->sta_xmitpriv.txseq_tid[pattrib->priority]++;
+			ptdls_sta->sta_xmitpriv.txseq_tid[pattrib->priority] &= 0xFFF;
+			pattrib->seqnum = ptdls_sta->sta_xmitpriv.txseq_tid[pattrib->priority];
+			SetSeqNum(hdr, pattrib->seqnum);
+
+			if (pattrib->encrypt) {
+				pattrib->encrypt = _AES_;
+				if (padapter->dvobj->phl_com->dev_cap.sec_cap.hw_form_hdr)
+					pattrib->iv_len = 0;
+				else
+					pattrib->iv_len = 8;
+				pattrib->icv_len = 8;
+				pattrib->bswenc = _FALSE;
+			}
+			pattrib->mac_id = ptdls_sta->phl_sta->macid;
+		} else {
+			res = _FAIL;
+			goto exit;
+		}
+	} else if (psta) {
+		psta->sta_xmitpriv.txseq_tid[pattrib->priority]++;
+		psta->sta_xmitpriv.txseq_tid[pattrib->priority] &= 0xFFF;
+		pattrib->seqnum = psta->sta_xmitpriv.txseq_tid[pattrib->priority];
+		SetSeqNum(hdr, pattrib->seqnum);
+	}
+
+
+exit:
+
+
+	return res;
+}
+
+s32 rtw_xmit_tdls_coalesce(_adapter *padapter, struct xmit_frame *pxmitframe, struct tdls_txmgmt *ptxmgmt)
+{
+	s32 llc_sz;
+
+	u8 *pframe, *mem_start;
+
+	struct sta_info		*psta;
+	struct sta_priv		*pstapriv = &padapter->stapriv;
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_mlme_priv	*pmlmepriv = &padapter_link->mlmepriv;
+	u8 *pbuf_start;
+	s32 bmcst = IS_MCAST(pattrib->ra);
+	s32 res = _SUCCESS;
+
+
+	if (pattrib->psta)
+		psta = pattrib->psta;
+	else {
+		if (bmcst)
+			psta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+		else
+			psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+	}
+
+	if (psta == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	if (pxmitframe->buf_addr == NULL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	pbuf_start = pxmitframe->buf_addr;
+	mem_start = pbuf_start + TXDESC_OFFSET;
+
+	if (rtw_make_tdls_wlanhdr(padapter, mem_start, pattrib, ptxmgmt) == _FAIL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	pframe = mem_start;
+	pframe += pattrib->hdrlen;
+
+	/* adding icv, if necessary... */
+	if (pattrib->iv_len) {
+		if (psta != NULL) {
+			switch (pattrib->encrypt) {
+			case _WEP40_:
+			case _WEP104_:
+				WEP_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+				break;
+			case _TKIP_:
+				if (bmcst)
+					TKIP_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+				else
+					TKIP_IV(pattrib->iv, psta->dot11txpn, 0);
+				break;
+			case _AES_:
+				if (bmcst)
+					AES_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
+				else
+					AES_IV(pattrib->iv, psta->dot11txpn, 0);
+				break;
+			}
+		}
+
+		_rtw_memcpy(pframe, pattrib->iv, pattrib->iv_len);
+		pframe += pattrib->iv_len;
+
+	}
+
+	llc_sz = rtw_put_snap(pframe, pattrib->ether_type);
+	pframe += llc_sz;
+
+	/* pattrib->pktlen will be counted in rtw_build_tdls_ies */
+	pattrib->pktlen = 0;
+
+	rtw_build_tdls_ies(padapter, pxmitframe, pframe, ptxmgmt);
+
+	if ((pattrib->icv_len > 0) && (pattrib->bswenc)) {
+		pframe += pattrib->pktlen;
+		_rtw_memcpy(pframe, pattrib->icv, pattrib->icv_len);
+		pframe += pattrib->icv_len;
+	}
+
+	pattrib->nr_frags = 1;
+	pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len + llc_sz +
+		((pattrib->bswenc) ? pattrib->icv_len : 0) + pattrib->pktlen;
+
+	if (xmitframe_addmic(padapter, pxmitframe) == _FAIL) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	update_attrib_vcs_info(padapter, pxmitframe);
+
+	pattrib->pktlen = pattrib->last_txcmdsz;
+
+exit:
+
+
+	return res;
+}
+#endif /* CONFIG_TDLS */
+
+/*
+ * Calculate wlan 802.11 packet MAX size from pkt_attrib
+ * This function doesn't consider fragment case
+ */
+u32 rtw_calculate_wlan_pkt_size_by_attribue(struct pkt_attrib *pattrib)
+{
+	u32	len = 0;
+
+	len = pattrib->hdrlen /* WLAN Header */
+		+ pattrib->iv_len /* IV */
+		+ XATTRIB_GET_MCTRL_LEN(pattrib)
+		+ SNAP_SIZE + sizeof(u16) /* LLC */
+		+ pattrib->pktlen
+		+ (pattrib->encrypt == _TKIP_ ? 8 : 0) /* MIC */
+		+ (pattrib->bswenc ? pattrib->icv_len : 0) /* ICV */
+		;
+
+	return len;
+}
+
+#ifdef CONFIG_TX_AMSDU
+s32 check_amsdu(struct xmit_frame *pxmitframe)
+{
+	struct pkt_attrib *pattrib;
+	struct sta_info *psta = NULL;
+	s32 ret = _TRUE;
+
+	if (!pxmitframe)
+		ret = _FALSE;
+
+	pattrib = &pxmitframe->attrib;
+
+	psta = rtw_get_stainfo(&pxmitframe->padapter->stapriv, &pattrib->ra[0]);
+	if (psta) {
+		if (psta->flags & WLAN_STA_AMSDU_DISABLE)
+			ret =_FALSE;
+	}
+
+	if (IS_MCAST(pattrib->ra))
+		ret = _FALSE;
+
+	if ((pattrib->ether_type == 0x888e) ||
+		(pattrib->ether_type == 0x0806) ||
+		(pattrib->ether_type == 0x88b4) ||
+		(pattrib->dhcp_pkt == 1))
+		ret = _FALSE;
+
+	if ((pattrib->encrypt == _WEP40_) ||
+	    (pattrib->encrypt == _WEP104_) ||
+	    (pattrib->encrypt == _TKIP_))
+		ret = _FALSE;
+
+	if (!pattrib->qos_en)
+		ret = _FALSE;
+
+	if (IS_AMSDU_AMPDU_NOT_VALID(pattrib))
+		ret = _FALSE;
+
+	return ret;
+}
+
+s32 check_amsdu_tx_support(_adapter *padapter, struct pkt_attrib *pattrib)
+{
+	struct dvobj_priv *pdvobjpriv;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	int tx_amsdu;
+	int tx_amsdu_rate;
+	int current_tx_rate;
+	s32 ret = _FALSE;
+
+	if(pmlmeext->cur_wireless_mode < WLAN_MD_11N)
+		return _FALSE;
+
+	pdvobjpriv = adapter_to_dvobj(padapter);
+	tx_amsdu = padapter->tx_amsdu;
+	tx_amsdu_rate = padapter->tx_amsdu_rate;
+	current_tx_rate = pdvobjpriv->traffic_stat.cur_tx_tp;
+
+	if (tx_amsdu == 1)
+		ret = _TRUE;
+	else if (tx_amsdu >= 2 && (tx_amsdu_rate == 0 || current_tx_rate > tx_amsdu_rate))
+		ret = _TRUE;
+	else
+		ret = _FALSE;
+
+	if (pattrib->tx_amsdu_en == _FALSE)
+		ret = _FALSE;
+
+	return ret;
+}
+
+s32 rtw_xmitframe_coalesce_amsdu(_adapter *padapter, struct xmit_frame *pxmitframe, struct xmit_frame *pxmitframe_queue)
+{
+
+	struct pkt_file pktfile;
+	struct pkt_attrib *pattrib;
+	struct sk_buff *pkt;
+
+	struct pkt_file pktfile_queue;
+	struct pkt_attrib *pattrib_queue;
+	struct sk_buff *pkt_queue;
+
+	s32 llc_sz, mem_sz;
+
+	s32 padding = 0;
+
+	u8 *pframe, *mem_start;
+	u8 hw_hdr_offset;
+
+	u16 *len;
+	u8 *pbuf_start;
+	s32 res = _SUCCESS;
+
+	if (pxmitframe->buf_addr == NULL) {
+		RTW_INFO("==> %s buf_addr==NULL\n", __FUNCTION__);
+		return _FAIL;
+	}
+
+
+	pbuf_start = pxmitframe->buf_addr;
+
+#ifdef CONFIG_USB_TX_AGGREGATION
+	hw_hdr_offset =  TXDESC_SIZE + (pxmitframe->pkt_offset * PACKET_OFFSET_SZ);
+#else
+#ifdef CONFIG_TX_EARLY_MODE /* for SDIO && Tx Agg */
+	hw_hdr_offset = TXDESC_OFFSET + EARLY_MODE_INFO_SIZE;
+#else
+	hw_hdr_offset = TXDESC_OFFSET;
+#endif
+#endif
+
+	mem_start = pbuf_start + hw_hdr_offset; //for DMA
+
+	pattrib = &pxmitframe->attrib;
+
+	pattrib->amsdu = 1;
+
+	if (rtw_make_wlanhdr(padapter, mem_start, pattrib) == _FAIL) {
+		RTW_INFO("%s: rtw_make_wlanhdr fail; drop pkt\n", __func__);
+		res = _FAIL;
+		goto exit;
+	}
+
+	llc_sz = 0;
+
+	pframe = mem_start;
+
+	//SetMFrag(mem_start);
+	ClearMFrag(mem_start);
+
+	pframe += pattrib->hdrlen;
+
+	/* adding icv, if necessary... */
+	if (pattrib->iv_len) {
+		update_attrib_sec_iv_info(padapter, pattrib);
+		_rtw_memcpy(pframe, pattrib->iv, pattrib->iv_len); // queue or new?
+
+		RTW_DBG("%s: keyid=%d pattrib->iv[3]=%.2x pframe=%.2x %.2x %.2x %.2x\n",
+			__func__, padapter->securitypriv.dot11PrivacyKeyIndex,
+			pattrib->iv[3], *pframe, *(pframe + 1), *(pframe + 2), *(pframe + 3));
+
+		pframe += pattrib->iv_len;
+	}
+
+	pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len;
+
+	if (pxmitframe_queue) {
+		pattrib_queue = &pxmitframe_queue->attrib;
+		pkt_queue = pxmitframe_queue->pkt;
+
+		_rtw_open_pktfile(pkt_queue, &pktfile_queue);
+		_rtw_pktfile_read(&pktfile_queue, NULL, pattrib_queue->pkt_hdrlen);
+
+		#ifdef CONFIG_RTW_MESH
+		if (MLME_IS_MESH(padapter)) {
+			/* mDA(6), mSA(6), len(2), mctrl */
+			_rtw_memcpy(pframe, pattrib_queue->mda, ETH_ALEN);
+			pframe += ETH_ALEN;
+			_rtw_memcpy(pframe, pattrib_queue->msa, ETH_ALEN);
+			pframe += ETH_ALEN;
+			len = (u16 *)pframe;
+			pframe += 2;
+			rtw_mesh_tx_build_mctrl(padapter, pattrib_queue, pframe);
+			pframe += XATTRIB_GET_MCTRL_LEN(pattrib_queue);
+		} else
+		#endif
+		{
+			/* 802.3 MAC Header DA(6)  SA(6)  Len(2)*/
+			_rtw_memcpy(pframe, pattrib_queue->dst, ETH_ALEN);
+			pframe += ETH_ALEN;
+			_rtw_memcpy(pframe, pattrib_queue->src, ETH_ALEN);
+			pframe += ETH_ALEN;
+			len = (u16 *)pframe;
+			pframe += 2;
+		}
+
+		llc_sz = rtw_put_snap(pframe, pattrib_queue->ether_type);
+		pframe += llc_sz;
+
+		mem_sz = _rtw_pktfile_read(&pktfile_queue, pframe, pattrib_queue->pktlen);
+		pframe += mem_sz;
+
+		*len = htons(XATTRIB_GET_MCTRL_LEN(pattrib_queue) + llc_sz + mem_sz);
+
+		//calc padding
+		padding = 4 - ((ETH_HLEN + XATTRIB_GET_MCTRL_LEN(pattrib_queue) + llc_sz + mem_sz) & (4-1));
+		if (padding == 4)
+			padding = 0;
+
+		//_rtw_memset(pframe,0xaa, padding);
+		pframe += padding;
+
+		pattrib->last_txcmdsz += ETH_HLEN + XATTRIB_GET_MCTRL_LEN(pattrib_queue) + llc_sz + mem_sz + padding ;
+	}
+
+	//2nd mpdu
+
+	pkt = pxmitframe->pkt;
+	_rtw_open_pktfile(pkt, &pktfile);
+	_rtw_pktfile_read(&pktfile, NULL, pattrib->pkt_hdrlen);
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		/* mDA(6), mSA(6), len(2), mctrl */
+		_rtw_memcpy(pframe, pattrib->mda, ETH_ALEN);
+		pframe += ETH_ALEN;
+		_rtw_memcpy(pframe, pattrib->msa, ETH_ALEN);
+		pframe += ETH_ALEN;
+		len = (u16 *)pframe;
+		pframe += 2;
+		rtw_mesh_tx_build_mctrl(padapter, pattrib, pframe);
+		pframe += XATTRIB_GET_MCTRL_LEN(pattrib);
+	} else
+#endif
+	{
+		/* 802.3 MAC Header  DA(6)  SA(6)  Len(2) */
+		_rtw_memcpy(pframe, pattrib->dst, ETH_ALEN);
+		pframe += ETH_ALEN;
+		_rtw_memcpy(pframe, pattrib->src, ETH_ALEN);
+		pframe += ETH_ALEN;
+		len = (u16 *)pframe;
+		pframe += 2;
+	}
+
+	llc_sz = rtw_put_snap(pframe, pattrib->ether_type);
+	pframe += llc_sz;
+
+	mem_sz = _rtw_pktfile_read(&pktfile, pframe, pattrib->pktlen);
+
+	pframe += mem_sz;
+
+	*len = htons(XATTRIB_GET_MCTRL_LEN(pattrib) + llc_sz + mem_sz);
+
+	//the last ampdu has no padding
+	padding = 0;
+
+	pattrib->nr_frags = 1;
+
+	pattrib->last_txcmdsz += ETH_HLEN + XATTRIB_GET_MCTRL_LEN(pattrib) + llc_sz + mem_sz + padding +
+		((pattrib->bswenc) ? pattrib->icv_len : 0) ;
+
+	if ((pattrib->icv_len > 0) && (pattrib->bswenc)) {
+		_rtw_memcpy(pframe, pattrib->icv, pattrib->icv_len);
+		pframe += pattrib->icv_len;
+	}
+
+	if (xmitframe_addmic(padapter, pxmitframe) == _FAIL) {
+		RTW_INFO("xmitframe_addmic(padapter, pxmitframe)==_FAIL\n");
+		res = _FAIL;
+		goto exit;
+	}
+
+	xmitframe_swencrypt(padapter, pxmitframe);
+
+	update_attrib_vcs_info(padapter, pxmitframe);
+
+exit:
+	return res;
+}
+#endif /* CONFIG_TX_AMSDU */
+
+/*
+
+This sub-routine will perform all the following:
+
+1. remove 802.3 header.
+2. create wlan_header, based on the info in pxmitframe
+3. append sta's iv/ext-iv
+4. append LLC
+5. move frag chunk from pframe to pxmitframe->mem
+6. apply sw-encrypt, if necessary.
+
+*/
+s32 rtw_xmitframe_coalesce(_adapter *padapter, struct sk_buff *pkt, struct xmit_frame *pxmitframe)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct pkt_file pktfile;
+
+	s32 frg_inx, frg_len, mpdu_len, llc_sz, mem_sz;
+
+	SIZE_PTR addr;
+
+	u8 *pframe, *mem_start;
+	u8 hw_hdr_offset;
+
+	/* struct sta_info		*psta; */
+	/* struct sta_priv		*pstapriv = &padapter->stapriv; */
+	/* struct mlme_priv	*pmlmepriv = &padapter->mlmepriv; */
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+
+	u8 *pbuf_start;
+
+	s32 bmcst = IS_MCAST(pattrib->ra);
+	s32 res = _SUCCESS;
+
+
+	/*
+		if (pattrib->psta)
+		{
+			psta = pattrib->psta;
+		} else
+		{
+			RTW_INFO("%s, call rtw_get_stainfo()\n", __func__);
+			psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+		}
+
+		if(psta==NULL)
+		{
+
+			RTW_INFO("%s, psta==NUL\n", __func__);
+			return _FAIL;
+		}
+
+
+		if(!(psta->state &WIFI_ASOC_STATE))
+		{
+			RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+			return _FAIL;
+		}
+	*/
+	if (pxmitframe->buf_addr == NULL) {
+		RTW_INFO("==> %s buf_addr==NULL\n", __FUNCTION__);
+		return _FAIL;
+	}
+
+	pbuf_start = pxmitframe->buf_addr;
+
+#if 0
+#ifdef CONFIG_USB_TX_AGGREGATION
+	hw_hdr_offset =  TXDESC_SIZE + (pxmitframe->pkt_offset * PACKET_OFFSET_SZ);
+#else
+#ifdef CONFIG_TX_EARLY_MODE /* for SDIO && Tx Agg */
+	hw_hdr_offset = TXDESC_OFFSET + EARLY_MODE_INFO_SIZE;
+#else
+	hw_hdr_offset = TXDESC_OFFSET;
+#endif
+#endif
+#endif
+	hw_hdr_offset = rtw_hal_get_txdesc_len(GET_PHL_COM(dvobj), pattrib); /*FPGA_test*/
+
+	mem_start = pbuf_start +	hw_hdr_offset;
+
+	if (rtw_make_wlanhdr(padapter, mem_start, pattrib) == _FAIL) {
+		RTW_INFO("%s: rtw_make_wlanhdr fail; drop pkt\n", __func__);
+		res = _FAIL;
+		goto exit;
+	}
+
+	_rtw_open_pktfile(pkt, &pktfile);
+	_rtw_pktfile_read(&pktfile, NULL, pattrib->pkt_hdrlen);
+
+	frg_inx = 0;
+	frg_len = pxmitpriv->frag_len - 4;/* 2346-4 = 2342 */
+
+	while (1) {
+		llc_sz = 0;
+
+		mpdu_len = frg_len;
+
+		pframe = mem_start;
+
+		SetMFrag(mem_start);
+
+		pframe += pattrib->hdrlen;
+		mpdu_len -= pattrib->hdrlen;
+
+		/* adding icv, if necessary... */
+		if (pattrib->iv_len) {
+			update_attrib_sec_iv_info(padapter, pattrib);
+			_rtw_memcpy(pframe, pattrib->iv, pattrib->iv_len);
+
+
+			pframe += pattrib->iv_len;
+
+			mpdu_len -= pattrib->iv_len;
+		}
+
+		if (frg_inx == 0) {
+			#ifdef CONFIG_RTW_MESH
+			if (MLME_IS_MESH(padapter)) {
+				rtw_mesh_tx_build_mctrl(padapter, pattrib, pframe);
+				pframe += XATTRIB_GET_MCTRL_LEN(pattrib);
+				mpdu_len -= XATTRIB_GET_MCTRL_LEN(pattrib);
+			}
+			#endif
+
+			llc_sz = rtw_put_snap(pframe, pattrib->ether_type);
+			pframe += llc_sz;
+			mpdu_len -= llc_sz;
+		}
+
+		if ((pattrib->icv_len > 0) && (pattrib->bswenc))
+			mpdu_len -= pattrib->icv_len;
+
+
+		if (bmcst) {
+			/* don't do fragment to broadcat/multicast packets */
+			mem_sz = _rtw_pktfile_read(&pktfile, pframe, pattrib->pktlen);
+		} else
+			mem_sz = _rtw_pktfile_read(&pktfile, pframe, mpdu_len);
+
+		pframe += mem_sz;
+
+		if ((pattrib->icv_len > 0) && (pattrib->bswenc)) {
+			_rtw_memcpy(pframe, pattrib->icv, pattrib->icv_len);
+			pframe += pattrib->icv_len;
+		}
+
+		frg_inx++;
+
+		if (bmcst || (rtw_endofpktfile(&pktfile) == _TRUE)) {
+			pattrib->nr_frags = frg_inx;
+
+			pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len +
+				((pattrib->nr_frags == 1) ? (XATTRIB_GET_MCTRL_LEN(pattrib) + llc_sz) : 0) +
+				((pattrib->bswenc) ? pattrib->icv_len : 0) + mem_sz;
+
+			ClearMFrag(mem_start);
+
+			break;
+		}
+
+		addr = (SIZE_PTR)(pframe);
+
+		mem_start = (unsigned char *)RND4(addr) + hw_hdr_offset;
+		_rtw_memcpy(mem_start, pbuf_start + hw_hdr_offset, pattrib->hdrlen);
+
+	}
+
+	if (xmitframe_addmic(padapter, pxmitframe) == _FAIL) {
+		RTW_INFO("xmitframe_addmic(padapter, pxmitframe)==_FAIL\n");
+		res = _FAIL;
+		goto exit;
+	}
+
+	xmitframe_swencrypt(padapter, pxmitframe);
+
+	if (bmcst == _FALSE)
+		update_attrib_vcs_info(padapter, pxmitframe);
+	else
+		pattrib->vcs_mode = NONE_VCS;
+
+exit:
+
+
+	return res;
+}
+
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+/*
+ * CCMP encryption for unicast robust mgmt frame and broadcast group privicy action
+ * BIP for broadcast robust mgmt frame
+ */
+s32 rtw_mgmt_xmitframe_coalesce(_adapter *padapter, struct sk_buff *pkt, struct xmit_frame *pxmitframe)
+{
+#define DBG_MGMT_XMIT_COALESEC_DUMP 0
+#define DBG_MGMT_XMIT_BIP_DUMP 0
+#define DBG_MGMT_XMIT_ENC_DUMP 0
+
+	struct pkt_file pktfile;
+	s32 frg_inx, frg_len, mpdu_len, llc_sz, mem_sz;
+	SIZE_PTR addr;
+	u8 *pframe, *mem_start = NULL, *tmp_buf = NULL;
+	u8 hw_hdr_offset, subtype ;
+	u8 category = 0xFF;
+	struct sta_info		*psta = NULL;
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	u8 *pbuf_start;
+	s32 bmcst = IS_MCAST(pattrib->ra);
+	s32 res = _FAIL;
+	u8 *BIP_AAD = NULL;
+	u8 *MGMT_body = NULL;
+
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct rtw_ieee80211_hdr	*pwlanhdr;
+	u8 mme_cont[_MME_IE_LENGTH_ - 2];
+	u8 mme_clen;
+
+	u32	ori_len;
+	union pn48 *pn = NULL;
+	enum security_type cipher = _NO_PRIVACY_;
+	u8 kid;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	u8 hw_security_zero_hdrlen = _FALSE;
+
+	if (pxmitframe->buf_addr == NULL) {
+		RTW_WARN(FUNC_ADPT_FMT" pxmitframe->buf_addr\n"
+			, FUNC_ADPT_ARG(padapter));
+		return _FAIL;
+	}
+
+	mem_start = pframe = (u8 *)(pxmitframe->buf_addr) + TXDESC_OFFSET;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	subtype = get_frame_sub_type(pframe); /* bit(7)~bit(2) */
+
+	/* check if robust mgmt frame */
+	if (subtype != WIFI_DEAUTH && subtype != WIFI_DISASSOC && subtype != WIFI_ACTION)
+		return _SUCCESS;
+	if (subtype == WIFI_ACTION) {
+		category = *(pframe + sizeof(struct rtw_ieee80211_hdr_3addr));
+		if (CATEGORY_IS_NON_ROBUST(category))
+			return _SUCCESS;
+	}
+	if (!bmcst) {
+		if (pattrib->psta)
+			psta = pattrib->psta;
+		else
+			pattrib->psta = psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+		if (psta == NULL) {
+			RTW_INFO(FUNC_ADPT_FMT" unicast sta == NULL\n", FUNC_ADPT_ARG(padapter));
+			return _FAIL;
+		}
+		if (!(psta->flags & WLAN_STA_MFP)) {
+			/* peer is not MFP capable, no need to encrypt */
+			return _SUCCESS;
+		}
+		if (psta->bpairwise_key_installed != _TRUE) {
+			RTW_INFO(FUNC_ADPT_FMT" PTK is not installed\n"
+				, FUNC_ADPT_ARG(padapter));
+			return _FAIL;
+		}
+	}
+
+	ori_len = BIP_AAD_SIZE + pattrib->pktlen + _MME_IE_LENGTH_;
+	tmp_buf = BIP_AAD = rtw_zmalloc(ori_len);
+	if (BIP_AAD == NULL)
+		return _FAIL;
+
+	_rtw_spinlock_bh(&padapter->security_key_mutex);
+
+	if (bmcst) {
+		if (subtype == WIFI_ACTION && CATEGORY_IS_GROUP_PRIVACY(category)) {
+			/* broadcast group privacy action frame */
+			#if DBG_MGMT_XMIT_COALESEC_DUMP
+			RTW_INFO(FUNC_ADPT_FMT" broadcast gp action(%u)\n"
+				, FUNC_ADPT_ARG(padapter), category);
+			#endif
+
+			if (pattrib->psta)
+				psta = pattrib->psta;
+			else
+				pattrib->psta = psta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+			if (psta == NULL) {
+				RTW_INFO(FUNC_ADPT_FMT" broadcast sta == NULL\n"
+					, FUNC_ADPT_ARG(padapter));
+				goto xmitframe_coalesce_fail;
+			}
+			if (padapter_link->securitypriv.binstallGrpkey != _TRUE) {
+				RTW_INFO(FUNC_ADPT_FMT" GTK is not installed\n"
+					, FUNC_ADPT_ARG(padapter));
+				goto xmitframe_coalesce_fail;
+			}
+
+			pn = &psta->dot11txpn;
+			cipher = padapter->securitypriv.dot118021XGrpPrivacy;
+			kid = padapter_link->securitypriv.dot118021XGrpKeyid;
+		} else {
+			#ifdef CONFIG_IEEE80211W
+			/* broadcast robust mgmt frame, using BIP */
+			int frame_body_len;
+			u8 mic[16];
+
+			/* IGTK key is not install ex: mesh MFP without IGTK */
+			if (SEC_IS_BIP_KEY_INSTALLED(&padapter_link->securitypriv) != _TRUE)
+				goto xmitframe_coalesce_success;
+
+			#if DBG_MGMT_XMIT_COALESEC_DUMP
+			if (subtype == WIFI_DEAUTH)
+				RTW_INFO(FUNC_ADPT_FMT" braodcast deauth\n", FUNC_ADPT_ARG(padapter));
+			else if (subtype == WIFI_DISASSOC)
+				RTW_INFO(FUNC_ADPT_FMT" braodcast disassoc\n", FUNC_ADPT_ARG(padapter));
+			else if (subtype == WIFI_ACTION) {
+				RTW_INFO(FUNC_ADPT_FMT" braodcast action(%u)\n"
+					, FUNC_ADPT_ARG(padapter), category);
+			}
+			#endif
+
+			/*HW encrypt need to record encrypt type*/
+			pattrib->encrypt = padapter->securitypriv.dot11wCipher;
+
+			_rtw_memset(mme_cont, 0, _MME_IE_LENGTH_ - 2);
+			mme_clen = padapter->securitypriv.dot11wCipher == _BIP_CMAC_128_ ? 16 : 24;
+
+			MGMT_body = pframe + sizeof(struct rtw_ieee80211_hdr_3addr);
+			pframe += pattrib->pktlen;
+
+			/* octent 0 and 1 is key index ,BIP keyid is 4 or 5, LSB only need octent 0 */
+			mme_cont[0] = padapter_link->securitypriv.dot11wBIPKeyid;
+			/* increase PN and apply to packet */
+			padapter_link->securitypriv.dot11wBIPtxpn.val++;
+			RTW_PUT_LE64(&mme_cont[2], padapter_link->securitypriv.dot11wBIPtxpn.val);
+
+			/* add MME IE with MIC all zero, MME string doesn't include element id and length */
+			pframe = rtw_set_ie(pframe, _MME_IE_, mme_clen, mme_cont, &(pattrib->pktlen));
+			pattrib->last_txcmdsz = pattrib->pktlen;
+
+			if (pattrib->encrypt &&
+				(padapter->securitypriv.sw_encrypt == _TRUE || padapter->securitypriv.hw_decrypted == _FALSE)) {
+				pattrib->bswenc = _TRUE;
+			} else {
+				/* currently HW only support _BIP_CMAC_128_ */
+				if (pattrib->encrypt == _BIP_CMAC_128_)
+					pattrib->bswenc = _FALSE;
+				else
+					pattrib->bswenc = _TRUE;
+			}
+
+			if (!pattrib->bswenc) {
+				pattrib->key_idx = padapter_link->securitypriv.dot11wBIPKeyid;
+				/*Don't need to append MIC part of MME*/
+				pattrib->pktlen -= (mme_clen == 16 ? 8 : 16);
+				pattrib->last_txcmdsz = pattrib->pktlen;
+				goto xmitframe_coalesce_success;
+			}
+
+			/* total frame length - header length */
+			frame_body_len = pattrib->pktlen - sizeof(struct rtw_ieee80211_hdr_3addr);
+
+			/* conscruct AAD, copy frame control field */
+			_rtw_memcpy(BIP_AAD, &pwlanhdr->frame_ctl, 2);
+			ClearRetry(BIP_AAD);
+			ClearPwrMgt(BIP_AAD);
+			ClearMData(BIP_AAD);
+			/* conscruct AAD, copy address 1 to address 3 */
+			_rtw_memcpy(BIP_AAD + 2, GetAddr1Ptr((u8 *)pwlanhdr), 18);
+			/* copy management fram body */
+			_rtw_memcpy(BIP_AAD + BIP_AAD_SIZE, MGMT_body, frame_body_len);
+
+			#if DBG_MGMT_XMIT_BIP_DUMP
+			/* dump total packet include MME with zero MIC */
+			{
+				int i;
+				printk("Total packet: ");
+				for (i = 0; i < BIP_AAD_SIZE + frame_body_len; i++)
+					printk(" %02x ", BIP_AAD[i]);
+				printk("\n");
+			}
+			#endif
+
+			/* calculate mic */
+			if (rtw_calculate_bip_mic(padapter->securitypriv.dot11wCipher,
+				(u8 *)pwlanhdr, pattrib->pktlen,
+				padapter_link->securitypriv.dot11wBIPKey[padapter_link->securitypriv.dot11wBIPKeyid].skey,
+				BIP_AAD, (BIP_AAD_SIZE + frame_body_len), mic) == _FAIL)
+				goto xmitframe_coalesce_fail;
+
+			#if DBG_MGMT_XMIT_BIP_DUMP
+			/* dump calculated mic result */
+			{
+				int i;
+				printk("Calculated mic result: ");
+				for (i = 0; i < 16; i++)
+					printk(" %02x ", mic[i]);
+				printk("\n");
+			}
+			#endif
+
+			/* copy right BIP mic value, total is 128bits, we use the 0~63 bits */
+			if (padapter->securitypriv.dot11wCipher == _BIP_CMAC_128_)
+				_rtw_memcpy(pframe - 8, mic, 8);
+			else
+				_rtw_memcpy(pframe - 16, mic, 16);
+
+			#if DBG_MGMT_XMIT_BIP_DUMP
+			/*dump all packet after mic ok */
+			{
+				int pp;
+				printk("pattrib->pktlen = %d\n", pattrib->pktlen);
+				for (pp = 0; pp < pattrib->pktlen; pp++)
+					printk(" %02x ", mem_start[pp]);
+				printk("\n");
+			}
+			#endif
+
+			#endif /* CONFIG_IEEE80211W */
+
+			goto xmitframe_coalesce_success;
+		}
+	} else {
+		/* unicast robust mgmt frame */
+		#if DBG_MGMT_XMIT_COALESEC_DUMP
+		if (subtype == WIFI_DEAUTH) {
+			RTW_INFO(FUNC_ADPT_FMT" unicast deauth to "MAC_FMT"\n"
+				, FUNC_ADPT_ARG(padapter), MAC_ARG(pattrib->ra));
+		} else if (subtype == WIFI_DISASSOC) {
+			RTW_INFO(FUNC_ADPT_FMT" unicast disassoc to "MAC_FMT"\n"
+				, FUNC_ADPT_ARG(padapter), MAC_ARG(pattrib->ra));
+		} else if (subtype == WIFI_ACTION) {
+			RTW_INFO(FUNC_ADPT_FMT" unicast action(%u) to "MAC_FMT"\n"
+				, FUNC_ADPT_ARG(padapter), category, MAC_ARG(pattrib->ra));
+		}
+		#endif
+
+		pn = &psta->dot11txpn;
+		cipher = psta->dot118021XPrivacy;
+		kid = 0;
+
+		_rtw_memcpy(pattrib->dot118021x_UncstKey.skey
+			, psta->dot118021x_UncstKey.skey
+			, (cipher & _SEC_TYPE_256_) ? 32 : 16);
+
+		/* To use wrong key */
+		if (pattrib->key_type == IEEE80211W_WRONG_KEY) {
+			RTW_INFO("use wrong key\n");
+			pattrib->dot118021x_UncstKey.skey[0] = 0xff;
+		}
+	}
+
+	#if DBG_MGMT_XMIT_ENC_DUMP
+	/* before encrypt dump the management packet content */
+	{
+		int i;
+		printk("Management pkt: ");
+		for (i = 0; i < pattrib->pktlen; i++)
+			printk(" %02x ", pframe[i]);
+		printk("=======\n");
+	}
+	#endif
+
+	/* bakeup original management packet */
+	_rtw_memcpy(tmp_buf, pframe, pattrib->pktlen);
+	/* move to data portion */
+	pframe += pattrib->hdrlen;
+
+	if (pattrib->key_type != IEEE80211W_NO_KEY) {
+		pattrib->encrypt = cipher;
+		pattrib->bswenc = _TRUE;
+	}
+
+	/*
+	* 802.11w encrypted management packet must be:
+	* _AES_, _CCMP_256_, _GCMP_, _GCMP_256_
+	*/
+	switch (pattrib->encrypt) {
+	case _AES_:
+		pattrib->iv_len = 8;
+		pattrib->icv_len = 8;
+		AES_IV(pattrib->iv, (*pn), kid);
+		hw_security_zero_hdrlen = _TRUE;
+		break;
+	case _CCMP_256_:
+		pattrib->iv_len = 8;
+		pattrib->icv_len = 16;
+		AES_IV(pattrib->iv, (*pn), kid);
+		hw_security_zero_hdrlen = _TRUE;
+		break;
+	case _GCMP_:
+	case _GCMP_256_:
+		pattrib->iv_len = 8;
+		pattrib->icv_len = 16;
+		GCMP_IV(pattrib->iv, (*pn), kid);
+		hw_security_zero_hdrlen = _TRUE;
+		break;
+	default:
+		goto xmitframe_coalesce_fail;
+	}
+
+	if (pattrib->encrypt &&
+	(padapter->securitypriv.sw_encrypt == _TRUE || psta->hw_decrypted == _FALSE)) {
+		pattrib->bswenc = _TRUE;
+	} else {
+		/* only right key can use HW encrypt */
+		if (pattrib->key_type == IEEE80211W_RIGHT_KEY)
+			pattrib->bswenc = _FALSE;
+		else
+			pattrib->bswenc = _TRUE;
+	}
+
+	/* at the moment the security CAM may be cleaned already --> use SW encryption */
+	if (subtype == WIFI_DEAUTH || subtype == WIFI_DISASSOC)
+		pattrib->bswenc = _TRUE;
+
+	if ((pattrib->bswenc == _FALSE) &&
+	    (hw_security_zero_hdrlen == _TRUE) &&
+	    (padapter->dvobj->phl_com->dev_cap.sec_cap.hw_form_hdr)) {
+		pattrib->iv_len = 0;
+	}
+
+	/* insert iv header into management frame */
+	_rtw_memcpy(pframe, pattrib->iv, pattrib->iv_len);
+	pframe += pattrib->iv_len;
+	/* copy mgmt data portion after CCMP header */
+	_rtw_memcpy(pframe, tmp_buf + pattrib->hdrlen, pattrib->pktlen - pattrib->hdrlen);
+	/* move pframe to end of mgmt pkt */
+	pframe += pattrib->pktlen - pattrib->hdrlen;
+	/* add 8 bytes CCMP IV header to length */
+	pattrib->pktlen += pattrib->iv_len;
+
+	#if DBG_MGMT_XMIT_ENC_DUMP
+	/* dump management packet include AES IV header */
+	{
+		int i;
+		printk("Management pkt + IV: ");
+		/* for(i=0; i<pattrib->pktlen; i++) */
+
+		printk("@@@@@@@@@@@@@\n");
+	}
+	#endif
+
+	if (!pattrib->bswenc) {
+		pattrib->key_idx = kid;
+		pattrib->last_txcmdsz = pattrib->pktlen;
+		SetPrivacy(mem_start);
+		goto xmitframe_coalesce_success;
+	}
+
+	if ((pattrib->icv_len > 0) && (pattrib->bswenc)) {
+		_rtw_memcpy(pframe, pattrib->icv, pattrib->icv_len);
+		pframe += pattrib->icv_len;
+	}
+	/* add 8 bytes MIC */
+	pattrib->pktlen += pattrib->icv_len;
+	/* set final tx command size */
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	/* set protected bit must be beofre SW encrypt */
+	SetPrivacy(mem_start);
+
+	#if DBG_MGMT_XMIT_ENC_DUMP
+	/* dump management packet include AES header */
+	{
+		int i;
+		printk("prepare to enc Management pkt + IV: ");
+		for (i = 0; i < pattrib->pktlen; i++)
+			printk(" %02x ", mem_start[i]);
+		printk("@@@@@@@@@@@@@\n");
+	}
+	#endif
+
+	/* software encrypt */
+	/* move to core_wlan_sw_encrypt() because of new txreq architecture */
+
+xmitframe_coalesce_success:
+	_rtw_spinunlock_bh(&padapter->security_key_mutex);
+	rtw_mfree(BIP_AAD, ori_len);
+	return _SUCCESS;
+
+xmitframe_coalesce_fail:
+	_rtw_spinunlock_bh(&padapter->security_key_mutex);
+	rtw_mfree(BIP_AAD, ori_len);
+
+	return _FAIL;
+}
+#endif /* defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH) */
+
+/* Logical Link Control(LLC) SubNetwork Attachment Point(SNAP) header
+ * IEEE LLC/SNAP header contains 8 octets
+ * First 3 octets comprise the LLC portion
+ * SNAP portion, 5 octets, is divided into two fields:
+ *	Organizationally Unique Identifier(OUI), 3 octets,
+ *	type, defined by that organization, 2 octets.
+ */
+s32 rtw_put_snap(u8 *data, u16 h_proto)
+{
+	struct ieee80211_snap_hdr *snap;
+	u8 *oui;
+
+
+	snap = (struct ieee80211_snap_hdr *)data;
+	snap->dsap = 0xaa;
+	snap->ssap = 0xaa;
+	snap->ctrl = 0x03;
+
+	if (h_proto == 0x8137 || h_proto == 0x80f3)
+		oui = P802_1H_OUI;
+	else
+		oui = RFC1042_OUI;
+
+	snap->oui[0] = oui[0];
+	snap->oui[1] = oui[1];
+	snap->oui[2] = oui[2];
+
+	*(u16 *)(data + SNAP_SIZE) = htons(h_proto);
+
+
+	return SNAP_SIZE + sizeof(u16);
+}
+
+void rtw_update_protection(_adapter *padapter, u8 *ie, uint ie_len)
+{
+
+	uint	protection;
+	u8	*perp;
+	sint	 erp_len;
+	struct	xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct	registry_priv *pregistrypriv = &padapter->registrypriv;
+
+
+	switch (pxmitpriv->vcs_setting) {
+	case DISABLE_VCS:
+		pxmitpriv->vcs = NONE_VCS;
+		break;
+
+	case ENABLE_VCS:
+		break;
+
+	case AUTO_VCS:
+	default:
+		perp = rtw_get_ie(ie, _ERPINFO_IE_, &erp_len, ie_len);
+		if (perp == NULL)
+			pxmitpriv->vcs = NONE_VCS;
+		else {
+			protection = (*(perp + 2)) & BIT(1);
+			if (protection) {
+				if (pregistrypriv->vcs_type == RTS_CTS)
+					pxmitpriv->vcs = RTS_CTS;
+				else
+					pxmitpriv->vcs = CTS_TO_SELF;
+			} else
+				pxmitpriv->vcs = NONE_VCS;
+		}
+
+		break;
+
+	}
+
+
+}
+
+#ifdef CONFIG_CORE_TXSC
+void rtw_count_tx_stats_tx_req(_adapter *padapter, struct rtw_xmit_req *txreq, struct sta_info *psta)
+{
+	struct stainfo_stats *pstats = NULL;
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	u32 sz = 0;
+
+	if (txreq->mdata.type == RTW_PHL_PKT_TYPE_DATA) {
+		pmlmepriv->LinkDetectInfo.NumTxOkInPeriod++;
+		pxmitpriv->tx_pkts++;
+		sz = txreq->mdata.pktlen - RTW_SZ_LLC - txreq->mdata.mac_hdr_len;
+		switch (txreq->mdata.sec_type) {
+		case RTW_ENC_WEP104:
+		case RTW_ENC_WEP40:
+			sz -= 4;
+			break;
+		case RTW_ENC_TKIP:
+			sz -= 8;
+			break;
+		case RTW_ENC_CCMP:
+			sz -= 8;
+			break;
+		case RTW_ENC_WAPI:
+			sz -= 18;
+			break;
+		case RTW_ENC_GCMP256:
+		case RTW_ENC_GCMP:
+		case RTW_ENC_CCMP256:
+			sz -= 8;
+			break;
+		default:
+			break;
+		}
+		pxmitpriv->tx_bytes += sz;
+		if (psta) {
+			pstats = &psta->sta_stats;
+			pstats->tx_pkts++;
+			pstats->tx_bytes += sz;
+			#if 0
+			if (is_multicast_mac_addr(psta->phl_sta->mac_addr))
+				pxmitpriv->tx_mc_pkts++;
+			else if (is_broadcast_mac_addr(psta->phl_sta->mac_addr))
+				pxmitpriv->tx_bc_pkts++;
+			else
+				pxmitpriv->tx_uc_pkts++;
+			#endif
+		}
+	}
+}
+#endif
+
+void rtw_count_tx_stats(_adapter *padapter, struct xmit_frame *pxmitframe, int sz)
+{
+	struct sta_info *psta = NULL;
+	struct stainfo_stats *pstats = NULL;
+	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+
+
+	if (pxmitframe->xftype == RTW_TX_OS) {
+		pmlmepriv->LinkDetectInfo.NumTxOkInPeriod++;
+		pxmitpriv->tx_pkts++;
+		pxmitpriv->tx_bytes += sz;
+
+		psta = pxmitframe->attrib.psta;
+		if (psta) {
+			pstats = &psta->sta_stats;
+
+			pstats->tx_pkts++;
+			pstats->tx_bytes += sz;
+		}
+	}
+}
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+static struct xmit_buf *__rtw_alloc_cmd_xmitbuf(struct xmit_priv *pxmitpriv,
+		enum cmdbuf_type buf_type)
+{
+	struct xmit_buf *pxmitbuf =  NULL;
+
+
+	pxmitbuf = &pxmitpriv->pcmd_xmitbuf[buf_type];
+	if (pxmitbuf !=  NULL) {
+		pxmitbuf->priv_data = NULL;
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+		pxmitbuf->len = 0;
+		pxmitbuf->pdata = pxmitbuf->ptail = pxmitbuf->phead;
+		pxmitbuf->agg_num = 0;
+		pxmitbuf->pg_num = 0;
+#endif
+#ifdef CONFIG_PCI_HCI
+		pxmitbuf->len = 0;
+#ifdef CONFIG_TRX_BD_ARCH
+		/*pxmitbuf->buf_desc = NULL;*/
+#else
+		pxmitbuf->desc = NULL;
+#endif
+#endif
+
+		if (pxmitbuf->sctx) {
+			RTW_INFO("%s pxmitbuf->sctx is not NULL\n", __func__);
+			rtw_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_ALLOC);
+		}
+	} else
+		RTW_INFO("%s fail, no xmitbuf available !!!\n", __func__);
+
+	return pxmitbuf;
+}
+
+struct xmit_frame *__rtw_alloc_cmdxmitframe(struct xmit_priv *pxmitpriv,
+		enum cmdbuf_type buf_type)
+{
+	struct xmit_frame		*pcmdframe;
+	struct xmit_buf		*pxmitbuf;
+
+	pcmdframe = rtw_alloc_xmitframe(pxmitpriv);
+	if (pcmdframe == NULL) {
+		RTW_INFO("%s, alloc xmitframe fail\n", __FUNCTION__);
+		return NULL;
+	}
+
+	pxmitbuf = __rtw_alloc_cmd_xmitbuf(pxmitpriv, buf_type);
+	if (pxmitbuf == NULL) {
+		RTW_INFO("%s, alloc xmitbuf fail\n", __FUNCTION__);
+		rtw_free_xmitframe(pxmitpriv, pcmdframe);
+		return NULL;
+	}
+
+	pcmdframe->frame_tag = MGNT_FRAMETAG;
+
+	pcmdframe->pxmitbuf = pxmitbuf;
+
+	pcmdframe->buf_addr = pxmitbuf->pbuf;
+
+	/* initial memory to zero */
+	_rtw_memset(pcmdframe->buf_addr, 0, MAX_CMDBUF_SZ);
+
+	pxmitbuf->priv_data = pcmdframe;
+
+	return pcmdframe;
+
+}
+
+struct xmit_buf *rtw_alloc_xmitbuf_ext(struct xmit_priv *pxmitpriv)
+{
+	struct xmit_buf *pxmitbuf =  NULL;
+	_list *plist, *phead;
+	_queue *pfree_queue = &pxmitpriv->free_xmit_extbuf_queue;
+	unsigned long sp_flags;
+
+	_rtw_spinlock_irq(&pfree_queue->lock, &sp_flags);
+
+	if (_rtw_queue_empty(pfree_queue) == _TRUE)
+		pxmitbuf = NULL;
+	else {
+
+		phead = get_list_head(pfree_queue);
+
+		plist = get_next(phead);
+
+		pxmitbuf = LIST_CONTAINOR(plist, struct xmit_buf, list);
+
+		rtw_list_delete(&(pxmitbuf->list));
+	}
+
+	if (pxmitbuf !=  NULL) {
+		pxmitpriv->free_xmit_extbuf_cnt--;
+#ifdef DBG_XMIT_BUF_EXT
+		RTW_INFO("DBG_XMIT_BUF_EXT ALLOC no=%d,  free_xmit_extbuf_cnt=%d\n", pxmitbuf->no, pxmitpriv->free_xmit_extbuf_cnt);
+#endif
+
+
+		pxmitbuf->priv_data = NULL;
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+		pxmitbuf->len = 0;
+		pxmitbuf->pdata = pxmitbuf->ptail = pxmitbuf->phead;
+		pxmitbuf->agg_num = 1;
+#endif
+#ifdef CONFIG_PCI_HCI
+		pxmitbuf->len = 0;
+#ifdef CONFIG_TRX_BD_ARCH
+		/*pxmitbuf->buf_desc = NULL;*/
+#else
+		pxmitbuf->desc = NULL;
+#endif
+#endif
+
+		if (pxmitbuf->sctx) {
+			RTW_INFO("%s pxmitbuf->sctx is not NULL\n", __func__);
+			rtw_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_ALLOC);
+		}
+
+	}
+
+	_rtw_spinunlock_irq(&pfree_queue->lock, &sp_flags);
+
+
+	return pxmitbuf;
+}
+
+s32 rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
+{
+	_queue *pfree_queue = &pxmitpriv->free_xmit_extbuf_queue;
+	unsigned long sp_flags;
+
+	if (pxmitbuf == NULL)
+		return _FAIL;
+
+	_rtw_spinlock_irq(&pfree_queue->lock, &sp_flags);
+
+	rtw_list_delete(&pxmitbuf->list);
+
+	rtw_list_insert_tail(&(pxmitbuf->list), get_list_head(pfree_queue));
+	pxmitpriv->free_xmit_extbuf_cnt++;
+#ifdef DBG_XMIT_BUF_EXT
+	RTW_INFO("DBG_XMIT_BUF_EXT FREE no=%d, free_xmit_extbuf_cnt=%d\n", pxmitbuf->no, pxmitpriv->free_xmit_extbuf_cnt);
+#endif
+
+	_rtw_spinunlock_irq(&pfree_queue->lock, &sp_flags);
+
+
+	return _SUCCESS;
+}
+
+struct xmit_buf *rtw_alloc_xmitbuf(struct xmit_priv *pxmitpriv)
+{
+	struct xmit_buf *pxmitbuf =  NULL;
+	_list *plist, *phead;
+	_queue *pfree_xmitbuf_queue = &pxmitpriv->free_xmitbuf_queue;
+	unsigned long sp_flags;
+
+	/* RTW_INFO("+rtw_alloc_xmitbuf\n"); */
+
+	_rtw_spinlock_irq(&pfree_xmitbuf_queue->lock, &sp_flags);
+
+	if (_rtw_queue_empty(pfree_xmitbuf_queue) == _TRUE)
+		pxmitbuf = NULL;
+	else {
+
+		phead = get_list_head(pfree_xmitbuf_queue);
+
+		plist = get_next(phead);
+
+		pxmitbuf = LIST_CONTAINOR(plist, struct xmit_buf, list);
+
+		rtw_list_delete(&(pxmitbuf->list));
+	}
+
+	if (pxmitbuf !=  NULL) {
+		pxmitpriv->free_xmitbuf_cnt--;
+#ifdef DBG_XMIT_BUF
+		RTW_INFO("DBG_XMIT_BUF ALLOC no=%d,  free_xmitbuf_cnt=%d\n", pxmitbuf->no, pxmitpriv->free_xmitbuf_cnt);
+#endif
+		/* RTW_INFO("alloc, free_xmitbuf_cnt=%d\n", pxmitpriv->free_xmitbuf_cnt); */
+
+		pxmitbuf->priv_data = NULL;
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+		pxmitbuf->len = 0;
+		pxmitbuf->pdata = pxmitbuf->ptail = pxmitbuf->phead;
+		pxmitbuf->agg_num = 0;
+		pxmitbuf->pg_num = 0;
+#endif
+#ifdef CONFIG_PCI_HCI
+		pxmitbuf->len = 0;
+#ifdef CONFIG_TRX_BD_ARCH
+		/*pxmitbuf->buf_desc = NULL;*/
+#else
+		pxmitbuf->desc = NULL;
+#endif
+#endif
+
+		if (pxmitbuf->sctx) {
+			RTW_INFO("%s pxmitbuf->sctx is not NULL\n", __func__);
+			rtw_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_ALLOC);
+		}
+	}
+#ifdef DBG_XMIT_BUF
+	else
+		RTW_INFO("DBG_XMIT_BUF rtw_alloc_xmitbuf return NULL\n");
+#endif
+
+	_rtw_spinunlock_irq(&pfree_xmitbuf_queue->lock, &sp_flags);
+
+
+	return pxmitbuf;
+}
+
+s32 rtw_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
+{
+	_queue *pfree_xmitbuf_queue = &pxmitpriv->free_xmitbuf_queue;
+	unsigned long sp_flags;
+
+	/* RTW_INFO("+rtw_free_xmitbuf\n"); */
+
+	if (pxmitbuf == NULL)
+		return _FAIL;
+
+	if (pxmitbuf->sctx) {
+		RTW_INFO("%s pxmitbuf->sctx is not NULL\n", __func__);
+		rtw_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_FREE);
+	}
+
+	if (pxmitbuf->buf_tag == XMITBUF_CMD) {
+	} else if (pxmitbuf->buf_tag == XMITBUF_MGNT)
+		rtw_free_xmitbuf_ext(pxmitpriv, pxmitbuf);
+	else {
+		_rtw_spinlock_irq(&pfree_xmitbuf_queue->lock, &sp_flags);
+
+		rtw_list_delete(&pxmitbuf->list);
+
+		rtw_list_insert_tail(&(pxmitbuf->list), get_list_head(pfree_xmitbuf_queue));
+
+		pxmitpriv->free_xmitbuf_cnt++;
+		/* RTW_INFO("FREE, free_xmitbuf_cnt=%d\n", pxmitpriv->free_xmitbuf_cnt); */
+#ifdef DBG_XMIT_BUF
+		RTW_INFO("DBG_XMIT_BUF FREE no=%d, free_xmitbuf_cnt=%d\n", pxmitbuf->no, pxmitpriv->free_xmitbuf_cnt);
+#endif
+		_rtw_spinunlock_irq(&pfree_xmitbuf_queue->lock, &sp_flags);
+	}
+
+
+	return _SUCCESS;
+}
+#endif
+
+void rtw_init_xmitframe(struct xmit_frame *pxframe)
+{
+	if (pxframe !=  NULL) { /* default value setting */
+		#if 0 /*CONFIG_CORE_XMITBUF*/
+		pxframe->buf_addr = NULL;
+		pxframe->pxmitbuf = NULL;
+		#endif
+
+		_rtw_memset(&pxframe->attrib, 0, sizeof(struct pkt_attrib));
+		/* pxframe->attrib.psta = NULL; */
+
+		pxframe->frame_tag = DATA_FRAMETAG;
+
+#ifdef CONFIG_USB_HCI
+		pxframe->pkt = NULL;
+#ifdef USB_PACKET_OFFSET_SZ
+		pxframe->pkt_offset = (PACKET_OFFSET_SZ / 8);
+#else
+		pxframe->pkt_offset = 1;/* default use pkt_offset to fill tx desc */
+#endif
+
+#ifdef CONFIG_USB_TX_AGGREGATION
+		pxframe->agg_num = 1;
+#endif
+
+#endif /* #ifdef CONFIG_USB_HCI */
+
+#ifdef CONFIG_XMIT_ACK
+		pxframe->ack_report = 0;
+#endif
+		pxframe->txfree_cnt = 0;
+	}
+}
+
+/*
+Calling context:
+1. OS_TXENTRY
+2. RXENTRY (rx_thread or RX_ISR/RX_CallBack)
+
+If we turn on USE_RXTHREAD, then, no need for critical section.
+Otherwise, we must use _enter/_exit critical to protect free_xmit_queue...
+
+Must be very very cautious...
+
+*/
+
+#ifdef RTW_PHL_TX
+void core_tx_init_xmitframe(struct xmit_frame *pxframe)
+{
+	if (!pxframe)
+		return;
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	pxframe->pxmitbuf = NULL;
+	#endif
+	_rtw_memset(&pxframe->attrib, 0, sizeof(struct pkt_attrib));
+	/* TXREQ_QMGT */
+	pxframe->ptxreq_buf = NULL;
+	pxframe->phl_txreq = NULL;
+
+	pxframe->txreq_cnt = 0;
+	pxframe->txfree_cnt = 0;
+}
+
+s32 core_tx_alloc_xmitframe(_adapter *padapter, struct xmit_frame **pxmitframe, u16 os_qid)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pxframe = NULL;
+	_queue *pfree_xmit_queue = &pxmitpriv->free_xmit_queue;
+	_list *plist, *phead;
+
+	PHLTX_LOG;
+
+	_rtw_spinlock_bh(&pfree_xmit_queue->lock);
+
+	if (_rtw_queue_empty(pfree_xmit_queue) == _TRUE) {
+		_rtw_spinunlock_bh(&pfree_xmit_queue->lock);
+		return FAIL;
+	} else {
+		phead = get_list_head(pfree_xmit_queue);
+
+		plist = get_next(phead);
+
+		pxframe = LIST_CONTAINOR(plist, struct xmit_frame, list);
+
+		rtw_list_delete(&pxframe->list);
+		pxmitpriv->free_xmitframe_cnt--;
+		pxframe->os_qid = os_qid;
+	}
+
+	_rtw_spinunlock_bh(&pfree_xmit_queue->lock);
+	rtw_os_check_stop_queue(pxmitpriv->adapter, os_qid);
+	core_tx_init_xmitframe(pxframe);
+
+	*pxmitframe = pxframe;
+	return SUCCESS;
+}
+
+s32 core_tx_free_xmitframe(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct xmit_priv 	*pxmitpriv = &padapter->xmitpriv;
+	_queue *queue = NULL;
+	/* TXREQ_QMGT */
+	struct xmit_txreq_buf *ptxreq_buf = NULL;
+	int i;
+	struct rtw_xmit_req *txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+
+	PHLTX_LOG;
+
+	if (pxframe == NULL)
+		goto exit;
+
+	/* TXREQ_QMGT */
+	ptxreq_buf = pxframe->ptxreq_buf;
+
+	pxframe->txfree_cnt++;
+
+	/* ?? shall detail check, like free 1 2 3, not free 2 2 3 */
+	/* ?? rtw_alloc_xmitframe_once case, seems no one use*/
+
+	if (pxframe->txfree_cnt < pxframe->txreq_cnt)
+		goto exit;
+
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	if (pxframe->pxmitbuf)
+		rtw_free_xmitbuf(pxmitpriv, pxframe->pxmitbuf);
+	#endif
+
+	for (i = 0; i < pxframe->txreq_cnt; i++) {
+		if (!pxframe->buf_need_free)
+			break;
+		if (!(pxframe->buf_need_free & BIT(i)))
+			continue;
+		pxframe->buf_need_free &= ~BIT(i);
+
+		txreq = &pxframe->phl_txreq[i];
+		rtw_warn_on(txreq->pkt_cnt != 1);
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		if (pkt_list->vir_addr && pkt_list->length)
+			rtw_mfree(pkt_list->vir_addr, pkt_list->length);
+	}
+
+	if (ptxreq_buf) {
+		queue = &padapter->free_txreq_queue;
+		_rtw_spinlock_bh(&queue->lock);
+
+		rtw_list_delete(&ptxreq_buf->list);
+		rtw_list_insert_tail(&ptxreq_buf->list, get_list_head(queue));
+
+		padapter->free_txreq_cnt++;
+		_rtw_spinunlock_bh(&queue->lock);
+	} else {
+		if (pxframe->ext_tag == 0)
+			;//printk("%s:tx recyele: ptxreq_buf=NULL\n", __FUNCTION__);
+	}
+
+	rtw_os_xmit_complete(padapter, pxframe);
+
+	if (pxframe->ext_tag == 0)
+		queue = &pxmitpriv->free_xmit_queue;
+	else if (pxframe->ext_tag == 1)
+		queue = &pxmitpriv->free_xframe_ext_queue;
+	else
+		rtw_warn_on(1);
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	rtw_list_delete(&pxframe->list);
+	rtw_list_insert_tail(&pxframe->list, get_list_head(queue));
+
+	if (pxframe->ext_tag == 0)
+		pxmitpriv->free_xmitframe_cnt++;
+	else if (pxframe->ext_tag == 1)
+		pxmitpriv->free_xframe_ext_cnt++;
+
+	_rtw_spinunlock_bh(&queue->lock);
+
+	if (queue == &pxmitpriv->free_xmit_queue)
+		rtw_os_check_wakup_queue(padapter, pxframe->os_qid);
+
+exit:
+	return _SUCCESS;
+}
+
+#endif
+
+struct xmit_frame *rtw_alloc_xmitframe(struct xmit_priv *pxmitpriv, u16 os_qid)/* (_queue *pfree_xmit_queue) */
+{
+	/*
+		Please remember to use all the osdep_service api,
+		and lock/unlock or _enter/_exit critical to protect
+		pfree_xmit_queue
+	*/
+
+	struct xmit_frame *pxframe = NULL;
+	_list *plist, *phead;
+	_queue *pfree_xmit_queue = &pxmitpriv->free_xmit_queue;
+
+
+	_rtw_spinlock_bh(&pfree_xmit_queue->lock);
+
+	if (_rtw_queue_empty(pfree_xmit_queue) == _TRUE) {
+		pxframe =  NULL;
+	} else {
+		phead = get_list_head(pfree_xmit_queue);
+
+		plist = get_next(phead);
+
+		pxframe = LIST_CONTAINOR(plist, struct xmit_frame, list);
+
+		rtw_list_delete(&(pxframe->list));
+		pxmitpriv->free_xmitframe_cnt--;
+		pxframe->os_qid = os_qid;
+	}
+
+	_rtw_spinunlock_bh(&pfree_xmit_queue->lock);
+
+	if (pxframe)
+		rtw_os_check_stop_queue(pxmitpriv->adapter, os_qid);
+
+	rtw_init_xmitframe(pxframe);
+
+
+	return pxframe;
+}
+
+struct xmit_frame *rtw_alloc_xmitframe_ext(struct xmit_priv *pxmitpriv)
+{
+	struct xmit_frame *pxframe = NULL;
+	_list *plist, *phead;
+	_queue *queue = &pxmitpriv->free_xframe_ext_queue;
+
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	if (_rtw_queue_empty(queue) == _TRUE) {
+		pxframe =  NULL;
+	} else {
+		phead = get_list_head(queue);
+		plist = get_next(phead);
+		pxframe = LIST_CONTAINOR(plist, struct xmit_frame, list);
+
+		rtw_list_delete(&(pxframe->list));
+		pxmitpriv->free_xframe_ext_cnt--;
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+
+	rtw_init_xmitframe(pxframe);
+
+
+	return pxframe;
+}
+
+struct xmit_frame *rtw_alloc_xmitframe_once(struct xmit_priv *pxmitpriv)
+{
+	struct xmit_frame *pxframe = NULL;
+	u8 *alloc_addr;
+
+	alloc_addr = rtw_zmalloc(sizeof(struct xmit_frame) + 4);
+
+	if (alloc_addr == NULL)
+		goto exit;
+
+	pxframe = (struct xmit_frame *)N_BYTE_ALIGMENT((SIZE_PTR)(alloc_addr), 4);
+	pxframe->alloc_addr = alloc_addr;
+
+	pxframe->padapter = pxmitpriv->adapter;
+	pxframe->frame_tag = NULL_FRAMETAG;
+
+	pxframe->pkt = NULL;
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	pxframe->buf_addr = NULL;
+	pxframe->pxmitbuf = NULL;
+	#endif
+
+	rtw_init_xmitframe(pxframe);
+
+	RTW_INFO("################## %s ##################\n", __func__);
+
+exit:
+	return pxframe;
+}
+
+s32 rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitframe)
+{
+	_queue *queue = NULL;
+	_adapter *padapter = pxmitpriv->adapter;
+	struct sk_buff *pndis_pkt = NULL;
+
+
+	if (pxmitframe == NULL) {
+		goto exit;
+	}
+
+	if (pxmitframe->pkt) {
+		pndis_pkt = pxmitframe->pkt;
+		pxmitframe->pkt = NULL;
+	}
+
+	if (pxmitframe->alloc_addr) {
+		RTW_INFO("################## %s with alloc_addr ##################\n", __func__);
+		rtw_mfree(pxmitframe->alloc_addr, sizeof(struct xmit_frame) + 4);
+		goto check_pkt_complete;
+	}
+
+	if (pxmitframe->ext_tag == 0)
+		queue = &pxmitpriv->free_xmit_queue;
+	else if (pxmitframe->ext_tag == 1)
+		queue = &pxmitpriv->free_xframe_ext_queue;
+	else
+		rtw_warn_on(1);
+
+	_rtw_spinlock_bh(&queue->lock);
+
+	rtw_list_delete(&pxmitframe->list);
+	rtw_list_insert_tail(&pxmitframe->list, get_list_head(queue));
+	if (pxmitframe->ext_tag == 0) {
+		pxmitpriv->free_xmitframe_cnt++;
+	} else if (pxmitframe->ext_tag == 1) {
+		pxmitpriv->free_xframe_ext_cnt++;
+	} else {
+	}
+
+	_rtw_spinunlock_bh(&queue->lock);
+	if (queue == &pxmitpriv->free_xmit_queue)
+		rtw_os_check_wakup_queue(padapter, pxmitframe->os_qid);
+
+check_pkt_complete:
+
+	if (pndis_pkt)
+		rtw_os_pkt_complete(padapter, pndis_pkt);
+
+exit:
+
+
+	return _SUCCESS;
+}
+
+void rtw_free_xmitframe_queue(struct xmit_priv *pxmitpriv, _queue *pframequeue)
+{
+	_list	*plist, *phead;
+	struct	xmit_frame	*pxmitframe;
+
+
+	_rtw_spinlock_bh(&(pframequeue->lock));
+
+	phead = get_list_head(pframequeue);
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+
+		pxmitframe = LIST_CONTAINOR(plist, struct xmit_frame, list);
+
+		plist = get_next(plist);
+
+		core_tx_free_xmitframe(pxmitpriv->adapter, pxmitframe);
+
+	}
+	_rtw_spinunlock_bh(&(pframequeue->lock));
+
+}
+
+s32 rtw_xmitframe_enqueue(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	DBG_COUNTER(padapter->tx_logs.core_tx_enqueue);
+	if (rtw_xmit_classifier(padapter, pxmitframe) == _FAIL) {
+		/*		pxmitframe->pkt = NULL; */
+		return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+static struct xmit_frame *dequeue_one_xmitframe(struct xmit_priv *pxmitpriv, struct hw_xmit *phwxmit, struct tx_servq *ptxservq, _queue *pframe_queue)
+{
+	_list	*xmitframe_plist, *xmitframe_phead;
+	struct	xmit_frame	*pxmitframe = NULL;
+
+	xmitframe_phead = get_list_head(pframe_queue);
+	xmitframe_plist = get_next(xmitframe_phead);
+
+	if ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE) {
+		pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+		rtw_list_delete(&pxmitframe->list);
+
+		ptxservq->qcnt--;
+	}
+
+	return pxmitframe;
+}
+
+static struct xmit_frame *get_one_xmitframe(struct xmit_priv *pxmitpriv, struct hw_xmit *phwxmit, struct tx_servq *ptxservq, _queue *pframe_queue)
+{
+	_list	*xmitframe_plist, *xmitframe_phead;
+	struct	xmit_frame	*pxmitframe = NULL;
+
+	xmitframe_phead = get_list_head(pframe_queue);
+	xmitframe_plist = get_next(xmitframe_phead);
+
+	if ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE)
+		pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+	return pxmitframe;
+}
+
+struct xmit_frame *rtw_get_xframe(struct xmit_priv *pxmitpriv, int *num_frame)
+{
+	_list *sta_plist, *sta_phead;
+	struct hw_xmit *phwxmit_i = pxmitpriv->hwxmits;
+	sint entry =  pxmitpriv->hwxmit_entry;
+
+	struct hw_xmit *phwxmit;
+	struct tx_servq *ptxservq = NULL;
+	_queue *pframe_queue = NULL;
+	struct xmit_frame *pxmitframe = NULL;
+	_adapter *padapter = pxmitpriv->adapter;
+	int i, inx[4];
+
+	inx[0] = 0;
+	inx[1] = 1;
+	inx[2] = 2;
+	inx[3] = 3;
+
+	*num_frame = 0;
+
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	for (i = 0; i < entry; i++) {
+		phwxmit = phwxmit_i + inx[i];
+
+		sta_phead = get_list_head(phwxmit->sta_queue);
+		sta_plist = get_next(sta_phead);
+
+		while ((rtw_end_of_queue_search(sta_phead, sta_plist)) == _FALSE) {
+
+			ptxservq = LIST_CONTAINOR(sta_plist, struct tx_servq, tx_pending);
+			pframe_queue = &ptxservq->sta_pending;
+
+			if (ptxservq->qcnt) {
+				*num_frame = ptxservq->qcnt;
+				pxmitframe = get_one_xmitframe(pxmitpriv, phwxmit, ptxservq, pframe_queue);
+				goto exit;
+			}
+			sta_plist = get_next(sta_plist);
+		}
+	}
+
+exit:
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+	return pxmitframe;
+}
+
+
+struct xmit_frame *rtw_dequeue_xframe(struct xmit_priv *pxmitpriv, struct hw_xmit *phwxmit_i, sint entry)
+{
+	_list *sta_plist, *sta_phead;
+	struct hw_xmit *phwxmit;
+	struct tx_servq *ptxservq = NULL;
+	_queue *pframe_queue = NULL;
+	struct xmit_frame *pxmitframe = NULL;
+	_adapter *padapter = pxmitpriv->adapter;
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	int i, inx[4];
+
+	inx[0] = 0;
+	inx[1] = 1;
+	inx[2] = 2;
+	inx[3] = 3;
+
+	if (pregpriv->wifi_spec == 1) {
+		int j;
+#if 0
+		if (flags < XMIT_QUEUE_ENTRY) {
+			/* priority exchange according to the completed xmitbuf flags. */
+			inx[flags] = 0;
+			inx[0] = flags;
+		}
+#endif
+
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI) || defined(CONFIG_PCI_HCI)
+		for (j = 0; j < 4; j++)
+			inx[j] = pxmitpriv->wmm_para_seq[j];
+#endif
+	}
+
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	for (i = 0; i < entry; i++) {
+		phwxmit = phwxmit_i + inx[i];
+
+		/* _rtw_spinlock_irq(&phwxmit->sta_queue->lock, &sp_flags); */
+
+		sta_phead = get_list_head(phwxmit->sta_queue);
+		sta_plist = get_next(sta_phead);
+
+		while ((rtw_end_of_queue_search(sta_phead, sta_plist)) == _FALSE) {
+
+			ptxservq = LIST_CONTAINOR(sta_plist, struct tx_servq, tx_pending);
+
+			pframe_queue = &ptxservq->sta_pending;
+
+			pxmitframe = dequeue_one_xmitframe(pxmitpriv, phwxmit, ptxservq, pframe_queue);
+
+			if (pxmitframe) {
+				phwxmit->accnt--;
+
+				/* Remove sta node when there is no pending packets. */
+				if (_rtw_queue_empty(pframe_queue)) /* must be done after get_next and before break */
+					rtw_list_delete(&ptxservq->tx_pending);
+
+				/* _rtw_spinunlock_irq(&phwxmit->sta_queue->lock, sp_flags); */
+
+				goto exit;
+			}
+
+			sta_plist = get_next(sta_plist);
+
+		}
+
+		/* _rtw_spinunlock_irq(&phwxmit->sta_queue->lock, sp_flags); */
+
+	}
+
+exit:
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+	return pxmitframe;
+}
+
+#if 1
+struct tx_servq *rtw_get_sta_pending(_adapter *padapter, struct sta_info *psta, sint up, u8 *ac)
+{
+	struct tx_servq *ptxservq = NULL;
+
+
+	switch (up) {
+	case 1:
+	case 2:
+		ptxservq = &(psta->sta_xmitpriv.bk_q);
+		*(ac) = 3;
+		break;
+
+	case 4:
+	case 5:
+		ptxservq = &(psta->sta_xmitpriv.vi_q);
+		*(ac) = 1;
+		break;
+
+	case 6:
+	case 7:
+		ptxservq = &(psta->sta_xmitpriv.vo_q);
+		*(ac) = 0;
+		break;
+
+	case 0:
+	case 3:
+	default:
+		ptxservq = &(psta->sta_xmitpriv.be_q);
+		*(ac) = 2;
+		break;
+
+	}
+
+
+	return ptxservq;
+}
+#else
+__inline static struct tx_servq *rtw_get_sta_pending
+(_adapter *padapter, _queue **ppstapending, struct sta_info *psta, sint up) {
+	struct tx_servq *ptxservq;
+	struct hw_xmit *phwxmits =  padapter->xmitpriv.hwxmits;
+
+
+#ifdef CONFIG_RTL8711
+
+	if (IS_MCAST(psta->phl_sta->mac_addr)) {
+		ptxservq = &(psta->sta_xmitpriv.be_q); /* we will use be_q to queue bc/mc frames in BCMC_stainfo */
+		*ppstapending = &padapter->xmitpriv.bm_pending;
+	} else
+#endif
+	{
+		switch (up) {
+		case 1:
+		case 2:
+			ptxservq = &(psta->sta_xmitpriv.bk_q);
+			*ppstapending = &padapter->xmitpriv.bk_pending;
+			(phwxmits + 3)->accnt++;
+			break;
+
+		case 4:
+		case 5:
+			ptxservq = &(psta->sta_xmitpriv.vi_q);
+			*ppstapending = &padapter->xmitpriv.vi_pending;
+			(phwxmits + 1)->accnt++;
+			break;
+
+		case 6:
+		case 7:
+			ptxservq = &(psta->sta_xmitpriv.vo_q);
+			*ppstapending = &padapter->xmitpriv.vo_pending;
+			(phwxmits + 0)->accnt++;
+			break;
+
+		case 0:
+		case 3:
+		default:
+			ptxservq = &(psta->sta_xmitpriv.be_q);
+			*ppstapending = &padapter->xmitpriv.be_pending;
+			(phwxmits + 2)->accnt++;
+			break;
+
+		}
+
+	}
+
+
+	return ptxservq;
+}
+#endif
+
+/*
+ * Will enqueue pxmitframe to the proper queue,
+ * and indicate it to xx_pending list.....
+ */
+s32 rtw_xmit_classifier(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	u8	ac_index;
+	struct sta_info	*psta;
+	struct tx_servq	*ptxservq;
+	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
+	struct hw_xmit	*phwxmits =  padapter->xmitpriv.hwxmits;
+	sint res = _SUCCESS;
+
+
+	DBG_COUNTER(padapter->tx_logs.core_tx_enqueue_class);
+
+	/*
+		if (pattrib->psta) {
+			psta = pattrib->psta;
+		} else {
+			RTW_INFO("%s, call rtw_get_stainfo()\n", __func__);
+			psta = rtw_get_stainfo(pstapriv, pattrib->ra);
+		}
+	*/
+
+	psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+	if (pattrib->psta != psta) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_enqueue_class_err_sta);
+		RTW_INFO("%s, pattrib->psta(%p) != psta(%p)\n", __func__, pattrib->psta, psta);
+		return _FAIL;
+	}
+
+	if (psta == NULL) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_enqueue_class_err_nosta);
+		res = _FAIL;
+		RTW_INFO("rtw_xmit_classifier: psta == NULL\n");
+		goto exit;
+	}
+
+	if (!(psta->state & WIFI_ASOC_STATE)) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_enqueue_class_err_fwlink);
+		RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+		return _FAIL;
+	}
+
+	ptxservq = rtw_get_sta_pending(padapter, psta, pattrib->priority, (u8 *)(&ac_index));
+
+	/* _rtw_spinlock_irq(&pstapending->lock, &flags); */
+
+	if (rtw_is_list_empty(&ptxservq->tx_pending))
+		rtw_list_insert_tail(&ptxservq->tx_pending, get_list_head(phwxmits[ac_index].sta_queue));
+
+	/* _rtw_spinlock_irq(&ptxservq->sta_pending.lock, &sp_flags); */
+
+	rtw_list_insert_tail(&pxmitframe->list, get_list_head(&ptxservq->sta_pending));
+	ptxservq->qcnt++;
+	phwxmits[ac_index].accnt++;
+
+	/* _rtw_spinunlock_irq(&ptxservq->sta_pending.lock, &sp_flags); */
+
+	/* _rtw_spinunlock_irq(&pstapending->lock, &flags); */
+
+exit:
+
+
+	return res;
+}
+
+void rtw_alloc_hwxmits(_adapter *padapter)
+{
+	struct hw_xmit *hwxmits;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	pxmitpriv->hwxmit_entry = HWXMIT_ENTRY;
+
+	pxmitpriv->hwxmits = NULL;
+
+	pxmitpriv->hwxmits = (struct hw_xmit *)rtw_zmalloc(sizeof(struct hw_xmit) * pxmitpriv->hwxmit_entry);
+
+	if (pxmitpriv->hwxmits == NULL) {
+		RTW_INFO("alloc hwxmits fail!...\n");
+		return;
+	}
+
+	hwxmits = pxmitpriv->hwxmits;
+
+	if (pxmitpriv->hwxmit_entry == 5) {
+		/* pxmitpriv->bmc_txqueue.head = 0; */
+		/* hwxmits[0] .phwtxqueue = &pxmitpriv->bmc_txqueue; */
+		hwxmits[0] .sta_queue = &pxmitpriv->bm_pending;
+
+		/* pxmitpriv->vo_txqueue.head = 0; */
+		/* hwxmits[1] .phwtxqueue = &pxmitpriv->vo_txqueue; */
+		hwxmits[1] .sta_queue = &pxmitpriv->vo_pending;
+
+		/* pxmitpriv->vi_txqueue.head = 0; */
+		/* hwxmits[2] .phwtxqueue = &pxmitpriv->vi_txqueue; */
+		hwxmits[2] .sta_queue = &pxmitpriv->vi_pending;
+
+		/* pxmitpriv->bk_txqueue.head = 0; */
+		/* hwxmits[3] .phwtxqueue = &pxmitpriv->bk_txqueue; */
+		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
+
+		/* pxmitpriv->be_txqueue.head = 0; */
+		/* hwxmits[4] .phwtxqueue = &pxmitpriv->be_txqueue; */
+		hwxmits[4] .sta_queue = &pxmitpriv->be_pending;
+
+	} else if (pxmitpriv->hwxmit_entry == 4) {
+
+		/* pxmitpriv->vo_txqueue.head = 0; */
+		/* hwxmits[0] .phwtxqueue = &pxmitpriv->vo_txqueue; */
+		hwxmits[0] .sta_queue = &pxmitpriv->vo_pending;
+
+		/* pxmitpriv->vi_txqueue.head = 0; */
+		/* hwxmits[1] .phwtxqueue = &pxmitpriv->vi_txqueue; */
+		hwxmits[1] .sta_queue = &pxmitpriv->vi_pending;
+
+		/* pxmitpriv->be_txqueue.head = 0; */
+		/* hwxmits[2] .phwtxqueue = &pxmitpriv->be_txqueue; */
+		hwxmits[2] .sta_queue = &pxmitpriv->be_pending;
+
+		/* pxmitpriv->bk_txqueue.head = 0; */
+		/* hwxmits[3] .phwtxqueue = &pxmitpriv->bk_txqueue; */
+		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
+	} else {
+
+
+	}
+
+
+}
+
+void rtw_free_hwxmits(_adapter *padapter)
+{
+	struct hw_xmit *hwxmits;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	hwxmits = pxmitpriv->hwxmits;
+	if (hwxmits)
+		rtw_mfree((u8 *)hwxmits, (sizeof(struct hw_xmit) * pxmitpriv->hwxmit_entry));
+}
+
+void rtw_init_hwxmits(struct hw_xmit *phwxmit, sint entry)
+{
+	sint i;
+	for (i = 0; i < entry; i++, phwxmit++) {
+		/* _rtw_spinlock_init(&phwxmit->xmit_lock); */
+		/* _rtw_init_listhead(&phwxmit->pending);		 */
+		/* phwxmit->txcmdcnt = 0; */
+		phwxmit->accnt = 0;
+	}
+}
+
+#ifdef CONFIG_BR_EXT
+int rtw_br_client_tx(_adapter *padapter, struct sk_buff **pskb)
+{
+	struct sk_buff *skb = *pskb;
+	/* if(MLME_IS_STA(adapter) */
+	{
+		void dhcp_flag_bcast(_adapter *priv, struct sk_buff *skb);
+		int res, is_vlan_tag = 0, i, do_nat25 = 1;
+		unsigned short vlan_hdr = 0;
+		void *br_port = NULL;
+
+		/* mac_clone_handle_frame(priv, skb); */
+
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35))
+		br_port = padapter->pnetdev->br_port;
+#else   /* (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35)) */
+		rcu_read_lock();
+		br_port = rcu_dereference(padapter->pnetdev->rx_handler_data);
+		rcu_read_unlock();
+#endif /* (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35)) */
+		_rtw_spinlock_bh(&padapter->br_ext_lock);
+		if (!(skb->data[0] & 1) &&
+		    br_port &&
+		    !_rtw_memcmp(skb->data + MACADDRLEN, padapter->br_mac, MACADDRLEN) &&
+		    *((unsigned short *)(skb->data + MACADDRLEN * 2)) != __constant_htons(ETH_P_8021Q) &&
+		    *((unsigned short *)(skb->data + MACADDRLEN * 2)) == __constant_htons(ETH_P_IP) &&
+		    _rtw_memcmp(padapter->scdb_mac, skb->data + MACADDRLEN, MACADDRLEN) && padapter->scdb_entry) {
+			_rtw_memcpy(skb->data + MACADDRLEN, GET_MY_HWADDR(padapter), MACADDRLEN);
+			padapter->scdb_entry->ageing_timer = jiffies;
+			_rtw_spinunlock_bh(&padapter->br_ext_lock);
+		} else
+			/* if (!priv->pmib->ethBrExtInfo.nat25_disable)		 */
+		{
+			/*			if (priv->dev->br_port &&
+			 *				 _rtw_memcmp(skb->data+MACADDRLEN, priv->br_mac, MACADDRLEN)) { */
+#if 1
+			if (*((unsigned short *)(skb->data + MACADDRLEN * 2)) == __constant_htons(ETH_P_8021Q)) {
+				is_vlan_tag = 1;
+				vlan_hdr = *((unsigned short *)(skb->data + MACADDRLEN * 2 + 2));
+				for (i = 0; i < 6; i++)
+					*((unsigned short *)(skb->data + MACADDRLEN * 2 + 2 - i * 2)) = *((unsigned short *)(skb->data + MACADDRLEN * 2 - 2 - i * 2));
+				skb_pull(skb, 4);
+			}
+			/* if SA == br_mac && skb== IP  => copy SIP to br_ip ?? why */
+			if (_rtw_memcmp(skb->data + MACADDRLEN, padapter->br_mac, MACADDRLEN) &&
+			    (*((unsigned short *)(skb->data + MACADDRLEN * 2)) == __constant_htons(ETH_P_IP)))
+				_rtw_memcpy(padapter->br_ip, skb->data + WLAN_ETHHDR_LEN + 12, 4);
+
+			if (*((unsigned short *)(skb->data + MACADDRLEN * 2)) == __constant_htons(ETH_P_IP)) {
+				if (!_rtw_memcmp(padapter->scdb_mac, skb->data + MACADDRLEN, MACADDRLEN)) {
+					void *scdb_findEntry(_adapter *priv, unsigned char *macAddr, unsigned char *ipAddr);
+
+					padapter->scdb_entry = (struct nat25_network_db_entry *)scdb_findEntry(padapter,
+						skb->data + MACADDRLEN, skb->data + WLAN_ETHHDR_LEN + 12);
+					if (padapter->scdb_entry != NULL) {
+						_rtw_memcpy(padapter->scdb_mac, skb->data + MACADDRLEN, MACADDRLEN);
+						_rtw_memcpy(padapter->scdb_ip, skb->data + WLAN_ETHHDR_LEN + 12, 4);
+						padapter->scdb_entry->ageing_timer = jiffies;
+						do_nat25 = 0;
+					}
+				} else {
+					if (padapter->scdb_entry) {
+						padapter->scdb_entry->ageing_timer = jiffies;
+						do_nat25 = 0;
+					} else {
+						_rtw_memset(padapter->scdb_mac, 0, MACADDRLEN);
+						_rtw_memset(padapter->scdb_ip, 0, 4);
+					}
+				}
+			}
+			_rtw_spinunlock_bh(&padapter->br_ext_lock);
+#endif /* 1 */
+			if (do_nat25) {
+				int nat25_db_handle(_adapter *priv, struct sk_buff *skb, int method);
+				if (nat25_db_handle(padapter, skb, NAT25_CHECK) == 0) {
+					struct sk_buff *newskb;
+
+					if (is_vlan_tag) {
+						skb_push(skb, 4);
+						for (i = 0; i < 6; i++)
+							*((unsigned short *)(skb->data + i * 2)) = *((unsigned short *)(skb->data + 4 + i * 2));
+						*((unsigned short *)(skb->data + MACADDRLEN * 2)) = __constant_htons(ETH_P_8021Q);
+						*((unsigned short *)(skb->data + MACADDRLEN * 2 + 2)) = vlan_hdr;
+					}
+
+					newskb = rtw_skb_copy(skb);
+					if (newskb == NULL) {
+						/* priv->ext_stats.tx_drops++; */
+						DEBUG_ERR("TX DROP: rtw_skb_copy fail!\n");
+						/* goto stop_proc; */
+						return -1;
+					}
+					rtw_skb_free(skb);
+
+					*pskb = skb = newskb;
+					if (is_vlan_tag) {
+						vlan_hdr = *((unsigned short *)(skb->data + MACADDRLEN * 2 + 2));
+						for (i = 0; i < 6; i++)
+							*((unsigned short *)(skb->data + MACADDRLEN * 2 + 2 - i * 2)) = *((unsigned short *)(skb->data + MACADDRLEN * 2 - 2 - i * 2));
+						skb_pull(skb, 4);
+					}
+				}
+
+				if (skb_is_nonlinear(skb))
+					DEBUG_ERR("%s(): skb_is_nonlinear!!\n", __FUNCTION__);
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18))
+				res = skb_linearize(skb, GFP_ATOMIC);
+#else	/* (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)) */
+				res = skb_linearize(skb);
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)) */
+				if (res < 0) {
+					DEBUG_ERR("TX DROP: skb_linearize fail!\n");
+					/* goto free_and_stop; */
+					return -1;
+				}
+
+				res = nat25_db_handle(padapter, skb, NAT25_INSERT);
+				if (res < 0) {
+					if (res == -2) {
+						/* priv->ext_stats.tx_drops++; */
+						DEBUG_ERR("TX DROP: nat25_db_handle fail!\n");
+						/* goto free_and_stop; */
+						return -1;
+
+					}
+					/* we just print warning message and let it go */
+					/* DEBUG_WARN("%s()-%d: nat25_db_handle INSERT Warning!\n", __FUNCTION__, __LINE__); */
+					/* return -1; */ /* return -1 will cause system crash on 2011/08/30! */
+					return 0;
+				}
+			}
+
+			_rtw_memcpy(skb->data + MACADDRLEN, GET_MY_HWADDR(padapter), MACADDRLEN);
+
+			dhcp_flag_bcast(padapter, skb);
+
+			if (is_vlan_tag) {
+				skb_push(skb, 4);
+				for (i = 0; i < 6; i++)
+					*((unsigned short *)(skb->data + i * 2)) = *((unsigned short *)(skb->data + 4 + i * 2));
+				*((unsigned short *)(skb->data + MACADDRLEN * 2)) = __constant_htons(ETH_P_8021Q);
+				*((unsigned short *)(skb->data + MACADDRLEN * 2 + 2)) = vlan_hdr;
+			}
+		}
+#if 0
+		else {
+			if (*((unsigned short *)(skb->data + MACADDRLEN * 2)) == __constant_htons(ETH_P_8021Q))
+				is_vlan_tag = 1;
+
+			if (is_vlan_tag) {
+				if (ICMPV6_MCAST_MAC(skb->data) && ICMPV6_PROTO1A_VALN(skb->data))
+					_rtw_memcpy(skb->data + MACADDRLEN, GET_MY_HWADDR(padapter), MACADDRLEN);
+			} else {
+				if (ICMPV6_MCAST_MAC(skb->data) && ICMPV6_PROTO1A(skb->data))
+					_rtw_memcpy(skb->data + MACADDRLEN, GET_MY_HWADDR(padapter), MACADDRLEN);
+			}
+		}
+#endif /* 0 */
+
+		/* check if SA is equal to our MAC */
+		if (!_rtw_memcmp(skb->data + MACADDRLEN, GET_MY_HWADDR(padapter), MACADDRLEN)) {
+			/* priv->ext_stats.tx_drops++; */
+			DEBUG_ERR("TX DROP: untransformed frame SA:%02X%02X%02X%02X%02X%02X!\n",
+				skb->data[6], skb->data[7], skb->data[8], skb->data[9], skb->data[10], skb->data[11]);
+			/* goto free_and_stop; */
+			return -1;
+		}
+	}
+	return 0;
+}
+#endif /* CONFIG_BR_EXT */
+
+#if 0
+static void do_queue_select(_adapter	*padapter, struct pkt_attrib *pattrib)
+{
+	u8 qsel;
+
+	qsel = pattrib->priority;
+
+	/* high priority packet */
+	if (pattrib->hipriority_pkt) {
+		pattrib->qsel = rtw_hal_get_qsel(padapter, QSLT_VO_ID);
+		pattrib->priority  = rtw_hal_get_qsel(padapter, QSLT_VO_ID);
+	}
+}
+#endif
+/*
+ * The main transmit(tx) entry
+ *
+ * Return
+ *	1	enqueue
+ *	0	success, hardware will handle this xmit frame(packet)
+ *	<0	fail
+ */
+ #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+s32 rtw_monitor_xmit_entry(struct sk_buff *skb, struct net_device *ndev)
+{
+	u16 frame_ctl;
+	struct ieee80211_radiotap_header rtap_hdr;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct pkt_file pktfile;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	struct pkt_attrib	*pattrib;
+	struct xmit_frame		*pmgntframe;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	unsigned char	*pframe;
+	u8 dummybuf[32];
+	int len = skb->len, rtap_len;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+
+	rtw_mstat_update(MSTAT_TYPE_SKB, MSTAT_ALLOC_SUCCESS, skb->truesize);
+
+#ifndef CONFIG_CUSTOMER_ALIBABA_GENERAL
+	if (unlikely(skb->len < sizeof(struct ieee80211_radiotap_header)))
+		goto fail;
+
+	_rtw_open_pktfile((struct sk_buff *)skb, &pktfile);
+	_rtw_pktfile_read(&pktfile, (u8 *)(&rtap_hdr), sizeof(struct ieee80211_radiotap_header));
+	rtap_len = ieee80211_get_radiotap_len((u8 *)(&rtap_hdr));
+	if (unlikely(rtap_hdr.it_version))
+		goto fail;
+
+	if (unlikely(skb->len < rtap_len))
+		goto fail;
+
+	if (rtap_len != 12) {
+		RTW_INFO("radiotap len (should be 14): %d\n", rtap_len);
+		goto fail;
+	}
+	_rtw_pktfile_read(&pktfile, dummybuf, rtap_len-sizeof(struct ieee80211_radiotap_header));
+	len = len - rtap_len;
+#endif
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		rtw_udelay_os(500);
+		goto fail;
+	}
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+//	_rtw_memcpy(pframe, (void *)checking, len);
+	_rtw_pktfile_read(&pktfile, pframe, len);
+
+
+	/* Check DATA/MGNT frames */
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	frame_ctl = le16_to_cpu(pwlanhdr->frame_ctl);
+	if ((frame_ctl & RTW_IEEE80211_FCTL_FTYPE) == RTW_IEEE80211_FTYPE_DATA) {
+
+		pattrib = &pmgntframe->attrib;
+		update_monitor_frame_attrib(padapter, padapter_link, pattrib);
+
+		if (is_broadcast_mac_addr(pwlanhdr->addr3) || is_broadcast_mac_addr(pwlanhdr->addr1))
+			pattrib->rate = MGN_24M;
+
+	} else {
+
+		pattrib = &pmgntframe->attrib;
+		update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	}
+	pattrib->retry_ctrl = _FALSE;
+	pattrib->pktlen = len;
+	pmlmeext->mgnt_seq = GetSequence(pwlanhdr);
+	pattrib->seqnum = pmlmeext->mgnt_seq;
+	pmlmeext->mgnt_seq++;
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	dump_mgntframe(padapter, pmgntframe);
+
+fail:
+	rtw_skb_free(skb);
+	return 0;
+}
+#endif
+
+/*
+ * The main transmit(tx) entry post handle
+ *
+ * Return
+ *	1	enqueue
+ *	0	success, hardware will handle this xmit frame(packet)
+ *	<0	fail
+ */
+s32 rtw_xmit_posthandle(_adapter *padapter, struct xmit_frame *pxmitframe,
+							struct sk_buff *pkt)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	s32 res;
+
+	res = update_attrib(padapter, pkt, &pxmitframe->attrib);
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (pxmitframe->attrib.ether_type != 0x88B4) {
+		if (rtw_wapi_drop_for_key_absent(padapter, pxmitframe->attrib.ra)) {
+			WAPI_TRACE(WAPI_RX, "drop for key absend when tx\n");
+			res = _FAIL;
+		}
+	}
+#endif
+	if (res == _FAIL) {
+		/*RTW_INFO("%s-"ADPT_FMT" update attrib fail\n", __func__, ADPT_ARG(padapter));*/
+#ifdef DBG_TX_DROP_FRAME
+		RTW_INFO("DBG_TX_DROP_FRAME %s update attrib fail\n", __FUNCTION__);
+#endif
+		rtw_free_xmitframe(pxmitpriv, pxmitframe);
+		return -1;
+	}
+	pxmitframe->pkt = pkt;
+
+	rtw_led_tx_control(padapter, pxmitframe->attrib.dst);
+
+	/*do_queue_select(padapter, &pxmitframe->attrib);*/
+
+#ifdef CONFIG_AP_MODE
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+	if (xmitframe_enqueue_for_sleeping_sta(padapter, pxmitframe) == _TRUE) {
+		_rtw_spinunlock_bh(&pxmitpriv->lock);
+		DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue);
+		return 1;
+	}
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+#endif
+
+	/* pre_xmitframe */
+	/*if (rtw_hal_xmit(padapter, pxmitframe) == _FALSE)*/
+	if (rtw_intf_data_xmit(padapter, pxmitframe) == _FALSE)
+		return 1;
+
+	return 0;
+}
+
+/*
+ * The main transmit(tx) entry
+ *
+ * Return
+ *	1	enqueue
+ *	0	success, hardware will handle this xmit frame(packet)
+ *	<0	fail
+ */
+s32 rtw_xmit(_adapter *padapter, struct sk_buff **ppkt, u16 os_qid)
+{
+	static systime start = 0;
+	static u32 drop_cnt = 0;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pxmitframe = NULL;
+	s32 res;
+
+	DBG_COUNTER(padapter->tx_logs.core_tx);
+
+	if (IS_CH_WAITING(adapter_to_rfctl(padapter)))
+		return -1;
+
+	if (rtw_linked_check(padapter) == _FALSE)
+		return -1;
+
+	if (start == 0)
+		start = rtw_get_current_time();
+
+	pxmitframe = rtw_alloc_xmitframe(pxmitpriv, os_qid);
+
+	if (rtw_get_passing_time_ms(start) > 2000) {
+		if (drop_cnt)
+			RTW_INFO("DBG_TX_DROP_FRAME %s no more pxmitframe, drop_cnt:%u\n", __FUNCTION__, drop_cnt);
+		start = rtw_get_current_time();
+		drop_cnt = 0;
+	}
+
+	if (pxmitframe == NULL) {
+		drop_cnt++;
+		/*RTW_INFO("%s-"ADPT_FMT" no more xmitframe\n", __func__, ADPT_ARG(padapter));*/
+		DBG_COUNTER(padapter->tx_logs.core_tx_err_pxmitframe);
+		return -1;
+	}
+
+#ifdef CONFIG_BR_EXT
+	if (MLME_IS_STA(padapter) || MLME_IS_ADHOC(padapter)) {
+		void *br_port = NULL;
+
+		#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35))
+		br_port = padapter->pnetdev->br_port;
+		#else
+		rcu_read_lock();
+		br_port = rcu_dereference(padapter->pnetdev->rx_handler_data);
+		rcu_read_unlock();
+		#endif
+
+		if (br_port) {
+			res = rtw_br_client_tx(padapter, ppkt);
+			if (res == -1) {
+				rtw_free_xmitframe(pxmitpriv, pxmitframe);
+				DBG_COUNTER(padapter->tx_logs.core_tx_err_brtx);
+				return -1;
+			}
+		}
+	}
+#endif /* CONFIG_BR_EXT */
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		_list f_list;
+
+		res = rtw_mesh_addr_resolve(padapter, pxmitframe, *ppkt, &f_list);
+		if (res == RTW_RA_RESOLVING)
+			return 1;
+		if (res == _FAIL)
+			return -1;
+
+		#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+		if (!rtw_is_list_empty(&f_list)) {
+			_list *list = get_next(&f_list);
+			struct xmit_frame *fframe;
+
+			while ((rtw_end_of_queue_search(&f_list, list)) == _FALSE) {
+				fframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+				list = get_next(list);
+				rtw_list_delete(&fframe->list);
+
+				fframe->pkt = rtw_skb_copy(*ppkt);
+				if (!fframe->pkt) {
+					if (res == RTW_ORI_NO_NEED)
+						res = _SUCCESS;
+					rtw_free_xmitframe(pxmitpriv, fframe);
+					continue;
+				}
+
+				rtw_xmit_posthandle(padapter, fframe, fframe->pkt);
+			}
+		}
+		#endif /* CONFIG_RTW_MESH_DATA_BMC_TO_UC */
+
+		if (res == RTW_ORI_NO_NEED) {
+			rtw_free_xmitframe(&padapter->xmitpriv, pxmitframe);
+			return 0;
+		}
+	}
+#endif /* CONFIG_RTW_MESH */
+
+	pxmitframe->pkt = NULL; /* let rtw_xmit_posthandle not to free pkt inside */
+	res = rtw_xmit_posthandle(padapter, pxmitframe, *ppkt);
+
+	return res;
+}
+
+#ifdef RTW_PHL_TX
+
+#ifdef RTW_PHL_TEST_FPGA
+u32 test_seq;
+#endif
+
+u8 *get_head_from_txreq(_adapter *padapter, struct xmit_frame *pxframe, u8 frag_idx)
+{
+	return 0;
+}
+
+u8 *get_tail_from_txreq(_adapter *padapter, struct xmit_frame *pxframe, u8 frag_idx)
+{
+	return 0;
+}
+
+void dump_pkt(u8 *start, u32 len)
+{
+	u32 idx = 0;
+	for (idx = 0; idx < len; idx++) {
+		printk("%02x ", start[idx]);
+		if ((idx % 20) == 19)
+			printk("\n");
+	}
+	printk("\n");
+}
+
+/* TXREQ_QMGT */
+u8 *get_txreq_buffer(_adapter *padapter, u8 **txreq, u8 **pkt_list, u8 **head, u8 **tail)
+{
+	struct xmit_txreq_buf *ptxreq_buf = NULL;
+	_list *plist, *phead;
+	_queue *pfree_txreq_queue = &padapter->free_txreq_queue;
+#ifdef CONFIG_CORE_TXSC
+	u8 i = 0;
+#endif
+
+	_rtw_spinlock_bh(&pfree_txreq_queue->lock);
+	if (_rtw_queue_empty(pfree_txreq_queue) == _TRUE) {
+		padapter->txreq_full_cnt++;
+	} else {
+		phead = get_list_head(pfree_txreq_queue);
+		plist = get_next(phead);
+		ptxreq_buf = LIST_CONTAINOR(plist, struct xmit_txreq_buf, list);
+		rtw_list_delete(&ptxreq_buf->list);
+
+		padapter->free_txreq_cnt--;
+	}
+	_rtw_spinunlock_bh(&pfree_txreq_queue->lock);
+
+	if (ptxreq_buf) {
+
+		if (txreq)
+			*txreq = ptxreq_buf->txreq;
+
+		if (head)
+			*head = ptxreq_buf->head;
+
+		if (tail)
+			*tail = ptxreq_buf->tail;
+
+		if (pkt_list)
+			*pkt_list = ptxreq_buf->pkt_list;
+
+#ifdef CONFIG_CORE_TXSC
+		for (i = 0; i < MAX_TXSC_SKB_NUM; i++)
+			ptxreq_buf->pkt[i] = NULL;
+		ptxreq_buf->pkt_cnt = 0;
+#endif
+	}
+
+	return (u8 *)ptxreq_buf;
+}
+
+void get_txreq_resources(_adapter *padapter, struct xmit_frame *pxframe,
+	u8 **txreq, u8 **pkt_list, u8 **head, u8 **tail)
+{
+	u32 offset_head = (sizeof(struct rtw_xmit_req) * RTW_MAX_FRAG_NUM);
+	u32 offset_tail = offset_head + (SZ_HEAD_BUF * RTW_MAX_FRAG_NUM);
+	u32 offset_list = offset_tail + (SZ_TAIL_BUF * RTW_MAX_FRAG_NUM);
+	u8 *pbuf = NULL;
+
+	PHLTX_ENTER;
+
+	//rtw_phl_tx todo: error handle, max tx req limit
+	padapter->tx_ring_idx++;
+	padapter->tx_ring_idx = (padapter->tx_ring_idx % MAX_TX_RING_NUM);
+
+	pbuf = padapter->tx_pool_ring[padapter->tx_ring_idx];
+	//memset(pbuf, 0, (SZ_TX_RING*RTW_MAX_FRAG_NUM));
+
+	if (txreq)
+		*txreq = pbuf;
+
+	if (head)
+		*head = pbuf + offset_head;
+
+	if (tail)
+		*tail = pbuf + offset_tail;
+
+	if (pkt_list)
+		*pkt_list = pbuf + offset_list;
+}
+
+void dump_xmitframe_txreq(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct rtw_xmit_req *txreq = pxframe->phl_txreq;
+	u32 idx, idx1 = 0;
+
+	PHLTX_ENTER;
+	printk("total txreq=%d \n", pxframe->txreq_cnt);
+
+	for (idx = 0; idx < pxframe->txreq_cnt; idx++) {
+		struct rtw_pkt_buf_list *pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		printk("txreq[%d] with %d pkts =====\n", idx, txreq->pkt_cnt);
+		for (idx1 = 0; idx1 < txreq->pkt_cnt; idx1++) {
+			printk("pkt[%d] 0x%p len=%d\n", idx1, (void *)pkt_list->vir_addr, pkt_list->length);
+			dump_pkt(pkt_list->vir_addr, pkt_list->length);
+			pkt_list++;
+		}
+		txreq++;
+	}
+	printk("\n");
+}
+
+#ifdef CONFIG_PCI_HCI
+void core_recycle_txreq_phyaddr(_adapter *padapter, struct rtw_xmit_req *txreq)
+{
+	PPCI_DATA pci_data = dvobj_to_pci(padapter->dvobj);
+	struct pci_dev *pdev = pci_data->ppcidev;
+	struct rtw_pkt_buf_list *pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+	u32 idx = 0;
+
+	for (idx = 0; idx < txreq->pkt_cnt; idx++) {
+		dma_addr_t phy_addr = (pkt_list->phy_addr_l);
+
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+		{
+			u64 phy_addr_h = pkt_list->phy_addr_h;
+			phy_addr |= (phy_addr_h << 32);
+		}
+#endif
+		pci_unmap_bus_addr(pdev, &phy_addr, pkt_list->length, DMA_TO_DEVICE);
+		pkt_list++;
+	}
+}
+
+void fill_txreq_phyaddr(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	PPCI_DATA pci_data = dvobj_to_pci(padapter->dvobj);
+	struct pci_dev *pdev = pci_data->ppcidev;
+	struct rtw_xmit_req *txreq = pxframe->phl_txreq;
+	u32 idx, idx1 = 0;
+
+	PHLTX_ENTER;
+
+	for (idx = 0; idx < pxframe->txreq_cnt; idx++) {
+		struct rtw_pkt_buf_list *pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+
+		for (idx1 = 0; idx1 < txreq->pkt_cnt; idx1++) {
+			dma_addr_t phy_addr = 0;
+			pci_get_bus_addr(pdev, pkt_list->vir_addr, &phy_addr, pkt_list->length, DMA_TO_DEVICE);
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+			pkt_list->phy_addr_h =  phy_addr >> 32;
+#else
+			pkt_list->phy_addr_h = 0x0;
+#endif
+			pkt_list->phy_addr_l = phy_addr & 0xFFFFFFFF;
+			pkt_list++;
+		}
+		txreq++;
+	}
+}
+#endif
+
+#ifdef CONFIG_XMIT_ACK_BY_REL_RPT
+static void fill_txreq_ack_txfb(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct rtw_xmit_req *txreq = pxframe->phl_txreq;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	if (pxframe->ack_report) {
+		#ifdef CONFIG_XMIT_ACK_BY_REL_RPT_DBG
+		pxmitpriv->ack_txfb.ctx = (void *)pxframe;
+		#endif
+		txreq->txfb = &pxmitpriv->ack_txfb;
+	}
+	else {
+		txreq->txfb = NULL;
+	}
+}
+#ifdef CONFIG_XMIT_ACK_BY_REL_RPT_DBG
+static void core_ack_txfb_hdl_dbg(_adapter *adapter, struct rtw_txfb_t *txfb)
+{
+	struct xmit_frame *xframe = (struct xmit_frame *)txfb->ctx;
+	struct pkt_attrib *pattrib;
+	unsigned char *pframe;
+
+	if (txfb->txsts == TX_STATUS_TX_DONE)
+		RTW_INFO("%s TX_STATUS_TX_DONE ("ADPT_FMT")\n", __func__, ADPT_ARG(adapter));
+	else if (txfb->txsts == TX_STATUS_TX_FAIL_REACH_RTY_LMT)
+		RTW_ERR("%s TX_STATUS_TX_FAIL_REACH_RTY_LMT ("ADPT_FMT")\n", __func__, ADPT_ARG(adapter));
+	else if (txfb->txsts == TX_STATUS_TX_FAIL_LIFETIME_DROP)
+		RTW_ERR("%s TX_STATUS_TX_FAIL_LIFETIME_DROP ("ADPT_FMT")\n", __func__, ADPT_ARG(adapter));
+	else if (txfb->txsts == TX_STATUS_TX_FAIL_MACID_DROP)
+		RTW_ERR("%s TX_STATUS_TX_FAIL_MACID_DROP ("ADPT_FMT")\n", __func__, ADPT_ARG(adapter));
+	else if (txfb->txsts == TX_STATUS_TX_FAIL_SW_DROP)
+		RTW_ERR("%s TX_STATUS_TX_FAIL_SW_DROP ("ADPT_FMT")\n", __func__, ADPT_ARG(adapter));
+	else if (txfb->txsts == TX_STATUS_TX_FAIL_FORCE_DROP_BY_STUCK)
+		RTW_ERR("%s TX_STATUS_TX_FAIL_FORCE_DROP_BY_STUCK ("ADPT_FMT")\n", __func__, ADPT_ARG(adapter));
+	else
+		RTW_ERR("%s txfb->txsts(%d) unknow ("ADPT_FMT")\n", __func__, txfb->txsts, ADPT_ARG(adapter));
+
+	if (xframe) {
+		pattrib = &xframe->attrib;
+		RTW_INFO("[TX-STS] macid: %d\n", pattrib->mac_id);
+		RTW_INFO("[TX-STS] seqnum: %d\n", pattrib->seqnum);
+
+		pframe = (u8 *)(xframe->buf_addr) + TXDESC_OFFSET;
+
+		if ((get_frame_sub_type(pframe) & WIFI_DATA_NULL) == WIFI_DATA_NULL)
+			RTW_INFO("[TX-STS] WIFI_DATA_NULL\n");
+		if ((get_frame_sub_type(pframe) & WIFI_QOS_DATA_TYPE) == WIFI_QOS_DATA_TYPE)
+			RTW_INFO("[TX-STS] WIFI_QOS_DATA_TYPE\n");
+		else if (get_frame_sub_type(pframe) == WIFI_PROBEREQ)
+			RTW_INFO("[TX-STS] WIFI_PROBEREQ\n");
+		else if (get_frame_sub_type(pframe) == WIFI_PROBERSP)
+			RTW_INFO("[TX-STS] WIFI_PROBERSP\n");
+	}
+}
+#endif
+
+static void core_ack_txfb_hdl(struct rtw_txfb_t *txfb)
+{
+	_adapter *adapter = NULL;
+
+	if (!txfb) {
+		RTW_ERR("%s: txfb is NULL!\n", __func__);
+		return;
+	}
+
+	adapter = (struct _ADAPTER *)txfb->drvpriv;
+#ifdef CONFIG_XMIT_ACK_BY_REL_RPT_DBG
+	core_ack_txfb_hdl_dbg(adapter, txfb);
+#endif
+
+	if (txfb->txsts == TX_STATUS_TX_DONE)
+		rtw_ack_tx_done(&adapter->xmitpriv, RTW_SCTX_DONE_SUCCESS);
+	else
+		rtw_ack_tx_done(&adapter->xmitpriv, RTW_SCTX_DONE_CCX_PKT_FAIL);
+}
+
+void rtw_ack_txfb_init(_adapter *padapter, struct rtw_txfb_t *txfb)
+{
+	txfb->drvpriv = padapter;
+	txfb->txfb_cb = core_ack_txfb_hdl;
+}
+#endif
+
+static void _fill_txreq_list_skb(_adapter *padapter,
+	struct rtw_xmit_req *txreq, struct rtw_pkt_buf_list **pkt_list,
+	struct sk_buff *skb, u32 *req_sz, s32 *req_offset)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+	#define skb_frag_off(f)	((f)->page_offset)
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 2, 0))
+	#define skb_frag_page(f) ((f)->page)
+	#define skb_frag_size(f) ((f)->size)
+#endif
+#define PKT_LIST_APPEND(_addr, _len)	do {				\
+		u32 __len = _len;					\
+		if (__len == 0)						\
+			break;						\
+		list->vir_addr = _addr;					\
+		list->length = __len;					\
+		txreq->pkt_cnt++;					\
+		txreq->total_len += __len;				\
+		list++;							\
+		*pkt_list = list;					\
+	} while (0)
+
+	struct rtw_pkt_buf_list *list = *pkt_list;
+	u8 nr_frags = skb_shinfo(skb)->nr_frags;
+	s32 offset = *req_offset;
+	u32 rem_sz = *req_sz;
+	u32 cur_frag_total, cur_frag_rem;
+	int i;
+
+	/* skb head frag */
+	cur_frag_total = skb_headlen(skb);
+
+	if (cur_frag_total > offset) {
+		cur_frag_rem = rtw_min(cur_frag_total - offset, rem_sz);
+		PKT_LIST_APPEND(skb->data + offset, cur_frag_rem);
+		rem_sz -= cur_frag_rem;
+		offset = 0;
+	} else {
+		offset -= cur_frag_total;
+	}
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		u8 *addr;
+
+		addr = ((void *)page_address(skb_frag_page(frag))) + skb_frag_off(frag);
+		cur_frag_total = skb_frag_size(frag);
+
+		if (offset < cur_frag_total) {
+			cur_frag_rem = cur_frag_total - offset;
+
+			if (rem_sz < cur_frag_rem) {
+				PKT_LIST_APPEND(addr + offset, rem_sz);
+				RTW_WARN("%s:%d, size(rem_sz)=%d cur_frag_rem=%d txreq->total_length = %d\n",
+					 __func__, __LINE__, rem_sz, cur_frag_rem, txreq->total_len);
+				rem_sz = 0;
+				break;
+			} else {
+				PKT_LIST_APPEND(addr + offset, cur_frag_rem);
+				RTW_DBG("%s:%d, size=%d txreq->total_length = %d\n",
+					__func__, __LINE__, cur_frag_rem, txreq->total_len);
+				rem_sz -= cur_frag_rem;
+			}
+
+			offset = 0;
+		} else {
+			offset -= cur_frag_total;
+		}
+	}
+
+	*req_sz = rem_sz;
+	*req_offset = offset;
+
+#undef PKT_LIST_APPEND
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
+	#undef skb_frag_off
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 2, 0))
+	#undef skb_frag_page
+	#undef skb_frag_size
+#endif
+}
+
+static int skb_total_frag_nr(struct sk_buff *head_skb)
+{
+	struct sk_buff *skb;
+	int nr;
+
+	nr = 1 + skb_shinfo(head_skb)->nr_frags;
+
+	skb_walk_frags(head_skb, skb)
+		nr += 1 + skb_shinfo(skb)->nr_frags;
+
+	return nr;
+}
+
+static void fill_txreq_list_skb(_adapter *padapter,
+	struct rtw_xmit_req *txreq, struct rtw_pkt_buf_list **pkt_list,
+	struct sk_buff *head_skb, u32 req_sz, s32 offset)
+{
+	struct sk_buff *skb;
+
+	if (skb_total_frag_nr(head_skb) > NUM_PKT_LIST_PER_TXREQ - 2) {
+		rtw_skb_linearize(head_skb);
+		RTW_WARN_LMT("skb total frag nr over %d\n", NUM_PKT_LIST_PER_TXREQ - 2);
+	}
+
+	_fill_txreq_list_skb(padapter, txreq, pkt_list, head_skb, &req_sz, &offset);
+
+	skb_walk_frags(head_skb, skb)
+		_fill_txreq_list_skb(padapter, txreq, pkt_list, skb, &req_sz, &offset);
+
+	if (req_sz != 0)
+		RTW_WARN("remain req_sz=%d should be zero\n", req_sz);
+}
+
+s32 rtw_core_replace_skb(struct sk_buff **pskb, u32 need_head, u32 need_tail)
+{
+	struct sk_buff *newskb;
+	struct sk_buff *skb = *pskb;
+
+	newskb = rtw_skb_copy(skb);
+
+	if (newskb == NULL)
+		return FAIL;
+
+	rtw_skb_free(skb);
+	*pskb = newskb;
+
+	return SUCCESS;
+}
+
+#ifdef CONFIG_BR_EXT
+s32 core_br_client_tx(_adapter *padapter, struct xmit_frame *pxframe, struct sk_buff **pskb)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	if (!adapter_use_wds(padapter) && check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE | WIFI_ADHOC_STATE) == _TRUE) {
+		void *br_port = NULL;
+
+		#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35))
+		br_port = padapter->pnetdev->br_port;
+		#else
+		rcu_read_lock();
+		br_port = rcu_dereference(padapter->pnetdev->rx_handler_data);
+		rcu_read_unlock();
+		#endif
+
+		if (br_port) {
+			if (rtw_br_client_tx(padapter, pskb) == FAIL) {
+				core_tx_free_xmitframe(padapter, pxframe);
+				DBG_COUNTER(padapter->tx_logs.core_tx_err_brtx);
+				return FAIL;
+			}
+		}
+	}
+	return SUCCESS;
+}
+#endif
+
+s32 core_tx_update_pkt(_adapter *padapter, struct xmit_frame *pxframe, struct sk_buff **pskb)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct sk_buff *skb_orig = *pskb;
+
+	PHLTX_LOG;
+
+//rtw_phl_tx todo, BR EXT
+	if (core_br_client_tx(padapter, pxframe, pskb) == FAIL)
+		return FAIL;
+
+	return SUCCESS;
+}
+
+s32 core_tx_update_xmitframe(_adapter *padapter,
+	struct xmit_frame *pxframe, struct sk_buff **pskb, struct sta_info *psta, u8 type)
+{
+	pxframe->xftype = type;
+	pxframe->pkt = *pskb;
+
+	PHLTX_LOG;
+
+#if 1
+	if (pxframe->xftype == RTW_TX_OS) {
+		if (update_attrib(padapter, *pskb, &pxframe->attrib) != _SUCCESS)
+			return FAIL;
+	}
+#else
+	pxframe->pkt = *pskb;
+
+	if (update_xmitframe_from_hdr(padapter, pxframe) == FAIL)
+		return FAIL;
+
+	PHLTX_LOG;
+
+	if (update_xmitframe_qos(padapter, pxframe) == FAIL)
+		return FAIL;
+
+	PHLTX_LOG;
+
+	if (update_xmitframe_security(padapter, pxframe) == FAIL)
+		return FAIL;
+
+	PHLTX_LOG;
+
+	//if (update_xmitframe_hw(padapter, pxframe) == FAIL)
+		//return FAIL;
+
+	PHLTX_LOG;
+
+	if (pxframe->xftype == RTW_TX_OS) {
+		if (pxframe->attrib.bswenc
+			&& (skb_shared(*pskb) || skb_cloned(*pskb))
+			&& (rtw_core_replace_skb(pskb, RTW_MAX_WL_HEAD, RTW_MAX_WL_TAIL) == FAIL))
+		return FAIL;
+	}
+#endif
+
+	PHLTX_LOG;
+
+	return SUCCESS;
+}
+
+
+
+void get_wl_frag_paras(_adapter *padapter, struct xmit_frame *pxframe,
+	u32 *frag_perfr, u32 *wl_frags)
+{
+	u32 wl_head, wl_tail, payload_totalsz, payload_fragsz, wl_frag_num;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	wl_head = wl_tail = payload_totalsz = 0;
+
+	wl_head += pxframe->attrib.hdrlen;
+	wl_tail += RTW_SZ_FCS;
+	if (pxframe->attrib.encrypt) {
+		wl_head += pxframe->attrib.iv_len;
+		wl_tail += pxframe->attrib.icv_len;
+	}
+
+	payload_fragsz = pxmitpriv->frag_len - wl_head - wl_tail;
+
+	payload_totalsz = pxframe->attrib.pktlen;
+	if (pxframe->xftype == RTW_TX_OS)
+		payload_totalsz += RTW_SZ_LLC;
+	if (pxframe->attrib.encrypt == _TKIP_)
+		payload_totalsz += RTW_TKIP_MIC_LEN;
+
+	if (pxframe->attrib.amsdu)
+		wl_frag_num = 1;
+	else if (payload_fragsz < payload_totalsz)
+		wl_frag_num = RTW_DIV_ROUND_UP(payload_totalsz, payload_fragsz);
+	else
+		wl_frag_num = 1;
+
+	pxframe->attrib.frag_datalen = *frag_perfr = payload_fragsz;
+	pxframe->attrib.nr_frags = *wl_frags = wl_frag_num;
+#ifdef CONFIG_CORE_TXSC
+	pxframe->attrib.frag_len_txsc = payload_fragsz - (payload_totalsz - pxframe->attrib.pktlen);
+#endif
+}
+
+u8 fill_txreq_pkt_perfrag_txos(struct _ADAPTER *padapter,
+			       struct xmit_frame *pxframe,
+			       u32 frag_perfr, u32 wl_frags)
+{
+	struct rtw_xmit_req *xf_txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	struct sk_buff *skb = pxframe->pkt;
+	u8 *txreq, *head, *tail, *list;
+	u32 head_sz, tail_sz, wlan_tail;
+	u32 payload_sz, payload_offset;
+	u8 idx;
+	u8 *wlhdr[RTW_MAX_FRAG_NUM] = {NULL};
+	u8 *wltail[RTW_MAX_FRAG_NUM] = {NULL};
+	/* TXREQ_QMGT */
+	struct xmit_txreq_buf *txreq_buf = NULL;
+
+	PHLTX_ENTER;
+
+	//printk("pxframe->attrib.pkt_hdrlen=%d pxframe->attrib.hdrlen=%d pxframe->attrib.iv_len=%d \n", pxframe->attrib.pkt_hdrlen, pxframe->attrib.hdrlen, pxframe->attrib.iv_len);
+
+	if (wl_frags > RTW_MAX_FRAG_NUM)
+		goto fail;
+	pxframe->txreq_cnt = wl_frags;
+
+	head_sz = pxframe->attrib.hdrlen + (pxframe->attrib.amsdu ? 0 : RTW_SZ_LLC);
+	tail_sz = 0;
+	if (pxframe->attrib.encrypt) {
+		head_sz += pxframe->attrib.iv_len;
+		if (pxframe->attrib.encrypt == _TKIP_)
+			tail_sz += RTW_TKIP_MIC_LEN;
+		if (pxframe->attrib.bswenc)
+			tail_sz += pxframe->attrib.icv_len;
+	}
+
+	PHLTX_LOG;
+
+	//get_txreq_resources(padapter, pxframe, &txreq, &list, &head, &tail);
+	/* TXREQ_QMGT */
+	txreq_buf = (struct xmit_txreq_buf *)get_txreq_buffer(padapter, &txreq, &list, &head, &tail);
+	if (txreq_buf == NULL) {
+		//do this in core_tx_init_xmitframe
+		//pxframe->phl_txreq = NULL;
+		//pxframe->ptxreq_buf = NULL;
+
+		//free in rtw_core_tx
+		//pxframe->pkt = NULL;//for not recycle in abort_core_tx
+		goto fail;
+	}
+#ifdef USE_PREV_WLHDR_BUF /* CONFIG_CORE_TXSC */
+	txreq_buf->macid = 0xff;
+	txreq_buf->txsc_id = 0xff;
+#endif
+	pxframe->ptxreq_buf = txreq_buf;
+
+	PHLTX_LOG;
+
+#if 0
+	payload = skb->data+pxframe->attrib.pkt_hdrlen;
+	printk("num_txreq=%d, hw_head=%d, hw_tail=%d, list=0x%p\n",
+		num_txreq, hw_head, hw_tail, (void *)list);
+
+	printk("p:txreq=0x%p, head=0x%p, tail=0x%p, payload=0x%p\n",
+		(void *)txreq, (void *)head, (void *)tail, (void *)payload);
+#endif
+
+	pxframe->phl_txreq = xf_txreq = (struct rtw_xmit_req *)txreq;
+	pkt_list = (struct rtw_pkt_buf_list *)list;
+#ifdef CONFIG_CORE_TXSC
+	xf_txreq->shortcut_id = 0;
+	xf_txreq->treq_type = RTW_PHL_TREQ_TYPE_NORMAL;
+#endif
+
+	PHLTX_LOG;
+
+	/* move to first payload position */
+	payload_offset = pxframe->attrib.pkt_hdrlen;
+
+	for (idx = 0; idx < wl_frags; idx++) {
+		/* for no memset */
+		xf_txreq->pkt_cnt = 0;
+		xf_txreq->total_len = 0;
+		xf_txreq->pkt_list = (u8 *)pkt_list;
+
+		/* fill head into txreq */
+		wlhdr[idx] = head;
+		pkt_list->vir_addr = head;
+		pkt_list->length = head_sz;
+		if (idx) {
+			/* deduct LLC size if not first fragment */
+			pkt_list->length -= RTW_SZ_LLC;
+		}
+		head += pkt_list->length;
+		xf_txreq->pkt_cnt++;
+		xf_txreq->total_len += pkt_list->length;
+		pkt_list++;
+
+		/* fill payload into txreq */
+		if (idx == (wl_frags - 1)) {
+			/* last payload size */
+			payload_sz = skb->len - payload_offset;
+		} else if (idx == 0) {
+			/* first payload size should deduct LLC size */
+			payload_sz = frag_perfr - RTW_SZ_LLC;
+		} else {
+			payload_sz = frag_perfr;
+		}
+		/* xf_txreq would be update and pkt_list++ inside */
+		fill_txreq_list_skb(padapter, xf_txreq, &pkt_list, skb,
+				    payload_sz, payload_offset);
+		payload_offset += payload_sz;
+
+		/* fill tail(if alloc) into txreq */
+		if (tail_sz) {
+			wlan_tail = tail_sz;
+			if ((pxframe->attrib.encrypt == _TKIP_) && (idx != (wl_frags - 1))) {
+				/* deduct MIC size if not last fragment with TKIP */
+				wlan_tail -= RTW_TKIP_MIC_LEN;
+			}
+			if (wlan_tail) {
+				wltail[idx] = tail;
+				pkt_list->vir_addr = tail;
+				pkt_list->length = wlan_tail;
+				tail += pkt_list->length;
+				xf_txreq->pkt_cnt++;
+				xf_txreq->total_len += pkt_list->length;
+				pkt_list++;
+			}
+		}
+
+		if (xf_txreq->pkt_cnt > NUM_PKT_LIST_PER_TXREQ)
+			RTW_WARN("xf_txreq->pkt_cnt=%d > NUM_PKT_LIST_PER_TXREQ\n",
+				 xf_txreq->pkt_cnt);
+
+		xf_txreq++;
+	}
+
+	_rtw_memcpy(pxframe->wlhdr, wlhdr, sizeof(wlhdr));
+	_rtw_memcpy(pxframe->wltail, wltail, sizeof(wltail));
+	PHLTX_EXIT;
+	return _SUCCESS;
+
+fail:
+	PHLTX_EXIT;
+	return _FAIL;
+}
+
+/* TXREQ_QMGT, MGT_TXREQ_QMGT */
+u8 fill_txreq_pkt_mgmt(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct rtw_xmit_req *xf_txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	//u8 *txreq, *head, *tail, *list, *mgmt = NULL;
+
+	PHLTX_ENTER;
+
+	if (!pxframe->phl_txreq)
+		goto fail;
+
+	xf_txreq = pxframe->phl_txreq;
+	pkt_list = (struct rtw_pkt_buf_list *)xf_txreq->pkt_list;
+
+	//get_txreq_resources(padapter, pxframe,
+	//	(u8 **)&xf_txreq, (u8 **)&pkt_list, NULL, NULL);
+	//printk("p:txreq=0x%p, pkt_list=0x%p \n", (void *)xf_txreq, (void *)pkt_list);
+
+	//for no memset
+	xf_txreq->pkt_cnt = 0;
+	xf_txreq->total_len = 0;
+#ifdef CONFIG_CORE_TXSC
+	xf_txreq->shortcut_id = 0;
+#endif
+
+	pkt_list->vir_addr = pxframe->buf_addr;
+	pkt_list->length = pxframe->attrib.pktlen;
+
+	xf_txreq->pkt_cnt = 1;
+	//xf_txreq->pkt_list = (u8 *)pkt_list;
+	xf_txreq->treq_type = RTW_PHL_TREQ_TYPE_NORMAL;
+
+	pxframe->txreq_cnt = 1;
+	//pxframe->phl_txreq = xf_txreq;
+
+	xf_txreq->total_len = xf_txreq->total_len + pxframe->attrib.pktlen;
+	//RTW_INFO("%s,%d, xf_txreq->total_length = %d\n", __func__, __LINE__, xf_txreq->total_len);
+
+#ifdef RTW_PHL_TEST_FPGA
+{
+	struct rtw_ieee80211_hdr *p = (struct rtw_ieee80211_hdr *)pxframe->buf_addr;
+
+	test_seq++;
+	test_seq = test_seq%0xFFF;
+	SetSeqNum(p, test_seq);
+}
+#endif
+
+exit:
+	return _SUCCESS;
+
+fail:
+	return _FAIL;
+}
+
+static u8 merge_txreq_to_one_piece(struct _ADAPTER *a,
+				   struct xmit_frame *xf)
+{
+	struct rtw_xmit_req *txreq = NULL;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	int i, j;
+	u32 total_sz;
+	u8 *buf, *ptr;
+
+
+	for (i = 0; i < xf->txreq_cnt; i++) {
+		txreq = &xf->phl_txreq[i];
+		total_sz = txreq->total_len;
+		buf = rtw_zmalloc(total_sz);
+		if (!buf)
+			return _FAIL;
+		xf->buf_need_free |= BIT(i);
+
+		ptr = buf;
+		for (j = 0; j < txreq->pkt_cnt; j++) {
+			pkt_list = &((struct rtw_pkt_buf_list *)txreq->pkt_list)[j];
+			_rtw_memcpy(ptr, pkt_list->vir_addr, pkt_list->length);
+			ptr += pkt_list->length;
+		}
+		txreq->pkt_cnt = 1;
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		pkt_list->vir_addr = buf;
+		pkt_list->length = total_sz;
+	}
+
+	return _SUCCESS;
+}
+
+#ifdef RTW_PHL_TEST_FPGA
+#define F_TX_MACID	(0)
+#define F_TX_TID		(1)
+#define F_TX_TYPE	RTW_PHL_PKT_TYPE_DATA
+#define F_TX_RATE	(0x8F) //HRATE_MCS15
+#define F_TX_BW		(1)
+#define F_TX_DMACH	(0)
+#endif
+
+static u8 get_security_cam_id(struct _ADAPTER *padapter, struct xmit_frame *pxframe, u8 keyid)
+{
+	struct dvobj_priv *d;
+	void *phl;
+	u8 sec_cam_id = 0;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *sta;
+	sint bmcast = IS_MCAST(pxframe->attrib.ra);
+	struct link_mlme_priv *pmlmepriv = &(pxframe->attrib.adapter_link->mlmepriv);
+	WLAN_BSSID_EX *pbss_network = (WLAN_BSSID_EX *)&pmlmepriv->cur_network.network;
+
+	if (bmcast == _TRUE) {
+		/* WEP: use unicast key type to match halmac rule (see: setkey_hdl) */
+		if (pxframe->attrib.encrypt == _WEP40_ || pxframe->attrib.encrypt == _WEP104_)
+			bmcast = _FALSE;
+
+		sta = rtw_get_stainfo(pstapriv, pbss_network->MacAddress);
+	} else {
+		sta = rtw_get_stainfo(pstapriv, pxframe->attrib.ra);
+	}
+
+	if (!sta) {
+		RTW_ERR("%s sta not found\n", __func__);
+		rtw_warn_on(1);
+		return sec_cam_id;
+	}
+
+	d = adapter_to_dvobj(padapter);
+	phl = GET_PHL_INFO(d);
+
+	if (keyid >= 4)
+		sec_cam_id = rtw_phl_get_sec_cam_idx(phl, sta->phl_sta, keyid,
+			RTW_SEC_KEY_BIP);
+	else
+		sec_cam_id = rtw_phl_get_sec_cam_idx(phl, sta->phl_sta, keyid,
+			bmcast ? RTW_SEC_KEY_MULTICAST : RTW_SEC_KEY_UNICAST);
+
+	return sec_cam_id;
+}
+
+/* Todo: HE rate mapping not ready */
+static const enum rtw_data_rate mrate2phlrate_tbl[] = {
+	[MGN_1M] = RTW_DATA_RATE_CCK1,
+	[MGN_2M] = RTW_DATA_RATE_CCK2,
+	[MGN_5_5M] = RTW_DATA_RATE_CCK5_5,
+	[MGN_11M] = RTW_DATA_RATE_CCK11,
+	[MGN_6M] = RTW_DATA_RATE_OFDM6,
+	[MGN_9M] = RTW_DATA_RATE_OFDM9,
+	[MGN_12M] = RTW_DATA_RATE_OFDM12,
+	[MGN_18M] = RTW_DATA_RATE_OFDM18,
+	[MGN_24M] = RTW_DATA_RATE_OFDM24,
+	[MGN_36M] = RTW_DATA_RATE_OFDM36,
+	[MGN_48M] = RTW_DATA_RATE_OFDM48,
+	[MGN_54M] = RTW_DATA_RATE_OFDM54,
+	[MGN_MCS0] = RTW_DATA_RATE_MCS0,
+	[MGN_MCS1] = RTW_DATA_RATE_MCS1,
+	[MGN_MCS2] = RTW_DATA_RATE_MCS2,
+	[MGN_MCS3] = RTW_DATA_RATE_MCS3,
+	[MGN_MCS4] = RTW_DATA_RATE_MCS4,
+	[MGN_MCS5] = RTW_DATA_RATE_MCS5,
+	[MGN_MCS6] = RTW_DATA_RATE_MCS6,
+	[MGN_MCS7] = RTW_DATA_RATE_MCS7,
+	[MGN_MCS8] = RTW_DATA_RATE_MCS8,
+	[MGN_MCS9] = RTW_DATA_RATE_MCS9,
+	[MGN_MCS10] = RTW_DATA_RATE_MCS10,
+	[MGN_MCS11] = RTW_DATA_RATE_MCS11,
+	[MGN_MCS12] = RTW_DATA_RATE_MCS12,
+	[MGN_MCS13] = RTW_DATA_RATE_MCS13,
+	[MGN_MCS14] = RTW_DATA_RATE_MCS14,
+	[MGN_MCS15] = RTW_DATA_RATE_MCS15,
+	[MGN_MCS16] = RTW_DATA_RATE_MCS16,
+	[MGN_MCS17] = RTW_DATA_RATE_MCS17,
+	[MGN_MCS18] = RTW_DATA_RATE_MCS18,
+	[MGN_MCS19] = RTW_DATA_RATE_MCS19,
+	[MGN_MCS20] = RTW_DATA_RATE_MCS20,
+	[MGN_MCS21] = RTW_DATA_RATE_MCS21,
+	[MGN_MCS22] = RTW_DATA_RATE_MCS22,
+	[MGN_MCS23] = RTW_DATA_RATE_MCS23,
+	[MGN_MCS24] = RTW_DATA_RATE_MCS24,
+	[MGN_MCS25] = RTW_DATA_RATE_MCS25,
+	[MGN_MCS26] = RTW_DATA_RATE_MCS26,
+	[MGN_MCS27] = RTW_DATA_RATE_MCS27,
+	[MGN_MCS28] = RTW_DATA_RATE_MCS28,
+	[MGN_MCS29] = RTW_DATA_RATE_MCS29,
+	[MGN_MCS30] = RTW_DATA_RATE_MCS30,
+	[MGN_MCS31] = RTW_DATA_RATE_MCS31,
+	[MGN_VHT1SS_MCS0] = RTW_DATA_RATE_VHT_NSS1_MCS0,
+	[MGN_VHT1SS_MCS1] = RTW_DATA_RATE_VHT_NSS1_MCS1,
+	[MGN_VHT1SS_MCS2] = RTW_DATA_RATE_VHT_NSS1_MCS2,
+	[MGN_VHT1SS_MCS3] = RTW_DATA_RATE_VHT_NSS1_MCS3,
+	[MGN_VHT1SS_MCS4] = RTW_DATA_RATE_VHT_NSS1_MCS4,
+	[MGN_VHT1SS_MCS5] = RTW_DATA_RATE_VHT_NSS1_MCS5,
+	[MGN_VHT1SS_MCS6] = RTW_DATA_RATE_VHT_NSS1_MCS6,
+	[MGN_VHT1SS_MCS7] = RTW_DATA_RATE_VHT_NSS1_MCS7,
+	[MGN_VHT1SS_MCS8] = RTW_DATA_RATE_VHT_NSS1_MCS8,
+	[MGN_VHT1SS_MCS9] = RTW_DATA_RATE_VHT_NSS1_MCS9,
+	[MGN_VHT2SS_MCS0] = RTW_DATA_RATE_VHT_NSS2_MCS0,
+	[MGN_VHT2SS_MCS1] = RTW_DATA_RATE_VHT_NSS2_MCS1,
+	[MGN_VHT2SS_MCS2] = RTW_DATA_RATE_VHT_NSS2_MCS2,
+	[MGN_VHT2SS_MCS3] = RTW_DATA_RATE_VHT_NSS2_MCS3,
+	[MGN_VHT2SS_MCS4] = RTW_DATA_RATE_VHT_NSS2_MCS4,
+	[MGN_VHT2SS_MCS5] = RTW_DATA_RATE_VHT_NSS2_MCS5,
+	[MGN_VHT2SS_MCS6] = RTW_DATA_RATE_VHT_NSS2_MCS6,
+	[MGN_VHT2SS_MCS7] = RTW_DATA_RATE_VHT_NSS2_MCS7,
+	[MGN_VHT2SS_MCS8] = RTW_DATA_RATE_VHT_NSS2_MCS8,
+	[MGN_VHT2SS_MCS9] = RTW_DATA_RATE_VHT_NSS2_MCS9,
+	[MGN_VHT3SS_MCS0] = RTW_DATA_RATE_VHT_NSS3_MCS0,
+	[MGN_VHT3SS_MCS1] = RTW_DATA_RATE_VHT_NSS3_MCS1,
+	[MGN_VHT3SS_MCS2] = RTW_DATA_RATE_VHT_NSS3_MCS2,
+	[MGN_VHT3SS_MCS3] = RTW_DATA_RATE_VHT_NSS3_MCS3,
+	[MGN_VHT3SS_MCS4] = RTW_DATA_RATE_VHT_NSS3_MCS4,
+	[MGN_VHT3SS_MCS5] = RTW_DATA_RATE_VHT_NSS3_MCS5,
+	[MGN_VHT3SS_MCS6] = RTW_DATA_RATE_VHT_NSS3_MCS6,
+	[MGN_VHT3SS_MCS7] = RTW_DATA_RATE_VHT_NSS3_MCS7,
+	[MGN_VHT3SS_MCS8] = RTW_DATA_RATE_VHT_NSS3_MCS8,
+	[MGN_VHT3SS_MCS9] = RTW_DATA_RATE_VHT_NSS3_MCS9,
+	[MGN_VHT4SS_MCS0] = RTW_DATA_RATE_VHT_NSS4_MCS0,
+	[MGN_VHT4SS_MCS1] = RTW_DATA_RATE_VHT_NSS4_MCS1,
+	[MGN_VHT4SS_MCS2] = RTW_DATA_RATE_VHT_NSS4_MCS2,
+	[MGN_VHT4SS_MCS3] = RTW_DATA_RATE_VHT_NSS4_MCS3,
+	[MGN_VHT4SS_MCS4] = RTW_DATA_RATE_VHT_NSS4_MCS4,
+	[MGN_VHT4SS_MCS5] = RTW_DATA_RATE_VHT_NSS4_MCS5,
+	[MGN_VHT4SS_MCS6] = RTW_DATA_RATE_VHT_NSS4_MCS6,
+	[MGN_VHT4SS_MCS7] = RTW_DATA_RATE_VHT_NSS4_MCS7,
+	[MGN_VHT4SS_MCS8] = RTW_DATA_RATE_VHT_NSS4_MCS8,
+	[MGN_VHT4SS_MCS9] = RTW_DATA_RATE_VHT_NSS4_MCS9,
+	[MGN_HE1SS_MCS0] = RTW_DATA_RATE_HE_NSS1_MCS0,
+	[MGN_HE1SS_MCS1] = RTW_DATA_RATE_HE_NSS1_MCS1,
+	[MGN_HE1SS_MCS2] = RTW_DATA_RATE_HE_NSS1_MCS2,
+	[MGN_HE1SS_MCS3] = RTW_DATA_RATE_HE_NSS1_MCS3,
+	[MGN_HE1SS_MCS4] = RTW_DATA_RATE_HE_NSS1_MCS4,
+	[MGN_HE1SS_MCS5] = RTW_DATA_RATE_HE_NSS1_MCS5,
+	[MGN_HE1SS_MCS6] = RTW_DATA_RATE_HE_NSS1_MCS6,
+	[MGN_HE1SS_MCS7] = RTW_DATA_RATE_HE_NSS1_MCS7,
+	[MGN_HE1SS_MCS8] = RTW_DATA_RATE_HE_NSS1_MCS8,
+	[MGN_HE1SS_MCS9] = RTW_DATA_RATE_HE_NSS1_MCS9,
+	[MGN_HE1SS_MCS10] = RTW_DATA_RATE_HE_NSS1_MCS10,
+	[MGN_HE1SS_MCS11] = RTW_DATA_RATE_HE_NSS1_MCS11,
+	[MGN_HE2SS_MCS0] = RTW_DATA_RATE_HE_NSS2_MCS0,
+	[MGN_HE2SS_MCS1] = RTW_DATA_RATE_HE_NSS2_MCS1,
+	[MGN_HE2SS_MCS2] = RTW_DATA_RATE_HE_NSS2_MCS2,
+	[MGN_HE2SS_MCS3] = RTW_DATA_RATE_HE_NSS2_MCS3,
+	[MGN_HE2SS_MCS4] = RTW_DATA_RATE_HE_NSS2_MCS4,
+	[MGN_HE2SS_MCS5] = RTW_DATA_RATE_HE_NSS2_MCS5,
+	[MGN_HE2SS_MCS6] = RTW_DATA_RATE_HE_NSS2_MCS6,
+	[MGN_HE2SS_MCS7] = RTW_DATA_RATE_HE_NSS2_MCS7,
+	[MGN_HE2SS_MCS8] = RTW_DATA_RATE_HE_NSS2_MCS8,
+	[MGN_HE2SS_MCS9] = RTW_DATA_RATE_HE_NSS2_MCS9,
+	[MGN_HE2SS_MCS10] = RTW_DATA_RATE_HE_NSS2_MCS10,
+	[MGN_HE2SS_MCS11] = RTW_DATA_RATE_HE_NSS2_MCS11,
+	[MGN_HE3SS_MCS0] = RTW_DATA_RATE_HE_NSS3_MCS0,
+	[MGN_HE3SS_MCS1] = RTW_DATA_RATE_HE_NSS3_MCS1,
+	[MGN_HE3SS_MCS2] = RTW_DATA_RATE_HE_NSS3_MCS2,
+	[MGN_HE3SS_MCS3] = RTW_DATA_RATE_HE_NSS3_MCS3,
+	[MGN_HE3SS_MCS4] = RTW_DATA_RATE_HE_NSS3_MCS4,
+	[MGN_HE3SS_MCS5] = RTW_DATA_RATE_HE_NSS3_MCS5,
+	[MGN_HE3SS_MCS6] = RTW_DATA_RATE_HE_NSS3_MCS6,
+	[MGN_HE3SS_MCS7] = RTW_DATA_RATE_HE_NSS3_MCS7,
+	[MGN_HE3SS_MCS8] = RTW_DATA_RATE_HE_NSS3_MCS8,
+	[MGN_HE3SS_MCS9] = RTW_DATA_RATE_HE_NSS3_MCS9,
+	[MGN_HE3SS_MCS10] = RTW_DATA_RATE_HE_NSS3_MCS10,
+	[MGN_HE3SS_MCS11] = RTW_DATA_RATE_HE_NSS3_MCS11,
+	[MGN_HE4SS_MCS0] = RTW_DATA_RATE_HE_NSS4_MCS0,
+	[MGN_HE4SS_MCS1] = RTW_DATA_RATE_HE_NSS4_MCS1,
+	[MGN_HE4SS_MCS2] = RTW_DATA_RATE_HE_NSS4_MCS2,
+	[MGN_HE4SS_MCS3] = RTW_DATA_RATE_HE_NSS4_MCS3,
+	[MGN_HE4SS_MCS4] = RTW_DATA_RATE_HE_NSS4_MCS4,
+	[MGN_HE4SS_MCS5] = RTW_DATA_RATE_HE_NSS4_MCS5,
+	[MGN_HE4SS_MCS6] = RTW_DATA_RATE_HE_NSS4_MCS6,
+	[MGN_HE4SS_MCS7] = RTW_DATA_RATE_HE_NSS4_MCS7,
+	[MGN_HE4SS_MCS8] = RTW_DATA_RATE_HE_NSS4_MCS8,
+	[MGN_HE4SS_MCS9] = RTW_DATA_RATE_HE_NSS4_MCS9,
+	[MGN_HE4SS_MCS10] = RTW_DATA_RATE_HE_NSS4_MCS10,
+	[MGN_HE4SS_MCS11] = RTW_DATA_RATE_HE_NSS4_MCS11,
+};
+
+/*
+ * _rate_mrate2phl() - convert data rate from mrate to PHL(MAC)
+ * @sta:	struct sta_info *
+ * @mrate:	date rate of mrate type, enum MGN_RATE
+ *
+ * Convert data rate from MGN_RATE definition to PHL's definition.
+ *
+ * Return PHL's data rate definition "enum rtw_data_rate".
+ * 0x0~0xB: CCK 1M ~ OFDM 54M
+ * 0x80~0x9F: HT MCS0~MCS31
+ * 0x100~0x109: VHT 1SS MCS0~MCS9
+ * 0x110~0x119: VHT 2SS MCS0~MCS9
+ * 0x120~0x129: VHT 3SS MCS0~MCS9
+ * 0x130~0x139: VHT 4SS MCS0~MCS9
+ * 0x180~0x18B: HE 1SS MCS0~MCS11
+ * 0x190~0x19B: HE 2SS MCS0~MCS11
+ * 0x1A0~0x1AB: HE 3SS MCS0~MCS11
+ * 0x1B0~0x1BB: HE 4SS MCS0~MCS11
+ */
+enum rtw_data_rate _rate_mrate2phl(enum MGN_RATE mrate)
+{
+	enum rtw_data_rate phl = RTW_DATA_RATE_CCK1;
+
+
+	if (mrate < ARRAY_SIZE(mrate2phlrate_tbl))
+		phl = mrate2phlrate_tbl[mrate];
+
+	if ((mrate != MGN_1M) && (phl == RTW_DATA_RATE_CCK1))
+		RTW_WARN("%s: Invalid rate 0x%x\n", __func__, mrate);
+
+	return phl;
+}
+
+/*
+ * _rate_drv2phl() - convert data rate from drive to PHL(MAC)
+ * @sta:	struct sta_info *
+ * @rate:	date rate of driver
+ *		0x0~0xB: CCK 1M ~ OFDM 54M
+ *		>0xB: HT/VHT/HE use the same bits field to represent each
+ *		      data rate, so these bits's real definition depended on
+ *		      sta's wireless mode.
+ *
+ * Convert driver's data rate definition to PHL's definition.
+ *
+ * Return PHL's data rate definition "enum rtw_data_rate".
+ */
+static enum rtw_data_rate _rate_drv2phl(struct sta_info *sta, u8 rate)
+{
+	enum rtw_data_rate phl = RTW_DATA_RATE_CCK1;
+	u8 ht_support = 0, vht_support = 0, he_support = 0;
+
+
+	if (rate < 12) {
+		/* B/G mode, CCK/OFDM rate */
+		return (enum rtw_data_rate)rate;
+	}
+
+#ifdef CONFIG_80211N_HT
+	if (sta->htpriv.ht_option == _TRUE)
+		ht_support = 1;
+#ifdef CONFIG_80211AC_VHT
+	if (sta->vhtpriv.vht_option == _TRUE)
+		vht_support = 1;
+#ifdef CONFIG_80211AX_HE
+	if (sta->hepriv.he_option == _TRUE)
+		he_support = 1;
+#endif /* CONFIG_80211AX_HE */
+#endif /* CONFIG_80211AC_VHT */
+#endif /* CONFIG_80211N_HT */
+
+	rate -= 12;
+	if (he_support) {
+		if (rate < 12)
+			phl = RTW_DATA_RATE_HE_NSS1_MCS0 + rate;
+		else if (rate < 24)
+			phl = RTW_DATA_RATE_HE_NSS2_MCS0 + (rate - 12);
+		else if (rate < 36)
+			phl = RTW_DATA_RATE_HE_NSS3_MCS0 + (rate - 24);
+		else
+			phl = RTW_DATA_RATE_HE_NSS4_MCS0 + (rate - 36);
+	} else if (vht_support) {
+		if (rate < 10)
+			phl = RTW_DATA_RATE_VHT_NSS1_MCS0 + rate;
+		else if (rate < 20)
+			phl = RTW_DATA_RATE_VHT_NSS2_MCS0 + (rate - 10);
+		else if (rate < 30)
+			phl = RTW_DATA_RATE_VHT_NSS3_MCS0 + (rate - 20);
+		else
+			phl = RTW_DATA_RATE_VHT_NSS4_MCS0 + (rate - 30);
+	} else if (ht_support) {
+		phl = RTW_DATA_RATE_MCS0 + rate;
+	}
+
+	return phl;
+}
+
+void dbg_dump_txreq_mdata(struct rtw_t_meta_data *mdata, const char *func)
+{
+#ifdef DBG_DUMP_TX_COUNTER
+	if (1) {
+		RTW_PRINT("[%s]\n", func);
+
+		RTW_PRINT("da: %02x%02x%02x%02x%02x%02x\n",
+			mdata->da[0], mdata->da[1], mdata->da[2], mdata->da[3], mdata->da[4], mdata->da[5]);
+		RTW_PRINT("sa: %02x%02x%02x%02x%02x%02x\n",
+			mdata->sa[0], mdata->sa[1], mdata->sa[2], mdata->sa[3], mdata->sa[4], mdata->sa[5]);
+		RTW_PRINT("to_ds: %d\n", mdata->to_ds);
+		RTW_PRINT("from_ds: %d\n", mdata->from_ds);
+		RTW_PRINT("band: %d\n", mdata->band);
+		RTW_PRINT("wmm: %d\n", mdata->wmm);
+		RTW_PRINT("type: %d\n", mdata->type);
+		RTW_PRINT("tid: %d\n", mdata->tid);
+		RTW_PRINT("pktlen: %d\n", mdata->pktlen);
+		RTW_PRINT("cat: %d\n", mdata->cat);
+		/* body section start */
+		RTW_PRINT("macid: %d\n", mdata->macid);
+		/* sequence */
+		RTW_PRINT("hw_seq_mode: %d\n", mdata->hw_seq_mode);
+		RTW_PRINT("hw_ssn_sel: %d\n", mdata->hw_ssn_sel);
+		RTW_PRINT("sw_seq: %d\n", mdata->sw_seq);
+
+		/* hdr conversion & hw amsdu & checksum offload*/
+		RTW_PRINT("hw_hdr_conv: %d\n", mdata->hw_hdr_conv);
+		RTW_PRINT("hw_amsdu: %d\n", mdata->hw_amsdu);
+		RTW_PRINT("chk_en: %d\n", mdata->chk_en);
+
+		RTW_PRINT("msdu_type: %d\n", mdata->msdu_type);
+		RTW_PRINT("mac_hdr_len: %d\n", mdata->mac_hdr_len);
+		RTW_PRINT("a4_hdr: %d\n", mdata->a4_hdr);
+		RTW_PRINT("with_llc: %d\n", mdata->with_llc);
+		RTW_PRINT("with_vlantag: %d\n", mdata->with_vlantag);
+		RTW_PRINT("sec_hdr_len: %d\n", mdata->sec_hdr_len);
+
+		RTW_PRINT("wp_offset: %d\n", mdata->wp_offset);
+
+		/*tx shortcut*/
+		RTW_PRINT("shcut_camid: %d\n", mdata->shcut_camid);
+
+		RTW_PRINT("msdu_num: %d\n", mdata->msdu_num);
+		RTW_PRINT("reuse_start_num: %d\n", mdata->reuse_start_num);
+		RTW_PRINT("reuse_size: %d\n", mdata->reuse_size);
+		RTW_PRINT("hci_seqnum_mode: %d\n", mdata->hci_seqnum_mode);
+
+		/* dma */
+		RTW_PRINT("dma_ch: %d\n", mdata->dma_ch);
+		RTW_PRINT("wd_page_size: %d\n", mdata->wd_page_size);
+		RTW_PRINT("wdinfo_en: %d\n", mdata->wdinfo_en);
+		RTW_PRINT("addr_info_num: %d\n", mdata->addr_info_num);
+		RTW_PRINT("usb_pkt_ofst: %d\n", mdata->usb_pkt_ofst);
+		RTW_PRINT("usb_txagg_num: %d\n", mdata->usb_txagg_num);
+
+		/* ampdu */
+		RTW_PRINT("ampdu_en: %d\n", mdata->ampdu_en);
+		RTW_PRINT("bk: %d\n", mdata->bk);
+
+		/* sec */
+		RTW_PRINT("hw_sec_iv: %d\n", mdata->hw_sec_iv);
+		RTW_PRINT("sw_sec_iv: %d\n", mdata->sw_sec_iv);
+		RTW_PRINT("sec_keyid: %d\n", mdata->sec_keyid);
+		RTW_PRINT_DUMP("iv", mdata->iv, 6);
+
+		/* mlo */
+		RTW_PRINT("is_mld_sw_en: %d\n", mdata->is_mld_sw_en);
+		RTW_PRINT("is_mld: %d\n", mdata->is_mld);
+
+		/* misc */
+		RTW_PRINT("no_ack: %d\n", mdata->no_ack);
+		RTW_PRINT("eosp_bit: %d\n", mdata->eosp_bit);
+		RTW_PRINT("more_data: %d\n", mdata->more_data);
+
+		/* body or info section start */
+		/* rate */
+		RTW_PRINT("data_bw_er: %d\n", mdata->data_bw_er);
+		RTW_PRINT("f_dcm: %d\n", mdata->f_dcm);
+		RTW_PRINT("f_er: %d\n", mdata->f_er);
+		RTW_PRINT("f_rate: %d\n", mdata->f_rate);
+		RTW_PRINT("f_gi_ltf: %d\n", mdata->f_gi_ltf);
+		RTW_PRINT("f_bw: %d\n", mdata->f_bw);
+		RTW_PRINT("userate_sel: %d\n", mdata->userate_sel);
+
+		/* a ctrl */
+		RTW_PRINT("a_ctrl_uph: %d\n", mdata->a_ctrl_uph);
+
+		/* sec */
+		RTW_PRINT("sec_hw_enc: %d\n", mdata->sec_hw_enc);
+		RTW_PRINT("sec_type: %d\n", mdata->sec_type);
+		RTW_PRINT("force_key_en: %d\n", mdata->force_key_en);
+
+		/* misc */
+		RTW_PRINT("bc: %d\n", mdata->bc);
+		RTW_PRINT("mc: %d\n", mdata->mc);
+		/* body or info section end */
+
+		/* info section start */
+		/* rate */
+		RTW_PRINT("f_ldpc: %d\n", mdata->f_ldpc);
+		RTW_PRINT("f_stbc: %d\n", mdata->f_stbc);
+
+		/* tx cnt & rty rate */
+		RTW_PRINT("dis_rts_rate_fb: %d\n", mdata->dis_rts_rate_fb);
+		RTW_PRINT("dis_data_rate_fb: %d\n", mdata->dis_data_rate_fb);
+		RTW_PRINT("data_rty_lowest_rate: %d\n", mdata->data_rty_lowest_rate);
+		RTW_PRINT("data_tx_cnt_lmt: %d\n", mdata->data_tx_cnt_lmt);
+		RTW_PRINT("data_tx_cnt_lmt_en: %d\n", mdata->data_tx_cnt_lmt_en);
+
+		/* ampdu */
+		RTW_PRINT("max_agg_num: %d\n", mdata->max_agg_num);
+		RTW_PRINT("ampdu_density: %d\n", mdata->ampdu_density);
+
+		/* a ctrl */
+		RTW_PRINT("a_ctrl_bqr: %d\n", mdata->a_ctrl_bqr);
+		RTW_PRINT("a_ctrl_bsr: %d\n", mdata->a_ctrl_bsr);
+		RTW_PRINT("a_ctrl_cas: %d\n", mdata->a_ctrl_cas);
+
+		/* sec */
+		RTW_PRINT("sec_cam_idx: %d\n", mdata->sec_cam_idx);
+
+		/* protection */
+		RTW_PRINT("rts_en: %d\n", mdata->rts_en);
+		RTW_PRINT("cts2self: %d\n", mdata->cts2self);
+		RTW_PRINT("rts_cca_mode: %d\n", mdata->rts_cca_mode);
+		RTW_PRINT("hw_rts_en: %d\n", mdata->hw_rts_en);
+
+		/* misc */
+		RTW_PRINT("mbssid: %d\n", mdata->mbssid);
+		RTW_PRINT("hal_port: %d\n", mdata->hal_port);
+		RTW_PRINT("nav_use_hdr: %d\n", mdata->nav_use_hdr);
+		RTW_PRINT("ack_ch_info: %d\n", mdata->ack_ch_info);
+		RTW_PRINT("life_time_sel: %d\n", mdata->life_time_sel);
+		RTW_PRINT("ndpa: %d\n", mdata->ndpa);
+
+		RTW_PRINT("f_bypass_punc: %d\n", mdata->f_bypass_punc);
+		RTW_PRINT("puncture_pattern_en: %d\n", mdata->puncture_pattern_en);
+		RTW_PRINT("puncture_pattern: %d\n", mdata->puncture_pattern);
+		RTW_PRINT("signaling_ta_pkt_en: %d\n", mdata->signaling_ta_pkt_en);
+		RTW_PRINT("snd_pkt_sel: %d\n", mdata->snd_pkt_sel);
+
+		RTW_PRINT("sifs_tx: %d\n", mdata->sifs_tx);
+		RTW_PRINT("rtt_en: %d\n", mdata->rtt_en);
+		RTW_PRINT("spe_rpt: %d\n", mdata->spe_rpt);
+		RTW_PRINT("raw: %d\n", mdata->raw);
+		RTW_PRINT("sw_define: %d\n", mdata->sw_define);
+		RTW_PRINT("sw_tx_ok: %d\n", mdata->sw_tx_ok);
+		RTW_PRINT("sr_en: %d\n", mdata->sr_en);
+		RTW_PRINT("sr_rate: %d\n", mdata->sr_rate);
+		/* info section end */
+	}
+#endif /*DBG_DUMP_TX_COUNTER*/
+}
+
+u8 fill_txreq_mdata(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct rtw_xmit_req *txreq = pxframe->phl_txreq;
+	struct sta_info *psta = pxframe->attrib.psta;
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	struct rtw_t_meta_data *mdata = &(txreq->mdata);
+#ifdef BMC_ON_HIQ
+       struct sta_priv *pstapriv = &padapter->stapriv;
+#endif
+	u32 idx = 0;
+	u8 htc_option = _FALSE;
+#ifdef CONFIG_XMIT_ACK_BY_CCX_RPT
+	struct xmit_priv *pxmitpriv = &(GET_PRIMARY_ADAPTER(padapter))->xmitpriv;
+#endif
+	struct _ADAPTER_LINK *padapter_link = pxframe->attrib.adapter_link;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+
+	PHLTX_LOG;
+	if (pxframe->attrib.order)
+		htc_option = _TRUE;
+
+	/* packet identify */
+	if (pxframe->attrib.type == WIFI_MGT_TYPE)
+		mdata->type = RTW_PHL_PKT_TYPE_MGNT;
+	else
+		mdata->type = RTW_PHL_PKT_TYPE_DATA;
+
+	mdata->macid = pxframe->attrib.mac_id;
+
+	/* enable wd info by default */
+	mdata->wdinfo_en = 1;
+
+	mdata->msdu_type = RTW_PHL_MSDU_TYPE_80211;
+	mdata->mac_hdr_len = pxframe->attrib.hdrlen;
+	mdata->a4_hdr = pxframe->attrib.a4_hdr;
+
+	if (mdata->type == RTW_PHL_PKT_TYPE_DATA) {
+		mdata->with_llc = 1;
+		/*mdata->with_vlantag = 0;*/
+		mdata->sec_hdr_len = pxframe->attrib.iv_len;
+	}
+
+	#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	mdata->chk_en = 1;
+	#endif
+
+	mdata->hw_seq_mode = 0;
+	mdata->sw_seq = pxframe->attrib.seqnum;
+	mdata->nav_use_hdr = 0;
+	mdata->hw_sec_iv = padapter->dvobj->phl_com->dev_cap.sec_cap.hw_sec_iv;
+	mdata->sw_sec_iv = 0;
+
+	/* packet security */
+	if (pxframe->attrib.encrypt == _NO_PRIVACY_ || pxframe->attrib.bswenc == _TRUE) {
+		mdata->sec_hw_enc = _FALSE;
+		mdata->sec_type = RTW_ENC_NONE;
+	} else {
+		mdata->sec_hw_enc = _TRUE;
+		mdata->sec_type = rtw_sec_algo_drv2phl(pxframe->attrib.encrypt);
+		if (padapter->dvobj->phl_com->dev_cap.sec_cap.hw_form_hdr &&
+		    mdata->sec_type != RTW_ENC_WAPI && mdata->hw_sec_iv == 0) {
+			mdata->sec_keyid = pxframe->attrib.key_idx;
+			if (!psta)
+				return _FAIL;
+			/* Fill PN of IV */
+			for (idx = 0; idx < 6; idx++)
+				mdata->iv[idx] = (psta->dot11txpn.val >> 8 * idx) & 0xff;
+			RTW_DBG("%s: keyid=%d, iv=" PN_FMT "\n", __func__, mdata->sec_keyid, PN_ARG(mdata->iv));
+			update_attrib_sec_iv_info(padapter, &pxframe->attrib);
+		}
+
+		if (padapter->dvobj->phl_com->dev_cap.sec_cap.hw_tx_search_key) {
+			mdata->force_key_en = false;
+		} else {
+			mdata->force_key_en = true;
+			mdata->sec_cam_idx = get_security_cam_id(padapter, pxframe, pxframe->attrib.key_idx);
+		}
+	}
+	/* Currently dump secrity settings for dbg */
+	RTW_DBG("sec_type= %d sec_hw_enc= %d sec_cam_idx= %d \n",
+		mdata->sec_type, mdata->sec_hw_enc, mdata->sec_cam_idx);
+
+	/* packet capability */
+	if (pxframe->attrib.ampdu_en == _TRUE) {
+		mdata->ampdu_en = 1;
+		mdata->bk = 0;
+		mdata->ampdu_density = pxframe->attrib.ampdu_spacing;
+		if (!psta->phl_sta)
+			return _FAIL;
+		mdata->max_agg_num = psta->phl_sta->asoc_cap.num_ampdu;
+	} else {
+		mdata->ampdu_en = 0;
+		mdata->bk = 1;
+	}
+	mdata->dis_data_rate_fb = 0;
+	mdata->dis_rts_rate_fb = 0;
+	mdata->data_tx_cnt_lmt_en = 0;
+	mdata->data_tx_cnt_lmt = 0;
+
+	/* Set DATA_RTY_LOWEST_RATE: 2.4G to CCK1M & 5G to OFDM6M */
+	if (rtw_get_oper_band(padapter, padapter_link) == BAND_ON_24G)
+		mdata->data_rty_lowest_rate = RTW_DATA_RATE_CCK1;
+	else if ((rtw_get_oper_band(padapter, padapter_link) == BAND_ON_5G) ||
+		 (rtw_get_oper_band(padapter, padapter_link) == BAND_ON_6G))
+		mdata->data_rty_lowest_rate = RTW_DATA_RATE_OFDM6;
+	else
+		RTW_WARN("%s: mdata->data_rty_lowest_rate is not set.\n", __func__);
+
+#ifdef CONFIG_TX_AMSDU
+	if (pxframe->attrib.tx_amsdu_en)
+		mdata->data_rty_lowest_rate = RTW_DATA_RATE_MCS0;
+#endif
+
+	mdata->life_time_sel = 0;
+	mdata->rts_en = pxframe->attrib.rtsen;
+	mdata->cts2self = pxframe->attrib.cts2self;
+	mdata->hw_rts_en = pxframe->attrib.hw_rts_en;
+	mdata->rts_cca_mode = 0;
+
+	mdata->f_bw = pxframe->attrib.bwmode;
+	/* Todo: GI and LTF not ready for HE */
+	mdata->f_gi_ltf = pxframe->attrib.sgi;
+
+	mdata->mc = IS_MCAST(pxframe->attrib.ra) ? 1 : 0;
+	mdata->bc = MacAddr_isBcst(pxframe->attrib.ra) ? 1 : 0;
+
+#ifdef CONFIG_80211AX_HE
+	if (psta && htc_option)
+		mdata->a_ctrl_bsr = 1;
+#endif
+	mdata->raw = 0;
+
+	if (pxframe->attrib.type == WIFI_MGT_TYPE) {
+		mdata->cat = RTW_PHL_RING_CAT_MGNT;
+		mdata->tid = rtw_phl_cvt_cat_to_tid(mdata->cat);
+
+	} else if (pxframe->attrib.type == WIFI_DATA_TYPE) {
+		#ifdef BMC_ON_HIQ
+		if ((mdata->mc || mdata->bc) &&
+		    (rtw_tim_map_anyone_be_set(padapter, pstapriv->sta_dz_bitmap))) {
+			mdata->cat = RTW_PHL_RING_CAT_HIQ; /* HIQ */
+			mdata->tid = rtw_phl_cvt_cat_to_tid(mdata->cat);
+			mdata->mbssid = 0; /* ToDo: Consider MBSSID */
+		} else
+		#endif
+		{
+			/*pxframe->attrib.subtype = WIFI_QOS_DATA_TYPE*/
+			mdata->tid = pxframe->attrib.priority;
+			mdata->cat = rtw_phl_cvt_tid_to_cat(pxframe->attrib.priority);
+		}
+	} else { /*WIFI_CTRL_TYPE*/
+		RTW_ERR("%s invalid attrib.type :%d", __func__, pxframe->attrib.type);
+		rtw_warn_on(1);
+	}
+
+
+#ifdef CONFIG_CORE_TXSC
+	mdata->ampdu_density = 0;
+	mdata->userate_sel = 0;
+#endif
+
+	if (pxframe->xftype == RTW_TX_DRV_MGMT) {
+		mdata->userate_sel = 1;
+		mdata->f_rate = _rate_mrate2phl(pxframe->attrib.rate);
+	} else {
+		/* low rate for EAPOL/ARP/DHCP */
+		if ((pxframe->attrib.ether_type == 0x888e) ||
+			(pxframe->attrib.ether_type == 0x0806) ||
+			(pxframe->attrib.dhcp_pkt == 1)) {
+
+			mdata->userate_sel = 1;
+			if (IS_CCK_RATE(padapter_link->mlmeextpriv.tx_rate))
+				mdata->f_rate = RTW_DATA_RATE_CCK1;
+			else
+				mdata->f_rate = RTW_DATA_RATE_OFDM6;
+			mdata->f_bw = CHANNEL_WIDTH_20;
+		} else {
+			/* fix rate for non specail packet */
+			if (padapter->fix_rate != NO_FIX_RATE) {
+				mdata->userate_sel = 1;
+				mdata->f_rate = GET_FIX_RATE(padapter->fix_rate);
+				mdata->f_gi_ltf = GET_FIX_RATE_SGI(padapter->fix_rate);
+				if (!padapter->data_fb)
+					mdata->dis_data_rate_fb = 1;
+			} else {
+				mdata->userate_sel = 0;
+			}
+
+			if (padapter->fix_bw != NO_FIX_BW)
+				mdata->f_bw = padapter->fix_bw;
+		}
+	}
+	mdata->f_er = 0;
+	mdata->f_dcm = 0;
+	mdata->f_stbc = pxframe->attrib.stbc;
+	mdata->f_ldpc = pxframe->attrib.ldpc;
+
+	mdata->band = 0;
+	mdata->dma_ch = 0;
+	mdata->spe_rpt = 0;
+	mdata->sw_define = 0;
+
+#ifdef CONFIG_XMIT_ACK_BY_CCX_RPT
+	if (pxframe->ack_report) {
+		mdata->spe_rpt = 1;
+		mdata->sw_define = pxmitpriv->ack_tx_seq_no;
+#ifdef RTW_WKARD_CCX_RPT_LIMIT_CTRL
+		mdata->data_tx_cnt_lmt_en = 1;
+		mdata->data_tx_cnt_lmt = 8;
+#endif
+	}
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+	mdata->pktlen = txreq->total_len;
+#endif
+
+#ifdef RTW_PHL_TEST_FPGA
+	mdata->type = F_TX_TYPE;
+	mdata->macid = F_TX_MACID;
+	mdata->tid = F_TX_TID;
+	mdata->dma_ch = F_TX_DMACH;
+	//mdata->band = cap->band;
+	mdata->f_rate = F_TX_RATE;
+	mdata->f_bw = F_TX_BW;
+	mdata->f_gi_ltf = 0;
+	mdata->f_stbc = 0;
+	mdata->f_ldpc = 0;
+
+	mdata->userate_sel = 1;
+	mdata->dis_data_rate_fb = 1;
+	mdata->dis_rts_rate_fb = 1;
+#endif
+
+#ifdef RTW_PHL_DBG_CMD
+	if (pxframe->xftype != RTW_TX_DRV_MGMT) {
+		if (padapter->txForce_enable) {
+			if (padapter->txForce_rate != INV_TXFORCE_VAL)
+				mdata->f_rate = padapter->txForce_rate;
+			if (padapter->txForce_agg != INV_TXFORCE_VAL)
+				mdata->ampdu_en = padapter->txForce_agg;
+			if (padapter->txForce_aggnum != INV_TXFORCE_VAL)
+				mdata->max_agg_num = padapter->txForce_aggnum;
+			if (padapter->txForce_gi != INV_TXFORCE_VAL)
+				mdata->f_gi_ltf = padapter->txForce_gi;
+		}
+	}
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+	dbg_dump_txreq_mdata(mdata, __func__);
+#endif
+
+	if (pxframe->txreq_cnt > 1) {
+		struct rtw_t_meta_data *mdata_tmp;
+		txreq++;
+		for (idx = 1; idx < pxframe->txreq_cnt; idx++) {
+#ifdef CONFIG_CORE_TXSC
+			mdata->pktlen = txreq->total_len;
+#endif
+			mdata_tmp = &(txreq->mdata);
+			memcpy(mdata_tmp, mdata, sizeof(struct rtw_t_meta_data));
+			txreq++;
+		}
+	}
+	return _SUCCESS;
+
+}
+
+
+void fill_txreq_others(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct rtw_xmit_req *txreq = pxframe->phl_txreq;
+	u32 idx = 0;
+
+	PHLTX_ENTER;
+
+	for (idx = 0; idx < pxframe->txreq_cnt; idx++) {
+		txreq->os_priv = pxframe;
+		txreq++;
+	}
+}
+
+u8 core_wlan_fill_txreq_pre(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	u32 frag_perfr, wl_frags = 0;
+
+	if (pxframe->xftype == RTW_TX_OS) {
+		get_wl_frag_paras(padapter, pxframe, &frag_perfr, &wl_frags);
+		if (fill_txreq_pkt_perfrag_txos(padapter, pxframe, frag_perfr, wl_frags) == _FAIL)
+			return _FAIL;
+	} else if (pxframe->xftype == RTW_TX_DRV_MGMT) {
+		if (fill_txreq_pkt_mgmt(padapter, pxframe) == _FAIL)
+			return _FAIL;
+	}
+
+	return _SUCCESS;
+}
+
+u8 core_wlan_fill_txreq_post(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	if (fill_txreq_mdata(padapter, pxframe) == _FAIL)
+		return _FAIL;
+	fill_txreq_others(padapter, pxframe);
+
+#ifdef CONFIG_PCI_HCI
+	/*must be called after all pkt contents modified (cache sync)*/
+	fill_txreq_phyaddr(padapter, pxframe);
+#ifdef CONFIG_XMIT_ACK_BY_REL_RPT
+	fill_txreq_ack_txfb(padapter, pxframe);
+#endif
+#endif
+	return _SUCCESS;
+}
+
+s32 core_wlan_fill_head(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	sint res = _SUCCESS;
+	u32 idx = 0;
+	if (pxframe->xftype == RTW_TX_OS) {
+		for (idx = 0; idx < pxframe->attrib.nr_frags; idx++) {
+			u8 *pwlanhdr = pxframe->wlhdr[idx];
+
+			if (!pwlanhdr) {
+				PHLTX_ERR;
+				continue;
+			}
+
+			if ( _FAIL == rtw_make_wlanhdr(padapter, pwlanhdr, &pxframe->attrib)) //rtw_core_make_wlanhdr(padapter, pwlanhdr, pxframe);
+				return _FAIL;
+
+			if (idx == (pxframe->attrib.nr_frags - 1))
+				ClearMFrag(pwlanhdr);
+			else
+				SetMFrag(pwlanhdr);
+
+			if (pxframe->attrib.iv_len) {
+				update_attrib_sec_iv_info(padapter, &pxframe->attrib);
+				_rtw_memcpy((pwlanhdr+pxframe->attrib.hdrlen), pxframe->attrib.iv, pxframe->attrib.iv_len);
+			}
+
+			if (idx == 0 && !pxframe->attrib.amsdu) {
+				/* Add LLC/SNAP to first fragment */
+				rtw_put_snap(pwlanhdr+pxframe->attrib.hdrlen+pxframe->attrib.iv_len,
+					     pxframe->attrib.ether_type);
+			}
+
+#ifdef RTW_PHL_TEST_FPGA
+{
+	struct rtw_ieee80211_hdr *p = (struct rtw_ieee80211_hdr *)pwlanhdr;
+	unsigned short *fctrl;
+	unsigned int pktlen = 0;
+	u16 *qc;
+
+	test_seq++;
+	test_seq = test_seq%0xFFF;
+	SetSeqNum(p, test_seq);
+}
+#endif
+
+		}
+	}
+	return res;
+}
+
+
+void core_wlan_fill_tail(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	;
+
+}
+
+
+u8 core_wlan_fill_tkip_mic(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	u8 *llc = NULL;
+	u8 *payload = NULL;
+	u8 mic[8] = {0x0};
+	struct mic_data micdata;
+	struct pkt_attrib *pattrib = &pxframe->attrib;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	s8 bmcst = IS_MCAST(pattrib->ra);
+	u8 priority[4] = {0x0};
+	int i = 0;
+	struct rtw_xmit_req *xf_txreq = pxframe->phl_txreq;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+	if (pattrib->encrypt == _TKIP_) {
+		u8 null_key[16] = {0x0};
+
+		/* set TKIP MIC key */
+		if (bmcst) {
+			if (_rtw_memcmp(lsecuritypriv->dot118021XGrptxmickey[lsecuritypriv->dot118021XGrpKeyid].skey,
+					null_key, 16) == _TRUE)
+				return _FAIL;
+
+			rtw_secmicsetkey(&micdata, lsecuritypriv->dot118021XGrptxmickey[lsecuritypriv->dot118021XGrpKeyid].skey);
+		} else {
+			if (_rtw_memcmp(
+					&pattrib->dot11tkiptxmickey.skey[0],
+					null_key, 16) == _TRUE)
+				return _FAIL;
+
+			rtw_secmicsetkey(&micdata, &pattrib->dot11tkiptxmickey.skey[0]);
+		}
+
+		/* set DA, SA */
+		rtw_secmicappend(&micdata, &pattrib->dst[0], 6);
+		rtw_secmicappend(&micdata, &pattrib->src[0], 6);
+
+		if (pattrib->qos_en)
+			priority[0] = pattrib->priority;
+
+		/* set priority */
+		rtw_secmicappend(&micdata, &priority[0], 4);
+
+		/* set LLC; TBD: should check if LLC is existed or not */
+		llc = pxframe->wlhdr[0] + pxframe->attrib.hdrlen + pxframe->attrib.iv_len;
+		rtw_secmicappend(&micdata, llc, SNAP_SIZE + sizeof(u16));
+
+		/* set MSDU payload */
+		pkt_list = (struct rtw_pkt_buf_list *) xf_txreq->pkt_list;
+
+		/*ignore hdr move to payload*/
+		pkt_list ++;
+		/*for loop ignore tail*/
+		for (i = 1; i < xf_txreq->pkt_cnt - 1; i++) {
+			rtw_secmicappend(&micdata, pkt_list->vir_addr, pkt_list->length);
+			pkt_list ++;
+		}
+
+		/* calculate MIC */
+		rtw_secgetmic(&micdata, &mic[0]);
+
+		/* append MIC to the last tail */
+		_rtw_memcpy(pxframe->wltail[pxframe->attrib.nr_frags-1], &(mic[0]), 8);
+	}
+
+	return _SUCCESS;
+}
+
+
+static void core_wlan_sw_encrypt(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct pkt_attrib *attrib;
+	u8 res;
+
+
+	attrib = &pxframe->attrib;
+	if (!attrib->encrypt)
+		return;
+	if (!attrib->bswenc)
+		return;
+
+	/* convert txreq to one piece */
+	res = merge_txreq_to_one_piece(padapter, pxframe);
+	if (res != _SUCCESS) {
+		RTW_ERR("%s: fail alloc buffer for sw enc!\n", __func__);
+		return;
+	}
+	xmitframe_swencrypt(padapter, pxframe);
+}
+
+#ifdef CONFIG_TX_AMSDU_SW_MODE
+static bool core_tx_amsdu_timeout(_adapter *padapter, struct pkt_attrib *pattrib)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	u8 amsdu_timeout;
+
+	amsdu_timeout = rtw_amsdu_get_timer_status(padapter, pattrib->priority);
+
+	if (amsdu_timeout == RTW_AMSDU_TIMER_UNSET) {
+		rtw_amsdu_set_timer_status(padapter,
+			pattrib->priority, RTW_AMSDU_TIMER_SETTING);
+		rtw_amsdu_set_timer(padapter, pattrib->priority);
+		pxmitpriv->amsdu_debug_set_timer++;
+		return false;
+	} else if (amsdu_timeout == RTW_AMSDU_TIMER_SETTING) {
+		return false;
+	} else if (amsdu_timeout == RTW_AMSDU_TIMER_TIMEOUT) {
+		rtw_amsdu_set_timer_status(padapter,
+			pattrib->priority, RTW_AMSDU_TIMER_UNSET);
+		pxmitpriv->amsdu_debug_timeout++;
+		return true;
+	}
+
+	return false;
+}
+
+/* 'pxframes[]' is array to store xframe to do AMSDU whose size is 'max_xf_nr',
+ * and return value is real used size. If return size is 1, then set 'amsdu' to
+ * decide normal frame or AMSDU one.
+ */
+static int core_tx_amsdu_dequeue(_adapter *padapter, struct xmit_frame *pxframes[],
+				 int max_xf_nr, bool *amsdu)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	struct xmit_frame *pxframe;
+	int tx_amsdu = rtw_min(padapter->tx_amsdu, max_xf_nr);
+	int tx_amsdu_rate = padapter->tx_amsdu_rate;
+	int current_tx_rate = dvobj->traffic_stat.cur_tx_tp;
+	int num_frame;
+	int nr_xf;
+
+	if (tx_amsdu == 0)
+		goto dequeue_normal_pkt;
+
+	if (!MLME_IS_STA(padapter))
+		goto dequeue_normal_pkt;
+
+	if (tx_amsdu >= 2 && tx_amsdu_rate && current_tx_rate < tx_amsdu_rate)
+		goto dequeue_normal_pkt;
+
+	/*No amsdu when wifi_spec on*/
+	if (pregpriv->wifi_spec == 1)
+		goto dequeue_normal_pkt;
+
+	pxframe = rtw_get_xframe(pxmitpriv, &num_frame);
+	if (num_frame == 0 || !pxframe)
+		return 0;
+
+	if (num_frame < tx_amsdu) {	/* Not enough MSDU for specific A-MSDU */
+		if (!core_tx_amsdu_timeout(padapter, &pxframe->attrib))
+			return 0;	/* Not timeout yet */
+	}
+
+	for (nr_xf = 0; nr_xf < tx_amsdu; nr_xf++) {
+		pxframe = rtw_get_xframe(pxmitpriv, &num_frame);
+
+		if (num_frame == 0 || !pxframe)
+			break;
+
+		if (!check_amsdu(pxframe))
+			break;
+
+		/* TODO: check if size is over peer's capability */
+
+		pxframe = rtw_dequeue_xframe(pxmitpriv, pxmitpriv->hwxmits,
+					     pxmitpriv->hwxmit_entry);
+
+		pxframes[nr_xf] = pxframe;
+	}
+
+	if (nr_xf == 0) {
+		if (num_frame > 0)
+			goto dequeue_normal_pkt;
+		RTW_WARN("%s: nr_xf=0, num_frame=%d\n", __func__, num_frame);
+		return 0;
+	}
+
+	if (nr_xf < AMSDU_DEBUG_MAX_COUNT)
+		pxmitpriv->amsdu_debug_coalesce[nr_xf-1]++;
+	else
+		pxmitpriv->amsdu_debug_coalesce[AMSDU_DEBUG_MAX_COUNT-1]++;
+
+	*amsdu = (nr_xf == 1 && tx_amsdu >= 2) ? false : true;
+
+	return nr_xf;
+
+dequeue_normal_pkt:
+	pxframe = rtw_dequeue_xframe(pxmitpriv, pxmitpriv->hwxmits,
+				     pxmitpriv->hwxmit_entry);
+	if (!pxframe)
+		return 0;
+
+	pxframes[0] = pxframe;
+	*amsdu = false;
+
+	return 1;
+}
+
+static bool core_tx_amsdu_dump(_adapter *padapter, struct xmit_frame *pxframes[],
+			       int xf_nr, bool amsdu)
+{
+	struct xmit_frame *head_xframe;
+	struct pkt_attrib *head_attrib;
+	u32 pktlen;
+
+	/* prepare head xmitframe */
+	head_xframe = pxframes[0];
+	head_attrib = &head_xframe->attrib;
+
+	if (xf_nr == 1 && !amsdu)
+		goto dump_pkt;
+
+	rtw_coalesce_tx_amsdu(padapter, pxframes, xf_nr, amsdu, &pktlen);
+
+	/* update proper attribute */
+	head_attrib->amsdu = 1;
+	head_attrib->pkt_hdrlen = 0;
+	head_attrib->pktlen = pktlen;
+
+dump_pkt:
+	if (core_tx_prepare_phl(padapter, head_xframe) == FAIL)
+		goto abort_core_tx;
+
+	if (core_tx_call_phl(padapter, head_xframe, NULL) == FAIL)
+		goto abort_core_tx;
+
+	return true;
+
+abort_core_tx:
+	core_tx_free_xmitframe(padapter, head_xframe);
+
+	return true;
+}
+
+void core_tx_amsdu_tasklet(unsigned long priv)
+{
+	_adapter *padapter = (_adapter *)priv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_frame *pxframes[5];
+	int xf_nr;
+	bool amsdu;
+
+	pxmitpriv->amsdu_debug_tasklet++;
+
+	while (1) {
+		xf_nr = core_tx_amsdu_dequeue(padapter, pxframes, ARRAY_SIZE(pxframes),
+					      &amsdu);
+		if (xf_nr == 0)
+			break;
+
+		pxmitpriv->amsdu_debug_dequeue++;
+
+		core_tx_amsdu_dump(padapter, pxframes, xf_nr, amsdu);
+	}
+}
+
+static s32 core_tx_amsdu_enqueue(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct pkt_attrib *pattrib = &pxframe->attrib;
+	int tx_amsdu = padapter->tx_amsdu;
+	u8 amsdu_timeout;
+	s32 res;
+
+	if (MLME_IS_STA(padapter) && check_amsdu_tx_support(padapter, pattrib)) {
+		if (IS_AMSDU_AMPDU_VALID(pattrib))
+			goto enqueue;
+	}
+
+	return FAIL;
+
+enqueue:
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	res = rtw_xmitframe_enqueue(padapter, pxframe);
+	if (res == _FAIL) {
+		_rtw_spinunlock_bh(&pxmitpriv->lock);
+		return FAIL;
+	}
+
+	pxmitpriv->amsdu_debug_enqueue++;
+
+	if (tx_amsdu >= 2) {
+		amsdu_timeout = rtw_amsdu_get_timer_status(padapter, pattrib->priority);
+		if (amsdu_timeout == RTW_AMSDU_TIMER_SETTING) {
+			rtw_amsdu_cancel_timer(padapter, pattrib->priority);
+			rtw_amsdu_set_timer_status(padapter, pattrib->priority,
+				RTW_AMSDU_TIMER_UNSET);
+		}
+	}
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+	rtw_tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
+
+	return _TRUE;
+}
+#endif /* CONFIG_TX_AMSDU_SW_MODE */
+
+s32 core_tx_prepare_phl(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	if (core_wlan_fill_txreq_pre(padapter, pxframe) == _FAIL)
+		return FAIL;
+
+	if (pxframe->xftype == RTW_TX_OS) {
+		if (core_wlan_fill_head(padapter, pxframe) == _FAIL) {
+			RTW_ERR("core_wlan_fill_head FAIL\n");
+			return FAIL;
+		}
+		if (core_wlan_fill_tkip_mic(padapter, pxframe) == _FAIL) {
+			RTW_ERR("core_wlan_fill_tkip_mic FAIL\n");
+			return FAIL;
+		}
+	}
+	core_wlan_fill_tail(padapter, pxframe);
+	core_wlan_sw_encrypt(padapter, pxframe);
+
+	if (core_wlan_fill_txreq_post(padapter, pxframe) == _FAIL)
+		return FAIL;
+
+	return SUCCESS;
+}
+
+
+s32 core_tx_call_phl(_adapter *padapter, struct xmit_frame *pxframe, void *txsc_pkt)
+{
+	struct rtw_xmit_req *txreq = NULL;
+	void *phl = padapter->dvobj->phl;
+	u32 idx = 0;
+	u8 txreq_cnt = 0;
+#ifdef CONFIG_CORE_TXSC
+	struct rtw_xmit_req *ptxsc_txreq = NULL;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+	struct txsc_pkt_entry *ptxsc_pkt = (struct txsc_pkt_entry *)txsc_pkt;
+	if (ptxsc_pkt)
+		ptxsc_txreq = ptxsc_pkt->ptxreq;
+
+	txreq = pxframe ? pxframe->phl_txreq : ptxsc_txreq;
+	txreq_cnt = pxframe ? pxframe->txreq_cnt : 1;
+#else
+	txreq = pxframe->phl_txreq;
+	txreq_cnt = pxframe->txreq_cnt;
+#endif
+
+	for (idx = 0; idx < txreq_cnt; idx++) {
+
+#ifdef RTW_PHL_TEST_FPGA
+		core_add_record(padapter, REC_TX_PHL, txreq);
+#endif
+
+		if (rtw_phl_add_tx_req(phl, txreq) != RTW_PHL_STATUS_SUCCESS)
+			return FAIL;
+
+		rtw_phl_tx_req_notify(phl);
+
+
+		txreq++;
+	}
+
+	/* should count tx status after add tx req is success */
+#ifdef CONFIG_CORE_TXSC
+	if (ptxsc_txreq != NULL)
+		rtw_count_tx_stats_tx_req(padapter, ptxsc_txreq, ptxsc_pkt->psta);
+	else
+#endif
+		rtw_count_tx_stats(padapter, pxframe, pxframe->attrib.pktlen);
+
+	return SUCCESS;
+}
+
+s32 core_tx_per_packet(_adapter *padapter, struct xmit_frame *pxframe,
+			   struct sk_buff **pskb, struct sta_info *psta)
+{
+#if defined(CONFIG_AP_MODE) || defined(CONFIG_CORE_TXSC)
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+#endif
+
+	if (core_tx_update_xmitframe(padapter, pxframe, pskb, psta, RTW_TX_OS) == FAIL)
+		goto abort_tx_per_packet;
+
+#ifdef CONFIG_80211N_HT
+	if ((pxframe->attrib.ether_type != 0x0806)
+	    && (pxframe->attrib.ether_type != 0x888e)
+	    && (pxframe->attrib.dhcp_pkt != 1))
+		rtw_issue_addbareq_cmd(padapter, pxframe, _TRUE);
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_TX_AMSDU_SW_MODE
+	if (core_tx_amsdu_enqueue(padapter, pxframe) == _TRUE)
+		return SUCCESS;	/* queued to do AMSDU */
+#endif
+
+	if (core_tx_prepare_phl(padapter, pxframe) == FAIL)
+		goto abort_tx_per_packet;
+
+#ifdef CONFIG_AP_MODE
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+	if (xmitframe_enqueue_for_sleeping_sta(padapter, pxframe) == _TRUE) {
+		_rtw_spinunlock_bh(&pxmitpriv->lock);
+		DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue);
+		return SUCCESS;
+	}
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+#endif
+
+#if !defined(CONFIG_CORE_TXSC) || defined(CONFIG_RTW_DATA_BMC_TO_UC)
+	if (core_tx_call_phl(padapter, pxframe, NULL) == SUCCESS)
+#endif
+		return SUCCESS;
+
+
+abort_tx_per_packet:
+	if (pxframe->pkt == NULL)
+		rtw_os_pkt_complete(padapter, *pskb);
+	core_tx_free_xmitframe(padapter, pxframe);
+
+	return FAIL;
+}
+
+s32 rtw_core_tx(_adapter *padapter, struct sk_buff **pskb, struct sta_info *psta, u16 os_qid)
+{
+	struct xmit_frame *pxframe = NULL;
+#if defined(CONFIG_AP_MODE) || defined(CONFIG_CORE_TXSC)
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+#endif
+	s32 res = 0;
+#ifdef CONFIG_CORE_TXSC
+	struct txsc_pkt_entry txsc_pkt;
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+	if (txsc_get_sc_cached_entry(padapter, *pskb, &txsc_pkt) == _SUCCESS)
+		goto core_txsc;
+#endif
+
+	if (core_tx_alloc_xmitframe(padapter, &pxframe, os_qid) == FAIL)
+		goto abort_core_tx;
+
+	if (core_tx_update_pkt(padapter, pxframe, pskb) == FAIL)
+		goto abort_core_tx;
+
+#if defined(CONFIG_AP_MODE)
+	if (MLME_STATE(padapter) & WIFI_AP_STATE) {
+		_list f_list;
+
+		res = rtw_ap_addr_resolve(padapter, os_qid, pxframe, *pskb, &f_list);
+		if (res == _FAIL)
+			goto abort_core_tx;
+
+		#if defined(CONFIG_RTW_WDS) || CONFIG_RTW_DATA_BMC_TO_UC
+		if (!rtw_is_list_empty(&f_list)) {
+			_list *list = get_next(&f_list);
+			struct xmit_frame *fframe;
+
+			while ((rtw_end_of_queue_search(&f_list, list)) == _FALSE) {
+				fframe = LIST_CONTAINOR(list, struct xmit_frame, list);
+				list = get_next(list);
+				rtw_list_delete(&fframe->list);
+
+				if (res == RTW_ORI_NO_NEED && rtw_is_list_empty(&f_list)) {
+					fframe->pkt = pxframe->pkt; /*last frame */
+					pxframe->pkt = NULL;
+				} else {
+					fframe->pkt = rtw_skb_copy(*pskb);
+				}
+
+				if (!fframe->pkt) {
+					if (res == RTW_ORI_NO_NEED && IS_MCAST(pxframe->attrib.dst))
+						res = _SUCCESS;
+
+					core_tx_free_xmitframe(padapter, fframe);
+					continue;
+				}
+
+				core_tx_per_packet(padapter, fframe, &fframe->pkt, NULL);
+			}
+		}
+		#endif
+
+		if (res == RTW_ORI_NO_NEED) {
+			core_tx_free_xmitframe(padapter, pxframe);
+			return SUCCESS;
+		}
+	}
+#endif /* defined(CONFIG_AP_MODE) */
+#ifdef CONFIG_LAYER2_ROAMING
+        if ((padapter->mlmepriv.roam_network) && ((*pskb)->protocol != htons(0x888e))) {        /* eapol never enqueue.*/
+		pxframe->pkt = *pskb;
+		rtw_list_delete(&pxframe->list);
+		_rtw_spinlock_bh(&pxmitpriv->rpkt_queue.lock);
+		rtw_list_insert_tail(&(pxframe->list), get_list_head(&(pxmitpriv->rpkt_queue)));
+		_rtw_spinunlock_bh(&pxmitpriv->rpkt_queue.lock);
+		return SUCCESS;
+	}
+#endif
+
+	res = core_tx_per_packet(padapter, pxframe, pskb, psta);
+	if (res == FAIL)
+		return FAIL;
+
+#ifdef CONFIG_CORE_TXSC
+	txsc_add_sc_cache_entry(padapter, pxframe, &txsc_pkt);
+
+core_txsc:
+
+	if (txsc_apply_sc_cached_entry(padapter, &txsc_pkt) == _FAIL)
+		goto abort_core_tx;
+
+	if (core_tx_call_phl(padapter, pxframe, &txsc_pkt) == FAIL)
+		goto abort_core_tx;
+#endif
+
+	return SUCCESS;
+
+abort_core_tx:
+	if (pxframe == NULL) {
+#ifdef CONFIG_CORE_TXSC
+		if (txsc_pkt.ptxreq)
+			txsc_free_txreq(padapter, txsc_pkt.ptxreq);
+		else
+#endif
+			rtw_os_pkt_complete(padapter, *pskb);
+	} else {
+		if (pxframe->pkt == NULL)
+			rtw_os_pkt_complete(padapter, *pskb);
+
+		core_tx_free_xmitframe(padapter, pxframe);
+	}
+
+	return FAIL;
+}
+
+enum rtw_phl_status
+rtw_core_tx_recycle(void *drv_priv, struct rtw_xmit_req *txreq)
+{
+	_adapter *padapter = NULL;
+	struct xmit_frame *pxframe = NULL;
+#ifdef CONFIG_CORE_TXSC
+	struct xmit_txreq_buf *ptxreq_buf = NULL;
+#endif
+
+	if (txreq->os_priv == NULL) {
+		RTW_ERR("NULL txreq!\n");
+		return RTW_PHL_STATUS_FAILURE;
+	}
+
+#ifdef CONFIG_CORE_TXSC
+	if (txreq->treq_type == RTW_PHL_TREQ_TYPE_CORE_TXSC) {
+		ptxreq_buf = (struct xmit_txreq_buf *)txreq->os_priv;
+		padapter = ptxreq_buf->adapter;
+		#ifdef RTW_PHL_DBG_CMD
+		core_add_record(padapter, REC_TX_PHL_RCC, txreq);
+		#endif
+		txsc_free_txreq(padapter, txreq);
+		return RTW_PHL_STATUS_SUCCESS;
+	}
+#endif /* CONFIG_CORE_TXSC */
+
+	pxframe = (struct xmit_frame *)txreq->os_priv;
+	if (pxframe == NULL) {
+		RTW_ERR("%s: NULL xmitframe !!\n", __func__);
+		rtw_warn_on(1);
+		return RTW_PHL_STATUS_FAILURE;
+	}
+
+	padapter = pxframe->padapter;
+
+	#ifdef RTW_PHL_DBG_CMD
+	core_add_record(padapter, REC_TX_PHL_RCC, txreq);
+	#endif
+
+	#ifdef CONFIG_PCI_HCI
+	core_recycle_txreq_phyaddr(padapter, txreq);
+	#endif
+	core_tx_free_xmitframe(padapter, pxframe);
+
+	return RTW_PHL_STATUS_SUCCESS;
+}
+#endif
+
+
+#ifdef CONFIG_TDLS
+sint xmitframe_enqueue_for_tdls_sleeping_sta(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	sint ret = _FALSE;
+
+	struct sta_info *ptdls_sta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	int i;
+
+	ptdls_sta = rtw_get_stainfo(pstapriv, pattrib->dst);
+	if (ptdls_sta == NULL)
+		return ret;
+	else if (ptdls_sta->tdls_sta_state & TDLS_LINKED_STATE) {
+
+		if (pattrib->triggered == 1) {
+			ret = _TRUE;
+			return ret;
+		}
+
+		_rtw_spinlock_bh(&ptdls_sta->sleep_q.lock);
+
+		if (ptdls_sta->state & WIFI_SLEEP_STATE) {
+			rtw_list_delete(&pxmitframe->list);
+
+			/* _rtw_spinlock_bh(&psta->sleep_q.lock);	 */
+
+			rtw_list_insert_tail(&pxmitframe->list, get_list_head(&ptdls_sta->sleep_q));
+
+			ptdls_sta->sleepq_len++;
+			ptdls_sta->sleepq_ac_len++;
+
+			/* indicate 4-AC queue bit in TDLS peer traffic indication */
+			switch (pattrib->priority) {
+			case 1:
+			case 2:
+				ptdls_sta->uapsd_bk |= BIT(1);
+				break;
+			case 4:
+			case 5:
+				ptdls_sta->uapsd_vi |= BIT(1);
+				break;
+			case 6:
+			case 7:
+				ptdls_sta->uapsd_vo |= BIT(1);
+				break;
+			case 0:
+			case 3:
+			default:
+				ptdls_sta->uapsd_be |= BIT(1);
+				break;
+			}
+
+			/* Transmit TDLS PTI via AP */
+			if (ptdls_sta->sleepq_len == 1)
+				rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_ISSUE_PTI, 0);
+
+			ret = _TRUE;
+		}
+
+		_rtw_spinunlock_bh(&ptdls_sta->sleep_q.lock);
+	}
+
+	return ret;
+
+}
+#endif /* CONFIG_TDLS */
+
+#define RTW_HIQ_FILTER_ALLOW_ALL 0
+#define RTW_HIQ_FILTER_ALLOW_SPECIAL 1
+#define RTW_HIQ_FILTER_DENY_ALL 2
+
+inline bool xmitframe_hiq_filter(struct xmit_frame *xmitframe)
+{
+	bool allow = _FALSE;
+	_adapter *adapter = xmitframe->padapter;
+	struct registry_priv *registry = &adapter->registrypriv;
+
+	if (adapter->registrypriv.wifi_spec == 1)
+		allow = _TRUE;
+	else if (registry->hiq_filter == RTW_HIQ_FILTER_ALLOW_SPECIAL) {
+
+		struct pkt_attrib *attrib = &xmitframe->attrib;
+
+		if (attrib->ether_type == 0x0806
+		    || attrib->ether_type == 0x888e
+#ifdef CONFIG_WAPI_SUPPORT
+		    || attrib->ether_type == 0x88B4
+#endif
+		    || attrib->dhcp_pkt
+		   ) {
+			if (0)
+				RTW_INFO(FUNC_ADPT_FMT" ether_type:0x%04x%s\n", FUNC_ADPT_ARG(xmitframe->padapter)
+					, attrib->ether_type, attrib->dhcp_pkt ? " DHCP" : "");
+			allow = _TRUE;
+		}
+	} else if (registry->hiq_filter == RTW_HIQ_FILTER_ALLOW_ALL)
+		allow = _TRUE;
+	else if (registry->hiq_filter == RTW_HIQ_FILTER_DENY_ALL)
+		allow = _FALSE;
+	else
+		rtw_warn_on(1);
+
+	return allow;
+}
+
+#if defined(CONFIG_AP_MODE) || defined(CONFIG_TDLS)
+
+sint xmitframe_enqueue_for_sleeping_sta(_adapter *padapter, struct xmit_frame *pxmitframe)
+{
+	sint ret = _FALSE;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct pkt_attrib *pattrib = &pxmitframe->attrib;
+	sint bmcst = IS_MCAST(pattrib->ra);
+	bool update_tim = _FALSE;
+	struct _ADAPTER_LINK *padapter_link = pattrib->adapter_link;
+#ifdef CONFIG_TDLS
+
+	if (padapter->tdlsinfo.link_established == _TRUE)
+		ret = xmitframe_enqueue_for_tdls_sleeping_sta(padapter, pxmitframe);
+#endif /* CONFIG_TDLS */
+
+	if (!MLME_IS_AP(padapter) && !MLME_IS_MESH(padapter)) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue_warn_fwstate);
+		return ret;
+	}
+	/*
+		if(pattrib->psta)
+		{
+			psta = pattrib->psta;
+		}
+		else
+		{
+			RTW_INFO("%s, call rtw_get_stainfo()\n", __func__);
+			psta=rtw_get_stainfo(pstapriv, pattrib->ra);
+		}
+	*/
+	psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+	if (pattrib->psta != psta) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue_warn_sta);
+		RTW_INFO("%s, pattrib->psta(%p) != psta(%p)\n", __func__, pattrib->psta, psta);
+		return _FALSE;
+	}
+
+	if (psta == NULL) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue_warn_nosta);
+		RTW_INFO("%s, psta==NUL\n", __func__);
+		return _FALSE;
+	}
+
+	if (!(psta->state & WIFI_ASOC_STATE)) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue_warn_link);
+		RTW_INFO("%s, psta->state(0x%x) != WIFI_ASOC_STATE\n", __func__, psta->state);
+		return _FALSE;
+	}
+
+	if (pattrib->triggered == 1) {
+		DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue_warn_trigger);
+		/* RTW_INFO("directly xmit pspoll_triggered packet\n"); */
+
+		/* pattrib->triggered=0; */
+		#if 0
+		if (bmcst && xmitframe_hiq_filter(pxmitframe) == _TRUE)
+			pattrib->qsel = rtw_hal_get_qsel(padapter, QSLT_HIGH_ID);/* HIQ */
+		#endif
+		return ret;
+	}
+
+
+	if (bmcst) {
+#ifndef BMC_ON_HIQ
+		_rtw_spinlock_bh(&psta->sleep_q.lock);
+
+		if (rtw_tim_map_anyone_be_set(padapter, pstapriv->sta_dz_bitmap)) { /* if anyone sta is in ps mode */
+			/* pattrib->qsel = rtw_hal_get_qsel(padapter,QSLT_HIGH_ID);*/ /* HIQ */
+
+			rtw_list_delete(&pxmitframe->list);
+
+			/*_rtw_spinlock_bh(&psta->sleep_q.lock);*/
+
+			rtw_list_insert_tail(&pxmitframe->list, get_list_head(&psta->sleep_q));
+
+			psta->sleepq_len++;
+
+			if (!(rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, 0)))
+				update_tim = _TRUE;
+
+			rtw_tim_map_set(padapter, pstapriv->tim_bitmap, 0);
+			rtw_tim_map_set(padapter, pstapriv->sta_dz_bitmap, 0);
+
+			/* RTW_INFO("enqueue, sq_len=%d\n", psta->sleepq_len); */
+			/* RTW_INFO_DUMP("enqueue, tim=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len); */
+			if (update_tim == _TRUE) {
+				if (is_broadcast_mac_addr(pattrib->ra))
+					_update_beacon(padapter, padapter_link, _TIM_IE_, NULL, _TRUE, 0, "buffer BC");
+				else
+					_update_beacon(padapter, padapter_link, _TIM_IE_, NULL, _TRUE, 0, "buffer MC");
+			} else
+				chk_bmc_sleepq_cmd(padapter);
+
+			/*_rtw_spinunlock_bh(&psta->sleep_q.lock);*/
+
+			ret = _TRUE;
+
+			DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue_mcast);
+		}
+
+		_rtw_spinunlock_bh(&psta->sleep_q.lock);
+#endif
+		return ret;
+
+	}
+
+
+	_rtw_spinlock_bh(&psta->sleep_q.lock);
+
+	if (psta->state & WIFI_SLEEP_STATE) {
+		u8 wmmps_ac = 0;
+
+		if (rtw_tim_map_is_set(padapter, pstapriv->sta_dz_bitmap, psta->phl_sta->aid)) {
+			rtw_list_delete(&pxmitframe->list);
+
+			/* _rtw_spinlock_bh(&psta->sleep_q.lock);	 */
+
+			rtw_list_insert_tail(&pxmitframe->list, get_list_head(&psta->sleep_q));
+
+			psta->sleepq_len++;
+
+			switch (pattrib->priority) {
+			case 1:
+			case 2:
+				wmmps_ac = psta->uapsd_bk & BIT(0);
+				break;
+			case 4:
+			case 5:
+				wmmps_ac = psta->uapsd_vi & BIT(0);
+				break;
+			case 6:
+			case 7:
+				wmmps_ac = psta->uapsd_vo & BIT(0);
+				break;
+			case 0:
+			case 3:
+			default:
+				wmmps_ac = psta->uapsd_be & BIT(0);
+				break;
+			}
+
+			if (wmmps_ac)
+				psta->sleepq_ac_len++;
+
+			if (((psta->has_legacy_ac) && (!wmmps_ac)) || ((!psta->has_legacy_ac) && (wmmps_ac))) {
+				if (!(rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid)))
+					update_tim = _TRUE;
+
+				rtw_tim_map_set(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid);
+
+				/* RTW_INFO("enqueue, sq_len=%d\n", psta->sleepq_len); */
+				/* RTW_INFO_DUMP("enqueue, tim=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len); */
+
+				if (update_tim == _TRUE) {
+					/* RTW_INFO("sleepq_len==1, update BCNTIM\n"); */
+					/* upate BCN for TIM IE */
+					_update_beacon(padapter, padapter_link, _TIM_IE_, NULL, _TRUE, 0, "buffer UC");
+				}
+			}
+
+			/* _rtw_spinunlock_bh(&psta->sleep_q.lock);			 */
+
+			/* if(psta->sleepq_len > (NR_XMITFRAME>>3)) */
+			/* { */
+			/*	wakeup_sta_to_xmit(padapter, psta); */
+			/* }	 */
+
+			ret = _TRUE;
+
+			DBG_COUNTER(padapter->tx_logs.core_tx_ap_enqueue_ucast);
+		}
+
+	}
+
+	_rtw_spinunlock_bh(&psta->sleep_q.lock);
+
+	return ret;
+
+}
+
+static void dequeue_xmitframes_to_sleeping_queue(_adapter *padapter, struct sta_info *psta, _queue *pframequeue)
+{
+	sint ret;
+	_list	*plist, *phead;
+	u8	ac_index;
+	struct tx_servq	*ptxservq;
+	struct pkt_attrib	*pattrib;
+	struct xmit_frame	*pxmitframe;
+	struct hw_xmit *phwxmits =  padapter->xmitpriv.hwxmits;
+
+	phead = get_list_head(pframequeue);
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		pxmitframe = LIST_CONTAINOR(plist, struct xmit_frame, list);
+
+		plist = get_next(plist);
+
+		pattrib = &pxmitframe->attrib;
+
+		pattrib->triggered = 0;
+
+		ret = xmitframe_enqueue_for_sleeping_sta(padapter, pxmitframe);
+
+		if (_TRUE == ret) {
+			ptxservq = rtw_get_sta_pending(padapter, psta, pattrib->priority, (u8 *)(&ac_index));
+
+			ptxservq->qcnt--;
+			phwxmits[ac_index].accnt--;
+		} else {
+			/* RTW_INFO("xmitframe_enqueue_for_sleeping_sta return _FALSE\n"); */
+		}
+
+	}
+
+}
+
+void stop_sta_xmit(_adapter *padapter, struct sta_info *psta)
+{
+	struct sta_info *psta_bmc;
+	struct sta_xmit_priv *pstaxmitpriv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	pstaxmitpriv = &psta->sta_xmitpriv;
+
+	/* for BC/MC Frames */
+	psta_bmc = rtw_get_bcmc_stainfo(padapter, psta->padapter_link);
+	if (!psta_bmc)
+		rtw_warn_on(1);
+
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	psta->state |= WIFI_SLEEP_STATE;
+
+#ifdef CONFIG_TDLS
+	if (!(psta->tdls_sta_state & TDLS_LINKED_STATE))
+#endif /* CONFIG_TDLS */
+		rtw_tim_map_set(padapter, pstapriv->sta_dz_bitmap, psta->phl_sta->aid);
+
+	dequeue_xmitframes_to_sleeping_queue(padapter, psta, &pstaxmitpriv->vo_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->vo_q.tx_pending));
+	dequeue_xmitframes_to_sleeping_queue(padapter, psta, &pstaxmitpriv->vi_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->vi_q.tx_pending));
+	dequeue_xmitframes_to_sleeping_queue(padapter, psta, &pstaxmitpriv->be_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->be_q.tx_pending));
+	dequeue_xmitframes_to_sleeping_queue(padapter, psta, &pstaxmitpriv->bk_q.sta_pending);
+	rtw_list_delete(&(pstaxmitpriv->bk_q.tx_pending));
+
+	if (psta_bmc != NULL
+		#ifdef CONFIG_TDLS
+		&& !(psta->tdls_sta_state & TDLS_LINKED_STATE)
+		#endif
+	) {
+		/* for BC/MC Frames */
+#ifndef BMC_ON_HIQ
+		pstaxmitpriv = &psta_bmc->sta_xmitpriv;
+		dequeue_xmitframes_to_sleeping_queue(padapter, psta_bmc, &pstaxmitpriv->vo_q.sta_pending);
+		rtw_list_delete(&(pstaxmitpriv->vo_q.tx_pending));
+		dequeue_xmitframes_to_sleeping_queue(padapter, psta_bmc, &pstaxmitpriv->vi_q.sta_pending);
+		rtw_list_delete(&(pstaxmitpriv->vi_q.tx_pending));
+		dequeue_xmitframes_to_sleeping_queue(padapter, psta_bmc, &pstaxmitpriv->be_q.sta_pending);
+		rtw_list_delete(&(pstaxmitpriv->be_q.tx_pending));
+		dequeue_xmitframes_to_sleeping_queue(padapter, psta_bmc, &pstaxmitpriv->bk_q.sta_pending);
+		rtw_list_delete(&(pstaxmitpriv->bk_q.tx_pending));
+#endif
+	}
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+
+}
+
+void wakeup_sta_to_xmit(_adapter *padapter, struct sta_info *psta)
+{
+	u8 update_mask = 0, wmmps_ac = 0;
+	struct sta_info *psta_bmc;
+	_list	*xmitframe_plist, *xmitframe_phead;
+	struct xmit_frame *pxmitframe = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	psta_bmc = rtw_get_bcmc_stainfo(padapter, psta->padapter_link);
+
+
+	/* _rtw_spinlock_bh(&psta->sleep_q.lock); */
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	xmitframe_phead = get_list_head(&psta->sleep_q);
+	xmitframe_plist = get_next(xmitframe_phead);
+
+	while ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE) {
+		pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+		xmitframe_plist = get_next(xmitframe_plist);
+
+		rtw_list_delete(&pxmitframe->list);
+
+		switch (pxmitframe->attrib.priority) {
+		case 1:
+		case 2:
+			wmmps_ac = psta->uapsd_bk & BIT(1);
+			break;
+		case 4:
+		case 5:
+			wmmps_ac = psta->uapsd_vi & BIT(1);
+			break;
+		case 6:
+		case 7:
+			wmmps_ac = psta->uapsd_vo & BIT(1);
+			break;
+		case 0:
+		case 3:
+		default:
+			wmmps_ac = psta->uapsd_be & BIT(1);
+			break;
+		}
+
+		psta->sleepq_len--;
+		if (psta->sleepq_len > 0)
+			pxmitframe->attrib.mdata = 1;
+		else
+			pxmitframe->attrib.mdata = 0;
+
+		if (wmmps_ac) {
+			psta->sleepq_ac_len--;
+			if (psta->sleepq_ac_len > 0) {
+				pxmitframe->attrib.mdata = 1;
+				pxmitframe->attrib.eosp = 0;
+			} else {
+				pxmitframe->attrib.mdata = 0;
+				pxmitframe->attrib.eosp = 1;
+			}
+		}
+
+		pxmitframe->attrib.triggered = 1;
+
+		/*
+				_rtw_spinunlock_bh(&psta->sleep_q.lock);
+				//rtw_intf_data_xmit
+				if(rtw_hal_xmit(padapter, pxmitframe) == _TRUE)
+				{
+					rtw_os_xmit_complete(padapter, pxmitframe);
+				}
+				_rtw_spinlock_bh(&psta->sleep_q.lock);
+		*/
+		rtw_intf_xmitframe_enqueue(padapter, pxmitframe);
+
+
+	}
+
+	if (psta->sleepq_len == 0) {
+#ifdef CONFIG_TDLS
+		if (psta->tdls_sta_state & TDLS_LINKED_STATE) {
+			if (psta->state & WIFI_SLEEP_STATE)
+				psta->state ^= WIFI_SLEEP_STATE;
+
+			_rtw_spinunlock_bh(&pxmitpriv->lock);
+			return;
+		}
+#endif /* CONFIG_TDLS */
+
+		if (rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid)) {
+			/* RTW_INFO("wakeup to xmit, qlen==0\n"); */
+			/* RTW_INFO_DUMP("update_BCNTIM, tim=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len); */
+			/* upate BCN for TIM IE */
+			/* update_BCNTIM(padapter); */
+			update_mask = BIT(0);
+		}
+
+		rtw_tim_map_clear(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid);
+
+		if (psta->state & WIFI_SLEEP_STATE)
+			psta->state ^= WIFI_SLEEP_STATE;
+
+		if (psta->state & WIFI_STA_ALIVE_CHK_STATE) {
+			RTW_INFO("%s alive check\n", __func__);
+			psta->expire_to = pstapriv->expire_to;
+			psta->state ^= WIFI_STA_ALIVE_CHK_STATE;
+		}
+
+		rtw_tim_map_clear(padapter, pstapriv->sta_dz_bitmap, psta->phl_sta->aid);
+	}
+
+	/* for BC/MC Frames */
+	if (!psta_bmc)
+		goto _exit;
+
+	if (!(rtw_tim_map_anyone_be_set_exclude_aid0(padapter, pstapriv->sta_dz_bitmap))) { /* no any sta in ps mode */
+		xmitframe_phead = get_list_head(&psta_bmc->sleep_q);
+		xmitframe_plist = get_next(xmitframe_phead);
+
+		while ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE) {
+			pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+			xmitframe_plist = get_next(xmitframe_plist);
+
+			rtw_list_delete(&pxmitframe->list);
+
+			psta_bmc->sleepq_len--;
+			if (psta_bmc->sleepq_len > 0)
+				pxmitframe->attrib.mdata = 1;
+			else
+				pxmitframe->attrib.mdata = 0;
+
+
+			pxmitframe->attrib.triggered = 1;
+			/*
+						_rtw_spinunlock_bh(&psta_bmc->sleep_q.lock);
+						//rtw_intf_data_xmit
+						if(rtw_hal_xmit(padapter, pxmitframe) == _TRUE)
+						{
+							rtw_os_xmit_complete(padapter, pxmitframe);
+						}
+						_rtw_spinlock_bh(&psta_bmc->sleep_q.lock);
+
+			*/
+			rtw_intf_xmitframe_enqueue(padapter, pxmitframe);
+
+		}
+
+		if (psta_bmc->sleepq_len == 0) {
+			if (rtw_tim_map_is_set(padapter, pstapriv->tim_bitmap, 0)) {
+				/* RTW_INFO("wakeup to xmit, qlen==0\n"); */
+				/* RTW_INFO_DUMP("update_BCNTIM, tim=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len); */
+				/* upate BCN for TIM IE */
+				/* update_BCNTIM(padapter); */
+				update_mask |= BIT(1);
+			}
+			rtw_tim_map_clear(padapter, pstapriv->tim_bitmap, 0);
+			rtw_tim_map_clear(padapter, pstapriv->sta_dz_bitmap, 0);
+		}
+
+	}
+
+_exit:
+
+	/* _rtw_spinunlock_bh(&psta_bmc->sleep_q.lock);	 */
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+	if (update_mask) {
+		/* update_BCNTIM(padapter); */
+		if ((update_mask & (BIT(0) | BIT(1))) == (BIT(0) | BIT(1)))
+			_update_beacon(padapter, psta->padapter_link, _TIM_IE_, NULL, _TRUE, 0, "clear UC&BMC");
+		else if ((update_mask & BIT(1)) == BIT(1))
+			_update_beacon(padapter, psta->padapter_link, _TIM_IE_, NULL, _TRUE, 0, "clear BMC");
+		else
+			_update_beacon(padapter, psta->padapter_link, _TIM_IE_, NULL, _TRUE, 0, "clear UC");
+	}
+
+}
+
+void xmit_delivery_enabled_frames(_adapter *padapter, struct sta_info *psta)
+{
+	u8 wmmps_ac = 0;
+	_list	*xmitframe_plist, *xmitframe_phead;
+	struct xmit_frame *pxmitframe = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+
+	/* _rtw_spinlock_bh(&psta->sleep_q.lock); */
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	xmitframe_phead = get_list_head(&psta->sleep_q);
+	xmitframe_plist = get_next(xmitframe_phead);
+
+	while ((rtw_end_of_queue_search(xmitframe_phead, xmitframe_plist)) == _FALSE) {
+		pxmitframe = LIST_CONTAINOR(xmitframe_plist, struct xmit_frame, list);
+
+		xmitframe_plist = get_next(xmitframe_plist);
+
+		switch (pxmitframe->attrib.priority) {
+		case 1:
+		case 2:
+			wmmps_ac = psta->uapsd_bk & BIT(1);
+			break;
+		case 4:
+		case 5:
+			wmmps_ac = psta->uapsd_vi & BIT(1);
+			break;
+		case 6:
+		case 7:
+			wmmps_ac = psta->uapsd_vo & BIT(1);
+			break;
+		case 0:
+		case 3:
+		default:
+			wmmps_ac = psta->uapsd_be & BIT(1);
+			break;
+		}
+
+		if (!wmmps_ac)
+			continue;
+
+		rtw_list_delete(&pxmitframe->list);
+
+		psta->sleepq_len--;
+		psta->sleepq_ac_len--;
+
+		if (psta->sleepq_ac_len > 0) {
+			pxmitframe->attrib.mdata = 1;
+			pxmitframe->attrib.eosp = 0;
+		} else {
+			pxmitframe->attrib.mdata = 0;
+			pxmitframe->attrib.eosp = 1;
+		}
+
+		pxmitframe->attrib.triggered = 1;
+		rtw_intf_xmitframe_enqueue(padapter, pxmitframe);
+
+		if ((psta->sleepq_ac_len == 0) && (!psta->has_legacy_ac) && (wmmps_ac)) {
+#ifdef CONFIG_TDLS
+			if (psta->tdls_sta_state & TDLS_LINKED_STATE) {
+				/* _rtw_spinunlock_bh(&psta->sleep_q.lock); */
+				goto exit;
+			}
+#endif /* CONFIG_TDLS */
+			rtw_tim_map_clear(padapter, pstapriv->tim_bitmap, psta->phl_sta->aid);
+
+			/* RTW_INFO("wakeup to xmit, qlen==0\n"); */
+			/* RTW_INFO_DUMP("update_BCNTIM, tim=", pstapriv->tim_bitmap, pstapriv->aid_bmp_len); */
+			/* upate BCN for TIM IE */
+			/* update_BCNTIM(padapter); */
+			rtw_update_beacon(padapter, psta->padapter_link, _TIM_IE_, NULL, _TRUE, 0);
+			/* update_mask = BIT(0); */
+		}
+
+	}
+
+#ifdef CONFIG_TDLS
+exit:
+#endif
+	/* _rtw_spinunlock_bh(&psta->sleep_q.lock);	 */
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+
+	return;
+}
+
+#endif /* defined(CONFIG_AP_MODE) || defined(CONFIG_TDLS) */
+
+#if 0 /*#ifdef CONFIG_XMIT_THREAD_MODE*/
+void enqueue_pending_xmitbuf(
+	struct xmit_priv *pxmitpriv,
+	struct xmit_buf *pxmitbuf)
+{
+	_queue *pqueue;
+	_adapter *pri_adapter = pxmitpriv->adapter;
+
+	pqueue = &pxmitpriv->pending_xmitbuf_queue;
+
+	_rtw_spinlock_bh(&pqueue->lock);
+	rtw_list_delete(&pxmitbuf->list);
+	rtw_list_insert_tail(&pxmitbuf->list, get_list_head(pqueue));
+	_rtw_spinunlock_bh(&pqueue->lock);
+
+#if defined(CONFIG_SDIO_HCI) && defined(CONFIG_CONCURRENT_MODE)
+	pri_adapter = GET_PRIMARY_ADAPTER(pri_adapter);
+#endif /*SDIO_HCI + CONCURRENT*/
+	_rtw_up_sema(&(pri_adapter->xmitpriv.xmit_sema));
+}
+
+void enqueue_pending_xmitbuf_to_head(
+	struct xmit_priv *pxmitpriv,
+	struct xmit_buf *pxmitbuf)
+{
+	_queue *pqueue = &pxmitpriv->pending_xmitbuf_queue;
+
+	_rtw_spinlock_bh(&pqueue->lock);
+	rtw_list_delete(&pxmitbuf->list);
+	rtw_list_insert_head(&pxmitbuf->list, get_list_head(pqueue));
+	_rtw_spinunlock_bh(&pqueue->lock);
+}
+
+struct xmit_buf *dequeue_pending_xmitbuf(
+	struct xmit_priv *pxmitpriv)
+{
+	struct xmit_buf *pxmitbuf;
+	_queue *pqueue;
+
+
+	pxmitbuf = NULL;
+	pqueue = &pxmitpriv->pending_xmitbuf_queue;
+
+	_rtw_spinlock_bh(&pqueue->lock);
+
+	if (_rtw_queue_empty(pqueue) == _FALSE) {
+		_list *plist, *phead;
+
+		phead = get_list_head(pqueue);
+		plist = get_next(phead);
+		pxmitbuf = LIST_CONTAINOR(plist, struct xmit_buf, list);
+		rtw_list_delete(&pxmitbuf->list);
+	}
+
+	_rtw_spinunlock_bh(&pqueue->lock);
+
+	return pxmitbuf;
+}
+
+static struct xmit_buf *dequeue_pending_xmitbuf_ext(
+	struct xmit_priv *pxmitpriv)
+{
+	struct xmit_buf *pxmitbuf;
+	_queue *pqueue;
+
+	pxmitbuf = NULL;
+	pqueue = &pxmitpriv->pending_xmitbuf_queue;
+
+	_rtw_spinlock_bh(&pqueue->lock);
+
+	if (_rtw_queue_empty(pqueue) == _FALSE) {
+		_list *plist, *phead;
+
+		phead = get_list_head(pqueue);
+		plist = phead;
+		do {
+			plist = get_next(plist);
+			if (plist == phead)
+				break;
+
+			pxmitbuf = LIST_CONTAINOR(plist, struct xmit_buf, list);
+
+			if (pxmitbuf->buf_tag == XMITBUF_MGNT) {
+				rtw_list_delete(&pxmitbuf->list);
+				break;
+			}
+			pxmitbuf = NULL;
+		} while (1);
+	}
+
+	_rtw_spinunlock_bh(&pqueue->lock);
+
+	return pxmitbuf;
+}
+
+struct xmit_buf *select_and_dequeue_pending_xmitbuf(_adapter *padapter)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct xmit_buf *pxmitbuf = NULL;
+
+	if (_TRUE == rtw_is_xmit_blocked(padapter))
+		return pxmitbuf;
+
+	pxmitbuf = dequeue_pending_xmitbuf_ext(pxmitpriv);
+	if (pxmitbuf == NULL && rtw_xmit_ac_blocked(padapter) != _TRUE)
+		pxmitbuf = dequeue_pending_xmitbuf(pxmitpriv);
+
+	return pxmitbuf;
+}
+
+sint check_pending_xmitbuf(
+	struct xmit_priv *pxmitpriv)
+{
+	_queue *pqueue;
+	sint	ret = _FALSE;
+
+	pqueue = &pxmitpriv->pending_xmitbuf_queue;
+
+	_rtw_spinlock_bh(&pqueue->lock);
+
+	if (_rtw_queue_empty(pqueue) == _FALSE)
+		ret = _TRUE;
+
+	_rtw_spinunlock_bh(&pqueue->lock);
+
+	return ret;
+}
+
+thread_return rtw_xmit_thread(thread_context context)
+{
+	s32 err;
+	_adapter *adapter;
+#ifdef RTW_XMIT_THREAD_HIGH_PRIORITY
+#ifdef PLATFORM_LINUX
+	struct sched_param param = { .sched_priority = 1 };
+
+	sched_setscheduler(current, SCHED_FIFO, &param);
+#endif /* PLATFORM_LINUX */
+#endif /* RTW_XMIT_THREAD_HIGH_PRIORITY */
+
+	err = _SUCCESS;
+	adapter = (_adapter *)context;
+
+	rtw_thread_enter("RTW_XMIT_THREAD");
+
+	do {
+		err = rtw_intf_xmit_buf_handler(adapter);
+		flush_signals_thread();
+	} while (_SUCCESS == err);
+
+	RTW_INFO(FUNC_ADPT_FMT " Exit\n", FUNC_ADPT_ARG(adapter));
+
+	rtw_thread_wait_stop();
+
+	return 0;
+}
+#endif
+
+#ifdef DBG_XMIT_BLOCK
+void dump_xmit_block(void *sel, _adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	RTW_PRINT_SEL(sel, "[XMIT-BLOCK] xmit_block :0x%02x\n", dvobj->xmit_block);
+	if (dvobj->xmit_block & XMIT_BLOCK_REDLMEM)
+		RTW_PRINT_SEL(sel, "Reason:%s\n", "XMIT_BLOCK_REDLMEM");
+	if (dvobj->xmit_block & XMIT_BLOCK_SUSPEND)
+		RTW_PRINT_SEL(sel, "Reason:%s\n", "XMIT_BLOCK_SUSPEND");
+	if (dvobj->xmit_block == XMIT_BLOCK_NONE)
+		RTW_PRINT_SEL(sel, "Reason:%s\n", "XMIT_BLOCK_NONE");
+}
+void dump_xmit_block_info(void *sel, const char *fun_name, _adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	RTW_INFO("\n"ADPT_FMT" call %s\n", ADPT_ARG(padapter), fun_name);
+	dump_xmit_block(sel, padapter);
+}
+#define DBG_XMIT_BLOCK_DUMP(adapter)	dump_xmit_block_info(RTW_DBGDUMP, __func__, adapter)
+#endif
+
+void rtw_set_xmit_block(_adapter *padapter, enum XMIT_BLOCK_REASON reason)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	_rtw_spinlock_bh(&dvobj->xmit_block_lock);
+	dvobj->xmit_block |= reason;
+	_rtw_spinunlock_bh(&dvobj->xmit_block_lock);
+
+	#ifdef DBG_XMIT_BLOCK
+	DBG_XMIT_BLOCK_DUMP(padapter);
+	#endif
+}
+
+void rtw_clr_xmit_block(_adapter *padapter, enum XMIT_BLOCK_REASON reason)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	_rtw_spinlock_bh(&dvobj->xmit_block_lock);
+	dvobj->xmit_block &= ~reason;
+	_rtw_spinunlock_bh(&dvobj->xmit_block_lock);
+
+	#ifdef DBG_XMIT_BLOCK
+	DBG_XMIT_BLOCK_DUMP(padapter);
+	#endif
+}
+bool rtw_is_xmit_blocked(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	#ifdef DBG_XMIT_BLOCK
+	DBG_XMIT_BLOCK_DUMP(padapter);
+	#endif
+	return ((dvobj->xmit_block) ? _TRUE : _FALSE);
+}
+
+bool rtw_xmit_ac_blocked(_adapter *adapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	_adapter *iface;
+	struct mlme_ext_priv *mlmeext;
+	bool blocked = _FALSE;
+	int i;
+
+	if (rfctl->offch_state != OFFCHS_NONE
+		#if CONFIG_DFS
+		|| IS_RADAR_DETECTED(rfctl)
+		#ifdef CONFIG_ECSA_PHL
+		|| rtw_mr_is_ecsa_running(adapter)
+		#endif
+		#endif
+	) {
+		blocked = _TRUE;
+		goto exit;
+	}
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+
+		if (!rtw_iface_at_same_hwband(adapter, iface))
+			continue;
+
+		mlmeext = &iface->mlmeextpriv;
+		/* check scan state */
+		if (mlmeext_scan_state(mlmeext) != SCAN_DISABLE
+			&& mlmeext_scan_state(mlmeext) != SCAN_BACK_OP
+		) {
+			blocked = _TRUE;
+			goto exit;
+		}
+
+		if (mlmeext_scan_state(mlmeext) == SCAN_BACK_OP
+			&& !mlmeext_chk_scan_backop_flags(mlmeext, SS_BACKOP_TX_RESUME)
+		) {
+			blocked = _TRUE;
+			goto exit;
+		}
+	}
+
+exit:
+	return blocked;
+}
+
+#ifdef CONFIG_LAYER2_ROAMING
+/*	dequeuq + xmit the cache skb during the roam procedure	*/
+void dequeuq_roam_pkt(_adapter *padapter)
+{
+	struct xmit_frame *rframe;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	_list *plist = NULL, *phead = NULL;
+
+	if (padapter->mlmepriv.roam_network) {
+		padapter->mlmepriv.roam_network = NULL;
+		_rtw_spinlock_bh(&pxmitpriv->rpkt_queue.lock);
+		phead = get_list_head(&pxmitpriv->rpkt_queue);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			rframe = LIST_CONTAINOR(plist, struct xmit_frame, list);
+			plist = get_next(plist);
+			rtw_list_delete(&rframe->list);
+			core_tx_per_packet(padapter, rframe, &rframe->pkt, NULL);
+		}
+		_rtw_spinunlock_bh(&pxmitpriv->rpkt_queue.lock);
+	}
+}
+#endif
+
+#ifdef CONFIG_TX_AMSDU
+void rtw_amsdu_vo_timeout_handler(void *FunctionContext)
+{
+	_adapter *adapter = (_adapter *)FunctionContext;
+
+	adapter->xmitpriv.amsdu_vo_timeout = RTW_AMSDU_TIMER_TIMEOUT;
+
+	rtw_tasklet_hi_schedule(&adapter->xmitpriv.xmit_tasklet);
+}
+
+void rtw_amsdu_vi_timeout_handler(void *FunctionContext)
+{
+	_adapter *adapter = (_adapter *)FunctionContext;
+
+	adapter->xmitpriv.amsdu_vi_timeout = RTW_AMSDU_TIMER_TIMEOUT;
+
+	rtw_tasklet_hi_schedule(&adapter->xmitpriv.xmit_tasklet);
+}
+
+void rtw_amsdu_be_timeout_handler(void *FunctionContext)
+{
+	_adapter *adapter = (_adapter *)FunctionContext;
+
+	adapter->xmitpriv.amsdu_be_timeout = RTW_AMSDU_TIMER_TIMEOUT;
+
+	rtw_tasklet_hi_schedule(&adapter->xmitpriv.xmit_tasklet);
+}
+
+void rtw_amsdu_bk_timeout_handler(void *FunctionContext)
+{
+	_adapter *adapter = (_adapter *)FunctionContext;
+
+	adapter->xmitpriv.amsdu_bk_timeout = RTW_AMSDU_TIMER_TIMEOUT;
+
+	rtw_tasklet_hi_schedule(&adapter->xmitpriv.xmit_tasklet);
+}
+
+u8 rtw_amsdu_get_timer_status(_adapter *padapter, u8 priority)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	u8 status =  RTW_AMSDU_TIMER_UNSET;
+
+	switch (priority) {
+	case 1:
+	case 2:
+		status = pxmitpriv->amsdu_bk_timeout;
+		break;
+	case 4:
+	case 5:
+		status = pxmitpriv->amsdu_vi_timeout;
+		break;
+	case 6:
+	case 7:
+		status = pxmitpriv->amsdu_vo_timeout;
+		break;
+	case 0:
+	case 3:
+	default:
+		status = pxmitpriv->amsdu_be_timeout;
+		break;
+	}
+	return status;
+}
+
+void rtw_amsdu_set_timer_status(_adapter *padapter, u8 priority, u8 status)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	switch (priority) {
+	case 1:
+	case 2:
+		pxmitpriv->amsdu_bk_timeout = status;
+		break;
+	case 4:
+	case 5:
+		pxmitpriv->amsdu_vi_timeout = status;
+		break;
+	case 6:
+	case 7:
+		pxmitpriv->amsdu_vo_timeout = status;
+		break;
+	case 0:
+	case 3:
+	default:
+		pxmitpriv->amsdu_be_timeout = status;
+		break;
+	}
+}
+
+void rtw_amsdu_set_timer(_adapter *padapter, u8 priority)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	_timer *amsdu_timer = NULL;
+
+	switch (priority) {
+	case 1:
+	case 2:
+		amsdu_timer = &pxmitpriv->amsdu_bk_timer;
+		break;
+	case 4:
+	case 5:
+		amsdu_timer = &pxmitpriv->amsdu_vi_timer;
+		break;
+	case 6:
+	case 7:
+		amsdu_timer = &pxmitpriv->amsdu_vo_timer;
+		break;
+	case 0:
+	case 3:
+	default:
+		amsdu_timer = &pxmitpriv->amsdu_be_timer;
+		break;
+	}
+	_set_timer(amsdu_timer, 1);
+}
+
+void rtw_amsdu_cancel_timer(_adapter *padapter, u8 priority)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	_timer *amsdu_timer = NULL;
+
+	switch (priority) {
+	case 1:
+	case 2:
+		amsdu_timer = &pxmitpriv->amsdu_bk_timer;
+		break;
+	case 4:
+	case 5:
+		amsdu_timer = &pxmitpriv->amsdu_vi_timer;
+		break;
+	case 6:
+	case 7:
+		amsdu_timer = &pxmitpriv->amsdu_vo_timer;
+		break;
+	case 0:
+	case 3:
+	default:
+		amsdu_timer = &pxmitpriv->amsdu_be_timer;
+		break;
+	}
+	_cancel_timer_ex(amsdu_timer);
+}
+#endif /* CONFIG_TX_AMSDU */
+
+
+#ifdef CONFIG_PCI_TX_POLLING
+void rtw_tx_poll_init(_adapter *padapter)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	_timer *timer = &pxmitpriv->tx_poll_timer;
+
+	if (!is_primary_adapter(padapter))
+		return;
+
+	if (timer->function != NULL) {
+		RTW_INFO("tx polling timer has been init.\n");
+		return;
+	}
+
+	rtw_init_timer(timer, rtw_tx_poll_timeout_handler, padapter);
+	rtw_tx_poll_timer_set(padapter, 1);
+	RTW_INFO("Tx poll timer init!\n");
+}
+
+void rtw_tx_poll_timeout_handler(void *FunctionContext)
+{
+	_adapter *adapter = (_adapter *)FunctionContext;
+
+	rtw_tx_poll_timer_set(adapter, 1);
+
+	if (adapter->dvobj->hal_func.tx_poll_handler)
+		adapter->dvobj->hal_func.tx_poll_handler(adapter);
+	else
+		RTW_WARN("hal ops: tx_poll_handler is NULL\n");
+}
+
+void rtw_tx_poll_timer_set(_adapter *padapter, u32 delay)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	_timer *timer = NULL;
+
+	timer = &pxmitpriv->tx_poll_timer;
+	_set_timer(timer, delay);
+}
+
+void rtw_tx_poll_timer_cancel(_adapter *padapter)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	_timer *timer = NULL;
+
+	if (!is_primary_adapter(padapter))
+		return;
+
+	timer = &pxmitpriv->tx_poll_timer;
+	_cancel_timer_ex(timer);
+	timer->function = NULL;
+	RTW_INFO("Tx poll timer cancel !\n");
+}
+#endif /* CONFIG_PCI_TX_POLLING */
+
+void rtw_sctx_init(struct submit_ctx *sctx, int timeout_ms)
+{
+	sctx->timeout_ms = timeout_ms;
+	sctx->submit_time = rtw_get_current_time();
+	_rtw_init_completion(&sctx->done);
+	sctx->status = RTW_SCTX_SUBMITTED;
+	sctx->rsp = NULL;
+}
+
+int rtw_sctx_wait(struct submit_ctx *sctx, const char *msg)
+{
+	int ret = _FAIL;
+	unsigned long expire;
+	int status = 0;
+
+#ifdef PLATFORM_LINUX
+	expire = sctx->timeout_ms ? msecs_to_jiffies(sctx->timeout_ms) : MAX_SCHEDULE_TIMEOUT;
+	if (!_rtw_wait_for_comp_timeout(&sctx->done, expire)) {
+		/* timeout, do something?? */
+		status = RTW_SCTX_DONE_TIMEOUT;
+		RTW_INFO("%s timeout: %s\n", __func__, msg);
+	} else
+		status = sctx->status;
+#endif
+
+	if (status == RTW_SCTX_DONE_SUCCESS)
+		ret = _SUCCESS;
+
+	return ret;
+}
+
+bool rtw_sctx_chk_waring_status(int status)
+{
+	switch (status) {
+	case RTW_SCTX_DONE_UNKNOWN:
+	case RTW_SCTX_DONE_BUF_ALLOC:
+	case RTW_SCTX_DONE_BUF_FREE:
+
+	case RTW_SCTX_DONE_DRV_STOP:
+	case RTW_SCTX_DONE_DEV_REMOVE:
+		return _TRUE;
+	default:
+		return _FALSE;
+	}
+}
+
+void rtw_sctx_done_err(struct submit_ctx **sctx, int status)
+{
+	if (*sctx) {
+		if (rtw_sctx_chk_waring_status(status))
+			RTW_INFO("%s status:%d\n", __func__, status);
+		(*sctx)->status = status;
+#ifdef PLATFORM_LINUX
+		complete(&((*sctx)->done));
+#endif
+		*sctx = NULL;
+	}
+}
+
+void rtw_sctx_done(struct submit_ctx **sctx)
+{
+	rtw_sctx_done_err(sctx, RTW_SCTX_DONE_SUCCESS);
+}
+
+#ifdef CONFIG_XMIT_ACK
+int rtw_ack_tx_wait(struct xmit_priv *pxmitpriv, u32 timeout_ms)
+{
+	struct submit_ctx *pack_tx_ops = &pxmitpriv->ack_tx_ops;
+
+	pack_tx_ops->submit_time = rtw_get_current_time();
+	pack_tx_ops->timeout_ms = timeout_ms;
+	pack_tx_ops->status = RTW_SCTX_SUBMITTED;
+
+	return rtw_sctx_wait(pack_tx_ops, __func__);
+}
+
+void rtw_ack_tx_done(struct xmit_priv *pxmitpriv, int status)
+{
+	struct submit_ctx *pack_tx_ops = &pxmitpriv->ack_tx_ops;
+
+	if (pxmitpriv->ack_tx)
+		rtw_sctx_done_err(&pack_tx_ops, status);
+	else
+		RTW_INFO("%s ack_tx not set\n", __func__);
+}
+#endif /* CONFIG_XMIT_ACK */
+
+#ifdef CONFIG_PCI_HCI
+void rtw_os_query_local_buf(void *priv, struct tx_local_buf *buf)
+{
+	struct dvobj_priv *obj = (struct dvobj_priv *)priv;
+	PPCI_DATA pci_data = dvobj_to_pci(obj);
+	struct pci_dev *pdev = pci_data->ppcidev;
+	dma_addr_t phy_addr;
+	size_t alloc_size = 12 * 1024; /* MPDU maximum size in spec: 12k */
+
+	buf->vir_addr = pci_alloc_cache_mem(pdev, &phy_addr, alloc_size,
+					    DMA_TO_DEVICE);
+	if (!buf->vir_addr)
+		return;
+
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	buf->phy_addr_h =  phy_addr >> 32;
+#else
+	buf->phy_addr_h = 0x0;
+#endif
+	buf->phy_addr_l = phy_addr & 0xFFFFFFFF;
+	buf->buf_max_size = alloc_size;
+	buf->cache = true;
+}
+
+void rtw_os_return_local_buf(void *priv, struct tx_local_buf *buf)
+{
+	struct dvobj_priv *obj = (struct dvobj_priv *)priv;
+	PPCI_DATA pci_data = dvobj_to_pci(obj);
+	struct pci_dev *pdev = pci_data->ppcidev;
+	dma_addr_t phy_addr = buf->phy_addr_l;
+
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	{
+		u64 phy_addr_h = buf->phy_addr_h;
+
+		phy_addr |= (phy_addr_h << 32);
+	}
+#endif
+
+	pci_free_cache_mem(pdev, buf->vir_addr, &phy_addr, buf->buf_max_size,
+			   DMA_TO_DEVICE);
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/rtw_xmit_shortcut.c b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_xmit_shortcut.c
new file mode 100644
index 000000000000..b1a55f84e823
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/rtw_xmit_shortcut.c
@@ -0,0 +1,838 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+
+#ifdef CONFIG_CORE_TXSC
+u8 DBG_PRINT_TXREQ_ONCE;
+void _print_txreq_pklist(struct xmit_frame *pxframe, struct rtw_xmit_req *ptxreq, struct sk_buff *pskb, const char *func)
+{
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	struct rtw_xmit_req *txreq = NULL;
+	u8 pkt_cnt = 0, i;
+
+	if (DBG_PRINT_TXREQ_ONCE == 1) {
+		RTW_PRINT("%s\n", func);
+		RTW_PRINT("[%s] pxframe=%p txreq=%p\n", func, pxframe, ptxreq);
+
+		if (pskb)
+			txsc_dump_data(pskb->data, ETH_HLEN, "ETHHDR");
+
+		if (ptxreq != NULL)
+			txreq = ptxreq;
+		else
+			txreq = pxframe->phl_txreq;
+
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		pkt_cnt = txreq->pkt_cnt;
+
+		RTW_PRINT("os_priv:%p, treq_type:%d, pkt_cnt:%d, total_len:%d, shortcut_id:%d\n\n",
+				txreq->os_priv, txreq->treq_type, txreq->pkt_cnt, txreq->total_len, txreq->shortcut_id);
+
+		for (i = 0; i < pkt_cnt; i++) {
+			RTW_PRINT("pkt_list[%d]\n", i);
+			txsc_dump_data(pkt_list->vir_addr, pkt_list->length, "pkt_list");
+			pkt_list++;
+		}
+		DBG_PRINT_TXREQ_ONCE = 0;
+	}
+}
+
+void txsc_init(_adapter *padapter)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	pxmitpriv->txsc_enable = 1; /* default TXSC on */
+	pxmitpriv->txsc_debug_mode = 0;
+	pxmitpriv->txsc_debug_mask = 0x3;
+}
+
+void txsc_clear(_adapter *padapter)
+{
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct sta_info *psta = NULL;
+	int i;
+	u8 tmp = 0;
+
+	RTW_PRINT("[TXSC] clear txsc entry\n");
+
+	_rtw_spinlock_bh(&pxmitpriv->txsc_lock);
+
+	if (pxmitpriv->txsc_enable) {
+		tmp = pxmitpriv->txsc_enable;
+		pxmitpriv->txsc_enable = 0;
+	}
+
+	pxmitpriv->ptxsc_sta_cached = NULL;
+
+	for (i = 0; i < pstapriv->max_num_sta; i++) {
+		psta = pstapriv->sta_aid[i];
+
+		if (!psta)
+			continue;
+
+		psta->txsc_cache_num = 0;
+		psta->txsc_cur_idx = 0;
+		psta->txsc_cache_idx = 0;
+		psta->txsc_cache_hit = 0;
+		psta->txsc_cache_miss = 0;
+		psta->txsc_path_slow = 0;
+		_rtw_memset(psta->txsc_entry_cache, 0x0, sizeof(struct txsc_entry) * CORE_TXSC_ENTRY_NUM);
+
+	}
+
+	pxmitpriv->txsc_enable = tmp;
+
+	_rtw_spinunlock_bh(&pxmitpriv->txsc_lock);
+}
+
+void txsc_dump(_adapter *padapter)
+{
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct sta_info *psta = NULL;
+	int i, j;
+
+	RTW_PRINT("[txsc][core] (txsc,enable) txsc_enable:%x\n", pxmitpriv->txsc_enable);
+	RTW_PRINT("[txsc][core] (txsc,debug) txsc_debug_mode:%x\n", pxmitpriv->txsc_debug_mode);
+
+	RTW_PRINT("[txsc][core] txsc_phl_err_cnt1:%d\n", pxmitpriv->txsc_phl_err_cnt1);
+	RTW_PRINT("[txsc][core] txsc_phl_err_cnt2:%d\n", pxmitpriv->txsc_phl_err_cnt2);
+	RTW_PRINT("\n");
+	for (i = 0; i < pstapriv->max_num_sta; i++) {
+		psta = pstapriv->sta_aid[i];
+		if (!psta)
+			continue;
+		RTW_PRINT("[%d] STA["MAC_FMT"]\n", i,
+			MAC_ARG(psta->phl_sta->mac_addr));
+
+		RTW_PRINT("[txsc] cur_idx:%d\n", psta->txsc_cur_idx);
+		RTW_PRINT("[txsc][core] txsc_path_slow:%d\n", psta->txsc_path_slow);
+		RTW_PRINT("[txsc][core] txsc_path_ps:%d\n", psta->txsc_path_ps);
+		RTW_PRINT("[txsc][core] txsc_cache_hit:%d\n", psta->txsc_cache_hit);
+		RTW_PRINT("[txsc][core] txsc_cache_miss:%d\n", psta->txsc_cache_miss);
+		RTW_PRINT("\n");
+		for (j = 0 ; j < CORE_TXSC_ENTRY_NUM; j++) {
+			if (!psta->txsc_entry_cache[j].txsc_is_used)
+				continue;
+
+			RTW_PRINT(" [%d][txsc][core] txsc_core_hit:%d\n", j, psta->txsc_entry_cache[j].txsc_cache_hit);
+			#ifdef CONFIG_PHL_TXSC
+			RTW_PRINT(" [%d][txsc][phl]  txsc_phl_hit:%d\n", j, psta->phl_sta->phl_txsc[j].txsc_cache_hit);
+			#endif
+			RTW_PRINT("\n");
+		}
+	}
+}
+
+void txsc_dump_data(u8 *buf, u16 buf_len, const char *prefix)
+{
+	int i = 0, j;
+
+	RTW_PRINT("[txsc_dump] [%s (%uB)]@%p:\n", prefix, buf_len, buf);
+
+	if (buf == NULL) {
+		RTW_PRINT("[txsc_dump] NULL!\n");
+		return;
+	}
+
+	while (i < buf_len) {
+		RTW_PRINT("[txsc_dump] %04X -", i);
+		for (j = 0; (j < 4) && (i < buf_len); j++, i += 4)
+			RTW_PRINT("  %02X %02X %02X %02X", buf[i], buf[i+1], buf[i+2], buf[i+3]);
+		RTW_PRINT("\n");
+	}
+}
+
+#ifdef CONFIG_PCI_HCI
+void txsc_recycle_txreq_phyaddr(_adapter *padapter, struct rtw_xmit_req *txreq)
+{
+	PPCI_DATA pci_data = dvobj_to_pci(padapter->dvobj);
+	struct pci_dev *pdev = pci_data->ppcidev;
+	struct rtw_pkt_buf_list *pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+	dma_addr_t phy_addr = 0;
+
+	/* only recycle pkt_list[1] = skb->data for SW TXSC */
+	pkt_list++;
+
+	phy_addr = (pkt_list->phy_addr_l);
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	phy_addr |= ((u64)pkt_list->phy_addr_h << 32);
+#endif
+	pci_unmap_bus_addr(pdev, &phy_addr, pkt_list->length, DMA_TO_DEVICE);
+}
+
+void txsc_fill_txreq_phyaddr(_adapter *padapter, struct rtw_pkt_buf_list *pkt_list)
+{
+	PPCI_DATA pci_data = dvobj_to_pci(padapter->dvobj);
+	struct pci_dev *pdev = pci_data->ppcidev;
+	dma_addr_t phy_addr = 0;
+
+	pci_get_bus_addr(pdev, pkt_list->vir_addr, &phy_addr, pkt_list->length, DMA_TO_DEVICE);
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	pkt_list->phy_addr_h =  phy_addr >> 32;
+#else
+	pkt_list->phy_addr_h = 0x0;
+#endif
+	pkt_list->phy_addr_l = phy_addr & 0xFFFFFFFF;
+}
+#endif
+
+static void txsc_init_pkt_entry(_adapter *padapter, struct sk_buff *pskb, struct txsc_pkt_entry *txsc_pkt)
+{
+	u8 priority = 0, i;
+
+	txsc_pkt->step = TXSC_NONE;
+	txsc_pkt->txsc_id = 0xff;
+	txsc_pkt->ptxreq = NULL;
+
+	for (i = 0; i < MAX_TXSC_SKB_NUM; i++)
+		txsc_pkt->xmit_skb[i] = NULL;
+
+	txsc_pkt->psta = NULL;
+	txsc_pkt->xmit_skb[0] = pskb;
+	txsc_pkt->skb_cnt = 1;
+
+	priority = *(pskb->data + ETH_HLEN + 1);
+	txsc_pkt->priority = tos_to_up(priority);
+}
+
+static void txsc_add_sc_check(_adapter *padapter, struct xmit_frame *pxframe, struct txsc_pkt_entry *txsc_pkt)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct pkt_attrib *pattrib = &pxframe->attrib;
+	struct sta_info *psta = pxframe->attrib.psta;
+
+	if (!pxmitpriv->txsc_enable) {
+		if (psta->txsc_cache_num > 0)
+			txsc_pkt->step = TXSC_SKIP;
+		else
+			txsc_pkt->step = TXSC_NONE;
+		goto exit;
+	}
+
+	if (pxframe->attrib.nr_frags > 1 || pxframe->attrib.bswenc == 1)
+		goto exit;
+
+	if (txsc_pkt->step != TXSC_NONE)
+		goto exit;
+
+	if (pattrib->qos_en &&
+		pattrib->ampdu_en == 1 &&
+		pattrib->ether_type == ETH_P_IP &&
+		!IS_MCAST(pattrib->ra) &&
+		!pattrib->icmp_pkt &&
+		!pattrib->dhcp_pkt) {
+
+		RTW_PRINT("[%s] sta[%02x] add eth_type=0x%x pkt to txsc\n",
+			__func__, pattrib->psta->phl_sta->mac_addr[5], pattrib->ether_type);
+
+		txsc_pkt->step = TXSC_ADD;
+	}
+
+exit:
+	return;
+}
+
+static u8 txsc_get_sc_entry(_adapter *padapter, struct sk_buff *pskb, struct txsc_pkt_entry *txsc_pkt)
+{
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct sta_info *psta = NULL;
+	u8 *ptxsc_ethdr = NULL;
+	u8 i, res, da[6], offset, res2 = 0, sta_hit = 0;
+
+	res = _FAIL;
+	offset = 6;
+
+	if (!pxmitpriv->txsc_enable)
+		return res;
+
+	if (pxmitpriv->ptxsc_sta_cached) {
+		if (pxmitpriv->ptxsc_sta_cached->phl_sta)
+			sta_hit = _rtw_memcmp(pxmitpriv->ptxsc_sta_cached->phl_sta->mac_addr, pskb->data, 6);
+
+		if (sta_hit)
+			psta = pxmitpriv->ptxsc_sta_cached;
+	}
+
+	if (!sta_hit) {
+		_rtw_memcpy(da, pskb->data, 6);
+		if (IS_MCAST(da)) {
+			res = _FAIL;
+		} else {
+			if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
+				psta = rtw_get_stainfo(pstapriv, da);
+			else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
+				psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+
+			if (!psta)
+				res = _FAIL;
+		}
+	}
+
+	if (!psta) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	/* skip power saving mode */
+	if (psta->state & WIFI_SLEEP_STATE) {
+		res = _FAIL;
+		txsc_pkt->step = TXSC_SKIP;
+		psta->txsc_path_ps++;
+		goto exit;
+	}
+
+
+	if (psta->txsc_cache_num == 0) {
+		res = _FAIL;
+		goto exit;
+	}
+
+	pxmitpriv->ptxsc_sta_cached = psta;
+	txsc_pkt->step = TXSC_NONE;
+	ptxsc_ethdr = (u8 *)&psta->txsc_entry_cache[psta->txsc_cache_idx].txsc_ethdr;
+	res2 = _rtw_memcmp((pskb->data + offset), (ptxsc_ethdr + offset), (ETH_HLEN - offset));
+
+	if (res2 &&
+		(pskb->len <= psta->txsc_entry_cache[psta->txsc_cache_idx].txsc_frag_len)) {
+
+		txsc_pkt->step = TXSC_APPLY;
+		txsc_pkt->psta = psta;
+		txsc_pkt->txsc_id = psta->txsc_cache_idx;
+
+		res = _SUCCESS;
+	} else {
+
+		for (i = 0; i < CORE_TXSC_ENTRY_NUM; i++) {
+			if (i != psta->txsc_cache_idx && psta->txsc_entry_cache[i].txsc_is_used) {
+
+				ptxsc_ethdr = (u8 *)&(psta->txsc_entry_cache[i].txsc_ethdr);
+				if (_rtw_memcmp((pskb->data + offset), (ptxsc_ethdr + offset), (ETH_HLEN - offset)) &&
+					(pskb->len <= psta->txsc_entry_cache[i].txsc_frag_len)) {
+
+					txsc_pkt->step = TXSC_APPLY;
+					txsc_pkt->txsc_id = i;
+					txsc_pkt->psta = psta;
+					psta->txsc_cache_idx = i;
+					res = _SUCCESS;
+
+					break;
+				}
+			}
+		}
+	}
+
+	if ((res == _SUCCESS) && (pxmitpriv->txsc_debug_mode == 1)) {
+		txsc_pkt->step = TXSC_DEBUG;
+		res = _FAIL;
+	}
+
+exit:
+	return res;
+}
+
+static void txsc_prepare_sc_entry(_adapter *padapter, struct xmit_frame *pxframe, struct txsc_pkt_entry *txsc_pkt)
+{
+	struct sta_priv	*pstapriv = &padapter->stapriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct sta_info *psta = pxframe->attrib.psta;
+	struct pkt_attrib *pattrib = &pxframe->attrib;
+	struct rtw_xmit_req *txreq = pxframe->phl_txreq;
+	struct rtw_pkt_buf_list *pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+	struct rtw_pkt_buf_list *ptxsc_pkt_list0 = NULL;
+	struct rtw_t_meta_data *ptxsc_mdata = NULL;
+	struct sk_buff *pskb = txsc_pkt->xmit_skb[0];
+	u8 i, idx;
+	u8 *ptxsc_ethdr = NULL;
+	u8 *ptxsc_wlhdr = NULL;
+	u8 *ptxsc_wlhdr_len = NULL;
+
+	if (!psta) {
+		RTW_ERR("%s: fetal err, XF_STA = NULL, please check.\n", __func__);
+		return;
+	}
+
+	if (txsc_pkt->step == TXSC_SKIP) {
+		if (pxmitpriv->txsc_enable && psta->txsc_cache_num > 0)
+			psta->txsc_cache_miss++;
+		else
+			psta->txsc_path_slow++;
+	} else if (txsc_pkt->step == TXSC_NONE) {
+		psta->txsc_path_slow++;
+	}
+
+	if (txsc_pkt->step != TXSC_ADD)
+		return;
+
+	idx = psta->txsc_cur_idx;
+
+	ptxsc_ethdr = (u8 *)&psta->txsc_entry_cache[idx].txsc_ethdr;
+	#ifdef USE_ONE_WLHDR
+	ptxsc_wlhdr = psta->txsc_entry_cache[idx].txsc_wlhdr;
+	#else
+	ptxsc_wlhdr = (u8 *)&psta->txsc_entry_cache[idx].txsc_wlhdr;
+	#endif
+	ptxsc_pkt_list0 = &psta->txsc_entry_cache[idx].txsc_pkt_list0;
+	ptxsc_mdata = &psta->txsc_entry_cache[idx].txsc_mdata;
+	ptxsc_wlhdr_len = &psta->txsc_entry_cache[idx].txsc_wlhdr_len;
+
+	_rtw_spinlock_bh(&pxmitpriv->txsc_lock);
+
+	if (psta->txsc_entry_cache[idx].txsc_is_used == 1)
+		RTW_PRINT("[CORE_TXSC] txsc entry is full, replace rentry[%d]\n", idx);
+
+	/* ALLOC WLHDR in DMA addr */
+	#ifdef USE_ONE_WLHDR
+	if (!ptxsc_wlhdr) {
+		ptxsc_wlhdr = rtw_zmalloc(CORE_TXSC_WLHDR_SIZE);
+		psta->txsc_entry_cache[idx].txsc_wlhdr = ptxsc_wlhdr;
+	}
+	#endif
+
+	/* ETH HDR */
+	_rtw_memcpy(ptxsc_ethdr, pskb->data, ETH_HLEN);
+
+	/* WLAN HDR + LLC */
+	_rtw_memcpy(ptxsc_wlhdr, pkt_list->vir_addr, pkt_list->length);
+	*ptxsc_wlhdr_len = pkt_list->length;
+
+	/* pkt_list[0] */
+	ptxsc_pkt_list0->vir_addr = ptxsc_wlhdr;
+	ptxsc_pkt_list0->length = pkt_list->length;
+	#ifdef CONFIG_PCI_HCI
+	txsc_fill_txreq_phyaddr(padapter, ptxsc_pkt_list0);
+	#endif
+
+	/* META DATA */
+	_rtw_memcpy(ptxsc_mdata, &txreq->mdata, sizeof(*ptxsc_mdata));
+
+	/* FRAGE_LEN */
+	psta->txsc_entry_cache[idx].txsc_frag_len = pxframe->attrib.frag_len_txsc;
+
+	psta->txsc_entry_cache[idx].txsc_is_used = 1;
+	psta->txsc_cache_idx = idx;
+	psta->txsc_cur_idx = (psta->txsc_cur_idx + 1) % CORE_TXSC_ENTRY_NUM;
+	if (psta->txsc_cache_num < CORE_TXSC_ENTRY_NUM)
+		psta->txsc_cache_num++;
+
+	psta->txsc_path_slow++;
+
+	pxmitpriv->ptxsc_sta_cached = psta;
+
+	txreq->treq_type = RTW_PHL_TREQ_TYPE_PHL_ADD_TXSC | RTW_PHL_TREQ_TYPE_PHL_UPDATE_TXSC;
+
+	/* set shortcut id  */
+	txreq->shortcut_id = idx;
+	psta->txsc_entry_cache[idx].txsc_phl_id = idx;
+
+	/* disable phl txsc  */
+	if (!(pxmitpriv->txsc_debug_mask&BIT1))
+		txreq->shortcut_id = 0;
+
+	RTW_PRINT("[CORE_TXSC][ADD] core_txsc_idx:%d(cur_idx:%d), txreq_sc_id:%d, txsc_frag_len:%d\n",
+		idx, psta->txsc_cur_idx, txreq->shortcut_id, psta->txsc_entry_cache[idx].txsc_frag_len);
+
+	/* for debug */
+	dbg_dump_txreq_mdata(ptxsc_mdata, __func__);
+
+	_rtw_spinunlock_bh(&pxmitpriv->txsc_lock);
+}
+
+u8 txsc_get_sc_cached_entry(_adapter *padapter, struct sk_buff *pskb, struct txsc_pkt_entry *txsc_pkt)
+{
+	txsc_init_pkt_entry(padapter, pskb, txsc_pkt);
+	return txsc_get_sc_entry(padapter, pskb, txsc_pkt);
+}
+
+void txsc_add_sc_cache_entry(_adapter *padapter, struct xmit_frame *pxframe, struct txsc_pkt_entry *txsc_pkt)
+{
+	txsc_add_sc_check(padapter, pxframe, txsc_pkt);
+	txsc_prepare_sc_entry(padapter, pxframe, txsc_pkt);
+}
+
+u8 txsc_apply_sc_cached_entry(_adapter *padapter, struct txsc_pkt_entry *txsc_pkt)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct rtw_pkt_buf_list	*pkt_list = NULL;
+	struct rtw_pkt_buf_list *pkt_list0 = NULL;
+	struct rtw_t_meta_data	*mdata = NULL;
+	struct rtw_xmit_req	*txreq = NULL;
+	struct xmit_txreq_buf *txreq_buf = NULL;
+	struct txsc_entry *txsc;
+	struct sta_info *psta = txsc_pkt->psta;
+	struct sk_buff *xmit_skb[MAX_TXSC_SKB_NUM];
+	u8 *hdr = NULL;
+	u16 *qc;
+	int wlhdr_copy_len, payload_offset = ETH_HLEN;
+	u8 idx, priority = 0, res = _SUCCESS;
+	u8 i, skb_cnt = 0, is_hdr_need_update = 0, txsc_wlhdr_len;
+	#ifndef USE_ONE_WLHDR
+	u8 *head, *ptxsc_wlhdr;
+	#endif
+
+	if (!pxmitpriv->txsc_enable || !psta)
+		return _SUCCESS;
+
+	if (txsc_pkt->step != TXSC_APPLY &&
+		txsc_pkt->step != TXSC_AMSDU_APPLY &&
+		txsc_pkt->step != TXSC_DEBUG)
+		return _SUCCESS;
+
+	_rtw_memset(xmit_skb, 0x0, sizeof(xmit_skb));
+
+	/* get cached entry */
+	idx = txsc_pkt->txsc_id;
+	txsc = &psta->txsc_entry_cache[idx];
+
+	pkt_list0 = &txsc->txsc_pkt_list0;
+	mdata = &txsc->txsc_mdata;
+	#ifdef USE_ONE_WLHDR
+	txsc_wlhdr_len = pkt_list0->length;
+	#else
+	ptxsc_wlhdr = (u8 *)txsc->txsc_wlhdr;
+	txsc_wlhdr_len = txsc->txsc_wlhdr_len;
+	#endif
+
+	if (txsc_pkt->step == TXSC_DEBUG) {
+		_rtw_memset(psta->debug_buf, 0x0, (sizeof(struct rtw_xmit_req)+sizeof(struct rtw_pkt_buf_list)*2));
+		txreq = (struct rtw_xmit_req *)psta->debug_buf;
+		pkt_list = (struct rtw_pkt_buf_list *)psta->debug_buf + sizeof(struct rtw_xmit_req);
+	} else {
+		#ifdef USE_ONE_WLHDR
+		txreq_buf = (struct xmit_txreq_buf *)get_txreq_buffer(padapter, (u8 **)&txreq, (u8 **)&pkt_list, NULL, NULL);
+		#else
+		txreq_buf = (struct xmit_txreq_buf *)get_txreq_buffer(padapter, (u8 **)&txreq, (u8 **)&pkt_list, (u8 **)&head, NULL);
+		#endif
+
+		if (txreq_buf == NULL) {
+			res = _FAIL;
+			goto exit;
+		}
+	}
+
+	/* init txreq buf */
+	txreq_buf->adapter = padapter;
+
+	/* fill txreq other */
+	txreq->os_priv = (void *)txreq_buf;
+	txreq->pkt_list = (u8 *)pkt_list;
+	txreq->pkt_cnt = 0;
+	txreq->total_len = 0;
+
+	/* fill txsc pkt entry */
+	txsc_pkt->ptxreq = txreq;
+
+	{
+		/* for no tx_amsdu case */
+		xmit_skb[0] = txsc_pkt->xmit_skb[0];
+		skb_cnt = txsc_pkt->skb_cnt;
+	}
+
+	if (skb_cnt == 0)
+		RTW_PRINT("[ERR][%s:%d] skb_cnt = 0 is a fatel error, plz check\n", __func__, __LINE__);
+
+	/* fill_txreq_mdata */
+	_rtw_memcpy(&txreq->mdata, mdata, sizeof(txreq->mdata));
+
+	/* Update TID from IP header */
+	/* priority = *(xmit_skb[0]->data + ETH_HLEN + 1); */
+	/*txreq->mdata.tid = tos_to_up(priority); */
+	priority = txreq->mdata.tid = txsc_pkt->priority;
+	txreq->mdata.cat = rtw_phl_cvt_tid_to_cat(priority);
+
+	/* SW shortcut --- */
+	/* rtw_core_wlan_fill_head */
+	/* use swseq in amsdu */
+	if (txreq->mdata.hw_seq_mode == 0) {
+		/* generate sw seq */
+		//priority = txreq->mdata.tid;
+		psta->sta_xmitpriv.txseq_tid[priority]++;
+		psta->sta_xmitpriv.txseq_tid[priority] &= 0xFFF;
+		txreq->mdata.sw_seq = psta->sta_xmitpriv.txseq_tid[priority];
+
+		hdr = txsc->txsc_wlhdr;
+		SetSeqNum(hdr, txreq->mdata.sw_seq);
+
+		is_hdr_need_update = 1;
+	}
+
+	wlhdr_copy_len = txsc_wlhdr_len;
+
+	/* WLAN header from cache */
+	#ifdef USE_ONE_WLHDR
+	_rtw_memcpy(pkt_list, pkt_list0, sizeof(struct rtw_pkt_buf_list));
+	#else
+	#ifdef USE_PREV_WLHDR_BUF
+	if (txreq_buf->macid == txreq->mdata.macid && txreq_buf->txsc_id == idx) {
+		if (is_hdr_need_update)
+			SetSeqNum(head, txreq->mdata.sw_seq);/* set sw seq */
+	} else
+	#endif /* USE_PREV_WLHDR_BUF */
+	{
+		_rtw_memcpy(head, ptxsc_wlhdr, wlhdr_copy_len);
+	}
+
+	/* fill wlhdr in pkt_list[0] */
+	pkt_list->vir_addr = head;
+	pkt_list->length = wlhdr_copy_len;
+	#ifdef CONFIG_PCI_HCI
+	txsc_fill_txreq_phyaddr(padapter, pkt_list);
+	#endif
+	#endif/* USE_ONE_WLHDR */
+
+	txreq->total_len += pkt_list->length;
+	txreq->pkt_cnt++;
+
+	#ifdef USE_PREV_WLHDR_BUF
+	txreq_buf->macid = txreq->mdata.macid;
+	txreq_buf->txsc_id = idx;
+	#endif/* USE_PREV_WLHDR_BUF */
+
+	/* Payload w.o. ether header */ /* CONFIG_TXSC_AMSDU for multiple skb */
+	for (i = 0; i < skb_cnt; i++) {
+		pkt_list++;
+
+		pkt_list->vir_addr = xmit_skb[i]->data + payload_offset;
+		pkt_list->length = xmit_skb[i]->len - payload_offset;
+
+		txreq->total_len += pkt_list->length;
+		txreq->pkt_cnt++;
+		#ifdef CONFIG_PCI_HCI
+		txsc_fill_txreq_phyaddr(padapter, pkt_list);
+		#endif
+
+		txreq_buf->pkt[i] = (u8 *)xmit_skb[i];
+	}
+
+	txreq->treq_type = RTW_PHL_TREQ_TYPE_CORE_TXSC;
+
+	if (txsc_pkt->step == TXSC_APPLY) {
+		psta->txsc_cache_hit++;
+		txsc->txsc_cache_hit++;
+	} else
+		psta->txsc_path_slow++;
+	/* SW shortcut --- */
+
+	txreq_buf->pkt_cnt = skb_cnt;/* for recycle multiple skb */
+	txreq->mdata.pktlen = txreq->total_len;
+	txreq->shortcut_id = psta->txsc_entry_cache[idx].txsc_phl_id;
+
+	if (!(pxmitpriv->txsc_debug_mask&BIT1))
+		txreq->shortcut_id = 0;
+
+	/* send addbareq */
+	txsc_issue_addbareq_cmd(padapter, priority, psta, _TRUE);
+
+#ifdef RTW_PHL_DBG_CMD
+	/* Update force rate settings so force rate takes effects
+	 * after shortcut cached
+	 */
+	if (padapter->txForce_enable) {
+		if (padapter->txForce_rate != INV_TXFORCE_VAL) {
+			txreq->mdata.f_rate = padapter->txForce_rate;
+			txreq->mdata.userate_sel = 1;
+			txreq->mdata.dis_data_rate_fb = 1;
+			txreq->mdata.dis_rts_rate_fb = 1;
+		}
+		if (padapter->txForce_agg != INV_TXFORCE_VAL)
+			txreq->mdata.ampdu_en = padapter->txForce_agg;
+		if (padapter->txForce_aggnum != INV_TXFORCE_VAL)
+			txreq->mdata.max_agg_num = padapter->txForce_aggnum;
+		if (padapter->txForce_gi != INV_TXFORCE_VAL)
+			txreq->mdata.f_gi_ltf = padapter->txForce_gi;
+		if (padapter->txForce_ampdu_density != INV_TXFORCE_VAL)
+			txreq->mdata.ampdu_density = padapter->txForce_ampdu_density;
+		if (padapter->txForce_bw != INV_TXFORCE_VAL)
+			txreq->mdata.f_bw = padapter->txForce_bw;
+	}
+#endif /* RTW_PHL_DBG_CMD */
+
+	/* for tx debug */
+	dbg_dump_txreq_mdata(&txreq->mdata, __func__);
+	_print_txreq_pklist(NULL, txsc_pkt->ptxreq, xmit_skb[0], __func__);
+
+exit:
+	return res;
+}
+
+void txsc_free_txreq(_adapter *padapter, struct rtw_xmit_req *txreq)
+{
+	struct xmit_txreq_buf *ptxreq_buf = NULL;
+	struct sk_buff *tx_skb = NULL;
+	_queue *queue = NULL;
+	u8 i;
+
+	if (txreq != NULL)
+		ptxreq_buf = txreq->os_priv;
+	else
+		return;
+
+	#ifdef CONFIG_PCI_HCI
+	#ifdef USE_ONE_WLHDR
+	txsc_recycle_txreq_phyaddr(padapter, txreq);
+	#else
+	core_recycle_txreq_phyaddr(padapter, txreq);
+	#endif
+	#endif
+
+	if (!ptxreq_buf) {
+		RTW_ERR("%s: NULL ptxreq_buf !!\n", __func__);
+		rtw_warn_on(1);
+		return;
+	}
+
+	queue = &padapter->free_txreq_queue;
+	_rtw_spinlock_bh(&queue->lock);
+
+	rtw_list_delete(&ptxreq_buf->list);
+	rtw_list_insert_tail(&ptxreq_buf->list, get_list_head(queue));
+
+	txreq->os_priv = NULL;
+	txreq->pkt_list = NULL;
+	txreq->treq_type = RTW_PHL_TREQ_TYPE_NORMAL;
+
+	/* this must be protected in  spinlock section */
+	if (ptxreq_buf->pkt_cnt == 0)
+		RTW_PRINT("[ERR][%s:%d] pkt_cnt = 0 is a fatel error, plz check\n", __func__, __LINE__);
+
+	for (i = 0; i < ptxreq_buf->pkt_cnt; i++) {
+		tx_skb = (struct sk_buff *)ptxreq_buf->pkt[i];
+		if (tx_skb)
+			rtw_os_pkt_complete(padapter, tx_skb);
+		else
+			RTW_DBG("%s:tx recyele: tx_skb=NULL\n", __func__);
+		ptxreq_buf->pkt[i] = NULL;
+	}
+	padapter->free_txreq_cnt++;
+
+	_rtw_spinunlock_bh(&queue->lock);
+}
+
+void txsc_debug_sc_entry(_adapter *padapter, struct xmit_frame *pxframe, struct txsc_pkt_entry *txsc_pkt)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct rtw_pkt_buf_list *pkt_list = NULL;
+	struct rtw_pkt_buf_list *txsc_pkt_list = NULL;
+	struct rtw_xmit_req *txreq = pxframe->phl_txreq;
+	struct rtw_xmit_req *ptxsc_txreq = txsc_pkt->ptxreq;
+	u8 i;
+
+	if (txsc_pkt->step != TXSC_DEBUG)
+		return;
+
+	if (pxmitpriv->txsc_debug_mask&BIT2) {
+
+		RTW_PRINT("\n\nNormal Path TXREQ: %p\n\n", txreq);
+		txsc_dump_data((u8 *)txreq, sizeof(struct rtw_xmit_req), "txreq:");
+		RTW_PRINT("os_priv=%p\n", txreq->os_priv);
+		RTW_PRINT("treq_type=%d shortcut_id=%d\n", txreq->treq_type, txreq->shortcut_id);
+		RTW_PRINT("total_len=%d pkt_cnt=%d\n", txreq->total_len, txreq->pkt_cnt);
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		for (i = 0; i < txreq->pkt_cnt; i++) {
+			RTW_PRINT("[%d]", i);
+			txsc_dump_data((u8 *)pkt_list, sizeof(struct rtw_pkt_buf_list), "pklist");
+			txsc_dump_data((u8 *)pkt_list->vir_addr, pkt_list->length, "pkt_list->vir_addr");
+			pkt_list++;
+		}
+		RTW_PRINT("mdata: pktlen=%d sw_seq=%d\n", txreq->mdata.pktlen, txreq->mdata.sw_seq);
+
+		RTW_PRINT("\n\nShortcut Path TXREQ: %p\n\n", ptxsc_txreq);
+		txsc_dump_data((u8 *)ptxsc_txreq, sizeof(struct rtw_xmit_req), "ptxsc_txreq:");
+		RTW_PRINT("os_priv=%p\n", ptxsc_txreq->os_priv);
+		RTW_PRINT("treq_type=%d shortcut_id=%d\n", ptxsc_txreq->treq_type, ptxsc_txreq->shortcut_id);
+		RTW_PRINT("total_len=%d pkt_cnt=%d\n", ptxsc_txreq->total_len, ptxsc_txreq->pkt_cnt);
+		txsc_pkt_list = (struct rtw_pkt_buf_list *)ptxsc_txreq->pkt_list;
+		for (i = 0; i < txreq->pkt_cnt; i++) {
+			RTW_PRINT("[%d]", i);
+			txsc_dump_data((u8 *)txsc_pkt_list, sizeof(struct rtw_pkt_buf_list), "pklist");
+			txsc_dump_data((u8 *)txsc_pkt_list->vir_addr, txsc_pkt_list->length, "pkt_list->vir_addr");
+			txsc_pkt_list++;
+		}
+		RTW_PRINT("mdata: pktlen=%d sw_seq=%d\n", ptxsc_txreq->mdata.pktlen, ptxsc_txreq->mdata.sw_seq);
+
+	} else {
+		if (!_rtw_memcmp(&txreq->mdata, &ptxsc_txreq->mdata, sizeof(struct rtw_t_meta_data))) {
+			txsc_dump_data((u8 *)&txreq->mdata, sizeof(struct rtw_t_meta_data), "txreq->mdata");
+			txsc_dump_data((u8 *)&ptxsc_txreq->mdata, sizeof(struct rtw_t_meta_data), "ptxsc_txreq->mdata");
+		}
+
+		pkt_list = (struct rtw_pkt_buf_list *)txreq->pkt_list;
+		txsc_pkt_list = (struct rtw_pkt_buf_list *)ptxsc_txreq->pkt_list;
+		if (pkt_list->length != txsc_pkt_list->length) {
+			txsc_dump_data((u8 *)pkt_list, sizeof(struct rtw_pkt_buf_list), "pkt_list[0]");
+			txsc_dump_data((u8 *)txsc_pkt_list, sizeof(struct rtw_pkt_buf_list), "txsc_pkt_list[0]");
+			txsc_dump_data((u8 *)pkt_list->vir_addr, pkt_list->length, "pkt_list[0]->vir_addr");
+			txsc_dump_data(txsc_pkt_list->vir_addr, pkt_list->length, "txsc_pkt_list[0]->vir_addr");
+		} else if (pkt_list->length == txsc_pkt_list->length) {
+			if (!_rtw_memcmp(pkt_list->vir_addr, txsc_pkt_list->vir_addr, pkt_list->length)) {
+				txsc_dump_data((u8 *)pkt_list->vir_addr, pkt_list->length, "pkt_list[0]->vir_addr");
+				txsc_dump_data((u8 *)txsc_pkt_list->vir_addr, pkt_list->length, "txsc_pkt_list[0]->vir_addr");
+			}
+		}
+	}
+
+	/* DO NOT WD CACHE */
+	txreq->shortcut_id = 0;
+	txreq->treq_type = RTW_PHL_TREQ_TYPE_NORMAL;
+}
+
+#ifdef CONFIG_80211N_HT
+static u8 txsc_issue_addbareq_check(_adapter *padapter, u8 issue_when_busy)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct registry_priv *pregistry = &padapter->registrypriv;
+
+	if (pregistry->tx_quick_addba_req == 0) {
+		if ((issue_when_busy == _TRUE) && (pmlmepriv->LinkDetectInfo.bBusyTraffic == _FALSE))
+			return _FALSE;
+
+		if (pmlmepriv->LinkDetectInfo.NumTxOkInPeriod < 100)
+			return _FALSE;
+	}
+
+	return _TRUE;
+}
+
+void txsc_issue_addbareq_cmd(_adapter *padapter, u8 priority, struct sta_info *psta, u8 issue_when_busy)
+{
+	u8 issued;
+	struct ht_priv	*phtpriv;
+	struct ampdu_priv *ampdu_priv;
+
+	if (txsc_issue_addbareq_check(padapter, issue_when_busy) == _FALSE)
+		return;
+
+	phtpriv = &psta->htpriv;
+	ampdu_priv = &psta->ampdu_priv;
+
+	if ((phtpriv->ht_option == _TRUE) && (ampdu_priv->ampdu_enable == _TRUE)) {
+		issued = (ampdu_priv->agg_enable_bitmap >> priority) & 0x1;
+		issued |= (ampdu_priv->candidate_tid_bitmap >> priority) & 0x1;
+
+		if (issued == 0) {
+			RTW_INFO("rtw_issue_addbareq_cmd, p=%d\n", priority);
+			psta->ampdu_priv.candidate_tid_bitmap |= BIT((u8)priority);
+			rtw_addbareq_cmd(padapter, (u8) priority, psta->phl_sta->mac_addr);
+		}
+	}
+
+}
+#endif /* CONFIG_80211N_HT */
+#endif /* CONFIG_CORE_TXSC */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/wds/rtw_wds.c b/drivers/net/wireless/realtek/rtl8852bs/core/wds/rtw_wds.c
new file mode 100644
index 000000000000..929b1bc00add
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/wds/rtw_wds.c
@@ -0,0 +1,786 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_WDS_C_
+
+#include <drv_types.h>
+
+#if defined(CONFIG_RTW_WDS)
+#include <linux/jhash.h>
+
+#if defined(CONFIG_AP_MODE)
+
+#ifdef PLATFORM_LINUX
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+static void rtw_wpath_free_rcu(struct rtw_wds_path *wpath)
+{
+	kfree_rcu(wpath, rcu);
+	rtw_mstat_update(MSTAT_TYPE_PHY, MSTAT_FREE, sizeof(struct rtw_wds_path));
+}
+#else
+static void rtw_wpath_free_rcu_callback(rtw_rcu_head *head)
+{
+	struct rtw_wds_path *wpath;
+
+	wpath = container_of(head, struct rtw_wds_path, rcu);
+	rtw_mfree(wpath, sizeof(struct rtw_wds_path));
+}
+
+static void rtw_wpath_free_rcu(struct rtw_wds_path *wpath)
+{
+	call_rcu(&wpath->rcu, rtw_wpath_free_rcu_callback);
+}
+#endif
+#endif /* PLATFORM_LINUX */
+
+static void rtw_wds_path_free_rcu(struct rtw_wds_table *tbl, struct rtw_wds_path *wpath);
+
+static u32 rtw_wds_table_hash(const void *addr, u32 len, u32 seed)
+{
+	/* Use last four bytes of hw addr as hash index */
+	return jhash_1word(*(u32 *)(addr+2), seed);
+}
+
+static const rtw_rhashtable_params rtw_wds_rht_params = {
+	.nelem_hint = 2,
+	.automatic_shrinking = true,
+	.key_len = ETH_ALEN,
+	.key_offset = offsetof(struct rtw_wds_path, dst),
+	.head_offset = offsetof(struct rtw_wds_path, rhash),
+	.hashfn = rtw_wds_table_hash,
+};
+
+static void rtw_wds_path_rht_free(void *ptr, void *tblptr)
+{
+	struct rtw_wds_path *wpath = ptr;
+	struct rtw_wds_table *tbl = tblptr;
+
+	rtw_wds_path_free_rcu(tbl, wpath);
+}
+
+static struct rtw_wds_table *rtw_wds_table_alloc(void)
+{
+	struct rtw_wds_table *newtbl;
+
+	newtbl = rtw_malloc(sizeof(struct rtw_wds_table));
+	if (!newtbl)
+		return NULL;
+
+	return newtbl;
+}
+
+static void rtw_wds_table_free(struct rtw_wds_table *tbl)
+{
+	rtw_rhashtable_free_and_destroy(&tbl->rhead,
+				    rtw_wds_path_rht_free, tbl);
+	rtw_mfree(tbl, sizeof(struct rtw_wds_table));
+}
+
+void rtw_wds_path_assign_nexthop(struct rtw_wds_path *wpath, struct sta_info *sta)
+{
+	rtw_rcu_assign_pointer(wpath->next_hop, sta);
+}
+
+static struct rtw_wds_path *rtw_wpath_lookup(struct rtw_wds_table *tbl, const u8 *dst)
+{
+	struct rtw_wds_path *wpath;
+
+	if (!tbl)
+		return NULL;
+
+	wpath = rtw_rhashtable_lookup_fast(&tbl->rhead, dst, rtw_wds_rht_params);
+
+	return wpath;
+}
+
+struct rtw_wds_path *rtw_wds_path_lookup(_adapter *adapter, const u8 *dst)
+{
+	return rtw_wpath_lookup(adapter->wds_paths, dst);
+}
+
+static struct rtw_wds_path *
+__rtw_wds_path_lookup_by_idx(struct rtw_wds_table *tbl, int idx)
+{
+	int i = 0, ret;
+	struct rtw_wds_path *wpath = NULL;
+	rtw_rhashtable_iter iter;
+
+	if (!tbl)
+		return NULL;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return NULL;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto err;
+
+	while ((wpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(wpath) && PTR_ERR(wpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(wpath))
+			break;
+		if (i++ == idx)
+			break;
+	}
+err:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+
+	if (IS_ERR(wpath) || !wpath)
+		return NULL;
+
+	return wpath;
+}
+
+/**
+ * Locking: must be called within a read rcu section.
+ */
+struct rtw_wds_path *
+rtw_wds_path_lookup_by_idx(_adapter *adapter, int idx)
+{
+	return __rtw_wds_path_lookup_by_idx(adapter->wds_paths, idx);
+}
+
+void dump_wpath(void *sel, _adapter *adapter)
+{
+	struct rtw_wds_path *wpath;
+	int idx = 0;
+	char dst[ETH_ALEN];
+	char next_hop[ETH_ALEN];
+	u32 age_ms;
+
+	RTW_PRINT_SEL(sel, "num:%d\n", ATOMIC_READ(&adapter->wds_path_num));
+	RTW_PRINT_SEL(sel, "%-17s %-17s %-6s\n"
+		, "dst", "next_hop", "age"
+	);
+
+	do {
+		rtw_rcu_read_lock();
+
+		wpath = rtw_wds_path_lookup_by_idx(adapter, idx);
+		if (wpath) {
+			_rtw_memcpy(dst, wpath->dst, ETH_ALEN);
+			_rtw_memcpy(next_hop, wpath->next_hop->phl_sta->mac_addr, ETH_ALEN);
+			age_ms = rtw_get_passing_time_ms(wpath->last_update);
+		}
+
+		rtw_rcu_read_unlock();
+
+		if (wpath) {
+			RTW_PRINT_SEL(sel, MAC_FMT" "MAC_FMT" %6u\n"
+				, MAC_ARG(dst), MAC_ARG(next_hop)
+				, age_ms < 999999 ? age_ms : 999999
+			);
+		}
+
+		idx++;
+	} while (wpath);
+}
+
+static
+struct rtw_wds_path *rtw_wds_path_new(_adapter *adapter,
+				const u8 *dst)
+{
+	struct rtw_wds_path *new_wpath;
+
+	new_wpath = rtw_zmalloc(sizeof(struct rtw_wds_path));
+	if (!new_wpath)
+		return NULL;
+
+	new_wpath->adapter = adapter;
+	_rtw_memcpy(new_wpath->dst, dst, ETH_ALEN);
+	new_wpath->last_update = rtw_get_current_time();
+
+	return new_wpath;
+}
+
+/**
+ * Returns: 0 on success
+ *
+ * State: the initial state of the new path is set to 0
+ */
+struct rtw_wds_path *rtw_wds_path_add(_adapter *adapter,
+	const u8 *dst, struct sta_info *next_hop)
+{
+	struct rtw_wds_table *tbl = adapter->wds_paths;
+	struct rtw_wds_path *wpath, *new_wpath;
+	int ret;
+
+	if (!tbl)
+		return ERR_PTR(-ENOTSUPP);
+
+	if (_rtw_memcmp(dst, adapter_mac_addr(adapter), ETH_ALEN) == _TRUE)
+		/* never add ourselves as neighbours */
+		return ERR_PTR(-ENOTSUPP);
+
+	if (IS_MCAST(dst))
+		return ERR_PTR(-ENOTSUPP);
+
+	if (ATOMIC_INC_UNLESS(&adapter->wds_path_num, RTW_WDS_MAX_PATHS) == 0)
+		return ERR_PTR(-ENOSPC);
+
+	new_wpath = rtw_wds_path_new(adapter, dst);
+	if (!new_wpath)
+		return ERR_PTR(-ENOMEM);
+
+	do {
+		ret = rtw_rhashtable_lookup_insert_fast(&tbl->rhead,
+						    &new_wpath->rhash,
+						    rtw_wds_rht_params);
+
+		if (ret == -EEXIST)
+			wpath = rtw_rhashtable_lookup_fast(&tbl->rhead,
+						       dst,
+						       rtw_wds_rht_params);
+
+	} while (unlikely(ret == -EEXIST && !wpath));
+
+	if (ret && ret != -EEXIST)
+		return ERR_PTR(ret);
+
+	/* At this point either new_wpath was added, or we found a
+	 * matching entry already in the table; in the latter case
+	 * free the unnecessary new entry.
+	 */
+	if (ret == -EEXIST) {
+		rtw_mfree(new_wpath, sizeof(struct rtw_wds_path));
+		new_wpath = wpath;
+	}
+	rtw_wds_path_assign_nexthop(new_wpath, next_hop);
+
+	return new_wpath;
+}
+
+static void rtw_wds_path_free_rcu(struct rtw_wds_table *tbl,
+			       struct rtw_wds_path *wpath)
+{
+	_adapter *adapter = wpath->adapter;
+
+	ATOMIC_DEC(&adapter->wds_path_num);
+
+	rtw_wpath_free_rcu(wpath);
+}
+
+static void __rtw_wds_path_del(struct rtw_wds_table *tbl, struct rtw_wds_path *wpath)
+{
+	rtw_rhashtable_remove_fast(&tbl->rhead, &wpath->rhash, rtw_wds_rht_params);
+	rtw_wds_path_free_rcu(tbl, wpath);
+}
+
+void rtw_wds_path_flush_by_nexthop(struct sta_info *sta)
+{
+	_adapter *adapter = sta->padapter;
+	struct rtw_wds_table *tbl = adapter->wds_paths;
+	struct rtw_wds_path *wpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((wpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(wpath) && PTR_ERR(wpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(wpath))
+			break;
+
+		if (rtw_rcu_access_pointer(wpath->next_hop) == sta)
+			__rtw_wds_path_del(tbl, wpath);
+	}
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+static void rtw_wds_table_flush_by_iface(struct rtw_wds_table *tbl)
+{
+	struct rtw_wds_path *wpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((wpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(wpath) && PTR_ERR(wpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(wpath))
+			break;
+		__rtw_wds_path_del(tbl, wpath);
+	}
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+void rtw_wds_path_flush_by_iface(_adapter *adapter)
+{
+	rtw_wds_table_flush_by_iface(adapter->wds_paths);
+}
+
+static int rtw_wds_table_path_del(struct rtw_wds_table *tbl,
+			  const u8 *addr)
+{
+	struct rtw_wds_path *wpath;
+
+	if (!tbl)
+		return -ENXIO;
+
+	rtw_rcu_read_lock();
+	wpath = rtw_rhashtable_lookup_fast(&tbl->rhead, addr, rtw_wds_rht_params);
+	if (!wpath) {
+		rtw_rcu_read_unlock();
+		return -ENXIO;
+	}
+
+	__rtw_wds_path_del(tbl, wpath);
+	rtw_rcu_read_unlock();
+	return 0;
+}
+
+int rtw_wds_path_del(_adapter *adapter, const u8 *addr)
+{
+	int err;
+
+	err = rtw_wds_table_path_del(adapter->wds_paths, addr);
+	return err;
+}
+
+int rtw_wds_pathtbl_init(_adapter *adapter)
+{
+	struct rtw_wds_table *tbl_path;
+	int ret;
+
+	tbl_path = rtw_wds_table_alloc();
+	if (!tbl_path)
+		return -ENOMEM;
+
+	rtw_rhashtable_init(&tbl_path->rhead, &rtw_wds_rht_params);
+
+	ATOMIC_SET(&adapter->wds_path_num, 0);
+	adapter->wds_paths = tbl_path;
+
+	return 0;
+}
+
+static
+void rtw_wds_path_tbl_expire(_adapter *adapter,
+			  struct rtw_wds_table *tbl)
+{
+	struct rtw_wds_path *wpath;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((wpath = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(wpath) && PTR_ERR(wpath) == -EAGAIN)
+			continue;
+		if (IS_ERR(wpath))
+			break;
+		if (rtw_time_after(rtw_get_current_time(), wpath->last_update + RTW_WDS_PATH_EXPIRE))
+			__rtw_wds_path_del(tbl, wpath);
+	}
+
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+void rtw_wds_path_expire(_adapter *adapter)
+{
+	rtw_wds_path_tbl_expire(adapter, adapter->wds_paths);
+}
+
+void rtw_wds_pathtbl_unregister(_adapter *adapter)
+{
+	if (adapter->wds_paths) {
+		rtw_wds_table_free(adapter->wds_paths);
+		adapter->wds_paths = NULL;
+	}
+}
+
+int rtw_wds_nexthop_lookup(_adapter *adapter, const u8 *da, u8 *ra)
+{
+	struct rtw_wds_path *wpath;
+	struct sta_info *next_hop;
+	int err = -ENOENT;
+
+	rtw_rcu_read_lock();
+	wpath = rtw_wds_path_lookup(adapter, da);
+
+	if (!wpath)
+		goto endlookup;
+
+	next_hop = rtw_rcu_dereference(wpath->next_hop);
+	if (next_hop) {
+		_rtw_memcpy(ra, next_hop->phl_sta->mac_addr, ETH_ALEN);
+		err = 0;
+	}
+
+endlookup:
+	rtw_rcu_read_unlock();
+	return err;
+}
+
+#endif /* defined(CONFIG_AP_MODE) */
+
+/* WDS group adddressed proxy TX record */
+struct rtw_wds_gptr {
+	u8 src[ETH_ALEN];
+	systime last_update;
+	rtw_rhash_head rhash;
+	_adapter *adapter;
+	rtw_rcu_head rcu;
+};
+
+#define RTW_WDS_GPTR_EXPIRE (2 * HZ)
+
+/* Maximum number of gptrs per interface */
+#define RTW_WDS_MAX_GPTRS		1024
+
+#ifdef PLATFORM_LINUX
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+static void rtw_wgptr_free_rcu(struct rtw_wds_gptr *wgptr)
+{
+	kfree_rcu(wgptr, rcu);
+	rtw_mstat_update(MSTAT_TYPE_PHY, MSTAT_FREE, sizeof(struct rtw_wds_gptr));
+}
+#else
+static void rtw_wgptr_free_rcu_callback(rtw_rcu_head *head)
+{
+	struct rtw_wds_gptr *wgptr;
+
+	wgptr = container_of(head, struct rtw_wds_gptr, rcu);
+	rtw_mfree(wgptr, sizeof(struct rtw_wds_gptr));
+}
+
+static void rtw_wgptr_free_rcu(struct rtw_wds_gptr *wgptr)
+{
+	call_rcu(&wgptr->rcu, rtw_wgptr_free_rcu_callback);
+}
+#endif
+#endif /* PLATFORM_LINUX */
+
+static void rtw_wds_gptr_free_rcu(struct rtw_wds_gptr_table *tbl, struct rtw_wds_gptr *wgptr)
+{
+	_adapter *adapter = wgptr->adapter;
+
+	ATOMIC_DEC(&adapter->wds_gpt_record_num);
+
+	rtw_wgptr_free_rcu(wgptr);
+}
+
+static u32 rtw_wds_gptr_table_hash(const void *addr, u32 len, u32 seed)
+{
+	/* Use last four bytes of hw addr as hash index */
+	return jhash_1word(*(u32 *)(addr+2), seed);
+}
+
+static const rtw_rhashtable_params rtw_wds_gptr_rht_params = {
+	.nelem_hint = 2,
+	.automatic_shrinking = true,
+	.key_len = ETH_ALEN,
+	.key_offset = offsetof(struct rtw_wds_gptr, src),
+	.head_offset = offsetof(struct rtw_wds_gptr, rhash),
+	.hashfn = rtw_wds_gptr_table_hash,
+};
+
+static void rtw_wds_gptr_rht_free(void *ptr, void *tblptr)
+{
+	struct rtw_wds_gptr *wgptr = ptr;
+	struct rtw_wds_gptr_table *tbl = tblptr;
+
+	rtw_wds_gptr_free_rcu(tbl, wgptr);
+}
+
+static struct rtw_wds_gptr_table *rtw_wds_gptr_table_alloc(void)
+{
+	struct rtw_wds_gptr_table *newtbl;
+
+	newtbl = rtw_malloc(sizeof(struct rtw_wds_gptr_table));
+	if (!newtbl)
+		return NULL;
+
+	return newtbl;
+}
+
+static void rtw_wds_gptr_table_free(struct rtw_wds_gptr_table *tbl)
+{
+	rtw_rhashtable_free_and_destroy(&tbl->rhead,
+				    rtw_wds_gptr_rht_free, tbl);
+	rtw_mfree(tbl, sizeof(struct rtw_wds_gptr_table));
+}
+
+static struct rtw_wds_gptr *rtw_wds_gptr_lookup(_adapter *adapter, const u8 *src)
+{
+	struct rtw_wds_gptr_table *tbl = adapter->wds_gpt_records;
+
+	if (!tbl)
+		return NULL;
+
+	return rtw_rhashtable_lookup_fast(&tbl->rhead, src, rtw_wds_gptr_rht_params);
+}
+
+/**
+ * Locking: must be called within a read rcu section.
+ */
+static struct rtw_wds_gptr *rtw_wds_gptr_lookup_by_idx(_adapter *adapter, int idx)
+{
+	int i = 0, ret;
+	struct rtw_wds_gptr_table *tbl = adapter->wds_gpt_records;
+	struct rtw_wds_gptr *wgptr = NULL;
+	rtw_rhashtable_iter iter;
+
+	if (!tbl)
+		return NULL;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return NULL;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto err;
+
+	while ((wgptr = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(wgptr) && PTR_ERR(wgptr) == -EAGAIN)
+			continue;
+		if (IS_ERR(wgptr))
+			break;
+		if (i++ == idx)
+			break;
+	}
+err:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+
+	if (IS_ERR(wgptr) || !wgptr)
+		return NULL;
+
+	return wgptr;
+}
+
+void dump_wgptr(void *sel, _adapter *adapter)
+{
+	struct rtw_wds_gptr *wgptr;
+	int idx = 0;
+	char src[ETH_ALEN];
+	u32 age_ms;
+
+	RTW_PRINT_SEL(sel, "num:%d\n", ATOMIC_READ(&adapter->wds_gpt_record_num));
+	RTW_PRINT_SEL(sel, "%-17s %-6s\n"
+		, "src", "age"
+	);
+
+	do {
+		rtw_rcu_read_lock();
+
+		wgptr = rtw_wds_gptr_lookup_by_idx(adapter, idx);
+		if (wgptr) {
+			_rtw_memcpy(src, wgptr->src, ETH_ALEN);
+			age_ms = rtw_get_passing_time_ms(wgptr->last_update);
+		}
+
+		rtw_rcu_read_unlock();
+
+		if (wgptr) {
+			RTW_PRINT_SEL(sel, MAC_FMT" %6u\n"
+				, MAC_ARG(src)
+				, age_ms < 999999 ? age_ms : 999999
+			);
+		}
+
+		idx++;
+	} while (wgptr);
+}
+
+static struct rtw_wds_gptr *rtw_wds_gptr_new(_adapter *adapter, const u8 *src)
+{
+	struct rtw_wds_gptr *new_wgptr;
+
+	new_wgptr = rtw_zmalloc(sizeof(struct rtw_wds_gptr));
+	if (!new_wgptr)
+		return NULL;
+
+	new_wgptr->adapter = adapter;
+	_rtw_memcpy(new_wgptr->src, src, ETH_ALEN);
+	new_wgptr->last_update = rtw_get_current_time();
+
+	return new_wgptr;
+}
+
+static struct rtw_wds_gptr *rtw_wds_gptr_add(_adapter *adapter, const u8 *src)
+{
+	struct rtw_wds_gptr_table *tbl = adapter->wds_gpt_records;
+	struct rtw_wds_gptr *wgptr, *new_wgptr;
+	int ret;
+
+	if (!tbl)
+		return ERR_PTR(-ENOTSUPP);
+
+	if (ATOMIC_INC_UNLESS(&adapter->wds_gpt_record_num, RTW_WDS_MAX_PATHS) == 0)
+		return ERR_PTR(-ENOSPC);
+
+	new_wgptr = rtw_wds_gptr_new(adapter, src);
+	if (!new_wgptr)
+		return ERR_PTR(-ENOMEM);
+
+	do {
+		ret = rtw_rhashtable_lookup_insert_fast(&tbl->rhead,
+						    &new_wgptr->rhash,
+						    rtw_wds_gptr_rht_params);
+
+		if (ret == -EEXIST)
+			wgptr = rtw_rhashtable_lookup_fast(&tbl->rhead,
+						       src,
+						       rtw_wds_gptr_rht_params);
+
+	} while (unlikely(ret == -EEXIST && !wgptr));
+
+	if (ret && ret != -EEXIST)
+		return ERR_PTR(ret);
+
+	/* At this point either new_wgptr was added, or we found a
+	 * matching entry already in the table; in the latter case
+	 * free the unnecessary new entry.
+	 */
+	if (ret == -EEXIST) {
+		rtw_mfree(new_wgptr, sizeof(struct rtw_wds_gptr));
+		new_wgptr = wgptr;
+	}
+
+	return new_wgptr;
+}
+
+bool rtw_rx_wds_gptr_check(_adapter *adapter, const u8 *src)
+{
+	struct rtw_wds_gptr *wgptr;
+	bool ret = 0;
+
+	rtw_rcu_read_lock();
+
+	wgptr = rtw_wds_gptr_lookup(adapter, src);
+	if (wgptr)
+		ret = rtw_time_after(wgptr->last_update + RTW_WDS_GPTR_EXPIRE, rtw_get_current_time());
+
+	rtw_rcu_read_unlock();
+
+	return ret;
+}
+
+void rtw_tx_wds_gptr_update(_adapter *adapter, const u8 *src)
+{
+	struct rtw_wds_gptr *wgptr;
+
+	rtw_rcu_read_lock();
+	wgptr = rtw_wds_gptr_lookup(adapter, src);
+	if (!wgptr)
+		rtw_wds_gptr_add(adapter, src);
+	else
+		wgptr->last_update = rtw_get_current_time();
+	rtw_rcu_read_unlock();
+}
+
+static void __rtw_wds_gptr_del(struct rtw_wds_gptr_table *tbl, struct rtw_wds_gptr *wgptr)
+{
+	rtw_rhashtable_remove_fast(&tbl->rhead, &wgptr->rhash, rtw_wds_gptr_rht_params);
+	rtw_wds_gptr_free_rcu(tbl, wgptr);
+}
+
+void rtw_wds_gptr_expire(_adapter *adapter)
+{
+	struct rtw_wds_gptr_table *tbl = adapter->wds_gpt_records;
+	struct rtw_wds_gptr *wgptr;
+	rtw_rhashtable_iter iter;
+	int ret;
+
+	if (!tbl)
+		return;
+
+	ret = rtw_rhashtable_walk_enter(&tbl->rhead, &iter);
+	if (ret)
+		return;
+
+	ret = rtw_rhashtable_walk_start(&iter);
+	if (ret && ret != -EAGAIN)
+		goto out;
+
+	while ((wgptr = rtw_rhashtable_walk_next(&iter))) {
+		if (IS_ERR(wgptr) && PTR_ERR(wgptr) == -EAGAIN)
+			continue;
+		if (IS_ERR(wgptr))
+			break;
+		if (rtw_time_after(rtw_get_current_time(), wgptr->last_update + RTW_WDS_GPTR_EXPIRE))
+			__rtw_wds_gptr_del(tbl, wgptr);
+	}
+
+out:
+	rtw_rhashtable_walk_stop(&iter);
+	rtw_rhashtable_walk_exit(&iter);
+}
+
+int rtw_wds_gptr_tbl_init(_adapter *adapter)
+{
+	struct rtw_wds_gptr_table *tbl;
+	int ret;
+
+	tbl = rtw_wds_gptr_table_alloc();
+	if (!tbl)
+		return -ENOMEM;
+
+	rtw_rhashtable_init(&tbl->rhead, &rtw_wds_gptr_rht_params);
+
+	ATOMIC_SET(&adapter->wds_gpt_record_num, 0);
+	adapter->wds_gpt_records = tbl;
+
+	return 0;
+}
+
+void rtw_wds_gptr_tbl_unregister(_adapter *adapter)
+{
+	if (adapter->wds_gpt_records) {
+		rtw_wds_gptr_table_free(adapter->wds_gpt_records);
+		adapter->wds_gpt_records = NULL;
+	}
+}
+#endif /* defined(CONFIG_RTW_WDS) */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/core/wds/rtw_wds.h b/drivers/net/wireless/realtek/rtl8852bs/core/wds/rtw_wds.h
new file mode 100644
index 000000000000..f1312c740107
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/core/wds/rtw_wds.h
@@ -0,0 +1,65 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_WDS_H_
+#define __RTW_WDS_H_
+
+#ifdef CONFIG_AP_MODE
+struct rtw_wds_path {
+	u8 dst[ETH_ALEN];
+	rtw_rhash_head rhash;
+	_adapter *adapter;
+	struct sta_info __rcu *next_hop;
+	rtw_rcu_head rcu;
+	systime last_update;
+};
+
+struct rtw_wds_table {
+	rtw_rhashtable rhead;
+};
+
+#define RTW_WDS_PATH_EXPIRE (600 * HZ)
+
+/* Maximum number of paths per interface */
+#define RTW_WDS_MAX_PATHS		1024
+
+int rtw_wds_nexthop_lookup(_adapter *adapter, const u8 *da, u8 *ra);
+
+struct rtw_wds_path *rtw_wds_path_lookup(_adapter *adapter, const u8 *dst);
+void dump_wpath(void *sel, _adapter *adapter);
+
+void rtw_wds_path_expire(_adapter *adapter);
+
+struct rtw_wds_path *rtw_wds_path_add(_adapter *adapter, const u8 *dst, struct sta_info *next_hop);
+void rtw_wds_path_assign_nexthop(struct rtw_wds_path *path, struct sta_info *sta);
+
+int rtw_wds_pathtbl_init(_adapter *adapter);
+void rtw_wds_pathtbl_unregister(_adapter *adapter);
+
+void rtw_wds_path_flush_by_nexthop(struct sta_info *sta);
+#endif /* CONFIG_AP_MODE */
+
+struct rtw_wds_gptr_table {
+	rtw_rhashtable rhead;
+};
+
+void dump_wgptr(void *sel, _adapter *adapter);
+bool rtw_rx_wds_gptr_check(_adapter *adapter, const u8 *src);
+void rtw_tx_wds_gptr_update(_adapter *adapter, const u8 *src);
+void rtw_wds_gptr_expire(_adapter *adapter);
+int rtw_wds_gptr_tbl_init(_adapter *adapter);
+void rtw_wds_gptr_tbl_unregister(_adapter *adapter);
+
+#endif /* __RTW_WDSH_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/ifcfg-wlan0 b/drivers/net/wireless/realtek/rtl8852bs/ifcfg-wlan0
new file mode 100755
index 000000000000..7ecb7ae62c7f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/ifcfg-wlan0
@@ -0,0 +1,4 @@
+#DHCP client
+DEVICE=wlan0
+BOOTPROTO=dhcp
+ONBOOT=yes
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/_hal_api_tmp.h b/drivers/net/wireless/realtek/rtl8852bs/include/_hal_api_tmp.h
new file mode 100644
index 000000000000..59ebedb97a8f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/_hal_api_tmp.h
@@ -0,0 +1,347 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_TMP_H_
+#define _HAL_API_TMP_H_
+
+
+/**** may be get from hal_com **********************************/
+
+#define H2C_MSR_ROLE_RSVD	0
+#define H2C_MSR_ROLE_STA	1
+#define H2C_MSR_ROLE_AP	2
+#define H2C_MSR_ROLE_GC	3
+#define H2C_MSR_ROLE_GO	4
+#define H2C_MSR_ROLE_TDLS	5
+#define H2C_MSR_ROLE_ADHOC	6
+#define H2C_MSR_ROLE_MESH	7
+#define H2C_MSR_ROLE_MAX	8
+
+/*************************************************************************************/
+typedef enum _HW_VARIABLES {
+	HW_VAR_NET_TYPE,
+	HW_VAR_SET_OPMODE,
+	HW_VAR_MAC_ADDR,
+	HW_VAR_BSSID,
+	HW_VAR_BASIC_RATE,
+	HW_VAR_TXPAUSE,
+	HW_VAR_BCN_FUNC,
+	HW_VAR_CORRECT_TSF,
+	HW_VAR_RCR,
+	HW_VAR_MLME_DISCONNECT,
+	HW_VAR_MLME_SITESURVEY,
+	HW_VAR_MLME_JOIN,
+	HW_VAR_ON_RCR_AM,
+	HW_VAR_OFF_RCR_AM,
+	HW_VAR_BEACON_INTERVAL,
+	HW_VAR_SLOT_TIME,
+	HW_VAR_RESP_SIFS,
+	HW_VAR_ACK_PREAMBLE,
+	HW_VAR_SEC_CFG,
+	HW_VAR_SEC_DK_CFG,
+	HW_VAR_BCN_VALID,
+	HW_VAR_FREECNT,
+	HW_VAR_STOP_BCN,
+	HW_VAR_RESUME_BCN,
+	/* PHYDM odm->SupportAbility */
+	HW_VAR_CAM_EMPTY_ENTRY,
+	HW_VAR_CAM_INVALID_ALL,
+	HW_VAR_AC_PARAM_VO,
+	HW_VAR_AC_PARAM_VI,
+	HW_VAR_AC_PARAM_BE,
+	HW_VAR_AC_PARAM_BK,
+	HW_VAR_ACM_CTRL,
+	HW_VAR_AMPDU_MIN_SPACE,
+#ifdef CONFIG_80211N_HT
+	HW_VAR_AMPDU_FACTOR,
+#endif /* CONFIG_80211N_HT */
+	HW_VAR_RXDMA_AGG_PG_TH,
+	HW_VAR_SET_RPWM,
+	HW_VAR_CPWM,
+	HW_VAR_H2C_FW_PWRMODE,
+	HW_VAR_H2C_INACTIVE_IPS,
+	HW_VAR_H2C_FW_JOINBSSRPT,
+	HW_VAR_TRIGGER_GPIO_0,
+	HW_VAR_BT_SET_COEXIST,
+	HW_VAR_BT_ISSUE_DELBA,
+	HW_VAR_FIFO_CLEARN_UP,
+	HW_VAR_RESTORE_HW_SEQ,
+	HW_VAR_CHECK_TXBUF,
+	HW_VAR_PCIE_STOP_TX_DMA,
+	HW_VAR_APFM_ON_MAC, /* Auto FSM to Turn On, include clock, isolation, power control for MAC only */
+	/* The valid upper nav range for the HW updating, if the true value is larger than the upper range, the HW won't update it. */
+	/* Unit in microsecond. 0 means disable this function. */
+#if defined(CONFIG_WOWLAN) || defined(CONFIG_AP_WOWLAN)
+	HW_VAR_WOWLAN,
+	HW_VAR_WAKEUP_REASON,
+#endif
+	HW_VAR_RPWM_TOG,
+#ifdef CONFIG_GPIO_WAKEUP
+	HW_VAR_WOW_OUTPUT_GPIO,
+	HW_VAR_WOW_INPUT_GPIO,
+	HW_SET_GPIO_WL_CTRL,
+#endif
+	HW_VAR_SYS_CLKR,
+	HW_VAR_NAV_UPPER,
+	HW_VAR_CHK_HI_QUEUE_EMPTY,
+	HW_VAR_CHK_MGQ_CPU_EMPTY,
+	HW_VAR_DL_BCN_SEL,
+	HW_VAR_AMPDU_MAX_TIME,
+	HW_VAR_WIRELESS_MODE,
+	HW_VAR_USB_MODE,
+	HW_VAR_PORT_SWITCH,
+	HW_VAR_PORT_CFG,
+	HW_VAR_SOUNDING_ENTER,
+	HW_VAR_SOUNDING_LEAVE,
+	HW_VAR_SOUNDING_RATE,
+	HW_VAR_SOUNDING_STATUS,
+	HW_VAR_SOUNDING_FW_NDPA,
+	HW_VAR_SOUNDING_CLK,
+	HW_VAR_SOUNDING_SET_GID_TABLE,
+	HW_VAR_SOUNDING_CSI_REPORT,
+	HW_VAR_DL_RSVD_PAGE,
+	HW_VAR_DUMP_MAC_QUEUE_INFO,
+	HW_VAR_ASIX_IOT,
+	HW_VAR_CH_SW_NEED_TO_TAKE_CARE_IQK_INFO,
+	HW_VAR_CH_SW_IQK_INFO_BACKUP,
+	HW_VAR_CH_SW_IQK_INFO_RESTORE,
+
+	HW_VAR_DBI,
+	HW_VAR_MDIO,
+	HW_VAR_L1OFF_CAPABILITY,
+	HW_VAR_L1OFF_NIC_SUPPORT,
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+	HW_VAR_TDLS_BCN_EARLY_C2H_RPT,
+#endif
+#endif
+	HW_VAR_DUMP_MAC_TXFIFO,
+	HW_VAR_PWR_CMD,
+
+	HW_VAR_SET_SOML_PARAM,
+	HW_VAR_ENABLE_RX_BAR,
+	HW_VAR_TSF_AUTO_SYNC,
+	HW_VAR_LPS_STATE_CHK,
+	#ifdef CONFIG_RTS_FULL_BW
+	HW_VAR_SET_RTS_BW,
+	#endif
+#if defined(CONFIG_PCI_HCI)
+	HW_VAR_ENSWBCN,
+#endif
+	HW_VAR_ACKTO,
+	HW_VAR_ACKTO_CCK,
+} HW_VARIABLES;
+
+static inline u8 rtw_hal_set_hwreg(_adapter *padapter, u8 var, u8 *val)
+{
+	return 0;
+}
+static inline void rtw_hal_get_hwreg(_adapter *padapter, u8 var, u8 *val)
+{}
+typedef enum _HAL_DEF_VARIABLE {
+	HAL_DEF_IS_SUPPORT_ANT_DIV,
+	HAL_DEF_DBG_DUMP_RXPKT,/* for dbg */
+	HAL_DEF_BEAMFORMER_CAP,
+	HAL_DEF_BEAMFORMEE_CAP,
+	HW_VAR_MAX_RX_AMPDU_FACTOR,
+	HW_DEF_RA_INFO_DUMP,
+	HAL_DEF_DBG_DUMP_TXPKT,
+	HAL_DEF_TX_PAGE_SIZE,
+	HW_VAR_BEST_AMPDU_DENSITY,
+} HAL_DEF_VARIABLE;
+
+static inline u8 rtw_hal_set_def_var(_adapter *padapter, HAL_DEF_VARIABLE def_var, void *val)
+{
+	return 0;
+}
+
+u8 rtw_hal_get_def_var(struct _ADAPTER *a, struct _ADAPTER_LINK *alink,
+				enum _HAL_DEF_VARIABLE def_var, void *val);
+
+#if 0
+static inline void rtw_hal_sec_read_cam_ent(_adapter *adapter, u8 id, u8 *ctrl, u8 *mac, u8 *key)
+{}
+static inline void rtw_hal_sec_write_cam_ent(_adapter *adapter, u8 id, u16 ctrl, u8 *mac, u8 *key)
+{}
+static inline void rtw_hal_sec_clr_cam_ent(_adapter *adapter, u8 id)
+{}
+static inline bool rtw_hal_sec_read_cam_is_gk(_adapter *adapter, u8 id)
+{
+	return _TRUE;
+}
+#endif
+
+static inline void rtw_hal_linked_info_dump(_adapter *padapter, u8 benable)
+{}
+
+static inline bool rtw_hal_get_phy_edcca_flag(_adapter *adapter)
+{
+	return _TRUE;
+}
+
+static inline u64 rtw_hal_get_tsftr_by_port(_adapter *adapter, u8 port)
+{
+	return 1;
+}
+
+static inline void rtw_hal_dump_rsvd_page(void *sel, _adapter *adapter, u8 page_offset, u8 page_num)
+{}
+
+
+/*u8 beamforming_get_htndp_tx_rate(void *dm_void, u8 bfer_str_num);*/
+static inline u8 rtw_hal_get_htndp_tx_rate(_adapter *adapter, u8 bfer_str_num)
+{
+	return 0;
+}
+/*u8 beamforming_get_vht_ndp_tx_rate(void *dm_void, u8 bfer_str_num);*/
+static inline u8 rtw_hal_get_vht_ndp_tx_rate(_adapter *adapter, u8 bfer_str_num)
+{
+	return 0;
+}
+
+static inline u8 rtw_hal_get_sounding_info(_adapter *adapter,u16 *throughput,
+				       u8 total_bfee_num, u16 *tx_rate)
+{
+	return 0;
+}
+
+static inline void rtw_hal_dump_target_tx_power(void *sel, _adapter *adapter)
+{}
+
+static inline void rtw_hal_dump_trx_mode(void *sel, _adapter *adapter)
+{}
+
+#if defined(CONFIG_RTW_LED) && defined(CONFIG_RTW_SW_LED)
+#ifndef CONFIG_RTW_SW_LED_TRX_DA_CLASSIFY
+#define CONFIG_RTW_SW_LED_TRX_DA_CLASSIFY 0
+#endif
+
+#if CONFIG_RTW_SW_LED_TRX_DA_CLASSIFY
+void rtw_sw_led_blink_uc_trx_only(LED_DATA *led);
+void rtw_sw_led_ctl_mode_uc_trx_only(_adapter *adapter, LED_CTL_MODE ctl);
+#endif
+void rtw_led_control(_adapter *adapter, LED_CTL_MODE ctl);
+void rtw_led_tx_control(_adapter *adapter, const u8 *da);
+void rtw_led_rx_control(_adapter *adapter, const u8 *da);
+void rtw_led_set_iface_en(_adapter *adapter, u8 en);
+void rtw_led_set_iface_en_mask(_adapter *adapter, u8 mask);
+void rtw_led_set_ctl_en_mask(_adapter *adapter, u32 ctl_mask);
+void rtw_led_set_ctl_en_mask_primary(_adapter *adapter);
+void rtw_led_set_ctl_en_mask_virtual(_adapter *adapter);
+#else
+#define rtw_led_control(adapter, ctl) do {} while (0)
+#define rtw_led_tx_control(adapter, da) do {} while (0)
+#define rtw_led_rx_control(adapter, da) do {} while (0)
+#define rtw_led_set_iface_en(adapter, en) do {} while (0)
+#define rtw_led_set_iface_en_mask(adapter, mask) do {} while (0)
+#define rtw_led_set_ctl_en_mask(adapter, ctl_mask) do {} while (0)
+#define rtw_led_set_ctl_en_mask_primary(adapter) do {} while (0)
+#define rtw_led_set_ctl_en_mask_virtual(adapter) do {} while (0)
+#endif /* defined(CONFIG_RTW_LED) && defined(CONFIG_RTW_SW_LED) */
+
+#ifdef CONFIG_PCI_HCI
+static inline u8 rtw_hal_pci_dbi_read(_adapter *padapter, u16 addr)
+{	return 0;}
+static inline u8 rtw_hal_pci_l1off_nic_support(_adapter *padapter)
+{	return 0;}
+
+static inline u8 rtw_hal_pci_l1off_capability(_adapter *padapter)
+{	return 0;}
+
+static inline void rtw_hal_unmap_beacon_icf(_adapter *padapter)
+{
+	//hal->hal_ops.unmap_beacon_icf(padapter);
+}
+#endif
+
+#if defined(CONFIG_PCI_HCI)
+static inline u8 rtw_hal_check_nic_enough_desc_all(_adapter *padapter)
+{ return _SUCCESS;}
+#endif
+
+static inline s32 rtw_hal_macid_sleep(_adapter *adapter, u8 macid)
+{	return 0;}
+static inline s32 rtw_hal_macid_wakeup(_adapter *adapter, u8 macid)
+{	return 0;}
+static inline s32 rtw_hal_macid_sleep_all_used(_adapter *adapter)
+{	return 0;}
+static inline s32 rtw_hal_macid_wakeup_all_used(_adapter *adapter)
+{	return 0;}
+
+static void rtw_hal_c2h_pkt_hdl(_adapter *adapter, u8 *buf, u16 len)
+{
+	//adapter->dvobj->hal_func.hal_mac_c2h_handler(adapter, buf, len);
+}
+static inline s32 rtw_hal_fill_h2c_cmd(_adapter *padapter, u8 ElementID, u32 CmdLen, u8 *pCmdBuffer)
+{
+/*
+	_adapter *pri_adapter = GET_PRIMARY_ADAPTER(padapter);
+
+	if (GET_PHL_COM(pri_adapter)->fw_ready == _TRUE)
+		return hal->hal_ops.fill_h2c_cmd(padapter, ElementID, CmdLen, pCmdBuffer);
+	else if (padapter->registrypriv.mp_mode == 0)
+		RTW_PRINT(FUNC_ADPT_FMT" FW doesn't exit when no MP mode, by pass H2C id:0x%02x\n"
+			  , FUNC_ADPT_ARG(padapter), ElementID);
+*/
+	return 0;
+}
+
+static inline u8 rtw_hal_get_port(_adapter *adapter)
+{	return 0;}
+
+static inline void rtw_hal_read_edca(_adapter *adapter, u16 *vo_params, u16 *vi_params,
+			u16 *be_params, u16 *bk_params)
+{
+	//hal->hal_func.read_wmmedca_reg(padapter, vo_params, vi_params, be_params, bk_params);
+}
+
+/************************ xmit *******************/
+static inline void rtw_hal_bcn_param_setting(_adapter *padapter)
+{
+	//hal->hal_ops.set_beacon_param_handler(padapter);
+}
+
+static inline void rtw_hal_set_tx_power_level(_adapter *adapter, u8 channel)
+{}
+
+
+/****************** GEORGIA_TODO_REDEFINE_IO ************************/
+static inline u32 rtw_hal_get_ltsf(_adapter *adapter)/*get tst low 4 bytes */
+{
+	return 0;
+}
+
+static inline u32 rtw_hal_get_dma_statu(_adapter *adapter)
+{
+	return 0;
+}
+
+#ifdef RTW_SUPPORT_PLATFORM_SHUTDOWN
+static inline u8 rtw_hal_sdio_leave_suspend(_adapter *adapter)
+{
+	return 0;
+}
+#endif
+
+static inline void rtw_hal_get_version(char *str, u32 len)
+{
+	//get hal version
+	//rtw_halmac_get_version(str, 30);
+	// get fw version
+	// get phy (bb/rf) version
+	// get btc version
+}
+
+
+#endif /*_HAL_API_TMP_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/_hal_rate.h b/drivers/net/wireless/realtek/rtl8852bs/include/_hal_rate.h
new file mode 100644
index 000000000000..9f6668230832
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/_hal_rate.h
@@ -0,0 +1,1202 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __HAL_RATE_H__
+#define __HAL_RATE_H__
+
+/*-----------------------------------------------------------
+ *	HW Rate def
+ * -----------------------------------------------------------
+ * CCK Rates, TxHT = 0 */
+#define DESC_RATE1M				0x00
+#define DESC_RATE2M				0x01
+#define DESC_RATE5_5M			0x02
+#define DESC_RATE11M			0x03
+
+/* OFDM Rates, TxHT = 0 */
+#define DESC_RATE6M				0x04
+#define DESC_RATE9M				0x05
+#define DESC_RATE12M			0x06
+#define DESC_RATE18M			0x07
+#define DESC_RATE24M			0x08
+#define DESC_RATE36M			0x09
+#define DESC_RATE48M			0x0A
+#define DESC_RATE54M			0x0B
+
+/* MCS Rates, TxHT = 1 */
+#define DESC_RATEMCS0			0x0C
+#define DESC_RATEMCS1			0x0D
+#define DESC_RATEMCS2			0x0E
+#define DESC_RATEMCS3			0x0F
+#define DESC_RATEMCS4			0x10
+#define DESC_RATEMCS5			0x11
+#define DESC_RATEMCS6			0x12
+#define DESC_RATEMCS7			0x13
+#define DESC_RATEMCS8			0x14
+#define DESC_RATEMCS9			0x15
+#define DESC_RATEMCS10			0x16
+#define DESC_RATEMCS11			0x17
+#define DESC_RATEMCS12			0x18
+#define DESC_RATEMCS13			0x19
+#define DESC_RATEMCS14			0x1A
+#define DESC_RATEMCS15			0x1B
+#define DESC_RATEMCS16			0x1C
+#define DESC_RATEMCS17			0x1D
+#define DESC_RATEMCS18			0x1E
+#define DESC_RATEMCS19			0x1F
+#define DESC_RATEMCS20			0x20
+#define DESC_RATEMCS21			0x21
+#define DESC_RATEMCS22			0x22
+#define DESC_RATEMCS23			0x23
+#define DESC_RATEMCS24			0x24
+#define DESC_RATEMCS25			0x25
+#define DESC_RATEMCS26			0x26
+#define DESC_RATEMCS27			0x27
+#define DESC_RATEMCS28			0x28
+#define DESC_RATEMCS29			0x29
+#define DESC_RATEMCS30			0x2A
+#define DESC_RATEMCS31			0x2B
+#define DESC_RATEVHTSS1MCS0		0x2C
+#define DESC_RATEVHTSS1MCS1		0x2D
+#define DESC_RATEVHTSS1MCS2		0x2E
+#define DESC_RATEVHTSS1MCS3		0x2F
+#define DESC_RATEVHTSS1MCS4		0x30
+#define DESC_RATEVHTSS1MCS5		0x31
+#define DESC_RATEVHTSS1MCS6		0x32
+#define DESC_RATEVHTSS1MCS7		0x33
+#define DESC_RATEVHTSS1MCS8		0x34
+#define DESC_RATEVHTSS1MCS9		0x35
+#define DESC_RATEVHTSS2MCS0		0x36
+#define DESC_RATEVHTSS2MCS1		0x37
+#define DESC_RATEVHTSS2MCS2		0x38
+#define DESC_RATEVHTSS2MCS3		0x39
+#define DESC_RATEVHTSS2MCS4		0x3A
+#define DESC_RATEVHTSS2MCS5		0x3B
+#define DESC_RATEVHTSS2MCS6		0x3C
+#define DESC_RATEVHTSS2MCS7		0x3D
+#define DESC_RATEVHTSS2MCS8		0x3E
+#define DESC_RATEVHTSS2MCS9		0x3F
+#define DESC_RATEVHTSS3MCS0		0x40
+#define DESC_RATEVHTSS3MCS1		0x41
+#define DESC_RATEVHTSS3MCS2		0x42
+#define DESC_RATEVHTSS3MCS3		0x43
+#define DESC_RATEVHTSS3MCS4		0x44
+#define DESC_RATEVHTSS3MCS5		0x45
+#define DESC_RATEVHTSS3MCS6		0x46
+#define DESC_RATEVHTSS3MCS7		0x47
+#define DESC_RATEVHTSS3MCS8		0x48
+#define DESC_RATEVHTSS3MCS9		0x49
+#define DESC_RATEVHTSS4MCS0		0x4A
+#define DESC_RATEVHTSS4MCS1		0x4B
+#define DESC_RATEVHTSS4MCS2		0x4C
+#define DESC_RATEVHTSS4MCS3		0x4D
+#define DESC_RATEVHTSS4MCS4		0x4E
+#define DESC_RATEVHTSS4MCS5		0x4F
+#define DESC_RATEVHTSS4MCS6		0x50
+#define DESC_RATEVHTSS4MCS7		0x51
+#define DESC_RATEVHTSS4MCS8		0x52
+#define DESC_RATEVHTSS4MCS9		0x53
+#define DESC_RATEHESS1MCS0		0x54
+#define DESC_RATEHESS1MCS1		0x55
+#define DESC_RATEHESS1MCS2		0x56
+#define DESC_RATEHESS1MCS3		0x57
+#define DESC_RATEHESS1MCS4		0x58
+#define DESC_RATEHESS1MCS5		0x59
+#define DESC_RATEHESS1MCS6		0x5A
+#define DESC_RATEHESS1MCS7		0x5B
+#define DESC_RATEHESS1MCS8		0x5C
+#define DESC_RATEHESS1MCS9		0x5D
+#define DESC_RATEHESS1MCS10		0x5E
+#define DESC_RATEHESS1MCS11		0x5F
+#define DESC_RATEHESS2MCS0		0x60
+#define DESC_RATEHESS2MCS1		0x61
+#define DESC_RATEHESS2MCS2		0x62
+#define DESC_RATEHESS2MCS3		0x63
+#define DESC_RATEHESS2MCS4		0x64
+#define DESC_RATEHESS2MCS5		0x65
+#define DESC_RATEHESS2MCS6		0x66
+#define DESC_RATEHESS2MCS7		0x67
+#define DESC_RATEHESS2MCS8		0x68
+#define DESC_RATEHESS2MCS9		0x69
+#define DESC_RATEHESS2MCS10		0x6A
+#define DESC_RATEHESS2MCS11		0x6B
+#define DESC_RATEHESS3MCS0		0x6C
+#define DESC_RATEHESS3MCS1		0x6D
+#define DESC_RATEHESS3MCS2		0x6E
+#define DESC_RATEHESS3MCS3		0x6F
+#define DESC_RATEHESS3MCS4		0x70
+#define DESC_RATEHESS3MCS5		0x71
+#define DESC_RATEHESS3MCS6		0x72
+#define DESC_RATEHESS3MCS7		0x73
+#define DESC_RATEHESS3MCS8		0x74
+#define DESC_RATEHESS3MCS9		0x75
+#define DESC_RATEHESS3MCS10		0x76
+#define DESC_RATEHESS3MCS11		0x77
+#define DESC_RATEHESS4MCS0		0x78
+#define DESC_RATEHESS4MCS1		0x79
+#define DESC_RATEHESS4MCS2		0x7A
+#define DESC_RATEHESS4MCS3		0x7B
+#define DESC_RATEHESS4MCS4		0x7C
+#define DESC_RATEHESS4MCS5		0x7D
+#define DESC_RATEHESS4MCS6		0x7E
+#define DESC_RATEHESS4MCS7		0x7F
+#define DESC_RATEHESS4MCS8		0x80
+#define DESC_RATEHESS4MCS9		0x81
+#define DESC_RATEHESS4MCS10		0x82
+#define DESC_RATEHESS4MCS11		0x83
+
+#define HDATA_RATE(rate)\
+	(rate == DESC_RATE1M) ? "CCK_1M" :\
+	(rate == DESC_RATE2M) ? "CCK_2M" :\
+	(rate == DESC_RATE5_5M) ? "CCK5_5M" :\
+	(rate == DESC_RATE11M) ? "CCK_11M" :\
+	(rate == DESC_RATE6M) ? "OFDM_6M" :\
+	(rate == DESC_RATE9M) ? "OFDM_9M" :\
+	(rate == DESC_RATE12M) ? "OFDM_12M" :\
+	(rate == DESC_RATE18M) ? "OFDM_18M" :\
+	(rate == DESC_RATE24M) ? "OFDM_24M" :\
+	(rate == DESC_RATE36M) ? "OFDM_36M" :\
+	(rate == DESC_RATE48M) ? "OFDM_48M" :\
+	(rate == DESC_RATE54M) ? "OFDM_54M" :\
+	(rate == DESC_RATEMCS0) ? "MCS0" :\
+	(rate == DESC_RATEMCS1) ? "MCS1" :\
+	(rate == DESC_RATEMCS2) ? "MCS2" :\
+	(rate == DESC_RATEMCS3) ? "MCS3" :\
+	(rate == DESC_RATEMCS4) ? "MCS4" :\
+	(rate == DESC_RATEMCS5) ? "MCS5" :\
+	(rate == DESC_RATEMCS6) ? "MCS6" :\
+	(rate == DESC_RATEMCS7) ? "MCS7" :\
+	(rate == DESC_RATEMCS8) ? "MCS8" :\
+	(rate == DESC_RATEMCS9) ? "MCS9" :\
+	(rate == DESC_RATEMCS10) ? "MCS10" :\
+	(rate == DESC_RATEMCS11) ? "MCS11" :\
+	(rate == DESC_RATEMCS12) ? "MCS12" :\
+	(rate == DESC_RATEMCS13) ? "MCS13" :\
+	(rate == DESC_RATEMCS14) ? "MCS14" :\
+	(rate == DESC_RATEMCS15) ? "MCS15" :\
+	(rate == DESC_RATEMCS16) ? "MCS16" :\
+	(rate == DESC_RATEMCS17) ? "MCS17" :\
+	(rate == DESC_RATEMCS18) ? "MCS18" :\
+	(rate == DESC_RATEMCS19) ? "MCS19" :\
+	(rate == DESC_RATEMCS20) ? "MCS20" :\
+	(rate == DESC_RATEMCS21) ? "MCS21" :\
+	(rate == DESC_RATEMCS22) ? "MCS22" :\
+	(rate == DESC_RATEMCS23) ? "MCS23" :\
+	(rate == DESC_RATEMCS24) ? "MCS24" :\
+	(rate == DESC_RATEMCS25) ? "MCS25" :\
+	(rate == DESC_RATEMCS26) ? "MCS26" :\
+	(rate == DESC_RATEMCS27) ? "MCS27" :\
+	(rate == DESC_RATEMCS28) ? "MCS28" :\
+	(rate == DESC_RATEMCS29) ? "MCS29" :\
+	(rate == DESC_RATEMCS30) ? "MCS30" :\
+	(rate == DESC_RATEMCS31) ? "MCS31" :\
+	(rate == DESC_RATEVHTSS1MCS0) ? "VHTSS1MCS0" :\
+	(rate == DESC_RATEVHTSS1MCS1) ? "VHTSS1MCS1" :\
+	(rate == DESC_RATEVHTSS1MCS2) ? "VHTSS1MCS2" :\
+	(rate == DESC_RATEVHTSS1MCS3) ? "VHTSS1MCS3" :\
+	(rate == DESC_RATEVHTSS1MCS4) ? "VHTSS1MCS4" :\
+	(rate == DESC_RATEVHTSS1MCS5) ? "VHTSS1MCS5" :\
+	(rate == DESC_RATEVHTSS1MCS6) ? "VHTSS1MCS6" :\
+	(rate == DESC_RATEVHTSS1MCS7) ? "VHTSS1MCS7" :\
+	(rate == DESC_RATEVHTSS1MCS8) ? "VHTSS1MCS8" :\
+	(rate == DESC_RATEVHTSS1MCS9) ? "VHTSS1MCS9" :\
+	(rate == DESC_RATEVHTSS2MCS0) ? "VHTSS2MCS0" :\
+	(rate == DESC_RATEVHTSS2MCS1) ? "VHTSS2MCS1" :\
+	(rate == DESC_RATEVHTSS2MCS2) ? "VHTSS2MCS2" :\
+	(rate == DESC_RATEVHTSS2MCS3) ? "VHTSS2MCS3" :\
+	(rate == DESC_RATEVHTSS2MCS4) ? "VHTSS2MCS4" :\
+	(rate == DESC_RATEVHTSS2MCS5) ? "VHTSS2MCS5" :\
+	(rate == DESC_RATEVHTSS2MCS6) ? "VHTSS2MCS6" :\
+	(rate == DESC_RATEVHTSS2MCS7) ? "VHTSS2MCS7" :\
+	(rate == DESC_RATEVHTSS2MCS8) ? "VHTSS2MCS8" :\
+	(rate == DESC_RATEVHTSS2MCS9) ? "VHTSS2MCS9" :\
+	(rate == DESC_RATEVHTSS3MCS0) ? "VHTSS3MCS0" :\
+	(rate == DESC_RATEVHTSS3MCS1) ? "VHTSS3MCS1" :\
+	(rate == DESC_RATEVHTSS3MCS2) ? "VHTSS3MCS2" :\
+	(rate == DESC_RATEVHTSS3MCS3) ? "VHTSS3MCS3" :\
+	(rate == DESC_RATEVHTSS3MCS4) ? "VHTSS3MCS4" :\
+	(rate == DESC_RATEVHTSS3MCS5) ? "VHTSS3MCS5" :\
+	(rate == DESC_RATEVHTSS3MCS6) ? "VHTSS3MCS6" :\
+	(rate == DESC_RATEVHTSS3MCS7) ? "VHTSS3MCS7" :\
+	(rate == DESC_RATEVHTSS3MCS8) ? "VHTSS3MCS8" :\
+	(rate == DESC_RATEVHTSS3MCS9) ? "VHTSS3MCS9" :\
+	(rate == DESC_RATEVHTSS4MCS0) ? "VHTSS4MCS0" :\
+	(rate == DESC_RATEVHTSS4MCS1) ? "VHTSS4MCS1" :\
+	(rate == DESC_RATEVHTSS4MCS2) ? "VHTSS4MCS2" :\
+	(rate == DESC_RATEVHTSS4MCS3) ? "VHTSS4MCS3" :\
+	(rate == DESC_RATEVHTSS4MCS4) ? "VHTSS4MCS4" :\
+	(rate == DESC_RATEVHTSS4MCS5) ? "VHTSS4MCS5" :\
+	(rate == DESC_RATEVHTSS4MCS6) ? "VHTSS4MCS6" :\
+	(rate == DESC_RATEVHTSS4MCS7) ? "VHTSS4MCS7" :\
+	(rate == DESC_RATEVHTSS4MCS8) ? "VHTSS4MCS8" :\
+	(rate == DESC_RATEVHTSS4MCS9) ? "VHTSS4MCS9" :\
+	"UNKNOWN"
+
+enum hw_data_rate {
+	HRATE_CCK1		= 0x0,
+	HRATE_CCK2		= 0x1,
+	HRATE_CCK5_5	= 0x2,
+	HRATE_CCK11		= 0x3,
+	HRATE_OFDM6		= 0x4,
+	HRATE_OFDM9		= 0x5,
+	HRATE_OFDM12	= 0x6,
+	HRATE_OFDM18	= 0x7,
+	HRATE_OFDM24	= 0x8,
+	HRATE_OFDM36	= 0x9,
+	HRATE_OFDM48	= 0xA,
+	HRATE_OFDM54	= 0xB,
+	HRATE_MCS0		= 0x80,
+	HRATE_MCS1		= 0x81,
+	HRATE_MCS2		= 0x82,
+	HRATE_MCS3		= 0x83,
+	HRATE_MCS4		= 0x84,
+	HRATE_MCS5		= 0x85,
+	HRATE_MCS6		= 0x86,
+	HRATE_MCS7		= 0x87,
+	HRATE_MCS8		= 0x88,
+	HRATE_MCS9		= 0x89,
+	HRATE_MCS10		= 0x8A,
+	HRATE_MCS11		= 0x8B,
+	HRATE_MCS12		= 0x8C,
+	HRATE_MCS13		= 0x8D,
+	HRATE_MCS14		= 0x8E,
+	HRATE_MCS15		= 0x8F,
+	HRATE_MCS16		= 0x90,
+	HRATE_MCS17		= 0x91,
+	HRATE_MCS18		= 0x92,
+	HRATE_MCS19		= 0x93,
+	HRATE_MCS20		= 0x94,
+	HRATE_MCS21		= 0x95,
+	HRATE_MCS22		= 0x96,
+	HRATE_MCS23		= 0x97,
+	HRATE_MCS24		= 0x98,
+	HRATE_MCS25		= 0x99,
+	HRATE_MCS26		= 0x9A,
+	HRATE_MCS27		= 0x9B,
+	HRATE_MCS28		= 0x9C,
+	HRATE_MCS29		= 0x9D,
+	HRATE_MCS30		= 0x9E,
+	HRATE_MCS31		= 0x9F,
+	HRATE_VHT_NSS1_MCS0	= 0x100,
+	HRATE_VHT_NSS1_MCS1	= 0x101,
+	HRATE_VHT_NSS1_MCS2	= 0x102,
+	HRATE_VHT_NSS1_MCS3	= 0x103,
+	HRATE_VHT_NSS1_MCS4	= 0x104,
+	HRATE_VHT_NSS1_MCS5	= 0x105,
+	HRATE_VHT_NSS1_MCS6	= 0x106,
+	HRATE_VHT_NSS1_MCS7	= 0x107,
+	HRATE_VHT_NSS1_MCS8	= 0x108,
+	HRATE_VHT_NSS1_MCS9	= 0x109,
+	HRATE_VHT_NSS2_MCS0	= 0x110,
+	HRATE_VHT_NSS2_MCS1	= 0x111,
+	HRATE_VHT_NSS2_MCS2	= 0x112,
+	HRATE_VHT_NSS2_MCS3	= 0x113,
+	HRATE_VHT_NSS2_MCS4	= 0x114,
+	HRATE_VHT_NSS2_MCS5	= 0x115,
+	HRATE_VHT_NSS2_MCS6	= 0x116,
+	HRATE_VHT_NSS2_MCS7	= 0x117,
+	HRATE_VHT_NSS2_MCS8	= 0x118,
+	HRATE_VHT_NSS2_MCS9	= 0x119,
+	HRATE_VHT_NSS3_MCS0	= 0x120,
+	HRATE_VHT_NSS3_MCS1	= 0x121,
+	HRATE_VHT_NSS3_MCS2	= 0x122,
+	HRATE_VHT_NSS3_MCS3	= 0x123,
+	HRATE_VHT_NSS3_MCS4	= 0x124,
+	HRATE_VHT_NSS3_MCS5	= 0x125,
+	HRATE_VHT_NSS3_MCS6	= 0x126,
+	HRATE_VHT_NSS3_MCS7	= 0x127,
+	HRATE_VHT_NSS3_MCS8	= 0x128,
+	HRATE_VHT_NSS3_MCS9	= 0x129,
+	HRATE_VHT_NSS4_MCS0	= 0x130,
+	HRATE_VHT_NSS4_MCS1	= 0x131,
+	HRATE_VHT_NSS4_MCS2	= 0x132,
+	HRATE_VHT_NSS4_MCS3	= 0x133,
+	HRATE_VHT_NSS4_MCS4	= 0x134,
+	HRATE_VHT_NSS4_MCS5	= 0x135,
+	HRATE_VHT_NSS4_MCS6	= 0x136,
+	HRATE_VHT_NSS4_MCS7	= 0x137,
+	HRATE_VHT_NSS4_MCS8	= 0x138,
+	HRATE_VHT_NSS4_MCS9	= 0x139,
+	HRATE_HE_NSS1_MCS0	= 0x180,
+	HRATE_HE_NSS1_MCS1	= 0x181,
+	HRATE_HE_NSS1_MCS2	= 0x182,
+	HRATE_HE_NSS1_MCS3	= 0x183,
+	HRATE_HE_NSS1_MCS4	= 0x184,
+	HRATE_HE_NSS1_MCS5	= 0x185,
+	HRATE_HE_NSS1_MCS6	= 0x186,
+	HRATE_HE_NSS1_MCS7	= 0x187,
+	HRATE_HE_NSS1_MCS8	= 0x188,
+	HRATE_HE_NSS1_MCS9	= 0x189,
+	HRATE_HE_NSS1_MCS10	= 0x18A,
+	HRATE_HE_NSS1_MCS11	= 0x18B,
+	HRATE_HE_NSS2_MCS0	= 0x190,
+	HRATE_HE_NSS2_MCS1	= 0x191,
+	HRATE_HE_NSS2_MCS2	= 0x192,
+	HRATE_HE_NSS2_MCS3	= 0x193,
+	HRATE_HE_NSS2_MCS4	= 0x194,
+	HRATE_HE_NSS2_MCS5	= 0x195,
+	HRATE_HE_NSS2_MCS6	= 0x196,
+	HRATE_HE_NSS2_MCS7	= 0x197,
+	HRATE_HE_NSS2_MCS8	= 0x198,
+	HRATE_HE_NSS2_MCS9	= 0x199,
+	HRATE_HE_NSS2_MCS10	= 0x19A,
+	HRATE_HE_NSS2_MCS11	= 0x19B,
+	HRATE_HE_NSS3_MCS0	= 0x1A0,
+	HRATE_HE_NSS3_MCS1	= 0x1A1,
+	HRATE_HE_NSS3_MCS2	= 0x1A2,
+	HRATE_HE_NSS3_MCS3	= 0x1A3,
+	HRATE_HE_NSS3_MCS4	= 0x1A4,
+	HRATE_HE_NSS3_MCS5	= 0x1A5,
+	HRATE_HE_NSS3_MCS6	= 0x1A6,
+	HRATE_HE_NSS3_MCS7	= 0x1A7,
+	HRATE_HE_NSS3_MCS8	= 0x1A8,
+	HRATE_HE_NSS3_MCS9	= 0x1A9,
+	HRATE_HE_NSS3_MCS10	= 0x1AA,
+	HRATE_HE_NSS3_MCS11	= 0x1AB,
+	HRATE_HE_NSS4_MCS0	= 0x1B0,
+	HRATE_HE_NSS4_MCS1	= 0x1B1,
+	HRATE_HE_NSS4_MCS2	= 0x1B2,
+	HRATE_HE_NSS4_MCS3	= 0x1B3,
+	HRATE_HE_NSS4_MCS4	= 0x1B4,
+	HRATE_HE_NSS4_MCS5	= 0x1B5,
+	HRATE_HE_NSS4_MCS6	= 0x1B6,
+	HRATE_HE_NSS4_MCS7	= 0x1B7,
+	HRATE_HE_NSS4_MCS8	= 0x1B8,
+	HRATE_HE_NSS4_MCS9	= 0x1B9,
+	HRATE_HE_NSS4_MCS10	= 0x1BA,
+	HRATE_HE_NSS4_MCS11	= 0x1BB
+};
+
+static inline u16 mrate_to_hwrate(u16 rate)
+{
+	u16 ret = DESC_RATE1M;
+
+	switch (rate) {
+	case MGN_1M:
+		ret = RTW_DATA_RATE_CCK1;
+		break;
+	case MGN_2M:
+		ret = RTW_DATA_RATE_CCK2;
+		break;
+	case MGN_5_5M:
+		ret = RTW_DATA_RATE_CCK5_5;
+		break;
+	case MGN_11M:
+		ret = RTW_DATA_RATE_CCK11;
+		break;
+	case MGN_6M:
+		ret = RTW_DATA_RATE_OFDM6;
+		break;
+	case MGN_9M:
+		ret = RTW_DATA_RATE_OFDM9;
+		break;
+	case MGN_12M:
+		ret = RTW_DATA_RATE_OFDM12;
+		break;
+	case MGN_18M:
+		ret = RTW_DATA_RATE_OFDM18;
+		break;
+	case MGN_24M:
+		ret = RTW_DATA_RATE_OFDM24;
+		break;
+	case MGN_36M:
+		ret = RTW_DATA_RATE_OFDM36;
+		break;
+	case MGN_48M:
+		ret = RTW_DATA_RATE_OFDM48;
+		break;
+	case MGN_54M:
+		ret = RTW_DATA_RATE_OFDM54;
+		break;
+
+	case MGN_MCS0:
+		ret = RTW_DATA_RATE_MCS0;
+		break;
+	case MGN_MCS1:
+		ret = RTW_DATA_RATE_MCS1;
+		break;
+	case MGN_MCS2:
+		ret = RTW_DATA_RATE_MCS2;
+		break;
+	case MGN_MCS3:
+		ret = RTW_DATA_RATE_MCS3;
+		break;
+	case MGN_MCS4:
+		ret = RTW_DATA_RATE_MCS4;
+		break;
+	case MGN_MCS5:
+		ret = RTW_DATA_RATE_MCS5;
+		break;
+	case MGN_MCS6:
+		ret = RTW_DATA_RATE_MCS6;
+		break;
+	case MGN_MCS7:
+		ret = RTW_DATA_RATE_MCS7;
+		break;
+	case MGN_MCS8:
+		ret = RTW_DATA_RATE_MCS8;
+		break;
+	case MGN_MCS9:
+		ret = RTW_DATA_RATE_MCS9;
+		break;
+	case MGN_MCS10:
+		ret = RTW_DATA_RATE_MCS10;
+		break;
+	case MGN_MCS11:
+		ret = RTW_DATA_RATE_MCS11;
+		break;
+	case MGN_MCS12:
+		ret = RTW_DATA_RATE_MCS12;
+		break;
+	case MGN_MCS13:
+		ret = RTW_DATA_RATE_MCS13;
+		break;
+	case MGN_MCS14:
+		ret = RTW_DATA_RATE_MCS14;
+		break;
+	case MGN_MCS15:
+		ret = RTW_DATA_RATE_MCS15;
+		break;
+	case MGN_MCS16:
+		ret = RTW_DATA_RATE_MCS16;
+		break;
+	case MGN_MCS17:
+		ret = RTW_DATA_RATE_MCS17;
+		break;
+	case MGN_MCS18:
+		ret = RTW_DATA_RATE_MCS18;
+		break;
+	case MGN_MCS19:
+		ret = RTW_DATA_RATE_MCS19;
+		break;
+	case MGN_MCS20:
+		ret = RTW_DATA_RATE_MCS20;
+		break;
+	case MGN_MCS21:
+		ret = RTW_DATA_RATE_MCS21;
+		break;
+	case MGN_MCS22:
+		ret = RTW_DATA_RATE_MCS22;
+		break;
+	case MGN_MCS23:
+		ret = RTW_DATA_RATE_MCS23;
+		break;
+	case MGN_MCS24:
+		ret = RTW_DATA_RATE_MCS24;
+		break;
+	case MGN_MCS25:
+		ret = RTW_DATA_RATE_MCS25;
+		break;
+	case MGN_MCS26:
+		ret = RTW_DATA_RATE_MCS26;
+		break;
+	case MGN_MCS27:
+		ret = RTW_DATA_RATE_MCS27;
+		break;
+	case MGN_MCS28:
+		ret = RTW_DATA_RATE_MCS28;
+		break;
+	case MGN_MCS29:
+		ret = RTW_DATA_RATE_MCS29;
+		break;
+	case MGN_MCS30:
+		ret = RTW_DATA_RATE_MCS30;
+		break;
+	case MGN_MCS31:
+		ret = RTW_DATA_RATE_MCS31;
+		break;
+
+	case MGN_VHT1SS_MCS0:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS0;
+		break;
+	case MGN_VHT1SS_MCS1:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS1;
+		break;
+	case MGN_VHT1SS_MCS2:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS2;
+		break;
+	case MGN_VHT1SS_MCS3:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS3;
+		break;
+	case MGN_VHT1SS_MCS4:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS4;
+		break;
+	case MGN_VHT1SS_MCS5:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS5;
+		break;
+	case MGN_VHT1SS_MCS6:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS6;
+		break;
+	case MGN_VHT1SS_MCS7:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS7;
+		break;
+	case MGN_VHT1SS_MCS8:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS8;
+		break;
+	case MGN_VHT1SS_MCS9:
+		ret = RTW_DATA_RATE_VHT_NSS1_MCS9;
+		break;
+	case MGN_VHT2SS_MCS0:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS0;
+		break;
+	case MGN_VHT2SS_MCS1:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS1;
+		break;
+	case MGN_VHT2SS_MCS2:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS2;
+		break;
+	case MGN_VHT2SS_MCS3:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS3;
+		break;
+	case MGN_VHT2SS_MCS4:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS4;
+		break;
+	case MGN_VHT2SS_MCS5:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS5;
+		break;
+	case MGN_VHT2SS_MCS6:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS6;
+		break;
+	case MGN_VHT2SS_MCS7:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS7;
+		break;
+	case MGN_VHT2SS_MCS8:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS8;
+		break;
+	case MGN_VHT2SS_MCS9:
+		ret = RTW_DATA_RATE_VHT_NSS2_MCS9;
+		break;
+	case MGN_VHT3SS_MCS0:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS0;
+		break;
+	case MGN_VHT3SS_MCS1:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS1;
+		break;
+	case MGN_VHT3SS_MCS2:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS2;
+		break;
+	case MGN_VHT3SS_MCS3:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS3;
+		break;
+	case MGN_VHT3SS_MCS4:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS4;
+		break;
+	case MGN_VHT3SS_MCS5:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS5;
+		break;
+	case MGN_VHT3SS_MCS6:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS6;
+		break;
+	case MGN_VHT3SS_MCS7:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS7;
+		break;
+	case MGN_VHT3SS_MCS8:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS8;
+		break;
+	case MGN_VHT3SS_MCS9:
+		ret = RTW_DATA_RATE_VHT_NSS3_MCS9;
+		break;
+	case MGN_VHT4SS_MCS0:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS0;
+		break;
+	case MGN_VHT4SS_MCS1:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS1;
+		break;
+	case MGN_VHT4SS_MCS2:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS2;
+		break;
+	case MGN_VHT4SS_MCS3:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS3;
+		break;
+	case MGN_VHT4SS_MCS4:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS4;
+		break;
+	case MGN_VHT4SS_MCS5:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS5;
+		break;
+	case MGN_VHT4SS_MCS6:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS6;
+		break;
+	case MGN_VHT4SS_MCS7:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS7;
+		break;
+	case MGN_VHT4SS_MCS8:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS8;
+		break;
+	case MGN_VHT4SS_MCS9:
+		ret = RTW_DATA_RATE_VHT_NSS4_MCS9;
+		break;
+
+	case MGN_HE1SS_MCS0:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS0;
+		break;
+	case MGN_HE1SS_MCS1:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS1;
+		break;
+	case MGN_HE1SS_MCS2:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS2;
+		break;
+	case MGN_HE1SS_MCS3:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS3;
+		break;
+	case MGN_HE1SS_MCS4:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS4;
+		break;
+	case MGN_HE1SS_MCS5:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS5;
+		break;
+	case MGN_HE1SS_MCS6:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS6;
+		break;
+	case MGN_HE1SS_MCS7:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS7;
+		break;
+	case MGN_HE1SS_MCS8:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS8;
+		break;
+	case MGN_HE1SS_MCS9:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS9;
+		break;
+	case MGN_HE1SS_MCS10:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS10;
+		break;
+	case MGN_HE1SS_MCS11:
+		ret = RTW_DATA_RATE_HE_NSS1_MCS11;
+		break;
+	case MGN_HE2SS_MCS0:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS0;
+		break;
+	case MGN_HE2SS_MCS1:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS1;
+		break;
+	case MGN_HE2SS_MCS2:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS2;
+		break;
+	case MGN_HE2SS_MCS3:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS3;
+		break;
+	case MGN_HE2SS_MCS4:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS4;
+		break;
+	case MGN_HE2SS_MCS5:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS5;
+		break;
+	case MGN_HE2SS_MCS6:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS6;
+		break;
+	case MGN_HE2SS_MCS7:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS7;
+		break;
+	case MGN_HE2SS_MCS8:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS8;
+		break;
+	case MGN_HE2SS_MCS9:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS9;
+		break;
+	case MGN_HE2SS_MCS10:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS10;
+		break;
+	case MGN_HE2SS_MCS11:
+		ret = RTW_DATA_RATE_HE_NSS2_MCS11;
+		break;
+	case MGN_HE3SS_MCS0:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS0;
+		break;
+	case MGN_HE3SS_MCS1:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS1;
+		break;
+	case MGN_HE3SS_MCS2:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS2;
+		break;
+	case MGN_HE3SS_MCS3:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS3;
+		break;
+	case MGN_HE3SS_MCS4:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS4;
+		break;
+	case MGN_HE3SS_MCS5:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS5;
+		break;
+	case MGN_HE3SS_MCS6:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS6;
+		break;
+	case MGN_HE3SS_MCS7:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS7;
+		break;
+	case MGN_HE3SS_MCS8:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS8;
+		break;
+	case MGN_HE3SS_MCS9:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS9;
+		break;
+	case MGN_HE3SS_MCS10:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS10;
+		break;
+	case MGN_HE3SS_MCS11:
+		ret = RTW_DATA_RATE_HE_NSS3_MCS11;
+		break;
+	case MGN_HE4SS_MCS0:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS0;
+		break;
+	case MGN_HE4SS_MCS1:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS1;
+		break;
+	case MGN_HE4SS_MCS2:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS2;
+		break;
+	case MGN_HE4SS_MCS3:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS3;
+		break;
+	case MGN_HE4SS_MCS4:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS4;
+		break;
+	case MGN_HE4SS_MCS5:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS5;
+		break;
+	case MGN_HE4SS_MCS6:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS6;
+		break;
+	case MGN_HE4SS_MCS7:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS7;
+		break;
+	case MGN_HE4SS_MCS8:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS8;
+		break;
+	case MGN_HE4SS_MCS9:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS9;
+		break;
+	case MGN_HE4SS_MCS10:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS10;
+		break;
+	case MGN_HE4SS_MCS11:
+		ret = RTW_DATA_RATE_HE_NSS4_MCS11;
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static inline u8 hwrate_to_mrate(u16 rate)
+{
+	u16 ret_rate = MGN_1M;
+
+	switch (rate) {
+
+	case RTW_DATA_RATE_CCK1:
+		ret_rate = MGN_1M;
+		break;
+	case RTW_DATA_RATE_CCK2:
+		ret_rate = MGN_2M;
+		break;
+	case RTW_DATA_RATE_CCK5_5:
+		ret_rate = MGN_5_5M;
+		break;
+	case RTW_DATA_RATE_CCK11:
+		ret_rate = MGN_11M;
+		break;
+	case RTW_DATA_RATE_OFDM6:
+		ret_rate = MGN_6M;
+		break;
+	case RTW_DATA_RATE_OFDM9:
+		ret_rate = MGN_9M;
+		break;
+	case RTW_DATA_RATE_OFDM12:
+		ret_rate = MGN_12M;
+		break;
+	case RTW_DATA_RATE_OFDM18:
+		ret_rate = MGN_18M;
+		break;
+	case RTW_DATA_RATE_OFDM24:
+		ret_rate = MGN_24M;
+		break;
+	case RTW_DATA_RATE_OFDM36:
+		ret_rate = MGN_36M;
+		break;
+	case RTW_DATA_RATE_OFDM48:
+		ret_rate = MGN_48M;
+		break;
+	case RTW_DATA_RATE_OFDM54:
+		ret_rate = MGN_54M;
+		break;
+	case RTW_DATA_RATE_MCS0:
+		ret_rate = MGN_MCS0;
+		break;
+	case RTW_DATA_RATE_MCS1:
+		ret_rate = MGN_MCS1;
+		break;
+	case RTW_DATA_RATE_MCS2:
+		ret_rate = MGN_MCS2;
+		break;
+	case RTW_DATA_RATE_MCS3:
+		ret_rate = MGN_MCS3;
+		break;
+	case RTW_DATA_RATE_MCS4:
+		ret_rate = MGN_MCS4;
+		break;
+	case RTW_DATA_RATE_MCS5:
+		ret_rate = MGN_MCS5;
+		break;
+	case RTW_DATA_RATE_MCS6:
+		ret_rate = MGN_MCS6;
+		break;
+	case RTW_DATA_RATE_MCS7:
+		ret_rate = MGN_MCS7;
+		break;
+	case RTW_DATA_RATE_MCS8:
+		ret_rate = MGN_MCS8;
+		break;
+	case RTW_DATA_RATE_MCS9:
+		ret_rate = MGN_MCS9;
+		break;
+	case RTW_DATA_RATE_MCS10:
+		ret_rate = MGN_MCS10;
+		break;
+	case RTW_DATA_RATE_MCS11:
+		ret_rate = MGN_MCS11;
+		break;
+	case RTW_DATA_RATE_MCS12:
+		ret_rate = MGN_MCS12;
+		break;
+	case RTW_DATA_RATE_MCS13:
+		ret_rate = MGN_MCS13;
+		break;
+	case RTW_DATA_RATE_MCS14:
+		ret_rate = MGN_MCS14;
+		break;
+	case RTW_DATA_RATE_MCS15:
+		ret_rate = MGN_MCS15;
+		break;
+	case RTW_DATA_RATE_MCS16:
+		ret_rate = MGN_MCS16;
+		break;
+	case RTW_DATA_RATE_MCS17:
+		ret_rate = MGN_MCS17;
+		break;
+	case RTW_DATA_RATE_MCS18:
+		ret_rate = MGN_MCS18;
+		break;
+	case RTW_DATA_RATE_MCS19:
+		ret_rate = MGN_MCS19;
+		break;
+	case RTW_DATA_RATE_MCS20:
+		ret_rate = MGN_MCS20;
+		break;
+	case RTW_DATA_RATE_MCS21:
+		ret_rate = MGN_MCS21;
+		break;
+	case RTW_DATA_RATE_MCS22:
+		ret_rate = MGN_MCS22;
+		break;
+	case RTW_DATA_RATE_MCS23:
+		ret_rate = MGN_MCS23;
+		break;
+	case RTW_DATA_RATE_MCS24:
+		ret_rate = MGN_MCS24;
+		break;
+	case RTW_DATA_RATE_MCS25:
+		ret_rate = MGN_MCS25;
+		break;
+	case RTW_DATA_RATE_MCS26:
+		ret_rate = MGN_MCS26;
+		break;
+	case RTW_DATA_RATE_MCS27:
+		ret_rate = MGN_MCS27;
+		break;
+	case RTW_DATA_RATE_MCS28:
+		ret_rate = MGN_MCS28;
+		break;
+	case RTW_DATA_RATE_MCS29:
+		ret_rate = MGN_MCS29;
+		break;
+	case RTW_DATA_RATE_MCS30:
+		ret_rate = MGN_MCS30;
+		break;
+	case RTW_DATA_RATE_MCS31:
+		ret_rate = MGN_MCS31;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS0:
+		ret_rate = MGN_VHT1SS_MCS0;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS1:
+		ret_rate = MGN_VHT1SS_MCS1;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS2:
+		ret_rate = MGN_VHT1SS_MCS2;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS3:
+		ret_rate = MGN_VHT1SS_MCS3;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS4:
+		ret_rate = MGN_VHT1SS_MCS4;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS5:
+		ret_rate = MGN_VHT1SS_MCS5;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS6:
+		ret_rate = MGN_VHT1SS_MCS6;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS7:
+		ret_rate = MGN_VHT1SS_MCS7;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS8:
+		ret_rate = MGN_VHT1SS_MCS8;
+		break;
+	case RTW_DATA_RATE_VHT_NSS1_MCS9:
+		ret_rate = MGN_VHT1SS_MCS9;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS0:
+		ret_rate = MGN_VHT2SS_MCS0;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS1:
+		ret_rate = MGN_VHT2SS_MCS1;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS2:
+		ret_rate = MGN_VHT2SS_MCS2;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS3:
+		ret_rate = MGN_VHT2SS_MCS3;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS4:
+		ret_rate = MGN_VHT2SS_MCS4;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS5:
+		ret_rate = MGN_VHT2SS_MCS5;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS6:
+		ret_rate = MGN_VHT2SS_MCS6;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS7:
+		ret_rate = MGN_VHT2SS_MCS7;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS8:
+		ret_rate = MGN_VHT2SS_MCS8;
+		break;
+	case RTW_DATA_RATE_VHT_NSS2_MCS9:
+		ret_rate = MGN_VHT2SS_MCS9;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS0:
+		ret_rate = MGN_VHT3SS_MCS0;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS1:
+		ret_rate = MGN_VHT3SS_MCS1;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS2:
+		ret_rate = MGN_VHT3SS_MCS2;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS3:
+		ret_rate = MGN_VHT3SS_MCS3;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS4:
+		ret_rate = MGN_VHT3SS_MCS4;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS5:
+		ret_rate = MGN_VHT3SS_MCS5;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS6:
+		ret_rate = MGN_VHT3SS_MCS6;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS7:
+		ret_rate = MGN_VHT3SS_MCS7;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS8:
+		ret_rate = MGN_VHT3SS_MCS8;
+		break;
+	case RTW_DATA_RATE_VHT_NSS3_MCS9:
+		ret_rate = MGN_VHT3SS_MCS9;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS0:
+		ret_rate = MGN_VHT4SS_MCS0;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS1:
+		ret_rate = MGN_VHT4SS_MCS1;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS2:
+		ret_rate = MGN_VHT4SS_MCS2;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS3:
+		ret_rate = MGN_VHT4SS_MCS3;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS4:
+		ret_rate = MGN_VHT4SS_MCS4;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS5:
+		ret_rate = MGN_VHT4SS_MCS5;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS6:
+		ret_rate = MGN_VHT4SS_MCS6;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS7:
+		ret_rate = MGN_VHT4SS_MCS7;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS8:
+		ret_rate = MGN_VHT4SS_MCS8;
+		break;
+	case RTW_DATA_RATE_VHT_NSS4_MCS9:
+		ret_rate = MGN_VHT4SS_MCS9;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS0:
+		ret_rate = MGN_HE1SS_MCS0;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS1:
+		ret_rate = MGN_HE1SS_MCS1;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS2:
+		ret_rate = MGN_HE1SS_MCS2;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS3:
+		ret_rate = MGN_HE1SS_MCS3;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS4:
+		ret_rate = MGN_HE1SS_MCS4;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS5:
+		ret_rate = MGN_HE1SS_MCS5;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS6:
+		ret_rate = MGN_HE1SS_MCS6;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS7:
+		ret_rate = MGN_HE1SS_MCS7;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS8:
+		ret_rate = MGN_HE1SS_MCS8;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS9:
+		ret_rate = MGN_HE1SS_MCS9;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS10:
+		ret_rate = MGN_HE1SS_MCS10;
+		break;
+	case RTW_DATA_RATE_HE_NSS1_MCS11:
+		ret_rate = MGN_HE1SS_MCS11;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS0:
+		ret_rate = MGN_HE2SS_MCS0;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS1:
+		ret_rate = MGN_HE2SS_MCS1;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS2:
+		ret_rate = MGN_HE2SS_MCS2;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS3:
+		ret_rate = MGN_HE2SS_MCS3;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS4:
+		ret_rate = MGN_HE2SS_MCS4;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS5:
+		ret_rate = MGN_HE2SS_MCS5;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS6:
+		ret_rate = MGN_HE2SS_MCS6;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS7:
+		ret_rate = MGN_HE2SS_MCS7;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS8:
+		ret_rate = MGN_HE2SS_MCS8;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS9:
+		ret_rate = MGN_HE2SS_MCS9;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS10:
+		ret_rate = MGN_HE2SS_MCS10;
+		break;
+	case RTW_DATA_RATE_HE_NSS2_MCS11:
+		ret_rate = MGN_HE2SS_MCS11;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS0:
+		ret_rate = MGN_HE3SS_MCS0;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS1:
+		ret_rate = MGN_HE3SS_MCS1;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS2:
+		ret_rate = MGN_HE3SS_MCS2;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS3:
+		ret_rate = MGN_HE3SS_MCS3;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS4:
+		ret_rate = MGN_HE3SS_MCS4;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS5:
+		ret_rate = MGN_HE3SS_MCS5;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS6:
+		ret_rate = MGN_HE3SS_MCS6;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS7:
+		ret_rate = MGN_HE3SS_MCS7;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS8:
+		ret_rate = MGN_HE3SS_MCS8;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS9:
+		ret_rate = MGN_HE3SS_MCS9;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS10:
+		ret_rate = MGN_HE3SS_MCS10;
+		break;
+	case RTW_DATA_RATE_HE_NSS3_MCS11:
+		ret_rate = MGN_HE3SS_MCS11;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS0:
+		ret_rate = MGN_HE4SS_MCS0;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS1:
+		ret_rate = MGN_HE4SS_MCS1;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS2:
+		ret_rate = MGN_HE4SS_MCS2;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS3:
+		ret_rate = MGN_HE4SS_MCS3;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS4:
+		ret_rate = MGN_HE4SS_MCS4;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS5:
+		ret_rate = MGN_HE4SS_MCS5;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS6:
+		ret_rate = MGN_HE4SS_MCS6;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS7:
+		ret_rate = MGN_HE4SS_MCS7;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS8:
+		ret_rate = MGN_HE4SS_MCS8;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS9:
+		ret_rate = MGN_HE4SS_MCS9;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS10:
+		ret_rate = MGN_HE4SS_MCS10;
+		break;
+	case RTW_DATA_RATE_HE_NSS4_MCS11:
+		ret_rate = MGN_HE4SS_MCS11;
+		break;
+
+	default:
+		break;
+	}
+
+	return ret_rate;
+}
+
+#endif /* __HAL_RATE_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/autoconf.h b/drivers/net/wireless/realtek/rtl8852bs/include/autoconf.h
new file mode 100644
index 000000000000..fcaba24b2aa5
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/autoconf.h
@@ -0,0 +1,345 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _AUTOCONF_H_
+#define _AUTOCONF_H_
+
+/***** temporarily flag *******/
+/*#define CONFIG_NO_FW*/
+/*#define CONFIG_DISABLE_ODM*/
+
+/*
+ * Debug Related Config
+ */
+#ifdef CONFIG_RTW_DEBUG
+	#define DBG	1	/* for ODM & BTC debug */
+#else /* !CONFIG_RTW_DEBUG */
+	#define DBG	0	/* for ODM & BTC debug */
+#endif /* CONFIG_RTW_DEBUG */
+
+
+/*#define DBG_XMIT_BUF*/
+/*#define DBG_XMIT_BUF_EXT*/
+/*#define CONFIG_FW_C2H_DEBUG*/
+
+/* debug memory management */
+#if 0
+	#define DBG_MEM_ALLOC
+	#define DBG_PHL_MEM_ALLOC
+	#define DBG_HAL_MAC_MEM_MOINTOR
+	#define DBG_HAL_MEM_MOINTOR
+#endif
+/*#define CONFIG_PHL_USE_KMEM_ALLOC*/
+
+/* debug TX/RX */
+/*#define DBG_XMIT_BLOCK*/
+/*#define DBG_RX_COUNTER_DUMP*/
+
+/*#define DBG_C2H_MAC_HIDDEN_RPT_HANDLE	1*/
+/*#define DBG_H2C_CONTENT*/
+
+#define CONFIG_ERROR_STATE_MONITOR
+/*#define CONFIG_MONITOR_OVERFLOW*/
+
+/*#define CONFIG_SDIO_INDIRECT_ACCESS*/
+/*#define DBG_SDIO_INDIRECT_ACCESS*/
+
+
+/*
+ * Work around Config
+ */
+#define RTW_WKARD_DIS_PROBE_REQ_RPT_TO_HOSTAPD
+
+#define RTW_WKARD_CORE_RSSI_V1
+#ifdef RTW_WKARD_CORE_RSSI_V1
+#define CONFIG_RX_PSTS_PER_PKT
+#define CONFIG_SIGNAL_STAT_PROCESS
+/*#define DBG_RX_SIGNAL_DISPLAY_PROCESSING*/
+#endif
+
+#ifdef CONFIG_BTC
+#define RTK_WKARD_CORE_BTC_STBC_CAP
+#endif
+
+/* #define RTW_WKARD_SD_RESET_DISABLE */
+
+/*
+ * Public General Config
+ */
+#define AUTOCONF_INCLUDED
+#define DRV_NAME "rtl8852bs"
+
+#define CONFIG_SDIO_HCI
+
+
+/*
+ * Wi-Fi Functions Config
+ */
+
+/*#define CONFIG_RECV_REORDERING_CTRL*/
+
+#define CONFIG_80211N_HT
+#define CONFIG_80211AC_VHT
+#define CONFIG_80211AX_HE
+#ifdef CONFIG_80211AC_VHT
+	#ifndef CONFIG_80211N_HT
+		#define CONFIG_80211N_HT
+	#endif
+#endif
+#ifdef CONFIG_80211AX_HE
+	#ifndef CONFIG_80211N_HT
+		#define CONFIG_80211N_HT
+	#endif
+	#ifndef CONFIG_80211AC_VHT
+		#define CONFIG_80211AC_VHT
+	#endif
+#endif
+
+#define CONFIG_TX_AMSDU
+#ifdef CONFIG_TX_AMSDU
+	#define CONFIG_TX_AMSDU_SW_MODE
+#endif
+
+#define CONFIG_BEAMFORMING
+#ifdef CONFIG_BEAMFORMING
+/*#define RTW_WKARD_TX_DISABLE_BFEE*/
+#endif
+
+/* Set CONFIG_IOCTL_CFG80211 from Makefile */
+#ifdef CONFIG_IOCTL_CFG80211
+	/*
+	 * Indecate new sta asoc through cfg80211_new_sta
+	 * If kernel version >= 3.2 or
+	 * version < 3.2 but already apply cfg80211 patch,
+	 * RTW_USE_CFG80211_STA_EVENT must be defiend!
+	 */
+	/* Set RTW_USE_CFG80211_STA_EVENT from Makefile */
+	#define CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER
+	/* #define CONFIG_DEBUG_CFG80211 */
+	#define CONFIG_SET_SCAN_DENY_TIMER
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+#define CONFIG_AP_MODE
+#ifdef CONFIG_AP_MODE
+	#define CONFIG_NATIVEAP_MLME
+	#ifndef CONFIG_NATIVEAP_MLME
+		#define CONFIG_HOSTAPD_MLME
+	#endif
+	/*#define CONFIG_RTW_HOSTAPD_ACS*/
+	/*#define CONFIG_FIND_BEST_CHANNEL*/
+#endif
+
+
+#define CONFIG_P2P
+
+#ifdef CONFIG_P2P
+	#define CONFIG_WFD	/* Wi-Fi display */
+	#define CONFIG_P2P_REMOVE_GROUP_INFO
+	/*#define CONFIG_DBG_P2P*/
+	#define CONFIG_P2P_PS
+	/*#define CONFIG_P2P_IPS*/
+	#define CONFIG_P2P_OP_CHK_SOCIAL_CH
+	#define CONFIG_CFG80211_ONECHANNEL_UNDER_CONCURRENT  /* Replace CONFIG_P2P_CHK_INVITE_CH_LIST flag */
+	/*#define CONFIG_P2P_INVITE_IOT*/
+#endif /* CONFIG_P2P */
+
+/* Set CONFIG_TDLS from Makefile */
+#ifdef CONFIG_TDLS
+	#define CONFIG_TDLS_DRIVER_SETUP
+/*	#ifndef CONFIG_WFD
+		#define CONFIG_WFD
+	#endif*/
+/*	#define CONFIG_TDLS_AUTOSETUP*/
+	#define CONFIG_TDLS_AUTOCHECKALIVE
+	/*
+	 * Enable "CONFIG_TDLS_CH_SW" by default,
+	 * however limit it to only work in wifi logo test mode
+	 * but not in normal mode currently
+	 */
+	#define CONFIG_TDLS_CH_SW
+#endif /* CONFIG_TDLS */
+
+#define CONFIG_HW_RTS
+
+/*
+ * Hareware/Firmware Related Config
+ */
+/* Set CONFIG_ANTENNA_DIVERSITY from Makefile */
+
+/*#define CONFIG_RTW_LED*/
+#ifdef CONFIG_RTW_LED
+	/*#define CONFIG_RTW_SW_LED*/
+#endif /* CONFIG_RTW_LED */
+
+#define DISABLE_BB_RF		0
+
+#define CONFIG_TCP_CSUM_OFFLOAD_RX
+
+/*
+ * Interface Related Config
+ */
+/*#define CONFIG_SDIO_TX_ENABLE_AVAL_INT => Related MAC reg must setting => HAL-MAC ?? */
+#define CONFIG_SDIO_RX_COPY
+
+#define CONFIG_SDIO_RX_NETBUF_ALLOC_IN_PHL
+#define CONFIG_SDIO_READ_RXFF_IN_INT
+
+/*#define RTW_XMIT_THREAD_HIGH_PRIORITY*/
+/*#define RTW_XMIT_THREAD_CB_HIGH_PRIORITY*/
+/*#define RTW_RECV_THREAD_HIGH_PRIORITY*/
+
+#ifdef CONFIG_RTW_NAPI
+/*#define CONFIG_RTW_NAPI_DYNAMIC*/
+#define CONFIG_RTW_NAPI_V2
+#ifdef CONFIG_RTW_NAPI_V2
+#define CONFIG_RX_BATCH_IND
+#endif
+#endif
+
+#define CONFIG_QUOTA_TURBO_ENABLE
+
+#define MAX_XMITBUF_SZ	26624	/* 26 KB */
+#define MAX_RECVBUF_SZ	68608	/* 68 KB*/
+#define MAX_PHL_TX_RING_ENTRY_NUM 512
+#define MAX_PHL_RX_RING_ENTRY_NUM 536
+#define CONFIG_RTW_REDUCE_MEM
+#define CONFIG_SCAN_BACKOP_STA
+#define NR_XMITFRAME	MAX_PHL_TX_RING_ENTRY_NUM
+#define MAX_TX_RING_NUM		MAX_PHL_TX_RING_ENTRY_NUM
+#define RTW_MAX_FRAG_NUM 1
+#define NR_XMITFRAME_EXT 120
+#define RTW_MAX_FW_SIZE 0x80000
+
+
+/*fw reduce code size*/
+#define CONFIG_FW_SPECIFY_FROM_CORE
+#ifdef CONFIG_FW_SPECIFY_FROM_CORE
+#ifdef CONFIG_WOWLAN
+/* #define MAC_FW_CATEGORY_WOWLAN */
+/* Support APFv6 & MDNS OFFLOAD*/
+#define MAC_FW_CATEGORY_WOWLANCE
+#endif /*CONFIG_WOWLAN*/
+#define MAC_FW_8852B_U2
+#define MAC_FW_8852B_U3
+
+#define MAC_FW_CATEGORY_NIC  /*pwr gating*/
+/*#define MAC_FW_CATEGORY_NICCE*/  /*clock gating*/
+/*#define MAC_FW_CATEGORY_NIC_PLE*/
+#endif /*CONFIG_FW_SPECIFY_FROM_CORE*/
+
+/*
+ * Others
+ */
+/*#define CONFIG_MAC_LOOPBACK_DRIVER*/
+#define CONFIG_SKB_COPY		/* for amsdu */
+#define CONFIG_EMBEDDED_FWIMG
+/*#define CONFIG_FILE_FWIMG*/
+#define CONFIG_LONG_DELAY_ISSUE
+
+
+/*
+ * Platform
+ */
+
+
+/*
+ * Auto Config Section
+ */
+#ifdef CONFIG_MAC_LOOPBACK_DRIVER
+#undef CONFIG_IOCTL_CFG80211
+#undef CONFIG_AP_MODE
+#undef CONFIG_NATIVEAP_MLME
+#undef CONFIG_ANTENNA_DIVERSITY
+#endif /* CONFIG_MAC_LOOPBACK_DRIVER */
+
+#ifdef CONFIG_MP_INCLUDED
+	#define MP_DRIVER	1
+	/* disable unnecessary functions for MP*/
+	/*#undef CONFIG_ANTENNA_DIVERSITY*/
+#else /* !CONFIG_MP_INCLUDED */
+	#define MP_DRIVER	0
+#endif /* !CONFIG_MP_INCLUDED */
+
+#ifdef CONFIG_POWER_SAVE
+	#define CONFIG_RTW_IPS
+	#define CONFIG_RTW_LPS
+	#ifdef CONFIG_RTW_IPS
+		#define CONFIG_FWIPS
+	#endif
+	#if defined(CONFIG_RTW_IPS) || defined(CONFIG_RTW_LPS)
+		#define CONFIG_RTW_WKARD_PS_DEFAULT_OFF
+		#define CONFIG_PS_FW_DBG
+	#endif
+	#ifdef CONFIG_WOWLAN
+		#define CONFIG_RTW_IPS_WOW
+		#ifdef CONFIG_RTW_IPS_WOW
+			#define CONFIG_FWIPS_WOW
+		#endif /* CONFIG_RTW_IPS_WOW */
+		#define CONFIG_RTW_LPS_WOW
+	#endif /* CONFIG_WOWLAN */
+#endif /* CONFIG_POWER_SAVE */
+
+#ifdef CONFIG_RTW_WNM
+#define RTW_MBO_VALID_TIME 12000 /*large than full scan time */
+#endif /*CONFIG_RTW_WNM*/
+
+#ifdef CONFIG_WOWLAN
+	#define CONFIG_GTK_OL
+	/* #define CONFIG_ARP_KEEP_ALIVE */
+#ifdef CONFIG_WOW_APF
+	#define CONFIG_APF_VERSION 6
+	#define CONFIG_APF_RAM_SIZE 1792
+#endif
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_GPIO_WAKEUP
+	#ifndef WAKEUP_GPIO_IDX
+		#define WAKEUP_GPIO_IDX	10	/* WIFI Chip Side */
+	#endif /*!WAKEUP_GPIO_IDX*/
+#endif /* CONFIG_GPIO_WAKEUP */
+
+#ifdef CONFIG_ANTENNA_DIVERSITY
+#define CONFIG_HW_ANTENNA_DIVERSITY
+#endif /* CONFIG_ANTENNA_DIVERSITY */
+
+
+#define CONFIG_RTW_DISABLE_PHL_LOG
+#define CONFIG_LOGO_MODE_ADJUST_AMSDU_RXFIFO
+
+#define CONFIG_MSG_NUM 100
+#define SCAN_PER_CH_EX_TIME 350
+
+#define CONFIG_XMIT_ACK
+#ifdef CONFIG_XMIT_ACK
+	/*#define DBG_XMIT_ACK*/
+	#define CONFIG_XMIT_ACK_BY_CCX_RPT
+	#ifdef CONFIG_XMIT_ACK_BY_CCX_RPT
+		#define RTW_WKARD_CCX_RPT_LIMIT_CTRL
+		#define CONFIG_PHL_DEFAULT_MGNT_Q_RPT_EN
+	#endif
+	#define CONFIG_ACTIVE_KEEP_ALIVE_CHECK
+	#define RTW_WKARD_TX_NULL_WD_RP
+	#define RTW_MAX_MGMT_TX_MS_GAS 2000
+#endif /* CONFIG_XMIT_ACK */
+
+/*#define CONFIG_PHL_CPU_BALANCE_THREAD*/
+#ifdef CONFIG_PHL_CPU_BALANCE_THREAD
+#define CPU_ID_TX 2
+#define CPU_ID_TX_CB 3
+#define CPU_ID_RX_CB 1
+/*#define DBG_THREAD_PID*/
+/*#define DBG_CPU_INFO*/
+#endif /*CONFIG_PHL_CPU_BALANCE_THREAD*/
+
+#endif /* _AUTOCONF_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/basic_types.h b/drivers/net/wireless/realtek/rtl8852bs/include/basic_types.h
new file mode 100644
index 000000000000..c531df7ccdc1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/basic_types.h
@@ -0,0 +1,367 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __BASIC_TYPES_H__
+#define __BASIC_TYPES_H__
+
+
+#define SUCCESS	0
+#define FAIL	(-1)
+
+#ifndef TRUE
+	#define _TRUE	1
+#else
+	#define _TRUE	TRUE
+#endif
+
+#ifndef FALSE
+	#define _FALSE	0
+#else
+	#define _FALSE	FALSE
+#endif
+
+
+#ifdef PLATFORM_LINUX
+	#include <linux/version.h>
+	#include <linux/types.h>
+	#include <linux/module.h>
+	#include <linux/kernel.h>
+	#include <linux/init.h>
+	#include <linux/utsname.h>
+
+	typedef	signed int sint;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19))
+typedef _Bool bool;
+
+enum {
+	false	= 0,
+	true	= 1
+};
+#endif
+
+	typedef void (*proc_t)(void *);
+
+	typedef	__kernel_size_t	SIZE_T;
+	typedef	__kernel_ssize_t	SSIZE_T;
+	#define FIELD_OFFSET(s, field)	((SSIZE_T)&((s *)(0))->field)
+
+#endif /*PLATFORM_LINUX*/
+
+
+#ifdef PLATFORM_FREEBSD
+
+	typedef signed char s8;
+	typedef unsigned char u8;
+
+	typedef signed short s16;
+	typedef unsigned short u16;
+
+	typedef signed int s32;
+	typedef unsigned int u32;
+
+	typedef unsigned int	uint;
+	typedef	signed int		sint;
+
+	typedef signed long long s64;
+	typedef unsigned long long u64;
+
+	typedef u32 dma_addr_t;
+	typedef long atomic_t;
+
+	typedef void (*proc_t)(void *);
+
+	typedef unsigned int __kernel_size_t;
+	typedef int __kernel_ssize_t;
+
+	typedef	__kernel_size_t	SIZE_T;
+	typedef	__kernel_ssize_t	SSIZE_T;
+	#define FIELD_OFFSET(s, field)	((SSIZE_T)&((s *)(0))->field)
+
+#endif
+
+#ifndef U8_MAX
+#define U8_MAX		((u8)~0U)
+#endif
+#ifndef S8_MAX
+#define S8_MAX		((s8)(U8_MAX>>1))
+#endif
+#ifndef S8_MIN
+#define S8_MIN		((s8)(-S8_MAX - 1))
+#endif
+#ifndef U16_MAX
+#define U16_MAX		((u16)~0U)
+#endif
+#ifndef S16_MAX
+#define S16_MAX		((s16)(U16_MAX>>1))
+#endif
+#ifndef S16_MIN
+#define S16_MIN		((s16)(-S16_MAX - 1))
+#endif
+#ifndef U32_MAX
+#define U32_MAX		((u32)~0U)
+#endif
+#ifndef S32_MAX
+#define S32_MAX		((s32)(U32_MAX>>1))
+#endif
+#ifndef S32_MIN
+#define S32_MIN		((s32)(-S32_MAX - 1))
+#endif
+#ifndef U64_MAX
+#define U64_MAX		((u64)~0ULL)
+#endif
+#ifndef S64_MAX
+#define S64_MAX		((s64)(U64_MAX>>1))
+#endif
+#ifndef S64_MIN
+#define S64_MIN		((s64)(-S64_MAX - 1))
+#endif
+
+#define MEM_ALIGNMENT_OFFSET	(sizeof (SIZE_T))
+#define MEM_ALIGNMENT_PADDING	(sizeof(SIZE_T) - 1)
+
+#define SIZE_PTR SIZE_T
+#define SSIZE_PTR SSIZE_T
+
+/*
+* Continuous bits starting from least significant bit
+* Example:
+* BIT_LEN_MASK_32(0) => 0x00000000
+* BIT_LEN_MASK_32(1) => 0x00000001
+* BIT_LEN_MASK_32(2) => 0x00000003
+* BIT_LEN_MASK_32(32) => 0xFFFFFFFF
+*/
+#define BIT_LEN_MASK_32(__BitLen) ((u32)(0xFFFFFFFF >> (32 - (__BitLen))))
+#define BIT_LEN_MASK_16(__BitLen) ((u16)(0xFFFF >> (16 - (__BitLen))))
+#define BIT_LEN_MASK_8(__BitLen) ((u8)(0xFF >> (8 - (__BitLen))))
+
+/*
+* Continuous bits starting from least significant bit
+* Example:
+* BIT_OFFSET_LEN_MASK_32(0, 2) => 0x00000003
+* BIT_OFFSET_LEN_MASK_32(16, 2) => 0x00030000
+*/
+#define BIT_OFFSET_LEN_MASK_32(__BitOffset, __BitLen) ((u32)(BIT_LEN_MASK_32(__BitLen) << (__BitOffset)))
+#define BIT_OFFSET_LEN_MASK_16(__BitOffset, __BitLen) ((u16)(BIT_LEN_MASK_16(__BitLen) << (__BitOffset)))
+#define BIT_OFFSET_LEN_MASK_8(__BitOffset, __BitLen) ((u8)(BIT_LEN_MASK_8(__BitLen) << (__BitOffset)))
+
+/*
+* Convert LE data to host byte order
+*/
+#define EF1Byte (u8)
+#define EF2Byte le16_to_cpu
+#define EF4Byte le32_to_cpu
+
+/*
+* Read LE data from memory to host byte order
+*/
+#define ReadLE4Byte(_ptr)	le32_to_cpu(*((u32 *)(_ptr)))
+#define ReadLE2Byte(_ptr)	le16_to_cpu(*((u16 *)(_ptr)))
+#define ReadLE1Byte(_ptr)	(*((u8 *)(_ptr)))
+
+/*
+* Read BE data from memory to host byte order
+*/
+#define ReadBEE4Byte(_ptr)	be32_to_cpu(*((u32 *)(_ptr)))
+#define ReadBE2Byte(_ptr)	be16_to_cpu(*((u16 *)(_ptr)))
+#define ReadBE1Byte(_ptr)	(*((u8 *)(_ptr)))
+
+/*
+* Write host byte order data to memory in LE order
+*/
+#define WriteLE4Byte(_ptr, _val)	((*((u32 *)(_ptr))) = cpu_to_le32(_val))
+#define WriteLE2Byte(_ptr, _val)	((*((u16 *)(_ptr))) = cpu_to_le16(_val))
+#define WriteLE1Byte(_ptr, _val)	((*((u8 *)(_ptr))) = ((u8)(_val)))
+
+/*
+* Write host byte order data to memory in BE order
+*/
+#define WriteBE4Byte(_ptr, _val)	((*((u32 *)(_ptr))) = cpu_to_be32(_val))
+#define WriteBE2Byte(_ptr, _val)	((*((u16 *)(_ptr))) = cpu_to_be16(_val))
+#define WriteBE1Byte(_ptr, _val)	((*((u8 *)(_ptr))) = ((u8)(_val)))
+
+/*
+* Return 4-byte value in host byte ordering from 4-byte pointer in litten-endian system.
+*/
+#define LE_P4BYTE_TO_HOST_4BYTE(__pStart) (le32_to_cpu(*((u32 *)(__pStart))))
+#define LE_P2BYTE_TO_HOST_2BYTE(__pStart) (le16_to_cpu(*((u16 *)(__pStart))))
+#define LE_P1BYTE_TO_HOST_1BYTE(__pStart) ((*((u8 *)(__pStart))))
+
+/*
+* Return 4-byte value in host byte ordering from 4-byte pointer in big-endian system.
+*/
+#define BE_P4BYTE_TO_HOST_4BYTE(__pStart) (be32_to_cpu(*((u32 *)(__pStart))))
+#define BE_P2BYTE_TO_HOST_2BYTE(__pStart) (be16_to_cpu(*((u16 *)(__pStart))))
+#define BE_P1BYTE_TO_HOST_1BYTE(__pStart) ((*((u8 *)(__pStart))))
+
+/*
+* Translate subfield (continuous bits in little-endian) of 4-byte value in LE byte to
+* 4-byte value in host byte ordering.
+*/
+#define LE_BITS_TO_4BYTE(__pStart, __BitOffset, __BitLen) \
+	((LE_P4BYTE_TO_HOST_4BYTE(__pStart) >> (__BitOffset)) & BIT_LEN_MASK_32(__BitLen))
+
+#define LE_BITS_TO_2BYTE(__pStart, __BitOffset, __BitLen) \
+	((LE_P2BYTE_TO_HOST_2BYTE(__pStart) >> (__BitOffset)) & BIT_LEN_MASK_16(__BitLen))
+
+#define LE_BITS_TO_1BYTE(__pStart, __BitOffset, __BitLen) \
+	((LE_P1BYTE_TO_HOST_1BYTE(__pStart) >> (__BitOffset)) & BIT_LEN_MASK_8(__BitLen))
+
+/*
+* Translate subfield (continuous bits in big-endian) of 4-byte value in BE byte to
+* 4-byte value in host byte ordering.
+*/
+#define BE_BITS_TO_4BYTE(__pStart, __BitOffset, __BitLen) \
+	((BE_P4BYTE_TO_HOST_4BYTE(__pStart) >> (__BitOffset)) & BIT_LEN_MASK_32(__BitLen))
+
+#define BE_BITS_TO_2BYTE(__pStart, __BitOffset, __BitLen) \
+	((BE_P2BYTE_TO_HOST_2BYTE(__pStart) >> (__BitOffset)) & BIT_LEN_MASK_16(__BitLen))
+
+#define BE_BITS_TO_1BYTE(__pStart, __BitOffset, __BitLen) \
+	((BE_P1BYTE_TO_HOST_1BYTE(__pStart) >> (__BitOffset)) & BIT_LEN_MASK_8(__BitLen))
+
+/*
+* Mask subfield (continuous bits in little-endian) of 4-byte value in LE byte oredering
+* and return the result in 4-byte value in host byte ordering.
+*/
+#define LE_BITS_CLEARED_TO_4BYTE(__pStart, __BitOffset, __BitLen) \
+	(LE_P4BYTE_TO_HOST_4BYTE(__pStart) & (~BIT_OFFSET_LEN_MASK_32(__BitOffset, __BitLen)))
+
+#define LE_BITS_CLEARED_TO_2BYTE(__pStart, __BitOffset, __BitLen) \
+	(LE_P2BYTE_TO_HOST_2BYTE(__pStart) & (~BIT_OFFSET_LEN_MASK_16(__BitOffset, __BitLen)))
+
+#define LE_BITS_CLEARED_TO_1BYTE(__pStart, __BitOffset, __BitLen) \
+	(LE_P1BYTE_TO_HOST_1BYTE(__pStart) & ((u8)(~BIT_OFFSET_LEN_MASK_8(__BitOffset, __BitLen))))
+
+/*
+* Mask subfield (continuous bits in big-endian) of 4-byte value in BE byte oredering
+* and return the result in 4-byte value in host byte ordering.
+*/
+#define BE_BITS_CLEARED_TO_4BYTE(__pStart, __BitOffset, __BitLen) \
+	(BE_P4BYTE_TO_HOST_4BYTE(__pStart) & (~BIT_OFFSET_LEN_MASK_32(__BitOffset, __BitLen)))
+
+#define BE_BITS_CLEARED_TO_2BYTE(__pStart, __BitOffset, __BitLen) \
+	(BE_P2BYTE_TO_HOST_2BYTE(__pStart) & (~BIT_OFFSET_LEN_MASK_16(__BitOffset, __BitLen)))
+
+#define BE_BITS_CLEARED_TO_1BYTE(__pStart, __BitOffset, __BitLen) \
+	(BE_P1BYTE_TO_HOST_1BYTE(__pStart) & (~BIT_OFFSET_LEN_MASK_8(__BitOffset, __BitLen)))
+
+/*
+* Set subfield of little-endian 4-byte value to specified value.
+*/
+#define SET_BITS_TO_LE_4BYTE(__pStart, __BitOffset, __BitLen, __Value) \
+	do { \
+		if (__BitOffset == 0 && __BitLen == 32) \
+			WriteLE4Byte(__pStart, __Value); \
+		else { \
+			WriteLE4Byte(__pStart, \
+				LE_BITS_CLEARED_TO_4BYTE(__pStart, __BitOffset, __BitLen) \
+				| \
+				((((u32)__Value) & BIT_LEN_MASK_32(__BitLen)) << (__BitOffset)) \
+			); \
+		} \
+	} while (0)
+
+#define SET_BITS_TO_LE_2BYTE(__pStart, __BitOffset, __BitLen, __Value) \
+	do { \
+		if (__BitOffset == 0 && __BitLen == 16) \
+			WriteLE2Byte(__pStart, __Value); \
+		else { \
+			WriteLE2Byte(__pStart, \
+				LE_BITS_CLEARED_TO_2BYTE(__pStart, __BitOffset, __BitLen) \
+				| \
+				((((u16)__Value) & BIT_LEN_MASK_16(__BitLen)) << (__BitOffset)) \
+			); \
+		} \
+	} while (0)
+
+#define SET_BITS_TO_LE_1BYTE(__pStart, __BitOffset, __BitLen, __Value) \
+	do { \
+		if (__BitOffset == 0 && __BitLen == 8) \
+			WriteLE1Byte(__pStart, __Value); \
+		else { \
+			WriteLE1Byte(__pStart, \
+				LE_BITS_CLEARED_TO_1BYTE(__pStart, __BitOffset, __BitLen) \
+				| \
+				((((u8)__Value) & BIT_LEN_MASK_8(__BitLen)) << (__BitOffset)) \
+			); \
+		} \
+	} while (0)
+
+/*
+* Set subfield of big-endian 4-byte value to specified value.
+*/
+#define SET_BITS_TO_BE_4BYTE(__pStart, __BitOffset, __BitLen, __Value) \
+	do { \
+		if (__BitOffset == 0 && __BitLen == 32) \
+			WriteBE4Byte(__pStart, __Value); \
+		else { \
+			WriteBE4Byte(__pStart, \
+				BE_BITS_CLEARED_TO_4BYTE(__pStart, __BitOffset, __BitLen) \
+				| \
+				((((u32)__Value) & BIT_LEN_MASK_32(__BitLen)) << (__BitOffset)) \
+			); \
+		} \
+	} while (0)
+
+#define SET_BITS_TO_BE_2BYTE(__pStart, __BitOffset, __BitLen, __Value) \
+	do { \
+		if (__BitOffset == 0 && __BitLen == 16) \
+			WriteBE2Byte(__pStart, __Value); \
+		else { \
+			WriteBE2Byte(__pStart, \
+				BE_BITS_CLEARED_TO_2BYTE(__pStart, __BitOffset, __BitLen) \
+				| \
+				((((u16)__Value) & BIT_LEN_MASK_16(__BitLen)) << (__BitOffset)) \
+			); \
+		} \
+	} while (0)
+
+#define SET_BITS_TO_BE_1BYTE(__pStart, __BitOffset, __BitLen, __Value) \
+	do { \
+		if (__BitOffset == 0 && __BitLen == 8) \
+			WriteBE1Byte(__pStart, __Value); \
+		else { \
+			WriteBE1Byte(__pStart, \
+				BE_BITS_CLEARED_TO_1BYTE(__pStart, __BitOffset, __BitLen) \
+				| \
+				((((u8)__Value) & BIT_LEN_MASK_8(__BitLen)) << (__BitOffset)) \
+			); \
+		} \
+	} while (0)
+
+/* Get the N-bytes aligment offset from the current length */
+#define N_BYTE_ALIGMENT(__Value, __Aligment) ((__Aligment == 1) ? (__Value) : (((__Value + __Aligment - 1) / __Aligment) * __Aligment))
+#define N_BYTE_ALIGNMENT(__Value, __Alignment) ((__Alignment == 1)\
+? (__Value) : (((__Value + __Alignment - 1) / __Alignment) * __Alignment))
+
+typedef unsigned char	BOOLEAN, *PBOOLEAN, boolean;
+
+#define _FAIL				0
+#define _SUCCESS			1
+
+#undef _TRUE
+#define _TRUE		1
+
+#undef _FALSE
+#define _FALSE		0
+
+
+#define TEST_FLAG(__Flag, __testFlag)		(((__Flag) & (__testFlag)) != 0)
+#define SET_FLAG(__Flag, __setFlag)			((__Flag) |= __setFlag)
+#define CLEAR_FLAG(__Flag, __clearFlag)		((__Flag) &= ~(__clearFlag))
+#define CLEAR_FLAGS(__Flag)					((__Flag) = 0)
+#define TEST_FLAGS(__Flag, __testFlags)		(((__Flag) & (__testFlags)) == (__testFlags))
+
+#endif /* __BASIC_TYPES_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/big_endian.h b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/big_endian.h
new file mode 100644
index 000000000000..d2336a5e13d8
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/big_endian.h
@@ -0,0 +1,82 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _LINUX_BYTEORDER_BIG_ENDIAN_H
+#define _LINUX_BYTEORDER_BIG_ENDIAN_H
+
+#ifndef __BIG_ENDIAN
+	#define __BIG_ENDIAN 4321
+#endif
+#ifndef __BIG_ENDIAN_BITFIELD
+	#define __BIG_ENDIAN_BITFIELD
+#endif
+
+#include <byteorder/swab.h>
+
+#define __constant_htonl(x) ((__u32)(x))
+#define __constant_ntohl(x) ((__u32)(x))
+#define __constant_htons(x) ((__u16)(x))
+#define __constant_ntohs(x) ((__u16)(x))
+#define __constant_cpu_to_le64(x) ___constant_swab64((x))
+#define __constant_le64_to_cpu(x) ___constant_swab64((x))
+#define __constant_cpu_to_le32(x) ___constant_swab32((x))
+#define __constant_le32_to_cpu(x) ___constant_swab32((x))
+#define __constant_cpu_to_le16(x) ___constant_swab16((x))
+#define __constant_le16_to_cpu(x) ___constant_swab16((x))
+#define __constant_cpu_to_be64(x) ((__u64)(x))
+#define __constant_be64_to_cpu(x) ((__u64)(x))
+#define __constant_cpu_to_be32(x) ((__u32)(x))
+#define __constant_be32_to_cpu(x) ((__u32)(x))
+#define __constant_cpu_to_be16(x) ((__u16)(x))
+#define __constant_be16_to_cpu(x) ((__u16)(x))
+#define __cpu_to_le64(x) __swab64((x))
+#define __le64_to_cpu(x) __swab64((x))
+#define __cpu_to_le32(x) __swab32((x))
+#define __le32_to_cpu(x) __swab32((x))
+#define __cpu_to_le16(x) __swab16((x))
+#define __le16_to_cpu(x) __swab16((x))
+#define __cpu_to_be64(x) ((__u64)(x))
+#define __be64_to_cpu(x) ((__u64)(x))
+#define __cpu_to_be32(x) ((__u32)(x))
+#define __be32_to_cpu(x) ((__u32)(x))
+#define __cpu_to_be16(x) ((__u16)(x))
+#define __be16_to_cpu(x) ((__u16)(x))
+#define __cpu_to_le64p(x) __swab64p((x))
+#define __le64_to_cpup(x) __swab64p((x))
+#define __cpu_to_le32p(x) __swab32p((x))
+#define __le32_to_cpup(x) __swab32p((x))
+#define __cpu_to_le16p(x) __swab16p((x))
+#define __le16_to_cpup(x) __swab16p((x))
+#define __cpu_to_be64p(x) (*(__u64 *)(x))
+#define __be64_to_cpup(x) (*(__u64 *)(x))
+#define __cpu_to_be32p(x) (*(__u32 *)(x))
+#define __be32_to_cpup(x) (*(__u32 *)(x))
+#define __cpu_to_be16p(x) (*(__u16 *)(x))
+#define __be16_to_cpup(x) (*(__u16 *)(x))
+#define __cpu_to_le64s(x) __swab64s((x))
+#define __le64_to_cpus(x) __swab64s((x))
+#define __cpu_to_le32s(x) __swab32s((x))
+#define __le32_to_cpus(x) __swab32s((x))
+#define __cpu_to_le16s(x) __swab16s((x))
+#define __le16_to_cpus(x) __swab16s((x))
+#define __cpu_to_be64s(x) do {} while (0)
+#define __be64_to_cpus(x) do {} while (0)
+#define __cpu_to_be32s(x) do {} while (0)
+#define __be32_to_cpus(x) do {} while (0)
+#define __cpu_to_be16s(x) do {} while (0)
+#define __be16_to_cpus(x) do {} while (0)
+
+#include <byteorder/generic.h>
+
+#endif /* _LINUX_BYTEORDER_BIG_ENDIAN_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/generic.h b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/generic.h
new file mode 100644
index 000000000000..35d07379a3e7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/generic.h
@@ -0,0 +1,191 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _LINUX_BYTEORDER_GENERIC_H
+#define _LINUX_BYTEORDER_GENERIC_H
+
+/*
+ * linux/byteorder_generic.h
+ * Generic Byte-reordering support
+ *
+ * Francois-Rene Rideau <fare@tunes.org> 19970707
+ *    gathered all the good ideas from all asm-foo/byteorder.h into one file,
+ *    cleaned them up.
+ *    I hope it is compliant with non-GCC compilers.
+ *    I decided to put __BYTEORDER_HAS_U64__ in byteorder.h,
+ *    because I wasn't sure it would be ok to put it in types.h
+ *    Upgraded it to 2.1.43
+ * Francois-Rene Rideau <fare@tunes.org> 19971012
+ *    Upgraded it to 2.1.57
+ *    to please Linus T., replaced huge #ifdef's between little/big endian
+ *    by nestedly #include'd files.
+ * Francois-Rene Rideau <fare@tunes.org> 19971205
+ *    Made it to 2.1.71; now a facelift:
+ *    Put files under include/linux/byteorder/
+ *    Split swab from generic support.
+ *
+ * TODO:
+ *   = Regular kernel maintainers could also replace all these manual
+ *    byteswap macros that remain, disseminated among drivers,
+ *    after some grep or the sources...
+ *   = Linus might want to rename all these macros and files to fit his taste,
+ *    to fit his personal naming scheme.
+ *   = it seems that a few drivers would also appreciate
+ *    nybble swapping support...
+ *   = every architecture could add their byteswap macro in asm/byteorder.h
+ *    see how some architectures already do (i386, alpha, ppc, etc)
+ *   = cpu_to_beXX and beXX_to_cpu might some day need to be well
+ *    distinguished throughout the kernel. This is not the case currently,
+ *    since little endian, big endian, and pdp endian machines needn't it.
+ *    But this might be the case for, say, a port of Linux to 20/21 bit
+ *    architectures (and F21 Linux addict around?).
+ */
+
+/*
+ * The following macros are to be defined by <asm/byteorder.h>:
+ *
+ * Conversion of long and short int between network and host format
+ *	ntohl(__u32 x)
+ *	ntohs(__u16 x)
+ *	htonl(__u32 x)
+ *	htons(__u16 x)
+ * It seems that some programs (which? where? or perhaps a standard? POSIX?)
+ * might like the above to be functions, not macros (why?).
+ * if that's true, then detect them, and take measures.
+ * Anyway, the measure is: define only ___ntohl as a macro instead,
+ * and in a separate file, have
+ * unsigned long inline ntohl(x){return ___ntohl(x);}
+ *
+ * The same for constant arguments
+ *	__constant_ntohl(__u32 x)
+ *	__constant_ntohs(__u16 x)
+ *	__constant_htonl(__u32 x)
+ *	__constant_htons(__u16 x)
+ *
+ * Conversion of XX-bit integers (16- 32- or 64-)
+ * between native CPU format and little/big endian format
+ * 64-bit stuff only defined for proper architectures
+ *	cpu_to_[bl]eXX(__uXX x)
+ *	[bl]eXX_to_cpu(__uXX x)
+ *
+ * The same, but takes a pointer to the value to convert
+ *	cpu_to_[bl]eXXp(__uXX x)
+ *	[bl]eXX_to_cpup(__uXX x)
+ *
+ * The same, but change in situ
+ *	cpu_to_[bl]eXXs(__uXX x)
+ *	[bl]eXX_to_cpus(__uXX x)
+ *
+ * See asm-foo/byteorder.h for examples of how to provide
+ * architecture-optimized versions
+ *
+ */
+
+
+#if defined(PLATFORM_LINUX) || defined(PLATFORM_MPIXEL) || defined(PLATFORM_FREEBSD)
+	/*
+	* inside the kernel, we can use nicknames;
+	* outside of it, we must avoid POSIX namespace pollution...
+	*/
+	#define cpu_to_le64 __cpu_to_le64
+	#define le64_to_cpu __le64_to_cpu
+	#define cpu_to_le32 __cpu_to_le32
+	#define le32_to_cpu __le32_to_cpu
+	#define cpu_to_le16 __cpu_to_le16
+	#define le16_to_cpu __le16_to_cpu
+	#define cpu_to_be64 __cpu_to_be64
+	#define be64_to_cpu __be64_to_cpu
+	#define cpu_to_be32 __cpu_to_be32
+	#define be32_to_cpu __be32_to_cpu
+	#define cpu_to_be16 __cpu_to_be16
+	#define be16_to_cpu __be16_to_cpu
+	#define cpu_to_le64p __cpu_to_le64p
+	#define le64_to_cpup __le64_to_cpup
+	#define cpu_to_le32p __cpu_to_le32p
+	#define le32_to_cpup __le32_to_cpup
+	#define cpu_to_le16p __cpu_to_le16p
+	#define le16_to_cpup __le16_to_cpup
+	#define cpu_to_be64p __cpu_to_be64p
+	#define be64_to_cpup __be64_to_cpup
+	#define cpu_to_be32p __cpu_to_be32p
+	#define be32_to_cpup __be32_to_cpup
+	#define cpu_to_be16p __cpu_to_be16p
+	#define be16_to_cpup __be16_to_cpup
+	#define cpu_to_le64s __cpu_to_le64s
+	#define le64_to_cpus __le64_to_cpus
+	#define cpu_to_le32s __cpu_to_le32s
+	#define le32_to_cpus __le32_to_cpus
+	#define cpu_to_le16s __cpu_to_le16s
+	#define le16_to_cpus __le16_to_cpus
+	#define cpu_to_be64s __cpu_to_be64s
+	#define be64_to_cpus __be64_to_cpus
+	#define cpu_to_be32s __cpu_to_be32s
+	#define be32_to_cpus __be32_to_cpus
+	#define cpu_to_be16s __cpu_to_be16s
+	#define be16_to_cpus __be16_to_cpus
+#endif
+
+
+/*
+ * Handle ntohl and suches. These have various compatibility
+ * issues - like we want to give the prototype even though we
+ * also have a macro for them in case some strange program
+ * wants to take the address of the thing or something..
+ *
+ * Note that these used to return a "long" in libc5, even though
+ * long is often 64-bit these days.. Thus the casts.
+ *
+ * They have to be macros in order to do the constant folding
+ * correctly - if the argument passed into a inline function
+ * it is no longer constant according to gcc..
+ */
+
+#undef ntohl
+#undef ntohs
+#undef htonl
+#undef htons
+
+/*
+ * Do the prototypes. Somebody might want to take the
+ * address or some such sick thing..
+ */
+#if defined(PLATFORM_LINUX) || (defined(__GLIBC__) && __GLIBC__ >= 2)
+	extern __u32			ntohl(__u32);
+	extern __u32			htonl(__u32);
+#else /* defined(PLATFORM_LINUX) || (defined (__GLIBC__) && __GLIBC__ >= 2) */
+	extern unsigned long int	ntohl(unsigned long int);
+	extern unsigned long int	htonl(unsigned long int);
+#endif
+	extern unsigned short int	ntohs(unsigned short int);
+	extern unsigned short int	htons(unsigned short int);
+
+#if defined(__GNUC__) && (__GNUC__ >= 2) && defined(__OPTIMIZE__) || defined(PLATFORM_MPIXEL)
+
+	#define ___htonl(x) __cpu_to_be32(x)
+	#define ___htons(x) __cpu_to_be16(x)
+	#define ___ntohl(x) __be32_to_cpu(x)
+	#define ___ntohs(x) __be16_to_cpu(x)
+
+	#if defined(PLATFORM_LINUX) || (defined(__GLIBC__) && __GLIBC__ >= 2)
+		#define htonl(x) ___htonl(x)
+		#define ntohl(x) ___ntohl(x)
+	#else
+		#define htonl(x) ((unsigned long)___htonl(x))
+		#define ntohl(x) ((unsigned long)___ntohl(x))
+	#endif
+	#define htons(x) ___htons(x)
+	#define ntohs(x) ___ntohs(x)
+
+#endif /* OPTIMIZE */
+#endif /* _LINUX_BYTEORDER_GENERIC_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/little_endian.h b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/little_endian.h
new file mode 100644
index 000000000000..a08e8d7acc43
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/little_endian.h
@@ -0,0 +1,84 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _LINUX_BYTEORDER_LITTLE_ENDIAN_H
+#define _LINUX_BYTEORDER_LITTLE_ENDIAN_H
+
+#ifndef __LITTLE_ENDIAN
+	#define __LITTLE_ENDIAN 1234
+#endif
+#ifndef __LITTLE_ENDIAN_BITFIELD
+	#define __LITTLE_ENDIAN_BITFIELD
+#endif
+
+#include <byteorder/swab.h>
+
+#ifndef __constant_htonl
+	#define __constant_htonl(x) ___constant_swab32((x))
+	#define __constant_ntohl(x) ___constant_swab32((x))
+	#define __constant_htons(x) ___constant_swab16((x))
+	#define __constant_ntohs(x) ___constant_swab16((x))
+	#define __constant_cpu_to_le64(x) ((__u64)(x))
+	#define __constant_le64_to_cpu(x) ((__u64)(x))
+	#define __constant_cpu_to_le32(x) ((__u32)(x))
+	#define __constant_le32_to_cpu(x) ((__u32)(x))
+	#define __constant_cpu_to_le16(x) ((__u16)(x))
+	#define __constant_le16_to_cpu(x) ((__u16)(x))
+	#define __constant_cpu_to_be64(x) ___constant_swab64((x))
+	#define __constant_be64_to_cpu(x) ___constant_swab64((x))
+	#define __constant_cpu_to_be32(x) ___constant_swab32((x))
+	#define __constant_be32_to_cpu(x) ___constant_swab32((x))
+	#define __constant_cpu_to_be16(x) ___constant_swab16((x))
+	#define __constant_be16_to_cpu(x) ___constant_swab16((x))
+	#define __cpu_to_le64(x) ((__u64)(x))
+	#define __le64_to_cpu(x) ((__u64)(x))
+	#define __cpu_to_le32(x) ((__u32)(x))
+	#define __le32_to_cpu(x) ((__u32)(x))
+	#define __cpu_to_le16(x) ((__u16)(x))
+	#define __le16_to_cpu(x) ((__u16)(x))
+	#define __cpu_to_be64(x) __swab64((x))
+	#define __be64_to_cpu(x) __swab64((x))
+	#define __cpu_to_be32(x) __swab32((x))
+	#define __be32_to_cpu(x) __swab32((x))
+	#define __cpu_to_be16(x) __swab16((x))
+	#define __be16_to_cpu(x) __swab16((x))
+	#define __cpu_to_le64p(x) (*(__u64 *)(x))
+	#define __le64_to_cpup(x) (*(__u64 *)(x))
+	#define __cpu_to_le32p(x) (*(__u32 *)(x))
+	#define __le32_to_cpup(x) (*(__u32 *)(x))
+	#define __cpu_to_le16p(x) (*(__u16 *)(x))
+	#define __le16_to_cpup(x) (*(__u16 *)(x))
+	#define __cpu_to_be64p(x) __swab64p((x))
+	#define __be64_to_cpup(x) __swab64p((x))
+	#define __cpu_to_be32p(x) __swab32p((x))
+	#define __be32_to_cpup(x) __swab32p((x))
+	#define __cpu_to_be16p(x) __swab16p((x))
+	#define __be16_to_cpup(x) __swab16p((x))
+	#define __cpu_to_le64s(x) do {} while (0)
+	#define __le64_to_cpus(x) do {} while (0)
+	#define __cpu_to_le32s(x) do {} while (0)
+	#define __le32_to_cpus(x) do {} while (0)
+	#define __cpu_to_le16s(x) do {} while (0)
+	#define __le16_to_cpus(x) do {} while (0)
+	#define __cpu_to_be64s(x) __swab64s((x))
+	#define __be64_to_cpus(x) __swab64s((x))
+	#define __cpu_to_be32s(x) __swab32s((x))
+	#define __be32_to_cpus(x) __swab32s((x))
+	#define __cpu_to_be16s(x) __swab16s((x))
+	#define __be16_to_cpus(x) __swab16s((x))
+#endif /* __constant_htonl */
+
+#include <byteorder/generic.h>
+
+#endif /* _LINUX_BYTEORDER_LITTLE_ENDIAN_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/swab.h b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/swab.h
new file mode 100644
index 000000000000..ee9752068104
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/swab.h
@@ -0,0 +1,136 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _LINUX_BYTEORDER_SWAB_H
+#define _LINUX_BYTEORDER_SWAB_H
+
+#if !defined(CONFIG_PLATFORM_MSTAR)
+#ifndef __u16
+	typedef unsigned short __u16;
+#endif
+
+#ifndef __u32
+	typedef unsigned int	__u32;
+#endif
+
+#ifndef __u8
+	typedef unsigned char __u8;
+#endif
+
+#ifndef __u64
+	typedef unsigned long long	__u64;
+#endif
+
+
+__inline static __u16  ___swab16(__u16 x)
+{
+	__u16 __x = x;
+	return
+		 (__u16)(
+			 (((__u16)(__x)&(__u16)0x00ffU) << 8) |
+			 (((__u16)(__x)&(__u16)0xff00U) >> 8));
+
+}
+
+__inline static __u32  ___swab32(__u32 x)
+{
+	__u32 __x = (x);
+	return  (__u32)(
+			(((__u32)(__x)&(__u32)0x000000ffUL) << 24) |
+			(((__u32)(__x)&(__u32)0x0000ff00UL) <<  8) |
+			(((__u32)(__x)&(__u32)0x00ff0000UL) >>  8) |
+			(((__u32)(__x)&(__u32)0xff000000UL) >> 24));
+}
+
+__inline static __u64  ___swab64(__u64 x)
+{
+	__u64 __x = (x);
+
+	return
+		 (__u64)(\
+		 (__u64)(((__u64)(__x)&(__u64)0x00000000000000ffULL) << 56) | \
+		 (__u64)(((__u64)(__x)&(__u64)0x000000000000ff00ULL) << 40) | \
+		 (__u64)(((__u64)(__x)&(__u64)0x0000000000ff0000ULL) << 24) | \
+		 (__u64)(((__u64)(__x)&(__u64)0x00000000ff000000ULL) <<  8) | \
+		 (__u64)(((__u64)(__x)&(__u64)0x000000ff00000000ULL) >>  8) | \
+		 (__u64)(((__u64)(__x)&(__u64)0x0000ff0000000000ULL) >> 24) | \
+		 (__u64)(((__u64)(__x)&(__u64)0x00ff000000000000ULL) >> 40) | \
+		 (__u64)(((__u64)(__x)&(__u64)0xff00000000000000ULL) >> 56));
+	\
+}
+#endif /* CONFIG_PLATFORM_MSTAR */
+
+#ifndef __arch__swab16
+__inline static __u16 __arch__swab16(__u16 x)
+{
+	return ___swab16(x);
+}
+
+#endif
+
+#ifndef __arch__swab32
+__inline static __u32 __arch__swab32(__u32 x)
+{
+	__u32 __tmp = (x) ;
+	return ___swab32(__tmp);
+}
+#endif
+
+#ifndef __arch__swab64
+
+__inline static __u64 __arch__swab64(__u64 x)
+{
+	__u64 __tmp = (x) ;
+	return ___swab64(__tmp);
+}
+
+
+#endif
+
+#ifndef __swab16
+	#define __swab16(x) __fswab16(x)
+	#define __swab32(x) __fswab32(x)
+	#define __swab64(x) __fswab64(x)
+#endif /* __swab16 */
+
+#ifdef PLATFORM_FREEBSD
+	__inline static __u16 __fswab16(__u16 x)
+#else
+	__inline static const __u16 __fswab16(__u16 x)
+#endif /* PLATFORM_FREEBSD */
+{
+	return __arch__swab16(x);
+}
+#ifdef PLATFORM_FREEBSD
+	__inline static __u32 __fswab32(__u32 x)
+#else
+	__inline static const __u32 __fswab32(__u32 x)
+#endif /* PLATFORM_FREEBSD */
+{
+	return __arch__swab32(x);
+}
+
+#if defined(PLATFORM_LINUX)
+	#define swab16 __swab16
+	#define swab32 __swab32
+	#define swab64 __swab64
+	#define swab16p __swab16p
+	#define swab32p __swab32p
+	#define swab64p __swab64p
+	#define swab16s __swab16s
+	#define swab32s __swab32s
+	#define swab64s __swab64s
+#endif
+
+#endif /* _LINUX_BYTEORDER_SWAB_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/swabb.h b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/swabb.h
new file mode 100644
index 000000000000..6be7617af69f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/byteorder/swabb.h
@@ -0,0 +1,151 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _LINUX_BYTEORDER_SWABB_H
+#define _LINUX_BYTEORDER_SWABB_H
+
+/*
+ * linux/byteorder/swabb.h
+ * SWAp Bytes Bizarrely
+ *	swaHHXX[ps]?(foo)
+ *
+ * Support for obNUXIous pdp-endian and other bizarre architectures.
+ * Will Linux ever run on such ancient beasts? if not, this file
+ * will be but a programming pearl. Still, it's a reminder that we
+ * shouldn't be making too many assumptions when trying to be portable.
+ *
+ */
+
+/*
+ * Meaning of the names I chose (vaxlinux people feel free to correct them):
+ * swahw32	swap 16-bit half-words in a 32-bit word
+ * swahb32	swap 8-bit halves of each 16-bit half-word in a 32-bit word
+ *
+ * No 64-bit support yet. I don't know NUXI conventions for long longs.
+ * I guarantee it will be a mess when it's there, though :->
+ * It will be even worse if there are conflicting 64-bit conventions.
+ * Hopefully, no one ever used 64-bit objects on NUXI machines.
+ *
+ */
+
+#define ___swahw32(x) \
+	({ \
+		__u32 __x = (x); \
+		((__u32)(\
+			 (((__u32)(__x) & (__u32)0x0000ffffUL) << 16) | \
+			 (((__u32)(__x) & (__u32)0xffff0000UL) >> 16))); \
+	})
+#define ___swahb32(x) \
+	({ \
+		__u32 __x = (x); \
+		((__u32)(\
+			 (((__u32)(__x) & (__u32)0x00ff00ffUL) << 8) | \
+			 (((__u32)(__x) & (__u32)0xff00ff00UL) >> 8))); \
+	})
+
+#define ___constant_swahw32(x) \
+	((__u32)(\
+		 (((__u32)(x) & (__u32)0x0000ffffUL) << 16) | \
+		 (((__u32)(x) & (__u32)0xffff0000UL) >> 16)))
+#define ___constant_swahb32(x) \
+	((__u32)(\
+		 (((__u32)(x) & (__u32)0x00ff00ffUL) << 8) | \
+		 (((__u32)(x) & (__u32)0xff00ff00UL) >> 8)))
+
+/*
+ * provide defaults when no architecture-specific optimization is detected
+ */
+#ifndef __arch__swahw32
+	#define __arch__swahw32(x) ___swahw32(x)
+#endif
+#ifndef __arch__swahb32
+	#define __arch__swahb32(x) ___swahb32(x)
+#endif
+
+#ifndef __arch__swahw32p
+	#define __arch__swahw32p(x) __swahw32(*(x))
+#endif
+#ifndef __arch__swahb32p
+	#define __arch__swahb32p(x) __swahb32(*(x))
+#endif
+
+#ifndef __arch__swahw32s
+	#define __arch__swahw32s(x) do { *(x) = __swahw32p((x)); } while (0)
+#endif
+#ifndef __arch__swahb32s
+	#define __arch__swahb32s(x) do { *(x) = __swahb32p((x)); } while (0)
+#endif
+
+
+/*
+ * Allow constant folding
+ */
+#if defined(__GNUC__) && (__GNUC__ >= 2) && defined(__OPTIMIZE__)
+#  define __swahw32(x) \
+	(__builtin_constant_p((__u32)(x)) ? \
+	 ___swahw32((x)) : \
+	 __fswahw32((x)))
+#  define __swahb32(x) \
+	(__builtin_constant_p((__u32)(x)) ? \
+	 ___swahb32((x)) : \
+	 __fswahb32((x)))
+#else
+#  define __swahw32(x) __fswahw32(x)
+#  define __swahb32(x) __fswahb32(x)
+#endif /* OPTIMIZE */
+
+
+__inline static__ __const__ __u32 __fswahw32(__u32 x)
+{
+	return __arch__swahw32(x);
+}
+__inline static__ __u32 __swahw32p(__u32 *x)
+{
+	return __arch__swahw32p(x);
+}
+__inline static__ void __swahw32s(__u32 *addr)
+{
+	__arch__swahw32s(addr);
+}
+
+
+__inline static__ __const__ __u32 __fswahb32(__u32 x)
+{
+	return __arch__swahb32(x);
+}
+__inline static__ __u32 __swahb32p(__u32 *x)
+{
+	return __arch__swahb32p(x);
+}
+__inline static__ void __swahb32s(__u32 *addr)
+{
+	__arch__swahb32s(addr);
+}
+
+#ifdef __BYTEORDER_HAS_U64__
+	/*
+	* Not supported yet
+	*/
+#endif /* __BYTEORDER_HAS_U64__ */
+
+#if defined(PLATFORM_LINUX)
+	#define swahw32 __swahw32
+	#define swahb32 __swahb32
+	#define swahw32p __swahw32p
+	#define swahb32p __swahb32p
+	#define swahw32s __swahw32s
+	#define swahb32s __swahb32s
+#endif
+
+#endif /* _LINUX_BYTEORDER_SWABB_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/circ_buf.h b/drivers/net/wireless/realtek/rtl8852bs/include/circ_buf.h
new file mode 100644
index 000000000000..8d83cb81c321
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/circ_buf.h
@@ -0,0 +1,23 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __CIRC_BUF_H_
+#define __CIRC_BUF_H_ 1
+
+#define CIRC_CNT(head,tail,size) (((head) - (tail)) & ((size)-1))
+
+#define CIRC_SPACE(head,tail,size) CIRC_CNT((tail),((head)+1),(size))
+
+#endif //_CIRC_BUF_H_
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/custom_gpio.h b/drivers/net/wireless/realtek/rtl8852bs/include/custom_gpio.h
new file mode 100644
index 000000000000..3c67735d689a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/custom_gpio.h
@@ -0,0 +1,34 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __CUSTOM_GPIO_H__
+#define __CUSTOM_GPIO_H___
+
+#include <drv_conf.h>
+#include <osdep_service.h>
+
+typedef enum cust_gpio_modes {
+	WLAN_PWDN_ON,
+	WLAN_PWDN_OFF,
+	WLAN_POWER_ON,
+	WLAN_POWER_OFF,
+	WLAN_BT_PWDN_ON,
+	WLAN_BT_PWDN_OFF
+} cust_gpio_modes_t;
+
+extern int rtw_wifi_gpio_init(void);
+extern int rtw_wifi_gpio_deinit(void);
+extern void rtw_wifi_gpio_wlan_ctrl(int onoff);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/drv_conf.h b/drivers/net/wireless/realtek/rtl8852bs/include/drv_conf.h
new file mode 100644
index 000000000000..d263c2e87515
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/drv_conf.h
@@ -0,0 +1,860 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __DRV_CONF_H__
+#define __DRV_CONF_H__
+#include "autoconf.h"
+
+#define CONFIG_RSSI_PRIORITY
+#define CONFIG_POST_CORE_KEEP_ALIVE
+
+/*temporary defination,this flag will be removed later*/
+#define CONFIG_PHL_WACHDOG_REFINE
+
+/*
+ * RTW_BUSY_DENY_SCAN control if scan would be denied by busy traffic.
+ * When this defined, BUSY_TRAFFIC_SCAN_DENY_PERIOD would be used to judge if
+ * scan request coming from scan UI. Scan request from scan UI would be
+ * exception and never be denied by busy traffic.
+ */
+#define RTW_BUSY_DENY_SCAN
+
+#if defined(CONFIG_MCC_MODE) && (!defined(CONFIG_CONCURRENT_MODE))
+
+	#error "Enable CONCURRENT_MODE before enable MCC MODE\n"
+
+#endif
+
+#if defined(CONFIG_MCC_MODE) && defined(CONFIG_TDLS)
+
+	#error "Disable TDLS before enable MCC MODE\n"
+
+#endif
+
+#if defined(CONFIG_RTW_80211R) && !defined(CONFIG_LAYER2_ROAMING)
+
+	#error "Enable CONFIG_LAYER2_ROAMING before enable CONFIG_RTW_80211R\n"
+
+#endif
+
+#if (defined(__ANDROID_COMMON_KERNEL__) && !defined(CONFIG_RTW_ANDROID))
+	#error "Set CONFIG_RTW_ANDROID in Makefile while build with Android Common Kernel!!"
+#endif
+
+#ifdef CONFIG_RTW_ANDROID
+
+	#include <linux/version.h>
+
+	#ifndef CONFIG_PLATFORM_ANDROID
+	#define CONFIG_PLATFORM_ANDROID
+	#endif
+
+	#ifndef CONFIG_IOCTL_CFG80211
+	#define CONFIG_IOCTL_CFG80211
+	#endif
+
+	#ifndef RTW_USE_CFG80211_STA_EVENT
+	#define RTW_USE_CFG80211_STA_EVENT
+	#endif
+
+	#if (CONFIG_RTW_ANDROID > 4)
+	#ifndef CONFIG_RADIO_WORK
+	#define CONFIG_RADIO_WORK
+	#endif
+	#endif
+
+	#if (CONFIG_RTW_ANDROID >= 8)
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,0))
+		#ifndef CONFIG_RTW_WIFI_HAL
+		#define CONFIG_RTW_WIFI_HAL
+		#endif
+		#else
+ 		#error "Linux kernel version is too old\n"
+		#endif
+	#endif
+
+	#if (CONFIG_RTW_ANDROID >= 11)
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0))
+			#ifndef CONFIG_RTW_ANDROID_GKI
+			#define CONFIG_RTW_ANDROID_GKI
+			#endif
+		#endif
+
+		#ifdef CONFIG_RTW_ANDROID_GKI
+			#ifdef CONFIG_ADAPTOR_INFO_CACHING_FILE
+			#undef CONFIG_ADAPTOR_INFO_CACHING_FILE
+			#endif
+		#endif
+	#endif
+
+	#ifdef CONFIG_RTW_WIFI_HAL
+	#ifndef CONFIG_RTW_WIFI_HAL_DEBUG
+	//#define CONFIG_RTW_WIFI_HAL_DEBUG
+	#endif
+	#if (CONFIG_RTW_ANDROID < 11)
+	#ifndef CONFIG_RTW_CFGVENDOR_LLSTATS
+	#define CONFIG_RTW_CFGVENDOR_LLSTATS
+	#endif
+	#endif
+	#if (CONFIG_RTW_ANDROID < 11)
+	#ifndef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+	#define CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+	#endif
+	#else /* CONFIG_RTW_ANDROID >= 11 */
+	#ifndef CONFIG_RTW_SCAN_RAND
+	#define CONFIG_RTW_SCAN_RAND
+	#endif
+	#endif /* CONFIG_RTW_ANDROID */
+	#ifndef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+	#define CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+	#endif
+	#ifndef CONFIG_RTW_CFGVENDOR_WIFI_LOGGER
+	#define CONFIG_RTW_CFGVENDOR_WIFI_LOGGER
+	#endif
+	#if (CONFIG_RTW_ANDROID >= 10)
+	#ifndef CONFIG_RTW_CFGVENDOR_WIFI_OFFLOAD
+	//#define CONFIG_RTW_CFGVENDOR_WIFI_OFFLOAD
+	#endif
+	#ifndef CONFIG_KERNEL_PATCH_EXTERNAL_AUTH
+	#define CONFIG_KERNEL_PATCH_EXTERNAL_AUTH
+	#endif
+	#endif
+	#endif // CONFIG_RTW_WIFI_HAL
+
+
+	/* Some Android build will restart the UI while non-printable ascii is passed
+	* between java and c/c++ layer (JNI). We force CONFIG_VALIDATE_SSID
+	* for Android here. If you are sure there is no risk on your system about this,
+	* mask this macro define to support non-printable ascii ssid.
+	* #define CONFIG_VALIDATE_SSID */
+
+	/* Android expect dbm as the rx signal strength unit */
+	#define CONFIG_SIGNAL_DISPLAY_DBM
+#else // for Linux
+
+	#ifndef CONFIG_RTW_SCAN_RAND
+	#define CONFIG_RTW_SCAN_RAND
+	#endif
+
+#endif // CONFIG_RTW_ANDROID
+
+#define RTW_PER_ADAPTER_WIPHY 0
+
+#if defined(CONFIG_REGD_SRC_FROM_OS) && RTW_PER_ADAPTER_WIPHY
+#error "CONFIG_REGD_SRC_FROM_OS is not supported when enable RTW_PER_ADAPTER_WIPHY"
+#endif
+
+/*
+#if defined(CONFIG_HAS_EARLYSUSPEND) && defined(CONFIG_RESUME_IN_WORKQUEUE)
+	#warning "You have CONFIG_HAS_EARLYSUSPEND enabled in your system, we disable CONFIG_RESUME_IN_WORKQUEUE automatically"
+	#undef CONFIG_RESUME_IN_WORKQUEUE
+#endif
+
+#if defined(CONFIG_ANDROID_POWER) && defined(CONFIG_RESUME_IN_WORKQUEUE)
+	#warning "You have CONFIG_ANDROID_POWER enabled in your system, we disable CONFIG_RESUME_IN_WORKQUEUE automatically"
+	#undef CONFIG_RESUME_IN_WORKQUEUE
+#endif
+*/
+
+#ifdef CONFIG_RESUME_IN_WORKQUEUE /* this can be removed, because there is no case for this... */
+	#if !defined(CONFIG_WAKELOCK) && !defined(CONFIG_ANDROID_POWER)
+		#error "enable CONFIG_RESUME_IN_WORKQUEUE without CONFIG_WAKELOCK or CONFIG_ANDROID_POWER will suffer from the danger of wifi's unfunctionality..."
+		#error "If you still want to enable CONFIG_RESUME_IN_WORKQUEUE in this case, mask this preprossor checking and GOOD LUCK..."
+	#endif
+#endif
+
+/* About USB VENDOR REQ */
+#if defined(CONFIG_USB_VENDOR_REQ_BUFFER_PREALLOC) && !defined(CONFIG_USB_VENDOR_REQ_MUTEX)
+	#warning "define CONFIG_USB_VENDOR_REQ_MUTEX for CONFIG_USB_VENDOR_REQ_BUFFER_PREALLOC automatically"
+	#define CONFIG_USB_VENDOR_REQ_MUTEX
+#endif
+#if defined(CONFIG_VENDOR_REQ_RETRY) &&  !defined(CONFIG_USB_VENDOR_REQ_MUTEX)
+	#warning "define CONFIG_USB_VENDOR_REQ_MUTEX for CONFIG_VENDOR_REQ_RETRY automatically"
+	#define CONFIG_USB_VENDOR_REQ_MUTEX
+#endif
+
+#ifdef CONFIG_WIFI_MONITOR
+	/*	#define CONFIG_MONITOR_MODE_XMIT	*/
+#endif
+
+#ifdef CONFIG_CUSTOMER_ALIBABA_GENERAL
+	#ifndef CONFIG_WIFI_MONITOR
+		#define CONFIG_WIFI_MONITOR
+	#endif
+	#ifndef CONFIG_MONITOR_MODE_XMIT
+		#define CONFIG_MONITOR_MODE_XMIT
+	#endif
+#endif
+
+#ifndef CONFIG_RTW_DATA_BMC_TO_UC
+#define CONFIG_RTW_DATA_BMC_TO_UC 0
+#endif
+
+#ifdef CONFIG_AP_MODE
+	#define CONFIG_LIMITED_AP_NUM 1
+	#define BMC_ON_HIQ
+
+	#ifndef CONFIG_RTW_MAX_AP_ASSOC_STA
+	#define CONFIG_RTW_MAX_AP_ASSOC_STA 0 /* 0: not specified */
+	#endif
+
+	#ifndef CONFIG_RTW_AP_DATA_BMC_TO_UC
+	#define CONFIG_RTW_AP_DATA_BMC_TO_UC 1
+	#endif
+	#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+	#undef CONFIG_RTW_DATA_BMC_TO_UC
+	#define CONFIG_RTW_DATA_BMC_TO_UC 1
+	#endif
+	#ifndef CONFIG_RTW_AP_SRC_B2U_FLAGS
+	#define CONFIG_RTW_AP_SRC_B2U_FLAGS 0x8 /* see RTW_AP_B2U_XXX */
+	#endif
+	#ifndef CONFIG_RTW_AP_FWD_B2U_FLAGS
+	#define CONFIG_RTW_AP_FWD_B2U_FLAGS 0x8 /* see RTW_AP_B2U_XXX */
+	#endif
+#endif
+
+#ifdef CONFIG_RTW_MULTI_AP
+	#ifndef CONFIG_AP_MODE
+	#error "enable CONFIG_RTW_MULTI_AP without CONFIG_AP_MODE"
+	#endif
+	#ifndef CONFIG_RTW_WDS
+	#define CONFIG_RTW_WDS
+	#endif
+	#ifndef CONFIG_RTW_UNASOC_STA_MODE_OF_STYPE
+	#define CONFIG_RTW_UNASOC_STA_MODE_OF_STYPE {2, 1} /* BMC:2 for all, NMY_UC:1 for interested target */
+	#endif
+	#ifndef CONFIG_RTW_NLRTW
+	#define CONFIG_RTW_NLRTW
+	#endif
+	#ifndef CONFIG_RTW_WNM
+	#define CONFIG_RTW_WNM
+	#endif
+	#ifndef CONFIG_RTW_FSM_RRM
+	#define CONFIG_RTW_80211K
+	#endif
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	#ifndef CONFIG_RTW_MESH_ACNODE_PREVENT
+	#define CONFIG_RTW_MESH_ACNODE_PREVENT 1
+	#endif
+
+	#ifndef CONFIG_RTW_MESH_OFFCH_CAND
+	#define CONFIG_RTW_MESH_OFFCH_CAND 1
+	#endif
+
+	#ifndef CONFIG_RTW_MESH_PEER_BLACKLIST
+	#define CONFIG_RTW_MESH_PEER_BLACKLIST 1
+	#endif
+
+	#ifndef CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	#define CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST 1
+	#endif
+	#ifndef CONFIG_RTW_MESH_CTO_MGATE_CARRIER
+	#define CONFIG_RTW_MESH_CTO_MGATE_CARRIER CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	#endif
+
+	#ifndef CONFIG_RTW_MPM_TX_IES_SYNC_BSS
+	#define CONFIG_RTW_MPM_TX_IES_SYNC_BSS 1
+	#endif
+	#if CONFIG_RTW_MPM_TX_IES_SYNC_BSS
+		#ifndef CONFIG_RTW_MESH_AEK
+		#define CONFIG_RTW_MESH_AEK
+		#endif
+	#endif
+
+	#ifndef CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	#define CONFIG_RTW_MESH_DATA_BMC_TO_UC 1
+	#endif
+	#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	#undef CONFIG_RTW_DATA_BMC_TO_UC
+	#define CONFIG_RTW_DATA_BMC_TO_UC 1
+	#endif
+	#ifndef CONFIG_RTW_MSRC_B2U_FLAGS
+	#define CONFIG_RTW_MSRC_B2U_FLAGS 0x0 /* see RTW_MESH_B2U_XXX */
+	#endif
+	#ifndef CONFIG_RTW_MFWD_B2U_FLAGS
+	#define CONFIG_RTW_MFWD_B2U_FLAGS 0x2 /* see RTW_MESH_B2U_XXX */
+	#endif
+#endif
+
+#if !defined(CONFIG_SCAN_BACKOP) && defined(CONFIG_AP_MODE)
+#define CONFIG_SCAN_BACKOP
+#endif
+
+#define RTW_SCAN_SPARSE_MIRACAST 1
+#define RTW_SCAN_SPARSE_BG 0
+
+#ifndef CONFIG_RTW_HIQ_FILTER
+	#define CONFIG_RTW_HIQ_FILTER 1
+#endif
+
+#ifndef CONFIG_RTW_EDCCA_MODE_SEL
+#define CONFIG_RTW_EDCCA_MODE_SEL 0 /* 0:RTW_EDCCA_NORM, 255:RTW_EDCCA_AUTO */
+#endif
+
+#ifndef CONFIG_RTW_ADAPTIVITY_EN
+#define CONFIG_RTW_ADAPTIVITY_EN 0
+#endif
+
+#ifndef CONFIG_RTW_ADAPTIVITY_MODE
+#define CONFIG_RTW_ADAPTIVITY_MODE 0
+#endif
+
+#ifndef CONFIG_RTW_EXCL_CHS
+#define CONFIG_RTW_EXCL_CHS {0}
+#endif
+
+#ifndef CONFIG_RTW_EXCL_CHS_6G
+#define CONFIG_RTW_EXCL_CHS_6G {0}
+#endif
+
+#ifndef CONFIG_RTW_DIS_CH_FLAGS
+#define CONFIG_RTW_DIS_CH_FLAGS NULL
+#endif
+
+#ifndef CONFIG_RTW_BCN_HINT_VALID_MS
+#define CONFIG_RTW_BCN_HINT_VALID_MS (60 * 1000)
+#endif
+
+#ifndef CONFIG_RTW_ENV
+#define CONFIG_RTW_ENV 0 /* 0:ANY */
+#endif
+
+#ifndef CONFIG_RTW_COUNTRY_IE_SLAVE_EN_MODE
+#define CONFIG_RTW_COUNTRY_IE_SLAVE_EN_MODE 0 /* 0: disable */
+#endif
+
+#ifndef CONFIG_RTW_COUNTRY_IE_SLAVE_FLAGS
+#define CONFIG_RTW_COUNTRY_IE_SLAVE_FLAGS 0x00
+#endif
+
+#ifndef CONFIG_RTW_COUNTRY_IE_SLAVE_EN_ROLE
+#define CONFIG_RTW_COUNTRY_IE_SLAVE_EN_ROLE 0x03 /* BIT0 for pure STA mode, BIT1 for P2P group client */
+#endif
+
+#ifndef CONFIG_RTW_COUNTRY_IE_SLAVE_EN_IFBMP
+#define CONFIG_RTW_COUNTRY_IE_SLAVE_EN_IFBMP 0xFF /* all iface */
+#endif
+
+#ifndef CONFIG_RTW_COUNTRY_IE_SLAVE_SCAN_INT_MS
+#define CONFIG_RTW_COUNTRY_IE_SLAVE_SCAN_INT_MS (60 * 1000)
+#endif
+
+#define CONFIG_IEEE80211_BAND_5GHZ 1
+
+#ifndef CONFIG_IEEE80211_BAND_6GHZ
+#define CONFIG_IEEE80211_BAND_6GHZ 0
+#endif
+
+#ifndef CONFIG_DFS
+#define CONFIG_DFS 1
+#endif
+
+#if CONFIG_DFS
+#define CONFIG_ECSA_PHL /* Process ECSA by PHL cmd dispatcher */
+#ifdef CONFIG_ECSA_PHL
+	#define CONFIG_ECSA
+	#define CONFIG_ECSA_EXTEND_OPTION
+#endif
+#endif
+
+#if CONFIG_IEEE80211_BAND_5GHZ && CONFIG_DFS && defined(CONFIG_AP_MODE)
+	#if !defined(CONFIG_DFS_SLAVE_WITH_RADAR_DETECT)
+	#define CONFIG_DFS_SLAVE_WITH_RADAR_DETECT 0
+	#endif
+	#if !defined(CONFIG_DFS_MASTER) || CONFIG_DFS_SLAVE_WITH_RADAR_DETECT
+	#define CONFIG_DFS_MASTER
+	#endif
+	#if defined(CONFIG_DFS_MASTER) && !defined(CONFIG_RTW_DFS_REGION_DOMAIN)
+	#define CONFIG_RTW_DFS_REGION_DOMAIN 0
+	#endif
+#else
+	#undef CONFIG_DFS_MASTER
+	#undef CONFIG_RTW_DFS_REGION_DOMAIN
+	#define CONFIG_RTW_DFS_REGION_DOMAIN 0
+	#undef CONFIG_DFS_SLAVE_WITH_RADAR_DETECT
+	#define CONFIG_DFS_SLAVE_WITH_RADAR_DETECT 0
+#endif
+
+#ifndef CONFIG_AP_REGU_FORBID
+	#if CONFIG_IEEE80211_BAND_6GHZ && defined(CONFIG_AP_MODE)
+	#define CONFIG_AP_REGU_FORBID 1
+	#else
+	#define CONFIG_AP_REGU_FORBID 0
+	#endif
+#endif
+
+#ifndef CONFIG_TXPWR_BY_RATE_EN
+#define CONFIG_TXPWR_BY_RATE_EN 2 /* by efuse */
+#endif
+#ifndef CONFIG_TXPWR_LIMIT_EN
+#define CONFIG_TXPWR_LIMIT_EN 2 /* by efuse */
+#endif
+
+#ifndef CONFIG_RTW_INIT_REGD_ALWAYS_APPLY
+#define CONFIG_RTW_INIT_REGD_ALWAYS_APPLY 0
+#endif
+
+#ifndef CONFIG_RTW_USER_REGD_ALWAYS_APPLY
+#define CONFIG_RTW_USER_REGD_ALWAYS_APPLY 0
+#endif
+
+#ifndef CONFIG_RTW_COUNTRY_CODE
+#define CONFIG_RTW_COUNTRY_CODE NULL
+#endif
+
+#ifndef CONFIG_RTW_CHPLAN
+#define CONFIG_RTW_CHPLAN 0xFFFF /* RTW_CHPLAN_IOCTL_UNSPECIFIED */
+#endif
+
+#ifndef CONFIG_RTW_CHPLAN_6G
+#define CONFIG_RTW_CHPLAN_6G 0xFFFF /* RTW_CHPLAN_IOCTL_UNSPECIFIED */
+#endif
+
+/* compatible with old fashion configuration */
+#if defined(CONFIG_CALIBRATE_TX_POWER_BY_REGULATORY)
+	#undef CONFIG_TXPWR_BY_RATE_EN
+	#undef CONFIG_TXPWR_LIMIT_EN
+	#define CONFIG_TXPWR_BY_RATE_EN 1
+	#define CONFIG_TXPWR_LIMIT_EN 1
+#elif defined(CONFIG_CALIBRATE_TX_POWER_TO_MAX)
+	#undef CONFIG_TXPWR_BY_RATE_EN
+	#undef CONFIG_TXPWR_LIMIT_EN
+	#define CONFIG_TXPWR_BY_RATE_EN 1
+	#define CONFIG_TXPWR_LIMIT_EN 0
+#endif
+
+#ifndef CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE
+#define CONFIG_COUNTRY_CHPLAN_EDCCA_OVERRIDE 0
+#endif
+
+#ifndef RTW_DEF_MODULE_REGULATORY_CERT
+	#define RTW_DEF_MODULE_REGULATORY_CERT 0
+#endif
+
+#if RTW_DEF_MODULE_REGULATORY_CERT
+	#ifdef CONFIG_REGD_SRC_FROM_OS
+	#error "CONFIG_REGD_SRC_FROM_OS is not supported when enable RTW_DEF_MODULE_REGULATORY_CERT"
+	#endif
+	/* force enable TX power by rate and TX power limit */
+	#undef CONFIG_TXPWR_BY_RATE_EN
+	#undef CONFIG_TXPWR_LIMIT_EN
+	#define CONFIG_TXPWR_BY_RATE_EN 1
+	#define CONFIG_TXPWR_LIMIT_EN 1
+#endif
+
+#if !CONFIG_TXPWR_LIMIT && CONFIG_TXPWR_LIMIT_EN
+	#undef CONFIG_TXPWR_LIMIT
+	#define CONFIG_TXPWR_LIMIT 1
+#endif
+
+#if CONFIG_TXPWR_LIMIT || CONFIG_AP_REGU_FORBID
+#ifndef CONFIG_80211D
+#define CONFIG_80211D /* define 802.11d for regulatory check */
+#endif
+#endif
+
+#ifndef CONFIG_RTW_REGD_SRC
+#define CONFIG_RTW_REGD_SRC 1 /* 0:RTK_PRIV, 1:OS */
+#endif
+
+#ifndef CONFIG_RTW_REGD_SRC_OS_11D
+#define CONFIG_RTW_REGD_SRC_OS_11D 1 /* 0:disable, 1:enable */
+#endif
+
+#ifdef CONFIG_RTW_IPCAM_APPLICATION
+	#undef CONFIG_TXPWR_BY_RATE_EN
+	#define CONFIG_TXPWR_BY_RATE_EN 1
+	#define CONFIG_RTW_CUSTOMIZE_BEEDCA		0x0000431C
+	#define CONFIG_RTW_CUSTOMIZE_BWMODE		0x00
+	#define CONFIG_RTW_CUSTOMIZE_RLSTA		0x7
+#if defined(CONFIG_RTL8822B)
+	#define CONFIG_RTW_TX_NPATH_EN		/*mutually incompatible with STBC_TX & Beamformer	*/
+#endif
+#endif
+/* #define CONFIG_RTW_TOKEN_BASED_XMIT */
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	#define NR_TBTX_SLOT			4
+	#define NR_MAXSTA_INSLOT		5
+	#define TBTX_TX_DURATION		30
+
+	#define MAX_TXPAUSE_DURATION	(TBTX_TX_DURATION*NR_TBTX_SLOT)
+#endif
+
+/*#define CONFIG_EXTEND_LOWRATE_TXOP			*/
+
+#ifndef CONFIG_RTW_RX_AMPDU_SZ_LIMIT_1SS
+	#define CONFIG_RTW_RX_AMPDU_SZ_LIMIT_1SS {0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF}
+#endif
+#ifndef CONFIG_RTW_RX_AMPDU_SZ_LIMIT_2SS
+	#define CONFIG_RTW_RX_AMPDU_SZ_LIMIT_2SS {0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF}
+#endif
+#ifndef CONFIG_RTW_RX_AMPDU_SZ_LIMIT_3SS
+	#define CONFIG_RTW_RX_AMPDU_SZ_LIMIT_3SS {0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF}
+#endif
+#ifndef CONFIG_RTW_RX_AMPDU_SZ_LIMIT_4SS
+	#define CONFIG_RTW_RX_AMPDU_SZ_LIMIT_4SS {0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_2G_A
+	#define CONFIG_RTW_TARGET_TX_PWR_2G_A {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_2G_B
+	#define CONFIG_RTW_TARGET_TX_PWR_2G_B {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_2G_C
+	#define CONFIG_RTW_TARGET_TX_PWR_2G_C {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_2G_D
+	#define CONFIG_RTW_TARGET_TX_PWR_2G_D {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_5G_A
+	#define CONFIG_RTW_TARGET_TX_PWR_5G_A {-1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_5G_B
+	#define CONFIG_RTW_TARGET_TX_PWR_5G_B {-1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_5G_C
+	#define CONFIG_RTW_TARGET_TX_PWR_5G_C {-1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_TARGET_TX_PWR_5G_D
+	#define CONFIG_RTW_TARGET_TX_PWR_5G_D {-1, -1, -1, -1, -1, -1, -1, -1, -1}
+#endif
+
+#ifndef CONFIG_RTW_AMPLIFIER_TYPE_2G
+	#define CONFIG_RTW_AMPLIFIER_TYPE_2G 0
+#endif
+
+#ifndef CONFIG_RTW_AMPLIFIER_TYPE_5G
+	#define CONFIG_RTW_AMPLIFIER_TYPE_5G 0
+#endif
+
+#ifndef CONFIG_RTW_RFE_TYPE
+	#define CONFIG_RTW_RFE_TYPE 0xFF
+#endif
+
+#ifndef CONFIG_RTW_GLNA_TYPE
+	#define CONFIG_RTW_GLNA_TYPE 0
+#endif
+
+#ifndef CONFIG_RTW_PLL_REF_CLK_SEL
+	#define CONFIG_RTW_PLL_REF_CLK_SEL 0x0F
+#endif
+
+#ifndef CONFIG_IFACE_NUMBER
+	#ifdef CONFIG_CONCURRENT_MODE
+		#define CONFIG_IFACE_NUMBER	2
+	#else
+		#define CONFIG_IFACE_NUMBER	1
+	#endif
+#endif
+
+#ifdef CONFIG_CONCURRENT_MODE
+	#if (CONFIG_IFACE_NUMBER < 2)
+		#error "CONFIG_IFACE_NUMBER less 2,but CONFIG_CONCURRENT_MODE defined"
+	#endif
+#else
+	#if (CONFIG_IFACE_NUMBER > 1)
+		#error "CONFIG_IFACE_NUMBER over 1,but CONFIG_CONCURRENT_MODE not defined"
+	#endif
+#endif
+
+#if (CONFIG_IFACE_NUMBER == 0)
+	#error "CONFIG_IFACE_NUMBER cound not be 0 !!"
+#endif
+
+
+#if (CONFIG_IFACE_NUMBER > 4)
+	#error "Not support over 4 interfaces yet !!"
+#endif
+
+#if (CONFIG_IFACE_NUMBER > 8)	/*IFACE_ID_MAX*/
+	#error "HW count not support over 8 interfaces !!"
+#endif
+
+#if (CONFIG_IFACE_NUMBER > 2)
+	#ifdef CONFIG_AP_MODE
+		#undef CONFIG_LIMITED_AP_NUM
+		#define CONFIG_LIMITED_AP_NUM	2
+	#endif /*CONFIG_AP_MODE*/
+
+#endif/*(CONFIG_IFACE_NUMBER > 2)*/
+
+#if defined(CONFIG_RTL8922A)
+#define STA_NUM_SW_LIMIT 32
+#elif defined(CONFIG_RTL8952A)
+#define STA_NUM_SW_LIMIT 128
+#else
+#define STA_NUM_SW_LIMIT 32
+#endif
+
+/*
+Mark CONFIG_DEAUTH_BEFORE_CONNECT by Arvin 2015/07/20
+If the failure of Wi-Fi connection is due to some irregular disconnection behavior (like unplug dongle,
+power down etc.) in last time, we can unmark this flag to avoid some unpredictable response from AP.
+*/
+/*#define CONFIG_DEAUTH_BEFORE_CONNECT */
+
+/*#define CONFIG_WEXT_DONT_JOIN_BYSSID	*/
+/* #include <rtw_byteorder.h> */
+
+
+/*#define CONFIG_DOSCAN_IN_BUSYTRAFFIC	*/
+
+/*Don't release SDIO irq in suspend/resume procedure*/
+#define CONFIG_RTW_SDIO_KEEP_IRQ	0
+
+#ifdef CONFIG_RTW_IO_RECORDS
+	#ifndef CONFIG_RTW_IO_RECORDS_STATIC
+	#define CONFIG_RTW_IO_RECORDS_STATIC 0
+	#endif
+	#ifndef CONFIG_RTW_IO_RECORDS_NUM
+	#define CONFIG_RTW_IO_RECORDS_NUM 30000
+	#endif
+	#ifndef CONFIG_RTW_IO_RECORDS_ENABLE
+	#define CONFIG_RTW_IO_RECORDS_ENABLE true
+	#endif
+	#ifndef CONFIG_RTW_IO_RECORDS_LOOP
+	#define CONFIG_RTW_IO_RECORDS_LOOP true
+	#endif
+#endif /* CONFIG_RTW_IO_RECORDS */
+
+#ifdef CONFIG_RTW_HOSTAPD_ACS
+	#ifdef CONFIG_FIND_BEST_CHANNEL
+		#undef CONFIG_FIND_BEST_CHANNEL
+	#endif
+	#ifndef CONFIG_RTW_ACS
+		#define CONFIG_RTW_ACS
+	#endif
+#endif
+
+#ifdef CONFIG_POWER_SAVE
+#ifdef CONFIG_RTW_IPS
+#ifdef RTW_IPS_MODE
+	#if (RTW_IPS_MODE > 4 || RTW_IPS_MODE < 0)
+		#error "The CONFIG_IPS_MODE value is wrong. Please follow HowTo_enable_the_power_saving_functionality.pdf.\n"
+	#endif
+	#ifndef CONFIG_FWIPS
+		#if (RTW_IPS_MODE > 1)
+			#undef RTW_IPS_MODE
+			#define RTW_IPS_MODE PS_PWR_OFF
+		#endif /* CONFIG_FWIPS */
+	#endif
+#else /* RTW_IPS_MODE */
+	#ifdef CONFIG_FWIPS
+		#define RTW_IPS_MODE PS_IPS_PWR_GATED
+	#else
+		#define RTW_IPS_MODE PS_PWR_OFF
+	#endif /* CONFIG_FWIPS */
+#endif /* RTW_IPS_MODE */
+#endif /* CONFIG_RTW_IPS */
+
+#ifdef CONFIG_RTW_LPS
+#ifdef RTW_LPS_MODE
+	#if (RTW_LPS_MODE > 3 || RTW_LPS_MODE < 0)
+		#error "The CONFIG_LPS_MODE value is wrong. Please follow HowTo_enable_the_power_saving_functionality.pdf.\n"
+	#endif
+#else
+	#define RTW_LPS_MODE PS_LPS_PWR_GATED
+#endif /* RTW_LPS_MODE */
+#endif /* CONFIG_RTW_LPS */
+
+#ifdef CONFIG_WOWLAN
+
+#ifndef CONFIG_APF_VERSION
+#undef CONFIG_WOW_APF
+#undef CONFIG_WOW_APF_DBG
+#endif /* CONFIG_APF_VERSION */
+
+#ifdef CONFIG_RTW_IPS_WOW
+#ifdef RTW_WOW_IPS_MODE
+	#if (RTW_WOW_IPS_MODE > 4 || RTW_WOW_IPS_MODE < 0 || RTW_WOW_IPS_MODE == 1)
+		#error "The CONFIG_WOW_IPS_MODE value is wrong. Please follow HowTo_enable_the_power_saving_functionality.pdf.\n"
+	#endif
+	#ifndef CONFIG_FWIPS_WOW
+		#undef RTW_WOW_IPS_MODE
+		#define RTW_WOW_IPS_MODE PS_IPS_NONE
+	#endif
+#else
+	#ifdef CONFIG_FWIPS_WOW
+		#define RTW_WOW_IPS_MODE PS_IPS_PWR_GATED
+	#else
+		#define RTW_WOW_IPS_MODE PS_IPS_NONE
+	#endif
+#endif /* RTW_WOW_IPS_MODE */
+#endif /* CONFIG_RTW_IPS_WOW */
+
+#ifdef CONFIG_RTW_LPS_WOW
+#ifdef RTW_WOW_LPS_MODE
+	#if (RTW_WOW_LPS_MODE > 3 || RTW_WOW_LPS_MODE < 0)
+		#error "The CONFIG_WOW_LPS_MODE value is wrong. Please follow HowTo_enable_the_power_saving_functionality.pdf.\n"
+	#endif
+#else
+	#define RTW_WOW_LPS_MODE PS_LPS_PWR_GATED
+#endif /* RTW_WOW_LPS_MODE */
+#endif /* CONFIG_RTW_LPS_WOW */
+#endif /* CONFIG_WOWLAN */
+#endif /* CONFIG_POWER_SAVE */
+
+
+#ifdef RTW_REDUCE_SCAN_SWITCH_CH_TIME
+#ifndef CONFIG_RTL8822B
+	#error "Only 8822B support RTW_REDUCE_SCAN_SWITCH_CH_TIME"
+#endif
+	#ifndef RTW_CHANNEL_SWITCH_OFFLOAD
+		#define RTW_CHANNEL_SWITCH_OFFLOAD
+	#endif
+#endif
+
+#ifdef CONFIG_PCI_BCN_POLLING
+#define CONFIG_BCN_ICF
+#endif
+
+#if !defined (CONFIG_PCI_MSI) || defined (CONFIG_RTW_FORCE_PCI_MSI_DISABLE)
+#define CONFIG_RTW_PCI_MSI_DISABLE
+#endif
+
+#if 0
+/* Debug related compiler flags */
+#define DBG_THREAD_PID	/* Add thread pid to debug message prefix */
+#define DBG_CPU_INFO	/* Add CPU info to debug message prefix */
+#endif
+
+#if defined(CONFIG_RTL8852C) || defined(CONFIG_RTL8842A)
+#define CONFIG_HW_FORM_SEC_HEADER
+#endif
+
+#if defined(CONFIG_TX_AMSDU_HW_MODE) || defined(CONFIG_HW_FORM_SEC_HEADER)
+#define CONFIG_HW_HDR_CONVERSION
+#elif defined(CONFIG_TX_AMSDU_SW_MODE)
+#else
+#define CONFIG_HW_HDR_CONVERSION	/* TODO: should be 'unchange' */
+#endif
+
+/* for phl illegal mac io access check*/
+//#define CONFIG_MAC_REG_RW_CHK
+
+/* To enable the CONFIG_PHL_P2PPS definition in phl_config.h */
+#ifdef CONFIG_P2P_PS
+#define CONFIG_P2PPS
+#endif
+
+#define CONFIG_CMD_DISP
+#ifdef CONFIG_CMD_DISP
+	/*#define DBG_CONFIG_CMD_DISP*/
+
+	#ifdef CONFIG_IOCTL_CFG80211
+	#define CONFIG_PHL_CMD_SCAN_BKOP_TIME
+	#endif
+	/* Extra time for scan hidden AP in passive channel */
+	#define RTW_EXTEND_ACTIVE_SCAN_PERIOD	30	/* unit: ms */
+
+	#define CONFIG_CMD_SER
+	#define CONFIG_STA_CMD_DISPR
+	#define CONFIG_AP_CMD_DISPR
+	/*#define CONFIG_IBSS_CMD_DISPR*/
+#endif
+
+#ifdef PRIVATE_R
+	#define CONFIG_RELEASE_RPT
+	/* #define CONFIG_WAKEUP_GPIO_INPUT_MODE
+	#define CONFIG_RA_TXSTS_DBG */
+	#define CONFIG_SNR_RPT
+	#ifndef CONFIG_ARP_KEEP_ALIVE
+		#define CONFIG_ARP_KEEP_ALIVE
+	#endif
+	/* #define CONFIG_WRC_WOW_MAGIC */
+	#define CONFIG_WOW_PERIODIC_WAKE
+	#ifndef CONFIG_PNO_SECURITY_OFFLOAD
+		#define CONFIG_PNO_SECURITY_OFFLOAD
+	#endif
+	#define DBG_RX_DFRAME_RAW_DATA
+	#define DBG_RX_SIGNAL_DISPLAY_RAW_DATA
+	#define CONFIG_GO_APPEND_COUNTRY_IE
+	#define RTW_IOT_ID 0x112;
+	#define CONFIG_RTW_HIDDEN_MAC_ADDR
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	#define CONFIG_STA_MULTIPLE_BSSID
+#endif
+
+#ifndef CONFIG_ALLOW_FUNC_2G_5G_ONLY
+#define CONFIG_ALLOW_FUNC_2G_5G_ONLY 1
+#endif
+
+#if !CONFIG_ALLOW_FUNC_2G_5G_ONLY
+#define RTW_FUNC_2G_5G_ONLY __attribute__ ((deprecated("ch utility consider only 2G/5G is not allowed")))
+#else
+#define RTW_FUNC_2G_5G_ONLY /* tag for channel functions/macros consider only 2G/5G, place at the same line with symbol name */
+#endif
+
+#ifndef RTW_IOT_ID
+	#define  RTW_IOT_ID 0x0
+#endif
+
+#ifdef CONFIG_TDLS_CH_SW
+	#ifndef CONFIG_CHSW_OFLD
+	#define CONFIG_CHSW_OFLD
+	#endif
+	#ifndef CONFIG_PHL_BCN_ERLY_RPT
+	#define CONFIG_PHL_BCN_ERLY_RPT
+	#endif
+#endif
+
+/*
+ * Channel switch FW offload config:
+ * drv needs to call function rtw_phl_set_chsw_ofld_info
+ * to effectively enable channel switch fw offload
+*/
+#ifdef CONFIG_CHSW_OFLD
+	#ifndef CONFIG_PHL_CHSWOFLD
+	#define CONFIG_PHL_CHSWOFLD
+	#endif
+#endif
+
+#ifdef CONFIG_BTC
+#define CONFIG_BTC_TRXSS_CHG
+#endif
+
+/*
+ * Work around Config
+ */
+/*
+ * RTW_WKARD_UPDATE_PHL_ROLE_CAP
+ * Update adapter->phl_role.cap & proto_role_cap by driver parameters(registry).
+ */
+#define RTW_WKARD_UPDATE_PHL_ROLE_CAP
+
+/*
+*RTW_WKARD_TRIGGER_FRAME_PARSER-OFDMA UL TB control
+*/
+#define RTW_WKARD_TRIGGER_FRAME_PARSER
+
+#ifdef CONFIG_CORE_DBG_NONE
+#ifdef RTW_DETECT_HANG
+#warning "CONFIG_CORE_DBG_NONE has been enabled, disable RTW_DETECT_HANG automatically"
+#undef RTW_DETECT_HANG
+#endif
+#endif
+
+#endif /* __DRV_CONF_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/drv_types.h b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types.h
new file mode 100644
index 000000000000..6e6f58ee8ebf
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types.h
@@ -0,0 +1,2136 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+/*-------------------------------------------------------------------------------
+
+	For type defines and data structure defines
+
+--------------------------------------------------------------------------------*/
+#ifndef __DRV_TYPES_H__
+#define __DRV_TYPES_H__
+
+#include <drv_conf.h>
+#include <basic_types.h>
+#include <osdep_service.h>
+#include <rtw_byteorder.h>
+#include "../phl/rtw_general_def.h"
+#include <wlan_bssdef.h>
+#include <wifi.h>
+#include <ieee80211.h>
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+	#include <net/neighbour.h>
+	#include <net/arp.h>
+#endif
+
+typedef struct _ADAPTER _adapter;
+struct _ADAPTER_LINK;
+/* connection interface of drv and hal */
+
+#include <rtw_debug.h>
+#include <rtw_rf.h>
+#include "../core/rtw_chset.h"
+#include "../core/rtw_chplan.h"
+
+#include "../phl/phl_headers_core.h"
+
+#ifdef CONFIG_80211N_HT
+	#include <rtw_ht.h>
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	#include <rtw_vht.h>
+#endif
+
+
+#include <rtw_security.h>
+#include <rtw_xmit.h>
+#include <xmit_osdep.h>
+#include <rtw_recv.h>
+#include <rtw_rm.h>
+
+#ifdef CONFIG_80211AX_HE
+	#include <rtw_he.h>
+#ifdef CONFIG_TWT
+	#include <rtw_twt.h>
+#endif
+#endif
+
+#ifdef CONFIG_BEAMFORMING
+	#include <rtw_beamforming.h>
+#endif
+
+#include <recv_osdep.h>
+
+/*CONFIG_PHL_ARCH*/
+#include "rtw_cmd.h"
+#include "rtw_phl_cmd.h"
+#include "phl_api_tmp.h"
+#include "rtw_phl.h"
+
+/*GEORGIA_TODO_FIXIT*/
+#include "_hal_rate.h"
+#include "_hal_api_tmp.h"
+
+#include "platform_ops.h"
+#include "rtw_scan.h"
+#ifdef CONFIG_RTW_80211R
+#include <rtw_ft.h>
+#endif
+#if defined(CONFIG_RTW_WNM) || defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+#include <rtw_wnm.h>
+#endif
+#ifdef CONFIG_RTW_MBO
+#include <rtw_mbo.h>
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+	#include <rtw_eht.h>
+#endif
+
+#include <rtw_qos.h>
+#include <rtw_wow.h>
+#include <rtw_pwrctrl.h>
+#include <rtw_mlme.h>
+#include <mlme_osdep.h>
+#include <rtw_io.h>
+
+#ifdef CONFIG_RTW_CORE_RXSC
+#include <rtw_recv_shortcut.h>
+#endif
+#ifdef CONFIG_CORE_TXSC
+#include <rtw_xmit_shortcut.h>
+#endif
+
+#include <rtw_ioctl.h>
+#include <rtw_ioctl_set.h>
+#include <rtw_ioctl_query.h>
+#include "rtw_cfg.h"
+#include <osdep_intf.h>
+#include <sta_info.h>
+#include <rtw_event.h>
+#include <rtw_mlme_ext.h>
+#include "../core/rtw_dfs.h"
+#include "../core/rtw_txpwr.h"
+#include <rtw_sec_cam.h>
+#include <rtw_mi.h>
+#include <rtw_ap.h>
+#include <rtw_csa.h>
+#ifdef CONFIG_RTW_WDS
+#include "../core/wds/rtw_wds.h"
+#endif
+#ifdef CONFIG_RTW_MESH
+#include "../core/mesh/rtw_mesh.h"
+#endif
+#ifdef CONFIG_WIFI_MONITOR
+#include "../core/monitor/rtw_radiotap.h"
+#endif
+
+#ifdef CONFIG_RTW_CSI_CHANNEL_INFO
+#include "../core/rtw_csi.h"
+#endif
+
+#include <rtw_version.h>
+
+#include <rtw_p2p.h>
+
+#ifdef CONFIG_TDLS
+	#include <rtw_tdls.h>
+#endif /* CONFIG_TDLS */
+
+#ifdef CONFIG_WAPI_SUPPORT
+	#include <rtw_wapi.h>
+#endif /* CONFIG_WAPI_SUPPORT */
+
+#ifdef CONFIG_MP_INCLUDED
+	#include <rtw_mp.h>
+	#include <rtw_efuse.h>
+#endif /* CONFIG_MP_INCLUDED */
+
+#ifdef CONFIG_BR_EXT
+	#include <rtw_br_ext.h>
+#endif /* CONFIG_BR_EXT */
+
+#ifdef CONFIG_FPGA_INCLUDED
+	#include <rtw_fpga.h>
+#endif /* CONFIG_FPGA_INCLUDED */
+
+#include <ip.h>
+#include <if_ether.h>
+#include <ethernet.h>
+#include <circ_buf.h>
+
+#include <rtw_android.h>
+
+#include <rtw_btc.h>
+#ifdef CONFIG_RTW_FSM
+#include <rtw_fsm.h>
+#endif
+
+#define SPEC_DEV_ID_NONE BIT(0)
+#define SPEC_DEV_ID_DISABLE_HT BIT(1)
+#define SPEC_DEV_ID_ENABLE_PS BIT(2)
+#define SPEC_DEV_ID_RF_CONFIG_1T1R BIT(3)
+#define SPEC_DEV_ID_RF_CONFIG_2T2R BIT(4)
+#define SPEC_DEV_ID_ASSIGN_IFNAME BIT(5)
+
+#if defined(RTW_PHL_TX) || defined(RTW_PHL_RX)
+//#define PHLRX_LOG(fmt, args...) printk("phl-rx [%s][%d]"fmt, __FUNCTION__,__LINE__, ## args)
+#define PHLRX_LOG		printk("phl-rx [%s][%d] \n", __FUNCTION__, __LINE__);
+#define PHLRX_ENTER		printk("phl-rx [%s][%d] ++\n", __FUNCTION__, __LINE__);
+#define PHLRX_EXIT		printk("phl-rx [%s][%d] --\n", __FUNCTION__, __LINE__);
+#endif
+
+struct specific_device_id {
+
+	u32		flags;
+
+	u16		idVendor;
+	u16		idProduct;
+
+};
+
+struct registry_priv {
+	u8	chip_version;
+	u8	rfintfs;
+	u8	lbkmode;
+	u8	hci;
+	NDIS_802_11_SSID	ssid;
+	u8	network_mode;	/* infra, ad-hoc, auto */
+	u8	band;
+	u8	channel;/* ad-hoc support requirement */
+	u8	wireless_mode;/* A, B, G, auto */
+	u8	band_type;
+	enum rtw_phl_scan_type	scan_mode;/*scan methods - active, passive */
+	u8	radio_enable;
+	u8	preamble;/* long, short, auto */
+	u8	vrtl_carrier_sense;/* Enable, Disable, Auto */
+	u8	vcs_type;/* RTS/CTS, CTS-to-self */
+	u16	rts_thresh;
+	u8	hw_rts_en;
+	u16  frag_thresh;
+	u8	adhoc_tx_pwr;
+	u8	soft_ap;
+#ifdef CONFIG_POWER_SAVE
+#ifdef CONFIG_RTW_IPS
+	u8	ips_mode;
+	u8	ips_cap;
+#endif /* CONFIG_RTW_IPS */
+#ifdef CONFIG_RTW_LPS
+	u8	lps_mode;
+	u8	lps_cap;
+	enum rtw_lps_listen_bcn_mode lps_listen_bcn_mode;
+	u8	lps_awake_interval;
+#endif /* CONFIG_RTW_LPS */
+#ifdef CONFIG_WOWLAN
+#ifdef CONFIG_RTW_IPS_WOW
+	u8	ips_wow_mode;
+	u8	ips_wow_cap;
+#endif /* CONFIG_RTW_IPS_WOW */
+#ifdef CONFIG_RTW_LPS_WOW
+	u8	lps_wow_mode;
+	u8	lps_wow_cap;
+	enum rtw_lps_listen_bcn_mode lps_wow_listen_bcn_mode;
+	u8	lps_wow_awake_interval;
+#endif /* CONFIG_RTW_LPS_WOW */
+#endif /* CONFIG_WOWLAN */
+#endif /* CONFIG_POWER_SAVE */
+	u8	smart_ps;
+	u8   usb_rxagg_mode;
+	u8	dynamic_agg_enable;
+	u8	long_retry_lmt;
+	u8	short_retry_lmt;
+	u16	busy_thresh;
+	u16	max_bss_cnt;
+	u8	ack_policy;
+	u8	mp_mode;
+#if defined(CONFIG_MP_INCLUDED) && defined(CONFIG_RTW_CUSTOMER_STR)
+	u8 mp_customer_str;
+#endif
+	u8  mp_dm;
+	u8	software_encrypt;
+	u8	software_decrypt;
+#ifdef CONFIG_TX_EARLY_MODE
+	u8   early_mode;
+#endif
+#ifdef CONFIG_NARROWBAND_SUPPORTING
+	u8	rtw_nb_config;
+#endif
+	u8	acm_method;
+	/* WMM */
+	u8	wmm_enable;
+
+	WLAN_BSSID_EX    dev_network;
+
+	u16 tx_bw_mode;
+#ifdef CONFIG_AP_MODE
+	u8 max_ap_assoc_sta;
+	u8 bmc_tx_rate;
+	#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+	u8 ap_src_b2u_flags;
+	u8 ap_fwd_b2u_flags;
+	#endif
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	u8 msrc_b2u_flags;
+	u8 mfwd_b2u_flags;
+	#endif
+#endif
+
+#ifdef CONFIG_80211N_HT
+	u8	ht_enable;
+	/* 0: 20 MHz, 1: 40 MHz, 2: 80 MHz, 3: 160MHz */
+	/* 2.4G use bit 0 ~ 3, 5G use bit 4 ~ 7,  6G use bit 8 ~ 11*/
+	/* 0x321 means enable 2.4G 40MHz & 5G 80MHz & 6G 160MHz*/
+	u16 bw_mode;
+	u8	ampdu_enable;/* for tx */
+	u8	rx_ampdu_amsdu;/* Rx A-MPDU Supports A-MSDU is permitted */
+	u8	tx_ampdu_amsdu;/* Tx A-MPDU Supports A-MSDU is permitted */
+	u8	tx_ampdu_num;
+	u8	tx_quick_addba_req;
+	u16 rx_ampdu_sz_limit_by_nss_bw[4][4]; /* 1~4SS, BW20~BW160 */
+	/* Short GI support Bit Map */
+	/* BIT0 - 20MHz, 1: support, 0: non-support */
+	/* BIT1 - 40MHz, 1: support, 0: non-support */
+	/* BIT2 - 80MHz, 1: support, 0: non-support */
+	/* BIT3 - 160MHz, 1: support, 0: non-support */
+	u8	short_gi;
+	/*
+	  * BIT0: Enable VHT LDPC Rx, BIT1: Enable VHT LDPC Tx
+	  * BIT2: Enable HE LDPC Rx, BIT3: Enable HE LDPC Tx
+	  * BIT4: Enable HT LDPC Rx, BIT5: Enable HT LDPC Tx
+	  */
+	u8	ldpc_cap;
+	/*
+	 * BIT0: Enable VHT STBC Rx, BIT1: Enable VHT STBC Tx
+	 * BIT4: Enable HT STBC Rx, BIT5: Enable HT STBC Tx
+	 * BIT8: Enable HE STBC Rx, BIT9: Enable HE STBC Rx(greater than 80M)
+	 * BIT10: Enable HE STBC Tx, BIT11: Enable HE STBC Tx(greater than 80M)
+	 */
+	u16	stbc_cap;
+	#if defined(CONFIG_RTW_TX_NPATH_EN)
+	u8	tx_npath;
+	#endif
+	#if defined(CONFIG_RTW_PATH_DIV)
+	u8 path_div;
+	#endif
+	/*
+	 * BIT0: Enable VHT SU Beamformer
+	 * BIT1: Enable VHT SU Beamformee
+	 * BIT2: Enable VHT MU Beamformer, depend on VHT SU Beamformer
+	 * BIT3: Enable VHT MU Beamformee, depend on VHT SU Beamformee
+	 * BIT4: Enable HT Beamformer
+	 * BIT5: Enable HT Beamformee
+	 * BIT6: Enable HE SU Beamformer
+	 * BIT7: Enable HE SU Beamformee
+	 * BIT8: Enable HE MU Beamformer
+	 * BIT9: Enable HE MU Beamformee
+	 */
+	u16	beamform_cap;
+	u8	dyn_txbf;
+	u8	beamformer_rf_num;
+	u8	beamformee_rf_num;
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_80211AC_VHT
+	u8	vht_enable; /* 0:disable, 1:enable, 2:auto */
+	u8	vht_24g_enable; /* 0:disable, 1:enable */
+	u8	ampdu_factor;
+	u8 vht_rx_mcs_map[2];
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+	u8	he_enable; /* 0:disable, 1:enable, 2:auto */
+#endif
+
+	u8	lowrate_two_xmit;
+
+	u8	low_power ;
+
+	u8	wifi_spec;/* !turbo_mode */
+
+	u8 rf_path; /*rf_config*/
+	u8 tx_nss;
+	u8 rx_nss;
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+	enum regd_src_t regd_src;
+	bool regd_src_os_11d;
+#endif
+	bool init_regd_always_apply;
+	bool user_regd_always_apply;
+	char alpha2[2];
+	u8	channel_plan;
+	u8	excl_chs[MAX_CHANNEL_NUM_2G_5G];
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 channel_plan_6g;
+	u8 excl_chs_6g[MAX_CHANNEL_NUM_6G];
+	enum rtw_env_t env;
+#endif
+	u8 dis_ch_flags;
+	u32 bcn_hint_valid_ms;
+
+	u8	full_ch_in_p2p_handshake; /* 0: reply only softap channel, 1: reply full channel list*/
+
+	BOOLEAN	bAcceptAddbaReq;
+
+	u8	antdiv_cfg;
+	u8	antdiv_type;
+	u8	drv_ant_band_switch;
+
+	u8	switch_usb_mode;
+
+	u8	hw_wps_pbc;/* 0:disable,1:enable */
+
+#ifdef CONFIG_ADAPTOR_INFO_CACHING_FILE
+	char	adaptor_info_caching_file_path[PATH_LENGTH_MAX];
+#endif
+
+#ifdef CONFIG_LAYER2_ROAMING
+	u8	max_roaming_times; /* the max number driver will try to roaming */
+#endif
+
+#ifdef CONFIG_80211D
+	u8 country_ie_slave_en_mode;
+	u8 country_ie_slave_flags;
+	u8 country_ie_slave_en_role;
+	u8 country_ie_slave_en_ifbmp;
+	u32 country_ie_slave_scan_int_ms;
+#endif
+
+	u8 ifname[16];
+	u8 if2name[16];
+#if defined(CONFIG_PLATFORM_ANDROID) && (CONFIG_IFACE_NUMBER > 2)
+	u8 if3name[16];
+#endif
+	/* for pll reference clock selction */
+	u8 pll_ref_clk_sel;
+
+	u8 target_tx_pwr_valid;
+	s8 target_tx_pwr_2g[RF_PATH_MAX][RATE_SECTION_NUM];
+#if CONFIG_IEEE80211_BAND_5GHZ
+	s8 target_tx_pwr_5g[RF_PATH_MAX][RATE_SECTION_NUM - 1];
+#endif
+
+	s8	TxBBSwing_2G;
+	s8	TxBBSwing_5G;
+	u8	AmplifierType_2G;
+	u8	AmplifierType_5G;
+	u8	bEn_RFE;
+	u8	PowerTracking_Type;
+	u8	GLNA_Type;
+	u8	RegPwrTrimEnable;
+
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	u8	load_phy_file;
+	u8	RegDecryptCustomFile;
+#endif
+#if defined(CONFIG_CONCURRENT_MODE) && !RTW_P2P_GROUP_INTERFACE
+#ifdef CONFIG_P2P
+	u8 sel_p2p_iface;
+#endif
+#endif
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+	u8 ignore_go_in_scan;
+	u8 ignore_low_rssi_in_scan;
+#endif
+	u32 vo_edca;
+
+	u8 qos_opt_enable;
+
+	u8 hiq_filter;
+	u8 edcca_mode_sel;
+	u8 adaptivity_idle_probability;
+
+	u8 boffefusemask;
+	BOOLEAN bFileMaskEfuse;
+	BOOLEAN bBTFileMaskEfuse;
+#ifdef CONFIG_RTW_ACS
+	u8 acs_auto_scan;
+	u8 acs_mode;
+#endif
+
+	u32	reg_rxgain_offset_2g;
+	u32	reg_rxgain_offset_5gl;
+	u32	reg_rxgain_offset_5gm;
+	u32	reg_rxgain_offset_5gh;
+
+#ifdef CONFIG_DFS_MASTER
+	u8 dfs_region_domain;
+#endif
+
+#ifdef CONFIG_RTW_NAPI
+	u8 en_napi;
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+	u32 napi_threshold;	/* unit: Mbps */
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+#ifdef CONFIG_RTW_GRO
+	u8 en_gro;
+#endif /* CONFIG_RTW_GRO */
+#endif /* CONFIG_RTW_NAPI */
+
+#ifdef CONFIG_WOWLAN
+	u8 wowlan_enable;
+	u8 wakeup_event;
+	u8 suspend_type;
+#endif
+
+	u8 check_hw_status;
+	u8 wowlan_sta_mix_mode;
+
+#ifdef CONFIG_PCI_HCI
+	u32 pci_aspm_config;
+	u32 pci_dynamic_aspm_linkctrl;
+#endif
+
+	u8 iqk_fw_offload;
+	u8 ch_switch_offload;
+
+#ifdef CONFIG_TDLS
+	u8 en_tdls;
+#endif
+
+#ifdef CONFIG_FW_OFFLOAD_PARAM_INIT
+	u8 fw_param_init;
+#endif
+
+#ifdef DBG_LA_MODE
+	u8 la_mode_en;
+#endif
+	u32 phydm_ability;
+	u32 halrf_ability;
+#ifdef CONFIG_TDMADIG
+	u8 tdmadig_en;
+	u8 tdmadig_mode;
+	u8 tdmadig_dynamic;
+#endif/*CONFIG_TDMADIG*/
+	u8 en_dyn_rrsr;
+	u32 set_rrsr_value;
+#ifdef CONFIG_RTW_MESH
+	u8 peer_alive_based_preq;
+#endif
+
+#ifdef RTW_BUSY_DENY_SCAN
+	/*
+	 * scan_interval_thr means scan interval threshold which is used to
+	 * judge if user is in scan page or not.
+	 * If scan interval < scan_interval_thr we guess user is in scan page,
+	 * and driver won't deny any scan request at that time.
+	 * Its default value comes from compiler flag
+	 * BUSY_TRAFFIC_SCAN_DENY_PERIOD, and unit is ms.
+	 */
+	u32 scan_interval_thr;
+#endif
+	u16 scan_pch_ex_time;
+	u8 deny_legacy;
+#ifdef CONFIG_RTW_MULTI_AP
+	u8 unassoc_sta_mode_of_stype[UNASOC_STA_SRC_NUM];
+	u16 max_unassoc_sta_cnt;
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	u16 roch_min_home_dur; /* min duration for op channel */
+	u16 roch_max_away_dur; /* max acceptable away duration for remain on channel */
+	u16 roch_extend_dur; /* minimum duration to stay in roch when mgnt tx */
+#endif
+
+#if defined(PRIVATE_R) && defined(CONFIG_P2P)
+	unsigned long go_hidden_ssid_mode;
+	ATOMIC_T set_hide_ssid_timer;
+#endif
+	u8 amsdu_mode;
+
+	u8 p2p_go_skip_keep_alive;
+
+#if CONFIG_DFS
+#ifdef CONFIG_ECSA_PHL
+	u8 en_ecsa;
+#endif
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	bool split_scan_6ghz;
+#endif
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	u8 twt_en;
+#endif
+#endif
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+	u8 dbcc_lg_sim;
+#endif
+};
+
+/* For registry parameters */
+#define RGTRY_OFT(field) ((u32)FIELD_OFFSET(struct registry_priv, field))
+#define RGTRY_SZ(field)   sizeof(((struct registry_priv *) 0)->field)
+
+#define WOWLAN_IS_STA_MIX_MODE(_Adapter)	(_Adapter->registrypriv.wowlan_sta_mix_mode)
+#define BSSID_OFT(field) ((u32)FIELD_OFFSET(WLAN_BSSID_EX, field))
+#define BSSID_SZ(field)   sizeof(((PWLAN_BSSID_EX) 0)->field)
+
+#define BW_MODE_2G(bw_mode) ((bw_mode) & 0x0F)
+#define BW_MODE_5G(bw_mode) (((bw_mode) >> 4) & 0x0F)
+#define BW_MODE_6G(bw_mode) (((bw_mode) >> 8) & 0x0F)
+#ifdef CONFIG_80211N_HT
+#define REGSTY_BW_2G(regsty) BW_MODE_2G((regsty)->bw_mode)
+#define REGSTY_BW_5G(regsty) BW_MODE_5G((regsty)->bw_mode)
+#define REGSTY_BW_6G(regsty) BW_MODE_6G((regsty)->bw_mode)
+#else
+#define REGSTY_BW_2G(regsty) CHANNEL_WIDTH_20
+#define REGSTY_BW_5G(regsty) CHANNEL_WIDTH_20
+#define REGSTY_BW_6G(regsty) CHANNEL_WIDTH_20
+#endif
+#define REGSTY_IS_BW_2G_SUPPORT(regsty, bw) (REGSTY_BW_2G((regsty)) >= (bw))
+#define REGSTY_IS_BW_5G_SUPPORT(regsty, bw) (REGSTY_BW_5G((regsty)) >= (bw))
+#define REGSTY_IS_BW_6G_SUPPORT(regsty, bw) (REGSTY_BW_6G((regsty)) >= (bw))
+
+#ifdef CONFIG_80211AC_VHT
+#define REGSTY_IS_11AC_ENABLE(regsty) ((regsty)->vht_enable != 0)
+#define REGSTY_IS_11AC_AUTO(regsty) ((regsty)->vht_enable == 2)
+#define REGSTY_IS_11AC_24G_ENABLE(regsty) ((regsty)->vht_24g_enable != 0)
+#else
+#define REGSTY_IS_11AC_ENABLE(regsty) 0
+#define REGSTY_IS_11AC_AUTO(regsty) 0
+#define REGSTY_IS_11AC_24G_ENABLE(regsty) 0
+#endif
+
+#define REGSTY_IS_11AX_ENABLE(regsty) ((regsty)->he_enable != 0)
+#define REGSTY_IS_11AX_AUTO(regsty) ((regsty)->he_enable == 2)
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+#define REGSTY_REGD_SRC_FROM_OS(regsty) ((regsty)->regd_src == REGD_SRC_OS)
+#define REGSTY_REGD_SRC_OS_11D(regsty) (REGSTY_REGD_SRC_FROM_OS(regsty) && (regsty)->regd_src_os_11d)
+#else
+#define REGSTY_REGD_SRC_FROM_OS(regsty) 0
+#define REGSTY_REGD_SRC_OS_11D(regsty) 0
+#endif
+
+#ifdef CONFIG_SDIO_HCI
+	#include <drv_types_sdio.h>
+#endif
+#ifdef CONFIG_GSPI_HCI
+	#include <drv_types_gspi.h>
+#endif
+#ifdef CONFIG_PCI_HCI
+	#include <drv_types_pci.h>
+#endif
+#ifdef CONFIG_USB_HCI
+	#include <drv_types_usb.h>
+#endif
+
+#include <rtw_trx.h>
+
+#ifdef CONFIG_CONCURRENT_MODE
+	#define is_primary_adapter(adapter) (adapter->adapter_type == PRIMARY_ADAPTER)
+	#define is_vir_adapter(adapter) (adapter->adapter_type == VIRTUAL_ADAPTER)
+#else
+	#define is_primary_adapter(adapter) (1)
+	#define is_vir_adapter(adapter) (0)
+#endif
+#define GET_PRIMARY_ADAPTER(padapter) (((_adapter *)padapter)->dvobj->padapters[IFACE_ID0])
+#define GET_IFACE_NUMS(padapter) (((_adapter *)padapter)->dvobj->iface_nums)
+#define GET_ADAPTER(padapter, iface_id) (((_adapter *)padapter)->dvobj->padapters[iface_id])
+#define GET_PRIMARY_LINK(padapter) (((_adapter *)padapter)->adapter_link[RTW_RLINK_PRIMARY])
+#define GET_LINK(padapter, link_idx) (((_adapter *)padapter)->adapter_link[link_idx])
+
+
+#ifdef RTW_PHL_TX
+
+#if 1
+#define	PHLTX_ENTER //printk("eric-tx [%s][%d] ++\n", __FUNCTION__, __LINE__)
+#define	PHLTX_LOG 	//printk("eric-tx [%s][%d]\n", __FUNCTION__, __LINE__)
+#define	PHLTX_EXIT 	//printk("eric-tx [%s][%d] --\n", __FUNCTION__, __LINE__)
+#define	PHLTX_ERR 	//printk("PHLTX_ERR [%s][%d]\n", __FUNCTION__, __LINE__)
+#else
+#define	PHLTX_ENTER printk("eric-tx [%s][%d] ++\n", __FUNCTION__, __LINE__)
+#define	PHLTX_LOG 	printk("eric-tx [%s][%d]\n", __FUNCTION__, __LINE__)
+#define	PHLTX_EXIT 	printk("eric-tx [%s][%d] --\n", __FUNCTION__, __LINE__)
+
+#define	PHLTX_ERR 	printk("PHLTX_ERR [%s][%d]\n", __FUNCTION__, __LINE__)
+#endif
+
+
+#define SZ_TXREQ 	(sizeof(struct rtw_xmit_req))
+#define SZ_HEAD_BUF	100
+#define SZ_TAIL_BUF	30
+
+#define NUM_PKT_LIST_PER_TXREQ	8
+#define SZ_PKT_LIST (sizeof(struct rtw_pkt_buf_list))
+
+
+#define SZ_TX_RING 		(SZ_TXREQ+SZ_HEAD_BUF+SZ_TAIL_BUF+(SZ_PKT_LIST*NUM_PKT_LIST_PER_TXREQ))
+#define SZ_MGT_RING		(SZ_TXREQ + SZ_PKT_LIST)/* MGT_TXREQ_QMGT */
+
+#ifndef MAX_TX_RING_NUM
+#define MAX_TX_RING_NUM 	4096
+#endif /*MAX_TX_RING_NUM*/
+#endif
+
+
+enum _IFACE_ID {
+	IFACE_ID0, /*PRIMARY_ADAPTER*/
+	IFACE_ID1,
+	IFACE_ID2,
+	IFACE_ID3,
+	IFACE_ID4,
+	IFACE_ID5,
+	IFACE_ID6,
+	IFACE_ID7,
+	IFACE_ID_MAX,
+};
+
+#define VIF_START_ID	1
+
+#ifdef CONFIG_DBG_COUNTER
+struct rx_logs {
+	u32 intf_rx;
+	u32 intf_rx_err_recvframe;
+	u32 intf_rx_err_skb;
+	u32 intf_rx_report;
+	u32 core_rx;
+	u32 core_rx_pre;
+	u32 core_rx_pre_ver_err;
+	u32 core_rx_pre_mgmt;
+	u32 core_rx_pre_mgmt_err_80211w;
+	u32 core_rx_pre_mgmt_err;
+	u32 core_rx_pre_ctrl;
+	u32 core_rx_pre_ctrl_err;
+	u32 core_rx_pre_data;
+	u32 core_rx_pre_data_wapi_seq_err;
+	u32 core_rx_pre_data_wapi_key_err;
+	u32 core_rx_pre_data_handled;
+	u32 core_rx_pre_data_err;
+	u32 core_rx_pre_data_unknown;
+	u32 core_rx_pre_unknown;
+	u32 core_rx_enqueue;
+	u32 core_rx_dequeue;
+	u32 core_rx_post;
+	u32 core_rx_post_decrypt;
+	u32 core_rx_post_decrypt_wep;
+	u32 core_rx_post_decrypt_tkip;
+	u32 core_rx_post_decrypt_aes;
+	u32 core_rx_post_decrypt_wapi;
+	u32 core_rx_post_decrypt_gcmp;
+	u32 core_rx_post_decrypt_hw;
+	u32 core_rx_post_decrypt_unknown;
+	u32 core_rx_post_decrypt_err;
+	u32 core_rx_post_defrag_err;
+	u32 core_rx_post_portctrl_err;
+	u32 core_rx_post_indicate;
+	u32 core_rx_post_indicate_in_oder;
+	u32 core_rx_post_indicate_reoder;
+	u32 core_rx_post_indicate_err;
+	u32 os_indicate;
+	u32 os_indicate_ap_mcast;
+	u32 os_indicate_ap_forward;
+	u32 os_indicate_ap_self;
+	u32 os_indicate_err;
+	u32 os_netif_ok;
+	u32 os_netif_err;
+};
+
+struct tx_logs {
+	u32 os_tx;
+	u32 os_tx_err_up;
+	u32 os_tx_err_xmit;
+	u32 os_tx_m2u;
+	u32 os_tx_m2u_ignore_fw_linked;
+	u32 os_tx_m2u_ignore_self;
+	u32 os_tx_m2u_entry;
+	u32 os_tx_m2u_entry_err_xmit;
+	u32 os_tx_m2u_entry_err_skb;
+	u32 os_tx_m2u_stop;
+	u32 core_tx;
+	u32 core_tx_err_pxmitframe;
+	u32 core_tx_err_brtx;
+	u32 core_tx_upd_attrib;
+	u32 core_tx_upd_attrib_adhoc;
+	u32 core_tx_upd_attrib_sta;
+	u32 core_tx_upd_attrib_ap;
+	u32 core_tx_upd_attrib_unknown;
+	u32 core_tx_upd_attrib_dhcp;
+	u32 core_tx_upd_attrib_icmp;
+	u32 core_tx_upd_attrib_active;
+	u32 core_tx_upd_attrib_err_ucast_sta;
+	u32 core_tx_upd_attrib_err_ucast_ap_link;
+	u32 core_tx_upd_attrib_err_sta;
+	u32 core_tx_upd_attrib_err_link;
+	u32 core_tx_upd_attrib_err_sec;
+	u32 core_tx_ap_enqueue_warn_fwstate;
+	u32 core_tx_ap_enqueue_warn_sta;
+	u32 core_tx_ap_enqueue_warn_nosta;
+	u32 core_tx_ap_enqueue_warn_link;
+	u32 core_tx_ap_enqueue_warn_trigger;
+	u32 core_tx_ap_enqueue_mcast;
+	u32 core_tx_ap_enqueue_ucast;
+	u32 core_tx_ap_enqueue;
+	u32 intf_tx;
+	u32 intf_tx_pending_ac;
+	u32 intf_tx_pending_fw_under_survey;
+	u32 intf_tx_pending_fw_under_linking;
+	u32 intf_tx_pending_xmitbuf;
+	u32 intf_tx_enqueue;
+	u32 core_tx_enqueue;
+	u32 core_tx_enqueue_class;
+	u32 core_tx_enqueue_class_err_sta;
+	u32 core_tx_enqueue_class_err_nosta;
+	u32 core_tx_enqueue_class_err_fwlink;
+	u32 intf_tx_direct;
+	u32 intf_tx_direct_err_coalesce;
+	u32 intf_tx_dequeue;
+	u32 intf_tx_dequeue_err_coalesce;
+	u32 intf_tx_dump_xframe;
+	u32 intf_tx_dump_xframe_err_txdesc;
+	u32 intf_tx_dump_xframe_err_port;
+};
+
+struct int_logs {
+	u32 all;
+	u32 known;	/* PHL recognized */
+	u32 err;	/* PHL handle fail */
+};
+
+#endif /* CONFIG_DBG_COUNTER */
+
+#ifdef RTW_DETECT_HANG
+struct fw_hang_info {
+	u8 dbg_is_fw_hang;
+	u8 dbg_is_fw_gone;
+};
+
+struct rxff_hang_info {
+	u8 dbg_is_rxff_hang;
+	u8 rx_ff_hang_cnt;
+	/* increase when HCI Rx */
+	u16 rx_cnt;
+	/* record in rtw_is_rxff_hang() */
+	u16 last_rx_cnt;
+};
+
+struct hang_info {
+	u8 is_stop;
+	u32 enter_cnt;
+	struct rxff_hang_info dbg_rxff_hang_info;
+	struct fw_hang_info dbg_fw_hang_info;
+};
+#endif /* RTW_DETECT_HANG */
+
+struct debug_priv {
+	u32 dbg_sdio_free_irq_error_cnt;
+	u32 dbg_sdio_alloc_irq_error_cnt;
+	u32 dbg_sdio_free_irq_cnt;
+	u32 dbg_sdio_alloc_irq_cnt;
+	u32 dbg_sdio_deinit_error_cnt;
+	u32 dbg_sdio_init_error_cnt;
+	u32 dbg_suspend_error_cnt;
+	u32 dbg_suspend_cnt;
+	u32 dbg_resume_cnt;
+	u32 dbg_resume_error_cnt;
+	u32 dbg_deinit_fail_cnt;
+	u32 dbg_carddisable_cnt;
+	u32 dbg_carddisable_error_cnt;
+	u32 dbg_ps_insuspend_cnt;
+	u32 dbg_dev_unload_inIPS_cnt;
+	u32 dbg_wow_leave_ps_fail_cnt;
+	u32 dbg_scan_pwr_state_cnt;
+	u32 dbg_downloadfw_pwr_state_cnt;
+	u32 dbg_fw_read_ps_state_fail_cnt;
+	u32 dbg_leave_ips_fail_cnt;
+	u32 dbg_leave_lps_fail_cnt;
+	u32 dbg_h2c_leave32k_fail_cnt;
+	u32 dbg_diswow_dload_fw_fail_cnt;
+	u32 dbg_enwow_dload_fw_fail_cnt;
+	u32 dbg_ips_drvopen_fail_cnt;
+	u32 dbg_poll_fail_cnt;
+	u32 dbg_rpwm_toogle_cnt;
+	u32 dbg_rpwm_timeout_fail_cnt;
+	u32 dbg_fw_mem_dl_error_cnt;
+	u64 dbg_rx_fifo_last_overflow;
+	u64 dbg_rx_fifo_curr_overflow;
+	u64 dbg_rx_fifo_diff_overflow;
+#ifdef RTW_DETECT_HANG
+	struct hang_info dbg_hang_info;
+#endif
+};
+
+struct rtw_traffic_statistics {
+	/* tx statistics */
+	u64	tx_bytes;
+	u64	tx_pkts;
+	u64	tx_drop;
+	u64	cur_tx_bytes;
+	u64	last_tx_bytes;
+	u32	cur_tx_tp; /* Tx throughput in Mbps. */
+
+	/* rx statistics */
+	u64	rx_bytes;
+	u64	rx_pkts;
+	u64	rx_drop;
+	u64	cur_rx_bytes;
+	u64	last_rx_bytes;
+	u32	cur_rx_tp; /* Rx throughput in Mbps. */
+};
+
+#define SEC_CAP_CHK_EXTRA_SEC	BIT1 /* 256 bit */
+
+#define KEY_FMT "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
+#define KEY_ARG(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2], ((u8 *)(x))[3], ((u8 *)(x))[4], ((u8 *)(x))[5], \
+	((u8 *)(x))[6], ((u8 *)(x))[7], ((u8 *)(x))[8], ((u8 *)(x))[9], ((u8 *)(x))[10], ((u8 *)(x))[11], \
+	((u8 *)(x))[12], ((u8 *)(x))[13], ((u8 *)(x))[14], ((u8 *)(x))[15]
+
+
+
+/* used for rf_ctl_t.rate_bmp_cck_ofdm */
+#define RATE_BMP_CCK		0x000F
+#define RATE_BMP_OFDM		0xFFF0
+#define RATE_BMP_HAS_CCK(_bmp_cck_ofdm)		(_bmp_cck_ofdm & RATE_BMP_CCK)
+#define RATE_BMP_HAS_OFDM(_bmp_cck_ofdm)	(_bmp_cck_ofdm & RATE_BMP_OFDM)
+#define RATE_BMP_GET_CCK(_bmp_cck_ofdm)		(_bmp_cck_ofdm & RATE_BMP_CCK)
+#define RATE_BMP_GET_OFDM(_bmp_cck_ofdm)	((_bmp_cck_ofdm & RATE_BMP_OFDM) >> 4)
+
+/* used for rf_ctl_t.rate_bmp_ht_by_bw */
+#define RATE_BMP_HT_1SS		0x000000FF
+#define RATE_BMP_HT_2SS		0x0000FF00
+#define RATE_BMP_HT_3SS		0x00FF0000
+#define RATE_BMP_HT_4SS		0xFF000000
+#define RATE_BMP_HAS_HT_1SS(_bmp_ht)		(_bmp_ht & RATE_BMP_HT_1SS)
+#define RATE_BMP_HAS_HT_2SS(_bmp_ht)		(_bmp_ht & RATE_BMP_HT_2SS)
+#define RATE_BMP_HAS_HT_3SS(_bmp_ht)		(_bmp_ht & RATE_BMP_HT_3SS)
+#define RATE_BMP_HAS_HT_4SS(_bmp_ht)		(_bmp_ht & RATE_BMP_HT_4SS)
+#define RATE_BMP_GET_HT_1SS(_bmp_ht)		(_bmp_ht & RATE_BMP_HT_1SS)
+#define RATE_BMP_GET_HT_2SS(_bmp_ht)		((_bmp_ht & RATE_BMP_HT_2SS) >> 8)
+#define RATE_BMP_GET_HT_3SS(_bmp_ht)		((_bmp_ht & RATE_BMP_HT_3SS) >> 16)
+#define RATE_BMP_GET_HT_4SS(_bmp_ht)		((_bmp_ht & RATE_BMP_HT_4SS) >> 24)
+
+/* used for rf_ctl_t.rate_bmp_vht_by_bw */
+#define RATE_BMP_VHT_1SS	0x00000003FF
+#define RATE_BMP_VHT_2SS	0x00000FFC00
+#define RATE_BMP_VHT_3SS	0x003FF00000
+#define RATE_BMP_VHT_4SS	0xFFC0000000
+#define RATE_BMP_HAS_VHT_1SS(_bmp_vht)		(_bmp_vht & RATE_BMP_VHT_1SS)
+#define RATE_BMP_HAS_VHT_2SS(_bmp_vht)		(_bmp_vht & RATE_BMP_VHT_2SS)
+#define RATE_BMP_HAS_VHT_3SS(_bmp_vht)		(_bmp_vht & RATE_BMP_VHT_3SS)
+#define RATE_BMP_HAS_VHT_4SS(_bmp_vht)		(_bmp_vht & RATE_BMP_VHT_4SS)
+#define RATE_BMP_GET_VHT_1SS(_bmp_vht)		((u16)(_bmp_vht & RATE_BMP_VHT_1SS))
+#define RATE_BMP_GET_VHT_2SS(_bmp_vht)		((u16)((_bmp_vht & RATE_BMP_VHT_2SS) >> 10))
+#define RATE_BMP_GET_VHT_3SS(_bmp_vht)		((u16)((_bmp_vht & RATE_BMP_VHT_3SS) >> 20))
+#define RATE_BMP_GET_VHT_4SS(_bmp_vht)		((u16)((_bmp_vht & RATE_BMP_VHT_4SS) >> 30))
+
+#define TXPWR_LMT_REF_VHT_FROM_HT	BIT0
+#define TXPWR_LMT_REF_HT_FROM_VHT	BIT1
+
+#define TXPWR_LMT_HAS_CCK_1T	BIT0
+#define TXPWR_LMT_HAS_CCK_2T	BIT1
+#define TXPWR_LMT_HAS_CCK_3T	BIT2
+#define TXPWR_LMT_HAS_CCK_4T	BIT3
+#define TXPWR_LMT_HAS_OFDM_1T	BIT4
+#define TXPWR_LMT_HAS_OFDM_2T	BIT5
+#define TXPWR_LMT_HAS_OFDM_3T	BIT6
+#define TXPWR_LMT_HAS_OFDM_4T	BIT7
+
+#define OFFCHS_NONE			0
+#define OFFCHS_LEAVING_OP	1
+#define OFFCHS_LEAVE_OP		2
+#define OFFCHS_BACKING_OP	3
+
+#define COUNTRY_IE_SLAVE_EN_ROLE_STA	BIT0 /* pure STA mode */
+#define COUNTRY_IE_SLAVE_EN_ROLE_GC		BIT1 /* P2P group client */
+
+struct rf_ctl_t {
+	bool disable_sw_chplan;
+
+	enum regd_src_t regd_src;
+	u8 regd_inr_bmp;
+	char alpha2[2];
+	u8 domain_code;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 domain_code_6g;
+	u8 env_bmp;
+#endif
+
+#ifdef CONFIG_CHPLAN_PROTO_EN
+	u8 proto_en;
+#endif
+	u8 dis_ch_flags;
+	u8 excl_chs[MAX_CHANNEL_NUM_2G_5G];
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 excl_chs_6g[MAX_CHANNEL_NUM_6G];
+#endif
+
+	_mutex regd_req_mutex;
+	_list regd_req_list;
+	u8 regd_req_num;
+	bool init_regd_always_apply;
+	bool user_regd_always_apply;
+	struct regd_req_t init_req;
+	struct regd_req_t *user_req;
+
+	u32 bcn_hint_valid_ms; /* the length of time beacon hint continue */
+
+#ifdef CONFIG_80211D
+	enum country_ie_slave_en_mode cis_en_mode;
+	u8 cis_flags; /* bitmap of enum country_ie_slave_flags */
+
+	u8 cis_en_role; /* per link cis enable role, see COUNTRY_IE_SLAVE_EN_ROLE_XXX, used when CISF_ENV_BSS is not set */
+	u8 cis_en_ifbmp; /* per link cis enable iface bitmap, used when CISF_ENV_BSS is not set */
+	u32 cis_scan_int_ms; /* 0 means no env BSS scan triggerred by driver self, used when CISF_ENV_BSS is set */
+
+	bool init_user_req_is_ww;
+	bool cis_enabled; /* enable status */
+	bool collect_link_cisr;
+	bool collect_network_cisr;
+
+	/* per link cis status */
+	u8 *recv_country_ie[CONFIG_IFACE_NUMBER][RTW_RLINK_MAX];
+	u32 recv_country_ie_len[CONFIG_IFACE_NUMBER][RTW_RLINK_MAX];
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	enum country_ie_slave_6g_reg_info recv_6g_reg_info[CONFIG_IFACE_NUMBER][RTW_RLINK_MAX];
+	#endif
+	struct country_ie_slave_record cisr[CONFIG_IFACE_NUMBER][RTW_RLINK_MAX];
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 default_chplan_cate_6g_map; /* bitmap of CHPLAN_6G_CATE_XXX */
+
+	/* txpwr_lmt_6g_cate_map intersection of links */
+	u8 txpwr_lmt_6g_cate_map_int_link_num;
+	u8 txpwr_lmt_6g_cate_map_int_all_link;
+	#endif
+#endif
+
+	struct rtw_chset chset;
+	struct op_class_pref_t **spt_op_class_ch;
+	u8 cap_spt_op_class_num;
+	u8 reg_spt_op_class_num;
+	u8 cur_spt_op_class_num;
+	struct p2p_channels channel_list;
+
+	u8 op_class;
+	u8 op_ch;
+	s16 op_txpwr_max; /* mBm */
+	u8 if_op_class[CONFIG_IFACE_NUMBER];
+	u8 if_op_ch[CONFIG_IFACE_NUMBER];
+
+	_mutex offch_mutex;
+	u8 offch_state;
+
+	/* used for debug or by tx power limit */
+	u16 rate_bmp_cck_ofdm;		/* 20MHz */
+	u32 rate_bmp_ht_by_bw[2];	/* 20MHz, 40MHz. 4SS supported */
+	u64 rate_bmp_vht_by_bw[4];	/* 20MHz, 40MHz, 80MHz, 160MHz. 4SS supported */
+
+#if CONFIG_TXPWR_LIMIT
+	u8 highest_ht_rate_bw_bmp;
+	u8 highest_vht_rate_bw_bmp;
+#endif
+	u8 tpc_mode;
+	u16 tpc_manual_constraint; /* mB */
+
+	bool ch_sel_within_same_band;
+
+	u8 edcca_mode_sel;
+	u8 edcca_mode_2g;
+#if CONFIG_IEEE80211_BAND_5GHZ
+	u8 edcca_mode_5g;
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	u8 edcca_mode_6g;
+#endif
+	enum band_type last_edcca_mode_op_band;
+
+#if CONFIG_DFS
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	bool radar_detect_by_others[HW_BAND_MAX];
+	/* 5G band is implicit */
+	u8 radar_detect_cch[HW_BAND_MAX];
+	u8 radar_detect_bw[HW_BAND_MAX];
+	u32 radar_detect_freq_hi[HW_BAND_MAX];
+	u32 radar_detect_freq_lo[HW_BAND_MAX];
+
+#ifdef CONFIG_DFS_MASTER
+	enum rtw_dfs_regd dfs_region_domain;
+	_timer radar_detect_timer;
+	u8 radar_detect_enabled;
+
+	bool radar_detected;
+
+	enum phl_band_idx radar_detect_hwband;
+
+	bool non_ocp_finished;
+	systime cac_start_time;
+	systime cac_end_time;
+	u8 cac_force_stop;
+
+#if CONFIG_DFS_SLAVE_WITH_RADAR_DETECT
+	u8 dfs_slave_with_rd;
+#endif
+	u8 dfs_ch_sel_e_flags;
+	u8 dfs_ch_sel_d_flags;
+
+	u8 dbg_dfs_fake_radar_detect_cnt;
+	u8 dbg_dfs_radar_detect_trigger_non;
+	u8 dbg_dfs_choose_dfs_ch_first;
+#endif /* CONFIG_DFS_MASTER */
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+#endif /* CONFIG_DFS */
+#ifdef CONFIG_RTW_MBO
+	struct npref_ch_rtp ch_rtp;
+#endif
+#if defined(PRIVATE_R) && defined(CONFIG_ECSA_PHL)
+	/* decide the parking channel of GO after leaving DFS channel */
+	u8 p2p_park_ch;
+#endif
+};
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+#define RFCTL_REGD_SRC_FROM_OS(rfctl) ((rfctl)->regd_src == REGD_SRC_OS)
+#else
+#define RFCTL_REGD_SRC_FROM_OS(rfctl) false
+#endif
+
+#define RFCTL_REG_WORLDWIDE(rfctl)	(IS_ALPHA2_WORLDWIDE(rfctl->alpha2))
+#define RFCTL_REG_ALPHA2_UNSPEC(rfctl)	(IS_ALPHA2_UNSPEC(rfctl->alpha2)) /* ex: only domain code is specified */
+#define RFCTL_REG_INTERSECTED(rfctl)	(IS_ALPHA2_INTERSECTED(rfctl->alpha2))
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+#define RFCTL_REG_EN_11A(rfctl) (((rfctl)->proto_en & CHPLAN_PROTO_EN_A) ? true : false)
+#else
+#define RFCTL_REG_EN_11A(rfctl) false
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+#define RFCTL_REG_EN_11AC(rfctl) (((rfctl)->proto_en & CHPLAN_PROTO_EN_AC) ? true : false)
+#else
+#define RFCTL_REG_EN_11AC(rfctl) false
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#define RFCTL_REG_EN_11AX(rfctl) (((rfctl)->proto_en & CHPLAN_PROTO_EN_AX) ? true : false)
+#else
+#define RFCTL_REG_EN_11AX(rfctl) false
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+#define RFCTL_REG_EN_11BE(rfctl) (((rfctl)->proto_en & CHPLAN_PROTO_EN_BE) ? true : false)
+#else
+#define RFCTL_REG_EN_11BE(rfctl) false
+#endif
+
+#if defined(CONFIG_80211D) && CONFIG_IEEE80211_BAND_6GHZ
+#define RFCTL_RECV_6G_REG_INFO(rfctl, iface_id, alink_id) ((rfctl)->recv_6g_reg_info[iface_id][alink_id])
+#else
+#define RFCTL_RECV_6G_REG_INFO(rfctl, iface_id, alink_id) CIS_6G_REG_RSVD
+#endif
+
+#ifdef CONFIG_DFS_MASTER
+#define IS_CAC_STOPPED(rfctl) ((rfctl)->cac_end_time == RTW_CAC_STOPPED)
+#define IS_CH_WAITING(rfctl) (!IS_CAC_STOPPED(rfctl) && rtw_time_after((rfctl)->cac_end_time, rtw_get_current_time()))
+#define IS_UNDER_CAC(rfctl) (IS_CH_WAITING(rfctl) && rtw_time_after_eq(rtw_get_current_time(), (rfctl)->cac_start_time))
+#define IS_RADAR_DETECTED(rfctl) ((rfctl)->radar_detected)
+#else
+#define IS_CAC_STOPPED(rfctl) 1
+#define IS_CH_WAITING(rfctl) 0
+#define IS_UNDER_CAC(rfctl) 0
+#define IS_RADAR_DETECTED(rfctl) 0
+#endif /* CONFIG_DFS_MASTER */
+
+#if CONFIG_DFS_SLAVE_WITH_RADAR_DETECT
+#define IS_DFS_SLAVE_WITH_RD(rfctl) ((rfctl)->dfs_slave_with_rd)
+#else
+#define IS_DFS_SLAVE_WITH_RD(rfctl) 0
+#endif
+
+#define HWBAND_FMT "HB%d"
+#define HWBAND_ARG(band_idx) (band_idx)
+#define FUNC_HWBAND_FMT "%s("HWBAND_FMT")"
+#define FUNC_HWBAND_ARG(band_idx) __func__, HWBAND_ARG(band_idx)
+
+#ifdef CONFIG_USB_HCI
+
+struct trx_urb_buf_q {
+	_queue free_urb_buf_queue;
+	u8 *alloc_urb_buf;
+	u8 *urb_buf;
+	uint free_urb_buf_cnt;
+};
+
+struct data_urb {
+	_list	list;
+	struct urb *urb;
+	u8 bulk_id;
+	u8 minlen;
+};
+
+#endif
+
+struct trx_data_buf_q {
+	_queue free_data_buf_queue;
+	u8 *alloc_data_buf;
+	u8 *data_buf;
+	uint free_data_buf_cnt;
+};
+
+
+struct lite_data_buf {
+	_list	list;
+	struct dvobj_priv *dvobj;
+	u16 buf_tag;
+	u8 *pbuf;
+	u8 *phl_buf_ptr; /*point to phl rtw_usb_buf from phl*/
+#ifdef CONFIG_USB_HCI
+	struct data_urb *dataurb;
+#endif
+	struct submit_ctx *sctx;
+
+};
+
+#ifdef CONFIG_DRV_FAKE_AP
+struct fake_ap {
+	struct sk_buff_head rxq;	/* RX queue */
+	_workitem work;
+	struct rtw_timer_list bcn_timer;
+};
+#endif /* CONFIG_DRV_FAKE_AP */
+
+/*device object*/
+struct dvobj_priv {
+	/*-------- below is common data --------*/
+	ATOMIC_T bSurpriseRemoved;
+	ATOMIC_T bDriverStopped;
+	ATOMIC_T hw_start;
+	s32	processing_dev_remove;
+
+	_mutex hw_init_mutex;
+	_mutex ioctrl_mutex;
+	_mutex setch_mutex;
+	_mutex setbw_mutex;
+
+	_adapter *padapters[CONFIG_IFACE_NUMBER];/*IFACE_ID_MAX*/
+	u8 virtual_iface_num;/*from registary*/
+	u8 iface_nums; /* total number of ifaces used runtime */
+	struct mi_state iface_state[MAX_BAND_NUM];
+
+	enum rtl_ic_id ic_id;
+	enum rtw_hci_type interface_type;/*USB,SDIO,SPI,PCI*/
+
+	/*CONFIG_PHL_ARCH*/
+	void *phl;
+	struct rtw_phl_com_t *phl_com;
+	#ifdef DBG_PHL_MEM_ALLOC
+	ATOMIC_T phl_mem;
+	#endif
+
+	struct rf_ctl_t rf_ctl;
+#if 0
+	/* move to phl */
+	/* struct macid_ctl_t macid_ctl; *//*shared HW resource*/
+	struct cam_ctl_t cam_ctl;/*sec-cam shared HW resource*/
+	struct sec_cam_ent cam_cache[SEC_CAM_ENT_NUM_SW_LIMIT];
+#endif
+	/****** Band info may be x 2*********/
+	/* saved channel info when call set_channel_bw */
+	systime on_oper_ch_time;
+
+	u32 fa_cnt_acc[HW_BAND_MAX];
+
+	/****** hal dep info*********/
+
+
+	ATOMIC_T continual_io_error;
+	ATOMIC_T disable_func;
+
+	u8 xmit_block;
+	_lock xmit_block_lock;
+
+	struct pwrctrl_priv pwrctl_priv;
+	struct cmd_priv	cmdpriv;
+	struct recv_priv recvpriv;
+#ifdef CONFIG_WOWLAN
+	struct wow_priv wowlan_priv;
+#endif /* CONFIG_WOWLAN */
+
+	struct rtw_traffic_statistics	traffic_stat;
+
+	#ifdef PLATFORM_LINUX
+	_thread_hdl_ rtnl_lock_holder;
+
+	#if defined(CONFIG_IOCTL_CFG80211)
+	struct wiphy *wiphy;
+	#endif
+	#endif /* PLATFORM_LINUX */
+
+	#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+	_timer dynamic_chk_timer; /* dynamic/periodic check timer */
+	#endif
+
+	#ifdef CONFIG_RTW_NAPI_DYNAMIC
+	u8 en_napi_dynamic;
+	#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+
+	#ifdef CONFIG_RTW_WIFI_HAL
+	u32 nodfs;
+	#endif
+
+	/*-------- below is for PCIE/USB/SDIO INTERFACE --------*/
+	#ifdef CONFIG_SDIO_HCI
+	struct sdio_data sdio_data;
+	#endif
+	#ifdef CONFIG_GSPI_HCI
+	GSPI_DATA gspi_data;
+	#endif
+	#ifdef CONFIG_PCI_HCI
+	PCI_DATA pci_data;
+	#endif
+	#ifdef CONFIG_USB_HCI
+	USB_DATA usb_data;
+	#endif
+
+	struct rtw_intf_ops *intf_ops;
+
+	struct trx_data_buf_q  litexmitbuf_q;
+	struct trx_data_buf_q  litexmit_extbuf_q;
+	struct trx_data_buf_q  literecvbuf_q;
+
+	/*-------- below is for USB INTERFACE --------*/
+	#ifdef CONFIG_USB_HCI
+	u8	Queue2Pipe[HW_QUEUE_ENTRY];/* for out pipe mapping */
+	struct trx_urb_buf_q xmit_urb_q;
+	struct trx_urb_buf_q recv_urb_q;
+	#ifdef CONFIG_USB_INTERRUPT_IN_PIPE
+	struct trx_data_buf_q  intin_buf_q;
+	struct trx_urb_buf_q intin_urb_q;
+	ATOMIC_T rx_pending_cnt;/* urb counts for sumit to host  */
+	#endif
+	#endif/* CONFIG_USB_HCI */
+
+	/*-------- below is for PCIE INTERFACE --------*/
+	#ifdef CONFIG_PCI_HCI
+
+
+	#endif/* CONFIG_PCI_HCI */
+
+	/* also for RTK T/P Testing Mode */
+	u8 scan_deny;
+
+
+	#ifdef CONFIG_RTW_CUSTOMER_STR
+	_mutex customer_str_mutex;
+	struct submit_ctx *customer_str_sctx;
+	u8 customer_str[RTW_CUSTOMER_STR_LEN];
+	#endif
+
+	struct debug_priv drv_dbg;
+
+#ifdef CONFIG_DRV_FAKE_AP
+	struct fake_ap fakeap;
+#endif /* CONFIG_DRV_FAKE_AP */
+
+	/* WPAS maintain from w1.fi */
+#define RTW_WPAS_W1FI		0x00
+	/* WPAS maintain from android */
+#define RTW_WPAS_ANDROID	0x01
+	u8 wpas_type;
+#ifdef CONFIG_CSI_TIMER_POLLING
+	_timer csi_poll_timer;
+#endif
+#ifdef CONFIG_RTW_CSI_NETLINK
+	struct sock *csi_nl_sk;
+#endif
+#ifdef CONFIG_DBCC_SUPPORT
+	u8 dis_dbcc_scan;
+#endif
+#ifdef CONFIG_SELF_DIAG_INFO
+	u8 bb_self_diag_en;
+#endif
+};
+
+#ifdef CONFIG_MCC_MODE
+#define DEV_MCC_CAPABLE(dvobj) (GET_PHL_COM(dvobj)->dev_cap.mcc_sup)
+#else
+#define DEV_MCC_CAPABLE(dvobj) false
+#endif
+
+#define HWBAND_STA_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_STA_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_STA_LD_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_STA_LD_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_STA_LG_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_STA_LG_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_TDLS_LD_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_TDLS_LD_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_AP_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_AP_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_AP_STARTING_NUM(_dvobj, _band_idx)	((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_AP_STARTING_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_AP_LD_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_AP_LD_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_ADHOC_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_ADHOC_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_ADHOC_LD_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_ADHOC_LD_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_MESH_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_MESH_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_MESH_LD_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_MESH_LD_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_P2P_DV_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_P2P_DV_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_P2P_GC_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_P2P_GC_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_P2P_GO_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_P2P_GO_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_SCAN_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_SCAN_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_WPS_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_WPS_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_ROCH_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_ROCH_NUM(&((_dvobj)->iface_state[_band_idx])))
+#define HWBAND_MGMT_TX_NUM(_dvobj, _band_idx)		((_band_idx) >= HW_BAND_MAX ? 0 : MSTATE_MGMT_TX_NUM(&((_dvobj)->iface_state[_band_idx])))
+
+#define dvobj_to_pwrctl(dvobj) (&(dvobj->pwrctl_priv))
+#ifdef CONFIG_WOWLAN
+#define dvobj_to_wowlan(dvobj) (&(dvobj->wowlan_priv))
+#endif /* CONFIG_WOWLAN */
+#define pwrctl_to_dvobj(pwrctl) container_of(pwrctl, struct dvobj_priv, pwrctl_priv)
+/*#define dvobj_to_macidctl(dvobj) (&(dvobj->macid_ctl))*/
+/*#define dvobj_to_sec_camctl(dvobj) (&(dvobj->cam_ctl))*/
+#define dvobj_to_regsty(dvobj) (&(dvobj->padapters[IFACE_ID0]->registrypriv))
+#if defined(CONFIG_IOCTL_CFG80211)
+#define dvobj_to_wiphy(dvobj) ((dvobj)->wiphy)
+#endif
+#define dvobj_to_rfctl(dvobj) (&(dvobj->rf_ctl))
+#define rfctl_to_dvobj(rfctl) container_of((rfctl), struct dvobj_priv, rf_ctl)
+#define dvobj_to_chset(dvobj) (&dvobj_to_rfctl(dvobj)->chset)
+
+#ifdef CONFIG_PCI_HCI
+static inline PCI_DATA *dvobj_to_pci(struct dvobj_priv *dvobj)
+{
+	return &dvobj->pci_data;
+}
+#endif
+#ifdef CONFIG_USB_HCI
+static inline USB_DATA *dvobj_to_usb(struct dvobj_priv *dvobj)
+{
+	return &dvobj->usb_data;
+}
+#endif
+#ifdef CONFIG_SDIO_HCI
+#define dvobj_to_sdio(d)		(&(d)->sdio_data)
+#define rtw_is_sdio30(d)		(!(!dvobj_to_sdio(d)->sd3_bus_mode))
+#define rtw_sdio_get_clock(d)		(dvobj_to_sdio(d)->clock)		/* unit: HZ */
+#define rtw_sdio_get_block_size(d)	(dvobj_to_sdio(d)->block_transfer_len)	/* unit: byte */
+#define rtw_sdio_set_irq_thd(d, id)	do {dvobj_to_sdio(d)->sys_sdio_irq_thd = id;} while (0)
+#ifdef PLATFORM_LINUX
+#define dvobj_to_sdio_func(d)		(dvobj_to_sdio(d)->func)
+#endif /* PLATFORM_LINUX */
+#endif /* CONFIG_SDIO_HCI */
+#ifdef CONFIG_GSPI_HCI
+static inline GSPI_DATA *dvobj_to_gspi(struct dvobj_priv *dvobj)
+{
+	return &dvobj->gspi_data;
+}
+#endif
+
+
+#ifdef PLATFORM_LINUX
+static inline struct device *dvobj_to_dev(struct dvobj_priv *dvobj)
+{
+	/* todo: get interface type from dvobj and the return the dev accordingly */
+#ifdef RTW_DVOBJ_CHIP_HW_TYPE
+#endif
+
+#ifdef CONFIG_USB_HCI
+	return &dvobj->usb_data.pusbintf->dev;
+#endif
+#ifdef CONFIG_SDIO_HCI
+	return &dvobj->sdio_data.func->dev;
+#endif
+#ifdef CONFIG_GSPI_HCI
+	return &dvobj->gspi_data.func->dev;
+#endif
+#ifdef CONFIG_PCI_HCI
+	return &dvobj->pci_data.ppcidev->dev;
+#endif
+}
+#endif
+
+_adapter *dvobj_get_unregisterd_adapter(struct dvobj_priv *dvobj);
+_adapter *dvobj_get_adapter_by_addr(struct dvobj_priv *dvobj, u8 *addr);
+#define dvobj_get_primary_adapter(dvobj)	((dvobj)->padapters[IFACE_ID0])
+
+
+enum _ADAPTER_TYPE {
+	PRIMARY_ADAPTER,
+	VIRTUAL_ADAPTER,
+	MAX_ADAPTER = 0xFF,
+};
+
+#ifdef CONFIG_RTW_NAPI
+enum _NAPI_STATE {
+	NAPI_DISABLE = 0,
+	NAPI_ENABLE = 1,
+};
+#endif
+
+#if 0 /*#ifdef CONFIG_MAC_LOOPBACK_DRIVER*/
+typedef struct loopbackdata {
+	_sema	sema;
+	_thread_hdl_ lbkthread;
+	u8 bstop;
+	u32 cnt;
+	u16 size;
+	u16 txsize;
+	u8 txbuf[0x8000];
+	u16 rxsize;
+	u8 rxbuf[0x8000];
+	u8 msg[100];
+
+} LOOPBACKDATA, *PLOOPBACKDATA;
+#endif
+
+#define ADAPTER_TX_BW_2G(adapter) BW_MODE_2G((adapter)->driver_tx_bw_mode)
+#define ADAPTER_TX_BW_5G(adapter) BW_MODE_5G((adapter)->driver_tx_bw_mode)
+#define ADAPTER_TX_BW_6G(adapter) BW_MODE_6G((adapter)->driver_tx_bw_mode)
+
+#ifdef RTW_PHL_DBG_CMD
+#define	CORE_LOG_NUM 	(100)
+#define MAX_FRAG		(4)
+#define INV_TXFORCE_VAL	(0xFFFF)
+
+/*
+enum _CORE_REC_DUMP {
+	REC_DUMP_NO = 0,
+	REC_DUMP_ALL,
+	REC_DUMP_TX,
+	REC_DUMP_RX,
+};
+*/
+
+enum _CORE_REC_TYPE {
+	REC_TX_MGMT = 0,
+	REC_TX_DATA,
+	REC_TX_PHL,
+	REC_TX_PHL_RCC,
+
+	REC_RX_PHL,
+	REC_RX_PHL_RCC,
+	REC_RX_MGMT,
+	REC_RX_DATA,
+	REC_RX_DATA_RETRY,
+};
+
+struct core_record {
+	u32 type;
+	u32 totalSz;
+
+	u32 wl_seq;
+	u32 wl_type;
+	u32 wl_subtype;
+
+	u8 	fragNum;
+	u32 fragLen[MAX_FRAG];
+	void* virtAddr[MAX_FRAG];
+	void* phyAddrL[MAX_FRAG];
+	void* phyAddrH[MAX_FRAG];
+};
+
+struct core_logs {
+	u32 txCnt_all;
+	u32 txCnt_data;
+	u32 txCnt_mgmt;
+	u32 txCnt_phl;
+	u32 txSize_phl;
+	u32 txCnt_recycle;
+	u32 txSize_recycle;
+	struct core_record drvTx[CORE_LOG_NUM];
+	struct core_record phlTx[CORE_LOG_NUM];
+	struct core_record txRcycle[CORE_LOG_NUM];
+
+	u32 rxCnt_phl;
+	u32 rxSize_phl;
+	u32 rxCnt_recycle;
+	u32 rxSize_recycle;
+	u32 rxCnt_data;
+	u32 rxCnt_data_retry;
+	u32 rxCnt_mgmt;
+	u32 rxCnt_all;
+	struct core_record drvRx[CORE_LOG_NUM];
+	struct core_record phlRx[CORE_LOG_NUM];
+	struct core_record rxRcycle[CORE_LOG_NUM];
+#ifdef CONFIG_RTW_CORE_RXSC
+	u32 rxCnt_data_orig;
+	u32 rxCnt_data_shortcut;
+#endif
+};
+
+#define MAX_TXBD_SIZE	40
+#define MAX_TXWD_SIZE	128
+#define MAX_RXWD_SIZE	32
+
+
+enum _PHL_REC_TYPE {
+	REC_TXBD = 0,
+	REC_TXWD,
+	REC_RXWD,
+	REC_WP_RCC,
+	REC_RX_MAP,
+	REC_RX_UNMAP,
+	REC_RX_AMPDU,
+};
+
+struct record_txbd {
+	u32 bd_len;
+	u8	bd_buf[MAX_TXBD_SIZE];
+};
+
+struct record_txwd {
+	u32 wp_seq;
+	u32 wd_len;
+	u8	wd_buf[MAX_TXWD_SIZE];
+};
+
+struct record_rxwd {
+	u32 wd_len;
+	u8	wd_buf[MAX_RXWD_SIZE];
+};
+
+struct record_pci {
+	u32 map_len;
+	void *virtAddr;
+	void* phyAddrL;
+	void* phyAddrH;
+};
+
+struct record_wp_rcc {
+	u32 wp_seq;
+};
+
+struct phl_logs {
+	u32 txCnt_bd;
+	u32 txCnt_wd;
+	u32 txCnt_recycle;
+
+	struct record_txbd txBd[CORE_LOG_NUM];
+	struct record_txwd txWd[CORE_LOG_NUM];
+	struct record_wp_rcc wpRecycle[CORE_LOG_NUM];
+
+	u32 rxCnt_map;
+	u32 rxSize_map;
+	u32 rxCnt_unmap;
+	u32 rxSize_unmap;
+	struct record_pci rxPciMap[CORE_LOG_NUM];
+	struct record_pci rxPciUnmap[CORE_LOG_NUM];
+
+	u32 rxCnt_wd;
+	struct record_rxwd rxWd[CORE_LOG_NUM];
+
+	u32 rxCnt_ampdu;
+	u32	rxAmpdu[CORE_LOG_NUM];
+};
+
+#endif
+
+enum _DIS_TURBO_EDCA {
+	EN_TURBO = 0,
+	DIS_TURBO,
+	DIS_TURBO_USE_MANUAL,
+};
+
+enum _DIS_RT_CAP {
+	DIS_MBO = 0,
+};
+
+struct _ADAPTER_LINK {
+	_adapter *adapter;
+	u8 mac_addr[ETH_ALEN]; /* link address */
+
+#ifdef CONFIG_AP_MODE
+	bool is_ap_chan_ctx_added;
+#endif
+	struct rtw_wifi_role_link_t *wrlink;
+
+	struct link_mlme_priv mlmepriv;
+	struct link_mlme_ext_priv mlmeextpriv;
+	struct link_security_priv securitypriv;
+};
+
+#define ALINK_GET_HWBAND(alink) ((alink)->wrlink->hw_band)
+#define ALINK_GET_BAND(alink) ((alink)->mlmeextpriv.chandef.band)
+#define ALINK_GET_CH(alink) ((alink)->mlmeextpriv.chandef.chan)
+#define ALINK_GET_BW(alink) ((alink)->mlmeextpriv.chandef.bw)
+#define ALINK_GET_OFFSET(alink) ((alink)->mlmeextpriv.chandef.offset)
+
+struct _ADAPTER {
+	int	pid[3];/*process id from UI, 0:wpa_supplicant, 1:hostapd, 2:dhcpcd*/
+
+	/*extend to support multi interface*/
+	u8 iface_id;
+	u8 isprimary; /* is primary adapter or not */
+	/* notes:
+	**	if isprimary is true, the adapter_type value is 0, iface_id is IFACE_ID0 for PRIMARY_ADAPTER
+	**	if isprimary is false, the adapter_type value is 1, iface_id is IFACE_ID1 for VIRTUAL_ADAPTER
+	**	refer to iface_id if iface_nums>2 and isprimary is false and the adapter_type value is 0xff.*/
+	u8 adapter_type;/*be used in  Multi-interface to recognize whether is PRIMARY_ADAPTER  or not(PRIMARY_ADAPTER/VIRTUAL_ADAPTER) .*/
+
+	u8 mac_addr[ETH_ALEN];
+	/*CONFIG_PHL_ARCH*/
+	struct rtw_wifi_role_t *phl_role;
+	ATOMIC_T need_tsf_sync_done;
+
+#ifdef CONFIG_HWSIM
+	int bup_hwsim;
+#endif
+
+	u8 netif_up;
+
+	u8 registered;
+	u8 ndev_unregistering;
+
+	struct _ADAPTER_LINK *adapter_link[RTW_RLINK_MAX];
+	u8 adapter_link_num;
+
+	struct dvobj_priv *dvobj;
+	struct mlme_priv mlmepriv;
+	struct mlme_ext_priv mlmeextpriv;
+	struct xmit_priv xmitpriv;
+	struct recv_info recvinfo;/*rssi*/
+	struct sta_priv	stapriv;
+	struct security_priv securitypriv;
+	_lock   security_key_mutex; /* add for CONFIG_IEEE80211W, none 11w also can use */
+	struct registry_priv	registrypriv;
+
+	#ifdef CONFIG_RTW_80211K
+	struct rm_priv	rmpriv;
+	#endif
+	#ifdef CONFIG_RTW_FSM
+	struct fsm_priv fsmpriv;
+	#endif
+
+	#ifdef CONFIG_MP_INCLUDED
+	struct mp_priv	mppriv;
+	#endif
+
+	#ifdef CONFIG_AP_MODE
+	struct hostapd_priv	*phostapdpriv;
+	u8 bmc_tx_rate;
+
+	struct ap_cmd_dispr_priv *apcmd_dipsr_priv;
+	u32 ap_stop_cmd_token;
+	enum phl_band_idx ap_start_cmd_bidx;
+	enum phl_band_idx ap_stop_cmd_bidx;
+	u8 ap_start_cmd_state;
+	u8 ap_stop_cmd_state;
+	_lock ap_stop_st_lock;
+	u8 ap_stop_state;
+
+#define AP_STOP_ST_IDLE		0
+#define AP_STOP_ST_START	1
+#define AP_STOP_ST_REQUESTING	2
+#define AP_STOP_ST_ACQUIRED	3
+#define AP_STOP_ST_DONE		4
+
+	struct phl_cmd_token_req add_del_sta_req;
+	_lock ap_add_del_sta_lock;
+#define ADD_DEL_STA_ST_IDLE		1
+#define ADD_DEL_STA_ST_REQUESTING	2
+#define ADD_DEL_STA_ST_ACQUIRED		3
+	u32 ap_add_del_sta_cmd_token;
+	u8 ap_add_del_sta_cmd_state;
+
+	#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+	u8 b2u_flags_ap_src;
+	u8 b2u_flags_ap_fwd;
+	#endif
+	#endif/*CONFIG_AP_MODE*/
+
+	u32	setband;
+	ATOMIC_T bandskip;
+
+	#ifdef CONFIG_P2P
+	struct wifidirect_info	wdinfo;
+	#endif /* CONFIG_P2P */
+
+	#ifdef CONFIG_TDLS
+	struct tdls_info	tdlsinfo;
+	#endif /* CONFIG_TDLS */
+
+	#ifdef CONFIG_WFD
+	struct wifi_display_info wfd_info;
+	#endif /* CONFIG_WFD */
+
+	#ifdef CONFIG_RTW_NAPI
+	struct	napi_struct napi;
+	u8	napi_state;
+	#endif
+
+	#ifdef CONFIG_GPIO_API
+	u8	pre_gpio_pin;
+	struct gpio_int_priv {
+		u8 interrupt_mode;
+		u8 interrupt_enable_mask;
+		void (*callback[8])(u8 level);
+	} gpiointpriv;
+	#endif
+
+	#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	_thread_hdl_ cmdThread;
+	#endif
+
+	#if 0 /*def CONFIG_XMIT_THREAD_MODE*/
+	_thread_hdl_ xmitThread;
+	#endif
+	#ifdef CONFIG_RECV_THREAD_MODE
+	_thread_hdl_ recvThread;
+	#endif
+
+	#ifdef PLATFORM_LINUX
+	#ifdef CONFIG_IOCTL_CFG80211
+	struct cfg80211_roch_info cfg80211_rochinfo;
+	#endif /* CONFIG_IOCTL_CFG80211 */
+
+	_nic_hdl pnetdev;
+	char old_ifname[IFNAMSIZ];
+
+	struct net_device_stats stats;
+	struct iw_statistics iwstats;
+	struct proc_dir_entry *dir_dev;/* for proc directory */
+	struct proc_dir_entry *dir_odm;
+
+	#ifdef CONFIG_IOCTL_CFG80211
+	struct wireless_dev *rtw_wdev;
+	struct rtw_wdev_priv wdev_data;
+
+	#endif /* CONFIG_IOCTL_CFG80211 */
+
+	#endif /* PLATFORM_LINUX */
+
+	#ifdef CONFIG_TX_AMSDU
+	u8 tx_amsdu;
+	u16 tx_amsdu_rate;
+	#endif
+
+	#ifdef CONFIG_RTW_WDS
+	bool use_wds; /* for STA, AP mode */
+
+	/* for STA mode */
+	struct rtw_wds_gptr_table *wds_gpt_records;
+	ATOMIC_T wds_gpt_record_num;
+
+	/* for AP mode */
+	#ifdef CONFIG_AP_MODE
+	struct rtw_wds_table *wds_paths;
+	ATOMIC_T wds_path_num;
+	#endif
+	#endif /* CONFIG_RTW_WDS */
+
+	#ifdef CONFIG_RTW_MULTI_AP
+	u8 multi_ap;
+	u8 ch_util_threshold;
+	#endif
+
+	#ifdef CONFIG_RTW_MESH
+	struct rtw_mesh_cfg mesh_cfg;
+	struct rtw_mesh_info mesh_info;
+	_timer mesh_path_timer;
+	_timer mesh_path_root_timer;
+	_timer mesh_atlm_param_req_timer; /* airtime link metrics param request timer */
+	_workitem mesh_work;
+	unsigned long wrkq_flags;
+	#endif /* CONFIG_RTW_MESH */
+
+	#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	ATOMIC_T tbtx_tx_pause;
+	ATOMIC_T tbtx_remove_tx_pause;
+	u8 	tbtx_capability;
+	u32	tbtx_duration;
+	#endif /* CONFIG_RTW_TOKEN_BASED_XMIT */
+
+	#ifdef CONFIG_WAPI_SUPPORT
+	u8	WapiSupport;
+	RT_WAPI_T wapiInfo;
+	#endif
+
+	#ifdef CONFIG_BR_EXT
+	_lock				br_ext_lock;
+	/* unsigned int			macclone_completed; */
+	struct nat25_network_db_entry	*nethash[NAT25_HASH_SIZE];
+	int				pppoe_connection_in_progress;
+	unsigned char			pppoe_addr[MACADDRLEN];
+	unsigned char			scdb_mac[MACADDRLEN];
+	unsigned char			scdb_ip[4];
+	struct nat25_network_db_entry	*scdb_entry;
+	unsigned char			br_mac[MACADDRLEN];
+	unsigned char			br_ip[4];
+
+	struct br_ext_info			ethBrExtInfo;
+	#endif /* CONFIG_BR_EXT */
+
+	#if 0 /*#ifdef CONFIG_MAC_LOOPBACK_DRIVER*/
+	PLOOPBACKDATA ploopback;
+	#endif
+
+	#ifdef PLATFORM_FREEBSD
+	_nic_hdl pifp;
+	_lock glock;
+	#endif /* PLATFORM_FREEBSD */
+
+	/* for debug purpose */
+#define NO_FIX_RATE		0xFFFF
+#define GET_FIX_RATE(v)		((v) & 0x0FFF)
+#define GET_FIX_RATE_SGI(v)	(((v) & 0x7000) >> 12)
+	u16 fix_rate;
+#define NO_FIX_BW		0xFF
+	u8 fix_bw;
+	u8 data_fb; /* data rate fallback, valid only when fix_rate is not 0xffff */
+	u8 power_offset;
+	u16 driver_tx_bw_mode;
+	u8 rsvd_page_offset;
+	u8 rsvd_page_num;
+	u8 ch_clm_ratio;
+	u8 ch_nhm_ratio;
+	#ifdef CONFIG_SUPPORT_FIFO_DUMP
+	u8 fifo_sel;
+	u32 fifo_addr;
+	u32 fifo_size;
+	#endif
+
+	u8 bLinkInfoDump;
+	/*	Added by Albert 2012/10/26 */
+	/*	The driver will show up the desired channel number when this flag is 1. */
+	u8 bNotifyChannelChange;
+	u8 bsta_tp_dump;
+	#ifdef CONFIG_P2P
+	/*	Added by Albert 2012/12/06 */
+	/*	The driver will show the current P2P status when the upper application reads it. */
+	u8 bShowGetP2PState;
+	#endif
+	u8 driver_vcs_en; /* Enable=1, Disable=0 driver control vrtl_carrier_sense for tx */
+	u8 driver_vcs_type;/* force 0:disable VCS, 1:RTS-CTS, 2:CTS-to-self when vcs_en=1. */
+	u8 driver_ampdu_spacing;/* driver control AMPDU Density for peer sta's rx */
+	u8 driver_rx_ampdu_factor;/* 0xff: disable drv ctrl, 0:8k, 1:16k, 2:32k, 3:64k; */
+	u8 driver_rx_ampdu_spacing;  /* driver control Rx AMPDU Density */
+	u8 fix_rx_ampdu_accept;
+	u16 fix_rx_ampdu_size; /* 0~1024, TODO:consider each sta and each TID */
+
+	#ifdef DBG_RX_COUNTER_DUMP
+	u8 dump_rx_cnt_mode;/*BIT0:drv,BIT1:mac,BIT2:phy*/
+	u32 drv_rx_cnt_ok;
+	u32 drv_rx_cnt_crcerror;
+	u32 drv_rx_cnt_drop;
+	#endif
+
+	#ifdef CONFIG_DBG_COUNTER
+	struct rx_logs rx_logs;
+	struct tx_logs tx_logs;
+	struct int_logs int_logs;
+	#endif
+
+#ifdef RTW_PHL_DBG_CMD
+
+	struct core_logs core_logs;
+	struct phl_logs phl_logs;
+
+	u32 txForce_enable;
+	u32 txForce_rate;
+	u32 txForce_agg;
+	u32 txForce_aggnum;
+	u32 txForce_gi;
+
+	u32 sniffer_enable;
+	u8	record_enable;
+#endif
+#ifdef RTW_PHL_TX
+	u8 *pxmit_txreq_buf;
+	_queue	free_txreq_queue;
+	u32 free_txreq_cnt;
+	u32 txreq_full_cnt;
+
+	u32 tx_ring_idx;
+	u8 *tx_pool_ring [MAX_TX_RING_NUM];
+#endif
+#ifdef CONFIG_RTW_CORE_RXSC
+	u8 enable_rxsc;
+#endif
+#ifdef DBG_CONFIG_CMD_DISP
+	enum phl_cmd_type cmd_type;
+	u32 cmd_timeout;
+#endif
+	u8 dis_turboedca;/*	1: disable turboedca,
+					2. disable turboedca,and setting EDCA parameter based on the input parameter*/
+	u32 dis_rt_cap; /* runtime disable CAP for testing */
+	u32 edca_param_mode;
+	u32 last_edca;
+	u8 edca_reset;/*1:be reset 0:not reset*/
+
+	_lock connect_st_lock;
+	u8 connect_state;
+#define CONNECT_ST_IDLE		0
+#define CONNECT_ST_REQUESTING	1
+#define CONNECT_ST_ACQUIRED	2
+	bool connect_abort;
+	struct phl_cmd_token_req connect_req;
+	u32 connect_token;
+	enum phl_band_idx connect_bidx;/*connect band idx*/
+
+	_lock disconnect_lock;
+	u8 disconnect_state;
+#define DISCON_ST_IDLE		0
+#define DISCON_ST_ONGOING	1
+#define DISCON_ST_ABORT		2
+	struct phl_cmd_token_req disconnect_req;
+	u32 disconnect_token;
+	enum phl_band_idx disconnect_bidx;/*disconnect band idx*/
+	struct cmd_obj *discon_cmd;
+
+#ifdef CONFIG_ECSA_PHL
+	struct core_ecsa_info ecsa_info;
+#endif
+
+	/*
+	* Indicates whether the current TSF command should be enqueued.
+	* true  - enqueue the command
+	* false - do not enqueue
+	*/
+	bool cur_tsf_cmd_enqueue;
+};
+
+#define adapter_to_dvobj(adapter) ((adapter)->dvobj)
+#define adapter_to_regsty(adapter) dvobj_to_regsty(adapter_to_dvobj((adapter)))
+#define adapter_to_pwrctl(adapter) dvobj_to_pwrctl(adapter_to_dvobj((adapter)))
+#ifdef CONFIG_WOWLAN
+#define adapter_to_wowlan(adapter) dvobj_to_wowlan(adapter_to_dvobj((adapter)))
+#endif /* CONFIG_WOWLAN */
+
+#define adapter_wdev_data(adapter) (&((adapter)->wdev_data))
+#define adapter_to_wiphy(adapter) dvobj_to_wiphy(adapter_to_dvobj(adapter))
+
+#define adapter_to_rfctl(adapter) dvobj_to_rfctl(adapter_to_dvobj((adapter)))
+
+#ifdef CONFIG_RTW_WDS
+#define adapter_use_wds(adapter) (adapter->use_wds)
+#define adapter_set_use_wds(adapter, en) do { \
+		(adapter)->use_wds = (en) ? 1 : 0; \
+		RTW_INFO(FUNC_ADPT_FMT" set use_wds=%d\n", FUNC_ADPT_ARG(adapter), (adapter)->use_wds); \
+	} while (0)
+#else
+#define adapter_use_wds(adapter) 0
+#endif
+
+#define adapter_mac_addr(adapter) (adapter->mac_addr)
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+#define adapter_pno_mac_addr(adapter) \
+	((adapter_wdev_data(adapter))->pno_mac_addr)
+#endif
+
+#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+#define adapter_pno_mac_addr(adapter) \
+	((adapter_wdev_data(adapter))->pno_mac_addr)
+#endif
+
+#define adapter_to_chset(adapter) (&adapter_to_rfctl((adapter))->chset)
+
+#define ADAPTER_GET_LINK_NUM(adapter) ((adapter)->adapter_link_num)
+
+#define mlme_to_adapter(mlme) container_of((mlme), _adapter, mlmepriv)
+#define tdls_info_to_adapter(tdls) container_of((tdls), _adapter, tdlsinfo)
+
+#define rtw_get_chip_id(adapter) (((_adapter *)adapter)->dvobj->chip_id)
+#define rtw_get_intf_type(adapter) (((_adapter *)adapter)->dvobj->interface_type)
+
+#define rtw_get_mi_nums(adapter) (((_adapter *)adapter)->dvobj->iface_nums)
+
+static inline void dev_set_surprise_removed(struct dvobj_priv *dvobj)
+{
+	ATOMIC_SET(&dvobj->bSurpriseRemoved, _TRUE);
+	if (dvobj->phl)
+		rtw_phl_dev_terminate_ntf(dvobj->phl);
+}
+static inline void dev_clr_surprise_removed(struct dvobj_priv *dvobj)
+{
+	ATOMIC_SET(&dvobj->bSurpriseRemoved, _FALSE);
+	if (dvobj->phl_com)
+		CLEAR_STATUS_FLAG(dvobj->phl_com->dev_state, RTW_DEV_SURPRISE_REMOVAL);
+}
+static inline void dev_set_drv_stopped(struct dvobj_priv *dvobj)
+{
+	ATOMIC_SET(&dvobj->bDriverStopped, _TRUE);
+}
+static inline void dev_clr_drv_stopped(struct dvobj_priv *dvobj)
+{
+	ATOMIC_SET(&dvobj->bDriverStopped, _FALSE);
+}
+static inline void dev_set_hw_start(struct dvobj_priv *dvobj)
+{
+	ATOMIC_SET(&dvobj->hw_start, _TRUE);
+}
+static inline void dev_clr_hw_start(struct dvobj_priv *dvobj)
+{
+	ATOMIC_SET(&dvobj->hw_start, _FALSE);
+}
+
+#define dev_is_surprise_removed(dvobj)	(ATOMIC_READ(&dvobj->bSurpriseRemoved) == _TRUE)
+#define dev_is_drv_stopped(dvobj)		(ATOMIC_READ(&dvobj->bDriverStopped) == _TRUE)
+#define dev_is_hw_start(dvobj)		(ATOMIC_READ(&dvobj->hw_start) == _TRUE)
+/*
+ * Function disabled.
+ *   */
+#define DF_TX_BIT		BIT0			/*rtw_usb_write_port_cancel*/
+#define DF_RX_BIT		BIT1			/*rtw_usb_read_port_cancel*/
+#define DF_IO_BIT		BIT2
+
+/* #define RTW_DISABLE_FUNC(padapter, func) (ATOMIC_ADD(&dvobj->disable_func, (func))) */
+/* #define RTW_ENABLE_FUNC(padapter, func) (ATOMIC_SUB(&dvobj->disable_func, (func))) */
+__inline static void RTW_DISABLE_FUNC(struct dvobj_priv *dvobj, int func_bit)
+{
+	int df = ATOMIC_READ(&dvobj->disable_func);
+	df |= func_bit;
+	ATOMIC_SET(&dvobj->disable_func, df);
+}
+
+__inline static void RTW_ENABLE_FUNC(struct dvobj_priv *dvobj, int func_bit)
+{
+	int df = ATOMIC_READ(&dvobj->disable_func);
+	df &= ~(func_bit);
+	ATOMIC_SET(&dvobj->disable_func, df);
+}
+
+#define RTW_CANNOT_RUN(dvobj) \
+	(dev_is_surprise_removed(dvobj) || \
+	dev_is_drv_stopped(dvobj))
+
+#define RTW_IS_FUNC_DISABLED(dvobj, func_bit) \
+	(ATOMIC_READ(&dvobj->disable_func) & (func_bit))
+
+#define RTW_CANNOT_IO(dvobj) \
+	(dev_is_surprise_removed(dvobj) || \
+	 RTW_IS_FUNC_DISABLED((dvobj), DF_IO_BIT))
+
+#define RTW_CANNOT_RX(dvobj) \
+	(RTW_CANNOT_RUN(dvobj) || \
+	 RTW_IS_FUNC_DISABLED((dvobj), DF_RX_BIT))
+
+#define RTW_CANNOT_TX(dvobj) \
+	(RTW_CANNOT_RUN(dvobj) || \
+	 RTW_IS_FUNC_DISABLED((dvobj), DF_TX_BIT))
+
+
+/* HCI Related header file */
+#ifdef CONFIG_USB_HCI
+	#include <usb_ops.h>
+#endif
+
+#ifdef CONFIG_SDIO_HCI
+	#include <sdio_ops.h>
+#endif
+
+#ifdef CONFIG_GSPI_HCI
+	#include <gspi_ops.h>
+#endif
+
+#ifdef CONFIG_PCI_HCI
+	#include <pci_ops.h>
+#endif
+#include <rtw_trx_ops.h>
+
+
+#endif /* __DRV_TYPES_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_gspi.h b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_gspi.h
new file mode 100644
index 000000000000..c22c4972232d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_gspi.h
@@ -0,0 +1,49 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __DRV_TYPES_GSPI_H__
+#define __DRV_TYPES_GSPI_H__
+
+/* SPI Header Files */
+#ifdef PLATFORM_LINUX
+	#include <linux/platform_device.h>
+	#include <linux/spi/spi.h>
+	#include <linux/gpio.h>
+	/* #include <mach/ldo.h> */
+	#include <asm/mach-types.h>
+	#include <asm/gpio.h>
+	#include <asm/io.h>
+	#include <mach/board.h>
+	#include <mach/hardware.h>
+	#include <mach/irqs.h>
+	#include <custom_gpio.h>
+#endif
+
+
+typedef struct gspi_data {
+	u8  func_number;
+
+	u8  tx_block_mode;
+	u8  rx_block_mode;
+	u32 block_transfer_len;
+
+#ifdef PLATFORM_LINUX
+	struct spi_device *func;
+
+	struct workqueue_struct *priv_wq;
+	struct delayed_work irq_work;
+#endif
+} GSPI_DATA, *PGSPI_DATA;
+
+#endif /*  #ifndef __DRV_TYPES_GSPI_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_pci.h b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_pci.h
new file mode 100644
index 000000000000..78513abf9698
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_pci.h
@@ -0,0 +1,96 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __DRV_TYPES_PCI_H__
+#define __DRV_TYPES_PCI_H__
+
+#ifdef PLATFORM_LINUX
+	#include <linux/pci.h>
+#endif
+
+#define	INTEL_VENDOR_ID				0x8086
+#define	SIS_VENDOR_ID					0x1039
+#define	ATI_VENDOR_ID					0x1002
+#define	ATI_DEVICE_ID					0x7914
+#define	AMD_VENDOR_ID					0x1022
+
+#define PCI_VENDER_ID_REALTEK		0x10ec
+
+enum aspm_mode {
+	ASPM_MODE_UND,
+	ASPM_MODE_PERF,
+	ASPM_MODE_PS,
+	ASPM_MODE_DEF,
+};
+
+struct pci_priv {
+	BOOLEAN		pci_clk_req;
+
+	u8	pciehdr_offset;
+
+	u8	linkctrl_reg;
+	u8	pcibridge_linkctrlreg;
+
+	u8	amd_l1_patch;
+};
+
+typedef struct _RT_ISR_CONTENT {
+	union {
+		u32			IntArray[2];
+		u32			IntReg4Byte;
+		u16			IntReg2Byte;
+	};
+} RT_ISR_CONTENT, *PRT_ISR_CONTENT;
+
+typedef struct pci_data {
+#ifdef PLATFORM_LINUX
+	struct pci_dev *ppcidev;
+
+	/* PCI MEM map */
+	unsigned long	pci_mem_end;	/* shared mem end	*/
+	unsigned long	pci_mem_start;	/* shared mem start	*/
+
+	/* PCI IO map */
+	unsigned long	pci_base_addr;	/* device I/O address	*/
+
+	#ifdef RTK_129X_PLATFORM
+	unsigned long	ctrl_start;
+	/* PCI MASK addr */
+	unsigned long	mask_addr;
+
+	/* PCI TRANSLATE addr */
+	unsigned long	tran_addr;
+
+	_lock	io_reg_lock;
+	#endif
+
+	/* PciBridge */
+	struct pci_priv pcipriv;
+
+	u8 irq_alloc;
+	unsigned int irq; /* get from pci_dev.irq, store to net_device.irq */
+	u16	irqline;
+	u8	irq_enabled;
+	RT_ISR_CONTENT	isr_content;
+	_lock	irq_th_lock;
+
+	u8	bdma64;
+
+#ifndef CONFIG_RTW_PCI_MSI_DISABLE
+	u32 msi_en;
+#endif
+#endif/* PLATFORM_LINUX */
+} PCI_DATA, *PPCI_DATA;
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_sdio.h b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_sdio.h
new file mode 100644
index 000000000000..fea501e0dd96
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_sdio.h
@@ -0,0 +1,90 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __DRV_TYPES_SDIO_H__
+#define __DRV_TYPES_SDIO_H__
+
+#ifdef PLATFORM_LINUX
+#include <linux/mmc/sdio_func.h>
+#include <linux/mmc/sdio_ids.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/card.h>
+#endif /* PLATFORM_LINUX */
+
+#define RTW_SDIO_CLK_33M	33000000
+#define RTW_SDIO_CLK_40M	40000000
+#define RTW_SDIO_CLK_80M	80000000
+#define RTW_SDIO_CLK_160M	160000000
+
+struct sdio_data {
+	u8  func_number;
+
+	u8  tx_block_mode;
+	u8  rx_block_mode;
+	u32 block_transfer_len;
+	u16 max_byte_size;	/* MAX size for cmd53 byte mode */
+	u8 irq_alloc;
+
+#ifdef PLATFORM_LINUX
+	struct mmc_card *card;
+	struct sdio_func *func;
+	u8 *tmpbuf;		/* buffer for register access */
+	u8 tmpbuf_sz;
+#endif
+	_thread_hdl_ sys_sdio_irq_thd;
+	unsigned int clock;
+	unsigned int timing;
+	u8	sd3_bus_mode;
+
+#ifdef DBG_SDIO
+#ifdef PLATFORM_LINUX
+	struct proc_dir_entry *proc_sdio_dbg;
+#endif /* PLATFORM_LINUX */
+
+	u32 cmd52_err_cnt;	/* CMD52 I/O error count */
+	u32 cmd53_err_cnt;	/* CMD53 I/O error count */
+
+#if (DBG_SDIO >= 1)
+	u32 reg_dump_mark;	/* reg dump at specific error count */
+#endif /* DBG_SDIO >= 1 */
+
+#if (DBG_SDIO >= 2)
+	u8 *dbg_msg;		/* Messages for debug */
+	u8 dbg_msg_size;
+	u8 *reg_mac;		/* Device MAC register, 0x0~0x800 */
+	u8 *reg_mac_ext;	/* Device MAC extend register, 0x1000~0x1800 */
+	u8 *reg_local;		/* Device SDIO local register, 0x0~0xFF */
+	u8 *reg_cia;		/* SDIO CIA(CCCR, FBR and etc.), 0x0~0x1FF */
+#endif /* DBG_SDIO >= 2 */
+
+#if (DBG_SDIO >= 3)
+	u8 dbg_enable;		/* 0/1: disable/enable debug mode */
+	u8 err_stop;		/* Stop(surprise remove) when I/O error happen */
+	u8 err_test;		/* Simulate error happen */
+	u8 err_test_triggered;	/* Simulate error already triggered */
+#endif /* DBG_SDIO >= 3 */
+#endif /* DBG_SDIO */
+};
+
+#define RTW_SDIO_ADDR_CMD52_BIT	(1<<17)
+#define RTW_SDIO_ADDR_CMD52_GEN(a)	(a | RTW_SDIO_ADDR_CMD52_BIT)
+#define RTW_SDIO_ADDR_CMD52_CLR(a)	(a&~RTW_SDIO_ADDR_CMD52_BIT)
+#define RTW_SDIO_ADDR_CMD52_CHK(a)	(a&RTW_SDIO_ADDR_CMD52_BIT ? 1 : 0)
+
+#define RTW_SDIO_ADDR_F0_BIT		(1<<18)
+#define RTW_SDIO_ADDR_F0_GEN(a)	(a | RTW_SDIO_ADDR_F0_BIT)
+#define RTW_SDIO_ADDR_F0_CLR(a)	(a&~RTW_SDIO_ADDR_F0_BIT)
+#define RTW_SDIO_ADDR_F0_CHK(a)	(a&RTW_SDIO_ADDR_F0_BIT ? 1 : 0)
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_usb.h b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_usb.h
new file mode 100644
index 000000000000..dd4e0e2b04b6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/drv_types_usb.h
@@ -0,0 +1,72 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __DRV_TYPES_USB_H__
+#define __DRV_TYPES_USB_H__
+
+/*FOR 8852AU*/
+#define MAX_ENDPOINT_NUM 9 /* 2 x Bulk-IN + 7 x Bulk-OUT */
+#define MAX_BULKOUT_NUM 7
+#define MAX_BULKIN_NUM 2 /*EP4 for Bulk-IN, EP8 for Bulk-IN interrupt or Bulk-INT*/
+
+#ifdef CONFIG_SELF_DIAG_INFO
+#define RX_STATS_MAX_NUM MAX_BULKIN_NUM
+#define TX_STATS_MAX_NUM MAX_BULKOUT_NUM
+
+struct trx_stats {
+	u32 rx_submit_cnt[RX_STATS_MAX_NUM];
+	u32 rx_submit_fail[RX_STATS_MAX_NUM];
+	u32 rx_complete_cnt[RX_STATS_MAX_NUM];
+	u32 rx_complete_fail[RX_STATS_MAX_NUM];
+	u32 tx_submit_cnt[TX_STATS_MAX_NUM];
+	u32 tx_submit_fail[TX_STATS_MAX_NUM];
+	u32 tx_complete_cnt[TX_STATS_MAX_NUM];
+	u32 tx_complete_fail[TX_STATS_MAX_NUM];
+	u32 last_rx_submit_cnt[RX_STATS_MAX_NUM];
+	u32 last_tx_submit_cnt[TX_STATS_MAX_NUM];
+};
+#endif
+
+typedef struct usb_data {
+	u8 usb_intf_start;
+
+	u8 usb_speed; /* 1.1, 2.0 or 3.0 */
+	u16 usb_bulkout_size;
+	u8 nr_endpoint; /*MAX_ENDPOINT_NUM*/
+
+	/* Bulk In , Out Pipe information */
+	int RtInPipe[MAX_BULKIN_NUM];
+	u8 inpipe_type[MAX_BULKIN_NUM];
+	u8 RtNumInPipes;
+	int RtOutPipe[MAX_BULKOUT_NUM];
+	u8 RtNumOutPipes;
+#ifdef CONFIG_SELF_DIAG_INFO
+	struct trx_stats trx_stats;
+#endif
+
+#ifdef CONFIG_USB_VENDOR_REQ_MUTEX
+	_mutex usb_vendor_req_mutex;
+#endif
+
+#ifdef CONFIG_USB_VENDOR_REQ_BUFFER_PREALLOC
+	u8 *usb_alloc_vendor_req_buf;
+	u8 *usb_vendor_req_buf;
+#endif
+
+#ifdef PLATFORM_LINUX
+	struct usb_interface *pusbintf;
+	struct usb_device *pusbdev;
+#endif/* PLATFORM_LINUX */
+} USB_DATA, *PUSB_DATA;
+#endif /*__DRV_TYPES_USB_H__*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/ethernet.h b/drivers/net/wireless/realtek/rtl8852bs/include/ethernet.h
new file mode 100644
index 000000000000..141512ec451d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/ethernet.h
@@ -0,0 +1,36 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+/*! \file */
+#ifndef __INC_ETHERNET_H
+#define __INC_ETHERNET_H
+
+#define ETHERNET_ADDRESS_LENGTH				6		/* !< Ethernet Address Length */
+#define ETHERNET_HEADER_SIZE				14		/* !< Ethernet Header Length */
+#define LLC_HEADER_SIZE						6		/* !< LLC Header Length */
+#define TYPE_LENGTH_FIELD_SIZE				2		/* !< Type/Length Size */
+#define MINIMUM_ETHERNET_PACKET_SIZE		60		/* !< Minimum Ethernet Packet Size */
+#define MAXIMUM_ETHERNET_PACKET_SIZE		1514	/* !< Maximum Ethernet Packet Size */
+
+#define RT_ETH_IS_MULTICAST(_pAddr)	((((u8 *)(_pAddr))[0]&0x01) != 0)		/* !< Is Multicast Address? */
+#define RT_ETH_IS_BROADCAST(_pAddr)	(\
+		((u8 *)(_pAddr))[0] == 0xff	&&		\
+		((u8 *)(_pAddr))[1] == 0xff	&&		\
+		((u8 *)(_pAddr))[2] == 0xff	&&		\
+		((u8 *)(_pAddr))[3] == 0xff	&&		\
+		((u8 *)(_pAddr))[4] == 0xff	&&		\
+		((u8 *)(_pAddr))[5] == 0xff)	/* !< Is Broadcast Address? */
+
+
+#endif /*  #ifndef __INC_ETHERNET_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/gspi_hal.h b/drivers/net/wireless/realtek/rtl8852bs/include/gspi_hal.h
new file mode 100644
index 000000000000..6b0561c6ee1a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/gspi_hal.h
@@ -0,0 +1,21 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __GSPI_HAL_H__
+#define __GSPI_HAL_H__
+
+
+void spi_int_dpc(_adapter *padapter, u32 sdio_hisr);
+
+#endif /* __GSPI_HAL_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/gspi_ops.h b/drivers/net/wireless/realtek/rtl8852bs/include/gspi_ops.h
new file mode 100644
index 000000000000..92d0413e56b6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/gspi_ops.h
@@ -0,0 +1,144 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __GSPI_OPS_H__
+#define __GSPI_OPS_H__
+
+/* follwing defination is based on
+ * GSPI spec of RTL8723, we temp
+ * suppose that it will be the same
+ * for diff chips of GSPI, if not
+ * we should move it to HAL folder */
+#define SPI_LOCAL_DOMAIN				0x0
+#define WLAN_IOREG_DOMAIN			0x8
+#define FW_FIFO_DOMAIN				0x4
+#define TX_HIQ_DOMAIN					0xc
+#define TX_MIQ_DOMAIN					0xd
+#define TX_LOQ_DOMAIN					0xe
+#define RX_RXFIFO_DOMAIN				0x1f
+
+/* IO Bus domain address mapping */
+#define DEFUALT_OFFSET					0x0
+#define SPI_LOCAL_OFFSET				0x10250000
+#define WLAN_IOREG_OFFSET			0x10260000
+#define FW_FIFO_OFFSET				0x10270000
+#define TX_HIQ_OFFSET					0x10310000
+#define TX_MIQ_OFFSET					0x1032000
+#define TX_LOQ_OFFSET					0x10330000
+#define RX_RXOFF_OFFSET				0x10340000
+
+/* SPI Local registers */
+#define SPI_REG_TX_CTRL					0x0000 /* SPI Tx Control */
+#define SPI_REG_STATUS_RECOVERY		0x0004
+#define SPI_REG_INT_TIMEOUT			0x0006
+#define SPI_REG_HIMR					0x0014 /* SPI Host Interrupt Mask */
+#define SPI_REG_HISR					0x0018 /* SPI Host Interrupt Service Routine */
+#define SPI_REG_RX0_REQ_LEN			0x001C /* RXDMA Request Length */
+#define SPI_REG_FREE_TXPG				0x0020 /* Free Tx Buffer Page */
+#define SPI_REG_HCPWM1					0x0024 /* HCI Current Power Mode 1 */
+#define SPI_REG_HCPWM2					0x0026 /* HCI Current Power Mode 2 */
+#define SPI_REG_HTSFR_INFO				0x0030 /* HTSF Informaion */
+#define SPI_REG_HRPWM1					0x0080 /* HCI Request Power Mode 1 */
+#define SPI_REG_HRPWM2					0x0082 /* HCI Request Power Mode 2 */
+#define SPI_REG_HPS_CLKR				0x0084 /* HCI Power Save Clock */
+#define SPI_REG_HSUS_CTRL				0x0086 /* SPI HCI Suspend Control */
+#define SPI_REG_HIMR_ON				0x0090 /* SPI Host Extension Interrupt Mask Always */
+#define SPI_REG_HISR_ON				0x0091 /* SPI Host Extension Interrupt Status Always */
+#define SPI_REG_CFG						0x00F0 /* SPI Configuration Register */
+
+#define SPI_TX_CTRL				(SPI_REG_TX_CTRL | SPI_LOCAL_OFFSET)
+#define SPI_STATUS_RECOVERY			(SPI_REG_STATUS_RECOVERY | SPI_LOCAL_OFFSET)
+#define SPI_INT_TIMEOUT					(SPI_REG_INT_TIMEOUT | SPI_LOCAL_OFFSET)
+#define SPI_HIMR				(SPI_REG_HIMR | SPI_LOCAL_OFFSET)
+#define SPI_HISR				(SPI_REG_HISR | SPI_LOCAL_OFFSET)
+#define SPI_RX0_REQ_LEN_1_BYTE		(SPI_REG_RX0_REQ_LEN | SPI_LOCAL_OFFSET)
+#define SPI_FREE_TXPG			(SPI_REG_FREE_TXPG | SPI_LOCAL_OFFSET)
+
+#define	SPI_HIMR_DISABLED				0
+
+/* SPI HIMR MASK diff with SDIO */
+#define SPI_HISR_RX_REQUEST			BIT(0)
+#define SPI_HISR_AVAL					BIT(1)
+#define SPI_HISR_TXERR					BIT(2)
+#define SPI_HISR_RXERR					BIT(3)
+#define SPI_HISR_TXFOVW				BIT(4)
+#define SPI_HISR_RXFOVW				BIT(5)
+#define SPI_HISR_TXBCNOK				BIT(6)
+#define SPI_HISR_TXBCNERR				BIT(7)
+#define SPI_HISR_BCNERLY_INT			BIT(16)
+#define SPI_HISR_ATIMEND				BIT(17)
+#define SPI_HISR_ATIMEND_E				BIT(18)
+#define SPI_HISR_CTWEND				BIT(19)
+#define SPI_HISR_C2HCMD				BIT(20)
+#define SPI_HISR_CPWM1					BIT(21)
+#define SPI_HISR_CPWM2					BIT(22)
+#define SPI_HISR_HSISR_IND				BIT(23)
+#define SPI_HISR_GTINT3_IND				BIT(24)
+#define SPI_HISR_GTINT4_IND				BIT(25)
+#define SPI_HISR_PSTIMEOUT				BIT(26)
+#define SPI_HISR_OCPINT					BIT(27)
+#define SPI_HISR_TSF_BIT32_TOGGLE		BIT(29)
+
+#define MASK_SPI_HISR_CLEAR		(SPI_HISR_TXERR |\
+		SPI_HISR_RXERR |\
+		SPI_HISR_TXFOVW |\
+		SPI_HISR_RXFOVW |\
+		SPI_HISR_TXBCNOK |\
+		SPI_HISR_TXBCNERR |\
+		SPI_HISR_C2HCMD |\
+		SPI_HISR_CPWM1 |\
+		SPI_HISR_CPWM2 |\
+		SPI_HISR_HSISR_IND |\
+		SPI_HISR_GTINT3_IND |\
+		SPI_HISR_GTINT4_IND |\
+		SPI_HISR_PSTIMEOUT |\
+		SPI_HISR_OCPINT)
+
+#define REG_LEN_FORMAT(pcmd, x) 			SET_BITS_TO_LE_4BYTE(pcmd, 0, 8, x)/* (x<<(unsigned int)24) */
+#define REG_ADDR_FORMAT(pcmd, x) 			SET_BITS_TO_LE_4BYTE(pcmd, 8, 16, x)/* (x<<(unsigned int)16) */
+#define REG_DOMAIN_ID_FORMAT(pcmd, x) 		SET_BITS_TO_LE_4BYTE(pcmd, 24, 5, x)/* (x<<(unsigned int)0) */
+#define REG_FUN_FORMAT(pcmd, x) 			SET_BITS_TO_LE_4BYTE(pcmd, 29, 2, x)/* (x<<(unsigned int)5) */
+#define REG_RW_FORMAT(pcmd, x) 				SET_BITS_TO_LE_4BYTE(pcmd, 31, 1, x)/* (x<<(unsigned int)7) */
+
+#define FIFO_LEN_FORMAT(pcmd, x) 			SET_BITS_TO_LE_4BYTE(pcmd, 0, 16, x)/* (x<<(unsigned int)24)
+ * #define FIFO_ADDR_FORMAT(pcmd,x) 			SET_BITS_TO_LE_4BYTE(pcmd, 8, 16, x) */ /* (x<<(unsigned int)16) */
+#define FIFO_DOMAIN_ID_FORMAT(pcmd, x) 	SET_BITS_TO_LE_4BYTE(pcmd, 24, 5, x)/* (x<<(unsigned int)0) */
+#define FIFO_FUN_FORMAT(pcmd, x) 			SET_BITS_TO_LE_4BYTE(pcmd, 29, 2, x)/* (x<<(unsigned int)5) */
+#define FIFO_RW_FORMAT(pcmd, x) 			SET_BITS_TO_LE_4BYTE(pcmd, 31, 1, x)/* (x<<(unsigned int)7) */
+
+
+/* get status dword0 */
+#define GET_STATUS_PUB_PAGE_NUM(status)		LE_BITS_TO_4BYTE(status, 24, 8)
+#define GET_STATUS_HI_PAGE_NUM(status)		LE_BITS_TO_4BYTE(status, 18, 6)
+#define GET_STATUS_MID_PAGE_NUM(status)		LE_BITS_TO_4BYTE(status, 12, 6)
+#define GET_STATUS_LOW_PAGE_NUM(status)		LE_BITS_TO_4BYTE(status, 6, 6)
+#define GET_STATUS_HISR_HI6BIT(status)			LE_BITS_TO_4BYTE(status, 0, 6)
+
+/* get status dword1 */
+#define GET_STATUS_HISR_MID8BIT(status)		LE_BITS_TO_4BYTE(status + 4, 24, 8)
+#define GET_STATUS_HISR_LOW8BIT(status)		LE_BITS_TO_4BYTE(status + 4, 16, 8)
+#define GET_STATUS_ERROR(status)				LE_BITS_TO_4BYTE(status + 4, 17, 1)
+#define GET_STATUS_INT(status)				LE_BITS_TO_4BYTE(status + 4, 16, 1)
+#define GET_STATUS_RX_LENGTH(status)			LE_BITS_TO_4BYTE(status + 4, 0, 16)
+
+
+#define RXDESC_SIZE	24
+
+
+struct spi_more_data {
+	unsigned long more_data;
+	unsigned long len;
+};
+
+#endif /* __GSPI_OPS_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/gspi_ops_linux.h b/drivers/net/wireless/realtek/rtl8852bs/include/gspi_ops_linux.h
new file mode 100644
index 000000000000..0ba263de8fd7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/gspi_ops_linux.h
@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __SDIO_OPS_LINUX_H__
+#define __SDIO_OPS_LINUX_H__
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/ieee80211.h b/drivers/net/wireless/realtek/rtl8852bs/include/ieee80211.h
new file mode 100644
index 000000000000..6b165d23ca7e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/ieee80211.h
@@ -0,0 +1,2071 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __IEEE80211_H
+#define __IEEE80211_H
+
+#define MGMT_QUEUE_NUM 5
+#define MAX_RF_PATH 4
+#define RF_PATH_MAX MAX_RF_PATH
+
+#define ETH_ALEN	6
+#define ETH_TYPE_LEN		2
+#define PAYLOAD_TYPE_LEN	1
+
+#define NET80211_TU_TO_US	1024		/* unit:us */
+#define DEFAULT_BCN_INTERVAL 100 /* 100 ms */
+
+#ifdef CONFIG_AP_MODE
+
+#define RTL_IOCTL_HOSTAPD (SIOCIWFIRSTPRIV + 28)
+
+/* RTL871X_IOCTL_HOSTAPD ioctl() cmd: */
+enum {
+	RTL871X_HOSTAPD_FLUSH = 1,
+	RTL871X_HOSTAPD_ADD_STA = 2,
+	RTL871X_HOSTAPD_REMOVE_STA = 3,
+	RTL871X_HOSTAPD_GET_INFO_STA = 4,
+	/* REMOVED: PRISM2_HOSTAPD_RESET_TXEXC_STA = 5, */
+	RTL871X_HOSTAPD_GET_WPAIE_STA = 5,
+	RTL871X_SET_ENCRYPTION = 6,
+	RTL871X_GET_ENCRYPTION = 7,
+	RTL871X_HOSTAPD_SET_FLAGS_STA = 8,
+	RTL871X_HOSTAPD_GET_RID = 9,
+	RTL871X_HOSTAPD_SET_RID = 10,
+	RTL871X_HOSTAPD_SET_ASSOC_AP_ADDR = 11,
+	RTL871X_HOSTAPD_SET_GENERIC_ELEMENT = 12,
+	RTL871X_HOSTAPD_MLME = 13,
+	RTL871X_HOSTAPD_SCAN_REQ = 14,
+	RTL871X_HOSTAPD_STA_CLEAR_STATS = 15,
+	RTL871X_HOSTAPD_SET_BEACON = 16,
+	RTL871X_HOSTAPD_SET_WPS_BEACON = 17,
+	RTL871X_HOSTAPD_SET_WPS_PROBE_RESP = 18,
+	RTL871X_HOSTAPD_SET_WPS_ASSOC_RESP = 19,
+	RTL871X_HOSTAPD_SET_HIDDEN_SSID = 20,
+	RTL871X_HOSTAPD_SET_MACADDR_ACL = 21,
+	RTL871X_HOSTAPD_ACL_ADD_STA = 22,
+	RTL871X_HOSTAPD_ACL_REMOVE_STA = 23,
+};
+
+/* STA flags */
+#define WLAN_STA_AUTH BIT(0)
+#define WLAN_STA_ASSOC BIT(1)
+#define WLAN_STA_PS BIT(2)
+#define WLAN_STA_TIM BIT(3)
+#define WLAN_STA_PERM BIT(4)
+#define WLAN_STA_AUTHORIZED BIT(5)
+#define WLAN_STA_PENDING_POLL BIT(6) /* pending activity poll not ACKed */
+#define WLAN_STA_SHORT_PREAMBLE BIT(7)
+#define WLAN_STA_PREAUTH BIT(8)
+#define WLAN_STA_WME BIT(9)
+#define WLAN_STA_MFP BIT(10)
+#define WLAN_STA_HT BIT(11)
+#define WLAN_STA_WPS BIT(12)
+#define WLAN_STA_MAYBE_WPS BIT(13)
+#define WLAN_STA_VHT BIT(14)
+#define WLAN_STA_HE BIT(15)
+#define WLAN_STA_WDS BIT(16)
+#define WLAN_STA_MULTI_AP BIT(17)
+#define WLAN_STA_AMSDU_DISABLE BIT(18)
+#define WLAN_STA_6G_BAND_CAP BIT(19)
+#define WLAN_STA_NONERP BIT(31)
+
+#endif
+
+#define IEEE_CMD_SET_WPA_PARAM			1
+#define IEEE_CMD_SET_WPA_IE				2
+#define IEEE_CMD_SET_ENCRYPTION			3
+#define IEEE_CMD_MLME						4
+
+#define IEEE_PARAM_WPA_ENABLED				1
+#define IEEE_PARAM_TKIP_COUNTERMEASURES		2
+#define IEEE_PARAM_DROP_UNENCRYPTED			3
+#define IEEE_PARAM_PRIVACY_INVOKED			4
+#define IEEE_PARAM_AUTH_ALGS					5
+#define IEEE_PARAM_IEEE_802_1X				6
+#define IEEE_PARAM_WPAX_SELECT				7
+
+#define AUTH_ALG_OPEN_SYSTEM			0x1
+#define AUTH_ALG_SHARED_KEY			0x2
+#define AUTH_ALG_LEAP				0x00000004
+
+#define IEEE_MLME_STA_DEAUTH				1
+#define IEEE_MLME_STA_DISASSOC			2
+
+#define IEEE_CRYPT_ERR_UNKNOWN_ALG			2
+#define IEEE_CRYPT_ERR_UNKNOWN_ADDR			3
+#define IEEE_CRYPT_ERR_CRYPT_INIT_FAILED		4
+#define IEEE_CRYPT_ERR_KEY_SET_FAILED			5
+#define IEEE_CRYPT_ERR_TX_KEY_SET_FAILED		6
+#define IEEE_CRYPT_ERR_CARD_CONF_FAILED		7
+
+
+#define	IEEE_CRYPT_ALG_NAME_LEN			16
+
+#define WPA_CIPHER_NONE	BIT(0)
+#define WPA_CIPHER_WEP40	BIT(1)
+#define WPA_CIPHER_WEP104 BIT(2)
+#define WPA_CIPHER_TKIP	BIT(3)
+#define WPA_CIPHER_CCMP	BIT(4)
+#define WPA_CIPHER_GCMP	BIT(5)
+#define WPA_CIPHER_GCMP_256	BIT(6)
+#define WPA_CIPHER_CCMP_256	BIT(7)
+#define WPA_CIPHER_BIP_CMAC_128	BIT(8)
+#define WPA_CIPHER_BIP_GMAC_128	BIT(9)
+#define WPA_CIPHER_BIP_GMAC_256	BIT(10)
+#define WPA_CIPHER_BIP_CMAC_256	BIT(11)
+
+
+#define WPA_SELECTOR_LEN 4
+extern u8 RTW_WPA_OUI_TYPE[] ;
+extern u16 RTW_WPA_VERSION ;
+extern u8 WPA_AUTH_KEY_MGMT_NONE[];
+extern u8 WPA_AUTH_KEY_MGMT_UNSPEC_802_1X[];
+extern u8 WPA_AUTH_KEY_MGMT_PSK_OVER_802_1X[];
+extern u8 WPA_CIPHER_SUITE_NONE[];
+extern u8 WPA_CIPHER_SUITE_WEP40[];
+extern u8 WPA_CIPHER_SUITE_TKIP[];
+extern u8 WPA_CIPHER_SUITE_WRAP[];
+extern u8 WPA_CIPHER_SUITE_CCMP[];
+extern u8 RSN_CIPHER_SUITE_GCMP[];
+extern u8 RSN_CIPHER_SUITE_GCMP_256[];
+extern u8 RSN_CIPHER_SUITE_CCMP_256[];
+extern u8 WPA_CIPHER_SUITE_WEP104[];
+
+
+#define RSN_HEADER_LEN 4
+#define RSN_SELECTOR_LEN 4
+#define RSN_PMKID_LEN 16
+
+extern u16 RSN_VERSION_BSD;
+extern u8 RSN_CIPHER_SUITE_NONE[];
+extern u8 RSN_CIPHER_SUITE_WEP40[];
+extern u8 RSN_CIPHER_SUITE_TKIP[];
+extern u8 RSN_CIPHER_SUITE_WRAP[];
+extern u8 RSN_CIPHER_SUITE_CCMP[];
+extern u8 RSN_CIPHER_SUITE_WEP104[];
+
+/* AKM suite type */
+extern u8 WLAN_AKM_8021X[];
+extern u8 WLAN_AKM_PSK[];
+extern u8 WLAN_AKM_FT_8021X[];
+extern u8 WLAN_AKM_FT_PSK[];
+extern u8 WLAN_AKM_8021X_SHA256[];
+extern u8 WLAN_AKM_PSK_SHA256[];
+extern u8 WLAN_AKM_TDLS[];
+extern u8 WLAN_AKM_SAE[];
+extern u8 WLAN_AKM_FT_OVER_SAE[];
+extern u8 WLAN_AKM_8021X_SUITE_B[];
+extern u8 WLAN_AKM_8021X_SUITE_B_192[];
+extern u8 WLAN_AKM_FILS_SHA256[];
+extern u8 WLAN_AKM_FILS_SHA384[];
+extern u8 WLAN_AKM_FT_FILS_SHA256[];
+extern u8 WLAN_AKM_FT_FILS_SHA384[];
+
+#define WLAN_AKM_TYPE_8021X BIT(0)
+#define WLAN_AKM_TYPE_PSK BIT(1)
+#define WLAN_AKM_TYPE_FT_8021X BIT(2)
+#define WLAN_AKM_TYPE_FT_PSK BIT(3)
+#define WLAN_AKM_TYPE_8021X_SHA256 BIT(4)
+#define WLAN_AKM_TYPE_PSK_SHA256 BIT(5)
+#define WLAN_AKM_TYPE_TDLS BIT(6)
+#define WLAN_AKM_TYPE_SAE BIT(7)
+#define WLAN_AKM_TYPE_FT_OVER_SAE BIT(8)
+#define WLAN_AKM_TYPE_8021X_SUITE_B BIT(9)
+#define WLAN_AKM_TYPE_8021X_SUITE_B_192 BIT(10)
+#define WLAN_AKM_TYPE_FILS_SHA256 BIT(11)
+#define WLAN_AKM_TYPE_FILS_SHA384 BIT(12)
+#define WLAN_AKM_TYPE_FT_FILS_SHA256 BIT(13)
+#define WLAN_AKM_TYPE_FT_FILS_SHA384 BIT(14)
+
+/* IEEE 802.11i */
+#define PMKID_LEN 16
+#define PMK_LEN 32
+#define PMK_LEN_SUITE_B_192 48
+#define PMK_LEN_MAX 48
+#define WPA_REPLAY_COUNTER_LEN 8
+#define WPA_NONCE_LEN 32
+#define WPA_KEY_RSC_LEN 8
+#define WPA_GMK_LEN 32
+#define WPA_GTK_MAX_LEN 32
+
+/* IEEE 802.11, 8.5.2 EAPOL-Key frames */
+#define WPA_KEY_INFO_TYPE_MASK ((u16) (BIT(0) | BIT(1) | BIT(2)))
+#define WPA_KEY_INFO_TYPE_AKM_DEFINED 0
+#define WPA_KEY_INFO_TYPE_HMAC_MD5_RC4 BIT(0)
+#define WPA_KEY_INFO_TYPE_HMAC_SHA1_AES BIT(1)
+#define WPA_KEY_INFO_TYPE_AES_128_CMAC 3
+#define WPA_KEY_INFO_KEY_TYPE BIT(3) /* 1 = Pairwise, 0 = Group key */
+/* bit4..5 is used in WPA, but is reserved in IEEE 802.11i/RSN */
+#define WPA_KEY_INFO_KEY_INDEX_MASK (BIT(4) | BIT(5))
+#define WPA_KEY_INFO_KEY_INDEX_SHIFT 4
+#define WPA_KEY_INFO_INSTALL BIT(6) /* pairwise */
+#define WPA_KEY_INFO_TXRX BIT(6) /* group */
+#define WPA_KEY_INFO_ACK BIT(7)
+#define WPA_KEY_INFO_MIC BIT(8)
+#define WPA_KEY_INFO_SECURE BIT(9)
+#define WPA_KEY_INFO_ERROR BIT(10)
+#define WPA_KEY_INFO_REQUEST BIT(11)
+#define WPA_KEY_INFO_ENCR_KEY_DATA BIT(12) /* IEEE 802.11i/RSN only */
+#define WPA_KEY_INFO_SMK_MESSAGE BIT(13)
+
+struct ieee802_1x_hdr {
+	u8 version;
+	u8 type;
+	u16 length;
+	/* followed by length octets of data */
+};
+
+struct wpa_eapol_key {
+	u8 type;
+	/* Note: key_info, key_length, and key_data_length are unaligned */
+	u8 key_info[2]; /* big endian */
+	u8 key_length[2]; /* big endian */
+	u8 replay_counter[WPA_REPLAY_COUNTER_LEN];
+	u8 key_nonce[WPA_NONCE_LEN];
+	u8 key_iv[16];
+	u8 key_rsc[WPA_KEY_RSC_LEN];
+	u8 key_id[8]; /* Reserved in IEEE 802.11i/RSN */
+	/* variable length Key MIC field */
+	/* big endian 2-octet Key Data Length field */
+	/* followed by Key Data Length bytes of Key Data */
+};
+
+#define is_legacy_only(net_type)  ((net_type) == ((net_type) & (WLAN_MD_11BG | WLAN_MD_11A)))
+#define is_supported_24g(band_type) ((band_type) & BAND_CAP_2G ? _TRUE : _FALSE)
+#define is_supported_5g(band_type) ((band_type) & BAND_CAP_5G ? _TRUE : _FALSE)
+#define is_supported_6g(band_type) ((band_type) & BAND_CAP_6G ? _TRUE : _FALSE)
+#define is_supported_tx_cck(net_type) ((net_type) & (WLAN_MD_11B) ? _TRUE : _FALSE)
+#define is_suuported_tx_ofdm(net_type) ((net_type) & (WLAN_MD_11G | WLAN_MD_11A) ? _TRUE : _FALSE)
+#define is_supported_ht(net_type) ((net_type) & (WLAN_MD_11N) ? _TRUE : _FALSE)
+#define is_supported_vht(net_type) ((net_type) & (WLAN_MD_11AC) ? _TRUE : _FALSE)
+#define is_supported_he(net_type) ((net_type) & (WLAN_MD_11AX) ? _TRUE : _FALSE)
+
+
+
+typedef struct ieee_param {
+	u32 cmd;
+	u8 sta_addr[ETH_ALEN];
+	union {
+		struct {
+			u8 name;
+			u32 value;
+		} wpa_param;
+		struct {
+			u32 len;
+			u8 reserved[32];
+			DECLARE_FLEX_ARRAY(u8, data);
+		} wpa_ie;
+		struct {
+			int command;
+			int reason_code;
+		} mlme;
+		struct {
+			u8 alg[IEEE_CRYPT_ALG_NAME_LEN];
+			u8 set_tx;
+			u32 err;
+			u8 idx;
+			u8 seq[8]; /* sequence counter (set: RX, get: TX) */
+			u16 key_len;
+			DECLARE_FLEX_ARRAY(u8, key);
+		} crypt;
+#ifdef CONFIG_AP_MODE
+		struct {
+			u16 aid;
+			u16 capability;
+			int flags;
+			u8 tx_supp_rates[16];
+			struct rtw_ieee80211_ht_cap ht_cap;
+		} add_sta;
+		struct {
+			u8	reserved[2];/* for set max_num_sta */
+			DECLARE_FLEX_ARRAY(u8, buf);
+		} bcn_ie;
+#endif
+
+	} u;
+} ieee_param;
+
+#ifdef CONFIG_AP_MODE
+typedef struct ieee_param_ex {
+	u32 cmd;
+	u8 sta_addr[ETH_ALEN];
+	DECLARE_FLEX_ARRAY(u8, data);
+} ieee_param_ex;
+
+struct sta_data {
+	u16 aid;
+	u16 capability;
+	int flags;
+	u32 sta_set;
+	u8 tx_supp_rates[16];
+	u32 tx_supp_rates_len;
+	struct rtw_ieee80211_ht_cap ht_cap;
+	u64	rx_pkts;
+	u64	rx_bytes;
+	u64	rx_drops;
+	u64	tx_pkts;
+	u64	tx_bytes;
+	u64	tx_drops;
+};
+#endif
+
+
+#if WIRELESS_EXT < 17
+	#define IW_QUAL_QUAL_INVALID   0x10
+	#define IW_QUAL_LEVEL_INVALID  0x20
+	#define IW_QUAL_NOISE_INVALID  0x40
+	#define IW_QUAL_QUAL_UPDATED   0x1
+	#define IW_QUAL_LEVEL_UPDATED  0x2
+	#define IW_QUAL_NOISE_UPDATED  0x4
+#endif
+
+#define IEEE80211_DATA_LEN		2304
+/* Maximum size for the MA-UNITDATA primitive, 802.11 standard section
+   6.2.1.1.2.
+
+   The figure in section 7.1.2 suggests a body size of up to 2312
+   bytes is allowed, which is a bit confusing, I suspect this
+   represents the 2304 bytes of real data, plus a possible 8 bytes of
+   WEP IV and ICV. (this interpretation suggested by Ramiro Barreiro) */
+
+
+#define IEEE80211_HLEN			30
+#define IEEE80211_FRAME_LEN		(IEEE80211_DATA_LEN + IEEE80211_HLEN)
+
+
+/* this is stolen from ipw2200 driver */
+#define IEEE_IBSS_MAC_HASH_SIZE 31
+
+struct ieee_ibss_seq {
+	u8 mac[ETH_ALEN];
+	u16 seq_num;
+	u16 frag_num;
+	unsigned long packet_time;
+	_list	list;
+};
+
+#if defined(PLATFORM_LINUX) || defined(PLATFORM_FREEBSD)
+
+struct rtw_ieee80211_hdr {
+	u16 frame_ctl;
+	u16 duration_id;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	u16 seq_ctl;
+	u8 addr4[ETH_ALEN];
+} __attribute__((packed));
+
+struct rtw_ieee80211_hdr_3addr {
+	u16 frame_ctl;
+	u16 duration_id;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	u16 seq_ctl;
+} __attribute__((packed));
+
+
+struct rtw_ieee80211_hdr_qos {
+	u16 frame_ctl;
+	u16 duration_id;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	u16 seq_ctl;
+	u8 addr4[ETH_ALEN];
+	u16	qc;
+}  __attribute__((packed));
+
+struct rtw_ieee80211_hdr_3addr_qos {
+	u16 frame_ctl;
+	u16 duration_id;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	u16 seq_ctl;
+	u16     qc;
+}  __attribute__((packed));
+
+struct eapol {
+	u8 snap[6];
+	u16 ethertype;
+	u8 version;
+	u8 type;
+	u16 length;
+} __attribute__((packed));
+
+struct rtw_ieee80211s_hdr {
+	u8 flags;
+	u8 ttl;
+	u32 seqnum;
+	u8 eaddr1[ETH_ALEN];
+	u8 eaddr2[ETH_ALEN];
+} __attribute__((packed));
+
+/**
+ * struct rtw_ieee80211_rann_ie
+ *
+ * This structure refers to "Root Announcement information element"
+ */
+ struct rtw_ieee80211_rann_ie {
+	u8 rann_flags;
+	u8 rann_hopcount;
+	u8 rann_ttl;
+	u8 rann_addr[ETH_ALEN];
+	u32 rann_seq;
+	u32 rann_interval;
+	u32 rann_metric;
+} __attribute__((packed));
+#endif
+
+/* Some IEEE 802.11x packet types are corresponding to parsing_eapol_packet() */
+enum eap_type {
+	EAP_PACKET = 0,
+	NON_EAPOL,
+	EAPOL_START,
+	EAPOL_LOGOFF,
+	EAPOL_KEY,
+	EAPOL_ENCAP_ASF_ALERT,
+	EAPOL_PACKET,
+	EAPOL_WPA_GROUP_KEY_1_2,
+	EAPOL_WPA_GROUP_KEY_2_2,
+	EAPOL_1_4,
+	EAPOL_2_4,
+	EAPOL_3_4,
+	EAPOL_4_4,
+};
+
+#define IEEE80211_3ADDR_LEN 24
+#define IEEE80211_4ADDR_LEN 30
+#define IEEE80211_FCS_LEN    4
+
+#define MIN_FRAG_THRESHOLD     256U
+#define	MAX_FRAG_THRESHOLD     2346U
+
+/* Frame control field constants */
+#define RTW_IEEE80211_FCTL_VERS		0x0003
+#define RTW_IEEE80211_FCTL_FTYPE		0x000c
+#define RTW_IEEE80211_FCTL_STYPE		0x00f0
+#define RTW_IEEE80211_FCTL_TODS		0x0100
+#define RTW_IEEE80211_FCTL_FROMDS	0x0200
+#define RTW_IEEE80211_FCTL_MOREFRAGS	0x0400
+#define RTW_IEEE80211_FCTL_RETRY		0x0800
+#define RTW_IEEE80211_FCTL_PM		0x1000
+#define RTW_IEEE80211_FCTL_MOREDATA	0x2000
+#define RTW_IEEE80211_FCTL_PROTECTED	0x4000
+#define RTW_IEEE80211_FCTL_ORDER		0x8000
+#define RTW_IEEE80211_FCTL_CTL_EXT	0x0f00
+
+#define RTW_IEEE80211_FTYPE_MGMT		0x0000
+#define RTW_IEEE80211_FTYPE_CTL		0x0004
+#define RTW_IEEE80211_FTYPE_DATA		0x0008
+#define RTW_IEEE80211_FTYPE_EXT		0x000c
+
+/* management */
+#define RTW_IEEE80211_STYPE_ASSOC_REQ	0x0000
+#define RTW_IEEE80211_STYPE_ASSOC_RESP	0x0010
+#define RTW_IEEE80211_STYPE_REASSOC_REQ	0x0020
+#define RTW_IEEE80211_STYPE_REASSOC_RESP	0x0030
+#define RTW_IEEE80211_STYPE_PROBE_REQ	0x0040
+#define RTW_IEEE80211_STYPE_PROBE_RESP	0x0050
+#define RTW_IEEE80211_STYPE_BEACON		0x0080
+#define RTW_IEEE80211_STYPE_ATIM		0x0090
+#define RTW_IEEE80211_STYPE_DISASSOC	0x00A0
+#define RTW_IEEE80211_STYPE_AUTH		0x00B0
+#define RTW_IEEE80211_STYPE_DEAUTH		0x00C0
+#define RTW_IEEE80211_STYPE_ACTION		0x00D0
+
+/* control */
+#define RTW_IEEE80211_STYPE_CTL_EXT		0x0060
+#define RTW_IEEE80211_STYPE_BACK_REQ		0x0080
+#define RTW_IEEE80211_STYPE_BACK		0x0090
+#define RTW_IEEE80211_STYPE_PSPOLL		0x00A0
+#define RTW_IEEE80211_STYPE_RTS		0x00B0
+#define RTW_IEEE80211_STYPE_CTS		0x00C0
+#define RTW_IEEE80211_STYPE_ACK		0x00D0
+#define RTW_IEEE80211_STYPE_CFEND		0x00E0
+#define RTW_IEEE80211_STYPE_CFENDACK		0x00F0
+
+/* data */
+#define RTW_IEEE80211_STYPE_DATA		0x0000
+#define RTW_IEEE80211_STYPE_DATA_CFACK	0x0010
+#define RTW_IEEE80211_STYPE_DATA_CFPOLL	0x0020
+#define RTW_IEEE80211_STYPE_DATA_CFACKPOLL	0x0030
+#define RTW_IEEE80211_STYPE_NULLFUNC	0x0040
+#define RTW_IEEE80211_STYPE_CFACK		0x0050
+#define RTW_IEEE80211_STYPE_CFPOLL		0x0060
+#define RTW_IEEE80211_STYPE_CFACKPOLL	0x0070
+#define RTW_IEEE80211_STYPE_QOS_DATA		0x0080
+#define RTW_IEEE80211_STYPE_QOS_DATA_CFACK		0x0090
+#define RTW_IEEE80211_STYPE_QOS_DATA_CFPOLL		0x00A0
+#define RTW_IEEE80211_STYPE_QOS_DATA_CFACKPOLL	0x00B0
+#define RTW_IEEE80211_STYPE_QOS_NULLFUNC	0x00C0
+#define RTW_IEEE80211_STYPE_QOS_CFACK		0x00D0
+#define RTW_IEEE80211_STYPE_QOS_CFPOLL		0x00E0
+#define RTW_IEEE80211_STYPE_QOS_CFACKPOLL	0x00F0
+
+/* sequence control field */
+#define RTW_IEEE80211_SCTL_FRAG	0x000F
+#define RTW_IEEE80211_SCTL_SEQ	0xFFF0
+
+
+#define RTW_ERP_INFO_NON_ERP_PRESENT BIT(0)
+#define RTW_ERP_INFO_USE_PROTECTION BIT(1)
+#define RTW_ERP_INFO_BARKER_PREAMBLE_MODE BIT(2)
+
+/* QoS,QOS */
+#define NORMAL_ACK			0
+#define NO_ACK				1
+#define NON_EXPLICIT_ACK	2
+#define BLOCK_ACK			3
+
+#ifndef ETH_P_PAE
+	#define ETH_P_PAE 0x888E /* Port Access Entity (IEEE 802.1X) */
+#endif /* ETH_P_PAE */
+
+#define ETH_P_PREAUTH 0x88C7 /* IEEE 802.11i pre-authentication */
+
+#define ETH_P_ECONET	0x0018
+
+#ifndef ETH_P_80211_RAW
+	#define ETH_P_80211_RAW (ETH_P_ECONET + 1)
+#endif
+
+/* IEEE 802.11 defines */
+
+#define P80211_OUI_LEN 3
+
+#if defined(PLATFORM_LINUX) || defined(PLATFORM_FREEBSD)
+
+struct ieee80211_snap_hdr {
+
+	u8    dsap;   /* always 0xAA */
+	u8    ssap;   /* always 0xAA */
+	u8    ctrl;   /* always 0x03 */
+	u8    oui[P80211_OUI_LEN];    /* organizational universal id */
+
+} __attribute__((packed));
+
+#endif
+
+#define SNAP_SIZE sizeof(struct ieee80211_snap_hdr)
+#ifdef CONFIG_CORE_TXSC
+#define WLHDR_SIZE sizeof(struct rtw_ieee80211_hdr)
+#endif
+
+#define WLAN_FC_GET_TYPE(fc) ((fc) & RTW_IEEE80211_FCTL_FTYPE)
+#define WLAN_FC_GET_STYPE(fc) ((fc) & RTW_IEEE80211_FCTL_STYPE)
+
+#define WLAN_QC_GET_TID(qc) ((qc) & 0x0f)
+
+#define WLAN_GET_SEQ_FRAG(seq) ((seq) & RTW_IEEE80211_SCTL_FRAG)
+#define WLAN_GET_SEQ_SEQ(seq)  ((seq) & RTW_IEEE80211_SCTL_SEQ)
+
+/* Authentication algorithms */
+#define WLAN_AUTH_OPEN 0
+#define WLAN_AUTH_SHARED_KEY 1
+#define WLAN_AUTH_FT 2
+#define WLAN_AUTH_SAE 3
+
+#define WLAN_AUTH_CHALLENGE_LEN 128
+
+#define WLAN_CAPABILITY_BSS (1<<0)
+#define WLAN_CAPABILITY_IBSS (1<<1)
+#define WLAN_CAPABILITY_CF_POLLABLE (1<<2)
+#define WLAN_CAPABILITY_CF_POLL_REQUEST (1<<3)
+#define WLAN_CAPABILITY_PRIVACY (1<<4)
+#define WLAN_CAPABILITY_SHORT_PREAMBLE (1<<5)
+#define WLAN_CAPABILITY_PBCC (1<<6)
+#define WLAN_CAPABILITY_CHANNEL_AGILITY (1<<7)
+#define WLAN_CAPABILITY_SHORT_SLOT (1<<10)
+
+/* Status codes */
+#define WLAN_STATUS_SUCCESS 0
+#define WLAN_STATUS_UNSPECIFIED_FAILURE 1
+#define WLAN_STATUS_CAPS_UNSUPPORTED 10
+#define WLAN_STATUS_REASSOC_NO_ASSOC 11
+#define WLAN_STATUS_ASSOC_DENIED_UNSPEC 12
+#define WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG 13
+#define WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION 14
+#define WLAN_STATUS_CHALLENGE_FAIL 15
+#define WLAN_STATUS_AUTH_TIMEOUT 16
+#define WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA 17
+#define WLAN_STATUS_ASSOC_DENIED_RATES 18
+/* 802.11b */
+#define WLAN_STATUS_ASSOC_DENIED_NOSHORT 19
+#define WLAN_STATUS_ASSOC_DENIED_NOPBCC 20
+#define WLAN_STATUS_ASSOC_DENIED_NOAGILITY 21
+
+/* Reason codes */
+#define WLAN_REASON_UNSPECIFIED 1
+#define WLAN_REASON_PREV_AUTH_NOT_VALID 2
+#define WLAN_REASON_DEAUTH_LEAVING 3
+#define WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY 4
+#define WLAN_REASON_DISASSOC_AP_BUSY 5
+#define WLAN_REASON_CLASS2_FRAME_FROM_NONAUTH_STA 6
+#define WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA 7
+#define WLAN_REASON_DISASSOC_STA_HAS_LEFT 8
+#define WLAN_REASON_STA_REQ_ASSOC_WITHOUT_AUTH 9
+#define WLAN_REASON_IEEE_802_1X_AUTH_FAILED 23
+#define WLAN_REASON_MESH_PEER_CANCELED 52
+#define WLAN_REASON_MESH_MAX_PEERS 53
+#define WLAN_REASON_MESH_CONFIG 54
+#define WLAN_REASON_MESH_CLOSE 55
+#define WLAN_REASON_MESH_MAX_RETRIES 56
+#define WLAN_REASON_MESH_CONFIRM_TIMEOUT 57
+#define WLAN_REASON_MESH_INVALID_GTK 58
+#define WLAN_REASON_MESH_INCONSISTENT_PARAM 59
+#define WLAN_REASON_MESH_INVALID_SECURITY 60
+#define WLAN_REASON_MESH_PATH_NOPROXY 61
+#define WLAN_REASON_MESH_PATH_NOFORWARD 62
+#define WLAN_REASON_MESH_PATH_DEST_UNREACHABLE 63
+#define WLAN_REASON_MAC_EXISTS_IN_MBSS 64
+#define WLAN_REASON_MESH_CHAN_REGULATORY 65
+#define WLAN_REASON_MESH_CHAN 66
+#define WLAN_REASON_SA_QUERY_TIMEOUT 65532
+#define WLAN_REASON_ACTIVE_ROAM 65533
+#define WLAN_REASON_JOIN_WRONG_CHANNEL       65534
+#define WLAN_REASON_EXPIRATION_CHK 65535
+
+#define WLAN_REASON_IS_PRIVATE(reason) ( \
+	reason == WLAN_REASON_EXPIRATION_CHK \
+	|| reason == WLAN_REASON_JOIN_WRONG_CHANNEL \
+	|| reason == WLAN_REASON_ACTIVE_ROAM \
+	|| reason == WLAN_REASON_SA_QUERY_TIMEOUT \
+	)
+
+/* Information Element IDs */
+#define WLAN_EID_SSID 0
+#define WLAN_EID_SUPP_RATES 1
+#define WLAN_EID_FH_PARAMS 2
+#define WLAN_EID_DS_PARAMS 3
+#define WLAN_EID_CF_PARAMS 4
+#define WLAN_EID_TIM 5
+#define WLAN_EID_IBSS_PARAMS 6
+#define WLAN_EID_COUNTRY 7
+#define WLAN_EID_CHALLENGE 16
+/* EIDs defined by IEEE 802.11h - START */
+#define WLAN_EID_PWR_CONSTRAINT 32
+#define WLAN_EID_PWR_CAPABILITY 33
+#define WLAN_EID_TPC_REQUEST 34
+#define WLAN_EID_TPC_REPORT 35
+#define WLAN_EID_SUPPORTED_CHANNELS 36
+#define WLAN_EID_CHANNEL_SWITCH 37
+#define WLAN_EID_MEASURE_REQUEST 38
+#define WLAN_EID_MEASURE_REPORT 39
+#define WLAN_EID_QUITE 40
+#define WLAN_EID_IBSS_DFS 41
+/* EIDs defined by IEEE 802.11h - END */
+#define WLAN_EID_ERP_INFO 42
+#define WLAN_EID_HT_CAP 45
+#define WLAN_EID_RSN 48
+#define WLAN_EID_EXT_SUPP_RATES 50
+#define WLAN_EID_AP_CHANNEL_RPT 51
+#define WLAN_EID_MOBILITY_DOMAIN 54
+#define WLAN_EID_FAST_BSS_TRANSITION 55
+#define WLAN_EID_TIMEOUT_INTERVAL 56
+#define WLAN_EID_RIC_DATA 57
+#define WLAN_EID_SUPPORT_OP_CLASS 59
+#define WLAN_EID_ECSA 60
+#define WLAN_EID_HT_OPERATION 61
+#define WLAN_EID_SECONDARY_CHANNEL_OFFSET 62
+#define WLAN_EID_MULTIPLE_BSSID 71
+#define WLAN_EID_20_40_BSS_COEXISTENCE 72
+#define WLAN_EID_20_40_BSS_INTOLERANT 73
+#define WLAN_EID_OVERLAPPING_BSS_SCAN_PARAMS 74
+#define WLAN_EID_MMIE 76
+#define WLAN_EID_NON_TX_BSSID_CAP 83
+#define WLAN_EID_MULTI_BSSID_IDX 85
+#define WLAN_EID_MESH_CONFIG 113
+#define WLAN_EID_MESH_ID 114
+#define WLAN_EID_MPM 117
+#define WLAN_EID_RANN 126
+#define WLAN_EID_EXT_CAP 127
+#define WLAN_EID_PREQ 130
+#define WLAN_EID_PREP 131
+#define WLAN_EID_PERR 132
+#define WLAN_EID_AMPE 139
+#define WLAN_EID_MIC 140
+#define WLAN_EID_TWT 216
+#define WLAN_EID_VENDOR_SPECIFIC 221
+#define WLAN_EID_GENERIC (WLAN_EID_VENDOR_SPECIFIC)
+#define WLAN_EID_VHT_CAPABILITY 191
+#define WLAN_EID_VHT_OPERATION 192
+#define WLAN_EID_VHT_WIDE_BW_CHSWITCH 194
+#define WLAN_EID_VHT_TX_POWER_ENVELOPE 195
+#define WLAN_EID_CHANNEL_SWITCH_WRAPPER 196
+#define WLAN_EID_VHT_OP_MODE_NOTIFY 199
+#define WLAN_EID_RSNX 244
+#define WLAN_EID_EXTENSION 255
+#define WLAN_EID_EXT_OWE_DH_PARAM 32
+
+/* EID Extension */
+#define WLAN_EID_EXTENSION_HE_CAPABILITY	35
+#define WLAN_EID_EXTENSION_HE_OPERATION	36
+#define WLAN_EID_EXTENSION_HE_MU_EDCA	38
+#define WLAN_EID_EXT_HE_6G_CAP 59
+
+#define WLAN_EID_EXT_CAP_MAX_LEN 10
+
+#define IEEE80211_MGMT_HDR_LEN 24
+#define IEEE80211_DATA_HDR3_LEN 24
+#define IEEE80211_DATA_HDR4_LEN 30
+
+
+#define IEEE80211_STATMASK_SIGNAL (1<<0)
+#define IEEE80211_STATMASK_RSSI (1<<1)
+#define IEEE80211_STATMASK_NOISE (1<<2)
+#define IEEE80211_STATMASK_RATE (1<<3)
+#define IEEE80211_STATMASK_WEMASK 0x7
+
+
+#define IEEE80211_CCK_MODULATION    (1<<0)
+#define IEEE80211_OFDM_MODULATION   (1<<1)
+
+#define IEEE80211_24GHZ_BAND     (1<<0)
+#define IEEE80211_52GHZ_BAND     (1<<1)
+
+#define IEEE80211_CCK_RATE_LEN		4
+#define IEEE80211_NUM_OFDM_RATESLEN	8
+
+
+
+#define IEEE80211_CCK_RATE_1MB		        0x02
+#define IEEE80211_CCK_RATE_2MB		        0x04
+#define IEEE80211_CCK_RATE_5MB		        0x0B
+#define IEEE80211_CCK_RATE_11MB		        0x16
+#define IEEE80211_OFDM_RATE_LEN		8
+#define IEEE80211_OFDM_RATE_6MB		        0x0C
+#define IEEE80211_OFDM_RATE_9MB		        0x12
+#define IEEE80211_OFDM_RATE_12MB		0x18
+#define IEEE80211_OFDM_RATE_18MB		0x24
+#define IEEE80211_OFDM_RATE_24MB		0x30
+#define IEEE80211_PBCC_RATE_22MB		0x2C
+#define IEEE80211_FREAK_RATE_22_5MB		0x2D
+#define IEEE80211_OFDM_RATE_36MB		0x48
+#define IEEE80211_OFDM_RATE_48MB		0x60
+#define IEEE80211_OFDM_RATE_54MB		0x6C
+/* Supported rates membership selectors */
+#ifndef BSS_MEMBERSHIP_SELECTOR_HT_PHY
+#define BSS_MEMBERSHIP_SELECTOR_HT_PHY		127
+#endif
+#ifndef BSS_MEMBERSHIP_SELECTOR_VHT_PHY
+#define BSS_MEMBERSHIP_SELECTOR_VHT_PHY		126
+#endif
+#ifndef BSS_MEMBERSHIP_SELECTOR_HE_PHY
+#define BSS_MEMBERSHIP_SELECTOR_HE_PHY		122
+#endif
+#ifndef BSS_MEMBERSHIP_SELECTOR_SAE_H2E
+#define BSS_MEMBERSHIP_SELECTOR_SAE_H2E		123
+#endif
+#define IEEE80211_BASIC_RATE_MASK		0x80
+
+#define IEEE80211_CCK_RATE_1MB_MASK		(1<<0)
+#define IEEE80211_CCK_RATE_2MB_MASK		(1<<1)
+#define IEEE80211_CCK_RATE_5MB_MASK		(1<<2)
+#define IEEE80211_CCK_RATE_11MB_MASK		(1<<3)
+#define IEEE80211_OFDM_RATE_6MB_MASK		(1<<4)
+#define IEEE80211_OFDM_RATE_9MB_MASK		(1<<5)
+#define IEEE80211_OFDM_RATE_12MB_MASK		(1<<6)
+#define IEEE80211_OFDM_RATE_18MB_MASK		(1<<7)
+#define IEEE80211_OFDM_RATE_24MB_MASK		(1<<8)
+#define IEEE80211_OFDM_RATE_36MB_MASK		(1<<9)
+#define IEEE80211_OFDM_RATE_48MB_MASK		(1<<10)
+#define IEEE80211_OFDM_RATE_54MB_MASK		(1<<11)
+
+#define IEEE80211_CCK_RATES_MASK	        0x0000000F
+#define IEEE80211_CCK_BASIC_RATES_MASK	(IEEE80211_CCK_RATE_1MB_MASK | \
+		IEEE80211_CCK_RATE_2MB_MASK)
+#define IEEE80211_CCK_DEFAULT_RATES_MASK	(IEEE80211_CCK_BASIC_RATES_MASK | \
+		IEEE80211_CCK_RATE_5MB_MASK | \
+		IEEE80211_CCK_RATE_11MB_MASK)
+
+#define IEEE80211_OFDM_RATES_MASK		0x00000FF0
+#define IEEE80211_OFDM_BASIC_RATES_MASK	(IEEE80211_OFDM_RATE_6MB_MASK | \
+		IEEE80211_OFDM_RATE_12MB_MASK | \
+		IEEE80211_OFDM_RATE_24MB_MASK)
+#define IEEE80211_OFDM_DEFAULT_RATES_MASK	(IEEE80211_OFDM_BASIC_RATES_MASK | \
+		IEEE80211_OFDM_RATE_9MB_MASK  | \
+		IEEE80211_OFDM_RATE_18MB_MASK | \
+		IEEE80211_OFDM_RATE_36MB_MASK | \
+		IEEE80211_OFDM_RATE_48MB_MASK | \
+		IEEE80211_OFDM_RATE_54MB_MASK)
+#define IEEE80211_DEFAULT_RATES_MASK (IEEE80211_OFDM_DEFAULT_RATES_MASK | \
+				      IEEE80211_CCK_DEFAULT_RATES_MASK)
+
+#define IEEE80211_NUM_OFDM_RATES	    8
+#define IEEE80211_NUM_CCK_RATES	            4
+#define IEEE80211_OFDM_SHIFT_MASK_A         4
+
+enum subelement_of_multiple_bssid_id {
+	MBSSID_NONTRANSMITTED_BSSID_PROFILE_ID = 0,
+	MBSSID_VENDOR_SPECIFIC_ID = 221
+};
+
+enum MGN_RATE {
+	MGN_1M		= 0x02,
+	MGN_2M		= 0x04,
+	MGN_5_5M	= 0x0B,
+	MGN_6M		= 0x0C,
+	MGN_9M		= 0x12,
+	MGN_11M	= 0x16,
+	MGN_12M	= 0x18,
+	MGN_18M	= 0x24,
+	MGN_24M	= 0x30,
+	MGN_36M	= 0x48,
+	MGN_48M	= 0x60,
+	MGN_54M	= 0x6C,
+	MGN_MCS32	= 0x7F,
+	MGN_MCS0,
+	MGN_MCS1,
+	MGN_MCS2,
+	MGN_MCS3,
+	MGN_MCS4,
+	MGN_MCS5,
+	MGN_MCS6,
+	MGN_MCS7,
+	MGN_MCS8,
+	MGN_MCS9,
+	MGN_MCS10,
+	MGN_MCS11,
+	MGN_MCS12,
+	MGN_MCS13,
+	MGN_MCS14,
+	MGN_MCS15,
+	MGN_MCS16,
+	MGN_MCS17,
+	MGN_MCS18,
+	MGN_MCS19,
+	MGN_MCS20,
+	MGN_MCS21,
+	MGN_MCS22,
+	MGN_MCS23,
+	MGN_MCS24,
+	MGN_MCS25,
+	MGN_MCS26,
+	MGN_MCS27,
+	MGN_MCS28,
+	MGN_MCS29,
+	MGN_MCS30,
+	MGN_MCS31,
+	MGN_VHT1SS_MCS0,
+	MGN_VHT1SS_MCS1,
+	MGN_VHT1SS_MCS2,
+	MGN_VHT1SS_MCS3,
+	MGN_VHT1SS_MCS4,
+	MGN_VHT1SS_MCS5,
+	MGN_VHT1SS_MCS6,
+	MGN_VHT1SS_MCS7,
+	MGN_VHT1SS_MCS8,
+	MGN_VHT1SS_MCS9,
+	MGN_VHT2SS_MCS0,
+	MGN_VHT2SS_MCS1,
+	MGN_VHT2SS_MCS2,
+	MGN_VHT2SS_MCS3,
+	MGN_VHT2SS_MCS4,
+	MGN_VHT2SS_MCS5,
+	MGN_VHT2SS_MCS6,
+	MGN_VHT2SS_MCS7,
+	MGN_VHT2SS_MCS8,
+	MGN_VHT2SS_MCS9,
+	MGN_VHT3SS_MCS0,
+	MGN_VHT3SS_MCS1,
+	MGN_VHT3SS_MCS2,
+	MGN_VHT3SS_MCS3,
+	MGN_VHT3SS_MCS4,
+	MGN_VHT3SS_MCS5,
+	MGN_VHT3SS_MCS6,
+	MGN_VHT3SS_MCS7,
+	MGN_VHT3SS_MCS8,
+	MGN_VHT3SS_MCS9,
+	MGN_VHT4SS_MCS0,
+	MGN_VHT4SS_MCS1,
+	MGN_VHT4SS_MCS2,
+	MGN_VHT4SS_MCS3,
+	MGN_VHT4SS_MCS4,
+	MGN_VHT4SS_MCS5,
+	MGN_VHT4SS_MCS6,
+	MGN_VHT4SS_MCS7,
+	MGN_VHT4SS_MCS8,
+	MGN_VHT4SS_MCS9,
+	MGN_HE1SS_MCS0 = 0xd0,
+	MGN_HE1SS_MCS1,
+	MGN_HE1SS_MCS2,
+	MGN_HE1SS_MCS3,
+	MGN_HE1SS_MCS4,
+	MGN_HE1SS_MCS5,
+	MGN_HE1SS_MCS6,
+	MGN_HE1SS_MCS7,
+	MGN_HE1SS_MCS8,
+	MGN_HE1SS_MCS9,
+	MGN_HE1SS_MCS10,
+	MGN_HE1SS_MCS11,
+	MGN_HE2SS_MCS0,
+	MGN_HE2SS_MCS1,
+	MGN_HE2SS_MCS2,
+	MGN_HE2SS_MCS3,
+	MGN_HE2SS_MCS4,
+	MGN_HE2SS_MCS5,
+	MGN_HE2SS_MCS6,
+	MGN_HE2SS_MCS7,
+	MGN_HE2SS_MCS8,
+	MGN_HE2SS_MCS9,
+	MGN_HE2SS_MCS10,
+	MGN_HE2SS_MCS11,
+	MGN_HE3SS_MCS0,
+	MGN_HE3SS_MCS1,
+	MGN_HE3SS_MCS2,
+	MGN_HE3SS_MCS3,
+	MGN_HE3SS_MCS4,
+	MGN_HE3SS_MCS5,
+	MGN_HE3SS_MCS6,
+	MGN_HE3SS_MCS7,
+	MGN_HE3SS_MCS8,
+	MGN_HE3SS_MCS9,
+	MGN_HE3SS_MCS10,
+	MGN_HE3SS_MCS11,
+	MGN_HE4SS_MCS0,
+	MGN_HE4SS_MCS1,
+	MGN_HE4SS_MCS2,
+	MGN_HE4SS_MCS3,
+	MGN_HE4SS_MCS4,
+	MGN_HE4SS_MCS5,
+	MGN_HE4SS_MCS6,
+	MGN_HE4SS_MCS7,
+	MGN_HE4SS_MCS8,
+	MGN_HE4SS_MCS9,
+	MGN_HE4SS_MCS10,
+	MGN_HE4SS_MCS11 = 0xff,
+	MGN_UNKNOWN = 0xcc
+};
+
+#define IS_HT_RATE(_rate)	((_rate) >= MGN_MCS0 && (_rate) <= MGN_MCS31)
+#define IS_VHT_RATE(_rate)	((_rate) >= MGN_VHT1SS_MCS0 && (_rate) <= MGN_VHT4SS_MCS9)
+#define IS_CCK_RATE(_rate)	((_rate) == MGN_1M || (_rate) == MGN_2M || (_rate) == MGN_5_5M || (_rate) == MGN_11M)
+#define IS_OFDM_RATE(_rate)	((_rate) >= MGN_6M && (_rate) <= MGN_54M  && (_rate) != MGN_11M)
+
+#define IS_HT1SS_RATE(_rate) ((_rate) >= MGN_MCS0 && (_rate) <= MGN_MCS7)
+#define IS_HT2SS_RATE(_rate) ((_rate) >= MGN_MCS8 && (_rate) <= MGN_MCS15)
+#define IS_HT3SS_RATE(_rate) ((_rate) >= MGN_MCS16 && (_rate) <= MGN_MCS23)
+#define IS_HT4SS_RATE(_rate) ((_rate) >= MGN_MCS24 && (_rate) <= MGN_MCS31)
+
+#define IS_VHT1SS_RATE(_rate) ((_rate) >= MGN_VHT1SS_MCS0 && (_rate) <= MGN_VHT1SS_MCS9)
+#define IS_VHT2SS_RATE(_rate) ((_rate) >= MGN_VHT2SS_MCS0 && (_rate) <= MGN_VHT2SS_MCS9)
+#define IS_VHT3SS_RATE(_rate) ((_rate) >= MGN_VHT3SS_MCS0 && (_rate) <= MGN_VHT3SS_MCS9)
+#define IS_VHT4SS_RATE(_rate) ((_rate) >= MGN_VHT4SS_MCS0 && (_rate) <= MGN_VHT4SS_MCS9)
+
+#define IS_1T_RATE(_rate)	(IS_CCK_RATE((_rate)) || IS_OFDM_RATE((_rate)) || IS_HT1SS_RATE((_rate)) || IS_VHT1SS_RATE((_rate)))
+#define IS_2T_RATE(_rate)	(IS_HT2SS_RATE((_rate)) || IS_VHT2SS_RATE((_rate)))
+#define IS_3T_RATE(_rate)	(IS_HT3SS_RATE((_rate)) || IS_VHT3SS_RATE((_rate)))
+#define IS_4T_RATE(_rate)	(IS_HT4SS_RATE((_rate)) || IS_VHT4SS_RATE((_rate)))
+
+#define MGN_RATE_STR(_rate) \
+	(_rate == MGN_1M) ? "CCK_1M" : \
+	(_rate == MGN_2M) ? "CCK_2M" : \
+	(_rate == MGN_5_5M) ? "CCK_5.5M" : \
+	(_rate == MGN_11M) ? "CCK_11M" : \
+	(_rate == MGN_6M) ? "OFDM_6M" : \
+	(_rate == MGN_9M) ? "OFDM_9M" : \
+	(_rate == MGN_12M) ? "OFDM_12M" : \
+	(_rate == MGN_18M) ? "OFDM_18M" : \
+	(_rate == MGN_24M) ? "OFDM_24M" : \
+	(_rate == MGN_36M) ? "OFDM_36M" : \
+	(_rate == MGN_48M) ? "OFDM_48M" : \
+	(_rate == MGN_54M) ? "OFDM_54M" : \
+	(_rate == MGN_MCS32) ? "MCS32" : \
+	(_rate == MGN_MCS0) ? "MCS0" : \
+	(_rate == MGN_MCS1) ? "MCS1" : \
+	(_rate == MGN_MCS2) ? "MCS2" : \
+	(_rate == MGN_MCS3) ? "MCS3" : \
+	(_rate == MGN_MCS4) ? "MCS4" : \
+	(_rate == MGN_MCS5) ? "MCS5" : \
+	(_rate == MGN_MCS6) ? "MCS6" : \
+	(_rate == MGN_MCS7) ? "MCS7" : \
+	(_rate == MGN_MCS8) ? "MCS8" : \
+	(_rate == MGN_MCS9) ? "MCS9" : \
+	(_rate == MGN_MCS10) ? "MCS10" : \
+	(_rate == MGN_MCS11) ? "MCS11" : \
+	(_rate == MGN_MCS12) ? "MCS12" : \
+	(_rate == MGN_MCS13) ? "MCS13" : \
+	(_rate == MGN_MCS14) ? "MCS14" : \
+	(_rate == MGN_MCS15) ? "MCS15" : \
+	(_rate == MGN_MCS16) ? "MCS16" : \
+	(_rate == MGN_MCS17) ? "MCS17" : \
+	(_rate == MGN_MCS18) ? "MCS18" : \
+	(_rate == MGN_MCS19) ? "MCS19" : \
+	(_rate == MGN_MCS20) ? "MCS20" : \
+	(_rate == MGN_MCS21) ? "MCS21" : \
+	(_rate == MGN_MCS22) ? "MCS22" : \
+	(_rate == MGN_MCS23) ? "MCS23" : \
+	(_rate == MGN_MCS24) ? "MCS24" : \
+	(_rate == MGN_MCS25) ? "MCS25" : \
+	(_rate == MGN_MCS26) ? "MCS26" : \
+	(_rate == MGN_MCS27) ? "MCS27" : \
+	(_rate == MGN_MCS28) ? "MCS28" : \
+	(_rate == MGN_MCS29) ? "MCS29" : \
+	(_rate == MGN_MCS30) ? "MCS30" : \
+	(_rate == MGN_MCS31) ? "MCS31" : \
+	(_rate == MGN_VHT1SS_MCS0) ? "VHT1SMCS0" : \
+	(_rate == MGN_VHT1SS_MCS1) ? "VHT1SMCS1" : \
+	(_rate == MGN_VHT1SS_MCS2) ? "VHT1SMCS2" : \
+	(_rate == MGN_VHT1SS_MCS3) ? "VHT1SMCS3" : \
+	(_rate == MGN_VHT1SS_MCS4) ? "VHT1SMCS4" : \
+	(_rate == MGN_VHT1SS_MCS5) ? "VHT1SMCS5" : \
+	(_rate == MGN_VHT1SS_MCS6) ? "VHT1SMCS6" : \
+	(_rate == MGN_VHT1SS_MCS7) ? "VHT1SMCS7" : \
+	(_rate == MGN_VHT1SS_MCS8) ? "VHT1SMCS8" : \
+	(_rate == MGN_VHT1SS_MCS9) ? "VHT1SMCS9" : \
+	(_rate == MGN_VHT2SS_MCS0) ? "VHT2SMCS0" : \
+	(_rate == MGN_VHT2SS_MCS1) ? "VHT2SMCS1" : \
+	(_rate == MGN_VHT2SS_MCS2) ? "VHT2SMCS2" : \
+	(_rate == MGN_VHT2SS_MCS3) ? "VHT2SMCS3" : \
+	(_rate == MGN_VHT2SS_MCS4) ? "VHT2SMCS4" : \
+	(_rate == MGN_VHT2SS_MCS5) ? "VHT2SMCS5" : \
+	(_rate == MGN_VHT2SS_MCS6) ? "VHT2SMCS6" : \
+	(_rate == MGN_VHT2SS_MCS7) ? "VHT2SMCS7" : \
+	(_rate == MGN_VHT2SS_MCS8) ? "VHT2SMCS8" : \
+	(_rate == MGN_VHT2SS_MCS9) ? "VHT2SMCS9" : \
+	(_rate == MGN_VHT3SS_MCS0) ? "VHT3SMCS0" : \
+	(_rate == MGN_VHT3SS_MCS1) ? "VHT3SMCS1" : \
+	(_rate == MGN_VHT3SS_MCS2) ? "VHT3SMCS2" : \
+	(_rate == MGN_VHT3SS_MCS3) ? "VHT3SMCS3" : \
+	(_rate == MGN_VHT3SS_MCS4) ? "VHT3SMCS4" : \
+	(_rate == MGN_VHT3SS_MCS5) ? "VHT3SMCS5" : \
+	(_rate == MGN_VHT3SS_MCS6) ? "VHT3SMCS6" : \
+	(_rate == MGN_VHT3SS_MCS7) ? "VHT3SMCS7" : \
+	(_rate == MGN_VHT3SS_MCS8) ? "VHT3SMCS8" : \
+	(_rate == MGN_VHT3SS_MCS9) ? "VHT3SMCS9" : \
+	(_rate == MGN_VHT4SS_MCS0) ? "VHT4SMCS0" : \
+	(_rate == MGN_VHT4SS_MCS1) ? "VHT4SMCS1" : \
+	(_rate == MGN_VHT4SS_MCS2) ? "VHT4SMCS2" : \
+	(_rate == MGN_VHT4SS_MCS3) ? "VHT4SMCS3" : \
+	(_rate == MGN_VHT4SS_MCS4) ? "VHT4SMCS4" : \
+	(_rate == MGN_VHT4SS_MCS5) ? "VHT4SMCS5" : \
+	(_rate == MGN_VHT4SS_MCS6) ? "VHT4SMCS6" : \
+	(_rate == MGN_VHT4SS_MCS7) ? "VHT4SMCS7" : \
+	(_rate == MGN_VHT4SS_MCS8) ? "VHT4SMCS8" : \
+	(_rate == MGN_VHT4SS_MCS9) ? "VHT4SMCS9" : "UNKNOWN"
+
+typedef enum _RATE_SECTION {
+	CCK = 0,
+	OFDM = 1,
+	HT_MCS0_MCS7 = 2,
+	HT_MCS8_MCS15 = 3,
+	HT_MCS16_MCS23 = 4,
+	HT_MCS24_MCS31 = 5,
+	HT_1SS = HT_MCS0_MCS7,
+	HT_2SS = HT_MCS8_MCS15,
+	HT_3SS = HT_MCS16_MCS23,
+	HT_4SS = HT_MCS24_MCS31,
+	VHT_1SSMCS0_1SSMCS9 = 6,
+	VHT_2SSMCS0_2SSMCS9 = 7,
+	VHT_3SSMCS0_3SSMCS9 = 8,
+	VHT_4SSMCS0_4SSMCS9 = 9,
+	VHT_1SS = VHT_1SSMCS0_1SSMCS9,
+	VHT_2SS = VHT_2SSMCS0_2SSMCS9,
+	VHT_3SS = VHT_3SSMCS0_3SSMCS9,
+	VHT_4SS = VHT_4SSMCS0_4SSMCS9,
+	HE_1SSMCS0_1SSMCS11 = 10,
+	HE_2SSMCS0_2SSMCS11 = 11,
+	HE_3SSMCS0_3SSMCS11 = 12,
+	HE_4SSMCS0_4SSMCS11 = 13,
+	HE_1SS = HE_1SSMCS0_1SSMCS11,
+	HE_2SS = HE_2SSMCS0_2SSMCS11,
+	HE_3SS = HE_3SSMCS0_3SSMCS11,
+	HE_4SS = HE_4SSMCS0_4SSMCS11,
+	DCM_1SSMCS0_1SSMC4 = 14,
+	DCM_2SSMCS0_2SSMC4 = 15,
+	DCM_3SSMCS0_3SSMC4 = 16,
+	DCM_4SSMCS0_4SSMC4 = 17,
+	DCM_1SS = DCM_1SSMCS0_1SSMC4,
+	DCM_2SS = DCM_2SSMCS0_2SSMC4,
+	DCM_3SS = DCM_3SSMCS0_3SSMC4,
+	DCM_4SS = DCM_4SSMCS0_4SSMC4,
+	RATE_SECTION_NUM,
+} RATE_SECTION;
+
+RATE_SECTION mgn_rate_to_rs(enum MGN_RATE rate);
+
+const char *rate_section_str(u8 section);
+
+#define IS_CCK_RATE_SECTION(section) ((section) == CCK)
+#define IS_OFDM_RATE_SECTION(section) ((section) == OFDM)
+#define IS_HT_RATE_SECTION(section) ((section) >= HT_1SS && (section) <= HT_4SS)
+#define IS_VHT_RATE_SECTION(section) ((section) >= VHT_1SS && (section) <= VHT_4SS)
+#define IS_HE_RATE_SECTION(section) ((section) >= HE_1SS && (section) <= HE_4SS)
+#define IS_DCM_RATE_SECTION(section) ((section) >= DCM_1SS && (section) <= DCM_4SS)
+
+#define IS_1T_RATE_SECTION(section) ((section) == CCK || (section) == OFDM || (section) == HT_1SS || (section) == VHT_1SS)
+#define IS_2T_RATE_SECTION(section) ((section) == HT_2SS || (section) == VHT_2SS)
+#define IS_3T_RATE_SECTION(section) ((section) == HT_3SS || (section) == VHT_3SS)
+#define IS_4T_RATE_SECTION(section) ((section) == HT_4SS || (section) == VHT_4SS)
+
+extern u8 mgn_rates_cck[];
+extern u8 mgn_rates_ofdm[];
+extern u8 mgn_rates_mcs0_7[];
+extern u8 mgn_rates_mcs8_15[];
+extern u8 mgn_rates_mcs16_23[];
+extern u8 mgn_rates_mcs24_31[];
+extern u8 mgn_rates_vht1ss[];
+extern u8 mgn_rates_vht2ss[];
+extern u8 mgn_rates_vht3ss[];
+extern u8 mgn_rates_vht4ss[];
+extern u8 mgn_rates_offset[];
+
+struct rate_section_ent {
+	u8 tx_num; /* value of RF_TX_NUM */
+	u8 rate_num;
+	u8 *rates;
+};
+
+extern struct rate_section_ent rates_by_sections[];
+
+#define rate_section_to_tx_num(section) (rates_by_sections[(section)].tx_num)
+#define rate_section_rate_num(section) (rates_by_sections[(section)].rate_num)
+
+/* NOTE: This data is for statistical purposes; not all hardware provides this
+ *       information for frames received.  Not setting these will not cause
+ *       any adverse affects. */
+struct ieee80211_rx_stats {
+	/* u32 mac_time[2]; */
+	s8 rssi;
+	u8 signal;
+	u8 noise;
+	u8 received_channel;
+	u16 rate; /* in 100 kbps */
+	/* u8 control; */
+	u8 mask;
+	u8 freq;
+	u16 len;
+};
+
+/* IEEE 802.11 requires that STA supports concurrent reception of at least
+ * three fragmented frames. This define can be increased to support more
+ * concurrent frames, but it should be noted that each entry can consume about
+ * 2 kB of RAM and increasing cache size will slow down frame reassembly. */
+#define IEEE80211_FRAG_CACHE_LEN 4
+
+struct ieee80211_frag_entry {
+	u32 first_frag_time;
+	uint seq;
+	uint last_frag;
+	uint qos;   /* jackson */
+	uint tid;	/* jackson */
+	struct sk_buff *skb;
+	u8 src_addr[ETH_ALEN];
+	u8 dst_addr[ETH_ALEN];
+};
+
+#define SEC_KEY_1         (1<<0)
+#define SEC_KEY_2         (1<<1)
+#define SEC_KEY_3         (1<<2)
+#define SEC_KEY_4         (1<<3)
+#define SEC_ACTIVE_KEY    (1<<4)
+#define SEC_AUTH_MODE     (1<<5)
+#define SEC_UNICAST_GROUP (1<<6)
+#define SEC_LEVEL         (1<<7)
+#define SEC_ENABLED       (1<<8)
+
+#define SEC_LEVEL_0      0 /* None */
+#define SEC_LEVEL_1      1 /* WEP 40 and 104 bit */
+#define SEC_LEVEL_2      2 /* Level 1 + TKIP */
+#define SEC_LEVEL_2_CKIP 3 /* Level 1 + CKIP */
+#define SEC_LEVEL_3      4 /* Level 2 + CCMP */
+
+#define WEP_KEYS 4
+#define WEP_KEY_LEN 13
+#define BIP_MAX_KEYID 5
+#define BIP_AAD_SIZE  20
+
+#if defined(PLATFORM_LINUX)
+struct ieee80211_security {
+	u16 active_key:2,
+	    enabled:1,
+	    auth_mode:2,
+	    auth_algo:4,
+	    unicast_uses_group:1;
+	u8 key_sizes[WEP_KEYS];
+	u8 keys[WEP_KEYS][WEP_KEY_LEN];
+	u8 level;
+	u16 flags;
+} __attribute__((packed));
+
+#endif
+
+/*
+
+ 802.11 data frame from AP
+
+      ,-------------------------------------------------------------------.
+Bytes |  2   |  2   |    6    |    6    |    6    |  2   | 0..2312 |   4  |
+      |------|------|---------|---------|---------|------|---------|------|
+Desc. | ctrl | dura |  DA/RA  |   TA    |    SA   | Sequ |  frame  |  fcs |
+      |      | tion | (BSSID) |         |         | ence |  data   |      |
+      `-------------------------------------------------------------------'
+
+Total: 28-2340 bytes
+
+*/
+
+struct ieee80211_header_data {
+	u16 frame_ctl;
+	u16 duration_id;
+	u8 addr1[6];
+	u8 addr2[6];
+	u8 addr3[6];
+	u16 seq_ctrl;
+};
+
+#define BEACON_PROBE_SSID_ID_POSITION 12
+
+/* Management Frame Information Element Types */
+#define MFIE_TYPE_SSID       0
+#define MFIE_TYPE_RATES      1
+#define MFIE_TYPE_FH_SET     2
+#define MFIE_TYPE_DS_SET     3
+#define MFIE_TYPE_CF_SET     4
+#define MFIE_TYPE_TIM        5
+#define MFIE_TYPE_IBSS_SET   6
+#define MFIE_TYPE_CHALLENGE  16
+#define MFIE_TYPE_ERP        42
+#define MFIE_TYPE_RSN	     48
+#define MFIE_TYPE_RATES_EX   50
+#define MFIE_TYPE_GENERIC    221
+
+#if defined(PLATFORM_LINUX)
+struct ieee80211_info_element_hdr {
+	u8 id;
+	u8 len;
+} __attribute__((packed));
+
+struct ieee80211_info_element {
+	u8 id;
+	u8 len;
+	DECLARE_FLEX_ARRAY(u8, data);
+} __attribute__((packed));
+#endif
+
+
+/*
+ * These are the data types that can make up management packets
+ *
+	u16 auth_algorithm;
+	u16 auth_sequence;
+	u16 beacon_interval;
+	u16 capability;
+	u8 current_ap[ETH_ALEN];
+	u16 listen_interval;
+	struct {
+		u16 association_id:14, reserved:2;
+	} __attribute__ ((packed));
+	u32 time_stamp[2];
+	u16 reason;
+	u16 status;
+*/
+
+#define IEEE80211_DEFAULT_TX_ESSID "Penguin"
+#define IEEE80211_DEFAULT_BASIC_RATE 10
+
+
+#if defined(PLATFORM_LINUX)
+struct ieee80211_authentication {
+	struct ieee80211_header_data header;
+	u16 algorithm;
+	u16 transaction;
+	u16 status;
+	/* struct ieee80211_info_element_hdr info_element; */
+} __attribute__((packed));
+
+
+struct ieee80211_probe_response {
+	struct ieee80211_header_data header;
+	u32 time_stamp[2];
+	u16 beacon_interval;
+	u16 capability;
+	struct ieee80211_info_element info_element;
+} __attribute__((packed));
+
+struct ieee80211_probe_request {
+	struct ieee80211_header_data header;
+	/*struct ieee80211_info_element info_element;*/
+} __attribute__((packed));
+
+struct ieee80211_assoc_request_frame {
+	struct rtw_ieee80211_hdr_3addr header;
+	u16 capability;
+	u16 listen_interval;
+	/* u8 current_ap[ETH_ALEN]; */
+	struct ieee80211_info_element_hdr info_element;
+} __attribute__((packed));
+
+struct ieee80211_assoc_response_frame {
+	struct rtw_ieee80211_hdr_3addr header;
+	u16 capability;
+	u16 status;
+	u16 aid;
+	/*	struct ieee80211_info_element info_element;  supported rates  */
+} __attribute__((packed));
+#endif
+
+struct ieee80211_txb {
+	u8 nr_frags;
+	u8 encrypted;
+	u16 reserved;
+	u16 frag_size;
+	u16 payload_size;
+	DECLARE_FLEX_ARRAY(struct sk_buff *, fragments);
+};
+
+
+/* SWEEP TABLE ENTRIES NUMBER*/
+#define MAX_SWEEP_TAB_ENTRIES		  42
+#define MAX_SWEEP_TAB_ENTRIES_PER_PACKET  7
+/* MAX_RATES_LENGTH needs to be 12.  The spec says 8, and many APs
+ * only use 8, and then use extended rates for the remaining supported
+ * rates.  Other APs, however, stick all of their supported rates on the
+ * main rates information element... */
+#define MAX_RATES_LENGTH                  ((u8)12)
+#define MAX_RATES_EX_LENGTH               ((u8)16)
+#define MAX_NETWORK_COUNT                  128
+#define IEEE80211_SOFTMAC_SCAN_TIME	  400
+/* (HZ / 2) */
+#define IEEE80211_SOFTMAC_ASSOC_RETRY_TIME (HZ * 2)
+
+#define CRC_LENGTH                 4U
+
+#define MAX_WPA_IE_LEN (256)
+#define MAX_WPS_IE_LEN (512)
+#define MAX_OWE_IE_LEN (128)
+#define MAX_P2P_IE_LEN (256)
+#define MAX_WFD_IE_LEN (128)
+#define MAX_RSNX_IE_LEN (16)
+
+#define NETWORK_EMPTY_ESSID (1<<0)
+#define NETWORK_HAS_OFDM    (1<<1)
+#define NETWORK_HAS_CCK     (1<<2)
+
+#define IEEE80211_DTIM_MBCAST 4
+#define IEEE80211_DTIM_UCAST 2
+#define IEEE80211_DTIM_VALID 1
+#define IEEE80211_DTIM_INVALID 0
+
+#define IEEE80211_PS_DISABLED 0
+#define IEEE80211_PS_UNICAST IEEE80211_DTIM_UCAST
+#define IEEE80211_PS_MBCAST IEEE80211_DTIM_MBCAST
+#define IW_ESSID_MAX_SIZE 32
+
+
+#define DEFAULT_MAX_SCAN_AGE (15 * HZ)
+#define DEFAULT_FTS 2346
+
+#ifdef CONFIG_RTW_HIDDEN_MAC_ADDR
+#define MAC_FMT "%02x:%02x:%02x:xx:xx:xx"
+#define MAC_ARG(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2]
+#else
+#define MAC_FMT "%02x:%02x:%02x:%02x:%02x:%02x"
+#define MAC_ARG(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2], ((u8 *)(x))[3], ((u8 *)(x))[4], ((u8 *)(x))[5]
+#endif /* CONFIG_RTW_HIDDEN_MAC_ADDR */
+
+#define MAC_SFMT "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx"
+#define MAC_SARG(x) ((u8*)(x)),((u8*)(x)) + 1,((u8*)(x)) + 2,((u8*)(x)) + 3,((u8*)(x)) + 4,((u8*)(x)) + 5
+#define IP_FMT "%d.%d.%d.%d"
+#define IP_ARG(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2], ((u8 *)(x))[3]
+#define PORT_FMT "%u"
+#define PORT_ARG(x) ntohs(*((u16 *)(x)))
+
+#define is_multicast_mac_addr(Addr) ((((Addr[0]) & 0x01) == 0x01) && ((Addr[0]) != 0xff))
+#define is_broadcast_mac_addr(Addr) ((((Addr[0]) & 0xff) == 0xff) && (((Addr[1]) & 0xff) == 0xff) && \
+	(((Addr[2]) & 0xff) == 0xff) && (((Addr[3]) & 0xff) == 0xff) && (((Addr[4]) & 0xff) == 0xff) && \
+				     (((Addr[5]) & 0xff) == 0xff))
+#define is_zero_mac_addr(Addr)	((Addr[0] == 0x00) && (Addr[1] == 0x00) && (Addr[2] == 0x00) &&   \
+                (Addr[3] == 0x00) && (Addr[4] == 0x00) && (Addr[5] == 0x00))
+
+
+#define CFG_IEEE80211_RESERVE_FCS (1<<0)
+#define CFG_IEEE80211_COMPUTE_FCS (1<<1)
+
+typedef struct tx_pending_t {
+	int frag;
+	struct ieee80211_txb *txb;
+} tx_pending_t;
+
+
+
+#define TID_NUM	16
+
+#define IEEE_A            (1<<0)
+#define IEEE_B            (1<<1)
+#define IEEE_G            (1<<2)
+#define IEEE_MODE_MASK    (IEEE_A | IEEE_B | IEEE_G)
+
+/* Baron move to ieee80211.c */
+int ieee80211_is_empty_essid(const char *essid, int essid_len);
+int ieee80211_get_hdrlen(u16 fc);
+
+#if 0
+	/* Action frame categories (IEEE 802.11-2007, 7.3.1.11, Table 7-24) */
+	#define WLAN_ACTION_SPECTRUM_MGMT 0
+	#define WLAN_ACTION_QOS 1
+	#define WLAN_ACTION_DLS 2
+	#define WLAN_ACTION_BLOCK_ACK 3
+	#define WLAN_ACTION_RADIO_MEASUREMENT 5
+	#define WLAN_ACTION_FT 6
+	#define WLAN_ACTION_SA_QUERY 8
+	#define WLAN_ACTION_WMM 17
+#endif
+
+
+/* Action category code */
+enum rtw_ieee80211_category {
+	RTW_WLAN_CATEGORY_SPECTRUM_MGMT = 0,
+	RTW_WLAN_CATEGORY_QOS = 1,
+	RTW_WLAN_CATEGORY_DLS = 2,
+	RTW_WLAN_CATEGORY_BACK = 3,
+	RTW_WLAN_CATEGORY_PUBLIC = 4, /* IEEE 802.11 public action frames */
+	RTW_WLAN_CATEGORY_RADIO_MEAS = 5,
+	RTW_WLAN_CATEGORY_FT = 6,
+	RTW_WLAN_CATEGORY_HT = 7,
+	RTW_WLAN_CATEGORY_SA_QUERY = 8,
+	RTW_WLAN_CATEGORY_WNM = 10,
+	RTW_WLAN_CATEGORY_UNPROTECTED_WNM = 11, /* add for CONFIG_IEEE80211W, none 11w also can use */
+	RTW_WLAN_CATEGORY_TDLS = 12,
+	RTW_WLAN_CATEGORY_MESH = 13,
+	RTW_WLAN_CATEGORY_MULTIHOP = 14,
+	RTW_WLAN_CATEGORY_SELF_PROTECTED = 15,
+	RTW_WLAN_CATEGORY_WMM = 17,
+	RTW_WLAN_CATEGORY_VHT = 21,
+	RTW_WLAN_CATEGORY_UNP_S1G = 22,
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	RTW_WLAN_CATEGORY_TBTX = 25,
+#endif
+	RTW_WLAN_CATEGORY_HE = 30,
+	RTW_WLAN_CATEGORY_PROTECTED_HE = 31,
+	RTW_WLAN_CATEGORY_P2P = 0x7f,/* P2P action frames */
+};
+
+#define CATEGORY_IS_GROUP_PRIVACY(cat) \
+	(cat == RTW_WLAN_CATEGORY_MESH || cat == RTW_WLAN_CATEGORY_MULTIHOP)
+
+#define CATEGORY_IS_NON_ROBUST(cat) \
+	(cat == RTW_WLAN_CATEGORY_PUBLIC \
+	|| cat == RTW_WLAN_CATEGORY_HT \
+	|| cat == RTW_WLAN_CATEGORY_UNPROTECTED_WNM \
+	|| cat == RTW_WLAN_CATEGORY_SELF_PROTECTED \
+	|| cat == RTW_WLAN_CATEGORY_VHT \
+	|| cat == RTW_WLAN_CATEGORY_UNP_S1G \
+	|| cat == RTW_WLAN_CATEGORY_P2P)
+
+#define CATEGORY_IS_ROBUST(cat) !CATEGORY_IS_NON_ROBUST(cat)
+
+/* SPECTRUM_MGMT action code */
+enum rtw_ieee80211_spectrum_mgmt_actioncode {
+	RTW_WLAN_ACTION_SPCT_MSR_REQ = 0,
+	RTW_WLAN_ACTION_SPCT_MSR_RPRT = 1,
+	RTW_WLAN_ACTION_SPCT_TPC_REQ = 2,
+	RTW_WLAN_ACTION_SPCT_TPC_RPRT = 3,
+	RTW_WLAN_ACTION_SPCT_CHL_SWITCH = 4,
+	RTW_WLAN_ACTION_SPCT_EXT_CHL_SWITCH = 5,
+};
+
+/* SELF_PROTECTED action code */
+enum rtw_ieee80211_self_protected_actioncode {
+	RTW_ACT_SELF_PROTECTED_RSVD = 0,
+	RTW_ACT_SELF_PROTECTED_MESH_OPEN = 1,
+	RTW_ACT_SELF_PROTECTED_MESH_CONF = 2,
+	RTW_ACT_SELF_PROTECTED_MESH_CLOSE = 3,
+	RTW_ACT_SELF_PROTECTED_MESH_GK_INFORM = 4,
+	RTW_ACT_SELF_PROTECTED_MESH_GK_ACK = 5,
+	RTW_ACT_SELF_PROTECTED_NUM,
+};
+
+/* MESH action code */
+enum rtw_ieee80211_mesh_actioncode {
+	RTW_ACT_MESH_LINK_METRIC_REPORT,
+	RTW_ACT_MESH_HWMP_PATH_SELECTION,
+	RTW_ACT_MESH_GATE_ANNOUNCEMENT,
+	RTW_ACT_MESH_CONGESTION_CONTROL_NOTIFICATION,
+	RTW_ACT_MESH_MCCA_SETUP_REQUEST,
+	RTW_ACT_MESH_MCCA_SETUP_REPLY,
+	RTW_ACT_MESH_MCCA_ADVERTISEMENT_REQUEST,
+	RTW_ACT_MESH_MCCA_ADVERTISEMENT,
+	RTW_ACT_MESH_MCCA_TEARDOWN,
+	RTW_ACT_MESH_TBTT_ADJUSTMENT_REQUEST,
+	RTW_ACT_MESH_TBTT_ADJUSTMENT_RESPONSE,
+};
+
+enum _PUBLIC_ACTION {
+	ACT_PUBLIC_BSSCOEXIST = 0, /* 20/40 BSS Coexistence */
+	ACT_PUBLIC_DSE_ENABLE = 1,
+	ACT_PUBLIC_DSE_DEENABLE = 2,
+	ACT_PUBLIC_DSE_REG_LOCATION = 3,
+	ACT_PUBLIC_EXT_CHL_SWITCH = 4,
+	ACT_PUBLIC_DSE_MSR_REQ = 5,
+	ACT_PUBLIC_DSE_MSR_RPRT = 6,
+	ACT_PUBLIC_MP = 7, /* Measurement Pilot */
+	ACT_PUBLIC_DSE_PWR_CONSTRAINT = 8,
+	ACT_PUBLIC_VENDOR = 9, /* for WIFI_DIRECT */
+	ACT_PUBLIC_GAS_INITIAL_REQ = 10,
+	ACT_PUBLIC_GAS_INITIAL_RSP = 11,
+	ACT_PUBLIC_GAS_COMEBACK_REQ = 12,
+	ACT_PUBLIC_GAS_COMEBACK_RSP = 13,
+	ACT_PUBLIC_TDLS_DISCOVERY_RSP = 14,
+	ACT_PUBLIC_LOCATION_TRACK = 15,
+	ACT_PUBLIC_MAX
+};
+
+#ifdef CONFIG_TDLS
+enum TDLS_ACTION_FIELD {
+	TDLS_SETUP_REQUEST = 0,
+	TDLS_SETUP_RESPONSE = 1,
+	TDLS_SETUP_CONFIRM = 2,
+	TDLS_TEARDOWN = 3,
+	TDLS_PEER_TRAFFIC_INDICATION = 4,
+	TDLS_CHANNEL_SWITCH_REQUEST = 5,
+	TDLS_CHANNEL_SWITCH_RESPONSE = 6,
+	TDLS_PEER_PSM_REQUEST = 7,
+	TDLS_PEER_PSM_RESPONSE = 8,
+	TDLS_PEER_TRAFFIC_RESPONSE = 9,
+	TDLS_DISCOVERY_REQUEST = 10,
+	TDLS_DISCOVERY_RESPONSE = 14,	/* it's used in public action frame */
+};
+
+#define	TUNNELED_PROBE_REQ	15
+#define	TUNNELED_PROBE_RSP	16
+#endif /* CONFIG_TDLS */
+
+/* BACK action code */
+enum rtw_ieee80211_back_actioncode {
+	RTW_WLAN_ACTION_ADDBA_REQ = 0,
+	RTW_WLAN_ACTION_ADDBA_RESP = 1,
+	RTW_WLAN_ACTION_DELBA = 2,
+};
+
+/* HT features action code */
+enum rtw_ieee80211_ht_actioncode {
+	RTW_WLAN_ACTION_HT_NOTI_CHNL_WIDTH = 0,
+	RTW_WLAN_ACTION_HT_SM_PS = 1,
+	RTW_WLAN_ACTION_HT_PSMP = 2,
+	RTW_WLAN_ACTION_HT_SET_PCO_PHASE = 3,
+	RTW_WLAN_ACTION_HT_CSI = 4,
+	RTW_WLAN_ACTION_HT_NON_COMPRESS_BEAMFORMING = 5,
+	RTW_WLAN_ACTION_HT_COMPRESS_BEAMFORMING = 6,
+	RTW_WLAN_ACTION_HT_ASEL_FEEDBACK = 7,
+};
+
+/* BACK (block-ack) parties */
+enum rtw_ieee80211_back_parties {
+	RTW_WLAN_BACK_RECIPIENT = 0,
+	RTW_WLAN_BACK_INITIATOR = 1,
+	RTW_WLAN_BACK_TIMER = 2,
+};
+
+/*20/40 BSS Coexistence element */
+#define RTW_WLAN_20_40_BSS_COEX_INFO_REQ            BIT(0)
+#define RTW_WLAN_20_40_BSS_COEX_40MHZ_INTOL         BIT(1)
+#define RTW_WLAN_20_40_BSS_COEX_20MHZ_WIDTH_REQ     BIT(2)
+#define RTW_WLAN_20_40_BSS_COEX_OBSS_EXEMPT_REQ     BIT(3)
+#define RTW_WLAN_20_40_BSS_COEX_OBSS_EXEMPT_GRNT    BIT(4)
+
+/* VHT features action code */
+enum rtw_ieee80211_vht_actioncode {
+	RTW_WLAN_ACTION_VHT_COMPRESSED_BEAMFORMING = 0,
+	RTW_WLAN_ACTION_VHT_GROUPID_MANAGEMENT = 1,
+	RTW_WLAN_ACTION_VHT_OPMODE_NOTIFICATION = 2,
+};
+
+enum rtw_ieee80211_unp_s1g_actioncode {
+	RTW_WLAN_ACTION_UNP_S1G_TWT_SETUP = 6,
+	RTW_WLAN_ACTION_UNP_S1G_TWT_TEARDOWN = 7,
+	RTW_WLAN_ACTION_UNP_S1G_TWT_INFO = 11,
+};
+
+enum EXT_CAP_INFO{
+	BSS_COEXT = 0, /* 20/40 BSS Coexistence Management Support */
+	EXT_CH_SWITCH = 2, /* Extended Channel Switching */
+	WNM_SLEEP_MODE = 17, /* WNM Sleep Mode */
+	BSS_TRANSITION = 19, /* BSS Transition */
+	MULTI_BSSID = 22, /* Multiple BSSID */
+	TIME_MEASUREMENT = 23, /* Timing Measurement */
+	SSID_LIST = 25, /* SSID List */
+	TDLS_PSM = 29, /* TDLS Peer PSM Support */
+	TDLS_CH_SWITCH = 30, /* TDLS channel switching */
+	INTERWORKING = 31, /* Interworking */
+	TDLS_SUPPORT = 37, /* TDLS Support */
+	WNM_NOTIFICATION = 46, /* WNM Notification */
+	OP_MODE_NOTIFICATION = 62, /* Operating Mode Notification */
+	FTM_RESPONDER = 70, /* Fine Timing Measurement Responder */
+	FTM_INITIATOR = 71, /* Fine Timing Measurement Initiator */
+	TWT_REQ_SUPPORT = 77, /* TWT Requester Support */
+	TWT_RSP_SUPPORT = 78  /* TWT Responder Support */
+};
+
+#define OUI_MICROSOFT 0x0050f2 /* Microsoft (also used in Wi-Fi specs)
+				* 00:50:F2 */
+#define WME_OUI_TYPE 2
+#define WME_OUI_SUBTYPE_INFORMATION_ELEMENT 0
+#define WME_OUI_SUBTYPE_PARAMETER_ELEMENT 1
+#define WME_OUI_SUBTYPE_TSPEC_ELEMENT 2
+#define WME_VERSION 1
+
+#define WME_ACTION_CODE_SETUP_REQUEST 0
+#define WME_ACTION_CODE_SETUP_RESPONSE 1
+#define WME_ACTION_CODE_TEARDOWN 2
+
+#define WME_SETUP_RESPONSE_STATUS_ADMISSION_ACCEPTED 0
+#define WME_SETUP_RESPONSE_STATUS_INVALID_PARAMETERS 1
+#define WME_SETUP_RESPONSE_STATUS_REFUSED 3
+
+#define WME_TSPEC_DIRECTION_UPLINK 0
+#define WME_TSPEC_DIRECTION_DOWNLINK 1
+#define WME_TSPEC_DIRECTION_BI_DIRECTIONAL 3
+
+
+#define OUI_BROADCOM 0x00904c /* Broadcom (Epigram) */
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+#define OUI_REALTEK	0x00e04c /* Realtek */
+#endif
+#define VENDOR_HT_CAPAB_OUI_TYPE 0x33 /* 00-90-4c:0x33 */
+
+#define MBO_OUI_TYPE 0x16
+
+enum rtw_ieee80211_rann_flags {
+	RTW_RANN_FLAG_IS_GATE = 1 << 0,
+};
+
+/**
+ * enum rtw_ieee80211_preq_flags - mesh PREQ element flags
+ *
+ * @RTW_IEEE80211_PREQ_IS_GATE_FLAG: Gate Announcement subfield
+ * @RTW_IEEE80211_PREQ_PROACTIVE_PREP_FLAG: proactive PREP subfield
+ */
+enum rtw_ieee80211_preq_flags {
+	RTW_IEEE80211_PREQ_IS_GATE_FLAG = 1 << 0,
+	RTW_IEEE80211_PREQ_PROACTIVE_PREP_FLAG	= 1 << 2,
+};
+
+/**
+ * enum rtw_ieee80211_preq_target_flags - mesh PREQ element per target flags
+ *
+ * @RTW_IEEE80211_PREQ_TO_FLAG: target only subfield
+ * @RTW_IEEE80211_PREQ_USN_FLAG: unknown target HWMP sequence number subfield
+ */
+enum rtw_ieee80211_preq_target_flags {
+	RTW_IEEE80211_PREQ_TO_FLAG	= 1<<0,
+	RTW_IEEE80211_PREQ_USN_FLAG	= 1<<2,
+};
+
+/**
+ * enum rtw_ieee80211_root_mode_identifier - root mesh STA mode identifier
+ *
+ * These attribute are used by dot11MeshHWMPRootMode to set root mesh STA mode
+ *
+ * @RTW_IEEE80211_ROOTMODE_NO_ROOT: the mesh STA is not a root mesh STA (default)
+ * @RTW_IEEE80211_ROOTMODE_ROOT: the mesh STA is a root mesh STA if greater than
+ *	this value
+ * @RTW_IEEE80211_PROACTIVE_PREQ_NO_PREP: the mesh STA is a root mesh STA supports
+ *	the proactive PREQ with proactive PREP subfield set to 0
+ * @RTW_IEEE80211_PROACTIVE_PREQ_WITH_PREP: the mesh STA is a root mesh STA
+ *	supports the proactive PREQ with proactive PREP subfield set to 1
+ * @RTW_IEEE80211_PROACTIVE_RANN: the mesh STA is a root mesh STA supports
+ *	the proactive RANN
+ */
+enum rtw_ieee80211_root_mode_identifier {
+	RTW_IEEE80211_ROOTMODE_NO_ROOT = 0,
+	RTW_IEEE80211_ROOTMODE_ROOT = 1,
+	RTW_IEEE80211_PROACTIVE_PREQ_NO_PREP = 2,
+	RTW_IEEE80211_PROACTIVE_PREQ_WITH_PREP = 3,
+	RTW_IEEE80211_PROACTIVE_RANN = 4,
+};
+
+/**
+ * enum rtw_ieee80211_channel_flags - channel flags
+ *
+ * Channel flags set by the regulatory control code.
+ *
+ * @RTW_IEEE80211_CHAN_DISABLED: This channel is disabled.
+ * @RTW_IEEE80211_CHAN_PASSIVE_SCAN: Only passive scanning is permitted
+ *      on this channel.
+ * @RTW_IEEE80211_CHAN_NO_IBSS: IBSS is not allowed on this channel.
+ * @RTW_IEEE80211_CHAN_RADAR: Radar detection is required on this channel.
+ * @RTW_IEEE80211_CHAN_NO_HT40PLUS: extension channel above this channel
+ *      is not permitted.
+ * @RTW_IEEE80211_CHAN_NO_HT40MINUS: extension channel below this channel
+ *      is not permitted.
+ */
+enum rtw_ieee80211_channel_flags {
+	RTW_IEEE80211_CHAN_DISABLED         = 1 << 0,
+	RTW_IEEE80211_CHAN_PASSIVE_SCAN     = 1 << 1,
+	RTW_IEEE80211_CHAN_NO_IBSS          = 1 << 2,
+	RTW_IEEE80211_CHAN_RADAR            = 1 << 3,
+	RTW_IEEE80211_CHAN_NO_HT40PLUS      = 1 << 4,
+	RTW_IEEE80211_CHAN_NO_HT40MINUS     = 1 << 5,
+};
+
+#define RTW_IEEE80211_CHAN_NO_HT40 \
+	(RTW_IEEE80211_CHAN_NO_HT40PLUS | RTW_IEEE80211_CHAN_NO_HT40MINUS)
+
+/* Represent channel details, subset of ieee80211_channel */
+struct rtw_ieee80211_channel {
+	u8 band; /*enum band_type band;*//* enum ieee80211_band band; or enum nl80211_band band; */
+	/* u32 center_freq; */
+	/* u16 freq_offset; */
+	u16 hw_value;
+	u32 flags;
+	/* int max_antenna_gain; */
+	/* int max_power; */
+	/* int max_reg_power; */
+	/* bool beacon_found; */
+	/* u32 orig_flags; */
+	/* int orig_mag; */
+	/* int orig_mpwr; */
+	/* enum nl80211_dfs_state dfs_state; */
+	/* unsigned long dfs_state_entered; */
+	/* unsigned int dfs_cac_ms; */
+};
+
+#define CHAN_FMT \
+	"band:%d, " \
+	/*"center_freq:%u, "*/ \
+	"hw_value:%u, " \
+	"flags:0x%08x" \
+	/*"max_antenna_gain:%d\n"*/ \
+	/*"max_power:%d\n"*/ \
+	/*"max_reg_power:%d\n"*/ \
+	/*"beacon_found:%u\n"*/ \
+	/*"orig_flags:0x%08x\n"*/ \
+	/*"orig_mag:%d\n"*/ \
+	/*"orig_mpwr:%d\n"*/
+
+#define CHAN_ARG(channel) \
+	(channel)->band \
+	/*, (channel)->center_freq*/ \
+	, (channel)->hw_value \
+	, (channel)->flags \
+	/*, (channel)->max_antenna_gain*/ \
+	/*, (channel)->max_power*/ \
+	/*, (channel)->max_reg_power*/ \
+	/*, (channel)->beacon_found*/ \
+	/*, (channel)->orig_flags*/ \
+	/*, (channel)->orig_mag*/ \
+	/*, (channel)->orig_mpwr*/ \
+
+/* Parsed Information Elements */
+struct rtw_ieee802_11_elems {
+	u8 *ssid;
+	u8 ssid_len;
+	u8 *supp_rates;
+	u8 supp_rates_len;
+	u8 *fh_params;
+	u8 fh_params_len;
+	u8 *ds_params;
+	u8 ds_params_len;
+	u8 *cf_params;
+	u8 cf_params_len;
+	u8 *tim;
+	u8 tim_len;
+	u8 *ibss_params;
+	u8 ibss_params_len;
+	u8 *challenge;
+	u8 challenge_len;
+	u8 *erp_info;
+	u8 erp_info_len;
+	u8 *ext_supp_rates;
+	u8 ext_supp_rates_len;
+	u8 *wpa_ie;
+	u8 wpa_ie_len;
+	u8 *rsn_ie;
+	u8 rsn_ie_len;
+	u8 *wme;
+	u8 wme_len;
+	u8 *wme_tspec;
+	u8 wme_tspec_len;
+	u8 *wps_ie;
+	u8 wps_ie_len;
+	u8 *power_cap;
+	u8 power_cap_len;
+	u8 *supp_channels;
+	u8 supp_channels_len;
+	u8 *mdie;
+	u8 mdie_len;
+	u8 *ftie;
+	u8 ftie_len;
+	u8 *timeout_int;
+	u8 timeout_int_len;
+	u8 *ht_capabilities;
+	u8 ht_capabilities_len;
+	u8 *ht_operation;
+	u8 ht_operation_len;
+	u8 *vendor_ht_cap;
+	u8 vendor_ht_cap_len;
+	u8 *vht_capabilities;
+	u8 vht_capabilities_len;
+	u8 *vht_operation;
+	u8 vht_operation_len;
+	u8 *vht_op_mode_notify;
+	u8 vht_op_mode_notify_len;
+	u8 *he_capabilities;
+	u8 he_capabilities_len;
+	u8 *he_operation;
+	u8 *he_6g_band_cap;
+	u8 he_6g_band_cap_len;
+	u8 he_operation_len;
+	u8 *rm_en_cap;
+	u8 rm_en_cap_len;
+#ifdef CONFIG_RTW_MESH
+	u8 *preq;
+	u8 preq_len;
+	u8 *prep;
+	u8 prep_len;
+	u8 *perr;
+	u8 perr_len;
+	u8 *rann;
+	u8 rann_len;
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	u8 *tbtx_cap;
+	u8 tbtx_cap_len;
+#endif
+	u8 *ap_channel_rpt;
+	u8 ap_channel_rpt_len;
+	u8 *country_info;
+	u8 country_info_len;
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+	u8 *mbssid;
+	u8 mbssid_len;
+
+	/* exist in nontransmitted bssid profile */
+	u8 *non_tx_bssid_cap;
+	u8 non_tx_bssid_cap_len;
+#endif
+};
+
+typedef enum { ParseOK = 0, ParseUnknown = 1, ParseFailed = -1 } ParseRes;
+
+ParseRes rtw_ieee802_11_parse_elems(u8 *start, uint len,
+				struct rtw_ieee802_11_elems *elems,
+				int show_errors);
+
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+ParseRes rtw_ieee802_11_override_elems_by_mbssid(
+	u8 *mbssid_ie, uint mbssid_ie_len, u8 mbssid_idx, struct rtw_ieee802_11_elems *elems
+	, int show_errors);
+#endif
+
+u8 *rtw_set_fixed_ie(unsigned char *pbuf, unsigned int len, unsigned char *source, unsigned int *frlen);
+u8 *rtw_set_ie(u8 *pbuf, sint index, uint len, const u8 *source, uint *frlen);
+
+/*
+ * Represent Extention Channel Offset in HT Capabilities
+ * This is available only in 40Mhz mode.
+ *   */
+#if 0
+typedef enum _EXTCHNL_OFFSET {
+	EXTCHNL_OFFSET_NO_EXT = 0,
+	EXTCHNL_OFFSET_UPPER = 1,
+	EXTCHNL_OFFSET_NO_DEF = 2,
+	EXTCHNL_OFFSET_LOWER = 3,
+} EXTCHNL_OFFSET, *PEXTCHNL_OFFSET;
+#endif
+
+/*enum chan_offset*/
+enum secondary_ch_offset {
+	IEEE80211_SCN = 0, /* no secondary channel */
+	IEEE80211_SCA = 1, /* the secondary channel is above the primary channel */
+	IEEE80211_SCB = 3,  /* the secondary channel is below the primary channel */
+};
+u8 secondary_ch_offset_to_hal_ch_offset(u8 ch_offset);
+u8 hal_ch_offset_to_secondary_ch_offset(u8 ch_offset);
+u8 *rtw_set_ie_ch_switch(u8 *buf, u32 *buf_len, u8 ch_switch_mode, u8 new_ch, u8 ch_switch_cnt);
+u8 *rtw_set_ie_secondary_ch_offset(u8 *buf, u32 *buf_len, u8 secondary_ch_offset);
+u8 *rtw_set_ie_wide_bw_ch_switch(u8 *buf, u32 *buf_len,
+	u8 ch_width, u8 seg_0, u8 seg_1);
+u8 *rtw_set_ie_mesh_ch_switch_parm(u8 *buf, u32 *buf_len, u8 ttl, u8 flags, u16 reason, u16 precedence);
+
+u8 *rtw_get_ie(const u8 *pbuf, sint index, sint *len, sint limit);
+u8 *rtw_get_ext_ie(const u8 *pbuf, sint ext_id, sint *len, sint limit);
+u8 rtw_update_rate_bymode(WLAN_BSSID_EX *pbss_network, u32 mode);
+
+u8 *rtw_get_ie_ex(const u8 *in_ie, uint in_len, u8 eid, const u8 *oui, u8 oui_len, u8 *ie, uint *ielen);
+u8 rtw_ies_update_ie(u8 *ies, uint *ies_len, uint ies_offset, u8 eid, const u8 *content, u8 content_len);
+u8 rtw_ies_update_ie_ex(u8 *ies, uint *ies_len, uint ies_offset, u8 eid_ex, const u8 *content, u8 content_len);
+u8 rtw_ies_add_ie(u8 *ies, uint *ies_len, uint ies_offset, u8 eid,
+		  const u8 *content, u8 content_len);
+int rtw_ies_remove_ie(u8 *ies, uint *ies_len, uint offset, u8 eid, u8 *oui, u8 oui_len);
+
+void rtw_set_supported_rate(u8 *SupportedRates, uint mode, u8 ch, enum band_type band);
+
+#define GET_RSN_CAP_MFP_OPTION(cap)	LE_BITS_TO_2BYTE(((u8 *)(cap)), 6, 2)
+
+#define MFP_NO			0
+#define MFP_INVALID		1
+#define MFP_OPTIONAL	2
+#define MFP_REQUIRED	3
+
+/*For amsdu mode */
+#define GET_RSN_CAP_SPP_OPT(cap) LE_BITS_TO_2BYTE(((u8 *)(cap)), 10, 2)
+#define SET_RSN_CAP_SPP(cap, spp) SET_BITS_TO_LE_2BYTE(((u8 *)(cap)), 10, 2, spp)
+#define SPP_CAP BIT(0)
+#define SPP_REQ BIT(1)
+
+enum rtw_amsdu_mode {
+	RTW_AMSDU_MODE_NON_SPP	= 0,
+	RTW_AMSDU_MODE_SPP	= 1,
+	RTW_AMSDU_MODE_ALL_DROP	= 2,
+	};
+
+
+struct rsne_info {
+	u8 *gcs;
+	u16 pcs_cnt;
+	u8 *pcs_list;
+	u16 akm_cnt;
+	u8 *akm_list;
+	u8 *cap;
+	u16 pmkid_cnt;
+	u8 *pmkid_list;
+	u8 *gmcs;
+
+	u8 err;
+};
+int rtw_rsne_info_parse(const u8 *ie, uint ie_len, struct rsne_info *info);
+
+unsigned char *rtw_get_wpa_ie(unsigned char *pie, int *wpa_ie_len, int limit);
+unsigned char *rtw_get_wpa2_ie(unsigned char *pie, int *rsn_ie_len, int limit);
+int rtw_get_wpa_cipher_suite(u8 *s);
+int rtw_get_rsn_cipher_suite(u8 *s);
+int rtw_get_wapi_ie(u8 *in_ie, uint in_len, u8 *wapi_ie, u16 *wapi_len);
+int rtw_parse_wpa_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwise_cipher, u32 *akm);
+int rtw_parse_wpa2_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwise_cipher, int *gmcs, u32 *akm, u8 *mfp_opt, u8* spp_opt);
+
+int rtw_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len, u8 *wpa_ie, u16 *wpa_len);
+
+u8 rtw_is_wps_ie(u8 *ie_ptr, uint *wps_ielen);
+u8 *rtw_get_wps_ie_from_scan_queue(u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen, enum bss_type frame_type);
+u8 *rtw_get_wps_ie(const u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen);
+u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id , u8 *buf_attr, u32 *len_attr);
+u8 *rtw_get_wps_attr_content(u8 *wps_ie, uint wps_ielen, u16 target_attr_id , u8 *buf_content, uint *len_content);
+
+u8 *rtw_get_owe_ie(const u8 *in_ie, uint in_len, u8 *owe_ie, uint *owe_ielen);
+
+void rtw_add_ext_cap_info(u8 *ext_cap_data, u8 *ext_cap_data_len, u8 cap_info);
+void rtw_remove_ext_cap_info(u8 *ext_cap_data, u8 *ext_cap_data_len, u8 cap_info);
+u8 rtw_update_ext_cap_ie(u8 *ext_cap_data, u8 ext_cap_data_len, u8 *ies, u32 *ies_len, u8 ies_offset);
+void rtw_parse_ext_cap_ie(u8 *ext_cap_data, u8 *ext_cap_data_len, u8 *ies, u32 ies_len, u8 ies_offset);
+
+/**
+ * for_each_ie - iterate over continuous IEs
+ * @ie:
+ * @buf:
+ * @buf_len:
+ */
+#define for_each_ie(ie, buf, buf_len) \
+	for (ie = (void *)buf; (((u8 *)ie) - ((u8 *)buf) + 1) < buf_len; ie = (void *)(((u8 *)ie) + *(((u8 *)ie)+1) + 2))
+
+void dump_ies(void *sel, const u8 *buf, u32 buf_len);
+
+#ifdef CONFIG_80211N_HT
+#define HT_SC_OFFSET_MAX 4
+extern const char *const _ht_sc_offset_str[];
+#define ht_sc_offset_str(sc) (((sc) >= HT_SC_OFFSET_MAX) ? _ht_sc_offset_str[2] : _ht_sc_offset_str[(sc)])
+
+void dump_ht_cap_ie_content(void *sel, const u8 *buf, u32 buf_len);
+#endif
+
+void dump_wps_ie(void *sel, const u8 *ie, u32 ie_len);
+
+RTW_FUNC_2G_5G_ONLY void rtw_ies_get_chbw(u8 *ies, int ies_len, u8 *ch, u8 *bw, u8 *offset, u8 ht, u8 vht);
+u8 *rtw_ies_get_he_6g_op_info_ie(u8 *ies, int ies_len);
+void rtw_ies_get_bchbw(u8 *ies, int ies_len, enum band_type *band, u8 *chan, u8 *bw,
+	u8 *offset, u8 *freq0, u8 *freq1, u8 ht, u8 vht, u8 he);
+void rtw_bss_get_chbw(WLAN_BSSID_EX *bss, enum band_type *band, u8 *ch, u8 *bw, u8 *offset, u8 ht, u8 vht, u8 he);
+
+u32 rtw_get_p2p_merged_ies_len(u8 *in_ie, u32 in_len);
+int rtw_p2p_merge_ies(u8 *in_ie, u32 in_len, u8 *merge_ie);
+void dump_p2p_ie(void *sel, const u8 *ie, u32 ie_len);
+u8 *rtw_get_p2p_ie(const u8 *in_ie, int in_len, u8 *p2p_ie, uint *p2p_ielen);
+u8 *rtw_get_vendor_ie(const u8 *in_ie, int in_len, u8 *vendor_ie, uint *vendor_ielen);
+u8 *rtw_get_p2p_attr(u8 *p2p_ie, uint p2p_ielen, u8 target_attr_id, u8 *buf_attr, u32 *len_attr);
+u8 *rtw_get_p2p_attr_content(u8 *p2p_ie, uint p2p_ielen, u8 target_attr_id, u8 *buf_content, uint *len_content);
+u32 rtw_set_p2p_attr_content(u8 *pbuf, u8 attr_id, u16 attr_len, u8 *pdata_attr);
+uint rtw_del_p2p_ie(u8 *ies, uint ies_len_ori, const char *msg);
+uint rtw_del_p2p_attr(u8 *ie, uint ielen_ori, u8 attr_id);
+u8 *rtw_bss_ex_get_p2p_ie(WLAN_BSSID_EX *bss_ex, u8 *p2p_ie, uint *p2p_ielen);
+void rtw_bss_ex_del_p2p_ie(WLAN_BSSID_EX *bss_ex);
+void rtw_bss_ex_del_p2p_attr(WLAN_BSSID_EX *bss_ex, u8 attr_id);
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+int rtw_chk_p2p_wildcard_ssid(WLAN_BSSID_EX *bss_ex);
+int rtw_chk_p2p_ie(WLAN_BSSID_EX *bss_ex);
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+
+void dump_wfd_ie(void *sel, const u8 *ie, u32 ie_len);
+u8 *rtw_get_wfd_ie(const u8 *in_ie, int in_len, u8 *wfd_ie, uint *wfd_ielen);
+u8 *rtw_get_wfd_attr(u8 *wfd_ie, uint wfd_ielen, u8 target_attr_id, u8 *buf_attr, u32 *len_attr);
+u8 *rtw_get_wfd_attr_content(u8 *wfd_ie, uint wfd_ielen, u8 target_attr_id, u8 *buf_content, uint *len_content);
+uint rtw_del_wfd_ie(u8 *ies, uint ies_len_ori, const char *msg);
+uint rtw_del_wfd_attr(u8 *ie, uint ielen_ori, u8 attr_id);
+u8 *rtw_bss_ex_get_wfd_ie(WLAN_BSSID_EX *bss_ex, u8 *wfd_ie, uint *wfd_ielen);
+void rtw_bss_ex_del_wfd_ie(WLAN_BSSID_EX *bss_ex);
+void rtw_bss_ex_del_wfd_attr(WLAN_BSSID_EX *bss_ex, u8 attr_id);
+
+#define MULTI_AP_SUB_ELEM_TYPE 0x06
+#define MULTI_AP_TEAR_DOWN BIT(4)
+#define MULTI_AP_FRONTHAUL_BSS BIT(5)
+#define MULTI_AP_BACKHAUL_BSS BIT(6)
+#define MULTI_AP_BACKHAUL_STA BIT(7)
+#ifdef CONFIG_RTW_MULTI_AP
+void dump_multi_ap_ie(void *sel, const u8 *ie, u32 ie_len);
+u8 rtw_get_multi_ap_ie_ext(const u8 *ies, int ies_len);
+u8 *rtw_set_multi_ap_ie_ext(u8 *pbuf, uint *frlen, u8 val);
+#endif
+
+uint	rtw_get_rateset_len(u8	*rateset);
+
+struct registry_priv;
+int rtw_generate_ie(struct registry_priv *pregistrypriv);
+
+int rtw_get_bit_value_from_ieee_value(u8 val);
+
+uint	rtw_is_cckrates_included(u8 *rate);
+
+uint	rtw_is_cckratesonly_included(u8 *rate);
+uint rtw_get_cckrate_size(u8 *rate,u32 rate_length);
+int rtw_check_network_type(unsigned char *rate, int ratelen, int channel);
+
+u8 rtw_check_invalid_mac_address(u8 *mac_addr, u8 check_local_bit);
+void rtw_macaddr_cfg(u8 *out, const u8 *hw_mac_addr);
+
+u16 rtw_ht_mcs_rate(u8 bw_40MHz, u8 short_GI, unsigned char *MCS_rate);
+u8	rtw_ht_mcsset_to_nss(u8 *supp_mcs_set);
+u32	rtw_ht_mcs_set_to_bitmap(u8 *mcs_set, u8 nss);
+
+int rtw_action_frame_parse(const u8 *frame, u32 frame_len, u8 *category, u8 *action);
+const char *action_public_str(u8 action);
+
+u8 key_char2num(u8 ch);
+u8 key_2char2num(u8 hch, u8 lch);
+u8 str_2char2num(u8 hch, u8 lch);
+void macstr2num(u8 *dst, u8 *src);
+u8 convert_ip_addr(u8 hch, u8 mch, u8 lch);
+int wifirate2_ratetbl_inx(unsigned char rate);
+
+/* For amsdu mode. */
+/*void rtw_set_spp_amsdu_mode(u8 mode, u8 *rsn_ie, int rsn_ie_len);*/
+u8 rtw_check_amsdu_disable(u8 mode, u8 spp_opt);
+
+char *get_macaddr_str(char *str, void *sel, const u8 *addr);
+#endif /* IEEE80211_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/ieee80211_ext.h b/drivers/net/wireless/realtek/rtl8852bs/include/ieee80211_ext.h
new file mode 100644
index 000000000000..77ccf8344e7e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/ieee80211_ext.h
@@ -0,0 +1,312 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __IEEE80211_EXT_H
+#define __IEEE80211_EXT_H
+
+#include <drv_conf.h>
+#include <osdep_service.h>
+#include <drv_types.h>
+
+#define WMM_OUI_TYPE 2
+#define WMM_OUI_SUBTYPE_INFORMATION_ELEMENT 0
+#define WMM_OUI_SUBTYPE_PARAMETER_ELEMENT 1
+#define WMM_OUI_SUBTYPE_TSPEC_ELEMENT 2
+#define WMM_VERSION 1
+
+#define WPA_PROTO_WPA BIT(0)
+#define WPA_PROTO_RSN BIT(1)
+
+#define WPA_KEY_MGMT_IEEE8021X BIT(0)
+#define WPA_KEY_MGMT_PSK BIT(1)
+#define WPA_KEY_MGMT_NONE BIT(2)
+#define WPA_KEY_MGMT_IEEE8021X_NO_WPA BIT(3)
+#define WPA_KEY_MGMT_WPA_NONE BIT(4)
+
+
+#define WPA_CAPABILITY_PREAUTH BIT(0)
+#define WPA_CAPABILITY_MGMT_FRAME_PROTECTION BIT(6)
+#define WPA_CAPABILITY_PEERKEY_ENABLED BIT(9)
+
+
+#define PMKID_LEN 16
+
+
+#ifdef PLATFORM_LINUX
+struct wpa_ie_hdr {
+	u8 elem_id;
+	u8 len;
+	u8 oui[4]; /* 24-bit OUI followed by 8-bit OUI type */
+	u8 version[2]; /* little endian */
+} __attribute__((packed));
+
+struct rsn_ie_hdr {
+	u8 elem_id; /* WLAN_EID_RSN */
+	u8 len;
+	u8 version[2]; /* little endian */
+} __attribute__((packed));
+
+struct wme_ac_parameter {
+#if defined(CONFIG_LITTLE_ENDIAN)
+	/* byte 1 */
+	u8	aifsn:4,
+	     acm:1,
+	     aci:2,
+	     reserved:1;
+
+	/* byte 2 */
+	u8	eCWmin:4,
+	     eCWmax:4;
+#elif defined(CONFIG_BIG_ENDIAN)
+	/* byte 1 */
+	u8	reserved:1,
+	     aci:2,
+	     acm:1,
+	     aifsn:4;
+
+	/* byte 2 */
+	u8	eCWmax:4,
+	     eCWmin:4;
+#else
+#error	"Please fix <endian.h>"
+#endif
+
+	/* bytes 3 & 4 */
+	u16 txopLimit;
+} __attribute__((packed));
+
+struct wme_parameter_element {
+	/* required fields for WME version 1 */
+	u8 oui[3];
+	u8 oui_type;
+	u8 oui_subtype;
+	u8 version;
+	u8 acInfo;
+	u8 reserved;
+	struct wme_ac_parameter ac[4];
+
+} __attribute__((packed));
+
+#endif
+
+#define WPA_PUT_LE16(a, val)			\
+	do {					\
+		(a)[1] = ((u16) (val)) >> 8;	\
+		(a)[0] = ((u16) (val)) & 0xff;	\
+	} while (0)
+
+#define WPA_PUT_BE32(a, val)					\
+	do {							\
+		(a)[0] = (u8) ((((u32) (val)) >> 24) & 0xff);	\
+		(a)[1] = (u8) ((((u32) (val)) >> 16) & 0xff);	\
+		(a)[2] = (u8) ((((u32) (val)) >> 8) & 0xff);	\
+		(a)[3] = (u8) (((u32) (val)) & 0xff);		\
+	} while (0)
+
+#define WPA_PUT_LE32(a, val)					\
+	do {							\
+		(a)[3] = (u8) ((((u32) (val)) >> 24) & 0xff);	\
+		(a)[2] = (u8) ((((u32) (val)) >> 16) & 0xff);	\
+		(a)[1] = (u8) ((((u32) (val)) >> 8) & 0xff);	\
+		(a)[0] = (u8) (((u32) (val)) & 0xff);		\
+	} while (0)
+
+#define RSN_SELECTOR_PUT(a, val) WPA_PUT_BE32((u8 *) (a), (val))
+/* #define RSN_SELECTOR_PUT(a, val) WPA_PUT_LE32((u8 *) (a), (val)) */
+
+
+
+/* Action category code */
+enum ieee80211_category {
+	WLAN_CATEGORY_SPECTRUM_MGMT = 0,
+	WLAN_CATEGORY_QOS = 1,
+	WLAN_CATEGORY_DLS = 2,
+	WLAN_CATEGORY_BACK = 3,
+	WLAN_CATEGORY_HT = 7,
+	WLAN_CATEGORY_WMM = 17,
+};
+
+/* SPECTRUM_MGMT action code */
+enum ieee80211_spectrum_mgmt_actioncode {
+	WLAN_ACTION_SPCT_MSR_REQ = 0,
+	WLAN_ACTION_SPCT_MSR_RPRT = 1,
+	WLAN_ACTION_SPCT_TPC_REQ = 2,
+	WLAN_ACTION_SPCT_TPC_RPRT = 3,
+	WLAN_ACTION_SPCT_CHL_SWITCH = 4,
+	WLAN_ACTION_SPCT_EXT_CHL_SWITCH = 5,
+};
+
+/* BACK action code */
+enum ieee80211_back_actioncode {
+	WLAN_ACTION_ADDBA_REQ = 0,
+	WLAN_ACTION_ADDBA_RESP = 1,
+	WLAN_ACTION_DELBA = 2,
+};
+
+/* HT features action code */
+enum ieee80211_ht_actioncode {
+	WLAN_ACTION_NOTIFY_CH_WIDTH = 0,
+	WLAN_ACTION_SM_PS = 1,
+	WLAN_ACTION_PSPM = 2,
+	WLAN_ACTION_PCO_PHASE = 3,
+	WLAN_ACTION_MIMO_CSI_MX = 4,
+	WLAN_ACTION_MIMO_NONCP_BF = 5,
+	WLAN_ACTION_MIMP_CP_BF = 6,
+	WLAN_ACTION_ASEL_INDICATES_FB = 7,
+	WLAN_ACTION_HI_INFO_EXCHG = 8,
+};
+
+/* BACK (block-ack) parties */
+enum ieee80211_back_parties {
+	WLAN_BACK_RECIPIENT = 0,
+	WLAN_BACK_INITIATOR = 1,
+	WLAN_BACK_TIMER = 2,
+};
+
+#ifdef PLATFORM_LINUX
+
+struct ieee80211_mgmt {
+	u16 frame_control;
+	u16 duration;
+	u8 da[6];
+	u8 sa[6];
+	u8 bssid[6];
+	u16 seq_ctrl;
+	union {
+		struct {
+			u16 auth_alg;
+			u16 auth_transaction;
+			u16 status_code;
+			/* possibly followed by Challenge text */
+			u8 variable[0];
+		}  __attribute__((packed)) auth;
+		struct {
+			u16 reason_code;
+		}  __attribute__((packed)) deauth;
+		struct {
+			u16 capab_info;
+			u16 listen_interval;
+			/* followed by SSID and Supported rates */
+			u8 variable[0];
+		}  __attribute__((packed)) assoc_req;
+		struct {
+			u16 capab_info;
+			u16 status_code;
+			u16 aid;
+			/* followed by Supported rates */
+			u8 variable[0];
+		}  __attribute__((packed)) assoc_resp, reassoc_resp;
+		struct {
+			u16 capab_info;
+			u16 listen_interval;
+			u8 current_ap[6];
+			/* followed by SSID and Supported rates */
+			u8 variable[0];
+		}  __attribute__((packed)) reassoc_req;
+		struct {
+			u16 reason_code;
+		}  __attribute__((packed)) disassoc;
+		struct {
+			__le64 timestamp;
+			u16 beacon_int;
+			u16 capab_info;
+			/* followed by some of SSID, Supported rates,
+			 * FH Params, DS Params, CF Params, IBSS Params, TIM */
+			u8 variable[0];
+		}  __attribute__((packed)) beacon;
+		struct {
+			/* only variable items: SSID, Supported rates */
+			u8 variable[0];
+		}  __attribute__((packed)) probe_req;
+		struct {
+			__le64 timestamp;
+			u16 beacon_int;
+			u16 capab_info;
+			/* followed by some of SSID, Supported rates,
+			 * FH Params, DS Params, CF Params, IBSS Params */
+			u8 variable[0];
+		}  __attribute__((packed)) probe_resp;
+		struct {
+			u8 category;
+			union {
+				struct {
+					u8 action_code;
+					u8 dialog_token;
+					u8 status_code;
+					u8 variable[0];
+				}  __attribute__((packed)) wme_action;
+#if 0
+				struct {
+					u8 action_code;
+					u8 element_id;
+					u8 length;
+					struct ieee80211_channel_sw_ie sw_elem;
+				}  __attribute__((packed)) chan_switch;
+				struct {
+					u8 action_code;
+					u8 dialog_token;
+					u8 element_id;
+					u8 length;
+					struct ieee80211_msrment_ie msr_elem;
+				}  __attribute__((packed)) measurement;
+#endif
+				struct {
+					u8 action_code;
+					u8 dialog_token;
+					u16 capab;
+					u16 timeout;
+					u16 start_seq_num;
+				}  __attribute__((packed)) addba_req;
+				struct {
+					u8 action_code;
+					u8 dialog_token;
+					u16 status;
+					u16 capab;
+					u16 timeout;
+				}  __attribute__((packed)) addba_resp;
+				struct {
+					u8 action_code;
+					u16 params;
+					u16 reason_code;
+				}  __attribute__((packed)) delba;
+				struct {
+					u8 action_code;
+					/* capab_info for open and confirm,
+					 * reason for close
+					 */
+					u16 aux;
+					/* Followed in plink_confirm by status
+					 * code, AID and supported rates,
+					 * and directly by supported rates in
+					 * plink_open and plink_close
+					 */
+					u8 variable[0];
+				}  __attribute__((packed)) plink_action;
+				struct {
+					u8 action_code;
+					u8 variable[0];
+				}  __attribute__((packed)) mesh_action;
+			} __attribute__((packed)) u;
+		}  __attribute__((packed)) action;
+	} __attribute__((packed)) u;
+} __attribute__((packed));
+
+#endif
+
+/* mgmt header + 1 byte category code */
+#define IEEE80211_MIN_ACTION_SIZE FIELD_OFFSET(struct ieee80211_mgmt, u.action.u)
+
+
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/if_ether.h b/drivers/net/wireless/realtek/rtl8852bs/include/if_ether.h
new file mode 100644
index 000000000000..0778cbd43a25
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/if_ether.h
@@ -0,0 +1,106 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef _LINUX_IF_ETHER_H
+#define _LINUX_IF_ETHER_H
+
+/*
+ *	IEEE 802.3 Ethernet magic constants.  The frame sizes omit the preamble
+ *	and FCS/CRC (frame check sequence).
+ */
+
+#define ETH_ALEN	6		/* Octets in one ethernet addr	 */
+#define ETH_HLEN	14		/* Total octets in header.	 */
+#define ETH_ZLEN	60		/* Min. octets in frame sans FCS */
+#define ETH_DATA_LEN	1500		/* Max. octets in payload	 */
+#define ETH_FRAME_LEN	1514		/* Max. octets in frame sans FCS */
+
+/*
+ *	These are the defined Ethernet Protocol ID's.
+ */
+
+#define ETH_P_LOOP	0x0060		/* Ethernet Loopback packet	*/
+#define ETH_P_PUP	0x0200		/* Xerox PUP packet		*/
+#define ETH_P_PUPAT	0x0201		/* Xerox PUP Addr Trans packet	*/
+#define ETH_P_IP	0x0800		/* Internet Protocol packet	*/
+#define ETH_P_X25	0x0805		/* CCITT X.25			*/
+#define ETH_P_ARP	0x0806		/* Address Resolution packet	*/
+#define	ETH_P_BPQ	0x08FF		/* G8BPQ AX.25 Ethernet Packet	[ NOT AN OFFICIALLY REGISTERED ID ] */
+#define ETH_P_IEEEPUP	0x0a00		/* Xerox IEEE802.3 PUP packet */
+#define ETH_P_IEEEPUPAT	0x0a01		/* Xerox IEEE802.3 PUP Addr Trans packet */
+#define ETH_P_DEC       0x6000          /* DEC Assigned proto          */
+#define ETH_P_DNA_DL    0x6001          /* DEC DNA Dump/Load           */
+#define ETH_P_DNA_RC    0x6002          /* DEC DNA Remote Console      */
+#define ETH_P_DNA_RT    0x6003          /* DEC DNA Routing             */
+#define ETH_P_LAT       0x6004          /* DEC LAT                     */
+#define ETH_P_DIAG      0x6005          /* DEC Diagnostics             */
+#define ETH_P_CUST      0x6006          /* DEC Customer use            */
+#define ETH_P_SCA       0x6007          /* DEC Systems Comms Arch      */
+#define ETH_P_RARP      0x8035		/* Reverse Addr Res packet	*/
+#define ETH_P_ATALK	0x809B		/* Appletalk DDP		*/
+#define ETH_P_AARP	0x80F3		/* Appletalk AARP		*/
+#define ETH_P_8021Q	0x8100          /* 802.1Q VLAN Extended Header */
+#define ETH_P_IPX	0x8137		/* IPX over DIX			*/
+#define ETH_P_IPV6	0x86DD		/* IPv6 over bluebook		*/
+#define ETH_P_PPP_DISC	0x8863		/* PPPoE discovery messages    */
+#define ETH_P_PPP_SES	0x8864		/* PPPoE session messages	*/
+#define ETH_P_ATMMPOA	0x884c		/* MultiProtocol Over ATM	*/
+#define ETH_P_ATMFATE	0x8884		/* Frame-based ATM Transport
+					 * over Ethernet
+					 */
+
+/*
+ *	Non DIX types. Won't clash for 1500 types.
+ */
+
+#define ETH_P_802_3	0x0001		/* Dummy type for 802.3 frames */
+#define ETH_P_AX25	0x0002		/* Dummy protocol id for AX.25 */
+#define ETH_P_ALL	0x0003		/* Every packet (be careful!!!) */
+#define ETH_P_802_2	0x0004		/* 802.2 frames 		*/
+#define ETH_P_SNAP	0x0005		/* Internal only		*/
+#define ETH_P_DDCMP     0x0006          /* DEC DDCMP: Internal only    */
+#define ETH_P_WAN_PPP   0x0007          /* Dummy type for WAN PPP frames*/
+#define ETH_P_PPP_MP    0x0008          /* Dummy type for PPP MP frames */
+#define ETH_P_LOCALTALK 0x0009		/* Localtalk pseudo type 	*/
+#define ETH_P_PPPTALK	0x0010		/* Dummy type for Atalk over PPP*/
+#define ETH_P_TR_802_2	0x0011		/* 802.2 frames 		*/
+#define ETH_P_MOBITEX	0x0015		/* Mobitex (kaz@cafe.net)	*/
+#define ETH_P_CONTROL	0x0016		/* Card specific control frames */
+#define ETH_P_IRDA	0x0017		/* Linux-IrDA			*/
+#define ETH_P_ECONET	0x0018		/* Acorn Econet			*/
+
+/*
+ *	This is an Ethernet frame header.
+ */
+
+struct ethhdr {
+	unsigned char	h_dest[ETH_ALEN];	/* destination eth addr	*/
+	unsigned char	h_source[ETH_ALEN];	/* source ether addr	*/
+	unsigned short	h_proto;		/* packet type ID field	*/
+};
+
+struct _vlan {
+	unsigned short       h_vlan_TCI;                /* Encapsulates priority and VLAN ID */
+	unsigned short       h_vlan_encapsulated_proto;
+};
+
+
+
+#define get_vlan_id(pvlan) ((ntohs((unsigned short)pvlan->h_vlan_TCI)) & 0xfff)
+#define get_vlan_priority(pvlan) ((ntohs((unsigned short)pvlan->h_vlan_TCI))>>13)
+#define get_vlan_encap_proto(pvlan) (ntohs((unsigned short)pvlan->h_vlan_encapsulated_proto))
+
+
+#endif	/* _LINUX_IF_ETHER_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/ip.h b/drivers/net/wireless/realtek/rtl8852bs/include/ip.h
new file mode 100644
index 000000000000..6b609a2fd677
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/ip.h
@@ -0,0 +1,135 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _LINUX_IP_H
+#define _LINUX_IP_H
+
+/* SOL_IP socket options */
+
+#define IPTOS_TOS_MASK		0x1E
+#define IPTOS_TOS(tos)		((tos)&IPTOS_TOS_MASK)
+#define	IPTOS_LOWDELAY		0x10
+#define	IPTOS_THROUGHPUT	0x08
+#define	IPTOS_RELIABILITY	0x04
+#define	IPTOS_MINCOST		0x02
+
+#define IPTOS_PREC_MASK		0xE0
+#define IPTOS_PREC(tos)		((tos)&IPTOS_PREC_MASK)
+#define IPTOS_PREC_NETCONTROL           0xe0
+#define IPTOS_PREC_INTERNETCONTROL      0xc0
+#define IPTOS_PREC_CRITIC_ECP           0xa0
+#define IPTOS_PREC_FLASHOVERRIDE        0x80
+#define IPTOS_PREC_FLASH                0x60
+#define IPTOS_PREC_IMMEDIATE            0x40
+#define IPTOS_PREC_PRIORITY             0x20
+#define IPTOS_PREC_ROUTINE              0x00
+
+
+/* IP options */
+#define IPOPT_COPY		0x80
+#define IPOPT_CLASS_MASK	0x60
+#define IPOPT_NUMBER_MASK	0x1f
+
+#define	IPOPT_COPIED(o)		((o)&IPOPT_COPY)
+#define	IPOPT_CLASS(o)		((o)&IPOPT_CLASS_MASK)
+#define	IPOPT_NUMBER(o)		((o)&IPOPT_NUMBER_MASK)
+
+#define	IPOPT_CONTROL		0x00
+#define	IPOPT_RESERVED1		0x20
+#define	IPOPT_MEASUREMENT	0x40
+#define	IPOPT_RESERVED2		0x60
+
+#define IPOPT_END	(0 | IPOPT_CONTROL)
+#define IPOPT_NOOP	(1 | IPOPT_CONTROL)
+#define IPOPT_SEC	(2 | IPOPT_CONTROL | IPOPT_COPY)
+#define IPOPT_LSRR	(3 | IPOPT_CONTROL | IPOPT_COPY)
+#define IPOPT_TIMESTAMP	(4 | IPOPT_MEASUREMENT)
+#define IPOPT_RR	(7 | IPOPT_CONTROL)
+#define IPOPT_SID	(8 | IPOPT_CONTROL | IPOPT_COPY)
+#define IPOPT_SSRR	(9 | IPOPT_CONTROL | IPOPT_COPY)
+#define IPOPT_RA	(20 | IPOPT_CONTROL | IPOPT_COPY)
+
+#define IPVERSION	4
+#define MAXTTL		255
+#define IPDEFTTL	64
+
+/* struct timestamp, struct route and MAX_ROUTES are removed.
+
+   REASONS: it is clear that nobody used them because:
+   - MAX_ROUTES value was wrong.
+   - "struct route" was wrong.
+   - "struct timestamp" had fatally misaligned bitfields and was completely unusable.
+ */
+
+#define IPOPT_OPTVAL 0
+#define IPOPT_OLEN   1
+#define IPOPT_OFFSET 2
+#define IPOPT_MINOFF 4
+#define MAX_IPOPTLEN 40
+#define IPOPT_NOP IPOPT_NOOP
+#define IPOPT_EOL IPOPT_END
+#define IPOPT_TS  IPOPT_TIMESTAMP
+
+#define	IPOPT_TS_TSONLY		0		/* timestamps only */
+#define	IPOPT_TS_TSANDADDR	1		/* timestamps and addresses */
+#define	IPOPT_TS_PRESPEC	3		/* specified modules only */
+
+#ifdef PLATFORM_LINUX
+
+struct ip_options {
+	__u32		faddr;				/* Saved first hop address */
+	unsigned char	optlen;
+	unsigned char srr;
+	unsigned char rr;
+	unsigned char ts;
+	unsigned char is_setbyuser:1,			/* Set by setsockopt?			*/
+		 is_data:1,			/* Options in __data, rather than skb	*/
+		 is_strictroute:1,		/* Strict source route			*/
+		 srr_is_hit:1,			/* Packet destination addr was our one	*/
+		 is_changed:1,			/* IP checksum more not valid		*/
+		 rr_needaddr:1,			/* Need to record addr of outgoing dev	*/
+		 ts_needtime:1,			/* Need to record timestamp		*/
+		 ts_needaddr:1;			/* Need to record addr of outgoing dev */
+	unsigned char router_alert;
+	unsigned char __pad1;
+	unsigned char __pad2;
+	unsigned char __data[0];
+};
+
+#define optlength(opt) (sizeof(struct ip_options) + opt->optlen)
+#endif
+
+struct iphdr {
+#if defined(__LITTLE_ENDIAN_BITFIELD)
+	__u8	ihl:4,
+		version:4;
+#elif defined (__BIG_ENDIAN_BITFIELD)
+	__u8	version:4,
+		ihl:4;
+#else
+#error	"Please fix <asm/byteorder.h>"
+#endif
+	__u8	tos;
+	__u16	tot_len;
+	__u16	id;
+	__u16	frag_off;
+	__u8	ttl;
+	__u8	protocol;
+	__u16	check;
+	__u32	saddr;
+	__u32	daddr;
+	/*The options start here. */
+};
+
+#endif	/* _LINUX_IP_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/linux/wireless.h b/drivers/net/wireless/realtek/rtl8852bs/include/linux/wireless.h
new file mode 100644
index 000000000000..37a6061d0885
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/linux/wireless.h
@@ -0,0 +1,87 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef _LINUX_WIRELESS_H
+#define _LINUX_WIRELESS_H
+
+/***************************** INCLUDES *****************************/
+
+#if 0
+	#include <linux/types.h>		/* for __u* and __s* typedefs */
+	#include <linux/socket.h>		/* for "struct sockaddr" et al	*/
+	#include <linux/if.h>			/* for IFNAMSIZ and co... */
+#else
+	#define __user
+	/* typedef uint16_t	__u16; */
+	#include <sys/socket.h>			/* for "struct sockaddr" et al	*/
+	#include <net/if.h>			/* for IFNAMSIZ and co... */
+#endif
+
+/****************************** TYPES ******************************/
+#ifdef CONFIG_COMPAT
+struct compat_iw_point {
+	compat_caddr_t pointer;
+	__u16 length;
+	__u16 flags;
+};
+#endif
+/* --------------------------- SUBTYPES --------------------------- */
+/*
+ *	For all data larger than 16 octets, we need to use a
+ *	pointer to memory allocated in user space.
+ */
+struct	iw_point {
+	void __user	*pointer;	/* Pointer to the data  (in user space) */
+	__u16		length;		/* number of fields or size in bytes */
+	__u16		flags;		/* Optional params */
+};
+
+
+/* ------------------------ IOCTL REQUEST ------------------------ */
+/*
+ * This structure defines the payload of an ioctl, and is used
+ * below.
+ *
+ * Note that this structure should fit on the memory footprint
+ * of iwreq (which is the same as ifreq), which mean a max size of
+ * 16 octets = 128 bits. Warning, pointers might be 64 bits wide...
+ * You should check this when increasing the structures defined
+ * above in this file...
+ */
+union	iwreq_data {
+	/* Config - generic */
+	char		name[IFNAMSIZ];
+	/* Name : used to verify the presence of  wireless extensions.
+	 * Name of the protocol/provider... */
+
+	struct iw_point	data;		/* Other large parameters */
+};
+
+/*
+ * The structure to exchange data for ioctl.
+ * This structure is the same as 'struct ifreq', but (re)defined for
+ * convenience...
+ * Do I need to remind you about structure size (32 octets) ?
+ */
+struct	iwreq {
+	union {
+		char	ifrn_name[IFNAMSIZ];	/* if name, e.g. "eth0" */
+	} ifr_ifrn;
+
+	/* Data part (defined just above) */
+	union	iwreq_data	u;
+};
+
+#endif	/* _LINUX_WIRELESS_H */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/mlme_osdep.h b/drivers/net/wireless/realtek/rtl8852bs/include/mlme_osdep.h
new file mode 100644
index 000000000000..f5dd32941cf2
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/mlme_osdep.h
@@ -0,0 +1,42 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef	__MLME_OSDEP_H_
+#define __MLME_OSDEP_H_
+
+extern void rtw_os_indicate_disconnect(_adapter *adapter, u16 reason, u8 locally_generated);
+extern void rtw_os_indicate_connect(_adapter *adapter);
+void rtw_os_indicate_scan_done(_adapter *padapter, bool aborted);
+extern void rtw_report_sec_ie(_adapter *adapter, u8 authmode, u8 *sec_ie);
+
+void rtw_reset_securitypriv(_adapter *adapter);
+
+#ifdef CONFIG_DFS_MASTER
+void rtw_os_indicate_radar_detected(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 cch, enum channel_width bw);
+void rtw_os_indicate_cac_started(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_os_indicate_cac_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_os_indicate_cac_aborted(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_os_force_cac_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_os_indicate_nop_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw);
+void rtw_os_indicate_nop_started(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw, bool called_on_cmd_thd);
+#endif
+
+#endif /* _MLME_OSDEP_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/osdep_intf.h b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_intf.h
new file mode 100644
index 000000000000..745618631b0a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_intf.h
@@ -0,0 +1,100 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __OSDEP_INTF_H_
+#define __OSDEP_INTF_H_
+
+struct dvobj_priv *devobj_init(void);
+void devobj_deinit(struct dvobj_priv *pdvobj);
+u8 devobj_data_init(struct dvobj_priv *dvobj);
+void devobj_data_deinit(struct dvobj_priv *dvobj);
+u8 devobj_trx_resource_init(struct dvobj_priv *dvobj);
+void devobj_trx_resource_deinit(struct dvobj_priv *dvobj);
+
+u8 rtw_init_drv_sw(_adapter *padapter);
+u8 rtw_free_drv_sw(_adapter *padapter);
+u8 rtw_reset_drv_sw(_adapter *padapter);
+void rtw_drv_stop_prim_iface(_adapter *adapter);
+
+u8 rtw_adapter_link_init(struct dvobj_priv *dvobj);
+void rtw_adapter_link_deinit(struct dvobj_priv *dvobj);
+int rtw_init_link_capab(struct dvobj_priv *dvobj);
+
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+u32 rtw_start_drv_threads(_adapter *padapter);
+void rtw_stop_drv_threads(_adapter *padapter);
+#endif
+
+#if defined(CONFIG_WOWLAN) || defined(CONFIG_AP_WOWLAN)
+void rtw_cancel_dynamic_chk_timer(_adapter *padapter);
+#endif
+void rtw_cancel_all_timer(_adapter *padapter);
+
+
+#ifdef PLATFORM_LINUX
+int rtw_ioctl(struct net_device *dev, struct ifreq *rq, int cmd);
+
+int rtw_init_netdev_name(struct net_device *pnetdev, const char *ifname);
+struct net_device *rtw_init_netdev(_adapter *padapter);
+
+void rtw_os_ndev_free(_adapter *adapter);
+int rtw_os_ndev_init(_adapter *adapter, const char *name);
+void rtw_os_ndev_deinit(_adapter *adapter);
+void rtw_os_ndev_unregister(_adapter *adapter);
+void rtw_os_ndevs_unregister(struct dvobj_priv *dvobj);
+int rtw_os_ndevs_init(struct dvobj_priv *dvobj);
+void rtw_os_ndevs_deinit(struct dvobj_priv *dvobj);
+
+u16 rtw_os_recv_select_queue(u8 *msdu, enum rtw_rx_llc_hdl llc_hdl);
+
+int rtw_ndev_notifier_register(void);
+void rtw_ndev_notifier_unregister(void);
+void rtw_inetaddr_notifier_register(void);
+void rtw_inetaddr_notifier_unregister(void);
+
+#include "../os_dep/linux/rtw_proc.h"
+#include "../os_dep/linux/nlrtw.h"
+
+#ifdef CONFIG_IOCTL_CFG80211
+	#include "../os_dep/linux/ioctl_cfg80211.h"
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+u8 rtw_rtnl_lock_needed(struct dvobj_priv *dvobj);
+void rtw_set_rtnl_lock_holder(struct dvobj_priv *dvobj, _thread_hdl_ thd_hdl);
+
+#endif /* PLATFORM_LINUX */
+
+
+#ifdef PLATFORM_FREEBSD
+extern int rtw_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
+#endif
+
+#ifdef CONFIG_CONCURRENT_MODE
+struct _io_ops;
+struct dvobj_priv;
+u8 rtw_drv_add_vir_ifaces(struct dvobj_priv *dvobj);
+void rtw_drv_stop_vir_ifaces(struct dvobj_priv *dvobj);
+void rtw_drv_free_vir_ifaces(struct dvobj_priv *dvobj);
+#endif
+
+void rtw_ndev_destructor(_nic_hdl ndev);
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+int rtw_gw_addr_query(_adapter *padapter);
+#endif
+
+int rtw_suspend_common(_adapter *padapter);
+int rtw_resume_common(_adapter *padapter);
+int rtw_suspend_free_assoc_resource(_adapter *padapter);
+
+#endif /* _OSDEP_INTF_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service.h b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service.h
new file mode 100644
index 000000000000..93543ceef194
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service.h
@@ -0,0 +1,777 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __OSDEP_SERVICE_H_
+#define __OSDEP_SERVICE_H_
+
+#define RTW_RX_HANDLED		2
+#define RTW_RFRAME_UNAVAIL	3
+#define RTW_RFRAME_PKT_UNAVAIL	4
+#define RTW_RBUF_UNAVAIL	5
+#define RTW_RBUF_PKT_UNAVAIL	6
+#define RTW_SDIO_RECV_FAIL	7
+#define RTW_ALREADY		8
+#define RTW_RA_RESOLVING	9
+#define RTW_ORI_NO_NEED		10
+#define RTW_NOT_SUPPORT		15
+#define RTW_ABORT_LINKING	17
+
+/* #define RTW_STATUS_TIMEDOUT -110 */
+
+
+
+#ifdef PLATFORM_FREEBSD
+	#include <osdep_service_bsd.h>
+#endif
+
+#ifdef PLATFORM_LINUX
+	#include <linux/version.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0))
+	#include <linux/sched/signal.h>
+	#include <linux/sched/types.h>
+#endif
+	#include <osdep_service_linux.h>
+#endif
+
+/* #include <rtw_byteorder.h> */
+
+#ifndef BIT
+	#define BIT(x)	(1 << (x))
+#endif
+#ifndef BIT_ULL
+#define BIT_ULL(x)	(1ULL << (x))
+#endif
+
+#define CHECK_BIT(a, b) (!!((a) & (b)))
+
+#define BIT0	0x00000001
+#define BIT1	0x00000002
+#define BIT2	0x00000004
+#define BIT3	0x00000008
+#define BIT4	0x00000010
+#define BIT5	0x00000020
+#define BIT6	0x00000040
+#define BIT7	0x00000080
+#define BIT8	0x00000100
+#define BIT9	0x00000200
+#define BIT10	0x00000400
+#define BIT11	0x00000800
+#define BIT12	0x00001000
+#define BIT13	0x00002000
+#define BIT14	0x00004000
+#define BIT15	0x00008000
+#define BIT16	0x00010000
+#define BIT17	0x00020000
+#define BIT18	0x00040000
+#define BIT19	0x00080000
+#define BIT20	0x00100000
+#define BIT21	0x00200000
+#define BIT22	0x00400000
+#define BIT23	0x00800000
+#define BIT24	0x01000000
+#define BIT25	0x02000000
+#define BIT26	0x04000000
+#define BIT27	0x08000000
+#define BIT28	0x10000000
+#define BIT29	0x20000000
+#define BIT30	0x40000000
+#define BIT31	0x80000000
+#define BIT32	0x0100000000
+#define BIT33	0x0200000000
+#define BIT34	0x0400000000
+#define BIT35	0x0800000000
+#define BIT36	0x1000000000
+
+#ifndef GENMASK
+#define GENMASK(h, l) \
+	(((~0UL) - (1UL << (l)) + 1) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
+#endif
+
+extern int RTW_STATUS_CODE(int error_code);
+extern s32 rtw_warn_on_cnt;
+
+#ifndef RTK_DMP_PLATFORM
+	#define CONFIG_USE_VMALLOC
+#endif
+
+/* flags used for rtw_mstat_update() */
+enum mstat_f {
+	/* type: 0x00ff */
+	MSTAT_TYPE_VIR = 0x00,
+	MSTAT_TYPE_PHY = 0x01,
+	MSTAT_TYPE_SKB = 0x02,
+	MSTAT_TYPE_USB = 0x03,
+	MSTAT_TYPE_MAX = 0x04,
+
+	/* func: 0xff00 */
+	MSTAT_FUNC_UNSPECIFIED = 0x00 << 8,
+	MSTAT_FUNC_IO = 0x01 << 8,
+	MSTAT_FUNC_TX_IO = 0x02 << 8,
+	MSTAT_FUNC_RX_IO = 0x03 << 8,
+	MSTAT_FUNC_TX = 0x04 << 8,
+	MSTAT_FUNC_RX = 0x05 << 8,
+	MSTAT_FUNC_CFG_VENDOR = 0x06 << 8,
+	MSTAT_FUNC_MAX = 0x07 << 8,
+};
+
+#define mstat_tf_idx(flags) ((flags) & 0xff)
+#define mstat_ff_idx(flags) (((flags) & 0xff00) >> 8)
+
+typedef enum mstat_status {
+	MSTAT_ALLOC_SUCCESS = 0,
+	MSTAT_ALLOC_FAIL,
+	MSTAT_FREE
+} MSTAT_STATUS;
+
+
+#ifdef DBG_MEM_ALLOC
+void rtw_mstat_update(const enum mstat_f flags, const MSTAT_STATUS status, u32 sz);
+void rtw_mstat_dump(void *sel);
+bool match_mstat_sniff_rules(const enum mstat_f flags, const size_t size);
+void *dbg_rtw_vmalloc(u32 sz, const enum mstat_f flags, const char *func, const int line);
+void *dbg_rtw_zvmalloc(u32 sz, const enum mstat_f flags, const char *func, const int line);
+void dbg_rtw_vmfree(void *pbuf, u32 sz, const enum mstat_f flags, const char *func, const int line);
+void *dbg_rtw_malloc(u32 sz, const enum mstat_f flags, const char *func, const int line);
+void *dbg_rtw_zmalloc(u32 sz, const enum mstat_f flags, const char *func, const int line);
+void dbg_rtw_mfree(void *pbuf, u32 sz, const enum mstat_f flags, const char *func, const int line);
+
+struct sk_buff *dbg_rtw_skb_alloc(unsigned int size, const enum mstat_f flags, const char *func, const int line);
+void dbg_rtw_skb_free(struct sk_buff *skb, const enum mstat_f flags, const char *func, const int line);
+struct sk_buff *dbg_rtw_skb_copy(const struct sk_buff *skb, const enum mstat_f flags, const char *func, const int line);
+struct sk_buff *dbg_rtw_skb_clone(struct sk_buff *skb, const enum mstat_f flags, const char *func, const int line);
+int dbg_rtw_skb_linearize(struct sk_buff *skb, const enum mstat_f flags, const char *func, int line);
+int dbg_rtw_netif_rx(_nic_hdl ndev, struct sk_buff *skb, const enum mstat_f flags, const char *func, int line);
+#ifdef CONFIG_RTW_NAPI
+int dbg_rtw_netif_receive_skb(_nic_hdl ndev, struct sk_buff *skb, const enum mstat_f flags, const char *func, int line);
+#ifdef CONFIG_RTW_GRO
+gro_result_t dbg_rtw_napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb, const enum mstat_f flags, const char *func, int line);
+#endif
+#endif /* CONFIG_RTW_NAPI */
+void dbg_rtw_skb_queue_purge(struct sk_buff_head *list, enum mstat_f flags, const char *func, int line);
+#ifdef CONFIG_USB_HCI
+void *dbg_rtw_usb_buffer_alloc(struct usb_device *dev, size_t size, dma_addr_t *dma, const enum mstat_f flags, const char *func, const int line);
+void dbg_rtw_usb_buffer_free(struct usb_device *dev, size_t size, void *addr, dma_addr_t dma, const enum mstat_f flags, const char *func, const int line);
+#endif /* CONFIG_USB_HCI */
+
+#ifdef CONFIG_USE_VMALLOC
+#define rtw_vmalloc(sz)			dbg_rtw_vmalloc((sz), MSTAT_TYPE_VIR, __FUNCTION__, __LINE__)
+#define rtw_zvmalloc(sz)			dbg_rtw_zvmalloc((sz), MSTAT_TYPE_VIR, __FUNCTION__, __LINE__)
+#define rtw_vmfree(pbuf, sz)			dbg_rtw_vmfree((pbuf), (sz), MSTAT_TYPE_VIR, __FUNCTION__, __LINE__)
+#define rtw_vmalloc_f(sz, mstat_f)		dbg_rtw_vmalloc((sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_VIR, __FUNCTION__, __LINE__)
+#define rtw_zvmalloc_f(sz, mstat_f)		dbg_rtw_zvmalloc((sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_VIR, __FUNCTION__, __LINE__)
+#define rtw_vmfree_f(pbuf, sz, mstat_f)	dbg_rtw_vmfree((pbuf), (sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_VIR, __FUNCTION__, __LINE__)
+#else /* CONFIG_USE_VMALLOC */
+#define rtw_vmalloc(sz)			dbg_rtw_malloc((sz), MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_zvmalloc(sz)			dbg_rtw_zmalloc((sz), MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_vmfree(pbuf, sz)			dbg_rtw_mfree((pbuf), (sz), MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_vmalloc_f(sz, mstat_f)		dbg_rtw_malloc((sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_zvmalloc_f(sz, mstat_f)		dbg_rtw_zmalloc((sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_vmfree_f(pbuf, sz, mstat_f)	dbg_rtw_mfree((pbuf), (sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#endif /* CONFIG_USE_VMALLOC */
+#define rtw_malloc(sz)				dbg_rtw_malloc((sz), MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_zmalloc(sz)			dbg_rtw_zmalloc((sz), MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_mfree(pbuf, sz)			dbg_rtw_mfree((pbuf), (sz), MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_malloc_f(sz, mstat_f)		dbg_rtw_malloc((sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_zmalloc_f(sz, mstat_f)		dbg_rtw_zmalloc((sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+#define rtw_mfree_f(pbuf, sz, mstat_f)	dbg_rtw_mfree((pbuf), (sz), ((mstat_f) & 0xff00) | MSTAT_TYPE_PHY, __FUNCTION__, __LINE__)
+
+#define rtw_skb_alloc(size)			dbg_rtw_skb_alloc((size), MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_free(skb)			dbg_rtw_skb_free((skb), MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_alloc_f(size, mstat_f)	dbg_rtw_skb_alloc((size), ((mstat_f) & 0xff00) | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_free_f(skb, mstat_f)		dbg_rtw_skb_free((skb), ((mstat_f) & 0xff00) | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_copy(skb)			dbg_rtw_skb_copy((skb), MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_clone(skb)			dbg_rtw_skb_clone((skb), MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_copy_f(skb, mstat_f)		dbg_rtw_skb_copy((skb), ((mstat_f) & 0xff00) | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_clone_f(skb, mstat_f)		dbg_rtw_skb_clone((skb), ((mstat_f) & 0xff00) | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#define rtw_skb_linearize(skb)			dbg_rtw_skb_linearize((skb), MSTAT_TYPE_SKB, __func__, __LINE__)
+#define rtw_netif_rx(ndev, skb)		dbg_rtw_netif_rx(ndev, skb, MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#ifdef CONFIG_RTW_NAPI
+#define rtw_netif_receive_skb(ndev, skb)	dbg_rtw_netif_receive_skb(ndev, skb, MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#ifdef CONFIG_RTW_GRO
+#define rtw_napi_gro_receive(napi, skb)	dbg_rtw_napi_gro_receive(napi, skb, MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#endif
+#endif /* CONFIG_RTW_NAPI */
+#define rtw_skb_queue_purge(sk_buff_head)	dbg_rtw_skb_queue_purge(sk_buff_head, MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#ifdef CONFIG_USB_HCI
+#define rtw_usb_buffer_alloc(dev, size, dma)		dbg_rtw_usb_buffer_alloc((dev), (size), (dma), MSTAT_TYPE_USB, __FUNCTION__, __LINE__)
+#define rtw_usb_buffer_free(dev, size, addr, dma)	dbg_rtw_usb_buffer_free((dev), (size), (addr), (dma), MSTAT_TYPE_USB, __FUNCTION__, __LINE__)
+#define rtw_usb_buffer_alloc_f(dev, size, dma, mstat_f)			dbg_rtw_usb_buffer_alloc((dev), (size), (dma), ((mstat_f) & 0xff00) | MSTAT_TYPE_USB, __FUNCTION__, __LINE__)
+#define rtw_usb_buffer_free_f(dev, size, addr, dma, mstat_f)	dbg_rtw_usb_buffer_free((dev), (size), (addr), (dma), ((mstat_f) & 0xff00) | MSTAT_TYPE_USB, __FUNCTION__, __LINE__)
+#endif /* CONFIG_USB_HCI */
+
+#else /* DBG_MEM_ALLOC */
+#define rtw_mstat_update(flag, status, sz) do {} while (0)
+#define rtw_mstat_dump(sel) do {} while (0)
+#define match_mstat_sniff_rules(flags, size) _FALSE
+void *_rtw_vmalloc(u32 sz);
+void *_rtw_zvmalloc(u32 sz);
+void _rtw_vmfree(void *pbuf, u32 sz);
+void *_rtw_zmalloc(u32 sz);
+void *_rtw_malloc(u32 sz);
+void _rtw_mfree(void *pbuf, u32 sz);
+
+struct sk_buff *_rtw_skb_alloc(u32 sz);
+void _rtw_skb_free(struct sk_buff *skb);
+#ifdef CONFIG_RTW_NAPI
+int _rtw_netif_receive_skb(_nic_hdl ndev, struct sk_buff *skb);
+#ifdef CONFIG_RTW_GRO
+gro_result_t _rtw_napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
+#endif
+#endif /* CONFIG_RTW_NAPI */
+void _rtw_skb_queue_purge(struct sk_buff_head *list);
+
+#ifdef CONFIG_USE_VMALLOC
+#define rtw_vmalloc(sz)			_rtw_vmalloc((sz))
+#define rtw_zvmalloc(sz)			_rtw_zvmalloc((sz))
+#define rtw_vmfree(pbuf, sz)			_rtw_vmfree((pbuf), (sz))
+#define rtw_vmalloc_f(sz, mstat_f)		_rtw_vmalloc((sz))
+#define rtw_zvmalloc_f(sz, mstat_f)		_rtw_zvmalloc((sz))
+#define rtw_vmfree_f(pbuf, sz, mstat_f)	_rtw_vmfree((pbuf), (sz))
+#else /* CONFIG_USE_VMALLOC */
+#define rtw_vmalloc(sz)			_rtw_malloc((sz))
+#define rtw_zvmalloc(sz)			_rtw_zmalloc((sz))
+#define rtw_vmfree(pbuf, sz)			_rtw_mfree((pbuf), (sz))
+#define rtw_vmalloc_f(sz, mstat_f)		_rtw_malloc((sz))
+#define rtw_zvmalloc_f(sz, mstat_f)		_rtw_zmalloc((sz))
+#define rtw_vmfree_f(pbuf, sz, mstat_f)	_rtw_mfree((pbuf), (sz))
+#endif /* CONFIG_USE_VMALLOC */
+#define rtw_malloc(sz)				_rtw_malloc((sz))
+#define rtw_zmalloc(sz)			_rtw_zmalloc((sz))
+#define rtw_mfree(pbuf, sz)			_rtw_mfree((pbuf), (sz))
+#define rtw_malloc_f(sz, mstat_f)		_rtw_malloc((sz))
+#define rtw_zmalloc_f(sz, mstat_f)		_rtw_zmalloc((sz))
+#define rtw_mfree_f(pbuf, sz, mstat_f)	_rtw_mfree((pbuf), (sz))
+
+#define rtw_skb_alloc(size) 			_rtw_skb_alloc((size))
+#define rtw_skb_free(skb) 			_rtw_skb_free((skb))
+#define rtw_skb_alloc_f(size, mstat_f)	_rtw_skb_alloc((size))
+#define rtw_skb_free_f(skb, mstat_f)		_rtw_skb_free((skb))
+#define rtw_skb_copy(skb)			_rtw_skb_copy((skb))
+#define rtw_skb_clone(skb)			_rtw_skb_clone((skb))
+#define rtw_skb_copy_f(skb, mstat_f)		_rtw_skb_copy((skb))
+#define rtw_skb_clone_f(skb, mstat_f)		_rtw_skb_clone((skb))
+#define rtw_skb_linearize(skb)			_rtw_skb_linearize(skb)
+#define rtw_netif_rx(ndev, skb)		_rtw_netif_rx(ndev, skb)
+#ifdef CONFIG_RTW_NAPI
+#define rtw_netif_receive_skb(ndev, skb) _rtw_netif_receive_skb(ndev, skb)
+#ifdef CONFIG_RTW_GRO
+#define rtw_napi_gro_receive(napi, skb) _rtw_napi_gro_receive(napi, skb)
+#endif
+#endif /* CONFIG_RTW_NAPI */
+#define rtw_skb_queue_purge(sk_buff_head) _rtw_skb_queue_purge(sk_buff_head)
+#ifdef CONFIG_USB_HCI
+#define rtw_usb_buffer_alloc(dev, size, dma) _rtw_usb_buffer_alloc((dev), (size), (dma))
+#define rtw_usb_buffer_free(dev, size, addr, dma) _rtw_usb_buffer_free((dev), (size), (addr), (dma))
+#define rtw_usb_buffer_alloc_f(dev, size, dma, mstat_f) _rtw_usb_buffer_alloc((dev), (size), (dma))
+#define rtw_usb_buffer_free_f(dev, size, addr, dma, mstat_f) _rtw_usb_buffer_free((dev), (size), (addr), (dma))
+#endif /* CONFIG_USB_HCI */
+#endif /* DBG_MEM_ALLOC */
+
+
+void *rtw_malloc2d(int h, int w, size_t size);
+void rtw_mfree2d(void *pbuf, int h, int w, int size);
+void _rtw_memcpy(void *dec, const void *sour, u32 sz);
+void _rtw_memmove(void *dst, const void *src, u32 sz);
+int _rtw_memcmp(const void *dst, const void *src, u32 sz);
+int _rtw_memcmp2(const void *dst, const void *src, u32 sz);
+void _rtw_memset(void *pbuf, int c, u32 sz);
+
+void _rtw_init_listhead(_list *list);
+u32 rtw_is_list_empty(_list *phead);
+void rtw_list_insert_head(_list *plist, _list *phead);
+void rtw_list_insert_tail(_list *plist, _list *phead);
+void rtw_list_splice(_list *list, _list *head);
+void rtw_list_splice_init(_list *list, _list *head);
+void rtw_list_splice_tail(_list *list, _list *head);
+
+void rtw_list_delete(_list *plist);
+
+void rtw_hlist_head_init(rtw_hlist_head *h);
+void rtw_hlist_add_head(rtw_hlist_node *n, rtw_hlist_head *h);
+void rtw_hlist_del(rtw_hlist_node *n);
+void rtw_hlist_add_head_rcu(rtw_hlist_node *n, rtw_hlist_head *h);
+void rtw_hlist_del_rcu(rtw_hlist_node *n);
+
+void _rtw_init_queue(_queue *pqueue);
+void _rtw_deinit_queue(_queue *pqueue);
+u32 _rtw_queue_empty(_queue	*pqueue);
+u32 rtw_end_of_queue_search(_list *queue, _list *pelement);
+
+systime _rtw_get_current_time(void);
+u32 _rtw_systime_to_us(systime stime);
+u32 _rtw_systime_to_ms(systime stime);
+systime _rtw_ms_to_systime(u32 ms);
+systime _rtw_us_to_systime(u32 us);
+s32 _rtw_get_passing_time_ms(systime start);
+s32 _rtw_get_remaining_time_ms(systime end);
+s32 _rtw_get_time_interval_ms(systime start, systime end);
+bool _rtw_time_after(systime a, systime b);
+bool _rtw_time_after_eq(systime a, systime b);
+
+#ifdef DBG_SYSTIME
+#define rtw_get_current_time() ({systime __stime = _rtw_get_current_time(); __stime;})
+#define rtw_systime_to_us(stime) ({u32 __us = _rtw_systime_to_us(stime); typecheck(systime, stime); __us;})
+#define rtw_systime_to_ms(stime) ({u32 __ms = _rtw_systime_to_ms(stime); typecheck(systime, stime); __ms;})
+#define rtw_ms_to_systime(ms) ({systime __stime = _rtw_ms_to_systime(ms); __stime;})
+#define rtw_us_to_systime(us) ({systime __stime = _rtw_us_to_systime(us); __stime;})
+#define rtw_get_passing_time_ms(start) ({u32 __ms = _rtw_get_passing_time_ms(start); typecheck(systime, start); __ms;})
+#define rtw_get_remaining_time_ms(end) ({u32 __ms = _rtw_get_remaining_time_ms(end); typecheck(systime, end); __ms;})
+#define rtw_get_time_interval_ms(start, end) ({u32 __ms = _rtw_get_time_interval_ms(start, end); typecheck(systime, start); typecheck(systime, end); __ms;})
+#define rtw_time_after(a, b) ({bool __r = _rtw_time_after(a, b); typecheck(systime, a); typecheck(systime, b); __r;})
+#define rtw_time_after_eq(a, b) ({bool __r = _rtw_time_after_eq(a, b); typecheck(systime, a); typecheck(systime, b); __r;})
+#define rtw_time_before(a, b) ({bool __r = _rtw_time_after(b, a); typecheck(systime, a); typecheck(systime, b); __r;})
+#define rtw_time_before_eq(a, b) ({bool __r = _rtw_time_after_eq(b, a); typecheck(systime, a); typecheck(systime, b); __r;})
+#else
+#define rtw_get_current_time() _rtw_get_current_time()
+#define rtw_systime_to_us(stime) _rtw_systime_to_us(stime)
+#define rtw_systime_to_ms(stime) _rtw_systime_to_ms(stime)
+#define rtw_ms_to_systime(ms) _rtw_ms_to_systime(ms)
+#define rtw_us_to_systime(us) _rtw_us_to_systime(us)
+#define rtw_get_passing_time_ms(start) _rtw_get_passing_time_ms(start)
+#define rtw_get_remaining_time_ms(end) _rtw_get_remaining_time_ms(end)
+#define rtw_get_time_interval_ms(start, end) _rtw_get_time_interval_ms(start, end)
+#define rtw_time_after(a, b) _rtw_time_after(a, b)
+#define rtw_time_after_eq(a, b) _rtw_time_after(a, b)
+#define rtw_time_before(a, b) _rtw_time_after(b, a)
+#define rtw_time_before_eq(a, b) _rtw_time_after_eq(b, a)
+#endif
+
+#ifndef CONFIG_OSDEP_SPTIME_API
+sysptime rtw_sptime_get(void);
+sysptime rtw_sptime_get_raw(void);
+sysptime rtw_sptime_set(s64 secs, const u32 nsecs);
+sysptime rtw_sptime_zero(void);
+
+int rtw_sptime_cmp(const sysptime cmp1, const sysptime cmp2);
+sysptime rtw_sptime_sub(const sysptime lhs, const sysptime rhs);
+sysptime rtw_sptime_add(const sysptime lhs, const sysptime rhs);
+
+s64 rtw_sptime_to_ms(const sysptime sptime);
+sysptime rtw_ms_to_sptime(u64 ms);
+s64 rtw_sptime_to_us(const sysptime sptime);
+sysptime rtw_us_to_sptime(u64 us);
+s64 rtw_sptime_to_ns(const sysptime sptime);
+sysptime rtw_ns_to_sptime(u64 ns);
+#endif /* !CONFIG_OSDEP_SPTIME_API */
+
+#define rtw_sptime_eql(cmp1, cmp2) (rtw_sptime_cmp(cmp1, cmp2) == 0)
+#define rtw_sptime_after(cmp1, cmp2) (rtw_sptime_cmp(cmp1, cmp2) > 0)
+#define rtw_sptime_before(cmp1, cmp2) (rtw_sptime_cmp(cmp1, cmp2) < 0)
+#define rtw_sptime_is_zero(sptime) (rtw_sptime_cmp(sptime, rtw_sptime_zero()) == 0)
+#define rtw_sptime_diff_ms(start, end) rtw_sptime_to_ms(rtw_sptime_sub(end, start))
+#define rtw_sptime_diff_us(start, end) rtw_sptime_to_us(rtw_sptime_sub(end, start))
+#define rtw_sptime_diff_ns(start, end) rtw_sptime_to_ns(rtw_sptime_sub(end, start))
+#define rtw_sptime_pass_ms(start) rtw_sptime_diff_ms(start, rtw_sptime_get())
+#define rtw_sptime_pass_us(start) rtw_sptime_diff_us(start, rtw_sptime_get())
+#define rtw_sptime_pass_ns(start) rtw_sptime_diff_ns(start, rtw_sptime_get())
+
+void rtw_sleep_schedulable(int ms);
+
+void rtw_msleep_os(int ms);
+void rtw_usleep_os(int us);
+
+u32 rtw_atoi(u8 *s);
+
+#ifdef DBG_DELAY_OS
+#define rtw_mdelay_os(ms) _rtw_mdelay_os((ms), __FUNCTION__, __LINE__)
+#define rtw_udelay_os(ms) _rtw_udelay_os((ms), __FUNCTION__, __LINE__)
+void _rtw_mdelay_os(int ms, const char *func, const int line);
+void _rtw_udelay_os(int us, const char *func, const int line);
+#else
+void rtw_mdelay_os(int ms);
+void rtw_udelay_os(int us);
+#endif
+
+void rtw_yield_os(void);
+
+void rtw_init_timer(_timer *ptimer, void *pfunc, void *ctx);
+__inline static unsigned char _cancel_timer_ex(_timer *ptimer)
+{
+	u8 bcancelled;
+
+	_cancel_timer(ptimer, &bcancelled);
+
+	return bcancelled;
+}
+
+__inline static void _cancel_timer_nowait(_timer *ptimer)
+{
+	_cancel_timer_async(ptimer);
+}
+
+#define _RND(sz, r) ((((sz)+((r)-1))/(r))*(r))
+#define RND4(x)	(((x >> 2) + (((x & 3) == 0) ? 0 : 1)) << 2)
+
+__inline static u32 _RND4(u32 sz)
+{
+
+	u32	val;
+
+	val = ((sz >> 2) + ((sz & 3) ? 1 : 0)) << 2;
+
+	return val;
+
+}
+
+__inline static u32 _RND8(u32 sz)
+{
+
+	u32	val;
+
+	val = ((sz >> 3) + ((sz & 7) ? 1 : 0)) << 3;
+
+	return val;
+
+}
+
+__inline static u32 _RND128(u32 sz)
+{
+
+	u32	val;
+
+	val = ((sz >> 7) + ((sz & 127) ? 1 : 0)) << 7;
+
+	return val;
+
+}
+
+__inline static u32 _RND256(u32 sz)
+{
+
+	u32	val;
+
+	val = ((sz >> 8) + ((sz & 255) ? 1 : 0)) << 8;
+
+	return val;
+
+}
+
+__inline static u32 _RND512(u32 sz)
+{
+
+	u32	val;
+
+	val = ((sz >> 9) + ((sz & 511) ? 1 : 0)) << 9;
+
+	return val;
+
+}
+
+__inline static u32 bitshift(u32 bitmask)
+{
+	u32 i;
+
+	for (i = 0; i <= 31; i++)
+		if (((bitmask >> i) &  0x1) == 1)
+			break;
+
+	return i;
+}
+
+static inline int largest_bit(u32 bitmask)
+{
+	int i;
+
+	for (i = 31; i >= 0; i--)
+		if (bitmask & BIT(i))
+			break;
+
+	return i;
+}
+
+static inline int largest_bit_64(u64 bitmask)
+{
+	int i;
+
+	for (i = 63; i >= 0; i--)
+		if (bitmask & BIT_ULL(i))
+			break;
+
+	return i;
+}
+
+#define rtw_abs(a) ((a) < 0 ? -(a) : (a))
+#define rtw_min(a, b) (((a) > (b)) ? (b) : (a))
+#define rtw_max(a, b) (((a) > (b)) ? (a) : (b))
+
+#define rtw_is_range_empty(hi, lo) ((hi) == (lo))
+#define rtw_is_range_a_in_b(a_hi, a_lo, b_hi, b_lo) (((a_hi) <= (b_hi)) && ((a_lo) >= (b_lo)))
+#define rtw_is_range_adjacent(a_hi, a_lo, b_hi, b_lo) (((a_hi) == (b_lo)) || ((a_lo) == (b_hi)))
+#define rtw_is_range_overlap(a_hi, a_lo, b_hi, b_lo) (((a_hi) > (b_lo)) && ((a_lo) < (b_hi)))
+
+/*
+* Combine two ranges if possible (hilo_s is empty or adjcent/overlap with hilo)
+* @hi_s, @lo_s: range parameters to store combined range
+* @hi, @lo: range parameters to be combined, if combined, set to 0 (empty)
+*/
+#define rtw_range_combine(hi_s, lo_s, hi, lo) \
+	do { \
+		if (rtw_is_range_empty(hi, lo)) {} \
+		else if (rtw_is_range_empty(hi_s, lo_s)) { \
+			(hi_s) = (hi); \
+			(lo_s) = (lo); \
+			(hi) = 0; (lo) = 0; \
+		} else if (rtw_is_range_adjacent(hi_s, lo_s, hi, lo) \
+			|| rtw_is_range_overlap(hi_s, lo_s, hi, lo) \
+		) { \
+			(hi_s) = rtw_max(hi_s, hi); \
+			(lo_s) = rtw_min(lo_s, lo); \
+			(hi) = 0; (lo) = 0; \
+		} \
+	} while (0)
+
+/*
+* Merge two ranges (no need to adjcent/overlap with each other)
+* @hi_s, @lo_s: range parameters to store merged range
+* @hi, @lo: range parameters to be merged
+*/
+#define rtw_range_merge(hi_s, lo_s, hi, lo) \
+	do { \
+		if (rtw_is_range_empty(hi, lo)) {} \
+		else if (rtw_is_range_empty(hi_s, lo_s)) { \
+			(hi_s) = (hi); \
+			(lo_s) = (lo); \
+		} else { \
+			(hi_s) = rtw_max(hi_s, hi); \
+			(lo_s) = rtw_min(lo_s, lo); \
+		} \
+	} while (0)
+
+#define MAC_FMT_LEN 18
+#ifdef CONFIG_RTW_HIDDEN_MAC_ADDR
+#ifndef MAC_FMT
+#define MAC_FMT "%02x:%02x:%02x:xx:xx:xx"
+#endif
+#ifndef MAC_ARG
+#define MAC_ARG(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2]
+#endif
+#else /* CONFIG_RTW_HIDDEN_MAC_ADDR */
+#ifndef MAC_FMT
+#define MAC_FMT "%02x:%02x:%02x:%02x:%02x:%02x"
+#endif
+#ifndef MAC_ARG
+#define MAC_ARG(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2], ((u8 *)(x))[3], ((u8 *)(x))[4], ((u8 *)(x))[5]
+#endif
+#endif /* CONFIG_RTW_HIDDEN_MAC_ADDR */
+
+#ifndef MAC_FMT_SEL
+#define MAC_FMT_SEL "%02x:%02x:%02x:%02x:%02x:%02x"
+#endif
+#ifndef MAC_ARG_SEL
+#define MAC_ARG_SEL(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2], ((u8 *)(x))[3], ((u8 *)(x))[4], ((u8 *)(x))[5]
+#endif
+
+bool rtw_macaddr_is_larger(const u8 *a, const u8 *b);
+
+void rtw_suspend_lock_init(void);
+void rtw_suspend_lock_uninit(void);
+void rtw_lock_suspend(void);
+void rtw_unlock_suspend(void);
+void rtw_lock_suspend_timeout(u32 timeout_ms);
+void rtw_lock_traffic_suspend_timeout(u32 timeout_ms);
+void rtw_resume_lock_suspend(void);
+void rtw_resume_unlock_suspend(void);
+#ifdef CONFIG_AP_WOWLAN
+void rtw_softap_lock_suspend(void);
+void rtw_softap_unlock_suspend(void);
+#endif
+
+void rtw_set_bit(int nr, unsigned long *addr);
+void rtw_clear_bit(int nr, unsigned long *addr);
+int rtw_test_and_clear_bit(int nr, unsigned long *addr);
+int rtw_test_and_set_bit(int nr, unsigned long *addr);
+
+/* File operation APIs, just for linux now */
+#if !defined(CONFIG_RTW_ANDROID_GKI)
+int rtw_is_dir_readable(const char *path);
+int rtw_store_to_file(const char *path, u8 *buf, u32 sz);
+#endif /* !defined(CONFIG_RTW_ANDROID_GKI) */
+int rtw_is_file_readable(const char *path);
+int rtw_is_file_readable_with_size(const char *path, u32 *sz);
+int rtw_readable_file_sz_chk(const char *path, u32 sz);
+int rtw_retrieve_from_file(const char *path, u8 *buf, u32 sz);
+void rtw_free_netdev(struct net_device *netdev);
+
+#ifndef CONFIG_OSDEP_DIV64_API
+u64 rtw_modular64(u64 x, u64 y);
+u64 rtw_division64(u64 x, u64 y);
+s64 rtw_division64_s64(s64 x, s64 y);
+#endif
+
+u32 rtw_random32(void);
+
+void rtw_wiphy_rfkill_set_hw_state(struct wiphy *wiphy, bool blocked);
+
+/* Macros for handling unaligned memory accesses */
+
+#define RTW_GET_BE16(a) ((u16) (((a)[0] << 8) | (a)[1]))
+#define RTW_PUT_BE16(a, val)			\
+	do {					\
+		(a)[0] = ((u16) (val)) >> 8;	\
+		(a)[1] = ((u16) (val)) & 0xff;	\
+	} while (0)
+
+#define RTW_GET_LE16(a) ((u16) (((a)[1] << 8) | (a)[0]))
+#define RTW_PUT_LE16(a, val)			\
+	do {					\
+		(a)[1] = ((u16) (val)) >> 8;	\
+		(a)[0] = ((u16) (val)) & 0xff;	\
+	} while (0)
+
+#define RTW_GET_BE24(a) ((((u32) (a)[0]) << 16) | (((u32) (a)[1]) << 8) | \
+			 ((u32) (a)[2]))
+#define RTW_PUT_BE24(a, val)					\
+	do {							\
+		(a)[0] = (u8) ((((u32) (val)) >> 16) & 0xff);	\
+		(a)[1] = (u8) ((((u32) (val)) >> 8) & 0xff);	\
+		(a)[2] = (u8) (((u32) (val)) & 0xff);		\
+	} while (0)
+
+#define RTW_GET_BE32(a) ((((u32) (a)[0]) << 24) | (((u32) (a)[1]) << 16) | \
+			 (((u32) (a)[2]) << 8) | ((u32) (a)[3]))
+#define RTW_PUT_BE32(a, val)					\
+	do {							\
+		(a)[0] = (u8) ((((u32) (val)) >> 24) & 0xff);	\
+		(a)[1] = (u8) ((((u32) (val)) >> 16) & 0xff);	\
+		(a)[2] = (u8) ((((u32) (val)) >> 8) & 0xff);	\
+		(a)[3] = (u8) (((u32) (val)) & 0xff);		\
+	} while (0)
+
+#define RTW_GET_LE32(a) ((((u32) (a)[3]) << 24) | (((u32) (a)[2]) << 16) | \
+			 (((u32) (a)[1]) << 8) | ((u32) (a)[0]))
+#define RTW_PUT_LE32(a, val)					\
+	do {							\
+		(a)[3] = (u8) ((((u32) (val)) >> 24) & 0xff);	\
+		(a)[2] = (u8) ((((u32) (val)) >> 16) & 0xff);	\
+		(a)[1] = (u8) ((((u32) (val)) >> 8) & 0xff);	\
+		(a)[0] = (u8) (((u32) (val)) & 0xff);		\
+	} while (0)
+
+#define RTW_GET_BE64(a) ((((u64) (a)[0]) << 56) | (((u64) (a)[1]) << 48) | \
+			 (((u64) (a)[2]) << 40) | (((u64) (a)[3]) << 32) | \
+			 (((u64) (a)[4]) << 24) | (((u64) (a)[5]) << 16) | \
+			 (((u64) (a)[6]) << 8) | ((u64) (a)[7]))
+#define RTW_PUT_BE64(a, val)				\
+	do {						\
+		(a)[0] = (u8) (((u64) (val)) >> 56);	\
+		(a)[1] = (u8) (((u64) (val)) >> 48);	\
+		(a)[2] = (u8) (((u64) (val)) >> 40);	\
+		(a)[3] = (u8) (((u64) (val)) >> 32);	\
+		(a)[4] = (u8) (((u64) (val)) >> 24);	\
+		(a)[5] = (u8) (((u64) (val)) >> 16);	\
+		(a)[6] = (u8) (((u64) (val)) >> 8);	\
+		(a)[7] = (u8) (((u64) (val)) & 0xff);	\
+	} while (0)
+
+#define RTW_GET_LE64(a) ((((u64) (a)[7]) << 56) | (((u64) (a)[6]) << 48) | \
+			 (((u64) (a)[5]) << 40) | (((u64) (a)[4]) << 32) | \
+			 (((u64) (a)[3]) << 24) | (((u64) (a)[2]) << 16) | \
+			 (((u64) (a)[1]) << 8) | ((u64) (a)[0]))
+#define RTW_PUT_LE64(a, val)					\
+	do {							\
+		(a)[7] = (u8) ((((u64) (val)) >> 56) & 0xff);	\
+		(a)[6] = (u8) ((((u64) (val)) >> 48) & 0xff);	\
+		(a)[5] = (u8) ((((u64) (val)) >> 40) & 0xff);	\
+		(a)[4] = (u8) ((((u64) (val)) >> 32) & 0xff);	\
+		(a)[3] = (u8) ((((u64) (val)) >> 24) & 0xff);	\
+		(a)[2] = (u8) ((((u64) (val)) >> 16) & 0xff);	\
+		(a)[1] = (u8) ((((u64) (val)) >> 8) & 0xff);	\
+		(a)[0] = (u8) (((u64) (val)) & 0xff);		\
+	} while (0)
+
+void rtw_buf_free(u8 **buf, u32 *buf_len);
+void rtw_buf_update(u8 **buf, u32 *buf_len, const u8 *src, u32 src_len);
+
+struct rtw_cbuf {
+	u32 write;
+	u32 read;
+	u32 size;
+	void *bufs[0];
+};
+
+bool rtw_cbuf_full(struct rtw_cbuf *cbuf);
+bool rtw_cbuf_empty(struct rtw_cbuf *cbuf);
+bool rtw_cbuf_push(struct rtw_cbuf *cbuf, void *buf);
+void *rtw_cbuf_pop(struct rtw_cbuf *cbuf);
+struct rtw_cbuf *rtw_cbuf_alloc(u32 size);
+void rtw_cbuf_free(struct rtw_cbuf *cbuf);
+
+struct map_seg_t {
+	u16 sa;
+	u16 len;
+	u8 *c;
+};
+
+struct map_t {
+	u16 len;
+	u16 seg_num;
+	u8 init_value;
+	struct map_seg_t *segs;
+};
+
+#define MAPSEG_ARRAY_ENT(_sa, _len, _c, arg...) \
+	{ .sa = _sa, .len = _len, .c = (u8[_len]){ _c, ##arg}, }
+
+#define MAPSEG_PTR_ENT(_sa, _len, _p) \
+	{ .sa = _sa, .len = _len, .c = _p, }
+
+#define MAP_ENT(_len, _seg_num, _init_v, _seg, arg...) \
+	{ .len = _len, .seg_num = _seg_num, .init_value = _init_v, .segs = (struct map_seg_t[_seg_num]){ _seg, ##arg}, }
+
+int map_readN(const struct map_t *map, u16 offset, u16 len, u8 *buf);
+u8 map_read8(const struct map_t *map, u16 offset);
+
+struct blacklist_ent {
+	_list list;
+	u8 addr[ETH_ALEN];
+	systime exp_time;
+};
+
+int rtw_blacklist_add(_queue *blist, const u8 *addr, u32 timeout_ms);
+int rtw_blacklist_del(_queue *blist, const u8 *addr);
+int rtw_blacklist_search(_queue *blist, const u8 *addr);
+void rtw_blacklist_flush(_queue *blist);
+void dump_blacklist(void *sel, _queue *blist, const char *title);
+
+/* String handler */
+
+BOOLEAN is_null(char c);
+BOOLEAN is_all_null(char *c, int len);
+BOOLEAN is_eol(char c);
+BOOLEAN is_space(char c);
+BOOLEAN is_decimal(char chTmp);
+BOOLEAN IsHexDigit(char chTmp);
+BOOLEAN is_alpha(char chTmp);
+char alpha_to_upper(char c);
+
+int hex2num_i(char c);
+int hex2byte_i(const char *hex);
+int hexstr2bin(const char *hex, u8 *buf, size_t len);
+
+/*
+* ustrs
+* str_0      str_1      str_2      str_3
+* |          |          |          |          |
+* |---------------- ustrs_len ----------------|
+*/
+#define ustrs_for_each_str(ustrs, ustrs_len, str) \
+	for ((str) = (ustrs); (str) < (ustrs) + (ustrs_len); (str) += strlen(str) + 1)
+
+void ustrs_add(char **ustrs, int *ustrs_len, const char *str);
+
+/*
+ * Write formatted output to sized buffer
+ */
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service_bsd.h b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service_bsd.h
new file mode 100644
index 000000000000..adee6f3a8062
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service_bsd.h
@@ -0,0 +1,1044 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __OSDEP_BSD_SERVICE_H_
+#define __OSDEP_BSD_SERVICE_H_
+
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/param.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/mbuf.h>
+#include <sys/kernel.h>
+#include <sys/socket.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kdb.h>
+#include <sys/kthread.h>
+#include <sys/malloc.h>
+#include <sys/time.h>
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/rman.h>
+
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_arp.h>
+#include <net/ethernet.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+#include <net/route.h>
+
+
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/in_var.h>
+#include <netinet/if_ether.h>
+#include <if_ether.h>
+
+#include <net80211/ieee80211_var.h>
+#include <net80211/ieee80211_regdomain.h>
+#include <net80211/ieee80211_radiotap.h>
+#include <net80211/ieee80211_ratectl.h>
+
+#include <dev/usb/usb.h>
+#include <dev/usb/usbdi.h>
+#include "usbdevs.h"
+
+#define	USB_DEBUG_VAR rum_debug
+#include <dev/usb/usb_debug.h>
+
+#if 1 //Baron porting from linux, it's all temp solution, needs to check again
+#include <sys/sema.h>
+#include <sys/pcpu.h> /* XXX for PCPU_GET */
+//	typedef struct 	semaphore _sema;
+	typedef struct 	sema _sema;
+//	typedef	spinlock_t	_lock;
+	typedef	struct mtx	_lock;
+	typedef struct mtx 		_mutex;
+	typedef struct rtw_timer_list _timer;
+	struct list_head {
+	struct list_head *next, *prev;
+	};
+	struct	__queue	{
+		struct	list_head	queue;	
+		_lock	lock;
+	};
+
+	typedef struct mbuf	_buffer;
+	
+	typedef struct	__queue	_queue;
+	typedef struct	list_head	_list;
+
+	typedef	struct	ifnet * _nic_hdl;
+	
+	typedef pid_t		_thread_hdl_;
+//	typedef struct thread		_thread_hdl_;
+	typedef void		thread_return;
+	typedef void*	thread_context;
+
+	typedef void timer_hdl_return;
+	typedef void* timer_hdl_context;
+	typedef struct work_struct _workitem;
+	typedef struct task _tasklet;
+
+#define   KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))
+/* emulate a modern version */
+#define LINUX_VERSION_CODE KERNEL_VERSION(2, 6, 35)
+
+#define WIRELESS_EXT -1
+#define HZ hz
+
+//#define IFT_RTW	0xf9 //ifnet allocate type for RTW
+#define free_netdev if_free
+#define LIST_CONTAINOR(ptr, type, member) \
+        ((type *)((char *)(ptr)-(SIZE_T)(&((type *)0)->member)))
+#define container_of(p,t,n) (t*)((p)-&(((t*)0)->n))
+
+/*lock - spinlock*/
+static inline void _rtw_spinlock_init(_lock *plock)
+{
+	mtx_init(plock, "", NULL, MTX_DEF | MTX_RECURSE);
+}
+
+static inline void _rtw_spinlock_free(_lock *plock)
+{
+	mtx_destroy(plock);
+}
+
+static inline void _rtw_spinlock(_lock	*plock)
+{
+	mtx_lock(plock);
+}
+
+static inline void _rtw_spinunlock(_lock *plock)
+{
+	mtx_unlock(plock);
+}
+
+#if 0
+static inline void _rtw_spinlock_ex(_lock *plock)
+{
+	mtx_lock(plock);
+}
+
+static inline void _rtw_spinunlock_ex(_lock *plock)
+{
+	mtx_unlock(plock);
+}
+#endif
+
+__inline static void _rtw_spinlock_irq(_lock *plock, unsigned long *flags)
+{
+	mtx_lock(plock); /*{local_irq_save((x)); mtx_lock_spin((lock));}*/
+}
+
+__inline static void _rtw_spinunlock_irq(_lock *plock, unsigned long *flags)
+{
+	mtx_unlock(plock);
+}
+
+__inline static void _rtw_spinlock_bh(_lock *plock)
+{
+	mtx_lock(plock);/*{local_irq_save((x)); mtx_lock_spin((lock));}*/
+}
+
+__inline static void _rtw_spinunlock_bh(_lock *plock)
+{
+	mtx_unlock(plock);
+}
+
+/*lock - semaphore*/
+static inline void _rtw_init_sema(_sema *sema, int init_val)
+{
+	sema_init(sema, init_val, "rtw_drv");
+}
+
+static inline void _rtw_free_sema(_sema *sema)
+{
+	sema_destroy(sema);
+}
+
+static inline void _rtw_up_sema(_sema *sema)
+{
+	sema_post(sema);
+}
+
+static inline u32 _rtw_down_sema(_sema *sema)
+{
+	sema_wait(sema);
+	return  _SUCCESS;
+}
+
+/*lock - mutex*/
+static inline void _rtw_mutex_init(_mutex *pmutex)
+{
+	mtx_init(pmutex, "", NULL, MTX_DEF | MTX_RECURSE);
+}
+
+static inline void _rtw_mutex_free(_mutex *pmutex)
+{
+	sema_destroy(pmutex);
+}
+
+__inline static void _rtw_mutex_lock_interruptible(_mutex *pmutex)
+{
+
+	mtx_lock(pmutex);
+
+}
+__inline static void _rtw_mutex_lock(_mutex *pmutex)
+{
+
+	mtx_lock(pmutex);
+
+}
+
+__inline static void _rtw_mutex_unlock(_mutex *pmutex)
+{
+	mtx_unlock(pmutex);
+}
+
+static inline void *_rtw_vmalloc(u32 sz)
+{
+	void *pbuf;
+	pbuf = malloc(sz, M_DEVBUF, M_NOWAIT);
+
+	return pbuf;
+}
+
+static inline void *_rtw_zvmalloc(u32 sz)
+{
+	void *pbuf;
+
+	pbuf = malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT);
+	return pbuf;
+}
+
+static inline void _rtw_vmfree(void *pbuf, u32 sz)
+{
+	free(pbuf, M_DEVBUF);
+}
+
+static inline void *_rtw_malloc(u32 sz)
+{
+	void *pbuf = NULL;
+
+	pbuf = malloc(sz, M_DEVBUF, M_NOWAIT);
+	return pbuf;
+}
+static inline void *_rtw_zmalloc(u32 sz)
+{
+	return malloc(sz, M_DEVBUF, M_ZERO | M_NOWAIT);
+}
+static inline void _rtw_mfree(void *pbuf, u32 sz)
+{
+	free(pbuf, M_DEVBUF);
+}
+#ifdef CONFIG_USB_HCI
+static inline void *_rtw_usb_buffer_alloc(struct usb_device *dev, size_t size, dma_addr_t *dma)
+{
+	return malloc(size, M_USBDEV, M_NOWAIT | M_ZERO);
+}
+static inline void _rtw_usb_buffer_free(struct usb_device *dev, size_t size, void *addr, dma_addr_t dma)
+{
+	free(addr, M_USBDEV);
+}
+#endif /* CONFIG_USB_HCI */
+
+struct sk_buff *_rtw_skb_alloc(u32 sz);
+void _rtw_skb_free(struct sk_buff *skb);
+static inline struct sk_buff *_rtw_skb_copy(const struct sk_buff *skb)
+{
+	return NULL;
+}
+
+static inline struct sk_buff *_rtw_skb_clone(struct sk_buff *skb)
+{
+	return skb_clone(skb);
+}
+
+static inline int _rtw_netif_rx(_nic_hdl ndev, struct sk_buff *skb)
+{
+	return (*ndev->if_input)(ndev, skb);
+}
+
+#ifdef CONFIG_RTW_NAPI
+static inline int _rtw_netif_receive_skb(_nic_hdl ndev, struct sk_buff *skb)
+{
+	rtw_warn_on(1);
+	return -1;
+}
+
+#ifdef CONFIG_RTW_GRO
+static inline gro_result_t _rtw_napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+{
+	rtw_warn_on(1);
+	return -1;
+}
+#endif /* CONFIG_RTW_GRO */
+#endif /* CONFIG_RTW_NAPI */
+
+
+/* 
+ * Linux timers are emulated using FreeBSD callout functions
+ * (and taskqueue functionality).
+ *
+ * Currently no timer stats functionality.
+ *
+ * See (linux_compat) processes.c
+ *
+ */
+struct rtw_timer_list {
+	struct callout callout;
+	void (*function)(void *);
+	void *arg;
+};
+
+struct workqueue_struct;
+struct work_struct;
+typedef void (*work_func_t)(struct work_struct *work);
+/* Values for the state of an item of work (work_struct) */
+typedef enum work_state {
+        WORK_STATE_UNSET = 0,
+        WORK_STATE_CALLOUT_PENDING = 1,
+        WORK_STATE_TASK_PENDING = 2,
+        WORK_STATE_WORK_CANCELLED = 3        
+} work_state_t;
+
+struct work_struct {
+        struct task task; /* FreeBSD task */
+        work_state_t state; /* the pending or otherwise state of work. */
+        work_func_t func;       
+};
+
+//modify private structure to match freebsd
+#define BITS_PER_LONG 32
+union ktime {
+	s64	tv64;
+#if BITS_PER_LONG != 64 && !defined(CONFIG_KTIME_SCALAR)
+	struct {
+#ifdef __BIG_ENDIAN
+	s32	sec, nsec;
+#else
+	s32	nsec, sec;
+#endif
+	} tv;
+#endif
+};
+#define kmemcheck_bitfield_begin(name)
+#define kmemcheck_bitfield_end(name)
+#define CHECKSUM_NONE 0
+typedef unsigned char *sk_buff_data_t;
+typedef union ktime ktime_t;		/* Kill this */
+
+void rtw_mtx_lock(_lock *plock);
+	
+void rtw_mtx_unlock(_lock *plock);
+
+/** 
+ *	struct sk_buff - socket buffer
+ *	@next: Next buffer in list
+ *	@prev: Previous buffer in list
+ *	@sk: Socket we are owned by
+ *	@tstamp: Time we arrived
+ *	@dev: Device we arrived on/are leaving by
+ *	@transport_header: Transport layer header
+ *	@network_header: Network layer header
+ *	@mac_header: Link layer header
+ *	@_skb_refdst: destination entry (with norefcount bit)
+ *	@sp: the security path, used for xfrm
+ *	@cb: Control buffer. Free for use by every layer. Put private vars here
+ *	@len: Length of actual data
+ *	@data_len: Data length
+ *	@mac_len: Length of link layer header
+ *	@hdr_len: writable header length of cloned skb
+ *	@csum: Checksum (must include start/offset pair)
+ *	@csum_start: Offset from skb->head where checksumming should start
+ *	@csum_offset: Offset from csum_start where checksum should be stored
+ *	@local_df: allow local fragmentation
+ *	@cloned: Head may be cloned (check refcnt to be sure)
+ *	@nohdr: Payload reference only, must not modify header
+ *	@pkt_type: Packet class
+ *	@fclone: skbuff clone status
+ *	@ip_summed: Driver fed us an IP checksum
+ *	@priority: Packet queueing priority
+ *	@users: User count - see {datagram,tcp}.c
+ *	@protocol: Packet protocol from driver
+ *	@truesize: Buffer size 
+ *	@head: Head of buffer
+ *	@data: Data head pointer
+ *	@tail: Tail pointer
+ *	@end: End pointer
+ *	@destructor: Destruct function
+ *	@mark: Generic packet mark
+ *	@nfct: Associated connection, if any
+ *	@ipvs_property: skbuff is owned by ipvs
+ *	@peeked: this packet has been seen already, so stats have been
+ *		done for it, don't do them again
+ *	@nf_trace: netfilter packet trace flag
+ *	@nfctinfo: Relationship of this skb to the connection
+ *	@nfct_reasm: netfilter conntrack re-assembly pointer
+ *	@nf_bridge: Saved data about a bridged frame - see br_netfilter.c
+ *	@skb_iif: ifindex of device we arrived on
+ *	@rxhash: the packet hash computed on receive
+ *	@queue_mapping: Queue mapping for multiqueue devices
+ *	@tc_index: Traffic control index
+ *	@tc_verd: traffic control verdict
+ *	@ndisc_nodetype: router type (from link layer)
+ *	@dma_cookie: a cookie to one of several possible DMA operations
+ *		done by skb DMA functions
+ *	@secmark: security marking
+ *	@vlan_tci: vlan tag control information
+ */
+
+struct sk_buff {
+	/* These two members must be first. */
+	struct sk_buff		*next;
+	struct sk_buff		*prev;
+
+	ktime_t			tstamp;
+
+	struct sock		*sk;
+	//struct net_device	*dev;
+	struct ifnet *dev;
+
+	/*
+	 * This is the control buffer. It is free to use for every
+	 * layer. Please put your private variables there. If you
+	 * want to keep them across layers you have to do a skb_clone()
+	 * first. This is owned by whoever has the skb queued ATM.
+	 */
+	char			cb[48] __aligned(8);
+
+	unsigned long		_skb_refdst;
+#ifdef CONFIG_XFRM
+	struct	sec_path	*sp;
+#endif
+	unsigned int		len,
+				data_len;
+	u16			mac_len,
+				hdr_len;
+	union {
+		u32		csum;
+		struct {
+			u16	csum_start;
+			u16	csum_offset;
+		}smbol2;
+	}smbol1;
+	u32			priority;
+	kmemcheck_bitfield_begin(flags1);
+	u8			local_df:1,
+				cloned:1,
+				ip_summed:2,
+				nohdr:1,
+				nfctinfo:3;
+	u8			pkt_type:3,
+				fclone:2,
+				ipvs_property:1,
+				peeked:1,
+				nf_trace:1;
+	kmemcheck_bitfield_end(flags1);
+	u16			protocol;
+
+	void			(*destructor)(struct sk_buff *skb);
+#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
+	struct nf_conntrack	*nfct;
+	struct sk_buff		*nfct_reasm;
+#endif
+#ifdef CONFIG_BRIDGE_NETFILTER
+	struct nf_bridge_info	*nf_bridge;
+#endif
+
+	int			skb_iif;
+#ifdef CONFIG_NET_SCHED
+	u16			tc_index;	/* traffic control index */
+#ifdef CONFIG_NET_CLS_ACT
+	u16			tc_verd;	/* traffic control verdict */
+#endif
+#endif
+
+	u32			rxhash;
+
+	kmemcheck_bitfield_begin(flags2);
+	u16			queue_mapping:16;
+#ifdef CONFIG_IPV6_NDISC_NODETYPE
+	u8			ndisc_nodetype:2,
+				deliver_no_wcard:1;
+#else
+	u8			deliver_no_wcard:1;
+#endif
+	kmemcheck_bitfield_end(flags2);
+
+	/* 0/14 bit hole */
+
+#ifdef CONFIG_NET_DMA
+	dma_cookie_t		dma_cookie;
+#endif
+#ifdef CONFIG_NETWORK_SECMARK
+	u32			secmark;
+#endif
+	union {
+		u32		mark;
+		u32		dropcount;
+	}symbol3;
+
+	u16			vlan_tci;
+
+	sk_buff_data_t		transport_header;
+	sk_buff_data_t		network_header;
+	sk_buff_data_t		mac_header;
+	/* These elements must be at the end, see alloc_skb() for details.  */
+	sk_buff_data_t		tail;
+	sk_buff_data_t		end;
+	unsigned char		*head,
+				*data;
+	unsigned int		truesize;
+	ATOMIC_T		users;
+};
+struct sk_buff_head {
+	/* These two members must be first. */
+	struct sk_buff	*next;
+	struct sk_buff	*prev;
+
+	u32		qlen;
+	_lock	lock;
+};
+#define skb_tail_pointer(skb)	skb->tail
+static inline unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
+{
+	unsigned char *tmp = skb_tail_pointer(skb);
+	//SKB_LINEAR_ASSERT(skb);
+	skb->tail += len;
+	skb->len  += len;
+	return tmp;
+}
+
+static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
+{
+	skb->len -= len;
+	if(skb->len < skb->data_len)
+		printf("%s(),%d,error!\n",__FUNCTION__,__LINE__);
+	return skb->data += len;
+}
+static inline unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
+{
+	return __skb_pull(skb, len);
+}
+static inline u32 skb_queue_len(const struct sk_buff_head *list_)
+{
+	return list_->qlen;
+}
+static inline void __skb_insert(struct sk_buff *newsk,
+				struct sk_buff *prev, struct sk_buff *next,
+				struct sk_buff_head *list)
+{
+	newsk->next = next;
+	newsk->prev = prev;
+	next->prev  = prev->next = newsk;
+	list->qlen++;
+}
+static inline void __skb_queue_before(struct sk_buff_head *list,
+				      struct sk_buff *next,
+				      struct sk_buff *newsk)
+{
+	__skb_insert(newsk, next->prev, next, list);
+}
+static inline void skb_queue_tail(struct sk_buff_head *list,
+				   struct sk_buff *newsk)
+{
+	mtx_lock(&list->lock);
+	__skb_queue_before(list, (struct sk_buff *)list, newsk);
+	mtx_unlock(&list->lock);
+}
+static inline struct sk_buff *skb_peek(struct sk_buff_head *list_)
+{
+	struct sk_buff *list = ((struct sk_buff *)list_)->next;
+	if (list == (struct sk_buff *)list_)
+		list = NULL;
+	return list;
+}
+static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
+{
+	struct sk_buff *next, *prev;
+
+	list->qlen--;
+	next	   = skb->next;
+	prev	   = skb->prev;
+	skb->next  = skb->prev = NULL;
+	next->prev = prev;
+	prev->next = next;
+}
+
+static inline struct sk_buff *skb_dequeue(struct sk_buff_head *list)
+{
+	mtx_lock(&list->lock);
+
+	struct sk_buff *skb = skb_peek(list);
+	if (skb)
+		__skb_unlink(skb, list);
+
+	mtx_unlock(&list->lock);
+
+	return skb;
+}
+static inline void skb_reserve(struct sk_buff *skb, int len)
+{
+	skb->data += len;
+	skb->tail += len;
+}
+static inline void __skb_queue_head_init(struct sk_buff_head *list)
+{
+	list->prev = list->next = (struct sk_buff *)list;
+	list->qlen = 0;
+}
+/*
+ * This function creates a split out lock class for each invocation;
+ * this is needed for now since a whole lot of users of the skb-queue
+ * infrastructure in drivers have different locking usage (in hardirq)
+ * than the networking core (in softirq only). In the long run either the
+ * network layer or drivers should need annotation to consolidate the
+ * main types of usage into 3 classes.
+ */
+static inline void skb_queue_head_init(struct sk_buff_head *list)
+{
+	_rtw_spinlock_init(&list->lock);
+	__skb_queue_head_init(list);
+}
+static inline u8 *rtw_skb_data(struct sk_buff *pkt)
+{
+	return pkt->data;
+}
+
+static inline u32 rtw_skb_len(struct sk_buff *pkt)
+{
+	return pkt->len;
+}
+
+unsigned long copy_from_user(void *to, const void *from, unsigned long n);
+unsigned long copy_to_user(void *to, const void *from, unsigned long n);
+struct sk_buff * dev_alloc_skb(unsigned int size);
+struct sk_buff *skb_clone(const struct sk_buff *skb);
+void dev_kfree_skb_any(struct sk_buff *skb);
+#endif //Baron porting from linux, it's all temp solution, needs to check again
+
+
+#if 1 // kenny add Linux compatibility code for Linux USB driver
+#include <dev/usb/usb_compat_linux.h>
+
+#define __init		// __attribute ((constructor))
+#define __exit		// __attribute ((destructor))
+
+/*
+ * Definitions for module_init and module_exit macros.
+ *
+ * These macros will use the SYSINIT framework to call a specified
+ * function (with no arguments) on module loading or unloading.
+ * 
+ */
+
+void module_init_exit_wrapper(void *arg);
+
+#define module_init(initfn)                             \
+        SYSINIT(mod_init_ ## initfn,                    \
+                SI_SUB_KLD, SI_ORDER_FIRST,             \
+                module_init_exit_wrapper, initfn)
+
+#define module_exit(exitfn)                             \
+        SYSUNINIT(mod_exit_ ## exitfn,                  \
+                  SI_SUB_KLD, SI_ORDER_ANY,             \
+                  module_init_exit_wrapper, exitfn)
+
+/*
+ * The usb_register and usb_deregister functions are used to register
+ * usb drivers with the usb subsystem. 
+ */
+int usb_register(struct usb_driver *driver);
+int usb_deregister(struct usb_driver *driver);
+
+/*
+ * usb_get_dev and usb_put_dev - increment/decrement the reference count 
+ * of the usb device structure.
+ *
+ * Original body of usb_get_dev:
+ *
+ *       if (dev)
+ *               get_device(&dev->dev);
+ *       return dev;
+ *
+ * Reference counts are not currently used in this compatibility
+ * layer. So these functions will do nothing.
+ */
+static inline struct usb_device *
+usb_get_dev(struct usb_device *dev)
+{
+        return dev;
+}
+
+static inline void 
+usb_put_dev(struct usb_device *dev)
+{
+        return;
+}
+
+
+// rtw_usb_compat_linux
+int rtw_usb_submit_urb(struct urb *urb, uint16_t mem_flags);
+int rtw_usb_unlink_urb(struct urb *urb);
+int rtw_usb_clear_halt(struct usb_device *dev, struct usb_host_endpoint *uhe);
+int rtw_usb_control_msg(struct usb_device *dev, struct usb_host_endpoint *uhe,
+    uint8_t request, uint8_t requesttype,
+    uint16_t value, uint16_t index, void *data,
+    uint16_t size, usb_timeout_t timeout);
+int rtw_usb_set_interface(struct usb_device *dev, uint8_t iface_no, uint8_t alt_index);
+int rtw_usb_setup_endpoint(struct usb_device *dev,
+    struct usb_host_endpoint *uhe, usb_size_t bufsize);
+struct urb *rtw_usb_alloc_urb(uint16_t iso_packets, uint16_t mem_flags);
+struct usb_host_endpoint *rtw_usb_find_host_endpoint(struct usb_device *dev, uint8_t type, uint8_t ep);
+struct usb_host_interface *rtw_usb_altnum_to_altsetting(const struct usb_interface *intf, uint8_t alt_index);
+struct usb_interface *rtw_usb_ifnum_to_if(struct usb_device *dev, uint8_t iface_no);
+void *rtw_usb_get_intfdata(struct usb_interface *intf);
+void rtw_usb_linux_register(void *arg);
+void rtw_usb_linux_deregister(void *arg);
+void rtw_usb_linux_free_device(struct usb_device *dev);
+void rtw_usb_free_urb(struct urb *urb);
+void rtw_usb_init_urb(struct urb *urb);
+void rtw_usb_kill_urb(struct urb *urb);
+void rtw_usb_set_intfdata(struct usb_interface *intf, void *data);
+void rtw_usb_fill_bulk_urb(struct urb *urb, struct usb_device *udev,
+    struct usb_host_endpoint *uhe, void *buf,
+    int length, usb_complete_t callback, void *arg);
+int rtw_usb_bulk_msg(struct usb_device *udev, struct usb_host_endpoint *uhe,
+    void *data, int len, uint16_t *pactlen, usb_timeout_t timeout);
+void *usb_get_intfdata(struct usb_interface *intf);
+int usb_linux_init_endpoints(struct usb_device *udev);
+
+
+
+typedef struct urb *  PURB;
+
+typedef unsigned gfp_t;
+#define __GFP_WAIT      ((gfp_t)0x10u)  /* Can wait and reschedule? */
+#define __GFP_HIGH      ((gfp_t)0x20u)  /* Should access emergency pools? */
+#define __GFP_IO        ((gfp_t)0x40u)  /* Can start physical IO? */
+#define __GFP_FS        ((gfp_t)0x80u)  /* Can call down to low-level FS? */
+#define __GFP_COLD      ((gfp_t)0x100u) /* Cache-cold page required */
+#define __GFP_NOWARN    ((gfp_t)0x200u) /* Suppress page allocation failure warning */
+#define __GFP_REPEAT    ((gfp_t)0x400u) /* Retry the allocation.  Might fail */
+#define __GFP_NOFAIL    ((gfp_t)0x800u) /* Retry for ever.  Cannot fail */
+#define __GFP_NORETRY   ((gfp_t)0x1000u)/* Do not retry.  Might fail */
+#define __GFP_NO_GROW   ((gfp_t)0x2000u)/* Slab internal usage */
+#define __GFP_COMP      ((gfp_t)0x4000u)/* Add compound page metadata */
+#define __GFP_ZERO      ((gfp_t)0x8000u)/* Return zeroed page on success */
+#define __GFP_NOMEMALLOC ((gfp_t)0x10000u) /* Don't use emergency reserves */
+#define __GFP_HARDWALL   ((gfp_t)0x20000u) /* Enforce hardwall cpuset memory allocs */
+
+/* This equals 0, but use constants in case they ever change */
+#define GFP_NOWAIT      (GFP_ATOMIC & ~__GFP_HIGH)
+/* GFP_ATOMIC means both !wait (__GFP_WAIT not set) and use emergency pool */
+#define GFP_ATOMIC      (__GFP_HIGH)
+#define GFP_NOIO        (__GFP_WAIT)
+#define GFP_NOFS        (__GFP_WAIT | __GFP_IO)
+#define GFP_KERNEL      (__GFP_WAIT | __GFP_IO | __GFP_FS)
+#define GFP_USER        (__GFP_WAIT | __GFP_IO | __GFP_FS | __GFP_HARDWALL)
+#define GFP_HIGHUSER    (__GFP_WAIT | __GFP_IO | __GFP_FS | __GFP_HARDWALL | \
+                         __GFP_HIGHMEM)
+
+
+#endif // kenny add Linux compatibility code for Linux USB
+
+__inline static _list *get_next(_list	*list)
+{
+	return list->next;
+}	
+
+__inline static _list	*get_list_head(_queue	*queue)
+{
+	return (&(queue->queue));
+}
+
+	
+#define LIST_CONTAINOR(ptr, type, member) \
+        ((type *)((char *)(ptr)-(SIZE_T)(&((type *)0)->member)))	
+
+        
+static inline void __list_del(struct list_head * prev, struct list_head * next)
+{
+	next->prev = prev;
+	prev->next = next;
+}
+static inline void INIT_LIST_HEAD(struct list_head *list)
+{
+	list->next = list;
+	list->prev = list;
+}
+__inline static void rtw_list_delete(_list *plist)
+{
+	__list_del(plist->prev, plist->next);
+	INIT_LIST_HEAD(plist);
+}
+
+static inline void timer_hdl(void *ctx)
+{
+	_timer *timer = (_timer *)ctx;
+
+	rtw_mtx_lock(NULL);
+	if (callout_pending(&timer->callout)) {
+		/* callout was reset */
+		rtw_mtx_unlock(NULL);
+		return;
+	}
+
+	if (!callout_active(&timer->callout)) {
+		/* callout was stopped */
+		rtw_mtx_unlock(NULL);
+		return;
+	}
+
+	callout_deactivate(&timer->callout);
+
+	timer->function(timer->arg);
+
+	rtw_mtx_unlock(NULL);
+}
+
+static inline void _init_timer(_timer *ptimer, void *pfunc, void *cntx)
+{
+	ptimer->function = pfunc;
+	ptimer->arg = cntx;
+	callout_init(&ptimer->callout, CALLOUT_MPSAFE);
+}
+
+__inline static int _check_timer_is_active(_timer *ptimer)
+{
+	rtw_mtx_lock(NULL);
+	if (callout_pending(&ptimer->callout)) {
+		/* callout was reset */
+		rtw_mtx_unlock(NULL);
+		return _TRUE;
+	}
+	rtw_mtx_unlock(NULL);
+		return _FALSE;
+}
+
+__inline static void _set_timer(_timer *ptimer,u32 delay_time)
+{	
+	if (ptimer->function && ptimer->arg) {
+		rtw_mtx_lock(NULL);
+		callout_reset(&ptimer->callout, delay_time, timer_hdl, ptimer);
+		rtw_mtx_unlock(NULL);
+	}
+}
+
+__inline static void _cancel_timer(_timer *ptimer,u8 *bcancelled)
+{
+	rtw_mtx_lock(NULL);
+	callout_drain(&ptimer->callout);
+	rtw_mtx_unlock(NULL);
+	*bcancelled = 1; /* assume an pending timer to be canceled */
+}
+
+__inline static void _init_workitem(_workitem *pwork, void *pfunc, PVOID cntx)
+{
+	printf("%s Not implement yet! \n",__FUNCTION__);
+}
+
+__inline static void _set_workitem(_workitem *pwork)
+{
+	printf("%s Not implement yet! \n",__FUNCTION__);
+//	schedule_work(pwork);
+}
+
+//
+// Global Mutex: can only be used at PASSIVE level.
+//
+
+#define ACQUIRE_GLOBAL_MUTEX(_MutexCounter)                              \
+{                                                               \
+}
+
+#define RELEASE_GLOBAL_MUTEX(_MutexCounter)                              \
+{                                                               \
+}
+
+/* Atomic integer operations */
+#define ATOMIC_T atomic_t
+
+#define ATOMIC_INIT(i)  { (i) }
+static inline void ATOMIC_SET(ATOMIC_T *v, int i)
+{
+	atomic_set_int(v, i);
+}
+
+static inline int ATOMIC_READ(ATOMIC_T *v)
+{
+	return atomic_load_acq_32(v);
+}
+
+static inline void ATOMIC_ADD(ATOMIC_T *v, int i)
+{
+	atomic_add_int(v, i);
+}
+static inline void ATOMIC_SUB(ATOMIC_T *v, int i)
+{
+	atomic_subtract_int(v, i);
+}
+
+static inline void ATOMIC_INC(ATOMIC_T *v)
+{
+	atomic_add_int(v, 1);
+}
+
+static inline void ATOMIC_DEC(ATOMIC_T *v)
+{
+	atomic_subtract_int(v, 1);
+}
+
+static inline int ATOMIC_ADD_RETURN(ATOMIC_T *v, int i)
+{
+	atomic_add_int(v, i);
+	return atomic_load_acq_32(v);
+}
+
+static inline int ATOMIC_SUB_RETURN(ATOMIC_T *v, int i)
+{
+	atomic_subtract_int(v, i);
+	return atomic_load_acq_32(v);
+}
+
+static inline int ATOMIC_INC_RETURN(ATOMIC_T *v)
+{
+	atomic_add_int(v, 1);
+	return atomic_load_acq_32(v);
+}
+
+static inline int ATOMIC_DEC_RETURN(ATOMIC_T *v)
+{
+	atomic_subtract_int(v, 1);
+	return atomic_load_acq_32(v);
+}
+
+static inline bool ATOMIC_INC_UNLESS(ATOMIC_T *v, int u)
+{
+	#error "TBD\n"
+}
+
+/*task*/
+typedef void (*task_fn_t)(void *context, int pending);
+#if 0 /*taskqueue -- asynchronous task execution*/
+
+TASK_INIT(struct task *task, int priority, task_fn_t func,
+	 void *context);
+
+TASK_INITIALIZER(int priority, task_fn_t func, void *context);
+
+TASKQUEUE_DECLARE(name);
+
+TASKQUEUE_DEFINE(name, taskqueue_enqueue_fn enqueue, void *context,
+	 init);
+
+#endif
+static inline void rtw_tasklet_init(_tasklet *t,task_fn_t func,
+							unsigned long data)
+{
+	TASK_INIT(t, 0, func, padapter);
+}
+static inline void rtw_tasklet_kill(_tasklet *t)
+
+
+}
+
+static inline void rtw_tasklet_schedule(_tasklet *t)
+{
+
+}
+static inline void rtw_tasklet_hi_schedule(_tasklet *t)
+{
+
+}
+
+
+/*thread*/
+static inline void rtw_thread_enter(char *name)
+{
+	printf("%s", "RTKTHREAD_enter");
+}
+
+static inline void rtw_thread_exit(_completion *comp)
+{
+	printf("%s", "RTKTHREAD_exit");
+}
+
+#include <sys/unistd.h>		/* for RFHIGHPID */
+static inline _thread_hdl_ rtw_thread_start(int (*threadfn)(void *data),
+			void *data, const char namefmt[])
+{
+	_thread_hdl_ _rtw_thread = NULL;
+	struct proc *p;
+	struct thread *td;
+
+	_rtw_thread = kproc_kthread_add(mp_xmit_packet_thread, data,
+			&p, &td, RFHIGHPID, 0, namefmt, namefmt);
+
+	if (_rtw_thread < 0)
+		_rtw_thread = NULL;
+	return _rtw_thread;
+}
+
+static inline bool rtw_thread_stop(_thread_hdl_ th)
+{
+	return _FALSE;
+}
+static inline void rtw_thread_wait_stop(void)
+{
+
+}
+__inline static void flush_signals_thread(void)
+{
+
+}
+
+#define rtw_dump_stack(void) do {} while (0)
+#define rtw_bug_on(condition) do {} while (0)
+#define rtw_warn_on(condition) do {} while (0)
+#define rtw_sprintf(buf, size, format, arg...) do {} while (0)
+
+#define rtw_netdev_priv(netdev) (((struct ifnet *)netdev)->if_softc)
+#define rtw_free_netdev(netdev) if_free((netdev))
+
+#define RTW_DIV_ROUND_UP(n, d)	(((n) + (d - 1)) / d)
+
+#define NDEV_FMT "%s"
+#define NDEV_ARG(ndev) ""
+#define ADPT_FMT "%s"
+#define ADPT_ARG(adapter) ""
+#define FUNC_NDEV_FMT "%s"
+#define FUNC_NDEV_ARG(ndev) __func__
+#define FUNC_ADPT_FMT "%s"
+#define FUNC_ADPT_ARG(adapter) __func__
+
+#define STRUCT_PACKED
+
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service_linux.h b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service_linux.h
new file mode 100644
index 000000000000..70f5d5501c98
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/osdep_service_linux.h
@@ -0,0 +1,1270 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __OSDEP_LINUX_SERVICE_H_
+#define __OSDEP_LINUX_SERVICE_H_
+
+#include <linux/version.h>
+#include <linux/spinlock.h>
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/namei.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 5))
+#include <linux/kref.h>
+#endif
+/* #include <linux/smp_lock.h> */
+#include <linux/netdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/skbuff.h>
+#include <linux/circ_buf.h>
+#include <asm/uaccess.h>
+#include <asm/byteorder.h>
+#include <asm/atomic.h>
+#include <asm/io.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26))
+#include <asm/semaphore.h>
+#else
+#include <linux/semaphore.h>
+#endif
+#include <linux/sem.h>
+#include <linux/sched.h>
+#include <linux/etherdevice.h>
+#include <linux/wireless.h>
+#include <net/iw_handler.h>
+#include <net/addrconf.h>
+#include <linux/if_arp.h>
+#include <linux/rtnetlink.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>	/* for struct tasklet_struct */
+#include <linux/ip.h>
+#include <linux/kthread.h>
+#include <linux/list.h>
+#include <linux/vmalloc.h>
+
+#ifdef CONFIG_RTKM
+#include <rtw_mem.h>
+#endif /* CONFIG_RTKM */
+
+#if defined(RTW_XMIT_THREAD_HIGH_PRIORITY) || \
+    defined(RTW_XMIT_THREAD_CB_HIGH_PRIORITY) || \
+    defined(RTW_RECV_THREAD_HIGH_PRIORITY)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+#include <uapi/linux/sched/types.h>	/* struct sched_param */
+#endif
+#endif
+
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 5, 41))
+#include <linux/tqueue.h>
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+#include <uapi/linux/limits.h>
+#else
+#include <linux/limits.h>
+#endif
+
+#ifdef RTK_DMP_PLATFORM
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 12))
+#include <linux/pageremap.h>
+#endif
+#include <asm/io.h>
+#endif
+
+#ifdef CONFIG_NET_RADIO
+#define CONFIG_WIRELESS_EXT
+#endif
+
+/* Monitor mode */
+#include <net/ieee80211_radiotap.h>
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+#include <linux/ieee80211.h>
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25) && \
+	 LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
+#define CONFIG_IEEE80211_HT_ADDT_INFO
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	/*	#include <linux/ieee80211.h> */
+	#include <net/cfg80211.h>
+#else
+	#ifdef CONFIG_REGD_SRC_FROM_OS
+	#error "CONFIG_REGD_SRC_FROM_OS requires CONFIG_IOCTL_CFG80211"
+	#endif
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+
+#ifdef CONFIG_HAS_EARLYSUSPEND
+#include <linux/earlysuspend.h>
+#endif /* CONFIG_HAS_EARLYSUSPEND */
+
+#ifdef CONFIG_EFUSE_CONFIG_FILE
+#include <linux/fs.h>
+#endif
+
+#ifdef CONFIG_PCI_HCI
+#include <linux/pci_regs.h>
+#endif
+
+#ifdef CONFIG_USB_HCI
+#include <linux/usb.h>
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 21))
+#include <linux/usb_ch9.h>
+#else
+#include <linux/usb/ch9.h>
+#endif
+#endif
+
+
+#if defined(CONFIG_RTW_GRO) && (!defined(CONFIG_RTW_NAPI))
+
+	#error "Enable NAPI before enable GRO\n"
+
+#endif
+
+
+#if (KERNEL_VERSION(2, 6, 29) > LINUX_VERSION_CODE && defined(CONFIG_RTW_NAPI))
+
+	#undef CONFIG_RTW_NAPI
+	/*#warning "Linux Kernel version too old to support NAPI (should newer than 2.6.29)\n"*/
+
+#endif
+
+#if (KERNEL_VERSION(2, 6, 33) > LINUX_VERSION_CODE && defined(CONFIG_RTW_GRO))
+
+	#undef CONFIG_RTW_GRO
+	/*#warning "Linux Kernel version too old to support GRO(should newer than 2.6.33)\n"*/
+
+#endif
+
+/*
+ * MLD related linux kernel patch in
+ * Android Common Kernel android13-5.15
+ * refs/heads/common-android13-5.15-2023-04 (5.15.94)
+ * refs/heads/android13-5.15-lts (5.15.106)
+ */
+#if (defined(__ANDROID_COMMON_KERNEL__) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 94)))
+        #define CONFIG_ACK_5_15_LTS_KERNEL
+#endif
+
+#define ATOMIC_T atomic_t
+
+#ifdef DBG_MEMORY_LEAK
+extern ATOMIC_T _malloc_cnt;
+extern ATOMIC_T _malloc_size;
+#endif
+
+static inline void *_rtw_vmalloc(u32 sz)
+{
+	void *pbuf;
+
+	pbuf = vmalloc(sz);
+
+#ifdef DBG_MEMORY_LEAK
+	if (pbuf != NULL) {
+		atomic_inc(&_malloc_cnt);
+		atomic_add(sz, &_malloc_size);
+	}
+#endif /* DBG_MEMORY_LEAK */
+
+	return pbuf;
+}
+
+static inline void *_rtw_zvmalloc(u32 sz)
+{
+	void *pbuf;
+
+	pbuf = _rtw_vmalloc(sz);
+	if (pbuf != NULL)
+		memset(pbuf, 0, sz);
+
+	return pbuf;
+}
+
+static inline void _rtw_vmfree(void *pbuf, u32 sz)
+{
+	vfree(pbuf);
+
+#ifdef DBG_MEMORY_LEAK
+	atomic_dec(&_malloc_cnt);
+	atomic_sub(sz, &_malloc_size);
+#endif /* DBG_MEMORY_LEAK */
+}
+
+static inline void *_rtw_malloc(u32 sz)
+{
+	void *pbuf = NULL;
+
+	#ifdef RTK_DMP_PLATFORM
+	if (sz > 0x4000)
+		pbuf = dvr_malloc(sz);
+	else
+	#endif
+	{
+#ifdef CONFIG_RTKM
+		pbuf = rtkm_kmalloc(sz, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+#else /* !CONFIG_RTKM */
+		pbuf = kmalloc(sz, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+#endif /* CONFIG_RTKM */
+	}
+
+#ifdef DBG_MEMORY_LEAK
+	if (pbuf != NULL) {
+		atomic_inc(&_malloc_cnt);
+		atomic_add(sz, &_malloc_size);
+	}
+#endif /* DBG_MEMORY_LEAK */
+
+	return pbuf;
+
+}
+
+static inline void *_rtw_zmalloc(u32 sz)
+{
+#if 0
+	void *pbuf = _rtw_malloc(sz);
+
+	if (pbuf != NULL)
+		memset(pbuf, 0, sz);
+#else
+#ifdef CONFIG_RTKM
+	void *pbuf = rtkm_kzalloc(sz, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+#else /* !CONFIG_RTKM */
+	/*kzalloc in KERNEL_VERSION(2, 6, 14)*/
+	void *pbuf = kzalloc( sz, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+#endif /* CONFIG_RTKM */
+
+#endif
+	return pbuf;
+}
+
+static inline void _rtw_mfree(void *pbuf, u32 sz)
+{
+	#ifdef RTK_DMP_PLATFORM
+	if (sz > 0x4000)
+		dvr_free(pbuf);
+	else
+	#endif
+	{
+#ifdef CONFIG_RTKM
+		rtkm_kfree(pbuf, sz);
+#else /* !CONFIG_RTKM */
+		kfree(pbuf);
+#endif /* CONFIG_RTKM */
+	}
+
+#ifdef DBG_MEMORY_LEAK
+	atomic_dec(&_malloc_cnt);
+	atomic_sub(sz, &_malloc_size);
+#endif /* DBG_MEMORY_LEAK */
+
+}
+
+#ifdef CONFIG_USB_HCI
+typedef struct urb *PURB;
+
+static inline void *_rtw_usb_buffer_alloc(struct usb_device *dev, size_t size, dma_addr_t *dma)
+{
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	return usb_alloc_coherent(dev, size, (in_interrupt() ? GFP_ATOMIC : GFP_KERNEL), dma);
+	#else
+	return usb_buffer_alloc(dev, size, (in_interrupt() ? GFP_ATOMIC : GFP_KERNEL), dma);
+	#endif
+}
+static inline void _rtw_usb_buffer_free(struct usb_device *dev, size_t size, void *addr, dma_addr_t dma)
+{
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	usb_free_coherent(dev, size, addr, dma);
+	#else
+	usb_buffer_free(dev, size, addr, dma);
+	#endif
+}
+#endif /* CONFIG_USB_HCI */
+
+
+/*lock - spinlock*/
+typedef	spinlock_t _lock;
+static inline void _rtw_spinlock_init(_lock *plock)
+{
+	spin_lock_init(plock);
+}
+static inline void _rtw_spinlock_free(_lock *plock)
+{
+}
+static inline void _rtw_spinlock(_lock *plock)
+{
+	spin_lock(plock);
+}
+static inline void _rtw_spinunlock(_lock *plock)
+{
+	spin_unlock(plock);
+}
+
+#if 0
+static inline void _rtw_spinlock_ex(_lock *plock)
+{
+	spin_lock(plock);
+}
+
+static inline void _rtw_spinunlock_ex(_lock *plock)
+{
+
+	spin_unlock(plock);
+}
+#endif
+__inline static void _rtw_spinlock_irq(_lock *plock, unsigned long *flags)
+{
+	spin_lock_irqsave(plock, *flags);
+}
+__inline static void _rtw_spinunlock_irq(_lock *plock, unsigned long *flags)
+{
+	spin_unlock_irqrestore(plock, *flags);
+}
+__inline static void _rtw_spinlock_bh(_lock *plock)
+{
+	spin_lock_bh(plock);
+}
+__inline static void _rtw_spinunlock_bh(_lock *plock)
+{
+	spin_unlock_bh(plock);
+}
+
+__inline static int _rtw_spin_is_locked(_lock *plock)
+{
+	return spin_is_locked(plock);
+}
+
+/*lock - semaphore*/
+typedef struct	semaphore _sema;
+static inline void _rtw_init_sema(_sema *sema, int init_val)
+{
+	sema_init(sema, init_val);
+}
+static inline void _rtw_free_sema(_sema *sema)
+{
+}
+static inline void _rtw_up_sema(_sema *sema)
+{
+	up(sema);
+}
+static inline u32 _rtw_down_sema(_sema *sema)
+{
+	if (down_interruptible(sema))
+		return _FAIL;
+	else
+		return _SUCCESS;
+}
+
+/*lock - mutex*/
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	typedef struct mutex		_mutex;
+#else
+	typedef struct semaphore	_mutex;
+#endif
+static inline void _rtw_mutex_init(_mutex *pmutex)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	mutex_init(pmutex);
+#else
+	init_MUTEX(pmutex);
+#endif
+}
+
+static inline void _rtw_mutex_free(_mutex *pmutex)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	mutex_destroy(pmutex);
+#else
+#endif
+}
+__inline static int _rtw_mutex_lock_interruptible(_mutex *pmutex)
+{
+	int ret = 0;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	/* mutex_lock(pmutex); */
+	ret = mutex_lock_interruptible(pmutex);
+#else
+	ret = down_interruptible(pmutex);
+#endif
+	return ret;
+}
+
+__inline static int _rtw_mutex_lock(_mutex *pmutex)
+{
+	int ret = 0;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	mutex_lock(pmutex);
+#else
+	down(pmutex);
+#endif
+	return ret;
+}
+
+__inline static void _rtw_mutex_unlock(_mutex *pmutex)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	mutex_unlock(pmutex);
+#else
+	up(pmutex);
+#endif
+}
+
+
+/*completion*/
+typedef struct completion _completion;
+static inline void _rtw_init_completion(_completion *comp)
+{
+	init_completion(comp);
+}
+static inline unsigned long _rtw_wait_for_comp_timeout(_completion *comp, unsigned long timeout)
+{
+	return wait_for_completion_timeout(comp, timeout);
+}
+static inline void _rtw_wait_for_comp(_completion *comp)
+{
+	return wait_for_completion(comp);
+}
+
+struct	__queue	{
+	struct	list_head	queue;
+	_lock	lock;
+};
+
+typedef unsigned char	_buffer;
+
+typedef struct	__queue	_queue;
+
+
+/*list*/
+#define LIST_CONTAINOR(ptr, type, member) \
+	((type *)((char *)(ptr)-(SIZE_T)(&((type *)0)->member)))
+
+
+typedef struct	list_head	_list;
+/* Caller must check if the list is empty before calling rtw_list_delete*/
+__inline static void rtw_list_delete(_list *plist)
+{
+	list_del_init(plist);
+}
+
+__inline static _list *get_next(_list	*list)
+{
+	return list->next;
+}
+__inline static _list	*get_list_head(_queue *queue)
+{
+	return &(queue->queue);
+}
+#define rtw_list_first_entry(ptr, type, member) list_first_entry(ptr, type, member)
+
+/* hlist */
+typedef struct	hlist_head	rtw_hlist_head;
+typedef struct	hlist_node	rtw_hlist_node;
+#define rtw_hlist_for_each_entry(pos, head, member) hlist_for_each_entry(pos, head, member)
+#define rtw_hlist_for_each_safe(pos, n, head) hlist_for_each_safe(pos, n, head)
+#define rtw_hlist_entry(ptr, type, member) hlist_entry(ptr, type, member)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+#define rtw_hlist_for_each_entry_safe(pos, np, n, head, member) hlist_for_each_entry_safe(pos, n, head, member)
+#define rtw_hlist_for_each_entry_rcu(pos, node, head, member) hlist_for_each_entry_rcu(pos, head, member)
+#else
+#define rtw_hlist_for_each_entry_safe(pos, np, n, head, member) hlist_for_each_entry_safe(pos, np, n, head, member)
+#define rtw_hlist_for_each_entry_rcu(pos, node, head, member) hlist_for_each_entry_rcu(pos, node, head, member)
+#endif
+
+/* RCU */
+typedef struct rcu_head rtw_rcu_head;
+#define rtw_rcu_dereference(p) rcu_dereference((p))
+#define rtw_rcu_dereference_protected(p, c) rcu_dereference_protected(p, c)
+#define rtw_rcu_assign_pointer(p, v) rcu_assign_pointer((p), (v))
+#define rtw_rcu_read_lock() rcu_read_lock()
+#define rtw_rcu_read_unlock() rcu_read_unlock()
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34))
+#define rtw_rcu_access_pointer(p) rcu_access_pointer(p)
+#endif
+
+/* rhashtable */
+#include "../os_dep/linux/rtw_rhashtable.h"
+
+
+/*thread*/
+typedef void *_thread_hdl_;
+typedef int thread_return;
+typedef void *thread_context;
+struct thread_hdl{
+	_thread_hdl_ thread_handler;
+	u8 thread_status;
+	u8 cpu_id;
+	u8 en_assign_cpuid;
+};
+#define THREAD_STATUS_STARTED BIT(0)
+#define THREAD_STATUS_STOPPED BIT(1)
+#define RST_THREAD_STATUS(t) (t->thread_status = 0)
+#define SET_THREAD_STATUS(t, s) 	(t->thread_status |= s)
+#define CLR_THREAD_STATUS(t, cl)	(t->thread_status &= ~(cl))
+#define CHK_THREAD_STATUS(t, ck) (t->thread_status & ck)
+
+#define RTW_TASK_COMM_LEN TASK_COMM_LEN
+static inline void rtw_get_task_comm(char *dst) { memcpy(dst, current->comm, RTW_TASK_COMM_LEN); }
+static inline unsigned int rtw_get_cpu_id(void) { return smp_processor_id(); }
+
+typedef void timer_hdl_return;
+typedef void *timer_hdl_context;
+
+static inline void rtw_thread_enter(char *name)
+{
+	allow_signal(SIGTERM);
+}
+
+static inline void rtw_thread_exit(_completion *comp)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+	complete_and_exit(comp, 0);
+#else
+	kthread_complete_and_exit(comp, 0);
+#endif
+}
+
+#ifdef CONFIG_PHL_CPU_BALANCE_THREAD
+static inline _thread_hdl_ rtw_thread_cpu_start(int (*threadfn)(void *data),
+			void *data, const char namefmt[], u8 cpu_id, u8 en_cpuid)
+{
+	_thread_hdl_ _rtw_thread = NULL;
+
+	_rtw_thread = kthread_create(threadfn, data, namefmt);
+	if (IS_ERR(_rtw_thread)) {
+		WARN_ON(!_rtw_thread);
+		_rtw_thread = NULL;
+	}
+	else {
+		/* Specific CPU */
+		if(en_cpuid == _TRUE)
+			kthread_bind(_rtw_thread, cpu_id);
+		wake_up_process(_rtw_thread);
+	}
+	return _rtw_thread;
+}
+#endif /*CONFIG_PHL_CPU_BALANCE_THREAD*/
+
+static inline _thread_hdl_ rtw_thread_start(int (*threadfn)(void *data),
+			void *data, const char namefmt[])
+{
+	_thread_hdl_ _rtw_thread = NULL;
+
+	_rtw_thread = kthread_run(threadfn, data, namefmt);
+	if (IS_ERR(_rtw_thread)) {
+		WARN_ON(!_rtw_thread);
+		_rtw_thread = NULL;
+	}
+	return _rtw_thread;
+}
+
+static inline bool rtw_thread_stop(_thread_hdl_ th)
+{
+
+	return kthread_stop(th);
+}
+
+static inline void rtw_thread_wait_stop(void)
+{
+	#if 0
+	while (!kthread_should_stop())
+		rtw_msleep_os(10);
+	#else
+	set_current_state(TASK_INTERRUPTIBLE);
+	while (!kthread_should_stop()) {
+		schedule();
+		set_current_state(TASK_INTERRUPTIBLE);
+	}
+	__set_current_state(TASK_RUNNING);
+	#endif
+}
+
+static inline void flush_signals_thread(void)
+{
+	if (signal_pending(current))
+		flush_signals(current);
+}
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24))
+	#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
+#endif
+
+#define CONFIG_OSDEP_DIV64_API
+
+static inline u64 rtw_modular64(u64 x, u64 y)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0))
+	u64 r = 0;
+#else
+	u32 r = 0;
+#endif
+
+	if (0 == y)
+		return 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0))
+	div64_u64_rem(x, y, &r);
+#else
+	WARN_ON(y > U32_MAX);
+	div_u64_rem(x, y, &r);
+#endif
+
+	return r;
+}
+
+static inline u64 rtw_division64(u64 x, u64 y)
+{
+	return (0 != y) ? div64_u64(x, y) : 0;
+}
+
+static inline s64 rtw_division64_s64(s64 x, s64 y)
+{
+	return (0 != y) ? div64_s64(x, y) : 0;
+}
+
+typedef unsigned long systime;
+typedef ktime_t sysptime;
+
+#define CONFIG_OSDEP_SPTIME_API
+
+static inline sysptime rtw_sptime_get(void)
+{
+	return ktime_get(); /* CLOCK_MONOTONIC */
+}
+
+static inline sysptime rtw_sptime_get_raw(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0))
+	return ktime_get_raw(); /* CLOCK_MONOTONIC_RAW */
+#else
+	return ktime_get(); /* CLOCK_MONOTONIC */
+#endif
+}
+
+static inline sysptime rtw_sptime_set(s64 secs, const u32 nsecs)
+{
+	return ktime_set(secs, nsecs);
+}
+
+static inline sysptime rtw_sptime_zero(void)
+{
+	return ktime_set(0, 0);
+}
+
+/*
+ *   cmp1  < cmp2: return <0
+ *   cmp1 == cmp2: return 0
+ *   cmp1  > cmp2: return >0
+ */
+static inline int rtw_sptime_cmp(const sysptime cmp1, const sysptime cmp2)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	return ktime_compare(cmp1, cmp2);
+#else
+	if (cmp1.tv64 < cmp2.tv64)
+		return -1;
+	if (cmp1.tv64 > cmp2.tv64)
+		return 1;
+	return 0;
+#endif
+}
+
+/*
+ * sub = lhs - rhs, in normalized form
+ */
+static inline sysptime rtw_sptime_sub(const sysptime lhs, const sysptime rhs)
+{
+	return ktime_sub(lhs, rhs);
+}
+
+/*
+ * add = lhs + rhs, in normalized form
+ */
+static inline sysptime rtw_sptime_add(const sysptime lhs, const sysptime rhs)
+{
+	return ktime_add(lhs, rhs);
+}
+
+static inline s64 rtw_sptime_to_ms(const sysptime sptime)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	return ktime_to_ms(sptime);
+#else
+	struct timeval tv = ktime_to_timeval(sptime);
+
+	return (s64) tv.tv_sec * MSEC_PER_SEC + tv.tv_usec / USEC_PER_MSEC;
+#endif
+}
+
+static inline sysptime rtw_ms_to_sptime(u64 ms)
+{
+	return ns_to_ktime(ms * NSEC_PER_MSEC);
+}
+
+static inline s64 rtw_sptime_to_us(const sysptime sptime)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22))
+	return ktime_to_us(sptime);
+#else
+	struct timeval tv = ktime_to_timeval(sptime);
+
+	return (s64) tv.tv_sec * USEC_PER_SEC + tv.tv_usec;
+#endif
+}
+
+static inline sysptime rtw_us_to_sptime(u64 us)
+{
+	return ns_to_ktime(us * NSEC_PER_USEC);
+}
+
+static inline s64 rtw_sptime_to_ns(const sysptime sptime)
+{
+	return ktime_to_ns(sptime);
+}
+
+static inline sysptime rtw_ns_to_sptime(u64 ns)
+{
+	return ns_to_ktime(ns);
+}
+
+/*tasklet*/
+typedef struct tasklet_struct _tasklet;
+typedef void (*tasklet_fn_t)(unsigned long);
+
+#if 1
+static inline void rtw_tasklet_init(_tasklet *t, tasklet_fn_t func,
+							unsigned long data)
+{
+	tasklet_init(t, func, data);
+}
+static inline void rtw_tasklet_kill(_tasklet *t)
+{
+	tasklet_kill(t);
+}
+
+static inline void rtw_tasklet_schedule(_tasklet *t)
+{
+	tasklet_schedule(t);
+}
+static inline void rtw_tasklet_hi_schedule(_tasklet *t)
+{
+	tasklet_hi_schedule(t);
+}
+#else
+#define rtw_tasklet_init tasklet_init
+#define rtw_tasklet_kill tasklet_kill
+#define rtw_tasklet_schedule tasklet_schedule
+#define rtw_tasklet_hi_schedule tasklet_hi_schedule
+#endif
+
+/*skb_buffer*/
+static inline struct sk_buff *_rtw_skb_alloc(u32 sz)
+{
+	return __dev_alloc_skb(sz, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+}
+
+static inline void _rtw_skb_free(struct sk_buff *skb)
+{
+	dev_kfree_skb_any(skb);
+}
+
+static inline struct sk_buff *_rtw_skb_copy(const struct sk_buff *skb)
+{
+	return skb_copy(skb, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+}
+
+static inline struct sk_buff *_rtw_skb_clone(struct sk_buff *skb)
+{
+	return skb_clone(skb, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+}
+
+static inline int _rtw_skb_linearize(struct sk_buff *skb)
+{
+	return skb_linearize(skb);
+}
+
+static inline struct sk_buff *_rtw_pskb_copy(struct sk_buff *skb)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36))
+	return pskb_copy(skb, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+#else
+	return skb_clone(skb, in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+#endif
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 22))
+/* Porting from linux kernel, for compatible with old kernel. */
+static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
+{
+	return skb->tail;
+}
+
+static inline void skb_reset_tail_pointer(struct sk_buff *skb)
+{
+	skb->tail = skb->data;
+}
+
+static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
+{
+	skb->tail = skb->data + offset;
+}
+
+static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
+{
+	return skb->end;
+}
+#endif
+static inline u8 *rtw_skb_data(struct sk_buff *pkt)
+{
+	return pkt->data;
+}
+
+static inline u32 rtw_skb_len(struct sk_buff *pkt)
+{
+	return pkt->len;
+}
+
+static inline void *rtw_skb_put_zero(struct sk_buff *skb, unsigned int len)
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+	return skb_put_zero(skb, len);
+#else
+	void *tmp = skb_put(skb, len);
+
+	memset(tmp, 0, len);
+
+	return tmp;
+#endif
+}
+
+/*timer*/
+typedef struct rtw_timer_list _timer;
+struct rtw_timer_list {
+	struct timer_list timer;
+	void (*function)(void *);
+	void *arg;
+};
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+static inline void timer_hdl(struct timer_list *in_timer)
+{
+	_timer *ptimer = timer_container_of(ptimer, in_timer, timer);
+
+	ptimer->function(ptimer->arg);
+}
+#else
+static inline void timer_hdl(unsigned long cntx)
+{
+	_timer *ptimer = (_timer *)cntx;
+
+	ptimer->function(ptimer->arg);
+}
+#endif
+
+__inline static void _init_timer(_timer *ptimer, void *pfunc, void *cntx)
+{
+	ptimer->function = pfunc;
+	ptimer->arg = cntx;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+	timer_setup(&ptimer->timer, timer_hdl, 0);
+#else
+	/* setup_timer(ptimer, pfunc,(u32)cntx);	 */
+	ptimer->timer.function = timer_hdl;
+	ptimer->timer.data = (unsigned long)ptimer;
+	init_timer(&ptimer->timer);
+#endif
+}
+
+__inline static int _check_timer_is_active(_timer *ptimer)
+{
+	return timer_pending(&ptimer->timer);
+}
+
+__inline static void _set_timer(_timer *ptimer, u32 delay_time)
+{
+	mod_timer(&ptimer->timer , (jiffies + (delay_time * HZ / 1000)));
+}
+
+__inline static void _cancel_timer(_timer *ptimer, u8 *bcancelled)
+{
+	*bcancelled = timer_delete_sync(&ptimer->timer) == 1 ? 1 : 0;
+}
+
+__inline static void _cancel_timer_async(_timer *ptimer)
+{
+	timer_delete(&ptimer->timer);
+}
+
+/*work*/
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 5, 41))
+typedef struct work_struct _workitem;
+#else
+typedef struct tq_struct _workitem;
+#endif
+
+static inline void _init_workitem(_workitem *pwork, void *pfunc, void *cntx)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20))
+	INIT_WORK(pwork, pfunc);
+#elif (LINUX_VERSION_CODE > KERNEL_VERSION(2, 5, 41))
+	INIT_WORK(pwork, pfunc, pwork);
+#else
+	INIT_TQUEUE(pwork, pfunc, pwork);
+#endif
+}
+
+__inline static void _set_workitem(_workitem *pwork)
+{
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 5, 41))
+	schedule_work(pwork);
+#else
+	schedule_task(pwork);
+#endif
+}
+
+__inline static void _cancel_workitem_sync(_workitem *pwork)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22))
+	cancel_work_sync(pwork);
+#elif (LINUX_VERSION_CODE > KERNEL_VERSION(2, 5, 41))
+	flush_scheduled_work();
+#else
+	flush_scheduled_tasks();
+#endif
+}
+/*
+ * Global Mutex: can only be used at PASSIVE level.
+ *   */
+#define ACQUIRE_GLOBAL_MUTEX(_MutexCounter)                              \
+	{                                                               \
+		while (atomic_inc_return((atomic_t *)&(_MutexCounter)) != 1) { \
+			atomic_dec((atomic_t *)&(_MutexCounter));        \
+			msleep(10);                          \
+		}                                                           \
+	}
+
+#define RELEASE_GLOBAL_MUTEX(_MutexCounter)                              \
+	{                                                               \
+		atomic_dec((atomic_t *)&(_MutexCounter));        \
+	}
+
+
+typedef	struct	net_device *_nic_hdl;
+static inline int rtw_netif_queue_stopped(struct net_device *pnetdev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	return (netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 0)) &&
+		netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 1)) &&
+		netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 2)) &&
+		netif_tx_queue_stopped(netdev_get_tx_queue(pnetdev, 3)));
+#else
+	return netif_queue_stopped(pnetdev);
+#endif
+}
+
+#ifdef CONFIG_HWSIM
+int _rtw_netif_rx(_nic_hdl ndev, struct sk_buff *skb);
+#else
+static inline int _rtw_netif_rx(_nic_hdl ndev, struct sk_buff *skb)
+{
+#if defined(CONFIG_RTW_FC_FASTFWD)
+extern int fwdEngine_wifi_rx(struct sk_buff *skb);
+enum {
+	RE8670_RX_STOP=0,
+	RE8670_RX_CONTINUE,
+	RE8670_RX_STOP_SKBNOFREE,
+	RE8670_RX_END
+};
+int ret = 0;
+
+	skb->dev = ndev;
+	skb->data-=14;
+	skb->len+=14;
+
+	ret = fwdEngine_wifi_rx(skb);
+
+	if(ret==RE8670_RX_CONTINUE)
+	{
+		skb->data+=14;
+		skb->len-=14;
+	return netif_rx(skb);
+}
+	else if(ret==RE8670_RX_STOP)
+	{
+		kfree_skb(skb);
+	}
+
+	return 0;
+#else
+	skb->dev = ndev;
+	return netif_rx(skb);
+#endif
+}
+#endif
+
+#ifdef CONFIG_RTW_NAPI
+static inline int _rtw_netif_receive_skb(_nic_hdl ndev, struct sk_buff *skb)
+{
+	skb->dev = ndev;
+	return netif_receive_skb(skb);
+}
+
+#ifdef CONFIG_RTW_GRO
+static inline gro_result_t _rtw_napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
+{
+	return napi_gro_receive(napi, skb);
+}
+#endif /* CONFIG_RTW_GRO */
+#endif /* CONFIG_RTW_NAPI */
+
+static inline void rtw_netif_wake_queue(struct net_device *pnetdev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	netif_tx_wake_all_queues(pnetdev);
+#else
+	netif_wake_queue(pnetdev);
+#endif
+}
+
+static inline void rtw_netif_start_queue(struct net_device *pnetdev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	netif_tx_start_all_queues(pnetdev);
+#else
+	netif_start_queue(pnetdev);
+#endif
+}
+
+static inline void rtw_netif_stop_queue(struct net_device *pnetdev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	netif_tx_stop_all_queues(pnetdev);
+#else
+	netif_stop_queue(pnetdev);
+#endif
+}
+static inline void rtw_netif_device_attach(struct net_device *pnetdev)
+{
+	netif_device_attach(pnetdev);
+}
+static inline void rtw_netif_device_detach(struct net_device *pnetdev)
+{
+	netif_device_detach(pnetdev);
+}
+static inline void rtw_netif_carrier_on(struct net_device *pnetdev)
+{
+	netif_carrier_on(pnetdev);
+}
+static inline void rtw_netif_carrier_off(struct net_device *pnetdev)
+{
+	netif_carrier_off(pnetdev);
+}
+
+static inline int rtw_merge_string(char *dst, int dst_len, const char *src1, const char *src2)
+{
+	int	len = 0;
+	len += snprintf(dst + len, dst_len - len, "%s", src1);
+	len += snprintf(dst + len, dst_len - len, "%s", src2);
+
+	return len;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27))
+	#define rtw_signal_process(pid, sig) kill_pid(find_vpid((pid)), (sig), 1)
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) */
+	#define rtw_signal_process(pid, sig) kill_proc((pid), (sig), 1)
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) */
+
+
+/* Suspend lock prevent system from going suspend */
+#ifdef CONFIG_WAKELOCK
+	#include <linux/wakelock.h>
+#elif defined(CONFIG_ANDROID_POWER)
+	#include <linux/android_power.h>
+#endif
+
+/* limitation of path length */
+#define PATH_LENGTH_MAX PATH_MAX
+
+/* Atomic integer operations */
+static inline void ATOMIC_SET(ATOMIC_T *v, int i)
+{
+	atomic_set(v, i);
+}
+
+static inline int ATOMIC_READ(ATOMIC_T *v)
+{
+	return atomic_read(v);
+}
+
+static inline void ATOMIC_ADD(ATOMIC_T *v, int i)
+{
+	atomic_add(i, v);
+}
+static inline void ATOMIC_SUB(ATOMIC_T *v, int i)
+{
+	atomic_sub(i, v);
+}
+
+static inline void ATOMIC_INC(ATOMIC_T *v)
+{
+	atomic_inc(v);
+}
+
+static inline void ATOMIC_DEC(ATOMIC_T *v)
+{
+	atomic_dec(v);
+}
+
+static inline int ATOMIC_ADD_RETURN(ATOMIC_T *v, int i)
+{
+	return atomic_add_return(i, v);
+}
+
+static inline int ATOMIC_SUB_RETURN(ATOMIC_T *v, int i)
+{
+	return atomic_sub_return(i, v);
+}
+
+static inline int ATOMIC_INC_RETURN(ATOMIC_T *v)
+{
+	return atomic_inc_return(v);
+}
+
+static inline int ATOMIC_DEC_RETURN(ATOMIC_T *v)
+{
+	return atomic_dec_return(v);
+}
+
+static inline bool ATOMIC_INC_UNLESS(ATOMIC_T *v, int u)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 15))
+	return atomic_add_unless(v, 1, u);
+#else
+	/* only make sure not exceed after this function */
+	if (ATOMIC_INC_RETURN(v) > u) {
+		ATOMIC_DEC(v);
+		return 0;
+	}
+	return 1;
+#endif
+}
+
+#define NDEV_FMT "%s"
+#define NDEV_ARG(ndev) ndev->name
+#define ADPT_FMT "%s"
+#define ADPT_ARG(adapter) (adapter->pnetdev ? adapter->pnetdev->name : NULL)
+#define FUNC_NDEV_FMT "%s(%s)"
+#define FUNC_NDEV_ARG(ndev) __func__, ndev->name
+#define FUNC_ADPT_FMT "%s(%s)"
+#define FUNC_ADPT_ARG(adapter) __func__, (adapter->pnetdev ? adapter->pnetdev->name : NULL)
+
+#define rtw_netdev_priv(netdev) (((struct rtw_netdev_priv_indicator *)netdev_priv(netdev))->priv)
+struct rtw_netdev_priv_indicator {
+	void *priv;
+	u32 sizeof_priv;
+};
+struct net_device *rtw_alloc_etherdev_with_old_priv(int sizeof_priv, void *old_priv);
+extern struct net_device *rtw_alloc_etherdev(int sizeof_priv);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24))
+#define rtw_get_same_net_ndev_by_name(ndev, name) dev_get_by_name(name)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26))
+#define rtw_get_same_net_ndev_by_name(ndev, name) dev_get_by_name(ndev->nd_net, name)
+#else
+#define rtw_get_same_net_ndev_by_name(ndev, name) dev_get_by_name(dev_net(ndev), name)
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24))
+#define rtw_get_bridge_ndev_by_name(name) dev_get_by_name(name)
+#else
+#define rtw_get_bridge_ndev_by_name(name) dev_get_by_name(&init_net, name)
+#endif
+
+static inline void rtw_dump_stack(void)
+{
+	dump_stack();
+}
+#define rtw_bug_on(condition) BUG_ON(condition)
+#define RTW_DIV_ROUND_UP(n, d)	DIV_ROUND_UP(n, d)
+#define rtw_sprintf(buf, size, format, arg...) snprintf(buf, size, format, ##arg)
+
+#define STRUCT_PACKED __attribute__ ((packed))
+
+#ifndef fallthrough
+#if __GNUC__ >= 5 || defined(__clang__)
+#ifndef __has_attribute
+#define __has_attribute(x) 0
+#endif
+#if __has_attribute(__fallthrough__)
+#define fallthrough __attribute__((__fallthrough__))
+#endif
+#endif
+#ifndef fallthrough
+#define fallthrough do {} while (0) /* fallthrough */
+#endif
+#endif
+
+#ifndef static_assert
+#define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
+#define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
+#endif
+
+#ifdef CONFIG_PCI_HCI
+/* Extended Capabilities (PCI-X 2.0 and Express) */
+#ifndef PCI_EXT_CAP_ID_L1SS
+#define PCI_EXT_CAP_ID_L1SS  0x1E	/* L1 PM Substates */
+#endif
+/* L1 PM Substates */
+#ifndef PCI_L1SS_CAP
+#define PCI_L1SS_CAP		    4	/* capability register */
+#define  PCI_L1SS_CAP_PCIPM_L1_2	 1	/* PCI PM L1.2 Support */
+#define  PCI_L1SS_CAP_PCIPM_L1_1	 2	/* PCI PM L1.1 Support */
+#define  PCI_L1SS_CAP_ASPM_L1_2		 4	/* ASPM L1.2 Support */
+#define  PCI_L1SS_CAP_ASPM_L1_1		 8	/* ASPM L1.1 Support */
+#define  PCI_L1SS_CAP_L1_PM_SS		16	/* L1 PM Substates Support */
+#endif
+#ifndef PCI_L1SS_CTL1
+#define PCI_L1SS_CTL1		    8	/* Control Register 1 */
+#define  PCI_L1SS_CTL1_PCIPM_L1_2	1	/* PCI PM L1.2 Enable */
+#define  PCI_L1SS_CTL1_PCIPM_L1_1	2	/* PCI PM L1.1 Support */
+#define  PCI_L1SS_CTL1_ASPM_L1_2	4	/* ASPM L1.2 Support */
+#define  PCI_L1SS_CTL1_ASPM_L1_1	8	/* ASPM L1.1 Support */
+#define  PCI_L1SS_CTL1_L1SS_MASK	0x0000000F
+#endif
+#endif /* CONFIG_PCI_HCI */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0))
+#define dev_addr_mod(dev, offset, addr, len) _rtw_memcpy(&dev->dev_addr[offset], addr, len)
+#endif
+
+#define rtw_warn_on(condition) \
+	do { \
+		if (condition) { \
+			RTW_WARN("RTW_WARN_ON triggered: =====> START\n"); \
+			WARN_ON(1); \
+			ATOMIC_INC((ATOMIC_T *)&rtw_warn_on_cnt); \
+			RTW_WARN("RTW_WARN_ON triggered: <===== END\n"); \
+		} \
+	} while (0)
+
+#endif /* __OSDEP_LINUX_SERVICE_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/pci_ops.h b/drivers/net/wireless/realtek/rtl8852bs/include/pci_ops.h
new file mode 100644
index 000000000000..260b3dce6487
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/pci_ops.h
@@ -0,0 +1,44 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __PCI_OPS_H_
+#define __PCI_OPS_H_
+
+#ifdef RTK_129X_PLATFORM
+#define PCIE_SLOT1_MEM_START	0x9804F000
+#define PCIE_SLOT1_MEM_LEN	0x1000
+#define PCIE_SLOT1_CTRL_START	0x9804EC00
+
+#define PCIE_SLOT2_MEM_START	0x9803C000
+#define PCIE_SLOT2_MEM_LEN	0x1000
+#define PCIE_SLOT2_CTRL_START	0x9803BC00
+
+#define PCIE_MASK_OFFSET	0x100 /* mask offset from CTRL_START */
+#define PCIE_TRANSLATE_OFFSET	0x104 /* translate offset from CTRL_START */
+#endif
+
+#define PCI_BC_CLK_REQ		BIT0
+#define PCI_BC_ASPM_L0s		BIT1
+#define PCI_BC_ASPM_L1		BIT2
+#define PCI_BC_ASPM_L1Off	BIT3
+//#define PCI_BC_ASPM_LTR	BIT4
+//#define PCI_BC_ASPM_OBFF	BIT5
+
+void	PlatformClearPciPMEStatus(_adapter *adapter);
+
+#ifdef CONFIG_64BIT_DMA
+u8 PlatformEnableDMA64(struct pci_dev *pdev);
+#endif
+
+#endif /*__PCI_OPS_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/pci_ops_linux.h b/drivers/net/wireless/realtek/rtl8852bs/include/pci_ops_linux.h
new file mode 100644
index 000000000000..37f7c1281308
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/pci_ops_linux.h
@@ -0,0 +1,24 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __PCI_OPS_LINUX_H__
+#define __PCI_OPS_LINUX_H__
+
+u8 os_pci_read8(struct dvobj_priv *dvobj, u32 addr);
+u16 os_pci_read16(struct dvobj_priv *dvobj, u32 addr);
+u32 os_pci_read32(struct dvobj_priv *dvobj, u32 addr);
+int os_pci_write8(struct dvobj_priv *dvobj, u32 addr, u8 val);
+int os_pci_write16(struct dvobj_priv *dvobj, u32 addr, u16 val);
+int os_pci_write32(struct dvobj_priv *dvobj, u32 addr, u32 val);
+#endif /*__PCI_OPS_LINUX_H__*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/phl_api_tmp.h b/drivers/net/wireless/realtek/rtl8852bs/include/phl_api_tmp.h
new file mode 100644
index 000000000000..3e160e4abf9b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/phl_api_tmp.h
@@ -0,0 +1,28 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __PHL_API_TMP_H_
+#define __PHL_API_TMP_H_
+
+
+#ifdef CONFIG_SDIO_HCI
+static inline void rtw_hal_sd_f0_reg_dump(void *sel, void *h){ };
+static inline void rtw_hal_sdio_local_reg_dump(void *sel, void *h){ };
+#endif
+
+static inline u32 rtw_hal_get_txdesc_len(void *h,
+		struct pkt_attrib *attrib){return 0;};
+
+
+#endif /*__PHL_API_TMP_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/recv_osdep.h b/drivers/net/wireless/realtek/rtl8852bs/include/recv_osdep.h
new file mode 100644
index 000000000000..df8b0866541b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/recv_osdep.h
@@ -0,0 +1,74 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RECV_OSDEP_H_
+#define __RECV_OSDEP_H_
+
+u8 rtw_init_recv_info(_adapter *adapter);
+sint rtw_init_recv_priv(struct dvobj_priv *dvobj);
+void rtw_free_recv_priv(struct dvobj_priv *dvobj);
+
+#if 0
+extern s32  rtw_recv_entry(union recv_frame *precv_frame);
+#endif
+void rtw_rframe_set_os_pkt(union recv_frame *rframe);
+extern int rtw_recv_indicatepkt(_adapter *adapter, union recv_frame *precv_frame);
+extern void rtw_recv_returnpacket(_nic_hdl cnxt, struct sk_buff *preturnedpkt);
+
+#ifdef CONFIG_WIFI_MONITOR
+extern int rtw_recv_monitor(_adapter *padapter, union recv_frame *precv_frame);
+#endif /* CONFIG_WIFI_MONITOR */
+
+#ifdef CONFIG_HOSTAPD_MLME
+extern void rtw_hostapd_mlme_rx(_adapter *padapter, union recv_frame *precv_frame);
+#endif
+
+struct sta_info;
+extern void rtw_handle_tkip_mic_err(_adapter *padapter, struct sta_info *sta, u8 bgroup);
+
+
+int rtw_os_recv_resource_init(struct recv_priv *precvpriv);
+int rtw_os_recv_resource_alloc(union recv_frame *precvframe);
+void rtw_os_recv_resource_free(struct recv_priv *precvpriv);
+
+
+int rtw_os_alloc_recvframe(_adapter *padapter, union recv_frame *precvframe,
+				u8 *pdata, struct sk_buff *pskb);
+int rtw_os_recvframe_duplicate_skb(_adapter *padapter,
+				union recv_frame *pcloneframe, struct sk_buff *pskb);
+void rtw_os_free_recvframe(union recv_frame *precvframe);
+
+
+#if 0
+int rtw_os_recvbuf_resource_alloc(_adapter *padapter, struct recv_buf *precvbuf);
+int rtw_os_recvbuf_resource_free(_adapter *padapter, struct recv_buf *precvbuf);
+#endif
+
+struct sk_buff *rtw_os_alloc_msdu_pkt(union recv_frame *prframe, const u8 *da, const u8 *sa
+	, u8 *msdu ,u16 msdu_len, enum rtw_rx_llc_hdl llc_hdl);
+void rtw_os_recv_indicate_pkt(_adapter *padapter, struct sk_buff *pkt,
+				union recv_frame *rframe);
+
+#ifdef PLATFORM_LINUX
+#ifdef CONFIG_RTW_NAPI
+#include <linux/netdevice.h>	/* struct napi_struct */
+
+int rtw_recv_napi_poll(struct napi_struct *, int budget);
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+void dynamic_napi_th_chk (_adapter *adapter);
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+#endif /* CONFIG_RTW_NAPI */
+#endif /* PLATFORM_LINUX */
+
+#endif /*  */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_android.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_android.h
new file mode 100644
index 000000000000..8d3b481631d9
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_android.h
@@ -0,0 +1,106 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __RTW_ANDROID_H__
+#define __RTW_ANDROID_H__
+
+enum ANDROID_WIFI_CMD {
+	ANDROID_WIFI_CMD_START,
+	ANDROID_WIFI_CMD_STOP,
+	ANDROID_WIFI_CMD_SCAN_ACTIVE,
+	ANDROID_WIFI_CMD_SCAN_PASSIVE,
+	ANDROID_WIFI_CMD_RSSI,
+	ANDROID_WIFI_CMD_LINKSPEED,
+	ANDROID_WIFI_CMD_RXFILTER_START,
+	ANDROID_WIFI_CMD_RXFILTER_STOP,
+	ANDROID_WIFI_CMD_RXFILTER_ADD,
+	ANDROID_WIFI_CMD_RXFILTER_REMOVE,
+	ANDROID_WIFI_CMD_BTCOEXSCAN_START,
+	ANDROID_WIFI_CMD_BTCOEXSCAN_STOP,
+	ANDROID_WIFI_CMD_BTCOEXMODE,
+	ANDROID_WIFI_CMD_SETSUSPENDMODE,
+	ANDROID_WIFI_CMD_SETSUSPENDOPT,
+	ANDROID_WIFI_CMD_P2P_DEV_ADDR,
+	ANDROID_WIFI_CMD_SETFWPATH,
+	ANDROID_WIFI_CMD_SETBAND,
+	ANDROID_WIFI_CMD_GETBAND,
+	ANDROID_WIFI_CMD_COUNTRY,
+	ANDROID_WIFI_CMD_P2P_SET_NOA,
+	ANDROID_WIFI_CMD_P2P_GET_NOA,
+	ANDROID_WIFI_CMD_P2P_SET_PS,
+	ANDROID_WIFI_CMD_SET_AP_WPS_P2P_IE,
+
+	ANDROID_WIFI_CMD_MIRACAST,
+
+#ifdef CONFIG_PNO_SUPPORT
+	ANDROID_WIFI_CMD_PNOSSIDCLR_SET,
+	ANDROID_WIFI_CMD_PNOSETUP_SET,
+	ANDROID_WIFI_CMD_PNOENABLE_SET,
+	ANDROID_WIFI_CMD_PNODEBUG_SET,
+#endif
+
+	ANDROID_WIFI_CMD_MACADDR,
+
+	ANDROID_WIFI_CMD_BLOCK_SCAN,
+	ANDROID_WIFI_CMD_BLOCK,
+
+	ANDROID_WIFI_CMD_WFD_ENABLE,
+	ANDROID_WIFI_CMD_WFD_DISABLE,
+
+	ANDROID_WIFI_CMD_WFD_SET_TCPPORT,
+	ANDROID_WIFI_CMD_WFD_SET_MAX_TPUT,
+	ANDROID_WIFI_CMD_WFD_SET_DEVTYPE,
+	ANDROID_WIFI_CMD_HOSTAPD_SET_MACADDR_ACL,
+	ANDROID_WIFI_CMD_HOSTAPD_ACL_ADD_STA,
+	ANDROID_WIFI_CMD_HOSTAPD_ACL_REMOVE_STA,
+#if defined(CONFIG_GTK_OL) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0))
+	ANDROID_WIFI_CMD_GTK_REKEY_OFFLOAD,
+#endif /* CONFIG_GTK_OL */
+	ANDROID_WIFI_CMD_P2P_DISABLE,
+	ANDROID_WIFI_CMD_SET_AEK,
+	ANDROID_WIFI_CMD_EXT_AUTH_STATUS,
+	ANDROID_WIFI_CMD_DRIVERVERSION,
+#ifdef PRIVATE_R
+	ANDROID_WIFI_CMD_ROKU_FIND_REMOTE,
+#endif
+	ANDROID_WIFI_CMD_MAX
+};
+
+int rtw_android_cmdstr_to_num(char *cmdstr);
+int rtw_android_priv_cmd(struct net_device *net, struct ifreq *ifr, int cmd);
+
+#if defined(RTW_ENABLE_WIFI_CONTROL_FUNC)
+int rtw_android_wifictrl_func_add(void);
+void rtw_android_wifictrl_func_del(void);
+void *wl_android_prealloc(int section, unsigned long size);
+
+int wifi_get_irq_number(unsigned long *irq_flags_ptr);
+int wifi_set_power(int on, unsigned long msec);
+int wifi_get_mac_addr(unsigned char *buf);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0))
+void *wifi_get_country_code(char *ccode, u32 flags);
+#else /* Linux kernel < 3.18 */
+void *wifi_get_country_code(char *ccode);
+#endif /* Linux kernel < 3.18 */
+#else
+static inline int rtw_android_wifictrl_func_add(void)
+{
+	return 0;
+}
+static inline void rtw_android_wifictrl_func_del(void) {}
+#endif /* defined(RTW_ENABLE_WIFI_CONTROL_FUNC) */
+
+
+#endif /* __RTW_ANDROID_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ap.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ap.h
new file mode 100644
index 000000000000..fadd1b223ef7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ap.h
@@ -0,0 +1,180 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_AP_H_
+#define __RTW_AP_H_
+
+
+#ifdef CONFIG_AP_MODE
+
+/* external function */
+extern void rtw_indicate_sta_assoc_event(_adapter *padapter, struct sta_info *psta);
+extern void rtw_indicate_sta_disassoc_event(_adapter *padapter, struct sta_info *psta);
+
+
+void init_mlme_ap_info(_adapter *padapter);
+void free_mlme_ap_info(_adapter *padapter);
+u8 rtw_set_tim_ie(u8 dtim_cnt, u8 dtim_period
+	, const u8 *tim_bmp, u8 tim_bmp_len, u8 *tim_ie);
+/* void update_BCNTIM(_adapter *padapter); */
+void rtw_add_bcn_ie(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 index, u8 *data, u8 len);
+void rtw_remove_bcn_ie(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 index);
+void rtw_add_bcn_ie_ex(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 eid_ex, u8 *data, u8 len);
+void rtw_remove_bcn_ie_ex(_adapter *padapter, WLAN_BSSID_EX *pnetwork, u8 index, u8* pindex_ex, u8 index_ex_len);
+void _update_beacon(_adapter *padapter,struct _ADAPTER_LINK *padapter_link,
+		u8 ie_id, u8 *oui, u8 tx, u8 flags, const char *tag);
+#define rtw_update_beacon(adapter, adapter_link, ie_id, oui, tx, flags) _update_beacon((adapter), (adapter_link), (ie_id), (oui), (tx), (flags), __func__)
+/*rtw_update_beacon - (flags) can set to normal enqueue (0) and RTW_CMDF_WAIT_ACK enqueue.
+ (flags) = RTW_CMDF_DIRECTLY  is not currently implemented, it will do normal enqueue.*/
+
+void expire_timeout_post_chk(_adapter *padapter, u32 ap_chk_sta_bmp);
+void expire_timeout_chk(_adapter *padapter, u32 *ap_chk_sta_bmp);
+void update_sta_info_apmode(_adapter *padapter, struct sta_info *psta);
+void rtw_start_bss_hdl_after_chbw_decided(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+void rtw_core_ap_prepare(_adapter *padapter, struct createbss_parm *parm);
+int rtw_check_beacon_data(_adapter *padapter, u8 *pbuf,  int len);
+void rtw_ap_restore_network(_adapter *padapter);
+
+#if CONFIG_RTW_MACADDR_ACL
+void rtw_macaddr_acl_init(_adapter *adapter, u8 period);
+void rtw_macaddr_acl_deinit(_adapter *adapter, u8 period);
+void rtw_macaddr_acl_clear(_adapter *adapter, u8 period);
+void rtw_set_macaddr_acl(_adapter *adapter, u8 period, int mode);
+int rtw_acl_add_sta(_adapter *adapter, u8 period, const u8 *addr);
+int rtw_acl_remove_sta(_adapter *adapter, u8 period, const u8 *addr);
+#endif /* CONFIG_RTW_MACADDR_ACL */
+
+u8 rtw_ap_set_sta_key(_adapter *adapter, const u8 *addr, u8 alg, const u8 *key, u8 keyid, u8 gk);
+u8 rtw_ap_set_pairwise_key(_adapter *padapter, struct sta_info *psta);
+int rtw_ap_set_group_key(_adapter *padapter,struct _ADAPTER_LINK *padapter_link,
+			u8 *key, u8 alg, int keyid);
+int rtw_ap_set_wep_key(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *key, u8 keylen, int keyid, u8 set_tx);
+
+#ifdef CONFIG_NATIVEAP_MLME
+void associated_clients_update(_adapter *padapter, u8 updated, u32 sta_info_type);
+void bss_cap_update_on_sta_join(_adapter *padapter, struct sta_info *psta);
+u8 bss_cap_update_on_sta_leave(_adapter *padapter, struct sta_info *psta);
+void sta_info_update(_adapter *padapter, struct sta_info *psta);
+u8 ap_free_sta(_adapter *padapter, struct sta_info *psta, bool active, u8 subtype, u16 reason, bool enqueue);
+int rtw_sta_flush(_adapter *padapter, bool enqueue);
+void start_ap_mode(_adapter *padapter);
+void stop_ap_mode(_adapter *padapter);
+#endif
+
+void rtw_ap_update_clients_rainfo(struct _ADAPTER *a, u8 flag);
+void rtw_ap_update_bss_bchbw(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+				WLAN_BSSID_EX *bss, struct rtw_chan_def *chandef);
+u8 rtw_ap_bchbw_decision(_adapter *adapter, u8 ifbmp, u8 excl_ifbmp, s8 req_band
+	, s16 req_ch, s8 req_bw, s8 req_offset, struct rtw_chan_def *chdef);
+
+#ifdef CONFIG_AUTO_AP_MODE
+void rtw_auto_ap_rx_msg_dump(_adapter *padapter, union recv_frame *precv_frame, u8 *ehdr_pos);
+extern void rtw_start_auto_ap(_adapter *adapter);
+#endif /* CONFIG_AUTO_AP_MODE */
+
+void rtw_ap_parse_sta_capability(_adapter *adapter, struct sta_info *sta, u8 *cap);
+u16 rtw_ap_parse_sta_supported_rates(_adapter *adapter, struct sta_info *sta, u8 *tlv_ies, u16 tlv_ies_len);
+u16 rtw_ap_parse_sta_security_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems);
+void rtw_ap_parse_sta_wmm_ie(_adapter *adapter, struct sta_info *sta, u8 *tlv_ies, u16 tlv_ies_len);
+void rtw_ap_parse_sta_ht_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems);
+void rtw_ap_parse_sta_vht_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems);
+void rtw_ap_parse_sta_he_ie(_adapter *adapter, struct sta_info *sta, struct rtw_ieee802_11_elems *elems);
+void rtw_ap_parse_sta_multi_ap_ie(_adapter *adapter, struct sta_info *sta, u8 *ies, int ies_len);
+
+
+void rtw_core_ap_start(_adapter *padapter, struct createbss_parm *parm);
+void rtw_core_ap_swch_start(_adapter *padapter, struct createbss_parm *parm);
+
+/* b2u flags */
+#define RTW_AP_B2U_ALL		BIT0
+#define RTW_AP_B2U_GA_UCAST	BIT1 /* WDS group addressed unicast frame, forward only */
+#define RTW_AP_B2U_BCAST	BIT2
+#define RTW_AP_B2U_IP_MCAST	BIT3
+
+#define rtw_ap_src_b2u_policy_chk(flags, da) ( \
+	(flags & RTW_AP_B2U_ALL) \
+	|| ((flags & RTW_AP_B2U_BCAST) && is_broadcast_mac_addr(da)) \
+	|| ((flags & RTW_AP_B2U_IP_MCAST) && (IP_MCAST_MAC(da) || ICMPV6_MCAST_MAC(da))) \
+	)
+
+#define rtw_ap_fwd_b2u_policy_chk(flags, da, gaucst) ( \
+	(flags & RTW_AP_B2U_ALL) \
+	|| ((flags & RTW_AP_B2U_GA_UCAST) && gaucst) \
+	|| ((flags & RTW_AP_B2U_BCAST) && is_broadcast_mac_addr(da)) \
+	|| ((flags & RTW_AP_B2U_IP_MCAST) && (IP_MCAST_MAC(da) || ICMPV6_MCAST_MAC(da))) \
+	)
+
+void dump_ap_b2u_flags(void *sel, _adapter *adapter);
+
+int rtw_ap_addr_resolve(_adapter *adapter, u16 os_qid, struct xmit_frame *xframe, struct sk_buff *pkt, _list *f_list);
+int rtw_ap_rx_data_validate_hdr(_adapter *adapter, union recv_frame *rframe, struct sta_info **sta);
+int rtw_ap_rx_msdu_act_check(union recv_frame *rframe
+	, const u8 *da, const u8 *sa
+	, u8 *msdu, enum rtw_rx_llc_hdl llc_hdl
+	, struct xmit_frame **fwd_frame, _list *f_list);
+
+#ifdef CONFIG_BMC_TX_RATE_SELECT
+void rtw_update_bmc_sta_tx_rate(_adapter *adapter);
+#endif
+
+void rtw_process_ht_action_smps(_adapter *padapter, u8 *ta, u8 ctrl_field);
+void rtw_process_public_act_bsscoex(_adapter *padapter, u8 *pframe, uint frame_len);
+#ifdef CONFIG_80211N_HT
+int rtw_ht_operation_update(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+#endif /* CONFIG_80211N_HT */
+u8 rtw_ap_sta_states_check(_adapter *adapter);
+
+void rtw_ap_set_sta_wmode(_adapter *padapter, struct sta_info *sta);
+
+#if defined(CONFIG_RTW_ACS) && defined(WKARD_ACS)
+void rtw_acs_start(_adapter *padapter);
+void rtw_acs_stop(_adapter *padapter);
+#endif /* defined(CONFIG_RTW_ACS) && defined(WKARD_ACS) */
+void rtw_ap_set_edca(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, enum rtw_ac ac, u32 param);
+
+#define CMD_APSTART_ACQUIRE BIT0
+#define CMD_APSTOP_ACQUIRE BIT1
+#define	CMD_APSTOP_STARTED BIT2
+
+enum rtw_phl_status rtw_ap_start_cmd(struct cmd_obj *p);
+enum rtw_phl_status rtw_ap_stop_cmd(struct cmd_obj *p);
+void rtw_ap_stop_set_state(struct _ADAPTER *a, u8 st);
+int rtw_ap_stop_wait(struct _ADAPTER *a);
+enum rtw_phl_status rtw_ap_add_del_sta_cmd(struct _ADAPTER *padapter);
+enum rtw_phl_status rtw_free_bcn_entry(struct _ADAPTER *padapter);
+
+bool rtw_add_del_sta_cmd_check(struct _ADAPTER *padapter, unsigned char *MacAddr, bool add_sta);
+void rtw_cmd_ap_add_del_sta_req_init(struct _ADAPTER *padapter);
+void rtw_cmd_ap_add_del_sta_req_free(struct _ADAPTER *padapter);
+void rtw_update_probe_rsp_basic_rate_and_ext(struct xmit_frame *xframe);
+
+#if CONFIG_AP_REGU_FORBID
+bool rtw_ap_link_regu_forbid_apply(struct _ADAPTER_LINK *alink, bool forbid, bool enqueue);
+bool rtw_ap_link_regu_forbid_update(struct _ADAPTER_LINK *alink, bool enqueue);
+#else
+static inline
+bool rtw_ap_link_regu_forbid_apply(struct _ADAPTER_LINK *alink, bool forbid, bool enqueue)
+{
+	return false;
+}
+static inline
+bool rtw_ap_link_regu_forbid_update(struct _ADAPTER_LINK *alink, bool enqueue)
+{
+	return false;
+}
+#endif
+
+#endif /* end of CONFIG_AP_MODE */
+#endif /*__RTW_AP_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_beamforming.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_beamforming.h
new file mode 100644
index 000000000000..14ccc3982f63
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_beamforming.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_BEAMFORMING_H_
+#define __RTW_BEAMFORMING_H_
+
+#ifdef CONFIG_BEAMFORMING
+
+#define	BEAMFORMING_HT_BEAMFORMER_ENABLE	BIT(0)	/*Declare sta support beamformer*/
+#define	BEAMFORMING_HT_BEAMFORMEE_ENABLE	BIT(1)	/*Declare sta support beamformee*/
+#define	BEAMFORMING_HT_BEAMFORMER_TEST		BIT(2)	/*Transmiting Beamforming no matter the target supports it or not*/
+#define	BEAMFORMING_HT_BEAMFORMER_STEER_NUM		(BIT(4)|BIT(5))		/*Sta Bfer's capability*/
+#define	BEAMFORMING_HT_BEAMFORMEE_CHNL_EST_CAP	(BIT(6)|BIT(7))		/*Sta BFee's capability*/
+
+#define	BEAMFORMING_VHT_BEAMFORMER_ENABLE	BIT(0)	/*Declare sta support beamformer*/
+#define	BEAMFORMING_VHT_BEAMFORMEE_ENABLE	BIT(1)	/*Declare sta support beamformee*/
+#define	BEAMFORMING_VHT_MU_MIMO_AP_ENABLE	BIT(2)	/*Declare sta support MU beamformer*/
+#define	BEAMFORMING_VHT_MU_MIMO_STA_ENABLE	BIT(3)	/*Declare sta support MU beamformer*/
+#define	BEAMFORMING_VHT_BEAMFORMER_TEST		BIT(4)	/*Transmiting Beamforming no matter the target supports it or not*/
+#define	BEAMFORMING_VHT_BEAMFORMER_STS_CAP		(BIT(8)|BIT(9)|BIT(10))		/*Sta BFee's capability*/
+#define	BEAMFORMING_VHT_BEAMFORMEE_SOUND_DIM	(BIT(12)|BIT(13)|BIT(14))	/*Sta Bfer's capability*/
+
+#define BEAMFORMING_HE_BEAMFORMER_ENABLE        BIT(0)  /*Declare sta support beamformer*/
+#define BEAMFORMING_HE_BEAMFORMEE_ENABLE        BIT(1)  /*Declare sta support beamformee*/
+#define BEAMFORMING_HE_MU_MIMO_AP_ENABLE        BIT(2)  /*Declare sta support MU beamformer*/
+#define BEAMFORMING_HE_MU_MIMO_STA_ENABLE       BIT(3)  /*Declare sta support MU beamformer*/
+#define BEAMFORMING_HE_BEAMFORMER_TEST          BIT(4)  /*Transmiting Beamforming no matter the target supports it or not*/
+#define BEAMFORMING_HE_BEAMFORMER_STS_CAP               (BIT(8)|BIT(9)|BIT(10))         /*Sta BFee's capability*/
+#define BEAMFORMING_HE_BEAMFORMEE_SOUND_DIM     (BIT(12)|BIT(13)|BIT(14))       /*Sta Bfer's capability*/
+
+void rtw_core_bf_watchdog(_adapter *padapter);
+
+int rtw_bf_get_vht_gid_mgnt_packet(struct _ADAPTER *a, union recv_frame *rframe);
+#define rtw_beamforming_get_vht_gid_mgnt_frame		rtw_bf_get_vht_gid_mgnt_packet
+
+#endif
+#endif /*__RTW_BEAMFORMING_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_br_ext.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_br_ext.h
new file mode 100644
index 000000000000..177b270c8304
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_br_ext.h
@@ -0,0 +1,69 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_BR_EXT_H_
+#define _RTW_BR_EXT_H_
+
+#if 1	/* rtw_wifi_driver */
+#define CL_IPV6_PASS	1
+#define MACADDRLEN		6
+#define _DEBUG_ERR		RTW_INFO
+#define _DEBUG_INFO		/* RTW_INFO */
+#define DEBUG_WARN		RTW_INFO
+#define DEBUG_INFO		/* RTW_INFO */
+#define DEBUG_ERR		RTW_INFO
+/* #define GET_MY_HWADDR		((GET_MIB(priv))->dot11OperationEntry.hwaddr) */
+#define GET_MY_HWADDR(padapter)		(adapter_mac_addr(padapter))
+#endif /* rtw_wifi_driver */
+
+#define NAT25_HASH_BITS		4
+#define NAT25_HASH_SIZE		(1 << NAT25_HASH_BITS)
+#define NAT25_AGEING_TIME	300
+
+#ifdef CL_IPV6_PASS
+	#define MAX_NETWORK_ADDR_LEN	17
+#else
+	#define MAX_NETWORK_ADDR_LEN	11
+#endif
+
+struct nat25_network_db_entry {
+	struct nat25_network_db_entry	*next_hash;
+	struct nat25_network_db_entry	**pprev_hash;
+	ATOMIC_T						use_count;
+	unsigned char					macAddr[6];
+	unsigned long					ageing_timer;
+	unsigned char				networkAddr[MAX_NETWORK_ADDR_LEN];
+};
+
+enum NAT25_METHOD {
+	NAT25_MIN,
+	NAT25_CHECK,
+	NAT25_INSERT,
+	NAT25_LOOKUP,
+	NAT25_PARSE,
+	NAT25_MAX
+};
+
+struct br_ext_info {
+	unsigned int	nat25_disable;
+	unsigned int	macclone_enable;
+	unsigned int	dhcp_bcst_disable;
+	int		addPPPoETag;		/* 1: Add PPPoE relay-SID, 0: disable */
+	unsigned char	nat25_dmzMac[MACADDRLEN];
+	unsigned int	nat25sc_disable;
+};
+
+void nat25_db_cleanup(_adapter *priv);
+
+#endif /* _RTW_BR_EXT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_btc.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_btc.h
new file mode 100644
index 000000000000..5885a145e93a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_btc.h
@@ -0,0 +1,37 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2013 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifdef CONFIG_BTC
+
+#ifndef __RTW_BTC_H__
+#define __RTW_BTC_H__
+
+enum rtw_core_btc_cmd_id {
+	RTW_CORE_BTC_CMD_TRXSS_LMT,
+	RTW_CORE_BTC_CMD_TRXSS_NO_LMT,
+	RTW_CORE_BTC_CMD_MAX
+};
+
+enum rtw_btc_esoc_type rtw_btc_update_ext_soc_type(u8 esoc_type);
+
+u8 rtw_core_btc_hdl(_adapter *padapter, enum rtw_core_btc_cmd_id btc_cmd_id);
+u8 rtw_core_btc_cmd(_adapter *padapter, enum rtw_core_btc_cmd_id btc_cmd_id, u8 flags);
+
+#ifdef CONFIG_BTC_TRXSS_CHG
+u8 rtw_btc_trxss_chg_hdl(struct dvobj_priv *dvobj, struct phl_msg *msg, u16 evt_id);
+#endif
+
+#endif /* __RTW_BTC_H__ */
+#endif /* CONFIG_BTC */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_byteorder.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_byteorder.h
new file mode 100644
index 000000000000..8ac3b9284d56
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_byteorder.h
@@ -0,0 +1,33 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTL871X_BYTEORDER_H_
+#define _RTL871X_BYTEORDER_H_
+
+
+#if defined(CONFIG_LITTLE_ENDIAN) && defined (CONFIG_BIG_ENDIAN)
+	#error "Shall be CONFIG_LITTLE_ENDIAN or CONFIG_BIG_ENDIAN, but not both!\n"
+#endif
+
+#if defined(CONFIG_LITTLE_ENDIAN)
+	#ifndef CONFIG_PLATFORM_MSTAR389
+		#include <byteorder/little_endian.h>
+	#endif
+#elif defined (CONFIG_BIG_ENDIAN)
+	#include <byteorder/big_endian.h>
+#else
+	#  error "Must be LITTLE/BIG Endian Host"
+#endif
+
+#endif /* _RTL871X_BYTEORDER_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_cfg.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_cfg.h
new file mode 100644
index 000000000000..be5e7deabf44
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_cfg.h
@@ -0,0 +1,57 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_CFG_H_
+#define _RTW_CFG_H_
+
+enum rtw_pwr_by_rate_setting {
+	RTW_PW_BY_RATE_ON = 0,
+	RTW_PW_BY_RATE_ALL_SAME = 1
+};
+
+enum rtw_pwr_limit_type {
+	RTW_PWLMT_BY_EFUSE = 0,
+	RTW_PWLMT_DISABLE = 1,
+	RTW_PWBYRATE_AND_PWLMT = 2
+};
+
+#if defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW)
+enum rtw_ps_proto_type {
+	RTW_PS_PROTO_LISTEN_BEACON_MODE = 0,
+	RTW_PS_PROTO_MAX
+};
+void rtw_update_lps_proto_cap_setting(enum rtw_ps_proto_type proto, u8 mode,
+				      u8 para, struct rtw_ps_cap_t *ps_cap,
+				      bool is_wowlan);
+void rtw_update_lps_proto_registry_setting(enum rtw_ps_proto_type proto,
+					   u8 mode, u8 para,
+					   struct registry_priv *registry,
+					   bool is_wowlan);
+#endif /* defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW) */
+
+u8 rtw_load_dvobj_registry(struct dvobj_priv *dvobj);
+uint rtw_load_registry(_adapter *adapter);
+
+void rtw_core_update_default_setting (struct dvobj_priv *dvobj);
+
+#define RTW_ADAPTIVITY_EN_DISABLE 0
+#define RTW_ADAPTIVITY_EN_ENABLE 1
+#define RTW_ADAPTIVITY_EN_AUTO 2
+
+#define RTW_ADAPTIVITY_MODE_NORMAL 0
+#define RTW_ADAPTIVITY_MODE_CARRIER_SENSE 1
+
+void rtw_cfg_adaptivity_config_msg(void *sel, _adapter *adapter);
+
+#endif /*_RTW_CFG_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_cmd.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_cmd.h
new file mode 100644
index 000000000000..4f39b4f33147
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_cmd.h
@@ -0,0 +1,639 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_CMD_H_
+#define __RTW_CMD_H_
+
+#define C2H_MEM_SZ (16*1024)
+
+#define MAX_EVTSZ	1024
+
+struct cmd_obj {
+	_adapter *padapter;
+	/*
+	* band_idx, could be used when padapter is not up and operated on specific hwband
+	* >= HW_BAND_MAX: not specified (get hwband by adapter/alink)
+	*/
+	u8 band_idx;
+
+	u16	cmdcode;
+	u32 cmd_token;
+	u8	res;
+	u8	*parmbuf;
+	u32	cmdsz;
+	u8	*rsp;
+	u32	rspsz;
+	struct submit_ctx *sctx;
+	/*
+	* buffer allocated by handler/callback in cmd_thread, will
+	* 1. got and accessed by caller or
+	* 2. freed by .sctx_rsp_buf_free when caller timeout
+	*/
+	void *sctx_rsp_buf;
+	/* set to function to free sctx_rsp_buf when caller is timeout */
+	void (*sctx_rsp_buf_free)(void *);
+	u8 no_io;
+	/* _sema 	cmd_sem; */
+	_list	list;
+};
+
+#define CMD_OBJ_SET_HWBAND(cmdobj, hwband) do { (cmdobj)->band_idx = hwband; } while (0)
+
+/* cmd flags */
+enum {
+	RTW_CMDF_DIRECTLY = BIT0,
+	RTW_CMDF_WAIT_ACK = BIT1,
+};
+
+struct cmd_priv {
+	u8	cmd_seq;
+	u32	cmd_issued_cnt;
+
+	struct dvobj_priv *dvobj;
+	_mutex sctx_mutex;
+	ATOMIC_T event_seq;
+	u32 evt_done_cnt;
+
+	#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	_queue	cmd_queue;
+	_sema	cmd_queue_sema;
+	_sema	start_cmdthread_sema;
+	ATOMIC_T cmdthd_running;
+	#endif
+};
+
+struct back_op_param {
+	unsigned int off_ch_dur;
+	unsigned int off_ch_ext_dur; /* extend when MGNT_TX */
+	unsigned int on_ch_dur;
+};
+
+#define init_h2fwcmd_w_parm_no_rsp(pcmd, pparm, code) \
+	do {\
+		_rtw_init_listhead(&pcmd->list);\
+		CMD_OBJ_SET_HWBAND(pcmd, HW_BAND_MAX);\
+		pcmd->cmdcode = code;\
+		pcmd->parmbuf = (u8 *)(pparm);\
+		pcmd->cmdsz = sizeof (*pparm);\
+		pcmd->rsp = NULL;\
+		pcmd->rspsz = 0;\
+	} while (0)
+
+#define init_h2fwcmd_w_parm_no_parm_rsp(pcmd, code) \
+	do {\
+		_rtw_init_listhead(&pcmd->list);\
+		CMD_OBJ_SET_HWBAND(pcmd, HW_BAND_MAX);\
+		pcmd->cmdcode = code;\
+		pcmd->parmbuf = NULL;\
+		pcmd->cmdsz = 0;\
+		pcmd->rsp = NULL;\
+		pcmd->rspsz = 0;\
+	} while (0)
+
+struct P2P_PS_Offload_t {
+	u8 Offload_En:1;
+	u8 role:1; /* 1: Owner, 0: Client */
+	u8 CTWindow_En:1;
+	u8 NoA0_En:1;
+	u8 NoA1_En:1;
+	u8 AllStaSleep:1; /* Only valid in Owner */
+	u8 discovery:1;
+	u8 rsvd:1;
+#ifdef CONFIG_P2P_PS_NOA_USE_MACID_SLEEP
+	u8 p2p_macid:7;
+	u8 disable_close_rf:1; /*1: not close RF but just pause p2p_macid when NoA duration*/
+#endif /* CONFIG_P2P_PS_NOA_USE_MACID_SLEEP */
+};
+
+struct P2P_PS_CTWPeriod_t {
+	u8 CTWPeriod;	/* TU */
+};
+
+#ifdef CONFIG_P2P_WOWLAN
+
+struct P2P_WoWlan_Offload_t {
+	u8 Disconnect_Wkup_Drv:1;
+	u8 role:2;
+	u8 Wps_Config[2];
+};
+
+#endif /* CONFIG_P2P_WOWLAN */
+
+extern u32 rtw_enqueue_cmd(struct cmd_priv *pcmdpriv, struct cmd_obj *obj);
+
+extern void rtw_free_cmd_obj(struct cmd_obj *pcmd);
+
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+void rtw_stop_cmd_thread(_adapter *adapter);
+thread_return rtw_cmd_thread(thread_context context);
+#endif
+
+u32 rtw_init_cmd_priv(struct dvobj_priv *dvobj);
+void rtw_free_cmd_priv(struct dvobj_priv *dvobj);
+
+#ifdef CONFIG_IOCTL_CFG80211
+u8 rtw_mgnt_tx_cmd(_adapter *adapter, u8 tx_band, u8 tx_ch, u8 no_cck, const u8 *buf, size_t len, int wait_ack, u8 flags);
+struct mgnt_tx_parm {
+	u8 tx_band;
+	u8 tx_ch;
+	u8 no_cck;
+	const u8 *buf;
+	size_t len;
+	int wait_ack;
+};
+#endif
+
+enum rtw_drvextra_cmd_id {
+	NONE_WK_CID, /*MCC_CMD_WK_CID*/
+	#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+	DYNAMIC_CHK_WK_CID,
+	#endif
+	DM_CTRL_WK_CID,
+	PBC_POLLING_WK_CID,
+	LPS_CTRL_WK_CID,
+	ANT_SELECT_WK_CID,
+	P2P_PS_WK_CID,
+	CHECK_HIQ_WK_CID,/* for softap mode, check hi queue if empty */
+	C2H_WK_CID,
+	RESET_SECURITYPRIV, /* add for CONFIG_IEEE80211W, none 11w also can use */
+	FREE_ASSOC_RESOURCES, /* add for CONFIG_IEEE80211W, none 11w also can use */
+	DM_IN_LPS_WK_CID,
+	LPS_CHANGE_DTIM_CID,
+	DFS_RADAR_DETECT_WK_CID,
+	DFS_RADAR_DETECT_EN_DEC_WK_CID,
+	SESSION_TRACKER_WK_CID,
+	TEST_H2C_CID,
+	MP_CMD_WK_CID,
+	CUSTOMER_STR_WK_CID,
+	MGNT_TX_WK_CID,
+	REQ_PER_CMD_WK_CID,
+	SSMPS_WK_CID,
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	TXSS_WK_CID,
+#endif
+	AC_PARM_CMD_WK_CID,
+#ifdef CONFIG_AP_MODE
+	STOP_AP_WK_CID,
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	TBTX_CONTROL_TX_WK_CID,
+#endif
+	TSF_SYNC_DONE_WK_CID,
+#ifdef PRIVATE_R
+	FIND_REMOTE_WK_CID,
+#ifdef CONFIG_P2P
+	HIDE_SSID_WK_CID,
+#endif
+#endif
+#ifdef CONFIG_BTC
+	CORE_BTC_CID,
+#endif
+	MAX_WK_CID
+};
+
+enum LPS_CTRL_TYPE {
+	LPS_CTRL_SCAN = 0,
+	LPS_CTRL_JOINBSS = 1,
+	LPS_CTRL_CONNECT = 2,
+	LPS_CTRL_DISCONNECT = 3,
+	LPS_CTRL_SPECIAL_PACKET = 4,
+	LPS_CTRL_LEAVE = 5,
+	LPS_CTRL_TRAFFIC_BUSY = 6,
+	LPS_CTRL_TX_TRAFFIC_LEAVE = 7,
+	LPS_CTRL_RX_TRAFFIC_LEAVE = 8,
+	LPS_CTRL_ENTER = 9,
+	LPS_CTRL_LEAVE_CFG80211_PWRMGMT = 10,
+	LPS_CTRL_LEAVE_SET_OPTION = 11,
+};
+
+enum STAKEY_TYPE {
+	GROUP_KEY		= 0,
+	UNICAST_KEY		= 1,
+	TDLS_KEY		= 2,
+};
+
+enum RFINTFS {
+	SWSI,
+	HWSI,
+	HWPI,
+};
+
+/*
+Caller Mode: Infra, Ad-Hoc
+
+Notes: To join the specified bss
+
+Command Event Mode
+
+*/
+struct joinbss_parm {
+	WLAN_BSSID_EX network;
+};
+
+/*
+Caller Mode: Infra, Ad-HoC(C)
+
+Notes: To disconnect the current associated BSS
+
+Command Mode
+
+*/
+struct disconnect_parm {
+	u32 deauth_timeout_ms;
+};
+
+/*
+Caller Mode: AP, Ad-HoC(M)
+
+Notes: To create a BSS
+
+Command Mode
+*/
+struct createbss_parm {
+	bool adhoc;
+
+	/* used by AP/Mesh mode now */
+	u8 ifbmp;
+	u8 excl_ifbmp;
+	s8 req_band;
+	s16 req_ch;
+	s8 req_bw;
+	s8 req_offset;
+	u8 ifbmp_ch_changed;
+	u8 ch_to_set;
+	u8 offset_to_set;
+	u8 bw_to_set;
+	u8 is_change_chbw;
+	#ifdef CONFIG_DBCC_SUPPORT
+	u8 is_trigger_dbcc;
+	#endif
+};
+
+
+struct	setopmode_parm {
+	u8	mode;
+	u8	rsvd[3];
+};
+
+
+/*
+Caller Mode: Any
+
+Notes: To set the auth type of RTL8711. open/shared/802.1x
+
+Command Mode
+
+*/
+struct setauth_parm {
+	u8 mode;  /* 0: legacy open, 1: legacy shared 2: 802.1x */
+	u8 _1x;   /* 0: PSK, 1: TLS */
+	u8 rsvd[2];
+};
+
+/*
+Caller Mode: Infra
+
+a. algorithm: wep40, wep104, tkip & aes
+b. keytype: grp key/unicast key
+c. key contents
+
+when shared key ==> keyid is the camid
+when 802.1x ==> keyid [0:1] ==> grp key
+when 802.1x ==> keyid > 2 ==> unicast key
+
+*/
+struct setkey_parm {
+	u8	algorithm;	/* encryption algorithm, could be none, wep40, TKIP, CCMP, wep104 */
+	u8	keyid;
+	u8	set_tx;		/* 1: main tx key for wep. 0: other key. */
+	u8	key[32];	/* this could be 40 or 104 */
+};
+
+/*
+When in AP or Ad-Hoc mode, this is used to
+allocate an sw/hw entry for a newly associated sta.
+
+Command
+
+when shared key ==> algorithm/keyid
+
+*/
+struct set_stakey_parm {
+	u8 addr[ETH_ALEN];
+	u8 algorithm;
+	u8 keyid;
+	u8 key[32];
+	u8 gk;
+};
+
+struct set_stakey_rsp {
+	u8	addr[ETH_ALEN];
+	u8	keyid;
+	u8	rsvd;
+};
+
+struct Tx_Beacon_param {
+	WLAN_BSSID_EX network;
+};
+
+/*
+	Notes: This command is used for H2C/C2H loopback testing
+
+	mac[0] == 0
+	==> CMD mode, return H2C_SUCCESS.
+	The following condition must be ture under CMD mode
+		mac[1] == mac[4], mac[2] == mac[3], mac[0]=mac[5]= 0;
+		s0 == 0x1234, s1 == 0xabcd, w0 == 0x78563412, w1 == 0x5aa5def7;
+		s2 == (b1 << 8 | b0);
+
+	mac[0] == 1
+	==> CMD_RSP mode, return H2C_SUCCESS_RSP
+
+	The rsp layout shall be:
+	rsp:			parm:
+		mac[0]  =   mac[5];
+		mac[1]  =   mac[4];
+		mac[2]  =   mac[3];
+		mac[3]  =   mac[2];
+		mac[4]  =   mac[1];
+		mac[5]  =   mac[0];
+		s0		=   s1;
+		s1		=   swap16(s0);
+		w0		=	swap32(w1);
+		b0		=	b1
+		s2		=	s0 + s1
+		b1		=	b0
+		w1		=	w0
+
+	mac[0] ==	2
+	==> CMD_EVENT mode, return	H2C_SUCCESS
+	The event layout shall be:
+	event:			parm:
+		mac[0]  =   mac[5];
+		mac[1]  =   mac[4];
+		mac[2]  =   event's sequence number, starting from 1 to parm's marc[3]
+		mac[3]  =   mac[2];
+		mac[4]  =   mac[1];
+		mac[5]  =   mac[0];
+		s0		=   swap16(s0) - event.mac[2];
+		s1		=   s1 + event.mac[2];
+		w0		=	swap32(w0);
+		b0		=	b1
+		s2		=	s0 + event.mac[2]
+		b1		=	b0
+		w1		=	swap32(w1) - event.mac[2];
+
+		parm->mac[3] is the total event counts that host requested.
+
+
+	event will be the same with the cmd's param.
+
+*/
+
+/* CMD param Formart for driver extra cmd handler */
+struct drvextra_cmd_parm {
+	struct _ADAPTER_LINK *adapter_link;
+	int ec_id; /* extra cmd id */
+	int type; /* Can use this field as the type id or command size */
+	int size; /* buffer size */
+	unsigned char *pbuf;
+};
+
+/*------------------- Below are used for RF/BB tunning ---------------------*/
+struct addBaReq_parm {
+	unsigned int tid;
+	u8	addr[ETH_ALEN];
+};
+
+struct addBaRsp_parm {
+	unsigned int tid;
+	unsigned int start_seq;
+	u8 addr[ETH_ALEN];
+	u8 status;
+	u16 size;
+	struct ADDBA_request preq;
+};
+
+struct set_ch_parm {
+	enum band_type band;
+	u8 ch;
+	u8 bw;
+	u8 ch_offset;
+	enum rfk_tri_type rt_type;
+	u8 link_idx;
+};
+
+
+struct LedBlink_param {
+	void *pLed;
+};
+
+struct TDLSoption_param {
+	u8 addr[ETH_ALEN];
+	u8 option;
+};
+
+struct RunInThread_param {
+	void (*func)(void *);
+	void *context;
+};
+
+
+#define GEN_CMD_CODE(cmd)	cmd ## _CMD_
+
+
+/*
+
+Result:
+0x00: success
+0x01: sucess, and check Response.
+0x02: cmd ignored due to duplicated sequcne number
+0x03: cmd dropped due to invalid cmd code
+0x04: reserved.
+
+*/
+
+#define H2C_RSP_OFFSET			512
+
+#define H2C_SUCCESS			0x00
+#define H2C_SUCCESS_RSP			0x01
+#define H2C_DUPLICATED			0x02
+#define H2C_DROPPED			0x03
+#define H2C_PARAMETERS_ERROR		0x04
+#define H2C_REJECTED			0x05
+#define H2C_CMD_OVERFLOW		0x06
+#define H2C_RESERVED			0x07
+#define H2C_ENQ_HEAD			0x08
+#define H2C_ENQ_HEAD_FAIL		0x09
+#define H2C_CMD_FAIL			0x0A
+#define H2C_MEMORY				0x0B
+
+u8 rtw_create_ibss_cmd(_adapter *adapter, int flags);
+u8 rtw_startbss_cmd(_adapter *adapter, int flags);
+
+#define REQ_BAND_NONE		-1
+#define REQ_CH_NONE		-1
+#define REQ_CH_INT_INFO	-2
+#define REQ_BW_NONE		-1
+#define REQ_BW_ORI		-2
+#define REQ_OFFSET_NONE	-1
+
+u8 rtw_change_bss_bchbw_cmd(_adapter *adapter, int flags
+	, u32 iflbmp, u32 excl_iflbmp, s8 req_band, s16 req_ch, s8 req_bw, s8 req_offset);
+
+struct sta_info;
+extern u8 rtw_setstakey_cmd(_adapter  *padapter, struct sta_info *sta, u8 key_type, bool enqueue);
+extern u8 rtw_clearstakey_cmd(_adapter *padapter, struct sta_info *sta, u8 enqueue);
+
+extern u8 rtw_joinbss_cmd(_adapter  *padapter, struct wlan_network *pnetwork);
+u8 rtw_disassoc_cmd(_adapter *padapter, u32 deauth_timeout_ms, int flags);
+#ifdef CONFIG_AP_MODE
+u8 rtw_stop_ap_cmd(_adapter *adapter, u8 flags);
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+u8 rtw_tx_control_cmd(_adapter *adapter);
+#endif
+extern u8 rtw_setopmode_cmd(_adapter  *padapter, NDIS_802_11_NETWORK_INFRASTRUCTURE networktype, u8 flags);
+
+extern u8 rtw_addbareq_cmd(_adapter *padapter, u8 tid, u8 *addr);
+extern u8 rtw_addbarsp_cmd(_adapter *padapter, u8 *addr, u16 tid,
+			   struct ADDBA_request *paddba_req, u8 status, u16 size,
+			   u16 start_seq);
+extern u8 rtw_delba_cmd(struct _ADAPTER *a, u8 *addr, u16 tid);
+/* add for CONFIG_IEEE80211W, none 11w also can use */
+extern u8 rtw_reset_securitypriv_cmd(_adapter *padapter);
+extern u8 rtw_free_assoc_resources_cmd(_adapter *padapter, u8 lock_scanned_queue, int flags);
+#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+extern u8 rtw_dynamic_chk_wk_cmd(_adapter *adapter);
+#endif
+#ifdef PRIVATE_R
+extern u8 rtw_find_remote_wk_cmd(_adapter *adapter);
+#ifdef CONFIG_P2P
+extern u8 rtw_hide_ssid_wk_cmd(_adapter *adapter);
+#endif
+#endif
+
+#ifdef CONFIG_ANTENNA_DIVERSITY
+extern  u8 rtw_antenna_select_cmd(_adapter *padapter, u8 antenna, u8 enqueue);
+#endif
+
+#ifdef CONFIG_AP_MODE
+u8 rtw_chk_hi_queue_cmd(_adapter *padapter);
+#endif /* CONFIG_AP_MODE */
+
+u8 rtw_test_h2c_cmd(_adapter *adapter, u8 *buf, u8 len);
+#if defined(RTW_PHL_DBG_CMD)
+void core_cmd_phl_handler(_adapter *adapter, char *extra);
+void core_add_record(_adapter *adapter, u8 type, void *p);
+void phl_add_record(void *d, u8 type, void *p, u32 size);
+#endif
+
+u8 rtw_set_chbw_cmd(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				struct rtw_chan_def *chdef, u8 flags, enum rfk_tri_type rt_type);
+
+#ifdef CONFIG_RTW_LED_HANDLED_BY_CMD_THREAD
+u8 rtw_led_blink_cmd(_adapter *padapter, void *pLed);
+#endif
+extern u8 rtw_tdls_cmd(_adapter *padapter, u8 *addr, u8 option, u8 flags);
+
+u8 rtw_mp_cmd(_adapter *adapter, u8 mp_cmd_id, u8 flags);
+
+#ifdef CONFIG_RTW_CUSTOMER_STR
+u8 rtw_customer_str_req_cmd(_adapter *adapter);
+u8 rtw_customer_str_write_cmd(_adapter *adapter, const u8 *cstr);
+#endif
+
+u8 rtw_c2h_packet_wk_cmd(_adapter *adapter, u8 *c2h_evt, u16 length);
+
+u8 rtw_run_in_thread_cmd(_adapter *adapter, void (*func)(void *), void *context);
+u8 rtw_run_in_thread_cmd_wait(_adapter *adapter, void (*func)(void *), void *context, s32 timeout_ms);
+
+struct ssmps_cmd_parm {
+	struct sta_info *sta;
+	u8 smps;
+};
+u8 rtw_ssmps_wk_cmd(_adapter *adapter, struct sta_info *sta, u8 smps, u8 enqueue);
+
+u8 session_tracker_chk_cmd(_adapter *adapter, struct sta_info *sta);
+u8 session_tracker_add_cmd(_adapter *adapter, struct sta_info *sta, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port);
+u8 session_tracker_del_cmd(_adapter *adapter, struct sta_info *sta, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port);
+
+u8 set_txq_params_cmd(_adapter *adapter, struct _ADAPTER_LINK *padapter_link, u32 ac_parm, u8 ac_type);
+
+#if defined(CONFIG_RTW_MESH) && defined(RTW_PER_CMD_SUPPORT_FW)
+u8 rtw_req_per_cmd(_adapter * adapter);
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+u8 rtw_tbtx_chk_cmd(_adapter *adapter);
+u8 rtw_tbtx_token_dispatch_cmd(_adapter *adapter);
+#endif
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+struct txss_cmd_parm {
+	struct sta_info *sta;
+	bool tx_1ss;
+};
+
+void rtw_ctrl_txss_update(_adapter *adapter, struct sta_info *sta);
+u8 rtw_ctrl_txss(_adapter *adapter, struct sta_info *sta, bool tx_1ss);
+void rtw_ctrl_tx_ss_by_tp(_adapter *adapter);
+
+#ifdef DBG_CTRL_TXSS
+void dbg_ctrl_txss(_adapter *adapter, bool tx_1ss);
+#endif
+#endif
+
+u8 rtw_drvextra_cmd_hdl(_adapter *padapter, unsigned char *pbuf);
+
+extern void rtw_disassoc_cmd_callback(_adapter  *padapter, struct cmd_obj *pcmd);
+void rtw_create_ibss_post_hdl(_adapter *padapter, int status);
+extern void rtw_readtssi_cmdrsp_callback(_adapter	*padapter,  struct cmd_obj *pcmd);
+
+extern void rtw_getrttbl_cmdrsp_callback(_adapter  *padapter,  struct cmd_obj *pcmd);
+void rtw_run_cmd(_adapter *padapter, struct cmd_obj *pcmd, bool discard);
+u32 rtw_get_turbo_edca(_adapter *padapter, u8 aifs, u8 ecwmin, u8 ecwmax, u8 txop);
+
+enum rtw_cmd_id {
+	CMD_JOINBSS,
+	CMD_DISCONNECT,
+	CMD_CREATE_BSS,
+	CMD_SET_OPMODE,
+	CMD_SET_AUTH,
+	CMD_ADD_BAREQ,
+	CMD_SET_CHANNEL,
+	CMD_TX_BEACON,
+	CMD_SET_MLME_EVT,
+	CMD_SET_DRV_EXTRA,
+	CMD_SET_CHANPLAN,
+	CMD_LEDBLINK,
+	CMD_TDLS,
+	CMD_CHK_BMCSLEEPQ,
+	CMD_RUN_INTHREAD,
+	CMD_ADD_BARSP,
+	CMD_RM_POST_EVENT,
+	CMD_SET_MESH_PLINK_STATE,
+	CMD_DELBA,
+	CMD_GET_CHANPLAN,
+	CMD_TWT,
+	CMD_ID_MAX
+};
+
+#define CMD_FMT "cmd=%d,%d,%d"
+#define CMD_ARG(cmd) \
+	(cmd)->cmdcode, \
+	(cmd)->cmdcode == CMD_SET_DRV_EXTRA ? ((struct drvextra_cmd_parm *)(cmd)->parmbuf)->ec_id : ((cmd)->cmdcode == CMD_SET_MLME_EVT ? ((struct rtw_evt_header *)(cmd)->parmbuf)->id : 0), \
+	(cmd)->cmdcode == CMD_SET_DRV_EXTRA ? ((struct drvextra_cmd_parm *)(cmd)->parmbuf)->type : 0
+
+void rtw_dynamic_chk_wk_sw_hdl(_adapter *padapter);
+void rtw_dynamic_chk_wk_hw_hdl(_adapter *padapter);
+#endif /* _CMD_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_csa.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_csa.h
new file mode 100644
index 000000000000..8bc672c23e36
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_csa.h
@@ -0,0 +1,148 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_CSA_H_
+#define __RTW_CSA_H_
+
+#if CONFIG_DFS
+#define CSA_IE_LEN 3 /* Length of Channel Switch Announcement element */
+#define ECSA_IE_LEN 4 /* Length of Extended Channel Switch Announcement element */
+#define CSA_SWITCH_MODE 0
+#define CSA_NEW_CH 1
+#define CSA_SWITCH_COUNT 2
+#define MAX_CSA_CNT 10
+
+#define WAIT_BCN_TIMES 5
+
+#define CS_WR_DATA_LEN 5 /* Length of Channel Switch Wrapper element */
+
+#ifdef CONFIG_ECSA_PHL
+#define MCC_ECSA_DELAY_START_TIME 3000 /* ms */
+
+enum ecsa_state_t {
+	ECSA_ST_NONE,
+	ECSA_ST_SW_START, /* ready to switch */
+	ECSA_ST_SW_DONE /* switch done */
+};
+
+enum csa_trigger_type {
+	/* AP mode trigger channel switch via driver core layer */
+	CSA_AP_CORE_SWITCH_CH,
+
+	/* AP mode trigger channel switch via chan_switch command from hostapd or wpa_supplicant */
+	CSA_AP_CFG80211_SWITCH_CH,
+
+	/* STA mode receive CSA/ECSA IE from associating AP */
+	CSA_STA_RX_CSA_IE,
+
+	/* STA mode disconnect with DFS AP and DFS region not supported, so switching AP mode to non-DFS channel */
+	CSA_STA_DISCONNECT_ON_DFS
+};
+
+enum ecsa_mr_case {
+	ECSA_SINGLE_ROLE,
+	ECSA_MR_SCC_STA_AP,
+	ECSA_MR_UNSUPPORTED
+};
+
+enum core_layer_ecsa_type{
+	ECSA_CORE_TYPE_NONE = 0,
+	ECSA_CORE_TYPE_AP = 1,
+	ECSA_CORE_TYPE_STA = 2,
+	ECSA_CORE_TYPE_SCC_STA_AP = 3 /* Concurrent: one STA mode and one AP mode*/
+};
+
+struct core_ecsa_info {
+	enum ecsa_state_t state;
+	enum core_layer_ecsa_type core_ecsa_type;
+	enum phl_ecsa_start_reason ecsa_allow_case;
+	u32 ecsa_delay_time;
+
+	/* @channel_width defined in 802.11-2016, Table 9-252 VHT operation information subfields
+	* The format of the Wide Bandwidth Channel Switch subelement is the same as the
+	* Wide Bandwidth Channel Switch element (see 9.4.2.161) except for the following:
+	* A value 0 in the New Channel Width field signifies only a 40 MHz BSS bandwidth
+	* 0 for 40 MHz
+	* 1 for 80 MHz, 160 MHz or 80+80 MHz
+	* 2 for 160 MHz (deprecated)
+	* 3 for non-contiguous 80+80 MHz (deprecated)
+	* 255 for initial value, defined by driver
+	*/
+	u8 channel_width;
+	struct createbss_parm *bss_param;
+	struct rtw_phl_ecsa_param phl_ecsa_param;
+	struct _ADAPTER *trigger_iface; /* interface which trigger core layer ECSA flow */
+	struct _ADAPTER *execute_iface; /* interface which actually execute PHL ECSA FG */
+};
+
+__inline static void set_ecsa_state(struct core_ecsa_info *ecsa_info, enum ecsa_state_t state)
+{
+	ecsa_info->state = state;
+}
+
+__inline static bool check_ecsa_state(struct core_ecsa_info *ecsa_info, enum ecsa_state_t state)
+{
+	if (ecsa_info->state == state)
+		return _TRUE;
+	return _FALSE;
+}
+
+#define SET_ECSA_STATE(adapter, state) set_ecsa_state(&((adapter)->ecsa_info), (state))
+#define CHK_ECSA_STATE(adapter, state) check_ecsa_state(&((adapter)->ecsa_info), (state))
+
+void reset_ecsa_param(struct _ADAPTER *a);
+bool rtw_is_ecsa_enabled(struct _ADAPTER *a);
+bool rtw_mr_is_ecsa_running(struct _ADAPTER *a);
+bool rtw_hal_is_csa_support(struct _ADAPTER *a);
+void rtw_ecsa_update_sta_mlme(struct _ADAPTER_LINK *alink, u8 *pframe, u32 packet_len);
+void rtw_build_css_ie(struct _ADAPTER *a, struct rtw_phl_ecsa_param *ecsa_param);
+void rtw_build_sec_offset_ie(struct _ADAPTER *a, u8 seconday_offset);
+void rtw_build_wide_bw_cs_ie(struct _ADAPTER *a, struct rtw_chan_def new_chandef);
+void rtw_cfg80211_build_csa_beacon(
+	struct _ADAPTER *a,
+	struct cfg80211_csa_settings *params,
+	struct rtw_chan_def csa_chdef
+);
+void rtw_ecsa_update_probe_resp(struct xmit_frame *xframe);
+void rtw_ecsa_update_beacon(void *priv, struct rtw_wifi_role_t *role, struct rtw_wifi_role_link_t *rlink);
+bool rtw_ap_check_ecsa_allow(
+	void *priv,
+	struct rtw_wifi_role_t *role,
+	struct rtw_chan_def chan_def,
+	enum phl_ecsa_start_reason reason,
+#ifdef CONFIG_PHL_ECSA_EXTEND_OPTION
+	u32 *extend_option,
+#endif
+	u32 *delay_start_ms
+);
+void rtw_ecsa_mr_update_chan_info_by_role(
+	void *priv,
+	struct rtw_wifi_role_t *role,
+	struct rtw_wifi_role_link_t *rlink,
+	struct rtw_chan_def new_chan_def
+);
+bool rtw_ecsa_check_tx_resume_allow(void *priv, struct rtw_wifi_role_t *role);
+void rtw_ecsa_complete(void *priv, struct rtw_wifi_role_t *role);
+bool rtw_trigger_phl_ecsa_start(struct _ADAPTER *trigger_iface, enum csa_trigger_type trigger_type);
+bool rtw_hal_trigger_csa_start(struct _ADAPTER *a, enum csa_trigger_type trigger_type,
+	u8 csa_mode, u8 ecsa_op_class, u8 csa_switch_cnt,
+	u8 csa_band, u8 csa_ch, u8 csa_bw, u8 csa_offset,
+	u8 csa_ch_width, u8 csa_ch_freq_seg0, u8 csa_ch_freq_seg1);
+bool rtw_hal_dfs_trigger_csa(struct _ADAPTER *a,
+	enum csa_trigger_type trigger_type,
+	u8 band_idx, u8 u_ch, u8 u_bw, u8 u_offset);
+#endif /* CONFIG_ECSA_PHL */
+#endif /* CONFIG_DFS */
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_debug.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_debug.h
new file mode 100644
index 000000000000..b237d4e36b8d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_debug.h
@@ -0,0 +1,821 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_DEBUG_H__
+#define __RTW_DEBUG_H__
+
+/* driver log level*/
+enum {
+	_DRV_NONE_ = 0,
+	_DRV_ALWAYS_ = 1,
+	_DRV_ERR_ = 2,
+	_DRV_WARNING_ = 3,
+	_DRV_INFO_ = 4,
+	_DRV_DEBUG_ = 5,
+	_DRV_MAX_ = 6
+};
+
+#define DRIVER_PREFIX "RTW: "
+
+#define RTW_PRINT(x, ...) do {} while (0)
+#define RTW_ERR(x, ...) do {} while (0)
+#define RTW_WARN(x,...) do {} while (0)
+#define RTW_WARN_LMT(x,...) do {} while (0)
+#define RTW_INFO(x,...) do {} while (0)
+#define RTW_DBG(x,...) do {} while (0)
+#define RTW_PRINT_SEL(x,...) do {} while (0)
+#define _RTW_PRINT(x, ...) do {} while (0)
+#define _RTW_ERR(x, ...) do {} while (0)
+#define _RTW_WARN(x,...) do {} while (0)
+#define _RTW_INFO(x,...) do {} while (0)
+#define _RTW_DBG(x,...) do {} while (0)
+#define _RTW_PRINT_SEL(x,...) do {} while (0)
+
+#define RTW_INFO_DUMP(_TitleString, _HexData, _HexDataLen) do {} while (0)
+#define RTW_DBG_DUMP(_TitleString, _HexData, _HexDataLen) do {} while (0)
+#define RTW_PRINT_DUMP(_TitleString, _HexData, _HexDataLen) do {} while (0)
+
+#define RTW_DBG_EXPR(EXPR) do {} while (0)
+
+#define RTW_DBGDUMP 0 /* 'stream' for _dbgdump */
+
+
+
+#undef _dbgdump
+#undef _seqdump
+
+#if defined(PLATFORM_LINUX)
+	#ifdef DBG_THREAD_PID
+	#define T_PID_FMT	"(%5u) "
+	#define T_PID_ARG	current->pid
+	#else /* !DBG_THREAD_PID */
+	#define T_PID_FMT	"%s"
+	#define T_PID_ARG	""
+	#endif /* !DBG_THREAD_PID */
+
+	#ifdef DBG_CPU_INFO
+	#define CPU_INFO_FMT	"[%u] "
+	#define CPU_INFO_ARG	task_cpu(current)
+	#else /* !DBG_CPU_INFO */
+	#define CPU_INFO_FMT	"%s"
+	#define CPU_INFO_ARG	""
+	#endif /* !DBG_CPU_INFO */
+
+	/* Extra information in prefix */
+	#define EX_INFO_FMT	T_PID_FMT CPU_INFO_FMT
+	#define EX_INFO_ARG	T_PID_ARG, CPU_INFO_ARG
+
+	#ifdef RTW_APPEND_LOGLEVEL
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31))
+	#define KERN_DEFAULT	""
+	#endif
+
+	/* Log level mapping from driver to Linux kernel */
+	#ifndef RTW_LL_PRINT
+	#define RTW_LL_PRINT	KERN_CRIT
+	#endif
+	#ifndef RTW_LL_ERR
+	#define RTW_LL_ERR	KERN_ERR
+	#endif
+	#ifndef RTW_LL_WARN
+	#define RTW_LL_WARN	KERN_WARNING
+	#endif
+	#ifndef RTW_LL_INFO
+	#define RTW_LL_INFO	KERN_INFO
+	#endif
+	#ifndef RTW_LL_DBG
+	#define RTW_LL_DBG	KERN_DEBUG
+	#endif
+	#ifndef RTW_LL_DEFAULT
+	#define RTW_LL_DEFAULT	KERN_DEFAULT
+	#endif
+
+	#define _dbgdump_lv(lv, fmt, arg...)		printk(lv EX_INFO_FMT fmt, EX_INFO_ARG, ##arg)
+	#define _dbgdump_print(fmt, arg...)		_dbgdump_lv(RTW_LL_PRINT, fmt, ##arg)
+	#define _dbgdump_err(fmt, arg...)		_dbgdump_lv(RTW_LL_ERR, fmt, ##arg)
+	#define _dbgdump_warn(fmt, arg...)		_dbgdump_lv(RTW_LL_WARN, fmt, ##arg)
+	#define _dbgdump_info(fmt, arg...)		_dbgdump_lv(RTW_LL_INFO, fmt, ##arg)
+	#define _dbgdump_dbg(fmt, arg...)		_dbgdump_lv(RTW_LL_DBG, fmt, ##arg)
+	#define _dbgdump(fmt, arg...)			_dbgdump_lv(RTW_LL_DEFAULT, fmt, ##arg)
+
+	#define _dbgdump_lv_lmt(lv, fmt, arg...)	printk_ratelimited(lv EX_INFO_FMT fmt, EX_INFO_ARG, ##arg)
+	#define _dbgdump_print_lmt(fmt, arg...)		_dbgdump_lv_lmt(RTW_LL_PRINT, fmt, ##arg)
+	#define _dbgdump_err_lmt(fmt, arg...)		_dbgdump_lv_lmt(RTW_LL_ERR, fmt, ##arg)
+	#define _dbgdump_warn_lmt(fmt, arg...)		_dbgdump_lv_lmt(RTW_LL_WARN, fmt, ##arg)
+	#define _dbgdump_info_lmt(fmt, arg...)		_dbgdump_lv_lmt(RTW_LL_INFO, fmt, ##arg)
+	#define _dbgdump_dbg_lmt(fmt, arg...)		_dbgdump_lv_lmt(RTW_LL_DBG, fmt, ##arg)
+	#define _dbgdump_lmt(fmt, arg...)		_dbgdump_lv_lmt(RTW_LL_DEFAULT, fmt, ##arg)
+
+	#else /* !RTW_APPEND_LOGLEVEL */
+
+	#define _dbgdump(fmt, arg...)	printk(EX_INFO_FMT fmt, EX_INFO_ARG, ##arg)
+	#define _dbgdump_print		_dbgdump
+	#define _dbgdump_err		_dbgdump
+	#define _dbgdump_warn		_dbgdump
+	#define _dbgdump_info		_dbgdump
+	#define _dbgdump_dbg		_dbgdump
+
+	#define _dbgdump_lmt(fmt, arg...)	printk_ratelimited(EX_INFO_FMT fmt, EX_INFO_ARG, ##arg)
+	#define _dbgdump_print_lmt		_dbgdump_lmt
+	#define _dbgdump_err_lmt		_dbgdump_lmt
+	#define _dbgdump_warn_lmt		_dbgdump_lmt
+	#define _dbgdump_info_lmt		_dbgdump_lmt
+	#define _dbgdump_dbg_lmt		_dbgdump_lmt
+	#endif /* !RTW_APPEND_LOGLEVEL */
+
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24))
+	#define KERN_CONT
+	#endif
+	/*
+	 * _dbgdump_c() default use KERN_CONT flag, so it would not print
+	 * messages with a new line. Usually use it if you want to continue
+	 * a line.
+	 */
+	#define _dbgdump_c(fmt, arg...)	printk(KERN_CONT fmt, ##arg)
+	#define _dbgdump_c_lmt(fmt, arg...)	printk_ratelimited(KERN_CONT fmt, ##arg)
+	#define _seqdump seq_printf
+#elif defined(PLATFORM_FREEBSD)
+	#define _dbgdump printf
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24))
+	#define KERN_CONT
+	#endif
+	#define _dbgdump_c(fmt, arg...)	printf(KERN_CONT fmt, ##arg)
+	#define _seqdump(sel, fmt, arg...) _dbgdump(fmt, ##arg)
+#endif
+
+void RTW_BUF_DUMP_SEL(uint _loglevel, void *sel, u8 *_titlestring,
+			bool _idx_show, const u8 *_hexdata, int _hexdatalen);
+void RTW_BUF_DUMP_SEL_ALWAYS(void *sel, u8 *_titlestring,
+			bool _idx_show, const u8 *_hexdata, int _hexdatalen);
+
+#ifdef CONFIG_RTW_DEBUG
+
+/*#ifndef _OS_INTFS_C_*/
+extern uint rtw_drv_log_level;
+/*#endif*/
+
+#if defined(_dbgdump)
+
+/* with driver-defined prefix */
+#undef RTW_PRINT
+#define RTW_PRINT(fmt, arg...)     \
+	do {\
+		if (_DRV_ALWAYS_ <= rtw_drv_log_level) {\
+			_dbgdump_print(DRIVER_PREFIX fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_PRINT_LMT
+#define RTW_PRINT_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_ALWAYS_ <= rtw_drv_log_level) {\
+			_dbgdump_print_lmt(DRIVER_PREFIX fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_ERR
+#define RTW_ERR(fmt, arg...)     \
+	do {\
+		if (_DRV_ERR_ <= rtw_drv_log_level) {\
+			_dbgdump_err(DRIVER_PREFIX "ERROR " fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_ERR_LMT
+#define RTW_ERR_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_ERR_ <= rtw_drv_log_level) {\
+			_dbgdump_err_lmt(DRIVER_PREFIX "ERROR " fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_WARN
+#define RTW_WARN(fmt, arg...)     \
+	do {\
+		if (_DRV_WARNING_ <= rtw_drv_log_level) {\
+			_dbgdump_warn(DRIVER_PREFIX "WARN " fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_WARN_LMT
+#define RTW_WARN_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_WARNING_ <= rtw_drv_log_level) {\
+			_dbgdump_warn_lmt(DRIVER_PREFIX "WARN " fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_INFO
+#define RTW_INFO(fmt, arg...)     \
+	do {\
+		if (_DRV_INFO_ <= rtw_drv_log_level) {\
+			_dbgdump_info(DRIVER_PREFIX fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_INFO_LMT
+#define RTW_INFO_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_INFO_ <= rtw_drv_log_level) {\
+			_dbgdump_info_lmt(DRIVER_PREFIX fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_DBG
+#define RTW_DBG(fmt, arg...)     \
+	do {\
+		if (_DRV_DEBUG_ <= rtw_drv_log_level) {\
+			_dbgdump_dbg(DRIVER_PREFIX fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_DBG_LMT
+#define RTW_DBG_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_DEBUG_ <= rtw_drv_log_level) {\
+			_dbgdump_dbg_lmt(DRIVER_PREFIX fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef RTW_INFO_DUMP
+#define RTW_INFO_DUMP(_TitleString, _HexData, _HexDataLen)	\
+	RTW_BUF_DUMP_SEL(_DRV_INFO_, RTW_DBGDUMP, _TitleString, _FALSE, _HexData, _HexDataLen)
+
+#undef RTW_DBG_DUMP
+#define RTW_DBG_DUMP(_TitleString, _HexData, _HexDataLen)	\
+	RTW_BUF_DUMP_SEL(_DRV_DEBUG_, RTW_DBGDUMP, _TitleString, _FALSE, _HexData, _HexDataLen)
+
+
+#undef RTW_PRINT_DUMP
+#define RTW_PRINT_DUMP(_TitleString, _HexData, _HexDataLen)	\
+	RTW_BUF_DUMP_SEL(_DRV_ALWAYS_, RTW_DBGDUMP, _TitleString, _FALSE, _HexData, _HexDataLen)
+
+/* without driver-defined prefix */
+#undef _RTW_PRINT
+#define _RTW_PRINT(fmt, arg...)     \
+	do {\
+		if (_DRV_ALWAYS_ <= rtw_drv_log_level) {\
+			_dbgdump_c(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_PRINT_LMT
+#define _RTW_PRINT_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_ALWAYS_ <= rtw_drv_log_level) {\
+			_dbgdump_c_lmt(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_ERR
+#define _RTW_ERR(fmt, arg...)     \
+	do {\
+		if (_DRV_ERR_ <= rtw_drv_log_level) {\
+			_dbgdump_c(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_ERR_LMT
+#define _RTW_ERR_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_ERR_ <= rtw_drv_log_level) {\
+			_dbgdump_c_lmt(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_WARN
+#define _RTW_WARN(fmt, arg...)     \
+	do {\
+		if (_DRV_WARNING_ <= rtw_drv_log_level) {\
+			_dbgdump_c(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_WARN_LMT
+#define _RTW_WARN_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_WARNING_ <= rtw_drv_log_level) {\
+			_dbgdump_c_lmt(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_INFO
+#define _RTW_INFO(fmt, arg...)     \
+	do {\
+		if (_DRV_INFO_ <= rtw_drv_log_level) {\
+			_dbgdump_c(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_INFO_LMT
+#define _RTW_INFO_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_INFO_ <= rtw_drv_log_level) {\
+			_dbgdump_c_lmt(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_DBG
+#define _RTW_DBG(fmt, arg...)     \
+	do {\
+		if (_DRV_DEBUG_ <= rtw_drv_log_level) {\
+			_dbgdump_c(fmt, ##arg);\
+		} \
+	} while (0)
+
+#undef _RTW_DBG_LMT
+#define _RTW_DBG_LMT(fmt, arg...)     \
+	do {\
+		if (_DRV_DEBUG_ <= rtw_drv_log_level) {\
+			_dbgdump_c_lmt(fmt, ##arg);\
+		} \
+	} while (0)
+
+
+/* other debug APIs */
+#undef RTW_DBG_EXPR
+#define RTW_DBG_EXPR(EXPR) do { if (_DRV_DEBUG_ <= rtw_drv_log_level) EXPR; } while (0)
+
+#endif /* defined(_dbgdump) */
+#endif /* CONFIG_RTW_DEBUG */
+
+
+#if defined(_seqdump)
+/* dump message to selected 'stream' with driver-defined prefix */
+#undef RTW_PRINT_SEL
+#define RTW_PRINT_SEL(sel, fmt, arg...) \
+	do {\
+		if (sel == RTW_DBGDUMP)\
+			RTW_PRINT(fmt, ##arg); \
+		else {\
+			_seqdump(sel, fmt, ##arg) /*rtw_warn_on(1)*/; \
+		} \
+	} while (0)
+
+/* dump message to selected 'stream' */
+#undef _RTW_PRINT_SEL
+#define _RTW_PRINT_SEL(sel, fmt, arg...) \
+	do {\
+		if (sel == RTW_DBGDUMP)\
+			_RTW_PRINT(fmt, ##arg); \
+		else {\
+			_seqdump(sel, fmt, ##arg) /*rtw_warn_on(1)*/; \
+		} \
+	} while (0)
+
+/* dump message to selected 'stream' */
+#undef RTW_DUMP_SEL
+#define RTW_DUMP_SEL(sel, _HexData, _HexDataLen) \
+	RTW_BUF_DUMP_SEL(_DRV_ALWAYS_, sel, NULL, _FALSE, _HexData, _HexDataLen)
+
+#define RTW_MAP_DUMP_SEL(sel, _TitleString, _HexData, _HexDataLen) \
+	RTW_BUF_DUMP_SEL(_DRV_ALWAYS_, sel, _TitleString, _TRUE, _HexData, _HexDataLen)
+
+#define RTW_MAP_DUMP_SEL_ALWAYS(sel, _TitleString, _HexData, _HexDataLen) \
+	RTW_BUF_DUMP_SEL_ALWAYS(sel, _TitleString, _TRUE, _HexData, _HexDataLen)
+
+#endif /* defined(_seqdump) */
+
+
+#ifdef CONFIG_DBG_COUNTER
+	#define DBG_COUNTER(counter) counter++
+#else
+	#define DBG_COUNTER(counter)
+#endif
+
+void dump_drv_version(void *sel);
+void dump_log_level(void *sel);
+void dump_drv_cfg(void *sel);
+
+void rtw_sink_rtp_seq_dbg(_adapter *adapter, u8 *ehdr_pos);
+
+#ifdef CONFIG_RECV_REORDERING_CTRL
+struct sta_info;
+void sta_rx_reorder_ctl_dump(void *sel, struct sta_info *sta);
+#endif
+struct dvobj_priv;
+void dump_tx_rate_bmp(void *sel, struct dvobj_priv *dvobj);
+void dump_adapters_status(void *sel, struct dvobj_priv *dvobj);
+
+bool rtw_del_rx_ampdu_test_trigger_no_tx_fail(void);
+u32 rtw_get_wait_hiq_empty_ms(void);
+void rtw_sta_linking_test_set_start(void);
+bool rtw_sta_linking_test_wait_done(void);
+bool rtw_sta_linking_test_force_fail(void);
+#ifdef CONFIG_AP_MODE
+u16 rtw_ap_linking_test_force_auth_fail(void);
+u16 rtw_ap_linking_test_force_asoc_fail(void);
+#endif
+
+#ifdef CONFIG_PROC_DEBUG
+ssize_t proc_set_write_reg(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_read_reg(struct seq_file *m, void *v);
+ssize_t proc_set_read_reg(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+int proc_get_ignore_go_and_low_rssi_in_scan(struct seq_file *m, void *v);
+ssize_t proc_set_ignore_go_and_low_rssi_in_scan(struct file *file,
+	const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+
+ssize_t proc_set_mac_dbg_status_dump(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_fwstate(struct seq_file *m, void *v);
+int proc_get_sec_info(struct seq_file *m, void *v);
+int proc_get_mlmext_state(struct seq_file *m, void *v);
+#ifdef CONFIG_LAYER2_ROAMING
+int proc_get_roam_flags(struct seq_file *m, void *v);
+ssize_t proc_set_roam_flags(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_roam_param(struct seq_file *m, void *v);
+ssize_t proc_set_roam_param(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+ssize_t proc_set_roam_tgt_addr(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif /* CONFIG_LAYER2_ROAMING */
+int proc_get_qos_option(struct seq_file *m, void *v);
+int proc_get_ht_option(struct seq_file *m, void *v);
+int proc_get_rf_info(struct seq_file *m, void *v);
+int proc_get_scan_param(struct seq_file *m, void *v);
+ssize_t proc_set_scan_param(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_scan_abort(struct seq_file *m, void *v);
+ssize_t proc_set_scan_abort(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_survey_info(struct seq_file *m, void *v);
+ssize_t proc_set_survey_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_ap_info(struct seq_file *m, void *v);
+#ifdef PRIVATE_R
+int proc_get_infra_ap(struct seq_file *m, void *v);
+#endif /* PRIVATE_R */
+ssize_t proc_reset_trx_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_trx_info(struct seq_file *m, void *v);
+ssize_t proc_set_tx_power_offset(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_tx_power_offset(struct seq_file *m, void *v);
+int proc_get_rate_ctl(struct seq_file *m, void *v);
+int proc_get_wifi_spec(struct seq_file *m, void *v);
+ssize_t proc_set_rate_ctl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_bw_ctl(struct seq_file *m, void *v);
+ssize_t proc_set_bw_ctl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#ifdef DBG_RX_COUNTER_DUMP
+int proc_get_rx_cnt_dump(struct seq_file *m, void *v);
+ssize_t proc_set_rx_cnt_dump(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+
+#ifdef CONFIG_AP_MODE
+int proc_get_bmc_tx_rate(struct seq_file *m, void *v);
+ssize_t proc_set_bmc_tx_rate(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif /*CONFIG_AP_MODE*/
+
+int proc_get_ps_dbg_info(struct seq_file *m, void *v);
+ssize_t proc_set_ps_dbg_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+ssize_t proc_set_del_rx_ampdu_test_case(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+ssize_t proc_set_wait_hiq_empty(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+ssize_t proc_set_sta_linking_test(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#ifdef CONFIG_AP_MODE
+ssize_t proc_set_ap_linking_test(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+
+struct sta_info;
+int update_sta_stats(_adapter *adapter, struct sta_info *psta);
+int proc_get_rx_stat(struct seq_file *m, void *v);
+int proc_get_tx_stat(struct seq_file *m, void *v);
+int proc_get_sta_tx_stat(struct seq_file *m, void *v);
+ssize_t proc_set_sta_tx_stat(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#ifdef CONFIG_AP_MODE
+int proc_get_sta_active_time(struct seq_file *m, void *v);
+int proc_get_all_sta_info(struct seq_file *m, void *v);
+#endif /* CONFIG_AP_MODE */
+
+#ifdef DBG_MEMORY_LEAK
+int proc_get_malloc_cnt(struct seq_file *m, void *v);
+#endif /* DBG_MEMORY_LEAK */
+
+#ifdef CONFIG_FIND_BEST_CHANNEL
+int proc_get_best_channel(struct seq_file *m, void *v);
+ssize_t proc_set_best_channel(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif /* CONFIG_FIND_BEST_CHANNEL */
+
+int proc_get_trx_info_debug(struct seq_file *m, void *v);
+
+#ifdef CONFIG_HUAWEI_PROC
+int proc_get_huawei_trx_info(struct seq_file *m, void *v);
+#endif
+
+int proc_get_rx_signal(struct seq_file *m, void *v);
+ssize_t proc_set_rx_signal(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_hw_status(struct seq_file *m, void *v);
+ssize_t proc_set_hw_status(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_mac_rptbuf(struct seq_file *m, void *v);
+
+#ifdef CONFIG_80211N_HT
+int proc_get_ht_enable(struct seq_file *m, void *v);
+ssize_t proc_set_ht_enable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_bw_mode(struct seq_file *m, void *v);
+ssize_t proc_set_bw_mode(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_ampdu_enable(struct seq_file *m, void *v);
+ssize_t proc_set_ampdu_enable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+void dump_regsty_rx_ampdu_size_limit(void *sel, _adapter *adapter);
+int proc_get_rx_ampdu(struct seq_file *m, void *v);
+ssize_t proc_set_rx_ampdu(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+void rtw_dump_dft_phy_cap(void *sel, _adapter *adapter);
+void rtw_get_dft_phy_cap(void *sel, _adapter *adapter);
+void rtw_dump_drv_phy_cap(void *sel, _adapter *adapter);
+void rtw_dump_macaddr(void *sel, _adapter *adapter);
+
+int proc_get_rx_stbc(struct seq_file *m, void *v);
+ssize_t proc_set_rx_stbc(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_stbc_cap(struct seq_file *m, void *v);
+ssize_t proc_set_stbc_cap(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_ldpc_cap(struct seq_file *m, void *v);
+ssize_t proc_set_ldpc_cap(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#ifdef CONFIG_BEAMFORMING
+int proc_get_txbf_cap(struct seq_file *m, void *v);
+ssize_t proc_set_txbf_cap(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+int proc_get_rx_ampdu_factor(struct seq_file *m, void *v);
+ssize_t proc_set_rx_ampdu_factor(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_tx_ampdu_num(struct seq_file *m, void *v);
+ssize_t proc_set_tx_ampdu_num(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_rx_ampdu_density(struct seq_file *m, void *v);
+ssize_t proc_set_rx_ampdu_density(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_tx_ampdu_density(struct seq_file *m, void *v);
+ssize_t proc_set_tx_ampdu_density(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_tx_quick_addba_req(struct seq_file *m, void *v);
+ssize_t proc_set_tx_quick_addba_req(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#ifdef CONFIG_TX_AMSDU
+int proc_get_tx_amsdu(struct seq_file *m, void *v);
+ssize_t proc_set_tx_amsdu(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_tx_amsdu_rate(struct seq_file *m, void *v);
+ssize_t proc_set_tx_amsdu_rate(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+int proc_get_tx_max_agg_time(struct seq_file *m, void *v);
+ssize_t proc_set_tx_max_agg_time(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_80211AC_VHT
+int proc_get_vht_24g_enable(struct seq_file *m, void *v);
+ssize_t proc_set_vht_24g_enable(struct file *file, const char __user *buffer,
+				size_t count, loff_t *pos, void *data);
+#endif
+
+ssize_t proc_set_dyn_rrsr(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_dyn_rrsr(struct seq_file *m, void *v);
+
+#if 0
+int proc_get_two_path_rssi(struct seq_file *m, void *v);
+int proc_get_rssi_disp(struct seq_file *m, void *v);
+ssize_t proc_set_rssi_disp(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+
+#ifdef CONFIG_DBG_COUNTER
+int proc_get_rx_logs(struct seq_file *m, void *v);
+int proc_get_tx_logs(struct seq_file *m, void *v);
+int proc_get_int_logs(struct seq_file *m, void *v);
+#endif
+
+#ifdef CONFIG_PCI_HCI
+int proc_get_rx_ring(struct seq_file *m, void *v);
+int proc_get_tx_ring(struct seq_file *m, void *v);
+int proc_get_pci_aspm(struct seq_file *m, void *v);
+int proc_get_pci_conf_space(struct seq_file *m, void *v);
+ssize_t proc_set_pci_conf_space(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_pci_bridge_conf_space(struct seq_file *m, void *v);
+ssize_t proc_set_pci_bridge_conf_space(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+
+#ifdef DBG_TXBD_DESC_DUMP
+int proc_get_tx_ring_ext(struct seq_file *m, void *v);
+ssize_t proc_set_tx_ring_ext(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+#endif
+
+#ifdef CONFIG_WOWLAN
+int proc_get_wow_enable(struct seq_file *m, void *v);
+ssize_t proc_set_wow_enable(struct file *file, const char __user *buffer,
+		size_t count, loff_t *pos, void *data);
+int proc_get_pattern_info(struct seq_file *m, void *v);
+ssize_t proc_set_pattern_info(struct file *file, const char __user *buffer,
+		size_t count, loff_t *pos, void *data);
+int proc_get_wakeup_event(struct seq_file *m, void *v);
+ssize_t proc_set_wakeup_event(struct file *file, const char __user *buffer,
+		size_t count, loff_t *pos, void *data);
+int proc_get_wakeup_reason(struct seq_file *m, void *v);
+int proc_get_wake_indication(struct seq_file *m, void *v);
+#ifdef CONFIG_GPIO_WAKEUP
+int proc_get_wowlan_gpio_info(struct seq_file *m, void *v);
+ssize_t proc_set_wowlan_gpio_info(struct file *file, const char __user *buffer,
+		size_t count, loff_t *pos, void *data);
+ssize_t proc_set_wow_gpio_duration(struct file *file, const char __user *buffer,
+		size_t count, loff_t *pos, void *data);
+int proc_get_wow_gpio_duration(struct seq_file *m, void *v);
+#endif /*CONFIG_GPIO_WAKEUP*/
+#ifdef CONFIG_MDNS_OFFLOAD
+ssize_t proc_set_wow_mdns_resp(struct file *file, const char __user *buffer,
+		size_t count, loff_t *pos, void *data);
+int proc_get_wow_mdns_resp(struct seq_file *m, void *v);
+ssize_t proc_set_wow_mdns_match_criteria(struct file *file,
+		const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_wow_mdns_match_criteria(struct seq_file *m, void *v);
+ssize_t proc_set_wow_mdns_passthru_list(struct file *file,
+		const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_wow_mdns_passthru_list(struct seq_file *m, void *v);
+ssize_t proc_set_wow_mdns_offload_state(struct file *file,
+		const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_wow_mdns_offload_state(struct seq_file *m, void *v);
+ssize_t proc_set_wow_mdns_passthru_behavior(struct file *file,
+		const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_wow_mdns_passthru_behavior(struct seq_file *m, void *v);
+#endif /* CONFIG_MDNS_OFFLOAD */
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+int proc_get_wow_pw_info(struct seq_file *m, void *v);
+ssize_t proc_set_wow_pw_info(struct file *file, const char __user *buffer,
+			     size_t count, loff_t *pos, void *data);
+#endif
+
+#ifdef CONFIG_WOW_APF
+ssize_t proc_set_wow_apf(struct file *file, const char __user *buffer,
+		size_t count, loff_t *pos, void *data);
+int proc_get_wow_apf(struct seq_file *m, void *v);
+
+#endif
+
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_P2P_WOWLAN
+int proc_get_p2p_wowlan_info(struct seq_file *m, void *v);
+#endif /* CONFIG_P2P_WOWLAN */
+
+#ifdef CONFIG_POWER_SAVE
+int proc_get_ps_info(struct seq_file *m, void *v);
+ssize_t proc_set_ps_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#if defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW)
+ssize_t proc_set_ps_dtim(struct file *file, const char __user *buffer,
+			 size_t count, loff_t *pos, void *data);
+#endif /* defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW) */
+#endif /* CONFIG_POWER_SAVE */
+
+int proc_get_new_bcn_max(struct seq_file *m, void *v);
+ssize_t proc_set_new_bcn_max(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+#ifdef CONFIG_TDLS
+int proc_get_tdls_enable(struct seq_file *m, void *v);
+ssize_t proc_set_tdls_enable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_tdls_info(struct seq_file *m, void *v);
+#endif
+
+int proc_get_monitor(struct seq_file *m, void *v);
+ssize_t proc_set_monitor(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+#ifdef DBG_XMIT_BLOCK
+int proc_get_xmit_block(struct seq_file *m, void *v);
+ssize_t proc_set_xmit_block(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+
+#ifdef CONFIG_RTKM
+int proc_get_rtkm_info(struct seq_file *m, void *v);
+ssize_t proc_set_rtkm_info(struct file *file, const char __user *buffer,
+			   size_t count, loff_t *pos, void *data);
+#endif /* CONFIG_RTKM */
+
+#ifdef CONFIG_IEEE80211W
+ssize_t proc_set_tx_sa_query(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_tx_sa_query(struct seq_file *m, void *v);
+ssize_t proc_set_tx_deauth(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_tx_deauth(struct seq_file *m, void *v);
+ssize_t proc_set_tx_auth(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_tx_auth(struct seq_file *m, void *v);
+#endif /* CONFIG_IEEE80211W */
+
+#endif /* CONFIG_PROC_DEBUG */
+
+int proc_get_efuse_map(struct seq_file *m, void *v);
+ssize_t proc_set_efuse_map(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_ack_timeout(struct seq_file *m, void *v);
+ssize_t proc_set_ack_timeout(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+int proc_get_fw_info(struct seq_file *m, void *v);
+
+int proc_get_fw_offload(struct seq_file *m, void *v);
+ssize_t proc_set_fw_offload(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+
+#ifdef CONFIG_DBG_RF_CAL
+int proc_get_iqk_info(struct seq_file *m, void *v);
+ssize_t proc_set_iqk(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_lck_info(struct seq_file *m, void *v);
+ssize_t proc_set_lck(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif /*CONFIG_DBG_RF_CAL*/
+
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+ssize_t proc_set_txss_tp(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_txss_tp(struct seq_file *m, void *v);
+#ifdef DBG_CTRL_TXSS
+ssize_t proc_set_txss_ctrl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_txss_ctrl(struct seq_file *m, void *v);
+#endif
+#endif
+
+#ifdef CONFIG_SUPPORT_STATIC_SMPS
+ssize_t proc_set_smps(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_smps(struct seq_file *m, void *v);
+#endif
+
+int proc_get_defs_param(struct seq_file *m, void *v);
+ssize_t proc_set_defs_param(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+#ifdef RTW_DETECT_HANG
+ssize_t proc_set_hang_info(struct file *file, const char __user *buffer,
+			   size_t count, loff_t *pos, void *data);
+int proc_get_hang_info(struct seq_file *m, void *v);
+#endif
+
+int proc_get_disconnect_info(struct seq_file *m, void *v);
+ssize_t proc_set_disconnect_info(struct file *file, const char __user *buffer,
+				 size_t count, loff_t *pos, void *data);
+
+int proc_get_chan(struct seq_file *m, void *v);
+ssize_t proc_set_chan(struct file *file, const char __user *buffer,
+				size_t count, loff_t *pos, void *data);
+
+int proc_get_mr_test(struct seq_file *m, void *v);
+ssize_t proc_set_mr_test(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_deny_legacy(struct seq_file *m, void *v);
+ssize_t proc_set_deny_legacy(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_tx_ul_mu_disable(struct seq_file *m, void *v);
+ssize_t proc_set_tx_ul_mu_disable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_vcs(struct seq_file *m, void *v);
+ssize_t proc_set_vcs(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+int proc_get_cur_tsf(struct seq_file *m, void *v);
+int proc_get_cur_tsf_cmd_enqueue(struct seq_file *m, void *v);
+ssize_t proc_set_cur_tsf_cmd_enqueue(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+
+#ifdef CONFIG_RTW_CSI_CHANNEL_INFO
+int proc_get_csi_ch_info(struct seq_file *m, void *v);
+ssize_t proc_set_csi_ch_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+
+#define _drv_always_		1
+#define _drv_emerg_			2
+#define _drv_alert_			3
+#define _drv_crit_			4
+#define _drv_err_			5
+#define _drv_warning_		6
+#define _drv_notice_		7
+#define _drv_info_			8
+#define _drv_dump_			9
+#define _drv_debug_			10
+
+#define _module_rtl871x_xmit_c_		BIT(0)
+#define _module_xmit_osdep_c_		BIT(1)
+#define _module_rtl871x_recv_c_		BIT(2)
+#define _module_recv_osdep_c_		BIT(3)
+#define _module_rtl871x_mlme_c_		BIT(4)
+#define _module_mlme_osdep_c_		BIT(5)
+#define _module_rtl871x_sta_mgt_c_		BIT(6)
+#define _module_rtl871x_cmd_c_			BIT(7)
+#define _module_cmd_osdep_c_		BIT(8)
+#define _module_rtl871x_io_c_				BIT(9)
+#define _module_io_osdep_c_		BIT(10)
+#define _module_os_intfs_c_			BIT(11)
+#define _module_rtl871x_security_c_		BIT(12)
+#define _module_rtl871x_eeprom_c_			BIT(13)
+#define _module_hal_init_c_		BIT(14)
+#define _module_hci_hal_init_c_		BIT(15)
+#define _module_rtl871x_ioctl_c_		BIT(16)
+#define _module_rtl871x_ioctl_set_c_		BIT(17)
+#define _module_rtl871x_ioctl_query_c_	BIT(18)
+#define _module_rtl871x_pwrctrl_c_			BIT(19)
+#define _module_hci_intfs_c_			BIT(20)
+#define _module_hci_ops_c_			BIT(21)
+#define _module_osdep_service_c_			BIT(22)
+#define _module_mp_			BIT(23)
+#define _module_hci_ops_os_c_			BIT(24)
+#define _module_rtl871x_ioctl_os_c		BIT(25)
+#define _module_rtl8712_cmd_c_		BIT(26)
+/* #define _module_efuse_			BIT(27) */
+#define	_module_rtl8192c_xmit_c_ BIT(28)
+#define _module_hal_xmit_c_	BIT(28)
+#define _module_efuse_			BIT(29)
+#define _module_rtl8712_recv_c_		BIT(30)
+#define _module_rtl8712_led_c_		BIT(31)
+
+#endif /* __RTW_DEBUG_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_efuse.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_efuse.h
new file mode 100644
index 000000000000..6475c9f8f37e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_efuse.h
@@ -0,0 +1,139 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#if defined(CONFIG_MP_INCLUDED)
+#ifndef __RTW_EFUSE_H__
+#define __RTW_EFUSE_H__
+#include <drv_types.h>
+#include <rtw_mp.h>
+
+#define RTW_MAX_EFUSE_MAP_LEN 2048
+
+enum rtw_efuse_type {
+	RTW_EFUSE_WIFI = 0,
+	RTW_EFUSE_BT,
+	RTW_EFUSE_NONE,
+};
+
+/* PHL efuse command */
+enum rtw_efuse_phl_cmdid {
+	RTW_EFUSE_CMD_WIFI_READ = 0,
+	RTW_EFUSE_CMD_WIFI_WRITE = 1,
+	RTW_EFUSE_CMD_WIFI_UPDATE = 2,
+	RTW_EFUSE_CMD_WIFI_UPDATE_MAP = 3,
+	RTW_EFUSE_CMD_WIFI_GET_OFFSET_MASK = 4,
+	RTW_EFUSE_CMD_WIFI_GET_USAGE = 5,
+	RTW_EFUSE_CMD_BT_READ = 6,
+	RTW_EFUSE_CMD_BT_WRITE = 7,
+	RTW_EFUSE_CMD_BT_UPDATE = 8,
+	RTW_EFUSE_CMD_BT_UPDATE_MAP = 9,
+	RTW_EFUSE_CMD_BT_GET_OFFSET_MASK = 10,
+	RTW_EFUSE_CMD_BT_GET_USAGE = 11,
+	RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE = 12,
+	RTW_EFUSE_CMD_WIFI_GET_SIZE = 13,
+	RTW_EFUSE_CMD_WIFI_GET_AVL_SIZE = 14,
+	RTW_EFUSE_CMD_AUTOLOAD_STATUS = 15,
+	RTW_EFUSE_CMD_SHADOW_MAP2BUF = 16,
+	RTW_EFUSE_CMD_FILE_MAP_LOAD = 17,
+	RTW_EFUSE_CMD_FILE_MASK_LOAD = 18,
+	RTW_MP_EFUSE_CMD_GET_INFO = 19,
+	/* BT */
+	RTW_EFUSE_CMD_BT_GET_LOG_SIZE = 20,
+	RTW_EFUSE_CMD_BT_GET_SIZE = 21,
+	RTW_EFUSE_CMD_BT_GET_AVL_SIZE = 22,
+	RTW_EFUSE_CMD_BT_SHADOW_MAP2BUF = 23,
+	RTW_EFUSE_CMD_BT_FILE_MAP_LOAD = 24,
+	RTW_EFUSE_CMD_BT_FILE_MASK_LOAD = 25,
+	RTW_EFUSE_CMD_BT_READ_HIDDEN = 26,
+	RTW_EFUSE_CMD_BT_WRITE_HIDDEN = 27,
+	RTW_MP_EFUSE_CMD_WIFI_GET_MAP_FROM =28,
+	RTW_EFUSE_CMD_WIFI_GET_PHY_MAP = 29,
+	RTW_EFUSE_CMD_BT_GET_PHY_MAP = 30,
+	RTW_MP_EFUSE_CMD_WIFI_SET_RENEW = 31,
+	RTW_EFUSE_CMD_WIFI_GET_MASK_BUF = 32,
+	RTW_EFUSE_CMD_BT_GET_MASK_BUF = 33,
+	RTW_EFUSE_CMD_MAX,
+};
+
+struct rtw_efuse_phl_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+	u8 io_type;
+	u16 io_offset;
+	u32 io_value;
+	u8 autoload;
+	u8 pfile_path[200];
+	u16 buf_len;
+	u8 poutbuf[1536];
+};
+
+enum RTW_EFUSE_MAP_STATUS {
+	RTW_DEFAULT_MAP = 0,
+	RTW_HW_LOG_MAP = 1,
+	RTW_FILE_MAP = 2,
+	RTW_EFUSE_UNKNOWN,
+};
+
+#define RTW_EFUSE_FROM2STR(status)\
+(status == RTW_DEFAULT_MAP) ? "DEFAULT" :\
+(status == RTW_HW_LOG_MAP) ? "HW_LOG_EFUSE" :\
+(status == RTW_FILE_MAP) ? "FILE_EFUSE" :\
+"UNknow"
+
+s8 rtw_efuse_get_map_from(_adapter *padapter);
+
+u32 rtw_efuse_get_map_size(_adapter *padapter , u16 *size , enum rtw_efuse_phl_cmdid cmdid);
+
+u32 rtw_efuse_get_available_size(_adapter *padapter , u16 *size, u8 efuse_type);
+
+u8 rtw_efuse_read_map2shadow(_adapter *padapter, u8 efuse_type);
+
+u8 rtw_efuse_map_read(_adapter * adapter, u16 addr, u16 cnts, u8 *data, u8 efuse_type);
+
+u8 rtw_efuse_map_write(_adapter * adapter, u16 addr, u16 cnts, u8 *data, u8 efuse_type, u8 bpg);
+
+int rtw_ioctl_efuse_get(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra);
+
+int rtw_ioctl_efuse_set(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wdata, char *extra);
+
+int rtw_ioctl_efuse_file_map_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra);
+
+int rtw_ioctl_efuse_file_mask_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra);
+
+int rtw_ioctl_efuse_bt_file_map_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra);
+
+int rtw_ioctl_efuse_bt_file_mask_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra);
+
+u8 rtw_efuse_raw_map_read(_adapter * adapter, u16 addr,
+	            u16 cnts, u8 *data, u8 efuse_type);
+
+u8 rtw_efuse_bt_write_raw_hidden(_adapter * adapter, u16 addr,
+				u16 cnts, u8 *data);
+#endif
+
+#endif /*#if defined(CONFIG_MP_INCLUDED)*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_eht.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_eht.h
new file mode 100644
index 000000000000..91f0ef28f61a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_eht.h
@@ -0,0 +1,82 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_EHT_H_
+#define _RTW_EHT_H_
+
+struct mlo_priv {
+	u8	mlo_option;
+	u8	tid2link_nego;
+	/* ToDo */
+};
+
+struct eht_priv {
+	u8	eht_option;
+	/* ToDo */
+};
+
+#define EHT_ML_CONTROL_LEN 2
+
+/* Min length of common info in basic ML
+** Common info length field: 1 */
+#define EHT_BASIC_COMMON_INFO_MIN_LEN 1 + ETH_ALEN
+
+/* Min length of a legal basic ML element
+** Element ID + Element length fields: 2
+** Element ID extension field: 1 */
+#define EHT_ML_BASIC_MIN_LEN 2 + 1 + EHT_ML_CONTROL_LEN + EHT_BASIC_COMMON_INFO_MIN_LEN
+
+/* Min length of a legal probe req ML element
+** Element ID + Element length fields: 2
+** Element ID extension field: 1
+** Common info length field: 1 */
+#define EHT_ML_PROBEREQ_MIN_LEN 2 + 1 + EHT_ML_CONTROL_LEN + 1
+
+#define EHT_ML_STA_CONTROL_LEN 2
+
+void rtw_set_assoc_status(_adapter *padapter, struct rtw_phl_mld_t *pmld, u8 link_id, u16 status);
+
+/*
+  rtw_update_ies_with_inheritance(): API for core to update the current IEs with the inherited IEs
+  @ cur_ies: current IEs
+  @ cur_ies_len: length of the current IEs
+  @ inh_ies: the inherited IEs
+  @ inh_ies_len: length of the inherited IEs
+*/
+void rtw_update_ies_with_inheritance(u8 *cur_ies, u32 *cur_ies_len, u8 *inh_ies, u8 inh_ies_len);
+
+u32 rtw_parse_ml_ie(_adapter *padapter, u8 *ies, u32 ies_len,
+		struct rtw_phl_ml_element *ml_ele);
+
+/*
+  rtw_eht_ml_ies_handler(): API for core to parse ML related element in Assoc Rsp
+  @ ies: start point of IEs in Assoc Rsp
+  @ ies_len: length of the IEs in Assoc Rsp
+*/
+void rtw_eht_ml_ies_handler(struct _ADAPTER *padapter, struct _ADAPTER_LINK *padapter_link,
+				u8 *ies, u32 ies_len);
+
+/*
+  rtw_build_ml_ie(): API for core to build ML element
+  @ pkt_type: packet type, e.g., PACKET_AUTH, PACKET_ASSOC_REQUEST, etc
+  @ primary_ies: IEs built in the primary link
+  @ primary_ies_len: length of the IEs in the primary link
+*/
+u8 *rtw_build_ml_ies(_adapter *padapter, struct sta_info *psta, u8 *pframe, struct pkt_attrib *pattrib,
+			enum phl_packet_type pkt_type, u8 *primary_ies, u32 primary_ies_len, u8 reserved);
+
+u32 rtw_restructure_ml_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *in_ie, u8 *out_ie, uint in_len, uint *pout_len);
+
+#endif /* _RTW_EHT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_event.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_event.h
new file mode 100644
index 000000000000..1009c74dea52
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_event.h
@@ -0,0 +1,77 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_EVENT_H_
+#define _RTW_EVENT_H_
+
+/*
+Used to report a bss has been scanned
+
+*/
+struct survey_event	{
+	WLAN_BSSID_EX bss;
+};
+
+/*
+Used to report that the requested site survey has been done.
+
+bss_cnt indicates the number of bss that has been reported.
+
+
+*/
+struct surveydone_event {
+	unsigned int	bss_cnt;
+	u8 activate_ch_cnt;
+	u8 reason;
+	bool complete; /* scan done w/o aborting */
+	bool acs; /* aim to trigger channel selection */
+};
+
+/*
+Used to report the link result of joinning the given bss
+
+
+join_res:
+-1: authentication fail
+-2: association fail
+> 0: TID
+
+*/
+struct joinbss_event {
+	struct	wlan_network	network;
+};
+
+/*
+Used to report a given STA has joinned the created BSS.
+It is used in AP/Ad-HoC(M) mode.
+
+
+*/
+struct stassoc_event {
+	unsigned char macaddr[6];
+};
+
+struct stadel_event {
+	unsigned char macaddr[6];
+	unsigned char rsvd[2]; /* for reason */
+	unsigned char locally_generated;
+	int mac_id;
+};
+
+struct wmm_event {
+	unsigned char wmm;
+};
+
+
+#endif /* _WLANEVENT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm.h
new file mode 100644
index 000000000000..47d462450401
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm.h
@@ -0,0 +1,338 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_FSM_H__
+#define __RTW_FSM_H__
+
+#include "rtw_fsm_xxx.h" /* xxx_priv reference template */
+#include "rtw_fsm_rrm.h" /* rrm_priv */
+#include "rtw_fsm_wnm.h" /* btm_priv */
+
+u8 fsm_post_event_hdl(_adapter *padapter, u8 *pbuf);
+
+/* #define RTW_DEBUG_FSM */
+/* #define FSM_DBG_MEM_OVERWRITE */
+
+#ifdef FSM_DBG_MEM_OVERWRITE
+void *fsm_kmalloc(u32 sz);
+void fsm_kfree(void *ptr, u32 sz);
+#endif
+#define FSM_NAME_LEN		32
+#define CLOCK_UNIT              10 /* ms */
+
+/* event map
+ */
+#define FSM_EV_MASK	0xff00
+#define FSM_USR_EV_MASK	0x0100
+#define FSM_INT_EV_MASK	0x0200
+#define FSM_GBL_EV_MASK	0x0400
+#define FSM_EV_UNKNOWN	0xffff
+
+/* FSM EVENT */
+enum FSM_EV_ID {
+	/* Expose to all FSM service */
+	FSM_INT_EV_MASK_ = FSM_INT_EV_MASK,
+	FSM_EV_ABORT,
+	FSM_EV_TIMER_EXPIRE,
+	FSM_EV_END, /* for reference */
+
+	FSM_EV_STATE_IN,
+	FSM_EV_STATE_OUT,
+
+	FSM_EV_NEW_OBJ,
+	FSM_EV_DEL_OBJ,
+
+	FSM_EV_INT_MAX,
+
+	FSM_GBL_EV_MASK_ = FSM_GBL_EV_MASK,
+	FSM_EV_CONNECTED,
+	FSM_EV_CONNECT_FAIL,
+	FSM_EV_DISCONNECTED,
+	FSM_EV_SCAN_START,
+	FSM_EV_SCAN_DONE,
+
+	FSM_EV_GBL_MAX
+};
+
+struct fsm_root;
+struct fsm_main;
+struct fsm_obj;
+
+struct fsm_priv {
+	struct fsm_root *root;
+#ifdef CONFIG_RTW_FSM_XXX
+	struct xxx_priv xxxpriv;
+#endif
+#ifdef CONFIG_RTW_FSM_RRM
+	struct rrm_priv rmpriv;
+#endif
+#ifdef CONFIG_RTW_FSM_BTM
+	struct btm_priv btmpriv;
+#endif
+};
+
+/* @oid: object id
+ * @event: event id
+ * @msg: additional message of the event
+ * @msg_sz: message size
+ */
+struct fsm_msg {
+	_os_list list;
+	u32 oid; /* receiver */
+	u16 cid; /* receiver */
+	u16 event; /* event id */
+	struct fsm_main *fsm;
+
+	void *param;
+	int param_sz;
+};
+
+enum fsm_dbg_level {
+	FSM_LV_NONE,
+	FSM_LV_PRINT,
+	FSM_LV_ERR,
+	FSM_LV_WARN,
+	FSM_LV_INFO, /* dbg_level: dump normal info msg */
+	FSM_LV_DBG, /* dbg_level: dump state change info */
+	FSM_LV_TRACE,
+	FSM_LV_MAX
+};
+
+#define EV_ENT(ev) {ev, #ev, FSM_LV_INFO}
+#define EV_WRN(ev) {ev, #ev, FSM_LV_WARN}
+#define EV_INF(ev) {ev, #ev, FSM_LV_INFO}
+#define EV_DBG(ev) {ev, #ev, FSM_LV_DBG}
+
+struct fsm_event_ent {
+	u16 event;
+	char *name;
+	u8 evt_level;
+};
+
+#define ST_ENT(st, hdl) {st, #st, hdl}
+struct fsm_state_ent {
+	u8 state;
+	char *name;
+	int (*fsm_func)(void *priv, u16 event, void *param);
+};
+
+/* struct of rtw_fsm_register_fsm() */
+struct rtw_fsm_tb {
+	void *priv;
+	u8 dbg_level;
+	u8 evt_level;
+	u8 max_state;
+	u16 max_event;
+	struct fsm_state_ent *state_tbl;
+	struct fsm_event_ent *evt_tbl;
+
+	int (*init_priv)(void *priv); /* optional */
+	int (*deinit_priv)(void *priv); /* optional */
+	int (*active_obj)(void *priv, void *custom_obj); /* optional */
+	int (*deactive_obj)(void *priv, void *custom_obj); /* optional */
+	/* debug function */
+	void (*dump_obj)(void *obj, char *p, int *sz); /* optional */
+	void (*dump_fsm)(void *fsm, char *p, int *sz); /* optional */
+	void (*debug)(void *custom_obj, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 *out_len);
+};
+
+/* fsm init funciton */
+int rtw_fsm_init(struct fsm_priv *fsmpriv, void *priv);
+void rtw_fsm_deinit(struct fsm_priv *fsmpriv);
+void rtw_fsm_start(struct fsm_priv *fsmpriv);
+void rtw_fsm_stop(struct fsm_priv *fsmpriv);
+
+struct fsm_main *rtw_fsm_register_fsm(struct fsm_root *root,
+	const char *name, struct rtw_fsm_tb *tb);
+
+void *rtw_fsm_new_obj(struct fsm_main *fsm,
+	struct sta_info *psta, u16 cid, void **custom_obj, int obj_sz);
+void fsm_activate_obj(struct fsm_obj *obj);
+void fsm_deactivate_obj(struct fsm_obj *obj);
+#ifdef RTW_DEBUG_FSM
+void rtw_fsm_dbg(void *a, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 out_len);
+#endif
+
+/* fsm operating funciton */
+struct fsm_msg *fsm_new_msg(struct fsm_obj *obj, u16 event);
+int fsm_send_msg(struct fsm_main *fsm, struct fsm_msg *msg);
+int fsm_abort_obj(struct fsm_obj *obj);
+void fsm_state_goto(struct fsm_obj *obj, u8 new_state);
+void fsm_set_alarm(struct fsm_obj *obj, int ms, u16 event);
+void fsm_set_alarm_ext(struct fsm_obj *obj,
+	int ms, u16 event, u8 id, void *priv);
+void fsm_cancel_alarm(struct fsm_obj *obj);
+void fsm_cancel_alarm_ext(struct fsm_obj *obj, u8 id);
+void fsm_del_obj(struct fsm_obj *obj);
+#if 0
+void fsm_pause_alarm(struct fsm_obj *obj);
+void fsm_pause_alarm_ext(struct fsm_obj *obj, u8 id);
+void fsm_resume_alarm(struct fsm_obj *obj);
+void fsm_resume_alarm_ext(struct fsm_obj *obj, u8 id);
+bool fsm_is_alarm_off(struct fsm_obj *obj);
+bool fsm_is_alarm_off_ext(struct fsm_obj *obj, u8 id);
+void fsm_extend_alarm_ext(struct fsm_obj *obj, int ms, u8 id);
+#endif
+u8 fsm_dbg_lv(struct fsm_main *fsm, u8 level);
+u8 fsm_evt_lv(struct fsm_main *fsm, u8 level);
+int fsm_gen_msg(struct fsm_obj *obj, void *pbuf, u32 sz, u16 event);
+int rtw_fsm_gen_cmd(struct fsm_main *fsm, void *pbuf, u32 sz, u16 event);
+
+/* function to manipulate extra queue */
+int rtw_fsm_enqueue_ext(struct fsm_main *fsm, struct fsm_msg *msg, u8 to_head);
+struct fsm_msg *rtw_fsm_dequeue_ext(struct fsm_main *fsm);
+int rtw_fsm_is_ext_queue_empty(struct fsm_main *fsm);
+
+/* util function */
+void *rtw_fsm_get_obj(struct fsm_main *fsm, u16 cid);
+char *fsm_obj_name(struct fsm_obj *obj);
+char *fsm_evt_name(struct fsm_main *fsm, u16 event);
+
+void *fsm_to_priv(struct fsm_main *fsm);
+struct sta_info *obj_to_sta(struct fsm_obj *obj);
+_adapter *fsm_to_adapter(struct fsm_main *fsm);
+
+/* Notify function */
+void rtw_fsm_notify_connect(struct fsm_priv *fsmpriv, struct sta_info *psta, int res);
+void rtw_fsm_notify_disconnect(struct fsm_priv *fsmpriv, struct sta_info *psta);
+void rtw_fsm_notify_scan_start(struct fsm_priv *fsmpriv, struct sta_info *psta);
+void rtw_fsm_notify_scan_done(struct fsm_priv *fsmpriv, struct sta_info *psta);
+
+_adapter *fsm_to_adapter(struct fsm_main *fsm);
+
+#define obj2adp(c) fsm_to_adapter(c->fsm)
+#define obj2sta(c) obj_to_sta(c->obj)
+#define obj2mac(c) obj_to_sta(c->obj)->phl_sta->mac_addr
+#define obj2priv(c) fsm_to_priv(c->fsm)
+
+#define rtw_fsm_st_goto(c, d) fsm_state_goto(c->obj, d)
+#define rtw_fsm_set_alarm(c, d, e) fsm_set_alarm(c->obj, d, e)
+#define rtw_fsm_cancel_alarm(c) fsm_cancel_alarm(c->obj)
+
+#define rtw_fsm_abort_obj(c) fsm_abort_obj(c->obj)
+
+#define rtw_fsm_new_msg(c, d) fsm_new_msg(c->obj, d)
+#define rtw_fsm_send_msg(c, d) fsm_send_msg(c->fsm, d)
+#define rtw_fsm_gen_msg(c, d, e, f) fsm_gen_msg(c->obj, d, e, f)
+#define rtw_fsm_activate_obj(c) fsm_activate_obj(c->obj)
+#define rtw_fsm_deactivate_obj(c) fsm_deactivate_obj(c->obj)
+#define rtw_fsm_obj_name(c) fsm_obj_name(c->obj)
+#define rtw_fsm_evt_name(c, d) fsm_evt_name(c->fsm, d)
+#define rtw_fsm_cancel_alarm_ext(c, d) fsm_cancel_alarm_ext(c->obj, d)
+#define rtw_fsm_set_alarm_ext(c, d, e, f, g) fsm_set_alarm_ext(c->obj, d, e, f, g)
+#define rtw_fsm_del_obj(c) fsm_del_obj(c->obj)
+
+#if 1
+#define FSM_PRINT(fsm, fmt, ...) \
+	do {\
+		if (!fsm || fsm_dbg_lv(fsm, FSM_LV_PRINT)) \
+			RTW_PRINT(fmt, ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_ERR_(fsm, fmt, ...) \
+	do {\
+		if (!fsm || fsm_dbg_lv(fsm, FSM_LV_ERR)) \
+			RTW_ERR(fmt, ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_ERR(c, fmt, ...) \
+	do {\
+		if (!c->fsm || fsm_dbg_lv(c->fsm, FSM_LV_ERR)) \
+			RTW_ERR("%s " fmt, fsm_obj_name(c->obj), ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_WARN_(fsm, fmt, ...) \
+	do {\
+		if (!fsm || fsm_dbg_lv(fsm, FSM_LV_WARN)) \
+			RTW_WARN(fmt, ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_WARN(c, fmt, ...) \
+	do {\
+		if (!c->fsm || fsm_dbg_lv(c->fsm, FSM_LV_WARN)) \
+			RTW_WARN("%s " fmt, fsm_obj_name(c->obj), ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_INFO_(fsm, fmt, ...) \
+	do {\
+		if (!fsm || fsm_dbg_lv(fsm, FSM_LV_INFO)) \
+			RTW_INFO(fmt, ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_INFO(c, fmt, ...) \
+	do {\
+		if (!c->fsm || fsm_dbg_lv(c->fsm, FSM_LV_INFO)) \
+			RTW_INFO("%s " fmt, fsm_obj_name(c->obj), ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_DBG_(fsm, fmt, ...) \
+	do {\
+		if (!fsm || fsm_dbg_lv(fsm, FSM_LV_DBG)) \
+			RTW_INFO(fmt, ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_DBG(c, fmt, ...) \
+	do {\
+		if (!c->fsm || fsm_dbg_lv(c->fsm, FSM_LV_DBG)) \
+			RTW_INFO("%s " fmt, fsm_obj_name(c->obj), ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_TRACE_(fsm, fmt, ...) \
+	do {\
+		if (!fsm || fsm_dbg_lv(fsm, FSM_LV_TRACE)) \
+			RTW_INFO(fmt, ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_TRACE(c, fmt, ...) \
+	do {\
+		if (!c->fsm || fsm_dbg_lv(c->fsm, FSM_LV_TRACE)) \
+			RTW_INFO("%s " fmt, fsm_obj_name(c->obj), ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_MSG_(fsm, level_, fmt, ...) \
+	do {\
+		if (!fsm || fsm_dbg_lv(fsm, level_)) \
+			RTW_INFO(fmt, ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_MSG(c, level_, fmt, ...) \
+	do {\
+		if (!c->fsm || fsm_dbg_lv(fsm, level_)) \
+			RTW_PRINT("%s ", fmt, fsm_obj_name(c->obj), ##__VA_ARGS__); \
+	} while (0)
+
+#define FSM_EV_MSG_(fsm, level_, fmt, ...) \
+	do {\
+		if (!fsm || fsm_evt_lv(fsm, level_)) \
+			RTW_INFO(fmt, ##__VA_ARGS__); \
+	} while (0)
+#else
+#undef FSM_PRINT
+#define FSM_PRINT(fsm, fmt, ...)
+#undef FSM_ERR
+#define FSM_ERR(fsm, fmt, ...)
+#undef FSM_WARN
+#define FSM_WARN(fsm, fmt, ...)
+#undef FSM_INFO
+#define FSM_INFO(fsm, fmt, ...)
+#undef FSM_DBG
+#define FSM_DBG(fsm, fmt, ...)
+#undef FSM_MSG
+#define FSM_MSG(fsm, level, fmt, ...)
+#undef FSM_EV_MSG
+#define FSM_EV_MSG(fsm, level, fmt, ...)
+#endif  /* CONFIG_PHL_WPP */
+#endif /* __PHL_FSM_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_rrm.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_rrm.h
new file mode 100644
index 000000000000..e923fd842291
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_rrm.h
@@ -0,0 +1,51 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_FSM_RRM_H__
+#define __RTW_FSM_RRM_H__
+
+#ifdef CONFIG_RTW_FSM_RRM
+u8 rm_post_event_hdl(_adapter *padapter, u8 *pbuf);
+
+struct fsm_priv;
+struct rrm_obj;
+
+/* Header file for application to invoke rrm service */
+
+struct rrm_priv {
+	u8 enable;
+	_queue rm_queue;
+
+	u8 rm_en_cap_def[5];
+	u8 rm_en_cap_assoc[5];
+	u8 meas_token;
+
+	struct fsm_main *fsm;
+	/* rm debug */
+	void *prm_sel;
+};
+
+int rtw_rrm_reg_fsm(struct fsm_priv *fsmpriv);
+
+u8 rm_add_nb_req(_adapter *a, struct sta_info *psta);
+void rtw_ap_parse_sta_rm_en_cap(_adapter *a,
+	struct sta_info *psta, struct rtw_ieee802_11_elems *elem);
+unsigned int rrm_on_action(_adapter *a, union recv_frame *precv_frame);
+
+void rrm_update_cap(u8 *frame_head, _adapter *pa, u32 pktlen, int offset);
+void RM_IE_handler(_adapter *padapter, PNDIS_802_11_VARIABLE_IEs pIE);
+
+#endif /* CONFIG_RTW_FSM_RRM */
+#endif /* __RTW_FSM_RRM_H__ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_wnm.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_wnm.h
new file mode 100644
index 000000000000..cbe01deedd5e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_wnm.h
@@ -0,0 +1,36 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2023 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_FSM_BTM_H__
+#define __RTW_FSM_BTM_H__
+
+#ifdef CONFIG_RTW_FSM_BTM
+struct fsm_priv;
+struct btm_obj;
+
+/* Header file for application to invoke btm service */
+
+struct btm_priv {
+	struct fsm_main *fsm;
+	struct btm_obj *btm;
+};
+
+int rtw_btm_reg_fsm(struct fsm_priv *fsmpriv);
+int rtw_btm_new_obj(_adapter *a, struct sta_info *psta,
+	struct roam_nb_info *pnb, u8 roam_reason);
+void rtw_btm_notify_action_resp(struct btm_obj *pbtm);
+void rtw_btm_notify_scan_done(void *obj);
+void rtw_btm_notify_scan_found_candidate(struct btm_obj *pbtm, struct wlan_network *pnetwork);
+#endif /* CONFIG_RTW_FSM_BTM */
+#endif /* __RTW_BTM_FSM_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_xxx.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_xxx.h
new file mode 100644
index 000000000000..a9093d00b211
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_fsm_xxx.h
@@ -0,0 +1,33 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2023 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_FSM_XXX_H__
+#define __RTW_FSM_XXX_H__
+
+#ifdef CONFIG_RTW_FSM_XXX
+struct fsm_priv;
+struct xxx_obj;
+
+/* Header file for application to invoke xxx service */
+
+struct xxx_priv {
+	struct fsm_main *fsm;
+	struct xxx_obj *xxx;
+};
+
+int rtw_xxx_reg_fsm(struct fsm_priv *fsmpriv);
+struct xxx_obj *rtw_xxx_new_obj(_adapter *a, struct sta_info *psta, u16 cid);
+void rtw_xxx_abort(struct xxx_priv *pxxxpriv);
+#endif /* CONFIG_RTW_FSM_XXX */
+#endif /* __RTW_FSM_XXX_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ft.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ft.h
new file mode 100644
index 000000000000..b8e31d7bc455
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ft.h
@@ -0,0 +1,203 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __RTW_FT_H_
+#define __RTW_FT_H_
+
+enum rtw_ieee80211_ft_actioncode {
+	RTW_WLAN_ACTION_FT_RESV,
+	RTW_WLAN_ACTION_FT_REQ,
+	RTW_WLAN_ACTION_FT_RSP,
+	RTW_WLAN_ACTION_FT_CONF,
+	RTW_WLAN_ACTION_FT_ACK,
+	RTW_WLAN_ACTION_FT_MAX,
+};
+
+enum _rtw_ft_sta_status {
+	RTW_FT_UNASSOCIATED_STA = 0,
+	RTW_FT_AUTHENTICATING_STA,
+	RTW_FT_AUTHENTICATED_STA,
+	RTW_FT_ASSOCIATING_STA,
+	RTW_FT_ASSOCIATED_STA,
+	RTW_FT_REQUESTING_STA,
+	RTW_FT_REQUESTED_STA,
+	RTW_FT_CONFIRMED_STA,
+	RTW_FT_UNSPECIFIED_STA
+};
+
+#define RTW_FT_ACTION_REQ_LMT	4
+
+#define RTW_FT_MAX_IE_SZ	256
+
+#define rtw_ft_chk_status(a, s) \
+	((a)->mlmepriv.ft_roam.ft_status == (s))
+
+#define rtw_ft_roam_status(a, s)	\
+	((rtw_to_roam(a) > 0) && rtw_ft_chk_status(a, s))
+
+#define rtw_ft_authed_sta(a)	\
+	((rtw_ft_chk_status(a, RTW_FT_AUTHENTICATED_STA)) ||	\
+	(rtw_ft_chk_status(a, RTW_FT_ASSOCIATING_STA)) ||	\
+	(rtw_ft_chk_status(a, RTW_FT_ASSOCIATED_STA)))
+
+#define rtw_ft_set_status(a, s) \
+	do { \
+		((a)->mlmepriv.ft_roam.ft_status = (s)); \
+	} while (0)
+
+#define rtw_ft_lock_set_status(a, s) \
+	do { \
+		_rtw_spinlock_bh(&(a)->mlmepriv.lock);	\
+		((a)->mlmepriv.ft_roam.ft_status = (s));	\
+		_rtw_spinunlock_bh(&(a)->mlmepriv.lock);	\
+	} while (0)
+
+#define rtw_ft_reset_status(a) \
+	do { \
+		((a)->mlmepriv.ft_roam.ft_status = RTW_FT_UNASSOCIATED_STA); \
+	} while (0)
+
+enum rtw_ft_capability {
+	RTW_FT_EN = BIT0,
+	RTW_FT_OTD_EN = BIT1,
+	RTW_FT_PEER_EN = BIT2,
+	RTW_FT_PEER_OTD_EN = BIT3,
+	RTW_FT_BTM_ROAM = BIT4,
+	RTW_FT_TEST_RSSI_ROAM = BIT7,
+};
+
+#define rtw_ft_chk_flags(a, f) \
+	((a)->mlmepriv.ft_roam.ft_flags & (f))
+
+#define rtw_ft_set_flags(a, f) \
+	do { \
+		((a)->mlmepriv.ft_roam.ft_flags |= (f)); \
+	} while (0)
+
+#define rtw_ft_clr_flags(a, f) \
+	do { \
+		((a)->mlmepriv.ft_roam.ft_flags &= ~(f)); \
+	} while (0)
+
+#define rtw_ft_roam(a)	\
+	((rtw_to_roam(a) > 0) && rtw_ft_chk_flags(a, RTW_FT_PEER_EN))
+
+#define rtw_ft_valid_akm(a, t)	\
+	((rtw_ft_chk_flags(a, RTW_FT_EN)) && \
+	(((t) == 3) || ((t) == 4)))
+
+/* allow OTD while driver disconnect with current AP */
+#define rtw_ft_otd_roam_en(a)	\
+	((rtw_ft_chk_flags(a, RTW_FT_OTD_EN))	\
+	&& ((a)->mlmepriv.ft_roam.ft_cap & 0x01))
+
+#define rtw_ft_otd_roam(a) \
+	rtw_ft_chk_flags(a, RTW_FT_PEER_OTD_EN)
+
+#define rtw_ft_valid_otd_candidate(a, p)	\
+	((rtw_ft_chk_flags(a, RTW_FT_OTD_EN)) 	\
+	&& ((rtw_ft_chk_flags(a, RTW_FT_PEER_OTD_EN)	\
+	&& ((*((p)+4) & 0x01) == 0))	\
+	|| ((rtw_ft_chk_flags(a, RTW_FT_PEER_OTD_EN) == 0)	\
+	&& (*((p)+4) & 0x01))))
+
+struct ft_roam_info {
+	u16	mdid;
+	u8	ft_cap;
+	/*b0: FT over DS, b1: Resource Req Protocol Cap, b2~b7: Reserved*/
+	u8	updated_ft_ies[RTW_FT_MAX_IE_SZ];
+	u16	updated_ft_ies_len;
+	u8	ft_action[RTW_FT_MAX_IE_SZ];
+	u16	ft_action_len;
+	struct cfg80211_ft_event_params ft_event;
+	u8	ft_flags;
+	u32 ft_status;
+	u32 ft_req_retry_cnt;
+	bool ft_updated_bcn;
+};
+
+struct rtw_sta_ft_info_t {
+	u8 *rsn_ie;
+	u32 rsn_len;
+	u8 *md_ie;
+	u32 md_len;
+	u8 *ft_ie;
+	u32 ft_len;
+};
+
+void rtw_ft_info_init(struct ft_roam_info *pft);
+
+int rtw_ft_proc_flags_get(struct seq_file *m, void *v);
+
+ssize_t rtw_ft_proc_flags_set(struct file *file, const char __user *buffer,
+	size_t count, loff_t *pos, void *data);
+
+u8 rtw_ft_chk_roaming_candidate(
+	_adapter *padapter, struct wlan_network *competitor);
+
+void rtw_ft_update_stainfo(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork);
+
+void rtw_ft_reassoc_event_callback(_adapter *padapter, u8 *pbuf);
+
+void rtw_ft_validate_akm_type(_adapter  *padapter,
+	WLAN_BSSID_EX *network);
+
+void rtw_ft_update_bcn(_adapter *padapter, union recv_frame *precv_frame);
+
+void rtw_ft_start_clnt_join(_adapter *padapter);
+
+u8 rtw_ft_update_rsnie(
+	_adapter *padapter, u8 bwrite,
+	struct pkt_attrib *pattrib, u8 **pframe);
+
+void rtw_ft_build_auth_req_ies(_adapter *padapter,
+	struct pkt_attrib *pattrib, u8 **pframe);
+
+void rtw_ft_build_assoc_req_ies(_adapter *padapter,
+	u8 is_reassoc, struct pkt_attrib *pattrib, u8 **pframe);
+
+u8 rtw_ft_update_auth_rsp_ies(_adapter *padapter, u8 *pframe, u32 len);
+
+void rtw_ft_start_roam(_adapter *padapter, u8 *pTargetAddr);
+
+void rtw_ft_issue_action_req(_adapter *padapter, u8 *pTargetAddr);
+
+void rtw_ft_report_evt(_adapter *padapter);
+
+void rtw_ft_report_reassoc_evt(_adapter *padapter, u8 *pMacAddr);
+
+void rtw_ft_link_timer_hdl(void *ctx);
+
+void rtw_ft_roam_timer_hdl(void *ctx);
+
+void rtw_ft_roam_status_reset(_adapter *padapter);
+
+void rtw_ft_peer_info_init(struct sta_info *psta);
+
+void rtw_ft_peer_info_free(struct sta_info *psta);
+
+int rtw_ft_update_sta_ies(_adapter *padapter,
+	struct cfg80211_update_ft_ies_params *pie);
+
+void rtw_ft_update_assocresp_ies(struct net_device *net,
+	struct cfg80211_ap_settings *settings);
+
+void rtw_ft_process_ft_auth_rsp(_adapter *padapter, u8 *pframe, u32 len);
+
+void rtw_ft_build_assoc_rsp_ies(_adapter *padapter,
+	struct sta_info *psta, struct pkt_attrib *pattrib, u8 **pframe);
+
+void rtw_ft_start_clnt_action(_adapter *padapter, u8 *pTargetAddr);
+#endif /* __RTW_FT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_he.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_he.h
new file mode 100644
index 000000000000..f3760ad1ef7f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_he.h
@@ -0,0 +1,1115 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_HE_H_
+#define _RTW_HE_H_
+
+
+/* ********************************************* */
+/* IEEE802.11 BA BUFFER SIZE IN ADDBA DEFINITION */
+/* ********************************************* */
+#define MAX_BA_BUFFER_SIZE_HE 256
+
+/* Set HE MAC Capabilities Information */
+#define SET_HE_MAC_CAP_HTC_HE_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 1, _val)
+#define SET_HE_MAC_CAP_TWT_REQUESTER_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 1, 1, _val)
+#define SET_HE_MAC_CAP_TWT_RESPONDER_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 2, 1, _val)
+#define SET_HE_MAC_CAP_DYNAMIC_FRAG_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 3, 2, _val)
+#define SET_HE_MAC_CAP_MAX_FRAG_MSDU_EXP(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 5, 3, _val)
+
+#define SET_HE_MAC_CAP_MIN_FRAG_SIZE(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 0, 2, _val)
+#define SET_HE_MAC_CAP_TRI_FRAME_PADDING_DUR(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 2, 2, _val)
+#define SET_HE_MAC_CAP_MULTI_TID_AGG_RX_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 4, 3, _val)
+#define SET_HE_MAC_CAP_LINK_ADAPT_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_2BYTE((_pEleStart) + 1, 7, 2, _val)
+
+#define SET_HE_MAC_CAP_ALL_ACK_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 1, 1, _val)
+#define SET_HE_MAC_CAP_TRS_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 2, 1, _val)
+#define SET_HE_MAC_CAP_BRS_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 3, 1, _val)
+#define SET_HE_MAC_CAP_BC_TWT_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 4, 1, _val)
+#define SET_HE_MAC_CAP_32_BIT_BMP_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 5, 1, _val)
+#define SET_HE_MAC_CAP_MU_CASCADE_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 6, 1, _val)
+#define SET_HE_MAC_CAP_ACK_ENABLED_AGG_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 7, 1, _val)
+
+#define SET_HE_MAC_CAP_OM_CTRL_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 1, 1, _val)
+#define SET_HE_MAC_CAP_OFDMA_RA_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 2, 1, _val)
+#define SET_HE_MAC_CAP_MAX_AMPDU_LEN_EXP_EXT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 3, 2, _val)
+#define SET_HE_MAC_CAP_AMSDU_FRAG_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 5, 1, _val)
+#define SET_HE_MAC_CAP_FLEX_TWT_SCHED_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 6, 1, _val)
+#define SET_HE_MAC_CAP_RX_CTRL_FRAME_TO_MULTI_BSS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 7, 1, _val)
+
+#define SET_HE_MAC_CAP_BSRP_BQRP_AMPDU_AGG(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 0, 1, _val)
+#define SET_HE_MAC_CAP_QTP_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 1, 1, _val)
+#define SET_HE_MAC_CAP_BQR_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 2, 1, _val)
+#define SET_HE_MAC_CAP_PSR_RESPONDER(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 3, 1, _val)
+#define SET_HE_MAC_CAP_NDP_FEEDBACK_RPT_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 4, 1, _val)
+#define SET_HE_MAC_CAP_OPS_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 5, 1, _val)
+#define SET_HE_MAC_CAP_AMSDU_NOT_UNDER_BA_IN_ACK_EN_AMPDU(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 6, 1, _val)
+#define SET_HE_MAC_CAP_MULTI_AID_AGG_TX_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_2BYTE((_pEleStart) + 4, 7, 3, _val)
+
+#define SET_HE_MAC_CAP_HE_SUB_CH_SELECTIVE_TX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 2, 1, _val)
+#define SET_HE_MAC_CAP_UL_2_996_TONE_RU_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 3, 1, _val)
+#define SET_HE_MAC_CAP_OM_CTRL_UL_MU_DATA_DISABLE_RX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 4, 1, _val)
+#define SET_HE_MAC_CAP_HE_DYNAMIC_SM_POWER_SAVE(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 5, 1, _val)
+#define SET_HE_MAC_CAP_PUNCTURED_SND_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 6, 1, _val)
+#define SET_HE_MAC_CAP_HT_VHT_TRIG_FRAME_RX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 7, 1, _val)
+
+/* Set HE PHY Capabilities Information */
+#define SET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 1, 7, _val)
+
+#define SET_HE_PHY_CAP_PUNCTURED_PREAMBLE_RX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 0, 4, _val)
+#define SET_HE_PHY_CAP_DEVICE_CLASS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 4, 1, _val)
+#define SET_HE_PHY_CAP_LDPC_IN_PAYLOAD(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 5, 1, _val)
+#define SET_HE_PHY_CAP_SU_PPDU_1X_LTF_0_POINT_8_GI(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 6, 1, _val)
+#define SET_HE_PHY_CAP_MIDAMBLE_TRX_MAX_NSTS(_pEleStart, _val) \
+	SET_BITS_TO_LE_2BYTE((_pEleStart) + 1, 7, 2, _val)
+
+#define SET_HE_PHY_CAP_NDP_4X_LTF_3_POINT_2_GI(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 1, 1, _val)
+#define SET_HE_PHY_CAP_STBC_TX_LESS_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 2, 1, _val)
+#define SET_HE_PHY_CAP_STBC_RX_LESS_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 3, 1, _val)
+#define SET_HE_PHY_CAP_DOPPLER_TX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 4, 1, _val)
+#define SET_HE_PHY_CAP_DOPPLER_RX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 5, 1, _val)
+#define SET_HE_PHY_CAP_FULL_BW_UL_MUMIMO(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 6, 1, _val)
+#define SET_HE_PHY_CAP_PARTIAL_BW_UL_MUMIMO(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 7, 1, _val)
+
+#define SET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_TX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 0, 2, _val)
+#define SET_HE_PHY_CAP_DCM_MAX_NSS_TX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 2, 1, _val)
+#define SET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_RX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 3, 2, _val)
+#define SET_HE_PHY_CAP_DCM_MAX_NSS_RX(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 5, 1, _val)
+#define SET_HE_PHY_CAP_RX_PARTIAL_BW_SU_IN_20MHZ_MUPPDU(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 6, 1, _val)
+#define SET_HE_PHY_CAP_SU_BFER(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 3, 7, 1, _val)
+
+#define SET_HE_PHY_CAP_SU_BFEE(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 0, 1, _val)
+#define SET_HE_PHY_CAP_MU_BFER(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 1, 1, _val)
+#define SET_HE_PHY_CAP_BFEE_STS_LESS_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 2, 3, _val)
+#define SET_HE_PHY_CAP_BFEE_STS_GREATER_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 4, 5, 3, _val)
+
+#define SET_HE_PHY_CAP_NUM_SND_DIMEN_LESS_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 0, 3, _val)
+#define SET_HE_PHY_CAP_NUM_SND_DIMEN_GREATER_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 3, 3, _val)
+#define SET_HE_PHY_CAP_NG_16_SU_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 6, 1, _val)
+#define SET_HE_PHY_CAP_NG_16_MU_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 5, 7, 1, _val)
+
+#define SET_HE_PHY_CAP_CODEBOOK_4_2_SU_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 0, 1, _val)
+#define SET_HE_PHY_CAP_CODEBOOK_7_5_MU_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 1, 1, _val)
+#define SET_HE_PHY_CAP_TRIG_SUBF_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 2, 1, _val)
+#define SET_HE_PHY_CAP_TRIG_MUBF_PARTIAL_BW_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 3, 1, _val)
+#define SET_HE_PHY_CAP_TRIG_CQI_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 4, 1, _val)
+#define SET_HE_PHY_CAP_PARTIAL_BW_EXT_RANGE(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 5, 1, _val)
+#define SET_HE_PHY_CAP_PARTIAL_BW_DL_MU_MIMO(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 6, 1, _val)
+#define SET_HE_PHY_CAP_PPE_THRESHOLD_PRESENT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 6, 7, 1, _val)
+
+#define SET_HE_PHY_CAP_PSR_BASED_SR_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 7, 0, 1, _val)
+#define SET_HE_PHY_CAP_PWR_BOOST_FACTOR_SUPPORT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 7, 1, 1, _val)
+#define SET_HE_PHY_CAP_SU_MU_PPDU_4X_LTF_0_POINT_8_GI(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 7, 2, 1, _val)
+#define SET_HE_PHY_CAP_MAX_NC(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 7, 3, 3, _val)
+#define SET_HE_PHY_CAP_STBC_TX_GREATER_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 7, 6, 1, _val)
+#define SET_HE_PHY_CAP_STBC_RX_GREATER_THAN_80MHZ(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 7, 7, 1, _val)
+
+#define SET_HE_PHY_CAP_ERSU_PPDU_4X_LTF_0_POINT_8_GI(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 8, 0, 1, _val)
+#define SET_HE_PHY_CAP_20M_IN_40M_HE_PPDU_IN_2G4(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 8, 1, 1, _val)
+#define SET_HE_PHY_CAP_20M_IN_160C_160NC_HE_PPDU(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 8, 2, 1, _val)
+#define SET_HE_PHY_CAP_80M_IN_160C_160NC_HE_PPDU(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 8, 3, 1, _val)
+#define SET_HE_PHY_CAP_ERSU_PPDU_1X_LTF_0_POINT_8_GI(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 8, 4, 1, _val)
+#define SET_HE_PHY_CAP_MIDAMBLE_TRX_2X_1X_LTF(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 8, 5, 1, _val)
+#define SET_HE_PHY_CAP_DCM_MAX_RU(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 8, 6, 2, _val)
+
+#define SET_HE_PHY_CAP_LONGER_THAN_16_HESIGB_OFDM_SYM(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 9, 0, 1, _val)
+#define SET_HE_PHY_CAP_NON_TRIGGER_CQI_FEEDBACK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 9, 1, 1, _val)
+#define SET_HE_PHY_CAP_TX_1024_QAM_LESS_THAN_242_TONE_RU(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 9, 2, 1, _val)
+#define SET_HE_PHY_CAP_RX_1024_QAM_LESS_THAN_242_TONE_RU(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 9, 3, 1, _val)
+#define SET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_CMP_SIGB(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 9, 4, 1, _val)
+#define SET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_NONCMP_SIGB(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 9, 5, 1, _val)
+#define SET_HE_PHY_CAP_NOMINAL_PACKET_PADDING(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 9, 6, 2, _val)
+
+/* Set Supported HE-MCS And NSS Set Information */
+#define SET_HE_CAP_MCS_1SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 2, _val)
+#define SET_HE_CAP_MCS_2SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 2, 2, _val)
+#define SET_HE_CAP_MCS_3SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 4, 2, _val)
+#define SET_HE_CAP_MCS_4SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 6, 2, _val)
+#define SET_HE_CAP_MCS_5SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 0, 2, _val)
+#define SET_HE_CAP_MCS_6SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 2, 2, _val)
+#define SET_HE_CAP_MCS_7SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 4, 2, _val)
+#define SET_HE_CAP_MCS_8SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 6, 2, _val)
+
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_1SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_1SS(_pEleStart, _val)
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_2SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_2SS(_pEleStart, _val)
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_3SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_3SS(_pEleStart, _val)
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_4SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_4SS(_pEleStart, _val)
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_5SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_5SS(_pEleStart, _val)
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_6SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_6SS(_pEleStart, _val)
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_7SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_7SS(_pEleStart, _val)
+#define SET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_8SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_8SS(_pEleStart, _val)
+
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_1SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_1SS(_pEleStart + 2, _val)
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_2SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_2SS(_pEleStart + 2, _val)
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_3SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_3SS(_pEleStart + 2, _val)
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_4SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_4SS(_pEleStart + 2, _val)
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_5SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_5SS(_pEleStart + 2, _val)
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_6SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_6SS(_pEleStart + 2, _val)
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_7SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_7SS(_pEleStart + 2, _val)
+#define SET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_8SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_8SS(_pEleStart + 2, _val)
+
+#define SET_HE_CAP_RX_MCS_160MHZ_1SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_1SS(_pEleStart + 4, _val)
+#define SET_HE_CAP_RX_MCS_160MHZ_2SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_2SS(_pEleStart + 4, _val)
+#define SET_HE_CAP_RX_MCS_160MHZ_3SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_3SS(_pEleStart + 4, _val)
+#define SET_HE_CAP_RX_MCS_160MHZ_4SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_4SS(_pEleStart + 4, _val)
+#define SET_HE_CAP_RX_MCS_160MHZ_5SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_5SS(_pEleStart + 4, _val)
+#define SET_HE_CAP_RX_MCS_160MHZ_6SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_6SS(_pEleStart + 4, _val)
+#define SET_HE_CAP_RX_MCS_160MHZ_7SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_7SS(_pEleStart + 4, _val)
+#define SET_HE_CAP_RX_MCS_160MHZ_8SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_8SS(_pEleStart + 4, _val)
+
+#define SET_HE_CAP_TX_MCS_160MHZ_1SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_1SS(_pEleStart + 6, _val)
+#define SET_HE_CAP_TX_MCS_160MHZ_2SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_2SS(_pEleStart + 6, _val)
+#define SET_HE_CAP_TX_MCS_160MHZ_3SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_3SS(_pEleStart + 6, _val)
+#define SET_HE_CAP_TX_MCS_160MHZ_4SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_4SS(_pEleStart + 6, _val)
+#define SET_HE_CAP_TX_MCS_160MHZ_5SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_5SS(_pEleStart + 6, _val)
+#define SET_HE_CAP_TX_MCS_160MHZ_6SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_6SS(_pEleStart + 6, _val)
+#define SET_HE_CAP_TX_MCS_160MHZ_7SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_7SS(_pEleStart + 6, _val)
+#define SET_HE_CAP_TX_MCS_160MHZ_8SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_8SS(_pEleStart + 6, _val)
+
+#define SET_HE_CAP_RX_MCS_80_80MHZ_1SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_1SS(_pEleStart + 8, _val)
+#define SET_HE_CAP_RX_MCS_80_80MHZ_2SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_2SS(_pEleStart + 8, _val)
+#define SET_HE_CAP_RX_MCS_80_80MHZ_3SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_3SS(_pEleStart + 8, _val)
+#define SET_HE_CAP_RX_MCS_80_80MHZ_4SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_4SS(_pEleStart + 8, _val)
+#define SET_HE_CAP_RX_MCS_80_80MHZ_5SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_5SS(_pEleStart + 8, _val)
+#define SET_HE_CAP_RX_MCS_80_80MHZ_6SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_6SS(_pEleStart + 8, _val)
+#define SET_HE_CAP_RX_MCS_80_80MHZ_7SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_7SS(_pEleStart + 8, _val)
+#define SET_HE_CAP_RX_MCS_80_80MHZ_8SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_8SS(_pEleStart + 8, _val)
+
+#define SET_HE_CAP_TX_MCS_80_80MHZ_1SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_1SS(_pEleStart + 10, _val)
+#define SET_HE_CAP_TX_MCS_80_80MHZ_2SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_2SS(_pEleStart + 10, _val)
+#define SET_HE_CAP_TX_MCS_80_80MHZ_3SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_3SS(_pEleStart + 10, _val)
+#define SET_HE_CAP_TX_MCS_80_80MHZ_4SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_4SS(_pEleStart + 10, _val)
+#define SET_HE_CAP_TX_MCS_80_80MHZ_5SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_5SS(_pEleStart + 10, _val)
+#define SET_HE_CAP_TX_MCS_80_80MHZ_6SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_6SS(_pEleStart + 10, _val)
+#define SET_HE_CAP_TX_MCS_80_80MHZ_7SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_7SS(_pEleStart + 10, _val)
+#define SET_HE_CAP_TX_MCS_80_80MHZ_8SS(_pEleStart, _val) \
+	SET_HE_CAP_MCS_8SS(_pEleStart + 10, _val)
+
+/* Set PPE Threshold */
+#define SET_HE_CAP_PPE_NSTS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 3, _val)
+#define SET_HE_CAP_PPE_PU_IDX_BITMASK(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 3, 4, _val)
+
+/* Get HE MAC Capabilities Information */
+#define GET_HE_MAC_CAP_HTC_HE_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 1)
+#define GET_HE_MAC_CAP_TWT_REQUESTER_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 1, 1)
+#define GET_HE_MAC_CAP_TWT_RESPONDER_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 2, 1)
+#define GET_HE_MAC_CAP_DYNAMIC_FRAG_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 3, 2)
+#define GET_HE_MAC_CAP_MAX_FRAG_MSDU_EXP(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 5, 3)
+
+#define GET_HE_MAC_CAP_MIN_FRAG_SIZE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 0, 2)
+#define GET_HE_MAC_CAP_TRI_FRAME_PADDING_DUR(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 2, 2)
+#define GET_HE_MAC_CAP_MULTI_TID_AGG_RX_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 4, 3)
+#define GET_HE_MAC_CAP_LINK_ADAPT_SUPPORT(_pEleStart) \
+	LE_BITS_TO_2BYTE((_pEleStart) + 1, 7, 2)
+
+#define GET_HE_MAC_CAP_ALL_ACK_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 1, 1)
+#define GET_HE_MAC_CAP_TRS_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 2, 1)
+#define GET_HE_MAC_CAP_BRS_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 3, 1)
+#define GET_HE_MAC_CAP_BC_TWT_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 4, 1)
+#define GET_HE_MAC_CAP_32_BIT_BMP_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 5, 1)
+#define GET_HE_MAC_CAP_MU_CASCADE_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 6, 1)
+#define GET_HE_MAC_CAP_ACK_ENABLED_AGG_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 7, 1)
+
+#define GET_HE_MAC_CAP_OM_CTRL_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 1, 1)
+#define GET_HE_MAC_CAP_OFDMA_RA_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 2, 1)
+#define GET_HE_MAC_CAP_MAX_AMPDU_LEN_EXP_EXT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 3, 2)
+#define GET_HE_MAC_CAP_AMSDU_FRAG_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 5, 1)
+#define GET_HE_MAC_CAP_FLEX_TWT_SCHED_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 6, 1)
+#define GET_HE_MAC_CAP_RX_CTRL_FRAME_TO_MULTI_BSS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 7, 1)
+
+#define GET_HE_MAC_CAP_BSRP_BQRP_AMPDU_AGG(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 0, 1)
+#define GET_HE_MAC_CAP_QTP_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 1, 1)
+#define GET_HE_MAC_CAP_BQR_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 2, 1)
+#define GET_HE_MAC_CAP_PSR_RESPONDER(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 3, 1)
+#define GET_HE_MAC_CAP_NDP_FEEDBACK_RPT_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 4, 1)
+#define GET_HE_MAC_CAP_OPS_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 5, 1)
+#define GET_HE_MAC_CAP_AMSDU_NOT_UNDER_BA_IN_ACK_EN_AMPDU(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 6, 1)
+#define GET_HE_MAC_CAP_MULTI_AID_AGG_TX_SUPPORT(_pEleStart) \
+	LE_BITS_TO_2BYTE((_pEleStart) + 4, 7, 3)
+
+#define GET_HE_MAC_CAP_HE_SUB_CH_SELECTIVE_TX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 2, 1)
+#define GET_HE_MAC_CAP_UL_2_996_TONE_RU_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 3, 1)
+#define GET_HE_MAC_CAP_OM_CTRL_UL_MU_DATA_DISABLE_RX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 4, 1)
+#define GET_HE_MAC_CAP_HE_DYNAMIC_SM_POWER_SAVE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 5, 1)
+#define GET_HE_MAC_CAP_PUNCTURED_SND_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 6, 1)
+#define GET_HE_MAC_CAP_HT_VHT_TRIG_FRAME_RX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 7, 1)
+
+/* Get HE PHY Capabilities Information */
+#define GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 1, 7)
+
+#define GET_HE_PHY_CAP_PUNCTURED_PREAMBLE_RX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 0, 4)
+#define GET_HE_PHY_CAP_DEVICE_CLASS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 4, 1)
+#define GET_HE_PHY_CAP_LDPC_IN_PAYLOAD(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 5, 1)
+#define GET_HE_PHY_CAP_SU_PPDU_1X_LTF_0_POINT_8_GI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 6, 1)
+#define GET_HE_PHY_CAP_MIDAMBLE_TRX_MAX_NSTS(_pEleStart) \
+	LE_BITS_TO_2BYTE((_pEleStart) + 1, 7, 2)
+
+#define GET_HE_PHY_CAP_NDP_4X_LTF_3_POINT_2_GI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 1, 1)
+#define GET_HE_PHY_CAP_STBC_TX_LESS_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 2, 1)
+#define GET_HE_PHY_CAP_STBC_RX_LESS_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 3, 1)
+#define GET_HE_PHY_CAP_DOPPLER_TX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 4, 1)
+#define GET_HE_PHY_CAP_DOPPLER_RX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 5, 1)
+#define GET_HE_PHY_CAP_FULL_BW_UL_MUMIMO(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 6, 1)
+#define GET_HE_PHY_CAP_PARTIAL_BW_UL_MUMIMO(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 7, 1)
+
+#define GET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_TX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 0, 2)
+#define GET_HE_PHY_CAP_DCM_MAX_NSS_TX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 2, 1)
+#define GET_HE_PHY_CAP_DCM_MAX_CONSTELLATION_RX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 3, 2)
+#define GET_HE_PHY_CAP_DCM_MAX_NSS_RX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 5, 1)
+#define GET_HE_PHY_CAP_RX_PARTIAL_BW_SU_IN_20MHZ_MUPPDU(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 6, 1)
+#define GET_HE_PHY_CAP_SU_BFER(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 7, 1)
+
+#define GET_HE_PHY_CAP_SU_BFEE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 0, 1)
+#define GET_HE_PHY_CAP_MU_BFER(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 1, 1)
+#define GET_HE_PHY_CAP_BFEE_STS_LESS_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 2, 3)
+#define GET_HE_PHY_CAP_BFEE_STS_GREATER_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 5, 3)
+
+#define GET_HE_PHY_CAP_NUM_SND_DIMEN_LESS_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 0, 3)
+#define GET_HE_PHY_CAP_NUM_SND_DIMEN_GREATER_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 3, 3)
+#define GET_HE_PHY_CAP_NG_16_SU_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 6, 1)
+#define GET_HE_PHY_CAP_NG_16_MU_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 7, 1)
+
+#define GET_HE_PHY_CAP_CODEBOOK_4_2_SU_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 0, 1)
+#define GET_HE_PHY_CAP_CODEBOOK_7_5_MU_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 1, 1)
+#define GET_HE_PHY_CAP_TRIG_SUBF_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 2, 1)
+#define GET_HE_PHY_CAP_TRIG_MUBF_PARTIAL_BW_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 3, 1)
+#define GET_HE_PHY_CAP_TRIG_CQI_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 4, 1)
+#define GET_HE_PHY_CAP_PARTIAL_BW_EXT_RANGE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 5, 1)
+#define GET_HE_PHY_CAP_PARTIAL_BW_DL_MU_MIMO(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 6, 1)
+#define GET_HE_PHY_CAP_PPE_THRESHOLD_PRESENT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 7, 1)
+
+#define GET_HE_PHY_CAP_PSR_BASED_SR_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 0, 1)
+#define GET_HE_PHY_CAP_PWR_BOOST_FACTOR_SUPPORT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 1, 1)
+#define GET_HE_PHY_CAP_SU_MU_PPDU_4X_LTF_0_POINT_8_GI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 2, 1)
+#define GET_HE_PHY_CAP_MAX_NC(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 3, 3)
+#define GET_HE_PHY_CAP_STBC_TX_GREATER_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 6, 1)
+#define GET_HE_PHY_CAP_STBC_RX_GREATER_THAN_80MHZ(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 7, 1)
+
+#define GET_HE_PHY_CAP_ERSU_PPDU_4X_LTF_0_POINT_8_GI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 0, 1)
+#define GET_HE_PHY_CAP_20M_IN_40M_HE_PPDU_IN_2G4(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 1, 1)
+#define GET_HE_PHY_CAP_20M_IN_160C_160NC_HE_PPDU(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 2, 1)
+#define GET_HE_PHY_CAP_80M_IN_160C_160NC_HE_PPDU(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 3, 1)
+#define GET_HE_PHY_CAP_ERSU_PPDU_1X_LTF_0_POINT_8_GI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 4, 1)
+#define GET_HE_PHY_CAP_MIDAMBLE_TRX_2X_1X_LTF(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 5, 1)
+#define GET_HE_PHY_CAP_DCM_MAX_RU(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 6, 2)
+
+#define GET_HE_PHY_CAP_LONGER_THAN_16_HESIGB_OFDM_SYM(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 0, 1)
+#define GET_HE_PHY_CAP_NON_TRIGGER_CQI_FEEDBACK(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 1, 1)
+#define GET_HE_PHY_CAP_TX_1024_QAM_LESS_THAN_242_TONE_RU(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 2, 1)
+#define GET_HE_PHY_CAP_RX_1024_QAM_LESS_THAN_242_TONE_RU(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 3, 1)
+#define GET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_CMP_SIGB(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 4, 1)
+#define GET_HE_PHY_CAP_RX_FULLBW_SU_USE_MUPPDU_NONCMP_SIGB(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 5, 1)
+#define GET_HE_PHY_CAP_NOMINAL_PACKET_PADDING(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 6, 2)
+
+/* Get Supported HE-MCS And NSS Set Information */
+#define GET_HE_CAP_MCS_1SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 2)
+#define GET_HE_CAP_MCS_2SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 2, 2)
+#define GET_HE_CAP_MCS_3SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 4, 2)
+#define GET_HE_CAP_MCS_4SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 6, 2)
+#define GET_HE_CAP_MCS_5SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 0, 2)
+#define GET_HE_CAP_MCS_6SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 2, 2)
+#define GET_HE_CAP_MCS_7SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 4, 2)
+#define GET_HE_CAP_MCS_8SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 6, 2)
+
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_1SS(_pEleStart) \
+	GET_HE_CAP_MCS_1SS(_pEleStart)
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_2SS(_pEleStart) \
+	GET_HE_CAP_MCS_2SS(_pEleStart)
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_3SS(_pEleStart) \
+	GET_HE_CAP_MCS_3SS(_pEleStart)
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_4SS(_pEleStart) \
+	GET_HE_CAP_MCS_4SS(_pEleStart)
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_5SS(_pEleStart) \
+	GET_HE_CAP_MCS_5SS(_pEleStart)
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_6SS(_pEleStart) \
+	GET_HE_CAP_MCS_6SS(_pEleStart)
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_7SS(_pEleStart) \
+	GET_HE_CAP_MCS_7SS(_pEleStart)
+#define GET_HE_CAP_RX_MCS_LESS_THAN_80MHZ_8SS(_pEleStart) \
+	GET_HE_CAP_MCS_8SS(_pEleStart)
+
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_1SS(_pEleStart) \
+	GET_HE_CAP_MCS_1SS(_pEleStart + 2)
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_2SS(_pEleStart) \
+	GET_HE_CAP_MCS_2SS(_pEleStart + 2)
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_3SS(_pEleStart) \
+	GET_HE_CAP_MCS_3SS(_pEleStart + 2)
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_4SS(_pEleStart) \
+	GET_HE_CAP_MCS_4SS(_pEleStart + 2)
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_5SS(_pEleStart) \
+	GET_HE_CAP_MCS_5SS(_pEleStart + 2)
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_6SS(_pEleStart) \
+	GET_HE_CAP_MCS_6SS(_pEleStart + 2)
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_7SS(_pEleStart) \
+	GET_HE_CAP_MCS_7SS(_pEleStart + 2)
+#define GET_HE_CAP_TX_MCS_LESS_THAN_80MHZ_8SS(_pEleStart) \
+	GET_HE_CAP_MCS_8SS(_pEleStart + 2)
+
+#define GET_HE_CAP_RX_MCS_160MHZ_1SS(_pEleStart) \
+	GET_HE_CAP_MCS_1SS(_pEleStart + 4)
+#define GET_HE_CAP_RX_MCS_160MHZ_2SS(_pEleStart) \
+	GET_HE_CAP_MCS_2SS(_pEleStart + 4)
+#define GET_HE_CAP_RX_MCS_160MHZ_3SS(_pEleStart) \
+	GET_HE_CAP_MCS_3SS(_pEleStart + 4)
+#define GET_HE_CAP_RX_MCS_160MHZ_4SS(_pEleStart) \
+	GET_HE_CAP_MCS_4SS(_pEleStart + 4)
+#define GET_HE_CAP_RX_MCS_160MHZ_5SS(_pEleStart) \
+	GET_HE_CAP_MCS_5SS(_pEleStart + 4)
+#define GET_HE_CAP_RX_MCS_160MHZ_6SS(_pEleStart) \
+	GET_HE_CAP_MCS_6SS(_pEleStart + 4)
+#define GET_HE_CAP_RX_MCS_160MHZ_7SS(_pEleStart) \
+	GET_HE_CAP_MCS_7SS(_pEleStart + 4)
+#define GET_HE_CAP_RX_MCS_160MHZ_8SS(_pEleStart) \
+	GET_HE_CAP_MCS_8SS(_pEleStart + 4)
+
+#define GET_HE_CAP_TX_MCS_160MHZ_1SS(_pEleStart) \
+	GET_HE_CAP_MCS_1SS(_pEleStart + 6)
+#define GET_HE_CAP_TX_MCS_160MHZ_2SS(_pEleStart) \
+	GET_HE_CAP_MCS_2SS(_pEleStart + 6)
+#define GET_HE_CAP_TX_MCS_160MHZ_3SS(_pEleStart) \
+	GET_HE_CAP_MCS_3SS(_pEleStart + 6)
+#define GET_HE_CAP_TX_MCS_160MHZ_4SS(_pEleStart) \
+	GET_HE_CAP_MCS_4SS(_pEleStart + 6)
+#define GET_HE_CAP_TX_MCS_160MHZ_5SS(_pEleStart) \
+	GET_HE_CAP_MCS_5SS(_pEleStart + 6)
+#define GET_HE_CAP_TX_MCS_160MHZ_6SS(_pEleStart) \
+	GET_HE_CAP_MCS_6SS(_pEleStart + 6)
+#define GET_HE_CAP_TX_MCS_160MHZ_7SS(_pEleStart) \
+	GET_HE_CAP_MCS_7SS(_pEleStart + 6)
+#define GET_HE_CAP_TX_MCS_160MHZ_8SS(_pEleStart) \
+	GET_HE_CAP_MCS_8SS(_pEleStart + 6)
+
+#define GET_HE_CAP_RX_MCS_80_80MHZ_1SS(_pEleStart) \
+	GET_HE_CAP_MCS_1SS(_pEleStart + 8)
+#define GET_HE_CAP_RX_MCS_80_80MHZ_2SS(_pEleStart) \
+	GET_HE_CAP_MCS_2SS(_pEleStart + 8)
+#define GET_HE_CAP_RX_MCS_80_80MHZ_3SS(_pEleStart) \
+	GET_HE_CAP_MCS_3SS(_pEleStart + 8)
+#define GET_HE_CAP_RX_MCS_80_80MHZ_4SS(_pEleStart) \
+	GET_HE_CAP_MCS_4SS(_pEleStart + 8)
+#define GET_HE_CAP_RX_MCS_80_80MHZ_5SS(_pEleStart) \
+	GET_HE_CAP_MCS_5SS(_pEleStart + 8)
+#define GET_HE_CAP_RX_MCS_80_80MHZ_6SS(_pEleStart) \
+	GET_HE_CAP_MCS_6SS(_pEleStart + 8)
+#define GET_HE_CAP_RX_MCS_80_80MHZ_7SS(_pEleStart) \
+	GET_HE_CAP_MCS_7SS(_pEleStart + 8)
+#define GET_HE_CAP_RX_MCS_80_80MHZ_8SS(_pEleStart) \
+	GET_HE_CAP_MCS_8SS(_pEleStart + 8)
+
+#define GET_HE_CAP_TX_MCS_80_80MHZ_1SS(_pEleStart) \
+	GET_HE_CAP_MCS_1SS(_pEleStart + 10)
+#define GET_HE_CAP_TX_MCS_80_80MHZ_2SS(_pEleStart) \
+	GET_HE_CAP_MCS_2SS(_pEleStart + 10)
+#define GET_HE_CAP_TX_MCS_80_80MHZ_3SS(_pEleStart) \
+	GET_HE_CAP_MCS_3SS(_pEleStart + 10)
+#define GET_HE_CAP_TX_MCS_80_80MHZ_4SS(_pEleStart) \
+	GET_HE_CAP_MCS_4SS(_pEleStart + 10)
+#define GET_HE_CAP_TX_MCS_80_80MHZ_5SS(_pEleStart) \
+	GET_HE_CAP_MCS_5SS(_pEleStart + 10)
+#define GET_HE_CAP_TX_MCS_80_80MHZ_6SS(_pEleStart) \
+	GET_HE_CAP_MCS_6SS(_pEleStart + 10)
+#define GET_HE_CAP_TX_MCS_80_80MHZ_7SS(_pEleStart) \
+	GET_HE_CAP_MCS_7SS(_pEleStart + 10)
+#define GET_HE_CAP_TX_MCS_80_80MHZ_8SS(_pEleStart) \
+	GET_HE_CAP_MCS_8SS(_pEleStart + 10)
+
+/* Get PPE Threshold */
+#define GET_HE_CAP_PPE_NSTS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 3)
+#define GET_HE_CAP_PPE_PU_IDX_BITMASK(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 3, 4)
+
+/* Set HE Operation element */
+#define SET_HE_OP_PARA_DEFAULT_PE_DUR(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 3, _val)
+#define SET_HE_OP_PARA_TWT_REQUIRED(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 3, 1, _val)
+#define SET_HE_OP_PARA_TXOP_DUR_RTS_THRESHOLD(_pEleStart, _val) \
+	SET_BITS_TO_LE_2BYTE(_pEleStart, 4, 10, _val)
+
+#define SET_HE_OP_PARA_VHT_OP_INFO_PRESENT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 6, 1, _val)
+#define SET_HE_OP_PARA_CO_HOSTED_BSS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 7, 1, _val)
+
+#define SET_HE_OP_PARA_ER_SU_DISABLE(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 0, 1, _val)
+#define SET_HE_OP_PARA_6GHZ_OP_INFO_PRESENT(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 2, 1, 1, _val)
+
+#define SET_HE_OP_BSS_COLOR_INFO_BSS_COLOR(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 6, _val)
+#define SET_HE_OP_BSS_COLOR_INFO_PARTIAL_BSS_COLOR(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 6, 1, _val)
+#define SET_HE_OP_BSS_COLOR_INFO_BSS_COLOR_DISABLE(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 7, 1, _val)
+
+#define SET_HE_OP_BASIC_MCS_1SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 2, _val)
+#define SET_HE_OP_BASIC_MCS_2SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 2, 2, _val)
+#define SET_HE_OP_BASIC_MCS_3SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 4, 2, _val)
+#define SET_HE_OP_BASIC_MCS_4SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE(_pEleStart, 6, 2, _val)
+#define SET_HE_OP_BASIC_MCS_5SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 0, 2, _val)
+#define SET_HE_OP_BASIC_MCS_6SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 2, 2, _val)
+#define SET_HE_OP_BASIC_MCS_7SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 4, 2, _val)
+#define SET_HE_OP_BASIC_MCS_8SS(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 6, 2, _val)
+
+#define SET_HE_OP_INFO_PRIMARY_CHAN(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart), 0, 8, _val)
+#define SET_HE_OP_INFO_CHAN_WIDTH(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 0, 2, _val)
+#define SET_HE_OP_INFO_DUPL_BCN(_pEleStart) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 2, 1, _val)
+#define SET_HE_OP_INFO_REGULATORY_INFO(_pEleStart) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 3, 3, _val)
+
+#define SET_HE_OP_INFO_CENTER_FREQ_0(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart)+ 2, 0, 8, _val)
+#define SET_HE_OP_INFO_CENTER_FREQ_1(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart)+ 3, 0, 8, _val)
+#define SET_HE_OP_INFO_MIN_RATE(_pEleStart, _val) \
+	SET_BITS_TO_LE_1BYTE((_pEleStart)+ 4, 0, 8, _val)
+
+/* Values in HE spec */
+#define TXOP_DUR_RTS_TH_DISABLED	1023
+
+/* Get HE Operation element */
+#define GET_HE_OP_PARA_DEFAULT_PE_DUR(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 3)
+#define GET_HE_OP_PARA_TWT_REQUIRED(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 3, 1)
+#define GET_HE_OP_PARA_TXOP_DUR_RTS_THRESHOLD(_pEleStart) \
+	LE_BITS_TO_2BYTE(_pEleStart, 4, 10)
+
+#define GET_HE_OP_PARA_VHT_OP_INFO_PRESENT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 6, 1)
+#define GET_HE_OP_PARA_CO_HOSTED_BSS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 7, 1)
+
+#define GET_HE_OP_PARA_ER_SU_DISABLE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 0, 1)
+#define GET_HE_OP_PARA_6GHZ_OP_INFO_PRESENT(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 1, 1)
+
+#define GET_HE_OP_BSS_COLOR_INFO_BSS_COLOR(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 0, 6)
+#define GET_HE_OP_BSS_COLOR_INFO_PARTIAL_BSS_COLOR(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 6, 1)
+#define GET_HE_OP_BSS_COLOR_INFO_BSS_COLOR_DISABLE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 7, 1)
+
+#define GET_HE_OP_BASIC_MCS_1SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 2)
+#define GET_HE_OP_BASIC_MCS_2SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 2, 2)
+#define GET_HE_OP_BASIC_MCS_3SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 4, 2)
+#define GET_HE_OP_BASIC_MCS_4SS(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 6, 2)
+#define GET_HE_OP_BASIC_MCS_5SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 0, 2)
+#define GET_HE_OP_BASIC_MCS_6SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 2, 2)
+#define GET_HE_OP_BASIC_MCS_7SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 4, 2)
+#define GET_HE_OP_BASIC_MCS_8SS(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 6, 2)
+
+#define GET_HE_OP_INFO_PRIMARY_CHAN(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 8)
+#define GET_HE_OP_INFO_CHAN_WIDTH(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 0, 2)
+#define GET_HE_OP_INFO_DUPL_BCN(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 2, 1)
+#define GET_HE_OP_INFO_REG_INFO(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 3, 3)
+#define GET_HE_OP_INFO_CHAN_CTR_FREQ_SEG0(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 0, 8)
+#define GET_HE_OP_INFO_CHAN_CTR_FREQ_SEG1(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 0, 8)
+#define GET_HE_OP_INFO_MIN_RATE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 0, 8)
+
+/* Get MU EDCA Parameter Set element */
+#define GET_HE_MU_EDCA_QOS_INFO(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 8)
+#define GET_HE_MU_EDCA_QOS_INFO_UPDATE_CNT(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 4)
+#define GET_HE_MU_EDCA_BE_AIFSN(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 0, 4)
+#define GET_HE_MU_EDCA_BE_ACI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 1, 5, 2)
+#define GET_HE_MU_EDCA_BE_ECW_MIN_MAX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 0, 8)
+#define GET_HE_MU_EDCA_BE_TIMER(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 3, 0, 8)
+#define GET_HE_MU_EDCA_BK_AIFSN(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 0, 4)
+#define GET_HE_MU_EDCA_BK_ACI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 4, 5, 2)
+#define GET_HE_MU_EDCA_BK_ECW_MIN_MAX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 5, 0, 8)
+#define GET_HE_MU_EDCA_BK_TIMER(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 6, 0, 8)
+#define GET_HE_MU_EDCA_VI_AIFSN(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 0, 4)
+#define GET_HE_MU_EDCA_VI_ACI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 7, 5, 2)
+#define GET_HE_MU_EDCA_VI_ECW_MIN_MAX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 8, 0, 8)
+#define GET_HE_MU_EDCA_VI_TIMER(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 9, 0, 8)
+#define GET_HE_MU_EDCA_VO_AIFSN(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 10, 0, 4)
+#define GET_HE_MU_EDCA_VO_ACI(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 10, 5, 2)
+#define GET_HE_MU_EDCA_VO_ECW_MIN_MAX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 11, 0, 8)
+#define GET_HE_MU_EDCA_VO_TIMER(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 12, 0, 8)
+
+#define GET_HE_6G_BAND_CAP_MIN_MPDU_SPACING(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 0, 3)
+#define GET_HE_6G_BAND_CAP_MAX_AMPDU_LEN_EXP(_pEleStart) \
+	LE_BITS_TO_1BYTE(_pEleStart, 3, 3)
+#define GET_HE_6G_BAND_CAP_MAX_MPDU_LEN(_pEleStart) \
+        LE_BITS_TO_1BYTE(_pEleStart, 6, 2)
+#define GET_HE_6G_BAND_CAP_SM_PS(_pEleStart) \
+        LE_BITS_TO_1BYTE((_pEleStart) + 1, 1, 2)
+#define GET_HE_6G_BAND_CAP_RD_RESP(_pEleStart) \
+        LE_BITS_TO_1BYTE((_pEleStart) + 1, 3, 1)
+#define GET_HE_6G_BAND_CAP_RX_ANT_PATTERN(_pEleStart) \
+        LE_BITS_TO_1BYTE((_pEleStart) + 1, 4, 1)
+#define GET_HE_6G_BAND_CAP_TX_ANT_PATTERN(_pEleStart) \
+        LE_BITS_TO_1BYTE((_pEleStart) + 1, 5, 1)
+
+#define SET_HE_6G_BAND_CAP_MIN_MPDU_SPACING(_pEleStart, _val) \
+        SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 3, _val)
+#define SET_HE_6G_BAND_CAP_MAX_AMPDU_LEN_EXP(_pEleStart, _val) \
+        SET_BITS_TO_LE_1BYTE(_pEleStart, 3, 3, _val)
+#define SET_HE_6G_BAND_CAP_MAX_MPDU_LEN(_pEleStart, _val) \
+        SET_BITS_TO_LE_1BYTE(_pEleStart, 6, 2, _val)
+#define SET_HE_6G_BAND_CAP_SM_PS(_pEleStart, _val) \
+        SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 1, 2, _val)
+#define SET_HE_6G_BAND_CAP_RD_RESP(_pEleStart, _val) \
+        SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 3, 1, _val)
+#define SET_HE_6G_BAND_CAP_RX_ANT_PATTERN(_pEleStart, _val) \
+        SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 4, 1, _val)
+#define SET_HE_6G_BAND_CAP_TX_ANT_PATTERN(_pEleStart, _val) \
+        SET_BITS_TO_LE_1BYTE((_pEleStart) + 1, 5, 1, _val)
+
+/* HE variant HT Control */
+#define HE_VAR_HTC	3
+
+#define HE_VAR_HTC_CID_TRS		0
+#define HE_VAR_HTC_CID_OM		1
+#define HE_VAR_HTC_CID_HLA		2
+#define HE_VAR_HTC_CID_BSR		3
+#define HE_VAR_HTC_CID_UPH		4
+#define HE_VAR_HTC_CID_BQR		5
+#define HE_VAR_HTC_CID_CAS		6
+
+/* Set HE variant HT Control field */
+#define SET_HE_VAR_HTC(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 0, 2, HE_VAR_HTC)
+
+#define SET_HE_VAR_HTC_CID_TRS(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 2, 4, HE_VAR_HTC_CID_TRS)
+#define SET_HE_VAR_HTC_CID_OM(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 2, 4, HE_VAR_HTC_CID_OM)
+#define SET_HE_VAR_HTC_CID_HLA(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 2, 4, HE_VAR_HTC_CID_HLA)
+#define SET_HE_VAR_HTC_CID_BSR(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 2, 4, HE_VAR_HTC_CID_BSR)
+#define SET_HE_VAR_HTC_CID_UPH(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 2, 4, HE_VAR_HTC_CID_UPH)
+#define SET_HE_VAR_HTC_CID_BQR(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 2, 4, HE_VAR_HTC_CID_BQR)
+#define SET_HE_VAR_HTC_CID_CAS(_pStart) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 2, 4, HE_VAR_HTC_CID_CAS)
+
+#define SET_HE_VAR_HTC_OM_RX_NSS(_pStart, _val) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 6, 3, _val)
+#define SET_HE_VAR_HTC_OM_CH_WIDTH(_pStart, _val) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 6 + 3, 2, _val)
+#define SET_HE_VAR_HTC_OM_UL_MU_DIS(_pStart, _val) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 6 + 5, 1, _val)
+#define SET_HE_VAR_HTC_OM_TX_NSTS(_pStart, _val) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 6 + 6, 3, _val)
+#define SET_HE_VAR_HTC_OM_ER_SU_DIS(_pStart, _val) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 6 + 9, 1, _val)
+#define SET_HE_VAR_HTC_OM_DL_MU_MIMO_RR(_pStart, _val) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 6 + 10, 1, _val)
+#define SET_HE_VAR_HTC_OM_UL_MU_DATA_DIS(_pStart, _val) \
+	SET_BITS_TO_LE_4BYTE((u8 *)_pStart, 6 + 11, 1, _val)
+
+/* Get HE variant HT Control field */
+#define GET_VAR_HTC(_pStart) \
+	LE_BITS_TO_1BYTE(_pStart, 0, 2)
+#define GET_HE_VAR_HTC_CID(_pStart) \
+	LE_BITS_TO_1BYTE(_pStart, 2, 4)
+
+#define HE_MCS_SUPP_MSC0_TO_MSC7		0x0	/* 2b00 */
+#define HE_MCS_SUPP_MSC0_TO_MSC9		0x1	/* 2b01 */
+#define HE_MCS_SUPP_MSC0_TO_MSC11	0x2	/* 2b10 */
+#define HE_MSC_NOT_SUPP				0x3	/* 2b11 */
+#define HE_MSC_NOT_SUPP_BYTE			((HE_MSC_NOT_SUPP << 6) | (HE_MSC_NOT_SUPP << 4) \
+											| (HE_MSC_NOT_SUPP << 2) | HE_MSC_NOT_SUPP)
+
+#define HE_DEV_CLASS_A					1
+#define HE_DEV_CLASS_B					0
+
+/*
+ * HE_MAC_Cap (6)
+ * HE_PHY_Cap (11)
+ * HE_Support_MCS (4, 8 or 12)
+ * PPE_Thres (variable, max = 25)
+ */
+#define HE_CAP_ELE_MAC_CAP_LEN			6
+#define HE_CAP_ELE_PHY_CAP_LEN			11
+
+#define HE_CAP_ELE_SUPP_MCS_LEN_RX_80M		2
+#define HE_CAP_ELE_SUPP_MCS_LEN_TX_80M		2
+#define HE_CAP_ELE_SUPP_MCS_LEN_RX_160M		2
+#define HE_CAP_ELE_SUPP_MCS_LEN_TX_160M		2
+#define HE_CAP_ELE_SUPP_MCS_LEN_RX_80M_80M	2
+#define HE_CAP_ELE_SUPP_MCS_LEN_TX_80M_80M	2
+#define HE_CAP_ELE_SUPP_MCS_MAX_LEN		(HE_CAP_ELE_SUPP_MCS_LEN_RX_80M \
+	+ HE_CAP_ELE_SUPP_MCS_LEN_TX_80M + HE_CAP_ELE_SUPP_MCS_LEN_RX_160M \
+	+ HE_CAP_ELE_SUPP_MCS_LEN_TX_160M + HE_CAP_ELE_SUPP_MCS_LEN_RX_80M_80M \
+	+ HE_CAP_ELE_SUPP_MCS_LEN_TX_80M_80M)
+
+#define HE_CAP_ELE_PPE_THRE_MAX_LEN		25
+
+#define HE_CAP_ELE_MAX_LEN (1 + HE_CAP_ELE_MAC_CAP_LEN + HE_CAP_ELE_PHY_CAP_LEN \
+	+ HE_CAP_ELE_SUPP_MCS_MAX_LEN + HE_CAP_ELE_PPE_THRE_MAX_LEN)
+/* #define HE_CAP_MAC_CAP_OFFSET 0
+#define HE_CAP_PHY_CAP_OFFSET 6
+#define HE_CAP_SUPPORT_MCS_OFFSET 17
+*/
+
+/*
+ * HE_Ope_Para (3)
+ * BSS_Color (1)
+ * Basic_MCS (2)
+ * VHT_Op (0 or 3)
+ * CoHosted_Bssid_Ind (0 or 1)
+ * 6Ghz_Ope_Info (0 or 5)
+ */
+#define HE_OPER_PARAMS_LEN 				3
+#define HE_OPER_BSS_COLOR_INFO_LEN 		1
+#define HE_OPER_BASIC_MCS_LEN				2
+#define HE_OPER_VHT_OPER_INFO_LEN		3
+#define HE_OPER_MAX_COHOST_BSSID_LEN 	1
+#define HE_OPER_6G_OPER_INFO_LEN			5
+ 
+#define HE_OPER_ELE_MAX_LEN (1 + HE_OPER_PARAMS_LEN + HE_OPER_BSS_COLOR_INFO_LEN \
+	+ HE_OPER_BASIC_MCS_LEN + HE_OPER_VHT_OPER_INFO_LEN \
+	+ HE_OPER_MAX_COHOST_BSSID_LEN + HE_OPER_6G_OPER_INFO_LEN)
+/* #define HE_OPER_PARAS_OFFSET 0
+#define HE_OPER_BSS_COLOR_OFFSET 3
+#define HE_OPER_BASIC_MCS_OFFSET 4
+*/
+
+#define HE_6G_BAND_CAP_INFO_LEN		2
+#define HE_6G_BAND_CAP_MAX_LEN	(1 + HE_6G_BAND_CAP_INFO_LEN)
+
+#define MAX_HE_GI_TYPE 3
+#define MAX_HE_MCS_INDEX 12 * 2 /* 1SS + 2SS */
+
+enum rtw_he_actrl_om_mask {
+	OM_RX_NSS = BIT0,
+	OM_CH_BW = BIT1,
+	OM_UL_MU_DIS = BIT2,
+	OM_TX_NSTS = BIT3,
+	OM_ER_SU_DIS = BIT4,
+	OM_DL_MU_RR = BIT5,
+	OM_UL_MU_DATA_DIS = BIT6
+};
+
+struct rtw_he_actrl_om_ele {
+	u8 rx_nss;
+	u8 channel_width;
+	u8 ul_mu_disable;
+	u8 tx_nsts;
+	u8 er_su_disable;
+	u8 dl_mu_mimo_rr;
+	u8 ul_mu_data_disable;
+};
+
+
+struct rtw_he_actrl_om {
+	/* om ctrl flag for normal tx pkt */
+	u8 actrl_om_normal_tx;
+	u8 actrl_om_normal_tx_cnt;
+	/* current om ctrl element content */
+	struct rtw_he_actrl_om_ele om_actrl_ele;
+};
+
+struct he_priv {
+	u8	he_option;
+	u8 	he_cap[HE_CAP_ELE_MAX_LEN];
+	u8	he_op[HE_OPER_ELE_MAX_LEN];
+	u8 	he_6g_band_cap[HE_6G_BAND_CAP_MAX_LEN];
+	u8 	op_present;
+	u8	he_highest_rate;
+	u8	pre_he_muedca_cnt;
+	u8	mpdu_min_spacing;
+	struct rtw_he_actrl_om om_info;
+#ifdef CONFIG_TWT
+	struct rtw_phl_twt_element twt_ele;
+#endif
+};
+
+/*trigger frame*/
+#define TRIGGER_FRAME_USER_INFO_SZ 5 /* byte */
+#define TRIGGER_FRAME_MIN_LENGTH 24 + TRIGGER_FRAME_USER_INFO_SZ /* byte , aleast one user info !!! */
+/*basic tigger frame with 1 byte trigger dependent info */
+#define TRIGGER_FRAME_BASIC_USER_INFO_SZ TRIGGER_FRAME_USER_INFO_SZ + 1
+
+enum trigger_ul_bw {
+	TRIGGER_UL_BW_20	= 0,
+	TRIGGER_UL_BW_40	= 1,
+	TRIGGER_UL_BW_80	= 2,
+	TRIGGER_UL_BW_80_80_160	= 3,
+};
+
+/*trigger frame Common Info*/
+#define GET_TRIGGER_FRAME_TYPE(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart + 16), 0, 4)
+
+#define GET_TRIGGER_FRAME_COM_INFO_BW(_pEleStart) \
+	LE_BITS_TO_2BYTE((_pEleStart + 18), 2, 2)
+
+/*trigger frame User Info*/
+#define GET_TRIGGER_FRAME_USER_INFO_AID12(_user_info) \
+	LE_BITS_TO_2BYTE(_user_info, 0, 12)
+
+#define GET_TRIGGER_FRAME_USER_INFO_RUA(_user_info) \
+	LE_BITS_TO_2BYTE((_user_info + 1), 4, 8)
+
+#define GET_TRIGGER_FRAME_USER_INFO_UL_MCS(_user_info) \
+	LE_BITS_TO_2BYTE((_user_info + 2), 5, 4)
+
+
+enum rtw_he_trigger_frame_type {
+	TRIGGER_FRAME_T_BASIC = 0,
+	TRIGGER_FRAME_T_BFRP,
+	TRIGGER_FRAME_T_MUBAR,
+	TRIGGER_FRAME_T_MURTS,
+	TRIGGER_FRAME_T_BSRP,
+	TRIGGER_FRAME_T_GCR_MUBAR,
+	TRIGGER_FRAME_T_BQRP,
+	TRIGGER_FRAME_T_NFRP = 7,
+	TRIGGER_FRAME_T_RSVD = 8,
+};
+
+u16 rtw_he_mcs_to_data_rate(u8 bw, u8 gi, u8 he_mcs_rate);
+void	rtw_he_use_default_setting(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+void	update_sta_he_info_apmode(_adapter *padapter, void *sta);
+#if CONFIG_IEEE80211_BAND_6GHZ
+void update_sta_6ghz_info_apmode(_adapter *padapter, void *sta);
+#endif
+void	update_hw_he_param(_adapter *padapter);
+void HE_caps_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE);
+struct rtw_phl_stainfo_t;
+void HE_mac_caps_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start);
+void HE_phy_caps_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 *supp_mcs_len);
+void HE_supp_mcs_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 supp_mcs_len);
+void HE_ppe_thre_handler(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start);
+void rtw_he_set_asoc_cap_supp_mcs(_adapter *padapter, struct rtw_phl_stainfo_t *phl_sta, u8 *ele_start, u8 supp_mcs_len);
+u8 rtw_he_get_highest_rate(u8 *he_mcs_map);
+void HE_operation_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				PNDIS_802_11_VARIABLE_IEs pIE);
+void HE_mu_edca_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE, u8 first);
+void HE_6g_bandcap_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE);
+u32 rtw_build_he_cap_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *pbuf, enum band_type band);
+u32 rtw_build_he_operation_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *pbuf, struct rtw_chan_def *chandef);
+
+struct protocol_cap_t;
+struct phy_cap_t;
+u32 rtw_get_dft_he_cap_ie(_adapter *padapter, struct phy_cap_t *phy_cap,
+		struct protocol_cap_t *proto_cap, u8 *pbuf, enum band_type band);
+u32 rtw_build_he_6g_band_cap_ie_by_proto(_adapter *padapter, enum role_type role_type,
+		struct protocol_cap_t *proto_cap, u8 *pbuf);
+
+u32 rtw_restructure_he_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				u8 *in_ie, u8 *out_ie, uint in_len, uint *pout_len, enum band_type band);
+void HEOnAssocRsp(_adapter *padapter);
+void rtw_he_ies_attach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork, enum band_type band);
+void rtw_he_ies_detach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork);
+u8 rtw_he_htc_en(_adapter *padapter, struct sta_info *psta);
+void rtw_he_fill_htc(_adapter *padapter, struct pkt_attrib *pattrib, u32 *phtc_buf);
+void rtw_he_set_om_info(_adapter *padapter, struct _ADAPTER_LINK *alink, u8 om_mask, struct rtw_he_actrl_om *om_info);
+void rtw_he_init_om_info(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+void rtw_update_he_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork, struct rtw_chan_def *chandef);
+void rtw_update_probe_rsp_he_cap_and_op(struct _ADAPTER *a, u8 *ies, sint *ies_len, enum band_type band);
+void rtw_he_om_ctrl_trx_ss(_adapter *adapter, struct _ADAPTER_LINK *alink,
+							struct sta_info *sta, u8 chg_ss, bool need_update_ra);
+void rtw_process_he_triggerframe(_adapter *padapter,union recv_frame *precv_frame);
+#endif /* _RTW_HE_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ht.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ht.h
new file mode 100644
index 000000000000..8cc0e5857a9e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ht.h
@@ -0,0 +1,225 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_HT_H_
+#define _RTW_HT_H_
+
+/* ********************************************* */
+/* IEEE802.11 BA BUFFER SIZE IN ADDBA DEFINITION */
+/* ********************************************* */
+#define MAX_BA_BUFFER_SIZE_HT 64 /* HT & VHT */
+
+#define HT_CAP_IE_LEN 26
+#define HT_OP_IE_LEN 22
+
+struct ampdu_priv {
+	u8	ampdu_enable;/* for enable Tx A-MPDU */
+	u8	tx_amsdu_enable;/* for enable Tx A-MSDU */
+
+	/* u8	baddbareq_issued[16]; */
+	u32	rx_ampdu_maxlen; /* for rx reordering ctrl win_sz, updated when join_callback. */
+	u8	rx_ampdu_min_spacing;
+
+	/* for processing Tx A-MPDU */
+	u8	agg_enable_bitmap;
+	/* u8	ADDBA_retry_count; */
+	u8	candidate_tid_bitmap;
+};
+
+struct ht_priv {
+	u8	ht_option;
+	u8	bss_coexist;/* for 20/40 Bss coexist */
+
+	u32	tx_amsdu_maxlen; /* 1: 8k, 0:4k ; default:8k, for tx */
+
+	u8	ch_offset;/* PRIME_CHNL_OFFSET */
+	u8	sgi_20m;
+	u8	sgi_40m;
+
+	u8	ldpc_cap;
+	u8	stbc_cap;
+	u8	beamform_cap;
+
+	u8 op_present:1; /* ht_op is present */
+
+	struct rtw_ieee80211_ht_cap ht_cap;
+	u8 ht_op[HT_OP_IE_LEN];
+
+};
+
+#ifdef PRIVATE_R
+struct ht_priv_infra_ap {
+
+	/*Infra mode, only store AP's info , not intersection of STA and AP*/
+	u8	channel_width_infra_ap;
+	u8	sgi_20m_infra_ap;
+	u8	sgi_40m_infra_ap;
+	u8	ldpc_cap_infra_ap;
+	u8	stbc_cap_infra_ap;
+	u8	MCS_set_infra_ap[16];
+	u8	Rx_ss_infra_ap;
+	u16	rx_highest_data_rate_infra_ap;
+};
+#endif /* PRIVATE_R */
+
+typedef enum AGGRE_SIZE {
+	HT_AGG_SIZE_8K = 0,
+	HT_AGG_SIZE_16K = 1,
+	HT_AGG_SIZE_32K = 2,
+	HT_AGG_SIZE_64K = 3,
+	VHT_AGG_SIZE_128K = 4,
+	VHT_AGG_SIZE_256K = 5,
+	VHT_AGG_SIZE_512K = 6,
+	VHT_AGG_SIZE_1024K = 7,
+} AGGRE_SIZE_E, *PAGGRE_SIZE_E;
+
+#define	LDPC_HT_ENABLE_RX			BIT0
+#define	LDPC_HT_ENABLE_TX			BIT1
+#define	LDPC_HT_TEST_TX_ENABLE		BIT2
+#define	LDPC_HT_CAP_TX				BIT3
+
+#define	STBC_HT_ENABLE_RX			BIT0
+#define	STBC_HT_ENABLE_TX			BIT1
+#define	STBC_HT_TEST_TX_ENABLE		BIT2
+#define	STBC_HT_CAP_TX				BIT3
+
+/* ------------------------------------------------------------
+ * The HT Control field
+ * ------------------------------------------------------------ */
+#define SET_HT_CTRL_CSI_STEERING(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart))+2, 6, 2, _val)
+#define SET_HT_CTRL_NDP_ANNOUNCEMENT(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart))+3, 0, 1, _val)
+#define GET_HT_CTRL_NDP_ANNOUNCEMENT(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+3, 0, 1)
+
+/* 20/40 BSS Coexist */
+#define SET_EXT_CAPABILITY_ELE_BSS_COEXIST(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 0, 1, _val)
+#define GET_EXT_CAPABILITY_ELE_BSS_COEXIST(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 0, 1)
+
+/* HT Capabilities Info field */
+#define HT_CAP_ELE_CAP_INFO(_pEleStart)					((u8 *)(_pEleStart))
+#define GET_HT_CAP_ELE_LDPC_CAP(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 0, 1)
+#define GET_HT_CAP_ELE_CHL_WIDTH(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 1, 1)
+#define GET_HT_CAP_ELE_SM_PS(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 2, 2)
+#define GET_HT_CAP_ELE_GREENFIELD(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 4, 1)
+#define GET_HT_CAP_ELE_SHORT_GI20M(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 5, 1)
+#define GET_HT_CAP_ELE_SHORT_GI40M(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 6, 1)
+#define GET_HT_CAP_ELE_TX_STBC(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 7, 1)
+#define GET_HT_CAP_ELE_RX_STBC(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+1, 0, 2)
+#define GET_HT_CAP_ELE_DELAYED_BA(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+1, 2, 1)
+#define GET_HT_CAP_ELE_MAX_AMSDU_LENGTH(_pEleStart)		LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+1, 3, 1)
+#define GET_HT_CAP_ELE_DSSS_CCK_40M(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+1, 4, 1)
+#define GET_HT_CAP_ELE_FORTY_INTOLERANT(_pEleStart)		LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+1, 6, 1)
+#define GET_HT_CAP_ELE_LSIG_TXOP_PROTECT(_pEleStart)	LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+1, 7, 1)
+
+#define SET_HT_CAP_ELE_LDPC_CAP(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 0, 1, _val)
+#define SET_HT_CAP_ELE_CHL_WIDTH(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 1, 1, _val)
+#define SET_HT_CAP_ELE_SM_PS(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 2, 2, _val)
+#define SET_HT_CAP_ELE_GREENFIELD(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 4, 1, _val)
+#define SET_HT_CAP_ELE_SHORT_GI20M(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 5, 1, _val)
+#define SET_HT_CAP_ELE_SHORT_GI40M(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 6, 1, _val)
+#define SET_HT_CAP_ELE_TX_STBC(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 7, 1, _val)
+#define SET_HT_CAP_ELE_RX_STBC(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 0, 2, _val)
+#define SET_HT_CAP_ELE_DELAYED_BA(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 2, 1, _val)
+#define SET_HT_CAP_ELE_MAX_AMSDU_LENGTH(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 3, 1, _val)
+#define SET_HT_CAP_ELE_DSSS_CCK_40M(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 4, 1, _val)
+#define SET_HT_CAP_ELE_FORTY_INTOLERANT(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 6, 1, _val)
+#define SET_HT_CAP_ELE_LSIG_TXOP_PROTECT(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 7, 1, _val)
+
+/* A-MPDU Parameters field */
+#define HT_CAP_ELE_AMPDU_PARA(_pEleStart)				(((u8 *)(_pEleStart))+2)
+#define GET_HT_CAP_ELE_MAX_AMPDU_LEN_EXP(_pEleStart)	LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+2, 0, 2)
+#define GET_HT_CAP_ELE_MIN_MPDU_S_SPACE(_pEleStart)		LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+2, 2, 3)
+
+#define HT_AMPDU_PARA_FMT "%02x " \
+	"MAX AMPDU len:%u bytes, MIN MPDU Start Spacing:%u"
+
+#define HT_AMPDU_PARA_ARG(x) \
+	*((u8 *)(x)) \
+	, (1 << (13+GET_HT_CAP_ELE_MAX_AMPDU_LEN_EXP(((u8 *)x)-2)))-1 \
+	, GET_HT_CAP_ELE_MIN_MPDU_S_SPACE(((u8 *)x)-2)
+
+#define SET_HT_CAP_ELE_MAX_AMPDU_LEN_EXP(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 2, 0, 2, _val)
+#define SET_HT_CAP_ELE_MIN_MPDU_S_SPACE(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 2, 2, 3, _val)
+
+/* Supported MCS Set field */
+#define HT_CAP_ELE_SUP_MCS_SET(_pEleStart)				(((u8 *)(_pEleStart))+3)
+#define HT_CAP_ELE_RX_MCS_MAP(_pEleStart)				HT_CAP_ELE_SUP_MCS_SET(_pEleStart)
+#define GET_HT_CAP_ELE_RX_HIGHEST_DATA_RATE(_pEleStart)	LE_BITS_TO_2BYTE(((u8 *)(_pEleStart))+13, 0, 10)
+#define GET_HT_CAP_ELE_TX_MCS_DEF(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+15, 0, 1)
+#define GET_HT_CAP_ELE_TRX_MCS_NEQ(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+15, 1, 1)
+#define GET_HT_CAP_ELE_TX_MAX_SS(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+15, 2, 2)
+#define GET_HT_CAP_ELE_TX_UEQM(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart))+15, 4, 1)
+
+#define HT_RX_MCS_BMP_FMT "%02x %02x %02x %02x %02x%02x%02x%02x%02x%02x"
+#define HT_RX_MCS_BMP_ARG(x) ((u8 *)(x))[0], ((u8 *)(x))[1], ((u8 *)(x))[2], ((u8 *)(x))[3], ((u8 *)(x))[4], ((u8 *)(x))[5], \
+	((u8 *)(x))[6], ((u8 *)(x))[7], ((u8 *)(x))[8], ((u8 *)(x))[9]
+
+#define HT_SUP_MCS_SET_FMT HT_RX_MCS_BMP_FMT \
+	/* "\n%02x%02x%02x%02x%02x%02x" */\
+	" %uMbps %s%s%s"
+#define HT_SUP_MCS_SET_ARG(x) HT_RX_MCS_BMP_ARG(x) \
+	/*,((u8 *)(x))[10], ((u8 *)(x))[11], ((u8 *)(x))[12], ((u8 *)(x))[13], ((u8 *)(x))[14], ((u8 *)(x))[15] */\
+	, GET_HT_CAP_ELE_RX_HIGHEST_DATA_RATE(((u8 *)x)-3) \
+	, GET_HT_CAP_ELE_TX_MCS_DEF(((u8 *)x)-3) ? "TX_MCS_DEF " : "" \
+	, GET_HT_CAP_ELE_TRX_MCS_NEQ(((u8 *)x)-3) ? "TRX_MCS_NEQ " : "" \
+	, GET_HT_CAP_ELE_TX_UEQM(((u8 *)x)-3) ? "TX_UEQM " : ""
+
+/* TXBF Capabilities */
+#define SET_HT_CAP_TXBF_RECEIVE_NDP_CAP(_pEleStart, _val)				SET_BITS_TO_LE_4BYTE(((u8 *)(_pEleStart))+21, 3, 1, ((u8)_val))
+#define SET_HT_CAP_TXBF_TRANSMIT_NDP_CAP(_pEleStart, _val)				SET_BITS_TO_LE_4BYTE(((u8 *)(_pEleStart))+21, 4, 1, ((u8)_val))
+#define SET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP(_pEleStart, _val)	SET_BITS_TO_LE_4BYTE(((u8 *)(_pEleStart))+21, 10, 1, ((u8)_val))
+#define SET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP(_pEleStart, _val)	SET_BITS_TO_LE_4BYTE(((u8 *)(_pEleStart))+21, 15, 2, ((u8)_val))
+#define SET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(_pEleStart, _val)	SET_BITS_TO_LE_4BYTE(((u8 *)(_pEleStart))+21, 23, 2, ((u8)_val))
+#define SET_HT_CAP_TXBF_CHNL_ESTIMATION_NUM_ANTENNAS(_pEleStart, _val)	SET_BITS_TO_LE_4BYTE(((u8 *)(_pEleStart))+21, 27, 2, ((u8)_val))
+
+
+#define GET_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP(_pEleStart)			LE_BITS_TO_4BYTE(((u8 *)(_pEleStart))+21, 10, 1)
+#define GET_HT_CAP_TXBF_EXPLICIT_COMP_FEEDBACK_CAP(_pEleStart)			LE_BITS_TO_4BYTE(((u8 *)(_pEleStart))+21, 15, 2)
+#define GET_HT_CAP_TXBF_COMP_STEERING_NUM_ANTENNAS(_pEleStart)		LE_BITS_TO_4BYTE(((u8 *)(_pEleStart))+21, 23, 2)
+#define GET_HT_CAP_TXBF_CHNL_ESTIMATION_NUM_ANTENNAS(_pEleStart)		LE_BITS_TO_4BYTE(((u8 *)(_pEleStart))+21, 27, 2)
+
+/* HT Operation element */
+
+#define GET_HT_OP_ELE_PRI_CHL(_pEleStart)					LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)), 0, 8)
+#define SET_HT_OP_ELE_PRI_CHL(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)), 0, 8, _val)
+
+/* HT Operation Info field */
+#define HT_OP_ELE_OP_INFO(_pEleStart)						(((u8 *)(_pEleStart)) + 1)
+#define GET_HT_OP_ELE_2ND_CHL_OFFSET(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 1, 0, 2)
+#define GET_HT_OP_ELE_STA_CHL_WIDTH(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 1, 2, 1)
+#define GET_HT_OP_ELE_RIFS_MODE(_pEleStart)					LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 1, 3, 1)
+#define GET_HT_OP_ELE_HT_PROTECT(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 2, 0, 2)
+#define GET_HT_OP_ELE_NON_GREEN_PRESENT(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 2, 2, 1)
+#define GET_HT_OP_ELE_OBSS_NON_HT_PRESENT(_pEleStart)		LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 2, 4, 1)
+#define GET_HT_OP_ELE_CHL_CENTER_FREQ_SEGMENT2(_pEleStart)		LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 2, 5, 8)
+#define GET_HT_OP_ELE_DUAL_BEACON(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 4, 6, 1)
+#define GET_HT_OP_ELE_DUAL_CTS(_pEleStart)					LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 4, 7, 1)
+#define GET_HT_OP_ELE_STBC_BEACON(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 5, 0, 1)
+#define GET_HT_OP_ELE_LSIG_TXOP_PROTECT(_pEleStart)			LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 5, 1, 1)
+#define GET_HT_OP_ELE_PCO_ACTIVE(_pEleStart)				LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 5, 2, 1)
+#define GET_HT_OP_ELE_PCO_PHASE(_pEleStart)					LE_BITS_TO_1BYTE(((u8 *)(_pEleStart)) + 5, 3, 1)
+
+#define SET_HT_OP_ELE_2ND_CHL_OFFSET(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 0, 2, _val)
+#define SET_HT_OP_ELE_STA_CHL_WIDTH(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 2, 1, _val)
+#define SET_HT_OP_ELE_RIFS_MODE(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 1, 3, 1, _val)
+#define SET_HT_OP_ELE_HT_PROTECT(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 2, 0, 2, _val)
+#define SET_HT_OP_ELE_NON_GREEN_PRESENT(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 2, 2, 1, _val)
+#define SET_HT_OP_ELE_OBSS_NON_HT_PRESENT(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 2, 4, 1, _val)
+#define SET_HT_OP_ELE_DUAL_BEACON(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 4, 6, 1, _val)
+#define SET_HT_OP_ELE_DUAL_CTS(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 4, 7, 1, _val)
+#define SET_HT_OP_ELE_STBC_BEACON(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 5, 0, 1, _val)
+#define SET_HT_OP_ELE_LSIG_TXOP_PROTECT(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 5, 1, 1, _val)
+#define SET_HT_OP_ELE_PCO_ACTIVE(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 5, 2, 1, _val)
+#define SET_HT_OP_ELE_PCO_PHASE(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(((u8 *)(_pEleStart)) + 5, 3, 1, _val)
+
+#endif /* _RTL871X_HT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_hwsim_intf.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_hwsim_intf.h
new file mode 100644
index 000000000000..608f548b5f9a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_hwsim_intf.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright(c) 2018 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA
+ */
+
+#ifndef __RTW_HWSIM_INTF_H_
+#define __RTW_HWSIM_INTF_H_
+
+#include <linux/skbuff.h>
+
+#include <drv_types.h>
+
+int rtw_hwsim_medium_tx(struct _ADAPTER *adapter, const void *tx_ctx,
+                        u8 *buf, size_t buflen);
+
+void rtw_hwsim_medium_pre_netif_rx(struct sk_buff *skb);
+	
+
+#endif	/* __RTW_HWSIM_INTF_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_io.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_io.h
new file mode 100644
index 000000000000..cf4cbb29b135
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_io.h
@@ -0,0 +1,40 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_IO_H_
+#define _RTW_IO_H_
+
+#ifdef CONFIG_PCI_HCI
+#define MAX_CONTINUAL_IO_ERR 4
+#endif
+
+#ifdef CONFIG_USB_HCI
+#define MAX_CONTINUAL_IO_ERR 4
+#endif
+
+#ifdef CONFIG_SDIO_HCI
+#define SD_IO_TRY_CNT (8)
+#define MAX_CONTINUAL_IO_ERR SD_IO_TRY_CNT
+#endif
+
+#ifdef CONFIG_GSPI_HCI
+#define SD_IO_TRY_CNT (8)
+#define MAX_CONTINUAL_IO_ERR SD_IO_TRY_CNT
+#endif
+
+
+int rtw_inc_and_chk_continual_io_error(struct dvobj_priv *dvobj);
+void rtw_reset_continual_io_error(struct dvobj_priv *dvobj);
+
+#endif /* _RTW_IO_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_io_records.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_io_records.h
new file mode 100644
index 000000000000..ca8ebec453e2
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_io_records.h
@@ -0,0 +1,215 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2025 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_IO_RECORDS_H_
+#define __RTW_IO_RECORDS_H_
+
+/*
+* Data struct and utilities for HCI ops
+*/
+#ifdef CONFIG_RTW_IO_RECORDS
+#ifndef DBG_RTW_IO_RECORD_DATA_LEN
+#define DBG_RTW_IO_RECORD_DATA_LEN 4 /* record 4-byte at most by default */
+#endif
+
+#ifndef DBG_RTW_IO_RECORD_TASK_INFO
+#define DBG_RTW_IO_RECORD_TASK_INFO 1
+#endif
+
+#ifndef DBG_RTW_IO_RECORD_CPU_INFO
+#define DBG_RTW_IO_RECORD_CPU_INFO 1
+#endif
+
+struct rtw_io_record {
+	bool valid;
+	sysptime stime;
+	sysptime etime;
+	u8 type; /* enum, defined by each hci */
+	u32 addr;
+	u32 cnt;
+	int err;
+#if DBG_RTW_IO_RECORD_DATA_LEN
+	u8 data[DBG_RTW_IO_RECORD_DATA_LEN];
+#endif
+#if DBG_RTW_IO_RECORD_TASK_INFO
+	char task_comm[RTW_TASK_COMM_LEN]; /* get when io done */
+#endif
+#if DBG_RTW_IO_RECORD_CPU_INFO
+	unsigned int cpu; /* get when io done */
+#endif
+};
+
+struct rtw_io_records {
+#if CONFIG_RTW_IO_RECORDS_STATIC
+	struct rtw_io_record record[CONFIG_RTW_IO_RECORDS_NUM];
+#else
+	struct rtw_io_record *record;
+#endif
+
+	size_t record_num;
+	size_t pos;
+	bool enable;
+	bool loop;
+};
+
+extern struct rtw_io_records dbg_rtw_io_records;
+
+#define __dbg_rtw_io_record_warn(_r) do {} while (0)
+
+#if DBG_RTW_IO_RECORD_DATA_LEN
+#define __dbg_rtw_io_record_fill_data_1(_r, _data) (_r)->data[0] = *((u8 *)_data)
+#define __dbg_rtw_io_record_fill_data_n(_r, _cnt, _data) _rtw_memcpy((_r)->data, _data, rtw_min(_cnt, DBG_RTW_IO_RECORD_DATA_LEN))
+#define __dbg_rtw_io_record_fill_data_w(_r, _w) *((u16 *)(_r)->data) = cpu_to_le16(_w)
+#define __dbg_rtw_io_record_fill_data_dw(_r, _dw) *((u32 *)(_r)->data) = cpu_to_le32(_dw)
+#else
+#define __dbg_rtw_io_record_fill_data_1(_r, _data) do {} while (0)
+#define __dbg_rtw_io_record_fill_data_n(_r, _cnt, _data) do {} while (0)
+#define __dbg_rtw_io_record_fill_data_w(_r, _w) do {} while (0)
+#define __dbg_rtw_io_record_fill_data_dw(_r, _l) do {} while (0)
+#endif
+
+#if DBG_RTW_IO_RECORD_TASK_INFO
+#define __dbg_rtw_io_record_fill_task_info(_r) \
+	do { \
+		rtw_get_task_comm((_r)->task_comm); \
+	} while (0)
+#else
+#define __dbg_rtw_io_record_fill_task_info(_r) do {} while (0)
+#endif
+
+#if DBG_RTW_IO_RECORD_CPU_INFO
+#define __dbg_rtw_io_record_fill_cpu_info(_r) \
+	do { \
+		(_r)->cpu = rtw_get_cpu_id(); \
+	} while (0)
+#else
+#define __dbg_rtw_io_record_fill_cpu_info(_r) do {} while (0)
+#endif
+
+#define DECLARE_DBG_RTW_IO_RECORD_P(_r) struct rtw_io_record *_r
+
+#define dbg_rtw_io_record_get_and_advance(_r) \
+	do { \
+		if (dbg_rtw_io_records.enable) { \
+			if (dbg_rtw_io_records.loop || !dbg_rtw_io_records.record[dbg_rtw_io_records.pos].valid) { \
+				(_r) = &dbg_rtw_io_records.record[dbg_rtw_io_records.pos]; \
+				dbg_rtw_io_records.pos++; \
+				if (dbg_rtw_io_records.pos >= dbg_rtw_io_records.record_num) \
+					dbg_rtw_io_records.pos = 0; \
+				(_r)->valid = true; \
+			} else { \
+				RTW_INFO("%s record full, disable\n", __func__); \
+				dbg_rtw_io_records.enable = false; \
+				(_r) = NULL; \
+			} \
+		} else \
+			(_r) = NULL; \
+	} while (0)
+
+#define dbg_rtw_io_record_fill_stime(_r) \
+	do { \
+		if (_r) { \
+			(_r)->stime = rtw_sptime_get_raw(); \
+		} \
+	} while (0)
+
+#define dbg_rtw_io_record_fill_1(_r, _type, _addr, _err, _data) \
+	do { \
+		if (_r) { \
+			(_r)->etime = rtw_sptime_get_raw(); \
+			(_r)->type = _type; \
+			(_r)->addr = _addr; \
+			(_r)->cnt = 1; \
+			(_r)->err = _err; \
+			__dbg_rtw_io_record_fill_task_info(_r); \
+			__dbg_rtw_io_record_fill_cpu_info(_r); \
+			__dbg_rtw_io_record_fill_data_1(_r, _data); \
+			__dbg_rtw_io_record_warn(_r); \
+		} \
+	} while (0)
+
+#define dbg_rtw_io_record_fill_n(_r, _type, _addr, _cnt, _err, _data) \
+	do { \
+		if (_r) { \
+			(_r)->etime = rtw_sptime_get_raw(); \
+			(_r)->type = _type; \
+			(_r)->addr = _addr; \
+			(_r)->cnt = _cnt; \
+			(_r)->err = _err; \
+			__dbg_rtw_io_record_fill_task_info(_r); \
+			__dbg_rtw_io_record_fill_cpu_info(_r); \
+			__dbg_rtw_io_record_fill_data_n(_r, _cnt, _data); \
+			__dbg_rtw_io_record_warn(_r); \
+		} \
+	} while (0)
+
+#define dbg_rtw_io_record_fill_w(_r, _type, _addr, _err, _w) \
+	do { \
+		if (_r) { \
+			(_r)->etime = rtw_sptime_get_raw(); \
+			(_r)->type = _type; \
+			(_r)->addr = _addr; \
+			(_r)->cnt = 2; \
+			(_r)->err = _err; \
+			__dbg_rtw_io_record_fill_task_info(_r); \
+			__dbg_rtw_io_record_fill_cpu_info(_r); \
+			__dbg_rtw_io_record_fill_data_w(_r, _w); \
+			__dbg_rtw_io_record_warn(_r); \
+		} \
+	} while (0)
+
+#define dbg_rtw_io_record_fill_dw(_r, _type, _addr, _err, _dw) \
+	do { \
+		if (_r) { \
+			(_r)->etime = rtw_sptime_get_raw(); \
+			(_r)->type = _type; \
+			(_r)->addr = _addr; \
+			(_r)->cnt = 4; \
+			(_r)->err = _err; \
+			__dbg_rtw_io_record_fill_task_info(_r); \
+			__dbg_rtw_io_record_fill_cpu_info(_r); \
+			__dbg_rtw_io_record_fill_data_dw(_r, _dw); \
+			__dbg_rtw_io_record_warn(_r); \
+		} \
+	} while (0)
+#else
+#define DECLARE_DBG_RTW_IO_RECORD_P(_r)
+#define dbg_rtw_io_record_get_and_advance(_r) do {} while (0)
+#define dbg_rtw_io_record_fill_stime(_r) do {} while (0)
+#define dbg_rtw_io_record_fill_1(_r, _type, _addr, _err, _data) do {} while (0)
+#define dbg_rtw_io_record_fill_n(_r, _type, _addr, _cnt, _err, _data) do {} while (0)
+#define dbg_rtw_io_record_fill_w(_r, _type, _addr, _err, _w) do {} while (0)
+#define dbg_rtw_io_record_fill_dw(_r, _type, _addr, _err, _dw) do {} while (0)
+#endif /* CONFIG_RTW_IO_RECORDS */
+
+/*
+* APIs for common code
+*/
+#ifdef CONFIG_RTW_IO_RECORDS
+static inline void rtw_io_records_dump_title(void *sel, bool tab) { _io_records_dump_title(sel, tab); }
+static inline void rtw_io_records_dump_value_by_seq(void *sel, bool tab, size_t seq) { _io_records_dump_value_by_seq(sel, tab, seq); }
+static inline void rtw_io_records_summary_dump(void *sel) { _io_records_summary_dump(sel); }
+static inline void rtw_io_records_claim_and_enable(struct dvobj_priv *d, bool enable) { _io_records_claim_and_enable(d, enable); }
+
+bool rtw_io_records_enabled(void);
+void rtw_io_records_clear(void);
+bool rtw_io_record_valid(size_t seq);
+int rtw_io_records_init(void);
+void rtw_io_records_deinit(void);
+#else
+static inline int rtw_io_records_init(void) {return _FAIL;}
+static inline void rtw_io_records_deinit(void) {}
+#endif
+
+#endif /* __RTW_IO_RECORDS_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl.h
new file mode 100644
index 000000000000..a4a2866e7714
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl.h
@@ -0,0 +1,47 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_IOCTL_H_
+#define _RTW_IOCTL_H_
+
+enum oid_type {
+	QUERY_OID,
+	SET_OID
+};
+
+struct oid_par_priv {
+	void		*adapter_context;
+	uint		oid;
+	void		*information_buf;
+	u32		information_buf_len;
+	u32		*bytes_rw;
+	u32		*bytes_needed;
+	enum oid_type	type_of_oid;
+	u32		dbg;
+};
+
+#if defined(PLATFORM_LINUX) && defined(CONFIG_WIRELESS_EXT)
+extern struct iw_handler_def  rtw_handlers_def;
+#endif
+
+extern void rtw_request_wps_pbc_event(_adapter *padapter);
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+extern int rtw_vendor_ie_get_raw_data(struct net_device *, u32, char *, u32);
+extern int rtw_vendor_ie_get_data(struct net_device*, int , char*);
+extern int rtw_vendor_ie_get(struct net_device *, struct iw_request_info *, union iwreq_data *, char *);
+extern int rtw_vendor_ie_set(struct net_device*, struct iw_request_info*, union iwreq_data*, char*);
+#endif
+
+#endif /*  #ifndef __INC_CEINFO_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl_query.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl_query.h
new file mode 100644
index 000000000000..db3ac9b68a12
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl_query.h
@@ -0,0 +1,19 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_IOCTL_QUERY_H_
+#define _RTW_IOCTL_QUERY_H_
+
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl_set.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl_set.h
new file mode 100644
index 000000000000..71e10ae70c77
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_ioctl_set.h
@@ -0,0 +1,41 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_IOCTL_SET_H_
+#define __RTW_IOCTL_SET_H_
+
+u8 rtw_set_802_11_authentication_mode(_adapter *pdapter, NDIS_802_11_AUTHENTICATION_MODE authmode);
+u8 rtw_set_802_11_bssid(_adapter *padapter, u8 *bssid);
+u8 rtw_set_802_11_add_wep(_adapter *padapter, NDIS_802_11_WEP *wep);
+u8 rtw_set_802_11_disassociate(_adapter *padapter);
+u8 rtw_set_802_11_infrastructure_mode(_adapter *padapter, NDIS_802_11_NETWORK_INFRASTRUCTURE networktype, u8 flags);
+u8 rtw_set_802_11_ssid(_adapter *padapter, NDIS_802_11_SSID *ssid);
+u8 rtw_set_802_11_connect(_adapter *padapter, const u8 *bssid, NDIS_802_11_SSID *ssid,
+			  u16 ch, enum band_type band);
+
+u8 rtw_validate_bssid(const u8 *bssid);
+u8 rtw_validate_ssid(NDIS_802_11_SSID *ssid);
+
+u16 rtw_get_cur_max_rate(_adapter *adapter);
+int rtw_set_scan_mode(_adapter *adapter, enum rtw_phl_scan_type scan_mode);
+int rtw_set_channel_plan(_adapter *adapter, u8 channel_plan, u8 chplan_6g
+	, enum rtw_env_t env, enum rtw_regd_inr inr);
+int rtw_set_country(_adapter *adapter, const char *country_code
+	, enum rtw_env_t env, enum rtw_regd_inr inr);
+#if CONFIG_IEEE80211_BAND_6GHZ
+int rtw_set_env(_adapter *adapter, enum rtw_env_t env, enum rtw_regd_inr inr);
+#endif
+int rtw_set_band(_adapter *adapter, u8 band);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mbo.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mbo.h
new file mode 100644
index 000000000000..c181f4ceface
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mbo.h
@@ -0,0 +1,210 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_MBO_H_
+#define _RTW_MBO_H_
+
+#define MBO_CH_LIST_MAX_NUM 247
+#define MBO_OCE_ELEMENT_MAX_LEN 255
+
+#define MBO_CH_PREFER_NON_OP 0
+#define MBO_CH_PREFER_NOT 1
+#define MBO_CH_PREFER_OK 255
+
+#define rtw_mbo_wifi_logo_test(a)	\
+	(((a)->registrypriv.wifi_spec) == 1)
+
+#define rtw_mbo_wifi_spec_test(a)	\
+	(((a)->mlmepriv.mbo_attr.mbo_spec_test) == 1)
+
+#define rtw_mbo_ap_assoc_disallow(a)	\
+	(((a)->mlmepriv.mbo_attr.assoc_disallow) > 0)
+
+#define rtw_mbo_add_internw_ext_cap(d, l)	\
+	rtw_add_ext_cap_info(d, l, INTERWORKING)
+
+#define rtw_mbo_wnm_notification_req(c, a) \
+	(((c) == RTW_WLAN_CATEGORY_WNM) &&	\
+	(((a) == RTW_WLAN_ACTION_WNM_NOTIF_REQ)))
+
+/* IEEE Std 802.11-2016 Table 9-46 - Status codes */
+#define RTW_ASSOC_DENIED_NO_MORE_STAS	17
+#define RTW_ASSOC_REFUSED_TEMPORARILY	30
+
+/* MBO-OCE Information Element */
+#define RTW_MBO_EID		WLAN_EID_VENDOR_SPECIFIC
+#define RTW_MBO_OUI		0x506F9A
+#define RTW_MBO_OUI_TYPE	0x16
+
+/* MBO AP Capability Indication */
+#define RTW_MBO_ATTR_AP_CAP_ID			0x1
+/* Non-preferred Channel Report */
+#define RTW_MBO_ATTR_NPREF_CH_RPT_ID	0x2
+/* Cellular Data Capabilities */
+#define RTW_MBO_ATTR_CELL_DATA_CAP_ID	0x3
+/* Association Disallowed */
+#define RTW_MBO_ATTR_ASSOC_DISABLED_ID	0x4
+/* Transition Reason Code */
+#define RTW_MBO_ATTR_TRANS_RES_ID		0x6
+/* Transition Rejection Reason Code */
+#define RTW_MBO_ATTR_TRANS_REJ_ID		0x7
+/* Association Retry Delay */
+#define RTW_MBO_ATTR_ASSOC_RETRY_DELAY_ID	0x8
+
+#define RTW_MBO_MAX_CH_LIST_NUM MAX_CHANNEL_NUM
+
+#define RTW_MBO_MAX_CH_RPT_NUM 32
+
+#define RTW_MBO_TEST_CMD_REST		0x00
+#define RTW_MBO_TEST_CMD_BTM_REQ_SEND	0xfd
+#define RTW_MBO_TEST_CMD_BTM_REQ_SET	0xfe
+#define RTW_MBO_TEST_CMD_NB_BSS_ADD	0xff
+
+enum rtw_mbo_attri_type {
+	MBO_AP_CAPABILITY = 1,
+	NON_PREFER_CHANNEL_RPT = 2,
+	CELLULAR_DATA_CAPABILITY = 3,
+	ASSOCIATION_DISALLOW = 4,
+	CELLULAR_DATA_CONNECT_PREFER = 5,
+	TRANS_REASON_CODE = 6,
+	TRANS_REJECT_REASON_CODE = 7,
+	ASSOCIATION_RETRY_DELAY = 8
+};
+
+struct rtw_mbo_ch_list {
+	u8 op_class;
+	u8 channel;
+	u8 preference;
+};
+
+struct mbo_priv {
+	u8 enable;
+	u8 assoc_disallow;
+	u8 cellular_aware;
+	struct rtw_mbo_ch_list ch_list[MBO_CH_LIST_MAX_NUM];
+	u8 ch_list_num;
+	u8 mbo_oce_element[MBO_OCE_ELEMENT_MAX_LEN];
+	u8 mbo_oce_element_len;
+};
+
+struct mbo_user_btm_req_pkt {
+	struct btm_req_hdr hdr;
+	u32 candidate_cnt;
+	struct wnm_btm_cant btm_cants[RTW_MAX_NB_RPT_NUM];
+	u8 append_mbo_ie;
+};
+
+struct mbo_attr_info {
+	u8 mbo_spec_test;
+	u8 ap_cap_ind;
+	u8 assoc_disallow;
+	u8 cell_data_cap;
+	u8 reason;
+	u16 delay;
+	struct mbo_user_btm_req_pkt user_raw;
+};
+
+struct npref_ch {
+	u8 op_class;
+	u8 chs[RTW_MBO_MAX_CH_LIST_NUM];
+	size_t nm_of_ch;
+	u8 preference;
+	u8 reason;
+};
+
+struct npref_ch_rtp {
+	struct npref_ch ch_rpt[RTW_MBO_MAX_CH_RPT_NUM];
+	size_t nm_of_rpt;
+};
+
+
+void rtw_mbo_ie_handler(_adapter *padapter, struct mbo_priv *mbopriv,
+						const u8 *pbuf, uint limit_len);
+struct sta_info;
+void rtw_ap_parse_sta_mbo_element(_adapter *padapter,
+						struct sta_info *psta, u8 *ies_buf, u16 ies_len);
+void rtw_mbo_fill_non_prefer_channel_list(_adapter *padapter, struct mbo_priv *mbopriv,
+											const u8 *pbuf, u8 len);
+
+
+
+void rtw_mbo_build_cell_data_cap_attr(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_update_ie_data(
+	_adapter *padapter, u8 *pie, u32 ie_len);
+
+void rtw_mbo_build_npref_ch_rpt_attr(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_build_trans_reject_reason_attr(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib, u8 *pres);
+
+u8 rtw_mbo_disallowed_network(struct wlan_network *pnetwork);
+
+void rtw_mbo_build_extended_cap(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+ssize_t rtw_mbo_proc_non_pref_chans_set(
+	struct file *pfile, const char __user *buffer,
+	size_t count, loff_t *pos, void *pdata);
+
+int rtw_mbo_proc_non_pref_chans_get(
+	struct seq_file *m, void *v);
+
+ssize_t rtw_mbo_proc_cell_data_set(
+	struct file *pfile, const char __user *buffer,
+	size_t count, loff_t *pos, void *pdata);
+
+int rtw_mbo_proc_cell_data_get(
+	struct seq_file *m, void *v);
+
+ssize_t rtw_mbo_proc_attr_set(
+	struct file *pfile, const char __user *buffer,
+	size_t count, loff_t *pos, void *pdata);
+
+int rtw_mbo_proc_attr_get(
+	struct seq_file *m, void *v);
+
+void rtw_mbo_wnm_notification_parsing(
+	_adapter *padapter, const u8 *pdata, size_t data_len);
+
+void rtw_mbo_build_wnm_notification(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_build_probe_req_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_build_assoc_req_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_attr_info_init(_adapter *padapter);
+
+void rtw_mbo_process_assoc_req(
+        _adapter *padapter, u8 *pie, int ie_len);
+
+void rtw_mbo_build_beacon_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_build_probe_rsp_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_build_assoc_rsp_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+void rtw_mbo_build_wnm_btmreq_reason_ies(
+	_adapter *padapter, u8 **pframe, struct pkt_attrib *pattrib);
+
+#endif /* _RTW_MBO_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mcc.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mcc.h
new file mode 100644
index 000000000000..d7e34037a16f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mcc.h
@@ -0,0 +1,14 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2015 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mem.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mem.h
new file mode 100644
index 000000000000..675c75218ce1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mem.h
@@ -0,0 +1,29 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_MEM_H__
+#define __RTW_MEM_H__
+
+#define RTKM_MGMT_SIZE PAGE_SIZE
+
+void *rtkm_kmalloc(size_t size, gfp_t flags);
+void *rtkm_kzalloc(size_t size, gfp_t flags);
+void rtkm_kfree(const void *objp, size_t size);
+
+int rtkm_prealloc_init(void);
+void rtkm_prealloc_destroy(void);
+void rtkm_dump_mstatus(void *sel);
+void rtkm_set_trace(unsigned int mask);
+
+#endif /* __RTW_MEM_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mi.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mi.h
new file mode 100644
index 000000000000..9b8c0fe9bb12
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mi.h
@@ -0,0 +1,267 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_MI_H_
+#define __RTW_MI_H_
+
+int rtw_mi_get_ch_setting_union_by_ifbmp(_adapter *adapter, u8 ifbmp, enum band_type *band, u8 *ch, u8 *bw, u8 *offset);
+int rtw_mi_get_ch_setting_union(_adapter *adapter, enum band_type *band, u8 *ch, u8 *bw, u8 *offset);
+int rtw_mi_get_ch_setting_union_no_self(_adapter *adapter, enum band_type *band, u8 *ch, u8 *bw, u8 *offset);
+int rtw_mi_get_bch_setting_union_by_hwband(struct dvobj_priv *dvobj, u8 band_idx
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset);
+int rtw_mi_get_bch_setting_union_by_hwband_ifbmp(struct dvobj_priv *dvobj, u8 band_idx, u8 ifbmp
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset);
+
+struct mi_state {
+	u8 sta_num;			/* WIFI_STATION_STATE */
+	u8 ld_sta_num;		/* WIFI_STATION_STATE && WIFI_ASOC_STATE */
+	u8 lg_sta_num;		/* WIFI_STATION_STATE && WIFI_UNDER_LINKING */
+#ifdef CONFIG_TDLS
+	u8 ld_tdls_num;		/* adapter.tdlsinfo.link_established */
+#endif
+#ifdef CONFIG_AP_MODE
+	u8 ap_num;			/* WIFI_AP_STATE && WIFI_ASOC_STATE */
+	u8 starting_ap_num;	/*WIFI_FW_AP_STATE*/
+	u8 ld_ap_num;		/* WIFI_AP_STATE && WIFI_ASOC_STATE && asoc_sta_count >= 2 */
+#endif
+	u8 adhoc_num;		/* (WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE) && WIFI_ASOC_STATE */
+	u8 ld_adhoc_num;	/* (WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE) && WIFI_ASOC_STATE && asoc_sta_count > 2 */
+#ifdef CONFIG_RTW_MESH
+	u8 mesh_num;		/* WIFI_MESH_STATE &&  WIFI_ASOC_STATE */
+	u8 ld_mesh_num;		/* WIFI_MESH_STATE &&  WIFI_ASOC_STATE && asoc_sta_count > 2 */
+#endif
+	u8 scan_num;		/* WIFI_UNDER_SURVEY */
+	u8 scan_enter_num;	/* WIFI_UNDER_SURVEY && !SCAN_DISABLE && !SCAN_BACK_OP */
+	u8 uwps_num;		/* WIFI_UNDER_WPS */
+#ifdef CONFIG_IOCTL_CFG80211
+	#ifdef CONFIG_P2P
+	u8 roch_num;
+	#endif
+	u8 mgmt_tx_num;
+#endif
+#ifdef CONFIG_P2P
+	u8 p2p_device_num;
+	u8 p2p_gc;
+	u8 p2p_go;
+#endif
+};
+
+#define MSTATE_STA_NUM(_mstate)			((_mstate)->sta_num)
+#define MSTATE_STA_LD_NUM(_mstate)		((_mstate)->ld_sta_num)
+#define MSTATE_STA_LG_NUM(_mstate)		((_mstate)->lg_sta_num)
+
+#ifdef CONFIG_TDLS
+#define MSTATE_TDLS_LD_NUM(_mstate)		((_mstate)->ld_tdls_num)
+#else
+#define MSTATE_TDLS_LD_NUM(_mstate)		0
+#endif
+
+#ifdef CONFIG_AP_MODE
+#define MSTATE_AP_NUM(_mstate)			((_mstate)->ap_num)
+#define MSTATE_AP_STARTING_NUM(_mstate)		((_mstate)->starting_ap_num)
+#define MSTATE_AP_LD_NUM(_mstate)		((_mstate)->ld_ap_num)
+#else
+#define MSTATE_AP_NUM(_mstate)			0
+#define MSTATE_AP_STARTING_NUM(_mstate)	0
+#define MSTATE_AP_LD_NUM(_mstate)		0
+#endif
+
+#define MSTATE_ADHOC_NUM(_mstate)		((_mstate)->adhoc_num)
+#define MSTATE_ADHOC_LD_NUM(_mstate)		((_mstate)->ld_adhoc_num)
+
+#ifdef CONFIG_RTW_MESH
+#define MSTATE_MESH_NUM(_mstate)		((_mstate)->mesh_num)
+#define MSTATE_MESH_LD_NUM(_mstate)		((_mstate)->ld_mesh_num)
+#else
+#define MSTATE_MESH_NUM(_mstate)		0
+#define MSTATE_MESH_LD_NUM(_mstate)		0
+#endif
+
+#define MSTATE_SCAN_NUM(_mstate)		((_mstate)->scan_num)
+#define MSTATE_SCAN_ENTER_NUM(_mstate)	((_mstate)->scan_enter_num)
+#define MSTATE_WPS_NUM(_mstate)			((_mstate)->uwps_num)
+
+#if defined(CONFIG_IOCTL_CFG80211) && defined(CONFIG_P2P)
+#define MSTATE_ROCH_NUM(_mstate)		((_mstate)->roch_num)
+#else
+#define MSTATE_ROCH_NUM(_mstate)		0
+#endif
+
+#ifdef CONFIG_P2P
+#define MSTATE_P2P_DV_NUM(_mstate)		((_mstate)->p2p_device_num)
+#define MSTATE_P2P_GC_NUM(_mstate)		((_mstate)->p2p_gc)
+#define MSTATE_P2P_GO_NUM(_mstate)		((_mstate)->p2p_go)
+#else
+#define MSTATE_P2P_DV_NUM(_mstate)		0
+#define MSTATE_P2P_GC_NUM(_mstate)		0
+#define MSTATE_P2P_GO_NUM(_mstate)		0
+#endif
+
+#if defined(CONFIG_IOCTL_CFG80211)
+#define MSTATE_MGMT_TX_NUM(_mstate)		((_mstate)->mgmt_tx_num)
+#else
+#define MSTATE_MGMT_TX_NUM(_mstate)		0
+#endif
+
+/* For now, not return union_ch/bw/offset */
+void rtw_mi_status_by_ifbmp(_adapter *adapter, u8 ifbmp, struct mi_state *mstate);
+void rtw_mi_status(_adapter *adapter, struct mi_state *mstate);
+void rtw_mi_status_no_self(_adapter *adapter, struct mi_state *mstate);
+void rtw_mi_status_no_others(_adapter *adapter, struct mi_state *mstate);
+void rtw_mi_status_by_hwband(struct dvobj_priv *dvobj, u8 band_idx, struct mi_state *mstate);
+void rtw_mi_status_by_hwband_ifbmp(struct dvobj_priv *dvobj, u8 band_idx, u8 ifbmp, struct mi_state *mstate);
+
+/* For now, not handle union_ch/bw/offset */
+void rtw_mi_status_merge(struct mi_state *d, struct mi_state *a);
+
+void rtw_mi_update_iface_status(struct mlme_priv *pmlmepriv, sint state);
+
+u8 rtw_mi_netif_stop_queue(_adapter *padapter);
+u8 rtw_mi_buddy_netif_stop_queue(_adapter *padapter);
+
+u8 rtw_mi_netif_wake_queue(_adapter *padapter);
+u8 rtw_mi_buddy_netif_wake_queue(_adapter *padapter);
+
+u8 rtw_mi_netif_carrier_on(_adapter *padapter);
+u8 rtw_mi_buddy_netif_carrier_on(_adapter *padapter);
+u8 rtw_mi_netif_carrier_off(_adapter *padapter);
+u8 rtw_mi_buddy_netif_carrier_off(_adapter *padapter);
+
+u8 rtw_mi_netif_caroff_qstop(_adapter *padapter);
+u8 rtw_mi_buddy_netif_caroff_qstop(_adapter *padapter);
+u8 rtw_mi_netif_caron_qstart(_adapter *padapter);
+u8 rtw_mi_buddy_netif_caron_qstart(_adapter *padapter);
+
+void rtw_mi_scan_abort(_adapter *adapter, bool bwait);
+void rtw_mi_buddy_scan_abort(_adapter *adapter, bool bwait);
+#if 0
+u32 rtw_mi_start_drv_threads(_adapter *adapter);
+u32 rtw_mi_buddy_start_drv_threads(_adapter *adapter);
+void rtw_mi_stop_drv_threads(_adapter *adapter);
+void rtw_mi_buddy_stop_drv_threads(_adapter *adapter);
+#endif
+
+void rtw_mi_cancel_all_timer(_adapter *adapter);
+void rtw_mi_buddy_cancel_all_timer(_adapter *adapter);
+void rtw_mi_reset_drv_sw(_adapter *adapter);
+void rtw_mi_buddy_reset_drv_sw(_adapter *adapter);
+
+void rtw_mi_suspend_free_assoc_resource(_adapter *adapter);
+void rtw_mi_buddy_suspend_free_assoc_resource(_adapter *adapter);
+
+#ifdef CONFIG_SET_SCAN_DENY_TIMER
+void rtw_mi_set_scan_deny(_adapter *adapter, u32 ms);
+void rtw_mi_buddy_set_scan_deny(_adapter *adapter, u32 ms);
+#else
+#define rtw_mi_set_scan_deny(adapter, ms) do {} while (0)
+#define rtw_mi_buddy_set_scan_deny(adapter, ms) do {} while (0)
+#endif
+
+u8 rtw_mi_is_scan_deny(_adapter *adapter);
+u8 rtw_mi_buddy_is_scan_deny(_adapter *adapter);
+
+
+u8 rtw_mi_busy_traffic_check(_adapter *padapter);
+u8 rtw_mi_buddy_busy_traffic_check(_adapter *padapter);
+
+u8 rtw_mi_check_mlmeinfo_state(_adapter *padapter, u32 state);
+u8 rtw_mi_buddy_check_mlmeinfo_state(_adapter *padapter, u32 state);
+
+u8 rtw_mi_check_fwstate(_adapter *padapter, sint state);
+u8 rtw_mi_buddy_check_fwstate(_adapter *padapter, sint state);
+u8 rtw_mi_check_fwstate_by_hwband(struct dvobj_priv *dvobj, u8 band_idx, sint state);
+
+enum {
+	MI_LINKED,
+	MI_ASSOC,
+	MI_UNDER_WPS,
+	MI_AP_MODE,
+	MI_AP_ASSOC,
+	MI_ADHOC,
+	MI_ADHOC_ASSOC,
+	MI_MESH,
+	MI_MESH_ASSOC,
+	MI_STA_NOLINK, /* this is misleading, but not used now */
+	MI_STA_LINKED,
+	MI_STA_LINKING,
+};
+u8 rtw_mi_check_status(_adapter *adapter, u8 type);
+
+void dump_dvobj_mi_status(void *sel, const char *fun_name, _adapter *adapter);
+#ifdef DBG_IFACE_STATUS
+#define DBG_IFACE_STATUS_DUMP(adapter)	dump_dvobj_mi_status(RTW_DBGDUMP, __func__, adapter)
+#endif
+void dump_mi_status(void *sel, struct dvobj_priv *dvobj);
+
+u8 rtw_mi_traffic_statistics(_adapter *padapter);
+u8 rtw_mi_check_miracast_enabled(_adapter *padapter);
+
+#ifdef CONFIG_XMIT_THREAD_MODE
+u8 rtw_mi_check_pending_xmitbuf(_adapter *padapter);
+u8 rtw_mi_buddy_check_pending_xmitbuf(_adapter *padapter);
+#endif
+
+
+u8 rtw_mi_dynamic_check_handlder(struct _ADAPTER *padapter);
+u8 rtw_mi_keep_alive_pre_chk_hdl(struct _ADAPTER *padapter);
+#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+u8 rtw_mi_dynamic_check_timer_handlder(_adapter *padapter);
+u8 rtw_mi_buddy_dynamic_check_timer_handlder(_adapter *padapter);
+#endif
+
+extern void rtw_iface_dynamic_chk_wk_hdl(_adapter *padapter);
+u8 rtw_mi_dynamic_chk_wk_hdl(_adapter *padapter);
+u8 rtw_mi_buddy_dynamic_chk_wk_hdl(_adapter *padapter);
+
+u8 rtw_mi_os_xmit_schedule(_adapter *padapter);
+u8 rtw_mi_buddy_os_xmit_schedule(_adapter *padapter);
+
+u8 rtw_mi_report_survey_event(_adapter *padapter, union recv_frame *precv_frame);
+u8 rtw_mi_buddy_report_survey_event(_adapter *padapter, union recv_frame *precv_frame);
+
+u8 rtw_mi_tx_beacon_hdl(_adapter *padapter);
+u8 rtw_mi_buddy_tx_beacon_hdl(_adapter *padapter);
+
+u8 rtw_mi_set_tx_beacon_cmd(_adapter *padapter);
+u8 rtw_mi_buddy_set_tx_beacon_cmd(_adapter *padapter);
+
+_adapter *rtw_get_iface_by_macddr(_adapter *padapter, const u8 *mac_addr);
+
+void rtw_mi_buddy_clone_bcmc_packet(_adapter *padapter, union recv_frame *precvframe);
+
+u8 rtw_mi_get_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx);
+_adapter *rtw_mi_get_iface_by_hwband(struct dvobj_priv *dvobj, u8 band_idx);
+u8 rtw_mi_get_ld_sta_ifbmp(_adapter *adapter);
+u8 rtw_mi_get_ld_sta_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx);
+u8 rtw_mi_get_lgd_sta_ifbmp(_adapter *adapter);
+u8 rtw_mi_get_lgd_sta_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx);
+u8 rtw_mi_get_ap_mesh_ifbmp(_adapter *adapter);
+u8 rtw_mi_get_ap_mesh_ifbmp_by_hwband(struct dvobj_priv *dvobj, u8 band_idx);
+_adapter *rtw_mi_get_ap_mesh_iface_by_hwband(struct dvobj_priv *dvobj, u8 band_idx);
+#ifdef	PHL_MR_PROC_CMD
+u8 rtw_mi_dump_mac_addr(_adapter *adapter);
+#endif
+
+u8 rtw_mi_disconnect(_adapter *adapter);
+u8 rtw_mi_buddy_disconnect(_adapter *adapter, struct disconnect_data *disc_data);
+
+bool rtw_iface_is_operate_at_hwband(_adapter *adapter, u8 band_idx);
+bool rtw_iface_at_same_hwband(_adapter *adapter, _adapter *iface);
+
+u8 rtw_mi_get_hw_port(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+
+u32 ifbmp_to_iflbmp(u8 ifbmp);
+u8 iflbmp_to_ifbmp(u32 iflbmp);
+
+#endif /*__RTW_MI_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mlme.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mlme.h
new file mode 100644
index 000000000000..8e1a0c6c5c8f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mlme.h
@@ -0,0 +1,1256 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_MLME_H_
+#define __RTW_MLME_H_
+
+#ifndef MAX_BSS_CNT
+#define MAX_BSS_CNT 128
+#endif
+/* #define   MAX_JOIN_TIMEOUT	2000 */
+/* #define   MAX_JOIN_TIMEOUT	2500 */
+#define   MAX_JOIN_TIMEOUT	6500
+#define   PS_ANNC_DRV_RETRY_INT_MS	30
+
+/*	Commented by Albert 20101105
+ *	Increase the scanning timeout because of increasing the SURVEY_TO value. */
+#ifndef SCANQUEUE_LIFETIME
+#define	SCANQUEUE_LIFETIME 20000 /* 20sec, unit:msec */
+#endif /*SCANQUEUE_LIFETIME*/
+
+#define MAX_UNASSOC_STA_CNT 128
+#define UNASSOC_STA_LIFETIME_MS 60000
+
+/*pmlmepriv->fw_state*/
+#define WIFI_NULL_STATE			0x00000000
+#define WIFI_ASOC_STATE			0x00000001 /* Linked */
+#define WIFI_REASOC_STATE			0x00000002
+#define WIFI_SLEEP_STATE			0x00000004
+#define WIFI_STATION_STATE			0x00000008
+#define WIFI_AP_STATE				0x00000010
+#define WIFI_ADHOC_STATE			0x00000020
+#define WIFI_ADHOC_MASTER_STATE		0x00000040
+#define WIFI_UNDER_LINKING			0x00000080
+#define WIFI_UNDER_WPS				0x00000100
+#define WIFI_MESH_STATE			0x00000200
+#define WIFI_STA_ALIVE_CHK_STATE		0x00000400
+#define WIFI_UNDER_SURVEY			0x00000800 /* under site surveying */
+#define WIFI_REGU_FORBID			0x00001000
+/*#define WIFI_UNDEFINED_STATE			0x00002000*/
+/*#define WIFI_UNDEFINED_STATE			0x00004000*/
+/*#define WIFI_UNDEFINED_STATE			0x00008000*/
+#define WIFI_MP_STATE				0x00010000
+/*#define WIFI_UNDEFINED_STATE			0x00020000*/
+/*#define WIFI_UNDEFINED_STATE			0x00040000*/
+/*#define WIFI_UNDEFINED_STATE			0x00080000*/
+/*#define WIFI_UNDEFINED_STATE			0x00100000*/
+/*#define WIFI_UNDEFINED_STATE			0x00200000*/
+/*#define WIFI_UNDEFINED_STATE			0x00400000*/
+#define WIFI_OP_CH_SWITCHING			0x00800000
+#define WIFI_UNDER_KEY_HANDSHAKE		0x01000000
+/*#define WIFI_UNDEFINED_STATE			0x02000000*/
+/*#define WIFI_UNDEFINED_STATE			0x04000000*/
+/*#define WIFI_UNDEFINED_STATE			0x08000000*/
+/*#define WIFI_UNDEFINED_STATE			0x10000000*/
+#define WIFI_CSA_SKIP_CHECK_BEACON		0x20000000
+#define WIFI_CSA_UPDATE_BEACON			0x40000000
+#define WIFI_MONITOR_STATE			0x80000000
+
+
+#define MIRACAST_DISABLED	0
+#define MIRACAST_SOURCE		BIT0
+#define MIRACAST_SINK		BIT1
+
+#define MIRACAST_MODE_REVERSE(mode) \
+	((((mode) & MIRACAST_SOURCE) ? MIRACAST_SINK : 0) | (((mode) & MIRACAST_SINK) ? MIRACAST_SOURCE : 0))
+
+bool is_miracast_enabled(_adapter *adapter);
+bool rtw_chk_miracast_mode(_adapter *adapter, u8 mode);
+const char *get_miracast_mode_str(int mode);
+void rtw_wfd_st_switch(struct sta_info *sta, bool on);
+
+#define MLME_STATE(adapter) get_fwstate(&((adapter)->mlmepriv))
+#define CHK_MLME_STATE(adapter, state) check_fwstate(&((adapter)->mlmepriv), (state))
+#define SET_MLME_STATE(adapter, state) set_fwstate(&((adapter)->mlmepriv), (state))
+#define CLR_MLME_STATE(adapter, state) clr_fwstate(&((adapter)->mlmepriv), (state))
+#define _CLR_MLME_STATE_(adapter, state) _clr_fwstate_(&((adapter)->mlmepriv), (state))
+
+/* this driver has only one adapter link */
+#define LINK_MLME_STATE(adapter_link) MLME_STATE((adapter_link)->adapter)
+#define CHK_LINK_MLME_STATE(adapter_link, state) CHK_MLME_STATE((adapter_link)->adapter, (state))
+#define SET_LINK_MLME_STATE(adapter_link, state) SET_MLME_STATE((adapter_link)->adapter, (state))
+#define CLR_LINK_MLME_STATE(adapter_link, state) CLR_MLME_STATE((adapter_link)->adapter, (state))
+#define _CLR_LINK_MLME_STATE_(adapter_link, state) _CLR_MLME_STATE_((adapter_link)->adapter, (state))
+
+#define MLME_IS_NULL(adapter) CHK_MLME_STATE(adapter, WIFI_NULL_STATE)
+#define MLME_IS_STA(adapter) CHK_MLME_STATE(adapter, WIFI_STATION_STATE)
+#define MLME_IS_AP(adapter) CHK_MLME_STATE(adapter, WIFI_AP_STATE)
+#define MLME_IS_ADHOC(adapter) CHK_MLME_STATE(adapter, WIFI_ADHOC_STATE)
+#define MLME_IS_ADHOC_MASTER(adapter) CHK_MLME_STATE(adapter, WIFI_ADHOC_MASTER_STATE)
+#define MLME_IS_MESH(adapter) CHK_MLME_STATE(adapter, WIFI_MESH_STATE)
+#define MLME_IS_MONITOR(adapter) CHK_MLME_STATE(adapter, WIFI_MONITOR_STATE)
+#define MLME_IS_MP(adapter) CHK_MLME_STATE(adapter, WIFI_MP_STATE)
+#ifdef CONFIG_P2P
+	#define MLME_IS_PD(adapter) rtw_p2p_chk_role(&(adapter)->wdinfo, P2P_ROLE_DEVICE)
+	#define MLME_IS_GC(adapter) rtw_p2p_chk_role(&(adapter)->wdinfo, P2P_ROLE_CLIENT)
+	#define MLME_IS_GO(adapter) rtw_p2p_chk_role(&(adapter)->wdinfo, P2P_ROLE_GO)
+#else /* !CONFIG_P2P */
+	#define MLME_IS_PD(adapter) 0
+	#define MLME_IS_GC(adapter) 0
+	#define MLME_IS_GO(adapter) 0
+#endif /* !CONFIG_P2P */
+
+#define MLME_IS_MSRC(adapter) rtw_chk_miracast_mode((adapter), MIRACAST_SOURCE)
+#define MLME_IS_MSINK(adapter) rtw_chk_miracast_mode((adapter), MIRACAST_SINK)
+
+#define MLME_IS_SCAN(adapter) CHK_MLME_STATE(adapter, WIFI_UNDER_SURVEY)
+#define MLME_IS_LINKING(adapter) CHK_MLME_STATE(adapter, WIFI_UNDER_LINKING)
+#define MLME_IS_ASOC(adapter) CHK_MLME_STATE(adapter, WIFI_ASOC_STATE)
+#define MLME_IS_OPCH_SW(adapter) CHK_MLME_STATE(adapter, WIFI_OP_CH_SWITCHING)
+#define MLME_IS_WPS(adapter) CHK_MLME_STATE(adapter, WIFI_UNDER_WPS)
+
+#define LINK_MLME_IS_SCAN(adapter_link) CHK_LINK_MLME_STATE(adapter_link, WIFI_UNDER_SURVEY)
+#define LINK_MLME_IS_ASOC(adapter_link) CHK_LINK_MLME_STATE(adapter_link, WIFI_ASOC_STATE)
+#define LINK_MLME_IS_OPCH_SW(adapter_link) CHK_LINK_MLME_STATE(adapter_link, WIFI_OP_CH_SWITCHING)
+#define LINK_MLME_IS_CSA_UPBCN(adapter_link) CHK_LINK_MLME_STATE(adapter_link, WIFI_CSA_UPDATE_BEACON)
+
+#if CONFIG_AP_REGU_FORBID
+#define MLME_IS_REGU_FORBID(adapter) CHK_MLME_STATE(adapter, WIFI_REGU_FORBID)
+#define LINK_MLME_IS_REGU_FORBID(adapter_link) CHK_LINK_MLME_STATE(adapter_link, WIFI_REGU_FORBID)
+#else
+#define MLME_IS_REGU_FORBID(adapter) false
+#define LINK_MLME_IS_REGU_FORBID(adapter_link) false
+#endif
+
+#if defined(CONFIG_IOCTL_CFG80211) && defined(CONFIG_P2P)
+#define MLME_IS_ROCH(adapter) (rtw_cfg80211_get_is_roch(adapter) == _TRUE)
+#else
+#define MLME_IS_ROCH(adapter) 0
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+#define MLME_IS_MGMT_TX(adapter) rtw_cfg80211_get_is_mgmt_tx(adapter)
+#else
+#define MLME_IS_MGMT_TX(adapter) 0
+#endif
+
+#define MLME_STATE_FMT "%s%s%s%s%s%s%s%s%s%s%s%s%s"
+#define MLME_STATE_ARG(adapter) \
+	MLME_IS_STA((adapter)) ? (MLME_IS_GC((adapter)) ? " GC" : " STA") : \
+	MLME_IS_AP((adapter)) ? (MLME_IS_GO((adapter)) ? " GO" : " AP") : \
+	MLME_IS_ADHOC((adapter)) ? " ADHOC" : \
+	MLME_IS_ADHOC_MASTER((adapter)) ? " ADHOC_M" : \
+	MLME_IS_MESH((adapter)) ? " MESH" : \
+	MLME_IS_MONITOR((adapter)) ? " MONITOR" : \
+	MLME_IS_MP((adapter)) ? " MP" : "", \
+	MLME_IS_PD((adapter)) ? " PD" : "", \
+	MLME_IS_MSRC((adapter)) ? " MSRC" : "", \
+	MLME_IS_MSINK((adapter)) ? " MSINK" : "", \
+	MLME_IS_SCAN((adapter)) ? " SCAN" : "", \
+	MLME_IS_LINKING((adapter)) ? " LINKING" : "", \
+	MLME_IS_ASOC((adapter)) ? " ASOC" : "", \
+	MLME_IS_OPCH_SW((adapter)) ? " OPCH_SW" : "", \
+	MLME_IS_REGU_FORBID((adapter)) ? " REGU_FORBID" : "", \
+	MLME_IS_WPS((adapter)) ? " WPS" : "", \
+	MLME_IS_ROCH((adapter)) ? " ROCH" : "", \
+	MLME_IS_MGMT_TX((adapter)) ? " MGMT_TX" : "", \
+	(MLME_STATE((adapter)) & WIFI_SLEEP_STATE) ? " SLEEP" : ""
+
+enum {
+	MLME_ACTION_UNKNOWN,
+	MLME_ACTION_NONE,
+	MLME_SCAN_ENABLE, /* WIFI_UNDER_SURVEY */
+	MLME_SCAN_ENTER, /* WIFI_UNDER_SURVEY && !SCAN_DISABLE && !SCAN_BACK_OP */
+	MLME_SCAN_DONE, /*  WIFI_UNDER_SURVEY && (SCAN_DISABLE || SCAN_BACK_OP) */
+	MLME_SCAN_DISABLE, /* WIFI_UNDER_SURVEY is going to be cleared */
+	MLME_STA_CONNECTING,
+	MLME_STA_CONNECTED,
+	MLME_STA_DISCONNECTED,
+	MLME_TDLS_LINKED,
+	MLME_TDLS_NOLINK,
+	MLME_AP_STARTED,
+	MLME_AP_STOPPED,
+	MLME_ADHOC_STARTED,
+	MLME_ADHOC_STOPPED,
+	MLME_MESH_STARTED,
+	MLME_MESH_STOPPED,
+	MLME_OPCH_SWITCH,
+};
+
+enum dot11AuthAlgrthmNum {
+	dot11AuthAlgrthm_Open = 0,
+	dot11AuthAlgrthm_Shared,
+	dot11AuthAlgrthm_8021X,
+	dot11AuthAlgrthm_Auto,
+	dot11AuthAlgrthm_WAPI,
+	dot11AuthAlgrthm_MaxNum
+};
+
+/**
+ * enum mlme_auth_type - AuthenticationType
+ *
+ * @MLME_AUTHTYPE_OPEN_SYSTEM: Open System authentication
+ * @MLME_AUTHTYPE_SHARED_KEY: Shared Key authentication (WEP only)
+ * @MLME_AUTHTYPE_FT: Fast BSS Transition (IEEE 802.11r)
+ * @MLME_AUTHTYPE_NETWORK_EAP: Network EAP (some Cisco APs and mainly LEAP)
+ * @MLME_AUTHTYPE_SAE: Simultaneous authentication of equals
+ * @MLME_AUTHTYPE_FILS_SK: Fast Initial Link Setup shared key
+ * @MLME_AUTHTYPE_FILS_SK_PFS: Fast Initial Link Setup shared key with PFS
+ * @MLME_AUTHTYPE_FILS_PK: Fast Initial Link Setup public key
+ * @__MLME_AUTHTYPE_NUM: internal
+ * @MLME_AUTHTYPE_MAX: maximum valid auth algorithm
+ * @MLME_AUTHTYPE_AUTOMATIC: determine automatically (if necessary by trying
+ *      multiple times); this is invalid in netlink -- leave out the attribute
+ *      for this on CONNECT commands.
+ */
+enum mlme_auth_type {
+	MLME_AUTHTYPE_OPEN_SYSTEM,
+	MLME_AUTHTYPE_SHARED_KEY,
+	MLME_AUTHTYPE_FT,
+	MLME_AUTHTYPE_NETWORK_EAP,
+	MLME_AUTHTYPE_SAE,
+	MLME_AUTHTYPE_FILS_SK,
+	MLME_AUTHTYPE_FILS_SK_PFS,
+	MLME_AUTHTYPE_FILS_PK,
+
+	/* keep last */
+	__MLME_AUTHTYPE_NUM,
+	MLME_AUTHTYPE_MAX = __MLME_AUTHTYPE_NUM - 1,
+	MLME_AUTHTYPE_AUTOMATIC
+};
+
+
+#define WIFI_FREQUENCY_BAND_AUTO 0
+#define WIFI_FREQUENCY_BAND_5GHZ 1
+#define WIFI_FREQUENCY_BAND_2GHZ 2
+
+#define rtw_band_valid(band) ((band) <= WIFI_FREQUENCY_BAND_2GHZ)
+
+enum SCAN_RESULT_TYPE {
+	SCAN_RESULT_P2P_ONLY = 0,		/*	Will return all the P2P devices. */
+	SCAN_RESULT_ALL = 1,			/*	Will return all the scanned device, include AP. */
+	SCAN_RESULT_WFD_TYPE = 2		/*	Will just return the correct WFD device. */
+									/*	If this device is Miracast sink device, it will just return all the Miracast source devices. */
+};
+
+/*
+
+there are several "locks" in mlme_priv,
+since mlme_priv is a shared resource between many threads,
+like ISR/Call-Back functions, the OID handlers, and even timer functions.
+
+
+Each _queue has its own locks, already.
+Other items are protected by mlme_priv.lock.
+
+To avoid possible dead lock, any thread trying to modifiying mlme_priv
+SHALL not lock up more than one locks at a time!
+
+*/
+
+
+#define traffic_threshold	10
+#define	traffic_scan_period	500
+
+typedef struct _RT_LINK_DETECT_T {
+	u32				NumTxOkInPeriod;
+	u32				NumRxOkInPeriod;
+	u32				NumRxUnicastOkInPeriod;
+	BOOLEAN			bBusyTraffic;
+	BOOLEAN			bTxBusyTraffic;
+	BOOLEAN			bRxBusyTraffic;
+	BOOLEAN			bHigherBusyTraffic; /* For interrupt migration purpose. */
+	BOOLEAN			bHigherBusyRxTraffic; /* We may disable Tx interrupt according as Rx traffic. */
+	BOOLEAN			bHigherBusyTxTraffic; /* We may disable Tx interrupt according as Tx traffic. */
+	/* u8 TrafficBusyState; */
+	u8 TrafficTransitionCount;
+	u32 LowPowerTransitionCount;
+} RT_LINK_DETECT_T, *PRT_LINK_DETECT_T;
+
+#ifdef CONFIG_WFD
+
+struct wifi_display_info {
+	u16							wfd_enable;			/*	Eanble/Disable the WFD function. */
+	u16							init_rtsp_ctrlport;	/* init value of rtsp_ctrlport when WFD enable */
+	u16							rtsp_ctrlport;		/* TCP port number at which the this WFD device listens for RTSP messages, 0 when WFD disable */
+	u16							tdls_rtsp_ctrlport;	/* rtsp_ctrlport used by tdls, will sync when rtsp_ctrlport is changed by user */
+	u16							peer_rtsp_ctrlport;	/*	TCP port number at which the peer WFD device listens for RTSP messages */
+													/*	This filed should be filled when receiving the gropu negotiation request */
+
+	u8							peer_session_avail;	/*	WFD session is available or not for the peer wfd device. */
+													/*	This variable will be set when sending the provisioning discovery request to peer WFD device. */
+													/*	And this variable will be reset when it is read by using the iwpriv p2p_get wfd_sa command. */
+	u8							ip_address[4];
+	u8							peer_ip_address[4];
+	u8							wfd_pc;				/*	WFD preferred connection */
+													/*	0 -> Prefer to use the P2P for WFD connection on peer side. */
+													/*	1 -> Prefer to use the TDLS for WFD connection on peer side. */
+
+	u8							wfd_device_type;	/*	WFD Device Type */
+													/*	0 -> WFD Source Device */
+													/*	1 -> WFD Primary Sink Device */
+	enum	SCAN_RESULT_TYPE	scan_result_type;	/*	Used when P2P is enable. This parameter will impact the scan result. */
+	u8 op_wfd_mode;
+	u8 stack_wfd_mode;
+};
+#endif /* CONFIG_WFD */
+
+#ifdef CONFIG_IOCTL_CFG80211
+struct cfg80211_roch_info {
+	u8						restore_channel;
+	struct ieee80211_channel	remain_on_ch_channel;
+	enum nl80211_channel_type	remain_on_ch_type;
+	unsigned int duration;
+	ATOMIC_T ro_ch_cookie_gen;
+	u64 remain_on_ch_cookie;
+	bool is_ro_ch;
+	struct wireless_dev *ro_ch_wdev;
+	systime last_ro_ch_time; /* this will be updated at the beginning and end of ro_ch */
+};
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+#ifdef CONFIG_P2P_WOWLAN
+
+enum P2P_WOWLAN_RECV_FRAME_TYPE {
+	P2P_WOWLAN_RECV_NEGO_REQ = 0,
+	P2P_WOWLAN_RECV_INVITE_REQ = 1,
+	P2P_WOWLAN_RECV_PROVISION_REQ = 2,
+};
+
+struct p2p_wowlan_info {
+
+	u8						is_trigger;
+	enum P2P_WOWLAN_RECV_FRAME_TYPE	wowlan_recv_frame_type;
+	u8						wowlan_peer_addr[ETH_ALEN];
+	u16						wowlan_peer_wpsconfig;
+	u8						wowlan_peer_is_persistent;
+	u8						wowlan_peer_invitation_type;
+};
+
+#endif /* CONFIG_P2P_WOWLAN */
+
+struct wifidirect_info {
+	_adapter				*padapter;
+
+#ifdef CONFIG_WFD
+	struct wifi_display_info		*wfd_info;
+#endif
+
+#ifdef CONFIG_P2P_WOWLAN
+	struct p2p_wowlan_info		p2p_wow_info;
+#endif /* CONFIG_P2P_WOWLAN */
+
+	enum P2P_ROLE			role;
+	u8						listen_channel;
+	u8						support_rate[8];
+	u8						p2p_wildcard_ssid[P2P_WILDCARD_SSID_LEN];
+	u8						wfd_tdls_enable;			/*	Flag to enable or disable the TDLS by WFD Sigma */
+														/*	0: disable */
+														/*	1: enable */
+	u8						wfd_tdls_weaksec;			/*	Flag to enable or disable the weak security function for TDLS by WFD Sigma */
+														/*	0: disable */
+														/*	In this case, the driver can't issue the tdsl setup request frame. */
+														/*	1: enable */
+														/*	In this case, the driver can issue the tdls setup request frame */
+														/*	even the current security is weak security. */
+
+#ifdef CONFIG_P2P_PS
+	enum P2P_PS_MODE		p2p_ps_mode; /* indicate p2p ps mode */
+	enum P2P_PS_STATE		p2p_ps_state; /* indicate p2p ps state */
+	u8						noa_index; /* Identifies and instance of Notice of Absence timing. */
+	u8						ctwindow; /* Client traffic window. A period of time in TU after TBTT. */
+	u8						opp_ps; /* opportunistic power save. */
+	u8						noa_num; /* number of NoA descriptor in P2P IE. */
+	u8						noa_count[P2P_MAX_NOA_NUM]; /* Count for owner, Type of client. */
+	u32						noa_duration[P2P_MAX_NOA_NUM]; /* Max duration for owner, preferred or min acceptable duration for client. */
+	u32						noa_interval[P2P_MAX_NOA_NUM]; /* Length of interval for owner, preferred or max acceptable interval of client. */
+	u32						noa_start_time[P2P_MAX_NOA_NUM]; /* schedule expressed in terms of the lower 4 bytes of the TSF timer. */
+#endif /* CONFIG_P2P_PS */
+#ifdef PRIVATE_R
+	u8						remote_mac_address[48]; /* For Roku find remote function */
+	u32						num_of_remote;
+#endif
+};
+
+struct tdls_ss_record {	/* signal strength record */
+	u8		macaddr[ETH_ALEN];
+	u8		RxPWDBAll;
+	u8		is_tdls_sta;	/* _TRUE: direct link sta, _FALSE: else */
+};
+
+struct tdls_temp_mgmt {
+	u8	initiator;	/* 0: None, 1: we initiate, 2: peer initiate */
+	u8	peer_addr[ETH_ALEN];
+};
+
+#ifdef CONFIG_TDLS_CH_SW
+struct tdls_ch_switch {
+	u32	ch_sw_state;
+	ATOMIC_T	chsw_on;
+	u8	addr[ETH_ALEN];
+	u8	off_ch_num;
+	u8	ch_offset;
+	u32	cur_time;
+	u8	delay_switch_back;
+	u8	dump_stack;
+	struct submit_ctx	chsw_sctx;
+};
+#endif
+
+struct tdls_info {
+	u8					ap_prohibited;
+	u8					ch_switch_prohibited;
+	u8					link_established;
+	u8					sta_cnt;
+	u8					sta_maximum;	/* 1:tdls sta is equal (NUM_STA-1), reach max direct link number; 0: else; */
+	struct tdls_ss_record	ss_record;
+#ifdef CONFIG_TDLS_CH_SW
+	struct tdls_ch_switch	chsw_info;
+#endif
+
+	u8					ch_sensing;
+	u8					cur_channel;
+	u8					collect_pkt_num[MAX_CHANNEL_NUM];
+	_lock				cmd_lock;
+	_lock				hdl_lock;
+	u8					watchdog_count;
+	u8					dev_discovered;		/* WFD_TDLS: for sigma test */
+
+	/* Let wpa_supplicant to setup*/
+	u8					driver_setup;
+#ifdef CONFIG_WFD
+	struct wifi_display_info		*wfd_info;
+#endif
+
+	struct submit_ctx	*tdls_sctx;
+};
+
+struct tdls_txmgmt {
+	u8 peer[ETH_ALEN];
+	u8 action_code;
+	u8 dialog_token;
+	u16 status_code;
+	u8 *buf;
+	size_t len;
+};
+
+/* used for mlme_priv.roam_flags */
+enum {
+	RTW_ROAM_ON_EXPIRED = BIT0,
+	RTW_ROAM_ON_RESUME = BIT1,
+	RTW_ROAM_ACTIVE = BIT2,
+	RTW_ROAM_QUICK_SCAN = BIT3,
+	RTW_ROAM_BTM = BIT4,
+};
+
+#define UNASOC_STA_SRC_RX_BMC		0
+#define UNASOC_STA_SRC_RX_NMY_UC	1
+#define UNASOC_STA_SRC_NUM			2
+
+#define UNASOC_STA_MODE_DISABLED	0
+#define UNASOC_STA_MODE_INTERESTED	1
+#define UNASOC_STA_MODE_ALL			2
+#define UNASOC_STA_MODE_NUM			3
+
+#define UNASOC_STA_DEL_CHK_SKIP		0
+#define UNASOC_STA_DEL_CHK_ALIVE	1
+#define UNASOC_STA_DEL_CHK_DELETED	2
+
+#ifdef CONFIG_RTW_MULTI_AP
+struct unassoc_sta_info {
+	_list list;
+	u8 addr[ETH_ALEN];
+	u8 interested;
+	s8 recv_signal_power;
+	systime time;
+};
+#endif
+
+struct	wlan_network {
+	_list	list;
+	int	network_type;	/* refer to ieee80211.h for WIRELESS_11A/B/G */
+	int	fixed;			/* set to fixed when not to be removed as site-surveying */
+	systime last_scanned; /* timestamp for the network */
+	systime last_non_hidden_ssid_ap;
+#ifdef CONFIG_RTW_MESH
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	systime acnode_stime;
+	systime acnode_notify_etime;
+#endif
+#endif
+	int	aid;			/* will only be valid when a BSS is joinned. */
+	int	join_res;
+	struct beacon_keys bcn_keys;
+	bool bcn_keys_valid;
+#ifdef CONFIG_80211D
+	struct country_ie_slave_record cisr;
+#endif
+	WLAN_BSSID_EX	network; /* must be the last item */
+};
+
+#ifdef PRIVATE_R
+#define MAX_VENDOR_IE_NUM 10
+#define MAX_VENDOR_IE_LEN 255
+#define MAX_VENDOR_IE_PARAM_LEN MAX_VENDOR_IE_LEN + 2	/* vendor ie filter index + content maximum length */
+
+#define MAX_NUM_DIS_BCN_INFO 3
+struct dis_bcn_key {
+	_list	list;
+	struct beacon_keys bcn_content;
+};
+#endif
+
+struct link_mlme_priv {
+	u8 	link_id;		/* valid for connectint to MLD network
+					** is parsed from RN and ML IEs */
+	u8 	is_accepted;	/* Accepted by AP or not */
+	u8	to_join;	/* Desired link or not */
+	struct wlan_network	cur_network;
+
+	/* bcn check info */
+	struct beacon_keys cur_beacon_keys; /* save current beacon keys */
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+	struct beacon_keys new_beacon_keys; /* save new beacon keys */
+	u8 new_beacon_cnts; /* if new_beacon_cnts >= threshold, ap beacon is changed */
+#endif
+#ifdef PRIVATE_R
+	_queue idle_dis_bcn_queue;
+	_queue busy_dis_bcn_queue;
+	struct dis_bcn_key dis_bcn_key_info[MAX_NUM_DIS_BCN_INFO];
+#endif
+#ifdef CONFIG_80211N_HT
+
+	/* Number of non-HT AP/stations */
+	int num_sta_no_ht;
+
+	/* Number of HT AP/stations 20 MHz */
+	/* int num_sta_ht_20mhz; */
+
+
+	int num_FortyMHzIntolerant;
+	u8 sw_to_20mhz; /*switch to 20Mhz BW*/
+
+	struct ht_priv	htpriv;
+	struct ampdu_priv ampdu_priv;
+#endif
+	/* VHT or HE IE is configured by upper layer or not (hostapd or wpa_supplicant) */
+	u8 upper_layer_setting;
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv	vhtpriv;
+#ifdef PRIVATE_R
+	/*infra mode, used to store AP's info*/
+	struct vht_priv_infra_ap vhtpriv_infra_ap;
+#endif /* PRIVATE_R */
+#endif
+
+#ifdef PRIVATE_R
+	struct ht_priv_infra_ap htpriv_infra_ap;
+#endif /* PRIVATE_R */
+
+#ifdef CONFIG_80211AX_HE
+	struct he_priv hepriv;
+#endif
+	struct qos_priv qospriv;
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+	/* Number of associated Non-ERP stations (i.e., stations using 802.11b
+	 * in 802.11g BSS) */
+	int num_sta_non_erp;
+
+	/* Number of associated stations that do not support Short Slot Time */
+	int num_sta_no_short_slot_time;
+
+	/* Number of associated stations that do not support Short Preamble */
+	int num_sta_no_short_preamble;
+
+	ATOMIC_T olbc; /* Overlapping Legacy BSS Condition (Legacy b/g)*/
+
+	/* Number of HT associated stations that do not support greenfield */
+	int num_sta_ht_no_gf;
+
+	/* Number of associated non-HT stations */
+	/* int num_sta_no_ht; */
+
+	/* Number of HT associated stations 20 MHz */
+	int num_sta_ht_20mhz;
+
+	/* number of associated stations 40MHz intolerant */
+	int num_sta_40mhz_intolerant;
+
+	/* Overlapping BSS information */
+	ATOMIC_T olbc_ht;
+
+	_lock	bcn_update_lock;
+	u8 update_bcn;
+	struct rtw_chan_def ori_chandef;
+#endif
+	u8 ext_capab_ie_data[WLAN_EID_EXT_CAP_MAX_LEN];/*currently for ap mode only*/
+	u8 ext_capab_ie_len;
+
+#ifdef CONFIG_80211N_HT
+	int ht_20mhz_width_req;
+	int ht_intolerant_ch_reported;
+	u16 ht_op_mode;
+#endif /* CONFIG_80211N_HT */
+};
+
+struct mlme_priv {
+
+	_lock	lock;
+	sint	fw_state;	/* shall we protect this variable? maybe not necessarily... */
+	u8	to_join; /* flag */
+	u16 join_status;
+#ifdef CONFIG_LAYER2_ROAMING
+	u8 roam_scan_order[4];
+	u8 to_roam; /* roaming trying times */
+	struct wlan_network *roam_network; /* the target of active roam */
+	u8 roam_flags;
+	u8 roam_rssi_diff_th; /* rssi difference threshold for active scan candidate selection */
+	u32 roam_scan_int; 		/* scan interval for active roam (Unit:2 second)*/
+	u32 roam_scanr_exp_ms; /* scan result expire time in ms  for roam */
+	u8 roam_tgt_addr[ETH_ALEN]; /* request to roam to speicific target without other consideration */
+	u8 roam_rssi_threshold;
+	systime last_roaming;
+	bool need_to_roam;
+#endif
+
+	u32 defs_lmt_sta;
+	u32 defs_lmt_time;
+
+	u8	*nic_hdl;
+	u32	max_bss_cnt;		/*	The size of scan queue	*/
+	_list	*pscanned;
+	_queue	free_bss_pool;
+	_queue	scanned_queue;
+	u8		*free_bss_buf;
+	u32		num_of_scanned;
+
+	_queue	free_mld_bss_pool;
+	_queue	scanned_mld_queue;
+	u8		*free_mld_bss_buf;
+	u32		num_of_scanned_mld;
+
+	NDIS_802_11_SSID	assoc_ssid;
+	u8	assoc_bssid[6];
+	u16	assoc_ch;		/* 0 reserved for no specific channel */
+	enum band_type	assoc_band;
+
+	struct wlan_network	dev_cur_network;
+	struct wlan_network *cur_network_scanned;
+
+#if CONFIG_DFS
+	u8 bcn_cnts_after_csa;
+#endif
+
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+	/* for arp offload keep alive */
+	u8 bGetGateway;
+	u8	GetGatewayTryCnt;
+	u8	gw_mac_addr[ETH_ALEN];
+	u8	gw_ip[4];
+#endif
+
+	/* uint wireless_mode; no used, remove it */
+
+	u32	auto_scan_int_ms;
+
+	_timer assoc_timer;
+
+	uint assoc_by_bssid;
+	uint assoc_by_rssi;
+
+	systime scan_start_time; /* used to evaluate the time spent in scanning */
+
+#ifdef CONFIG_SET_SCAN_DENY_TIMER
+	_timer set_scan_deny_timer;
+	ATOMIC_T set_scan_deny; /* 0: allowed, 1: deny */
+#endif
+	u8 wpa_phase;/*wpa_phase after wps finished*/
+
+#ifdef CONFIG_80211N_HT
+	struct ht_priv	dev_htpriv;
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv	dev_vhtpriv;
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	struct he_priv dev_hepriv;
+#endif
+
+#ifdef CONFIG_80211BE_EHT
+	struct mlo_priv mlopriv;
+#endif
+
+#ifdef CONFIG_RTW_MBO
+	struct mbo_priv mbopriv;
+#endif
+
+#ifdef CONFIG_RTW_80211R
+	struct ft_roam_info ft_roam;
+#endif
+#if defined(CONFIG_RTW_WNM) || defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	struct roam_nb_info nb_info;
+	u8 ch_cnt;
+#endif
+
+	RT_LINK_DETECT_T	LinkDetectInfo;
+
+	u8	acm_mask; /* for wmm acm mask */
+	enum rtw_phl_scan_type scan_mode; /* active: 1, passive: 0 */
+
+	u8 *wps_probe_req_ie;
+	u32 wps_probe_req_ie_len;
+
+#if defined(CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME)
+#ifdef CONFIG_RTW_80211R
+	u8 *auth_rsp;
+	u32 auth_rsp_len;
+#endif
+	u8 *assoc_req;
+	u32 assoc_req_len;
+
+	u8 *assoc_rsp;
+	u32 assoc_rsp_len;
+
+	u8 *probe_rsp;
+	u32 probe_rsp_len;
+
+	/* u8 *wps_probe_req_ie; */
+	/* u32 wps_probe_req_ie_len; */
+
+	u8 *beacon_head_ie;
+	u32 beacon_head_ie_len;
+	u8 *beacon_tail_ie;
+	u32 beacon_tail_ie_len;
+
+	u8 *wps_beacon_ie;
+	u32 wps_beacon_ie_len;
+
+	u8 *wps_probe_resp_ie;
+	u32 wps_probe_resp_ie_len;
+
+	u8 *wps_assoc_resp_ie;
+	u32 wps_assoc_resp_ie_len;
+
+	u8 *p2p_beacon_ie;
+	u32 p2p_beacon_ie_len;
+
+	u8 *p2p_probe_req_ie;
+	u32 p2p_probe_req_ie_len;
+
+	u8 *p2p_probe_resp_ie;
+	u32 p2p_probe_resp_ie_len;
+
+	u8 *p2p_go_probe_resp_ie;		/* for GO */
+	u32 p2p_go_probe_resp_ie_len;	/* for GO */
+
+	u8 *p2p_assoc_req_ie;
+	u32 p2p_assoc_req_ie_len;
+
+	u8 *p2p_assoc_resp_ie;
+	u32 p2p_assoc_resp_ie_len;
+
+	_lock	bcn_update_lock;
+	u8 update_bcn;
+
+	struct rtw_chan_def ori_chandef;
+	#ifdef CONFIG_80211AC_VHT
+	u8 ori_vht_en;
+	#endif
+	u8 ap_isolate;
+#endif /* #if defined (CONFIG_AP_MODE) && defined (CONFIG_NATIVEAP_MLME) */
+
+#if defined(CONFIG_WFD) && defined(CONFIG_IOCTL_CFG80211)
+	u8 *wfd_beacon_ie;
+	u32 wfd_beacon_ie_len;
+
+	u8 *wfd_probe_req_ie;
+	u32 wfd_probe_req_ie_len;
+
+	u8 *wfd_probe_resp_ie;
+	u32 wfd_probe_resp_ie_len;
+
+	u8 *wfd_go_probe_resp_ie;		/* for GO */
+	u32 wfd_go_probe_resp_ie_len;	/* for GO */
+
+	u8 *wfd_assoc_req_ie;
+	u32 wfd_assoc_req_ie_len;
+
+	u8 *wfd_assoc_resp_ie;
+	u32 wfd_assoc_resp_ie_len;
+#endif
+
+#ifdef CONFIG_RTW_MBO
+	u8 *pcell_data_cap_ie;
+	u32 cell_data_cap_len;
+	struct mbo_attr_info mbo_attr;
+#endif
+
+#ifdef RTK_DMP_PLATFORM
+	/* DMP kobject_hotplug function  signal need in passive level */
+	_workitem	Linkup_workitem;
+	_workitem	Linkdown_workitem;
+#endif
+
+#ifdef RTW_BUSY_DENY_SCAN
+	systime lastscantime;
+#endif
+
+#ifdef CONFIG_CONCURRENT_MODE
+	u8	scanning_via_buddy_intf;
+#endif
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	u32 vendor_ie_mask[WLAN_MAX_VENDOR_IE_NUM];
+	u8 vendor_ie[WLAN_MAX_VENDOR_IE_NUM][WLAN_MAX_VENDOR_IE_LEN];
+	u32 vendor_ielen[WLAN_MAX_VENDOR_IE_NUM];
+#endif
+#ifdef CONFIG_RTW_MULTI_AP
+	u8 unassoc_sta_mode_of_stype[UNASOC_STA_SRC_NUM];
+	_queue unassoc_sta_queue;
+	_queue free_unassoc_sta_queue;
+	u8 *free_unassoc_sta_buf;
+	u32 interested_unassoc_sta_cnt;
+	u32 max_unassoc_sta_cnt;
+#endif
+#ifdef PRIVATE_R
+	u8 vendor_ie_filter[MAX_VENDOR_IE_NUM][MAX_VENDOR_IE_LEN];
+	u8 vendor_ie_len[MAX_VENDOR_IE_NUM];
+	u8 vendor_ie_filter_enable;
+#endif
+};
+
+#define rtw_mlme_set_auto_scan_int(adapter, ms) \
+	do { \
+		adapter->mlmepriv.auto_scan_int_ms = ms; \
+	} while (0)
+
+#define set_assoc_timer(mlme, ms) \
+	do { \
+		/*RTW_INFO("%s set_assoc_timer(%p, %d)\n", __FUNCTION__, (mlme), (ms));*/ \
+		_set_timer(&(mlme)->assoc_timer, (ms)); \
+	} while (0)
+#define cancel_assoc_timer(mlme) \
+	do { \
+		/*RTW_INFO("%s cancel_assoc_timer(%p)\n", __FUNCTION__, (mlme));*/ \
+		_cancel_timer_ex(&(mlme)->assoc_timer); \
+	} while (0)
+
+#define RTW_AUTO_SCAN_REASON_UNSPECIFIED	0
+#define RTW_AUTO_SCAN_REASON_2040_BSS		BIT0
+#define RTW_AUTO_SCAN_REASON_ACS		BIT1
+#define RTW_AUTO_SCAN_REASON_ROAM		BIT2
+#define RTW_AUTO_SCAN_REASON_ROAM_ACTIVE	BIT3
+#define RTW_AUTO_SCAN_REASON_MESH_OFFCH_CAND	BIT4
+#define RTW_AUTO_SCAN_REASON_CIS_ENV_BSS	BIT5
+
+void rtw_mlme_reset_auto_scan_int(_adapter *adapter, u8 *reason);
+
+#ifdef CONFIG_AP_MODE
+
+struct hostapd_priv {
+	_adapter *padapter;
+
+#ifdef CONFIG_HOSTAPD_MLME
+	struct net_device *pmgnt_netdev;
+	struct usb_anchor anchored;
+#endif
+
+};
+
+extern int hostapd_mode_init(_adapter *padapter);
+extern void hostapd_mode_unload(_adapter *padapter);
+#endif
+
+
+extern void rtw_joinbss_event_prehandle(_adapter *adapter, u8 *pbuf, u16 status);
+extern void rtw_joinbss_event_callback(_adapter *adapter, u8 *pbuf);
+extern void rtw_stassoc_event_callback(_adapter *adapter, u8 *pbuf);
+extern void rtw_stadel_event_callback(_adapter *adapter, u8 *pbuf);
+extern void rtw_wmm_event_callback(_adapter *padapter, u8 *pbuf);
+#ifdef CONFIG_IEEE80211W
+void rtw_sta_timeout_event_callback(_adapter *adapter, u8 *pbuf);
+#endif /* CONFIG_IEEE80211W */
+
+extern void rtw_free_network_queue(_adapter *adapter, u8 isfreeall);
+
+#ifdef PRIVATE_R
+struct dis_bcn_key *_rtw_alloc_dis_bcn_key(_queue *pqueue);
+struct dis_bcn_key *rtw_alloc_dis_bcn_key(_queue *pidle_queue, _queue *pbusy_queue);
+int rtw_enqueue_dis_bcn_key(struct  dis_bcn_key  *pdis_bcn_key, _queue *pbusy_queue);
+#endif
+
+extern int rtw_init_mlme_priv(_adapter *adapter);/* (struct mlme_priv *pmlmepriv); */
+
+extern void rtw_free_mlme_priv(struct mlme_priv *pmlmepriv);
+
+extern sint rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv);
+extern sint rtw_set_key(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+			sint keyid, u8 set_tx, bool enqueue);
+extern sint rtw_set_auth(_adapter *adapter, struct security_priv *psecuritypriv);
+
+__inline static u8 *get_link_bssid(struct link_mlme_priv *pmlmepriv)
+{
+	/* if sta_mode:pmlmepriv->cur_network.network.MacAddress=> bssid */
+	/* if adhoc_mode:pmlmepriv->cur_network.network.MacAddress=> ibss mac address */
+	return pmlmepriv->cur_network.network.MacAddress;
+}
+
+__inline static u8 *get_bssid(struct mlme_priv *pmlmepriv)
+{
+	/* if sta_mode:pmlmepriv->cur_network.network.MacAddress=> bssid */
+	/* if adhoc_mode:pmlmepriv->cur_network.network.MacAddress=> ibss mac address */
+	return pmlmepriv->dev_cur_network.network.MacAddress;
+}
+
+__inline static sint check_fwstate(struct mlme_priv *pmlmepriv, sint state)
+{
+	if ((state == WIFI_NULL_STATE) &&
+		(pmlmepriv->fw_state == WIFI_NULL_STATE))
+		return _TRUE;
+
+	if (pmlmepriv->fw_state & state)
+		return _TRUE;
+
+	return _FALSE;
+}
+
+__inline static sint get_fwstate(struct mlme_priv *pmlmepriv)
+{
+	return pmlmepriv->fw_state;
+}
+
+/*
+ * No Limit on the calling context,
+ * therefore set it to be the critical section...
+ *
+ * ### NOTE:#### (!!!!)
+ * MUST TAKE CARE THAT BEFORE CALLING THIS FUNC, YOU SHOULD HAVE LOCKED pmlmepriv->lock
+ */
+extern void rtw_mi_update_iface_status(struct mlme_priv *pmlmepriv, sint state);
+
+static inline void set_fwstate(struct mlme_priv *pmlmepriv, sint state)
+{
+	pmlmepriv->fw_state |= state;
+	rtw_mi_update_iface_status(pmlmepriv, state);
+}
+static inline void init_fwstate(struct mlme_priv *pmlmepriv, sint state)
+{
+	pmlmepriv->fw_state = state;
+	rtw_mi_update_iface_status(pmlmepriv, state);
+}
+
+static inline void _clr_fwstate_(struct mlme_priv *pmlmepriv, sint state)
+{
+	pmlmepriv->fw_state &= ~state;
+	rtw_mi_update_iface_status(pmlmepriv, state);
+}
+
+/*
+ * No Limit on the calling context,
+ * therefore set it to be the critical section...
+ */
+static inline void clr_fwstate(struct mlme_priv *pmlmepriv, sint state)
+{
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	_clr_fwstate_(pmlmepriv, state);
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+}
+
+static inline void up_scanned_network(struct mlme_priv *pmlmepriv)
+{
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	pmlmepriv->num_of_scanned++;
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+}
+u8 rtw_is_adapter_up(_adapter *padapter);
+
+__inline static void down_scanned_network(struct mlme_priv *pmlmepriv)
+{
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	pmlmepriv->num_of_scanned--;
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+}
+
+__inline static void set_scanned_network_val(struct mlme_priv *pmlmepriv, sint val)
+{
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	pmlmepriv->num_of_scanned = val;
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+}
+
+__inline static void set_scanned_mld_network_val(struct mlme_priv *pmlmepriv, sint val)
+{
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	pmlmepriv->num_of_scanned_mld = val;
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+}
+
+extern u16 rtw_get_capability(WLAN_BSSID_EX *bss);
+void dump_scanned_queue(void *sel, _adapter *adapter);
+extern void rtw_disconnect_hdl_under_linked(_adapter *adapter, struct sta_info *psta, u8 free_assoc);
+extern void rtw_generate_random_ibss(u8 *pibss);
+struct wlan_network *_rtw_find_network(_queue *scanned_queue, const u8 *addr);
+struct wlan_network *rtw_find_network(_queue *scanned_queue, const u8 *addr);
+extern struct wlan_network *rtw_get_oldest_wlan_network(_queue *scanned_queue);
+struct wlan_network *_rtw_find_same_network(_queue *scanned_queue, struct wlan_network *network);
+struct wlan_network *rtw_find_same_network(_queue *scanned_queue, struct wlan_network *network);
+
+extern void rtw_free_assoc_resources(_adapter *adapter, u8 lock_scanned_queue);
+extern void rtw_indicate_disconnect(_adapter *adapter, u16 reason, u8 locally_generated);
+extern void rtw_indicate_connect(_adapter *adapter);
+void rtw_indicate_scan_done(_adapter *padapter, bool aborted);
+
+u32 rtw_join_abort_timeout(_adapter *adapter, u32 timeout_ms);
+
+int rtw_cached_pmkid(_adapter *adapter, u8 *bssid);
+int rtw_pmkid_sync_rsn(_adapter *adapter, u8 *ie, uint ie_len, int i_ent);
+
+extern int rtw_restruct_sec_ie(_adapter *adapter, u8 *out_ie);
+extern int rtw_restruct_wmm_ie(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+					u8 *in_ie, u8 *out_ie, uint in_len, uint initial_out_len);
+extern void rtw_init_registrypriv_dev_network(_adapter *adapter);
+
+extern void rtw_update_registrypriv_dev_network(_adapter *adapter);
+
+extern void rtw_get_encrypt_decrypt_from_registrypriv(_adapter *adapter);
+
+extern void rtw_join_timeout_handler(void *ctx);
+
+extern void rtw_iface_dynamic_check_handlder(struct _ADAPTER *a);
+
+void rtw_core_watchdog_sw_post_hdlr(void *drv_priv);
+void rtw_core_watchdog_sw_hdlr(void *drv_priv);
+void rtw_core_watchdog_hw_hdlr(void *drv_priv);
+
+#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+extern void rtw_dynamic_check_timer_handlder(void *ctx);
+extern void rtw_iface_dynamic_check_timer_handlder(_adapter *adapter);
+#endif
+
+
+void rtw_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv);
+
+#define MLME_BEACON_IE			0
+#define MLME_PROBE_REQ_IE		1
+#define MLME_PROBE_RESP_IE		2
+#define MLME_GO_PROBE_RESP_IE	3
+#define MLME_ASSOC_REQ_IE		4
+#define MLME_ASSOC_RESP_IE		5
+
+#if defined(CONFIG_WFD) && defined(CONFIG_IOCTL_CFG80211)
+int rtw_mlme_update_wfd_ie_data(struct mlme_priv *mlme, u8 type, u8 *ie, u32 ie_len);
+#endif
+
+
+/* extern struct wlan_network* _rtw_dequeue_network(_queue *queue); */
+
+extern struct wlan_network *_rtw_alloc_network(struct mlme_priv *pmlmepriv);
+
+
+extern void _rtw_free_network(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork, u8 isfreeall);
+extern void _rtw_free_network_nolock(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork);
+extern int rtw_init_link_mlme_priv(struct _ADAPTER_LINK *padapter_link);
+extern void rtw_free_mld_network_queue(_adapter *adapter, u8 isfreeall);
+#ifdef CONFIG_80211BE_EHT
+void rtw_link_mld_network(_adapter *padapter, struct wlan_mld_network *mld_network, struct wlan_network *network);
+struct wlan_mld_network *rtw_clone_to_join_mld_network(_adapter *padapter, struct wlan_mld_network *pmld_network);
+void rtw_free_cloned_mld_network(struct wlan_mld_network *mld_network);
+extern struct wlan_network *rtw_get_link_network_by_linkid(struct wlan_mld_network *mld_network, u8 link_id);
+void rtw_unlink_network_by_linkid(struct wlan_mld_network *mld_network, u8 link_id);
+#endif
+struct wlan_network *rtw_clone_network(_adapter *padapter, WLAN_BSSID_EX *pnetwork);
+struct wlan_mld_network *rtw_alloc_mld_network(_adapter *adapter, const u8 *mac_addr);
+extern struct _ADAPTER_LINK *rtw_get_adapter_link_by_linkid(_adapter *padapter, u8 link_id);
+struct _ADAPTER_LINK *rtw_get_adapter_link_by_hwband(_adapter *padapter, u8 band_idx);
+u8 rtw_adapter_link_get_id(struct _ADAPTER_LINK *alink);
+void rtw_update_link_ht_cap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *pie, uint ie_len, u8 channel);
+void rtw_update_join_priv(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			  WLAN_BSSID_EX *network, WLAN_BSSID_EX *psecnetwork);
+void rtw_clear_to_join_status(_adapter *padapter);
+void rtw_clear_is_accepted_status(_adapter *padapter);
+
+extern void _rtw_free_network_queue(_adapter *padapter, u8 isfreeall);
+
+extern sint rtw_if_up(_adapter *padapter);
+
+sint rtw_linked_check(_adapter *padapter);
+
+u8 *rtw_get_capability_from_ie(u8 *ie);
+u8 *rtw_get_timestampe_from_ie(u8 *ie);
+u8 *rtw_get_beacon_interval_from_ie(u8 *ie);
+
+
+void rtw_joinbss_reset(_adapter *padapter);
+
+#ifdef CONFIG_80211N_HT
+void rtw_ht_get_dft_setting(_adapter *padapter,
+				struct protocol_cap_t *dft_proto_cap,
+				struct role_link_cap_t *dft_cap);
+void	rtw_ht_use_default_setting(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+void rtw_build_wmm_ie_ht(_adapter *padapter, u8 *out_ie, uint *pout_len);
+unsigned int rtw_restructure_ht_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+						u8 *in_ie, u8 *out_ie, uint in_len, uint *pout_len,
+						u8 channel);
+void rtw_issue_addbareq_cmd(_adapter *padapter, struct xmit_frame *pxmitframe, u8 issue_when_busy);
+#endif
+
+void rtw_append_extended_cap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+					u8 *out_ie, uint *pout_len);
+
+int rtw_is_same_ibss(_adapter *adapter, struct wlan_network *pnetwork);
+int is_same_network(WLAN_BSSID_EX *src, WLAN_BSSID_EX *dst);
+int rtw_check_join_candidate(struct mlme_priv *mlme,
+	struct wlan_network **candidate, struct wlan_network *competitor);
+u8 rtw_do_join(_adapter *padapter);
+
+#ifdef CONFIG_LAYER2_ROAMING
+#define rtw_roam_flags(adapter) ((adapter)->mlmepriv.roam_flags)
+#define rtw_chk_roam_flags(adapter, flags) ((adapter)->mlmepriv.roam_flags & flags)
+#define rtw_clr_roam_flags(adapter, flags) \
+	do { \
+		((adapter)->mlmepriv.roam_flags &= ~flags); \
+	} while (0)
+
+#define rtw_set_roam_flags(adapter, flags) \
+	do { \
+		((adapter)->mlmepriv.roam_flags |= flags); \
+	} while (0)
+
+#define rtw_assign_roam_flags(adapter, flags) \
+	do { \
+		((adapter)->mlmepriv.roam_flags = flags); \
+	} while (0)
+
+void _rtw_roaming(_adapter *adapter, struct wlan_network *tgt_network, u8 reason);
+void rtw_roaming(_adapter *adapter, struct wlan_network *tgt_network, u8 reason);
+void rtw_set_to_roam(_adapter *adapter, u8 to_roam);
+u8 rtw_dec_to_roam(_adapter *adapter);
+u8 rtw_to_roam(_adapter *adapter);
+struct wlan_network *rtw_select_roaming_candidate(struct mlme_priv *pmlmepriv);
+int rtw_check_roaming_candidate(struct mlme_priv *mlme, struct wlan_network **candidate,
+	struct wlan_network *competitor);
+#else
+#define rtw_roam_flags(adapter) 0
+#define rtw_chk_roam_flags(adapter, flags) 0
+#define rtw_clr_roam_flags(adapter, flags) do {} while (0)
+#define rtw_set_roam_flags(adapter, flags) do {} while (0)
+#define rtw_assign_roam_flags(adapter, flags) do {} while (0)
+#define _rtw_roaming(adapter, tgt_network) do {} while (0)
+#define rtw_roaming(adapter, tgt_network) do {} while (0)
+#define rtw_set_to_roam(adapter, to_roam) do {} while (0)
+#define rtw_dec_to_roam(adapter) 0
+#define rtw_to_roam(adapter) 0
+#define rtw_select_roaming_candidate(mlme) NULL
+#endif /* CONFIG_LAYER2_ROAMING */
+
+RTW_FUNC_2G_5G_ONLY bool rtw_adjust_chbw(_adapter *adapter, u8 req_ch, u8 *req_bw, u8 *req_offset);
+bool rtw_adjust_bchbw(_adapter *adapter, enum band_type req_band, u8 req_ch, u8 *req_bw, u8 *req_offset);
+
+#ifdef CONFIG_RTW_MULTI_AP
+void rtw_map_config_monitor_act_non(_adapter *adapter);
+void rtw_map_config_monitor(_adapter *adapter, u8 self_act);
+void rtw_unassoc_sta_set_mode(_adapter *adapter, u8 stype, u8 mode);
+bool rtw_unassoc_sta_src_chk(_adapter *adapter, u8 stype);
+void dump_unassoc_sta(void *sel, _adapter *adapter);
+void rtw_del_unassoc_sta_queue(_adapter *adapter);
+void rtw_del_unassoc_sta(_adapter *adapter, u8 *addr);
+void rtw_rx_add_unassoc_sta(_adapter *adapter, u8 stype, u8 *addr, s8 recv_signal_power);
+void rtw_add_interested_unassoc_sta(_adapter *adapter, u8 *addr);
+void rtw_undo_interested_unassoc_sta(_adapter *adapter, u8 *addr);
+void rtw_undo_all_interested_unassoc_sta(_adapter *adapter);
+u8 rtw_search_unassoc_sta(_adapter *adapter, u8 *addr, struct unassoc_sta_info *ret_sta);
+#endif
+
+void rtw_sta_traffic_info(void *sel, _adapter *adapter);
+
+#define GET_ARP_HTYPE(_arp)	BE_BITS_TO_2BYTE(((u8 *)(_arp)) + 0, 0, 16)
+#define GET_ARP_PTYPE(_arp)	BE_BITS_TO_2BYTE(((u8 *)(_arp)) + 2, 0, 16)
+#define GET_ARP_HLEN(_arp)	BE_BITS_TO_1BYTE(((u8 *)(_arp)) + 4, 0, 8)
+#define GET_ARP_PLEN(_arp)	BE_BITS_TO_1BYTE(((u8 *)(_arp)) + 5, 0, 8)
+#define GET_ARP_OPER(_arp)	BE_BITS_TO_2BYTE(((u8 *)(_arp)) + 6, 0, 16)
+
+#define SET_ARP_HTYPE(_arp, _val)	SET_BITS_TO_BE_2BYTE(((u8 *)(_arp)) + 0, 0, 16, _val)
+#define SET_ARP_PTYPE(_arp, _val)	SET_BITS_TO_BE_2BYTE(((u8 *)(_arp)) + 2, 0, 16, _val)
+#define SET_ARP_HLEN(_arp, _val)	SET_BITS_TO_BE_1BYTE(((u8 *)(_arp)) + 4, 0, 8, _val)
+#define SET_ARP_PLEN(_arp, _val)	SET_BITS_TO_BE_1BYTE(((u8 *)(_arp)) + 5, 0, 8, _val)
+#define SET_ARP_OPER(_arp, _val)	SET_BITS_TO_BE_2BYTE(((u8 *)(_arp)) + 6, 0, 16, _val)
+
+#define ARP_SHA(_arp, _hlen, _plen)	(((u8 *)(_arp)) + 8)
+#define ARP_SPA(_arp, _hlen, _plen)	(((u8 *)(_arp)) + 8 + (_hlen))
+#define ARP_THA(_arp, _hlen, _plen)	(((u8 *)(_arp)) + 8 + (_hlen) + (_plen))
+#define ARP_TPA(_arp, _hlen, _plen)	(((u8 *)(_arp)) + 8 + 2 * (_hlen) + (_plen))
+
+#define ARP_SENDER_MAC_ADDR(_arp)	ARP_SHA(_arp, ETH_ALEN, RTW_IP_ADDR_LEN)
+#define ARP_SENDER_IP_ADDR(_arp)	ARP_SPA(_arp, ETH_ALEN, RTW_IP_ADDR_LEN)
+#define ARP_TARGET_MAC_ADDR(_arp)	ARP_THA(_arp, ETH_ALEN, RTW_IP_ADDR_LEN)
+#define ARP_TARGET_IP_ADDR(_arp)	ARP_TPA(_arp, ETH_ALEN, RTW_IP_ADDR_LEN)
+
+#define GET_ARP_SENDER_MAC_ADDR(_arp, _val)	_rtw_memcpy(_val, ARP_SENDER_MAC_ADDR(_arp), ETH_ALEN)
+#define GET_ARP_SENDER_IP_ADDR(_arp, _val)	_rtw_memcpy(_val, ARP_SENDER_IP_ADDR(_arp), RTW_IP_ADDR_LEN)
+#define GET_ARP_TARGET_MAC_ADDR(_arp, _val)	_rtw_memcpy(_val, ARP_TARGET_MAC_ADDR(_arp), ETH_ALEN)
+#define GET_ARP_TARGET_IP_ADDR(_arp, _val)	_rtw_memcpy(_val, ARP_TARGET_IP_ADDR(_arp), RTW_IP_ADDR_LEN)
+
+#define SET_ARP_SENDER_MAC_ADDR(_arp, _val)	_rtw_memcpy(ARP_SENDER_MAC_ADDR(_arp), _val, ETH_ALEN)
+#define SET_ARP_SENDER_IP_ADDR(_arp, _val)	_rtw_memcpy(ARP_SENDER_IP_ADDR(_arp), _val, RTW_IP_ADDR_LEN)
+#define SET_ARP_TARGET_MAC_ADDR(_arp, _val)	_rtw_memcpy(ARP_TARGET_MAC_ADDR(_arp), _val, ETH_ALEN)
+#define SET_ARP_TARGET_IP_ADDR(_arp, _val)	_rtw_memcpy(ARP_TARGET_IP_ADDR(_arp), _val, RTW_IP_ADDR_LEN)
+
+void dump_arp_pkt(void *sel, u8 *da, u8 *sa, u8 *arp, bool tx);
+
+#define IPV4_SRC(_iphdr)			(((u8 *)(_iphdr)) + 12)
+#define IPV4_DST(_iphdr)			(((u8 *)(_iphdr)) + 16)
+#define GET_IPV4_IHL(_iphdr)		BE_BITS_TO_1BYTE(((u8 *)(_iphdr)) + 0, 0, 4)
+#define GET_IPV4_PROTOCOL(_iphdr)	BE_BITS_TO_1BYTE(((u8 *)(_iphdr)) + 9, 0, 8)
+#define GET_IPV4_SRC(_iphdr)		BE_BITS_TO_4BYTE(((u8 *)(_iphdr)) + 12, 0, 32)
+#define GET_IPV4_DST(_iphdr)		BE_BITS_TO_4BYTE(((u8 *)(_iphdr)) + 16, 0, 32)
+
+#define GET_UDP_SRC(_udphdr)			BE_BITS_TO_2BYTE(((u8 *)(_udphdr)) + 0, 0, 16)
+#define GET_UDP_DST(_udphdr)			BE_BITS_TO_2BYTE(((u8 *)(_udphdr)) + 2, 0, 16)
+#define GET_UDP_SIG1(_udphdr)			BE_BITS_TO_1BYTE(((u8 *)(_udphdr)) + 8, 0, 8)
+#define GET_UDP_SIG2(_udphdr)			BE_BITS_TO_1BYTE(((u8 *)(_udphdr)) + 23, 0, 8)
+
+#define TCP_SRC(_tcphdr)				(((u8 *)(_tcphdr)) + 0)
+#define TCP_DST(_tcphdr)				(((u8 *)(_tcphdr)) + 2)
+#define GET_TCP_SRC(_tcphdr)			BE_BITS_TO_2BYTE(((u8 *)(_tcphdr)) + 0, 0, 16)
+#define GET_TCP_DST(_tcphdr)			BE_BITS_TO_2BYTE(((u8 *)(_tcphdr)) + 2, 0, 16)
+#define GET_TCP_SEQ(_tcphdr)			BE_BITS_TO_4BYTE(((u8 *)(_tcphdr)) + 4, 0, 32)
+#define GET_TCP_ACK_SEQ(_tcphdr)		BE_BITS_TO_4BYTE(((u8 *)(_tcphdr)) + 8, 0, 32)
+#define GET_TCP_DOFF(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 12, 4, 4)
+#define GET_TCP_FIN(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 0, 1)
+#define GET_TCP_SYN(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 1, 1)
+#define GET_TCP_RST(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 2, 1)
+#define GET_TCP_PSH(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 3, 1)
+#define GET_TCP_ACK(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 4, 1)
+#define GET_TCP_URG(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 5, 1)
+#define GET_TCP_ECE(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 6, 1)
+#define GET_TCP_CWR(_tcphdr)			BE_BITS_TO_1BYTE(((u8 *)(_tcphdr)) + 13, 7, 1)
+
+enum rtw_phl_status rtw_connect_cmd(struct _ADAPTER *a,
+				    struct _WLAN_BSSID_EX *network);
+void rtw_connect_abort(struct _ADAPTER *a);
+int rtw_connect_abort_wait(struct _ADAPTER *a);
+void rtw_connect_req_free(struct _ADAPTER *a);
+void rtw_connect_req_init(struct _ADAPTER *a);
+enum rtw_phl_status rtw_connect_disconnect_prepare(struct _ADAPTER *a);
+
+enum rtw_phl_status rtw_disconnect_cmd(struct _ADAPTER *a,
+				       struct cmd_obj *pcmd);
+bool rtw_disconnect_wait_complete(struct _ADAPTER *a, u32 timeout);
+int rtw_disconnect_abort_wait(struct _ADAPTER *a);
+void rtw_disconnect_req_free(struct _ADAPTER *a);
+void rtw_disconnect_req_init(struct _ADAPTER *a);
+
+#endif /* __RTW_MLME_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mlme_ext.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mlme_ext.h
new file mode 100644
index 000000000000..ac50f4cc97f9
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mlme_ext.h
@@ -0,0 +1,1115 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_MLME_EXT_H_
+#define __RTW_MLME_EXT_H_
+
+/*
+ * SURVEY_TO: dwell time (ms) for passive channel scan or other scan functions
+ * in addition to normal scan.
+ * ACTIVE_CH_SURVEY_TO: dwell time (ms) for active scan channel in nomal scan.
+ */
+#ifndef SURVEY_TO
+#define SURVEY_TO		(100)
+#endif /*SURVEY_TO*/
+#define ACTIVE_CH_SURVEY_TO	(50)
+
+#define REAUTH_TO		(300) /* (50) */
+#define REASSOC_TO		(300) /* (50) */
+#define SAE_AUTH_TIMEOUT	(2400)
+/* #define DISCONNECT_TO	(3000) */
+#define ADDBA_TO			(2000)
+
+#define LINKED_TO (1) /* unit:2 sec, 1x2 = 2 sec */
+
+#define REAUTH_LIMIT	(4)
+#define REASSOC_LIMIT	(4)
+#define READDBA_LIMIT	(2)
+
+#ifdef CONFIG_GSPI_HCI
+	#define ROAMING_LIMIT	5
+#else
+	#define ROAMING_LIMIT	8
+#endif
+
+/*net_type, pmlmeinfo->state, pstat->state*/
+#define	_HW_STATE_NOLINK_	0x00
+#define	_HW_STATE_ADHOC_	0x01
+#define	_HW_STATE_STATION_	0x02
+#define	_HW_STATE_AP_		0x03
+#define	_HW_STATE_MONITOR_ 	0x04
+
+#define	WIFI_FW_NULL_STATE		_HW_STATE_NOLINK_
+#define	WIFI_FW_STATION_STATE		_HW_STATE_STATION_
+#define	WIFI_FW_AP_STATE		_HW_STATE_AP_
+#define	WIFI_FW_ADHOC_STATE		_HW_STATE_ADHOC_
+
+#define 	WIFI_FW_PRE_LINK		0x00000800
+#define	WIFI_FW_AUTH_NULL		0x00000100
+#define	WIFI_FW_AUTH_STATE		0x00000200
+#define	WIFI_FW_AUTH_SUCCESS		0x00000400
+
+#define	WIFI_FW_ASSOC_STATE		0x00002000
+#define	WIFI_FW_ASSOC_SUCCESS		0x00004000
+
+#define	WIFI_FW_LINKING_STATE		(WIFI_FW_AUTH_NULL | WIFI_FW_AUTH_STATE | WIFI_FW_AUTH_SUCCESS | WIFI_FW_ASSOC_STATE)
+
+
+#define		_1M_RATE_	0
+#define		_2M_RATE_	1
+#define		_5M_RATE_	2
+#define		_11M_RATE_	3
+#define		_6M_RATE_	4
+#define		_9M_RATE_	5
+#define		_12M_RATE_	6
+#define		_18M_RATE_	7
+#define		_24M_RATE_	8
+#define		_36M_RATE_	9
+#define		_48M_RATE_	10
+#define		_54M_RATE_	11
+
+/********************************************************
+MCS rate definitions
+*********************************************************/
+#define MCS_RATE_1R	(0x000000ff)
+#define MCS_RATE_2R	(0x0000ffff)
+#define MCS_RATE_3R	(0x00ffffff)
+#define MCS_RATE_4R	(0xffffffff)
+#define MCS_RATE_2R_13TO15_OFF	(0x00001fff)
+
+
+extern unsigned char RTW_WPA_OUI[];
+extern unsigned char WMM_OUI[];
+extern unsigned char WPS_OUI[];
+extern unsigned char WFD_OUI[];
+extern unsigned char P2P_OUI[];
+extern unsigned char MULTI_AP_OUI[];
+
+extern unsigned char WMM_INFO_OUI[];
+extern unsigned char WMM_PARA_OUI[];
+
+#if defined(PRIVATE_R) && defined(CONFIG_P2P)
+extern int rtw_go_hidden_ssid_mode;
+enum {
+	MIRACAST = 0,
+	PAIRING_MODE = 1,
+	CAPTIVE_PORTAL = 2,
+};
+#define ALL_HIDE_SSID 0xff
+#define MIRACAST_UNHIDE_SSID 0xfe
+#endif
+
+typedef enum _HT_IOT_PEER {
+	HT_IOT_PEER_UNKNOWN			= 0,
+	HT_IOT_PEER_REALTEK			= 1,
+	HT_IOT_PEER_REALTEK_92SE		= 2,
+	HT_IOT_PEER_BROADCOM		= 3,
+	HT_IOT_PEER_RALINK			= 4,
+	HT_IOT_PEER_ATHEROS			= 5,
+	HT_IOT_PEER_CISCO				= 6,
+	HT_IOT_PEER_MERU				= 7,
+	HT_IOT_PEER_MARVELL			= 8,
+	HT_IOT_PEER_REALTEK_SOFTAP 	= 9,/* peer is RealTek SOFT_AP, by Bohn, 2009.12.17 */
+	HT_IOT_PEER_SELF_SOFTAP 		= 10, /* Self is SoftAP */
+	HT_IOT_PEER_AIRGO				= 11,
+	HT_IOT_PEER_INTEL				= 12,
+	HT_IOT_PEER_RTK_APCLIENT		= 13,
+	HT_IOT_PEER_REALTEK_81XX		= 14,
+	HT_IOT_PEER_REALTEK_WOW		= 15,
+	HT_IOT_PEER_REALTEK_JAGUAR_CBVAP = 16,
+	HT_IOT_PEER_REALTEK_JAGUAR_CCVAP = 17,
+	HT_IOT_PEER_MAX				= 18
+} HT_IOT_PEER_E, *PHTIOT_PEER_E;
+
+
+typedef enum _RT_HT_INF0_CAP {
+	RT_HT_CAP_USE_TURBO_AGGR = 0x01,
+	RT_HT_CAP_USE_LONG_PREAMBLE = 0x02,
+	RT_HT_CAP_USE_AMPDU = 0x04,
+	RT_HT_CAP_USE_WOW = 0x8,
+	RT_HT_CAP_USE_SOFTAP = 0x10,
+	RT_HT_CAP_USE_92SE = 0x20,
+	RT_HT_CAP_USE_88C_92C = 0x40,
+	RT_HT_CAP_USE_AP_CLIENT_MODE = 0x80,	/* AP team request to reserve this bit, by Emily */
+} RT_HT_INF0_CAPBILITY, *PRT_HT_INF0_CAPBILITY;
+
+typedef enum _RT_HT_INF1_CAP {
+	RT_HT_CAP_USE_VIDEO_CLIENT = 0x01,
+	RT_HT_CAP_USE_JAGUAR_CBV = 0x02,
+	RT_HT_CAP_USE_JAGUAR_CCV = 0x04,
+} RT_HT_INF1_CAPBILITY, *PRT_HT_INF1_CAPBILITY;
+
+struct mlme_handler {
+	unsigned int   num;
+	char *str;
+	unsigned int (*func)(_adapter *padapter, union recv_frame *precv_frame);
+};
+
+struct action_handler {
+	unsigned int   num;
+	char *str;
+	unsigned int (*func)(_adapter *padapter, union recv_frame *precv_frame);
+};
+
+#ifdef CONFIG_TDLS
+enum TDLS_option {
+	TDLS_ESTABLISHED = 1,
+	TDLS_ISSUE_PTI,
+	TDLS_CH_SW_RESP,
+	TDLS_CH_SW_PREPARE,
+	TDLS_CH_SW_START,
+	TDLS_CH_SW_TO_OFF_CHNL,
+	TDLS_CH_SW_TO_BASE_CHNL_UNSOLICITED,
+	TDLS_CH_SW_TO_BASE_CHNL,
+	TDLS_CH_SW_END_TO_BASE_CHNL,
+	TDLS_CH_SW_END,
+	TDLS_RS_RCR,
+	TDLS_TEARDOWN_STA,
+	TDLS_TEARDOWN_STA_TOOFAR,
+	TDLS_TEARDOWN_STA_NO_WAIT,
+	TDLS_TEARDOWN_STA_LOCALLY,
+	TDLS_TEARDOWN_STA_LOCALLY_POST,
+	maxTDLS,
+};
+
+#endif /* CONFIG_TDLS */
+
+typedef enum {
+	DISCONNECTION_NOT_YET_OCCUR,
+	DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND,
+	DISCONNECTION_BY_SYSTEM_DUE_TO_NET_DEVICE_DOWN,
+	DISCONNECTION_BY_SYSTEM_DUE_TO_SYSTEM_IN_SUSPEND,
+	DISCONNECTION_BY_DRIVER_DUE_TO_CONNECTION_EXIST,
+	DISCONNECTION_BY_DRIVER_DUE_TO_EACH_IFACE_CHBW_NOT_SYNC,
+	DISCONNECTION_BY_DRIVER_DUE_TO_DFS_DETECTION,
+	DISCONNECTION_BY_DRIVER_DUE_TO_IOCTL_DBG_PORT,
+	DISCONNECTION_BY_DRIVER_DUE_TO_AP_BEACON_CHANGED,
+	DISCONNECTION_BY_DRIVER_DUE_TO_KEEPALIVE_TIMEOUT,
+	DISCONNECTION_BY_DRIVER_DUE_TO_LAYER2_ROAMING_TERMINATE,
+	DISCONNECTION_BY_DRIVER_DUE_TO_JOINBSS_TIMEOUT,
+	DISCONNECTION_BY_FW_DUE_TO_FW_DECISION_IN_WOW_RESUME,
+	DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DISASSOC_IN_WOW_RESUME,
+	DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DEAUTH_IN_WOW_RESUME,
+	DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DEAUTH,
+	DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DISASSOC,
+	DISCONNECTION_BY_DRIVER_DUE_TO_RECEIVE_CSA_NON_DFS,
+	DISCONNECTION_BY_DRIVER_DUE_TO_RECEIVE_CSA_DFS,
+	DISCONNECTION_BY_DRIVER_DUE_TO_RECEIVE_INVALID_CSA,
+	DISCONNECTION_BY_DRIVER_DUE_TO_JOIN_WRONG_CHANNEL,
+	DISCONNECTION_BY_DRIVER_DUE_TO_FT,
+	DISCONNECTION_BY_DRIVER_DUE_TO_ROAMING,
+	DISCONNECTION_BY_DRIVER_DUE_TO_SA_QUERY_TIMEOUT,
+} Disconnect_type;
+
+struct disconnect_data {
+	struct rtw_chan_def *buddy_chdef;
+	Disconnect_type disc_code;
+};
+
+#define SSID_CHANGED BIT0
+#define SSID_LENGTH_CHANGED BIT1
+#define BEACON_CHANNEL_CHANGED BIT2
+#define ENCRYPT_PROTOCOL_CHANGED BIT3
+#define PAIRWISE_CIPHER_CHANGED BIT4
+#define GROUP_CIPHER_CHANGED BIT5
+#define IS_8021X_CHANGED BIT6
+
+/*
+ * Usage:
+ * When one iface acted as AP mode and the other iface is STA mode and scanning,
+ * it should switch back to AP's operating channel periodically.
+ * Parameters info:
+ * When the driver scanned RTW_SCAN_NUM_OF_CH channels, it would switch back to AP's operating channel for
+ * RTW_BACK_OP_CH_MS milliseconds.
+ * Example:
+ * For chip supports 2.4G + 5GHz and AP mode is operating in channel 1,
+ * RTW_SCAN_NUM_OF_CH is 8, RTW_BACK_OP_CH_MS is 300
+ * When it's STA mode gets set_scan command,
+ * it would
+ * 1. Doing the scan on channel 1.2.3.4.5.6.7.8
+ * 2. Back to channel 1 for 300 milliseconds
+ * 3. Go through doing site survey on channel 9.10.11.36.40.44.48.52
+ * 4. Back to channel 1 for 300 milliseconds
+ * 5. ... and so on, till survey done.
+ */
+#if defined(CONFIG_ATMEL_RC_PATCH)
+	#define RTW_SCAN_NUM_OF_CH 2
+	#define RTW_BACK_OP_CH_MS 200
+#else
+	#ifndef RTW_SCAN_NUM_OF_CH
+	#define RTW_SCAN_NUM_OF_CH 3
+	#endif /*RTW_SCAN_NUM_OF_CH*/
+
+	#ifndef RTW_BACK_OP_CH_MS
+	#define RTW_BACK_OP_CH_MS 400
+	#endif /*RTW_BACK_OP_CH_MS*/
+#endif
+
+#define RTW_IP_ADDR_LEN 4
+#define RTW_IPv6_ADDR_LEN 16
+
+struct link_mlme_ext_info {
+	u16	bcn_interval;
+	u16	capability;
+	u8	slotTime;
+	u8	preamble_mode;
+	u8	WMM_enable;
+	u8	ERP_enable;
+	u8	ERP_IE;
+	u8	HT_enable;
+	u8	HT_caps_enable;
+	u8	HT_info_enable;
+	u8	HT_protection;
+	u8	SM_PS;
+	u8	bwmode_updated;
+	u8	VHT_enable;
+
+	u8	HE_enable;
+
+	struct WMM_para_element	WMM_param;
+	struct HT_caps_element	HT_caps;
+	struct HT_info_element		HT_info;
+	WLAN_BSSID_EX			network;/* join network or bss_network, if in ap mode, it is the same to cur_network.network */
+#ifdef PRIVATE_R
+	/*infra mode, store supported rates from AssocRsp*/
+	NDIS_802_11_RATES_EX	SupportedRates_infra_ap;
+	u8 ht_vht_received;/*ht_vht_received used to show debug msg BIT(0):HT BIT(1):VHT */
+#endif /* PRIVATE_R */
+};
+
+struct mlme_ext_info {
+	u32	state;
+	u32	reauth_count;
+	u32	reassoc_count;
+	u32	link_count;
+	u32	auth_seq;
+	u32	auth_algo;	/* 802.11 auth, could be open, shared, auto */
+	u16 auth_status;
+	u32	authModeToggle;
+	u32	enc_algo;/* encrypt algorithm; */
+	u32	key_index;	/* this is only valid for legendary wep, 0~3 for key id. */
+	u32	iv;
+	u8	chg_txt[128];
+	u8	assoc_AP_vendor;
+	u8	agg_enable_bitmap;
+	u8	ADDBA_retry_count;
+	u8	candidate_tid_bitmap;
+	u8	dialogToken;
+	/* Accept ADDBA Request */
+	BOOLEAN bAcceptAddbaReq;
+	u8	hidden_ssid_mode;
+
+	u8 ip_addr[RTW_IP_ADDR_LEN];
+	u8 ip6_addr[RTW_IPv6_ADDR_LEN];
+
+	struct ADDBA_request		ADDBA_req;
+	WLAN_BSSID_EX			dev_network;/* join network or bss_network, if in ap mode, it is the same to cur_network.network */
+
+	Disconnect_type disconnect_code;
+	u32 illegal_beacon_code;
+	u32 wifi_reason_code;
+	u32 disconnect_occurred_time;
+};
+
+void init_channel_list(_adapter *padapter);
+
+int rtw_rfctl_init(struct dvobj_priv *dvobj);
+void rtw_rfctl_deinit(struct dvobj_priv *dvobj);
+void rtw_rfctl_update_op_mode(struct rf_ctl_t *rfctl, u8 ifbmp_mod, u8 if_op, u8 ifbmp_excl);
+
+bool rtw_mlme_band_check(_adapter *adapter, const u32 ch);
+
+
+enum {
+	BAND_24G = BIT0,
+	BAND_5G = BIT1,
+};
+void RTW_SET_SCAN_BAND_SKIP(_adapter *padapter, int skip_band);
+void RTW_CLR_SCAN_BAND_SKIP(_adapter *padapter, int skip_band);
+int RTW_GET_SCAN_BAND_SKIP(_adapter *padapter);
+
+bool rtw_mlme_ignore_chan(_adapter *adapter, const u32 ch);
+
+/* P2P_MAX_REG_CLASSES - Maximum number of regulatory classes */
+#define P2P_MAX_REG_CLASSES 10
+
+/* P2P_MAX_REG_CLASS_CHANNELS - Maximum number of channels per regulatory class */
+#define P2P_MAX_REG_CLASS_CHANNELS 20
+
+/* struct p2p_channels - List of supported channels */
+struct p2p_channels {
+	/* struct p2p_reg_class - Supported regulatory class */
+	struct p2p_reg_class {
+		/* reg_class - Regulatory class (IEEE 802.11-2007, Annex J) */
+		u8 reg_class;
+
+		/* channel - Supported channels */
+		u8 channel[P2P_MAX_REG_CLASS_CHANNELS];
+
+		/* channels - Number of channel entries in use */
+		size_t channels;
+	} reg_class[P2P_MAX_REG_CLASSES];
+
+	/* reg_classes - Number of reg_class entries in use */
+	size_t reg_classes;
+};
+
+struct p2p_oper_class_map {
+	enum hw_mode {IEEE80211G, IEEE80211A} mode;
+	u8 op_class;
+	u8 min_chan;
+	u8 max_chan;
+	u8 inc;
+	enum { BW20, BW40PLUS, BW40MINUS } bw;
+};
+
+struct channel_info_triplet {
+	u8 first_ch;
+	u8 num_of_ch;
+	s8 power_level;
+};
+
+struct link_mlme_ext_priv {
+	struct rtw_chan_def chandef;
+	unsigned char	cur_wireless_mode;	/* NETWORK_TYPE */
+	unsigned char	basicrate[NumRates];
+	unsigned char	datarate[NumRates];
+#ifdef CONFIG_80211N_HT
+	unsigned char default_supported_mcs_set[16];
+#endif
+	struct link_mlme_ext_info	mlmext_info;/* for sta/adhoc mode, including current scanning/connecting/connected related info.
+					   * for ap mode, network includes ap's cap_info */
+	u8	tx_rate; /* TXRATE when USERATE is set. */
+	u64 TSFValue;
+	u32 bcn_cnt;
+	u32 last_bcn_cnt;
+	u8 cur_bcn_cnt;/*2s*/
+	u8 dtim;/*DTIM Period*/
+#ifdef DBG_RX_BCN
+	u8 tim[4];
+#endif
+#ifdef CONFIG_BCN_RECV_TIME
+	u16 bcn_rx_time;
+#endif
+#ifdef CONFIG_AP_MODE
+	unsigned char bstart_bss;
+#endif
+	/* Below record country element*/
+	char country[3];
+	struct channel_info_triplet ch_info[MAX_CHANNEL_NUM];
+	/* Below record channel rpt element */
+	u8 op_class;
+	u8 channel_list[MAX_CHANNEL_NUM];
+};
+
+struct mlme_ext_priv {
+	_adapter	*padapter;
+	ATOMIC_T		event_seq;
+	u16	mgnt_seq;
+#ifdef CONFIG_IEEE80211W
+	u16	sa_query_seq;
+#endif
+	/* struct fw_priv 	fwpriv; */
+
+	unsigned char	dev_cur_wireless_mode;	/* NETWORK_TYPE */
+
+	struct ss_res		sitesurvey_res;
+	struct mlme_ext_info	mlmext_info;/* for sta/adhoc mode, including current scanning/connecting/connected related info.
+                                                      * for ap mode, network includes ap's cap_info */
+	/*_timer		survey_timer;*/
+	_timer		link_timer;
+
+#ifdef CONFIG_RTW_80211R
+	_timer		ft_link_timer;
+	_timer		ft_roam_timer;
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	_timer		tbtx_xmit_timer;
+	_timer		tbtx_token_dispatch_timer;
+#endif
+#ifdef PRIVATE_R
+	_timer		find_remote_timer;
+#ifdef CONFIG_P2P
+	_timer		hide_ssid_timer;
+#endif
+#endif
+
+	systime last_scan_time;
+	u8 scan_abort;
+	bool scan_abort_to;
+	u8 join_abort;
+
+	u32	retry; /* retry for issue probereq */
+
+#ifdef CONFIG_AP_MODE
+	unsigned char bstart_bss;
+#endif
+
+	/* recv_decache check for Action_public frame */
+	u8 action_public_dialog_token;
+	u16	 action_public_rxseq;
+
+	/* #ifdef CONFIG_ACTIVE_KEEP_ALIVE_CHECK */
+	u8 active_keep_alive_check;
+	/* #endif */
+#ifdef DBG_FIXED_CHAN
+	u8 fixed_chan;
+#endif
+
+#ifdef CONFIG_SUPPORT_STATIC_SMPS
+	u8 ssmps_en;
+	u16 ssmps_tx_tp_th;/*Mbps*/
+	u16 ssmps_rx_tp_th;/*Mbps*/
+	#ifdef DBG_STATIC_SMPS
+	u8 ssmps_test;
+	u8 ssmps_test_en;
+	#endif
+#endif
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	u8 txss_ctrl_en;
+	u16 txss_tp_th;/*Mbps*/
+	u8 txss_tp_chk_cnt;/*unit 2s*/
+	bool txss_1ss;
+#endif
+	u8 txss_bk;
+#ifdef CONFIG_ECSA_PHL
+	_timer csa_wait_bcn_timer;
+#endif
+};
+
+struct support_rate_handler {
+	u8 rate;
+	bool basic;
+	bool existence;
+};
+
+static inline u8 check_mlmeinfo_state(struct mlme_ext_priv *plmeext, sint state)
+{
+	if ((plmeext->mlmext_info.state & 0x03) == state)
+		return _TRUE;
+
+	return _FALSE;
+}
+
+#define mlmeext_msr(mlmeext) ((mlmeext)->mlmext_info.state & 0x03)
+
+int init_mlme_ext_priv(_adapter *padapter);
+void free_mlme_ext_priv(struct mlme_ext_priv *pmlmeext);
+int rtw_init_link_mlme_ext_priv(struct _ADAPTER_LINK *padapter_link);
+extern struct xmit_frame *alloc_mgtxmitframe(struct xmit_priv *pxmitpriv);
+struct xmit_frame *alloc_mgtxmitframe_once(struct xmit_priv *pxmitpriv);
+
+void get_rate_set(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *pbssrate, int *bssrate_len);
+void set_mcs_rate_by_mask(u8 *mcs_set, u32 mask);
+void UpdateBrateTbl(_adapter *padapter, u8 *mBratesOS);
+void UpdateBrateTblForSoftAP(u8 *bssrateset, u32 bssratelen);
+void change_band_update_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				WLAN_BSSID_EX *pnetwork, struct rtw_chan_def *chandef);
+
+void rtw_set_external_auth_status(_adapter *padapter, const void *data, int len);
+
+void rtw_get_oper_chdef(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+				struct rtw_chan_def *chandef);
+u8 rtw_get_oper_band(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+u8 rtw_get_oper_ch(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+u8 rtw_get_oper_bw(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+u8 rtw_get_oper_choffset(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+int rtw_get_oper_chdef_by_hwband(struct dvobj_priv *dvobj, enum phl_band_idx band_idx
+	, struct rtw_chan_def *chandef);
+int rtw_get_oper_bchbw_by_hwband(struct dvobj_priv *dvobj, enum phl_band_idx band_idx
+	, enum band_type *band, u8 *ch, u8 *bw, u8 *offset);
+
+systime rtw_get_on_oper_ch_time(_adapter *adapter);
+systime rtw_get_on_cur_ch_time(_adapter *adapter);
+
+RTW_FUNC_2G_5G_ONLY void set_channel_bwmode(_adapter *padapter,
+		struct _ADAPTER_LINK *padapter_link,
+		unsigned char channel,
+		unsigned char channel_offset,
+		unsigned short bwmode,
+		enum rfk_tri_type rt_type);
+
+void set_bch_bwmode(_adapter *padapter,
+		struct _ADAPTER_LINK *padapter_link,
+		enum band_type band,
+		unsigned char channel,
+		unsigned char channel_offset,
+		unsigned short bwmode,
+		enum rfk_tri_type rt_type);
+
+unsigned int decide_wait_for_beacon_timeout(unsigned int bcn_interval);
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+BOOLEAN Is6GHzPreferScanChannel(u8 channel);
+#endif
+BOOLEAN IsLegal5GChannel(_adapter *adapter, u8 channel);
+
+u8 collect_bss_info(_adapter *padapter, union recv_frame *precv_frame, WLAN_BSSID_EX *bssid);
+void rtw_update_adapter_network(WLAN_BSSID_EX *dst, WLAN_BSSID_EX *src, _adapter *padapter, bool update_ie);
+void rtw_update_network(WLAN_BSSID_EX *dst, WLAN_BSSID_EX *src, _adapter *padapter, bool update_ie);
+
+u8 *get_my_bssid(WLAN_BSSID_EX *pnetwork);
+u16 get_beacon_interval(WLAN_BSSID_EX *bss);
+
+int is_client_associated_to_ap(_adapter *padapter);
+int is_client_associated_to_ibss(_adapter *padapter);
+int is_IBSS_empty(_adapter *padapter);
+
+unsigned char check_assoc_AP(u8 *pframe, uint len);
+void get_assoc_AP_Vendor(char *vendor, u8 assoc_AP_vendor);
+#ifdef CONFIG_RTS_FULL_BW
+void rtw_parse_sta_vendor_ie_8812(_adapter *adapter, struct sta_info *sta, u8 *tlv_ies, u16 tlv_ies_len);
+#endif/*CONFIG_RTS_FULL_BW*/
+#ifdef CONFIG_80211AC_VHT
+void get_vht_bf_cap(u8 *pframe, uint len, struct vht_bf_cap *bf_cap);
+#endif
+
+int WMM_param_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs	pIE);
+#ifdef CONFIG_WFD
+void rtw_process_wfd_ie(_adapter *adapter, u8 *ie, u8 ie_len, const char *tag);
+void rtw_process_wfd_ies(_adapter *adapter, u8 *ies, u8 ies_len, const char *tag);
+#endif
+void WMMOnAssocRsp(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+u8 rtw_is_tbtx_capabilty(u8 *p, u8 len);
+#endif
+
+void HT_caps_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE);
+#ifdef PRIVATE_R
+void HT_caps_handler_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE);
+#endif
+void HT_info_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE);
+void HTOnAssocRsp(_adapter *padapter);
+
+#ifdef PRIVATE_R
+void Supported_rate_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				PNDIS_802_11_VARIABLE_IEs pIE);
+void Extended_Supported_rate_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				PNDIS_802_11_VARIABLE_IEs pIE);
+#endif
+
+void ERP_IE_handler(_adapter *padapter,struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE);
+void VCS_update(_adapter *padapter, struct sta_info *psta);
+void	update_ldpc_stbc_cap(struct sta_info *psta);
+
+bool rtw_validate_value(u16 EID, u8 *p, u16 len);
+bool is_hidden_ssid(char *ssid, int len);
+bool hidden_ssid_ap(WLAN_BSSID_EX *snetwork);
+void rtw_absorb_ssid_ifneed(_adapter *padapter, WLAN_BSSID_EX *bssid, u8 *pframe);
+
+int rtw_get_bcn_keys(_adapter *adapter, struct _ADAPTER_LINK *adapter_link
+	, u8 *whdr, u32 flen, struct beacon_keys *bcn_keys);
+int rtw_get_bcn_keys_from_bss(WLAN_BSSID_EX *bss, struct beacon_keys *bcn_keys);
+int rtw_update_bcn_keys_of_network(struct wlan_network *network);
+
+int validate_beacon_len(u8 *pframe, uint len);
+void rtw_dump_bcn_keys(void *sel, struct beacon_keys *recv_beacon);
+bool rtw_bcn_key_compare(struct beacon_keys *cur, struct beacon_keys *recv);
+int rtw_check_bcn_info(_adapter *adapter, struct _ADAPTER_LINK *adapter_link,
+		u8 *pframe, u32 packet_len);
+void update_beacon_info(_adapter *padapter, u8 *pframe, uint len, struct sta_info *psta);
+#ifdef CONFIG_ECSA_PHL
+void process_csa_ie(_adapter *padapter, u8 *ies, uint ies_len);
+#endif
+void update_capinfo(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, u16 updateCap);
+void update_wireless_mode(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+void update_tx_basic_rate(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 modulation);
+#if 0
+void update_sta_basic_rate(struct sta_info *psta, u8 wireless_mode);
+#endif
+int rtw_ies_get_supported_rate(u8 *ies, uint ies_len, u8 *rate_set, u8 *rate_num);
+int rtw_elems_get_supported_rate(struct rtw_ieee802_11_elems *elems, u8 *rate_set, u8 *rate_num);
+
+/* for sta/adhoc mode */
+void update_sta_info(_adapter *padapter, struct sta_info *psta);
+void update_sta_ra_info(_adapter *padapter, struct sta_info *psta);
+
+s8 rtw_get_sta_rx_nss(_adapter *adapter, struct sta_info *psta);
+s8 rtw_get_sta_tx_nss(_adapter *adapter, struct sta_info *psta);
+
+
+unsigned int update_basic_rate(unsigned char *ptn, unsigned int ptn_sz);
+unsigned int update_supported_rate(unsigned char *ptn, unsigned int ptn_sz);
+
+unsigned int receive_disconnect(_adapter *padapter, unsigned char *MacAddr, unsigned short reason, u8 locally_generated);
+
+unsigned char get_highest_rate_idx(u64 mask);
+
+#if 0
+unsigned char get_lowest_rate_idx_ex(u64 mask, int start_bit);
+#define get_lowest_rate_idx(mask) get_lowest_rate_idx_ex(mask, 0)
+u8 get_highest_bw_cap(u8 bwmode);
+#endif
+
+int support_short_GI(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			struct HT_caps_element *pHT_caps, u8 bwmode);
+unsigned int is_ap_in_tkip(struct _ADAPTER_LINK *padapter_link);
+unsigned int is_ap_in_wep(struct _ADAPTER_LINK *padapter_link);
+unsigned int should_forbid_n_rate(_adapter *padapter);
+
+enum eap_type parsing_eapol_packet(_adapter *padapter, u8 *key_payload, struct sta_info *psta, u8 trx_type);
+
+bool rtw_bmp_is_set(const u8 *bmp, u8 bmp_len, u8 id);
+void rtw_bmp_set(u8 *bmp, u8 bmp_len, u8 id);
+void rtw_bmp_clear(u8 *bmp, u8 bmp_len, u8 id);
+bool rtw_bmp_not_empty(const u8 *bmp, u8 bmp_len);
+bool rtw_bmp_not_empty_exclude_bit0(const u8 *bmp, u8 bmp_len);
+
+#ifdef CONFIG_AP_MODE
+bool rtw_tim_map_is_set(_adapter *padapter, const u8 *map, u8 id);
+void rtw_tim_map_set(_adapter *padapter, u8 *map, u8 id);
+void rtw_tim_map_clear(_adapter *padapter, u8 *map, u8 id);
+bool rtw_tim_map_anyone_be_set(_adapter *padapter, const u8 *map);
+bool rtw_tim_map_anyone_be_set_exclude_aid0(_adapter *padapter, const u8 *map);
+#endif /* CONFIG_AP_MODE */
+
+u32 report_join_res(_adapter *padapter, int aid_res, u16 status);
+void report_survey_event(_adapter *padapter, union recv_frame *precv_frame);
+void report_surveydone_event(_adapter *padapter, u8 reason, bool complete, bool acs, u8 flags);
+u32 report_del_sta_event(_adapter *padapter, unsigned char *MacAddr, unsigned short reason, bool enqueue, u8 locally_generated);
+void report_add_sta_event(_adapter *padapter, unsigned char *MacAddr);
+void report_wmm_edca_update(_adapter *padapter);
+
+void beacon_timing_control(_adapter *padapter);
+u8 chk_bmc_sleepq_cmd(_adapter *padapter);
+extern u8 set_tx_beacon_cmd(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 flags);
+unsigned int setup_beacon_frame(_adapter *padapter, unsigned char *beacon_frame);
+void update_mgnt_tx_rate(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 rate);
+void update_monitor_frame_attrib(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			struct pkt_attrib *pattrib);
+void update_mgntframe_attrib(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			struct pkt_attrib *pattrib);
+void update_mgntframe_attrib_addr(_adapter *padapter, struct xmit_frame *pmgntframe);
+void dump_mgntframe(_adapter *padapter, struct xmit_frame *pmgntframe);
+s32 dump_mgntframe_and_wait(_adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms);
+s32 dump_mgntframe_and_wait_ack(_adapter *padapter, struct xmit_frame *pmgntframe);
+s32 dump_mgntframe_and_wait_ack_timeout(_adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms);
+
+#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+u8 *rtw_set_channel_triplet(struct rtw_chset *chset, u8 *triplet, u8 *trip_len, char alpha2[2]);
+u8 rtw_set_ie_country(_adapter *adapter, u8 *pframe, u32 *pkt_len);
+#endif
+
+#ifdef CONFIG_P2P
+void issue_probersp_p2p(_adapter *padapter, unsigned char *da);
+void issue_probereq_p2p(_adapter *padapter, u8 *da);
+#endif /* CONFIG_P2P */
+int rtw_build_assocreq_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 *pframe, struct pkt_attrib *pattrib, u8 is_reassoc);
+u8 *rtw_build_probersp_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		u8 *pframe, struct pkt_attrib *pattrib, u8 is_valid_p2p_probereq);
+
+void issue_beacon(_adapter *padapter, int timeout_ms);
+void issue_probersp(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		unsigned char *da, u8 is_valid_p2p_probereq);
+#ifdef CONFIG_80211BE_EHT
+void issue_ml_probersp(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		unsigned char *da, u8 is_valid_p2p_probereq);
+#endif
+void _issue_assocreq(_adapter *padapter, u8 is_assoc);
+void issue_assocreq(_adapter *padapter);
+void issue_reassocreq(_adapter *padapter);
+void issue_asocrsp(_adapter *padapter, unsigned short status, struct sta_info *pstat, int pkt_type);
+void issue_auth(_adapter *padapter, struct sta_info *psta, unsigned short status);
+void issue_probereq(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, const NDIS_802_11_SSID *pssid, const u8 *da);
+s32 issue_probereq_ex(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, const NDIS_802_11_SSID *pssid, const u8 *da, u8 ch, bool append_wps, int try_cnt, int wait_ms);
+int rtw_match_self_addr(_adapter *padapter, const u8 *mac_addr);
+int rtw_match_bssid(_adapter *padapter, const u8 *mac_addr);
+int rtw_match_bssid_ext(_adapter *padapter, const u8 *mac_addr);
+int issue_nulldata(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *da, unsigned int power_mode, int try_cnt, int wait_ms);
+bool rtw_core_issu_null_data(void *priv, u8 ridx, u8 link_idx, bool ps, u8 module_id);
+
+int issue_qos_nulldata(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		unsigned char *da, u16 tid, u8 ps, int try_cnt, int wait_ms, u8 need_om);
+int issue_disasoc(_adapter *padapter, unsigned char *da, unsigned short reason);
+int issue_deauth(_adapter *padapter, unsigned char *da, unsigned short reason);
+int issue_deauth_ex(_adapter *padapter, u8 *da, unsigned short reason, int try_cnt, int wait_ms);
+void issue_addba_req(_adapter *adapter, unsigned char *ra, u8 tid);
+void issue_addba_rsp(_adapter *adapter, unsigned char *ra, u8 tid, u16 status,
+	             u16 size, struct ADDBA_request *paddba_req);
+u8 issue_addba_rsp_wait_ack(_adapter *adapter, unsigned char *ra, u8 tid,
+	                    u16 status, u16 size, struct ADDBA_request *paddba_req,
+	                    int try_cnt, int wait_ms);
+void issue_del_ba(_adapter *adapter, unsigned char *ra, u8 tid, u16 reason, u8 initiator);
+int issue_del_ba_ex(_adapter *adapter, unsigned char *ra, u8 tid, u16 reason, u8 initiator, int try_cnt, int wait_ms);
+void issue_action_BSSCoexistPacket(_adapter *padapter);
+
+#ifdef CONFIG_IEEE80211W
+void issue_action_SA_Query(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			unsigned char *raddr, unsigned char action, unsigned short tid, u8 key_type);
+int issue_deauth_11w(_adapter *padapter, unsigned char *da, unsigned short reason, u8 key_type);
+#endif /* CONFIG_IEEE80211W */
+int issue_action_SM_PS(_adapter *padapter ,  unsigned char *raddr , u8 NewMimoPsMode);
+int issue_action_SM_PS_wait_ack(_adapter *padapter, unsigned char *raddr, u8 NewMimoPsMode, int try_cnt, int wait_ms);
+void issue_action_spct_ch_switch(_adapter *padapter,
+		struct _ADAPTER_LINK *adapter_link, u8 *ra);
+#ifdef PRIVATE_R
+int issue_action_find_remote(_adapter *padapter);
+#endif
+
+unsigned int send_delba_sta_tid(_adapter *adapter, u8 initiator, struct sta_info *sta, u8 tid, u8 force);
+unsigned int send_delba_sta_tid_wait_ack(_adapter *adapter, u8 initiator, struct sta_info *sta, u8 tid, u8 force);
+
+unsigned int send_delba(_adapter *padapter, u8 initiator, u8 *addr);
+unsigned int send_beacon(_adapter *padapter);
+
+void start_clnt_assoc(_adapter *padapterf);
+void start_clnt_auth(_adapter *padapter);
+void start_clnt_join(_adapter *padapter);
+void start_create_ibss(_adapter *padapter);
+#if defined(CONFIG_LAYER2_ROAMING) && (defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM))
+void rtw_roam_nb_discover(_adapter *padapter, u8 bfroce);
+#endif
+void rtw_ie_handler(struct _ADAPTER *padapter, struct _ADAPTER_LINK *padapter_link, u8 *ie, u32 ie_len);
+unsigned int OnAssocReq(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAssocRsp(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnProbeReq(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnProbeRsp(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int DoReserved(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnBeacon(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAtim(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnDisassoc(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAuth(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAuthClient(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnDeAuth(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction(_adapter *padapter, union recv_frame *precv_frame);
+
+unsigned int on_action_spct(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_qos(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_dls(_adapter *padapter, union recv_frame *precv_frame);
+#ifdef CONFIG_RTW_WNM
+unsigned int on_action_wnm(_adapter *adapter, union recv_frame *rframe);
+#endif
+
+#define RX_AMPDU_ACCEPT_INVALID (0xFF)
+#define RX_AMPDU_SIZE_INVALID (0xFFFF)
+
+enum rx_ampdu_reason {
+	RX_AMPDU_DRV_FIXED = 1,
+	RX_AMPDU_BTCOEX = 2, /* not used, because BTCOEX has its own variable management */
+	RX_AMPDU_DRV_SCAN = 3,
+};
+u16 rtw_rx_ampdu_size(_adapter *adapter);
+bool rtw_rx_ampdu_is_accept(_adapter *adapter);
+bool rtw_rx_ampdu_set_size(_adapter *adapter, u16 size, u8 reason);
+bool rtw_rx_ampdu_set_accept(_adapter *adapter, u8 accept, u8 reason);
+u8 rx_ampdu_apply_sta_tid(_adapter *adapter, struct sta_info *sta, u8 tid, u8 accept, u16 size);
+u16 rx_ampdu_size_sta_limit(_adapter *adapter, struct sta_info *sta);
+u8 rx_ampdu_apply_sta(_adapter *adapter, struct sta_info *sta, u8 accept, u16 size);
+u16 rtw_rx_ampdu_apply(_adapter *adapter);
+
+unsigned int OnAction_back(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int on_action_public(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_ft(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_ht(_adapter *padapter, union recv_frame *precv_frame);
+#ifdef CONFIG_IEEE80211W
+unsigned int OnAction_sa_query(_adapter *padapter, union recv_frame *precv_frame);
+#endif /* CONFIG_IEEE80211W */
+unsigned int on_action_rm(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_wmm(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_vht(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_he(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_protected_he(_adapter *padapter, union recv_frame *precv_frame);
+unsigned int OnAction_p2p(_adapter *padapter, union recv_frame *precv_frame);
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+unsigned int OnAction_tbtx_token(_adapter *padapter, union recv_frame *precv_frame);
+#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+void rtw_issue_action_token_req(_adapter *padapter, struct sta_info *pstat);
+void rtw_issue_action_token_rel(_adapter *padapter);
+#endif
+
+unsigned int on_action_s1g(_adapter *padapter, union recv_frame *precv_frame);
+
+void rtw_mlmeext_disconnect(_adapter *padapter);
+int rtw_set_hw_after_join(struct _ADAPTER *a, int join_res);
+void mlmeext_sta_del_event_callback(_adapter *padapter);
+void mlmeext_sta_add_event_callback(_adapter *padapter, struct sta_info *psta);
+
+#ifdef CONFIG_POST_CORE_KEEP_ALIVE
+void rtw_core_keep_alive_post_hdlr(void *drv_priv, struct rtw_keep_alive_param *klive);
+#endif
+int rtw_get_rx_chk_limit(_adapter *adapter);
+void rtw_set_rx_chk_limit(_adapter *adapter, int limit);
+void linked_status_post_chk(_adapter *padapter, u8 if_sta_chk_rx_bmp, u8 if_sta_chk_tx_bmp);
+void linked_status_chk(_adapter *padapter, u8 *if_sta_chk_rx_bmp, u8 *if_sta_chk_tx_bmp);
+void dynamic_update_bcn_check(_adapter *padapter);
+
+#define rtw_get_bcn_cnt(adapter_link)	(adapter_link->mlmeextpriv.cur_bcn_cnt)
+#define rtw_get_bcn_dtim_period(adapter_link)	(adapter_link->mlmeextpriv.dtim)
+void rtw_collect_bcn_info(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+
+void _linked_info_dump(_adapter *padapter);
+
+void link_timer_hdl(void *ctx);
+void addba_timer_hdl(void *ctx);
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+void rtw_tbtx_xmit_timer_hdl(void *ctx);
+void rtw_tbtx_token_dispatch_timer_hdl(void *ctx);
+#endif
+#ifdef CONFIG_IEEE80211W
+void sa_query_timer_hdl(void *ctx);
+#endif /* CONFIG_IEEE80211W */
+#if 0
+void reauth_timer_hdl(_adapter *padapter);
+void reassoc_timer_hdl(_adapter *padapter);
+#endif
+#ifdef PRIVATE_R
+void find_remote_timer_hdl(void *ctx);
+#ifdef CONFIG_P2P
+void hide_ssid_hdl(void *ctx);
+#endif
+#endif
+#ifdef CONFIG_ECSA_PHL
+void csa_wait_bcn_timer_hdl(void *ctx);
+#endif
+
+#define set_link_timer(mlmeext, ms) \
+	do { \
+		/*RTW_INFO("%s set_link_timer(%p, %d)\n", __FUNCTION__, (mlmeext), (ms));*/ \
+		_set_timer(&(mlmeext)->link_timer, (ms)); \
+	} while (0)
+#define cancel_link_timer(mlmeext) \
+	do { \
+		/*RTW_INFO("%s cancel_link_timer(%p)\n", __FUNCTION__, (mlmeext));*/ \
+		_cancel_timer_ex(&(mlmeext)->link_timer); \
+	} while (0)
+#ifdef PRIVATE_R
+#define set_find_remote_timer(mlmeext, ms) \
+	do { \
+		/*RTW_INFO("%s set_find_remote_timer(%p, %d)\n", __FUNCTION__, (mlmeext), (ms));*/ \
+		_set_timer(&(mlmeext)->find_remote_timer, (ms)); \
+	} while (0)
+#endif
+
+bool rtw_is_cck_rate(u8 rate);
+bool rtw_is_ofdm_rate(u8 rate);
+bool rtw_is_basic_rate_cck(u8 rate);
+bool rtw_is_basic_rate_ofdm(u8 rate);
+bool rtw_is_basic_rate_mix(u8 rate);
+
+extern int cckrates_included(unsigned char *rate, int ratelen);
+extern int cckratesonly_included(unsigned char *rate, int ratelen);
+
+extern void process_addba_req(_adapter *padapter, u8 *paddba_req, u8 *addr);
+
+extern void update_TSF(struct link_mlme_ext_priv *pmlmeext, u8 *pframe, uint len);
+
+#ifdef CONFIG_BCN_RECV_TIME
+void rtw_rx_bcn_time_update(_adapter *adapter, struct _ADAPTER_LINK *adapter_link, uint bcn_len, u8 data_rate);
+#endif
+extern u8 traffic_status_watchdog(_adapter *padapter);
+
+void rtw_process_bar_frame(_adapter *padapter, union recv_frame *precv_frame);
+void rtw_join_done_chk_ch(_adapter *padapter, int join_res);
+
+void survey_done_set_ch_bw(_adapter *padapter);
+
+#ifdef RTW_BUSY_DENY_SCAN
+#ifndef BUSY_TRAFFIC_SCAN_DENY_PERIOD
+#ifdef CONFIG_RTW_ANDROID
+#ifdef CONFIG_PLATFORM_ARM_SUN8I
+	#define BUSY_TRAFFIC_SCAN_DENY_PERIOD	8000
+#else
+	#define BUSY_TRAFFIC_SCAN_DENY_PERIOD	12000
+#endif
+#else /* !CONFIG_ANDROID */
+#define BUSY_TRAFFIC_SCAN_DENY_PERIOD	16000
+#endif /* !CONFIG_ANDROID */
+#endif /* !BUSY_TRAFFIC_SCAN_DENY_PERIOD */
+#endif /* RTW_BUSY_DENY_SCAN */
+
+void rtw_leave_opch(_adapter *adapter);
+void rtw_back_opch(_adapter *adapter);
+
+void update_join_info(struct _ADAPTER *a, struct _WLAN_BSSID_EX *pbuf);
+void update_join_link_info(struct _ADAPTER *a, struct _ADAPTER_LINK *alink, struct _WLAN_BSSID_EX *pbuf);
+/*int rtw_chk_start_clnt_join(_adapter *adapter, u8 *ch, u8 *bw, u8 *offset);*/
+
+u8 rtw_join_cmd_hdl(_adapter *padapter, u8 *pbuf);
+u8 disconnect_hdl(_adapter *padapter, u8 *pbuf);
+u8 createbss_hdl(_adapter *padapter, u8 *pbuf);
+void rtw_disconnect_ch_switch(_adapter *adapter);
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+u32 rtw_build_vendor_ie(_adapter *padapter , unsigned char **pframe , u8 mgmt_frame_tyte);
+#endif
+#ifdef CONFIG_AP_MODE
+u8 stop_ap_hdl(_adapter *adapter);
+#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+u8 tx_control_hdl(_adapter *adapter);
+#endif
+u8 setopmode_hdl(_adapter *padapter, u8 *pbuf);
+
+u8 setauth_hdl(_adapter *padapter, u8 *pbuf);
+
+u8 setkey_hdl(struct _ADAPTER *a, struct _ADAPTER_LINK *alink,
+		struct setkey_parm *key, enum phl_cmd_type cmd_type,  u32 cmd_timeout);
+u8 set_stakey_hdl(_adapter *padapter, struct set_stakey_parm *key,
+		enum phl_cmd_type cmd_type,  u32 cmd_timeout);
+
+u8 set_assocsta_hdl(_adapter *padapter, u8 *pbuf);
+u8 del_assocsta_hdl(_adapter *padapter, u8 *pbuf);
+u8 add_ba_hdl(_adapter *padapter, unsigned char *pbuf);
+u8 add_ba_rsp_hdl(_adapter *padapter, unsigned char *pbuf);
+u8 delba_hdl(struct _ADAPTER *a, unsigned char *pbuf);
+
+void rtw_ap_wep_pk_setting(_adapter *adapter, struct sta_info *psta);
+
+u8 mlme_evt_hdl(_adapter *padapter, unsigned char *pbuf);
+u8 chk_bmc_sleepq_hdl(_adapter *padapter, unsigned char *pbuf);
+u8 tx_beacon_hdl(_adapter *padapter, unsigned char *pbuf);
+u8 rtw_set_chbw_hdl(_adapter *padapter, u8 *pbuf);
+u8 led_blink_hdl(_adapter *padapter, unsigned char *pbuf);
+u8 tdls_hdl(_adapter *padapter, unsigned char *pbuf);
+u8 run_in_thread_hdl(_adapter *padapter, u8 *pbuf);
+
+
+int rtw_sae_preprocess(_adapter *adapter, const u8 *buf, u32 len, u8 tx);
+
+u32 rtw_desc_rate_to_bitrate(u8 bw, u16 rate_idx, u8 sgi);
+u16 rtw_get_current_tx_rate(_adapter *padapter, struct sta_info *psta);
+u8 rtw_get_current_tx_sgi(_adapter *padapter, struct sta_info *psta);
+bool rtw_chk_phy_can_append_actrl(_adapter *padapter, struct sta_info *psta);
+void rtw_get_current_rx_info(_adapter *adapter, struct sta_info *psta,
+	u16 *rate, u8 *bw, u8 *gi_ltf);
+
+#ifdef CONFIG_RTW_MESH
+extern u8 rtw_mesh_set_plink_state_cmd_hdl(_adapter *adapter, u8 *parmbuf);
+#else
+static inline u8 rtw_mesh_set_plink_state_cmd_hdl(_adapter *adapter, u8 *parmbuf) { return H2C_CMD_FAIL; };
+#endif /*CONFIG_RTW_MESH*/
+
+#if defined(CONFIG_RTW_ACS) && defined(WKARD_ACS)
+#define IS_ACS_ENABLE(dvobj)					(_FALSE)
+
+enum rtw_nhm_pid {
+	NHM_PID_IEEE_11K_HIGH,
+	NHM_PID_ACS,
+};
+void rtw_acs_select_best_chan(_adapter *adapter);
+void rtw_acs_trigger(_adapter *padapter, u32 scan_ms, u8 scan_ch, u8 pid);
+void rtw_acs_get_rst(_adapter *adapter);
+#endif /* defined(CONFIG_RTW_ACS) && defined(WKARD_ACS) */
+
+#ifdef CONFIG_RTW_MULTI_AP
+u8 rtw_get_ch_utilization(_adapter *adapter);
+void rtw_ch_util_rpt(_adapter *adapter);
+#endif
+
+u8 rtw_set_mac_addr(_adapter *padapter, const u8 *mac_addr, bool cfg_hw, bool directly);
+
+#define GEN_MLME_EXT_HANDLER(name, cmd, callback)	{name, cmd, callback},
+
+struct rtw_cmd {
+	char name[32];
+	u8(*cmd_hdl)(_adapter *padapter, u8 *pbuf);
+	void (*callback)(_adapter  *padapter, struct cmd_obj *cmd);
+};
+
+
+#ifdef CONFIG_RTW_FSM_RRM
+u8 fsm_post_event_hdl(_adapter *padapter, u8 *pbuf);
+#endif
+#ifdef _RTW_CMD_C_
+struct rtw_cmd wlancmds[] = {
+	GEN_MLME_EXT_HANDLER("CMD_JOINBSS", rtw_join_cmd_hdl, NULL) /*CMD_JOINBSS*/
+	GEN_MLME_EXT_HANDLER("CMD_DISCONNECT", disconnect_hdl, rtw_disassoc_cmd_callback) /*CMD_DISCONNECT*/
+	GEN_MLME_EXT_HANDLER("CMD_CREATE_BSS", createbss_hdl, NULL) /*CMD_CREATE_BSS*/
+	GEN_MLME_EXT_HANDLER("CMD_SET_OPMODE", setopmode_hdl, NULL) /*CMD_SET_OPMODE*/
+	GEN_MLME_EXT_HANDLER("CMD_SET_AUTH", setauth_hdl, NULL) /*CMD_SET_AUTH*/
+	GEN_MLME_EXT_HANDLER("CMD_ADD_BAREQ", add_ba_hdl, NULL) /*CMD_ADD_BAREQ*/
+	GEN_MLME_EXT_HANDLER("CMD_SET_CHANNEL", rtw_set_chbw_hdl, NULL) /*CMD_SET_CHANNEL*/
+	GEN_MLME_EXT_HANDLER("CMD_TX_BEACON", tx_beacon_hdl, NULL) /*CMD_TX_BEACON*/
+	GEN_MLME_EXT_HANDLER("CMD_SET_MLME_EVT", mlme_evt_hdl, NULL) /*CMD_SET_MLME_EVT*/
+	GEN_MLME_EXT_HANDLER("CMD_SET_DRV_EXTRA", rtw_drvextra_cmd_hdl, NULL) /*CMD_SET_DRV_EXTRA*/
+	GEN_MLME_EXT_HANDLER("CMD_SET_CHANPLAN", rtw_set_chplan_hdl, NULL) /*CMD_SET_CHANPLAN*/
+	GEN_MLME_EXT_HANDLER("CMD_LEDBLINK", led_blink_hdl, NULL) /*CMD_LEDBLINK*/
+	GEN_MLME_EXT_HANDLER("CMD_TDLS", tdls_hdl, NULL) /*CMD_TDLS*/
+	GEN_MLME_EXT_HANDLER("CMD_CHK_BMCSLEEPQ", chk_bmc_sleepq_hdl, NULL) /*CMD_CHK_BMCSLEEPQ*/
+	GEN_MLME_EXT_HANDLER("CMD_RUN_INTHREAD", run_in_thread_hdl, NULL) /*CMD_RUN_INTHREAD*/
+	GEN_MLME_EXT_HANDLER("CMD_ADD_BARSP", add_ba_rsp_hdl, NULL) /*CMD_ADD_BARSP*/
+#ifdef CONFIG_RTW_80211K
+	GEN_MLME_EXT_HANDLER("CMD_RM_POST_EVENT", rm_post_event_hdl, NULL) /*CMD_RM_POST_EVENT*/
+#endif
+#ifdef CONFIG_RTW_FSM_RRM
+	GEN_MLME_EXT_HANDLER("CMD_RM_POST_EVENT", fsm_post_event_hdl, NULL) /*CMD_RM_POST_EVENT*/
+#endif
+	GEN_MLME_EXT_HANDLER("CMD_SET_MESH_PLINK_STATE", rtw_mesh_set_plink_state_cmd_hdl, NULL) /*CMD_SET_MESH_PLINK_STATE*/
+	GEN_MLME_EXT_HANDLER("CMD_DELBA", delba_hdl, NULL) /*CMD_DELBA*/
+	GEN_MLME_EXT_HANDLER("CMD_GET_CHANPLAN", rtw_get_chplan_hdl, rtw_get_chplan_callback) /* CMD_GET_CHANPLAN */
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	GEN_MLME_EXT_HANDLER("CMD_TWT", rtw_core_twt_hdl, NULL) /* CMD_TWT */
+#endif
+#endif
+};
+#endif /*_RTW_CMD_C_*/
+char *rtw_cmd_name(struct cmd_obj *pcmd);
+
+/*rtw_event*/
+struct rtw_evt_header {
+	u8 id;
+	u8 seq;
+	u16 len;
+};
+
+enum rtw_event_id {
+	EVT_SURVEY, /*0*/
+	EVT_SURVEY_DONE, /*1*/
+	EVT_JOINBSS, /*2*/
+	EVT_ADD_STA, /*3*/
+	EVT_DEL_STA, /*4*/
+	EVT_WMM_UPDATE, /*5*/
+#ifdef CONFIG_IEEE80211W
+	EVT_TIMEOUT_STA, /*6*/
+#endif /* CONFIG_IEEE80211W */
+#ifdef CONFIG_RTW_80211R
+	EVT_FT_REASSOC, /*7*/
+#endif
+	EVT_ID_MAX
+};
+
+struct rtw_event {
+	char name[32];
+	u32 parmsize;
+	void (*event_callback)(_adapter *dev, u8 *pbuf);
+};
+#ifdef _RTW_MLME_EXT_C_
+static struct rtw_event wlanevents[] = {
+	{"EVT_SURVEY", sizeof(struct survey_event), &rtw_survey_event_callback}, /*EVT_SURVEY*/
+	{"EVT_SURVEY_DONE", sizeof(struct surveydone_event), &rtw_surveydone_event_callback}, /*EVT_SURVEY_DONE*/
+	{"EVT_JOINBSS", sizeof(struct joinbss_event), &rtw_joinbss_event_callback}, /*EVT_JOINBSS*/
+	{"EVT_ADD_STA", sizeof(struct stassoc_event), &rtw_stassoc_event_callback}, /*EVT_ADD_STA*/
+	{"EVT_DEL_STA", sizeof(struct stadel_event), &rtw_stadel_event_callback}, /*EVT_DEL_STA*/
+	{"EVT_WMM_UPDATE", sizeof(struct wmm_event), &rtw_wmm_event_callback}, /*EVT_WMM_UPDATE*/
+	#ifdef CONFIG_IEEE80211W
+	{"EVT_TIMEOUT_STA", sizeof(struct stadel_event), &rtw_sta_timeout_event_callback}, /*EVT_TIMEOUT_STA*/
+	#endif /* CONFIG_IEEE80211W */
+	#ifdef CONFIG_RTW_80211R
+	{"EVT_FT_REASSOC", sizeof(struct stassoc_event), &rtw_ft_reassoc_event_callback}, /*EVT_FT_REASSOC*/
+	#endif
+};
+#endif/*_RTW_MLME_EXT_C_*/
+char *rtw_evt_name(struct rtw_evt_header *pev);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mp.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mp.h
new file mode 100644
index 000000000000..ee066543b61a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_mp.h
@@ -0,0 +1,1746 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_MP_H_
+#define _RTW_MP_H_
+
+#include <drv_types.h>
+
+#define RTWPRIV_VER_INFO	1
+
+#define MAX_MP_XMITBUF_SZ	2048
+#define NR_MP_XMITFRAME		8
+#define MP_READ_REG_MAX_OFFSET 0x4FFF
+
+#define TX_POWER_BASE 4  /* dbm * 4 */
+#define TX_POWER_CODE_WORD_BASE 8 /* dbm * 8 */
+
+#define RTW_IWD_MAX_LEN	128
+
+struct rtw_mp_ax_tx_cnt {
+	u8 band;
+	u8 sel;
+	u16 txcnt[11];
+};
+
+struct mp_xmit_frame {
+	_list	list;
+
+	struct pkt_attrib attrib;
+
+	struct sk_buff *pkt;
+
+	int frame_tag;
+
+	_adapter *padapter;
+
+#ifdef CONFIG_USB_HCI
+
+	/* insert urb, irp, and irpcnt info below... */
+	/* max frag_cnt = 8 */
+	u8 *mem_addr;
+	u32 sz[8];
+	u8 bpending[8];
+	sint ac_tag[8];
+	sint last[8];
+	uint irpcnt;
+	uint fragcnt;
+#endif /* CONFIG_USB_HCI */
+
+	uint mem[(MAX_MP_XMITBUF_SZ >> 2)];
+};
+
+struct mp_wiparam {
+	u32 bcompleted;
+	u32 act_type;
+	u32 io_offset;
+	u32 io_value;
+};
+
+typedef void(*wi_act_func)(void *padapter);
+
+struct mp_tx {
+	u8 stop;
+	u32 count, sended;
+	u8 payload;
+	struct pkt_attrib attrib;
+	/* struct tx_desc desc; */
+	/* u8 resvdtx[7]; */
+	u8 desc[TXDESC_SIZE];
+	u8 *pallocated_buf;
+	u8 *buf;
+	u32 buf_size, write_size;
+	_thread_hdl_ PktTxThread;
+};
+
+#define MP_MAX_LINES		1000
+#define MP_MAX_LINES_BYTES	256
+
+
+typedef struct _RT_PMAC_PKT_INFO {
+	u8			MCS;
+	u8			Nss;
+	u8			Nsts;
+	u32			N_sym;
+	u8			SIGA2B3;
+} RT_PMAC_PKT_INFO, *PRT_PMAC_PKT_INFO;
+
+typedef struct _RT_PMAC_TX_INFO {
+	u8			bEnPMacTx:1;		/* 0: Disable PMac 1: Enable PMac */
+	u8			Mode:3;				/* 0: Packet TX 3:Continuous TX */
+	u8			Ntx:4;				/* 0-7 */
+	u8			TX_RATE;			/* MPT_RATE_E */
+	u8			TX_RATE_HEX;
+	u8			TX_SC;
+	u8			bSGI:1;
+	u8			bSPreamble:1;
+	u8			bSTBC:1;
+	u8			bLDPC:1;
+	u8			NDP_sound:1;
+	u8			BandWidth:3;		/* 0: 20 1:40 2:80Mhz */
+	u8			m_STBC;			/* bSTBC + 1 */
+	u16			PacketPeriod;
+	u32		PacketCount;
+	u32		PacketLength;
+	u8			PacketPattern;
+	u16			SFD;
+	u8			SignalField;
+	u8			ServiceField;
+	u16			LENGTH;
+	u8			CRC16[2];
+	u8			LSIG[3];
+	u8			HT_SIG[6];
+	u8			VHT_SIG_A[6];
+	u8			VHT_SIG_B[4];
+	u8			VHT_SIG_B_CRC;
+	u8			VHT_Delimiter[4];
+	u8			MacAddress[6];
+} RT_PMAC_TX_INFO, *PRT_PMAC_TX_INFO;
+
+struct rtw_mp_giltf_data {
+	u8 gi;
+	u8 ltf;
+	char type_str[8];
+};
+
+typedef void (*MPT_WORK_ITEM_HANDLER)(void *adapter);
+typedef struct _MPT_CONTEXT {
+	/* Indicate if we have started Mass Production Test. */
+	BOOLEAN			bMassProdTest;
+
+	/* Indicate if the driver is unloading or unloaded. */
+	BOOLEAN			bMptDrvUnload;
+
+	_sema			MPh2c_Sema;
+	_timer			MPh2c_timeout_timer;
+	/* Event used to sync H2c for BT control */
+
+	BOOLEAN		MptH2cRspEvent;
+	BOOLEAN		MptBtC2hEvent;
+	BOOLEAN		bMPh2c_timeout;
+
+	/* 8190 PCI does not support NDIS_WORK_ITEM. */
+	/* Work Item for Mass Production Test. */
+	/* NDIS_WORK_ITEM	MptWorkItem;
+	*	RT_WORK_ITEM		MptWorkItem; */
+	/* Event used to sync the case unloading driver and MptWorkItem is still in progress.
+	*	NDIS_EVENT		MptWorkItemEvent; */
+	/* To protect the following variables.
+	*	NDIS_SPIN_LOCK		MptWorkItemSpinLock; */
+	/* Indicate a MptWorkItem is scheduled and not yet finished. */
+	BOOLEAN			bMptWorkItemInProgress;
+	/* An instance which implements function and context of MptWorkItem. */
+	MPT_WORK_ITEM_HANDLER	CurrMptAct;
+
+	/* 1=Start, 0=Stop from UI. */
+	u32			MptTestStart;
+	/* _TEST_MODE, defined in MPT_Req2.h */
+	u32			MptTestItem;
+	/* Variable needed in each implementation of CurrMptAct. */
+	u32			MptActType;	/* Type of action performed in CurrMptAct. */
+	/* The Offset of IO operation is depend of MptActType. */
+	u32			MptIoOffset;
+	/* The Value of IO operation is depend of MptActType. */
+	u32			MptIoValue;
+	/* The RfPath of IO operation is depend of MptActType. */
+
+	u32			mpt_rf_path;
+
+	u8			MptChannelToSw;	/* Channel to switch. */
+	u8			MptInitGainToSet;	/* Initial gain to set. */
+	/* u32			bMptAntennaA;		 */ /* TRUE if we want to use antenna A. */
+	u32			MptBandWidth;		/* bandwidth to switch. */
+
+	u32			mpt_rate_index;/* rate index. */
+
+	/* Register value kept for Single Carrier Tx test. */
+	u8			btMpCckTxPower;
+	/* Register value kept for Single Carrier Tx test. */
+	u8			btMpOfdmTxPower;
+	/* For MP Tx Power index */
+	u8			TxPwrLevel[4];	/* rf-A, rf-B*/
+	u32			RegTxPwrLimit;
+	/* Content of RCR Regsiter for Mass Production Test. */
+	u32			MptRCR;
+	/* TRUE if we only receive packets with specific pattern. */
+	BOOLEAN			bMptFilterPattern;
+	/* Rx OK count, statistics used in Mass Production Test. */
+	u32			MptRxOkCnt;
+	/* Rx CRC32 error count, statistics used in Mass Production Test. */
+	u32			MptRxCrcErrCnt;
+
+	BOOLEAN			bCckContTx;	/* TRUE if we are in CCK Continuous Tx test. */
+	BOOLEAN			bOfdmContTx;	/* TRUE if we are in OFDM Continuous Tx test. */
+		/* TRUE if we have start Continuous Tx test. */
+	BOOLEAN			is_start_cont_tx;
+
+	/* TRUE if we are in Single Carrier Tx test. */
+	BOOLEAN			bSingleCarrier;
+	/* TRUE if we are in Carrier Suppression Tx Test. */
+
+	BOOLEAN			is_carrier_suppression;
+
+	/* TRUE if we are in Single Tone Tx test. */
+
+	BOOLEAN			is_single_tone;
+
+
+	/* ACK counter asked by K.Y.. */
+	BOOLEAN			bMptEnableAckCounter;
+	u32			MptAckCounter;
+
+	/* SD3 Willis For 8192S to save 1T/2T RF table for ACUT	Only fro ACUT delete later ~~~! */
+	/* s8		BufOfLines[2][MAX_LINES_HWCONFIG_TXT][MAX_BYTES_LINE_HWCONFIG_TXT]; */
+	/* s8			BufOfLines[2][MP_MAX_LINES][MP_MAX_LINES_BYTES]; */
+	/* s32			RfReadLine[2]; */
+
+	u8		APK_bound[2];	/* for APK	path A/path B */
+	BOOLEAN		bMptIndexEven;
+
+	u8		backup0xc50;
+	u8		backup0xc58;
+	u8		backup0xc30;
+	u8		backup0x52_RF_A;
+	u8		backup0x52_RF_B;
+
+	u32			backup0x58_RF_A;
+	u32			backup0x58_RF_B;
+
+	u8			h2cReqNum;
+	u8			c2hBuf[32];
+
+	u8          btInBuf[100];
+	u32			mptOutLen;
+	u8          mptOutBuf[100];
+	RT_PMAC_TX_INFO	PMacTxInfo;
+	RT_PMAC_PKT_INFO	PMacPktInfo;
+	u8 HWTxmode;
+
+	BOOLEAN			bldpc;
+	BOOLEAN			bstbc;
+} MPT_CONTEXT, *PMPT_CONTEXT;
+/* #endif */
+
+
+/* #define RTPRIV_IOCTL_MP					( SIOCIWFIRSTPRIV + 0x17) */
+enum {
+	WRITE_REG = 1,
+	READ_REG,
+	WRITE_RF,
+	READ_RF,
+	MP_START,
+	MP_STOP,
+	MP_RATE,
+	MP_CHANNEL,
+	MP_TRXSC_OFFSET,
+	MP_BANDWIDTH,
+	MP_TXPOWER,
+	MP_ANT_TX,
+	MP_ANT_RX,
+	MP_CTX,
+	MP_QUERY,
+	MP_ARX,
+	MP_PSD,
+	MP_PWRTRK,
+	MP_THER,
+	MP_IOCTL,
+	EFUSE_GET,
+	EFUSE_SET,
+	MP_RESET_STATS,
+	MP_DUMP,
+	MP_PHYPARA,
+	MP_SetRFPathSwh,
+	MP_QueryDrvStats,
+	CTA_TEST,
+	MP_DISABLE_BT_COEXIST,
+	MP_PwrCtlDM,
+	MP_GETVER,
+	MP_MON,
+	EFUSE_BT_MASK,
+	EFUSE_MASK,
+	EFUSE_FILE,
+	EFUSE_FILE_STORE,
+	MP_TX,
+	MP_RX,
+	MP_IQK,
+	MP_LCK,
+	MP_HW_TX_MODE,
+	MP_GET_TXPOWER_INX,
+	MP_CUSTOMER_STR,
+	MP_PWRLMT,
+	MP_PWRBYRATE,
+	BT_EFUSE_FILE,
+	MP_SWRFPath,
+	MP_LINK,
+	MP_DPK_TRK,
+	MP_DPK,
+	MP_GET_TSSIDE,
+	MP_SET_TSSIDE,
+	MP_GET_PHL_TEST,
+	MP_SET_PHL_TEST,
+	MP_SET_PHL_TX_PATTERN,
+	MP_SET_PHL_PLCP_TX_DATA,
+	MP_SET_PHL_PLCP_TX_USER,
+	MP_SET_PHL_TX_METHOD,
+	MP_SET_PHL_CONIFG_PHY_NUM,
+	MP_PHL_RFK,
+	MP_PHL_BTC_PATH,
+	MP_GET_HE,
+	MP_UUID,
+	MP_GPIO,
+	MP_BAND,
+	MP_MACLOOPBK,
+	MP_MAC_IOTEST,
+	MP_NULL,
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	VENDOR_IE_SET ,
+	VENDOR_IE_GET ,
+#endif
+#if defined(RTW_PHL_TX) || defined(RTW_PHL_RX) || defined(CONFIG_PHL_TEST_SUITE)
+	PHL_TEST_SET,
+	PHL_TEST_GET,
+#endif
+#ifdef CONFIG_WOWLAN
+	MP_WOW_ENABLE,
+	MP_WOW_SET_PATTERN,
+#endif
+#ifdef CONFIG_AP_WOWLAN
+	MP_AP_WOW_ENABLE,
+#endif
+	MP_SD_IREAD,
+	MP_SD_IWRITE,
+#ifdef CONFIG_FPGA_INCLUDED
+	FPGA_SET,
+#endif
+#ifdef CONFIG_WOW_APF
+	APF_SET,
+	APF_GET,
+	APF_DEFAULT_SET,
+	APF_DEFAULT_GET,
+#endif
+};
+
+struct rtw_plcp_user {
+	u8 plcp_usr_idx;
+	u16 plcp_mcs;
+	u8 coding;
+	u8 dcm;
+	u8 aid;
+	u32 plcp_txlen; /*apep*/
+	u32 ru_alloc;
+	u8 plcp_nss;
+	u8 txbf;
+	u8 pwr_boost_db;
+};
+
+struct mp_priv {
+	_adapter *papdater;
+
+	/* Testing Flag */
+	u32 mode;/* 0 for normal type packet, 1 for loopback packet (16bytes TXCMD) */
+
+	u32 prev_fw_state;
+
+	/* OID cmd handler */
+	struct mp_wiparam workparam;
+	/*	u8 act_in_progress; */
+
+	/* Tx Section */
+	u8 TID;
+	u32 tx_pktcount;
+	u32 pktInterval;
+	u32 pktLength;
+	struct mp_tx tx;
+
+	/* Rx Section */
+	u32 rx_bssidpktcount;
+	u32 rx_pktcount;
+	u32 rx_pktcount_filter_out;
+	u32 rx_crcerrpktcount;
+	u32 rx_pktloss;
+	BOOLEAN  rx_bindicatePkt;
+	struct recv_stat rxstat;
+	BOOLEAN brx_filter_beacon;
+
+	/* RF/BB relative */
+	u8 band;
+	u32 channel;
+	u8 bandwidth;
+	u8 prime_channel_offset;
+	u8 txpoweridx;
+	s16 txpowerdbm;
+	u16 rateidx;
+	s16 pre_refcw_cck_pwridxa;
+	s16 pre_refcw_cck_pwridxb;
+	s16 pre_refcw_ofdm_pwridxa;
+	s16 pre_refcw_ofdm_pwridxb;
+
+	u32 preamble;
+	/*	u8 modem; */
+	u32 CrystalCap;
+	/*	u32 curr_crystalcap; */
+
+	u8 antenna_tx;
+	u8 antenna_rx;
+	u8 antenna_trx;
+	u8 curr_rfpath;
+	u8 ant_sw;
+
+	u8 check_mp_pkt;
+
+	u8 bSetTxPower;
+	/*	uint ForcedDataRate; */
+	u8 mp_dm;
+	u8 mac_filter[ETH_ALEN];
+	u8 bmac_filter;
+
+	/* RF PATH Setting for WLG WLA BTG BT */
+	u8 rf_path_cfg;
+	u8 btc_path; /* BTC_MODE_NORMAL, BTC_MODE_WL,BTC_MODE_BT */
+
+	struct wlan_network mp_network;
+	NDIS_802_11_MAC_ADDRESS network_macaddr;
+
+	u8 *pallocated_mp_xmitframe_buf;
+	u8 *pmp_xmtframe_buf;
+	_queue free_mp_xmitqueue;
+	u32 free_mp_xmitframe_cnt;
+	BOOLEAN bSetRxBssid;
+	BOOLEAN bTxBufCkFail;
+	BOOLEAN bRTWSmbCfg;
+	BOOLEAN bloopback;
+	BOOLEAN bloadefusemap;
+	BOOLEAN bloadBTefusemap;
+	BOOLEAN bprocess_mp_mode;
+
+	MPT_CONTEXT	mpt_ctx;
+
+	u8		*TXradomBuffer;
+	u8		mp_keep_btc_mode;
+	u8		mplink_buf[2048];
+	u32		mplink_rx_len;
+	BOOLEAN mplink_brx;
+	BOOLEAN mplink_btx;
+
+	bool tssitrk_on;
+	bool bspecif_tssi_de;
+	u32 specif_tsside_val;
+
+	u8 tssi_mode;
+	u8 rtw_mp_cur_phy;
+	u8 rtw_mp_dbcc;
+	s16 path_pwr_offset[4];	/* rf-A, rf-B*/
+	u8 rtw_mp_tx_method;
+	u16 rtw_mp_tx_time;
+	u8 rtw_mp_tx_state;
+	u8 rtw_mp_pmact_patt_idx;
+	u8 rtw_mp_pmact_ppdu_type;
+	u8 rtw_mp_data_bandwidth;
+	u8 rtw_mp_stbc;
+	u8 rtw_mp_plcp_gi;
+	u8 rtw_mp_plcp_ltf;
+	u8 rtw_mp_he_sigb;
+	u8 rtw_mp_he_sigb_dcm;
+	u32 rtw_mp_plcp_tx_time;
+	u8 rtw_mp_plcp_tx_mode;
+
+	u8 rtw_mp_he_er_su_ru_106_en;
+	u8 rtw_mp_trxsc;
+	u16 rtw_mp_plcp_rualloc;
+	u8 rtw_mp_plcp_tx_user;
+	u32 rtw_mp_ru_tone;
+	u8 ru_tone_sel_list[6];
+	u8 ru_alloc_list[68];
+	u8 rtw_coding;
+
+	struct rtw_mp_giltf_data st_giltf[5];
+	struct rtw_plcp_user mp_plcp_user[4];
+	u8 mp_plcp_useridx;
+
+	u8 keep_ips_status;
+	u8 keep_lps_status;
+	u8 tx_shape_idx;
+	u8 gpio_id;
+	u8 gpio_enable;
+	u8 is_tmac_mode;
+
+	_thread_hdl_ rx_cal_thread;
+	u8 rx_cal_stop;
+	u8 rx_cal_process;
+
+	u8 loopbk_speed;
+	u8 mac_iotest_res;
+	u16 rx_rate;
+};
+
+#define PPDU_TYPE_STR(idx)\
+	(idx == RTW_MP_TYPE_CCK) ? "CCK" :\
+	(idx == RTW_MP_TYPE_LEGACY) ? "LEGACY" :\
+	(idx == RTW_MP_TYPE_HT_MF) ? "HT_MF" :\
+	(idx == RTW_MP_TYPE_HT_GF) ? "HT_GF" :\
+	(idx == RTW_MP_TYPE_VHT) ? "VHT" :\
+	(idx == RTW_MP_TYPE_HE_SU) ? "HE_SU" :\
+	(idx == RTW_MP_TYPE_HE_ER_SU) ? "HE_ER_SU" :\
+	(idx == RTW_MP_TYPE_HE_MU_OFDMA) ? "HE_MU" :\
+	(idx == RTW_MP_TYPE_HE_TB) ? "HE_TB" :\
+	"UNknow"
+
+
+typedef struct _IOCMD_STRUCT_ {
+	u8	cmdclass;
+	u16	value;
+	u8	index;
+} IOCMD_STRUCT;
+
+struct rf_reg_param {
+	u32 path;
+	u32 offset;
+	u32 value;
+};
+
+struct bb_reg_param {
+	u32 offset;
+	u32 value;
+};
+
+/* *********************************************************************** */
+
+#define LOWER	_TRUE
+#define RAISE	_FALSE
+
+/* Hardware Registers */
+#if 0
+#if 0
+#define IOCMD_CTRL_REG			0x102502C0
+#define IOCMD_DATA_REG			0x102502C4
+#else
+#define IOCMD_CTRL_REG			0x10250370
+#define IOCMD_DATA_REG			0x10250374
+#endif
+
+#define IOCMD_GET_THERMAL_METER		0xFD000028
+
+#define IOCMD_CLASS_BB_RF		0xF0
+#define IOCMD_BB_READ_IDX		0x00
+#define IOCMD_BB_WRITE_IDX		0x01
+#define IOCMD_RF_READ_IDX		0x02
+#define IOCMD_RF_WRIT_IDX		0x03
+#endif
+#define BB_REG_BASE_ADDR		0x800
+
+/* MP variables */
+#if 0
+#define _2MAC_MODE_	0
+#define _LOOPBOOK_MODE_	1
+#endif
+
+typedef enum _MP_MODE_ {
+	MP_OFF,
+	MP_ON,
+	MP_ERR,
+	MP_CONTINUOUS_TX,
+	MP_SINGLE_CARRIER_TX,
+	MP_CARRIER_SUPPRISSION_TX,
+	MP_SINGLE_TONE_TX,
+	MP_PACKET_TX,
+	MP_PACKET_RX
+} MP_MODE;
+
+typedef enum _TEST_MODE {
+	TEST_NONE                 ,
+	PACKETS_TX                ,
+	PACKETS_RX                ,
+	CONTINUOUS_TX             ,
+	OFDM_Single_Tone_TX       ,
+	CCK_Carrier_Suppression_TX
+} TEST_MODE;
+
+typedef enum _MPT_BANDWIDTH {
+	MPT_BW_20MHZ = 0,
+	MPT_BW_40MHZ_DUPLICATE = 1,
+	MPT_BW_40MHZ_ABOVE = 2,
+	MPT_BW_40MHZ_BELOW = 3,
+	MPT_BW_40MHZ = 4,
+	MPT_BW_80MHZ = 5,
+	MPT_BW_80MHZ_20_ABOVE = 6,
+	MPT_BW_80MHZ_20_BELOW = 7,
+	MPT_BW_80MHZ_20_BOTTOM = 8,
+	MPT_BW_80MHZ_20_TOP = 9,
+	MPT_BW_80MHZ_40_ABOVE = 10,
+	MPT_BW_80MHZ_40_BELOW = 11,
+} MPT_BANDWIDTHE, *PMPT_BANDWIDTH;
+
+#define MAX_RF_PATH_NUMS	RF_PATH_MAX
+
+
+extern u8 mpdatarate[NumRates];
+
+/* MP set force data rate base on the definition. */
+typedef enum _MPT_RATE_INDEX {
+	/* CCK rate. */
+	MPT_RATE_1M = 1 ,	/* 0 */
+	MPT_RATE_2M,
+	MPT_RATE_55M,
+	MPT_RATE_11M,	/* 3 */
+
+	/* OFDM rate. */
+	MPT_RATE_6M,	/* 4 */
+	MPT_RATE_9M,
+	MPT_RATE_12M,
+	MPT_RATE_18M,
+	MPT_RATE_24M,
+	MPT_RATE_36M,
+	MPT_RATE_48M,
+	MPT_RATE_54M,	/* 11 */
+
+	/* HT rate. */
+	MPT_RATE_MCS0,	/* 12 */
+	MPT_RATE_MCS1,
+	MPT_RATE_MCS2,
+	MPT_RATE_MCS3,
+	MPT_RATE_MCS4,
+	MPT_RATE_MCS5,
+	MPT_RATE_MCS6,
+	MPT_RATE_MCS7,	/* 19 */
+	MPT_RATE_MCS8,
+	MPT_RATE_MCS9,
+	MPT_RATE_MCS10,
+	MPT_RATE_MCS11,
+	MPT_RATE_MCS12,
+	MPT_RATE_MCS13,
+	MPT_RATE_MCS14,
+	MPT_RATE_MCS15,	/* 27 */
+	MPT_RATE_MCS16,
+	MPT_RATE_MCS17, /*  #29 */
+	MPT_RATE_MCS18,
+	MPT_RATE_MCS19,
+	MPT_RATE_MCS20,
+	MPT_RATE_MCS21,
+	MPT_RATE_MCS22, /*  #34 */
+	MPT_RATE_MCS23,
+	MPT_RATE_MCS24,
+	MPT_RATE_MCS25,
+	MPT_RATE_MCS26,
+	MPT_RATE_MCS27, /*  #39 */
+	MPT_RATE_MCS28, /*  #40 */
+	MPT_RATE_MCS29, /*  #41 */
+	MPT_RATE_MCS30, /*  #42 */
+	MPT_RATE_MCS31, /*  #43 */
+	/* VHT rate. Total: 20*/
+	MPT_RATE_VHT1SS_MCS0 = 100,/*  #44*/
+	MPT_RATE_VHT1SS_MCS1, /*  # */
+	MPT_RATE_VHT1SS_MCS2,
+	MPT_RATE_VHT1SS_MCS3,
+	MPT_RATE_VHT1SS_MCS4,
+	MPT_RATE_VHT1SS_MCS5,
+	MPT_RATE_VHT1SS_MCS6, /*  # */
+	MPT_RATE_VHT1SS_MCS7,
+	MPT_RATE_VHT1SS_MCS8,
+	MPT_RATE_VHT1SS_MCS9, /* #53 */
+	MPT_RATE_VHT2SS_MCS0, /* #54 */
+	MPT_RATE_VHT2SS_MCS1,
+	MPT_RATE_VHT2SS_MCS2,
+	MPT_RATE_VHT2SS_MCS3,
+	MPT_RATE_VHT2SS_MCS4,
+	MPT_RATE_VHT2SS_MCS5,
+	MPT_RATE_VHT2SS_MCS6,
+	MPT_RATE_VHT2SS_MCS7,
+	MPT_RATE_VHT2SS_MCS8,
+	MPT_RATE_VHT2SS_MCS9, /* #63 */
+	MPT_RATE_VHT3SS_MCS0,
+	MPT_RATE_VHT3SS_MCS1,
+	MPT_RATE_VHT3SS_MCS2,
+	MPT_RATE_VHT3SS_MCS3,
+	MPT_RATE_VHT3SS_MCS4,
+	MPT_RATE_VHT3SS_MCS5,
+	MPT_RATE_VHT3SS_MCS6, /*  #126 */
+	MPT_RATE_VHT3SS_MCS7,
+	MPT_RATE_VHT3SS_MCS8,
+	MPT_RATE_VHT3SS_MCS9,
+	MPT_RATE_VHT4SS_MCS0,
+	MPT_RATE_VHT4SS_MCS1, /*  #131 */
+	MPT_RATE_VHT4SS_MCS2,
+	MPT_RATE_VHT4SS_MCS3,
+	MPT_RATE_VHT4SS_MCS4,
+	MPT_RATE_VHT4SS_MCS5,
+	MPT_RATE_VHT4SS_MCS6, /*  #136 */
+	MPT_RATE_VHT4SS_MCS7,
+	MPT_RATE_VHT4SS_MCS8,
+	MPT_RATE_VHT4SS_MCS9,
+	MPT_RATE_LAST
+} MPT_RATE_E, *PMPT_RATE_E;
+
+#define MAX_TX_PWR_INDEX_N_MODE 64	/* 0x3F */
+
+#define MPT_IS_CCK_RATE(_value)		(MPT_RATE_1M <= _value && _value <= MPT_RATE_11M)
+#define MPT_IS_OFDM_RATE(_value)	(MPT_RATE_6M <= _value && _value <= MPT_RATE_54M)
+#define MPT_IS_HT_RATE(_value)		(MPT_RATE_MCS0 <= _value && _value <= MPT_RATE_MCS31)
+#define MPT_IS_HT_1S_RATE(_value)	(MPT_RATE_MCS0 <= _value && _value <= MPT_RATE_MCS7)
+#define MPT_IS_HT_2S_RATE(_value)	(MPT_RATE_MCS8 <= _value && _value <= MPT_RATE_MCS15)
+#define MPT_IS_HT_3S_RATE(_value)	(MPT_RATE_MCS16 <= _value && _value <= MPT_RATE_MCS23)
+#define MPT_IS_HT_4S_RATE(_value)	(MPT_RATE_MCS24 <= _value && _value <= MPT_RATE_MCS31)
+
+#define MPT_IS_VHT_RATE(_value)		(MPT_RATE_VHT1SS_MCS0 <= _value && _value <= MPT_RATE_VHT4SS_MCS9)
+#define MPT_IS_VHT_1S_RATE(_value)	(MPT_RATE_VHT1SS_MCS0 <= _value && _value <= MPT_RATE_VHT1SS_MCS9)
+#define MPT_IS_VHT_2S_RATE(_value)	(MPT_RATE_VHT2SS_MCS0 <= _value && _value <= MPT_RATE_VHT2SS_MCS9)
+#define MPT_IS_VHT_3S_RATE(_value)	(MPT_RATE_VHT3SS_MCS0 <= _value && _value <= MPT_RATE_VHT3SS_MCS9)
+#define MPT_IS_VHT_4S_RATE(_value)	(MPT_RATE_VHT4SS_MCS0 <= _value && _value <= MPT_RATE_VHT4SS_MCS9)
+
+#define MPT_IS_2SS_RATE(_rate) ((MPT_RATE_MCS8 <= _rate && _rate <= MPT_RATE_MCS15) || \
+	(MPT_RATE_VHT2SS_MCS0 <= _rate && _rate <= MPT_RATE_VHT2SS_MCS9))
+#define MPT_IS_3SS_RATE(_rate) ((MPT_RATE_MCS16 <= _rate && _rate <= MPT_RATE_MCS23) || \
+	(MPT_RATE_VHT3SS_MCS0 <= _rate && _rate <= MPT_RATE_VHT3SS_MCS9))
+#define MPT_IS_4SS_RATE(_rate) ((MPT_RATE_MCS24 <= _rate && _rate <= MPT_RATE_MCS31) || \
+	(MPT_RATE_VHT4SS_MCS0 <= _rate && _rate <= MPT_RATE_VHT4SS_MCS9))
+
+typedef enum _POWER_MODE_ {
+	POWER_LOW = 0,
+	POWER_NORMAL
+} POWER_MODE;
+
+/* The following enumeration is used to define the value of Reg0xD00[30:28] or JaguarReg0x914[18:16]. */
+typedef enum _OFDM_TX_MODE {
+	OFDM_ALL_OFF		= 0,
+	OFDM_ContinuousTx	= 1,
+	OFDM_SingleCarrier	= 2,
+	OFDM_SingleTone	= 4,
+} OFDM_TX_MODE;
+
+
+#define RX_PKT_BROADCAST	1
+#define RX_PKT_DEST_ADDR	2
+#define RX_PKT_PHY_MATCH	3
+
+typedef enum _ENCRY_CTRL_STATE_ {
+	HW_CONTROL,		/* hw encryption& decryption */
+	SW_CONTROL,		/* sw encryption& decryption */
+	HW_ENCRY_SW_DECRY,	/* hw encryption & sw decryption */
+	SW_ENCRY_HW_DECRY	/* sw encryption & hw decryption */
+} ENCRY_CTRL_STATE;
+
+typedef enum	_MPT_TXPWR_DEF {
+	MPT_CCK,
+	MPT_OFDM, /* L and HT OFDM */
+	MPT_OFDM_AND_HT,
+	MPT_HT,
+	MPT_VHT
+} MPT_TXPWR_DEF;
+
+
+#define IS_MPT_HT_RATE(_rate)			(_rate >= MPT_RATE_MCS0 && _rate <= MPT_RATE_MCS31)
+#define IS_MPT_VHT_RATE(_rate)			(_rate >= MPT_RATE_VHT1SS_MCS0 && _rate <= MPT_RATE_VHT4SS_MCS9)
+#define IS_MPT_CCK_RATE(_rate)			(_rate >= MPT_RATE_1M && _rate <= MPT_RATE_11M)
+#define IS_MPT_OFDM_RATE(_rate)			(_rate >= MPT_RATE_6M && _rate <= MPT_RATE_54M)
+
+typedef enum _mp_tx_pkt_payload{
+	MP_TX_Payload_00 = 0,
+	MP_TX_Payload_a5,
+	MP_TX_Payload_5a,
+	MP_TX_Payload_ff,
+	MP_TX_Payload_prbs9,
+	MP_TX_Payload_default_random
+} mp_tx_pkt_payload;
+
+/*************************************************************************/
+#if 0
+extern struct mp_xmit_frame *alloc_mp_xmitframe(struct mp_priv *pmp_priv);
+extern int free_mp_xmitframe(struct xmit_priv *pxmitpriv, struct mp_xmit_frame *pmp_xmitframe);
+#endif
+/* SYNC to PHL MP define*/
+/* mp command class */
+enum rtw_mp_class {
+	RTW_MP_CLASS_CONFIG = 0,
+	RTW_MP_CLASS_TX = 1,
+	RTW_MP_CLASS_RX = 2,
+	RTW_MP_CLASS_EFUSE = 3,
+	RTW_MP_CLASS_REG = 4,
+	RTW_MP_CLASS_TXPWR = 5,
+	RTW_MP_CLASS_CAL = 6,
+	RTW_MP_CLASS_FLASH = 7,
+	RTW_MP_CLASS_MAX,
+};
+
+enum rtw_mp_tx_method {
+	RTW_MP_SW_TX = 0,
+	RTW_MP_PMACT_TX,
+	RTW_MP_TMACT_TX,
+	RTW_MP_FW_PMACT_TX,
+};
+
+enum rtw_mp_tx_cmd {
+	RTW_MP_TX_NONE = 0,
+	RTW_MP_TX_PACKETS,
+	RTW_MP_TX_CONTINUOUS,
+	RTW_MP_TX_SINGLE_TONE,
+	RTW_MP_TX_CCK_Carrier_Suppression,
+	RTW_MP_TX_CONFIG_PLCP_COMMON_INFO,
+	RTW_MP_TX_CMD_PHY_OK,
+	RTW_MP_TX_CONFIG_PLCP_PATTERN,
+	RTW_MP_TX_CONFIG_PLCP_USER_INFO,
+	RTW_MP_TX_TB_TEST,
+	RTW_MP_TX_DPD_BYPASS,
+	RTW_MP_TX_CHECK_TX_IDLE,
+	RTW_MP_TX_CMD_BB_LOOPBCK,
+	RTW_MP_TX_CMD_MAX,
+};
+
+/* mp config command */
+enum rtw_mp_config_cmdid {
+	RTW_MP_CONFIG_CMD_GET_BW,
+	RTW_MP_CONFIG_CMD_GET_RF_STATUS,
+	RTW_MP_CONFIG_CMD_SET_RATE_IDX,
+	RTW_MP_CONFIG_CMD_SET_RF_TXRX_PATH,
+	RTW_MP_CONFIG_CMD_SET_RESET_PHY_COUNT,
+	RTW_MP_CONFIG_CMD_SET_RESET_MAC_COUNT,
+	RTW_MP_CONFIG_CMD_SET_RESET_DRV_COUNT,
+	RTW_MP_CONFIG_CMD_SET_TXRX_MODE,
+	RTW_MP_CONFIG_CMD_PBC,
+	RTW_MP_CONFIG_CMD_START_DUT,
+	RTW_MP_CONFIG_CMD_STOP_DUT,
+	RTW_MP_CONFIG_CMD_GET_MIMPO_RSSI,
+	RTW_MP_CONFIG_CMD_GET_BOARD_TYPE,
+	RTW_MP_CONFIG_CMD_GET_MODULATION,
+	RTW_MP_CONFIG_CMD_GET_RF_MODE,
+	RTW_MP_CONFIG_CMD_GET_RF_PATH,
+	RTW_MP_CONFIG_CMD_SET_MODULATION,
+	RTW_MP_CONFIG_CMD_GET_DEVICE_INFO,
+	RTW_MP_CONFIG_CMD_SET_PHY_INDEX,
+	RTW_MP_CONFIG_CMD_GET_MAC_ADDR,
+	RTW_MP_CONFIG_CMD_SET_MAC_ADDR,
+	RTW_MP_CONFIG_CMD_SET_CH_BW,
+	RTW_MP_CONFIG_CMD_GET_TX_NSS,
+	RTW_MP_CONFIG_CMD_GET_RX_NSS,
+	RTW_MP_CONFIG_CMD_SWITCH_BT_PATH,
+	RTW_MP_CONFIG_CMD_GET_RFE_TYPE,
+	RTW_MP_CONFIG_CMD_GET_DEV_IDX,
+	RTW_MP_CONFIG_CMD_TRIGGER_FW_CONFLICT,
+	RTW_MP_CONFIG_CMD_GET_UUID,
+	RTW_MP_CONFIG_CMD_SET_REGULATION,
+	RTW_MP_CONFIG_CMD_GET_DRV_VER,
+	RTW_MP_CONFIG_CMD_SET_BT_UART,
+	RTW_MP_CONFIG_CMD_SWITCH_ANTENNA,
+	RTW_MP_CONFIG_CMD_SET_MAC_LOOPBK_ENTER,
+	RTW_MP_CONFIG_CMD_SET_HCI_SPEED,
+	RTW_MP_CONFIG_CMD_GET_HCI_SPEED,
+	RTW_MP_CONFIG_CMD_SET_MAC_GENERNAL_IO_TEST,
+	RTW_MP_CONFIG_CMD_SET_MAC_L1SS_ENABLE,
+	RTW_MP_CONFIG_CMD_SET_MAC_ASPM_STATE,
+	RTW_MP_CONFIG_CMD_SET_GPIO,
+	RTW_MP_CONFIG_CMD_SET_MAC_LOOPBK_SPEED,
+	RTW_MP_CONFIG_CMD_MAX,
+};
+
+/* mp rx command */
+enum rtw_mp_rx_cmd {
+	RTW_MP_RX_CMD_PHY_CRC_OK = 0,
+	RTW_MP_RX_CMD_PHY_CRC_ERR = 1,
+	RTW_MP_RX_CMD_MAC_CRC_OK = 2,
+	RTW_MP_RX_CMD_MAC_CRC_ERR = 3,
+	RTW_MP_RX_CMD_DRV_CRC_OK = 4,
+	RTW_MP_RX_CMD_DRV_CRC_ERR = 5,
+	RTW_MP_RX_CMD_GET_RSSI = 6,
+	RTW_MP_RX_CMD_GET_RXEVM = 7,
+	RTW_MP_RX_CMD_GET_PHYSTS = 8,
+	RTW_MP_RX_CMD_TRIGGER_RXEVM = 9,
+	RTW_MP_RX_CMD_SET_GAIN_OFFSET = 10,
+	RTW_MP_RX_CMD_GET_RSSI_EX = 11,
+	RTW_MP_RX_CMD_SET_RX_FLTR = 12,
+	RTW_MP_RX_CMD_MAX,
+
+};
+
+/* mp reg command */
+enum rtw_mp_reg_cmd {
+	RTW_MP_REG_CMD_READ_MAC = 0,
+	RTW_MP_REG_CMD_WRITE_MAC = 1,
+	RTW_MP_REG_CMD_READ_RF = 2,
+	RTW_MP_REG_CMD_WRITE_RF = 3,
+	RTW_MP_REG_CMD_READ_SYN = 4,
+	RTW_MP_REG_CMD_WRITE_SYN = 5,
+	RTW_MP_REG_CMD_READ_BB = 6,
+	RTW_MP_REG_CMD_WRITE_BB = 7,
+	RTW_MP_REG_CMD_SET_XCAP = 8,
+	RTW_MP_REG_CMD_GET_XCAP = 9,
+	RTW_MP_REG_CMD_MAX,
+};
+
+/* mp tx power command */
+enum rtw_mp_txpwr_cmd {
+	RTW_MP_TXPWR_CMD_READ_PWR_TABLE = 0,
+	RTW_MP_TXPWR_CMD_GET_PWR_TRACK_STATUS = 1,
+	RTW_MP_TXPWR_CMD_SET_PWR_TRACK_STATUS = 2,
+	RTW_MP_TXPWR_CMD_SET_TXPWR = 3,
+	RTW_MP_TXPWR_CMD_GET_TXPWR = 4,
+	RTW_MP_TXPWR_CMD_GET_TXPWR_INDEX = 5,
+	RTW_MP_TXPWR_CMD_GET_THERMAL = 6,
+	RTW_MP_TXPWR_CMD_GET_TSSI = 7,
+	RTW_MP_TXPWR_CMD_SET_TSSI = 8,
+	RTW_MP_TXPWR_CMD_GET_TXPWR_REF = 9,
+	RTW_MP_TXPWR_CMD_GET_TXPWR_REF_CW = 10,
+	RTW_MP_TXPWR_CMD_SET_TXPWR_INDEX = 11,
+	RTW_MP_TXPWR_CMD_GET_TXINFOPWR = 12,
+	RTW_MP_TXPWR_CMD_SET_RFMODE = 13,
+	RTW_MP_TXPWR_CMD_SET_TSSI_OFFSET = 14,
+	RTW_MP_TXPWR_CMD_GET_ONLINE_TSSI_DE = 15,
+	RTW_MP_TXPWR_CMD_SET_PWR_LMT_EN = 16,
+	RTW_MP_TXPWR_CMD_GET_PWR_LMT_EN = 17,
+	RTW_MP_TXPWR_CMD_SET_TX_POW_PATTERN_SHARP = 18,
+	RTW_MP_TXPWR_CMD_SET_TX_POW_TABLE_SWITCH = 19,
+	RTW_MP_TXPWR_CMD_MAX,
+};
+
+/*
+enum rtw_mp_flash_cmd {
+	RTW_MP_FLASH_CMD_WIFI_READ,
+	RTW_MP_FLASH_CMD_WIFI_WRITE,
+	RTW_MP_FLASH_CMD_MAX,
+};
+*/
+
+enum rtw_mp_cal_cmd {
+	RTW_MP_CAL_CMD_TRIGGER_CAL = 0,
+	RTW_MP_CAL_CMD_SET_CAPABILITY_CAL = 1,
+	RTW_MP_CAL_CMD_GET_CAPABILITY_CAL = 2,
+	RTW_MP_CAL_CMD_GET_TSSI_DE_VALUE = 3,
+	RTW_MP_CAL_CMD_SET_TSSI_DE_TX_VERIFY = 4,
+	RTW_MP_CAL_CMD_GET_TXPWR_FINAL_ABS = 5,
+	RTW_MP_CAL_CMD_TRIGGER_DPK_TRACKING = 6,
+	RTW_MP_CAL_CMD_SET_TSSI_AVG = 7,
+	RTW_MP_CAL_CMD_PSD_INIT = 8,
+	RTW_MP_CAL_CMD_PSD_RESTORE = 9,
+	RTW_MP_CAL_CMD_PSD_GET_POINT_DATA = 10,
+	RTW_MP_CAL_CMD_PSD_QUERY = 11,
+	RTW_MP_CAL_CMD_EVENT_TRIGGER = 12,
+	RTW_MP_CAL_CMD_TRIGGER_WATCHDOG_CAL = 13,
+	RTW_MP_CAL_CMD_MAX,
+};
+
+enum rtw_mp_pmac_mode {
+	RTW_MP_PMAC_NONE_TEST,
+	RTW_MP_PMAC_PKTS_TX,
+	RTW_MP_PMAC_PKTS_RX,
+	RTW_MP_PMAC_CONT_TX,
+	RTW_MP_PMAC_FW_TRIG_TX,
+	RTW_MP_PMAC_OFDM_SINGLE_TONE_TX,
+	RTW_MP_PMAC_CCK_CARRIER_SIPPRESSION_TX
+};
+
+enum rtw_mp_calibration_type {
+	RTW_MP_CAL_CHL_RFK = 0,
+	RTW_MP_CAL_DACK = 1,
+	RTW_MP_CAL_IQK = 2,
+	RTW_MP_CAL_LCK = 3,
+	RTW_MP_CAL_DPK = 4,
+	RTW_MP_CAL_DPK_TRACK = 5,
+	RTW_MP_CAL_TSSI = 6,
+	RTW_MP_CAL_GAPK = 7,
+	RTW_MP_CAL_MAX,
+};
+
+enum rtw_mp_ppdu_type {
+	RTW_MP_TYPE_CCK			= 0,
+	RTW_MP_TYPE_LEGACY,
+	RTW_MP_TYPE_HT_MF,
+	RTW_MP_TYPE_HT_GF,
+	RTW_MP_TYPE_VHT,
+	RTW_MP_TYPE_HE_SU,
+	RTW_MP_TYPE_HE_ER_SU,
+	RTW_MP_TYPE_HE_MU_OFDMA,
+	RTW_MP_TYPE_HE_TB
+};
+
+typedef enum _mp_ant_path {
+	MP_ANTENNA_NONE	= 0,
+	MP_ANTENNA_D	= 1,
+	MP_ANTENNA_C	= 2,
+	MP_ANTENNA_CD	= 3,
+	MP_ANTENNA_B	= 4,
+	MP_ANTENNA_BD	= 5,
+	MP_ANTENNA_BC	= 6,
+	MP_ANTENNA_BCD	= 7,
+	MP_ANTENNA_A	= 8,
+	MP_ANTENNA_AD	= 9,
+	MP_ANTENNA_AC	= 10,
+	MP_ANTENNA_ACD	= 11,
+	MP_ANTENNA_AB	= 12,
+	MP_ANTENNA_ABD	= 13,
+	MP_ANTENNA_ABC	= 14,
+	MP_ANTENNA_ABCD	= 15
+} mp_ant_path;
+
+struct rtw_mp_mac_lbk_tx_rpt {
+	u32 total_cnt;
+	u32 idle_cnt;
+	u32 busy_cnt;
+};
+
+struct rtw_gpio_config_arg {
+	u8 gpio_mode;
+	u8 gpio_id;
+	u8 gpio_enable;
+};
+#ifdef CONFIG_POWER_SAVE
+struct rtw_pwr_config_arg {
+	u8 pwr_state;
+	u8 pwr_lvl;
+};
+#endif
+struct rtw_mp_cmd_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+};
+
+#define RTW_MP_TEST_NAME_LEN		32
+#define RTW_MP_TEST_RPT_RSN_LEN	32
+
+struct rtw_mp_test_rpt {
+	char name[RTW_MP_TEST_NAME_LEN];
+	u8 status;
+	char rsn[RTW_MP_TEST_RPT_RSN_LEN];
+	u32 total_time; // in ms
+};
+
+
+
+struct rtw_mp_config_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+	u8 channel;
+	u8 bandwidth;
+	u8 rate_idx;
+	u8 ant_tx;
+	u8 tx_rfpath;
+	u8 ant_rx;
+	u8 rx_rfpath;
+	u8 rf_path;
+	u8 get_rfstats;
+	u8 modulation;
+	u8 bustype;
+	u32 chipid;
+	u8 cur_phy;
+	u8 mac_addr[6];
+	u8 sc_idx;
+	u8 dbcc_en;
+	u8 btc_mode;
+	u8 rfe_type;
+	u8 dev_id;
+	u32 offset;
+	u8 voltag;
+	u8 band;
+	u32 uuid;
+	u8 regulation;
+	u8 frc_switch;
+	u8 is_tmac_mode;
+	u32 drv_ver;
+	u8 phy_idx;
+	u8 is_bt_uart;
+	u8 ant_sw;
+	u8 hci_speed;
+	struct rtw_gpio_config_arg gpio_cfg;
+#ifdef CONFIG_POWER_SAVE
+	struct rtw_pwr_config_arg pwr_cfg;
+#endif
+		u8 en_phy;
+};
+
+struct rtw_mp_tx_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+	u8 tx_method;
+	u8 plcp_ppdu_type;	/*offline gen*/
+	u16 plcp_case_id;	/*offline gen*/
+	u8 bCarrierSuppression;
+	u8 is_cck;
+	u8 start_tx;
+	u16 tx_cnt;
+	u16 period;		/* us */
+	u16 tx_time;	/* us */
+	u32 tx_ok;
+	u8 tx_path;
+	u8 tx_mode;		/* mode: 0 = tmac, 1 = pmac */
+	u8 tx_concurrent_en;	/* concurrent tx */
+	u8 phy_idx;
+	u8 dpd_bypass;
+	/* plcp info */
+	u32 dbw; /*0:BW20, 1:BW40, 2:BW80, 3:BW160/BW80+80*/
+	u32 source_gen_mode;
+	u32 locked_clk;
+	u32 dyn_bw;
+	u32 ndp_en;
+	u32 long_preamble_en; /*bmode*/
+	u32 stbc;
+	u32 gi; /*0:0.4,1:0.8,2:1.6,3:3.2*/
+	u32 tb_l_len;
+	u32 tb_ru_tot_sts_max;
+	u32 vht_txop_not_allowed;
+	u32 tb_disam;
+	u32 doppler;
+	u32 he_ltf_type; /*0:1x,1:2x,2:4x*/
+	u32 ht_l_len;
+	u32 preamble_puncture;
+	u32 he_mcs_sigb;/*0~5*/
+	u32 he_dcm_sigb;
+	u32 he_sigb_compress_en;
+	u32 max_tx_time_0p4us;
+	u32 ul_flag;
+	u32 tb_ldpc_extra;
+	u32 bss_color;
+	u32 sr;
+	u32 beamchange_en;
+	u32 he_er_u106ru_en;
+	u32 ul_srp1;
+	u32 ul_srp2;
+	u32 ul_srp3;
+	u32 ul_srp4;
+	u32 mode;
+	u32 group_id;
+	u32 ppdu_type;/*0: bmode,1:Legacy,2:HT_MF,3:HT_GF,4:VHT,5:HE_SU,6:HE_ER_SU,7:HE_MU,8:HE_TB*/
+	u32 txop;
+	u32 tb_strt_sts;
+	u32 tb_pre_fec_padding_factor;
+	u32 cbw;
+	u32 txsc;
+	u32 tb_mumimo_mode_en;
+	u32 nominal_t_pe; /* def = 2*/
+	u32 ness; /* def = 0*/
+	u32 n_user;
+	u32 tb_rsvd;/*def = 0*/
+	/* plcp user info */
+	u32 plcp_usr_idx;
+	u32 mcs;
+	u32 mpdu_len;
+	u32 n_mpdu;
+	u32 fec;
+	u32 dcm;
+	u32 aid;
+	u32 scrambler_seed; /* rand (1~255)*/
+	u32 random_init_seed; /* rand (1~255)*/
+	u32 apep;
+	u32 ru_alloc;
+	u32 nss;
+	u32 txbf;
+	u32 pwr_boost_db;
+	//struct mp_plcp_param_t plcp_param;	/*online gen*/
+	u32 data_rate;
+	u8 plcp_sts;
+
+	/*HE-TB Test*/
+	u8 bSS_id_addr0;
+	u8 bSS_id_addr1;
+	u8 bSS_id_addr2;
+	u8 bSS_id_addr3;
+	u8 bSS_id_addr4;
+	u8 bSS_id_addr5;
+	u8 is_link_mode;
+
+	/* tx state*/
+	u8 tx_state;
+
+	/* bb loop back*/
+	u8 enable;
+	u8 is_dgt;
+	u8 cck_lbk_en;
+	u8 is_bt_link;
+
+	u32 puncture;
+	/* txsb */
+	u32 txsb;
+	u32 eht_mcs_sig;
+
+	/* sw tx*/
+	u8 mac_addr_0;
+	u8 mac_addr_1;
+	u8 mac_addr_2;
+	u8 mac_addr_3;
+	u8 mac_addr_4;
+	u8 mac_addr_5;
+	u32 sw_tx_payload_size;
+
+	/* ampdu control */
+	u8 ampdu_num;
+	u8 sw_tx_en;
+
+	/* mac loop back */
+	struct rtw_mp_mac_lbk_tx_rpt tx_rpt;
+};
+
+struct rtw_mp_rx_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+	u32 rx_ok;
+	u32 rx_err;
+	u8 rssi;
+	u8 rx_path;
+	u8 rx_evm;
+	u8 user;
+	u8 strm;
+	u8 rxevm_table;
+	u8 enable;
+	u32 phy_user0_rxevm;
+	u32 phy_user1_rxevm;
+	u32 phy_user2_rxevm;
+	u32 phy_user3_rxevm;
+	s8 offset;
+	u8 rf_path;
+	u8 iscck;
+	s32 rssi_ex[4];
+	u8 rx_phy_idx;
+	u8 rx_fltr_addr[6];
+	u8 rx_fltr_enable;
+};
+
+enum rtw_mp_tssi_pwrtrk_type{
+	RTW_MP_TSSI_OFF = 0,
+	RTW_MP_TSSI_ON,
+	RTW_MP_TSSI_CAL
+};
+
+
+struct rtw_mp_reg_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+	u32 io_offset;
+	u32 io_value;
+	u8 io_type;
+	u8 ofdm;
+	u8 rfpath;
+	u8 sc_xo;
+	u8 xsi_offset;
+	u8 xsi_value;
+};
+
+struct rtw_mp_txpwr_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+	s16 txpwr;
+	u16 txpwr_index;
+	u8 txpwr_track_status;
+	u8 txpwr_status;
+	u32 tssi;
+	u8 thermal;
+	u8 rfpath;
+	u8 ofdm;
+	u8 tx_path;
+	u16 rate;
+	u8 bandwidth;
+	u8 channel;
+	s16 table_item; /*get an element of power table*/
+	u8 dcm;
+	u8 beamforming;
+	u8 offset;
+	s16 txpwr_ref;
+	u8 is_cck;
+	u8 rf_mode;
+	u32 tssi_de_offset;
+	s32 dbm;
+	s32 pout;
+	s32 online_tssi_de;
+	bool pwr_lmt_en;
+	u8 sharp_id;
+	u8 cur_phy;
+};
+
+struct rtw_mp_cal_arg {
+	u8 mp_class;
+	u8 cmd;
+	u8 cmd_ok;
+	u8 status;
+	u8 cal_type;
+	u8 enable;
+	u8 rfpath;
+	u16 io_value;
+	u8 channel;
+	u8 bandwidth;
+	s32 xdbm;
+	u8 path;
+	u8 iq_path;
+	u32 avg;
+	u32 fft;
+	s32 point;
+	u32 upoint;
+	u32 start_point;
+	u32 stop_point;
+	u32 buf;
+	u32 outbuf[400];
+	u8 event;
+	u8 func;
+};
+
+enum RTW_TEST_SUB_MODULE {
+	RTW_TEST_SUB_MODULE_MP = 0,
+	RTW_TEST_SUB_MODULE_FPGA = 1,
+	RTW_TEST_SUB_MODULE_VERIFY = 2,
+	RTW_TEST_SUB_MODULE_TOOL = 3,
+	RTW_TEST_SUB_MODULE_TRX = 4,
+	RTW_TEST_SUB_MODULE_UNKNOWN,
+};
+
+struct rtw_test_module_info {
+	u8 tm_type;
+	u8 tm_mode;
+};
+
+#define RTW_MAX_TEST_CMD_BUF 2000
+struct rtw_mp_test_cmdbuf {
+	u8 type;
+	u16 len;
+	u8 buf[RTW_MAX_TEST_CMD_BUF];
+};
+
+enum rtw_mp_nss
+ {
+	MP_NSS1,
+	MP_NSS2,
+	MP_NSS3,
+	MP_NSS4
+ };
+
+#define RU_TONE_STR(idx)\
+	(idx == MP_RU_TONE_26) ? "26-Tone" :\
+	(idx == MP_RU_TONE_52) ? "52-Tone" :\
+	(idx == MP_RU_TONE_106) ? "106-Tone" :\
+	(idx == MP_RU_TONE_242) ? "242-Tone" :\
+	(idx == MP_RU_TONE_484) ? "484-Tone" :\
+	(idx == MP_RU_TONE_996) ? "996-Tone" :\
+	"UNknow"
+
+enum rtw_mp_resourceUnit
+{
+	MP_RU_TONE_26,
+	MP_RU_TONE_52,
+	MP_RU_TONE_106,
+	MP_RU_TONE_242,
+	MP_RU_TONE_484,
+	MP_RU_TONE_996,
+	MP_RU_TONE_996X2,
+	MP_RU_TONE_HESIGB,
+	MP_RU_TONE_996X4,
+	MP_RU_TONE_52_26,
+	MP_RU_TONE_106_26,
+	MP_RU_TONE_484_242,
+	MP_RU_TONE_996_484,
+	MP_RU_TONE_996_484_242,
+	MP_RU_TONE_996X2_484,
+	MP_RU_TONE_996X3,
+	MP_RU_TONE_996X3_484
+};
+/* SYNC to PHL MP define END */
+
+
+
+#define MP_IS_HT_HRATE(_rate)	((_rate) >= HRATE_MCS0 && (_rate) <= HRATE_MCS31)
+#define MP_IS_VHT_HRATE(_rate)	((_rate) >= HRATE_VHT_NSS1_MCS0 && (_rate) <= HRATE_VHT_NSS4_MCS9)
+#define MP_IS_CCK_HRATE(_rate)	((_rate) == HRATE_CCK1 || (_rate) == HRATE_CCK2 || \
+								 (_rate) == HRATE_CCK5_5 || (_rate) == HRATE_CCK11)
+
+#define MP_IS_OFDM_HRATE(_rate)	((_rate) >= HRATE_OFDM6 && (_rate) <= HRATE_OFDM54)
+#define MP_IS_HE_HRATE(_rate)	((_rate) >= HRATE_HE_NSS1_MCS0 && (_rate) <= HRATE_HE_NSS4_MCS11)
+
+#define MP_IS_HT1SS_HRATE(_rate) ((_rate) >= HRATE_MCS0 && (_rate) <= HRATE_MCS7)
+#define MP_IS_HT2SS_HRATE(_rate) ((_rate) >= HRATE_MCS8 && (_rate) <= HRATE_MCS15)
+#define MP_IS_HT3SS_HRATE(_rate) ((_rate) >= HRATE_MCS16 && (_rate) <= HRATE_MCS23)
+#define MP_IS_HT4SS_HRATE(_rate) ((_rate) >= HRATE_MCS24 && (_rate) <= HRATE_MCS31)
+
+#define MP_IS_VHT1SS_HRATE(_rate) ((_rate) >= HRATE_VHT_NSS1_MCS0 && (_rate) <= HRATE_VHT_NSS1_MCS9)
+#define MP_IS_VHT2SS_HRATE(_rate) ((_rate) >= HRATE_VHT_NSS2_MCS0 && (_rate) <= HRATE_VHT_NSS2_MCS9)
+#define MP_IS_VHT3SS_HRATE(_rate) ((_rate) >= HRATE_VHT_NSS3_MCS0 && (_rate) <= HRATE_VHT_NSS3_MCS9)
+#define MP_IS_VHT4SS_HRATE(_rate) ((_rate) >= HRATE_VHT_NSS4_MCS0 && (_rate) <= HRATE_VHT_NSS4_MCS9)
+
+#define MP_IS_HE1SS_HRATE(_rate) ((_rate) >= HRATE_HE_NSS1_MCS0 && (_rate) <= HRATE_HE_NSS1_MCS11)
+#define MP_IS_HE2SS_HRATE(_rate) ((_rate) >= HRATE_HE_NSS2_MCS0 && (_rate) <= HRATE_HE_NSS2_MCS11)
+#define MP_IS_HE3SS_HRATE(_rate) ((_rate) >= HRATE_HE_NSS3_MCS0 && (_rate) <= HRATE_HE_NSS3_MCS11)
+#define MP_IS_HE4SS_HRATE(_rate) ((_rate) >= HRATE_HE_NSS4_MCS0 && (_rate) <= HRATE_HE_NSS4_MCS11)
+
+#define MP_IS_1T_HRATE(_rate)	(MP_IS_CCK_HRATE((_rate)) || MP_IS_OFDM_HRATE((_rate)) \
+								|| MP_IS_HT1SS_HRATE((_rate)) || MP_IS_VHT1SS_HRATE((_rate)) \
+								|| MP_IS_HE1SS_HRATE((_rate)))
+
+#define MP_IS_2T_HRATE(_rate)	(MP_IS_HT2SS_HRATE((_rate)) || MP_IS_VHT2SS_HRATE((_rate)) \
+								|| MP_IS_HE2SS_HRATE((_rate)))
+
+#define MP_IS_3T_HRATE(_rate)	(MP_IS_HT3SS_HRATE((_rate)) || MP_IS_VHT3SS_HRATE((_rate)) \
+								|| MP_IS_HE3SS_HRATE((_rate)))
+
+#define MP_IS_4T_HRATE(_rate)	(MP_IS_HT4SS_HRATE((_rate)) || MP_IS_VHT4SS_HRATE((_rate)) \
+								|| MP_IS_HE4SS_HRATE((_rate)))
+
+
+
+void rtw_mp_get_phl_cmd(_adapter *padapter, void* buf, u32 buflen);
+void rtw_mp_set_phl_cmd(_adapter *padapter, void* buf, u32 buflen);
+
+bool rtw_mp_phl_config_arg(_adapter *padapter, enum rtw_mp_config_cmdid cmdid);
+void rtw_mp_phl_rx_physts(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg, bool bstart);
+void rtw_mp_phl_rx_rssi(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg);
+void rtw_mp_phl_rx_gain_offset(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg, u8 path_num);
+void rtw_mp_phl_query_rx(_adapter *padapter, struct rtw_mp_rx_arg *rx_arg ,u8 rx_qurey_type);
+u8 rtw_mp_phl_txpower(_adapter *padapter, struct rtw_mp_txpwr_arg	*ptxpwr_arg, u8 cmdid);
+void rtw_mp_set_crystal_cap(_adapter *padapter, u32 xcapvalue);
+u8 rtw_mp_phl_calibration(_adapter *padapter, struct rtw_mp_cal_arg	*pcal_arg, u8 cmdid);
+u8 rtw_mp_phl_reg(_adapter *padapter, struct rtw_mp_reg_arg	*reg_arg, u8 cmdid);
+u8 rtw_mp_phl_rx_cal_cmd(_adapter *padapter);
+
+
+u8 rtw_update_giltf(_adapter *padapter);
+void rtw_mp_update_coding(_adapter *padapter);
+u8 rtw_mp_update_ru_tone(_adapter *padapter);
+u8 rtw_mp_update_ru_alloc(_adapter *padapter);
+
+bool rtw_mp_is_cck_rate(u16 rate);
+
+extern s32 init_mp_priv(_adapter *padapter);
+extern s32 MPT_InitializeAdapter(_adapter *padapter, u8 Channel);
+extern void MPT_DeInitAdapter(_adapter *padapter);
+extern s32 mp_start_test(_adapter *padapter);
+extern void mp_stop_test(_adapter *padapter);
+
+
+extern void write_bbreg(_adapter *padapter, u32 addr, u32 bitmask, u32 val);
+extern u32 read_rfreg(_adapter *padapter, u8 rfpath, u32 addr);
+extern void write_rfreg(_adapter *padapter, u8 rfpath, u32 addr, u32 val);
+#ifdef CONFIG_ANTENNA_DIVERSITY
+u8 rtw_mp_set_antdiv(_adapter *padapter, BOOLEAN bMain);
+#endif
+void	SetChannel(_adapter *adapter);
+void	SetBandwidth(_adapter *adapter);
+
+u16 rtw_mp_txpower_dbm(_adapter *adapter, u8 rf_path);
+
+void	SetAntenna(_adapter *adapter);
+void	SetDataRate(_adapter *adapter);
+
+
+u8 rtw_mp_rfpath2txnss(_adapter *padapter, enum rf_path path);
+
+void	GetThermalMeter(_adapter *adapter, u8 rfpath ,u8 *value);
+void	GetUuid(_adapter *adapter, u32 *uuid);
+void SetGpio(_adapter *padapter);
+void	rtw_mp_continuous_tx(_adapter *adapter, u8 bstart);
+void	rtw_mp_singlecarrier_tx(_adapter *adapter, u8 bstart);
+void	rtw_mp_singletone_tx(_adapter *adapter, u8 bstart);
+void	rtw_mp_carriersuppr_tx(_adapter *adapter, u8 bstart);
+void	rtw_mp_txpwr_level(_adapter *adapter);
+
+void	rtw_set_phl_packet_tx(_adapter *padapter, u8 bStart);
+void rtw_pre_phl_packet_tx(_adapter *padapter, u8 bStart);
+
+u8	rtw_phl_mp_tx_cmd(_adapter *padapter, enum rtw_mp_tx_cmd cmdid,
+						enum rtw_mp_tx_method tx_method, boolean bstart);
+
+void	rtw_mp_set_packet_tx(_adapter *padapter);
+void	rtw_mp_reset_phy_count(_adapter *adapter);
+
+u32	mp_query_psd(_adapter *adapter, u8 *data);
+void	rtw_mp_trigger_iqk(_adapter *padapter);
+void	rtw_mp_trigger_lck(_adapter *padapter);
+void	rtw_mp_trigger_dpk(_adapter *padapter);
+u8 rtw_mp_mode_check(_adapter *padapter);
+
+
+void mpt_ProSetPMacTx(_adapter *adapter);
+void rtw_mp_set_rfpath_switch(_adapter *adapter);
+
+u8 mpt_to_mgnt_rate(u32	MptRateIdx);
+u16 rtw_mp_rate_parse(_adapter *adapter, u8 *target_str);
+u32 mp_join(_adapter *padapter, u8 mode);
+u32 hal_mpt_query_phytxok(_adapter *adapter);
+u32 rtw_mpt_raw2dec_dbm(u32 val);
+u32 mpt_get_tx_power_finalabs_val(_adapter *padapter, u8 rf_path);
+void mpt_trigger_tssi_tracking(_adapter *adapter, u8 rf_path);
+u8 rtw_mpt_set_power_limit_en(_adapter *padapter, bool en_val);
+bool rtw_mpt_get_power_limit_en(_adapter *padapter);
+
+u32 rtw_mp_get_tssi_de(_adapter *padapter, u8 rf_path);
+s32 rtw_mp_get_online_tssi_de(_adapter *padapter, s32 out_pwr, s32 tgdbm, u8 rf_path);
+u8 rtw_mp_set_tsside2verify(_adapter *padapter, u32 tssi_de, u8 rf_path);
+
+u8 rtw_mp_set_tssi_pwrtrk(_adapter *padapter, u8 tssi_state);
+u8 rtw_mp_get_tssi_pwrtrk(_adapter *padapter);
+u8 rtw_mp_set_tx_shape_idx(_adapter *padapter);
+
+void rtw_mp_cal_trigger(_adapter *padapter, u8 cal_tye);
+void rtw_mp_cal_capab(_adapter *padapter, u8 cal_tye, u8 benable);
+
+void rtw_mp_rx_phl_cal_timer(_adapter *padapter);
+
+void rtw_mp_phl_set_mac_loopbk(_adapter *padapter);
+void rtw_mp_phl_set_mac_loopbk_speed(_adapter *padapter);
+void rtw_mp_phl_set_mac_io_test(_adapter *padapter);
+
+thread_return mp_xmit_phl_packet_thread(thread_context context);
+
+void
+PMAC_Get_Pkt_Param(
+	PRT_PMAC_TX_INFO	pPMacTxInfo,
+	PRT_PMAC_PKT_INFO	pPMacPktInfo
+);
+void
+CCK_generator(
+	PRT_PMAC_TX_INFO	pPMacTxInfo,
+	PRT_PMAC_PKT_INFO	pPMacPktInfo
+);
+void
+PMAC_Nsym_generator(
+	PRT_PMAC_TX_INFO	pPMacTxInfo,
+	PRT_PMAC_PKT_INFO	pPMacPktInfo
+);
+void
+L_SIG_generator(
+	u32	N_SYM,		/* Max: 750*/
+	PRT_PMAC_TX_INFO	pPMacTxInfo,
+	PRT_PMAC_PKT_INFO	pPMacPktInfo
+);
+
+void HT_SIG_generator(
+	PRT_PMAC_TX_INFO	pPMacTxInfo,
+	PRT_PMAC_PKT_INFO	pPMacPktInfo);
+
+void VHT_SIG_A_generator(
+	PRT_PMAC_TX_INFO	pPMacTxInfo,
+	PRT_PMAC_PKT_INFO	pPMacPktInfo);
+
+void VHT_SIG_B_generator(
+	PRT_PMAC_TX_INFO	pPMacTxInfo);
+
+void VHT_Delimiter_generator(
+	PRT_PMAC_TX_INFO	pPMacTxInfo);
+
+u8 rtw_do_mp_iwdata_len_chk(const char *caller, u32 len);
+
+int rtw_mp_write_reg(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_read_reg(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_write_rf(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_read_rf(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_start(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_stop(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_rate(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_channel(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_trxsc_offset(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_bandwidth(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_txpower_index(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_txpower(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_ant_tx(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_ant_rx(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_set_ctx_destAddr(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_ctx(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_disable_bt_coexist(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_arx(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_trx_query(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_pwrtrk(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_psd(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_thermal(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_UUID(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_reset_stats(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_dump(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_phypara(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_SetRFPath(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_switch_rf_path(struct net_device *dev,
+			struct iw_request_info *info,
+			struct iw_point *wrqu, char *extra);
+int rtw_mp_link(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_QueryDrv(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_PwrCtlDM(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_getver(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_mon(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_pwrlmt(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_dpk_track(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wrqu, char *extra);
+int rtw_mp_dpk(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wrqu, char *extra);
+int rtw_mp_mac_loopbk(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+int rtw_mp_mac_iotest(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+
+int rtw_mp_SetBT(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_pretx_proc(_adapter *padapter, u8 bStartTest, char *extra);
+int rtw_mp_tx(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_rx(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+int rtw_mp_hwtx(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra);
+u8 rtw_mp_hwrate2mptrate(u8 rate);
+int rtw_mp_iqk(struct net_device *dev,
+		 struct iw_request_info *info,
+		 struct iw_point *wrqu, char *extra);
+int rtw_mp_lck(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_get_tsside(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+int rtw_mp_set_tsside(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra);
+
+int rtw_priv_mp_set(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wdata, char *extra);
+
+int rtw_priv_mp_get(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wdata, char *extra);
+
+int rtw_mp_set_phl_io(struct net_device *dev,
+			 struct iw_request_info *info,
+			 struct iw_point *wrqu, char *extra);
+
+int rtw_mp_get_phl_io(struct net_device *dev,
+			 struct iw_request_info *info,
+			 struct iw_point *wrqu, char *extra);
+
+int rtw_mp_tx_pattern_idx(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+
+int rtw_mp_tx_plcp_tx_data(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+
+int rtw_mp_tx_plcp_tx_user(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+
+int rtw_mp_tx_method(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+
+int rtw_mp_config_phy(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+
+int rtw_mp_phl_rfk(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+int rtw_mp_phl_btc_path(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+int rtw_mp_get_he(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+int rtw_mp_band(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra);
+void rtw_mp_phl_rx_reset_fltr(_adapter *padapter,
+			struct rtw_mp_rx_arg *rx_arg,
+			bool bstart);
+#endif /* _RTW_MP_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_p2p.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_p2p.h
new file mode 100644
index 000000000000..91e1a4c3fdcd
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_p2p.h
@@ -0,0 +1,94 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_P2P_H_
+#define __RTW_P2P_H_
+
+
+u32 build_prov_disc_request_p2p_ie(struct wifidirect_info *pwdinfo, u8 *pbuf, u8 *pssid, u8 ussidlen, u8 *pdev_raddr);
+#ifdef CONFIG_WFD
+int rtw_init_wifi_display_info(_adapter *padapter);
+void rtw_wfd_enable(_adapter *adapter, bool on);
+void rtw_wfd_set_ctrl_port(_adapter *adapter, u16 port);
+void rtw_tdls_wfd_enable(_adapter *adapter, bool on);
+
+u32 build_probe_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_probe_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf, u8 tunneled);
+u32 build_beacon_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_nego_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_nego_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_nego_confirm_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_invitation_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_invitation_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_assoc_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_assoc_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_provdisc_req_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+u32 build_provdisc_resp_wfd_ie(struct wifidirect_info *pwdinfo, u8 *pbuf);
+
+u32 rtw_append_beacon_wfd_ie(_adapter *adapter, u8 *pbuf);
+u32 rtw_append_probe_req_wfd_ie(_adapter *adapter, u8 *pbuf);
+u32 rtw_append_probe_resp_wfd_ie(_adapter *adapter, u8 *pbuf);
+u32 rtw_append_assoc_req_wfd_ie(_adapter *adapter, u8 *pbuf);
+u32 rtw_append_assoc_resp_wfd_ie(_adapter *adapter, u8 *pbuf);
+#endif /*CONFIG_WFD */
+
+void rtw_xframe_chk_wfd_ie(struct xmit_frame *xframe);
+
+u32 process_probe_req_p2p_ie(struct wifidirect_info *pwdinfo, u8 *pframe, uint len);
+u32 process_assoc_req_p2p_ie(struct wifidirect_info *pwdinfo, u8 *pframe, uint len, struct sta_info *psta);
+u8 process_p2p_presence_req(struct wifidirect_info *pwdinfo, u8 *pframe, uint len);
+int process_p2p_cross_connect_ie(_adapter *padapter, u8 *IEs, u32 IELength);
+
+#ifdef CONFIG_P2P_PS
+void	process_p2p_ps_ie(_adapter *padapter, u8 *IEs, u32 IELength);
+void	p2p_ps_wk_hdl(_adapter *padapter, u8 p2p_ps_state);
+u8	p2p_ps_wk_cmd(_adapter *padapter, u8 p2p_ps_state, u8 enqueue);
+u8 *rtw_append_p2p_go_noa_ie(struct _ADAPTER *adapter, u8 *frame, u32 *len);
+void rtw_core_register_p2pps_ops(struct dvobj_priv *dvobj);
+void rtw_append_probe_resp_p2p_go_noa(struct xmit_frame *xframe);
+#endif /* CONFIG_P2P_PS */
+#ifdef CONFIG_P2P
+void rtw_append_probe_resp_p2p_ie(struct xmit_frame *xframe);
+void rtw_append_probe_resp_vendor_ie(struct xmit_frame *xframe);
+#endif
+#ifdef CONFIG_IOCTL_CFG80211
+int rtw_p2p_check_frames(_adapter *padapter, const u8 *buf, u32 len, u8 tx);
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+void reset_global_wifidirect_info(_adapter *padapter);
+void init_wifidirect_info(_adapter *padapter, enum P2P_ROLE role);
+int rtw_p2p_enable(_adapter *padapter, enum P2P_ROLE role);
+
+void _rtw_p2p_set_role(struct wifidirect_info *wdinfo, enum P2P_ROLE role);
+
+static inline int _rtw_p2p_role(struct wifidirect_info *wdinfo)
+{
+	return wdinfo->role;
+}
+static inline bool _rtw_p2p_chk_role(struct wifidirect_info *wdinfo, enum P2P_ROLE role)
+{
+	return wdinfo->role == role;
+}
+
+#ifdef CONFIG_DBG_P2P
+void dbg_rtw_p2p_set_role(struct wifidirect_info *wdinfo, enum P2P_ROLE role, const char *caller, int line);
+#define rtw_p2p_set_role(wdinfo, role) dbg_rtw_p2p_set_role(wdinfo, role, __FUNCTION__, __LINE__)
+#else /* CONFIG_DBG_P2P */
+#define rtw_p2p_set_role(wdinfo, role) _rtw_p2p_set_role(wdinfo, role)
+#endif /* CONFIG_DBG_P2P */
+
+#define rtw_p2p_role(wdinfo) _rtw_p2p_role(wdinfo)
+#define rtw_p2p_chk_role(wdinfo, role) _rtw_p2p_chk_role(wdinfo, role)
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_phl.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_phl.h
new file mode 100644
index 000000000000..626d8c352fa0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_phl.h
@@ -0,0 +1,256 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_PHL_H_
+#define _RTW_PHL_H_
+
+typedef struct rtw_phl_com_t HAL_DATA_TYPE; /*, *PHAL_DATA_TYPE;*/
+
+#define GET_PHL_INFO(_dvobj)	(_dvobj->phl)
+#define GET_PHL_COM(_dvobj)	(_dvobj->phl_com)
+#define GET_HAL_SPEC(_dvobj)	(&(GET_PHL_COM(_dvobj)->hal_spec))
+
+
+#define GET_PHY_CAP(_dvobj, _band)     (&(GET_PHL_COM(_dvobj))->phy_cap[_band])
+/* Using the macro when band info not ready. */
+#define GET_PHY_TX_NSS_BY_BAND(_dvobj, _band) ((GET_PHY_CAP(_dvobj, _band))->txss)
+#define GET_PHY_RX_NSS_BY_BAND(_dvobj, _band) ((GET_PHY_CAP(_dvobj, _band))->rxss)
+
+#define GET_TX_PATH_NUM(_dvobj, _band) ((GET_PHY_CAP(_dvobj, _band))->tx_path_num)
+#define GET_RX_PATH_NUM(_dvobj, _band) ((GET_PHY_CAP(_dvobj, _band))->rx_path_num)
+
+#define GET_HAL_RFPATH_NUM(_dvobj) ((GET_PHL_COM(_dvobj))->rf_path_num)
+/* refer to (hal_data->version_id.RFType / registrypriv->rf_path / 8814a from efuse or registrypriv)*/
+#define GET_HAL_RFPATH(_dvobj) ((GET_PHL_COM(_dvobj))->rf_type)
+
+#define GET_WIFI_ROLE_LINK_CURRENT_CH(_adapter_link) (_adapter_link->wrlink->chandef.chan)
+#define GET_WIFI_ROLE_LINK_CURRENT_BAND(_adapter_link) (_adapter_link->wrlink->chandef.band)
+#define WIFI_ROLE_LINK_IS_ON_24G(_adapter_link) (_adapter_link->wrlink->chandef.band == BAND_ON_24G)
+#define WIFI_ROLE_LINK_IS_ON_5G(_adapter_link) (_adapter_link->wrlink->chandef.band == BAND_ON_5G)
+#define WIFI_ROLE_LINK_IS_ON_6G(_adapter_link) (_adapter_link->wrlink->chandef.band == BAND_ON_6G)
+
+#ifdef CONFIG_USB_HCI
+#define RTW_LITEXMITBUF_NR 256
+#define RTW_XMITURB_NR 256
+/*
+ * NR_RECV_URB is the number of maximum bulk-in URBs submitted by the driver.
+ * Since PHL will resubmit the bulk-in URB before the last completed URB and
+ * literecvbuf are available, the driver needs to allocate NR_RECV_URB+1 URB
+ * and literecvbuf to prevent URB resubmission failure.
+ */
+#define RTW_LITERECVBUF_NR (NR_RECV_URB + 1)
+#define RTW_RECVURB_NR (NR_RECV_URB + 1)
+#define RTW_INTINBUF_NR 1
+#define RTW_INTINURB_NR 1
+#endif
+
+#ifdef CONFIG_PCI_HCI
+#define IOT_BASE	0x11
+#elif defined(CONFIG_USB_HCI)
+#define IOT_BASE	0x12
+#elif defined(CONFIG_SDIO_HCI)
+#define IOT_BASE	0x13
+#else
+#define IOT_BASE	0x10
+#endif
+
+#ifndef IOT_VENDOR
+#define IOT_VENDOR	0
+#endif
+
+#define IOT_ID(T)	(((T) << 16) | (IOT_VENDOR << 8) | IOT_BASE)
+
+s8 rtw_phl_rssi_to_dbm(u8 rssi);
+
+void rtw_hw_dump_hal_spec(void *sel, struct dvobj_priv *dvobj);
+void rtw_dump_phl_sta_info(void *sel, struct sta_info *sta);
+
+bool rtw_hw_chk_band_cap(struct dvobj_priv *dvobj, u8 cap);
+bool rtw_hw_chk_bw_cap(struct dvobj_priv *dvobj, u8 cap);
+bool rtw_hw_chk_proto_cap(struct dvobj_priv *dvobj, u8 cap);
+bool rtw_hw_chk_wl_func(struct dvobj_priv *dvobj, u8 func);
+bool rtw_hw_is_band_support(struct dvobj_priv *dvobj, u8 band);
+bool rtw_hw_is_bw_support(struct dvobj_priv *dvobj, u8 bw);
+bool rtw_hw_is_wireless_mode_support(struct dvobj_priv *dvobj, u8 mode);
+u8 rtw_hw_get_wireless_mode(struct dvobj_priv *dvobj);
+u8 rtw_hw_get_band_cap(struct dvobj_priv *dvobj);
+u8 rtw_hw_get_mac_addr(struct dvobj_priv *dvobj, u8 *hw_mac_addr);
+
+bool rtw_hw_is_mimo_support(_adapter *adapter);
+u8 rtw_hw_largest_bw(struct dvobj_priv *dvobj, u8 in_bw);
+u8 rtw_hw_init(struct dvobj_priv *dvobj);
+void rtw_hw_deinit(struct dvobj_priv *dvobj);
+
+
+u8 rtw_hw_start(struct dvobj_priv *dvobj);
+void rtw_hw_stop(struct dvobj_priv *dvobj);
+bool rtw_hw_get_init_completed(struct dvobj_priv *dvobj);
+bool rtw_hw_is_init_completed(struct dvobj_priv *dvobj);
+void rtw_hw_cap_init(struct dvobj_priv *dvobj);
+void rtw_dump_rfe_type(struct dvobj_priv *d);
+
+void rtw_collect_adapter_link_mac_addr(_adapter *adapter, u8 **mac);
+u8 rtw_hw_iface_init(_adapter *adapter);
+u8 rtw_hw_iface_type_change(_adapter *adapter, u8 iface_type);
+void rtw_hw_iface_deinit(_adapter *adapter);
+
+/* security */
+u8 rtw_sec_algo_drv2phl(enum security_type drv_algo);
+u8 rtw_sec_algo_phl2drv(enum rtw_enc_algo phl_algo);
+int rtw_hw_add_key(struct _ADAPTER *a, struct sta_info *sta,
+		u8 keyid, enum security_type keyalgo, u8 keytype, u8 *key,
+		u8 spp, enum phl_cmd_type cmd_type, u32 cmd_timeout);
+int rtw_hw_del_key(struct _ADAPTER *a, struct sta_info *sta,
+		u8 keyid, u8 keytype, enum phl_cmd_type cmd_type, u32 cmd_timeout);
+int rtw_hw_del_all_key(struct _ADAPTER *a, struct sta_info *sta,
+		enum phl_cmd_type cmd_type, u32 cmd_timeout);
+
+/* settting */
+int rtw_hw_set_ch_bw(struct _ADAPTER *a, struct _ADAPTER_LINK *alink, enum band_type band,
+		u8 ch, enum channel_width bw, u8 offset, enum rfk_tri_type rt_type);
+int rtw_hw_set_edca(struct _ADAPTER *a, struct _ADAPTER_LINK *alink, u8 ac, u32 param);
+
+/* connect */
+#ifdef RTW_WKARD_UPDATE_PHL_ROLE_CAP
+void rtw_update_phl_cap_by_rgstry(struct _ADAPTER *a, struct _ADAPTER_LINK *alink);
+#endif
+void rtw_update_phl_sta_cap(struct _ADAPTER *a, struct sta_info *sta,
+			    struct protocol_cap_t *cap);
+void rtw_update_phl_sta_edca(struct _ADAPTER *a, struct _ADAPTER_LINK *alink, enum rtw_ac ac, u32 param);
+#if 0
+int rtw_hw_prepare_connect(struct _ADAPTER *a, struct sta_info *sta, u8 *target_addr);
+#endif
+
+int rtw_hw_connect_remove_sta(struct _ADAPTER *a, struct sta_info *sta);
+int rtw_hw_connect_abort(struct _ADAPTER *a);
+int rtw_hw_connected(struct _ADAPTER *a);
+int rtw_hw_connected_apmode(struct _ADAPTER *a, struct sta_info *sta);
+int rtw_hw_disconnect(struct _ADAPTER *a, struct sta_info *sta);
+
+void rtw_update_roch_chan_def(struct _ADAPTER_LINK *adapter_link,
+				u8 remain_ch, enum channel_width remain_bw,
+				enum chan_offset offset,
+				enum band_type band);
+
+void rtw_hw_update_chan_def(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+
+u8 rtw_sta_hal_media_status_rpt_cmd(_adapter *a, struct sta_info *sta, bool connected, u8 flag);
+u8 rtw_sta_hal_ra_mask_update_cmd(_adapter *a, struct sta_info *sta, u8 flag);
+
+u8 rtw_link_hal_core_stop_beacon(struct _ADAPTER_LINK *alink, bool stop, u8 flag);
+
+#ifdef RTW_DETECT_HANG
+void rtw_is_hang_check(struct _ADAPTER *a);
+#endif
+
+/*
+ * define _HT_IOT_PEER here is a workaround, because fail to include
+ * rtw_mlme_ext.h before declaring rtw_update_phl_iot().
+ */
+enum _HT_IOT_PEER;
+void rtw_update_phl_iot(struct _ADAPTER *a, enum _HT_IOT_PEER peer);
+int rtw_get_sta_tx_stat(_adapter *adapter, struct sta_info *sta);
+
+#ifdef CONFIG_RTW_ACS
+u8 rtw_acs_get_clm_ratio(struct _ADAPTER *a, enum band_type band, u8 idx);
+u8 rtw_acs_get_nhm_ratio(struct _ADAPTER *a, enum band_type band, u8 ch);
+s8 rtw_acs_get_noise_dbm(struct _ADAPTER *a, enum band_type band, u8 idx);
+int rtw_acs_get_report(struct _ADAPTER *a, enum band_type band, u8 ch, struct rtw_acs_info_parm *rpt);
+#endif /* CONFIG_RTW_ACS */
+
+void rtw_dump_env_rpt(struct _ADAPTER *a, void *sel);
+
+#ifdef DBG_RX_DFRAME_RAW_DATA
+void rtw_dump_rx_dframe_info(struct _ADAPTER *padapter, void *sel);
+#endif
+
+#ifdef CONFIG_WOWLAN
+u8 rtw_hw_wow(struct _ADAPTER *a, u8 wow_en);
+#endif
+
+#ifdef CONFIG_MCC_MODE
+u8 rtw_hw_mcc_chk_inprogress(struct _ADAPTER *a, struct _ADAPTER_LINK *adapter_link);
+#endif
+
+void rtw_edcca_hal_update(struct dvobj_priv *dvobj);
+
+#if CONFIG_TXPWR_LIMIT
+enum txpwr_lmt_reg_exc_match {
+	TXPWR_LMT_REG_EXC_MATCH_NONE = 0,
+	TXPWR_LMT_REG_EXC_MATCH_COUNTRY,
+	TXPWR_LMT_REG_EXC_MATCH_DOMAIN,
+};
+
+enum txpwr_lmt_reg_exc_match rtw_txpwr_hal_lmt_reg_exc_search(struct dvobj_priv* dvobj
+	, const char *country, u8 domain, const char **reg_name);
+#if CONFIG_IEEE80211_BAND_6GHZ
+enum txpwr_lmt_reg_exc_match rtw_txpwr_hal_lmt_reg_exc_6g_search(struct dvobj_priv* dvobj
+	, const char *country, u8 domain, const char **reg_name);
+#endif
+bool rtw_txpwr_hal_lmt_reg_search(struct dvobj_priv* dvobj, enum band_type band, const char *name);
+void rtw_txpwr_hal_set_current_lmt_regs_by_name(struct dvobj_priv* dvobj, char *names_of_band[], int names_len_of_band[]);
+void rtw_txpwr_hal_get_current_lmt_regs_name(struct dvobj_priv* dvobj, char *names_of_band[], int names_len_of_band[]);
+void dump_txpwr_lmt(void *sel, _adapter *adapter);
+#ifdef CONFIG_80211AX_HE
+void dump_txpwr_lmt_ru(void *sel, _adapter *adapter);
+#endif
+#endif /* CONFIG_TXPWR_LIMIT */
+
+void dump_txpwr_by_rate(void *sel, _adapter *adapter);
+
+#ifdef CONFIG_DFS_MASTER
+void rtw_dfs_hal_radar_detect_disable(struct dvobj_priv *dvobj, u8 band_idx);
+void rtw_dfs_hal_radar_detect_enable(struct dvobj_priv *dvobj, u8 band_idx, bool cac, u32 rd_freq_hi, u32 rd_freq_lo);
+void rtw_dfs_hal_set_cac_status(struct dvobj_priv *dvobj, u8 band_idx, bool cac);
+void rtw_dfs_hal_csa_mg_tx_pause(struct dvobj_priv *dvobj, u8 band_idx, bool pause);
+bool rtw_dfs_hal_region_supported(struct dvobj_priv* dvobj, enum rtw_dfs_regd domain);
+void rtw_dfs_hal_update_region(struct dvobj_priv *dvobj, u8 band_idx, enum rtw_dfs_regd domain);
+u8 rtw_dfs_hal_radar_detect_polling_int_ms(struct dvobj_priv *dvobj);
+#endif /* CONFIG_DFS_MASTER */
+
+#define SPF_PHL_RF_019_SAR /* for HALRF PW_LMT_REGU_EXT_PWR mechanism */
+
+bool rtw_txpwr_hal_is_txpwr_limit_needed(struct dvobj_priv *dvobj);
+bool rtw_txpwr_hal_get_pwr_lmt_en(struct dvobj_priv *dvobj);
+struct tx_power_ext_info;
+bool rtw_txpwr_hal_get_ext_info(struct dvobj_priv *dvobj, struct tx_power_ext_info *info);
+void rtw_txpwr_hal_update_pwr(struct dvobj_priv *dvobj, enum phl_band_idx band_idx);
+
+u8 get_phy_tx_nss(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+u8 get_phy_rx_nss(_adapter *adapter, struct _ADAPTER_LINK *adapter_link);
+bool phy_is_txpwr_user_mbm_valid(_adapter *adapter, s16 mbm);
+u8 rtw_backup_and_get_final_ss(_adapter *adapter, struct sta_info *sta, u8 chg_ss);
+void rtw_ctrl_and_backup_assoc_cap_rx_nss(_adapter *adapter, struct sta_info *sta, u8 rx_nss);
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN
+bool rtw_dbcc_b0_sta_chan_chk(struct _ADAPTER *a);
+
+enum dbcc_chk_pcd {
+	DBCC_CHK_STA_CON,
+	DBCC_CHK_STA_DIS,
+	DBCC_CHK_AP_START,
+	DBCC_CHK_AP_STOP
+};
+bool rtw_dbcc_chk_enable_hdl(_adapter *adapter,
+				enum phl_cmd_type cmd_type,
+				enum dbcc_chk_pcd chk_pcd);
+bool rtw_dbcc_chk_disable_hdl(_adapter *adapter,
+				struct rtw_chan_def *new_chdef,
+				enum phl_cmd_type cmd_type,
+				enum dbcc_chk_pcd chk_pcd);
+enum rtw_phl_status
+rtw_discon_end_dbcc_en_notify(struct _ADAPTER *a, enum phl_module_id mdl_id);
+
+#endif
+
+#endif /* _RTW_HW_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_phl_cmd.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_phl_cmd.h
new file mode 100644
index 000000000000..ec5c30fa72ea
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_phl_cmd.h
@@ -0,0 +1,30 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_PHL_CMD__
+#define __RTW_PHL_CMD__
+u32 rtw_enqueue_phl_cmd(struct cmd_obj *pcmd);
+#ifdef CONFIG_TDLS
+enum rtw_phl_status rtw_send_tdls_sync_msg(_adapter *padapter);
+#endif
+#if defined (CONFIG_PCIE_TRX_MIT)
+u8 rtw_pcie_trx_mit_cmd(_adapter *padapter, u32 tx_timer, u8 tx_counter,
+			u32 rx_timer, u8 rx_counter, u8 fixed_mit);
+#endif
+#ifdef CONFIG_POST_CORE_KEEP_ALIVE
+u8 rtw_set_post_keep_alive_param(_adapter *padapter, u8 if_sta_chk_rx_bmp,
+			   u8 if_sta_chk_tx_bmp, u32 *if_ap_chk_sta_bmp);
+#endif
+#endif /* __RTW_PHL_CMD__ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_pwrctrl.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_pwrctrl.h
new file mode 100644
index 000000000000..eea3e3e1bc67
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_pwrctrl.h
@@ -0,0 +1,133 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_PWRCTRL_H_
+#define __RTW_PWRCTRL_H_
+
+typedef _sema _pwrlock;
+/*
+	BIT[2:0] = HW state
+	BIT[3] = Protocol PS state,   0: register active state , 1: register sleep state
+	BIT[4] = sub-state
+*/
+
+enum ps_ips_level {
+	PS_IPS_NONE = 0,
+	PS_PWR_OFF = 1,
+	PS_IPS_RF_OFF = 2,
+	PS_IPS_CLK_GATED = 3,
+	PS_IPS_PWR_GATED = 4,
+	PS_IPS_MAX = 5
+};
+
+enum ps_lps_level {
+	PS_LPS_NONE = 0,
+	PS_LPS_RF_OFF = 1,
+	PS_LPS_CLK_GATED = 2,
+	PS_LPS_PWR_GATED = 3,
+	PS_LPS_MAX = 4
+};
+
+struct pwrctrl_priv {
+	_pwrlock	lock;
+	u32	alives;
+
+	_workitem dma_event; /*for handle un-synchronized tx dma*/
+	u8		bInSuspend;
+#ifdef CONFIG_BTC
+	u8		bAutoResume;
+	u8		autopm_cnt;
+#endif
+	u8		bSupportRemoteWakeup;
+	u8		wowlan_ap_mode;
+	u8		wowlan_mode;
+	u8		wowlan_p2p_mode;
+	u8		wowlan_no_link_mode;
+	u8		wowlan_pno_enable;
+	u8		wowlan_in_resume;
+
+#ifdef CONFIG_GPIO_WAKEUP
+#endif /* CONFIG_GPIO_WAKEUP */
+	u8		hst2dev_high_active;
+#ifdef CONFIG_WOWLAN
+#ifdef CONFIG_IPV6
+	u8		wowlan_ns_offload_en;
+#endif /*CONFIG_IPV6*/
+	u8		wowlan_txpause_status;
+	u8		wowlan_pattern_idx;
+	u64		wowlan_fw_iv;
+	struct rtl_priv_pattern	patterns[MAX_WKFM_CAM_NUM];
+	_mutex	wowlan_pattern_cam_mutex;
+	u8		wowlan_aoac_rpt_loc;
+	struct aoac_report wowlan_aoac_rpt;
+#endif /* CONFIG_WOWLAN */
+	u8		bkeepfwalive;
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+	struct workqueue_struct *rtw_workqueue;
+	_workitem resume_work;
+#endif
+
+#ifdef CONFIG_HAS_EARLYSUSPEND
+	struct early_suspend early_suspend;
+	u8 do_late_resume;
+#endif /* CONFIG_HAS_EARLYSUSPEND */
+
+#ifdef CONFIG_ANDROID_POWER
+	android_early_suspend_t early_suspend;
+	u8 do_late_resume;
+#endif
+#ifdef CONFIG_RTW_CFGVENDOR_LLSTATS
+	systime radio_on_start_time;
+	systime pwr_saving_start_time;
+	u32 pwr_saving_time;
+	u32 on_time;
+	u32 tx_time;
+	u32 rx_time;
+#endif /* CONFIG_RTW_CFGVENDOR_LLSTATS */
+	enum rtw_rf_state rfkill_state;
+};
+
+#ifdef CONFIG_RTW_IPS
+bool rtw_core_set_ips_state(void *drv_priv, enum rtw_rf_state state);
+#endif
+extern void rtw_init_pwrctrl_priv(_adapter *adapter);
+extern void rtw_free_pwrctrl_priv(_adapter *adapter);
+
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+void rtw_resume_in_workqueue(struct pwrctrl_priv *pwrpriv);
+#endif /* CONFIG_RESUME_IN_WORKQUEUE */
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) || defined(CONFIG_ANDROID_POWER)
+bool rtw_is_earlysuspend_registered(struct pwrctrl_priv *pwrpriv);
+bool rtw_is_do_late_resume(struct pwrctrl_priv *pwrpriv);
+void rtw_set_do_late_resume(struct pwrctrl_priv *pwrpriv, bool enable);
+void rtw_register_early_suspend(struct pwrctrl_priv *pwrpriv);
+void rtw_unregister_early_suspend(struct pwrctrl_priv *pwrpriv);
+#else
+#define rtw_is_earlysuspend_registered(pwrpriv) _FALSE
+#define rtw_is_do_late_resume(pwrpriv) _FALSE
+#define rtw_set_do_late_resume(pwrpriv, enable) do {} while (0)
+#define rtw_register_early_suspend(pwrpriv) do {} while (0)
+#define rtw_unregister_early_suspend(pwrpriv) do {} while (0)
+#endif /* CONFIG_HAS_EARLYSUSPEND || CONFIG_ANDROID_POWER */
+void rtw_ssmps_enter(_adapter *adapter, struct sta_info *sta);
+void rtw_ssmps_leave(_adapter *adapter, struct sta_info *sta);
+void rtw_update_ips_setting(int make_level, int ins_level, u8 *mode, u8 *cap, bool is_wow);
+void rtw_update_lps_setting(int make_level, int ins_level, u8 *mode, u8 *cap, bool is_wow);
+#if defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW)
+void rtw_update_lps_listen_beacon_mode(
+	u8 bcn_mode, u8 awake_interval,
+	enum rtw_lps_listen_bcn_mode *lps_bcn_mode, u8 *lps_awake_interval);
+#endif /* defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW) */
+#endif /* __RTL871X_PWRCTRL_H_ */
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_qos.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_qos.h
new file mode 100644
index 000000000000..74cbac84de31
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_qos.h
@@ -0,0 +1,52 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+
+#ifndef _RTW_QOS_H_
+#define _RTW_QOS_H_
+
+#define DRV_CFG_UAPSD_VO 	BIT0
+#define DRV_CFG_UAPSD_VI 	BIT1
+#define DRV_CFG_UAPSD_BK 	BIT2
+#define DRV_CFG_UAPSD_BE 	BIT3
+
+#define WMM_IE_UAPSD_VO 	BIT0
+#define WMM_IE_UAPSD_VI 	BIT1
+#define WMM_IE_UAPSD_BK 	BIT2
+#define WMM_IE_UAPSD_BE 	BIT3
+
+#define WMM_TID0 	BIT0
+#define WMM_TID1 	BIT1
+#define WMM_TID2 	BIT2
+#define WMM_TID3 	BIT3
+#define WMM_TID4 	BIT4
+#define WMM_TID5 	BIT5
+#define WMM_TID6 	BIT6
+#define WMM_TID7 	BIT7
+
+#define AP_SUPPORTED_UAPSD BIT7
+/* TC = Traffic Category,  TID0~7 represents TC */
+#define BIT_MASK_TID_TC 0xff
+/* TS = Traffic Stream,  TID8~15 represents TS */
+#define BIT_MASK_TID_TS 0xff00
+#define ALL_TID_TC_SUPPORTED_UAPSD 0xff
+
+struct	qos_priv	{
+
+	unsigned int	  qos_option;	/* bit mask option: u-apsd, s-apsd, ts, block ack...		 */
+};
+
+
+#endif /* _RTL871X_QOS_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_recv.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_recv.h
new file mode 100644
index 000000000000..d7baa4a21d3a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_recv.h
@@ -0,0 +1,659 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_RECV_H_
+#define _RTW_RECV_H_
+
+#define RTW_RX_MSDU_ACT_NONE		0
+#define RTW_RX_MSDU_ACT_INDICATE	BIT0
+#define RTW_RX_MSDU_ACT_FORWARD		BIT1
+
+#ifdef CONFIG_RTW_NAPI
+	#define RTL_NAPI_WEIGHT (32)
+#endif
+
+
+#define NR_RECVFRAME 256
+
+#define RXFRAME_ALIGN	8
+#define RXFRAME_ALIGN_SZ	(1<<RXFRAME_ALIGN)
+
+#define DRVINFO_SZ	4 /* unit is 8bytes */
+
+#define MAX_RXFRAME_CNT	512
+#define MAX_RX_NUMBLKS		(32)
+#define RECVFRAME_HDR_ALIGN 128
+#define MAX_CONTINUAL_NORXPACKET_COUNT 4    /*  In MAX_CONTINUAL_NORXPACKET_COUNT*2 sec  , no rx traffict would issue DELBA*/
+
+#define PHY_RSSI_SLID_WIN_MAX				100
+#define PHY_LINKQUALITY_SLID_WIN_MAX		20
+
+
+#define SNAP_SIZE sizeof(struct ieee80211_snap_hdr)
+
+#define MAX_SUBFRAME_COUNT	64
+/* Bridge-Tunnel header (for EtherTypes ETH_P_AARP and ETH_P_IPX) */
+extern u8 rtw_bridge_tunnel_header[];
+extern u8 rtw_rfc1042_header[];
+
+enum addba_rsp_ack_state {
+	RTW_RECV_ACK_OR_TIMEOUT,
+};
+
+#ifdef RTW_PHL_RX
+enum rtw_core_rx_state {
+	CORE_RX_CONTINUE = _SUCCESS,
+	CORE_RX_DONE,
+	CORE_RX_DROP,
+	CORE_RX_FAIL,
+#ifdef CONFIG_RTW_CORE_RXSC
+	CORE_RX_GO_SHORTCUT,
+#endif
+	CORE_RX_DEFRAG,
+};
+#endif
+
+/* for Rx reordering buffer control */
+struct recv_reorder_ctrl {
+	_adapter	*padapter;
+	u8 tid;
+	u8 enable;
+	u16 indicate_seq;/* =wstart_b, init_value=0xffff */
+	u16 ampdu_size;
+	unsigned long rec_abba_rsp_ack;
+
+	#ifdef CONFIG_RECV_REORDERING_CTRL
+	_queue pending_recvframe_queue;
+	u8 wsize_b;
+	_timer reordering_ctrl_timer;
+	u8 bReorderWaiting;
+	#endif
+};
+
+struct	stainfo_rxcache	{
+	u16	tid_rxseq[16];
+	u8 iv[16][8];
+	u8 last_tid;
+};
+
+
+struct smooth_rssi_data {
+	u32	elements[100];	/* array to store values */
+	u32	index;			/* index to current array to store */
+	u32	total_num;		/* num of valid elements */
+	u32	total_val;		/* sum of valid elements */
+};
+
+struct signal_stat {
+	u8	update_req;		/* used to indicate */
+	u8	avg_val;		/* avg of valid elements */
+	u32	total_num;		/* num of valid elements */
+	u32	total_val;		/* sum of valid elements	 */
+};
+
+/*TODO get phyinfo from PHL PPDU status - RTW_WKARD_CORE_RSSI_V1*/
+struct phydm_phyinfo_struct {
+	bool is_valid;
+	u8 rx_pwdb_all;
+	u8 signal_quality;	/* OFDM: signal_quality=rx_mimo_signal_quality[0], CCK: signal qualityin 0-100 index. */
+
+	s8 rx_power;		/* in dBm Translate from PWdB */
+	s8 recv_signal_power;	/* Real power in dBm for this packet, no beautification and aggregation. Keep this raw info to be used for the other procedures. */
+	u8 signal_strength;	/* in 0-100 index. */
+	s8 rx_pwr[RTW_PHL_MAX_RF_PATH];		/* per-path's pwdb */
+	s8 rx_snr[RTW_PHL_MAX_RF_PATH];		/* per-path's SNR */
+	u8 rx_count:2;		/* RX path counter---*/
+	u8 snr_fd_avg;
+	u8 snr_fd[RTW_PHL_MAX_RF_PATH];
+	u8 snr_td_avg;
+	u8 snr_td[RTW_PHL_MAX_RF_PATH];
+};
+
+
+struct rx_pkt_attrib {
+	u16	pkt_len;
+	u8	physt;
+	u8	drvinfo_sz;
+	u8	shift_sz;
+	u8	hdrlen; /* the WLAN Header Len */
+	u8	to_fr_ds;
+	u8	amsdu;
+	u8	qos;
+	u8	priority;
+	u8	pw_save;
+	u8	mdata;
+	u16	seq_num;
+	u8	frag_num;
+	u8	mfrag;
+	u8	order;
+	u8	privacy; /* in frame_ctrl field */
+	u8	bdecrypted;
+	u8	encrypt; /* when 0 indicate no encrypt. when non-zero, indicate the encrypt algorith */
+	u8	iv_len;
+	u8	icv_len;
+	u8	crc_err;
+	u8	icv_err;
+
+#ifdef CONFIG_RTW_CORE_RXSC
+	u16	eth_type;
+#endif
+
+	u8	dst[ETH_ALEN];
+	u8	src[ETH_ALEN];
+	u8	ta[ETH_ALEN];
+	u8	ra[ETH_ALEN];
+	u8	bssid[ETH_ALEN];
+#ifdef CONFIG_RTW_MESH
+	u8	msa[ETH_ALEN]; /* mesh sa */
+	u8	mda[ETH_ALEN]; /* mesh da */
+	u8 mesh_ctrl_present;
+	u8	mesh_ctrl_len; /* length of mesh control field */
+#endif
+
+	u8	ack_policy;
+
+	u8	key_index;
+	u16	data_rate; /* enum rtw_data_rate */
+	u8	gi_ltf;
+	u8 ch; /* RX channel */
+	u8	bw;
+	u8	stbc;
+	u8	ldpc;
+	u8	sgi;
+	u8	pkt_rpt_type;
+	u8	ampdu;
+	u8	ppdu_cnt;
+	u8	ampdu_eof;
+	u32 	free_cnt;		/* free run counter */
+	struct phydm_phyinfo_struct phy_info;
+#ifdef CONFIG_WIFI_MONITOR
+	u8 moif[16];
+#endif
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_RX
+	/* checksum offload realted varaiables */
+	u8 csum_valid;		/* Checksum valid, 0: not check, 1: checked */
+	u8 csum_err;		/* Checksum Error occurs */
+#endif /* CONFIG_TCP_CSUM_OFFLOAD_RX */
+
+#ifdef RTW_PHL_DBG_CMD
+	u8	wl_type;
+	u8	wl_subtype;
+#endif
+#ifdef CONFIG_RTW_CORE_RXSC
+	u8	bsnaphdr;
+#endif
+	u8	addr_cam_vld;
+	u16	macid;
+#ifdef CONFIG_TDLS
+	u8 is_tdls_frame;
+#endif
+};
+
+#ifdef CONFIG_RTW_MESH
+#define RATTRIB_GET_MCTRL_LEN(rattrib) ((rattrib)->mesh_ctrl_len)
+#else
+#define RATTRIB_GET_MCTRL_LEN(rattrib) 0
+#endif
+
+/* These definition is used for Rx packet reordering. */
+#define SN_LESS(a, b)		(((a-b) & 0x800) != 0)
+#define SN_EQUAL(a, b)	(a == b)
+/* #define REORDER_WIN_SIZE	128 */
+/* #define REORDER_ENTRY_NUM	128 */
+#define REORDER_WAIT_TIME	(50) /* (ms) */
+
+#if defined(CONFIG_PLATFORM_RTK390X) && defined(CONFIG_USB_HCI)
+	#define RECVBUFF_ALIGN_SZ 32
+#else
+	#define RECVBUFF_ALIGN_SZ 8
+#endif
+
+/*GEORGIA_TODO_FIXIT_IC_DEPENDENCE*/
+#define RXDESC_SIZE	24
+#define RXDESC_OFFSET RXDESC_SIZE
+
+#ifdef CONFIG_TRX_BD_ARCH
+struct rx_buf_desc {
+	/* RX has exactly one segment */
+#ifdef CONFIG_64BIT_DMA
+	unsigned int dword[4];
+#else
+	unsigned int dword[2];
+#endif
+};
+
+struct recv_stat {
+	unsigned int rxdw[8];
+};
+#else
+struct recv_stat {
+	unsigned int rxdw0;
+
+	unsigned int rxdw1;
+
+#if !(defined(CONFIG_RTL8822B) || defined(CONFIG_RTL8821C) || defined(CONFIG_RTL8822C)) && defined(CONFIG_PCI_HCI)  /* exclude 8822be, 8821ce ,8822ce*/
+	unsigned int rxdw2;
+
+	unsigned int rxdw3;
+#endif
+
+#ifndef BUF_DESC_ARCH
+	unsigned int rxdw4;
+
+	unsigned int rxdw5;
+
+#ifdef CONFIG_PCI_HCI
+	unsigned int rxdw6;
+
+	unsigned int rxdw7;
+#endif
+#endif /* if BUF_DESC_ARCH is defined, rx_buf_desc occupy 4 double words */
+};
+#endif
+
+#define EOR BIT(30)
+
+
+/*
+accesser of recv_priv: rtw_recv_entry(dispatch / passive level); recv_thread(passive) ; returnpkt(dispatch)
+; halt(passive) ;
+
+using enter_critical section to protect
+*/
+
+#ifdef PRIVATE_R
+	/* total data rate index = 132 (refer to DESC_RATEHESS4MCS11 = 0x83) */
+	#define DESC_RATE_MAX 0x84
+#endif
+
+struct recv_info {
+	u64 rx_bytes;
+	u64 rx_pkts;
+	u64 rx_drop;
+
+#ifdef PRIVATE_R
+	u16 rx_vo_pkt_count_per_data_rate[DESC_RATE_MAX];
+	u64 rx_vo_pkt_retry_count;
+	/* @last_rx_uc_data counts unicast data packets, it will be reset if user reads it by proc */
+	u64 last_rx_uc_data;
+	/* @last_rx_uc_data_rate_avg sums receive data rate of unicast data packets */
+	u64 last_rx_uc_data_rate_avg;
+#endif
+	u64 dbg_rx_drop_count;
+	u64 dbg_rx_ampdu_drop_count;
+	u64 dbg_rx_ampdu_forced_indicate_count;
+	u64 dbg_rx_ampdu_loss_count;
+	u64 dbg_rx_dup_mgt_frame_drop_count;
+	u64 dbg_rx_ampdu_window_shift_cnt;
+	u64 dbg_rx_conflic_mac_addr_cnt;
+
+	/* For display the phy informatiom */
+	u8 is_signal_dbg;	/* for debug */
+	u8 signal_strength_dbg;	/* for debug */
+
+	/*RTW_WKARD_CORE_RSSI_V1 - GEORGIA MUST REFINE*/
+	u8 signal_strength;
+	u8 signal_qual;
+	s8 rssi;	/* rtw_phl_rssi_to_dbm(ptarget_wlan->network.PhyInfo.SignalStrength); */
+
+
+	#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	_timer signal_stat_timer;
+	u32 signal_stat_sampling_interval;
+	#endif
+
+	/* u32 signal_stat_converging_constant; */
+	struct signal_stat signal_qual_data;
+	struct signal_stat signal_strength_data;
+
+
+	u16 sink_udpport, pre_rtp_rxseq, cur_rtp_rxseq;
+
+	boolean store_law_data_flag;
+};
+#ifdef CONFIG_SIGNAL_STAT_PROCESS
+#define rtw_set_signal_stat_timer(recvinfo) _set_timer(&(recvinfo)->signal_stat_timer, (recvinfo)->signal_stat_sampling_interval)
+#endif
+
+struct sta_recv_priv {
+
+	_lock	lock;
+	sint	option;
+
+	/* _queue	blk_strms[MAX_RX_NUMBLKS]; */
+	_queue defrag_q;	 /* keeping the fragment frame until defrag */
+
+	struct	stainfo_rxcache rxcache;
+	u16	bmc_tid_rxseq[16];
+	u16	nonqos_rxseq;
+	u16	nonqos_bmc_rxseq;
+
+	/* uint	sta_rx_bytes; */
+	/* uint	sta_rx_pkts; */
+	/* uint	sta_rx_fail; */
+
+};
+
+#if 0
+struct recv_buf {
+	_list list;
+
+	_lock recvbuf_lock;
+
+	u32	ref_cnt;
+
+	_adapter *adapter;
+
+	u8	*pbuf;
+	u8	*pallocated_buf;
+
+	u32	len;
+	u8	*phead;
+	u8	*pdata;
+	u8	*ptail;
+	u8	*pend;
+
+#ifdef CONFIG_USB_HCI
+	PURB	purb;
+	dma_addr_t dma_transfer_addr;	/* (in) dma addr for transfer_buffer */
+	u32 alloc_sz;
+
+	u8  irp_pending;
+	int  transfer_len;
+#endif
+	struct sk_buff *pskb;
+};
+#endif
+
+struct recv_frame_hdr {
+	_list	list;
+	struct sk_buff *pkt;
+
+	_adapter  *adapter;
+	struct _ADAPTER_LINK *adapter_link;
+	struct dvobj_priv *dvobj;
+
+	u8 fragcnt;
+
+	int frame_tag;
+	int keytrack;
+	struct rx_pkt_attrib attrib;
+
+	uint  len;
+	u8 *rx_head;
+	u8 *rx_data;
+	u8 *rx_tail;
+	u8 *rx_end;
+
+	void *precvbuf;
+
+
+	/*  */
+	struct sta_info *psta;
+#ifdef CONFIG_RECV_REORDERING_CTRL
+	/* for A-MPDU Rx reordering buffer control */
+	struct recv_reorder_ctrl *preorder_ctrl;
+#endif
+#ifdef RTW_PHL_RX
+	void *rx_req;
+#endif
+#ifdef CONFIG_RTW_CORE_RXSC
+	struct core_rxsc_entry *rxsc_entry;
+#endif
+
+#ifdef CONFIG_WAPI_SUPPORT
+	u8 UserPriority;
+	u8 WapiTempPN[16];
+	u8 WapiSrcAddr[6];
+	u8 bWapiCheckPNInDecrypt;
+	u8 bIsWaiPacket;
+#endif
+
+};
+
+
+union recv_frame {
+	union {
+		_list list;
+		struct recv_frame_hdr hdr;
+		uint mem[RECVFRAME_HDR_ALIGN >> 2];
+	} u;
+	/* uint mem[MAX_RXSZ>>2]; */
+};
+
+enum rtw_rx_llc_hdl {
+	RTW_RX_LLC_KEEP		= 0,
+	RTW_RX_LLC_REMOVE	= 1,
+	RTW_RX_LLC_VLAN		= 2,
+};
+
+struct recv_priv {
+	struct dvobj_priv *dvobj;
+
+	#ifdef CONFIG_RECV_THREAD_MODE
+	_sema recv_sema;
+	#endif
+	_queue free_recv_queue; /*recv_frame*/
+	#if 0
+	_queue uc_swdec_pending_queue;
+	#endif
+
+	u8 *pallocated_frame_buf;
+	u8 *precv_frame_buf;
+
+	uint free_recvframe_cnt;
+
+#if defined(PLATFORM_LINUX) || defined(PLATFORM_FREEBSD)
+#ifdef CONFIG_RTW_NAPI
+	struct sk_buff_head rx_napi_skb_queue;
+#endif
+#endif /* defined(PLATFORM_LINUX) || defined(PLATFORM_FREEBSD) */
+
+};
+
+bool rtw_rframe_del_wfd_ie(union recv_frame *rframe, u8 ies_offset);
+#ifdef RTW_PHL_RX
+extern void dump_recv_frame(_adapter *adapter, union recv_frame *prframe);
+extern sint validate_recv_frame(_adapter *adapter, union recv_frame **pprecv_frame);
+extern s32 rtw_core_rx_data_pre_process(_adapter *adapter, union recv_frame **prframe);
+extern s32 rtw_core_rx_data_post_process(_adapter *adapter, union recv_frame **pprframe);
+
+enum rtw_phl_status rtw_core_rx_process(void *drv_priv);
+void process_pwrbit_data(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *psta);
+void process_wmmps_data(_adapter *padapter, union recv_frame *precv_frame, struct sta_info *psta);
+#ifdef CONFIG_RTW_CORE_RXSC
+sint recv_ucast_pn_decache(union recv_frame *precv_frame);
+sint recv_bcast_pn_decache(union recv_frame *precv_frame);
+#endif /* CONFIG_RTW_CORE_RXSC */
+#endif
+
+extern void rtw_init_recvframe(union recv_frame *precvframe);
+extern int rtw_free_recvframe(union recv_frame *precvframe);
+union recv_frame *rtw_alloc_recvframe(_queue *pfree_recv_queue);
+
+#if 0
+u32 rtw_free_uc_swdec_pending_queue(struct dvobj_priv *dvobj);
+#endif
+
+#if defined(CONFIG_80211N_HT) && defined(CONFIG_RECV_REORDERING_CTRL)
+void rtw_reordering_ctrl_timeout_handler(void *pcontext);
+#endif
+
+#if 0
+void rx_query_phy_status(union recv_frame *rframe, u8 *phy_stat);
+#endif
+
+int rtw_inc_and_chk_continual_no_rx_packet(struct sta_info *sta, int tid_index);
+void rtw_reset_continual_no_rx_packet(struct sta_info *sta, int tid_index);
+
+#ifdef CONFIG_RECV_THREAD_MODE
+thread_return rtw_recv_thread(thread_context context);
+#endif
+#ifdef RTW_WKARD_CORE_RSSI_V1
+void rx_process_phy_info(union recv_frame *precvframe);
+#endif
+__inline static u8 *get_rxmem(union recv_frame *precvframe)
+{
+	/* always return rx_head... */
+	if (precvframe == NULL)
+		return NULL;
+
+	return precvframe->u.hdr.rx_head;
+}
+
+__inline static u8 *get_rx_status(union recv_frame *precvframe)
+{
+
+	return get_rxmem(precvframe);
+
+}
+
+__inline static u8 *get_recvframe_data(union recv_frame *precvframe)
+{
+
+	/* alwasy return rx_data */
+	if (precvframe == NULL)
+		return NULL;
+
+	return precvframe->u.hdr.rx_data;
+
+}
+
+__inline static u8 *recvframe_push(union recv_frame *precvframe, sint sz)
+{
+	/* append data before rx_data */
+
+	/* add data to the start of recv_frame
+	*
+	*      This function extends the used data area of the recv_frame at the buffer
+	*      start. rx_data must be still larger than rx_head, after pushing.
+	*/
+
+	if (precvframe == NULL)
+		return NULL;
+
+
+	precvframe->u.hdr.rx_data -= sz ;
+	if (precvframe->u.hdr.rx_data < precvframe->u.hdr.rx_head) {
+		precvframe->u.hdr.rx_data += sz ;
+		return NULL;
+	}
+
+	precvframe->u.hdr.len += sz;
+
+	return precvframe->u.hdr.rx_data;
+
+}
+
+
+__inline static u8 *recvframe_pull(union recv_frame *precvframe, sint sz)
+{
+	/* rx_data += sz; move rx_data sz bytes  hereafter */
+
+	/* used for extract sz bytes from rx_data, update rx_data and return the updated rx_data to the caller */
+
+
+	if (precvframe == NULL)
+		return NULL;
+
+
+	precvframe->u.hdr.rx_data += sz;
+
+	if (precvframe->u.hdr.rx_data > precvframe->u.hdr.rx_tail) {
+		precvframe->u.hdr.rx_data -= sz;
+		return NULL;
+	}
+
+	precvframe->u.hdr.len -= sz;
+
+	return precvframe->u.hdr.rx_data;
+
+}
+
+__inline static u8 *recvframe_put(union recv_frame *precvframe, sint sz)
+{
+	/* rx_tai += sz; move rx_tail sz bytes  hereafter */
+
+	/* used for append sz bytes from ptr to rx_tail, update rx_tail and return the updated rx_tail to the caller */
+	/* after putting, rx_tail must be still larger than rx_end. */
+	unsigned char *prev_rx_tail;
+
+	/* RTW_INFO("recvframe_put: len=%d\n", sz); */
+
+	if (precvframe == NULL)
+		return NULL;
+
+	prev_rx_tail = precvframe->u.hdr.rx_tail;
+
+	precvframe->u.hdr.rx_tail += sz;
+
+	if (precvframe->u.hdr.rx_tail > precvframe->u.hdr.rx_end) {
+		precvframe->u.hdr.rx_tail -= sz;
+		return NULL;
+	}
+
+	precvframe->u.hdr.len += sz;
+
+	return precvframe->u.hdr.rx_tail;
+
+}
+
+
+
+__inline static u8 *recvframe_pull_tail(union recv_frame *precvframe, sint sz)
+{
+	/* rmv data from rx_tail (by yitsen) */
+
+	/* used for extract sz bytes from rx_end, update rx_end and return the updated rx_end to the caller */
+	/* after pulling, rx_end must be still larger than rx_data. */
+
+	if (precvframe == NULL)
+		return NULL;
+
+	precvframe->u.hdr.rx_tail -= sz;
+
+	if (precvframe->u.hdr.rx_tail < precvframe->u.hdr.rx_data) {
+		precvframe->u.hdr.rx_tail += sz;
+		return NULL;
+	}
+
+	precvframe->u.hdr.len -= sz;
+
+	return precvframe->u.hdr.rx_tail;
+
+}
+
+__inline static sint get_recvframe_len(union recv_frame *precvframe)
+{
+	return precvframe->u.hdr.len;
+}
+
+
+struct sta_info;
+extern void _rtw_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv);
+
+extern void  mgt_dispatcher(_adapter *padapter, union recv_frame *precv_frame);
+
+u8 adapter_allow_bmc_data_rx(_adapter *adapter);
+#if 0
+s32 pre_recv_entry(union recv_frame *precvframe, u8 *pphy_status);
+#endif
+#ifdef PRIVATE_R
+u8 rtw_get_idx_by_rx_rate(u16 data_rate);
+#endif /* PRIVATE_R */
+void count_rx_stats(_adapter *padapter, union recv_frame *prframe, struct sta_info *sta);
+u8 rtw_init_lite_recv_resource(struct dvobj_priv *dvobj);
+void rtw_free_lite_recv_resource(struct dvobj_priv *dvobj);
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_recv_shortcut.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_recv_shortcut.h
new file mode 100644
index 000000000000..2dc9886b0e36
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_recv_shortcut.h
@@ -0,0 +1,60 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef _RTW_RECV_SHORTCUT_H_
+#define _RTW_RECV_SHORTCUT_H_
+
+#ifdef CONFIG_RTW_CORE_RXSC
+
+#define NUM_RXSC_ENTRY	4
+
+enum rxsc_entry_status  {
+	RXSC_ENTRY_INVALID = 0,
+	RXSC_ENTRY_VALID,
+	RXSC_ENTRY_APPLYING,
+};
+
+struct rxsc_wlan_hdr {
+	unsigned short	fmctrl;
+	unsigned short	duration;
+	unsigned char	addr1[ETH_ALEN];
+	unsigned char	addr2[ETH_ALEN];
+	unsigned char	addr3[ETH_ALEN];
+	unsigned short	sequence;
+	unsigned char	addr4[ETH_ALEN];
+	unsigned short	qosctrl;
+	unsigned char	iv[8];
+};
+
+struct core_rxsc_entry {
+	u8 status;
+	u32 rxsc_payload_offset;
+	u32 rxsc_trim_pad;
+
+	struct rxsc_wlan_hdr rxsc_wlanhdr;
+	struct ethhdr rxsc_ethhdr;
+	struct rx_pkt_attrib rxsc_attrib;
+
+	u8 rxsc_wd[64];
+};
+
+struct core_rxsc_entry *core_rxsc_alloc_entry(_adapter *adapter, union recv_frame *prframe);
+s32 core_rxsc_apply_check(_adapter *adapter, union recv_frame *prframe);
+s32 core_rxsc_apply_shortcut(_adapter *adapter, union recv_frame *prframe);
+void core_rxsc_clear_entry(_adapter *adapter, struct sta_info *psta);
+
+#endif /* CONFIG_RTW_CORE_RXSC */
+#endif /* _RTW_RECV_SHORTCUT_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rf.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rf.h
new file mode 100644
index 000000000000..0765a11caaf9
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rf.h
@@ -0,0 +1,302 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef	__RTW_RF_H_
+#define __RTW_RF_H_
+
+#define NumRates	(13)
+#define	B_MODE_RATE_NUM	(4)
+#define	G_MODE_RATE_NUM	(8)
+#define	G_MODE_BASIC_RATE_NUM	(3)
+/* slot time for 11g */
+#define SHORT_SLOT_TIME					9
+#define NON_SHORT_SLOT_TIME				20
+
+#define CENTER_CH_2G_NUM		14
+#define CENTER_CH_2G_40M_NUM	9
+
+#define CENTER_CH_5G_20M_NUM	28	/* 20M center channels */
+#define CENTER_CH_5G_40M_NUM	14	/* 40M center channels */
+#define CENTER_CH_5G_80M_NUM	7	/* 80M center channels */
+#define CENTER_CH_5G_160M_NUM	3	/* 160M center channels */
+#define CENTER_CH_5G_ALL_NUM	(CENTER_CH_5G_20M_NUM + CENTER_CH_5G_40M_NUM + CENTER_CH_5G_80M_NUM)
+
+#define CENTER_CH_6G_20M_NUM	64	/* 20M center channels */
+#define CENTER_CH_6G_40M_NUM	32	/* 40M center channels */
+#define CENTER_CH_6G_80M_NUM	16	/* 80M center channels */
+#define CENTER_CH_6G_160M_NUM	8	/* 160M center channels */
+
+#define	MAX_CHANNEL_NUM_2G		CENTER_CH_2G_NUM
+#define	MAX_CHANNEL_NUM_5G		CENTER_CH_5G_20M_NUM
+#define MAX_CHANNEL_NUM_6G		CENTER_CH_6G_20M_NUM
+#define MAX_CHANNEL_NUM_2G_5G	(MAX_CHANNEL_NUM_2G + MAX_CHANNEL_NUM_5G)
+
+#define	MAX_CHANNEL_NUM		( \
+	MAX_CHANNEL_NUM_2G \
+	+ (CONFIG_IEEE80211_BAND_5GHZ ? MAX_CHANNEL_NUM_5G : 0) \
+	+ (CONFIG_IEEE80211_BAND_6GHZ ? MAX_CHANNEL_NUM_6G : 0) \
+	)
+
+extern u8 center_ch_2g[CENTER_CH_2G_NUM];
+extern u8 center_ch_2g_40m[CENTER_CH_2G_40M_NUM];
+
+u8 center_chs_2g_num(u8 bw);
+u8 center_chs_2g(u8 bw, u8 id);
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+extern u8 center_ch_5g_20m[CENTER_CH_5G_20M_NUM];
+extern u8 center_ch_5g_40m[CENTER_CH_5G_40M_NUM];
+extern u8 center_ch_5g_80m[CENTER_CH_5G_80M_NUM];
+extern u8 center_ch_5g_160m[CENTER_CH_5G_160M_NUM];
+extern u8 center_ch_5g_all[CENTER_CH_5G_ALL_NUM];
+
+u8 center_chs_5g_num(u8 bw);
+u8 center_chs_5g(u8 bw, u8 id);
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+extern u8 center_ch_6g_20m[CENTER_CH_6G_20M_NUM];
+extern u8 center_ch_6g_40m[CENTER_CH_6G_40M_NUM];
+extern u8 center_ch_6g_80m[CENTER_CH_6G_80M_NUM];
+extern u8 center_ch_6g_160m[CENTER_CH_6G_160M_NUM];
+
+u8 center_chs_6g_num(u8 bw);
+u8 center_chs_6g(u8 bw, u8 id);
+#endif
+
+extern u8 (*center_chs_num_of_band[BAND_MAX])(u8 bw);
+extern u8 (*center_chs_of_band[BAND_MAX])(u8 bw, u8 id);
+
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_scch_by_cch_offset(u8 cch, u8 bw, u8 offset);
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_scch_by_cch_opch(u8 cch, u8 bw, u8 opch);
+
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_op_chs_by_cch_bw(u8 cch, u8 bw, u8 **op_chs, u8 *op_ch_num);
+
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_offset_by_chbw(u8 ch, u8 bw, u8 *r_offset);
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_center_ch(u8 ch, u8 bw, u8 offset);
+
+RTW_FUNC_2G_5G_ONLY bool rtw_is_chbw_grouped(u8 ch_a, u8 bw_a, u8 offset_a, u8 ch_b, u8 bw_b, u8 offset_b);
+RTW_FUNC_2G_5G_ONLY void rtw_sync_chbw(u8 *req_ch, u8 *req_bw, u8 *req_offset, u8 *g_ch, u8 *g_bw, u8 *g_offset);
+
+u8 rtw_get_scch_by_bcch_offset(enum band_type band, u8 cch, u8 bw, u8 offset);
+u8 rtw_get_scch_by_bcch_opch(enum band_type band, u8 cch, u8 bw, u8 opch);
+
+u8 rtw_get_op_chs_by_bcch_bw(enum band_type band, u8 cch, u8 bw, u8 **op_chs, u8 *op_ch_num);
+
+u8 rtw_get_offset_by_bchbw(enum band_type band, u8 ch, u8 bw, u8 *r_offset);
+u8 rtw_get_offsets_by_bchbw(enum band_type band, u8 ch, u8 bw, u8 *r_offset, u8 *r_offset_num);
+u8 rtw_get_center_ch_by_band(enum band_type band, u8 ch, u8 bw, u8 offset);
+
+bool rtw_is_bchbw_grouped(enum band_type band_a, u8 ch_a, u8 bw_a, u8 offset_a
+	, enum band_type band_b, u8 ch_b, u8 bw_b, u8 offset_b);
+void rtw_sync_bchbw(enum band_type *req_band, u8 *req_ch, u8 *req_bw, u8 *req_offset
+	, enum band_type *g_band, u8 *g_ch, u8 *g_bw, u8 *g_offset);
+
+typedef enum _CAPABILITY {
+	cESS			= 0x0001,
+	cIBSS			= 0x0002,
+	cPollable		= 0x0004,
+	cPollReq			= 0x0008,
+	cPrivacy		= 0x0010,
+	cShortPreamble	= 0x0020,
+	cPBCC			= 0x0040,
+	cChannelAgility	= 0x0080,
+	cSpectrumMgnt	= 0x0100,
+	cQos			= 0x0200,	/* For HCCA, use with CF-Pollable and CF-PollReq */
+	cShortSlotTime	= 0x0400,
+	cAPSD			= 0x0800,
+	cRM				= 0x1000,	/* RRM (Radio Request Measurement) */
+	cDSSS_OFDM	= 0x2000,
+	cDelayedBA		= 0x4000,
+	cImmediateBA	= 0x8000,
+} CAPABILITY, *PCAPABILITY;
+
+enum	_REG_PREAMBLE_MODE {
+	PREAMBLE_LONG	= 1,
+	PREAMBLE_AUTO	= 2,
+	PREAMBLE_SHORT	= 3,
+};
+
+#define rf_path_char(path) (((path) >= RF_PATH_MAX) ? 'X' : 'A' + (path))
+
+#ifdef CONFIG_NARROWBAND_SUPPORTING
+enum nb_config {
+	RTW_NB_CONFIG_NONE		= 0,
+	RTW_NB_CONFIG_WIDTH_5	= 5,
+	RTW_NB_CONFIG_WIDTH_10	= 6,
+};
+#endif
+
+extern const char *const _rtw_band_str[];
+#define band_str(band) (((band) >= BAND_MAX) ? _rtw_band_str[BAND_MAX] : _rtw_band_str[(band)])
+
+extern const u8 _band_to_band_cap[];
+#define band_to_band_cap(band) (((band) >= BAND_MAX) ? _band_to_band_cap[BAND_MAX] : _band_to_band_cap[(band)])
+
+
+extern const char *const _ch_width_str[];
+#define ch_width_str(bw) (((bw) < CHANNEL_WIDTH_MAX) ? _ch_width_str[(bw)] : "CHANNEL_WIDTH_MAX")
+
+extern const u8 _ch_width_to_bw_cap[];
+#define ch_width_to_bw_cap(bw) (((bw) < CHANNEL_WIDTH_MAX) ? _ch_width_to_bw_cap[(bw)] : 0)
+
+enum opc_bw {
+	OPC_BW20		= 0,
+	OPC_BW40PLUS	= 1,
+	OPC_BW40MINUS	= 2,
+	OPC_BW40		= 3,
+	OPC_BW80		= 4,
+	OPC_BW160		= 5,
+	OPC_BW80P80		= 6,
+	OPC_BW_NUM,
+};
+
+extern const char *const _opc_bw_str[OPC_BW_NUM];
+#define opc_bw_str(bw) (((bw) < OPC_BW_NUM) ? _opc_bw_str[(bw)] : "N/A")
+
+extern const u8 _opc_bw_to_ch_width[OPC_BW_NUM];
+#define opc_bw_to_ch_width(bw) (((bw) < OPC_BW_NUM) ? _opc_bw_to_ch_width[(bw)] : CHANNEL_WIDTH_MAX)
+
+/* global op class APIs */
+struct op_class_t {
+	u8 class_id;
+	enum band_type band;
+	enum opc_bw bw;
+	u8 *len_ch_attr;
+};
+
+#define OPC_CH_LIST_LEN(_opc) (_opc.len_ch_attr[0])
+#define OPC_CH_LIST_CH(_opc, _i) (_opc.len_ch_attr[_i + 1])
+
+extern const struct op_class_t global_op_class[];
+extern const int global_op_class_num;
+const struct op_class_t *get_global_op_class_by_id(u8 gid);
+bool is_valid_global_op_class_id(u8 gid);
+bool is_valid_global_op_class_ch(const struct op_class_t *opc, u8 ch);
+s16 get_sub_op_class(u8 gid, u8 ch);
+void dump_global_op_class(void *sel);
+RTW_FUNC_2G_5G_ONLY u8 rtw_get_op_class_by_chbw(u8 ch, u8 bw, u8 offset);
+u8 rtw_get_op_class_by_bchbw(enum band_type band, u8 ch, u8 bw, u8 offset);
+u8 rtw_get_bw_offset_by_op_class_ch(u8 gid, u8 ch, u8 *bw, u8 *offset);
+enum band_type rtw_get_band_by_op_class(u8 op_class);
+#ifdef CONFIG_ECSA_PHL
+int alink_get_supported_op_class(struct _ADAPTER_LINK *padapter_link, u8 *op_set, int len);
+int get_supported_op_class(_adapter *padapter, u8 *op_set, int len);
+#endif
+
+struct op_ch_t {
+	u8 ch;
+	u8 static_non_op:1; /* not in channel list */
+	u8 no_ir:1;
+	s16 max_txpwr; /* mBm */
+};
+
+struct op_class_pref_t {
+	u8 class_id;
+	enum band_type band;
+	enum opc_bw bw;
+	u8 ch_num; /* number of chs */
+	u8 op_ch_num; /* channel number which is not static non operable */
+	u8 ir_ch_num; /* channel number which can init radiation */
+	struct op_ch_t chs[];
+};
+
+struct rf_ctl_t;
+int rtw_rfctl_op_class_pref_init(struct rf_ctl_t *rfctl, u8 band_bmp, u8 bw_bmp[]);
+void rtw_rfctl_op_class_pref_deinit(struct rf_ctl_t *rfctl);
+
+#define REG_BEACON_HINT		0
+#define REG_TXPWR_CHANGE	1
+#define REG_CHANGE			2
+
+void op_class_pref_apply_regulatory(struct rf_ctl_t *rfctl, u8 reason);
+
+struct rf_ctl_t;
+void dump_cap_spt_op_class_ch(void *sel, struct rf_ctl_t *rfctl, bool detail);
+void dump_reg_spt_op_class_ch(void *sel, struct rf_ctl_t *rfctl, bool detail);
+void dump_cur_spt_op_class_ch(void *sel, struct rf_ctl_t *rfctl, bool detail);
+
+typedef enum _VHT_DATA_SC {
+	VHT_DATA_SC_DONOT_CARE = 0,
+	VHT_DATA_SC_20_UPPER_OF_80MHZ = 1,
+	VHT_DATA_SC_20_LOWER_OF_80MHZ = 2,
+	VHT_DATA_SC_20_UPPERST_OF_80MHZ = 3,
+	VHT_DATA_SC_20_LOWEST_OF_80MHZ = 4,
+	VHT_DATA_SC_20_RECV1 = 5,
+	VHT_DATA_SC_20_RECV2 = 6,
+	VHT_DATA_SC_20_RECV3 = 7,
+	VHT_DATA_SC_20_RECV4 = 8,
+	VHT_DATA_SC_40_UPPER_OF_80MHZ = 9,
+	VHT_DATA_SC_40_LOWER_OF_80MHZ = 10,
+} VHT_DATA_SC, *PVHT_DATA_SC_E;
+
+typedef enum _PROTECTION_MODE {
+	PROTECTION_MODE_AUTO = 0,
+	PROTECTION_MODE_FORCE_ENABLE = 1,
+	PROTECTION_MODE_FORCE_DISABLE = 2,
+} PROTECTION_MODE, *PPROTECTION_MODE;
+
+typedef enum _RF_TX_NUM {
+	RF_1TX = 0,
+	RF_2TX,
+	RF_3TX,
+	RF_4TX,
+	RF_MAX_TX_NUM,
+	RF_TX_NUM_NONIMPLEMENT,
+} RF_TX_NUM;
+
+#define RF_TYPE_VALID(rf_type) (rf_type < RF_TYPE_MAX)
+
+extern const u8 _rf_type_to_rf_tx_cnt[];
+#define rf_type_to_rf_tx_cnt(rf_type) (RF_TYPE_VALID(rf_type) ? _rf_type_to_rf_tx_cnt[rf_type] : 0)
+
+extern const u8 _rf_type_to_rf_rx_cnt[];
+#define rf_type_to_rf_rx_cnt(rf_type) (RF_TYPE_VALID(rf_type) ? _rf_type_to_rf_rx_cnt[rf_type] : 0)
+
+extern const char *const _rf_type_to_rfpath_str[];
+#define rf_type_to_rfpath_str(rf_type) (RF_TYPE_VALID(rf_type) ? _rf_type_to_rfpath_str[rf_type] : "UNKNOWN")
+
+enum rf_type trx_num_to_rf_type(u8 tx_num, u8 rx_num);
+enum rf_type trx_bmp_to_rf_type(u8 tx_bmp, u8 rx_bmp);
+bool rf_type_is_a_in_b(enum rf_type a, enum rf_type b);
+u8 rtw_restrict_trx_path_bmp_by_rftype(u8 trx_path_bmp, enum rf_type type, u8 *tx_num, u8 *rx_num);
+
+RTW_FUNC_2G_5G_ONLY int rtw_ch2freq(int chan);
+int rtw_bch2freq(enum band_type band, int ch);
+int rtw_freq2ch(int freq);
+enum band_type rtw_freq2band(int freq);
+enum channel_width rtw_frange_to_bw(u32 hi, u32 lo);
+bool rtw_freq_consecutive(int a, int b);
+bool rtw_bcchbw_to_freq_range(enum band_type band, u8 c_ch, u8 bw, u32 *hi, u32 *lo);
+bool rtw_bchbw_to_freq_range(enum band_type band, u8 ch, u8 bw, u8 offset, u32 *hi, u32 *lo);
+RTW_FUNC_2G_5G_ONLY bool rtw_chbw_to_freq_range(u8 ch, u8 bw, u8 offset, u32 *hi, u32 *lo);
+
+struct rf_ctl_t;
+
+#if CONFIG_TXPWR_LIMIT
+void dump_regd_exc_list(void *sel, struct rf_ctl_t *rfctl);
+#endif
+
+RTW_FUNC_2G_5G_ONLY static inline bool rtw_is_2g_ch(u8 ch) { return ch >= 1 && ch <= 14; }
+RTW_FUNC_2G_5G_ONLY static inline bool rtw_is_5g_ch(u8 ch) { return ch >= 36 && ch <= 177; }
+RTW_FUNC_2G_5G_ONLY static inline enum band_type rtw_get_band_type(u8 chan)
+{
+	return (chan > 14) ? BAND_ON_5G : BAND_ON_24G;
+}
+
+bool rtw_is_long_cac_range(u32 hi, u32 lo, u8 dfs_region);
+RTW_FUNC_2G_5G_ONLY bool rtw_is_long_cac_ch(u8 ch, u8 bw, u8 offset, u8 dfs_region);
+bool rtw_is_long_cac_bch(enum band_type band, u8 ch, u8 bw, u8 offset, u8 dfs_region);
+
+#endif /* _RTL8711_RF_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm.h
new file mode 100644
index 000000000000..d5fa19d7ef2b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm.h
@@ -0,0 +1,111 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __RTW_RM_H_
+#define __RTW_RM_H_
+
+u8 rm_post_event_hdl(_adapter *padapter, u8 *pbuf);
+
+#define RM_TIMER_NUM 		32
+#define RM_ALL_MEAS		BIT(1)
+#define RM_ID_FOR_ALL(aid)	((aid<<16)|RM_ALL_MEAS)
+
+#define RM_CAP_ARG(x) ((u8 *)(x))[4], ((u8 *)(x))[3], ((u8 *)(x))[2], ((u8 *)(x))[1], ((u8 *)(x))[0]
+#define RM_CAP_FMT "%02x %02x%02x %02x%02x"
+
+#ifndef MIN
+#define MIN(x, y) (((x) < (y)) ? (x) : (y))
+#endif
+
+#if defined(CONFIG_RTW_80211K) && !defined(CONFIG_RTW_FSM_RRM)
+/* remember to modify rm_event_name() when adding new event */
+enum RM_EV_ID {
+	RM_EV_state_in,
+	RM_EV_busy_timer_expire,
+	RM_EV_delay_timer_expire,
+	RM_EV_meas_timer_expire,
+	RM_EV_retry_timer_expire,
+	RM_EV_repeat_delay_expire,
+	RM_EV_request_timer_expire,
+	RM_EV_wait_report,
+	RM_EV_start_meas,
+	RM_EV_survey_done,
+	RM_EV_recv_rep,
+	RM_EV_cancel,
+	RM_EV_state_out,
+	RM_EV_max
+};
+
+struct rm_event {
+	u32 rmid;
+	enum RM_EV_ID evid;
+	_list list;
+};
+
+struct rm_clock {
+	struct rm_obj *prm;
+	ATOMIC_T counter;
+	enum RM_EV_ID evid;
+};
+
+struct rm_priv {
+	u8 enable;
+	_queue ev_queue;
+	_queue rm_queue;
+	_timer rm_timer;
+
+	struct rm_clock clock[RM_TIMER_NUM];
+	u8 rm_en_cap_def[5];
+	u8 rm_en_cap_assoc[5];
+	u8 meas_token;
+
+	/* rm debug */
+	void *prm_sel;
+};
+
+#define	MAX_CH_NUM_IN_OP_CLASS	11
+typedef struct _RT_OPERATING_CLASS {
+	int	global_op_class;
+	int	Len;
+	u8	Channel[MAX_CH_NUM_IN_OP_CLASS];
+} RT_OPERATING_CLASS, *PRT_OPERATING_CLASS;
+
+int rtw_init_rm(_adapter *padapter);
+int rtw_free_rm_priv(_adapter *padapter);
+
+unsigned int rm_on_action(_adapter *padapter, union recv_frame *precv_frame);
+void rtw_ap_parse_sta_rm_en_cap(_adapter *padapter,
+	struct sta_info *psta, struct rtw_ieee802_11_elems *elems);
+
+int rm_post_event(_adapter *padapter, u32 rmid, enum RM_EV_ID evid);
+void rm_handler(_adapter *padapter, struct rm_event *pev);
+int rm_get_chset(struct rm_obj *prm);
+
+u8 rm_add_nb_req(_adapter *padapter, struct sta_info *psta);
+/* from ioctl */
+int rm_send_bcn_reqs(_adapter *padapter, u8 *sta_addr, u8 op_class, u8 ch,
+	u16 measure_duration, u8 measure_mode, u8 *bssid, u8 *ssid,
+	u8 reporting_detail,
+	u8 n_ap_ch_rpt, struct _RT_OPERATING_CLASS *rpt,
+	u8 n_elem_id, u8 *elem_id_list);
+void indicate_beacon_report(u8 *sta_addr,
+	u8 n_measure_rpt, u32 elem_len, u8 *elem);
+
+void RM_IE_handler(_adapter *padapter, PNDIS_802_11_VARIABLE_IEs pIE);
+void rm_update_cap(u8 *frame_head, _adapter *pa, u32 pktlen, int offset);
+#endif /*CONFIG_RTW_80211K */
+
+
+#endif /* __RTW_RM_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm_fsm.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm_fsm.h
new file mode 100644
index 000000000000..8921303239df
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm_fsm.h
@@ -0,0 +1,406 @@
+
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __RTW_RM_FSM_H_
+#define __RTW_RM_FSM_H_
+
+#ifdef CONFIG_RTW_80211K
+
+#define RM_SUPPORT_IWPRIV_DBG	1
+#define RM_MORE_DBG_MSG		0
+
+#define DBG_BCN_REQ_DETAIL	0
+#define DBG_BCN_REQ_WILDCARD	0
+#define DBG_BCN_REQ_SSID	0
+#define DBG_BCN_REQ_SSID_NAME	"RealKungFu"
+
+#define RM_REQ_TIMEOUT		10000	/* 10 seconds */
+#define RM_MEAS_TIMEOUT		10000	/* 10 seconds */
+#define RM_REPT_SCAN_INTVL	5000	/*  5 seconds */
+#define RM_REPT_POLL_INTVL	2000	/*  2 seconds */
+#define RM_COND_INTVL		2000	/*  2 seconds */
+#define RM_SCAN_DENY_TIMES	10
+#define RM_BUSY_TRAFFIC_TIMES	10
+#define RM_WAIT_BUSY_TIMEOUT	1000	/*  1 seconds */
+
+#define MEAS_REQ_MOD_PARALLEL	BIT(0)
+#define MEAS_REQ_MOD_ENABLE	BIT(1)
+#define MEAS_REQ_MOD_REQUEST	BIT(2)
+#define MEAS_REQ_MOD_REPORT	BIT(3)
+#define MEAS_REQ_MOD_DUR_MAND	BIT(4)
+
+#define MEAS_REP_MOD_LATE	BIT(0)
+#define MEAS_REP_MOD_INCAP	BIT(1)
+#define MEAS_REP_MOD_REFUSE	BIT(2)
+
+#define RM_MASTER		BIT(0)	/* STA who issue meas_req */
+#define RM_SLAVE		0	/* STA who do measurement */
+
+#define CLOCK_UNIT		10	/* ms */
+#define RTW_MAX_NB_RPT_IE_NUM	16
+
+#define RM_GET_AID(rmid)	((rmid&0xffff0000)>>16)
+#define RM_IS_ID_FOR_ALL(rmid)	(rmid&RM_ALL_MEAS)
+
+/* IEEE 802.11-2012 Table 8-59 Measurement Type definitions
+*  for measurement request
+*  modify rm_meas_type_req_name() when adding new type
+*/
+enum meas_type_of_req {
+	basic_req,	/* spectrum measurement */
+	cca_req,
+	rpi_histo_req,
+	ch_load_req,
+	noise_histo_req,
+	bcn_req,
+	frame_req,
+	sta_statis_req,
+	lci_req,
+	meas_type_req_max,
+};
+
+/* IEEE 802.11-2012 Table 8-81 Measurement Type definitions
+*  for measurement report
+*  modify rm_type_rep_name() when adding new type
+*/
+enum meas_type_of_rep {
+	basic_rep,	/* spectrum measurement */
+	cca_rep,
+	rpi_histo_rep,
+	ch_load_rep,	/* radio measurement */
+	noise_histo_rep,
+	bcn_rep,
+	frame_rep,
+	sta_statis_rep,	/* Radio measurement and WNM */
+	lci_rep,
+	meas_type_rep_max
+};
+
+/*
+* Beacon request
+*/
+/* IEEE 802.11-2012 Table 8-64 Measurement mode for Beacon Request element */
+enum bcn_req_meas_mode {
+	bcn_req_passive,
+	bcn_req_active,
+	bcn_req_bcn_table
+};
+
+/* IEEE 802.11-2012 Table 8-65 optional subelement IDs for Beacon Request */
+enum bcn_req_opt_sub_id{
+	bcn_req_ssid = 0,		/* len 0-32 */
+	bcn_req_rep_info = 1,		/* len 2 */
+	bcn_req_rep_detail = 2,		/* len 1 */
+	bcn_req_req = 10,		/* len 0-237 */
+	bcn_req_ap_ch_rep = 51,		/* len 1-237 */
+	bcn_req_last_bcn_rpt_ind = 164	/* len 1 */
+};
+
+/* IEEE 802.11-2012 Table 8-66 Reporting condition of Beacon Report */
+enum bcn_rep_cound_id{
+	bcn_rep_cond_immediately,	/* default */
+	bcn_req_cond_rcpi_greater,
+	bcn_req_cond_rcpi_less,
+	bcn_req_cond_rsni_greater,
+	bcn_req_cond_rsni_less,
+	bcn_req_cond_max
+};
+
+struct opt_rep_info {
+	u8 cond;
+	u8 threshold;
+};
+
+#define BCN_REQ_OPT_MAX_NUM		16
+#define BCN_REQ_REQ_OPT_MAX_NUM		16
+#define BCN_REQ_OPT_AP_CH_RPT_MAX_NUM	12
+struct bcn_req_opt {
+	/* all req cmd id */
+	u8 opt_id[BCN_REQ_OPT_MAX_NUM];
+	u8 opt_id_num;
+	u8 req_id_num;
+	u8 req_id[BCN_REQ_REQ_OPT_MAX_NUM];
+	u8 rep_detail;
+	u8 rep_last_bcn_ind;
+	NDIS_802_11_SSID ssid;
+
+	/* bcn report condition */
+	struct opt_rep_info rep_cond;
+
+	u8 ap_ch_rpt_num;
+	struct _RT_OPERATING_CLASS *ap_ch_rpt[BCN_REQ_OPT_AP_CH_RPT_MAX_NUM];
+
+	/* 0:default(Report to be issued after each measurement) */
+	u8 *req_start;	/*id : 10 request;start  */
+	u8 req_len;	/*id : 10 request;length */
+};
+
+/*
+* channel load
+*/
+/* IEEE 802.11-2012 Table 8-60 optional subelement IDs for channel load request */
+enum ch_load_opt_sub_id{
+	ch_load_rsvd,
+	ch_load_rep_info
+};
+
+/* IEEE 802.11-2012 Table 8-61 Reporting condition for channel load Report */
+enum ch_load_cound_id{
+	ch_load_cond_immediately,	/* default */
+	ch_load_cond_anpi_equal_greater,
+	ch_load_cond_anpi_equal_less,
+	ch_load_cond_max
+};
+
+/*
+* Noise histogram
+*/
+/* IEEE 802.11-2012 Table 8-62 optional subelement IDs for noise histogram */
+enum noise_histo_opt_sub_id{
+	noise_histo_rsvd,
+	noise_histo_rep_info
+};
+
+/* IEEE 802.11-2012 Table 8-63 Reporting condition for noise historgarm Report */
+enum noise_histo_cound_id{
+	noise_histo_cond_immediately,	/* default */
+	noise_histo_cond_anpi_equal_greater,
+	noise_histo_cond_anpi_equal_less,
+	noise_histo_cond_max
+};
+
+struct meas_req_opt {
+	/* report condition */
+	struct opt_rep_info rep_cond;
+};
+
+/*
+* State machine
+*/
+
+enum RM_STATE {
+	RM_ST_IDLE,
+	RM_ST_DO_MEAS,
+	RM_ST_WAIT_MEAS,
+	RM_ST_SEND_REPORT,
+	RM_ST_RECV_REPORT,
+	RM_ST_END,
+	RM_ST_MAX
+};
+
+struct rm_meas_req {
+	u8 category;
+	u8 action_code;		/* T8-206  */
+	u8 diag_token;
+	u16 rpt;
+
+	u8 e_id;
+	u8 len;
+	u8 m_token;
+	u8 m_mode;		/* req:F8-105, rep:F8-141 */
+	u8 m_type;		/* T8-59 */
+	u8 op_class;
+	u8 ch_num;
+	u16 rand_intvl;		/* units of TU */
+	u16 meas_dur;		/* units of TU */
+
+	u8 scan_mode;
+	u8 bssid[6];		/* for bcn_req */
+
+	u8 *pssid;
+	u8 *opt_s_elem_start;
+	int opt_s_elem_len;
+
+	s8 tx_pwr_used;		/* for link measurement */
+	s8 tx_pwr_max;		/* for link measurement */
+
+	union {
+		struct bcn_req_opt bcn;
+		struct meas_req_opt clm;
+		struct meas_req_opt nhm;
+	}opt;
+
+	struct rtw_ieee80211_channel ch_set[RTW_CHANNEL_SCAN_AMOUNT];
+	u8 ch_set_ch_amount;
+	s8 rx_pwr;		/* in dBm */
+	u8 rx_bw;
+	u8 rx_rsni;
+	u16 rx_rate;
+	u8 band;
+};
+
+struct rm_meas_rep {
+	u8 category;
+	u8 action_code;		/* T8-206  */
+	u8 diag_token;
+
+	u8 e_id;		/* T8-54, 38 request; 39 report */
+	u8 len;
+	u8 m_token;
+	u8 m_mode;		/* req:F8-105, rep:F8-141 */
+	u8 m_type;		/* T8-59 */
+	u8 op_class;
+	u8 ch_num;
+
+	u8 ch_load;
+	u8 anpi;
+	u8 ipi[11];
+
+	u16 rpt;
+	u8 scan_mode;
+	u8 bssid[6];		/* for bcn_req */
+	u8 band;
+};
+
+#define MAX_RM_PKT_NUM	32
+#define MAX_RM_AP_NUM	128
+struct data_buf {
+	u8 *pbuf;
+	u16 len;
+};
+
+struct rm_obj {
+
+	/* aid << 16 
+		|diag_token << 8
+		|B(1) 1/0:All_AID/UNIC
+		|B(0) 1/0:RM_MASTER/RM_SLAVE */
+	u32 rmid;
+
+	enum RM_STATE state;
+	struct rm_meas_req q;
+	struct rm_meas_rep p;
+	struct sta_info *psta;
+	struct rm_clock *pclock;
+
+	/* meas report */
+	u64 meas_start_time;
+	u64 meas_end_time;
+	int wait_busy;
+	u8 poll_mode;
+
+	struct data_buf buf[MAX_RM_PKT_NUM];
+	bool from_ioctl;
+
+	struct wlan_network *ap[MAX_RM_AP_NUM];
+	u8 ap_num;
+
+	_list list;
+};
+
+/*
+* Measurement
+*/
+struct opt_subelement {
+	u8 id;
+	u8 length;
+	u8 *data;
+};
+
+/* 802.11-2012 Table 8-206 Radio Measurment Action field */
+enum rm_action_code {
+	RM_ACT_RADIO_MEAS_REQ,
+	RM_ACT_RADIO_MEAS_REP,
+	RM_ACT_LINK_MEAS_REQ,
+	RM_ACT_LINK_MEAS_REP,
+	RM_ACT_NB_REP_REQ,	/* 4 */
+	RM_ACT_NB_REP_RESP,
+	RM_ACT_RESV,
+	RM_ACT_MAX
+};
+
+/* 802.11-2012 Table 8-119 RM Enabled Capabilities definition */
+enum rm_cap_en {
+	RM_LINK_MEAS_CAP_EN,
+	RM_NB_REP_CAP_EN,		/* neighbor report */
+	RM_PARAL_MEAS_CAP_EN,		/* parallel report */
+	RM_REPEAT_MEAS_CAP_EN,
+	RM_BCN_PASSIVE_MEAS_CAP_EN,
+	RM_BCN_ACTIVE_MEAS_CAP_EN,
+	RM_BCN_TABLE_MEAS_CAP_EN,
+	RM_BCN_MEAS_REP_COND_CAP_EN,	/* conditions */
+
+	RM_FRAME_MEAS_CAP_EN,
+	RM_CH_LOAD_CAP_EN,
+	RM_NOISE_HISTO_CAP_EN,		/* noise historgram */
+	RM_STATIS_MEAS_CAP_EN,		/* statistics */
+	RM_LCI_MEAS_CAP_EN,		/* 12 */
+	RM_LCI_AMIMUTH_CAP_EN,
+	RM_TRANS_STREAM_CAT_MEAS_CAP_EN,
+	RM_TRIG_TRANS_STREAM_CAT_MEAS_CAP_EN,
+
+	RM_AP_CH_REP_CAP_EN,
+	RM_RM_MIB_CAP_EN,
+	RM_OP_CH_MAX_MEAS_DUR0,		/* 18-20 */
+	RM_OP_CH_MAX_MEAS_DUR1,
+	RM_OP_CH_MAX_MEAS_DUR2,
+	RM_NONOP_CH_MAX_MEAS_DUR0,	/* 21-23 */
+	RM_NONOP_CH_MAX_MEAS_DUR1,
+	RM_NONOP_CH_MAX_MEAS_DUR2,
+
+	RM_MEAS_PILOT_CAP0,		/* 24-26 */
+	RM_MEAS_PILOT_CAP1,
+	RM_MEAS_PILOT_CAP2,
+	RM_MEAS_PILOT_TRANS_INFO_CAP_EN,
+	RM_NB_REP_TSF_OFFSET_CAP_EN,
+	RM_RCPI_MEAS_CAP_EN,		/* 29 */
+	RM_RSNI_MEAS_CAP_EN,
+	RM_BSS_AVG_ACCESS_DELAY_CAP_EN,
+
+	RM_AVALB_ADMIS_CAPACITY_CAP_EN,
+	RM_ANT_CAP_EN,
+	RM_RSVD,			/* 34-39 */
+	RM_MAX
+};
+
+char *rm_state_name(enum RM_STATE state);
+char *rm_event_name(enum RM_EV_ID evid);
+char *rm_type_req_name(u8 meas_type);
+int _rm_post_event(_adapter *padapter, u32 rmid, enum RM_EV_ID evid);
+int rm_enqueue_rmobj(_adapter *padapter, struct rm_obj *obj, bool to_head);
+
+void rm_free_rmobj(struct rm_obj *prm);
+struct rm_obj *rm_alloc_rmobj(_adapter *padapter);
+struct rm_obj *rm_get_rmobj(_adapter *padapter, u32 rmid);
+struct sta_info *rm_get_psta(_adapter *padapter, u32 rmid);
+
+int retrieve_radio_meas_result(struct rm_obj *prm);
+int rm_radio_meas_report_cond(struct rm_obj *prm);
+int rm_recv_radio_mens_req(_adapter *padapter,
+	union recv_frame *precv_frame,struct sta_info *psta);
+int rm_recv_radio_mens_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta);
+int rm_recv_link_mens_req(_adapter *padapter,
+	union recv_frame *precv_frame,struct sta_info *psta);
+int rm_recv_link_mens_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta);
+int rm_radio_mens_nb_rep(_adapter *padapter,
+	union recv_frame *precv_frame, struct sta_info *psta);
+int issue_null_reply(struct rm_obj *prm);
+int issue_beacon_rep(struct rm_obj *prm);
+int issue_nb_req(struct rm_obj *prm);
+int issue_radio_meas_req(struct rm_obj *prm);
+int issue_radio_meas_rep(struct rm_obj *prm);
+int issue_link_meas_req(struct rm_obj *prm);
+int issue_link_meas_rep(struct rm_obj *prm);
+
+void rm_set_rep_mode(struct rm_obj *prm, u8 mode);
+
+int ready_for_scan(struct rm_obj *prm);
+int rm_sitesurvey(struct rm_obj *prm);
+
+#endif /*CONFIG_RTW_80211K*/
+#endif /*__RTW_RM_FSM_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm_util.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm_util.h
new file mode 100644
index 000000000000..ae133b07243a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_rm_util.h
@@ -0,0 +1,46 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef _RTW_RM_UTIL_H_
+#define _RTW_RM_UTIL_H_
+/*
+ * define the following channels as the max channels in each channel plan.
+ * 2G, total 14 chnls
+ * {1,2,3,4,5,6,7,8,9,10,11,12,13,14}
+ * 5G, total 25 chnls
+ * {36,40,44,48,52,56,60,64,100,104,108,112,116,120,124,128,132,136,140,144,149,153,157,161,165}
+ */
+#ifndef MAX
+#define MAX(x, y) (((x) > (y)) ? (x) : (y))
+#endif
+
+u8 rm_get_ch_set(u8 op_class, struct rtw_ieee80211_channel *pch_set, u8 ch_num);
+u8 rm_get_ch_set_from_bcn_req_opt(struct bcn_req_opt *opt,
+	struct rtw_ieee80211_channel *pch_set, u8 pch_num);
+u8 rm_get_bcn_rsni(struct rm_obj *prm, struct wlan_network *pnetwork);
+u8 rm_get_bcn_rcpi(struct rm_obj *prm, struct wlan_network *pnetwork);
+u8 rm_get_frame_rsni(struct rm_obj *prm, union recv_frame *pframe);
+u8 translate_percentage_to_rcpi(u32 SignalStrengthIndex);
+u8 translate_dbm_to_rcpi(s8 SignalPower);
+u8 rm_gen_dialog_token(_adapter *padapter);
+u8 rm_gen_meas_token(_adapter *padapter);
+u32 rm_gen_rmid(_adapter *padapter, struct rm_obj *prm, u8 role);
+int is_wildcard_bssid(u8 *bssid);
+
+int rm_get_path_a_max_tx_power(_adapter *adapter, s8 *path_a);
+int rm_get_tx_power(_adapter *adapter, enum band_type band, enum MGN_RATE rate, s8 *pwr);
+int rm_get_rx_sensitivity(_adapter *adapter, enum channel_width bw, enum MGN_RATE rate, s8 *pwr);
+
+#endif /* _RTW_RM_UTIL_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_scan.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_scan.h
new file mode 100644
index 000000000000..b04d57f5c895
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_scan.h
@@ -0,0 +1,304 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_SCAN_H_
+#define __RTW_SCAN_H_
+
+/*rtw_mlme.h*/
+void rtw_drv_scan_by_self(_adapter *padapter, u8 reason);
+void rtw_scan_abort_no_wait(_adapter *adapter);
+u32 rtw_scan_abort(_adapter *adapter, u32 timeout_ms);
+
+void rtw_survey_event_callback(_adapter *adapter, u8 *pbuf);
+void rtw_surveydone_event_callback(_adapter *adapter, u8 *pbuf);
+int rtw_scan_ch_decision(_adapter *padapter, struct rtw_ieee80211_channel *out,
+	u32 out_num, struct rtw_ieee80211_channel *in, u32 in_num, bool no_sparse);
+
+enum {
+	SS_DENY_MP_MODE,
+	SS_DENY_RSON_SCANING,
+	SS_DENY_BLOCK_SCAN,
+	SS_DENY_BY_DRV,
+	SS_DENY_SELF_AP_UNDER_WPS,
+	SS_DENY_SELF_AP_UNDER_LINKING,
+	SS_DENY_SELF_AP_UNDER_SURVEY,
+	/*SS_DENY_SELF_STA_UNDER_WPS,*/
+	SS_DENY_SELF_STA_UNDER_LINKING,
+	SS_DENY_SELF_STA_UNDER_SURVEY,
+	SS_DENY_BUDDY_UNDER_LINK_WPS,
+	SS_DENY_BUDDY_UNDER_SURVEY,
+	SS_DENY_BUSY_TRAFFIC,
+	SS_ALLOW,
+#ifdef DBG_LA_MODE
+	SS_DENY_LA_MODE,
+#endif
+	SS_DENY_ADAPTIVITY,
+};
+u8 _rtw_sitesurvey_condition_check(const char *caller, _adapter *adapter, bool check_sc_interval);
+#define rtw_sitesurvey_condition_check(adapter, check_sc_interval) _rtw_sitesurvey_condition_check(__func__, adapter, check_sc_interval)
+
+#ifdef CONFIG_SET_SCAN_DENY_TIMER
+bool rtw_is_scan_deny(_adapter *adapter);
+void rtw_clear_scan_deny(_adapter *adapter);
+void rtw_set_scan_deny_timer_hdl(void *ctx);
+void rtw_set_scan_deny(_adapter *adapter, u32 ms);
+#else
+#define rtw_is_scan_deny(adapter) _FALSE
+#define rtw_clear_scan_deny(adapter) do {} while (0)
+#define rtw_set_scan_deny(adapter, ms) do {} while (0)
+#endif
+
+#ifdef CONFIG_RTW_ACS
+u8 rtw_set_acs_sitesurvey(_adapter *adapter);
+#endif
+
+/*rtw_mlme_ext.h*/
+void sitesurvey_set_offch_state(_adapter *adapter, u8 scan_state);
+
+/*const char *scan_state_str(u8 state);*/
+#define mlmeext_scan_state(mlmeext) ((mlmeext)->sitesurvey_res.state)
+/*#define mlmeext_scan_state_str(mlmeext) scan_state_str((mlmeext)->sitesurvey_res.state)*/
+#define mlmeext_chk_scan_state(mlmeext, _state) ((mlmeext)->sitesurvey_res.state == (_state))
+#define mlmeext_set_scan_state(mlmeext, _state) \
+	do { \
+		((mlmeext)->sitesurvey_res.state = (_state)); \
+		((mlmeext)->sitesurvey_res.next_state = (_state)); \
+		rtw_mi_update_iface_status(&((container_of(mlmeext, _adapter, mlmeextpriv)->mlmepriv)), 0); \
+		/* RTW_INFO("set_scan_state:%s\n", scan_state_str(_state)); */ \
+		sitesurvey_set_offch_state(container_of(mlmeext, _adapter, mlmeextpriv), _state); \
+	} while (0)
+
+#if 0
+#define mlmeext_scan_next_state(mlmeext) ((mlmeext)->sitesurvey_res.next_state)
+#define mlmeext_set_scan_next_state(mlmeext, _state) \
+	do { \
+		((mlmeext)->sitesurvey_res.next_state = (_state)); \
+		/* RTW_INFO("set_scan_next_state:%s\n", scan_state_str(_state)); */ \
+	} while (0)
+#endif
+#ifdef CONFIG_SCAN_BACKOP
+#define mlmeext_scan_backop_flags(mlmeext) ((mlmeext)->sitesurvey_res.backop_flags)
+#define mlmeext_chk_scan_backop_flags(mlmeext, flags) ((mlmeext)->sitesurvey_res.backop_flags & (flags))
+#define mlmeext_assign_scan_backop_flags(mlmeext, flags) \
+	do { \
+		((mlmeext)->sitesurvey_res.backop_flags = (flags)); \
+		RTW_INFO("assign_scan_backop_flags:0x%02x\n", (mlmeext)->sitesurvey_res.backop_flags); \
+	} while (0)
+
+#define mlmeext_scan_backop_flags_sta(mlmeext) ((mlmeext)->sitesurvey_res.backop_flags_sta)
+#define mlmeext_chk_scan_backop_flags_sta(mlmeext, flags) ((mlmeext)->sitesurvey_res.backop_flags_sta & (flags))
+#define mlmeext_assign_scan_backop_flags_sta(mlmeext, flags) \
+	do { \
+		((mlmeext)->sitesurvey_res.backop_flags_sta = (flags)); \
+	} while (0)
+#else
+#define mlmeext_scan_backop_flags(mlmeext) (0)
+#define mlmeext_chk_scan_backop_flags(mlmeext, flags) (0)
+#define mlmeext_assign_scan_backop_flags(mlmeext, flags) do {} while (0)
+
+#define mlmeext_scan_backop_flags_sta(mlmeext) (0)
+#define mlmeext_chk_scan_backop_flags_sta(mlmeext, flags) (0)
+#define mlmeext_assign_scan_backop_flags_sta(mlmeext, flags) do {} while (0)
+#endif /* CONFIG_SCAN_BACKOP */
+
+#if defined(CONFIG_SCAN_BACKOP) && defined(CONFIG_AP_MODE)
+#define mlmeext_scan_backop_flags_ap(mlmeext) ((mlmeext)->sitesurvey_res.backop_flags_ap)
+#define mlmeext_chk_scan_backop_flags_ap(mlmeext, flags) ((mlmeext)->sitesurvey_res.backop_flags_ap & (flags))
+#define mlmeext_assign_scan_backop_flags_ap(mlmeext, flags) \
+	do { \
+		((mlmeext)->sitesurvey_res.backop_flags_ap = (flags)); \
+	} while (0)
+#else
+#define mlmeext_scan_backop_flags_ap(mlmeext) (0)
+#define mlmeext_chk_scan_backop_flags_ap(mlmeext, flags) (0)
+#define mlmeext_assign_scan_backop_flags_ap(mlmeext, flags) do {} while (0)
+#endif /* defined(CONFIG_SCAN_BACKOP) && defined(CONFIG_AP_MODE) */
+
+#if defined(CONFIG_SCAN_BACKOP) && defined(CONFIG_RTW_MESH)
+#define mlmeext_scan_backop_flags_mesh(mlmeext) ((mlmeext)->sitesurvey_res.backop_flags_mesh)
+#define mlmeext_chk_scan_backop_flags_mesh(mlmeext, flags) ((mlmeext)->sitesurvey_res.backop_flags_mesh & (flags))
+#define mlmeext_assign_scan_backop_flags_mesh(mlmeext, flags) \
+	do { \
+		((mlmeext)->sitesurvey_res.backop_flags_mesh = (flags)); \
+	} while (0)
+#else
+#define mlmeext_scan_backop_flags_mesh(mlmeext) (0)
+#define mlmeext_chk_scan_backop_flags_mesh(mlmeext, flags) (0)
+#define mlmeext_assign_scan_backop_flags_mesh(mlmeext, flags) do {} while (0)
+#endif /* defined(CONFIG_SCAN_BACKOP) && defined(CONFIG_RTW_MESH) */
+
+#if 0
+void survey_timer_hdl(void *ctx);
+#define set_survey_timer(mlmeext, ms) \
+	do { \
+		/*RTW_INFO("%s set_survey_timer(%p, %d)\n", __FUNCTION__, (mlmeext), (ms));*/ \
+		_set_timer(&(mlmeext)->survey_timer, (ms)); \
+	} while (0)
+#define cancel_survey_timer(mlmeext) \
+	do { \
+		/*RTW_INFO("%s cancel_survey_timer(%p)\n", __FUNCTION__, (mlmeext));*/ \
+		_cancel_timer_ex(&(mlmeext)->survey_timer); \
+	} while (0)
+#endif
+
+enum SCAN_STATE {
+	SCAN_DISABLE = 0,
+	SCAN_START = 1,
+	SCAN_PS_ANNC_WAIT = 2,
+	SCAN_ENTER = 3,
+	SCAN_PROCESS = 4,
+
+	/* backop */
+	SCAN_BACKING_OP = 5,
+	SCAN_BACK_OP = 6,
+	SCAN_LEAVING_OP = 7,
+	SCAN_LEAVE_OP = 8,
+
+	/* SW antenna diversity (before linked) */
+	SCAN_SW_ANTDIV_BL = 9,
+
+	/* legacy p2p */
+	SCAN_TO_P2P_LISTEN = 10,
+	SCAN_P2P_LISTEN = 11,
+
+	SCAN_COMPLETE = 12,
+	SCAN_STATE_MAX,
+};
+
+
+enum ss_backop_flag {
+	SS_BACKOP_EN = BIT0, /* backop when linked */
+	SS_BACKOP_EN_NL = BIT1, /* backop even when no linked */
+
+	SS_BACKOP_PS_ANNC = BIT4,
+	SS_BACKOP_TX_RESUME = BIT5,
+};
+
+#define RTW_SSID_SCAN_AMOUNT 9 /* for WEXT_CSCAN_AMOUNT 9 */
+#define RTW_CHANNEL_SCAN_AMOUNT MAX_CHANNEL_NUM
+
+struct ss_res {
+	u8 state;
+	u8 next_state; /* will set to state on next cmd hdl */
+	int bss_cnt;
+	u8 activate_ch_cnt;
+	_lock scan_param_lock;
+	struct rtw_phl_scan_param *scan_param;
+	struct submit_ctx sctx;
+
+	u16 scan_ch_ms;
+
+	u8 rx_ampdu_accept;
+	u16 rx_ampdu_size;
+#if 0
+	int channel_idx;
+	u8 force_ssid_scan;
+	int scan_mode;
+	u8 igi_scan;
+	u8 igi_before_scan; /* used for restoring IGI value without enable DIG & FA_CNT */
+#endif
+
+#ifdef CONFIG_SCAN_BACKOP
+	u8 backop_flags_sta; /* policy for station mode*/
+	#ifdef CONFIG_AP_MODE
+	u8 backop_flags_ap; /* policy for ap mode */
+	#endif
+	#ifdef CONFIG_RTW_MESH
+	u8 backop_flags_mesh; /* policy for mesh mode */
+	#endif
+	u8 backop_flags; /* per backop runtime decision */
+	#if 0
+	u8 scan_cnt;
+	#endif
+	u8 scan_cnt_max;
+	systime backop_time; /* the start time of backop */
+	u16 backop_ms;
+#endif
+#if defined(CONFIG_ANTENNA_DIVERSITY) || defined(DBG_SCAN_SW_ANTDIV_BL)
+	u8 is_sw_antdiv_bl_scan;
+#endif
+	u8 ssid_num;
+	u8 ch_num;
+	NDIS_802_11_SSID ssid[RTW_SSID_SCAN_AMOUNT];
+	struct rtw_ieee80211_channel ch[RTW_CHANNEL_SCAN_AMOUNT];
+
+	u32 token; 	/* 0: use to identify caller */
+	u16 duration;	/* 0: use default */
+	u8 igi;		/* 0: use defalut */
+	u8 bw;		/* 0: use default */
+
+	bool acs; /* aim to trigger channel selection when scan done */
+};
+
+enum rtw_scan_type {
+	RTW_SCAN_NORMAL,
+	RTW_SCAN_P2P,
+	RTW_SCAN_RRM,
+	RTW_SCAN_ROAM
+};
+
+#define RTW_MAX_NB_RPT_NUM 8
+#if defined(CONFIG_RTW_FSM_RRM) || defined(CONFIG_RTW_FSM_BTM)
+struct nb_bssid{
+	u8 ch;
+	u8 bssid[ETH_ALEN];
+};
+#endif
+
+struct sitesurvey_parm {
+	enum rtw_phl_scan_type scan_mode;	/* active: 1, passive: 0 */
+	/* sint bsslimit;	// 1 ~ 48 */
+	u8 ssid_num;
+	u8 ch_num;
+	NDIS_802_11_SSID ssid[RTW_SSID_SCAN_AMOUNT];
+	struct rtw_ieee80211_channel ch[RTW_CHANNEL_SCAN_AMOUNT];
+
+	u32 rrm_token;	/* 80211k use it to identify caller */
+	u16 duration;	/* 0: use default, otherwise: channel scan time */
+	u8 bw;		/* 0: use default */
+
+	bool acs; /* aim to trigger channel selection when scan done */
+	u8 reason;
+
+	enum rtw_scan_type scan_type;
+#ifdef CONFIG_RTW_FSM
+	struct sta_info *psta;
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	bool pending_6ghz_scan; /* indicate separate scan for 6GHz channels later */
+	bool split_scan_6ghz; /* indicate if current scan is the second scan for 6GHz */
+#endif
+	bool scan_6ghz_only; /* indicate only 6GHz channels in scan request */
+#endif
+#if defined(CONFIG_RTW_FSM_RRM) || defined(CONFIG_RTW_FSM_BTM)
+	u8 nb_num;
+	struct nb_bssid nb[RTW_MAX_NB_RPT_NUM];
+#endif
+};
+
+void rtw_init_sitesurvey_parm(_adapter *padapter, struct sitesurvey_parm *pparm);
+u8 rtw_sitesurvey_cmd(_adapter *padapter, struct sitesurvey_parm *pparm);
+
+#ifdef CONFIG_IOCTL_CFG80211
+u8 rtw_phl_remain_on_ch_cmd(_adapter *padapter, u64 cookie, struct wireless_dev *wdev,
+	struct ieee80211_channel *ch, u8 ch_type, unsigned int duration,
+	struct back_op_param *bkop_parm);
+#endif
+
+u32 rtw_parse_reduced_nb_rpt(_adapter *adapter, u8 *ies, u32 ies_len,
+	struct rtw_phl_rnb_rpt_element *rnb_ele);
+
+#endif /* __RTW_SCAN_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_sdio.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_sdio.h
new file mode 100644
index 000000000000..306061db793d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_sdio.h
@@ -0,0 +1,32 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_SDIO_H_
+#define _RTW_SDIO_H_
+
+#include <drv_types.h>		/* struct dvobj_priv and etc. */
+
+#define SDIO_ERR_VAL8	0xFF
+#define SDIO_ERR_VAL16	0xFFFF
+#define SDIO_ERR_VAL32	0xFFFFFFFF
+
+u8 rtw_sdio_read_cmd52(struct dvobj_priv *, u32 addr, void *buf, size_t len);
+u8 rtw_sdio_read_cmd53(struct dvobj_priv *, u32 addr, void *buf, size_t len);
+u8 rtw_sdio_write_cmd52(struct dvobj_priv *, u32 addr, void *buf, size_t len);
+u8 rtw_sdio_write_cmd53(struct dvobj_priv *, u32 addr, void *buf, size_t len);
+u8 rtw_sdio_f0_read(struct dvobj_priv *, u32 addr, void *buf, size_t len);
+size_t rtw_sdio_cmd53_align_size(struct dvobj_priv *d, size_t len);
+
+
+#endif /* _RTW_SDIO_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_sec_cam.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_sec_cam.h
new file mode 100644
index 000000000000..7d4bdacd57e3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_sec_cam.h
@@ -0,0 +1,210 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_SEC_CAM_H__
+#define __RTW_SEC_CAM_H__
+
+#if 0
+#define SEC_CAM_ENT_NUM_SW_LIMIT 32
+
+#define SEC_STATUS_STA_PK_GK_CONFLICT_DIS_BMC_SEARCH BIT0
+#define SEC_CAP_CHK_BMC	BIT0
+
+struct sec_cam_bmp {
+	u32 m0;
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 32)
+	u32 m1;
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 64)
+	u32 m2;
+#endif
+#if (SEC_CAM_ENT_NUM_SW_LIMIT > 96)
+	u32 m3;
+#endif
+};
+
+struct cam_ctl_t {
+	_lock lock;
+
+	u8 sec_cap;
+	u32 flags;
+
+	u8 num;
+	struct sec_cam_bmp used;
+
+	_mutex sec_cam_access_mutex;
+};
+
+struct sec_cam_ent {
+	u16 ctrl;
+	u8 mac[ETH_ALEN];
+	u8 key[16];
+};
+
+void dump_sec_cam_ent(void *sel, struct sec_cam_ent *ent, int id);
+void dump_sec_cam_ent_title(void *sel, u8 has_id);
+void dump_sec_cam(void *sel, _adapter *adapter);
+void dump_sec_cam_cache(void *sel, _adapter *adapter);
+
+bool _rtw_camctl_chk_cap(_adapter *adapter, u8 cap);
+void _rtw_camctl_set_flags(_adapter *adapter, u32 flags);
+void rtw_camctl_set_flags(_adapter *adapter, u32 flags);
+void _rtw_camctl_clr_flags(_adapter *adapter, u32 flags);
+void rtw_camctl_clr_flags(_adapter *adapter, u32 flags);
+bool _rtw_camctl_chk_flags(_adapter *adapter, u32 flags);
+
+void dump_sec_cam_map(void *sel, struct sec_cam_bmp *map, u8 max_num);
+void rtw_sec_cam_map_set(struct sec_cam_bmp *map, u8 id);
+void rtw_sec_cam_map_clr(struct sec_cam_bmp *map, u8 id);
+void rtw_sec_cam_map_clr_all(struct sec_cam_bmp *map);
+
+bool rtw_sec_camid_is_used(struct cam_ctl_t *cam_ctl, u8 id);
+
+bool _rtw_camid_is_gk(_adapter *adapter, u8 cam_id);
+bool rtw_camid_is_gk(_adapter *adapter, u8 cam_id);
+s16 rtw_camid_search(_adapter *adapter, u8 *addr, s16 kid, s8 gk);
+s16 rtw_camid_alloc(_adapter *adapter, struct sta_info *sta, u8 kid, u8 gk, bool ext_sec, bool *used);
+void rtw_camid_free(_adapter *adapter, u8 cam_id);
+u8 rtw_get_sec_camid(_adapter *adapter, u8 max_bk_key_num, u8 *sec_key_id);
+
+void _clear_cam_entry(_adapter *padapter, u8 entry);
+void write_cam_from_cache(_adapter *adapter, u8 id);
+void rtw_sec_cam_swap(_adapter *adapter, u8 cam_id_a, u8 cam_id_b);
+void rtw_clean_dk_section(_adapter *adapter);
+void rtw_clean_hw_dk_cam(_adapter *adapter);
+
+/* modify both HW and cache */
+void write_cam(_adapter *padapter, u8 id, u16 ctrl, u8 *mac, u8 *key);
+void clear_cam_entry(_adapter *padapter, u8 id);
+
+/* modify cache only */
+void write_cam_cache(_adapter *adapter, u8 id, u16 ctrl, u8 *mac, u8 *key);
+void clear_cam_cache(_adapter *adapter, u8 id);
+
+void invalidate_cam_all(_adapter *padapter);
+#endif
+
+void flush_all_cam_entry(_adapter *padapter, enum phl_cmd_type cmd_type, u32 cmd_timeout);
+
+
+#ifdef  CONFIG_DBG_AX_CAM
+
+#define INDIRECT_ACCESS_ADDR	0xC04
+#define INDIRECT_ACCESS_VALUE	0x40000
+#define SEC_CAM_BASE_ADDR	0x18814000
+#define BSSID_CAM_BASE_ADDR	0x18853000
+#define ADDR_CAM_BASE_ADDR	0x18850000
+
+enum CAM_KEY_TYPE {
+	KEY_TYPE_UNI = 0,
+	KEY_TYPE_GROUP = 1,
+	KEY_TYPE_BIP = 2,
+	KEY_TYPE_NONE = 3
+};
+
+/*Address cam field info DW0 - DW9*/
+
+/* DWORD 0 ; Offset 00h */
+#define GET_AX_ADDR_CAM_VALID(__pCAM)		LE_BITS_TO_4BYTE(__pCAM, 0, 1)
+#define GET_AX_ADDR_CAM_NET_TYPE(__pCAM)	LE_BITS_TO_4BYTE(__pCAM, 1, 2)
+#define GET_AX_ADDR_CAM_BCN_HD_COND(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 3, 2)
+#define GET_AX_ADDR_CAM_HIT_RULE(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 5, 2)
+#define GET_AX_ADDR_CAM_BB_SEL(__pCAM)		LE_BITS_TO_4BYTE(__pCAM, 7, 1)
+#define GET_AX_ADDR_CAM_ADDR_MASK(__pCAM)	LE_BITS_TO_4BYTE(__pCAM, 8, 6)
+#define GET_AX_ADDR_CAM_MASK_SEL(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 14, 2)
+#define GET_AX_ADDR_CAM_SMA_HASH(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 16, 8)
+#define GET_AX_ADDR_CAM_TMA_HASH(__pCAM)	LE_BITS_TO_4BYTE(__pCAM, 24, 8)
+
+/* DWORD 1 ; Offset 04h */
+#define GET_AX_ADDR_CAM_BSSID_CAM_IDX(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+4, 0, 6)
+/* DWORD 2 ; Offset 08h */
+#define GET_AX_ADDR_CAM_SMA_0(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+8, 0, 8)
+#define GET_AX_ADDR_CAM_SMA_1(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+8, 8, 8)
+#define GET_AX_ADDR_CAM_SMA_2(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+8, 16, 8)
+#define GET_AX_ADDR_CAM_SMA_3(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+8, 24, 8)
+/* DWORD 3 ; Offset 0Ch */
+#define GET_AX_ADDR_CAM_SMA_4(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+12, 8, 6)
+#define GET_AX_ADDR_CAM_SMA_5(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+12, 14, 2)
+#define GET_AX_ADDR_CAM_TMA_0(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+12, 16, 8)
+#define GET_AX_ADDR_CAM_TMA_1(__pCAM)		LE_BITS_TO_4BYTE(__pCAM+12, 24, 8)
+/* DWORD 4 ; Offset 10h */
+#define GET_AX_ADDR_CAM_TMA_2(__pCAM)		LE_BITS_TO_4BYTE(__pCAM+16, 0, 8)
+#define GET_AX_ADDR_CAM_TMA_3(__pCAM)		LE_BITS_TO_4BYTE(__pCAM+16, 8, 8)
+#define GET_AX_ADDR_CAM_TMA_4(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+16, 16, 8)
+#define GET_AX_ADDR_CAM_TMA_5(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+16, 24, 8)
+/* DWORD 5 ; Offset 14h : RSVD ALL*/
+/* DWORD 6 ; Offset 18h */
+
+#define GET_AX_ADDR_CAM_MACID(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+24, 0, 8)
+#define GET_AX_ADDR_CAM_PORT_INT(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+24, 8, 3)
+#define GET_AX_ADDR_CAM_TSF_SYNC(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+24, 11, 3)
+#define GET_AX_ADDR_CAM_TF(__pCAM)		LE_BITS_TO_4BYTE(__pCAM+24, 14, 2)
+#define GET_AX_ADDR_CAM_LSIG_TXOP(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+24, 16, 2)
+#define GET_AX_ADDR_CAM_CTRLCNT_IDX(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+24, 18, 4)
+#define GET_AX_ADDR_CAM_CTRLCNT_VALID(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+24, 20, 1)
+#define GET_AX_ADDR_CAM_TARGET_IND(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+24, 24, 3)
+#define GET_AX_ADDR_CAM_FRM_TARGET_IND(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+24, 27, 3)
+
+/* DWORD 7 ; Offset 1Ch */
+#define GET_AX_ADDR_CAM_AID12_0(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+28, 0, 8)
+#define GET_AX_ADDR_CAM_AID12_1(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+28, 8, 4)
+#define GET_AX_ADDR_CAM_WOL_PATTERN(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+28, 12, 1)
+#define GET_AX_ADDR_CAM_WOL_UC(__pCAM)		LE_BITS_TO_4BYTE(__pCAM+28, 13, 1)
+#define GET_AX_ADDR_CAM_WOL_MAGIC(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+28, 14, 1)
+#define GET_AX_ADDR_CAM_WAPI(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM+28, 15, 1)
+#define GET_AX_ADDR_CAM_SEC_ENT_MODE(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 16, 2)
+#define GET_AX_ADDR_CAM_SEC_ENT0_KEYID(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 18, 2)
+#define GET_AX_ADDR_CAM_SEC_ENT1_KEYID(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 20, 2)
+#define GET_AX_ADDR_CAM_SEC_ENT2_KEYID(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 22, 2)
+#define GET_AX_ADDR_CAM_SEC_ENT3_KEYID(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 24, 2)
+#define GET_AX_ADDR_CAM_SEC_ENT4_KEYID(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 26, 2)
+#define GET_AX_ADDR_CAM_SEC_ENT5_KEYID(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 28, 2)
+#define GET_AX_ADDR_CAM_SEC_ENT6_KEYID(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+28, 30, 2)
+/* DWORD 8 ; Offset 20h */
+#define GET_AX_ADDR_CAM_SEC_ENT_VALID(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+32, 0, 8)
+#define GET_AX_ADDR_CAM_SEC_ENT0(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+32, 8, 8)
+#define GET_AX_ADDR_CAM_SEC_ENT1(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+32, 16, 8)
+#define GET_AX_ADDR_CAM_SEC_ENT2(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+32, 24, 8)
+/* DWORD 9 ; Offset 24h */
+#define GET_AX_ADDR_CAM_SEC_ENT3(__pCAM)	LE_BITS_TO_4BYTE(__pCAM+36, 0, 8)
+#define GET_AX_ADDR_CAM_SEC_ENT4(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+36, 8, 8)
+#define GET_AX_ADDR_CAM_SEC_ENT5(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+36, 16, 8)
+#define GET_AX_ADDR_CAM_SEC_ENT6(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM+36, 24, 8)
+
+/* Security cam */
+/*DWORD 0 ; offset 00h*/
+#define GET_AX_SEC_CAM_TYPE(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM, 0, 4)
+#define GET_AX_SEC_CAM_EXT_KEY(__pCAM)		LE_BITS_TO_4BYTE(__pCAM, 4, 1)
+#define GET_AX_SEC_SPP_MODE_(__pCAM) 		LE_BITS_TO_4BYTE(__pCAM, 5, 1)
+
+/*BSSID cam*/
+/*DWORD 0 ; offset 00h*/
+#define GET_AX_BSSID_CAM_VALID(__pCAM)		LE_BITS_TO_4BYTE(__pCAM, 0, 1)
+#define GET_AX_BSSID_CAM_BB_SEL(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 1, 1)
+#define GET_AX_BSSID_CAM_COLOR_(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 8, 6)
+#define GET_AX_BSSID_CAM_BSSID_0(__pCAM)	LE_BITS_TO_4BYTE(__pCAM, 16, 8)
+#define GET_AX_BSSID_CAM_BSSID_1(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 24, 8)
+/*DWORD 1 ; offset 04h*/
+#define GET_AX_BSSID_CAM_BSSID_2(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 0, 8)
+#define GET_AX_BSSID_CAM_BSSID_3(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 8, 8)
+#define GET_AX_BSSID_CAM_BSSID_4(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 16, 8)
+#define GET_AX_BSSID_CAM_BSSID_5(__pCAM) 	LE_BITS_TO_4BYTE(__pCAM, 24, 8)
+
+int get_ax_sec_cam(void* sel, struct _ADAPTER *a);
+int get_ax_address_cam(void* sel, struct _ADAPTER *a);
+int get_ax_valid_key(void* sel, struct _ADAPTER *a);
+
+#endif /* CONFIG_DBG_AX_CAM */
+
+#endif /* __RTW_SEC_CAM_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_security.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_security.h
new file mode 100644
index 000000000000..543fa8b0dee0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_security.h
@@ -0,0 +1,440 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_SECURITY_H_
+#define __RTW_SECURITY_H_
+
+enum security_type {
+	/* TYPE */
+	_NO_PRIVACY_	= 0x00,
+	_WEP40_		= 0x01,
+	_TKIP_		= 0x02,
+	_TKIP_WTMIC_	= 0x03,
+	_AES_		= 0x04,
+	_WEP104_	= 0x05,
+	_SMS4_		= 0x06,
+	_GCMP_		= 0x07,
+	_SEC_TYPE_MAX_,
+
+	/* EXT_SECTYPE=1 */
+	_SEC_TYPE_256_	= 0x10,
+	_CCMP_256_	= (_AES_ | _SEC_TYPE_256_),
+	_GCMP_256_	= (_GCMP_ | _SEC_TYPE_256_),
+
+#ifdef CONFIG_IEEE80211W
+	/* EXT_SECTYPE=0, MGNT=1, GK=0/1, KEYID=00/01 */
+	_SEC_TYPE_BIT_	= 0x20,
+	_BIP_CMAC_128_	= (_SEC_TYPE_BIT_),
+	_BIP_GMAC_128_	= (_SEC_TYPE_BIT_ + 1),
+	_BIP_GMAC_256_	= (_SEC_TYPE_BIT_ + 2),
+	/* EXT_SECTYPE=1, MGNT=1, GK=1, KEYID=00/01 */
+	_BIP_CMAC_256_	= (_SEC_TYPE_BIT_ + 3),
+	_BIP_MAX_,
+#endif
+};
+
+
+/* 802.11W use wrong key */
+#define IEEE80211W_RIGHT_KEY	0x0
+#define IEEE80211W_WRONG_KEY	0x1
+#define IEEE80211W_NO_KEY		0x2
+
+#define CCMPH_2_PN(ch)	((ch) & 0x000000000000ffff) \
+			| (((ch) & 0xffffffff00000000) >> 16)
+
+#define is_wep_enc(alg) (((alg) == _WEP40_) || ((alg) == _WEP104_))
+
+const char *security_type_str(u8 value);
+#ifdef CONFIG_IEEE80211W
+u32 security_type_bip_to_gmcs(enum security_type type);
+#endif
+
+#define _WPA_IE_ID_	0xdd
+#define _WPA2_IE_ID_	0x30
+
+#define RTW_KEK_LEN 16
+#define RTW_KCK_LEN 16
+#define RTW_TKIP_MIC_LEN 8
+#define RTW_REPLAY_CTR_LEN 8
+/* For CCMP-128 only */
+#define RTW_PTK_LEN 16
+/* For BIP-CMAC-128 only */
+#define RTW_IGTK_LEN 16
+
+#define INVALID_SEC_MAC_CAM_ID	0xFF
+
+typedef enum {
+	ENCRYP_PROTOCOL_OPENSYS,   /* open system */
+	ENCRYP_PROTOCOL_WEP,       /* WEP */
+	ENCRYP_PROTOCOL_WPA,       /* WPA */
+	ENCRYP_PROTOCOL_RSN,       /* RSN(WPA2/WPA3) */
+	ENCRYP_PROTOCOL_WAPI,      /* WAPI: Not support in this version */
+	ENCRYP_PROTOCOL_MAX
+} ENCRYP_PROTOCOL_E;
+
+
+#ifndef Ndis802_11AuthModeWPA2
+#define Ndis802_11AuthModeWPA2 (Ndis802_11AuthModeWPANone + 1)
+#endif
+
+#ifndef Ndis802_11AuthModeWPA2PSK
+#define Ndis802_11AuthModeWPA2PSK (Ndis802_11AuthModeWPANone + 2)
+#endif
+
+union pn48	{
+
+	u64	val;
+
+#ifdef CONFIG_LITTLE_ENDIAN
+
+struct {
+	u8 TSC0;
+	u8 TSC1;
+	u8 TSC2;
+	u8 TSC3;
+	u8 TSC4;
+	u8 TSC5;
+	u8 TSC6;
+	u8 TSC7;
+} _byte_;
+
+#elif defined(CONFIG_BIG_ENDIAN)
+
+struct {
+	u8 TSC7;
+	u8 TSC6;
+	u8 TSC5;
+	u8 TSC4;
+	u8 TSC3;
+	u8 TSC2;
+	u8 TSC1;
+	u8 TSC0;
+} _byte_;
+
+#endif
+
+};
+
+union Keytype {
+	u8 skey[32];
+};
+
+typedef struct _RT_PMKID_LIST {
+	u8						bUsed;
+	u8						Bssid[6];
+	u8						PMKID[16];
+	u8						SsidBuf[33];
+	u8						*ssid_octet;
+	u16						ssid_length;
+} RT_PMKID_LIST, *PRT_PMKID_LIST;
+
+struct link_security_priv {
+	u32	dot118021XGrpKeyid;		/* key id used for Grp Key ( tx key index) */
+	union Keytype	dot118021XGrpKey[6];	/* 802.1x Group Key, for inx0 and inx1	 */
+	union Keytype	dot118021XGrptxmickey[6];
+	union Keytype	dot118021XGrprxmickey[6];
+	union pn48		dot11Grptxpn;			/* PN48 used for Grp Key xmit. */
+	union pn48		dot11Grprxpn;			/* PN48 used for Grp Key recv. */
+	u8				iv_seq[4][8];
+#ifdef CONFIG_IEEE80211W
+	u32	dot11wBIPKeyid;						/* key id used for BIP Key ( tx key index) */
+	union Keytype	dot11wBIPKey[6];		/* BIP Key, for index4 and index5 */
+	union pn48		dot11wBIPtxpn;			/* PN48 used for BIP xmit. */
+	union pn48		dot11wBIPrxpn;			/* PN48 used for BIP recv. */
+	u8	binstallBIPkey;
+#endif /* CONFIG_IEEE80211W */
+	u8	binstallGrpkey;
+	u8	bcheck_grpkey;
+	u8	bgrpkey_handshake;
+};
+
+struct security_priv {
+	u32	  dot11AuthAlgrthm;		/* 802.11 auth, could be open, shared, 8021x and authswitch */
+	u32	  dot11PrivacyAlgrthm;	/* This specify the privacy for shared auth. algorithm. */
+
+	/* WEP */
+	u32	  dot11PrivacyKeyIndex;	/* this is only valid for legendary wep, 0~3 for key id. (tx key index) */
+	union Keytype dot11DefKey[6];			/* this is only valid for def. key	 */
+	u32	dot11DefKeylen[6];
+	u8	dot11Def_camid[6];
+	u8 	key_mask; /* use to restore wep key after hal_init */
+
+	u32 dot118021XGrpPrivacy;	/* This specify the privacy algthm. used for Grp key */
+#ifdef CONFIG_IEEE80211W
+	enum security_type dot11wCipher;
+#endif
+#ifdef CONFIG_AP_MODE
+	/* extend security capabilities for AP_MODE */
+	unsigned int dot8021xalg;/* 0:disable, 1:psk, 2:802.1x */
+	unsigned int wpa_psk;/* 0:disable, bit(0): WPA, bit(1):WPA2 */
+	unsigned int wpa_group_cipher;
+	unsigned int wpa2_group_cipher;
+	unsigned int wpa_pairwise_cipher;
+	unsigned int wpa2_pairwise_cipher;
+	unsigned int akmp; /* An authentication and key management protocol */
+	u8 mfp_opt;
+#endif
+#ifdef CONFIG_CONCURRENT_MODE
+	u8	dot118021x_bmc_cam_id;
+#endif
+	/*IEEE802.11-2012 Std. Table 8-101 AKM Suite Selectors*/
+	u32	rsn_akm_suite_type;
+
+	u8 wps_ie[MAX_WPS_IE_LEN];/* added in assoc req */
+	int wps_ie_len;
+
+	u8 owe_ie[MAX_OWE_IE_LEN];/* added in assoc req */
+	int owe_ie_len;
+
+	u8 rsnx_ie[MAX_RSNX_IE_LEN];
+	int rsnx_ie_len;
+
+#ifdef CONFIG_GTK_OL
+	u8	binstallKCK_KEK;
+#endif /* CONFIG_GTK_OL */
+	u8	busetkipkey;
+
+	u8	auth_alg;
+	u8	auth_type;
+	u8	extauth_status;
+	/* u8	packet_cnt; */ /* unused, removed */
+
+	s32	sw_encrypt;/* from registry_priv */
+	s32	sw_decrypt;/* from registry_priv */
+
+	s32 hw_decrypted; /* Broadcast HW security is ready or not */
+
+
+	/* keeps the auth_type & enc_status from upper layer ioctl(wpa_supplicant or wzc) */
+	u32 ndisauthtype;	/* NDIS_802_11_AUTHENTICATION_MODE */
+	u32 ndisencryptstatus;	/* NDIS_802_11_ENCRYPTION_STATUS */
+
+	NDIS_802_11_WEP ndiswep;
+
+	u8 supplicant_ie[256];  /* store sta security information element */
+
+
+	/* for tkip countermeasure */
+	systime last_mic_err_time;
+	u8	btkip_countermeasure;
+	u8	btkip_wait_report;
+	systime btkip_countermeasure_time;
+
+	/* --------------------------------------------------------------------------- */
+	/* For WPA2 Pre-Authentication. */
+	/* --------------------------------------------------------------------------- */
+	/* u8				RegEnablePreAuth;				 */ /* Default value: Pre-Authentication enabled or not, from registry "EnablePreAuth". Added by Annie, 2005-11-01. */
+	/* u8				EnablePreAuthentication;			 */ /* Current Value: Pre-Authentication enabled or not. */
+	RT_PMKID_LIST		PMKIDList[NUM_PMKID_CACHE];	/* Renamed from PreAuthKey[NUM_PRE_AUTH_KEY]. Annie, 2006-10-13. */
+	u8				PMKIDIndex;
+	/* u32				PMKIDCount;						 */ /* Added by Annie, 2006-10-13. */
+	/* u8				szCapability[256];				 */ /* For WPA2-PSK using zero-config, by Annie, 2005-09-20. */
+
+	u8 bWepDefaultKeyIdxSet;
+
+#define DBG_SW_SEC_CNT
+#ifdef DBG_SW_SEC_CNT
+	u64 wep_sw_enc_cnt_bc;
+	u64 wep_sw_enc_cnt_mc;
+	u64 wep_sw_enc_cnt_uc;
+	u64 wep_sw_dec_cnt_bc;
+	u64 wep_sw_dec_cnt_mc;
+	u64 wep_sw_dec_cnt_uc;
+
+	u64 tkip_sw_enc_cnt_bc;
+	u64 tkip_sw_enc_cnt_mc;
+	u64 tkip_sw_enc_cnt_uc;
+	u64 tkip_sw_dec_cnt_bc;
+	u64 tkip_sw_dec_cnt_mc;
+	u64 tkip_sw_dec_cnt_uc;
+
+	u64 aes_sw_enc_cnt_bc;
+	u64 aes_sw_enc_cnt_mc;
+	u64 aes_sw_enc_cnt_uc;
+	u64 aes_sw_dec_cnt_bc;
+	u64 aes_sw_dec_cnt_mc;
+	u64 aes_sw_dec_cnt_uc;
+
+	u64 gcmp_sw_enc_cnt_bc;
+	u64 gcmp_sw_enc_cnt_mc;
+	u64 gcmp_sw_enc_cnt_uc;
+	u64 gcmp_sw_dec_cnt_bc;
+	u64 gcmp_sw_dec_cnt_mc;
+	u64 gcmp_sw_dec_cnt_uc;
+#endif /* DBG_SW_SEC_CNT */
+};
+
+#ifdef CONFIG_IEEE80211W
+#define SEC_IS_BIP_KEY_INSTALLED(sec) ((sec)->binstallBIPkey)
+#else
+#define SEC_IS_BIP_KEY_INSTALLED(sec) _FALSE
+#endif
+
+#define GET_ENCRY_ALGO(psecuritypriv, psta, encry_algo, bmcst)\
+	do {\
+		switch (psecuritypriv->dot11AuthAlgrthm) {\
+		case dot11AuthAlgrthm_Open:\
+		case dot11AuthAlgrthm_Shared:\
+		case dot11AuthAlgrthm_Auto:\
+			encry_algo = (u8)psecuritypriv->dot11PrivacyAlgrthm;\
+			break;\
+		case dot11AuthAlgrthm_8021X:\
+			if (bmcst)\
+				encry_algo = (u8)psecuritypriv->dot118021XGrpPrivacy;\
+			else\
+				encry_algo = (u8) psta->dot118021XPrivacy;\
+			break;\
+		case dot11AuthAlgrthm_WAPI:\
+			encry_algo = (u8)psecuritypriv->dot11PrivacyAlgrthm;\
+			break;\
+		} \
+	} while (0)
+
+#define _AES_IV_LEN_ 8
+
+#define SET_ICE_IV_LEN(iv_len, icv_len, encrypt)\
+	do {\
+		switch (encrypt) {\
+		case _WEP40_:\
+		case _WEP104_:\
+			iv_len = 4;\
+			icv_len = 4;\
+			break;\
+		case _TKIP_:\
+			iv_len = 8;\
+			icv_len = 4;\
+			break;\
+		case _AES_:\
+			iv_len = 8;\
+			icv_len = 8;\
+			break;\
+		case _GCMP_:\
+		case _GCMP_256_:\
+			iv_len = 8;\
+			icv_len = 16;\
+			break;\
+		case _CCMP_256_:\
+			iv_len = 8;\
+			icv_len = 16;\
+			break;\
+		case _SMS4_:\
+			iv_len = 18;\
+			icv_len = 16;\
+			break;\
+		default:\
+			iv_len = 0;\
+			icv_len = 0;\
+			break;\
+		} \
+	} while (0)
+
+
+#define GET_TKIP_PN(iv, dot11txpn)\
+	do {\
+		dot11txpn._byte_.TSC0 = iv[2];\
+		dot11txpn._byte_.TSC1 = iv[0];\
+		dot11txpn._byte_.TSC2 = iv[4];\
+		dot11txpn._byte_.TSC3 = iv[5];\
+		dot11txpn._byte_.TSC4 = iv[6];\
+		dot11txpn._byte_.TSC5 = iv[7];\
+	} while (0)
+
+
+#define ROL32(A, n)	(((A) << (n)) | (((A)>>(32-(n)))  & ((1UL << (n)) - 1)))
+#define ROR32(A, n)	ROL32((A), 32-(n))
+
+struct mic_data {
+	u32  K0, K1;         /* Key */
+	u32  L, R;           /* Current state */
+	u32  M;              /* Message accumulator (single word) */
+	u32     nBytesInM;      /*  # bytes in M */
+};
+
+void rtw_secmicsetkey(struct mic_data *pmicdata, u8 *key);
+void rtw_secmicappendbyte(struct mic_data *pmicdata, u8 b);
+void rtw_secmicappend(struct mic_data *pmicdata, u8 *src, u32 nBytes);
+void rtw_secgetmic(struct mic_data *pmicdata, u8 *dst);
+
+void rtw_seccalctkipmic(
+	u8 *key,
+	u8 *header,
+	u8 *data,
+	u32 data_len,
+	u8 *Miccode,
+	u8   priority);
+
+u32 rtw_aes_encrypt(_adapter *padapter, u8 *pxmitframe);
+u32 rtw_tkip_encrypt(_adapter *padapter, u8 *pxmitframe);
+void rtw_wep_encrypt(_adapter *padapter, u8  *pxmitframe);
+
+u32 rtw_aes_decrypt(_adapter *padapter, u8  *precvframe);
+u32 rtw_tkip_decrypt(_adapter *padapter, u8  *precvframe);
+void rtw_wep_decrypt(_adapter *padapter, u8  *precvframe);
+
+u32 rtw_gcmp_encrypt(_adapter *padapter, u8 *pxmitframe);
+u32 rtw_gcmp_decrypt(_adapter *padapter, u8 *precvframe);
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+u32 rtw_core_aes_encrypt(_adapter *padapter, u8 *pxframe);
+#endif
+
+#ifdef CONFIG_RTW_MESH_AEK
+int rtw_aes_siv_encrypt(const u8 *key, size_t key_len,
+	const u8 *pw, size_t pwlen, size_t num_elem,
+	const u8 *addr[], const size_t *len, u8 *out);
+int rtw_aes_siv_decrypt(const u8 *key, size_t key_len,
+	const u8 *iv_crypt, size_t iv_c_len, size_t num_elem,
+	const u8 *addr[], const size_t *len, u8 *out);
+#endif /* CONFIG_RTW_MESH_AEK */
+
+#ifdef CONFIG_IEEE80211W
+u8 rtw_calculate_bip_mic(enum security_type gmcs, u8 *whdr_pos, s32 len,
+	const u8 *key, const u8 *data, size_t data_len, u8 *mic);
+u32 rtw_bip_verify(enum security_type gmcs, u16 pkt_len,
+	u8 *whdr_pos, sint flen, const u8 *key, u16 keyid, u64 *ipn,
+	u8 *precvframe);
+#endif
+#ifdef CONFIG_TDLS
+void wpa_tdls_generate_tpk(_adapter *padapter, void *sta);
+int wpa_tdls_ftie_mic(u8 *kck, u8 trans_seq,
+			u8 *lnkid, u8 *rsnie, u8 *timeoutie, u8 *ftie,
+			u8 *mic);
+int wpa_tdls_teardown_ftie_mic(u8 *kck, u8 *lnkid, u16 reason,
+			u8 dialog_token, u8 trans_seq, u8 *ftie, u8 *mic);
+int tdls_verify_mic(u8 *kck, u8 trans_seq,
+			u8 *lnkid, u8 *rsnie, u8 *timeoutie, u8 *ftie);
+#endif /* CONFIG_TDLS */
+
+void rtw_sec_restore_wep_key(_adapter *adapter);
+u8 rtw_handle_tkip_countermeasure(_adapter *adapter, const char *caller);
+
+#ifdef CONFIG_WOWLAN
+u16 rtw_calc_crc(u8  *pdata, int length);
+#endif /*CONFIG_WOWLAN*/
+
+#define rtw_sec_chk_auth_alg(a, s) \
+	((a)->securitypriv.auth_alg == (s))
+
+#define rtw_sec_chk_auth_type(a, s) \
+	((a)->securitypriv.auth_type == (s))
+
+#define IV_FMT "0x%02x%02x%02x%02x%02x%02x%02x%02x"
+#define IV_ARG(iv) iv[7], iv[6], iv[5], iv[4], iv[3], iv[2], iv[1], iv[0]
+#define PN_FMT "0x%02x%02x%02x%02x%02x%02x"
+#define PN_ARG(pn) pn[5], pn[4], pn[3], pn[2], pn[1], pn[0]
+u8 rtw_iv_to_pn(u8 *iv, u8 *pn, u8 *key_id, u32 enc_algo);
+u8 rtw_pn_to_iv(u8 *pn, u8 *iv, u8 key_id, u32 enc_algo);
+#endif /* __RTL871X_SECURITY_H_ */
+
+u32 rtw_calc_crc32(u8 *data, size_t len);
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_swcrypto.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_swcrypto.h
new file mode 100644
index 000000000000..aa578feae4f1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_swcrypto.h
@@ -0,0 +1,54 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __RTW_SWCRYPTO_H_
+#define __RTW_SWCRYPTO_H_
+
+#define NEW_CRYPTO 1
+
+#if 0 //RTW_PHL_TX: mark un-finished codes for reading
+int _rtw_core_ccmp_encrypt(u8 *key, u32 key_len, uint hdrlen, u8 *phdr, uint datalen, u8 *pdata);
+#endif
+
+
+int _rtw_ccmp_encrypt(_adapter *padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame, uint plen);
+int _rtw_ccmp_decrypt(_adapter *padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame, uint plen);
+
+int _rtw_gcmp_encrypt(_adapter *padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame, uint plen);
+int _rtw_gcmp_decrypt(_adapter *padapter, u8 *key, u32 key_len, uint hdrlen, u8 *frame, uint plen);
+
+#ifdef CONFIG_RTW_MESH_AEK
+int _aes_siv_encrypt(const u8 *key, size_t key_len,
+	const u8 *pw, size_t pwlen,
+	size_t num_elem, const u8 *addr[], const size_t *len, u8 *out);
+int _aes_siv_decrypt(const u8 *key, size_t key_len,
+	const u8 *iv_crypt, size_t iv_c_len,
+	size_t num_elem, const u8 *addr[], const size_t *len, u8 *out);
+#endif
+
+#if defined(CONFIG_IEEE80211W) | defined(CONFIG_TDLS)
+u8 _bip_ccmp_protect(const u8 *key, size_t key_len,
+	const u8 *data, size_t data_len, u8 *mic);
+u8 _bip_gcmp_protect(u8 *whdr_pos, size_t len,
+	const u8 *key, size_t key_len,
+	const u8 *data, size_t data_len, u8 *mic);
+#endif /* CONFIG_IEEE80211W */
+
+#ifdef CONFIG_TDLS
+void _tdls_generate_tpk(void *sta, const u8 *own_addr, const u8 *bssid);
+#endif /* CONFIG_TDLS */
+
+#endif /* __RTW_SWCRYPTO_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_tdls.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_tdls.h
new file mode 100644
index 000000000000..5f78033059c4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_tdls.h
@@ -0,0 +1,190 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_TDLS_H_
+#define __RTW_TDLS_H_
+
+
+#ifdef CONFIG_TDLS
+/* TDLS STA state */
+
+
+/* TDLS Diect Link Establishment */
+#define	TDLS_STATE_NONE				0x00000000		/* Default state */
+#define	TDLS_INITIATOR_STATE		BIT(28)			/* 0x10000000 */
+#define	TDLS_RESPONDER_STATE		BIT(29)			/* 0x20000000 */
+#define	TDLS_LINKED_STATE			BIT(30)			/* 0x40000000 */
+#define	TDLS_RESETUP_STATE			BIT(31)			/* 0x80000000 */
+/* TDLS PU Buffer STA */
+#define	TDLS_WAIT_PTR_STATE			BIT(24)			/* 0x01000000 */	/* Waiting peer's TDLS_PEER_TRAFFIC_RESPONSE frame */
+/* TDLS Check ALive */
+#define	TDLS_ALIVE_STATE			BIT(20)			/* 0x00100000 */	/* Check if peer sta is alived. */
+/* TDLS Channel Switch */
+#define TDLS_WAIT_CH_SW_LAUNCH_STATE	BIT(14)			/* 0x00004000 */
+#define	TDLS_CH_SWITCH_PREPARE_STATE	BIT(15)			/* 0x00008000 */
+#define	TDLS_CH_SWITCH_ON_STATE			BIT(16)			/* 0x00010000 */
+#define	TDLS_PEER_AT_OFF_STATE			BIT(17)			/* 0x00020000 */	/* Could send pkt on target ch */
+#define	TDLS_CH_SW_INITIATOR_STATE		BIT(18)			/* 0x00040000 */	/* Avoid duplicated or unconditional ch. switch rsp. */
+#define	TDLS_WAIT_CH_RSP_STATE			BIT(19)			/* 0x00080000 */	/* Wait Ch. response as we are TDLS channel switch initiator */
+
+
+#define	TDLS_TPK_RESEND_COUNT			86400	/*Unit: seconds */
+#define	TDLS_CH_SWITCH_TIME				15
+#define	TDLS_CH_SWITCH_TIMEOUT			30
+#define	TDLS_CH_SWITCH_OPER_OFFLOAD_TIMEOUT	10
+#define	TDLS_SIGNAL_THRESH			0x20
+#define	TDLS_WATCHDOG_PERIOD		10	/* Periodically sending tdls discovery request in TDLS_WATCHDOG_PERIOD * 2 sec */
+#define	TDLS_HANDSHAKE_TIME			3000
+#define	TDLS_PTI_TIME				7000
+
+#define TDLS_CH_SW_STAY_ON_BASE_CHNL_TIMEOUT	20		/* ms */
+#define TDLS_CH_SW_MONITOR_TIMEOUT				2000	/*ms */
+
+#define TDLS_MIC_LEN 16
+#define WPA_NONCE_LEN 32
+#define TDLS_TIMEOUT_LEN 4
+
+enum TDLS_CH_SW_CHNL {
+	TDLS_CH_SW_BASE_CHNL = 0,
+	TDLS_CH_SW_OFF_CHNL
+};
+
+#define TDLS_MIC_CTRL_LEN 2
+#define TDLS_FTIE_DATA_LEN (TDLS_MIC_CTRL_LEN + TDLS_MIC_LEN + \
+							WPA_NONCE_LEN + WPA_NONCE_LEN)
+struct wpa_tdls_ftie {
+	u8 ie_type; /* FTIE */
+	u8 ie_len;
+	union {
+		struct {
+			u8 mic_ctrl[TDLS_MIC_CTRL_LEN];
+			u8 mic[TDLS_MIC_LEN];
+			u8 Anonce[WPA_NONCE_LEN]; /* Responder Nonce in TDLS */
+			u8 Snonce[WPA_NONCE_LEN]; /* Initiator Nonce in TDLS */
+		};
+		struct {
+			u8 data[TDLS_FTIE_DATA_LEN];
+		};
+	};
+	/* followed by optional elements */
+} ;
+
+struct wpa_tdls_lnkid {
+	u8 ie_type; /* Link Identifier IE */
+	u8 ie_len;
+	u8 bssid[ETH_ALEN];
+	u8 init_sta[ETH_ALEN];
+	u8 resp_sta[ETH_ALEN];
+} ;
+
+static u8 TDLS_RSNIE[20] = {	0x01, 0x00,	/* Version shall be set to 1 */
+				0x00, 0x0f, 0xac, 0x07,	/* Group sipher suite */
+				0x01, 0x00,	/* Pairwise cipher suite count */
+	0x00, 0x0f, 0xac, 0x04,	/* Pairwise cipher suite list; CCMP only */
+				0x01, 0x00,	/* AKM suite count */
+				0x00, 0x0f, 0xac, 0x07,	/* TPK Handshake */
+				0x0c, 0x02,
+				/* PMKID shall not be present */
+			   };
+
+static u8 TDLS_WMMIE[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01, 0x00};	/* Qos info all set zero */
+
+static u8 TDLS_WMM_PARAM_IE[] = {0x00, 0x00, 0x03, 0xa4, 0x00, 0x00, 0x27, 0xa4, 0x00, 0x00, 0x42, 0x43, 0x5e, 0x00, 0x62, 0x32, 0x2f, 0x00};
+
+static u8 TDLS_EXT_CAPIE[] = {0x00, 0x00, 0x00, 0x50, 0x20, 0x00, 0x00, 0x00};	/* bit(28), bit(30), bit(37) */
+
+/* SRC: Supported Regulatory Classes */
+static u8 TDLS_SRC[] = { 0x01, 0x01, 0x02, 0x03, 0x04, 0x0c, 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1d, 0x1e, 0x20, 0x21 };
+
+bool rtw_check_tdls_frame(void *priv, struct rtw_phl_stainfo_t *phl_sta,
+		struct rtw_recv_pkt *phlrx);
+int check_ap_tdls_prohibited(u8 *pframe, u8 pkt_len);
+int check_ap_tdls_ch_switching_prohibited(u8 *pframe, u8 pkt_len);
+
+void rtw_set_tdls_enable(_adapter *padapter, u8 enable);
+u8 rtw_is_tdls_enabled(_adapter *padapter);
+u8 rtw_is_tdls_sta_existed(_adapter *padapter);
+u8 rtw_tdls_is_setup_allowed(_adapter *padapter);
+#ifdef CONFIG_TDLS_CH_SW
+u8 rtw_tdls_is_chsw_allowed(_adapter *padapter);
+#endif
+
+void rtw_tdls_set_link_established(_adapter *adapter, bool en);
+void rtw_reset_tdls_info(_adapter *padapter);
+int rtw_init_tdls_info(_adapter *padapter);
+void rtw_free_tdls_info(struct tdls_info *ptdlsinfo);
+void rtw_free_all_tdls_sta(_adapter *padapter, u8 enqueue_cmd);
+void rtw_enable_tdls_func(_adapter *padapter);
+void rtw_disable_tdls_func(_adapter *padapter, u8 enqueue_cmd);
+int issue_nulldata_to_TDLS_peer_STA(_adapter *padapter, struct sta_info *sta,
+		unsigned int power_mode, int try_cnt, int wait_ms);
+void rtw_init_tdls_timer(_adapter *padapter, struct sta_info *psta);
+void	rtw_cancel_tdls_timer(struct sta_info *psta);
+void rtw_tdls_teardown_pre_hdl(_adapter *padapter, struct sta_info *psta);
+void rtw_tdls_teardown_post_hdl(_adapter *padapter, struct sta_info *psta, u8 enqueue_cmd);
+
+#ifdef CONFIG_TDLS_CH_SW
+void rtw_tdls_set_ch_sw_oper_control(_adapter *padapter, u8 enable);
+void rtw_tdls_ch_sw_back_to_base_chnl(_adapter *padapter);
+s32 rtw_tdls_do_ch_sw(_adapter *padapter, struct sta_info *ptdls_sta, u8 chnl_type, u8 channel, u8 channel_offset, u16 bwmode, u16 ch_switch_time);
+void rtw_tdls_chsw_oper_done(_adapter *padapter);
+#endif
+
+#ifdef CONFIG_WFD
+int issue_tunneled_probe_req(_adapter *padapter);
+int issue_tunneled_probe_rsp(_adapter *padapter, union recv_frame *precv_frame);
+#endif /* CONFIG_WFD */
+int issue_tdls_dis_req(_adapter *padapter, struct tdls_txmgmt *ptxmgmt);
+int issue_tdls_setup_req(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, int wait_ack);
+int issue_tdls_setup_rsp(_adapter *padapter, struct tdls_txmgmt *ptxmgmt);
+int issue_tdls_setup_cfm(_adapter *padapter, struct tdls_txmgmt *ptxmgmt);
+int issue_tdls_dis_rsp(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, u8 privacy);
+int issue_tdls_teardown(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, u8 wait_ack);
+int issue_tdls_peer_traffic_rsp(_adapter *padapter, struct sta_info *psta, struct tdls_txmgmt *ptxmgmt);
+int issue_tdls_peer_traffic_indication(_adapter *padapter, struct sta_info *psta);
+#ifdef CONFIG_TDLS_CH_SW
+int issue_tdls_ch_switch_req(_adapter *padapter, struct sta_info *ptdls_sta);
+int issue_tdls_ch_switch_rsp(_adapter *padapter, struct tdls_txmgmt *ptxmgmt, int wait_ack);
+#endif
+sint On_TDLS_Dis_Rsp(_adapter *adapter, union recv_frame *precv_frame);
+sint On_TDLS_Setup_Req(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+int On_TDLS_Setup_Rsp(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+int On_TDLS_Setup_Cfm(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+int On_TDLS_Dis_Req(_adapter *adapter, union recv_frame *precv_frame);
+int On_TDLS_Teardown(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+int On_TDLS_Peer_Traffic_Indication(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+int On_TDLS_Peer_Traffic_Rsp(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+#ifdef CONFIG_TDLS_CH_SW
+sint On_TDLS_Ch_Switch_Req(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+sint On_TDLS_Ch_Switch_Rsp(_adapter *adapter, union recv_frame *precv_frame, struct sta_info *ptdls_sta);
+void rtw_build_tdls_ch_switch_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+void rtw_build_tdls_ch_switch_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+#endif
+void rtw_build_tdls_setup_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+void rtw_build_tdls_setup_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+void rtw_build_tdls_setup_cfm_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+void rtw_build_tdls_teardown_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+void rtw_build_tdls_dis_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt);
+void rtw_build_tdls_dis_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, u8 privacy);
+void rtw_build_tdls_peer_traffic_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+void rtw_build_tdls_peer_traffic_indication_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe, struct tdls_txmgmt *ptxmgmt, struct sta_info *ptdls_sta);
+void rtw_build_tunneled_probe_req_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe);
+void rtw_build_tunneled_probe_rsp_ies(_adapter *padapter, struct xmit_frame *pxmitframe, u8 *pframe);
+
+int rtw_tdls_is_driver_setup(_adapter *padapter);
+void rtw_tdls_set_key(_adapter *padapter, struct sta_info *ptdls_sta);
+const char *rtw_tdls_action_txt(enum TDLS_ACTION_FIELD action);
+#endif /* CONFIG_TDLS */
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx.h
new file mode 100644
index 000000000000..de31744d9c28
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx.h
@@ -0,0 +1,35 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_TRX_H_
+#define _RTW_TRX_H_
+
+struct dvobj_priv;
+struct rtw_intf_ops {
+	/*** xmit section ***/
+	s32(*init_xmit_priv)(_adapter *adapter);
+	void(*free_xmit_priv)(_adapter *adapter);
+
+	s32(*data_xmit)(_adapter *adapter, struct xmit_frame *pxmitframe);
+
+	/*** recv section ***/
+	s32(*init_recv_priv)(struct dvobj_priv *dvobj);
+	void(*free_recv_priv)(struct dvobj_priv *dvobj);
+
+};
+
+s32 rtw_mgnt_xmit(_adapter *adapter, struct xmit_frame *pmgntframe);
+
+
+#endif /* _RTW_TRX_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_ops.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_ops.h
new file mode 100644
index 000000000000..beeeffb6ab74
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_ops.h
@@ -0,0 +1,63 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_TRX_OPS_H_
+#define _RTW_TRX_OPS_H_
+#include <drv_types.h>
+
+static inline s32 rtw_intf_init_xmit_priv(_adapter *adapter)
+{
+	return adapter_to_dvobj(adapter)->intf_ops->init_xmit_priv(adapter);
+}
+
+static inline void rtw_intf_free_xmit_priv(_adapter *adapter)
+{
+
+	adapter_to_dvobj(adapter)->intf_ops->free_xmit_priv(adapter);
+}
+static inline s32 rtw_intf_data_xmit(_adapter *adapter,
+					struct xmit_frame *pxmitframe)
+{
+	return adapter_to_dvobj(adapter)->intf_ops->data_xmit(adapter, pxmitframe);
+}
+
+static inline s32 rtw_intf_xmitframe_enqueue(_adapter *adapter,
+						struct xmit_frame *pxmitframe)
+{
+	u32 rtn;
+
+	/* enqueue is not necessary, casuse phl use sw queue to save xmitframe */
+	rtn = core_tx_call_phl(adapter, pxmitframe, NULL);
+
+	if (rtn == FAIL)
+		core_tx_free_xmitframe(adapter, pxmitframe);
+
+	return rtn;
+}
+
+/************************ recv *******************/
+static inline s32 rtw_intf_init_recv_priv(struct dvobj_priv *dvobj)
+{
+	return dvobj->intf_ops->init_recv_priv(dvobj);
+}
+static inline void rtw_intf_free_recv_priv(struct dvobj_priv *dvobj)
+{
+	return dvobj->intf_ops->free_recv_priv(dvobj);
+}
+
+struct lite_data_buf *rtw_alloc_litedatabuf(struct trx_data_buf_q *data_buf_q);
+s32 rtw_free_litedatabuf(struct trx_data_buf_q *data_buf_q,
+		struct lite_data_buf *lite_data_buf);
+
+#endif /* _RTW_TRX_OPS_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_pci.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_pci.h
new file mode 100644
index 000000000000..e78bfe8b8f96
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_pci.h
@@ -0,0 +1,26 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_TRX_PCI_H_
+#define _RTW_TRX_PCI_H_
+
+extern struct rtw_intf_ops pci_ops;
+
+static inline u8 is_pci_support_dma64(struct dvobj_priv *dvobj)
+{
+	PPCI_DATA pci_data = dvobj_to_pci(dvobj);
+
+	return (pci_data->bdma64 == _TRUE) ? _TRUE : _FALSE;
+}
+#endif /* _RTW_TRX_PCI_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_usb.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_usb.h
new file mode 100644
index 000000000000..3f0e1012aa66
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_trx_usb.h
@@ -0,0 +1,21 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_TRX_USB_H_
+#define _RTW_TRX_USB_H_
+#include <drv_types.h>		/* struct dvobj_priv and etc. */
+
+extern struct rtw_intf_ops usb_ops;
+
+#endif /* _RTW_TRX_USB_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_twt.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_twt.h
new file mode 100644
index 000000000000..c58a45b7b58d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_twt.h
@@ -0,0 +1,106 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __RTW_TWT_H_
+#define __RTW_TWT_H_
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+
+#define TWT_REQ_SUPP(twt_cap) (twt_cap & RTW_PHL_TWT_REQ_SUP)
+#define TWT_RSP_SUPP(twt_cap) (twt_cap & RTW_PHL_TWT_RSP_SUP)
+
+#define TWT_NEGO_TYPE(twt_ele) (twt_ele)->twt_ctrl.nego_type
+#define TWT_NEGO_TYPE_I(twt_ele) (((twt_ele)->twt_ctrl.nego_type == RTW_PHL_INDIV_TWT) \
+				  || ((twt_ele)->twt_ctrl.nego_type == RTW_PHL_WAKE_TBTT_INR))
+#define TWT_NEGO_TYPE_B(twt_ele) (((twt_ele)->twt_ctrl.nego_type == RTW_PHL_BCAST_TWT) \
+				  || ((twt_ele)->twt_ctrl.nego_type == RTW_PHL_MANAGE_BCAST_TWT))
+#define TWT_REQER_I(twt_ele) (twt_ele)->info.i_twt_para_set.req_type.twt_request
+#define TWT_REQER_B(twt_ele, bidx) (twt_ele)->info.b_twt_para_set[bidx].req_type.twt_request
+#define TWT_SETUP_CMD_I(twt_ele) (twt_ele)->info.i_twt_para_set.req_type.twt_setup_cmd
+#define TWT_SETUP_CMD_B(twt_ele, bidx) (twt_ele)->info.b_twt_para_set[bidx].req_type.twt_setup_cmd
+
+enum rtw_core_twt_cmd_id {
+	CORE_TWT_CMD_ISETUP,
+	CORE_TWT_CMD_BSETUP,
+	CORE_TWT_CMD_ITEARDOWN,
+	CORE_TWT_CMD_BTEARDOWN,
+	CORE_TWT_CMD_SET_INFO,
+	CORE_TWT_CMD_ON_ISETUP,
+	CORE_TWT_CMD_ON_BSETUP,
+	CORE_TWT_CMD_ON_TEARDOWN
+};
+
+struct rtw_core_twt_cmd_obj {
+	_adapter *padapter;
+	struct _ADAPTER_LINK *padapter_link;
+	struct sta_info *psta;
+	enum rtw_core_twt_cmd_id cmd_id;
+	void *pparm;
+	u32 parm_sz;
+	bool pparm_need_free;
+};
+
+struct rtw_core_twt_set_info {
+	bool suspend;
+	u32 dur;
+};
+
+u8 rtw_core_twt_init(_adapter *padapter);
+u8 rtw_core_twt_reset(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+u8 rtw_core_twt_hdl(_adapter *padapter, u8 *pbuf);
+u8 rtw_core_twt_cmd(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		    struct sta_info *psta, enum rtw_core_twt_cmd_id cmd_id,
+		    void *pparm, u32 pparm_sz, bool parm_need_free);
+u8 rtw_core_twt_set_ctrl(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 u32 ndp_pag_ind, u32 rsp_pm_mode, u32 nego_type,
+			 u32 twt_info_dis, u32 wk_dur_unit);
+u8 rtw_core_twt_set_ireq(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 u32 request, u32 setup_cmd, u32 trigger, u32 implicit,
+			 u32 flow_type, u32 wk_intvl_exp, u32 protection);
+u8 rtw_core_twt_set_isetup(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			   struct sta_info *psta, u32 target_wk_time,
+			   u32 nom_min_wk_dur, u32 wk_int_mantissa, u32 twt_chnl);
+u8 rtw_core_twt_set_iteardown(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      struct sta_info *psta, u32 nego_type,
+			      u32 flow_id, u32 trdwn_all);
+u8 rtw_core_twt_set_breq(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 u32 request, u32 setup_cmd, u32 trigger, u32 lst_bcparm_set,
+			 u32 flow_type, u32 btwt_rcmd, u32 wk_intvl_exp);
+u8 rtw_core_twt_set_bsetup(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			   struct sta_info *psta, u32 target_wk_time,
+			   u32 nom_min_wk_dur, u32 wk_int_mantissa, u32 btwt_id,
+			   u32 btwt_prstnc);
+u8 rtw_core_twt_set_bteardown(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      struct sta_info *psta, u32 nego_type,
+			      u32 btwt_id, u32 trdwn_all);
+u8 rtw_core_twt_set_info(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			 struct sta_info *psta, bool suspend, u32 dur);
+u8 rtw_issue_twt_setup(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		       u8 *da, struct rtw_phl_twt_setup_info *twt_setup_info);
+u8 rtw_issue_twt_teardown(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			  u8 *da, struct rtw_phl_twt_flow_field *twt_flow_info,
+			  int try_cnt, int wait_ms);
+u8 rtw_issue_twt_teardown_cmn(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			      u8 *da, struct rtw_phl_twt_element *twt_ele,
+			      int try_cnt, int wait_ms, bool individual);
+u8 rtw_issue_twt_info(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+		       u8 *da, struct rtw_phl_twt_info_f *twt_info);
+u8 rtw_core_twt_on_action(_adapter *padapter, u8 *raddr, u8 *pframe_body, u32 frame_len);
+
+#endif /*CONFIG_TWT */
+#endif /* CONFIG_80211AX_HE */
+
+#endif /* __RTW_TWT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_version.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_version.h
new file mode 100644
index 000000000000..c2539ce95451
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_version.h
@@ -0,0 +1 @@
+#define DRIVERVERSION	"v1.19.17-248-g19f2c2a389.20250825"
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_vht.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_vht.h
new file mode 100644
index 000000000000..3fb6aefa348a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_vht.h
@@ -0,0 +1,250 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_VHT_H_
+#define _RTW_VHT_H_
+
+#define VHT_CAP_IE_LEN 12
+#define VHT_OP_IE_LEN 5
+
+/* channel width defined in 802.11-2016, Table 9-252 VHT operation information subfields
+* 0 for 20 MHz or 40 MHz
+* 1 for 80 MHz, 160 MHz or 80+80 MHz
+* 2 for 160 MHz (deprecated)
+* 3 for non-contiguous 80+80 MHz (deprecated)
+*/
+#define CH_WIDTH_20_40M 0
+#define CH_WIDTH_80_160M 1
+
+#define	LDPC_VHT_ENABLE_RX			BIT0
+#define	LDPC_VHT_ENABLE_TX			BIT1
+#define	LDPC_VHT_TEST_TX_ENABLE		BIT2
+#define	LDPC_VHT_CAP_TX				BIT3
+
+#define	STBC_VHT_ENABLE_RX			BIT0
+#define	STBC_VHT_ENABLE_TX			BIT1
+#define	STBC_VHT_TEST_TX_ENABLE		BIT2
+#define	STBC_VHT_CAP_TX				BIT3
+
+/* VHT capability info */
+#define SET_VHT_CAPABILITY_ELE_MAX_MPDU_LENGTH(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 2, _val)
+#define SET_VHT_CAPABILITY_ELE_CHL_WIDTH(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(_pEleStart, 2, 2, _val)
+#define SET_VHT_CAPABILITY_ELE_RX_LDPC(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(_pEleStart, 4, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_SHORT_GI80M(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE(_pEleStart, 5, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_SHORT_GI160M(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE(_pEleStart, 6, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_TX_STBC(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE(_pEleStart, 7, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_RX_STBC(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+1, 0, 3, _val)
+#define SET_VHT_CAPABILITY_ELE_SU_BFER(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+1, 3, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_SU_BFEE(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+1, 4, 1, _val)
+/* #define SET_VHT_CAPABILITY_ELE_BFER_ANT_SUPP(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE((_pEleStart)+1, 5, 3, _val) */
+#define SET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE((_pEleStart)+1, 5, 3, _val)
+#define SET_VHT_CAPABILITY_ELE_SOUNDING_DIMENSIONS(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE((_pEleStart)+2, 0, 3, _val) /* B16~B18 */
+
+#define SET_VHT_CAPABILITY_ELE_MU_BFER(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+2, 3, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_MU_BFEE(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+2, 4, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_TXOP_PS(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+2, 5, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_HTC_VHT(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+2, 6, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(_pEleStart, _val)		SET_BITS_TO_LE_2BYTE((_pEleStart)+2, 7, 3, _val) /* B23~B25 */
+#define SET_VHT_CAPABILITY_ELE_LINK_ADAPTION(_pEleStart, _val)				SET_BITS_TO_LE_1BYTE((_pEleStart)+2, 2, 2, _val)
+#define SET_VHT_CAPABILITY_ELE_MCS_RX_MAP(_pEleStart, _val)				SET_BITS_TO_LE_2BYTE((_pEleStart)+4, 0, 16, _val)   /* B0~B15 indicate Rx MCS MAP, we write 0 to indicate MCS0~7. by page */
+#define SET_VHT_CAPABILITY_ELE_MCS_RX_HIGHEST_RATE(_pEleStart, _val)				SET_BITS_TO_LE_2BYTE((_pEleStart)+6, 0, 13, _val)
+#define SET_VHT_CAPABILITY_ELE_MCS_TX_MAP(_pEleStart, _val)				SET_BITS_TO_LE_2BYTE((_pEleStart)+8, 0, 16, _val)   /* B0~B15 indicate Tx MCS MAP, we write 0 to indicate MCS0~7. by page */
+#define SET_VHT_CAPABILITY_ELE_MCS_TX_HIGHEST_RATE(_pEleStart, _val)				SET_BITS_TO_LE_2BYTE((_pEleStart)+10, 0, 13, _val)
+#define SET_VHT_CAPABILITY_ELE_MCS_TX_EXT_NSS_BW_CAP(_pEleStart, _val)				SET_BITS_TO_LE_2BYTE((_pEleStart)+10, 13, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_RX_ANT_PATTERN(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE((_pEleStart)+3, 4, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_TX_ANT_PATTERN(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE((_pEleStart)+3, 5, 1, _val)
+#define SET_VHT_CAPABILITY_ELE_EXT_NSS_BW(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE((_pEleStart)+3, 6, 2, _val)
+
+#define GET_VHT_CAPABILITY_ELE_MAX_MPDU_LENGTH(_pEleStart)			LE_BITS_TO_1BYTE(_pEleStart, 0, 2)
+#define GET_VHT_CAPABILITY_ELE_CHL_WIDTH(_pEleStart)				LE_BITS_TO_1BYTE(_pEleStart, 2, 2)
+#define GET_VHT_CAPABILITY_ELE_RX_LDPC(_pEleStart)			LE_BITS_TO_1BYTE(_pEleStart, 4, 1)
+#define GET_VHT_CAPABILITY_ELE_SHORT_GI80M(_pEleStart)				LE_BITS_TO_1BYTE(_pEleStart, 5, 1)
+#define GET_VHT_CAPABILITY_ELE_SHORT_GI160M(_pEleStart)				LE_BITS_TO_1BYTE(_pEleStart, 6, 1)
+#define GET_VHT_CAPABILITY_ELE_TX_STBC(_pEleStart)				LE_BITS_TO_1BYTE(_pEleStart, 7, 1)
+#define GET_VHT_CAPABILITY_ELE_RX_STBC(_pEleStart)				LE_BITS_TO_1BYTE((_pEleStart)+1, 0, 3)
+#define GET_VHT_CAPABILITY_ELE_SU_BFER(_pEleStart)					LE_BITS_TO_1BYTE((_pEleStart)+1, 3, 1)
+#define GET_VHT_CAPABILITY_ELE_SU_BFEE(_pEleStart)					LE_BITS_TO_1BYTE((_pEleStart)+1, 4, 1)
+/*phydm-beamforming*/
+#define GET_VHT_CAPABILITY_ELE_SU_BFEE_STS_CAP(_pEleStart)	LE_BITS_TO_1BYTE((_pEleStart)+1, 5, 3)
+#define GET_VHT_CAPABILITY_ELE_SU_BFER_SOUND_DIM_NUM(_pEleStart)	LE_BITS_TO_2BYTE((_pEleStart)+2, 0, 3)
+#define GET_VHT_CAPABILITY_ELE_MU_BFER(_pEleStart)				LE_BITS_TO_1BYTE((_pEleStart)+2, 3, 1)
+#define GET_VHT_CAPABILITY_ELE_MU_BFEE(_pEleStart)				LE_BITS_TO_1BYTE((_pEleStart)+2, 4, 1)
+#define GET_VHT_CAPABILITY_ELE_TXOP_PS(_pEleStart)				LE_BITS_TO_1BYTE((_pEleStart)+2, 5, 1)
+#define GET_VHT_CAPABILITY_ELE_HTC_VHT(_pEleStart)			LE_BITS_TO_1BYTE((_pEleStart)+2, 6, 1)
+#define GET_VHT_CAPABILITY_ELE_MAX_RXAMPDU_FACTOR(_pEleStart)	LE_BITS_TO_2BYTE((_pEleStart)+2, 7, 3)
+#define GET_VHT_CAPABILITY_ELE_LINK_ADAPTION(_pEleStart)			LE_BITS_TO_1BYTE((_pEleStart)+3, 2, 2)
+#define GET_VHT_CAPABILITY_ELE_RX_ANT_PATTERN(_pEleStart)		LE_BITS_TO_1BYTE((_pEleStart)+3, 4, 1)
+#define GET_VHT_CAPABILITY_ELE_TX_ANT_PATTERN(_pEleStart)		LE_BITS_TO_1BYTE((_pEleStart)+3, 5, 1)
+#define GET_VHT_CAPABILITY_ELE_EXT_NSS_BW(_pEleStart)			LE_BITS_TO_1BYTE((_pEleStart)+3, 6, 2)
+#define GET_VHT_CAPABILITY_ELE_RX_MCS(_pEleStart)					       ((_pEleStart)+4)
+#define GET_VHT_CAPABILITY_ELE_MCS_RX_HIGHEST_RATE(_pEleStart)			LE_BITS_TO_2BYTE((_pEleStart)+6, 0, 13)
+#define GET_VHT_CAPABILITY_ELE_TX_MCS(_pEleStart)					       ((_pEleStart)+8)
+#define GET_VHT_CAPABILITY_ELE_MCS_TX_HIGHEST_RATE(_pEleStart)			LE_BITS_TO_2BYTE((_pEleStart)+10, 0, 13)
+
+
+/* VHT Operation Information Element */
+#define SET_VHT_OPERATION_ELE_CHL_WIDTH(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 8, _val)
+#define SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(_pEleStart+1, 0, 8, _val)
+#define SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(_pEleStart+2, 0, 8, _val)
+#define SET_VHT_OPERATION_ELE_BASIC_MCS_SET(_pEleStart, _val)			SET_BITS_TO_LE_2BYTE((_pEleStart)+3, 0, 16, _val)
+
+#define GET_VHT_OPERATION_ELE_CHL_WIDTH(_pEleStart)		LE_BITS_TO_1BYTE(_pEleStart, 0, 8)
+#define GET_VHT_OPERATION_ELE_CENTER_FREQ1(_pEleStart)	LE_BITS_TO_1BYTE((_pEleStart)+1, 0, 8)
+#define GET_VHT_OPERATION_ELE_CENTER_FREQ2(_pEleStart)     LE_BITS_TO_1BYTE((_pEleStart)+2, 0, 8)
+#define GET_VHT_OPERATION_ELE_BASIC_MCS_SET(_pEleStart)	LE_BITS_TO_2BYTE((_pEleStart)+3, 0, 16)
+
+/* VHT Operating Mode */
+#define SET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(_pEleStart, 0, 2, _val)
+#define SET_VHT_OPERATING_MODE_FIELD_80_80_160_BW(_pEleStart, _val)		SET_BITS_TO_LE_1BYTE(_pEleStart, 2, 1, _val)
+#define SET_VHT_OPERATING_MODE_FIELD_RX_NSS(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE(_pEleStart, 4, 3, _val)
+#define SET_VHT_OPERATING_MODE_FIELD_RX_NSS_TYPE(_pEleStart, _val)	SET_BITS_TO_LE_1BYTE(_pEleStart, 7, 1, _val)
+#define GET_VHT_OPERATING_MODE_FIELD_CHNL_WIDTH(_pEleStart)			LE_BITS_TO_1BYTE(_pEleStart, 0, 2)
+#define GET_VHT_OPERATING_MODE_FIELD_80_80_160_BW(_pEleStart)			LE_BITS_TO_1BYTE(_pEleStart, 2, 1)
+#define GET_VHT_OPERATING_MODE_FIELD_RX_NSS(_pEleStart)				LE_BITS_TO_1BYTE(_pEleStart, 4, 3)
+#define GET_VHT_OPERATING_MODE_FIELD_RX_NSS_TYPE(_pEleStart)		LE_BITS_TO_1BYTE(_pEleStart, 7, 1)
+
+#define SET_EXT_CAPABILITY_ELE_OP_MODE_NOTIF(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE((_pEleStart)+7, 6, 1, _val)
+#define GET_EXT_CAPABILITY_ELE_OP_MODE_NOTIF(_pEleStart)				LE_BITS_TO_1BYTE((_pEleStart)+7, 6, 1)
+
+#define VHT_MAX_MPDU_LEN_MAX 3
+extern const u16 _vht_max_mpdu_len[];
+#define vht_max_mpdu_len(val) (((val) >= VHT_MAX_MPDU_LEN_MAX) ? _vht_max_mpdu_len[VHT_MAX_MPDU_LEN_MAX] : _vht_max_mpdu_len[(val)])
+
+#define VHT_SUP_CH_WIDTH_SET_MAX 3
+extern const u8 _vht_sup_ch_width_set_to_bw_cap[];
+#define vht_sup_ch_width_set_to_bw_cap(set) (((set) >= VHT_SUP_CH_WIDTH_SET_MAX) ? _vht_sup_ch_width_set_to_bw_cap[VHT_SUP_CH_WIDTH_SET_MAX] : _vht_sup_ch_width_set_to_bw_cap[(set)])
+extern const char *const _vht_sup_ch_width_set_str[];
+#define vht_sup_ch_width_set_str(set) (((set) >= VHT_SUP_CH_WIDTH_SET_MAX) ? _vht_sup_ch_width_set_str[VHT_SUP_CH_WIDTH_SET_MAX] : _vht_sup_ch_width_set_str[(set)])
+
+#define VHT_MAX_AMPDU_LEN(f) ((1 << (13 + f)) - 1)
+void dump_vht_cap_ie(void *sel, const u8 *ie, u32 ie_len);
+
+#define VHT_OP_CH_WIDTH_MAX 4
+extern const char *const _vht_op_ch_width_str[];
+#define vht_op_ch_width_str(ch_width) (((ch_width) >= VHT_OP_CH_WIDTH_MAX) ? _vht_op_ch_width_str[VHT_OP_CH_WIDTH_MAX] : _vht_op_ch_width_str[(ch_width)])
+
+void dump_vht_op_ie(void *sel, const u8 *ie, u32 ie_len);
+
+struct vht_bf_cap {
+	u8 is_mu_bfer;
+	u8 su_sound_dim;
+};
+
+struct vht_priv {
+	u8 vht_option;
+
+	u8 ldpc_cap;
+	u8 stbc_cap;
+	u8 rx_stbc_nss; /* Support nss spatial stream */
+	u16 beamform_cap;
+	struct vht_bf_cap ap_bf_cap;
+	u8 num_snd_dim;
+	u8 bfme_sts;
+	u8 sgi_80m;
+	u8 sgi_160m;
+	u8 ampdu_len;/* A-MPDU length exponent : 0 to 7 */
+
+	/*
+	 * max_amsdu_len
+	 * 0: 3895 bytes
+	 * 1: 7991 bytes
+	 * 2: 11454 bytes
+	 */
+	u8 max_mpdu_len;
+
+	u8 vht_highest_rate;
+	u8 vht_mcs_map[2];
+
+	u8 txop_ps; /* TXOP power save mode*/
+	u8 htc_vht; /* VHT variant HT Control*/
+	u8 link_adap_cap; /* VHT Link Adaptation Capable */
+	u8 tx_ant_pattern; /* Tx Antenna Pattern Consistency */
+	u8 rx_ant_pattern; /* Rx Antenna Pattern Consistency */
+	u8 ext_nss_bw; /* Extended NSS BW Support */
+
+	u8 op_present:1; /* vht_op is present */
+	u8 notify_present:1; /* vht_op_mode_notify is present */
+
+	u8 vht_cap[32];
+	u8 vht_op[VHT_OP_IE_LEN];
+	u8 vht_op_mode_notify;
+
+	/* Backup these two VHT IEs from hostapd/wpa_supplicant for restore usage */
+	u8 vht_cap_ie_backup[VHT_CAP_IE_LEN];
+	u8 vht_op_ie_backup[VHT_OP_IE_LEN];
+};
+
+#ifdef PRIVATE_R
+struct vht_priv_infra_ap {
+
+	/* Infra mode, only store for AP's info, not intersection of STA and AP*/
+	u8	ldpc_cap_infra_ap;
+	u8	stbc_cap_infra_ap;
+	u16	beamform_cap_infra_ap;
+	u8	vht_mcs_map_infra_ap[2];
+	u8	vht_mcs_map_tx_infra_ap[2];
+	u8	channel_width_infra_ap;
+	u8	number_of_streams_infra_ap;
+};
+#endif /* PRIVATE_R */
+
+u8	rtw_get_vht_highest_rate(u8 *pvht_mcs_map);
+u16	rtw_vht_mcs_to_data_rate(u8 bw, u8 short_GI, u8 vht_mcs_rate);
+u64	rtw_vht_mcs_map_to_bitmap(u8 *mcs_map, u8 nss);
+struct protocol_cap_t;
+struct role_cap_t;
+struct role_link_cap_t;
+u32	rtw_get_dft_vht_cap_ie(_adapter *padapter, u8 *pbuf);
+void rtw_vht_get_dft_setting(_adapter *padapter,
+			struct protocol_cap_t *dft_proto_cap,
+			struct role_link_cap_t *dft_cap);
+void	rtw_vht_get_real_setting(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+u32	rtw_build_vht_operation_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+						u8 *pbuf, enum band_type band, u8 channel);
+u32	rtw_build_vht_op_mode_notify_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+						u8 *pbuf, u8 bw);
+u32	rtw_build_vht_cap_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, u8 *pbuf);
+void	update_sta_vht_info_apmode(_adapter *padapter, void *psta);
+void	VHT_caps_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				PNDIS_802_11_VARIABLE_IEs pIE);
+#ifdef PRIVATE_R
+void	VHT_caps_handler_infra_ap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			PNDIS_802_11_VARIABLE_IEs pIE);
+#endif /* PRIVATE_R */
+void	VHT_operation_handler(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+					PNDIS_802_11_VARIABLE_IEs pIE);
+void	rtw_process_vht_op_mode_notify(_adapter *padapter, u8 *pframe, void *sta);
+u8 rtw_vht_get_oper_bw(u8 *ht_op_ie, u8 *vht_cap_ie, u8 *vht_op_ie, u8 oper_bw);
+u32	rtw_restructure_vht_ie(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+				u8 *in_ie, u8 *out_ie, uint in_len, uint *pout_len, enum band_type band);
+void	VHTOnAssocRsp(_adapter *padapter);
+u8	rtw_vht_mcsmap_to_nss(u8 *pvht_mcs_map);
+void rtw_vht_nss_to_mcsmap(u8 nss, u8 *target_mcs_map, u8 *cur_mcs_map);
+void rtw_vht_ies_attach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			WLAN_BSSID_EX *pcur_network);
+void rtw_vht_ies_detach(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			WLAN_BSSID_EX *pcur_network);
+void rtw_check_for_vht20(_adapter *adapter, u8 *ies, int ies_len);
+void rtw_update_drv_vht_cap(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			u8 *vht_cap_ie);
+void rtw_check_vht_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link,
+			WLAN_BSSID_EX *pnetwork);
+void rtw_update_probe_rsp_vht_cap(struct _ADAPTER *a, u8 *ies, sint ies_len);
+void rtw_reattach_vht_ies(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, WLAN_BSSID_EX *pnetwork);
+u8 rtw_issue_op_mode_notify_frame(_adapter *a, struct _ADAPTER_LINK *a_link,
+		u8 *ra, u8 rx_nss, enum channel_width bw, u8 try_cnt, u8 wait_ms);
+struct sta_info;
+void rtw_vht_op_mode_ctrl_rx_nss(_adapter *adapter, struct _ADAPTER_LINK *a_link,
+		struct sta_info *sta, u8 final_rx_nss, bool need_update_ra);
+#endif /* _RTW_VHT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wapi.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wapi.h
new file mode 100644
index 000000000000..4b53bd161755
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wapi.h
@@ -0,0 +1,230 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __INC_WAPI_H
+#define __INC_WAPI_H
+
+
+#define CONFIG_WAPI_SW_SMS4
+#define WAPI_DEBUG
+
+#define SMS4_MIC_LEN                16
+#define WAPI_EXT_LEN                18
+#define MAX_WAPI_IE_LEN		    256
+#define sMacHdrLng				24		/* octets in data header, no WEP */
+
+#ifdef WAPI_DEBUG
+
+/* WAPI trace debug */
+extern u32 wapi_debug_component;
+
+static inline void wapi_dump_buf(u8 *buf, u32 len)
+{
+	u32 i;
+	printk("-----------------Len %d----------------\n", len);
+	for (i = 0; i < len; i++)
+		printk(KERN_CONT"%2.2x-", *(buf + i));
+	printk("\n");
+}
+
+#define WAPI_TRACE(component, x, args...) \
+	do { if (wapi_debug_component & (component)) \
+			printk(KERN_DEBUG "WAPI" ":" x "" , \
+			       ##args);\
+	} while (0);
+
+#define WAPI_DATA(component, x, buf, len) \
+	do { if (wapi_debug_component & (component)) { \
+			printk("%s:\n", x);\
+			wapi_dump_buf((buf), (len)); } \
+	} while (0);
+
+#define RT_ASSERT_RET(_Exp)								\
+	if (!(_Exp)) {									\
+		printk("RTWLAN: ");					\
+		printk("Assertion failed! %s,%s, line=%d\n", \
+		       #_Exp, __FUNCTION__, __LINE__);          \
+		return;						\
+	}
+#define RT_ASSERT_RET_VALUE(_Exp, Ret)								\
+	if (!(_Exp)) {									\
+		printk("RTWLAN: ");					\
+		printk("Assertion failed! %s,%s, line=%d\n", \
+		       #_Exp, __FUNCTION__, __LINE__);          \
+		return Ret;						\
+	}
+
+#else
+#define RT_ASSERT_RET(_Exp) do {} while (0)
+#define RT_ASSERT_RET_VALUE(_Exp, Ret) do {} while (0)
+#define WAPI_TRACE(component, x, args...) do {} while (0)
+#define WAPI_DATA(component, x, buf, len) do {} while (0)
+#endif
+
+
+enum WAPI_DEBUG {
+	WAPI_INIT				= 1,
+	WAPI_API				= 1 << 1,
+	WAPI_TX				= 1 << 2,
+	WAPI_RX				= 1 << 3,
+	WAPI_MLME				= 1 << 4,
+	WAPI_IOCTL				= 1 << 5,
+	WAPI_ERR			= 1 << 31
+};
+
+#define			WAPI_MAX_BKID_NUM				4
+#define			WAPI_MAX_STAINFO_NUM			4
+#define			WAPI_CAM_ENTRY_NUM			14	/* 28/2 = 14 */
+
+typedef struct  _RT_WAPI_BKID {
+	struct list_head	list;
+	u8				bkid[16];
+} RT_WAPI_BKID, *PRT_WAPI_BKID;
+
+typedef struct  _RT_WAPI_KEY {
+	u8			dataKey[16];
+	u8			micKey[16];
+	u8			keyId;
+	bool			bSet;
+	bool             bTxEnable;
+} RT_WAPI_KEY, *PRT_WAPI_KEY;
+
+typedef enum _RT_WAPI_PACKET_TYPE {
+	WAPI_NONE = 0,
+	WAPI_PREAUTHENTICATE = 1,
+	WAPI_STAKEY_REQUEST = 2,
+	WAPI_AUTHENTICATE_ACTIVE = 3,
+	WAPI_ACCESS_AUTHENTICATE_REQUEST = 4,
+	WAPI_ACCESS_AUTHENTICATE_RESPONSE = 5,
+	WAPI_CERTIFICATE_AUTHENTICATE_REQUEST = 6,
+	WAPI_CERTIFICATE_AUTHENTICATE_RESPONSE = 7,
+	WAPI_USK_REQUEST = 8,
+	WAPI_USK_RESPONSE = 9,
+	WAPI_USK_CONFIRM = 10,
+	WAPI_MSK_NOTIFICATION = 11,
+	WAPI_MSK_RESPONSE = 12
+} RT_WAPI_PACKET_TYPE;
+
+typedef struct	_RT_WAPI_STA_INFO {
+	struct list_head		list;
+	u8					PeerMacAddr[6];
+	RT_WAPI_KEY		      wapiUsk;
+	RT_WAPI_KEY		      wapiUskUpdate;
+	RT_WAPI_KEY		      wapiMsk;
+	RT_WAPI_KEY		      wapiMskUpdate;
+	u8					lastRxUnicastPN[16];
+	u8					lastTxUnicastPN[16];
+	u8					lastRxMulticastPN[16];
+	u8					lastRxUnicastPNBEQueue[16];
+	u8					lastRxUnicastPNBKQueue[16];
+	u8					lastRxUnicastPNVIQueue[16];
+	u8					lastRxUnicastPNVOQueue[16];
+	bool					bSetkeyOk;
+	bool					bAuthenticateInProgress;
+	bool					bAuthenticatorInUpdata;
+} RT_WAPI_STA_INFO, *PRT_WAPI_STA_INFO;
+
+/* Added for HW wapi en/decryption */
+typedef struct _RT_WAPI_CAM_ENTRY {
+	/* RT_LIST_ENTRY		list; */
+	u8			IsUsed;
+	u8			entry_idx;/* for cam entry */
+	u8			keyidx;	/* 0 or 1,new or old key */
+	u8			PeerMacAddr[6];
+	u8			type;	/* should be 110,wapi */
+} RT_WAPI_CAM_ENTRY, *PRT_WAPI_CAM_ENTRY;
+
+typedef struct _RT_WAPI_T {
+	/* BKID */
+	RT_WAPI_BKID		wapiBKID[WAPI_MAX_BKID_NUM];
+	struct list_head		wapiBKIDIdleList;
+	struct list_head		wapiBKIDStoreList;
+	/* Key for Tx Multicast/Broadcast */
+	RT_WAPI_KEY		      wapiTxMsk;
+
+	/* sec related */
+	u8				lastTxMulticastPN[16];
+	/* STA list */
+	RT_WAPI_STA_INFO	wapiSta[WAPI_MAX_STAINFO_NUM];
+	struct list_head		wapiSTAIdleList;
+	struct list_head		wapiSTAUsedList;
+	/*  */
+	bool				bWapiEnable;
+
+	/* store WAPI IE */
+	u8				wapiIE[256];
+	u8				wapiIELength;
+	bool				bWapiPSK;
+	/* last sequece number for wai packet */
+	u16				wapiSeqnumAndFragNum;
+	int extra_prefix_len;
+	int extra_postfix_len;
+
+	RT_WAPI_CAM_ENTRY	wapiCamEntry[WAPI_CAM_ENTRY_NUM];
+} RT_WAPI_T, *PRT_WAPI_T;
+
+typedef struct _WLAN_HEADER_WAPI_EXTENSION {
+	u8      KeyIdx;
+	u8      Reserved;
+	u8      PN[16];
+} WLAN_HEADER_WAPI_EXTENSION, *PWLAN_HEADER_WAPI_EXTENSION;
+
+u32 WapiComparePN(u8 *PN1, u8 *PN2);
+
+
+void rtw_wapi_init(_adapter *padapter);
+
+void rtw_wapi_free(_adapter *padapter);
+
+void rtw_wapi_disable_tx(_adapter *padapter);
+
+u8 rtw_wapi_is_wai_packet(_adapter *padapter, u8 *pkt_data);
+
+void rtw_wapi_update_info(_adapter *padapter, union recv_frame *precv_frame);
+
+u8 rtw_wapi_check_for_drop(_adapter *padapter, union recv_frame *precv_frame, u8 *ehdr_ops);
+
+void rtw_build_probe_resp_wapi_ie(_adapter *padapter, unsigned char *pframe, struct pkt_attrib *pattrib);
+
+void rtw_build_beacon_wapi_ie(_adapter *padapter, unsigned char *pframe, struct pkt_attrib *pattrib);
+
+void rtw_build_assoc_req_wapi_ie(_adapter *padapter, unsigned char *pframe, struct pkt_attrib *pattrib);
+
+void rtw_wapi_on_assoc_ok(_adapter *padapter, PNDIS_802_11_VARIABLE_IEs pIE);
+
+void rtw_wapi_return_one_sta_info(_adapter *padapter, u8 *MacAddr);
+
+void rtw_wapi_return_all_sta_info(_adapter *padapter);
+
+void rtw_wapi_clear_cam_entry(_adapter *padapter, u8 *pMacAddr);
+
+void rtw_wapi_clear_all_cam_entry(_adapter *padapter);
+
+void rtw_wapi_set_key(_adapter *padapter, RT_WAPI_KEY *pWapiKey, RT_WAPI_STA_INFO *pWapiSta, u8 bGroupKey);
+
+int rtw_wapi_create_event_send(_adapter *padapter, u8 EventId, u8 *MacAddr, u8 *Buff, u16 BufLen);
+
+u32	rtw_sms4_encrypt(_adapter *padapter, u8 *pxmitframe);
+
+u32	rtw_sms4_decrypt(_adapter *padapter, u8 *precvframe);
+
+void rtw_wapi_get_iv(_adapter *padapter, u8 *pRA, u8 *IV);
+
+u8 WapiIncreasePN(u8 *PN, u8 AddCount);
+
+bool rtw_wapi_drop_for_key_absent(_adapter *padapter, u8 *pRA);
+
+void rtw_wapi_set_set_encryption(_adapter *padapter, struct ieee_param *param);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wnm.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wnm.h
new file mode 100644
index 000000000000..6f3d9727d396
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wnm.h
@@ -0,0 +1,225 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __RTW_WNM_H_
+#define __RTW_WNM_H_
+
+#define RTW_RRM_NB_RPT_EN		BIT(1)
+
+#define RTW_WNM_FEATURE_BTM_REQ_EN		BIT(0)
+
+#define rtw_roam_busy_scan(a, nb)	\
+	(((a)->mlmepriv.LinkDetectInfo.bBusyTraffic == _TRUE) && \
+	(((a)->mlmepriv.ch_cnt) < ((nb)->nb_rpt_ch_list_num)))
+
+#define rtw_wnm_btm_preference_cap(b) \
+	(b->preference_en == _TRUE)
+
+#define rtw_wnm_btm_roam_triggered(a) \
+	(((a)->mlmepriv.nb_info.preference_en == _TRUE) \
+	&& (rtw_ft_chk_flags((a), RTW_FT_BTM_ROAM))	\
+	)
+
+#define rtw_wnm_btm_diff_bss(a, b) \
+	((is_zero_mac_addr(b->roam_target_addr) == _FALSE) && \
+	(_rtw_memcmp(b->roam_target_addr,\
+		(a)->mlmepriv.dev_cur_network.network.MacAddress, ETH_ALEN) == _FALSE))
+
+#define rtw_wnm_btm_roam_candidate(a, b, c) \
+	((is_zero_mac_addr((a)->mlmepriv.nb_info.roam_target_addr) == _FALSE) && \
+	(_rtw_memcmp(b->roam_target_addr,\
+		(c)->network.MacAddress, ETH_ALEN)))
+
+#define rtw_wnm_add_btm_ext_cap(d, l)	rtw_add_ext_cap_info(d, l, BSS_TRANSITION)
+
+#define wnm_btm_bss_term_inc(p) (*((u8 *)((p)+3)) & BSS_TERMINATION_INCLUDED)
+
+#define wnm_btm_ess_disassoc_im(p) (*((u8 *)((p)+3)) & ESS_DISASSOC_IMMINENT)
+
+#define wnm_btm_dialog_token(p) (*((u8 *)((p)+2)))
+
+#define wnm_btm_query_reason(p) (*((u8 *)((p)+3)))
+
+#define wnm_btm_req_mode(p) (*((u8 *)((p)+3)))
+
+#define wnm_btm_disassoc_timer(p) (*((u16 *)((p)+4)))
+
+#define wnm_btm_valid_interval(p) (*((u8 *)((p)+6)))
+
+#define wnm_btm_term_duration_offset(p) ((p)+7)
+
+#define wnm_btm_rsp_status(p) (*((u8 *)((p)+3)))
+
+#define wnm_btm_rsp_term_delay(p) (*((u8 *)((p)+4)))
+
+#define RTW_WLAN_ACTION_WNM_NB_RPT_ELEM	0x34
+
+enum rtw_ieee80211_wnm_actioncode {
+	RTW_WLAN_ACTION_WNM_BTM_QUERY = 6,
+	RTW_WLAN_ACTION_WNM_BTM_REQ = 7,
+	RTW_WLAN_ACTION_WNM_BTM_RSP = 8,
+	RTW_WLAN_ACTION_WNM_NOTIF_REQ = 26,
+	RTW_WLAN_ACTION_WNM_NOTIF_RSP = 27,
+};
+
+/*IEEE Std 80211k Figure 7-95b Neighbor Report element format*/
+struct nb_rpt_hdr {
+	u8 id; /*0x34: Neighbor Report Element ID*/
+	u8 len;
+	u8 bssid[ETH_ALEN];
+	u32 bss_info;
+	u8 reg_class;
+	u8 ch_num;
+	u8 phy_type;
+};
+
+/*IEEE Std 80211v, Figure 7-9 BSS Termination Duration subelement field format */
+struct btm_term_duration {
+	u8 id;
+	u8 len;
+	u64 tsf;		/* value of the TSF counter when BSS termination will occur in the future */
+	u16 duration;		/* number of minutes for which the BSS is not present*/
+};
+
+/*IEEE Std 80211v, Figure 7-10 BSS Transition Management Request frame body format */
+struct btm_req_hdr {
+	u8 dialog_token;
+	u8 req_mode;
+	/* number of TBTTs until the AP sends a Disassociation frame to this STA */
+	u16 disassoc_timer;
+	/* number of TBTTs until the BSS transition candidate list is no longer valid */
+	u8 validity_interval;
+	struct btm_term_duration term_duration;
+};
+
+struct btm_rsp_hdr {
+	u8 dialog_token;
+	u8 status;
+	/* the number of minutes that
+		the responding STA requests the BSS to delay termination */
+	u8 termination_delay;
+	u8 bssid[ETH_ALEN];
+	u8 *pcandidates;
+	u32 candidates_num;
+};
+
+struct btm_rpt_cache {
+	u8 dialog_token;
+	u8 req_mode;
+	u16 disassoc_timer;
+	u8 validity_interval;
+	struct btm_term_duration term_duration;
+
+	/* from BTM req */
+	u32 validity_time;
+	u32 disassoc_time;
+
+	systime req_stime;
+};
+
+/*IEEE Std 80211v,  Table 7-43b Optional Subelement IDs for Neighbor Report*/
+/* BSS Transition Candidate Preference */
+#define WNM_BTM_CAND_PREF_SUBEID 0x03
+
+/* BSS Termination Duration */
+#define WNM_BTM_TERM_DUR_SUBEID		0x04
+
+struct wnm_btm_cant {
+	struct nb_rpt_hdr nb_rpt;
+	u8 preference;	/* BSS Transition Candidate Preference */
+};
+
+enum rtw_btm_req_mod {
+	PREFERRED_CANDIDATE_LIST_INCLUDED = BIT0,
+	ABRIDGED = BIT1,
+	DISASSOC_IMMINENT = BIT2,
+	BSS_TERMINATION_INCLUDED = BIT3,
+	ESS_DISASSOC_IMMINENT = BIT4,
+};
+
+struct roam_nb_info {
+	struct nb_rpt_hdr nb_rpt[RTW_MAX_NB_RPT_NUM];
+	struct rtw_ieee80211_channel nb_rpt_ch_list[RTW_MAX_NB_RPT_NUM];
+	struct btm_rpt_cache btm_cache;
+	bool	nb_rpt_valid;
+	u8	nb_rpt_ch_list_num;
+	u8 preference_en;
+	u8 roam_target_addr[ETH_ALEN];
+	u32	last_nb_rpt_entries;
+	u8 nb_rpt_is_same;
+	s8 disassoc_waiting;
+	_timer roam_scan_timer;
+	_timer disassoc_chk_timer;
+
+	u32 features;
+};
+
+void rtw_wnm_candidate_info(struct wlan_network *pnetwork, struct wlan_network *cnetwork, u8 *reason);
+
+u8 rtw_wnm_btm_reassoc_req(_adapter *padapter);
+
+void rtw_wnm_roam_scan_hdl(void *ctx);
+
+void rtw_wnm_disassoc_chk_hdl(void *ctx);
+
+u8 rtw_wnm_try_btm_roam_imnt(_adapter *padapter);
+
+void rtw_wnm_process_btm_query(_adapter *padapter,
+	u8* pframe, u32 frame_len);
+
+void rtw_wnm_process_btm_req(_adapter *padapter,
+	struct sta_info *psta, u8* pframe, u32 frame_len);
+
+void rtw_wnm_process_notification_req(
+	_adapter *padapter, u8* pframe, u32 frame_len);
+
+void rtw_wnm_reset_btm_candidate(struct roam_nb_info *pnb);
+
+void rtw_wnm_reset_btm_state(_adapter *padapter);
+
+u32 rtw_wnm_btm_rsp_candidates_sz_get(
+	_adapter *padapter, u8* pframe, u32 frame_len);
+
+void rtw_wnm_process_btm_rsp(_adapter *padapter,
+	u8* pframe, u32 frame_len, struct btm_rsp_hdr *prsp);
+
+void rtw_wnm_issue_btm_req(_adapter *padapter,
+	u8 *pmac, struct btm_req_hdr *phdr, u8 *purl, u32 url_len,
+	u8 *pcandidates, u8 candidate_cnt);
+
+void rtw_wnm_reset_btm_cache(_adapter *padapter);
+
+void rtw_wnm_issue_action(_adapter *padapter,
+	struct roam_nb_info *pnb, u8 action, u8 reason, u8 dialog);
+
+void rtw_wnm_update_reassoc_req_ie(_adapter *padapter);
+
+void rtw_roam_nb_info_init(_adapter *padapter);
+
+u8 rtw_roam_nb_scan_list_set(_adapter *padapter,
+	struct sitesurvey_parm *pparm);
+
+u32 rtw_wnm_btm_candidates_survey(_adapter *padapter,
+	u8* pframe, u32 elem_len, u8 is_preference);
+
+struct wlan_network *rtw_wnm_btm_candidate_select(
+	_adapter *padapter, struct roam_nb_info *pnb);
+struct wlan_network * rtw_wnm_btm_candidate_check(_adapter *padapter,
+	struct roam_nb_info *pnb, struct wlan_network *pnetwork);
+
+//u8 rtw_wmn_btm_rsp_reason_decision(_adapter *padapter, u8* req_mode);
+u8 rtw_wmn_btm_rsp_reason_decision(_adapter *padapter, struct roam_nb_info *pnb);
+#endif /* __RTW_WNM_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wow.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wow.h
new file mode 100644
index 000000000000..af0950071917
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_wow.h
@@ -0,0 +1,221 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_WOW_H_
+#define __RTW_WOW_H_
+
+#ifdef CONFIG_WOWLAN
+/* WoWLAN Info define */
+/*=========================== Remote Control Info =========================== */
+#define REMOTECTRL_INFO_VALID_CHECK		0xdd
+#define REMOTECTRL_INFO_SYMBOL_CHK_PTK		BIT0
+#define REMOTECTRL_INFO_SYMBOL_CHK_GTK		BIT1
+/*============================================================================*/
+	#ifdef CONFIG_PLATFORM_ANDROID_INTEL_X86
+		/* TCP/ICMP/UDP multicast with specific IP addr */
+		#define DEFAULT_PATTERN_NUM 4
+	#else
+		/* TCP/ICMP */
+		#define DEFAULT_PATTERN_NUM 3
+	#endif
+
+#ifdef CONFIG_GOOGLE_CAST_WAKEUP
+	#define GOOGLE_CAST_PATTERN_NUM 3
+#endif
+#define MAX_WKFM_CAM_NUM	18 /* same as MAX_WOW_CAM_NUM */
+
+#define MAX_WKFM_SIZE	16 /* (16 bytes for WKFM bit mask, 16*8 = 128 bits) */
+#define MAX_WKFM_PATTERN_SIZE	128
+#define MAX_IN_PATTERN_SIZE	512
+
+/*
+ * MAX_WKFM_PATTERN_STR_LEN : the max. length of wow pattern string
+ *	e.g. echo 00:01:02:...:7f > /proc/net/rtl88x2bu/wlan0/wow_pattern_info
+ *	- each byte of pattern is represented as 2-bytes ascii : MAX_WKFM_PATTERN_SIZE * 2
+ *	- the number of common ':' in pattern string : MAX_WKFM_PATTERN_SIZE - 1
+ *	- 1 byte '\n'(0x0a) is generated at the end when we use echo command
+ *	so total max. length is (MAX_WKFM_PATTERN_SIZE * 3)
+ */
+#define MAX_WKFM_PATTERN_STR_LEN (MAX_WKFM_PATTERN_SIZE * 3)
+
+#define WKFMCAM_ADDR_NUM 6
+#define WKFMCAM_SIZE 24 /* each entry need 6*4 bytes */
+
+#define IS_DISCONNECT_WOW_REASON(rsn) \
+	(rsn == RTW_MAC_WOW_RX_DISASSOC || \
+	 rsn == RTW_MAC_WOW_RX_DEAUTH || \
+	 rsn == RTW_MAC_WOW_FW_DECISION_DISCONNECT || \
+	 rsn == RTW_MAC_WOW_NO_WAKE_RX_PAIRWISEKEY || \
+	 rsn == RTW_MAC_WOW_NO_WAKE_RX_GTK || \
+	 rsn == RTW_MAC_WOW_NO_WAKE_RX_DISASSOC || \
+	 rsn == RTW_MAC_WOW_NO_WAKE_RX_DEAUTH || \
+	 rsn == RTW_MAC_WOW_NO_WAKE_RX_EAPREQ_IDENTIFY || \
+	 rsn == RTW_MAC_WOW_NO_WAKE_FW_DECISION_DISCONNECT || \
+	 0)
+
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+#define WOW_DEFAULT_WAKE_PERIOD 300
+#define WOW_DEFAULT_WAKE_DURATION 30
+#endif
+
+struct aoac_report {
+	u8 iv[8];
+	u8 replay_counter_eapol_key[8];
+	u8 group_key[32];
+	u8 key_index;
+	u8 security_type;
+	u8 wow_pattern_idx;
+	u8 version_info;
+	u8 rekey_ok:1;
+	u8 dummy:7;
+	u8 reserved[3];
+	u8 rxptk_iv[8];
+	u8 rxgtk_iv[4][8];
+};
+
+enum pattern_type {
+	RTW_INVALID_PATTERN,
+	RTW_DEFAULT_PATTERN,
+	RTW_CUSTOMIZED_PATTERN,
+};
+
+enum rtw_wow_dev2hst_gpio {
+	DEV2HST_GPIO_OUTPUT = 0,
+	DEV2HST_GPIO_INPUT = 1
+};
+
+enum rtw_wow_dev2hst_active {
+	DEV2HST_LOW_ACTIVE = 0,
+	DEV2HST_HIGH_ACTIVE = 1
+};
+
+enum rtw_wow_dev2hst_toggle_pulse {
+	DEV2HST_TOGGLE = 0,
+	DEV2HST_PULSE = 1
+};
+
+enum rtw_wow_dev2hst_time_unit {
+	DEV2HST_32US = 0,
+	DEV2HST_4MS = 1
+};
+
+typedef struct rtl_priv_pattern {
+	int len;
+	char content[MAX_WKFM_PATTERN_SIZE];
+	char mask[MAX_WKFM_SIZE];
+} rtl_priv_pattern_t;
+
+struct wow_priv {
+	enum rtw_mac_wow_wake_reason wow_wake_reason;
+
+	struct rtw_wow_gpio_info wow_gpio;
+	struct rtw_disc_det_info wow_disc;
+#ifdef CONFIG_PNO_SUPPORT
+	struct rtw_nlo_info wow_nlo;
+#endif
+#ifdef CONFIG_MDNS_OFFLOAD
+	struct rtw_mdns_ofld_info mdns_ofld_info; //ryan
+#endif
+#ifdef CONFIG_WOW_APF
+		struct rtw_apf_info apf_info;
+#endif
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+	struct rtw_periodic_wake_info wow_periodic_wake;
+#endif
+	enum pattern_type wow_ptrn_valid[MAX_WKFM_CAM_NUM];
+};
+
+void rtw_init_wow(_adapter *padapter);
+void rtw_free_wow(_adapter *adapter);
+void rtw_get_sec_iv(_adapter *padapter, u8 *pcur_dot11txpn, u8 *StaAddr);
+bool rtw_wowlan_parser_pattern_cmd(u8 *input, char *pattern,
+				int *pattern_len, char *bit_mask);
+u8 rtw_wow_pattern_set(_adapter *adapter,
+		       struct rtw_wowcam_upd_info * wowcam_info,
+		       enum pattern_type set_type);
+void rtw_wow_pattern_clean(_adapter *adapter, enum pattern_type clean_type);
+#ifdef CONFIG_GOOGLE_CAST_WAKEUP
+void rtw_set_google_cast_mdns_wow_pattern(_adapter *padapter);
+#endif
+void rtw_set_default_pattern(_adapter *adapter);
+void rtw_wow_pattern_sw_dump(_adapter *adapter);
+void rtw_construct_remote_control_info(_adapter *adapter,
+				       struct rtw_remote_wake_ctrl_info *ctrl_info);
+void rtw_core_wow_handle_wake_up_rsn(void *drv_priv, u8 rsn);
+#ifdef CONFIG_GTK_OL
+void rtw_update_gtk_ofld_info(void *drv_priv, struct rtw_aoac_report *aoac_info,
+			      u8 aoac_report_get_ok, u8 phase);
+#endif
+void rtw_wowlan_set_pattern_cast_type(_adapter *adapter, struct rtw_wowcam_upd_info *wowcam_info);
+#ifdef CONFIG_WRC_WOW_MAGIC
+/**
+ * rtw_cfg_wrc_wol_magic - Enable/disable WRC magic packet wake up feature
+ *
+ * This function uses a non-wowlan adapter to set the address cam required by
+ * the WRC (wireless remote control), and receives packets with A1 and A3 as the
+ * MAC address of the wowlan adapter.
+ *
+ * When this feature is disabled, the driver will reallocate the wifi role of
+ * the non-wowlan adapter.
+ */
+u8 rtw_cfg_wrc_wol_magic(_adapter *padapter, u8 enable);
+#endif
+
+#ifdef CONFIG_MDNS_OFFLOAD
+int rtw_wow_add_mdns_resp(_adapter *padapter, u8 index, u8 *resp_content, u16 content_len);
+int rtw_wow_del_mdns_resp(_adapter *padapter, u8 index);
+int rtw_wow_get_mdns_resp_ent(_adapter *padapter, u8 index, struct rtw_mdns_resp_entry **resp_entry);
+int rtw_wow_add_mdns_match_crit(_adapter *padapter, u8 index, u16 match_type, u16 name_offset, u16 name_len);
+int rtw_wow_del_mdns_match_crit(_adapter *padapter, u8 index);
+int rtw_wow_add_mdns_passthru_name(_adapter *padapter, u8 *name, u8 name_len);
+void rtw_wow_clr_mdns_passthru_name(_adapter *padapter);
+void rtw_wow_get_mdns_passthru_list(_adapter *padapter, struct rtw_mdns_passthru_list **passthru_list);
+#endif
+
+#ifdef CONFIG_WOW_APF
+u8 rtw_wow_set_apf(struct _ADAPTER *padapter, u8 *apf_prog, u16 apf_prog_len);
+u8 rtw_wow_get_apf(struct _ADAPTER *padapter, u8 *apf_prog, u16 *apf_prog_len);
+u8 rtw_wow_clear_apf(struct _ADAPTER *padapter);
+#endif
+
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_PNO_SUPPORT
+#ifdef CONFIG_PNO_SECURITY_OFFLOAD
+#define CIPHER_IE "key_mgmt="
+#define CIPHER_NONE "NONE"
+#define CIPHER_WPA_PSK "WPA-PSK"
+#define CIPHER_WPA_EAP "WPA-EAP IEEE8021X"
+enum nlo_cipher_suite {
+	NLO_CIPHER_OPEN      = 0,
+	NLO_CIPHER_WEP       = BIT(0),
+	NLO_CIPHER_WPA_TKIP  = BIT(1),
+	NLO_CIPHER_WPA_AES   = BIT(2),
+	NLO_CIPHER_WPA2_TKIP = BIT(5),
+	NLO_CIPHER_WPA2_AES  = BIT(6),
+};
+#endif
+#define MAX_NLO_SCAN_PLANS 2
+#define MAX_NLO_SCAN_PERIOD 600
+#define MAX_NLO_NORMAL_SCAN_CYCLE 255
+#define NLO_DEFAULT_SCAN_DELAY 3
+int rtw_nlo_enable(struct net_device *net, struct cfg80211_ssid *ssids,
+		   int n_ssids, struct ieee80211_channel **channels,
+		   u32 n_channels, u32 delay, u32 interval, u32 iterations,
+		   u32 slow_interval);
+int rtw_nlo_disable(struct net_device *net);
+void rtw_nlo_debug(struct net_device * net);
+#endif /* CONFIG_PNO_SUPPORT */
+
+#endif /* __RTW_WOW_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_xmit.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_xmit.h
new file mode 100644
index 000000000000..bae440b5c226
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_xmit.h
@@ -0,0 +1,1173 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_XMIT_H_
+#define _RTW_XMIT_H_
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+#ifdef CONFIG_PCI_HCI
+	#define XMITBUF_ALIGN_SZ 4
+#else
+	#ifdef USB_XMITBUF_ALIGN_SZ
+		#define XMITBUF_ALIGN_SZ (USB_XMITBUF_ALIGN_SZ)
+	#else
+		#define XMITBUF_ALIGN_SZ 512
+	#endif
+#endif
+
+#define MAX_CMDBUF_SZ	(5120)	/* (4096) */
+#endif
+
+#define MAX_BEACON_LEN	512
+
+#define MAX_NUMBLKS		(1)
+
+#define XMIT_VO_QUEUE (0)
+#define XMIT_VI_QUEUE (1)
+#define XMIT_BE_QUEUE (2)
+#define XMIT_BK_QUEUE (3)
+
+#define VO_QUEUE_INX		0
+#define VI_QUEUE_INX		1
+#define BE_QUEUE_INX		2
+#define BK_QUEUE_INX		3
+#define BCN_QUEUE_INX		4
+#define MGT_QUEUE_INX		5
+#define HIGH_QUEUE_INX		6
+#define TXCMD_QUEUE_INX	7
+
+#define HW_QUEUE_ENTRY	8
+
+
+#ifdef RTW_PHL_TX
+#ifndef RTW_MAX_FRAG_NUM
+#define RTW_MAX_FRAG_NUM 10 //max scatter number of a packet to xmit
+#endif /*RTW_MAX_FRAG_NUM*/
+#define RTW_MAX_WL_HEAD	100
+#define RTW_MAX_WL_TAIL 100
+#define RTW_SZ_LLC	(SNAP_SIZE + sizeof(u16))
+#define RTW_SZ_FCS	4
+#endif
+
+#define WEP_IV(pattrib_iv, dot11txpn, keyidx)\
+	do {\
+		dot11txpn.val = (dot11txpn.val == 0xffffff) ? 0 : (dot11txpn.val + 1);\
+		pattrib_iv[0] = dot11txpn._byte_.TSC0;\
+		pattrib_iv[1] = dot11txpn._byte_.TSC1;\
+		pattrib_iv[2] = dot11txpn._byte_.TSC2;\
+		pattrib_iv[3] = ((keyidx & 0x3)<<6);\
+	} while (0)
+
+
+#define TKIP_IV(pattrib_iv, dot11txpn, keyidx)\
+	do {\
+		dot11txpn.val = dot11txpn.val == 0xffffffffffffULL ? 0 : (dot11txpn.val + 1);\
+		pattrib_iv[0] = dot11txpn._byte_.TSC1;\
+		pattrib_iv[1] = (dot11txpn._byte_.TSC1 | 0x20) & 0x7f;\
+		pattrib_iv[2] = dot11txpn._byte_.TSC0;\
+		pattrib_iv[3] = BIT(5) | ((keyidx & 0x3)<<6);\
+		pattrib_iv[4] = dot11txpn._byte_.TSC2;\
+		pattrib_iv[5] = dot11txpn._byte_.TSC3;\
+		pattrib_iv[6] = dot11txpn._byte_.TSC4;\
+		pattrib_iv[7] = dot11txpn._byte_.TSC5;\
+	} while (0)
+
+#define AES_IV(pattrib_iv, dot11txpn, keyidx)\
+	do {\
+		dot11txpn.val = dot11txpn.val == 0xffffffffffffULL ? 0 : (dot11txpn.val + 1);\
+		pattrib_iv[0] = dot11txpn._byte_.TSC0;\
+		pattrib_iv[1] = dot11txpn._byte_.TSC1;\
+		pattrib_iv[2] = 0;\
+		pattrib_iv[3] = BIT(5) | ((keyidx & 0x3)<<6);\
+		pattrib_iv[4] = dot11txpn._byte_.TSC2;\
+		pattrib_iv[5] = dot11txpn._byte_.TSC3;\
+		pattrib_iv[6] = dot11txpn._byte_.TSC4;\
+		pattrib_iv[7] = dot11txpn._byte_.TSC5;\
+	} while (0)
+
+#define GCMP_IV(a, b, c) AES_IV(a, b, c)
+
+/* Check if AMPDU Tx is supported or not. If it is supported,
+* it need to check "amsdu in ampdu" is supported or not.
+* (ampdu_en, amsdu_ampdu_en) =
+* (0, x) : AMPDU is not enable, but AMSDU is valid to send.
+* (1, 0) : AMPDU is enable, AMSDU in AMPDU is not enable. So, AMSDU is not valid to send.
+* (1, 1) : AMPDU and AMSDU in AMPDU are enable. So, AMSDU is valid to send.
+*/
+#define IS_AMSDU_AMPDU_NOT_VALID(pattrib)\
+	 ((pattrib->ampdu_en == _TRUE) && (pattrib->amsdu_ampdu_en == _FALSE))
+
+#define IS_AMSDU_AMPDU_VALID(pattrib)\
+	 !((pattrib->ampdu_en == _TRUE) && (pattrib->amsdu_ampdu_en == _FALSE))
+
+#define HWXMIT_ENTRY	4
+
+/* For Buffer Descriptor ring architecture */
+#if defined(BUF_DESC_ARCH) || defined(CONFIG_TRX_BD_ARCH)
+	#define TX_BUFFER_SEG_NUM	1 /* 0:2 seg, 1: 4 seg, 2: 8 seg. */
+#endif
+
+/*GEORGIA_TODO_FIXIT_MOVE_TO_HAL*/
+#if defined(CONFIG_RTL8822B) || defined(CONFIG_RTL8822C)
+	#define TXDESC_SIZE 48		/* HALMAC_TX_DESC_SIZE_8822B */
+#elif defined(CONFIG_RTL8821C)
+	#define TXDESC_SIZE 48		/* HALMAC_TX_DESC_SIZE_8821C */
+#elif defined(CONFIG_RTL8814B)
+	#define TXDESC_SIZE (16 + 32)
+#else
+	#define TXDESC_SIZE 32 /* old IC (ex: 8188E) */
+#endif
+
+#ifdef CONFIG_TX_EARLY_MODE
+	#define EARLY_MODE_INFO_SIZE	8
+#endif
+
+#ifdef CONFIG_USB_HCI
+	#ifdef USB_PACKET_OFFSET_SZ
+		#define PACKET_OFFSET_SZ (USB_PACKET_OFFSET_SZ)
+	#else
+		#define PACKET_OFFSET_SZ (8)
+	#endif
+	#define TXDESC_OFFSET (TXDESC_SIZE + PACKET_OFFSET_SZ)
+#endif
+
+#ifdef CONFIG_PCI_HCI
+	#if defined(CONFIG_RTL8822B) || defined(CONFIG_RTL8821C) || defined(CONFIG_RTL8822C) || defined(CONFIG_TRX_BD_ARCH)
+		/* this section is defined for buffer descriptor ring architecture */
+		#define TX_WIFI_INFO_SIZE (TXDESC_SIZE) /* it may add 802.11 hdr or others... */
+		/* tx desc and payload are in the same buf */
+		#define TXDESC_OFFSET (TX_WIFI_INFO_SIZE)
+	#else
+		/* tx desc and payload are NOT in the same buf */
+		#define TXDESC_OFFSET (0)
+		/* 8188ee/8723be/8812ae/8821ae has extra PCI DMA info in tx desc */
+	#endif
+#endif /* CONFIG_PCI_HCI */
+
+#ifdef RTW_PHL_TX
+#ifdef TXDESC_OFFSET
+#undef TXDESC_OFFSET
+#endif
+#define TXDESC_OFFSET (0)
+#endif
+
+#ifdef RTW_PHL_TX
+enum CORE_TX_TYPE {
+	RTW_TX_OS = 0,
+	RTW_TX_OS_MAC80211,
+	RTW_TX_DRV_MGMT,
+};
+#endif
+
+enum TXDESC_SC {
+	SC_DONT_CARE = 0x00,
+	SC_UPPER = 0x01,
+	SC_LOWER = 0x02,
+	SC_DUPLICATE = 0x03
+};
+
+#ifdef CONFIG_PCI_HCI
+	#ifndef CONFIG_TRX_BD_ARCH	/* CONFIG_TRX_BD_ARCH doesn't need this */
+		#define TXDESC_64_BYTES
+	#endif
+#endif
+
+/*GEORGIA_TODO_FIXIT_IC_DEPENDENCE*/
+#ifdef CONFIG_TRX_BD_ARCH
+struct tx_buf_desc {
+#ifdef CONFIG_64BIT_DMA
+#define TX_BUFFER_SEG_SIZE	4	/* in unit of DWORD */
+#else
+#define TX_BUFFER_SEG_SIZE	2	/* in unit of DWORD */
+#endif
+	unsigned int dword[TX_BUFFER_SEG_SIZE * (2 << TX_BUFFER_SEG_NUM)];
+} __packed;
+#elif (defined(CONFIG_RTL8822B) || defined(CONFIG_RTL8822C)) && defined(CONFIG_PCI_HCI) /* 8192ee or 8814ae */
+/* 8192EE_TODO */
+struct tx_desc {
+	unsigned int txdw0;
+	unsigned int txdw1;
+	unsigned int txdw2;
+	unsigned int txdw3;
+	unsigned int txdw4;
+	unsigned int txdw5;
+	unsigned int txdw6;
+	unsigned int txdw7;
+};
+#else
+struct tx_desc {
+	unsigned int txdw0;
+	unsigned int txdw1;
+	unsigned int txdw2;
+	unsigned int txdw3;
+	unsigned int txdw4;
+	unsigned int txdw5;
+	unsigned int txdw6;
+	unsigned int txdw7;
+
+#if defined(TXDESC_40_BYTES) || defined(TXDESC_64_BYTES)
+	unsigned int txdw8;
+	unsigned int txdw9;
+#endif /* TXDESC_40_BYTES */
+
+#ifdef TXDESC_64_BYTES
+	unsigned int txdw10;
+	unsigned int txdw11;
+
+	/* 2008/05/15 MH Because PCIE HW memory R/W 4K limit. And now,  our descriptor */
+	/* size is 40 bytes. If you use more than 102 descriptor( 103*40>4096), HW will execute */
+	/* memoryR/W CRC error. And then all DMA fetch will fail. We must decrease descriptor */
+	/* number or enlarge descriptor size as 64 bytes. */
+	unsigned int txdw12;
+	unsigned int txdw13;
+	unsigned int txdw14;
+	unsigned int txdw15;
+#endif
+};
+#endif
+
+#ifndef CONFIG_TRX_BD_ARCH
+union txdesc {
+	struct tx_desc txdesc;
+	unsigned int value[TXDESC_SIZE >> 2];
+};
+#endif
+
+#ifdef CONFIG_PCI_HCI
+#define PCI_MAX_TX_QUEUE_COUNT	8	/* == HW_QUEUE_ENTRY */
+
+struct rtw_tx_ring {
+	unsigned char	qid;
+#ifdef CONFIG_TRX_BD_ARCH
+	struct tx_buf_desc	*buf_desc;
+#else
+	struct tx_desc	*desc;
+#endif
+	dma_addr_t	dma;
+	unsigned int	idx;
+	unsigned int	entries;
+	_queue		queue;
+	u32		qlen;
+#ifdef CONFIG_TRX_BD_ARCH
+	u16		hw_rp_cache;
+#endif
+};
+
+#ifdef DBG_TXBD_DESC_DUMP
+
+#define TX_BAK_FRMAE_CNT	10
+#define TX_BAK_DESC_LEN	48	/* byte */
+#define TX_BAK_DATA_LEN		30	/* byte */
+
+struct rtw_tx_desc_backup {
+	int tx_bak_rp;
+	int tx_bak_wp;
+	u8 tx_bak_desc[TX_BAK_DESC_LEN];
+	u8 tx_bak_data_hdr[TX_BAK_DATA_LEN];
+	u8 tx_desc_size;
+};
+#endif
+#endif
+
+struct	hw_xmit	{
+	/* _lock xmit_lock; */
+	/* _list	pending; */
+	_queue *sta_queue;
+	/* struct hw_txqueue *phwtxqueue; */
+	/* sint	txcmdcnt; */
+	int	accnt;
+};
+
+
+#if 1 //def RTW_PHL_TX
+struct pkt_attrib {
+//updated by rtw_core_update_xmitframe
+	u32 sz_payload_per_frag;
+
+	u32 sz_wlan_head;
+	u32 sz_wlan_tail;
+
+	u32 sz_phl_head;
+	u32 sz_phl_tail;
+
+	u8	nr_frags;
+	u32 frag_len;
+	u32 frag_datalen;
+#ifdef CONFIG_CORE_TXSC
+	u32 frag_len_txsc;
+#endif
+
+//updated by
+	u16	ether_type;
+
+	u8	src[ETH_ALEN];
+	u8	dst[ETH_ALEN];
+	u8	ta[ETH_ALEN];
+	u8	ra[ETH_ALEN];
+
+	u16	pkt_hdrlen;	/* the original 802.3 pkt header len */
+	u32 sz_payload;
+
+	u8	dhcp_pkt;
+	u8	icmp_pkt;
+	u8	hipriority_pkt; /* high priority packet */
+
+//WLAN HDR
+	u16	hdrlen;		/* the WLAN Header Len */
+	u8	a4_hdr;
+	u8	type;
+	u8	subtype;
+	u8	qos_en;
+	u16	seqnum;
+	u8	ampdu_en;/* tx ampdu enable */
+	u8	ack_policy;
+	u8	amsdu;
+	u8	mdata;/* more data bit */
+	u8	eosp;
+	u8	priority;
+
+//Security
+	u8	bswenc;
+	/*
+	 * encrypt
+	 * indicate the encrypt algorithm, ref: enum security_type.
+	 * 0: indicate no encrypt.
+	 */
+	u8	encrypt;
+	u8	iv_len;
+	u8	icv_len;
+	u8	iv[18];
+	u8	icv[16];
+	u8	key_idx;
+	union Keytype	dot11tkiptxmickey;
+	/* union Keytype	dot11tkiprxmickey; */
+	union Keytype	dot118021x_UncstKey;
+
+//updated by rtw_core_update_xmitframe
+	u8	hw_ssn_sel;	/* for HW_SEQ0,1,2,3 */
+	u32	pktlen;		/* the original 802.3 pkt raw_data len (not include ether_hdr data) */
+	u32	last_txcmdsz;
+
+#if defined(CONFIG_CONCURRENT_MODE)
+	u8	bmc_camid;
+#endif
+
+
+
+	u8	mac_id;
+	u8	vcs_mode;	/* virtual carrier sense method */
+#ifdef CONFIG_RTW_WDS
+	u8	wds;
+#endif
+#ifdef CONFIG_RTW_MESH
+	u8	mda[ETH_ALEN];	/* mesh da */
+	u8	msa[ETH_ALEN];	/* mesh sa */
+	u8	meshctrl_len;	/* Length of Mesh Control field */
+	u8	mesh_frame_mode;
+	#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	u8 mb2u;
+	#endif
+	u8 mfwd_ttl;
+	u32 mseq;
+#endif
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	u8	hw_csum;
+#endif
+
+
+	u8	ht_en;
+	u8	raid;/* rate adpative id */
+	u8	bwmode;
+	u8	ch_offset;/* PRIME_CHNL_OFFSET */
+	u8	sgi;/* short GI */
+	u8	ampdu_spacing; /* ampdu_min_spacing for peer sta's rx */
+	u8	amsdu_ampdu_en;/* tx amsdu in ampdu enable */
+#ifdef CONFIG_TX_AMSDU
+	u8	tx_amsdu_en;/* tx amsdu enable */
+#endif
+	u8	pctrl;/* per packet txdesc control enable */
+	u8	triggered;/* for ap mode handling Power Saving sta */
+	/*u8	qsel;*/
+	u8	order;/* order bit */
+	u8	rate;
+	u8	intel_proxim;
+	u8	retry_ctrl;
+	u8   mbssid;
+	u8	ldpc;
+	u8	stbc;
+
+	struct sta_info *psta;
+
+	struct _ADAPTER_LINK *adapter_link;
+
+	u8 rtsen;
+	u8 cts2self;
+	u8 hw_rts_en;
+
+#ifdef CONFIG_TDLS
+	u8 direct_link;
+	struct sta_info *ptdls_sta;
+#endif /* CONFIG_TDLS */
+	u8 key_type;
+
+
+
+#ifdef CONFIG_BEAMFORMING
+	u16 txbf_p_aid;/*beamforming Partial_AID*/
+	u16 txbf_g_id;/*beamforming Group ID*/
+
+	/*
+	 * 2'b00: Unicast NDPA
+	 * 2'b01: Broadcast NDPA
+	 * 2'b10: Beamforming Report Poll
+	 * 2'b11: Final Beamforming Report Poll
+	 */
+	u8 bf_pkt_type;
+#endif
+	u8 wdinfo_en;/*FPGA_test*/
+	u8 dma_ch;/*FPGA_test*/
+};
+#endif
+
+#if 0 //ndef RTW_PHL_TX
+/* reduce size */
+struct pkt_attrib {
+	u8	type;
+	u8	subtype;
+	u8	bswenc;
+	u8	dhcp_pkt;
+	u16	ether_type;
+	u16	seqnum;
+	u8	hw_ssn_sel;	/* for HW_SEQ0,1,2,3 */
+	u16	pkt_hdrlen;	/* the original 802.3 pkt header len */
+	u16	hdrlen;		/* the WLAN Header Len */
+	u32	pktlen;		/* the original 802.3 pkt raw_data len (not include ether_hdr data) */
+	u32	last_txcmdsz;
+	u8	nr_frags;
+	u8	encrypt;	/* when 0 indicate no encrypt. when non-zero, indicate the encrypt algorith */
+#if defined(CONFIG_CONCURRENT_MODE)
+	u8	bmc_camid;
+#endif
+	u8	iv_len;
+	u8	icv_len;
+	u8	iv[18];
+	u8	icv[16];
+	u8	priority;
+	u8	ack_policy;
+	u8	mac_id;
+	u8	vcs_mode;	/* virtual carrier sense method */
+	u8	dst[ETH_ALEN];
+	u8	src[ETH_ALEN];
+	u8	ta[ETH_ALEN];
+	u8	ra[ETH_ALEN];
+#ifdef CONFIG_RTW_MESH
+	u8	mda[ETH_ALEN];	/* mesh da */
+	u8	msa[ETH_ALEN];	/* mesh sa */
+	u8	meshctrl_len;	/* Length of Mesh Control field */
+	u8	mesh_frame_mode;
+	#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	u8 mb2u;
+	#endif
+	u8 mfwd_ttl;
+	u32 mseq;
+#endif
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	u8	hw_csum;
+#endif
+	u8	key_idx;
+	u8	qos_en;
+	u8	ht_en;
+	u8	raid;/* rate adpative id */
+	u8	bwmode;
+	u8	ch_offset;/* PRIME_CHNL_OFFSET */
+	u8	sgi;/* short GI */
+	u8	ampdu_en;/* tx ampdu enable */
+	u8	ampdu_spacing; /* ampdu_min_spacing for peer sta's rx */
+	u8	amsdu;
+	u8	amsdu_ampdu_en;/* tx amsdu in ampdu enable */
+	u8	mdata;/* more data bit */
+	u8	pctrl;/* per packet txdesc control enable */
+	u8	triggered;/* for ap mode handling Power Saving sta */
+	u8	qsel;
+	u8	order;/* order bit */
+	u8	eosp;
+	u8	rate;
+	u8	intel_proxim;
+	u8	retry_ctrl;
+	u8   mbssid;
+	u8	ldpc;
+	u8	stbc;
+
+	struct sta_info *psta;
+
+	u8 rtsen;
+	u8 cts2self;
+	union Keytype	dot11tkiptxmickey;
+	/* union Keytype	dot11tkiprxmickey; */
+	union Keytype	dot118021x_UncstKey;
+
+#ifdef CONFIG_TDLS
+	u8 direct_link;
+	struct sta_info *ptdls_sta;
+#endif /* CONFIG_TDLS */
+	u8 key_type;
+
+	u8 icmp_pkt;
+	u8 hipriority_pkt; /* high priority packet */
+
+#ifdef CONFIG_BEAMFORMING
+	u16 txbf_p_aid;/*beamforming Partial_AID*/
+	u16 txbf_g_id;/*beamforming Group ID*/
+
+	/*
+	 * 2'b00: Unicast NDPA
+	 * 2'b01: Broadcast NDPA
+	 * 2'b10: Beamforming Report Poll
+	 * 2'b11: Final Beamforming Report Poll
+	 */
+	u8 bf_pkt_type;
+#endif
+	u8 wdinfo_en;/*FPGA_test*/
+	u8 dma_ch;/*FPGA_test*/
+};
+#endif
+
+#ifdef CONFIG_RTW_WDS
+#define XATTRIB_GET_WDS(xattrib) ((xattrib)->wds)
+#else
+#define XATTRIB_GET_WDS(xattrib) 0
+#endif
+
+#ifdef CONFIG_RTW_MESH
+#define XATTRIB_GET_MCTRL_LEN(xattrib) ((xattrib)->meshctrl_len)
+#else
+#define XATTRIB_GET_MCTRL_LEN(xattrib) 0
+#endif
+
+#ifdef CONFIG_TX_AMSDU
+enum {
+	RTW_AMSDU_TIMER_UNSET = 0,
+	RTW_AMSDU_TIMER_SETTING,
+	RTW_AMSDU_TIMER_TIMEOUT,
+};
+#endif
+
+#define WLANHDR_OFFSET	64
+
+#define NULL_FRAMETAG		(0x0)
+#define DATA_FRAMETAG		0x01
+#define L2_FRAMETAG		0x02
+#define MGNT_FRAMETAG		0x03
+#define AMSDU_FRAMETAG	0x04
+
+#define EII_FRAMETAG		0x05
+#define IEEE8023_FRAMETAG  0x06
+
+#define MP_FRAMETAG		0x07
+
+#define TXAGG_FRAMETAG	0x08
+
+enum {
+	XMITBUF_DATA = 0,
+	XMITBUF_MGNT = 1,
+	XMITBUF_CMD = 2,
+};
+
+bool rtw_xmit_ac_blocked(_adapter *adapter);
+
+struct  submit_ctx {
+	systime submit_time; /* */
+	u32 timeout_ms; /* <0: not synchronous, 0: wait forever, >0: up to ms waiting */
+	int status; /* status for operation */
+	void *rsp; /* rsp buffer allocated by handler */
+	_completion done;
+};
+
+enum {
+	RTW_SCTX_SUBMITTED = -1,
+	RTW_SCTX_DONE_SUCCESS = 0,
+	RTW_SCTX_DONE_UNKNOWN,
+	RTW_SCTX_DONE_TIMEOUT,
+	RTW_SCTX_DONE_BUF_ALLOC,
+	RTW_SCTX_DONE_BUF_FREE,
+	RTW_SCTX_DONE_WRITE_PORT_ERR,
+	RTW_SCTX_DONE_TX_DESC_NA,
+	RTW_SCTX_DONE_TX_DENY,
+	RTW_SCTX_DONE_CCX_PKT_FAIL,
+	RTW_SCTX_DONE_DRV_STOP,
+	RTW_SCTX_DONE_DEV_REMOVE,
+	RTW_SCTX_DONE_CMD_ERROR,
+	RTW_SCTX_DONE_CMD_DROP,
+	RTX_SCTX_CSTR_WAIT_RPT2,
+};
+
+
+void rtw_sctx_init(struct submit_ctx *sctx, int timeout_ms);
+int rtw_sctx_wait(struct submit_ctx *sctx, const char *msg);
+void rtw_sctx_done_err(struct submit_ctx **sctx, int status);
+void rtw_sctx_done(struct submit_ctx **sctx);
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+struct xmit_buf {
+	_list	list;
+
+	_adapter *padapter;
+
+	u8 *pallocated_buf;
+
+	u8 *pbuf;
+
+	void *priv_data;
+
+	u16 buf_tag; /* 0: Normal xmitbuf, 1: extension xmitbuf, 2:cmd xmitbuf */
+	u16 flags;
+	u32 alloc_sz;
+
+	u32  len;
+
+	struct submit_ctx *sctx;
+
+#ifdef CONFIG_USB_HCI
+
+	/* u32 sz[8]; */
+	u32	ff_hwaddr;
+	u8 bulkout_id; /* for halmac */
+
+	PURB	pxmit_urb[8];
+	dma_addr_t dma_transfer_addr;	/* (in) dma addr for transfer_buffer */
+
+	u8 bpending[8];
+
+	sint last[8];
+
+#endif
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+	u8 *phead;
+	u8 *pdata;
+	u8 *ptail;
+	u8 *pend;
+	u32 ff_hwaddr;
+	u8	pg_num;
+	u8	agg_num;
+#endif
+
+#ifdef CONFIG_PCI_HCI
+#ifdef CONFIG_TRX_BD_ARCH
+	/*struct tx_buf_desc *buf_desc;*/
+#else
+	struct tx_desc *desc;
+#endif
+#endif
+
+#if defined(DBG_XMIT_BUF) || defined(DBG_XMIT_BUF_EXT)
+	u8 no;
+#endif
+
+};
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+#define MAX_TXSC_SKB_NUM 6
+#endif
+
+struct xmit_txreq_buf {
+	_list	list;
+	u8 *txreq;
+	u8 *head;
+	u8 *tail;
+	u8 *pkt_list;
+#ifdef CONFIG_CORE_TXSC
+	u8 *pkt[MAX_TXSC_SKB_NUM];
+	u8 pkt_cnt;
+	_adapter *adapter;
+	u8 macid;
+	u8 txsc_id;
+#endif
+};
+
+struct xmit_frame {
+	_list	list;
+
+	struct pkt_attrib attrib;
+
+	u16 os_qid;
+
+	struct sk_buff *pkt;
+
+	int	frame_tag;
+
+	_adapter *padapter;
+
+	/*Only for MGNT Frame*/
+	u8 *prealloc_buf_addr;
+	#ifdef CONFIG_USB_HCI
+	dma_addr_t dma_transfer_addr;
+	#endif
+	u8 *buf_addr;
+	#if 0 /*CONFIG_CORE_XMITBUF*/
+	struct xmit_buf *pxmitbuf;
+	#endif
+
+#ifdef CONFIG_USB_HCI
+#ifdef CONFIG_USB_TX_AGGREGATION
+	u8	agg_num;
+#endif
+	s8	pkt_offset;
+#endif
+
+#ifdef CONFIG_XMIT_ACK
+	u8 ack_report;
+#endif
+
+	u8 *alloc_addr; /* the actual address this xmitframe allocated */
+	u8 ext_tag; /* 0:data, 1:mgmt */
+
+#ifdef RTW_PHL_TX
+	u8 xftype;
+
+	//struct sk_buff		*skb;
+	//struct sta_info 		*psta;
+	//struct pkt_attrib	tx_attrib;
+
+	u8 alloc_hdr;
+	u8 alloc_tail;
+	u8 *wlhdr[RTW_MAX_FRAG_NUM];
+	u8 *wltail[RTW_MAX_FRAG_NUM];
+
+	u32 txring_idx;
+	u32 txreq_cnt;
+	struct rtw_xmit_req 	*phl_txreq;
+	u32 txfree_cnt;
+
+	struct xmit_txreq_buf	*ptxreq_buf;/* TXREQ_QMGT for recycle*/
+
+	u16 buf_need_free; /* size is realted to RTW_MAX_FRAG_NUM */
+#endif
+
+};
+
+struct tx_servq {
+	_list	tx_pending;
+	_queue	sta_pending;
+	int qcnt;
+};
+
+
+struct sta_xmit_priv {
+	_lock	lock;
+	sint	option;
+	sint	apsd_setting;	/* When bit mask is on, the associated edca queue supports APSD. */
+
+
+	/* struct tx_servq blk_q[MAX_NUMBLKS]; */
+	struct tx_servq	be_q;			/* priority == 0,3 */
+	struct tx_servq	bk_q;			/* priority == 1,2 */
+	struct tx_servq	vi_q;			/* priority == 4,5 */
+	struct tx_servq	vo_q;			/* priority == 6,7 */
+	_list	legacy_dz;
+	_list  apsd;
+
+	u16 txseq_tid[16];
+
+	/* uint	sta_tx_bytes; */
+	/* u64	sta_tx_pkts; */
+	/* uint	sta_tx_fail; */
+
+
+};
+
+
+struct	hw_txqueue	{
+	volatile sint	head;
+	volatile sint	tail;
+	volatile sint 	free_sz;	/* in units of 64 bytes */
+	volatile sint      free_cmdsz;
+	volatile sint	 txsz[8];
+	uint	ff_hwaddr;
+	uint	cmd_hwaddr;
+	sint	ac_tag;
+};
+
+struct agg_pkt_info {
+	u16 offset;
+	u16 pkt_len;
+};
+#if 0 /*CONFIG_CORE_XMITBUF*/
+enum cmdbuf_type {
+	CMDBUF_BEACON = 0x00,
+	CMDBUF_RSVD,
+	CMDBUF_MAX
+};
+#endif
+struct	xmit_priv	{
+
+	_lock	lock;
+	#if 0 /*def CONFIG_XMIT_THREAD_MODE*/
+	_sema	xmit_sema;
+	#endif
+
+	/* _queue	blk_strms[MAX_NUMBLKS]; */
+	_queue	be_pending;
+	_queue	bk_pending;
+	_queue	vi_pending;
+	_queue	vo_pending;
+	_queue	bm_pending;
+
+	/* _queue	legacy_dz_queue; */
+	/* _queue	apsd_queue; */
+
+	u8 *pallocated_frame_buf;
+	u8 *pxmit_frame_buf;
+	uint free_xmitframe_cnt;
+	_queue	free_xmit_queue;
+
+	/* uint mapping_addr; */
+	/* uint pkt_sz; */
+
+	u8 *xframe_ext_alloc_addr;
+	u8 *xframe_ext;
+	uint free_xframe_ext_cnt;
+	_queue free_xframe_ext_queue;
+
+	/* MGT_TXREQ_QMGT */
+	u8 *xframe_ext_txreq_alloc_addr;
+	u8 *xframe_ext_txreq;
+
+	/* struct	hw_txqueue	be_txqueue; */
+	/* struct	hw_txqueue	bk_txqueue; */
+	/* struct	hw_txqueue	vi_txqueue; */
+	/* struct	hw_txqueue	vo_txqueue; */
+	/* struct	hw_txqueue	bmc_txqueue; */
+
+	uint	frag_len;
+
+	_adapter	*adapter;
+
+	u8   vcs_setting;
+	u8	vcs;
+	u8	vcs_type;
+	/* u16  rts_thresh; */
+
+	u64	tx_bytes;
+	u64	tx_pkts;
+	u64	tx_drop;
+	u64	last_tx_pkts;
+
+	struct hw_xmit *hwxmits;
+	u8	hwxmit_entry;
+
+	u8	wmm_para_seq[4];/* sequence for wmm ac parameter strength from large to small. it's value is 0->vo, 1->vi, 2->be, 3->bk. */
+
+#ifdef CONFIG_USB_HCI
+	_sema	tx_retevt;/* all tx return event; */
+	u8		txirp_cnt;
+
+	_tasklet xmit_tasklet;
+
+	/* per AC pending irp */
+	int beq_cnt;
+	int bkq_cnt;
+	int viq_cnt;
+	int voq_cnt;
+#endif
+
+#ifdef PRIVATE_R
+        u64 tx_be_drop_cnt;
+        u64 tx_bk_drop_cnt;
+        u64 tx_vi_drop_cnt;
+        u64 tx_vo_drop_cnt;
+#endif
+
+#ifdef CONFIG_PCI_HCI
+	/* Tx */
+	struct rtw_tx_ring	tx_ring[PCI_MAX_TX_QUEUE_COUNT];
+	int	txringcount[PCI_MAX_TX_QUEUE_COUNT];
+	u8 	beaconDMAing;		/* flag of indicating beacon is transmiting to HW by DMA */
+	_tasklet xmit_tasklet;
+#endif
+
+#if defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)
+#ifdef CONFIG_TX_AMSDU_SW_MODE
+	_tasklet xmit_tasklet;
+#endif
+#endif /* CONFIG_SDIO_HCI */
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+	_queue free_xmitbuf_queue;
+	_queue pending_xmitbuf_queue;
+	u8 *pallocated_xmitbuf;
+	u8 *pxmitbuf;
+	uint free_xmitbuf_cnt;
+
+	_queue free_xmit_extbuf_queue;
+	u8 *pallocated_xmit_extbuf;
+	u8 *pxmit_extbuf;
+	uint free_xmit_extbuf_cnt;
+
+	struct xmit_buf	pcmd_xmitbuf[CMDBUF_MAX];
+#endif
+	u8   hw_ssn_seq_no;/* mapping to REG_HW_SEQ 0,1,2,3 */
+	u16	nqos_ssn;
+#ifdef CONFIG_TX_EARLY_MODE
+
+#ifdef CONFIG_SDIO_HCI
+#define MAX_AGG_PKT_NUM 20
+#else
+#define MAX_AGG_PKT_NUM 256 /* Max tx ampdu coounts		 */
+#endif
+
+	struct agg_pkt_info agg_pkt[MAX_AGG_PKT_NUM];
+#endif
+
+#ifdef CONFIG_XMIT_ACK
+	int	ack_tx;
+	_mutex ack_tx_mutex;
+	struct submit_ctx ack_tx_ops;
+	u8 ack_tx_seq_no;
+	#ifdef CONFIG_XMIT_ACK_BY_REL_RPT
+	struct rtw_txfb_t ack_txfb;
+	#endif
+#endif
+
+#ifdef CONFIG_TX_AMSDU
+	_timer amsdu_vo_timer;
+	u8 amsdu_vo_timeout;
+
+	_timer amsdu_vi_timer;
+	u8 amsdu_vi_timeout;
+
+	_timer amsdu_be_timer;
+	u8 amsdu_be_timeout;
+
+	_timer amsdu_bk_timer;
+	u8 amsdu_bk_timeout;
+
+	u32 amsdu_debug_set_timer;
+	u32 amsdu_debug_timeout;
+
+#ifndef AMSDU_DEBUG_MAX_COUNT
+#define AMSDU_DEBUG_MAX_COUNT 5
+#endif
+	u32 amsdu_debug_coalesce[AMSDU_DEBUG_MAX_COUNT];
+	u32 amsdu_debug_tasklet;
+	u32 amsdu_debug_enqueue;
+	u32 amsdu_debug_dequeue;
+#endif
+#ifdef DBG_TXBD_DESC_DUMP
+	BOOLEAN	 dump_txbd_desc;
+#endif
+#ifdef CONFIG_PCI_TX_POLLING
+	_timer tx_poll_timer;
+#endif
+#ifdef CONFIG_LAYER2_ROAMING
+	_queue	rpkt_queue;
+#endif
+	_lock lock_sctx;
+#ifdef CONFIG_CORE_TXSC
+	_lock txsc_lock;
+	u8 txsc_enable;
+	u8 txsc_debug_mode;
+	u8 txsc_debug_mask;/* BIT0:core txsc(no use), BIT1: phl txsc enable, BIT2: debug_print */
+
+	struct sta_info *ptxsc_sta_cached;
+
+	/* for debug */
+	u32 txsc_phl_err_cnt1;
+	u32 txsc_phl_err_cnt2;
+#endif /* CONFIG_CORE_TXSC */
+
+	u16 max_agg_time;
+};
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+extern struct xmit_frame *__rtw_alloc_cmdxmitframe(struct xmit_priv *pxmitpriv,
+		enum cmdbuf_type buf_type);
+#define rtw_alloc_cmdxmitframe(p) __rtw_alloc_cmdxmitframe(p, CMDBUF_RSVD)
+#define rtw_alloc_bcnxmitframe(p) __rtw_alloc_cmdxmitframe(p, CMDBUF_BEACON)
+
+extern struct xmit_buf *rtw_alloc_xmitbuf_ext(struct xmit_priv *pxmitpriv);
+extern s32 rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
+
+extern struct xmit_buf *rtw_alloc_xmitbuf(struct xmit_priv *pxmitpriv);
+extern s32 rtw_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
+#endif
+enum rtw_data_rate _rate_mrate2phl(enum MGN_RATE mrate);
+void rtw_count_tx_stats(_adapter *padapter, struct xmit_frame *pxmitframe, int sz);
+extern void rtw_update_protection(_adapter *padapter, u8 *ie, uint ie_len);
+
+extern s32 rtw_make_wlanhdr(_adapter *padapter, u8 *hdr, struct pkt_attrib *pattrib);
+extern s32 rtw_put_snap(u8 *data, u16 h_proto);
+
+extern struct xmit_frame *rtw_alloc_xmitframe(struct xmit_priv *pxmitpriv, u16 os_qid);
+struct xmit_frame *rtw_alloc_xmitframe_ext(struct xmit_priv *pxmitpriv);
+struct xmit_frame *rtw_alloc_xmitframe_once(struct xmit_priv *pxmitpriv);
+extern s32 rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitframe);
+extern void rtw_free_xmitframe_queue(struct xmit_priv *pxmitpriv, _queue *pframequeue);
+s32 core_tx_free_xmitframe(_adapter *padapter, struct xmit_frame *pxframe);
+struct tx_servq *rtw_get_sta_pending(_adapter *padapter, struct sta_info *psta, sint up, u8 *ac);
+extern s32 rtw_xmitframe_enqueue(_adapter *padapter, struct xmit_frame *pxmitframe);
+extern struct xmit_frame *rtw_dequeue_xframe(struct xmit_priv *pxmitpriv, struct hw_xmit *phwxmit_i, sint entry);
+
+extern s32 rtw_xmit_classifier(_adapter *padapter, struct xmit_frame *pxmitframe);
+extern u32 rtw_calculate_wlan_pkt_size_by_attribue(struct pkt_attrib *pattrib);
+#define rtw_wlan_pkt_size(f) rtw_calculate_wlan_pkt_size_by_attribue(&f->attrib)
+extern s32 rtw_xmitframe_coalesce(_adapter *padapter, struct sk_buff *pkt,
+						struct xmit_frame *pxmitframe);
+#if defined(CONFIG_IEEE80211W) || defined(CONFIG_RTW_MESH)
+extern s32 rtw_mgmt_xmitframe_coalesce(_adapter *padapter,
+			struct sk_buff *pkt, struct xmit_frame *pxmitframe);
+#endif
+#ifdef CONFIG_TDLS
+extern struct tdls_txmgmt *ptxmgmt;
+s32 rtw_xmit_tdls_coalesce(_adapter *padapter, struct xmit_frame *pxmitframe, struct tdls_txmgmt *ptxmgmt);
+s32 update_tdls_attrib(_adapter *padapter, struct _ADAPTER_LINK *padapter_link, struct pkt_attrib *pattrib);
+#endif
+s32 _rtw_init_hw_txqueue(struct hw_txqueue *phw_txqueue, u8 ac_tag);
+void _rtw_init_sta_xmit_priv(struct sta_xmit_priv *psta_xmitpriv);
+
+
+s32 rtw_txframes_pending(_adapter *padapter);
+s32 rtw_txframes_sta_ac_pending(_adapter *padapter, struct pkt_attrib *pattrib);
+void rtw_init_hwxmits(struct hw_xmit *phwxmit, sint entry);
+
+
+s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, _adapter *padapter);
+void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv);
+
+u8 rtw_init_lite_xmit_resource(struct dvobj_priv *dvobj);
+void rtw_free_lite_xmit_resource(struct dvobj_priv *dvobj);
+
+void rtw_alloc_hwxmits(_adapter *padapter);
+void rtw_free_hwxmits(_adapter *padapter);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+s32 rtw_monitor_xmit_entry(struct sk_buff *skb, struct net_device *ndev);
+#endif
+s32 rtw_xmit_posthandle(_adapter *padapter, struct xmit_frame *pxmitframe, struct sk_buff *pkt);
+s32 rtw_xmit(_adapter *padapter, struct sk_buff **pkt, u16 os_qid);
+bool xmitframe_hiq_filter(struct xmit_frame *xmitframe);
+#if defined(CONFIG_AP_MODE) || defined(CONFIG_TDLS)
+sint xmitframe_enqueue_for_sleeping_sta(_adapter *padapter, struct xmit_frame *pxmitframe);
+void stop_sta_xmit(_adapter *padapter, struct sta_info *psta);
+void wakeup_sta_to_xmit(_adapter *padapter, struct sta_info *psta);
+void xmit_delivery_enabled_frames(_adapter *padapter, struct sta_info *psta);
+#endif
+
+#ifdef RTW_PHL_TX
+void dbg_dump_txreq_mdata(struct rtw_t_meta_data *mdata, const char *func);
+s32 core_tx_prepare_phl(_adapter *padapter, struct xmit_frame *pxframe);
+s32 core_tx_call_phl(_adapter *padapter, struct xmit_frame *pxframe, void *txsc_pkt);
+s32 core_tx_per_packet(_adapter *padapter, struct xmit_frame *pxframe,
+		       struct sk_buff **pskb, struct sta_info *psta);
+s32 rtw_core_tx(_adapter *padapter, struct sk_buff **ppkt, struct sta_info *psta, u16 os_qid);
+enum rtw_phl_status rtw_core_tx_recycle(void *drv_priv, struct rtw_xmit_req *txreq);
+s32 core_tx_alloc_xmitframe(_adapter *padapter, struct xmit_frame **pxmitframe, u16 os_qid);
+#ifdef CONFIG_PCI_HCI
+struct tx_local_buf;
+void rtw_os_query_local_buf(void *priv, struct tx_local_buf *buf);
+void rtw_os_return_local_buf(void *priv, struct tx_local_buf *buf);
+#endif
+#ifdef CONFIG_CORE_TXSC
+void core_recycle_txreq_phyaddr(_adapter *padapter, struct rtw_xmit_req *txreq);
+s32 core_tx_free_xmitframe(_adapter *padapter, struct xmit_frame *pxframe);
+u8 *get_txreq_buffer(_adapter *padapter, u8 **txreq, u8 **pkt_list, u8 **head, u8 **tail);
+u8 tos_to_up(u8 tos);
+#endif
+#endif
+
+void core_tx_amsdu_tasklet(unsigned long priv);
+
+u8 rtw_get_tx_bw_mode(_adapter *adapter, struct sta_info *sta);
+
+void rtw_update_tx_rate_bmp(struct dvobj_priv *dvobj);
+u8 rtw_get_tx_bw_bmp_of_ht_rate(struct dvobj_priv *dvobj, u8 rate, u8 max_bw);
+u8 rtw_get_tx_bw_bmp_of_vht_rate(struct dvobj_priv *dvobj, u8 rate, u8 max_bw);
+s16 rtw_rfctl_get_oper_txpwr_max_mbm(struct rf_ctl_t *rfctl, u8 ch, u8 bw, u8 offset, u8 ifbmp_mod, u8 if_op, bool eirp);
+s16 rtw_rfctl_get_reg_max_txpwr_mbm(struct rf_ctl_t *rfctl, enum band_type band, u8 ch, u8 bw, u8 offset, bool eirp);
+
+u8 query_ra_short_GI(struct sta_info *psta, u8 bw);
+
+u8	qos_acm(u8 acm_mask, u8 priority);
+
+#if 0 /*def CONFIG_XMIT_THREAD_MODE*/
+void	enqueue_pending_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
+void enqueue_pending_xmitbuf_to_head(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
+struct xmit_buf	*dequeue_pending_xmitbuf(struct xmit_priv *pxmitpriv);
+struct xmit_buf	*select_and_dequeue_pending_xmitbuf(_adapter *padapter);
+sint	check_pending_xmitbuf(struct xmit_priv *pxmitpriv);
+thread_return	rtw_xmit_thread(thread_context context);
+#endif
+
+#ifdef CONFIG_TX_AMSDU
+extern void rtw_amsdu_vo_timeout_handler(void *FunctionContext);
+extern void rtw_amsdu_vi_timeout_handler(void *FunctionContext);
+extern void rtw_amsdu_be_timeout_handler(void *FunctionContext);
+extern void rtw_amsdu_bk_timeout_handler(void *FunctionContext);
+
+extern u8 rtw_amsdu_get_timer_status(_adapter *padapter, u8 priority);
+extern void rtw_amsdu_set_timer_status(_adapter *padapter, u8 priority, u8 status);
+extern void rtw_amsdu_set_timer(_adapter *padapter, u8 priority);
+extern void rtw_amsdu_cancel_timer(_adapter *padapter, u8 priority);
+
+extern s32 rtw_xmitframe_coalesce_amsdu(_adapter *padapter, struct xmit_frame *pxmitframe, struct xmit_frame *pxmitframe_queue);
+extern s32 check_amsdu(struct xmit_frame *pxmitframe);
+extern s32 check_amsdu_tx_support(_adapter *padapter, struct pkt_attrib *pattrib);
+extern struct xmit_frame *rtw_get_xframe(struct xmit_priv *pxmitpriv, int *num_frame);
+#endif
+
+#ifdef DBG_TXBD_DESC_DUMP
+void rtw_tx_desc_backup(_adapter *padapter, struct xmit_frame *pxmitframe, u8 desc_size, u8 hwq);
+void rtw_tx_desc_backup_reset(void);
+u8 rtw_get_tx_desc_backup(_adapter *padapter, u8 hwq, struct rtw_tx_desc_backup **pbak);
+#endif
+
+#ifdef CONFIG_PCI_TX_POLLING
+void rtw_tx_poll_init(_adapter *padapter);
+void rtw_tx_poll_timeout_handler(void *FunctionContext);
+void rtw_tx_poll_timer_set(_adapter *padapter, u32 delay);
+void rtw_tx_poll_timer_cancel(_adapter *padapter);
+#endif
+
+#ifdef CONFIG_XMIT_ACK
+int rtw_ack_tx_wait(struct xmit_priv *pxmitpriv, u32 timeout_ms);
+void rtw_ack_tx_done(struct xmit_priv *pxmitpriv, int status);
+
+#ifdef CONFIG_XMIT_ACK_BY_REL_RPT
+void rtw_ack_txfb_init(_adapter *padapter, struct rtw_txfb_t *txfb);
+#endif
+
+#endif /* CONFIG_XMIT_ACK */
+
+enum XMIT_BLOCK_REASON {
+	XMIT_BLOCK_NONE = 0,
+	XMIT_BLOCK_REDLMEM = BIT0, /*LPS-PG*/
+	XMIT_BLOCK_SUSPEND = BIT1, /*WOW*/
+	XMIT_BLOCK_MAX = 0xFF,
+};
+void rtw_init_xmit_block(_adapter *padapter);
+void rtw_deinit_xmit_block(_adapter *padapter);
+
+#ifdef DBG_XMIT_BLOCK
+void dump_xmit_block(void *sel, _adapter *padapter);
+#endif
+void rtw_set_xmit_block(_adapter *padapter, enum XMIT_BLOCK_REASON reason);
+void rtw_clr_xmit_block(_adapter *padapter, enum XMIT_BLOCK_REASON reason);
+bool rtw_is_xmit_blocked(_adapter *padapter);
+#ifdef CONFIG_LAYER2_ROAMING
+void dequeuq_roam_pkt(_adapter *padapter);
+#endif
+/* include after declaring struct xmit_buf, in order to avoid warning */
+#include <xmit_osdep.h>
+
+#endif /* _RTL871X_XMIT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/rtw_xmit_shortcut.h b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_xmit_shortcut.h
new file mode 100644
index 000000000000..4b539a7533c6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/rtw_xmit_shortcut.h
@@ -0,0 +1,94 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _RTW_XMIT_SHORTCUT_H_
+#define _RTW_XMIT_SHORTCUT_H_
+
+#ifdef CONFIG_CORE_TXSC
+
+#ifndef USE_ONE_WLHDR
+#define USE_PREV_WLHDR_BUF
+#endif
+
+#define CORE_TXSC_ENTRY_NUM 8
+#define CORE_TXSC_WLHDR_SIZE (WLHDR_SIZE + SNAP_SIZE + 2 + _AES_IV_LEN_)
+#define CORE_TXSC_DEBUG_BUF_SIZE (sizeof(struct rtw_xmit_req) + sizeof(struct rtw_pkt_buf_list)*2)
+
+enum txsc_action_type {
+	TXSC_NONE,
+	TXSC_SKIP,
+	TXSC_ADD,
+	TXSC_APPLY,
+	TXSC_AMSDU_APPLY,
+	TXSC_DEBUG,
+};
+
+enum full_cnt_type {
+	PHL_WD_EMPTY,
+	PHL_BD_FULL,
+	PHL_WD_RECYCLE_NOTHING,
+	PHL_WD_RECYCLE_OK,
+};
+
+struct txsc_pkt_entry {
+	enum txsc_action_type step;
+	struct sta_info *psta;
+	struct rtw_xmit_req *ptxreq;
+
+	u8 txsc_id;
+	u8 priority;
+
+	struct sk_buff *xmit_skb[MAX_TXSC_SKB_NUM];
+	u8 skb_cnt;
+};
+
+struct txsc_entry {
+	u8	txsc_is_used;
+	u8	txsc_ethdr[ETH_HLEN];
+
+	/* wlhdr --- */
+	#ifdef USE_ONE_WLHDR
+	u8	*txsc_wlhdr;
+	#else
+	u8	txsc_wlhdr[CORE_TXSC_WLHDR_SIZE];
+	#endif
+	u8	txsc_wlhdr_len;
+	struct rtw_pkt_buf_list	txsc_pkt_list0;
+	/* wlhdr --- */
+
+	struct rtw_t_meta_data	txsc_mdata;
+	u32	txsc_frag_len;/* for pkt frag check */
+
+	u8	txsc_phl_id; /* CONFIG_PHL_TXSC */
+	u32	txsc_cache_hit;
+};
+
+void _print_txreq_pklist(struct xmit_frame *pxframe, struct rtw_xmit_req *ptxsc_txreq, struct sk_buff *pskb, const char *func);
+void txsc_init(_adapter *padapter);
+void txsc_clear(_adapter *padapter);
+void txsc_dump(_adapter *padapter);
+void txsc_dump_data(u8 *buf, u16 buf_len, const char *prefix);
+u8 txsc_get_sc_cached_entry(_adapter *padapter, struct sk_buff *pskb, struct txsc_pkt_entry *txsc_pkt);
+void txsc_add_sc_cache_entry(_adapter *padapter, struct xmit_frame *pxframe, struct txsc_pkt_entry *txsc_pkt);
+u8 txsc_apply_sc_cached_entry(_adapter *padapter, struct txsc_pkt_entry *txsc_pkt);
+#ifdef CONFIG_PCI_HCI
+void txsc_fill_txreq_phyaddr(_adapter *padapter, struct rtw_pkt_buf_list *pkt_list);
+void txsc_recycle_txreq_phyaddr(_adapter *padapter, struct rtw_xmit_req *txreq);
+#endif
+void txsc_free_txreq(_adapter *padapter, struct rtw_xmit_req *txreq);
+void txsc_debug_sc_entry(_adapter *padapter, struct xmit_frame *pxframe, struct txsc_pkt_entry *txsc_pkt);
+void txsc_issue_addbareq_cmd(_adapter *padapter, u8 priority, struct sta_info *psta, u8 issue_when_busy);
+#endif /* CONFIG_CORE_TXSC */
+#endif /* _RTW_XMIT_SHORTCUT_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/sdio_ops.h b/drivers/net/wireless/realtek/rtl8852bs/include/sdio_ops.h
new file mode 100644
index 000000000000..211928224ab0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/sdio_ops.h
@@ -0,0 +1,44 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __SDIO_OPS_H__
+#define __SDIO_OPS_H__
+
+#include <drv_types.h>		/* struct dvobj_priv, struct sdio_data and etc. */
+
+#ifdef CONFIG_RTW_IO_RECORDS
+void rtw_sdio_records_dump_title(void *sel, bool tab);
+void rtw_sdio_records_dump_value_by_seq(void *sel, bool tab, size_t seq);
+void rtw_sdio_records_summary_dump(void *sel);
+void rtw_sdio_records_claim_and_enable(struct dvobj_priv *d, bool enable);
+#define _io_records_dump_title(sel, tab) rtw_sdio_records_dump_title(sel, tab)
+#define _io_records_dump_value_by_seq(sel, tab, seq) rtw_sdio_records_dump_value_by_seq(sel, tab, seq)
+#define _io_records_summary_dump(sel) rtw_sdio_records_summary_dump(sel)
+#define _io_records_claim_and_enable(d, enable) rtw_sdio_records_claim_and_enable(d, enable)
+#endif /* CONFIG_RTW_IO_RECORDS */
+
+int __must_check rtw_sdio_raw_read(struct dvobj_priv *d, unsigned int addr,
+				   void *buf, size_t len, bool fixed);
+int __must_check rtw_sdio_raw_write(struct dvobj_priv *d, unsigned int addr,
+				    void *buf, size_t len, bool fixed);
+
+void dump_sdio_card_info(void *sel, struct dvobj_priv *dvobj);
+
+u32 rtw_sdio_init(struct dvobj_priv *dvobj);
+void rtw_sdio_deinit(struct dvobj_priv *dvobj);
+int rtw_sdio_alloc_irq(struct dvobj_priv *dvobj);
+void rtw_sdio_free_irq(struct dvobj_priv *dvobj);
+u8 rtw_sdio_get_num_of_func(struct dvobj_priv *dvobj);
+
+#endif /* !__SDIO_OPS_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/sdio_ops_linux.h b/drivers/net/wireless/realtek/rtl8852bs/include/sdio_ops_linux.h
new file mode 100644
index 000000000000..def934699106
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/sdio_ops_linux.h
@@ -0,0 +1,20 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __SDIO_OPS_LINUX_H__
+#define __SDIO_OPS_LINUX_H__
+
+
+#endif /* __SDIO_OPS_LINUX_H__ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/sta_info.h b/drivers/net/wireless/realtek/rtl8852bs/include/sta_info.h
new file mode 100644
index 000000000000..e25e2ee16757
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/sta_info.h
@@ -0,0 +1,872 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __STA_INFO_H_
+#define __STA_INFO_H_
+
+#ifdef CONFIG_CORE_TXSC
+#include <rtw_xmit_shortcut.h>
+#endif
+
+#ifdef CONFIG_RTW_CORE_RXSC
+#include <rtw_recv_shortcut.h>
+#endif
+
+#define NUM_STA STA_NUM_SW_LIMIT
+
+#ifndef CONFIG_RTW_MACADDR_ACL
+	#define CONFIG_RTW_MACADDR_ACL 1
+#endif
+
+#ifndef CONFIG_RTW_PRE_LINK_STA
+	#define CONFIG_RTW_PRE_LINK_STA 0
+#endif
+
+#define NUM_ACL 16
+
+#define RTW_ACL_PERIOD_DEV 0
+#define RTW_ACL_PERIOD_BSS 1
+#define RTW_ACL_PERIOD_NUM 2
+
+#define RTW_ACL_MODE_DISABLED				0
+#define RTW_ACL_MODE_ACCEPT_UNLESS_LISTED	1
+#define RTW_ACL_MODE_DENY_UNLESS_LISTED		2
+#define RTW_ACL_MODE_MAX					3
+
+#if CONFIG_RTW_MACADDR_ACL
+extern const char *const _acl_period_str[RTW_ACL_PERIOD_NUM];
+#define acl_period_str(mode) (((mode) >= RTW_ACL_PERIOD_NUM) ? "INVALID" : _acl_period_str[(mode)])
+extern const char *const _acl_mode_str[RTW_ACL_MODE_MAX];
+#define acl_mode_str(mode) (((mode) >= RTW_ACL_MODE_MAX) ? "INVALID" : _acl_mode_str[(mode)])
+#endif
+
+#ifndef RTW_PRE_LINK_STA_NUM
+	#define RTW_PRE_LINK_STA_NUM 8
+#endif
+
+struct pre_link_sta_node_t {
+	u8 valid;
+	u8 addr[ETH_ALEN];
+};
+
+struct pre_link_sta_ctl_t {
+	_lock lock;
+	u8 num;
+	struct pre_link_sta_node_t node[RTW_PRE_LINK_STA_NUM];
+};
+
+#ifdef CONFIG_TDLS
+#define MAX_ALLOWED_TDLS_STA_NUM	4
+#endif
+
+enum sta_info_update_type {
+	STA_INFO_UPDATE_NONE = 0,
+	STA_INFO_UPDATE_BW = BIT(0),
+	STA_INFO_UPDATE_RATE = BIT(1),
+	STA_INFO_UPDATE_PROTECTION_MODE = BIT(2),
+	STA_INFO_UPDATE_CAP = BIT(3),
+	STA_INFO_UPDATE_HT_CAP = BIT(4),
+	STA_INFO_UPDATE_VHT_CAP = BIT(5),
+	STA_INFO_UPDATE_ALL = STA_INFO_UPDATE_BW
+			      | STA_INFO_UPDATE_RATE
+			      | STA_INFO_UPDATE_PROTECTION_MODE
+			      | STA_INFO_UPDATE_CAP
+			      | STA_INFO_UPDATE_HT_CAP
+			      | STA_INFO_UPDATE_VHT_CAP,
+	STA_INFO_UPDATE_MAX
+};
+
+struct rtw_wlan_acl_node {
+	_list		        list;
+	u8       addr[ETH_ALEN];
+	u8       valid;
+};
+
+struct wlan_acl_pool {
+	int mode;
+	int num;
+	struct rtw_wlan_acl_node aclnode[NUM_ACL];
+	_queue	acl_node_q;
+};
+
+struct	stainfo_stats	{
+	systime last_rx_time;
+
+	u64 rx_mgnt_pkts;
+	u64 rx_beacon_pkts;
+	u64 rx_probereq_pkts;
+	u64 rx_probersp_pkts; /* unicast to self */
+	u64 rx_probersp_bm_pkts;
+	u64 rx_probersp_uo_pkts; /* unicast to others */
+	u64 rx_ctrl_pkts;
+	u64 rx_data_pkts;
+	u64 rx_data_bc_pkts;
+	u64 rx_data_mc_pkts;
+	u64 rx_data_qos_pkts[TID_NUM]; /* unicast only */
+
+	u64	last_rx_mgnt_pkts;
+	u64 last_rx_beacon_pkts;
+	u64 last_rx_probereq_pkts;
+	u64 last_rx_probersp_pkts; /* unicast to self */
+	u64 last_rx_probersp_bm_pkts;
+	u64 last_rx_probersp_uo_pkts; /* unicast to others */
+	u64	last_rx_ctrl_pkts;
+	u64	last_rx_data_pkts;
+	u64 last_rx_data_bc_pkts;
+	u64 last_rx_data_mc_pkts;
+	u64 last_rx_data_qos_pkts[TID_NUM]; /* unicast only */
+
+#ifdef CONFIG_TDLS
+	u64 rx_tdls_disc_rsp_pkts;
+	u64 last_rx_tdls_disc_rsp_pkts;
+#endif
+
+	u64	rx_bytes;
+	u64	rx_bc_bytes;
+	u64	rx_mc_bytes;
+	u64	last_rx_bytes;
+	u64 last_rx_bc_bytes;
+	u64 last_rx_mc_bytes;
+	u64	rx_drops; /* TBD */
+	u32 rx_tp_kbits;
+	u32 smooth_rx_tp_kbits;
+
+	u64	tx_pkts;
+	u64	last_tx_pkts;
+
+	u64	tx_bytes;
+	u64	last_tx_bytes;
+	u64 tx_drops; /* TBD */
+	u32 tx_tp_kbits;
+	u32 smooth_tx_tp_kbits;
+
+	/* unicast only */
+	u64 last_rx_data_uc_pkts; /* For Read & Clear requirement in proc_get_rx_stat() */
+	u32 duplicate_cnt;	/* Read & Clear, in proc_get_rx_stat() */
+	u32 rxratecnt[128];	/* Read & Clear, in proc_get_rx_stat() */
+	u32 tx_ok_cnt;		/* Read & Clear, in proc_get_tx_stat() */
+	u32 tx_fail_cnt;	/* Read & Clear, in proc_get_tx_stat() */
+	u32 tx_fail_cnt_sum;	/* cumulative counts */
+	u32 tx_retry_cnt;	/* Read & Clear, in proc_get_tx_stat() */
+	u32 tx_retry_cnt_sum;	/* cumulative counts */
+	u64 total_tx_retry_cnt;
+	u32 rx_retry_cnt;
+#ifdef CONFIG_RTW_MESH
+	u32 rx_hwmp_pkts;
+	u32 last_rx_hwmp_pkts;
+#endif
+};
+
+#ifndef DBG_SESSION_TRACKER
+#define DBG_SESSION_TRACKER 0
+#endif
+
+/* session tracker status */
+#define ST_STATUS_NONE		0
+#define ST_STATUS_CHECK		BIT0
+#define ST_STATUS_ESTABLISH	BIT1
+#define ST_STATUS_EXPIRE	BIT2
+
+#define ST_EXPIRE_MS (10 * 1000)
+
+struct session_tracker {
+	_list list; /* session_tracker_queue */
+	u32 local_naddr;
+	u16 local_port;
+	u32 remote_naddr;
+	u16 remote_port;
+	systime set_time;
+	u8 status;
+};
+
+/* session tracker cmd */
+#define ST_CMD_ADD 0
+#define ST_CMD_DEL 1
+#define ST_CMD_CHK 2
+
+struct st_cmd_parm {
+	u8 cmd;
+	struct sta_info *sta;
+	u32 local_naddr; /* TODO: IPV6 */
+	u16 local_port;
+	u32 remote_naddr; /* TODO: IPV6 */
+	u16 remote_port;
+};
+
+typedef bool (*st_match_rule)(_adapter *adapter, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port);
+
+struct st_register {
+	u8 s_proto;
+	st_match_rule rule;
+};
+
+#define SESSION_TRACKER_REG_ID_WFD 0
+#define SESSION_TRACKER_REG_ID_NUM 1
+
+struct st_ctl_t {
+	struct st_register reg[SESSION_TRACKER_REG_ID_NUM];
+	_queue tracker_q;
+};
+
+void rtw_st_ctl_init(struct st_ctl_t *st_ctl);
+void rtw_st_ctl_deinit(struct st_ctl_t *st_ctl);
+void rtw_st_ctl_register(struct st_ctl_t *st_ctl, u8 st_reg_id, struct st_register *reg);
+void rtw_st_ctl_unregister(struct st_ctl_t *st_ctl, u8 st_reg_id);
+bool rtw_st_ctl_chk_reg_s_proto(struct st_ctl_t *st_ctl, u8 s_proto);
+bool rtw_st_ctl_chk_reg_rule(struct st_ctl_t *st_ctl, _adapter *adapter, u8 *local_naddr, u8 *local_port, u8 *remote_naddr, u8 *remote_port);
+void rtw_st_ctl_rx(struct sta_info *sta, u8 *ehdr_pos);
+void dump_st_ctl(void *sel, struct st_ctl_t *st_ctl);
+
+#ifdef CONFIG_TDLS
+struct TDLS_PeerKey {
+	u8 kck[16]; /* TPK-KCK */
+	u8 tk[16]; /* TPK-TK; only CCMP will be used */
+} ;
+#endif /* CONFIG_TDLS */
+
+#ifdef DBG_RX_DFRAME_RAW_DATA
+struct sta_recv_dframe_info {
+
+	u8 sta_data_rate;
+	u8 sta_sgi;
+	u8 sta_bw_mode;
+	s8 sta_mimo_signal_strength[4];
+	s8 sta_RxPwr[4];
+	u8 sta_ofdm_snr[4];
+};
+#endif
+
+#ifdef CONFIG_RTW_MESH
+struct mesh_plink_ent;
+struct rtw_ewma_err_rate {
+	unsigned long internal;
+};
+
+/* Mesh airtime link metrics parameters */
+struct rtw_atlm_param {
+	struct rtw_ewma_err_rate err_rate; /* Now is PACKET error rate */
+	u16 data_rate; /* The unit is 100Kbps */
+	u16 total_pkt;
+	u16 overhead; /* Channel access overhead */
+};
+#endif
+
+struct rtw_add_del_sta_obj {
+	_list list;
+	struct sta_info *sta;
+	u8 is_add_sta;
+	u16 aid;
+};
+
+struct sta_info {
+
+	_lock	lock;
+	_list	list; /* free_sta_queue */
+	_list	hash_list; /* sta_hash */
+	bool is_freeing;
+	/* _list asoc_list; */ /* 20061114 */
+	/* _list sleep_list; */ /* sleep_q */
+	/* _list wakeup_list; */ /* wakeup_q */
+	_adapter *padapter;
+	struct _ADAPTER_LINK *padapter_link;
+
+	struct rtw_phl_stainfo_t *phl_sta;
+
+	/* move to phl station info */
+	/* struct cmn_sta_info cmn; */
+
+	struct sta_xmit_priv sta_xmitpriv;
+	struct sta_recv_priv sta_recvpriv;
+
+#ifdef DBG_RX_DFRAME_RAW_DATA
+	struct sta_recv_dframe_info  sta_dframe_info;
+	struct sta_recv_dframe_info  sta_dframe_info_bmc;
+#endif
+	_queue sleep_q;
+	unsigned int sleepq_len;
+
+	uint state;
+	uint qos_option;
+	u16 hwseq;
+
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	u8 rm_en_cap[5];
+	u8 rm_diag_token;
+#endif /* CONFIG_RTW_80211K */
+
+	systime	resp_nonenc_eapol_key_starttime;
+	uint	ieee8021x_blocked;	/* 0: allowed, 1:blocked */
+	uint	dot118021XPrivacy; /* aes, tkip... */
+	union Keytype	dot11tkiptxmickey;
+	union Keytype	dot11tkiprxmickey;
+	union Keytype	dot118021x_UncstKey;
+	union pn48		dot11txpn;			/* PN48 used for Unicast xmit */
+	union pn48		dot11rxpn;			/* PN48 used for Unicast recv. */
+	s8		hw_decrypted; /* STA HW security is ready or not */
+
+#ifdef RTW_PHL_TX
+	u8	iv_len;
+	u8	icv_len;
+	u8	iv[18];
+	u8	icv[16];
+#endif
+
+#ifdef CONFIG_RTW_CORE_RXSC
+	u32 rxsc_idx_new;
+	u32 rxsc_idx_cached;
+	struct core_rxsc_entry rxsc_entry[NUM_RXSC_ENTRY];
+#endif
+	ATOMIC_T	keytrack;
+#ifdef CONFIG_RTW_MESH
+	/* peer's GTK, RX only */
+	u8 group_privacy;
+	u8 gtk_bmp;
+	union Keytype gtk;
+	union pn48 gtk_pn;
+	#ifdef CONFIG_IEEE80211W
+	/* peer's IGTK, RX only */
+	enum security_type dot11wCipher;
+	u8 igtk_bmp;
+	u8 igtk_id;
+	union Keytype igtk;
+	union pn48 igtk_pn;
+	#endif /* CONFIG_IEEE80211W */
+#endif /* CONFIG_RTW_MESH */
+#ifdef CONFIG_GTK_OL
+	u8 kek[RTW_KEK_LEN];
+	u8 kck[RTW_KCK_LEN];
+	u8 replay_ctr[RTW_REPLAY_CTR_LEN];
+#endif /* CONFIG_GTK_OL */
+#ifdef CONFIG_IEEE80211W
+	_timer dot11w_expire_timer;
+#endif /* CONFIG_IEEE80211W */
+
+	u8	bssrateset[16];
+	u32	bssratelen;
+
+	u8	cts2self;
+	u8	rtsen;
+	u8	hw_rts_en;
+
+	u8	init_rate;
+
+	struct stainfo_stats sta_stats;
+
+#ifdef CONFIG_TDLS
+	u32	tdls_sta_state;
+	u8	SNonce[32];
+	u8	ANonce[32];
+	u32	TDLS_PeerKey_Lifetime;
+	u32	TPK_count;
+	_timer	TPK_timer;
+	struct TDLS_PeerKey	tpk;
+#ifdef CONFIG_TDLS_CH_SW
+	u16	ch_switch_time;
+	u16	ch_switch_timeout;
+	/* u8	option; */
+	_timer	ch_sw_timer;
+	_timer	delay_timer;
+	_timer	stay_on_base_chnl_timer;
+	_timer	ch_sw_monitor_timer;
+#endif
+	_timer handshake_timer;
+	u8 alive_count;
+	_timer	pti_timer;
+	u8	TDLS_RSNIE[20];	/* Save peer's RSNIE, used for sending TDLS_SETUP_RSP */
+#endif /* CONFIG_TDLS */
+
+	/* for A-MPDU TX, ADDBA timeout check	 */
+	_timer addba_retry_timer;
+
+	/* for A-MPDU Rx reordering buffer control */
+	struct recv_reorder_ctrl recvreorder_ctrl[TID_NUM];
+	ATOMIC_T continual_no_rx_packet[TID_NUM];
+	/* for A-MPDU Tx */
+	/* unsigned char		ampdu_txen_bitmap; */
+	u16	BA_starting_seqctrl[16];
+
+
+#ifdef CONFIG_80211N_HT
+	struct ht_priv	htpriv;
+	struct ampdu_priv ampdu_priv;
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	struct vht_priv	vhtpriv;
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	struct he_priv	hepriv;
+#endif
+	u8 smps_mode; /*spatial multiplexing power save mode. 0:static SMPS, 1:dynamic SMPS, 3:SMPS disabled, 2:reserved*/
+
+#ifdef CONFIG_RTW_MBO
+	struct mbo_priv mbopriv;
+#endif /* CONFIG_RTW_MBO */
+
+	/* Notes:	 */
+	/* STA_Mode: */
+	/* curr_network(mlme_priv/security_priv/qos/ht) + sta_info: (STA & AP) CAP/INFO	 */
+	/* scan_q: AP CAP/INFO */
+
+	/* AP_Mode: */
+	/* curr_network(mlme_priv/security_priv/qos/ht) : AP CAP/INFO */
+	/* sta_info: (AP & STA) CAP/INFO */
+
+	unsigned int expire_to;
+
+#ifdef CONFIG_AP_MODE
+
+	_list asoc_list;
+	_list auth_list;
+
+	unsigned int auth_seq;
+	unsigned int authalg;
+	unsigned char chg_txt[128];
+
+	u16 capability;
+	int flags;
+
+	int dot8021xalg;/* 0:disable, 1:psk, 2:802.1x */
+	int wpa_psk;/* 0:disable, bit(0): WPA, bit(1):WPA2 */
+	int wpa_group_cipher;
+	int wpa2_group_cipher;
+	int wpa_pairwise_cipher;
+	int wpa2_pairwise_cipher;
+
+	u32 akm_suite_type;
+
+	u8 bpairwise_key_installed;
+#ifdef CONFIG_RTW_80211R
+	struct rtw_sta_ft_info_t ft_peer;
+	u8 ft_pairwise_key_installed;
+#endif
+#ifdef CONFIG_RTW_FSM_BTM
+	struct btm_obj *btm;
+#endif
+
+#ifdef CONFIG_NATIVEAP_MLME
+	u8 wpa_ie[32];
+
+	u8 nonerp_set;
+	u8 no_short_slot_time_set;
+	u8 no_short_preamble_set;
+	u8 no_ht_gf_set;
+	u8 no_ht_set;
+	u8 ht_20mhz_set;
+	u8 ht_40mhz_intolerant;
+#endif /* CONFIG_NATIVEAP_MLME */
+
+#ifdef CONFIG_ATMEL_RC_PATCH
+	u8 flag_atmel_rc;
+#endif
+
+	u8 qos_info;
+
+	u8 max_sp_len;
+	u8 uapsd_bk;/* BIT(0): Delivery enabled, BIT(1): Trigger enabled */
+	u8 uapsd_be;
+	u8 uapsd_vi;
+	u8 uapsd_vo;
+
+	u8 has_legacy_ac;
+	unsigned int sleepq_ac_len;
+
+#ifdef CONFIG_P2P
+	/* p2p priv data */
+	u8 is_p2p_device;
+	u8 p2p_status_code;
+
+	/* p2p client info */
+	u8 dev_addr[ETH_ALEN];
+	/* u8 iface_addr[ETH_ALEN]; */ /* = hwaddr[ETH_ALEN] */
+	u8 dev_cap;
+	u16 config_methods;
+	u8 primary_dev_type[8];
+	u8 num_of_secdev_type;
+	u8 secdev_types_list[32];/* 32/8 == 4; */
+	u16 dev_name_len;
+	u8 dev_name[32];
+#endif /* CONFIG_P2P */
+
+#ifdef CONFIG_WFD
+	u8 op_wfd_mode;
+#endif
+
+#if !defined(CONFIG_ACTIVE_KEEP_ALIVE_CHECK) && defined(CONFIG_80211N_HT)
+	u8 under_exist_checking;
+#endif
+
+	u8 keep_alive_trycnt;
+
+#ifdef CONFIG_AUTO_AP_MODE
+	u8 isrc; /* this device is rc */
+	u16 pid; /* pairing id */
+#endif
+
+	struct rtw_add_del_sta_obj *add_del_sta_obj;
+
+#endif /* CONFIG_AP_MODE	 */
+
+#ifdef CONFIG_RTW_MESH
+	struct mesh_plink_ent *plink;
+
+	u8 local_mps;
+	u8 peer_mps;
+	u8 nonpeer_mps;
+
+	struct rtw_atlm_param metrics;
+	/* The reference for nexthop_lookup */
+	BOOLEAN alive;
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	u8 *pauth_frame;
+	u32 auth_len;
+	u8 *passoc_req;
+	u32 assoc_req_len;
+#endif
+
+	u8		IOTPeer;			/* Enum value.	HT_IOT_PEER_E */
+
+	/* To store the sequence number of received management frame */
+	u16 RxMgmtFrameSeqNum;
+
+	struct st_ctl_t st_ctl;
+	u8 max_agg_num_minimal_record; /*keep minimal tx desc max_agg_num setting*/
+	u8 curr_rx_gi_ltf;
+	u16 curr_rx_rate;
+	u16 curr_rx_rate_bmc;
+#ifdef CONFIG_RTS_FULL_BW
+	bool vendor_8812;
+#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	u8 tbtx_enable;			/* Does this sta_info support & enable TBTX function? */
+//	u8 tbtx_timeslot;		/* This sta_info belong to which time slot.	*/
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+	u32 txsc_cache_hit;
+	u32 txsc_cache_miss;
+	u32 txsc_path_slow;
+	u32 txsc_path_ps;
+	u8 txsc_cur_idx; /* next entry to add */
+	u8 txsc_cache_idx; /* latest cache idx */
+	u8 txsc_cache_num; /* num of txsc entry */
+	struct txsc_entry txsc_entry_cache[CORE_TXSC_ENTRY_NUM];
+	u8 debug_buf[CORE_TXSC_DEBUG_BUF_SIZE];
+#endif /* CONFIG_CORE_TXSC */
+	u32 snr_fd_total[4];
+	u32 snr_td_total[4];
+	u8 snr_fd_avg[4];
+	u8 snr_td_avg[4];
+	u32 snr_num;
+
+	u8 start_active;/*Used to note the first time receive data frame*/
+	u32 start_active_time;
+	u32 latest_active_time;
+
+	ATOMIC_T deleting;
+};
+
+#define STA_MACADDR(sta) (sta)->phl_sta->mac_addr
+
+#ifdef CONFIG_RTW_MESH
+#define STA_SET_MESH_PLINK(sta, link) (sta)->plink = link
+#else
+#define STA_SET_MESH_PLINK(sta, link) do {} while (0)
+#endif
+
+#define sta_tx_pkts(sta) \
+	(sta->sta_stats.tx_pkts)
+
+#define sta_last_tx_pkts(sta) \
+	(sta->sta_stats.last_tx_pkts)
+
+#define sta_rx_pkts(sta) \
+	(sta->sta_stats.rx_mgnt_pkts \
+	 + sta->sta_stats.rx_ctrl_pkts \
+	 + sta->sta_stats.rx_data_pkts)
+
+#define sta_last_rx_pkts(sta) \
+	(sta->sta_stats.last_rx_mgnt_pkts \
+	 + sta->sta_stats.last_rx_ctrl_pkts \
+	 + sta->sta_stats.last_rx_data_pkts)
+
+#define sta_rx_data_pkts(sta) (sta->sta_stats.rx_data_pkts)
+#define sta_last_rx_data_pkts(sta) (sta->sta_stats.last_rx_data_pkts)
+
+#define sta_rx_data_uc_pkts(sta) (sta->sta_stats.rx_data_pkts - sta->sta_stats.rx_data_bc_pkts - sta->sta_stats.rx_data_mc_pkts)
+#define sta_last_rx_data_uc_pkts(sta) (sta->sta_stats.last_rx_data_pkts - sta->sta_stats.last_rx_data_bc_pkts - sta->sta_stats.last_rx_data_mc_pkts)
+
+#define sta_rx_data_qos_pkts(sta, i) \
+	(sta->sta_stats.rx_data_qos_pkts[i])
+
+#define sta_last_rx_data_qos_pkts(sta, i) \
+	(sta->sta_stats.last_rx_data_qos_pkts[i])
+
+#define sta_rx_mgnt_pkts(sta) \
+	(sta->sta_stats.rx_mgnt_pkts)
+
+#define sta_last_rx_mgnt_pkts(sta) \
+	(sta->sta_stats.last_rx_mgnt_pkts)
+
+#define sta_rx_beacon_pkts(sta) \
+	(sta->sta_stats.rx_beacon_pkts)
+
+#define sta_last_rx_beacon_pkts(sta) \
+	(sta->sta_stats.last_rx_beacon_pkts)
+
+#define sta_rx_probereq_pkts(sta) \
+	(sta->sta_stats.rx_probereq_pkts)
+
+#define sta_last_rx_probereq_pkts(sta) \
+	(sta->sta_stats.last_rx_probereq_pkts)
+
+#define sta_rx_probersp_pkts(sta) \
+	(sta->sta_stats.rx_probersp_pkts)
+
+#define sta_last_rx_probersp_pkts(sta) \
+	(sta->sta_stats.last_rx_probersp_pkts)
+
+#define sta_rx_probersp_bm_pkts(sta) \
+	(sta->sta_stats.rx_probersp_bm_pkts)
+
+#define sta_last_rx_probersp_bm_pkts(sta) \
+	(sta->sta_stats.last_rx_probersp_bm_pkts)
+
+#define sta_rx_probersp_uo_pkts(sta) \
+	(sta->sta_stats.rx_probersp_uo_pkts)
+
+#define sta_last_rx_probersp_uo_pkts(sta) \
+	(sta->sta_stats.last_rx_probersp_uo_pkts)
+
+#ifdef CONFIG_RTW_MESH
+#define update_last_rx_hwmp_pkts(sta) \
+	do { \
+		sta->sta_stats.last_rx_hwmp_pkts = sta->sta_stats.rx_hwmp_pkts; \
+	} while(0)
+#else
+#define update_last_rx_hwmp_pkts(sta) do {} while(0)
+#endif
+
+#define sta_update_last_rx_pkts(sta) \
+	do { \
+		int __i; \
+		\
+		sta->sta_stats.last_rx_mgnt_pkts = sta->sta_stats.rx_mgnt_pkts; \
+		sta->sta_stats.last_rx_beacon_pkts = sta->sta_stats.rx_beacon_pkts; \
+		sta->sta_stats.last_rx_probereq_pkts = sta->sta_stats.rx_probereq_pkts; \
+		sta->sta_stats.last_rx_probersp_pkts = sta->sta_stats.rx_probersp_pkts; \
+		sta->sta_stats.last_rx_probersp_bm_pkts = sta->sta_stats.rx_probersp_bm_pkts; \
+		sta->sta_stats.last_rx_probersp_uo_pkts = sta->sta_stats.rx_probersp_uo_pkts; \
+		sta->sta_stats.last_rx_ctrl_pkts = sta->sta_stats.rx_ctrl_pkts; \
+		update_last_rx_hwmp_pkts(sta); \
+		\
+		sta->sta_stats.last_rx_data_pkts = sta->sta_stats.rx_data_pkts; \
+		sta->sta_stats.last_rx_data_bc_pkts = sta->sta_stats.rx_data_bc_pkts; \
+		sta->sta_stats.last_rx_data_mc_pkts = sta->sta_stats.rx_data_mc_pkts; \
+		for (__i = 0; __i < TID_NUM; __i++) \
+			sta->sta_stats.last_rx_data_qos_pkts[__i] = sta->sta_stats.rx_data_qos_pkts[__i]; \
+	} while (0)
+
+#define STA_RX_PKTS_ARG(sta) \
+	sta->sta_stats.rx_mgnt_pkts \
+	, sta->sta_stats.rx_ctrl_pkts \
+	, sta->sta_stats.rx_data_pkts
+
+#define STA_LAST_RX_PKTS_ARG(sta) \
+	sta->sta_stats.last_rx_mgnt_pkts \
+	, sta->sta_stats.last_rx_ctrl_pkts \
+	, sta->sta_stats.last_rx_data_pkts
+
+#define STA_RX_PKTS_DIFF_ARG(sta) \
+	sta->sta_stats.rx_mgnt_pkts - sta->sta_stats.last_rx_mgnt_pkts \
+	, sta->sta_stats.rx_ctrl_pkts - sta->sta_stats.last_rx_ctrl_pkts \
+	, sta->sta_stats.rx_data_pkts - sta->sta_stats.last_rx_data_pkts
+
+#define STA_PKTS_FMT "(m:%llu, c:%llu, d:%llu)"
+
+#define sta_rx_uc_bytes(sta) (sta->sta_stats.rx_bytes - sta->sta_stats.rx_bc_bytes - sta->sta_stats.rx_mc_bytes)
+#define sta_last_rx_uc_bytes(sta) (sta->sta_stats.last_rx_bytes - sta->sta_stats.last_rx_bc_bytes - sta->sta_stats.last_rx_mc_bytes)
+
+#ifdef CONFIG_WFD
+#define STA_OP_WFD_MODE(sta) (sta)->op_wfd_mode
+#define STA_SET_OP_WFD_MODE(sta, mode) (sta)->op_wfd_mode = (mode)
+#else
+#define STA_OP_WFD_MODE(sta) 0
+#define STA_SET_OP_WFD_MODE(sta, mode) do {} while (0)
+#endif
+
+#define AID_BMP_LEN(max_aid) ((max_aid + 1) / 8 + (((max_aid + 1) % 8) ? 1 : 0))
+
+struct	sta_priv {
+
+	u8 *pallocated_stainfo_buf;
+	u8 *pstainfo_buf;
+	_queue	free_sta_queue;
+
+	_lock sta_hash_lock;
+	_list   sta_hash[NUM_STA];
+#ifdef PRIVATE_R
+	_lock last_rx_uc_data_lock;
+#endif
+	int asoc_sta_count;
+	_queue sleep_q;
+	_queue wakeup_q;
+
+	_adapter *padapter;
+
+	u32 adhoc_expire_to;
+
+	int rx_chk_limit;
+
+#ifdef CONFIG_AP_MODE
+	_list asoc_list;
+	_list auth_list;
+	_lock asoc_list_lock;
+	_lock auth_list_lock;
+	_lock active_time_lock;
+	u8 asoc_list_cnt;
+	u8 auth_list_cnt;
+
+	_list add_sta_list;
+	u8 add_sta_list_cnt;
+
+	unsigned int auth_to;  /* sec, time to expire in authenticating. */
+	unsigned int assoc_to; /* sec, time to expire before associating. */
+	unsigned int expire_to; /* sec , time to expire after associated. */
+
+	/*
+	* pointers to STA info; based on allocated AID or NULL if AID free
+	* AID is in the range 1-2007, so sta_aid[0] corresponders to AID 1
+	*/
+	struct sta_info **sta_aid;
+	u16 max_aid;
+	u16 started_aid; /* started AID for allocation search */
+	bool rr_aid; /* round robin AID allocation, will modify started_aid */
+	u8 aid_bmp_len; /* in byte */
+	u8 *sta_dz_bitmap;
+	u8 *tim_bitmap;
+
+	u16 max_num_sta;
+
+#if CONFIG_RTW_MACADDR_ACL
+	struct wlan_acl_pool acl_list[RTW_ACL_PERIOD_NUM];
+#endif
+
+	#if CONFIG_RTW_PRE_LINK_STA
+	struct pre_link_sta_ctl_t pre_link_sta_ctl;
+	#endif
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	u8 tbtx_asoc_list_cnt;
+	struct sta_info *token_holder[NR_MAXSTA_INSLOT];
+	struct sta_info *last_token_holder;
+	ATOMIC_T nr_token_keeper;
+#endif
+#endif /* CONFIG_AP_MODE */
+
+#ifdef CONFIG_ATMEL_RC_PATCH
+	u8 atmel_rc_pattern[6];
+#endif
+	u8 c2h_sta_mac[ETH_ALEN];
+	u8 c2h_adapter_id;
+	struct submit_ctx *gotc2h;
+};
+
+
+__inline static u32 wifi_mac_hash(const u8 *mac)
+{
+	u32 x;
+
+	x = mac[0];
+	x = (x << 2) ^ mac[1];
+	x = (x << 2) ^ mac[2];
+	x = (x << 2) ^ mac[3];
+	x = (x << 2) ^ mac[4];
+	x = (x << 2) ^ mac[5];
+
+	x ^= x >> 8;
+	x  = x & (NUM_STA - 1);
+
+	return x;
+}
+
+
+extern u32	_rtw_init_sta_priv(struct sta_priv *pstapriv);
+extern u32	_rtw_free_sta_priv(struct sta_priv *pstapriv);
+
+#define stainfo_offset_valid(offset) (offset < NUM_STA && offset >= 0)
+int rtw_stainfo_offset(struct sta_priv *stapriv, struct sta_info *sta);
+struct sta_info *rtw_get_stainfo_by_offset(struct sta_priv *stapriv, int offset);
+
+extern struct sta_info *rtw_alloc_stainfo(struct sta_priv *stapriv, const u8 *hwaddr,
+					enum rtw_device_type dtype,
+					u16 main_id,
+					const u8 link_idx,
+					enum phl_cmd_type cmd_type);
+
+extern struct sta_info *rtw_alloc_stainfo_sw(struct sta_priv *stapriv,
+					enum rtw_device_type dtype,
+					u16 main_id,
+					const u8 link_idx,
+					const u8 *hwaddr);
+extern u32 rtw_alloc_stainfo_hw(struct	sta_priv *stapriv, struct sta_info *psta);
+
+extern u32 rtw_free_stainfo(_adapter *padapter , struct sta_info *psta);
+u32	rtw_free_stainfo_sw(_adapter *padapter, struct sta_info *psta);
+extern void rtw_free_all_stainfo(_adapter *padapter);
+bool rtw_is_self_addr_stainfo(_adapter *adapter, struct sta_info *sta);
+extern bool rtw_is_self_stainfo(_adapter *padapter, struct sta_info *sta);
+extern u32 rtw_free_mld_stainfo(_adapter *padapter, struct rtw_phl_mld_t *mld);
+extern struct sta_info *rtw_get_stainfo_by_macid(struct sta_priv *pstapriv, u16 macid);
+extern struct sta_info *rtw_get_bcmc_stainfo(_adapter *padapter, struct _ADAPTER_LINK *padapter_link);
+extern struct sta_info *rtw_get_stainfo(struct sta_priv *stapriv, const u8 *hwaddr);
+extern struct sta_info *rtw_get_link_stainfo(struct sta_priv *stapriv, struct rtw_phl_mld_t *pmld, u8 lidx);
+extern struct sta_info *rtw_get_primary_stainfo(struct sta_priv *stapriv, struct sta_info *psta);
+extern struct sta_info *rtw_get_primary_stainfo_by_addr(struct sta_priv *stapriv, const u8 *hwaddr);
+
+u32	rtw_free_self_stainfo(_adapter *adapter);
+u32 rtw_init_self_stainfo(_adapter *adapter, enum phl_cmd_type cmd_type);
+
+struct sta_info *rtw_get_stainfo_to_free(struct sta_priv *stapriv, const u8 *hwaddr);
+void rtw_stainfo_claim_to_free_no_lock(struct sta_info *sta);
+void rtw_stainfo_claim_to_free(struct sta_info *sta);
+
+#ifdef CONFIG_AP_MODE
+u16 rtw_aid_alloc(_adapter *adapter, struct sta_info *sta);
+void dump_aid_status(void *sel, _adapter *adapter);
+void rtw_stapriv_asoc_list_lock(struct sta_priv *stapriv);
+void rtw_stapriv_asoc_list_unlock(struct sta_priv *stapriv);
+void rtw_stapriv_asoc_list_add(struct sta_priv *stapriv, struct sta_info *sta);
+void rtw_stapriv_asoc_list_del(struct sta_priv *stapriv, struct sta_info *sta);
+#endif
+
+#if CONFIG_RTW_MACADDR_ACL
+extern u8 rtw_access_ctrl(_adapter *adapter, const u8 *mac_addr);
+void dump_macaddr_acl(void *sel, _adapter *adapter);
+#endif
+
+#if CONFIG_RTW_PRE_LINK_STA
+bool rtw_is_pre_link_sta(struct sta_priv *stapriv, u8 *addr);
+struct sta_info *rtw_pre_link_sta_add(struct sta_priv *stapriv, u8 *hwaddr);
+void rtw_pre_link_sta_del(struct sta_priv *stapriv, u8 *hwaddr);
+void rtw_pre_link_sta_ctl_reset(struct sta_priv *stapriv);
+void rtw_pre_link_sta_ctl_init(struct sta_priv *stapriv);
+void rtw_pre_link_sta_ctl_deinit(struct sta_priv *stapriv);
+void dump_pre_link_sta_ctl(void *sel, struct sta_priv *stapriv);
+#endif /* CONFIG_RTW_PRE_LINK_STA */
+
+#endif /* _STA_INFO_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/usb_ops.h b/drivers/net/wireless/realtek/rtl8852bs/include/usb_ops.h
new file mode 100644
index 000000000000..6558d38dda46
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/usb_ops.h
@@ -0,0 +1,67 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __USB_OPS_H_
+#define __USB_OPS_H_
+
+
+#define REALTEK_USB_VENQT_READ		0xC0
+#define REALTEK_USB_VENQT_WRITE	0x40
+#define REALTEK_USB_VENQT_CMD_REQ	0x05
+#define REALTEK_USB_VENQT_CMD_IDX	0x00
+#define REALTEK_USB_BULK_IN_EP_IDX	0
+#define REALTEK_USB_IN_INT_EP_IDX	1
+
+enum {
+	VENDOR_WRITE = 0x00,
+	VENDOR_READ = 0x01,
+};
+#define ALIGNMENT_UNIT				16
+#define MAX_VENDOR_REQ_CMD_SIZE	254		/* 8188cu SIE Support */
+#define MAX_USB_IO_CTL_SIZE		(MAX_VENDOR_REQ_CMD_SIZE + ALIGNMENT_UNIT)
+
+#ifdef PLATFORM_LINUX
+#include <usb_ops_linux.h>
+#endif /* PLATFORM_LINUX */
+
+#define IS_FULL_SPEED_USB(_dvobj)	(dvobj_to_usb(_dvobj)->usb_speed == RTW_USB_SPEED_FULL)
+#define IS_HIGH_SPEED_USB(_dvobj)	(dvobj_to_usb(_dvobj)->usb_speed == RTW_USB_SPEED_HIGH)
+#define IS_SUPER_SPEED_USB(_dvobj)	(dvobj_to_usb(_dvobj)->usb_speed == RTW_USB_SPEED_SUPER)
+#define IS_SUPER_PLUS_SPEED_USB(_dvobj)	(dvobj_to_usb(_dvobj)->usb_speed == RTW_USB_SPEED_SUPER_10G)
+
+static inline u16 rtw_usb_bulkout_size(struct dvobj_priv *dvobj)
+{
+	if (IS_SUPER_SPEED_USB(dvobj))
+		return USB_SUPER_SPEED_BULK_SIZE;
+	else if (IS_HIGH_SPEED_USB(dvobj))
+		return USB_HIGH_SPEED_BULK_SIZE;
+	else
+		return USB_FULL_SPEED_BULK_SIZE;
+}
+
+static inline u8 rtw_usb_bulkout_size_boundary(struct dvobj_priv *dvobj, int buf_len)
+{
+	u8 rst = _TRUE;
+
+	if (IS_SUPER_SPEED_USB(dvobj))
+		rst = (0 == (buf_len) % USB_SUPER_SPEED_BULK_SIZE) ? _TRUE : _FALSE;
+	else if (IS_HIGH_SPEED_USB(dvobj))
+		rst = (0 == (buf_len) % USB_HIGH_SPEED_BULK_SIZE) ? _TRUE : _FALSE;
+	else
+		rst = (0 == (buf_len) % USB_FULL_SPEED_BULK_SIZE) ? _TRUE : _FALSE;
+	return rst;
+}
+
+
+#endif /* __USB_OPS_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/usb_ops_linux.h b/drivers/net/wireless/realtek/rtl8852bs/include/usb_ops_linux.h
new file mode 100644
index 000000000000..5b5702261344
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/usb_ops_linux.h
@@ -0,0 +1,66 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __USB_OPS_LINUX_H__
+#define __USB_OPS_LINUX_H__
+
+#define RTW_USB_CONTROL_MSG_TIMEOUT_TEST	10/* ms */
+#define RTW_USB_CONTROL_MSG_TIMEOUT	500/* ms */
+
+#define RECV_BULK_IN_ADDR		0x80/* assign by drv, not real address */
+#define RECV_INT_IN_ADDR		0x81/* assign by drv, not real address */
+
+#define INTERRUPT_MSG_FORMAT_LEN 60
+
+#if defined(CONFIG_VENDOR_REQ_RETRY) && defined(CONFIG_USB_VENDOR_REQ_MUTEX)
+	/* vendor req retry should be in the situation when each vendor req is atomically submitted from others */
+	#define MAX_USBCTRL_VENDORREQ_TIMES	10
+#else
+	#define MAX_USBCTRL_VENDORREQ_TIMES	1
+#endif
+
+#define RTW_USB_BULKOUT_TIMEOUT	5000/* ms */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 12))
+#define rtw_usb_control_msg(dev, pipe, request, requesttype, value, index, data, size, timeout_ms) \
+	usb_control_msg((dev), (pipe), (request), (requesttype), (value), (index), (data), (size), (timeout_ms))
+#define rtw_usb_bulk_msg(usb_dev, pipe, data, len, actual_length, timeout_ms) \
+	usb_bulk_msg((usb_dev), (pipe), (data), (len), (actual_length), (timeout_ms))
+#else
+#define rtw_usb_control_msg(dev, pipe, request, requesttype, value, index, data, size, timeout_ms) \
+	usb_control_msg((dev), (pipe), (request), (requesttype), (value), (index), (data), (size), \
+		((timeout_ms) == 0) || ((timeout_ms) * HZ / 1000 > 0) ? ((timeout_ms) * HZ / 1000) : 1)
+#define rtw_usb_bulk_msg(usb_dev, pipe, data, len, actual_length, timeout_ms) \
+	usb_bulk_msg((usb_dev), (pipe), (data), (len), (actual_length), \
+		((timeout_ms) == 0) || ((timeout_ms) * HZ / 1000 > 0) ? ((timeout_ms) * HZ / 1000) : 1)
+#endif
+
+
+int rtw_os_urb_resource_alloc(struct data_urb *dataurb);
+void rtw_os_urb_resource_free(struct data_urb *dataurb);
+
+int usbctrl_vendorreq(struct dvobj_priv *pdvobjpriv, u8 request, u16 value, u16 index,
+				void *pdata, u16 len, u8 requesttype);
+
+u32 rtw_usb_write_port(void *d, u8 *phl_tx_buf_ptr,
+	u8  bulk_id, u32 len, u8 *pkt_data_buf);
+
+void rtw_usb_write_port_cancel(void *d);
+
+u32 rtw_usb_read_port(void *d, void *rxobj,
+	u8 *inbuf, u32 inbuf_len, u8 bulk_id, u8 minlen);
+
+void rtw_usb_read_port_cancel(void *d);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/wifi.h b/drivers/net/wireless/realtek/rtl8852bs/include/wifi.h
new file mode 100644
index 000000000000..33de68235df7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/wifi.h
@@ -0,0 +1,1312 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _WIFI_H_
+#define _WIFI_H_
+
+
+#ifndef BIT
+#define BIT(x)	(1 << (x))
+#endif
+#ifndef BIT_ULL
+#define BIT_ULL(x)	(1ULL << (x))
+#endif
+
+
+#define WLAN_ETHHDR_LEN		14
+#define WLAN_ETHADDR_LEN	6
+#define WLAN_IEEE_OUI_LEN	3
+#define WLAN_ADDR_LEN		6
+#define WLAN_CRC_LEN		4
+#define WLAN_BSSID_LEN		6
+#define WLAN_BSS_TS_LEN		8
+#define WLAN_HDR_A3_LEN		24
+#define WLAN_HDR_A4_LEN		30
+#define WLAN_HDR_A3_QOS_LEN	26
+
+#define WLAN_HDR_A3_HTC_LEN	28
+#define WLAN_HDR_A3_QOS_HTC_LEN	30
+#define WLAN_HDR_A4_HTC_LEN	34
+#define WLAN_HDR_A4_QOS_HTC_LEN	36
+
+#define WLAN_HDR_A4_QOS_LEN	32
+#define WLAN_SSID_MAXLEN	32
+#define WLAN_DATA_MAXLEN	2312
+
+#define WLAN_A3_PN_OFFSET	24
+#define WLAN_A4_PN_OFFSET	30
+
+#define WLAN_MIN_ETHFRM_LEN	60
+#define WLAN_MAX_ETHFRM_LEN	1514
+#define WLAN_ETHHDR_LEN		14
+#define WLAN_WMM_LEN		24
+#define VENDOR_NAME_LEN		20
+#define WLAN_IE_ID_LEN 1
+#define WLAN_IE_LEN_LEN 1
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+#define WLAN_MAX_VENDOR_IE_LEN 255
+#define WLAN_MAX_VENDOR_IE_NUM 5
+#define WIFI_BEACON_VENDOR_IE_BIT BIT(0)
+#define WIFI_PROBEREQ_VENDOR_IE_BIT BIT(1)
+#define WIFI_PROBERESP_VENDOR_IE_BIT BIT(2)
+#define WIFI_ASSOCREQ_VENDOR_IE_BIT BIT(3)
+#define WIFI_ASSOCRESP_VENDOR_IE_BIT BIT(4)
+#ifdef CONFIG_P2P
+#define WIFI_P2P_PROBEREQ_VENDOR_IE_BIT BIT(5)
+#define WIFI_P2P_PROBERESP_VENDOR_IE_BIT BIT(6)
+#define WLAN_MAX_VENDOR_IE_MASK_MAX 7
+#else
+#define WLAN_MAX_VENDOR_IE_MASK_MAX 5
+#endif
+#endif
+
+#define P80211CAPTURE_VERSION	0x80211001
+
+/* This value is tested by WiFi 11n Test Plan 5.2.3.
+ * This test verifies the WLAN NIC can update the NAV through sending the CTS with large duration. */
+#define	WiFiNavUpperUs				30000	/* 30 ms */
+
+#ifdef GREEN_HILL
+#pragma pack(1)
+#endif
+
+enum WIFI_FRAME_TYPE {
+	WIFI_MGT_TYPE  =	(0),
+	WIFI_CTRL_TYPE =	(BIT(2)),
+	WIFI_DATA_TYPE =	(BIT(3)),
+	WIFI_QOS_DATA_TYPE	= (BIT(7) | BIT(3)),	/* !< QoS Data	 */
+};
+
+enum WIFI_FRAME_SUBTYPE {
+
+	/* below is for mgt frame */
+	WIFI_ASSOCREQ       = (0 | WIFI_MGT_TYPE),
+	WIFI_ASSOCRSP       = (BIT(4) | WIFI_MGT_TYPE),
+	WIFI_REASSOCREQ     = (BIT(5) | WIFI_MGT_TYPE),
+	WIFI_REASSOCRSP     = (BIT(5) | BIT(4) | WIFI_MGT_TYPE),
+	WIFI_PROBEREQ       = (BIT(6) | WIFI_MGT_TYPE),
+	WIFI_PROBERSP       = (BIT(6) | BIT(4) | WIFI_MGT_TYPE),
+	WIFI_BEACON         = (BIT(7) | WIFI_MGT_TYPE),
+	WIFI_ATIM           = (BIT(7) | BIT(4) | WIFI_MGT_TYPE),
+	WIFI_DISASSOC       = (BIT(7) | BIT(5) | WIFI_MGT_TYPE),
+	WIFI_AUTH           = (BIT(7) | BIT(5) | BIT(4) | WIFI_MGT_TYPE),
+	WIFI_DEAUTH         = (BIT(7) | BIT(6) | WIFI_MGT_TYPE),
+	WIFI_ACTION         = (BIT(7) | BIT(6) | BIT(4) | WIFI_MGT_TYPE),
+	WIFI_ACTION_NOACK = (BIT(7) | BIT(6) | BIT(5) | WIFI_MGT_TYPE),
+
+	/* below is for control frame */
+	WIFI_TRIGGER = (BIT(5) | WIFI_CTRL_TYPE),
+	WIFI_BF_REPORT_POLL = (BIT(6) | WIFI_CTRL_TYPE),
+	WIFI_NDPA         = (BIT(6) | BIT(4) | WIFI_CTRL_TYPE),
+	WIFI_BAR            = (BIT(7) | WIFI_CTRL_TYPE),
+	WIFI_PSPOLL         = (BIT(7) | BIT(5) | WIFI_CTRL_TYPE),
+	WIFI_RTS            = (BIT(7) | BIT(5) | BIT(4) | WIFI_CTRL_TYPE),
+	WIFI_CTS            = (BIT(7) | BIT(6) | WIFI_CTRL_TYPE),
+	WIFI_ACK            = (BIT(7) | BIT(6) | BIT(4) | WIFI_CTRL_TYPE),
+	WIFI_CFEND          = (BIT(7) | BIT(6) | BIT(5) | WIFI_CTRL_TYPE),
+	WIFI_CFEND_CFACK    = (BIT(7) | BIT(6) | BIT(5) | BIT(4) | WIFI_CTRL_TYPE),
+
+	/* below is for data frame */
+	WIFI_DATA           = (0 | WIFI_DATA_TYPE),
+	WIFI_DATA_CFACK     = (BIT(4) | WIFI_DATA_TYPE),
+	WIFI_DATA_CFPOLL    = (BIT(5) | WIFI_DATA_TYPE),
+	WIFI_DATA_CFACKPOLL = (BIT(5) | BIT(4) | WIFI_DATA_TYPE),
+	WIFI_DATA_NULL      = (BIT(6) | WIFI_DATA_TYPE),
+	WIFI_CF_ACK         = (BIT(6) | BIT(4) | WIFI_DATA_TYPE),
+	WIFI_CF_POLL        = (BIT(6) | BIT(5) | WIFI_DATA_TYPE),
+	WIFI_CF_ACKPOLL     = (BIT(6) | BIT(5) | BIT(4) | WIFI_DATA_TYPE),
+	WIFI_QOS_DATA_NULL	= (BIT(6) | WIFI_QOS_DATA_TYPE),
+};
+
+enum WIFI_REASON_CODE	{
+	_RSON_RESERVED_					= 0,
+	_RSON_UNSPECIFIED_				= 1,
+	_RSON_AUTH_NO_LONGER_VALID_		= 2,
+	_RSON_DEAUTH_STA_LEAVING_		= 3,
+	_RSON_INACTIVITY_				= 4,
+	_RSON_UNABLE_HANDLE_			= 5,
+	_RSON_CLS2_						= 6,
+	_RSON_CLS3_						= 7,
+	_RSON_DISAOC_STA_LEAVING_		= 8,
+	_RSON_ASOC_NOT_AUTH_			= 9,
+
+	/* WPA reason */
+	_RSON_INVALID_IE_				= 13,
+	_RSON_MIC_FAILURE_				= 14,
+	_RSON_4WAY_HNDSHK_TIMEOUT_		= 15,
+	_RSON_GROUP_KEY_UPDATE_TIMEOUT_	= 16,
+	_RSON_DIFF_IE_					= 17,
+	_RSON_MLTCST_CIPHER_NOT_VALID_	= 18,
+	_RSON_UNICST_CIPHER_NOT_VALID_	= 19,
+	_RSON_AKMP_NOT_VALID_			= 20,
+	_RSON_UNSUPPORT_RSNE_VER_		= 21,
+	_RSON_INVALID_RSNE_CAP_			= 22,
+	_RSON_IEEE_802DOT1X_AUTH_FAIL_	= 23,
+
+	/* belowing are Realtek definition */
+	_RSON_PMK_NOT_AVAILABLE_		= 24,
+	_RSON_TDLS_TEAR_TOOFAR_			= 25,
+	_RSON_TDLS_TEAR_UN_RSN_			= 26,
+};
+
+/* Reason codes (IEEE 802.11-2007, 7.3.1.7, Table 7-22) */
+#if 0
+#define WLAN_REASON_UNSPECIFIED 1
+#define WLAN_REASON_PREV_AUTH_NOT_VALID 2
+#define WLAN_REASON_DEAUTH_LEAVING 3
+#define WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY 4
+#define WLAN_REASON_DISASSOC_AP_BUSY 5
+#define WLAN_REASON_CLASS2_FRAME_FROM_NONAUTH_STA 6
+#define WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA 7
+#define WLAN_REASON_DISASSOC_STA_HAS_LEFT 8
+#define WLAN_REASON_STA_REQ_ASSOC_WITHOUT_AUTH 9
+#endif
+/* IEEE 802.11h */
+#define WLAN_REASON_PWR_CAPABILITY_NOT_VALID 10
+#define WLAN_REASON_SUPPORTED_CHANNEL_NOT_VALID 11
+#if 0
+/* IEEE 802.11i */
+#define WLAN_REASON_INVALID_IE 13
+#define WLAN_REASON_MICHAEL_MIC_FAILURE 14
+#define WLAN_REASON_4WAY_HANDSHAKE_TIMEOUT 15
+#define WLAN_REASON_GROUP_KEY_UPDATE_TIMEOUT 16
+#define WLAN_REASON_IE_IN_4WAY_DIFFERS 17
+#define WLAN_REASON_GROUP_CIPHER_NOT_VALID 18
+#define WLAN_REASON_PAIRWISE_CIPHER_NOT_VALID 19
+#define WLAN_REASON_AKMP_NOT_VALID 20
+#define WLAN_REASON_UNSUPPORTED_RSN_IE_VERSION 21
+#define WLAN_REASON_INVALID_RSN_IE_CAPAB 22
+#define WLAN_REASON_IEEE_802_1X_AUTH_FAILED 23
+#define WLAN_REASON_CIPHER_SUITE_REJECTED 24
+#endif
+
+enum WIFI_STATUS_CODE {
+	_STATS_SUCCESSFUL_			= 0,
+	_STATS_FAILURE_				= 1,
+	_STATS_SEC_DISABLED_			= 5,
+	_STATS_NOT_IN_SAME_BSS_		= 7,
+	_STATS_CAP_FAIL_			= 10,
+	_STATS_NO_ASOC_				= 11,
+	_STATS_OTHER_				= 12,
+	_STATS_NO_SUPP_ALG_			= 13,
+	_STATS_OUT_OF_AUTH_SEQ_		= 14,
+	_STATS_CHALLENGE_FAIL_		= 15,
+	_STATS_AUTH_TIMEOUT_		= 16,
+	_STATS_UNABLE_HANDLE_STA_	= 17,
+	_STATS_RATE_FAIL_			= 18,
+	_STATS_REFUSED_TEMPORARILY_ = 30,
+	_STATS_DECLINE_REQ_			= 37,
+	_STATS_INVALID_PARAMETERS_	= 38,
+	_STATS_INVALID_RSNIE_			= 72,
+};
+
+/* Status codes (IEEE 802.11-2007, 7.3.1.9, Table 7-23) */
+#if 0
+#define WLAN_STATUS_SUCCESS 0
+#define WLAN_STATUS_UNSPECIFIED_FAILURE 1
+#define WLAN_STATUS_CAPS_UNSUPPORTED 10
+#define WLAN_STATUS_REASSOC_NO_ASSOC 11
+#define WLAN_STATUS_ASSOC_DENIED_UNSPEC 12
+#define WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG 13
+#define WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION 14
+#define WLAN_STATUS_CHALLENGE_FAIL 15
+#define WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA 17
+#define WLAN_STATUS_ASSOC_DENIED_RATES 18
+#endif
+#define WLAN_STATUS_AUTH_TIMEOUT 16
+
+/* entended */
+/* IEEE 802.11b */
+#define WLAN_STATUS_ASSOC_DENIED_NOSHORT 19
+#define WLAN_STATUS_ASSOC_DENIED_NOPBCC 20
+#define WLAN_STATUS_ASSOC_DENIED_NOAGILITY 21
+/* IEEE 802.11h */
+#define WLAN_STATUS_SPEC_MGMT_REQUIRED 22
+#define WLAN_STATUS_PWR_CAPABILITY_NOT_VALID 23
+#define WLAN_STATUS_SUPPORTED_CHANNEL_NOT_VALID 24
+/* IEEE 802.11g */
+#define WLAN_STATUS_ASSOC_DENIED_NO_SHORT_SLOT_TIME 25
+#define WLAN_STATUS_ASSOC_DENIED_NO_ER_PBCC 26
+#define WLAN_STATUS_ASSOC_DENIED_NO_DSSS_OFDM 27
+/* IEEE 802.11w */
+#define WLAN_STATUS_ASSOC_REJECTED_TEMPORARILY 30
+#define WLAN_STATUS_ROBUST_MGMT_FRAME_POLICY_VIOLATION 31
+/* IEEE 802.11i */
+#define WLAN_STATUS_INVALID_IE 40
+#define WLAN_STATUS_GROUP_CIPHER_NOT_VALID 41
+#define WLAN_STATUS_PAIRWISE_CIPHER_NOT_VALID 42
+#define WLAN_STATUS_AKMP_NOT_VALID 43
+#define WLAN_STATUS_UNSUPPORTED_RSN_IE_VERSION 44
+#define WLAN_STATUS_INVALID_RSN_IE_CAPAB 45
+#define WLAN_STATUS_CIPHER_REJECTED_PER_POLICY 46
+#define WLAN_STATUS_TS_NOT_CREATED 47
+#define WLAN_STATUS_DIRECT_LINK_NOT_ALLOWED 48
+#define WLAN_STATUS_DEST_STA_NOT_PRESENT 49
+#define WLAN_STATUS_DEST_STA_NOT_QOS_STA 50
+#define WLAN_STATUS_ASSOC_DENIED_LISTEN_INT_TOO_LARGE 51
+/* IEEE 802.11r */
+#define WLAN_STATUS_INVALID_FT_ACTION_FRAME_COUNT 52
+#define WLAN_STATUS_INVALID_PMKID 53
+#define WLAN_STATUS_INVALID_MDIE 54
+#define WLAN_STATUS_INVALID_FTIE 55
+
+
+enum WIFI_REG_DOMAIN {
+	DOMAIN_FCC		= 1,
+	DOMAIN_IC		= 2,
+	DOMAIN_ETSI		= 3,
+	DOMAIN_SPAIN	= 4,
+	DOMAIN_FRANCE	= 5,
+	DOMAIN_MKK		= 6,
+	DOMAIN_ISRAEL	= 7,
+	DOMAIN_MKK1		= 8,
+	DOMAIN_MKK2		= 9,
+	DOMAIN_MKK3		= 10,
+	DOMAIN_MAX
+};
+
+#define _TO_DS_		BIT(8)
+#define _FROM_DS_	BIT(9)
+#define _MORE_FRAG_	BIT(10)
+#define _RETRY_		BIT(11)
+#define _PWRMGT_	BIT(12)
+#define _MORE_DATA_	BIT(13)
+#define _PRIVACY_	BIT(14)
+#define _ORDER_			BIT(15)
+
+#define SetToDs(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) |= cpu_to_le16(_TO_DS_); \
+	} while (0)
+
+#define GetToDs(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_TO_DS_)) != 0)
+
+#define ClearToDs(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) &= (~cpu_to_le16(_TO_DS_)); \
+	} while (0)
+
+#define SetFrDs(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) |= cpu_to_le16(_FROM_DS_); \
+	} while (0)
+
+#define GetFrDs(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_FROM_DS_)) != 0)
+
+#define ClearFrDs(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) &= (~cpu_to_le16(_FROM_DS_)); \
+	} while (0)
+
+#define get_tofr_ds(pframe)	((GetToDs(pframe) << 1) | GetFrDs(pframe))
+
+
+#define SetMFrag(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) |= cpu_to_le16(_MORE_FRAG_); \
+	} while (0)
+
+#define GetMFrag(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_MORE_FRAG_)) != 0)
+
+#define ClearMFrag(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) &= (~cpu_to_le16(_MORE_FRAG_)); \
+	} while (0)
+
+#define SetRetry(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) |= cpu_to_le16(_RETRY_); \
+	} while (0)
+
+#define GetRetry(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_RETRY_)) != 0)
+
+#define ClearRetry(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) &= (~cpu_to_le16(_RETRY_)); \
+	} while (0)
+
+#define SetPwrMgt(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) |= cpu_to_le16(_PWRMGT_); \
+	} while (0)
+
+#define GetPwrMgt(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_PWRMGT_)) != 0)
+
+#define ClearPwrMgt(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) &= (~cpu_to_le16(_PWRMGT_)); \
+	} while (0)
+
+#define SetMData(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) |= cpu_to_le16(_MORE_DATA_); \
+	} while (0)
+
+#define GetMData(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_MORE_DATA_)) != 0)
+
+#define ClearMData(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) &= (~cpu_to_le16(_MORE_DATA_)); \
+	} while (0)
+
+#define SetPrivacy(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) |= cpu_to_le16(_PRIVACY_); \
+	} while (0)
+
+#define GetPrivacy(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_PRIVACY_)) != 0)
+
+#define ClearPrivacy(pbuf)	\
+	do	{	\
+		*(unsigned short *)(pbuf) &= (~cpu_to_le16(_PRIVACY_)); \
+	} while (0)
+
+
+#define GetOrder(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_ORDER_)) != 0)
+
+#define GetFrameType(pbuf)	(le16_to_cpu(*(unsigned short *)(pbuf)) & (BIT(3) | BIT(2)))
+
+#define SetFrameType(pbuf, type)	\
+	do {	\
+		*(unsigned short *)(pbuf) &= __constant_cpu_to_le16(~(BIT(3) | BIT(2))); \
+		*(unsigned short *)(pbuf) |= __constant_cpu_to_le16(type); \
+	} while (0)
+
+#define get_frame_sub_type(pbuf)	(cpu_to_le16(*(unsigned short *)(pbuf)) & (BIT(7) | BIT(6) | BIT(5) | BIT(4) | BIT(3) | BIT(2)))
+
+
+#define set_frame_sub_type(pbuf, type) \
+	do {    \
+		*(unsigned short *)(pbuf) &= cpu_to_le16(~(BIT(7) | BIT(6) | BIT(5) | BIT(4) | BIT(3) | BIT(2))); \
+		*(unsigned short *)(pbuf) |= cpu_to_le16(type); \
+	} while (0)
+
+
+#define GetSequence(pbuf)	(cpu_to_le16(*(unsigned short *)((SIZE_PTR)(pbuf) + 22)) >> 4)
+
+#define GetFragNum(pbuf)	(cpu_to_le16(*(unsigned short *)((SIZE_PTR)(pbuf) + 22)) & 0x0f)
+
+#define GetTupleCache(pbuf)	(cpu_to_le16(*(unsigned short *)((SIZE_PTR)(pbuf) + 22)))
+
+#define SetFragNum(pbuf, num) \
+	do {    \
+		*(unsigned short *)((SIZE_PTR)(pbuf) + 22) = \
+			((*(unsigned short *)((SIZE_PTR)(pbuf) + 22)) & le16_to_cpu(~(0x000f))) | \
+				cpu_to_le16(0x0f & (num));     \
+	} while (0)
+
+#define SetSeqNum(pbuf, num) \
+	do {    \
+		*(unsigned short *)((SIZE_PTR)(pbuf) + 22) = \
+			((*(unsigned short *)((SIZE_PTR)(pbuf) + 22)) & le16_to_cpu((unsigned short)~0xfff0)) | \
+			le16_to_cpu((unsigned short)(0xfff0 & (num << 4))); \
+	} while (0)
+
+#define set_duration(pbuf, dur) \
+	do {    \
+		*(unsigned short *)((SIZE_PTR)(pbuf) + 2) = cpu_to_le16(0xffff & (dur)); \
+	} while (0)
+
+
+/* QoS control field */
+#define SetPriority(qc, tid)	SET_BITS_TO_LE_2BYTE(((u8 *)(qc)), 0, 4, tid)
+#define SetEOSP(qc, eosp)		SET_BITS_TO_LE_2BYTE(((u8 *)(qc)), 4, 1, eosp)
+#define SetAckpolicy(qc, ack)	SET_BITS_TO_LE_2BYTE(((u8 *)(qc)), 5, 2, ack)
+#define SetAMsdu(qc, amsdu)		SET_BITS_TO_LE_2BYTE(((u8 *)(qc)), 7, 1, amsdu)
+
+#define GetPriority(qc)		LE_BITS_TO_2BYTE(((u8 *)(qc)), 0, 4)
+#define GetEOSP(qc)			LE_BITS_TO_2BYTE(((u8 *)(qc)), 4, 1)
+#define GetAckpolicy(qc)	LE_BITS_TO_2BYTE(((u8 *)(qc)), 5, 2)
+#define GetAMsdu(qc)		LE_BITS_TO_2BYTE(((u8 *)(qc)), 7, 1)
+
+/* QoS control field (MSTA only) */
+#define set_mctrl_present(qc, p)	SET_BITS_TO_LE_2BYTE(((u8 *)(qc)), 8, 1, p)
+#define set_mps_lv(qc, lv)			SET_BITS_TO_LE_2BYTE(((u8 *)(qc)), 9, 1, lv)
+#define set_rspi(qc, rspi)			SET_BITS_TO_LE_2BYTE(((u8 *)(qc)), 10, 1, rspi)
+
+#define get_mctrl_present(qc)	LE_BITS_TO_2BYTE(((u8 *)(qc)), 8, 1)
+#define get_mps_lv(qc)			LE_BITS_TO_2BYTE(((u8 *)(qc)), 9, 1)
+#define get_rspi(qc)			LE_BITS_TO_2BYTE(((u8 *)(qc)), 10, 1)
+
+
+#define GetAid(pbuf)	(cpu_to_le16(*(unsigned short *)((SIZE_PTR)(pbuf) + 2)) & 0x3fff)
+
+#define GetTid(pbuf)	(cpu_to_le16(*(unsigned short *)((SIZE_PTR)(pbuf) + (((GetToDs(pbuf)<<1) | GetFrDs(pbuf)) == 3 ? 30 : 24))) & 0x000f)
+
+#define GetAddr1Ptr(pbuf)	((unsigned char *)((SIZE_PTR)(pbuf) + 4))
+
+#define get_addr2_ptr(pbuf)	((unsigned char *)((SIZE_PTR)(pbuf) + 10))
+
+#define GetAddr3Ptr(pbuf)	((unsigned char *)((SIZE_PTR)(pbuf) + 16))
+
+#define GetAddr4Ptr(pbuf)	((unsigned char *)((SIZE_PTR)(pbuf) + 24))
+
+
+#define MacAddr_isBcst(addr) \
+	(\
+	 ((addr[0] == 0xff) && (addr[1] == 0xff) && \
+	  (addr[2] == 0xff) && (addr[3] == 0xff) && \
+	  (addr[4] == 0xff) && (addr[5] == 0xff)) ? _TRUE : _FALSE \
+	)
+
+__inline static int IS_MCAST(const u8 *da)
+{
+	if ((*da) & 0x01)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+__inline static unsigned char *get_ra(unsigned char *pframe)
+{
+	unsigned char	*ra;
+	ra = GetAddr1Ptr(pframe);
+	return ra;
+}
+__inline static unsigned char *get_ta(unsigned char *pframe)
+{
+	unsigned char	*ta;
+	ta = get_addr2_ptr(pframe);
+	return ta;
+}
+
+/* can't apply to mesh mode */
+__inline static unsigned char *get_da(unsigned char *pframe)
+{
+	unsigned char	*da;
+	unsigned int	to_fr_ds	= (GetToDs(pframe) << 1) | GetFrDs(pframe);
+
+	switch (to_fr_ds) {
+	case 0x00:	/* ToDs=0, FromDs=0 */
+		da = GetAddr1Ptr(pframe);
+		break;
+	case 0x01:	/* ToDs=0, FromDs=1 */
+		da = GetAddr1Ptr(pframe);
+		break;
+	case 0x02:	/* ToDs=1, FromDs=0 */
+		da = GetAddr3Ptr(pframe);
+		break;
+	default:	/* ToDs=1, FromDs=1 */
+		da = GetAddr3Ptr(pframe);
+		break;
+	}
+
+	return da;
+}
+
+/* can't apply to mesh mode */
+__inline static unsigned char *get_sa(unsigned char *pframe)
+{
+	unsigned char	*sa;
+	unsigned int	to_fr_ds	= (GetToDs(pframe) << 1) | GetFrDs(pframe);
+
+	switch (to_fr_ds) {
+	case 0x00:	/* ToDs=0, FromDs=0 */
+		sa = get_addr2_ptr(pframe);
+		break;
+	case 0x01:	/* ToDs=0, FromDs=1 */
+		sa = GetAddr3Ptr(pframe);
+		break;
+	case 0x02:	/* ToDs=1, FromDs=0 */
+		sa = get_addr2_ptr(pframe);
+		break;
+	default:	/* ToDs=1, FromDs=1 */
+		sa = GetAddr4Ptr(pframe);
+		break;
+	}
+
+	return sa;
+}
+
+/* can't apply to mesh mode */
+__inline static unsigned char *get_hdr_bssid(unsigned char *pframe)
+{
+	unsigned char	*sa = NULL;
+	unsigned int	to_fr_ds	= (GetToDs(pframe) << 1) | GetFrDs(pframe);
+
+	switch (to_fr_ds) {
+	case 0x00:	/* ToDs=0, FromDs=0 */
+		sa = GetAddr3Ptr(pframe);
+		break;
+	case 0x01:	/* ToDs=0, FromDs=1 */
+		sa = get_addr2_ptr(pframe);
+		break;
+	case 0x02:	/* ToDs=1, FromDs=0 */
+		sa = GetAddr1Ptr(pframe);
+		break;
+	case 0x03:	/* ToDs=1, FromDs=1 */
+		sa = GetAddr1Ptr(pframe);
+		break;
+	}
+
+	return sa;
+}
+
+
+__inline static int IsFrameTypeCtrl(unsigned char *pframe)
+{
+	if (WIFI_CTRL_TYPE == GetFrameType(pframe))
+		return _TRUE;
+	else
+		return _FALSE;
+}
+static inline int IsFrameTypeMgnt(unsigned char *pframe)
+{
+	if (GetFrameType(pframe) == WIFI_MGT_TYPE)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+static inline int IsFrameTypeData(unsigned char *pframe)
+{
+	if (GetFrameType(pframe) == WIFI_DATA_TYPE)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+
+/*-----------------------------------------------------------------------------
+			Below is for common definition
+------------------------------------------------------------------------------*/
+#define _ASOCREQ_IE_OFFSET_		4	/* excluding wlan_hdr */
+#define	_ASOCRSP_IE_OFFSET_		6
+#define _REASOCREQ_IE_OFFSET_	10
+#define _REASOCRSP_IE_OFFSET_	6
+#define _PROBEREQ_IE_OFFSET_	0
+#define	_PROBERSP_IE_OFFSET_	12
+#define _AUTH_IE_OFFSET_		6
+#define _DEAUTH_IE_OFFSET_		0
+#define _BEACON_IE_OFFSET_		12
+#define _PUBLIC_ACTION_IE_OFFSET_	8
+
+#define _FIXED_IE_LENGTH_			_BEACON_IE_OFFSET_
+
+#define _SSID_IE_				0
+#define _SUPPORTEDRATES_IE_	1
+#define _DSSET_IE_				3
+#define _TIM_IE_					5
+#define _IBSS_PARA_IE_			6
+#define _CHLGETXT_IE_			16
+#define _SUPPORTED_CH_IE_		36
+#define	_MEAS_REQ_IE_		38
+#define	_MEAS_RSP_IE_		39
+#define _RSN_IE_2_				48
+#define _SSN_IE_1_					221
+#define _ERPINFO_IE_			42
+#define _EXT_SUPPORTEDRATES_IE_	50
+
+#define _HT_CAPABILITY_IE_			45
+#define _NEIGHBOR_REPORT_IE_			52
+#define _MDIE_					54
+#define _FTIE_					55
+#define _TIMEOUT_ITVL_IE_			56
+#define _SRC_IE_				59
+#define _HT_EXTRA_INFO_IE_			61 /*HT Operation Information*/
+#define _HT_ADD_INFO_IE_			61 /* _HT_EXTRA_INFO_IE_ */
+#define _WAPI_IE_				68
+#define _EID_RRM_EN_CAP_IE_			70
+#define _EID_MULTIPLEBSSID_IE_			71
+
+
+/* #define EID_BSSCoexistence			72 */ /* 20/40 BSS Coexistence
+ * #define EID_BSSIntolerantChlReport	73 */
+#define _RIC_Descriptor_IE_			75
+#ifdef CONFIG_IEEE80211W
+#define _MME_IE_					76 /* 802.11w Management MIC element */
+#endif /* CONFIG_IEEE80211W */
+#define _LINK_ID_IE_					101
+#define _CH_SWITCH_TIMING_		104
+#define _PTI_BUFFER_STATUS_		106
+#define _VENDOR_SPECIFIC_IE_		221
+
+#define	_RESERVED47_				47
+
+typedef	enum _ELEMENT_ID {
+	EID_SsId					= 0, /* service set identifier (0:32) */
+	EID_SupRates				= 1, /* supported rates (1:8) */
+	EID_FHParms				= 2, /* FH parameter set (5) */
+	EID_DSParms				= 3, /* DS parameter set (1) */
+	EID_CFParms				= 4, /* CF parameter set (6) */
+	EID_Tim						= 5, /* Traffic Information Map (4:254) */
+	EID_IbssParms				= 6, /* IBSS parameter set (2) */
+	EID_Country					= 7, /* */
+
+	/* Form 7.3.2: Information elements in 802.11E/D13.0, page 46. */
+	EID_QBSSLoad				= 11,
+	EID_EDCAParms				= 12,
+	EID_TSpec					= 13,
+	EID_TClass					= 14,
+	EID_Schedule				= 15,
+	/*  */
+
+	EID_Ctext					= 16, /* challenge text*/
+	EID_POWER_CONSTRAINT		= 32, /* Power Constraint*/
+
+	/* vivi for WIFITest, 802.11h AP, 20100427 */
+	/* 2010/12/26 MH The definition we can declare always!! */
+	EID_PowerCap				= 33,
+	EID_TPC				= 35,
+	EID_SupportedChannels		= 36,
+	EID_ChlSwitchAnnounce		= 37,
+
+	EID_MeasureRequest			= 38, /* Measurement Request */
+	EID_MeasureReport			= 39, /* Measurement Report */
+
+	EID_ERPInfo				= 42,
+
+	/* Form 7.3.2: Information elements in 802.11E/D13.0, page 46. */
+	EID_TSDelay				= 43,
+	EID_TCLASProc				= 44,
+	EID_HTCapability			= 45,
+	EID_QoSCap					= 46,
+	/*  */
+
+	EID_WPA2					= 48,
+	EID_ExtSupRates			= 50,
+
+	EID_FTIE					= 55, /* Defined in 802.11r */
+	EID_Timeout				= 56, /* Defined in 802.11r */
+
+	EID_SupRegulatory			= 59, /* Supported Requlatory Classes 802.11y */
+	EID_HTInfo					= 61,
+	EID_SecondaryChnlOffset		= 62,
+
+	EID_RMEnabledCapability     = 70, /* RM Enabled Capability */
+
+	EID_BSSCoexistence			= 72, /* 20/40 BSS Coexistence */
+	EID_BSSIntolerantChlReport	= 73,
+	EID_OBSS					= 74, /* Overlapping BSS Scan Parameters */
+
+	EID_LinkIdentifier			= 101, /* Defined in 802.11z */
+	EID_WakeupSchedule		= 102, /* Defined in 802.11z */
+	EID_ChnlSwitchTimeing		= 104, /* Defined in 802.11z */
+	EID_PTIControl				= 105, /* Defined in 802.11z */
+	EID_PUBufferStatus			= 106, /* Defined in 802.11z */
+
+	/* From S19:Aironet IE and S21:AP IP address IE in CCX v1.13, p16 and p18. */
+	EID_Aironet					= 133, /* 0x85: Aironet Element for Cisco CCX */
+	EID_CiscoIP					= 149, /* 0x95: IP Address IE for Cisco CCX */
+
+	EID_CellPwr					= 150, /* 0x96: Cell Power Limit IE. Ref. 0x96. */
+
+	EID_CCKM					= 156,
+
+	EID_Vendor					= 221, /* 0xDD: Vendor Specific */
+
+	EID_WAPI					= 68,
+	EID_VHTCapability 			= 191, /* Based on 802.11ac D2.0 */
+	EID_VHTOperation 			= 192, /* Based on 802.11ac D2.0 */
+	EID_VHTTransmitPower 		= 195,
+	EID_OpModeNotification		= 199, /* Based on 802.11ac D3.0 */
+} ELEMENT_ID, *PELEMENT_ID;
+
+/* ---------------------------------------------------------------------------
+					Below is the fixed elements...
+-----------------------------------------------------------------------------*/
+#define _AUTH_ALGM_NUM_			2
+#define _AUTH_SEQ_NUM_			2
+#define _BEACON_ITERVAL_		2
+#define _CAPABILITY_			2
+#define _CURRENT_APADDR_		6
+#define _LISTEN_INTERVAL_		2
+#define _RSON_CODE_				2
+#define _ASOC_ID_				2
+#define _STATUS_CODE_			2
+#define _TIMESTAMP_				8
+
+#define AUTH_ODD_TO				0
+#define AUTH_EVEN_TO			1
+
+#define WLAN_ETHCONV_ENCAP		1
+#define WLAN_ETHCONV_RFC1042	2
+#define WLAN_ETHCONV_8021h		3
+
+#define cap_ESS BIT(0)
+#define cap_IBSS BIT(1)
+#define cap_CFPollable BIT(2)
+#define cap_CFRequest BIT(3)
+#define cap_Privacy BIT(4)
+#define cap_ShortPremble BIT(5)
+#define cap_PBCC	BIT(6)
+#define cap_ChAgility	BIT(7)
+#define cap_SpecMgmt	BIT(8)
+#define cap_QoS	BIT(9)
+#define cap_ShortSlot	BIT(10)
+
+/*-----------------------------------------------------------------------------
+				Below is the definition for 802.11i / 802.1x
+------------------------------------------------------------------------------*/
+#define _IEEE8021X_MGT_			1		/* WPA */
+#define _IEEE8021X_PSK_			2		/* WPA with pre-shared key */
+
+#if 0
+#define _NO_PRIVACY_			0
+#define _WEP_40_PRIVACY_		1
+#define _TKIP_PRIVACY_			2
+#define _WRAP_PRIVACY_			3
+#define _CCMP_PRIVACY_			4
+#define _WEP_104_PRIVACY_		5
+#define _WEP_WPA_MIXED_PRIVACY_ 6	/*  WEP + WPA */
+#endif
+
+#define _MME_IE_LENGTH_  26
+
+/*-----------------------------------------------------------------------------
+				Below is the definition for WMM
+------------------------------------------------------------------------------*/
+#define _WMM_IE_Length_				7  /* for WMM STA */
+
+
+/*-----------------------------------------------------------------------------
+				Below is the definition for 802.11n
+------------------------------------------------------------------------------*/
+
+/* #ifdef CONFIG_80211N_HT */
+
+#define set_htc_order_bit(pbuf)	\
+		do	{	\
+			*(unsigned short *)(pbuf) |= cpu_to_le16(_ORDER_); \
+		} while (0)
+
+
+
+#define GetOrderBit(pbuf)	(((*(unsigned short *)(pbuf)) & le16_to_cpu(_ORDER_)) != 0)
+
+#define ACT_CAT_VENDOR				0x7F/* 127 */
+
+/**
+ * struct rtw_ieee80211_bar - HT Block Ack Request
+ *
+ * This structure refers to "HT BlockAckReq" as
+ * described in 802.11n draft section 7.2.1.7.1
+ */
+#if defined(PLATFORM_LINUX)
+struct rtw_ieee80211_bar {
+	unsigned short frame_control;
+	unsigned short duration;
+	unsigned char ra[6];
+	unsigned char ta[6];
+	unsigned short control;
+	unsigned short start_seq_num;
+} __attribute__((packed));
+#endif
+
+/* 802.11 BAR control masks */
+#define IEEE80211_BAR_CTRL_ACK_POLICY_NORMAL     0x0000
+#define IEEE80211_BAR_CTRL_CBMTID_COMPRESSED_BA  0x0004
+
+
+#if defined(PLATFORM_LINUX) || defined(PLATFORM_FREEBSD)
+
+
+
+/**
+* struct rtw_ieee80211_ht_cap - HT capabilities
+*
+* This structure refers to "HT capabilities element" as
+* described in 802.11n draft section 7.3.2.52
+*/
+
+struct rtw_ieee80211_ht_cap {
+	unsigned short	cap_info;
+	unsigned char	ampdu_params_info;
+	unsigned char	supp_mcs_set[16];
+	unsigned short	extended_ht_cap_info;
+	unsigned int		tx_BF_cap_info;
+	unsigned char	       antenna_selection_info;
+} __attribute__((packed));
+
+/**
+ * struct rtw_ieee80211_ht_cap - HT additional information
+ *
+ * This structure refers to "HT information element" as
+ * described in 802.11n draft section 7.3.2.53
+ */
+#ifndef CONFIG_IEEE80211_HT_ADDT_INFO
+struct ieee80211_ht_addt_info {
+	unsigned char	control_chan;
+	unsigned char		ht_param;
+	unsigned short	operation_mode;
+	unsigned short	stbc_param;
+	unsigned char		basic_set[16];
+} __attribute__((packed));
+#endif
+
+#define HT_CAPS_IE_LEN 26 /*struct rtw_ieee80211_ht_cap */
+struct HT_caps_element {
+	union {
+		struct {
+			unsigned short	HT_caps_info;
+			unsigned char	AMPDU_para;
+			unsigned char	MCS_rate[16];
+			unsigned short	HT_ext_caps;
+			unsigned int	Beamforming_caps;
+			unsigned char	ASEL_caps;
+		} HT_cap_element;
+		unsigned char HT_cap[HT_CAPS_IE_LEN];
+	} u;
+} __attribute__((packed));
+
+struct HT_info_element {
+	unsigned char	primary_channel;
+	unsigned char	infos[5];
+	unsigned char	MCS_rate[16];
+}  __attribute__((packed));
+
+struct AC_param {
+	unsigned char		ACI_AIFSN;
+	unsigned char		CW;
+	unsigned short	TXOP_limit;
+}  __attribute__((packed));
+
+struct WMM_para_element {
+	unsigned char		QoS_info;
+	unsigned char		reserved;
+	struct AC_param	ac_param[4];
+}  __attribute__((packed));
+
+struct ADDBA_request {
+	unsigned char		dialog_token;
+	unsigned short	BA_para_set;
+	unsigned short	BA_timeout_value;
+	unsigned short	BA_starting_seqctrl;
+}  __attribute__((packed));
+
+
+
+#endif
+
+
+typedef enum _HT_CAP_AMPDU_FACTOR {
+	MAX_AMPDU_FACTOR_8K		= 0,
+	MAX_AMPDU_FACTOR_16K	= 1,
+	MAX_AMPDU_FACTOR_32K	= 2,
+	MAX_AMPDU_FACTOR_64K	= 3,
+} HT_CAP_AMPDU_FACTOR;
+
+typedef enum _VHT_CAP_AMPDU_FACTOR {
+	MAX_AMPDU_FACTOR_128K = 4,
+	MAX_AMPDU_FACTOR_256K = 5,
+	MAX_AMPDU_FACTOR_512K = 6,
+	MAX_AMPDU_FACTOR_1M = 7,
+} VHT_CAP_AMPDU_FACTOR;
+
+
+typedef enum _HT_CAP_AMPDU_DENSITY {
+	AMPDU_DENSITY_VALUE_0 = 0 , /* For no restriction */
+	AMPDU_DENSITY_VALUE_1 = 1 , /* For 1/4 us */
+	AMPDU_DENSITY_VALUE_2 = 2 , /* For 1/2 us */
+	AMPDU_DENSITY_VALUE_3 = 3 , /* For 1 us */
+	AMPDU_DENSITY_VALUE_4 = 4 , /* For 2 us */
+	AMPDU_DENSITY_VALUE_5 = 5 , /* For 4 us */
+	AMPDU_DENSITY_VALUE_6 = 6 , /* For 8 us */
+	AMPDU_DENSITY_VALUE_7 = 7 , /* For 16 us */
+} HT_CAP_AMPDU_DENSITY;
+
+/* 802.11n HT capabilities masks */
+#define IEEE80211_HT_CAP_LDPC_CODING		0x0001
+#define IEEE80211_HT_CAP_SUP_WIDTH		0x0002
+#define IEEE80211_HT_CAP_SM_PS			0x000C
+#define IEEE80211_HT_CAP_GRN_FLD		0x0010
+#define IEEE80211_HT_CAP_SGI_20			0x0020
+#define IEEE80211_HT_CAP_SGI_40			0x0040
+#define IEEE80211_HT_CAP_TX_STBC			0x0080
+#define IEEE80211_HT_CAP_RX_STBC_1R		0x0100
+#define IEEE80211_HT_CAP_RX_STBC_2R		0x0200
+#define IEEE80211_HT_CAP_RX_STBC_3R		0x0300
+#define IEEE80211_HT_CAP_DELAY_BA		0x0400
+#define IEEE80211_HT_CAP_MAX_AMSDU		0x0800
+#define IEEE80211_HT_CAP_DSSSCCK40		0x1000
+#define RTW_IEEE80211_HT_CAP_40MHZ_INTOLERANT	((u16) BIT(14))
+/* 802.11n HT capability AMPDU settings */
+#define IEEE80211_HT_CAP_AMPDU_FACTOR		0x03
+#define IEEE80211_HT_CAP_AMPDU_DENSITY		0x1C
+/* 802.11n HT capability MSC set */
+#define IEEE80211_SUPP_MCS_SET_UEQM		4
+#define IEEE80211_HT_CAP_MAX_STREAMS		4
+#define IEEE80211_SUPP_MCS_SET_LEN		10
+/* maximum streams the spec allows */
+#define IEEE80211_HT_CAP_MCS_TX_DEFINED		0x01
+#define IEEE80211_HT_CAP_MCS_TX_RX_DIFF		0x02
+#define IEEE80211_HT_CAP_MCS_TX_STREAMS		0x0C
+#define IEEE80211_HT_CAP_MCS_TX_UEQM		0x10
+/* 802.11n HT capability TXBF capability */
+#define IEEE80211_HT_CAP_TXBF_RX_NDP		0x00000008
+#define IEEE80211_HT_CAP_TXBF_TX_NDP		0x00000010
+#define IEEE80211_HT_CAP_TXBF_EXPLICIT_COMP_STEERING_CAP	0x00000400
+
+/* 802.11n HT IE masks */
+#define IEEE80211_HT_IE_CHA_SEC_OFFSET		0x03
+#define IEEE80211_HT_IE_CHA_SEC_NONE		0x00
+#define IEEE80211_HT_IE_CHA_SEC_ABOVE		0x01
+#define IEEE80211_HT_IE_CHA_SEC_BELOW		0x03
+#define IEEE80211_HT_IE_CHA_WIDTH		0x04
+#define IEEE80211_HT_IE_HT_PROTECTION		0x0003
+#define IEEE80211_HT_IE_NON_GF_STA_PRSNT	0x0004
+#define IEEE80211_HT_IE_NON_HT_STA_PRSNT	0x0010
+
+/* block-ack parameters */
+#define IEEE80211_ADDBA_PARAM_POLICY_MASK 0x0002
+#define IEEE80211_ADDBA_PARAM_TID_MASK 0x003C
+#define RTW_IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK 0xFFC0
+#define IEEE80211_DELBA_PARAM_TID_MASK 0xF000
+#define IEEE80211_DELBA_PARAM_INITIATOR_MASK 0x0800
+
+#define OP_MODE_PURE                    0
+#define OP_MODE_MAY_BE_LEGACY_STAS      1
+#define OP_MODE_20MHZ_HT_STA_ASSOCED    2
+#define OP_MODE_MIXED                   3
+
+#define HT_INFO_HT_PARAM_SECONDARY_CHNL_OFF_MASK	((u8) BIT(0) | BIT(1))
+#define HT_INFO_HT_PARAM_SECONDARY_CHNL_ABOVE		((u8) BIT(0))
+#define HT_INFO_HT_PARAM_SECONDARY_CHNL_BELOW		((u8) BIT(0) | BIT(1))
+#define HT_INFO_HT_PARAM_REC_TRANS_CHNL_WIDTH		((u8) BIT(2))
+#define HT_INFO_HT_PARAM_RIFS_MODE			((u8) BIT(3))
+#define HT_INFO_HT_PARAM_CTRL_ACCESS_ONLY		((u8) BIT(4))
+#define HT_INFO_HT_PARAM_SRV_INTERVAL_GRANULARITY	((u8) BIT(5))
+
+#define HT_INFO_OPERATION_MODE_OP_MODE_MASK	\
+	((u16) (0x0001 | 0x0002))
+#define HT_INFO_OPERATION_MODE_OP_MODE_OFFSET		0
+#define HT_INFO_OPERATION_MODE_NON_GF_DEVS_PRESENT	((u8) BIT(2))
+#define HT_INFO_OPERATION_MODE_TRANSMIT_BURST_LIMIT	((u8) BIT(3))
+#define HT_INFO_OPERATION_MODE_NON_HT_STA_PRESENT	((u8) BIT(4))
+
+#define HT_INFO_STBC_PARAM_DUAL_BEACON			((u16) BIT(6))
+#define HT_INFO_STBC_PARAM_DUAL_STBC_PROTECT		((u16) BIT(7))
+#define HT_INFO_STBC_PARAM_SECONDARY_BCN		((u16) BIT(8))
+#define HT_INFO_STBC_PARAM_LSIG_TXOP_PROTECT_ALLOWED	((u16) BIT(9))
+#define HT_INFO_STBC_PARAM_PCO_ACTIVE			((u16) BIT(10))
+#define HT_INFO_STBC_PARAM_PCO_PHASE			((u16) BIT(11))
+
+
+
+/* #endif */
+
+/*	===============WPS Section=============== */
+/*	For WPSv1.0 */
+#define WPSOUI							0x0050f204
+/*	WPS attribute ID */
+#define WPS_ATTR_VER1					0x104A
+#define WPS_ATTR_SIMPLE_CONF_STATE	0x1044
+#define WPS_ATTR_RESP_TYPE			0x103B
+#define WPS_ATTR_UUID_E				0x1047
+#define WPS_ATTR_MANUFACTURER		0x1021
+#define WPS_ATTR_MODEL_NAME			0x1023
+#define WPS_ATTR_MODEL_NUMBER		0x1024
+#define WPS_ATTR_SERIAL_NUMBER		0x1042
+#define WPS_ATTR_PRIMARY_DEV_TYPE	0x1054
+#define WPS_ATTR_SEC_DEV_TYPE_LIST	0x1055
+#define WPS_ATTR_DEVICE_NAME			0x1011
+#define WPS_ATTR_CONF_METHOD			0x1008
+#define WPS_ATTR_RF_BANDS				0x103C
+#define WPS_ATTR_DEVICE_PWID			0x1012
+#define WPS_ATTR_REQUEST_TYPE			0x103A
+#define WPS_ATTR_ASSOCIATION_STATE	0x1002
+#define WPS_ATTR_CONFIG_ERROR			0x1009
+#define WPS_ATTR_VENDOR_EXT			0x1049
+#define WPS_ATTR_SELECTED_REGISTRAR	0x1041
+
+/*	Value of WPS attribute "WPS_ATTR_DEVICE_NAME */
+#define WPS_MAX_DEVICE_NAME_LEN		32
+
+/*	Value of WPS Request Type Attribute */
+#define WPS_REQ_TYPE_ENROLLEE_INFO_ONLY			0x00
+#define WPS_REQ_TYPE_ENROLLEE_OPEN_8021X		0x01
+#define WPS_REQ_TYPE_REGISTRAR					0x02
+#define WPS_REQ_TYPE_WLAN_MANAGER_REGISTRAR	0x03
+
+/*	Value of WPS Response Type Attribute */
+#define WPS_RESPONSE_TYPE_INFO_ONLY	0x00
+#define WPS_RESPONSE_TYPE_8021X		0x01
+#define WPS_RESPONSE_TYPE_REGISTRAR	0x02
+#define WPS_RESPONSE_TYPE_AP			0x03
+
+/*	Value of WPS WiFi Simple Configuration State Attribute */
+#define WPS_WSC_STATE_NOT_CONFIG	0x01
+#define WPS_WSC_STATE_CONFIG			0x02
+
+/*	Value of WPS Version Attribute */
+#define WPS_VERSION_1					0x10
+
+/*	Value of WPS Configuration Method Attribute */
+#define WPS_CONFIG_METHOD_FLASH		0x0001
+#define WPS_CONFIG_METHOD_ETHERNET	0x0002
+#define WPS_CONFIG_METHOD_LABEL		0x0004
+#define WPS_CONFIG_METHOD_DISPLAY	0x0008
+#define WPS_CONFIG_METHOD_E_NFC		0x0010
+#define WPS_CONFIG_METHOD_I_NFC		0x0020
+#define WPS_CONFIG_METHOD_NFC		0x0040
+#define WPS_CONFIG_METHOD_PBC		0x0080
+#define WPS_CONFIG_METHOD_KEYPAD	0x0100
+#define WPS_CONFIG_METHOD_VPBC		0x0280
+#define WPS_CONFIG_METHOD_PPBC		0x0480
+#define WPS_CONFIG_METHOD_VDISPLAY	0x2008
+#define WPS_CONFIG_METHOD_PDISPLAY	0x4008
+
+/*	Value of Category ID of WPS Primary Device Type Attribute */
+#define WPS_PDT_CID_DISPLAYS			0x0007
+#define WPS_PDT_CID_MULIT_MEDIA		0x0008
+#define WPS_PDT_CID_RTK_WIDI			WPS_PDT_CID_MULIT_MEDIA
+
+/*	Value of Sub Category ID of WPS Primary Device Type Attribute */
+#define WPS_PDT_SCID_MEDIA_SERVER	0x0005
+#define WPS_PDT_SCID_RTK_DMP			WPS_PDT_SCID_MEDIA_SERVER
+
+/*	Value of Device Password ID */
+#define WPS_DPID_PIN					0x0000
+#define WPS_DPID_USER_SPEC			0x0001
+#define WPS_DPID_MACHINE_SPEC			0x0002
+#define WPS_DPID_REKEY					0x0003
+#define WPS_DPID_PBC					0x0004
+#define WPS_DPID_REGISTRAR_SPEC		0x0005
+
+/*	Value of WPS RF Bands Attribute */
+#define WPS_RF_BANDS_2_4_GHZ		0x01
+#define WPS_RF_BANDS_5_GHZ		0x02
+
+/*	Value of WPS Association State Attribute */
+#define WPS_ASSOC_STATE_NOT_ASSOCIATED			0x00
+#define WPS_ASSOC_STATE_CONNECTION_SUCCESS		0x01
+#define WPS_ASSOC_STATE_CONFIGURATION_FAILURE	0x02
+#define WPS_ASSOC_STATE_ASSOCIATION_FAILURE		0x03
+#define WPS_ASSOC_STATE_IP_FAILURE				0x04
+
+/*	=====================P2P Section===================== */
+/*	For P2P */
+#define	P2POUI							0x506F9A09
+
+/*	P2P Attribute ID */
+#define	P2P_ATTR_STATUS					0x00
+#define	P2P_ATTR_MINOR_REASON_CODE		0x01
+#define	P2P_ATTR_CAPABILITY				0x02
+#define	P2P_ATTR_DEVICE_ID				0x03
+#define	P2P_ATTR_GO_INTENT				0x04
+#define	P2P_ATTR_CONF_TIMEOUT			0x05
+#define	P2P_ATTR_LISTEN_CH				0x06
+#define	P2P_ATTR_GROUP_BSSID				0x07
+#define	P2P_ATTR_EX_LISTEN_TIMING		0x08
+#define	P2P_ATTR_INTENDED_IF_ADDR		0x09
+#define	P2P_ATTR_MANAGEABILITY			0x0A
+#define	P2P_ATTR_CH_LIST					0x0B
+#define	P2P_ATTR_NOA						0x0C
+#define	P2P_ATTR_DEVICE_INFO				0x0D
+#define	P2P_ATTR_GROUP_INFO				0x0E
+#define	P2P_ATTR_GROUP_ID					0x0F
+#define	P2P_ATTR_INTERFACE				0x10
+#define	P2P_ATTR_OPERATING_CH			0x11
+#define	P2P_ATTR_INVITATION_FLAGS		0x12
+
+/*	Value of Status Attribute */
+#define	P2P_STATUS_SUCCESS						0x00
+#define	P2P_STATUS_FAIL_INFO_UNAVAILABLE		0x01
+#define	P2P_STATUS_FAIL_INCOMPATIBLE_PARAM		0x02
+#define	P2P_STATUS_FAIL_LIMIT_REACHED			0x03
+#define	P2P_STATUS_FAIL_INVALID_PARAM			0x04
+#define	P2P_STATUS_FAIL_REQUEST_UNABLE			0x05
+#define	P2P_STATUS_FAIL_PREVOUS_PROTO_ERR		0x06
+#define	P2P_STATUS_FAIL_NO_COMMON_CH			0x07
+#define	P2P_STATUS_FAIL_UNKNOWN_P2PGROUP		0x08
+#define	P2P_STATUS_FAIL_BOTH_GOINTENT_15		0x09
+#define	P2P_STATUS_FAIL_INCOMPATIBLE_PROVSION	0x0A
+#define	P2P_STATUS_FAIL_USER_REJECT				0x0B
+
+/*	Value of Inviation Flags Attribute */
+#define	P2P_INVITATION_FLAGS_PERSISTENT			BIT(0)
+
+#define	DMP_P2P_DEVCAP_SUPPORT	(P2P_DEVCAP_SERVICE_DISCOVERY | \
+				 P2P_DEVCAP_CLIENT_DISCOVERABILITY | \
+				 P2P_DEVCAP_CONCURRENT_OPERATION | \
+				 P2P_DEVCAP_INVITATION_PROC)
+
+#define	DMP_P2P_GRPCAP_SUPPORT	(P2P_GRPCAP_INTRABSS)
+
+/*	Value of Device Capability Bitmap */
+#define	P2P_DEVCAP_SERVICE_DISCOVERY		BIT(0)
+#define	P2P_DEVCAP_CLIENT_DISCOVERABILITY	BIT(1)
+#define	P2P_DEVCAP_CONCURRENT_OPERATION	BIT(2)
+#define	P2P_DEVCAP_INFRA_MANAGED			BIT(3)
+#define	P2P_DEVCAP_DEVICE_LIMIT				BIT(4)
+#define	P2P_DEVCAP_INVITATION_PROC			BIT(5)
+
+/*	Value of Group Capability Bitmap */
+#define	P2P_GRPCAP_GO							BIT(0)
+#define	P2P_GRPCAP_PERSISTENT_GROUP			BIT(1)
+#define	P2P_GRPCAP_GROUP_LIMIT				BIT(2)
+#define	P2P_GRPCAP_INTRABSS					BIT(3)
+#define	P2P_GRPCAP_CROSS_CONN				BIT(4)
+#define	P2P_GRPCAP_PERSISTENT_RECONN		BIT(5)
+#define	P2P_GRPCAP_GROUP_FORMATION			BIT(6)
+
+/*	P2P Public Action Frame ( Management Frame ) */
+#define	P2P_PUB_ACTION_ACTION				0x09
+
+/*	P2P Public Action Frame Type */
+#define	P2P_GO_NEGO_REQ						0
+#define	P2P_GO_NEGO_RESP						1
+#define	P2P_GO_NEGO_CONF						2
+#define	P2P_INVIT_REQ							3
+#define	P2P_INVIT_RESP							4
+#define	P2P_DEVDISC_REQ						5
+#define	P2P_DEVDISC_RESP						6
+#define	P2P_PROVISION_DISC_REQ				7
+#define	P2P_PROVISION_DISC_RESP				8
+
+/*	P2P Action Frame Type */
+#define	P2P_NOTICE_OF_ABSENCE	0
+#define	P2P_PRESENCE_REQUEST		1
+#define	P2P_PRESENCE_RESPONSE	2
+#define	P2P_GO_DISC_REQUEST		3
+
+
+#define	P2P_MAX_PERSISTENT_GROUP_NUM		10
+
+#define	P2P_PROVISIONING_SCAN_CNT			3
+
+#define	P2P_WILDCARD_SSID_LEN				7
+
+#define	P2P_FINDPHASE_EX_NONE				0	/* default value, used when: (1)p2p disabed or (2)p2p enabled but only do 1 scan phase */
+#define	P2P_FINDPHASE_EX_FULL				1	/* used when p2p enabled and want to do 1 scan phase and P2P_FINDPHASE_EX_MAX-1 find phase */
+#define	P2P_FINDPHASE_EX_SOCIAL_FIRST		(P2P_FINDPHASE_EX_FULL+1)
+#define	P2P_FINDPHASE_EX_MAX					4
+#define	P2P_FINDPHASE_EX_SOCIAL_LAST		P2P_FINDPHASE_EX_MAX
+
+#define	P2P_PROVISION_TIMEOUT				5000	/*	5 seconds timeout for sending the provision discovery request */
+#define	P2P_CONCURRENT_PROVISION_TIMEOUT	3000	/*	3 seconds timeout for sending the provision discovery request under concurrent mode */
+#define	P2P_GO_NEGO_TIMEOUT					5000	/*	5 seconds timeout for receiving the group negotation response */
+#define	P2P_CONCURRENT_GO_NEGO_TIMEOUT		3000	/*	3 seconds timeout for sending the negotiation request under concurrent mode */
+#define	P2P_TX_PRESCAN_TIMEOUT				100		/*	100ms */
+#define	P2P_INVITE_TIMEOUT					5000	/*	5 seconds timeout for sending the invitation request */
+#define	P2P_CONCURRENT_INVITE_TIMEOUT		3000	/*	3 seconds timeout for sending the invitation request under concurrent mode */
+#define	P2P_RESET_SCAN_CH						25000	/*	25 seconds timeout to reset the scan channel (based on channel plan) */
+#define	P2P_MAX_INTENT						15
+
+#define	P2P_MAX_NOA_NUM						2
+
+/*	WPS Configuration Method */
+#define	WPS_CM_NONE							0x0000
+#define	WPS_CM_LABEL							0x0004
+#define	WPS_CM_DISPLYA						0x0008
+#define	WPS_CM_EXTERNAL_NFC_TOKEN			0x0010
+#define	WPS_CM_INTEGRATED_NFC_TOKEN		0x0020
+#define	WPS_CM_NFC_INTERFACE					0x0040
+#define	WPS_CM_PUSH_BUTTON					0x0080
+#define	WPS_CM_KEYPAD						0x0100
+#define	WPS_CM_SW_PUHS_BUTTON				0x0280
+#define	WPS_CM_HW_PUHS_BUTTON				0x0480
+#define	WPS_CM_SW_DISPLAY_PIN				0x2008
+#define	WPS_CM_LCD_DISPLAY_PIN				0x4008
+
+enum P2P_ROLE {
+	P2P_ROLE_DISABLE = 0,
+	P2P_ROLE_DEVICE = 1,
+	P2P_ROLE_CLIENT = 2,
+	P2P_ROLE_GO = 3
+};
+
+enum P2P_WPSINFO {
+	P2P_NO_WPSINFO						= 0,
+	P2P_GOT_WPSINFO_PEER_DISPLAY_PIN	= 1,
+	P2P_GOT_WPSINFO_SELF_DISPLAY_PIN	= 2,
+	P2P_GOT_WPSINFO_PBC					= 3,
+};
+
+#define	P2P_PRIVATE_IOCTL_SET_LEN		64
+
+#ifdef CONFIG_P2P_PS
+enum P2P_PS_STATE {
+	P2P_PS_DISABLE = 0,
+	P2P_PS_ENABLE = 1,
+	P2P_PS_SCAN = 2,
+	P2P_PS_SCAN_DONE = 3,
+	P2P_PS_ALLSTASLEEP = 4, /* for P2P GO */
+};
+
+enum P2P_PS_MODE {
+	P2P_PS_NONE = 0,
+	P2P_PS_CTWINDOW = 1,
+	P2P_PS_NOA	 = 2,
+	P2P_PS_MIX = 3, /* CTWindow and NoA */
+};
+#endif /* CONFIG_P2P_PS */
+
+/*	=====================WFD Section=====================
+ *	For Wi-Fi Display */
+#define	WFD_ATTR_DEVICE_INFO			0x00
+#define	WFD_ATTR_ASSOC_BSSID			0x01
+#define	WFD_ATTR_COUPLED_SINK_INFO	0x06
+#define	WFD_ATTR_LOCAL_IP_ADDR		0x08
+#define	WFD_ATTR_SESSION_INFO		0x09
+#define	WFD_ATTR_ALTER_MAC			0x0a
+
+/*	For WFD Device Information Attribute */
+#define	WFD_DEVINFO_SOURCE					0x0000
+#define	WFD_DEVINFO_PSINK					0x0001
+#define	WFD_DEVINFO_SSINK					0x0002
+#define	WFD_DEVINFO_DUAL					0x0003
+
+#define	WFD_DEVINFO_SESSION_AVAIL			0x0010
+#define	WFD_DEVINFO_WSD						0x0040
+#define	WFD_DEVINFO_PC_TDLS					0x0080
+#define	WFD_DEVINFO_HDCP_SUPPORT			0x0100
+
+#define IP_MCAST_MAC(mac)		((mac[0] == 0x01) && (mac[1] == 0x00) && (mac[2] == 0x5e))
+#define ICMPV6_MCAST_MAC(mac)	((mac[0] == 0x33) && (mac[1] == 0x33) && (mac[2] != 0xff))
+
+#ifdef CONFIG_IOCTL_CFG80211
+/* Regulatroy Domain */
+struct regd_pair_mapping {
+	u16 reg_dmnenum;
+	u16 reg_5ghz_ctl;
+	u16 reg_2ghz_ctl;
+};
+
+struct rtw_regulatory {
+	char alpha2[2];
+	u16 country_code;
+	u16 max_power_level;
+	u32 tp_scale;
+	u16 current_rd;
+	u16 current_rd_ext;
+	int16_t power_limit;
+	struct regd_pair_mapping *regpair;
+};
+#endif
+
+#ifdef CONFIG_WAPI_SUPPORT
+#ifndef IW_AUTH_WAPI_VERSION_1
+#define IW_AUTH_WAPI_VERSION_1		0x00000008
+#endif
+#ifndef IW_AUTH_KEY_MGMT_WAPI_PSK
+#define IW_AUTH_KEY_MGMT_WAPI_PSK	0x04
+#endif
+#ifndef IW_AUTH_WAPI_ENABLED
+#define IW_AUTH_WAPI_ENABLED		0x20
+#endif
+#ifndef IW_ENCODE_ALG_SM4
+#define IW_ENCODE_ALG_SM4			0x20
+#endif
+#endif
+
+#define GET_MBSSID_MAX_BSSID_INDOCATOR(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 0, 8)
+
+#define MBSSID_MAX_BSSID_INDICATOR_OFFSET 3
+
+#define GET_MULTIPLE_BSSID_IDX_INDEX(_pEleStart) \
+	LE_BITS_TO_1BYTE((_pEleStart) + 2, 0, 8)
+
+#endif /* _WIFI_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/wlan_bssdef.h b/drivers/net/wireless/realtek/rtl8852bs/include/wlan_bssdef.h
new file mode 100644
index 000000000000..69f6699241d8
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/wlan_bssdef.h
@@ -0,0 +1,328 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __WLAN_BSSDEF_H__
+#define __WLAN_BSSDEF_H__
+
+
+#define MAX_IE_SZ	768
+
+
+#ifdef PLATFORM_LINUX
+
+#define NDIS_802_11_LENGTH_SSID         32
+#define NDIS_802_11_LENGTH_RATES        8
+#define NDIS_802_11_LENGTH_RATES_EX     16
+
+typedef unsigned char   NDIS_802_11_MAC_ADDRESS[ETH_ALEN];
+typedef long    		NDIS_802_11_RSSI;           /* in dBm */
+typedef unsigned char   NDIS_802_11_RATES[NDIS_802_11_LENGTH_RATES];        /* Set of 8 data rates */
+typedef unsigned char   NDIS_802_11_RATES_EX[NDIS_802_11_LENGTH_RATES_EX];  /* Set of 16 data rates */
+
+typedef struct _NDIS_802_11_SSID {
+	u32  SsidLength;
+	u8  Ssid[32];
+} NDIS_802_11_SSID, *PNDIS_802_11_SSID;
+
+/*
+	FW will only save the channel number in DSConfig.
+	ODI Handler will convert the channel number to freq. number.
+*/
+typedef struct _NDIS_802_11_CONFIGURATION {
+	u32	Length;		/* Length of structure */
+	u32	BeaconPeriod;	/* units are Kusec */
+	u32	ATIMWindow;	/* units are Kusec */
+	u32	Band;		/* wifi band */
+	u32	DSConfig;	/* channel number */
+} NDIS_802_11_CONFIGURATION, *PNDIS_802_11_CONFIGURATION;
+
+typedef enum _NDIS_802_11_NETWORK_INFRASTRUCTURE {
+	Ndis802_11IBSS,
+	Ndis802_11Infrastructure,
+	Ndis802_11AutoUnknown,
+	Ndis802_11InfrastructureMax,     /* Not a real value, defined as upper bound */
+	Ndis802_11APMode,
+	Ndis802_11Monitor,
+	Ndis802_11_mesh,
+} NDIS_802_11_NETWORK_INFRASTRUCTURE, *PNDIS_802_11_NETWORK_INFRASTRUCTURE;
+
+
+typedef struct _NDIS_802_11_VARIABLE_IEs {
+	u8  ElementID;
+	u8  Length;
+	DECLARE_FLEX_ARRAY(u8, data);
+} NDIS_802_11_VARIABLE_IEs, *PNDIS_802_11_VARIABLE_IEs;
+
+typedef enum _NDIS_802_11_AUTHENTICATION_MODE {
+	Ndis802_11AuthModeOpen,
+	Ndis802_11AuthModeShared,
+	Ndis802_11AuthModeAutoSwitch,
+	Ndis802_11AuthModeWPA,
+	Ndis802_11AuthModeWPAPSK,
+	Ndis802_11AuthModeWPANone,
+	Ndis802_11AuthModeWAPI,
+	Ndis802_11AuthModeMax               /* Not a real mode, defined as upper bound */
+} NDIS_802_11_AUTHENTICATION_MODE, *PNDIS_802_11_AUTHENTICATION_MODE;
+
+typedef enum _NDIS_802_11_WEP_STATUS {
+	Ndis802_11WEPEnabled,
+	Ndis802_11Encryption1Enabled = Ndis802_11WEPEnabled,
+	Ndis802_11WEPDisabled,
+	Ndis802_11EncryptionDisabled = Ndis802_11WEPDisabled,
+	Ndis802_11WEPKeyAbsent,
+	Ndis802_11Encryption1KeyAbsent = Ndis802_11WEPKeyAbsent,
+	Ndis802_11WEPNotSupported,
+	Ndis802_11EncryptionNotSupported = Ndis802_11WEPNotSupported,
+	Ndis802_11Encryption2Enabled,
+	Ndis802_11Encryption2KeyAbsent,
+	Ndis802_11Encryption3Enabled,
+	Ndis802_11Encryption3KeyAbsent,
+	Ndis802_11_EncrypteionWAPI
+} NDIS_802_11_WEP_STATUS, *PNDIS_802_11_WEP_STATUS,
+NDIS_802_11_ENCRYPTION_STATUS, *PNDIS_802_11_ENCRYPTION_STATUS;
+
+typedef struct _NDIS_802_11_WEP {
+	u32     Length;        /* Length of this structure */
+	u32     KeyIndex;      /* 0 is the per-client key, 1-N are the global keys */
+	u32     KeyLength;     /* length of key in bytes */
+	u8     KeyMaterial[16];/* variable length depending on above field */
+} NDIS_802_11_WEP, *PNDIS_802_11_WEP;
+
+#endif /* end of #ifdef PLATFORM_LINUX */
+
+#ifdef PLATFORM_FREEBSD
+
+#define NDIS_802_11_LENGTH_SSID         32
+#define NDIS_802_11_LENGTH_RATES        8
+#define NDIS_802_11_LENGTH_RATES_EX     16
+
+typedef unsigned char   NDIS_802_11_MAC_ADDRESS[ETH_ALEN];
+typedef long    		NDIS_802_11_RSSI;           /* in dBm */
+typedef unsigned char   NDIS_802_11_RATES[NDIS_802_11_LENGTH_RATES];        /* Set of 8 data rates */
+typedef unsigned char   NDIS_802_11_RATES_EX[NDIS_802_11_LENGTH_RATES_EX];  /* Set of 16 data rates */
+
+
+typedef struct _NDIS_802_11_SSID {
+	u32  SsidLength;
+	u8  Ssid[32];
+} NDIS_802_11_SSID, *PNDIS_802_11_SSID;
+
+/*
+	FW will only save the channel number in DSConfig.
+	ODI Handler will convert the channel number to freq. number.
+*/
+typedef struct _NDIS_802_11_CONFIGURATION {
+	u32           Length;             /* Length of structure */
+	u32           BeaconPeriod;       /* units are Kusec */
+	u32           ATIMWindow;         /* units are Kusec */
+	u32           DSConfig;           /* channel number */
+} NDIS_802_11_CONFIGURATION, *PNDIS_802_11_CONFIGURATION;
+
+typedef enum _NDIS_802_11_NETWORK_INFRASTRUCTURE {
+	Ndis802_11IBSS,
+	Ndis802_11Infrastructure,
+	Ndis802_11AutoUnknown,
+	Ndis802_11InfrastructureMax,     /* Not a real value, defined as upper bound */
+	Ndis802_11APMode
+} NDIS_802_11_NETWORK_INFRASTRUCTURE, *PNDIS_802_11_NETWORK_INFRASTRUCTURE;
+
+typedef struct _NDIS_802_11_VARIABLE_IEs {
+	u8  ElementID;
+	u8  Length;
+	DECLARE_FLEX_ARRAY(u8, data);
+} NDIS_802_11_VARIABLE_IEs, *PNDIS_802_11_VARIABLE_IEs;
+
+typedef enum _NDIS_802_11_AUTHENTICATION_MODE {
+	Ndis802_11AuthModeOpen,
+	Ndis802_11AuthModeShared,
+	Ndis802_11AuthModeAutoSwitch,
+	Ndis802_11AuthModeWPA,
+	Ndis802_11AuthModeWPAPSK,
+	Ndis802_11AuthModeWPANone,
+	Ndis802_11AuthModeMax               /* Not a real mode, defined as upper bound */
+} NDIS_802_11_AUTHENTICATION_MODE, *PNDIS_802_11_AUTHENTICATION_MODE;
+
+typedef enum _NDIS_802_11_WEP_STATUS {
+	Ndis802_11WEPEnabled,
+	Ndis802_11Encryption1Enabled = Ndis802_11WEPEnabled,
+	Ndis802_11WEPDisabled,
+	Ndis802_11EncryptionDisabled = Ndis802_11WEPDisabled,
+	Ndis802_11WEPKeyAbsent,
+	Ndis802_11Encryption1KeyAbsent = Ndis802_11WEPKeyAbsent,
+	Ndis802_11WEPNotSupported,
+	Ndis802_11EncryptionNotSupported = Ndis802_11WEPNotSupported,
+	Ndis802_11Encryption2Enabled,
+	Ndis802_11Encryption2KeyAbsent,
+	Ndis802_11Encryption3Enabled,
+	Ndis802_11Encryption3KeyAbsent
+} NDIS_802_11_WEP_STATUS, *PNDIS_802_11_WEP_STATUS,
+NDIS_802_11_ENCRYPTION_STATUS, *PNDIS_802_11_ENCRYPTION_STATUS;
+
+
+typedef struct _NDIS_802_11_WEP {
+	u32     Length;        /* Length of this structure */
+	u32     KeyIndex;      /* 0 is the per-client key, 1-N are the global keys */
+	u32     KeyLength;     /* length of key in bytes */
+	u8     KeyMaterial[16];/* variable length depending on above field */
+} NDIS_802_11_WEP, *PNDIS_802_11_WEP;
+
+#endif /* PLATFORM_FREEBSD */
+
+
+/*RTW_WKARD_CORE_RSSI_V1 - GEORGIA MUST REFINE*/
+typedef struct _WLAN_PHY_INFO {
+	u8	isValid;
+	u8	SignalStrength;/* (in percentage) */
+	u8	SignalQuality;/* (in percentage) */
+	s8 	rssi; /*dbm*/
+	u8	Optimum_antenna;  /* for Antenna diversity */
+	u8	is_cck_rate;	/* 1:cck_rate */
+	s8	rx_snr[4];
+#if defined (CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	u32	free_cnt; 	/* freerun counter */
+	u8	rm_en_cap[5];
+#endif
+} WLAN_PHY_INFO, *PWLAN_PHY_INFO;
+
+typedef struct _WLAN_BCN_INFO {
+	/* these infor get from rtw_get_encrypt_info when
+	 *	 * translate scan to UI */
+	u8 encryp_protocol;/* ENCRYP_PROTOCOL_E: OPEN/WEP/WPA/WPA2/WAPI */
+	int group_cipher; /* WPA/WPA2 group cipher */
+	int pairwise_cipher;/* //WPA/WPA2/WEP pairwise cipher */
+	int is_8021x;
+
+	/* bwmode 20/40 and ch_offset UP/LOW */
+	unsigned short	ht_cap_info;
+	unsigned char	ht_info_infos_0;
+} WLAN_BCN_INFO, *PWLAN_BCN_INFO;
+
+enum bss_type {
+	BSS_TYPE_UNDEF,
+	BSS_TYPE_PROB_REQ = 1,
+	BSS_TYPE_BCN = 2,
+	BSS_TYPE_PROB_RSP = 3,
+};
+
+struct wlan_mld_network {
+	_list	list;
+	u8 mac_addr[ETH_ALEN];
+	systime last_scanned; /* timestamp for the mld network */
+	struct wlan_network *link_network[RTW_NETWORK_LINK_MAX];
+	u8 network_num;
+};
+
+#define GET_LINK_NETWORK(mld_network, lidx) (((struct wlan_mld_network *) mld_network)->link_network[lidx])
+
+/* temporally add #pragma pack for structure alignment issue of
+*   WLAN_BSSID_EX and get_WLAN_BSSID_EX_sz()
+*/
+typedef struct _WLAN_BSSID_EX {
+	u32  Length;
+	NDIS_802_11_MAC_ADDRESS  MacAddress;
+	u8  Reserved[2];/* [0]: IS beacon frame , bss_type*/
+	NDIS_802_11_SSID  Ssid;
+	NDIS_802_11_SSID  mesh_id;
+	u32  Privacy;
+	NDIS_802_11_CONFIGURATION  Configuration;
+	NDIS_802_11_NETWORK_INFRASTRUCTURE  InfrastructureMode;
+	NDIS_802_11_RATES_EX  SupportedRates;
+	WLAN_PHY_INFO	PhyInfo;
+	u8 is_mld;
+#ifdef CONFIG_80211BE_EHT
+	struct wlan_mld_network *mld_network;
+	u8 link_id; /* parse from ML IEs of AP */
+	u8 is_complete_profile;
+#endif
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+	u8 is_mbssid;/*is_nontx = non-transmitted BSSID*/
+	u8 tx_bssid[ETH_ALEN];/*transmitted BSSID*/
+	u8 mbssid_index;
+#endif
+	u32  IELength;
+	u8  IEs[MAX_IE_SZ];	/* (timestamp, beacon interval, and capability information) */
+}
+__attribute__((packed)) WLAN_BSSID_EX, *PWLAN_BSSID_EX;
+
+#define BSS_EX_OP_CH(bss_ex) ((bss_ex)->Configuration.DSConfig)
+#define BSS_EX_OP_BAND(bss_ex) ((bss_ex)->Configuration.Band)
+#ifdef CONFIG_STA_MULTIPLE_BSSID
+#define BSS_EX_MBSSID_IDX(bss_ex) ((bss_ex)->mbssid_index)
+#else
+#define BSS_EX_MBSSID_IDX(bss_ex) 0
+#endif
+#define BSS_EX_IES(bss_ex) ((bss_ex)->IEs)
+#define BSS_EX_IES_LEN(bss_ex) ((bss_ex)->IELength)
+#define BSS_EX_FIXED_IE_OFFSET(bss_ex) ((bss_ex)->Reserved[0] == BSS_TYPE_PROB_REQ ? 0 : 12)
+#define BSS_EX_TLV_IES(bss_ex) (BSS_EX_IES((bss_ex)) + BSS_EX_FIXED_IE_OFFSET((bss_ex)))
+#define BSS_EX_TLV_IES_LEN(bss_ex) (BSS_EX_IES_LEN((bss_ex)) - BSS_EX_FIXED_IE_OFFSET((bss_ex)))
+
+__inline  static uint get_WLAN_BSSID_EX_sz(WLAN_BSSID_EX *bss)
+{
+	return sizeof(WLAN_BSSID_EX) - MAX_IE_SZ + bss->IELength;
+}
+
+struct beacon_keys {
+	u8 ssid[IW_ESSID_MAX_SIZE];
+	u32 ssid_len;
+	enum band_type band;
+	u8 ch;
+	u8 bw;
+	u8 offset;
+	u8 proto_cap; /* PROTO_CAP_XXX */
+	u8 rate_set[12];
+	u8 rate_num;
+	int encryp_protocol;
+	int pairwise_cipher;
+	int group_cipher;
+	u32 akm;
+};
+
+struct wlan_network;
+
+enum VRTL_CARRIER_SENSE {
+	DISABLE_VCS,
+	ENABLE_VCS,
+	AUTO_VCS
+};
+
+enum VCS_TYPE {
+	NONE_VCS,
+	RTS_CTS,
+	CTS_TO_SELF
+};
+
+
+
+
+#define PWR_CAM 0
+#define PWR_MINPS 1
+#define PWR_MAXPS 2
+#define PWR_UAPSD 3
+#define PWR_VOIP 4
+
+
+enum UAPSD_MAX_SP {
+	NO_LIMIT,
+	TWO_MSDU,
+	FOUR_MSDU,
+	SIX_MSDU
+};
+
+
+/* john */
+#define NUM_PRE_AUTH_KEY 16
+#define NUM_PMKID_CACHE NUM_PRE_AUTH_KEY
+
+#endif /* #ifndef WLAN_BSSDEF_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/include/xmit_osdep.h b/drivers/net/wireless/realtek/rtl8852bs/include/xmit_osdep.h
new file mode 100644
index 000000000000..1a5b39178857
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/include/xmit_osdep.h
@@ -0,0 +1,104 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __XMIT_OSDEP_H_
+#define __XMIT_OSDEP_H_
+
+
+struct pkt_file {
+	struct sk_buff *pkt;
+	SIZE_T pkt_len;	 /* the remainder length of the open_file */
+	_buffer *cur_buffer;
+	u8 *buf_start;
+	u8 *cur_addr;
+	SIZE_T buf_len;
+};
+#ifndef NR_XMITFRAME
+#define NR_XMITFRAME		1256
+#endif /*NR_XMITFRAME*/
+
+#ifndef NR_XMITFRAME_EXT
+#define NR_XMITFRAME_EXT	32
+#endif /*NR_XMITFRAME_EXT*/
+
+#define SZ_XMITFRAME_EXT	1536	/*MGNT frame*/
+
+#ifdef CONFIG_PCI_HCI
+	#define SZ_ALIGN_XMITFRAME_EXT	4
+#else
+	#ifdef USB_XMITBUF_ALIGN_SZ
+		#define SZ_ALIGN_XMITFRAME_EXT (USB_XMITBUF_ALIGN_SZ)
+	#else
+		#define SZ_ALIGN_XMITFRAME_EXT 512
+	#endif
+#endif
+
+
+struct xmit_priv;
+struct pkt_attrib;
+struct sta_xmit_priv;
+struct xmit_frame;
+struct xmit_buf;
+
+
+#ifdef PLATFORM_FREEBSD
+extern int rtw_xmit_entry(struct sk_buff *pkt, _nic_hdl pnetdev);
+extern void rtw_xmit_entry_wrap(_nic_hdl pifp);
+#endif /* PLATFORM_FREEBSD */
+
+#ifdef PLATFORM_LINUX
+extern int _rtw_xmit_entry(struct sk_buff *pkt, _nic_hdl pnetdev);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32))
+extern netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, _nic_hdl pnetdev);
+#else
+extern int rtw_xmit_entry(struct sk_buff *pkt, _nic_hdl pnetdev);
+#endif
+
+#ifdef RTW_PHL_TX
+extern int rtw_os_tx(struct sk_buff *pkt, _nic_hdl pnetdev);
+#endif
+#endif /* PLATFORM_LINUX */
+
+void rtw_os_xmit_schedule(_adapter *padapter);
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+int rtw_os_xmit_resource_alloc(_adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz, u8 flag);
+void rtw_os_xmit_resource_free(_adapter *padapter, struct xmit_buf *pxmitbuf, u32 free_sz, u8 flag);
+#else
+u8 rtw_os_xmit_resource_alloc(_adapter *padapter, struct xmit_frame *pxframe);
+void rtw_os_xmit_resource_free(_adapter *padapter, struct xmit_frame *pxframe);
+#endif
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+void rtw_set_tx_chksum_offload(struct sk_buff *pkt, struct pkt_attrib *pattrib);
+#endif
+
+extern uint rtw_remainder_len(struct pkt_file *pfile);
+extern void _rtw_open_pktfile(struct sk_buff *pkt, struct pkt_file *pfile);
+extern uint _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, uint rlen);
+extern sint rtw_endofpktfile(struct pkt_file *pfile);
+
+extern void rtw_os_pkt_complete(_adapter *padapter, struct sk_buff *pkt);
+extern void rtw_os_xmit_complete(_adapter *padapter, struct xmit_frame *pxframe);
+
+void rtw_os_check_wakup_queue(_adapter *padapter, u16 os_qid);
+bool rtw_os_check_stop_queue(_adapter *padapter, u16 os_qid);
+void rtw_os_wake_queue_at_free_stainfo(_adapter *padapter, int *qcnt_freed);
+
+void dump_os_queue(void *sel, _adapter *padapter);
+
+void rtw_coalesce_tx_amsdu(_adapter *padapter, struct xmit_frame *pxframes[],
+			   int xf_nr, bool amsdu, u32 *pktlen);
+
+#endif /* __XMIT_OSDEP_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/custom_gpio_linux.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/custom_gpio_linux.c
new file mode 100644
index 000000000000..c95bbedfa427
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/custom_gpio_linux.c
@@ -0,0 +1,245 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include "drv_types.h"
+
+#ifdef CONFIG_PLATFORM_SPRD
+
+/* gspi func & GPIO define */
+#include <mach/gpio.h>/* 0915 */
+#include <mach/board.h>
+
+#if !(defined ANDROID_2X)
+
+#ifndef GPIO_WIFI_POWER
+#define GPIO_WIFI_POWER -1
+#endif /* !GPIO_WIFI_POWER */
+
+#ifndef GPIO_WIFI_RESET
+#define GPIO_WIFI_RESET -1
+#endif /* !GPIO_WIFI_RESET */
+
+#ifndef GPIO_WIFI_PWDN
+#define GPIO_WIFI_PWDN -1
+#endif /* !GPIO_WIFI_RESET */
+#ifdef CONFIG_GSPI_HCI
+extern unsigned int oob_irq;
+#endif /* CONFIG_GSPI_HCI */
+
+#ifdef CONFIG_SDIO_HCI
+extern int rtw_mp_mode;
+#else /* !CONFIG_SDIO_HCI */
+#endif /* !CONFIG_SDIO_HCI */
+
+int rtw_wifi_gpio_init(void)
+{
+#ifdef CONFIG_GSPI_HCI
+	if (GPIO_WIFI_IRQ > 0) {
+		gpio_request(GPIO_WIFI_IRQ, "oob_irq");
+		gpio_direction_input(GPIO_WIFI_IRQ);
+
+		oob_irq = gpio_to_irq(GPIO_WIFI_IRQ);
+
+		RTW_INFO("%s oob_irq:%d\n", __func__, oob_irq);
+	}
+#endif
+	if (GPIO_WIFI_RESET > 0)
+		gpio_request(GPIO_WIFI_RESET , "wifi_rst");
+	if (GPIO_WIFI_POWER > 0)
+		gpio_request(GPIO_WIFI_POWER, "wifi_power");
+
+	return 0;
+}
+
+int rtw_wifi_gpio_deinit(void)
+{
+#ifdef CONFIG_GSPI_HCI
+	if (GPIO_WIFI_IRQ > 0)
+		gpio_free(GPIO_WIFI_IRQ);
+#endif
+	if (GPIO_WIFI_RESET > 0)
+		gpio_free(GPIO_WIFI_RESET);
+	if (GPIO_WIFI_POWER > 0)
+		gpio_free(GPIO_WIFI_POWER);
+	return 0;
+}
+
+/* Customer function to control hw specific wlan gpios */
+void rtw_wifi_gpio_wlan_ctrl(int onoff)
+{
+	switch (onoff) {
+	case WLAN_PWDN_OFF:
+		RTW_INFO("%s: call customer specific GPIO(%d) to set wifi power down pin to 0\n",
+			 __FUNCTION__, GPIO_WIFI_RESET);
+
+#ifndef CONFIG_DONT_BUS_SCAN
+		if (GPIO_WIFI_RESET > 0)
+			gpio_direction_output(GPIO_WIFI_RESET , 0);
+#endif
+		break;
+
+	case WLAN_PWDN_ON:
+		RTW_INFO("%s: callc customer specific GPIO(%d) to set wifi power down pin to 1\n",
+			 __FUNCTION__, GPIO_WIFI_RESET);
+
+		if (GPIO_WIFI_RESET > 0)
+			gpio_direction_output(GPIO_WIFI_RESET , 1);
+		break;
+
+	case WLAN_POWER_OFF:
+		break;
+
+	case WLAN_POWER_ON:
+		break;
+	}
+}
+
+#else /* ANDROID_2X */
+
+#include <mach/ldo.h>
+
+extern int sprd_3rdparty_gpio_wifi_pwd;
+
+int rtw_wifi_gpio_init(void)
+{
+	return 0;
+}
+
+int rtw_wifi_gpio_deinit(void)
+{
+	return 0;
+}
+
+/* Customer function to control hw specific wlan gpios */
+void rtw_wifi_gpio_wlan_ctrl(int onoff)
+{
+	switch (onoff) {
+	case WLAN_PWDN_OFF:
+		RTW_INFO("%s: call customer specific GPIO to set wifi power down pin to 0\n",
+			 __FUNCTION__);
+		if (sprd_3rdparty_gpio_wifi_pwd > 0)
+			gpio_set_value(sprd_3rdparty_gpio_wifi_pwd, 0);
+
+		if (sprd_3rdparty_gpio_wifi_pwd == 60) {
+			RTW_INFO("%s: turn off VSIM2 2.8V\n", __func__);
+			LDO_TurnOffLDO(LDO_LDO_SIM2);
+		}
+		break;
+
+	case WLAN_PWDN_ON:
+		RTW_INFO("%s: callc customer specific GPIO to set wifi power down pin to 1\n",
+			 __FUNCTION__);
+		if (sprd_3rdparty_gpio_wifi_pwd == 60) {
+			RTW_INFO("%s: turn on VSIM2 2.8V\n", __func__);
+			LDO_SetVoltLevel(LDO_LDO_SIM2, LDO_VOLT_LEVEL0);
+			LDO_TurnOnLDO(LDO_LDO_SIM2);
+		}
+		if (sprd_3rdparty_gpio_wifi_pwd > 0)
+			gpio_set_value(sprd_3rdparty_gpio_wifi_pwd, 1);
+		break;
+
+	case WLAN_POWER_OFF:
+		break;
+
+	case WLAN_POWER_ON:
+		break;
+
+	case WLAN_BT_PWDN_OFF:
+		RTW_INFO("%s: call customer specific GPIO to set bt power down pin to 0\n",
+			 __FUNCTION__);
+		break;
+
+	case WLAN_BT_PWDN_ON:
+		RTW_INFO("%s: callc customer specific GPIO to set bt power down pin to 1\n",
+			 __FUNCTION__);
+		break;
+	}
+}
+#endif /* ANDROID_2X */
+
+#elif defined(CONFIG_PLATFORM_ARM_RK3066)
+#include <mach/iomux.h>
+
+#define GPIO_WIFI_IRQ		RK30_PIN2_PC2
+extern unsigned int oob_irq;
+int rtw_wifi_gpio_init(void)
+{
+#ifdef CONFIG_GSPI_HCI
+	if (GPIO_WIFI_IRQ > 0) {
+		rk30_mux_api_set(GPIO2C2_LCDC1DATA18_SMCBLSN1_HSADCDATA5_NAME, GPIO2C_GPIO2C2);/* jacky_test */
+		gpio_request(GPIO_WIFI_IRQ, "oob_irq");
+		gpio_direction_input(GPIO_WIFI_IRQ);
+
+		oob_irq = gpio_to_irq(GPIO_WIFI_IRQ);
+
+		RTW_INFO("%s oob_irq:%d\n", __func__, oob_irq);
+	}
+#endif
+	return 0;
+}
+
+
+int rtw_wifi_gpio_deinit(void)
+{
+#ifdef CONFIG_GSPI_HCI
+	if (GPIO_WIFI_IRQ > 0)
+		gpio_free(GPIO_WIFI_IRQ);
+#endif
+	return 0;
+}
+
+void rtw_wifi_gpio_wlan_ctrl(int onoff)
+{
+}
+
+#ifdef CONFIG_GPIO_API
+/* this is a demo for extending GPIO pin[7] as interrupt mode */
+struct net_device *rtl_net;
+extern int rtw_register_gpio_interrupt(struct net_device *netdev, int gpio_num, void(*callback)(u8 level));
+extern int rtw_disable_gpio_interrupt(struct net_device *netdev, int gpio_num);
+void gpio_int(u8 is_high)
+{
+	RTW_INFO("%s level=%d\n", __func__, is_high);
+}
+int register_net_gpio_init(void)
+{
+	rtl_net = dev_get_by_name(&init_net, "wlan0");
+	if (!rtl_net) {
+		RTW_PRINT("rtl_net init fail!\n");
+		return -1;
+	}
+	return rtw_register_gpio_interrupt(rtl_net, 7, gpio_int);
+}
+int unregister_net_gpio_init(void)
+{
+	rtl_net = dev_get_by_name(&init_net, "wlan0");
+	if (!rtl_net) {
+		RTW_PRINT("rtl_net init fail!\n");
+		return -1;
+	}
+	return rtw_disable_gpio_interrupt(rtl_net, 7);
+}
+#endif
+
+#else
+
+int rtw_wifi_gpio_init(void)
+{
+	return 0;
+}
+
+void rtw_wifi_gpio_wlan_ctrl(int onoff)
+{
+}
+#endif /* CONFIG_PLATFORM_SPRD */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_cfg80211.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_cfg80211.c
new file mode 100644
index 000000000000..d4a456cdda04
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_cfg80211.c
@@ -0,0 +1,11611 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define  _IOCTL_CFG80211_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_IOCTL_CFG80211
+
+#ifndef DBG_RTW_CFG80211_STA_PARAM
+#define DBG_RTW_CFG80211_STA_PARAM 0
+#endif
+
+#ifndef DBG_RTW_CFG80211_MESH_CONF
+#define DBG_RTW_CFG80211_MESH_CONF 0
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+#define STATION_INFO_INACTIVE_TIME	BIT(NL80211_STA_INFO_INACTIVE_TIME)
+#define STATION_INFO_RX_BYTES		BIT(NL80211_STA_INFO_RX_BYTES)
+#define STATION_INFO_TX_BYTES		BIT(NL80211_STA_INFO_TX_BYTES)
+#define STATION_INFO_LLID			BIT(NL80211_STA_INFO_LLID)
+#define STATION_INFO_PLID			BIT(NL80211_STA_INFO_PLID)
+#define STATION_INFO_PLINK_STATE	BIT(NL80211_STA_INFO_PLINK_STATE)
+#define STATION_INFO_SIGNAL			BIT(NL80211_STA_INFO_SIGNAL)
+#define STATION_INFO_TX_BITRATE		BIT(NL80211_STA_INFO_TX_BITRATE)
+#define STATION_INFO_RX_PACKETS		BIT(NL80211_STA_INFO_RX_PACKETS)
+#define STATION_INFO_TX_PACKETS		BIT(NL80211_STA_INFO_TX_PACKETS)
+#define STATION_INFO_TX_RETRIES		BIT(NL80211_STA_INFO_TX_RETRIES)
+#define STATION_INFO_TX_FAILED		BIT(NL80211_STA_INFO_TX_FAILED)
+#define STATION_INFO_RX_BITRATE		BIT(NL80211_STA_INFO_RX_BITRATE)
+#define STATION_INFO_LOCAL_PM		BIT(NL80211_STA_INFO_LOCAL_PM)
+#define STATION_INFO_PEER_PM		BIT(NL80211_STA_INFO_PEER_PM)
+#define STATION_INFO_NONPEER_PM		BIT(NL80211_STA_INFO_NONPEER_PM)
+#define STATION_INFO_RX_BYTES64		BIT(NL80211_STA_INFO_RX_BYTES64)
+#define STATION_INFO_TX_BYTES64		BIT(NL80211_STA_INFO_TX_BYTES64)
+#define STATION_INFO_ASSOC_REQ_IES	0
+#endif /* Linux kernel >= 4.0.0 */
+
+#define RTW_MAX_MGMT_TX_CNT (8)
+#ifndef RTW_MAX_MGMT_TX_MS_GAS
+#define RTW_MAX_MGMT_TX_MS_GAS (500)
+#endif /*RTW_MAX_MGMT_TX_MS_GAS*/
+#define RTW_SCAN_IE_LEN_MAX      2304
+#define RTW_MAX_REMAIN_ON_CHANNEL_DURATION 5000 /* ms */
+#define RTW_MAX_NUM_PMKIDS 4
+
+#define RTW_CH_MAX_2G_CHANNEL               14      /* Max channel in 2G band */
+
+#ifdef CONFIG_WAPI_SUPPORT
+
+#ifndef WLAN_CIPHER_SUITE_SMS4
+#define WLAN_CIPHER_SUITE_SMS4          0x00147201
+#endif
+
+#ifndef WLAN_AKM_SUITE_WAPI_PSK
+#define WLAN_AKM_SUITE_WAPI_PSK         0x000FAC04
+#endif
+
+#ifndef WLAN_AKM_SUITE_WAPI_CERT
+#define WLAN_AKM_SUITE_WAPI_CERT        0x000FAC12
+#endif
+
+#ifndef NL80211_WAPI_VERSION_1
+#define NL80211_WAPI_VERSION_1          (1 << 2)
+#endif
+
+#endif /* CONFIG_WAPI_SUPPORT */
+
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(4, 11, 12))
+#ifdef CONFIG_RTW_80211R
+#define WLAN_AKM_SUITE_FT_8021X		0x000FAC03
+#define WLAN_AKM_SUITE_FT_PSK			0x000FAC04
+#endif
+#endif
+
+#define WIFI_CIPHER_SUITE_GCMP		0x000FAC08
+#define WIFI_CIPHER_SUITE_GCMP_256	0x000FAC09
+#define WIFI_CIPHER_SUITE_CCMP_256	0x000FAC0A
+#define WIFI_CIPHER_SUITE_BIP_GMAC_128	0x000FAC0B
+#define WIFI_CIPHER_SUITE_BIP_GMAC_256	0x000FAC0C
+#define WIFI_CIPHER_SUITE_BIP_CMAC_256	0x000FAC0D
+
+/*
+ * If customer need, defining this flag will make driver 
+ * always return -EBUSY at the condition of scan deny.
+ */
+/* #define CONFIG_NOTIFY_SCAN_ABORT_WITH_BUSY */
+
+static const u32 rtw_cipher_suites[] = {
+	WLAN_CIPHER_SUITE_WEP40,
+	WLAN_CIPHER_SUITE_WEP104,
+	WLAN_CIPHER_SUITE_TKIP,
+	WLAN_CIPHER_SUITE_CCMP,
+#ifdef CONFIG_WAPI_SUPPORT
+	WLAN_CIPHER_SUITE_SMS4,
+#endif /* CONFIG_WAPI_SUPPORT */
+#ifdef CONFIG_IEEE80211W
+	WLAN_CIPHER_SUITE_AES_CMAC,
+	WIFI_CIPHER_SUITE_GCMP,
+	WIFI_CIPHER_SUITE_GCMP_256,
+	WIFI_CIPHER_SUITE_CCMP_256,
+	WIFI_CIPHER_SUITE_BIP_GMAC_128,
+	WIFI_CIPHER_SUITE_BIP_GMAC_256,
+	WIFI_CIPHER_SUITE_BIP_CMAC_256,
+#endif /* CONFIG_IEEE80211W */
+};
+
+#define RATETAB_ENT(_rate, _rateid, _flags) \
+	{								\
+		.bitrate	= (_rate),				\
+		.hw_value	= (_rateid),				\
+		.flags		= (_flags),				\
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+/* if wowlan is not supported, kernel generate a disconnect at each suspend
+ * cf: /net/wireless/sysfs.c, so register a stub wowlan.
+ * Moreover wowlan has to be enabled via a the nl80211_set_wowlan callback.
+ * (from user space, e.g. iw phy0 wowlan enable)
+ */
+static const struct wiphy_wowlan_support wowlan_stub = {
+	.flags = WIPHY_WOWLAN_ANY,
+	.n_patterns = 0,
+	.pattern_max_len = 0,
+	.pattern_min_len = 0,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+	.max_pkt_offset = 0,
+#endif
+};
+#endif
+
+static const struct ieee80211_rate rtw_rates[] = {
+	RATETAB_ENT(10,  0x1,   0),
+	RATETAB_ENT(20,  0x2,   0),
+	RATETAB_ENT(55,  0x4,   0),
+	RATETAB_ENT(110, 0x8,   0),
+	RATETAB_ENT(60,  0x10,  0),
+	RATETAB_ENT(90,  0x20,  0),
+	RATETAB_ENT(120, 0x40,  0),
+	RATETAB_ENT(180, 0x80,  0),
+	RATETAB_ENT(240, 0x100, 0),
+	RATETAB_ENT(360, 0x200, 0),
+	RATETAB_ENT(480, 0x400, 0),
+	RATETAB_ENT(540, 0x800, 0),
+};
+
+#define rtw_a_rates		(rtw_rates + 4)
+#define RTW_A_RATES_NUM	8
+#define rtw_g_rates		(rtw_rates + 0)
+#define RTW_G_RATES_NUM	12
+
+static int rtw_cfg80211_set_assocresp_ies(struct net_device *net, const u8 *buf, int len);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+static u8 rtw_chdef_to_cfg80211_chan_def(struct wiphy *wiphy,
+		struct cfg80211_chan_def *chdef,
+		struct rtw_chan_def *rtw_chdef,
+		u8 ht)
+{
+	return rtw_bchbw_to_cfg80211_chan_def(wiphy, chdef
+		, rtw_chdef->band, rtw_chdef->chan, rtw_chdef->bw, rtw_chdef->offset, ht);
+}
+
+static void rtw_get_chdef_from_cfg80211_chan_def(struct cfg80211_chan_def *chdef,
+		u8 *ht, struct rtw_chan_def *rtw_chdef)
+{
+	u8 bw, offset;
+
+	rtw_get_bchbw_from_cfg80211_chan_def(chdef
+		, ht, &rtw_chdef->band, &rtw_chdef->chan, &bw, &offset);
+
+	rtw_chdef->bw = bw;
+	rtw_chdef->offset = offset;
+}
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+static enum nl80211_channel_type rtw_chdef_to_nl80211_channel_type(struct rtw_chan_def *rtw_chdef, u8 ht)
+{
+	return rtw_bchbw_to_nl80211_channel_type(rtw_chdef->band, rtw_chdef->chan, rtw_chdef->bw, rtw_chdef->offset, ht);
+}
+
+static void rtw_get_chdef_from_nl80211_channel_type(struct ieee80211_channel *chan,
+			enum nl80211_channel_type ctype,
+			u8 *ht, struct rtw_chan_def *rtw_chdef)
+{
+	u8 bw, offset;
+
+	rtw_get_bchbw_from_nl80211_channel_type(chan, ctype
+		, &rtw_chdef->band, ht, &rtw_chdef->chan, &bw, &offset);
+
+	rtw_chdef->bw = bw;
+	rtw_chdef->offset = offset;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+bool rtw_cfg80211_allow_ch_switch_notify(_adapter *adapter)
+{
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))
+	if ((!MLME_IS_AP(adapter))
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
+		&& (!MLME_IS_ADHOC(adapter))
+		&& (!MLME_IS_ADHOC_MASTER(adapter))
+		&& (!MLME_IS_MESH(adapter))
+#elif defined(CONFIG_RTW_MESH)
+		&& (!MLME_IS_MESH(adapter))
+#endif
+		)
+		return 0;
+#endif
+	return 1;
+}
+
+u8 rtw_cfg80211_ch_switch_notify(_adapter *adapter,
+					struct _ADAPTER_LINK *alink,
+					struct rtw_chan_def *rtw_chdef,
+					u8 ht, bool started)
+{
+	struct wiphy *wiphy = adapter_to_wiphy(adapter);
+	u8 ret = _SUCCESS;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	unsigned int link_id = 0;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	struct cfg80211_chan_def chdef = {0};
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	u16 punct_bitmap = 0; /*TBD*/
+#endif
+
+	ret = rtw_chdef_to_cfg80211_chan_def(wiphy, &chdef, rtw_chdef, ht);
+	if (ret != _SUCCESS)
+		goto exit;
+
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+	if (started) {
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0))
+		cfg80211_ch_switch_started_notify(adapter->pnetdev, &chdef, link_id, 0, false);
+		#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+		cfg80211_ch_switch_started_notify(adapter->pnetdev, &chdef, link_id, 0, false, punct_bitmap);
+		#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0))
+		cfg80211_ch_switch_started_notify(adapter->pnetdev, &chdef, link_id, 0, false);
+		#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 11, 0))
+		/* --- cfg80211_ch_switch_started_notfiy() ---
+		 *  A new parameter, bool quiet, is added from Linux kernel v5.11,
+		 *  to see if block-tx was requested by the AP. since currently,
+		 *  the API is used for station before connected in rtw_chk_start_clnt_join()
+		 *  the quiet is set to false here first. May need to refine it if
+		 *  called by others with block-tx.
+		 */
+		cfg80211_ch_switch_started_notify(adapter->pnetdev, &chdef, 0, false);
+		#else
+		cfg80211_ch_switch_started_notify(adapter->pnetdev, &chdef, 0);
+		#endif
+		goto exit;
+	}
+	#endif
+
+	if (!rtw_cfg80211_allow_ch_switch_notify(adapter))
+		goto exit;
+
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0))
+	cfg80211_ch_switch_notify(adapter->pnetdev, &chdef, link_id);
+	#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	cfg80211_ch_switch_notify(adapter->pnetdev, &chdef, link_id, punct_bitmap);
+	#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2))
+	/* ToDo CONFIG_RTW_MLD */
+	cfg80211_ch_switch_notify(adapter->pnetdev, &chdef, link_id);
+	#else
+	cfg80211_ch_switch_notify(adapter->pnetdev, &chdef);
+	#endif
+
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) */
+	int freq = rtw_bch2freq(rtw_chdef->band, rtw_chdef->chan);
+	enum nl80211_channel_type ctype;
+
+	if (!rtw_cfg80211_allow_ch_switch_notify(adapter))
+		goto exit;
+
+	if (!freq) {
+		ret = _FAIL;
+		goto exit;
+	}
+
+	ctype = rtw_chdef_to_nl80211_channel_type(rtw_chdef, ht);
+	cfg80211_ch_switch_notify(adapter->pnetdev, freq, ctype);
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) */
+
+exit:
+	return ret;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) */
+
+static void rtw_2g_channels_init(struct ieee80211_channel *channels)
+{
+	_rtw_memcpy(channels, rtw_2ghz_channels, sizeof(rtw_2ghz_channels));
+}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static void rtw_5g_channels_init(struct ieee80211_channel *channels)
+{
+	_rtw_memcpy(channels, rtw_5ghz_a_channels, sizeof(rtw_5ghz_a_channels));
+}
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static void rtw_6g_channels_init(struct ieee80211_channel *channels)
+{
+	_rtw_memcpy(channels, rtw_6ghz_channels, sizeof(rtw_6ghz_channels));
+}
+#endif
+
+void rtw_2g_rates_init(struct ieee80211_rate *rates)
+{
+	_rtw_memcpy(rates, rtw_g_rates,
+		sizeof(struct ieee80211_rate) * RTW_G_RATES_NUM
+	);
+}
+
+void rtw_5g_rates_init(struct ieee80211_rate *rates)
+{
+	_rtw_memcpy(rates, rtw_a_rates,
+		sizeof(struct ieee80211_rate) * RTW_A_RATES_NUM
+	);
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+void rtw_6g_rates_init(struct ieee80211_rate *rates)
+{
+	_rtw_memcpy(rates, rtw_a_rates,
+		sizeof(struct ieee80211_rate) * RTW_A_RATES_NUM
+	);
+}
+#endif
+
+struct ieee80211_supported_band *rtw_spt_band_alloc(enum band_type band)
+{
+	struct ieee80211_supported_band *spt_band = NULL;
+	int n_channels, n_bitrates;
+
+	if (rtw_band_to_nl80211_band(band) == NUM_NL80211_BANDS)
+		goto exit;
+
+	if (band == BAND_ON_24G) {
+		n_channels = MAX_CHANNEL_NUM_2G;
+		n_bitrates = RTW_G_RATES_NUM;
+	} else if (band == BAND_ON_5G) {
+		n_channels = MAX_CHANNEL_NUM_5G;
+		n_bitrates = RTW_A_RATES_NUM;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	} else if (band == BAND_ON_6G) {
+		n_channels = MAX_CHANNEL_NUM_6G;
+		n_bitrates = RTW_A_RATES_NUM;
+#endif
+	} else
+		goto exit;
+
+	spt_band = (struct ieee80211_supported_band *)rtw_zmalloc(
+		sizeof(struct ieee80211_supported_band)
+		+ sizeof(struct ieee80211_channel) * n_channels
+		+ sizeof(struct ieee80211_rate) * n_bitrates
+#if defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)))
+		+ sizeof(struct ieee80211_sband_iftype_data) * 2
+#endif /* defined(CONFIG_80211AX_HE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) */
+	);
+	if (!spt_band)
+		goto exit;
+
+	spt_band->channels = (struct ieee80211_channel *)(((u8 *)spt_band) + sizeof(struct ieee80211_supported_band));
+	spt_band->bitrates = (struct ieee80211_rate *)(((u8 *)spt_band->channels) + sizeof(struct ieee80211_channel) * n_channels);
+	spt_band->band = rtw_band_to_nl80211_band(band);
+	spt_band->n_channels = n_channels;
+	spt_band->n_bitrates = n_bitrates;
+#if defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)))
+	spt_band->iftype_data = (struct ieee80211_sband_iftype_data *)(((u8 *)spt_band->bitrates)
+	                        + sizeof(struct ieee80211_rate) * n_bitrates);
+	spt_band->n_iftype_data = 0;
+#endif /* defined(CONFIG_80211AX_HE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) */
+
+	if (band == BAND_ON_24G) {
+		rtw_2g_channels_init(spt_band->channels);
+		rtw_2g_rates_init(spt_band->bitrates);
+	}
+#if CONFIG_IEEE80211_BAND_5GHZ
+	else if (band == BAND_ON_5G) {
+		rtw_5g_channels_init(spt_band->channels);
+		rtw_5g_rates_init(spt_band->bitrates);
+	}
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (band == BAND_ON_6G) {
+		rtw_6g_channels_init(spt_band->channels);
+		rtw_6g_rates_init(spt_band->bitrates);
+	}
+#endif
+
+	/* spt_band.ht_cap */
+
+exit:
+
+	return spt_band;
+}
+
+void rtw_spt_band_free(struct ieee80211_supported_band *spt_band)
+{
+	u32 size = 0;
+
+	if (!spt_band)
+		return;
+
+	if (spt_band->band == NL80211_BAND_2GHZ) {
+		size = sizeof(struct ieee80211_supported_band)
+			+ sizeof(struct ieee80211_channel) * MAX_CHANNEL_NUM_2G
+			+ sizeof(struct ieee80211_rate) * RTW_G_RATES_NUM;
+	} else if (spt_band->band == NL80211_BAND_5GHZ) {
+		size = sizeof(struct ieee80211_supported_band)
+			+ sizeof(struct ieee80211_channel) * MAX_CHANNEL_NUM_5G
+			+ sizeof(struct ieee80211_rate) * RTW_A_RATES_NUM;
+	}
+#if CONFIG_IEEE80211_BAND_6GHZ
+	else if (spt_band->band == NL80211_BAND_6GHZ) {
+		size = sizeof(struct ieee80211_supported_band)
+			+ sizeof(struct ieee80211_channel) * MAX_CHANNEL_NUM_6G
+			+ sizeof(struct ieee80211_rate) * RTW_A_RATES_NUM;
+	}
+#endif
+
+#if defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)))
+	size +=	sizeof(struct ieee80211_sband_iftype_data) * 2;
+#endif /* defined(CONFIG_80211AX_HE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) */
+
+	rtw_mfree((u8 *)spt_band, size);
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+static const struct ieee80211_txrx_stypes
+	rtw_cfg80211_default_mgmt_stypes[NUM_NL80211_IFTYPES] = {
+	[NL80211_IFTYPE_ADHOC] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ACTION >> 4)
+	},
+	[NL80211_IFTYPE_STATION] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
+		BIT(IEEE80211_STYPE_AUTH >> 4) |
+		BIT(IEEE80211_STYPE_PROBE_REQ >> 4)
+	},
+	[NL80211_IFTYPE_AP] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
+		BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
+		BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
+		BIT(IEEE80211_STYPE_DISASSOC >> 4) |
+		BIT(IEEE80211_STYPE_AUTH >> 4) |
+		BIT(IEEE80211_STYPE_DEAUTH >> 4) |
+		BIT(IEEE80211_STYPE_ACTION >> 4)
+	},
+	[NL80211_IFTYPE_AP_VLAN] = {
+		/* copy AP */
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
+		BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
+		BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
+		BIT(IEEE80211_STYPE_DISASSOC >> 4) |
+		BIT(IEEE80211_STYPE_AUTH >> 4) |
+		BIT(IEEE80211_STYPE_DEAUTH >> 4) |
+		BIT(IEEE80211_STYPE_ACTION >> 4)
+	},
+	[NL80211_IFTYPE_P2P_CLIENT] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
+		BIT(IEEE80211_STYPE_AUTH >> 4) |
+		BIT(IEEE80211_STYPE_PROBE_REQ >> 4)
+	},
+	[NL80211_IFTYPE_P2P_GO] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) |
+		BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) |
+		BIT(IEEE80211_STYPE_PROBE_REQ >> 4) |
+		BIT(IEEE80211_STYPE_DISASSOC >> 4) |
+		BIT(IEEE80211_STYPE_AUTH >> 4) |
+		BIT(IEEE80211_STYPE_DEAUTH >> 4) |
+		BIT(IEEE80211_STYPE_ACTION >> 4)
+	},
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+	[NL80211_IFTYPE_P2P_DEVICE] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
+			BIT(IEEE80211_STYPE_PROBE_REQ >> 4)
+	},
+#endif
+#if defined(CONFIG_RTW_MESH)
+	[NL80211_IFTYPE_MESH_POINT] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ACTION >> 4)
+			| BIT(IEEE80211_STYPE_AUTH >> 4)
+	},
+#endif
+
+};
+#endif
+
+NDIS_802_11_NETWORK_INFRASTRUCTURE nl80211_iftype_to_rtw_network_type(enum nl80211_iftype type)
+{
+	switch (type) {
+	case NL80211_IFTYPE_ADHOC:
+		return Ndis802_11IBSS;
+
+	#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_CLIENT:
+	#endif
+	case NL80211_IFTYPE_STATION:
+		return Ndis802_11Infrastructure;
+
+#ifdef CONFIG_AP_MODE
+	#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_GO:
+	#endif
+	case NL80211_IFTYPE_AP:
+		return Ndis802_11APMode;
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	case NL80211_IFTYPE_MESH_POINT:
+		return Ndis802_11_mesh;
+#endif
+
+#ifdef CONFIG_WIFI_MONITOR
+	case NL80211_IFTYPE_MONITOR:
+		return Ndis802_11Monitor;
+#endif /* CONFIG_WIFI_MONITOR */
+
+	default:
+		return Ndis802_11InfrastructureMax;
+	}
+}
+
+u32 nl80211_iftype_to_rtw_mlme_state(enum nl80211_iftype type)
+{
+	switch (type) {
+	case NL80211_IFTYPE_ADHOC:
+		return WIFI_ADHOC_STATE;
+
+	#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_CLIENT:
+	#endif
+	case NL80211_IFTYPE_STATION:
+		return WIFI_STATION_STATE;
+
+#ifdef CONFIG_AP_MODE
+	#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_GO:
+	#endif
+	case NL80211_IFTYPE_AP:
+		return WIFI_AP_STATE;
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	case NL80211_IFTYPE_MESH_POINT:
+		return WIFI_MESH_STATE;
+#endif
+
+	case NL80211_IFTYPE_MONITOR:
+		return WIFI_MONITOR_STATE;
+
+	default:
+		return WIFI_NULL_STATE;
+	}
+}
+
+static
+enum role_type nl80211_iftype_to_rtw_phl_role_type(enum nl80211_iftype type)
+{
+	switch (type) {
+	case NL80211_IFTYPE_ADHOC:
+		return PHL_RTYPE_ADHOC;
+	#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_CLIENT:
+		return PHL_RTYPE_P2P_GC;
+	#endif
+	case NL80211_IFTYPE_STATION:
+		return PHL_RTYPE_STATION;
+
+#ifdef CONFIG_AP_MODE
+	#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_GO:
+		return PHL_RTYPE_P2P_GO;
+	#endif
+	case NL80211_IFTYPE_AP:
+		return PHL_RTYPE_AP;
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	case NL80211_IFTYPE_MESH_POINT:
+		return PHL_RTYPE_MESH;
+#endif
+
+	case NL80211_IFTYPE_MONITOR:
+		return PHL_RTYPE_MONITOR;
+
+	default:
+		return PHL_RTYPE_NONE;
+	}
+}
+
+static int rtw_cfg80211_sync_iftype(_adapter *adapter)
+{
+	struct wireless_dev *rtw_wdev = adapter->rtw_wdev;
+
+	if (!(nl80211_iftype_to_rtw_mlme_state(rtw_wdev->iftype) & MLME_STATE(adapter))) {
+		/* iftype and mlme state is not syc */
+		NDIS_802_11_NETWORK_INFRASTRUCTURE network_type;
+
+		network_type = nl80211_iftype_to_rtw_network_type(rtw_wdev->iftype);
+		if (network_type != Ndis802_11InfrastructureMax) {
+
+			rtw_set_802_11_infrastructure_mode(adapter, network_type, 0);
+			rtw_setopmode_cmd(adapter, network_type, RTW_CMDF_WAIT_ACK);
+		} else {
+			rtw_warn_on(1);
+			RTW_WARN(FUNC_ADPT_FMT" iftype:%u is not support\n", FUNC_ADPT_ARG(adapter), rtw_wdev->iftype);
+			return _FAIL;
+		}
+	}
+
+	return _SUCCESS;
+}
+
+static u64 rtw_get_systime_us(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0))
+	return ktime_to_us(ktime_get_boottime());
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+	struct timespec ts;
+	get_monotonic_boottime(&ts);
+	return ((u64)ts.tv_sec * 1000000) + ts.tv_nsec / 1000;
+#else
+	struct timeval tv;
+	do_gettimeofday(&tv);
+	return ((u64)tv.tv_sec * 1000000) + tv.tv_usec;
+#endif
+}
+
+/* Try to remove non target BSS's SR to reduce PBC overlap rate */
+static int rtw_cfg80211_clear_wps_sr_of_non_target_bss(_adapter *padapter, struct wlan_network *pnetwork, struct cfg80211_ssid *req_ssid)
+{
+	int ret = 0;
+	u8 *psr = NULL, sr = 0;
+	NDIS_802_11_SSID *pssid = &pnetwork->network.Ssid;
+	u32 wpsielen = 0;
+	u8 *wpsie = NULL;
+
+	if (pssid->SsidLength == req_ssid->ssid_len
+		&& _rtw_memcmp(pssid->Ssid, req_ssid->ssid, req_ssid->ssid_len) == _TRUE)
+		goto exit;
+
+	wpsie = rtw_get_wps_ie(pnetwork->network.IEs + _FIXED_IE_LENGTH_
+		, pnetwork->network.IELength - _FIXED_IE_LENGTH_, NULL, &wpsielen);
+	if (wpsie && wpsielen > 0)
+		psr = rtw_get_wps_attr_content(wpsie, wpsielen, WPS_ATTR_SELECTED_REGISTRAR, &sr, NULL);
+
+	if (psr && sr) {
+		if (0)
+			RTW_INFO("clear sr of non target bss:%s("MAC_FMT")\n"
+				, pssid->Ssid, MAC_ARG(pnetwork->network.MacAddress));
+		*psr = 0; /* clear sr */
+		ret = 1;
+	}
+
+exit:
+	return ret;
+}
+
+#define MAX_BSSINFO_LEN 1000
+struct cfg80211_bss *rtw_cfg80211_inform_bss(_adapter *padapter, struct wlan_network *pnetwork)
+{
+	struct ieee80211_channel *notify_channel;
+	struct cfg80211_bss *bss = NULL;
+	u16 band;
+	u16 channel;
+	u32 freq;
+	u64 notify_timestamp;
+	u16 notify_capability;
+	u16 notify_interval;
+	u8 *notify_ie;
+	size_t notify_ielen;
+	s32 notify_signal;
+	/* u8 buf[MAX_BSSINFO_LEN]; */
+
+	u8 *pbuf;
+	size_t buf_size = MAX_BSSINFO_LEN;
+	size_t len, bssinf_len = 0;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	unsigned short *fctrl;
+	u8	bc_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+	struct wireless_dev *wdev = padapter->rtw_wdev;
+	struct wiphy *wiphy = wdev->wiphy;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	pbuf = rtw_zmalloc(buf_size);
+	if (pbuf == NULL) {
+		RTW_INFO("%s pbuf allocate failed  !!\n", __FUNCTION__);
+		return bss;
+	}
+
+	/* RTW_INFO("%s\n", __func__); */
+
+	bssinf_len = pnetwork->network.IELength + sizeof(struct rtw_ieee80211_hdr_3addr);
+	if (bssinf_len > buf_size) {
+		RTW_INFO("%s IE Length too long > %zu byte\n", __FUNCTION__, buf_size);
+		goto exit;
+	}
+
+#ifndef CONFIG_WAPI_SUPPORT
+	{
+		u16 wapi_len = 0;
+
+		if (rtw_get_wapi_ie(pnetwork->network.IEs, pnetwork->network.IELength, NULL, &wapi_len) > 0) {
+			if (wapi_len > 0) {
+				RTW_INFO("%s, no support wapi!\n", __FUNCTION__);
+				goto exit;
+			}
+		}
+	}
+#endif /* !CONFIG_WAPI_SUPPORT */
+
+	band = pnetwork->network.Configuration.Band;
+	channel = pnetwork->network.Configuration.DSConfig;
+	freq = rtw_bch2freq(band, channel);
+	notify_channel = ieee80211_get_channel(wiphy, freq);
+
+	if (0)
+		notify_timestamp = le64_to_cpu(*(u64 *)rtw_get_timestampe_from_ie(pnetwork->network.IEs));
+	else
+		notify_timestamp = rtw_get_systime_us();
+
+	notify_interval = le16_to_cpu(*(u16 *)rtw_get_beacon_interval_from_ie(pnetwork->network.IEs));
+	notify_capability = le16_to_cpu(*(u16 *)rtw_get_capability_from_ie(pnetwork->network.IEs));
+
+	notify_ie = pnetwork->network.IEs + _FIXED_IE_LENGTH_;
+	notify_ielen = pnetwork->network.IELength - _FIXED_IE_LENGTH_;
+
+	/*RTW_WKARD_CORE_RSSI_V1*/
+	/* We've set wiphy's signal_type as CFG80211_SIGNAL_TYPE_MBM: signal strength in mBm (100*dBm) */
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE &&
+	    is_same_network(&pmlmepriv->dev_cur_network.network, &pnetwork->network)) {
+		notify_signal = 100 * rtw_phl_rssi_to_dbm(padapter->recvinfo.signal_strength); /* dbm */
+	} else {
+		notify_signal = 100 * rtw_phl_rssi_to_dbm(pnetwork->network.PhyInfo.SignalStrength); /* pnetwork->network.PhyInfo.rssi -dbm */
+	}
+#if 0
+	RTW_INFO("bssid: "MAC_FMT", rssi:%d\t", MAC_ARG(pnetwork->network.MacAddress),notify_signal);
+	RTW_INFO("ss:%d, sq:%d, orssi:%d\n",
+		pnetwork->network.PhyInfo.SignalStrength,
+		pnetwork->network.PhyInfo.SignalQuality,
+		pnetwork->network.PhyInfo.rssi);
+#endif
+#if 0
+	RTW_INFO("bssid: "MAC_FMT"\n", MAC_ARG(pnetwork->network.MacAddress));
+	RTW_INFO("Channel: %d(%d)\n", channel, freq);
+	RTW_INFO("Capability: %X\n", notify_capability);
+	RTW_INFO("Beacon interval: %d\n", notify_interval);
+	RTW_INFO("Signal: %d\n", notify_signal);
+	RTW_INFO("notify_timestamp: %llu\n", notify_timestamp);
+#endif
+
+	/* pbuf = buf; */
+
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pbuf;
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	SetSeqNum(pwlanhdr, 0/*pmlmeext->mgnt_seq*/);
+	/* pmlmeext->mgnt_seq++; */
+
+	if (pnetwork->network.Reserved[0] == BSS_TYPE_BCN) { /* WIFI_BEACON */
+		_rtw_memcpy(pwlanhdr->addr1, bc_addr, ETH_ALEN);
+		set_frame_sub_type(pbuf, WIFI_BEACON);
+	} else {
+		_rtw_memcpy(pwlanhdr->addr1, adapter_mac_addr(padapter), ETH_ALEN);
+		set_frame_sub_type(pbuf, WIFI_PROBERSP);
+	}
+
+	_rtw_memcpy(pwlanhdr->addr2, pnetwork->network.MacAddress, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, pnetwork->network.MacAddress, ETH_ALEN);
+
+
+	/* pbuf += sizeof(struct rtw_ieee80211_hdr_3addr); */
+	len = sizeof(struct rtw_ieee80211_hdr_3addr);
+	_rtw_memcpy((pbuf + len), pnetwork->network.IEs, pnetwork->network.IELength);
+	*((u64 *)(pbuf + len)) = cpu_to_le64(notify_timestamp);
+
+	len += pnetwork->network.IELength;
+
+	#if defined(CONFIG_P2P) && 0
+	if(rtw_get_p2p_ie(pnetwork->network.IEs+12, pnetwork->network.IELength-12, NULL, NULL))
+		RTW_INFO("%s, got p2p_ie\n", __func__);
+	#endif
+
+#if 1
+	bss = cfg80211_inform_bss_frame(wiphy, notify_channel, (struct ieee80211_mgmt *)pbuf,
+					len, notify_signal, GFP_ATOMIC);
+#else
+
+	bss = cfg80211_inform_bss(wiphy, notify_channel, (const u8 *)pnetwork->network.MacAddress,
+		notify_timestamp, notify_capability, notify_interval, notify_ie,
+		notify_ielen, notify_signal, GFP_ATOMIC/*GFP_KERNEL*/);
+#endif
+
+	if (unlikely(!bss)) {
+		RTW_INFO(FUNC_ADPT_FMT" bss NULL\n", FUNC_ADPT_ARG(padapter));
+		goto exit;
+	}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 38))
+#ifndef COMPAT_KERNEL_RELEASE
+	/* patch for cfg80211, update beacon ies to information_elements */
+	if (pnetwork->network.Reserved[0] == BSS_TYPE_BCN) { /* WIFI_BEACON */
+
+		if (bss->len_information_elements != bss->len_beacon_ies) {
+			bss->information_elements = bss->beacon_ies;
+			bss->len_information_elements =  bss->len_beacon_ies;
+		}
+	}
+#endif /* COMPAT_KERNEL_RELEASE */
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 38) */
+
+#if 0
+	{
+		if (bss->information_elements == bss->proberesp_ies) {
+			if (bss->len_information_elements !=  bss->len_proberesp_ies)
+				RTW_INFO("error!, len_information_elements != bss->len_proberesp_ies\n");
+		} else if (bss->len_information_elements <  bss->len_beacon_ies) {
+			bss->information_elements = bss->beacon_ies;
+			bss->len_information_elements =  bss->len_beacon_ies;
+		}
+	}
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
+	cfg80211_put_bss(wiphy, bss);
+#else
+	cfg80211_put_bss(bss);
+#endif
+
+exit:
+	if (pbuf)
+		rtw_mfree(pbuf, buf_size);
+	return bss;
+
+}
+
+/*
+	Check the given bss is valid by kernel API cfg80211_get_bss()
+	@padapter : the given adapter
+
+	return _TRUE if bss is valid,  _FALSE for not found.
+*/
+int rtw_cfg80211_check_bss(_adapter *padapter)
+{
+	WLAN_BSSID_EX  *pnetwork = &(padapter->mlmeextpriv.mlmext_info.dev_network);
+	struct cfg80211_bss *bss = NULL;
+	struct ieee80211_channel *notify_channel = NULL;
+	u32 freq;
+
+	if (!(pnetwork) || !(padapter->rtw_wdev))
+		return _FALSE;
+
+	freq = rtw_bch2freq(pnetwork->Configuration.Band, pnetwork->Configuration.DSConfig);
+	notify_channel = ieee80211_get_channel(padapter->rtw_wdev->wiphy, freq);
+
+	RTW_INFO("%s freq=%d band=%d dsconfig=%d\n", __func__, freq, pnetwork->Configuration.Band, pnetwork->Configuration.DSConfig);
+	if (!notify_channel) {
+		RTW_INFO(FUNC_ADPT_FMT" notify_channel null!\n", FUNC_ADPT_ARG(padapter));
+		return _FALSE;
+	}
+
+	if (!(padapter->rtw_wdev->wiphy)) {
+		RTW_INFO(FUNC_ADPT_FMT"padapter->rtw_wdev->wiphy null!\n", FUNC_ADPT_ARG(padapter));
+		return _FALSE;
+	}
+
+	bss = cfg80211_get_bss(padapter->rtw_wdev->wiphy, notify_channel,
+			pnetwork->MacAddress, pnetwork->Ssid.Ssid,
+			pnetwork->Ssid.SsidLength,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
+			pnetwork->InfrastructureMode == Ndis802_11Infrastructure?IEEE80211_BSS_TYPE_ESS:IEEE80211_BSS_TYPE_IBSS,
+			IEEE80211_PRIVACY(pnetwork->Privacy));
+#else
+			pnetwork->InfrastructureMode == Ndis802_11Infrastructure?WLAN_CAPABILITY_ESS:WLAN_CAPABILITY_IBSS, pnetwork->InfrastructureMode == Ndis802_11Infrastructure?WLAN_CAPABILITY_ESS:WLAN_CAPABILITY_IBSS);
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
+	cfg80211_put_bss(padapter->rtw_wdev->wiphy, bss);
+#else
+	cfg80211_put_bss(bss);
+#endif
+
+	return bss != NULL;
+}
+
+void rtw_cfg80211_ibss_indicate_connect(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network  *cur_network = &(pmlmepriv->dev_cur_network);
+	struct wireless_dev *pwdev = padapter->rtw_wdev;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0))
+	struct wiphy *wiphy = pwdev->wiphy;
+	int freq = 2412;
+	struct ieee80211_channel *notify_channel;
+#endif
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+	if (pwdev->iftype != NL80211_IFTYPE_ADHOC)
+		return;
+
+	if (!rtw_cfg80211_check_bss(padapter)) {
+		WLAN_BSSID_EX  *pnetwork = &(padapter->mlmeextpriv.mlmext_info.dev_network);
+		struct wlan_network *scanned = pmlmepriv->cur_network_scanned;
+
+		if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE) {
+
+			_rtw_memcpy(&cur_network->network, pnetwork, sizeof(WLAN_BSSID_EX));
+			if (cur_network) {
+				if (!rtw_cfg80211_inform_bss(padapter, cur_network))
+					RTW_INFO(FUNC_ADPT_FMT" inform fail !!\n", FUNC_ADPT_ARG(padapter));
+				else
+					RTW_INFO(FUNC_ADPT_FMT" inform success !!\n", FUNC_ADPT_ARG(padapter));
+			} else {
+				RTW_INFO("cur_network is not exist!!!\n");
+				return ;
+			}
+		} else {
+			if (scanned == NULL) {
+				rtw_warn_on(1);
+				return;
+			}
+
+			if (_rtw_memcmp(&(scanned->network.Ssid), &(pnetwork->Ssid), sizeof(NDIS_802_11_SSID)) == _TRUE
+				&& _rtw_memcmp(scanned->network.MacAddress, pnetwork->MacAddress, sizeof(NDIS_802_11_MAC_ADDRESS)) == _TRUE
+			) {
+				if (!rtw_cfg80211_inform_bss(padapter, scanned)){
+					RTW_INFO(FUNC_ADPT_FMT" inform fail !!\n", FUNC_ADPT_ARG(padapter));
+				} else {
+					/* RTW_INFO(FUNC_ADPT_FMT" inform success !!\n", FUNC_ADPT_ARG(padapter)); */
+				}
+			} else {
+				RTW_INFO("scanned & pnetwork compare fail\n");
+				rtw_warn_on(1);
+			}
+		}
+
+		if (!rtw_cfg80211_check_bss(padapter))
+			RTW_PRINT(FUNC_ADPT_FMT" BSS not found !!\n", FUNC_ADPT_ARG(padapter));
+	}
+	/* notify cfg80211 that device joined an IBSS */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0))
+	freq = rtw_bch2freq(cur_network->network.Configuration.Band, cur_network->network.Configuration.DSConfig);
+	if (1)
+		RTW_INFO("band: %d, chan: %d, freq: %d\n", cur_network->network.Configuration.Band,
+				cur_network->network.Configuration.DSConfig, freq);
+	notify_channel = ieee80211_get_channel(wiphy, freq);
+	cfg80211_ibss_joined(padapter->pnetdev, cur_network->network.MacAddress, notify_channel, GFP_ATOMIC);
+#else
+	cfg80211_ibss_joined(padapter->pnetdev, cur_network->network.MacAddress, GFP_ATOMIC);
+#endif
+}
+
+void rtw_cfg80211_indicate_connect(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct wlan_network  *cur_network = &(pmlmepriv->dev_cur_network);
+	struct wireless_dev *pwdev = padapter->rtw_wdev;
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif
+#if defined(CPTCFG_VERSION) || LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
+	struct cfg80211_roam_info roam_info ={};
+#endif
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+	if (pwdev->iftype != NL80211_IFTYPE_STATION
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+		&& pwdev->iftype != NL80211_IFTYPE_P2P_CLIENT
+		#endif
+	)
+		return;
+
+	if (!MLME_IS_STA(padapter))
+		return;
+
+	if (check_fwstate(pmlmepriv, WIFI_MONITOR_STATE) != _TRUE) {
+		WLAN_BSSID_EX  *pnetwork = &(padapter->mlmeextpriv.mlmext_info.dev_network);
+		struct wlan_network *scanned = pmlmepriv->cur_network_scanned;
+
+		/* RTW_INFO(FUNC_ADPT_FMT" BSS not found\n", FUNC_ADPT_ARG(padapter)); */
+
+		if (scanned == NULL) {
+			rtw_warn_on(1);
+			goto check_bss;
+		}
+
+		if (_rtw_memcmp(scanned->network.MacAddress, pnetwork->MacAddress, sizeof(NDIS_802_11_MAC_ADDRESS)) == _TRUE
+			&& _rtw_memcmp(&(scanned->network.Ssid), &(pnetwork->Ssid), sizeof(NDIS_802_11_SSID)) == _TRUE
+		) {
+			if (!rtw_cfg80211_inform_bss(padapter, scanned))
+				RTW_INFO(FUNC_ADPT_FMT" inform fail !!\n", FUNC_ADPT_ARG(padapter));
+			else {
+				/* RTW_INFO(FUNC_ADPT_FMT" inform success !!\n", FUNC_ADPT_ARG(padapter)); */
+			}
+		} else {
+			RTW_INFO("scanned: %s("MAC_FMT"), cur: %s("MAC_FMT")\n",
+				scanned->network.Ssid.Ssid, MAC_ARG(scanned->network.MacAddress),
+				pnetwork->Ssid.Ssid, MAC_ARG(pnetwork->MacAddress)
+			);
+			rtw_warn_on(1);
+		}
+	}
+
+check_bss:
+	if (!rtw_cfg80211_check_bss(padapter))
+		RTW_PRINT(FUNC_ADPT_FMT" BSS not found !!\n", FUNC_ADPT_ARG(padapter));
+
+	_rtw_spinlock_bh(&pwdev_priv->connect_req_lock);
+
+	if (rtw_to_roam(padapter) > 0) {
+		#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39) || defined(COMPAT_KERNEL_RELEASE)
+		struct wiphy *wiphy = pwdev->wiphy;
+		struct ieee80211_channel *notify_channel;
+		u32 freq;
+
+		freq = rtw_bch2freq(cur_network->network.Configuration.Band,
+				cur_network->network.Configuration.DSConfig);
+		notify_channel = ieee80211_get_channel(wiphy, freq);
+		#endif
+
+		#if defined(CPTCFG_VERSION) || LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
+		#if defined(CONFIG_ACK_5_15_LTS_KERNEL) || LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)
+		/* ToDo CONFIG_RTW_MLD */
+		roam_info.links[0].bssid = cur_network->network.MacAddress;
+		#else
+		roam_info.bssid = cur_network->network.MacAddress;
+		#endif
+		roam_info.req_ie = pmlmepriv->assoc_req + sizeof(struct rtw_ieee80211_hdr_3addr) + 2;
+		roam_info.req_ie_len = pmlmepriv->assoc_req_len - sizeof(struct rtw_ieee80211_hdr_3addr) - 2;
+		roam_info.resp_ie = pmlmepriv->assoc_rsp + sizeof(struct rtw_ieee80211_hdr_3addr) + 6;
+		roam_info.resp_ie_len = pmlmepriv->assoc_rsp_len - sizeof(struct rtw_ieee80211_hdr_3addr) - 6;
+
+		cfg80211_roamed(padapter->pnetdev, &roam_info, GFP_ATOMIC);
+		#else
+		cfg80211_roamed(padapter->pnetdev
+			#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 39) || defined(COMPAT_KERNEL_RELEASE)
+			, notify_channel
+			#endif
+			, cur_network->network.MacAddress
+			, pmlmepriv->assoc_req + sizeof(struct rtw_ieee80211_hdr_3addr) + 2
+			, pmlmepriv->assoc_req_len - sizeof(struct rtw_ieee80211_hdr_3addr) - 2
+			, pmlmepriv->assoc_rsp + sizeof(struct rtw_ieee80211_hdr_3addr) + 6
+			, pmlmepriv->assoc_rsp_len - sizeof(struct rtw_ieee80211_hdr_3addr) - 6
+			, GFP_ATOMIC);
+		#endif /*LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)*/
+
+		RTW_INFO(FUNC_ADPT_FMT" call cfg80211_roamed\n", FUNC_ADPT_ARG(padapter));
+
+#ifdef CONFIG_RTW_80211R
+		if (rtw_ft_roam(padapter))
+			rtw_ft_set_status(padapter, RTW_FT_ASSOCIATED_STA);
+#endif
+	} else {
+		#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) || defined(COMPAT_KERNEL_RELEASE)
+		RTW_INFO("pwdev->sme_state(b)=%d\n", pwdev->sme_state);
+		#endif
+
+		if (check_fwstate(pmlmepriv, WIFI_MONITOR_STATE) != _TRUE)
+			rtw_cfg80211_connect_result(pwdev, cur_network->network.MacAddress
+				, pmlmepriv->assoc_req + sizeof(struct rtw_ieee80211_hdr_3addr) + 2
+				, pmlmepriv->assoc_req_len - sizeof(struct rtw_ieee80211_hdr_3addr) - 2
+				, pmlmepriv->assoc_rsp + sizeof(struct rtw_ieee80211_hdr_3addr) + 6
+				, pmlmepriv->assoc_rsp_len - sizeof(struct rtw_ieee80211_hdr_3addr) - 6
+				, WLAN_STATUS_SUCCESS, GFP_ATOMIC);
+		#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) || defined(COMPAT_KERNEL_RELEASE)
+		RTW_INFO("pwdev->sme_state(a)=%d\n", pwdev->sme_state);
+		#endif
+	}
+
+	rtw_wdev_free_connect_req(pwdev_priv);
+
+	_rtw_spinunlock_bh(&pwdev_priv->connect_req_lock);
+}
+
+void rtw_cfg80211_indicate_disconnect(_adapter *padapter, u16 reason, u8 locally_generated)
+{
+	struct wireless_dev *pwdev = padapter->rtw_wdev;
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif
+
+	RTW_INFO(FUNC_ADPT_FMT" ,reason = %d\n", FUNC_ADPT_ARG(padapter), reason);
+
+	/*always replace privated definitions with wifi reserved value 0*/
+	if (WLAN_REASON_IS_PRIVATE(reason))
+		reason = 0;
+
+	if (pwdev->iftype != NL80211_IFTYPE_STATION
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+		&& pwdev->iftype != NL80211_IFTYPE_P2P_CLIENT
+		#endif
+	)
+		return;
+
+	if (!MLME_IS_STA(padapter))
+		return;
+
+	_rtw_spinlock_bh(&pwdev_priv->connect_req_lock);
+
+	if (padapter->ndev_unregistering || !rtw_wdev_not_indic_disco(pwdev_priv)) {
+		#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) || defined(COMPAT_KERNEL_RELEASE)
+		RTW_INFO("pwdev->sme_state(b)=%d\n", pwdev->sme_state);
+
+		if (pwdev->sme_state == CFG80211_SME_CONNECTING) {
+			RTW_INFO(FUNC_ADPT_FMT" call cfg80211_connect_result, reason:%d\n", FUNC_ADPT_ARG(padapter), reason);
+			rtw_cfg80211_connect_result(pwdev, NULL, NULL, 0, NULL, 0,
+				reason?reason:WLAN_STATUS_UNSPECIFIED_FAILURE,
+				GFP_ATOMIC);
+		} else if (pwdev->sme_state == CFG80211_SME_CONNECTED) {
+			RTW_INFO(FUNC_ADPT_FMT" call cfg80211_disconnected, reason:%d\n", FUNC_ADPT_ARG(padapter), reason);
+			rtw_cfg80211_disconnected(pwdev, reason?reason:WLAN_STATUS_UNSPECIFIED_FAILURE, NULL, 0, locally_generated, GFP_ATOMIC);
+		}
+
+		RTW_INFO("pwdev->sme_state(a)=%d\n", pwdev->sme_state);
+		#else
+		if (pwdev_priv->connect_req) {
+			RTW_INFO(FUNC_ADPT_FMT" call cfg80211_connect_result, reason:%d\n", FUNC_ADPT_ARG(padapter), reason);
+			rtw_cfg80211_connect_result(pwdev, NULL, NULL, 0, NULL, 0,
+				reason?reason:WLAN_STATUS_UNSPECIFIED_FAILURE,
+				GFP_ATOMIC);
+		} else {
+			RTW_INFO(FUNC_ADPT_FMT" call cfg80211_disconnected, reason:%d\n", FUNC_ADPT_ARG(padapter), reason);
+			rtw_cfg80211_disconnected(pwdev, reason?reason:WLAN_STATUS_UNSPECIFIED_FAILURE, NULL, 0, locally_generated, GFP_ATOMIC);
+		}
+		#endif
+	}
+
+	rtw_wdev_free_connect_req(pwdev_priv);
+
+	_rtw_spinunlock_bh(&pwdev_priv->connect_req_lock);
+}
+
+
+#ifdef CONFIG_AP_MODE
+static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_param *param)
+{
+	int ret = 0;
+	u32 wep_key_idx, wep_key_len;
+	struct sta_info *psta = NULL, *pbcmc_sta = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	param->u.crypt.err = 0;
+	param->u.crypt.alg[IEEE_CRYPT_ALG_NAME_LEN - 1] = '\0';
+
+	if (is_broadcast_mac_addr(param->sta_addr)) {
+		if (param->u.crypt.idx >= WEP_KEYS
+			#ifdef CONFIG_IEEE80211W
+			&& param->u.crypt.idx > BIP_MAX_KEYID
+			#endif
+		) {
+			ret = -EINVAL;
+			goto exit;
+		}
+	} else {
+		psta = rtw_get_stainfo(pstapriv, param->sta_addr);
+		if (!psta) {
+			ret = -EINVAL;
+			RTW_INFO(FUNC_ADPT_FMT", sta "MAC_FMT" not found\n"
+				, FUNC_ADPT_ARG(padapter), MAC_ARG(param->sta_addr));
+			goto exit;
+		}
+	#ifdef CONFIG_RTW_80211R_AP
+		if ((psta->authalg == WLAN_AUTH_FT) &&
+			!(psta->state & WIFI_FW_ASSOC_SUCCESS)) {
+			ret = -EINVAL;
+			RTW_INFO(FUNC_ADPT_FMT", sta "MAC_FMT
+				" not ready to setkey before assoc success!\n"
+				, FUNC_ADPT_ARG(padapter), MAC_ARG(param->sta_addr));
+			goto exit;
+		}
+	#endif
+	}
+
+	if (strcmp(param->u.crypt.alg, "none") == 0 && (psta == NULL)) {
+		/* todo:clear default encryption keys */
+
+		RTW_INFO("clear default encryption keys, keyid=%d\n", param->u.crypt.idx);
+
+		goto exit;
+	}
+
+
+	if (strcmp(param->u.crypt.alg, "WEP") == 0 && (psta == NULL)) {
+		RTW_INFO("r871x_set_encryption, crypt.alg = WEP\n");
+
+		wep_key_idx = param->u.crypt.idx;
+		wep_key_len = param->u.crypt.key_len;
+
+		RTW_INFO("r871x_set_encryption, wep_key_idx=%d, len=%d\n", wep_key_idx, wep_key_len);
+
+		if ((wep_key_idx >= WEP_KEYS) || (wep_key_len <= 0)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+
+		if (wep_key_len > 0)
+			wep_key_len = wep_key_len <= WLAN_KEY_LEN_WEP40 ? WLAN_KEY_LEN_WEP40 : WLAN_KEY_LEN_WEP104;
+
+		if (psecuritypriv->bWepDefaultKeyIdxSet == 0) {
+			/* wep default key has not been set, so use this key index as default key. */
+
+			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+			psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+			psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
+
+			if (wep_key_len == WLAN_KEY_LEN_WEP104) {
+				psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+				psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
+			}
+
+			psecuritypriv->dot11PrivacyKeyIndex = wep_key_idx;
+		}
+
+		_rtw_memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), param->u.crypt.key, wep_key_len);
+
+		psecuritypriv->dot11DefKeylen[wep_key_idx] = wep_key_len;
+
+		rtw_ap_set_wep_key(padapter, padapter_link,
+				param->u.crypt.key, wep_key_len, wep_key_idx, 1);
+
+		goto exit;
+
+	}
+
+	if (!psta) { /* group key */
+		if (param->u.crypt.set_tx == 0) { /* group key, TX only */
+			if (strcmp(param->u.crypt.alg, "WEP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set WEP TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				if (param->u.crypt.key_len == 13)
+					psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
+
+			} else if (strcmp(param->u.crypt.alg, "TKIP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set TKIP TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _TKIP_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				/* set mic key */
+				_rtw_memcpy(lsecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
+				_rtw_memcpy(lsecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+				psecuritypriv->busetkipkey = _TRUE;
+
+			} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CCMP TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _AES_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+
+			} else if (strcmp(param->u.crypt.alg, "GCMP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GCMP TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _GCMP_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,
+					param->u.crypt.key,
+					(param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+
+			} else if (strcmp(param->u.crypt.alg, "GCMP_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GCMP_256 TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _GCMP_256_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,
+					param->u.crypt.key,
+					(param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+
+			} else if (strcmp(param->u.crypt.alg, "CCMP_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CCMP_256 TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _CCMP_256_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,
+					param->u.crypt.key,
+					(param->u.crypt.key_len > 32 ? 32: param->u.crypt.key_len));
+
+			#ifdef CONFIG_IEEE80211W
+			} else if (strcmp(param->u.crypt.alg, "BIP") == 0) {
+				psecuritypriv->dot11wCipher = _BIP_CMAC_128_;
+				RTW_INFO(FUNC_ADPT_FMT" set TX CMAC-128 IGTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+				lsecuritypriv->dot11wBIPtxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				lsecuritypriv->binstallBIPkey = _TRUE;
+				rtw_ap_set_group_key(padapter, padapter_link, param->u.crypt.key, psecuritypriv->dot11wCipher, param->u.crypt.idx);
+				goto exit;
+			} else if (strcmp(param->u.crypt.alg, "BIP_GMAC_128") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set TX GMAC-128 IGTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot11wCipher = _BIP_GMAC_128_;
+				_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,
+					param->u.crypt.key, param->u.crypt.key_len);
+				lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+				lsecuritypriv->dot11wBIPtxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				lsecuritypriv->binstallBIPkey = _TRUE;
+				goto exit;
+			} else if (strcmp(param->u.crypt.alg, "BIP_GMAC_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set TX GMAC-256 IGTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot11wCipher = _BIP_GMAC_256_;
+				_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,
+					param->u.crypt.key, param->u.crypt.key_len);
+				lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+				lsecuritypriv->dot11wBIPtxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				lsecuritypriv->binstallBIPkey = _TRUE;
+				goto exit;
+			} else if (strcmp(param->u.crypt.alg, "BIP_CMAC_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set TX CMAC-256 IGTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot11wCipher = _BIP_CMAC_256_;
+				_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,
+					param->u.crypt.key, param->u.crypt.key_len);
+				lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+				lsecuritypriv->dot11wBIPtxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				lsecuritypriv->binstallBIPkey = _TRUE;
+				goto exit;
+			#endif /* CONFIG_IEEE80211W */
+
+			} else if (strcmp(param->u.crypt.alg, "none") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" clear group key, idx:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx);
+				psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
+			} else {
+				RTW_WARN(FUNC_ADPT_FMT" set group key, not support\n"
+					, FUNC_ADPT_ARG(padapter));
+				goto exit;
+			}
+
+			lsecuritypriv->dot118021XGrpKeyid = param->u.crypt.idx;
+			pbcmc_sta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+			if (pbcmc_sta) {
+				pbcmc_sta->dot11txpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				pbcmc_sta->ieee8021x_blocked = _FALSE;
+				pbcmc_sta->dot118021XPrivacy = psecuritypriv->dot118021XGrpPrivacy; /* rx will use bmc_sta's dot118021XPrivacy			 */
+			}
+			lsecuritypriv->binstallGrpkey = _TRUE;
+			psecuritypriv->dot11PrivacyAlgrthm = psecuritypriv->dot118021XGrpPrivacy;/* !!! */
+
+			rtw_ap_set_group_key(padapter, padapter_link, param->u.crypt.key, psecuritypriv->dot118021XGrpPrivacy, param->u.crypt.idx);
+		}
+
+		goto exit;
+
+	}
+
+	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X && psta) { /* psk/802_1x */
+		if (param->u.crypt.set_tx == 1) {
+			u8 iv[IV_LENGTH];
+
+			/* pairwise key */
+			if (param->u.crypt.key_len == 32)
+				_rtw_memcpy(psta->dot118021x_UncstKey.skey,
+						param->u.crypt.key,
+						(param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+			else
+				_rtw_memcpy(psta->dot118021x_UncstKey.skey,
+						param->u.crypt.key,
+						(param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+
+			if (strcmp(param->u.crypt.alg, "WEP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set WEP PTK of "MAC_FMT" idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot118021XPrivacy = _WEP40_;
+				if (param->u.crypt.key_len == 13)
+					psta->dot118021XPrivacy = _WEP104_;
+
+			} else if (strcmp(param->u.crypt.alg, "TKIP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set TKIP PTK of "MAC_FMT" idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot118021XPrivacy = _TKIP_;
+				/* set mic key */
+				_rtw_memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
+				_rtw_memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+				psecuritypriv->busetkipkey = _TRUE;
+
+			} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CCMP PTK of "MAC_FMT" idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot118021XPrivacy = _AES_;
+
+			} else if (strcmp(param->u.crypt.alg, "GCMP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GCMP PTK of "MAC_FMT" idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot118021XPrivacy = _GCMP_;
+
+			} else if (strcmp(param->u.crypt.alg, "GCMP_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GCMP_256 PTK of "MAC_FMT" idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot118021XPrivacy = _GCMP_256_;
+
+			} else if (strcmp(param->u.crypt.alg, "CCMP_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CCMP_256 PTK of "MAC_FMT" idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot118021XPrivacy = _CCMP_256_;
+
+			} else if (strcmp(param->u.crypt.alg, "none") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" clear pairwise key of "MAC_FMT" idx:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx);
+				psta->dot118021XPrivacy = _NO_PRIVACY_;
+			} else {
+				RTW_WARN(FUNC_ADPT_FMT" set pairwise key of "MAC_FMT", not support\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr));
+				goto exit;
+			}
+
+			psta->dot11txpn.val = RTW_GET_LE64(param->u.crypt.seq);
+			psta->dot11rxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+			if (rtw_pn_to_iv(param->u.crypt.seq,
+			    iv, param->u.crypt.idx,
+			    padapter->securitypriv.dot11PrivacyAlgrthm)) {
+				struct stainfo_rxcache *prxcache = &psta->sta_recvpriv.rxcache;
+				int i;
+
+				for (i = 0; i < RTW_MAX_TID_NUM; i++)
+					_rtw_memcpy(prxcache->iv[i], iv, IV_LENGTH);
+			}
+			psta->ieee8021x_blocked = _FALSE;
+
+			if (psta->dot118021XPrivacy != _NO_PRIVACY_) {
+				psta->bpairwise_key_installed = _TRUE;
+
+				/* WPA2 key-handshake has completed */
+				if (psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPA2PSK)
+					psta->state &= (~WIFI_UNDER_KEY_HANDSHAKE);
+			}
+
+			rtw_ap_set_pairwise_key(padapter, psta);
+		} else {
+			/* peer's group key, RX only */
+			#ifdef CONFIG_RTW_MESH
+			if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CCMP GTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->group_privacy = _AES_;
+				_rtw_memcpy(psta->gtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				psta->gtk_bmp |= BIT(param->u.crypt.idx);
+				psta->gtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+
+			} else if (strcmp(param->u.crypt.alg, "GCMP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GCMP GTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->group_privacy = _GCMP_;
+				_rtw_memcpy(psta->gtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				psta->gtk_bmp |= BIT(param->u.crypt.idx);
+				psta->gtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+
+			} else if (strcmp(param->u.crypt.alg, "CCMP_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CCMP_256 GTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->group_privacy = _CCMP_256_;
+				_rtw_memcpy(psta->gtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+				psta->gtk_bmp |= BIT(param->u.crypt.idx);
+				psta->gtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+
+			} else if (strcmp(param->u.crypt.alg, "GCMP_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GCMP_256 GTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->group_privacy = _GCMP_256_;
+				_rtw_memcpy(psta->gtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+				psta->gtk_bmp |= BIT(param->u.crypt.idx);
+				psta->gtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+
+			#ifdef CONFIG_IEEE80211W
+			} else if (strcmp(param->u.crypt.alg, "BIP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CMAC-128 IGTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot11wCipher = _BIP_CMAC_128_;
+				_rtw_memcpy(psta->igtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				psta->igtk_bmp |= BIT(param->u.crypt.idx);
+				psta->igtk_id = param->u.crypt.idx;
+				psta->igtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+				goto exit;
+
+			} else if (strcmp(param->u.crypt.alg, "BIP_GMAC_128") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GMAC-128 IGTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot11wCipher = _BIP_GMAC_128_;
+				_rtw_memcpy(psta->igtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				psta->igtk_bmp |= BIT(param->u.crypt.idx);
+				psta->igtk_id = param->u.crypt.idx;
+				psta->igtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+				goto exit;
+
+			} else if (strcmp(param->u.crypt.alg, "BIP_CMAC_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CMAC-256 IGTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot11wCipher = _BIP_CMAC_256_;
+				_rtw_memcpy(psta->igtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+				psta->igtk_bmp |= BIT(param->u.crypt.idx);
+				psta->igtk_id = param->u.crypt.idx;
+				psta->igtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+				goto exit;
+
+			} else if (strcmp(param->u.crypt.alg, "BIP_GMAC_256") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set GMAC-256 IGTK of "MAC_FMT", idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx, param->u.crypt.key_len);
+				psta->dot11wCipher = _BIP_GMAC_256_;
+				_rtw_memcpy(psta->igtk.skey, param->u.crypt.key, (param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+				psta->igtk_bmp |= BIT(param->u.crypt.idx);
+				psta->igtk_id = param->u.crypt.idx;
+				psta->igtk_pn.val = RTW_GET_LE64(param->u.crypt.seq);
+				goto exit;
+			#endif /* CONFIG_IEEE80211W */
+
+			} else if (strcmp(param->u.crypt.alg, "none") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" clear group key of "MAC_FMT", idx:%u\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+					, param->u.crypt.idx);
+				psta->group_privacy = _NO_PRIVACY_;
+				psta->gtk_bmp &= ~BIT(param->u.crypt.idx);
+			} else
+			#endif /* CONFIG_RTW_MESH */
+			{
+				RTW_WARN(FUNC_ADPT_FMT" set group key of "MAC_FMT", not support\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr));
+				goto exit;
+			}
+
+			#ifdef CONFIG_RTW_MESH
+			rtw_ap_set_sta_key(padapter, psta->phl_sta->mac_addr, psta->group_privacy
+				, param->u.crypt.key, param->u.crypt.idx, 1);
+			#endif
+		}
+
+	}
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_AP_MODE */
+
+static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param *param)
+{
+	int ret = 0;
+	u32 wep_key_idx, wep_key_len;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv *lmlmepriv = &padapter_link->mlmepriv;
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+#endif /* CONFIG_P2P */
+
+	RTW_INFO("%s\n", __func__);
+
+	param->u.crypt.err = 0;
+	param->u.crypt.alg[IEEE_CRYPT_ALG_NAME_LEN - 1] = '\0';
+
+	if (is_broadcast_mac_addr(param->sta_addr)) {
+		if (param->u.crypt.idx >= WEP_KEYS
+			#ifdef CONFIG_IEEE80211W
+			&& param->u.crypt.idx > BIP_MAX_KEYID
+			#endif
+		) {
+			ret = -EINVAL;
+			goto exit;
+		}
+	} else {
+#ifdef CONFIG_WAPI_SUPPORT
+		if (strcmp(param->u.crypt.alg, "SMS4"))
+#endif
+		{
+			ret = -EINVAL;
+			goto exit;
+		}
+	}
+
+	if (strcmp(param->u.crypt.alg, "WEP") == 0) {
+		RTW_INFO("wpa_set_encryption, crypt.alg = WEP\n");
+
+		wep_key_idx = param->u.crypt.idx;
+		wep_key_len = param->u.crypt.key_len;
+
+		if ((wep_key_idx >= WEP_KEYS) || (wep_key_len <= 0)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+
+		if (psecuritypriv->bWepDefaultKeyIdxSet == 0) {
+			/* wep default key has not been set, so use this key index as default key. */
+
+			wep_key_len = wep_key_len <= 5 ? 5 : 13;
+
+			psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+			psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
+
+			if (wep_key_len == 13) {
+				psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+				psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
+			}
+
+			psecuritypriv->dot11PrivacyKeyIndex = wep_key_idx;
+		}
+
+		_rtw_memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), param->u.crypt.key, wep_key_len);
+
+		psecuritypriv->dot11DefKeylen[wep_key_idx] = wep_key_len;
+
+		rtw_set_key(padapter, padapter_link, wep_key_idx, 0, _TRUE);
+
+		goto exit;
+	}
+
+	if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) { /* 802_1x */
+		struct sta_info *psta, *pbcmc_sta;
+		struct sta_priv *pstapriv = &padapter->stapriv;
+
+		/* RTW_INFO("%s, : dot11AuthAlgrthm == dot11AuthAlgrthm_8021X\n", __func__); */
+
+		if (MLME_IS_STA(padapter) || MLME_IS_MP(padapter)) {/* sta mode */
+#ifdef CONFIG_RTW_80211R
+			if (rtw_ft_roam(padapter))
+				psta = rtw_get_stainfo(pstapriv, pmlmepriv->assoc_bssid);
+			else
+#endif
+				psta = rtw_get_stainfo(pstapriv, get_link_bssid(lmlmepriv));
+			if (psta == NULL) {
+				/* DEBUG_ERR( ("Set wpa_set_encryption: Obtain Sta_info fail\n")); */
+				RTW_INFO("%s, : Obtain Sta_info fail\n", __func__);
+			} else {
+				/* Jeff: don't disable ieee8021x_blocked while clearing key */
+				if (strcmp(param->u.crypt.alg, "none") != 0)
+					psta->ieee8021x_blocked = _FALSE;
+
+				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled) ||
+				    (padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled))
+					psta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+
+				if (param->u.crypt.set_tx == 1) { /* pairwise key */
+					u8 iv[IV_LENGTH];
+
+					RTW_INFO(FUNC_ADPT_FMT" set %s PTK idx:%u, len:%u\n"
+						, FUNC_ADPT_ARG(padapter), param->u.crypt.alg, param->u.crypt.idx, param->u.crypt.key_len);
+
+					if (strcmp(param->u.crypt.alg, "GCMP_256") == 0
+						|| strcmp(param->u.crypt.alg, "CCMP_256") == 0) {
+						_rtw_memcpy(psta->dot118021x_UncstKey.skey,
+							param->u.crypt.key,
+							((param->u.crypt.key_len > 32) ?
+								32 : param->u.crypt.key_len));
+					} else
+						_rtw_memcpy(psta->dot118021x_UncstKey.skey,
+							param->u.crypt.key,
+							(param->u.crypt.key_len > 16 ?
+								16 : param->u.crypt.key_len));
+
+					if (strcmp(param->u.crypt.alg, "TKIP") == 0) { /* set mic key */
+						_rtw_memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
+						_rtw_memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+						padapter->securitypriv.busetkipkey = _FALSE;
+					}
+					psta->dot11txpn.val = RTW_GET_LE64(param->u.crypt.seq);
+					psta->dot11rxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+					if (rtw_pn_to_iv(param->u.crypt.seq,
+					    iv, param->u.crypt.idx,
+					    padapter->securitypriv.dot11PrivacyAlgrthm)) {
+						struct stainfo_rxcache *prxcache = &psta->sta_recvpriv.rxcache;
+						int i;
+
+						for (i = 0; i < RTW_MAX_TID_NUM; i++)
+							_rtw_memcpy(prxcache->iv[i], iv, IV_LENGTH);
+					}
+					psta->bpairwise_key_installed = _TRUE;
+					#ifdef CONFIG_RTW_80211R
+					psta->ft_pairwise_key_installed = _TRUE;
+					#endif
+					rtw_setstakey_cmd(padapter, psta, UNICAST_KEY, _TRUE);
+
+				} else { /* group key */
+					if (strcmp(param->u.crypt.alg, "TKIP") == 0
+						|| strcmp(param->u.crypt.alg, "CCMP") == 0
+						|| strcmp(param->u.crypt.alg, "GCMP") == 0) {
+						RTW_INFO(FUNC_ADPT_FMT" set %s GTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.alg, param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,
+							param->u.crypt.key,
+							(param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+						_rtw_memcpy(lsecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
+						_rtw_memcpy(lsecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+						lsecuritypriv->binstallGrpkey = _TRUE;
+						if (param->u.crypt.idx < 4)
+							_rtw_memcpy(lsecuritypriv->iv_seq[param->u.crypt.idx], param->u.crypt.seq, 8);
+						lsecuritypriv->dot118021XGrpKeyid = param->u.crypt.idx;
+						rtw_set_key(padapter, padapter_link, param->u.crypt.idx, 1, _TRUE);
+					} else if (strcmp(param->u.crypt.alg, "GCMP_256") == 0
+						|| strcmp(param->u.crypt.alg, "CCMP_256") == 0) {
+						RTW_INFO(FUNC_ADPT_FMT" set %s GTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.alg, param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(
+							lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,
+							param->u.crypt.key,
+							(param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+						lsecuritypriv->binstallGrpkey = _TRUE;
+						lsecuritypriv->dot118021XGrpKeyid = param->u.crypt.idx;
+						rtw_set_key(padapter, padapter_link, param->u.crypt.idx, 1, _TRUE);
+					#ifdef CONFIG_IEEE80211W
+					} else if (strcmp(param->u.crypt.alg, "BIP") == 0) {
+						psecuritypriv->dot11wCipher = _BIP_CMAC_128_;
+						RTW_INFO(FUNC_ADPT_FMT" set CMAC-128 IGTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,
+							param->u.crypt.key,
+							(param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+						lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+						lsecuritypriv->dot11wBIPrxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+						lsecuritypriv->binstallBIPkey = _TRUE;
+						rtw_set_key(padapter, padapter_link, param->u.crypt.idx, 1, _TRUE);
+					} else if (strcmp(param->u.crypt.alg, "BIP_GMAC_128") == 0) {
+						psecuritypriv->dot11wCipher = _BIP_GMAC_128_;
+						RTW_INFO(FUNC_ADPT_FMT" set GMAC-128 IGTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,
+							param->u.crypt.key,
+							(param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+						lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+						lsecuritypriv->dot11wBIPrxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+						lsecuritypriv->binstallBIPkey = _TRUE;
+					} else if (strcmp(param->u.crypt.alg, "BIP_GMAC_256") == 0) {
+						psecuritypriv->dot11wCipher = _BIP_GMAC_256_;
+						RTW_INFO(FUNC_ADPT_FMT" set GMAC-256 IGTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,
+							param->u.crypt.key,
+							(param->u.crypt.key_len > 32 ? 32 : param->u.crypt.key_len));
+						lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+						lsecuritypriv->dot11wBIPrxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+						lsecuritypriv->binstallBIPkey = _TRUE;
+					} else if (strcmp(param->u.crypt.alg, "BIP_CMAC_256") == 0) {
+						psecuritypriv->dot11wCipher = _BIP_CMAC_256_;
+						RTW_INFO(FUNC_ADPT_FMT" set CMAC-256 IGTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,
+							param->u.crypt.key, param->u.crypt.key_len);
+						lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+						lsecuritypriv->dot11wBIPrxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+						lsecuritypriv->binstallBIPkey = _TRUE;
+					#endif /* CONFIG_IEEE80211W */
+
+					}
+
+					/* WPA/WPA2 key-handshake has completed */
+					clr_fwstate(pmlmepriv, WIFI_UNDER_KEY_HANDSHAKE);
+
+				}
+			}
+
+			pbcmc_sta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+			if (pbcmc_sta == NULL) {
+				/* DEBUG_ERR( ("Set OID_802_11_ADD_KEY: bcmc stainfo is null\n")); */
+			} else {
+				/* Jeff: don't disable ieee8021x_blocked while clearing key */
+				if (strcmp(param->u.crypt.alg, "none") != 0)
+					pbcmc_sta->ieee8021x_blocked = _FALSE;
+
+				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled) ||
+				    (padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled))
+					pbcmc_sta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+			}
+		} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) { /* adhoc mode */
+		}
+	}
+
+	#ifdef CONFIG_WAPI_SUPPORT
+	if (strcmp(param->u.crypt.alg, "SMS4") == 0)
+		rtw_wapi_set_set_encryption(padapter, param);
+	#endif
+
+exit:
+
+	RTW_INFO("%s, ret=%d\n", __func__, ret);
+
+
+	return ret;
+}
+
+static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	, int link_id
+#endif
+	, u8 key_index
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	, bool pairwise
+#endif
+	, const u8 *mac_addr, struct key_params *params)
+{
+	char *alg_name;
+	u32 param_len;
+	struct ieee_param *param = NULL;
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct wireless_dev *rtw_wdev = padapter->rtw_wdev;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+#ifdef CONFIG_TDLS
+	struct sta_info *ptdls_sta;
+#endif /* CONFIG_TDLS */
+
+	if (mac_addr)
+		RTW_INFO(FUNC_NDEV_FMT" adding key for " MAC_FMT "\n", FUNC_NDEV_ARG(ndev), MAC_ARG(mac_addr));
+	RTW_INFO(FUNC_NDEV_FMT" cipher=0x%x\n", FUNC_NDEV_ARG(ndev), params->cipher);
+	RTW_INFO(FUNC_NDEV_FMT" key_len=%d, key_index=%d\n", FUNC_NDEV_ARG(ndev), params->key_len, key_index);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	RTW_INFO(FUNC_NDEV_FMT" pairwise=%d\n", FUNC_NDEV_ARG(ndev), pairwise);
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	RTW_INFO(FUNC_NDEV_FMT" link_id=%d\n", FUNC_NDEV_ARG(ndev), link_id);
+#endif
+
+	if (rtw_cfg80211_sync_iftype(padapter) != _SUCCESS) {
+		ret = -ENOTSUPP;
+		goto addkey_end;
+	}
+
+	param_len = sizeof(struct ieee_param) + params->key_len;
+	param = rtw_malloc(param_len);
+	if (param == NULL)
+		return -1;
+
+	_rtw_memset(param, 0, param_len);
+
+	param->cmd = IEEE_CMD_SET_ENCRYPTION;
+	_rtw_memset(param->sta_addr, 0xff, ETH_ALEN);
+
+	switch (params->cipher) {
+	case IW_AUTH_CIPHER_NONE:
+		/* todo: remove key */
+		/* remove = 1;	 */
+		alg_name = "none";
+		break;
+	case WLAN_CIPHER_SUITE_WEP40:
+	case WLAN_CIPHER_SUITE_WEP104:
+		alg_name = "WEP";
+		break;
+	case WLAN_CIPHER_SUITE_TKIP:
+		alg_name = "TKIP";
+		break;
+	case WLAN_CIPHER_SUITE_CCMP:
+		alg_name = "CCMP";
+		break;
+	case WIFI_CIPHER_SUITE_GCMP:
+		alg_name = "GCMP";
+		break;
+	case WIFI_CIPHER_SUITE_GCMP_256:
+		alg_name = "GCMP_256";
+		break;
+	case WIFI_CIPHER_SUITE_CCMP_256:
+		alg_name = "CCMP_256";
+		break;
+#ifdef CONFIG_IEEE80211W
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+		alg_name = "BIP";
+		break;
+	case WIFI_CIPHER_SUITE_BIP_GMAC_128:
+		alg_name = "BIP_GMAC_128";
+		break;
+	case WIFI_CIPHER_SUITE_BIP_GMAC_256:
+		alg_name = "BIP_GMAC_256";
+		break;
+	case WIFI_CIPHER_SUITE_BIP_CMAC_256:
+		alg_name = "BIP_CMAC_256";
+		break;
+#endif /* CONFIG_IEEE80211W */
+#ifdef CONFIG_WAPI_SUPPORT
+	case WLAN_CIPHER_SUITE_SMS4:
+		alg_name = "SMS4";
+		if (pairwise == NL80211_KEYTYPE_PAIRWISE) {
+			if (key_index != 0 && key_index != 1) {
+				ret = -ENOTSUPP;
+				goto addkey_end;
+			}
+			_rtw_memcpy((void *)param->sta_addr, (void *)mac_addr, ETH_ALEN);
+		} else
+			RTW_INFO("mac_addr is null\n");
+		RTW_INFO("rtw_wx_set_enc_ext: SMS4 case\n");
+		break;
+#endif
+
+	default:
+		ret = -ENOTSUPP;
+		goto addkey_end;
+	}
+
+	strncpy((char *)param->u.crypt.alg, alg_name, IEEE_CRYPT_ALG_NAME_LEN);
+
+
+	if (!mac_addr || is_broadcast_ether_addr(mac_addr)
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+		|| !pairwise
+		#endif
+	) {
+		param->u.crypt.set_tx = 0; /* for wpa/wpa2 group key */
+	} else {
+		param->u.crypt.set_tx = 1; /* for wpa/wpa2 pairwise key */
+	}
+
+	param->u.crypt.idx = key_index;
+
+	if (params->seq_len && params->seq) {
+		_rtw_memcpy(param->u.crypt.seq, (u8 *)params->seq, params->seq_len);
+		RTW_INFO(FUNC_NDEV_FMT" seq_len:%u, seq:0x%llx\n", FUNC_NDEV_ARG(ndev)
+			, params->seq_len, RTW_GET_LE64(param->u.crypt.seq));
+	}
+
+	if (params->key_len && params->key) {
+		param->u.crypt.key_len = params->key_len;
+		_rtw_memcpy(param->u.crypt.key, (u8 *)params->key, params->key_len);
+	}
+
+	if (MLME_IS_STA(padapter)) {
+#ifdef CONFIG_TDLS
+		if (rtw_tdls_is_driver_setup(padapter) == _FALSE && mac_addr) {
+			ptdls_sta = rtw_get_stainfo(&padapter->stapriv, (void *)mac_addr);
+			if (ptdls_sta != NULL && ptdls_sta->tdls_sta_state) {
+				_rtw_memcpy(ptdls_sta->tpk.tk, params->key, params->key_len);
+				goto addkey_end;
+			}
+		}
+#endif /* CONFIG_TDLS */
+		ret = rtw_cfg80211_set_encryption(ndev, param);
+	} else if (MLME_IS_AP(padapter) || MLME_IS_MESH(padapter)) {
+#ifdef CONFIG_AP_MODE
+		if (mac_addr)
+			_rtw_memcpy(param->sta_addr, (void *)mac_addr, ETH_ALEN);
+
+		ret = rtw_cfg80211_ap_set_encryption(ndev, param);
+#endif
+	} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE
+		|| check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE
+	) {
+		/* RTW_INFO("@@@@@@@@@@ fw_state=0x%x, iftype=%d\n", pmlmepriv->fw_state, rtw_wdev->iftype); */
+		ret = rtw_cfg80211_set_encryption(ndev, param);
+	} else
+		RTW_INFO("error! fw_state=0x%x, iftype=%d\n", pmlmepriv->fw_state, rtw_wdev->iftype);
+
+
+addkey_end:
+	if (param)
+		rtw_mfree(param, param_len);
+
+	return ret;
+
+}
+
+static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	, int link_id
+#endif
+	, u8 keyid
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	, bool pairwise
+#endif
+	, const u8 *mac_addr, void *cookie
+	, void (*callback)(void *cookie, struct key_params *))
+{
+#define GET_KEY_PARAM_FMT_S " keyid=%d"
+#define GET_KEY_PARAM_ARG_S , keyid
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	#define GET_KEY_PARAM_FMT_2_6_37 ", pairwise=%d"
+	#define GET_KEY_PARAM_ARG_2_6_37 , pairwise
+#else
+	#define GET_KEY_PARAM_FMT_2_6_37 ""
+	#define GET_KEY_PARAM_ARG_2_6_37
+#endif
+#define GET_KEY_PARAM_FMT_E ", addr=%pM"
+#define GET_KEY_PARAM_ARG_E , mac_addr
+
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct security_priv *sec = &adapter->securitypriv;
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct sta_info *sta = NULL;
+	u32 cipher = _NO_PRIVACY_;
+	union Keytype *key = NULL;
+	u8 key_len = 0;
+	u64 *pn = NULL;
+	u8 pn_len = 0;
+	u8 pn_val[8] = {0};
+
+	struct key_params params;
+	int ret = -ENOENT;
+
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_security_priv *lsec = &adapter_link->securitypriv;
+
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	RTW_INFO(FUNC_NDEV_FMT" link_id=%d\n", FUNC_NDEV_ARG(ndev), link_id);
+#endif
+
+	if (keyid >= WEP_KEYS
+		#ifdef CONFIG_IEEE80211W
+		&& keyid > BIP_MAX_KEYID
+		#endif
+	)
+		goto exit;
+
+	if (!mac_addr || is_broadcast_ether_addr(mac_addr)
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+		|| (MLME_IS_STA(adapter) && !pairwise)
+		#endif
+	) {
+		/* WEP key, TX GTK/IGTK, RX GTK/IGTK(for STA mode) */
+		if (is_wep_enc(sec->dot118021XGrpPrivacy))
+		{
+			if (keyid >= WEP_KEYS)
+				goto exit;
+			if (!(sec->key_mask & BIT(keyid)))
+				goto exit;
+			cipher = sec->dot118021XGrpPrivacy;
+			key = &sec->dot11DefKey[keyid];
+		} else {
+			if (keyid < WEP_KEYS) {
+				if (lsec->binstallGrpkey != _TRUE)
+					goto exit;
+				cipher = sec->dot118021XGrpPrivacy;
+				key = &lsec->dot118021XGrpKey[keyid];
+				sta = rtw_get_bcmc_stainfo(adapter, adapter_link);
+				if (sta)
+					pn = &sta->dot11txpn.val;
+			#ifdef CONFIG_IEEE80211W
+			} else if (keyid <= BIP_MAX_KEYID) {
+				if (SEC_IS_BIP_KEY_INSTALLED(lsec) != _TRUE)
+					goto exit;
+				cipher = sec->dot11wCipher;
+				key = &lsec->dot11wBIPKey[keyid];
+				pn = &lsec->dot11wBIPtxpn.val;
+			#endif
+			}
+		}
+	} else {
+		/* Pairwise key, RX GTK/IGTK for specific peer */
+		sta = rtw_get_stainfo(stapriv, mac_addr);
+		if (!sta)
+			goto exit;
+
+		if (keyid < WEP_KEYS && pairwise) {
+			if (sta->bpairwise_key_installed != _TRUE)
+				goto exit;
+			cipher = sta->dot118021XPrivacy;
+			key = &sta->dot118021x_UncstKey;
+		#ifdef CONFIG_RTW_MESH
+		} else if (keyid < WEP_KEYS && !pairwise) {
+			if (!(sta->gtk_bmp & BIT(keyid)))
+				goto exit;
+			cipher = sta->group_privacy;
+			key = &sta->gtk;
+		#ifdef CONFIG_IEEE80211W
+		} else if (keyid <= BIP_MAX_KEYID && !pairwise) {
+			if (!(sta->igtk_bmp & BIT(keyid)))
+				goto exit;
+			cipher = sta->dot11wCipher;
+			key = &sta->igtk;
+			pn = &sta->igtk_pn.val;
+		#endif
+		#endif /* CONFIG_RTW_MESH */
+		}
+	}
+
+	if (!key)
+		goto exit;
+
+	if (cipher == _WEP40_) {
+		cipher = WLAN_CIPHER_SUITE_WEP40;
+		key_len = sec->dot11DefKeylen[keyid];
+	} else if (cipher == _WEP104_) {
+		cipher = WLAN_CIPHER_SUITE_WEP104;
+		key_len = sec->dot11DefKeylen[keyid];
+	} else if (cipher == _TKIP_ || cipher == _TKIP_WTMIC_) {
+		cipher = WLAN_CIPHER_SUITE_TKIP;
+		key_len = 16;
+	} else if (cipher == _AES_) {
+		cipher = WLAN_CIPHER_SUITE_CCMP;
+		key_len = 16;
+#ifdef CONFIG_WAPI_SUPPORT
+	} else if (cipher == _SMS4_) {
+		cipher = WLAN_CIPHER_SUITE_SMS4;
+		key_len = 16;
+#endif
+	} else if (cipher == _GCMP_) {
+		cipher = WIFI_CIPHER_SUITE_GCMP;
+		key_len = 16;
+	} else if (cipher == _CCMP_256_) {
+		cipher = WIFI_CIPHER_SUITE_CCMP_256;
+		key_len = 32;
+	} else if (cipher == _GCMP_256_) {
+		cipher = WIFI_CIPHER_SUITE_GCMP_256;
+		key_len = 32;
+	#ifdef CONFIG_IEEE80211W
+	} else if (cipher == _BIP_CMAC_128_) {
+		cipher = WLAN_CIPHER_SUITE_AES_CMAC;
+		key_len = 16;
+	} else if (cipher == _BIP_GMAC_128_) {
+		cipher = WIFI_CIPHER_SUITE_BIP_GMAC_128;
+		key_len = 16;
+	} else if (cipher == _BIP_GMAC_256_) {
+		cipher = WIFI_CIPHER_SUITE_BIP_GMAC_256;
+		key_len = 32;
+	} else if (cipher == _BIP_CMAC_256_) {
+		cipher = WIFI_CIPHER_SUITE_BIP_CMAC_256;
+		key_len = 32;
+	#endif
+	} else {
+		RTW_WARN(FUNC_NDEV_FMT" unknown cipher:%u\n", FUNC_NDEV_ARG(ndev), cipher);
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (pn) {
+		*((u64 *)pn_val) = cpu_to_le64(*pn);
+		pn_len = 6;
+	}
+
+	ret = 0;
+	
+exit:
+	RTW_INFO(FUNC_NDEV_FMT
+		GET_KEY_PARAM_FMT_S
+		GET_KEY_PARAM_FMT_2_6_37
+		GET_KEY_PARAM_FMT_E
+		" ret %d\n", FUNC_NDEV_ARG(ndev)
+		GET_KEY_PARAM_ARG_S
+		GET_KEY_PARAM_ARG_2_6_37
+		GET_KEY_PARAM_ARG_E
+		, ret);
+	if (pn)
+		RTW_INFO(FUNC_NDEV_FMT " seq:0x%llx\n", FUNC_NDEV_ARG(ndev), *pn);
+
+	if (ret == 0) {
+		_rtw_memset(&params, 0, sizeof(params));
+
+		params.cipher = cipher;
+		params.key = key->skey;
+		params.key_len = key_len;
+		if (pn) {
+			params.seq = pn_val;
+			params.seq_len = pn_len;
+		}
+
+		callback(cookie, &params);
+	}
+
+	return ret;
+}
+
+static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct net_device *ndev
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	, int link_id
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	, u8 key_index, bool pairwise, const u8 *mac_addr)
+#else /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) */
+	, u8 key_index, const u8 *mac_addr)
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) */
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+
+	if(mac_addr){
+		RTW_INFO(FUNC_NDEV_FMT" key_index=%d, addr=" MAC_FMT "\n", FUNC_NDEV_ARG(ndev), key_index, MAC_ARG(mac_addr));
+	}else{
+		RTW_INFO(FUNC_NDEV_FMT" key_index=%d, addr=%pM\n", FUNC_NDEV_ARG(ndev), key_index, mac_addr);
+	}
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	RTW_INFO(FUNC_NDEV_FMT" link_id=%d\n", FUNC_NDEV_ARG(ndev), link_id);
+#endif
+
+	if (key_index == psecuritypriv->dot11PrivacyKeyIndex) {
+		/* clear the flag of wep default key set. */
+		psecuritypriv->bWepDefaultKeyIdxSet = 0;
+	}
+
+	return 0;
+}
+
+static int cfg80211_rtw_set_default_key(struct wiphy *wiphy, struct net_device *ndev
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	, int link_id
+#endif
+	, u8 key_index
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) || defined(COMPAT_KERNEL_RELEASE)
+	, bool unicast, bool multicast
+	#endif
+)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+
+#define SET_DEF_KEY_PARAM_FMT " key_index=%d"
+#define SET_DEF_KEY_PARAM_ARG , key_index
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) || defined(COMPAT_KERNEL_RELEASE)
+	#define SET_DEF_KEY_PARAM_FMT_2_6_38 ", unicast=%d, multicast=%d"
+	#define SET_DEF_KEY_PARAM_ARG_2_6_38 , unicast, multicast
+#else
+	#define SET_DEF_KEY_PARAM_FMT_2_6_38 ""
+	#define SET_DEF_KEY_PARAM_ARG_2_6_38
+#endif
+
+	RTW_INFO(FUNC_NDEV_FMT
+		SET_DEF_KEY_PARAM_FMT
+		SET_DEF_KEY_PARAM_FMT_2_6_38
+		"\n", FUNC_NDEV_ARG(ndev)
+		SET_DEF_KEY_PARAM_ARG
+		SET_DEF_KEY_PARAM_ARG_2_6_38
+	);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	RTW_INFO(FUNC_NDEV_FMT" link_id=%d\n", FUNC_NDEV_ARG(ndev), link_id);
+#endif
+
+	/* ToDo CONFIG_RTW_MLD */
+	if ((key_index < WEP_KEYS) && ((psecuritypriv->dot11PrivacyAlgrthm == _WEP40_) || (psecuritypriv->dot11PrivacyAlgrthm == _WEP104_))) { /* set wep default key */
+		psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
+
+		psecuritypriv->dot11PrivacyKeyIndex = key_index;
+
+		psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+		psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
+		if (psecuritypriv->dot11DefKeylen[key_index] == WLAN_KEY_LEN_WEP104) {
+			psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+			psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
+		}
+
+		psecuritypriv->bWepDefaultKeyIdxSet = 1; /* set the flag to represent that wep default key has been set */
+	}
+
+	return 0;
+
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30))
+int cfg80211_rtw_set_default_mgmt_key(struct wiphy *wiphy, struct net_device *ndev
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	, int link_id
+#endif
+	, u8 key_index)
+{
+#define SET_DEF_KEY_PARAM_FMT " key_index=%d"
+#define SET_DEF_KEY_PARAM_ARG , key_index
+
+	RTW_INFO(FUNC_NDEV_FMT
+		SET_DEF_KEY_PARAM_FMT
+		"\n", FUNC_NDEV_ARG(ndev)
+		SET_DEF_KEY_PARAM_ARG
+	);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	RTW_INFO(FUNC_NDEV_FMT" link_id=%d\n", FUNC_NDEV_ARG(ndev), link_id);
+#endif
+
+	/* ToDo CONFIG_RTW_MLD */
+	return 0;
+}
+#endif
+
+#if defined(CONFIG_GTK_OL) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0))
+static int cfg80211_rtw_set_rekey_data(struct wiphy *wiphy,
+	struct net_device *ndev,
+	struct cfg80211_gtk_rekey_data *data)
+{
+	/*int i;*/
+	struct sta_info *psta;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct mlme_priv   *pmlmepriv = &padapter->mlmepriv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+
+	psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+	if (psta == NULL) {
+		RTW_INFO("%s, : Obtain Sta_info fail\n", __func__);
+		return -1;
+	}
+
+	_rtw_memcpy(psta->kek, data->kek, NL80211_KEK_LEN);
+	/*printk("\ncfg80211_rtw_set_rekey_data KEK:");
+	for(i=0;i<NL80211_KEK_LEN; i++)
+		printk(" %02x ", psta->kek[i]);*/
+	_rtw_memcpy(psta->kck, data->kck, NL80211_KCK_LEN);
+	/*printk("\ncfg80211_rtw_set_rekey_data KCK:");
+	for(i=0;i<NL80211_KCK_LEN; i++)
+		printk(" %02x ", psta->kck[i]);*/
+	_rtw_memcpy(psta->replay_ctr, data->replay_ctr, NL80211_REPLAY_CTR_LEN);
+	psecuritypriv->binstallKCK_KEK = _TRUE;
+	/*printk("\nREPLAY_CTR: ");
+	for(i=0;i<RTW_REPLAY_CTR_LEN; i++)
+		printk(" %02x ", psta->replay_ctr[i]);*/
+
+	return 0;
+}
+#endif /*CONFIG_GTK_OL*/
+
+#ifdef CONFIG_RTW_MESH
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+static enum nl80211_mesh_power_mode rtw_mesh_ps_to_nl80211_mesh_power_mode(u8 ps)
+{
+	if (ps == RTW_MESH_PS_UNKNOWN)
+		return NL80211_MESH_POWER_UNKNOWN;
+	if (ps == RTW_MESH_PS_ACTIVE)
+		return NL80211_MESH_POWER_ACTIVE;
+	if (ps == RTW_MESH_PS_LSLEEP)
+		return NL80211_MESH_POWER_LIGHT_SLEEP;
+	if (ps == RTW_MESH_PS_DSLEEP)
+		return NL80211_MESH_POWER_DEEP_SLEEP;
+
+	rtw_warn_on(1);
+	return NL80211_MESH_POWER_UNKNOWN;
+}
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+enum nl80211_plink_state rtw_plink_state_to_nl80211_plink_state(u8 plink_state)
+{
+	if (plink_state == RTW_MESH_PLINK_UNKNOWN)
+		return NUM_NL80211_PLINK_STATES;
+	if (plink_state == RTW_MESH_PLINK_LISTEN)
+		return NL80211_PLINK_LISTEN;
+	if (plink_state == RTW_MESH_PLINK_OPN_SNT)
+		return NL80211_PLINK_OPN_SNT;
+	if (plink_state == RTW_MESH_PLINK_OPN_RCVD)
+		return NL80211_PLINK_OPN_RCVD;
+	if (plink_state == RTW_MESH_PLINK_CNF_RCVD)
+		return NL80211_PLINK_CNF_RCVD;
+	if (plink_state == RTW_MESH_PLINK_ESTAB)
+		return NL80211_PLINK_ESTAB;
+	if (plink_state == RTW_MESH_PLINK_HOLDING)
+		return NL80211_PLINK_HOLDING;
+	if (plink_state == RTW_MESH_PLINK_BLOCKED)
+		return NL80211_PLINK_BLOCKED;
+
+	rtw_warn_on(1);
+	return NUM_NL80211_PLINK_STATES;
+}
+
+u8 nl80211_plink_state_to_rtw_plink_state(enum nl80211_plink_state plink_state)
+{
+	if (plink_state == NL80211_PLINK_LISTEN)
+		return RTW_MESH_PLINK_LISTEN;
+	if (plink_state == NL80211_PLINK_OPN_SNT)
+		return RTW_MESH_PLINK_OPN_SNT;
+	if (plink_state == NL80211_PLINK_OPN_RCVD)
+		return RTW_MESH_PLINK_OPN_RCVD;
+	if (plink_state == NL80211_PLINK_CNF_RCVD)
+		return RTW_MESH_PLINK_CNF_RCVD;
+	if (plink_state == NL80211_PLINK_ESTAB)
+		return RTW_MESH_PLINK_ESTAB;
+	if (plink_state == NL80211_PLINK_HOLDING)
+		return RTW_MESH_PLINK_HOLDING;
+	if (plink_state == NL80211_PLINK_BLOCKED)
+		return RTW_MESH_PLINK_BLOCKED;
+
+	rtw_warn_on(1);
+	return RTW_MESH_PLINK_UNKNOWN;
+}
+#endif
+
+static void rtw_cfg80211_fill_mesh_only_sta_info(struct mesh_plink_ent *plink, struct sta_info *sta, struct station_info *sinfo)
+{
+	sinfo->filled |= STATION_INFO_LLID;
+	sinfo->llid = plink->llid;
+	sinfo->filled |= STATION_INFO_PLID;
+	sinfo->plid = plink->plid;
+	sinfo->filled |= STATION_INFO_PLINK_STATE;
+	sinfo->plink_state = rtw_plink_state_to_nl80211_plink_state(plink->plink_state);
+	if (!sta && plink->scanned) {
+		sinfo->filled |= STATION_INFO_SIGNAL;
+		sinfo->signal = rtw_phl_rssi_to_dbm(plink->scanned->network.PhyInfo.SignalStrength);
+		sinfo->filled |= STATION_INFO_INACTIVE_TIME;
+		if (plink->plink_state == RTW_MESH_PLINK_UNKNOWN)
+			sinfo->inactive_time = 0 - 1;
+		else
+			sinfo->inactive_time = rtw_get_passing_time_ms(plink->scanned->last_scanned);
+	}
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	if (sta) {
+		sinfo->filled |= STATION_INFO_LOCAL_PM;
+		sinfo->local_pm = rtw_mesh_ps_to_nl80211_mesh_power_mode(sta->local_mps);
+		sinfo->filled |= STATION_INFO_PEER_PM;
+		sinfo->peer_pm = rtw_mesh_ps_to_nl80211_mesh_power_mode(sta->peer_mps);
+		sinfo->filled |= STATION_INFO_NONPEER_PM;
+		sinfo->nonpeer_pm = rtw_mesh_ps_to_nl80211_mesh_power_mode(sta->nonpeer_mps);
+	}
+#endif
+}
+#endif /* CONFIG_RTW_MESH */
+
+static void rtw_desc_rate_to_nss_mcs(u16 rate_idx, u8 *rate_mode, u8 *nss, u8 *mcs)
+{
+	u8 mcs_in = 0, nss_in = 0;
+
+	/* enum rtw_data_rate */
+	if ((RTW_DATA_RATE_MCS0 <= rate_idx) &&
+	   (rate_idx <= RTW_DATA_RATE_MCS31)) {
+		*rate_mode = RTW_HT_MODE;
+		mcs_in = rate_idx - RTW_DATA_RATE_MCS0;
+	} else if ((RTW_DATA_RATE_VHT_NSS1_MCS0 <= rate_idx) &&
+		   (rate_idx <= RTW_DATA_RATE_VHT_NSS4_MCS9)) {
+		*rate_mode = RTW_VHT_MODE;
+		mcs_in = rate_idx & 0xF;
+		nss_in = ((rate_idx - RTW_DATA_RATE_VHT_NSS1_MCS0) >> 4) + 1;
+	} else if ((RTW_DATA_RATE_HE_NSS1_MCS0 <= rate_idx) &&
+		   (rate_idx <= RTW_DATA_RATE_HE_NSS4_MCS11)) {
+		*rate_mode = RTW_HE_MODE;
+		mcs_in = rate_idx & 0xF;
+		nss_in = ((rate_idx - RTW_DATA_RATE_HE_NSS1_MCS0) >> 4) + 1;
+	} else {
+		*rate_mode = RTW_LEGACY_MODE;
+	}
+
+	if (nss)
+		*nss = nss_in;
+	if (mcs)
+		*mcs = mcs_in;
+}
+
+static void sta_set_rate_info(_adapter *adapter, struct rate_info *rinfo,
+			      u16 rtw_rate_idx, u8 sgi, u8 bw)
+{
+	u8 mcs = 0;
+	u8 nss = 0;
+	u8 mod = 0;
+
+	rinfo->flags = 0;
+	rtw_desc_rate_to_nss_mcs(rtw_rate_idx, &mod, &nss, &mcs);
+
+	if (sgi)
+		rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0))
+	if (mod == RTW_HE_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_HE_MCS;
+		rinfo->bw = bw == CHANNEL_WIDTH_160 ? RATE_INFO_BW_160 :
+			    bw == CHANNEL_WIDTH_80 ? RATE_INFO_BW_80 :
+			    bw == CHANNEL_WIDTH_40 ? RATE_INFO_BW_40 : RATE_INFO_BW_20;
+		rinfo->nss = nss;
+		rinfo->mcs = mcs;
+	} else if (mod == RTW_VHT_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_VHT_MCS;
+		rinfo->bw = bw == CHANNEL_WIDTH_160 ? RATE_INFO_BW_160 :
+			    bw == CHANNEL_WIDTH_80 ? RATE_INFO_BW_80 :
+			    bw == CHANNEL_WIDTH_40 ? RATE_INFO_BW_40 : RATE_INFO_BW_20;
+		rinfo->nss = nss;
+		rinfo->mcs = mcs;
+	} else if (mod == RTW_HT_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_MCS;
+		rinfo->bw = bw ? RATE_INFO_BW_40 : RATE_INFO_BW_20;
+		rinfo->mcs = mcs;
+	}
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+	if (mod == RTW_VHT_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_VHT_MCS;
+		rinfo->bw = bw == CHANNEL_WIDTH_160 ? RATE_INFO_BW_160 :
+			    bw == CHANNEL_WIDTH_80 ? RATE_INFO_BW_80 :
+			    bw == CHANNEL_WIDTH_40 ? RATE_INFO_BW_40 : RATE_INFO_BW_20;
+		rinfo->nss = nss;
+		rinfo->mcs = mcs;
+	} else if (mod == RTW_HT_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_MCS;
+		rinfo->bw = bw ? RATE_INFO_BW_40 : RATE_INFO_BW_20;
+		rinfo->mcs = mcs;
+	}
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	if (mod == RTW_VHT_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_VHT_MCS;
+		rinfo->flags |= bw == CHANNEL_WIDTH_160 ? RATE_INFO_FLAGS_160_MHZ_WIDTH :
+				bw == CHANNEL_WIDTH_80 ? RATE_INFO_FLAGS_80_MHZ_WIDTH :
+				bw == CHANNEL_WIDTH_40 ? RATE_INFO_FLAGS_40_MHZ_WIDTH : 0;
+		rinfo->nss = nss;
+		rinfo->mcs = mcs;
+	} else if (mod == RTW_HT_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_MCS;
+		rinfo->flags |= bw ? RATE_INFO_FLAGS_40_MHZ_WIDTH : 0;
+		rinfo->mcs = mcs;
+	}
+#else
+	if (mod == RTW_VHT_MODE) {
+		rinfo->legacy = 0;
+		RTW_INFO("Cannot report VHT rate in current kernel version\n");
+	} else if (mod == RTW_HT_MODE) {
+		rinfo->flags |= RATE_INFO_FLAGS_MCS;
+		rinfo->flags |= bw ? RATE_INFO_FLAGS_40_MHZ_WIDTH : 0;
+		rinfo->mcs = mcs;
+	}
+#endif
+	else {
+		rinfo->legacy = rtw_desc_rate_to_bitrate(0, rtw_rate_idx, 0);
+	}
+}
+
+static int cfg80211_rtw_get_station(struct wiphy *wiphy,
+	struct net_device *ndev,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0))
+	u8 *mac,
+#else
+	const u8 *mac,
+#endif
+	struct station_info *sinfo)
+{
+	int ret = 0;
+	u8 bw, sgi;
+	u16 rtw_rate_idx;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sta_info *psta = NULL;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+#ifdef CONFIG_RTW_MESH
+	struct mesh_plink_ent *plink = NULL;
+#endif
+	sinfo->filled = 0;
+
+	if (!mac) {
+		RTW_INFO(FUNC_NDEV_FMT" mac==%p\n", FUNC_NDEV_ARG(ndev), mac);
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	psta = rtw_get_stainfo(pstapriv, mac);
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		if (psta)
+			plink = psta->plink;
+		if (!plink)
+			plink = rtw_mesh_plink_get(padapter, mac);
+	}
+#endif /* CONFIG_RTW_MESH */
+
+	if ((!MLME_IS_MESH(padapter) && !psta)
+		#ifdef CONFIG_RTW_MESH
+		|| (MLME_IS_MESH(padapter) && !plink)
+		#endif
+	) {
+		RTW_INFO(FUNC_NDEV_FMT" no sta info for mac="MAC_FMT"\n"
+			, FUNC_NDEV_ARG(ndev), MAC_ARG(mac));
+		ret = -ENOENT;
+		goto exit;
+	}
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO(FUNC_NDEV_FMT" mac="MAC_FMT"\n", FUNC_NDEV_ARG(ndev), MAC_ARG(mac));
+#endif
+
+	/* for infra./P2PClient mode */
+	if (MLME_IS_STA(padapter)
+		&& check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+		struct wlan_network  *cur_network = &(pmlmepriv->dev_cur_network);
+
+		if (_rtw_memcmp((u8 *)mac, cur_network->network.MacAddress, ETH_ALEN) == _FALSE) {
+			RTW_INFO("%s, mismatch bssid="MAC_FMT"\n", __func__, MAC_ARG(cur_network->network.MacAddress));
+			ret = -ENOENT;
+			goto exit;
+		}
+
+		sinfo->filled |= STATION_INFO_SIGNAL;
+		sinfo->signal = rtw_phl_rssi_to_dbm(padapter->recvinfo.signal_strength);
+
+		sinfo->filled |= STATION_INFO_TX_BITRATE;
+		sinfo->txrate.legacy = rtw_get_cur_max_rate(padapter);
+	}
+
+	if (psta) {
+		if (!MLME_IS_STA(padapter)
+			|| check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _FALSE
+		) {
+			sinfo->filled |= STATION_INFO_SIGNAL;
+			/* ToDo: need API to query hal_sta->rssi_stat.rssi */
+			/* sinfo->signal = rtw_phl_rssi_to_dbm(psta->phl_sta->rssi_stat.rssi); */
+		}
+		sinfo->filled |= STATION_INFO_INACTIVE_TIME;
+		sinfo->inactive_time = rtw_get_passing_time_ms(psta->sta_stats.last_rx_time);
+		sinfo->filled |= STATION_INFO_RX_PACKETS;
+		sinfo->rx_packets = sta_rx_data_pkts(psta);
+		sinfo->filled |= STATION_INFO_TX_PACKETS;
+		sinfo->tx_packets = psta->sta_stats.tx_pkts;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+		sinfo->filled |= STATION_INFO_RX_BYTES64;
+		sinfo->filled |= STATION_INFO_TX_BYTES64;
+#else
+		sinfo->filled |= STATION_INFO_RX_BYTES;
+		sinfo->filled |= STATION_INFO_TX_BYTES;
+#endif
+		sinfo->rx_bytes = psta->sta_stats.rx_bytes;
+		sinfo->tx_bytes = psta->sta_stats.tx_bytes;
+
+		/* Although according to cfg80211.h struct station_info */
+		/* @txrate: current unicast bitrate from this station */
+		/* We still report sinfo->txrate as bitrate to this station */
+		sinfo->filled |= STATION_INFO_TX_BITRATE;
+		rtw_rate_idx = rtw_get_current_tx_rate(padapter, psta);
+		sgi = rtw_get_current_tx_sgi(padapter, psta);
+		bw = psta->phl_sta->rlink->chandef.bw;
+		sta_set_rate_info(padapter, &sinfo->txrate, rtw_rate_idx, sgi, bw);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39))
+		/* Although @rxrate: current unicast bitrate to this station */
+		/* We report sinfo->rxrate as bitrate from this station */
+		sinfo->filled |= STATION_INFO_RX_BITRATE;
+		rtw_get_current_rx_info(padapter, psta, &rtw_rate_idx, &bw, &sgi);
+		sta_set_rate_info(padapter, &sinfo->rxrate, rtw_rate_idx, sgi, bw);
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+		if (rtw_get_sta_tx_stat(padapter, psta) != RTW_NOT_SUPPORT) {
+			sinfo->filled |= STATION_INFO_TX_FAILED;
+			sinfo->filled |= STATION_INFO_TX_RETRIES;
+			sinfo->tx_failed = psta->sta_stats.tx_fail_cnt_sum;
+			sinfo->tx_retries = psta->sta_stats.tx_retry_cnt_sum;
+		}
+#ifdef CONFIG_RTW_ANDROID
+		else {
+			sinfo->filled |= STATION_INFO_TX_FAILED;
+			sinfo->tx_failed = 0;
+		}
+#endif /* CONFIG_RTW_ANDROID */
+#endif
+	}
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter))
+		rtw_cfg80211_fill_mesh_only_sta_info(plink, psta, sinfo);
+#endif
+
+exit:
+	return ret;
+}
+
+extern int netdev_open(struct net_device *pnetdev);
+
+#if 0
+enum nl80211_iftype {
+	NL80211_IFTYPE_UNSPECIFIED,
+	NL80211_IFTYPE_ADHOC, /* 1 */
+	NL80211_IFTYPE_STATION, /* 2 */
+	NL80211_IFTYPE_AP, /* 3 */
+	NL80211_IFTYPE_AP_VLAN,
+	NL80211_IFTYPE_WDS,
+	NL80211_IFTYPE_MONITOR, /* 6 */
+	NL80211_IFTYPE_MESH_POINT,
+	NL80211_IFTYPE_P2P_CLIENT, /* 8 */
+	NL80211_IFTYPE_P2P_GO, /* 9 */
+	/* keep last */
+	NUM_NL80211_IFTYPES,
+	NL80211_IFTYPE_MAX = NUM_NL80211_IFTYPES - 1
+};
+#endif
+static int cfg80211_rtw_change_iface(struct wiphy *wiphy,
+				     struct net_device *ndev,
+				     enum nl80211_iftype type,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) && !defined(CPTCFG_VERSION)
+				     u32 *flags,
+#endif
+				     struct vif_params *params)
+{
+	enum nl80211_iftype old_type;
+	NDIS_802_11_NETWORK_INFRASTRUCTURE networkType;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct wireless_dev *rtw_wdev = padapter->rtw_wdev;
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif
+#ifdef CONFIG_MONITOR_MODE_XMIT
+	struct mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+#endif
+	int ret = 0;
+	u8 change = _FALSE;
+
+	RTW_INFO(FUNC_NDEV_FMT" type=%d\n", FUNC_NDEV_ARG(ndev), type);
+
+	if (adapter_to_dvobj(padapter)->processing_dev_remove == _TRUE) {
+		ret = -EPERM;
+		goto exit;
+	}
+
+
+	RTW_INFO(FUNC_NDEV_FMT" call netdev_open\n", FUNC_NDEV_ARG(ndev));
+	if (netdev_open(ndev) != 0) {
+		RTW_INFO(FUNC_NDEV_FMT" call netdev_open fail\n", FUNC_NDEV_ARG(ndev));
+		ret = -EPERM;
+		goto exit;
+	}
+
+	old_type = rtw_wdev->iftype;
+	RTW_INFO(FUNC_NDEV_FMT" old_iftype=%d, new_iftype=%d\n",
+		FUNC_NDEV_ARG(ndev), old_type, type);
+
+	if (old_type != type) {
+		change = _TRUE;
+		pmlmeext->action_public_rxseq = 0xffff;
+		pmlmeext->action_public_dialog_token = 0xff;
+	}
+
+#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	if (type != NL80211_IFTYPE_P2P_CLIENT && type != NL80211_IFTYPE_P2P_GO) {
+		if (!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE)) {
+			if (!rtw_p2p_enable(padapter, P2P_ROLE_DISABLE)) {
+				ret = -EOPNOTSUPP;
+				goto exit;
+			}
+		}
+	}
+#endif
+
+	/* initial default type */
+	ndev->type = ARPHRD_ETHER;
+
+	switch (type) {
+	case NL80211_IFTYPE_ADHOC:
+		networkType = Ndis802_11IBSS;
+		break;
+
+	case NL80211_IFTYPE_STATION:
+		networkType = Ndis802_11Infrastructure;
+		break;
+
+	case NL80211_IFTYPE_AP:
+		networkType = Ndis802_11APMode;
+		break;
+
+#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_CLIENT:
+		networkType = Ndis802_11Infrastructure;
+		if (change) {
+			if (!rtw_p2p_enable(padapter, P2P_ROLE_CLIENT)) {
+				ret = -EOPNOTSUPP;
+				goto exit;
+			}
+		}
+		break;
+
+	case NL80211_IFTYPE_P2P_GO:
+		networkType = Ndis802_11APMode;
+		if (change) {
+			if (!rtw_p2p_enable(padapter, P2P_ROLE_GO)) {
+				ret = -EOPNOTSUPP;
+				goto exit;
+			}
+		}
+		break;
+#endif /* defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)) */
+
+#ifdef CONFIG_RTW_MESH
+	case NL80211_IFTYPE_MESH_POINT:
+		networkType = Ndis802_11_mesh;
+		break;
+#endif
+
+#ifdef CONFIG_WIFI_MONITOR
+	case NL80211_IFTYPE_MONITOR:
+		networkType = Ndis802_11Monitor;
+
+#ifdef CONFIG_CUSTOMER_ALIBABA_GENERAL
+		ndev->type = ARPHRD_IEEE80211; /* IEEE 802.11 : 801 */
+#else
+		ndev->type = ARPHRD_IEEE80211_RADIOTAP; /* IEEE 802.11 + radiotap header : 803 */
+#endif
+		break;
+#endif /* CONFIG_WIFI_MONITOR */
+	default:
+		ret = -EOPNOTSUPP;
+		goto exit;
+	}
+
+	rtw_wdev->iftype = type;
+
+	if (rtw_set_802_11_infrastructure_mode(padapter, networkType, 0) == _FALSE) {
+		rtw_wdev->iftype = old_type;
+		ret = -EPERM;
+		goto exit;
+	}
+#ifdef CONFIG_HWSIM
+	rtw_setopmode_cmd(padapter, networkType, RTW_CMDF_DIRECTLY);
+#else
+	rtw_setopmode_cmd(padapter, networkType, RTW_CMDF_WAIT_ACK);
+#endif
+
+#ifdef CONFIG_MONITOR_MODE_XMIT
+	if (check_fwstate(pmlmepriv, WIFI_MONITOR_STATE) == _TRUE)
+		rtw_indicate_connect(padapter);
+#endif
+
+	#if defined(CONFIG_RTW_WDS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 33))
+	if (params->use_4addr != -1) {
+		RTW_INFO(FUNC_NDEV_FMT" use_4addr=%d\n",
+			 FUNC_NDEV_ARG(ndev), params->use_4addr);
+		adapter_set_use_wds(padapter, params->use_4addr);
+	}
+	#endif
+
+exit:
+
+	RTW_INFO(FUNC_NDEV_FMT" ret:%d\n", FUNC_NDEV_ARG(ndev), ret);
+	return ret;
+}
+
+void rtw_cfg80211_indicate_scan_done(_adapter *adapter, bool aborted)
+{
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(adapter);
+
+#if defined(CPTCFG_VERSION) || (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE)
+	struct cfg80211_scan_info info;
+
+	memset(&info, 0, sizeof(info));
+	info.aborted = aborted;
+#endif
+
+	_rtw_spinlock_bh(&pwdev_priv->scan_req_lock);
+	if (pwdev_priv->scan_request != NULL) {
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("%s with scan req\n", __FUNCTION__);
+		#endif
+
+		/* avoid WARN_ON(request != wiphy_to_dev(request->wiphy)->scan_req); */
+		if (pwdev_priv->scan_request->wiphy != pwdev_priv->rtw_wdev->wiphy)
+			RTW_INFO("error wiphy compare\n");
+		else
+#if defined(CPTCFG_VERSION) || (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE)
+			cfg80211_scan_done(pwdev_priv->scan_request, &info);
+#else
+			cfg80211_scan_done(pwdev_priv->scan_request, aborted);
+#endif
+
+		pwdev_priv->scan_request = NULL;
+	} else {
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("%s without scan req\n", __FUNCTION__);
+		#endif
+	}
+	_rtw_spinunlock_bh(&pwdev_priv->scan_req_lock);
+}
+
+u32 rtw_cfg80211_wait_scan_req_empty(_adapter *adapter, u32 timeout_ms)
+{
+	struct rtw_wdev_priv *wdev_priv = adapter_wdev_data(adapter);
+	u8 empty = _FALSE;
+	systime start;
+	u32 pass_ms;
+
+	start = rtw_get_current_time();
+
+	while (rtw_get_passing_time_ms(start) <= timeout_ms) {
+
+		if (RTW_CANNOT_RUN(adapter_to_dvobj(adapter)))
+			break;
+
+		if (!wdev_priv->scan_request) {
+			empty = _TRUE;
+			break;
+		}
+
+		rtw_msleep_os(10);
+	}
+
+	pass_ms = rtw_get_passing_time_ms(start);
+
+	if (empty == _FALSE && pass_ms > timeout_ms)
+		RTW_PRINT(FUNC_ADPT_FMT" pass_ms:%u, timeout\n"
+			, FUNC_ADPT_ARG(adapter), pass_ms);
+
+	return pass_ms;
+}
+
+void rtw_cfg80211_unlink_bss(_adapter *padapter, struct wlan_network *pnetwork)
+{
+	struct wireless_dev *pwdev = padapter->rtw_wdev;
+	struct wiphy *wiphy = pwdev->wiphy;
+	struct cfg80211_bss *bss = NULL;
+	WLAN_BSSID_EX select_network = pnetwork->network;
+
+	bss = cfg80211_get_bss(wiphy, NULL/*notify_channel*/,
+		select_network.MacAddress, select_network.Ssid.Ssid,
+		select_network.Ssid.SsidLength,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
+		select_network.InfrastructureMode == Ndis802_11Infrastructure?IEEE80211_BSS_TYPE_ESS:IEEE80211_BSS_TYPE_IBSS,
+		IEEE80211_PRIVACY(select_network.Privacy));
+#else
+		select_network.InfrastructureMode == Ndis802_11Infrastructure?WLAN_CAPABILITY_ESS:WLAN_CAPABILITY_IBSS,
+		select_network.InfrastructureMode == Ndis802_11Infrastructure?WLAN_CAPABILITY_ESS:WLAN_CAPABILITY_IBSS);
+#endif
+
+	if (bss) {
+		cfg80211_unlink_bss(wiphy, bss);
+		RTW_INFO("%s(): cfg80211_unlink %s!!\n", __func__, select_network.Ssid.Ssid);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
+		cfg80211_put_bss(padapter->rtw_wdev->wiphy, bss);
+#else
+		cfg80211_put_bss(bss);
+#endif
+	}
+	return;
+}
+
+/* if target wps scan ongoing, target_ssid is filled */
+int rtw_cfg80211_is_target_wps_scan(struct cfg80211_scan_request *scan_req, struct cfg80211_ssid *target_ssid)
+{
+	int ret = 0;
+
+	if (scan_req->n_ssids != 1
+		|| scan_req->ssids[0].ssid_len == 0
+		|| scan_req->n_channels != 1
+	)
+		goto exit;
+
+	/* under target WPS scan */
+	_rtw_memcpy(target_ssid, scan_req->ssids, sizeof(struct cfg80211_ssid));
+	ret = 1;
+
+exit:
+	return ret;
+}
+
+static void _rtw_cfg80211_surveydone_event_callback(_adapter *padapter, struct cfg80211_scan_request *scan_req)
+{
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	_list					*plist, *phead;
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	_queue				*queue	= &(pmlmepriv->scanned_queue);
+	struct	wlan_network	*pnetwork = NULL;
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	struct cfg80211_ssid target_ssid;
+	u8 target_wps_scan = 0;
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("%s\n", __func__);
+#endif
+
+	if (scan_req)
+		target_wps_scan = rtw_cfg80211_is_target_wps_scan(scan_req, &target_ssid);
+	else {
+		_rtw_spinlock_bh(&pwdev_priv->scan_req_lock);
+		if (pwdev_priv->scan_request != NULL)
+			target_wps_scan = rtw_cfg80211_is_target_wps_scan(pwdev_priv->scan_request, &target_ssid);
+		_rtw_spinunlock_bh(&pwdev_priv->scan_req_lock);
+	}
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	while (1) {
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+		/* report network only if the current channel set contains the channel to which this network belongs */
+		if (rtw_network_chk_opch_status(rfctl, pnetwork)
+			&& rtw_network_chk_regu_ies(rfctl, pnetwork)
+			&& rtw_mlme_band_check(padapter, BSS_EX_OP_CH(&pnetwork->network)) == _TRUE
+			&& _TRUE == rtw_validate_ssid(&(pnetwork->network.Ssid))
+		) {
+			if (target_wps_scan)
+				rtw_cfg80211_clear_wps_sr_of_non_target_bss(padapter, pnetwork, &target_ssid);
+			rtw_cfg80211_inform_bss(padapter, pnetwork);
+		}
+
+		plist = get_next(plist);
+	}
+
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+}
+
+inline void rtw_cfg80211_surveydone_event_callback(_adapter *padapter)
+{
+	_rtw_cfg80211_surveydone_event_callback(padapter, NULL);
+}
+
+static int rtw_cfg80211_set_probe_req_wpsp2pie(_adapter *padapter, char *buf, int len)
+{
+	int ret = 0;
+	uint wps_ielen = 0;
+	u8 *wps_ie;
+	u32	p2p_ielen = 0;
+	u8 *p2p_ie;
+	u32	wfd_ielen = 0;
+	u8 *wfd_ie;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("%s, ielen=%d\n", __func__, len);
+#endif
+
+	if (len > 0) {
+		wps_ie = rtw_get_wps_ie(buf, len, NULL, &wps_ielen);
+		if (wps_ie) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("probe_req_wps_ielen=%d\n", wps_ielen);
+			#endif
+
+			if (pmlmepriv->wps_probe_req_ie) {
+				u32 free_len = pmlmepriv->wps_probe_req_ie_len;
+				pmlmepriv->wps_probe_req_ie_len = 0;
+				rtw_mfree(pmlmepriv->wps_probe_req_ie, free_len);
+				pmlmepriv->wps_probe_req_ie = NULL;
+			}
+
+			pmlmepriv->wps_probe_req_ie = rtw_malloc(wps_ielen);
+			if (pmlmepriv->wps_probe_req_ie == NULL) {
+				RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				return -EINVAL;
+
+			}
+			_rtw_memcpy(pmlmepriv->wps_probe_req_ie, wps_ie, wps_ielen);
+			pmlmepriv->wps_probe_req_ie_len = wps_ielen;
+		}
+
+		/* buf += wps_ielen; */
+		/* len -= wps_ielen; */
+
+		#ifdef CONFIG_P2P
+		p2p_ie = rtw_get_p2p_ie(buf, len, NULL, &p2p_ielen);
+		if (p2p_ie) {
+			struct wifidirect_info *wdinfo = &padapter->wdinfo;
+			u32 attr_contentlen = 0;
+			u8 listen_ch_attr[5];
+
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("probe_req_p2p_ielen=%d\n", p2p_ielen);
+			#endif
+
+			if (pmlmepriv->p2p_probe_req_ie) {
+				u32 free_len = pmlmepriv->p2p_probe_req_ie_len;
+				pmlmepriv->p2p_probe_req_ie_len = 0;
+				rtw_mfree(pmlmepriv->p2p_probe_req_ie, free_len);
+				pmlmepriv->p2p_probe_req_ie = NULL;
+			}
+
+			pmlmepriv->p2p_probe_req_ie = rtw_malloc(p2p_ielen);
+			if (pmlmepriv->p2p_probe_req_ie == NULL) {
+				RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				return -EINVAL;
+
+			}
+			_rtw_memcpy(pmlmepriv->p2p_probe_req_ie, p2p_ie, p2p_ielen);
+			pmlmepriv->p2p_probe_req_ie_len = p2p_ielen;
+
+			attr_contentlen = sizeof(listen_ch_attr);
+			if (rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_LISTEN_CH, (u8 *)listen_ch_attr, (uint *) &attr_contentlen)) {
+				if (wdinfo->listen_channel !=  listen_ch_attr[4]) {
+					RTW_INFO(FUNC_ADPT_FMT" listen channel - country:%c%c%c, class:%u, ch:%u\n",
+						FUNC_ADPT_ARG(padapter), listen_ch_attr[0], listen_ch_attr[1], listen_ch_attr[2],
+						listen_ch_attr[3], listen_ch_attr[4]);
+					wdinfo->listen_channel = listen_ch_attr[4];
+				}
+			}
+		}
+		#endif /* CONFIG_P2P */
+
+		#ifdef CONFIG_WFD
+		wfd_ie = rtw_get_wfd_ie(buf, len, NULL, &wfd_ielen);
+		if (wfd_ie) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("probe_req_wfd_ielen=%d\n", wfd_ielen);
+			#endif
+
+			if (rtw_mlme_update_wfd_ie_data(pmlmepriv, MLME_PROBE_REQ_IE, wfd_ie, wfd_ielen) != _SUCCESS)
+				return -EINVAL;
+		}
+		#endif /* CONFIG_WFD */
+
+		#ifdef CONFIG_RTW_MBO
+		rtw_mbo_update_ie_data(padapter, buf, len);
+		#endif
+	}
+
+	return ret;
+
+}
+
+#ifdef CONFIG_CONCURRENT_MODE
+u8 rtw_cfg80211_scan_via_buddy(_adapter *padapter, struct cfg80211_scan_request *request)
+{
+	int i;
+	u8 ret = _FALSE;
+	_adapter *iface = NULL;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		struct mlme_priv *buddy_mlmepriv;
+		struct rtw_wdev_priv *buddy_wdev_priv;
+
+		iface = dvobj->padapters[i];
+		if (iface == NULL)
+			continue;
+
+		if (iface == padapter)
+			continue;
+
+		if (rtw_is_adapter_up(iface) == _FALSE)
+			continue;
+
+		buddy_mlmepriv = &iface->mlmepriv;
+		if (!check_fwstate(buddy_mlmepriv, WIFI_UNDER_SURVEY))
+			continue;
+
+		buddy_wdev_priv = adapter_wdev_data(iface);
+		_rtw_spinlock_bh(&pwdev_priv->scan_req_lock);
+		_rtw_spinlock_bh(&buddy_wdev_priv->scan_req_lock);
+		if (buddy_wdev_priv->scan_request) {
+			pmlmepriv->scanning_via_buddy_intf = _TRUE;
+			_rtw_spinlock_bh(&pmlmepriv->lock);
+			set_fwstate(pmlmepriv, WIFI_UNDER_SURVEY);
+			_rtw_spinunlock_bh(&pmlmepriv->lock);
+			pwdev_priv->scan_request = request;
+			ret = _TRUE;
+		}
+		_rtw_spinunlock_bh(&buddy_wdev_priv->scan_req_lock);
+		_rtw_spinunlock_bh(&pwdev_priv->scan_req_lock);
+
+		if (ret == _TRUE)
+			goto exit;
+	}
+
+exit:
+	return ret;
+}
+
+void rtw_cfg80211_indicate_scan_done_for_buddy(_adapter *padapter, bool bscan_aborted)
+{
+	int i;
+	_adapter *iface = NULL;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct mlme_priv *mlmepriv;
+	struct rtw_wdev_priv *wdev_priv;
+	bool indicate_buddy_scan;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && rtw_is_adapter_up(iface)) {
+
+			if (iface == padapter)
+				continue;
+
+			mlmepriv = &(iface->mlmepriv);
+			wdev_priv = adapter_wdev_data(iface);
+
+			indicate_buddy_scan = _FALSE;
+			_rtw_spinlock_bh(&wdev_priv->scan_req_lock);
+			if (mlmepriv->scanning_via_buddy_intf == _TRUE) {
+				mlmepriv->scanning_via_buddy_intf = _FALSE;
+				clr_fwstate(mlmepriv, WIFI_UNDER_SURVEY);
+				if (wdev_priv->scan_request)
+					indicate_buddy_scan = _TRUE;
+			}
+			_rtw_spinunlock_bh(&wdev_priv->scan_req_lock);
+
+			if (indicate_buddy_scan == _TRUE) {
+				rtw_cfg80211_surveydone_event_callback(iface);
+				rtw_indicate_scan_done(iface, bscan_aborted);
+			}
+
+		}
+	}
+}
+#endif /* CONFIG_CONCURRENT_MODE */
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+int rtw_cfg80211_split_scan_6ghz(_adapter *padapter) /* second scan of two part scan */
+{
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	struct cfg80211_scan_request *request = pwdev_priv->scan_request;
+	struct cfg80211_ssid *ssids = request->ssids;
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct rtw_phl_rnb_rpt_element *rnb_ie;
+	_queue *queue = &(pmlmepriv->scanned_queue);
+	_list *plist, *phead;
+	struct wlan_network *pnetwork;
+	struct sitesurvey_parm *parm;
+	u8 non_psc_chan[MAX_CHANNEL_NUM_6G] = {0};
+	int non_psc_num = 0;
+	int ret;
+	int i, j;
+
+	rnb_ie = rtw_zmalloc(sizeof(struct rtw_phl_rnb_rpt_element));
+	if (rnb_ie == NULL)
+		return -ENOMEM;
+
+	parm = rtw_malloc(sizeof(struct sitesurvey_parm));
+	if (parm == NULL) {
+		rtw_mfree(rnb_ie, sizeof(struct rtw_phl_rnb_rpt_element));
+		return -ENOMEM;
+	}
+
+	rtw_init_sitesurvey_parm(padapter, parm);
+	parm->scan_6ghz_only = true;
+	parm->split_scan_6ghz = true;
+
+	/* parsing request ssids, n_ssids */
+	for (i = 0; i < request->n_ssids && ssids && i < RTW_SSID_SCAN_AMOUNT; i++) {
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("%s: ssid=%s, len=%d\n", __func__, ssids[i].ssid, ssids[i].ssid_len);
+		#endif
+		_rtw_memcpy(&parm->ssid[i].Ssid, ssids[i].ssid, ssids[i].ssid_len);
+		parm->ssid[i].SsidLength = ssids[i].ssid_len;
+		parm->ssid_num++;
+	}
+
+	/* if no ssid entry, set the scan type as passive */
+	if (request->n_ssids == 0)
+		parm->scan_mode = RTW_PHL_SCAN_PASSIVE;
+
+	/* Find non-PSC 6GHz channels found in RNR IEs from first scan */
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	while (1) {
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+		if (rtw_parse_reduced_nb_rpt(padapter, pnetwork->network.IEs, pnetwork->network.IELength, rnb_ie)) {
+			for (i = 0; i < rnb_ie->nb_ap_num; i++) {
+				if ((rnb_ie->nb_aps[i].chan_def.band == BAND_ON_6G) &&
+				    !Is6GHzPreferScanChannel(rnb_ie->nb_aps[i].chan_def.chan)) {
+					/* skip non colocated APs */
+					if (!rnb_ie->nb_aps[i].tbtt_infos[0].bss_param.colated_ap)
+						continue;
+
+					for (j = 0; j < MAX_CHANNEL_NUM_6G; j++) {
+						if (non_psc_chan[j] == 0) {
+							non_psc_chan[j] = rnb_ie->nb_aps[i].chan_def.chan;
+							non_psc_num++;
+							break;
+						} else if (non_psc_chan[j] == rnb_ie->nb_aps[i].chan_def.chan) {
+							break;
+						}
+					}
+				}
+			}
+		}
+		plist = get_next(plist);
+	}
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	/* Add 6GHz PSC channels and non-PSC channels found in RNR IEs above into scan list */
+	for (i = 0; i < request->n_channels && i < RTW_CHANNEL_SCAN_AMOUNT; i++) {
+		if (nl80211_band_to_rtw_band(request->channels[i]->band) != BAND_ON_6G)
+			continue;
+
+		if (!Is6GHzPreferScanChannel(request->channels[i]->hw_value)) {
+			for (j = 0; j < non_psc_num; j++)
+				if (non_psc_chan[j] == request->channels[i]->hw_value)
+					break;
+			if (j == non_psc_num)
+				continue;
+		}
+
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO(FUNC_ADPT_FMT CHAN_FMT"\n", FUNC_ADPT_ARG(padapter), CHAN_ARG(request->channels[i]));
+		#endif
+		parm->ch[parm->ch_num].hw_value = request->channels[i]->hw_value;
+		parm->ch[parm->ch_num].flags = request->channels[i]->flags;
+		parm->ch[parm->ch_num].band = BAND_ON_6G;
+		parm->ch_num++;
+	}
+
+	ret = rtw_sitesurvey_cmd(padapter, parm);
+
+	rtw_mfree(rnb_ie, sizeof(struct rtw_phl_rnb_rpt_element));
+	rtw_mfree(parm, sizeof(struct sitesurvey_parm));
+
+	return ret;
+}
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static int cfg80211_rtw_scan(struct wiphy *wiphy
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))
+	, struct net_device *ndev
+	#endif
+	, struct cfg80211_scan_request *request)
+{
+	int i;
+	u8 _status = _FALSE;
+	int ret = 0;
+	struct sitesurvey_parm *parm = NULL;
+	u8 survey_times = 3;
+	u8 survey_times_for_one_ch = 6;
+	struct cfg80211_ssid *ssids = request->ssids;
+	int social_channel = 0, j = 0;
+	bool need_indicate_scan_done = _FALSE;
+	bool ps_denied = _FALSE;
+	u8 ssc_chk;
+	_adapter *padapter;
+	struct wireless_dev *wdev;
+	struct rtw_wdev_priv *pwdev_priv;
+	struct mlme_priv *pmlmepriv = NULL;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo;
+#endif /* CONFIG_P2P */
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	bool scan_11ac_chan = false;
+#endif
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	wdev = request->wdev;
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev == wiphy_to_pd_wdev(wiphy))
+		padapter = wiphy_to_adapter(wiphy);
+	else
+	#endif
+	if (wdev_to_ndev(wdev))
+		padapter = (_adapter *)rtw_netdev_priv(wdev_to_ndev(wdev));
+	else {
+		ret = -EINVAL;
+		goto exit;
+	}
+#else
+	if (ndev == NULL) {
+		ret = -EINVAL;
+		goto exit;
+	}
+	padapter = (_adapter *)rtw_netdev_priv(ndev);
+	wdev = ndev_to_wdev(ndev);
+#endif
+
+	pwdev_priv = adapter_wdev_data(padapter);
+	pmlmepriv = &padapter->mlmepriv;
+#ifdef CONFIG_P2P
+	pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+
+	RTW_INFO(FUNC_ADPT_FMT"%s\n", FUNC_ADPT_ARG(padapter)
+		, wdev == wiphy_to_pd_wdev(wiphy) ? " PD" : "");
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		RTW_DBG(FUNC_ADPT_FMT "- bDriverStopped(%s) bSurpriseRemoved(%s)\n",
+			FUNC_ADPT_ARG(padapter),
+			dev_is_drv_stopped(adapter_to_dvobj(padapter)) ? "True" : "False",
+			dev_is_surprise_removed(adapter_to_dvobj(padapter)) ? "True" : "False");
+		ret = -EBUSY;
+		goto exit;
+	}
+
+	if (adapter_to_pwrctl(padapter)->bInSuspend == _TRUE) {
+		RTW_INFO("%s return -EBUSY since bInSuspend is TRUE\n", __func__);
+		ret = -EBUSY;
+		goto exit;
+	}
+
+	parm = rtw_malloc(sizeof(*parm));
+	if (parm == NULL) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	rtw_init_sitesurvey_parm(padapter, parm);
+
+	ssc_chk = rtw_sitesurvey_condition_check(padapter, _TRUE);
+
+	if (ssc_chk == SS_DENY_MP_MODE)
+		goto bypass_p2p_chk;
+#ifdef DBG_LA_MODE
+	if (ssc_chk == SS_DENY_LA_MODE)
+		goto bypass_p2p_chk;
+#endif
+
+#ifdef CONFIG_P2P
+	if (request->n_ssids && ssids
+		&& _rtw_memcmp(ssids[0].ssid, "DIRECT-", 7)
+		&& rtw_get_p2p_ie((u8 *)request->ie, request->ie_len, NULL, NULL)
+	) {
+		if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE)) {
+			if (!rtw_p2p_enable(padapter, P2P_ROLE_DEVICE)) {
+				ret = -EOPNOTSUPP;
+				goto exit;
+			}
+		} else {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("%s, role=%d\n", __func__, rtw_p2p_role(pwdinfo));
+			#endif
+		}
+
+		if (request->n_channels == 3 &&
+			request->channels[0]->hw_value == 1 &&
+			request->channels[1]->hw_value == 6 &&
+			request->channels[2]->hw_value == 11
+		)
+		social_channel = 1;
+		parm->scan_type = RTW_SCAN_P2P;
+	}
+#endif /*CONFIG_P2P*/
+
+	if (request->ie && request->ie_len > 0)
+		rtw_cfg80211_set_probe_req_wpsp2pie(padapter, (u8 *)request->ie, request->ie_len);
+
+bypass_p2p_chk:
+
+	switch (ssc_chk) {
+		case SS_ALLOW :
+			break;
+
+		case SS_DENY_MP_MODE:
+			ret = -EPERM;
+			goto exit;
+		#ifdef DBG_LA_MODE
+		case SS_DENY_LA_MODE:
+			ret = -EPERM;
+			goto exit;
+		#endif
+		case SS_DENY_BLOCK_SCAN :
+		case SS_DENY_SELF_AP_UNDER_WPS :
+		case SS_DENY_SELF_AP_UNDER_LINKING :
+		case SS_DENY_SELF_AP_UNDER_SURVEY :
+		case SS_DENY_SELF_STA_UNDER_SURVEY :
+		#ifdef CONFIG_CONCURRENT_MODE
+		case SS_DENY_BUDDY_UNDER_LINK_WPS :
+		#endif
+		case SS_DENY_BUSY_TRAFFIC :
+		case SS_DENY_ADAPTIVITY:
+			need_indicate_scan_done = _TRUE;
+			goto check_need_indicate_scan_done;
+
+		case SS_DENY_BY_DRV :
+			#ifdef CONFIG_NOTIFY_SCAN_ABORT_WITH_BUSY
+			ret = -EBUSY;
+			goto exit;
+			#else
+			need_indicate_scan_done = _TRUE;
+			goto check_need_indicate_scan_done;
+			#endif
+			break;
+
+		case SS_DENY_SELF_STA_UNDER_LINKING :
+			ret = -EBUSY;
+			goto check_need_indicate_scan_done;
+
+		#ifdef CONFIG_CONCURRENT_MODE
+		case SS_DENY_BUDDY_UNDER_SURVEY :
+			{
+				bool scan_via_buddy = rtw_cfg80211_scan_via_buddy(padapter, request);
+
+				if (scan_via_buddy == _FALSE)
+					need_indicate_scan_done = _TRUE;
+
+				goto check_need_indicate_scan_done;
+			}
+		#endif
+
+		default :
+			RTW_ERR("site survey check code (%d) unknown\n", ssc_chk);
+			need_indicate_scan_done = _TRUE;
+			goto check_need_indicate_scan_done;
+	}
+
+#ifdef CONFIG_RTW_SCAN_RAND
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+		if (request->flags & NL80211_SCAN_FLAG_RANDOM_ADDR
+			&& MLME_IS_STA(padapter)
+			&& (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == _FALSE) ) {
+			pwdev_priv->random_mac_enabled = true;
+			get_random_mask_addr(pwdev_priv->pno_mac_addr, request->mac_addr,
+						 request->mac_addr_mask);
+			RTW_INFO("%s random mac: " MAC_FMT " \n",
+				 __func__, MAC_ARG(pwdev_priv->pno_mac_addr));
+		} else {
+			pwdev_priv->random_mac_enabled = false;
+		}
+#endif /* LINUX_VERSION_CODE */
+#endif /* CONFIG_RTW_SCAN_RAND */
+
+	/* parsing request ssids, n_ssids */
+	for (i = 0; i < request->n_ssids && ssids && i < RTW_SSID_SCAN_AMOUNT; i++) {
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("ssid=%s, len=%d\n", ssids[i].ssid, ssids[i].ssid_len);
+		#endif
+		_rtw_memcpy(&parm->ssid[i].Ssid, ssids[i].ssid, ssids[i].ssid_len);
+		parm->ssid[i].SsidLength = ssids[i].ssid_len;
+	}
+	parm->ssid_num = i;
+
+	/* no ssid entry, set the scan type as passive */
+	if (request->n_ssids == 0)
+		parm->scan_mode = RTW_PHL_SCAN_PASSIVE;
+
+	/* parsing channels, n_channels */
+	for (i = 0; i < request->n_channels && i < RTW_CHANNEL_SCAN_AMOUNT; i++) {
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+		if (padapter->registrypriv.split_scan_6ghz) {
+			/* Perform two part scan only if both 2.4G/5G and 6G channels exist*/
+			if (request->channels[i]->band == NL80211_BAND_6GHZ) {
+				/* if scan_11ac_chan = false, then perform scan with
+				 * all 6GHz channels listed in scan request directly */
+				if (scan_11ac_chan) {
+					parm->pending_6ghz_scan = true;
+					continue;
+				}
+			} else {
+				scan_11ac_chan = true;
+				if (parm->pending_6ghz_scan)
+					RTW_WARN("%s: 6GHz chan in scan channels[] before 11ac chan !!\n", __func__);
+			}
+		}
+		#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+		#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO(FUNC_ADPT_FMT CHAN_FMT"\n", FUNC_ADPT_ARG(padapter), CHAN_ARG(request->channels[i]));
+		#endif
+		parm->ch[parm->ch_num].hw_value = request->channels[i]->hw_value;
+		parm->ch[parm->ch_num].flags = request->channels[i]->flags;
+		parm->ch[parm->ch_num].band = nl80211_band_to_rtw_band(request->channels[i]->band);
+		parm->ch_num++;
+	}
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+	parm->scan_6ghz_only = request->scan_6ghz;
+	#else
+	if (!scan_11ac_chan)
+		parm->scan_6ghz_only = true;
+	#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) */
+	#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+	_rtw_spinlock_bh(&pwdev_priv->scan_req_lock);
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+
+	_status = rtw_sitesurvey_cmd(padapter, parm);
+	if (_status == _SUCCESS)
+		pwdev_priv->scan_request = request;
+	else
+		ret = -1;
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+	_rtw_spinunlock_bh(&pwdev_priv->scan_req_lock);
+
+check_need_indicate_scan_done:
+	if (_TRUE == need_indicate_scan_done) {
+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE)
+		struct cfg80211_scan_info info;
+
+		memset(&info, 0, sizeof(info));
+		info.aborted = 0;
+#endif
+		/* the process time of scan results must be over at least 1ms in the newly Android */
+		rtw_msleep_os(1);
+
+		_rtw_cfg80211_surveydone_event_callback(padapter, request);
+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE)
+		cfg80211_scan_done(request, &info);
+#else
+		cfg80211_scan_done(request, 0);
+#endif
+	}
+
+exit:
+	if (parm)
+		rtw_mfree(parm, sizeof(*parm));
+
+#ifdef RTW_BUSY_DENY_SCAN
+	if (pmlmepriv)
+		pmlmepriv->lastscantime = rtw_get_current_time();
+#endif
+
+	return ret;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
+static void cfg80211_rtw_abort_scan(struct wiphy *wiphy,
+				    struct wireless_dev *wdev)
+{
+	_adapter *padapter = wiphy_to_adapter(wiphy);
+
+	RTW_INFO("=>"FUNC_NDEV_FMT" - Abort Scan\n", FUNC_ADPT_ARG(padapter));
+	if (wdev->iftype != NL80211_IFTYPE_STATION) {
+		RTW_ERR("abort scan ignored, iftype(%d)\n", wdev->iftype);
+		return;
+	}
+	rtw_scan_abort(padapter, 0);
+}
+#endif /* LINUX_VERSION_CODE >= 4.5.0 */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0))
+static int cfg80211_rtw_set_wiphy_params(struct wiphy *wiphy, int radio_idx,
+					 u32 changed)
+#else
+static int cfg80211_rtw_set_wiphy_params(struct wiphy *wiphy, u32 changed)
+#endif
+{
+#if 0
+	struct iwm_priv *iwm = wiphy_to_iwm(wiphy);
+
+	if (changed & WIPHY_PARAM_RTS_THRESHOLD &&
+	    (iwm->conf.rts_threshold != wiphy->rts_threshold)) {
+		int ret;
+
+		iwm->conf.rts_threshold = wiphy->rts_threshold;
+
+		ret = iwm_umac_set_config_fix(iwm, UMAC_PARAM_TBL_CFG_FIX,
+				CFG_RTS_THRESHOLD,
+				iwm->conf.rts_threshold);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (changed & WIPHY_PARAM_FRAG_THRESHOLD &&
+	    (iwm->conf.frag_threshold != wiphy->frag_threshold)) {
+		int ret;
+
+		iwm->conf.frag_threshold = wiphy->frag_threshold;
+
+		ret = iwm_umac_set_config_fix(iwm, UMAC_PARAM_TBL_FA_CFG_FIX,
+				CFG_FRAG_THRESHOLD,
+				iwm->conf.frag_threshold);
+		if (ret < 0)
+			return ret;
+	}
+#endif
+	RTW_INFO("%s\n", __func__);
+	return 0;
+}
+
+
+
+static int rtw_cfg80211_set_wpa_version(struct security_priv *psecuritypriv, u32 wpa_version)
+{
+	RTW_INFO("%s, wpa_version=%d\n", __func__, wpa_version);
+
+	if (!wpa_version) {
+		psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
+		return 0;
+	}
+
+
+	if (wpa_version & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2))
+		psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPAPSK;
+
+#if 0
+	if (wpa_version & NL80211_WPA_VERSION_2)
+		psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPA2PSK;
+#endif
+
+	#ifdef CONFIG_WAPI_SUPPORT
+	if (wpa_version & NL80211_WAPI_VERSION_1)
+		psecuritypriv->ndisauthtype = Ndis802_11AuthModeWAPI;
+	#endif
+
+	return 0;
+
+}
+
+static int rtw_cfg80211_set_auth_type(struct security_priv *psecuritypriv,
+		enum nl80211_auth_type sme_auth_type)
+{
+	RTW_INFO("%s, nl80211_auth_type=%d\n", __func__, sme_auth_type);
+
+	if (NL80211_AUTHTYPE_MAX <= (int)MLME_AUTHTYPE_SAE) {
+		if (MLME_AUTHTYPE_SAE == psecuritypriv->auth_type) {
+			/* This case pre handle in
+			 * rtw_check_connect_sae_compat()
+			 */
+			psecuritypriv->auth_alg = WLAN_AUTH_SAE;
+			return 0;
+		}
+	} else if (sme_auth_type == (int)MLME_AUTHTYPE_SAE) {
+		psecuritypriv->auth_type = MLME_AUTHTYPE_SAE;
+		psecuritypriv->auth_alg = WLAN_AUTH_SAE;
+		return 0;
+	}
+
+	psecuritypriv->auth_type = sme_auth_type;
+
+	switch (sme_auth_type) {
+	case NL80211_AUTHTYPE_AUTOMATIC:
+
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+
+		break;
+	case NL80211_AUTHTYPE_OPEN_SYSTEM:
+
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
+
+		if (psecuritypriv->ndisauthtype > Ndis802_11AuthModeWPA)
+			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+
+#ifdef CONFIG_WAPI_SUPPORT
+		if (psecuritypriv->ndisauthtype == Ndis802_11AuthModeWAPI)
+			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_WAPI;
+#endif
+
+		break;
+	case NL80211_AUTHTYPE_SHARED_KEY:
+
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Shared;
+
+		psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
+
+
+		break;
+	default:
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
+		/* return -ENOTSUPP; */
+	}
+
+	return 0;
+
+}
+
+static int rtw_cfg80211_set_cipher(struct security_priv *psecuritypriv, u32 cipher, bool ucast)
+{
+	u32 ndisencryptstatus = Ndis802_11EncryptionDisabled;
+
+	u32 *profile_cipher = ucast ? &psecuritypriv->dot11PrivacyAlgrthm :
+		&psecuritypriv->dot118021XGrpPrivacy;
+
+	RTW_INFO("%s, ucast=%d, cipher=0x%x\n", __func__, ucast, cipher);
+
+
+	if (!cipher) {
+		*profile_cipher = _NO_PRIVACY_;
+		psecuritypriv->ndisencryptstatus = ndisencryptstatus;
+		return 0;
+	}
+
+	switch (cipher) {
+	case IW_AUTH_CIPHER_NONE:
+		*profile_cipher = _NO_PRIVACY_;
+		ndisencryptstatus = Ndis802_11EncryptionDisabled;
+#ifdef CONFIG_WAPI_SUPPORT
+		if (psecuritypriv->dot11PrivacyAlgrthm == _SMS4_)
+			*profile_cipher = _SMS4_;
+#endif
+		break;
+	case WLAN_CIPHER_SUITE_WEP40:
+		*profile_cipher = _WEP40_;
+		ndisencryptstatus = Ndis802_11Encryption1Enabled;
+		break;
+	case WLAN_CIPHER_SUITE_WEP104:
+		*profile_cipher = _WEP104_;
+		ndisencryptstatus = Ndis802_11Encryption1Enabled;
+		break;
+	case WLAN_CIPHER_SUITE_TKIP:
+		*profile_cipher = _TKIP_;
+		ndisencryptstatus = Ndis802_11Encryption2Enabled;
+		break;
+	case WLAN_CIPHER_SUITE_CCMP:
+		*profile_cipher = _AES_;
+		ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WIFI_CIPHER_SUITE_GCMP:
+		*profile_cipher = _GCMP_;
+		ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WIFI_CIPHER_SUITE_GCMP_256:
+		*profile_cipher = _GCMP_256_;
+		ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WIFI_CIPHER_SUITE_CCMP_256:
+		*profile_cipher = _CCMP_256_;
+		ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+#ifdef CONFIG_WAPI_SUPPORT
+	case WLAN_CIPHER_SUITE_SMS4:
+		*profile_cipher = _SMS4_;
+		ndisencryptstatus = Ndis802_11_EncrypteionWAPI;
+		break;
+#endif
+	default:
+		RTW_INFO("Unsupported cipher: 0x%x\n", cipher);
+		return -ENOTSUPP;
+	}
+
+	if (ucast) {
+		psecuritypriv->ndisencryptstatus = ndisencryptstatus;
+
+		/* if(psecuritypriv->dot11PrivacyAlgrthm >= _AES_) */
+		/*	psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPA2PSK; */
+	}
+
+	return 0;
+}
+
+static int rtw_cfg80211_set_key_mgt(struct security_priv *psecuritypriv, u32 key_mgt)
+{
+	RTW_INFO("%s, key_mgt=0x%x\n", __func__, key_mgt);
+
+	if (key_mgt == WLAN_AKM_SUITE_8021X) {
+		/* *auth_type = UMAC_AUTH_TYPE_8021X; */
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+		psecuritypriv->rsn_akm_suite_type = 1;
+	} else if (key_mgt == WLAN_AKM_SUITE_PSK) {
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+		psecuritypriv->rsn_akm_suite_type = 2;
+	} else if (key_mgt == WLAN_AKM_SUITE_SAE) {
+		psecuritypriv->rsn_akm_suite_type = 8;
+	}
+#ifdef CONFIG_WAPI_SUPPORT
+	else if (key_mgt == WLAN_AKM_SUITE_WAPI_PSK)
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_WAPI;
+	else if (key_mgt == WLAN_AKM_SUITE_WAPI_CERT)
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_WAPI;
+#endif
+#ifdef CONFIG_RTW_80211R
+	else if (key_mgt == WLAN_AKM_SUITE_FT_8021X) {
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+		psecuritypriv->rsn_akm_suite_type = 3;
+	} else if (key_mgt == WLAN_AKM_SUITE_FT_PSK) {
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+		psecuritypriv->rsn_akm_suite_type = 4;
+	}
+#endif
+	else {
+		RTW_INFO("Invalid key mgt: 0x%x\n", key_mgt);
+		/* return -EINVAL; */
+	}
+
+	return 0;
+}
+
+static int rtw_cfg80211_set_wpa_ie(_adapter *padapter, u8 *pie, size_t ielen)
+{
+	u8 *buf = NULL, *pos = NULL;
+	int group_cipher = 0, pairwise_cipher = 0;
+	u8 mfp_opt = MFP_NO;
+	int ret = 0;
+	int wpa_ielen = 0;
+	int wpa2_ielen = 0;
+	int rsnx_ielen = 0;
+	u8 *pwpa, *pwpa2, *prsnx;
+	u8 null_addr[] = {0, 0, 0, 0, 0, 0};
+
+	if (pie == NULL || !ielen) {
+		/* Treat this as normal case, but need to clear WIFI_UNDER_WPS */
+		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
+		goto exit;
+	}
+
+	if (ielen > MAX_WPA_IE_LEN + MAX_WPS_IE_LEN + MAX_P2P_IE_LEN) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	buf = rtw_zmalloc(ielen);
+	if (buf == NULL) {
+		ret =  -ENOMEM;
+		goto exit;
+	}
+
+	_rtw_memcpy(buf, pie , ielen);
+
+	RTW_INFO("set wpa_ie(length:%zu):\n", ielen);
+	RTW_INFO_DUMP(NULL, buf, ielen);
+
+	pos = buf;
+	if (ielen < RSN_HEADER_LEN) {
+		ret  = -1;
+		goto exit;
+	}
+
+	pwpa = rtw_get_wpa_ie(buf, &wpa_ielen, ielen);
+	if (pwpa && wpa_ielen > 0) {
+		if (rtw_parse_wpa_ie(pwpa, wpa_ielen + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
+			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPAPSK;
+			_rtw_memcpy(padapter->securitypriv.supplicant_ie, &pwpa[0], wpa_ielen + 2);
+
+			RTW_INFO("got wpa_ie, wpa_ielen:%u\n", wpa_ielen);
+		}
+	}
+
+	pwpa2 = rtw_get_wpa2_ie(buf, &wpa2_ielen, ielen);
+	if (pwpa2 && wpa2_ielen > 0) {
+		if (rtw_parse_wpa2_ie(pwpa2, wpa2_ielen + 2, &group_cipher, &pairwise_cipher, NULL, NULL, &mfp_opt, NULL) == _SUCCESS) {
+			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPA2PSK;
+			_rtw_memcpy(padapter->securitypriv.supplicant_ie, &pwpa2[0], wpa2_ielen + 2);
+
+			RTW_INFO("got wpa2_ie, wpa2_ielen:%u\n", wpa2_ielen);
+		}
+
+		prsnx = rtw_get_ie(buf, WLAN_EID_RSNX, &rsnx_ielen, ielen);
+		if (prsnx && (rsnx_ielen > 0)) {
+			if ((rsnx_ielen + 2) <= MAX_RSNX_IE_LEN) {
+				_rtw_memset(padapter->securitypriv.rsnx_ie, 0,
+					MAX_RSNX_IE_LEN);
+				padapter->securitypriv.rsnx_ie_len = \
+					(rsnx_ielen + 2);
+				_rtw_memcpy(padapter->securitypriv.rsnx_ie,
+					prsnx,
+					padapter->securitypriv.rsnx_ie_len);
+			} else
+				RTW_ERR("%s:no more buf to save RSNX Cap!\n",
+					__func__);
+		} else {
+			_rtw_memset(padapter->securitypriv.rsnx_ie, 0,
+				MAX_RSNX_IE_LEN);
+			padapter->securitypriv.rsnx_ie_len = 0;
+		}
+
+	}
+
+
+
+	if (group_cipher == 0)
+		group_cipher = WPA_CIPHER_NONE;
+	if (pairwise_cipher == 0)
+		pairwise_cipher = WPA_CIPHER_NONE;
+
+	switch (group_cipher) {
+	case WPA_CIPHER_NONE:
+		padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
+		break;
+	case WPA_CIPHER_WEP40:
+		padapter->securitypriv.dot118021XGrpPrivacy = _WEP40_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+		break;
+	case WPA_CIPHER_TKIP:
+		padapter->securitypriv.dot118021XGrpPrivacy = _TKIP_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled;
+		break;
+	case WPA_CIPHER_CCMP:
+		padapter->securitypriv.dot118021XGrpPrivacy = _AES_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_GCMP:
+		padapter->securitypriv.dot118021XGrpPrivacy = _GCMP_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_GCMP_256:
+		padapter->securitypriv.dot118021XGrpPrivacy = _GCMP_256_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_CCMP_256:
+		padapter->securitypriv.dot118021XGrpPrivacy = _CCMP_256_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_WEP104:
+		padapter->securitypriv.dot118021XGrpPrivacy = _WEP104_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+		break;
+	}
+
+	switch (pairwise_cipher) {
+	case WPA_CIPHER_NONE:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
+		break;
+	case WPA_CIPHER_WEP40:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+		break;
+	case WPA_CIPHER_TKIP:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _TKIP_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled;
+		break;
+	case WPA_CIPHER_CCMP:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _AES_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_GCMP:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _GCMP_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_GCMP_256:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _GCMP_256_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_CCMP_256:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _CCMP_256_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+		break;
+	case WPA_CIPHER_WEP104:
+		padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_;
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+		break;
+	}
+
+	if (mfp_opt == MFP_INVALID) {
+		RTW_INFO(FUNC_ADPT_FMT" invalid MFP setting\n", FUNC_ADPT_ARG(padapter));
+		ret = -EINVAL;
+		goto exit;
+	}
+	padapter->securitypriv.mfp_opt = mfp_opt;
+
+	{/* handle wps_ie */
+		uint wps_ielen;
+		u8 *wps_ie;
+
+		wps_ie = rtw_get_wps_ie(buf, ielen, NULL, &wps_ielen);
+		if (wps_ie && wps_ielen > 0) {
+			RTW_INFO("got wps_ie, wps_ielen:%u\n", wps_ielen);
+			padapter->securitypriv.wps_ie_len = wps_ielen < MAX_WPS_IE_LEN ? wps_ielen : MAX_WPS_IE_LEN;
+			_rtw_memcpy(padapter->securitypriv.wps_ie, wps_ie, padapter->securitypriv.wps_ie_len);
+			set_fwstate(&padapter->mlmepriv, WIFI_UNDER_WPS);
+		} else
+			_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
+	}
+
+	{/* handle owe_ie */
+		uint owe_ielen;
+		u8 *owe_ie;
+
+		owe_ie = rtw_get_owe_ie(buf, ielen, NULL, &owe_ielen);
+		if (owe_ie && owe_ielen > 0) {
+			RTW_INFO("got owe_ie, owe_ielen:%u\n", owe_ielen);
+			padapter->securitypriv.owe_ie_len = owe_ielen < MAX_OWE_IE_LEN ? owe_ielen : MAX_OWE_IE_LEN;
+			_rtw_memcpy(padapter->securitypriv.owe_ie, owe_ie, padapter->securitypriv.owe_ie_len);
+		}
+	}
+
+	#ifdef CONFIG_P2P
+	{/* check p2p_ie for assoc req; */
+		uint p2p_ielen = 0;
+		u8 *p2p_ie;
+		struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+		p2p_ie = rtw_get_p2p_ie(buf, ielen, NULL, &p2p_ielen);
+		if (p2p_ie) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("%s p2p_assoc_req_ielen=%d\n", __FUNCTION__, p2p_ielen);
+			#endif
+
+			if (pmlmepriv->p2p_assoc_req_ie) {
+				u32 free_len = pmlmepriv->p2p_assoc_req_ie_len;
+				pmlmepriv->p2p_assoc_req_ie_len = 0;
+				rtw_mfree(pmlmepriv->p2p_assoc_req_ie, free_len);
+				pmlmepriv->p2p_assoc_req_ie = NULL;
+			}
+
+			pmlmepriv->p2p_assoc_req_ie = rtw_malloc(p2p_ielen);
+			if (pmlmepriv->p2p_assoc_req_ie == NULL) {
+				RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				goto exit;
+			}
+			_rtw_memcpy(pmlmepriv->p2p_assoc_req_ie, p2p_ie, p2p_ielen);
+			pmlmepriv->p2p_assoc_req_ie_len = p2p_ielen;
+		}
+	}
+	#endif /* CONFIG_P2P */
+
+	#ifdef CONFIG_WFD
+	{
+		uint wfd_ielen = 0;
+		u8 *wfd_ie;
+		struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+		wfd_ie = rtw_get_wfd_ie(buf, ielen, NULL, &wfd_ielen);
+		if (wfd_ie) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("%s wfd_assoc_req_ielen=%d\n", __FUNCTION__, wfd_ielen);
+			#endif
+
+			if (rtw_mlme_update_wfd_ie_data(pmlmepriv, MLME_ASSOC_REQ_IE, wfd_ie, wfd_ielen) != _SUCCESS)
+				goto exit;
+		}
+	}
+	#endif /* CONFIG_WFD */
+
+	#ifdef CONFIG_RTW_MULTI_AP
+	padapter->multi_ap = rtw_get_multi_ap_ie_ext(buf, ielen) & MULTI_AP_BACKHAUL_STA;
+	if (padapter->multi_ap)
+		adapter_set_use_wds(padapter, 1);
+	#endif
+
+exit:
+	if (buf)
+		rtw_mfree(buf, ielen);
+	if (ret)
+		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
+
+	return ret;
+}
+
+static int cfg80211_rtw_join_ibss(struct wiphy *wiphy, struct net_device *ndev,
+				  struct cfg80211_ibss_params *params)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	NDIS_802_11_SSID ndis_ssid;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	struct cfg80211_chan_def *pch_def;
+#endif
+	struct ieee80211_channel *pch;
+	int ret = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	pch_def = (struct cfg80211_chan_def *)(&params->chandef);
+	pch = (struct ieee80211_channel *) pch_def->chan;
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 31))
+	pch = (struct ieee80211_channel *)(params->channel);
+#endif
+
+	if (!params->ssid || !params->ssid_len) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (params->ssid_len > IW_ESSID_MAX_SIZE) {
+		ret = -E2BIG;
+		goto exit;
+	}
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_LINKING)) {
+		RTW_INFO("%s, but buddy_intf is under linking\n", __FUNCTION__);
+		ret = -EINVAL;
+		goto exit;
+	}
+	rtw_mi_buddy_scan_abort(padapter, _TRUE); /* OR rtw_mi_scan_abort(padapter, _TRUE);*/
+#endif /*CONFIG_CONCURRENT_MODE*/
+
+
+	_rtw_memset(&ndis_ssid, 0, sizeof(NDIS_802_11_SSID));
+	ndis_ssid.SsidLength = params->ssid_len;
+	_rtw_memcpy(ndis_ssid.Ssid, (u8 *)params->ssid, params->ssid_len);
+
+	/* RTW_INFO("ssid=%s, len=%zu\n", ndis_ssid.Ssid, params->ssid_len); */
+
+	psecuritypriv->ndisencryptstatus = Ndis802_11EncryptionDisabled;
+	psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+	psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
+	psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+	psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
+
+	ret = rtw_cfg80211_set_auth_type(psecuritypriv, NL80211_AUTHTYPE_OPEN_SYSTEM);
+	rtw_set_802_11_authentication_mode(padapter, psecuritypriv->ndisauthtype);
+
+	RTW_INFO("%s: center_freq = %d\n", __func__, pch->center_freq);
+	pmlmeext->chandef.chan = rtw_freq2ch(pch->center_freq);
+
+	if (rtw_set_802_11_ssid(padapter, &ndis_ssid) == _FALSE) {
+		ret = -1;
+		goto exit;
+	}
+
+exit:
+	return ret;
+}
+
+static int cfg80211_rtw_leave_ibss(struct wiphy *wiphy, struct net_device *ndev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct wireless_dev *rtw_wdev = padapter->rtw_wdev;
+	enum nl80211_iftype old_type;
+	int ret = 0;
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+#if (RTW_CFG80211_BLOCK_STA_DISCON_EVENT & RTW_CFG80211_BLOCK_DISCON_WHEN_DISCONNECT)
+	rtw_wdev_set_not_indic_disco(adapter_wdev_data(padapter), 1);
+#endif
+
+	old_type = rtw_wdev->iftype;
+
+	rtw_set_to_roam(padapter, 0);
+
+	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) {
+		rtw_scan_abort(padapter, 0);
+
+		rtw_wdev->iftype = NL80211_IFTYPE_STATION;
+
+		if (rtw_set_802_11_infrastructure_mode(padapter, Ndis802_11Infrastructure, 0) == _FALSE) {
+			rtw_wdev->iftype = old_type;
+			ret = -EPERM;
+			goto leave_ibss;
+		}
+		rtw_setopmode_cmd(padapter, Ndis802_11Infrastructure, RTW_CMDF_WAIT_ACK);
+	}
+
+leave_ibss:
+#if (RTW_CFG80211_BLOCK_STA_DISCON_EVENT & RTW_CFG80211_BLOCK_DISCON_WHEN_DISCONNECT)
+	rtw_wdev_set_not_indic_disco(adapter_wdev_data(padapter), 0);
+#endif
+
+	return 0;
+}
+
+bool rtw_cfg80211_is_connect_requested(_adapter *adapter)
+{
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(adapter);
+	bool requested;
+
+	_rtw_spinlock_bh(&pwdev_priv->connect_req_lock);
+	requested = pwdev_priv->connect_req ? 1 : 0;
+	_rtw_spinunlock_bh(&pwdev_priv->connect_req_lock);
+
+	return requested;
+}
+
+static int _rtw_disconnect(struct wiphy *wiphy, struct net_device *ndev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	u8 ret = _FAIL;
+
+	/* if(check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) */
+	{
+		rtw_scan_abort(padapter, 0);
+		rtw_join_abort_timeout(padapter, MAX_JOIN_TIMEOUT);
+
+		ret = rtw_disassoc_cmd(padapter, 500, RTW_CMDF_WAIT_ACK);
+		if (ret == _FAIL && padapter->disconnect_token)
+			return 0;
+
+		if ((MLME_IS_ASOC(padapter) == _TRUE) && (MLME_IS_STA(padapter) == _FALSE))
+			rtw_free_assoc_resources_cmd(padapter, _TRUE, RTW_CMDF_WAIT_ACK);
+
+		RTW_INFO("%s...call rtw_indicate_disconnect\n", __func__);
+		/* indicate locally_generated = 0 when suspend */
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0))
+		rtw_indicate_disconnect(padapter, 0, wiphy->dev.power.is_prepared ? _FALSE : _TRUE);
+		#else
+		/*
+		* for kernel < 4.2, DISCONNECT event is hardcoded with
+		* NL80211_ATTR_DISCONNECTED_BY_AP=1 in NL80211 layer
+		* no need to judge if under suspend
+		*/
+		rtw_indicate_disconnect(padapter, 0, _TRUE);
+		#endif
+
+		pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+		pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND;
+		pmlmeinfo->wifi_reason_code = WLAN_REASON_DEAUTH_LEAVING;
+	}
+	return 0;
+}
+
+#if (KERNEL_VERSION(4, 17, 0) > LINUX_VERSION_CODE) \
+    && !defined(CONFIG_KERNEL_PATCH_EXTERNAL_AUTH)
+static bool rtw_check_connect_sae_compat(struct cfg80211_connect_params *sme)
+{
+	struct rtw_ieee802_11_elems elems;
+	struct rsne_info info;
+	u8 AKM_SUITE_SAE[] = {0x00, 0x0f, 0xac, 8};
+	int i;
+
+	if (sme->auth_type != (int)MLME_AUTHTYPE_SHARED_KEY)
+		return false;
+
+	if (rtw_ieee802_11_parse_elems((u8 *)sme->ie, sme->ie_len, &elems, 0)
+	    == ParseFailed)
+		return false;
+
+	if (!elems.rsn_ie)
+		return false;
+
+	if (rtw_rsne_info_parse(elems.rsn_ie - 2, elems.rsn_ie_len + 2, &info) == _FAIL)
+		return false;
+
+	for (i = 0; i < info.akm_cnt; i++)
+		if (_rtw_memcmp(info.akm_list + i * RSN_SELECTOR_LEN,
+				AKM_SUITE_SAE, RSN_SELECTOR_LEN) == _TRUE)
+			return true;
+
+	return false;
+}
+#else
+#define rtw_check_connect_sae_compat(sme)	false
+#endif
+
+/* todo: move to rtw_security.c ? */
+static int rtw_set_security(struct _ADAPTER *a,
+			    struct cfg80211_connect_params *sme)
+{
+	struct security_priv *sec = &a->securitypriv;
+	int ret = 0;
+
+
+	sec->ndisencryptstatus = Ndis802_11EncryptionDisabled;
+	sec->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+	sec->dot118021XGrpPrivacy = _NO_PRIVACY_;
+	sec->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+	sec->ndisauthtype = Ndis802_11AuthModeOpen;
+	sec->auth_alg = WLAN_AUTH_OPEN;
+	sec->extauth_status = WLAN_STATUS_UNSPECIFIED_FAILURE;
+
+	ret = rtw_cfg80211_set_wpa_version(sec, sme->crypto.wpa_versions);
+	if (ret < 0)
+		return -1;
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (sme->crypto.wpa_versions & NL80211_WAPI_VERSION_1) {
+		a->wapiInfo.bWapiEnable = true;
+		a->wapiInfo.extra_prefix_len = WAPI_EXT_LEN;
+		a->wapiInfo.extra_postfix_len = SMS4_MIC_LEN;
+	} else {
+		a->wapiInfo.bWapiEnable = false;
+	}
+#endif
+
+	ret = rtw_cfg80211_set_auth_type(sec, sme->auth_type);
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (sec->dot11AuthAlgrthm == dot11AuthAlgrthm_WAPI)
+		a->mlmeextpriv.mlmext_info.auth_algo = sec->dot11AuthAlgrthm;
+#endif
+	if (ret < 0)
+		return -1;
+
+	if (sme->crypto.n_ciphers_pairwise) {
+		ret = rtw_cfg80211_set_cipher(sec, sme->crypto.ciphers_pairwise[0], _TRUE);
+		if (ret < 0)
+			return -1;
+	}
+
+	/* For WEP Shared auth */
+	if (sme->key_len > 0 && sme->key) {
+		u32 wep_key_idx, wep_key_len, wep_total_len;
+		NDIS_802_11_WEP	*pwep = NULL;
+		RTW_INFO("%s(): Shared/Auto WEP\n", __FUNCTION__);
+
+		wep_key_idx = sme->key_idx;
+		wep_key_len = sme->key_len;
+
+		if (sme->key_idx > WEP_KEYS)
+			return -EINVAL;
+
+		if (!wep_key_len)
+			return -EINVAL;
+		wep_key_len = wep_key_len <= 5 ? 5 : 13;
+		wep_total_len = wep_key_len + FIELD_OFFSET(NDIS_802_11_WEP, KeyMaterial);
+		pwep = (NDIS_802_11_WEP *) rtw_malloc(wep_total_len);
+		if (pwep == NULL) {
+			RTW_INFO(" wpa_set_encryption: pwep allocate fail !!!\n");
+			return -ENOMEM;
+		}
+
+		_rtw_memset(pwep, 0, wep_total_len);
+
+		pwep->KeyLength = wep_key_len;
+		pwep->Length = wep_total_len;
+
+		if (wep_key_len == 13) {
+			a->securitypriv.dot11PrivacyAlgrthm = _WEP104_;
+			a->securitypriv.dot118021XGrpPrivacy = _WEP104_;
+		}
+
+		pwep->KeyIndex = wep_key_idx;
+		pwep->KeyIndex |= 0x80000000;
+
+		_rtw_memcpy(pwep->KeyMaterial, (void *)sme->key, pwep->KeyLength);
+
+		if (rtw_set_802_11_add_wep(a, pwep) == (u8)_FAIL)
+			ret = -EOPNOTSUPP ;
+
+		if (pwep)
+			rtw_mfree((u8 *)pwep, wep_total_len);
+
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = rtw_cfg80211_set_cipher(sec, sme->crypto.cipher_group, _FALSE);
+	if (ret < 0)
+		return ret;
+
+	if (sme->crypto.n_akm_suites) {
+		ret = rtw_cfg80211_set_key_mgt(sec, sme->crypto.akm_suites[0]);
+		if (ret < 0)
+			return ret;
+	}
+#ifdef CONFIG_8011R
+	else {
+		/*It could be a connection without RSN IEs*/
+		sec->rsn_akm_suite_type = 0;
+	}
+#endif
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_WAPI_PSK)
+		a->wapiInfo.bWapiPSK = true;
+	else if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_WAPI_CERT)
+		a->wapiInfo.bWapiPSK = false;
+#endif
+
+	rtw_set_802_11_authentication_mode(a, sec->ndisauthtype);
+
+	/* rtw_set_802_11_encryption_mode(a, a->securitypriv.ndisencryptstatus); */
+
+	return 0;
+}
+
+static int rtw_set_wpa_ie(struct _ADAPTER *a,
+			  struct cfg80211_connect_params *sme)
+{
+	return rtw_cfg80211_set_wpa_ie(a, (u8 *)sme->ie, sme->ie_len);
+}
+
+static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
+				struct cfg80211_connect_params *sme)
+{
+	int ret = 0;
+	const u8 *bssid = NULL;
+	NDIS_802_11_SSID ndis_ssid;
+	/* u8 matched_by_bssid=_FALSE; */
+	/* u8 matched_by_ssid=_FALSE; */
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u16 ch_hw_vlue = 0;
+	enum band_type ch_band = BAND_MAX;
+
+#if (RTW_CFG80211_BLOCK_STA_DISCON_EVENT & RTW_CFG80211_BLOCK_DISCON_WHEN_CONNECT)
+	rtw_wdev_set_not_indic_disco(pwdev_priv, 1);
+#endif
+
+	RTW_INFO("=>"FUNC_NDEV_FMT" - Start to Connection\n", FUNC_NDEV_ARG(ndev));
+	RTW_INFO("privacy=%d, key=%p, key_len=%d, key_idx=%d, auth_type=%d\n",
+		sme->privacy, sme->key, sme->key_len, sme->key_idx, sme->auth_type);
+
+	if (rtw_check_connect_sae_compat(sme)) {
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+		sme->auth_type = NL80211_AUTHTYPE_SAE;
+#else /* LINUX_VERSION_CODE < 3.8 */
+		/*
+		 * When the linux kernel version is less than v3.8, there is no
+		 * NL80211_AUTHTYPE_SAE, also the driver checks psecuritypriv->auth_type
+		 * in rtw_cfg80211_set_auth_type() instead of sme's auth_type in this
+		 * case already. So NL80211_AUTHTYPE_AUTOMATIC is set here when there
+		 * is no NL80211_AUTHTYPE_SAE in this case.
+		 *
+		 * PS: NL80211_AUTHTYPE_AUTOMATIC is NL80211_AUTHTYPE_MAX + 1
+		 *     So it won't conflicts with other defined type in this case.
+		 *
+		 * PS: The driver supports from Linux Kernel 2.6.35. and there is
+		 *     NL80211_AUTHTYPE_MAX / NL80211_AUTHTYPE_AUTOMATIC defined
+		 *     in linux kernel 2.6.35 already.
+		 */
+		sme->auth_type = NL80211_AUTHTYPE_AUTOMATIC;
+#endif /* LINUX_VERSION_CODE */
+
+		psecuritypriv->auth_type = MLME_AUTHTYPE_SAE;
+		psecuritypriv->auth_alg = WLAN_AUTH_SAE;
+		RTW_INFO("%s set sme->auth_type for SAE compat\n", __FUNCTION__);
+	}
+
+	if (pwdev_priv->block == _TRUE) {
+		ret = -EBUSY;
+		RTW_INFO("%s wdev_priv.block is set\n", __FUNCTION__);
+		goto exit;
+	}
+
+       if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE | WIFI_UNDER_LINKING) == _TRUE) {
+
+		_rtw_disconnect(wiphy, ndev);
+		RTW_INFO("%s disconnect before connecting! fw_state=0x%x\n",
+			__FUNCTION__, pmlmepriv->fw_state);
+	}
+
+	if (!sme->ssid || !sme->ssid_len) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (sme->ssid_len > IW_ESSID_MAX_SIZE) {
+		ret = -E2BIG;
+		goto exit;
+	}
+
+	rtw_mi_scan_abort(padapter, _TRUE);
+
+	rtw_join_abort_timeout(padapter, MAX_JOIN_TIMEOUT);
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_LINKING)) {
+		ret = -EINVAL;
+		goto exit;
+	}
+#endif
+
+	_rtw_memset(&ndis_ssid, 0, sizeof(NDIS_802_11_SSID));
+	ndis_ssid.SsidLength = sme->ssid_len;
+	_rtw_memcpy(ndis_ssid.Ssid, (u8 *)sme->ssid, sme->ssid_len);
+
+	RTW_INFO("ssid=%s, len=%zu\n", ndis_ssid.Ssid, sme->ssid_len);
+
+
+	if (sme->bssid) {
+		RTW_INFO("bssid="MAC_FMT"\n", MAC_ARG(sme->bssid));
+		bssid = sme->bssid;
+	}
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0))
+	else if (sme->bssid_hint) {
+		RTW_INFO("bssid_hint="MAC_FMT"\n", MAC_ARG(sme->bssid_hint));
+		bssid = sme->bssid_hint;
+	}
+#endif
+
+	ret = rtw_set_security(padapter, sme);
+	if (ret < 0)
+		goto exit;
+
+	if (!is_wep_enc(psecuritypriv->dot11PrivacyAlgrthm) && (psecuritypriv->dot11PrivacyAlgrthm != _SMS4_)) {
+		ret = rtw_set_wpa_ie(padapter, sme);
+		if (ret < 0)
+			goto exit;
+	}
+#ifdef CONFIG_RTW_MBO
+	rtw_mbo_update_ie_data(padapter, (u8 *)sme->ie, sme->ie_len);
+#endif
+
+	if (sme->channel) {
+		ch_hw_vlue = sme->channel->hw_value;
+		ch_band = nl80211_band_to_rtw_band(sme->channel->band);
+	}
+
+	if (rtw_set_802_11_connect(padapter, bssid, &ndis_ssid,
+				   ch_hw_vlue, ch_band) == _FALSE) {
+		ret = -1;
+		goto exit;
+	}
+
+
+	_rtw_spinlock_bh(&pwdev_priv->connect_req_lock);
+
+	if (pwdev_priv->connect_req) {
+		rtw_wdev_free_connect_req(pwdev_priv);
+		RTW_INFO(FUNC_NDEV_FMT" free existing connect_req\n", FUNC_NDEV_ARG(ndev));
+	}
+
+	pwdev_priv->connect_req = (struct cfg80211_connect_params *)rtw_malloc(sizeof(*pwdev_priv->connect_req));
+	if (pwdev_priv->connect_req)
+		_rtw_memcpy(pwdev_priv->connect_req, sme, sizeof(*pwdev_priv->connect_req));
+	else
+		RTW_WARN(FUNC_NDEV_FMT" alloc connect_req fail\n", FUNC_NDEV_ARG(ndev));
+
+	_rtw_spinunlock_bh(&pwdev_priv->connect_req_lock);
+
+	RTW_INFO("set ssid:dot11AuthAlgrthm=%d, dot11PrivacyAlgrthm=%d, dot118021XGrpPrivacy=%d\n", psecuritypriv->dot11AuthAlgrthm, psecuritypriv->dot11PrivacyAlgrthm,
+		psecuritypriv->dot118021XGrpPrivacy);
+
+exit:
+	RTW_INFO("<=%s, ret %d\n", __FUNCTION__, ret);
+
+#if (RTW_CFG80211_BLOCK_STA_DISCON_EVENT & RTW_CFG80211_BLOCK_DISCON_WHEN_CONNECT)
+	rtw_wdev_set_not_indic_disco(pwdev_priv, 0);
+#endif
+
+	return ret;
+}
+
+static int cfg80211_rtw_disconnect(struct wiphy *wiphy, struct net_device *ndev,
+				   u16 reason_code)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+
+	RTW_INFO(FUNC_NDEV_FMT" - Start to Disconnect\n", FUNC_NDEV_ARG(ndev));
+
+#if (RTW_CFG80211_BLOCK_STA_DISCON_EVENT & RTW_CFG80211_BLOCK_DISCON_WHEN_DISCONNECT)
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	if (!wiphy->dev.power.is_prepared)
+	#endif
+		rtw_wdev_set_not_indic_disco(adapter_wdev_data(padapter), 1);
+#endif
+
+	rtw_set_to_roam(padapter, 0);
+
+	/* if(check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) */
+	{
+		_rtw_disconnect(wiphy, ndev);
+	}
+
+#if (RTW_CFG80211_BLOCK_STA_DISCON_EVENT & RTW_CFG80211_BLOCK_DISCON_WHEN_DISCONNECT)
+	rtw_wdev_set_not_indic_disco(adapter_wdev_data(padapter), 0);
+#endif
+
+	RTW_INFO(FUNC_NDEV_FMT" return 0\n", FUNC_NDEV_ARG(ndev));
+	return 0;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static const char *nl80211_tx_power_setting_str(int type)
+{
+	switch (type) {
+	case NL80211_TX_POWER_AUTOMATIC:
+		return "AUTO";
+	case NL80211_TX_POWER_LIMITED:
+		return "LIMIT";
+	case NL80211_TX_POWER_FIXED:
+		return "FIX";
+	default:
+		return "UNKNOWN";
+	};
+}
+#endif	/*	CONFIG_RTW_DEBUG	*/
+
+static int cfg80211_rtw_set_txpower(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	struct wireless_dev *wdev,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)) || defined(COMPAT_KERNEL_RELEASE)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0))
+	int radio_idx,
+#endif
+	enum nl80211_tx_power_setting type, int mbm)
+#else
+	enum tx_power_setting type, int dbm)
+#endif
+{
+#if !((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)) || defined(COMPAT_KERNEL_RELEASE))
+	int mbm = dbm * 100;
+#endif
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	_adapter *adapter = wiphy_to_adapter(wiphy);
+	int ret = -EOPNOTSUPP;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	if (wdev) {
+		RTW_WARN(FUNC_WIPHY_FMT" wdev specific control is not supported\n", FUNC_WIPHY_ARG(wiphy));
+		goto exit;
+	}
+#endif
+
+	RTW_INFO(FUNC_WIPHY_FMT" type:%s(%u) mbm:%d\n", FUNC_WIPHY_ARG(wiphy)
+		, nl80211_tx_power_setting_str(type), type, mbm);
+
+	switch (type) {
+	case NL80211_TX_POWER_AUTOMATIC:
+		wiphy_data->txpwr_total_lmt_mbm = UNSPECIFIED_MBM;
+		ret = 0;
+		break;
+	case NL80211_TX_POWER_LIMITED:
+		if (!phy_is_txpwr_user_mbm_valid(adapter, mbm)) {
+			RTW_WARN(FUNC_WIPHY_FMT" mbm:%d not support\n", FUNC_WIPHY_ARG(wiphy), mbm);
+			goto exit;
+		}
+		wiphy_data->txpwr_total_lmt_mbm = mbm;
+		ret = 0;
+		break;
+	case NL80211_TX_POWER_FIXED:
+		if (!phy_is_txpwr_user_mbm_valid(adapter, mbm)) {
+			RTW_WARN(FUNC_WIPHY_FMT" mbm:%d not support\n", FUNC_WIPHY_ARG(wiphy), mbm);
+			goto exit;
+		}
+		//TODO
+		RTW_INFO("%s Not supported set txpower fixed\n", __func__);
+		ret = -ENOSYS;
+		break;
+	default:
+		RTW_WARN(FUNC_WIPHY_FMT" unknown type:%d\n", FUNC_WIPHY_ARG(wiphy), type);
+	}
+
+	if (ret == 0)
+		rtw_run_in_thread_cmd_wait(adapter, ((void *)(rtw_update_txpwr_level_all_hwband)), adapter_to_dvobj(adapter), 2000);
+
+exit:
+	return ret;
+}
+
+static int cfg80211_rtw_get_txpower(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	struct wireless_dev *wdev,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0))
+	int radio_idx,
+	unsigned int link_id,
+#endif
+#endif
+	int *dbm)
+{
+	RTW_INFO("%s\n", __func__);
+
+	*dbm = (12);
+
+	return 0;
+}
+
+inline bool rtw_cfg80211_pwr_mgmt(_adapter *adapter)
+{
+	struct rtw_wdev_priv *rtw_wdev_priv = adapter_wdev_data(adapter);
+	return rtw_wdev_priv->power_mgmt;
+}
+
+static int cfg80211_rtw_set_power_mgmt(struct wiphy *wiphy,
+				       struct net_device *ndev,
+				       bool enabled, int timeout)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct rtw_wdev_priv *rtw_wdev_priv = adapter_wdev_data(padapter);
+#ifdef CONFIG_POWER_SAVE
+#ifdef CONFIG_RTW_LPS
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_phl_com_t *phl_com = dvobj->phl_com;
+	struct rtw_ps_cap_t *ps_cap_p = &phl_com->dev_cap.ps_cap;
+#endif /* CONFIG_RTW_LPS */
+#endif /* CONFIG_POWER_SAVE */
+
+	RTW_INFO(FUNC_NDEV_FMT" enabled:%u, timeout:%d\n", FUNC_NDEV_ARG(ndev),
+		enabled, timeout);
+
+	rtw_wdev_priv->power_mgmt = enabled;
+
+#ifdef CONFIG_POWER_SAVE
+#ifdef CONFIG_RTW_LPS
+	if (adapter_to_regsty(padapter)->lps_mode != PS_OP_MODE_DISABLED) {
+		if (enabled) {
+			if (ps_cap_p->lps_en == PS_OP_MODE_DISABLED)
+				rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, PS_OP_MODE_AUTO);
+		} else {
+			if (ps_cap_p->lps_en != PS_OP_MODE_DISABLED)
+				rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, PS_OP_MODE_DISABLED);
+		}
+	}
+#endif /* CONFIG_RTW_LPS */
+#endif /* CONFIG_POWER_SAVE */
+
+	return 0;
+}
+
+static void _rtw_set_pmksa(struct net_device *ndev,
+	u8 *bssid, u8 *pmkid)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	u8 index, blInserted = _FALSE;
+
+	/* overwrite PMKID */
+	for (index = 0 ; index < NUM_PMKID_CACHE; index++) {
+		if (_rtw_memcmp(psecuritypriv->PMKIDList[index].Bssid, bssid, ETH_ALEN) == _TRUE) {
+			/* BSSID is matched, the same AP => rewrite with new PMKID. */
+			RTW_INFO("BSSID("MAC_FMT") exists in the PMKList.\n", MAC_ARG(bssid));
+
+			_rtw_memcpy(psecuritypriv->PMKIDList[index].PMKID, pmkid, WLAN_PMKID_LEN);
+			psecuritypriv->PMKIDList[index].bUsed = _TRUE;
+			blInserted = _TRUE;
+			break;
+		}
+	}
+
+	if (!blInserted) {
+		/* Find a new entry */
+		RTW_INFO("Use the new entry index = %d for this PMKID.\n",
+			psecuritypriv->PMKIDIndex);
+
+		_rtw_memcpy(psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].Bssid, bssid, ETH_ALEN);
+		_rtw_memcpy(psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].PMKID, pmkid, WLAN_PMKID_LEN);
+
+		psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].bUsed = _TRUE;
+		psecuritypriv->PMKIDIndex++ ;
+		if (psecuritypriv->PMKIDIndex == 16)
+			psecuritypriv->PMKIDIndex = 0;
+	}
+}
+
+static int cfg80211_rtw_set_pmksa(struct wiphy *wiphy,
+				  struct net_device *ndev,
+				  struct cfg80211_pmksa *pmksa)
+{
+	u8	index, blInserted = _FALSE;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct mlme_priv *mlme = &padapter->mlmepriv;
+	struct security_priv	*psecuritypriv = &padapter->securitypriv;
+	u8	strZeroMacAddress[ETH_ALEN] = { 0x00 };
+	bool sae_auth = rtw_sec_chk_auth_type(padapter, MLME_AUTHTYPE_SAE);
+
+	RTW_INFO(FUNC_NDEV_FMT" "MAC_FMT" "KEY_FMT"\n", FUNC_NDEV_ARG(ndev)
+		, MAC_ARG(pmksa->bssid), KEY_ARG(pmksa->pmkid));
+
+	if (_rtw_memcmp((u8 *)pmksa->bssid, strZeroMacAddress, ETH_ALEN) == _TRUE)
+		return -EINVAL;
+
+	_rtw_set_pmksa(ndev, (u8 *)pmksa->bssid, (u8 *)pmksa->pmkid);
+
+	if (sae_auth &&
+		(psecuritypriv->extauth_status == WLAN_STATUS_SUCCESS)) {
+		RTW_PRINT("SAE: auth success, start assoc\n");
+		start_clnt_assoc(padapter);
+	}
+
+	return 0;
+}
+
+static int cfg80211_rtw_del_pmksa(struct wiphy *wiphy,
+				  struct net_device *ndev,
+				  struct cfg80211_pmksa *pmksa)
+{
+	u8	index, bMatched = _FALSE;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct security_priv	*psecuritypriv = &padapter->securitypriv;
+
+	RTW_INFO(FUNC_NDEV_FMT" "MAC_FMT" "KEY_FMT"\n", FUNC_NDEV_ARG(ndev)
+		, MAC_ARG(pmksa->bssid), KEY_ARG(pmksa->pmkid));
+
+	for (index = 0 ; index < NUM_PMKID_CACHE; index++) {
+		if (_rtw_memcmp(psecuritypriv->PMKIDList[index].Bssid, (u8 *)pmksa->bssid, ETH_ALEN) == _TRUE) {
+			/* BSSID is matched, the same AP => Remove this PMKID information and reset it. */
+			_rtw_memset(psecuritypriv->PMKIDList[index].Bssid, 0x00, ETH_ALEN);
+			_rtw_memset(psecuritypriv->PMKIDList[index].PMKID, 0x00, WLAN_PMKID_LEN);
+			psecuritypriv->PMKIDList[index].bUsed = _FALSE;
+			bMatched = _TRUE;
+			RTW_INFO(FUNC_NDEV_FMT" clear id:%hhu\n", FUNC_NDEV_ARG(ndev), index);
+			break;
+		}
+	}
+
+	if (_FALSE == bMatched) {
+		RTW_INFO(FUNC_NDEV_FMT" do not have matched BSSID\n"
+			, FUNC_NDEV_ARG(ndev));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int cfg80211_rtw_flush_pmksa(struct wiphy *wiphy,
+				    struct net_device *ndev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct security_priv	*psecuritypriv = &padapter->securitypriv;
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	_rtw_memset(&psecuritypriv->PMKIDList[0], 0x00, sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
+	psecuritypriv->PMKIDIndex = 0;
+
+	return 0;
+}
+
+#ifdef CONFIG_AP_MODE
+void rtw_cfg80211_indicate_sta_assoc(_adapter *padapter, u8 *pmgmt_frame, uint frame_len)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	enum band_type band;
+#if !defined(RTW_USE_CFG80211_STA_EVENT) && !defined(COMPAT_KERNEL_RELEASE)
+	s32 freq;
+	int channel;
+	struct wireless_dev *pwdev = padapter->rtw_wdev;
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+#endif
+	struct net_device *ndev = padapter->pnetdev;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+#if defined(RTW_USE_CFG80211_STA_EVENT) || defined(COMPAT_KERNEL_RELEASE)
+	{
+		struct station_info sinfo;
+		u8 ie_offset;
+		if (get_frame_sub_type(pmgmt_frame) == WIFI_ASSOCREQ)
+			ie_offset = _ASOCREQ_IE_OFFSET_;
+		else /* WIFI_REASSOCREQ */
+			ie_offset = _REASOCREQ_IE_OFFSET_;
+
+		memset(&sinfo, 0, sizeof(sinfo));
+		sinfo.filled = STATION_INFO_ASSOC_REQ_IES;
+		sinfo.assoc_req_ies = pmgmt_frame + WLAN_HDR_A3_LEN + ie_offset;
+		sinfo.assoc_req_ies_len = frame_len - WLAN_HDR_A3_LEN - ie_offset;
+		cfg80211_new_sta(ndev, get_addr2_ptr(pmgmt_frame), &sinfo, GFP_ATOMIC);
+	}
+#else /* defined(RTW_USE_CFG80211_STA_EVENT) */
+	channel = pmlmeext->chandef.chan;
+	band = pmlmeext->chandef.band;
+	freq = rtw_bch2freq(band, channel);
+
+	#ifdef COMPAT_KERNEL_RELEASE
+	rtw_cfg80211_rx_mgmt(pwdev, freq, 0, pmgmt_frame, frame_len, GFP_ATOMIC);
+	#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) && !defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER)
+	rtw_cfg80211_rx_mgmt(pwdev, freq, 0, pmgmt_frame, frame_len, GFP_ATOMIC);
+	#else /* COMPAT_KERNEL_RELEASE */
+	{
+		/* to avoid WARN_ON(wdev->iftype != NL80211_IFTYPE_STATION)  when calling cfg80211_send_rx_assoc() */
+		#ifndef CONFIG_PLATFORM_MSTAR
+		pwdev->iftype = NL80211_IFTYPE_STATION;
+		#endif /* CONFIG_PLATFORM_MSTAR */
+		RTW_INFO("iftype=%d before call cfg80211_send_rx_assoc()\n", pwdev->iftype);
+		rtw_cfg80211_send_rx_assoc(padapter, NULL, pmgmt_frame, frame_len);
+		RTW_INFO("iftype=%d after call cfg80211_send_rx_assoc()\n", pwdev->iftype);
+		pwdev->iftype = NL80211_IFTYPE_AP;
+		/* cfg80211_rx_action(padapter->pnetdev, freq, pmgmt_frame, frame_len, GFP_ATOMIC); */
+	}
+	#endif /* COMPAT_KERNEL_RELEASE */
+#endif /* defined(RTW_USE_CFG80211_STA_EVENT) */
+
+}
+
+void rtw_cfg80211_indicate_sta_disassoc(_adapter *padapter, const u8 *da, unsigned short reason)
+{
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	enum band_type band;
+#if !defined(RTW_USE_CFG80211_STA_EVENT) && !defined(COMPAT_KERNEL_RELEASE)
+	s32 freq;
+	int channel;
+	u8 *pmgmt_frame;
+	uint frame_len;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+	unsigned short *fctrl;
+	u8 mgmt_buf[128] = {0};
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct wireless_dev *wdev = padapter->rtw_wdev;
+#endif
+	struct net_device *ndev = padapter->pnetdev;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+#if defined(RTW_USE_CFG80211_STA_EVENT) || defined(COMPAT_KERNEL_RELEASE)
+	cfg80211_del_sta(ndev, da, GFP_ATOMIC);
+#else /* defined(RTW_USE_CFG80211_STA_EVENT) */
+	channel = padapter_link->mlmeext.chandef.chan;
+	band = padapter_link->mlmeext.chandef.chan;
+	freq = rtw_bch2freq(band, channel);
+
+	pmgmt_frame = mgmt_buf;
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pmgmt_frame;
+
+	fctrl = &(pwlanhdr->frame_ctl);
+	*(fctrl) = 0;
+
+	_rtw_memcpy(pwlanhdr->addr1, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr2, da, ETH_ALEN);
+	_rtw_memcpy(pwlanhdr->addr3, get_my_bssid(&(pmlmeinfo->dev_network)), ETH_ALEN);
+
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pmlmeext->mgnt_seq++;
+	set_frame_sub_type(pmgmt_frame, WIFI_DEAUTH);
+
+	pmgmt_frame += sizeof(struct rtw_ieee80211_hdr_3addr);
+	frame_len = sizeof(struct rtw_ieee80211_hdr_3addr);
+
+	reason = cpu_to_le16(reason);
+	pmgmt_frame = rtw_set_fixed_ie(pmgmt_frame, _RSON_CODE_ , (unsigned char *)&reason, &frame_len);
+
+	#ifdef COMPAT_KERNEL_RELEASE
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, mgmt_buf, frame_len, GFP_ATOMIC);
+	#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) && !defined(CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER)
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, mgmt_buf, frame_len, GFP_ATOMIC);
+	#else /* COMPAT_KERNEL_RELEASE */
+	cfg80211_send_disassoc(padapter->pnetdev, mgmt_buf, frame_len);
+	/* cfg80211_rx_action(padapter->pnetdev, freq, mgmt_buf, frame_len, GFP_ATOMIC); */
+	#endif /* COMPAT_KERNEL_RELEASE */
+#endif /* defined(RTW_USE_CFG80211_STA_EVENT) */
+}
+
+static int rtw_cfg80211_monitor_if_open(struct net_device *ndev)
+{
+	int ret = 0;
+
+	RTW_INFO("%s\n", __func__);
+
+	return ret;
+}
+
+static int rtw_cfg80211_monitor_if_close(struct net_device *ndev)
+{
+	int ret = 0;
+
+	RTW_INFO("%s\n", __func__);
+
+	return ret;
+}
+
+static int rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struct net_device *ndev)
+{
+	int ret = 0;
+	int rtap_len;
+	int qos_len = 0;
+	int dot11_hdr_len = 24;
+	int snap_len = 6;
+	unsigned char *pdata;
+	u16 frame_ctl;
+	unsigned char src_mac_addr[ETH_ALEN];
+	unsigned char dst_mac_addr[ETH_ALEN];
+	struct rtw_ieee80211_hdr *dot11_hdr;
+	struct ieee80211_radiotap_header *rtap_hdr;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	rtw_mstat_update(MSTAT_TYPE_SKB, MSTAT_ALLOC_SUCCESS, skb->truesize);
+
+	if (alink_regu_block_tx(padapter_link))
+		goto fail;
+
+	if (unlikely(skb->len < sizeof(struct ieee80211_radiotap_header)))
+		goto fail;
+
+	rtap_hdr = (struct ieee80211_radiotap_header *)skb->data;
+	if (unlikely(rtap_hdr->it_version))
+		goto fail;
+
+	rtap_len = ieee80211_get_radiotap_len(skb->data);
+	if (unlikely(skb->len < rtap_len))
+		goto fail;
+
+	if (rtap_len != 14) {
+		RTW_INFO("radiotap len (should be 14): %d\n", rtap_len);
+		goto fail;
+	}
+
+	/* Skip the ratio tap header */
+	skb_pull(skb, rtap_len);
+
+	dot11_hdr = (struct rtw_ieee80211_hdr *)skb->data;
+	frame_ctl = le16_to_cpu(dot11_hdr->frame_ctl);
+	/* Check if the QoS bit is set */
+	if ((frame_ctl & RTW_IEEE80211_FCTL_FTYPE) == RTW_IEEE80211_FTYPE_DATA) {
+		/* Check if this ia a Wireless Distribution System (WDS) frame
+		 * which has 4 MAC addresses
+		 */
+		if (dot11_hdr->frame_ctl & 0x0080)
+			qos_len = 2;
+		if ((dot11_hdr->frame_ctl & 0x0300) == 0x0300)
+			dot11_hdr_len += 6;
+
+		_rtw_memcpy(dst_mac_addr, dot11_hdr->addr1, sizeof(dst_mac_addr));
+		_rtw_memcpy(src_mac_addr, dot11_hdr->addr2, sizeof(src_mac_addr));
+
+		/* Skip the 802.11 header, QoS (if any) and SNAP, but leave spaces for
+		 * for two MAC addresses
+		 */
+		skb_pull(skb, dot11_hdr_len + qos_len + snap_len - sizeof(src_mac_addr) * 2);
+		pdata = (unsigned char *)skb->data;
+		_rtw_memcpy(pdata, dst_mac_addr, sizeof(dst_mac_addr));
+		_rtw_memcpy(pdata + sizeof(dst_mac_addr), src_mac_addr, sizeof(src_mac_addr));
+
+		RTW_INFO("should be eapol packet\n");
+
+		/* Use the real net device to transmit the packet */
+		ret = _rtw_xmit_entry(skb, padapter->pnetdev);
+
+		return ret;
+
+	} else if ((frame_ctl & (RTW_IEEE80211_FCTL_FTYPE | RTW_IEEE80211_FCTL_STYPE))
+		== (RTW_IEEE80211_FTYPE_MGMT | RTW_IEEE80211_STYPE_ACTION)
+	) {
+		/* only for action frames */
+		struct xmit_frame		*pmgntframe;
+		struct pkt_attrib	*pattrib;
+		unsigned char	*pframe;
+		/* u8 category, action, OUI_Subtype, dialogToken=0; */
+		/* unsigned char	*frame_body; */
+		struct rtw_ieee80211_hdr *pwlanhdr;
+		struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+		struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+		u8 *buf = skb->data;
+		u32 len = skb->len;
+		u8 category, action;
+		int type = -1;
+
+		if (rtw_action_frame_parse(buf, len, &category, &action) == _FALSE) {
+			RTW_INFO(FUNC_NDEV_FMT" frame_control:0x%x\n", FUNC_NDEV_ARG(ndev),
+				le16_to_cpu(((struct rtw_ieee80211_hdr_3addr *)buf)->frame_ctl));
+			goto fail;
+		}
+
+		RTW_INFO("RTW_Tx:da="MAC_FMT" via "FUNC_NDEV_FMT"\n",
+			MAC_ARG(GetAddr1Ptr(buf)), FUNC_NDEV_ARG(ndev));
+		#ifdef CONFIG_P2P
+		type = rtw_p2p_check_frames(padapter, buf, len, _TRUE);
+		if (type >= 0)
+			goto dump;
+		#endif
+		if (category == RTW_WLAN_CATEGORY_PUBLIC)
+			RTW_INFO("RTW_Tx:%s\n", action_public_str(action));
+		else
+			RTW_INFO("RTW_Tx:category(%u), action(%u)\n", category, action);
+#ifdef CONFIG_P2P
+dump:
+#endif
+		/* starting alloc mgmt frame to dump it */
+		pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+		if (pmgntframe == NULL)
+			goto fail;
+
+		/* update attribute */
+		pattrib = &pmgntframe->attrib;
+		update_mgntframe_attrib(padapter, padapter_link, pattrib);
+		pattrib->retry_ctrl = _FALSE;
+
+		_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+		pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+
+		_rtw_memcpy(pframe, (void *)buf, len);
+		pattrib->pktlen = len;
+
+#ifdef CONFIG_P2P
+		if (type >= 0)
+			rtw_xframe_chk_wfd_ie(pmgntframe);
+#endif /* CONFIG_P2P */
+
+		pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+		/* update seq number */
+		pmlmeext->mgnt_seq = GetSequence(pwlanhdr);
+		pattrib->seqnum = pmlmeext->mgnt_seq;
+		pmlmeext->mgnt_seq++;
+
+
+		pattrib->last_txcmdsz = pattrib->pktlen;
+
+		dump_mgntframe(padapter, pmgntframe);
+
+	} else
+		RTW_INFO("frame_ctl=0x%x\n", frame_ctl & (RTW_IEEE80211_FCTL_FTYPE | RTW_IEEE80211_FCTL_STYPE));
+
+
+fail:
+
+	rtw_skb_free(skb);
+
+	return 0;
+
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 2, 0))
+static void rtw_cfg80211_monitor_if_set_multicast_list(struct net_device *ndev)
+{
+	RTW_INFO("%s\n", __func__);
+}
+#endif
+static int rtw_cfg80211_monitor_if_set_mac_address(struct net_device *ndev, void *addr)
+{
+	int ret = 0;
+
+	RTW_INFO("%s\n", __func__);
+
+	return ret;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+static const struct net_device_ops rtw_cfg80211_monitor_if_ops = {
+	.ndo_open = rtw_cfg80211_monitor_if_open,
+	.ndo_stop = rtw_cfg80211_monitor_if_close,
+	.ndo_start_xmit = rtw_cfg80211_monitor_if_xmit_entry,
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 2, 0))
+	.ndo_set_multicast_list = rtw_cfg80211_monitor_if_set_multicast_list,
+	#endif
+	.ndo_set_mac_address = rtw_cfg80211_monitor_if_set_mac_address,
+};
+#endif
+
+static int rtw_cfg80211_add_monitor_if(_adapter *padapter, char *name, struct net_device **ndev)
+{
+	int ret = 0;
+	struct net_device *mon_ndev = NULL;
+	struct wireless_dev *mon_wdev = NULL;
+	struct rtw_netdev_priv_indicator *pnpi;
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+
+	if (!name) {
+		RTW_INFO(FUNC_ADPT_FMT" without specific name\n", FUNC_ADPT_ARG(padapter));
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (pwdev_priv->pmon_ndev) {
+		RTW_INFO(FUNC_ADPT_FMT" monitor interface exist: "NDEV_FMT"\n",
+			FUNC_ADPT_ARG(padapter), NDEV_ARG(pwdev_priv->pmon_ndev));
+		ret = -EBUSY;
+		goto out;
+	}
+
+	mon_ndev = alloc_etherdev(sizeof(struct rtw_netdev_priv_indicator));
+	if (!mon_ndev) {
+		RTW_INFO(FUNC_ADPT_FMT" allocate ndev fail\n", FUNC_ADPT_ARG(padapter));
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	mon_ndev->type = ARPHRD_IEEE80211_RADIOTAP;
+	strncpy(mon_ndev->name, name, IFNAMSIZ);
+	mon_ndev->name[IFNAMSIZ - 1] = 0;
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 11, 8))
+	mon_ndev->priv_destructor = rtw_ndev_destructor;
+#else
+	mon_ndev->destructor = rtw_ndev_destructor;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+	mon_ndev->netdev_ops = &rtw_cfg80211_monitor_if_ops;
+#else
+	mon_ndev->open = rtw_cfg80211_monitor_if_open;
+	mon_ndev->stop = rtw_cfg80211_monitor_if_close;
+	mon_ndev->hard_start_xmit = rtw_cfg80211_monitor_if_xmit_entry;
+	mon_ndev->set_mac_address = rtw_cfg80211_monitor_if_set_mac_address;
+#endif
+
+	pnpi = netdev_priv(mon_ndev);
+	pnpi->priv = padapter;
+	pnpi->sizeof_priv = sizeof(_adapter);
+
+	/*  wdev */
+	mon_wdev = (struct wireless_dev *)rtw_zmalloc(sizeof(struct wireless_dev));
+	if (!mon_wdev) {
+		RTW_INFO(FUNC_ADPT_FMT" allocate mon_wdev fail\n", FUNC_ADPT_ARG(padapter));
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	mon_wdev->wiphy = padapter->rtw_wdev->wiphy;
+	mon_wdev->netdev = mon_ndev;
+	mon_wdev->iftype = NL80211_IFTYPE_MONITOR;
+	mon_ndev->ieee80211_ptr = mon_wdev;
+
+	ret = cfg80211_register_netdevice(mon_ndev);
+	if (ret)
+		goto out;
+
+	*ndev = pwdev_priv->pmon_ndev = mon_ndev;
+	_rtw_memcpy(pwdev_priv->ifname_mon, name, IFNAMSIZ + 1);
+
+out:
+	if (ret && mon_wdev) {
+		rtw_mfree((u8 *)mon_wdev, sizeof(struct wireless_dev));
+		mon_wdev = NULL;
+	}
+
+	if (ret && mon_ndev) {
+		free_netdev(mon_ndev);
+		*ndev = mon_ndev = NULL;
+	}
+
+	return ret;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+static struct wireless_dev *
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) || defined(COMPAT_KERNEL_RELEASE)
+static struct net_device *
+#else
+static int
+#endif
+	cfg80211_rtw_add_virtual_intf(
+		struct wiphy *wiphy,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+		const char *name,
+		#else
+		char *name,
+		#endif
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+		unsigned char name_assign_type,
+		#endif
+		enum nl80211_iftype type,
+		#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) && !defined(CPTCFG_VERSION)
+		u32 *flags,
+		#endif
+		struct vif_params *params)
+{
+	int ret = 0;
+	struct wireless_dev *wdev = NULL;
+	struct net_device *ndev = NULL;
+	_adapter *padapter;
+	struct dvobj_priv *dvobj = wiphy_to_dvobj(wiphy);
+
+	rtw_set_rtnl_lock_holder(dvobj, current);
+
+	RTW_INFO(FUNC_WIPHY_FMT" name:%s, type:%d\n", FUNC_WIPHY_ARG(wiphy), name, type);
+
+	switch (type) {
+	case NL80211_IFTYPE_MONITOR:
+		padapter = wiphy_to_adapter(wiphy); /* TODO: get ap iface ? */
+		ret = rtw_cfg80211_add_monitor_if(padapter, (char *)name, &ndev);
+		if (ret == 0)
+			wdev = ndev->ieee80211_ptr;
+		break;
+
+#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+	case NL80211_IFTYPE_P2P_CLIENT:
+	case NL80211_IFTYPE_P2P_GO:
+#if !RTW_P2P_GROUP_INTERFACE
+		RTW_ERR("%s, can't add GO/GC interface when RTW_P2P_GROUP_INTERFACE is not defined\n", __func__);
+		ret = -EOPNOTSUPP;
+		break;
+#endif
+#endif
+	case NL80211_IFTYPE_STATION:
+	case NL80211_IFTYPE_AP:
+#ifdef CONFIG_RTW_MESH
+	case NL80211_IFTYPE_MESH_POINT:
+#endif
+		padapter = dvobj_get_unregisterd_adapter(dvobj);
+		if (!padapter) {
+			RTW_WARN("adapter pool empty!\n");
+			ret = -ENODEV;
+			break;
+		}
+
+		if (rtw_os_ndev_init(padapter, name) != _SUCCESS) {
+			RTW_WARN("ndev init fail!\n");
+			ret = -ENODEV;
+			break;
+		}
+
+		#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+		if (type == NL80211_IFTYPE_P2P_CLIENT)
+			rtw_p2p_enable(padapter, P2P_ROLE_CLIENT);
+		else if (type == NL80211_IFTYPE_P2P_GO)
+			rtw_p2p_enable(padapter, P2P_ROLE_GO);
+		#endif
+
+		ndev = padapter->pnetdev;
+		wdev = ndev->ieee80211_ptr;
+		break;
+
+#if defined(CONFIG_P2P) && defined(RTW_DEDICATED_P2P_DEVICE)
+	case NL80211_IFTYPE_P2P_DEVICE:
+		ret = rtw_pd_iface_alloc(wiphy, name, &wdev);
+		break;
+#endif
+
+	case NL80211_IFTYPE_ADHOC:
+	case NL80211_IFTYPE_AP_VLAN:
+	case NL80211_IFTYPE_WDS:
+	default:
+		ret = -ENODEV;
+		RTW_INFO("Unsupported interface type\n");
+		break;
+	}
+
+	if (ndev)
+		RTW_INFO(FUNC_WIPHY_FMT" ndev:%p, ret:%d\n", FUNC_WIPHY_ARG(wiphy), ndev, ret);
+	else
+		RTW_INFO(FUNC_WIPHY_FMT" wdev:%p, ret:%d\n", FUNC_WIPHY_ARG(wiphy), wdev, ret);
+
+	rtw_set_rtnl_lock_holder(dvobj, NULL);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	return wdev ? wdev : ERR_PTR(ret);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) || defined(COMPAT_KERNEL_RELEASE)
+	return ndev ? ndev : ERR_PTR(ret);
+#else
+	return ret;
+#endif
+}
+
+static int cfg80211_rtw_del_virtual_intf(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct wireless_dev *wdev
+#else
+	struct net_device *ndev
+#endif
+)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct net_device *ndev = wdev_to_ndev(wdev);
+#endif
+	int ret = 0;
+	struct dvobj_priv *dvobj = wiphy_to_dvobj(wiphy);
+	_adapter *adapter;
+	struct rtw_wdev_priv *pwdev_priv;
+
+	rtw_set_rtnl_lock_holder(dvobj, current);
+
+	if (ndev) {
+		adapter = (_adapter *)rtw_netdev_priv(ndev);
+		pwdev_priv = adapter_wdev_data(adapter);
+
+		if (ndev == pwdev_priv->pmon_ndev) {
+			cfg80211_unregister_netdevice(ndev);
+			pwdev_priv->pmon_ndev = NULL;
+			pwdev_priv->ifname_mon[0] = '\0';
+			RTW_INFO(FUNC_NDEV_FMT" remove monitor ndev\n", FUNC_NDEV_ARG(ndev));
+		} else {
+			RTW_INFO(FUNC_NDEV_FMT" unregister ndev\n", FUNC_NDEV_ARG(ndev));
+			rtw_os_ndev_unregister(adapter);
+		}
+#ifdef CONFIG_P2P
+		if (!rtw_p2p_chk_role(&adapter->wdinfo, P2P_ROLE_DISABLE))
+			rtw_p2p_enable(adapter, P2P_ROLE_DISABLE);
+#endif
+	} else
+#if defined(CONFIG_P2P) && defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev->iftype == NL80211_IFTYPE_P2P_DEVICE) {
+		if (wdev == wiphy_to_pd_wdev(wiphy))
+			rtw_pd_iface_free(wiphy);
+		else {
+			RTW_ERR(FUNC_WIPHY_FMT" unknown P2P Device wdev:%p\n", FUNC_WIPHY_ARG(wiphy), wdev);
+			rtw_warn_on(1);
+		}
+	} else
+#endif
+	{
+		ret = -EINVAL;
+		goto exit;
+	}
+
+exit:
+	rtw_set_rtnl_lock_holder(dvobj, NULL);
+	return ret;
+}
+
+static int rtw_add_beacon(_adapter *adapter, const u8 *head, size_t head_len, const u8 *tail, size_t tail_len)
+{
+	int ret = 0;
+	u8 *pbuf = NULL;
+	uint len, wps_ielen = 0;
+	uint p2p_ielen = 0;
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	/* struct sta_priv *pstapriv = &padapter->stapriv; */
+
+
+	RTW_INFO("%s beacon_head_len=%zu, beacon_tail_len=%zu\n", __FUNCTION__, head_len, tail_len);
+
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	if (head_len < 24)
+		return -EINVAL;
+
+	pbuf = rtw_zmalloc(head_len + tail_len);
+	if (!pbuf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	_rtw_memcpy(pbuf, (void *)head + 24, head_len - 24); /* 24=beacon header len. */
+	_rtw_memcpy(pbuf + head_len - 24, (void *)tail, tail_len);
+
+	len = head_len + tail_len - 24;
+
+	/* check wps ie if inclued */
+	if (rtw_get_wps_ie(pbuf + _FIXED_IE_LENGTH_, len - _FIXED_IE_LENGTH_, NULL, &wps_ielen))
+		RTW_INFO("add bcn, wps_ielen=%d\n", wps_ielen);
+
+#ifdef CONFIG_P2P
+	/* check p2p if enable */
+	if (rtw_get_p2p_ie(pbuf + _FIXED_IE_LENGTH_, len - _FIXED_IE_LENGTH_, NULL, &p2p_ielen)) {
+		struct wifidirect_info *pwdinfo = &(adapter->wdinfo);
+
+		RTW_INFO("got p2p_ie, len=%d\n", p2p_ielen);
+
+		if (!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) {
+			RTW_INFO("add p2p beacon whitout GO mode, p2p_role=%d\n",
+				 rtw_p2p_role(pwdinfo));
+			ret = -EOPNOTSUPP;
+			goto exit;
+		}
+	}
+#endif /* CONFIG_P2P */
+
+	if (adapter_to_dvobj(adapter)->wpas_type == RTW_WPAS_ANDROID) {
+		/* pbss_network->IEs will not include p2p_ie, wfd ie */
+		rtw_ies_remove_ie(pbuf, &len, _BEACON_IE_OFFSET_, _VENDOR_SPECIFIC_IE_, P2P_OUI, 4);
+		rtw_ies_remove_ie(pbuf, &len, _BEACON_IE_OFFSET_, _VENDOR_SPECIFIC_IE_, WFD_OUI, 4);
+	}
+
+	if (rtw_check_beacon_data(adapter, pbuf,  len) == _SUCCESS) {
+		ret = 0;
+	} else
+		ret = -EINVAL;
+
+exit:
+	if (pbuf)
+		rtw_mfree(pbuf, head_len + tail_len);
+
+	return ret;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) && !defined(COMPAT_KERNEL_RELEASE)
+static int cfg80211_rtw_add_beacon(struct wiphy *wiphy, struct net_device *ndev,
+		struct beacon_parameters *info)
+{
+	int ret = 0;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	if (rtw_cfg80211_sync_iftype(adapter) != _SUCCESS) {
+		ret = -ENOTSUPP;
+		goto exit;
+	}
+	rtw_mi_scan_abort(adapter, _TRUE);
+	rtw_mi_buddy_set_scan_deny(adapter, 300);
+	ret = rtw_add_beacon(adapter, info->head, info->head_len, info->tail, info->tail_len);
+
+exit:
+	return ret;
+}
+
+static int cfg80211_rtw_set_beacon(struct wiphy *wiphy, struct net_device *ndev,
+		struct beacon_parameters *info)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_ext_priv *pmlmeext = &(adapter_link->mlmeextpriv);
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	pmlmeext->bstart_bss = _TRUE;
+
+	cfg80211_rtw_add_beacon(wiphy, ndev, info);
+
+	return 0;
+}
+
+static int	cfg80211_rtw_del_beacon(struct wiphy *wiphy, struct net_device *ndev)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	rtw_mi_buddy_set_scan_deny(adapter, 300);
+	rtw_mi_scan_abort(adapter, _TRUE);
+	rtw_stop_ap_cmd(adapter, RTW_CMDF_WAIT_ACK);
+	return 0;
+}
+#else
+static int rtw_cfg80211_set_beacon_ies(struct net_device *net, const u8 *head,
+				       int head_len, const u8 *tail, int tail_len);
+static int cfg80211_rtw_start_ap(struct wiphy *wiphy, struct net_device *ndev,
+		struct cfg80211_ap_settings *settings)
+{
+	int ret = 0;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	if (adapter_to_dvobj(adapter)->wpas_type == RTW_WPAS_W1FI) {
+		/* turn on the beacon send */
+		adapter_link->mlmeextpriv.bstart_bss = _TRUE;
+	}
+
+	RTW_INFO(FUNC_NDEV_FMT" hidden_ssid:%d, auth_type:%d\n", FUNC_NDEV_ARG(ndev),
+		settings->hidden_ssid, settings->auth_type);
+
+	if (rtw_cfg80211_sync_iftype(adapter) != _SUCCESS) {
+		ret = -ENOTSUPP;
+		goto exit;
+	}
+
+#ifdef RTW_PHL_BCN
+	{
+		struct rtw_wifi_role_t *wrole = adapter->phl_role;
+		struct rtw_wifi_role_link_t *rlink = adapter_link->wrlink;
+		if (rlink)
+			rlink->bcn_cmn.bcn_added = 0;
+	}
+#endif
+
+	/* Kernel < v5.x, the auth_type set as NL80211_AUTHTYPE_AUTOMATIC. if
+	 * the AKM SAE in the RSN IE, we have to update the auth_type for SAE in
+	 * rtw_check_beacon_data().
+	 *
+	 * we only update auth_type when rtw_check_beacon_data()
+	 */
+	/* rtw_cfg80211_set_auth_type(&adapter->securitypriv, settings->auth_type); */
+
+	rtw_mi_scan_abort(adapter, _TRUE);
+	rtw_mi_buddy_set_scan_deny(adapter, 300);
+	ret = rtw_add_beacon(adapter, settings->beacon.head, settings->beacon.head_len,
+		settings->beacon.tail, settings->beacon.tail_len);
+
+	pmlmeinfo->hidden_ssid_mode = settings->hidden_ssid;
+
+	rtw_cfg80211_set_beacon_ies(ndev, settings->beacon.head,
+				    settings->beacon.head_len,
+				    settings->beacon.tail,
+				    settings->beacon.tail_len);
+
+#ifdef CONFIG_RTW_80211R_AP
+	rtw_ft_update_assocresp_ies(ndev, settings);
+#endif
+
+	if (settings->beacon.assocresp_ies &&
+		settings->beacon.assocresp_ies_len > 0) {
+			rtw_cfg80211_set_assocresp_ies(ndev,
+			settings->beacon.assocresp_ies,
+			settings->beacon.assocresp_ies_len);
+	}
+
+	if (settings->ssid && settings->ssid_len) {
+		WLAN_BSSID_EX *pbss_network = &adapter->mlmepriv.dev_cur_network.network;
+		WLAN_BSSID_EX *pbss_network_ext = &pmlmeinfo->dev_network;
+
+		if (0)
+			RTW_INFO(FUNC_ADPT_FMT" ssid:(%s,%zu), from ie:(%s,%d)\n", FUNC_ADPT_ARG(adapter),
+				settings->ssid, settings->ssid_len,
+				pbss_network->Ssid.Ssid, pbss_network->Ssid.SsidLength);
+
+		_rtw_memcpy(pbss_network->Ssid.Ssid, (void *)settings->ssid, settings->ssid_len);
+		pbss_network->Ssid.SsidLength = settings->ssid_len;
+		_rtw_memcpy(pbss_network_ext->Ssid.Ssid, (void *)settings->ssid, settings->ssid_len);
+		pbss_network_ext->Ssid.SsidLength = settings->ssid_len;
+
+		if (0)
+			RTW_INFO(FUNC_ADPT_FMT" after ssid:(%s,%d), (%s,%d)\n", FUNC_ADPT_ARG(adapter),
+				pbss_network->Ssid.Ssid, pbss_network->Ssid.SsidLength,
+				pbss_network_ext->Ssid.Ssid, pbss_network_ext->Ssid.SsidLength);
+	}
+	pmlmeinfo->assoc_AP_vendor = HT_IOT_PEER_REALTEK;
+
+exit:
+	return ret;
+}
+
+static int rtw_cfg80211_set_assocresp_ies(struct net_device *net, const u8 *buf, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	RTW_INFO("%s, ielen=%d\n", __func__, len);
+
+	if (len <= 0)
+		goto exit;
+
+	if (pmlmepriv->assoc_rsp) {
+		u32 free_len = pmlmepriv->assoc_rsp_len;
+
+		pmlmepriv->assoc_rsp_len = 0;
+		rtw_mfree(pmlmepriv->assoc_rsp, free_len);
+		pmlmepriv->assoc_rsp = NULL;
+	}
+
+	pmlmepriv->assoc_rsp = rtw_malloc(len);
+	if (pmlmepriv->assoc_rsp == NULL) {
+		RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+		return -EINVAL;
+	}
+	_rtw_memcpy(pmlmepriv->assoc_rsp, buf, len);
+	pmlmepriv->assoc_rsp_len = len;
+
+exit:
+	return ret;
+}
+
+
+static int rtw_cfg80211_check_beacon_ies(struct net_device *net, const u8 *head,
+					 int head_len, const u8 *tail,
+					 int tail_len)
+{
+	int ret = 2;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	if (pmlmepriv->beacon_head_ie &&
+	    (pmlmepriv->beacon_head_ie_len == head_len)) {
+		if (_rtw_memcmp(pmlmepriv->beacon_head_ie, head, head_len) ==
+		    _TRUE)
+			ret--;
+	}
+	if (pmlmepriv->beacon_tail_ie &&
+	    (pmlmepriv->beacon_tail_ie_len == tail_len)) {
+		if (_rtw_memcmp(pmlmepriv->beacon_tail_ie, tail, tail_len) ==
+		    _TRUE)
+			ret--;
+	}
+	return ret;
+}
+
+static int rtw_cfg80211_set_beacon_ies(struct net_device *net, const u8 *head,
+				       int head_len, const u8 *tail,
+				       int tail_len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	RTW_INFO("%s, len(head, tail)=(%d, %d)\n", __func__, head_len,
+		 tail_len);
+
+	if (pmlmepriv->beacon_head_ie) {
+		u32 free_len = pmlmepriv->beacon_head_ie_len;
+
+		pmlmepriv->beacon_head_ie_len = 0;
+		rtw_mfree(pmlmepriv->beacon_head_ie, free_len);
+		pmlmepriv->beacon_head_ie = NULL;
+	}
+
+	if (head_len) {
+		pmlmepriv->beacon_head_ie = rtw_malloc(head_len);
+		if (pmlmepriv->beacon_head_ie == NULL) {
+			RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__,
+				 __LINE__);
+			return -EINVAL;
+		}
+		_rtw_memcpy(pmlmepriv->beacon_head_ie, head, head_len);
+		pmlmepriv->beacon_head_ie_len = head_len;
+	}
+
+	if (pmlmepriv->beacon_tail_ie) {
+		u32 free_len = pmlmepriv->beacon_tail_ie_len;
+
+		pmlmepriv->beacon_tail_ie_len = 0;
+		rtw_mfree(pmlmepriv->beacon_tail_ie, free_len);
+		pmlmepriv->beacon_tail_ie = NULL;
+	}
+
+	if (tail_len) {
+		pmlmepriv->beacon_tail_ie = rtw_malloc(tail_len);
+		if (pmlmepriv->beacon_tail_ie == NULL) {
+			RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__,
+				 __LINE__);
+			return -EINVAL;
+		}
+		_rtw_memcpy(pmlmepriv->beacon_tail_ie, tail, tail_len);
+		pmlmepriv->beacon_tail_ie_len = tail_len;
+	}
+
+	return ret;
+}
+
+static int cfg80211_rtw_change_beacon(struct wiphy *wiphy, struct net_device *ndev,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 7, 0))
+				      struct cfg80211_ap_update *param
+#else
+				      struct cfg80211_beacon_data *param
+#endif
+)
+
+{
+	int ret = 0;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 7, 0))
+	struct cfg80211_beacon_data *info = &param->beacon;
+#else
+	struct cfg80211_beacon_data *info = param;
+#endif
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+#ifdef not_yet
+	/*
+	 * @proberesp_ies: extra information element(s) to add into Probe Response
+	 *	frames or %NULL
+	 * @proberesp_ies_len: length of proberesp_ies in octets
+	 */
+	if (info->proberesp_ies_len > 0)
+		rtw_cfg80211_set_proberesp_ies(ndev, info->proberesp_ies, info->proberesp_ies_len);
+#endif /* not_yet */
+
+	if (info->assocresp_ies_len > 0)
+		rtw_cfg80211_set_assocresp_ies(ndev, info->assocresp_ies, info->assocresp_ies_len);
+
+	if (rtw_cfg80211_check_beacon_ies(ndev, info->head, info->head_len,
+					  info->tail, info->tail_len) != 0) {
+		ret = rtw_add_beacon(adapter, info->head, info->head_len,
+				     info->tail, info->tail_len);
+		rtw_cfg80211_set_beacon_ies(ndev, info->head, info->head_len,
+					    info->tail, info->tail_len);
+	}
+	return ret;
+}
+
+static int cfg80211_rtw_stop_ap(struct wiphy *wiphy, struct net_device *ndev
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	, unsigned int link_id
+#endif
+)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	RTW_INFO(FUNC_NDEV_FMT" link_id:%d\n", FUNC_NDEV_ARG(ndev), link_id);
+#else
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+#endif
+	/* ToDo CONFIG_RTW_MLD */
+
+	rtw_ap_stop_set_state(adapter, AP_STOP_ST_START);
+	rtw_mi_buddy_set_scan_deny(adapter, 300);
+	rtw_mi_scan_abort(adapter, _TRUE);
+	rtw_sta_flush(adapter, _TRUE);
+	rtw_stop_ap_cmd(adapter, RTW_CMDF_WAIT_ACK);
+	return 0;
+}
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) */
+
+#if CONFIG_RTW_MACADDR_ACL && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+static int cfg80211_rtw_set_mac_acl(struct wiphy *wiphy, struct net_device *ndev,
+		const struct cfg80211_acl_data *params)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+	u8 acl_mode = RTW_ACL_MODE_DISABLED;
+	int ret = -1;
+	int i;
+
+	if (!params) {
+		RTW_WARN(FUNC_ADPT_FMT" params NULL\n", FUNC_ADPT_ARG(adapter));
+		rtw_macaddr_acl_clear(adapter, RTW_ACL_PERIOD_BSS);
+		goto exit;
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT" acl_policy:%d, entry_num:%d\n"
+		, FUNC_ADPT_ARG(adapter), params->acl_policy, params->n_acl_entries);
+
+	if (params->acl_policy == NL80211_ACL_POLICY_ACCEPT_UNLESS_LISTED)
+		acl_mode = RTW_ACL_MODE_ACCEPT_UNLESS_LISTED;
+	else if (params->acl_policy == NL80211_ACL_POLICY_DENY_UNLESS_LISTED)
+		acl_mode = RTW_ACL_MODE_DENY_UNLESS_LISTED;
+
+	rtw_macaddr_acl_clear(adapter, RTW_ACL_PERIOD_BSS);
+
+	rtw_set_macaddr_acl(adapter, RTW_ACL_PERIOD_BSS, acl_mode);
+
+	for (i = 0; i < params->n_acl_entries; i++)
+		rtw_acl_add_sta(adapter, RTW_ACL_PERIOD_BSS, params->mac_addrs[i].addr);
+
+	ret = 0;
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_RTW_MACADDR_ACL && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)) */
+
+const char *_nl80211_sta_flags_str[] = {
+	"INVALID",
+	"AUTHORIZED",
+	"SHORT_PREAMBLE",
+	"WME",
+	"MFP",
+	"AUTHENTICATED",
+	"TDLS_PEER",
+	"ASSOCIATED",
+};
+
+#define nl80211_sta_flags_str(_f) ((_f <= NL80211_STA_FLAG_MAX) ? _nl80211_sta_flags_str[_f] : _nl80211_sta_flags_str[0])
+
+const char *_nl80211_plink_state_str[] = {
+	"LISTEN",
+	"OPN_SNT",
+	"OPN_RCVD",
+	"CNF_RCVD",
+	"ESTAB",
+	"HOLDING",
+	"BLOCKED",
+	"UNKNOWN",
+};
+
+#define nl80211_plink_state_str(_s) ((_s < NUM_NL80211_PLINK_STATES) ? _nl80211_plink_state_str[_s] : _nl80211_plink_state_str[NUM_NL80211_PLINK_STATES])
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0))
+#define NL80211_PLINK_ACTION_NO_ACTION PLINK_ACTION_INVALID
+#define NL80211_PLINK_ACTION_OPEN PLINK_ACTION_OPEN
+#define NL80211_PLINK_ACTION_BLOCK PLINK_ACTION_BLOCK
+#define NUM_NL80211_PLINK_ACTIONS 3
+#endif
+
+const char *_nl80211_plink_actions_str[] = {
+	"NO_ACTION",
+	"OPEN",
+	"BLOCK",
+	"UNKNOWN",
+};
+
+#define nl80211_plink_actions_str(_a) ((_a < NUM_NL80211_PLINK_ACTIONS) ? _nl80211_plink_actions_str[_a] : _nl80211_plink_actions_str[NUM_NL80211_PLINK_ACTIONS])
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+const char *_nl80211_mesh_power_mode_str[] = {
+	"UNKNOWN",
+	"ACTIVE",
+	"LIGHT_SLEEP",
+	"DEEP_SLEEP",
+};
+
+#define nl80211_mesh_power_mode_str(_p) ((_p <= NL80211_MESH_POWER_MAX) ? _nl80211_mesh_power_mode_str[_p] : _nl80211_mesh_power_mode_str[0])
+#endif
+
+void dump_station_parameters(void *sel, struct wiphy *wiphy, const struct station_parameters *params)
+{
+#if DBG_RTW_CFG80211_STA_PARAM
+	if (params->supported_rates_len) {
+		#define SUPP_RATES_BUF_LEN (3 * RTW_G_RATES_NUM + 1)
+		int i;
+		char supp_rates_buf[SUPP_RATES_BUF_LEN] = {0};
+		u8 cnt = 0;
+
+		rtw_warn_on(params->supported_rates_len > RTW_G_RATES_NUM);
+
+		for (i = 0; i < params->supported_rates_len; i++) {
+			if (i >= RTW_G_RATES_NUM)
+				break;
+			cnt += snprintf(supp_rates_buf + cnt, SUPP_RATES_BUF_LEN - cnt -1
+				, "%02X ", params->supported_rates[i]);
+			if (cnt >= SUPP_RATES_BUF_LEN - 1)
+				break;
+		}
+
+		RTW_PRINT_SEL(sel, "supported_rates:%s\n", supp_rates_buf);
+	}
+
+	if (params->vlan)
+		RTW_PRINT_SEL(sel, "vlan:"NDEV_FMT"\n", NDEV_ARG(params->vlan));
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 31))
+	if (params->sta_flags_mask) {
+		#define STA_FLAGS_BUF_LEN 128
+		int i = 0;
+		char sta_flags_buf[STA_FLAGS_BUF_LEN] = {0};
+		u8 cnt = 0;
+
+		for (i = 1; i <= NL80211_STA_FLAG_MAX; i++) {
+			if (params->sta_flags_mask & BIT(i)) {
+				cnt += snprintf(sta_flags_buf + cnt, STA_FLAGS_BUF_LEN - cnt -1, "%s=%u "
+					, nl80211_sta_flags_str(i), (params->sta_flags_set & BIT(i)) ? 1 : 0);
+				if (cnt >= STA_FLAGS_BUF_LEN - 1)
+					break;
+			}
+		}
+
+		RTW_PRINT_SEL(sel, "sta_flags:%s\n", sta_flags_buf);
+	}
+#else
+	u32 station_flags;
+	#error "TBD\n"
+#endif
+
+	if (params->listen_interval != -1)
+		RTW_PRINT_SEL(sel, "listen_interval:%d\n", params->listen_interval);
+
+	if (params->aid)
+		RTW_PRINT_SEL(sel, "aid:%u\n", params->aid);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
+	if (params->peer_aid)
+		RTW_PRINT_SEL(sel, "peer_aid:%u\n", params->peer_aid);
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26))
+	if (params->plink_action != NL80211_PLINK_ACTION_NO_ACTION)
+		RTW_PRINT_SEL(sel, "plink_action:%s\n", nl80211_plink_actions_str(params->plink_action));
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+	if (params->sta_modify_mask & STATION_PARAM_APPLY_PLINK_STATE)
+	#endif
+		RTW_PRINT_SEL(sel, "plink_state:%s\n"
+			, nl80211_plink_state_str(params->plink_state));
+#endif
+
+#if 0 /* TODO */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28))
+	const struct ieee80211_ht_cap *ht_capa;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	const struct ieee80211_vht_cap *vht_capa;
+#endif
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	if (params->sta_modify_mask & STATION_PARAM_APPLY_UAPSD)
+		RTW_PRINT_SEL(sel, "uapsd_queues:0x%02x\n", params->uapsd_queues);
+	if (params->max_sp)
+		RTW_PRINT_SEL(sel, "max_sp:%u\n", params->max_sp);
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	if (params->local_pm != NL80211_MESH_POWER_UNKNOWN) {
+		RTW_PRINT_SEL(sel, "local_pm:%s\n"
+			, nl80211_mesh_power_mode_str(params->local_pm));
+	}
+
+	if (params->sta_modify_mask & STATION_PARAM_APPLY_CAPABILITY)
+		RTW_PRINT_SEL(sel, "capability:0x%04x\n", params->capability);
+
+#if 0 /* TODO */
+	const u8 *ext_capab;
+	u8 ext_capab_len;
+#endif
+#endif
+
+#if 0 /* TODO */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0))
+	const u8 *supported_channels;
+	u8 supported_channels_len;
+	const u8 *supported_oper_classes;
+	u8 supported_oper_classes_len;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0))
+	u8 opmode_notif;
+	bool opmode_notif_used;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0))
+	int support_p2p_ps;
+#endif
+#endif
+#endif /* DBG_RTW_CFG80211_STA_PARAM */
+}
+
+static int	cfg80211_rtw_add_station(struct wiphy *wiphy, struct net_device *ndev,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0))
+	u8 *mac,
+#else
+	const u8 *mac,
+#endif
+	struct station_parameters *params)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+#if defined(CONFIG_TDLS) || defined(CONFIG_RTW_MESH)
+	struct sta_priv *pstapriv = &padapter->stapriv;
+#endif
+#ifdef CONFIG_TDLS
+	struct rtw_wifi_role_t *wrole = padapter->phl_role;
+	struct sta_info *psta;
+	enum role_type rtype = PHL_RTYPE_TDLS;
+	enum rtw_phl_status pstatus = RTW_PHL_STATUS_SUCCESS;
+#endif /* CONFIG_TDLS */
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	u16 main_id = rtw_phl_get_macid_max_num(phl);
+	struct rtw_phl_mld_t *pmld = NULL;
+
+	RTW_INFO(FUNC_NDEV_FMT" mac:"MAC_FMT"\n", FUNC_NDEV_ARG(ndev), MAC_ARG(mac));
+
+#if CONFIG_RTW_MACADDR_ACL
+	if (rtw_access_ctrl(padapter, mac) == _FALSE) {
+		RTW_INFO(FUNC_NDEV_FMT" deny by macaddr ACL\n", FUNC_NDEV_ARG(ndev));
+		ret = -EINVAL;
+		goto exit;
+	}
+#endif
+
+	dump_station_parameters(RTW_DBGDUMP, wiphy, params);
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		struct rtw_mesh_cfg *mcfg = &padapter->mesh_cfg;
+		struct rtw_mesh_info *minfo = &padapter->mesh_info;
+		struct mesh_plink_pool *plink_ctl = &minfo->plink_ctl;
+		struct mesh_plink_ent *plink = NULL;
+		struct wlan_network *scanned = NULL;
+		bool acnode = 0;
+		u8 add_new_sta = 0, probe_req = 0;
+
+		if (params->plink_state != NL80211_PLINK_LISTEN) {
+			RTW_WARN(FUNC_NDEV_FMT" %s\n", FUNC_NDEV_ARG(ndev), nl80211_plink_state_str(params->plink_state));
+			rtw_warn_on(1);
+		}
+		if (!params->aid || params->aid > pstapriv->max_aid) {
+			RTW_WARN(FUNC_NDEV_FMT" invalid aid:%u\n", FUNC_NDEV_ARG(ndev), params->aid);
+			rtw_warn_on(1);
+			ret = -EINVAL;
+			goto exit;
+		}
+
+		_rtw_spinlock_bh(&(plink_ctl->lock));
+
+		plink = _rtw_mesh_plink_get(padapter, mac);
+		if (plink)
+			goto release_plink_ctl;
+
+		#if CONFIG_RTW_MESH_PEER_BLACKLIST
+		if (rtw_mesh_peer_blacklist_search(padapter, mac)) {
+			RTW_INFO(FUNC_NDEV_FMT" deny by peer blacklist\n"
+				, FUNC_NDEV_ARG(ndev));
+			ret = -EINVAL;
+			goto release_plink_ctl;
+		}
+		#endif
+
+		scanned = rtw_find_network(&padapter->mlmepriv.scanned_queue, mac);
+		if (!scanned
+			|| rtw_get_passing_time_ms(scanned->last_scanned) >= mcfg->peer_sel_policy.scanr_exp_ms
+		) {
+			if (!scanned)
+				RTW_INFO(FUNC_NDEV_FMT" corresponding network not found\n", FUNC_NDEV_ARG(ndev));
+			else
+				RTW_INFO(FUNC_NDEV_FMT" corresponding network too old\n", FUNC_NDEV_ARG(ndev));
+
+			if (adapter_to_rfctl(padapter)->offch_state == OFFCHS_NONE)
+				probe_req = 1;
+
+			ret = -EINVAL;
+			goto release_plink_ctl;
+		}
+
+		#if CONFIG_RTW_MESH_ACNODE_PREVENT
+		if (plink_ctl->acnode_rsvd)
+			acnode = rtw_mesh_scanned_is_acnode_confirmed(padapter, scanned);
+		#endif
+
+		/* wpa_supplicant's auto peer will initiate peering when candidate peer is reported without max_peer_links consideration */
+		if (plink_ctl->num >= mcfg->max_peer_links + acnode ? 1 : 0) {
+			RTW_INFO(FUNC_NDEV_FMT" exceed max_peer_links:%u%s\n"
+				, FUNC_NDEV_ARG(ndev), mcfg->max_peer_links, acnode ? " acn" : "");
+			ret = -EINVAL;
+			goto release_plink_ctl;
+		}
+
+		if (!rtw_bss_is_candidate_mesh_peer(padapter, &scanned->network, 1, 1)) {
+			RTW_WARN(FUNC_NDEV_FMT" corresponding network is not candidate with same ch\n"
+				, FUNC_NDEV_ARG(ndev));
+			ret = -EINVAL;
+			goto release_plink_ctl;
+		}
+
+		#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+		if (!rtw_mesh_cto_mgate_network_filter(padapter, scanned)) {
+			RTW_INFO(FUNC_NDEV_FMT" peer filtered out by cto_mgate check\n"
+				, FUNC_NDEV_ARG(ndev));
+			ret = -EINVAL;
+			goto release_plink_ctl;
+		}
+		#endif
+
+		if (_rtw_mesh_plink_add(padapter, mac) == _SUCCESS) {
+			/* hook corresponding network in scan queue */
+			plink = _rtw_mesh_plink_get(padapter, mac);
+			if (params->aid > 0)
+				plink->aid = params->aid;
+			plink->scanned = scanned;
+
+			#if CONFIG_RTW_MESH_ACNODE_PREVENT
+			if (acnode) {
+				RTW_INFO(FUNC_ADPT_FMT" acnode "MAC_FMT"\n"
+				, FUNC_ADPT_ARG(padapter), MAC_ARG(scanned->network.MacAddress));
+			}
+			#endif
+
+			add_new_sta = 1;
+		} else {
+			RTW_WARN(FUNC_NDEV_FMT" rtw_mesh_plink_add not success\n"
+				, FUNC_NDEV_ARG(ndev));
+			ret = -EINVAL;
+		}
+release_plink_ctl:
+		_rtw_spinunlock_bh(&(plink_ctl->lock));
+
+		if (probe_req)
+			issue_probereq(padapter, &padapter->mlmepriv.dev_cur_network.dev_network.mesh_id, mac);
+
+		if (add_new_sta) {
+			struct station_info sinfo;
+
+			#ifdef CONFIG_DFS_MASTER
+			if (IS_UNDER_CAC(adapter_to_rfctl(padapter)))
+				rtw_force_stop_cac(adapter_to_rfctl(padapter), 300);
+			#endif
+
+			/* indicate new sta */
+			_rtw_memset(&sinfo, 0, sizeof(sinfo));
+			cfg80211_new_sta(ndev, mac, &sinfo, GFP_ATOMIC);
+		}
+		goto exit;
+	}
+#endif /* CONFIG_RTW_MESH */
+
+#ifdef CONFIG_TDLS
+	if (params->sta_flags_set & BIT(NL80211_STA_FLAG_TDLS_PEER)) {
+		if (wrole->type != PHL_RTYPE_TDLS) {
+			pstatus = rtw_phl_cmd_wrole_change(phl, wrole, NULL, WR_CHG_TYPE,
+							   (u8*)&rtype, sizeof(enum role_type),
+							   PHL_CMD_DIRECTLY, 0);
+			if (pstatus != RTW_PHL_STATUS_SUCCESS) {
+				RTW_ERR("%s - change to phl role type = %d fail with error = %d\n",
+					__func__, rtype, pstatus);
+				rtw_warn_on(1);
+				ret = -EOPNOTSUPP;
+				goto exit;
+			}
+		}
+		psta = rtw_get_stainfo(pstapriv, (u8 *)mac);
+		if (psta == NULL) {
+			pmld = rtw_phl_alloc_mld(phl, padapter->phl_role, (u8 *)mac, DTYPE);
+			if (pmld == NULL) {
+				RTW_INFO("[%s] Alloc mld for "MAC_FMT" fail\n", __FUNCTION__, MAC_ARG(mac));
+				ret = -EOPNOTSUPP;
+				goto exit;
+			}
+			psta = rtw_alloc_stainfo(pstapriv, (u8 *)mac, DTYPE, main_id, padapter_link->wrlink->id, PHL_CMD_WAIT);
+			if (psta == NULL) {
+				RTW_INFO("[%s] Alloc station for "MAC_FMT" fail\n", __FUNCTION__, MAC_ARG(mac));
+				ret = -EOPNOTSUPP;
+				goto exit;
+			}
+			rtw_phl_link_mld_stainfo(pmld, psta->phl_sta);
+		}
+	}
+#endif /* CONFIG_TDLS */
+
+exit:
+	return ret;
+}
+
+static int	cfg80211_rtw_del_station(struct wiphy *wiphy, struct net_device *ndev,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0))
+	u8 *mac
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))
+	const u8 *mac
+#else
+	struct station_del_parameters *params
+#endif
+)
+{
+	int ret = 0;
+	_list	*phead, *plist;
+	u8 updated = _FALSE;
+	const u8 *target_mac;
+	struct sta_info *psta = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	u8 subtype = 0;
+	u16 reason_code = 0;
+	bool is_ucast = 1;
+	u8 ignore_del = _FALSE;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))
+	target_mac = mac;
+#else
+	target_mac = params->mac;
+	subtype = params->subtype;
+	reason_code = params->reason_code;
+#endif
+
+	if(target_mac){
+		RTW_INFO("+"FUNC_NDEV_FMT" mac=" MAC_FMT "\n", FUNC_NDEV_ARG(ndev), MAC_ARG(target_mac));
+	}else{
+		RTW_INFO("+"FUNC_NDEV_FMT" mac=%pM\n", FUNC_NDEV_ARG(ndev), target_mac);
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+	if (subtype)
+		RTW_INFO(FUNC_NDEV_FMT" subtype=0x%x\n", FUNC_NDEV_ARG(ndev), subtype);
+	if (reason_code)
+		RTW_INFO(FUNC_NDEV_FMT" reason_code=%u\n", FUNC_NDEV_ARG(ndev), reason_code);
+#endif
+
+	if (check_fwstate(pmlmepriv, (WIFI_ASOC_STATE | WIFI_AP_STATE | WIFI_MESH_STATE)) != _TRUE) {
+		RTW_INFO("%s, fw_state != FW_LINKED|WIFI_AP_STATE|WIFI_MESH_STATE\n", __func__);
+		return -EINVAL;
+	}
+
+
+	if (!target_mac) {
+		RTW_INFO("flush all sta, and cam_entry\n");
+
+		flush_all_cam_entry(padapter, PHL_CMD_WAIT, 50);	/* clear CAM */
+
+#ifdef CONFIG_AP_MODE
+		ret = rtw_sta_flush(padapter, _TRUE);
+#endif
+		return ret;
+	}
+
+
+	RTW_INFO("free sta macaddr =" MAC_FMT "\n", MAC_ARG(target_mac));
+
+	/* broadcast deauth */
+	if (is_broadcast_mac_addr(target_mac)) {
+		if (MLME_IS_AP(padapter) && MLME_IS_ASOC(padapter)) {
+			is_ucast = 0;
+			issue_deauth(padapter, (u8 *)target_mac, WLAN_REASON_PREV_AUTH_NOT_VALID);
+		}
+		else {
+			return -EINVAL;
+		}
+	}
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+
+	/* check asoc_queue */
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+
+		plist = get_next(plist);
+
+		if ((_rtw_memcmp((u8 *)target_mac, psta->phl_sta->mac_addr, ETH_ALEN) || is_ucast == 0)
+			&& !_rtw_memcmp((u8 *)target_mac, padapter->mac_addr, ETH_ALEN)
+			&& !is_broadcast_mac_addr(psta->phl_sta->mac_addr)) {
+
+			if (ATOMIC_READ(&psta->deleting) == 0) {
+				ATOMIC_INC(&psta->deleting);
+				RTW_INFO("psta->deleting=%u, block auth\n", ATOMIC_READ(&psta->deleting));
+			} else {
+				ignore_del = _TRUE;
+				continue;
+			}
+
+			if (psta->dot8021xalg == 1 && psta->bpairwise_key_installed == _FALSE) {
+				RTW_INFO("%s, sta's dot8021xalg = 1 and key_installed = _FALSE\n", __func__);
+
+				if (MLME_IS_AP(padapter)) {
+					rtw_stapriv_asoc_list_del(pstapriv, psta);
+					reason_code = reason_code == 0 ? WLAN_REASON_IEEE_802_1X_AUTH_FAILED : reason_code;
+					RTW_INFO("%s, asoc_list_cnt=%d\n", __func__, pstapriv->asoc_list_cnt);
+
+					updated |= ap_free_sta(padapter, psta, is_ucast, subtype, reason_code, _TRUE);
+					psta = NULL;
+
+					if (is_ucast)
+						break;
+				}
+			} else {
+				RTW_INFO("free psta=%p, aid=%d\n", psta, psta->phl_sta->aid);
+				rtw_stapriv_asoc_list_del(pstapriv, psta);
+				RTW_INFO("%s, asoc_list_cnt=%d\n", __func__, pstapriv->asoc_list_cnt);
+
+				/* rtw_stapriv_asoc_list_unlock(pstapriv); */
+				if (MLME_IS_AP(padapter)) {
+					reason_code = reason_code == 0 ? WLAN_REASON_PREV_AUTH_NOT_VALID : reason_code;
+					updated |= ap_free_sta(padapter, psta, is_ucast, subtype, reason_code, _TRUE);
+				} else {
+					reason_code = reason_code == 0 ? WLAN_REASON_DEAUTH_LEAVING : reason_code;
+					updated |= ap_free_sta(padapter, psta, is_ucast, subtype, reason_code, _TRUE);
+				}
+				/* rtw_stapriv_asoc_list_lock(pstapriv); */
+
+				psta = NULL;
+
+				if (is_ucast)
+					break;
+			}
+		}
+	}
+
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+	if (ignore_del == _FALSE)
+		associated_clients_update(padapter, updated, STA_INFO_UPDATE_ALL);
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter))
+		rtw_mesh_plink_del(padapter, target_mac);
+#endif
+
+	RTW_INFO("-"FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	return ret;
+
+}
+
+static int cfg80211_rtw_change_station(struct wiphy *wiphy,
+	struct net_device *ndev,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0))
+	u8 *mac,
+#else
+	const u8 *mac,
+#endif
+	struct station_parameters *params)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+	int ret = 0;
+
+	RTW_INFO(FUNC_ADPT_FMT" mac:"MAC_FMT"\n", FUNC_ADPT_ARG(adapter), MAC_ARG(mac));
+
+	dump_station_parameters(RTW_DBGDUMP, wiphy, params);
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		enum cfg80211_station_type sta_type = CFG80211_STA_MESH_PEER_USER;
+		u8 plink_state = nl80211_plink_state_to_rtw_plink_state(params->plink_state);
+
+		ret = cfg80211_check_station_change(wiphy, params, sta_type);
+		if (ret) {
+			RTW_INFO("cfg80211_check_station_change return %d\n", ret);
+			goto exit;
+		}
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+		if (!(params->sta_modify_mask & STATION_PARAM_APPLY_PLINK_STATE))
+			goto exit;
+		#endif
+
+		if (rtw_mesh_set_plink_state_cmd(adapter, mac, plink_state) != _SUCCESS)
+			ret = -ENOENT;
+	}
+
+exit:
+#endif /* CONFIG_RTW_MESH */
+
+	if (ret)
+		RTW_INFO(FUNC_ADPT_FMT" mac:"MAC_FMT" ret:%d\n",
+			FUNC_ADPT_ARG(adapter), MAC_ARG(mac), ret);
+	return ret;
+}
+
+struct sta_info *rtw_sta_info_get_by_idx(struct sta_priv *pstapriv, const int idx, u8 *asoc_list_num)
+{
+	_list	*phead, *plist;
+	struct sta_info *psta = NULL;
+	int i = 0;
+
+	phead = &pstapriv->asoc_list;
+	plist = get_next(phead);
+
+	/* check asoc_queue */
+	while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+		if (idx == i)
+			psta = LIST_CONTAINOR(plist, struct sta_info, asoc_list);
+		plist = get_next(plist);
+		i++;
+	}
+
+	if (asoc_list_num)
+		*asoc_list_num = i;
+
+	return psta;
+}
+
+static int	cfg80211_rtw_dump_station(struct wiphy *wiphy, struct net_device *ndev,
+		int idx, u8 *mac, struct station_info *sinfo)
+{
+#define DBG_DUMP_STATION 0
+
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info *psta = NULL;
+#ifdef CONFIG_RTW_MESH
+	struct mesh_plink_ent *plink = NULL;
+#endif
+	u8 asoc_list_num;
+
+	if (DBG_DUMP_STATION)
+		RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	rtw_stapriv_asoc_list_lock(pstapriv);
+	psta = rtw_sta_info_get_by_idx(pstapriv, idx, &asoc_list_num);
+	rtw_stapriv_asoc_list_unlock(pstapriv);
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		if (psta)
+			plink = psta->plink;
+		if (!plink)
+			plink = rtw_mesh_plink_get_no_estab_by_idx(padapter, idx - asoc_list_num);
+	}
+#endif /* CONFIG_RTW_MESH */
+
+	if ((!MLME_IS_MESH(padapter) && !psta)
+		#ifdef CONFIG_RTW_MESH
+		|| (MLME_IS_MESH(padapter) && !plink)
+		#endif
+	) {
+		if (DBG_DUMP_STATION)
+			RTW_INFO(FUNC_NDEV_FMT" end with idx:%d\n", FUNC_NDEV_ARG(ndev), idx);
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	if (psta)
+		_rtw_memcpy(mac, psta->phl_sta->mac_addr, ETH_ALEN);
+	#ifdef CONFIG_RTW_MESH
+	else
+		_rtw_memcpy(mac, plink->addr, ETH_ALEN);
+	#endif
+	
+	sinfo->filled = 0;
+
+	if (psta) {
+		sinfo->filled |= STATION_INFO_SIGNAL;
+		/* ToDo: need API to query hal_sta->rssi_stat.rssi */
+		/* sinfo->signal = rtw_phl_rssi_to_dbm(psta->phl_sta->rssi_stat.rssi); */
+		sinfo->filled |= STATION_INFO_INACTIVE_TIME;
+		sinfo->inactive_time = rtw_get_passing_time_ms(psta->sta_stats.last_rx_time);
+	}
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter))
+		rtw_cfg80211_fill_mesh_only_sta_info(plink, psta, sinfo);
+#endif
+
+exit:
+	return ret;
+}
+
+static int	cfg80211_rtw_change_bss(struct wiphy *wiphy, struct net_device *ndev,
+		struct bss_parameters *params)
+{
+	_adapter *adapter = rtw_netdev_priv(ndev);
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	if (params->ap_isolate != -1) {
+		RTW_INFO("ap_isolate=%d\n", params->ap_isolate);
+		adapter->mlmepriv.ap_isolate = params->ap_isolate ? 1 : 0;
+	}
+#endif
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+static int	cfg80211_rtw_set_txq_params(struct wiphy *wiphy
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	, struct net_device *ndev
+#endif
+	, struct ieee80211_txq_params *params)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	_adapter *padapter = rtw_netdev_priv(ndev);
+#else
+	_adapter *padapter = wiphy_to_adapter(wiphy);
+#endif
+	u8	ac, AIFS, ECWMin, ECWMax, aSifsTime, vo_cw;
+	u16	TXOP;
+	u8	shift_count = 0;
+	u32	acParm;
+	struct registry_priv *pregpriv = &padapter->registrypriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	ac = params->ac;
+#else
+	ac = params->queue;
+#endif
+
+	switch (ac) {
+	case NL80211_TXQ_Q_VO:
+		ac = XMIT_VO_QUEUE;
+		vo_cw = (pregpriv->vo_edca >> 8) & 0xff;
+		if(vo_cw) {
+			/* change vo contention window */
+			params->cwmin = vo_cw & 0xf;
+			params->cwmax = vo_cw >> 4;
+		}
+		break;
+
+	case NL80211_TXQ_Q_VI:
+		ac = XMIT_VI_QUEUE;
+		break;
+
+	case NL80211_TXQ_Q_BE:
+		ac = XMIT_BE_QUEUE;
+		break;
+
+	case NL80211_TXQ_Q_BK:
+		ac = XMIT_BK_QUEUE;
+		break;
+
+	default:
+		break;
+	}
+
+#if 0
+	RTW_INFO("ac=%d\n", ac);
+	RTW_INFO("txop=%u\n", params->txop);
+	RTW_INFO("cwmin=%u\n", params->cwmin);
+	RTW_INFO("cwmax=%u\n", params->cwmax);
+	RTW_INFO("aifs=%u\n", params->aifs);
+#endif
+
+	if (WIFI_ROLE_LINK_IS_ON_5G(padapter_link) ||
+	    (pmlmeext->cur_wireless_mode & WLAN_MD_11N))
+		aSifsTime = 16;
+	else
+		aSifsTime = 10;
+
+	AIFS = params->aifs * pmlmeinfo->slotTime + aSifsTime;
+
+	while ((params->cwmin + 1) >> shift_count != 1) {
+		shift_count++;
+		if (shift_count == 15)
+			break;
+	}
+
+	ECWMin = shift_count;
+
+	shift_count = 0;
+	while ((params->cwmax + 1) >> shift_count != 1) {
+		shift_count++;
+		if (shift_count == 15)
+			break;
+	}
+
+	ECWMax = shift_count;
+
+	TXOP = params->txop;
+
+	acParm = AIFS | (ECWMin << 8) | (ECWMax << 12) | (TXOP << 16);
+
+	if (ac == XMIT_BE_QUEUE) {
+		padapter->last_edca = acParm;
+		acParm = rtw_get_turbo_edca(padapter, AIFS, ECWMin, ECWMax, TXOP);
+		if (acParm)
+			padapter->last_edca = acParm;
+		else
+			acParm = padapter->last_edca;
+	}
+
+	set_txq_params_cmd(padapter, padapter_link, acParm, ac);
+
+	return 0;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) */
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))
+static int	cfg80211_rtw_set_channel(struct wiphy *wiphy
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	, struct net_device *ndev
+	#endif
+	, struct ieee80211_channel *chan, enum nl80211_channel_type channel_type)
+{
+	_adapter *padapter;
+	struct _ADAPTER_LINK *padapter_link;
+	struct rtw_chan_def chdef = {0};
+	enum band_type chan_band = nl80211_band_to_rtw_band(chan->band);
+	int chan_target = rtw_freq2ch(chan->center_freq);
+	int chan_offset = CHAN_OFFSET_NO_EXT;
+	int chan_width = CHANNEL_WIDTH_20;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	if (ndev) {
+		RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+		padapter = (_adapter *)rtw_netdev_priv(ndev);
+	} else
+#endif
+		padapter = wiphy_to_adapter(wiphy);
+
+	padapter_link = GET_PRIMARY_LINK(padapter);
+
+	switch (channel_type) {
+	case NL80211_CHAN_NO_HT:
+	case NL80211_CHAN_HT20:
+		chan_width = CHANNEL_WIDTH_20;
+		chan_offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case NL80211_CHAN_HT40MINUS:
+		chan_width = CHANNEL_WIDTH_40;
+		chan_offset = CHAN_OFFSET_LOWER;
+		break;
+	case NL80211_CHAN_HT40PLUS:
+		chan_width = CHANNEL_WIDTH_40;
+		chan_offset = CHAN_OFFSET_UPPER;
+		break;
+	default:
+		chan_width = CHANNEL_WIDTH_20;
+		chan_offset = CHAN_OFFSET_NO_EXT;
+		break;
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT" band:%d, ch:%d, bw:%d, offset:%d\n"
+		, FUNC_ADPT_ARG(padapter), chan_band, chan_target, chan_width, chan_offset);
+
+	chdef.band = chan_band;
+	chdef.chan = chan_target;
+	chdef.bw = chan_width;
+	chdef.offset = chan_offset;
+
+	rtw_set_chbw_cmd(padapter, padapter_link, &chdef, RTW_CMDF_WAIT_ACK, RFK_TYPE_FORCE_NOT_DO);
+
+	return 0;
+}
+#endif /*#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))*/
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+static void rtw_get_chbwoff_from_cfg80211_chan_def(
+	struct cfg80211_chan_def *chandef,
+	u8 *ht, struct rtw_chan_def *rtw_chdef)
+{
+	struct ieee80211_channel *chan = chandef->chan;
+
+	rtw_chdef->chan = chan->hw_value;
+	rtw_chdef->band = nl80211_band_to_rtw_band(chan->band);
+	*ht = 1;
+
+	switch (chandef->width) {
+	case NL80211_CHAN_WIDTH_20_NOHT:
+		*ht = 0;
+		fallthrough;
+	case NL80211_CHAN_WIDTH_20:
+		rtw_chdef->bw = CHANNEL_WIDTH_20;
+		rtw_chdef->offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case NL80211_CHAN_WIDTH_40:
+		rtw_chdef->bw = CHANNEL_WIDTH_40;
+		rtw_chdef->offset = (chandef->center_freq1 > chan->center_freq)
+				? CHAN_OFFSET_UPPER
+				: CHAN_OFFSET_LOWER;
+		break;
+	case NL80211_CHAN_WIDTH_80:
+		rtw_chdef->bw = CHANNEL_WIDTH_80;
+		rtw_chdef->offset = (chandef->center_freq1 > chan->center_freq)
+				? CHAN_OFFSET_UPPER
+				: CHAN_OFFSET_LOWER;
+		break;
+	case NL80211_CHAN_WIDTH_160:
+		rtw_chdef->bw = CHANNEL_WIDTH_160;
+		rtw_chdef->offset = (chandef->center_freq1 > chan->center_freq)
+				? CHAN_OFFSET_UPPER
+				: CHAN_OFFSET_LOWER;
+		break;
+	case NL80211_CHAN_WIDTH_80P80:
+		rtw_chdef->bw = CHANNEL_WIDTH_80_80;
+		rtw_chdef->offset = (chandef->center_freq1 > chan->center_freq)
+				? CHAN_OFFSET_UPPER
+				: CHAN_OFFSET_LOWER;
+		break;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	case NL80211_CHAN_WIDTH_5:
+		rtw_chdef->bw = CHANNEL_WIDTH_5;
+		rtw_chdef->offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case NL80211_CHAN_WIDTH_10:
+		rtw_chdef->bw = CHANNEL_WIDTH_10;
+		rtw_chdef->offset = CHAN_OFFSET_NO_EXT;
+		break;
+#endif
+	default:
+		*ht = 0;
+		rtw_chdef->bw = CHANNEL_WIDTH_20;
+		rtw_chdef->offset = CHAN_OFFSET_NO_EXT;
+		RTW_INFO("unsupported cwidth:%u\n", chandef->width);
+		rtw_warn_on(1);
+	};
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) */
+
+static int cfg80211_rtw_set_monitor_channel(struct wiphy *wiphy
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0))
+	, struct net_device *dev
+#endif
+	, struct cfg80211_chan_def *chandef
+#else
+	, struct ieee80211_channel *chan
+	, enum nl80211_channel_type channel_type
+#endif
+	)
+{
+	_adapter *padapter = wiphy_to_adapter(wiphy);
+	struct rtw_chan_def target_chdef = {0};
+	u8 ht_option;
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("center_freq %u Mhz band %u ch %u width %u freq1 %u freq2 %u\n"
+		, chandef->chan->center_freq
+		, chandef->chan->band
+		, chandef->chan->hw_value
+		, chandef->width
+		, chandef->center_freq1
+		, chandef->center_freq2);
+#endif /* CONFIG_DEBUG_CFG80211 */
+
+	rtw_get_chbwoff_from_cfg80211_chan_def(chandef,
+		&ht_option, &target_chdef);
+#else
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("center_freq %u Mhz ch %u channel_type %u\n"
+		, chan->center_freq
+		, chan->hw_value
+		, channel_type);
+#endif /* CONFIG_DEBUG_CFG80211 */
+
+	rtw_get_chdef_from_nl80211_channel_type(chan, channel_type,
+		&ht_option, &target_chdef);
+#endif
+	RTW_INFO(FUNC_ADPT_FMT" band:%d, ch:%d, bw:%d, offset:%d\n",
+		FUNC_ADPT_ARG(padapter), target_chdef.band, target_chdef.chan,
+		target_chdef.bw, target_chdef.offset);
+
+	rtw_set_chbw_cmd(padapter, padapter_link,
+		&target_chdef, RTW_CMDF_WAIT_ACK, RFK_TYPE_FORCE_NOT_DO);
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+static int cfg80211_rtw_get_channel(struct wiphy *wiphy,
+	struct wireless_dev *wdev,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	unsigned int link_id,
+#endif
+	struct cfg80211_chan_def *chandef)
+{
+	_adapter *a = (_adapter *)rtw_netdev_priv(wdev_to_ndev(wdev));
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *a_link = GET_PRIMARY_LINK(a);
+	struct link_mlme_ext_priv *mlmeext = &(a_link->mlmeextpriv);
+	u8 ht_option = 0;
+	int ret = _FAIL;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	RTW_INFO(FUNC_ADPT_FMT" link_id:%d\n", FUNC_ADPT_ARG(a), link_id);
+#endif
+
+#ifdef CONFIG_80211N_HT
+	ht_option = a_link->mlmepriv.htpriv.ht_option;
+#endif /* CONFIG_80211N_HT */
+
+	ret = rtw_chdef_to_cfg80211_chan_def(wiphy, chandef,
+			&mlmeext->chandef, ht_option);
+
+	if (ret == _FAIL)
+		return -ENODATA;
+	else
+		return 0;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) */
+
+/*
+static int	cfg80211_rtw_auth(struct wiphy *wiphy, struct net_device *ndev,
+		struct cfg80211_auth_request *req)
+{
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	return 0;
+}
+
+static int	cfg80211_rtw_assoc(struct wiphy *wiphy, struct net_device *ndev,
+		struct cfg80211_assoc_request *req)
+{
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+	return 0;
+}
+*/
+#endif /* CONFIG_AP_MODE */
+
+void rtw_cfg80211_external_auth_request(_adapter *padapter, union recv_frame *rframe)
+{
+#if (KERNEL_VERSION(4, 17, 0) <= LINUX_VERSION_CODE) \
+    || defined(CONFIG_KERNEL_PATCH_EXTERNAL_AUTH)
+	struct cfg80211_external_auth_params params = {0};
+#endif
+	struct wireless_dev *wdev = padapter->rtw_wdev;
+	struct net_device *netdev = wdev_to_ndev(wdev);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+
+	u8 frame[256] = { 0 };
+	uint frame_len = 24;
+	s32 freq = 0;
+
+	/* rframe, in this case is null point */
+	freq = rtw_bch2freq(pmlmeext->chandef.band,
+				pmlmeext->chandef.chan);
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO(FUNC_ADPT_FMT": freq=%d channel=%d\n", FUNC_ADPT_ARG(padapter), freq, padapter_link->mlmeextpriv.chandef.chan);
+#endif
+
+#if (KERNEL_VERSION(4, 17, 0) <= LINUX_VERSION_CODE) \
+    || defined(CONFIG_KERNEL_PATCH_EXTERNAL_AUTH)
+	params.action = NL80211_EXTERNAL_AUTH_START;
+	_rtw_memcpy(params.bssid, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+	params.ssid.ssid_len = pmlmeinfo->network.Ssid.SsidLength;
+	_rtw_memcpy(params.ssid.ssid, pmlmeinfo->network.Ssid.Ssid,
+		pmlmeinfo->network.Ssid.SsidLength);
+	params.key_mgmt_suite = 0x8ac0f00;
+
+	RTW_INFO("external auth: use kernel API: cfg80211_external_auth_request()\n");
+	cfg80211_external_auth_request(netdev, &params, GFP_ATOMIC);
+#elif (KERNEL_VERSION(2, 6, 37) <= LINUX_VERSION_CODE)
+	set_frame_sub_type(frame, WIFI_AUTH);
+
+	_rtw_memcpy(frame + 4, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+	_rtw_memcpy(frame + 10, padapter_link->mac_addr, ETH_ALEN);
+	_rtw_memcpy(frame + 16, get_my_bssid(&pmlmeinfo->network), ETH_ALEN);
+	RTW_PUT_LE32((frame + 18), 0x8ac0f00);
+	RTW_PUT_LE32((frame + 24), 0x0003);
+
+	if (pmlmeinfo->network.Ssid.SsidLength) {
+		*(frame + 26) = pmlmeinfo->network.Ssid.SsidLength;
+		_rtw_memcpy(frame + 27, pmlmeinfo->network.Ssid.Ssid,
+			pmlmeinfo->network.Ssid.SsidLength);
+		frame_len = 27 + pmlmeinfo->network.Ssid.SsidLength;
+	}
+
+	RTW_INFO("external auth: with wpa_supplicant patch\n");
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, frame_len, GFP_ATOMIC);
+#endif
+}
+
+void rtw_cfg80211_rx_probe_request(_adapter *adapter, union recv_frame *rframe)
+{
+	struct wireless_dev *wdev = NULL;
+	struct wiphy *wiphy = NULL;
+	u8 *frame = get_recvframe_data(rframe);
+	uint frame_len = rframe->u.hdr.len;
+	s32 freq;
+	struct _ADAPTER_LINK *adapter_link = rframe->u.hdr.adapter_link;
+	enum band_type band = rtw_get_oper_band(adapter, adapter_link);
+	u8 ch, sch = rtw_get_oper_ch(adapter, adapter_link);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	wiphy = adapter->rtw_wdev->wiphy;
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+	wdev = wiphy_to_pd_wdev(wiphy);
+#endif
+#endif
+
+	if (wdev == NULL)
+		wdev = adapter->rtw_wdev;
+
+	ch = rframe->u.hdr.attrib.ch ? rframe->u.hdr.attrib.ch : sch;
+	freq = rtw_bch2freq(band, ch);
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("RTW_Rx: probe request, ch=%d(%d), ta="MAC_FMT"\n"
+		, ch, sch, MAC_ARG(get_addr2_ptr(frame)));
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,37)) || defined(COMPAT_KERNEL_RELEASE)
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, frame_len, GFP_ATOMIC);
+#else
+	cfg80211_rx_action(adapter->pnetdev, freq, frame, frame_len, GFP_ATOMIC);
+#endif
+}
+
+void rtw_cfg80211_rx_action_p2p(_adapter *adapter, union recv_frame *rframe)
+{
+	struct wireless_dev *wdev = adapter->rtw_wdev;
+	u8 *frame = get_recvframe_data(rframe);
+	uint frame_len = rframe->u.hdr.len;
+	s32 freq;
+	struct _ADAPTER_LINK *adapter_link = rframe->u.hdr.adapter_link;
+	enum band_type band = rtw_get_oper_band(adapter, adapter_link);
+	u8 ch, sch = rtw_get_oper_ch(adapter, adapter_link);
+	u8 category, action;
+	int type;
+
+	ch = rframe->u.hdr.attrib.ch ? rframe->u.hdr.attrib.ch : sch;
+	freq = rtw_bch2freq(band, ch);
+
+	RTW_INFO("RTW_Rx:band=%d, ch=%d(%d), ta="MAC_FMT"\n"
+		, band, ch, sch, MAC_ARG(get_addr2_ptr(frame)));
+#ifdef CONFIG_P2P
+	type = rtw_p2p_check_frames(adapter, frame, frame_len, _FALSE);
+	if (type >= 0)
+		goto indicate;
+#endif
+	rtw_action_frame_parse(frame, frame_len, &category, &action);
+	RTW_INFO("RTW_Rx:category(%u), action(%u)\n", category, action);
+#ifdef CONFIG_P2P
+indicate:
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, frame_len, GFP_ATOMIC);
+#else
+	cfg80211_rx_action(adapter->pnetdev, freq, frame, frame_len, GFP_ATOMIC);
+#endif
+}
+
+void rtw_cfg80211_rx_p2p_action_public(_adapter *adapter, union recv_frame *rframe)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct wireless_dev *wdev = adapter->rtw_wdev;
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(adapter);
+	u8 *frame = get_recvframe_data(rframe);
+	uint frame_len = rframe->u.hdr.len;
+	s32 freq;
+	struct _ADAPTER_LINK *adapter_link = rframe->u.hdr.adapter_link;
+	enum band_type band = rtw_get_oper_band(adapter, adapter_link);
+	u8 ch, sch = rtw_get_oper_ch(adapter, adapter_link);
+	u8 category, action;
+	int type;
+
+	if (MLME_IS_GO(adapter))
+		ch = sch;
+	else
+		ch = rframe->u.hdr.attrib.ch ? rframe->u.hdr.attrib.ch : sch;
+
+	freq = rtw_bch2freq(band, ch);
+
+	RTW_INFO("RTW_Rx:band=%d, ch=%d(%d), ta="MAC_FMT"\n"
+		, band, ch, sch, MAC_ARG(get_addr2_ptr(frame)));
+	#ifdef CONFIG_P2P
+	type = rtw_p2p_check_frames(adapter, frame, frame_len, _FALSE);
+	if (type >= 0) {
+		switch (type) {
+		case P2P_GO_NEGO_CONF:
+			if (0) {
+				RTW_INFO(FUNC_ADPT_FMT" Nego confirm. state=%u, status=%u, iaddr="MAC_FMT"\n"
+					, FUNC_ADPT_ARG(adapter), pwdev_priv->nego_info.state, pwdev_priv->nego_info.status
+					, MAC_ARG(pwdev_priv->nego_info.iface_addr));
+			}
+			if (pwdev_priv->nego_info.state == 2
+				&& pwdev_priv->nego_info.status == 0
+				&& rtw_check_invalid_mac_address(pwdev_priv->nego_info.iface_addr, _FALSE) == _FALSE
+			) {
+				_adapter *intended_iface = dvobj_get_adapter_by_addr(dvobj, pwdev_priv->nego_info.iface_addr);
+
+				if (intended_iface) {
+					RTW_INFO(FUNC_ADPT_FMT" Nego confirm. Allow only "ADPT_FMT" to scan for 2000 ms\n"
+						, FUNC_ADPT_ARG(adapter), ADPT_ARG(intended_iface));
+					/* allow only intended_iface to do scan for 2000 ms */
+					rtw_mi_set_scan_deny(adapter, 2000);
+					rtw_clear_scan_deny(intended_iface);
+				}
+			}
+			break;
+		case P2P_PROVISION_DISC_RESP:
+		case P2P_INVIT_RESP:
+			rtw_clear_scan_deny(adapter);
+			#if !RTW_P2P_GROUP_INTERFACE
+			rtw_mi_buddy_set_scan_deny(adapter, 2000);
+			#endif
+			break;
+		}
+		goto indicate;
+	}
+	#endif
+	rtw_action_frame_parse(frame, frame_len, &category, &action);
+	RTW_INFO("RTW_Rx:category(%u), action(%u)\n", category, action);
+#ifdef CONFIG_P2P
+indicate:
+#endif
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (rtw_cfg80211_redirect_pd_wdev(dvobj_to_wiphy(dvobj), get_ra(frame), &wdev))
+		if (0)
+			RTW_INFO("redirect to pd_wdev:%p\n", wdev);
+	#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, frame_len, GFP_ATOMIC);
+#else
+	cfg80211_rx_action(adapter->pnetdev, freq, frame, frame_len, GFP_ATOMIC);
+#endif
+}
+
+void rtw_cfg80211_rx_action(_adapter *adapter, union recv_frame *rframe, const char *msg)
+{
+	struct wireless_dev *wdev = adapter->rtw_wdev;
+	u8 *frame = get_recvframe_data(rframe);
+	uint frame_len = rframe->u.hdr.len;
+	s32 freq;
+	struct _ADAPTER_LINK *adapter_link = rframe->u.hdr.adapter_link;
+	enum band_type band = rtw_get_oper_band(adapter, adapter_link);
+	u8 ch, sch = rtw_get_oper_ch(adapter, adapter_link);
+	u8 category, action;
+	int type = -1;
+
+	if (MLME_IS_GO(adapter))
+		ch = sch;
+	else
+		ch = rframe->u.hdr.attrib.ch ? rframe->u.hdr.attrib.ch : sch;
+
+	freq = rtw_bch2freq(band, ch);
+
+	RTW_INFO("RTW_Rx:band=%d, ch=%d(%d), ta="MAC_FMT"\n"
+		, band, ch, sch, MAC_ARG(get_addr2_ptr(frame)));
+
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(adapter)) {
+		type = rtw_mesh_check_frames_rx(adapter, frame, frame_len);
+		if (type >= 0)
+			goto indicate;
+	}
+#endif
+	rtw_action_frame_parse(frame, frame_len, &category, &action);
+	if (category == RTW_WLAN_CATEGORY_PUBLIC) {
+		if (action == ACT_PUBLIC_GAS_INITIAL_REQ) {
+			rtw_mi_set_scan_deny(adapter, 200);
+			rtw_mi_scan_abort(adapter, _FALSE); /*rtw_scan_abort_no_wait*/
+		}
+	}
+#ifdef CONFIG_RTW_MESH
+indicate:
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, frame_len, GFP_ATOMIC);
+#else
+	cfg80211_rx_action(adapter->pnetdev, freq, frame, frame_len, GFP_ATOMIC);
+#endif
+
+	if (type == -1) {
+		if (msg)
+			RTW_INFO("RTW_Rx:%s\n", msg);
+		else
+			RTW_INFO("RTW_Rx:category(%u), action(%u)\n", category, action);
+	}
+}
+
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+void rtw_cfg80211_rx_rrm_action(_adapter *adapter, union recv_frame *rframe)
+{
+	struct wireless_dev *wdev = adapter->rtw_wdev;
+	u8 *frame = get_recvframe_data(rframe);
+	uint frame_len = rframe->u.hdr.len;
+	s32 freq;
+	struct _ADAPTER_LINK *adapter_link = rframe->u.hdr.adapter_link;
+	u8 ch, sch = rtw_get_oper_ch(adapter, adapter_link);
+	struct link_mlme_ext_priv *pmlmeext = &(adapter_link->mlmeextpriv);
+	u8 band = pmlmeext->chandef.band;
+
+	ch = rframe->u.hdr.attrib.ch ? rframe->u.hdr.attrib.ch : sch;
+	freq = rtw_bch2freq(band, ch);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, frame_len, GFP_ATOMIC);
+#else
+	cfg80211_rx_action(adapter->pnetdev, freq, frame, frame_len, GFP_ATOMIC);
+#endif
+	RTW_INFO("RTW_Rx:ch=%d(%d), ta="MAC_FMT"\n"
+		, ch, sch, MAC_ARG(get_addr2_ptr(frame)));
+}
+#endif /* CONFIG_RTW_80211K */
+
+int rtw_cfg80211_rx_mframe(_adapter *adapter, union recv_frame *rframe, const char *msg)
+{
+	struct wireless_dev *wdev = adapter->rtw_wdev;
+	u8 *frame = get_recvframe_data(rframe);
+	uint frame_len = rframe->u.hdr.len;
+	s32 freq;
+	struct _ADAPTER_LINK *adapter_link = rframe->u.hdr.adapter_link;
+	u8 ch, sch = rtw_get_oper_ch(adapter, adapter_link);
+	struct link_mlme_ext_priv *pmlmeext = &(adapter_link->mlmeextpriv);
+	u8 band = pmlmeext->chandef.band;
+
+	ch = rframe->u.hdr.attrib.ch ? rframe->u.hdr.attrib.ch : sch;
+	freq = rtw_bch2freq(band, ch);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	rtw_cfg80211_rx_mgmt(wdev, freq, 0, frame, frame_len, GFP_ATOMIC);
+#else
+	cfg80211_rx_action(adapter->pnetdev, freq, frame, frame_len, GFP_ATOMIC);
+#endif
+
+	RTW_INFO("RTW_Rx:ch=%d(%d), ta="MAC_FMT"\n", ch, sch, MAC_ARG(get_addr2_ptr(frame)));
+	if (!rtw_sae_preprocess(adapter, frame, frame_len, _FALSE)) {
+		if (msg)
+			RTW_INFO("RTW_Rx:%s\n", msg);
+		else
+			RTW_INFO("RTW_Rx:frame_control:0x%02x\n", le16_to_cpu(((struct rtw_ieee80211_hdr_3addr *)rframe)->frame_ctl));
+		return _FAIL;
+	}
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_RTW_80211R
+static s32 cfg80211_rtw_update_ft_ies(struct wiphy *wiphy,
+	struct net_device *ndev,
+	struct cfg80211_update_ft_ies_params *ftie)
+{
+	_adapter *padapter = NULL;
+	struct mlme_priv *pmlmepriv = NULL;
+	struct ft_roam_info *pft_roam = NULL;
+
+	if (ndev == NULL)
+		return  -EINVAL;
+
+	padapter = (_adapter *)rtw_netdev_priv(ndev);
+	pmlmepriv = &(padapter->mlmepriv);
+	pft_roam = &(pmlmepriv->ft_roam);
+
+#ifdef CONFIG_RTW_80211R_AP
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == _TRUE)
+		return rtw_ft_update_sta_ies(padapter, ftie);
+#endif
+
+	if (ftie->ie_len <= sizeof(pft_roam->updated_ft_ies)) {
+		_rtw_spinlock_bh(&pmlmepriv->lock);
+		_rtw_memcpy(pft_roam->updated_ft_ies, ftie->ie, ftie->ie_len);
+		pft_roam->updated_ft_ies_len = ftie->ie_len;
+		_rtw_spinunlock_bh(&pmlmepriv->lock);
+	} else {
+		RTW_ERR("FTIEs parsing fail!\n");
+		return -EINVAL;
+	}
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	if (rtw_ft_roam_status(padapter, RTW_FT_AUTHENTICATED_STA)) {
+		RTW_PRINT("auth success, start reassoc\n");
+		rtw_ft_set_status(padapter, RTW_FT_ASSOCIATING_STA);
+		start_clnt_assoc(padapter);
+	}
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+	return 0;
+}
+#endif
+
+#ifdef CONFIG_P2P
+inline int rtw_cfg80211_iface_has_p2p_group_cap(_adapter *adapter)
+{
+#if RTW_P2P_GROUP_INTERFACE
+	if (is_primary_adapter(adapter))
+		return 0;
+#endif
+	return 1;
+}
+
+inline int rtw_cfg80211_is_p2p_scan(_adapter *adapter)
+{
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wiphy_to_pd_wdev(adapter_to_wiphy(adapter))) /* pd_wdev exist */
+		return rtw_cfg80211_is_scan_by_pd_wdev(adapter);
+	#endif
+	{
+		/* Here are 2 cases:
+		 * 1. RTW_DEDICATED_P2P_DEVICE defined but upper layer don't use pd_wdev or
+		 * 2. RTW_DEDICATED_P2P_DEVICE not defined
+		 * Both cases check whether the contents of scan_request is p2p scan.
+		 */
+		struct rtw_wdev_priv *wdev_data = adapter_wdev_data(adapter);
+		int is_p2p_scan = 0;
+
+		_rtw_spinlock_bh(&wdev_data->scan_req_lock);
+		if (wdev_data->scan_request
+			&& wdev_data->scan_request->n_ssids
+			&& wdev_data->scan_request->ssids
+			&& wdev_data->scan_request->ie
+		) {
+			if (_rtw_memcmp(wdev_data->scan_request->ssids[0].ssid, "DIRECT-", 7)
+				&& rtw_get_p2p_ie((u8 *)wdev_data->scan_request->ie, wdev_data->scan_request->ie_len, NULL, NULL))
+				is_p2p_scan = 1;
+		}
+		_rtw_spinunlock_bh(&wdev_data->scan_req_lock);
+
+		return is_p2p_scan;
+	}
+}
+
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+int rtw_pd_iface_alloc(struct wiphy *wiphy, const char *name, struct wireless_dev **pd_wdev)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	struct wireless_dev *wdev = NULL;
+	struct rtw_netdev_priv_indicator *npi;
+	_adapter *primary_adpt = wiphy_to_adapter(wiphy);
+	int ret = 0;
+
+	if (wiphy_data->pd_wdev) {
+		RTW_WARN(FUNC_WIPHY_FMT" pd_wdev already exists\n", FUNC_WIPHY_ARG(wiphy));
+		ret = -EBUSY;
+		goto exit;
+	}
+
+	wdev = (struct wireless_dev *)rtw_zmalloc(sizeof(struct wireless_dev));
+	if (!wdev) {
+		RTW_WARN(FUNC_WIPHY_FMT" allocate wdev fail\n", FUNC_WIPHY_ARG(wiphy));
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	wdev->wiphy = wiphy;
+	wdev->iftype = NL80211_IFTYPE_P2P_DEVICE;
+	_rtw_memcpy(wdev->address, adapter_mac_addr(primary_adpt), ETH_ALEN);
+
+	wiphy_data->pd_wdev = wdev;
+	*pd_wdev = wdev;
+
+	RTW_INFO(FUNC_WIPHY_FMT" pd_wdev:%p, addr="MAC_FMT" added\n"
+		, FUNC_WIPHY_ARG(wiphy), wdev, MAC_ARG(wdev_address(wdev)));
+
+exit:
+	if (ret && wdev) {
+		rtw_mfree((u8 *)wdev, sizeof(struct wireless_dev));
+		wdev = NULL;
+	}
+
+	return ret;
+}
+
+void rtw_pd_iface_free(struct wiphy *wiphy)
+{
+	struct dvobj_priv *dvobj = wiphy_to_dvobj(wiphy);
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	u8 rtnl_lock_needed;
+
+	if (!wiphy_data->pd_wdev)
+		goto exit;
+
+	RTW_INFO(FUNC_WIPHY_FMT" pd_wdev:%p, addr="MAC_FMT"\n"
+		, FUNC_WIPHY_ARG(wiphy), wiphy_data->pd_wdev
+		, MAC_ARG(wdev_address(wiphy_data->pd_wdev)));
+
+	rtnl_lock_needed = rtw_rtnl_lock_needed(dvobj);
+	if (rtnl_lock_needed)
+		rtnl_lock();
+	cfg80211_unregister_wdev(wiphy_data->pd_wdev);
+	if (rtnl_lock_needed)
+		rtnl_unlock();
+
+	rtw_mfree((u8 *)wiphy_data->pd_wdev, sizeof(struct wireless_dev));
+	wiphy_data->pd_wdev = NULL;
+
+exit:
+	return;
+}
+
+static int cfg80211_rtw_start_p2p_device(struct wiphy *wiphy, struct wireless_dev *wdev)
+{
+	_adapter *adapter = wiphy_to_adapter(wiphy);
+
+	RTW_INFO(FUNC_WIPHY_FMT" wdev=%p\n", FUNC_WIPHY_ARG(wiphy), wdev);
+
+	rtw_p2p_enable(adapter, P2P_ROLE_DEVICE);
+	return 0;
+}
+
+static void cfg80211_rtw_stop_p2p_device(struct wiphy *wiphy, struct wireless_dev *wdev)
+{
+	_adapter *adapter = wiphy_to_adapter(wiphy);
+
+	RTW_INFO(FUNC_WIPHY_FMT" wdev=%p\n", FUNC_WIPHY_ARG(wiphy), wdev);
+
+	if (rtw_cfg80211_is_p2p_scan(adapter))
+		rtw_scan_abort(adapter, 0);
+
+	rtw_p2p_enable(adapter, P2P_ROLE_DISABLE);
+}
+
+inline int rtw_cfg80211_redirect_pd_wdev(struct wiphy *wiphy, u8 *ra, struct wireless_dev **wdev)
+{
+	struct wireless_dev *pd_wdev = wiphy_to_pd_wdev(wiphy);
+
+	if (pd_wdev && pd_wdev != *wdev
+		&& _rtw_memcmp(wdev_address(pd_wdev), ra, ETH_ALEN) == _TRUE
+	) {
+		*wdev = pd_wdev;
+		return 1;
+	}
+	return 0;
+}
+
+inline int rtw_cfg80211_is_scan_by_pd_wdev(_adapter *adapter)
+{
+	struct wiphy *wiphy = adapter_to_wiphy(adapter);
+	struct rtw_wdev_priv *wdev_data = adapter_wdev_data(adapter);
+	struct wireless_dev *wdev = NULL;
+
+	_rtw_spinlock_bh(&wdev_data->scan_req_lock);
+	if (wdev_data->scan_request)
+		wdev = wdev_data->scan_request->wdev;
+	_rtw_spinunlock_bh(&wdev_data->scan_req_lock);
+
+	if (wdev && wdev == wiphy_to_pd_wdev(wiphy))
+		return 1;
+
+	return 0;
+}
+#endif /* RTW_DEDICATED_P2P_DEVICE */
+#endif /* CONFIG_P2P */
+
+inline void rtw_cfg80211_set_is_roch(_adapter *adapter, bool val)
+{
+	adapter->cfg80211_rochinfo.is_ro_ch = val;
+	rtw_mi_update_iface_status(&(adapter->mlmepriv), 0);
+}
+
+inline bool rtw_cfg80211_get_is_roch(_adapter *adapter)
+{
+	return adapter->cfg80211_rochinfo.is_ro_ch;
+}
+
+inline bool rtw_cfg80211_is_ro_ch_once(_adapter *adapter)
+{
+	return adapter->cfg80211_rochinfo.last_ro_ch_time ? 1 : 0;
+}
+
+inline void rtw_cfg80211_set_last_ro_ch_time(_adapter *adapter)
+{
+	adapter->cfg80211_rochinfo.last_ro_ch_time = rtw_get_current_time();
+
+	if (!adapter->cfg80211_rochinfo.last_ro_ch_time)
+		adapter->cfg80211_rochinfo.last_ro_ch_time++;
+}
+
+inline s32 rtw_cfg80211_get_last_ro_ch_passing_ms(_adapter *adapter)
+{
+	return rtw_get_passing_time_ms(adapter->cfg80211_rochinfo.last_ro_ch_time);
+}
+
+#ifndef CONFIG_CFG80211_REPORT_PROBE_REQ
+static inline bool chk_is_p2p_device(_adapter *adapter, struct wireless_dev *wdev)
+{
+	struct wiphy *wiphy = adapter_to_wiphy(adapter);
+
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev == wiphy_to_pd_wdev(wiphy))
+		return _TRUE;
+#else
+	#if defined(CONFIG_CONCURRENT_MODE) && !RTW_P2P_GROUP_INTERFACE
+	if (adapter->iface_id == adapter->registrypriv.sel_p2p_iface)
+	#endif
+		return _TRUE;
+#endif
+	return _FALSE;
+}
+#endif
+
+#if 1 /*CONFIG_PHL_ARCH*/
+static s32 cfg80211_rtw_remain_on_channel(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct wireless_dev *wdev,
+#else
+	struct net_device *ndev,
+#endif
+	struct ieee80211_channel *channel,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+	enum nl80211_channel_type channel_type,
+#endif
+	unsigned int duration, u64 *cookie)
+{
+	s32 err = 0;
+	u8 remain_ch = (u8) rtw_freq2ch(channel->center_freq);
+	enum band_type ro_band = nl80211_band_to_rtw_band(channel->band);
+	_adapter *padapter = NULL;
+	struct registry_priv  *pregistrypriv = NULL;
+	//struct rtw_wdev_priv *pwdev_priv;
+	struct cfg80211_roch_info *pcfg80211_rochinfo;
+	struct back_op_param bkop_parm;
+	u8 lidx;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	u8 channel_type = 0;
+#endif
+#ifdef CONFIG_P2P
+	u8 is_p2p;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev == wiphy_to_pd_wdev(wiphy))
+		padapter = wiphy_to_adapter(wiphy);
+	else
+#endif
+	if (wdev_to_ndev(wdev))
+		padapter = (_adapter *)rtw_netdev_priv(wdev_to_ndev(wdev));
+	else
+		return -EINVAL;
+#else
+	struct wireless_dev *wdev;
+
+	if (ndev == NULL)
+		return -EINVAL;
+
+	padapter = (_adapter *)rtw_netdev_priv(ndev);
+	wdev = ndev_to_wdev(ndev);
+#endif
+	pregistrypriv = &padapter->registrypriv;
+	pcfg80211_rochinfo = &padapter->cfg80211_rochinfo;
+
+#ifndef CONFIG_CFG80211_REPORT_PROBE_REQ
+#ifdef CONFIG_P2P
+	is_p2p = chk_is_p2p_device(padapter, wdev);
+	if (is_p2p) {
+		struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+
+		if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE))
+			rtw_p2p_enable(padapter, P2P_ROLE_DEVICE);
+
+		pwdinfo->listen_channel = remain_ch;
+		RTW_INFO(FUNC_ADPT_FMT" init listen_channel %u\n",
+			FUNC_ADPT_ARG(padapter),
+			padapter->wdinfo.listen_channel);
+	}
+#endif
+#endif
+	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) != _TRUE) {
+		for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+			padapter_link = GET_LINK(padapter, lidx);
+			rtw_update_roch_chan_def(padapter_link,
+						 remain_ch,
+						 CHANNEL_WIDTH_20,
+						 CHAN_OFFSET_NO_EXT,
+						 ro_band);
+			rtw_hw_update_chan_def(padapter, padapter_link);
+		}
+	}
+
+	*cookie = ATOMIC_INC_RETURN(&pcfg80211_rochinfo->ro_ch_cookie_gen);
+
+	RTW_INFO(FUNC_ADPT_FMT"%s ch:%u duration:%d, cookie:0x%llx\n"
+		, FUNC_ADPT_ARG(padapter), wdev == wiphy_to_pd_wdev(wiphy) ? " PD" : ""
+		, remain_ch, duration, *cookie);
+
+	if (rtw_chset_search_bch(adapter_to_chset(padapter), ro_band, remain_ch) < 0) {
+		RTW_WARN(FUNC_ADPT_FMT" invalid ch:%u\n", FUNC_ADPT_ARG(padapter), remain_ch);
+		return -EFAULT;
+	}
+
+#ifdef CONFIG_MP_INCLUDED
+	if (rtw_mp_mode_check(padapter)) {
+		RTW_INFO("MP mode block remain_on_channel request\n");
+		return -EFAULT;
+	}
+#endif
+	if (adapter_to_pwrctl(padapter)->bInSuspend == _TRUE) {
+		RTW_INFO("%s return -EBUSY since bInSuspend is TRUE\n", __func__);
+		return -EBUSY;
+	}
+
+	rtw_scan_abort(padapter, 0);
+
+	bkop_parm.off_ch_dur = pregistrypriv->roch_max_away_dur;
+	bkop_parm.on_ch_dur = pregistrypriv->roch_min_home_dur;
+
+	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE))
+		bkop_parm.off_ch_ext_dur = pregistrypriv->roch_extend_dur;
+	else
+		bkop_parm.off_ch_ext_dur = pregistrypriv->roch_extend_dur * 6;
+
+	if(0)
+		RTW_INFO("%s bkop_parm.off_ch_ext_dur=%u\n", __func__, bkop_parm.off_ch_ext_dur);
+
+	rtw_phl_remain_on_ch_cmd(padapter, *cookie, wdev,
+		channel, channel_type, duration, &bkop_parm);
+
+	return 0;
+}
+
+#else /* !CONFIG_PHL_ARCH */
+
+static s32 cfg80211_rtw_remain_on_channel(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct wireless_dev *wdev,
+#else
+	struct net_device *ndev,
+#endif
+	struct ieee80211_channel *channel,
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+	enum nl80211_channel_type channel_type,
+#endif
+	unsigned int duration, u64 *cookie)
+{
+	s32 err = 0;
+	u8 remain_ch = (u8) rtw_freq2ch(channel->center_freq);
+	_adapter *padapter = NULL;
+	struct rtw_wdev_priv *pwdev_priv;
+	struct wifidirect_info *pwdinfo;
+	struct cfg80211_wifidirect_info *pcfg80211_wdinfo;
+#ifdef CONFIG_CONCURRENT_MODE
+	u8 is_p2p_find = _FALSE;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev == wiphy_to_pd_wdev(wiphy))
+		padapter = wiphy_to_adapter(wiphy);
+	else
+	#endif
+	if (wdev_to_ndev(wdev))
+		padapter = (_adapter *)rtw_netdev_priv(wdev_to_ndev(wdev));
+	else {
+		err = -EINVAL;
+		goto exit;
+	}
+#else
+	struct wireless_dev *wdev;
+
+	if (ndev == NULL) {
+		err = -EINVAL;
+		goto exit;
+	}
+	padapter = (_adapter *)rtw_netdev_priv(ndev);
+	wdev = ndev_to_wdev(ndev);
+#endif
+
+	pwdev_priv = adapter_wdev_data(padapter);
+	pwdinfo = &padapter->wdinfo;
+	pcfg80211_wdinfo = &padapter->cfg80211_wdinfo;
+#ifdef CONFIG_CONCURRENT_MODE
+	is_p2p_find = (duration < (pwdinfo->ext_listen_interval)) ? _TRUE : _FALSE;
+#endif
+
+	*cookie = ATOMIC_INC_RETURN(&pcfg80211_wdinfo->ro_ch_cookie_gen);
+
+	RTW_INFO(FUNC_ADPT_FMT"%s ch:%u duration:%d, cookie:0x%llx\n"
+		, FUNC_ADPT_ARG(padapter), wdev == wiphy_to_pd_wdev(wiphy) ? " PD" : ""
+		, remain_ch, duration, *cookie);
+
+	if (rtw_chset_search_ch(adapter_to_chset(padapter), remain_ch) < 0) {
+		RTW_WARN(FUNC_ADPT_FMT" invalid ch:%u\n", FUNC_ADPT_ARG(padapter), remain_ch);
+		err = -EFAULT;
+		goto exit;
+	}
+
+#ifdef CONFIG_MP_INCLUDED
+	if (rtw_mp_mode_check(padapter)) {
+		RTW_INFO("MP mode block remain_on_channel request\n");
+		err = -EFAULT;
+		goto exit;
+	}
+#endif
+
+	rtw_scan_abort(padapter);
+#ifdef CONFIG_CONCURRENT_MODE
+	/*don't scan_abort during p2p_listen.*/
+	if (is_p2p_find)
+		rtw_mi_buddy_scan_abort(padapter, _TRUE);
+#endif /*CONFIG_CONCURRENT_MODE*/
+
+	if (rtw_cfg80211_get_is_roch(padapter) == _TRUE) {
+		_cancel_timer_ex(&padapter->cfg80211_wdinfo.remain_on_ch_timer);
+		p2p_cancel_roch_cmd(padapter, 0, NULL, RTW_CMDF_WAIT_ACK);
+	}
+
+	/* if(!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_CLIENT) && !rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO)) */
+	if (rtw_p2p_chk_state(pwdinfo, P2P_STATE_NONE)
+		#if defined(CONFIG_CONCURRENT_MODE) && defined(CONFIG_P2P)
+		&& ((padapter->iface_id == padapter->registrypriv.sel_p2p_iface))
+		#endif
+	) {
+		rtw_p2p_enable(padapter, P2P_ROLE_DEVICE);
+		padapter->wdinfo.listen_channel = remain_ch;
+		RTW_INFO(FUNC_ADPT_FMT" init listen_channel %u\n"
+			, FUNC_ADPT_ARG(padapter), padapter->wdinfo.listen_channel);
+	} else if (rtw_p2p_chk_state(pwdinfo , P2P_STATE_LISTEN)
+		&& (time_after_eq(rtw_get_current_time(), pwdev_priv->probe_resp_ie_update_time)
+			&& rtw_get_passing_time_ms(pwdev_priv->probe_resp_ie_update_time) < 50)
+	) {
+		if (padapter->wdinfo.listen_channel != remain_ch) {
+			padapter->wdinfo.listen_channel = remain_ch;
+			RTW_INFO(FUNC_ADPT_FMT" update listen_channel %u\n"
+				, FUNC_ADPT_ARG(padapter), padapter->wdinfo.listen_channel);
+		}
+	} else {
+		rtw_p2p_set_pre_state(pwdinfo, rtw_p2p_state(pwdinfo));
+#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("%s, role=%d, p2p_state=%d\n", __func__, rtw_p2p_role(pwdinfo), rtw_p2p_state(pwdinfo));
+#endif
+	}
+
+	rtw_p2p_set_state(pwdinfo, P2P_STATE_LISTEN);
+
+	#ifdef RTW_ROCH_DURATION_ENLARGE
+	if (duration < 400)
+		duration = duration * 3; /* extend from exper */
+	#endif
+
+#if defined(RTW_ROCH_BACK_OP) && defined(CONFIG_CONCURRENT_MODE)
+	if (rtw_mi_check_status(padapter, MI_LINKED)) {
+		if (is_p2p_find) /* p2p_find , duration<1000 */
+			duration = duration + pwdinfo->ext_listen_interval;
+		else /* p2p_listen, duration=5000 */
+			duration = pwdinfo->ext_listen_interval + (pwdinfo->ext_listen_interval / 4);
+	}
+#endif /*defined (RTW_ROCH_BACK_OP) && defined(CONFIG_CONCURRENT_MODE) */
+
+	rtw_cfg80211_set_is_roch(padapter, _TRUE);
+	pcfg80211_wdinfo->ro_ch_wdev = wdev;
+	pcfg80211_wdinfo->remain_on_ch_cookie = *cookie;
+	pcfg80211_wdinfo->duration = duration;
+	rtw_cfg80211_set_last_ro_ch_time(padapter);
+	_rtw_memcpy(&pcfg80211_wdinfo->remain_on_ch_channel, channel, sizeof(struct ieee80211_channel));
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+	pcfg80211_wdinfo->remain_on_ch_type = channel_type;
+	#endif
+	pcfg80211_wdinfo->restore_channel = rtw_get_oper_ch(padapter);
+
+	p2p_roch_cmd(padapter, *cookie, wdev, channel, pcfg80211_wdinfo->remain_on_ch_type,
+		duration, RTW_CMDF_WAIT_ACK);
+
+	rtw_cfg80211_ready_on_channel(wdev, *cookie, channel, channel_type, duration, GFP_KERNEL);
+exit:
+	return err;
+}
+#endif /* CONFIG_PHL_ARCH */
+
+static s32 cfg80211_rtw_cancel_remain_on_channel(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct wireless_dev *wdev,
+#else
+	struct net_device *ndev,
+#endif
+	u64 cookie)
+{
+	s32 err = 0;
+	_adapter *padapter;
+	struct rtw_wdev_priv *pwdev_priv;
+	struct cfg80211_roch_info *pcfg80211_rochinfo;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev == wiphy_to_pd_wdev(wiphy))
+		padapter = wiphy_to_adapter(wiphy);
+	else
+	#endif
+	if (wdev_to_ndev(wdev))
+		padapter = (_adapter *)rtw_netdev_priv(wdev_to_ndev(wdev));
+	else {
+		err = -EINVAL;
+		goto exit;
+	}
+#else
+	struct wireless_dev *wdev;
+
+	if (ndev == NULL) {
+		err = -EINVAL;
+		goto exit;
+	}
+	padapter = (_adapter *)rtw_netdev_priv(ndev);
+	wdev = ndev_to_wdev(ndev);
+#endif
+
+	pwdev_priv = adapter_wdev_data(padapter);
+	pcfg80211_rochinfo = &padapter->cfg80211_rochinfo;
+
+	RTW_INFO(FUNC_ADPT_FMT"%s cookie:0x%llx\n"
+		, FUNC_ADPT_ARG(padapter), wdev == wiphy_to_pd_wdev(wiphy) ? " PD" : ""
+		, cookie);
+
+	if (rtw_cfg80211_get_is_roch(padapter) == _TRUE) {
+		rtw_scan_abort(padapter, 0);
+	}
+
+exit:
+	return err;
+}
+
+inline void rtw_cfg80211_set_is_mgmt_tx(_adapter *adapter, u8 val)
+{
+	struct rtw_wdev_priv *wdev_priv = adapter_wdev_data(adapter);
+
+	wdev_priv->is_mgmt_tx = val;
+	rtw_mi_update_iface_status(&(adapter->mlmepriv), 0);
+}
+
+inline u8 rtw_cfg80211_get_is_mgmt_tx(_adapter *adapter)
+{
+	struct rtw_wdev_priv *wdev_priv = adapter_wdev_data(adapter);
+
+	return wdev_priv->is_mgmt_tx;
+}
+
+static int _cfg80211_rtw_mgmt_tx(_adapter *padapter, u8 tx_band, u8 tx_ch, u8 no_cck, const u8 *buf, size_t len, int wait_ack)
+{
+	struct xmit_frame	*pmgntframe;
+	struct pkt_attrib	*pattrib;
+	unsigned char	*pframe;
+	int ret = _FAIL;
+	bool ack = _TRUE;
+	struct rtw_ieee80211_hdr *pwlanhdr;
+#if defined(RTW_ROCH_BACK_OP) && defined(CONFIG_P2P) && defined(CONFIG_CONCURRENT_MODE)
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+#endif
+	struct xmit_priv	*pxmitpriv = &(padapter->xmitpriv);
+	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_chan_def u_chdef = {0};
+	u8 leave_op = 0;
+#ifdef CONFIG_P2P
+	struct cfg80211_roch_info *pcfg80211_rochinfo = &padapter->cfg80211_rochinfo;
+	#ifdef CONFIG_CONCURRENT_MODE
+	struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+	#endif
+#endif
+#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+	struct wireless_dev *wdev = padapter->rtw_wdev;
+	u8 *p = NULL;
+	uint ie_len = 0;
+#endif
+	u8 frame_styp;
+	u8 hdr_len = sizeof(struct rtw_ieee80211_hdr_3addr);
+	u8 *ies;
+	sint ies_len;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter); 
+
+	rtw_cfg80211_set_is_mgmt_tx(padapter, 1);
+
+#if 0
+/* phl_scan
+ * Enable ext period when WIFI_ASOC_STATE
+ * cancel roch when ext_listen_period is up
+ * remove it, because phl_sacn will do extend
+ */
+#ifdef CONFIG_P2P
+	if (rtw_cfg80211_get_is_roch(padapter) == _TRUE) {
+		#ifdef CONFIG_CONCURRENT_MODE
+		if (!check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) {
+			RTW_INFO("%s, extend ro ch time\n", __func__);
+			_set_timer(&padapter->cfg80211_wdinfo.remain_on_ch_timer, pwdinfo->ext_listen_period);
+		}
+		#endif /* CONFIG_CONCURRENT_MODE */
+	}
+#endif /* CONFIG_P2P */
+#endif
+	if ((rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role,
+				padapter_link->wrlink, &u_chdef) != RTW_PHL_STATUS_SUCCESS)
+				|| (u_chdef.chan == 0)) {
+		RTW_ERR("%s get union chandef failed\n", __func__);
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (rtw_mi_check_status(padapter, MI_LINKED)
+		&& tx_ch != u_chdef.chan
+	) {
+
+		/* phl_scan will self leave opch */
+		if (!rtw_cfg80211_get_is_roch(padapter)) {
+			rtw_leave_opch(padapter);
+			leave_op = 1;
+		}
+	}
+
+	/* phl_scan will self remain on ch */
+	if (!rtw_cfg80211_get_is_roch(padapter)) {
+		if (tx_ch != rtw_get_oper_ch(padapter, padapter_link))
+			set_bch_bwmode(padapter, padapter_link, tx_band,
+				tx_ch, CHAN_OFFSET_NO_EXT, CHANNEL_WIDTH_20, RFK_TYPE_FORCE_NOT_DO);
+	}
+
+	/* starting alloc mgmt frame to dump it */
+	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
+	if (pmgntframe == NULL) {
+		/* ret = -ENOMEM; */
+		ret = _FAIL;
+		goto exit;
+	}
+
+	/* update attribute */
+	pattrib = &pmgntframe->attrib;
+	update_mgntframe_attrib(padapter, padapter_link, pattrib);
+
+	if (no_cck && IS_CCK_RATE(pattrib->rate)) {
+		/* force OFDM 6M rate*/
+		pattrib->rate = MGN_6M;
+	}
+
+	pattrib->retry_ctrl = _FALSE;
+
+	_rtw_memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
+
+	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
+
+	_rtw_memcpy(pframe, (void *)buf, len);
+	pattrib->pktlen = len;
+
+	pwlanhdr = (struct rtw_ieee80211_hdr *)pframe;
+	/* update seq number */
+	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
+	pattrib->seqnum = pmlmeext->mgnt_seq;
+	pmlmeext->mgnt_seq++;
+
+	frame_styp = get_frame_sub_type(pframe);
+	if (frame_styp == WIFI_PROBERSP) {
+		/* Update VHT capability IE, HE capability IE and HE operation IE */
+		ies = pmgntframe->buf_addr + TXDESC_OFFSET + hdr_len + _BEACON_IE_OFFSET_;
+		ies_len = pattrib->pktlen - hdr_len - _BEACON_IE_OFFSET_;
+		#ifdef CONFIG_80211AC_VHT
+		rtw_update_probe_rsp_vht_cap(padapter, ies, ies_len);
+		#endif
+		#ifdef CONFIG_80211AX_HE
+		rtw_update_probe_rsp_he_cap_and_op(padapter, ies, &ies_len, tx_band);
+		pattrib->pktlen = ies_len + hdr_len + _BEACON_IE_OFFSET_;
+		#endif
+
+		#ifdef CONFIG_P2P_PS
+		/* Add NoA IE in probe response */
+		rtw_append_probe_resp_p2p_go_noa(pmgntframe);
+		#endif
+
+		#ifdef CONFIG_ECSA_PHL
+		/* Update channel switch count of CSA/ECSA IE in probe response */
+		rtw_ecsa_update_probe_resp(pmgntframe);
+		#endif
+
+		#ifdef CONFIG_P2P
+		rtw_append_probe_resp_vendor_ie(pmgntframe);
+		rtw_append_probe_resp_p2p_ie(pmgntframe);
+		#endif
+
+		#ifdef CONFIG_GO_APPEND_COUNTRY_IE
+		if (wdev->iftype == NL80211_IFTYPE_P2P_GO) {
+			/* check if country IE is included */
+			p = rtw_get_ie(ies, WLAN_EID_COUNTRY, &ie_len, ies_len);
+
+			if (!p)
+				pframe += rtw_set_ie_country(padapter, ies - _BEACON_IE_OFFSET_, &(pattrib->pktlen));
+		}
+		#endif
+
+		#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+		//update ies address and ies_len
+                ies_len = pattrib->pktlen - hdr_len - _BEACON_IE_OFFSET_;
+		ies = pmgntframe->buf_addr + TXDESC_OFFSET + hdr_len + _BEACON_IE_OFFSET_ + ies_len;
+		pattrib->pktlen += rtw_build_vendor_ie(padapter , &ies , WIFI_PROBERESP_VENDOR_IE_BIT);
+		#endif
+
+                /* hostapd would not add/remove CCK rate when switching bands in our case,
+                   but there's no need to consider the CCK rate in P2P GO. */
+		if (MLME_IS_AP(padapter) && !MLME_IS_GO(padapter))
+			rtw_update_probe_rsp_basic_rate_and_ext(pmgntframe);
+	}
+
+#ifdef CONFIG_P2P
+	rtw_xframe_chk_wfd_ie(pmgntframe);
+#endif /* CONFIG_P2P */
+
+	pattrib->last_txcmdsz = pattrib->pktlen;
+
+	if (wait_ack) {
+		if (dump_mgntframe_and_wait_ack(padapter, pmgntframe) != _SUCCESS) {
+			ack = _FALSE;
+			ret = _FAIL;
+
+#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("%s, ack == _FAIL\n", __func__);
+#endif
+		} else {
+
+#ifdef CONFIG_XMIT_ACK
+			if (!MLME_IS_MESH(padapter)) /* TODO: remove this sleep for all mode */
+				rtw_msleep_os(50);
+#endif
+#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("%s, ack=%d, ok!\n", __func__, ack);
+#endif
+			ret = _SUCCESS;
+		}
+	} else {
+		dump_mgntframe(padapter, pmgntframe);
+		ret = _SUCCESS;
+	}
+
+exit:
+
+/* phl_scan will self return opch */
+#if 0
+	#ifdef CONFIG_P2P
+	if (rtw_cfg80211_get_is_roch(padapter)
+		&& !roch_stay_in_cur_chan(padapter)
+		&& pcfg80211_wdinfo->remain_on_ch_channel.hw_value != u_ch
+	) {
+		/* roch is ongoing, switch back to rch */
+		if (pcfg80211_wdinfo->remain_on_ch_channel.hw_value != tx_ch)
+			set_channel_bwmode(padapter,
+				pcfg80211_wdinfo->remain_on_ch_channel.hw_value,
+				CHAN_OFFSET_NO_EXT,
+				CHANNEL_WIDTH_20,
+				RFK_TYPE_FORCE_NOT_DO);
+	} else
+	#endif
+#endif
+	if (leave_op) {
+		if (rtw_mi_check_status(padapter, MI_LINKED))
+			set_bch_bwmode(padapter, padapter_link, u_chdef.band, u_chdef.chan, u_chdef.offset, u_chdef.bw, RFK_TYPE_FORCE_NOT_DO);
+
+		rtw_back_opch(padapter);
+	}
+	rtw_cfg80211_set_is_mgmt_tx(padapter, 0);
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("%s, ret=%d\n", __func__, ret);
+#endif
+
+	return ret;
+
+}
+
+u8 rtw_mgnt_tx_handler(_adapter *adapter, u8 *buf)
+{
+	u8 rst = H2C_CMD_FAIL;
+	struct mgnt_tx_parm *mgnt_parm = (struct mgnt_tx_parm *)buf;
+
+	if (_cfg80211_rtw_mgmt_tx(adapter, mgnt_parm->tx_band, mgnt_parm->tx_ch, mgnt_parm->no_cck,
+		mgnt_parm->buf, mgnt_parm->len, mgnt_parm->wait_ack) == _SUCCESS)
+		rst = H2C_SUCCESS;
+
+	return rst;
+}
+
+static int cfg80211_rtw_mgmt_tx(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct wireless_dev *wdev,
+#else
+	struct net_device *ndev,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0)) || defined(COMPAT_KERNEL_RELEASE)
+	struct ieee80211_channel *chan,
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) || defined(COMPAT_KERNEL_RELEASE)
+	bool offchan,
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+	enum nl80211_channel_type channel_type,
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+	bool channel_type_valid,
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) || defined(COMPAT_KERNEL_RELEASE)
+	unsigned int wait,
+	#endif
+	const u8 *buf, size_t len,
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	bool no_cck,
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	bool dont_wait_for_ack,
+	#endif
+#else
+	struct cfg80211_mgmt_tx_params *params,
+#endif
+	u64 *cookie)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) || defined(COMPAT_KERNEL_RELEASE)
+	struct ieee80211_channel *chan = params->chan;
+	const u8 *buf = params->buf;
+	size_t len = params->len;
+	bool no_cck = params->no_cck;
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 2, 0))
+	bool no_cck = 0;
+#endif
+	int ret = 0;
+	u8 tx_ret;
+	int wait_ack = 1;
+	const u8 *dump_buf = buf;
+	size_t dump_len = len;
+	u32 dump_limit = RTW_MAX_MGMT_TX_CNT;
+	u32 dump_cnt = 0;
+	u32 sleep_ms = 0;
+	u32 retry_guarantee_ms = 0;
+	bool ack = _TRUE;
+	u8 tx_band;
+	u8 tx_ch;
+	u8 category, action;
+	u8 frame_styp;
+#ifdef CONFIG_P2P
+	u8 is_p2p = 0;
+#endif
+	int type = (-1);
+	systime start = rtw_get_current_time();
+	_adapter *padapter;
+	struct dvobj_priv *dvobj;
+	struct rtw_wdev_priv *pwdev_priv;
+	struct rf_ctl_t *rfctl;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev == wiphy_to_pd_wdev(wiphy))
+		padapter = wiphy_to_adapter(wiphy);
+	else
+	#endif
+	if (wdev_to_ndev(wdev))
+		padapter = (_adapter *)rtw_netdev_priv(wdev_to_ndev(wdev));
+	else {
+		ret = -EINVAL;
+		goto exit;
+	}
+#else
+	struct wireless_dev *wdev;
+
+	if (ndev == NULL) {
+		ret = -EINVAL;
+		goto exit;
+	}
+	padapter = (_adapter *)rtw_netdev_priv(ndev);
+	wdev = ndev_to_wdev(ndev);
+#endif
+
+	if (chan == NULL) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	rfctl = adapter_to_rfctl(padapter);
+	tx_band = (u8)nl80211_band_to_rtw_band(chan->band);
+	tx_ch = (u8)rtw_freq2ch(chan->center_freq);
+	if (IS_CH_WAITING(rfctl)) {
+		#ifdef CONFIG_DFS_MASTER
+		if (rtw_rfctl_overlap_radar_detect_ch(rfctl, nl80211_band_to_rtw_band(chan->band), tx_ch, CHANNEL_WIDTH_20, CHAN_OFFSET_NO_EXT)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+		#endif
+	}
+
+	dvobj = adapter_to_dvobj(padapter);
+	pwdev_priv = adapter_wdev_data(padapter);
+
+	/* cookie generation */
+	*cookie = pwdev_priv->mgmt_tx_cookie++;
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO(FUNC_ADPT_FMT"%s len=%zu, ch=%d"
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+		", ch_type=%d"
+		#endif
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+		", channel_type_valid=%d"
+		#endif
+		"\n", FUNC_ADPT_ARG(padapter), wdev == wiphy_to_pd_wdev(wiphy) ? " PD" : ""
+		, len, tx_ch
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+		, channel_type
+		#endif
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+		, channel_type_valid
+		#endif
+	);
+#endif /* CONFIG_DEBUG_CFG80211 */
+
+	/* indicate ack before issue frame to avoid racing with rsp frame */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	rtw_cfg80211_mgmt_tx_status(wdev, *cookie, buf, len, ack, GFP_KERNEL);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34) && LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 36))
+	cfg80211_action_tx_status(ndev, *cookie, buf, len, ack, GFP_KERNEL);
+#endif
+
+	frame_styp = le16_to_cpu(((struct rtw_ieee80211_hdr_3addr *)buf)->frame_ctl) & IEEE80211_FCTL_STYPE;
+	if (IEEE80211_STYPE_PROBE_RESP == frame_styp) {
+#ifdef CONFIG_P2P
+		/* 36 = IEEE80211_3ADDR_LEN + _TIMESTAMP_ + _BEACON_ITERVAL_ + _CAPABILITY_ */
+		if(rtw_get_p2p_ie(dump_buf + 36, dump_len - 36, NULL, NULL)) {
+			struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+			if (rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE)) {
+				if (!rtw_p2p_enable(padapter, P2P_ROLE_DEVICE)) {
+					ret = -EOPNOTSUPP;
+					goto exit;
+				}
+			}
+			no_cck = 1;
+		}
+#endif
+#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("RTW_Tx: probe_resp tx_ch=%d, no_cck=%u, da="MAC_FMT"\n", tx_ch, no_cck, MAC_ARG(GetAddr1Ptr(buf)));
+#endif /* CONFIG_DEBUG_CFG80211 */
+		wait_ack = 0;
+		goto dump;
+	}
+	else if (frame_styp == RTW_IEEE80211_STYPE_AUTH) {
+		int retval = 0;
+
+		RTW_INFO("RTW_Tx:tx_ch=%d, no_cck=%u, da="MAC_FMT"\n", tx_ch, no_cck, MAC_ARG(GetAddr1Ptr(buf)));
+
+#ifdef CONFIG_RTW_80211R_AP
+		rtw_ft_process_ft_auth_rsp(padapter, (u8 *)buf, len);
+#endif
+
+		retval = rtw_sae_preprocess(padapter, buf, len, _TRUE);
+		if (retval == 2)
+			goto exit;
+		if (retval == 0)
+			RTW_INFO("RTW_Tx:AUTH\n");
+		dump_limit = 1;
+		goto dump;
+	}
+
+	if (rtw_action_frame_parse(buf, len, &category, &action) == _FALSE) {
+		RTW_INFO(FUNC_ADPT_FMT" frame_control:0x%02x\n", FUNC_ADPT_ARG(padapter),
+			le16_to_cpu(((struct rtw_ieee80211_hdr_3addr *)buf)->frame_ctl));
+		goto exit;
+	}
+
+	RTW_INFO("RTW_Tx:tx_ch=%d, no_cck=%u, da="MAC_FMT"\n", tx_ch, no_cck, MAC_ARG(GetAddr1Ptr(buf)));
+#ifdef CONFIG_P2P
+	type = rtw_p2p_check_frames(padapter, buf, len, _TRUE);
+	if (type >= 0) {
+		is_p2p = 1;
+		no_cck = 1; /* force no CCK for P2P frames */
+		goto dump;
+	}
+#endif
+#ifdef CONFIG_RTW_MESH
+	if (MLME_IS_MESH(padapter)) {
+		type = rtw_mesh_check_frames_tx(padapter, &dump_buf, &dump_len);
+		if (type >= 0) {
+			dump_limit = 1;
+			goto dump;
+		}
+	}
+#endif
+	if (category == RTW_WLAN_CATEGORY_PUBLIC) {
+		RTW_INFO("RTW_Tx:%s\n", action_public_str(action));
+		switch (action) {
+		case ACT_PUBLIC_GAS_INITIAL_REQ:
+		case ACT_PUBLIC_GAS_INITIAL_RSP:
+			sleep_ms = 50;
+			retry_guarantee_ms = RTW_MAX_MGMT_TX_MS_GAS;
+			break;
+		}
+	}
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+	else if (category == RTW_WLAN_CATEGORY_RADIO_MEAS)
+		RTW_INFO("RTW_Tx: RRM Action\n");
+#endif
+	else
+		RTW_INFO("RTW_Tx:category(%u), action(%u)\n", category, action);
+
+dump:
+
+	while (1) {
+		dump_cnt++;
+
+		/* ROCH uses phl scan to handle, cancel scan is not necessary */
+		if (!rtw_cfg80211_get_is_roch(padapter) &&
+		    frame_styp != IEEE80211_STYPE_PROBE_RESP) {
+			rtw_mi_set_scan_deny(padapter, 1000);
+			rtw_mi_scan_abort(padapter, _TRUE);
+		}
+
+		tx_ret = rtw_mgnt_tx_cmd(padapter, tx_band, tx_ch, no_cck, dump_buf, dump_len, wait_ack, RTW_CMDF_WAIT_ACK);
+		if (tx_ret == _SUCCESS
+			|| (dump_cnt >= dump_limit && rtw_get_passing_time_ms(start) >= retry_guarantee_ms))
+			break;
+
+		if (sleep_ms > 0)
+			rtw_msleep_os(sleep_ms);
+	}
+
+	if (tx_ret != _SUCCESS || dump_cnt > 1) {
+		RTW_INFO(FUNC_ADPT_FMT" %s (%d/%d) in %d ms\n", FUNC_ADPT_ARG(padapter),
+			tx_ret == _SUCCESS ? "OK" : "FAIL", dump_cnt, dump_limit, rtw_get_passing_time_ms(start));
+	}
+
+#ifdef CONFIG_P2P
+	if (is_p2p) {
+		switch (type) {
+		case P2P_GO_NEGO_CONF:
+			if (0) {
+				RTW_INFO(FUNC_ADPT_FMT" Nego confirm. state=%u, status=%u, iaddr="MAC_FMT"\n"
+					, FUNC_ADPT_ARG(padapter), pwdev_priv->nego_info.state, pwdev_priv->nego_info.status
+					, MAC_ARG(pwdev_priv->nego_info.iface_addr));
+			}
+			if (pwdev_priv->nego_info.state == 2
+				&& pwdev_priv->nego_info.status == 0
+				&& rtw_check_invalid_mac_address(pwdev_priv->nego_info.iface_addr, _FALSE) == _FALSE
+			) {
+				_adapter *intended_iface = dvobj_get_adapter_by_addr(dvobj, pwdev_priv->nego_info.iface_addr);
+
+				if (intended_iface) {
+					RTW_INFO(FUNC_ADPT_FMT" Nego confirm. Allow only "ADPT_FMT" to scan for 2000 ms\n"
+						, FUNC_ADPT_ARG(padapter), ADPT_ARG(intended_iface));
+					/* allow only intended_iface to do scan for 2000 ms */
+					rtw_mi_set_scan_deny(padapter, 2000);
+					rtw_clear_scan_deny(intended_iface);
+				}
+			}
+			break;
+		case P2P_INVIT_RESP:
+			if (pwdev_priv->invit_info.flags & BIT(0)
+				&& pwdev_priv->invit_info.status == 0
+			) {
+				rtw_clear_scan_deny(padapter);
+				RTW_INFO(FUNC_ADPT_FMT" agree with invitation of persistent group\n",
+					FUNC_ADPT_ARG(padapter));
+				#if !RTW_P2P_GROUP_INTERFACE
+				rtw_mi_buddy_set_scan_deny(padapter, 5000);
+				#endif
+			}
+			break;
+		}
+	}
+#endif /* CONFIG_P2P */
+
+	/* Driver will stuck at CONNECT_FG if STA TX auth frame fail, so trigger join timeout handler */
+	if (tx_ret != _SUCCESS && frame_styp == RTW_IEEE80211_STYPE_AUTH && MLME_IS_STA(padapter)) {
+		RTW_INFO("TX auth fail, trigger join timeout handler\n");
+		set_assoc_timer(&(padapter->mlmepriv), 1000);
+	}
+
+	if (dump_buf != buf)
+		rtw_mfree((u8 *)dump_buf, dump_len);
+exit:
+	return ret;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+static void cfg80211_rtw_mgmt_frame_register(struct wiphy *wiphy,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct wireless_dev *wdev,
+#else
+	struct net_device *ndev,
+#endif
+	u16 frame_type, bool reg)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	struct net_device *ndev = wdev_to_ndev(wdev);
+#endif
+	_adapter *adapter;
+	struct rtw_wdev_priv *pwdev_priv;
+
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+	if (wdev == wiphy_to_pd_wdev(wiphy)) {
+		adapter = wiphy_to_adapter(wiphy);
+	} else
+#endif
+	{
+		if (ndev == NULL)
+			goto exit;
+		adapter = (_adapter *)rtw_netdev_priv(ndev);
+	}
+	pwdev_priv = adapter_wdev_data(adapter);
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO(FUNC_ADPT_FMT" frame_type:%x, reg:%d\n", FUNC_ADPT_ARG(adapter),
+		frame_type, reg);
+#endif
+
+	switch (frame_type) {
+	case IEEE80211_STYPE_AUTH: /* 0x00B0 */
+		if (reg > 0)
+			SET_CFG80211_REPORT_MGMT(pwdev_priv, IEEE80211_STYPE_AUTH);
+		else
+			CLR_CFG80211_REPORT_MGMT(pwdev_priv, IEEE80211_STYPE_AUTH);
+		break;
+	case IEEE80211_STYPE_PROBE_REQ: /* 0x0040 */
+		if (reg > 0)
+			SET_CFG80211_REPORT_MGMT(pwdev_priv, IEEE80211_STYPE_PROBE_REQ);
+		else
+			CLR_CFG80211_REPORT_MGMT(pwdev_priv, IEEE80211_STYPE_PROBE_REQ);
+		break;
+#ifdef not_yet
+	case IEEE80211_STYPE_ACTION: /* 0x00D0 */
+		if (reg > 0)
+			SET_CFG80211_REPORT_MGMT(pwdev_priv, IEEE80211_STYPE_ACTION);
+		else
+			CLR_CFG80211_REPORT_MGMT(pwdev_priv, IEEE80211_STYPE_ACTION);
+		break;
+#endif
+	default:
+		break;
+	}
+
+exit:
+	return;
+}
+#else
+static void cfg80211_rtw_update_mgmt_frame_register(
+	struct wiphy *wiphy,
+	struct wireless_dev *wdev,
+	struct mgmt_frame_regs *upd)
+{
+	struct net_device *ndev;
+	_adapter *padapter;
+	struct rtw_wdev_priv *pwdev_priv;
+	u32 rtw_stypes_mask = 0;
+	u32 rtw_mstypes_mask = 0;
+
+	ndev = wdev_to_ndev(wdev);
+
+	if (ndev == NULL)
+		goto exit;
+
+	padapter = (_adapter *)rtw_netdev_priv(ndev);
+	pwdev_priv = adapter_wdev_data(padapter);
+
+	/* Driver only supports Auth and Probe request */
+	rtw_stypes_mask = BIT(IEEE80211_STYPE_AUTH >> 4) |
+			  BIT(IEEE80211_STYPE_PROBE_REQ >> 4);
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO(FUNC_ADPT_FMT " global_stypes:0x%08x interface_stypes:0x%08x\n",
+		FUNC_ADPT_ARG(padapter), upd->global_stypes, upd->interface_stypes);
+	RTW_INFO(FUNC_ADPT_FMT " global_mcast_stypes:0x%08x interface_mcast_stypes:0x%08x\n",
+		FUNC_ADPT_ARG(padapter), upd->global_mcast_stypes, upd->interface_mcast_stypes);
+	RTW_INFO(FUNC_ADPT_FMT " old_regs:0x%08x new_regs:0x%08x\n",
+		FUNC_ADPT_ARG(padapter), pwdev_priv->mgmt_regs,
+		(upd->interface_stypes & rtw_stypes_mask));
+#endif
+	if (pwdev_priv->mgmt_regs !=
+			(upd->interface_stypes & rtw_stypes_mask)) {
+		pwdev_priv->mgmt_regs = (upd->interface_stypes & rtw_stypes_mask);
+	}
+
+exit:
+	return;
+}
+#endif
+
+#if defined(CONFIG_TDLS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+static int cfg80211_rtw_tdls_mgmt(struct wiphy *wiphy,
+	struct net_device *ndev,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	const u8 *peer,
+#else
+	u8 *peer,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0))
+	int link_id,
+#endif
+	u8 action_code,
+	u8 dialog_token,
+	u16 status_code,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0))
+	u32 peer_capability,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0))
+	bool initiator,
+#endif
+	const u8 *buf,
+	size_t len)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
+	int ret = 0;
+	struct tdls_txmgmt txmgmt;
+
+	if (rtw_hw_chk_wl_func(dvobj, WL_FUNC_TDLS) == _FALSE) {
+		RTW_INFO("Discard tdls action:%d, since hal doesn't support tdls\n", action_code);
+		goto discard;
+	}
+
+	if (rtw_is_tdls_enabled(padapter) == _FALSE) {
+		RTW_INFO("TDLS is not enabled\n");
+		goto discard;
+	}
+
+	if (rtw_tdls_is_driver_setup(padapter)) {
+		RTW_INFO("Discard tdls action:%d, let driver to set up direct link\n", action_code);
+		goto discard;
+	}
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	_rtw_memcpy(txmgmt.peer, peer, ETH_ALEN);
+	txmgmt.action_code = action_code;
+	txmgmt.dialog_token = dialog_token;
+	txmgmt.status_code = status_code;
+	txmgmt.len = len;
+	txmgmt.buf = (u8 *)rtw_malloc(txmgmt.len);
+	if (txmgmt.buf == NULL) {
+		ret = -ENOMEM;
+		goto bad;
+	}
+	_rtw_memcpy(txmgmt.buf, (void *)buf, txmgmt.len);
+
+	/* Debug purpose */
+#if 1
+	RTW_INFO("%s %d\n", __FUNCTION__, __LINE__);
+	RTW_INFO("peer:"MAC_FMT", action code:%d, dialog:%d, status code:%d\n",
+		MAC_ARG(txmgmt.peer), txmgmt.action_code,
+		txmgmt.dialog_token, txmgmt.status_code);
+#if 0
+	if (txmgmt.len > 0) {
+		int i = 0;
+		for (; i < len; i++)
+			printk("%02x ", *(txmgmt.buf + i));
+		RTW_INFO("len:%d\n", (u32)txmgmt.len);
+	}
+#endif
+#endif
+
+	switch (txmgmt.action_code) {
+	case TDLS_SETUP_REQUEST:
+		issue_tdls_setup_req(padapter, &txmgmt, _TRUE);
+		break;
+	case TDLS_SETUP_RESPONSE:
+		issue_tdls_setup_rsp(padapter, &txmgmt);
+		break;
+	case TDLS_SETUP_CONFIRM:
+		issue_tdls_setup_cfm(padapter, &txmgmt);
+		break;
+	case TDLS_TEARDOWN:
+		issue_tdls_teardown(padapter, &txmgmt, _TRUE);
+		break;
+	case TDLS_DISCOVERY_REQUEST:
+		issue_tdls_dis_req(padapter, &txmgmt);
+		break;
+	case TDLS_DISCOVERY_RESPONSE:
+		issue_tdls_dis_rsp(padapter, &txmgmt, pmlmeinfo->enc_algo ? _TRUE : _FALSE);
+		break;
+	}
+
+bad:
+	if (txmgmt.buf)
+		rtw_mfree(txmgmt.buf, txmgmt.len);
+
+discard:
+	return ret;
+}
+
+static int cfg80211_rtw_tdls_oper(struct wiphy *wiphy,
+	struct net_device *ndev,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	const u8 *peer,
+#else
+	u8 *peer,
+#endif
+	enum nl80211_tdls_operation oper)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct tdls_txmgmt	txmgmt;
+	struct sta_info *ptdls_sta = NULL;
+	int ret = 0;
+
+	RTW_INFO(FUNC_NDEV_FMT", nl80211_tdls_operation:%d\n", FUNC_NDEV_ARG(ndev), oper);
+
+	if (rtw_hw_chk_wl_func(dvobj, WL_FUNC_TDLS) == _FALSE) {
+		RTW_INFO("Discard tdls oper:%d, since hal doesn't support tdls\n", oper);
+		return 0;
+	}
+
+	if (rtw_is_tdls_enabled(padapter) == _FALSE) {
+		RTW_INFO("TDLS is not enabled\n");
+		return 0;
+	}
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	if (peer)
+		_rtw_memcpy(txmgmt.peer, peer, ETH_ALEN);
+
+	if (rtw_tdls_is_driver_setup(padapter)) {
+		/* these two cases are done by driver itself */
+		if (oper == NL80211_TDLS_ENABLE_LINK || oper == NL80211_TDLS_DISABLE_LINK)
+			return 0;
+	}
+
+	switch (oper) {
+	case NL80211_TDLS_DISCOVERY_REQ:
+		issue_tdls_dis_req(padapter, &txmgmt);
+		break;
+	case NL80211_TDLS_SETUP:
+#ifdef CONFIG_WFD
+		if (_AES_ != padapter->securitypriv.dot11PrivacyAlgrthm) {
+			if (padapter->wdinfo.wfd_tdls_weaksec == _TRUE)
+				issue_tdls_setup_req(padapter, &txmgmt, _TRUE);
+			else
+				RTW_INFO("[%s] Current link is not AES, SKIP sending the tdls setup request!!\n", __FUNCTION__);
+		} else
+#endif /* CONFIG_WFD */
+		{
+			issue_tdls_setup_req(padapter, &txmgmt, _TRUE);
+		}
+		break;
+	case NL80211_TDLS_TEARDOWN:
+		ptdls_sta = rtw_get_stainfo(&(padapter->stapriv), txmgmt.peer);
+		if (ptdls_sta != NULL) {
+			txmgmt.status_code = _RSON_TDLS_TEAR_UN_RSN_;
+			issue_tdls_teardown(padapter, &txmgmt, _TRUE);
+		} else
+			RTW_INFO("TDLS peer not found\n");
+		break;
+	case NL80211_TDLS_ENABLE_LINK:
+		RTW_INFO(FUNC_NDEV_FMT", NL80211_TDLS_ENABLE_LINK;mac:"MAC_FMT"\n", FUNC_NDEV_ARG(ndev), MAC_ARG(peer));
+		ptdls_sta = rtw_get_stainfo(&(padapter->stapriv), (u8 *)peer);
+		if (ptdls_sta == NULL) {
+			ret = -ENOENT;
+			break;
+		}
+
+		rtw_tdls_set_link_established(padapter, _TRUE);
+		ptdls_sta->tdls_sta_state |= TDLS_LINKED_STATE;
+		ptdls_sta->state |= WIFI_ASOC_STATE;
+		if (_FAIL == rtw_tdls_cmd(padapter, txmgmt.peer, TDLS_ESTABLISHED, RTW_CMDF_WAIT_ACK)) {
+			RTW_ERR(FUNC_NDEV_FMT", nl80211_tdls_operation:%d fail\n", FUNC_NDEV_ARG(ndev), oper);
+			ret = -EINVAL;
+		}
+		break;
+	case NL80211_TDLS_DISABLE_LINK:
+		RTW_INFO(FUNC_NDEV_FMT", NL80211_TDLS_DISABLE_LINK;mac:"MAC_FMT"\n", FUNC_NDEV_ARG(ndev), MAC_ARG(peer));
+		ptdls_sta = rtw_get_stainfo(&(padapter->stapriv), (u8 *)peer);
+		if (ptdls_sta == NULL) {
+			ret = -ENOENT;
+			break;
+		}
+
+		if (!(ptdls_sta->tdls_sta_state & TDLS_RESETUP_STATE)) {
+			rtw_tdls_teardown_pre_hdl(padapter, ptdls_sta);
+			if (_FAIL == rtw_tdls_cmd(padapter, (u8 *)peer, TDLS_TEARDOWN_STA_LOCALLY_POST, RTW_CMDF_WAIT_ACK)) {
+				RTW_ERR(FUNC_NDEV_FMT", nl80211_tdls_operation:%d fail\n", FUNC_NDEV_ARG(ndev), oper);
+				ret = -EINVAL;
+			}
+		} else {
+			ptdls_sta->tdls_sta_state &= ~TDLS_RESETUP_STATE;
+		}
+		break;
+	}
+	return ret;
+}
+#endif /* CONFIG_TDLS */
+
+#if defined(CONFIG_RTW_MESH) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38))
+
+#if DBG_RTW_CFG80211_MESH_CONF
+#define LEGACY_RATES_STR_LEN (RTW_G_RATES_NUM * 5 + 1)
+int get_legacy_rates_str(struct wiphy *wiphy, enum nl80211_band band, u32 mask, char *buf)
+{
+	int i;
+	int cnt = 0;
+
+	for (i = 0; i < wiphy->bands[band]->n_bitrates; i++) {
+		if (mask & BIT(i)) {
+			cnt += snprintf(buf + cnt, LEGACY_RATES_STR_LEN - cnt -1, "%d.%d "
+				, wiphy->bands[band]->bitrates[i].bitrate / 10
+				, wiphy->bands[band]->bitrates[i].bitrate % 10);
+			if (cnt >= LEGACY_RATES_STR_LEN - 1)
+				break;
+		}
+	}
+
+	return cnt;
+}
+
+void dump_mesh_setup(void *sel, struct wiphy *wiphy, const struct mesh_setup *setup)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	struct cfg80211_chan_def *chdef = (struct cfg80211_chan_def *)(&setup->chandef);
+#endif
+	struct ieee80211_channel *chan;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	chan = (struct ieee80211_channel *)chdef->chan;
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	chan = (struct ieee80211_channel *)setup->channel;
+#endif
+
+	RTW_PRINT_SEL(sel, "mesh_id:\"%s\", len:%u\n", setup->mesh_id, setup->mesh_id_len);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	RTW_PRINT_SEL(sel, "sync_method:%u\n", setup->sync_method);
+#endif
+	RTW_PRINT_SEL(sel, "path_sel_proto:%u, path_metric:%u\n", setup->path_sel_proto, setup->path_metric);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	RTW_PRINT_SEL(sel, "auth_id:%u\n", setup->auth_id);
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	if (setup->ie && setup->ie_len) {
+		RTW_PRINT_SEL(sel, "ie:%p, len:%u\n", setup->ie, setup->ie_len);
+		dump_ies(RTW_DBGDUMP, setup->ie, setup->ie_len);
+	}
+#else
+	if (setup->vendor_ie && setup->vendor_ie_len) {
+		RTW_PRINT_SEL(sel, "ie:%p, len:%u\n", setup->vendor_ie, setup->vendor_ie_len);
+		dump_ies(RTW_DBGDUMP, setup->vendor_ie, setup->vendor_ie_len);
+	}
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	RTW_PRINT_SEL(sel, "is_authenticated:%d, is_secure:%d\n", setup->is_authenticated, setup->is_secure);
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+	RTW_PRINT_SEL(sel, "user_mpm:%d\n", setup->user_mpm);
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	RTW_PRINT_SEL(sel, "dtim_period:%u, beacon_interval:%u\n", setup->dtim_period, setup->beacon_interval);
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	RTW_PRINT_SEL(sel, "center_freq:%u, ch:%u, width:%s, cfreq1:%u, cfreq2:%u\n"
+		, chan->center_freq, chan->hw_value, nl80211_chan_width_str(chdef->width), chdef->center_freq1, chdef->center_freq2);
+#else
+	RTW_PRINT_SEL(sel, "center_freq:%u, ch:%u, channel_type:%s\n"
+		, chan->center_freq, chan->hw_value, nl80211_channel_type_str(setup->channel_type));
+#endif
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	if (setup->mcast_rate[chan->band]) {
+		RTW_PRINT_SEL(sel, "mcast_rate:%d.%d\n"
+			, wiphy->bands[chan->band]->bitrates[setup->mcast_rate[chan->band] - 1].bitrate / 10
+			, wiphy->bands[chan->band]->bitrates[setup->mcast_rate[chan->band] - 1].bitrate % 10
+		);
+	}
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	if (setup->basic_rates) {
+		char buf[LEGACY_RATES_STR_LEN] = {0};
+
+		get_legacy_rates_str(wiphy, chan->band, setup->basic_rates, buf);
+		RTW_PRINT_SEL(sel, "basic_rates:%s\n", buf);
+	}
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0))
+	if (setup->beacon_rate.control[chan->band].legacy) {
+		char buf[LEGACY_RATES_STR_LEN] = {0};
+
+		get_legacy_rates_str(wiphy, chan->band, setup->beacon_rate.control[chan->band].legacy, buf);
+		RTW_PRINT_SEL(sel, "beacon_rate.legacy:%s\n", buf);
+	}
+	if (*((u32 *)&(setup->beacon_rate.control[chan->band].ht_mcs[0]))
+		|| *((u32 *)&(setup->beacon_rate.control[chan->band].ht_mcs[4]))
+		|| *((u16 *)&(setup->beacon_rate.control[chan->band].ht_mcs[8]))
+	) {
+		RTW_PRINT_SEL(sel, "beacon_rate.ht_mcs:"HT_RX_MCS_BMP_FMT"\n"
+			, HT_RX_MCS_BMP_ARG(setup->beacon_rate.control[chan->band].ht_mcs));
+	}
+
+	if (setup->beacon_rate.control[chan->band].vht_mcs[0]
+		|| setup->beacon_rate.control[chan->band].vht_mcs[1]
+		|| setup->beacon_rate.control[chan->band].vht_mcs[2]
+		|| setup->beacon_rate.control[chan->band].vht_mcs[3]
+	) {
+		int i;
+
+		for (i = 0; i < 4; i++) {/* parsing up to 4SS */
+			u16 mcs_mask = setup->beacon_rate.control[chan->band].vht_mcs[i];
+
+			RTW_PRINT_SEL(sel, "beacon_rate.vht_mcs[%d]:%s\n", i
+				, mcs_mask == 0x00FF ? "0~7" : mcs_mask == 0x01FF ? "0~8" : mcs_mask == 0x03FF ? "0~9" : "invalid");
+		}
+	}
+
+	if (setup->beacon_rate.control[chan->band].gi) {
+		RTW_PRINT_SEL(sel, "beacon_rate.gi:%s\n"
+			, setup->beacon_rate.control[chan->band].gi == NL80211_TXRATE_FORCE_SGI ? "SGI" :
+				setup->beacon_rate.control[chan->band].gi == NL80211_TXRATE_FORCE_LGI ? "LGI" : "invalid"
+		);
+	}
+#endif
+}
+
+void dump_mesh_config(void *sel, const struct mesh_config *conf)
+{
+	RTW_PRINT_SEL(sel, "dot11MeshRetryTimeout:%u\n", conf->dot11MeshRetryTimeout);
+	RTW_PRINT_SEL(sel, "dot11MeshConfirmTimeout:%u\n", conf->dot11MeshConfirmTimeout);
+	RTW_PRINT_SEL(sel, "dot11MeshHoldingTimeout:%u\n", conf->dot11MeshHoldingTimeout);
+	RTW_PRINT_SEL(sel, "dot11MeshMaxPeerLinks:%u\n", conf->dot11MeshMaxPeerLinks);
+	RTW_PRINT_SEL(sel, "dot11MeshMaxRetries:%u\n", conf->dot11MeshMaxRetries);
+	RTW_PRINT_SEL(sel, "dot11MeshTTL:%u\n", conf->dot11MeshTTL);
+	RTW_PRINT_SEL(sel, "element_ttl:%u\n", conf->element_ttl);
+	RTW_PRINT_SEL(sel, "auto_open_plinks:%d\n", conf->auto_open_plinks);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	RTW_PRINT_SEL(sel, "dot11MeshNbrOffsetMaxNeighbor:%u\n", conf->dot11MeshNbrOffsetMaxNeighbor);
+#endif
+
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPmaxPREQretries:%u\n", conf->dot11MeshHWMPmaxPREQretries);
+	RTW_PRINT_SEL(sel, "path_refresh_time:%u\n", conf->path_refresh_time);
+	RTW_PRINT_SEL(sel, "min_discovery_timeout:%u\n", conf->min_discovery_timeout);
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPactivePathTimeout:%u\n", conf->dot11MeshHWMPactivePathTimeout);
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPpreqMinInterval:%u\n", conf->dot11MeshHWMPpreqMinInterval);	
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPperrMinInterval:%u\n", conf->dot11MeshHWMPperrMinInterval);
+#endif
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPnetDiameterTraversalTime:%u\n", conf->dot11MeshHWMPnetDiameterTraversalTime);
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPRootMode:%u\n", conf->dot11MeshHWMPRootMode);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPRannInterval:%u\n", conf->dot11MeshHWMPRannInterval);
+	RTW_PRINT_SEL(sel, "dot11MeshGateAnnouncementProtocol:%d\n", conf->dot11MeshGateAnnouncementProtocol);
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
+	RTW_PRINT_SEL(sel, "dot11MeshForwarding:%d\n", conf->dot11MeshForwarding);
+	RTW_PRINT_SEL(sel, "rssi_threshold:%d\n", conf->rssi_threshold);
+#endif
+	
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	RTW_PRINT_SEL(sel, "ht_opmode:0x%04x\n", conf->ht_opmode);
+#endif
+	
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPactivePathToRootTimeout:%u\n", conf->dot11MeshHWMPactivePathToRootTimeout);
+	RTW_PRINT_SEL(sel, "dot11MeshHWMProotInterval:%u\n", conf->dot11MeshHWMProotInterval);
+	RTW_PRINT_SEL(sel, "dot11MeshHWMPconfirmationInterval:%u\n", conf->dot11MeshHWMPconfirmationInterval);
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	RTW_PRINT_SEL(sel, "power_mode:%s\n", nl80211_mesh_power_mode_str(conf->power_mode));
+	RTW_PRINT_SEL(sel, "dot11MeshAwakeWindowDuration:%u\n", conf->dot11MeshAwakeWindowDuration);
+#endif
+	
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	RTW_PRINT_SEL(sel, "plink_timeout:%u\n", conf->plink_timeout);
+#endif
+}
+#endif /* DBG_RTW_CFG80211_MESH_CONF */
+
+static void rtw_cfg80211_mesh_info_set_profile(struct rtw_mesh_info *minfo, const struct mesh_setup *setup)
+{
+	_rtw_memcpy(minfo->mesh_id, setup->mesh_id, setup->mesh_id_len);
+	minfo->mesh_id_len = setup->mesh_id_len;
+	minfo->mesh_pp_id = setup->path_sel_proto;
+	minfo->mesh_pm_id = setup->path_metric;
+	minfo->mesh_cc_id = 0;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	minfo->mesh_sp_id = setup->sync_method;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	minfo->mesh_auth_id = setup->auth_id;
+#else
+	if (setup->is_authenticated) {
+		u8 *rsn_ie;
+		sint rsn_ie_len;
+		struct rsne_info info;
+		u8 *akm;
+		u8 AKM_SUITE_SAE[4] = {0x00, 0x0F, 0xAC, 0x08};
+
+		rsn_ie = rtw_get_ie(setup->ie, WLAN_EID_RSN, &rsn_ie_len, setup->ie_len);
+		if (!rsn_ie || !rsn_ie_len) {
+			rtw_warn_on(1);
+			return;
+		}
+
+		if (rtw_rsne_info_parse(rsn_ie, rsn_ie_len + 2, &info) != _SUCCESS) {
+			rtw_warn_on(1);
+			return;
+		}
+
+		if (!info.akm_list || !info.akm_cnt) {
+			rtw_warn_on(1);
+			return;
+		}
+
+		akm = info.akm_list;
+		while (akm < info.akm_list + info.akm_cnt * 4) {
+			if (_rtw_memcmp(akm, AKM_SUITE_SAE, 4) == _TRUE) {
+				minfo->mesh_auth_id = 0x01;
+				break;
+			}
+		}
+
+		if (!minfo->mesh_auth_id) {
+			rtw_warn_on(1);
+			return;
+		}
+	}
+#endif
+}
+
+static inline bool chk_mesh_attr(enum nl80211_meshconf_params parm, u32 mask)
+{
+	return (mask >> (parm - 1)) & 0x1;
+}
+
+static void rtw_cfg80211_mesh_cfg_set(_adapter *adapter, const struct mesh_config *conf, u32 mask)
+{
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+
+#if 0 /* driver MPM */
+	if (chk_mesh_attr(NL80211_MESHCONF_RETRY_TIMEOUT, mask));
+	if (chk_mesh_attr(NL80211_MESHCONF_CONFIRM_TIMEOUT, mask));
+	if (chk_mesh_attr(NL80211_MESHCONF_HOLDING_TIMEOUT, mask));
+	if (chk_mesh_attr(NL80211_MESHCONF_MAX_PEER_LINKS, mask));
+	if (chk_mesh_attr(NL80211_MESHCONF_MAX_RETRIES, mask));
+#endif
+
+	if (chk_mesh_attr(NL80211_MESHCONF_TTL, mask))
+		mcfg->dot11MeshTTL = conf->dot11MeshTTL;
+	if (chk_mesh_attr(NL80211_MESHCONF_ELEMENT_TTL, mask))
+		mcfg->element_ttl = conf->element_ttl;
+
+#if 0 /* driver MPM */
+	if (chk_mesh_attr(NL80211_MESHCONF_AUTO_OPEN_PLINKS, mask));
+#endif
+
+#if 0 /* TBD: synchronization */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_SYNC_OFFSET_MAX_NEIGHBOR, mask));
+#endif
+#endif
+
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_MAX_PREQ_RETRIES, mask))
+		mcfg->dot11MeshHWMPmaxPREQretries = conf->dot11MeshHWMPmaxPREQretries;
+	if (chk_mesh_attr(NL80211_MESHCONF_PATH_REFRESH_TIME, mask))
+		mcfg->path_refresh_time = conf->path_refresh_time;
+	if (chk_mesh_attr(NL80211_MESHCONF_MIN_DISCOVERY_TIMEOUT, mask))
+		mcfg->min_discovery_timeout = conf->min_discovery_timeout;
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_ACTIVE_PATH_TIMEOUT, mask))
+		mcfg->dot11MeshHWMPactivePathTimeout = conf->dot11MeshHWMPactivePathTimeout;
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_PREQ_MIN_INTERVAL, mask))
+		mcfg->dot11MeshHWMPpreqMinInterval = conf->dot11MeshHWMPpreqMinInterval;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_PERR_MIN_INTERVAL, mask))
+		mcfg->dot11MeshHWMPperrMinInterval = conf->dot11MeshHWMPperrMinInterval;
+#endif
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_NET_DIAM_TRVS_TIME, mask))
+		mcfg->dot11MeshHWMPnetDiameterTraversalTime = conf->dot11MeshHWMPnetDiameterTraversalTime;
+
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_ROOTMODE, mask))
+		mcfg->dot11MeshHWMPRootMode = conf->dot11MeshHWMPRootMode;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_GATE_ANNOUNCEMENTS, mask))
+		mcfg->dot11MeshGateAnnouncementProtocol = conf->dot11MeshGateAnnouncementProtocol;
+	/* our current gate annc implementation rides on root annc with gate annc bit in PREQ flags */
+	if (mcfg->dot11MeshGateAnnouncementProtocol
+		&& mcfg->dot11MeshHWMPRootMode <= RTW_IEEE80211_ROOTMODE_ROOT
+	) {
+		mcfg->dot11MeshHWMPRootMode = RTW_IEEE80211_PROACTIVE_RANN;
+		RTW_INFO(ADPT_FMT" enable PROACTIVE_RANN becaue gate annc is needed\n", ADPT_ARG(adapter));
+	}
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_RANN_INTERVAL, mask))
+		mcfg->dot11MeshHWMPRannInterval = conf->dot11MeshHWMPRannInterval;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_FORWARDING, mask))
+		mcfg->dot11MeshForwarding = conf->dot11MeshForwarding;
+
+	if (chk_mesh_attr(NL80211_MESHCONF_RSSI_THRESHOLD, mask))
+		mcfg->rssi_threshold = conf->rssi_threshold;
+#endif
+
+#if 0 /* controlled by driver */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_HT_OPMODE, mask));
+#endif
+#endif
+	
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_PATH_TO_ROOT_TIMEOUT, mask))
+		mcfg->dot11MeshHWMPactivePathToRootTimeout = conf->dot11MeshHWMPactivePathToRootTimeout;
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_ROOT_INTERVAL, mask))
+		mcfg->dot11MeshHWMProotInterval = conf->dot11MeshHWMProotInterval;
+	if (chk_mesh_attr(NL80211_MESHCONF_HWMP_CONFIRMATION_INTERVAL, mask))
+		mcfg->dot11MeshHWMPconfirmationInterval = conf->dot11MeshHWMPconfirmationInterval;	
+#endif
+
+#if 0 /* TBD */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_POWER_MODE, mask));
+	if (chk_mesh_attr(NL80211_MESHCONF_AWAKE_WINDOW, mask));
+#endif
+#endif
+
+#if 0 /* driver MPM */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	if (chk_mesh_attr(NL80211_MESHCONF_PLINK_TIMEOUT, mask));
+#endif
+#endif
+}
+
+u8 *rtw_cfg80211_construct_mesh_beacon_ies(struct wiphy *wiphy, _adapter *adapter
+	, const struct mesh_config *conf, const struct mesh_setup *setup
+	, uint *ies_len)
+{
+	struct rtw_mesh_info *minfo = &adapter->mesh_info;
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	struct cfg80211_chan_def *chdef = (struct cfg80211_chan_def *)(&setup->chandef);
+#endif
+	struct ieee80211_channel *chan;
+	struct rtw_chan_def rtw_chdef = {0};
+#endif
+	uint len;
+	u8 n_bitrates;
+	u8 ht = 0;
+	u8 vht = 0;
+	u8 *rsn_ie = NULL;
+	sint rsn_ie_len = 0;
+	u8 *ies = NULL, *c;
+	u8 supported_rates[RTW_G_RATES_NUM] = {0};
+	int i;
+
+	*ies_len = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	chan = (struct ieee80211_channel *)chdef->chan;
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	chan = (struct ieee80211_channel *)setup->channel;
+#endif
+
+	n_bitrates = wiphy->bands[chan->band]->n_bitrates;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	rtw_get_chdef_from_cfg80211_chan_def(chdef, &ht, &rtw_chdef);
+#else
+	rtw_get_chdef_from_nl80211_channel_type(chan, setup->channel_type, &ht, &rtw_chdef);
+#endif
+	if (!rtw_chdef->chan)
+		goto exit;
+
+#if defined(CONFIG_80211AC_VHT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	vht = ht && rtw_chdef->band != BAND_ON_24G && rtw_chdef->bw >= CHANNEL_WIDTH_80; /* VHT40/VHT20? */
+#endif
+
+	RTW_INFO(FUNC_ADPT_FMT" => ch:%u,%u,%u, ht:%u, vht:%u\n"
+		, FUNC_ADPT_ARG(adapter), rtw_chdef->chan, rtw_chdef->bw, rtw_chdef->offset, ht, vht);
+#endif
+
+	rsn_ie = rtw_get_ie(setup->ie, WLAN_EID_RSN, &rsn_ie_len, setup->ie_len);
+	if (rsn_ie && !rsn_ie_len) {
+		rtw_warn_on(1);
+		rsn_ie = NULL;
+	}
+
+	len = _BEACON_IE_OFFSET_
+		+ 2 /* 0-length SSID */
+		+ (n_bitrates >= 8 ? 8 : n_bitrates) + 2 /* Supported Rates */
+		+ 3 /* DS parameter set */
+		+ 6 /* TIM  */
+		+ (n_bitrates > 8 ? n_bitrates - 8 + 2 : 0) /* Extended Supported Rates */
+		+ (rsn_ie ? rsn_ie_len + 2 : 0) /* RSN */
+		#if defined(CONFIG_80211N_HT)
+		+ (ht ? HT_CAP_IE_LEN + 2 + HT_OP_IE_LEN + 2 : 0) /* HT */
+		#endif
+		#if defined(CONFIG_80211AC_VHT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+		+ (vht ? VHT_CAP_IE_LEN + 2 + VHT_OP_IE_LEN + 2 : 0) /* VHT */
+		#endif
+		+ minfo->mesh_id_len + 2 /* Mesh ID */
+		+ 9 /* Mesh configuration */
+		;
+
+	ies = rtw_zmalloc(len);
+	if (!ies)
+		goto exit;
+
+	/* timestamp */
+	c = ies + 8;
+
+	/* beacon interval */
+	RTW_PUT_LE16(c , setup->beacon_interval);
+	c += 2;
+
+	/* capability */
+	if (rsn_ie)
+		*((u16 *)c) |= cpu_to_le16(cap_Privacy);
+	c += 2;
+
+	/* SSID */
+	c = rtw_set_ie(c, WLAN_EID_SSID, 0, NULL, NULL);
+
+	/* Supported Rates */
+	for (i = 0; i < n_bitrates; i++) {
+		supported_rates[i] = wiphy->bands[chan->band]->bitrates[i].bitrate / 5;
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+		if (setup->basic_rates & BIT(i))
+		#else
+		if (rtw_is_basic_rate_mix(supported_rates[i]))
+		#endif
+			supported_rates[i] |= IEEE80211_BASIC_RATE_MASK;
+	}
+	c = rtw_set_ie(c, WLAN_EID_SUPP_RATES, (n_bitrates >= 8 ? 8 : n_bitrates), supported_rates, NULL);
+
+	/* DS parameter set */
+	c = rtw_set_ie(c, WLAN_EID_DS_PARAMS, 1, &rtw_chdef->chan, NULL);
+
+	/* TIM */
+	*c = WLAN_EID_TIM;
+	*(c + 1) = 4;
+	c += 6;
+	//c = rtw_set_ie(c, _TIM_IE_, 4, NULL, NULL);
+
+	/* Extended Supported Rates */
+	if (n_bitrates > 8)
+		c = rtw_set_ie(c, WLAN_EID_EXT_SUPP_RATES, n_bitrates - 8, supported_rates + 8, NULL);
+
+	/* RSN */
+	if (rsn_ie)
+		c = rtw_set_ie(c, WLAN_EID_RSN, rsn_ie_len, rsn_ie + 2, NULL);
+
+#if defined(CONFIG_80211N_HT)
+	if (ht) {
+		struct ieee80211_sta_ht_cap *sta_ht_cap = &wiphy->bands[chan->band]->ht_cap;
+		u8 ht_cap[HT_CAP_IE_LEN];
+		u8 ht_op[HT_OP_IE_LEN];
+
+		_rtw_memset(ht_cap, 0, HT_CAP_IE_LEN);
+		_rtw_memset(ht_op, 0, HT_OP_IE_LEN);
+
+		/* WLAN_EID_HT_CAP */
+		RTW_PUT_LE16(HT_CAP_ELE_CAP_INFO(ht_cap), sta_ht_cap->cap);
+		SET_HT_CAP_ELE_MAX_AMPDU_LEN_EXP(ht_cap, sta_ht_cap->ampdu_factor);
+		SET_HT_CAP_ELE_MIN_MPDU_S_SPACE(ht_cap, sta_ht_cap->ampdu_density);
+		_rtw_memcpy(HT_CAP_ELE_SUP_MCS_SET(ht_cap), &sta_ht_cap->mcs, 16);
+		c = rtw_set_ie(c, WLAN_EID_HT_CAP, HT_CAP_IE_LEN, ht_cap, NULL);
+
+		/* WLAN_EID_HT_OPERATION */
+		SET_HT_OP_ELE_PRI_CHL(ht_op, rtw_chdef->chan);
+		switch (rtw_chdef->offset) {
+		case CHAN_OFFSET_UPPER:
+			SET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op, IEEE80211_SCA);
+			break;
+		case CHAN_OFFSET_LOWER:
+			SET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op, IEEE80211_SCB);
+			break;
+		case CHAN_OFFSET_NO_EXT:
+		default:
+			SET_HT_OP_ELE_2ND_CHL_OFFSET(ht_op, IEEE80211_SCN);
+			break;
+		}
+		if (rtw_chdef->bw >= CHANNEL_WIDTH_40)
+			SET_HT_OP_ELE_STA_CHL_WIDTH(ht_op, 1);
+		else
+			SET_HT_OP_ELE_STA_CHL_WIDTH(ht_op, 0);
+		c = rtw_set_ie(c, WLAN_EID_HT_OPERATION, HT_OP_IE_LEN, ht_op, NULL);
+	}
+#endif /* defined(CONFIG_80211N_HT) */
+
+#if defined(CONFIG_80211AC_VHT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	if (vht) {
+		struct ieee80211_sta_vht_cap *sta_vht_cap = &wiphy->bands[chan->band]->vht_cap;
+		u8 vht_cap[VHT_CAP_IE_LEN];
+		u8 vht_op[VHT_OP_IE_LEN];
+		u8 cch = rtw_phl_get_center_ch(&rtw_chdef);
+
+		_rtw_memset(vht_op, 0, VHT_OP_IE_LEN);
+
+		/* WLAN_EID_VHT_CAPABILITY */
+		_rtw_memcpy(vht_cap, &sta_vht_cap->cap, 4);
+		_rtw_memcpy(vht_cap + 4, &sta_vht_cap->vht_mcs, 8);
+		c = rtw_set_ie(c, WLAN_EID_VHT_CAPABILITY, VHT_CAP_IE_LEN, vht_cap, NULL);
+
+		/* WLAN_EID_VHT_OPERATION */
+		if (rtw_chdef->bw < CHANNEL_WIDTH_80) {
+			SET_VHT_OPERATION_ELE_CHL_WIDTH(vht_op, CH_WIDTH_20_40M);
+			SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(vht_op, 0);
+			SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(vht_op, 0);
+		} else if (rtw_chdef->bw == CHANNEL_WIDTH_80) {
+			SET_VHT_OPERATION_ELE_CHL_WIDTH(vht_op, CH_WIDTH_80_160M);
+			SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ1(vht_op, cch);
+			SET_VHT_OPERATION_ELE_CHL_CENTER_FREQ2(vht_op, 0);
+		} else {
+			RTW_ERR(FUNC_ADPT_FMT" unsupported BW:%u\n", FUNC_ADPT_ARG(adapter), rtw_chdef->bw);
+			rtw_warn_on(1);
+			rtw_mfree(ies, len);
+			goto exit;
+		}
+
+		/* Hard code 1 stream, MCS0-7 is a min Basic VHT MCS rates */
+		vht_op[3] = 0xfc;
+		vht_op[4] = 0xff;
+		c = rtw_set_ie(c, WLAN_EID_VHT_OPERATION, VHT_OP_IE_LEN, vht_op, NULL);
+	}
+#endif /* defined(CONFIG_80211AC_VHT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0)) */
+
+	/* Mesh ID */
+	c = rtw_set_ie_mesh_id(c, NULL, minfo->mesh_id, minfo->mesh_id_len);
+
+	/* Mesh configuration */
+	c = rtw_set_ie_mesh_config(c, NULL
+		, minfo->mesh_pp_id
+		, minfo->mesh_pm_id
+		, minfo->mesh_cc_id
+		, minfo->mesh_sp_id
+		, minfo->mesh_auth_id
+		, 0, 0, 0
+		, 1
+		, 0, 0
+		, mcfg->dot11MeshForwarding
+		, 0, 0, 0
+	);
+
+#if DBG_RTW_CFG80211_MESH_CONF
+	RTW_INFO(FUNC_ADPT_FMT" ies_len:%u\n", FUNC_ADPT_ARG(adapter), len);
+	dump_ies(RTW_DBGDUMP, ies + _BEACON_IE_OFFSET_, len - _BEACON_IE_OFFSET_);
+#endif
+
+exit:
+	if (ies)
+		*ies_len = len;
+	return ies;
+}
+
+static int cfg80211_rtw_get_mesh_config(struct wiphy *wiphy, struct net_device *dev
+	, struct mesh_config *conf)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mesh_cfg *mesh_cfg = &adapter->mesh_cfg;
+	int ret = 0;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+
+	/* driver MPM */
+	conf->dot11MeshRetryTimeout = 0;
+	conf->dot11MeshConfirmTimeout = 0;
+	conf->dot11MeshHoldingTimeout = 0;
+	conf->dot11MeshMaxPeerLinks = mesh_cfg->max_peer_links;
+	conf->dot11MeshMaxRetries = 0;
+
+	conf->dot11MeshTTL = mesh_cfg->dot11MeshTTL;
+	conf->element_ttl = mesh_cfg->element_ttl;
+
+	/* driver MPM */
+	conf->auto_open_plinks = 0;
+
+	/* TBD: synchronization */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	conf->dot11MeshNbrOffsetMaxNeighbor = 0;
+#endif
+
+	conf->dot11MeshHWMPmaxPREQretries = mesh_cfg->dot11MeshHWMPmaxPREQretries;
+	conf->path_refresh_time = mesh_cfg->path_refresh_time;
+	conf->min_discovery_timeout = mesh_cfg->min_discovery_timeout;
+	conf->dot11MeshHWMPactivePathTimeout = mesh_cfg->dot11MeshHWMPactivePathTimeout;
+	conf->dot11MeshHWMPpreqMinInterval = mesh_cfg->dot11MeshHWMPpreqMinInterval;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	conf->dot11MeshHWMPperrMinInterval = mesh_cfg->dot11MeshHWMPperrMinInterval;
+#endif
+	conf->dot11MeshHWMPnetDiameterTraversalTime = mesh_cfg->dot11MeshHWMPnetDiameterTraversalTime;
+	conf->dot11MeshHWMPRootMode = mesh_cfg->dot11MeshHWMPRootMode;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	conf->dot11MeshHWMPRannInterval = mesh_cfg->dot11MeshHWMPRannInterval;
+#endif
+	conf->dot11MeshGateAnnouncementProtocol = mesh_cfg->dot11MeshGateAnnouncementProtocol;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
+	conf->dot11MeshForwarding = mesh_cfg->dot11MeshForwarding;
+	conf->rssi_threshold = mesh_cfg->rssi_threshold;
+#endif
+
+	/* TBD */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+	conf->ht_opmode = 0xffff;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	conf->dot11MeshHWMPactivePathToRootTimeout = mesh_cfg->dot11MeshHWMPactivePathToRootTimeout;
+	conf->dot11MeshHWMProotInterval = mesh_cfg->dot11MeshHWMProotInterval;
+	conf->dot11MeshHWMPconfirmationInterval = mesh_cfg->dot11MeshHWMPconfirmationInterval;
+#endif
+
+	/* TBD: power save */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	conf->power_mode = NL80211_MESH_POWER_ACTIVE;
+	conf->dot11MeshAwakeWindowDuration = 0;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	conf->plink_timeout = mesh_cfg->plink_timeout;
+#endif
+
+	return ret;
+}
+
+static void rtw_mbss_info_change_notify(_adapter *adapter, bool minfo_changed, bool need_work)
+{
+	if (need_work)
+		rtw_mesh_work(&adapter->mesh_work);
+}
+
+static int cfg80211_rtw_update_mesh_config(struct wiphy *wiphy, struct net_device *dev
+	, u32 mask, const struct mesh_config *nconf)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	int ret = 0;
+	bool minfo_changed = _FALSE, need_work = _FALSE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	RTW_INFO(FUNC_ADPT_FMT" mask:0x%08x\n", FUNC_ADPT_ARG(adapter), mask);
+
+	rtw_cfg80211_mesh_cfg_set(adapter, nconf, mask);
+	rtw_update_beacon(adapter, adapter_link, WLAN_EID_MESH_CONFIG, NULL, _TRUE, 0);
+#if CONFIG_RTW_MESH_CTO_MGATE_CARRIER
+	if (rtw_mesh_cto_mgate_required(adapter))
+		rtw_netif_carrier_off(adapter->pnetdev);
+	else
+		rtw_netif_carrier_on(adapter->pnetdev);
+#endif
+	need_work = rtw_ieee80211_mesh_root_setup(adapter);
+
+	rtw_mbss_info_change_notify(adapter, minfo_changed, need_work);
+
+	return ret;
+}
+
+static int cfg80211_rtw_join_mesh(struct wiphy *wiphy, struct net_device *dev,
+	const struct mesh_config *conf, const struct mesh_setup *setup)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 *ies = NULL;
+	uint ies_len;
+	int ret = 0;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+
+#if DBG_RTW_CFG80211_MESH_CONF
+	RTW_INFO(FUNC_ADPT_FMT" mesh_setup:\n", FUNC_ADPT_ARG(adapter));
+	dump_mesh_setup(RTW_DBGDUMP, wiphy, setup);
+	RTW_INFO(FUNC_ADPT_FMT" mesh_config:\n", FUNC_ADPT_ARG(adapter));
+	dump_mesh_config(RTW_DBGDUMP, conf);
+#endif
+
+	if (rtw_cfg80211_sync_iftype(adapter) != _SUCCESS) {
+		ret = -ENOTSUPP;
+		goto exit;
+	}
+
+	/* initialization */
+	rtw_mesh_init_mesh_info(adapter);
+
+	/* apply cfg80211 settings*/
+	rtw_cfg80211_mesh_info_set_profile(&adapter->mesh_info, setup);
+	rtw_cfg80211_mesh_cfg_set(adapter, conf, 0xFFFFFFFF);
+
+	/* apply cfg80211 settings (join only) */
+	rtw_mesh_cfg_init_max_peer_links(adapter, conf->dot11MeshMaxPeerLinks);
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	rtw_mesh_cfg_init_plink_timeout(adapter, conf->plink_timeout);
+	#endif
+
+	rtw_ieee80211_mesh_root_setup(adapter);
+
+	ies = rtw_cfg80211_construct_mesh_beacon_ies(wiphy, adapter, conf, setup, &ies_len);
+	if (!ies) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	/* start mbss */
+	if (rtw_check_beacon_data(adapter, ies,  ies_len) != _SUCCESS) {
+		ret = -EINVAL;
+		goto exit;
+	}
+	
+	rtw_mesh_work(&adapter->mesh_work);
+
+exit:
+	if (ies)
+		rtw_mfree(ies, ies_len);
+	if (ret)
+		rtw_mesh_deinit_mesh_info(adapter);
+
+	return ret;
+}
+
+static int cfg80211_rtw_leave_mesh(struct wiphy *wiphy, struct net_device *dev)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	int ret = 0;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+
+	rtw_mesh_deinit_mesh_info(adapter);
+
+	rtw_stop_ap_cmd(adapter, RTW_CMDF_WAIT_ACK);
+
+	return ret;
+}
+
+static int cfg80211_rtw_add_mpath(struct wiphy *wiphy, struct net_device *dev
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	, const u8 *dst, const u8 *next_hop
+	#else
+	, u8 *dst, u8 *next_hop
+	#endif
+)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct sta_info *sta;
+	struct rtw_mesh_path *mpath;
+	int ret = 0;
+
+	rtw_rcu_read_lock();
+
+	sta = rtw_get_stainfo(stapriv, next_hop);
+	if (!sta) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	mpath = rtw_mesh_path_add(adapter, dst);
+	if (!mpath) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	rtw_mesh_path_fix_nexthop(mpath, sta);
+
+exit:
+	rtw_rcu_read_unlock();
+
+	return ret;
+}
+
+static int cfg80211_rtw_del_mpath(struct wiphy *wiphy, struct net_device *dev
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	, const u8 *dst
+	#else
+	, u8 *dst
+	#endif
+)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	int ret = 0;
+
+	if (dst) {
+		if (rtw_mesh_path_del(adapter, dst)) {
+			ret = -ENOENT;
+			goto exit;
+		}
+	} else {
+		rtw_mesh_path_flush_by_iface(adapter);
+	}	
+
+exit:
+	return ret;
+}
+
+static int cfg80211_rtw_change_mpath(struct wiphy *wiphy, struct net_device *dev
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	, const u8 *dst, const u8 *next_hop
+	#else
+	, u8 *dst, u8 *next_hop
+	#endif
+)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *stapriv = &adapter->stapriv;
+	struct sta_info *sta;
+	struct rtw_mesh_path *mpath;
+	int ret = 0;
+
+	rtw_rcu_read_lock();
+
+	sta = rtw_get_stainfo(stapriv, next_hop);
+	if (!sta) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	mpath = rtw_mesh_path_lookup(adapter, dst);
+	if (!mpath) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	rtw_mesh_path_fix_nexthop(mpath, sta);
+
+exit:
+	rtw_rcu_read_unlock();
+
+	return ret;
+}
+
+static void rtw_cfg80211_mpath_set_pinfo(struct rtw_mesh_path *mpath, u8 *next_hop, struct mpath_info *pinfo)
+{
+	struct sta_info *next_hop_sta = rtw_rcu_dereference(mpath->next_hop);
+
+	if (next_hop_sta)
+		_rtw_memcpy(next_hop, next_hop_sta->phl_sta->mac_addr, ETH_ALEN);
+	else
+		_rtw_memset(next_hop, 0, ETH_ALEN);
+
+	_rtw_memset(pinfo, 0, sizeof(*pinfo));
+
+	pinfo->generation = mpath->adapter->mesh_info.mesh_paths_generation;
+
+	pinfo->filled = 0
+		| MPATH_INFO_FRAME_QLEN
+		| MPATH_INFO_SN
+		| MPATH_INFO_METRIC
+		| MPATH_INFO_EXPTIME
+		| MPATH_INFO_DISCOVERY_TIMEOUT
+		| MPATH_INFO_DISCOVERY_RETRIES
+		| MPATH_INFO_FLAGS
+		;
+
+	pinfo->frame_qlen = mpath->frame_queue_len;
+	pinfo->sn = mpath->sn;
+	pinfo->metric = mpath->metric;
+	if (rtw_time_after(mpath->exp_time, rtw_get_current_time()))
+		pinfo->exptime = rtw_get_remaining_time_ms(mpath->exp_time);
+	pinfo->discovery_timeout = rtw_systime_to_ms(mpath->discovery_timeout);
+	pinfo->discovery_retries = mpath->discovery_retries;
+	if (mpath->flags & RTW_MESH_PATH_ACTIVE)
+		pinfo->flags |= NL80211_MPATH_FLAG_ACTIVE;
+	if (mpath->flags & RTW_MESH_PATH_RESOLVING)
+		pinfo->flags |= NL80211_MPATH_FLAG_RESOLVING;
+	if (mpath->flags & RTW_MESH_PATH_SN_VALID)
+		pinfo->flags |= NL80211_MPATH_FLAG_SN_VALID;
+	if (mpath->flags & RTW_MESH_PATH_FIXED)
+		pinfo->flags |= NL80211_MPATH_FLAG_FIXED;
+	if (mpath->flags & RTW_MESH_PATH_RESOLVED)
+		pinfo->flags |= NL80211_MPATH_FLAG_RESOLVED;
+}
+
+static int cfg80211_rtw_get_mpath(struct wiphy *wiphy, struct net_device *dev, u8 *dst, u8 *next_hop, struct mpath_info *pinfo)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mesh_path *mpath;
+	int ret = 0;
+
+	rtw_rcu_read_lock();
+
+	mpath = rtw_mesh_path_lookup(adapter, dst);
+	if (!mpath) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	rtw_cfg80211_mpath_set_pinfo(mpath, next_hop, pinfo);
+
+exit:
+	rtw_rcu_read_unlock();
+
+	return ret;
+}
+
+static int cfg80211_rtw_dump_mpath(struct wiphy *wiphy, struct net_device *dev, int idx, u8 *dst, u8 *next_hop, struct mpath_info *pinfo)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mesh_path *mpath;
+	int ret = 0;
+
+	rtw_rcu_read_lock();
+
+	mpath = rtw_mesh_path_lookup_by_idx(adapter, idx);
+	if (!mpath) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	_rtw_memcpy(dst, mpath->dst, ETH_ALEN);
+	rtw_cfg80211_mpath_set_pinfo(mpath, next_hop, pinfo);
+
+exit:
+	rtw_rcu_read_unlock();
+
+	return ret;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+static void rtw_cfg80211_mpp_set_pinfo(struct rtw_mesh_path *mpath, u8 *mpp, struct mpath_info *pinfo)
+{
+	_rtw_memcpy(mpp, mpath->mpp, ETH_ALEN);
+
+	_rtw_memset(pinfo, 0, sizeof(*pinfo));
+	pinfo->generation = mpath->adapter->mesh_info.mpp_paths_generation;
+}
+
+static int cfg80211_rtw_get_mpp(struct wiphy *wiphy, struct net_device *dev, u8 *dst, u8 *mpp, struct mpath_info *pinfo)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mesh_path *mpath;
+	int ret = 0;
+
+	rtw_rcu_read_lock();
+
+	mpath = rtw_mpp_path_lookup(adapter, dst);
+	if (!mpath) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	rtw_cfg80211_mpp_set_pinfo(mpath, mpp, pinfo);
+
+exit:
+	rtw_rcu_read_unlock();
+
+	return ret;
+}
+
+static int cfg80211_rtw_dump_mpp(struct wiphy *wiphy, struct net_device *dev, int idx, u8 *dst, u8 *mpp, struct mpath_info *pinfo)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mesh_path *mpath;
+	int ret = 0;
+
+	rtw_rcu_read_lock();
+
+	mpath = rtw_mpp_path_lookup_by_idx(adapter, idx);
+	if (!mpath) {
+		ret = -ENOENT;
+		goto exit;
+	}
+
+	_rtw_memcpy(dst, mpath->dst, ETH_ALEN);
+	rtw_cfg80211_mpp_set_pinfo(mpath, mpp, pinfo);
+
+exit:
+	rtw_rcu_read_unlock();
+
+	return ret;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) */
+
+#endif /* defined(CONFIG_RTW_MESH) */
+
+#if defined(CONFIG_PNO_SUPPORT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+static int cfg80211_rtw_sched_scan_start(struct wiphy *wiphy,
+		struct net_device *dev,
+		struct cfg80211_sched_scan_request *request)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct cfg80211_ssid ssids[MAX_NLO_NUM];
+	int n_ssids = 0;
+	u32 interval = 0;
+	u32 iterations = 0;
+	u32 slow_interval = 0;
+	u32 delay = 0;
+	int i = 0;
+	u8 ret = 0;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+	if (padapter->netif_up == _FALSE) {
+		RTW_INFO("%s: net device is down.\n", __func__);
+		ret = -EIO;
+		goto exit;
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE ||
+		check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE  ||
+		check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE) {
+		RTW_INFO("%s: device is busy.\n", __func__);
+		rtw_scan_abort(padapter, 0);
+	}
+
+	if (request == NULL) {
+		RTW_INFO("%s: invalid cfg80211_requests parameters.\n", __func__);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
+	/* Delay in seconds to use before starting the first scan */
+	delay = request->delay ? request->delay : NLO_DEFAULT_SCAN_DELAY;
+#endif
+
+	/* Prepare scan interval (in second) */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)
+	interval = request->scan_plans[0].interval;
+	if (request->scan_plans[0].iterations) {
+		iterations = request->scan_plans[0].iterations;
+		if (request->n_scan_plans == MAX_NLO_SCAN_PLANS)
+			slow_interval = request->scan_plans[1].interval;
+		else
+			slow_interval = interval;
+	} else {
+		iterations = 1;
+		slow_interval = interval;
+	}
+#else
+	interval = request->interval;
+	iterations = 1;
+	slow_interval = interval;
+#endif
+
+	/* Prepare SSIDs to be scanned */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0)
+	n_ssids = rtw_min(request->n_match_sets, MAX_NLO_NUM);
+	for (i = 0; i < n_ssids; i++) {
+		_rtw_memcpy((void *)&ssids[i],
+			    (void *)&request->match_sets[i].ssid,
+			    sizeof(struct cfg80211_ssid));
+	}
+#else
+	n_ssids = rtw_min(request->n_ssids, MAX_NLO_NUM);
+	for (i = 0; i < n_ssids; i++) {
+		_rtw_memcpy((void *)&ssids[i],
+			    (void *)&request->ssids[i].ssid,
+			    sizeof(struct cfg80211_ssid));
+	}
+#endif
+
+	ret = rtw_nlo_enable(dev, ssids, n_ssids,
+			     request->channels, request->n_channels, delay,
+			     interval, iterations, slow_interval);
+
+exit:
+	return ret;
+}
+
+static int cfg80211_rtw_sched_scan_stop(struct wiphy *wiphy,
+		struct net_device *dev
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) || defined (COMPAT_KERNEL_RELEASE_4_19)
+		, u64 reqid
+#endif
+)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+	return rtw_nlo_disable(dev);
+}
+
+int	cfg80211_rtw_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wow) {
+	_adapter *padapter = wiphy_to_adapter(wiphy);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+	RTW_DBG("==> %s\n",__func__);
+
+	if (wowpriv->wow_nlo.nlo_en &&
+	    !check_fwstate(pmlmepriv, WIFI_ASOC_STATE))
+		pwrpriv->wowlan_pno_enable = _TRUE;
+
+	RTW_DBG("<== %s\n",__func__);
+	return 0;
+}
+
+int	cfg80211_rtw_resume(struct wiphy *wiphy) {
+
+	_adapter *padapter = wiphy_to_adapter(wiphy);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_nlo_info *wow_nlo = &wowpriv->wow_nlo;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sitesurvey_parm parm;
+	int i, len;
+
+
+	RTW_DBG("==> %s\n",__func__);
+	if (wowpriv->wow_wake_reason == RTW_MAC_WOW_NLO_SSID_MACH) {
+
+		struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+		int PNOWakeupScanWaitCnt = 0;
+
+		rtw_cfg80211_disconnected(padapter->rtw_wdev, 0, NULL, 0, 1, GFP_ATOMIC);
+
+		rtw_init_sitesurvey_parm(padapter, &parm);
+		for (i=0;i<wow_nlo->num_of_networks && i < RTW_SSID_SCAN_AMOUNT; i++) {
+			len = wow_nlo->ssidlen[i];
+			_rtw_memcpy(&parm.ssid[i].Ssid, wow_nlo->ssid[i], len);
+			parm.ssid[i].SsidLength = len;
+		}
+		parm.ssid_num = i;
+
+		_rtw_spinlock_bh(&pmlmepriv->lock);
+		//This modification fix PNO wakeup reconnect issue with hidden SSID AP.
+		//rtw_sitesurvey_cmd(padapter, NULL);
+		rtw_sitesurvey_cmd(padapter, &parm);
+		_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+		for (PNOWakeupScanWaitCnt = 0; PNOWakeupScanWaitCnt < 10; PNOWakeupScanWaitCnt++) {
+			if(check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _FALSE)
+				break;
+			rtw_msleep_os(1000);
+		}
+
+		_rtw_spinlock_bh(&pmlmepriv->lock);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)) || defined(COMPAT_KERNEL_RELEASE_4_19)
+		cfg80211_sched_scan_results(padapter->rtw_wdev->wiphy, 0);
+#else
+		cfg80211_sched_scan_results(padapter->rtw_wdev->wiphy);
+#endif
+		_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+		pwrpriv->wowlan_pno_enable = _FALSE;
+	}
+	RTW_DBG("<== %s\n",__func__);
+	return 0;
+}
+#endif /* CONFIG_PNO_SUPPORT */
+
+static int rtw_cfg80211_set_beacon_wpsp2pie(struct net_device *ndev, char *buf, int len)
+{
+	int ret = 0;
+	uint wps_ielen = 0;
+	u8 *wps_ie;
+	u32	p2p_ielen = 0;
+	u8 wps_oui[8] = {0x0, 0x50, 0xf2, 0x04};
+	u8 *p2p_ie;
+	u32	wfd_ielen = 0;
+	u8 *wfd_ie;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(ndev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+
+	RTW_INFO(FUNC_NDEV_FMT" ielen=%d\n", FUNC_NDEV_ARG(ndev), len);
+
+	if (len > 0) {
+		wps_ie = rtw_get_wps_ie(buf, len, NULL, &wps_ielen);
+		if (wps_ie) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("bcn_wps_ielen=%d\n", wps_ielen);
+			#endif
+
+			if (pmlmepriv->wps_beacon_ie) {
+				u32 free_len = pmlmepriv->wps_beacon_ie_len;
+				pmlmepriv->wps_beacon_ie_len = 0;
+				rtw_mfree(pmlmepriv->wps_beacon_ie, free_len);
+				pmlmepriv->wps_beacon_ie = NULL;
+			}
+
+			pmlmepriv->wps_beacon_ie = rtw_malloc(wps_ielen);
+			if (pmlmepriv->wps_beacon_ie == NULL) {
+				RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				return -EINVAL;
+
+			}
+
+			_rtw_memcpy(pmlmepriv->wps_beacon_ie, wps_ie, wps_ielen);
+			pmlmepriv->wps_beacon_ie_len = wps_ielen;
+
+			rtw_update_beacon(padapter, padapter_link,
+					_VENDOR_SPECIFIC_IE_, wps_oui, _TRUE, RTW_CMDF_WAIT_ACK);
+
+		}
+
+		/* buf += wps_ielen; */
+		/* len -= wps_ielen; */
+
+		#ifdef CONFIG_P2P
+		p2p_ie = rtw_get_p2p_ie(buf, len, NULL, &p2p_ielen);
+		if (p2p_ie) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("bcn_p2p_ielen=%d\n", p2p_ielen);
+			#endif
+
+			if (pmlmepriv->p2p_beacon_ie) {
+				u32 free_len = pmlmepriv->p2p_beacon_ie_len;
+				pmlmepriv->p2p_beacon_ie_len = 0;
+				rtw_mfree(pmlmepriv->p2p_beacon_ie, free_len);
+				pmlmepriv->p2p_beacon_ie = NULL;
+			}
+
+			pmlmepriv->p2p_beacon_ie = rtw_malloc(p2p_ielen);
+			if (pmlmepriv->p2p_beacon_ie == NULL) {
+				RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				return -EINVAL;
+
+			}
+
+			_rtw_memcpy(pmlmepriv->p2p_beacon_ie, p2p_ie, p2p_ielen);
+			pmlmepriv->p2p_beacon_ie_len = p2p_ielen;
+
+		}
+		#endif /* CONFIG_P2P */
+
+
+		#ifdef CONFIG_WFD
+		wfd_ie = rtw_get_wfd_ie(buf, len, NULL, &wfd_ielen);
+		if (wfd_ie) {
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("bcn_wfd_ielen=%d\n", wfd_ielen);
+			#endif
+
+			if (rtw_mlme_update_wfd_ie_data(pmlmepriv, MLME_BEACON_IE, wfd_ie, wfd_ielen) != _SUCCESS)
+				return -EINVAL;
+		}
+		#endif /* CONFIG_WFD */
+
+		pmlmeext->bstart_bss = _TRUE;
+
+	}
+
+	return ret;
+
+}
+
+static int rtw_cfg80211_set_probe_resp_wpsp2pie(struct net_device *net, char *buf, int len)
+{
+	int ret = 0;
+	uint wps_ielen = 0;
+	u8 *wps_ie;
+	u32	p2p_ielen = 0;
+	u8 *p2p_ie;
+	u32 vendor_ielen = 0;
+	u8 *vendor_ie;
+	u32	wfd_ielen = 0;
+	u8 *wfd_ie;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("%s, ielen=%d\n", __func__, len);
+#endif
+
+	if (len > 0) {
+		wps_ie = rtw_get_wps_ie(buf, len, NULL, &wps_ielen);
+		if (wps_ie) {
+			uint	attr_contentlen = 0;
+			u16	uconfig_method, *puconfig_method = NULL;
+
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("probe_resp_wps_ielen=%d\n", wps_ielen);
+			#endif
+
+			if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+				u8 sr = 0;
+				rtw_get_wps_attr_content(wps_ie,  wps_ielen, WPS_ATTR_SELECTED_REGISTRAR, (u8 *)(&sr), NULL);
+
+				if (sr != 0)
+					RTW_INFO("%s, got sr\n", __func__);
+				else {
+					RTW_INFO("GO mode process WPS under site-survey,  sr no set\n");
+					return ret;
+				}
+			}
+
+			if (pmlmepriv->wps_probe_resp_ie) {
+				u32 free_len = pmlmepriv->wps_probe_resp_ie_len;
+				pmlmepriv->wps_probe_resp_ie_len = 0;
+				rtw_mfree(pmlmepriv->wps_probe_resp_ie, free_len);
+				pmlmepriv->wps_probe_resp_ie = NULL;
+			}
+
+			pmlmepriv->wps_probe_resp_ie = rtw_malloc(wps_ielen);
+			if (pmlmepriv->wps_probe_resp_ie == NULL) {
+				RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				return -EINVAL;
+
+			}
+
+			/* add PUSH_BUTTON config_method by driver self in wpsie of probe_resp at GO Mode */
+			puconfig_method = (u16 *)rtw_get_wps_attr_content(wps_ie, wps_ielen, WPS_ATTR_CONF_METHOD , NULL, &attr_contentlen);
+			if (puconfig_method != NULL) {
+				/* struct registry_priv *pregistrypriv = &padapter->registrypriv; */
+				struct wireless_dev *wdev = padapter->rtw_wdev;
+
+				#ifdef CONFIG_DEBUG_CFG80211
+				/* printk("config_method in wpsie of probe_resp = 0x%x\n", be16_to_cpu(*puconfig_method)); */
+				#endif
+
+				#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+				/* for WIFI-DIRECT LOGO 4.2.2, AUTO GO can't set PUSH_BUTTON flags */
+				if (wdev->iftype == NL80211_IFTYPE_P2P_GO) {
+					uconfig_method = WPS_CM_PUSH_BUTTON;
+					uconfig_method = cpu_to_be16(uconfig_method);
+
+					*puconfig_method &= ~uconfig_method;
+				}
+				#endif
+			}
+
+			_rtw_memcpy(pmlmepriv->wps_probe_resp_ie, wps_ie, wps_ielen);
+			pmlmepriv->wps_probe_resp_ie_len = wps_ielen;
+
+		}
+
+		/* buf += wps_ielen; */
+		/* len -= wps_ielen; */
+
+		vendor_ie = rtw_get_vendor_ie(buf, len, NULL, &vendor_ielen);
+		if (vendor_ie) {
+			pmlmepriv->probe_rsp = rtw_malloc(vendor_ielen);
+			if (pmlmepriv->probe_rsp == NULL) {
+				RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				return -EINVAL;
+			}
+			_rtw_memcpy(pmlmepriv->probe_rsp, vendor_ie, vendor_ielen);
+			pmlmepriv->probe_rsp_len = vendor_ielen;
+		}
+
+		#ifdef CONFIG_P2P
+		p2p_ie = rtw_get_p2p_ie(buf, len, NULL, &p2p_ielen);
+		if (p2p_ie) {
+			u8 is_GO = _FALSE;
+			u32 attr_contentlen = 0;
+			u16 cap_attr = 0;
+
+			#ifdef CONFIG_DEBUG_CFG80211
+			RTW_INFO("probe_resp_p2p_ielen=%d\n", p2p_ielen);
+			#endif
+
+			/* Check P2P Capability ATTR */
+			attr_contentlen = sizeof(cap_attr);
+			if (rtw_get_p2p_attr_content(p2p_ie, p2p_ielen, P2P_ATTR_CAPABILITY, (u8 *)&cap_attr, (uint *) &attr_contentlen)) {
+				u8 grp_cap = 0;
+				/* RTW_INFO( "[%s] Got P2P Capability Attr!!\n", __FUNCTION__ ); */
+				cap_attr = le16_to_cpu(cap_attr);
+				grp_cap = (u8)((cap_attr >> 8) & 0xff);
+
+				is_GO = (grp_cap & BIT(0)) ? _TRUE : _FALSE;
+
+				if (is_GO)
+					RTW_INFO("Got P2P Capability Attr, grp_cap=0x%x, is_GO\n", grp_cap);
+			}
+
+
+			if (is_GO == _FALSE) {
+				if (pmlmepriv->p2p_probe_resp_ie) {
+					u32 free_len = pmlmepriv->p2p_probe_resp_ie_len;
+					pmlmepriv->p2p_probe_resp_ie_len = 0;
+					rtw_mfree(pmlmepriv->p2p_probe_resp_ie, free_len);
+					pmlmepriv->p2p_probe_resp_ie = NULL;
+				}
+
+				pmlmepriv->p2p_probe_resp_ie = rtw_malloc(p2p_ielen);
+				if (pmlmepriv->p2p_probe_resp_ie == NULL) {
+					RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+					return -EINVAL;
+
+				}
+				_rtw_memcpy(pmlmepriv->p2p_probe_resp_ie, p2p_ie, p2p_ielen);
+				pmlmepriv->p2p_probe_resp_ie_len = p2p_ielen;
+			} else {
+				if (pmlmepriv->p2p_go_probe_resp_ie) {
+					u32 free_len = pmlmepriv->p2p_go_probe_resp_ie_len;
+					pmlmepriv->p2p_go_probe_resp_ie_len = 0;
+					rtw_mfree(pmlmepriv->p2p_go_probe_resp_ie, free_len);
+					pmlmepriv->p2p_go_probe_resp_ie = NULL;
+				}
+
+				pmlmepriv->p2p_go_probe_resp_ie = rtw_malloc(p2p_ielen);
+				if (pmlmepriv->p2p_go_probe_resp_ie == NULL) {
+					RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+					return -EINVAL;
+
+				}
+				_rtw_memcpy(pmlmepriv->p2p_go_probe_resp_ie, p2p_ie, p2p_ielen);
+				pmlmepriv->p2p_go_probe_resp_ie_len = p2p_ielen;
+			}
+
+		}
+		#endif /* CONFIG_P2P */
+
+
+		#ifdef CONFIG_WFD
+		wfd_ie = rtw_get_wfd_ie(buf, len, NULL, &wfd_ielen);
+		#ifdef CONFIG_DEBUG_CFG80211
+		RTW_INFO("probe_resp_wfd_ielen=%d\n", wfd_ielen);
+		#endif
+
+		if (rtw_mlme_update_wfd_ie_data(pmlmepriv, MLME_PROBE_RESP_IE, wfd_ie, wfd_ielen) != _SUCCESS)
+			return -EINVAL;
+		#endif /* CONFIG_WFD */
+
+	}
+
+	return ret;
+
+}
+
+static int rtw_cfg80211_set_assoc_resp_wpsp2pie(struct net_device *net, char *buf, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u8 *ie;
+	u32 ie_len;
+
+	RTW_INFO("%s, ielen=%d\n", __func__, len);
+
+	if (len <= 0)
+		goto exit;
+
+	ie = rtw_get_wps_ie(buf, len, NULL, &ie_len);
+	if (ie && ie_len) {
+		if (pmlmepriv->wps_assoc_resp_ie) {
+			u32 free_len = pmlmepriv->wps_assoc_resp_ie_len;
+
+			pmlmepriv->wps_assoc_resp_ie_len = 0;
+			rtw_mfree(pmlmepriv->wps_assoc_resp_ie, free_len);
+			pmlmepriv->wps_assoc_resp_ie = NULL;
+		}
+
+		pmlmepriv->wps_assoc_resp_ie = rtw_malloc(ie_len);
+		if (pmlmepriv->wps_assoc_resp_ie == NULL) {
+			RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+			return -EINVAL;
+		}
+		_rtw_memcpy(pmlmepriv->wps_assoc_resp_ie, ie, ie_len);
+		pmlmepriv->wps_assoc_resp_ie_len = ie_len;
+	}
+
+	ie = rtw_get_p2p_ie(buf, len, NULL, &ie_len);
+	if (ie && ie_len) {
+		if (pmlmepriv->p2p_assoc_resp_ie) {
+			u32 free_len = pmlmepriv->p2p_assoc_resp_ie_len;
+
+			pmlmepriv->p2p_assoc_resp_ie_len = 0;
+			rtw_mfree(pmlmepriv->p2p_assoc_resp_ie, free_len);
+			pmlmepriv->p2p_assoc_resp_ie = NULL;
+		}
+
+		pmlmepriv->p2p_assoc_resp_ie = rtw_malloc(ie_len);
+		if (pmlmepriv->p2p_assoc_resp_ie == NULL) {
+			RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+			return -EINVAL;
+		}
+		_rtw_memcpy(pmlmepriv->p2p_assoc_resp_ie, ie, ie_len);
+		pmlmepriv->p2p_assoc_resp_ie_len = ie_len;
+	}
+
+#ifdef CONFIG_WFD
+	ie = rtw_get_wfd_ie(buf, len, NULL, &ie_len);
+	if (rtw_mlme_update_wfd_ie_data(pmlmepriv, MLME_ASSOC_RESP_IE, ie, ie_len) != _SUCCESS)
+		return -EINVAL;
+#endif
+
+exit:
+	return ret;
+}
+
+int rtw_cfg80211_set_mgnt_wpsp2pie(struct net_device *net, char *buf, int len,
+	int type)
+{
+	int ret = 0;
+	uint wps_ielen = 0;
+	u32	p2p_ielen = 0;
+
+#ifdef CONFIG_DEBUG_CFG80211
+	RTW_INFO("%s, ielen=%d\n", __func__, len);
+#endif
+
+	if ((rtw_get_wps_ie(buf, len, NULL, &wps_ielen) && (wps_ielen > 0))
+		#ifdef CONFIG_P2P
+		|| (rtw_get_p2p_ie(buf, len, NULL, &p2p_ielen) && (p2p_ielen > 0))
+		#endif
+	) {
+		if (net != NULL) {
+			switch (type) {
+			case 0x1: /* BEACON */
+				ret = rtw_cfg80211_set_beacon_wpsp2pie(net, buf, len);
+				break;
+			case 0x2: /* PROBE_RESP */
+				ret = rtw_cfg80211_set_probe_resp_wpsp2pie(net, buf, len);
+				#ifdef CONFIG_P2P
+				if (ret == 0)
+					adapter_wdev_data((_adapter *)rtw_netdev_priv(net))->probe_resp_ie_update_time = rtw_get_current_time();
+				#endif
+				break;
+			case 0x4: /* ASSOC_RESP */
+				ret = rtw_cfg80211_set_assoc_resp_wpsp2pie(net, buf, len);
+				break;
+			}
+		}
+	}
+
+	return ret;
+
+}
+
+#ifdef CONFIG_80211N_HT
+static void rtw_cfg80211_init_ht_capab_ex(_adapter *padapter,
+	struct ieee80211_sta_ht_cap *ht_cap, enum band_type band, u8 rf_type,
+	struct protocol_cap_t *dft_proto_cap,
+	struct role_link_cap_t *dft_cap)
+{
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct ht_priv		*phtpriv = &pmlmepriv->dev_htpriv;
+	u8 stbc_rx_enable = _FALSE;
+
+	rtw_ht_get_dft_setting(padapter, dft_proto_cap, dft_cap);
+
+	/* RX LDPC */
+	if (TEST_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_RX))
+		ht_cap->cap |= IEEE80211_HT_CAP_LDPC_CODING;
+
+	/* TX STBC */
+	if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX))
+		ht_cap->cap |= IEEE80211_HT_CAP_TX_STBC;
+
+	/* RX STBC */
+	if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX)) {
+		switch (rf_type) {
+		case RF_1T1R:
+			ht_cap->cap |= IEEE80211_HT_CAP_RX_STBC_1R;/*RX STBC One spatial stream*/
+			break;
+		case RF_2T2R:
+		case RF_1T2R:
+			ht_cap->cap |= IEEE80211_HT_CAP_RX_STBC_1R;/* Only one spatial-stream STBC RX is supported */
+			break;
+		case RF_3T3R:
+		case RF_3T4R:
+		case RF_4T4R:
+			ht_cap->cap |= IEEE80211_HT_CAP_RX_STBC_1R;/* Only one spatial-stream STBC RX is supported */
+			break;
+		default:
+			RTW_INFO("[warning] rf_type %d is not expected\n", rf_type);
+			break;
+		}
+	}
+}
+
+static void rtw_cfg80211_init_ht_capab(_adapter *padapter,
+	struct ieee80211_sta_ht_cap *ht_cap, enum band_type band, u8 rf_type,
+	struct protocol_cap_t *dft_proto_cap,
+	struct role_link_cap_t *dft_cap)
+{
+	struct registry_priv *regsty = &padapter->registrypriv;
+	u8 rx_nss = 0;
+
+	if (!regsty->ht_enable || !is_supported_ht(regsty->wireless_mode))
+		return;
+
+	ht_cap->ht_supported = 1;
+
+	ht_cap->cap = IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
+				IEEE80211_HT_CAP_SGI_40 | IEEE80211_HT_CAP_SGI_20 |
+				IEEE80211_HT_CAP_DSSSCCK40 | IEEE80211_HT_CAP_MAX_AMSDU;
+	rtw_cfg80211_init_ht_capab_ex(padapter, ht_cap, band, rf_type,
+							dft_proto_cap, dft_cap);
+
+	/*
+	 *Maximum length of AMPDU that the STA can receive.
+	 *Length = 2 ^ (13 + max_ampdu_length_exp) - 1 (octets)
+	 */
+	ht_cap->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K;
+
+	/*Minimum MPDU start spacing , */
+	ht_cap->ampdu_density = IEEE80211_HT_MPDU_DENSITY_16;
+
+	ht_cap->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED;
+
+	rx_nss = GET_PHY_RX_NSS_BY_BAND(adapter_to_dvobj(padapter), HW_BAND_0);
+	switch (rx_nss) {
+	case 1:
+		ht_cap->mcs.rx_mask[0] = 0xFF;
+		break;
+	case 2:
+		ht_cap->mcs.rx_mask[0] = 0xFF;
+		ht_cap->mcs.rx_mask[1] = 0xFF;
+		break;
+	case 3:
+		ht_cap->mcs.rx_mask[0] = 0xFF;
+		ht_cap->mcs.rx_mask[1] = 0xFF;
+		ht_cap->mcs.rx_mask[2] = 0xFF;
+		break;
+	case 4:
+		ht_cap->mcs.rx_mask[0] = 0xFF;
+		ht_cap->mcs.rx_mask[1] = 0xFF;
+		ht_cap->mcs.rx_mask[2] = 0xFF;
+		ht_cap->mcs.rx_mask[3] = 0xFF;
+		break;
+	default:
+		RTW_ERR("%s, error rf_type=%d, rx_nss=%d\n", __func__, rf_type, rx_nss);
+		rtw_warn_on(1);
+	};
+
+	ht_cap->mcs.rx_highest = cpu_to_le16(
+		rtw_ht_mcs_rate(rtw_hw_is_bw_support(adapter_to_dvobj(padapter), CHANNEL_WIDTH_40)
+			, rtw_hw_is_bw_support(adapter_to_dvobj(padapter), CHANNEL_WIDTH_40) ? ht_cap->cap & IEEE80211_HT_CAP_SGI_40 : ht_cap->cap & IEEE80211_HT_CAP_SGI_20
+			, ht_cap->mcs.rx_mask) / 10);
+}
+#endif /* CONFIG_80211N_HT */
+
+#if defined(CONFIG_80211AC_VHT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+static void rtw_cfg80211_init_vht_capab(_adapter *padapter,
+	struct ieee80211_sta_vht_cap *sta_vht_cap,
+	enum band_type band, u8 rf_type, struct protocol_cap_t *dft_proto_cap,
+	struct role_link_cap_t *dft_cap)
+{
+	struct registry_priv *regsty = &padapter->registrypriv;
+	u8 vht_cap_ie[2 + 12] = {0};
+
+	if (!REGSTY_IS_11AC_ENABLE(regsty) || !is_supported_vht(regsty->wireless_mode))
+		return;
+
+	rtw_vht_get_dft_setting(padapter, dft_proto_cap, dft_cap);
+
+	rtw_get_dft_vht_cap_ie(padapter, vht_cap_ie);
+
+	sta_vht_cap->vht_supported = 1;
+
+	_rtw_memcpy(&sta_vht_cap->cap, vht_cap_ie + 2, 4);
+	_rtw_memcpy(&sta_vht_cap->vht_mcs, vht_cap_ie + 2 + 4, 8);
+}
+#endif /* defined(CONFIG_80211AC_VHT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0)) */
+
+#if defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)))
+static int rtw_cfg80211_init_he_capab(_adapter *padapter,
+	struct ieee80211_sband_iftype_data *sta_iface_data,
+	enum nl80211_iftype iftype, struct phy_cap_t *phy_cap,
+	struct protocol_cap_t *dft_proto_cap, enum nl80211_band band)
+{
+	struct ieee80211_sta_he_cap *sta_he_cap = &(sta_iface_data->he_cap);
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	struct registry_priv *regsty = &padapter->registrypriv;
+	u8 cap_len = 0;
+	u8 he_mcs_set_ext_len = 0;
+	u8 he_cap_ie[HE_CAP_ELE_MAX_LEN] = {0};
+#if defined(CONFIG_IEEE80211_BAND_6GHZ) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	u8 he_6g_band_cap_ie[HE_6G_BAND_CAP_MAX_LEN] = {0};
+	enum role_type role_type;
+#endif /* CONFIG_IEEE80211_BAND_6GHZ && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)) */
+	u8 ofst_80m = 3 + HE_CAP_ELE_MAC_CAP_LEN + HE_CAP_ELE_PHY_CAP_LEN;
+	u8 ofst_160m = ofst_80m + 4;
+	u8 ofst_80p80m = ofst_80m + 4;
+
+	if (!REGSTY_IS_11AX_ENABLE(regsty) || !is_supported_he(regsty->wireless_mode))
+		return _FAIL;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	/* forbid before 6G AP regulatory ready */
+	if (rtw_txpwr_hal_is_txpwr_limit_needed(adapter_to_dvobj(padapter))
+		&& band == NL80211_BAND_6GHZ && iftype == NL80211_IFTYPE_AP)
+		return _FAIL;
+#endif
+
+	sta_iface_data->types_mask = BIT(iftype);
+
+	cap_len = rtw_get_dft_he_cap_ie(padapter, phy_cap, dft_proto_cap, he_cap_ie, nl80211_band_to_rtw_band(band));
+
+	sta_he_cap->has_he = 1;
+
+	/* mac & phy capability info */
+	_rtw_memcpy(&sta_he_cap->he_cap_elem.mac_cap_info, he_cap_ie + 3, HE_CAP_ELE_MAC_CAP_LEN);
+	_rtw_memcpy(&sta_he_cap->he_cap_elem.phy_cap_info,
+	            he_cap_ie + 3 + HE_CAP_ELE_MAC_CAP_LEN,
+				HE_CAP_ELE_PHY_CAP_LEN);
+
+#if defined(CONFIG_IEEE80211_BAND_6GHZ) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0))
+	/* 6G band cap */
+	role_type = nl80211_iftype_to_rtw_phl_role_type(iftype);
+
+	cap_len = rtw_build_he_6g_band_cap_ie_by_proto(padapter, role_type,
+						       dft_proto_cap, he_6g_band_cap_ie);
+
+	_rtw_memcpy(&sta_iface_data->he_6ghz_capa,
+	            he_6g_band_cap_ie + 1, HE_6G_BAND_CAP_INFO_LEN);
+#endif /* CONFIG_IEEE80211_BAND_6GHZ && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)) */
+
+	/* Supported HE-MCS And NSS Set */
+	/* HE supports 80M BW */
+	_rtw_memcpy(&sta_he_cap->he_mcs_nss_supp.rx_mcs_80, he_cap_ie + ofst_80m, 2);
+	_rtw_memcpy(&sta_he_cap->he_mcs_nss_supp.tx_mcs_80, he_cap_ie + ofst_80m + 2, 2);
+
+	/* HE supports 160M BW */
+	if(GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(he_cap_ie + 3 + HE_CAP_ELE_MAC_CAP_LEN) & BIT2) {
+		_rtw_memcpy(&sta_he_cap->he_mcs_nss_supp.rx_mcs_160, he_cap_ie + ofst_160m, 2);
+		_rtw_memcpy(&sta_he_cap->he_mcs_nss_supp.tx_mcs_160, he_cap_ie + ofst_160m + 2, 2);
+		ofst_80p80m += 4;
+	} else {
+		_rtw_memset(&sta_he_cap->he_mcs_nss_supp.rx_mcs_160, HE_MSC_NOT_SUPP_BYTE, 2);
+		_rtw_memset(&sta_he_cap->he_mcs_nss_supp.tx_mcs_160, HE_MSC_NOT_SUPP_BYTE, 2);
+	}
+
+	/* HE supports 80M+80M BW */
+	if(GET_HE_PHY_CAP_SUPPORT_CHAN_WIDTH_SET(he_cap_ie + 3 + HE_CAP_ELE_MAC_CAP_LEN) & BIT3) {
+		_rtw_memcpy(&sta_he_cap->he_mcs_nss_supp.rx_mcs_80p80, he_cap_ie + ofst_80p80m, 2);
+		_rtw_memcpy(&sta_he_cap->he_mcs_nss_supp.tx_mcs_80p80, he_cap_ie + ofst_80p80m + 2, 2);
+	} else {
+		_rtw_memset(&sta_he_cap->he_mcs_nss_supp.rx_mcs_80p80, HE_MSC_NOT_SUPP_BYTE, 2);
+		_rtw_memset(&sta_he_cap->he_mcs_nss_supp.tx_mcs_80p80, HE_MSC_NOT_SUPP_BYTE, 2);
+	}
+
+	return _SUCCESS;
+}
+
+
+static void rtw_cfg80211_init_sband_iftype_data(_adapter *padapter,
+	struct ieee80211_supported_band *band, struct phy_cap_t *phy_cap,
+		struct role_link_cap_t *dft_cap,
+		struct protocol_cap_t *dft_sta_proto_cap,
+		struct protocol_cap_t *dft_ap_proto_cap)
+{
+	struct ieee80211_sband_iftype_data *he_iftype = NULL;
+	void *phl = GET_PHL_INFO(adapter_to_dvobj(padapter));
+	int ret = _FAIL;
+
+	if (!rtw_hw_is_wireless_mode_support(adapter_to_dvobj(padapter),
+					    WLAN_MD_11AX))
+		return;
+
+	he_iftype = (struct ieee80211_sband_iftype_data *)(((u8 *)band->bitrates)
+			            + sizeof(struct ieee80211_rate) * band->n_bitrates);
+
+	ret = rtw_cfg80211_init_he_capab(padapter, he_iftype,
+					NL80211_IFTYPE_AP, phy_cap, dft_ap_proto_cap, band->band);
+	if (ret == _SUCCESS) {
+		band->n_iftype_data += 1;
+		he_iftype += 1;
+	}
+
+	ret = rtw_cfg80211_init_he_capab(padapter, he_iftype,
+					NL80211_IFTYPE_STATION, phy_cap, dft_sta_proto_cap, band->band);
+	if (ret == _SUCCESS)
+		band->n_iftype_data += 1;
+}
+#endif /* defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)) */
+
+static int rtw_cfg80211_init_wiphy_band(_adapter *padapter, struct wiphy *wiphy)
+{
+	u8 rf_type;
+	struct ieee80211_supported_band *band;
+	int ret = _FAIL;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	struct protocol_cap_t dft_sta_proto_cap = {0};
+	struct protocol_cap_t dft_ap_proto_cap = {0};
+	struct role_link_cap_t dft_cap = {0};
+	struct phy_cap_t *phy_cap;
+	u8 hw_band = HW_BAND_0;
+
+	/*init wiphy0 for band0*/
+	rtw_phl_get_dft_cap(dvobj->phl, hw_band, &dft_cap);
+
+	rtw_phl_get_dft_proto_cap(dvobj->phl, hw_band,
+			PHL_RTYPE_STATION, &dft_sta_proto_cap);
+	rtw_phl_get_dft_proto_cap(dvobj->phl, hw_band,
+				PHL_RTYPE_AP, &dft_ap_proto_cap);
+	phy_cap = &(dvobj->phl_com->phy_cap[hw_band]);
+
+	/*TODO init wiphy1 for band1*/
+	#if 0 /*#ifdef CONFIG_DBCC_SUPPORT*/
+	/*if (dvobj->phl_com->dev_cap.hw_sup_flags & HW_SUP_DBCC)*/
+	if (dvobj->phl_com->dev_cap.dbcc_sup) {
+		hw_band = HW_BAND_1;
+		rtw_phl_get_dft_cap(dvobj->phl, hw_band, &dft_cap);
+
+		rtw_phl_get_dft_proto_cap(dvobj->phl, hw_band,
+			PHL_RTYPE_STATION, &dft_sta_proto_cap);
+		rtw_phl_get_dft_proto_cap(dvobj->phl, hw_band,
+				PHL_RTYPE_AP, &dft_ap_proto_cap);
+		phy_cap = &(dvobj->phl_com->phy_cap[hw_band]);
+	}
+	#endif
+
+	rf_type = GET_HAL_RFPATH(dvobj);
+	RTW_INFO("%s:rf_type=%d\n", __func__, rf_type);
+
+
+	if (is_supported_24g(padapter->registrypriv.band_type) &&
+	    (rtw_hw_is_band_support(dvobj, BAND_ON_24G))) {
+		band = wiphy->bands[NL80211_BAND_2GHZ] = rtw_spt_band_alloc(BAND_ON_24G);
+		if (!band)
+			goto err;
+
+		rtw_2g_channels_init(band->channels);
+		rtw_2g_rates_init(band->bitrates);
+		#if defined(CONFIG_80211N_HT)
+		rtw_cfg80211_init_ht_capab(padapter, &band->ht_cap,
+					BAND_ON_24G, rf_type, &dft_sta_proto_cap, &dft_cap);
+		#endif
+		#if defined(CONFIG_80211AX_HE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0))
+		rtw_cfg80211_init_sband_iftype_data(padapter, band, phy_cap,
+			&dft_cap, &dft_sta_proto_cap, &dft_ap_proto_cap);
+		#endif /* defined(CONFIG_80211AX_HE) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) */
+	}
+
+	#if CONFIG_IEEE80211_BAND_5GHZ
+	if (is_supported_5g(padapter->registrypriv.band_type) &&
+	    (rtw_hw_is_band_support(dvobj, BAND_ON_5G))) {
+		band = wiphy->bands[NL80211_BAND_5GHZ] = rtw_spt_band_alloc(BAND_ON_5G);
+		if (!band)
+			goto err;
+
+		rtw_5g_channels_init(band->channels);
+		rtw_5g_rates_init(band->bitrates);
+		#if defined(CONFIG_80211N_HT)
+		rtw_cfg80211_init_ht_capab(padapter, &band->ht_cap,
+					BAND_ON_5G, rf_type, &dft_sta_proto_cap, &dft_cap);
+		#endif
+		#if defined(CONFIG_80211AC_VHT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+		rtw_cfg80211_init_vht_capab(padapter, &band->vht_cap,
+					BAND_ON_5G, rf_type, &dft_sta_proto_cap, &dft_cap);
+		#endif
+		#if defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)))
+		rtw_cfg80211_init_sband_iftype_data(padapter, band, phy_cap,
+			&dft_cap, &dft_sta_proto_cap, &dft_ap_proto_cap);
+		#endif /* defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)) */
+	}
+	#endif
+
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	if (is_supported_6g(padapter->registrypriv.band_type) &&
+	    (rtw_hw_is_band_support(dvobj, BAND_ON_6G))) {
+		band = wiphy->bands[NL80211_BAND_6GHZ] = rtw_spt_band_alloc(BAND_ON_6G);
+		if (!band)
+			goto err;
+
+		rtw_6g_channels_init(band->channels);
+		rtw_6g_rates_init(band->bitrates);
+
+		#if defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)))
+		rtw_cfg80211_init_sband_iftype_data(padapter, band, phy_cap,
+			&dft_cap, &dft_sta_proto_cap, &dft_ap_proto_cap);
+		#endif /* defined(CONFIG_80211AX_HE) && (defined(CPTCFG_VERSION) || LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)) */
+	}
+	#endif
+
+	return _SUCCESS;
+
+err:
+	if (wiphy->bands[NL80211_BAND_2GHZ]) {
+		rtw_spt_band_free(wiphy->bands[NL80211_BAND_2GHZ]);
+		wiphy->bands[NL80211_BAND_2GHZ] = NULL;
+	}
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (wiphy->bands[NL80211_BAND_5GHZ]) {
+		rtw_spt_band_free(wiphy->bands[NL80211_BAND_5GHZ]);
+		wiphy->bands[NL80211_BAND_5GHZ] = NULL;
+	}
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (wiphy->bands[NL80211_BAND_6GHZ]) {
+		rtw_spt_band_free(wiphy->bands[NL80211_BAND_6GHZ]);
+		wiphy->bands[NL80211_BAND_6GHZ] = NULL;
+	}
+#endif
+
+	return ret;
+}
+
+#if !defined(CONFIG_REGD_SRC_FROM_OS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
+void rtw_cfg80211_update_wiphy_max_txpower(_adapter *adapter, struct wiphy *wiphy)
+{
+	struct ieee80211_supported_band *band;
+	struct ieee80211_channel *channel;
+	s16 max_txpwr;
+	int i;
+
+	if (is_supported_24g(adapter->registrypriv.band_type)) {
+		band = wiphy->bands[NL80211_BAND_2GHZ];
+		if (band) {
+			max_txpwr = 30 * MBM_PDBM; //phy_get_txpwr_by_rate_total_max_mbm(adapter, BAND_ON_24G, 1, 1);
+			if (max_txpwr != UNSPECIFIED_MBM) {
+				for (i = 0; i < band->n_channels; i++) {
+					channel = &band->channels[i];
+					channel->max_power = max_txpwr / MBM_PDBM;
+				}
+			}
+		}
+	}
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (is_supported_5g(adapter->registrypriv.band_type)) {
+		band = wiphy->bands[NL80211_BAND_5GHZ];
+		if (band) {
+			max_txpwr = 30 * MBM_PDBM; //phy_get_txpwr_by_rate_total_max_mbm(adapter, BAND_ON_5G, 1, 1);
+			if (max_txpwr != UNSPECIFIED_MBM) {
+				for (i = 0; i < band->n_channels; i++) {
+					channel = &band->channels[i];
+					channel->max_power = max_txpwr / MBM_PDBM;
+				}
+			}
+		}
+	}
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (is_supported_6g(adapter->registrypriv.band_type)) {
+		band = wiphy->bands[NL80211_BAND_6GHZ];
+		if (band) {
+			max_txpwr = 30 * MBM_PDBM; //phy_get_txpwr_by_rate_total_max_mbm(adapter, BAND_ON_6G, 1, 1);
+			if (max_txpwr != UNSPECIFIED_MBM) {
+				for (i = 0; i < band->n_channels; i++) {
+					channel = &band->channels[i];
+					channel->max_power = max_txpwr / MBM_PDBM;
+				}
+			}
+		}
+	}
+#endif
+}
+#endif /* defined(CONFIG_REGD_SRC_FROM_OS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)) */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)) && (CONFIG_IFACE_NUMBER >= 2)
+struct ieee80211_iface_limit rtw_limits[] = {
+	{
+		.max = CONFIG_IFACE_NUMBER,
+		.types = BIT(NL80211_IFTYPE_STATION)
+			#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+			| BIT(NL80211_IFTYPE_P2P_CLIENT)
+			#endif
+	},
+	#ifdef CONFIG_AP_MODE
+	{
+		.max = rtw_min(CONFIG_IFACE_NUMBER, CONFIG_LIMITED_AP_NUM),
+		.types = BIT(NL80211_IFTYPE_AP)
+			#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+			| BIT(NL80211_IFTYPE_P2P_GO)
+			#endif
+	},
+	#endif
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_P2P_DEVICE)
+	},
+	#endif
+	#if defined(CONFIG_RTW_MESH)
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_MESH_POINT)
+	},
+	#endif
+};
+
+struct ieee80211_iface_combination rtw_combinations[] = {
+	{
+		.limits = rtw_limits,
+		.n_limits = ARRAY_SIZE(rtw_limits),
+		#if defined(RTW_DEDICATED_P2P_DEVICE)
+		.max_interfaces = CONFIG_IFACE_NUMBER + 1,
+		#else
+		.max_interfaces = CONFIG_IFACE_NUMBER,
+		#endif
+		#if defined(CONFIG_MCC_MODE) || defined(CONFIG_DBCC_SUPPORT)
+		.num_different_channels = 2,
+		#else
+		.num_different_channels = 1,
+		#endif
+
+	},
+};
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)) */
+
+static int rtw_cfg80211_init_wiphy(_adapter *adapter, struct wiphy *wiphy)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+	int ret = _FAIL;
+
+	/* copy mac_addr to wiphy */
+	_rtw_memcpy(wiphy->perm_addr, adapter_mac_addr(adapter), ETH_ALEN);
+
+	wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
+
+	wiphy->max_scan_ssids = RTW_SSID_SCAN_AMOUNT;
+	wiphy->max_scan_ie_len = RTW_SCAN_IE_LEN_MAX;
+	wiphy->max_num_pmkids = RTW_MAX_NUM_PMKIDS;
+#if defined(CONFIG_AP_MODE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0))
+	wiphy->max_ap_assoc_sta = adapter->stapriv.max_num_sta;
+#endif
+
+#if CONFIG_RTW_MACADDR_ACL && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	wiphy->max_acl_mac_addrs = NUM_ACL;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) || defined(COMPAT_KERNEL_RELEASE)
+	wiphy->max_remain_on_channel_duration = RTW_MAX_REMAIN_ON_CHANNEL_DURATION;
+#endif
+
+	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION)
+		| BIT(NL80211_IFTYPE_ADHOC)
+		#ifdef CONFIG_AP_MODE
+		| BIT(NL80211_IFTYPE_AP)
+		#ifdef CONFIG_WIFI_MONITOR
+		| BIT(NL80211_IFTYPE_MONITOR)
+		#endif
+		#endif
+		#if defined(CONFIG_P2P) && ((LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE))
+		| BIT(NL80211_IFTYPE_P2P_CLIENT)
+		| BIT(NL80211_IFTYPE_P2P_GO)
+		#if defined(RTW_DEDICATED_P2P_DEVICE)
+		| BIT(NL80211_IFTYPE_P2P_DEVICE)
+		#endif
+		#endif
+
+		#ifdef CONFIG_RTW_MESH
+		| BIT(NL80211_IFTYPE_MESH_POINT)	/* 2.6.26 */
+		#endif
+		;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+#ifdef CONFIG_AP_MODE
+	wiphy->mgmt_stypes = rtw_cfg80211_default_mgmt_stypes;
+#endif /* CONFIG_AP_MODE */
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	#ifdef CONFIG_WIFI_MONITOR
+	wiphy->software_iftypes |= BIT(NL80211_IFTYPE_MONITOR);
+	#endif
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)) && (CONFIG_IFACE_NUMBER >= 2)
+	wiphy->iface_combinations = rtw_combinations;
+	wiphy->n_iface_combinations = ARRAY_SIZE(rtw_combinations);
+#endif
+
+	wiphy->cipher_suites = rtw_cipher_suites;
+	wiphy->n_cipher_suites = ARRAY_SIZE(rtw_cipher_suites);
+
+	if (rtw_cfg80211_init_wiphy_band(adapter, wiphy) != _SUCCESS) {
+		RTW_ERR("rtw_cfg80211_init_wiphy_band fail\n");
+		goto exit;
+	}
+	#if !defined(CONFIG_REGD_SRC_FROM_OS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
+	rtw_cfg80211_update_wiphy_max_txpower(adapter, wiphy);
+	#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0))
+	wiphy->flags |= WIPHY_FLAG_SUPPORTS_SEPARATE_DEFAULT_KEYS;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL;
+	wiphy->flags |= WIPHY_FLAG_HAVE_AP_SME;
+	/* OFFCHAN_TX not ready, Mgmt tx depend on REMAIN_ON_CHANNEL */
+	/* wiphy->flags |= WIPHY_FLAG_OFFCHAN_TX; */
+#endif
+
+#if (KERNEL_VERSION(3, 2, 0) <= LINUX_VERSION_CODE)
+	wiphy->flags |= WIPHY_FLAG_AP_UAPSD;
+#endif
+
+#if defined(CONFIG_PM) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) || defined (COMPAT_KERNEL_RELEASE_4_19)
+	wiphy->max_sched_scan_reqs = 1;
+#else
+	wiphy->flags |= WIPHY_FLAG_SUPPORTS_SCHED_SCAN;
+#endif
+#ifdef CONFIG_PNO_SUPPORT
+	wiphy->max_sched_scan_ssids = MAX_NLO_NUM;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0)
+	wiphy->max_match_sets = MAX_NLO_NUM;
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)
+	wiphy->max_sched_scan_plans = MAX_NLO_SCAN_PLANS;
+	wiphy->max_sched_scan_plan_interval = MAX_NLO_SCAN_PERIOD;
+	wiphy->max_sched_scan_plan_iterations = MAX_NLO_NORMAL_SCAN_CYCLE;
+#endif
+#endif
+#endif
+
+#if defined(CONFIG_PM) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0))
+	wiphy->wowlan = wowlan_stub;
+#else
+	wiphy->wowlan = &wowlan_stub;
+#endif
+#endif
+
+#if defined(CONFIG_TDLS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS;
+#ifndef CONFIG_TDLS_DRIVER_SETUP
+	wiphy->flags |= WIPHY_FLAG_TDLS_EXTERNAL_SETUP;	/* Driver handles key exchange */
+	wiphy->flags |= NL80211_ATTR_HT_CAPABILITY;
+#endif /* CONFIG_TDLS_DRIVER_SETUP */
+#endif /* CONFIG_TDLS */
+
+#if 0 && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0))
+	wiphy->flags |= WIPHY_FLAG_SUPPORTS_FW_ROAM;
+#endif
+
+#ifdef CONFIG_RTW_WDS
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 33))
+	wiphy->flags |= WIPHY_FLAG_4ADDR_AP;
+	wiphy->flags |= WIPHY_FLAG_4ADDR_STATION;
+	#endif
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	wiphy->flags |= 0
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+		| WIPHY_FLAG_IBSS_RSN
+		#endif
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+		| WIPHY_FLAG_MESH_AUTH
+		#endif
+		;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	wiphy->features |= 0
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+		| NL80211_FEATURE_USERSPACE_MPM
+		#endif
+		;
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)) */
+#endif /* CONFIG_RTW_MESH */
+
+#if (defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)) && (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0))
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_RRM);
+#endif
+
+#if (KERNEL_VERSION(3, 8, 0) <= LINUX_VERSION_CODE)
+	wiphy->features |= NL80211_FEATURE_SAE;
+#endif
+
+#ifdef CONFIG_RTW_SCAN_RAND
+	#if (KERNEL_VERSION(3, 19, 0) <= LINUX_VERSION_CODE)
+	wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR;
+	#endif /* KERNEL_VERSION 3.19 */
+#endif /* CONFIG_RTW_SCAN_RAND */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+#ifdef CONFIG_WIFI_MONITOR
+	/* Currently only for Monitor debugging */
+	wiphy->flags |= WIPHY_FLAG_SUPPORTS_5_10_MHZ;
+#endif
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0)) */
+
+#ifdef CONFIG_ECSA_PHL
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0))
+	wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH;
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	wiphy->max_num_csa_counters = MAX_CSA_CNT;
+#endif
+#endif /* CONFIG_ECSA_PHL */
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
+	wiphy->flags |= WIPHY_FLAG_SPLIT_SCAN_6GHZ;
+#endif
+#endif
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+#ifdef CONFIG_RFKILL_POLL
+void rtw_cfg80211_init_rfkill(struct wiphy *wiphy)
+{
+	wiphy_rfkill_set_hw_state(wiphy, 0);
+	wiphy_rfkill_start_polling(wiphy);
+}
+
+void rtw_cfg80211_deinit_rfkill(struct wiphy *wiphy)
+{
+	wiphy_rfkill_stop_polling(wiphy);
+}
+
+static void cfg80211_rtw_rfkill_poll(struct wiphy *wiphy)
+{
+	_adapter *padapter = NULL;
+	bool blocked = _FALSE;
+	u8 valid = 0;
+
+	padapter = wiphy_to_adapter(wiphy);
+
+	if (adapter_to_dvobj(padapter)->processing_dev_remove == _TRUE) {
+		/*RTW_INFO("cfg80211_rtw_rfkill_poll: device is removed!\n");*/
+		return;
+	}
+
+	blocked = rtw_hal_rfkill_poll(padapter, &valid);
+	/*RTW_INFO("cfg80211_rtw_rfkill_poll: valid=%d, blocked=%d\n",
+			valid, blocked);*/
+
+	if (valid)
+		wiphy_rfkill_set_hw_state(wiphy, blocked);
+}
+#endif
+
+#if defined(CONFIG_RTW_HOSTAPD_ACS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 33))
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0))
+#define SURVEY_INFO_TIME			SURVEY_INFO_CHANNEL_TIME
+#define SURVEY_INFO_TIME_BUSY		SURVEY_INFO_CHANNEL_TIME_BUSY
+#define SURVEY_INFO_TIME_EXT_BUSY	SURVEY_INFO_CHANNEL_TIME_EXT_BUSY
+#define SURVEY_INFO_TIME_RX			SURVEY_INFO_CHANNEL_TIME_RX
+#define SURVEY_INFO_TIME_TX			SURVEY_INFO_CHANNEL_TIME_TX
+#endif
+
+#ifdef CONFIG_FIND_BEST_CHANNEL
+static void rtw_cfg80211_set_survey_info_with_find_best_channel(struct wiphy *wiphy
+	, struct net_device *netdev, int idx, struct survey_info *info)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(netdev);
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	u32 total_rx_cnt = 0;
+	int i;
+
+	s8 noise = -50;		/*channel noise in dBm. This and all following fields are optional */
+	u64 time = 100;		/*amount of time in ms the radio was turn on (on the channel)*/
+	u64 time_busy = 0;	/*amount of time the primary channel was sensed busy*/
+
+	info->filled  = SURVEY_INFO_NOISE_DBM
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+		| SURVEY_INFO_TIME | SURVEY_INFO_TIME_BUSY
+		#endif
+		;
+
+	for (i = 0; i < chset->chs_len; i++)
+		total_rx_cnt += chset->chs[i].rx_count;
+
+	time_busy = chset->chs[idx].rx_count * time / total_rx_cnt;
+	noise += chset->chs[idx].rx_count * 50 / total_rx_cnt;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0))
+	info->channel_time = time;
+	info->channel_time_busy = time_busy;
+	#else
+	info->time = time;
+	info->time_busy = time_busy;
+	#endif
+#endif
+	info->noise = noise;
+
+	/* reset if final channel is got */
+	if (idx == chset->chs_len - 1) {
+		for (i = 0; i < chset->chs_len; i++)
+			chset->chs[i].rx_count = 0;
+	}
+}
+#endif /* CONFIG_FIND_BEST_CHANNEL */
+
+#ifdef CONFIG_RTW_ACS
+static void rtw_cfg80211_set_survey_info_with_clm(_adapter *padapter, int idx, struct survey_info *pinfo)
+{
+	s8 noise = -50;			/*channel noise in dBm. This and all following fields are optional */
+	u8 time = SURVEY_TO;	/*amount of time in ms the radio was turn on (on the channel)*/
+	u8 time_busy = 0;		/*amount of time the primary channel was sensed busy*/
+
+	if ((idx < 0) || (pinfo == NULL))
+		return;
+
+	pinfo->filled  = SURVEY_INFO_NOISE_DBM
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+		| SURVEY_INFO_TIME | SURVEY_INFO_TIME_BUSY
+		#endif
+		;
+
+	time_busy = rtw_acs_get_clm_ratio(padapter, pinfo->channel->band, pinfo->channel->hw_value);
+	noise = rtw_acs_get_noise_dbm(padapter, pinfo->channel->band, pinfo->channel->hw_value);
+	RTW_INFO("[%d] ch=%d, band=%d, time=%d(ms), time_busy=%d(ms), noise=%d(dbm)\n",
+		idx, pinfo->channel->hw_value, pinfo->channel->band ,time, time_busy, noise);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37))
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0))
+	pinfo->channel_time = time;
+	pinfo->channel_time_busy = time_busy;
+	#else
+	pinfo->time = time;
+	pinfo->time_busy = time_busy;
+	#endif
+#endif
+	pinfo->noise = noise;
+}
+#endif
+
+int rtw_hostapd_acs_dump_survey(struct wiphy *wiphy, struct net_device *netdev, int idx, struct survey_info *info)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(netdev);
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	RT_CHANNEL_INFO *chinfo;
+	u32 freq = 0;
+	u8 ret = 0;
+	u16 channel = 0;
+	u8 band = 0;
+
+	if (!netdev || !info) {
+		RTW_INFO("%s: invial parameters.\n", __func__);
+		return -EINVAL;
+	}
+
+	_rtw_memset(info, 0, sizeof(struct survey_info));
+	if (padapter->netif_up == _FALSE) {
+		RTW_INFO("%s: net device is down.\n", __func__);
+		return -EIO;
+	}
+
+	if (idx >= MAX_CHANNEL_NUM)
+		return -ENOENT;
+
+	chinfo = &chset->chs[idx];
+	channel = chinfo->ChannelNum;
+	band = chinfo->band;
+	freq = rtw_bch2freq(band, channel);
+
+	info->channel = ieee80211_get_channel(wiphy, freq);
+	/* RTW_INFO("%s: channel %d, freq %d\n", __func__, channel, freq); */
+
+	if (!info->channel)
+		return -EINVAL;
+
+	if (info->channel->flags == IEEE80211_CHAN_DISABLED)
+		return ret;
+
+	rtw_cfg80211_set_survey_info_with_clm(padapter, idx, info);
+
+	return ret;
+}
+#endif /* defined(CONFIG_RTW_HOSTAPD_ACS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 33)) */
+
+#if defined(CPTCFG_VERSION) || (KERNEL_VERSION(4, 17, 0) <= LINUX_VERSION_CODE) \
+    || defined(CONFIG_KERNEL_PATCH_EXTERNAL_AUTH)
+int cfg80211_rtw_external_auth(struct wiphy *wiphy, struct net_device *dev,
+	struct cfg80211_external_auth_params *params)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(dev));
+
+	rtw_cfg80211_external_auth_status(wiphy, dev,
+		(struct rtw_external_auth_params *)params);
+
+	return 0;
+}
+#endif
+
+void rtw_cfg80211_external_auth_status(struct wiphy *wiphy, struct net_device *dev,
+	struct rtw_external_auth_params *params)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct sta_info	*psta = NULL;
+	u8 *buf = NULL;
+	u32 len = 0;
+
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(dev));
+
+	RTW_INFO("SAE: action: %u, status: %u\n", params->action, params->status);
+	if (params->status == WLAN_STATUS_SUCCESS) {
+		RTW_INFO("bssid: "MAC_FMT"\n", MAC_ARG(params->bssid));
+		RTW_INFO("SSID: [%s]\n",
+			((params->ssid.ssid_len == 0) ? "" : (char *)params->ssid.ssid));
+		RTW_INFO("suite: 0x%08x\n", params->key_mgmt_suite);
+	}
+
+	psta = rtw_get_stainfo(pstapriv, params->bssid);
+	if (psta && (params->status == WLAN_STATUS_SUCCESS)\
+		&& MLME_IS_AP(padapter)) {
+		/* AP mode */
+		RTW_INFO("station match\n");
+
+		psta->state &= ~WIFI_FW_AUTH_NULL;
+		psta->state |= WIFI_FW_AUTH_SUCCESS;
+		psta->expire_to = padapter->stapriv.assoc_to;
+
+		/* ToDo: Kernel v5.1 pmkid is pointer */
+		/* RTW_INFO_DUMP("PMKID:", params->pmkid, PMKID_LEN); */
+		_rtw_set_pmksa(dev, params->bssid, params->pmkid);
+
+		_rtw_spinlock_bh(&psta->lock);
+		if ((psta->auth_len != 0) && (psta->pauth_frame != NULL)) {
+			buf =  rtw_zmalloc(psta->auth_len);
+			if (buf) {
+				_rtw_memcpy(buf, psta->pauth_frame, psta->auth_len);
+				len = psta->auth_len;
+			}
+
+			rtw_mfree(psta->pauth_frame, psta->auth_len);
+			psta->pauth_frame = NULL;
+			psta->auth_len = 0;
+		}
+		_rtw_spinunlock_bh(&psta->lock);
+
+		if (buf) {
+			struct _ADAPTER_LINK *padapter_link = psta->padapter_link;
+			struct link_mlme_ext_priv *pmlmeext = &(padapter_link->mlmeextpriv);
+			/* send the SAE auth Confirm */
+
+			{
+				rtw_mi_set_scan_deny(padapter, 1000);
+				rtw_mi_scan_abort(padapter, _TRUE);
+
+				RTW_INFO("SAE: Tx auth Confirm\n");
+				rtw_mgnt_tx_cmd(padapter, pmlmeext->chandef.band, pmlmeext->chandef.chan, 1, buf, len, 0, RTW_CMDF_DIRECTLY);
+
+			}
+
+			rtw_mfree(buf, len);
+			buf = NULL;
+			len = 0;
+		}
+	} else {
+		/* STA mode */
+		psecuritypriv->extauth_status = params->status;
+	}
+}
+
+#ifdef CONFIG_AP_MODE
+#ifdef CONFIG_ECSA_PHL
+static bool rtw_ap_check_csa_setting(_adapter* a, u8 new_band, u8 new_ch, u8 new_bw, u8 new_offset)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	struct rtw_chset *chset = adapter_to_chset(a);
+	struct rtw_chan_def u_chdef = {0};
+	u8 c_band, c_ch, c_bw, c_offset, u_band, u_ch, u_bw, u_offset;
+#ifdef CONFIG_MCC_MODE
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(d);
+	u8 mcc_sup = phl_com->dev_cap.mcc_sup;
+#else
+	u8 mcc_sup = _FALSE;
+#endif
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(a);
+	struct link_mlme_ext_priv *pmlmeext = &(alink->mlmeextpriv);
+
+	if (rtw_phl_mr_get_chandef(d->phl, a->phl_role, alink->wrlink, &u_chdef)
+							!= RTW_PHL_STATUS_SUCCESS) {
+		RTW_ERR("CSA : "FUNC_ADPT_FMT" get union chandef failed\n", FUNC_ADPT_ARG(a));
+		rtw_warn_on(1);
+		return _FALSE;
+	}
+
+	u_ch = u_chdef.chan;
+	u_bw = u_chdef.bw;
+	u_band = u_chdef.band;
+	u_offset = u_chdef.offset;
+	c_ch = pmlmeext->chandef.chan;
+	c_bw = pmlmeext->chandef.bw;
+	c_band = pmlmeext->chandef.band;
+	c_offset = pmlmeext->chandef.offset;
+
+	if (rtw_rfctl_is_regu_forbid_bss(adapter_to_rfctl(a), new_band)) {
+		RTW_WARN("%s %s AP not allowed by regulatory\n", __func__, band_str(new_band));
+		return _FALSE;
+	}
+
+	if (rtw_chset_search_bch(chset, new_band, new_ch) < 0
+			|| rtw_chset_is_bch_non_ocp(chset, new_band, new_ch)) {
+		RTW_INFO("CSA : reject, channel not legal csa_setting:%u,%u,%u,%u\n",
+			new_band, new_ch, new_bw, new_offset);
+		return _FALSE;
+	} else if (rtw_chset_is_dfs_bch(chset, new_band, new_ch)
+			&& !rtw_rfctl_radar_detect_supported(adapter_to_rfctl(a))) {
+		RTW_INFO("CSA : reject, DFS channel %u can't be used\n", new_ch);
+		return _FALSE;
+	}
+
+	/* Need to group with chanctx if not support MCC */
+	if (mcc_sup == _FALSE &&
+			rtw_mi_get_ld_sta_ifbmp(a) &&
+			rtw_is_bchbw_grouped(new_band, new_ch, new_bw, new_offset,
+				u_band, u_ch, u_bw, u_offset) == _FALSE) {
+		RTW_INFO("CSA : reject, can't group with STA mode, csa_setting:%u,%u,%u,%u, union:%u,%u,%u,%u\n",
+			new_band, new_ch, new_bw, new_offset, u_band, u_ch, u_bw, u_offset);
+		return _FALSE;
+	}
+
+	return _TRUE;
+}
+#endif /* CONFIG_ECSA_PHL */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0))
+static int cfg80211_rtw_channel_switch(struct wiphy *wiphy,
+				struct net_device *dev,
+				struct cfg80211_csa_settings *params)
+{
+#ifdef CONFIG_ECSA_PHL
+	_adapter *a = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(a);
+	struct rtw_chan_def csa_chdef = {0};
+	u8 mode = 0, count = 0, ht = 0;
+	u8 ecsa_op_class;
+	u8 csa_ch_width = 0, csa_ch_freq_seg0 = 0, csa_ch_freq_seg1 = 0;
+
+	if (!(CHK_MLME_STATE(a, WIFI_AP_STATE | WIFI_MESH_STATE)
+			&& MLME_IS_ASOC(a))) {
+		RTW_ERR("CSA : "FUNC_ADPT_FMT" not AP/Mesh, so return -ENOTCONN\n", FUNC_ADPT_ARG(a));
+		return -ENOTCONN;
+	}
+
+	if (rtw_mr_is_ecsa_running(a)) {
+		RTW_INFO("CSA : "FUNC_ADPT_FMT" someone is switching channel, so return -EBUSY\n", FUNC_ADPT_ARG(a));
+		return -EBUSY;
+	}
+
+	rtw_get_chdef_from_cfg80211_chan_def(&params->chandef, &ht, &csa_chdef);
+
+	if (rtw_ap_check_csa_setting(a, csa_chdef.band, csa_chdef.chan, csa_chdef.bw, csa_chdef.offset) == _FALSE)
+		return -EINVAL;
+
+	mode = params->block_tx;
+	count = params->count;
+	RTW_INFO("CSA : Get from cfg80211_csa_settings, block_tx = %s, switch count = %u\n",
+			mode ? "True" : "Flase", count);
+
+	/* Parsing channel switch related IEs from cfg80211_csa_settings */
+	rtw_cfg80211_build_csa_beacon(a, params, csa_chdef);
+
+	ecsa_op_class = rtw_phl_get_operating_class(csa_chdef);
+	rtw_hal_trigger_csa_start(a, CSA_AP_CFG80211_SWITCH_CH,
+		mode, ecsa_op_class, count,
+		csa_chdef.band, csa_chdef.chan, csa_chdef.bw, csa_chdef.offset,
+		csa_ch_width, csa_ch_freq_seg0, csa_ch_freq_seg1);
+
+#endif /* CONFIG_ECSA_PHL */
+	return 0;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) */
+#endif /* CONFIG_AP_MODE */
+
+struct cfg80211_ops rtw_cfg80211_ops = {
+	.change_virtual_intf = cfg80211_rtw_change_iface,
+	.add_key = cfg80211_rtw_add_key,
+	.get_key = cfg80211_rtw_get_key,
+	.del_key = cfg80211_rtw_del_key,
+	.set_default_key = cfg80211_rtw_set_default_key,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30))
+	.set_default_mgmt_key = cfg80211_rtw_set_default_mgmt_key,
+#endif
+#if defined(CONFIG_GTK_OL) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0))
+	.set_rekey_data = cfg80211_rtw_set_rekey_data,
+#endif /*CONFIG_GTK_OL*/
+	.get_station = cfg80211_rtw_get_station,
+	.scan = cfg80211_rtw_scan,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0))
+	.abort_scan = cfg80211_rtw_abort_scan,
+#endif /* LINUX_VERSION_CODE 4.5.0 */
+	.set_wiphy_params = cfg80211_rtw_set_wiphy_params,
+	.connect = cfg80211_rtw_connect,
+	.disconnect = cfg80211_rtw_disconnect,
+	.join_ibss = cfg80211_rtw_join_ibss,
+	.leave_ibss = cfg80211_rtw_leave_ibss,
+	.set_tx_power = cfg80211_rtw_set_txpower,
+	.get_tx_power = cfg80211_rtw_get_txpower,
+	.set_power_mgmt = cfg80211_rtw_set_power_mgmt,
+	.set_pmksa = cfg80211_rtw_set_pmksa,
+	.del_pmksa = cfg80211_rtw_del_pmksa,
+	.flush_pmksa = cfg80211_rtw_flush_pmksa,
+
+#ifdef CONFIG_AP_MODE
+	.add_virtual_intf = cfg80211_rtw_add_virtual_intf,
+	.del_virtual_intf = cfg80211_rtw_del_virtual_intf,
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)) && !defined(COMPAT_KERNEL_RELEASE)
+	.add_beacon = cfg80211_rtw_add_beacon,
+	.set_beacon = cfg80211_rtw_set_beacon,
+	.del_beacon = cfg80211_rtw_del_beacon,
+#else
+	.start_ap = cfg80211_rtw_start_ap,
+	.change_beacon = cfg80211_rtw_change_beacon,
+	.stop_ap = cfg80211_rtw_stop_ap,
+#endif
+
+#if CONFIG_RTW_MACADDR_ACL && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	.set_mac_acl = cfg80211_rtw_set_mac_acl,
+#endif
+
+	.add_station = cfg80211_rtw_add_station,
+	.del_station = cfg80211_rtw_del_station,
+	.change_station = cfg80211_rtw_change_station,
+	.dump_station = cfg80211_rtw_dump_station,
+	.change_bss = cfg80211_rtw_change_bss,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+	.set_txq_params = cfg80211_rtw_set_txq_params,
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))
+	.set_channel = cfg80211_rtw_set_channel,
+#endif
+	/* .auth = cfg80211_rtw_auth, */
+	/* .assoc = cfg80211_rtw_assoc,	 */
+#endif /* CONFIG_AP_MODE */
+
+#if defined(CONFIG_RTW_MESH) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38))
+	.get_mesh_config = cfg80211_rtw_get_mesh_config,
+	.update_mesh_config = cfg80211_rtw_update_mesh_config,
+	.join_mesh = cfg80211_rtw_join_mesh,
+	.leave_mesh = cfg80211_rtw_leave_mesh,
+	.add_mpath = cfg80211_rtw_add_mpath,
+	.del_mpath = cfg80211_rtw_del_mpath,
+	.change_mpath = cfg80211_rtw_change_mpath,
+	.get_mpath = cfg80211_rtw_get_mpath,
+	.dump_mpath = cfg80211_rtw_dump_mpath,
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+	.get_mpp = cfg80211_rtw_get_mpp,
+	.dump_mpp = cfg80211_rtw_dump_mpp,
+	#endif
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	.set_monitor_channel = cfg80211_rtw_set_monitor_channel,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	.get_channel = cfg80211_rtw_get_channel,
+#endif
+
+	.remain_on_channel = cfg80211_rtw_remain_on_channel,
+	.cancel_remain_on_channel = cfg80211_rtw_cancel_remain_on_channel,
+#ifdef CONFIG_P2P
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	.start_p2p_device = cfg80211_rtw_start_p2p_device,
+	.stop_p2p_device = cfg80211_rtw_stop_p2p_device,
+	#endif
+#endif /* CONFIG_P2P */
+
+#ifdef CONFIG_RTW_80211R
+	.update_ft_ies = cfg80211_rtw_update_ft_ies,
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)) || defined(COMPAT_KERNEL_RELEASE)
+	.mgmt_tx = cfg80211_rtw_mgmt_tx,
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
+	.mgmt_frame_register = cfg80211_rtw_mgmt_frame_register,
+#else
+	.update_mgmt_frame_registrations = cfg80211_rtw_update_mgmt_frame_register,
+#endif
+
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34) && LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35))
+	.action = cfg80211_rtw_mgmt_tx,
+#endif
+
+#if defined(CONFIG_TDLS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0))
+	.tdls_mgmt = cfg80211_rtw_tdls_mgmt,
+	.tdls_oper = cfg80211_rtw_tdls_oper,
+#endif /* CONFIG_TDLS */
+
+#if defined(CONFIG_PNO_SUPPORT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+	.sched_scan_start = cfg80211_rtw_sched_scan_start,
+	.sched_scan_stop = cfg80211_rtw_sched_scan_stop,
+	.suspend = cfg80211_rtw_suspend,
+	.resume = cfg80211_rtw_resume,
+#endif /* CONFIG_PNO_SUPPORT */
+#ifdef CONFIG_RFKILL_POLL
+	.rfkill_poll = cfg80211_rtw_rfkill_poll,
+#endif
+#if defined(CONFIG_RTW_HOSTAPD_ACS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 33))
+	.dump_survey = rtw_hostapd_acs_dump_survey,
+#endif
+#if defined(CPTCFG_VERSION) || (KERNEL_VERSION(4, 17, 0) <= LINUX_VERSION_CODE) \
+    || defined(CONFIG_KERNEL_PATCH_EXTERNAL_AUTH)
+	.external_auth = cfg80211_rtw_external_auth,
+#endif
+#ifdef CONFIG_AP_MODE
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0))
+	.channel_switch = cfg80211_rtw_channel_switch,
+#endif
+#endif /* #ifdef CONFIG_AP_MODE */
+};
+
+struct wiphy *rtw_wiphy_alloc(_adapter *padapter, struct device *dev)
+{
+	struct wiphy *wiphy;
+	struct rtw_wiphy_data *wiphy_data;
+
+	/* wiphy */
+	wiphy = wiphy_new(&rtw_cfg80211_ops, sizeof(struct rtw_wiphy_data));
+	if (!wiphy) {
+		RTW_ERR("Couldn't allocate wiphy device\n");
+		goto exit;
+	}
+	set_wiphy_dev(wiphy, dev);
+
+	/* wiphy_data */
+	wiphy_data = rtw_wiphy_priv(wiphy);
+	wiphy_data->dvobj = adapter_to_dvobj(padapter);
+
+	wiphy_data->txpwr_total_lmt_mbm = UNSPECIFIED_MBM;
+	/*wiphy_data->txpwr_total_target_mbm = UNSPECIFIED_MBM;*/
+
+	if (rtw_cfg80211_init_wiphy(padapter, wiphy) != _SUCCESS) {
+		rtw_wiphy_free(wiphy);
+		wiphy = NULL;
+		goto exit;
+	}
+
+	rtw_regd_init(wiphy);
+
+	RTW_INFO(FUNC_WIPHY_FMT"\n", FUNC_WIPHY_ARG(wiphy));
+
+exit:
+	return wiphy;
+}
+
+void rtw_wiphy_free(struct wiphy *wiphy)
+{
+	if (!wiphy)
+		return;
+
+	RTW_INFO(FUNC_WIPHY_FMT"\n", FUNC_WIPHY_ARG(wiphy));
+
+	rtw_regd_deinit(wiphy);
+
+	if (wiphy->bands[NL80211_BAND_2GHZ]) {
+		rtw_spt_band_free(wiphy->bands[NL80211_BAND_2GHZ]);
+		wiphy->bands[NL80211_BAND_2GHZ] = NULL;
+	}
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (wiphy->bands[NL80211_BAND_5GHZ]) {
+		rtw_spt_band_free(wiphy->bands[NL80211_BAND_5GHZ]);
+		wiphy->bands[NL80211_BAND_5GHZ] = NULL;
+	}
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+	if (wiphy->bands[NL80211_BAND_6GHZ]) {
+		rtw_spt_band_free(wiphy->bands[NL80211_BAND_6GHZ]);
+		wiphy->bands[NL80211_BAND_6GHZ] = NULL;
+	}
+#endif
+
+	wiphy_free(wiphy);
+}
+
+int rtw_wiphy_register(struct wiphy *wiphy)
+{
+	struct rtw_chset *chset = dvobj_to_chset(wiphy_to_dvobj(wiphy));
+	struct get_chplan_resp *chplan;
+	int ret;
+
+	RTW_INFO(FUNC_WIPHY_FMT"\n", FUNC_WIPHY_ARG(wiphy));
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) || defined(RTW_VENDOR_EXT_SUPPORT)
+	rtw_cfgvendor_attach(wiphy);
+#endif
+
+	#if !RTW_PER_ADAPTER_WIPHY
+	rtw_chset_hook_os_channels(chset, wiphy);
+	#endif
+
+	ret = wiphy_register(wiphy);
+	if (ret != 0) {
+		RTW_INFO(FUNC_WIPHY_FMT" wiphy_register() return %d\n", FUNC_WIPHY_ARG(wiphy), ret);
+		goto exit;
+	}
+
+	if (rtw_get_chplan_cmd(wiphy_to_adapter(wiphy), RTW_CMDF_DIRECTLY, &chplan) == _SUCCESS)
+		rtw_regd_change_complete_sync(wiphy, chplan, 1);
+	else
+		rtw_warn_on(1);
+
+exit:
+	return ret;
+}
+
+void rtw_wiphy_unregister(struct wiphy *wiphy)
+{
+	RTW_INFO(FUNC_WIPHY_FMT"\n", FUNC_WIPHY_ARG(wiphy));
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) || defined(RTW_VENDOR_EXT_SUPPORT)
+	rtw_cfgvendor_detach(wiphy);
+#endif
+
+	#if defined(RTW_DEDICATED_P2P_DEVICE)
+	rtw_pd_iface_free(wiphy);
+	#endif
+
+	#if CONFIG_RTW_CFG80211_CAC_EVENT
+	rtw_regd_free_du_wdev(wiphy);
+	#endif
+
+	return wiphy_unregister(wiphy);
+}
+
+int rtw_wdev_alloc(_adapter *padapter, struct wiphy *wiphy)
+{
+	int ret = 0;
+	struct net_device *pnetdev = padapter->pnetdev;
+	struct wireless_dev *wdev;
+	struct rtw_wdev_priv *pwdev_priv;
+
+	RTW_INFO("%s(padapter=%p)\n", __func__, padapter);
+
+	/*  wdev */
+	wdev = (struct wireless_dev *)rtw_zmalloc(sizeof(struct wireless_dev));
+	if (!wdev) {
+		RTW_INFO("Couldn't allocate wireless device\n");
+		ret = -ENOMEM;
+		goto exit;
+	}
+	wdev->wiphy = wiphy;
+	wdev->netdev = pnetdev;
+	wdev->iftype = NL80211_IFTYPE_STATION;
+	padapter->rtw_wdev = wdev;
+	pnetdev->ieee80211_ptr = wdev;
+
+	/* init pwdev_priv */
+	pwdev_priv = adapter_wdev_data(padapter);
+	pwdev_priv->rtw_wdev = wdev;
+	pwdev_priv->pmon_ndev = NULL;
+	pwdev_priv->ifname_mon[0] = '\0';
+	pwdev_priv->padapter = padapter;
+	pwdev_priv->scan_request = NULL;
+	_rtw_spinlock_init(&pwdev_priv->scan_req_lock);
+	pwdev_priv->connect_req = NULL;
+	_rtw_spinlock_init(&pwdev_priv->connect_req_lock);
+
+	pwdev_priv->p2p_enabled = _FALSE;
+	pwdev_priv->probe_resp_ie_update_time = rtw_get_current_time();
+	rtw_wdev_invit_info_init(&pwdev_priv->invit_info);
+	rtw_wdev_nego_info_init(&pwdev_priv->nego_info);
+
+	_rtw_mutex_init(&pwdev_priv->roch_mutex);
+
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+        pwdev_priv->rssi_monitor_enable = 0;
+        pwdev_priv->rssi_monitor_max = 0;
+        pwdev_priv->rssi_monitor_min = 0;
+#endif
+
+
+exit:
+	return ret;
+}
+
+void rtw_wdev_free(struct wireless_dev *wdev)
+{
+	if (!wdev)
+		return;
+
+	RTW_INFO("%s(wdev=%p)\n", __func__, wdev);
+
+	if (wdev_to_ndev(wdev)) {
+		_adapter *adapter = (_adapter *)rtw_netdev_priv(wdev_to_ndev(wdev));
+		struct rtw_wdev_priv *wdev_priv = adapter_wdev_data(adapter);
+
+		_rtw_spinlock_free(&wdev_priv->scan_req_lock);
+
+		_rtw_spinlock_bh(&wdev_priv->connect_req_lock);
+		rtw_wdev_free_connect_req(wdev_priv);
+		_rtw_spinunlock_bh(&wdev_priv->connect_req_lock);
+		_rtw_spinlock_free(&wdev_priv->connect_req_lock);
+
+		_rtw_mutex_free(&wdev_priv->roch_mutex);
+	}
+
+	rtw_mfree((u8 *)wdev, sizeof(struct wireless_dev));
+}
+
+void rtw_wdev_unregister(struct wireless_dev *wdev)
+{
+	struct net_device *ndev;
+	_adapter *adapter;
+	struct rtw_wdev_priv *pwdev_priv;
+
+	if (!wdev)
+		return;
+
+	RTW_INFO("%s(wdev=%p)\n", __func__, wdev);
+
+	ndev = wdev_to_ndev(wdev);
+	if (!ndev)
+		return;
+
+	adapter = (_adapter *)rtw_netdev_priv(ndev);
+	pwdev_priv = adapter_wdev_data(adapter);
+
+	rtw_cfg80211_indicate_scan_done(adapter, _TRUE);
+
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0)) || defined(COMPAT_KERNEL_RELEASE)
+	#if (defined(CONFIG_ACK_5_15_LTS_KERNEL) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2)))
+	/* ToDo CONFIG_RTW_MLD */
+	if (wdev->valid_links && wdev->links[0].client.current_bss)
+	#else
+	if (wdev->current_bss)
+	#endif
+	{
+		RTW_INFO(FUNC_ADPT_FMT" clear current_bss by cfg80211_disconnected\n", FUNC_ADPT_ARG(adapter));
+		rtw_cfg80211_indicate_disconnect(adapter, 0, 1);
+	}
+	#endif
+
+	if (pwdev_priv->pmon_ndev) {
+		RTW_INFO("%s, unregister monitor interface\n", __func__);
+		unregister_netdev(pwdev_priv->pmon_ndev);
+	}
+}
+
+int rtw_cfg80211_ndev_res_alloc(_adapter *adapter)
+{
+	int ret = _FAIL;
+
+	if (rtw_wdev_alloc(adapter, adapter_to_wiphy(adapter)) == 0)
+		ret = _SUCCESS;
+
+	return ret;
+}
+
+void rtw_cfg80211_ndev_res_free(_adapter *adapter)
+{
+	rtw_wdev_free(adapter->rtw_wdev);
+	adapter->rtw_wdev = NULL;
+}
+
+int rtw_cfg80211_ndev_res_register(_adapter *adapter)
+{
+	return _SUCCESS;
+}
+
+void rtw_cfg80211_ndev_res_unregister(_adapter *adapter)
+{
+	rtw_wdev_unregister(adapter->rtw_wdev);
+}
+
+int rtw_cfg80211_dev_res_alloc(struct dvobj_priv *dvobj)
+{
+	int ret = _FAIL;
+	struct wiphy *wiphy;
+	struct device *dev = dvobj_to_dev(dvobj);
+
+	wiphy = rtw_wiphy_alloc(dvobj_get_primary_adapter(dvobj), dev);
+	if (wiphy == NULL)
+		return ret;
+
+	dvobj->wiphy = wiphy;
+
+	ret = _SUCCESS;
+	return ret;
+}
+
+void rtw_cfg80211_dev_res_free(struct dvobj_priv *dvobj)
+{
+	rtw_wiphy_free(dvobj_to_wiphy(dvobj));
+	dvobj->wiphy = NULL;
+}
+
+int rtw_cfg80211_dev_res_register(struct dvobj_priv *dvobj)
+{
+	int ret = _FAIL;
+
+	if (rtw_wiphy_register(dvobj_to_wiphy(dvobj)) != 0)
+		return ret;
+
+#ifdef CONFIG_RFKILL_POLL
+	rtw_cfg80211_init_rfkill(dvobj_to_wiphy(dvobj));
+#endif
+
+	ret = _SUCCESS;
+
+	return ret;
+}
+
+void rtw_cfg80211_dev_res_unregister(struct dvobj_priv *dvobj)
+{
+#ifdef CONFIG_RFKILL_POLL
+	rtw_cfg80211_deinit_rfkill(dvobj_to_wiphy(dvobj));
+#endif
+	rtw_wiphy_unregister(dvobj_to_wiphy(dvobj));
+}
+
+s16 rtw_cfg80211_dev_get_total_txpwr_lmt_mbm(struct dvobj_priv *dvobj)
+{
+	struct rtw_wiphy_data *wiphy_data;
+	s16 mbm = UNSPECIFIED_MBM;
+
+	wiphy_data = rtw_wiphy_priv(dvobj_to_wiphy(dvobj));
+	mbm = wiphy_data->txpwr_total_lmt_mbm;
+
+	return mbm;
+}
+
+#endif /* CONFIG_IOCTL_CFG80211 */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_cfg80211.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_cfg80211.h
new file mode 100644
index 000000000000..75e284cf6276
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_cfg80211.h
@@ -0,0 +1,501 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __IOCTL_CFG80211_H__
+#define __IOCTL_CFG80211_H__
+
+#define RTW_CFG80211_BLOCK_DISCON_WHEN_CONNECT		BIT0
+#define RTW_CFG80211_BLOCK_DISCON_WHEN_DISCONNECT	BIT1
+
+#define CONFIG_CFG80211_REPORT_PROBE_REQ
+#ifdef CONFIG_80211BE_EHT
+#undef CONFIG_CFG80211_REPORT_PROBE_REQ
+#endif
+#ifndef RTW_CFG80211_BLOCK_STA_DISCON_EVENT
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0))
+#define RTW_CFG80211_BLOCK_STA_DISCON_EVENT (RTW_CFG80211_BLOCK_DISCON_WHEN_CONNECT)
+#else
+#define RTW_CFG80211_BLOCK_STA_DISCON_EVENT (RTW_CFG80211_BLOCK_DISCON_WHEN_CONNECT | RTW_CFG80211_BLOCK_DISCON_WHEN_DISCONNECT)
+#endif
+#endif
+
+#if defined(RTW_USE_CFG80211_STA_EVENT)
+	#undef CONFIG_CFG80211_FORCE_COMPATIBLE_2_6_37_UNDER
+#endif
+
+#ifndef RTW_P2P_GROUP_INTERFACE
+	#define RTW_P2P_GROUP_INTERFACE 0
+#endif
+
+/*
+* (RTW_P2P_GROUP_INTERFACE, RTW_DEDICATED_P2P_DEVICE)
+* (0, 0): wlan0 + p2p0(PD+PG)
+* (1, 0): wlan0(with PD) + dynamic PGs
+* (1, 1): wlan0 (with dynamic PD wdev) + dynamic PGs
+*/
+
+#if RTW_P2P_GROUP_INTERFACE
+	#ifndef CONFIG_RTW_DYNAMIC_NDEV
+		#define CONFIG_RTW_DYNAMIC_NDEV
+	#endif
+
+	#ifndef CONFIG_RADIO_WORK
+		#define CONFIG_RADIO_WORK
+	#endif
+#endif
+
+#ifdef CONFIG_RTW_DYNAMIC_NDEV
+	#ifdef RTW_DEDICATED_P2P_DEVICE
+		#ifdef CONFIG_RTW_STATIC_NDEV_NUM
+		#undef CONFIG_RTW_STATIC_NDEV_NUM
+		#endif
+	#endif
+
+	#ifndef CONFIG_RTW_STATIC_NDEV_NUM
+		#define CONFIG_RTW_STATIC_NDEV_NUM 1
+	#endif
+#else
+	#ifdef CONFIG_RTW_STATIC_NDEV_NUM
+	#undef CONFIG_RTW_STATIC_NDEV_NUM
+	#endif
+#endif
+
+#ifndef CONFIG_RTW_STATIC_NDEV_NUM
+	#define CONFIG_RTW_STATIC_NDEV_NUM CONFIG_IFACE_NUMBER
+#endif
+
+#if !((CONFIG_RTW_STATIC_NDEV_NUM > 0) && \
+	(CONFIG_RTW_STATIC_NDEV_NUM <= CONFIG_IFACE_NUMBER))
+	#error "CONFIG_RTW_STATIC_NDEV_NUM our of range"
+#endif
+
+#if defined(CONFIG_P2P) && defined(CONFIG_SEL_P2P_IFACE) && \
+	!((CONFIG_SEL_P2P_IFACE >= 0) && (CONFIG_SEL_P2P_IFACE < CONFIG_RTW_STATIC_NDEV_NUM))
+	#error "CONFIG_SEL_P2P_IFACE our of range"
+#endif
+
+#ifndef CONFIG_RADIO_WORK
+#define RTW_ROCH_DURATION_ENLARGE
+#define RTW_ROCH_BACK_OP
+#endif
+
+#if defined(CONFIG_DFS_MASTER) && (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0))
+#ifndef CONFIG_RTW_CFG80211_CAC_EVENT
+#define CONFIG_RTW_CFG80211_CAC_EVENT 1
+#endif
+#else
+#ifdef CONFIG_RTW_CFG80211_CAC_EVENT
+#undef CONFIG_RTW_CFG80211_CAC_EVENT
+#endif
+#define CONFIG_RTW_CFG80211_CAC_EVENT 0
+#endif
+
+#if CONFIG_RTW_CFG80211_CAC_EVENT && RTW_PER_ADAPTER_WIPHY
+#error "CONFIG_RTW_CFG80211_CAC_EVENT is not supported when enable RTW_PER_ADAPTER_WIPHY"
+#endif
+
+#if !defined(CONFIG_P2P) && RTW_P2P_GROUP_INTERFACE
+	#error "RTW_P2P_GROUP_INTERFACE can't be enabled when CONFIG_P2P is disabled\n"
+#endif
+
+#ifdef CONFIG_SEL_P2P_IFACE
+	#if RTW_P2P_GROUP_INTERFACE
+		#error "CONFIG_SEL_P2P_IFACE has no effect when RTW_P2P_GROUP_INTERFACE is enabled"
+	#endif
+	#ifdef CONFIG_CFG80211_REPORT_PROBE_REQ
+		#error "CONFIG_SEL_P2P_IFACE has no effect when CONFIG_CFG80211_REPORT_PROBE_REQ is enabled"
+	#endif
+#endif
+
+#if !RTW_P2P_GROUP_INTERFACE && defined(RTW_DEDICATED_P2P_DEVICE)
+	#error "RTW_DEDICATED_P2P_DEVICE can't be enabled when RTW_P2P_GROUP_INTERFACE is disabled\n"
+#endif
+
+#if defined(RTW_DEDICATED_P2P_DEVICE) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0))
+	#error "RTW_DEDICATED_P2P_DEVICE can't be enabled when kernel < 3.7.0\n"
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0))
+		#error "CONFIG_RTW_MESH can't be enabled when kernel < 3.10.0\n"
+	#endif
+#endif
+
+struct rtw_wdev_invit_info {
+	u8 state; /* 0: req, 1:rep */
+	u8 peer_mac[ETH_ALEN];
+	u8 group_bssid[ETH_ALEN];
+	u8 active;
+	u8 token;
+	u8 flags;
+	u8 status;
+	u8 req_op_ch;
+	u8 rsp_op_ch;
+};
+
+#define rtw_wdev_invit_info_init(invit_info) \
+	do { \
+		(invit_info)->state = 0xff; \
+		_rtw_memset((invit_info)->peer_mac, 0, ETH_ALEN); \
+		_rtw_memset((invit_info)->group_bssid, 0, ETH_ALEN); \
+		(invit_info)->active = 0xff; \
+		(invit_info)->token = 0; \
+		(invit_info)->flags = 0x00; \
+		(invit_info)->status = 0xff; \
+		(invit_info)->req_op_ch = 0; \
+		(invit_info)->rsp_op_ch = 0; \
+	} while (0)
+
+struct rtw_wdev_nego_info {
+	u8 state; /* 0: req, 1:rep, 2:conf */
+	u8 iface_addr[ETH_ALEN];
+	u8 peer_mac[ETH_ALEN];
+	u8 peer_iface_addr[ETH_ALEN];
+	u8 active;
+	u8 token;
+	u8 status;
+	u8 req_intent;
+	u8 req_op_ch;
+	u8 req_listen_ch;
+	u8 rsp_intent;
+	u8 rsp_op_ch;
+	u8 conf_op_ch;
+};
+
+#define rtw_wdev_nego_info_init(nego_info) \
+	do { \
+		(nego_info)->state = 0xff; \
+		_rtw_memset((nego_info)->iface_addr, 0, ETH_ALEN); \
+		_rtw_memset((nego_info)->peer_mac, 0, ETH_ALEN); \
+		_rtw_memset((nego_info)->peer_iface_addr, 0, ETH_ALEN); \
+		(nego_info)->active = 0xff; \
+		(nego_info)->token = 0; \
+		(nego_info)->status = 0xff; \
+		(nego_info)->req_intent = 0xff; \
+		(nego_info)->req_op_ch = 0; \
+		(nego_info)->req_listen_ch = 0; \
+		(nego_info)->rsp_intent = 0xff; \
+		(nego_info)->rsp_op_ch = 0; \
+		(nego_info)->conf_op_ch = 0; \
+	} while (0)
+
+struct rtw_wdev_priv {
+	struct wireless_dev *rtw_wdev;
+
+	_adapter *padapter;
+
+	#if RTW_CFG80211_BLOCK_STA_DISCON_EVENT
+	u8 not_indic_disco;
+	#endif
+
+	struct cfg80211_scan_request *scan_request;
+	_lock scan_req_lock;
+
+	struct cfg80211_connect_params *connect_req;
+	_lock connect_req_lock;
+
+	struct net_device *pmon_ndev;/* for monitor interface */
+	char ifname_mon[IFNAMSIZ + 1]; /* interface name for monitor interface */
+
+	u8 p2p_enabled;
+	systime probe_resp_ie_update_time;
+
+	struct rtw_wdev_invit_info invit_info;
+	struct rtw_wdev_nego_info nego_info;
+
+	bool block;
+	bool block_scan;
+	bool power_mgmt;
+
+	/**
+	 * mgmt_regs: bitmap of management frame subtypes registered for the
+	 * 	given interface
+	 * mcast_mgmt_regs: mcast RX is needed on this interface for these
+	 * 	subtypes
+	 */
+	u32 mgmt_regs;
+	/* u32 mcast_mgmt_regs; */
+
+	u8 is_mgmt_tx;
+	u16 mgmt_tx_cookie;
+
+	_mutex roch_mutex;
+
+#if defined(CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI) || defined(CONFIG_RTW_SCAN_RAND)
+	bool random_mac_enabled;
+	u8 pno_mac_addr[ETH_ALEN];
+	u16 pno_scan_seq_num;
+#endif
+
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+        s8 rssi_monitor_max;
+        s8 rssi_monitor_min;
+        u8 rssi_monitor_enable;
+#endif
+
+};
+
+enum external_auth_action {
+	EXTERNAL_AUTH_START,
+	EXTERNAL_AUTH_ABORT,
+};
+
+struct rtw_external_auth_params {
+	enum external_auth_action action;
+	u8 bssid[ETH_ALEN]__aligned(2);
+	struct cfg80211_ssid ssid;
+	unsigned int key_mgmt_suite;
+	u16 status;
+	u8 pmkid[PMKID_LEN];
+};
+
+bool rtw_cfg80211_is_connect_requested(_adapter *adapter);
+
+#if RTW_CFG80211_BLOCK_STA_DISCON_EVENT
+#define rtw_wdev_not_indic_disco(rtw_wdev_data) ((rtw_wdev_data)->not_indic_disco)
+#define rtw_wdev_set_not_indic_disco(rtw_wdev_data, val) do { (rtw_wdev_data)->not_indic_disco = (val); } while (0)
+#else
+#define rtw_wdev_not_indic_disco(rtw_wdev_data) 0
+#define rtw_wdev_set_not_indic_disco(rtw_wdev_data, val) do {} while (0)
+#endif
+
+#define rtw_wdev_free_connect_req(rtw_wdev_data) \
+	do { \
+		if ((rtw_wdev_data)->connect_req) { \
+			rtw_mfree((u8 *)(rtw_wdev_data)->connect_req, sizeof(*(rtw_wdev_data)->connect_req)); \
+			(rtw_wdev_data)->connect_req = NULL; \
+		} \
+	} while (0)
+
+#define wdev_to_ndev(w) ((w)->netdev)
+#define wdev_to_wiphy(w) ((w)->wiphy)
+#define ndev_to_wdev(n) ((n)->ieee80211_ptr)
+
+struct rtw_wiphy_data {
+	struct dvobj_priv *dvobj;
+
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+	struct wireless_dev *pd_wdev; /* P2P device wdev */
+#endif
+
+	_list async_regd_change_list;
+	_mutex async_regd_change_mutex;
+	_workitem async_regd_change_work;
+
+#if CONFIG_RTW_CFG80211_CAC_EVENT
+	_list async_cac_change_list;
+	_mutex async_cac_change_mutex;
+	_workitem async_cac_change_work;
+
+	/* for DFS channel state sync */
+	struct wireless_dev *du_wdev;
+	struct cfg80211_chan_def du_chdef;
+#endif
+
+	s16 txpwr_total_lmt_mbm;	/* EIRP */
+};
+
+#define rtw_wiphy_priv(wiphy) ((struct rtw_wiphy_data *)wiphy_priv(wiphy))
+#define wiphy_to_dvobj(wiphy) (((struct rtw_wiphy_data *)wiphy_priv(wiphy))->dvobj)
+#define wiphy_to_adapter(wiphy) (dvobj_get_primary_adapter(wiphy_to_dvobj(wiphy)))
+
+
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+#define wiphy_to_pd_wdev(wiphy) (rtw_wiphy_priv(wiphy)->pd_wdev)
+#else
+#define wiphy_to_pd_wdev(wiphy) NULL
+#endif
+
+#define WIPHY_FMT "%s"
+#define WIPHY_ARG(wiphy) wiphy_name(wiphy)
+#define FUNC_WIPHY_FMT "%s("WIPHY_FMT")"
+#define FUNC_WIPHY_ARG(wiphy) __func__, WIPHY_ARG(wiphy)
+
+#define SET_CFG80211_MGMT_REGS(w, t) (w |= BIT(t >> 4))
+#define CLR_CFG80211_MGMT_REGS(w, t) (w &= (~BIT(t >> 4)))
+#define GET_CFG80211_MGMT_REGS(w, t) ((w & BIT(t >> 4)) > 0)
+
+#define SET_CFG80211_REPORT_MGMT(w, t) (SET_CFG80211_MGMT_REGS(w->mgmt_regs, t))
+#define CLR_CFG80211_REPORT_MGMT(w, t) (CLR_CFG80211_MGMT_REGS(w->mgmt_regs, t))
+#define GET_CFG80211_REPORT_MGMT(w, t) (GET_CFG80211_MGMT_REGS(w->mgmt_regs, t))
+
+struct wiphy *rtw_wiphy_alloc(_adapter *padapter, struct device *dev);
+void rtw_wiphy_free(struct wiphy *wiphy);
+int rtw_wiphy_register(struct wiphy *wiphy);
+void rtw_wiphy_unregister(struct wiphy *wiphy);
+
+int rtw_wdev_alloc(_adapter *padapter, struct wiphy *wiphy);
+void rtw_wdev_free(struct wireless_dev *wdev);
+void rtw_wdev_unregister(struct wireless_dev *wdev);
+
+int rtw_cfg80211_ndev_res_alloc(_adapter *adapter);
+void rtw_cfg80211_ndev_res_free(_adapter *adapter);
+int rtw_cfg80211_ndev_res_register(_adapter *adapter);
+void rtw_cfg80211_ndev_res_unregister(_adapter *adapter);
+
+int rtw_cfg80211_dev_res_alloc(struct dvobj_priv *dvobj);
+void rtw_cfg80211_dev_res_free(struct dvobj_priv *dvobj);
+int rtw_cfg80211_dev_res_register(struct dvobj_priv *dvobj);
+void rtw_cfg80211_dev_res_unregister(struct dvobj_priv *dvobj);
+s16 rtw_cfg80211_dev_get_total_txpwr_lmt_mbm(struct dvobj_priv *dvobj);
+
+void rtw_cfg80211_unlink_bss(_adapter *padapter, struct wlan_network *pnetwork);
+void rtw_cfg80211_surveydone_event_callback(_adapter *padapter);
+struct cfg80211_bss *rtw_cfg80211_inform_bss(_adapter *padapter, struct wlan_network *pnetwork);
+int rtw_cfg80211_check_bss(_adapter *padapter);
+void rtw_cfg80211_ibss_indicate_connect(_adapter *padapter);
+void rtw_cfg80211_indicate_connect(_adapter *padapter);
+void rtw_cfg80211_indicate_disconnect(_adapter *padapter, u16 reason, u8 locally_generated);
+void rtw_cfg80211_indicate_scan_done(_adapter *adapter, bool aborted);
+u32 rtw_cfg80211_wait_scan_req_empty(_adapter *adapter, u32 timeout_ms);
+
+#ifdef CONFIG_CONCURRENT_MODE
+u8 rtw_cfg80211_scan_via_buddy(_adapter *padapter, struct cfg80211_scan_request *request);
+void rtw_cfg80211_indicate_scan_done_for_buddy(_adapter *padapter, bool bscan_aborted);
+#endif
+
+#ifdef CONFIG_AP_MODE
+void rtw_cfg80211_indicate_sta_assoc(_adapter *padapter, u8 *pmgmt_frame, uint frame_len);
+void rtw_cfg80211_indicate_sta_disassoc(_adapter *padapter, const u8 *da, unsigned short reason);
+#endif /* CONFIG_AP_MODE */
+
+void rtw_cfg80211_set_is_roch(_adapter *adapter, bool val);
+bool rtw_cfg80211_get_is_roch(_adapter *adapter);
+bool rtw_cfg80211_is_ro_ch_once(_adapter *adapter);
+void rtw_cfg80211_set_last_ro_ch_time(_adapter *adapter);
+s32 rtw_cfg80211_get_last_ro_ch_passing_ms(_adapter *adapter);
+
+#ifdef CONFIG_P2P
+int rtw_cfg80211_iface_has_p2p_group_cap(_adapter *adapter);
+int rtw_cfg80211_is_p2p_scan(_adapter *adapter);
+#if defined(RTW_DEDICATED_P2P_DEVICE)
+int rtw_cfg80211_redirect_pd_wdev(struct wiphy *wiphy, u8 *ra, struct wireless_dev **wdev);
+int rtw_cfg80211_is_scan_by_pd_wdev(_adapter *adapter);
+int rtw_pd_iface_alloc(struct wiphy *wiphy, const char *name, struct wireless_dev **pd_wdev);
+void rtw_pd_iface_free(struct wiphy *wiphy);
+#endif
+#endif /* CONFIG_P2P */
+
+
+void rtw_cfg80211_set_is_mgmt_tx(_adapter *adapter, u8 val);
+u8 rtw_cfg80211_get_is_mgmt_tx(_adapter *adapter);
+u8 rtw_mgnt_tx_handler(_adapter *adapter, u8 *buf);
+
+void rtw_cfg80211_rx_p2p_action_public(_adapter *padapter, union recv_frame *rframe);
+void rtw_cfg80211_rx_action_p2p(_adapter *padapter, union recv_frame *rframe);
+void rtw_cfg80211_rx_action(_adapter *adapter, union recv_frame *rframe, const char *msg);
+int rtw_cfg80211_rx_mframe(_adapter *adapter, union recv_frame *rframe, const char *msg);
+void rtw_cfg80211_rx_probe_request(_adapter *padapter, union recv_frame *rframe);
+
+void rtw_cfg80211_external_auth_request(_adapter *padapter, union recv_frame *rframe);
+void rtw_cfg80211_external_auth_status(struct wiphy *wiphy, struct net_device *dev,
+	struct rtw_external_auth_params *params);
+
+int rtw_cfg80211_set_mgnt_wpsp2pie(struct net_device *net, char *buf, int len, int type);
+
+bool rtw_cfg80211_pwr_mgmt(_adapter *adapter);
+#if defined(CONFIG_RTW_80211K) || defined(CONFIG_RTW_FSM_RRM)
+void rtw_cfg80211_rx_rrm_action(_adapter *adapter, union recv_frame *rframe);
+#endif
+
+#ifdef CONFIG_RFKILL_POLL
+void rtw_cfg80211_init_rfkill(struct wiphy *wiphy);
+void rtw_cfg80211_deinit_rfkill(struct wiphy *wiphy);
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0))  && !defined(COMPAT_KERNEL_RELEASE)
+#define rtw_cfg80211_rx_mgmt(wdev, freq, sig_dbm, buf, len, gfp) cfg80211_rx_mgmt(wdev_to_ndev(wdev), freq, buf, len, gfp)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))
+#define rtw_cfg80211_rx_mgmt(wdev, freq, sig_dbm, buf, len, gfp) cfg80211_rx_mgmt(wdev_to_ndev(wdev), freq, sig_dbm, buf, len, gfp)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0))
+#define rtw_cfg80211_rx_mgmt(wdev, freq, sig_dbm, buf, len, gfp) cfg80211_rx_mgmt(wdev, freq, sig_dbm, buf, len, gfp)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3 , 18 , 0))
+#define rtw_cfg80211_rx_mgmt(wdev , freq , sig_dbm , buf , len , gfp) cfg80211_rx_mgmt(wdev , freq , sig_dbm , buf , len , 0 , gfp)
+#else
+#define rtw_cfg80211_rx_mgmt(wdev , freq , sig_dbm , buf , len , gfp) cfg80211_rx_mgmt(wdev , freq , sig_dbm , buf , len , 0)
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0))  && !defined(COMPAT_KERNEL_RELEASE)
+#define rtw_cfg80211_send_rx_assoc(adapter, bss, buf, len) cfg80211_send_rx_assoc((adapter)->pnetdev, buf, len)
+#else
+#define rtw_cfg80211_send_rx_assoc(adapter, bss, buf, len) cfg80211_send_rx_assoc((adapter)->pnetdev, bss, buf, len)
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))
+#define rtw_cfg80211_mgmt_tx_status(wdev, cookie, buf, len, ack, gfp) cfg80211_mgmt_tx_status(wdev_to_ndev(wdev), cookie, buf, len, ack, gfp)
+#else
+#define rtw_cfg80211_mgmt_tx_status(wdev, cookie, buf, len, ack, gfp) cfg80211_mgmt_tx_status(wdev, cookie, buf, len, ack, gfp)
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0))
+#define rtw_cfg80211_ready_on_channel(wdev, cookie, chan, channel_type, duration, gfp)  cfg80211_ready_on_channel(wdev_to_ndev(wdev), cookie, chan, channel_type, duration, gfp)
+#define rtw_cfg80211_remain_on_channel_expired(wdev, cookie, chan, chan_type, gfp) cfg80211_remain_on_channel_expired(wdev_to_ndev(wdev), cookie, chan, chan_type, gfp)
+#elif (LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0))
+#define rtw_cfg80211_ready_on_channel(wdev, cookie, chan, channel_type, duration, gfp)  cfg80211_ready_on_channel(wdev, cookie, chan, channel_type, duration, gfp)
+#define rtw_cfg80211_remain_on_channel_expired(wdev, cookie, chan, chan_type, gfp) cfg80211_remain_on_channel_expired(wdev, cookie, chan, chan_type, gfp)
+#else
+#define rtw_cfg80211_ready_on_channel(wdev, cookie, chan, channel_type, duration, gfp)  cfg80211_ready_on_channel(wdev, cookie, chan, duration, gfp)
+#define rtw_cfg80211_remain_on_channel_expired(wdev, cookie, chan, chan_type, gfp) cfg80211_remain_on_channel_expired(wdev, cookie, chan, gfp)
+#endif
+
+#define rtw_cfg80211_connect_result(wdev, bssid, req_ie, req_ie_len, resp_ie, resp_ie_len, status, gfp) cfg80211_connect_result(wdev_to_ndev(wdev), bssid, req_ie, req_ie_len, resp_ie, resp_ie_len, status, gfp)
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
+#define rtw_cfg80211_disconnected(wdev, reason, ie, ie_len, locally_generated, gfp) cfg80211_disconnected(wdev_to_ndev(wdev), reason, ie, ie_len, gfp)
+#else
+#define rtw_cfg80211_disconnected(wdev, reason, ie, ie_len, locally_generated, gfp) cfg80211_disconnected(wdev_to_ndev(wdev), reason, ie, ie_len, locally_generated, gfp)
+#endif
+
+#ifdef CONFIG_RTW_80211R
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+#define rtw_cfg80211_ft_event(adapter, parm)  cfg80211_ft_event((adapter)->pnetdev, parm)
+#else
+	#error "Cannot support FT for KERNEL_VERSION < 3.10\n"
+#endif
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ && LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
+#error "CONFIG_IEEE80211_BAND_6GHZ supported only on kernel versions 5.4.0 or newer"
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+#define rtw_cfg80211_notify_new_peer_candidate(wdev, addr, ie, ie_len, sig_dbm, gfp) cfg80211_notify_new_peer_candidate(wdev_to_ndev(wdev), addr, ie, ie_len, sig_dbm, gfp)
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0))
+#define rtw_cfg80211_notify_new_peer_candidate(wdev, addr, ie, ie_len, sig_dbm, gfp) cfg80211_notify_new_peer_candidate(wdev_to_ndev(wdev), addr, ie, ie_len, gfp)
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0))
+u8 rtw_cfg80211_ch_switch_notify(_adapter *adapter,
+					struct _ADAPTER_LINK *alink,
+					struct rtw_chan_def *rtw_chdef,
+					u8 ht, bool started);
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36))
+#define NL80211_TX_POWER_AUTOMATIC	TX_POWER_AUTOMATIC
+#define NL80211_TX_POWER_LIMITED	TX_POWER_LIMITED
+#define NL80211_TX_POWER_FIXED		TX_POWER_FIXED
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+int rtw_cfg80211_split_scan_6ghz(_adapter *padapter);
+#endif
+#endif
+
+#include "os_ch_utils.h"
+#include "wifi_regd.h"
+#include "rtw_cfgvendor.h"
+
+#endif /* __IOCTL_CFG80211_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_efuse.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_efuse.c
new file mode 100644
index 000000000000..71a4f0adebe3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_efuse.c
@@ -0,0 +1,1633 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#if defined(CONFIG_MP_INCLUDED)
+#include <rtw_efuse.h>
+
+static u8 rtw_efuse_cmd(_adapter *padapter,
+							struct rtw_efuse_phl_arg *pefuse_arg ,
+							enum rtw_efuse_phl_cmdid cmdid)
+{
+	u32 i = 0;
+	u8 ret = _FAIL;
+	pefuse_arg->mp_class = RTW_MP_CLASS_EFUSE;
+	pefuse_arg->cmd = cmdid;
+	pefuse_arg->cmd_ok = 0;
+
+
+	rtw_mp_set_phl_cmd(padapter, (void*)pefuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	while (i <= 500) {
+		rtw_msleep_os(10);
+		rtw_mp_get_phl_cmd(padapter, (void*)pefuse_arg, sizeof(struct rtw_efuse_phl_arg));
+		if (pefuse_arg->cmd_ok && pefuse_arg->status == RTW_PHL_STATUS_SUCCESS) {
+			RTW_INFO("%s,eFuse GET CMD OK !!!\n", __func__);
+			ret = _SUCCESS;
+			break;
+		} else if (pefuse_arg->cmd_ok && pefuse_arg->status == RTW_PHL_STATUS_FAILURE) { 
+			RTW_INFO("%s,eFuse GET CMD FAIL !!!\n", __func__);
+			ret = _FAIL;
+			break;
+		} else {
+			rtw_msleep_os(10);
+			if (i > 50) {
+				RTW_INFO("%s, timeout eFuse GET CMD FAIL !!!\n", __func__);
+				break;
+			}
+			RTW_INFO("%s, wait for eFuse GET CMD !!!\n", __func__);
+			i++;
+		}
+	}
+
+	return ret;
+}
+
+u32 rtw_efuse_get_map_size(_adapter *padapter , u16 *size , enum rtw_efuse_phl_cmdid cmdid)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _FAIL;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+	if (efuse_arg) {
+		_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+		rtw_efuse_cmd(padapter, efuse_arg, cmdid);
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS) {
+				*size = efuse_arg->io_value;
+				res = _SUCCESS;
+		} else {
+				*size = 0;
+				res = _FAIL;
+		}
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+u32 rtw_efuse_get_available_size(_adapter *padapter , u16 *size, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _FAIL;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+
+	if (efuse_arg) {
+		_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+
+		if (efuse_type == RTW_EFUSE_WIFI)
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_WIFI_GET_AVL_SIZE);
+		else
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_BT_GET_AVL_SIZE);
+
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS) {
+				*size = efuse_arg->io_value;
+				res = _SUCCESS;
+		} else {
+				*size = 0;
+				res = _FAIL;
+		}
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+static u8 rtw_efuse_fake2map(_adapter *padapter, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _SUCCESS;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+	if (efuse_arg) {
+		_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+
+		if (efuse_type == RTW_EFUSE_WIFI)
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_WIFI_UPDATE);
+		else if (efuse_type == RTW_EFUSE_BT)
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_BT_UPDATE);
+
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS)
+			res = _SUCCESS;
+		else
+			res = _FAIL;
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+u8 rtw_efuse_read_map2shadow(_adapter *padapter, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _SUCCESS;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+	if (efuse_arg) {
+		_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+		if (efuse_type == RTW_EFUSE_WIFI)
+		    rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_WIFI_UPDATE_MAP);
+		else if (efuse_type == RTW_EFUSE_BT)
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_BT_UPDATE_MAP);
+		else
+			RTW_INFO("%s,efuse_type unknow :%d!!!\n", __func__, efuse_type);
+
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS)
+				res = _SUCCESS;
+		else
+				res = _FAIL;
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+static u8 rtw_efuse_get_shadow_map(_adapter *padapter, u8 *map, u16 size, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _FAIL;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+	if (efuse_arg) {
+
+		efuse_arg->buf_len = size;
+		if (efuse_type == RTW_EFUSE_WIFI)
+		    rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_SHADOW_MAP2BUF);
+		else if (efuse_type == RTW_EFUSE_BT)
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_BT_SHADOW_MAP2BUF);
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS) {
+				_rtw_memcpy((void *)map, efuse_arg->poutbuf, size);
+				res = _SUCCESS;
+		} else
+				res = _FAIL;
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+static u8 rtw_efuse_renew_update(_adapter *padapter, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _SUCCESS;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+	if (efuse_arg) {
+		_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+
+		if (efuse_type == RTW_EFUSE_WIFI)
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_MP_EFUSE_CMD_WIFI_SET_RENEW);
+		else if (efuse_type == RTW_EFUSE_BT)
+			RTW_INFO("halmac_get_logical_efuse_size fail\n");
+
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS)
+			res = _SUCCESS;
+		else
+			res = _FAIL;
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+static bool rtw_file_efuse_ismasked(_adapter *padapter, u16 offset, u8 *maskbuf)
+{
+	int r = offset / 16;
+	int c = (offset % 16) / 2;
+	int result = 0;
+
+	if (padapter->registrypriv.boffefusemask)
+		return _FALSE;
+
+	if (c < 4) /* Upper double word */
+		result = (maskbuf[r] & (0x10 << c));
+	else
+		result = (maskbuf[r] & (0x01 << (c - 4)));
+
+	return (result > 0) ? 0 : 1;
+}
+
+static u8 rtw_efuse_compare_data(_adapter *padapter,
+				u16 addr, u32 len, u8 *dst, u8 *src, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	enum rtw_efuse_phl_cmdid cmdid = RTW_EFUSE_CMD_WIFI_GET_MASK_BUF;
+	u8 res = _FAIL;
+	u16 idx = 0;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+
+	if (efuse_arg) {
+		_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+
+
+		if (efuse_type == RTW_EFUSE_WIFI)
+			cmdid = RTW_EFUSE_CMD_WIFI_GET_MASK_BUF;
+		else if (efuse_type == RTW_EFUSE_BT)
+			cmdid = RTW_EFUSE_CMD_BT_GET_MASK_BUF;
+
+		efuse_arg->status = 0;
+		efuse_arg->cmd_ok = 0;
+		efuse_arg->buf_len = 0;
+
+		if (rtw_efuse_cmd(padapter, efuse_arg, cmdid) == _SUCCESS) {
+
+			RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "efuse mask buf:",
+				efuse_arg->poutbuf, efuse_arg->buf_len);
+
+			for (idx = 0; idx + addr < (addr + len); idx++) {
+					u16 chkaddr = idx + addr;
+
+				if (!rtw_file_efuse_ismasked(padapter, chkaddr, efuse_arg->poutbuf)) {
+					if (_rtw_memcmp((void *)dst + chkaddr , (void *)src + idx, 1)) {
+						RTW_INFO("mask cmp success addr: 0x%x cur dst: %02x <=> PG src:%02x\n",
+										idx, dst[chkaddr], src[idx]);
+						res = _SUCCESS;
+					} else {
+						RTW_INFO("mask cmp Fail addr: 0x%x cur dst: %02x <=> PG src:%02x\n",
+										chkaddr, dst[chkaddr], src[idx]);
+						res  = _FAIL;
+						break;
+					}
+				}
+			}
+		} else {
+			RTW_INFO("rtw_efuse_cmd:%d, status:%d,mask len: %d\n",
+				efuse_arg->cmd_ok, efuse_arg->status, efuse_arg->buf_len);
+			res  = _FAIL;
+		}
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+	return res;
+}
+
+u8 rtw_efuse_map_read(_adapter * adapter, u16 addr, u16 cnts, u8 *data, u8 efuse_type)
+{
+	struct dvobj_priv *d;
+	u8 *efuse = NULL;
+	u16 size, i;
+	int err = _FAIL;
+	u8 status = _SUCCESS;
+
+	if (efuse_type == RTW_EFUSE_WIFI)
+		err = rtw_efuse_get_map_size(adapter, &size, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+	else if (efuse_type == RTW_EFUSE_BT)
+		err = rtw_efuse_get_map_size(adapter, &size, RTW_EFUSE_CMD_BT_GET_LOG_SIZE);
+
+	if (err == _FAIL) {
+		status = _FAIL;
+		RTW_INFO("halmac_get_logical_efuse_size fail\n");
+		goto exit;
+	}
+	/* size error handle */
+	if ((addr + cnts) > size) {
+		if (addr < size)
+			cnts = size - addr;
+		else {
+			status = _FAIL;
+			RTW_INFO(" %s() ,addr + cnts) > size fail\n", __func__);
+			goto exit;
+		}
+	}
+
+	efuse = rtw_zmalloc(size);
+	if (efuse) {
+		if (rtw_efuse_read_map2shadow(adapter, efuse_type) == _SUCCESS) {
+			err = rtw_efuse_get_shadow_map(adapter, efuse, size, efuse_type);
+			if (err == _FAIL) {
+				rtw_mfree(efuse, size);
+				status = _FAIL;
+				RTW_INFO(" %s() ,halmac_read_logical_efus map fail\n", __func__);
+				goto exit;
+			}
+		} else {
+			RTW_INFO(" %s() ,rtw_efuse_read_map2shadow FAIL !!!\n", __func__);
+			rtw_mfree(efuse, size);
+			status = _FAIL;
+			goto exit;
+		}
+		if (efuse) {
+			RTW_INFO(" %s() ,cp efuse to data\n", __func__);
+			_rtw_memcpy(data, efuse + addr, cnts);
+			rtw_mfree(efuse, size);
+		}
+	} else {
+			RTW_INFO(" %s() ,alloc efuse fail\n", __func__);
+			goto exit;
+	}
+	status = _SUCCESS;
+exit:
+
+	return status;
+}
+
+u8 rtw_efuse_map_write(_adapter * adapter, u16 addr, u16 cnts, u8 *data, u8 efuse_type, u8 bpg)
+{
+	struct dvobj_priv *d;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	u16 size;
+	int err = _FAIL;
+	u8 mask_buf[64] = "";
+	u32 backupRegs[4] = {0};
+	u8 status = _SUCCESS;
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u16 i = 0;
+
+	efuse_arg = rtw_zmalloc(sizeof(struct rtw_efuse_phl_arg));
+
+	if (efuse_type == RTW_EFUSE_WIFI)
+		err = rtw_efuse_get_map_size(adapter, &size, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+	else if (efuse_type == RTW_EFUSE_BT)
+		err = rtw_efuse_get_map_size(adapter, &size, RTW_EFUSE_CMD_BT_GET_LOG_SIZE);
+
+	if (err == _FAIL) {
+		status = _FAIL;
+		goto exit;
+	}
+
+	if ((addr + cnts) > size) {
+		status = _FAIL;
+		goto exit;
+	}
+
+	if (efuse_type == RTW_EFUSE_WIFI) {
+		while (i != cnts) {
+			efuse_arg->io_type = 1;
+			efuse_arg->io_offset = addr + i;
+			efuse_arg->io_value = data[i];
+			rtw_efuse_cmd(adapter, efuse_arg, RTW_EFUSE_CMD_WIFI_WRITE);
+			if (i > cnts)
+				break;
+			i++;
+		}
+	} else if (efuse_type == RTW_EFUSE_BT) {
+		while (i != cnts) {
+			efuse_arg->io_type = 1;
+			efuse_arg->io_offset = addr + i;
+			efuse_arg->io_value = data[i];
+			rtw_efuse_cmd(adapter, efuse_arg, RTW_EFUSE_CMD_BT_WRITE);
+			if (i > cnts)
+				break;
+			i++;
+		}
+	}
+
+	if (bpg) {
+		RTW_INFO(" in PG state !!!\n");
+		if (efuse_type == RTW_EFUSE_WIFI)
+			err = rtw_efuse_fake2map(adapter, efuse_type);
+		else if (efuse_type == RTW_EFUSE_BT) {
+			u8 reg2chkbt = 0;
+
+			rtw_phl_write8(dvobj->phl, 0xa3, 0x05);
+			rtw_msleep_os(10);
+			reg2chkbt = rtw_phl_read8(dvobj->phl, 0xa0);
+
+			if (reg2chkbt != 0x04) {
+				RTW_ERR("BT State not Active:0x%x ,can't Write \n", reg2chkbt);
+				goto exit;
+			}
+
+			err = rtw_efuse_fake2map(adapter, efuse_type);
+		}
+
+		if (err == _FAIL) {
+			status = _FAIL;
+			goto exit;
+		}
+	}
+	status = _SUCCESS;
+exit :
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return status;
+}
+
+static u8 rtw_efuse_map_file_load(_adapter *padapter, u8 *filepath, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _FAIL;
+
+	if (filepath) {
+		RTW_INFO("efuse file path %s len %zu", filepath, strlen(filepath));
+
+		efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+		efuse_arg->status = RTW_PHL_STATUS_FAILURE;
+
+		if (efuse_arg) {
+			_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+			_rtw_memcpy(efuse_arg->pfile_path, filepath, strlen(filepath));
+			if (efuse_type == RTW_EFUSE_WIFI)
+				rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_FILE_MAP_LOAD);
+			else
+				rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_BT_FILE_MAP_LOAD);
+		}
+	}
+	if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS)
+			res = _SUCCESS;
+	else
+			res = _FAIL;
+
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+static u8 rtw_efuse_mask_file_load(_adapter *padapter, u8 *filepath, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _FAIL;
+
+	if (filepath) {
+		RTW_INFO("efuse file path %s len %zu", filepath, strlen(filepath));
+		efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+		efuse_arg->status = RTW_PHL_STATUS_FAILURE;
+
+		if (efuse_arg) {
+			_rtw_memset((void *)efuse_arg, 0, sizeof(struct rtw_efuse_phl_arg));
+			_rtw_memcpy(efuse_arg->pfile_path, filepath, strlen(filepath));
+			if (efuse_type == RTW_EFUSE_WIFI)
+				rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_FILE_MASK_LOAD);
+			else
+				rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_BT_FILE_MASK_LOAD);
+		}
+	}
+
+	if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS)
+			res = _SUCCESS;
+	else
+			res = _FAIL;
+
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+int rtw_ioctl_efuse_get(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	char *pch, *ptmp, *token, *tmp[3] = {0x00, 0x00, 0x00};
+	u16 i = 0, j = 0, mapLen = 0, addr = 0, cnts = 0;
+	int err = 0;
+	char *pextra = NULL;
+	u8 *pre_efuse_map = NULL;
+	u16 max_available_len = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length)) {
+		err = -EFAULT;
+		goto exit;
+	}
+
+	*(extra + wrqu->data.length) = '\0';
+
+	pch = extra;
+	RTW_INFO("%s: in=%s\n", __FUNCTION__, extra);
+
+	i = 0;
+	/* mac 16 "00e04c871200" rmap,00,2 */
+	while ((token = strsep(&pch, ",")) != NULL) {
+		if (i > 2)
+			break;
+		tmp[i] = token;
+		i++;
+	}
+	pre_efuse_map = rtw_zmalloc(RTW_MAX_EFUSE_MAP_LEN);
+	if (pre_efuse_map == NULL)
+			goto exit;
+
+	if (strcmp(tmp[0], "status") == 0) {
+		//sprintf(extra, "Load File efuse=%s,Load File MAC=%s"
+		//	, efuse->file_status == EFUSE_FILE_FAILED ? "FAIL" : "OK"
+		//	, pHalData->macaddr_file_status == MACADDR_FILE_FAILED ? "FAIL" : "OK"
+		//       );
+		goto exit;
+	} else if (strcmp(tmp[0], "drvmap") == 0) {
+		static u8 drvmaporder = 0;
+		u8 *efuse_data = NULL;
+		u32 shift, cnt;
+		u32 blksz = 0x200; /* The size of one time show, default 512 */
+		//EFUSE_GetEfuseDefinition(padapter, EFUSE_WIFI, TYPE_EFUSE_MAP_LEN, (void *)&mapLen, _FALSE);
+
+		//efuse_data = efuse->data;
+
+		shift = blksz * drvmaporder;
+		efuse_data += shift;
+		cnt = mapLen - shift;
+
+		if (cnt > blksz) {
+			cnt = blksz;
+			drvmaporder++;
+		} else
+			drvmaporder = 0;
+
+		sprintf(extra, "\n");
+		for (i = 0; i < cnt; i += 16) {
+			pextra = extra + strlen(extra);
+			pextra += sprintf(pextra, "0x%02x\t", shift + i);
+			for (j = 0; j < 8; j++)
+				pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+			pextra += sprintf(pextra, "\t");
+			for (; j < 16; j++)
+				pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+			pextra += sprintf(pextra, "\n");
+		}
+		if ((shift + cnt) < mapLen)
+			pextra += sprintf(pextra, "\t...more (left:%d/%d)\n", mapLen-(shift + cnt), mapLen);
+
+	} else if (strcmp(tmp[0], "realmap") == 0) {
+		static u8 order = 0;
+		u32 shift, cnt;
+		u32 blksz = 0x200; /* The size of one time show, default 512 */
+		u8 *efuse_data = NULL;
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+
+ 		if (pre_efuse_map) {
+			if (rtw_efuse_map_read(padapter, 0, mapLen, pre_efuse_map, RTW_EFUSE_WIFI) == _FAIL) {
+				RTW_INFO("%s: read realmap Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				efuse_data = pre_efuse_map;
+
+				_rtw_memset(extra, '\0', strlen(extra));
+				shift = blksz * order;
+				efuse_data += shift;
+				cnt = mapLen - shift;
+				if (cnt > blksz) {
+					cnt = blksz;
+					order++;
+				} else
+					order = 0;
+
+				sprintf(extra, "\n");
+				for (i = 0; i < cnt; i += 16) {
+					pextra = extra + strlen(extra);
+					pextra += sprintf(pextra, "0x%02x\t", shift + i);
+					for (j = 0; j < 8; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\t");
+					for (; j < 16; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\n");
+
+					if (strlen(extra) > 0x7FF)
+						break;
+				}
+				if ((shift + cnt) < mapLen)
+					pextra += sprintf(pextra, "\t...more (left:%d/%d)\n", mapLen - (shift + cnt), mapLen);
+				}
+		}
+	} else if (strcmp(tmp[0], "rmap") == 0) {
+		u8 *data = NULL;
+		if ((tmp[1] == NULL) || (tmp[2] == NULL)) {
+			RTW_INFO("%s: rmap Fail!! Parameters error!\n", __FUNCTION__);
+			err = -EINVAL;
+			goto exit;
+		}
+		/* rmap addr cnts */
+		addr = simple_strtoul(tmp[1], &ptmp, 16);
+		RTW_INFO("%s: addr=%x\n", __FUNCTION__, addr);
+
+		cnts = simple_strtoul(tmp[2], &ptmp, 10);
+		if (cnts == 0) {
+			RTW_INFO("%s: rmap Fail!! cnts error!\n", __FUNCTION__);
+			err = -EINVAL;
+			goto exit;
+		}
+		RTW_INFO("%s: cnts=%d\n", __FUNCTION__, cnts);
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+
+		if ((addr + cnts) > mapLen) {
+			RTW_INFO("%s: addr(0x%X)+cnts(%d) over mapLen %d parameter error!\n", __FUNCTION__, addr, cnts, mapLen);
+			err = -EINVAL;
+			goto exit;
+		}
+
+		if (pre_efuse_map) {
+			if (rtw_efuse_map_read(padapter, addr, cnts, pre_efuse_map, RTW_EFUSE_WIFI) == _FAIL) {
+				RTW_INFO("%s: rtw_efuse_map_read Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				data = pre_efuse_map;
+				*extra = 0;
+				pextra = extra;
+				for (i = 0; i < cnts; i++) {
+					pextra += sprintf(pextra, "0x%02X ", data[i]);
+				}
+			}
+		}
+	} else if (strcmp(tmp[0], "wlrfkmap") == 0) {
+		static u8 order = 0;
+		u32 shift, cnt;
+		u32 blksz = 0x200; /* The size of one time show, default 512 */
+		u8 *efuse_data = NULL;
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+
+ 		if (pre_efuse_map) {
+			if (rtw_efuse_get_shadow_map(padapter, pre_efuse_map, mapLen, RTW_EFUSE_WIFI) == _FAIL) {
+				RTW_INFO("%s: read wifi fake map Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				efuse_data = pre_efuse_map;
+
+				_rtw_memset(extra, '\0', strlen(extra));
+				shift = blksz * order;
+				efuse_data += shift;
+				cnt = mapLen - shift;
+				if (cnt > blksz) {
+					cnt = blksz;
+					order++;
+				} else
+					order = 0;
+
+				sprintf(extra, "\n");
+				for (i = 0; i < cnt; i += 16) {
+					pextra = extra + strlen(extra);
+					pextra += sprintf(pextra, "0x%02x\t", shift + i);
+					for (j = 0; j < 8; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\t");
+					for (; j < 16; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\n");
+
+					if (strlen(extra) > 0x7FF)
+						break;
+				}
+				if ((shift + cnt) < mapLen)
+					pextra += sprintf(pextra, "\t...more (left:%d/%d)\n", mapLen - (shift + cnt), mapLen);
+				}
+		}
+	} else if (strcmp(tmp[0], "wlrfkrmap") == 0) {
+		u8 *data = NULL;
+		if ((tmp[1] == NULL) || (tmp[2] == NULL)) {
+			RTW_INFO("%s: rmap Fail!! Parameters error!\n", __FUNCTION__);
+			err = -EINVAL;
+			goto exit;
+		}
+		/* rmap addr cnts */
+		addr = simple_strtoul(tmp[1], &ptmp, 16);
+		RTW_INFO("%s: addr=%x\n", __FUNCTION__, addr);
+
+		cnts = simple_strtoul(tmp[2], &ptmp, 10);
+		if (cnts == 0) {
+			RTW_INFO("%s: rmap Fail!! cnts error!\n", __FUNCTION__);
+			err = -EINVAL;
+			goto exit;
+		}
+		RTW_INFO("%s: cnts=%d\n", __FUNCTION__, cnts);
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+
+		if ((addr + cnts) > mapLen) {
+			RTW_INFO("%s: addr(0x%X)+cnts(%d) over mapLen %d parameter error!\n", __FUNCTION__, addr, cnts, mapLen);
+			err = -EINVAL;
+			goto exit;
+		}
+
+		if (pre_efuse_map) {
+			if (rtw_efuse_get_shadow_map(padapter, pre_efuse_map, mapLen, RTW_EFUSE_WIFI) == _FAIL) {
+				RTW_INFO("%s: read wifi fake map Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				data = pre_efuse_map;
+				*extra = 0;
+				pextra = extra;
+				for (i = addr; i < addr + cnts; i++)
+					pextra += sprintf(pextra, "0x%02X ", data[i]);
+			}
+		}
+	} else if (strcmp(tmp[0], "btrealmap") == 0) {
+		static u8 order = 0;
+		u32 shift, cnt;
+		u32 blksz = 0x200; /* The size of one time show, default 512 */
+		u8 *efuse_data = NULL;
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_BT_GET_LOG_SIZE);
+
+ 		if (pre_efuse_map) {
+			if (rtw_efuse_map_read(padapter, 0, mapLen, pre_efuse_map, RTW_EFUSE_BT) == _FAIL) {
+				RTW_INFO("%s: read BT realmap Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				efuse_data = pre_efuse_map;
+
+				_rtw_memset(extra, '\0', strlen(extra));
+				shift = blksz * order;
+				efuse_data += shift;
+				cnt = mapLen - shift;
+				if (cnt > blksz) {
+					cnt = blksz;
+					order++;
+				} else
+					order = 0;
+
+				sprintf(extra, "\n");
+				for (i = 0; i < cnt; i += 16) {
+					pextra = extra + strlen(extra);
+					pextra += sprintf(pextra, "0x%02x\t", shift + i);
+					for (j = 0; j < 8; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\t");
+					for (; j < 16; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\n");
+
+					if (strlen(extra) > 0x7FF)
+						break;
+				}
+				if ((shift + cnt) < mapLen)
+					pextra += sprintf(pextra, "\t...more (left:%d/%d)\n", mapLen - (shift + cnt), mapLen);
+			}
+ 		}
+	} else if (strcmp(tmp[0], "btrmap") == 0) {
+		u8 *data = NULL;
+		if ((tmp[1] == NULL) || (tmp[2] == NULL)) {
+			RTW_INFO("%s: btrmap Fail!! Parameters error!\n", __FUNCTION__);
+			err = -EINVAL;
+			goto exit;
+		}
+		/* rmap addr cnts */
+		addr = simple_strtoul(tmp[1], &ptmp, 16);
+		RTW_INFO("%s: addr=%x\n", __FUNCTION__, addr);
+
+		cnts = simple_strtoul(tmp[2], &ptmp, 10);
+		if (cnts == 0) {
+			RTW_INFO("%s: btrmap Fail!! cnts error!\n", __FUNCTION__);
+			err = -EINVAL;
+			goto exit;
+		}
+		RTW_INFO("%s: cnts=%d\n", __FUNCTION__, cnts);
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_BT_GET_LOG_SIZE);
+
+		if ((addr + cnts) > mapLen) {
+			RTW_INFO("%s: addr(0x%X)+cnts(%d) over mapLen %d parameter error!\n", __FUNCTION__, addr, cnts, mapLen);
+			err = -EINVAL;
+			goto exit;
+		}
+
+		if (pre_efuse_map) {
+			if (rtw_efuse_map_read(padapter, addr, cnts, pre_efuse_map, RTW_EFUSE_BT) == _FAIL) {
+				RTW_INFO("%s: rtw_efuse_map_read Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				data = pre_efuse_map;
+				*extra = 0;
+				pextra = extra;
+				for (i = 0; i < cnts; i++)
+					pextra += sprintf(pextra, "0x%02X ", data[i]);
+			}
+		}
+	} else if (strcmp(tmp[0], "btrfkmap") == 0) {
+		static u8 order = 0;
+		u32 shift, cnt;
+		u32 blksz = 0x200; /* The size of one time show, default 512 */
+		u8 *efuse_data = NULL;
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_BT_GET_LOG_SIZE);
+
+ 		if (pre_efuse_map) {
+			if (rtw_efuse_get_shadow_map(padapter, pre_efuse_map, mapLen, RTW_EFUSE_BT) == _FAIL) {
+				RTW_INFO("%s: read BT fake map Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				efuse_data = pre_efuse_map;
+
+				_rtw_memset(extra, '\0', strlen(extra));
+				shift = blksz * order;
+				efuse_data += shift;
+				cnt = mapLen - shift;
+				if (cnt > blksz) {
+					cnt = blksz;
+					order++;
+				} else
+					order = 0;
+
+				sprintf(extra, "\n");
+				for (i = 0; i < cnt; i += 16) {
+					pextra = extra + strlen(extra);
+					pextra += sprintf(pextra, "0x%02x\t", shift + i);
+					for (j = 0; j < 8; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\t");
+					for (; j < 16; j++)
+						pextra += sprintf(pextra, "%02X ", efuse_data[i + j]);
+					pextra += sprintf(pextra, "\n");
+
+					if (strlen(extra) > 0x7FF)
+						break;
+				}
+				if ((shift + cnt) < mapLen)
+					pextra += sprintf(pextra, "\t...more (left:%d/%d)\n", mapLen - (shift + cnt), mapLen);
+			}
+		}
+	} else if (strcmp(tmp[0], "ableraw") == 0) {
+			rtw_efuse_get_available_size(padapter, &max_available_len, RTW_EFUSE_WIFI);
+			sprintf(extra, "[WIFI available raw size]= %d bytes\n", max_available_len);
+	} else if (strcmp(tmp[0], "btableraw") == 0) {
+			rtw_efuse_get_available_size(padapter, &max_available_len, RTW_EFUSE_BT);
+			sprintf(extra, "[BT available raw size]= %d bytes\n", max_available_len);
+	} else if (strcmp(tmp[0], "realraw") == 0) {
+		static u8 raw_order = 0;
+		u32 shift, cnt;
+		u32 blksz = 0x200; /* The size of one time show, default 512 */
+		u8 *realraw_map_data = NULL;
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_WIFI_GET_SIZE);
+
+ 		if (pre_efuse_map) {
+			if (rtw_efuse_raw_map_read (padapter, 0, mapLen, pre_efuse_map, RTW_EFUSE_WIFI) == _FAIL) {
+				RTW_INFO("%s: read wifi raw map Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				realraw_map_data = pre_efuse_map;
+
+				_rtw_memset(extra, '\0', strlen(extra));
+				shift = blksz * raw_order;
+				realraw_map_data += shift;
+				cnt = mapLen - shift;
+				if (cnt > blksz) {
+					cnt = blksz;
+					raw_order++;
+				} else
+					raw_order = 0;
+
+				sprintf(extra, "\n");
+				for (i = 0; i < cnt; i += 16) {
+					pextra = extra + strlen(extra);
+					pextra += sprintf(pextra, "0x%02x\t", shift + i);
+					for (j = 0; j < 8; j++)
+						pextra += sprintf(pextra, "%02X ", realraw_map_data[i + j]);
+					pextra += sprintf(pextra, "\t");
+					for (; j < 16; j++)
+						pextra += sprintf(pextra, "%02X ", realraw_map_data[i + j]);
+					pextra += sprintf(pextra, "\n");
+
+					if (strlen(extra) > 0x7FF)
+						break;
+				}
+				if ((shift + cnt) < mapLen)
+					pextra += sprintf(pextra, "\t...more (left:%d/%d)\n", mapLen - (shift + cnt), mapLen);
+				}
+		}
+	} else if (strcmp(tmp[0], "btrealraw")== 0) {
+		static u8 raw_order = 0;
+		u32 shift, cnt;
+		u32 blksz = 0x200; /* The size of one time show, default 512 */
+		u8 *realraw_map_data = NULL;
+
+		rtw_efuse_get_map_size(padapter, &mapLen, RTW_EFUSE_CMD_BT_GET_SIZE);
+
+ 		if (pre_efuse_map) {
+			if (rtw_efuse_raw_map_read (padapter, 0, mapLen, pre_efuse_map, RTW_EFUSE_BT) == _FAIL) {
+				RTW_INFO("%s: read bt raw map Fail!!\n", __FUNCTION__);
+				err = -EFAULT;
+			} else {
+				realraw_map_data = pre_efuse_map;
+
+				_rtw_memset(extra, '\0', strlen(extra));
+				shift = blksz * raw_order;
+				realraw_map_data += shift;
+				cnt = mapLen - shift;
+				if (cnt > blksz) {
+					cnt = blksz;
+					raw_order++;
+				} else
+					raw_order = 0;
+
+				sprintf(extra, "\n");
+				for (i = 0; i < cnt; i += 16) {
+					pextra = extra + strlen(extra);
+					pextra += sprintf(pextra, "0x%02x\t", shift + i);
+					for (j = 0; j < 8; j++)
+						pextra += sprintf(pextra, "%02X ", realraw_map_data[i + j]);
+					pextra += sprintf(pextra, "\t");
+					for (; j < 16; j++)
+						pextra += sprintf(pextra, "%02X ", realraw_map_data[i + j]);
+					pextra += sprintf(pextra, "\n");
+
+					if (strlen(extra) > 0x7FF)
+						break;
+				}
+				if ((shift + cnt) < mapLen)
+					pextra += sprintf(pextra, "\t...more (left:%d/%d)\n", mapLen - (shift + cnt), mapLen);
+				}
+		}
+	} else
+		sprintf(extra, "Command not found!");
+
+exit:
+	if (pre_efuse_map)
+		rtw_mfree(pre_efuse_map, RTW_MAX_EFUSE_MAP_LEN);
+	if (!err)
+		wrqu->data.length = strlen(extra);
+
+	*(extra +  wrqu->data.length) = '\0';
+	RTW_INFO("%s: strlen(extra) =%zu\n", __FUNCTION__, strlen(extra));
+	if (copy_to_user(wrqu->data.pointer, extra, wrqu->data.length))
+		err = -EFAULT;
+
+	return err;
+}
+
+
+int rtw_ioctl_efuse_set(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wdata, char *extra)
+{
+	struct iw_point *wrqu;
+	struct pwrctrl_priv *pwrctrlpriv ;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	u32 i = 0, j = 0, jj = 0, kk = 0;
+	u8 *setdata = NULL;
+	u8 *shadowmap = NULL;
+	u8 *setrawdata = NULL;
+	char *pch, *ptmp, *token, *tmp[3] = {0x00, 0x00, 0x00};
+	u16 addr = 0xFF, cnts = 0, max_available_len = 0;
+	u16 wifimaplen = 0;
+	int err = 0;
+	boolean bcmpchk = _TRUE;
+	u8 status = _SUCCESS;
+	u16 size = 0;
+	u8 bpg = true;
+
+	wrqu = (struct iw_point *)wdata;
+	pwrctrlpriv = adapter_to_pwrctl(padapter);
+
+	if (copy_from_user(extra, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	*(extra + wrqu->length) = '\0';
+
+	setdata = rtw_zmalloc(RTW_MAX_EFUSE_MAP_LEN);
+	if (setdata == NULL) {
+		err = -ENOMEM;
+		goto exit;
+	}
+
+	pch = extra;
+	RTW_INFO("%s: in=%s\n", __FUNCTION__, extra);
+
+	i = 0;
+	while ((token = strsep(&pch, ",")) != NULL) {
+		if (i > 2)
+			break;
+		tmp[i] = token;
+		i++;
+	}
+
+	/* tmp[0],[1],[2] */
+	/* wmap,addr,00e04c871200 */
+	if ((strcmp(tmp[0], "wmap") == 0) || (strcmp(tmp[0], "wlwfake") == 0)) {
+		if ((tmp[1] == NULL) || (tmp[2] == NULL)) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		addr = simple_strtoul(tmp[1], &ptmp, 16);
+		addr &= 0xFFF;
+
+		cnts = strlen(tmp[2]);
+		if (cnts % 2) {
+			err = -EINVAL;
+			goto exit;
+		}
+		cnts /= 2;
+		if (cnts == 0) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		RTW_INFO("%s: addr=0x%X\n", __FUNCTION__, addr);
+		RTW_INFO("%s: cnts=%d\n", __FUNCTION__, cnts);
+		RTW_INFO("%s: map data=%s\n", __FUNCTION__, tmp[2]);
+
+		for (jj = 0, kk = 0; jj < cnts; jj++, kk += 2)
+			setdata[jj] = key_2char2num(tmp[2][kk], tmp[2][kk + 1]);
+
+		err = rtw_efuse_get_map_size(padapter, &size, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+		max_available_len = size;
+
+		if (err == _FAIL) {
+			status = _FAIL;
+			goto exit;
+		}
+
+		if ((addr + cnts) > max_available_len) {
+			RTW_INFO("%s: addr(0x%X)+cnts(%d) parameter error!\n", __FUNCTION__, addr, cnts);
+			err = -EFAULT;
+			goto exit;
+		}
+
+		if (strcmp(tmp[0], "wlwfake") == 0)
+			bpg = false;
+
+		if (rtw_efuse_map_write(padapter, addr, cnts, setdata, RTW_EFUSE_WIFI, bpg) == _FAIL) {
+			RTW_INFO("%s: rtw_efuse_map_write error!!\n", __FUNCTION__);
+			err = -EFAULT;
+			goto exit;
+		}
+
+		*extra = 0;
+		if (bpg) {
+			RTW_INFO("%s: after rtw_efuse_map_write to _rtw_memcmp\n", __func__);
+			shadowmap = rtw_zmalloc(size);
+			if (shadowmap == NULL) {
+				RTW_INFO("%s: ShadowMapWiFi rtw_zmalloc Fail\n", __func__);
+				err = 0;
+				goto exit;
+			}
+			if (rtw_efuse_get_shadow_map(padapter, shadowmap, size,RTW_EFUSE_WIFI) == _SUCCESS) {
+
+				if (rtw_efuse_compare_data(padapter, addr, cnts, shadowmap, setdata, RTW_EFUSE_WIFI)) {
+					RTW_INFO("%s: WiFi write map afterf compare success\n", __FUNCTION__);
+					sprintf(extra, "WiFi write map compare OK\n");
+					err = 0;
+					goto exit;
+				} else {
+					RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "shadowmap :", shadowmap + addr, cnts);
+					RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "map_pged :", setdata, cnts);
+					sprintf(extra, "WiFi write map compare FAIL\n");
+					RTW_INFO("%s: WiFi write map compare Fail\n", __FUNCTION__);
+					err = 0;
+					goto exit;
+				}
+			}
+		} else {
+			sprintf(extra, "WiFi write fake map OK\n");
+			RTW_INFO("%s: WiFi write fake map OK\n", __FUNCTION__);
+			err = 0;
+			goto exit;
+		}
+	} else if ((strcmp(tmp[0], "btwmap") == 0) || (strcmp(tmp[0], "btwfake") == 0)) {
+		if ((tmp[1] == NULL) || (tmp[2] == NULL)) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		addr = simple_strtoul(tmp[1], &ptmp, 16);
+		addr &= 0xFFF;
+
+		cnts = strlen(tmp[2]);
+		if (cnts % 2) {
+			err = -EINVAL;
+			goto exit;
+		}
+		cnts /= 2;
+		if (cnts == 0) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		RTW_INFO("%s: addr=0x%X\n", __FUNCTION__, addr);
+		RTW_INFO("%s: cnts=%d\n", __FUNCTION__, cnts);
+		RTW_INFO("%s: map data=%s\n", __FUNCTION__, tmp[2]);
+
+		for (jj = 0, kk = 0; jj < cnts; jj++, kk += 2)
+			setdata[jj] = key_2char2num(tmp[2][kk], tmp[2][kk + 1]);
+
+		err = rtw_efuse_get_map_size(padapter, &size, RTW_EFUSE_CMD_BT_GET_LOG_SIZE);
+		max_available_len = size;
+
+		if (err == _FAIL) {
+			status = _FAIL;
+			goto exit;
+		}
+
+		if ((addr + cnts) > max_available_len) {
+			RTW_INFO("%s: addr(0x%X)+cnts(%d) parameter error!\n", __FUNCTION__, addr, cnts);
+			err = -EFAULT;
+			goto exit;
+		}
+
+		if (strcmp(tmp[0], "btwfake") == 0)
+			bpg = false;
+
+		if (rtw_efuse_map_write(padapter, addr, cnts, setdata, RTW_EFUSE_BT, bpg) == _FAIL) {
+			RTW_INFO("%s: rtw_efuse_map_write error!!\n", __FUNCTION__);
+			err = -EFAULT;
+			goto exit;
+		}
+
+		*extra = 0;
+		RTW_INFO("%s: after rtw_efuse_map_write to _rtw_memcmp\n", __func__);
+		if (bpg) {
+			shadowmap = rtw_zmalloc(size);
+			if (shadowmap == NULL) {
+				RTW_INFO("%s: shadowmap rtw_zmalloc Fail\n", __func__);
+				err = 0;
+				goto exit;
+			}
+			if (rtw_efuse_get_shadow_map(padapter, shadowmap, size,RTW_EFUSE_BT) == _SUCCESS) {
+
+				if (rtw_efuse_compare_data(padapter, addr, cnts, shadowmap, setdata, RTW_EFUSE_BT)) {
+					RTW_INFO("%s: BT write map afterf compare success\n", __func__);
+					sprintf(extra, "BT write map compare OK\n");
+					err = 0;
+					goto exit;
+				} else {
+					RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "shadowmap :", shadowmap, cnts);
+					RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "map_pged :", setdata, cnts);
+					sprintf(extra, "BT write map compare FAIL\n");
+					RTW_INFO("%s: BT write map compare Fail\n", __func__);
+					err = 0;
+					goto exit;
+				}
+			}
+		} else {
+			sprintf(extra, "BT write fake map OK\n");
+			RTW_INFO("%s: BT write fake map OK\n", __FUNCTION__);
+			err = 0;
+			goto exit;
+		}
+	} else if (strcmp(tmp[0], "btwraw") == 0) {
+		u8 reg2chkbt = 0;
+
+		if ((tmp[1] == NULL) || (tmp[2] == NULL)) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		addr = simple_strtoul(tmp[1], &ptmp, 16);
+		addr &= 0xFFF;
+
+		cnts = strlen(tmp[2]);
+		if (cnts % 2) {
+			err = -EINVAL;
+			goto exit;
+		}
+		cnts /= 2;
+		if (cnts == 0) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		RTW_INFO("%s: addr=0x%X\n", __FUNCTION__, addr);
+		RTW_INFO("%s: cnts=%d\n", __FUNCTION__, cnts);
+		RTW_INFO("%s: raw data=%s\n", __FUNCTION__, tmp[2]);
+
+		rtw_phl_write8(dvobj->phl, 0xa3, 0x05);
+		rtw_msleep_os(10);
+		reg2chkbt = rtw_phl_read8(dvobj->phl, 0xa0);
+		RTW_INFO("%s: btfk2map before read 0xa0 BT Status =0x%x\n", __func__,
+			rtw_phl_read32(dvobj->phl, 0xa0));
+
+		if (reg2chkbt != 0x04) {
+			sprintf(extra, "Error ! BT State not Active:0x%x ,can't Write \n" , reg2chkbt);
+			goto exit;
+		}
+
+		for (jj = 0, kk = 0; jj < cnts; jj++, kk += 2)
+			setdata[jj] = key_2char2num(tmp[2][kk], tmp[2][kk + 1]);
+
+		if (rtw_efuse_bt_write_raw_hidden(padapter, addr, cnts, setdata) == _FAIL) {
+			RTW_INFO("%s: rtw_efuse_bt_write_hidden error!!\n", __FUNCTION__);
+			err = -EFAULT;
+			goto exit;
+		}
+	} else if (strcmp(tmp[0], "wlfk2map") == 0) {
+		u16 mapsize = 0;
+		u8 *map_pged = NULL;
+
+		RTW_INFO("%s:do rtw_efuse_fake2map\n", __func__);
+		rtw_efuse_get_map_size(padapter, &mapsize, RTW_EFUSE_CMD_WIFI_GET_LOG_SIZE);
+		shadowmap = rtw_zmalloc(mapsize);
+		if (shadowmap == NULL) {
+			RTW_INFO("%s: shadowmap rtw_zmalloc Fail\n", __func__);
+			err = 0;
+			goto exit;
+		}
+		if (rtw_efuse_get_shadow_map(padapter, shadowmap, mapsize, RTW_EFUSE_WIFI) == _FAIL) {
+			RTW_INFO("%s:rtw_efuse_get_shadow_map Fail\n", __func__);
+			err = 0;
+			goto exit;
+		}
+
+		status = rtw_efuse_fake2map(padapter, RTW_EFUSE_WIFI);
+		if (status == _SUCCESS)
+			RTW_INFO("WIFI write map OK\n");
+		else
+			RTW_INFO("WIFI write map FAIL\n");
+
+		map_pged = rtw_zmalloc(mapsize);
+		if (map_pged == NULL) {
+			RTW_INFO("%s: map_pged rtw_zmalloc Fail\n", __func__);
+			err = 0;
+			goto exit;
+		}
+		if (rtw_efuse_map_read(padapter, 0x00, mapsize, map_pged, RTW_EFUSE_WIFI) == _SUCCESS) {
+
+			if (rtw_efuse_compare_data(padapter, 0x00, mapsize, map_pged, shadowmap, RTW_EFUSE_WIFI)) {
+				RTW_INFO("%s: WiFi write map afterf compare success\n", __func__);
+				sprintf(extra, "WiFi write map compare OK\n");
+			} else {
+				RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "shadowmap :", shadowmap, mapsize);
+				RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "map_pged :", map_pged, mapsize);
+				sprintf(extra, "WiFi write map compare FAIL\n");
+				RTW_INFO("%s: WiFi write map compare Fail\n", __func__);
+			}
+		}
+
+		if (map_pged)
+			rtw_mfree(map_pged, mapsize);
+		err = 0;
+		goto exit;
+	} else if (strcmp(tmp[0], "btfk2map") == 0) {
+		u8 reg2chkbt = 0;
+		u16 mapsize = 0;
+		u8 *map_pged = NULL;
+
+		RTW_INFO("%s:do rtw_efuse_fake2map\n", __func__);
+		rtw_efuse_get_map_size(padapter, &mapsize, RTW_EFUSE_CMD_BT_GET_LOG_SIZE);
+		shadowmap = rtw_zmalloc(mapsize);
+		if (shadowmap == NULL) {
+			RTW_INFO("%s: shadowmap rtw_zmalloc Fail\n", __func__);
+			err = 0;
+			goto exit;
+		}
+		if (rtw_efuse_get_shadow_map(padapter, shadowmap, mapsize, RTW_EFUSE_BT) == _FAIL) {
+			RTW_INFO("%s:rtw_efuse_get_shadow_map Fail\n", __func__);
+			err = 0;
+			goto exit;
+		}
+
+		rtw_phl_write8(dvobj->phl, 0xa3, 0x05);
+		rtw_msleep_os(10);
+		reg2chkbt = rtw_phl_read8(dvobj->phl, 0xa0);
+		RTW_INFO("%s: btfk2map before read 0xa0 BT Status =0x%x\n", __func__,
+			rtw_phl_read32(dvobj->phl, 0xa0));
+
+		if (reg2chkbt != 0x04) {
+			sprintf(extra, "Error ! BT State not Active:0x%x ,can't Write \n" , reg2chkbt);
+			goto exit;
+		}
+
+		status = rtw_efuse_fake2map(padapter, RTW_EFUSE_BT);
+		if (status == _SUCCESS)
+			RTW_INFO("BT write map OK\n");
+		else
+			RTW_INFO("BT write map FAIL\n");
+
+		map_pged = rtw_zmalloc(mapsize);
+		if (map_pged == NULL) {
+			RTW_INFO("%s: map_pged rtw_zmalloc Fail\n", __func__);
+			err = 0;
+			goto exit;
+		}
+
+		if (rtw_efuse_map_read(padapter, 0x00, mapsize, map_pged, RTW_EFUSE_BT) == _SUCCESS) {
+
+			if (rtw_efuse_compare_data(padapter, 0x00, mapsize, map_pged, shadowmap, RTW_EFUSE_BT)) {
+				RTW_INFO("%s: BT write map afterf compare success\n", __func__);
+				sprintf(extra, "BT write map compare OK\n");
+			} else {
+				RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "shadowmap :", shadowmap, mapsize);
+				RTW_MAP_DUMP_SEL(RTW_DBGDUMP, "map_pged :", map_pged, mapsize);
+				sprintf(extra, "BT write map compare FAIL\n");
+				RTW_INFO("%s: BT write map compare Fail\n", __func__);
+			}
+		}
+
+		if (map_pged)
+			rtw_mfree(map_pged, mapsize);
+
+		err = 0;
+		goto exit;
+	} else if (strncmp(tmp[0], "update", 6) == 0) {
+		if (rtw_efuse_renew_update(padapter, RTW_EFUSE_WIFI) == _FAIL) {
+			RTW_INFO("%s: rtw_efuse_renew_update error!!\n", __FUNCTION__);
+			sprintf(extra, "WIFI update FAIL\n");
+			err = -EFAULT;
+			goto exit;
+		} else
+			sprintf(extra, "WIFI update OK\n");
+
+	}
+exit:
+	if (setdata)
+		rtw_mfree(setdata, RTW_MAX_EFUSE_MAP_LEN);
+	if (shadowmap)
+		rtw_mfree(shadowmap, size);
+
+	wrqu->length = strlen(extra);
+
+	return err;
+}
+
+int rtw_ioctl_efuse_file_map_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra)
+{
+	char *rtw_efuse_file_map_path;
+	u8 Status = 0;
+	_adapter *padapter= rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, ( wrqu->data.length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	extra[wrqu->data.length] = '\0';
+
+	rtw_efuse_file_map_path = extra;
+	if (rtw_is_file_readable(rtw_efuse_file_map_path) == _TRUE) {
+		RTW_INFO("%s do rtw_efuse_mask_file_read = %s!\n", __func__, rtw_efuse_file_map_path);
+		Status = rtw_efuse_map_file_load(padapter, rtw_efuse_file_map_path, RTW_EFUSE_WIFI);
+		if (Status == _TRUE) {
+			pmp_priv->bloadefusemap = _TRUE;
+			sprintf(extra, "efuse Map file file_read OK\n");
+		} else {
+			pmp_priv->bloadefusemap = _FALSE;
+			sprintf(extra, "efuse Map file file_read FAIL\n");
+		}
+	} else {
+		sprintf(extra, "efuse file readable FAIL\n");
+		RTW_INFO("%s rtw_is_file_readable fail!\n", __func__);
+	}
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_ioctl_efuse_file_mask_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra)
+{
+	char *rtw_efuse_file_map_path;
+	u8 Status = 0;
+	_adapter *padapter= rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, ( wrqu->data.length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	extra[wrqu->data.length] = '\0';
+
+	rtw_efuse_file_map_path = extra;
+	if (rtw_is_file_readable(rtw_efuse_file_map_path) == _TRUE) {
+		RTW_INFO("%s do rtw_efuse_mask_file_read = %s!\n", __func__, rtw_efuse_file_map_path);
+		Status = rtw_efuse_mask_file_load(padapter, rtw_efuse_file_map_path, RTW_EFUSE_WIFI);
+		if (Status == _TRUE) {
+			pmp_priv->bloadefusemap = _TRUE;
+			sprintf(extra, "efuse Mask file file_read OK\n");
+		} else {
+			pmp_priv->bloadefusemap = _FALSE;
+			sprintf(extra, "efuse Mask file file_read FAIL\n");
+		}
+	} else {
+		sprintf(extra, "efuse file readable FAIL\n");
+		RTW_INFO("%s rtw_is_file_readable fail!\n", __func__);
+	}
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_ioctl_efuse_bt_file_map_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra)
+{
+	char *rtw_efuse_file_map_path;
+	u8 Status = 0;
+	_adapter *padapter= rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, ( wrqu->data.length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	extra[wrqu->data.length] = '\0';
+
+	rtw_efuse_file_map_path = extra;
+	if (rtw_is_file_readable(rtw_efuse_file_map_path) == _TRUE) {
+		RTW_INFO("%s BT do rtw_efuse_mask_file_read = %s!\n", __func__, rtw_efuse_file_map_path);
+		Status = rtw_efuse_map_file_load(padapter, rtw_efuse_file_map_path, RTW_EFUSE_BT);
+		if (Status == _TRUE) {
+			pmp_priv->bloadefusemap = _TRUE;
+			sprintf(extra, "BT efuse Map file file_read OK\n");
+		} else {
+			pmp_priv->bloadefusemap = _FALSE;
+			sprintf(extra, "BT efuse Map file file_read FAIL\n");
+		}
+	} else {
+		sprintf(extra, "BT efuse file readable FAIL\n"
+				"indicate the file path and directory to CMD next space,\n"
+				"CMD format:rtwpriv wlan0 bt_efuse_file /xx/xx/xx/efuse.map\n");
+		RTW_INFO("%s BT rtw_is_file_readable fail!\n", __func__);
+	}
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_ioctl_efuse_bt_file_mask_load(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra)
+{
+	char *rtw_efuse_file_map_path;
+	u8 Status = 0;
+	_adapter *padapter= rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, ( wrqu->data.length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	extra[wrqu->data.length] = '\0';
+
+	rtw_efuse_file_map_path = extra;
+	if (rtw_is_file_readable(rtw_efuse_file_map_path) == _TRUE) {
+		RTW_INFO("%s BT mask do rtw_efuse_mask_file_read = %s!\n", __func__, rtw_efuse_file_map_path);
+		Status = rtw_efuse_mask_file_load(padapter, rtw_efuse_file_map_path, RTW_EFUSE_BT);
+		if (Status == _TRUE) {
+			pmp_priv->bloadefusemap = _TRUE;
+			sprintf(extra, "BT efuse Mask file file_read OK\n");
+		} else {
+			pmp_priv->bloadefusemap = _FALSE;
+			sprintf(extra, "BT efuse Mask file file_read FAIL\n");
+		}
+	} else {
+		sprintf(extra, "BT efuse file readable FAIL\n"
+			"indicate the file path and directory to CMD next space,\n"
+			"CMD format:rtwpriv wlan0 efuse_bt_mask /xx/xx/xx/efuse.mask\n");
+		RTW_INFO("%s BT rtw_is_file_readable fail!\n", __func__);
+	}
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+s8 rtw_efuse_get_map_from(_adapter *padapter)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 map_status = -1;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+	if (efuse_arg) {
+		_rtw_memset((void *)efuse_arg,
+				0,
+				sizeof(struct rtw_efuse_phl_arg));
+		rtw_efuse_cmd(padapter,
+				efuse_arg,
+				RTW_MP_EFUSE_CMD_WIFI_GET_MAP_FROM);
+
+		if (efuse_arg->cmd_ok && 
+			efuse_arg->status == RTW_PHL_STATUS_SUCCESS) {
+			map_status = efuse_arg->io_value;
+			RTW_INFO("%s: OK value:%d\n",
+				__func__, map_status);
+		} else {
+			map_status = -1;
+			RTW_INFO("%s: fail !\n", __func__);
+		}
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return map_status;
+}
+
+static u8 rtw_efuse_get_realraw(_adapter *padapter, u8 *map, u16 addr, u16 size, u8 efuse_type)
+{
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u8 res = _FAIL;
+
+	efuse_arg = _rtw_malloc(sizeof(struct rtw_efuse_phl_arg));
+	if (efuse_arg) {
+		efuse_arg->io_offset = addr;
+		efuse_arg->buf_len = size;
+		if (efuse_type == RTW_EFUSE_WIFI)
+		    rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_WIFI_GET_PHY_MAP);
+		else if (efuse_type == RTW_EFUSE_BT)
+			rtw_efuse_cmd(padapter, efuse_arg, RTW_EFUSE_CMD_BT_GET_PHY_MAP);
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS) {
+				_rtw_memcpy((void *)map, efuse_arg->poutbuf, size);
+				res = _SUCCESS;
+		} else
+				res = _FAIL;
+	}
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return res;
+}
+
+u8 rtw_efuse_raw_map_read(_adapter * adapter, u16 addr, u16 cnts, u8 *data, u8 efuse_type)
+{
+	struct dvobj_priv *d;
+	u8 *efuse = NULL;
+	u16 size, i;
+	int err = _FAIL;
+	u8 status = _SUCCESS;
+
+	if (efuse_type == RTW_EFUSE_WIFI)
+		err = rtw_efuse_get_map_size(adapter, &size, RTW_EFUSE_CMD_WIFI_GET_SIZE);
+	else if (efuse_type == RTW_EFUSE_BT)
+		err = rtw_efuse_get_map_size(adapter, &size, RTW_EFUSE_CMD_BT_GET_SIZE);
+
+	if (err == _FAIL) {
+		status = _FAIL;
+		RTW_INFO("get_efuse_size fail\n");
+		goto exit;
+	}
+	/* size error handle */
+	if ((addr + cnts) > size) {
+		if (addr < size)
+			cnts = size - addr;
+		else {
+			status = _FAIL;
+			RTW_INFO(" %s() ,addr + cnts) > size fail\n", __func__);
+			goto exit;
+		}
+	}
+
+	efuse = rtw_zmalloc(size);
+	if (efuse) {
+		err = rtw_efuse_get_realraw (adapter, efuse, addr, size, efuse_type);
+		if (err == _FAIL) {
+			rtw_mfree(efuse, size);
+			status = _FAIL;
+			RTW_INFO(" %s() , phl get realraw fail\n", __func__);
+			goto exit;
+		}
+		if (efuse) {
+			RTW_INFO(" %s() ,cp efuse to data\n", __func__);
+			_rtw_memcpy(data, efuse + addr, cnts);
+			rtw_mfree(efuse, size);
+		}
+	} else {
+			RTW_INFO(" %s() ,alloc efuse fail\n", __func__);
+			goto exit;
+	}
+	status = _SUCCESS;
+exit:
+
+	return status;
+}
+
+u8 rtw_efuse_bt_write_raw_hidden(_adapter * adapter, u16 addr, u16 cnts, u8 *data)
+{
+	u8 status = _SUCCESS;
+	struct rtw_efuse_phl_arg *efuse_arg = NULL;
+	u16 i = 0;
+
+	efuse_arg = _rtw_zmalloc(sizeof(struct rtw_efuse_phl_arg));
+
+	while (i < cnts) {
+		efuse_arg->io_type = 1;
+		efuse_arg->io_offset = addr + i;
+		efuse_arg->io_value = data[i];
+		rtw_efuse_cmd(adapter, efuse_arg, RTW_EFUSE_CMD_BT_WRITE_HIDDEN);
+		if (efuse_arg->cmd_ok && efuse_arg->status == RTW_PHL_STATUS_SUCCESS)
+			status = _SUCCESS;
+		else{
+			status = _FAIL;
+			break;
+		}
+		i++;
+	}
+exit :
+	if (efuse_arg)
+		_rtw_mfree(efuse_arg, sizeof(struct rtw_efuse_phl_arg));
+
+	return status;
+}
+
+#endif /*#if defined(CONFIG_MP_INCLUDED)*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_linux.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_linux.c
new file mode 100644
index 000000000000..3f0b461ed9e2
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_linux.c
@@ -0,0 +1,8412 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _IOCTL_LINUX_C_
+
+#include <drv_types.h>
+#include <rtw_mp.h>
+
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27))
+#define  iwe_stream_add_event(a, b, c, d, e)  iwe_stream_add_event(b, c, d, e)
+#define  iwe_stream_add_point(a, b, c, d, e)  iwe_stream_add_point(b, c, d, e)
+#endif
+
+#ifdef CONFIG_80211N_HT
+extern int rtw_ht_enable;
+#endif
+
+
+#define RTL_IOCTL_WPA_SUPPLICANT	(SIOCIWFIRSTPRIV+30)
+
+#define SCAN_ITEM_SIZE 768
+#define MAX_CUSTOM_LEN 64
+#define RATE_COUNT 4
+#define MAX_SCAN_BUFFER_LEN 65535
+
+#ifdef CONFIG_GLOBAL_UI_PID
+extern int ui_pid[3];
+#endif
+
+/* combo scan */
+#define WEXT_CSCAN_AMOUNT 9
+#define WEXT_CSCAN_BUF_LEN		360
+#define WEXT_CSCAN_HEADER		"CSCAN S\x01\x00\x00S\x00"
+#define WEXT_CSCAN_HEADER_SIZE		12
+#define WEXT_CSCAN_SSID_SECTION		'S'
+#define WEXT_CSCAN_CHANNEL_SECTION	'C'
+#define WEXT_CSCAN_NPROBE_SECTION	'N'
+#define WEXT_CSCAN_ACTV_DWELL_SECTION	'A'
+#define WEXT_CSCAN_PASV_DWELL_SECTION	'P'
+#define WEXT_CSCAN_HOME_DWELL_SECTION	'H'
+#define WEXT_CSCAN_TYPE_SECTION		'T'
+
+
+extern u8 key_2char2num(u8 hch, u8 lch);
+extern u8 str_2char2num(u8 hch, u8 lch);
+extern void macstr2num(u8 *dst, u8 *src);
+extern u8 convert_ip_addr(u8 hch, u8 mch, u8 lch);
+
+u32 rtw_rates[] = {1000000, 2000000, 5500000, 11000000,
+	6000000, 9000000, 12000000, 18000000, 24000000, 36000000, 48000000, 54000000};
+
+/**
+ * hwaddr_aton - Convert ASCII string to MAC address
+ * @txt: MAC address as a string (e.g., "00:11:22:33:44:55")
+ * @addr: Buffer for the MAC address (ETH_ALEN = 6 bytes)
+ * Returns: 0 on success, -1 on failure (e.g., string not a MAC address)
+ */
+static int hwaddr_aton_i(const char *txt, u8 *addr)
+{
+	int i;
+
+	for (i = 0; i < 6; i++) {
+		int a, b;
+
+		a = hex2num_i(*txt++);
+		if (a < 0)
+			return -1;
+		b = hex2num_i(*txt++);
+		if (b < 0)
+			return -1;
+		*addr++ = (a << 4) | b;
+		if (i < 5 && *txt++ != ':')
+			return -1;
+	}
+
+	return 0;
+}
+#ifdef CONFIG_RTW_ANDROID
+static void indicate_wx_custom_event(_adapter *padapter, char *msg)
+{
+	u8 *buff;
+	union iwreq_data wrqu;
+
+	if (strlen(msg) > IW_CUSTOM_MAX) {
+		RTW_INFO("%s strlen(msg):%zu > IW_CUSTOM_MAX:%u\n", __FUNCTION__ , strlen(msg), IW_CUSTOM_MAX);
+		return;
+	}
+
+	buff = rtw_zmalloc(IW_CUSTOM_MAX + 1);
+	if (!buff)
+		return;
+
+	_rtw_memcpy(buff, msg, strlen(msg));
+
+	_rtw_memset(&wrqu, 0, sizeof(wrqu));
+	wrqu.data.length = strlen(msg);
+
+	RTW_INFO("%s %s\n", __FUNCTION__, buff);
+#ifndef CONFIG_IOCTL_CFG80211
+	wireless_send_event(padapter->pnetdev, IWEVCUSTOM, &wrqu, buff);
+#endif
+
+	rtw_mfree(buff, IW_CUSTOM_MAX + 1);
+
+}
+#endif
+
+#if 0
+static void request_wps_pbc_event(_adapter *padapter)
+{
+	u8 *buff, *p;
+	union iwreq_data wrqu;
+
+
+	buff = rtw_malloc(IW_CUSTOM_MAX);
+	if (!buff)
+		return;
+
+	_rtw_memset(buff, 0, IW_CUSTOM_MAX);
+
+	p = buff;
+
+	p += sprintf(p, "WPS_PBC_START.request=TRUE");
+
+	_rtw_memset(&wrqu, 0, sizeof(wrqu));
+
+	wrqu.data.length = p - buff;
+
+	wrqu.data.length = (wrqu.data.length < IW_CUSTOM_MAX) ? wrqu.data.length : IW_CUSTOM_MAX;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+#ifndef CONFIG_IOCTL_CFG80211
+	wireless_send_event(padapter->pnetdev, IWEVCUSTOM, &wrqu, buff);
+#endif
+
+	if (buff)
+		rtw_mfree(buff, IW_CUSTOM_MAX);
+
+}
+#endif
+
+#ifdef CONFIG_SUPPORT_HW_WPS_PBC
+void rtw_request_wps_pbc_event(_adapter *padapter)
+{
+#ifdef RTK_DMP_PLATFORM
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 12))
+	kobject_uevent(&padapter->pnetdev->dev.kobj, KOBJ_NET_PBC);
+#else
+	kobject_hotplug(&padapter->pnetdev->class_dev.kobj, KOBJ_NET_PBC);
+#endif
+#else
+
+	if (padapter->pid[0] == 0) {
+		/*	0 is the default value and it means the application monitors the HW PBC doesn't privde its pid to driver. */
+		return;
+	}
+
+	rtw_signal_process(padapter->pid[0], SIGUSR1);
+
+#endif
+
+	rtw_led_control(padapter, LED_CTL_START_WPS_BOTTON);
+}
+#endif/* #ifdef CONFIG_SUPPORT_HW_WPS_PBC */
+
+void indicate_wx_scan_complete_event(_adapter *padapter)
+{
+	union iwreq_data wrqu;
+
+	_rtw_memset(&wrqu, 0, sizeof(union iwreq_data));
+
+	/* RTW_INFO("+rtw_indicate_wx_scan_complete_event\n"); */
+#ifndef CONFIG_IOCTL_CFG80211
+	wireless_send_event(padapter->pnetdev, SIOCGIWSCAN, &wrqu, NULL);
+#endif
+}
+
+
+void rtw_indicate_wx_assoc_event(_adapter *padapter)
+{
+	union iwreq_data wrqu;
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX		*pnetwork = (WLAN_BSSID_EX *)(&(pmlmeinfo->dev_network));
+
+	_rtw_memset(&wrqu, 0, sizeof(union iwreq_data));
+
+	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
+
+	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)
+		_rtw_memcpy(wrqu.ap_addr.sa_data, pnetwork->MacAddress, ETH_ALEN);
+	else
+		_rtw_memcpy(wrqu.ap_addr.sa_data, pmlmepriv->dev_cur_network.network.MacAddress, ETH_ALEN);
+
+	RTW_PRINT("assoc success\n");
+#ifndef CONFIG_IOCTL_CFG80211
+	wireless_send_event(padapter->pnetdev, SIOCGIWAP, &wrqu, NULL);
+#endif
+}
+
+void rtw_indicate_wx_disassoc_event(_adapter *padapter)
+{
+	union iwreq_data wrqu;
+
+	_rtw_memset(&wrqu, 0, sizeof(union iwreq_data));
+
+	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
+	_rtw_memset(wrqu.ap_addr.sa_data, 0, ETH_ALEN);
+
+#ifndef CONFIG_IOCTL_CFG80211
+	RTW_PRINT("indicate disassoc\n");
+	wireless_send_event(padapter->pnetdev, SIOCGIWAP, &wrqu, NULL);
+#endif
+}
+
+/*
+uint	rtw_is_cckrates_included(u8 *rate)
+{
+		u32	i = 0;
+
+		while(rate[i]!=0)
+		{
+			if  (  (((rate[i]) & 0x7f) == 2)	|| (((rate[i]) & 0x7f) == 4) ||
+			(((rate[i]) & 0x7f) == 11)  || (((rate[i]) & 0x7f) == 22) )
+			return _TRUE;
+			i++;
+		}
+
+		return _FALSE;
+}
+
+uint	rtw_is_cckratesonly_included(u8 *rate)
+{
+	u32 i = 0;
+
+	while(rate[i]!=0)
+	{
+			if  (  (((rate[i]) & 0x7f) != 2) && (((rate[i]) & 0x7f) != 4) &&
+				(((rate[i]) & 0x7f) != 11)  && (((rate[i]) & 0x7f) != 22) )
+			return _FALSE;
+			i++;
+	}
+
+	return _TRUE;
+}
+*/
+
+static inline char *iwe_stream_mac_addr_proess(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	/*  AP MAC address */
+	iwe->cmd = SIOCGIWAP;
+	iwe->u.ap_addr.sa_family = ARPHRD_ETHER;
+
+	_rtw_memcpy(iwe->u.ap_addr.sa_data, pnetwork->network.MacAddress, ETH_ALEN);
+	start = iwe_stream_add_event(info, start, stop, iwe, IW_EV_ADDR_LEN);
+	return start;
+}
+static inline char *iwe_stream_essid_proess(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+
+	/* Add the ESSID */
+	iwe->cmd = SIOCGIWESSID;
+	iwe->u.data.flags = 1;
+	iwe->u.data.length = min((u16)pnetwork->network.Ssid.SsidLength, (u16)32);
+	start = iwe_stream_add_point(info, start, stop, iwe, pnetwork->network.Ssid.Ssid);
+	return start;
+}
+
+static inline char *iwe_stream_chan_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	if (pnetwork->network.Configuration.DSConfig < 1 /*|| pnetwork->network.Configuration.DSConfig>14*/)
+		pnetwork->network.Configuration.DSConfig = 1;
+
+	/* Add frequency/channel */
+	iwe->cmd = SIOCGIWFREQ;
+	iwe->u.freq.m = rtw_bch2freq(pnetwork->network.Configuration.Band,
+				      pnetwork->network.Configuration.DSConfig) * 100000;
+	iwe->u.freq.e = 1;
+	iwe->u.freq.i = pnetwork->network.Configuration.DSConfig;
+	start = iwe_stream_add_event(info, start, stop, iwe, IW_EV_FREQ_LEN);
+	return start;
+}
+static inline char *iwe_stream_mode_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe, u16 cap)
+{
+	/* Add mode */
+	if (cap & (WLAN_CAPABILITY_IBSS | WLAN_CAPABILITY_BSS)) {
+		iwe->cmd = SIOCGIWMODE;
+		if (cap & WLAN_CAPABILITY_BSS)
+			iwe->u.mode = IW_MODE_MASTER;
+		else
+			iwe->u.mode = IW_MODE_ADHOC;
+
+		start = iwe_stream_add_event(info, start, stop, iwe, IW_EV_UINT_LEN);
+	}
+	return start;
+}
+static inline char *iwe_stream_encryption_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe, u16 cap)
+{
+
+	/* Add encryption capability */
+	iwe->cmd = SIOCGIWENCODE;
+	if (cap & WLAN_CAPABILITY_PRIVACY)
+		iwe->u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY;
+	else
+		iwe->u.data.flags = IW_ENCODE_DISABLED;
+	iwe->u.data.length = 0;
+	start = iwe_stream_add_point(info, start, stop, iwe, pnetwork->network.Ssid.Ssid);
+	return start;
+
+}
+
+static inline char *iwe_stream_protocol_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	u16 ht_cap = _FALSE, vht_cap = _FALSE;
+	u32 ht_ielen = 0, vht_ielen = 0;
+	char *p;
+	u8 ie_offset = (pnetwork->network.Reserved[0] == BSS_TYPE_PROB_REQ ? 0 : 12); /* Probe Request	 */
+
+#ifdef CONFIG_80211N_HT
+	/* parsing HT_CAP_IE	 */
+	if(padapter->registrypriv.ht_enable && is_supported_ht(padapter->registrypriv.wireless_mode)) {
+		p = rtw_get_ie(&pnetwork->network.IEs[ie_offset], _HT_CAPABILITY_IE_, &ht_ielen, pnetwork->network.IELength - ie_offset);
+		if (p && ht_ielen > 0)
+			ht_cap = _TRUE;
+	}
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	/* parsing VHT_CAP_IE */
+	if(is_supported_vht(padapter->registrypriv.wireless_mode)) {
+		p = rtw_get_ie(&pnetwork->network.IEs[ie_offset], EID_VHTCapability, &vht_ielen, pnetwork->network.IELength - ie_offset);
+		if (p && vht_ielen > 0)
+			vht_cap = _TRUE;
+	}
+#endif
+	/* Add the protocol name */
+	iwe->cmd = SIOCGIWNAME;
+	if ((rtw_is_cckratesonly_included((u8 *)&pnetwork->network.SupportedRates)) == _TRUE) {
+		if (ht_cap == _TRUE)
+			snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11bn");
+		else
+			snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11b");
+	} else if ((rtw_is_cckrates_included((u8 *)&pnetwork->network.SupportedRates)) == _TRUE) {
+		if (ht_cap == _TRUE)
+			snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11bgn");
+		else
+			snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11bg");
+	} else {
+		if (pnetwork->network.Configuration.DSConfig > 14) {
+			#ifdef CONFIG_80211AC_VHT
+			if (vht_cap == _TRUE)
+				snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11AC");
+			else
+			#endif
+			{
+				if (ht_cap == _TRUE)
+					snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11an");
+				else
+					snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11a");
+			}
+		} else {
+			if (ht_cap == _TRUE)
+				snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11gn");
+			else
+				snprintf(iwe->u.name, IFNAMSIZ, "IEEE 802.11g");
+		}
+	}
+	start = iwe_stream_add_event(info, start, stop, iwe, IW_EV_CHAR_LEN);
+	return start;
+}
+
+static inline char *iwe_stream_rate_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	u32 ht_ielen = 0, vht_ielen = 0;
+	char *p;
+	u16 max_rate = 0, rate, ht_cap = _FALSE, vht_cap = _FALSE;
+	u32 i = 0;
+	u8 bw_40MHz = 0, short_GI = 0, bw_160MHz = 0, vht_highest_rate = 0;
+	u16 mcs_rate = 0, vht_data_rate = 0;
+	char custom[MAX_CUSTOM_LEN] = {0};
+	u8 ie_offset = (pnetwork->network.Reserved[0] == BSS_TYPE_PROB_REQ ? 0 : 12); /* Probe Request	 */
+
+	/* parsing HT_CAP_IE	 */
+	if(is_supported_ht(padapter->registrypriv.wireless_mode)) {
+		p = rtw_get_ie(&pnetwork->network.IEs[ie_offset], _HT_CAPABILITY_IE_, &ht_ielen, pnetwork->network.IELength - ie_offset);
+		if (p && ht_ielen > 0) {
+			struct rtw_ieee80211_ht_cap *pht_capie;
+			ht_cap = _TRUE;
+			pht_capie = (struct rtw_ieee80211_ht_cap *)(p + 2);
+			_rtw_memcpy(&mcs_rate , pht_capie->supp_mcs_set, 2);
+			bw_40MHz = (pht_capie->cap_info & IEEE80211_HT_CAP_SUP_WIDTH) ? 1 : 0;
+			short_GI = (pht_capie->cap_info & (IEEE80211_HT_CAP_SGI_20 | IEEE80211_HT_CAP_SGI_40)) ? 1 : 0;
+		}
+	}
+#ifdef CONFIG_80211AC_VHT
+	/* parsing VHT_CAP_IE */
+	if(is_supported_vht(padapter->registrypriv.wireless_mode)){
+		p = rtw_get_ie(&pnetwork->network.IEs[ie_offset], EID_VHTCapability, &vht_ielen, pnetwork->network.IELength - ie_offset);
+		if (p && vht_ielen > 0) {
+			u8	mcs_map[2];
+
+			vht_cap = _TRUE;
+			bw_160MHz = GET_VHT_CAPABILITY_ELE_CHL_WIDTH(p + 2);
+			if (bw_160MHz)
+				short_GI = GET_VHT_CAPABILITY_ELE_SHORT_GI160M(p + 2);
+			else
+				short_GI = GET_VHT_CAPABILITY_ELE_SHORT_GI80M(p + 2);
+
+			_rtw_memcpy(mcs_map, GET_VHT_CAPABILITY_ELE_TX_MCS(p + 2), 2);
+
+			vht_highest_rate = rtw_get_vht_highest_rate(mcs_map);
+			vht_data_rate = rtw_vht_mcs_to_data_rate(CHANNEL_WIDTH_80, short_GI, vht_highest_rate);
+		}
+	}
+#endif
+
+	/*Add basic and extended rates */
+	p = custom;
+	p += snprintf(p, MAX_CUSTOM_LEN - (p - custom), " Rates (Mb/s): ");
+	while (pnetwork->network.SupportedRates[i] != 0) {
+		rate = pnetwork->network.SupportedRates[i] & 0x7F;
+		if (rate > max_rate)
+			max_rate = rate;
+		p += snprintf(p, MAX_CUSTOM_LEN - (p - custom),
+			      "%d%s ", rate >> 1, (rate & 1) ? ".5" : "");
+		i++;
+	}
+#ifdef CONFIG_80211AC_VHT
+	if (vht_cap == _TRUE)
+		max_rate = vht_data_rate;
+	else
+#endif
+		if (ht_cap == _TRUE) {
+			if (mcs_rate & 0x8000) /* MCS15 */
+				max_rate = (bw_40MHz) ? ((short_GI) ? 300 : 270) : ((short_GI) ? 144 : 130);
+
+			else if (mcs_rate & 0x0080) /* MCS7 */
+				max_rate = (bw_40MHz) ? ((short_GI) ? 150 : 135) : ((short_GI) ? 72 : 65);
+			else { /* default MCS7 */
+				/* RTW_INFO("wx_get_scan, mcs_rate_bitmap=0x%x\n", mcs_rate); */
+				max_rate = (bw_40MHz) ? ((short_GI) ? 150 : 135) : ((short_GI) ? 72 : 65);
+			}
+
+			max_rate = max_rate * 2; /* Mbps/2;		 */
+		}
+
+	iwe->cmd = SIOCGIWRATE;
+	iwe->u.bitrate.fixed = iwe->u.bitrate.disabled = 0;
+	iwe->u.bitrate.value = max_rate * 500000;
+	start = iwe_stream_add_event(info, start, stop, iwe, IW_EV_PARAM_LEN);
+	return start ;
+}
+
+static inline char *iwe_stream_wpa_wpa2_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	int buf_size = MAX_WPA_IE_LEN * 2;
+	/* u8 pbuf[buf_size]={0};	 */
+	u8 *pbuf = rtw_zmalloc(buf_size);
+
+	u8 wpa_ie[255] = {0}, rsn_ie[255] = {0};
+	u16 i, wpa_len = 0, rsn_len = 0;
+	u8 *p;
+	sint out_len = 0;
+
+
+	if (pbuf) {
+		p = pbuf;
+
+		/* parsing WPA/WPA2 IE */
+		if (pnetwork->network.Reserved[0] != BSS_TYPE_PROB_REQ) { /* Probe Request */
+			out_len = rtw_get_sec_ie(pnetwork->network.IEs , pnetwork->network.IELength, rsn_ie, &rsn_len, wpa_ie, &wpa_len);
+
+			if (wpa_len > 0) {
+
+				_rtw_memset(pbuf, 0, buf_size);
+				p += sprintf(p, "wpa_ie=");
+				for (i = 0; i < wpa_len; i++)
+					p += sprintf(p, "%02x", wpa_ie[i]);
+
+				if (wpa_len > 100) {
+					printk("-----------------Len %d----------------\n", wpa_len);
+					for (i = 0; i < wpa_len; i++)
+						printk("%02x ", wpa_ie[i]);
+					printk("\n");
+					printk("-----------------Len %d----------------\n", wpa_len);
+				}
+
+				_rtw_memset(iwe, 0, sizeof(*iwe));
+				iwe->cmd = IWEVCUSTOM;
+				iwe->u.data.length = strlen(pbuf);
+				start = iwe_stream_add_point(info, start, stop, iwe, pbuf);
+
+				_rtw_memset(iwe, 0, sizeof(*iwe));
+				iwe->cmd = IWEVGENIE;
+				iwe->u.data.length = wpa_len;
+				start = iwe_stream_add_point(info, start, stop, iwe, wpa_ie);
+			}
+			if (rsn_len > 0) {
+
+				_rtw_memset(pbuf, 0, buf_size);
+				p += sprintf(p, "rsn_ie=");
+				for (i = 0; i < rsn_len; i++)
+					p += sprintf(p, "%02x", rsn_ie[i]);
+				_rtw_memset(iwe, 0, sizeof(*iwe));
+				iwe->cmd = IWEVCUSTOM;
+				iwe->u.data.length = strlen(pbuf);
+				start = iwe_stream_add_point(info, start, stop, iwe, pbuf);
+
+				_rtw_memset(iwe, 0, sizeof(*iwe));
+				iwe->cmd = IWEVGENIE;
+				iwe->u.data.length = rsn_len;
+				start = iwe_stream_add_point(info, start, stop, iwe, rsn_ie);
+			}
+		}
+
+		rtw_mfree(pbuf, buf_size);
+	}
+	return start;
+}
+
+static inline char *iwe_stream_wps_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	/* parsing WPS IE */
+	uint cnt = 0, total_ielen;
+	u8 *wpsie_ptr = NULL;
+	uint wps_ielen = 0;
+	u8 ie_offset = (pnetwork->network.Reserved[0] == BSS_TYPE_PROB_REQ ? 0 : 12);
+
+	u8 *ie_ptr = pnetwork->network.IEs + ie_offset;
+	total_ielen = pnetwork->network.IELength - ie_offset;
+
+	if (pnetwork->network.Reserved[0] == BSS_TYPE_PROB_REQ) { /* Probe Request */
+		ie_ptr = pnetwork->network.IEs;
+		total_ielen = pnetwork->network.IELength;
+	} else { /* Beacon or Probe Respones */
+		ie_ptr = pnetwork->network.IEs + _FIXED_IE_LENGTH_;
+		total_ielen = pnetwork->network.IELength - _FIXED_IE_LENGTH_;
+	}
+	while (cnt < total_ielen) {
+		if (rtw_is_wps_ie(&ie_ptr[cnt], &wps_ielen) && (wps_ielen > 2)) {
+			wpsie_ptr = &ie_ptr[cnt];
+			iwe->cmd = IWEVGENIE;
+			iwe->u.data.length = (u16)wps_ielen;
+			start = iwe_stream_add_point(info, start, stop, iwe, wpsie_ptr);
+		}
+		cnt += ie_ptr[cnt + 1] + 2; /* goto next */
+	}
+	return start;
+}
+
+static inline char *iwe_stream_wapi_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+#ifdef CONFIG_WAPI_SUPPORT
+	char *p;
+
+	if (pnetwork->network.Reserved[0] != BSS_TYPE_PROB_REQ) { /* Probe Request */
+		sint out_len_wapi = 0;
+		/* here use static for stack size */
+		static u8 buf_wapi[MAX_WAPI_IE_LEN * 2] = {0};
+		static u8 wapi_ie[MAX_WAPI_IE_LEN] = {0};
+		u16 wapi_len = 0;
+		u16  i;
+
+		out_len_wapi = rtw_get_wapi_ie(pnetwork->network.IEs , pnetwork->network.IELength, wapi_ie, &wapi_len);
+
+		RTW_INFO("rtw_wx_get_scan: %s ", pnetwork->network.Ssid.Ssid);
+		RTW_INFO("rtw_wx_get_scan: ssid = %d ", wapi_len);
+
+
+		if (wapi_len > 0) {
+			p = buf_wapi;
+			/* _rtw_memset(buf_wapi, 0, MAX_WAPI_IE_LEN*2); */
+			p += sprintf(p, "wapi_ie=");
+			for (i = 0; i < wapi_len; i++)
+				p += sprintf(p, "%02x", wapi_ie[i]);
+
+			_rtw_memset(iwe, 0, sizeof(*iwe));
+			iwe->cmd = IWEVCUSTOM;
+			iwe->u.data.length = strlen(buf_wapi);
+			start = iwe_stream_add_point(info, start, stop, iwe, buf_wapi);
+
+			_rtw_memset(iwe, 0, sizeof(*iwe));
+			iwe->cmd = IWEVGENIE;
+			iwe->u.data.length = wapi_len;
+			start = iwe_stream_add_point(info, start, stop, iwe, wapi_ie);
+		}
+	}
+#endif/* #ifdef CONFIG_WAPI_SUPPORT */
+	return start;
+}
+
+static inline char   *iwe_stream_rssi_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	u8 ss, sq;
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	/* Add quality statistics */
+	iwe->cmd = IWEVQUAL;
+	iwe->u.qual.updated = IW_QUAL_QUAL_UPDATED | IW_QUAL_LEVEL_UPDATED
+#if 0 /*def CONFIG_BACKGROUND_NOISE_MONITOR*/
+			      | IW_QUAL_NOISE_UPDATED
+#else
+			      | IW_QUAL_NOISE_INVALID
+#endif
+#ifdef CONFIG_SIGNAL_DISPLAY_DBM
+			      | IW_QUAL_DBM
+#endif
+			      ;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE &&
+	    is_same_network(&pmlmepriv->dev_cur_network.network, &pnetwork->network)) {
+		ss = padapter->recvinfo.signal_strength;
+		sq = padapter->recvinfo.signal_qual;
+	} else {
+		ss = pnetwork->network.PhyInfo.SignalStrength;
+		sq = pnetwork->network.PhyInfo.SignalQuality;
+	}
+
+
+#ifdef CONFIG_SIGNAL_DISPLAY_DBM
+	iwe->u.qual.level = (u8) rtw_phl_rssi_to_dbm(ss); /* dbm */
+#else
+	iwe->u.qual.level = (u8)ss; /* % */
+#endif
+
+	iwe->u.qual.qual = (u8)sq;   /* signal quality */
+
+#ifdef CONFIG_PLATFORM_ROCKCHIPS
+	iwe->u.qual.noise = -100; /* noise level suggest by zhf@rockchips */
+#else
+	iwe->u.qual.noise = 0; /* noise level */
+#endif /* CONFIG_PLATFORM_ROCKCHIPS */
+
+	/* RTW_INFO("iqual=%d, ilevel=%d, inoise=%d, iupdated=%d\n", iwe.u.qual.qual, iwe.u.qual.level , iwe.u.qual.noise, iwe.u.qual.updated); */
+
+	start = iwe_stream_add_event(info, start, stop, iwe, IW_EV_QUAL_LEN);
+	return start;
+}
+
+static inline char   *iwe_stream_net_rsv_process(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop, struct iw_event *iwe)
+{
+	u8 buf[32] = {0};
+	u8 *p, *pos;
+	p = buf;
+	pos = pnetwork->network.Reserved;
+
+	p += sprintf(p, "fm=%02X%02X", pos[1], pos[0]);
+	_rtw_memset(iwe, 0, sizeof(*iwe));
+	iwe->cmd = IWEVCUSTOM;
+	iwe->u.data.length = strlen(buf);
+	start = iwe_stream_add_point(info, start, stop, iwe, buf);
+	return start;
+}
+
+static char *translate_scan(_adapter *padapter,
+		struct iw_request_info *info, struct wlan_network *pnetwork,
+		char *start, char *stop)
+{
+	struct iw_event iwe;
+	u16 cap = 0;
+	_rtw_memset(&iwe, 0, sizeof(iwe));
+
+	start = iwe_stream_mac_addr_proess(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_essid_proess(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_protocol_process(padapter, info, pnetwork, start, stop, &iwe);
+	if (pnetwork->network.Reserved[0] == BSS_TYPE_PROB_REQ) /* Probe Request */
+		cap = 0;
+	else {
+		_rtw_memcpy((u8 *)&cap, rtw_get_capability_from_ie(pnetwork->network.IEs), 2);
+		cap = le16_to_cpu(cap);
+	}
+
+	start = iwe_stream_mode_process(padapter, info, pnetwork, start, stop, &iwe, cap);
+	start = iwe_stream_chan_process(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_encryption_process(padapter, info, pnetwork, start, stop, &iwe, cap);
+	start = iwe_stream_rate_process(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_wpa_wpa2_process(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_wps_process(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_wapi_process(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_rssi_process(padapter, info, pnetwork, start, stop, &iwe);
+	start = iwe_stream_net_rsv_process(padapter, info, pnetwork, start, stop, &iwe);
+
+	return start;
+}
+
+static int wpa_set_auth_algs(struct net_device *dev, u32 value)
+{
+	_adapter *padapter = (_adapter *) rtw_netdev_priv(dev);
+	int ret = 0;
+
+	if ((value & AUTH_ALG_SHARED_KEY) && (value & AUTH_ALG_OPEN_SYSTEM)) {
+		RTW_INFO("wpa_set_auth_algs, AUTH_ALG_SHARED_KEY and  AUTH_ALG_OPEN_SYSTEM [value:0x%x]\n", value);
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+		padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeAutoSwitch;
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+	} else if (value & AUTH_ALG_SHARED_KEY) {
+		RTW_INFO("wpa_set_auth_algs, AUTH_ALG_SHARED_KEY  [value:0x%x]\n", value);
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+
+#ifdef CONFIG_PLATFORM_MT53XX
+		padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeAutoSwitch;
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+#else
+		padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeShared;
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Shared;
+#endif
+	} else if (value & AUTH_ALG_OPEN_SYSTEM) {
+		RTW_INFO("wpa_set_auth_algs, AUTH_ALG_OPEN_SYSTEM\n");
+		/* padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled; */
+		if (padapter->securitypriv.ndisauthtype < Ndis802_11AuthModeWPAPSK) {
+#ifdef CONFIG_PLATFORM_MT53XX
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeAutoSwitch;
+			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+#else
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
+			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
+#endif
+		}
+
+	} else if (value & AUTH_ALG_LEAP)
+		RTW_INFO("wpa_set_auth_algs, AUTH_ALG_LEAP\n");
+	else {
+		RTW_INFO("wpa_set_auth_algs, error!\n");
+		ret = -EINVAL;
+	}
+
+	return ret;
+
+}
+
+static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param, u32 param_len)
+{
+	int ret = 0;
+	u32 wep_key_idx, wep_key_len;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+#endif /* CONFIG_P2P */
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_security_priv *lsecuritypriv = &padapter_link->securitypriv;
+	struct link_mlme_priv *lmlmepriv = &padapter_link->mlmepriv;
+
+
+	param->u.crypt.err = 0;
+	param->u.crypt.alg[IEEE_CRYPT_ALG_NAME_LEN - 1] = '\0';
+
+	if (param_len < (u32)((u8 *) param->u.crypt.key - (u8 *) param) + param->u.crypt.key_len) {
+		ret =  -EINVAL;
+		goto exit;
+	}
+
+	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
+	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
+
+		if (param->u.crypt.idx >= WEP_KEYS
+#ifdef CONFIG_IEEE80211W
+		    && param->u.crypt.idx > BIP_MAX_KEYID
+#endif /* CONFIG_IEEE80211W */
+		   ) {
+			ret = -EINVAL;
+			goto exit;
+		}
+	} else {
+#ifdef CONFIG_WAPI_SUPPORT
+		if (strcmp(param->u.crypt.alg, "SMS4"))
+#endif
+		{
+			ret = -EINVAL;
+			goto exit;
+		}
+	}
+
+	if (strcmp(param->u.crypt.alg, "WEP") == 0) {
+		RTW_INFO("wpa_set_encryption, crypt.alg = WEP\n");
+
+		wep_key_idx = param->u.crypt.idx;
+		wep_key_len = param->u.crypt.key_len;
+
+		if ((wep_key_idx >= WEP_KEYS) || (wep_key_len <= 0)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+
+		if (psecuritypriv->bWepDefaultKeyIdxSet == 0) {
+			/* wep default key has not been set, so use this key index as default key.*/
+
+			wep_key_len = wep_key_len <= 5 ? 5 : 13;
+
+			psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+			psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
+
+			if (wep_key_len == 13) {
+				psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+				psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
+			}
+
+			psecuritypriv->dot11PrivacyKeyIndex = wep_key_idx;
+		}
+
+		_rtw_memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), param->u.crypt.key, wep_key_len);
+
+		psecuritypriv->dot11DefKeylen[wep_key_idx] = wep_key_len;
+
+		psecuritypriv->key_mask |= BIT(wep_key_idx);
+
+		padapter->mlmeextpriv.mlmext_info.key_index = wep_key_idx;
+		goto exit;
+	}
+
+	if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) { /* 802_1x */
+		struct sta_info *psta, *pbcmc_sta;
+		struct sta_priv *pstapriv = &padapter->stapriv;
+
+		if (MLME_IS_STA(padapter) || MLME_IS_MP(padapter)) { /* sta mode */
+			psta = rtw_get_stainfo(pstapriv, get_link_bssid(lmlmepriv));
+			if (psta == NULL) {
+				/* DEBUG_ERR( ("Set wpa_set_encryption: Obtain Sta_info fail\n")); */
+			} else {
+				/* Jeff: don't disable ieee8021x_blocked while clearing key */
+				if (strcmp(param->u.crypt.alg, "none") != 0)
+					psta->ieee8021x_blocked = _FALSE;
+
+				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled) ||
+				    (padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled))
+					psta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+
+				if (param->u.crypt.set_tx == 1) { /* pairwise key */
+					RTW_INFO(FUNC_ADPT_FMT" set %s PTK idx:%u, len:%u\n"
+						, FUNC_ADPT_ARG(padapter), param->u.crypt.alg, param->u.crypt.idx, param->u.crypt.key_len);
+					_rtw_memcpy(psta->dot118021x_UncstKey.skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+					if (strcmp(param->u.crypt.alg, "TKIP") == 0) { /* set mic key */
+						_rtw_memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
+						_rtw_memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+						padapter->securitypriv.busetkipkey = _FALSE;
+					}
+					psta->dot11txpn.val = RTW_GET_LE64(param->u.crypt.seq);
+					psta->dot11rxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+					psta->bpairwise_key_installed = _TRUE;
+					rtw_setstakey_cmd(padapter, psta, UNICAST_KEY, _TRUE);
+
+				} else { /* group key */
+					if (strcmp(param->u.crypt.alg, "TKIP") == 0 || strcmp(param->u.crypt.alg, "CCMP") == 0) {
+						RTW_INFO(FUNC_ADPT_FMT" set %s GTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.alg, param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key,
+							(param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+						/* only TKIP group key need to install this */
+						if (param->u.crypt.key_len > 16) {
+							_rtw_memcpy(lsecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
+							_rtw_memcpy(lsecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+						}
+						lsecuritypriv->binstallGrpkey = _TRUE;
+						if (param->u.crypt.idx < 4)
+							_rtw_memcpy(lsecuritypriv->iv_seq[param->u.crypt.idx], param->u.crypt.seq, 8);
+						lsecuritypriv->dot118021XGrpKeyid = param->u.crypt.idx;
+						rtw_set_key(padapter, padapter_link, param->u.crypt.idx, 1, _TRUE);
+					#ifdef CONFIG_IEEE80211W
+					} else if (strcmp(param->u.crypt.alg, "BIP") == 0) {
+						RTW_INFO(FUNC_ADPT_FMT" set IGTK idx:%u, len:%u\n"
+							, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+						_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey,  param->u.crypt.key,
+							(param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+						lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+						lsecuritypriv->dot11wBIPrxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+						lsecuritypriv->binstallBIPkey = _TRUE;
+						rtw_set_key(padapter, padapter_link, param->u.crypt.idx, 1, _TRUE);
+					#endif /* CONFIG_IEEE80211W */
+
+					}
+
+					/* WPA/WPA2 key-handshake has completed */
+					clr_fwstate(pmlmepriv, WIFI_UNDER_KEY_HANDSHAKE);
+				}
+			}
+
+			pbcmc_sta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+			if (pbcmc_sta == NULL) {
+				/* DEBUG_ERR( ("Set OID_802_11_ADD_KEY: bcmc stainfo is null\n")); */
+			} else {
+				/* Jeff: don't disable ieee8021x_blocked while clearing key */
+				if (strcmp(param->u.crypt.alg, "none") != 0)
+					pbcmc_sta->ieee8021x_blocked = _FALSE;
+
+				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled) ||
+				    (padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled))
+					pbcmc_sta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+			}
+		} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) { /* adhoc mode */
+		}
+	}
+
+#ifdef CONFIG_WAPI_SUPPORT
+	if (strcmp(param->u.crypt.alg, "SMS4") == 0)
+		rtw_wapi_set_set_encryption(padapter, param);
+#endif
+
+exit:
+
+
+	return ret;
+}
+
+static int rtw_set_wpa_ie(_adapter *padapter, char *pie, unsigned short ielen)
+{
+	u8 *buf = NULL, *pos = NULL;
+	int group_cipher = 0, pairwise_cipher = 0;
+	u8 mfp_opt = MFP_NO;
+	int ret = 0;
+	u8 null_addr[] = {0, 0, 0, 0, 0, 0};
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+#endif /* CONFIG_P2P */
+
+	if ((ielen > MAX_WPA_IE_LEN) || (pie == NULL)) {
+		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
+		if (pie == NULL)
+			return ret;
+		else
+			return -EINVAL;
+	}
+
+	if (ielen) {
+		buf = rtw_zmalloc(ielen);
+		if (buf == NULL) {
+			ret =  -ENOMEM;
+			goto exit;
+		}
+
+		_rtw_memcpy(buf, pie , ielen);
+
+		/* dump */
+		{
+			int i;
+			RTW_INFO("\n wpa_ie(length:%d):\n", ielen);
+			for (i = 0; i < ielen; i = i + 8)
+				RTW_INFO("0x%.2x 0x%.2x 0x%.2x 0x%.2x 0x%.2x 0x%.2x 0x%.2x 0x%.2x\n", buf[i], buf[i + 1], buf[i + 2], buf[i + 3], buf[i + 4], buf[i + 5], buf[i + 6], buf[i + 7]);
+		}
+
+		pos = buf;
+		if (ielen < RSN_HEADER_LEN) {
+			ret  = -1;
+			goto exit;
+		}
+
+		if (rtw_parse_wpa_ie(buf, ielen, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
+			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPAPSK;
+			_rtw_memcpy(padapter->securitypriv.supplicant_ie, &buf[0], ielen);
+		}
+
+		if (rtw_parse_wpa2_ie(buf, ielen, &group_cipher, &pairwise_cipher, NULL, NULL, &mfp_opt, NULL) == _SUCCESS) {
+			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPA2PSK;
+			_rtw_memcpy(padapter->securitypriv.supplicant_ie, &buf[0], ielen);
+		}
+
+		if (group_cipher == 0)
+			group_cipher = WPA_CIPHER_NONE;
+		if (pairwise_cipher == 0)
+			pairwise_cipher = WPA_CIPHER_NONE;
+
+		switch (group_cipher) {
+		case WPA_CIPHER_NONE:
+			padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
+			break;
+		case WPA_CIPHER_WEP40:
+			padapter->securitypriv.dot118021XGrpPrivacy = _WEP40_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			break;
+		case WPA_CIPHER_TKIP:
+			padapter->securitypriv.dot118021XGrpPrivacy = _TKIP_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled;
+			break;
+		case WPA_CIPHER_CCMP:
+			padapter->securitypriv.dot118021XGrpPrivacy = _AES_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+			break;
+		case WPA_CIPHER_WEP104:
+			padapter->securitypriv.dot118021XGrpPrivacy = _WEP104_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			break;
+		}
+
+		switch (pairwise_cipher) {
+		case WPA_CIPHER_NONE:
+			padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
+			break;
+		case WPA_CIPHER_WEP40:
+			padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			break;
+		case WPA_CIPHER_TKIP:
+			padapter->securitypriv.dot11PrivacyAlgrthm = _TKIP_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled;
+			break;
+		case WPA_CIPHER_CCMP:
+			padapter->securitypriv.dot11PrivacyAlgrthm = _AES_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+			break;
+		case WPA_CIPHER_WEP104:
+			padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_;
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			break;
+		}
+
+		if (mfp_opt == MFP_INVALID) {
+			RTW_INFO(FUNC_ADPT_FMT" invalid MFP setting\n", FUNC_ADPT_ARG(padapter));
+			ret = -EINVAL;
+			goto exit;
+		}
+		padapter->securitypriv.mfp_opt = mfp_opt;
+
+		_clr_fwstate_(&padapter->mlmepriv, WIFI_UNDER_WPS);
+		{/* set wps_ie	 */
+			u16 cnt = 0;
+			u8 eid, wps_oui[4] = {0x0, 0x50, 0xf2, 0x04};
+
+			while (cnt < ielen) {
+				eid = buf[cnt];
+
+				if ((eid == _VENDOR_SPECIFIC_IE_) && (_rtw_memcmp(&buf[cnt + 2], wps_oui, 4) == _TRUE)) {
+					RTW_INFO("SET WPS_IE\n");
+
+					padapter->securitypriv.wps_ie_len = ((buf[cnt + 1] + 2) < MAX_WPS_IE_LEN) ? (buf[cnt + 1] + 2) : MAX_WPS_IE_LEN;
+
+					_rtw_memcpy(padapter->securitypriv.wps_ie, &buf[cnt], padapter->securitypriv.wps_ie_len);
+
+					set_fwstate(&padapter->mlmepriv, WIFI_UNDER_WPS);
+
+					cnt += buf[cnt + 1] + 2;
+
+					break;
+				} else {
+					cnt += buf[cnt + 1] + 2; /* goto next	 */
+				}
+			}
+		}
+
+		#ifdef CONFIG_RTW_MULTI_AP
+		padapter->multi_ap = rtw_get_multi_ap_ie_ext(buf, ielen) & MULTI_AP_BACKHAUL_STA;
+		if (padapter->multi_ap)
+			adapter_set_use_wds(padapter, 1);
+		#endif
+	}
+
+	/* TKIP and AES disallow multicast packets until installing group key */
+	if (padapter->securitypriv.dot11PrivacyAlgrthm == _TKIP_
+	    || padapter->securitypriv.dot11PrivacyAlgrthm == _TKIP_WTMIC_
+	    || padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)
+		/* WPS open need to enable multicast
+		 * || check_fwstate(&padapter->mlmepriv, WIFI_UNDER_WPS) == _TRUE) */
+		rtw_hal_set_hwreg(padapter, HW_VAR_OFF_RCR_AM, null_addr);
+
+
+exit:
+
+	if (buf)
+		rtw_mfree(buf, ielen);
+
+	return ret;
+}
+
+static int rtw_wx_get_name(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u32 ht_ielen = 0;
+	char *p;
+	u8 ht_cap = _FALSE, vht_cap = _FALSE, he_cap = _FALSE;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct	link_mlme_priv	*pmlmepriv = &(padapter_link->mlmepriv);
+	WLAN_BSSID_EX  *pcur_bss = &pmlmepriv->cur_network.network;
+	NDIS_802_11_RATES_EX *prates = NULL;
+
+
+
+	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE | WIFI_ADHOC_MASTER_STATE) == _TRUE) {
+		/* parsing HT_CAP_IE */
+		if( is_supported_ht(padapter->registrypriv.wireless_mode)&&(padapter->registrypriv.ht_enable)) {
+			p = rtw_get_ie(&pcur_bss->IEs[12], _HT_CAPABILITY_IE_, &ht_ielen, pcur_bss->IELength - 12);
+			if (p && ht_ielen > 0 )
+				ht_cap = _TRUE;
+		}
+#ifdef CONFIG_80211AC_VHT
+		if (is_supported_vht(padapter->registrypriv.wireless_mode) &&
+			(pmlmepriv->vhtpriv.vht_option == _TRUE))
+			vht_cap = _TRUE;
+#endif
+#ifdef CONFIG_80211AX_HE
+		if (is_supported_he(padapter->registrypriv.wireless_mode) &&
+			(pmlmepriv->hepriv.he_option == _TRUE))
+			he_cap = _TRUE;
+#endif
+
+		prates = &pcur_bss->SupportedRates;
+#ifdef CONFIG_80211AX_HE
+		if (he_cap == _TRUE)
+			snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11AX");
+		else
+#endif
+		if (rtw_is_cckratesonly_included((u8 *)prates) == _TRUE) {
+			if (ht_cap == _TRUE)
+				snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11bn");
+			else
+				snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11b");
+		} else if ((rtw_is_cckrates_included((u8 *)prates)) == _TRUE) {
+			if (ht_cap == _TRUE)
+				snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11bgn");
+			else {
+				if(padapter->registrypriv.wireless_mode & WLAN_MD_11G)
+					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11bg");
+				else
+					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11b");
+			}
+		} else {
+			if (pcur_bss->Configuration.DSConfig > 14) {
+#ifdef CONFIG_80211AC_VHT
+				if (vht_cap == _TRUE)
+					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11AC");
+				else
+#endif
+				{
+					if (ht_cap == _TRUE)
+						snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11an");
+					else
+						snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11a");
+				}
+			} else {
+				if (ht_cap == _TRUE)
+					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11gn");
+				else
+					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11g");
+			}
+		}
+	} else {
+		/* prates = &padapter->registrypriv.dev_network.SupportedRates; */
+		/* snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11g"); */
+		snprintf(wrqu->name, IFNAMSIZ, "unassociated");
+	}
+
+
+	return 0;
+}
+
+static int rtw_wx_set_freq(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	int exp = 1, freq = 0, div = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	if (wrqu->freq.m <= 1000) {
+		/* allow 2G/5G channels only */
+		enum band_type band = wrqu->freq.m <= 14 ? BAND_ON_24G : BAND_ON_5G;
+
+		if (wrqu->freq.flags == IW_FREQ_AUTO) {
+			if (rtw_chset_search_bch(adapter_to_chset(padapter), band, wrqu->freq.m) > 0) {
+				padapter_link->mlmeextpriv.chandef.band = band;
+				padapter_link->mlmeextpriv.chandef.chan = wrqu->freq.m;
+				RTW_INFO("%s: channel is auto, set to channel %d\n", __func__, wrqu->freq.m);
+			} else {
+				padapter_link->mlmeextpriv.chandef.band = BAND_ON_24G;
+				padapter_link->mlmeextpriv.chandef.chan = 1;
+				RTW_INFO("%s: channel is auto, Channel Plan don't match just set to channel 1\n", __func__);
+			}
+		} else {
+			padapter_link->mlmeextpriv.chandef.band = band;
+			padapter_link->mlmeextpriv.chandef.chan = wrqu->freq.m;
+			RTW_INFO("%s: set to channel %d\n", __func__, padapter_link->mlmeextpriv.chandef.chan);
+		}
+	} else {
+		while (wrqu->freq.e) {
+			exp *= 10;
+			wrqu->freq.e--;
+		}
+
+		freq = wrqu->freq.m;
+
+		while (!(freq % 10)) {
+			freq /= 10;
+			exp *= 10;
+		}
+
+		/* freq unit is MHz here */
+		div = 1000000 / exp;
+
+		if (div)
+			freq /= div;
+		else {
+			div = exp / 1000000;
+			freq *= div;
+		}
+
+		/* If freq is invalid, rtw_freq2ch() will return channel 1 */
+		padapter_link->mlmeextpriv.chandef.band = rtw_freq2band(freq);
+		padapter_link->mlmeextpriv.chandef.chan = rtw_freq2ch(freq);
+		RTW_INFO("%s: set to channel %d\n", __func__, padapter_link->mlmeextpriv.chandef.chan);
+	}
+	set_bch_bwmode(padapter,
+				padapter_link,
+				padapter_link->mlmeextpriv.chandef.band,
+				padapter_link->mlmeextpriv.chandef.chan,
+				CHAN_OFFSET_NO_EXT,
+				CHANNEL_WIDTH_20,
+				RFK_TYPE_FORCE_NOT_DO);
+
+	return 0;
+}
+
+static int rtw_wx_get_freq(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	WLAN_BSSID_EX  *pcur_bss = &padapter_link->mlmepriv.cur_network.network;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE && check_fwstate(pmlmepriv, WIFI_MONITOR_STATE) != _TRUE) {
+		wrqu->freq.m = rtw_bch2freq(pcur_bss->Configuration.Band, pcur_bss->Configuration.DSConfig) * 100000;
+		wrqu->freq.e = 1;
+		wrqu->freq.i = pcur_bss->Configuration.DSConfig;
+
+	} else {
+		wrqu->freq.m = rtw_bch2freq(padapter_link->mlmeextpriv.chandef.band, padapter_link->mlmeextpriv.chandef.chan) * 100000;
+		wrqu->freq.e = 1;
+		wrqu->freq.i = padapter_link->mlmeextpriv.chandef.chan;
+	}
+
+	return 0;
+}
+
+u32 wext_mode_to_rtw_mlme_state(union iwreq_data *wrqu)
+{
+	switch (wrqu->mode) {
+	#ifdef CONFIG_WIFI_MONITOR
+	case IW_MODE_MONITOR:
+		return WIFI_MONITOR_STATE;
+	#endif
+	case IW_MODE_ADHOC:
+		return WIFI_ADHOC_STATE;
+	#ifdef CONFIG_AP_MODE
+	case IW_MODE_MASTER:
+		return WIFI_AP_STATE;
+	#endif
+	case IW_MODE_INFRA:
+		return WIFI_STATION_STATE;
+
+	#ifdef CONFIG_RTW_MESH
+	case IW_MODE_MESH:
+		return WIFI_MESH_STATE;
+	#endif
+
+	case IW_MODE_AUTO:
+	default:
+		return WIFI_STATION_STATE;
+	}
+}
+
+static int rtw_wx_set_mode(struct net_device *dev, struct iw_request_info *a,
+			   union iwreq_data *wrqu, char *b)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	NDIS_802_11_NETWORK_INFRASTRUCTURE networkType ;
+	int ret = 0;
+
+	if (!rtw_hw_is_init_completed(adapter_to_dvobj(padapter))) {
+		ret = -EPERM;
+		goto exit;
+	}
+
+	/* initial default type */
+	dev->type = ARPHRD_ETHER;
+
+	switch (wrqu->mode) {
+#ifdef CONFIG_WIFI_MONITOR
+	case IW_MODE_MONITOR:
+		networkType = Ndis802_11Monitor;
+#if 0
+		dev->type = ARPHRD_IEEE80211; /* IEEE 802.11 : 801 */
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+		dev->type = ARPHRD_IEEE80211_RADIOTAP; /* IEEE 802.11 + radiotap header : 803 */
+		RTW_INFO("set_mode = IW_MODE_MONITOR\n");
+#else
+		RTW_INFO("kernel version < 2.6.24 not support IW_MODE_MONITOR\n");
+#endif
+		break;
+#endif /* CONFIG_WIFI_MONITOR */
+	case IW_MODE_AUTO:
+		networkType = Ndis802_11AutoUnknown;
+		RTW_INFO("set_mode = IW_MODE_AUTO\n");
+		break;
+	case IW_MODE_ADHOC:
+		networkType = Ndis802_11IBSS;
+		RTW_INFO("set_mode = IW_MODE_ADHOC\n");
+		break;
+	case IW_MODE_MASTER:
+		networkType = Ndis802_11APMode;
+		RTW_INFO("set_mode = IW_MODE_MASTER\n");
+		break;
+	case IW_MODE_INFRA:
+		networkType = Ndis802_11Infrastructure;
+		RTW_INFO("set_mode = IW_MODE_INFRA\n");
+		break;
+
+	default:
+		ret = -EINVAL;;
+		goto exit;
+	}
+
+	if (rtw_set_802_11_infrastructure_mode(padapter, networkType, 0) == _FALSE) {
+
+		ret = -EPERM;
+		goto exit;
+
+	}
+
+	rtw_setopmode_cmd(padapter, networkType, RTW_CMDF_WAIT_ACK);
+
+	if (check_fwstate(pmlmepriv, WIFI_MONITOR_STATE) == _TRUE)
+		rtw_indicate_connect(padapter);
+
+exit:
+
+
+	return ret;
+
+}
+
+static int rtw_wx_get_mode(struct net_device *dev, struct iw_request_info *a,
+			   union iwreq_data *wrqu, char *b)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct	mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	if (MLME_IS_STA(padapter))
+		wrqu->mode = IW_MODE_INFRA;
+	else if (MLME_IS_ADHOC(padapter) || MLME_IS_ADHOC_MASTER(padapter))
+		wrqu->mode = IW_MODE_ADHOC;
+	else if (MLME_IS_AP(padapter))
+		wrqu->mode = IW_MODE_MASTER;
+	else if (MLME_IS_MONITOR(padapter))
+		wrqu->mode = IW_MODE_MONITOR;
+	else
+		wrqu->mode = IW_MODE_AUTO;
+
+
+	return 0;
+
+}
+
+
+static int rtw_wx_set_pmkid(struct net_device *dev,
+			    struct iw_request_info *a,
+			    union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8          j, blInserted = _FALSE;
+	int         intReturn = _FALSE;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	struct iw_pmksa  *pPMK = (struct iw_pmksa *) extra;
+	u8     strZeroMacAddress[ETH_ALEN] = { 0x00 };
+	u8     strIssueBssid[ETH_ALEN] = { 0x00 };
+
+#if 0
+	struct iw_pmksa {
+		__u32   cmd;
+		struct sockaddr bssid;
+		__u8    pmkid[IW_PMKID_LEN];   /* IW_PMKID_LEN=16 */
+	}
+	There are the BSSID information in the bssid.sa_data array.
+	If cmd is IW_PMKSA_FLUSH, it means the wpa_suppplicant wants to clear all the PMKID information.
+	If cmd is IW_PMKSA_ADD, it means the wpa_supplicant wants to add a PMKID / BSSID to driver.
+	If cmd is IW_PMKSA_REMOVE, it means the wpa_supplicant wants to remove a PMKID / BSSID from driver.
+#endif
+
+	_rtw_memcpy(strIssueBssid, pPMK->bssid.sa_data, ETH_ALEN);
+	if (pPMK->cmd == IW_PMKSA_ADD) {
+		RTW_INFO("[rtw_wx_set_pmkid] IW_PMKSA_ADD!\n");
+		if (_rtw_memcmp(strIssueBssid, strZeroMacAddress, ETH_ALEN) == _TRUE)
+			return intReturn ;
+		else
+			intReturn = _TRUE;
+		blInserted = _FALSE;
+
+		/* overwrite PMKID */
+		for (j = 0 ; j < NUM_PMKID_CACHE; j++) {
+			if (_rtw_memcmp(psecuritypriv->PMKIDList[j].Bssid, strIssueBssid, ETH_ALEN) == _TRUE) {
+				/* BSSID is matched, the same AP => rewrite with new PMKID. */
+
+				RTW_INFO("[rtw_wx_set_pmkid] BSSID exists in the PMKList.\n");
+
+				_rtw_memcpy(psecuritypriv->PMKIDList[j].PMKID, pPMK->pmkid, IW_PMKID_LEN);
+				psecuritypriv->PMKIDList[j].bUsed = _TRUE;
+				blInserted = _TRUE;
+				break;
+			}
+		}
+
+		if (!blInserted) {
+			/* Find a new entry */
+			RTW_INFO("[rtw_wx_set_pmkid] Use the new entry index = %d for this PMKID.\n",
+				 psecuritypriv->PMKIDIndex);
+
+			_rtw_memcpy(psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].Bssid, strIssueBssid, ETH_ALEN);
+			_rtw_memcpy(psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].PMKID, pPMK->pmkid, IW_PMKID_LEN);
+
+			psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].bUsed = _TRUE;
+			psecuritypriv->PMKIDIndex++ ;
+			if (psecuritypriv->PMKIDIndex == 16)
+				psecuritypriv->PMKIDIndex = 0;
+		}
+	} else if (pPMK->cmd == IW_PMKSA_REMOVE) {
+		RTW_INFO("[rtw_wx_set_pmkid] IW_PMKSA_REMOVE!\n");
+		intReturn = _TRUE;
+		for (j = 0 ; j < NUM_PMKID_CACHE; j++) {
+			if (_rtw_memcmp(psecuritypriv->PMKIDList[j].Bssid, strIssueBssid, ETH_ALEN) == _TRUE) {
+				/* BSSID is matched, the same AP => Remove this PMKID information and reset it. */
+				_rtw_memset(psecuritypriv->PMKIDList[j].Bssid, 0x00, ETH_ALEN);
+				psecuritypriv->PMKIDList[j].bUsed = _FALSE;
+				break;
+			}
+		}
+	} else if (pPMK->cmd == IW_PMKSA_FLUSH) {
+		RTW_INFO("[rtw_wx_set_pmkid] IW_PMKSA_FLUSH!\n");
+		_rtw_memset(&psecuritypriv->PMKIDList[0], 0x00, sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
+		psecuritypriv->PMKIDIndex = 0;
+		intReturn = _TRUE;
+	}
+	return intReturn ;
+}
+
+static int rtw_wx_get_sens(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+#ifdef CONFIG_PLATFORM_ROCKCHIPS
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	/*
+	*  20110311 Commented by Jeff
+	*  For rockchip platform's wpa_driver_wext_get_rssi
+	*/
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+		/* wrqu->sens.value=-padapter->recvinfo.signal_strength; */
+		wrqu->sens.value = -padapter->recvinfo.rssi;
+		/* RTW_INFO("%s: %d\n", __FUNCTION__, wrqu->sens.value); */
+		wrqu->sens.fixed = 0; /* no auto select */
+	} else
+#endif
+	{
+		wrqu->sens.value = 0;
+		wrqu->sens.fixed = 0;	/* no auto select */
+		wrqu->sens.disabled = 1;
+	}
+	return 0;
+}
+
+static int rtw_wx_get_range(struct net_device *dev,
+			    struct iw_request_info *info,
+			    union iwreq_data *wrqu, char *extra)
+{
+	struct iw_range *range = (struct iw_range *)extra;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	u16 val;
+	int i;
+
+
+
+	wrqu->data.length = sizeof(*range);
+	_rtw_memset(range, 0, sizeof(*range));
+
+	/* Let's try to keep this struct in the same order as in
+	 * linux/include/wireless.h
+	 */
+
+	/* TODO: See what values we can set, and remove the ones we can't
+	 * set, or fill them with some default data.
+	 */
+
+	/* ~5 Mb/s real (802.11b) */
+	range->throughput = 5 * 1000 * 1000;
+
+	/* TODO: Not used in 802.11b?
+	*	range->min_nwid;	 Minimal NWID we are able to set  */
+	/* TODO: Not used in 802.11b?
+	*	range->max_nwid;	 Maximal NWID we are able to set  */
+
+	/* Old Frequency (backward compat - moved lower ) */
+	/*	range->old_num_channels;
+	 *	range->old_num_frequency;
+	 * 	range->old_freq[6];  Filler to keep "version" at the same offset  */
+
+	/* signal level threshold range */
+
+	/* Quality of link & SNR stuff */
+	/* Quality range (link, level, noise)
+	 * If the quality is absolute, it will be in the range [0 ; max_qual],
+	 * if the quality is dBm, it will be in the range [max_qual ; 0].
+	 * Don't forget that we use 8 bit arithmetics...
+	 *
+	 * If percentage range is 0~100
+	 * Signal strength dbm range logical is -100 ~ 0
+	 * but usually value is -90 ~ -20
+	 */
+	range->max_qual.qual = 100;
+#ifdef CONFIG_SIGNAL_DISPLAY_DBM
+	range->max_qual.level = (u8)-100;
+	range->max_qual.noise = (u8)-100;
+	range->max_qual.updated = IW_QUAL_ALL_UPDATED; /* Updated all three */
+	range->max_qual.updated |= IW_QUAL_DBM;
+#else /* !CONFIG_SIGNAL_DISPLAY_DBM */
+	/* percent values between 0 and 100. */
+	range->max_qual.level = 100;
+	range->max_qual.noise = 100;
+	range->max_qual.updated = IW_QUAL_ALL_UPDATED; /* Updated all three */
+#endif /* !CONFIG_SIGNAL_DISPLAY_DBM */
+
+	/* This should contain the average/typical values of the quality
+	 * indicator. This should be the threshold between a "good" and
+	 * a "bad" link (example : monitor going from green to orange).
+	 * Currently, user space apps like quality monitors don't have any
+	 * way to calibrate the measurement. With this, they can split
+	 * the range between 0 and max_qual in different quality level
+	 * (using a geometric subdivision centered on the average).
+	 * I expect that people doing the user space apps will feedback
+	 * us on which value we need to put in each driver... */
+	range->avg_qual.qual = 92; /* > 8% missed beacons is 'bad' */
+#ifdef CONFIG_SIGNAL_DISPLAY_DBM
+	/* TODO: Find real 'good' to 'bad' threshold value for RSSI */
+	range->avg_qual.level = (u8)-70;
+	range->avg_qual.noise = 0;
+	range->avg_qual.updated = IW_QUAL_ALL_UPDATED; /* Updated all three */
+	range->avg_qual.updated |= IW_QUAL_DBM;
+#else /* !CONFIG_SIGNAL_DISPLAY_DBM */
+	/* TODO: Find real 'good' to 'bad' threshol value for RSSI */
+	range->avg_qual.level = 30;
+	range->avg_qual.noise = 100;
+	range->avg_qual.updated = IW_QUAL_ALL_UPDATED; /* Updated all three */
+#endif /* !CONFIG_SIGNAL_DISPLAY_DBM */
+
+	range->num_bitrates = RATE_COUNT;
+
+	for (i = 0; i < RATE_COUNT && i < IW_MAX_BITRATES; i++)
+		range->bitrate[i] = rtw_rates[i];
+
+	range->min_frag = MIN_FRAG_THRESHOLD;
+	range->max_frag = MAX_FRAG_THRESHOLD;
+
+	range->pm_capa = 0;
+
+	range->we_version_compiled = WIRELESS_EXT;
+	range->we_version_source = 16;
+
+	/*	range->retry_capa;	 What retry options are supported
+	 *	range->retry_flags;	 How to decode max/min retry limit
+	 *	range->r_time_flags;	 How to decode max/min retry life
+	 *	range->min_retry;	 Minimal number of retries
+	 *	range->max_retry;	 Maximal number of retries
+	 *	range->min_r_time;	 Minimal retry lifetime
+	 *	range->max_r_time;	 Maximal retry lifetime  */
+
+	for (i = 0, val = 0; i < chset->chs_len; i++) {
+
+		/* Include only legal frequencies for some countries */
+		if (chset->chs[i].flags & RTW_CHF_DIS)
+			continue;
+
+		range->freq[val].i = chset->chs[i].ChannelNum;
+		range->freq[val].m = rtw_bch2freq(chset->chs[i].band, chset->chs[i].ChannelNum) * 100000;
+		range->freq[val].e = 1;
+		val++;
+
+		if (val == IW_MAX_FREQUENCIES)
+			break;
+	}
+
+	range->num_channels = val;
+	range->num_frequency = val;
+
+	/* Commented by Albert 2009/10/13
+	 * The following code will proivde the security capability to network manager.
+	 * If the driver doesn't provide this capability to network manager,
+	 * the WPA/WPA2 routers can't be choosen in the network manager. */
+
+	/*
+	#define IW_SCAN_CAPA_NONE		0x00
+	#define IW_SCAN_CAPA_ESSID		0x01
+	#define IW_SCAN_CAPA_BSSID		0x02
+	#define IW_SCAN_CAPA_CHANNEL	0x04
+	#define IW_SCAN_CAPA_MODE		0x08
+	#define IW_SCAN_CAPA_RATE		0x10
+	#define IW_SCAN_CAPA_TYPE		0x20
+	#define IW_SCAN_CAPA_TIME		0x40
+	*/
+
+#if WIRELESS_EXT > 17
+	range->enc_capa = IW_ENC_CAPA_WPA | IW_ENC_CAPA_WPA2 |
+			  IW_ENC_CAPA_CIPHER_TKIP | IW_ENC_CAPA_CIPHER_CCMP;
+#endif
+
+#ifdef IW_SCAN_CAPA_ESSID /* WIRELESS_EXT > 21 */
+	range->scan_capa = IW_SCAN_CAPA_ESSID | IW_SCAN_CAPA_TYPE | IW_SCAN_CAPA_BSSID |
+		   IW_SCAN_CAPA_CHANNEL | IW_SCAN_CAPA_MODE | IW_SCAN_CAPA_RATE;
+#endif
+
+
+
+	return 0;
+
+}
+
+/* set bssid flow
+ * s1. rtw_set_802_11_infrastructure_mode()
+ * s2. rtw_set_802_11_authentication_mode()
+ * s3. set_802_11_encryption_mode()
+ * s4. rtw_set_802_11_bssid() */
+static int rtw_wx_set_wap(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *awrq,
+			  char *extra)
+{
+	uint ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sockaddr *temp = (struct sockaddr *)awrq;
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	_list	*phead;
+	u8 *dst_bssid, *src_bssid;
+	_queue	*queue	= &(pmlmepriv->scanned_queue);
+	struct	wlan_network	*pnetwork = NULL;
+	NDIS_802_11_AUTHENTICATION_MODE	authmode;
+
+	/*
+	#ifdef CONFIG_CONCURRENT_MODE
+		if(padapter->adapter_type > PRIMARY_IFACE)
+		{
+			ret = -EINVAL;
+			goto exit;
+		}
+	#endif
+	*/
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING) == _TRUE) {
+		RTW_INFO("set bssid, but buddy_intf is under scanning or linking\n");
+		ret = -EINVAL;
+		goto exit;
+	}
+#endif
+	if (!padapter->netif_up) {
+		ret = -1;
+		goto exit;
+	}
+
+
+	if (temp->sa_family != ARPHRD_ETHER) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	authmode = padapter->securitypriv.ndisauthtype;
+	_rtw_spinlock_bh(&queue->lock);
+	phead = get_list_head(queue);
+	pmlmepriv->pscanned = get_next(phead);
+
+	while (1) {
+
+		if ((rtw_end_of_queue_search(phead, pmlmepriv->pscanned)) == _TRUE) {
+#if 0
+			ret = -EINVAL;
+			goto exit;
+
+			if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) {
+				rtw_set_802_11_bssid(padapter, temp->sa_data);
+				goto exit;
+			} else {
+				ret = -EINVAL;
+				goto exit;
+			}
+#endif
+
+			break;
+		}
+
+		pnetwork = LIST_CONTAINOR(pmlmepriv->pscanned, struct wlan_network, list);
+
+		pmlmepriv->pscanned = get_next(pmlmepriv->pscanned);
+
+		dst_bssid = pnetwork->network.MacAddress;
+
+		src_bssid = temp->sa_data;
+
+		if ((_rtw_memcmp(dst_bssid, src_bssid, ETH_ALEN)) == _TRUE) {
+			if (!rtw_set_802_11_infrastructure_mode(padapter, pnetwork->network.InfrastructureMode, 0)) {
+				ret = -1;
+				_rtw_spinunlock_bh(&queue->lock);
+				goto exit;
+			}
+
+			break;
+		}
+
+	}
+	_rtw_spinunlock_bh(&queue->lock);
+
+	rtw_set_802_11_authentication_mode(padapter, authmode);
+	/* set_802_11_encryption_mode(padapter, padapter->securitypriv.ndisencryptstatus); */
+	if (rtw_set_802_11_bssid(padapter, temp->sa_data) == _FALSE) {
+		ret = -1;
+	}
+
+exit:
+	return ret;
+}
+
+static int rtw_wx_get_wap(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	WLAN_BSSID_EX  *pcur_bss = &pmlmepriv->dev_cur_network.network;
+
+	wrqu->ap_addr.sa_family = ARPHRD_ETHER;
+
+	_rtw_memset(wrqu->ap_addr.sa_data, 0, ETH_ALEN);
+
+
+
+	if (((check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) == _TRUE) ||
+	    ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) == _TRUE) ||
+	    ((check_fwstate(pmlmepriv, WIFI_AP_STATE)) == _TRUE))
+
+		_rtw_memcpy(wrqu->ap_addr.sa_data, pcur_bss->MacAddress, ETH_ALEN);
+	else
+		_rtw_memset(wrqu->ap_addr.sa_data, 0, ETH_ALEN);
+
+
+	return 0;
+
+}
+
+static int rtw_wx_set_mlme(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+#if 0
+	/* SIOCSIWMLME data */
+	struct	iw_mlme {
+		__u16		cmd; /* IW_MLME_* */
+		__u16		reason_code;
+		struct sockaddr	addr;
+	};
+#endif
+
+	int ret = 0;
+	u16 reason;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct iw_mlme *mlme = (struct iw_mlme *) extra;
+
+
+	if (mlme == NULL)
+		return -1;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	reason = cpu_to_le16(mlme->reason_code);
+
+
+	RTW_INFO("%s, cmd=%d, reason=%d\n", __FUNCTION__, mlme->cmd, reason);
+
+
+	switch (mlme->cmd) {
+	case IW_MLME_DEAUTH:
+		if (!rtw_set_802_11_disassociate(padapter))
+			ret = -1;
+		break;
+
+	case IW_MLME_DISASSOC:
+		if (!rtw_set_802_11_disassociate(padapter))
+			ret = -1;
+
+		break;
+
+	default:
+		return -EOPNOTSUPP;
+	}
+	return ret;
+}
+
+static int rtw_wx_set_scan(struct net_device *dev, struct iw_request_info *a,
+			   union iwreq_data *wrqu, char *extra)
+{
+	u8 _status = _FALSE;
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/*struct mlme_priv *pmlmepriv = &padapter->mlmepriv;*/
+	struct sitesurvey_parm *parm = NULL;
+	u8 ssc_chk;
+#ifdef CONFIG_P2P
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_P2P */
+
+#ifdef DBG_IOCTL
+	RTW_INFO("DBG_IOCTL %s:%d\n", __FUNCTION__, __LINE__);
+#endif
+
+#if 1
+	ssc_chk = rtw_sitesurvey_condition_check(padapter, _FALSE);
+
+	#ifdef CONFIG_DOSCAN_IN_BUSYTRAFFIC
+	if ((ssc_chk != SS_ALLOW) && (ssc_chk != SS_DENY_BUSY_TRAFFIC))
+	#else
+	/* When Busy Traffic, driver do not site survey. So driver return success. */
+	/* wpa_supplicant will not issue SIOCSIWSCAN cmd again after scan timeout. */
+	/* modify by thomas 2011-02-22. */
+	if (ssc_chk != SS_ALLOW)
+	#endif
+	{
+		if (ssc_chk == SS_DENY_MP_MODE)
+			ret = -EPERM;
+		#ifdef DBG_LA_MODE
+		else if (ssc_chk == SS_DENY_LA_MODE)
+			ret = -EPERM;
+		#endif
+		else
+			indicate_wx_scan_complete_event(padapter);
+
+		goto exit;
+	} else
+		RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+
+	if (!rtw_is_adapter_up(padapter)) {
+		ret = -1;
+		goto exit;
+	}
+#else
+
+#ifdef CONFIG_MP_INCLUDED
+	if (rtw_mp_mode_check(padapter)) {
+		RTW_INFO("MP mode block Scan request\n");
+		ret = -EPERM;
+		goto exit;
+	}
+#endif
+	if (rtw_is_scan_deny(padapter)) {
+		indicate_wx_scan_complete_event(padapter);
+		goto exit;
+	}
+
+	if (!rtw_is_adapter_up(padapter)) {
+		ret = -1;
+		goto exit;
+	}
+
+#ifndef CONFIG_DOSCAN_IN_BUSYTRAFFIC
+	/* When Busy Traffic, driver do not site survey. So driver return success. */
+	/* wpa_supplicant will not issue SIOCSIWSCAN cmd again after scan timeout. */
+	/* modify by thomas 2011-02-22. */
+	if (rtw_mi_busy_traffic_check(padapter)) {
+		indicate_wx_scan_complete_event(padapter);
+		goto exit;
+	}
+#endif
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) && check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+		RTW_INFO("AP mode process WPS\n");
+		indicate_wx_scan_complete_event(padapter);
+		goto exit;
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING) == _TRUE) {
+		indicate_wx_scan_complete_event(padapter);
+		goto exit;
+	}
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_mi_buddy_check_fwstate(padapter,
+		       WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING | WIFI_UNDER_WPS)) {
+
+		indicate_wx_scan_complete_event(padapter);
+		goto exit;
+	}
+#endif
+#endif
+
+	parm = rtw_malloc(sizeof(*parm));
+	if (parm == NULL) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+#if WIRELESS_EXT >= 17
+	if (wrqu->data.length == sizeof(struct iw_scan_req)) {
+		struct iw_scan_req *req = (struct iw_scan_req *)extra;
+
+		if (wrqu->data.flags & IW_SCAN_THIS_ESSID) {
+			int len = min((int)req->essid_len, IW_ESSID_MAX_SIZE);
+
+			rtw_init_sitesurvey_parm(padapter, parm);
+			_rtw_memcpy(&parm->ssid[0].Ssid, &req->essid, len);
+			parm->ssid[0].SsidLength = len;
+			parm->ssid_num = 1;
+
+			RTW_INFO("IW_SCAN_THIS_ESSID, ssid=%s, len=%d\n", req->essid, req->essid_len);
+
+			_status = rtw_sitesurvey_cmd(padapter, parm);
+
+		} else if (req->scan_type == IW_SCAN_TYPE_PASSIVE)
+			RTW_INFO("rtw_wx_set_scan, req->scan_type == IW_SCAN_TYPE_PASSIVE\n");
+
+	} else
+#endif
+
+		if (wrqu->data.length >= WEXT_CSCAN_HEADER_SIZE
+		    && _rtw_memcmp(extra, WEXT_CSCAN_HEADER, WEXT_CSCAN_HEADER_SIZE) == _TRUE
+		   ) {
+			int len = wrqu->data.length - WEXT_CSCAN_HEADER_SIZE;
+			char *pos = extra + WEXT_CSCAN_HEADER_SIZE;
+			char section;
+			char sec_len;
+			int ssid_index = 0;
+
+			/* RTW_INFO("%s COMBO_SCAN header is recognized\n", __FUNCTION__); */
+			rtw_init_sitesurvey_parm(padapter, parm);
+
+			while (len >= 1) {
+				section = *(pos++);
+				len -= 1;
+
+				switch (section) {
+				case WEXT_CSCAN_SSID_SECTION:
+					/* RTW_INFO("WEXT_CSCAN_SSID_SECTION\n"); */
+					if (len < 1) {
+						len = 0;
+						break;
+					}
+
+					sec_len = *(pos++);
+					len -= 1;
+
+					if (sec_len > 0 && sec_len <= len) {
+
+						parm->ssid[ssid_index].SsidLength = sec_len;
+						_rtw_memcpy(&parm->ssid[ssid_index].Ssid, pos, sec_len);
+
+						/* RTW_INFO("%s COMBO_SCAN with specific parm->ssid:%s, %d\n", __FUNCTION__ */
+						/*	, parm->ssid[ssid_index].Ssid, parm->ssid[ssid_index].SsidLength); */
+						ssid_index++;
+					}
+
+					pos += sec_len;
+					len -= sec_len;
+					break;
+
+
+				case WEXT_CSCAN_CHANNEL_SECTION:
+					/* RTW_INFO("WEXT_CSCAN_CHANNEL_SECTION\n"); */
+					pos += 1;
+					len -= 1;
+					break;
+				case WEXT_CSCAN_ACTV_DWELL_SECTION:
+					/* RTW_INFO("WEXT_CSCAN_ACTV_DWELL_SECTION\n"); */
+					pos += 2;
+					len -= 2;
+					break;
+				case WEXT_CSCAN_PASV_DWELL_SECTION:
+					/* RTW_INFO("WEXT_CSCAN_PASV_DWELL_SECTION\n"); */
+					pos += 2;
+					len -= 2;
+					break;
+				case WEXT_CSCAN_HOME_DWELL_SECTION:
+					/* RTW_INFO("WEXT_CSCAN_HOME_DWELL_SECTION\n"); */
+					pos += 2;
+					len -= 2;
+					break;
+				case WEXT_CSCAN_TYPE_SECTION:
+					/* RTW_INFO("WEXT_CSCAN_TYPE_SECTION\n"); */
+					pos += 1;
+					len -= 1;
+					break;
+#if 0
+				case WEXT_CSCAN_NPROBE_SECTION:
+					RTW_INFO("WEXT_CSCAN_NPROBE_SECTION\n");
+					break;
+#endif
+
+				default:
+					/* RTW_INFO("Unknown CSCAN section %c\n", section); */
+					len = 0; /* stop parsing */
+				}
+				/* RTW_INFO("len:%d\n", len); */
+
+			}
+			parm->ssid_num = ssid_index;
+
+			/* jeff: it has still some scan paramater to parse, we only do this now... */
+			_status = rtw_sitesurvey_cmd(padapter, parm);
+
+		} else
+
+			_status = rtw_sitesurvey_cmd(padapter, NULL);
+
+	if (_status == _FALSE)
+		ret = -1;
+
+	rtw_mfree(parm, sizeof(*parm));
+
+exit:
+#ifdef DBG_IOCTL
+	RTW_INFO("DBG_IOCTL %s:%d return %d\n", __FUNCTION__, __LINE__, ret);
+#endif
+
+	return ret;
+}
+
+static int rtw_wx_get_scan(struct net_device *dev, struct iw_request_info *a,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_list					*plist, *phead;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(padapter);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	_queue				*queue	= &(pmlmepriv->scanned_queue);
+	struct	wlan_network	*pnetwork = NULL;
+	char *ev = extra;
+	char *stop = ev + wrqu->data.length;
+	u32 ret = 0;
+	u32 wait_for_surveydone;
+	sint wait_status;
+
+#ifdef CONFIG_P2P
+	struct	wifidirect_info	*pwdinfo = &padapter->wdinfo;
+#endif /* CONFIG_P2P */
+
+
+#ifdef DBG_IOCTL
+	RTW_INFO("DBG_IOCTL %s:%d\n", __FUNCTION__, __LINE__);
+#endif
+
+	if (dev_is_drv_stopped(adapter_to_dvobj(padapter))) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	wait_for_surveydone = 100;
+
+#if 1 /* Wireless Extension use EAGAIN to try */
+	wait_status = WIFI_UNDER_SURVEY
+#ifndef CONFIG_RTW_ANDROID
+		      | WIFI_UNDER_LINKING
+#endif
+		      ;
+
+	while (check_fwstate(pmlmepriv, wait_status) == _TRUE)
+		return -EAGAIN;
+#else
+	wait_status = WIFI_UNDER_SURVEY
+#ifndef CONFIG_RTW_ANDROID
+		      | WIFI_UNDER_LINKING
+#endif
+		      ;
+
+	while (check_fwstate(pmlmepriv, wait_status) == _TRUE) {
+		rtw_msleep_os(30);
+		cnt++;
+		if (cnt > wait_for_surveydone)
+			break;
+	}
+#endif
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	while (1) {
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+		if ((stop - ev) < SCAN_ITEM_SIZE) {
+			if(wrqu->data.length == MAX_SCAN_BUFFER_LEN){ /*max buffer len defined by iwlist*/
+				ret = 0;
+				RTW_INFO("%s: Scan results incomplete\n", __FUNCTION__);
+				break;
+			}
+			ret = -E2BIG;
+			break;
+		}
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+		/* report network only if the current channel set contains the channel to which this network belongs */
+		if (rtw_network_chk_opch_status(rfctl, pnetwork)
+			&& rtw_network_chk_regu_ies(rfctl, pnetwork)
+			&& rtw_mlme_band_check(padapter, BSS_EX_OP_CH(&pnetwork->network)) == _TRUE
+			&& _TRUE == rtw_validate_ssid(&(pnetwork->network.Ssid))
+		)
+			ev = translate_scan(padapter, a, pnetwork, ev, stop);
+
+		plist = get_next(plist);
+
+	}
+
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	wrqu->data.length = ev - extra;
+	wrqu->data.flags = 0;
+
+exit:
+
+
+#ifdef DBG_IOCTL
+	RTW_INFO("DBG_IOCTL %s:%d return %d\n", __FUNCTION__, __LINE__, ret);
+#endif
+
+	return ret ;
+
+}
+
+/* set ssid flow
+ * s1. rtw_set_802_11_infrastructure_mode()
+ * s2. set_802_11_authenticaion_mode()
+ * s3. set_802_11_encryption_mode()
+ * s4. rtw_set_802_11_ssid() */
+static int rtw_wx_set_essid(struct net_device *dev,
+			    struct iw_request_info *a,
+			    union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	_queue *queue = &pmlmepriv->scanned_queue;
+	_list *phead;
+	struct wlan_network *pnetwork = NULL;
+	NDIS_802_11_AUTHENTICATION_MODE authmode;
+	NDIS_802_11_SSID ndis_ssid;
+	u8 *dst_ssid, *src_ssid;
+
+	uint ret = 0, len;
+
+
+#ifdef DBG_IOCTL
+	RTW_INFO("DBG_IOCTL %s:%d\n", __FUNCTION__, __LINE__);
+#endif
+#ifdef CONFIG_WEXT_DONT_JOIN_BYSSID
+	RTW_INFO("%s: CONFIG_WEXT_DONT_JOIN_BYSSID be defined!! only allow bssid joining\n", __func__);
+	return -EPERM;
+#endif
+
+#if WIRELESS_EXT <= 20
+	if ((wrqu->essid.length - 1) > IW_ESSID_MAX_SIZE) {
+#else
+	if (wrqu->essid.length > IW_ESSID_MAX_SIZE) {
+#endif
+		ret = -E2BIG;
+		goto exit;
+	}
+
+	if (!padapter->netif_up) {
+		ret = -1;
+		goto exit;
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+		ret = -1;
+		goto exit;
+	}
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_mi_buddy_check_fwstate(padapter, WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING)) {
+		RTW_INFO("set ssid, but buddy_intf is under scanning or linking\n");
+		ret = -EINVAL;
+		goto exit;
+	}
+#endif
+	authmode = padapter->securitypriv.ndisauthtype;
+	RTW_INFO("=>%s\n", __FUNCTION__);
+	if (wrqu->essid.flags && wrqu->essid.length) {
+		/* Commented by Albert 20100519 */
+		/* We got the codes in "set_info" function of iwconfig source code. */
+		/*	========================================= */
+		/*	wrq.u.essid.length = strlen(essid) + 1; */
+		/*	if(we_kernel_version > 20) */
+		/*		wrq.u.essid.length--; */
+		/*	========================================= */
+		/*	That means, if the WIRELESS_EXT less than or equal to 20, the correct ssid len should subtract 1. */
+#if WIRELESS_EXT <= 20
+		len = ((wrqu->essid.length - 1) < IW_ESSID_MAX_SIZE) ? (wrqu->essid.length - 1) : IW_ESSID_MAX_SIZE;
+#else
+		len = (wrqu->essid.length < IW_ESSID_MAX_SIZE) ? wrqu->essid.length : IW_ESSID_MAX_SIZE;
+#endif
+
+		if (wrqu->essid.length != 33)
+			RTW_INFO("ssid=%s, len=%d\n", extra, wrqu->essid.length);
+
+		_rtw_memset(&ndis_ssid, 0, sizeof(NDIS_802_11_SSID));
+		ndis_ssid.SsidLength = len;
+		_rtw_memcpy(ndis_ssid.Ssid, extra, len);
+		src_ssid = ndis_ssid.Ssid;
+
+		_rtw_spinlock_bh(&queue->lock);
+		phead = get_list_head(queue);
+		pmlmepriv->pscanned = get_next(phead);
+
+		while (1) {
+			if (rtw_end_of_queue_search(phead, pmlmepriv->pscanned) == _TRUE) {
+#if 0
+				if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) {
+					rtw_set_802_11_ssid(padapter, &ndis_ssid);
+
+					goto exit;
+				} else {
+					ret = -EINVAL;
+					goto exit;
+				}
+#endif
+
+				break;
+			}
+
+			pnetwork = LIST_CONTAINOR(pmlmepriv->pscanned, struct wlan_network, list);
+
+			pmlmepriv->pscanned = get_next(pmlmepriv->pscanned);
+
+			dst_ssid = pnetwork->network.Ssid.Ssid;
+
+
+			if ((_rtw_memcmp(dst_ssid, src_ssid, ndis_ssid.SsidLength) == _TRUE) &&
+			    (pnetwork->network.Ssid.SsidLength == ndis_ssid.SsidLength)) {
+
+				if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE) {
+					if (pnetwork->network.InfrastructureMode != pmlmepriv->dev_cur_network.network.InfrastructureMode)
+						continue;
+				}
+
+				if (rtw_set_802_11_infrastructure_mode(padapter, pnetwork->network.InfrastructureMode, 0) == _FALSE) {
+					ret = -1;
+					_rtw_spinunlock_bh(&queue->lock);
+					goto exit;
+				}
+
+				break;
+			}
+		}
+		_rtw_spinunlock_bh(&queue->lock);
+		rtw_set_802_11_authentication_mode(padapter, authmode);
+		/* set_802_11_encryption_mode(padapter, padapter->securitypriv.ndisencryptstatus); */
+		if (rtw_set_802_11_ssid(padapter, &ndis_ssid) == _FALSE) {
+			ret = -1;
+			goto exit;
+		}
+	}
+
+exit:
+	RTW_INFO("<=%s, ret %d\n", __FUNCTION__, ret);
+
+#ifdef DBG_IOCTL
+	RTW_INFO("DBG_IOCTL %s:%d return %d\n", __FUNCTION__, __LINE__, ret);
+#endif
+
+
+	return ret;
+}
+
+static int rtw_wx_get_essid(struct net_device *dev,
+			    struct iw_request_info *a,
+			    union iwreq_data *wrqu, char *extra)
+{
+	u32 len, ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	WLAN_BSSID_EX  *pcur_bss = &pmlmepriv->dev_cur_network.network;
+
+
+
+	if ((check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) ||
+	    (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE)) {
+		len = pcur_bss->Ssid.SsidLength;
+
+		wrqu->essid.length = len;
+
+		_rtw_memcpy(extra, pcur_bss->Ssid.Ssid, len);
+
+		wrqu->essid.flags = 1;
+	} else {
+		ret = -1;
+		goto exit;
+	}
+
+exit:
+
+
+	return ret;
+
+}
+
+static int rtw_wx_set_rate(struct net_device *dev,
+			   struct iw_request_info *a,
+			   union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+#if 0
+	int i;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8	datarates[NumRates];
+	u32	target_rate = wrqu->bitrate.value;
+	u32	fixed = wrqu->bitrate.fixed;
+	u32	ratevalue = 0;
+	u8 mpdatarate[NumRates] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0xff};
+
+
+
+	if (target_rate == -1) {
+		ratevalue = 11;
+		goto set_rate;
+	}
+	target_rate = target_rate / 100000;
+
+	switch (target_rate) {
+	case 10:
+		ratevalue = 0;
+		break;
+	case 20:
+		ratevalue = 1;
+		break;
+	case 55:
+		ratevalue = 2;
+		break;
+	case 60:
+		ratevalue = 3;
+		break;
+	case 90:
+		ratevalue = 4;
+		break;
+	case 110:
+		ratevalue = 5;
+		break;
+	case 120:
+		ratevalue = 6;
+		break;
+	case 180:
+		ratevalue = 7;
+		break;
+	case 240:
+		ratevalue = 8;
+		break;
+	case 360:
+		ratevalue = 9;
+		break;
+	case 480:
+		ratevalue = 10;
+		break;
+	case 540:
+		ratevalue = 11;
+		break;
+	default:
+		ratevalue = 11;
+		break;
+	}
+
+set_rate:
+
+	for (i = 0; i < NumRates; i++) {
+		if (ratevalue == mpdatarate[i]) {
+			datarates[i] = mpdatarate[i];
+			if (fixed == 0)
+				break;
+		} else
+			datarates[i] = 0xff;
+
+	}
+
+	if (rtw_setdatarate_cmd(padapter, datarates) != _SUCCESS) {
+		ret = -1;
+	}
+
+#endif
+	return ret;
+}
+
+static int rtw_wx_get_rate(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	u16 max_rate = 0;
+
+	max_rate = rtw_get_cur_max_rate((_adapter *)rtw_netdev_priv(dev));
+
+	if (max_rate == 0)
+		return -EPERM;
+
+	wrqu->bitrate.fixed = 0;	/* no auto select */
+	wrqu->bitrate.value = max_rate * 100000;
+	/*RTW_INFO("%s, max_rate=%d, wrqu->bitrate.value=%d\n", __func__, max_rate, wrqu->bitrate.value);*/
+	/*Note: wrqu->bitrate.value was s32 , so 160M 2SS max rate will overflow*/
+	if (wrqu->bitrate.value < 0)
+		wrqu->bitrate.value = 0x7FFFFFFF;
+
+	return 0;
+}
+
+static int rtw_wx_set_rts(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	if (wrqu->rts.disabled)
+		padapter->registrypriv.rts_thresh = 2347;
+	else {
+		if (wrqu->rts.value < 0 ||
+		    wrqu->rts.value > 2347)
+			return -EINVAL;
+
+		padapter->registrypriv.rts_thresh = wrqu->rts.value;
+	}
+
+	RTW_INFO("%s, rts_thresh=%d\n", __func__, padapter->registrypriv.rts_thresh);
+
+
+	return 0;
+
+}
+
+static int rtw_wx_get_rts(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	RTW_INFO("%s, rts_thresh=%d\n", __func__, padapter->registrypriv.rts_thresh);
+
+	wrqu->rts.value = padapter->registrypriv.rts_thresh;
+	wrqu->rts.fixed = 0;	/* no auto select */
+	/* wrqu->rts.disabled = (wrqu->rts.value == DEFAULT_RTS_THRESHOLD); */
+
+
+	return 0;
+}
+
+static int rtw_wx_set_frag(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	if (wrqu->frag.disabled)
+		padapter->xmitpriv.frag_len = MAX_FRAG_THRESHOLD;
+	else {
+		if (wrqu->frag.value < MIN_FRAG_THRESHOLD ||
+		    wrqu->frag.value > MAX_FRAG_THRESHOLD)
+			return -EINVAL;
+
+		padapter->xmitpriv.frag_len = wrqu->frag.value & ~0x1;
+	}
+
+	RTW_INFO("%s, frag_len=%d\n", __func__, padapter->xmitpriv.frag_len);
+
+
+	return 0;
+
+}
+
+static int rtw_wx_get_frag(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	RTW_INFO("%s, frag_len=%d\n", __func__, padapter->xmitpriv.frag_len);
+
+	wrqu->frag.value = padapter->xmitpriv.frag_len;
+	wrqu->frag.fixed = 0;	/* no auto select */
+	/* wrqu->frag.disabled = (wrqu->frag.value == DEFAULT_FRAG_THRESHOLD); */
+
+
+	return 0;
+}
+
+static int rtw_wx_get_retry(struct net_device *dev,
+			    struct iw_request_info *info,
+			    union iwreq_data *wrqu, char *extra)
+{
+	/* _adapter *padapter = (_adapter *)rtw_netdev_priv(dev); */
+
+
+	wrqu->retry.value = 7;
+	wrqu->retry.fixed = 0;	/* no auto select */
+	wrqu->retry.disabled = 1;
+
+	return 0;
+
+}
+
+#if 0
+	#define IW_ENCODE_INDEX		0x00FF	/* Token index (if needed) */
+	#define IW_ENCODE_FLAGS		0xFF00	/* Flags defined below */
+	#define IW_ENCODE_MODE		0xF000	/* Modes defined below */
+	#define IW_ENCODE_DISABLED	0x8000	/* Encoding disabled */
+	#define IW_ENCODE_ENABLED	0x0000	/* Encoding enabled */
+	#define IW_ENCODE_RESTRICTED	0x4000	/* Refuse non-encoded packets */
+	#define IW_ENCODE_OPEN		0x2000	/* Accept non-encoded packets */
+	#define IW_ENCODE_NOKEY		0x0800  /* Key is write only, so not present */
+	#define IW_ENCODE_TEMP		0x0400  /* Temporary key */
+	/*
+	iwconfig wlan0 key on->flags = 0x6001->maybe it means auto
+	iwconfig wlan0 key off->flags = 0x8800
+	iwconfig wlan0 key open->flags = 0x2800
+	iwconfig wlan0 key open 1234567890->flags = 0x2000
+	iwconfig wlan0 key restricted->flags = 0x4800
+	iwconfig wlan0 key open [3] 1234567890->flags = 0x2003
+	iwconfig wlan0 key restricted [2] 1234567890->flags = 0x4002
+	iwconfig wlan0 key open [3] -> flags = 0x2803
+	iwconfig wlan0 key restricted [2] -> flags = 0x4802
+	*/
+#endif
+
+static int rtw_wx_set_enc(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *keybuf)
+{
+	u32 key, ret = 0;
+	u32 keyindex_provided;
+	NDIS_802_11_WEP	 wep;
+	NDIS_802_11_AUTHENTICATION_MODE authmode;
+
+	struct iw_point *erq = &(wrqu->encoding);
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	RTW_INFO("+rtw_wx_set_enc, flags=0x%x\n", erq->flags);
+
+	_rtw_memset(&wep, 0, sizeof(NDIS_802_11_WEP));
+
+	key = erq->flags & IW_ENCODE_INDEX;
+
+
+	if (erq->flags & IW_ENCODE_DISABLED) {
+		RTW_INFO("EncryptionDisabled\n");
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
+		padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+		authmode = Ndis802_11AuthModeOpen;
+		padapter->securitypriv.ndisauthtype = authmode;
+
+		goto exit;
+	}
+
+	if (key) {
+		if (key > WEP_KEYS)
+			return -EINVAL;
+		key--;
+		keyindex_provided = 1;
+	} else {
+		keyindex_provided = 0;
+		key = padapter->securitypriv.dot11PrivacyKeyIndex;
+		RTW_INFO("rtw_wx_set_enc, key=%d\n", key);
+	}
+
+	/* set authentication mode	 */
+	if (erq->flags & IW_ENCODE_OPEN) {
+		RTW_INFO("rtw_wx_set_enc():IW_ENCODE_OPEN\n");
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;/* Ndis802_11EncryptionDisabled; */
+
+#ifdef CONFIG_PLATFORM_MT53XX
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+#else
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
+#endif
+
+		padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
+		authmode = Ndis802_11AuthModeOpen;
+		padapter->securitypriv.ndisauthtype = authmode;
+	} else if (erq->flags & IW_ENCODE_RESTRICTED) {
+		RTW_INFO("rtw_wx_set_enc():IW_ENCODE_RESTRICTED\n");
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
+
+#ifdef CONFIG_PLATFORM_MT53XX
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+#else
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Shared;
+#endif
+
+		padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_;
+		padapter->securitypriv.dot118021XGrpPrivacy = _WEP40_;
+		authmode = Ndis802_11AuthModeShared;
+		padapter->securitypriv.ndisauthtype = authmode;
+	} else {
+		RTW_INFO("rtw_wx_set_enc():erq->flags=0x%x\n", erq->flags);
+
+		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;/* Ndis802_11EncryptionDisabled; */
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+		padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
+		authmode = Ndis802_11AuthModeOpen;
+		padapter->securitypriv.ndisauthtype = authmode;
+	}
+
+	wep.KeyIndex = key;
+	if (erq->length > 0) {
+		wep.KeyLength = erq->length <= 5 ? 5 : 13;
+
+		wep.Length = wep.KeyLength + FIELD_OFFSET(NDIS_802_11_WEP, KeyMaterial);
+	} else {
+		wep.KeyLength = 0 ;
+
+		if (keyindex_provided == 1) { /* set key_id only, no given KeyMaterial(erq->length==0). */
+			padapter->securitypriv.dot11PrivacyKeyIndex = key;
+
+			RTW_INFO("(keyindex_provided == 1), keyid=%d, key_len=%d\n", key, padapter->securitypriv.dot11DefKeylen[key]);
+
+			switch (padapter->securitypriv.dot11DefKeylen[key]) {
+			case 5:
+				padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_;
+				break;
+			case 13:
+				padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_;
+				break;
+			default:
+				padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+				break;
+			}
+
+			goto exit;
+
+		}
+
+	}
+
+	wep.KeyIndex |= 0x80000000;
+
+	_rtw_memcpy(wep.KeyMaterial, keybuf, wep.KeyLength);
+
+	if (rtw_set_802_11_add_wep(padapter, &wep) == _FALSE) {
+		ret = -EOPNOTSUPP;
+		goto exit;
+	}
+
+
+exit:
+
+
+	return ret;
+
+}
+
+static int rtw_wx_get_enc(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *keybuf)
+{
+	uint key, ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct iw_point *erq = &(wrqu->encoding);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) != _TRUE) {
+		if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) != _TRUE) {
+			erq->length = 0;
+			erq->flags |= IW_ENCODE_DISABLED;
+			return 0;
+		}
+	}
+
+
+	key = erq->flags & IW_ENCODE_INDEX;
+
+	if (key) {
+		if (key > WEP_KEYS)
+			return -EINVAL;
+		key--;
+	} else
+		key = padapter->securitypriv.dot11PrivacyKeyIndex;
+
+	erq->flags = key + 1;
+
+	/* if(padapter->securitypriv.ndisauthtype == Ndis802_11AuthModeOpen) */
+	/* { */
+	/* erq->flags |= IW_ENCODE_OPEN; */
+	/* }	  */
+
+	switch (padapter->securitypriv.ndisencryptstatus) {
+	case Ndis802_11EncryptionNotSupported:
+	case Ndis802_11EncryptionDisabled:
+
+		erq->length = 0;
+		erq->flags |= IW_ENCODE_DISABLED;
+
+		break;
+
+	case Ndis802_11Encryption1Enabled:
+
+		erq->length = padapter->securitypriv.dot11DefKeylen[key];
+
+		if (erq->length) {
+			_rtw_memcpy(keybuf, padapter->securitypriv.dot11DefKey[key].skey, padapter->securitypriv.dot11DefKeylen[key]);
+
+			erq->flags |= IW_ENCODE_ENABLED;
+
+			if (padapter->securitypriv.ndisauthtype == Ndis802_11AuthModeOpen)
+				erq->flags |= IW_ENCODE_OPEN;
+			else if (padapter->securitypriv.ndisauthtype == Ndis802_11AuthModeShared)
+				erq->flags |= IW_ENCODE_RESTRICTED;
+		} else {
+			erq->length = 0;
+			erq->flags |= IW_ENCODE_DISABLED;
+		}
+
+		break;
+
+	case Ndis802_11Encryption2Enabled:
+	case Ndis802_11Encryption3Enabled:
+
+		erq->length = 16;
+		erq->flags |= (IW_ENCODE_ENABLED | IW_ENCODE_OPEN | IW_ENCODE_NOKEY);
+
+		break;
+
+	default:
+		erq->length = 0;
+		erq->flags |= IW_ENCODE_DISABLED;
+
+		break;
+
+	}
+
+
+	return ret;
+
+}
+
+static int rtw_wx_get_power(struct net_device *dev,
+			    struct iw_request_info *info,
+			    union iwreq_data *wrqu, char *extra)
+{
+	/* _adapter *padapter = (_adapter *)rtw_netdev_priv(dev); */
+
+	wrqu->power.value = 0;
+	wrqu->power.fixed = 0;	/* no auto select */
+	wrqu->power.disabled = 1;
+
+	return 0;
+
+}
+
+static int rtw_wx_set_gen_ie(struct net_device *dev,
+			     struct iw_request_info *info,
+			     union iwreq_data *wrqu, char *extra)
+{
+	int ret;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	ret = rtw_set_wpa_ie(padapter, extra, wrqu->data.length);
+
+	return ret;
+}
+
+static int rtw_wx_set_auth(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct iw_param *param = (struct iw_param *)&(wrqu->param);
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+#ifdef CONFIG_WAPI_SUPPORT
+#ifndef CONFIG_IOCTL_CFG80211
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+	u32 value = param->value;
+#endif
+#endif
+	int ret = 0;
+
+	switch (param->flags & IW_AUTH_INDEX) {
+
+	case IW_AUTH_WPA_VERSION:
+#ifdef CONFIG_WAPI_SUPPORT
+#ifndef CONFIG_IOCTL_CFG80211
+		padapter->wapiInfo.bWapiEnable = false;
+		if (value == IW_AUTH_WAPI_VERSION_1) {
+			padapter->wapiInfo.bWapiEnable = true;
+			psecuritypriv->dot11PrivacyAlgrthm = _SMS4_;
+			psecuritypriv->dot118021XGrpPrivacy = _SMS4_;
+			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_WAPI;
+			pmlmeinfo->auth_algo = psecuritypriv->dot11AuthAlgrthm;
+			padapter->wapiInfo.extra_prefix_len = WAPI_EXT_LEN;
+			padapter->wapiInfo.extra_postfix_len = SMS4_MIC_LEN;
+		}
+#endif
+#endif
+		break;
+	case IW_AUTH_CIPHER_PAIRWISE:
+
+		break;
+	case IW_AUTH_CIPHER_GROUP:
+
+		break;
+	case IW_AUTH_KEY_MGMT:
+#ifdef CONFIG_WAPI_SUPPORT
+#ifndef CONFIG_IOCTL_CFG80211
+		RTW_INFO("rtw_wx_set_auth: IW_AUTH_KEY_MGMT case\n");
+		if (value == IW_AUTH_KEY_MGMT_WAPI_PSK)
+			padapter->wapiInfo.bWapiPSK = true;
+		else
+			padapter->wapiInfo.bWapiPSK = false;
+		RTW_INFO("rtw_wx_set_auth: IW_AUTH_KEY_MGMT bwapipsk %d\n", padapter->wapiInfo.bWapiPSK);
+#endif
+#endif
+		/*
+		 *  ??? does not use these parameters
+		 */
+		break;
+
+	case IW_AUTH_TKIP_COUNTERMEASURES: {
+		if (param->value) {
+			/* wpa_supplicant is enabling the tkip countermeasure. */
+			padapter->securitypriv.btkip_countermeasure = _TRUE;
+		} else {
+			/* wpa_supplicant is disabling the tkip countermeasure. */
+			padapter->securitypriv.btkip_countermeasure = _FALSE;
+		}
+		break;
+	}
+	case IW_AUTH_DROP_UNENCRYPTED: {
+		/* HACK:
+		 *
+		 * wpa_supplicant calls set_wpa_enabled when the driver
+		 * is loaded and unloaded, regardless of if WPA is being
+		 * used.  No other calls are made which can be used to
+		 * determine if encryption will be used or not prior to
+		 * association being expected.  If encryption is not being
+		 * used, drop_unencrypted is set to false, else true -- we
+		 * can use this to determine if the CAP_PRIVACY_ON bit should
+		 * be set.
+		 */
+
+		if (padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption1Enabled) {
+			break;/* it means init value, or using wep, ndisencryptstatus = Ndis802_11Encryption1Enabled, */
+			/* then it needn't reset it; */
+		}
+
+		if (param->value) {
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
+			padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+			padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
+			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
+		}
+
+		break;
+	}
+
+	case IW_AUTH_80211_AUTH_ALG:
+
+#if defined(CONFIG_RTW_ANDROID) || 1
+		/*
+		 *  It's the starting point of a link layer connection using wpa_supplicant
+		*/
+		if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) {
+			rtw_disassoc_cmd(padapter, 500, RTW_CMDF_WAIT_ACK);
+			if (MLME_IS_STA(padapter) == _FALSE)
+				rtw_free_assoc_resources_cmd(padapter, _TRUE, RTW_CMDF_WAIT_ACK);
+			RTW_INFO("%s...call rtw_indicate_disconnect\n ", __FUNCTION__);
+			rtw_indicate_disconnect(padapter, 0, _FALSE);
+
+			pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_HIGH_LAYER_COMMAND;
+			pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+		}
+#endif
+
+
+		ret = wpa_set_auth_algs(dev, (u32)param->value);
+
+		break;
+
+	case IW_AUTH_WPA_ENABLED:
+
+		/* if(param->value) */
+		/* padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X; */ /* 802.1x */
+		/* else */
+		/* padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Open; */ /* open system */
+
+		/* _disassociate(priv); */
+
+		break;
+
+	case IW_AUTH_RX_UNENCRYPTED_EAPOL:
+		/* ieee->ieee802_1x = param->value; */
+		break;
+
+	case IW_AUTH_PRIVACY_INVOKED:
+		/* ieee->privacy_invoked = param->value; */
+		break;
+
+#ifdef CONFIG_WAPI_SUPPORT
+#ifndef CONFIG_IOCTL_CFG80211
+	case IW_AUTH_WAPI_ENABLED:
+		break;
+#endif
+#endif
+
+	default:
+		return -EOPNOTSUPP;
+
+	}
+
+	return ret;
+
+}
+
+static int rtw_wx_set_enc_ext(struct net_device *dev,
+			      struct iw_request_info *info,
+			      union iwreq_data *wrqu, char *extra)
+{
+	char *alg_name;
+	u32 param_len;
+	struct ieee_param *param = NULL;
+	struct iw_point *pencoding = &wrqu->encoding;
+	struct iw_encode_ext *pext = (struct iw_encode_ext *)extra;
+	int ret = 0;
+
+	param_len = sizeof(struct ieee_param) + pext->key_len;
+	param = (struct ieee_param *)rtw_malloc(param_len);
+	if (param == NULL)
+		return -1;
+
+	_rtw_memset(param, 0, param_len);
+
+	param->cmd = IEEE_CMD_SET_ENCRYPTION;
+	_rtw_memset(param->sta_addr, 0xff, ETH_ALEN);
+
+
+	switch (pext->alg) {
+	case IW_ENCODE_ALG_NONE:
+		/* todo: remove key */
+		/* remove = 1;	 */
+		alg_name = "none";
+		break;
+	case IW_ENCODE_ALG_WEP:
+		alg_name = "WEP";
+		break;
+	case IW_ENCODE_ALG_TKIP:
+		alg_name = "TKIP";
+		break;
+	case IW_ENCODE_ALG_CCMP:
+		alg_name = "CCMP";
+		break;
+#ifdef CONFIG_IEEE80211W
+	case IW_ENCODE_ALG_AES_CMAC:
+		alg_name = "BIP";
+		break;
+#endif /* CONFIG_IEEE80211W */
+#ifdef CONFIG_WAPI_SUPPORT
+#ifndef CONFIG_IOCTL_CFG80211
+	case IW_ENCODE_ALG_SM4:
+		alg_name = "SMS4";
+		_rtw_memcpy(param->sta_addr, pext->addr.sa_data, ETH_ALEN);
+		RTW_INFO("rtw_wx_set_enc_ext: SMS4 case\n");
+		break;
+#endif
+#endif
+	default:
+		ret = -1;
+		goto exit;
+	}
+
+	strncpy((char *)param->u.crypt.alg, alg_name, IEEE_CRYPT_ALG_NAME_LEN);
+
+	if (pext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY)
+		param->u.crypt.set_tx = 1;
+
+	/* cliW: WEP does not have group key
+	 * just not checking GROUP key setting
+	 */
+	if ((pext->alg != IW_ENCODE_ALG_WEP) &&
+	    ((pext->ext_flags & IW_ENCODE_EXT_GROUP_KEY)
+#ifdef CONFIG_IEEE80211W
+	     || (pext->ext_flags & IW_ENCODE_ALG_AES_CMAC)
+#endif /* CONFIG_IEEE80211W */
+	    ))
+		param->u.crypt.set_tx = 0;
+
+	param->u.crypt.idx = (pencoding->flags & 0x00FF) - 1 ;
+
+	if (pext->ext_flags & IW_ENCODE_EXT_RX_SEQ_VALID) {
+#ifdef CONFIG_WAPI_SUPPORT
+#ifndef CONFIG_IOCTL_CFG80211
+		if (pext->alg == IW_ENCODE_ALG_SM4)
+			_rtw_memcpy(param->u.crypt.seq, pext->rx_seq, 16);
+		else
+#endif /* CONFIG_IOCTL_CFG80211 */
+#endif /* CONFIG_WAPI_SUPPORT */
+			_rtw_memcpy(param->u.crypt.seq, pext->rx_seq, 8);
+	}
+
+	if (pext->key_len) {
+		param->u.crypt.key_len = pext->key_len;
+		/* _rtw_memcpy(param + 1, pext + 1, pext->key_len); */
+		_rtw_memcpy(param->u.crypt.key, pext + 1, pext->key_len);
+	}
+
+	if (pencoding->flags & IW_ENCODE_DISABLED) {
+		/* todo: remove key */
+		/* remove = 1; */
+	}
+
+	ret =  wpa_set_encryption(dev, param, param_len);
+
+exit:
+	if (param)
+		rtw_mfree((u8 *)param, param_len);
+
+	return ret;
+}
+
+
+static int rtw_wx_get_nick(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	/* _adapter *padapter = (_adapter *)rtw_netdev_priv(dev); */
+	/* struct mlme_priv *pmlmepriv = &(padapter->mlmepriv); */
+	/* struct security_priv *psecuritypriv = &padapter->securitypriv; */
+
+	if (extra) {
+		wrqu->data.length = 14;
+		wrqu->data.flags = 1;
+		_rtw_memcpy(extra, "<WIFI@REALTEK>", 14);
+	}
+
+	/* rtw_signal_process(pid, SIGUSR1); */ /* for test */
+
+	/* dump debug info here	 */
+#if 0
+	u32 dot11AuthAlgrthm;		/*  802.11 auth, could be open, shared, and 8021x */
+	u32 dot11PrivacyAlgrthm;	/*  This specify the privacy for shared auth. algorithm. */
+	u32 dot118021XGrpPrivacy;	/*  This specify the privacy algthm. used for Grp key */
+	u32 ndisauthtype;
+	u32 ndisencryptstatus;
+#endif
+
+	/* RTW_INFO("auth_alg=0x%x, enc_alg=0x%x, auth_type=0x%x, enc_type=0x%x\n",  */
+	/*		psecuritypriv->dot11AuthAlgrthm, psecuritypriv->dot11PrivacyAlgrthm, */
+	/*		psecuritypriv->ndisauthtype, psecuritypriv->ndisencryptstatus); */
+
+	/* RTW_INFO("enc_alg=0x%x\n", psecuritypriv->dot11PrivacyAlgrthm); */
+	/* RTW_INFO("auth_type=0x%x\n", psecuritypriv->ndisauthtype); */
+	/* RTW_INFO("enc_type=0x%x\n", psecuritypriv->ndisencryptstatus); */
+
+	return 0;
+
+}
+
+static int rtw_wx_read32(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+
+	_adapter *padapter;
+	struct dvobj_priv *dvobj;
+	struct iw_point *p;
+	u16 len;
+	u32 addr;
+	u32 data32;
+	u32 bytes;
+	u8 *ptmp;
+	int ret;
+
+
+	ret = 0;
+	padapter = (_adapter *)rtw_netdev_priv(dev);
+	dvobj = adapter_to_dvobj(padapter);
+	p = &wrqu->data;
+	len = p->length;
+	if (0 == len)
+		return -EINVAL;
+
+	ptmp = (u8 *)rtw_malloc(len);
+	if (NULL == ptmp)
+		return -ENOMEM;
+
+	if (copy_from_user(ptmp, p->pointer, len)) {
+		ret = -EFAULT;
+		goto exit;
+	}
+
+	bytes = 0;
+	addr = 0;
+	sscanf(ptmp, "%d,%x", &bytes, &addr);
+
+	switch (bytes) {
+	case 1:
+		data32 = rtw_phl_read8(dvobj->phl, addr);
+		sprintf(extra, "0x%02X", data32);
+		break;
+	case 2:
+		data32 = rtw_phl_read16(dvobj->phl, addr);
+		sprintf(extra, "0x%04X", data32);
+		break;
+	case 4:
+		data32 = rtw_phl_read32(dvobj->phl, addr);
+		sprintf(extra, "0x%08X", data32);
+		break;
+#if 0
+	#if defined(CONFIG_SDIO_HCI) && defined(CONFIG_SDIO_INDIRECT_ACCESS) && defined(DBG_SDIO_INDIRECT_ACCESS)
+	case 11:
+		data32 = rtw_sd_iread8(padapter, addr);
+		sprintf(extra, "0x%02X", data32);
+		break;
+	case 12:
+		data32 = rtw_sd_iread16(padapter, addr);
+		sprintf(extra, "0x%04X", data32);
+		break;
+	case 14:
+		data32 = rtw_sd_iread32(padapter, addr);
+		sprintf(extra, "0x%08X", data32);
+		break;
+	#endif
+#endif
+	default:
+		RTW_INFO("%s: usage> read [bytes],[address(hex)]\n", __func__);
+		ret = -EINVAL;
+		goto exit;
+	}
+	RTW_INFO("%s: addr=0x%08X data=%s\n", __func__, addr, extra);
+
+exit:
+	rtw_mfree(ptmp, len);
+	return 0;
+}
+
+static int rtw_wx_write32(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	u32 addr;
+	u32 data32;
+	u32 bytes;
+
+
+	bytes = 0;
+	addr = 0;
+	data32 = 0;
+	sscanf(extra, "%d,%x,%x", &bytes, &addr, &data32);
+
+	switch (bytes) {
+	case 1:
+		rtw_phl_write8(dvobj->phl, addr, (u8)data32);
+		RTW_INFO("%s: addr=0x%08X data=0x%02X\n", __func__, addr, (u8)data32);
+		break;
+	case 2:
+		rtw_phl_write16(dvobj->phl, addr, (u16)data32);
+		RTW_INFO("%s: addr=0x%08X data=0x%04X\n", __func__, addr, (u16)data32);
+		break;
+	case 4:
+		rtw_phl_write32(dvobj->phl, addr, data32);
+		RTW_INFO("%s: addr=0x%08X data=0x%08X\n", __func__, addr, data32);
+		break;
+	default:
+		RTW_INFO("%s: usage> write [bytes],[address(hex)],[data(hex)]\n", __func__);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int rtw_wx_read_rf(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u32 path, addr, data32;
+
+
+	path = *(u32 *)extra;
+	addr = *((u32 *)extra + 1);
+	data32 = rtw_phl_read_rfreg(GET_PHL_INFO(dvobj), path, addr, 0xFFFFF);
+	/*	RTW_INFO("%s: path=%d addr=0x%02x data=0x%05x\n", __func__, path, addr, data32); */
+	/*
+	 * IMPORTANT!!
+	 * Only when wireless private ioctl is at odd order,
+	 * "extra" would be copied to user space.
+	 */
+	sprintf(extra, "0x%05x", data32);
+
+	return 0;
+}
+
+static int rtw_wx_write_rf(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u32 path, addr, data32;
+
+
+	path = *(u32 *)extra;
+	addr = *((u32 *)extra + 1);
+	data32 = *((u32 *)extra + 2);
+	/*	RTW_INFO("%s: path=%d addr=0x%02x data=0x%05x\n", __func__, path, addr, data32); */
+	rtw_phl_write_rfreg(GET_PHL_INFO(dvobj), path, addr, 0xFFFFF, data32);
+
+	return 0;
+}
+
+static int rtw_wx_priv_null(struct net_device *dev, struct iw_request_info *a,
+			    union iwreq_data *wrqu, char *b)
+{
+	return -1;
+}
+
+#if defined(CONFIG_RTW_80211K) //|| defined(CONFIG_RTW_FSM_RRM) TODO
+extern void rm_dbg_cmd(_adapter *padapter, char *s);
+static int rtw_wx_priv_rrm(struct net_device *dev, struct iw_request_info *a,
+			    union iwreq_data *wrqu, char *b)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u32 path, addr, data32;
+
+
+	rm_dbg_cmd(padapter, b);
+	wrqu->data.length = strlen(b);
+
+	return 0;
+}
+#endif
+
+static int dummy(struct net_device *dev, struct iw_request_info *a,
+		 union iwreq_data *wrqu, char *b)
+{
+	/* _adapter *padapter = (_adapter *)rtw_netdev_priv(dev);	 */
+	/* struct mlme_priv *pmlmepriv = &(padapter->mlmepriv); */
+
+	/* RTW_INFO("cmd_code=%x, fwstate=0x%x\n", a->cmd, get_fwstate(pmlmepriv)); */
+
+	return -1;
+
+}
+
+static int rtw_wx_set_channel_plan(struct net_device *dev,
+				   struct iw_request_info *info,
+				   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u16 channel_plan_req = (u16)(*((int *)wrqu));
+
+	rtw_chplan_ioctl_input_mapping(&channel_plan_req, NULL);
+
+	if (_SUCCESS != rtw_set_channel_plan(padapter, channel_plan_req, RTW_CHPLAN_6G_UNSPECIFIED, RTW_ENV_NUM, RTW_REGD_SET_BY_USER))
+		return -EPERM;
+
+	return 0;
+}
+
+static int rtw_wx_set_mtk_wps_probe_ie(struct net_device *dev,
+				       struct iw_request_info *a,
+				       union iwreq_data *wrqu, char *b)
+{
+#ifdef CONFIG_PLATFORM_MT53XX
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+#endif
+	return 0;
+}
+
+static int rtw_wx_get_sensitivity(struct net_device *dev,
+				  struct iw_request_info *info,
+				  union iwreq_data *wrqu, char *buf)
+{
+#ifdef CONFIG_PLATFORM_MT53XX
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	/*	Modified by Albert 20110914 */
+	/*	This is in dbm format for MTK platform. */
+	wrqu->qual.level = padapter->recvinfo.rssi;
+	RTW_INFO(" level = %u\n",  wrqu->qual.level);
+#endif
+	return 0;
+}
+
+static int rtw_wx_set_mtk_wps_ie(struct net_device *dev,
+				 struct iw_request_info *info,
+				 union iwreq_data *wrqu, char *extra)
+{
+#ifdef CONFIG_PLATFORM_MT53XX
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	return rtw_set_wpa_ie(padapter, wrqu->data.pointer, wrqu->data.length);
+#else
+	return 0;
+#endif
+}
+
+#ifdef MP_IOCTL_HDL
+static void rtw_dbg_mode_hdl(_adapter *padapter, u32 id, u8 *pdata, u32 len)
+{
+	pRW_Reg	RegRWStruct;
+	struct rf_reg_param *prfreg;
+	struct dvobj_priv dvobj = adapter_to_dvobj(padapter);
+	u8 path;
+	u8 offset;
+	u32 value;
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	switch (id) {
+	case GEN_MP_IOCTL_SUBCODE(MP_START):
+		RTW_INFO("871x_driver is only for normal mode, can't enter mp mode\n");
+		break;
+	case GEN_MP_IOCTL_SUBCODE(READ_REG):
+		RegRWStruct = (pRW_Reg)pdata;
+		switch (RegRWStruct->width) {
+		case 1:
+			RegRWStruct->value = rtw_read8(padapter, RegRWStruct->offset);
+			break;
+		case 2:
+			RegRWStruct->value = rtw_read16(padapter, RegRWStruct->offset);
+			break;
+		case 4:
+			RegRWStruct->value = rtw_read32(padapter, RegRWStruct->offset);
+			break;
+		default:
+			break;
+		}
+
+		break;
+	case GEN_MP_IOCTL_SUBCODE(WRITE_REG):
+		RegRWStruct = (pRW_Reg)pdata;
+		switch (RegRWStruct->width) {
+		case 1:
+			rtw_write8(padapter, RegRWStruct->offset, (u8)RegRWStruct->value);
+			break;
+		case 2:
+			rtw_write16(padapter, RegRWStruct->offset, (u16)RegRWStruct->value);
+			break;
+		case 4:
+			rtw_write32(padapter, RegRWStruct->offset, (u32)RegRWStruct->value);
+			break;
+		default:
+			break;
+		}
+
+		break;
+	case GEN_MP_IOCTL_SUBCODE(READ_RF_REG):
+
+		prfreg = (struct rf_reg_param *)pdata;
+
+		path = (u8)prfreg->path;
+		offset = (u8)prfreg->offset;
+
+		value = rtw_phl_read_rfreg(GET_PHL_INFO(dvobj), path, offset, 0xffffffff);
+
+		prfreg->value = value;
+
+		break;
+	case GEN_MP_IOCTL_SUBCODE(WRITE_RF_REG):
+
+		prfreg = (struct rf_reg_param *)pdata;
+
+		path = (u8)prfreg->path;
+		offset = (u8)prfreg->offset;
+		value = prfreg->value;
+
+		rtw_phl_write_rfreg(GET_PHL_INFO(dvobj), path, offset, 0xffffffff, value);
+
+		break;
+	case GEN_MP_IOCTL_SUBCODE(TRIGGER_GPIO):
+		RTW_INFO("==> trigger gpio 0\n");
+		rtw_hal_set_hwreg(padapter, HW_VAR_TRIGGER_GPIO_0, 0);
+		break;
+#ifdef CONFIG_BTC
+	case GEN_MP_IOCTL_SUBCODE(SET_DM_BT):
+		RTW_INFO("==> set dm_bt_coexist:%x\n", *(u8 *)pdata);
+		rtw_hal_set_hwreg(padapter, HW_VAR_BT_SET_COEXIST, pdata);
+		break;
+	case GEN_MP_IOCTL_SUBCODE(DEL_BA):
+		RTW_INFO("==> delete ba:%x\n", *(u8 *)pdata);
+		rtw_hal_set_hwreg(padapter, HW_VAR_BT_ISSUE_DELBA, pdata);
+		break;
+#endif
+
+	default:
+		break;
+	}
+
+}
+static int rtw_mp_ioctl_hdl(struct net_device *dev, struct iw_request_info *info,
+			    union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+	u32 BytesRead, BytesWritten, BytesNeeded;
+	struct oid_par_priv	oid_par;
+	struct mp_ioctl_handler	*phandler;
+	struct mp_ioctl_param	*poidparam;
+	uint status = 0;
+	u16 len;
+	u8 *pparmbuf = NULL, bset;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct iw_point *p = &wrqu->data;
+
+	/* RTW_INFO("+rtw_mp_ioctl_hdl\n"); */
+
+	/* mutex_lock(&ioctl_mutex); */
+
+	if ((!p->length) || (!p->pointer)) {
+		ret = -EINVAL;
+		goto _rtw_mp_ioctl_hdl_exit;
+	}
+
+	pparmbuf = NULL;
+	bset = (u8)(p->flags & 0xFFFF);
+	len = p->length;
+	pparmbuf = (u8 *)rtw_malloc(len);
+	if (pparmbuf == NULL) {
+		ret = -ENOMEM;
+		goto _rtw_mp_ioctl_hdl_exit;
+	}
+
+	if (copy_from_user(pparmbuf, p->pointer, len)) {
+		ret = -EFAULT;
+		goto _rtw_mp_ioctl_hdl_exit;
+	}
+
+	poidparam = (struct mp_ioctl_param *)pparmbuf;
+
+	if (poidparam->subcode >= MAX_MP_IOCTL_SUBCODE) {
+		ret = -EINVAL;
+		goto _rtw_mp_ioctl_hdl_exit;
+	}
+
+	/* RTW_INFO("%s: %d\n", __func__, poidparam->subcode); */
+#ifdef CONFIG_MP_INCLUDED
+	if (padapter->registrypriv.mp_mode == 1) {
+		phandler = mp_ioctl_hdl + poidparam->subcode;
+
+		if ((phandler->paramsize != 0) && (poidparam->len < phandler->paramsize)) {
+			ret = -EINVAL;
+			goto _rtw_mp_ioctl_hdl_exit;
+		}
+
+		if (phandler->handler) {
+			oid_par.adapter_context = padapter;
+			oid_par.oid = phandler->oid;
+			oid_par.information_buf = poidparam->data;
+			oid_par.information_buf_len = poidparam->len;
+			oid_par.dbg = 0;
+
+			BytesWritten = 0;
+			BytesNeeded = 0;
+
+			if (bset) {
+				oid_par.bytes_rw = &BytesRead;
+				oid_par.bytes_needed = &BytesNeeded;
+				oid_par.type_of_oid = SET_OID;
+			} else {
+				oid_par.bytes_rw = &BytesWritten;
+				oid_par.bytes_needed = &BytesNeeded;
+				oid_par.type_of_oid = QUERY_OID;
+			}
+
+			status = phandler->handler(&oid_par);
+
+			/* todo:check status, BytesNeeded, etc. */
+		} else {
+			RTW_INFO("rtw_mp_ioctl_hdl(): err!, subcode=%d, oid=%d, handler=%p\n",
+				poidparam->subcode, phandler->oid, phandler->handler);
+			ret = -EFAULT;
+			goto _rtw_mp_ioctl_hdl_exit;
+		}
+	} else
+#endif
+	{
+		rtw_dbg_mode_hdl(padapter, poidparam->subcode, poidparam->data, poidparam->len);
+	}
+
+	if (bset == 0x00) {/* query info */
+		if (copy_to_user(p->pointer, pparmbuf, len))
+			ret = -EFAULT;
+	}
+
+	if (status) {
+		ret = -EFAULT;
+		goto _rtw_mp_ioctl_hdl_exit;
+	}
+
+_rtw_mp_ioctl_hdl_exit:
+
+	if (pparmbuf)
+		rtw_mfree(pparmbuf, len);
+
+	/* mutex_unlock(&ioctl_mutex); */
+
+	return ret;
+}
+#endif /*MP_IOCTL_HDL*/
+static int rtw_get_ap_info(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+	u32 cnt = 0, wpa_ielen;
+	_list	*plist, *phead;
+	unsigned char *pbuf;
+	u8 bssid[ETH_ALEN];
+	char data[32];
+	struct wlan_network *pnetwork = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	_queue *queue = &(pmlmepriv->scanned_queue);
+	struct iw_point *pdata = &wrqu->data;
+
+	RTW_INFO("+rtw_get_aplist_info\n");
+
+	if (dev_is_drv_stopped(adapter_to_dvobj(padapter)) || (pdata == NULL)) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	while ((check_fwstate(pmlmepriv, (WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING))) == _TRUE) {
+		rtw_msleep_os(30);
+		cnt++;
+		if (cnt > 100)
+			break;
+	}
+
+
+	/* pdata->length = 0; */ /* ?	 */
+	pdata->flags = 0;
+	if (pdata->length >= 32) {
+		if (copy_from_user(data, pdata->pointer, 32)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+	} else {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	_rtw_spinlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	phead = get_list_head(queue);
+	plist = get_next(phead);
+
+	while (1) {
+		if (rtw_end_of_queue_search(phead, plist) == _TRUE)
+			break;
+
+
+		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
+
+		/* if(hwaddr_aton_i(pdata->pointer, bssid)) */
+		if (hwaddr_aton_i(data, bssid)) {
+			RTW_INFO("Invalid BSSID '%s'.\n", (u8 *)data);
+			_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+			return -EINVAL;
+		}
+
+
+		if (_rtw_memcmp(bssid, pnetwork->network.MacAddress, ETH_ALEN) == _TRUE) { /* BSSID match, then check if supporting wpa/wpa2 */
+			RTW_INFO("BSSID:" MAC_FMT "\n", MAC_ARG(bssid));
+
+			pbuf = rtw_get_wpa_ie(&pnetwork->network.IEs[12], &wpa_ielen, pnetwork->network.IELength - 12);
+			if (pbuf && (wpa_ielen > 0)) {
+				pdata->flags = 1;
+				break;
+			}
+
+			pbuf = rtw_get_wpa2_ie(&pnetwork->network.IEs[12], &wpa_ielen, pnetwork->network.IELength - 12);
+			if (pbuf && (wpa_ielen > 0)) {
+				pdata->flags = 2;
+				break;
+			}
+
+		}
+
+		plist = get_next(plist);
+
+	}
+
+	_rtw_spinunlock_bh(&(pmlmepriv->scanned_queue.lock));
+
+	if (pdata->length >= 34) {
+		if (copy_to_user((u8 *)pdata->pointer + 32, (u8 *)&pdata->flags, 1)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+	}
+
+exit:
+
+	return ret;
+
+}
+
+static int rtw_set_pid(struct net_device *dev,
+		       struct iw_request_info *info,
+		       union iwreq_data *wrqu, char *extra)
+{
+
+	int ret = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	int *pdata = (int *)wrqu;
+	int selector;
+
+	if (dev_is_drv_stopped(adapter_to_dvobj(padapter)) || (pdata == NULL)) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	selector = *pdata;
+	if (selector < 3 && selector >= 0) {
+		padapter->pid[selector] = *(pdata + 1);
+#ifdef CONFIG_GLOBAL_UI_PID
+		ui_pid[selector] = *(pdata + 1);
+#endif
+		RTW_INFO("%s set pid[%d]=%d\n", __FUNCTION__, selector , padapter->pid[selector]);
+	} else
+		RTW_INFO("%s selector %d error\n", __FUNCTION__, selector);
+
+exit:
+
+	return ret;
+
+}
+
+static int rtw_wps_start(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct iw_point *pdata = &wrqu->data;
+	u32   u32wps_start = 0;
+	unsigned int uintRet = 0;
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter)) || (NULL == pdata)) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	uintRet = copy_from_user((void *) &u32wps_start, pdata->pointer, 4);
+	if (u32wps_start == 0)
+		u32wps_start = *extra;
+
+	RTW_INFO("[%s] wps_start = %d\n", __FUNCTION__, u32wps_start);
+
+	if (u32wps_start == 1)   /* WPS Start */
+		rtw_led_control(padapter, LED_CTL_START_WPS);
+	else if (u32wps_start == 2)   /* WPS Stop because of wps success */
+		rtw_led_control(padapter, LED_CTL_STOP_WPS);
+	else if (u32wps_start == 3)   /* WPS Stop because of wps fail */
+		rtw_led_control(padapter, LED_CTL_STOP_WPS_FAIL);
+
+exit:
+
+	return ret;
+
+}
+
+#ifdef DBG_CMD_QUEUE
+u8 dump_cmd_id = 0;
+#endif
+
+static int rtw_dbg_port(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+	return ret;
+}
+
+static int wpa_set_param(struct net_device *dev, u8 name, u32 value)
+{
+	uint ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	switch (name) {
+	case IEEE_PARAM_WPA_ENABLED:
+
+		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X; /* 802.1x */
+
+		/* ret = ieee80211_wpa_enable(ieee, value); */
+
+		switch ((value) & 0xff) {
+		case 1: /* WPA */
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPAPSK; /* WPA_PSK */
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled;
+			break;
+		case 2: /* WPA2 */
+			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPA2PSK; /* WPA2_PSK */
+			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
+			break;
+		}
+
+
+		break;
+
+	case IEEE_PARAM_TKIP_COUNTERMEASURES:
+		/* ieee->tkip_countermeasures=value; */
+		break;
+
+	case IEEE_PARAM_DROP_UNENCRYPTED: {
+		/* HACK:
+		 *
+		 * wpa_supplicant calls set_wpa_enabled when the driver
+		 * is loaded and unloaded, regardless of if WPA is being
+		 * used.  No other calls are made which can be used to
+		 * determine if encryption will be used or not prior to
+		 * association being expected.  If encryption is not being
+		 * used, drop_unencrypted is set to false, else true -- we
+		 * can use this to determine if the CAP_PRIVACY_ON bit should
+		 * be set.
+		 */
+
+#if 0
+		struct ieee80211_security sec = {
+			.flags = SEC_ENABLED,
+			.enabled = value,
+		};
+		ieee->drop_unencrypted = value;
+		/* We only change SEC_LEVEL for open mode. Others
+		 * are set by ipw_wpa_set_encryption.
+		 */
+		if (!value) {
+			sec.flags |= SEC_LEVEL;
+			sec.level = SEC_LEVEL_0;
+		} else {
+			sec.flags |= SEC_LEVEL;
+			sec.level = SEC_LEVEL_1;
+		}
+		if (ieee->set_security)
+			ieee->set_security(ieee->dev, &sec);
+#endif
+		break;
+
+	}
+	case IEEE_PARAM_PRIVACY_INVOKED:
+
+		/* ieee->privacy_invoked=value; */
+
+		break;
+
+	case IEEE_PARAM_AUTH_ALGS:
+
+		ret = wpa_set_auth_algs(dev, value);
+
+		break;
+
+	case IEEE_PARAM_IEEE_802_1X:
+
+		/* ieee->ieee802_1x=value;		 */
+
+		break;
+
+	case IEEE_PARAM_WPAX_SELECT:
+
+		/* added for WPA2 mixed mode */
+		/*RTW_WARN("------------------------>wpax value = %x\n", value);*/
+		/*
+		spin_lock_irqsave(&ieee->wpax_suitlist_lock,flags);
+		ieee->wpax_type_set = 1;
+		ieee->wpax_type_notify = value;
+		spin_unlock_irqrestore(&ieee->wpax_suitlist_lock,flags);
+		*/
+
+		break;
+
+	default:
+
+
+
+		ret = -EOPNOTSUPP;
+
+
+		break;
+
+	}
+
+	return ret;
+
+}
+
+static int wpa_mlme(struct net_device *dev, u32 command, u32 reason)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	switch (command) {
+	case IEEE_MLME_STA_DEAUTH:
+
+		if (!rtw_set_802_11_disassociate(padapter))
+			ret = -1;
+
+		break;
+
+	case IEEE_MLME_STA_DISASSOC:
+
+		if (!rtw_set_802_11_disassociate(padapter))
+			ret = -1;
+
+		break;
+
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+	return ret;
+
+}
+
+static int wpa_supplicant_ioctl(struct net_device *dev, struct iw_point *p)
+{
+	struct ieee_param *param;
+	uint ret = 0;
+
+	/* down(&ieee->wx_sem);	 */
+
+	if (p->length < sizeof(struct ieee_param) || !p->pointer) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	param = (struct ieee_param *)rtw_malloc(p->length);
+	if (param == NULL) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	if (copy_from_user(param, p->pointer, p->length)) {
+		rtw_mfree((u8 *)param, p->length);
+		ret = -EFAULT;
+		goto out;
+	}
+
+	switch (param->cmd) {
+
+	case IEEE_CMD_SET_WPA_PARAM:
+		ret = wpa_set_param(dev, param->u.wpa_param.name, param->u.wpa_param.value);
+		break;
+
+	case IEEE_CMD_SET_WPA_IE:
+		/* ret = wpa_set_wpa_ie(dev, param, p->length); */
+		ret =  rtw_set_wpa_ie((_adapter *)rtw_netdev_priv(dev), (char *)param->u.wpa_ie.data, (u16)param->u.wpa_ie.len);
+		break;
+
+	case IEEE_CMD_SET_ENCRYPTION:
+		ret = wpa_set_encryption(dev, param, p->length);
+		break;
+
+	case IEEE_CMD_MLME:
+		ret = wpa_mlme(dev, param->u.mlme.command, param->u.mlme.reason_code);
+		break;
+
+	default:
+		RTW_INFO("Unknown WPA supplicant request: %d\n", param->cmd);
+		ret = -EOPNOTSUPP;
+		break;
+
+	}
+
+	if (ret == 0 && copy_to_user(p->pointer, param, p->length))
+		ret = -EFAULT;
+
+	rtw_mfree((u8 *)param, p->length);
+
+out:
+
+	/* up(&ieee->wx_sem); */
+
+	return ret;
+
+}
+
+#ifdef CONFIG_AP_MODE
+static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param, u32 param_len)
+{
+	int ret = 0;
+	u32 wep_key_idx, wep_key_len, wep_total_len = 0;
+	NDIS_802_11_WEP	*pwep = NULL;
+	struct sta_info *psta = NULL, *pbcmc_sta = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_priv	*lmlmepriv = &padapter_link->mlmepriv;
+	struct link_security_priv *lsecuritypriv = &(padapter_link->securitypriv);
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	param->u.crypt.err = 0;
+	param->u.crypt.alg[IEEE_CRYPT_ALG_NAME_LEN - 1] = '\0';
+
+	/* sizeof(struct ieee_param) = 64 bytes; */
+	/* if (param_len !=  (u32) ((u8 *) param->u.crypt.key - (u8 *) param) + param->u.crypt.key_len) */
+	if (param_len !=  sizeof(struct ieee_param) + param->u.crypt.key_len) {
+		ret =  -EINVAL;
+		goto exit;
+	}
+
+	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
+	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
+		if (param->u.crypt.idx >= WEP_KEYS
+#ifdef CONFIG_IEEE80211W
+		    && param->u.crypt.idx > BIP_MAX_KEYID
+#endif /* CONFIG_IEEE80211W */
+		   ) {
+			ret = -EINVAL;
+			goto exit;
+		}
+	} else {
+		psta = rtw_get_stainfo(pstapriv, param->sta_addr);
+		if (!psta) {
+			/* ret = -EINVAL; */
+			RTW_INFO("rtw_set_encryption(), sta has already been removed or never been added\n");
+			goto exit;
+		}
+	}
+
+	if (strcmp(param->u.crypt.alg, "none") == 0 && (psta == NULL)) {
+		/* todo:clear default encryption keys */
+
+		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
+		psecuritypriv->ndisencryptstatus = Ndis802_11EncryptionDisabled;
+		psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
+
+		RTW_INFO("clear default encryption keys, keyid=%d\n", param->u.crypt.idx);
+
+		goto exit;
+	}
+
+
+	if (strcmp(param->u.crypt.alg, "WEP") == 0 && (psta == NULL)) {
+		RTW_INFO("r871x_set_encryption, crypt.alg = WEP\n");
+
+		wep_key_idx = param->u.crypt.idx;
+		wep_key_len = param->u.crypt.key_len;
+
+		RTW_INFO("r871x_set_encryption, wep_key_idx=%d, len=%d\n", wep_key_idx, wep_key_len);
+
+		if ((wep_key_idx >= WEP_KEYS) || (wep_key_len <= 0)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+
+
+		if (wep_key_len > 0) {
+			wep_key_len = wep_key_len <= 5 ? 5 : 13;
+			wep_total_len = wep_key_len + FIELD_OFFSET(NDIS_802_11_WEP, KeyMaterial);
+			pwep = (NDIS_802_11_WEP *)rtw_malloc(wep_total_len);
+			if (pwep == NULL) {
+				RTW_INFO(" r871x_set_encryption: pwep allocate fail !!!\n");
+				goto exit;
+			}
+
+			_rtw_memset(pwep, 0, wep_total_len);
+
+			pwep->KeyLength = wep_key_len;
+			pwep->Length = wep_total_len;
+
+		}
+
+		pwep->KeyIndex = wep_key_idx;
+
+		_rtw_memcpy(pwep->KeyMaterial,  param->u.crypt.key, pwep->KeyLength);
+
+		if (param->u.crypt.set_tx) {
+			RTW_INFO("wep, set_tx=1\n");
+
+			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+			psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
+			psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+			psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
+
+			if (pwep->KeyLength == 13) {
+				psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+				psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
+			}
+
+
+			psecuritypriv->dot11PrivacyKeyIndex = wep_key_idx;
+
+			_rtw_memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), pwep->KeyMaterial, pwep->KeyLength);
+
+			psecuritypriv->dot11DefKeylen[wep_key_idx] = pwep->KeyLength;
+
+			rtw_ap_set_wep_key(padapter,padapter_link,
+					pwep->KeyMaterial, pwep->KeyLength, wep_key_idx, 1);
+		} else {
+			RTW_INFO("wep, set_tx=0\n");
+
+			/* don't update "psecuritypriv->dot11PrivacyAlgrthm" and  */
+			/* "psecuritypriv->dot11PrivacyKeyIndex=keyid", but can rtw_set_key to cam */
+
+			_rtw_memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), pwep->KeyMaterial, pwep->KeyLength);
+
+			psecuritypriv->dot11DefKeylen[wep_key_idx] = pwep->KeyLength;
+
+			rtw_ap_set_wep_key(padapter, padapter_link,
+					pwep->KeyMaterial, pwep->KeyLength, wep_key_idx, 0);
+		}
+
+		goto exit;
+
+	}
+
+
+	if (!psta && check_fwstate(pmlmepriv, WIFI_AP_STATE)) /*  */ { /* group key */
+		if (param->u.crypt.set_tx == 1) {
+			if (strcmp(param->u.crypt.alg, "WEP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set WEP TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				if (param->u.crypt.key_len == 13)
+					psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
+
+			} else if (strcmp(param->u.crypt.alg, "TKIP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set TKIP TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _TKIP_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				/* set mic key */
+				_rtw_memcpy(lsecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
+				_rtw_memcpy(lsecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+				psecuritypriv->busetkipkey = _TRUE;
+
+			} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set CCMP TX GTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				psecuritypriv->dot118021XGrpPrivacy = _AES_;
+				_rtw_memcpy(lsecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+
+			#ifdef CONFIG_IEEE80211W
+			} else if (strcmp(param->u.crypt.alg, "BIP") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" set TX IGTK idx:%u, len:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx, param->u.crypt.key_len);
+				_rtw_memcpy(lsecuritypriv->dot11wBIPKey[param->u.crypt.idx].skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+				lsecuritypriv->dot11wBIPKeyid = param->u.crypt.idx;
+				lsecuritypriv->dot11wBIPtxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				lsecuritypriv->binstallBIPkey = _TRUE;
+				goto exit;
+			#endif /* CONFIG_IEEE80211W */
+
+			} else if (strcmp(param->u.crypt.alg, "none") == 0) {
+				RTW_INFO(FUNC_ADPT_FMT" clear group key, idx:%u\n"
+					, FUNC_ADPT_ARG(padapter), param->u.crypt.idx);
+				psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
+			} else {
+				RTW_WARN(FUNC_ADPT_FMT" set group key, not support\n"
+					, FUNC_ADPT_ARG(padapter));
+				goto exit;
+			}
+
+			lsecuritypriv->dot118021XGrpKeyid = param->u.crypt.idx;
+			pbcmc_sta = rtw_get_bcmc_stainfo(padapter, padapter_link);
+			if (pbcmc_sta) {
+				pbcmc_sta->dot11txpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				pbcmc_sta->ieee8021x_blocked = _FALSE;
+				pbcmc_sta->dot118021XPrivacy = psecuritypriv->dot118021XGrpPrivacy; /* rx will use bmc_sta's dot118021XPrivacy			 */
+			}
+			lsecuritypriv->binstallGrpkey = _TRUE;
+			psecuritypriv->dot11PrivacyAlgrthm = psecuritypriv->dot118021XGrpPrivacy;/* !!! */
+
+			rtw_ap_set_group_key(padapter, padapter_link, param->u.crypt.key, psecuritypriv->dot118021XGrpPrivacy, param->u.crypt.idx);
+		}
+
+		goto exit;
+
+	}
+
+	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X && psta) { /* psk/802_1x */
+		if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+			if (param->u.crypt.set_tx == 1) {
+				_rtw_memcpy(psta->dot118021x_UncstKey.skey,  param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
+
+				if (strcmp(param->u.crypt.alg, "WEP") == 0) {
+					RTW_INFO(FUNC_ADPT_FMT" set WEP PTK of "MAC_FMT" idx:%u, len:%u\n"
+						, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+						, param->u.crypt.idx, param->u.crypt.key_len);
+					psta->dot118021XPrivacy = _WEP40_;
+					if (param->u.crypt.key_len == 13)
+						psta->dot118021XPrivacy = _WEP104_;
+
+				} else if (strcmp(param->u.crypt.alg, "TKIP") == 0) {
+					RTW_INFO(FUNC_ADPT_FMT" set TKIP PTK of "MAC_FMT" idx:%u, len:%u\n"
+						, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+						, param->u.crypt.idx, param->u.crypt.key_len);
+					psta->dot118021XPrivacy = _TKIP_;
+					/* set mic key */
+					_rtw_memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
+					_rtw_memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+					psecuritypriv->busetkipkey = _TRUE;
+
+				} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
+					RTW_INFO(FUNC_ADPT_FMT" set CCMP PTK of "MAC_FMT" idx:%u, len:%u\n"
+						, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+						, param->u.crypt.idx, param->u.crypt.key_len);
+					psta->dot118021XPrivacy = _AES_;
+
+				} else if (strcmp(param->u.crypt.alg, "none") == 0) {
+					RTW_INFO(FUNC_ADPT_FMT" clear pairwise key of "MAC_FMT" idx:%u\n"
+						, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr)
+						, param->u.crypt.idx);
+					psta->dot118021XPrivacy = _NO_PRIVACY_;
+
+				} else {
+					RTW_WARN(FUNC_ADPT_FMT" set pairwise key of "MAC_FMT", not support\n"
+						, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr));
+					goto exit;
+				}
+
+				psta->dot11txpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				psta->dot11rxpn.val = RTW_GET_LE64(param->u.crypt.seq);
+				psta->ieee8021x_blocked = _FALSE;
+
+				if (psta->dot118021XPrivacy != _NO_PRIVACY_) {
+					psta->bpairwise_key_installed = _TRUE;
+
+					/* WPA2 key-handshake has completed */
+					if (psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPA2PSK)
+						psta->state &= (~WIFI_UNDER_KEY_HANDSHAKE);
+				}
+
+				rtw_ap_set_pairwise_key(padapter, psta);
+			} else {
+				RTW_WARN(FUNC_ADPT_FMT" set group key of "MAC_FMT", not support\n"
+					, FUNC_ADPT_ARG(padapter), MAC_ARG(psta->phl_sta->mac_addr));
+				goto exit;
+			}
+
+		}
+
+	}
+
+exit:
+
+	if (pwep)
+		rtw_mfree((u8 *)pwep, wep_total_len);
+
+	return ret;
+
+}
+
+static int rtw_set_beacon(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	unsigned char *pbuf = param->u.bcn_ie.buf;
+
+
+	RTW_INFO("%s, len=%d\n", __FUNCTION__, len);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	_rtw_memcpy(&pstapriv->max_num_sta, param->u.bcn_ie.reserved, 2);
+
+	if ((pstapriv->max_num_sta > NUM_STA) || (pstapriv->max_num_sta <= 0))
+		pstapriv->max_num_sta = NUM_STA;
+
+
+	if (rtw_check_beacon_data(padapter, pbuf, (len - 12 - 2)) == _SUCCESS) /* 12 = param header, 2:no packed */
+		ret = 0;
+	else
+		ret = -EINVAL;
+
+
+	return ret;
+
+}
+
+static int rtw_hostapd_sta_flush(struct net_device *dev)
+{
+	/* _list	*phead, *plist; */
+	int ret = 0;
+	/* struct sta_info *psta = NULL; */
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	/* struct sta_priv *pstapriv = &padapter->stapriv; */
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	flush_all_cam_entry(padapter, PHL_CMD_WAIT, 50);	/* clear CAM */
+#ifdef CONFIG_AP_MODE
+	ret = rtw_sta_flush(padapter, _TRUE);
+#endif
+	return ret;
+
+}
+
+static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
+{
+	int ret = 0;
+	struct sta_info *psta = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct link_mlme_priv *pmlmepriv;
+
+	RTW_INFO("rtw_add_sta(aid=%d)=" MAC_FMT "\n", param->u.add_sta.aid, MAC_ARG(param->sta_addr));
+
+	if (check_fwstate(&padapter->mlmepriv, (WIFI_ASOC_STATE | WIFI_AP_STATE)) != _TRUE)
+		return -EINVAL;
+
+	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
+	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
+		return -EINVAL;
+
+#if 0
+	psta = rtw_get_stainfo(pstapriv, param->sta_addr);
+	if (psta) {
+		RTW_INFO("rtw_add_sta(), free has been added psta=%p\n", psta);
+		/* _rtw_spinlock_bh(&(pstapriv->sta_hash_lock));		 */
+		rtw_free_stainfo(padapter,  psta);
+		/* _rtw_spinunlock_bh(&(pstapriv->sta_hash_lock)); */
+
+		psta = NULL;
+	}
+#endif
+	/* psta = rtw_alloc_stainfo(pstapriv, param->sta_addr); */
+	psta = rtw_get_stainfo(pstapriv, param->sta_addr);
+	if (psta) {
+		int flags = param->u.add_sta.flags;
+		pmlmepriv = &psta->padapter_link->mlmepriv;
+
+		/* RTW_INFO("rtw_add_sta(), init sta's variables, psta=%p\n", psta); */
+
+		psta->phl_sta->aid = param->u.add_sta.aid;/* aid=1~2007 */
+
+		_rtw_memcpy(psta->bssrateset, param->u.add_sta.tx_supp_rates, 16);
+
+
+		/* check wmm cap. */
+		if (WLAN_STA_WME & flags)
+			psta->qos_option = 1;
+		else
+			psta->qos_option = 0;
+
+		if (pmlmepriv->qospriv.qos_option == 0)
+			psta->qos_option = 0;
+
+
+#ifdef CONFIG_80211N_HT
+		/* chec 802.11n ht cap. */
+		if (padapter->registrypriv.ht_enable &&
+			is_supported_ht(padapter->registrypriv.wireless_mode) &&
+			(WLAN_STA_HT & flags)) {
+			psta->htpriv.ht_option = _TRUE;
+			psta->qos_option = 1;
+			_rtw_memcpy((void *)&psta->htpriv.ht_cap, (void *)&param->u.add_sta.ht_cap, sizeof(struct rtw_ieee80211_ht_cap));
+		} else
+			psta->htpriv.ht_option = _FALSE;
+
+		if (pmlmepriv->htpriv.ht_option == _FALSE)
+			psta->htpriv.ht_option = _FALSE;
+
+#endif
+
+
+		update_sta_info_apmode(padapter, psta);
+
+
+	} else
+		ret = -ENOMEM;
+
+	return ret;
+
+}
+
+static int rtw_del_sta(struct net_device *dev, struct ieee_param *param)
+{
+	int ret = 0;
+	struct sta_info *psta = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+
+	RTW_INFO("rtw_del_sta=" MAC_FMT "\n", MAC_ARG(param->sta_addr));
+
+	if (check_fwstate(pmlmepriv, (WIFI_ASOC_STATE | WIFI_AP_STATE)) != _TRUE)
+		return -EINVAL;
+
+	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
+	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
+		return -EINVAL;
+
+	psta = rtw_get_stainfo(pstapriv, param->sta_addr);
+	if (psta) {
+		u8 updated = _FALSE;
+
+		/* RTW_INFO("free psta=%p, aid=%d\n", psta, psta->phl_sta->aid); */
+
+		rtw_stapriv_asoc_list_lock(pstapriv);
+		if (rtw_is_list_empty(&psta->asoc_list) == _FALSE) {
+			rtw_stapriv_asoc_list_del(pstapriv, psta);
+			updated = ap_free_sta(padapter, psta, _TRUE, 0, WLAN_REASON_DEAUTH_LEAVING, _TRUE);
+
+		}
+		rtw_stapriv_asoc_list_unlock(pstapriv);
+
+		associated_clients_update(padapter, updated, STA_INFO_UPDATE_ALL);
+
+		psta = NULL;
+
+	} else {
+		RTW_INFO("rtw_del_sta(), sta has already been removed or never been added\n");
+
+		/* ret = -1; */
+	}
+
+
+	return ret;
+
+}
+
+static int rtw_ioctl_get_sta_data(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	struct sta_info *psta = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct ieee_param_ex *param_ex = (struct ieee_param_ex *)param;
+	struct sta_data *psta_data = (struct sta_data *)param_ex->data;
+
+	RTW_INFO("rtw_ioctl_get_sta_info, sta_addr: " MAC_FMT "\n", MAC_ARG(param_ex->sta_addr));
+
+	if (check_fwstate(pmlmepriv, (WIFI_ASOC_STATE | WIFI_AP_STATE)) != _TRUE)
+		return -EINVAL;
+
+	if (param_ex->sta_addr[0] == 0xff && param_ex->sta_addr[1] == 0xff &&
+	    param_ex->sta_addr[2] == 0xff && param_ex->sta_addr[3] == 0xff &&
+	    param_ex->sta_addr[4] == 0xff && param_ex->sta_addr[5] == 0xff)
+		return -EINVAL;
+
+	psta = rtw_get_stainfo(pstapriv, param_ex->sta_addr);
+	if (psta) {
+#if 0
+		struct {
+			u16 aid;
+			u16 capability;
+			int flags;
+			u32 sta_set;
+			u8 tx_supp_rates[16];
+			u32 tx_supp_rates_len;
+			struct rtw_ieee80211_ht_cap ht_cap;
+			u64	rx_pkts;
+			u64	rx_bytes;
+			u64	rx_drops;
+			u64	tx_pkts;
+			u64	tx_bytes;
+			u64	tx_drops;
+		} get_sta;
+#endif
+		psta_data->aid = (u16)psta->phl_sta->aid;
+		psta_data->capability = psta->capability;
+		psta_data->flags = psta->flags;
+
+		/*
+				nonerp_set : BIT(0)
+				no_short_slot_time_set : BIT(1)
+				no_short_preamble_set : BIT(2)
+				no_ht_gf_set : BIT(3)
+				no_ht_set : BIT(4)
+				ht_20mhz_set : BIT(5)
+		*/
+
+		psta_data->sta_set = ((psta->nonerp_set) |
+				      (psta->no_short_slot_time_set << 1) |
+				      (psta->no_short_preamble_set << 2) |
+				      (psta->no_ht_gf_set << 3) |
+				      (psta->no_ht_set << 4) |
+				      (psta->ht_20mhz_set << 5));
+
+		psta_data->tx_supp_rates_len =  psta->bssratelen;
+		_rtw_memcpy(psta_data->tx_supp_rates, psta->bssrateset, psta->bssratelen);
+#ifdef CONFIG_80211N_HT
+		if(padapter->registrypriv.ht_enable && is_supported_ht(padapter->registrypriv.wireless_mode))
+			_rtw_memcpy(&psta_data->ht_cap, &psta->htpriv.ht_cap, sizeof(struct rtw_ieee80211_ht_cap));
+#endif /* CONFIG_80211N_HT */
+		psta_data->rx_pkts = psta->sta_stats.rx_data_pkts;
+		psta_data->rx_bytes = psta->sta_stats.rx_bytes;
+		psta_data->rx_drops = psta->sta_stats.rx_drops;
+
+		psta_data->tx_pkts = psta->sta_stats.tx_pkts;
+		psta_data->tx_bytes = psta->sta_stats.tx_bytes;
+		psta_data->tx_drops = psta->sta_stats.tx_drops;
+
+
+	} else
+		ret = -1;
+
+	return ret;
+
+}
+
+static int rtw_get_sta_wpaie(struct net_device *dev, struct ieee_param *param)
+{
+	int ret = 0;
+	struct sta_info *psta = NULL;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+
+	RTW_INFO("rtw_get_sta_wpaie, sta_addr: " MAC_FMT "\n", MAC_ARG(param->sta_addr));
+
+	if (check_fwstate(pmlmepriv, (WIFI_ASOC_STATE | WIFI_AP_STATE)) != _TRUE)
+		return -EINVAL;
+
+	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
+	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
+		return -EINVAL;
+
+	psta = rtw_get_stainfo(pstapriv, param->sta_addr);
+	if (psta) {
+		if ((psta->wpa_ie[0] == WLAN_EID_RSN) || (psta->wpa_ie[0] == WLAN_EID_GENERIC)) {
+			int wpa_ie_len;
+			int copy_len;
+
+			wpa_ie_len = psta->wpa_ie[1];
+
+			copy_len = ((wpa_ie_len + 2) > sizeof(psta->wpa_ie)) ? (sizeof(psta->wpa_ie)) : (wpa_ie_len + 2);
+
+			param->u.wpa_ie.len = copy_len;
+
+			_rtw_memcpy(param->u.wpa_ie.reserved, psta->wpa_ie, copy_len);
+		} else {
+			/* ret = -1; */
+			RTW_INFO("sta's wpa_ie is NONE\n");
+		}
+	} else
+		ret = -1;
+
+	return ret;
+
+}
+
+static int rtw_set_wps_beacon(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	unsigned char wps_oui[4] = {0x0, 0x50, 0xf2, 0x04};
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct link_mlme_ext_priv	*pmlmeext = &(padapter_link->mlmeextpriv);
+	int ie_len;
+
+	RTW_INFO("%s, len=%d\n", __FUNCTION__, len);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	ie_len = len - 12 - 2; /* 12 = param header, 2:no packed */
+
+
+	if (pmlmepriv->wps_beacon_ie) {
+		rtw_mfree(pmlmepriv->wps_beacon_ie, pmlmepriv->wps_beacon_ie_len);
+		pmlmepriv->wps_beacon_ie = NULL;
+	}
+
+	if (ie_len > 0) {
+		pmlmepriv->wps_beacon_ie = rtw_malloc(ie_len);
+		pmlmepriv->wps_beacon_ie_len = ie_len;
+		if (pmlmepriv->wps_beacon_ie == NULL) {
+			RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+			return -EINVAL;
+		}
+
+		_rtw_memcpy(pmlmepriv->wps_beacon_ie, param->u.bcn_ie.buf, ie_len);
+
+		rtw_update_beacon(padapter, padapter_link,
+				_VENDOR_SPECIFIC_IE_, wps_oui, _TRUE, 0);
+
+		pmlmeext->bstart_bss = _TRUE;
+
+	}
+
+
+	return ret;
+
+}
+
+static int rtw_set_wps_probe_resp(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	int ie_len;
+
+	RTW_INFO("%s, len=%d\n", __FUNCTION__, len);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	ie_len = len - 12 - 2; /* 12 = param header, 2:no packed */
+
+
+	if (pmlmepriv->wps_probe_resp_ie) {
+		rtw_mfree(pmlmepriv->wps_probe_resp_ie, pmlmepriv->wps_probe_resp_ie_len);
+		pmlmepriv->wps_probe_resp_ie = NULL;
+	}
+
+	if (ie_len > 0) {
+		pmlmepriv->wps_probe_resp_ie = rtw_malloc(ie_len);
+		pmlmepriv->wps_probe_resp_ie_len = ie_len;
+		if (pmlmepriv->wps_probe_resp_ie == NULL) {
+			RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+			return -EINVAL;
+		}
+		_rtw_memcpy(pmlmepriv->wps_probe_resp_ie, param->u.bcn_ie.buf, ie_len);
+	}
+
+
+	return ret;
+
+}
+
+static int rtw_set_wps_assoc_resp(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	int ie_len;
+
+	RTW_INFO("%s, len=%d\n", __FUNCTION__, len);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	ie_len = len - 12 - 2; /* 12 = param header, 2:no packed */
+
+
+	if (pmlmepriv->wps_assoc_resp_ie) {
+		rtw_mfree(pmlmepriv->wps_assoc_resp_ie, pmlmepriv->wps_assoc_resp_ie_len);
+		pmlmepriv->wps_assoc_resp_ie = NULL;
+	}
+
+	if (ie_len > 0) {
+		pmlmepriv->wps_assoc_resp_ie = rtw_malloc(ie_len);
+		pmlmepriv->wps_assoc_resp_ie_len = ie_len;
+		if (pmlmepriv->wps_assoc_resp_ie == NULL) {
+			RTW_INFO("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+			return -EINVAL;
+		}
+
+		_rtw_memcpy(pmlmepriv->wps_assoc_resp_ie, param->u.bcn_ie.buf, ie_len);
+	}
+
+
+	return ret;
+
+}
+
+static int rtw_set_hidden_ssid(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlmepriv = &(adapter->mlmepriv);
+	struct mlme_ext_priv	*mlmeext = &(adapter->mlmeextpriv);
+	struct mlme_ext_info	*mlmeinfo = &(mlmeext->mlmext_info);
+	int ie_len;
+	u8 *ssid_ie;
+	char ssid[NDIS_802_11_LENGTH_SSID + 1];
+	sint ssid_len = 0;
+	u8 ignore_broadcast_ssid;
+
+	if (check_fwstate(mlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EPERM;
+
+	if (param->u.bcn_ie.reserved[0] != 0xea)
+		return -EINVAL;
+
+	mlmeinfo->hidden_ssid_mode = ignore_broadcast_ssid = param->u.bcn_ie.reserved[1];
+
+	ie_len = len - 12 - 2; /* 12 = param header, 2:no packed */
+	ssid_ie = rtw_get_ie(param->u.bcn_ie.buf,  WLAN_EID_SSID, &ssid_len, ie_len);
+
+	if (ssid_ie && ssid_len > 0 && ssid_len <= NDIS_802_11_LENGTH_SSID) {
+		WLAN_BSSID_EX *pbss_network = &mlmepriv->dev_cur_network.network;
+		WLAN_BSSID_EX *pbss_network_ext = &mlmeinfo->dev_network;
+
+		_rtw_memcpy(ssid, ssid_ie + 2, ssid_len);
+		ssid[ssid_len] = 0x0;
+
+		if (0)
+			RTW_INFO(FUNC_ADPT_FMT" ssid:(%s,%d), from ie:(%s,%d), (%s,%d)\n", FUNC_ADPT_ARG(adapter),
+				ssid, ssid_len,
+				pbss_network->Ssid.Ssid, pbss_network->Ssid.SsidLength,
+				pbss_network_ext->Ssid.Ssid, pbss_network_ext->Ssid.SsidLength);
+
+		_rtw_memcpy(pbss_network->Ssid.Ssid, (void *)ssid, ssid_len);
+		pbss_network->Ssid.SsidLength = ssid_len;
+		_rtw_memcpy(pbss_network_ext->Ssid.Ssid, (void *)ssid, ssid_len);
+		pbss_network_ext->Ssid.SsidLength = ssid_len;
+
+		if (0)
+			RTW_INFO(FUNC_ADPT_FMT" after ssid:(%s,%d), (%s,%d)\n", FUNC_ADPT_ARG(adapter),
+				pbss_network->Ssid.Ssid, pbss_network->Ssid.SsidLength,
+				pbss_network_ext->Ssid.Ssid, pbss_network_ext->Ssid.SsidLength);
+	}
+
+	RTW_INFO(FUNC_ADPT_FMT" ignore_broadcast_ssid:%d, %s,%d\n", FUNC_ADPT_ARG(adapter),
+		ignore_broadcast_ssid, ssid, ssid_len);
+
+	return ret;
+}
+
+#if CONFIG_RTW_MACADDR_ACL
+static int rtw_ioctl_acl_remove_sta(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
+	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
+		return -EINVAL;
+
+	ret = rtw_acl_remove_sta(padapter, RTW_ACL_PERIOD_BSS, param->sta_addr);
+
+	return ret;
+
+}
+
+static int rtw_ioctl_acl_add_sta(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
+	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
+		return -EINVAL;
+
+	ret = rtw_acl_add_sta(padapter, RTW_ACL_PERIOD_BSS, param->sta_addr);
+
+	return ret;
+
+}
+
+static int rtw_ioctl_set_macaddr_acl(struct net_device *dev, struct ieee_param *param, int len)
+{
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != _TRUE)
+		return -EINVAL;
+
+	rtw_set_macaddr_acl(padapter, RTW_ACL_PERIOD_BSS, param->u.mlme.command);
+
+	return ret;
+}
+#endif /* CONFIG_RTW_MACADDR_ACL */
+
+static int rtw_hostapd_ioctl(struct net_device *dev, struct iw_point *p)
+{
+	struct ieee_param *param;
+	int ret = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	/*
+	* this function is expect to call in master mode, which allows no power saving
+	* so, we just check hw_init_completed
+	*/
+
+	if (!rtw_hw_is_init_completed(dvobj)) {
+		ret = -EPERM;
+		goto out;
+	}
+
+
+	/* if (p->length < sizeof(struct ieee_param) || !p->pointer){ */
+	if (!p->pointer) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	param = (struct ieee_param *)rtw_malloc(p->length);
+	if (param == NULL) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	if (copy_from_user(param, p->pointer, p->length)) {
+		rtw_mfree((u8 *)param, p->length);
+		ret = -EFAULT;
+		goto out;
+	}
+
+	/* RTW_INFO("%s, cmd=%d\n", __FUNCTION__, param->cmd); */
+
+	switch (param->cmd) {
+	case RTL871X_HOSTAPD_FLUSH:
+
+		ret = rtw_hostapd_sta_flush(dev);
+
+		break;
+
+	case RTL871X_HOSTAPD_ADD_STA:
+
+		ret = rtw_add_sta(dev, param);
+
+		break;
+
+	case RTL871X_HOSTAPD_REMOVE_STA:
+
+		ret = rtw_del_sta(dev, param);
+
+		break;
+
+	case RTL871X_HOSTAPD_SET_BEACON:
+
+		ret = rtw_set_beacon(dev, param, p->length);
+
+		break;
+
+	case RTL871X_SET_ENCRYPTION:
+
+		ret = rtw_set_encryption(dev, param, p->length);
+
+		break;
+
+	case RTL871X_HOSTAPD_GET_WPAIE_STA:
+
+		ret = rtw_get_sta_wpaie(dev, param);
+
+		break;
+
+	case RTL871X_HOSTAPD_SET_WPS_BEACON:
+
+		ret = rtw_set_wps_beacon(dev, param, p->length);
+
+		break;
+
+	case RTL871X_HOSTAPD_SET_WPS_PROBE_RESP:
+
+		ret = rtw_set_wps_probe_resp(dev, param, p->length);
+
+		break;
+
+	case RTL871X_HOSTAPD_SET_WPS_ASSOC_RESP:
+
+		ret = rtw_set_wps_assoc_resp(dev, param, p->length);
+
+		break;
+
+	case RTL871X_HOSTAPD_SET_HIDDEN_SSID:
+
+		ret = rtw_set_hidden_ssid(dev, param, p->length);
+
+		break;
+
+	case RTL871X_HOSTAPD_GET_INFO_STA:
+
+		ret = rtw_ioctl_get_sta_data(dev, param, p->length);
+
+		break;
+
+#if CONFIG_RTW_MACADDR_ACL
+	case RTL871X_HOSTAPD_SET_MACADDR_ACL:
+		ret = rtw_ioctl_set_macaddr_acl(dev, param, p->length);
+		break;
+	case RTL871X_HOSTAPD_ACL_ADD_STA:
+		ret = rtw_ioctl_acl_add_sta(dev, param, p->length);
+		break;
+	case RTL871X_HOSTAPD_ACL_REMOVE_STA:
+		ret = rtw_ioctl_acl_remove_sta(dev, param, p->length);
+		break;
+#endif /* CONFIG_RTW_MACADDR_ACL */
+
+	default:
+		RTW_INFO("Unknown hostapd request: %d\n", param->cmd);
+		ret = -EOPNOTSUPP;
+		break;
+
+	}
+
+	if (ret == 0 && copy_to_user(p->pointer, param, p->length))
+		ret = -EFAULT;
+
+
+	rtw_mfree((u8 *)param, p->length);
+
+out:
+
+	return ret;
+
+}
+#endif
+
+static int rtw_wx_set_priv(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *awrq,
+			   char *extra)
+{
+
+#ifdef CONFIG_DEBUG_RTW_WX_SET_PRIV
+	char *ext_dbg;
+#endif
+
+	int ret = 0;
+	int len = 0;
+	char *ext;
+#ifdef CONFIG_RTW_ANDROID
+	int i;
+#endif
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct iw_point *dwrq = (struct iw_point *)awrq;
+
+	if (dwrq->length == 0)
+		return -EFAULT;
+
+	len = dwrq->length;
+	ext = rtw_vmalloc(len);
+	if (!ext)
+		return -ENOMEM;
+
+	if (copy_from_user(ext, dwrq->pointer, len)) {
+		rtw_vmfree(ext, len);
+		return -EFAULT;
+	}
+
+
+
+#ifdef CONFIG_DEBUG_RTW_WX_SET_PRIV
+	ext_dbg = rtw_vmalloc(len);
+	if (!ext_dbg) {
+		rtw_vmfree(ext, len);
+		return -ENOMEM;
+	}
+
+	_rtw_memcpy(ext_dbg, ext, len);
+#endif
+
+	/* added for wps2.0 @20110524 */
+	if (dwrq->flags == 0x8766 && len > 8) {
+		u32 cp_sz;
+		struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+		u8 *probereq_wpsie = ext;
+		int probereq_wpsie_len = len;
+		u8 wps_oui[4] = {0x0, 0x50, 0xf2, 0x04};
+
+		if ((_VENDOR_SPECIFIC_IE_ == probereq_wpsie[0]) &&
+		    (_rtw_memcmp(&probereq_wpsie[2], wps_oui, 4) == _TRUE)) {
+			cp_sz = probereq_wpsie_len > MAX_WPS_IE_LEN ? MAX_WPS_IE_LEN : probereq_wpsie_len;
+
+			if (pmlmepriv->wps_probe_req_ie) {
+				u32 free_len = pmlmepriv->wps_probe_req_ie_len;
+				pmlmepriv->wps_probe_req_ie_len = 0;
+				rtw_mfree(pmlmepriv->wps_probe_req_ie, free_len);
+				pmlmepriv->wps_probe_req_ie = NULL;
+			}
+
+			pmlmepriv->wps_probe_req_ie = rtw_malloc(cp_sz);
+			if (pmlmepriv->wps_probe_req_ie == NULL) {
+				printk("%s()-%d: rtw_malloc() ERROR!\n", __FUNCTION__, __LINE__);
+				ret =  -EINVAL;
+				goto FREE_EXT;
+
+			}
+
+			_rtw_memcpy(pmlmepriv->wps_probe_req_ie, probereq_wpsie, cp_sz);
+			pmlmepriv->wps_probe_req_ie_len = cp_sz;
+
+		}
+
+		goto FREE_EXT;
+
+	}
+
+	if (len >= WEXT_CSCAN_HEADER_SIZE
+		&& _rtw_memcmp(ext, WEXT_CSCAN_HEADER, WEXT_CSCAN_HEADER_SIZE) == _TRUE
+	) {
+		ret = rtw_wx_set_scan(dev, info, awrq, ext);
+		goto FREE_EXT;
+	}
+
+#ifdef CONFIG_RTW_ANDROID
+	/* RTW_INFO("rtw_wx_set_priv: %s req=%s\n", dev->name, ext); */
+
+	i = rtw_android_cmdstr_to_num(ext);
+
+	switch (i) {
+	case ANDROID_WIFI_CMD_START:
+		indicate_wx_custom_event(padapter, "START");
+		break;
+	case ANDROID_WIFI_CMD_STOP:
+		indicate_wx_custom_event(padapter, "STOP");
+		break;
+	case ANDROID_WIFI_CMD_RSSI: {
+		struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+		struct	wlan_network	*pcur_network = &pmlmepriv->dev_cur_network;
+
+		if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+			sprintf(ext, "%s rssi %d", pcur_network->network.Ssid.Ssid, padapter->recvinfo.rssi);
+		else
+			sprintf(ext, "OK");
+	}
+		break;
+	case ANDROID_WIFI_CMD_LINKSPEED: {
+		u16 mbps = rtw_get_cur_max_rate(padapter) / 10;
+		sprintf(ext, "LINKSPEED %d", mbps);
+	}
+		break;
+	case ANDROID_WIFI_CMD_MACADDR:
+		sprintf(ext, "MACADDR = " MAC_FMT, MAC_ARG(dev->dev_addr));
+		break;
+	case ANDROID_WIFI_CMD_SCAN_ACTIVE: {
+		/* rtw_set_scan_mode(padapter, SCAN_ACTIVE); */
+		sprintf(ext, "OK");
+	}
+		break;
+	case ANDROID_WIFI_CMD_SCAN_PASSIVE: {
+		/* rtw_set_scan_mode(padapter, SCAN_PASSIVE); */
+		sprintf(ext, "OK");
+	}
+		break;
+
+	case ANDROID_WIFI_CMD_COUNTRY: {
+		char country_code[10];
+		sscanf(ext, "%*s %s", country_code);
+		rtw_set_country(padapter, country_code, RTW_ENV_NUM, RTW_REGD_SET_BY_USER);
+		sprintf(ext, "OK");
+	}
+		break;
+	default:
+		#ifdef CONFIG_DEBUG_RTW_WX_SET_PRIV
+		RTW_INFO("%s: %s unknowned req=%s\n", __FUNCTION__,
+			dev->name, ext_dbg);
+		#endif
+
+		sprintf(ext, "OK");
+
+	}
+
+	if (copy_to_user(dwrq->pointer, ext, min(dwrq->length, (u16)(strlen(ext) + 1))))
+		ret = -EFAULT;
+
+#ifdef CONFIG_DEBUG_RTW_WX_SET_PRIV
+	RTW_INFO("%s: %s req=%s rep=%s dwrq->length=%d, strlen(ext)+1=%d\n", __FUNCTION__,
+		dev->name, ext_dbg , ext, dwrq->length, (u16)(strlen(ext) + 1));
+#endif
+#endif /* end of CONFIG_ANDROID */
+
+
+FREE_EXT:
+
+	rtw_vmfree(ext, len);
+	#ifdef CONFIG_DEBUG_RTW_WX_SET_PRIV
+	rtw_vmfree(ext_dbg, len);
+	#endif
+
+	/* RTW_INFO("rtw_wx_set_priv: (SIOCSIWPRIV) %s ret=%d\n",  */
+	/*		dev->name, ret); */
+
+	return ret;
+
+}
+#ifdef CONFIG_WOWLAN
+static int rtw_wowlan_ctrl(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	int ret = 0;
+	systime start_time = rtw_get_current_time();
+
+	RTW_INFO("+rtw_wowlan_ctrl: %s\n", extra);
+
+	if (!check_fwstate(pmlmepriv, WIFI_ASOC_STATE) &&
+		MLME_IS_STA(padapter) &&
+		!WOWLAN_IS_STA_MIX_MODE(padapter)) {
+#ifdef CONFIG_PNO_SUPPORT
+		if (wowpriv->wow_nlo.nlo_en) {
+			pwrctrlpriv->wowlan_pno_enable = _TRUE;
+		} else
+#endif
+		{
+			RTW_INFO("[%s] WARNING: Please Connect With AP First!!\n",
+				 __func__);
+			goto _rtw_wowlan_ctrl_exit_free;
+		}
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY))
+		rtw_scan_abort(padapter, 0);
+
+	if (_rtw_memcmp(extra, "enable", 6))
+
+
+		rtw_suspend_common(padapter);
+
+	else if (_rtw_memcmp(extra, "disable", 7)) {
+#ifdef CONFIG_USB_HCI
+		RTW_ENABLE_FUNC(adapter_to_dvobj(padapter), DF_RX_BIT);
+		RTW_ENABLE_FUNC(adapter_to_dvobj(padapter), DF_TX_BIT);
+#endif
+		rtw_resume_common(padapter);
+
+#ifdef CONFIG_PNO_SUPPORT
+		pwrctrlpriv->wowlan_pno_enable = _FALSE;
+#endif /* CONFIG_PNO_SUPPORT */
+
+	} else {
+		RTW_INFO("[%s] Invalid Parameter.\n", __func__);
+		goto _rtw_wowlan_ctrl_exit_free;
+	}
+	/* mutex_lock(&ioctl_mutex); */
+_rtw_wowlan_ctrl_exit_free:
+	RTW_PRINT("%s in %d ms\n", __func__,
+		  rtw_get_passing_time_ms(start_time));
+	return ret;
+}
+
+/*
+ * IP filter This pattern if for a frame containing a ip packet:
+ * AA:AA:AA:AA:AA:AA:BB:BB:BB:BB:BB:BB:CC:CC:DD:-:-:-:-:-:-:-:-:EE:-:-:FF:FF:FF:FF:GG:GG:GG:GG:HH:HH:II:II
+ *
+ * A: Ethernet destination address
+ * B: Ethernet source address
+ * C: Ethernet protocol type
+ * D: IP header VER+Hlen, use: 0x45 (4 is for ver 4, 5 is for len 20)
+ * E: IP protocol
+ * F: IP source address ( 192.168.0.4: C0:A8:00:2C )
+ * G: IP destination address ( 192.168.0.4: C0:A8:00:2C )
+ * H: Source port (1024: 04:00)
+ * I: Destination port (1024: 04:00)
+ */
+
+static int rtw_wowlan_set_pattern(struct net_device *dev,
+				  struct iw_request_info *info,
+				  union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_wowcam_upd_info wowcam_info = {0};
+	u8 input[MAX_IN_PATTERN_SIZE];
+	int ret = 0;
+
+	if (!(registry_par->wakeup_event & BIT(3))) {
+		ret = -EOPNOTSUPP;
+		RTW_INFO("%s: customized pattern disabled, wakeup_event: %#2x\n",
+			 __func__, registry_par->wakeup_event);
+		goto _rtw_wowlan_set_pattern_exit;
+	}
+
+	if (!check_fwstate(pmlmepriv, WIFI_ASOC_STATE) &&
+	    MLME_IS_STA(padapter)) {
+		ret = -EOPNOTSUPP;
+		RTW_INFO("Please Connect With AP First!!\n");
+		goto _rtw_wowlan_set_pattern_exit;
+	}
+
+	if ((wrqu->data.length <= 0) || (wrqu->data.length > MAX_IN_PATTERN_SIZE)) {
+		ret = -EINVAL;
+		RTW_INFO("ERROR: parameter length error, len=%d\n", wrqu->data.length);
+		goto _rtw_wowlan_set_pattern_exit;
+	} else {
+		/* set pattern */
+		if (copy_from_user(input, wrqu->data.pointer, wrqu->data.length)) {
+			ret -EFAULT;
+			goto _rtw_wowlan_set_pattern_exit;
+		}
+
+		if (strncmp(input, "pattern=", 8) == 0) {
+			if (rtw_wowlan_parser_pattern_cmd(input,
+							  wowcam_info.ptrn,
+							  &wowcam_info.ptrn_len,
+							  wowcam_info.mask)) {
+				if (_FAIL == rtw_wow_pattern_set(padapter,
+								 &wowcam_info,
+								 RTW_CUSTOMIZED_PATTERN))
+					ret = -EFAULT;
+			} else {
+				ret = -EINVAL;
+			}
+		} else if (strncmp(input, "clean", 5) == 0) {
+			rtw_wow_pattern_clean(padapter, RTW_CUSTOMIZED_PATTERN);
+		} else if (strncmp(input, "show", 4) == 0) {
+			#if 0 /* WOW_ToDo */
+			rtw_wow_pattern_cam_dump(padapter);
+			rtw_wow_pattern_sw_dump(padapter);
+			#endif
+		} else {
+			RTW_INFO("ERROR: incorrect parameter!\n");
+			ret = -EINVAL;
+		}
+	}
+_rtw_wowlan_set_pattern_exit:
+	return ret;
+}
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_AP_WOWLAN
+static int rtw_ap_wowlan_ctrl(struct net_device *dev,
+			      struct iw_request_info *info,
+			      union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct wowlan_ioctl_param poidparam;
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sta_info	*psta = NULL;
+	int ret = 0;
+	systime start_time = rtw_get_current_time();
+	poidparam.subcode = 0;
+
+	RTW_INFO("+rtw_ap_wowlan_ctrl: %s\n", extra);
+
+	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+		RTW_INFO("[%s] It is not AP mode!!\n", __func__);
+		goto _rtw_ap_wowlan_ctrl_exit_free;
+	}
+
+	if (_rtw_memcmp(extra, "enable", 6)) {
+
+		pwrctrlpriv->wowlan_ap_mode = _TRUE;
+
+		rtw_suspend_common(padapter);
+	} else if (_rtw_memcmp(extra, "disable", 7)) {
+#ifdef CONFIG_USB_HCI
+		RTW_ENABLE_FUNC(adapter_to_dvobj(padapter), DF_RX_BIT);
+		RTW_ENABLE_FUNC(adapter_to_dvobj(padapter), DF_TX_BIT);
+#endif
+		rtw_resume_common(padapter);
+	} else {
+		RTW_INFO("[%s] Invalid Parameter.\n", __func__);
+		goto _rtw_ap_wowlan_ctrl_exit_free;
+	}
+	/* mutex_lock(&ioctl_mutex); */
+_rtw_ap_wowlan_ctrl_exit_free:
+	RTW_INFO("-rtw_ap_wowlan_ctrl( subcode = %d)\n", poidparam.subcode);
+	RTW_PRINT("%s in %d ms\n", __func__,
+		  rtw_get_passing_time_ms(start_time));
+	return ret;
+}
+#endif /* CONFIG_AP_WOWLAN */
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+
+int rtw_vendor_ie_get_raw_data(struct net_device *dev, u32 vendor_ie_num,
+							   char *extra, u32 length)
+{
+	int j;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u32 vendor_ie_mask = 0;
+	char *pstring;
+
+	if (vendor_ie_num >= WLAN_MAX_VENDOR_IE_NUM) {
+		RTW_INFO("[%s] only support %d vendor ie\n", __func__ ,
+				 WLAN_MAX_VENDOR_IE_NUM);
+		return -EFAULT;
+	}
+
+	if (pmlmepriv->vendor_ielen[vendor_ie_num] == 0) {
+		RTW_INFO("[%s]  Fail, vendor_ie_num: %d is not set\n", __func__,
+				 vendor_ie_num);
+		return -EFAULT;
+	}
+
+	if (length < 2 * pmlmepriv->vendor_ielen[vendor_ie_num] + 5) {
+		RTW_INFO("[%s]  Fail, buffer size is too small\n", __func__);
+		return -EFAULT;
+	}
+
+	vendor_ie_mask = pmlmepriv->vendor_ie_mask[vendor_ie_num];
+	_rtw_memset(extra, 0, length);
+
+	pstring = extra;
+	pstring += sprintf(pstring, "%d,%x,", vendor_ie_num, vendor_ie_mask);
+
+	for (j = 0; j < pmlmepriv->vendor_ielen[vendor_ie_num]; j++)
+		pstring += sprintf(pstring, "%02x", pmlmepriv->vendor_ie[vendor_ie_num][j]);
+
+	length = pstring - extra;
+	return length;
+}
+
+int rtw_vendor_ie_get_data(struct net_device *dev, int vendor_ie_num, char *extra)
+{
+	int j;
+	char *pstring;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u32 vendor_ie_mask = 0;
+	__u16 length = 0;
+
+	vendor_ie_mask = pmlmepriv->vendor_ie_mask[vendor_ie_num];
+	pstring = extra;
+	pstring += sprintf(pstring , "\nVendor IE num %d , Mask:%x " , vendor_ie_num , vendor_ie_mask);
+
+	if (vendor_ie_mask & WIFI_BEACON_VENDOR_IE_BIT)
+		pstring += sprintf(pstring , "[Beacon]");
+	if (vendor_ie_mask & WIFI_PROBEREQ_VENDOR_IE_BIT)
+		pstring += sprintf(pstring , "[Probe Req]");
+	if (vendor_ie_mask & WIFI_PROBERESP_VENDOR_IE_BIT)
+		pstring += sprintf(pstring , "[Probe Resp]");
+	if (vendor_ie_mask & WIFI_ASSOCREQ_VENDOR_IE_BIT)
+		pstring += sprintf(pstring , "[Assoc Req]");
+	if (vendor_ie_mask & WIFI_ASSOCRESP_VENDOR_IE_BIT)
+		pstring += sprintf(pstring , "[Assoc Resp]");
+#ifdef CONFIG_P2P
+	if (vendor_ie_mask & WIFI_P2P_PROBEREQ_VENDOR_IE_BIT)
+		pstring += sprintf(pstring , "[P2P_Probe Req]");
+	if (vendor_ie_mask & WIFI_P2P_PROBERESP_VENDOR_IE_BIT)
+		pstring += sprintf(pstring , "[P2P_Probe Resp]");
+#endif
+
+	pstring += sprintf(pstring , "\nVendor IE:\n");
+	for (j = 0 ; j < pmlmepriv->vendor_ielen[vendor_ie_num]  ; j++)
+		pstring += sprintf(pstring , "%02x" , pmlmepriv->vendor_ie[vendor_ie_num][j]);
+
+	length = pstring - extra;
+	return length;
+
+}
+
+int rtw_vendor_ie_get(struct net_device *dev, struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0, vendor_ie_num = 0, cmdlen;
+	struct iw_point *p;
+	u8 *ptmp;
+
+	p = &wrqu->data;
+	cmdlen = p->length;
+	if (0 == cmdlen)
+		return -EINVAL;
+
+	ptmp = (u8 *)rtw_malloc(cmdlen);
+	if (NULL == ptmp)
+		return -ENOMEM;
+
+	if (copy_from_user(ptmp, p->pointer, cmdlen)) {
+		ret = -EFAULT;
+		goto exit;
+	}
+	ret = sscanf(ptmp , "%d", &vendor_ie_num);
+	if (vendor_ie_num > WLAN_MAX_VENDOR_IE_NUM - 1) {
+		ret = -EFAULT;
+		goto exit;
+	}
+
+	wrqu->data.length = rtw_vendor_ie_get_data(dev, vendor_ie_num, extra);
+
+exit:
+	rtw_mfree(ptmp, cmdlen);
+
+	return 0;
+}
+
+int rtw_vendor_ie_set(struct net_device *dev, struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0, i , len = 0 , totoal_ie_len = 0 , total_ie_len_byte = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	u32 vendor_ie_mask = 0;
+	u32 vendor_ie_num = 0;
+	u32 vendor_ie_mask_max = BIT(WLAN_MAX_VENDOR_IE_MASK_MAX) - 1;
+	u32 id, elen;
+
+	ret = sscanf(extra, "%d,%x,%*s", &vendor_ie_num , &vendor_ie_mask);
+	if (strrchr(extra , ','))
+		extra = strrchr(extra , ',') + 1;
+	else
+		return -EINVAL;
+	totoal_ie_len = strlen(extra);
+	RTW_INFO("[%s] vendor_ie_num = %d , vendor_ie_mask = 0x%x , vendor_ie = %s , len = %d\n", __func__ , vendor_ie_num , vendor_ie_mask , extra  , totoal_ie_len);
+
+	if (vendor_ie_num  > WLAN_MAX_VENDOR_IE_NUM - 1) {
+		RTW_INFO("[%s] Fail, only support %d vendor ie\n", __func__ , WLAN_MAX_VENDOR_IE_NUM);
+		return -EFAULT;
+	}
+
+	if (totoal_ie_len > WLAN_MAX_VENDOR_IE_LEN) {
+		RTW_INFO("[%s] Fail , not support ie length extend %d\n", __func__ , WLAN_MAX_VENDOR_IE_LEN);
+		return -EFAULT;
+	}
+
+	if (vendor_ie_mask > vendor_ie_mask_max) {
+		RTW_INFO("[%s] Fail, not support vendor_ie_mask more than 0x%x\n", __func__ , vendor_ie_mask_max);
+		return -EFAULT;
+	}
+
+	if (vendor_ie_mask == 0) {
+		RTW_INFO("[%s] Clear vendor_ie_num %d group\n", __func__ , vendor_ie_num);
+		goto _clear_path;
+	}
+
+	if (totoal_ie_len % 2 != 0) {
+		RTW_INFO("[%s]  Fail , IE length = %zu is odd\n" , __func__ , strlen(extra));
+		return -EFAULT;
+	}
+
+	if (totoal_ie_len > 0) {
+		for (i = 0  ; i < strlen(extra) ; i += 2) {
+			pmlmepriv->vendor_ie[vendor_ie_num][len] = key_2char2num(extra[i] , extra[i + 1]);
+			if (len == 0) {
+				id = pmlmepriv->vendor_ie[vendor_ie_num][len];
+				if (id != WLAN_EID_VENDOR_SPECIFIC) {
+					RTW_INFO("[%s] Fail , VENDOR SPECIFIC IE ID \"%x\" was not correct\n", __func__ , id);
+					goto _clear_path;
+				}
+			} else if (len == 1) {
+				total_ie_len_byte = (totoal_ie_len / 2) - 2;
+				elen = pmlmepriv->vendor_ie[vendor_ie_num][len];
+				if (elen != total_ie_len_byte) {
+					RTW_INFO("[%s] Fail , Input IE length = \"%d\"(hex:%x) bytes , not match input total IE context length \"%d\" bytes\n", __func__ , elen , elen ,
+						 total_ie_len_byte);
+					goto _clear_path;
+				}
+			}
+			len++;
+		}
+		pmlmepriv->vendor_ielen[vendor_ie_num] = len;
+	} else
+		pmlmepriv->vendor_ielen[vendor_ie_num] = 0;
+
+
+
+	if (vendor_ie_mask & WIFI_BEACON_VENDOR_IE_BIT)
+		RTW_INFO("[%s] Beacon append vendor ie\n", __func__);
+	if (vendor_ie_mask & WIFI_PROBEREQ_VENDOR_IE_BIT)
+		RTW_INFO("[%s] Probe Req append vendor ie\n", __func__);
+	if (vendor_ie_mask & WIFI_PROBERESP_VENDOR_IE_BIT)
+		RTW_INFO("[%s] Probe Resp append vendor ie\n", __func__);
+	if (vendor_ie_mask & WIFI_ASSOCREQ_VENDOR_IE_BIT)
+		RTW_INFO("[%s] Assoc Req append vendor ie\n", __func__);
+	if (vendor_ie_mask & WIFI_ASSOCRESP_VENDOR_IE_BIT)
+		RTW_INFO("[%s] Assoc Resp append vendor ie\n", __func__);
+#ifdef CONFIG_P2P
+	if (vendor_ie_mask & WIFI_P2P_PROBEREQ_VENDOR_IE_BIT)
+		RTW_INFO("[%s] P2P Probe Req append vendor ie\n", __func__);
+	if (vendor_ie_mask & WIFI_P2P_PROBERESP_VENDOR_IE_BIT)
+		RTW_INFO("[%s] P2P Probe Resp append vendor ie\n", __func__);
+#endif
+
+	pmlmepriv->vendor_ie_mask[vendor_ie_num] = vendor_ie_mask;
+
+	return ret;
+
+_clear_path:
+	_rtw_memset(pmlmepriv->vendor_ie[vendor_ie_num] , 0 , sizeof(u32) * WLAN_MAX_VENDOR_IE_LEN);
+	pmlmepriv->vendor_ielen[vendor_ie_num] = 0;
+	pmlmepriv->vendor_ie_mask[vendor_ie_num] = 0;
+	return -EFAULT;
+}
+#endif
+
+#if defined(RTW_PHL_TX) || defined(RTW_PHL_RX) || defined(CONFIG_PHL_TEST_SUITE)
+int rtw_phl_test_set(struct net_device *dev,
+	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u32 mode = 0;
+	u32 bytes = 0;
+	int ret = 0;
+
+#if defined(RTW_PHL_DBG_CMD)
+	core_cmd_phl_handler(padapter, extra);
+#endif
+
+exit:
+	return 0;
+}
+#endif
+
+
+#ifdef CONFIG_RTW_CUSTOMER_STR
+static int rtw_mp_customer_str(
+	struct net_device *dev,
+	struct iw_request_info *info,
+	union iwreq_data *wrqu, char *extra)
+{
+	_adapter *adapter = rtw_netdev_priv(dev);
+	u32 len;
+	u8 *pbuf = NULL, *pch;
+	char *ptmp;
+	u8 param[RTW_CUSTOMER_STR_LEN];
+	u8 count = 0;
+	u8 tmp;
+	u8 i;
+	u32 pos;
+	u8 ret;
+	u8 read = 0;
+
+	if (adapter->registrypriv.mp_mode != 1
+		|| !adapter->registrypriv.mp_customer_str)
+		return -EFAULT;
+
+	len = wrqu->data.length + 1;
+
+	pbuf = (u8 *)rtw_zmalloc(len);
+	if (pbuf == NULL) {
+		RTW_WARN("%s: no memory!\n", __func__);
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(pbuf, wrqu->data.pointer, wrqu->data.length)) {
+		rtw_mfree(pbuf, len);
+		RTW_WARN("%s: copy from user fail!\n", __func__);
+		return -EFAULT;
+	}
+	RTW_INFO("%s: string=\"%s\"\n", __func__, pbuf);
+
+	ptmp = (char *)pbuf;
+	pch = strsep(&ptmp, ",");
+	if ((pch == NULL) || (strlen(pch) == 0)) {
+		rtw_mfree(pbuf, len);
+		RTW_INFO("%s: parameter error(no cmd)!\n", __func__);
+		return -EFAULT;
+	}
+
+	_rtw_memset(param, 0xFF, RTW_CUSTOMER_STR_LEN);
+
+	if (strcmp(pch, "read") == 0) {
+		read = 1;
+		ret = rtw_hal_customer_str_read(adapter, param);
+
+	} else if (strcmp(pch, "write") == 0) {
+		do {
+			pch = strsep(&ptmp, ":");
+			if ((pch == NULL) || (strlen(pch) == 0))
+				break;
+			if (strlen(pch) != 2
+				|| IsHexDigit(*pch) == _FALSE
+				|| IsHexDigit(*(pch + 1)) == _FALSE
+				|| sscanf(pch, "%hhx", &tmp) != 1
+			) {
+				RTW_WARN("%s: invalid 8-bit hex!\n", __func__);
+				rtw_mfree(pbuf, len);
+				return -EFAULT;
+			}
+
+			param[count++] = tmp;
+
+		} while (count < RTW_CUSTOMER_STR_LEN);
+
+		if (count == 0) {
+			rtw_mfree(pbuf, len);
+			RTW_WARN("%s: no input!\n", __func__);
+			return -EFAULT;
+		}
+		ret = rtw_hal_customer_str_write(adapter, param);
+	} else {
+		rtw_mfree(pbuf, len);
+		RTW_INFO("%s: parameter error(unknown cmd)!\n", __func__);
+		return -EFAULT;
+	}
+
+	pos = sprintf(extra, "%s: ", read ? "read" : "write");
+	if (read == 0 || ret == _SUCCESS) {
+		for (i = 0; i < RTW_CUSTOMER_STR_LEN; i++)
+			pos += sprintf(extra + pos, "%02x:", param[i]);
+		extra[pos] = 0;
+		pos--;
+	}
+	pos += sprintf(extra + pos, " %s", ret == _SUCCESS ? "OK" : "FAIL");
+
+	wrqu->data.length = strlen(extra) + 1;
+
+	rtw_mfree(pbuf, len);
+	return 0;
+}
+#endif /* CONFIG_RTW_CUSTOMER_STR */
+
+
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+#define DBG_MP_SDIO_INDIRECT_ACCESS 1
+static int rtw_mp_sd_iread(struct net_device *dev
+			   , struct iw_request_info *info
+			   , struct iw_point *wrqu
+			   , char *extra)
+{
+	char input[16];
+	u8 width;
+	unsigned long addr;
+	u32 ret = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	if (wrqu->length > 16) {
+		RTW_INFO(FUNC_ADPT_FMT" wrqu->length:%d\n", FUNC_ADPT_ARG(padapter), wrqu->length);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length)) {
+		RTW_INFO(FUNC_ADPT_FMT" copy_from_user fail\n", FUNC_ADPT_ARG(padapter));
+		ret = -EFAULT;
+		goto exit;
+	}
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	if (sscanf(input, "%hhu,%lx", &width, &addr) != 2) {
+		RTW_INFO(FUNC_ADPT_FMT" sscanf fail\n", FUNC_ADPT_ARG(padapter));
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (addr > 0x3FFF) {
+		RTW_INFO(FUNC_ADPT_FMT" addr:0x%lx\n", FUNC_ADPT_ARG(padapter), addr);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (DBG_MP_SDIO_INDIRECT_ACCESS)
+		RTW_INFO(FUNC_ADPT_FMT" width:%u, addr:0x%lx\n", FUNC_ADPT_ARG(padapter), width, addr);
+
+	switch (width) {
+	case 1:
+		sprintf(extra, "0x%02x", rtw_sd_iread8(padapter, addr));
+		wrqu->length = strlen(extra);
+		break;
+	case 2:
+		sprintf(extra, "0x%04x", rtw_sd_iread16(padapter, addr));
+		wrqu->length = strlen(extra);
+		break;
+	case 4:
+		sprintf(extra, "0x%08x", rtw_sd_iread32(padapter, addr));
+		wrqu->length = strlen(extra);
+		break;
+	default:
+		wrqu->length = 0;
+		ret = -EINVAL;
+		break;
+	}
+
+exit:
+	return ret;
+}
+
+static int rtw_mp_sd_iwrite(struct net_device *dev
+			    , struct iw_request_info *info
+			    , struct iw_point *wrqu
+			    , char *extra)
+{
+	char width;
+	unsigned long addr, data;
+	int ret = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	char input[32];
+
+	if (wrqu->length > 32) {
+		RTW_INFO(FUNC_ADPT_FMT" wrqu->length:%d\n", FUNC_ADPT_ARG(padapter), wrqu->length);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length)) {
+		RTW_INFO(FUNC_ADPT_FMT" copy_from_user fail\n", FUNC_ADPT_ARG(padapter));
+		ret = -EFAULT;
+		goto exit;
+	}
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	if (sscanf(input, "%hhu,%lx,%lx", &width, &addr, &data) != 3) {
+		RTW_INFO(FUNC_ADPT_FMT" sscanf fail\n", FUNC_ADPT_ARG(padapter));
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (addr > 0x3FFF) {
+		RTW_INFO(FUNC_ADPT_FMT" addr:0x%lx\n", FUNC_ADPT_ARG(padapter), addr);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	if (DBG_MP_SDIO_INDIRECT_ACCESS)
+		RTW_INFO(FUNC_ADPT_FMT" width:%u, addr:0x%lx, data:0x%lx\n", FUNC_ADPT_ARG(padapter), width, addr, data);
+
+	switch (width) {
+	case 1:
+		if (data > 0xFF) {
+			ret = -EINVAL;
+			break;
+		}
+		rtw_sd_iwrite8(padapter, addr, data);
+		break;
+	case 2:
+		if (data > 0xFFFF) {
+			ret = -EINVAL;
+			break;
+		}
+		rtw_sd_iwrite16(padapter, addr, data);
+		break;
+	case 4:
+		rtw_sd_iwrite32(padapter, addr, data);
+		break;
+	default:
+		wrqu->length = 0;
+		ret = -EINVAL;
+		break;
+	}
+
+exit:
+	return ret;
+}
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+
+#if defined(CONFIG_FPGA_INCLUDED)
+static int rtw_fpga_set(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	if (_rtw_memcmp(extra, "start", 5))
+		rtw_fpga_start(padapter);
+
+	if (_rtw_memcmp(extra, "stop", 4))
+		rtw_fpga_stop(padapter);
+
+	return ret;
+}
+#endif
+#ifdef CONFIG_WOW_APF
+enum access {
+	SET,
+	GET
+};
+/* access 0 : set/clear apf prog, 1*/
+static int rtw_apf(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra, u8 access)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	int ret = 0, len = 0, i;
+	u16 apf_prog_len = 0;
+	u8 *apf_prog = NULL;
+
+#if 0
+	if (extra) {
+		len = strlen(extra);
+		RTW_INFO("len: %d\n", len);
+		RTW_INFO("extra: %s\n", extra);
+		RTW_INFO_DUMP(NULL, extra, len);
+	}
+#endif
+
+	if (access == SET) {
+		if (extra) {
+			len = strlen(extra);
+			if (_rtw_memcmp(extra, "clear", 5)) {
+				if (rtw_wow_clear_apf(padapter) != _SUCCESS) {
+					RTW_ERR("%s() rtw_wow_clear_apf fail\n", __func__);
+					ret = -EFAULT;
+				}
+			} else { /* set apf prog */
+				if ((len % 2) != 0 || len > MAX_APF_PROG_SIZE * 2) {
+					RTW_ERR("%s() Invalid APF PROG, len(%d)\n", __func__, len);
+					RTW_ERR("Input APF PROG:%s\n", extra);
+					ret = -EINVAL;
+				} else {
+					apf_prog = rtw_zvmalloc(len / 2);
+					apf_prog_len = 0;
+					if (!apf_prog) {
+						RTW_ERR("%s() apf_prog is NULL\n", __func__);
+						ret = -ENOMEM;
+					} else {
+						for (i = 0; i < len; i += 2) {
+							apf_prog[apf_prog_len] = key_2char2num(extra[i], extra[i + 1]);
+							apf_prog_len++;
+						}
+#ifdef CONFIG_WOW_APF_DBG
+						RTW_INFO("%s() set apf prog(%d|%d)\n", __func__, len, apf_prog_len);
+						RTW_INFO("%s\n", extra);
+						RTW_INFO_DUMP("APF PROG HEX", apf_prog, apf_prog_len);
+#endif
+						if (rtw_wow_set_apf(padapter, apf_prog, apf_prog_len) != _SUCCESS) {
+							RTW_ERR("%s() rtw_wow_set_apf fail\n", __func__);
+							ret = -EFAULT;
+						}
+					}
+
+				}
+			}
+		}
+	} else { /* get apf prog */
+		apf_prog= rtw_zvmalloc(MAX_APF_PROG_SIZE);
+		if (!apf_prog) {
+			RTW_ERR("%s() apf_prog is NULL\n", __func__);
+			ret = -ENOMEM;
+		} else {
+			if (rtw_wow_get_apf(padapter, apf_prog, &apf_prog_len) != _SUCCESS) {
+					RTW_ERR("%s() rtw_wow_get_apf fail\n", __func__);
+					ret = -EFAULT;
+			} else {
+				if (!extra)
+					RTW_INFO("extra is NULL\n");
+
+				RTW_INFO_DUMP("APF PROG HEX", apf_prog, apf_prog_len);
+			}
+		}
+	}
+
+	if (ret != 0) {
+		RTW_ERR("%s() Incorrect ioctl cmd:\n", __func__);
+		RTW_ERR("%s\n", extra);
+		RTW_INFO("apf cmd:\n");
+		RTW_INFO("Set apf prog		: iwpriv wlan0 apf_set $APF_PROG\n");
+		RTW_INFO("Get apf prog		: iwpriv wlan0 apf_get\n");
+		RTW_INFO("Clear apf prog	: iwpriv wlan0 apf_set clear\n");
+	}
+
+	if (apf_prog)
+		rtw_vmfree(apf_prog, apf_prog_len);
+	return ret;
+}
+/* access 0 : set/clear apf prog, 1*/
+static int rtw_apf_default(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra, u8 access)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	static u16 apf_dft_prog_len = 0;
+	static u8 apf_dft_prog[2048];
+	int i, len, ret = 0;
+
+	if (access == SET) {
+		RTW_INFO("Set apf dft prog, exisit len(%u)\n", apf_dft_prog_len);
+		if (extra) {
+			if (_rtw_memcmp(extra, "clear", 5)) {
+				RTW_INFO("Clear apf dft prog\n");
+				apf_dft_prog_len = 0;
+				_rtw_memset(apf_dft_prog, 0, 2048);
+			} else if (_rtw_memcmp(extra, "set", 3)) {
+				RTW_INFO("Set apf dft prog\n");
+				rtw_wow_set_apf(padapter, apf_dft_prog, apf_dft_prog_len);
+			} else {
+				RTW_INFO("Fill apf dft prog\n");
+				len = strlen(extra);
+				if ((len % 2) != 0 || len > MAX_APF_PROG_SIZE * 2) {
+					RTW_ERR("%s() Invalid APF PROG, len(%d)\n", __func__, len);
+					RTW_ERR("Input APF PROG:%s\n", extra);
+					ret = -EINVAL;
+				} else {
+					for (i = 0; i < len; i += 2) {
+							apf_dft_prog[apf_dft_prog_len] = key_2char2num(extra[i], extra[i + 1]);
+							apf_dft_prog_len++;
+					}
+				}
+			}
+		}
+	} else {
+		RTW_INFO("Get apf dft prog len(%u)\n", apf_dft_prog_len);
+		RTW_INFO_DUMP("APF DFT PROG HEX:\n", apf_dft_prog, apf_dft_prog_len);
+	}
+
+	return 0;
+}
+
+#endif
+static int rtw_priv_set(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wdata, char *extra)
+{
+	struct iw_point *wrqu = (struct iw_point *)wdata;
+	u32 subcmd = wrqu->flags;
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	if (padapter == NULL)
+		return -ENETDOWN;
+
+	if (padapter->netif_up == _FALSE) {
+		RTW_INFO(" %s fail =>(padapter->netif_up == _FALSE )\n", __FUNCTION__);
+		return -ENETDOWN;
+	}
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		RTW_INFO("%s fail =>(bSurpriseRemoved == _TRUE) || ( bDriverStopped == _TRUE)\n", __func__);
+		return -ENETDOWN;
+	}
+
+	if (extra == NULL) {
+		wrqu->length = 0;
+		return -EIO;
+	}
+
+	if (subcmd < MP_NULL) {
+#ifdef CONFIG_MP_INCLUDED
+		rtw_priv_mp_set(dev, info, wdata, extra);
+#endif
+		return 0;
+	}
+
+	switch (subcmd) {
+#ifdef CONFIG_WOWLAN
+	case MP_WOW_ENABLE:
+		RTW_INFO("set case MP_WOW_ENABLE: %s\n", extra);
+
+		rtw_wowlan_ctrl(dev, info, wdata, extra);
+		break;
+	case MP_WOW_SET_PATTERN:
+		RTW_INFO("set case MP_WOW_SET_PATTERN: %s\n", extra);
+		rtw_wowlan_set_pattern(dev, info, wdata, extra);
+		break;
+#endif
+#ifdef CONFIG_AP_WOWLAN
+	case MP_AP_WOW_ENABLE:
+		RTW_INFO("set case MP_AP_WOW_ENABLE: %s\n", extra);
+		rtw_ap_wowlan_ctrl(dev, info, wdata, extra);
+		break;
+#endif
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	case VENDOR_IE_SET:
+		RTW_INFO("set case VENDOR_IE_SET\n");
+		rtw_vendor_ie_set(dev , info , wdata , extra);
+		break;
+#endif
+#if defined(RTW_PHL_TX) || defined(RTW_PHL_RX) || defined(CONFIG_PHL_TEST_SUITE)
+	case PHL_TEST_SET:
+		RTW_INFO("set case PHL_TEST_SET\n");
+		rtw_phl_test_set(dev , info , wdata , extra);
+		break;
+#endif
+#ifdef CONFIG_FPGA_INCLUDED
+	case FPGA_SET:
+		RTW_INFO("set case FPGA_SET\n");
+		rtw_fpga_set(dev , info , wdata , extra);
+		break;
+#endif
+#ifdef CONFIG_WOW_APF
+	case APF_SET:
+		RTW_INFO("set apf prog\n");
+		rtw_apf(dev, info, wdata, extra, SET);
+		break;
+	case APF_DEFAULT_SET:
+		RTW_INFO("<set dft apf prog>\n");
+		rtw_apf_default(dev, info, wdata, extra, SET);
+		break;
+#endif
+	default:
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int rtw_priv_get(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wdata, char *extra)
+{
+	struct iw_point *wrqu = (struct iw_point *)wdata;
+	u32 subcmd = wrqu->flags;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	int status = 0 ;
+#ifndef CONFIG_MP_INCLUDED
+
+	if (padapter == NULL)
+		return -ENETDOWN;
+
+	if (padapter->netif_up == _FALSE) {
+		RTW_INFO(" %s fail =>(padapter->netif_up == _FALSE )\n", __FUNCTION__);
+		return -ENETDOWN;
+	}
+
+	if (RTW_CANNOT_RUN(adapter_to_dvobj(padapter))) {
+		RTW_INFO("%s fail =>(padapter->bSurpriseRemoved == _TRUE) || ( padapter->bDriverStopped == _TRUE)\n", __func__);
+		return -ENETDOWN;
+	}
+
+	if (extra == NULL) {
+		wrqu->length = 0;
+		return -EIO;
+	}
+#endif
+	if (subcmd < MP_NULL) {
+#ifdef CONFIG_MP_INCLUDED
+		status = rtw_priv_mp_get(dev, info, wdata, extra);
+		rtw_msleep_os(10); /* delay 5ms for sending pkt before exit adb shell operation */
+#endif
+
+	} else {
+		switch (subcmd) {
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+			case MP_SD_IREAD:
+				status = rtw_mp_sd_iread(dev, info, wrqu, extra);
+				break;
+			case MP_SD_IWRITE:
+				status = rtw_mp_sd_iwrite(dev, info, wrqu, extra);
+				break;
+#endif
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+			case VENDOR_IE_GET:
+				RTW_INFO("get case VENDOR_IE_GET\n");
+				status = rtw_vendor_ie_get(dev , info , wdata , extra);
+				break;
+#endif
+#ifdef CONFIG_WOW_APF
+			case APF_GET:
+				RTW_INFO("get apf prog\n");
+				rtw_apf(dev, info, wdata, extra, GET);
+				break;
+			case APF_DEFAULT_GET:
+				RTW_INFO("get apf prog\n");
+				rtw_apf_default(dev, info, wdata, extra, GET);
+				break;
+#endif
+
+			default:
+				return -EIO;
+		}
+	}
+
+	return status;
+}
+
+
+#ifdef CONFIG_TDLS
+static int rtw_wx_tdls_wfd_enable(struct net_device *dev,
+				  struct iw_request_info *info,
+				  union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_WFD
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	if (extra[0] == '0')
+		rtw_tdls_wfd_enable(padapter, 0);
+	else
+		rtw_tdls_wfd_enable(padapter, 1);
+
+#endif /* CONFIG_WFD */
+
+	return ret;
+}
+
+static int rtw_tdls_weaksec(struct net_device *dev,
+			    struct iw_request_info *info,
+			    union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	u8 i, j;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	if (extra[0] == '0')
+		padapter->wdinfo.wfd_tdls_weaksec = 0;
+	else
+		padapter->wdinfo.wfd_tdls_weaksec = 1;
+
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+
+static int rtw_tdls_enable(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	if (extra[0] == '0')
+		rtw_disable_tdls_func(padapter, _TRUE);
+	else if (extra[0] == '1')
+		rtw_enable_tdls_func(padapter);
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_setup(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+#ifdef CONFIG_TDLS
+	u8 i, j;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_txmgmt txmgmt;
+#ifdef CONFIG_WFD
+	struct wifidirect_info *pwdinfo = &(padapter->wdinfo);
+#endif /* CONFIG_WFD */
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	if (wrqu->data.length - 1 != 17) {
+		RTW_INFO("[%s] length:%d != 17\n", __FUNCTION__, (wrqu->data.length - 1));
+		return ret;
+	}
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	for (i = 0, j = 0 ; i < ETH_ALEN; i++, j += 3)
+		txmgmt.peer[i] = key_2char2num(*(extra + j), *(extra + j + 1));
+
+#ifdef CONFIG_WFD
+	if (_AES_ != padapter->securitypriv.dot11PrivacyAlgrthm) {
+		/* Weak Security situation with AP. */
+		if (0 == pwdinfo->wfd_tdls_weaksec)	{
+			/* Can't send the tdls setup request out!! */
+			RTW_INFO("[%s] Current link is not AES, "
+				"SKIP sending the tdls setup request!!\n", __FUNCTION__);
+		} else
+			issue_tdls_setup_req(padapter, &txmgmt, _TRUE);
+	} else
+#endif /* CONFIG_WFD */
+	{
+		issue_tdls_setup_req(padapter, &txmgmt, _TRUE);
+	}
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_teardown(struct net_device *dev,
+			     struct iw_request_info *info,
+			     union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	u8 i, j;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_info *ptdls_sta = NULL;
+	struct tdls_txmgmt txmgmt;
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	if (wrqu->data.length - 1 != 17 && wrqu->data.length - 1 != 19) {
+		RTW_INFO("[%s] length:%d != 17 or 19\n",
+			 __FUNCTION__, (wrqu->data.length - 1));
+		return ret;
+	}
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	for (i = 0, j = 0; i < ETH_ALEN; i++, j += 3)
+		txmgmt.peer[i] = key_2char2num(*(extra + j), *(extra + j + 1));
+
+	ptdls_sta = rtw_get_stainfo(&(padapter->stapriv), txmgmt.peer);
+
+	if (ptdls_sta != NULL) {
+		txmgmt.status_code = _RSON_TDLS_TEAR_UN_RSN_;
+		if (wrqu->data.length - 1 == 19)
+			issue_tdls_teardown(padapter, &txmgmt, _FALSE);
+		else
+			issue_tdls_teardown(padapter, &txmgmt, _TRUE);
+	} else
+		RTW_INFO("TDLS peer not found\n");
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_discovery(struct net_device *dev,
+			      struct iw_request_info *info,
+			      union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_txmgmt	txmgmt;
+	int i = 0, j = 0;
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+	for (i = 0, j = 0 ; i < ETH_ALEN; i++, j += 3)
+		txmgmt.peer[i] = key_2char2num(*(extra + j), *(extra + j + 1));
+
+	issue_tdls_dis_req(padapter, &txmgmt);
+
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_ch_switch(struct net_device *dev,
+			      struct iw_request_info *info,
+			      union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+	u8 i, j;
+	struct sta_info *ptdls_sta = NULL;
+	u8 central_chnl;
+	struct rtw_chan_def chdef = {0};
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	if (rtw_tdls_is_chsw_allowed(padapter) == _FALSE) {
+		RTW_INFO("TDLS channel switch is not allowed\n");
+		return ret;
+	}
+
+	for (i = 0, j = 0 ; i < ETH_ALEN; i++, j += 3)
+		pchsw_info->addr[i] = key_2char2num(*(extra + j), *(extra + j + 1));
+
+	ptdls_sta = rtw_get_stainfo(&padapter->stapriv, pchsw_info->addr);
+	if (ptdls_sta == NULL)
+		return ret;
+
+	pchsw_info->ch_sw_state |= (TDLS_CH_SW_INITIATOR_STATE | TDLS_WAIT_CH_SW_LAUNCH_STATE);
+
+	if (ptdls_sta != NULL) {
+		if (pchsw_info->off_ch_num == 0)
+			pchsw_info->off_ch_num = 11;
+	} else
+		RTW_INFO("TDLS peer not found\n");
+
+	chdef.band = rtw_get_band_type(pchsw_info->off_ch_num);
+	chdef.chan = pchsw_info->off_ch_num;
+	chdef.bw = (pchsw_info->ch_offset) ? CHANNEL_WIDTH_40 : CHANNEL_WIDTH_20;
+	chdef.offset = pchsw_info->ch_offset;
+	central_chnl = rtw_phl_get_center_ch(&chdef);
+
+	rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_PREPARE, 0);
+
+	/* issue_tdls_ch_switch_req(padapter, ptdls_sta); */
+	/* RTW_INFO("issue tdls ch switch req\n"); */
+
+#endif /* CONFIG_TDLS_CH_SW */
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_ch_switch_off(struct net_device *dev,
+				  struct iw_request_info *info,
+				  union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_ch_switch *pchsw_info = &padapter->tdlsinfo.chsw_info;
+	u8 i, j, mac_addr[ETH_ALEN];
+	struct sta_info *ptdls_sta = NULL;
+	struct tdls_txmgmt txmgmt;
+
+	_rtw_memset(&txmgmt, 0x00, sizeof(struct tdls_txmgmt));
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	if (rtw_tdls_is_chsw_allowed(padapter) == _FALSE) {
+		RTW_INFO("TDLS channel switch is not allowed\n");
+		return ret;
+	}
+
+	if (wrqu->data.length >= 17) {
+		for (i = 0, j = 0 ; i < ETH_ALEN; i++, j += 3)
+			mac_addr[i] = key_2char2num(*(extra + j), *(extra + j + 1));
+		ptdls_sta = rtw_get_stainfo(&padapter->stapriv, mac_addr);
+	}
+
+	if (ptdls_sta == NULL)
+		return ret;
+
+	rtw_tdls_cmd(padapter, ptdls_sta->phl_sta->mac_addr, TDLS_CH_SW_END_TO_BASE_CHNL, 0);
+
+	pchsw_info->ch_sw_state &= ~(TDLS_CH_SWITCH_PREPARE_STATE |
+				   TDLS_CH_SW_INITIATOR_STATE |
+				   TDLS_CH_SWITCH_ON_STATE |
+				   TDLS_PEER_AT_OFF_STATE |
+				   TDLS_WAIT_CH_SW_LAUNCH_STATE);
+	_rtw_memset(pchsw_info->addr, 0x00, ETH_ALEN);
+
+	ptdls_sta->ch_switch_time = 0;
+	ptdls_sta->ch_switch_timeout = 0;
+	_cancel_timer_ex(&ptdls_sta->ch_sw_timer);
+	_cancel_timer_ex(&ptdls_sta->delay_timer);
+	_cancel_timer_ex(&ptdls_sta->stay_on_base_chnl_timer);
+	_cancel_timer_ex(&ptdls_sta->ch_sw_monitor_timer);
+
+#endif /* CONFIG_TDLS_CH_SW */
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_dump_ch(struct net_device *dev,
+			    struct iw_request_info *info,
+			    union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	RTW_INFO("[%s] dump_stack:%s\n", __FUNCTION__, extra);
+
+	extra[wrqu->data.length] = 0x00;
+	ptdlsinfo->chsw_info.dump_stack = rtw_atoi(extra);
+
+	return ret;
+
+#endif
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_off_ch_num(struct net_device *dev,
+			       struct iw_request_info *info,
+			       union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	RTW_INFO("[%s] off_ch_num:%s\n", __FUNCTION__, extra);
+
+	extra[wrqu->data.length] = 0x00;
+	ptdlsinfo->chsw_info.off_ch_num = rtw_atoi(extra);
+
+	return ret;
+
+#endif
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_ch_offset(struct net_device *dev,
+			      struct iw_request_info *info,
+			      union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_TDLS_CH_SW
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	RTW_INFO("[%s] ch_offset:%s\n", __FUNCTION__, extra);
+
+	extra[wrqu->data.length] = 0x00;
+	switch (rtw_atoi(extra)) {
+	case IEEE80211_SCA:
+		ptdlsinfo->chsw_info.ch_offset = CHAN_OFFSET_UPPER;
+		break;
+
+	case IEEE80211_SCB:
+		ptdlsinfo->chsw_info.ch_offset = CHAN_OFFSET_LOWER;
+		break;
+
+	default:
+		ptdlsinfo->chsw_info.ch_offset = CHAN_OFFSET_NO_EXT;
+		break;
+	}
+
+	return ret;
+
+#endif
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_pson(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 i, j, mac_addr[ETH_ALEN];
+	struct sta_info *ptdls_sta = NULL;
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	for (i = 0, j = 0; i < ETH_ALEN; i++, j += 3)
+		mac_addr[i] = key_2char2num(*(extra + j), *(extra + j + 1));
+
+	ptdls_sta = rtw_get_stainfo(&padapter->stapriv, mac_addr);
+
+	issue_nulldata_to_TDLS_peer_STA(padapter, ptdls_sta, 1, 3, PS_ANNC_DRV_RETRY_INT_MS);
+
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_psoff(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 i, j, mac_addr[ETH_ALEN];
+	struct sta_info *ptdls_sta = NULL;
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	for (i = 0, j = 0; i < ETH_ALEN; i++, j += 3)
+		mac_addr[i] = key_2char2num(*(extra + j), *(extra + j + 1));
+
+	ptdls_sta = rtw_get_stainfo(&padapter->stapriv, mac_addr);
+
+	if (ptdls_sta)
+		issue_nulldata_to_TDLS_peer_STA(padapter, ptdls_sta, 0, 3, PS_ANNC_DRV_RETRY_INT_MS);
+
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_setip(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_WFD
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct wifi_display_info *pwfd_info = ptdlsinfo->wfd_info;
+	u8 i = 0, j = 0, k = 0, tag = 0;
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__, extra, wrqu->data.length - 1);
+
+	while (i < 4) {
+		for (j = 0; j < 4; j++) {
+			if (*(extra + j + tag) == '.' || *(extra + j + tag) == '\0') {
+				if (j == 1)
+					pwfd_info->ip_address[i] = convert_ip_addr('0', '0', *(extra + (j - 1) + tag));
+				if (j == 2)
+					pwfd_info->ip_address[i] = convert_ip_addr('0', *(extra + (j - 2) + tag), *(extra + (j - 1) + tag));
+				if (j == 3)
+					pwfd_info->ip_address[i] = convert_ip_addr(*(extra + (j - 3) + tag), *(extra + (j - 2) + tag), *(extra + (j - 1) + tag));
+
+				tag += j + 1;
+				break;
+			}
+		}
+		i++;
+	}
+
+	RTW_INFO("[%s] Set IP = %u.%u.%u.%u\n", __FUNCTION__,
+		 ptdlsinfo->wfd_info->ip_address[0],
+		 ptdlsinfo->wfd_info->ip_address[1],
+		 ptdlsinfo->wfd_info->ip_address[2],
+		 ptdlsinfo->wfd_info->ip_address[3]);
+
+#endif /* CONFIG_WFD */
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_getip(struct net_device *dev,
+			  struct iw_request_info *info,
+			  union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_WFD
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct wifi_display_info *pwfd_info = ptdlsinfo->wfd_info;
+
+	RTW_INFO("[%s]\n", __FUNCTION__);
+
+	sprintf(extra, "\n\n%u.%u.%u.%u\n",
+		pwfd_info->peer_ip_address[0], pwfd_info->peer_ip_address[1],
+		pwfd_info->peer_ip_address[2], pwfd_info->peer_ip_address[3]);
+
+	RTW_INFO("[%s] IP=%u.%u.%u.%u\n", __FUNCTION__,
+		 pwfd_info->peer_ip_address[0], pwfd_info->peer_ip_address[1],
+		 pwfd_info->peer_ip_address[2], pwfd_info->peer_ip_address[3]);
+
+	wrqu->data.length = strlen(extra);
+
+#endif /* CONFIG_WFD */
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+static int rtw_tdls_getport(struct net_device *dev,
+			    struct iw_request_info *info,
+			    union iwreq_data *wrqu, char *extra)
+{
+
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_WFD
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+	struct wifi_display_info *pwfd_info = ptdlsinfo->wfd_info;
+
+	RTW_INFO("[%s]\n", __FUNCTION__);
+
+	sprintf(extra, "\n\n%d\n", pwfd_info->peer_rtsp_ctrlport);
+	RTW_INFO("[%s] remote port = %d\n",
+		 __FUNCTION__, pwfd_info->peer_rtsp_ctrlport);
+
+	wrqu->data.length = strlen(extra);
+
+#endif /* CONFIG_WFD */
+#endif /* CONFIG_TDLS */
+
+	return ret;
+
+}
+
+/* WFDTDLS, for sigma test */
+static int rtw_tdls_dis_result(struct net_device *dev,
+			       struct iw_request_info *info,
+			       union iwreq_data *wrqu, char *extra)
+{
+
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+#ifdef CONFIG_WFD
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	RTW_INFO("[%s]\n", __FUNCTION__);
+
+	if (ptdlsinfo->dev_discovered == _TRUE) {
+		sprintf(extra, "\n\nDis=1\n");
+		ptdlsinfo->dev_discovered = _FALSE;
+	}
+
+	wrqu->data.length = strlen(extra);
+
+#endif /* CONFIG_WFD */
+#endif /* CONFIG_TDLS */
+
+	return ret;
+
+}
+
+/* WFDTDLS, for sigma test */
+static int rtw_wfd_tdls_status(struct net_device *dev,
+			       struct iw_request_info *info,
+			       union iwreq_data *wrqu, char *extra)
+{
+
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct tdls_info *ptdlsinfo = &padapter->tdlsinfo;
+
+	RTW_INFO("[%s]\n", __FUNCTION__);
+
+	sprintf(extra, "\nlink_established:%d\n"
+		"sta_cnt:%d\n"
+		"sta_maximum:%d\n"
+		"cur_channel:%d\n"
+		"tdls_enable:%d"
+#ifdef CONFIG_TDLS_CH_SW
+		"ch_sw_state:%08x\n"
+		"chsw_on:%d\n"
+		"off_ch_num:%d\n"
+		"cur_time:%d\n"
+		"ch_offset:%d\n"
+		"delay_swtich_back:%d"
+#endif
+		,
+		ptdlsinfo->link_established, ptdlsinfo->sta_cnt,
+		ptdlsinfo->sta_maximum, ptdlsinfo->cur_channel,
+		rtw_is_tdls_enabled(padapter)
+#ifdef CONFIG_TDLS_CH_SW
+		,
+		ptdlsinfo->chsw_info.ch_sw_state,
+		ATOMIC_READ(&padapter->tdlsinfo.chsw_info.chsw_on),
+		ptdlsinfo->chsw_info.off_ch_num,
+		ptdlsinfo->chsw_info.cur_time,
+		ptdlsinfo->chsw_info.ch_offset,
+		ptdlsinfo->chsw_info.delay_switch_back
+#endif
+	       );
+
+	wrqu->data.length = strlen(extra);
+
+#endif /* CONFIG_TDLS */
+
+	return ret;
+
+}
+
+static int rtw_tdls_getsta(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wrqu, char *extra)
+{
+
+	int ret = 0;
+#ifdef CONFIG_TDLS
+	u8 i, j;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 addr[ETH_ALEN] = {0};
+	char charmac[17];
+	struct sta_info *ptdls_sta = NULL;
+
+	RTW_INFO("[%s] %s %d\n", __FUNCTION__,
+		 (char *)wrqu->data.pointer, wrqu->data.length - 1);
+
+	if (copy_from_user(charmac, wrqu->data.pointer + 9, 17)) {
+		ret = -EFAULT;
+		goto exit;
+	}
+
+	RTW_INFO("[%s] %d, charmac:%s\n", __FUNCTION__, __LINE__, charmac);
+	for (i = 0, j = 0 ; i < ETH_ALEN; i++, j += 3)
+		addr[i] = key_2char2num(*(charmac + j), *(charmac + j + 1));
+
+	RTW_INFO("[%s] %d, charmac:%s, addr:"MAC_FMT"\n",
+		 __FUNCTION__, __LINE__, charmac, MAC_ARG(addr));
+	ptdls_sta = rtw_get_stainfo(&padapter->stapriv, addr);
+	if (ptdls_sta) {
+		sprintf(extra, "\n\ntdls_sta_state=0x%08x\n", ptdls_sta->tdls_sta_state);
+		RTW_INFO("\n\ntdls_sta_state=%d\n", ptdls_sta->tdls_sta_state);
+	} else {
+		sprintf(extra, "\n\nNot found this sta\n");
+		RTW_INFO("\n\nNot found this sta\n");
+	}
+	wrqu->data.length = strlen(extra);
+
+exit:
+#endif /* CONFIG_TDLS */
+	return ret;
+
+}
+
+static int rtw_tdls_get_best_ch(struct net_device *dev,
+				struct iw_request_info *info,
+				union iwreq_data *wrqu, char *extra)
+{
+#ifdef CONFIG_FIND_BEST_CHANNEL
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_chset *chset = adapter_to_chset(padapter);
+	u32 i, best_channel_24G = 1, best_channel_5G = 36, index_24G = 0, index_5G = 0;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].flags & RTW_CHF_DIS)
+			continue;
+		if (chset->chs[i].ChannelNum == 1)
+			index_24G = i;
+		if (chset->chs[i].ChannelNum == 36)
+			index_5G = i;
+	}
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].flags & RTW_CHF_DIS)
+			continue;
+
+		/* 2.4G */
+		if (chset->chs[i].ChannelNum == 6 || chset->chs[i].ChannelNum == 11) {
+			if (chset->chs[i].rx_count < chset->chs[index_24G].rx_count) {
+				index_24G = i;
+				best_channel_24G = chset->chs[i].ChannelNum;
+			}
+		}
+
+		/* 5G */
+		if (chset->chs[i].ChannelNum >= 36
+		    && chset->chs[i].ChannelNum < 140) {
+			/* Find primary channel */
+			if (((chset->chs[i].ChannelNum - 36) % 8 == 0)
+			    && (chset->chs[i].rx_count < chset->chs[index_5G].rx_count)) {
+				index_5G = i;
+				best_channel_5G = chset->chs[i].ChannelNum;
+			}
+		}
+
+		if (chset->chs[i].ChannelNum >= 149
+		    && chset->chs[i].ChannelNum < 165) {
+			/* Find primary channel */
+			if (((chset->chs[i].ChannelNum - 149) % 8 == 0)
+			    && (chset->chs[i].rx_count < chset->chs[index_5G].rx_count)) {
+				index_5G = i;
+				best_channel_5G = chset->chs[i].ChannelNum;
+			}
+		}
+#if 1 /* debug */
+		RTW_INFO("The rx cnt of channel %3d = %d\n",
+			 chset->chs[i].ChannelNum,
+			 chset->chs[i].rx_count);
+#endif
+	}
+
+	sprintf(extra, "\nbest_channel_24G = %d\n", best_channel_24G);
+	RTW_INFO("best_channel_24G = %d\n", best_channel_24G);
+
+	if (index_5G != 0) {
+		sprintf(extra, "best_channel_5G = %d\n", best_channel_5G);
+		RTW_INFO("best_channel_5G = %d\n", best_channel_5G);
+	}
+
+	wrqu->data.length = strlen(extra);
+
+#endif
+
+	return 0;
+
+}
+#endif /*#ifdef CONFIG_TDLS*/
+static int rtw_tdls(struct net_device *dev,
+		    struct iw_request_info *info,
+		    union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_INFO("[%s] extra = %s\n", __FUNCTION__, extra);
+
+	if (rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_TDLS) == _FALSE) {
+		RTW_INFO("Discard tdls oper since hal doesn't support tdls\n");
+		return 0;
+	}
+
+	if (rtw_is_tdls_enabled(padapter) == _FALSE) {
+		RTW_INFO("TDLS is not enabled\n");
+		return 0;
+	}
+
+	/* WFD Sigma will use the tdls enable command to let the driver know we want to test the tdls now! */
+
+	if (rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST)) {
+		if (_rtw_memcmp(extra, "wfdenable=", 10)) {
+			wrqu->data.length -= 10;
+			rtw_wx_tdls_wfd_enable(dev, info, wrqu, &extra[10]);
+			return ret;
+		}
+	}
+
+	if (_rtw_memcmp(extra, "weaksec=", 8)) {
+		wrqu->data.length -= 8;
+		rtw_tdls_weaksec(dev, info, wrqu, &extra[8]);
+		return ret;
+	} else if (_rtw_memcmp(extra, "tdlsenable=", 11)) {
+		wrqu->data.length -= 11;
+		rtw_tdls_enable(dev, info, wrqu, &extra[11]);
+		return ret;
+	}
+
+	if (_rtw_memcmp(extra, "setup=", 6)) {
+		wrqu->data.length -= 6;
+		rtw_tdls_setup(dev, info, wrqu, &extra[6]);
+	} else if (_rtw_memcmp(extra, "tear=", 5)) {
+		wrqu->data.length -= 5;
+		rtw_tdls_teardown(dev, info, wrqu, &extra[5]);
+	} else if (_rtw_memcmp(extra, "dis=", 4)) {
+		wrqu->data.length -= 4;
+		rtw_tdls_discovery(dev, info, wrqu, &extra[4]);
+	} else if (_rtw_memcmp(extra, "swoff=", 6)) {
+		wrqu->data.length -= 6;
+		rtw_tdls_ch_switch_off(dev, info, wrqu, &extra[6]);
+	} else if (_rtw_memcmp(extra, "sw=", 3)) {
+		wrqu->data.length -= 3;
+		rtw_tdls_ch_switch(dev, info, wrqu, &extra[3]);
+	} else if (_rtw_memcmp(extra, "dumpstack=", 10)) {
+		wrqu->data.length -= 10;
+		rtw_tdls_dump_ch(dev, info, wrqu, &extra[10]);
+	} else if (_rtw_memcmp(extra, "offchnum=", 9)) {
+		wrqu->data.length -= 9;
+		rtw_tdls_off_ch_num(dev, info, wrqu, &extra[9]);
+	} else if (_rtw_memcmp(extra, "choffset=", 9)) {
+		wrqu->data.length -= 9;
+		rtw_tdls_ch_offset(dev, info, wrqu, &extra[9]);
+	} else if (_rtw_memcmp(extra, "pson=", 5)) {
+		wrqu->data.length -= 5;
+		rtw_tdls_pson(dev, info, wrqu, &extra[5]);
+	} else if (_rtw_memcmp(extra, "psoff=", 6)) {
+		wrqu->data.length -= 6;
+		rtw_tdls_psoff(dev, info, wrqu, &extra[6]);
+	}
+
+#ifdef CONFIG_WFD
+	if (rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST)) {
+		if (_rtw_memcmp(extra, "setip=", 6)) {
+			wrqu->data.length -= 6;
+			rtw_tdls_setip(dev, info, wrqu, &extra[6]);
+		} else if (_rtw_memcmp(extra, "tprobe=", 6))
+			issue_tunneled_probe_req((_adapter *)rtw_netdev_priv(dev));
+	}
+#endif /* CONFIG_WFD */
+
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+
+static int rtw_tdls_get(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wrqu, char *extra)
+{
+	int ret = 0;
+
+#ifdef CONFIG_TDLS
+
+	RTW_INFO("[%s] extra = %s\n", __FUNCTION__, (char *) wrqu->data.pointer);
+
+	if (_rtw_memcmp(wrqu->data.pointer, "ip", 2))
+		rtw_tdls_getip(dev, info, wrqu, extra);
+	else if (_rtw_memcmp(wrqu->data.pointer, "port", 4))
+		rtw_tdls_getport(dev, info, wrqu, extra);
+	/* WFDTDLS, for sigma test */
+	else if (_rtw_memcmp(wrqu->data.pointer, "dis", 3))
+		rtw_tdls_dis_result(dev, info, wrqu, extra);
+	else if (_rtw_memcmp(wrqu->data.pointer, "status", 6))
+		rtw_wfd_tdls_status(dev, info, wrqu, extra);
+	else if (_rtw_memcmp(wrqu->data.pointer, "tdls_sta=", 9))
+		rtw_tdls_getsta(dev, info, wrqu, extra);
+	else if (_rtw_memcmp(wrqu->data.pointer, "best_ch", 7))
+		rtw_tdls_get_best_ch(dev, info, wrqu, extra);
+#endif /* CONFIG_TDLS */
+
+	return ret;
+}
+
+#if 0 /*#ifdef CONFIG_MAC_LOOPBACK_DRIVER*/
+static s32 initLoopback(_adapter *padapter)
+{
+	PLOOPBACKDATA ploopback;
+
+
+	if (padapter->ploopback == NULL) {
+		ploopback = (PLOOPBACKDATA)rtw_zmalloc(sizeof(LOOPBACKDATA));
+		if (ploopback == NULL)
+			return -ENOMEM;
+
+		_rtw_init_sema(&ploopback->sema, 0);
+		ploopback->bstop = _TRUE;
+		ploopback->cnt = 0;
+		ploopback->size = 300;
+		_rtw_memset(ploopback->msg, 0, sizeof(ploopback->msg));
+
+		padapter->ploopback = ploopback;
+	}
+
+	return 0;
+}
+
+static void freeLoopback(_adapter *padapter)
+{
+	PLOOPBACKDATA ploopback;
+
+
+	ploopback = padapter->ploopback;
+	if (ploopback) {
+		rtw_mfree((u8 *)ploopback, sizeof(LOOPBACKDATA));
+		padapter->ploopback = NULL;
+	}
+}
+
+static s32 initpseudoadhoc(_adapter *padapter)
+{
+	NDIS_802_11_NETWORK_INFRASTRUCTURE networkType;
+	s32 err;
+
+	networkType = Ndis802_11IBSS;
+	err = rtw_set_802_11_infrastructure_mode(padapter, networkType, 0);
+	if (err == _FALSE)
+		return _FAIL;
+
+	err = rtw_setopmode_cmd(padapter, networkType, RTW_CMDF_WAIT_ACK);
+	if (err == _FAIL)
+		return _FAIL;
+
+	return _SUCCESS;
+}
+
+static s32 createpseudoadhoc(_adapter *padapter)
+{
+	NDIS_802_11_AUTHENTICATION_MODE authmode;
+	struct mlme_priv *pmlmepriv;
+	NDIS_802_11_SSID *passoc_ssid;
+	WLAN_BSSID_EX *pdev_network;
+	u8 *pibss;
+	u8 ssid[] = "pseduo_ad-hoc";
+	s32 err;
+
+
+	pmlmepriv = &padapter->mlmepriv;
+
+	authmode = Ndis802_11AuthModeOpen;
+	err = rtw_set_802_11_authentication_mode(padapter, authmode);
+	if (err == _FALSE)
+		return _FAIL;
+
+	passoc_ssid = &pmlmepriv->assoc_ssid;
+	_rtw_memset(passoc_ssid, 0, sizeof(NDIS_802_11_SSID));
+	passoc_ssid->SsidLength = sizeof(ssid) - 1;
+	_rtw_memcpy(passoc_ssid->Ssid, ssid, passoc_ssid->SsidLength);
+
+	pdev_network = &padapter->registrypriv.dev_network;
+	pibss = padapter->registrypriv.dev_network.MacAddress;
+	_rtw_memcpy(&pdev_network->Ssid, passoc_ssid, sizeof(NDIS_802_11_SSID));
+
+	rtw_update_registrypriv_dev_network(padapter);
+	rtw_generate_random_ibss(pibss);
+
+	_rtw_spinlock_bh(&pmlmepriv->lock);
+	/*pmlmepriv->fw_state = WIFI_ADHOC_MASTER_STATE;*/
+	init_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
+
+	_rtw_spinunlock_bh(&pmlmepriv->lock);
+
+#if 0
+	err = rtw_create_ibss_cmd(padapter, 0);
+	if (err == _FAIL)
+		return _FAIL;
+#else
+	{
+		struct wlan_network *pcur_network;
+		struct sta_info *psta;
+
+		/* 3  create a new psta */
+		pcur_network = &pmlmepriv->cur_network;
+
+		/* clear psta in the cur_network, if any */
+		psta = rtw_get_stainfo(&padapter->stapriv, pcur_network->network.MacAddress);
+		if (psta)
+			rtw_free_stainfo(padapter, psta);
+
+		psta = rtw_alloc_stainfo(&padapter->stapriv, pibss);
+		if (psta == NULL)
+			return _FAIL;
+
+		/* 3  join psudo AdHoc */
+		pcur_network->join_res = 1;
+		pcur_network->aid = psta->phl_sta->aid = 1;
+		_rtw_memcpy(&pcur_network->network, pdev_network, get_WLAN_BSSID_EX_sz(pdev_network));
+
+	}
+#endif
+
+	return _SUCCESS;
+}
+/*GEORGIA_TODO_FIXIT_MOVE_TO_HAL*/
+static struct xmit_frame *createloopbackpkt(_adapter *padapter, u32 size)
+{
+	struct xmit_priv *pxmitpriv;
+	struct xmit_frame *pframe;
+	struct xmit_buf *pxmitbuf;
+	struct pkt_attrib *pattrib;
+	struct tx_desc *desc;
+	u8 *pkt_start, *pkt_end, *ptr;
+	struct rtw_ieee80211_hdr *hdr;
+	s32 bmcast;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u16 xmitbuf_sz = GET_HAL_XMITBUF_SZ(dvobj);
+
+	if ((TXDESC_SIZE + WLANHDR_OFFSET + size) > xmitbuf_sz)
+		return NULL;
+
+	pxmitpriv = &padapter->xmitpriv;
+	pframe = NULL;
+
+	/* 2 1. allocate xmit frame */
+	pframe = rtw_alloc_xmitframe(pxmitpriv);
+	if (pframe == NULL)
+		return NULL;
+	pframe->padapter = padapter;
+
+	/* 2 2. allocate xmit buffer */
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+	pxmitbuf = rtw_alloc_xmitbuf(pxmitpriv);
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+	if (pxmitbuf == NULL) {
+		rtw_free_xmitframe(pxmitpriv, pframe);
+		return NULL;
+	}
+
+	pframe->pxmitbuf = pxmitbuf;
+	pframe->buf_addr = pxmitbuf->pbuf;
+	pxmitbuf->priv_data = pframe;
+
+	/* 2 3. update_attrib() */
+	pattrib = &pframe->attrib;
+
+	/* init xmitframe attribute */
+	_rtw_memset(pattrib, 0, sizeof(struct pkt_attrib));
+
+	pattrib->ether_type = 0x8723;
+	_rtw_memcpy(pattrib->src, adapter_mac_addr(padapter), ETH_ALEN);
+	_rtw_memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
+	_rtw_memset(pattrib->dst, 0xFF, ETH_ALEN);
+	_rtw_memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
+
+	/*	pattrib->dhcp_pkt = 0;
+	 *	pattrib->pktlen = 0; */
+	pattrib->ack_policy = 0;
+	/*	pattrib->pkt_hdrlen = ETH_HLEN; */
+	pattrib->hdrlen = WLAN_HDR_A3_LEN;
+	pattrib->subtype = WIFI_DATA;
+	pattrib->priority = 0;
+	pattrib->qsel = pattrib->priority;
+	/*	do_queue_select(padapter, pattrib); */
+	pattrib->nr_frags = 1;
+	pattrib->encrypt = 0;
+	pattrib->bswenc = _FALSE;
+	pattrib->qos_en = _FALSE;
+
+	bmcast = IS_MCAST(pattrib->ra);
+	if (bmcast)
+		pattrib->psta = rtw_get_bcmc_stainfo(padapter);
+	else
+		pattrib->psta = rtw_get_stainfo(&padapter->stapriv, get_bssid(&padapter->mlmepriv));
+
+	pattrib->mac_id = pattrib->psta->phl_sta->macid;
+	pattrib->pktlen = size;
+	pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->pktlen;
+
+	/* 2 4. fill TX descriptor */
+	desc = (struct tx_desc *)pframe->buf_addr;
+	_rtw_memset(desc, 0, TXDESC_SIZE);
+
+	fill_default_txdesc(pframe, (u8 *)desc);
+
+	/* Hw set sequence number */
+	((PTXDESC)desc)->hwseq_en = 0; /* HWSEQ_EN, 0:disable, 1:enable
+ * ((PTXDESC)desc)->hwseq_sel = 0;  */ /* HWSEQ_SEL */
+
+	((PTXDESC)desc)->disdatafb = 1;
+
+	/* convert to little endian */
+	desc->txdw0 = cpu_to_le32(desc->txdw0);
+	desc->txdw1 = cpu_to_le32(desc->txdw1);
+	desc->txdw2 = cpu_to_le32(desc->txdw2);
+	desc->txdw3 = cpu_to_le32(desc->txdw3);
+	desc->txdw4 = cpu_to_le32(desc->txdw4);
+	desc->txdw5 = cpu_to_le32(desc->txdw5);
+	desc->txdw6 = cpu_to_le32(desc->txdw6);
+	desc->txdw7 = cpu_to_le32(desc->txdw7);
+#ifdef CONFIG_PCI_HCI
+	desc->txdw8 = cpu_to_le32(desc->txdw8);
+	desc->txdw9 = cpu_to_le32(desc->txdw9);
+	desc->txdw10 = cpu_to_le32(desc->txdw10);
+	desc->txdw11 = cpu_to_le32(desc->txdw11);
+	desc->txdw12 = cpu_to_le32(desc->txdw12);
+	desc->txdw13 = cpu_to_le32(desc->txdw13);
+	desc->txdw14 = cpu_to_le32(desc->txdw14);
+	desc->txdw15 = cpu_to_le32(desc->txdw15);
+#endif
+
+	cal_txdesc_chksum(desc);
+
+	/* 2 5. coalesce */
+	pkt_start = pframe->buf_addr + TXDESC_SIZE;
+	pkt_end = pkt_start + pattrib->last_txcmdsz;
+
+	/* 3 5.1. make wlan header, make_wlanhdr() */
+	hdr = (struct rtw_ieee80211_hdr *)pkt_start;
+	set_frame_sub_type(&hdr->frame_ctl, pattrib->subtype);
+	_rtw_memcpy(hdr->addr1, pattrib->dst, ETH_ALEN); /* DA */
+	_rtw_memcpy(hdr->addr2, pattrib->src, ETH_ALEN); /* SA */
+	_rtw_memcpy(hdr->addr3, get_bssid(&padapter->mlmepriv), ETH_ALEN); /* RA, BSSID */
+
+	/* 3 5.2. make payload */
+	ptr = pkt_start + pattrib->hdrlen;
+	get_random_bytes(ptr, pkt_end - ptr);
+
+	pxmitbuf->len = TXDESC_SIZE + pattrib->last_txcmdsz;
+	pxmitbuf->ptail += pxmitbuf->len;
+
+	return pframe;
+}
+
+static void freeloopbackpkt(_adapter *padapter, struct xmit_frame *pframe)
+{
+	struct xmit_priv *pxmitpriv;
+	struct xmit_buf *pxmitbuf;
+
+
+	pxmitpriv = &padapter->xmitpriv;
+	pxmitbuf = pframe->pxmitbuf;
+
+	rtw_free_xmitframe(pxmitpriv, pframe);
+	rtw_free_xmitbuf(pxmitpriv, pxmitbuf);
+}
+
+static void printdata(u8 *pbuf, u32 len)
+{
+	u32 i, val;
+
+
+	for (i = 0; (i + 4) <= len; i += 4) {
+		printk("%08X", *(u32 *)(pbuf + i));
+		if ((i + 4) & 0x1F)
+			printk(" ");
+		else
+			printk("\n");
+	}
+
+	if (i < len) {
+#ifdef CONFIG_BIG_ENDIAN
+		for (; i < len, i++)
+			printk("%02X", pbuf + i);
+#else /* CONFIG_LITTLE_ENDIAN */
+#if 0
+		val = 0;
+		_rtw_memcpy(&val, pbuf + i, len - i);
+		printk("%8X", val);
+#else
+		u8 str[9];
+		u8 n;
+		val = 0;
+		n = len - i;
+		_rtw_memcpy(&val, pbuf + i, n);
+		sprintf(str, "%08X", val);
+		n = (4 - n) * 2;
+		printk("%8s", str + n);
+#endif
+#endif /* CONFIG_LITTLE_ENDIAN */
+	}
+	printk("\n");
+}
+
+static u8 pktcmp(_adapter *padapter, u8 *txbuf, u32 txsz, u8 *rxbuf, u32 rxsz)
+{
+	struct recv_stat *prxstat;
+	struct recv_stat report;
+	PRXREPORT prxreport;
+	u32 drvinfosize;
+	u32 rxpktsize;
+	u8 fcssize;
+	u8 ret = _FALSE;
+
+	prxstat = (struct recv_stat *)rxbuf;
+	report.rxdw0 = le32_to_cpu(prxstat->rxdw0);
+	report.rxdw1 = le32_to_cpu(prxstat->rxdw1);
+	report.rxdw2 = le32_to_cpu(prxstat->rxdw2);
+	report.rxdw3 = le32_to_cpu(prxstat->rxdw3);
+	report.rxdw4 = le32_to_cpu(prxstat->rxdw4);
+	report.rxdw5 = le32_to_cpu(prxstat->rxdw5);
+
+	prxreport = (PRXREPORT)&report;
+	drvinfosize = prxreport->drvinfosize << 3;
+	rxpktsize = prxreport->pktlen;
+
+	if (rtw_hal_rcr_check(padapter, RCR_APPFCS))
+		fcssize = IEEE80211_FCS_LEN;
+	else
+		fcssize = 0;
+
+	if ((txsz - TXDESC_SIZE) != (rxpktsize - fcssize)) {
+		RTW_INFO("%s: ERROR! size not match tx/rx=%d/%d !\n",
+			 __func__, txsz - TXDESC_SIZE, rxpktsize - fcssize);
+		ret = _FALSE;
+	} else {
+		ret = _rtw_memcmp(txbuf + TXDESC_SIZE, \
+				  rxbuf + RXDESC_SIZE + drvinfosize, \
+				  txsz - TXDESC_SIZE);
+		if (ret == _FALSE)
+			RTW_INFO("%s: ERROR! pkt content mismatch!\n", __func__);
+	}
+
+	if (ret == _FALSE) {
+		RTW_INFO("\n%s: TX PKT total=%d, desc=%d, content=%d\n",
+			 __func__, txsz, TXDESC_SIZE, txsz - TXDESC_SIZE);
+		RTW_INFO("%s: TX DESC size=%d\n", __func__, TXDESC_SIZE);
+		printdata(txbuf, TXDESC_SIZE);
+		RTW_INFO("%s: TX content size=%d\n", __func__, txsz - TXDESC_SIZE);
+		printdata(txbuf + TXDESC_SIZE, txsz - TXDESC_SIZE);
+
+		RTW_INFO("\n%s: RX PKT read=%d offset=%d(%d,%d) content=%d\n",
+			__func__, rxsz, RXDESC_SIZE + drvinfosize, RXDESC_SIZE, drvinfosize, rxpktsize);
+		if (rxpktsize != 0) {
+			RTW_INFO("%s: RX DESC size=%d\n", __func__, RXDESC_SIZE);
+			printdata(rxbuf, RXDESC_SIZE);
+			RTW_INFO("%s: RX drvinfo size=%d\n", __func__, drvinfosize);
+			printdata(rxbuf + RXDESC_SIZE, drvinfosize);
+			RTW_INFO("%s: RX content size=%d\n", __func__, rxpktsize);
+			printdata(rxbuf + RXDESC_SIZE + drvinfosize, rxpktsize);
+		} else {
+			RTW_INFO("%s: RX data size=%d\n", __func__, rxsz);
+			printdata(rxbuf, rxsz);
+		}
+	}
+
+	return ret;
+}
+
+thread_return lbk_thread(thread_context context)
+{
+#if 0
+	s32 err;
+	_adapter *padapter;
+	PLOOPBACKDATA ploopback;
+	struct xmit_frame *pxmitframe;
+	u32 cnt, ok, fail, headerlen;
+	u32 pktsize;
+	u32 ff_hwaddr;
+
+
+	padapter = (_adapter *)context;
+	ploopback = padapter->ploopback;
+	if (ploopback == NULL)
+		return -1;
+	cnt = 0;
+	ok = 0;
+	fail = 0;
+
+	daemonize("%s", "RTW_LBK_THREAD");
+	allow_signal(SIGTERM);
+
+	do {
+		if (ploopback->size == 0) {
+			get_random_bytes(&pktsize, 4);
+			pktsize = (pktsize % 1535) + 1; /* 1~1535 */
+		} else
+			pktsize = ploopback->size;
+
+		pxmitframe = createloopbackpkt(padapter, pktsize);
+		if (pxmitframe == NULL) {
+			sprintf(ploopback->msg, "loopback FAIL! 3. create Packet FAIL!");
+			break;
+		}
+
+		ploopback->txsize = TXDESC_SIZE + pxmitframe->attrib.last_txcmdsz;
+		_rtw_memcpy(ploopback->txbuf, pxmitframe->buf_addr, ploopback->txsize);
+		ff_hwaddr = rtw_get_ff_hwaddr(pxmitframe);
+		cnt++;
+		RTW_INFO("%s: wirte port cnt=%d size=%d\n", __func__, cnt, ploopback->txsize);
+		pxmitframe->pxmitbuf->pdata = ploopback->txbuf;
+		rtw_write_port(padapter, ff_hwaddr, ploopback->txsize, (u8 *)pxmitframe->pxmitbuf);
+
+		/* wait for rx pkt */
+		_rtw_down_sema(&ploopback->sema);
+
+		err = pktcmp(padapter, ploopback->txbuf, ploopback->txsize, ploopback->rxbuf, ploopback->rxsize);
+		if (err == _TRUE)
+			ok++;
+		else
+			fail++;
+
+		ploopback->txsize = 0;
+		_rtw_memset(ploopback->txbuf, 0, 0x8000);
+		ploopback->rxsize = 0;
+		_rtw_memset(ploopback->rxbuf, 0, 0x8000);
+
+		freeloopbackpkt(padapter, pxmitframe);
+		pxmitframe = NULL;
+
+		flush_signals_thread();
+
+		if ((ploopback->bstop == _TRUE) ||
+		    ((ploopback->cnt != 0) && (ploopback->cnt == cnt))) {
+			u32 ok_rate, fail_rate, all;
+			all = cnt;
+			ok_rate = (ok * 100) / all;
+			fail_rate = (fail * 100) / all;
+			sprintf(ploopback->msg, \
+				"loopback result: ok=%d%%(%d/%d),error=%d%%(%d/%d)", \
+				ok_rate, ok, all, fail_rate, fail, all);
+			break;
+		}
+	} while (1);
+
+	ploopback->bstop = _TRUE;
+
+	thread_exit(NULL);
+#endif
+	return 0;
+}
+
+static void loopbackTest(_adapter *padapter, u32 cnt, u32 size, u8 *pmsg)
+{
+	PLOOPBACKDATA ploopback;
+	u32 len;
+	s32 err;
+
+
+	ploopback = padapter->ploopback;
+
+	if (ploopback) {
+		if (ploopback->bstop == _FALSE) {
+			ploopback->bstop = _TRUE;
+			_rtw_up_sema(&ploopback->sema);
+		}
+		len = 0;
+		do {
+			len = strlen(ploopback->msg);
+			if (len)
+				break;
+			rtw_msleep_os(1);
+		} while (1);
+		_rtw_memcpy(pmsg, ploopback->msg, len + 1);
+		freeLoopback(padapter);
+
+		return;
+	}
+
+	/* disable dynamic algorithm	 */
+	rtw_phydm_ability_backup(padapter);
+	rtw_phydm_func_disable_all(padapter);
+
+	/* create pseudo ad-hoc connection */
+	err = initpseudoadhoc(padapter);
+	if (err == _FAIL) {
+		sprintf(pmsg, "loopback FAIL! 1.1 init ad-hoc FAIL!");
+		return;
+	}
+
+	err = createpseudoadhoc(padapter);
+	if (err == _FAIL) {
+		sprintf(pmsg, "loopback FAIL! 1.2 create ad-hoc master FAIL!");
+		return;
+	}
+
+	err = initLoopback(padapter);
+	if (err) {
+		sprintf(pmsg, "loopback FAIL! 2. init FAIL! error code=%d", err);
+		return;
+	}
+
+	ploopback = padapter->ploopback;
+
+	ploopback->bstop = _FALSE;
+	ploopback->cnt = cnt;
+	ploopback->size = size;
+	ploopback->lbkthread = rtw_thread_start(lbk_thread, padapter, "RTW_LBK_THREAD");
+	if (ploopback->lbkthread == NULL)) {
+		freeLoopback(padapter);
+		sprintf(pmsg, "loopback start FAIL! cnt=%d", cnt);
+		return;
+	}
+
+	sprintf(pmsg, "loopback start! cnt=%d", cnt);
+}
+#endif /* CONFIG_MAC_LOOPBACK_DRIVER */
+
+static int rtw_test(
+	struct net_device *dev,
+	struct iw_request_info *info,
+	union iwreq_data *wrqu, char *extra)
+{
+	u32 len;
+	u8 *pbuf, *pch;
+	char *ptmp;
+	u8 *delim = ",";
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+
+	RTW_INFO("+%s\n", __func__);
+	len = wrqu->data.length;
+
+	pbuf = (u8 *)rtw_zmalloc(len + 1);
+	if (pbuf == NULL) {
+		RTW_INFO("%s: no memory!\n", __func__);
+		return -ENOMEM;
+	}
+
+	if (copy_from_user(pbuf, wrqu->data.pointer, len)) {
+		rtw_mfree(pbuf, len + 1);
+		RTW_INFO("%s: copy from user fail!\n", __func__);
+		return -EFAULT;
+	}
+
+	pbuf[len] = '\0';
+
+	RTW_INFO("%s: string=\"%s\"\n", __func__, pbuf);
+
+	ptmp = (char *)pbuf;
+	pch = strsep(&ptmp, delim);
+	if ((pch == NULL) || (strlen(pch) == 0)) {
+		rtw_mfree(pbuf, len);
+		RTW_INFO("%s: parameter error(level 1)!\n", __func__);
+		return -EFAULT;
+	}
+
+#if 0 /*#ifdef CONFIG_MAC_LOOPBACK_DRIVER*/
+	if (strcmp(pch, "loopback") == 0) {
+		s32 cnt = 0;
+		u32 size = 64;
+
+		pch = strsep(&ptmp, delim);
+		if ((pch == NULL) || (strlen(pch) == 0)) {
+			rtw_mfree(pbuf, len);
+			RTW_INFO("%s: parameter error(level 2)!\n", __func__);
+			return -EFAULT;
+		}
+
+		sscanf(pch, "%d", &cnt);
+		RTW_INFO("%s: loopback cnt=%d\n", __func__, cnt);
+
+		pch = strsep(&ptmp, delim);
+		if ((pch == NULL) || (strlen(pch) == 0)) {
+			rtw_mfree(pbuf, len);
+			RTW_INFO("%s: parameter error(level 2)!\n", __func__);
+			return -EFAULT;
+		}
+
+		sscanf(pch, "%d", &size);
+		RTW_INFO("%s: loopback size=%d\n", __func__, size);
+
+		loopbackTest(padapter, cnt, size, extra);
+		wrqu->data.length = strlen(extra) + 1;
+
+		goto free_buf;
+	}
+#endif
+
+	if (strcmp(pch, "h2c") == 0) {
+		u8 param[8];
+		u8 count = 0;
+		u32 tmp;
+		u8 i;
+		u32 pos;
+		u8 ret;
+
+		do {
+			pch = strsep(&ptmp, delim);
+			if ((pch == NULL) || (strlen(pch) == 0))
+				break;
+
+			sscanf(pch, "%x", &tmp);
+			param[count++] = (u8)tmp;
+		} while (count < 8);
+
+		if (count == 0) {
+			rtw_mfree(pbuf, len);
+			RTW_INFO("%s: parameter error(level 2)!\n", __func__);
+			return -EFAULT;
+		}
+
+		ret = rtw_test_h2c_cmd(padapter, param, count);
+
+		pos = sprintf(extra, "H2C ID=0x%02x content=", param[0]);
+		for (i = 1; i < count; i++)
+			pos += sprintf(extra + pos, "%02x,", param[i]);
+		extra[pos] = 0;
+		pos--;
+		pos += sprintf(extra + pos, " %s", ret == _FAIL ? "FAIL" : "OK");
+
+		wrqu->data.length = strlen(extra) + 1;
+
+		goto free_buf;
+	}
+
+free_buf:
+	rtw_mfree(pbuf, len);
+	return 0;
+}
+
+static iw_handler rtw_handlers[] = {
+	NULL,					/* SIOCSIWCOMMIT */
+	rtw_wx_get_name,		/* SIOCGIWNAME */
+	dummy,					/* SIOCSIWNWID */
+	dummy,					/* SIOCGIWNWID */
+	rtw_wx_set_freq,		/* SIOCSIWFREQ */
+	rtw_wx_get_freq,		/* SIOCGIWFREQ */
+	rtw_wx_set_mode,		/* SIOCSIWMODE */
+	rtw_wx_get_mode,		/* SIOCGIWMODE */
+	dummy,					/* SIOCSIWSENS */
+	rtw_wx_get_sens,		/* SIOCGIWSENS */
+	NULL,					/* SIOCSIWRANGE */
+	rtw_wx_get_range,		/* SIOCGIWRANGE */
+	rtw_wx_set_priv,		/* SIOCSIWPRIV */
+	NULL,					/* SIOCGIWPRIV */
+	NULL,					/* SIOCSIWSTATS */
+	NULL,					/* SIOCGIWSTATS */
+	dummy,					/* SIOCSIWSPY */
+	dummy,					/* SIOCGIWSPY */
+	NULL,					/* SIOCGIWTHRSPY */
+	NULL,					/* SIOCWIWTHRSPY */
+	rtw_wx_set_wap,		/* SIOCSIWAP */
+	rtw_wx_get_wap,		/* SIOCGIWAP */
+	rtw_wx_set_mlme,		/* request MLME operation; uses struct iw_mlme */
+	dummy,					/* SIOCGIWAPLIST -- depricated */
+	rtw_wx_set_scan,		/* SIOCSIWSCAN */
+	rtw_wx_get_scan,		/* SIOCGIWSCAN */
+	rtw_wx_set_essid,		/* SIOCSIWESSID */
+	rtw_wx_get_essid,		/* SIOCGIWESSID */
+	dummy,					/* SIOCSIWNICKN */
+	rtw_wx_get_nick,		/* SIOCGIWNICKN */
+	NULL,					/* -- hole -- */
+	NULL,					/* -- hole -- */
+	rtw_wx_set_rate,		/* SIOCSIWRATE */
+	rtw_wx_get_rate,		/* SIOCGIWRATE */
+	rtw_wx_set_rts,			/* SIOCSIWRTS */
+	rtw_wx_get_rts,			/* SIOCGIWRTS */
+	rtw_wx_set_frag,		/* SIOCSIWFRAG */
+	rtw_wx_get_frag,		/* SIOCGIWFRAG */
+	dummy,					/* SIOCSIWTXPOW */
+	dummy,					/* SIOCGIWTXPOW */
+	dummy,					/* SIOCSIWRETRY */
+	rtw_wx_get_retry,		/* SIOCGIWRETRY */
+	rtw_wx_set_enc,			/* SIOCSIWENCODE */
+	rtw_wx_get_enc,			/* SIOCGIWENCODE */
+	dummy,					/* SIOCSIWPOWER */
+	rtw_wx_get_power,		/* SIOCGIWPOWER */
+	NULL,					/*---hole---*/
+	NULL,					/*---hole---*/
+	rtw_wx_set_gen_ie,		/* SIOCSIWGENIE */
+	NULL,					/* SIOCGWGENIE */
+	rtw_wx_set_auth,		/* SIOCSIWAUTH */
+	NULL,					/* SIOCGIWAUTH */
+	rtw_wx_set_enc_ext,		/* SIOCSIWENCODEEXT */
+	NULL,					/* SIOCGIWENCODEEXT */
+	rtw_wx_set_pmkid,		/* SIOCSIWPMKSA */
+	NULL,					/*---hole---*/
+};
+
+
+static const struct iw_priv_args rtw_private_args[] = {
+	{
+		SIOCIWFIRSTPRIV + 0x0,
+		IW_PRIV_TYPE_CHAR | 0x7FF, 0, "write"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x1,
+		IW_PRIV_TYPE_CHAR | 0x7FF,
+		IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_FIXED | IFNAMSIZ, "read"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x2, 0, 0, "driver_ext"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x3, 0, 0, "mp_ioctl"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x4,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "apinfo"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x5,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 2, 0, "setpid"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x6,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "wps_start"
+	},
+	/* for PLATFORM_MT53XX	 */
+	{
+		SIOCIWFIRSTPRIV + 0x7,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "get_sensitivity"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x8,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "wps_prob_req_ie"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x9,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "wps_assoc_req_ie"
+	},
+
+	/* for RTK_DMP_PLATFORM	 */
+	{
+		SIOCIWFIRSTPRIV + 0xA,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "channel_plan"
+	},
+
+	{
+		SIOCIWFIRSTPRIV + 0xB,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 2, 0, "dbg"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0xC,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 3, 0, "rfw"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0xD,
+		IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 2, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_FIXED | IFNAMSIZ, "rfr"
+	},
+#if 0
+	{
+		SIOCIWFIRSTPRIV + 0xE, 0, 0, "wowlan_ctrl"
+	},
+#endif
+	{
+		SIOCIWFIRSTPRIV + 0x10,
+		IW_PRIV_TYPE_CHAR | 1024, 0, "p2p_set"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x11,
+		IW_PRIV_TYPE_CHAR | 1024, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK , "p2p_get"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x12, 0, 0, "NULL"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x13,
+		IW_PRIV_TYPE_CHAR | 64, IW_PRIV_TYPE_CHAR | 64 , "p2p_get2"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x14,
+		IW_PRIV_TYPE_CHAR  | 64, 0, "tdls"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x15,
+		IW_PRIV_TYPE_CHAR | 1024, IW_PRIV_TYPE_CHAR | 1024 , "tdls_get"
+	},
+	{
+		SIOCIWFIRSTPRIV + 0x16,
+		IW_PRIV_TYPE_CHAR | 64, 0, "pm_set"
+	},
+#if defined(CONFIG_RTW_80211K) //|| defined(CONFIG_RTW_FSM_RRM) TODO
+	{
+		SIOCIWFIRSTPRIV + 0x17,
+		IW_PRIV_TYPE_CHAR | 1024, IW_PRIV_TYPE_CHAR | 1024 , "rrm"
+	},
+#else
+	{SIOCIWFIRSTPRIV + 0x17, IW_PRIV_TYPE_CHAR | 1024 , 0 , "NULL"},
+#endif
+	{SIOCIWFIRSTPRIV + 0x18, IW_PRIV_TYPE_CHAR | 1024 , 0 , "NULL"},
+#ifdef CONFIG_MP_INCLUDED
+	{SIOCIWFIRSTPRIV + 0x1A, IW_PRIV_TYPE_CHAR | 1024, 0,  "NULL"},
+	{SIOCIWFIRSTPRIV + 0x1B, IW_PRIV_TYPE_CHAR | 128, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "NULL"},
+#else
+	{SIOCIWFIRSTPRIV + 0x1A, IW_PRIV_TYPE_CHAR | 1024, 0,  "NULL"},
+	{SIOCIWFIRSTPRIV + 0x1B, IW_PRIV_TYPE_CHAR | 128, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "efuse_get"},
+#endif
+	{
+		SIOCIWFIRSTPRIV + 0x1D,
+		IW_PRIV_TYPE_CHAR | 40, IW_PRIV_TYPE_CHAR | 0x7FF, "test"
+	},
+
+	{ SIOCIWFIRSTPRIV + 0x0E, IW_PRIV_TYPE_CHAR | 1024, 0 , ""},  /* set  */
+	{ SIOCIWFIRSTPRIV + 0x0F, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK , ""},/* get
+ * --- sub-ioctls definitions --- */
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+	{ VENDOR_IE_SET, IW_PRIV_TYPE_CHAR | 1024 , 0 , "vendor_ie_set" },
+	{ VENDOR_IE_GET, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "vendor_ie_get" },
+#endif
+#if defined(RTW_PHL_TX) || defined(RTW_PHL_RX) || defined(CONFIG_PHL_TEST_SUITE)
+	{ PHL_TEST_SET, IW_PRIV_TYPE_CHAR | 1024 , 0 , "phl_test" },
+	{ PHL_TEST_GET, IW_PRIV_TYPE_CHAR | 1024, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "" },
+#endif
+#ifdef CONFIG_WOWLAN
+	{ MP_WOW_ENABLE , IW_PRIV_TYPE_CHAR | 1024, 0, "wow_mode" },
+	{ MP_WOW_SET_PATTERN , IW_PRIV_TYPE_CHAR | 1024, 0, "wow_set_pattern" },
+#endif
+#ifdef CONFIG_AP_WOWLAN
+	{ MP_AP_WOW_ENABLE , IW_PRIV_TYPE_CHAR | 1024, 0, "ap_wow_mode" }, /* set  */
+#endif
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+	{ MP_SD_IREAD, IW_PRIV_TYPE_CHAR | 1024, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "sd_iread" },
+	{ MP_SD_IWRITE, IW_PRIV_TYPE_CHAR | 1024, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "sd_iwrite" },
+#endif
+#ifdef CONFIG_MP_INCLUDED
+	{ MP_GET_PHL_TEST, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "phl_get_io" },
+	{ MP_SET_PHL_TEST, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "phl_set_io" },
+#endif
+#ifdef CONFIG_FPGA_INCLUDED
+	{ FPGA_SET, IW_PRIV_TYPE_CHAR | 1024 , 0 , "fpga_set" },
+#endif
+#ifdef CONFIG_WOW_APF
+	{ APF_SET, IW_PRIV_TYPE_CHAR | 1024, 0, "apf_set"},
+	{ APF_GET, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "apf_get" },
+	{ APF_DEFAULT_SET, IW_PRIV_TYPE_CHAR | 1024, 0, "apf_dft_set"},
+	{ APF_DEFAULT_GET, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "apf_dft_get" },
+#endif
+};
+
+/* --- sub-ioctls definitions --- */
+static const struct iw_priv_args rtw_mp_private_args[] = {
+#ifdef CONFIG_MP_INCLUDED
+	{ MP_START , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_start" },
+	{ MP_PHYPARA, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_phypara" },
+	{ MP_STOP , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_stop" },
+	{ MP_CHANNEL , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK ,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_channel" },
+	{ MP_TRXSC_OFFSET , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK ,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_trxsc" },
+	{ MP_BANDWIDTH , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_bandwidth"},
+	{ MP_RATE , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_rate" },
+	{ MP_RESET_STATS , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_reset_stats"},
+	{ MP_QUERY , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK , "mp_query"},
+	{ READ_REG , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "read_reg" },
+	{ MP_RATE , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_rate" },
+	{ READ_RF , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "read_rf" },
+	{ MP_PSD , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_psd"},
+	{ MP_DUMP, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_dump" },
+	{ MP_TXPOWER , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_txpower"},
+	{ MP_ANT_TX , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_ant_tx"},
+	{ MP_ANT_RX , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_ant_rx"},
+	{ WRITE_REG , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "write_reg" },
+	{ WRITE_RF , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "write_rf" },
+	{ MP_CTX , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_ctx"},
+	{ MP_ARX , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_arx"},
+	{ MP_THER , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_ther"},
+	{ EFUSE_SET, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "efuse_set" },
+	{ EFUSE_GET, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "efuse_get" },
+	{ MP_PWRTRK , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_pwrtrk"},
+	{ MP_QueryDrvStats, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_drvquery" },
+	{ MP_SetRFPathSwh, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_setrfpath" },
+	{ MP_PwrCtlDM, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_pwrctldm" },
+	{ MP_GET_TXPOWER_INX, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_get_txpower" },
+	{ MP_GETVER, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_priv_ver" },
+	{ MP_MON, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_mon" },
+	{ EFUSE_BT_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "efuse_bt_mask" },
+	{ EFUSE_MASK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "efuse_mask" },
+	{ EFUSE_FILE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "efuse_file" },
+	{ EFUSE_FILE_STORE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "efuse_store" },
+	{ MP_TX, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_tx" },
+	{ MP_RX, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_rx" },
+	{ MP_HW_TX_MODE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_hxtx" },
+	{ MP_PWRLMT, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_pwrlmt" },
+	{ MP_PWRBYRATE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_pwrbyrate" },
+	{ CTA_TEST, IW_PRIV_TYPE_CHAR | 1024, 0, "cta_test"},
+	{ MP_IQK, IW_PRIV_TYPE_CHAR | 1024, 0, "mp_iqk"},
+	{ MP_LCK, IW_PRIV_TYPE_CHAR | 1024, 0, "mp_lck"},
+	{ BT_EFUSE_FILE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "bt_efuse_file" },
+	{ MP_SWRFPath, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_swrfpath" },
+	{ MP_LINK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_link" },
+	{ MP_DPK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_dpk"},
+	{ MP_DPK_TRK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_dpk_trk" },
+	{ MP_GET_TSSIDE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_get_tsside" },
+	{ MP_SET_TSSIDE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_set_tsside" },
+	{ MP_GET_PHL_TEST, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "phl_get_io" },
+	{ MP_SET_PHL_TEST, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "phl_set_io" },
+	{ MP_SET_PHL_TX_PATTERN, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_tx_patt" },
+	{ MP_SET_PHL_TX_METHOD, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "tx_method" },
+	{ MP_SET_PHL_CONIFG_PHY_NUM, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_phy" },
+	{ MP_SET_PHL_PLCP_TX_DATA, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_plcp_data" },
+	{ MP_SET_PHL_PLCP_TX_USER, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_plcp_user" },
+	{ MP_PHL_RFK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_rfk" },
+	{ MP_PHL_BTC_PATH, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_btc_path" },
+	{ MP_GET_HE, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_get_he" },
+	{ MP_BAND, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_band" },
+	{ MP_MACLOOPBK, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mac_loopbk" },
+	{ MP_MAC_IOTEST, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mac_io_test" },
+#ifdef CONFIG_RTW_CUSTOMER_STR
+	{ MP_CUSTOMER_STR, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "customer_str" },
+#endif
+	{ MP_UUID , IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+				IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_uuid"},
+
+	{ MP_GPIO, IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK,
+		IW_PRIV_TYPE_CHAR | IW_PRIV_SIZE_MASK, "mp_gpio"},
+#endif /* CONFIG_MP_INCLUDED */
+};
+
+static iw_handler rtw_private_handler[] = {
+	rtw_wx_write32,					/* 0x00 */
+	rtw_wx_read32,					/* 0x01 */
+	NULL,					/* 0x02 */
+#ifdef MP_IOCTL_HDL
+	rtw_mp_ioctl_hdl,				/* 0x03 */
+#else
+	rtw_wx_priv_null,
+#endif
+	/* for MM DTV platform */
+	rtw_get_ap_info,					/* 0x04 */
+
+	rtw_set_pid,						/* 0x05 */
+	rtw_wps_start,					/* 0x06 */
+
+	/* for PLATFORM_MT53XX */
+	rtw_wx_get_sensitivity,			/* 0x07 */
+	rtw_wx_set_mtk_wps_probe_ie,	/* 0x08 */
+	rtw_wx_set_mtk_wps_ie,			/* 0x09 */
+
+	/* for RTK_DMP_PLATFORM
+	 * Set Channel depend on the country code */
+	rtw_wx_set_channel_plan,		/* 0x0A */
+
+	rtw_dbg_port,					/* 0x0B */
+	rtw_wx_write_rf,					/* 0x0C */
+	rtw_wx_read_rf,					/* 0x0D */
+
+	rtw_priv_set,					/*0x0E*/
+	rtw_priv_get,					/*0x0F*/
+
+	NULL, 						/* 0x10 */
+	NULL,						/* 0x11 */
+	NULL,						/* 0x12 */
+	NULL,						/* 0x13 */
+
+	rtw_tdls,						/* 0x14 */
+	rtw_tdls_get,					/* 0x15 */
+
+#if defined(CONFIG_RTW_80211K) //|| defined(CONFIG_RTW_FSM_RRM) TODO
+	rtw_wx_priv_rrm,				/* 0x17 */
+#else
+	rtw_wx_priv_null,				/* 0x17 */
+#endif
+	NULL,						/* 0x18 */
+	rtw_wx_priv_null,				/* 0x19 */
+#ifdef CONFIG_MP_INCLUDED
+	rtw_wx_priv_null,				/* 0x1A */
+	rtw_wx_priv_null,				/* 0x1B */
+#else
+	rtw_wx_priv_null,				/* 0x1A */
+	/*rtw_mp_efuse_get,*/				/* 0x1B */
+#endif
+	NULL,							/* 0x1C is reserved for hostapd */
+	rtw_test,						/* 0x1D */
+};
+
+#if WIRELESS_EXT >= 17
+static struct iw_statistics *rtw_get_wireless_stats(struct net_device *dev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct iw_statistics *piwstats = &padapter->iwstats;
+	int tmp_level = 0;
+	int tmp_qual = 0;
+	int tmp_noise = 0;
+
+	if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) != _TRUE) {
+		piwstats->qual.qual = 0;
+		piwstats->qual.level = 0;
+		piwstats->qual.noise = 0;
+		/* RTW_INFO("No link  level:%d, qual:%d, noise:%d\n", tmp_level, tmp_qual, tmp_noise); */
+	} else {
+#ifdef CONFIG_SIGNAL_DISPLAY_DBM
+		tmp_level = rtw_phl_rssi_to_dbm(padapter->recvinfo.signal_strength);
+#else
+		tmp_level = padapter->recvinfo.signal_strength;
+#endif
+
+		tmp_qual = padapter->recvinfo.signal_qual;
+		/* RTW_INFO("level:%d, qual:%d, noise:%d, rssi (%d)\n", tmp_level, tmp_qual, tmp_noise,padapter->recvinfo.rssi); */
+
+		piwstats->qual.level = tmp_level;
+		piwstats->qual.qual = tmp_qual;
+		piwstats->qual.noise = tmp_noise;
+	}
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 14))
+	piwstats->qual.updated = IW_QUAL_ALL_UPDATED ;/* |IW_QUAL_DBM; */
+#else
+#ifdef RTK_DMP_PLATFORM
+	/* IW_QUAL_DBM= 0x8, if driver use this flag, wireless extension will show value of dbm. */
+	/* remove this flag for show percentage 0~100 */
+	piwstats->qual.updated = 0x07;
+#else
+	piwstats->qual.updated = 0x0f;
+#endif
+#endif
+
+#ifdef CONFIG_SIGNAL_DISPLAY_DBM
+	piwstats->qual.updated = piwstats->qual.updated | IW_QUAL_DBM;
+#endif
+
+	return &padapter->iwstats;
+}
+#endif
+
+#ifdef CONFIG_WIRELESS_EXT
+struct iw_handler_def rtw_handlers_def = {
+	.standard = rtw_handlers,
+	.num_standard = sizeof(rtw_handlers) / sizeof(iw_handler),
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33)) || defined(CONFIG_WEXT_PRIV)
+	.private = rtw_private_handler,
+	.private_args = (struct iw_priv_args *)rtw_private_args,
+	.num_private = sizeof(rtw_private_handler) / sizeof(iw_handler),
+	.num_private_args = sizeof(rtw_private_args) / sizeof(struct iw_priv_args),
+#endif
+#if WIRELESS_EXT >= 17
+	.get_wireless_stats = rtw_get_wireless_stats,
+#endif
+};
+#endif
+
+/* copy from net/wireless/wext.c start
+ * ----------------------------------------------------------------
+ *
+ * Calculate size of private arguments
+ */
+static const char iw_priv_type_size[] = {
+	0,                              /* IW_PRIV_TYPE_NONE */
+	1,                              /* IW_PRIV_TYPE_BYTE */
+	1,                              /* IW_PRIV_TYPE_CHAR */
+	0,                              /* Not defined */
+	sizeof(__u32),                  /* IW_PRIV_TYPE_INT */
+	sizeof(struct iw_freq),         /* IW_PRIV_TYPE_FLOAT */
+	sizeof(struct sockaddr),        /* IW_PRIV_TYPE_ADDR */
+	0,                              /* Not defined */
+};
+
+static int get_priv_size(__u16 args)
+{
+	int num = args & IW_PRIV_SIZE_MASK;
+	int type = (args & IW_PRIV_TYPE_MASK) >> 12;
+
+	return num * iw_priv_type_size[type];
+}
+/* copy from net/wireless/wext.c end */
+
+
+static int _rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_data)
+{
+	int err = 0;
+	u8 *input = NULL;
+	u32 input_len = 0;
+	const char delim[] = " ";
+	u8 *output = NULL;
+	u32 output_len = 0;
+	u32 count = 0;
+	u8 *buffer = NULL;
+	u32 buffer_len = 0;
+	char *ptr = NULL;
+	u8 cmdname[17] = {0}; /* IFNAMSIZ+1 */
+	u32 cmdlen;
+	s32 len;
+	u8 *extra = NULL;
+	u32 extra_size = 0;
+
+	s32 k;
+	const iw_handler *priv;		/* Private ioctl */
+	const struct iw_priv_args *priv_args;	/* Private ioctl description */
+	const struct iw_priv_args *mp_priv_args;	/*MP Private ioctl description */
+	const struct iw_priv_args *sel_priv_args;	/*Selected Private ioctl description */
+	u32 num_priv;				/* Number of ioctl */
+	u32 num_priv_args;			/* Number of descriptions */
+	u32 num_mp_priv_args;			/*Number of MP descriptions */
+	u32 num_sel_priv_args;			/*Number of Selected descriptions */
+	iw_handler handler;
+	int temp;
+	int subcmd = 0;				/* sub-ioctl index */
+	int offset = 0;				/* Space for sub-ioctl index */
+
+	union iwreq_data wdata;
+
+	_rtw_memcpy(&wdata, wrq_data, sizeof(wdata));
+
+	input_len = wdata.data.length;
+	if (!input_len)
+		return -EINVAL;
+	input = rtw_zmalloc(input_len);
+
+	if (input == NULL) {
+		err = -EOPNOTSUPP;
+		goto exit;
+	}
+
+	if (copy_from_user(input, wdata.data.pointer, input_len)) {
+		err = -EFAULT;
+		goto exit;
+	}
+	input[input_len - 1] = '\0';
+	ptr = input;
+	len = input_len;
+
+	sscanf(ptr, "%16s", cmdname);
+	cmdlen = strlen(cmdname);
+	RTW_DBG("%s: cmd=%s\n", __func__, cmdname);
+
+	/* skip command string */
+	if (cmdlen > 0)
+		cmdlen += 1; /* skip one space */
+	ptr += cmdlen;
+	len -= cmdlen;
+	RTW_DBG("%s: parameters=%s\n", __func__, ptr);
+
+	priv = rtw_private_handler;
+	priv_args = rtw_private_args;
+	mp_priv_args = rtw_mp_private_args;
+	num_priv = sizeof(rtw_private_handler) / sizeof(iw_handler);
+	num_priv_args = sizeof(rtw_private_args) / sizeof(struct iw_priv_args);
+	num_mp_priv_args = sizeof(rtw_mp_private_args) / sizeof(struct iw_priv_args);
+
+	if (num_priv_args == 0) {
+		err = -EOPNOTSUPP;
+		goto exit;
+	}
+
+	/* Search the correct ioctl */
+	k = -1;
+	sel_priv_args = priv_args;
+	num_sel_priv_args = num_priv_args;
+	while
+	((++k < num_sel_priv_args) && strcmp(sel_priv_args[k].name, cmdname))
+		;
+
+	/* If not found... */
+	if (k == num_sel_priv_args) {
+		k = -1;
+		sel_priv_args = mp_priv_args;
+		num_sel_priv_args = num_mp_priv_args;
+		while
+		((++k < num_sel_priv_args) && strcmp(sel_priv_args[k].name, cmdname))
+			;
+
+		if (k == num_sel_priv_args) {
+			err = -EOPNOTSUPP;
+			goto exit;
+		}
+	}
+
+	/* Watch out for sub-ioctls ! */
+	if (sel_priv_args[k].cmd < SIOCDEVPRIVATE) {
+		int j = -1;
+
+		/* Find the matching *real* ioctl */
+		while ((++j < num_priv_args) && ((priv_args[j].name[0] != '\0') ||
+			 (priv_args[j].set_args != sel_priv_args[k].set_args) ||
+			 (priv_args[j].get_args != sel_priv_args[k].get_args)))
+			;
+
+		/* If not found... */
+		if (j == num_priv_args) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		/* Save sub-ioctl number */
+		subcmd = sel_priv_args[k].cmd;
+		/* Reserve one int (simplify alignment issues) */
+		offset = sizeof(__u32);
+		/* Use real ioctl definition from now on */
+		k = j;
+	}
+
+	buffer = rtw_zmalloc(4096);
+	if (NULL == buffer) {
+		err = -ENOMEM;
+		goto exit;
+	}
+
+	if (k >= num_priv_args) {
+		err = -EINVAL;
+		goto exit;
+	}
+
+	/* If we have to set some data */
+	if ((priv_args[k].set_args & IW_PRIV_TYPE_MASK) &&
+	    (priv_args[k].set_args & IW_PRIV_SIZE_MASK)) {
+		u8 *str;
+
+		switch (priv_args[k].set_args & IW_PRIV_TYPE_MASK) {
+		case IW_PRIV_TYPE_BYTE:
+			/* Fetch args */
+			count = 0;
+			do {
+				str = strsep(&ptr, delim);
+				if (NULL == str)
+					break;
+				sscanf(str, "%i", &temp);
+				buffer[count++] = (u8)temp;
+			} while (1);
+			buffer_len = count;
+
+			/* Number of args to fetch */
+			wdata.data.length = count;
+			if (wdata.data.length > (priv_args[k].set_args & IW_PRIV_SIZE_MASK))
+				wdata.data.length = priv_args[k].set_args & IW_PRIV_SIZE_MASK;
+
+			break;
+
+		case IW_PRIV_TYPE_INT:
+			/* Fetch args */
+			count = 0;
+			do {
+				str = strsep(&ptr, delim);
+				if (NULL == str)
+					break;
+				sscanf(str, "%i", &temp);
+				((s32 *)buffer)[count++] = (s32)temp;
+			} while (1);
+			buffer_len = count * sizeof(s32);
+
+			/* Number of args to fetch */
+			wdata.data.length = count;
+			if (wdata.data.length > (priv_args[k].set_args & IW_PRIV_SIZE_MASK))
+				wdata.data.length = priv_args[k].set_args & IW_PRIV_SIZE_MASK;
+
+			break;
+
+		case IW_PRIV_TYPE_CHAR:
+			if (len > 0) {
+				/* Size of the string to fetch */
+				wdata.data.length = len;
+				if (wdata.data.length > (priv_args[k].set_args & IW_PRIV_SIZE_MASK))
+					wdata.data.length = priv_args[k].set_args & IW_PRIV_SIZE_MASK;
+
+				/* Fetch string */
+				_rtw_memcpy(buffer, ptr, wdata.data.length);
+			} else {
+				wdata.data.length = 1;
+				buffer[0] = '\0';
+			}
+			buffer_len = wdata.data.length;
+			break;
+
+		default:
+			RTW_INFO("%s: Not yet implemented...\n", __func__);
+			err = -1;
+			goto exit;
+		}
+
+		if ((priv_args[k].set_args & IW_PRIV_SIZE_FIXED) &&
+		    (wdata.data.length != (priv_args[k].set_args & IW_PRIV_SIZE_MASK))) {
+			RTW_INFO("%s: The command %s needs exactly %d argument(s)...\n",
+				__func__, cmdname, priv_args[k].set_args & IW_PRIV_SIZE_MASK);
+			err = -EINVAL;
+			goto exit;
+		}
+	}   /* if args to set */
+	else
+		wdata.data.length = 0L;
+
+	/* Those two tests are important. They define how the driver
+	* will have to handle the data */
+	if ((priv_args[k].set_args & IW_PRIV_SIZE_FIXED) &&
+	    ((get_priv_size(priv_args[k].set_args) + offset) <= IFNAMSIZ)) {
+		/* First case : all SET args fit within wrq */
+		if (offset)
+			wdata.mode = subcmd;
+		_rtw_memcpy(wdata.name + offset, buffer, IFNAMSIZ - offset);
+	} else {
+		if ((priv_args[k].set_args == 0) &&
+		    (priv_args[k].get_args & IW_PRIV_SIZE_FIXED) &&
+		    (get_priv_size(priv_args[k].get_args) <= IFNAMSIZ)) {
+			/* Second case : no SET args, GET args fit within wrq */
+			if (offset)
+				wdata.mode = subcmd;
+		} else {
+			/* Third case : args won't fit in wrq, or variable number of args */
+			if (copy_to_user(wdata.data.pointer, buffer, buffer_len)) {
+				err = -EFAULT;
+				goto exit;
+			}
+			wdata.data.flags = subcmd;
+		}
+	}
+
+	rtw_mfree(input, input_len);
+	input = NULL;
+
+	extra_size = 0;
+	if (IW_IS_SET(priv_args[k].cmd)) {
+		/* Size of set arguments */
+		extra_size = get_priv_size(priv_args[k].set_args);
+
+		/* Does it fits in iwr ? */
+		if ((priv_args[k].set_args & IW_PRIV_SIZE_FIXED) &&
+		    ((extra_size + offset) <= IFNAMSIZ))
+			extra_size = 0;
+	} else {
+		/* Size of get arguments */
+		extra_size = get_priv_size(priv_args[k].get_args);
+
+		/* Does it fits in iwr ? */
+		if ((priv_args[k].get_args & IW_PRIV_SIZE_FIXED) &&
+		    (extra_size <= IFNAMSIZ))
+			extra_size = 0;
+	}
+
+	if (extra_size == 0) {
+		extra = (u8 *)&wdata;
+		rtw_mfree(buffer, 4096);
+		buffer = NULL;
+	} else
+		extra = buffer;
+
+	handler = priv[priv_args[k].cmd - SIOCIWFIRSTPRIV];
+	err = handler(dev, NULL, &wdata, extra);
+
+	/* If we have to get some data */
+	if ((priv_args[k].get_args & IW_PRIV_TYPE_MASK) &&
+	    (priv_args[k].get_args & IW_PRIV_SIZE_MASK)) {
+		int j;
+		int n = 0;	/* number of args */
+		u8 str[20] = {0};
+
+		/* Check where is the returned data */
+		if ((priv_args[k].get_args & IW_PRIV_SIZE_FIXED) &&
+		    (get_priv_size(priv_args[k].get_args) <= IFNAMSIZ))
+			n = priv_args[k].get_args & IW_PRIV_SIZE_MASK;
+		else
+			n = wdata.data.length;
+
+		output = rtw_zmalloc(4096);
+		if (NULL == output) {
+			err =  -ENOMEM;
+			goto exit;
+		}
+
+		switch (priv_args[k].get_args & IW_PRIV_TYPE_MASK) {
+		case IW_PRIV_TYPE_BYTE:
+			/* Display args */
+			for (j = 0; j < n; j++) {
+				sprintf(str, "%d  ", extra[j]);
+				len = strlen(str);
+				output_len = strlen(output);
+				if ((output_len + len + 1) > 4096) {
+					err = -E2BIG;
+					goto exit;
+				}
+				_rtw_memcpy(output + output_len, str, len);
+			}
+			break;
+
+		case IW_PRIV_TYPE_INT:
+			/* Display args */
+			for (j = 0; j < n; j++) {
+				sprintf(str, "%d  ", ((__s32 *)extra)[j]);
+				len = strlen(str);
+				output_len = strlen(output);
+				if ((output_len + len + 1) > 4096) {
+					err = -E2BIG;
+					goto exit;
+				}
+				_rtw_memcpy(output + output_len, str, len);
+			}
+			break;
+
+		case IW_PRIV_TYPE_CHAR:
+			/* Display args */
+			_rtw_memcpy(output, extra, n);
+			output_len = n;
+			break;
+
+		default:
+			RTW_INFO("%s: Not yet implemented...\n", __func__);
+			err = -1;
+			goto exit;
+		}
+
+		output_len ++;
+		wrq_data->data.length = output_len;
+		if (copy_to_user(wrq_data->data.pointer, output, output_len)) {
+			err = -EFAULT;
+			goto exit;
+		}
+	}   /* if args to set */
+	else
+		wrq_data->data.length = 0;
+
+exit:
+	if (input)
+		rtw_mfree(input, input_len);
+	if (buffer)
+		rtw_mfree(buffer, 4096);
+	if (output)
+		rtw_mfree(output, 4096);
+
+	return err;
+}
+
+#ifdef CONFIG_COMPAT
+static int rtw_ioctl_compat_wext_private(struct net_device *dev, struct ifreq *rq)
+{
+	struct compat_iw_point iwp_compat;
+	union iwreq_data wrq_data;
+	int err = 0;
+	RTW_DBG("%s:...\n", __func__);
+	if (copy_from_user(&iwp_compat, rq->ifr_ifru.ifru_data, sizeof(struct compat_iw_point)))
+		return -EFAULT;
+
+	wrq_data.data.pointer = compat_ptr(iwp_compat.pointer);
+	wrq_data.data.length = iwp_compat.length;
+	wrq_data.data.flags = iwp_compat.flags;
+
+	err = _rtw_ioctl_wext_private(dev, &wrq_data);
+
+	iwp_compat.pointer = ptr_to_compat(wrq_data.data.pointer);
+	iwp_compat.length = wrq_data.data.length;
+	iwp_compat.flags = wrq_data.data.flags;
+	if (copy_to_user(rq->ifr_ifru.ifru_data, &iwp_compat, sizeof(struct compat_iw_point)))
+		return -EFAULT;
+
+	return err;
+}
+#endif /* CONFIG_COMPAT */
+
+static int rtw_ioctl_standard_wext_private(struct net_device *dev, struct ifreq *rq)
+{
+	struct iw_point *iwp;
+	union iwreq_data wrq_data;
+	int err = 0;
+	iwp = &wrq_data.data;
+	RTW_DBG("%s:...\n", __func__);
+	if (copy_from_user(iwp, rq->ifr_ifru.ifru_data, sizeof(struct iw_point)))
+		return -EFAULT;
+
+	err = _rtw_ioctl_wext_private(dev, &wrq_data);
+
+	if (copy_to_user(rq->ifr_ifru.ifru_data, iwp, sizeof(struct iw_point)))
+		return -EFAULT;
+
+	return err;
+}
+
+static int rtw_ioctl_wext_private(struct net_device *dev, struct ifreq *rq)
+{
+#ifdef CONFIG_COMPAT
+#if (KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE)
+	if (is_compat_task())
+#else
+	if (in_compat_syscall())
+#endif
+		return rtw_ioctl_compat_wext_private(dev, rq);
+	else
+#endif /* CONFIG_COMPAT */
+		return rtw_ioctl_standard_wext_private(dev, rq);
+}
+
+int rtw_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
+{
+	struct iwreq *wrq = (struct iwreq *)rq;
+	int ret = 0;
+
+	switch (cmd) {
+	case RTL_IOCTL_WPA_SUPPLICANT:
+		ret = wpa_supplicant_ioctl(dev, &wrq->u.data);
+		break;
+#ifdef CONFIG_AP_MODE
+	case RTL_IOCTL_HOSTAPD:
+		ret = rtw_hostapd_ioctl(dev, &wrq->u.data);
+		break;
+#ifdef CONFIG_WIRELESS_EXT
+	case SIOCSIWMODE:
+		ret = rtw_wx_set_mode(dev, NULL, &wrq->u, NULL);
+		break;
+#endif
+#endif /* CONFIG_AP_MODE */
+	case SIOCDEVPRIVATE:
+		ret = rtw_ioctl_wext_private(dev, rq);
+		break;
+	case (SIOCDEVPRIVATE+1):
+		ret = rtw_android_priv_cmd(dev, rq, cmd);
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+
+	return ret;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_mp.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_mp.c
new file mode 100644
index 000000000000..d95cb411ae77
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/ioctl_mp.c
@@ -0,0 +1,4298 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#if defined(CONFIG_MP_INCLUDED)
+
+#include <drv_types.h>
+#include <rtw_mp.h>
+
+u8 rtw_do_mp_iwdata_len_chk(const char *caller, u32 len)
+{
+	u8 is_illegal = _FALSE;
+	if (len >= RTW_IWD_MAX_LEN) {
+		RTW_ERR("%s : iw data len(%u) > RTW_IWD_MAX_LEN(%u)",
+			caller, len, RTW_IWD_MAX_LEN);
+		is_illegal = _TRUE;
+	}
+	return is_illegal;
+}
+
+/*
+ * Input Format: %s,%d,%d
+ *	%s is width, could be
+ *		"b" for 1 byte
+ *		"w" for WORD (2 bytes)
+ *		"dw" for DWORD (4 bytes)
+ *	1st %d is address(offset)
+ *	2st %d is data to write
+ */
+
+int rtw_mp_write_reg(struct net_device *dev,
+		     struct iw_request_info *info,
+		     struct iw_point *wrqu, char *extra)
+{
+	char *pch, *pnext;
+	char *width_str;
+	char width;
+	u32 addr, data;
+	int ret;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	char input[RTW_IWD_MAX_LEN];
+	struct rtw_mp_reg_arg	reg_arg;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	pch = input;
+
+	pnext = strpbrk(pch, " ,.-");
+	if (pnext == NULL)
+		return -EINVAL;
+	*pnext = 0;
+	width_str = pch;
+
+	pch = pnext + 1;
+	pnext = strpbrk(pch, " ,.-");
+	if (pnext == NULL)
+		return -EINVAL;
+	*pnext = 0;
+
+	ret = sscanf(pch, "%x", &addr);
+
+	pch = pnext + 1;
+	pnext = strpbrk(pch, " ,.-");
+	if ((pch - input) >= wrqu->length)
+		return -EINVAL;
+
+	ret = sscanf(pch, "%x", &data);
+	RTW_INFO("data=%x,addr=%x\n", (u32)data, (u32)addr);
+	ret = 0;
+	width = width_str[0];
+	switch (width) {
+	case 'b':
+		/* 1 byte*/
+		if (data > 0xFF) {
+			ret = -EINVAL;
+			break;
+		}
+		reg_arg.io_offset = addr;
+		reg_arg.io_type = 1;
+		reg_arg.io_value = (u8)data;
+
+		if (rtw_mp_phl_reg(padapter, &reg_arg, RTW_MP_REG_CMD_WRITE_BB))
+			RTW_INFO("write data=%x,addr=%x OK\n", (u8)data, addr);
+		else
+			RTW_INFO("write data=%x,addr=%x fail\n", (u8)data, addr);
+		break;
+	case 'w':
+		/* 2 bytes*/
+		if (data > 0xFFFF) {
+			ret = -EINVAL;
+			break;
+		}
+		reg_arg.io_offset = addr;
+		reg_arg.io_type = 2;
+		reg_arg.io_value = (u16)data;
+
+		if (rtw_mp_phl_reg(padapter, &reg_arg, RTW_MP_REG_CMD_WRITE_BB))
+			RTW_INFO("write data=%x,addr=%x OK\n", (u16)data, addr);
+		else
+			RTW_INFO("write data=%x,addr=%x fail\n", (u16)data, addr);
+		break;
+	case 'd':
+		/* 4 bytes*/
+		reg_arg.io_offset = addr;
+		reg_arg.io_type = 4;
+		reg_arg.io_value = data;
+
+		if (rtw_mp_phl_reg(padapter, &reg_arg, RTW_MP_REG_CMD_WRITE_BB))
+			RTW_INFO("write data=%x,addr=%x OK\n", data, addr);
+		else
+			RTW_INFO("write data=%x,addr=%x fail\n", data, addr);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+
+/*
+ * Input Format: %s,%d
+ *	%s is width, could be
+ *		"b" for 1 byte
+ *		"w" for WORD (2 bytes)
+ *		"dw" for DWORD (4 bytes)
+ *	%d is address(offset)
+ *
+ * Return:
+ *	%d for data readed
+ */
+int rtw_mp_read_reg(struct net_device *dev,
+		    struct iw_request_info *info,
+		    struct iw_point *wrqu, char *extra)
+{
+	char input[RTW_IWD_MAX_LEN];
+	char *pch, *pnext;
+	char *width_str;
+	char width;
+	char data[20], tmp[20];
+	u32 addr = 0, strtout = 0;
+	u32 i = 0, j = 0, ret = 0, data32 = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_mp_reg_arg	reg_arg;
+	char *pextra = extra;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (wrqu->length > 128)
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	_rtw_memset(extra, 0, wrqu->length);
+	_rtw_memset(data, '\0', sizeof(data));
+	_rtw_memset(tmp, '\0', sizeof(tmp));
+	pch = input;
+	pnext = strpbrk(pch, " ,.-");
+	if (pnext == NULL)
+		return -EINVAL;
+	*pnext = 0;
+	width_str = pch;
+
+	pch = pnext + 1;
+
+	ret = sscanf(pch, "%x", &addr);
+
+	ret = 0;
+	width = width_str[0];
+
+	switch (width) {
+	case 'b':
+		reg_arg.io_offset = addr;
+		reg_arg.io_type = 1;
+
+		if (rtw_mp_phl_reg(padapter, &reg_arg, RTW_MP_REG_CMD_READ_BB)) {
+			data32 = reg_arg.io_value; //rtw_phl_read8(dvobj->phl, addr);
+			RTW_INFO("reg=%x\n", data32);
+			sprintf(extra, "%d", data32);
+		} else
+			sprintf(extra, "reg io fail\n");
+
+		wrqu->length = strlen(extra);
+		break;
+	case 'w':
+		/* 2 bytes*/
+		reg_arg.io_offset = addr;
+		reg_arg.io_type = 2;
+
+		if (rtw_mp_phl_reg(padapter, &reg_arg, RTW_MP_REG_CMD_READ_BB)) {
+			sprintf(data, "%04x", reg_arg.io_value);
+			RTW_INFO("reg=%s\n", data);
+		} else
+			sprintf(extra, "reg io fail\n");
+
+		for (i = 0 ; i <= strlen(data) ; i++) {
+			if (i % 2 == 0) {
+				tmp[j] = ' ';
+				j++;
+			}
+			if (data[i] != '\0')
+				tmp[j] = data[i];
+
+			j++;
+		}
+		tmp[j]='\0';
+
+		pch = tmp;
+		RTW_INFO("pch=%s", pch);
+
+		while (*pch != '\0') {
+			pnext = strpbrk(pch, " ");
+			if (!pnext || ((pnext - tmp) > 4))
+				break;
+
+			pnext++;
+			if (*pnext != '\0') {
+				/*strtout = simple_strtoul(pnext , &ptmp, 16);*/
+				ret = sscanf(pnext, "%x", &strtout);
+				pextra += sprintf(pextra, " %d", strtout);
+			} else
+				break;
+			pch = pnext;
+		}
+		wrqu->length = strlen(extra);
+		break;
+	case 'd':
+		/* 4 bytes */
+		reg_arg.io_offset = addr;
+		reg_arg.io_type = 4;
+
+		if (rtw_mp_phl_reg(padapter, &reg_arg, RTW_MP_REG_CMD_READ_BB)) {
+			sprintf(data, "%08x", reg_arg.io_value);
+			RTW_INFO("reg=%s\n", data);
+		} else
+			sprintf(extra, "reg io fail\n");
+
+		/*add read data format blank*/
+		for (i = 0 ; i <= strlen(data) ; i++) {
+			if (i % 2 == 0) {
+				tmp[j] = ' ';
+				j++;
+			}
+			if (data[i] != '\0')
+				tmp[j] = data[i];
+
+			j++;
+		}
+
+		pch = tmp;
+		RTW_INFO("pch=%s", pch);
+
+		while (*pch != '\0') {
+			pnext = strpbrk(pch, " ");
+			if (!pnext)
+				break;
+
+			pnext++;
+			if (*pnext != '\0') {
+				ret = sscanf(pnext, "%x", &strtout);
+				pextra += sprintf(pextra, " %d", strtout);
+			} else
+				break;
+			pch = pnext;
+		}
+		wrqu->length = strlen(extra);
+		break;
+
+	default:
+		wrqu->length = 0;
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+
+/*
+ * Input Format: %d,%x,%x
+ *	%d is RF path, should be smaller than MAX_RF_PATH_NUMS
+ *	1st %x is address(offset)
+ *	2st %x is data to write
+ */
+int rtw_mp_write_rf(struct net_device *dev,
+		    struct iw_request_info *info,
+		    struct iw_point *wrqu, char *extra)
+{
+
+	u32 path, addr, data;
+	int ret;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	char input[RTW_IWD_MAX_LEN];
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, wrqu->length);
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+
+	ret = sscanf(input, "%d,%x,%x", &path, &addr, &data);
+	if (ret < 3)
+		return -EINVAL;
+
+	if (path >= GET_HAL_RFPATH_NUM(adapter_to_dvobj(padapter)))
+		return -EINVAL;
+	if (addr > 0xFF)
+		return -EINVAL;
+	if (data > 0xFFFFF)
+		return -EINVAL;
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	rtw_phl_write_rfreg(GET_PHL_INFO(dvobj), path, addr, 0xFFFFF, data);
+
+	sprintf(extra, "write_rf completed\n");
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+
+/*
+ * Input Format: %d,%x
+ *	%d is RF path, should be smaller than MAX_RF_PATH_NUMS
+ *	%x is address(offset)
+ *
+ * Return:
+ *	%d for data readed
+ */
+int rtw_mp_read_rf(struct net_device *dev,
+		   struct iw_request_info *info,
+		   struct iw_point *wrqu, char *extra)
+{
+	char input[RTW_IWD_MAX_LEN];
+	char *pch, *pnext;
+	char data[20], tmp[20];
+	u32 path, addr, strtou;
+	u32 ret, i = 0 , j = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	char *pextra = extra;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (wrqu->length > 128)
+		return -EFAULT;
+	_rtw_memset(input, 0, wrqu->length);
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	ret = sscanf(input, "%d,%x", &path, &addr);
+	if (ret < 2)
+		return -EINVAL;
+
+	if (path >= GET_HAL_RFPATH_NUM(adapter_to_dvobj(padapter)))
+		return -EINVAL;
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	sprintf(data, "%08x",  rtw_phl_read_rfreg(GET_PHL_INFO(dvobj), path, addr , 0xFFFFF));
+	/*add read data format blank*/
+	for (i = 0 ; i <= strlen(data) ; i++) {
+		if (i % 2 == 0) {
+			tmp[j] = ' ';
+			j++;
+		}
+		tmp[j] = data[i];
+		j++;
+	}
+	pch = tmp;
+	RTW_INFO("pch=%s", pch);
+
+	while (*pch != '\0') {
+		pnext = strpbrk(pch, " ");
+		if (!pnext)
+			break;
+		pnext++;
+		if (*pnext != '\0') {
+			/*strtou =simple_strtoul(pnext , &ptmp, 16);*/
+			ret = sscanf(pnext, "%x", &strtou);
+			pextra += sprintf(pextra, " %d", strtou);
+		} else
+			break;
+		pch = pnext;
+	}
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+
+int rtw_mp_start(struct net_device *dev,
+		 struct iw_request_info *info,
+		 struct iw_point *wrqu, char *extra)
+{
+	int ret = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	char *pextra = NULL;
+
+	pmppriv->bprocess_mp_mode = _TRUE;
+
+	if (pmppriv->mode == MP_ON) {
+		sprintf(extra, "Already mp_start\n");
+		wrqu->length = strlen(extra);
+		return ret;
+	}
+
+	if (!rtw_is_adapter_up(padapter)) {
+		RTW_INFO("adapter_up fail !!!\n");
+		return -EPERM;
+	}
+
+	rtw_set_scan_deny(padapter, 5000);
+	rtw_mi_scan_abort(padapter, _TRUE);
+
+	if (rtw_mp_cmd(padapter, MP_START, RTW_CMDF_DIRECTLY) != _SUCCESS)
+		ret = -EPERM;
+
+	_rtw_memset(extra, 0, wrqu->length);
+	pextra = extra;
+	pextra += sprintf(extra, "mp_start %s\n", ret == 0 ? "ok" : "fail");
+	if (rtw_efuse_read_map2shadow(padapter, RTW_EFUSE_WIFI) == _SUCCESS){
+		pextra += sprintf(pextra, "EFUSE:%s\n",
+			RTW_EFUSE_FROM2STR(rtw_efuse_get_map_from(padapter)));
+	}
+
+	wrqu->length = strlen(extra);
+
+	return ret;
+}
+
+int rtw_mp_stop(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra)
+{
+	int ret = 0;
+	u8 status = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmppriv = &padapter->mppriv;
+
+
+	if (rtw_mp_cmd(padapter, MP_STOP, RTW_CMDF_DIRECTLY) != _SUCCESS)
+		ret = -EPERM;
+
+	if (pmppriv->mode != MP_OFF)
+		return -EPERM;
+
+	_rtw_memset(extra, 0, wrqu->length);
+	sprintf(extra, "mp_stop ok\n");
+	wrqu->length = strlen(extra);
+
+	pmppriv->bprocess_mp_mode = _FALSE;
+
+	return ret;
+}
+
+
+int rtw_mp_rate(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra)
+{
+	u16 rate = MPT_RATE_1M;
+	u8		input[RTW_IWD_MAX_LEN];
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	PMPT_CONTEXT		pMptCtx = &(padapter->mppriv.mpt_ctx);
+	u8 tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	char *pextra = extra;
+	u8 path_i = 0, i = 0;
+	u16 pwr_dbm = 0;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	rate = rtw_mp_rate_parse(padapter, input);
+	padapter->mppriv.rateidx = rate;
+
+	if (rate == 0 && strcmp(input, "1M") != 0) {
+		rate = rtw_atoi(input);
+#ifndef CONFIG_80211AX_HE
+		padapter->mppriv.rateidx = mrate_to_hwrate(rate);
+#endif
+		/*if (rate <= 0x7f)
+			rate = wifirate2_ratetbl_inx((u8)rate);
+		else if (rate < 0xC8)
+			rate = (rate - 0x79 + MPT_RATE_MCS0);
+		HT  rate 0x80(MCS0)  ~ 0x8F(MCS15) ~ 0x9F(MCS31) 128~159
+		VHT1SS~2SS rate 0xA0 (VHT1SS_MCS0 44) ~ 0xB3 (VHT2SS_MCS9 #63) 160~179
+		VHT rate 0xB4 (VHT3SS_MCS0 64) ~ 0xC7 (VHT2SS_MCS9 #83) 180~199
+		else
+		VHT rate 0x90(VHT1SS_MCS0) ~ 0x99(VHT1SS_MCS9) 144~153
+		rate =(rate - MPT_RATE_VHT1SS_MCS0);
+		*/
+	}
+	_rtw_memset(extra, 0, wrqu->length);
+
+	pextra += sprintf(pextra, "Set data rate to %s index %d\n" , input, padapter->mppriv.rateidx);
+	RTW_INFO("%s: %s rate index=%d\n", __func__, input, padapter->mppriv.rateidx);
+	pextra += sprintf(pextra, "PPDU Type %s\n", 
+								PPDU_TYPE_STR(pmp_priv->rtw_mp_pmact_ppdu_type));
+	pextra += sprintf(pextra, "CMD: [mp_plcp_datappdu=%%d]\nPLCP (PPDU Type):\n");
+	for (i = pmp_priv->rtw_mp_pmact_ppdu_type; i <= RTW_MP_TYPE_HE_TB; i++)
+		pextra += sprintf(pextra, "%d:%s\n", i,PPDU_TYPE_STR(i));
+
+	SetDataRate(padapter);
+#if 0
+	for (path_i = 0 ; path_i < tx_nss; path_i++) {
+		pwr_dbm = rtw_mp_get_pwrtab_dbm(padapter, path_i);
+		pextra += sprintf(pextra, "Path%d Pwrdbm:%d" , path_i, pwr_dbm);
+	}
+#endif
+	wrqu->length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_mp_channel(struct net_device *dev,
+		   struct iw_request_info *info,
+		   struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8		input[RTW_IWD_MAX_LEN];
+	int	channel = 1;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+
+	if (kstrtoint(input, 10, &channel) != 0) {
+		RTW_INFO("Failed to convert string to int\n");
+		return -EFAULT;
+	}
+
+	_rtw_memset(extra, 0, wrqu->length);
+	sprintf(extra, "Change channel %d to channel %d", pmp_priv->channel, channel);
+	pmp_priv->channel = channel;
+	RTW_INFO("%s:channel = %d\n", __func__, pmp_priv->channel);
+	SetChannel(padapter);
+
+	wrqu->length = strlen(extra);
+	return 0;
+}
+
+static void rtw_mp_update_trxsc(_adapter *padapter) {
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 trxsc_offset = pmp_priv->rtw_mp_trxsc;
+
+	switch (trxsc_offset) {
+	case 0:
+		pmp_priv->rtw_mp_data_bandwidth = pmp_priv->bandwidth;
+		RTW_INFO("%s:TRXSC %d, MP bandwidth = %d\n", __func__, trxsc_offset, pmp_priv->bandwidth);
+		break;
+	case 1:
+	case 2:
+	case 3:
+	case 4:
+		pmp_priv->rtw_mp_data_bandwidth = CHANNEL_WIDTH_20;
+		RTW_INFO("%s:TRXSC %d, MP bandwidth = %d\n", __func__, trxsc_offset, pmp_priv->bandwidth);
+		break;
+	case 9:
+	case 10:
+	case 11:
+	case 12:
+		pmp_priv->rtw_mp_data_bandwidth = CHANNEL_WIDTH_40;
+		RTW_INFO("%s:TRXSC %d, MP bandwidth = %d\n", __func__, trxsc_offset, pmp_priv->bandwidth);
+		break;
+	case 13:
+	case 14:
+		pmp_priv->rtw_mp_data_bandwidth = CHANNEL_WIDTH_40;
+		RTW_INFO("%s:TRXSC %d, MP bandwidth = %d\n", __func__, trxsc_offset, pmp_priv->bandwidth);
+		break;
+	}
+}
+
+int rtw_mp_trxsc_offset(struct net_device *dev,
+		   struct iw_request_info *info,
+		   struct iw_point *wrqu, char *extra)
+{
+
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8		input[RTW_IWD_MAX_LEN];
+	u32	trxsc_offset = 0;
+
+	_rtw_memset(input, 0, sizeof(input));
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	trxsc_offset = rtw_atoi(input);
+	RTW_INFO("%s: ch offset = %d\n", __func__, trxsc_offset);
+
+	pmp_priv->rtw_mp_trxsc = trxsc_offset;
+	rtw_mp_update_trxsc(padapter);
+	_rtw_memset(extra, 0, wrqu->length);
+	sprintf(extra, "change TRXSC to %d, current Bandwidth=%d\n", 
+						pmp_priv->rtw_mp_trxsc, pmp_priv->bandwidth);
+
+	wrqu->length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_mp_bandwidth(struct net_device *dev,
+		     struct iw_request_info *info,
+		     struct iw_point *wrqu, char *extra)
+{
+	u8 bandwidth = 0, sg = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8		input[RTW_IWD_MAX_LEN];
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	if (sscanf(input, "40M=%hhd,shortGI=%hhd", &bandwidth, &sg) > 0)
+		RTW_INFO("%s: bw=%d sg=%d\n", __func__, bandwidth , sg);
+#if 0
+	if (bandwidth == 1 && rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_40M))
+		bandwidth = CHANNEL_WIDTH_40;
+	else if (bandwidth == 2 && rtw_hw_chk_bw_cap(adapter_to_dvobj(padapter), BW_CAP_80M))
+		bandwidth = CHANNEL_WIDTH_80;
+	else
+		bandwidth = CHANNEL_WIDTH_20;
+
+	rtw_adjust_chbw(padapter, pmp_priv->channel, &bandwidth, &pmp_priv->prime_channel_offset);
+#else
+	pmp_priv->bandwidth = (u8)bandwidth;
+	pmp_priv->preamble = sg;
+	_rtw_memset(extra, 0, wrqu->length);
+	sprintf(extra, "Change BW %d to BW %d\n", pmp_priv->bandwidth , bandwidth);
+
+	SetBandwidth(padapter);
+	rtw_mp_update_trxsc(padapter);
+#endif
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_txpower_index(struct net_device *dev,
+			 struct iw_request_info *info,
+			 struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	char input[RTW_IWD_MAX_LEN];
+	u32 rfpath = 0 ;
+	u32 txpower_inx = 0, tarpowerdbm = 0;
+	char *pextra = extra;
+	u8 rf_type = GET_HAL_RFPATH(adapter_to_dvobj(padapter));
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(padapter);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 tx_nss = rtw_mp_rfpath2txnss(padapter, pmp_priv->curr_rfpath);
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	_rtw_memset(extra, 0, strlen(extra));
+
+	if (wrqu->length == 2) {
+		if (input[0] != '\0' ) {
+			rfpath = rtw_atoi(input);
+			tarpowerdbm = mpt_get_tx_power_finalabs_val(padapter, tx_nss);
+			if (tarpowerdbm > 0) {
+				pextra += sprintf(pextra, "dBm:%d.%d",
+				(tarpowerdbm / TX_POWER_BASE), rtw_mpt_raw2dec_dbm(tarpowerdbm));
+				padapter->mppriv.txpowerdbm = tarpowerdbm;
+				rtw_mp_txpower_dbm(padapter, rfpath);
+			}
+		}
+	} else {
+		u8 rfpath_i = 0;
+		u8 tx_nss = get_phy_tx_nss(padapter, adapter_link);
+
+		tarpowerdbm = mpt_get_tx_power_finalabs_val(padapter, MP_NSS1);
+		pextra += sprintf(pextra, "\n\t\t\t 1SS dBm:%d.%d",
+				(tarpowerdbm / TX_POWER_BASE), rtw_mpt_raw2dec_dbm(tarpowerdbm));
+		if (rf_type > RF_1T2R) {
+			tarpowerdbm = mpt_get_tx_power_finalabs_val(padapter, MP_NSS2);
+			pextra += sprintf(pextra, ",2SS dBm:%d.%d",
+			(tarpowerdbm / TX_POWER_BASE), rtw_mpt_raw2dec_dbm(tarpowerdbm));
+		}
+		if (rf_type > RF_2T4R) {
+			tarpowerdbm = mpt_get_tx_power_finalabs_val(padapter, MP_NSS3);
+			pextra += sprintf(pextra, ",3SS dBm:%d.%d",
+			(tarpowerdbm / TX_POWER_BASE), rtw_mpt_raw2dec_dbm(tarpowerdbm));
+		}
+		if (rf_type > RF_3T4R) {
+			tarpowerdbm = mpt_get_tx_power_finalabs_val(padapter, MP_NSS4);
+			pextra += sprintf(pextra, ",4SS dBm:%d.%d",
+				(tarpowerdbm / TX_POWER_BASE), rtw_mpt_raw2dec_dbm(tarpowerdbm));
+		}
+		padapter->mppriv.txpowerdbm = tarpowerdbm;
+		for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+			rtw_mp_txpower_dbm(padapter, rfpath_i);
+	}
+	wrqu->length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_mp_txpower(struct net_device *dev,
+		   struct iw_request_info *info,
+		   struct iw_point *wrqu, char *extra)
+{
+	u32 idx_a = 0, idx_b = 0, idx_c = 0, idx_d = 0;
+	int MsetPower = 1;
+	char pout_str_buf[7];
+	u8		input[RTW_IWD_MAX_LEN];
+	u8 rfpath_i = 0;
+	u16 agc_cw_val = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct mp_priv *pmppriv = &padapter ->mppriv;
+	u8 tx_nss = get_phy_tx_nss(padapter, padapter_link);
+	char *pextra = extra;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	_rtw_memset(extra, 0, strlen(extra));
+
+	MsetPower = strncmp(input, "off", 3);
+	if (MsetPower == 0) {
+		pmppriv->bSetTxPower = 0;
+		sprintf(pextra, "MP Set power off");
+	} else {
+		if (sscanf(input, "patha=%d,pathb=%d,pathc=%d,pathd=%d", &idx_a, &idx_b, &idx_c, &idx_d) >= 1) {
+				pextra += sprintf(pextra, "Set power offset path_A:%d path_B:%d path_C:%d path_D:%d\n", idx_a , idx_b , idx_c , idx_d);
+				pmppriv->path_pwr_offset[RF_PATH_A] = (u8)idx_a;
+				pmppriv->path_pwr_offset[RF_PATH_B] = (u8)idx_b;
+				pmppriv->path_pwr_offset[RF_PATH_C] = (u8)idx_c;
+				pmppriv->path_pwr_offset[RF_PATH_D]  = (u8)idx_d;
+				pmppriv->bSetTxPower = 1;
+		} else if (strncmp(input, "dbm", 3) == 0) {
+			u8 signed_flag = 0;
+			u8 ret = 0xff;
+			int int_num = 0;
+			u32 dec_num = 0;
+			s16 pout = 0;
+			int i;
+			u32 poutdbm = 0;
+			s32 db_temp = 0;
+			s16 pset = 0;
+			u8 rfpath;
+
+			if (sscanf(input, "dbm=%7s", pout_str_buf) == 1) {
+				ret = 0;
+			} else {
+				sprintf(extra, "[dbm = -30 ~ 30.00]");
+				goto invalid_param_format;
+			}
+
+			if(pout_str_buf[0] == '-')
+				signed_flag = 1;
+			i = sscanf(pout_str_buf, "%d.%3u", &int_num, &dec_num);
+			RTW_INFO("%s: pars input =%d.%d\n", __func__, int_num, dec_num);
+
+			if(i == 2)
+				dec_num = (dec_num < 10) ? dec_num * 10 : dec_num;
+
+			if (int_num >= 30 || ret == 0xff || dec_num > 99 || (dec_num % 25 != 0)) {
+				sprintf(extra, "CMD Format:[dbm= -30.00 ~ 30.00]\n"
+					" each scale step value must 0.25 or -0.25\n");
+				goto invalid_param_format;
+			}
+
+			pset = int_num * TX_POWER_BASE + ((dec_num * TX_POWER_BASE) / 100);
+			RTW_INFO("%s: pset=%d\n", __func__, pset);
+			pset = ((pset < 0 || signed_flag == 1) ? -pset : pset);
+
+
+			pextra += sprintf(pextra, "Set power dbm :%d.%d\n", int_num, dec_num);
+			pmppriv->txpowerdbm = pset;
+			pmppriv->bSetTxPower = 1;
+		} else {
+			pextra += sprintf(pextra, "Invalid format on line %s\n", input);
+			RTW_INFO("Invalid format on line %s\n", input );
+			wrqu->length = strlen(extra);
+			return 0;
+		}
+
+		for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++) {
+				agc_cw_val = rtw_mp_txpower_dbm(padapter, rfpath_i);
+				pextra += sprintf(pextra, "Path:%d PwrAGC:%d\n", rfpath_i,agc_cw_val);
+		}
+	}
+
+invalid_param_format:
+	wrqu->length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_mp_ant_tx(struct net_device *dev,
+		  struct iw_request_info *info,
+		  struct iw_point *wrqu, char *extra)
+{
+	u8 i;
+	u8		input[RTW_IWD_MAX_LEN];
+	u8 antenna = 0;
+	u16 pwr_dbm = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmppriv = &padapter ->mppriv;
+	char *pextra = extra;
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	pextra += sprintf(pextra, "switch Tx antenna to %s\n", input);
+
+	for (i = 0; i < strlen(input); i++) {
+		switch (input[i]) {
+		case 'a':
+			antenna |= MP_ANTENNA_A;
+			break;
+		case 'b':
+			antenna |= MP_ANTENNA_B;
+			break;
+		case 'c':
+			antenna |= MP_ANTENNA_C;
+			break;
+		case 'd':
+			antenna |= MP_ANTENNA_D;
+			break;
+		}
+	}
+	/*antenna |= BIT(extra[i]-'a');*/
+	RTW_INFO("%s: antenna=0x%x\n", __func__, antenna);
+	pmppriv->antenna_trx = antenna;
+
+	SetAntenna(padapter);
+	if(registry_par->mp_mode == 1) {
+		u8 tx_nss = rtw_mp_rfpath2txnss(padapter, pmppriv->curr_rfpath);
+		pwr_dbm = mpt_get_tx_power_finalabs_val(padapter, tx_nss);
+		if ( pwr_dbm > 0) {
+			padapter->mppriv.txpowerdbm = pwr_dbm;
+			pextra += sprintf(pextra, "read pwr dbm:%d.%d",
+			(pwr_dbm / TX_POWER_BASE), rtw_mpt_raw2dec_dbm(pwr_dbm));
+		}
+	}
+	wrqu->length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_mp_ant_rx(struct net_device *dev,
+		  struct iw_request_info *info,
+		  struct iw_point *wrqu, char *extra)
+{
+	u8 i;
+	u16 antenna = 0;
+	u8		input[RTW_IWD_MAX_LEN];
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	/*RTW_INFO("%s: input=%s\n", __func__, input);*/
+	_rtw_memset(extra, 0, wrqu->length);
+
+	sprintf(extra, "switch Rx antenna to %s", input);
+	for (i = 0; i < strlen(input); i++) {
+		switch (input[i]) {
+		case 'a':
+			antenna |= MP_ANTENNA_A;
+			break;
+		case 'b':
+			antenna |= MP_ANTENNA_B;
+			break;
+		case 'c':
+			antenna |= MP_ANTENNA_C;
+			break;
+		case 'd':
+			antenna |= MP_ANTENNA_D;
+			break;
+		}
+	}
+	RTW_INFO("%s: antenna=0x%x\n", __func__, antenna);
+
+	padapter->mppriv.antenna_trx = antenna;
+	SetAntenna(padapter);
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+
+int rtw_set_ctx_destAddr(struct net_device *dev,
+			 struct iw_request_info *info,
+			 struct iw_point *wrqu, char *extra)
+{
+	int jj, kk = 0;
+
+	struct pkt_attrib *pattrib;
+	struct mp_priv *pmp_priv;
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	pmp_priv = &padapter->mppriv;
+	pattrib = &pmp_priv->tx.attrib;
+
+	if (strlen(extra) < 5)
+		return _FAIL;
+
+	RTW_INFO("%s: in=%s\n", __func__, extra);
+	for (jj = 0, kk = 0; jj < ETH_ALEN; jj++, kk += 3)
+		pattrib->dst[jj] = key_2char2num(extra[kk], extra[kk + 1]);
+
+	RTW_INFO("pattrib->dst:%x %x %x %x %x %x\n", pattrib->dst[0], pattrib->dst[1], pattrib->dst[2], pattrib->dst[3], pattrib->dst[4], pattrib->dst[5]);
+	return 0;
+}
+
+
+
+int rtw_mp_ctx(struct net_device *dev,
+	       struct iw_request_info *info,
+	       struct iw_point *wrqu, char *extra)
+{
+	u32 pkTx = 1;
+	int countPkTx = 1, cotuTx = 1, CarrSprTx = 1, scTx = 1, sgleTx = 1, stop = 1, payload = 1;
+	u32 bStartTest = 1;
+	u32 count = 0, pktinterval = 0, pktlen = 0;
+	u8 status;
+	u8 tx_shape_idx = 255;
+
+	struct mp_priv *pmp_priv;
+	struct pkt_attrib *pattrib;
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	pmp_priv = &padapter->mppriv;
+	pattrib = &pmp_priv->tx.attrib;
+
+	if (padapter->registrypriv.mp_mode != 1 ) {
+		sprintf(extra, "Error: can't tx ,not in MP mode. \n");
+		wrqu->length = strlen(extra);
+		return 0;
+	}
+
+	if (copy_from_user(extra, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	*(extra + wrqu->length) = '\0';
+	RTW_INFO("%s: in=%s\n", __func__, extra);
+#ifdef CONFIG_CONCURRENT_MODE
+	if (!is_primary_adapter(padapter)) {
+		sprintf(extra, "Error: MP mode can't support Virtual adapter, Please to use main adapter.\n");
+		wrqu->length = strlen(extra);
+		return 0;
+	}
+#endif
+	countPkTx = strncmp(extra, "count=", 5); /* strncmp TRUE is 0*/
+	cotuTx = strncmp(extra, "background", 20);
+	CarrSprTx = strncmp(extra, "background,cs", 20);
+	scTx = strncmp(extra, "background,sc", 20);
+	sgleTx = strncmp(extra, "background,stone", 20);
+	pkTx = strncmp(extra, "background,pkt", 20);
+	stop = strncmp(extra, "stop", 4);
+	payload = strncmp(extra, "payload=", 8);
+
+	if (sscanf(extra, "count=%d,pkt", &count) > 0)
+		RTW_INFO("count= %d\n", count);
+	if (sscanf(extra, "pktinterval=%d", &pktinterval) > 0)
+		RTW_INFO("pktinterval= %d\n", pktinterval);
+	if (sscanf(extra, "pktlen=%d", &pktlen) > 0)
+		RTW_INFO("pktlen= %d\n", pktlen);
+	if (sscanf(extra, "tx_shape=%hhd", &tx_shape_idx) > 0)
+		RTW_INFO("tx_shape=%d\n", tx_shape_idx);
+
+	if (payload == 0) {
+			payload = MP_TX_Payload_default_random;
+			if (strncmp(extra, "payload=prbs9", 14) == 0) {
+				payload = MP_TX_Payload_prbs9;
+				sprintf(extra, "config payload PRBS9\n");
+			} else {
+				if (sscanf(extra, "payload=%x", &payload) > 0){
+					RTW_INFO("payload= %x\n", payload);
+					sprintf(extra, "config payload setting = %x\n"
+									"1. input payload=[]:\n		"
+									"[0]: 00, [1]: A5, [2]: 5A, [3]: FF, [4]: PRBS-9, [5]: Random\n"
+									"2. specified a hex payload: payload=0xee\n", payload);
+				 }
+			}
+			pmp_priv->tx.payload = payload;
+			wrqu->length = strlen(extra);
+			return 0;
+	}
+
+	if (_rtw_memcmp(extra, "destmac=", 8)) {
+		wrqu->length -= 8;
+		rtw_set_ctx_destAddr(dev, info, wrqu, &extra[8]);
+		sprintf(extra, "Set dest mac OK !\n");
+		return 0;
+	}
+	/*RTW_INFO("%s: count=%d countPkTx=%d cotuTx=%d CarrSprTx=%d scTx=%d sgleTx=%d pkTx=%d stop=%d\n", __func__, count, countPkTx, cotuTx, CarrSprTx, pkTx, sgleTx, scTx, stop);*/
+	_rtw_memset(extra, '\0', strlen(extra));
+
+	if (pktinterval != 0) {
+		sprintf(extra, "Pkt Interval = %d", pktinterval);
+		padapter->mppriv.pktInterval = pktinterval;
+		wrqu->length = strlen(extra);
+		return 0;
+
+	} else if (pktlen != 0) {
+		sprintf(extra, "Pkt len = %d", pktlen);
+		pattrib->pktlen = pktlen;
+		pmp_priv->mp_plcp_user[pmp_priv->mp_plcp_useridx].plcp_txlen = pktlen;
+		pmp_priv->rtw_mp_plcp_tx_mode = 0;
+		wrqu->length = strlen(extra);
+		return 0;
+
+	} else if (tx_shape_idx != 255) {
+		padapter->mppriv.tx_shape_idx = tx_shape_idx;
+
+		if (rtw_mp_set_tx_shape_idx(padapter))
+			sprintf(extra, "tx_shape idx = %d\n", tx_shape_idx);
+		else
+			sprintf(extra, "tx_shape %d Error\n", tx_shape_idx);
+
+		RTW_INFO("in tx_shape=%d\n", tx_shape_idx);
+		wrqu->length = strlen(extra);
+		return 0;
+
+	} else if (stop == 0) {
+		bStartTest = 0; /* To set Stop*/
+		pmp_priv->tx.stop = 1;
+		sprintf(extra, "Stop continuous Tx");
+	} else {
+		bStartTest = 1;
+
+		if (pmp_priv->mode != MP_ON) {
+			if (pmp_priv->tx.stop != 1) {
+				RTW_INFO("%s:Error MP_MODE %d != ON\n", __func__, pmp_priv->mode);
+				return	-EFAULT;
+			}
+		}
+	}
+
+	pmp_priv->tx.count = count;
+
+	if (pkTx == 0 || countPkTx == 0)
+		pmp_priv->mode = MP_PACKET_TX;
+	if (sgleTx == 0)
+		pmp_priv->mode = MP_SINGLE_TONE_TX;
+	if (cotuTx == 0)
+		pmp_priv->mode = MP_CONTINUOUS_TX;
+	if (CarrSprTx == 0)
+		pmp_priv->mode = MP_CARRIER_SUPPRISSION_TX;
+	if (scTx == 0)
+		pmp_priv->mode = MP_SINGLE_CARRIER_TX;
+
+	status = rtw_mp_pretx_proc(padapter, bStartTest, extra);
+
+	if (stop == 0)
+		pmp_priv->mode = MP_ON;
+
+	wrqu->length = strlen(extra);
+	return status;
+}
+
+
+
+int rtw_mp_disable_bt_coexist(struct net_device *dev,
+			      struct iw_request_info *info,
+			      union iwreq_data *wrqu, char *extra)
+{
+#ifdef CONFIG_BTC
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+#endif
+	u8 input[RTW_IWD_MAX_LEN];
+	u32 bt_coexist;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->data.length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+
+	if (copy_from_user(input, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	input[wrqu->data.length] = '\0';
+
+	bt_coexist = rtw_atoi(input);
+
+#if 0
+	if (bt_coexist == 0) {
+		RTW_INFO("Set OID_RT_SET_DISABLE_BT_COEXIST: disable BT_COEXIST\n");
+#ifdef CONFIG_BTC
+		rtw_btcoex_HaltNotify(padapter);
+		rtw_btcoex_SetManualControl(padapter, _TRUE);
+		/* Force to switch Antenna to WiFi*/
+		rtw_write16(padapter, 0x870, 0x300);
+		rtw_write16(padapter, 0x860, 0x110);
+#endif
+		/* CONFIG_BTC */
+	} else {
+#ifdef CONFIG_BTC
+		rtw_btcoex_SetManualControl(padapter, _FALSE);
+#endif
+	}
+#endif
+
+	return 0;
+}
+
+
+int rtw_mp_arx(struct net_device *dev,
+				struct iw_request_info *info,
+				struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmppriv = &padapter->mppriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	u8	input[RTW_IWD_MAX_LEN];
+	u32	ret;
+	char *pch, *token, *tmp[2] = {0x00, 0x00};
+	u32 i = 0, jj = 0, kk = 0, cnts = 0;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	RTW_INFO("%s: %s\n", __func__, input);
+	if (strncmp(input, "setbssid=", 8) == 0) {
+		pch = input;
+		while ((token = strsep(&pch, "=")) != NULL) {
+			if (i > 1)
+				break;
+			tmp[i] = token;
+			i++;
+		}
+		if ((tmp[0] != NULL) && (tmp[1] != NULL)) {
+			cnts = strlen(tmp[1]) / 2;
+			if (cnts < 1)
+				return -EFAULT;
+			RTW_INFO("%s: cnts=%d\n", __func__, cnts);
+			RTW_INFO("%s: data=%s\n", __func__, tmp[1]);
+			for (jj = 0, kk = 0; jj < cnts ; jj++, kk += 2) {
+				pmppriv->network_macaddr[jj] = 
+					key_2char2num(tmp[1][kk], tmp[1][kk + 1]);
+				RTW_INFO("network_macaddr[%d]=%x\n",
+					jj, pmppriv->network_macaddr[jj]);
+			}
+		} else
+			return -EFAULT;
+
+		pmppriv->bSetRxBssid = _TRUE;
+	} else if (strncmp(input, "frametype", 9) == 0) {
+		if (strncmp(input, "frametype beacon", 16) == 0)
+			pmppriv->brx_filter_beacon = _TRUE;
+		else
+			pmppriv->brx_filter_beacon = _FALSE;
+
+	} else if (strncmp(input, "accept_mac", 10) == 0) {
+		pmppriv->bmac_filter = _TRUE;
+		pch = input;
+		while ((token = strsep(&pch, "=")) != NULL) {
+			if (i > 1)
+				break;
+			tmp[i] = token;
+			i++;
+		}
+		if ((tmp[0] != NULL) && (tmp[1] != NULL)) {
+			cnts = strlen(tmp[1]) / 2;
+			if (cnts < 1)
+				return -EFAULT;
+			RTW_INFO("%s: cnts=%d\n", __func__, cnts);
+			RTW_INFO("%s: data=%s\n", __func__, tmp[1]);
+			for (jj = 0, kk = 0; jj < cnts ; jj++, kk += 2) {
+				pmppriv->mac_filter[jj] = key_2char2num(tmp[1][kk], tmp[1][kk + 1]);
+				RTW_INFO("%s mac_filter[%d]=%x\n", __func__, jj, pmppriv->mac_filter[jj]);
+			}
+		} else
+			return -EFAULT;
+
+	} else if (strncmp(input, "start", 5) == 0) {
+
+		if(pmppriv->bSetRxBssid == _TRUE) {
+			struct rtw_mp_rx_arg rx_arg;
+
+			pmppriv->brx_filter_beacon = _TRUE;
+			_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+			rtw_mp_phl_rx_reset_fltr(padapter, &rx_arg, _TRUE);
+		}
+
+		pmppriv->rx_cal_stop = 0;
+		rtw_mp_reset_phy_count(padapter);
+		rtw_mp_rx_phl_cal_timer(padapter);
+
+		sprintf(extra, "start");
+
+	} else if (strncmp(input, "stop", 5) == 0) {
+		struct rtw_mp_rx_arg rx_arg;
+
+		pmppriv->rx_cal_stop = 1;
+		_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+		rtw_mp_phl_query_rx(padapter, &rx_arg, 2);
+		if (rx_arg.cmd_ok) {
+			pmppriv->rx_pktcount = rx_arg.rx_ok;
+			pmppriv->rx_crcerrpktcount = rx_arg.rx_err;
+			RTW_INFO("phl_query_rx rx_ok=%d rx_err=%d\n",
+				pmppriv->rx_pktcount, pmppriv->rx_crcerrpktcount);
+		} else
+			RTW_WARN("phl_query_rx Fail !!!");
+
+		if(pmppriv->bSetRxBssid == _TRUE) {
+			pmppriv->brx_filter_beacon = _FALSE;
+			_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+			rtw_mp_phl_rx_reset_fltr(padapter, &rx_arg, _FALSE);
+		}
+
+		pmppriv->bmac_filter = _FALSE;
+		pmppriv->bSetRxBssid = _FALSE;
+		sprintf(extra, "Received packet OK:%d CRC error:%d ,Filter out:%d",
+			pmppriv->rx_pktcount, pmppriv->rx_crcerrpktcount,
+			pmppriv->rx_pktcount_filter_out);
+
+	} else if (strncmp(input, "phy", 3) == 0) {
+		struct rtw_mp_rx_arg rx_arg;
+
+		_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+		rtw_mp_phl_query_rx(padapter, &rx_arg, 0);
+		if (rx_arg.cmd_ok)
+			sprintf(extra, "Phy Received packet OK:%d CRC error:%d",
+					rx_arg.rx_ok, rx_arg.rx_err);
+		else
+			sprintf(extra, "PHL Phy Query Fail !!!");
+
+	} else if (strncmp(input, "mac", 3) == 0) {
+		struct rtw_mp_rx_arg rx_arg;
+
+		_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+		rtw_mp_phl_query_rx(padapter, &rx_arg, 1);
+		if (rx_arg.cmd_ok)
+			sprintf(extra, "Mac Received packet OK:%d CRC error:%d",
+					rx_arg.rx_ok, rx_arg.rx_err);
+		else
+			sprintf(extra, "Mac Phy Query Fail !!!");
+
+	} else if (strncmp(input, "mon=", 4) == 0) {
+		int bmon = 0;
+		ret = sscanf(input, "mon=%d", &bmon);
+
+		if (bmon == 1) {
+			pmppriv->rx_bindicatePkt = _TRUE;
+			sprintf(extra, "Indicating Receive Packet to network start\n");
+		} else {
+			pmppriv->rx_bindicatePkt = _FALSE;
+			sprintf(extra, "Indicating Receive Packet to network Stop\n");
+		}
+	} else if (strncmp(input, "loopbk", 6) == 0) {
+		u32 val32 = rtw_phl_read32(dvobj->phl, 0xCC20);
+		val32 |= BIT0;
+		rtw_phl_write32(dvobj->phl, 0xCC20 , val32);
+
+		val32 = rtw_phl_read32(dvobj->phl, 0xCE20);
+		val32 |= BIT0;
+		rtw_phl_write32(dvobj->phl, 0xCE20 , val32);
+		rtw_phl_write32(dvobj->phl, 0xce24 , 0x0);
+		rtw_phl_write32(dvobj->phl, 0xc624 , 0x4041010);
+
+		pmppriv->bloopback = _TRUE;
+		sprintf(extra , "Enter MAC LoopBack mode\n");
+
+	} else if (strncmp(input, "gain", 4) == 0) {
+		struct rtw_mp_rx_arg rx_arg;
+		u32 gain_val = 0xff;
+		u8 path_num = 0;
+		u8 rf_path = 0xff;
+		u8 iscck = 0xff;
+		u8 *pch = extra;
+
+		switch (input[4]) {
+		case 'a':
+			rf_path = RF_PATH_A;
+			break;
+		case 'b':
+			rf_path = RF_PATH_B;
+			break;
+		case 'c':
+			rf_path = RF_PATH_C;
+			break;
+		case 'd':
+			rf_path = RF_PATH_D;
+			break;
+		}
+
+		if ((sscanf(input + 5, "=0x%x,iscck=%hhd", &gain_val, &iscck) == 2) ||
+			(sscanf(input + 5, "=%d,iscck=%hhd", &gain_val, &iscck) == 2))
+			RTW_INFO("%s: read gain = %d , is cck =%d\n", __func__, gain_val, iscck);
+
+		else if ((sscanf(input + 4, "=0x%x", &gain_val) == 1) ||
+				(sscanf(input + 4, "=%d", &gain_val) == 1))
+				iscck = (u8)rtw_mp_is_cck_rate(pmppriv->rateidx);
+		else {
+				sprintf(pch, "error format: gain=[Dec/Hex]\n"
+						"\t\tgaina or gainb=[Dec/Hex],iscck=0/1\n");
+				wrqu->length = strlen(pch) + 1;
+				return 0;
+		}
+
+		if (rf_path == 0xff) {
+			rf_path = RF_PATH_A;
+			path_num = GET_HAL_RFPATH_NUM(adapter_to_dvobj(padapter)) - 1;
+		} else
+			path_num = rf_path;
+
+		if (gain_val == 0xff || iscck == 0xff) {
+			sprintf(extra, "error format: gaina or gainb=%d,iscck=%d\n", gain_val, iscck);
+			wrqu->length = strlen(extra) + 1;
+			return 0;
+		}
+
+		for (; rf_path <= path_num ; rf_path++) {
+			RTW_INFO("%s:set Path:%d gain_offset=%d iscck=%d\n",
+					__func__, rf_path, gain_val, iscck);
+
+			_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+			rx_arg.offset = (s8)gain_val;
+			rx_arg.iscck = iscck;
+			rtw_mp_phl_rx_gain_offset(padapter, &rx_arg, rf_path);
+
+			if (rx_arg.cmd_ok)
+				pch += sprintf(pch, "Path %s: 0x%hhx Rx Gain offset OK\n\t\t",
+				(rf_path == RF_PATH_A) ? "A":(rf_path == RF_PATH_B) ? "B":
+				(rf_path == RF_PATH_C) ? "C":"D", gain_val);
+		}
+	} else if (strncmp(input, "rssi", 4) == 0) {
+		struct rtw_mp_rx_arg rx_arg;
+		u8 rssi_path = 0, all_path_num = 0;
+		char *pcar = extra;
+		u8 i = 0;
+
+		rtw_mp_phl_rx_cal_cmd(padapter);
+
+		if (strncmp(input, "rssi a", 6) == 0)
+			rssi_path = 0;
+		else if (strncmp(input, "rssi b", 6) == 0)
+			rssi_path = 1;
+		else
+			all_path_num = GET_HAL_RFPATH_NUM(adapter_to_dvobj(padapter));
+
+		if (all_path_num > 1) {
+			rssi_path = 0;
+			all_path_num = all_path_num - 1;
+		} else 
+			all_path_num = rssi_path;
+
+		RTW_INFO("%s:Query RSSI Path:%d to %d\n", __func__, rssi_path, all_path_num);
+		_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+
+		for (i = rssi_path ; i <= all_path_num; i++) {
+			rx_arg.rf_path = i;
+
+			rtw_mp_phl_rx_rssi(padapter, &rx_arg);
+			if (rx_arg.cmd_ok) {
+				int result_int = 0;
+				int result_dec = 0;
+
+				result_int = (rx_arg.rssi / 2) -110;
+				result_dec = (rx_arg.rssi % 2);
+				if (result_dec == 1) {
+					result_dec = (result_dec * 10) / 2;
+					result_int += 1;
+				}
+				pcar += sprintf(pcar, "Path%d RSSI=%d.%d ", i, result_int, result_dec);
+			} else
+				pcar += sprintf(pcar, "Path%d RSSI Fail\n", i);
+
+		}
+	} else if (strncmp(input, "physts", 6) == 0) {
+		struct rtw_mp_rx_arg rx_arg;
+		bool bon;
+
+		if (strncmp(input, "physts on", 9) == 0)
+			bon = true;
+		else
+			bon = false;
+
+		_rtw_memset((void *)&rx_arg, 0, sizeof(struct rtw_mp_rx_arg));
+		rtw_mp_phl_rx_physts(padapter, &rx_arg, bon);
+		if (rx_arg.cmd_ok)
+			sprintf(extra, "start OK" );
+		else
+			sprintf(extra, "start Fail");
+	}
+
+	wrqu->length = strlen(extra) + 1;
+
+	return 0;
+}
+
+
+int rtw_mp_trx_query(struct net_device *dev,
+		     struct iw_request_info *info,
+		     struct iw_point *wrqu, char *extra)
+{
+	u32 txok, txfail, rxok, rxfail, rxfilterout;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+
+
+	if (pmp_priv->rtw_mp_tx_method == RTW_MP_PMACT_TX)
+		rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_CMD_PHY_OK, pmp_priv->rtw_mp_tx_method, _FALSE);
+
+	txok = padapter->mppriv.tx.sended;
+	txfail = 0;
+	rxok = padapter->mppriv.rx_pktcount;
+	rxfail = padapter->mppriv.rx_crcerrpktcount;
+	rxfilterout = padapter->mppriv.rx_pktcount_filter_out;
+
+	_rtw_memset(extra, '\0', 128);
+
+	sprintf(extra, "Tx OK:%d, Tx Fail:%d, Rx OK:%d, CRC error:%d ,Rx Filter out:%d\n", txok, txfail, rxok, rxfail, rxfilterout);
+
+	wrqu->length = strlen(extra) + 1;
+
+	return 0;
+}
+
+
+int rtw_mp_pwrtrk(struct net_device *dev,
+		  struct iw_request_info *info,
+		  struct iw_point *wrqu, char *extra)
+{
+	u8 enable;
+	u32 thermal;
+	s32 ret = 0;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 input[RTW_IWD_MAX_LEN];
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	enable = RTW_MP_TSSI_OFF;
+	if (wrqu->length > 1) {
+		/* not empty string*/
+		if (strncmp(input, "off", 3) == 0) {
+			enable = RTW_MP_TSSI_OFF;
+			sprintf(extra, "TSSI power tracking off");
+		} else if (strncmp(input, "on", 2) == 0) {
+			enable = RTW_MP_TSSI_ON;
+			sprintf(extra, "TSSI power tracking on");
+		} else if (strncmp(input, "cal", 3) == 0) {
+			enable = RTW_MP_TSSI_CAL;
+			sprintf(extra, "TSSI cal");
+		} else {
+			input[wrqu->length] = '\0';
+			enable = rtw_atoi(input);
+			sprintf(extra, "TSSI power tracking %d", enable);
+		}
+
+		if (enable <= RTW_MP_TSSI_CAL)
+			ret = rtw_mp_set_tssi_pwrtrk(padapter, enable);
+
+		if (ret == false)
+			sprintf(extra, "set TSSI power tracking fail");
+		else
+			pmp_priv->tssi_mode = enable;
+	} else {
+		enable = rtw_mp_get_tssi_pwrtrk(padapter);
+		sprintf(extra, "Get TSSI state: %d\n\
+		incput (int/str): [0]:off / [1]:on / [2]:cal for TSSI Tracking", enable);
+	}
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_psd(struct net_device *dev,
+	       struct iw_request_info *info,
+	       struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	u8		input[RTW_IWD_MAX_LEN];
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	strcpy(extra, input);
+
+	wrqu->length = mp_query_psd(padapter, extra);
+
+	return 0;
+}
+
+int rtw_mp_uuid(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	u32 uuid;
+
+	if (copy_from_user(extra, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	GetUuid(padapter, &uuid);
+
+	_rtw_memset(extra, 0, wrqu->length);
+	sprintf(extra, "%d", uuid);
+
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_thermal(struct net_device *dev,
+		   struct iw_request_info *info,
+		   struct iw_point *wrqu, char *extra)
+{
+	u8 val[4] = {0};
+	u8 ret = 0;
+	u16 ther_path_addr[4] = {0};
+	u16 cnt = 1;
+	_adapter *padapter = rtw_netdev_priv(dev);
+	int rfpath = RF_PATH_A;
+
+	if (copy_from_user(extra, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	if ((strncmp(extra, "write", 6) == 0)) {
+#if 0
+		int i;
+		u16 raw_cursize = 0, raw_maxsize = 0;
+
+		raw_maxsize = efuse_GetavailableSize(padapter);
+		RTW_INFO("[eFuse available raw size]= %d bytes\n", raw_maxsize - raw_cursize);
+		if (2 > raw_maxsize - raw_cursize) {
+			RTW_INFO("no available efuse!\n");
+			return -EFAULT;
+		}
+
+		for (i = 0; i < GET_HAL_RFPATH_NUM(adapter_to_dvobj(padapter)); i++) {
+				GetThermalMeter(padapter, i , &val[i]);
+				if (ther_path_addr[i] != 0 && val[i] != 0) {
+					if (rtw_efuse_map_write(padapter, ther_path_addr[i], cnt, &val[i]) == _FAIL) {
+						RTW_INFO("Error efuse write thermal addr 0x%x ,val = 0x%x\n", ther_path_addr[i], val[i]);
+						return -EFAULT;
+					}
+				} else {
+						RTW_INFO("Error efuse write thermal Null addr,val \n");
+						return -EFAULT;
+				}
+		}
+#endif
+		_rtw_memset(extra, 0, wrqu->length);
+		sprintf(extra, " efuse write ok :%d", val[0]);
+	} else {
+		ret = sscanf(extra, "%d", &rfpath);
+		if (ret < 1) {
+			rfpath = RF_PATH_A;
+			RTW_INFO("default thermal of path(%d)\n", rfpath);
+		}
+		if (rfpath >= GET_HAL_RFPATH_NUM(adapter_to_dvobj(padapter)))
+			return -EINVAL;
+
+		RTW_INFO("read thermal of path(%d)\n", rfpath);
+		GetThermalMeter(padapter, rfpath, &val[0]);
+
+		_rtw_memset(extra, 0, wrqu->length);
+		sprintf(extra, "%d [0x%hhx]", val[0], val[0]);
+	}
+
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+
+
+int rtw_mp_reset_stats(struct net_device *dev,
+		       struct iw_request_info *info,
+		       struct iw_point *wrqu, char *extra)
+{
+	struct mp_priv *pmp_priv;
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	pmp_priv = &padapter->mppriv;
+
+	pmp_priv->tx.sended = 0;
+	pmp_priv->tx_pktcount = 0;
+	pmp_priv->rx_pktcount = 0;
+	pmp_priv->rx_pktcount_filter_out = 0;
+	pmp_priv->rx_crcerrpktcount = 0;
+
+	rtw_mp_reset_phy_count(padapter);
+	rtw_mp_phl_rx_cal_cmd(padapter);
+
+	_rtw_memset(extra, 0, wrqu->length);
+	sprintf(extra, "mp_reset_stats ok\n");
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+
+int rtw_mp_dump(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra)
+{
+	struct mp_priv *pmp_priv;
+	u8		input[RTW_IWD_MAX_LEN];
+	_adapter *padapter = rtw_netdev_priv(dev);
+	pmp_priv = &padapter->mppriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	if (strncmp(input, "all", 4) == 0) {
+		//mac_reg_dump(RTW_DBGDUMP, padapter);
+		//bb_reg_dump(RTW_DBGDUMP, padapter);
+		//rf_reg_dump(RTW_DBGDUMP, padapter);
+	}
+	return 0;
+}
+
+
+int rtw_mp_phypara(struct net_device *dev,
+		   struct iw_request_info *info,
+		   struct iw_point *wrqu, char *extra)
+{
+
+	_adapter *padapter = rtw_netdev_priv(dev);
+	char	input[RTW_IWD_MAX_LEN];
+	u32		invalxcap = 0, ret = 0;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	RTW_INFO("%s:priv in=%s\n", __func__, input);
+
+	if (strncmp(input, "xcap=", 5) == 0) {
+		if ((sscanf(input+4, "=0x%x", &invalxcap) == 1) ||
+			(sscanf(input+4, "=%d", &invalxcap) == 1)) {
+			if (invalxcap < 255) {
+				rtw_mp_set_crystal_cap(padapter, invalxcap);
+				sprintf(extra, "Set xcap = %d [0x%hhx]", invalxcap, invalxcap);
+			} else
+				sprintf(extra, "Error formats , inpunt value over 255 !\n");
+
+			wrqu->length = strlen(extra);
+			return ret;
+		}
+	}
+
+	sprintf(extra, "Error formats , inpunt [xcap=%%d/0x%%x]\n");
+
+	wrqu->length = strlen(extra);
+	return ret;
+}
+
+
+int rtw_mp_SetRFPath(struct net_device *dev,
+		     struct iw_request_info *info,
+		     struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	char	input[RTW_IWD_MAX_LEN];
+	int		bMain = 1, bTurnoff = 1;
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	bMain = strncmp(input, "1", 2); /* strncmp TRUE is 0*/
+	bTurnoff = strncmp(input, "0", 3); /* strncmp TRUE is 0*/
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	if (bMain == 0) {
+		pmp_priv->ant_sw = true;
+		RTW_INFO("%s:rtw_mp_set_rfpath_switch=true\n", __func__);
+		sprintf(extra, "mp_setrfpath Main\n");
+	} else if (bTurnoff == 0) {
+		pmp_priv->ant_sw = false;
+		RTW_INFO("%s:rtw_mp_set_rfpath_switch=false\n", __func__);
+		sprintf(extra, "mp_setrfpath Aux\n");
+	}
+	rtw_mp_set_rfpath_switch(padapter);
+
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+
+int rtw_mp_switch_rf_path(struct net_device *dev,
+			struct iw_request_info *info,
+			struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv;
+	char	input[RTW_IWD_MAX_LEN];
+	int		bwlg = 1, bwla = 1, btg = 1, bbt=1;
+	u8 ret = 0;
+
+#if 0
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	pmp_priv = &padapter->mppriv;
+
+	RTW_INFO("%s: in=%s\n", __func__, input);
+
+#ifdef CONFIG_RTL8821C /* only support for 8821c wlg/wla/btg/bt RF switch path */
+	if ((strncmp(input, "WLG", 3) == 0) || (strncmp(input, "1", 1) == 0)) {
+		pmp_priv->rf_path_cfg = SWITCH_TO_WLG;
+		sprintf(extra, "switch rf path WLG\n");
+
+	} else if ((strncmp(input, "WLA", 3) == 0) || (strncmp(input, "2", 1) == 0)) {
+		pmp_priv->rf_path_cfg = SWITCH_TO_WLA;
+		sprintf(extra, "switch rf path WLA\n");
+
+	} else if ((strncmp(input, "BTG", 3) == 0) || (strncmp(input, "0", 1) == 0)) {
+		pmp_priv->rf_path_cfg = SWITCH_TO_BTG;
+		sprintf(extra, "switch rf path BTG\n");
+
+	} else if ((strncmp(input, "BT", 3) == 0) || (strncmp(input, "3", 1) == 0)) {
+		pmp_priv->rf_path_cfg = SWITCH_TO_BT;
+		sprintf(extra, "switch rf path BT\n");
+	} else {
+		pmp_priv->rf_path_cfg = SWITCH_TO_WLG;
+		sprintf(extra, "Error input, default set WLG\n");
+		return -EFAULT;
+	}
+
+	mp_phy_switch_rf_path_set(padapter, &pmp_priv->rf_path_cfg);
+#endif
+	wrqu->length = strlen(extra);
+#endif
+	return ret;
+
+}
+int rtw_mp_QueryDrv(struct net_device *dev,
+		    struct iw_request_info *info,
+		    union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	char	input[RTW_IWD_MAX_LEN];
+	int	qAutoLoad = 1;
+	//struct efuse_info *efuse = adapter_to_efuse(padapter);
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->data.length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	RTW_INFO("%s:iwpriv in=%s\n", __func__, input);
+
+	qAutoLoad = strncmp(input, "autoload", 8); /* strncmp TRUE is 0*/
+
+	if (qAutoLoad == 0) {
+		RTW_INFO("%s:qAutoLoad\n", __func__);
+
+		//if (efuse->is_autoload_fail)
+		//	sprintf(extra, "fail");
+		//else
+		//	sprintf(extra, "ok");
+	}
+	wrqu->data.length = strlen(extra) + 1;
+	return 0;
+}
+
+
+int rtw_mp_PwrCtlDM(struct net_device *dev,
+		    struct iw_request_info *info,
+		    struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	u8		input[RTW_IWD_MAX_LEN];
+	u8		pwrtrk_state = 0;
+	u8		pwtk_type[5][25] = {"Thermal tracking off","Thermal tracking on",
+					"TSSI tracking off","TSSI tracking on","TSSI calibration"};
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	input[wrqu->length] = '\0';
+	RTW_INFO("%s: in=%s\n", __func__, input);
+
+	if (wrqu->length == 2) {
+		if(input[0] >= '0' && input[0] <= '4') {
+			pwrtrk_state = rtw_atoi(input);
+			/*MPT_PwrCtlDM(padapter, pwrtrk_state);*/
+			sprintf(extra, "PwrCtlDM start %s\n" , pwtk_type[pwrtrk_state]);
+		} else {
+			sprintf(extra, "Error unknown number ! Please check your input number\n"
+				" 0 : Thermal tracking off\n 1 : Thermal tracking on\n 2 : TSSI tracking off\n"
+				" 3 : TSSI tracking on\n 4 : TSSI calibration\n");
+		}
+		wrqu->length = strlen(extra);
+
+		return 0;
+	}
+	if (strncmp(input, "start", 5) == 0 || strncmp(input, "thertrk on", 10) == 0) {/* strncmp TRUE is 0*/
+		pwrtrk_state = 1;
+		sprintf(extra, "PwrCtlDM start %s\n" , pwtk_type[pwrtrk_state]);
+	} else if (strncmp(input, "thertrk off", 11) == 0 || strncmp(input, "stop", 5) == 0) {
+		pwrtrk_state = 0;
+		sprintf(extra, "PwrCtlDM stop %s\n" , pwtk_type[pwrtrk_state]);
+	} else if (strncmp(input, "tssitrk off", 11) == 0){
+		pwrtrk_state = 2;
+		sprintf(extra, "PwrCtlDM stop %s\n" , pwtk_type[pwrtrk_state]);
+	} else if (strncmp(input, "tssitrk on", 10) == 0){
+		pwrtrk_state = 3;
+		sprintf(extra, "PwrCtlDM start %s\n" , pwtk_type[pwrtrk_state]);
+	} else if (strncmp(input, "tssik", 5) == 0){
+		pwrtrk_state = 4;
+		sprintf(extra, "PwrCtlDM start %s\n" , pwtk_type[pwrtrk_state]);
+	} else {
+		pwrtrk_state = 0;
+		sprintf(extra, "Error input, default PwrCtlDM stop\n"
+			" thertrk off : Thermal tracking off\n thertrk on : Thermal tracking on\n"
+			" tssitrk off : TSSI tracking off\n tssitrk on : TSSI tracking on\n tssik : TSSI calibration\n\n"
+			" 0 : Thermal tracking off\n 1 : Thermal tracking on\n 2 : TSSI tracking off\n"
+			" 3 : TSSI tracking on\n 4 : TSSI calibration\n");
+	}
+
+	/*MPT_PwrCtlDM(padapter, pwrtrk_state);*/
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_iqk(struct net_device *dev,
+		 struct iw_request_info *info,
+		 struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	rtw_mp_trigger_iqk(padapter);
+
+	return 0;
+}
+
+int rtw_mp_lck(struct net_device *dev,
+		 struct iw_request_info *info,
+		 struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+
+	rtw_mp_trigger_lck(padapter);
+
+	return 0;
+}
+
+int rtw_mp_dpk(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	//struct dm_struct *phydm = adapter_to_phydm(padapter);
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	*(extra + wrqu->data.length) = '\0';
+
+	if (strncmp(extra, "off", 3) == 0 && strlen(extra) < 4) {
+			//phydm->dpk_info.is_dpk_enable = 0;
+			//halrf_dpk_enable_disable(phydm);
+			sprintf(extra, "set dpk off\n");
+
+	} else if (strncmp(extra, "on", 2) == 0 && strlen(extra) < 3) {
+			//phydm->dpk_info.is_dpk_enable = 1;
+			//halrf_dpk_enable_disable(phydm);
+			sprintf(extra, "set dpk on\n");
+	} else	{
+			rtw_mp_trigger_dpk(padapter);
+			sprintf(extra, "set dpk trigger\n");
+	}
+
+	wrqu->data.length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_get_tsside(struct net_device *dev,
+			 struct iw_request_info *info,
+			 struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	char input[RTW_IWD_MAX_LEN];
+	u8 rfpath = 0xff;
+	s8 tssi_de = 0;
+	char pout_str_buf[7];
+	char tgr_str_buf[7];
+	u8 pout_signed_flag = 0 , tgrpwr_signed_flag = 0;
+	int int_num = 0;
+	u32 dec_num = 0;
+	s32 pout = 0;
+	s32 tgrpwr = 0;
+	int i;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+	input[wrqu->length] = '\0';
+
+	if (wrqu->length == 2) {
+		rfpath = rtw_atoi(input);
+		if (rfpath <= 3) {
+			tssi_de = rtw_mp_get_tssi_de(padapter, rfpath);
+		} else
+			sprintf(extra, "Invalid command format, please indicate RF path 0/1/2/3");
+
+	} else if (sscanf(input + 1, "dbm=%7s pwr=%7s", tgr_str_buf , pout_str_buf) == 2) {
+		/*
+		* rtwpriv wlan0 mp_get_tsside adbm=12 pwr=12
+		* [adbm] target power [pwr] output power
+		* rf_path : 0 = adbm , 1 = bdbm
+		* dbm : -15.00 ~ 25.00
+		* pwr : -15.00 ~ 25.00
+		* ex : rtwpriv wlan0 mp_get_tsside adbm=16 pwr=14.25
+		*/
+		RTW_INFO("%s: in=tgr_str %s  pout_str %s\n", __func__, tgr_str_buf , pout_str_buf);
+		switch (input[0]) {
+		case 'a':
+			rfpath = RF_PATH_A;
+			break;
+		case 'b':
+			rfpath = RF_PATH_B;
+			break;
+		case 'c':
+			rfpath = RF_PATH_C;
+			break;
+		case 'd':
+			rfpath = RF_PATH_D;
+			break;
+		default:
+			goto error;
+			break;
+		}
+
+		if(pout_str_buf[0] == '-')
+			pout_signed_flag = 1;
+
+		i = sscanf(pout_str_buf, "%d.%2u", &int_num, &dec_num);
+		pout = int_num * 100;
+		RTW_DBG("%s:pout %d int %d dec %d\n", __func__, pout, int_num , dec_num);
+
+		if (i == 2) {
+
+			u8 idx = 0;
+			u32 dec = 0;
+			u8 str_len = 0;
+			char *token, *tmp[3] = {};
+			char *pextra;
+
+			pextra = pout_str_buf;
+			while ((token = strsep (&pextra,".")) != NULL) {
+				tmp[idx] = token;
+				RTW_INFO("%s() token %d = %s strlen =%ld\n", __func__,
+					idx, tmp[idx], strlen(tmp[idx]));
+				idx++;
+			}
+
+			str_len = strlen(tmp[1]);
+			dec = rtw_atoi(tmp[1]);
+			dec_num = (str_len == 1) ? dec * 10: (str_len == 2) ? dec * 1: dec;
+			RTW_INFO("%s() decimal_num  = %d\n", __func__, dec_num);
+			pout += ((pout < 0 || pout_signed_flag == 1) ? -dec_num : dec_num);
+		}
+		if (pout < -1500 || pout > 2500)
+			goto error;
+		RTW_INFO("%s:pout %d\n", __func__, pout);
+
+		if (tgr_str_buf[0] == '-')
+			tgrpwr_signed_flag = 1;
+
+		int_num = 0;
+		dec_num = 0;
+
+		i = sscanf(tgr_str_buf, "%d.%2u", &int_num, &dec_num);
+		tgrpwr = int_num * 100;
+		RTW_DBG("%s:tgrpwr %d int %d dec %d\n", __func__, tgrpwr, int_num , dec_num);
+
+		if (i == 2) {
+
+			u8 idx = 0;
+			u32 dec = 0;
+			u8 str_len = 0;
+			char *token, *tmp[3] = {};
+			char *pextra;
+
+			pextra = tgr_str_buf;
+			while ((token = strsep (&pextra,".")) != NULL) {
+				tmp[idx] = token;
+				RTW_INFO("%s() token %d = %s strlen =%ld\n", __func__,
+					idx, tmp[idx], strlen(tmp[idx]));
+				idx++;
+			}
+
+			str_len = strlen(tmp[1]);
+			dec = rtw_atoi(tmp[1]);
+			dec_num = (str_len == 1) ? dec * 10: (str_len == 2) ? dec * 1: dec;
+			RTW_INFO("%s() decimal_num  = %d\n", __func__, dec_num);
+			tgrpwr += ((tgrpwr < 0 || tgrpwr_signed_flag == 1) ? -dec_num : dec_num);
+		}
+
+		if (tgrpwr < -1500 || 2500 < tgrpwr)
+			goto error;
+		RTW_INFO("%s:tgrpwr %d\n", __func__, tgrpwr);
+
+		tssi_de = (s8)rtw_mp_get_online_tssi_de(padapter, pout, tgrpwr, rfpath);
+	}
+
+	if (rfpath == 0)
+		sprintf(extra, "patha=%d hex:%x", tssi_de, (u8)tssi_de);
+	else if (rfpath == 1)
+		sprintf(extra, "pathb=%d hex:%x", tssi_de, (u8)tssi_de);
+	else if (rfpath == 2)
+		sprintf(extra, "pathc=%d hex:%x", tssi_de, (u8)tssi_de);
+	else if (rfpath == 3)
+		sprintf(extra, "pathd=%d hex:%x", tssi_de, (u8)tssi_de);
+	else
+		goto error;
+
+	wrqu->length = strlen(extra);
+	return 0;
+error:
+
+	sprintf(extra, "Invalid command format, please indicate RF path mp_get_tsside [0/1/2/3]\n\
+			GET ONLINE TSSI DE:\n\
+			mp_get_tsside adbm=-15.00 ~ 25.00 pwr=-15.00 ~ 25.00\n\
+			mp_get_tsside bdbm=-15.00 ~ 25.00 pwr=-15.00 ~ 25.00\n");
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_set_tsside(struct net_device *dev,
+		   struct iw_request_info *info,
+		   struct iw_point *wrqu, char *extra)
+{
+	int tsside_val = 0;
+	u8 rf_path = RF_PATH_A;
+	char input[RTW_IWD_MAX_LEN];
+
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+        u8 rfpath_i = 0;
+        struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+        u8 tx_nss = get_phy_tx_nss(padapter, padapter_link);
+        u8 bk_txpwr = pmp_priv->txpowerdbm;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	RTW_INFO("%s:input =[%s]\n", __func__, input);
+
+	switch (input[4]) {
+	case 'a':
+		rf_path = RF_PATH_A;
+		break;
+	case 'b':
+		rf_path = RF_PATH_B;
+		break;
+	case 'c':
+		rf_path = RF_PATH_C;
+		break;
+	case 'd':
+		rf_path = RF_PATH_D;
+		break;
+	default:
+		goto exit_err;
+	}
+
+	if ((sscanf(input+5, "=0x%x", &tsside_val) == 1) ||
+		(sscanf(input+5, "=%d", &tsside_val) == 1)) {
+
+		RTW_INFO("%s:got tsside val =[%d] 0x%x\n", __func__, tsside_val, (u32)tsside_val);
+		if (tsside_val > 255)
+			sprintf(extra, "Error TSSI DE value: %d" , tsside_val);
+		else {
+			sprintf(extra, "Set TSSI DE path_%s: %d",
+				rf_path == RF_PATH_A ? "A" : rf_path == RF_PATH_B ? "B" :
+				rf_path == RF_PATH_C ? "C":"D", tsside_val);
+
+			rtw_mp_set_tsside2verify(padapter, (u32)tsside_val, rf_path);
+			pmp_priv->bspecif_tssi_de = true;
+			pmp_priv->specif_tsside_val = tsside_val;
+			if (pmp_priv->tssi_mode >= RTW_MP_TSSI_ON && bk_txpwr > 17 * TX_POWER_BASE) {
+               			 pmp_priv->txpowerdbm = 16 * TX_POWER_BASE ;
+                		for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+                        		rtw_mp_txpower_dbm(padapter, rfpath_i);
+                		rtw_msleep_os(60);
+                		pmp_priv->txpowerdbm = bk_txpwr;
+                		for (rfpath_i = 0 ; rfpath_i < tx_nss; rfpath_i ++)
+                        		rtw_mp_txpower_dbm(padapter, rfpath_i);
+        		}
+
+		}
+	} else
+		goto exit_err;
+
+	wrqu->length = strlen(extra);
+
+	return 0;
+
+exit_err:
+	sprintf(extra, "Invalid command format,\n\t\t"
+			"please input TSSI DE value within patha/b/c/d=[decimal] or [hex:0xXX]");
+
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_getver(struct net_device *dev,
+		  struct iw_request_info *info,
+		  union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv;
+
+	pmp_priv = &padapter->mppriv;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	sprintf(extra, "rtwpriv=%d\n", RTWPRIV_VER_INFO);
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_mp_mon(struct net_device *dev,
+	       struct iw_request_info *info,
+	       union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	//struct hal_ops *pHalFunc = &hal->hal_func;
+	NDIS_802_11_NETWORK_INFRASTRUCTURE networkType;
+	int bstart = 1, bstop = 1;
+
+	networkType = Ndis802_11Infrastructure;
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	*(extra + wrqu->data.length) = '\0';
+
+#if 0 //def CONFIG_MP_INCLUDED
+	if (init_mp_priv(padapter) == _FAIL)
+		RTW_INFO("%s: initialize MP private data Fail!\n", __func__);
+	padapter->mppriv.channel = 6;
+
+	bstart = strncmp(extra, "start", 5); /* strncmp TRUE is 0*/
+	bstop = strncmp(extra, "stop", 4); /* strncmp TRUE is 0*/
+	if (bstart == 0) {
+		mp_join(padapter, WIFI_FW_ADHOC_STATE);
+		SetPacketRx(padapter, _TRUE, _FALSE);
+		SetChannel(padapter);
+		pmp_priv->rx_bindicatePkt = _TRUE;
+		pmp_priv->bRTWSmbCfg = _TRUE;
+		sprintf(extra, "monitor mode start\n");
+	} else if (bstop == 0) {
+		SetPacketRx(padapter, _FALSE, _FALSE);
+		pmp_priv->rx_bindicatePkt = _FALSE;
+		pmp_priv->bRTWSmbCfg = _FALSE;
+		padapter->registrypriv.mp_mode = 1;
+		pHalFunc->hal_deinit(padapter);
+		padapter->registrypriv.mp_mode = 0;
+		pHalFunc->hal_init(padapter);
+		/*rtw_disassoc_cmd(padapter, 0, 0);*/
+		if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+			rtw_disassoc_cmd(padapter, 500, 0);
+			rtw_indicate_disconnect(padapter, 0, _FALSE);
+			/*rtw_free_assoc_resources_cmd(padapter, _TRUE, 0);*/
+		}
+		sprintf(extra, "monitor mode Stop\n");
+	}
+#endif
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_pretx_proc(_adapter *padapter, u8 bstart, char *extra)
+{
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	char *pextra = extra;
+
+	switch (pmp_priv->mode) {
+
+	case MP_PACKET_TX:
+		if (bstart == 0) {
+			pmp_priv->tx.stop = 1;
+			pmp_priv->mode = MP_ON;
+			sprintf(extra, "Stop continuous Tx");
+		} else if (pmp_priv->tx.stop == 1) {
+			pextra = extra + strlen(extra);
+			pextra += sprintf(pextra, "\nStart continuous DA=ffffffffffff len=1500 count=%u\n", pmp_priv->tx.count);
+			pmp_priv->tx.stop = 0;
+			/*SetPacketTx(padapter);*/
+		} else {
+			RTW_INFO(" tx.stop != 1 or not STOP TX\n");
+			return -EFAULT;
+		}
+		rtw_set_phl_packet_tx(padapter, bstart);
+		return 0;
+	case MP_SINGLE_TONE_TX:
+		if (bstart != 0)
+			strcat(extra, "\nStart continuous DA=ffffffffffff len=1500\n infinite=yes.");
+		pmp_priv->rtw_mp_tx_method = RTW_MP_TMACT_TX;
+		pmp_priv->tx.count = 1;
+		rtw_pre_phl_packet_tx(padapter, (u8)bstart);
+		pmp_priv->rtw_mp_tx_method = RTW_MP_PMACT_TX;
+		rtw_mp_singletone_tx(padapter, (u8)bstart);
+		pmp_priv->tx.stop = 1;
+		break;
+	case MP_CONTINUOUS_TX:
+		if (bstart != 0)
+			strcat(extra, "\nStart continuous DA=ffffffffffff len=1500\n infinite=yes.");
+		rtw_mp_continuous_tx(padapter, (u8)bstart);
+		break;
+	case MP_CARRIER_SUPPRISSION_TX:
+		if (bstart != 0) {
+			if (rtw_mp_hwrate2mptrate(pmp_priv->rateidx) <= MPT_RATE_11M)
+				strcat(extra, "\nStart continuous DA=ffffffffffff len=1500\n infinite=yes.");
+			else
+				strcat(extra, "\nSpecify carrier suppression but not CCK rate");
+		}
+		rtw_mp_carriersuppr_tx(padapter, (u8)bstart);
+		break;
+	case MP_SINGLE_CARRIER_TX:
+		if (bstart != 0)
+			strcat(extra, "\nStart continuous DA=ffffffffffff len=1500\n infinite=yes.");
+		rtw_mp_singlecarrier_tx(padapter, (u8)bstart);
+		break;
+	default:
+		sprintf(extra, "Error! Continuous-Tx is not on-going.");
+		return -EFAULT;
+	}
+
+	if (bstart == 1 && pmp_priv->mode != MP_ON) {
+		struct mp_priv *pmp_priv = &padapter->mppriv;
+
+		if (pmp_priv->tx.stop == 0) {
+			pmp_priv->tx.stop = 1;
+			rtw_msleep_os(5);
+		}
+#ifdef CONFIG_80211N_HT
+		if(padapter->registrypriv.ht_enable &&
+			is_supported_ht(padapter->registrypriv.wireless_mode))
+			pmp_priv->tx.attrib.ht_en = 1;
+#endif
+
+	} else
+		pmp_priv->mode = MP_ON;
+
+	return 0;
+}
+
+
+int rtw_mp_tx(struct net_device *dev,
+	      struct iw_request_info *info,
+	      union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	PMPT_CONTEXT		pMptCtx = &(padapter->mppriv.mpt_ctx);
+	char *pextra = extra;
+	u32 bandwidth = 0, sg = 0, channel = 6, txpower = 40, rate = 108, ant = 0, txmode = 1, count = 0;
+	u8 bStartTest = 1, status = 0;
+#ifdef CONFIG_MP_VHT_HW_TX_MODE
+	u8 Idx = 0, tmpU1B;
+#endif
+	u16 antenna = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	*(extra + wrqu->data.length) = '\0';
+#if 0	
+#ifdef CONFIG_CONCURRENT_MODE
+	if (!is_primary_adapter(padapter)) {
+		sprintf(extra, "Error: MP mode can't support Virtual adapter, Please to use main adapter.\n");
+		wrqu->data.length = strlen(extra);
+		return 0;
+	}
+#endif
+
+	if (strncmp(extra, "stop", 3) == 0) {
+		bStartTest = 0; /* To set Stop*/
+		pmp_priv->tx.stop = 1;
+		sprintf(extra, "Stop continuous Tx");
+		status = rtw_mp_pretx_proc(padapter, bStartTest, extra);
+		wrqu->data.length = strlen(extra);
+		return status;
+	} else if (strncmp(extra, "count", 5) == 0) {
+		if (sscanf(extra, "count=%d", &count) < 1)
+			RTW_INFO("Got Count=%d]\n", count);
+		pmp_priv->tx.count = count;
+		return 0;
+	} else if (strncmp(extra, "setting", 7) == 0) {
+		_rtw_memset(extra, 0, wrqu->data.length);
+		pextra += sprintf(pextra, "Current Setting :\n Channel:%d", pmp_priv->channel);
+		pextra += sprintf(pextra, "\n Bandwidth:%d", pmp_priv->bandwidth);
+		pextra += sprintf(pextra, "\n Rate index:%d", pmp_priv->rateidx);
+		pextra += sprintf(pextra, "\n TxPower index:%d", pmp_priv->txpoweridx);
+		pextra += sprintf(pextra, "\n Antenna TxPath:%d", pmp_priv->antenna_tx);
+		pextra += sprintf(pextra, "\n Antenna RxPath:%d", pmp_priv->antenna_rx);
+		pextra += sprintf(pextra, "\n MP Mode:%d", pmp_priv->mode);
+		wrqu->data.length = strlen(extra);
+		return 0;
+#ifdef CONFIG_MP_VHT_HW_TX_MODE
+	} else if (strncmp(extra, "pmact", 5) == 0) {
+		if (strncmp(extra, "pmact=", 6) == 0) {
+			_rtw_memset(&pMptCtx->PMacTxInfo, 0, sizeof(pMptCtx->PMacTxInfo));
+			if (strncmp(extra, "pmact=start", 11) == 0) {
+				pMptCtx->PMacTxInfo.bEnPMacTx = _TRUE;
+				sprintf(extra, "Set PMac Tx Mode start\n");
+			} else {
+				pMptCtx->PMacTxInfo.bEnPMacTx = _FALSE;
+				sprintf(extra, "Set PMac Tx Mode Stop\n");
+			}
+			if (pMptCtx->bldpc == TRUE)
+				pMptCtx->PMacTxInfo.bLDPC = _TRUE;
+
+			if (pMptCtx->bstbc == TRUE)
+				pMptCtx->PMacTxInfo.bSTBC = _TRUE;
+
+			pMptCtx->PMacTxInfo.bSPreamble = pmp_priv->preamble;
+			pMptCtx->PMacTxInfo.bSGI = pmp_priv->preamble;
+			pMptCtx->PMacTxInfo.BandWidth = pmp_priv->bandwidth;
+			pMptCtx->PMacTxInfo.TX_RATE = HwRateToMPTRate(pmp_priv->rateidx);
+
+			pMptCtx->PMacTxInfo.Mode = pMptCtx->HWTxmode;
+
+			pMptCtx->PMacTxInfo.NDP_sound = FALSE;/*(adapter.PacketType == NDP_PKT)?TRUE:FALSE;*/
+
+			if (padapter->mppriv.pktInterval == 0)
+				pMptCtx->PMacTxInfo.PacketPeriod = 100;
+			else
+				pMptCtx->PMacTxInfo.PacketPeriod = padapter->mppriv.pktInterval;
+
+			if (padapter->mppriv.pktLength < 1000)
+				pMptCtx->PMacTxInfo.PacketLength = 1000;
+			else
+				pMptCtx->PMacTxInfo.PacketLength = padapter->mppriv.pktLength;
+
+			pMptCtx->PMacTxInfo.PacketPattern  = rtw_random32() % 0xFF;
+
+			if (padapter->mppriv.tx_pktcount != 0)
+				pMptCtx->PMacTxInfo.PacketCount = padapter->mppriv.tx_pktcount;
+
+			pMptCtx->PMacTxInfo.Ntx = 0;
+			for (Idx = 16; Idx < 20; Idx++) {
+				tmpU1B = (padapter->mppriv.antenna_tx >> Idx) & 1;
+				if (tmpU1B)
+					pMptCtx->PMacTxInfo.Ntx++;
+			}
+
+			_rtw_memset(pMptCtx->PMacTxInfo.MacAddress, 0xFF, ETH_ALEN);
+
+			PMAC_Get_Pkt_Param(&pMptCtx->PMacTxInfo, &pMptCtx->PMacPktInfo);
+
+			if (MPT_IS_CCK_RATE(pMptCtx->PMacTxInfo.TX_RATE))
+
+				CCK_generator(&pMptCtx->PMacTxInfo, &pMptCtx->PMacPktInfo);
+			else {
+				PMAC_Nsym_generator(&pMptCtx->PMacTxInfo, &pMptCtx->PMacPktInfo);
+				/* 24 BIT*/
+				L_SIG_generator(pMptCtx->PMacPktInfo.N_sym, &pMptCtx->PMacTxInfo, &pMptCtx->PMacPktInfo);
+			}
+			/*	48BIT*/
+			if (MPT_IS_HT_RATE(pMptCtx->PMacTxInfo.TX_RATE))
+				HT_SIG_generator(&pMptCtx->PMacTxInfo, &pMptCtx->PMacPktInfo);
+			else if (MPT_IS_VHT_RATE(pMptCtx->PMacTxInfo.TX_RATE)) {
+				/*	48BIT*/
+				VHT_SIG_A_generator(&pMptCtx->PMacTxInfo, &pMptCtx->PMacPktInfo);
+
+				/*	26/27/29 BIT  & CRC 8 BIT*/
+				VHT_SIG_B_generator(&pMptCtx->PMacTxInfo);
+
+				/* 32 BIT*/
+				VHT_Delimiter_generator(&pMptCtx->PMacTxInfo);
+			}
+
+			mpt_ProSetPMacTx(padapter);
+
+		} else if (strncmp(extra, "pmact,mode=", 11) == 0) {
+			int txmode = 0;
+
+			if (sscanf(extra, "pmact,mode=%d", &txmode) > 0) {
+				if (txmode == 1) {
+					pMptCtx->HWTxmode = CONTINUOUS_TX;
+					sprintf(extra, "\t Config HW Tx mode = CONTINUOUS_TX\n");
+				} else if (txmode == 2) {
+					pMptCtx->HWTxmode = OFDM_Single_Tone_TX;
+					sprintf(extra, "\t Config HW Tx mode = OFDM_Single_Tone_TX\n");
+				} else {
+					pMptCtx->HWTxmode = PACKETS_TX;
+					sprintf(extra, "\t Config HW Tx mode = PACKETS_TX\n");
+				}
+			} else {
+				pMptCtx->HWTxmode = PACKETS_TX;
+				sprintf(extra, "\t Config HW Tx mode=\n 0 = PACKETS_TX\n 1 = CONTINUOUS_TX\n 2 = OFDM_Single_Tone_TX");
+			}
+		} else if (strncmp(extra, "pmact,", 6) == 0) {
+			int PacketPeriod = 0, PacketLength = 0, PacketCout = 0;
+			int bldpc = 0, bstbc = 0;
+
+			if (sscanf(extra, "pmact,period=%d", &PacketPeriod) > 0) {
+				padapter->mppriv.pktInterval = PacketPeriod;
+				RTW_INFO("PacketPeriod=%d\n", padapter->mppriv.pktInterval);
+				sprintf(extra, "PacketPeriod [1~255]= %d\n", padapter->mppriv.pktInterval);
+
+			} else if (sscanf(extra, "pmact,length=%d", &PacketLength) > 0) {
+				padapter->mppriv.pktLength = PacketLength;
+				RTW_INFO("PacketPeriod=%d\n", padapter->mppriv.pktLength);
+				sprintf(extra, "PacketLength[~65535]=%d\n", padapter->mppriv.pktLength);
+
+			} else if (sscanf(extra, "pmact,count=%d", &PacketCout) > 0) {
+				padapter->mppriv.tx_pktcount = PacketCout;
+				RTW_INFO("Packet Cout =%d\n", padapter->mppriv.tx_pktcount);
+				sprintf(extra, "Packet Cout =%d\n", padapter->mppriv.tx_pktcount);
+
+			} else if (sscanf(extra, "pmact,ldpc=%d", &bldpc) > 0) {
+				pMptCtx->bldpc = bldpc;
+				RTW_INFO("Set LDPC =%d\n", pMptCtx->bldpc);
+				sprintf(extra, "Set LDPC =%d\n", pMptCtx->bldpc);
+
+			} else if (sscanf(extra, "pmact,stbc=%d", &bstbc) > 0) {
+				pMptCtx->bstbc = bstbc;
+				RTW_INFO("Set STBC =%d\n", pMptCtx->bstbc);
+				sprintf(extra, "Set STBC =%d\n", pMptCtx->bstbc);
+			} else
+				sprintf(extra, "\n period={1~255}\n length={1000~65535}\n count={0~}\n ldpc={0/1}\n stbc={0/1}");
+
+		}
+
+		wrqu->data.length = strlen(extra);
+		return 0;
+#endif
+	} else {
+
+		if (sscanf(extra, "ch=%d,bw=%d,rate=%d,pwr=%d,ant=%d,tx=%d", &channel, &bandwidth, &rate, &txpower, &ant, &txmode) < 6) {
+			RTW_INFO("Invalid format [ch=%d,bw=%d,rate=%d,pwr=%d,ant=%d,tx=%d]\n", channel, bandwidth, rate, txpower, ant, txmode);
+			_rtw_memset(extra, 0, wrqu->data.length);
+			pextra += sprintf(pextra, "\n Please input correct format as bleow:\n");
+			pextra += sprintf(pextra, "\t ch=%d,bw=%d,rate=%d,pwr=%d,ant=%d,tx=%d\n", channel, bandwidth, rate, txpower, ant, txmode);
+			pextra += sprintf(pextra, "\n [ ch : BGN = <1~14> , A or AC = <36~165> ]");
+			pextra += sprintf(pextra, "\n [ bw : Bandwidth: 0 = 20M, 1 = 40M, 2 = 80M ]");
+			pextra += sprintf(pextra, "\n [ rate :	CCK: 1 2 5.5 11M X 2 = < 2 4 11 22 >]");
+			pextra += sprintf(pextra, "\n [		OFDM: 6 9 12 18 24 36 48 54M X 2 = < 12 18 24 36 48 72 96 108>");
+			pextra += sprintf(pextra, "\n [		HT 1S2SS MCS0 ~ MCS15 : < [MCS0]=128 ~ [MCS7]=135 ~ [MCS15]=143 >");
+			pextra += sprintf(pextra, "\n [		HT 3SS MCS16 ~ MCS32 : < [MCS16]=144 ~ [MCS23]=151 ~ [MCS32]=159 >");
+			pextra += sprintf(pextra, "\n [		VHT 1SS MCS0 ~ MCS9 : < [MCS0]=160 ~ [MCS9]=169 >");
+			pextra += sprintf(pextra, "\n [ txpower : 1~63 power index");
+			pextra += sprintf(pextra, "\n [ ant : <A = 1, B = 2, C = 4, D = 8> ,2T ex: AB=3 BC=6 CD=12");
+			pextra += sprintf(pextra, "\n [ txmode : < 0 = CONTINUOUS_TX, 1 = PACKET_TX, 2 = SINGLE_TONE_TX, 3 = CARRIER_SUPPRISSION_TX, 4 = SINGLE_CARRIER_TX>\n");
+			wrqu->data.length = strlen(extra);
+			return status;
+
+		} else {
+			char *pextra = extra;
+			RTW_INFO("Got format [ch=%d,bw=%d,rate=%d,pwr=%d,ant=%d,tx=%d]\n", channel, bandwidth, rate, txpower, ant, txmode);
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "Change Current channel %d to channel %d", padapter->mppriv.channel , channel);
+			padapter->mppriv.channel = channel;
+			SetChannel(padapter);
+			pHalData->current_channel = channel;
+
+			if (bandwidth == 1)
+				bandwidth = CHANNEL_WIDTH_40;
+			else if (bandwidth == 2)
+				bandwidth = CHANNEL_WIDTH_80;
+			pextra = extra + strlen(pextra);
+			pextra += sprintf(pextra, "\nChange Current Bandwidth %d to Bandwidth %d", padapter->mppriv.bandwidth, bandwidth);
+			padapter->mppriv.bandwidth = (u8)bandwidth;
+			padapter->mppriv.preamble = sg;
+			SetBandwidth(padapter);
+			pHalData->current_channel_bw = bandwidth;
+
+			pextra += sprintf(pextra, "\nSet power level :%d", txpower);
+			padapter->mppriv.txpoweridx = (u8)txpower;
+			pMptCtx->TxPwrLevel[RF_PATH_A] = (u8)txpower;
+			pMptCtx->TxPwrLevel[RF_PATH_B] = (u8)txpower;
+			pMptCtx->TxPwrLevel[RF_PATH_C] = (u8)txpower;
+			pMptCtx->TxPwrLevel[RF_PATH_D]  = (u8)txpower;
+			SetTxPower(padapter);
+
+			RTW_INFO("%s: bw=%d sg=%d\n", __func__, bandwidth, sg);
+
+			if (rate <= 0x7f)
+				rate = wifirate2_ratetbl_inx((u8)rate);
+			else if (rate < 0xC8)
+				rate = (rate - 0x80 + MPT_RATE_MCS0);
+			/*HT  rate 0x80(MCS0)  ~ 0x8F(MCS15) ~ 0x9F(MCS31) 128~159
+			VHT1SS~2SS rate 0xA0 (VHT1SS_MCS0 44) ~ 0xB3 (VHT2SS_MCS9 #63) 160~179
+			VHT rate 0xB4 (VHT3SS_MCS0 64) ~ 0xC7 (VHT2SS_MCS9 #83) 180~199
+			else
+			VHT rate 0x90(VHT1SS_MCS0) ~ 0x99(VHT1SS_MCS9) 144~153
+			rate =(rate - MPT_RATE_VHT1SS_MCS0);
+			*/
+			RTW_INFO("%s: rate index=%d\n", __func__, rate);
+			if (rate >= MPT_RATE_LAST)
+				return -EINVAL;
+			pextra += sprintf(pextra, "\nSet data rate to %d index %d", padapter->mppriv.rateidx, rate);
+
+			padapter->mppriv.rateidx = rate;
+			pMptCtx->mpt_rate_index = rate;
+			SetDataRate(padapter);
+
+			pextra += sprintf(pextra, "\nSet Antenna Path :%d", ant);
+			switch (ant) {
+			case 1:
+				antenna = ANTENNA_A;
+				break;
+			case 2:
+				antenna = ANTENNA_B;
+				break;
+			case 4:
+				antenna = ANTENNA_C;
+				break;
+			case 8:
+				antenna = ANTENNA_D;
+				break;
+			case 3:
+				antenna = ANTENNA_AB;
+				break;
+			case 5:
+				antenna = ANTENNA_AC;
+				break;
+			case 9:
+				antenna = ANTENNA_AD;
+				break;
+			case 6:
+				antenna = ANTENNA_BC;
+				break;
+			case 10:
+				antenna = ANTENNA_BD;
+				break;
+			case 12:
+				antenna = ANTENNA_CD;
+				break;
+			case 7:
+				antenna = ANTENNA_ABC;
+				break;
+			case 14:
+				antenna = ANTENNA_BCD;
+				break;
+			case 11:
+				antenna = ANTENNA_ABD;
+				break;
+			case 15:
+				antenna = ANTENNA_ABCD;
+				break;
+			}
+			RTW_INFO("%s: antenna=0x%x\n", __func__, antenna);
+			padapter->mppriv.antenna_tx = antenna;
+			padapter->mppriv.antenna_rx = antenna;
+			pHalData->antenna_tx_path = antenna;
+			SetAntenna(padapter);
+
+			if (txmode == 0)
+				pmp_priv->mode = MP_CONTINUOUS_TX;
+			else if (txmode == 1) {
+				pmp_priv->mode = MP_PACKET_TX;
+				pmp_priv->tx.count = count;
+			} else if (txmode == 2)
+				pmp_priv->mode = MP_SINGLE_TONE_TX;
+			else if (txmode == 3)
+				pmp_priv->mode = MP_CARRIER_SUPPRISSION_TX;
+			else if (txmode == 4)
+				pmp_priv->mode = MP_SINGLE_CARRIER_TX;
+
+			status = rtw_mp_pretx_proc(padapter, bStartTest, extra);
+		}
+
+	}
+#endif
+	wrqu->data.length = strlen(extra);
+	return status;
+}
+
+
+int rtw_mp_rx(struct net_device *dev,
+	      struct iw_request_info *info,
+	      union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	char *pextra = extra;
+	u32 bandwidth = 0, sg = 0, channel = 6, ant = 0;
+	u16 antenna = 0;
+	u8 bStartRx = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	*(extra + wrqu->data.length) = '\0';
+#if 0
+#ifdef CONFIG_CONCURRENT_MODE
+	if (!is_primary_adapter(padapter)) {
+		sprintf(extra, "Error: MP mode can't support Virtual adapter, Please to use main adapter.\n");
+		wrqu->data.length = strlen(extra);
+		return 0;
+	}
+#endif
+
+	if (strncmp(extra, "stop", 4) == 0) {
+		_rtw_memset(extra, 0, wrqu->data.length);
+		SetPacketRx(padapter, bStartRx, _FALSE);
+		pmp_priv->bmac_filter = _FALSE;
+		sprintf(extra, "Received packet OK:%d CRC error:%d ,Filter out:%d", padapter->mppriv.rx_pktcount, padapter->mppriv.rx_crcerrpktcount, padapter->mppriv.rx_pktcount_filter_out);
+		wrqu->data.length = strlen(extra);
+		return 0;
+
+	} else if (sscanf(extra, "ch=%d,bw=%d,ant=%d", &channel, &bandwidth, &ant) < 3) {
+		RTW_INFO("Invalid format [ch=%d,bw=%d,ant=%d]\n", channel, bandwidth, ant);
+		_rtw_memset(extra, 0, wrqu->data.length);
+		pextra += sprintf(pextra, "\n Please input correct format as bleow:\n");
+		pextra += sprintf(pextra, "\t ch=%d,bw=%d,ant=%d\n", channel, bandwidth, ant);
+		pextra += sprintf(pextra, "\n [ ch : BGN = <1~14> , A or AC = <36~165> ]");
+		pextra += sprintf(pextra, "\n [ bw : Bandwidth: 0 = 20M, 1 = 40M, 2 = 80M ]");
+		pextra += sprintf(pextra, "\n [ ant : <A = 1, B = 2, C = 4, D = 8> ,2T ex: AB=3 BC=6 CD=12");
+		wrqu->data.length = strlen(extra);
+		return 0;
+
+	} else {
+		char *pextra = extra;
+		bStartRx = 1;
+		RTW_INFO("Got format [ch=%d,bw=%d,ant=%d]\n", channel, bandwidth, ant);
+		_rtw_memset(extra, 0, wrqu->data.length);
+		sprintf(extra, "Change Current channel %d to channel %d", padapter->mppriv.channel , channel);
+		padapter->mppriv.channel = channel;
+		SetChannel(padapter);
+		pHalData->current_channel = channel;
+
+		if (bandwidth == 1)
+			bandwidth = CHANNEL_WIDTH_40;
+		else if (bandwidth == 2)
+			bandwidth = CHANNEL_WIDTH_80;
+		pextra = extra + strlen(extra);
+		pextra += sprintf(pextra, "\nChange Current Bandwidth %d to Bandwidth %d", padapter->mppriv.bandwidth, bandwidth);
+		padapter->mppriv.bandwidth = (u8)bandwidth;
+		padapter->mppriv.preamble = sg;
+		SetBandwidth(padapter);
+		pHalData->current_channel_bw = bandwidth;
+
+		pextra += sprintf(pextra, "\nSet Antenna Path :%d", ant);
+		switch (ant) {
+		case 1:
+			antenna = ANTENNA_A;
+			break;
+		case 2:
+			antenna = ANTENNA_B;
+			break;
+		case 4:
+			antenna = ANTENNA_C;
+			break;
+		case 8:
+			antenna = ANTENNA_D;
+			break;
+		case 3:
+			antenna = ANTENNA_AB;
+			break;
+		case 5:
+			antenna = ANTENNA_AC;
+			break;
+		case 9:
+			antenna = ANTENNA_AD;
+			break;
+		case 6:
+			antenna = ANTENNA_BC;
+			break;
+		case 10:
+			antenna = ANTENNA_BD;
+			break;
+		case 12:
+			antenna = ANTENNA_CD;
+			break;
+		case 7:
+			antenna = ANTENNA_ABC;
+			break;
+		case 14:
+			antenna = ANTENNA_BCD;
+			break;
+		case 11:
+			antenna = ANTENNA_ABD;
+			break;
+		case 15:
+			antenna = ANTENNA_ABCD;
+			break;
+		}
+		RTW_INFO("%s: antenna=0x%x\n", __func__, antenna);
+		padapter->mppriv.antenna_tx = antenna;
+		padapter->mppriv.antenna_rx = antenna;
+		pHalData->antenna_tx_path = antenna;
+		SetAntenna(padapter);
+
+		strcat(extra, "\nstart Rx");
+		SetPacketRx(padapter, bStartRx, _FALSE);
+	}
+#endif	
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+
+int rtw_mp_hwtx(struct net_device *dev,
+		struct iw_request_info *info,
+		union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = &padapter->mppriv;
+	PMPT_CONTEXT mpt_ctx = &(padapter->mppriv.mpt_ctx);
+
+#if defined(CONFIG_RTL8821B) || defined(CONFIG_RTL8822B) || defined(CONFIG_RTL8821C) || defined(CONFIG_RTL8822C)
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	*(extra + wrqu->data.length) = '\0';
+
+	_rtw_memset(&mpt_ctx->PMacTxInfo, 0, sizeof(RT_PMAC_TX_INFO));
+	_rtw_memcpy((void *)&mpt_ctx->PMacTxInfo, (void *)extra, sizeof(RT_PMAC_TX_INFO));
+	_rtw_memset(extra, 0, wrqu->data.length);
+
+	if (mpt_ctx->PMacTxInfo.bEnPMacTx == 1 && pmp_priv->mode != MP_ON) {
+		sprintf(extra, "MP Tx Running, Please Set PMac Tx Mode Stop\n");
+		RTW_INFO("Error !!! MP Tx Running, Please Set PMac Tx Mode Stop\n");
+	} else {
+		RTW_INFO("To set MAC Tx mode\n");
+		mpt_ProSetPMacTx(padapter);
+		sprintf(extra, "Set PMac Tx Mode OK\n");
+	}
+	wrqu->data.length = strlen(extra);
+#endif
+	return 0;
+
+}
+
+int rtw_mp_pwrlmt(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+	u8 pwrlimtstat = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	*(extra + wrqu->data.length) = '\0';
+#if CONFIG_TXPWR_LIMIT
+	if (strncmp(extra, "off", 3) == 0 && strlen(extra) < 4) {
+		if (rtw_mpt_set_power_limit_en(padapter, _FALSE))
+			sprintf(extra, "Turn off Power Limit\n");
+		else 
+			sprintf(extra, "Turn off Power Limit Fail\n");
+	} else if (strncmp(extra, "on", 2) == 0 && strlen(extra) < 3) {
+		if (rtw_mpt_set_power_limit_en(padapter, _TRUE))
+			sprintf(extra, "Turn on Power Limit\n");
+		else 
+			sprintf(extra, "Turn on Power Limit Fail\n");
+	} else
+#endif
+	{
+		sprintf(extra, "PHL PWRLMT:%s\n",
+			(rtw_mpt_get_power_limit_en(padapter) == _TRUE) ?"ON" :"OFF");
+	}
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_dpk_track(struct net_device *dev,
+			struct iw_request_info *info,
+			union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	//struct dm_struct *phydm = adapter_to_phydm(padapter);
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	*(extra + wrqu->data.length) = '\0';
+
+	if (strncmp(extra, "off", 3) == 0 && strlen(extra) < 4) {
+		//halrf_set_dpk_track(phydm, FALSE);
+		sprintf(extra, "set dpk track off\n");
+
+	} else if (strncmp(extra, "on", 2) == 0 && strlen(extra) < 3) {
+		//halrf_set_dpk_track(phydm, TRUE);
+		sprintf(extra, "set dpk track on\n");
+	}
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_set_phl_io(struct net_device *dev,
+			 struct iw_request_info *info,
+			 struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_mp_cmd_arg *cmd_arg = NULL;
+	struct rtw_mp_test_cmdbuf *pcmdbuf = NULL;
+	u16 i = 0;
+
+	if (copy_from_user(extra, wrqu->pointer, wrqu->length))
+			return -EFAULT;
+
+	RTW_INFO("%s, wrqu->length %d !!!\n", __func__, wrqu->length);
+
+	rtw_phl_test_submodule_cmd_process(dvobj->phl_com, (void*)extra, wrqu->length);
+	pcmdbuf = (struct rtw_mp_test_cmdbuf *)extra;
+	while (1) {
+		if (pcmdbuf) {
+			cmd_arg = (struct rtw_mp_cmd_arg *)pcmdbuf->buf;
+			rtw_phl_test_submodule_get_rpt(dvobj->phl_com, (void *)extra, wrqu->length);
+		}
+		if (cmd_arg != NULL && cmd_arg->cmd_ok) {
+			RTW_INFO("%s,GET CMD OK !!!\n", __func__);
+			break;
+		} else {
+			i++;
+			rtw_msleep_os(100);
+			if (i == 3) {
+				RTW_INFO("%s,GET CMD FAIL !!!\n", __func__);
+				break;
+			}
+		}
+	}
+
+	if (copy_to_user(wrqu->pointer, extra, wrqu->length))
+		return -EFAULT;
+
+	return 0;
+}
+
+int rtw_mp_get_phl_io(struct net_device *dev,
+			 struct iw_request_info *info,
+			 struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	if (copy_from_user(extra, wrqu->pointer, wrqu->length))
+			return -EFAULT;
+	*(extra + wrqu->length) = '\0';
+
+	rtw_phl_test_submodule_get_rpt(dvobj->phl_com, (void *)&extra, wrqu->length);
+
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_tx_pattern_idx(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+
+	u32 tx_patt_idx = 0;
+	u32 ppdu_type = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	if (sscanf(extra, "index=%d,type=%d", &tx_patt_idx, &ppdu_type) > 0) {
+		RTW_INFO("%s: tx_patt_idx=%d ,ppdu_type=%d\n", __func__, tx_patt_idx , ppdu_type);
+		pmp_priv->rtw_mp_pmact_patt_idx = tx_patt_idx;
+		pmp_priv->rtw_mp_pmact_ppdu_type = ppdu_type;
+		rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_CONFIG_PLCP_PATTERN, pmp_priv->rtw_mp_tx_method, _TRUE);
+		_rtw_memset(extra, 0, wrqu->data.length);
+		sprintf(extra, "Config Tx Pattern idx %d to %d", pmp_priv->rtw_mp_pmact_patt_idx, tx_patt_idx);
+	} else if ((strncmp(extra, "stop", 4) == 0)) {
+		rtw_phl_mp_tx_cmd(padapter, RTW_MP_TX_CONFIG_PLCP_PATTERN, pmp_priv->rtw_mp_tx_method, _FALSE);
+		_rtw_memset(extra, 0, wrqu->data.length);
+		sprintf(extra, "Config Tx Pattern Stop");
+	} else {
+		u8 *pstr = extra;
+		_rtw_memset(pstr, 0, wrqu->data.length);
+	}
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_tx_plcp_tx_data(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 user_idx = pmp_priv->mp_plcp_useridx;
+
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	*(extra + wrqu->data.length) = '\0';
+
+	if ((strncmp(extra, "ppdu", 4) == 0)) {
+		u32 ppdu_type = 0;
+
+		if (sscanf(extra, "ppdu=%d", &ppdu_type) > 0) {
+				u8 *pextra = extra;
+
+				RTW_INFO("%s: ppdu_type=%d\n", __func__, ppdu_type);
+				_rtw_memset(extra, 0, wrqu->data.length);
+
+				pextra += sprintf(pextra, "Config PPDU Type %s to %s\n", 
+						PPDU_TYPE_STR(pmp_priv->rtw_mp_pmact_ppdu_type), PPDU_TYPE_STR(ppdu_type));
+				pmp_priv->rtw_mp_pmact_ppdu_type = ppdu_type;
+
+				rtw_mp_update_coding(padapter);
+
+				rtw_update_giltf(padapter);
+
+				if (ppdu_type >= RTW_MP_TYPE_HE_MU_OFDMA) {
+					u8 ru_num = 0 , rualloc_num = 0 ,i = 0;
+
+					ru_num = rtw_mp_update_ru_tone(padapter);
+					if (ru_num <= MP_RU_TONE_996) {
+						rualloc_num = rtw_mp_update_ru_alloc(padapter);
+						pextra += sprintf(pextra, "\nCurrent [%s] RU Alloc index:%d\n",
+													RU_TONE_STR(pmp_priv->rtw_mp_ru_tone),
+													pmp_priv->mp_plcp_user[user_idx].ru_alloc);
+
+						pextra += sprintf(pextra, "RU Alloc list:[");
+						for (i = 0;i <= rualloc_num - 1; i++)
+							pextra += sprintf(pextra, "%d ", pmp_priv->ru_alloc_list[i]);
+						pextra += sprintf(pextra, "]\n");
+
+						pextra += sprintf(pextra, "\nRU Tone support list(Refer Coding:%s):\n",
+													(pmp_priv->mp_plcp_user[user_idx].coding ? "LDPC":"BCC"));
+						for (i = 0;i <= ru_num; i++)
+							pextra += sprintf(pextra, "%d : [%s]\n",
+													pmp_priv->ru_tone_sel_list[i],
+													RU_TONE_STR(pmp_priv->ru_tone_sel_list[i]));
+						pextra += sprintf(pextra, "\n\nCodingCMD:[mp_plcp_user coding=%%d] (0:BCC 1:LDPC )");
+						pextra += sprintf(pextra, "\nRU Tone CMD:[ mp_plcp_user ru_tone=%%d ]");
+						pextra += sprintf(pextra, "\nRU Alloc CMD:[ mp_plcp_user ru_alloc=%%d ]");
+					}
+				}
+		} else {
+			u8 *pstr = extra;
+			_rtw_memset(pstr, 0, wrqu->data.length);
+			pstr += sprintf(pstr, "CMD: [mp_plcp_datappdu=%%d]\nPLCP (PPDU Type):\n\
+0:CCK\n1:LEGACY\n2:HT_MF\n3:HT_GF\n4:VHT\n5:HE_SU\n6:HE_ER_SU\n7:HE_MU_OFDMA\n8:HE_TB\n");
+			}
+	}else if ((strncmp(extra, "preamble", 8) == 0)) {
+		u8 preamble = 0;
+
+		if (sscanf(extra, "preamble=%hhd", &preamble) > 0) {
+			RTW_INFO("%s: preamble=%d\n", __func__, preamble);
+			_rtw_memset(extra, 0, wrqu->data.length);
+
+				pmp_priv->preamble = preamble;
+				sprintf(extra, "Config Preamble %d to %d", pmp_priv->preamble, preamble);
+
+		} else
+			sprintf(extra, "Error format ! input 'preamble=[num]'\n");
+
+	} else if ((strncmp(extra, "stbc", 4) == 0)) {
+		u8 stbc = 0;
+
+		if (sscanf(extra, "stbc=%hhd", &stbc) > 0) {
+				RTW_INFO("%s: stbc=%d\n", __func__, stbc);
+				_rtw_memset(extra, 0, wrqu->data.length);
+				sprintf(extra, "Config STBC enable: %d to %d", pmp_priv->rtw_mp_stbc, stbc);
+				pmp_priv->rtw_mp_stbc = stbc;
+				if (pmp_priv->rtw_mp_stbc)
+					pmp_priv->mp_plcp_user[user_idx].dcm = 0;
+		} else {
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "input [stbc=0/1]");
+		}
+
+	} else if ((strncmp(extra, "giltf", 5) == 0)) {
+		u8 idx = 0;
+		u8 giltf_num = rtw_update_giltf(padapter);
+
+		if (sscanf(extra, "giltf=%hhd", &idx) > 0) {
+			u8 gi = 0, ltf = 0;
+
+			RTW_INFO("%s: gi+ltf=%d\n", __func__, idx);
+			_rtw_memset(extra, 0, wrqu->data.length);
+
+			if (giltf_num != 0 && idx <= giltf_num) {
+				gi = pmp_priv->st_giltf[idx].gi;
+				ltf = pmp_priv->st_giltf[idx].ltf;
+				sprintf(extra, "Config GI+LTF to %s ", pmp_priv->st_giltf[idx].type_str);
+				pmp_priv->rtw_mp_plcp_gi = gi;
+				pmp_priv->rtw_mp_plcp_ltf = ltf;
+				RTW_INFO("%s: gi=%d ltf=%d\n", __func__, gi, ltf);
+			} else
+				sprintf(extra, "Not support GI+LTF index\n");
+
+		} else {
+			u8 *pextra = extra;
+			u8 i = 0;
+
+			if (giltf_num > 0) {
+				pextra += sprintf(pextra, "GI + LTF list:\n");
+				for (i = 0;i <= giltf_num; i++)
+							pextra += sprintf(pextra, "%d:[%s]\n", i, pmp_priv->st_giltf[i].type_str);
+			}
+			sprintf(pextra, "PPDU Type Not support GI+LTF.");
+		}
+
+	} else if ((strncmp(extra, "tx_time", 7) == 0)) {
+			u32 tx_time = 0;
+
+			if (sscanf(extra, "tx_time=%d", &tx_time) > 0) {
+				u32 tmp_tx_time = (tx_time * 10) / 4;
+
+				pmp_priv->rtw_mp_plcp_tx_time = tmp_tx_time;
+				pmp_priv->rtw_mp_plcp_tx_mode = 1;
+				sprintf(extra, "Config Tx Time:%d us", pmp_priv->rtw_mp_plcp_tx_time);
+			}
+
+	} else if ((strncmp(extra, "tx_len", 6) == 0)) {
+			u32 tx_len = 0;
+
+			if (sscanf(extra, "tx_len=%d", &tx_len) > 0) {
+				pmp_priv->mp_plcp_user[user_idx].plcp_txlen = tx_len;
+				pmp_priv->rtw_mp_plcp_tx_mode = 0;
+				sprintf(extra, "Config Tx Len:%d", pmp_priv->mp_plcp_user[user_idx].plcp_txlen);
+		}
+
+	} else if ((strncmp(extra, "he_sigb", 7) == 0)) {
+			u32 he_sigb = 0;
+
+			if (sscanf(extra, "he_sigb=%d", &he_sigb) > 0) {
+				if (he_sigb <= 5) {
+					pmp_priv->rtw_mp_he_sigb = he_sigb;
+					sprintf(extra, "Config HE SIGB:%d", he_sigb);
+				} else
+					sprintf(extra, "Error Config HE SIGB:[%d] (0~5)", he_sigb);
+			} else {
+				u8 *pstr = extra;
+
+				_rtw_memset(pstr, 0, wrqu->data.length);
+				pstr += sprintf(pstr, "invalid CMD Format! input: he_sigb=[Num]\n\
+					PLCP (HE SIGB):\n0\n\1\n2\n\3\n\4\n\5\n");
+			}
+
+	} else if ((strncmp(extra, "he_sigb_dcm", 7) == 0)) {
+			u32 he_sigb_dcm = 0;
+
+			if (sscanf(extra, "he_sigb_dcm=%d", &he_sigb_dcm) > 0) {
+				if (he_sigb_dcm <= 1) {
+					pmp_priv->rtw_mp_he_sigb_dcm = he_sigb_dcm;
+					sprintf(extra, "Config HE SIGB DCM:%d", he_sigb_dcm);
+				} else
+					sprintf(extra, "Error Config HE SIGB:[%d] (0:Disable 1:Enable)", he_sigb_dcm);
+
+			} else {
+				u8 *pstr = extra;
+
+				_rtw_memset(pstr, 0, wrqu->data.length);
+				pstr += sprintf(pstr, "invalid CMD Format! input: he_sigb_dcm=[Num]\n\
+					PLCP (HE SIGB DCM):\n0:Disable\n1:Enable");
+			}
+
+	} else if ((strncmp(extra, "er_su_ru106en", 7) == 0)) {
+			u32 ru106en = 0;
+
+			if (sscanf(extra, "er_su_ru106en=%d", &ru106en) > 0) {
+				if (ru106en <= 1) {
+					pmp_priv->rtw_mp_he_er_su_ru_106_en = ru106en;
+					sprintf(extra, "Config he_er_su_ru106:%d", ru106en);
+				} else
+					sprintf(extra, "Error!!! Config HE ER SU RU106 Enable:[%d] (0:Disable 1:Enable)",
+										ru106en);
+			} else {
+				u8 *pstr = extra;
+
+				_rtw_memset(pstr, 0, wrqu->data.length);
+				pstr += sprintf(pstr, "Error!!! Config HE ER SU RU106 Enable: Input number[ 0:Disable 1:Enable ]");
+			}
+
+	} else if ((strncmp(extra, "ru_tone", 7) == 0)) {
+		u32 ru_tone = 0;
+
+		if (sscanf(extra, "ru_tone=%d", &ru_tone) > 0) {
+				RTW_INFO("%s: RU Tone=%d\n", __func__, ru_tone);
+				_rtw_memset(extra, 0, wrqu->data.length);
+				sprintf(extra, "Config RU tone %d to %d", pmp_priv->rtw_mp_ru_tone, ru_tone);
+				pmp_priv->rtw_mp_ru_tone = ru_tone;
+		} else {
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "Error!!!\tinput , [ru_tone= number]");
+		}
+
+	} else {
+				char *pstr = extra;
+				u8 ppdu_idx = pmp_priv->rtw_mp_pmact_ppdu_type;
+
+				if (ppdu_idx < RTW_MP_TYPE_HT_MF) {
+						pstr += sprintf(pstr, "invalid PPDU Type ! input :ppdu=[Num] over the HT\n");
+				} else {
+					u8 i = 0;
+					u8 num = rtw_update_giltf(padapter);
+
+					_rtw_memset(extra, 0, wrqu->data.length);
+					pstr += sprintf(pstr, "invalid CMD Format !! please input: giltf=[Num]\n");
+					pstr += sprintf(pstr, "PPDU %s GI+LTF:\n", PPDU_TYPE_STR(ppdu_idx));
+					for (i = 0; i <= num; i++)
+						pstr += sprintf(pstr, "[%d]: %s\n" ,i , pmp_priv->st_giltf[i].type_str);
+				}
+		}
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_tx_plcp_tx_user(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *mpprv = (struct mp_priv *)&padapter->mppriv;
+
+	u32 tx_mcs = 0;
+	u8 user_idx = mpprv->mp_plcp_useridx;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	*(extra + wrqu->data.length) = '\0';
+
+	if (sscanf(extra, "mcs=%d", &tx_mcs) > 0) {
+		RTW_INFO("%s: mcs=%d\n", __func__, tx_mcs);
+		mpprv->mp_plcp_user[user_idx].plcp_mcs = tx_mcs;
+		_rtw_memset(extra, 0, wrqu->data.length);
+		sprintf(extra, "Config PLCP MCS idx %d to %d",
+						mpprv->mp_plcp_user[user_idx].plcp_mcs, tx_mcs);
+
+	} else if ((strncmp(extra, "dcm", 3) == 0)) {
+		u8 dcm = 0;
+
+		if (sscanf(extra, "dcm=%hhd", &dcm) > 0) {
+				RTW_INFO("%s: dcm=%d\n", __func__, dcm);
+				_rtw_memset(extra, 0, wrqu->data.length);
+				sprintf(extra, "Config DCM enable: %d to %d", mpprv->mp_plcp_user[user_idx].dcm, dcm);
+				mpprv->mp_plcp_user[user_idx].dcm = dcm;
+
+				if (mpprv->mp_plcp_user[user_idx].dcm)
+						mpprv->rtw_mp_stbc = 0;
+		} else {
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "Error !!! input [dcm=0/1]");
+		}
+
+	} else if ((strncmp(extra, "coding", 6) == 0)) {
+		u8 coding = 0;
+
+		if (sscanf(extra, "coding=%hhd", &coding) > 0) {
+
+				RTW_INFO("%s: coding=%d\n", __func__, coding);
+				_rtw_memset(extra, 0, wrqu->data.length);
+				mpprv->rtw_coding = coding;
+				rtw_mp_update_coding(padapter);
+				sprintf(extra, "Config coding to %s",
+								(mpprv->rtw_coding?"LDPC":"BCC"));
+		} else {
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "Error !!!\n0:BCC 1:LDPC \t input Number [coding=0/1]");
+		}
+
+	} else if ((strncmp(extra, "ru_alloc", 8) == 0)) {
+		u32 ru_alloc = 0;
+
+		if (sscanf(extra, "ru_alloc=%d", &ru_alloc) > 0) {
+
+				RTW_INFO("%s: RU alloc=%d\n", __func__, ru_alloc);
+				_rtw_memset(extra, 0, wrqu->data.length);
+				sprintf(extra, "Config RU alloc %d to %d",
+								mpprv->mp_plcp_user[user_idx].ru_alloc, ru_alloc);
+				mpprv->mp_plcp_user[user_idx].ru_alloc = ru_alloc;
+		} else {
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "Error!!!\tinput , [ru_alloc= number]");
+		}
+
+	} else if ((strncmp(extra, "txuser", 6) == 0)) {
+		u32 txuser = 0;
+
+		if (sscanf(extra, "txuser=%d", &txuser) > 0) {
+				RTW_INFO("%s: Sel User idx=%d\n", __func__, txuser);
+				_rtw_memset(extra, 0, wrqu->data.length);
+				sprintf(extra, "config Tx User %d to %d", mpprv->rtw_mp_plcp_tx_user, txuser);
+				mpprv->rtw_mp_plcp_tx_user = txuser;
+		} else {
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "Error!!!\tinput , [txuser= number]");
+		}
+
+	} else if ((strncmp(extra, "user", 4) == 0)) {
+		u32 user_idx = 0;
+
+		if (sscanf(extra, "user=%d", &user_idx) > 0) {
+				RTW_INFO("%s: Sel User idx=%d\n", __func__, user_idx);
+				_rtw_memset(extra, 0, wrqu->data.length);
+				sprintf(extra, "select User idx %d to %d", mpprv->mp_plcp_useridx, user_idx);
+				mpprv->mp_plcp_useridx = user_idx;
+		} else {
+			_rtw_memset(extra, 0, wrqu->data.length);
+			sprintf(extra, "Error!!!\tinput , [user= number]");
+		}
+
+	} else if ((strncmp(extra, "tx_len", 6) == 0)) {
+			u32 tx_len = 0;
+
+			if (sscanf(extra, "tx_len=%d", &tx_len) > 0) {
+				mpprv->mp_plcp_user[user_idx].plcp_txlen = tx_len;
+				mpprv->rtw_mp_plcp_tx_mode = 0;
+				sprintf(extra, "Config Tx Len:%d", mpprv->mp_plcp_user[user_idx].plcp_txlen);
+		}
+
+	} else {
+		u8 *pstr = extra;
+		_rtw_memset(pstr, 0, wrqu->data.length);
+
+		pstr += sprintf(pstr, "invalid CMD Format!\n \
+		\t input :\n\
+		\t user=%%d\n\
+		\t mcs=%%d\n\
+		\t dcm=%%d,\n\
+		\t coding=%%d\n\
+		\t ru_alloc=%%d\n");
+	}
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_tx_method(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, wrqu->data.length))
+		return -EFAULT;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	extra[wrqu->data.length] = '\0';
+
+	if ((strncmp(extra, "PMACT", 5) == 0)) {
+		pmp_priv->rtw_mp_tx_method = RTW_MP_PMACT_TX;
+		sprintf(extra, "set PMACT OK");
+	} else if ((strncmp(extra, "TMACT", 5) == 0)) {
+		pmp_priv->rtw_mp_tx_method = RTW_MP_TMACT_TX;
+		pmp_priv->is_tmac_mode = 1;
+		sprintf(extra, "set TMACT OK");
+	} else if ((strncmp(extra, "FWPMACT", 7) == 0)) {
+		pmp_priv->rtw_mp_tx_method = RTW_MP_FW_PMACT_TX;
+		pmp_priv->is_tmac_mode = 0;
+		sprintf(extra, "set FWPMACT OK");
+	}
+	wrqu->data.length = strlen(extra) + 1;
+	return 0;
+}
+
+int rtw_mp_config_phy(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8	set_phy = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	extra[wrqu->data.length] = '\0';
+	set_phy = rtw_atoi(extra);
+
+	if (set_phy < 2) {
+		sprintf(extra, "set current phy %d to %d", pmp_priv->rtw_mp_cur_phy, set_phy);
+		pmp_priv->rtw_mp_cur_phy = set_phy;
+		rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SET_PHY_INDEX);
+	} else
+		sprintf(extra, "Not suuport phy %d", set_phy);
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_phl_rfk(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 k_type = RTW_MP_CAL_MAX;
+	u8 k_cap_ctrl = false;
+	u8 k_cap_on = false;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+	*(extra + wrqu->data.length) = '\0';
+
+	if (strncmp(extra, "iqk", 3) == 0) {
+		k_type = RTW_MP_CAL_IQK;
+
+		if (strncmp(extra, "iqk on", 6) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set iqk on");
+
+		} else if ((strncmp(extra, "iqk off", 7) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set iqk off");
+		} else
+			sprintf(extra, "set iqk trigger");
+
+	} else if (strncmp(extra, "dpk", 3) == 0) {
+		k_type = RTW_MP_CAL_DPK;
+		if (strncmp(extra, "dpk on", 6) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set dpk on");
+		} else if ((strncmp(extra, "dpk off", 7) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set dpk off");
+		} else
+			sprintf(extra, "set dpk trigger");
+
+	} else if (strncmp(extra, "chk", 3) == 0) {
+		k_type = RTW_MP_CAL_CHL_RFK;
+		if (strncmp(extra, "chk on", 6) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set chk on");
+		} else if ((strncmp(extra, "chk off", 7) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set chk off");
+		} else
+			sprintf(extra, "set chk trigger");
+
+	} else if (strncmp(extra, "dack", 3) == 0) {
+		k_type = RTW_MP_CAL_DACK;
+		if (strncmp(extra, "dack on", 6) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set dack on");
+		} else if ((strncmp(extra, "dack off", 7) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set dack off");
+		} else
+			sprintf(extra, "set dack trigger");
+
+	} else if (strncmp(extra, "lck", 3) == 0) {
+		k_type = RTW_MP_CAL_LCK;
+		if (strncmp(extra, "lck on", 6) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set lck on");
+		} else if ((strncmp(extra, "lck off", 7) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set lck off");
+		} else
+			sprintf(extra, "set lck trigger");
+
+	} else if (strncmp(extra, "dpk_trk", 7) == 0) {
+		k_type = RTW_MP_CAL_DPK_TRACK;
+		if (strncmp(extra, "dpk_trk on", 10) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set dpk_trk on");
+		} else if ((strncmp(extra, "dpk_trk off", 11) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set dpk_trk off");
+		}
+
+	} else if (strncmp(extra, "tssi", 4) == 0) {
+		k_type = RTW_MP_CAL_TSSI;
+		if (strncmp(extra, "tssi on", 7) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set tssi on");
+		} else if ((strncmp(extra, "tssi off", 8) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set tssi off");
+		} else
+			sprintf(extra, "set tssi trigger");
+
+	} else if (strncmp(extra, "gapk", 4) == 0) {
+		k_type = RTW_MP_CAL_GAPK;
+		if (strncmp(extra, "gapk on", 7) == 0) {
+			k_cap_ctrl = true;
+			k_cap_on = true;
+			sprintf(extra, "set gapk on");
+		} else if ((strncmp(extra, "gapk off", 8) == 0)) {
+			k_cap_ctrl = true;
+			k_cap_on = false;
+			sprintf(extra, "set gapk off");
+		} else
+			sprintf(extra, "set gapk trigger");
+
+	} else
+		sprintf(extra, "Error! CMD Format:\n\
+				[trigger K] or Set K on/off\n\
+				chk\\chk on/off\n\
+				dack\\dack on/off\n\
+				iqk\\iqk on/off\n\
+				lck\\lck on/off\n\
+				dpk\\dpk on/off\n\
+				dpk_trk on/off\n\
+				tssi\\tssi on/off\n\
+				gapk\\gapk on/off\n");
+
+	if (k_cap_ctrl) {
+		rtw_mp_cal_capab(padapter, k_type, k_cap_on);
+	} else if (k_type < RTW_MP_CAL_MAX)
+		rtw_mp_cal_trigger(padapter, k_type);
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_phl_btc_path(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 btc_mode = 0;
+
+	if (strncmp(extra, "normal", 6) == 0) {
+
+		btc_mode = BTC_MODE_NORMAL;
+		RTW_INFO("set BTC Path Normal");
+	} else if (strncmp(extra, "wl", 2) == 0 || strncmp(extra, "WL", 2) == 0) {
+
+		btc_mode = BTC_MODE_WL;
+		RTW_INFO("set BTC Path WL");
+	} else if (strncmp(extra, "bt", 2) == 0 || strncmp(extra, "BT", 2) == 0) {
+
+		btc_mode = BTC_MODE_BT;
+		RTW_INFO("set BTC Path BT");
+	} else {
+		btc_mode = BTC_MODE_WL;
+		RTW_INFO("Default set BTC Path WL");
+	}
+
+	pmp_priv->btc_path = btc_mode;
+	if (rtw_mp_phl_config_arg(padapter, RTW_MP_CONFIG_CMD_SWITCH_BT_PATH)) {
+		sprintf(extra, "set BTC Path %s",
+				(btc_mode == 0)? "Normal":((btc_mode == 1)? "WL":
+				((btc_mode == 2)? "BT":"DEFAULT WL")));
+
+	} else
+		sprintf(extra, "set BTC Path Fail");
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_get_he(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct registry_priv *regsty = &padapter->registrypriv;
+
+#ifdef CONFIG_80211AX_HE
+	if (!REGSTY_IS_11AX_ENABLE(regsty) ||
+		!is_supported_he(regsty->wireless_mode))
+		sprintf(extra, "false");
+	 else
+		sprintf(extra, "true");
+#endif
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_mp_band(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct registry_priv *regsty = &padapter->registrypriv;
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 band = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	extra[wrqu->data.length] = '\0';
+	band = rtw_atoi(extra);
+
+	if (rtw_hw_is_band_support(adapter_to_dvobj(padapter), band)) {
+		pmp_priv->band = band;
+		sprintf(extra, "Set band to %s", band == 0 ? "2.4GHz" : (band == 1 ? "5GHz" : "6GHz"));
+	} else
+		sprintf(extra, "band not supported");
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+
+}
+
+static inline void dump_buf(u8 *buf, u32 len)
+{
+	u32 i;
+
+	RTW_INFO("-----------------Len %d----------------\n", len);
+	for (i = 0; i < len; i++)
+		RTW_INFO("%2.2x-", *(buf + i));
+	RTW_INFO("\n");
+}
+
+int rtw_mp_link(struct net_device *dev,
+			struct iw_request_info *info,
+			struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv;
+	char	input[RTW_IWD_MAX_LEN];
+	int		bgetrxdata = 0, btxdata = 0, bsetbt = 0;
+	int err = 0;
+	u32 i = 0, datalen = 0,jj, kk, waittime = 0;
+	u16 val = 0x00, ret = 0;
+	char *pextra = NULL;
+	u8 *setdata = NULL;
+	char *pch, *ptmp, *token, *tmp[4] = {0x00, 0x00, 0x00};
+
+	pmp_priv = &padapter->mppriv;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+
+	_rtw_memset(extra, 0, wrqu->length);
+
+	RTW_INFO("%s: in=%s\n", __func__, input);
+
+	bgetrxdata =  (strncmp(input, "rxdata", 6) == 0) ? 1 : 0; /* strncmp TRUE is 0*/
+	btxdata =  (strncmp(input, "txdata", 6) == 0) ? 1 : 0; /* strncmp TRUE is 0*/
+	bsetbt =  (strncmp(input, "setbt", 5) == 0) ? 1 : 0; /* strncmp TRUE is 0*/
+
+	if (bgetrxdata) {
+		RTW_INFO("%s: in= 1 \n", __func__);
+		if (pmp_priv->mplink_brx == _TRUE) {
+
+				while (waittime < 100 && pmp_priv->mplink_brx == _FALSE) {
+						if (pmp_priv->mplink_brx == _FALSE)
+							rtw_msleep_os(10);
+						/*else
+							break;*/
+						waittime++;
+				}
+
+					sprintf(extra, "\n");
+					pextra = extra + strlen(extra);
+					for (i = 0; i < pmp_priv->mplink_rx_len; i ++) {
+						pextra += sprintf(pextra, "%02x:", pmp_priv->mplink_buf[i]);
+					}
+					_rtw_memset(pmp_priv->mplink_buf, '\0' , sizeof(pmp_priv->mplink_buf));
+					pmp_priv->mplink_brx = _FALSE;
+	
+		}
+	} else if (btxdata) {
+		struct pkt_attrib *pattrib;
+
+		pch = input;
+		setdata = rtw_zmalloc(1024);
+		if (setdata == NULL) {
+			err = -ENOMEM;
+			goto exit;
+		}
+
+		i = 0;
+		while ((token = strsep(&pch, ",")) != NULL) {
+			if (i > 2)
+				break;
+			tmp[i] = token;
+			i++;
+		}
+
+		/* tmp[0],[1],[2] */
+		/* txdata,00e04c871200........... */
+		if (strcmp(tmp[0], "txdata") == 0) {
+			if (tmp[1] == NULL) {
+				err = -EINVAL;
+				goto exit;
+			}
+		}
+
+		datalen = strlen(tmp[1]);
+		if (datalen % 2) {
+			err = -EINVAL;
+			goto exit;
+		}
+		datalen /= 2;
+		if (datalen == 0) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		RTW_INFO("%s: data len=%d\n", __FUNCTION__, datalen);
+		RTW_INFO("%s: tx data=%s\n", __FUNCTION__, tmp[1]);
+
+		for (jj = 0, kk = 0; jj < datalen; jj++, kk += 2)
+			setdata[jj] = key_2char2num(tmp[1][kk], tmp[1][kk + 1]);
+
+		dump_buf(setdata, datalen);
+		_rtw_memset(pmp_priv->mplink_buf, '\0' , sizeof(pmp_priv->mplink_buf));
+		_rtw_memcpy(pmp_priv->mplink_buf, setdata, datalen);
+
+		pattrib = &pmp_priv->tx.attrib;
+		pattrib->pktlen = datalen;
+		pmp_priv->tx.count = 1;
+		pmp_priv->tx.stop = 0;
+		pmp_priv->mplink_btx = _TRUE;
+		rtw_mp_set_packet_tx(padapter);
+		pmp_priv->mode = MP_PACKET_TX;
+
+	} else if (bsetbt) {
+
+#if 0
+#ifdef CONFIG_BTC
+		pch = input;
+		i = 0;
+
+		while ((token = strsep(&pch, ",")) != NULL) {
+			if (i > 3)
+				break;
+			tmp[i] = token;
+			i++;
+		}
+
+		if (tmp[1] == NULL) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		if (strcmp(tmp[1], "scbd") == 0) {
+			u16 org_val = 0x8002, pre_val, read_score_board_val;
+			u8 state;
+
+			pre_val = (rtw_read16(padapter,(0xaa))) & 0x7fff;
+
+			if (tmp[2] != NULL) {
+				state = simple_strtoul(tmp[2], &ptmp, 10);
+
+				if (state)
+						org_val = org_val | BIT6;
+				else
+						org_val = org_val & (~BIT6);
+
+				if (org_val != pre_val) {
+					pre_val = org_val;
+					rtw_write16(padapter, 0xaa, org_val);
+					RTW_INFO("%s,setbt scbd write org_val = 0x%x , pre_val = 0x%x\n", __func__, org_val, pre_val);
+				} else {
+					RTW_INFO("%s,setbt scbd org_val = 0x%x ,pre_val = 0x%x\n", __func__, org_val, pre_val);
+				}
+			} else {
+					read_score_board_val = (rtw_read16(padapter,(0xaa))) & 0x7fff;
+					RTW_INFO("%s,read_score_board_val = 0x%x\n", __func__, read_score_board_val);
+			}
+			goto exit;
+
+		} else if (strcmp(tmp[1], "testmode") == 0) {
+
+			if (tmp[2] == NULL) {
+				err = -EINVAL;
+				goto exit;
+			}
+
+			val = simple_strtoul(tmp[2], &ptmp, 16);
+			RTW_INFO("get tmp, type  %s, val =0x%x!\n", tmp[1], val);
+
+			if (tmp[2] != NULL) {
+				_rtw_memset(extra, 0, wrqu->length);
+				ret = rtw_btcoex_btset_testmode(padapter, val);
+				if (!CHECK_STATUS_CODE_FROM_BT_MP_OPER_RET(ret, BT_STATUS_BT_OP_SUCCESS)) {
+					RTW_INFO("%s: BT_OP fail = 0x%x!\n", __FUNCTION__, val);
+					sprintf(extra, "BT_OP fail  0x%x!\n", val);
+				} else
+					sprintf(extra, "Set BT_OP 0x%x done!\n", val);
+			}
+
+		}
+#endif /* CONFIG_BTC */
+#endif
+	}
+
+exit:
+	if (setdata)
+		rtw_mfree(setdata, 1024);
+
+	wrqu->length = strlen(extra);
+	return err;
+
+}
+
+int rtw_mp_gpio(struct net_device *dev,
+		struct iw_request_info *info,
+		struct iw_point *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	char input[RTW_IWD_MAX_LEN];
+	u8 gpio_id, gpio_enable;
+	int ret = 0;
+
+	if (rtw_do_mp_iwdata_len_chk(__func__, (wrqu->length + 1)))
+		return -EFAULT;
+
+	if (wrqu->length > 128)
+		return -EFAULT;
+
+	_rtw_memset(input, 0, sizeof(input));
+
+	if (copy_from_user(input, wrqu->pointer, wrqu->length))
+		return -EFAULT;
+	input[wrqu->length] = '\0';
+
+	RTW_INFO("%s: input = %s\n", __func__, input);
+	_rtw_memset(extra, 0, wrqu->length);
+
+	ret = sscanf(input, "%hhd,%hhd", &gpio_id, &gpio_enable);
+	if (ret < 2){
+		return -EINVAL;
+	}
+	else if (gpio_id > 15) {
+		return -EINVAL;
+	}
+	else if (gpio_enable != 0 && gpio_enable != 1) {
+		return -EINVAL;
+	}
+
+	RTW_INFO("%s: gpio_id = %hhd, gpio_enable = %hhd\n", __func__, gpio_id , gpio_enable);
+
+	pmp_priv->gpio_id = gpio_id;
+	pmp_priv->gpio_enable = gpio_enable;
+
+	SetGpio(padapter);
+
+	sprintf(extra, "Set gpio_id:%d, gpio_enable:%d => done\n", gpio_id, gpio_enable);
+	wrqu->length = strlen(extra);
+
+	return 0;
+}
+
+int rtw_mp_mac_loopbk(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	u8 speed = 0x00;
+	u32 pktnum = 0, i = 0;
+	u8 bspeed = 0, bonly_speed = 0;
+
+	if (copy_from_user(extra, wrqu->data.pointer, wrqu->data.length))
+		return -EFAULT;
+
+	extra[wrqu->data.length] = '\0';
+
+	if(strncmp(extra, "speed=", 6) == 0){ /* strncmp TRUE is 0*/
+		if (sscanf(extra, "speed=%hhd,pkt=%d", &speed, &pktnum) > 1) {
+			RTW_INFO("speed=0x%02x, pkt=%d\n", speed, pktnum);
+			pmp_priv->loopbk_speed = speed;
+			bspeed = 1;
+		} else if (sscanf(extra, "speed=%hhd", &speed) > 0) {
+			RTW_INFO("only speed=0x%02x\n", speed);
+			bspeed = 1;
+			bonly_speed = 1;
+		}
+	} else if (sscanf(extra, "pkt=%d", &pktnum))
+		RTW_INFO("only pkt=%d\n", pktnum);
+
+	if (bspeed) {
+		rtw_mp_phl_set_mac_loopbk_speed(padapter);
+		if (bonly_speed == 1) {
+			sprintf(extra , "MAC config speed done\n");
+			wrqu->data.length = strlen(extra);
+			return 0;
+		}
+	}
+	rtw_mp_phl_set_mac_loopbk(padapter);
+	pmp_priv->bloopback = _TRUE;
+	RTW_INFO("Enter MAC Loopback mode !\n");
+
+	if (pktnum > 0) {
+		pmp_priv->rtw_mp_tx_method = RTW_MP_TMACT_TX;
+		pmp_priv->tx.sended = 0;
+		pmp_priv->tx.stop = 0;
+		pmp_priv->tx_pktcount = 0;
+		pmp_priv->tx.count = pktnum;
+		pmp_priv->rx_pktcount = 0;
+
+		if (pmp_priv->pktInterval == 100 || pmp_priv->pktInterval == 0)
+			 pmp_priv->pktInterval = 2000;
+
+		pmp_priv->tx.PktTxThread = rtw_thread_start(
+			mp_xmit_phl_packet_thread, pmp_priv, "RTW_MP_Tx_THREAD");
+		if (pmp_priv->tx.PktTxThread == NULL)
+			RTW_ERR("Create PktTx Thread Fail !!!!!\n");
+
+		while(1) {
+			rtw_msleep_os(10);
+			if (pmp_priv->rx_pktcount == pktnum) {
+				sprintf(extra , "MAC Loopback success\n");
+				break;
+			} else if (i > 200) {
+				if(pmp_priv->rx_pktcount != pktnum)
+					sprintf(extra , "MAC Loopback fail\n");
+				break;
+			}
+			i++;
+		}
+		RTW_INFO("Rx cnt=%d!\n", pmp_priv->rx_pktcount);
+#if 0
+#ifdef CONFIG_PCI_HCI
+		if (rtw_mp_get_tx_req_recycle(padapter) == 0)
+			sprintf(extra , "MAC Loopback [Tx Report] Fail\n");
+#endif
+#endif
+	} else {
+		sprintf(extra , "Error Format ! ,\
+			Please input : speed=[0xhex],pkt=[int]\n\
+			or pkt=[int]\n");
+	}
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+
+}
+
+int rtw_mp_mac_iotest(struct net_device *dev,
+			 struct iw_request_info *info,
+			 union iwreq_data *wrqu, char *extra)
+{
+	_adapter *padapter = rtw_netdev_priv(dev);
+	struct mp_priv *pmp_priv = (struct mp_priv *)&padapter->mppriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+
+	u32 val32 = rtw_phl_read32(dvobj->phl, 0xCC20);
+	val32 |= BIT0;
+	rtw_phl_write32(dvobj->phl, 0xCC20 , val32);
+
+	rtw_mp_phl_set_mac_io_test(padapter);
+
+	if (pmp_priv->mac_iotest_res)
+		sprintf(extra , "mac io test success\n");
+	else
+		sprintf(extra , "mac io test fail\n");
+
+	wrqu->data.length = strlen(extra);
+	return 0;
+}
+
+int rtw_priv_mp_get(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wdata, char *extra)
+{
+
+	struct iw_point *wrqu = (struct iw_point *)wdata;
+	u32 subcmd = wrqu->flags;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	int status = 0;
+	u8 i = 0;
+	char *pch = extra;
+
+	if (!is_primary_adapter(padapter)) {
+		RTW_INFO("MP mode only primary Adapter support, iface id = %d,\n", padapter->iface_id);
+		RTW_INFO("***Please use Primary Adapter:["ADPT_FMT"]***\n", ADPT_ARG(GET_PRIMARY_ADAPTER(padapter)));
+		pch +=sprintf(pch, "Only primary Adapter support MP CMD\n");
+		pch +=sprintf(pch, "Please use Primary Adapter:"ADPT_FMT"", ADPT_ARG(GET_PRIMARY_ADAPTER(padapter)));
+		wrqu->length = strlen(extra);
+		return status;
+	}
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		_adapter *iface = dvobj->padapters[i];
+
+		if (iface == NULL)
+			continue;
+		if (rtw_is_adapter_up(iface) == _FALSE)
+			continue;
+		if (MLME_IS_AP(iface)) {
+			RTW_INFO("Adapter:["ADPT_FMT"], Please Leave AP mode or Down Interface\n", ADPT_ARG(iface));
+			pch += sprintf(pch, "Check Adapter:"ADPT_FMT",\n\
+								Please Leave AP mode or Down Interface\n", ADPT_ARG(iface));
+			wrqu->length = strlen(extra);
+			return status;
+		}
+	}
+
+	RTW_INFO("%s mutx in %d\n", __func__, subcmd);
+
+	switch (subcmd) {
+	case MP_START:
+		RTW_INFO("set case mp_start\n");
+		status = rtw_mp_start(dev, info, wrqu, extra);
+		break;
+	case MP_STOP:
+		RTW_INFO("set case mp_stop\n");
+		status = rtw_mp_stop(dev, info, wrqu, extra);
+		break;
+	case MP_BANDWIDTH:
+		RTW_INFO("set case mp_bandwidth\n");
+		status = rtw_mp_bandwidth(dev, info, wrqu, extra);
+		break;
+	case MP_RESET_STATS:
+		RTW_INFO("set case MP_RESET_STATS\n");
+		status = rtw_mp_reset_stats(dev, info, wrqu, extra);
+		break;
+	case MP_SetRFPathSwh:
+		RTW_INFO("set MP_SetRFPathSwitch\n");
+		status = rtw_mp_SetRFPath(dev, info, wrqu, extra);
+		break;
+	case WRITE_REG:
+		status = rtw_mp_write_reg(dev, info, wrqu, extra);
+		break;
+	case WRITE_RF:
+		status = rtw_mp_write_rf(dev, info, wrqu, extra);
+		break;
+	case MP_PHYPARA:
+		RTW_INFO("mp_get  MP_PHYPARA\n");
+		status = rtw_mp_phypara(dev, info, wrqu, extra);
+		break;
+	case MP_CHANNEL:
+		RTW_INFO("set case mp_channel\n");
+		status = rtw_mp_channel(dev , info, wrqu, extra);
+		break;
+	case MP_TRXSC_OFFSET:
+		RTW_INFO("set case rtw_mp_trxsc_offset\n");
+		status = rtw_mp_trxsc_offset(dev , info, wrqu, extra);
+		break;
+	case READ_REG:
+		RTW_INFO("mp_get  READ_REG\n");
+		status = rtw_mp_read_reg(dev, info, wrqu, extra);
+		break;
+	case READ_RF:
+		RTW_INFO("mp_get  READ_RF\n");
+		status = rtw_mp_read_rf(dev, info, wrqu, extra);
+		break;
+	case MP_RATE:
+		RTW_INFO("set case mp_rate\n");
+		status = rtw_mp_rate(dev, info, wrqu, extra);
+		break;
+	case MP_TXPOWER:
+		RTW_INFO("set case MP_TXPOWER\n");
+		status = rtw_mp_txpower(dev, info, wrqu, extra);
+		break;
+	case MP_ANT_TX:
+		RTW_INFO("set case MP_ANT_TX\n");
+		status = rtw_mp_ant_tx(dev, info, wrqu, extra);
+		break;
+	case MP_ANT_RX:
+		RTW_INFO("set case MP_ANT_RX\n");
+		status = rtw_mp_ant_rx(dev, info, wrqu, extra);
+		break;
+	case MP_QUERY:
+		status = rtw_mp_trx_query(dev, info, wrqu, extra);
+		break;
+	case MP_CTX:
+		RTW_INFO("set case MP_CTX\n");
+		status = rtw_mp_ctx(dev, info, wrqu, extra);
+		break;
+	case MP_ARX:
+		RTW_INFO("set case MP_ARX\n");
+		status = rtw_mp_arx(dev, info, wrqu, extra);
+		break;
+	case MP_DUMP:
+		RTW_INFO("set case MP_DUMP\n");
+		status = rtw_mp_dump(dev, info, wrqu, extra);
+		break;
+	case MP_PSD:
+		RTW_INFO("set case MP_PSD\n");
+		status = rtw_mp_psd(dev, info, wrqu, extra);
+		break;
+	case MP_THER:
+		RTW_INFO("set case MP_THER\n");
+		status = rtw_mp_thermal(dev, info, wrqu, extra);
+		break;
+	case MP_PwrCtlDM:
+		RTW_INFO("set MP_PwrCtlDM\n");
+		status = rtw_mp_PwrCtlDM(dev, info, wrqu, extra);
+		break;
+	case MP_QueryDrvStats:
+		RTW_INFO("mp_get MP_QueryDrvStats\n");
+		status = rtw_mp_QueryDrv(dev, info, wdata, extra);
+		break;
+	case MP_PWRTRK:
+		RTW_INFO("set case MP_PWRTRK\n");
+		status = rtw_mp_pwrtrk(dev, info, wrqu, extra);
+		break;
+	case MP_SET_TSSIDE:
+		RTW_INFO("set case MP_TSSI_DE\n");
+		status = rtw_mp_set_tsside(dev, info, wrqu, extra);
+		break;
+	case EFUSE_SET:
+		RTW_INFO("set case efuse set\n");
+		status = rtw_ioctl_efuse_set(dev, info, wdata, extra);
+		break;
+	case EFUSE_GET:
+		RTW_INFO("efuse get EFUSE_GET\n");
+		status = rtw_ioctl_efuse_get(dev, info, wdata, extra);
+		break;
+	case MP_GET_TXPOWER_INX:
+		RTW_INFO("mp_get MP_GET_TXPOWER_INX\n");
+		status = rtw_mp_txpower_index(dev, info, wrqu, extra);
+		break;
+	case MP_GETVER:
+		RTW_INFO("mp_get MP_GETVER\n");
+		status = rtw_mp_getver(dev, info, wdata, extra);
+		break;
+	case MP_MON:
+		RTW_INFO("mp_get MP_MON\n");
+		status = rtw_mp_mon(dev, info, wdata, extra);
+		break;
+	case EFUSE_BT_MASK:
+		RTW_INFO("mp_get EFUSE_BT_MASK\n");
+		status = rtw_ioctl_efuse_bt_file_mask_load(dev, info, wdata, extra);
+		break;
+	case EFUSE_MASK:
+		RTW_INFO("mp_get EFUSE_MASK\n");
+		status = rtw_ioctl_efuse_file_mask_load(dev, info, wdata, extra);
+		break;
+	case EFUSE_FILE:
+		RTW_INFO("mp_get EFUSE_FILE\n");
+		status = rtw_ioctl_efuse_file_map_load(dev, info, wdata, extra);
+		break;
+	case EFUSE_FILE_STORE:
+		RTW_INFO("mp_get EFUSE_FILE_STORE\n");
+		/*status = rtw_efuse_file_map_store(dev, info, wdata, extra);*/
+		break;
+	case MP_TX:
+		RTW_INFO("mp_get MP_TX\n");
+		status = rtw_mp_tx(dev, info, wdata, extra);
+		break;
+	case MP_RX:
+		RTW_INFO("mp_get MP_RX\n");
+		status = rtw_mp_rx(dev, info, wdata, extra);
+		break;
+	case MP_HW_TX_MODE:
+		RTW_INFO("mp_get MP_HW_TX_MODE\n");
+		status = rtw_mp_hwtx(dev, info, wdata, extra);
+		break;
+	case MP_GET_TSSIDE:
+		RTW_INFO("mp_get TSSI_DE\n");
+		status = rtw_mp_get_tsside(dev, info, wrqu, extra);
+		break;
+#ifdef CONFIG_RTW_CUSTOMER_STR
+	case MP_CUSTOMER_STR:
+		RTW_INFO("customer str\n");
+		status = rtw_mp_customer_str(dev, info, wdata, extra);
+		break;
+#endif
+	case MP_PWRLMT:
+		RTW_INFO("mp_get MP_SETPWRLMT\n");
+		status = rtw_mp_pwrlmt(dev, info, wdata, extra);
+		break;
+	case  BT_EFUSE_FILE:
+		RTW_INFO("mp_get BT EFUSE_FILE\n");
+		status = rtw_ioctl_efuse_bt_file_map_load(dev, info, wdata, extra);
+		break;
+	case MP_SWRFPath:
+		RTW_INFO("mp_get MP_SWRFPath\n");
+		status = rtw_mp_switch_rf_path(dev, info, wrqu, extra);
+		break;
+	case MP_LINK:
+		RTW_INFO("mp_get MP_LINK\n");
+		status = rtw_mp_link(dev, info, wrqu, extra);
+		break;
+	case MP_DPK_TRK:
+		RTW_INFO("mp_get MP_DPK_TRK\n");
+		status = rtw_mp_dpk_track(dev, info, wdata, extra);
+		break;
+	case MP_DPK:
+		RTW_INFO("set MP_DPK\n");
+		status = rtw_mp_dpk(dev, info, wdata, extra);
+		break;
+	case MP_GET_PHL_TEST:
+		RTW_INFO("mp_get MP_GET_PHL_TEST\n");
+		status = rtw_mp_get_phl_io(dev, info, wrqu, extra);
+		break;
+	case MP_SET_PHL_TEST:
+		RTW_INFO("mp_get MP_SET_PHL_TEST\n");
+		status = rtw_mp_set_phl_io(dev, info, wrqu, extra);
+		break;
+	case MP_SET_PHL_TX_PATTERN:
+		RTW_INFO("mp_get MP_SET_PHL_TEST\n");
+		status = rtw_mp_tx_pattern_idx(dev, info, wdata, extra);
+		break;
+	case MP_SET_PHL_PLCP_TX_DATA:
+		RTW_INFO("mp_get MP_SET_PHL_PLCP_TX_DATA\n");
+		status = rtw_mp_tx_plcp_tx_data(dev, info, wdata, extra);
+		break;
+	case MP_SET_PHL_PLCP_TX_USER:
+		RTW_INFO("mp_get MP_SET_PHL_PLCP_TX_USER\n");
+		status = rtw_mp_tx_plcp_tx_user(dev, info, wdata, extra);
+		break;
+	case MP_SET_PHL_TX_METHOD:
+		RTW_INFO("mp_get MP_SET_PHL_TX_METHOD\n");
+		status = rtw_mp_tx_method(dev, info, wdata, extra);
+		break;
+	case MP_SET_PHL_CONIFG_PHY_NUM:
+		RTW_INFO("mp_get MP_SET_PHL_CONIFG_PHY_NUM\n");
+		status = rtw_mp_config_phy(dev, info, wdata, extra);
+		break;
+	case MP_PHL_RFK:
+		RTW_INFO("mp_get MP_PHL_RFK\n");
+		status = rtw_mp_phl_rfk(dev, info, wdata, extra);
+		break;
+	case MP_PHL_BTC_PATH:
+		RTW_INFO("mp_get MP_PHL_BTC_PATH\n");
+		status = rtw_mp_phl_btc_path(dev, info, wdata, extra);
+		break;
+	case MP_GET_HE:
+		RTW_INFO("mp_get MP_GET_HE\n");
+		status = rtw_mp_get_he(dev, info, wdata, extra);
+		break;
+	case MP_BAND:
+		RTW_INFO("mp_get MP_BAND\n");
+		status = rtw_mp_band(dev, info, wdata, extra);
+		break;
+	case MP_UUID:
+		RTW_INFO("set case MP_UUID\n");
+		status = rtw_mp_uuid(dev, info, wrqu, extra);
+		break;
+	case MP_GPIO:
+		RTW_INFO("set case MP_GPIO\n");
+		status = rtw_mp_gpio(dev, info, wrqu, extra);
+		break;
+	case MP_MACLOOPBK:
+		RTW_INFO("set case MP_MACLOOPBK\n");
+		status = rtw_mp_mac_loopbk(dev, info, wdata, extra);
+		break;
+	case MP_MAC_IOTEST:
+		RTW_INFO("set case MP_MAC_IOTEST\n");
+		status = rtw_mp_mac_iotest(dev, info, wdata, extra);
+		break;
+	default:
+		status = -EIO;
+	}
+
+	return status;
+}
+
+int rtw_priv_mp_set(struct net_device *dev,
+			   struct iw_request_info *info,
+			   union iwreq_data *wdata, char *extra)
+{
+
+	struct iw_point *wrqu = (struct iw_point *)wdata;
+	u32 subcmd = wrqu->flags;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	int status = 0;
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (!is_primary_adapter(padapter)) {
+		RTW_INFO("MP mode only primary Adapter support\n");
+		return -EIO;
+	}
+#endif
+
+	RTW_INFO("%s mutx in %d\n", __func__, subcmd);
+
+	switch (subcmd) {
+	case MP_DISABLE_BT_COEXIST:
+		RTW_INFO("set case MP_DISABLE_BT_COEXIST\n");
+		status = rtw_mp_disable_bt_coexist(dev, info, wdata, extra);
+		break;
+	case MP_IQK:
+		RTW_INFO("set MP_IQK\n");
+		status = rtw_mp_iqk(dev, info, wrqu, extra);
+		break;
+	case MP_LCK:
+		RTW_INFO("set MP_LCK\n");
+		status = rtw_mp_lck(dev, info, wrqu, extra);
+	break;
+
+	default:
+		status = -EIO;
+	}
+
+	RTW_INFO("%s mutx done %d\n", __func__, subcmd);
+
+	return status;
+}
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/mlme_linux.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/mlme_linux.c
new file mode 100644
index 000000000000..3563bd62c448
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/mlme_linux.c
@@ -0,0 +1,504 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+
+#define _MLME_OSDEP_C_
+
+#include <drv_types.h>
+
+
+#ifdef RTK_DMP_PLATFORM
+void Linkup_workitem_callback(struct work_struct *work)
+{
+	struct mlme_priv *pmlmepriv = container_of(work, struct mlme_priv, Linkup_workitem);
+	_adapter *padapter = container_of(pmlmepriv, _adapter, mlmepriv);
+
+
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 12))
+	kobject_uevent(&padapter->pnetdev->dev.kobj, KOBJ_LINKUP);
+#else
+	kobject_hotplug(&padapter->pnetdev->class_dev.kobj, KOBJ_LINKUP);
+#endif
+
+}
+
+void Linkdown_workitem_callback(struct work_struct *work)
+{
+	struct mlme_priv *pmlmepriv = container_of(work, struct mlme_priv, Linkdown_workitem);
+	_adapter *padapter = container_of(pmlmepriv, _adapter, mlmepriv);
+
+
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 12))
+	kobject_uevent(&padapter->pnetdev->dev.kobj, KOBJ_LINKDOWN);
+#else
+	kobject_hotplug(&padapter->pnetdev->class_dev.kobj, KOBJ_LINKDOWN);
+#endif
+
+}
+#endif
+
+extern void rtw_indicate_wx_assoc_event(_adapter *padapter);
+extern void rtw_indicate_wx_disassoc_event(_adapter *padapter);
+
+void rtw_os_indicate_connect(_adapter *adapter)
+{
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == _TRUE) ||
+	    (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == _TRUE))
+		rtw_cfg80211_ibss_indicate_connect(adapter);
+	else
+		rtw_cfg80211_indicate_connect(adapter);
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	rtw_indicate_wx_assoc_event(adapter);
+
+#ifdef CONFIG_RTW_MESH
+#if CONFIG_RTW_MESH_CTO_MGATE_CARRIER
+	if (!rtw_mesh_cto_mgate_required(adapter))
+#endif
+#endif
+		rtw_netif_carrier_on(adapter->pnetdev);
+
+	if (adapter->pid[2] != 0)
+		rtw_signal_process(adapter->pid[2], SIGALRM);
+
+#ifdef RTK_DMP_PLATFORM
+	_set_workitem(&adapter->mlmepriv.Linkup_workitem);
+#endif
+
+
+}
+
+extern void indicate_wx_scan_complete_event(_adapter *padapter);
+void rtw_os_indicate_scan_done(_adapter *padapter, bool aborted)
+{
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_indicate_scan_done(padapter, aborted);
+#endif
+	indicate_wx_scan_complete_event(padapter);
+}
+
+static RT_PMKID_LIST   backupPMKIDList[NUM_PMKID_CACHE];
+void rtw_reset_securitypriv(_adapter *adapter)
+{
+	u8	backupPMKIDIndex = 0;
+	u8	backupTKIPCountermeasure = 0x00;
+	u32	backupTKIPcountermeasure_time = 0;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	/* add for CONFIG_IEEE80211W, none 11w also can use */
+
+	_rtw_spinlock_bh(&adapter->security_key_mutex);
+
+	if (adapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) { /* 802.1x */
+		u8 backup_sw_encrypt, backup_sw_decrypt;
+
+		backup_sw_encrypt = adapter->securitypriv.sw_encrypt;
+		backup_sw_decrypt = adapter->securitypriv.sw_decrypt;
+
+		/* Added by Albert 2009/02/18 */
+		/* We have to backup the PMK information for WiFi PMK Caching test item. */
+		/*  */
+		/* Backup the btkip_countermeasure information. */
+		/* When the countermeasure is trigger, the driver have to disconnect with AP for 60 seconds. */
+
+		_rtw_memset(&backupPMKIDList[0], 0x00, sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
+
+		_rtw_memcpy(&backupPMKIDList[0], &adapter->securitypriv.PMKIDList[0], sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
+		backupPMKIDIndex = adapter->securitypriv.PMKIDIndex;
+		backupTKIPCountermeasure = adapter->securitypriv.btkip_countermeasure;
+		backupTKIPcountermeasure_time = adapter->securitypriv.btkip_countermeasure_time;
+		_rtw_memset((unsigned char *)&adapter->securitypriv, 0, sizeof(struct security_priv));
+		_rtw_memset((unsigned char *)&adapter_link->securitypriv, 0, sizeof(struct link_security_priv));
+
+		/* Added by Albert 2009/02/18 */
+		/* Restore the PMK information to securitypriv structure for the following connection. */
+		_rtw_memcpy(&adapter->securitypriv.PMKIDList[0], &backupPMKIDList[0], sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
+		adapter->securitypriv.PMKIDIndex = backupPMKIDIndex;
+		adapter->securitypriv.btkip_countermeasure = backupTKIPCountermeasure;
+		adapter->securitypriv.btkip_countermeasure_time = backupTKIPcountermeasure_time;
+
+		adapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
+		adapter->securitypriv.ndisencryptstatus = Ndis802_11WEPDisabled;
+
+		adapter->securitypriv.extauth_status = WLAN_STATUS_UNSPECIFIED_FAILURE;
+
+		adapter->securitypriv.sw_encrypt = backup_sw_encrypt;
+		adapter->securitypriv.sw_decrypt = backup_sw_decrypt;
+
+	} else { /* reset values in securitypriv */
+		/* if(adapter->mlmepriv.fw_state & WIFI_STATION_STATE) */
+		/* { */
+		struct security_priv *psec_priv = &adapter->securitypriv;
+		struct link_security_priv *lsec_priv = &adapter_link->securitypriv;
+
+		psec_priv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+		psec_priv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		psec_priv->dot11PrivacyKeyIndex = 0;
+
+		psec_priv->dot118021XGrpPrivacy = _NO_PRIVACY_;
+		/* ToDo CONFIG_RTW_MLD: loop for all links */
+		lsec_priv->dot118021XGrpKeyid = 1;
+
+		psec_priv->ndisauthtype = Ndis802_11AuthModeOpen;
+		psec_priv->ndisencryptstatus = Ndis802_11WEPDisabled;
+		/* } */
+
+		psec_priv->extauth_status = WLAN_STATUS_UNSPECIFIED_FAILURE;
+	}
+	/* add for CONFIG_IEEE80211W, none 11w also can use */
+	_rtw_spinunlock_bh(&adapter->security_key_mutex);
+
+	RTW_INFO(FUNC_ADPT_FMT" - End to Disconnect\n", FUNC_ADPT_ARG(adapter));
+}
+
+void rtw_os_indicate_disconnect(_adapter *adapter,  u16 reason, u8 locally_generated)
+{
+	/* RT_PMKID_LIST   backupPMKIDList[NUM_PMKID_CACHE]; */
+
+
+	rtw_netif_carrier_off(adapter->pnetdev); /* Do it first for tx broadcast pkt after disconnection issue! */
+
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_indicate_disconnect(adapter,  reason, locally_generated);
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	rtw_indicate_wx_disassoc_event(adapter);
+
+#ifdef RTK_DMP_PLATFORM
+	_set_workitem(&adapter->mlmepriv.Linkdown_workitem);
+#endif
+	rtw_reset_securitypriv(adapter);
+}
+
+void rtw_report_sec_ie(_adapter *adapter, u8 authmode, u8 *sec_ie)
+{
+	uint	len;
+	u8	*buff, *p, i;
+	union iwreq_data wrqu;
+
+
+
+	buff = NULL;
+	if (authmode == _WPA_IE_ID_) {
+
+		buff = rtw_zmalloc(IW_CUSTOM_MAX);
+		if (NULL == buff) {
+			RTW_INFO(FUNC_ADPT_FMT ": alloc memory FAIL!!\n",
+				 FUNC_ADPT_ARG(adapter));
+			return;
+		}
+		p = buff;
+
+		p += sprintf(p, "ASSOCINFO(ReqIEs=");
+
+		len = sec_ie[1] + 2;
+		len = (len < IW_CUSTOM_MAX) ? len : IW_CUSTOM_MAX;
+
+		for (i = 0; i < len; i++)
+			p += sprintf(p, "%02x", sec_ie[i]);
+
+		p += sprintf(p, ")");
+
+		_rtw_memset(&wrqu, 0, sizeof(wrqu));
+
+		wrqu.data.length = p - buff;
+
+		wrqu.data.length = (wrqu.data.length < IW_CUSTOM_MAX) ? wrqu.data.length : IW_CUSTOM_MAX;
+
+#ifndef CONFIG_IOCTL_CFG80211
+		wireless_send_event(adapter->pnetdev, IWEVCUSTOM, &wrqu, buff);
+#endif
+
+		rtw_mfree(buff, IW_CUSTOM_MAX);
+	}
+
+
+}
+
+#ifdef CONFIG_AP_MODE
+
+void rtw_indicate_sta_assoc_event(_adapter *padapter, struct sta_info *psta)
+{
+	union iwreq_data wrqu;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+
+	if (psta == NULL)
+		return;
+
+	if (psta->phl_sta->aid > pstapriv->max_aid)
+		return;
+
+	if (pstapriv->sta_aid[psta->phl_sta->aid - 1] != psta)
+		return;
+
+
+	wrqu.addr.sa_family = ARPHRD_ETHER;
+
+	_rtw_memcpy(wrqu.addr.sa_data, psta->phl_sta->mac_addr, ETH_ALEN);
+
+	RTW_INFO("+rtw_indicate_sta_assoc_event\n");
+
+#ifndef CONFIG_IOCTL_CFG80211
+	wireless_send_event(padapter->pnetdev, IWEVREGISTERED, &wrqu, NULL);
+#endif
+
+}
+
+void rtw_indicate_sta_disassoc_event(_adapter *padapter, struct sta_info *psta)
+{
+	union iwreq_data wrqu;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+
+	if (psta == NULL)
+		return;
+
+	if (psta->phl_sta->aid > pstapriv->max_aid)
+		return;
+
+	if (pstapriv->sta_aid[psta->phl_sta->aid - 1] != psta)
+		return;
+
+
+	wrqu.addr.sa_family = ARPHRD_ETHER;
+
+	_rtw_memcpy(wrqu.addr.sa_data, psta->phl_sta->mac_addr, ETH_ALEN);
+
+	RTW_INFO("+rtw_indicate_sta_disassoc_event\n");
+
+#ifndef CONFIG_IOCTL_CFG80211
+	wireless_send_event(padapter->pnetdev, IWEVEXPIRED, &wrqu, NULL);
+#endif
+
+}
+
+
+#ifdef CONFIG_HOSTAPD_MLME
+
+static int mgnt_xmit_entry(struct sk_buff *skb, struct net_device *pnetdev)
+{
+	struct hostapd_priv *phostapdpriv = rtw_netdev_priv(pnetdev);
+	_adapter *padapter = (_adapter *)phostapdpriv->padapter;
+
+	/* RTW_INFO("%s\n", __FUNCTION__); */
+
+	return rtw_hal_hostap_mgnt_xmit_entry(padapter, skb);
+}
+
+static int mgnt_netdev_open(struct net_device *pnetdev)
+{
+	struct hostapd_priv *phostapdpriv = rtw_netdev_priv(pnetdev);
+
+	RTW_INFO("mgnt_netdev_open: MAC Address:" MAC_FMT "\n", MAC_ARG(pnetdev->dev_addr));
+
+
+	init_usb_anchor(&phostapdpriv->anchored);
+
+	rtw_netif_wake_queue(pnetdev);
+
+	rtw_netif_carrier_on(pnetdev);
+
+	return 0;
+}
+static int mgnt_netdev_close(struct net_device *pnetdev)
+{
+	struct hostapd_priv *phostapdpriv = rtw_netdev_priv(pnetdev);
+
+	RTW_INFO("%s\n", __FUNCTION__);
+
+	usb_kill_anchored_urbs(&phostapdpriv->anchored);
+
+	rtw_netif_carrier_off(pnetdev);
+
+	rtw_netif_stop_queue(pnetdev);
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+static const struct net_device_ops rtl871x_mgnt_netdev_ops = {
+	.ndo_open = mgnt_netdev_open,
+	.ndo_stop = mgnt_netdev_close,
+	.ndo_start_xmit = mgnt_xmit_entry,
+	#if 0
+	.ndo_set_mac_address = r871x_net_set_mac_address,
+	.ndo_get_stats = r871x_net_get_stats,
+	.ndo_do_ioctl = r871x_mp_ioctl,
+	#endif
+};
+#endif
+
+int hostapd_mode_init(_adapter *padapter)
+{
+	unsigned char mac[ETH_ALEN];
+	struct hostapd_priv *phostapdpriv;
+	struct net_device *pnetdev;
+
+	pnetdev = rtw_alloc_etherdev(sizeof(struct hostapd_priv));
+	if (!pnetdev)
+		return -ENOMEM;
+
+	/* SET_MODULE_OWNER(pnetdev); */
+	ether_setup(pnetdev);
+
+	/* pnetdev->type = ARPHRD_IEEE80211; */
+
+	phostapdpriv = rtw_netdev_priv(pnetdev);
+	phostapdpriv->pmgnt_netdev = pnetdev;
+	phostapdpriv->padapter = padapter;
+	padapter->phostapdpriv = phostapdpriv;
+
+	/* pnetdev->init = NULL; */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+
+	RTW_INFO("register rtl871x_mgnt_netdev_ops to netdev_ops\n");
+
+	pnetdev->netdev_ops = &rtl871x_mgnt_netdev_ops;
+
+#else
+
+	pnetdev->open = mgnt_netdev_open;
+
+	pnetdev->stop = mgnt_netdev_close;
+
+	pnetdev->hard_start_xmit = mgnt_xmit_entry;
+
+	/* pnetdev->set_mac_address = r871x_net_set_mac_address; */
+
+	/* pnetdev->get_stats = r871x_net_get_stats; */
+
+	/* pnetdev->do_ioctl = r871x_mp_ioctl; */
+
+#endif
+
+	pnetdev->watchdog_timeo = HZ; /* 1 second timeout */
+
+	/* pnetdev->wireless_handlers = NULL; */
+
+
+
+
+	if (dev_alloc_name(pnetdev, "mgnt.wlan%d") < 0)
+		RTW_INFO("hostapd_mode_init(): dev_alloc_name, fail!\n");
+
+
+	/* SET_NETDEV_DEV(pnetdev, pintfpriv->udev); */
+
+
+	mac[0] = 0x00;
+	mac[1] = 0xe0;
+	mac[2] = 0x4c;
+	mac[3] = 0x87;
+	mac[4] = 0x11;
+	mac[5] = 0x12;
+
+	_rtw_memcpy(pnetdev->dev_addr, mac, ETH_ALEN);
+
+
+	rtw_netif_carrier_off(pnetdev);
+
+
+	/* Tell the network stack we exist */
+	if (register_netdev(pnetdev) != 0) {
+		RTW_INFO("hostapd_mode_init(): register_netdev fail!\n");
+
+		if (pnetdev)
+			rtw_free_netdev(pnetdev);
+	}
+
+	return 0;
+
+}
+
+void hostapd_mode_unload(_adapter *padapter)
+{
+	struct hostapd_priv *phostapdpriv = padapter->phostapdpriv;
+	struct net_device *pnetdev = phostapdpriv->pmgnt_netdev;
+
+	unregister_netdev(pnetdev);
+	rtw_free_netdev(pnetdev);
+
+}
+
+#endif
+#endif
+
+#ifdef CONFIG_DFS_MASTER
+void rtw_os_indicate_radar_detected(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 cch, enum channel_width bw)
+{
+	rtw_nlrtw_radar_detect_event(rfctl, band_idx, cch, bw);
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_radar_detected_event(rfctl, band_idx, cch, bw);
+#endif
+}
+
+void rtw_os_indicate_cac_started(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_cac_started_event(rfctl, band_idx, ifbmp, cch, bw);
+#endif
+}
+
+void rtw_os_indicate_cac_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	rtw_nlrtw_cac_finish_event(rfctl, band_idx, ifbmp, cch, bw);
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_cac_finished_event(rfctl, band_idx, ifbmp, cch, bw);
+#endif
+}
+
+void rtw_os_indicate_cac_aborted(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	rtw_nlrtw_cac_abort_event(rfctl, band_idx, ifbmp, cch, bw);
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_cac_aborted_event(rfctl, band_idx, ifbmp, cch, bw);
+#endif
+}
+
+void rtw_os_force_cac_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_cac_force_finished(rfctl, band_idx, ifbmp, cch, bw);
+#endif
+}
+
+void rtw_os_indicate_nop_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw)
+{
+	rtw_nlrtw_nop_finish_event(rfctl, band_idx, band, cch, bw);
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_nop_finished_event(rfctl, band_idx, band, cch, bw);
+#endif
+}
+
+void rtw_os_indicate_nop_started(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw, bool called_on_cmd_thd)
+{
+	rtw_nlrtw_nop_start_event(rfctl, band_idx, band, cch, bw);
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_nop_started_event(rfctl, band_idx, band, cch, bw, called_on_cmd_thd);
+#endif
+}
+#endif /* CONFIG_DFS_MASTER */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/nlrtw.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/nlrtw.c
new file mode 100644
index 000000000000..a1c3e6ef0378
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/nlrtw.c
@@ -0,0 +1,596 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_NLRTW_C_
+
+#include <drv_types.h>
+#include "nlrtw.h"
+
+#ifdef CONFIG_RTW_NLRTW
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0))
+#include <uapi/linux/netlink.h>
+#endif
+
+
+enum nlrtw_cmds {
+	NLRTW_CMD_UNSPEC,
+
+	NLRTW_CMD_CHANNEL_UTILIZATION,
+	NLRTW_CMD_REG_CHANGE,
+	NLRTW_CMD_REG_BEACON_HINT,
+	NLRTW_CMD_RADAR_EVENT,
+	NLRTW_CMD_RADIO_OPMODE,
+
+	__NLRTW_CMD_AFTER_LAST,
+	NLRTW_CMD_MAX = __NLRTW_CMD_AFTER_LAST - 1
+};
+
+enum nlrtw_attrs {
+	NLRTW_ATTR_UNSPEC,
+
+	NLRTW_ATTR_WIPHY_NAME,
+	NLRTW_ATTR_CHANNEL_UTILIZATIONS,
+	NLRTW_ATTR_CHANNEL_UTILIZATION_THRESHOLD,
+	NLRTW_ATTR_CHANNEL_CENTER,
+	NLRTW_ATTR_CHANNEL_WIDTH,
+	NLRTW_ATTR_RADAR_EVENT,
+	NLRTW_ATTR_OP_CLASS,
+	NLRTW_ATTR_OP_CHANNEL,
+	NLRTW_ATTR_OP_TXPWR_MAX,
+	NLRTW_ATTR_IF_OPMODES,
+	NLRTW_ATTR_CHANNEL_BAND,
+
+	__NLRTW_ATTR_AFTER_LAST,
+	NUM_NLRTW_ATTR = __NLRTW_ATTR_AFTER_LAST,
+	NLRTW_ATTR_MAX = __NLRTW_ATTR_AFTER_LAST - 1
+};
+
+enum nlrtw_ch_util_attrs {
+	__NLRTW_ATTR_CHANNEL_UTILIZATION_INVALID,
+
+	NLRTW_ATTR_CHANNEL_UTILIZATION_VALUE,
+	NLRTW_ATTR_CHANNEL_UTILIZATION_BSSID,
+
+	__NLRTW_ATTR_CHANNEL_UTILIZATION_AFTER_LAST,
+	NUM_NLRTW_ATTR_CHANNEL_UTILIZATION = __NLRTW_ATTR_CHANNEL_UTILIZATION_AFTER_LAST,
+	NLRTW_ATTR_CHANNEL_UTILIZATION_MAX = __NLRTW_ATTR_CHANNEL_UTILIZATION_AFTER_LAST - 1
+};
+
+enum nlrtw_radar_event {
+	NLRTW_RADAR_DETECTED,
+	NLRTW_RADAR_CAC_FINISHED,
+	NLRTW_RADAR_CAC_ABORTED,
+	NLRTW_RADAR_NOP_FINISHED,
+	NLRTW_RADAR_NOP_STARTED, /* NON_OCP started not by local radar detection */
+};
+
+enum nlrtw_if_opmode_attrs {
+	NLRTW_IF_OPMODE_UNSPEC,
+
+	NLRTW_IF_OPMODE_MACADDR,
+	NLRTW_IF_OPMODE_OP_CLASS,
+	NLRTW_IF_OPMODE_OP_CHANNEL,
+
+	__NLRTW_IF_OPMODE_ATTR_AFTER_LAST,
+	NUM_NLRTW_IF_OPMODE_ATTR = __NLRTW_IF_OPMODE_ATTR_AFTER_LAST,
+	NLRTW_IF_OPMODE_ATTR_MAX = __NLRTW_IF_OPMODE_ATTR_AFTER_LAST - 1
+};
+
+static int nlrtw_ch_util_set(struct sk_buff *skb, struct genl_info *info)
+{
+	unsigned int msg;
+
+	if (!info->attrs[NLRTW_ATTR_CHANNEL_UTILIZATION_THRESHOLD])
+		return -EINVAL;
+	msg = nla_get_u8(info->attrs[NLRTW_ATTR_CHANNEL_UTILIZATION_THRESHOLD]);
+
+	return 0;
+}
+
+static struct nla_policy nlrtw_genl_policy[NUM_NLRTW_ATTR] = {
+	[NLRTW_ATTR_CHANNEL_UTILIZATION_THRESHOLD] = { .type = NLA_U8 },
+};
+
+static struct genl_ops nlrtw_genl_ops[] = {
+	{
+		.cmd = NLRTW_CMD_CHANNEL_UTILIZATION,
+		.flags = 0,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0)
+		.policy = nlrtw_genl_policy,
+#endif
+		.doit = nlrtw_ch_util_set,
+		.dumpit = NULL,
+	},
+};
+
+enum nlrtw_multicast_groups {
+	NLRTW_MCGRP_DEFAULT,
+};
+static struct genl_multicast_group nlrtw_genl_mcgrp[] = {
+	[NLRTW_MCGRP_DEFAULT] = { .name = "nlrtw_default" },
+};
+
+/* family definition */
+static struct genl_family nlrtw_genl_family = {
+	.hdrsize = 0,
+	.name = "nlrtw_"DRV_NAME,
+	.version = 1,
+	.maxattr = NLRTW_ATTR_MAX,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 2, 0)
+	.policy = nlrtw_genl_policy,
+#endif
+	.netnsok = true,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 12)
+	.module = THIS_MODULE,
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
+	.ops = nlrtw_genl_ops,
+	.n_ops = ARRAY_SIZE(nlrtw_genl_ops),
+	.mcgrps = nlrtw_genl_mcgrp,
+	.n_mcgrps = ARRAY_SIZE(nlrtw_genl_mcgrp),
+#endif
+};
+
+static inline int nlrtw_multicast(const struct genl_family *family,
+				  struct sk_buff *skb, u32 portid,
+				  unsigned int group, gfp_t flags)
+{
+	int ret;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
+	ret = genlmsg_multicast(&nlrtw_genl_family, skb, portid, group, flags);
+#else
+	ret = genlmsg_multicast(skb, portid, nlrtw_genl_mcgrp[group].id, flags);
+#endif
+	return ret;
+}
+
+int rtw_nlrtw_ch_util_rpt(_adapter *adapter, u8 n_rpts, u8 *val, u8 **mac_addr)
+{
+	struct sk_buff *skb = NULL;
+	void *msg_header = NULL;
+	struct nlattr *nl_ch_util, *nl_ch_utils;
+	struct wiphy *wiphy;
+	u8 i;
+	int ret;
+
+	wiphy = adapter_to_wiphy(adapter);
+	if (!wiphy)
+		return -EINVAL;
+
+	/* allocate memory */
+	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb) {
+		nlmsg_free(skb);
+		return -ENOMEM;
+	}
+
+	/* create the message headers */
+	msg_header = genlmsg_put(skb, 0, 0, &nlrtw_genl_family, 0,
+				 NLRTW_CMD_CHANNEL_UTILIZATION);
+	if (!msg_header) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* add attributes */
+	ret = nla_put_string(skb, NLRTW_ATTR_WIPHY_NAME, wiphy_name(wiphy));
+
+	nl_ch_utils = nla_nest_start(skb, NLRTW_ATTR_CHANNEL_UTILIZATIONS);
+	if (!nl_ch_utils) {
+		ret = -EMSGSIZE;
+		goto err_out;
+	}
+
+	for (i = 0; i < n_rpts; i++) {
+		nl_ch_util = nla_nest_start(skb, i);
+		if (!nl_ch_util) {
+			ret = -EMSGSIZE;
+			goto err_out;
+		}
+
+		ret = nla_put(skb, NLRTW_ATTR_CHANNEL_UTILIZATION_BSSID, ETH_ALEN, *(mac_addr + i));
+		if (ret != 0)
+			goto err_out;
+
+		ret = nla_put_u8(skb, NLRTW_ATTR_CHANNEL_UTILIZATION_VALUE, *(val + i));
+		if (ret != 0)
+			goto err_out;
+
+		nla_nest_end(skb, nl_ch_util);
+	}
+
+	nla_nest_end(skb, nl_ch_utils);
+
+	/* finalize the message */
+	genlmsg_end(skb, msg_header);
+
+	ret = nlrtw_multicast(&nlrtw_genl_family, skb, 0, NLRTW_MCGRP_DEFAULT, GFP_KERNEL);
+	if (ret == -ESRCH) {
+		RTW_INFO("[%s] return ESRCH(No such process)."
+			 " Maybe no process waits for this msg\n", __func__);
+		return ret;
+	} else if (ret != 0) {
+		RTW_INFO("[%s] ret = %d\n", __func__, ret);
+		return ret;
+	}
+
+	return 0;
+err_out:
+	nlmsg_free(skb);
+	return ret;
+}
+
+int rtw_nlrtw_reg_change_event(_adapter *adapter)
+{
+	struct sk_buff *skb = NULL;
+	void *msg_header = NULL;
+	struct wiphy *wiphy;
+	u8 i;
+	int ret;
+
+	wiphy = adapter_to_wiphy(adapter);
+	if (!wiphy) {
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	/* allocate memory */
+	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* create the message headers */
+	msg_header = genlmsg_put(skb, 0, 0, &nlrtw_genl_family, 0, NLRTW_CMD_REG_CHANGE);
+	if (!msg_header) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* add attributes */
+	ret = nla_put_string(skb, NLRTW_ATTR_WIPHY_NAME, wiphy_name(wiphy));
+	if (ret)
+		goto err_out;
+
+	/* finalize the message */
+	genlmsg_end(skb, msg_header);
+
+	ret = nlrtw_multicast(&nlrtw_genl_family, skb, 0, NLRTW_MCGRP_DEFAULT, GFP_KERNEL);
+	if (ret == -ESRCH) {
+		RTW_DBG(FUNC_WIPHY_FMT" return -ESRCH(No such process)."
+			 " Maybe no process waits for this msg\n", FUNC_WIPHY_ARG(wiphy));
+		return ret;
+	} else if (ret != 0) {
+		RTW_WARN(FUNC_WIPHY_FMT" return %d\n", FUNC_WIPHY_ARG(wiphy), ret);
+		return ret;
+	}
+
+	return 0;
+
+err_out:
+	if (skb)
+		nlmsg_free(skb);
+	return ret;
+}
+
+int rtw_nlrtw_reg_beacon_hint_event(_adapter *adapter)
+{
+	struct sk_buff *skb = NULL;
+	void *msg_header = NULL;
+	struct wiphy *wiphy;
+	u8 i;
+	int ret;
+
+	wiphy = adapter_to_wiphy(adapter);
+	if (!wiphy) {
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	/* allocate memory */
+	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* create the message headers */
+	msg_header = genlmsg_put(skb, 0, 0, &nlrtw_genl_family, 0, NLRTW_CMD_REG_BEACON_HINT);
+	if (!msg_header) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* add attributes */
+	ret = nla_put_string(skb, NLRTW_ATTR_WIPHY_NAME, wiphy_name(wiphy));
+	if (ret)
+		goto err_out;
+
+	/* finalize the message */
+	genlmsg_end(skb, msg_header);
+
+	ret = nlrtw_multicast(&nlrtw_genl_family, skb, 0, NLRTW_MCGRP_DEFAULT, GFP_KERNEL);
+	if (ret == -ESRCH) {
+		RTW_DBG(FUNC_WIPHY_FMT" return -ESRCH(No such process)."
+			 " Maybe no process waits for this msg\n", FUNC_WIPHY_ARG(wiphy));
+		return ret;
+	} else if (ret != 0) {
+		RTW_WARN(FUNC_WIPHY_FMT" return %d\n", FUNC_WIPHY_ARG(wiphy), ret);
+		return ret;
+	}
+
+	return 0;
+
+err_out:
+	if (skb)
+		nlmsg_free(skb);
+	return ret;
+}
+
+#ifdef CONFIG_DFS_MASTER
+static int _rtw_nlrtw_radar_event(struct rf_ctl_t *rfctl, u8 band_idx, enum nlrtw_radar_event evt_type, u8 band, u8 cch, u8 bw)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct sk_buff *skb = NULL;
+	void *msg_header = NULL;
+	struct wiphy *wiphy;
+	u8 i;
+	int ret;
+
+	/* TODO: hwband specific wiphy mapping */
+	wiphy = dvobj_to_wiphy(dvobj);
+	if (!wiphy) {
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	/* allocate memory */
+	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* create the message headers */
+	msg_header = genlmsg_put(skb, 0, 0, &nlrtw_genl_family, 0, NLRTW_CMD_RADAR_EVENT);
+	if (!msg_header) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* add attributes */
+	ret = nla_put_string(skb, NLRTW_ATTR_WIPHY_NAME, wiphy_name(wiphy));
+	if (ret)
+		goto err_out;
+
+	ret = nla_put_u8(skb, NLRTW_ATTR_RADAR_EVENT, (uint8_t)evt_type);
+	if (ret != 0)
+		goto err_out;
+
+	ret = nla_put_u8(skb, NLRTW_ATTR_CHANNEL_BAND, band);
+	if (ret != 0)
+		goto err_out;
+
+	ret = nla_put_u8(skb, NLRTW_ATTR_CHANNEL_CENTER, cch);
+	if (ret != 0)
+		goto err_out;
+
+	ret = nla_put_u8(skb, NLRTW_ATTR_CHANNEL_WIDTH, bw);
+	if (ret != 0)
+		goto err_out;
+
+	/* finalize the message */
+	genlmsg_end(skb, msg_header);
+
+	ret = nlrtw_multicast(&nlrtw_genl_family, skb, 0, NLRTW_MCGRP_DEFAULT, GFP_KERNEL);
+	if (ret == -ESRCH) {
+		RTW_DBG(FUNC_WIPHY_FMT" return -ESRCH(No such process)."
+			 " Maybe no process waits for this msg\n", FUNC_WIPHY_ARG(wiphy));
+		return ret;
+	} else if (ret != 0) {
+		RTW_WARN(FUNC_WIPHY_FMT" return %d\n", FUNC_WIPHY_ARG(wiphy), ret);
+		return ret;
+	}
+
+	return 0;
+
+err_out:
+	if (skb)
+		nlmsg_free(skb);
+	return ret;
+}
+
+int rtw_nlrtw_radar_detect_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 cch, u8 bw)
+{
+	return _rtw_nlrtw_radar_event(rfctl, band_idx, NLRTW_RADAR_DETECTED, BAND_ON_5G, cch, bw);
+}
+
+int rtw_nlrtw_cac_finish_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 ifbmp, u8 cch, u8 bw)
+{
+	if (ifbmp != 0xFF)
+		return 0;
+	/* only accept all iface event */
+	return _rtw_nlrtw_radar_event(rfctl, band_idx, NLRTW_RADAR_CAC_FINISHED, BAND_ON_5G, cch, bw);
+}
+
+int rtw_nlrtw_cac_abort_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 ifbmp, u8 cch, u8 bw)
+{
+	if (ifbmp != 0xFF)
+		return 0;
+	/* only accept all iface event */
+	return _rtw_nlrtw_radar_event(rfctl, band_idx, NLRTW_RADAR_CAC_ABORTED, BAND_ON_5G, cch, bw);
+}
+
+int rtw_nlrtw_nop_finish_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 band, u8 cch, u8 bw)
+{
+	return _rtw_nlrtw_radar_event(rfctl, band_idx, NLRTW_RADAR_NOP_FINISHED, band, cch, bw);
+}
+
+int rtw_nlrtw_nop_start_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 band, u8 cch, u8 bw)
+{
+	return _rtw_nlrtw_radar_event(rfctl, band_idx, NLRTW_RADAR_NOP_STARTED, band, cch, bw);
+}
+#endif /* CONFIG_DFS_MASTER */
+
+int rtw_nlrtw_radio_opmode_notify(struct rf_ctl_t *rfctl)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	_adapter *iface;
+	struct sk_buff *skb = NULL;
+	void *msg_header = NULL;
+	struct nlattr *nl_if_opmodes, *nl_if_opmode;
+	struct wiphy *wiphy;
+	u16 op_txpwr_max_u16;
+	u8 i;
+	int ret;
+
+	wiphy = dvobj_to_wiphy(dvobj);
+	if (!wiphy) {
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	/* allocate memory */
+	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	/* create the message headers */
+	msg_header = genlmsg_put(skb, 0, 0, &nlrtw_genl_family, 0, NLRTW_CMD_RADIO_OPMODE);
+	if (!msg_header) {
+		ret = -ENOBUFS;
+		goto err_out;
+	}
+
+	/* add attributes */
+	ret = nla_put_string(skb, NLRTW_ATTR_WIPHY_NAME, wiphy_name(wiphy));
+	if (ret)
+		goto err_out;
+
+	ret = nla_put_u8(skb, NLRTW_ATTR_OP_CLASS, rfctl->op_class);
+	if (ret != 0)
+		goto err_out;
+
+	ret = nla_put_u8(skb, NLRTW_ATTR_OP_CHANNEL, rfctl->op_ch);
+	if (ret != 0)
+		goto err_out;
+
+	*((s16 *)&op_txpwr_max_u16) = rfctl->op_txpwr_max;
+	ret = nla_put_u16(skb, NLRTW_ATTR_OP_TXPWR_MAX, op_txpwr_max_u16);
+	if (ret != 0)
+		goto err_out;
+
+	if (0)
+		RTW_INFO("radio: %u,%u %d\n", rfctl->op_class, rfctl->op_ch, rfctl->op_txpwr_max);
+
+	nl_if_opmodes = nla_nest_start(skb, NLRTW_ATTR_IF_OPMODES);
+	if (!nl_if_opmodes) {
+		ret = -ENOBUFS;
+		goto err_out;
+	}
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		if (!dvobj->padapters[i])
+			continue;
+		iface = dvobj->padapters[i];
+
+		if (!rfctl->if_op_class[i] || !rfctl->if_op_ch[i])
+			continue;
+
+		if (0)
+			RTW_INFO(ADPT_FMT": %u,%u\n", ADPT_ARG(iface), rfctl->if_op_class[i], rfctl->if_op_ch[i]);
+
+		nl_if_opmode = nla_nest_start(skb, i + 1);
+		if (!nl_if_opmode) {
+			ret = -ENOBUFS;
+			goto err_out;
+		}
+
+		ret = nla_put(skb, NLRTW_IF_OPMODE_MACADDR, ETH_ALEN, adapter_mac_addr(iface));
+		if (ret != 0)
+			goto err_out;
+
+		ret = nla_put_u8(skb, NLRTW_IF_OPMODE_OP_CLASS, rfctl->if_op_class[i]);
+		if (ret != 0)
+			goto err_out;
+
+		ret = nla_put_u8(skb, NLRTW_IF_OPMODE_OP_CHANNEL, rfctl->if_op_ch[i]);
+		if (ret != 0)
+			goto err_out;
+
+		nla_nest_end(skb, nl_if_opmode);
+	}
+
+	nla_nest_end(skb, nl_if_opmodes);
+
+	/* finalize the message */
+	genlmsg_end(skb, msg_header);
+
+	ret = nlrtw_multicast(&nlrtw_genl_family, skb, 0, NLRTW_MCGRP_DEFAULT, GFP_KERNEL);
+	if (ret == -ESRCH) {
+		RTW_DBG(FUNC_WIPHY_FMT" return -ESRCH(No such process)."
+			 " Maybe no process waits for this msg\n", FUNC_WIPHY_ARG(wiphy));
+		return ret;
+	} else if (ret != 0) {
+		RTW_WARN(FUNC_WIPHY_FMT" return %d\n", FUNC_WIPHY_ARG(wiphy), ret);
+		return ret;
+	}
+
+	return 0;
+
+err_out:
+	if (skb)
+		nlmsg_free(skb);
+	return ret;
+}
+
+int rtw_nlrtw_init(void)
+{
+	int err;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
+	err = genl_register_family(&nlrtw_genl_family);
+	if (err)
+		return err;
+#else
+	err = genl_register_family_with_ops(&nlrtw_genl_family, nlrtw_genl_ops, ARRAY_SIZE(nlrtw_genl_ops));
+	if (err)
+		return err;
+
+	err = genl_register_mc_group(&nlrtw_genl_family, &nlrtw_genl_mcgrp[0]);
+	if (err) {
+		genl_unregister_family(&nlrtw_genl_family);
+		return err;
+	}
+#endif
+	RTW_INFO("[%s] %s\n", __func__, nlrtw_genl_family.name);
+	return 0;
+}
+
+int rtw_nlrtw_deinit(void)
+{
+	int err;
+
+	err = genl_unregister_family(&nlrtw_genl_family);
+	RTW_INFO("[%s] err = %d\n", __func__, err);
+
+	return err;
+}
+#endif /* CONFIG_RTW_NLRTW */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/nlrtw.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/nlrtw.h
new file mode 100644
index 000000000000..06d6be286d5e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/nlrtw.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_NLRTW_H_
+#define __RTW_NLRTW_H_
+
+#ifdef CONFIG_RTW_NLRTW
+int rtw_nlrtw_init(void);
+int rtw_nlrtw_deinit(void);
+int rtw_nlrtw_ch_util_rpt(_adapter *adapter, u8 n_rpts, u8 *val, u8 **mac_addr);
+int rtw_nlrtw_reg_change_event(_adapter *adapter);
+int rtw_nlrtw_reg_beacon_hint_event(_adapter *adapter);
+int rtw_nlrtw_radio_opmode_notify(struct rf_ctl_t *rfctl);
+#else
+static inline int rtw_nlrtw_init(void) {return _FAIL;}
+static inline int rtw_nlrtw_deinit(void) {return _FAIL;}
+static inline int rtw_nlrtw_ch_util_rpt(_adapter *adapter, u8 n_rpts, u8 *val, u8 **mac_addr) {return _FAIL;}
+static inline int rtw_nlrtw_reg_change_event(_adapter *adapter) {return _FAIL;}
+static inline int rtw_nlrtw_reg_beacon_hint_event(_adapter *adapter) {return _FAIL;}
+static inline int rtw_nlrtw_radio_opmode_notify(struct rf_ctl_t *rfctl) {return _FAIL;}
+#endif /* CONFIG_RTW_NLRTW */
+
+#if defined(CONFIG_RTW_NLRTW) && defined(CONFIG_DFS_MASTER)
+int rtw_nlrtw_radar_detect_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 cch, u8 bw);
+int rtw_nlrtw_cac_finish_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 ifbmp, u8 cch, u8 bw);
+int rtw_nlrtw_cac_abort_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 ifbmp, u8 cch, u8 bw);
+int rtw_nlrtw_nop_finish_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 band, u8 cch, u8 bw);
+int rtw_nlrtw_nop_start_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 band, u8 cch, u8 bw);
+#else
+static inline int rtw_nlrtw_radar_detect_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 cch, u8 bw) {return _FAIL;}
+static inline int rtw_nlrtw_cac_finish_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 ifbmp, u8 cch, u8 bw) {return _FAIL;}
+static inline int rtw_nlrtw_cac_abort_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 ifbmp, u8 cch, u8 bw) {return _FAIL;}
+static inline int rtw_nlrtw_nop_finish_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 band, u8 cch, u8 bw) {return _FAIL;}
+static inline int rtw_nlrtw_nop_start_event(struct rf_ctl_t *rfctl, u8 band_idx, u8 band, u8 cch, u8 bw) {return _FAIL;}
+#endif /* defined(CONFIG_RTW_NLRTW) && defined(CONFIG_DFS_MASTER) */
+
+#endif /* __RTW_NLRTW_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_ch_utils.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_ch_utils.c
new file mode 100644
index 000000000000..5ec049febd7d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_ch_utils.c
@@ -0,0 +1,444 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define  _OS_CH_UTILS_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_IOCTL_CFG80211
+
+#define CHAN2G(_channel, _freq, _flags) {			\
+		.band			= NL80211_BAND_2GHZ,	\
+		.center_freq		= (_freq),		\
+		.hw_value		= (_channel),		\
+		.flags			= (_flags),		\
+		.max_antenna_gain	= 0,			\
+		.max_power		= 0,			\
+	}
+
+#define CHAN5G(_channel, _flags) {					\
+		.band			= NL80211_BAND_5GHZ,		\
+		.center_freq		= 5000 + (5 * (_channel)),	\
+		.hw_value		= (_channel),			\
+		.flags			= (_flags),			\
+		.max_antenna_gain	= 0,				\
+		.max_power		= 0,				\
+	}
+
+#define CHAN6G(_channel, _flags) {					\
+		.band			= NL80211_BAND_6GHZ,		\
+		.center_freq		= 5950 + (5 * (_channel)),	\
+		.hw_value		= (_channel),			\
+		.flags			= (_flags),			\
+		.max_antenna_gain	= 0,				\
+		.max_power		= 0,				\
+	}
+
+/* from center_ch_2g */
+const struct ieee80211_channel rtw_2ghz_channels[MAX_CHANNEL_NUM_2G] = {
+	CHAN2G(1, 2412, 0),
+	CHAN2G(2, 2417, 0),
+	CHAN2G(3, 2422, 0),
+	CHAN2G(4, 2427, 0),
+	CHAN2G(5, 2432, 0),
+	CHAN2G(6, 2437, 0),
+	CHAN2G(7, 2442, 0),
+	CHAN2G(8, 2447, 0),
+	CHAN2G(9, 2452, 0),
+	CHAN2G(10, 2457, 0),
+	CHAN2G(11, 2462, 0),
+	CHAN2G(12, 2467, 0),
+	CHAN2G(13, 2472, 0),
+	CHAN2G(14, 2484, 0),
+};
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+/* from center_ch_5g_20m */
+const struct ieee80211_channel rtw_5ghz_a_channels[MAX_CHANNEL_NUM_5G] = {
+	CHAN5G(36, 0),	CHAN5G(40, 0),	CHAN5G(44, 0),	CHAN5G(48, 0),
+
+	CHAN5G(52, 0),	CHAN5G(56, 0),	CHAN5G(60, 0),	CHAN5G(64, 0),
+
+	CHAN5G(100, 0),	CHAN5G(104, 0),	CHAN5G(108, 0),	CHAN5G(112, 0),
+	CHAN5G(116, 0),	CHAN5G(120, 0),	CHAN5G(124, 0),	CHAN5G(128, 0),
+	CHAN5G(132, 0),	CHAN5G(136, 0),	CHAN5G(140, 0),	CHAN5G(144, 0),
+
+	CHAN5G(149, 0),	CHAN5G(153, 0),	CHAN5G(157, 0),	CHAN5G(161, 0),
+	CHAN5G(165, 0),	CHAN5G(169, 0),	CHAN5G(173, 0),	CHAN5G(177, 0),
+};
+#endif
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+/* from center_ch_6g_20m */
+const struct ieee80211_channel rtw_6ghz_channels[MAX_CHANNEL_NUM_6G] = {
+	CHAN6G(1, 0),	CHAN6G(5, 0),	CHAN6G(9, 0),	CHAN6G(13, 0),
+	CHAN6G(17, 0),	CHAN6G(21, 0),	CHAN6G(25, 0),	CHAN6G(29, 0),
+	CHAN6G(33, 0),	CHAN6G(37, 0),	CHAN6G(41, 0),	CHAN6G(45, 0),
+	CHAN6G(49, 0),	CHAN6G(53, 0),	CHAN6G(57, 0),	CHAN6G(61, 0),
+	CHAN6G(65, 0),	CHAN6G(69, 0),	CHAN6G(73, 0),	CHAN6G(77, 0),
+	CHAN6G(81, 0),	CHAN6G(85, 0),	CHAN6G(89, 0),	CHAN6G(93, 0),
+
+	CHAN6G(97, 0),	CHAN6G(101, 0),	CHAN6G(105, 0),	CHAN6G(109, 0),
+	CHAN6G(113, 0),	CHAN6G(117, 0),
+
+					CHAN6G(121, 0),	CHAN6G(125, 0),
+	CHAN6G(129, 0),	CHAN6G(133, 0),	CHAN6G(137, 0),	CHAN6G(141, 0),
+	CHAN6G(145, 0),	CHAN6G(149, 0),	CHAN6G(153, 0),	CHAN6G(157, 0),
+	CHAN6G(161, 0),	CHAN6G(165, 0),	CHAN6G(169, 0),	CHAN6G(173, 0),
+	CHAN6G(177, 0),	CHAN6G(181, 0),	CHAN6G(185, 0),	CHAN6G(189, 0),
+
+	CHAN6G(193, 0),	CHAN6G(197, 0),	CHAN6G(201, 0),	CHAN6G(205, 0),
+	CHAN6G(209, 0),	CHAN6G(213, 0),	CHAN6G(217, 0),	CHAN6G(221, 0),
+	CHAN6G(225, 0),	CHAN6G(229, 0),	CHAN6G(233, 0),	CHAN6G(237, 0),
+
+	CHAN6G(241, 0),	CHAN6G(245, 0),	CHAN6G(249, 0),	CHAN6G(253, 0),
+};
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+const enum nl80211_band _rtw_band_to_nl80211_band[] = {
+	[BAND_ON_24G]	= NL80211_BAND_2GHZ,
+	[BAND_ON_5G]	= NL80211_BAND_5GHZ,
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	[BAND_ON_6G]	= NL80211_BAND_6GHZ,
+	#else
+	[BAND_ON_6G]	= NUM_NL80211_BANDS,
+	#endif
+};
+static_assert(ARRAY_SIZE(_rtw_band_to_nl80211_band) >= BAND_MAX);
+
+const enum band_type _nl80211_band_to_rtw_band[] = {
+	[NL80211_BAND_2GHZ]	= BAND_ON_24G,
+	[NL80211_BAND_5GHZ]	= BAND_ON_5G,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	[NL80211_BAND_60GHZ]	= BAND_MAX,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
+	#if CONFIG_IEEE80211_BAND_6GHZ
+	[NL80211_BAND_6GHZ]	= BAND_ON_6G,
+	#else
+	[NL80211_BAND_6GHZ]	= BAND_MAX,
+	#endif
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	[NL80211_BAND_S1GHZ]	= BAND_MAX,	/* around 900MHz, supported by S1G PHYs */
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)) || (defined(__ANDROID_COMMON_KERNEL__) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	[NL80211_BAND_LC]	= BAND_MAX,	/* light communication band (placeholder) */
+#endif
+};
+static_assert(ARRAY_SIZE(_nl80211_band_to_rtw_band) >= NUM_NL80211_BANDS);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+const char *nl80211_chan_width_str(enum nl80211_chan_width cwidth)
+{
+	switch (cwidth) {
+	case NL80211_CHAN_WIDTH_20_NOHT:
+		return "20_NOHT";
+	case NL80211_CHAN_WIDTH_20:
+		return "20";
+	case NL80211_CHAN_WIDTH_40:
+		return "40";
+	case NL80211_CHAN_WIDTH_80:
+		return "80";
+	case NL80211_CHAN_WIDTH_80P80:
+		return "80+80";
+	case NL80211_CHAN_WIDTH_160:
+		return "160";
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	case NL80211_CHAN_WIDTH_5:
+		return "5";
+	case NL80211_CHAN_WIDTH_10:
+		return "10";
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	case NL80211_CHAN_WIDTH_1:
+		return "1";
+	case NL80211_CHAN_WIDTH_2:
+		return "2";
+	case NL80211_CHAN_WIDTH_4:
+		return "4";
+	case NL80211_CHAN_WIDTH_8:
+		return "8";
+	case NL80211_CHAN_WIDTH_16:
+		return "16";
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0))
+	case NL80211_CHAN_WIDTH_320:
+		return "320";
+#endif
+	default:
+		return "INVALID";
+	};
+}
+
+u8 rtw_bchbw_to_cfg80211_chan_def(struct wiphy *wiphy, struct cfg80211_chan_def *chdef
+	, enum band_type band, u8 ch, u8 bw, u8 offset, u8 ht)
+{
+	int freq, cfreq;
+	struct ieee80211_channel *chan;
+	u8 ret = _FAIL;
+
+	_rtw_memset(chdef, 0, sizeof(*chdef));
+
+	freq = rtw_bch2freq(band, ch);
+	if (!freq)
+		goto exit;
+
+	cfreq = rtw_get_center_ch_by_band(band, ch, bw, offset);
+	if (!cfreq)
+		goto exit;
+	cfreq = rtw_bch2freq(band, cfreq);
+	if (!cfreq)
+		goto exit;
+
+	chan = ieee80211_get_channel(wiphy, freq);
+	if (!chan)
+		goto exit;
+
+	if (bw == CHANNEL_WIDTH_20)
+		chdef->width = ht ? NL80211_CHAN_WIDTH_20 : NL80211_CHAN_WIDTH_20_NOHT;
+	else if (bw == CHANNEL_WIDTH_40)
+		chdef->width = NL80211_CHAN_WIDTH_40;
+	else if (bw == CHANNEL_WIDTH_80)
+		chdef->width = NL80211_CHAN_WIDTH_80;
+	else if (bw == CHANNEL_WIDTH_160)
+		chdef->width = NL80211_CHAN_WIDTH_160;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	else if (bw == CHANNEL_WIDTH_5)
+		chdef->width = NL80211_CHAN_WIDTH_5;
+	else if (bw == CHANNEL_WIDTH_10)
+		chdef->width = NL80211_CHAN_WIDTH_10;
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0)) */
+	else {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	chdef->chan = chan;
+	chdef->center_freq1 = cfreq;
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+void rtw_get_bchbw_from_cfg80211_chan_def(struct cfg80211_chan_def *chdef
+	, u8 *ht, enum band_type *band, u8 *ch, u8 *bw, u8 *offset)
+{
+	int pri_freq;
+	struct ieee80211_channel *chan = chdef->chan;
+
+	*ch = 0;
+	*band = nl80211_band_to_rtw_band(chan->band);
+	if (*band == BAND_MAX) {
+		RTW_WARN("invalid band:%d\n", chan->band);
+		rtw_warn_on(1);
+		return;
+	}
+
+	pri_freq = rtw_bch2freq(*band, chan->hw_value);
+	if (!pri_freq) {
+		RTW_WARN("invalid channel:%d on %s\n", chan->hw_value, band_str(*band));
+		rtw_warn_on(1);
+		return;
+	}
+
+	switch (chdef->width) {
+	case NL80211_CHAN_WIDTH_20_NOHT:
+		*ht = 0;
+		*bw = CHANNEL_WIDTH_20;
+		*offset = CHAN_OFFSET_NO_EXT;
+		*ch = chan->hw_value;
+		break;
+	case NL80211_CHAN_WIDTH_20:
+		*ht = 1;
+		*bw = CHANNEL_WIDTH_20;
+		*offset = CHAN_OFFSET_NO_EXT;
+		*ch = chan->hw_value;
+		break;
+	case NL80211_CHAN_WIDTH_40:
+		*ht = 1;
+		*bw = CHANNEL_WIDTH_40;
+		*offset = pri_freq > chdef->center_freq1 ? CHAN_OFFSET_LOWER : CHAN_OFFSET_UPPER;
+		if (!rtw_get_offset_by_bchbw(*band, chan->hw_value, *bw, offset)) {
+			RTW_WARN("invalid channel:%d bw:%s on %s\n", chan->hw_value, ch_width_str(*bw), band_str(*band));
+			rtw_warn_on(1);
+			return;
+		}
+		*ch = chan->hw_value;
+		break;
+	case NL80211_CHAN_WIDTH_80:
+		*ht = 1;
+		*bw = CHANNEL_WIDTH_80;
+		if (!rtw_get_offset_by_bchbw(*band, chan->hw_value, *bw, offset)) {
+			RTW_WARN("invalid channel:%d bw:%s on %s\n", chan->hw_value, ch_width_str(*bw), band_str(*band));
+			rtw_warn_on(1);
+			return;
+		}
+		*ch = chan->hw_value;
+		break;
+	case NL80211_CHAN_WIDTH_160:
+		*ht = 1;
+		*bw = CHANNEL_WIDTH_160;
+		if (!rtw_get_offset_by_bchbw(*band, chan->hw_value, *bw, offset)) {
+			RTW_WARN("invalid channel:%d bw:%s on %s\n", chan->hw_value, ch_width_str(*bw), band_str(*band));
+			rtw_warn_on(1);
+			return;
+		}
+		*ch = chan->hw_value;
+		break;
+	case NL80211_CHAN_WIDTH_80P80:
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	case NL80211_CHAN_WIDTH_5:
+	case NL80211_CHAN_WIDTH_10:
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 9, 0))
+	case NL80211_CHAN_WIDTH_1:
+	case NL80211_CHAN_WIDTH_2:
+	case NL80211_CHAN_WIDTH_4:
+	case NL80211_CHAN_WIDTH_8:
+	case NL80211_CHAN_WIDTH_16:
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0))
+	case NL80211_CHAN_WIDTH_320:
+	#endif
+	default:
+		*ht = 0;
+		*bw = CHANNEL_WIDTH_20;
+		*offset = CHAN_OFFSET_NO_EXT;
+		RTW_INFO("unsupported cwidth:%s\n", nl80211_chan_width_str(chdef->width));
+		rtw_warn_on(1);
+	};
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY u8 rtw_chbw_to_cfg80211_chan_def(struct wiphy *wiphy, struct cfg80211_chan_def *chdef
+	, u8 ch, u8 bw, u8 offset, u8 ht)
+{
+	return rtw_bchbw_to_cfg80211_chan_def(wiphy, chdef, rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch, bw, offset, ht);
+}
+
+RTW_FUNC_2G_5G_ONLY void rtw_get_chbw_from_cfg80211_chan_def(struct cfg80211_chan_def *chdef
+	, u8 *ht, u8 *ch, u8 *bw, u8 *offset)
+{
+	enum band_type band;
+
+	rtw_get_bchbw_from_cfg80211_chan_def(chdef, ht, &band, ch, bw, offset);
+}
+#endif
+
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+const char *nl80211_channel_type_str(enum nl80211_channel_type ctype)
+{
+	switch (ctype) {
+	case NL80211_CHAN_NO_HT:
+		return "NO_HT";
+	case NL80211_CHAN_HT20:
+		return "HT20";
+	case NL80211_CHAN_HT40MINUS:
+		return "HT40-";
+	case NL80211_CHAN_HT40PLUS:
+		return "HT40+";
+	default:
+		return "INVALID";
+	};
+}
+
+enum nl80211_channel_type rtw_bchbw_to_nl80211_channel_type(enum band_type band, u8 ch, u8 bw, u8 offset, u8 ht)
+{
+	rtw_warn_on(!ht && (bw >= CHANNEL_WIDTH_40 || offset != CHAN_OFFSET_NO_EXT));
+
+	if (!ht)
+		return NL80211_CHAN_NO_HT;
+	if (bw >= CHANNEL_WIDTH_40) {
+		if (offset == CHAN_OFFSET_LOWER)
+			return NL80211_CHAN_HT40MINUS;
+		else if (offset == CHAN_OFFSET_UPPER)
+			return NL80211_CHAN_HT40PLUS;
+		else
+			rtw_warn_on(1);
+	}
+	return NL80211_CHAN_HT20;
+}
+
+void rtw_get_bchbw_from_nl80211_channel_type(struct ieee80211_channel *chan, enum nl80211_channel_type ctype
+	, enum band_type *band, u8 *ht, u8 *ch, u8 *bw, u8 *offset)
+{
+	int pri_freq;
+
+	*ch = 0;
+	*band = nl80211_band_to_rtw_band(chan->band);
+	if (*band == BAND_MAX) {
+		RTW_WARN("invalid band:%d\n", chan->band);
+		rtw_warn_on(1);
+		return;
+	}
+
+	pri_freq = rtw_bch2freq(*band, chan->hw_value);
+	if (!pri_freq) {
+		RTW_WARN("invalid channel:%d on %s\n", chan->hw_value, band_str(*band));
+		rtw_warn_on(1);
+		return;
+	}
+	*ch = chan->hw_value;
+
+	switch (ctype) {
+	case NL80211_CHAN_NO_HT:
+		*ht = 0;
+		*bw = CHANNEL_WIDTH_20;
+		*offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case NL80211_CHAN_HT20:
+		*ht = 1;
+		*bw = CHANNEL_WIDTH_20;
+		*offset = CHAN_OFFSET_NO_EXT;
+		break;
+	case NL80211_CHAN_HT40MINUS:
+		*ht = 1;
+		*bw = CHANNEL_WIDTH_40;
+		*offset = CHAN_OFFSET_LOWER;
+		break;
+	case NL80211_CHAN_HT40PLUS:
+		*ht = 1;
+		*bw = CHANNEL_WIDTH_40;
+		*offset = CHAN_OFFSET_UPPER;
+		break;
+	default:
+		*ht = 0;
+		*bw = CHANNEL_WIDTH_20;
+		*offset = CHAN_OFFSET_NO_EXT;
+		RTW_INFO("unsupported ctype:%s\n", nl80211_channel_type_str(ctype));
+		rtw_warn_on(1);
+	};
+}
+
+#if CONFIG_ALLOW_FUNC_2G_5G_ONLY
+RTW_FUNC_2G_5G_ONLY enum nl80211_channel_type rtw_chbw_to_nl80211_channel_type(u8 ch, u8 bw, u8 offset, u8 ht)
+{
+	return rtw_bchbw_to_nl80211_channel_type(rtw_is_2g_ch(ch) ? BAND_ON_24G : BAND_ON_5G, ch, bw, offset, ht);
+}
+
+RTW_FUNC_2G_5G_ONLY void rtw_get_chbw_from_nl80211_channel_type(struct ieee80211_channel *chan, enum nl80211_channel_type ctype
+	, u8 *ht, u8 *ch, u8 *bw, u8 *offset)
+{
+	enum band_type band;
+
+	rtw_get_bchbw_from_nl80211_channel_type(chan, ctype, &band, ht, ch, bw, offset);
+}
+#endif
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) */
+
+#endif /* CONFIG_IOCTL_CFG80211 */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_ch_utils.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_ch_utils.h
new file mode 100644
index 000000000000..86220a12ee74
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_ch_utils.h
@@ -0,0 +1,63 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __OS_CH_UTILS_H__
+#define __OS_CH_UTILS_H__
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0))
+#define NL80211_BAND_2GHZ IEEE80211_BAND_2GHZ
+#define NL80211_BAND_5GHZ IEEE80211_BAND_5GHZ
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+#define NL80211_BAND_60GHZ IEEE80211_BAND_60GHZ
+#endif
+#define NUM_NL80211_BANDS IEEE80211_NUM_BANDS
+#endif
+
+extern const enum nl80211_band _rtw_band_to_nl80211_band[];
+#define rtw_band_to_nl80211_band(band) (((band) < BAND_MAX) ? _rtw_band_to_nl80211_band[(band)] : NUM_NL80211_BANDS)
+
+extern const enum band_type _nl80211_band_to_rtw_band[];
+#define nl80211_band_to_rtw_band(band) (((band) < NUM_NL80211_BANDS) ? _nl80211_band_to_rtw_band[(band)] : BAND_MAX)
+
+extern const struct ieee80211_channel rtw_2ghz_channels[MAX_CHANNEL_NUM_2G];
+#if CONFIG_IEEE80211_BAND_5GHZ
+extern const struct ieee80211_channel rtw_5ghz_a_channels[MAX_CHANNEL_NUM_5G];
+#endif
+#if CONFIG_IEEE80211_BAND_6GHZ
+extern const struct ieee80211_channel rtw_6ghz_channels[MAX_CHANNEL_NUM_6G];
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+const char *nl80211_chan_width_str(enum nl80211_chan_width cwidth);
+u8 rtw_bchbw_to_cfg80211_chan_def(struct wiphy *wiphy, struct cfg80211_chan_def *chdef
+	, enum band_type band, u8 ch, u8 bw, u8 offset, u8 ht);
+void rtw_get_bchbw_from_cfg80211_chan_def(struct cfg80211_chan_def *chdef
+	, u8 *ht, enum band_type *band, u8 *ch, u8 *bw, u8 *offset);
+RTW_FUNC_2G_5G_ONLY u8 rtw_chbw_to_cfg80211_chan_def(struct wiphy *wiphy, struct cfg80211_chan_def *chdef
+	, u8 ch, u8 bw, u8 offset, u8 ht);
+RTW_FUNC_2G_5G_ONLY void rtw_get_chbw_from_cfg80211_chan_def(struct cfg80211_chan_def *chdef
+	, u8 *ht, u8 *ch, u8 *bw, u8 *offset);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+const char *nl80211_channel_type_str(enum nl80211_channel_type ctype);
+enum nl80211_channel_type rtw_bchbw_to_nl80211_channel_type(enum band_type band, u8 ch, u8 bw, u8 offset, u8 ht);
+void rtw_get_bchbw_from_nl80211_channel_type(struct ieee80211_channel *chan, enum nl80211_channel_type ctype
+	, enum band_type *band, u8 *ht, u8 *ch, u8 *bw, u8 *offset);
+RTW_FUNC_2G_5G_ONLY enum nl80211_channel_type rtw_chbw_to_nl80211_channel_type(u8 ch, u8 bw, u8 offset, u8 ht);
+RTW_FUNC_2G_5G_ONLY void rtw_get_chbw_from_nl80211_channel_type(struct ieee80211_channel *chan, enum nl80211_channel_type ctype
+	, u8 *ht, u8 *ch, u8 *bw, u8 *offset);
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)) */
+
+#endif /* __OS_CH_UTILS_H__ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_intfs.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_intfs.c
new file mode 100644
index 000000000000..3debf3b3645d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/os_intfs.c
@@ -0,0 +1,3631 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _OS_INTFS_C_
+
+#include <drv_types.h>
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Realtek Wireless Lan Driver");
+MODULE_AUTHOR("Realtek Semiconductor Corp.");
+MODULE_VERSION(DRIVERVERSION);
+
+
+int netdev_open(struct net_device *pnetdev);
+static int netdev_close(struct net_device *pnetdev);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 8, 0)
+#define strlcpy(p, q, s) strscpy(p, q, s)
+#endif
+
+/**
+ * rtw_net_set_mac_address
+ * This callback function is used for the Media Access Control address
+ * of each net_device needs to be changed.
+ *
+ * Arguments:
+ * @pnetdev: net_device pointer.
+ * @addr: new MAC address.
+ *
+ * Return:
+ * ret = 0: Permit to change net_device's MAC address.
+ * ret = -1 (Default): Operation not permitted.
+ *
+ * Auther: Arvin Liu
+ * Date: 2015/05/29
+ */
+static int rtw_net_set_mac_address(struct net_device *pnetdev, void *addr)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct sockaddr *sa = (struct sockaddr *)addr;
+	int ret = -EPERM;
+	u8 result = _FAIL;
+	bool cfg_hw = false;
+
+
+	/* if the net_device is linked, it's not permit to modify mac addr */
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) ||
+	    check_fwstate(pmlmepriv, WIFI_ASOC_STATE) ||
+	    check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY)) {
+		RTW_INFO(FUNC_ADPT_FMT": The net_device's is not idle currently\n"
+			 , FUNC_ADPT_ARG(padapter));
+
+		return ret;
+	}
+
+	/* check whether the input mac address is valid to permit modifying mac addr */
+	if (rtw_check_invalid_mac_address(sa->sa_data, _FALSE) == _TRUE) {
+		RTW_INFO(FUNC_ADPT_FMT": Invalid Mac Addr for "MAC_FMT"\n"
+			 , FUNC_ADPT_ARG(padapter), MAC_ARG(sa->sa_data));
+
+		return ret;
+	}
+
+	if (pnetdev->flags & IFF_UP) {
+		RTW_INFO(FUNC_ADPT_FMT": The net_device's is not in down state\n"
+			 , FUNC_ADPT_ARG(padapter));
+		cfg_hw = true;
+	}
+
+	result = rtw_set_mac_addr(padapter, sa->sa_data, cfg_hw, false);
+	if (result == _SUCCESS) {
+		dev_addr_mod(pnetdev, 0, sa->sa_data, ETH_ALEN);
+		RTW_INFO(FUNC_ADPT_FMT": Set Mac Addr to "MAC_FMT" Successfully\n"
+			 , FUNC_ADPT_ARG(padapter), MAC_ARG(sa->sa_data));
+		ret = 0;
+	} else {
+		RTW_ERR("%s: Set Mac Addr to "MAC_FMT" Failed\n", __func__, MAC_ARG(sa->sa_data));
+		ret = -EFAULT;
+	}
+
+	return ret;
+}
+
+static struct net_device_stats *rtw_net_get_stats(struct net_device *pnetdev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
+	struct recv_info *precvinfo = &(padapter->recvinfo);
+
+	padapter->stats.tx_packets = pxmitpriv->tx_pkts;/* pxmitpriv->tx_pkts++; */
+	padapter->stats.rx_packets = precvinfo->rx_pkts;/* precvinfo->rx_pkts++; */
+	padapter->stats.tx_dropped = pxmitpriv->tx_drop;
+	padapter->stats.rx_dropped = precvinfo->rx_drop;
+	padapter->stats.tx_bytes = pxmitpriv->tx_bytes;
+	padapter->stats.rx_bytes = precvinfo->rx_bytes;
+
+	return &padapter->stats;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+/*
+ * AC to queue mapping
+ *
+ * AC_VO -> queue 0
+ * AC_VI -> queue 1
+ * AC_BE -> queue 2
+ * AC_BK -> queue 3
+ */
+static const u16 rtw_1d_to_queue[8] = { 2, 3, 3, 2, 1, 1, 0, 0 };
+
+/* Given a data frame determine the 802.1p/1d tag to use. */
+unsigned int rtw_classify8021d(struct sk_buff *skb)
+{
+	unsigned int dscp;
+
+	/* skb->priority values from 256->263 are magic values to
+	 * directly indicate a specific 802.1d priority.  This is used
+	 * to allow 802.1d priority to be passed directly in from VLAN
+	 * tags, etc.
+	 */
+	if (skb->priority >= 256 && skb->priority <= 263)
+		return skb->priority - 256;
+
+	switch (skb->protocol) {
+	case htons(ETH_P_IP):
+		dscp = ip_hdr(skb)->tos & 0xfc;
+		break;
+	default:
+	#ifdef PRIVATE_R
+		return skb->priority;
+	#else
+		return 0;
+	#endif
+	}
+
+	return dscp >> 5;
+}
+
+
+static u16 rtw_select_queue(struct net_device *dev, struct sk_buff *skb
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)
+	, struct net_device *sb_dev
+	#else
+	, void *accel_priv
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0))
+	, select_queue_fallback_t fallback
+	#endif
+#endif
+)
+{
+	_adapter	*padapter = rtw_netdev_priv(dev);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+
+	skb->priority = rtw_classify8021d(skb);
+
+	if (pmlmepriv->acm_mask != 0)
+		skb->priority = qos_acm(pmlmepriv->acm_mask, skb->priority);
+
+	return rtw_1d_to_queue[skb->priority];
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35)) */
+
+u16 rtw_os_recv_select_queue(u8 *msdu, enum rtw_rx_llc_hdl llc_hdl)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	u32 priority = 0;
+
+	if (llc_hdl == RTW_RX_LLC_REMOVE) {
+		u16 eth_type = RTW_GET_BE16(msdu + SNAP_SIZE);
+
+		if (eth_type == ETH_P_IP) {
+			struct iphdr *iphdr = (struct iphdr *)(msdu + SNAP_SIZE + 2);
+			unsigned int dscp = iphdr->tos & 0xfc;
+
+			priority = dscp >> 5;
+		}
+	}
+
+	return rtw_1d_to_queue[priority];
+#else
+	return 0;
+#endif
+}
+
+static u8 is_rtw_ndev(struct net_device *ndev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+	return ndev->netdev_ops
+		&& ndev->netdev_ops->ndo_do_ioctl
+		&& ndev->netdev_ops->ndo_do_ioctl == rtw_ioctl;
+#else
+	return ndev->do_ioctl
+		&& ndev->do_ioctl == rtw_ioctl;
+#endif
+}
+
+
+#define _netdev_status_msg(_ndev, state, sts_str)		\
+	RTW_INFO(FUNC_NDEV_FMT" state:%lu - %s\n", FUNC_NDEV_ARG(_ndev), state, sts_str);
+
+static int rtw_ndev_notifier_call(struct notifier_block *nb, unsigned long state, void *ptr)
+{
+	struct net_device *ndev;
+
+	if (ptr == NULL)
+		return NOTIFY_DONE;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	ndev = netdev_notifier_info_to_dev(ptr);
+#else
+	ndev = ptr;
+#endif
+
+	if (ndev == NULL)
+		return NOTIFY_DONE;
+
+	if (!is_rtw_ndev(ndev))
+		return NOTIFY_DONE;
+
+	switch (state) {
+	case NETDEV_CHANGE:
+		_netdev_status_msg(ndev, state, "netdev change");
+		break;
+	case NETDEV_GOING_DOWN:
+		_netdev_status_msg(ndev, state, "netdev going down");
+		break;
+	case NETDEV_DOWN:
+		_netdev_status_msg(ndev, state, "netdev down");
+		break;
+	case NETDEV_UP:
+		_netdev_status_msg(ndev, state, "netdev up");
+		break;
+	case NETDEV_REBOOT:
+		_netdev_status_msg(ndev, state, "netdev reboot");
+		break;
+	case NETDEV_CHANGENAME:
+		rtw_adapter_proc_replace(ndev);
+		_netdev_status_msg(ndev, state, "netdev chang ename");
+		break;
+	case NETDEV_PRE_UP :
+		_netdev_status_msg(ndev, state, "netdev pre up");
+		break;
+	case NETDEV_JOIN:
+		_netdev_status_msg(ndev, state, "netdev join");
+		break;
+	default:
+		_netdev_status_msg(ndev, state, " ");
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block rtw_ndev_notifier = {
+	.notifier_call = rtw_ndev_notifier_call,
+};
+
+int rtw_ndev_notifier_register(void)
+{
+	return register_netdevice_notifier(&rtw_ndev_notifier);
+}
+
+void rtw_ndev_notifier_unregister(void)
+{
+	unregister_netdevice_notifier(&rtw_ndev_notifier);
+}
+
+int rtw_ndev_init(struct net_device *dev)
+{
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	RTW_PRINT(FUNC_ADPT_FMT" if%d mac_addr="MAC_FMT"\n"
+		, FUNC_ADPT_ARG(adapter), (adapter->iface_id + 1), MAC_ARG(dev->dev_addr));
+	strncpy(adapter->old_ifname, dev->name, IFNAMSIZ);
+	adapter->old_ifname[IFNAMSIZ - 1] = '\0';
+#ifdef CONFIG_ARCH_CORTINA
+	dev->priv_flags = IFF_DOMAIN_WLAN;
+#endif
+	rtw_adapter_proc_init(dev);
+
+#ifdef CONFIG_RTW_NAPI
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0))
+	netif_napi_add(dev, &adapter->napi, rtw_recv_napi_poll);
+	#else
+	netif_napi_add(dev, &adapter->napi, rtw_recv_napi_poll, RTL_NAPI_WEIGHT);
+	#endif
+#endif /* CONFIG_RTW_NAPI */
+
+	return 0;
+}
+
+void rtw_ndev_uninit(struct net_device *dev)
+{
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	RTW_PRINT(FUNC_ADPT_FMT" if%d\n"
+		  , FUNC_ADPT_ARG(adapter), (adapter->iface_id + 1));
+	rtw_adapter_proc_deinit(dev);
+
+#ifdef CONFIG_RTW_NAPI
+	if(adapter->napi_state == NAPI_ENABLE) {
+		napi_disable(&adapter->napi);
+		adapter->napi_state = NAPI_DISABLE;
+	}
+	netif_napi_del(&adapter->napi);
+#endif /* CONFIG_RTW_NAPI */
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+static int rtw_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
+			      void __user *data, int cmd)
+{
+	/* handle cmd(s) between SIOCDEVPRIVATE and SIOCDEVPRIVATE + 15 */
+
+	return rtw_ioctl(dev, ifr, cmd);
+}
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+static const struct net_device_ops rtw_netdev_ops = {
+	.ndo_init = rtw_ndev_init,
+	.ndo_uninit = rtw_ndev_uninit,
+	.ndo_open = netdev_open,
+	.ndo_stop = netdev_close,
+	.ndo_start_xmit = rtw_xmit_entry,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	.ndo_select_queue	= rtw_select_queue,
+#endif
+	.ndo_set_mac_address = rtw_net_set_mac_address,
+	.ndo_get_stats = rtw_net_get_stats,
+	.ndo_do_ioctl = rtw_ioctl,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	.ndo_siocdevprivate = rtw_siocdevprivate,
+#endif
+};
+#endif
+
+int rtw_init_netdev_name(struct net_device *pnetdev, const char *ifname)
+{
+	if (dev_alloc_name(pnetdev, ifname) < 0)
+		RTW_ERR("dev_alloc_name, fail!\n");
+
+	rtw_netif_carrier_off(pnetdev);
+	/* rtw_netif_stop_queue(pnetdev); */
+
+	return 0;
+}
+
+void rtw_hook_if_ops(struct net_device *ndev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+	ndev->netdev_ops = &rtw_netdev_ops;
+#else
+	ndev->init = rtw_ndev_init;
+	ndev->uninit = rtw_ndev_uninit;
+	ndev->open = netdev_open;
+	ndev->stop = netdev_close;
+	ndev->hard_start_xmit = rtw_xmit_entry;
+	ndev->set_mac_address = rtw_net_set_mac_address;
+	ndev->get_stats = rtw_net_get_stats;
+	ndev->do_ioctl = rtw_ioctl;
+#endif
+}
+
+#ifdef CONFIG_CONCURRENT_MODE
+static void rtw_hook_vir_if_ops(struct net_device *ndev);
+#endif
+struct net_device *rtw_init_netdev(_adapter *old_padapter)
+{
+	_adapter *padapter;
+	struct net_device *pnetdev;
+
+	if (old_padapter != NULL) {
+		rtw_os_ndev_free(old_padapter);
+		pnetdev = rtw_alloc_etherdev_with_old_priv(sizeof(_adapter), (void *)old_padapter);
+	} else
+		pnetdev = rtw_alloc_etherdev(sizeof(_adapter));
+
+	if (!pnetdev)
+		return NULL;
+
+	padapter = rtw_netdev_priv(pnetdev);
+	padapter->pnetdev = pnetdev;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
+	SET_MODULE_OWNER(pnetdev);
+#endif
+
+	rtw_hook_if_ops(pnetdev);
+#ifdef CONFIG_CONCURRENT_MODE
+	if (!is_primary_adapter(padapter))
+		rtw_hook_vir_if_ops(pnetdev);
+#endif /* CONFIG_CONCURRENT_MODE */
+
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+        pnetdev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
+        pnetdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
+#endif
+#endif
+
+#ifdef CONFIG_RTW_NETIF_SG
+        pnetdev->features |= NETIF_F_SG;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
+        pnetdev->hw_features |= NETIF_F_SG;
+#endif
+#endif
+
+	if ((pnetdev->features & NETIF_F_SG) && (pnetdev->features & NETIF_F_IP_CSUM)) {
+		pnetdev->features |= (NETIF_F_TSO | NETIF_F_GSO);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
+		pnetdev->hw_features |= (NETIF_F_TSO | NETIF_F_GSO);
+#endif
+	}
+	/* pnetdev->tx_timeout = NULL; */
+	pnetdev->watchdog_timeo = HZ * 3; /* 3 second timeout */
+
+#ifdef CONFIG_WIRELESS_EXT
+	pnetdev->wireless_handlers = (struct iw_handler_def *)&rtw_handlers_def;
+#endif
+
+#ifdef WIRELESS_SPY
+	/* priv->wireless_data.spy_data = &priv->spy_data; */
+	/* pnetdev->wireless_data = &priv->wireless_data; */
+#endif
+
+#ifdef CONFIG_TX_AMSDU_SW_MODE
+	pnetdev->needed_headroom += 8;	/* +8 for rfc1042 header */
+	pnetdev->needed_headroom += 4;	/* +4 for padding */
+#endif
+
+	return pnetdev;
+}
+#ifdef CONFIG_PCI_HCI
+#include <rtw_trx_pci.h>
+#endif
+int rtw_os_ndev_alloc(_adapter *adapter)
+{
+	int ret = _FAIL;
+	struct net_device *ndev = NULL;
+
+	ndev = rtw_init_netdev(adapter);
+	if (ndev == NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 5, 0)
+	SET_NETDEV_DEV(ndev, dvobj_to_dev(adapter_to_dvobj(adapter)));
+#endif
+
+#ifdef CONFIG_PCI_HCI
+	if (is_pci_support_dma64(adapter_to_dvobj(adapter)))
+		ndev->features |= NETIF_F_HIGHDMA;
+	ndev->irq = dvobj_to_pci(adapter_to_dvobj(adapter))->irq;
+#endif
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	if (rtw_cfg80211_ndev_res_alloc(adapter) != _SUCCESS) {
+		rtw_warn_on(1);
+	} else
+#endif
+	ret = _SUCCESS;
+
+	if (ret != _SUCCESS && ndev)
+		rtw_free_netdev(ndev);
+exit:
+	return ret;
+}
+
+void rtw_os_ndev_free(_adapter *adapter)
+{
+#if defined(CONFIG_IOCTL_CFG80211)
+	rtw_cfg80211_ndev_res_free(adapter);
+#endif
+
+	if (adapter->pnetdev) {
+		rtw_free_netdev(adapter->pnetdev);
+		adapter->pnetdev = NULL;
+	}
+}
+
+/* For ethtool +++ */
+#ifdef CONFIG_IOCTL_CFG80211
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 8))
+static void rtw_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
+{
+	struct wireless_dev *wdev = NULL;
+	_adapter *padapter = NULL;
+
+	wdev = dev->ieee80211_ptr;
+	if (wdev) {
+		strlcpy(info->driver, wiphy_dev(wdev->wiphy)->driver->name,
+			sizeof(info->driver));
+	} else {
+		strlcpy(info->driver, "N/A", sizeof(info->driver));
+	}
+
+	strlcpy(info->version, DRIVERVERSION, sizeof(info->version));
+
+	padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	/*GEORGIA_TODO_FIXIT*/
+	#if 0
+	HAL_DATA_TYPE *hal_data = NULL;
+	if (padapter) {
+		hal_data = GET_PHL_COM(padapter);
+	}
+	if (hal_data) {
+		scnprintf(info->fw_version, sizeof(info->fw_version), "%d.%d",
+			  hal_data->firmware_version, hal_data->firmware_sub_version);
+	} else
+	#endif
+	{
+		strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
+	}
+
+	strlcpy(info->bus_info, dev_name(wiphy_dev(wdev->wiphy)),
+		sizeof(info->bus_info));
+}
+
+static const char rtw_ethtool_gstrings_sta_stats[][ETH_GSTRING_LEN] = {
+	"rx_packets", "rx_bytes", "rx_dropped",
+	"tx_packets", "tx_bytes", "tx_dropped",
+};
+
+#define RTW_ETHTOOL_STATS_LEN	ARRAY_SIZE(rtw_ethtool_gstrings_sta_stats)
+
+static int rtw_ethtool_get_sset_count(struct net_device *dev, int sset)
+{
+	int rv = 0;
+
+	if (sset == ETH_SS_STATS)
+		rv += RTW_ETHTOOL_STATS_LEN;
+
+	if (rv == 0)
+		return -EOPNOTSUPP;
+
+	return rv;
+}
+
+static void rtw_ethtool_get_strings(struct net_device *dev, u32 sset, u8 *data)
+{
+	int sz_sta_stats = 0;
+
+	if (sset == ETH_SS_STATS) {
+		sz_sta_stats = sizeof(rtw_ethtool_gstrings_sta_stats);
+		_rtw_memcpy(data, rtw_ethtool_gstrings_sta_stats, sz_sta_stats);
+	}
+}
+
+static void rtw_ethtool_get_stats(struct net_device *dev,
+				  struct ethtool_stats *stats,
+				  u64 *data)
+{
+	int i = 0;
+	_adapter *padapter = NULL;
+	struct xmit_priv *pxmitpriv = NULL;
+	struct recv_info *precvinfo = NULL;
+
+	memset(data, 0, sizeof(u64) * RTW_ETHTOOL_STATS_LEN);
+
+	padapter = (_adapter *)rtw_netdev_priv(dev);
+	if (padapter) {
+		pxmitpriv = &(padapter->xmitpriv);
+		precvinfo = &(padapter->recvinfo);
+
+		data[i++] = precvinfo->rx_pkts;
+		data[i++] = precvinfo->rx_bytes;
+		data[i++] = precvinfo->rx_drop;
+
+		data[i++] = pxmitpriv->tx_pkts;
+		data[i++] = pxmitpriv->tx_bytes;
+		data[i++] = pxmitpriv->tx_drop;
+	} else {
+		data[i++] = 0;
+		data[i++] = 0;
+		data[i++] = 0;
+
+		data[i++] = 0;
+		data[i++] = 0;
+		data[i++] = 0;
+	}
+}
+
+static const struct ethtool_ops rtw_ethtool_ops = {
+	.get_drvinfo = rtw_ethtool_get_drvinfo,
+	.get_link = ethtool_op_get_link,
+	.get_strings = rtw_ethtool_get_strings,
+	.get_ethtool_stats = rtw_ethtool_get_stats,
+	.get_sset_count = rtw_ethtool_get_sset_count,
+};
+#endif // LINUX_VERSION_CODE >= 3.7.8
+#endif /* CONFIG_IOCTL_CFG80211 */
+/* For ethtool --- */
+
+int rtw_os_ndev_register(_adapter *adapter, const char *name)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	int ret = _SUCCESS;
+	struct net_device *ndev = adapter->pnetdev;
+	u8 rtnl_lock_needed = rtw_rtnl_lock_needed(dvobj);
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	if (rtw_cfg80211_ndev_res_register(adapter) != _SUCCESS) {
+		rtw_warn_on(1);
+		ret = _FAIL;
+		goto exit;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 8))
+	netdev_set_default_ethtool_ops(ndev, &rtw_ethtool_ops);
+#endif /* LINUX_VERSION_CODE >= 3.7.8 */
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_PCI_HCI)
+	ndev->gro_flush_timeout = 100000;
+#endif
+	/* alloc netdev name */
+	rtw_init_netdev_name(ndev, name);
+
+	dev_addr_mod(ndev, 0, adapter_mac_addr(adapter), ETH_ALEN);
+
+	/* Tell the network stack we exist */
+
+	if (rtnl_lock_needed)
+		ret = (register_netdev(ndev) == 0) ? _SUCCESS : _FAIL;
+	else
+		ret = (cfg80211_register_netdevice(ndev) == 0) ? _SUCCESS : _FAIL;
+
+	if (ret == _SUCCESS)
+		adapter->registered = 1;
+	else
+		RTW_INFO(FUNC_NDEV_FMT" if%d Failed!\n", FUNC_NDEV_ARG(ndev), (adapter->iface_id + 1));
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	if (ret != _SUCCESS) {
+		rtw_cfg80211_ndev_res_unregister(adapter);
+	}
+#endif
+
+#if defined(CONFIG_IOCTL_CFG80211)
+exit:
+#endif
+
+	return ret;
+}
+
+void rtw_os_ndev_unregister(_adapter *adapter)
+{
+	struct net_device *netdev = NULL;
+
+	if (adapter == NULL || adapter->registered == 0)
+		return;
+
+	adapter->ndev_unregistering = 1;
+
+	netdev = adapter->pnetdev;
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	rtw_cfg80211_ndev_res_unregister(adapter);
+#endif
+
+	if (netdev) {
+		struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+		u8 rtnl_lock_needed = rtw_rtnl_lock_needed(dvobj);
+
+		if (rtnl_lock_needed)
+			unregister_netdev(netdev);
+		else
+			cfg80211_unregister_netdevice(netdev);
+	}
+
+	adapter->registered = 0;
+	adapter->ndev_unregistering = 0;
+}
+
+/**
+ * rtw_os_ndev_init - Allocate and register OS layer net device and relating structures for @adapter
+ * @adapter: the adapter on which this function applies
+ * @name: the requesting net device name
+ *
+ * Returns:
+ * _SUCCESS or _FAIL
+ */
+int rtw_os_ndev_init(_adapter *adapter, const char *name)
+{
+	int ret = _FAIL;
+
+	if (rtw_os_ndev_alloc(adapter) != _SUCCESS)
+		goto exit;
+
+	if (rtw_os_ndev_register(adapter, name) != _SUCCESS)
+		goto os_ndev_free;
+
+	ret = _SUCCESS;
+
+os_ndev_free:
+	if (ret != _SUCCESS)
+		rtw_os_ndev_free(adapter);
+exit:
+	return ret;
+}
+
+/**
+ * rtw_os_ndev_deinit - Unregister and free OS layer net device and relating structures for @adapter
+ * @adapter: the adapter on which this function applies
+ */
+void rtw_os_ndev_deinit(_adapter *adapter)
+{
+	rtw_os_ndev_unregister(adapter);
+	rtw_os_ndev_free(adapter);
+}
+
+int rtw_os_ndevs_alloc(struct dvobj_priv *dvobj)
+{
+	int i, status = _SUCCESS;
+	_adapter *adapter;
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	if (rtw_cfg80211_dev_res_alloc(dvobj) != _SUCCESS) {
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+#endif
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+
+		if (i >= CONFIG_IFACE_NUMBER) {
+			RTW_ERR("%s %d >= CONFIG_IFACE_NUMBER(%d)\n", __func__, i, CONFIG_IFACE_NUMBER);
+			rtw_warn_on(1);
+			continue;
+		}
+
+		adapter = dvobj->padapters[i];
+		if (adapter && !adapter->pnetdev) {
+
+			#ifdef CONFIG_RTW_DYNAMIC_NDEV
+			if (!is_primary_adapter(adapter) &&
+				(i >= CONFIG_RTW_STATIC_NDEV_NUM))
+				break;
+			#endif
+
+			status = rtw_os_ndev_alloc(adapter);
+			if (status != _SUCCESS) {
+				rtw_warn_on(1);
+				break;
+			}
+		}
+	}
+
+	if (status != _SUCCESS) {
+		for (; i >= 0; i--) {
+			adapter = dvobj->padapters[i];
+			if (adapter && adapter->pnetdev)
+				rtw_os_ndev_free(adapter);
+		}
+	}
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	if (status != _SUCCESS)
+		rtw_cfg80211_dev_res_free(dvobj);
+#endif
+
+	return status;
+}
+
+void rtw_os_ndevs_free(struct dvobj_priv *dvobj)
+{
+	int i;
+	_adapter *adapter = NULL;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+
+		if (i >= CONFIG_IFACE_NUMBER) {
+			RTW_ERR("%s %d >= CONFIG_IFACE_NUMBER(%d)\n", __func__, i, CONFIG_IFACE_NUMBER);
+			rtw_warn_on(1);
+			continue;
+		}
+
+		adapter = dvobj->padapters[i];
+
+		if (adapter == NULL)
+			continue;
+
+		rtw_os_ndev_free(adapter);
+	}
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	rtw_cfg80211_dev_res_free(dvobj);
+#endif
+}
+
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+u32 rtw_start_drv_threads(_adapter *padapter)
+{
+	u32 _status = _SUCCESS;
+
+	RTW_INFO(FUNC_ADPT_FMT" enter\n", FUNC_ADPT_ARG(padapter));
+
+#ifdef CONFIG_XMIT_THREAD_MODE
+#if defined(CONFIG_SDIO_HCI)
+	if (is_primary_adapter(padapter))
+#endif
+	{
+		if (padapter->xmitThread == NULL) {
+			RTW_INFO(FUNC_ADPT_FMT " start RTW_XMIT_THREAD\n", FUNC_ADPT_ARG(padapter));
+			padapter->xmitThread = rtw_thread_start(rtw_xmit_thread, padapter, "RTW_XMIT_THREAD");
+			if (padapter->xmitThread == NULL)
+				_status = _FAIL;
+		}
+	}
+#endif /* #ifdef CONFIG_XMIT_THREAD_MODE */
+
+#ifdef CONFIG_RECV_THREAD_MODE
+	if (is_primary_adapter(padapter)) {
+		if (padapter->recvThread == NULL) {
+			RTW_INFO(FUNC_ADPT_FMT " start RTW_RECV_THREAD\n", FUNC_ADPT_ARG(padapter));
+			padapter->recvThread = rtw_thread_start(rtw_recv_thread, padapter, "RTW_RECV_THREAD");
+			if (padapter->recvThread == NULL)
+				_status = _FAIL;
+		}
+	}
+#endif
+
+	if (is_primary_adapter(padapter)) {
+		if (padapter->cmdThread == NULL) {
+			RTW_INFO(FUNC_ADPT_FMT " start RTW_CMD_THREAD\n", FUNC_ADPT_ARG(padapter));
+			padapter->cmdThread = rtw_thread_start(rtw_cmd_thread, padapter, "RTW_CMD_THREAD");
+			if (padapter->cmdThread == NULL)
+				_status = _FAIL;
+			else
+				_rtw_down_sema(&padapter->cmdpriv.start_cmdthread_sema); /* wait for cmd_thread to run */
+		}
+	}
+
+	_status = rtw_intf_start_xmit_frame_thread(padapter);
+	return _status;
+
+}
+
+void rtw_stop_drv_threads(_adapter *padapter)
+{
+	RTW_INFO(FUNC_ADPT_FMT" enter\n", FUNC_ADPT_ARG(padapter));
+	if (is_primary_adapter(padapter))
+		rtw_stop_cmd_thread(padapter);
+
+#ifdef CONFIG_XMIT_THREAD_MODE
+	/* Below is to termindate tx_thread... */
+#if defined(CONFIG_SDIO_HCI)
+	/* Only wake-up primary adapter */
+	if (is_primary_adapter(padapter))
+#endif  /*SDIO_HCI */
+	{
+		if (padapter->xmitThread) {
+			_rtw_up_sema(&padapter->xmitpriv.xmit_sema);
+			rtw_thread_stop(padapter->xmitThread);
+			padapter->xmitThread = NULL;
+		}
+	}
+#endif
+
+#ifdef CONFIG_RECV_THREAD_MODE
+	if (is_primary_adapter(padapter) && padapter->recvThread) {
+		/* Below is to termindate rx_thread... */
+		_rtw_up_sema(&padapter->recvpriv.recv_sema);
+		rtw_thread_stop(padapter->recvThread);
+		padapter->recvThread = NULL;
+	}
+#endif
+
+	/*rtw_hal_stop_thread(padapter);*/
+	rtw_intf_cancel_xmit_frame_thread(padapter);
+
+}
+#endif
+
+u8 rtw_init_default_value(_adapter *padapter)
+{
+	u8 ret  = _SUCCESS;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
+
+	/* xmit_priv */
+	pxmitpriv->vcs_setting = pregistrypriv->vrtl_carrier_sense;
+	pxmitpriv->vcs = pregistrypriv->vcs_type;
+	pxmitpriv->vcs_type = pregistrypriv->vcs_type;
+	/* pxmitpriv->rts_thresh = pregistrypriv->rts_thresh; */
+	pxmitpriv->frag_len = pregistrypriv->frag_thresh;
+
+	/* security_priv */
+	/* rtw_get_encrypt_decrypt_from_registrypriv(padapter); */
+#ifdef CONFIG_GTK_OL
+	psecuritypriv->binstallKCK_KEK = _FAIL;
+#endif /* CONFIG_GTK_OL */
+	psecuritypriv->sw_encrypt = pregistrypriv->software_encrypt;
+	psecuritypriv->sw_decrypt = pregistrypriv->software_decrypt;
+
+	psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+	psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+
+	psecuritypriv->dot11PrivacyKeyIndex = 0;
+
+	psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
+
+	psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
+	psecuritypriv->ndisencryptstatus = Ndis802_11WEPDisabled;
+#ifdef CONFIG_CONCURRENT_MODE
+	psecuritypriv->dot118021x_bmc_cam_id = INVALID_SEC_MAC_CAM_ID;
+#endif
+
+	/* pwrctrl_priv */
+
+	/* registry_priv */
+	rtw_init_registrypriv_dev_network(padapter);
+	rtw_update_registrypriv_dev_network(padapter);
+
+	pregistrypriv->wireless_mode &= rtw_hw_get_wireless_mode(adapter_to_dvobj(padapter));
+	pregistrypriv->band_type &= rtw_hw_get_band_cap(adapter_to_dvobj(padapter));
+
+	/* misc. */
+	padapter->bLinkInfoDump = 0;
+	padapter->bNotifyChannelChange = _FALSE;
+#ifdef CONFIG_P2P
+	padapter->bShowGetP2PState = 1;
+#endif
+
+	/* for debug purpose */
+	padapter->fix_rate = NO_FIX_RATE;
+	padapter->data_fb = 0;
+	padapter->fix_bw = NO_FIX_BW;
+	padapter->power_offset = 0;
+	padapter->rsvd_page_offset = 0;
+	padapter->rsvd_page_num = 0;
+#ifdef CONFIG_AP_MODE
+	padapter->bmc_tx_rate = pregistrypriv->bmc_tx_rate;
+	#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+	padapter->b2u_flags_ap_src = pregistrypriv->ap_src_b2u_flags;
+	padapter->b2u_flags_ap_fwd = pregistrypriv->ap_fwd_b2u_flags;
+	#endif
+#endif
+	padapter->driver_tx_bw_mode = pregistrypriv->tx_bw_mode;
+
+	padapter->driver_ampdu_spacing = 0xFF;
+	padapter->driver_rx_ampdu_factor =  0xFF;
+	padapter->driver_rx_ampdu_spacing = 0xFF;
+	padapter->fix_rx_ampdu_accept = RX_AMPDU_ACCEPT_INVALID;
+	padapter->fix_rx_ampdu_size = RX_AMPDU_SIZE_INVALID;
+#ifdef CONFIG_TX_AMSDU
+	padapter->tx_amsdu = 2;
+	padapter->tx_amsdu_rate = 10;
+#endif
+	if (pregistrypriv->adaptivity_idle_probability == 1) {
+#ifdef CONFIG_TX_AMSDU
+		padapter->tx_amsdu = 0;
+		padapter->tx_amsdu_rate = 0;
+#endif
+		padapter->dis_turboedca = 1;
+	}
+
+#ifdef DBG_RX_COUNTER_DUMP
+	padapter->dump_rx_cnt_mode = 0;
+	padapter->drv_rx_cnt_ok = 0;
+	padapter->drv_rx_cnt_crcerror = 0;
+	padapter->drv_rx_cnt_drop = 0;
+#endif
+#ifdef CONFIG_RTW_NAPI
+	padapter->napi_state = NAPI_DISABLE;
+#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	ATOMIC_SET(&padapter->tbtx_tx_pause, _FALSE);
+	ATOMIC_SET(&padapter->tbtx_remove_tx_pause, _FALSE);
+	padapter->tbtx_capability = _TRUE;
+#endif
+
+#ifdef CONFIG_CORE_TXSC
+	txsc_init(padapter);
+#endif
+
+	return ret;
+}
+
+#ifdef CONFIG_DRV_FAKE_AP
+extern void rtw_fakeap_work(struct work_struct *work);
+extern void rtw_fakeap_bcn_timer_hdl(void*);
+#endif /* CONFIG_DRV_FAKE_AP */
+
+struct dvobj_priv *devobj_init(void)
+{
+	struct dvobj_priv *pdvobj = NULL;
+	struct rf_ctl_t *rfctl;
+
+	pdvobj = (struct dvobj_priv *)rtw_zmalloc(sizeof(*pdvobj));
+	if (pdvobj == NULL)
+		return NULL;
+
+	rfctl = dvobj_to_rfctl(pdvobj);
+
+	_rtw_mutex_init(&pdvobj->hw_init_mutex);
+	_rtw_mutex_init(&pdvobj->setch_mutex);
+	_rtw_mutex_init(&pdvobj->setbw_mutex);
+
+#ifdef CONFIG_RTW_CUSTOMER_STR
+	_rtw_mutex_init(&pdvobj->customer_str_mutex);
+	_rtw_memset(pdvobj->customer_str, 0xFF, RTW_CUSTOMER_STR_LEN);
+#endif
+
+	pdvobj->processing_dev_remove = _FALSE;
+
+	ATOMIC_SET(&pdvobj->disable_func, 0);
+	/* move to phl */
+	/* rtw_macid_ctl_init(&pdvobj->macid_ctl); */
+#if 0
+	_rtw_spinlock_init(&pdvobj->cam_ctl.lock);
+	_rtw_mutex_init(&pdvobj->cam_ctl.sec_cam_access_mutex);
+#endif
+#if defined(RTK_129X_PLATFORM) && defined(CONFIG_PCI_HCI)
+	_rtw_spinlock_init(&pdvobj->io_reg_lock);
+#endif
+
+	#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+	rtw_init_timer(&(pdvobj->dynamic_chk_timer), rtw_dynamic_check_timer_handlder, pdvobj);
+	#endif
+
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+	pdvobj->en_napi_dynamic = 0;
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+
+	_rtw_mutex_init(&rfctl->offch_mutex);
+
+	pdvobj->scan_deny = _FALSE;
+	rtw_load_dvobj_registry(pdvobj);
+
+#ifdef CONFIG_DRV_FAKE_AP
+	skb_queue_head_init(&pdvobj->fakeap.rxq);
+	_init_workitem(&pdvobj->fakeap.work, rtw_fakeap_work, pdvobj);
+	_init_timer(&pdvobj->fakeap.bcn_timer, rtw_fakeap_bcn_timer_hdl, pdvobj);
+#endif /* CONFIG_DRV_FAKE_AP */
+
+	/* wpas type default from w1.fi */
+	pdvobj->wpas_type = RTW_WPAS_W1FI;
+
+	return pdvobj;
+
+}
+
+void devobj_deinit(struct dvobj_priv *pdvobj)
+{
+	if (!pdvobj)
+		return;
+
+	/* TODO: use rtw_os_ndevs_deinit instead at the first stage of driver's dev deinit function */
+#if defined(CONFIG_IOCTL_CFG80211)
+	rtw_cfg80211_dev_res_free(pdvobj);
+#endif
+
+	_rtw_mutex_free(&pdvobj->hw_init_mutex);
+
+#ifdef CONFIG_RTW_CUSTOMER_STR
+	_rtw_mutex_free(&pdvobj->customer_str_mutex);
+#endif
+
+	_rtw_mutex_free(&pdvobj->setch_mutex);
+	_rtw_mutex_free(&pdvobj->setbw_mutex);
+	/* move to phl */
+	/* rtw_macid_ctl_deinit(&pdvobj->macid_ctl); */
+#if 0
+	_rtw_spinlock_free(&pdvobj->cam_ctl.lock);
+	_rtw_mutex_free(&pdvobj->cam_ctl.sec_cam_access_mutex);
+#endif
+#if defined(RTK_129X_PLATFORM) && defined(CONFIG_PCI_HCI)
+	_rtw_spinlock_free(&pdvobj->io_reg_lock);
+#endif
+
+	rtw_mfree((u8 *)pdvobj, sizeof(*pdvobj));
+}
+
+inline u8 rtw_rtnl_lock_needed(struct dvobj_priv *dvobj)
+{
+	if (dvobj->rtnl_lock_holder && dvobj->rtnl_lock_holder == current)
+		return 0;
+	return 1;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26))
+static inline int rtnl_is_locked(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 17))
+	if (unlikely(rtnl_trylock())) {
+		rtnl_unlock();
+#else
+	if (unlikely(down_trylock(&rtnl_sem) == 0)) {
+		up(&rtnl_sem);
+#endif
+		return 0;
+	}
+	return 1;
+}
+#endif
+
+inline void rtw_set_rtnl_lock_holder(struct dvobj_priv *dvobj, _thread_hdl_ thd_hdl)
+{
+	rtw_warn_on(!rtnl_is_locked());
+
+	if (!thd_hdl || rtnl_is_locked())
+		dvobj->rtnl_lock_holder = thd_hdl;
+
+	if (dvobj->rtnl_lock_holder && 0)
+		RTW_INFO("rtnl_lock_holder: %s:%d\n", current->comm, current->pid);
+}
+
+u8 rtw_reset_drv_sw(_adapter *padapter)
+{
+	u8	ret8 = _SUCCESS;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+
+	/* hal_priv */
+	rtw_hw_cap_init(adapter_to_dvobj(padapter));
+
+	RTW_ENABLE_FUNC(adapter_to_dvobj(padapter), DF_RX_BIT);
+	RTW_ENABLE_FUNC(adapter_to_dvobj(padapter), DF_TX_BIT);
+
+	padapter->bLinkInfoDump = 0;
+
+	padapter->xmitpriv.tx_pkts = 0;
+	padapter->recvinfo.rx_pkts = 0;
+#ifdef PRIVATE_R
+	padapter->recvinfo.last_rx_uc_data = 0;
+#endif
+	pmlmepriv->LinkDetectInfo.bBusyTraffic = _FALSE;
+
+	/* pmlmepriv->LinkDetectInfo.TrafficBusyState = _FALSE; */
+	pmlmepriv->LinkDetectInfo.TrafficTransitionCount = 0;
+	pmlmepriv->LinkDetectInfo.LowPowerTransitionCount = 0;
+
+	_clr_fwstate_(pmlmepriv, WIFI_UNDER_SURVEY | WIFI_UNDER_LINKING);
+
+
+	/* mlmeextpriv */
+	mlmeext_set_scan_state(&padapter->mlmeextpriv, SCAN_DISABLE);
+
+	#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	if (padapter->netif_up == _TRUE)
+		rtw_set_signal_stat_timer(&padapter->recvinfo);
+	#endif
+	return ret8;
+}
+
+static void devobj_decide_init_chplan(struct dvobj_priv *dvobj)
+{
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+	struct dev_cap_t *dev_cap = &GET_PHL_COM(dvobj)->dev_cap;
+	const char *alpha2 = NULL; /* TODO */
+	u8 chplan = dev_cap->domain;
+	u8 chplan_6g = RTW_CHPLAN_6G_NULL;
+	bool disable_sw_chplan = _FALSE; /* TODO */
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	chplan_6g = dev_cap->domain_6g;
+#endif
+
+	if (alpha2)
+		RTW_INFO("%s alpha2:{%d,%d}\n", __func__, alpha2[0], alpha2[1]);
+	RTW_INFO("%s chplan:0x%02x\n", __func__, chplan);
+	RTW_INFO("%s chplan_6g:0x%02x\n", __func__, chplan_6g);
+	RTW_INFO("%s disable_sw_chplan:%d\n", __func__, disable_sw_chplan);
+
+	/*
+	* treat {0xFF, 0xFF} as unspecified
+	*/
+	if (alpha2 && strncmp(alpha2, "\xFF\xFF", 2) == 0)
+		alpha2 = NULL;
+
+#ifdef CONFIG_FORCE_SW_CHANNEL_PLAN
+	disable_sw_chplan = _FALSE;
+#endif
+
+	rtw_rfctl_decide_init_chplan(rfctl, alpha2, chplan, chplan_6g, disable_sw_chplan);
+}
+
+u8 devobj_data_init(struct dvobj_priv *dvobj)
+{
+	u8 ret = _FAIL;
+
+	dev_set_drv_stopped(dvobj);/*init*/
+	dev_clr_hw_start(dvobj); /* init */
+
+	/*init data of dvobj*/
+	if (rtw_rfctl_init(dvobj) == _FAIL)
+		goto exit;
+
+	devobj_decide_init_chplan(dvobj);
+
+	rtw_rfctl_apply_init_chplan(dvobj_to_rfctl(dvobj), true);
+
+	op_class_pref_apply_regulatory(dvobj_to_rfctl(dvobj), REG_CHANGE);
+
+	init_channel_list(dvobj_get_primary_adapter(dvobj));
+
+	rtw_hw_cap_init(dvobj);
+
+	RTW_ENABLE_FUNC(dvobj, DF_RX_BIT);
+	RTW_ENABLE_FUNC(dvobj, DF_TX_BIT);
+
+	ret = _SUCCESS;
+
+exit:
+	return 	ret;
+}
+
+
+void devobj_data_deinit(struct dvobj_priv *dvobj)
+{
+}
+
+u8 devobj_trx_resource_init(struct dvobj_priv *dvobj)
+{
+	u8 ret = _SUCCESS;
+
+#ifdef CONFIG_USB_HCI
+	ret = rtw_init_lite_xmit_resource(dvobj);
+	if (ret == _FAIL)
+		goto exit;
+	ret = rtw_init_lite_recv_resource(dvobj);
+	if (ret == _FAIL)
+		goto exit;
+#endif
+	ret = rtw_init_recv_priv(dvobj);
+	if (ret == _FAIL) {
+		RTW_ERR("%s rtw_init_recv_priv failed\n", __func__);
+		goto exit;
+	}
+
+	ret = rtw_init_cmd_priv(dvobj);
+	if (ret == _FAIL) {
+		RTW_ERR("%s rtw_init_cmd_priv failed\n", __func__);
+		goto exit;
+	}
+exit:
+	return 	ret;
+}
+
+
+void devobj_trx_resource_deinit(struct dvobj_priv *dvobj)
+{
+#ifdef CONFIG_USB_HCI
+	rtw_free_lite_xmit_resource(dvobj);
+	rtw_free_lite_recv_resource(dvobj);
+#endif
+	rtw_free_recv_priv(dvobj);
+	rtw_free_cmd_priv(dvobj);
+}
+
+
+u8 rtw_init_drv_sw(_adapter *padapter)
+{
+	u8	ret8 = _SUCCESS;
+
+#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+#endif
+
+	ret8 = rtw_init_default_value(padapter);/*load registrypriv value*/
+
+	if (rtw_init_mlme_priv(padapter) == _FAIL) {
+		ret8 = _FAIL;
+		goto exit;
+	}
+#ifdef CONFIG_RTW_FSM
+	if (rtw_fsm_init(&padapter->fsmpriv, padapter) == _FAIL) {
+		ret8 = _FAIL;
+		goto exit;
+	}
+#endif
+#ifdef CONFIG_P2P
+	init_wifidirect_info(padapter, P2P_ROLE_DISABLE);
+	reset_global_wifidirect_info(padapter);
+#ifdef CONFIG_WFD
+	if (rtw_init_wifi_display_info(padapter) == _FAIL)
+		RTW_ERR("Can't init init_wifi_display_info\n");
+#endif
+#endif /* CONFIG_P2P */
+
+	if (init_mlme_ext_priv(padapter) == _FAIL) {
+		ret8 = _FAIL;
+		goto exit;
+	}
+
+#ifdef CONFIG_TDLS
+	if (rtw_init_tdls_info(padapter) == _FAIL) {
+		RTW_INFO("Can't rtw_init_tdls_info\n");
+		ret8 = _FAIL;
+		goto exit;
+	}
+#endif /* CONFIG_TDLS */
+
+#ifdef CONFIG_RTW_MESH
+	rtw_mesh_cfg_init(padapter);
+#endif
+
+	if (_rtw_init_xmit_priv(&padapter->xmitpriv, padapter) == _FAIL) {
+		RTW_INFO("Can't _rtw_init_xmit_priv\n");
+		ret8 = _FAIL;
+		goto exit;
+	}
+	if (rtw_init_recv_info(padapter) == _FAIL) {
+		RTW_INFO("Can't rtw_init_recv_info\n");
+		ret8 = _FAIL;
+		goto exit;
+	}
+
+	/* add for CONFIG_IEEE80211W, none 11w also can use */
+	_rtw_spinlock_init(&padapter->security_key_mutex);
+
+	/* We don't need to memset padapter->XXX to zero, because adapter is allocated by rtw_zvmalloc(). */
+	/* _rtw_memset((unsigned char *)&padapter->securitypriv, 0, sizeof (struct security_priv)); */
+
+	if (_rtw_init_sta_priv(&padapter->stapriv) == _FAIL) {
+		RTW_INFO("Can't _rtw_init_sta_priv\n");
+		ret8 = _FAIL;
+		goto exit;
+	}
+
+	padapter->setband = WIFI_FREQUENCY_BAND_AUTO;
+	padapter->fix_rate = NO_FIX_RATE;
+	padapter->power_offset = 0;
+	padapter->rsvd_page_offset = 0;
+	padapter->rsvd_page_num = 0;
+
+	padapter->cur_tsf_cmd_enqueue = true;
+	padapter->data_fb = 0;
+	padapter->fix_rx_ampdu_accept = RX_AMPDU_ACCEPT_INVALID;
+	padapter->fix_rx_ampdu_size = RX_AMPDU_SIZE_INVALID;
+#ifdef DBG_RX_COUNTER_DUMP
+	padapter->dump_rx_cnt_mode = 0;
+	padapter->drv_rx_cnt_ok = 0;
+	padapter->drv_rx_cnt_crcerror = 0;
+	padapter->drv_rx_cnt_drop = 0;
+#endif
+
+	rtw_init_pwrctrl_priv(padapter);
+#ifdef CONFIG_WOWLAN
+	rtw_init_wow(padapter);
+#endif /* CONFIG_WOWLAN */
+
+	/* _rtw_memset((u8 *)&padapter->qospriv, 0, sizeof (struct qos_priv)); */ /* move to mlme_priv */
+
+#ifdef CONFIG_WAPI_SUPPORT
+	padapter->WapiSupport = true; /* set true temp, will revise according to Efuse or Registry value later. */
+	rtw_wapi_init(padapter);
+#endif
+
+#ifdef CONFIG_BR_EXT
+	_rtw_spinlock_init(&padapter->br_ext_lock);
+#endif /* CONFIG_BR_EXT */
+
+#ifdef CONFIG_RTW_80211K
+	rtw_init_rm(padapter);
+#endif
+
+#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+	_rtw_memset(pwdev_priv->pno_mac_addr, 0xFF, ETH_ALEN);
+#endif
+
+	rtw_connect_req_init(padapter);
+	rtw_disconnect_req_init(padapter);
+
+#ifdef CONFIG_AP_MODE
+	_rtw_spinlock_init(&padapter->ap_stop_st_lock);
+	rtw_cmd_ap_add_del_sta_req_init(padapter);
+#endif
+
+#ifdef CONFIG_ECSA_PHL
+	reset_ecsa_param(padapter);
+#endif
+
+exit:
+
+	return ret8;
+
+}
+
+
+u8 rtw_init_link_default_value(struct _ADAPTER_LINK *padapter_link)
+{
+	struct link_security_priv *psecuritypriv = &padapter_link->securitypriv;
+	struct link_mlme_priv *pmlmepriv = &padapter_link->mlmepriv;
+	u8 ret  = _SUCCESS;
+
+#ifdef CONFIG_80211N_HT
+	pmlmepriv->ampdu_priv.ampdu_enable = _FALSE;//set to disabled
+#endif
+
+	/* ToDo CONFIG_RTW_MLD: TBD */
+	psecuritypriv->binstallGrpkey = _FAIL;
+	psecuritypriv->dot118021XGrpKeyid = 1;
+
+#ifdef CONFIG_AP_MODE
+	padapter_link->is_ap_chan_ctx_added = _FALSE;
+#endif
+	return ret;
+}
+
+u8 init_adapter_link(_adapter *padapter) {
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+
+	/* ToDo CONFIG_RTW_MLD: temporally initialize the adapter_link_num as RTW_ONE_LINK */
+	padapter->adapter_link_num = RTW_ONE_LINK;
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = (struct _ADAPTER_LINK *)rtw_zvmalloc(sizeof(*padapter_link));
+		if (padapter_link == NULL)
+			goto init_fail;
+		padapter_link->adapter = padapter;
+		padapter->adapter_link[lidx] = padapter_link;
+		/* ToDo CONFIG_RTW_MLD:
+		** temporally initialize the link's mac_addr as the adapter's mac_addr with offset */
+		_rtw_memcpy(padapter_link->mac_addr, adapter_mac_addr(padapter), ETH_ALEN);
+		padapter_link->mac_addr[ETH_ALEN - 1] += lidx;
+
+		/* default values, e.g., wireless_mode, band_type */
+		rtw_init_link_default_value(padapter_link);
+		/* mlmepriv */
+		rtw_init_link_mlme_priv(padapter_link);
+		/* mlmeextpriv */
+		rtw_init_link_mlme_ext_priv(padapter_link);
+	}
+
+	return _SUCCESS;
+
+init_fail:
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		if (padapter_link) {
+			rtw_vmfree((u8 *)padapter_link, sizeof(*padapter_link));
+			padapter_link = NULL;
+		}
+	}
+	return _FAIL;
+}
+
+u8 rtw_adapter_link_init(struct dvobj_priv *dvobj)
+{
+	_adapter *padapter = NULL;
+	u8 i;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		padapter = dvobj->padapters[i];
+		if (!padapter)
+			goto init_fail;
+		if (init_adapter_link(padapter) != _SUCCESS)
+			goto init_fail;
+	}
+	return _SUCCESS;
+
+init_fail:
+	rtw_adapter_link_deinit(dvobj);
+	return _FAIL;
+}
+
+void rtw_adapter_link_deinit(struct dvobj_priv *dvobj)
+{
+	_adapter *padapter = NULL;
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+	u8 i;
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		padapter = dvobj->padapters[i];
+		if (!padapter)
+			continue;
+		for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+			padapter_link = GET_LINK(padapter, lidx);
+			if (padapter_link) {
+#ifdef PRIVATE_R
+				_rtw_deinit_queue(&(padapter_link->mlmepriv.idle_dis_bcn_queue));
+				_rtw_deinit_queue(&(padapter_link->mlmepriv.busy_dis_bcn_queue));
+#endif
+				rtw_vmfree((u8 *)padapter_link, sizeof(*padapter_link));
+				padapter_link = NULL;
+			}
+		}
+	}
+}
+
+int init_link_capab_from_adapter(_adapter *padapter) {
+	struct _ADAPTER_LINK *padapter_link = NULL;
+	u8 lidx;
+
+	for (lidx = 0; lidx < padapter->adapter_link_num; lidx++) {
+		padapter_link = GET_LINK(padapter, lidx);
+		if (!padapter_link)
+			return _FAIL;
+		if (is_supported_24g(padapter->registrypriv.band_type)) {
+#ifdef CONFIG_80211N_HT
+			if (padapter->registrypriv.ht_enable)
+				_rtw_memcpy(&padapter_link->mlmepriv.htpriv, &padapter->mlmepriv.dev_htpriv, sizeof(struct ht_priv));
+#endif
+#ifdef CONFIG_80211AX_HE
+			if (padapter->registrypriv.he_enable)
+				_rtw_memcpy(&padapter_link->mlmepriv.hepriv, &padapter->mlmepriv.dev_hepriv, sizeof(struct he_priv));
+#endif
+		}
+
+		if (is_supported_5g(padapter->registrypriv.band_type)) {
+#ifdef CONFIG_80211N_HT
+			if (padapter->registrypriv.ht_enable)
+				_rtw_memcpy(&padapter_link->mlmepriv.htpriv, &padapter->mlmepriv.dev_htpriv, sizeof(struct ht_priv));
+#endif
+#ifdef CONFIG_80211AC_VHT
+			if (padapter->registrypriv.vht_enable)
+				_rtw_memcpy(&padapter_link->mlmepriv.vhtpriv, &padapter->mlmepriv.dev_vhtpriv, sizeof(struct vht_priv));
+#endif
+#ifdef CONFIG_80211AX_HE
+			if (padapter->registrypriv.he_enable)
+				_rtw_memcpy(&padapter_link->mlmepriv.hepriv, &padapter->mlmepriv.dev_hepriv, sizeof(struct he_priv));
+#endif
+		}
+
+		if (is_supported_6g(padapter->registrypriv.band_type)) {
+#ifdef CONFIG_80211AX_HE
+			if (padapter->registrypriv.he_enable)
+				_rtw_memcpy(&padapter_link->mlmepriv.hepriv, &padapter->mlmepriv.dev_hepriv, sizeof(struct he_priv));
+#endif
+			/* ToDo CONFIG_RTW_MLD: eht_priv */
+		}
+	}
+	return _SUCCESS;
+}
+
+int rtw_init_link_capab(struct dvobj_priv *dvobj)
+{
+	_adapter *padapter = NULL;
+	u8 i;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		padapter = dvobj->padapters[i];
+		if (!padapter)
+			return _FAIL;
+		if (init_link_capab_from_adapter(padapter) != _SUCCESS)
+			return _FAIL;
+	}
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_WOWLAN
+void rtw_cancel_dynamic_chk_timer(_adapter *padapter)
+{
+	#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+	_cancel_timer_ex(&adapter_to_dvobj(padapter)->dynamic_chk_timer);
+	#endif
+}
+#endif
+
+void rtw_cancel_all_timer(_adapter *padapter)
+{
+#ifdef CONFIG_RTW_WNM
+	_cancel_timer_ex(&padapter->mlmepriv.nb_info.roam_scan_timer);
+#endif
+
+#ifdef CONFIG_RTW_80211R
+	_cancel_timer_ex(&padapter->mlmeextpriv.ft_link_timer);
+	_cancel_timer_ex(&padapter->mlmeextpriv.ft_roam_timer);
+#endif
+
+	/*_cancel_timer_ex(&padapter->mlmepriv.assoc_timer);*/
+	cancel_assoc_timer(&padapter->mlmepriv);
+
+#ifdef CONFIG_DFS_MASTER
+	_cancel_timer_ex(&adapter_to_rfctl(padapter)->radar_detect_timer);
+#endif
+
+	#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+	_cancel_timer_ex(&adapter_to_dvobj(padapter)->dynamic_chk_timer);
+	#endif
+
+#ifdef CONFIG_RTW_SW_LED
+	/* cancel sw led timer */
+	rtw_hal_sw_led_deinit(padapter);
+#endif
+
+#ifdef CONFIG_TX_AMSDU
+	_cancel_timer_ex(&padapter->xmitpriv.amsdu_bk_timer);
+	_cancel_timer_ex(&padapter->xmitpriv.amsdu_be_timer);
+	_cancel_timer_ex(&padapter->xmitpriv.amsdu_vo_timer);
+	_cancel_timer_ex(&padapter->xmitpriv.amsdu_vi_timer);
+#endif
+
+#ifdef CONFIG_SET_SCAN_DENY_TIMER
+	_cancel_timer_ex(&padapter->mlmepriv.set_scan_deny_timer);
+	rtw_clear_scan_deny(padapter);
+#endif
+#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	_cancel_timer_ex(&padapter->recvinfo.signal_stat_timer);
+#endif
+
+#ifdef CONFIG_RTW_TOKEN_BASED_XMIT
+	_cancel_timer_ex(&padapter->mlmeextpriv.tbtx_xmit_timer);
+	_cancel_timer_ex(&padapter->mlmeextpriv.tbtx_token_dispatch_timer);
+#endif
+
+#ifdef CONFIG_PLATFORM_FS_MX61
+	msleep(50);
+#endif
+
+}
+
+u8 rtw_free_drv_sw(_adapter *padapter)
+{
+
+#ifdef CONFIG_WAPI_SUPPORT
+	rtw_wapi_free(padapter);
+#endif
+
+	/* we can call rtw_p2p_enable here, but: */
+	/* 1. rtw_p2p_enable may have IO operation */
+	/* 2. rtw_p2p_enable is bundled with wext interface */
+	#ifdef CONFIG_P2P
+	{
+		struct wifidirect_info *pwdinfo = &padapter->wdinfo;
+		if (!rtw_p2p_chk_role(pwdinfo, P2P_ROLE_DISABLE)) {
+			rtw_p2p_set_role(pwdinfo, P2P_ROLE_DISABLE);
+		}
+	}
+	#endif
+	/* add for CONFIG_IEEE80211W, none 11w also can use */
+	_rtw_spinlock_free(&padapter->security_key_mutex);
+
+#ifdef CONFIG_BR_EXT
+	_rtw_spinlock_free(&padapter->br_ext_lock);
+#endif /* CONFIG_BR_EXT */
+
+	free_mlme_ext_priv(&padapter->mlmeextpriv);
+
+#ifdef CONFIG_TDLS
+	/* rtw_free_tdls_info(&padapter->tdlsinfo); */
+#endif /* CONFIG_TDLS */
+
+#ifdef CONFIG_RTW_80211K
+	rtw_free_rm_priv(padapter);
+#endif
+
+	rtw_free_mlme_priv(&padapter->mlmepriv);
+	rtw_connect_req_free(padapter);
+	rtw_disconnect_req_free(padapter);
+
+#ifdef CONFIG_AP_MODE
+	_rtw_spinlock_free(&padapter->ap_stop_st_lock);
+	rtw_cmd_ap_add_del_sta_req_free(padapter);
+#endif
+
+	if (is_primary_adapter(padapter))
+		rtw_rfctl_deinit(adapter_to_dvobj(padapter));
+
+	/* free_io_queue(padapter); */
+
+	_rtw_free_xmit_priv(&padapter->xmitpriv);
+
+	_rtw_free_sta_priv(&padapter->stapriv); /* will free bcmc_stainfo here */
+
+	rtw_free_pwrctrl_priv(padapter);
+#ifdef CONFIG_WOWLAN
+	rtw_free_wow(padapter);
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_RTW_FSM
+	rtw_fsm_deinit(&padapter->fsmpriv);
+#endif
+
+	/* rtw_mfree((void *)padapter, sizeof (padapter)); */
+
+
+	return _SUCCESS;
+
+}
+void rtw_drv_stop_prim_iface(_adapter *adapter)
+{
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(adapter);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct debug_priv *pdbgpriv = &dvobj->drv_dbg;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		rtw_disassoc_cmd(adapter, 0, RTW_CMDF_DIRECTLY|RTW_CMDF_WAIT_ACK);
+
+#ifdef CONFIG_AP_MODE
+	if (MLME_IS_AP(adapter) || MLME_IS_MESH(adapter)) {
+		free_mlme_ap_info(adapter);
+		#ifdef CONFIG_HOSTAPD_MLME
+		hostapd_mode_unload(adapter);
+		#endif
+	}
+#endif
+
+	RTW_INFO("==> "FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+
+	if (adapter->netif_up == _TRUE) {
+		#ifdef CONFIG_XMIT_ACK
+		if (adapter->xmitpriv.ack_tx)
+			rtw_ack_tx_done(&adapter->xmitpriv, RTW_SCTX_DONE_DRV_STOP);
+		#endif
+		rtw_hw_iface_deinit(adapter);
+		if (!pwrctl->bInSuspend)
+			adapter->netif_up = _FALSE;
+	}
+	#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	rtw_stop_drv_threads(adapter);
+
+	if (ATOMIC_READ(&(pcmdpriv->cmdthd_running)) == _TRUE) {
+		RTW_ERR("cmd_thread not stop !!\n");
+		rtw_warn_on(1);
+	}
+	#endif
+
+	rtw_cancel_all_timer(adapter);
+	RTW_INFO("<== "FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+
+}
+
+#ifdef CONFIG_CONCURRENT_MODE
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+static const struct net_device_ops rtw_netdev_vir_if_ops = {
+	.ndo_init = rtw_ndev_init,
+	.ndo_uninit = rtw_ndev_uninit,
+	.ndo_open = netdev_open,
+	.ndo_stop = netdev_close,
+	.ndo_start_xmit = rtw_xmit_entry,
+	.ndo_set_mac_address = rtw_net_set_mac_address,
+	.ndo_get_stats = rtw_net_get_stats,
+	.ndo_do_ioctl = rtw_ioctl,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
+	.ndo_siocdevprivate = rtw_siocdevprivate,
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	.ndo_select_queue	= rtw_select_queue,
+#endif
+};
+#endif
+
+static void rtw_hook_vir_if_ops(struct net_device *ndev)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+	ndev->netdev_ops = &rtw_netdev_vir_if_ops;
+#else
+	ndev->init = rtw_ndev_init;
+	ndev->uninit = rtw_ndev_uninit;
+	ndev->open = netdev_open;
+	ndev->stop = netdev_close;
+
+	ndev->set_mac_address = rtw_net_set_mac_address;
+#endif
+}
+static _adapter *rtw_drv_add_vir_if(struct dvobj_priv *dvobj)
+{
+	int res = _FAIL;
+	_adapter *padapter = NULL;
+	_adapter *primary_padapter = dvobj_get_primary_adapter(dvobj);
+	u8 mac[ETH_ALEN];
+#ifdef CONFIG_MI_UNIQUE_MACADDR_BIT
+	u32 mi_unique_macaddr_bit = 0;
+	bool is_uniq_macaddr = _FALSE;
+	u8 i;
+#endif
+
+	/****** init adapter ******/
+	padapter = (_adapter *)rtw_zvmalloc(sizeof(*padapter));
+	if (padapter == NULL)
+		goto exit;
+
+	_rtw_memcpy(padapter, primary_padapter, sizeof(_adapter));
+
+	if (rtw_load_registry(padapter) != _SUCCESS)
+		goto free_adapter;
+
+	padapter->netif_up = _FALSE;
+	padapter->dir_dev = NULL;
+	padapter->dir_odm = NULL;
+
+	/*set adapter_type/iface type*/
+	padapter->isprimary = _FALSE;
+	padapter->adapter_type = VIRTUAL_ADAPTER;
+
+
+	/****** hook vir if into dvobj ******/
+	padapter->iface_id = dvobj->iface_nums;
+	dvobj->padapters[dvobj->iface_nums++] = padapter;
+
+	/*init drv data*/
+	if (rtw_init_drv_sw(padapter) != _SUCCESS)
+		goto free_drv_sw;
+
+
+	/*get mac address from primary_padapter*/
+	_rtw_memcpy(mac, adapter_mac_addr(primary_padapter), ETH_ALEN);
+
+#ifdef CONFIG_MI_UNIQUE_MACADDR_BIT
+	mi_unique_macaddr_bit = BIT(CONFIG_MI_UNIQUE_MACADDR_BIT) >> 24;
+	/* Find out CONFIG_MI_UNIQUE_MACADDR_BIT in which nic specific byte */
+	for(i = 3; i < 6; i++) {
+		if(((mi_unique_macaddr_bit >> 8) == 0) &&
+			((mac[i] & (u8)mi_unique_macaddr_bit) == 0)) {
+			is_uniq_macaddr = _TRUE;
+
+			RTW_INFO("%s() "MAC_FMT" : BIT%u is zero\n",
+				__func__, MAC_ARG(mac), CONFIG_MI_UNIQUE_MACADDR_BIT);
+			break;
+		}
+		mi_unique_macaddr_bit >>= 8;
+	}
+
+	if(is_uniq_macaddr) {
+		/* IFACE_ID1/IFACE_ID3 : set locally administered bit */
+		if(padapter->iface_id & BIT(0))
+			mac[0] |= BIT(1);
+		/* IFACE_ID2/IFACE_ID3 : set bit(CONFIG_MI_UNIQUE_MACADDR_BIT) */
+		if(padapter->iface_id >> 1)
+			mac[i] |= (u8)mi_unique_macaddr_bit;
+	} else
+#endif
+	{
+		/*
+		* If the BIT1 is 0, the address is universally administered.
+		* If it is 1, the address is locally administered
+		*/
+		mac[0] |= BIT(1);
+		if (padapter->iface_id > IFACE_ID1)
+			mac[0] ^= ((padapter->iface_id) << 2);
+	}
+
+	_rtw_memcpy(adapter_mac_addr(padapter), mac, ETH_ALEN);
+
+	RTW_INFO("%s if%d mac_addr : "MAC_FMT"\n", __func__, padapter->iface_id + 1, MAC_ARG(adapter_mac_addr(padapter)));
+
+	rtw_led_set_ctl_en_mask_virtual(padapter);
+	rtw_led_set_iface_en(padapter, 1);
+
+	res = _SUCCESS;
+
+free_drv_sw:
+	if (res != _SUCCESS && padapter)
+		rtw_free_drv_sw(padapter);
+free_adapter:
+	if (res != _SUCCESS && padapter) {
+		rtw_vmfree((u8 *)padapter, sizeof(*padapter));
+		padapter = NULL;
+	}
+exit:
+	return padapter;
+}
+u8 rtw_drv_add_vir_ifaces(struct dvobj_priv *dvobj)
+{
+	u8 i;
+	u8 rst = _FAIL;
+
+	if (dvobj->virtual_iface_num > (CONFIG_IFACE_NUMBER - 1))
+		dvobj->virtual_iface_num = (CONFIG_IFACE_NUMBER - 1);
+
+	for (i = 0; i < dvobj->virtual_iface_num; i++) {
+		if (rtw_drv_add_vir_if(dvobj) == NULL) {
+			RTW_ERR("rtw_drv_add_vir_if failed! (%d)\n", i);
+			goto _exit;
+		}
+	}
+	rst = _SUCCESS;
+_exit:
+	return rst;
+}
+
+static void rtw_drv_stop_vir_if(_adapter *padapter)
+{
+	struct net_device *pnetdev = NULL;
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(padapter);
+
+	if (padapter == NULL)
+		return;
+	RTW_INFO(FUNC_ADPT_FMT" enter\n", FUNC_ADPT_ARG(padapter));
+
+	pnetdev = padapter->pnetdev;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE)
+		rtw_disassoc_cmd(padapter, 0, RTW_CMDF_DIRECTLY|RTW_CMDF_WAIT_ACK);
+
+#ifdef CONFIG_AP_MODE
+	if (MLME_IS_AP(padapter) || MLME_IS_MESH(padapter)) {
+		free_mlme_ap_info(padapter);
+		#ifdef CONFIG_HOSTAPD_MLME
+		hostapd_mode_unload(padapter);
+		#endif
+	}
+#endif
+
+	if (padapter->netif_up == _TRUE) {
+		#ifdef CONFIG_XMIT_ACK
+		if (padapter->xmitpriv.ack_tx)
+			rtw_ack_tx_done(&padapter->xmitpriv, RTW_SCTX_DONE_DRV_STOP);
+		#endif
+		rtw_hw_iface_deinit(padapter);
+		if (!pwrctl->bInSuspend)
+			padapter->netif_up = _FALSE;
+	}
+	#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	rtw_stop_drv_threads(padapter);
+	#endif
+	/* cancel timer after thread stop */
+	rtw_cancel_all_timer(padapter);
+}
+void rtw_drv_stop_vir_ifaces(struct dvobj_priv *dvobj)
+{
+	int i;
+
+	for (i = VIF_START_ID; i < dvobj->iface_nums; i++)
+		rtw_drv_stop_vir_if(dvobj->padapters[i]);
+}
+
+static void rtw_drv_free_vir_if(_adapter *padapter)
+{
+	if (padapter == NULL)
+		return;
+
+	RTW_INFO(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(padapter));
+	rtw_free_drv_sw(padapter);
+
+	/* TODO: use rtw_os_ndevs_deinit instead at the first stage of driver's dev deinit function */
+	rtw_os_ndev_free(padapter);
+
+	rtw_vmfree((u8 *)padapter, sizeof(_adapter));
+}
+void rtw_drv_free_vir_ifaces(struct dvobj_priv *dvobj)
+{
+	int i;
+
+	for (i = VIF_START_ID; i < dvobj->iface_nums; i++)
+		rtw_drv_free_vir_if(dvobj->padapters[i]);
+}
+
+
+#endif /*end of CONFIG_CONCURRENT_MODE*/
+
+/* IPv4, IPv6 IP addr notifier */
+static int rtw_inetaddr_notifier_call(struct notifier_block *nb,
+				      unsigned long action, void *data)
+{
+	struct in_ifaddr *ifa = (struct in_ifaddr *)data;
+	struct net_device *ndev;
+	struct mlme_ext_priv *pmlmeext = NULL;
+	struct mlme_ext_info *pmlmeinfo = NULL;
+	_adapter *adapter = NULL;
+
+	if (!ifa || !ifa->ifa_dev || !ifa->ifa_dev->dev)
+		return NOTIFY_DONE;
+
+	ndev = ifa->ifa_dev->dev;
+
+	if (!is_rtw_ndev(ndev))
+		return NOTIFY_DONE;
+
+	adapter = (_adapter *)rtw_netdev_priv(ifa->ifa_dev->dev);
+
+	if (adapter == NULL)
+		return NOTIFY_DONE;
+
+	pmlmeext = &adapter->mlmeextpriv;
+	pmlmeinfo = &pmlmeext->mlmext_info;
+
+	switch (action) {
+	case NETDEV_UP:
+		_rtw_memcpy(pmlmeinfo->ip_addr, &ifa->ifa_address,
+					RTW_IP_ADDR_LEN);
+		RTW_DBG("%s[%s]: up IP: %pI4\n", __func__,
+					ifa->ifa_label, pmlmeinfo->ip_addr);
+	break;
+	case NETDEV_DOWN:
+		_rtw_memset(pmlmeinfo->ip_addr, 0, RTW_IP_ADDR_LEN);
+		RTW_DBG("%s[%s]: down IP: %pI4\n", __func__,
+					ifa->ifa_label, pmlmeinfo->ip_addr);
+	break;
+	default:
+		RTW_DBG("%s: default action\n", __func__);
+	break;
+	}
+	return NOTIFY_DONE;
+}
+
+#ifdef CONFIG_IPV6
+static int rtw_inet6addr_notifier_call(struct notifier_block *nb,
+				       unsigned long action, void *data)
+{
+	struct inet6_ifaddr *inet6_ifa = data;
+	struct net_device *ndev;
+	struct pwrctrl_priv *pwrctl = NULL;
+	struct mlme_ext_priv *pmlmeext = NULL;
+	struct mlme_ext_info *pmlmeinfo = NULL;
+	_adapter *adapter = NULL;
+
+	if (!inet6_ifa || !inet6_ifa->idev || !inet6_ifa->idev->dev)
+		return NOTIFY_DONE;
+
+	ndev = inet6_ifa->idev->dev;
+
+	if (!is_rtw_ndev(ndev))
+		return NOTIFY_DONE;
+
+	adapter = (_adapter *)rtw_netdev_priv(inet6_ifa->idev->dev);
+
+	if (adapter == NULL)
+		return NOTIFY_DONE;
+
+	pmlmeext =  &adapter->mlmeextpriv;
+	pmlmeinfo = &pmlmeext->mlmext_info;
+	pwrctl = adapter_to_pwrctl(adapter);
+
+	pmlmeext = &adapter->mlmeextpriv;
+	pmlmeinfo = &pmlmeext->mlmext_info;
+
+	switch (action) {
+	case NETDEV_UP:
+#ifdef CONFIG_WOWLAN
+		pwrctl->wowlan_ns_offload_en = _TRUE;
+#endif
+		_rtw_memcpy(pmlmeinfo->ip6_addr, &inet6_ifa->addr,
+					RTW_IPv6_ADDR_LEN);
+		RTW_DBG("%s: up IPv6 addrs: %pI6\n", __func__,
+					pmlmeinfo->ip6_addr);
+		break;
+	case NETDEV_DOWN:
+#ifdef CONFIG_WOWLAN
+		pwrctl->wowlan_ns_offload_en = _FALSE;
+#endif
+		_rtw_memset(pmlmeinfo->ip6_addr, 0, RTW_IPv6_ADDR_LEN);
+		RTW_DBG("%s: down IPv6 addrs: %pI6\n", __func__,
+					pmlmeinfo->ip6_addr);
+		break;
+	default:
+		RTW_DBG("%s: default action\n", __func__);
+		break;
+	}
+	return NOTIFY_DONE;
+}
+#endif
+
+static struct notifier_block rtw_inetaddr_notifier = {
+	.notifier_call = rtw_inetaddr_notifier_call
+};
+
+#ifdef CONFIG_IPV6
+static struct notifier_block rtw_inet6addr_notifier = {
+	.notifier_call = rtw_inet6addr_notifier_call
+};
+#endif
+
+void rtw_inetaddr_notifier_register(void)
+{
+	RTW_INFO("%s\n", __func__);
+	register_inetaddr_notifier(&rtw_inetaddr_notifier);
+#ifdef CONFIG_IPV6
+	register_inet6addr_notifier(&rtw_inet6addr_notifier);
+#endif
+}
+
+void rtw_inetaddr_notifier_unregister(void)
+{
+	RTW_INFO("%s\n", __func__);
+	unregister_inetaddr_notifier(&rtw_inetaddr_notifier);
+#ifdef CONFIG_IPV6
+	unregister_inet6addr_notifier(&rtw_inet6addr_notifier);
+#endif
+}
+
+int rtw_os_ndevs_register(struct dvobj_priv *dvobj)
+{
+	int i, status = _SUCCESS;
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+	_adapter *adapter;
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	if (rtw_cfg80211_dev_res_register(dvobj) != _SUCCESS) {
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+#endif
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+
+		if (i >= CONFIG_IFACE_NUMBER) {
+			RTW_ERR("%s %d >= CONFIG_IFACE_NUMBER(%d)\n", __func__, i, CONFIG_IFACE_NUMBER);
+			rtw_warn_on(1);
+			continue;
+		}
+
+		adapter = dvobj->padapters[i];
+		if (adapter) {
+			char *name;
+
+			#ifdef CONFIG_RTW_DYNAMIC_NDEV
+			if (!is_primary_adapter(adapter) &&
+				(i >= CONFIG_RTW_STATIC_NDEV_NUM))
+				break;
+			#endif
+
+			if (adapter->iface_id == IFACE_ID0)
+				name = regsty->ifname;
+			else if (adapter->iface_id == IFACE_ID1)
+				name = regsty->if2name;
+#if defined(CONFIG_PLATFORM_ANDROID) && (CONFIG_IFACE_NUMBER > 2)
+			else if (adapter->iface_id == IFACE_ID2)
+				name = regsty->if3name;
+#endif
+			else
+				name = "wlan%d";
+
+			status = rtw_os_ndev_register(adapter, name);
+
+			if (status != _SUCCESS) {
+				rtw_warn_on(1);
+				break;
+			}
+		}
+	}
+
+	if (status != _SUCCESS) {
+		for (; i >= 0; i--) {
+			adapter = dvobj->padapters[i];
+			if (adapter)
+				rtw_os_ndev_unregister(adapter);
+		}
+	}
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	if (status != _SUCCESS)
+		rtw_cfg80211_dev_res_unregister(dvobj);
+#endif
+	return status;
+}
+
+void rtw_os_ndevs_unregister(struct dvobj_priv *dvobj)
+{
+	int i;
+	_adapter *adapter = NULL;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		adapter = dvobj->padapters[i];
+
+		if (adapter == NULL)
+			continue;
+
+		rtw_os_ndev_unregister(adapter);
+	}
+
+#if defined(CONFIG_IOCTL_CFG80211)
+	rtw_cfg80211_dev_res_unregister(dvobj);
+#endif
+}
+
+/**
+ * rtw_os_ndevs_init - Allocate and register OS layer net devices and relating structures for @dvobj
+ * @dvobj: the dvobj on which this function applies
+ *
+ * Returns:
+ * _SUCCESS or _FAIL
+ */
+int rtw_os_ndevs_init(struct dvobj_priv *dvobj)
+{
+	int ret = _FAIL;
+
+	if (rtw_os_ndevs_alloc(dvobj) != _SUCCESS)
+		goto exit;
+
+	if (rtw_os_ndevs_register(dvobj) != _SUCCESS)
+		goto os_ndevs_free;
+
+	ret = _SUCCESS;
+
+os_ndevs_free:
+	if (ret != _SUCCESS)
+		rtw_os_ndevs_free(dvobj);
+exit:
+	return ret;
+}
+
+/**
+ * rtw_os_ndevs_deinit - Unregister and free OS layer net devices and relating structures for @dvobj
+ * @dvobj: the dvobj on which this function applies
+ */
+void rtw_os_ndevs_deinit(struct dvobj_priv *dvobj)
+{
+	rtw_os_ndevs_unregister(dvobj);
+	rtw_os_ndevs_free(dvobj);
+}
+
+#ifdef CONFIG_BR_EXT
+void netdev_br_init(struct net_device *netdev)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(netdev);
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 35))
+	rcu_read_lock();
+#endif
+
+	/* if(check_fwstate(pmlmepriv, WIFI_STATION_STATE|WIFI_ADHOC_STATE) == _TRUE) */
+	{
+		/* struct net_bridge	*br = netdev->br_port->br; */ /* ->dev->dev_addr; */
+		#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35))
+		if (netdev->br_port)
+		#else
+		if (rcu_dereference(adapter->pnetdev->rx_handler_data))
+		#endif
+		{
+			struct net_device *br_netdev;
+
+			br_netdev = rtw_get_bridge_ndev_by_name(CONFIG_BR_EXT_BRNAME);
+			if (br_netdev) {
+				_rtw_memcpy(adapter->br_mac, br_netdev->dev_addr, ETH_ALEN);
+				dev_put(br_netdev);
+				RTW_INFO(FUNC_NDEV_FMT" bind bridge dev "NDEV_FMT"("MAC_FMT")\n"
+					, FUNC_NDEV_ARG(netdev), NDEV_ARG(br_netdev), MAC_ARG(br_netdev->dev_addr));
+			} else {
+				RTW_INFO(FUNC_NDEV_FMT" can't get bridge dev by name \"%s\"\n"
+					, FUNC_NDEV_ARG(netdev), CONFIG_BR_EXT_BRNAME);
+			}
+		}
+
+		adapter->ethBrExtInfo.addPPPoETag = 1;
+	}
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 35))
+	rcu_read_unlock();
+#endif
+}
+#endif /* CONFIG_BR_EXT */
+
+#if 0
+/*FPGA_test*/
+static int _drv_enable_trx(struct dvobj_priv *d)
+{
+	struct _ADAPTER *adapter;
+	u32 status;
+
+
+	adapter = dvobj_get_primary_adapter(d);
+	if (adapter->netif_up == _FALSE) {
+		status = rtw_mi_start_drv_threads(adapter);
+
+		if (status == _FAIL) {
+			RTW_ERR("%s: Start threads Failed!\n", __FUNCTION__);
+			return -1;
+		}
+	}
+
+
+	return 0;
+}
+#endif
+
+static int _netdev_open(struct net_device *pnetdev)
+{
+	uint status;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	RTW_INFO(FUNC_NDEV_FMT" start\n", FUNC_NDEV_ARG(pnetdev));
+
+#ifdef DIRTY_FOR_WORK
+	if(pnetdev->priv_flags & IFF_DONT_BRIDGE)
+	{
+		RTW_INFO("Unable to be bridged !! Unlock for this iface !!\n");
+		pnetdev->priv_flags &= ~(IFF_DONT_BRIDGE);
+	}
+#endif
+
+	if (!dev_is_hw_start(dvobj)) {
+		dev_clr_surprise_removed(dvobj);
+		dev_clr_drv_stopped(dvobj);
+		RTW_ENABLE_FUNC(dvobj, DF_RX_BIT);
+		RTW_ENABLE_FUNC(dvobj, DF_TX_BIT);
+		status = rtw_hw_start(dvobj);
+		if (status == _FAIL)
+			goto netdev_open_error;
+		rtw_led_control(padapter, LED_CTL_NO_LINK);
+
+		#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+		if (0){
+			_set_timer(&dvobj->dynamic_chk_timer, 2000);
+		}
+		#endif
+		#if 0 /*CONFIG_CORE_THREAD*/
+		_drv_enable_trx(dvobj);/*FPGA_test*/
+		#endif
+	}
+
+	#ifdef CONFIG_RTW_FSM
+	rtw_fsm_start(&padapter->fsmpriv);
+	#endif
+
+	#ifdef CONFIG_RTW_NAPI
+	if(padapter->napi_state == NAPI_DISABLE) {
+		napi_enable(&padapter->napi);
+		padapter->napi_state = NAPI_ENABLE;
+	}
+	#endif
+
+	if (padapter->netif_up == _FALSE) {
+		if (rtw_hw_iface_init(padapter) == _FAIL) {
+			rtw_warn_on(1);
+			goto netdev_open_error;
+		}
+
+		/* rtw_netif_carrier_on(pnetdev); */ /* call this func when rtw_joinbss_event_callback return success */
+		rtw_netif_wake_queue(pnetdev);
+
+		#ifdef CONFIG_BR_EXT
+		if (is_primary_adapter(padapter))
+			netdev_br_init(pnetdev);
+		#endif /* CONFIG_BR_EXT */
+
+		padapter->netif_up = _TRUE;
+	}
+
+	#ifdef CONFIG_SIGNAL_STAT_PROCESS
+	rtw_set_signal_stat_timer(&padapter->recvinfo);
+	#endif
+
+	RTW_INFO(FUNC_NDEV_FMT" Success (netif_up=%d)\n", FUNC_NDEV_ARG(pnetdev), padapter->netif_up);
+	return 0;
+
+netdev_open_error:
+	padapter->netif_up = _FALSE;
+
+	#ifdef CONFIG_RTW_NAPI
+	if(padapter->napi_state == NAPI_ENABLE) {
+		napi_disable(&padapter->napi);
+		padapter->napi_state = NAPI_DISABLE;
+	}
+	#endif
+
+	rtw_netif_carrier_off(pnetdev);
+	rtw_netif_stop_queue(pnetdev);
+
+	RTW_ERR(FUNC_NDEV_FMT" Failed!! (netif_up=%d)\n", FUNC_NDEV_ARG(pnetdev), padapter->netif_up);
+
+	return -1;
+
+}
+
+int netdev_open(struct net_device *pnetdev)
+{
+	int ret = _FALSE;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+
+	if (pwrctrlpriv->bInSuspend == _TRUE) {
+		RTW_INFO(" [WARN] "ADPT_FMT" %s  failed, bInSuspend=%d\n", ADPT_ARG(padapter), __func__, pwrctrlpriv->bInSuspend);
+		return 0;
+	}
+
+	RTW_INFO(FUNC_NDEV_FMT" , netif_up=%d\n", FUNC_NDEV_ARG(pnetdev), padapter->netif_up);
+	/*rtw_dump_stack();*/
+	_rtw_mutex_lock_interruptible(&(adapter_to_dvobj(padapter)->hw_init_mutex));
+	ret = _netdev_open(pnetdev);
+	_rtw_mutex_unlock(&(adapter_to_dvobj(padapter)->hw_init_mutex));
+
+
+#ifdef CONFIG_AUTO_AP_MODE
+	if (padapter->iface_id == IFACE_ID2)
+		rtw_start_auto_ap(padapter);
+#endif
+
+	return ret;
+}
+
+int _pm_netdev_open(_adapter *padapter)
+{
+	uint status;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
+	struct net_device *pnetdev = padapter->pnetdev;
+
+	RTW_INFO(FUNC_NDEV_FMT" start\n", FUNC_NDEV_ARG(pnetdev));
+
+	if (!rtw_hw_is_init_completed(dvobj)) { // ips
+		dev_clr_surprise_removed(dvobj);
+		status = rtw_hw_start(dvobj);
+		if (status == _FAIL)
+			goto netdev_open_error;
+		rtw_led_control(padapter, LED_CTL_NO_LINK);
+		#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+		_set_timer(&dvobj->dynamic_chk_timer, 2000);
+		#endif
+
+	#if 0 /*ndef CONFIG_IPS_CHECK_IN_WD*/
+		rtw_set_pwr_state_check_timer(pwrctrlpriv);
+	#endif /*CONFIG_IPS_CHECK_IN_WD*/
+	}
+
+	/*if (padapter->netif_up == _FALSE) */
+	{
+		if (rtw_hw_iface_init(padapter) == _FAIL)
+			goto netdev_open_error;
+
+		padapter->netif_up = _TRUE;
+	}
+
+	RTW_INFO(FUNC_NDEV_FMT" Success (netif_up=%d)\n", FUNC_NDEV_ARG(pnetdev), padapter->netif_up);
+	return 0;
+
+netdev_open_error:
+	padapter->netif_up = _FALSE;
+
+	rtw_netif_carrier_off(pnetdev);
+	rtw_netif_stop_queue(pnetdev);
+
+	RTW_ERR(FUNC_NDEV_FMT" Failed!! (netif_up=%d)\n", FUNC_NDEV_ARG(pnetdev), padapter->netif_up);
+
+	return -1;
+
+}
+int _mi_pm_netdev_open(struct net_device *pnetdev)
+{
+	int i;
+	int status = 0;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	_adapter *iface;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (iface->netif_up) {
+			status = _pm_netdev_open(iface);
+			if (status == -1) {
+				RTW_ERR("%s failled\n", __func__);
+				break;
+			}
+		}
+	}
+
+	return status;
+}
+
+int pm_netdev_open(struct net_device *pnetdev, u8 bnormal)
+{
+	int status = 0;
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+
+	if (_TRUE == bnormal) {
+		_rtw_mutex_lock_interruptible(&(adapter_to_dvobj(padapter)->hw_init_mutex));
+		status = _mi_pm_netdev_open(pnetdev);
+
+		_rtw_mutex_unlock(&(adapter_to_dvobj(padapter)->hw_init_mutex));
+	}
+
+	return status;
+}
+
+static int netdev_close(struct net_device *pnetdev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(padapter);
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	RTW_INFO(FUNC_NDEV_FMT" , netif_up=%d\n", FUNC_NDEV_ARG(pnetdev), padapter->netif_up);
+
+	pmlmepriv->LinkDetectInfo.bBusyTraffic = _FALSE;
+
+	rtw_scan_abort(padapter, 0); /* stop scanning process before wifi is going to down */
+#ifdef CONFIG_IOCTL_CFG80211
+	rtw_cfg80211_wait_scan_req_empty(padapter, 200);
+	/* padapter->rtw_wdev->iftype = NL80211_IFTYPE_MONITOR; */ /* set this at the end */
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+	{
+		RTW_INFO("netif_up=%d, hw_init_completed=%s\n",
+			padapter->netif_up,
+			rtw_hw_is_init_completed(dvobj) ? "_TRUE" : "_FALSE");
+
+		/* s1. */
+		if (pnetdev)
+			rtw_netif_stop_queue(pnetdev);
+
+		rtw_join_abort_timeout(padapter, 300);
+
+		/* s2. */
+		if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+			rtw_disassoc_cmd(padapter, 500, RTW_CMDF_WAIT_ACK);
+			/* s2-2*/
+			if (MLME_IS_STA(padapter) == _FALSE)
+				rtw_free_assoc_resources_cmd(padapter, _TRUE, RTW_CMDF_WAIT_ACK);
+			/* s2-3.  indicate disconnect to os */
+			rtw_indicate_disconnect(padapter, 0, _FALSE);
+			/* s2-4. */
+			rtw_free_network_queue(padapter, _TRUE);
+			rtw_free_mld_network_queue(padapter, _TRUE);
+
+			pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_NET_DEVICE_DOWN;
+			pmlmeinfo->wifi_reason_code = WLAN_REASON_DEAUTH_LEAVING;
+		}
+
+		rtw_ap_stop_wait(padapter);
+	}
+
+#ifdef CONFIG_BR_EXT
+	/* if (OPMODE & (WIFI_STATION_STATE | WIFI_ADHOC_STATE)) */
+	{
+		/* void nat25_db_cleanup(_adapter *priv); */
+		nat25_db_cleanup(padapter);
+	}
+#endif /* CONFIG_BR_EXT */
+
+#ifdef CONFIG_P2P
+	if (!rtw_p2p_chk_role(&padapter->wdinfo, P2P_ROLE_DISABLE))
+		rtw_p2p_enable(padapter, P2P_ROLE_DISABLE);
+#endif /* CONFIG_P2P */
+
+#ifdef CONFIG_WAPI_SUPPORT
+	rtw_wapi_disable_tx(padapter);
+#endif
+
+#ifdef CONFIG_RTW_FSM
+	rtw_fsm_stop(&padapter->fsmpriv);
+#endif
+
+#ifdef CONFIG_RTW_NAPI
+	if (padapter->napi_state == NAPI_ENABLE) {
+		napi_disable(&padapter->napi);
+		padapter->napi_state = NAPI_DISABLE;
+	}
+#endif /* CONFIG_RTW_NAPI */
+	rtw_hw_iface_deinit(padapter);
+	padapter->netif_up = _FALSE;
+
+	RTW_INFO("-871x_drv - drv_close, netif_up=%d\n", padapter->netif_up);
+
+	return 0;
+
+}
+
+int pm_netdev_close(struct net_device *pnetdev, u8 bnormal)
+{
+	int status = 0;
+
+	status = netdev_close(pnetdev);
+
+	return status;
+}
+
+void rtw_ndev_destructor(struct net_device *ndev)
+{
+	RTW_INFO(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(ndev));
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (ndev->ieee80211_ptr)
+		rtw_mfree((u8 *)ndev->ieee80211_ptr, sizeof(struct wireless_dev));
+#endif
+	free_netdev(ndev);
+}
+
+#ifdef CONFIG_ARP_KEEP_ALIVE_GW
+struct route_info {
+	struct in_addr dst_addr;
+	struct in_addr src_addr;
+	struct in_addr gateway;
+	unsigned int dev_index;
+};
+
+static void parse_routes(struct nlmsghdr *nl_hdr, struct route_info *rt_info)
+{
+	struct rtmsg *rt_msg;
+	struct rtattr *rt_attr;
+	int rt_len;
+
+	rt_msg = (struct rtmsg *) NLMSG_DATA(nl_hdr);
+	if ((rt_msg->rtm_family != AF_INET) || (rt_msg->rtm_table != RT_TABLE_MAIN))
+		return;
+
+	rt_attr = (struct rtattr *) RTM_RTA(rt_msg);
+	rt_len = RTM_PAYLOAD(nl_hdr);
+
+	for (; RTA_OK(rt_attr, rt_len); rt_attr = RTA_NEXT(rt_attr, rt_len)) {
+		switch (rt_attr->rta_type) {
+		case RTA_OIF:
+			rt_info->dev_index = *(int *) RTA_DATA(rt_attr);
+			break;
+		case RTA_GATEWAY:
+			rt_info->gateway.s_addr = *(u_int *) RTA_DATA(rt_attr);
+			break;
+		case RTA_PREFSRC:
+			rt_info->src_addr.s_addr = *(u_int *) RTA_DATA(rt_attr);
+			break;
+		case RTA_DST:
+			rt_info->dst_addr.s_addr = *(u_int *) RTA_DATA(rt_attr);
+			break;
+		}
+	}
+}
+
+static int route_dump(u32 *gw_addr , int *gw_index)
+{
+	int err = 0;
+	struct socket *sock;
+	struct {
+		struct nlmsghdr nlh;
+		struct rtgenmsg g;
+	} req;
+	struct msghdr msg;
+	struct iovec iov;
+	struct sockaddr_nl nladdr;
+	mm_segment_t oldfs;
+	char *pg;
+	int size = 0;
+
+	err = sock_create(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE, &sock);
+	if (err) {
+		printk(": Could not create a datagram socket, error = %d\n", -ENXIO);
+		return err;
+	}
+
+	memset(&nladdr, 0, sizeof(nladdr));
+	nladdr.nl_family = AF_NETLINK;
+
+	req.nlh.nlmsg_len = sizeof(req);
+	req.nlh.nlmsg_type = RTM_GETROUTE;
+	req.nlh.nlmsg_flags = NLM_F_ROOT | NLM_F_MATCH | NLM_F_REQUEST;
+	req.nlh.nlmsg_pid = 0;
+	req.g.rtgen_family = AF_INET;
+
+	iov.iov_base = &req;
+	iov.iov_len = sizeof(req);
+
+	msg.msg_name = &nladdr;
+	msg.msg_namelen = sizeof(nladdr);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+	/* referece:sock_xmit in kernel code
+	 * WRITE for sock_sendmsg, READ for sock_recvmsg
+	 * third parameter for msg_iovlen
+	 * last parameter for iov_len
+	 */
+	iov_iter_init(&msg.msg_iter, WRITE, &iov, 1, sizeof(req));
+#else
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+#endif
+	msg.msg_control = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags = MSG_DONTWAIT;
+
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+	err = sock_sendmsg(sock, &msg);
+#else
+	err = sock_sendmsg(sock, &msg, sizeof(req));
+#endif
+	set_fs(oldfs);
+
+	if (err < 0)
+		goto out_sock;
+
+	pg = (char *) __get_free_page(GFP_KERNEL);
+	if (pg == NULL) {
+		err = -ENOMEM;
+		goto out_sock;
+	}
+
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+restart:
+#endif
+
+	for (;;) {
+		struct nlmsghdr *h;
+
+		iov.iov_base = pg;
+		iov.iov_len = PAGE_SIZE;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+		iov_iter_init(&msg.msg_iter, READ, &iov, 1, PAGE_SIZE);
+#endif
+
+		oldfs = get_fs();
+		set_fs(KERNEL_DS);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0))
+		err = sock_recvmsg(sock, &msg, MSG_DONTWAIT);
+#else
+		err = sock_recvmsg(sock, &msg, PAGE_SIZE, MSG_DONTWAIT);
+#endif
+		set_fs(oldfs);
+
+		if (err < 0)
+			goto out_sock_pg;
+
+		if (msg.msg_flags & MSG_TRUNC) {
+			err = -ENOBUFS;
+			goto out_sock_pg;
+		}
+
+		h = (struct nlmsghdr *) pg;
+
+		while (NLMSG_OK(h, err)) {
+			struct route_info rt_info;
+			if (h->nlmsg_type == NLMSG_DONE) {
+				err = 0;
+				goto done;
+			}
+
+			if (h->nlmsg_type == NLMSG_ERROR) {
+				struct nlmsgerr *errm = (struct nlmsgerr *) NLMSG_DATA(h);
+				err = errm->error;
+				printk("NLMSG error: %d\n", errm->error);
+				goto done;
+			}
+
+			if (h->nlmsg_type == RTM_GETROUTE)
+				printk("RTM_GETROUTE: NLMSG: %d\n", h->nlmsg_type);
+			if (h->nlmsg_type != RTM_NEWROUTE) {
+				printk("NLMSG: %d\n", h->nlmsg_type);
+				err = -EINVAL;
+				goto done;
+			}
+
+			memset(&rt_info, 0, sizeof(struct route_info));
+			parse_routes(h, &rt_info);
+			if (!rt_info.dst_addr.s_addr && rt_info.gateway.s_addr && rt_info.dev_index) {
+				*gw_addr = rt_info.gateway.s_addr;
+				*gw_index = rt_info.dev_index;
+
+			}
+			h = NLMSG_NEXT(h, err);
+		}
+
+		if (err) {
+			printk("!!!Remnant of size %d %d %d\n", err, h->nlmsg_len, h->nlmsg_type);
+			err = -EINVAL;
+			break;
+		}
+	}
+
+done:
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+	if (!err && req.g.rtgen_family == AF_INET) {
+		req.g.rtgen_family = AF_INET6;
+
+		iov.iov_base = &req;
+		iov.iov_len = sizeof(req);
+
+		msg.msg_name = &nladdr;
+		msg.msg_namelen = sizeof(nladdr);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+		iov_iter_init(&msg.msg_iter, WRITE, &iov, 1, sizeof(req));
+#else
+		msg.msg_iov = &iov;
+		msg.msg_iovlen = 1;
+#endif
+		msg.msg_control = NULL;
+		msg.msg_controllen = 0;
+		msg.msg_flags = MSG_DONTWAIT;
+
+		oldfs = get_fs();
+		set_fs(KERNEL_DS);
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+		err = sock_sendmsg(sock, &msg);
+#else
+		err = sock_sendmsg(sock, &msg, sizeof(req));
+#endif
+		set_fs(oldfs);
+
+		if (err > 0)
+			goto restart;
+	}
+#endif
+
+out_sock_pg:
+	free_page((unsigned long) pg);
+
+out_sock:
+	sock_release(sock);
+	return err;
+}
+
+static int arp_query(unsigned char *haddr, u32 paddr,
+		     struct net_device *dev)
+{
+	struct neighbour *neighbor_entry;
+	int	ret = 0;
+
+	neighbor_entry = neigh_lookup(&arp_tbl, &paddr, dev);
+
+	if (neighbor_entry != NULL) {
+		neighbor_entry->used = jiffies;
+		if (neighbor_entry->nud_state & NUD_VALID) {
+			_rtw_memcpy(haddr, neighbor_entry->ha, dev->addr_len);
+			ret = 1;
+		}
+		neigh_release(neighbor_entry);
+	}
+	return ret;
+}
+
+static int get_defaultgw(u32 *ip_addr , char mac[])
+{
+	int gw_index = 0; /* oif device index */
+	struct net_device *gw_dev = NULL; /* oif device */
+
+	route_dump(ip_addr, &gw_index);
+
+	if (!(*ip_addr) || !gw_index) {
+		/* RTW_INFO("No default GW\n"); */
+		return -1;
+	}
+
+	gw_dev = dev_get_by_index(&init_net, gw_index);
+
+	if (gw_dev == NULL) {
+		/* RTW_INFO("get Oif Device Fail\n"); */
+		return -1;
+	}
+
+	if (!arp_query(mac, *ip_addr, gw_dev)) {
+		/* RTW_INFO( "arp query failed\n"); */
+		dev_put(gw_dev);
+		return -1;
+
+	}
+	dev_put(gw_dev);
+
+	return 0;
+}
+
+int	rtw_gw_addr_query(_adapter *padapter)
+{
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct pwrctrl_priv *pwrctl = adapter_to_pwrctl(padapter);
+	u32 gw_addr = 0; /* default gw address */
+	unsigned char gw_mac[32] = {0}; /* default gw mac */
+	int i;
+	int res;
+
+	res = get_defaultgw(&gw_addr, gw_mac);
+	if (!res) {
+		pmlmepriv->gw_ip[0] = gw_addr & 0xff;
+		pmlmepriv->gw_ip[1] = (gw_addr & 0xff00) >> 8;
+		pmlmepriv->gw_ip[2] = (gw_addr & 0xff0000) >> 16;
+		pmlmepriv->gw_ip[3] = (gw_addr & 0xff000000) >> 24;
+		_rtw_memcpy(pmlmepriv->gw_mac_addr, gw_mac, ETH_ALEN);
+		RTW_INFO("%s Gateway Mac:\t" MAC_FMT "\n", __FUNCTION__, MAC_ARG(pmlmepriv->gw_mac_addr));
+		RTW_INFO("%s Gateway IP:\t" IP_FMT "\n", __FUNCTION__, IP_ARG(pmlmepriv->gw_ip));
+	} else
+		RTW_INFO("Get Gateway IP/MAC fail!\n");
+
+	return res;
+}
+#endif
+
+int rtw_suspend_free_assoc_resource(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+#ifdef CONFIG_P2P
+	struct wifidirect_info	*pwdinfo = &padapter->wdinfo;
+#endif /* CONFIG_P2P */
+
+	RTW_INFO("==> "FUNC_ADPT_FMT" entry....\n", FUNC_ADPT_ARG(padapter));
+
+	if (rtw_chk_roam_flags(padapter, RTW_ROAM_ON_RESUME)) {
+		if (MLME_IS_STA(padapter) &&
+		    #ifdef CONFIG_P2P
+		    !rtw_p2p_chk_role(pwdinfo, P2P_ROLE_CLIENT) &&
+		    !rtw_p2p_chk_role(pwdinfo, P2P_ROLE_GO) &&
+		    #endif /* CONFIG_P2P */
+		    check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+			RTW_INFO("%s %s(" MAC_FMT "), length:%d assoc_ssid.length:%d\n", __FUNCTION__,
+				pmlmepriv->dev_cur_network.network.Ssid.Ssid,
+				MAC_ARG(pmlmepriv->dev_cur_network.network.MacAddress),
+				pmlmepriv->dev_cur_network.network.Ssid.SsidLength,
+				pmlmepriv->assoc_ssid.SsidLength);
+			rtw_set_to_roam(padapter, 1);
+		}
+	}
+
+	if (MLME_IS_STA(padapter) && check_fwstate(pmlmepriv, WIFI_ASOC_STATE))
+		rtw_disassoc_cmd(padapter, 0, RTW_CMDF_DIRECTLY|RTW_CMDF_WAIT_ACK);
+#ifdef CONFIG_AP_MODE
+	else if (MLME_IS_AP(padapter) || MLME_IS_MESH(padapter))
+		rtw_sta_flush(padapter, _FALSE);
+#endif
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+		/* s2-2. */
+		if (MLME_IS_STA(padapter) == _FALSE)
+			rtw_free_assoc_resources(padapter, _TRUE);
+
+		/* s2-3.  indicate disconnect to os */
+		if (MLME_IS_STA(padapter)) {
+			rtw_indicate_disconnect(padapter, 0, _FALSE);
+			pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_SYSTEM_IN_SUSPEND;
+			pmlmeinfo->wifi_reason_code = WLAN_REASON_DEAUTH_LEAVING;
+		}
+	}
+	/* s2-4. */
+	rtw_free_network_queue(padapter, _TRUE);
+	rtw_free_mld_network_queue(padapter, _TRUE);
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY)) {
+		RTW_PRINT("%s: fw_under_survey\n", __func__);
+		rtw_indicate_scan_done(padapter, 1);
+		clr_fwstate(pmlmepriv, WIFI_UNDER_SURVEY);
+	}
+
+	if (check_fwstate(pmlmepriv, WIFI_UNDER_LINKING) == _TRUE) {
+		RTW_PRINT("%s: fw_under_linking\n", __FUNCTION__);
+		rtw_indicate_disconnect(padapter, 0, _FALSE);
+		pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+		pmlmeinfo->disconnect_code = DISCONNECTION_BY_SYSTEM_DUE_TO_SYSTEM_IN_SUSPEND;
+		pmlmeinfo->wifi_reason_code = WLAN_REASON_DEAUTH_LEAVING;
+	}
+
+	RTW_INFO("<== "FUNC_ADPT_FMT" exit....\n", FUNC_ADPT_ARG(padapter));
+	return _SUCCESS;
+}
+
+#ifdef CONFIG_WOWLAN
+int rtw_suspend_wow(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	int ret = _SUCCESS;
+
+	RTW_INFO("==> "FUNC_ADPT_FMT" entry....\n", FUNC_ADPT_ARG(padapter));
+
+
+	RTW_INFO("wowlan_mode: %d\n", pwrpriv->wowlan_mode);
+	RTW_INFO("wowlan_pno_enable: %d\n", pwrpriv->wowlan_pno_enable);
+#ifdef CONFIG_P2P_WOWLAN
+	RTW_INFO("wowlan_p2p_enable: %d\n", pwrpriv->wowlan_p2p_enable);
+#endif
+
+	if (pwrpriv->wowlan_mode == _TRUE) {
+		rtw_phl_watchdog_stop(dvobj->phl);
+		rtw_mi_netif_stop_queue(padapter);
+		#ifdef CONFIG_CONCURRENT_MODE
+		rtw_mi_buddy_netif_carrier_off(padapter);
+		#endif
+
+		rtw_hw_wow(padapter, _TRUE);
+
+		if (rtw_chk_roam_flags(padapter, RTW_ROAM_ON_RESUME)) {
+			if (MLME_IS_STA(padapter)
+			    && check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+				RTW_INFO("%s %s(" MAC_FMT "), length:%d assoc_ssid.length:%d\n", __FUNCTION__,
+					pmlmepriv->dev_cur_network.network.Ssid.Ssid,
+					MAC_ARG(pmlmepriv->dev_cur_network.network.MacAddress),
+					pmlmepriv->dev_cur_network.network.Ssid.SsidLength,
+					 pmlmepriv->assoc_ssid.SsidLength);
+
+				rtw_set_to_roam(padapter, 0);
+			}
+		}
+
+		RTW_PRINT("%s: wowmode suspending\n", __func__);
+
+		if (check_fwstate(pmlmepriv, WIFI_UNDER_SURVEY) == _TRUE) {
+			RTW_PRINT("%s: fw_under_survey\n", __func__);
+			rtw_indicate_scan_done(padapter, 1);
+			clr_fwstate(pmlmepriv, WIFI_UNDER_SURVEY);
+		}
+
+		#ifdef CONFIG_SDIO_HCI
+		#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+		rtw_sdio_free_irq(dvobj);
+		#endif
+		#endif/*CONFIG_SDIO_HCI*/
+
+#ifdef CONFIG_CONCURRENT_MODE
+		rtw_mi_buddy_suspend_free_assoc_resource(padapter);
+#endif
+	} else
+		RTW_PRINT("%s: ### ERROR ### wowlan_mode=%d\n", __FUNCTION__, pwrpriv->wowlan_mode);
+	RTW_INFO("<== "FUNC_ADPT_FMT" exit....\n", FUNC_ADPT_ARG(padapter));
+	return ret;
+}
+#endif /* #ifdef CONFIG_WOWLAN */
+
+#ifdef CONFIG_AP_WOWLAN
+int rtw_suspend_ap_wow(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	int ret = _SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("==> "FUNC_ADPT_FMT" entry....\n", FUNC_ADPT_ARG(padapter));
+
+	pwrpriv->wowlan_ap_mode = _TRUE;
+
+	RTW_INFO("wowlan_ap_mode: %d\n", pwrpriv->wowlan_ap_mode);
+
+	rtw_mi_netif_stop_queue(padapter);
+
+	/* 0. Power off LED */
+	rtw_led_control(padapter, LED_CTL_POWER_OFF);
+
+	/* 1. stop thread */
+	dev_set_drv_stopped(dvobj);	/*for stop thread*/
+	#if 0
+	rtw_mi_stop_drv_threads(padapter);
+	#endif
+	dev_clr_drv_stopped(dvobj);	/*for 32k command*/
+
+	#ifdef CONFIG_SDIO_HCI
+	/* 2.2 free irq */
+	#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+	rtw_sdio_free_irq(dvobj);
+	#endif
+	#endif/*CONFIG_SDIO_HCI*/
+
+	RTW_PRINT("%s: wowmode suspending\n", __func__);
+
+	if (rtw_mi_check_status(padapter, MI_LINKED)) {
+		struct rtw_chan_def u_chdef = {0};
+
+		if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role, padapter_link->wrlink, &u_chdef)
+						!= RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s get union chandef failed\n", __func__);
+			rtw_warn_on(1);
+	}
+
+		RTW_INFO("back to linked/linking union - ch:%u, bw:%u, offset:%u\n",
+			u_chdef.chan, u_chdef.bw, u_chdef.offset);
+		set_bch_bwmode(padapter, padapter_link, u_chdef.band, u_chdef.chan, u_chdef.offset, u_chdef.bw, RFK_TYPE_FORCE_NOT_DO);
+	}
+
+	/*FOR ONE AP - TODO :Multi-AP*/
+	{
+		int i;
+		_adapter *iface;
+		struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if ((iface) && rtw_is_adapter_up(iface)) {
+				if (check_fwstate(&iface->mlmepriv, WIFI_AP_STATE | WIFI_MESH_STATE) == _FALSE)
+					rtw_suspend_free_assoc_resource(iface);
+			}
+		}
+
+	}
+
+	RTW_INFO("<== "FUNC_ADPT_FMT" exit....\n", FUNC_ADPT_ARG(padapter));
+	return ret;
+}
+#endif /* #ifdef CONFIG_AP_WOWLAN */
+
+
+int rtw_suspend_normal(_adapter *padapter)
+{
+	int ret = _SUCCESS;
+
+	RTW_INFO("==> "FUNC_ADPT_FMT" entry....\n", FUNC_ADPT_ARG(padapter));
+
+	rtw_mi_netif_caroff_qstop(padapter);
+
+	rtw_mi_suspend_free_assoc_resource(padapter);
+
+	rtw_led_control(padapter, LED_CTL_POWER_OFF);
+
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	rtw_stop_cmd_thread(padapter);
+#endif
+#ifdef CONFIG_CONCURRENT_MODE
+	rtw_drv_stop_vir_ifaces(adapter_to_dvobj(padapter));
+#endif
+	rtw_drv_stop_prim_iface(padapter);
+
+	if (rtw_hw_is_init_completed(adapter_to_dvobj(padapter)))
+		rtw_hw_stop(adapter_to_dvobj(padapter));
+	dev_set_surprise_removed(adapter_to_dvobj(padapter));
+
+	#ifdef CONFIG_SDIO_HCI
+	rtw_sdio_deinit(adapter_to_dvobj(padapter));
+
+	#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+	rtw_sdio_free_irq(adapter_to_dvobj(padapter));
+	#endif
+	#endif /*CONFIG_SDIO_HCI*/
+
+	RTW_INFO("<== "FUNC_ADPT_FMT" exit....\n", FUNC_ADPT_ARG(padapter));
+	return ret;
+}
+
+int rtw_suspend_common(_adapter *padapter)
+{
+	struct dvobj_priv *dvobj = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &dvobj->drv_dbg;
+	struct pwrctrl_priv *pwrpriv = dvobj_to_pwrctl(dvobj);
+#ifdef CONFIG_WOWLAN
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct registry_priv *registry_par = &padapter->registrypriv;
+#endif
+
+	int ret = 0;
+	systime start_time = rtw_get_current_time();
+
+	RTW_PRINT(" suspend start\n");
+	RTW_INFO("==> %s (%s:%d)\n", __FUNCTION__, current->comm, current->pid);
+
+	pdbgpriv->dbg_suspend_cnt++;
+
+	pwrpriv->bInSuspend = _TRUE;
+
+	if ( RTW_CANNOT_RUN(dvobj)) {
+		RTW_INFO("%s bDriverStopped=%s bSurpriseRemoved = %s\n", __func__
+			 , dev_is_drv_stopped(adapter_to_dvobj(padapter)) ? "True" : "False"
+			, dev_is_surprise_removed(adapter_to_dvobj(padapter)) ? "True" : "False");
+		pdbgpriv->dbg_suspend_error_cnt++;
+		goto exit;
+	}
+
+	rtw_mi_scan_abort(padapter, _TRUE);
+
+	rtw_mi_cancel_all_timer(padapter);
+
+	if (rtw_mi_check_status(padapter, MI_AP_MODE) == _FALSE) {
+#ifdef CONFIG_WOWLAN
+		if (WOWLAN_IS_STA_MIX_MODE(padapter))
+			pwrpriv->wowlan_mode = _TRUE;
+		else if ( registry_par->wowlan_enable && check_fwstate(pmlmepriv, WIFI_ASOC_STATE))
+			pwrpriv->wowlan_mode = _TRUE;
+		else if (pwrpriv->wowlan_pno_enable == _TRUE)
+			pwrpriv->wowlan_mode |= pwrpriv->wowlan_pno_enable;
+
+#ifdef CONFIG_P2P_WOWLAN
+		if (rtw_p2p_chk_role(&padapter->wdinfo, P2P_ROLE_CLIENT) ||
+		    rtw_p2p_chk_role(&padapter->wdinfo, P2P_ROLE_GO))
+			pwrpriv->wowlan_p2p_mode = _TRUE;
+		if (_TRUE == pwrpriv->wowlan_p2p_mode)
+			pwrpriv->wowlan_mode |= pwrpriv->wowlan_p2p_mode;
+#endif /* CONFIG_P2P_WOWLAN */
+
+		if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE))
+			pwrpriv->wowlan_no_link_mode = _FALSE;
+		else
+			pwrpriv->wowlan_no_link_mode = _TRUE;
+
+		if (pwrpriv->wowlan_mode == _TRUE)
+			rtw_suspend_wow(padapter);
+		else
+#endif /* CONFIG_WOWLAN */
+			rtw_suspend_normal(padapter);
+	} else if (rtw_mi_check_status(padapter, MI_AP_MODE)) {
+#ifdef CONFIG_AP_WOWLAN
+		rtw_suspend_ap_wow(padapter);
+#else
+		rtw_suspend_normal(padapter);
+#endif /*CONFIG_AP_WOWLAN*/
+	}
+
+
+	RTW_PRINT("rtw suspend success in %d ms\n",
+		  rtw_get_passing_time_ms(start_time));
+
+exit:
+	RTW_INFO("<===  %s return %d.............. in %dms\n", __FUNCTION__
+		 , ret, rtw_get_passing_time_ms(start_time));
+
+	return ret;
+}
+
+#ifdef CONFIG_WOWLAN
+int rtw_resume_process_wow(_adapter *padapter)
+{
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct dvobj_priv *dvobj = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &dvobj->drv_dbg;
+	struct wow_priv *wowpriv = dvobj_to_wowlan(dvobj);
+	struct registry_priv  *registry_par = &padapter->registrypriv;
+	int ret = _SUCCESS;
+
+	RTW_INFO("==> "FUNC_ADPT_FMT" entry....\n", FUNC_ADPT_ARG(padapter));
+
+	if (padapter) {
+		pwrpriv = adapter_to_pwrctl(padapter);
+	} else {
+		pdbgpriv->dbg_resume_error_cnt++;
+		ret = -1;
+		goto exit;
+	}
+
+	if (RTW_CANNOT_RUN(dvobj)) {
+		RTW_INFO("%s pdapter %p bDriverStopped %s bSurpriseRemoved %s\n"
+			 , __func__, padapter
+			 , dev_is_drv_stopped(dvobj) ? "True" : "False"
+			, dev_is_surprise_removed(dvobj) ? "True" : "False");
+		goto exit;
+	}
+
+	pwrpriv->wowlan_in_resume = _TRUE;
+
+	if (pwrpriv->wowlan_mode == _TRUE) {
+		/*pwrpriv->bFwCurrentInPSMode = _FALSE;*/
+
+		#ifdef CONFIG_SDIO_HCI
+		#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+		if (rtw_sdio_alloc_irq(dvobj) != _SUCCESS) {
+			ret = -1;
+			goto exit;
+		}
+		#endif
+		#endif/*CONFIG_SDIO_HCI*/
+
+#ifdef CONFIG_SIGNAL_STAT_PROCESS
+		rtw_set_signal_stat_timer(&padapter->recvinfo);
+#endif
+#ifdef CONFIG_CONCURRENT_MODE
+		rtw_mi_buddy_reset_drv_sw(padapter);
+#endif
+		rtw_hw_wow(padapter, _FALSE);
+
+		dev_clr_drv_stopped(dvobj);
+		RTW_INFO("%s: wowmode resuming, DriverStopped:%s\n", __func__, dev_is_drv_stopped(dvobj) ? "True" : "False");
+
+		#if 0
+		rtw_mi_start_drv_threads(padapter);
+		#endif
+
+#ifdef CONFIG_CONCURRENT_MODE
+		rtw_mi_buddy_netif_carrier_on(padapter);
+#endif
+
+		/* start netif queue */
+		rtw_mi_netif_wake_queue(padapter);
+
+	} else
+
+		RTW_PRINT("%s: ### ERROR ### wowlan_mode=%d\n", __FUNCTION__, pwrpriv->wowlan_mode);
+
+	if (padapter->pid[1] != 0) {
+		RTW_INFO("pid[1]:%d\n", padapter->pid[1]);
+		rtw_signal_process(padapter->pid[1], SIGUSR2);
+	}
+
+	if (IS_DISCONNECT_WOW_REASON(wowpriv->wow_wake_reason)) {
+		RTW_INFO("%s: disconnect reason: %02x\n", __func__,
+			 wowpriv->wow_wake_reason);
+
+		rtw_disassoc_cmd(padapter, 0, RTW_CMDF_WAIT_ACK);
+		if (MLME_IS_ASOC(padapter) == _TRUE)
+			rtw_free_assoc_resources(padapter, _TRUE);
+
+		if (wowpriv->wow_wake_reason == RTW_MAC_WOW_RX_DISASSOC ||
+		    wowpriv->wow_wake_reason == RTW_MAC_WOW_RX_DEAUTH)
+			rtw_indicate_disconnect(padapter, pmlmeinfo->wifi_reason_code, _FALSE);
+		else
+			rtw_indicate_disconnect(padapter, 0, _FALSE);
+
+		pmlmeinfo->state = WIFI_FW_NULL_STATE;
+
+		pmlmeinfo->disconnect_occurred_time = rtw_systime_to_ms(rtw_get_current_time());
+		if (wowpriv->wow_wake_reason == RTW_MAC_WOW_FW_DECISION_DISCONNECT ||
+		    wowpriv->wow_wake_reason == RTW_MAC_WOW_NO_WAKE_FW_DECISION_DISCONNECT)
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_FW_DUE_TO_FW_DECISION_IN_WOW_RESUME;
+		else if (wowpriv->wow_wake_reason == RTW_MAC_WOW_RX_DISASSOC ||
+			 wowpriv->wow_wake_reason == RTW_MAC_WOW_NO_WAKE_RX_DISASSOC)
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DISASSOC_IN_WOW_RESUME;
+		else if (wowpriv->wow_wake_reason == RTW_MAC_WOW_RX_DEAUTH ||
+			 wowpriv->wow_wake_reason == RTW_MAC_WOW_NO_WAKE_RX_DEAUTH)
+			pmlmeinfo->disconnect_code = DISCONNECTION_BY_AP_DUE_TO_RECEIVE_DEAUTH_IN_WOW_RESUME;
+		pmlmeinfo->wifi_reason_code = WLAN_REASON_UNSPECIFIED;
+	} else {
+		if (rtw_chk_roam_flags(padapter, RTW_ROAM_ON_RESUME)) {
+			RTW_INFO("%s: do roaming\n", __func__);
+			rtw_roaming(padapter, NULL, RTW_ROAM_ON_RESUME);
+		}
+	}
+
+	if (pwrpriv->wowlan_mode == _TRUE) {
+		rtw_phl_watchdog_start(dvobj->phl);
+#if 0 /*ndef CONFIG_IPS_CHECK_IN_WD*/
+		rtw_set_pwr_state_check_timer(pwrpriv);
+#endif
+	} else
+		RTW_PRINT("do not reset timer\n");
+
+	pwrpriv->wowlan_mode = _FALSE;
+
+	/* Power On LED */
+#ifdef CONFIG_RTW_SW_LED
+
+	if (IS_DISCONNECT_WOW_REASON(wowpriv->wow_wake_reason))
+		rtw_led_control(padapter, LED_CTL_NO_LINK);
+	else
+		rtw_led_control(padapter, LED_CTL_LINK);
+#endif
+
+exit:
+	RTW_INFO("<== "FUNC_ADPT_FMT" exit....\n", FUNC_ADPT_ARG(padapter));
+	return ret;
+}
+#endif /* #ifdef CONFIG_WOWLAN */
+
+#ifdef CONFIG_AP_WOWLAN
+int rtw_resume_process_ap_wow(_adapter *padapter)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct debug_priv *pdbgpriv = &dvobj->drv_dbg;
+	struct sta_info	*psta = NULL;
+	int ret = _SUCCESS;
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+
+	RTW_INFO("==> "FUNC_ADPT_FMT" entry....\n", FUNC_ADPT_ARG(padapter));
+
+	if (padapter) {
+		pwrpriv = adapter_to_pwrctl(padapter);
+	} else {
+		pdbgpriv->dbg_resume_error_cnt++;
+		ret = -1;
+		goto exit;
+	}
+
+	/* pwrpriv->bFwCurrentInPSMode = _FALSE; */
+#if 0 /*GEORGIA_TODO_REMOVE_IT_FOR_PHL_ARCH*/
+
+	rtw_hal_disable_interrupt(GET_PHL_COM(dvobj));
+
+	rtw_hal_clear_interrupt(padapter);
+#endif
+
+	#ifdef CONFIG_SDIO_HCI
+	#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+	if (rtw_sdio_alloc_irq(dvobj) != _SUCCESS) {
+		ret = -1;
+		goto exit;
+	}
+	#endif
+	#endif/*CONFIG_SDIO_HCI*/
+
+	pwrpriv->wowlan_ap_mode = _FALSE;
+
+	dev_clr_drv_stopped(dvobj);
+	RTW_INFO("%s: wowmode resuming, DriverStopped:%s\n", __func__, dev_is_drv_stopped(dvobj) ? "True" : "False");
+
+	#if 0
+	rtw_mi_start_drv_threads(padapter);
+	#endif
+
+	if (rtw_mi_check_status(padapter, MI_LINKED)) {
+		struct rtw_chan_def u_chdef = {0};
+
+		if (rtw_phl_mr_get_chandef(dvobj->phl, padapter->phl_role, padapter_link->wrlink, &u_chdef)
+								!= RTW_PHL_STATUS_SUCCESS) {
+			RTW_ERR("%s get union chandef failed\n", __func__);
+			rtw_warn_on(1);
+	}
+
+		RTW_INFO(FUNC_ADPT_FMT" back to linked/linking union - ch:%u, bw:%u, offset:%u\n",
+			FUNC_ADPT_ARG(padapter), u_chdef.chan, u_chdef.bw, u_chdef.offset);
+		set_bch_bwmode(padapter, padapter_link, u_chdef.band, u_chdef.chan, u_chdef.offset, u_chdef.bw, RFK_TYPE_FORCE_NOT_DO);
+	}
+
+	/*FOR ONE AP - TODO :Multi-AP*/
+	{
+		int i;
+		_adapter *iface;
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if ((iface) && rtw_is_adapter_up(iface)) {
+				if (check_fwstate(&iface->mlmepriv, WIFI_AP_STATE | WIFI_MESH_STATE | WIFI_ASOC_STATE))
+					rtw_reset_drv_sw(iface);
+			}
+		}
+
+	}
+
+	/* start netif queue */
+	rtw_mi_netif_wake_queue(padapter);
+
+	if (padapter->pid[1] != 0) {
+		RTW_INFO("pid[1]:%d\n", padapter->pid[1]);
+		rtw_signal_process(padapter->pid[1], SIGUSR2);
+	}
+
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+	/* rtw_unlock_suspend(); */
+#endif /* CONFIG_RESUME_IN_WORKQUEUE */
+
+	#if 0 /*#ifdef CONFIG_CORE_DM_CHK_TIMER*/
+	_set_timer(&dvobj->dynamic_chk_timer, 2000);
+	#endif
+#if 0 /*ndef CONFIG_IPS_CHECK_IN_WD*/
+	rtw_set_pwr_state_check_timer(pwrpriv);
+#endif
+
+	/* Power On LED */
+#ifdef CONFIG_RTW_SW_LED
+
+	rtw_led_control(padapter, LED_CTL_LINK);
+#endif
+exit:
+	RTW_INFO("<== "FUNC_ADPT_FMT" exit....\n", FUNC_ADPT_ARG(padapter));
+	return ret;
+}
+#endif /* #ifdef CONFIG_APWOWLAN */
+
+void rtw_mi_resume_process_normal(_adapter *padapter)
+{
+	int i;
+	_adapter *iface;
+	struct mlme_priv *pmlmepriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if ((iface) && rtw_is_adapter_up(iface)) {
+			pmlmepriv = &iface->mlmepriv;
+
+			if (MLME_IS_STA(iface)) {
+				RTW_INFO(FUNC_ADPT_FMT" fwstate:0x%08x - WIFI_STATION_STATE\n", FUNC_ADPT_ARG(iface), get_fwstate(pmlmepriv));
+
+				if (rtw_chk_roam_flags(iface, RTW_ROAM_ON_RESUME))
+					rtw_roaming(iface, NULL, RTW_ROAM_ON_RESUME);
+
+			} else if (MLME_IS_AP(iface) || MLME_IS_MESH(iface)) {
+				RTW_INFO(FUNC_ADPT_FMT" %s\n", FUNC_ADPT_ARG(iface), MLME_IS_AP(iface) ? "AP" : "MESH");
+				rtw_ap_restore_network(iface);
+			} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE))
+				RTW_INFO(FUNC_ADPT_FMT" fwstate:0x%08x - WIFI_ADHOC_STATE\n", FUNC_ADPT_ARG(iface), get_fwstate(pmlmepriv));
+			else
+				RTW_INFO(FUNC_ADPT_FMT" fwstate:0x%08x - ???\n", FUNC_ADPT_ARG(iface), get_fwstate(pmlmepriv));
+		}
+	}
+}
+
+int rtw_resume_process_normal(_adapter *padapter)
+{
+	struct net_device *pnetdev;
+	struct pwrctrl_priv *pwrpriv;
+	struct dvobj_priv *dvobj;
+	struct debug_priv *pdbgpriv;
+
+	int ret = _SUCCESS;
+
+	if (!padapter) {
+		ret = -1;
+		goto exit;
+	}
+
+	pnetdev = padapter->pnetdev;
+	pwrpriv = adapter_to_pwrctl(padapter);
+	dvobj = padapter->dvobj;
+	pdbgpriv = &dvobj->drv_dbg;
+
+	RTW_INFO("==> "FUNC_ADPT_FMT" entry....\n", FUNC_ADPT_ARG(padapter));
+
+	#ifdef CONFIG_SDIO_HCI
+	/* interface init */
+	if (rtw_sdio_init(dvobj) != _SUCCESS) {
+		ret = -1;
+		goto exit;
+	}
+	#endif/*CONFIG_SDIO_HCI*/
+
+	dev_clr_surprise_removed(dvobj);
+#if 0 /*GEORGIA_TODO_REMOVE_IT_FOR_PHL_ARCH*/
+	rtw_hal_disable_interrupt(GET_PHL_COM(dvobj));
+#endif
+	#ifdef CONFIG_SDIO_HCI
+	#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+	if (rtw_sdio_alloc_irq(dvobj) != _SUCCESS) {
+		ret = -1;
+		goto exit;
+	}
+	#endif
+	#endif/*CONFIG_SDIO_HCI*/
+
+	rtw_mi_reset_drv_sw(padapter);
+
+	pwrpriv->bkeepfwalive = _FALSE;
+
+	RTW_INFO("bkeepfwalive(%x)\n", pwrpriv->bkeepfwalive);
+	if (pm_netdev_open(pnetdev, _TRUE) != 0) {
+		ret = -1;
+		pdbgpriv->dbg_resume_error_cnt++;
+		goto exit;
+	}
+
+	rtw_mi_netif_caron_qstart(padapter);
+
+	if (padapter->pid[1] != 0) {
+		RTW_INFO("pid[1]:%d\n", padapter->pid[1]);
+		rtw_signal_process(padapter->pid[1], SIGUSR2);
+	}
+
+	rtw_mi_resume_process_normal(padapter);
+
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+	/* rtw_unlock_suspend(); */
+#endif /* CONFIG_RESUME_IN_WORKQUEUE */
+	RTW_INFO("<== "FUNC_ADPT_FMT" exit....\n", FUNC_ADPT_ARG(padapter));
+
+exit:
+	return ret;
+}
+
+int rtw_resume_common(_adapter *padapter)
+{
+	int ret = 0;
+	systime start_time = rtw_get_current_time();
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+
+	if (pwrpriv == NULL)
+		return 0;
+
+	if (pwrpriv->bInSuspend == _FALSE)
+		return 0;
+
+	RTW_PRINT("resume start\n");
+	RTW_INFO("==> %s (%s:%d)\n", __FUNCTION__, current->comm, current->pid);
+
+	if (rtw_mi_check_status(padapter, MI_AP_MODE) == _FALSE) {
+#ifdef CONFIG_WOWLAN
+		if (pwrpriv->wowlan_mode == _TRUE)
+			rtw_resume_process_wow(padapter);
+		else
+#endif
+			rtw_resume_process_normal(padapter);
+
+	} else if (rtw_mi_check_status(padapter, MI_AP_MODE)) {
+#ifdef CONFIG_AP_WOWLAN
+		rtw_resume_process_ap_wow(padapter);
+#else
+		rtw_resume_process_normal(padapter);
+#endif /* CONFIG_AP_WOWLAN */
+	}
+
+
+	pwrpriv->bInSuspend = _FALSE;
+	pwrpriv->wowlan_in_resume = _FALSE;
+
+	RTW_PRINT("%s:%d in %d ms\n", __FUNCTION__ , ret,
+		  rtw_get_passing_time_ms(start_time));
+
+
+	return ret;
+}
+
+#ifdef CONFIG_GPIO_API
+u8 rtw_get_gpio(struct net_device *netdev, u8 gpio_num)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(netdev);
+	return rtw_hal_get_gpio(adapter, gpio_num);
+}
+EXPORT_SYMBOL(rtw_get_gpio);
+
+int  rtw_set_gpio_output_value(struct net_device *netdev, u8 gpio_num, bool isHigh)
+{
+	u8 direction = 0;
+	u8 res = -1;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(netdev);
+	return rtw_hal_set_gpio_output_value(adapter, gpio_num, isHigh);
+}
+EXPORT_SYMBOL(rtw_set_gpio_output_value);
+
+int rtw_config_gpio(struct net_device *netdev, u8 gpio_num, bool isOutput)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(netdev);
+	return rtw_hal_config_gpio(adapter, gpio_num, isOutput);
+}
+EXPORT_SYMBOL(rtw_config_gpio);
+int rtw_register_gpio_interrupt(struct net_device *netdev, int gpio_num, void(*callback)(u8 level))
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(netdev);
+	return rtw_hal_register_gpio_interrupt(adapter, gpio_num, callback);
+}
+EXPORT_SYMBOL(rtw_register_gpio_interrupt);
+
+int rtw_disable_gpio_interrupt(struct net_device *netdev, int gpio_num)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(netdev);
+	return rtw_hal_disable_gpio_interrupt(adapter, gpio_num);
+}
+EXPORT_SYMBOL(rtw_disable_gpio_interrupt);
+
+#endif /* #ifdef CONFIG_GPIO_API */
+
+#ifdef CONFIG_APPEND_VENDOR_IE_ENABLE
+
+int rtw_vendor_ie_get_api(struct net_device *dev, int ie_num, char *extra,
+		u16 extra_len)
+{
+	int ret = 0;
+
+	ret = rtw_vendor_ie_get_raw_data(dev, ie_num, extra, extra_len);
+	return ret;
+}
+EXPORT_SYMBOL(rtw_vendor_ie_get_api);
+
+int rtw_vendor_ie_set_api(struct net_device *dev, char *extra)
+{
+	return rtw_vendor_ie_set(dev, NULL, NULL, extra);
+}
+EXPORT_SYMBOL(rtw_vendor_ie_set_api);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/recv_linux.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/recv_linux.c
new file mode 100644
index 000000000000..5a7f8a384b4d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/recv_linux.c
@@ -0,0 +1,643 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RECV_OSDEP_C_
+
+#include <drv_types.h>
+
+int rtw_os_recvframe_duplicate_skb(_adapter *padapter,
+			union recv_frame *pcloneframe, struct sk_buff *pskb)
+{
+	int res = _SUCCESS;
+	struct sk_buff *pkt_copy = NULL;
+
+	if (pskb == NULL) {
+		RTW_INFO("%s [WARN] skb == NULL, drop frag frame\n", __func__);
+		return _FAIL;
+	}
+#if 1
+	pkt_copy = rtw_skb_copy(pskb);
+
+	if (pkt_copy == NULL) {
+		RTW_INFO("%s [WARN] rtw_skb_copy fail , drop frag frame\n", __func__);
+		return _FAIL;
+	}
+#else
+	pkt_copy = rtw_skb_clone(pskb);
+
+	if (pkt_copy == NULL) {
+		RTW_INFO("%s [WARN] rtw_skb_clone fail , drop frag frame\n", __func__);
+		return _FAIL;
+	}
+#endif
+	pkt_copy->dev = padapter->pnetdev;
+
+	pcloneframe->u.hdr.pkt = pkt_copy;
+	pcloneframe->u.hdr.rx_head = pkt_copy->head;
+	pcloneframe->u.hdr.rx_data = pkt_copy->data;
+	pcloneframe->u.hdr.rx_end = skb_end_pointer(pkt_copy);
+	pcloneframe->u.hdr.rx_tail = skb_tail_pointer(pkt_copy);
+	pcloneframe->u.hdr.len = pkt_copy->len;
+
+	return res;
+}
+
+int rtw_os_alloc_recvframe(_adapter *padapter,
+		union recv_frame *precvframe, u8 *pdata, struct sk_buff *pskb)
+{
+	int res = _SUCCESS;
+	u8	shift_sz = 0;
+	u32	skb_len, alloc_sz;
+	struct sk_buff	*pkt_copy = NULL;
+	struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;
+
+
+	if (pdata == NULL) {
+		precvframe->u.hdr.pkt = NULL;
+		res = _FAIL;
+		return res;
+	}
+
+
+	/*	Modified by Albert 20101213 */
+	/*	For 8 bytes IP header alignment. */
+	shift_sz = pattrib->qos ? 6 : 0; /*	Qos data, wireless lan header length is 26 */
+
+	skb_len = pattrib->pkt_len;
+
+	/* for first fragment packet, driver need allocate 1536+drvinfo_sz+RXDESC_SIZE to defrag packet. */
+	/* modify alloc_sz for recvive crc error packet by thomas 2011-06-02 */
+	if ((pattrib->mfrag == 1) && (pattrib->frag_num == 0)) {
+		/* alloc_sz = 1664;	 */ /* 1664 is 128 alignment. */
+		alloc_sz = (skb_len <= 1650) ? 1664 : (skb_len + 14);
+	} else {
+		alloc_sz = skb_len;
+		/*	6 is for IP header 8 bytes alignment in QoS packet case. */
+		/*	8 is for skb->data 4 bytes alignment. */
+		alloc_sz += 14;
+	}
+
+	pkt_copy = rtw_skb_alloc(alloc_sz);
+
+	if (pkt_copy) {
+		pkt_copy->dev = padapter->pnetdev;
+		pkt_copy->len = skb_len;
+		precvframe->u.hdr.pkt = pkt_copy;
+		precvframe->u.hdr.rx_head = pkt_copy->head;
+		precvframe->u.hdr.rx_end = pkt_copy->data + alloc_sz;
+		skb_reserve(pkt_copy, 8 - ((SIZE_PTR)(pkt_copy->data) & 7));  /* force pkt_copy->data at 8-byte alignment address */
+		skb_reserve(pkt_copy, shift_sz);/* force ip_hdr at 8-byte alignment address according to shift_sz. */
+		_rtw_memcpy(pkt_copy->data, pdata, skb_len);
+		precvframe->u.hdr.rx_data = precvframe->u.hdr.rx_tail = pkt_copy->data;
+	} else {
+
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_RX
+		RTW_INFO("%s:can not allocate memory for skb copy\n", __func__);
+
+		precvframe->u.hdr.pkt = NULL;
+
+		/* rtw_free_recvframe(precvframe); */
+		/*exit_rtw_os_recv_resource_alloc;*/
+
+		res = _FAIL;
+#else
+		if ((pattrib->mfrag == 1) && (pattrib->frag_num == 0)) {
+			RTW_INFO("%s: alloc_skb fail , drop frag frame\n", __FUNCTION__);
+			/* rtw_free_recvframe(precvframe); */
+			res = _FAIL;
+			goto exit_rtw_os_recv_resource_alloc;
+		}
+
+		if (pskb == NULL) {
+			res = _FAIL;
+			goto exit_rtw_os_recv_resource_alloc;
+		}
+
+		precvframe->u.hdr.pkt = rtw_skb_clone(pskb);
+		if (precvframe->u.hdr.pkt) {
+			precvframe->u.hdr.pkt->dev = padapter->pnetdev;
+			precvframe->u.hdr.rx_head = precvframe->u.hdr.rx_data = precvframe->u.hdr.rx_tail = pdata;
+			precvframe->u.hdr.rx_end =  pdata + alloc_sz;
+		} else {
+			RTW_INFO("%s: rtw_skb_clone fail\n", __FUNCTION__);
+			/* rtw_free_recvframe(precvframe); */
+			/*exit_rtw_os_recv_resource_alloc;*/
+			res = _FAIL;
+		}
+#endif
+	}
+
+exit_rtw_os_recv_resource_alloc:
+
+	return res;
+
+}
+
+void rtw_os_free_recvframe(union recv_frame *precvframe)
+{
+	if (precvframe->u.hdr.pkt) {
+		rtw_skb_free(precvframe->u.hdr.pkt);
+		precvframe->u.hdr.pkt = NULL;
+	}
+}
+
+/* init os related resource in struct recv_priv */
+int rtw_os_recv_resource_init(struct recv_priv *precvpriv)
+{
+	int res = _SUCCESS;
+
+#ifdef CONFIG_RTW_NAPI
+	skb_queue_head_init(&precvpriv->rx_napi_skb_queue);
+#endif /* CONFIG_RTW_NAPI */
+
+	return res;
+}
+
+/* alloc os related resource in union recv_frame */
+int rtw_os_recv_resource_alloc(union recv_frame *precvframe)
+{
+	int	res = _SUCCESS;
+
+	precvframe->u.hdr.pkt = NULL;
+
+	return res;
+}
+
+/* free os related resource in union recv_frame */
+void rtw_os_recv_resource_free(struct recv_priv *precvpriv)
+{
+	sint i;
+	union recv_frame *precvframe;
+	precvframe = (union recv_frame *) precvpriv->precv_frame_buf;
+
+
+#ifdef CONFIG_RTW_NAPI
+	if (skb_queue_len(&precvpriv->rx_napi_skb_queue))
+		RTW_WARN("rx_napi_skb_queue not empty\n");
+	rtw_skb_queue_purge(&precvpriv->rx_napi_skb_queue);
+#endif /* CONFIG_RTW_NAPI */
+
+	for (i = 0; i < NR_RECVFRAME; i++) {
+		rtw_os_free_recvframe(precvframe);
+		precvframe++;
+	}
+}
+
+#if 0
+/* alloc os related resource in struct recv_buf */
+int rtw_os_recvbuf_resource_alloc(_adapter *padapter, struct recv_buf *precvbuf)
+{
+	int res = _SUCCESS;
+
+#ifdef CONFIG_USB_HCI
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_RX
+	struct dvobj_priv	*pdvobjpriv = adapter_to_dvobj(padapter);
+	struct usb_device	*pusbd = dvobj_to_usb(pdvobjpriv)->pusbdev;
+#endif
+
+	precvbuf->irp_pending = _FALSE;
+	precvbuf->purb = usb_alloc_urb(0, GFP_KERNEL);
+	if (precvbuf->purb == NULL)
+		res = _FAIL;
+
+	precvbuf->pskb = NULL;
+
+	precvbuf->pallocated_buf  = precvbuf->pbuf = NULL;
+
+	precvbuf->pdata = precvbuf->phead = precvbuf->ptail = precvbuf->pend = NULL;
+
+	precvbuf->transfer_len = 0;
+
+	precvbuf->len = 0;
+
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_RX
+	precvbuf->pallocated_buf = rtw_usb_buffer_alloc(pusbd, (size_t)precvbuf->alloc_sz, &precvbuf->dma_transfer_addr);
+	precvbuf->pbuf = precvbuf->pallocated_buf;
+	if (precvbuf->pallocated_buf == NULL)
+		return _FAIL;
+#endif /* CONFIG_USE_USB_BUFFER_ALLOC_RX */
+
+#endif /* CONFIG_USB_HCI */
+
+	return res;
+}
+
+/* free os related resource in struct recv_buf */
+int rtw_os_recvbuf_resource_free(_adapter *adapter, struct recv_buf *precvbuf)
+{
+	int ret = _SUCCESS;
+
+#ifdef CONFIG_USB_HCI
+
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_RX
+
+	struct dvobj_priv	*dvobj = adapter_to_dvobj(adapter);
+	struct usb_device	*pusbd = dvobj_to_usb(dvobj)->pusbdev;
+
+	rtw_usb_buffer_free(pusbd, (size_t)precvbuf->alloc_sz, precvbuf->pallocated_buf, precvbuf->dma_transfer_addr);
+	precvbuf->pallocated_buf =  NULL;
+	precvbuf->dma_transfer_addr = 0;
+
+#endif /* CONFIG_USE_USB_BUFFER_ALLOC_RX */
+
+	if (precvbuf->purb) {
+		/* usb_kill_urb(precvbuf->purb); */
+		usb_free_urb(precvbuf->purb);
+	}
+
+#endif /* CONFIG_USB_HCI */
+
+
+	if (precvbuf->pskb) {
+#ifdef CONFIG_PREALLOC_RX_SKB_BUFFER
+		if (rtw_free_skb_premem(precvbuf->pskb) != 0)
+#endif
+			rtw_skb_free(precvbuf->pskb);
+	}
+	return ret;
+
+}
+#endif
+
+struct sk_buff *rtw_os_alloc_msdu_pkt(union recv_frame *prframe,
+		const u8 *da, const u8 *sa, u8 *msdu ,u16 msdu_len,
+		enum rtw_rx_llc_hdl llc_hdl)
+{
+	u8	*data_ptr;
+	struct sk_buff *sub_skb;
+	struct rx_pkt_attrib *pattrib;
+
+	pattrib = &prframe->u.hdr.attrib;
+
+#ifdef CONFIG_SKB_COPY
+	sub_skb = rtw_skb_alloc(msdu_len + 14);
+	if (sub_skb) {
+		skb_reserve(sub_skb, 14);
+		data_ptr = (u8 *)skb_put(sub_skb, msdu_len);
+		_rtw_memcpy(data_ptr, msdu, msdu_len);
+	} else
+#endif /* CONFIG_SKB_COPY */
+	{
+		sub_skb = rtw_skb_clone(prframe->u.hdr.pkt);
+		if (sub_skb) {
+			sub_skb->data = msdu;
+			sub_skb->len = msdu_len;
+			skb_set_tail_pointer(sub_skb, msdu_len);
+		} else {
+			RTW_INFO("%s(): rtw_skb_clone() Fail!!!\n", __FUNCTION__);
+			return NULL;
+		}
+	}
+
+	if (llc_hdl) {
+		/* remove RFC1042 or Bridge-Tunnel encapsulation and replace EtherType */
+		skb_pull(sub_skb, SNAP_SIZE);
+		_rtw_memcpy(skb_push(sub_skb, ETH_ALEN), sa, ETH_ALEN);
+		_rtw_memcpy(skb_push(sub_skb, ETH_ALEN), da, ETH_ALEN);
+	} else {
+		/* Leave Ethernet header part of hdr and full payload */
+		u16 len;
+
+		len = htons(sub_skb->len);
+		_rtw_memcpy(skb_push(sub_skb, 2), &len, 2);
+		_rtw_memcpy(skb_push(sub_skb, ETH_ALEN), sa, ETH_ALEN);
+		_rtw_memcpy(skb_push(sub_skb, ETH_ALEN), da, ETH_ALEN);
+	}
+
+	return sub_skb;
+}
+
+#ifdef CONFIG_RTW_NAPI
+static int napi_recv(_adapter *padapter, int budget)
+{
+	struct sk_buff *pskb;
+	struct recv_priv *precvpriv = &adapter_to_dvobj(padapter)->recvpriv;
+	int work_done = 0;
+	struct registry_priv *pregistrypriv = &padapter->registrypriv;
+	u8 rx_ok;
+
+
+	while ((work_done < budget) &&
+	       (!skb_queue_empty(&precvpriv->rx_napi_skb_queue))) {
+		pskb = skb_dequeue(&precvpriv->rx_napi_skb_queue);
+		if (!pskb)
+			break;
+
+		rx_ok = _FALSE;
+
+#ifdef CONFIG_RTW_GRO
+		if (pregistrypriv->en_gro && !skb_cloned(pskb)) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0))
+			rtw_napi_gro_receive(&padapter->napi, pskb);
+#else
+			if (rtw_napi_gro_receive(&padapter->napi, pskb) != GRO_DROP)
+#endif
+				rx_ok = _TRUE;
+			goto next;
+		}
+#endif /* CONFIG_RTW_GRO */
+
+		if (rtw_netif_receive_skb(padapter->pnetdev, pskb) == NET_RX_SUCCESS)
+			rx_ok = _TRUE;
+
+next:
+		if (rx_ok == _TRUE) {
+			work_done++;
+			DBG_COUNTER(padapter->rx_logs.os_netif_ok);
+		} else {
+			DBG_COUNTER(padapter->rx_logs.os_netif_err);
+		}
+	}
+
+	return work_done;
+}
+
+int rtw_recv_napi_poll(struct napi_struct *napi, int budget)
+{
+	_adapter *padapter = container_of(napi, _adapter, napi);
+	int work_done = 0;
+	struct recv_priv *precvpriv = &adapter_to_dvobj(padapter)->recvpriv;
+
+
+	work_done = napi_recv(padapter, budget);
+	if (work_done < budget) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_PCI_HCI)
+		napi_complete_done(napi, work_done);
+#else
+		napi_complete(napi);
+#endif
+		if (!skb_queue_empty(&precvpriv->rx_napi_skb_queue))
+			napi_schedule(napi);
+	}
+
+	return work_done;
+}
+
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+void dynamic_napi_th_chk (_adapter *adapter)
+{
+
+	if (adapter->registrypriv.en_napi) {
+		struct dvobj_priv *dvobj;
+		struct registry_priv *registry;
+	
+		dvobj = adapter_to_dvobj(adapter);
+		registry = &adapter->registrypriv;
+		if (dvobj->traffic_stat.cur_rx_tp > registry->napi_threshold)
+			dvobj->en_napi_dynamic = 1;
+		else
+			dvobj->en_napi_dynamic = 0;
+	}
+
+}
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+#endif /* CONFIG_RTW_NAPI */
+
+void rtw_os_recv_indicate_pkt(_adapter *padapter, struct sk_buff *pkt,
+						union recv_frame *rframe)
+{
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct recv_priv *precvpriv = &adapter_to_dvobj(padapter)->recvpriv;
+	struct registry_priv	*pregistrypriv = &padapter->registrypriv;
+#ifdef CONFIG_BR_EXT
+	void *br_port = NULL;
+#endif
+	int ret;
+
+	/* Indicat the packets to upper layer */
+	if (pkt) {
+		struct ethhdr *ehdr = (struct ethhdr *)pkt->data;
+
+		DBG_COUNTER(padapter->rx_logs.os_indicate);
+#ifdef CONFIG_BR_EXT
+		if (!adapter_use_wds(padapter) &&
+		    (MLME_IS_STA(padapter) || MLME_IS_ADHOC(padapter))) {
+			/* Insert NAT2.5 RX here! */
+			#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 35))
+			br_port = padapter->pnetdev->br_port;
+			#else
+			rcu_read_lock();
+			br_port = rcu_dereference(padapter->pnetdev->rx_handler_data);
+			rcu_read_unlock();
+			#endif
+
+			if (br_port) {
+				int nat25_handle_frame(_adapter *priv, struct sk_buff *skb);
+
+				if (nat25_handle_frame(padapter, pkt) == -1) {
+					/* priv->ext_stats.rx_data_drops++; */
+					/* DEBUG_ERR("RX DROP: nat25_handle_frame fail!\n"); */
+					/* return FAIL; */
+
+					#if 1
+					/* bypass this frame to upper layer!! */
+					#else
+					rtw_skb_free(sub_skb);
+					continue;
+					#endif
+				}
+			}
+		}
+#endif /* CONFIG_BR_EXT */
+
+		/* After eth_type_trans process , pkt->data pointer will move from ethrnet header to ip header */
+		pkt->protocol = eth_type_trans(pkt, padapter->pnetdev);
+		pkt->dev = padapter->pnetdev;
+		pkt->ip_summed = CHECKSUM_NONE; /* CONFIG_TCP_CSUM_OFFLOAD_RX */
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_RX
+		if ((rframe->u.hdr.attrib.csum_valid == 1)
+		    && (rframe->u.hdr.attrib.csum_err == 0))
+			pkt->ip_summed = CHECKSUM_UNNECESSARY;
+#endif /* CONFIG_TCP_CSUM_OFFLOAD_RX */
+
+#ifdef CONFIG_RTW_NAPI
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+		if (!skb_queue_empty(&precvpriv->rx_napi_skb_queue)
+			&& !adapter_to_dvobj(padapter)->en_napi_dynamic			
+			)
+			napi_recv(padapter, RTL_NAPI_WEIGHT);
+#endif
+
+		if (pregistrypriv->en_napi
+			#ifdef CONFIG_RTW_NAPI_DYNAMIC
+			&& adapter_to_dvobj(padapter)->en_napi_dynamic
+			#endif
+		) {
+			skb_queue_tail(&precvpriv->rx_napi_skb_queue, pkt);
+			#ifndef CONFIG_RTW_NAPI_V2
+			napi_schedule(&padapter->napi);
+			#endif
+			return;
+		}
+#endif /* CONFIG_RTW_NAPI */
+
+		ret = rtw_netif_rx(padapter->pnetdev, pkt);
+		if (ret == NET_RX_SUCCESS)
+			DBG_COUNTER(padapter->rx_logs.os_netif_ok);
+		else
+			DBG_COUNTER(padapter->rx_logs.os_netif_err);
+	}
+}
+
+void rtw_handle_tkip_mic_err(_adapter *padapter, struct sta_info *sta, u8 bgroup)
+{
+#ifdef CONFIG_IOCTL_CFG80211
+	enum nl80211_key_type key_type = 0;
+#endif
+	union iwreq_data wrqu;
+	struct iw_michaelmicfailure    ev;
+	struct security_priv	*psecuritypriv = &padapter->securitypriv;
+	systime cur_time = 0;
+
+	if (psecuritypriv->last_mic_err_time == 0)
+		psecuritypriv->last_mic_err_time = rtw_get_current_time();
+	else {
+		cur_time = rtw_get_current_time();
+
+		if (cur_time - psecuritypriv->last_mic_err_time < 60 * HZ) {
+			psecuritypriv->btkip_countermeasure = _TRUE;
+			psecuritypriv->last_mic_err_time = 0;
+			psecuritypriv->btkip_countermeasure_time = cur_time;
+		} else
+			psecuritypriv->last_mic_err_time = rtw_get_current_time();
+	}
+
+#ifdef CONFIG_IOCTL_CFG80211
+	if (bgroup)
+		key_type |= NL80211_KEYTYPE_GROUP;
+	else
+		key_type |= NL80211_KEYTYPE_PAIRWISE;
+
+	cfg80211_michael_mic_failure(padapter->pnetdev, sta->phl_sta->mac_addr, key_type, -1, NULL, GFP_ATOMIC);
+#endif
+
+	_rtw_memset(&ev, 0x00, sizeof(ev));
+	if (bgroup)
+		ev.flags |= IW_MICFAILURE_GROUP;
+	else
+		ev.flags |= IW_MICFAILURE_PAIRWISE;
+
+	ev.src_addr.sa_family = ARPHRD_ETHER;
+	_rtw_memcpy(ev.src_addr.sa_data, sta->phl_sta->mac_addr, ETH_ALEN);
+
+	_rtw_memset(&wrqu, 0x00, sizeof(wrqu));
+	wrqu.data.length = sizeof(ev);
+
+#ifndef CONFIG_IOCTL_CFG80211
+	wireless_send_event(padapter->pnetdev, IWEVMICHAELMICFAILURE, &wrqu, (char *) &ev);
+#endif
+}
+
+#ifdef CONFIG_HOSTAPD_MLME
+void rtw_hostapd_mlme_rx(_adapter *padapter, union recv_frame *precv_frame)
+{
+	struct sk_buff *skb;
+	struct hostapd_priv *phostapdpriv  = padapter->phostapdpriv;
+	struct net_device *pmgnt_netdev = phostapdpriv->pmgnt_netdev;
+
+
+	skb = precv_frame->u.hdr.pkt;
+
+	if (skb == NULL)
+		return;
+
+	skb->data = precv_frame->u.hdr.rx_data;
+	skb->tail = precv_frame->u.hdr.rx_tail;
+	skb->len = precv_frame->u.hdr.len;
+
+	/* pskb_copy = rtw_skb_copy(skb);
+	*	if(skb == NULL) goto _exit; */
+
+	skb->dev = pmgnt_netdev;
+	skb->ip_summed = CHECKSUM_NONE;
+	skb->pkt_type = PACKET_OTHERHOST;
+	/* skb->protocol = __constant_htons(0x0019); ETH_P_80211_RAW */
+	skb->protocol = __constant_htons(0x0003); /*ETH_P_80211_RAW*/
+
+	/* RTW_INFO("(1)data=0x%x, head=0x%x, tail=0x%x, mac_header=0x%x, len=%d\n", skb->data, skb->head, skb->tail, skb->mac_header, skb->len); */
+
+	/* skb->mac.raw = skb->data; */
+	skb_reset_mac_header(skb);
+
+	/* skb_pull(skb, 24); */
+	_rtw_memset(skb->cb, 0, sizeof(skb->cb));
+
+	rtw_netif_rx(pmgnt_netdev, skb);
+
+	precv_frame->u.hdr.pkt = NULL; /* set pointer to NULL before rtw_free_recvframe() if call rtw_netif_rx() */
+}
+#endif /* CONFIG_HOSTAPD_MLME */
+
+#ifdef CONFIG_WIFI_MONITOR
+/* 
+   precv_frame: impossible to be NULL
+   precv_frame: free by caller
+ */
+int rtw_recv_monitor(_adapter *padapter, union recv_frame *precv_frame)
+{
+	int ret = _FAIL;
+	struct sk_buff *skb;
+
+	skb = precv_frame->u.hdr.pkt;
+	if (skb == NULL) {
+		RTW_INFO("%s :skb==NULL something wrong!!!!\n", __func__);
+		goto _recv_drop;
+	}
+
+	skb->data = precv_frame->u.hdr.rx_data;
+	skb_set_tail_pointer(skb, precv_frame->u.hdr.len);
+	skb->len = precv_frame->u.hdr.len;
+	skb->ip_summed = CHECKSUM_NONE;
+	skb->pkt_type = PACKET_OTHERHOST;
+	skb->protocol = htons(0x0019); /* ETH_P_80211_RAW */
+
+	/* send to kernel */
+	rtw_netif_rx(padapter->pnetdev, skb);
+
+	/* pointers to NULL before rtw_free_recvframe() */
+	precv_frame->u.hdr.pkt = NULL;
+
+	ret = _SUCCESS;
+
+_recv_drop:
+	return ret;
+}
+#endif /* CONFIG_WIFI_MONITOR */
+
+inline void rtw_rframe_set_os_pkt(union recv_frame *rframe)
+{
+	struct sk_buff *skb = rframe->u.hdr.pkt;
+
+	skb->data = rframe->u.hdr.rx_data;
+	skb_set_tail_pointer(skb, rframe->u.hdr.len);
+	skb->len = rframe->u.hdr.len;
+}
+
+int rtw_recv_indicatepkt(_adapter *padapter, union recv_frame *precv_frame)
+{
+	if (precv_frame->u.hdr.pkt == NULL)
+		goto _recv_indicatepkt_drop;
+
+	rtw_os_recv_indicate_pkt(padapter, precv_frame->u.hdr.pkt, precv_frame);
+
+	precv_frame->u.hdr.pkt = NULL;
+	rtw_free_recvframe(precv_frame);
+	return _SUCCESS;
+
+_recv_indicatepkt_drop:
+	rtw_free_recvframe(precv_frame);
+	DBG_COUNTER(padapter->rx_logs.os_indicate_err);
+	return _FAIL;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rhashtable.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rhashtable.c
new file mode 100644
index 000000000000..dddbf3af04d0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rhashtable.c
@@ -0,0 +1,844 @@
+/*
+ * Resizable, Scalable, Concurrent Hash Table
+ *
+ * Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au>
+ * Copyright (c) 2014-2015 Thomas Graf <tgraf@suug.ch>
+ * Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
+ *
+ * Code partially derived from nft_hash
+ * Rewritten with rehash code from br_multicast plus single list
+ * pointer as suggested by Josh Triplett
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/atomic.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/log2.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/jhash.h>
+#include <linux/random.h>
+#include <linux/err.h>
+#include <linux/export.h>
+
+#define HASH_DEFAULT_SIZE	64UL
+#define HASH_MIN_SIZE		4U
+#define BUCKET_LOCKS_PER_CPU   128UL
+
+static u32 head_hashfn(struct rhashtable *ht,
+		       const struct bucket_table *tbl,
+		       const struct rhash_head *he)
+{
+	return rht_head_hashfn(ht, tbl, he, ht->p);
+}
+
+#ifdef CONFIG_PROVE_LOCKING
+#define ASSERT_RHT_MUTEX(HT) BUG_ON(!lockdep_rht_mutex_is_held(HT))
+
+int lockdep_rht_mutex_is_held(struct rhashtable *ht)
+{
+	return (debug_locks) ? lockdep_is_held(&ht->mutex) : 1;
+}
+
+int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash)
+{
+	spinlock_t *lock = rht_bucket_lock(tbl, hash);
+
+	return (debug_locks) ? lockdep_is_held(lock) : 1;
+}
+#else
+#define ASSERT_RHT_MUTEX(HT)
+#endif
+
+
+static int alloc_bucket_locks(struct rhashtable *ht, struct bucket_table *tbl,
+			      gfp_t gfp)
+{
+	unsigned int i, size;
+#if defined(CONFIG_PROVE_LOCKING)
+	unsigned int nr_pcpus = 2;
+#else
+	unsigned int nr_pcpus = num_possible_cpus();
+#endif
+
+	nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
+	size = roundup_pow_of_two(nr_pcpus * ht->p.locks_mul);
+
+	/* Never allocate more than 0.5 locks per bucket */
+	size = min_t(unsigned int, size, tbl->size >> 1);
+
+	if (sizeof(spinlock_t) != 0) {
+#ifdef CONFIG_NUMA
+		if (size * sizeof(spinlock_t) > PAGE_SIZE &&
+		    gfp == GFP_KERNEL)
+			tbl->locks = vmalloc(size * sizeof(spinlock_t));
+		else
+#endif
+		tbl->locks = kmalloc_array(size, sizeof(spinlock_t),
+					   gfp);
+		if (!tbl->locks)
+			return -ENOMEM;
+		for (i = 0; i < size; i++)
+			spin_lock_init(&tbl->locks[i]);
+	}
+	tbl->locks_mask = size - 1;
+
+	return 0;
+}
+
+static void bucket_table_free(const struct bucket_table *tbl)
+{
+	if (tbl)
+		kvfree(tbl->locks);
+
+	kvfree(tbl);
+}
+
+static void bucket_table_free_rcu(struct rcu_head *head)
+{
+	bucket_table_free(container_of(head, struct bucket_table, rcu));
+}
+
+static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
+					       size_t nbuckets,
+					       gfp_t gfp)
+{
+	struct bucket_table *tbl = NULL;
+	size_t size;
+	int i;
+
+	size = sizeof(*tbl) + nbuckets * sizeof(tbl->buckets[0]);
+	if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER) ||
+	    gfp != GFP_KERNEL)
+		tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY);
+	if (tbl == NULL && gfp == GFP_KERNEL)
+		tbl = vzalloc(size);
+	if (tbl == NULL)
+		return NULL;
+
+	tbl->size = nbuckets;
+
+	if (alloc_bucket_locks(ht, tbl, gfp) < 0) {
+		bucket_table_free(tbl);
+		return NULL;
+	}
+
+	INIT_LIST_HEAD(&tbl->walkers);
+
+	get_random_bytes(&tbl->hash_rnd, sizeof(tbl->hash_rnd));
+
+	for (i = 0; i < nbuckets; i++)
+		INIT_RHT_NULLS_HEAD(tbl->buckets[i], ht, i);
+
+	return tbl;
+}
+
+static struct bucket_table *rhashtable_last_table(struct rhashtable *ht,
+						  struct bucket_table *tbl)
+{
+	struct bucket_table *new_tbl;
+
+	do {
+		new_tbl = tbl;
+		tbl = rht_dereference_rcu(tbl->future_tbl, ht);
+	} while (tbl);
+
+	return new_tbl;
+}
+
+static int rhashtable_rehash_one(struct rhashtable *ht, unsigned int old_hash)
+{
+	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
+	struct bucket_table *new_tbl = rhashtable_last_table(ht,
+		rht_dereference_rcu(old_tbl->future_tbl, ht));
+	struct rhash_head __rcu **pprev = &old_tbl->buckets[old_hash];
+	int err = -ENOENT;
+	struct rhash_head *head, *next, *entry;
+	spinlock_t *new_bucket_lock;
+	unsigned int new_hash;
+
+	rht_for_each(entry, old_tbl, old_hash) {
+		err = 0;
+		next = rht_dereference_bucket(entry->next, old_tbl, old_hash);
+
+		if (rht_is_a_nulls(next))
+			break;
+
+		pprev = &entry->next;
+	}
+
+	if (err)
+		goto out;
+
+	new_hash = head_hashfn(ht, new_tbl, entry);
+
+	new_bucket_lock = rht_bucket_lock(new_tbl, new_hash);
+
+	spin_lock_nested(new_bucket_lock, SINGLE_DEPTH_NESTING);
+	head = rht_dereference_bucket(new_tbl->buckets[new_hash],
+				      new_tbl, new_hash);
+
+	RCU_INIT_POINTER(entry->next, head);
+
+	rcu_assign_pointer(new_tbl->buckets[new_hash], entry);
+	spin_unlock(new_bucket_lock);
+
+	rcu_assign_pointer(*pprev, next);
+
+out:
+	return err;
+}
+
+static void rhashtable_rehash_chain(struct rhashtable *ht,
+				    unsigned int old_hash)
+{
+	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
+	spinlock_t *old_bucket_lock;
+
+	old_bucket_lock = rht_bucket_lock(old_tbl, old_hash);
+
+	spin_lock_bh(old_bucket_lock);
+	while (!rhashtable_rehash_one(ht, old_hash))
+		;
+	old_tbl->rehash++;
+	spin_unlock_bh(old_bucket_lock);
+}
+
+static int rhashtable_rehash_attach(struct rhashtable *ht,
+				    struct bucket_table *old_tbl,
+				    struct bucket_table *new_tbl)
+{
+	/* Protect future_tbl using the first bucket lock. */
+	spin_lock_bh(old_tbl->locks);
+
+	/* Did somebody beat us to it? */
+	if (rcu_access_pointer(old_tbl->future_tbl)) {
+		spin_unlock_bh(old_tbl->locks);
+		return -EEXIST;
+	}
+
+	/* Make insertions go into the new, empty table right away. Deletions
+	 * and lookups will be attempted in both tables until we synchronize.
+	 */
+	rcu_assign_pointer(old_tbl->future_tbl, new_tbl);
+
+	/* Ensure the new table is visible to readers. */
+	smp_wmb();
+
+	spin_unlock_bh(old_tbl->locks);
+
+	return 0;
+}
+
+static int rhashtable_rehash_table(struct rhashtable *ht)
+{
+	struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
+	struct bucket_table *new_tbl;
+	struct rhashtable_walker *walker;
+	unsigned int old_hash;
+
+	new_tbl = rht_dereference(old_tbl->future_tbl, ht);
+	if (!new_tbl)
+		return 0;
+
+	for (old_hash = 0; old_hash < old_tbl->size; old_hash++)
+		rhashtable_rehash_chain(ht, old_hash);
+
+	/* Publish the new table pointer. */
+	rcu_assign_pointer(ht->tbl, new_tbl);
+
+	spin_lock(&ht->lock);
+	list_for_each_entry(walker, &old_tbl->walkers, list)
+		walker->tbl = NULL;
+	spin_unlock(&ht->lock);
+
+	/* Wait for readers. All new readers will see the new
+	 * table, and thus no references to the old table will
+	 * remain.
+	 */
+	call_rcu(&old_tbl->rcu, bucket_table_free_rcu);
+
+	return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0;
+}
+
+/**
+ * rhashtable_expand - Expand hash table while allowing concurrent lookups
+ * @ht:		the hash table to expand
+ *
+ * A secondary bucket array is allocated and the hash entries are migrated.
+ *
+ * This function may only be called in a context where it is safe to call
+ * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
+ *
+ * The caller must ensure that no concurrent resizing occurs by holding
+ * ht->mutex.
+ *
+ * It is valid to have concurrent insertions and deletions protected by per
+ * bucket locks or concurrent RCU protected lookups and traversals.
+ */
+static int rhashtable_expand(struct rhashtable *ht)
+{
+	struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);
+	int err;
+
+	ASSERT_RHT_MUTEX(ht);
+
+	old_tbl = rhashtable_last_table(ht, old_tbl);
+
+	new_tbl = bucket_table_alloc(ht, old_tbl->size * 2, GFP_KERNEL);
+	if (new_tbl == NULL)
+		return -ENOMEM;
+
+	err = rhashtable_rehash_attach(ht, old_tbl, new_tbl);
+	if (err)
+		bucket_table_free(new_tbl);
+
+	return err;
+}
+
+/**
+ * rhashtable_shrink - Shrink hash table while allowing concurrent lookups
+ * @ht:		the hash table to shrink
+ *
+ * This function shrinks the hash table to fit, i.e., the smallest
+ * size would not cause it to expand right away automatically.
+ *
+ * The caller must ensure that no concurrent resizing occurs by holding
+ * ht->mutex.
+ *
+ * The caller must ensure that no concurrent table mutations take place.
+ * It is however valid to have concurrent lookups if they are RCU protected.
+ *
+ * It is valid to have concurrent insertions and deletions protected by per
+ * bucket locks or concurrent RCU protected lookups and traversals.
+ */
+static int rhashtable_shrink(struct rhashtable *ht)
+{
+	struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);
+	unsigned int size;
+	int err;
+
+	ASSERT_RHT_MUTEX(ht);
+
+	size = roundup_pow_of_two(atomic_read(&ht->nelems) * 3 / 2);
+	if (size < ht->p.min_size)
+		size = ht->p.min_size;
+
+	if (old_tbl->size <= size)
+		return 0;
+
+	if (rht_dereference(old_tbl->future_tbl, ht))
+		return -EEXIST;
+
+	new_tbl = bucket_table_alloc(ht, size, GFP_KERNEL);
+	if (new_tbl == NULL)
+		return -ENOMEM;
+
+	err = rhashtable_rehash_attach(ht, old_tbl, new_tbl);
+	if (err)
+		bucket_table_free(new_tbl);
+
+	return err;
+}
+
+static void rht_deferred_worker(struct work_struct *work)
+{
+	struct rhashtable *ht;
+	struct bucket_table *tbl;
+	int err = 0;
+
+	ht = container_of(work, struct rhashtable, run_work);
+	mutex_lock(&ht->mutex);
+
+	tbl = rht_dereference(ht->tbl, ht);
+	tbl = rhashtable_last_table(ht, tbl);
+
+	if (rht_grow_above_75(ht, tbl))
+		rhashtable_expand(ht);
+	else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl))
+		rhashtable_shrink(ht);
+
+	err = rhashtable_rehash_table(ht);
+
+	mutex_unlock(&ht->mutex);
+
+	if (err)
+		schedule_work(&ht->run_work);
+}
+
+static bool rhashtable_check_elasticity(struct rhashtable *ht,
+					struct bucket_table *tbl,
+					unsigned int hash)
+{
+	unsigned int elasticity = ht->elasticity;
+	struct rhash_head *head;
+
+	rht_for_each(head, tbl, hash)
+		if (!--elasticity)
+			return true;
+
+	return false;
+}
+
+int rhashtable_insert_rehash(struct rhashtable *ht,
+			     struct bucket_table *tbl)
+{
+	struct bucket_table *old_tbl;
+	struct bucket_table *new_tbl;
+	unsigned int size;
+	int err;
+
+	old_tbl = rht_dereference_rcu(ht->tbl, ht);
+
+	size = tbl->size;
+
+	err = -EBUSY;
+
+	if (rht_grow_above_75(ht, tbl))
+		size *= 2;
+	/* Do not schedule more than one rehash */
+	else if (old_tbl != tbl)
+		goto fail;
+
+	err = -ENOMEM;
+
+	new_tbl = bucket_table_alloc(ht, size, GFP_ATOMIC);
+	if (new_tbl == NULL)
+		goto fail;
+
+	err = rhashtable_rehash_attach(ht, tbl, new_tbl);
+	if (err) {
+		bucket_table_free(new_tbl);
+		if (err == -EEXIST)
+			err = 0;
+	} else
+		schedule_work(&ht->run_work);
+
+	return err;
+
+fail:
+	/* Do not fail the insert if someone else did a rehash. */
+	if (likely(rcu_dereference_raw(tbl->future_tbl)))
+		return 0;
+
+	/* Schedule async rehash to retry allocation in process context. */
+	if (err == -ENOMEM)
+		schedule_work(&ht->run_work);
+
+	return err;
+}
+
+struct bucket_table *rhashtable_insert_slow(struct rhashtable *ht,
+					    const void *key,
+					    struct rhash_head *obj,
+					    struct bucket_table *tbl)
+{
+	struct rhash_head *head;
+	unsigned int hash;
+	int err;
+
+	tbl = rhashtable_last_table(ht, tbl);
+	hash = head_hashfn(ht, tbl, obj);
+	spin_lock_nested(rht_bucket_lock(tbl, hash), SINGLE_DEPTH_NESTING);
+
+	err = -EEXIST;
+	if (key && rhashtable_lookup_fast(ht, key, ht->p))
+		goto exit;
+
+	err = -E2BIG;
+	if (unlikely(rht_grow_above_max(ht, tbl)))
+		goto exit;
+
+	err = -EAGAIN;
+	if (rhashtable_check_elasticity(ht, tbl, hash) ||
+	    rht_grow_above_100(ht, tbl))
+		goto exit;
+
+	err = 0;
+
+	head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);
+
+	RCU_INIT_POINTER(obj->next, head);
+
+	rcu_assign_pointer(tbl->buckets[hash], obj);
+
+	atomic_inc(&ht->nelems);
+
+exit:
+	spin_unlock(rht_bucket_lock(tbl, hash));
+
+	if (err == 0)
+		return NULL;
+	else if (err == -EAGAIN)
+		return tbl;
+	else
+		return ERR_PTR(err);
+}
+
+/**
+ * rhashtable_walk_init - Initialise an iterator
+ * @ht:		Table to walk over
+ * @iter:	Hash table Iterator
+ *
+ * This function prepares a hash table walk.
+ *
+ * Note that if you restart a walk after rhashtable_walk_stop you
+ * may see the same object twice.  Also, you may miss objects if
+ * there are removals in between rhashtable_walk_stop and the next
+ * call to rhashtable_walk_start.
+ *
+ * For a completely stable walk you should construct your own data
+ * structure outside the hash table.
+ *
+ * This function may sleep so you must not call it from interrupt
+ * context or with spin locks held.
+ *
+ * You must call rhashtable_walk_exit if this function returns
+ * successfully.
+ */
+int rhashtable_walk_init(struct rhashtable *ht, struct rhashtable_iter *iter)
+{
+	iter->ht = ht;
+	iter->p = NULL;
+	iter->slot = 0;
+	iter->skip = 0;
+
+	iter->walker = kmalloc(sizeof(*iter->walker), GFP_KERNEL);
+	if (!iter->walker)
+		return -ENOMEM;
+
+	spin_lock(&ht->lock);
+	iter->walker->tbl =
+		rcu_dereference_protected(ht->tbl, lockdep_is_held(&ht->lock));
+	list_add(&iter->walker->list, &iter->walker->tbl->walkers);
+	spin_unlock(&ht->lock);
+
+	return 0;
+}
+
+/**
+ * rhashtable_walk_exit - Free an iterator
+ * @iter:	Hash table Iterator
+ *
+ * This function frees resources allocated by rhashtable_walk_init.
+ */
+void rhashtable_walk_exit(struct rhashtable_iter *iter)
+{
+	spin_lock(&iter->ht->lock);
+	if (iter->walker->tbl)
+		list_del(&iter->walker->list);
+	spin_unlock(&iter->ht->lock);
+	kfree(iter->walker);
+}
+
+/**
+ * rhashtable_walk_start - Start a hash table walk
+ * @iter:	Hash table iterator
+ *
+ * Start a hash table walk.  Note that we take the RCU lock in all
+ * cases including when we return an error.  So you must always call
+ * rhashtable_walk_stop to clean up.
+ *
+ * Returns zero if successful.
+ *
+ * Returns -EAGAIN if resize event occured.  Note that the iterator
+ * will rewind back to the beginning and you may use it immediately
+ * by calling rhashtable_walk_next.
+ */
+int rhashtable_walk_start(struct rhashtable_iter *iter)
+	__acquires(RCU)
+{
+	struct rhashtable *ht = iter->ht;
+
+	rcu_read_lock();
+
+	spin_lock(&ht->lock);
+	if (iter->walker->tbl)
+		list_del(&iter->walker->list);
+	spin_unlock(&ht->lock);
+
+	if (!iter->walker->tbl) {
+		iter->walker->tbl = rht_dereference_rcu(ht->tbl, ht);
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
+/**
+ * rhashtable_walk_next - Return the next object and advance the iterator
+ * @iter:	Hash table iterator
+ *
+ * Note that you must call rhashtable_walk_stop when you are finished
+ * with the walk.
+ *
+ * Returns the next object or NULL when the end of the table is reached.
+ *
+ * Returns -EAGAIN if resize event occured.  Note that the iterator
+ * will rewind back to the beginning and you may continue to use it.
+ */
+void *rhashtable_walk_next(struct rhashtable_iter *iter)
+{
+	struct bucket_table *tbl = iter->walker->tbl;
+	struct rhashtable *ht = iter->ht;
+	struct rhash_head *p = iter->p;
+
+	if (p) {
+		p = rht_dereference_bucket_rcu(p->next, tbl, iter->slot);
+		goto next;
+	}
+
+	for (; iter->slot < tbl->size; iter->slot++) {
+		int skip = iter->skip;
+
+		rht_for_each_rcu(p, tbl, iter->slot) {
+			if (!skip)
+				break;
+			skip--;
+		}
+
+next:
+		if (!rht_is_a_nulls(p)) {
+			iter->skip++;
+			iter->p = p;
+			return rht_obj(ht, p);
+		}
+
+		iter->skip = 0;
+	}
+
+	iter->p = NULL;
+
+	/* Ensure we see any new tables. */
+	smp_rmb();
+
+	iter->walker->tbl = rht_dereference_rcu(tbl->future_tbl, ht);
+	if (iter->walker->tbl) {
+		iter->slot = 0;
+		iter->skip = 0;
+		return ERR_PTR(-EAGAIN);
+	}
+
+	return NULL;
+}
+
+/**
+ * rhashtable_walk_stop - Finish a hash table walk
+ * @iter:	Hash table iterator
+ *
+ * Finish a hash table walk.
+ */
+void rhashtable_walk_stop(struct rhashtable_iter *iter)
+	__releases(RCU)
+{
+	struct rhashtable *ht;
+	struct bucket_table *tbl = iter->walker->tbl;
+
+	if (!tbl)
+		goto out;
+
+	ht = iter->ht;
+
+	spin_lock(&ht->lock);
+	if (tbl->rehash < tbl->size)
+		list_add(&iter->walker->list, &tbl->walkers);
+	else
+		iter->walker->tbl = NULL;
+	spin_unlock(&ht->lock);
+
+	iter->p = NULL;
+
+out:
+	rcu_read_unlock();
+}
+
+static size_t rounded_hashtable_size(const struct rhashtable_params *params)
+{
+	return max(roundup_pow_of_two(params->nelem_hint * 4 / 3),
+		   (unsigned long)params->min_size);
+}
+
+static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed)
+{
+	return jhash2(key, length, seed);
+}
+
+/**
+ * rhashtable_init - initialize a new hash table
+ * @ht:		hash table to be initialized
+ * @params:	configuration parameters
+ *
+ * Initializes a new hash table based on the provided configuration
+ * parameters. A table can be configured either with a variable or
+ * fixed length key:
+ *
+ * Configuration Example 1: Fixed length keys
+ * struct test_obj {
+ *	int			key;
+ *	void *			my_member;
+ *	struct rhash_head	node;
+ * };
+ *
+ * struct rhashtable_params params = {
+ *	.head_offset = offsetof(struct test_obj, node),
+ *	.key_offset = offsetof(struct test_obj, key),
+ *	.key_len = sizeof(int),
+ *	.hashfn = jhash,
+ *	.nulls_base = (1U << RHT_BASE_SHIFT),
+ * };
+ *
+ * Configuration Example 2: Variable length keys
+ * struct test_obj {
+ *	[...]
+ *	struct rhash_head	node;
+ * };
+ *
+ * u32 my_hash_fn(const void *data, u32 len, u32 seed)
+ * {
+ *	struct test_obj *obj = data;
+ *
+ *	return [... hash ...];
+ * }
+ *
+ * struct rhashtable_params params = {
+ *	.head_offset = offsetof(struct test_obj, node),
+ *	.hashfn = jhash,
+ *	.obj_hashfn = my_hash_fn,
+ * };
+ */
+int rhashtable_init(struct rhashtable *ht,
+		    const struct rhashtable_params *params)
+{
+	struct bucket_table *tbl;
+	size_t size;
+
+	size = HASH_DEFAULT_SIZE;
+
+	if ((!params->key_len && !params->obj_hashfn) ||
+	    (params->obj_hashfn && !params->obj_cmpfn))
+		return -EINVAL;
+
+	if (params->nulls_base && params->nulls_base < (1U << RHT_BASE_SHIFT))
+		return -EINVAL;
+
+	_rtw_memset(ht, 0, sizeof(*ht));
+	mutex_init(&ht->mutex);
+	spin_lock_init(&ht->lock);
+	_rtw_memcpy(&ht->p, params, sizeof(*params));
+
+	if (params->min_size)
+		ht->p.min_size = roundup_pow_of_two(params->min_size);
+
+	if (params->max_size)
+		ht->p.max_size = rounddown_pow_of_two(params->max_size);
+
+	if (params->insecure_max_entries)
+		ht->p.insecure_max_entries =
+			rounddown_pow_of_two(params->insecure_max_entries);
+	else
+		ht->p.insecure_max_entries = ht->p.max_size * 2;
+
+	ht->p.min_size = max(ht->p.min_size, HASH_MIN_SIZE);
+
+	if (params->nelem_hint)
+		size = rounded_hashtable_size(&ht->p);
+
+	/* The maximum (not average) chain length grows with the
+	 * size of the hash table, at a rate of (log N)/(log log N).
+	 * The value of 16 is selected so that even if the hash
+	 * table grew to 2^32 you would not expect the maximum
+	 * chain length to exceed it unless we are under attack
+	 * (or extremely unlucky).
+	 *
+	 * As this limit is only to detect attacks, we don't need
+	 * to set it to a lower value as you'd need the chain
+	 * length to vastly exceed 16 to have any real effect
+	 * on the system.
+	 */
+	if (!params->insecure_elasticity)
+		ht->elasticity = 16;
+
+	if (params->locks_mul)
+		ht->p.locks_mul = roundup_pow_of_two(params->locks_mul);
+	else
+		ht->p.locks_mul = BUCKET_LOCKS_PER_CPU;
+
+	ht->key_len = ht->p.key_len;
+	if (!params->hashfn) {
+		ht->p.hashfn = jhash;
+
+		if (!(ht->key_len & (sizeof(u32) - 1))) {
+			ht->key_len /= sizeof(u32);
+			ht->p.hashfn = rhashtable_jhash2;
+		}
+	}
+
+	tbl = bucket_table_alloc(ht, size, GFP_KERNEL);
+	if (tbl == NULL)
+		return -ENOMEM;
+
+	atomic_set(&ht->nelems, 0);
+
+	RCU_INIT_POINTER(ht->tbl, tbl);
+
+	INIT_WORK(&ht->run_work, rht_deferred_worker);
+
+	return 0;
+}
+
+/**
+ * rhashtable_free_and_destroy - free elements and destroy hash table
+ * @ht:		the hash table to destroy
+ * @free_fn:	callback to release resources of element
+ * @arg:	pointer passed to free_fn
+ *
+ * Stops an eventual async resize. If defined, invokes free_fn for each
+ * element to releasal resources. Please note that RCU protected
+ * readers may still be accessing the elements. Releasing of resources
+ * must occur in a compatible manner. Then frees the bucket array.
+ *
+ * This function will eventually sleep to wait for an async resize
+ * to complete. The caller is responsible that no further write operations
+ * occurs in parallel.
+ */
+void rhashtable_free_and_destroy(struct rhashtable *ht,
+				 void (*free_fn)(void *ptr, void *arg),
+				 void *arg)
+{
+	const struct bucket_table *tbl;
+	unsigned int i;
+
+	cancel_work_sync(&ht->run_work);
+
+	mutex_lock(&ht->mutex);
+	tbl = rht_dereference(ht->tbl, ht);
+	if (free_fn) {
+		for (i = 0; i < tbl->size; i++) {
+			struct rhash_head *pos, *next;
+
+			for (pos = rht_dereference(tbl->buckets[i], ht),
+			     next = !rht_is_a_nulls(pos) ?
+					rht_dereference(pos->next, ht) : NULL;
+			     !rht_is_a_nulls(pos);
+			     pos = next,
+			     next = !rht_is_a_nulls(pos) ?
+					rht_dereference(pos->next, ht) : NULL)
+				free_fn(rht_obj(ht, pos), arg);
+		}
+	}
+
+	bucket_table_free(tbl);
+	mutex_unlock(&ht->mutex);
+}
+
+void rhashtable_destroy(struct rhashtable *ht)
+{
+	return rhashtable_free_and_destroy(ht, NULL, NULL);
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rhashtable.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rhashtable.h
new file mode 100644
index 000000000000..15a91d11c5f3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rhashtable.h
@@ -0,0 +1,827 @@
+/*
+ * Resizable, Scalable, Concurrent Hash Table
+ *
+ * Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au>
+ * Copyright (c) 2014-2015 Thomas Graf <tgraf@suug.ch>
+ * Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
+ *
+ * Code partially derived from nft_hash
+ * Rewritten with rehash code from br_multicast plus single list
+ * pointer as suggested by Josh Triplett
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _LINUX_RHASHTABLE_H
+#define _LINUX_RHASHTABLE_H
+
+#include <linux/atomic.h>
+#include <linux/compiler.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/jhash.h>
+#include <linux/list_nulls.h>
+#include <linux/workqueue.h>
+#include <linux/mutex.h>
+#include <linux/rcupdate.h>
+
+/*
+ * The end of the chain is marked with a special nulls marks which has
+ * the following format:
+ *
+ * +-------+-----------------------------------------------------+-+
+ * | Base  |                      Hash                           |1|
+ * +-------+-----------------------------------------------------+-+
+ *
+ * Base (4 bits) : Reserved to distinguish between multiple tables.
+ *                 Specified via &struct rhashtable_params.nulls_base.
+ * Hash (27 bits): Full hash (unmasked) of first element added to bucket
+ * 1 (1 bit)     : Nulls marker (always set)
+ *
+ * The remaining bits of the next pointer remain unused for now.
+ */
+#define RHT_BASE_BITS		4
+#define RHT_HASH_BITS		27
+#define RHT_BASE_SHIFT		RHT_HASH_BITS
+
+/* Base bits plus 1 bit for nulls marker */
+#define RHT_HASH_RESERVED_SPACE	(RHT_BASE_BITS + 1)
+
+struct rhash_head {
+	struct rhash_head __rcu		*next;
+};
+
+/**
+ * struct bucket_table - Table of hash buckets
+ * @size: Number of hash buckets
+ * @rehash: Current bucket being rehashed
+ * @hash_rnd: Random seed to fold into hash
+ * @locks_mask: Mask to apply before accessing locks[]
+ * @locks: Array of spinlocks protecting individual buckets
+ * @walkers: List of active walkers
+ * @rcu: RCU structure for freeing the table
+ * @future_tbl: Table under construction during rehashing
+ * @buckets: size * hash buckets
+ */
+struct bucket_table {
+	unsigned int		size;
+	unsigned int		rehash;
+	u32			hash_rnd;
+	unsigned int		locks_mask;
+	spinlock_t		*locks;
+	struct list_head	walkers;
+	struct rcu_head		rcu;
+
+	struct bucket_table __rcu *future_tbl;
+
+	struct rhash_head __rcu	*buckets[] ____cacheline_aligned_in_smp;
+};
+
+/**
+ * struct rhashtable_compare_arg - Key for the function rhashtable_compare
+ * @ht: Hash table
+ * @key: Key to compare against
+ */
+struct rhashtable_compare_arg {
+	struct rhashtable *ht;
+	const void *key;
+};
+
+typedef u32 (*rht_hashfn_t)(const void *data, u32 len, u32 seed);
+typedef u32 (*rht_obj_hashfn_t)(const void *data, u32 len, u32 seed);
+typedef int (*rht_obj_cmpfn_t)(struct rhashtable_compare_arg *arg,
+			       const void *obj);
+
+struct rhashtable;
+
+/**
+ * struct rhashtable_params - Hash table construction parameters
+ * @nelem_hint: Hint on number of elements, should be 75% of desired size
+ * @key_len: Length of key
+ * @key_offset: Offset of key in struct to be hashed
+ * @head_offset: Offset of rhash_head in struct to be hashed
+ * @insecure_max_entries: Maximum number of entries (may be exceeded)
+ * @max_size: Maximum size while expanding
+ * @min_size: Minimum size while shrinking
+ * @nulls_base: Base value to generate nulls marker
+ * @insecure_elasticity: Set to true to disable chain length checks
+ * @automatic_shrinking: Enable automatic shrinking of tables
+ * @locks_mul: Number of bucket locks to allocate per cpu (default: 128)
+ * @hashfn: Hash function (default: jhash2 if !(key_len % 4), or jhash)
+ * @obj_hashfn: Function to hash object
+ * @obj_cmpfn: Function to compare key with object
+ */
+struct rhashtable_params {
+	size_t			nelem_hint;
+	size_t			key_len;
+	size_t			key_offset;
+	size_t			head_offset;
+	unsigned int		insecure_max_entries;
+	unsigned int		max_size;
+	unsigned int		min_size;
+	u32			nulls_base;
+	bool			insecure_elasticity;
+	bool			automatic_shrinking;
+	size_t			locks_mul;
+	rht_hashfn_t		hashfn;
+	rht_obj_hashfn_t	obj_hashfn;
+	rht_obj_cmpfn_t		obj_cmpfn;
+};
+
+/**
+ * struct rhashtable - Hash table handle
+ * @tbl: Bucket table
+ * @nelems: Number of elements in table
+ * @key_len: Key length for hashfn
+ * @elasticity: Maximum chain length before rehash
+ * @p: Configuration parameters
+ * @run_work: Deferred worker to expand/shrink asynchronously
+ * @mutex: Mutex to protect current/future table swapping
+ * @lock: Spin lock to protect walker list
+ */
+struct rhashtable {
+	struct bucket_table __rcu	*tbl;
+	atomic_t			nelems;
+	unsigned int			key_len;
+	unsigned int			elasticity;
+	struct rhashtable_params	p;
+	struct work_struct		run_work;
+	struct mutex                    mutex;
+	spinlock_t			lock;
+};
+
+/**
+ * struct rhashtable_walker - Hash table walker
+ * @list: List entry on list of walkers
+ * @tbl: The table that we were walking over
+ */
+struct rhashtable_walker {
+	struct list_head list;
+	struct bucket_table *tbl;
+};
+
+/**
+ * struct rhashtable_iter - Hash table iterator, fits into netlink cb
+ * @ht: Table to iterate through
+ * @p: Current pointer
+ * @walker: Associated rhashtable walker
+ * @slot: Current slot
+ * @skip: Number of entries to skip in slot
+ */
+struct rhashtable_iter {
+	struct rhashtable *ht;
+	struct rhash_head *p;
+	struct rhashtable_walker *walker;
+	unsigned int slot;
+	unsigned int skip;
+};
+
+static inline unsigned long rht_marker(const struct rhashtable *ht, u32 hash)
+{
+	return NULLS_MARKER(ht->p.nulls_base + hash);
+}
+
+#define INIT_RHT_NULLS_HEAD(ptr, ht, hash) \
+	((ptr) = (typeof(ptr)) rht_marker(ht, hash))
+
+static inline bool rht_is_a_nulls(const struct rhash_head *ptr)
+{
+	return ((unsigned long) ptr & 1);
+}
+
+static inline unsigned long rht_get_nulls_value(const struct rhash_head *ptr)
+{
+	return ((unsigned long) ptr) >> 1;
+}
+
+static inline void *rht_obj(const struct rhashtable *ht,
+			    const struct rhash_head *he)
+{
+	return (char *)he - ht->p.head_offset;
+}
+
+static inline unsigned int rht_bucket_index(const struct bucket_table *tbl,
+					    unsigned int hash)
+{
+	return (hash >> RHT_HASH_RESERVED_SPACE) & (tbl->size - 1);
+}
+
+static inline unsigned int rht_key_hashfn(
+	struct rhashtable *ht, const struct bucket_table *tbl,
+	const void *key, const struct rhashtable_params params)
+{
+	unsigned int hash;
+
+	/* params must be equal to ht->p if it isn't constant. */
+	if (!__builtin_constant_p(params.key_len))
+		hash = ht->p.hashfn(key, ht->key_len, tbl->hash_rnd);
+	else if (params.key_len) {
+		unsigned int key_len = params.key_len;
+
+		if (params.hashfn)
+			hash = params.hashfn(key, key_len, tbl->hash_rnd);
+		else if (key_len & (sizeof(u32) - 1))
+			hash = jhash(key, key_len, tbl->hash_rnd);
+		else
+			hash = jhash2(key, key_len / sizeof(u32),
+				      tbl->hash_rnd);
+	} else {
+		unsigned int key_len = ht->p.key_len;
+
+		if (params.hashfn)
+			hash = params.hashfn(key, key_len, tbl->hash_rnd);
+		else
+			hash = jhash(key, key_len, tbl->hash_rnd);
+	}
+
+	return rht_bucket_index(tbl, hash);
+}
+
+static inline unsigned int rht_head_hashfn(
+	struct rhashtable *ht, const struct bucket_table *tbl,
+	const struct rhash_head *he, const struct rhashtable_params params)
+{
+	const char *ptr = rht_obj(ht, he);
+
+	return likely(params.obj_hashfn) ?
+	       rht_bucket_index(tbl, params.obj_hashfn(ptr, params.key_len ?:
+							    ht->p.key_len,
+						       tbl->hash_rnd)) :
+	       rht_key_hashfn(ht, tbl, ptr + params.key_offset, params);
+}
+
+/**
+ * rht_grow_above_75 - returns true if nelems > 0.75 * table-size
+ * @ht:		hash table
+ * @tbl:	current table
+ */
+static inline bool rht_grow_above_75(const struct rhashtable *ht,
+				     const struct bucket_table *tbl)
+{
+	/* Expand table when exceeding 75% load */
+	return atomic_read(&ht->nelems) > (tbl->size / 4 * 3) &&
+	       (!ht->p.max_size || tbl->size < ht->p.max_size);
+}
+
+/**
+ * rht_shrink_below_30 - returns true if nelems < 0.3 * table-size
+ * @ht:		hash table
+ * @tbl:	current table
+ */
+static inline bool rht_shrink_below_30(const struct rhashtable *ht,
+				       const struct bucket_table *tbl)
+{
+	/* Shrink table beneath 30% load */
+	return atomic_read(&ht->nelems) < (tbl->size * 3 / 10) &&
+	       tbl->size > ht->p.min_size;
+}
+
+/**
+ * rht_grow_above_100 - returns true if nelems > table-size
+ * @ht:		hash table
+ * @tbl:	current table
+ */
+static inline bool rht_grow_above_100(const struct rhashtable *ht,
+				      const struct bucket_table *tbl)
+{
+	return atomic_read(&ht->nelems) > tbl->size &&
+		(!ht->p.max_size || tbl->size < ht->p.max_size);
+}
+
+/**
+ * rht_grow_above_max - returns true if table is above maximum
+ * @ht:		hash table
+ * @tbl:	current table
+ */
+static inline bool rht_grow_above_max(const struct rhashtable *ht,
+				      const struct bucket_table *tbl)
+{
+	return ht->p.insecure_max_entries &&
+	       atomic_read(&ht->nelems) >= ht->p.insecure_max_entries;
+}
+
+/* The bucket lock is selected based on the hash and protects mutations
+ * on a group of hash buckets.
+ *
+ * A maximum of tbl->size/2 bucket locks is allocated. This ensures that
+ * a single lock always covers both buckets which may both contains
+ * entries which link to the same bucket of the old table during resizing.
+ * This allows to simplify the locking as locking the bucket in both
+ * tables during resize always guarantee protection.
+ *
+ * IMPORTANT: When holding the bucket lock of both the old and new table
+ * during expansions and shrinking, the old bucket lock must always be
+ * acquired first.
+ */
+static inline spinlock_t *rht_bucket_lock(const struct bucket_table *tbl,
+					  unsigned int hash)
+{
+	return &tbl->locks[hash & tbl->locks_mask];
+}
+
+#ifdef CONFIG_PROVE_LOCKING
+int lockdep_rht_mutex_is_held(struct rhashtable *ht);
+int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, u32 hash);
+#else
+static inline int lockdep_rht_mutex_is_held(struct rhashtable *ht)
+{
+	return 1;
+}
+
+static inline int lockdep_rht_bucket_is_held(const struct bucket_table *tbl,
+					     u32 hash)
+{
+	return 1;
+}
+#endif /* CONFIG_PROVE_LOCKING */
+
+int rhashtable_init(struct rhashtable *ht,
+		    const struct rhashtable_params *params);
+
+struct bucket_table *rhashtable_insert_slow(struct rhashtable *ht,
+					    const void *key,
+					    struct rhash_head *obj,
+					    struct bucket_table *old_tbl);
+int rhashtable_insert_rehash(struct rhashtable *ht, struct bucket_table *tbl);
+
+int rhashtable_walk_init(struct rhashtable *ht, struct rhashtable_iter *iter);
+void rhashtable_walk_exit(struct rhashtable_iter *iter);
+int rhashtable_walk_start(struct rhashtable_iter *iter) __acquires(RCU);
+void *rhashtable_walk_next(struct rhashtable_iter *iter);
+void rhashtable_walk_stop(struct rhashtable_iter *iter) __releases(RCU);
+
+void rhashtable_free_and_destroy(struct rhashtable *ht,
+				 void (*free_fn)(void *ptr, void *arg),
+				 void *arg);
+void rhashtable_destroy(struct rhashtable *ht);
+
+#define rht_dereference(p, ht) \
+	rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
+
+#define rht_dereference_rcu(p, ht) \
+	rcu_dereference_check(p, lockdep_rht_mutex_is_held(ht))
+
+#define rht_dereference_bucket(p, tbl, hash) \
+	rcu_dereference_protected(p, lockdep_rht_bucket_is_held(tbl, hash))
+
+#define rht_dereference_bucket_rcu(p, tbl, hash) \
+	rcu_dereference_check(p, lockdep_rht_bucket_is_held(tbl, hash))
+
+#define rht_entry(tpos, pos, member) \
+	({ tpos = container_of(pos, typeof(*tpos), member); 1; })
+
+/**
+ * rht_for_each_continue - continue iterating over hash chain
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @head:	the previous &struct rhash_head to continue from
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ */
+#define rht_for_each_continue(pos, head, tbl, hash) \
+	for (pos = rht_dereference_bucket(head, tbl, hash); \
+	     !rht_is_a_nulls(pos); \
+	     pos = rht_dereference_bucket((pos)->next, tbl, hash))
+
+/**
+ * rht_for_each - iterate over hash chain
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ */
+#define rht_for_each(pos, tbl, hash) \
+	rht_for_each_continue(pos, (tbl)->buckets[hash], tbl, hash)
+
+/**
+ * rht_for_each_entry_continue - continue iterating over hash chain
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @head:	the previous &struct rhash_head to continue from
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ * @member:	name of the &struct rhash_head within the hashable struct.
+ */
+#define rht_for_each_entry_continue(tpos, pos, head, tbl, hash, member)	\
+	for (pos = rht_dereference_bucket(head, tbl, hash);		\
+	     (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member);	\
+	     pos = rht_dereference_bucket((pos)->next, tbl, hash))
+
+/**
+ * rht_for_each_entry - iterate over hash chain of given type
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ * @member:	name of the &struct rhash_head within the hashable struct.
+ */
+#define rht_for_each_entry(tpos, pos, tbl, hash, member)		\
+	rht_for_each_entry_continue(tpos, pos, (tbl)->buckets[hash],	\
+				    tbl, hash, member)
+
+/**
+ * rht_for_each_entry_safe - safely iterate over hash chain of given type
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @next:	the &struct rhash_head to use as next in loop cursor.
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ * @member:	name of the &struct rhash_head within the hashable struct.
+ *
+ * This hash chain list-traversal primitive allows for the looped code to
+ * remove the loop cursor from the list.
+ */
+#define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member)	    \
+	for (pos = rht_dereference_bucket((tbl)->buckets[hash], tbl, hash), \
+	     next = !rht_is_a_nulls(pos) ?				    \
+		       rht_dereference_bucket(pos->next, tbl, hash) : NULL; \
+	     (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member);	    \
+	     pos = next,						    \
+	     next = !rht_is_a_nulls(pos) ?				    \
+		       rht_dereference_bucket(pos->next, tbl, hash) : NULL)
+
+/**
+ * rht_for_each_rcu_continue - continue iterating over rcu hash chain
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @head:	the previous &struct rhash_head to continue from
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ *
+ * This hash chain list-traversal primitive may safely run concurrently with
+ * the _rcu mutation primitives such as rhashtable_insert() as long as the
+ * traversal is guarded by rcu_read_lock().
+ */
+#define rht_for_each_rcu_continue(pos, head, tbl, hash)			\
+	for (({barrier(); }),						\
+	     pos = rht_dereference_bucket_rcu(head, tbl, hash);		\
+	     !rht_is_a_nulls(pos);					\
+	     pos = rcu_dereference_raw(pos->next))
+
+/**
+ * rht_for_each_rcu - iterate over rcu hash chain
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ *
+ * This hash chain list-traversal primitive may safely run concurrently with
+ * the _rcu mutation primitives such as rhashtable_insert() as long as the
+ * traversal is guarded by rcu_read_lock().
+ */
+#define rht_for_each_rcu(pos, tbl, hash)				\
+	rht_for_each_rcu_continue(pos, (tbl)->buckets[hash], tbl, hash)
+
+/**
+ * rht_for_each_entry_rcu_continue - continue iterating over rcu hash chain
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @head:	the previous &struct rhash_head to continue from
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ * @member:	name of the &struct rhash_head within the hashable struct.
+ *
+ * This hash chain list-traversal primitive may safely run concurrently with
+ * the _rcu mutation primitives such as rhashtable_insert() as long as the
+ * traversal is guarded by rcu_read_lock().
+ */
+#define rht_for_each_entry_rcu_continue(tpos, pos, head, tbl, hash, member) \
+	for (({barrier(); }),						    \
+	     pos = rht_dereference_bucket_rcu(head, tbl, hash);		    \
+	     (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member);	    \
+	     pos = rht_dereference_bucket_rcu(pos->next, tbl, hash))
+
+/**
+ * rht_for_each_entry_rcu - iterate over rcu hash chain of given type
+ * @tpos:	the type * to use as a loop cursor.
+ * @pos:	the &struct rhash_head to use as a loop cursor.
+ * @tbl:	the &struct bucket_table
+ * @hash:	the hash value / bucket index
+ * @member:	name of the &struct rhash_head within the hashable struct.
+ *
+ * This hash chain list-traversal primitive may safely run concurrently with
+ * the _rcu mutation primitives such as rhashtable_insert() as long as the
+ * traversal is guarded by rcu_read_lock().
+ */
+#define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member)		\
+	rht_for_each_entry_rcu_continue(tpos, pos, (tbl)->buckets[hash],\
+					tbl, hash, member)
+
+static inline int rhashtable_compare(struct rhashtable_compare_arg *arg,
+				     const void *obj)
+{
+	struct rhashtable *ht = arg->ht;
+	const char *ptr = obj;
+
+	return (_rtw_memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len) == _TRUE)? 0: 1;
+}
+
+/**
+ * rhashtable_lookup_fast - search hash table, inlined version
+ * @ht:		hash table
+ * @key:	the pointer to the key
+ * @params:	hash table parameters
+ *
+ * Computes the hash value for the key and traverses the bucket chain looking
+ * for a entry with an identical key. The first matching entry is returned.
+ *
+ * Returns the first entry on which the compare function returned true.
+ */
+static inline void *rhashtable_lookup_fast(
+	struct rhashtable *ht, const void *key,
+	const struct rhashtable_params params)
+{
+	struct rhashtable_compare_arg arg = {
+		.ht = ht,
+		.key = key,
+	};
+	const struct bucket_table *tbl;
+	struct rhash_head *he;
+	unsigned int hash;
+
+	rcu_read_lock();
+
+	tbl = rht_dereference_rcu(ht->tbl, ht);
+restart:
+	hash = rht_key_hashfn(ht, tbl, key, params);
+	rht_for_each_rcu(he, tbl, hash) {
+		if (params.obj_cmpfn ?
+		    params.obj_cmpfn(&arg, rht_obj(ht, he)) :
+		    rhashtable_compare(&arg, rht_obj(ht, he)))
+			continue;
+		rcu_read_unlock();
+		return rht_obj(ht, he);
+	}
+
+	/* Ensure we see any new tables. */
+	smp_rmb();
+
+	tbl = rht_dereference_rcu(tbl->future_tbl, ht);
+	if (unlikely(tbl))
+		goto restart;
+	rcu_read_unlock();
+
+	return NULL;
+}
+
+/* Internal function, please use rhashtable_insert_fast() instead */
+static inline int __rhashtable_insert_fast(
+	struct rhashtable *ht, const void *key, struct rhash_head *obj,
+	const struct rhashtable_params params)
+{
+	struct rhashtable_compare_arg arg = {
+		.ht = ht,
+		.key = key,
+	};
+	struct bucket_table *tbl, *new_tbl;
+	struct rhash_head *head;
+	spinlock_t *lock;
+	unsigned int elasticity;
+	unsigned int hash;
+	int err;
+
+restart:
+	rcu_read_lock();
+
+	tbl = rht_dereference_rcu(ht->tbl, ht);
+
+	/* All insertions must grab the oldest table containing
+	 * the hashed bucket that is yet to be rehashed.
+	 */
+	for (;;) {
+		hash = rht_head_hashfn(ht, tbl, obj, params);
+		lock = rht_bucket_lock(tbl, hash);
+		spin_lock_bh(lock);
+
+		if (tbl->rehash <= hash)
+			break;
+
+		spin_unlock_bh(lock);
+		tbl = rht_dereference_rcu(tbl->future_tbl, ht);
+	}
+
+	new_tbl = rht_dereference_rcu(tbl->future_tbl, ht);
+	if (unlikely(new_tbl)) {
+		tbl = rhashtable_insert_slow(ht, key, obj, new_tbl);
+		if (!IS_ERR_OR_NULL(tbl))
+			goto slow_path;
+
+		err = PTR_ERR(tbl);
+		goto out;
+	}
+
+	err = -E2BIG;
+	if (unlikely(rht_grow_above_max(ht, tbl)))
+		goto out;
+
+	if (unlikely(rht_grow_above_100(ht, tbl))) {
+slow_path:
+		spin_unlock_bh(lock);
+		err = rhashtable_insert_rehash(ht, tbl);
+		rcu_read_unlock();
+		if (err)
+			return err;
+
+		goto restart;
+	}
+
+	err = -EEXIST;
+	elasticity = ht->elasticity;
+	rht_for_each(head, tbl, hash) {
+		if (key &&
+		    unlikely(!(params.obj_cmpfn ?
+			       params.obj_cmpfn(&arg, rht_obj(ht, head)) :
+			       rhashtable_compare(&arg, rht_obj(ht, head)))))
+			goto out;
+		if (!--elasticity)
+			goto slow_path;
+	}
+
+	err = 0;
+
+	head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);
+
+	RCU_INIT_POINTER(obj->next, head);
+
+	rcu_assign_pointer(tbl->buckets[hash], obj);
+
+	atomic_inc(&ht->nelems);
+	if (rht_grow_above_75(ht, tbl))
+		schedule_work(&ht->run_work);
+
+out:
+	spin_unlock_bh(lock);
+	rcu_read_unlock();
+
+	return err;
+}
+
+/**
+ * rhashtable_insert_fast - insert object into hash table
+ * @ht:		hash table
+ * @obj:	pointer to hash head inside object
+ * @params:	hash table parameters
+ *
+ * Will take a per bucket spinlock to protect against mutual mutations
+ * on the same bucket. Multiple insertions may occur in parallel unless
+ * they map to the same bucket lock.
+ *
+ * It is safe to call this function from atomic context.
+ *
+ * Will trigger an automatic deferred table resizing if the size grows
+ * beyond the watermark indicated by grow_decision() which can be passed
+ * to rhashtable_init().
+ */
+static inline int rhashtable_insert_fast(
+	struct rhashtable *ht, struct rhash_head *obj,
+	const struct rhashtable_params params)
+{
+	return __rhashtable_insert_fast(ht, NULL, obj, params);
+}
+
+/**
+ * rhashtable_lookup_insert_fast - lookup and insert object into hash table
+ * @ht:		hash table
+ * @obj:	pointer to hash head inside object
+ * @params:	hash table parameters
+ *
+ * Locks down the bucket chain in both the old and new table if a resize
+ * is in progress to ensure that writers can't remove from the old table
+ * and can't insert to the new table during the atomic operation of search
+ * and insertion. Searches for duplicates in both the old and new table if
+ * a resize is in progress.
+ *
+ * This lookup function may only be used for fixed key hash table (key_len
+ * parameter set). It will BUG() if used inappropriately.
+ *
+ * It is safe to call this function from atomic context.
+ *
+ * Will trigger an automatic deferred table resizing if the size grows
+ * beyond the watermark indicated by grow_decision() which can be passed
+ * to rhashtable_init().
+ */
+static inline int rhashtable_lookup_insert_fast(
+	struct rhashtable *ht, struct rhash_head *obj,
+	const struct rhashtable_params params)
+{
+	const char *key = rht_obj(ht, obj);
+
+	BUG_ON(ht->p.obj_hashfn);
+
+	return __rhashtable_insert_fast(ht, key + ht->p.key_offset, obj,
+					params);
+}
+
+/**
+ * rhashtable_lookup_insert_key - search and insert object to hash table
+ *				  with explicit key
+ * @ht:		hash table
+ * @key:	key
+ * @obj:	pointer to hash head inside object
+ * @params:	hash table parameters
+ *
+ * Locks down the bucket chain in both the old and new table if a resize
+ * is in progress to ensure that writers can't remove from the old table
+ * and can't insert to the new table during the atomic operation of search
+ * and insertion. Searches for duplicates in both the old and new table if
+ * a resize is in progress.
+ *
+ * Lookups may occur in parallel with hashtable mutations and resizing.
+ *
+ * Will trigger an automatic deferred table resizing if the size grows
+ * beyond the watermark indicated by grow_decision() which can be passed
+ * to rhashtable_init().
+ *
+ * Returns zero on success.
+ */
+static inline int rhashtable_lookup_insert_key(
+	struct rhashtable *ht, const void *key, struct rhash_head *obj,
+	const struct rhashtable_params params)
+{
+	BUG_ON(!ht->p.obj_hashfn || !key);
+
+	return __rhashtable_insert_fast(ht, key, obj, params);
+}
+
+/* Internal function, please use rhashtable_remove_fast() instead */
+static inline int __rhashtable_remove_fast(
+	struct rhashtable *ht, struct bucket_table *tbl,
+	struct rhash_head *obj, const struct rhashtable_params params)
+{
+	struct rhash_head __rcu **pprev;
+	struct rhash_head *he;
+	spinlock_t * lock;
+	unsigned int hash;
+	int err = -ENOENT;
+
+	hash = rht_head_hashfn(ht, tbl, obj, params);
+	lock = rht_bucket_lock(tbl, hash);
+
+	spin_lock_bh(lock);
+
+	pprev = &tbl->buckets[hash];
+	rht_for_each(he, tbl, hash) {
+		if (he != obj) {
+			pprev = &he->next;
+			continue;
+		}
+
+		rcu_assign_pointer(*pprev, obj->next);
+		err = 0;
+		break;
+	}
+
+	spin_unlock_bh(lock);
+
+	return err;
+}
+
+/**
+ * rhashtable_remove_fast - remove object from hash table
+ * @ht:		hash table
+ * @obj:	pointer to hash head inside object
+ * @params:	hash table parameters
+ *
+ * Since the hash chain is single linked, the removal operation needs to
+ * walk the bucket chain upon removal. The removal operation is thus
+ * considerable slow if the hash table is not correctly sized.
+ *
+ * Will automatically shrink the table via rhashtable_expand() if the
+ * shrink_decision function specified at rhashtable_init() returns true.
+ *
+ * Returns zero on success, -ENOENT if the entry could not be found.
+ */
+static inline int rhashtable_remove_fast(
+	struct rhashtable *ht, struct rhash_head *obj,
+	const struct rhashtable_params params)
+{
+	struct bucket_table *tbl;
+	int err;
+
+	rcu_read_lock();
+
+	tbl = rht_dereference_rcu(ht->tbl, ht);
+
+	/* Because we have already taken (and released) the bucket
+	 * lock in old_tbl, if we find that future_tbl is not yet
+	 * visible then that guarantees the entry to still be in
+	 * the old tbl if it exists.
+	 */
+	while ((err = __rhashtable_remove_fast(ht, tbl, obj, params)) &&
+	       (tbl = rht_dereference_rcu(tbl->future_tbl, ht)))
+		;
+
+	if (err)
+		goto out;
+
+	atomic_dec(&ht->nelems);
+	if (unlikely(ht->p.automatic_shrinking &&
+		     rht_shrink_below_30(ht, tbl)))
+		schedule_work(&ht->run_work);
+
+out:
+	rcu_read_unlock();
+
+	return err;
+}
+
+#endif /* _LINUX_RHASHTABLE_H */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_android.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_android.c
new file mode 100644
index 000000000000..11d52a94bce3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_android.c
@@ -0,0 +1,1084 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifdef CONFIG_GPIO_WAKEUP
+#include <linux/gpio.h>
+#endif
+
+#include <drv_types.h>
+
+#if defined(RTW_ENABLE_WIFI_CONTROL_FUNC)
+#include <linux/platform_device.h>
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	#include <linux/wlan_plat.h>
+#else
+	#include <linux/wifi_tiwlan.h>
+#endif
+#endif /* defined(RTW_ENABLE_WIFI_CONTROL_FUNC) */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+#define strnicmp	strncasecmp
+#endif /* Linux kernel >= 4.0.0 */
+
+#ifdef CONFIG_GPIO_WAKEUP
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#endif
+
+#include "rtw_version.h"
+
+extern void macstr2num(u8 *dst, u8 *src);
+
+const char *android_wifi_cmd_str[ANDROID_WIFI_CMD_MAX] = {
+	"START",
+	"STOP",
+	"SCAN-ACTIVE",
+	"SCAN-PASSIVE",
+	"RSSI",
+	"LINKSPEED",
+	"RXFILTER-START",
+	"RXFILTER-STOP",
+	"RXFILTER-ADD",
+	"RXFILTER-REMOVE",
+	"BTCOEXSCAN-START",
+	"BTCOEXSCAN-STOP",
+	"BTCOEXMODE",
+	"SETSUSPENDMODE",
+	"SETSUSPENDOPT",
+	"P2P_DEV_ADDR",
+	"SETFWPATH",
+	"SETBAND",
+	"GETBAND",
+	"COUNTRY",
+	"P2P_SET_NOA",
+	"P2P_GET_NOA",
+	"P2P_SET_PS",
+	"SET_AP_WPS_P2P_IE",
+
+	"MIRACAST",
+
+#ifdef CONFIG_PNO_SUPPORT
+	"PNOSSIDCLR",
+	"PNOSETUP",
+	"PNOFORCE",
+	"PNODEBUG",
+#endif
+
+	"MACADDR",
+
+	"BLOCK_SCAN",
+	"BLOCK",
+	"WFD-ENABLE",
+	"WFD-DISABLE",
+	"WFD-SET-TCPPORT",
+	"WFD-SET-MAXTPUT",
+	"WFD-SET-DEVTYPE",
+	"HOSTAPD_SET_MACADDR_ACL",
+	"HOSTAPD_ACL_ADD_STA",
+	"HOSTAPD_ACL_REMOVE_STA",
+#if defined(CONFIG_GTK_OL) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0))
+	"GTK_REKEY_OFFLOAD",
+#endif /* CONFIG_GTK_OL */
+/*	Private command for	P2P disable*/
+	"P2P_DISABLE",
+	"SET_AEK",
+	"EXT_AUTH_STATUS",
+	"DRIVER_VERSION"
+#ifdef PRIVATE_R
+	,"ROKU_FIND_REMOTE"
+#endif
+};
+
+typedef struct android_wifi_priv_cmd {
+	char *buf;
+	int used_len;
+	int total_len;
+} android_wifi_priv_cmd;
+
+#ifdef CONFIG_COMPAT
+typedef struct compat_android_wifi_priv_cmd {
+	compat_uptr_t buf;
+	int used_len;
+	int total_len;
+} compat_android_wifi_priv_cmd;
+#endif /* CONFIG_COMPAT */
+
+/**
+ * Local (static) functions and variables
+ */
+
+/* Initialize g_wifi_on to 1 so dhd_bus_start will be called for the first
+ * time (only) in dhd_open, subsequential wifi on will be handled by
+ * wl_android_wifi_on
+ */
+static int g_wifi_on = _TRUE;
+
+unsigned int oob_irq = 0;
+unsigned int oob_gpio = 0;
+
+int rtw_android_cmdstr_to_num(char *cmdstr)
+{
+	int cmd_num;
+	for (cmd_num = 0 ; cmd_num < ANDROID_WIFI_CMD_MAX; cmd_num++)
+		if (0 == strnicmp(cmdstr , android_wifi_cmd_str[cmd_num], strlen(android_wifi_cmd_str[cmd_num])))
+			break;
+
+	return cmd_num;
+}
+
+int rtw_android_get_rssi(struct net_device *net, char *command, int total_len)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	wlan_network	*pcur_network = &pmlmepriv->dev_cur_network;
+	int bytes_written = 0;
+
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == _TRUE) {
+		bytes_written += snprintf(&command[bytes_written], total_len, "%s rssi %d",
+			pcur_network->network.Ssid.Ssid, padapter->recvinfo.rssi);
+	}
+
+	return bytes_written;
+}
+
+int rtw_android_get_link_speed(struct net_device *net, char *command, int total_len)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	int bytes_written = 0;
+	u16 link_speed = 0;
+
+	link_speed = rtw_get_cur_max_rate(padapter) / 10;
+	bytes_written = snprintf(command, total_len, "LinkSpeed %d", link_speed);
+
+	return bytes_written;
+}
+
+int rtw_android_get_macaddr(struct net_device *net, char *command, int total_len)
+{
+	int bytes_written = 0;
+
+	bytes_written = snprintf(command, total_len, "Macaddr = "MAC_FMT, MAC_ARG(net->dev_addr));
+	return bytes_written;
+}
+
+int rtw_android_set_country(struct net_device *net, char *command, int total_len)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(net);
+	char *country_code = command + strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_COUNTRY]) + 1;
+	int ret = _FAIL;
+
+	ret = rtw_set_country(adapter, country_code, RTW_ENV_NUM, RTW_REGD_SET_BY_USER);
+
+	return (ret == _SUCCESS) ? 0 : -1;
+}
+
+int rtw_android_get_p2p_dev_addr(struct net_device *net, char *command, int total_len)
+{
+	int bytes_written = 0;
+
+	/* We use the same address as our HW MAC address */
+	_rtw_memcpy(command, net->dev_addr, ETH_ALEN);
+
+	bytes_written = ETH_ALEN;
+	return bytes_written;
+}
+
+int rtw_android_set_block_scan(struct net_device *net, char *command, int total_len)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(net);
+	char *block_value = command + strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_BLOCK_SCAN]) + 1;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	adapter_wdev_data(adapter)->block_scan = (*block_value == '0') ? _FALSE : _TRUE;
+#endif
+
+	return 0;
+}
+
+int rtw_android_set_block(struct net_device *net, char *command, int total_len)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(net);
+	char *block_value = command + strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_BLOCK]) + 1;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	adapter_wdev_data(adapter)->block = (*block_value == '0') ? _FALSE : _TRUE;
+#endif
+
+	return 0;
+}
+
+int rtw_android_setband(struct net_device *net, char *command, int total_len)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(net);
+	char *arg = command + strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_SETBAND]) + 1;
+	u32 band = WIFI_FREQUENCY_BAND_AUTO;
+	int ret = _FAIL;
+
+	if (sscanf(arg, "%u", &band) >= 1)
+		ret = rtw_set_band(adapter, band);
+
+	return (ret == _SUCCESS) ? 0 : -1;
+}
+
+int rtw_android_getband(struct net_device *net, char *command, int total_len)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(net);
+	int bytes_written = 0;
+
+	bytes_written = snprintf(command, total_len, "%u", adapter->setband);
+
+	return bytes_written;
+}
+
+#ifdef CONFIG_WFD
+int rtw_android_set_miracast_mode(struct net_device *net, char *command, int total_len)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(net);
+	struct wifi_display_info *wfd_info = &adapter->wfd_info;
+	char *arg = command + strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_MIRACAST]) + 1;
+	u8 mode;
+	int num;
+	int ret = _FAIL;
+
+	num = sscanf(arg, "%hhu", &mode);
+
+	if (num < 1)
+		goto exit;
+
+	switch (mode) {
+	case 1: /* soruce */
+		mode = MIRACAST_SOURCE;
+		break;
+	case 2: /* sink */
+		mode = MIRACAST_SINK;
+		break;
+	case 0: /* disabled */
+	default:
+		mode = MIRACAST_DISABLED;
+		break;
+	}
+	wfd_info->stack_wfd_mode = mode;
+	RTW_INFO("stack miracast mode: %s\n", get_miracast_mode_str(wfd_info->stack_wfd_mode));
+
+	ret = _SUCCESS;
+
+exit:
+	return (ret == _SUCCESS) ? 0 : -1;
+}
+#endif /* CONFIG_WFD */
+
+int get_int_from_command(char *pcmd)
+{
+	int i = 0;
+
+	for (i = 0; i < strlen(pcmd); i++) {
+		if (pcmd[i] == '=') {
+			/*	Skip the '=' and space characters. */
+			i += 2;
+			break;
+		}
+	}
+	return rtw_atoi(pcmd + i) ;
+}
+
+#if defined(CONFIG_GTK_OL) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0))
+int rtw_gtk_offload(struct net_device *net, u8 *cmd_ptr)
+{
+	int i;
+	/* u8 *cmd_ptr = priv_cmd.buf; */
+	struct sta_info *psta;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(net);
+	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+
+
+	if (psta == NULL)
+		RTW_INFO("%s, : Obtain Sta_info fail\n", __func__);
+	else {
+		/* string command length of "GTK_REKEY_OFFLOAD" */
+		cmd_ptr += 18;
+
+		_rtw_memcpy(psta->kek, cmd_ptr, RTW_KEK_LEN);
+		cmd_ptr += RTW_KEK_LEN;
+		/*
+		printk("supplicant KEK: ");
+		for(i=0;i<RTW_KEK_LEN; i++)
+			printk(" %02x ", psta->kek[i]);
+		printk("\n supplicant KCK: ");
+		*/
+		_rtw_memcpy(psta->kck, cmd_ptr, RTW_KCK_LEN);
+		cmd_ptr += RTW_KCK_LEN;
+		/*
+		for(i=0;i<RTW_KEK_LEN; i++)
+			printk(" %02x ", psta->kck[i]);
+		*/
+		_rtw_memcpy(psta->replay_ctr, cmd_ptr, RTW_REPLAY_CTR_LEN);
+		psecuritypriv->binstallKCK_KEK = _TRUE;
+
+		/* printk("\nREPLAY_CTR: "); */
+		/* for(i=0;i<RTW_REPLAY_CTR_LEN; i++) */
+		/* printk(" %02x ", psta->replay_ctr[i]); */
+	}
+
+	return _SUCCESS;
+}
+#endif /* CONFIG_GTK_OL */
+
+#ifdef CONFIG_RTW_MESH_AEK
+static int rtw_android_set_aek(struct net_device *ndev, char *command, int total_len)
+{
+#define SET_AEK_DATA_LEN (ETH_ALEN + 32)
+
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(ndev);
+	u8 *addr;
+	u8 *aek;
+	int err = 0;
+
+	if (total_len - strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_SET_AEK]) - 1 != SET_AEK_DATA_LEN) {
+		err = -EINVAL;
+		goto exit;
+	}
+
+	addr = command + strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_SET_AEK]) + 1;
+	aek = addr + ETH_ALEN;
+
+	RTW_PRINT(FUNC_NDEV_FMT" addr="MAC_FMT"\n"
+		, FUNC_NDEV_ARG(ndev), MAC_ARG(addr));
+	if (0)
+		RTW_PRINT(FUNC_NDEV_FMT" aek="KEY_FMT KEY_FMT"\n"
+			, FUNC_NDEV_ARG(ndev), KEY_ARG(aek), KEY_ARG(aek + 16));
+
+	if (rtw_mesh_plink_set_aek(adapter, addr, aek) != _SUCCESS)
+		err = -ENOENT;
+
+exit:
+	return err;
+}
+#endif /* CONFIG_RTW_MESH_AEK */
+
+int rtw_android_priv_cmd(struct net_device *net, struct ifreq *ifr, int cmd)
+{
+	#define PRIVATE_COMMAND_MAX_LEN		65536
+	int ret = 0;
+	char *command = NULL;
+	int cmd_num;
+	int bytes_written = 0;
+	android_wifi_priv_cmd priv_cmd;
+	_adapter	*padapter = (_adapter *) rtw_netdev_priv(net);
+#ifdef CONFIG_WFD
+	struct wifi_display_info		*pwfd_info;
+#endif
+
+	rtw_lock_suspend();
+
+	if (!ifr->ifr_data) {
+		ret = -EINVAL;
+		goto exit;
+	}
+	if (padapter->registrypriv.mp_mode == 1) {
+		ret = -EINVAL;
+		goto exit;
+	}
+#ifdef CONFIG_COMPAT
+#if (KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE)
+	if (is_compat_task()) {
+#else
+	if (in_compat_syscall()) {
+#endif
+		/* User space is 32-bit, use compat ioctl */
+		compat_android_wifi_priv_cmd compat_priv_cmd;
+
+		if (copy_from_user(&compat_priv_cmd, ifr->ifr_data, sizeof(compat_android_wifi_priv_cmd))) {
+			ret = -EFAULT;
+			goto exit;
+		}
+		priv_cmd.buf = compat_ptr(compat_priv_cmd.buf);
+		priv_cmd.used_len = compat_priv_cmd.used_len;
+		priv_cmd.total_len = compat_priv_cmd.total_len;
+	} else
+#endif /* CONFIG_COMPAT */
+		if (copy_from_user(&priv_cmd, ifr->ifr_data, sizeof(android_wifi_priv_cmd))) {
+			ret = -EFAULT;
+			goto exit;
+		}
+	if (padapter->registrypriv.mp_mode == 1) {
+		ret = -EFAULT;
+		goto exit;
+	}
+	/*RTW_INFO("%s priv_cmd.buf=%p priv_cmd.total_len=%d  priv_cmd.used_len=%d\n",__func__,priv_cmd.buf,priv_cmd.total_len,priv_cmd.used_len);*/
+	if (priv_cmd.total_len > PRIVATE_COMMAND_MAX_LEN || priv_cmd.total_len < 0) {
+		RTW_WARN("%s: invalid private command (%d)\n", __FUNCTION__,
+			priv_cmd.total_len);
+		ret = -EFAULT;
+		goto exit;
+	}
+
+	command = rtw_zmalloc(priv_cmd.total_len+1);
+	if (!command) {
+		RTW_INFO("%s: failed to allocate memory\n", __FUNCTION__);
+		ret = -ENOMEM;
+		goto exit;
+	}
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0))
+	if (!access_ok(priv_cmd.buf, priv_cmd.total_len)) {
+	#else
+	if (!access_ok(VERIFY_READ, priv_cmd.buf, priv_cmd.total_len)) {
+	#endif
+		RTW_INFO("%s: failed to access memory\n", __FUNCTION__);
+		ret = -EFAULT;
+		goto exit;
+	}
+	if (copy_from_user(command, (void *)priv_cmd.buf, priv_cmd.total_len)) {
+		ret = -EFAULT;
+		goto exit;
+	}
+	command[priv_cmd.total_len] = '\0';
+	RTW_INFO("%s: Android private cmd \"%s\" on %s\n"
+		 , __FUNCTION__, command, ifr->ifr_name);
+
+	cmd_num = rtw_android_cmdstr_to_num(command);
+
+	switch (cmd_num) {
+	case ANDROID_WIFI_CMD_START:
+		/* bytes_written = wl_android_wifi_on(net); */
+		goto response;
+	case ANDROID_WIFI_CMD_SETFWPATH:
+		goto response;
+	}
+
+	if (!g_wifi_on) {
+		RTW_INFO("%s: Ignore private cmd \"%s\" - iface %s is down\n"
+			 , __FUNCTION__, command, ifr->ifr_name);
+		ret = 0;
+		goto exit;
+	}
+
+	if (!rtw_hw_chk_wl_func(adapter_to_dvobj(padapter), WL_FUNC_MIRACAST)) {
+		switch (cmd_num) {
+		case ANDROID_WIFI_CMD_WFD_ENABLE:
+		case ANDROID_WIFI_CMD_WFD_DISABLE:
+		case ANDROID_WIFI_CMD_WFD_SET_TCPPORT:
+		case ANDROID_WIFI_CMD_WFD_SET_MAX_TPUT:
+		case ANDROID_WIFI_CMD_WFD_SET_DEVTYPE:
+			goto response;
+		}
+	}
+
+	switch (cmd_num) {
+
+	case ANDROID_WIFI_CMD_STOP:
+		/* bytes_written = wl_android_wifi_off(net); */
+		break;
+
+	case ANDROID_WIFI_CMD_SCAN_ACTIVE:
+		/* rtw_set_scan_mode((_adapter *)rtw_netdev_priv(net), SCAN_ACTIVE); */
+		break;
+	case ANDROID_WIFI_CMD_SCAN_PASSIVE:
+		/* rtw_set_scan_mode((_adapter *)rtw_netdev_priv(net), SCAN_PASSIVE); */
+		break;
+
+	case ANDROID_WIFI_CMD_RSSI:
+		bytes_written = rtw_android_get_rssi(net, command, priv_cmd.total_len);
+		break;
+	case ANDROID_WIFI_CMD_LINKSPEED:
+		bytes_written = rtw_android_get_link_speed(net, command, priv_cmd.total_len);
+		break;
+
+	case ANDROID_WIFI_CMD_MACADDR:
+		bytes_written = rtw_android_get_macaddr(net, command, priv_cmd.total_len);
+		break;
+
+	case ANDROID_WIFI_CMD_BLOCK_SCAN:
+		bytes_written = rtw_android_set_block_scan(net, command, priv_cmd.total_len);
+		break;
+
+	case ANDROID_WIFI_CMD_BLOCK:
+		bytes_written = rtw_android_set_block(net, command, priv_cmd.total_len);
+		break;
+
+	case ANDROID_WIFI_CMD_RXFILTER_START:
+		/* bytes_written = net_os_set_packet_filter(net, 1); */
+		break;
+	case ANDROID_WIFI_CMD_RXFILTER_STOP:
+		/* bytes_written = net_os_set_packet_filter(net, 0); */
+		break;
+	case ANDROID_WIFI_CMD_RXFILTER_ADD:
+		/* int filter_num = *(command + strlen(CMD_RXFILTER_ADD) + 1) - '0'; */
+		/* bytes_written = net_os_rxfilter_add_remove(net, TRUE, filter_num); */
+		break;
+	case ANDROID_WIFI_CMD_RXFILTER_REMOVE:
+		/* int filter_num = *(command + strlen(CMD_RXFILTER_REMOVE) + 1) - '0'; */
+		/* bytes_written = net_os_rxfilter_add_remove(net, FALSE, filter_num); */
+		break;
+
+	case ANDROID_WIFI_CMD_BTCOEXSCAN_START:
+		/* TBD: BTCOEXSCAN-START */
+		break;
+	case ANDROID_WIFI_CMD_BTCOEXSCAN_STOP:
+		/* TBD: BTCOEXSCAN-STOP */
+		break;
+	case ANDROID_WIFI_CMD_BTCOEXMODE:
+#if 0
+		uint mode = *(command + strlen(CMD_BTCOEXMODE) + 1) - '0';
+		if (mode == 1)
+			net_os_set_packet_filter(net, 0); /* DHCP starts */
+		else
+			net_os_set_packet_filter(net, 1); /* DHCP ends */
+#ifdef WL_CFG80211
+		bytes_written = wl_cfg80211_set_btcoex_dhcp(net, command);
+#endif
+#endif
+		break;
+
+	case ANDROID_WIFI_CMD_SETSUSPENDMODE:
+		break;
+
+	case ANDROID_WIFI_CMD_SETSUSPENDOPT:
+		/* bytes_written = wl_android_set_suspendopt(net, command, priv_cmd.total_len); */
+		break;
+
+	case ANDROID_WIFI_CMD_SETBAND:
+		bytes_written = rtw_android_setband(net, command, priv_cmd.total_len);
+		break;
+
+	case ANDROID_WIFI_CMD_GETBAND:
+		bytes_written = rtw_android_getband(net, command, priv_cmd.total_len);
+		break;
+
+	case ANDROID_WIFI_CMD_COUNTRY:
+		bytes_written = rtw_android_set_country(net, command, priv_cmd.total_len);
+		break;
+
+#ifdef CONFIG_PNO_SUPPORT
+	case ANDROID_WIFI_CMD_PNOSSIDCLR_SET:
+		break;
+	case ANDROID_WIFI_CMD_PNOSETUP_SET:
+		break;
+	case ANDROID_WIFI_CMD_PNOENABLE_SET:
+		break;
+#endif
+
+	case ANDROID_WIFI_CMD_P2P_DEV_ADDR:
+		bytes_written = rtw_android_get_p2p_dev_addr(net, command, priv_cmd.total_len);
+		break;
+	case ANDROID_WIFI_CMD_P2P_SET_NOA:
+		/* int skip = strlen(CMD_P2P_SET_NOA) + 1; */
+		/* bytes_written = wl_cfg80211_set_p2p_noa(net, command + skip, priv_cmd.total_len - skip); */
+		break;
+	case ANDROID_WIFI_CMD_P2P_GET_NOA:
+		/* bytes_written = wl_cfg80211_get_p2p_noa(net, command, priv_cmd.total_len); */
+		break;
+	case ANDROID_WIFI_CMD_P2P_SET_PS:
+		/* int skip = strlen(CMD_P2P_SET_PS) + 1; */
+		/* bytes_written = wl_cfg80211_set_p2p_ps(net, command + skip, priv_cmd.total_len - skip); */
+		break;
+
+#ifdef CONFIG_IOCTL_CFG80211
+	case ANDROID_WIFI_CMD_SET_AP_WPS_P2P_IE: {
+		int skip = strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_SET_AP_WPS_P2P_IE]) + 3;
+		bytes_written = rtw_cfg80211_set_mgnt_wpsp2pie(net, command + skip, priv_cmd.total_len - skip, *(command + skip - 2) - '0');
+
+		adapter_to_dvobj(padapter)->wpas_type = RTW_WPAS_ANDROID;
+		break;
+	}
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+#ifdef CONFIG_WFD
+
+	case ANDROID_WIFI_CMD_MIRACAST:
+		bytes_written = rtw_android_set_miracast_mode(net, command, priv_cmd.total_len);
+		break;
+
+	case ANDROID_WIFI_CMD_WFD_ENABLE: {
+		/*	Commented by Albert 2012/07/24 */
+		/*	We can enable the WFD function by using the following command: */
+		/*	wpa_cli driver wfd-enable */
+
+		rtw_wfd_enable(padapter, 1);
+		break;
+	}
+
+	case ANDROID_WIFI_CMD_WFD_DISABLE: {
+		/*	Commented by Albert 2012/07/24 */
+		/*	We can disable the WFD function by using the following command: */
+		/*	wpa_cli driver wfd-disable */
+
+		rtw_wfd_enable(padapter, 0);
+		break;
+	}
+	case ANDROID_WIFI_CMD_WFD_SET_TCPPORT: {
+		/*	Commented by Albert 2012/07/24 */
+		/*	We can set the tcp port number by using the following command: */
+		/*	wpa_cli driver wfd-set-tcpport = 554 */
+
+		rtw_wfd_set_ctrl_port(padapter, (u16)get_int_from_command(command));
+		break;
+	}
+	case ANDROID_WIFI_CMD_WFD_SET_MAX_TPUT: {
+		break;
+	}
+	case ANDROID_WIFI_CMD_WFD_SET_DEVTYPE: {
+		/*	Commented by Albert 2012/08/28 */
+		/*	Specify the WFD device type ( WFD source/primary sink ) */
+
+		pwfd_info = &padapter->wfd_info;
+
+		pwfd_info->wfd_device_type = (u8) get_int_from_command(command);
+		pwfd_info->wfd_device_type &= WFD_DEVINFO_DUAL;
+		break;
+	}
+#endif
+#if CONFIG_RTW_MACADDR_ACL
+	case ANDROID_WIFI_CMD_HOSTAPD_SET_MACADDR_ACL: {
+		rtw_set_macaddr_acl(padapter, RTW_ACL_PERIOD_BSS, get_int_from_command(command));
+		break;
+	}
+	case ANDROID_WIFI_CMD_HOSTAPD_ACL_ADD_STA: {
+		u8 addr[ETH_ALEN] = {0x00};
+		macstr2num(addr, command + strlen("HOSTAPD_ACL_ADD_STA") + 3);	/* 3 is space bar + "=" + space bar these 3 chars */
+		rtw_acl_add_sta(padapter, RTW_ACL_PERIOD_BSS, addr);
+		break;
+	}
+	case ANDROID_WIFI_CMD_HOSTAPD_ACL_REMOVE_STA: {
+		u8 addr[ETH_ALEN] = {0x00};
+		macstr2num(addr, command + strlen("HOSTAPD_ACL_REMOVE_STA") + 3);	/* 3 is space bar + "=" + space bar these 3 chars */
+		rtw_acl_remove_sta(padapter, RTW_ACL_PERIOD_BSS, addr);
+		break;
+	}
+#endif /* CONFIG_RTW_MACADDR_ACL */
+#if defined(CONFIG_GTK_OL) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0))
+	case ANDROID_WIFI_CMD_GTK_REKEY_OFFLOAD:
+		rtw_gtk_offload(net, (u8 *)command);
+		break;
+#endif /* CONFIG_GTK_OL		 */
+	case ANDROID_WIFI_CMD_P2P_DISABLE: {
+#ifdef CONFIG_P2P
+		/* This Android private command is no longer in use */
+		/* rtw_p2p_enable(padapter, P2P_ROLE_DISABLE); */
+#endif /* CONFIG_P2P */
+		break;
+	}
+
+#ifdef CONFIG_RTW_MESH_AEK
+	case ANDROID_WIFI_CMD_SET_AEK:
+		bytes_written = rtw_android_set_aek(net, command, priv_cmd.total_len);
+		break;
+#endif
+
+	case ANDROID_WIFI_CMD_EXT_AUTH_STATUS: {
+		rtw_set_external_auth_status(padapter,
+			command + strlen("EXT_AUTH_STATUS "),
+			priv_cmd.total_len - strlen("EXT_AUTH_STATUS "));
+		break;
+	}
+	case ANDROID_WIFI_CMD_DRIVERVERSION: {
+		bytes_written = strlen(DRIVERVERSION);
+		snprintf(command, bytes_written + 1, DRIVERVERSION);
+		break;
+	}
+#ifdef PRIVATE_R
+	case ANDROID_WIFI_CMD_ROKU_FIND_REMOTE: {
+		struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+		struct wifidirect_info	*pwdinfo = &(padapter->wdinfo);
+		u8 	num, cmdlen;
+		u8	*ptr,*remote_mac_addr;
+		int 	i;
+
+		cmdlen = strlen(android_wifi_cmd_str[ANDROID_WIFI_CMD_ROKU_FIND_REMOTE]);
+		num = *(command + cmdlen + 1) - '0';
+
+		if (num != 0) {
+			pwdinfo->num_of_remote = num;
+			remote_mac_addr = pwdinfo->remote_mac_address;
+			ptr = command + cmdlen + 3;
+
+			for (i = 0; i < num; i++) {
+				macstr2num(remote_mac_addr, ptr);
+				ptr += 18; /* skip space and go to next mac addr */
+				remote_mac_addr += 6;
+			}
+
+			set_find_remote_timer(pmlmeext, 1);
+		}
+		else
+			_cancel_timer_ex(&pmlmeext->find_remote_timer);
+		break;
+	}
+#endif
+	default:
+		RTW_INFO("Unknown PRIVATE command %s - ignored\n", command);
+		snprintf(command, 3, "OK");
+		bytes_written = strlen("OK");
+	}
+
+response:
+	if (bytes_written >= 0) {
+		if ((bytes_written == 0) && (priv_cmd.total_len > 0))
+			command[0] = '\0';
+		if (bytes_written >= priv_cmd.total_len) {
+			RTW_INFO("%s: bytes_written = %d\n", __FUNCTION__, bytes_written);
+			bytes_written = priv_cmd.total_len;
+		} else
+			bytes_written++;
+		priv_cmd.used_len = bytes_written;
+		if (copy_to_user((void *)priv_cmd.buf, command, bytes_written)) {
+			RTW_INFO("%s: failed to copy data to user buffer\n", __FUNCTION__);
+			ret = -EFAULT;
+		}
+	} else
+		ret = bytes_written;
+
+exit:
+	rtw_unlock_suspend();
+	if (command)
+		rtw_mfree(command, priv_cmd.total_len + 1);
+
+	return ret;
+}
+
+
+/**
+ * Functions for Android WiFi card detection
+ */
+#if defined(RTW_ENABLE_WIFI_CONTROL_FUNC)
+
+static int g_wifidev_registered = 0;
+static struct semaphore wifi_control_sem;
+static struct wifi_platform_data *wifi_control_data = NULL;
+static struct resource *wifi_irqres = NULL;
+
+static int wifi_add_dev(void);
+static void wifi_del_dev(void);
+
+int rtw_android_wifictrl_func_add(void)
+{
+	int ret = 0;
+	sema_init(&wifi_control_sem, 0);
+
+	ret = wifi_add_dev();
+	if (ret) {
+		RTW_INFO("%s: platform_driver_register failed\n", __FUNCTION__);
+		return ret;
+	}
+	g_wifidev_registered = 1;
+
+	/* Waiting callback after platform_driver_register is done or exit with error */
+	if (down_timeout(&wifi_control_sem,  msecs_to_jiffies(1000)) != 0) {
+		ret = -EINVAL;
+		RTW_INFO("%s: platform_driver_register timeout\n", __FUNCTION__);
+	}
+
+	return ret;
+}
+
+void rtw_android_wifictrl_func_del(void)
+{
+	if (g_wifidev_registered) {
+		wifi_del_dev();
+		g_wifidev_registered = 0;
+	}
+}
+
+void *wl_android_prealloc(int section, unsigned long size)
+{
+	void *alloc_ptr = NULL;
+	if (wifi_control_data && wifi_control_data->mem_prealloc) {
+		alloc_ptr = wifi_control_data->mem_prealloc(section, size);
+		if (alloc_ptr) {
+			RTW_INFO("success alloc section %d\n", section);
+			if (size != 0L)
+				memset(alloc_ptr, 0, size);
+			return alloc_ptr;
+		}
+	}
+
+	RTW_INFO("can't alloc section %d\n", section);
+	return NULL;
+}
+
+int wifi_get_irq_number(unsigned long *irq_flags_ptr)
+{
+	if (wifi_irqres) {
+		*irq_flags_ptr = wifi_irqres->flags & IRQF_TRIGGER_MASK;
+		return (int)wifi_irqres->start;
+	}
+#ifdef CUSTOM_OOB_GPIO_NUM
+	return CUSTOM_OOB_GPIO_NUM;
+#else
+	return -1;
+#endif
+}
+
+int wifi_set_power(int on, unsigned long msec)
+{
+	RTW_INFO("%s = %d\n", __FUNCTION__, on);
+	if (wifi_control_data && wifi_control_data->set_power)
+		wifi_control_data->set_power(on);
+	if (msec)
+		msleep(msec);
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+int wifi_get_mac_addr(unsigned char *buf)
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+	if (!buf)
+		return -EINVAL;
+	if (wifi_control_data && wifi_control_data->get_mac_addr)
+		return wifi_control_data->get_mac_addr(buf);
+	return -EOPNOTSUPP;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35)) */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)) || defined(COMPAT_KERNEL_RELEASE)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0))
+void *wifi_get_country_code(char *ccode, u32 flags)
+#else /* Linux kernel < 3.18 */
+void *wifi_get_country_code(char *ccode)
+#endif /* Linux kernel < 3.18 */
+{
+	RTW_INFO("%s\n", __FUNCTION__);
+	if (!ccode)
+		return NULL;
+	if (wifi_control_data && wifi_control_data->get_country_code)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0))
+		return wifi_control_data->get_country_code(ccode, flags);
+#else /* Linux kernel < 3.18 */
+		return wifi_control_data->get_country_code(ccode);
+#endif /* Linux kernel < 3.18 */
+	return NULL;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)) */
+
+static int wifi_set_carddetect(int on)
+{
+	RTW_INFO("%s = %d\n", __FUNCTION__, on);
+	if (wifi_control_data && wifi_control_data->set_carddetect)
+		wifi_control_data->set_carddetect(on);
+	return 0;
+}
+
+static int wifi_probe(struct platform_device *pdev)
+{
+	struct wifi_platform_data *wifi_ctrl =
+		(struct wifi_platform_data *)(pdev->dev.platform_data);
+	int wifi_wake_gpio = 0;
+
+	RTW_INFO("## %s\n", __FUNCTION__);
+	wifi_irqres = platform_get_resource_byname(pdev, IORESOURCE_IRQ, "bcmdhd_wlan_irq");
+
+	if (wifi_irqres == NULL)
+		wifi_irqres = platform_get_resource_byname(pdev,
+				IORESOURCE_IRQ, "bcm4329_wlan_irq");
+	else
+		wifi_wake_gpio = wifi_irqres->start;
+
+#ifdef CONFIG_GPIO_WAKEUP
+	RTW_INFO("%s: gpio:%d wifi_wake_gpio:%d\n", __func__,
+	       (int)wifi_irqres->start, wifi_wake_gpio);
+
+	if (wifi_wake_gpio > 0) {
+		gpio_request(wifi_wake_gpio, "oob_irq");
+		gpio_direction_input(wifi_wake_gpio);
+		oob_irq = gpio_to_irq(wifi_wake_gpio);
+		RTW_INFO("%s oob_irq:%d\n", __func__, oob_irq);
+	} else if (wifi_irqres) {
+		oob_irq = wifi_irqres->start;
+		RTW_INFO("%s oob_irq:%d\n", __func__, oob_irq);
+	}
+
+	platform_wifi_get_oob_irq(&oob_irq);
+#endif
+	wifi_control_data = wifi_ctrl;
+
+	wifi_set_power(1, 0);	/* Power On */
+	wifi_set_carddetect(1);	/* CardDetect (0->1) */
+
+	up(&wifi_control_sem);
+	return 0;
+}
+
+#ifdef RTW_SUPPORT_PLATFORM_SHUTDOWN
+extern _adapter * g_test_adapter;
+
+static void shutdown_card(void)
+{
+	u32 addr;
+	u8 tmp8, cnt = 0;
+
+	if (NULL == g_test_adapter) {
+		RTW_INFO("%s: padapter==NULL\n", __FUNCTION__);
+		return;
+	}
+
+#ifdef CONFIG_WOWLAN
+#ifdef CONFIG_GPIO_WAKEUP
+	/*default wake up pin change to BT*/
+	RTW_INFO("%s:default wake up pin change to BT\n", __FUNCTION__);
+	/* ToDo: clear pin mux code is not ready
+	rtw_hal_switch_gpio_wl_ctrl(g_test_adapter, WAKEUP_GPIO_IDX, _FALSE); */
+#endif /* CONFIG_GPIO_WAKEUP */
+#endif /* CONFIG_WOWLAN */
+
+	/* Leave SDIO HCI Suspend */
+	#if 0 /*GEORGIA_TODO_REDEFINE_IO*/
+	addr = 0x10250086;
+	rtw_write8(g_test_adapter, addr, 0);
+	do {
+
+		tmp8 = rtw_read8(g_test_adapter, addr);
+		cnt++;
+		RTW_INFO(FUNC_ADPT_FMT ": polling SDIO_HSUS_CTRL(0x%x)=0x%x, cnt=%d\n",
+			 FUNC_ADPT_ARG(g_test_adapter), addr, tmp8, cnt);
+
+		if (tmp8 & BIT(1))
+			break;
+
+		if (cnt >= 100) {
+			RTW_INFO(FUNC_ADPT_FMT ": polling 0x%x[1]==1 FAIL!!\n",
+				 FUNC_ADPT_ARG(g_test_adapter), addr);
+			break;
+		}
+
+		rtw_mdelay_os(10);
+	} while (1);
+
+	/* unlock register I/O */
+	rtw_write8(g_test_adapter, 0x1C, 0);
+
+	/* enable power down function */
+	/* 0x04[4] = 1 */
+	/* 0x05[7] = 1 */
+	addr = 0x04;
+	tmp8 = rtw_read8(g_test_adapter, addr);
+	tmp8 |= BIT(4);
+	rtw_write8(g_test_adapter, addr, tmp8);
+	RTW_INFO(FUNC_ADPT_FMT ": read after write 0x%x=0x%x\n",
+		FUNC_ADPT_ARG(g_test_adapter), addr, rtw_read8(g_test_adapter, addr));
+
+	addr = 0x05;
+	tmp8 = rtw_read8(g_test_adapter, addr);
+	tmp8 |= BIT(7);
+	rtw_write8(g_test_adapter, addr, tmp8);
+	RTW_INFO(FUNC_ADPT_FMT ": read after write 0x%x=0x%x\n",
+		FUNC_ADPT_ARG(g_test_adapter), addr, rtw_read8(g_test_adapter, addr));
+
+	/* lock register page0 0x0~0xB read/write */
+	rtw_write8(g_test_adapter, 0x1C, 0x0E);
+#else
+	rtw_hal_sdio_leave_suspend(g_test_adapter);
+#endif
+	dev_set_surprise_removed(adapter_to_dvobj(g_test_adapter));
+	RTW_INFO(FUNC_ADPT_FMT ": bSurpriseRemoved=%s\n",
+		FUNC_ADPT_ARG(g_test_adapter),
+		dev_is_surprise_removed(adapter_to_dvobj(g_test_adapter)) ? "True" : "False");
+}
+#endif /* RTW_SUPPORT_PLATFORM_SHUTDOWN */
+
+static int wifi_remove(struct platform_device *pdev)
+{
+	struct wifi_platform_data *wifi_ctrl =
+		(struct wifi_platform_data *)(pdev->dev.platform_data);
+
+	RTW_INFO("## %s\n", __FUNCTION__);
+	wifi_control_data = wifi_ctrl;
+
+	wifi_set_power(0, 0);	/* Power Off */
+	wifi_set_carddetect(0);	/* CardDetect (1->0) */
+
+	up(&wifi_control_sem);
+	return 0;
+}
+
+#ifdef RTW_SUPPORT_PLATFORM_SHUTDOWN
+static void wifi_shutdown(struct platform_device *pdev)
+{
+	struct wifi_platform_data *wifi_ctrl =
+		(struct wifi_platform_data *)(pdev->dev.platform_data);
+
+
+	RTW_INFO("## %s\n", __FUNCTION__);
+
+	wifi_control_data = wifi_ctrl;
+
+	shutdown_card();
+	wifi_set_power(0, 0);	/* Power Off */
+	wifi_set_carddetect(0);	/* CardDetect (1->0) */
+}
+#endif /* RTW_SUPPORT_PLATFORM_SHUTDOWN */
+
+static int wifi_suspend(struct platform_device *pdev, pm_message_t state)
+{
+	RTW_INFO("##> %s\n", __FUNCTION__);
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 39)) && defined(OOB_INTR_ONLY)
+	bcmsdh_oob_intr_set(0);
+#endif
+	return 0;
+}
+
+static int wifi_resume(struct platform_device *pdev)
+{
+	RTW_INFO("##> %s\n", __FUNCTION__);
+#if (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 39)) && defined(OOB_INTR_ONLY)
+	if (dhd_os_check_if_up(bcmsdh_get_drvdata()))
+		bcmsdh_oob_intr_set(1);
+#endif
+	return 0;
+}
+
+/* temporarily use these two */
+static struct platform_driver wifi_device = {
+	.probe          = wifi_probe,
+	.remove         = wifi_remove,
+	.suspend        = wifi_suspend,
+	.resume         = wifi_resume,
+#ifdef RTW_SUPPORT_PLATFORM_SHUTDOWN
+	.shutdown       = wifi_shutdown,
+#endif /* RTW_SUPPORT_PLATFORM_SHUTDOWN */
+	.driver         = {
+		.name   = "bcmdhd_wlan",
+	}
+};
+
+static struct platform_driver wifi_device_legacy = {
+	.probe          = wifi_probe,
+	.remove         = wifi_remove,
+	.suspend        = wifi_suspend,
+	.resume         = wifi_resume,
+	.driver         = {
+		.name   = "bcm4329_wlan",
+	}
+};
+
+static int wifi_add_dev(void)
+{
+	RTW_INFO("## Calling platform_driver_register\n");
+	platform_driver_register(&wifi_device);
+	platform_driver_register(&wifi_device_legacy);
+	return 0;
+}
+
+static void wifi_del_dev(void)
+{
+	RTW_INFO("## Unregister platform_driver_register\n");
+	platform_driver_unregister(&wifi_device);
+	platform_driver_unregister(&wifi_device_legacy);
+}
+#endif /* defined(RTW_ENABLE_WIFI_CONTROL_FUNC) */
+
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfg.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfg.c
new file mode 100644
index 000000000000..795cb6028ad7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfg.c
@@ -0,0 +1,2269 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTW_CFG_C_
+
+#include <drv_types.h>
+
+/* module param defaults */
+int rtw_chip_version = 0x00;
+int rtw_rfintfs = HWPI;
+int rtw_lbkmode = 0;/* RTL8712_AIR_TRX; */
+#ifdef DBG_LA_MODE
+int rtw_la_mode_en=0;
+module_param(rtw_la_mode_en, int, 0644);
+#endif
+int rtw_network_mode = Ndis802_11IBSS;/* Ndis802_11Infrastructure; */ /* infra, ad-hoc, auto */
+/* NDIS_802_11_SSID	ssid; */
+int rtw_band = BAND_ON_5G;
+int rtw_channel = 36;/* ad-hoc support requirement */
+int rtw_wireless_mode = WLAN_MD_MAX;
+int rtw_band_type = BAND_CAP_2G | BAND_CAP_5G | BAND_CAP_6G;
+module_param(rtw_wireless_mode, int, 0644);
+module_param(rtw_band_type, int, 0644);
+#ifdef CONFIG_HW_RTS
+int rtw_vrtl_carrier_sense = ENABLE_VCS;
+int rtw_vcs_type = RTS_CTS;
+int rtw_hw_rts_en = 1;
+#else
+int rtw_vrtl_carrier_sense = AUTO_VCS;
+int rtw_vcs_type = RTS_CTS;
+int rtw_hw_rts_en = 0;
+#endif
+int rtw_rts_thresh = 2347;
+int rtw_frag_thresh = 2346;
+int rtw_preamble = PREAMBLE_LONG;/* long, short, auto */
+int rtw_scan_mode = 1;/* active, passive */
+
+#ifdef CONFIG_SCANOFLD
+int rtw_scan_fw_ofld = 1;/* 0:drv scan, 1:fw scan */
+module_param(rtw_scan_fw_ofld, int, 0644);
+#endif
+
+#ifdef CONFIG_POWER_SAVE
+#ifdef CONFIG_RTW_IPS
+int rtw_ips_mode = PS_IPS_MAX;
+module_param(rtw_ips_mode, int, 0644);
+MODULE_PARM_DESC(rtw_ips_mode, "The default IPS mode");
+#endif /* CONFIG_RTW_IPS */
+
+#ifdef CONFIG_RTW_LPS
+int rtw_lps_mode = PS_LPS_MAX;
+module_param(rtw_lps_mode, int, 0644);
+MODULE_PARM_DESC(rtw_lps_mode, "The default LPS mode");
+
+int rtw_lps_listen_bcn_mode = RTW_LPS_LISTEN_BCN_MAX;
+module_param(rtw_lps_listen_bcn_mode, int, 0644);
+MODULE_PARM_DESC(rtw_lps_listen_bcn_mode, "The default lps dtim mode");
+
+int rtw_lps_awake_interval = 0;
+module_param(rtw_lps_awake_interval, int, 0644);
+MODULE_PARM_DESC(rtw_lps_awake_interval, "The default LPS awake interval");
+#endif /* CONFIG_RTW_LPS */
+
+#ifdef CONFIG_WOWLAN
+#ifdef CONFIG_RTW_IPS_WOW
+int rtw_ips_wow_mode = PS_IPS_MAX;
+module_param(rtw_ips_wow_mode, int, 0644);
+MODULE_PARM_DESC(rtw_ips_wow_mode, "The default IPS mode on wowlan mode");
+#endif /* CONFIG_RTW_IPS_WOW */
+
+#ifdef CONFIG_RTW_LPS_WOW
+int rtw_lps_wow_mode = PS_LPS_MAX;
+module_param(rtw_lps_wow_mode, int, 0644);
+MODULE_PARM_DESC(rtw_lps_wow_mode, "The default LPS mode on wowlan mode");
+
+int rtw_lps_wow_listen_bcn_mode = RTW_LPS_LISTEN_BCN_MAX;
+module_param(rtw_lps_wow_listen_bcn_mode, int, 0644);
+MODULE_PARM_DESC(rtw_lps_wow_listen_bcn_mode, "The default dtim on wowlan mode");
+
+int rtw_lps_wow_awake_interval = 0;
+module_param(rtw_lps_wow_awake_interval, int, 0644);
+MODULE_PARM_DESC(rtw_lps_wow_awake_interval,
+		 "The default LPS awake interval on wowlan mode");
+#endif /* CONFIG_RTW_LPS_WOW */
+#endif /* CONFIG_WOWLAN */
+#endif /* CONFIG_POWER_SAVE */
+
+#ifdef CONFIG_NARROWBAND_SUPPORTING
+int rtw_nb_config = CONFIG_NB_VALUE;
+module_param(rtw_nb_config, int, 0644);
+MODULE_PARM_DESC(rtw_nb_config, "5M/10M/Normal bandwidth configuration");
+#endif
+
+int rtw_max_bss_cnt = 0;
+module_param(rtw_max_bss_cnt, int, 0644);
+
+#ifdef CONFIG_TX_EARLY_MODE
+int rtw_early_mode = 1;
+#endif
+
+int rtw_usb_rxagg_mode = 2;/* RX_AGG_DMA=1, RX_AGG_USB=2 */
+module_param(rtw_usb_rxagg_mode, int, 0644);
+
+int rtw_dynamic_agg_enable = 1;
+module_param(rtw_dynamic_agg_enable, int, 0644);
+
+/* set log level when inserting driver module, default log level is _DRV_INFO_ = 4,
+* please refer to "How_to_set_driver_debug_log_level.doc" to set the available level.
+*/
+#ifdef CONFIG_RTW_DEBUG
+#ifdef RTW_LOG_LEVEL
+	uint rtw_drv_log_level = (uint)RTW_LOG_LEVEL; /* from Makefile */
+#else
+	uint rtw_drv_log_level = _DRV_INFO_;
+#endif
+module_param(rtw_drv_log_level, uint, 0644);
+MODULE_PARM_DESC(rtw_drv_log_level, "set log level when insert driver module, default log level is _DRV_INFO_ = 4");
+#ifdef RTW_PHL_LOG_LEVEL
+	uint rtw_phl_log_level = (uint)RTW_PHL_LOG_LEVEL; /* from Makefile */
+#else
+	uint rtw_phl_log_level = _PHL_INFO_;
+#endif
+module_param(rtw_phl_log_level, uint, 0644);
+MODULE_PARM_DESC(rtw_phl_log_level, "set phl log level when insert driver module, default log level is _PHL_INFO_ = 4");
+#endif
+int rtw_radio_enable = 1;
+int rtw_long_retry_lmt = 7;
+int rtw_short_retry_lmt = 7;
+int rtw_busy_thresh = 40;
+/* int qos_enable = 0; */ /* * */
+int rtw_ack_policy = NORMAL_ACK;
+
+int rtw_mp_mode = 0;
+
+#if defined(CONFIG_MP_INCLUDED) && defined(CONFIG_RTW_CUSTOMER_STR)
+uint rtw_mp_customer_str = 0;
+module_param(rtw_mp_customer_str, uint, 0644);
+MODULE_PARM_DESC(rtw_mp_customer_str, "Whether or not to enable customer str support on MP mode");
+#endif
+
+int rtw_software_encrypt = 0;
+int rtw_software_decrypt = 0;
+
+int rtw_acm_method = 0;/* 0:By SW 1:By HW. */
+
+int rtw_wmm_enable = 1;/* default is set to enable the wmm. */
+
+#if defined(CONFIG_RTL8821C) || defined(CONFIG_RTL8822B) || defined(CONFIG_RTL8822C)
+	/*PHYDM API, must enable by default*/
+	int rtw_pwrtrim_enable = 1;
+#else
+	int rtw_pwrtrim_enable = 0; /* Default Enalbe  power trim by efuse config */
+#endif
+
+uint rtw_tx_bw_mode = 0x331;
+module_param(rtw_tx_bw_mode, uint, 0644);
+MODULE_PARM_DESC(rtw_tx_bw_mode, "The max tx bw for 2G/5G/6G. format is the same as rtw_bw_mode");
+
+
+#ifdef CONFIG_80211N_HT
+int rtw_ht_enable = 1;
+/* 0: 20 MHz, 1: 40 MHz, 2: 80 MHz, 3: 160MHz, 4: 80+80MHz
+* 2.4G use bit 0 ~ 3, 5G use bit 4 ~ 7
+* 6GHz use bits 8 ~ 11
+* 0x21 means enable 2.4G 40MHz & 5G 80MHz */
+#ifdef CONFIG_RTW_CUSTOMIZE_BWMODE
+int rtw_bw_mode = CONFIG_RTW_CUSTOMIZE_BWMODE;
+#else
+int rtw_bw_mode = 0x331;
+#endif
+
+int rtw_ampdu_enable = 1;/* for enable tx_ampdu , */ /* 0: disable, 0x1:enable */
+
+int rtw_rx_ampdu_amsdu = 2;/* 0: disabled, 1:enabled, 2:auto . There is an IOT issu with DLINK DIR-629 when the flag turn on */
+
+/* 10.12 A-MSDU operation
+ * HT -   0: 3839, 1: 7935  octets - Maximum A-MSDU Length
+ * VHT - 0: 3895, 1: 7991, 2:11454  octets - Maximum MPDU Length
+ */
+
+#if defined(CONFIG_RTW_REDUCE_MEM) && defined(CONFIG_PCI_HCI)
+int rtw_max_amsdu_len = MAX_ASMDU_LEN;
+#else
+int rtw_max_amsdu_len = 1;
+#endif
+
+module_param(rtw_max_amsdu_len, uint, 0644);
+
+/*
+* 2: Follow the AMSDU filed in ADDBA Resp. (Deault)
+* 0: Force the AMSDU filed in ADDBA Resp. to be disabled.
+* 1: Force the AMSDU filed in ADDBA Resp. to be enabled.
+*/
+int rtw_tx_ampdu_amsdu = 2;
+
+/* 0: Default by halmac provided. */
+/* other: Setting by driver. The final decision will take a samller value from halmac or driver. */
+int rtw_tx_ampdu_num = 0;
+module_param(rtw_tx_ampdu_num, int, 0644);
+
+int rtw_quick_addba_req = 1;
+
+static uint rtw_rx_ampdu_sz_limit_1ss[4] = CONFIG_RTW_RX_AMPDU_SZ_LIMIT_1SS;
+static uint rtw_rx_ampdu_sz_limit_1ss_num = 0;
+module_param_array(rtw_rx_ampdu_sz_limit_1ss, uint, &rtw_rx_ampdu_sz_limit_1ss_num, 0644);
+MODULE_PARM_DESC(rtw_rx_ampdu_sz_limit_1ss, "RX AMPDU size limit for 1SS link of each BW, 0xFF: no limitation");
+
+static uint rtw_rx_ampdu_sz_limit_2ss[4] = CONFIG_RTW_RX_AMPDU_SZ_LIMIT_2SS;
+static uint rtw_rx_ampdu_sz_limit_2ss_num = 0;
+module_param_array(rtw_rx_ampdu_sz_limit_2ss, uint, &rtw_rx_ampdu_sz_limit_2ss_num, 0644);
+MODULE_PARM_DESC(rtw_rx_ampdu_sz_limit_2ss, "RX AMPDU size limit for 2SS link of each BW, 0xFF: no limitation");
+
+static uint rtw_rx_ampdu_sz_limit_3ss[4] = CONFIG_RTW_RX_AMPDU_SZ_LIMIT_3SS;
+static uint rtw_rx_ampdu_sz_limit_3ss_num = 0;
+module_param_array(rtw_rx_ampdu_sz_limit_3ss, uint, &rtw_rx_ampdu_sz_limit_3ss_num, 0644);
+MODULE_PARM_DESC(rtw_rx_ampdu_sz_limit_3ss, "RX AMPDU size limit for 3SS link of each BW, 0xFF: no limitation");
+
+static uint rtw_rx_ampdu_sz_limit_4ss[4] = CONFIG_RTW_RX_AMPDU_SZ_LIMIT_4SS;
+static uint rtw_rx_ampdu_sz_limit_4ss_num = 0;
+module_param_array(rtw_rx_ampdu_sz_limit_4ss, uint, &rtw_rx_ampdu_sz_limit_4ss_num, 0644);
+MODULE_PARM_DESC(rtw_rx_ampdu_sz_limit_4ss, "RX AMPDU size limit for 4SS link of each BW, 0xFF: no limitation");
+
+/* Short GI support Bit Map
+* BIT0 - 20MHz, 0: non-support, 1: support
+* BIT1 - 40MHz, 0: non-support, 1: support
+* BIT2 - 80MHz, 0: non-support, 1: support
+* BIT3 - 160MHz, 0: non-support, 1: support */
+int rtw_short_gi = 0xf;
+/* BIT0: Enable VHT LDPC Rx, BIT1: Enable VHT LDPC Tx
+ * BIT2: Enable HE LDPC Rx, BIT3: Enable HE LDPC Tx
+ * BIT4: Enable HT LDPC Rx, BIT5: Enable HT LDPC Tx
+ */
+int rtw_ldpc_cap = 0x3f;
+/* BIT0: Enable VHT STBC Rx, BIT1: Enable VHT STBC Tx
+* BIT4: Enable HT STBC Rx, BIT5: Enable HT STBC Tx
+* BIT8: Enable HE STBC Rx, BIT9: Enable HE STBC Rx(greater than 80M)
+* BIT10: Enable HE STBC Tx, BIT11: Enable HE STBC Tx(greater than 80M)
+*/
+int rtw_stbc_cap = 0x133;
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_BEAMFORMING
+/*
+* BIT0: Enable VHT SU Beamformer
+* BIT1: Enable VHT SU Beamformee
+* BIT2: Enable VHT MU Beamformer, depend on VHT SU Beamformer
+* BIT3: Enable VHT MU Beamformee, depend on VHT SU Beamformee
+* BIT4: Enable HT Beamformer
+* BIT5: Enable HT Beamformee
+* BIT6: Enable HE SU Beamformer
+* BIT7: Enable HE SU Beamformee
+* BIT8: Enable HE MU Beamformer
+* BIT9: Enable HE MU Beamformee
+*/
+#ifndef PRIVATE_R
+int rtw_beamform_cap = BIT(1) | BIT(7);  /* For sw role BF cap. */
+int rtw_sw_proto_bf_cap_phy0 = BIT(1) | BIT(7);
+int rtw_sw_proto_bf_cap_phy1 = BIT(1) | BIT(7);
+#else
+int rtw_beamform_cap = BIT(1) | BIT(3) | BIT(7) | BIT(9);  /* For sw role BF cap. */
+int rtw_sw_proto_bf_cap_phy0 = BIT(1) | BIT(3) | BIT(7) | BIT(9);
+int rtw_sw_proto_bf_cap_phy1 = BIT(1) | BIT(3) | BIT(7) | BIT(9);
+#endif /* PRIVATE_R */
+int rtw_dyn_txbf = 1;
+int rtw_bfer_rf_number = 0; /*BeamformerCapRfNum Rf path number, 0 for auto, others for manual*/
+int rtw_bfee_rf_number = 0; /*BeamformeeCapRfNum  Rf path number, 0 for auto, others for manual*/
+#endif /* CONFIG_BEAMFORMING */
+
+#ifdef CONFIG_80211AC_VHT
+int rtw_vht_enable = 1; /* 0:disable, 1:enable, 2:force auto enable */
+module_param(rtw_vht_enable, int, 0644);
+
+int rtw_vht_24g_enable = 0; /* 0:disable, 1:enable */
+module_param(rtw_vht_24g_enable, int, 0644);
+
+int rtw_ampdu_factor = 7;
+
+uint rtw_vht_rx_mcs_map = 0xaaaa;
+module_param(rtw_vht_rx_mcs_map, uint, 0644);
+MODULE_PARM_DESC(rtw_vht_rx_mcs_map, "VHT RX MCS map");
+#endif /* CONFIG_80211AC_VHT */
+
+#ifdef CONFIG_80211AX_HE
+int rtw_he_enable = 1; /* 0:disable, 1:enable, 2:force auto enable */
+module_param(rtw_he_enable, int, 0644);
+#endif
+
+int rtw_lowrate_two_xmit = 1;/* Use 2 path Tx to transmit MCS0~7 and legacy mode */
+
+
+/* 0: not check in watch dog, 1: check in watch dog  */
+int rtw_check_hw_status = 0;
+
+int rtw_low_power = 0;
+int rtw_wifi_spec = 0;
+
+#ifdef CONFIG_SPECIAL_RF_PATH /* configure Nss/xTxR IC to 1ss/1T1R */
+int rtw_rf_path = RF_1T1R;
+int rtw_tx_nss = 1;
+int rtw_rx_nss = 1;
+#else
+int rtw_rf_path = RF_TYPE_MAX;
+int rtw_tx_nss = 0;
+int rtw_rx_nss = 0;
+#endif
+module_param(rtw_rf_path, int, 0644);
+module_param(rtw_tx_nss, int, 0644);
+module_param(rtw_rx_nss, int, 0644);
+
+#if defined(CONFIG_RTW_IO_RECORDS) && !CONFIG_RTW_IO_RECORDS_STATIC
+uint rtw_io_records_num = CONFIG_RTW_IO_RECORDS_NUM;
+module_param(rtw_io_records_num, uint, 0644);
+uint rtw_io_records_enable = CONFIG_RTW_IO_RECORDS_ENABLE;
+module_param(rtw_io_records_enable, uint, 0644);
+uint rtw_io_records_loop = CONFIG_RTW_IO_RECORDS_LOOP;
+module_param(rtw_io_records_loop, uint, 0644);
+#endif /* CONFIG_RTW_IO_RECORDS  && !CONFIG_RTW_IO_RECORDS_STATIC */
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+static uint rtw_regd_src = CONFIG_RTW_REGD_SRC;
+module_param(rtw_regd_src, uint, 0644);
+MODULE_PARM_DESC(rtw_regd_src, "The default regd source selection, 0:RTK_PRIV, 1:OS");
+
+static uint rtw_regd_src_os_11d = CONFIG_RTW_REGD_SRC_OS_11D;
+module_param(rtw_regd_src_os_11d, uint, 0644);
+MODULE_PARM_DESC(rtw_regd_src_os_11d, "If enable 802.11d when regd source is OS, 0:disable, 1:enable");
+#endif
+
+uint rtw_init_regd_always_apply = CONFIG_RTW_INIT_REGD_ALWAYS_APPLY;
+module_param(rtw_init_regd_always_apply, uint, 0644);
+MODULE_PARM_DESC(rtw_init_regd_always_apply, "Whether INIT regd request is always applied"
+	" (being included when taking intersection together with higher priority requests)"
+	" when regd source is RTK_PRIV");
+
+uint rtw_user_regd_always_apply = CONFIG_RTW_USER_REGD_ALWAYS_APPLY;
+module_param(rtw_user_regd_always_apply, uint, 0644);
+MODULE_PARM_DESC(rtw_user_regd_always_apply, "Whether USER regd request is always applied"
+	" (being included when taking intersection together with higher priority requests)"
+	" when regd source is RTK_PRIV");
+
+char *rtw_country_code = CONFIG_RTW_COUNTRY_CODE;
+module_param(rtw_country_code, charp, 0644);
+MODULE_PARM_DESC(rtw_country_code, "The default country code (in alpha2)");
+
+uint rtw_channel_plan = CONFIG_RTW_CHPLAN;
+module_param(rtw_channel_plan, uint, 0644);
+MODULE_PARM_DESC(rtw_channel_plan, "The default chplan ID when rtw_alpha2 is not specified or valid");
+
+static uint rtw_excl_chs[MAX_CHANNEL_NUM_2G_5G] = CONFIG_RTW_EXCL_CHS;
+static int rtw_excl_chs_num = 0;
+module_param_array(rtw_excl_chs, uint, &rtw_excl_chs_num, 0644);
+MODULE_PARM_DESC(rtw_excl_chs, "Exclusive channel list of 2G and 5G band");
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+uint rtw_split_scan_6ghz = 1; /* 0:disable, 1:enable */
+module_param(rtw_split_scan_6ghz, uint, 0644);
+MODULE_PARM_DESC(rtw_split_scan_6ghz, "Perform a separate scan operation for specific 6GHz channels to reduce overall scan time.");
+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) */
+
+uint rtw_channel_plan_6g = CONFIG_RTW_CHPLAN_6G;
+module_param(rtw_channel_plan_6g, uint, 0644);
+MODULE_PARM_DESC(rtw_channel_plan_6g, "The default chplan_6g ID when rtw_alpha2 is not specified or valid");
+
+static uint rtw_excl_chs_6g[MAX_CHANNEL_NUM_6G] = CONFIG_RTW_EXCL_CHS_6G;
+static int rtw_excl_chs_6g_num = 0;
+module_param_array(rtw_excl_chs_6g, uint, &rtw_excl_chs_6g_num, 0644);
+MODULE_PARM_DESC(rtw_excl_chs_6g, "Exclusive channel list of 6G band");
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+char *rtw_dis_ch_flags = CONFIG_RTW_DIS_CH_FLAGS;
+module_param(rtw_dis_ch_flags, charp, 0644);
+MODULE_PARM_DESC(rtw_dis_ch_flags, "The flags with which channel is to be disabled");
+
+static uint rtw_bcn_hint_valid_ms = CONFIG_RTW_BCN_HINT_VALID_MS;
+module_param(rtw_bcn_hint_valid_ms, uint, 0644);
+MODULE_PARM_DESC(rtw_bcn_hint_valid_ms, "The length of time beacon hint continue");
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static uint rtw_env = CONFIG_RTW_ENV;
+module_param(rtw_env, uint, 0644);
+MODULE_PARM_DESC(rtw_env, "The default environment setting:"
+	" 0: ANY, 1: INDOOR, 2: OUTDOOR");
+#endif
+
+#ifdef CONFIG_RTW_CSI_CHANNEL_INFO
+static uint rtw_sensing_csi = 1;
+module_param(rtw_sensing_csi, uint, 0644);
+MODULE_PARM_DESC(rtw_sensing_csi, "Enable FW sensing csi");
+#endif
+
+#ifdef RTW_WKARD_6G_RTS_CHK
+static uint rtw_rts_chk_off = 1;
+module_param(rtw_rts_chk_off, uint, 0644);
+MODULE_PARM_DESC(rtw_rts_chk_off, "Disable RTS CHK");
+#endif
+
+#ifdef CONFIG_80211D
+static uint rtw_country_ie_slave_en_mode = CONFIG_RTW_COUNTRY_IE_SLAVE_EN_MODE;
+module_param(rtw_country_ie_slave_en_mode, uint, 0644);
+MODULE_PARM_DESC(rtw_country_ie_slave_en_mode, "802.11d country IE slave enable mode:"
+	" 0: disable, 1: enable, 2: enable when INIT/USER set world wide mode");
+
+static uint rtw_country_ie_slave_flags = CONFIG_RTW_COUNTRY_IE_SLAVE_FLAGS;
+module_param(rtw_country_ie_slave_flags, uint, 0644);
+MODULE_PARM_DESC(rtw_country_ie_slave_flags, "802.11d country IE slave flags:"
+	" BIT0: deprecated BIT"
+	", BIT1: consider all environment BSSs, otherwise associated BSSs only");
+
+static uint rtw_country_ie_slave_en_role = CONFIG_RTW_COUNTRY_IE_SLAVE_EN_ROLE;
+module_param(rtw_country_ie_slave_en_role, uint, 0644);
+MODULE_PARM_DESC(rtw_country_ie_slave_en_role, "802.11d country IE slave enable role: BIT0:pure STA mode, BIT1:P2P group client");
+
+static uint rtw_country_ie_slave_en_ifbmp = CONFIG_RTW_COUNTRY_IE_SLAVE_EN_IFBMP;
+module_param(rtw_country_ie_slave_en_ifbmp, uint, 0644);
+MODULE_PARM_DESC(rtw_country_ie_slave_en_ifbmp, "802.11d country IE slave enable iface bitmap");
+
+static uint rtw_country_ie_slave_scan_int_ms = CONFIG_RTW_COUNTRY_IE_SLAVE_SCAN_INT_MS;
+module_param(rtw_country_ie_slave_scan_int_ms, uint, 0644);
+MODULE_PARM_DESC(rtw_country_ie_slave_scan_int_ms, "802.11d country IE slave auto scan interval in ms to find environment BSSs."
+	" 0: no environment BSS auto scan triggered by driver self");
+#endif
+
+/*if concurrent softap + p2p(GO) is needed, this param lets p2p response full channel list.
+But Softap must be SHUT DOWN once P2P decide to set up connection and become a GO.*/
+#ifdef CONFIG_FULL_CH_IN_P2P_HANDSHAKE
+	int rtw_full_ch_in_p2p_handshake = 1; /* reply full channel list*/
+#else
+	int rtw_full_ch_in_p2p_handshake = 0; /* reply only softap channel*/
+#endif
+
+#ifdef CONFIG_BTC
+
+#ifdef CONFIG_BTC_EXT_CFG_ESOC_TYPE
+int rtw_btc_ext_soc_type = CONFIG_BTC_EXT_CFG_ESOC_TYPE;
+module_param(rtw_btc_ext_soc_type, int, 0644);
+MODULE_PARM_DESC(rtw_btc_ext_soc_type, "BTC external soc type");
+#endif
+
+#ifdef CONFIG_BTC_EXT_CFG_ANT_ISO_DB
+int rtw_btc_ant_iso_db = CONFIG_BTC_EXT_CFG_ANT_ISO_DB;
+module_param(rtw_btc_ant_iso_db, int, 0644);
+MODULE_PARM_DESC(rtw_btc_ant_iso_db, "BTC antenna isolation value in unit dB");
+#endif
+
+#ifdef CONFIG_BTC_TRXSS_CHG
+int rtw_btc_trxss_chg = 0;
+module_param(rtw_btc_trxss_chg, int, 0644);
+MODULE_PARM_DESC(rtw_btc_trxss_chg, "Tx/Rx SS change triggered by BTC");
+#endif
+
+#endif /* CONFIG_BTC */
+
+int rtw_AcceptAddbaReq = _TRUE;/* 0:Reject AP's Add BA req, 1:Accept AP's Add BA req. */
+
+int rtw_antdiv_cfg = 2; /* 0:OFF , 1:ON, 2:decide by Efuse config */
+int rtw_antdiv_type = 0; /* 0:decide by efuse  1: for 88EE, 1Tx and 1RxCG are diversity.(2 Ant with SPDT), 2:  for 88EE, 1Tx and 2Rx are diversity.( 2 Ant, Tx and RxCG are both on aux port, RxCS is on main port ), 3: for 88EE, 1Tx and 1RxCG are fixed.(1Ant, Tx and RxCG are both on aux port) */
+
+int rtw_drv_ant_band_switch = 1; /* 0:OFF , 1:ON, Driver control antenna band switch*/
+
+/* 0: doesn't switch, 1: switch to usb 3.0 , 2: switch to usb 2.0 */
+int rtw_switch_usb_mode = 0;
+
+
+
+#ifdef CONFIG_HW_PWRP_DETECTION
+int rtw_hwpwrp_detect = 1;
+#else
+int rtw_hwpwrp_detect = 0; /* HW power  ping detect 0:disable , 1:enable */
+#endif
+
+#ifdef CONFIG_USB_HCI
+int rtw_hw_wps_pbc = 1;
+#else
+int rtw_hw_wps_pbc = 0;
+#endif
+
+#ifdef CONFIG_PCI_ASPM
+/* CLK_REQ:BIT0 L0s:BIT1 ASPM_L1:BIT2 L1Off:BIT3*/
+int	rtw_pci_aspm_enable = 0x5;
+#else
+int	rtw_pci_aspm_enable;
+#endif
+
+#ifdef CONFIG_QOS_OPTIMIZATION
+int rtw_qos_opt_enable = 1; /* 0: disable,1:enable */
+#else
+int rtw_qos_opt_enable = 0; /* 0: disable,1:enable */
+#endif
+module_param(rtw_qos_opt_enable, int, 0644);
+
+#if defined(CONFIG_PCI_HCI) && !defined(CONFIG_RTW_PCI_MSI_DISABLE)
+int rtw_msi_en = 1;
+module_param(rtw_msi_en, int, 0644);
+#endif
+
+#ifdef CONFIG_RTW_ACS
+int rtw_acs_auto_scan = 0; /*0:disable, 1:enable*/
+module_param(rtw_acs_auto_scan, int, 0644);
+
+int rtw_acs = 1;
+module_param(rtw_acs, int, 0644);
+#endif
+
+char *ifname = "wlan%d";
+module_param(ifname, charp, 0644);
+MODULE_PARM_DESC(ifname, "The default name to allocate for first interface");
+
+#ifdef CONFIG_PLATFORM_ANDROID
+	char *if2name = "p2p%d";
+#else /* CONFIG_PLATFORM_ANDROID */
+	char *if2name = "wlan%d";
+#endif /* CONFIG_PLATFORM_ANDROID */
+module_param(if2name, charp, 0644);
+MODULE_PARM_DESC(if2name, "The default name to allocate for second interface");
+
+#if defined(CONFIG_PLATFORM_ANDROID) && (CONFIG_IFACE_NUMBER > 2)
+	char *if3name = "ap%d";
+module_param(if3name, charp, 0644);
+MODULE_PARM_DESC(if3name, "The default name to allocate for third interface");
+#endif
+
+char *rtw_initmac = 0;  /* temp mac address if users want to use instead of the mac address in Efuse */
+
+#ifdef CONFIG_CONCURRENT_MODE
+
+	#if (CONFIG_IFACE_NUMBER > 2)
+		int rtw_virtual_iface_num = CONFIG_IFACE_NUMBER - 1;
+		module_param(rtw_virtual_iface_num, int, 0644);
+	#else
+		int rtw_virtual_iface_num = 1;
+	#endif
+
+#if defined(CONFIG_CONCURRENT_MODE) && !RTW_P2P_GROUP_INTERFACE
+#ifdef CONFIG_P2P
+	#ifdef CONFIG_SEL_P2P_IFACE
+	int rtw_sel_p2p_iface = CONFIG_SEL_P2P_IFACE;
+	#else
+	int rtw_sel_p2p_iface = (CONFIG_RTW_STATIC_NDEV_NUM - 1);
+	#endif
+
+	module_param(rtw_sel_p2p_iface, int, 0644);
+
+#endif
+#endif
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+	int rtw_ignore_go_in_scan = 1;
+	uint rtw_ignore_low_rssi_in_scan = 30;
+
+	module_param(rtw_ignore_go_in_scan, int, 0644);
+	module_param(rtw_ignore_low_rssi_in_scan, uint, 0644);
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+
+#endif
+
+/* affect ap/go cw only so far , 0 is no change*/
+uint rtw_vo_edca = 0;
+module_param(rtw_vo_edca, uint, 0644);
+
+#ifdef CONFIG_AP_MODE
+uint rtw_max_ap_assoc_sta = CONFIG_RTW_MAX_AP_ASSOC_STA;
+module_param(rtw_max_ap_assoc_sta, uint, 0644);
+MODULE_PARM_DESC(rtw_max_ap_assoc_sta, "the maximum number of associated STAs of AP mode, 0: not specified");
+
+u8 rtw_bmc_tx_rate = MGN_UNKNOWN;
+
+#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+int rtw_ap_src_b2u_flags = CONFIG_RTW_AP_SRC_B2U_FLAGS;
+module_param(rtw_ap_src_b2u_flags, int, 0644);
+
+int rtw_ap_fwd_b2u_flags = CONFIG_RTW_AP_FWD_B2U_FLAGS;
+module_param(rtw_ap_fwd_b2u_flags, int, 0644);
+#endif /* CONFIG_RTW_AP_DATA_BMC_TO_UC */
+#endif /* CONFIG_AP_MODE */
+
+#ifdef CONFIG_RTW_MESH
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+int rtw_msrc_b2u_flags = CONFIG_RTW_MSRC_B2U_FLAGS;
+module_param(rtw_msrc_b2u_flags, int, 0644);
+
+int rtw_mfwd_b2u_flags = CONFIG_RTW_MFWD_B2U_FLAGS;
+module_param(rtw_mfwd_b2u_flags, int, 0644);
+#endif /* CONFIG_RTW_MESH_DATA_BMC_TO_UC */
+#endif /* CONFIG_RTW_MESH */
+
+#ifdef RTW_WOW_STA_MIX
+int rtw_wowlan_sta_mix_mode = 1;
+#else
+int rtw_wowlan_sta_mix_mode = 0;
+#endif
+module_param(rtw_wowlan_sta_mix_mode, int, 0644);
+module_param(rtw_pwrtrim_enable, int, 0644);
+module_param(rtw_initmac, charp, 0644);
+module_param(rtw_chip_version, int, 0644);
+module_param(rtw_rfintfs, int, 0644);
+module_param(rtw_lbkmode, int, 0644);
+module_param(rtw_network_mode, int, 0644);
+module_param(rtw_band, int, 0644);
+module_param(rtw_channel, int, 0644);
+module_param(rtw_mp_mode, int, 0644);
+module_param(rtw_wmm_enable, int, 0644);
+module_param(rtw_vrtl_carrier_sense, int, 0644);
+module_param(rtw_vcs_type, int, 0644);
+module_param(rtw_hw_rts_en, int, 0644);
+module_param(rtw_busy_thresh, int, 0644);
+
+#ifdef CONFIG_80211N_HT
+module_param(rtw_ht_enable, int, 0644);
+module_param(rtw_bw_mode, int, 0644);
+module_param(rtw_ampdu_enable, int, 0644);
+module_param(rtw_stbc_cap, int, 0644);
+module_param(rtw_rx_ampdu_amsdu, int, 0644);
+module_param(rtw_tx_ampdu_amsdu, int, 0644);
+module_param(rtw_quick_addba_req, int, 0644);
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_BEAMFORMING
+module_param(rtw_beamform_cap, int, 0644);
+module_param(rtw_sw_proto_bf_cap_phy0, int, 0644);
+module_param(rtw_sw_proto_bf_cap_phy1, int, 0644);
+module_param(rtw_dyn_txbf, int, 0644);
+#endif
+module_param(rtw_lowrate_two_xmit, int, 0644);
+
+module_param(rtw_low_power, int, 0644);
+module_param(rtw_wifi_spec, int, 0644);
+
+module_param(rtw_full_ch_in_p2p_handshake, int, 0644);
+module_param(rtw_antdiv_cfg, int, 0644);
+module_param(rtw_antdiv_type, int, 0644);
+
+module_param(rtw_drv_ant_band_switch, int, 0644);
+
+module_param(rtw_switch_usb_mode, int, 0644);
+
+module_param(rtw_hwpwrp_detect, int, 0644);
+
+module_param(rtw_hw_wps_pbc, int, 0644);
+module_param(rtw_check_hw_status, int, 0644);
+
+#if defined(CONFIG_PCI_HCI) && defined(CONFIG_PCI_ASPM)
+module_param(rtw_pci_aspm_enable, int, 0644);
+#endif
+
+#ifdef CONFIG_TX_EARLY_MODE
+module_param(rtw_early_mode, int, 0644);
+#endif
+#ifdef CONFIG_ADAPTOR_INFO_CACHING_FILE
+char *rtw_adaptor_info_caching_file_path = "/data/misc/wifi/rtw_cache";
+module_param(rtw_adaptor_info_caching_file_path, charp, 0644);
+MODULE_PARM_DESC(rtw_adaptor_info_caching_file_path, "The path of adapter info cache file");
+#endif /* CONFIG_ADAPTOR_INFO_CACHING_FILE */
+
+#ifdef CONFIG_LAYER2_ROAMING
+uint rtw_max_roaming_times = 2;
+module_param(rtw_max_roaming_times, uint, 0644);
+MODULE_PARM_DESC(rtw_max_roaming_times, "The max roaming times to try");
+#endif /* CONFIG_LAYER2_ROAMING */
+
+#ifdef CONFIG_FILE_FWIMG
+char *rtw_fw_file_path = CONFIG_FIRMWARE_PATH; /* "/system/etc/firmware/rtlwifi/FW_NIC.BIN"; */
+
+module_param(rtw_fw_file_path, charp, 0644);
+MODULE_PARM_DESC(rtw_fw_file_path, "The path of fw image");
+
+char *rtw_fw_wow_file_path = "/system/etc/firmware/rtlwifi/FW_WoWLAN.BIN";
+module_param(rtw_fw_wow_file_path, charp, 0644);
+MODULE_PARM_DESC(rtw_fw_wow_file_path, "The path of fw for Wake on Wireless image");
+
+#ifdef CONFIG_MP_INCLUDED
+char *rtw_fw_mp_bt_file_path = "";
+module_param(rtw_fw_mp_bt_file_path, charp, 0644);
+MODULE_PARM_DESC(rtw_fw_mp_bt_file_path, "The path of fw for MP-BT image");
+#endif /* CONFIG_MP_INCLUDED */
+#endif /* CONFIG_FILE_FWIMG */
+
+uint rtw_hiq_filter = CONFIG_RTW_HIQ_FILTER;
+module_param(rtw_hiq_filter, uint, 0644);
+MODULE_PARM_DESC(rtw_hiq_filter, "0:allow all, 1:allow special, 2:deny all");
+
+uint rtw_edcca_mode_sel = CONFIG_RTW_EDCCA_MODE_SEL;
+module_param(rtw_edcca_mode_sel, uint, 0644);
+MODULE_PARM_DESC(rtw_edcca_mode_sel, "0:NORMAL, 1:CS, 2:ADPT, 3:CBP, 255:auto");
+
+uint rtw_adaptivity_en = CONFIG_RTW_ADAPTIVITY_EN;
+module_param(rtw_adaptivity_en, uint, 0644);
+MODULE_PARM_DESC(rtw_adaptivity_en, "0:disable, 1:enable, 2:auto");
+
+uint rtw_adaptivity_mode = CONFIG_RTW_ADAPTIVITY_MODE;
+module_param(rtw_adaptivity_mode, uint, 0644);
+MODULE_PARM_DESC(rtw_adaptivity_mode, "0:normal, 1:carrier sense");
+
+int rtw_adaptivity_idle_probability = 0;
+module_param(rtw_adaptivity_idle_probability, int, 0644);
+MODULE_PARM_DESC(rtw_adaptivity_idle_probability, "rtw_adaptivity_idle_probability");
+
+#ifdef CONFIG_DFS_MASTER
+uint rtw_dfs_region_domain = CONFIG_RTW_DFS_REGION_DOMAIN;
+module_param(rtw_dfs_region_domain, uint, 0644);
+MODULE_PARM_DESC(rtw_dfs_region_domain, "0:NONE, 1:FCC, 2:MKK, 3:ETSI, 4:KCC");
+#endif
+
+uint rtw_amsdu_mode = RTW_AMSDU_MODE_NON_SPP;
+module_param(rtw_amsdu_mode, uint, 0644);
+MODULE_PARM_DESC(rtw_amsdu_mode, "0:non-spp, 1:spp, 2:all drop");
+
+uint rtw_amplifier_type_2g = CONFIG_RTW_AMPLIFIER_TYPE_2G;
+module_param(rtw_amplifier_type_2g, uint, 0644);
+MODULE_PARM_DESC(rtw_amplifier_type_2g, "BIT3:2G ext-PA, BIT4:2G ext-LNA");
+
+uint rtw_amplifier_type_5g = CONFIG_RTW_AMPLIFIER_TYPE_5G;
+module_param(rtw_amplifier_type_5g, uint, 0644);
+MODULE_PARM_DESC(rtw_amplifier_type_5g, "BIT6:5G ext-PA, BIT7:5G ext-LNA");
+
+uint rtw_RFE_type = CONFIG_RTW_RFE_TYPE;
+module_param(rtw_RFE_type, uint, 0644);
+MODULE_PARM_DESC(rtw_RFE_type, "default init value:64");
+
+uint rtw_rfe_type = CONFIG_RTW_RFE_TYPE;
+module_param(rtw_rfe_type, uint, 0644);
+MODULE_PARM_DESC(rtw_rfe_type, "default init value:64");
+
+#ifdef CONFIG_DBCC_SUPPORT
+/*0:disable ,1: enable*/
+int rtw_dbcc_en = 1;
+module_param(rtw_dbcc_en, int, 0644);
+MODULE_PARM_DESC(rtw_dbcc_en, "0:Disable, 1:Enable DBCC");
+
+int rtw_dbcc_itf_ctl = 0;
+module_param(rtw_dbcc_itf_ctl, int, 0644);
+MODULE_PARM_DESC(rtw_dbcc_itf_ctl, "0:can't control interference, 1:can control interference");
+
+#ifdef CONFIG_DBCC_FORCE
+/* rmap - bitmap of wrole(idx) - force roles (idx) operate in band1*/
+int rtw_dbcc_force_rmap = 0;
+module_param(rtw_dbcc_force_rmap, int, 0644);
+MODULE_PARM_DESC(rtw_dbcc_force_rmap, "Band-1's Role idx");
+
+int rtw_dbcc_force_cck_phyidx = 0;
+module_param(rtw_dbcc_force_cck_phyidx, int, 0644);
+MODULE_PARM_DESC(rtw_dbcc_force_cck_phyidx, "cck phy-idx");
+#endif /*CONFIG_DBCC_FORCE*/
+
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+int rtw_dbcc_lg_sim = 0;
+module_param(rtw_dbcc_lg_sim, int, 0644);
+#endif
+#endif /*CONFIG_DBCC_SUPPORT*/
+
+uint rtw_powertracking_type = 64;
+module_param(rtw_powertracking_type, uint, 0644);
+MODULE_PARM_DESC(rtw_powertracking_type, "default init value:64");
+
+uint rtw_GLNA_type = CONFIG_RTW_GLNA_TYPE;
+module_param(rtw_GLNA_type, uint, 0644);
+MODULE_PARM_DESC(rtw_GLNA_type, "default init value:0");
+
+uint rtw_TxBBSwing_2G = 0xFF;
+module_param(rtw_TxBBSwing_2G, uint, 0644);
+MODULE_PARM_DESC(rtw_TxBBSwing_2G, "default init value:0xFF");
+
+uint rtw_TxBBSwing_5G = 0xFF;
+module_param(rtw_TxBBSwing_5G, uint, 0644);
+MODULE_PARM_DESC(rtw_TxBBSwing_5G, "default init value:0xFF");
+
+uint rtw_OffEfuseMask = 0;
+module_param(rtw_OffEfuseMask, uint, 0644);
+MODULE_PARM_DESC(rtw_OffEfuseMask, "default open Efuse Mask value:0");
+
+uint rtw_FileMaskEfuse = 0;
+module_param(rtw_FileMaskEfuse, uint, 0644);
+MODULE_PARM_DESC(rtw_FileMaskEfuse, "default drv Mask Efuse value:0");
+
+uint rtw_rxgain_offset_2g = 0;
+module_param(rtw_rxgain_offset_2g, uint, 0644);
+MODULE_PARM_DESC(rtw_rxgain_offset_2g, "default RF Gain 2G Offset value:0");
+
+uint rtw_rxgain_offset_5gl = 0;
+module_param(rtw_rxgain_offset_5gl, uint, 0644);
+MODULE_PARM_DESC(rtw_rxgain_offset_5gl, "default RF Gain 5GL Offset value:0");
+
+uint rtw_rxgain_offset_5gm = 0;
+module_param(rtw_rxgain_offset_5gm, uint, 0644);
+MODULE_PARM_DESC(rtw_rxgain_offset_5gm, "default RF Gain 5GM Offset value:0");
+
+uint rtw_rxgain_offset_5gh = 0;
+module_param(rtw_rxgain_offset_5gh, uint, 0644);
+MODULE_PARM_DESC(rtw_rxgain_offset_5gm, "default RF Gain 5GL Offset value:0");
+
+uint rtw_pll_ref_clk_sel = CONFIG_RTW_PLL_REF_CLK_SEL;
+module_param(rtw_pll_ref_clk_sel, uint, 0644);
+MODULE_PARM_DESC(rtw_pll_ref_clk_sel, "force pll_ref_clk_sel, 0xF:use autoload value");
+
+int rtw_tx_pwr_by_rate = CONFIG_TXPWR_BY_RATE_EN;
+module_param(rtw_tx_pwr_by_rate, int, 0644);
+MODULE_PARM_DESC(rtw_tx_pwr_by_rate, "0:Disable, 1:Enable, 2: Depend on efuse");
+
+#if CONFIG_TXPWR_LIMIT
+int rtw_tx_pwr_lmt_enable = CONFIG_TXPWR_LIMIT_EN;
+module_param(rtw_tx_pwr_lmt_enable, int, 0644);
+MODULE_PARM_DESC(rtw_tx_pwr_lmt_enable, "0:Disable, 1:Enable, 2: Depend on efuse");
+#endif
+
+static int rtw_target_tx_pwr_2g_a[RATE_SECTION_NUM] = CONFIG_RTW_TARGET_TX_PWR_2G_A;
+static int rtw_target_tx_pwr_2g_a_num = 0;
+module_param_array(rtw_target_tx_pwr_2g_a, int, &rtw_target_tx_pwr_2g_a_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_2g_a, "2.4G target tx power (unit:dBm) of RF path A for each rate section, should match the real calibrate power, -1: undefined");
+
+static int rtw_target_tx_pwr_2g_b[RATE_SECTION_NUM] = CONFIG_RTW_TARGET_TX_PWR_2G_B;
+static int rtw_target_tx_pwr_2g_b_num = 0;
+module_param_array(rtw_target_tx_pwr_2g_b, int, &rtw_target_tx_pwr_2g_b_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_2g_b, "2.4G target tx power (unit:dBm) of RF path B for each rate section, should match the real calibrate power, -1: undefined");
+
+static int rtw_target_tx_pwr_2g_c[RATE_SECTION_NUM] = CONFIG_RTW_TARGET_TX_PWR_2G_C;
+static int rtw_target_tx_pwr_2g_c_num = 0;
+module_param_array(rtw_target_tx_pwr_2g_c, int, &rtw_target_tx_pwr_2g_c_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_2g_c, "2.4G target tx power (unit:dBm) of RF path C for each rate section, should match the real calibrate power, -1: undefined");
+
+static int rtw_target_tx_pwr_2g_d[RATE_SECTION_NUM] = CONFIG_RTW_TARGET_TX_PWR_2G_D;
+static int rtw_target_tx_pwr_2g_d_num = 0;
+module_param_array(rtw_target_tx_pwr_2g_d, int, &rtw_target_tx_pwr_2g_d_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_2g_d, "2.4G target tx power (unit:dBm) of RF path D for each rate section, should match the real calibrate power, -1: undefined");
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+static int rtw_target_tx_pwr_5g_a[RATE_SECTION_NUM - 1] = CONFIG_RTW_TARGET_TX_PWR_5G_A;
+static int rtw_target_tx_pwr_5g_a_num = 0;
+module_param_array(rtw_target_tx_pwr_5g_a, int, &rtw_target_tx_pwr_5g_a_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_5g_a, "5G target tx power (unit:dBm) of RF path A for each rate section, should match the real calibrate power, -1: undefined");
+
+static int rtw_target_tx_pwr_5g_b[RATE_SECTION_NUM - 1] = CONFIG_RTW_TARGET_TX_PWR_5G_B;
+static int rtw_target_tx_pwr_5g_b_num = 0;
+module_param_array(rtw_target_tx_pwr_5g_b, int, &rtw_target_tx_pwr_5g_b_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_5g_b, "5G target tx power (unit:dBm) of RF path B for each rate section, should match the real calibrate power, -1: undefined");
+
+static int rtw_target_tx_pwr_5g_c[RATE_SECTION_NUM - 1] = CONFIG_RTW_TARGET_TX_PWR_5G_C;
+static int rtw_target_tx_pwr_5g_c_num = 0;
+module_param_array(rtw_target_tx_pwr_5g_c, int, &rtw_target_tx_pwr_5g_c_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_5g_c, "5G target tx power (unit:dBm) of RF path C for each rate section, should match the real calibrate power, -1: undefined");
+
+static int rtw_target_tx_pwr_5g_d[RATE_SECTION_NUM - 1] = CONFIG_RTW_TARGET_TX_PWR_5G_D;
+static int rtw_target_tx_pwr_5g_d_num = 0;
+module_param_array(rtw_target_tx_pwr_5g_d, int, &rtw_target_tx_pwr_5g_d_num, 0644);
+MODULE_PARM_DESC(rtw_target_tx_pwr_5g_d, "5G target tx power (unit:dBm) of RF path D for each rate section, should match the real calibrate power, -1: undefined");
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+
+#ifdef CONFIG_RTW_TX_NPATH_EN
+/*0:disable ,1: 2path*/
+int rtw_tx_npath_enable = 1;
+module_param(rtw_tx_npath_enable, int, 0644);
+MODULE_PARM_DESC(rtw_tx_npath_enable, "0:Disable, 1:TX-2PATH");
+#endif
+
+#ifdef CONFIG_RTW_PATH_DIV
+/*0:disable ,1: path diversity*/
+int rtw_path_div_enable = 1;
+module_param(rtw_path_div_enable, int, 0644);
+MODULE_PARM_DESC(rtw_path_div_enable, "0:Disable, 1:Enable path diversity");
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+int rtw_twt_enable = 0;
+module_param(rtw_twt_enable, int, 0644);
+MODULE_PARM_DESC(rtw_twt_enable, "0:Disable, 1:Enable TWT");
+#endif
+#endif
+
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+char *rtw_phy_file_path = REALTEK_CONFIG_PATH;
+module_param(rtw_phy_file_path, charp, 0644);
+MODULE_PARM_DESC(rtw_phy_file_path, "The path of phy parameter");
+/* PHY FILE Bit Map
+* BIT0 - MAC_REG,			0: non-support, 1: support
+* BIT1 - BB_PHY_REG,			0: non-support, 1: support
+* BIT2 - BB_PHY_REG_MP,			0: non-support, 1: support
+* BIT3 - RF_RADIO,			0: non-support, 1: support
+* BIT4 - RF_TXPWR_BY_RATE,		0: non-support, 1: support
+* BIT5 - RF_TXPWR_TRACK,		0: non-support, 1: support
+* BIT6 - RF_TXPWR_LMT,			0: non-support, 1: support
+* BIT7 - RF_TXPWR_LMT_RU,		0: non-support, 1: support
+* BIT8 - BB_PHY_REG_GAIN_FILE		0: non-support, 1: support
+* BIT9 - RF_TXPWR_LMT_6G_FILE		0: non-support, 1: support
+* BIT10 -RF_TXPWR_LMT_RU_6G_FILE	0: non-support, 1: support
+*/
+int rtw_load_phy_file = ( BIT4 | BIT6 | BIT7 | BIT9 | BIT10);
+module_param(rtw_load_phy_file, int, 0644);
+MODULE_PARM_DESC(rtw_load_phy_file, "PHY File Bit Map");
+int rtw_decrypt_phy_file = 0;
+module_param(rtw_decrypt_phy_file, int, 0644);
+MODULE_PARM_DESC(rtw_decrypt_phy_file, "Enable Decrypt PHY File");
+bool rtw_is_free_param_info = true;
+#endif
+
+
+uint rtw_phydm_ability = 0xffffffff;
+module_param(rtw_phydm_ability, uint, 0644);
+
+uint rtw_halrf_ability = 0xffffffff;
+module_param(rtw_halrf_ability, uint, 0644);
+
+
+uint rtw_rfk_ability = 0xffffffff;
+module_param(rtw_rfk_ability, uint, 0644);
+
+#ifdef CONFIG_FW_IO_OFLD_SUPPORT
+uint rtw_fw_ofld_cap = 1;
+module_param(rtw_fw_ofld_cap, uint, 0644);
+#endif /*CONFIG_FW_IO_OFLD_SUPPORT*/
+
+#ifdef CONFIG_CHSW_OFLD
+uint rtw_chsw_ofld_cap = 1;
+module_param(rtw_chsw_ofld_cap, uint, 0644);
+#endif
+
+uint rtw_edcca_th_2g = 0;
+module_param(rtw_edcca_th_2g, uint, 0644);
+uint rtw_edcca_th_5g = 0;
+module_param(rtw_edcca_th_5g, uint, 0644);
+uint rtw_edcca_cs_th = 0;
+module_param(rtw_edcca_cs_th, uint, 0644);
+#if CONFIG_IEEE80211_BAND_6GHZ
+uint rtw_edcca_cbp_th_6g = 0;
+module_param(rtw_edcca_cbp_th_6g, uint, 0644);
+#endif
+
+#ifdef CONFIG_RTW_MESH
+uint rtw_peer_alive_based_preq = 1;
+module_param(rtw_peer_alive_based_preq, uint, 0644);
+MODULE_PARM_DESC(rtw_peer_alive_based_preq,
+	"On demand PREQ will reference peer alive status. 0: Off, 1: On");
+#endif
+
+#ifdef CONFIG_RTW_NAPI
+/*following setting should define NAPI in Makefile
+enable napi only = 1, disable napi = 0*/
+int rtw_en_napi = 1;
+module_param(rtw_en_napi, int, 0644);
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+int rtw_napi_threshold = 100; /* unit: Mbps */
+module_param(rtw_napi_threshold, int, 0644);
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+#ifdef CONFIG_RTW_GRO
+/*following setting should define GRO in Makefile
+enable gro = 1, disable gro = 0*/
+int rtw_en_gro = 1;
+module_param(rtw_en_gro, int, 0644);
+#endif /* CONFIG_RTW_GRO */
+#endif /* CONFIG_RTW_NAPI */
+
+#ifdef RTW_IQK_FW_OFFLOAD
+int rtw_iqk_fw_offload = 1;
+#else
+int rtw_iqk_fw_offload;
+#endif /* RTW_IQK_FW_OFFLOAD */
+module_param(rtw_iqk_fw_offload, int, 0644);
+
+#ifdef RTW_CHANNEL_SWITCH_OFFLOAD
+int rtw_ch_switch_offload = 0;
+#else
+int rtw_ch_switch_offload;
+#endif /* RTW_CHANNEL_SWITCH_OFFLOAD */
+module_param(rtw_ch_switch_offload, int, 0644);
+
+#ifdef CONFIG_TDLS
+int rtw_en_tdls = 1;
+module_param(rtw_en_tdls, int, 0644);
+#endif
+
+#ifdef CONFIG_FW_OFFLOAD_PARAM_INIT
+int rtw_fw_param_init = 1;
+module_param(rtw_fw_param_init, int, 0644);
+#endif
+
+#ifdef CONFIG_TDMADIG
+int rtw_tdmadig_en = 1;
+/*
+1:MODE_PERFORMANCE
+2:MODE_COVERAGE
+*/
+int rtw_tdmadig_mode = 1;
+int rtw_dynamic_tdmadig = 0;
+module_param(rtw_tdmadig_en, int, 0644);
+module_param(rtw_tdmadig_mode, int, 0644);
+module_param(rtw_dynamic_tdmadig, int, 0644);
+#endif/*CONFIG_TDMADIG*/
+
+/*dynamic RRSR default enable*/
+int rtw_en_dyn_rrsr = 1;
+int rtw_rrsr_value = 0xFFFFFFFF;
+module_param(rtw_en_dyn_rrsr, int, 0644);
+module_param(rtw_rrsr_value, int, 0644);
+
+#ifdef CONFIG_WOWLAN
+/*
+ * 0: disable, 1: enable
+ */
+uint rtw_wow_enable = 1;
+module_param(rtw_wow_enable, uint, 0644);
+/*
+ * bit[0]: magic packet wake up
+ * bit[1]: unucast packet(HW/FW unuicast)
+ * bit[2]: deauth wake up
+ */
+uint rtw_wakeup_event = RTW_WAKEUP_EVENT;
+module_param(rtw_wakeup_event, uint, 0644);
+/*
+ * 0: common WOWLAN
+ * bit[0]: disable BB RF
+ * bit[1]: For wireless remote controller with or without connection
+ */
+uint rtw_suspend_type = RTW_SUSPEND_TYPE;
+module_param(rtw_suspend_type, uint, 0644);
+#endif
+
+#if defined(PRIVATE_R) && defined(CONFIG_P2P)
+int rtw_go_hidden_ssid_mode = ALL_HIDE_SSID;
+module_param(rtw_go_hidden_ssid_mode, int, 0644);
+#endif
+
+#ifdef RTW_BUSY_DENY_SCAN
+uint rtw_scan_interval_thr = BUSY_TRAFFIC_SCAN_DENY_PERIOD;
+module_param(rtw_scan_interval_thr, uint, 0644);
+MODULE_PARM_DESC(rtw_scan_interval_thr, "Threshold used to judge if scan " \
+		 "request comes from scan UI, unit is ms.");
+#endif /* RTW_BUSY_DENY_SCAN */
+
+#ifdef CONFIG_HW_HDR_CONVERSION
+int rtw_hw_hdr_conv = true;
+#else
+int rtw_hw_hdr_conv = false;
+#endif
+
+#ifdef CONFIG_MCC_MODE
+int rtw_mcc_en = _TRUE;
+module_param(rtw_mcc_en, int, 0644);
+#endif
+
+#ifdef CONFIG_RTW_MULTI_AP
+static int rtw_unassoc_sta_mode_of_stype[UNASOC_STA_SRC_NUM] = CONFIG_RTW_UNASOC_STA_MODE_OF_STYPE;
+static int rtw_unassoc_sta_mode_of_stype_num = 0;
+module_param_array(rtw_unassoc_sta_mode_of_stype, int, &rtw_unassoc_sta_mode_of_stype_num, 0644);
+
+uint rtw_max_unassoc_sta_cnt = 0;
+module_param(rtw_max_unassoc_sta_cnt, uint, 0644);
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+uint rtw_roch_min_home_dur = 1500;
+uint rtw_roch_max_away_dur = 500;
+uint rtw_roch_extend_dur = 500;
+module_param(rtw_roch_min_home_dur, uint, 0644);
+module_param(rtw_roch_max_away_dur, uint, 0644);
+module_param(rtw_roch_extend_dur, uint, 0644);
+#endif
+
+#if CONFIG_DFS
+#ifdef CONFIG_ECSA_PHL
+uint rtw_en_ecsa = 1;
+module_param(rtw_en_ecsa, uint, 0644);
+#endif
+#endif
+
+#ifdef CONFIG_QUOTA_TURBO_ENABLE
+int rtw_quota_turbo_en = true;
+#else
+int rtw_quota_turbo_en = false;
+#endif
+module_param(rtw_quota_turbo_en, int, 0644);
+
+uint rtw_scan_pch_ex = 0;
+module_param(rtw_scan_pch_ex, uint, 0644);
+
+#ifdef CONFIG_THERMAL_PROTECT
+/*
+ * CONFIG_THERMAL_PROTECT
+ * NOTICE: Before enabling thermal protction mechanism please make sure the
+ * firmware already support related features, ex. TX duty control or other
+ * needed mechanism.
+ */
+/*
+ * rtw_thermal_threshold
+ * Threshold value to trigger thermal protection.
+ * Default value 0xFF means no software setting, just use hardware(HW) setting,
+ * and HW setting is different by different IC design.
+ */
+uint rtw_thermal_threshold = 0xFF;
+module_param(rtw_thermal_threshold, uint, 0644);
+MODULE_PARM_DESC(rtw_thermal_threshold, "Thermal threshold to trigger thermal protection");
+/*
+ * rtw_thermal_min_duty
+ * The minimum percentage of TX duty can be configured.
+ * This could be used to meet minimal TX throughput requirement.
+ * The valid value range is 0~100, and 100 means no TX duty control.
+ */
+uint rtw_thermal_min_duty = 50;
+module_param(rtw_thermal_min_duty, uint, 0644);
+MODULE_PARM_DESC(rtw_thermal_min_duty, "The minimum percentage of TX duty can be configured, 0~100");
+#endif /* CONFIG_THERMAL_PROTECT */
+
+static void rtw_regsty_load_target_tx_power(struct registry_priv *regsty)
+{
+	int path, rs;
+	int *target_tx_pwr;
+
+	for (path = RF_PATH_A; path < RF_PATH_MAX; path++) {
+		if (path == RF_PATH_A)
+			target_tx_pwr = rtw_target_tx_pwr_2g_a;
+		else if (path == RF_PATH_B)
+			target_tx_pwr = rtw_target_tx_pwr_2g_b;
+		else if (path == RF_PATH_C)
+			target_tx_pwr = rtw_target_tx_pwr_2g_c;
+		else if (path == RF_PATH_D)
+			target_tx_pwr = rtw_target_tx_pwr_2g_d;
+
+		for (rs = CCK; rs < RATE_SECTION_NUM; rs++)
+			regsty->target_tx_pwr_2g[path][rs] = target_tx_pwr[rs];
+	}
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	for (path = RF_PATH_A; path < RF_PATH_MAX; path++) {
+		if (path == RF_PATH_A)
+			target_tx_pwr = rtw_target_tx_pwr_5g_a;
+		else if (path == RF_PATH_B)
+			target_tx_pwr = rtw_target_tx_pwr_5g_b;
+		else if (path == RF_PATH_C)
+			target_tx_pwr = rtw_target_tx_pwr_5g_c;
+		else if (path == RF_PATH_D)
+			target_tx_pwr = rtw_target_tx_pwr_5g_d;
+
+		for (rs = OFDM; rs < RATE_SECTION_NUM; rs++)
+			regsty->target_tx_pwr_5g[path][rs - 1] = target_tx_pwr[rs - 1];
+	}
+#endif /* CONFIG_IEEE80211_BAND_5GHZ */
+}
+
+static inline void rtw_regsty_load_chplan(struct registry_priv *regsty)
+{
+	u16 chplan = RTW_CHPLAN_UNSPECIFIED;
+	u16 chplan_6g = RTW_CHPLAN_6G_UNSPECIFIED;
+
+	chplan = rtw_channel_plan;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	chplan_6g = rtw_channel_plan_6g;
+#endif
+
+	rtw_chplan_ioctl_input_mapping(&chplan, &chplan_6g);
+
+	regsty->channel_plan = chplan;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	regsty->channel_plan_6g = chplan_6g;
+#endif
+}
+
+static inline void rtw_regsty_load_alpha2(struct registry_priv *regsty)
+{
+	if (!rtw_country_code || strlen(rtw_country_code) != 2
+		|| (!IS_ALPHA2_WORLDWIDE(rtw_country_code)
+			&& (is_alpha(rtw_country_code[0]) == _FALSE
+				|| is_alpha(rtw_country_code[1]) == _FALSE)
+			)
+	) {
+		if (rtw_country_code && rtw_country_code[0] != '\0')
+			RTW_ERR("%s discard rtw_country_code not in alpha2 or \"%s\"\n", __func__, WORLDWIDE_ALPHA2);
+		SET_UNSPEC_ALPHA2(regsty->alpha2);
+	} else
+		_rtw_memcpy(regsty->alpha2, rtw_country_code, 2);
+}
+
+static void rtw_regsty_load_addl_ch_disable_conf(struct registry_priv *regsty)
+{
+	int i;
+	int ch_num = 0;
+
+	if (rtw_dis_ch_flags && strlen(rtw_dis_ch_flags)) {
+		char *buf = rtw_malloc(strlen(rtw_dis_ch_flags) + 1);
+
+		if (buf) {
+			char *c;
+			enum rtw_ch_type ch_type;
+
+			_rtw_memcpy(buf, rtw_dis_ch_flags, strlen(rtw_dis_ch_flags) + 1);
+			for (c = strsep(&buf, ","); c; c = strsep(&buf, ",")) {
+				ch_type = get_ch_type_from_str(c, strlen(c));
+				if (ch_type != RTW_CHT_NUM)
+					regsty->dis_ch_flags |= BIT(ch_type);
+			}
+			rtw_mfree(buf, strlen(rtw_dis_ch_flags) + 1);
+		} else
+			RTW_WARN("%s rtw_malloc(strlen(rtw_dis_ch_flags) fail\n", __func__);
+	}
+
+	for (i = 0; i < MAX_CHANNEL_NUM_2G_5G; i++)
+		if (((u8)rtw_excl_chs[i]) != 0)
+			regsty->excl_chs[ch_num++] = (u8)rtw_excl_chs[i];
+
+	if (ch_num < MAX_CHANNEL_NUM_2G_5G)
+		regsty->excl_chs[ch_num] = 0;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+	ch_num = 0;
+	for (i = 0; i < MAX_CHANNEL_NUM_6G; i++)
+		if (((u8)rtw_excl_chs_6g[i]) != 0)
+			regsty->excl_chs_6g[ch_num++] = (u8)rtw_excl_chs_6g[i];
+
+	if (ch_num < MAX_CHANNEL_NUM_6G)
+		regsty->excl_chs_6g[ch_num] = 0;
+#endif
+}
+
+static inline void rtw_regsty_load_env_settings(struct registry_priv *regsty)
+{
+#if CONFIG_IEEE80211_BAND_6GHZ
+	regsty->env = (u8)rtw_env;
+	if (regsty->env >= RTW_ENV_NUM) {
+		RTW_WARN("%s invalid rtw_env(%u), set to %s\n", __func__
+			, regsty->env, env_str(RTW_ENV_ANY));
+		regsty->env = RTW_ENV_ANY;
+	}
+#endif
+}
+
+#ifdef CONFIG_80211D
+inline void rtw_regsty_load_country_ie_slave_settings(struct registry_priv *regsty)
+{
+	regsty->country_ie_slave_en_mode = rtw_country_ie_slave_en_mode;
+	regsty->country_ie_slave_flags = rtw_country_ie_slave_flags;
+	regsty->country_ie_slave_en_role = rtw_country_ie_slave_en_role;
+	regsty->country_ie_slave_en_ifbmp = rtw_country_ie_slave_en_ifbmp;
+	regsty->country_ie_slave_scan_int_ms = rtw_country_ie_slave_scan_int_ms;
+}
+#endif
+
+static void rtw_regsty_load_edcca_mode_settings(struct registry_priv *regsty)
+{
+	regsty->edcca_mode_sel = (u8)rtw_edcca_mode_sel;
+	if (RTW_EDCCA_MODE_SEL_VALID(regsty->edcca_mode_sel)) {
+		if (regsty->edcca_mode_sel == RTW_EDCCA_NORM) {
+			/* consider old interfaces */
+			if (rtw_adaptivity_en == RTW_ADAPTIVITY_EN_ENABLE) {
+				if (rtw_adaptivity_mode == RTW_ADAPTIVITY_MODE_NORMAL)
+					regsty->edcca_mode_sel = RTW_EDCCA_ADAPT;
+				else if (rtw_adaptivity_mode == RTW_ADAPTIVITY_MODE_CARRIER_SENSE)
+					regsty->edcca_mode_sel = RTW_EDCCA_CS;
+			} else if (rtw_adaptivity_en == RTW_ADAPTIVITY_EN_AUTO)
+				regsty->edcca_mode_sel = RTW_EDCCA_AUTO;
+		}
+	} else {
+		RTW_WARN("%s invalid rtw_edcca_mode_sel(%u), set to %s\n", __func__
+			, regsty->edcca_mode_sel, rtw_edcca_mode_str(RTW_EDCCA_NORM));
+		regsty->edcca_mode_sel = RTW_EDCCA_NORM;
+	}
+}
+
+#ifdef CONFIG_DFS_MASTER
+static void rtw_regsty_load_dfs_region_domain_settings(struct registry_priv *regsty)
+{
+	regsty->dfs_region_domain = (u8)rtw_dfs_region_domain;
+	if (regsty->dfs_region_domain >= RTW_DFS_REGD_NUM) {
+		RTW_WARN("%s invalid DFS region domain(%u), set to %s\n", __func__
+			, regsty->dfs_region_domain, rtw_dfs_regd_str(RTW_DFS_REGD_NONE));
+		regsty->dfs_region_domain = RTW_DFS_REGD_NONE;
+		return;
+	}
+	#ifdef CONFIG_REGD_SRC_FROM_OS
+	if (rtw_regd_src == REGD_SRC_OS && regsty->dfs_region_domain != RTW_DFS_REGD_NONE) {
+		RTW_WARN("%s force disable radar detection capability when regd_src is OS\n", __func__);
+		regsty->dfs_region_domain = RTW_DFS_REGD_NONE;
+	}
+	#endif
+}
+#endif
+
+#ifdef CONFIG_80211N_HT
+static inline void rtw_regsty_init_rx_ampdu_sz_limit(struct registry_priv *regsty)
+{
+	int i, j;
+	uint *sz_limit;
+
+	for (i = 0; i < 4; i++) {
+		if (i == 0)
+			sz_limit = rtw_rx_ampdu_sz_limit_1ss;
+		else if (i == 1)
+			sz_limit = rtw_rx_ampdu_sz_limit_2ss;
+		else if (i == 2)
+			sz_limit = rtw_rx_ampdu_sz_limit_3ss;
+		else if (i == 3)
+			sz_limit = rtw_rx_ampdu_sz_limit_4ss;
+
+		for (j = 0; j < 4; j++)
+			regsty->rx_ampdu_sz_limit_by_nss_bw[i][j] = sz_limit[j];
+	}
+}
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_RTW_MULTI_AP
+inline void rtw_regsty_init_unassoc_sta_param(struct registry_priv *regsty)
+{
+	int i;
+
+	for (i = 0; i < UNASOC_STA_SRC_NUM; i++)
+		regsty->unassoc_sta_mode_of_stype[i] = rtw_unassoc_sta_mode_of_stype[i];
+
+	regsty->max_unassoc_sta_cnt = (u16)rtw_max_unassoc_sta_cnt;
+}
+#endif
+
+static void rtw_load_phy_file_path (struct dvobj_priv *dvobj)
+{
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	if (rtw_load_phy_file & LOAD_BB_PHY_REG_FILE) {
+		phl_com->phy_sw_cap[0].bb_phy_reg_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].bb_phy_reg_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].bb_phy_reg_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].bb_phy_reg_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_RF_RADIO_FILE) {
+		phl_com->phy_sw_cap[0].rf_radio_a_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_radio_a_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_radio_b_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_radio_b_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_radio_a_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_radio_a_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[0].rf_radio_b_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_radio_b_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_RF_TXPWR_BY_RATE) {
+		phl_com->phy_sw_cap[0].rf_txpwr_byrate_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_txpwr_byrate_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_txpwr_byrate_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_txpwr_byrate_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_RF_TXPWR_TRACK_FILE) {
+		phl_com->phy_sw_cap[0].rf_txpwrtrack_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_txpwrtrack_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_txpwrtrack_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_txpwrtrack_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_RF_TXPWR_LMT_FILE) {
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_RF_TXPWR_LMT_RU_FILE) {
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_ru_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_ru_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_ru_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_ru_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_BB_PHY_REG_GAIN_FILE) {
+		phl_com->phy_sw_cap[0].bb_phy_reg_gain_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].bb_phy_reg_gain_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].bb_phy_reg_gain_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].bb_phy_reg_gain_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_RF_TXPWR_LMT_6G_FILE) {
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_6g_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_6g_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_6g_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_6g_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+	if (rtw_load_phy_file & LOAD_RF_TXPWR_LMT_RU_6G_FILE) {
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_ru_6g_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_ru_6g_info.para_src = RTW_PARA_SRC_EXTNAL;
+		phl_com->phy_sw_cap[0].rf_txpwrlmt_ru_6g_info.hal_phy_folder = rtw_phy_file_path;
+		phl_com->phy_sw_cap[1].rf_txpwrlmt_ru_6g_info.hal_phy_folder = rtw_phy_file_path;
+	}
+
+#endif/* CONFIG_LOAD_PHY_PARA_FROM_FILE */
+}
+
+#if defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW)
+void rtw_update_lps_proto_cap_setting(enum rtw_ps_proto_type proto, u8 mode,
+				      u8 para, struct rtw_ps_cap_t *ps_cap,
+				      bool is_wowlan)
+{
+	switch (proto) {
+	case RTW_PS_PROTO_LISTEN_BEACON_MODE:
+		rtw_update_lps_listen_beacon_mode(
+			mode, para,
+			is_wowlan ? &ps_cap->lps_wow_listen_bcn_mode :
+				    &ps_cap->lps_listen_bcn_mode,
+			is_wowlan ? &ps_cap->lps_wow_awake_interval :
+				    &ps_cap->lps_awake_interval);
+		break;
+	default:
+		RTW_ERR("%s ps protocol (%d) invalid\n", __func__, proto);
+	}
+}
+
+void rtw_update_lps_proto_registry_setting(enum rtw_ps_proto_type proto,
+					   u8 mode, u8 para,
+					   struct registry_priv *registry,
+					   bool is_wowlan)
+{
+	switch (proto) {
+	case RTW_PS_PROTO_LISTEN_BEACON_MODE:
+		if (is_wowlan)
+#ifdef CONFIG_RTW_LPS_WOW
+			rtw_update_lps_listen_beacon_mode(
+				mode, para, &registry->lps_wow_listen_bcn_mode,
+				&registry->lps_wow_awake_interval);
+#else
+			RTW_INFO("%s WOWLAN not enabled\n", __func__);
+#endif /* CONFIG_RTW_LPS_WOW */
+		else if (!is_wowlan)
+#ifdef CONFIG_RTW_LPS
+			rtw_update_lps_listen_beacon_mode(
+				mode, para, &registry->lps_listen_bcn_mode,
+				&registry->lps_awake_interval);
+#else
+			RTW_INFO("%s LPS not enabled\n", __func__);
+#endif /* CONFIG_RTW_LPS */
+		break;
+	default:
+		RTW_ERR("%s ps protocol (%d) invalid\n", __func__, proto);
+	}
+}
+#endif /* defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW) */
+
+void rtw_core_update_default_setting (struct dvobj_priv *dvobj)
+{
+	int i;
+	struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);
+#ifdef CONFIG_PCIE_TRX_MIT_DYN
+	struct rtw_pcie_trx_mit_info_t mit_info = {0};
+	void *phl = GET_PHL_INFO(dvobj);
+#endif
+#ifdef CONFIG_PCI_HCI
+	PPCI_DATA pci_data = dvobj_to_pci(dvobj);
+#endif
+	u8 max_bw_mode = 0;
+
+	#ifdef CONFIG_RTW_DEBUG
+	rtw_phl_log_level_cfg(rtw_phl_log_level);
+	#endif
+
+	#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	rtw_load_phy_file_path(dvobj);
+	phl_com->dev_sw_cap.bfree_para_info = rtw_is_free_param_info;
+	#endif /* CONFIG_LOAD_PHY_PARA_FROM_FILE */
+
+	if (is_supported_24g(rtw_band_type))
+		max_bw_mode = BW_MODE_2G(rtw_bw_mode);
+	if (is_supported_5g(rtw_band_type))
+		max_bw_mode = rtw_max(BW_MODE_5G(rtw_bw_mode), max_bw_mode);
+	if (is_supported_6g(rtw_band_type))
+		max_bw_mode = rtw_max(BW_MODE_6G(rtw_bw_mode), max_bw_mode);
+
+	phl_com->dev_sw_cap.fw_cap.dlram_en = true;
+	phl_com->dev_sw_cap.fw_cap.dlrom_en = false;
+	#ifdef CONFIG_FILE_FWIMG
+	phl_com->dev_sw_cap.fw_cap.fw_src = RTW_FW_SRC_EXTNAL;
+	#else /* !CONFIG_FILE_FWIMG */
+	phl_com->dev_sw_cap.fw_cap.fw_src = RTW_FW_SRC_INTNAL;
+	#endif /* !CONFIG_FILE_FWIMG */
+
+	#ifdef CONFIG_FW_IO_OFLD_SUPPORT
+	#ifdef CONFIG_PHL_IO_OFLD
+	phl_com->dev_sw_cap.io_ofld = rtw_fw_ofld_cap;
+	#endif
+	#endif
+	#ifdef CONFIG_CHSW_OFLD
+	phl_com->dev_sw_cap.chsw_ofld = rtw_chsw_ofld_cap;
+	#endif
+
+	phl_com->phy_sw_cap[0].proto_sup = rtw_wireless_mode;
+	phl_com->phy_sw_cap[0].band_sup = rtw_band_type;
+	phl_com->phy_sw_cap[0].bw_sup = ch_width_to_bw_cap(max_bw_mode + 1) - 1; /* max supported bw */
+	phl_com->phy_sw_cap[0].txss = rtw_tx_nss;
+	phl_com->phy_sw_cap[0].rxss = rtw_rx_nss;
+	phl_com->phy_sw_cap[0].tx_path_num = rtw_rf_path;
+	phl_com->phy_sw_cap[0].rx_path_num = rtw_rf_path;
+	phl_com->phy_sw_cap[0].txagg_num = rtw_tx_ampdu_num;
+
+	phl_com->phy_sw_cap[1].proto_sup = rtw_wireless_mode;
+	phl_com->phy_sw_cap[1].band_sup = rtw_band_type;
+	phl_com->phy_sw_cap[1].bw_sup = ch_width_to_bw_cap(max_bw_mode + 1) - 1; /* max supported bw */
+	phl_com->phy_sw_cap[1].txss = rtw_tx_nss;
+	phl_com->phy_sw_cap[1].rxss = rtw_rx_nss;
+	phl_com->phy_sw_cap[1].tx_path_num = rtw_rf_path;
+	phl_com->phy_sw_cap[1].rx_path_num = rtw_rf_path;
+	phl_com->phy_sw_cap[1].txagg_num = rtw_tx_ampdu_num;
+
+	phl_com->phy_sw_cap[0].hw_rts_time_th = 88;
+	phl_com->phy_sw_cap[0].hw_rts_len_th = 4080;
+	phl_com->phy_sw_cap[1].hw_rts_time_th = 88;
+	phl_com->phy_sw_cap[1].hw_rts_len_th = 4080;
+
+	/*phl_com->dev_sw_cap.pkg_type = rtw_pkg_type;*/
+	if (rtw_RFE_type != CONFIG_RTW_RFE_TYPE)
+		rtw_rfe_type = rtw_RFE_type;
+	phl_com->dev_sw_cap.rfe_type = rtw_rfe_type;
+#ifdef DBG_LA_MODE
+	phl_com->dev_sw_cap.la_mode = rtw_la_mode_en;
+#endif
+
+#ifdef CONFIG_DBCC_SUPPORT
+	phl_com->dev_sw_cap.dbcc_sup = rtw_dbcc_en;
+	phl_com->dev_sw_cap.mcmb_itf_ctrl = rtw_dbcc_itf_ctl;
+#ifdef CONFIG_DBCC_FORCE
+	phl_com->dev_sw_cap.dbcc_force_rmap = rtw_dbcc_force_rmap;
+	phl_com->dev_sw_cap.dbcc_force_cck_phyidx = rtw_dbcc_force_cck_phyidx;
+#endif /*CONFIG_DBCC_FORCE*/
+#endif /*CONFIG_DBCC_SUPPORT*/
+
+	phl_com->dev_sw_cap.hw_hdr_conv = rtw_hw_hdr_conv;
+#if defined(CONFIG_LOGO_MODE_ADJUST_AMSDU_RXFIFO)
+	phl_com->proto_sw_cap[0].max_amsdu_len = rtw_wifi_spec ? 0 : rtw_max_amsdu_len;
+	phl_com->proto_sw_cap[1].max_amsdu_len = rtw_wifi_spec ? 0 : rtw_max_amsdu_len;
+#else
+	phl_com->proto_sw_cap[0].max_amsdu_len = rtw_max_amsdu_len;
+	phl_com->proto_sw_cap[1].max_amsdu_len = rtw_max_amsdu_len;
+#endif
+	phl_com->proto_sw_cap[0].amsdu_in_ampdu = 1;
+	phl_com->proto_sw_cap[1].amsdu_in_ampdu = 1;
+#if defined(CONFIG_PCI_HCI)
+	#if defined(CONFIG_PCI_ASPM)
+	phl_com->bus_sw_cap.l0s_ctrl = (rtw_pci_aspm_enable & BIT1) ? RTW_PCIE_BUS_FUNC_ENABLE : RTW_PCIE_BUS_FUNC_DISABLE;
+	phl_com->bus_sw_cap.l1_ctrl = (rtw_pci_aspm_enable & BIT2) ? RTW_PCIE_BUS_FUNC_ENABLE : RTW_PCIE_BUS_FUNC_DISABLE;
+	phl_com->bus_sw_cap.l1ss_ctrl = (rtw_pci_aspm_enable & BIT3) ? RTW_PCIE_BUS_FUNC_ENABLE : RTW_PCIE_BUS_FUNC_DISABLE;
+	phl_com->bus_sw_cap.wake_ctrl = RTW_PCIE_BUS_FUNC_DEFAULT;
+	phl_com->bus_sw_cap.crq_ctrl = (rtw_pci_aspm_enable & BIT0) ? RTW_PCIE_BUS_FUNC_ENABLE : RTW_PCIE_BUS_FUNC_DISABLE;
+	#else
+	phl_com->bus_sw_cap.l0s_ctrl = RTW_PCIE_BUS_FUNC_DISABLE;
+	phl_com->bus_sw_cap.l1_ctrl = RTW_PCIE_BUS_FUNC_DISABLE;
+	phl_com->bus_sw_cap.l1ss_ctrl = RTW_PCIE_BUS_FUNC_DISABLE;
+	phl_com->bus_sw_cap.wake_ctrl = RTW_PCIE_BUS_FUNC_DEFAULT;
+	phl_com->bus_sw_cap.crq_ctrl = RTW_PCIE_BUS_FUNC_DISABLE;
+	#endif
+
+#if defined(CONFIG_TX_WD_NUM) && (CONFIG_TX_WD_NUM > 0)
+	phl_com->bus_sw_cap.wd_num = CONFIG_TX_WD_NUM;
+#else
+	phl_com->bus_sw_cap.wd_num = 0;
+#endif
+
+	phl_com->bus_sw_cap.txbd_num = CORE_TXBD_NUM;
+	phl_com->bus_sw_cap.rxbd_num = CORE_RXBD_NUM;
+
+	phl_com->bus_sw_cap.rxbuf_size = CORE_RXBUF_SIZE;
+	phl_com->bus_sw_cap.rxbuf_num = CORE_RXBUF_NUM;
+	phl_com->bus_sw_cap.rpbuf_size = CORE_RPBUF_SIZE;
+	phl_com->bus_sw_cap.rpbuf_num = CORE_RPBUF_NUM;
+	phl_com->bus_sw_cap.rpbd_num = CORE_RPBD_NUM;
+
+#ifdef CONFIG_PCIE_TRX_MIT_DYN
+	mit_info.tx_timer = 0;
+	mit_info.tx_counter = 0;
+	mit_info.rx_timer = 0;
+	mit_info.rx_counter = 0;
+	mit_info.fixed_mitigation = 0;
+	mit_info.rx_mit_timer_high = PCIE_RX_INT_MIT_TIMER;
+	mit_info.rx_mit_counter_high = 0;
+	rtw_phl_pcie_trx_mit_cfg(phl, &mit_info); /* send to the phl cfg */
+#endif
+#ifndef CONFIG_RTW_PCI_MSI_DISABLE
+	pci_data->msi_en = rtw_msi_en;
+#endif
+#endif /*CONFIG_PCI_HCI*/
+
+#ifdef CONFIG_BTC
+	phl_com->dev_sw_cap.btc_mode = BTC_MODE_NORMAL;
+#ifdef CONFIG_BTC_EXT_CFG_ESOC_TYPE
+	phl_com->dev_sw_cap.btc_esoc_type = rtw_btc_update_ext_soc_type(rtw_btc_ext_soc_type);
+#endif
+#ifdef CONFIG_BTC_EXT_CFG_ANT_ISO_DB
+	phl_com->dev_sw_cap.btc_ant_iso_db = rtw_btc_ant_iso_db;
+#endif
+#ifdef CONFIG_BTC_TRXSS_CHG
+	if (rtw_btc_trxss_chg)
+		GET_DEV_SW_BTC_CAP(phl_com).btc_deg_wifi_cap |= BTC_DRG_WIFI_CAP_TRX1SS;
+#endif
+#else
+	phl_com->dev_sw_cap.btc_mode = BTC_MODE_WL;
+#endif
+#ifdef CONFIG_MCC_MODE
+	phl_com->dev_sw_cap.mcc_sup = rtw_mcc_en;
+#endif
+#ifdef CONFIG_USB_HCI
+	phl_com->bus_sw_cap.tx_buf_size = MAX_XMITBUF_SZ;
+	phl_com->bus_sw_cap.tx_buf_num = NR_XMITBUFF;
+	phl_com->bus_sw_cap.tx_mgnt_buf_size = MAX_MGNT_XMITBUF_SZ;
+	phl_com->bus_sw_cap.tx_mgnt_buf_num = NR_MGNT_XMITBUFF;
+	phl_com->bus_sw_cap.rx_buf_size = MAX_RECVBUF_SZ;
+	phl_com->bus_sw_cap.rx_buf_align_size = RECVBUF_SZ_ALIGN_SZ;
+	phl_com->bus_sw_cap.rx_buf_num = NR_RECVBUFF;
+	phl_com->bus_sw_cap.in_token_num = NR_RECV_URB;
+#endif
+#ifdef CONFIG_SDIO_HCI
+#ifdef MAX_XMITBUF_SZ
+	phl_com->bus_sw_cap.tx_buf_size = MAX_XMITBUF_SZ;
+#endif
+#ifdef NR_XMITBUFF
+	phl_com->bus_sw_cap.tx_buf_num = NR_XMITBUFF;
+#endif
+#ifdef MAX_MGNT_XMITBUF_SZ
+	phl_com->bus_sw_cap.tx_mgnt_buf_size = MAX_MGNT_XMITBUF_SZ;
+#endif
+#ifdef NR_MGNT_XMITBUFF
+	phl_com->bus_sw_cap.tx_mgnt_buf_num = NR_MGNT_XMITBUFF;
+#endif
+#ifdef MAX_RECVBUF_SZ
+	phl_com->bus_sw_cap.rx_buf_size = MAX_RECVBUF_SZ;
+#endif
+#ifdef NR_RECVBUFF
+	phl_com->bus_sw_cap.rx_buf_num = NR_RECVBUFF;
+#endif
+#endif /* CONFIG_SDIO_HCI */
+
+	/* Set STBC Tx/Rx sw role cap */
+	phl_com->role_sw_cap.stbc_cap = 0;
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT5) ? HW_CAP_STBC_HT_TX : 0;
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT1) ? HW_CAP_STBC_VHT_TX : 0;
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT10) ? HW_CAP_STBC_HE_TX : 0;
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT11) ? HW_CAP_STBC_HE_TX_GT_80M : 0;
+
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT4) ? HW_CAP_STBC_HT_RX : 0;
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT0) ? HW_CAP_STBC_VHT_RX : 0;
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT8) ? HW_CAP_STBC_HE_RX : 0;
+	phl_com->role_sw_cap.stbc_cap |=
+		(rtw_stbc_cap & BIT9) ? HW_CAP_STBC_HE_RX_GT_80M : 0;
+
+	/*Band0*/
+	phl_com->proto_sw_cap[0].stbc_ht_tx = (rtw_stbc_cap & BIT5) ? 1 : 0;
+	phl_com->proto_sw_cap[0].stbc_vht_tx = (rtw_stbc_cap & BIT1) ? 1 : 0;
+	phl_com->proto_sw_cap[0].stbc_he_tx = (rtw_stbc_cap & BIT10) ? 1 : 0;
+	phl_com->proto_sw_cap[0].stbc_tx_greater_80mhz = (rtw_stbc_cap & BIT11) ? 1 : 0;
+
+	phl_com->proto_sw_cap[0].stbc_ht_rx = (rtw_stbc_cap & BIT4) ? 1 : 0;
+	phl_com->proto_sw_cap[0].stbc_vht_rx = (rtw_stbc_cap & BIT0) ? 1 : 0;
+	phl_com->proto_sw_cap[0].stbc_he_rx = (rtw_stbc_cap & BIT8) ? 1 : 0;
+	phl_com->proto_sw_cap[0].stbc_rx_greater_80mhz = (rtw_stbc_cap & BIT9) ? 1 : 0;
+
+	/*Band1*/
+	phl_com->proto_sw_cap[1].stbc_ht_tx = (rtw_stbc_cap & BIT5) ? 1 : 0;
+	phl_com->proto_sw_cap[1].stbc_vht_tx = (rtw_stbc_cap & BIT1) ? 1 : 0;
+	phl_com->proto_sw_cap[1].stbc_he_tx = (rtw_stbc_cap & BIT10) ? 1 : 0;
+	phl_com->proto_sw_cap[1].stbc_tx_greater_80mhz = (rtw_stbc_cap & BIT11) ? 1 : 0;
+
+	phl_com->proto_sw_cap[1].stbc_ht_rx = (rtw_stbc_cap & BIT4) ? 1 : 0;
+	phl_com->proto_sw_cap[1].stbc_vht_rx = (rtw_stbc_cap & BIT0) ? 1 : 0;
+	phl_com->proto_sw_cap[1].stbc_he_rx = (rtw_stbc_cap & BIT8) ? 1 : 0;
+	phl_com->proto_sw_cap[1].stbc_rx_greater_80mhz = (rtw_stbc_cap & BIT9) ? 1 : 0;
+
+#ifdef CONFIG_BEAMFORMING
+	if (rtw_wifi_spec == 1) {
+		if ((rtw_beamform_cap & BIT(3)) == 0)
+			RTW_INFO("%s: enable VHT MU-MIMO Beamformee for Logo Test!\n", __func__);
+
+		rtw_beamform_cap |= BIT(3);
+		rtw_sw_proto_bf_cap_phy0 |= BIT(3);
+		rtw_sw_proto_bf_cap_phy1 |= BIT(3);
+	}
+
+	phl_com->role_sw_cap.bf_cap = 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT0) ? HW_CAP_BFER_VHT_SU : 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT1) ? HW_CAP_BFEE_VHT_SU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT2) ? HW_CAP_BFER_VHT_MU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT3) ? HW_CAP_BFEE_VHT_MU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT4) ? HW_CAP_BFER_HT_SU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT5) ? HW_CAP_BFEE_HT_SU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT6) ? HW_CAP_BFER_HE_SU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT7) ? HW_CAP_BFEE_HE_SU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT8) ? HW_CAP_BFER_HE_MU: 0;
+	phl_com->role_sw_cap.bf_cap |= (rtw_beamform_cap & BIT9) ? HW_CAP_BFEE_HE_MU: 0;
+
+	/*Band0*/
+	phl_com->proto_sw_cap[0].vht_su_bfmr = (rtw_sw_proto_bf_cap_phy0 & BIT0) ? 1 : 0;
+	phl_com->proto_sw_cap[0].vht_su_bfme = (rtw_sw_proto_bf_cap_phy0 & BIT1) ? 1 : 0;
+	phl_com->proto_sw_cap[0].vht_mu_bfmr = (rtw_sw_proto_bf_cap_phy0 & BIT2) ? 1 : 0;
+	phl_com->proto_sw_cap[0].vht_mu_bfme = (rtw_sw_proto_bf_cap_phy0 & BIT3) ? 1 : 0;
+	phl_com->proto_sw_cap[0].ht_su_bfmr = (rtw_sw_proto_bf_cap_phy0 & BIT4) ? 1 : 0;
+	phl_com->proto_sw_cap[0].ht_su_bfme = (rtw_sw_proto_bf_cap_phy0 & BIT5) ? 1 : 0;
+	phl_com->proto_sw_cap[0].he_su_bfmr = (rtw_sw_proto_bf_cap_phy0 & BIT6) ? 1 : 0;
+	phl_com->proto_sw_cap[0].he_su_bfme = (rtw_sw_proto_bf_cap_phy0 & BIT7) ? 1 : 0;
+	phl_com->proto_sw_cap[0].he_mu_bfmr = (rtw_sw_proto_bf_cap_phy0 & BIT8) ? 1 : 0;
+	phl_com->proto_sw_cap[0].he_mu_bfme = (rtw_sw_proto_bf_cap_phy0 & BIT9) ? 1 : 0;
+
+	/*Band1*/
+	phl_com->proto_sw_cap[1].vht_su_bfmr = (rtw_sw_proto_bf_cap_phy1 & BIT0) ? 1 : 0;
+	phl_com->proto_sw_cap[1].vht_su_bfme = (rtw_sw_proto_bf_cap_phy1 & BIT1) ? 1 : 0;
+	phl_com->proto_sw_cap[1].vht_mu_bfmr = (rtw_sw_proto_bf_cap_phy1 & BIT2) ? 1 : 0;
+	phl_com->proto_sw_cap[1].vht_mu_bfme = (rtw_sw_proto_bf_cap_phy1 & BIT3) ? 1 : 0;
+	phl_com->proto_sw_cap[1].ht_su_bfmr = (rtw_sw_proto_bf_cap_phy1 & BIT4) ? 1 : 0;
+	phl_com->proto_sw_cap[1].ht_su_bfme = (rtw_sw_proto_bf_cap_phy1 & BIT5) ? 1 : 0;
+	phl_com->proto_sw_cap[1].he_su_bfmr = (rtw_sw_proto_bf_cap_phy1 & BIT6) ? 1 : 0;
+	phl_com->proto_sw_cap[1].he_su_bfme = (rtw_sw_proto_bf_cap_phy1 & BIT7) ? 1 : 0;
+	phl_com->proto_sw_cap[1].he_mu_bfmr = (rtw_sw_proto_bf_cap_phy1 & BIT8) ? 1 : 0;
+	phl_com->proto_sw_cap[1].he_mu_bfme = (rtw_sw_proto_bf_cap_phy1 & BIT9) ? 1 : 0;
+#endif
+
+#if CONFIG_TXPWR_LIMIT
+	if (rtw_tx_pwr_lmt_enable == 2)
+		phl_com->dev_sw_cap.pwrlmt_type = RTW_PWLMT_BY_EFUSE;
+	else if (rtw_tx_pwr_lmt_enable == 1)
+		phl_com->dev_sw_cap.pwrlmt_type = RTW_PWBYRATE_AND_PWLMT;
+	else if (rtw_tx_pwr_lmt_enable == 0)
+		phl_com->dev_sw_cap.pwrlmt_type = RTW_PWLMT_DISABLE;
+#else
+	phl_com->dev_sw_cap.pwrlmt_type = RTW_PWLMT_DISABLE;
+#endif
+
+	if (rtw_tx_pwr_by_rate == 1)
+		phl_com->dev_sw_cap.pwrbyrate_off = RTW_PW_BY_RATE_ON;
+	else if (rtw_tx_pwr_by_rate == 0)
+		phl_com->dev_sw_cap.pwrbyrate_off = RTW_PW_BY_RATE_ALL_SAME;
+	else
+		phl_com->dev_sw_cap.pwrbyrate_off = RTW_PW_BY_RATE_ON;
+
+	/* If rf_board_opt is not assigned to specific value, it must be set to 0xFF as default. */
+	phl_com->dev_sw_cap.rf_board_opt = 0xFF;
+
+#ifdef CONFIG_POWER_SAVE
+	phl_com->dev_sw_cap.ps_cap.leave_fail_act = PS_LEAVE_FAIL_SER_L1_RECOVERY;
+#ifdef CONFIG_RTW_WKARD_PS_DEFAULT_OFF
+	phl_com->dev_sw_cap.ps_cap.init_rt_stop_rson = PS_RT_DEBUG;
+#endif /* CONFIG_RTW_WKARD_PS_DEFAULT_OFF */
+#ifdef CONFIG_RTW_IPS
+	rtw_update_ips_setting(RTW_IPS_MODE, rtw_ips_mode,
+			       &phl_com->dev_sw_cap.ps_cap.ips_en,
+			       &phl_com->dev_sw_cap.ps_cap.ips_cap, _FALSE);
+#endif /* CONFIG_RTW_IPS */
+#ifdef CONFIG_RTW_LPS
+	rtw_update_lps_setting(RTW_LPS_MODE, rtw_lps_mode,
+			       &phl_com->dev_sw_cap.ps_cap.lps_en,
+			       &phl_com->dev_sw_cap.ps_cap.lps_cap, _FALSE);
+	rtw_update_lps_proto_cap_setting(RTW_PS_PROTO_LISTEN_BEACON_MODE,
+					 (u8)rtw_lps_listen_bcn_mode,
+					 (u8)rtw_lps_awake_interval,
+					 &phl_com->dev_sw_cap.ps_cap, _FALSE);
+#endif /* CONFIG_RTW_LPS */
+#ifdef CONFIG_WOWLAN
+#ifdef CONFIG_RTW_IPS_WOW
+	rtw_update_ips_setting(RTW_WOW_IPS_MODE, rtw_ips_wow_mode,
+			       &phl_com->dev_sw_cap.ps_cap.ips_wow_en,
+			       &phl_com->dev_sw_cap.ps_cap.ips_wow_cap, _TRUE);
+#endif /* CONFIG_RTW_IPS_WOW */
+#ifdef CONFIG_RTW_LPS_WOW
+	rtw_update_lps_setting(RTW_WOW_LPS_MODE, rtw_lps_wow_mode,
+			       &phl_com->dev_sw_cap.ps_cap.lps_wow_en,
+			       &phl_com->dev_sw_cap.ps_cap.lps_wow_cap, _TRUE);
+	rtw_update_lps_proto_cap_setting(RTW_PS_PROTO_LISTEN_BEACON_MODE,
+					 (u8)rtw_lps_wow_listen_bcn_mode,
+					 (u8)rtw_lps_wow_awake_interval,
+					 &phl_com->dev_sw_cap.ps_cap, _TRUE);
+#endif /* CONFIG_RTW_LPS_WOW */
+#endif /* CONFIG_WOWLAN */
+#endif /* CONFIG_POWER_SAVE */
+
+	phl_com->dev_sw_cap.rfk_cap = rtw_rfk_ability;
+
+	phl_com->dev_sw_cap.edcca_cap.edcca_adap_th_2g = rtw_edcca_th_2g;
+	phl_com->dev_sw_cap.edcca_cap.edcca_adap_th_5g = rtw_edcca_th_5g;
+	phl_com->dev_sw_cap.edcca_cap.edcca_carrier_sense_th = rtw_edcca_cs_th;
+#if CONFIG_IEEE80211_BAND_6GHZ
+	phl_com->dev_sw_cap.edcca_cap.edcca_cbp_th_6g = rtw_edcca_cbp_th_6g;
+#endif
+
+	/* ref: rtw_update_phl_iot() */
+	for (i = 0; i < MAX_WIFI_ROLE_NUMBER; i++)
+		phl_com->id.iot_id[i] = IOT_ID(0);
+	phl_com->id.id = RTW_IOT_ID;
+#ifdef CONFIG_RTW_LED
+	rtw_phl_led_set_ctrl_mode(GET_PHL_INFO(dvobj), 0, RTW_LED_CTRL_HW_TX_MODE);
+#endif
+
+#if defined (CONFIG_RPQ_AGG_NUM) && (CONFIG_RPQ_AGG_NUM > 0)
+	phl_com->dev_sw_cap.rpq_agg_num = CONFIG_RPQ_AGG_NUM;
+#else
+	phl_com->dev_sw_cap.rpq_agg_num = 0; /* MAC default num: 121 for all IC */
+#endif
+
+	phl_com->dev_sw_cap.quota_turbo = rtw_quota_turbo_en;
+	if (rtw_wifi_spec == 1)
+		phl_com->drv_mode = RTW_DRV_MODE_LOGO_TEST;
+
+#ifdef CONFIG_HW_FORM_SEC_HEADER
+	dvobj->phl_com->dev_sw_cap.sec_cap.hw_form_hdr = true;
+#else
+	dvobj->phl_com->dev_sw_cap.sec_cap.hw_form_hdr = false;
+#endif
+	dvobj->phl_com->dev_sw_cap.sec_cap.hw_sec_iv = false;
+	dvobj->phl_com->dev_sw_cap.sec_cap.hw_tx_search_key = false;
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	/* currently we only support to be TWT Requester */
+	phl_com->dev_sw_cap.twt_sup = RTW_PHL_TWT_REQ_SUP | RTW_PHL_TWT_BC_SUP;
+#endif
+#endif
+
+#ifdef CONFIG_THERMAL_PROTECT
+	phl_com->dev_sw_cap.min_tx_duty = rtw_thermal_min_duty;
+	phl_com->dev_sw_cap.thermal_threshold = rtw_thermal_threshold;
+#endif /* CONFIG_THERMAL_PROTECT */
+
+#ifdef CONFIG_SCANOFLD
+	phl_com->dev_sw_cap.scan_ofld = rtw_scan_fw_ofld;
+#endif
+#ifdef CONFIG_RTW_CSI_CHANNEL_INFO
+	phl_com->dev_sw_cap.sensing_csi = rtw_sensing_csi;
+#endif
+
+#ifdef RTW_WKARD_6G_RTS_CHK
+	phl_com->rsp_static_rts_chk_off = rtw_rts_chk_off;
+#endif
+}
+
+u8 rtw_load_dvobj_registry(struct dvobj_priv *dvobj)
+{
+	/*struct rtw_phl_com_t *phl_com = GET_PHL_COM(dvobj);*/
+	#ifdef CONFIG_CONCURRENT_MODE
+	dvobj->virtual_iface_num = (u8)rtw_virtual_iface_num;
+	#endif
+	return _SUCCESS;
+}
+
+uint rtw_load_registry(_adapter *adapter)
+{
+	uint status = _SUCCESS;
+	struct registry_priv  *registry_par = &adapter->registrypriv;
+
+#ifdef CONFIG_DBCC_P2P_BG_LISTEN_SIM
+	registry_par->dbcc_lg_sim = rtw_dbcc_lg_sim;
+#endif
+
+#ifdef CONFIG_RTW_DEBUG
+	if (rtw_drv_log_level >= _DRV_MAX_)
+		rtw_drv_log_level = _DRV_DEBUG_;
+#endif
+
+	registry_par->chip_version = (u8)rtw_chip_version;
+	registry_par->rfintfs = (u8)rtw_rfintfs;
+	registry_par->lbkmode = (u8)rtw_lbkmode;
+	/* registry_par->hci = (u8)hci; */
+	registry_par->network_mode  = (u8)rtw_network_mode;
+
+	_rtw_memcpy(registry_par->ssid.Ssid, "ANY", 3);
+	registry_par->ssid.SsidLength = 3;
+
+	registry_par->band = (u8)rtw_band;
+	registry_par->channel = (u8)rtw_channel;
+#ifdef CONFIG_NARROWBAND_SUPPORTING
+	if (rtw_nb_config != RTW_NB_CONFIG_NONE)
+		rtw_wireless_mode &= ~WIRELESS_11B;
+#endif
+	registry_par->wireless_mode = (u8)rtw_wireless_mode;
+	registry_par->band_type = (u8)rtw_band_type;
+
+	if (is_supported_24g(registry_par->band_type) && (!is_supported_5g(registry_par->band_type))
+		&& (registry_par->channel > 14)) {
+		registry_par->band = BAND_ON_24G;
+		registry_par->channel = 1;
+	} else if (is_supported_5g(registry_par->band_type) && (!is_supported_24g(registry_par->band_type))
+		&& (registry_par->channel <= 14)) {
+		registry_par->band = BAND_ON_5G;
+		registry_par->channel = 36;
+	} else if (is_supported_6g(registry_par->band_type) && !is_supported_24g(registry_par->band_type)
+		&& !is_supported_5g(registry_par->band_type) && (registry_par->band != BAND_ON_6G)){
+		registry_par->band = BAND_ON_6G;
+		registry_par->channel = 33;
+	}
+
+	rtw_regsty_load_edcca_mode_settings(registry_par);
+
+	registry_par->adaptivity_idle_probability = (u8)rtw_adaptivity_idle_probability;
+	if (registry_par->adaptivity_idle_probability == 1) {
+		rtw_vrtl_carrier_sense = DISABLE_VCS;
+		rtw_vcs_type = NONE_VCS;
+		rtw_hw_rts_en = 0;
+	}
+
+	registry_par->vrtl_carrier_sense = (u8)rtw_vrtl_carrier_sense ;
+	registry_par->vcs_type = (u8)rtw_vcs_type;
+	registry_par->rts_thresh = (u16)rtw_rts_thresh;
+	registry_par->hw_rts_en = (u8)rtw_hw_rts_en;
+	registry_par->frag_thresh = (u16)rtw_frag_thresh;
+	registry_par->preamble = (u8)rtw_preamble;
+	registry_par->scan_mode = (u8)rtw_scan_mode;
+#ifdef CONFIG_TDMADIG
+	registry_par->tdmadig_en = (u8)rtw_tdmadig_en;
+	registry_par->tdmadig_mode = (u8)rtw_tdmadig_mode;
+	registry_par->tdmadig_dynamic = (u8) rtw_dynamic_tdmadig;
+#endif /* CONFIG_TDMADIG */
+#ifdef CONFIG_POWER_SAVE
+#ifdef CONFIG_RTW_IPS
+	rtw_update_ips_setting(RTW_IPS_MODE, rtw_ips_mode,
+			       &registry_par->ips_mode, &registry_par->ips_cap,
+			       _FALSE);
+#endif /* CONFIG_RTW_IPS */
+#ifdef CONFIG_RTW_LPS
+	rtw_update_lps_setting(RTW_LPS_MODE, rtw_lps_mode,
+			       &registry_par->lps_mode, &registry_par->lps_cap,
+			       _FALSE);
+	rtw_update_lps_proto_registry_setting(RTW_PS_PROTO_LISTEN_BEACON_MODE,
+					      (u8)rtw_lps_listen_bcn_mode,
+					      (u8)rtw_lps_awake_interval,
+					      registry_par, _FALSE);
+#endif /* CONFIG_RTW_LPS */
+#ifdef CONFIG_WOWLAN
+#ifdef CONFIG_RTW_IPS_WOW
+	rtw_update_ips_setting(RTW_WOW_IPS_MODE, rtw_ips_wow_mode,
+			       &registry_par->ips_wow_mode,
+			       &registry_par->ips_wow_cap, _TRUE);
+#endif /* CONFIG_RTW_IPS_WOW */
+#ifdef CONFIG_RTW_LPS_WOW
+	rtw_update_lps_setting(RTW_WOW_LPS_MODE, rtw_lps_wow_mode,
+			       &registry_par->lps_wow_mode,
+			       &registry_par->lps_wow_cap, _TRUE);
+	rtw_update_lps_proto_registry_setting(RTW_PS_PROTO_LISTEN_BEACON_MODE,
+					      (u8)rtw_lps_wow_listen_bcn_mode,
+					      (u8)rtw_lps_wow_awake_interval,
+					      registry_par, _TRUE);
+#endif /* CONFIG_RTW_LPS_WOW */
+#endif /* CONFIG_WOWLAN */
+#endif /* CONFIG_POWER_SAVE */
+	registry_par->en_dyn_rrsr = (u8)rtw_en_dyn_rrsr;
+	registry_par->set_rrsr_value = (u32)rtw_rrsr_value;
+
+	registry_par->radio_enable = (u8)rtw_radio_enable;
+	registry_par->long_retry_lmt = (u8)rtw_long_retry_lmt;
+	registry_par->short_retry_lmt = (u8)rtw_short_retry_lmt;
+	registry_par->busy_thresh = (u16)rtw_busy_thresh;
+	registry_par->max_bss_cnt = (u16)rtw_max_bss_cnt;
+	/* registry_par->qos_enable = (u8)rtw_qos_enable; */
+	registry_par->ack_policy = (u8)rtw_ack_policy;
+	registry_par->mp_mode = (u8)rtw_mp_mode;
+#if defined(CONFIG_MP_INCLUDED) && defined(CONFIG_RTW_CUSTOMER_STR)
+	registry_par->mp_customer_str = (u8)rtw_mp_customer_str;
+#endif
+	registry_par->software_encrypt = (u8)rtw_software_encrypt;
+	registry_par->software_decrypt = (u8)rtw_software_decrypt;
+
+	registry_par->acm_method = (u8)rtw_acm_method;
+	registry_par->usb_rxagg_mode = (u8)rtw_usb_rxagg_mode;
+	registry_par->dynamic_agg_enable = (u8)rtw_dynamic_agg_enable;
+
+	/* WMM */
+	registry_par->wmm_enable = (u8)rtw_wmm_enable;
+
+	registry_par->RegPwrTrimEnable = (u8)rtw_pwrtrim_enable;
+
+	registry_par->tx_bw_mode = (u16)rtw_tx_bw_mode;
+
+#ifdef CONFIG_80211N_HT
+	registry_par->ht_enable = (u8)rtw_ht_enable;
+	if (registry_par->ht_enable && is_supported_ht(registry_par->wireless_mode)) {
+#ifdef CONFIG_NARROWBAND_SUPPORTING
+	if (rtw_nb_config != RTW_NB_CONFIG_NONE)
+		rtw_bw_mode = 0;
+#endif
+		registry_par->bw_mode = (u16)rtw_bw_mode;
+		registry_par->ampdu_enable = (u8)rtw_ampdu_enable;
+		registry_par->rx_ampdu_amsdu = (u8)rtw_rx_ampdu_amsdu;
+#ifdef CONFIG_DISBALE_RX_AMSDU_FOR_BUS_LOW_SPEED
+#ifdef CONFIG_USB_HCI
+		if (dvobj_to_usb(adapter_to_dvobj(adapter))->usb_speed < RTW_USB_SPEED_SUPER)
+			registry_par->rx_ampdu_amsdu = 0;
+#endif
+#endif
+		registry_par->tx_ampdu_amsdu = (u8)rtw_tx_ampdu_amsdu;
+		registry_par->tx_ampdu_num = (u8)rtw_tx_ampdu_num;
+		registry_par->tx_quick_addba_req = (u8)rtw_quick_addba_req;
+		registry_par->short_gi = (u8)rtw_short_gi;
+		registry_par->ldpc_cap = (u8)rtw_ldpc_cap;
+
+#ifdef CONFIG_RTW_TX_NPATH_EN
+		registry_par->tx_npath = (u8)rtw_tx_npath_enable;
+#endif
+#ifdef CONFIG_RTW_PATH_DIV
+		registry_par->path_div = (u8)rtw_path_div_enable;
+#endif
+		registry_par->stbc_cap = (u16)rtw_stbc_cap;
+#ifdef CONFIG_BEAMFORMING
+		registry_par->beamform_cap = (u16)rtw_beamform_cap;
+		registry_par->dyn_txbf = (u8)rtw_dyn_txbf;
+		registry_par->beamformer_rf_num = (u8)rtw_bfer_rf_number;
+		registry_par->beamformee_rf_num = (u8)rtw_bfee_rf_number;
+#endif
+		rtw_regsty_init_rx_ampdu_sz_limit(registry_par);
+	}
+#endif
+
+#if 0
+int rtw_short_gi = 0xf;
+/* BIT0: Enable VHT LDPC Rx, BIT1: Enable VHT LDPC Tx, BIT4: Enable HT LDPC Rx, BIT5: Enable HT LDPC Tx */
+int rtw_ldpc_cap = 0x33;
+/* BIT0: Enable VHT STBC Rx, BIT1: Enable VHT STBC Tx, BIT4: Enable HT STBC Rx, BIT5: Enable HT STBC Tx */
+int rtw_stbc_cap = 0x13;
+#endif
+
+#ifdef DBG_LA_MODE
+	registry_par->la_mode_en = (u8)rtw_la_mode_en;
+#endif
+#ifdef CONFIG_NARROWBAND_SUPPORTING
+	registry_par->rtw_nb_config = (u8)rtw_nb_config;
+#endif
+
+#ifdef CONFIG_80211AC_VHT
+	registry_par->vht_enable = (u8)rtw_vht_enable;
+	registry_par->vht_24g_enable = (u8)rtw_vht_24g_enable;
+	registry_par->ampdu_factor = (u8)rtw_ampdu_factor;
+	registry_par->vht_rx_mcs_map[0] = (u8)(rtw_vht_rx_mcs_map & 0xFF);
+	registry_par->vht_rx_mcs_map[1] = (u8)((rtw_vht_rx_mcs_map & 0xFF00) >> 8);
+#endif
+
+#ifdef CONFIG_80211AX_HE
+	registry_par->he_enable = (u8)rtw_he_enable;
+#endif
+
+#ifdef CONFIG_TX_EARLY_MODE
+	registry_par->early_mode = (u8)rtw_early_mode;
+#endif
+	registry_par->lowrate_two_xmit = (u8)rtw_lowrate_two_xmit;
+	registry_par->rf_path = (u8)rtw_rf_path; /*rf_config/rtw_rf_config*/
+	registry_par->tx_nss = (u8)rtw_tx_nss;
+	registry_par->rx_nss = (u8)rtw_rx_nss;
+	registry_par->low_power = (u8)rtw_low_power;
+
+	registry_par->check_hw_status = (u8)rtw_check_hw_status;
+
+	registry_par->wifi_spec = (u8)rtw_wifi_spec;
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+	if (regd_src_is_valid(rtw_regd_src))
+		registry_par->regd_src = (u8)rtw_regd_src;
+	else {
+		RTW_WARN("%s invalid rtw_regd_src(%u), use REGD_SRC_RTK_PRIV instead\n", __func__, rtw_regd_src);
+		registry_par->regd_src = REGD_SRC_RTK_PRIV;
+	}
+	registry_par->regd_src_os_11d = !!rtw_regd_src_os_11d;
+#endif
+	registry_par->init_regd_always_apply = !!rtw_init_regd_always_apply;
+	registry_par->user_regd_always_apply = !!rtw_user_regd_always_apply;
+	rtw_regsty_load_alpha2(registry_par);
+	rtw_regsty_load_chplan(registry_par);
+	rtw_regsty_load_addl_ch_disable_conf(registry_par);
+	registry_par->bcn_hint_valid_ms = rtw_bcn_hint_valid_ms;
+	rtw_regsty_load_env_settings(registry_par);
+#ifdef CONFIG_80211D
+	rtw_regsty_load_country_ie_slave_settings(registry_par);
+#endif
+
+	registry_par->full_ch_in_p2p_handshake = (u8)rtw_full_ch_in_p2p_handshake;
+
+	registry_par->bAcceptAddbaReq = (u8)rtw_AcceptAddbaReq;
+
+	registry_par->antdiv_cfg = (u8)rtw_antdiv_cfg;
+	registry_par->antdiv_type = (u8)rtw_antdiv_type;
+
+	registry_par->drv_ant_band_switch = (u8) rtw_drv_ant_band_switch;
+
+	registry_par->switch_usb_mode = (u8)rtw_switch_usb_mode;
+
+	registry_par->hw_wps_pbc = (u8)rtw_hw_wps_pbc;
+
+#ifdef CONFIG_ADAPTOR_INFO_CACHING_FILE
+	snprintf(registry_par->adaptor_info_caching_file_path, PATH_LENGTH_MAX, "%s", rtw_adaptor_info_caching_file_path);
+	registry_par->adaptor_info_caching_file_path[PATH_LENGTH_MAX - 1] = 0;
+#endif
+
+#ifdef CONFIG_LAYER2_ROAMING
+	registry_par->max_roaming_times = (u8)rtw_max_roaming_times;
+#endif
+
+	snprintf(registry_par->ifname, 16, "%s", ifname);
+	snprintf(registry_par->if2name, 16, "%s", if2name);
+#if defined(CONFIG_PLATFORM_ANDROID) && (CONFIG_IFACE_NUMBER > 2)
+	snprintf(registry_par->if3name, 16, "%s", if3name);
+#endif
+
+#if defined(CONFIG_CONCURRENT_MODE) && !RTW_P2P_GROUP_INTERFACE
+#ifdef CONFIG_P2P
+	if (CONFIG_RTW_STATIC_NDEV_NUM <= rtw_sel_p2p_iface) {
+		RTW_ERR("rtw_sel_p2p_iface out of range\n");
+		rtw_sel_p2p_iface = CONFIG_RTW_STATIC_NDEV_NUM - 1;
+	}
+
+	registry_par->sel_p2p_iface = (u8)rtw_sel_p2p_iface;
+	RTW_INFO("%s, Select P2P interface: iface_id:%d\n", __func__, registry_par->sel_p2p_iface);
+#endif
+#endif
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+	registry_par->ignore_go_in_scan = (u8)rtw_ignore_go_in_scan;
+	registry_par->ignore_low_rssi_in_scan = rtw_ignore_low_rssi_in_scan;
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+	registry_par->vo_edca = rtw_vo_edca;
+
+	registry_par->pll_ref_clk_sel = (u8)rtw_pll_ref_clk_sel;
+
+	rtw_regsty_load_target_tx_power(registry_par);
+
+	registry_par->TxBBSwing_2G = (s8)rtw_TxBBSwing_2G;
+	registry_par->TxBBSwing_5G = (s8)rtw_TxBBSwing_5G;
+	registry_par->bEn_RFE = 1;
+
+	registry_par->PowerTracking_Type = (u8)rtw_powertracking_type;
+	registry_par->AmplifierType_2G = (u8)rtw_amplifier_type_2g;
+	registry_par->AmplifierType_5G = (u8)rtw_amplifier_type_5g;
+	registry_par->GLNA_Type = (u8)rtw_GLNA_type;
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	registry_par->load_phy_file = (u8)rtw_load_phy_file;
+	registry_par->RegDecryptCustomFile = (u8)rtw_decrypt_phy_file;
+#endif
+	registry_par->qos_opt_enable = (u8)rtw_qos_opt_enable;
+
+	registry_par->hiq_filter = (u8)rtw_hiq_filter;
+
+	registry_par->boffefusemask = (u8)rtw_OffEfuseMask;
+	registry_par->bFileMaskEfuse = (u8)rtw_FileMaskEfuse;
+	registry_par->bBTFileMaskEfuse = (u8)rtw_FileMaskEfuse;
+
+#ifdef CONFIG_RTW_ACS
+	registry_par->acs_mode = (u8)rtw_acs;
+	registry_par->acs_auto_scan = (u8)rtw_acs_auto_scan;
+#endif
+
+	registry_par->reg_rxgain_offset_2g = (u32) rtw_rxgain_offset_2g;
+	registry_par->reg_rxgain_offset_5gl = (u32) rtw_rxgain_offset_5gl;
+	registry_par->reg_rxgain_offset_5gm = (u32) rtw_rxgain_offset_5gm;
+	registry_par->reg_rxgain_offset_5gh = (u32) rtw_rxgain_offset_5gh;
+
+#ifdef CONFIG_DFS_MASTER
+	rtw_regsty_load_dfs_region_domain_settings(registry_par);
+#endif
+
+#ifdef CONFIG_WOWLAN
+	registry_par->wowlan_enable = rtw_wow_enable;
+	registry_par->wakeup_event = rtw_wakeup_event;
+	registry_par->suspend_type = rtw_suspend_type;
+#endif
+
+	registry_par->wowlan_sta_mix_mode = rtw_wowlan_sta_mix_mode;
+
+#ifdef CONFIG_PCI_HCI
+	registry_par->pci_aspm_config = rtw_pci_aspm_enable;
+#endif
+
+#ifdef CONFIG_RTW_NAPI
+	registry_par->en_napi = (u8)rtw_en_napi;
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+	registry_par->napi_threshold = (u32)rtw_napi_threshold;
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+#ifdef CONFIG_RTW_GRO
+	registry_par->en_gro = (u8)rtw_en_gro;
+	if (!registry_par->en_napi && registry_par->en_gro) {
+		registry_par->en_gro = 0;
+		RTW_WARN("Disable GRO because NAPI is not enabled\n");
+	}
+#endif /* CONFIG_RTW_GRO */
+#endif /* CONFIG_RTW_NAPI */
+
+	registry_par->iqk_fw_offload = (u8)rtw_iqk_fw_offload;
+	registry_par->ch_switch_offload = (u8)rtw_ch_switch_offload;
+
+#ifdef CONFIG_TDLS
+	registry_par->en_tdls = rtw_en_tdls;
+#endif
+
+
+#ifdef CONFIG_FW_OFFLOAD_PARAM_INIT
+	registry_par->fw_param_init = rtw_fw_param_init;
+#endif
+#ifdef CONFIG_AP_MODE
+	registry_par->max_ap_assoc_sta = (u8)rtw_max_ap_assoc_sta;
+	registry_par->bmc_tx_rate = rtw_bmc_tx_rate;
+	#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+	registry_par->ap_src_b2u_flags = rtw_ap_src_b2u_flags;
+	registry_par->ap_fwd_b2u_flags = rtw_ap_fwd_b2u_flags;
+	#endif
+#endif /* CONFIG_AP_MODE */
+
+#ifdef CONFIG_RTW_MESH
+	#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	registry_par->msrc_b2u_flags = rtw_msrc_b2u_flags;
+	registry_par->mfwd_b2u_flags = rtw_mfwd_b2u_flags;
+	#endif
+#endif /* CONFIG_RTW_MESH */
+
+	registry_par->phydm_ability = rtw_phydm_ability;
+	registry_par->halrf_ability = rtw_halrf_ability;
+#ifdef CONFIG_RTW_MESH
+	registry_par->peer_alive_based_preq = rtw_peer_alive_based_preq;
+#endif
+
+#ifdef RTW_BUSY_DENY_SCAN
+	registry_par->scan_interval_thr = rtw_scan_interval_thr;
+#endif
+
+#ifdef CONFIG_RTW_MULTI_AP
+	rtw_regsty_init_unassoc_sta_param(registry_par);
+#endif
+
+#ifdef CONFIG_IOCTL_CFG80211
+	registry_par->roch_min_home_dur = (u16)rtw_roch_min_home_dur;
+	registry_par->roch_max_away_dur = (u16)rtw_roch_max_away_dur;
+	registry_par->roch_extend_dur = (u16)rtw_roch_extend_dur;
+#endif
+
+#if defined(PRIVATE_R) && defined(CONFIG_P2P)
+	registry_par->go_hidden_ssid_mode = rtw_go_hidden_ssid_mode;
+	ATOMIC_SET(&registry_par->set_hide_ssid_timer, 0);
+	registry_par->p2p_go_skip_keep_alive = _TRUE;
+#else
+	registry_par->p2p_go_skip_keep_alive = _FALSE;
+#endif
+	registry_par->amsdu_mode = (u8)rtw_amsdu_mode;
+
+#if CONFIG_DFS
+#ifdef CONFIG_ECSA_PHL
+	registry_par->en_ecsa = rtw_en_ecsa;
+#endif
+#endif
+	registry_par->scan_pch_ex_time = (u16)rtw_scan_pch_ex;
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	registry_par->split_scan_6ghz = rtw_split_scan_6ghz ? true : false;
+#endif
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	registry_par->twt_en = rtw_twt_enable;
+#endif
+#endif
+
+	return status;
+}
+
+void rtw_cfg_adaptivity_config_msg(void *sel, _adapter *adapter)
+{
+	struct registry_priv *regsty = &adapter->registrypriv;
+
+	rtw_edcca_mode_sel_msg(sel, adapter_to_dvobj(adapter));
+	RTW_PRINT_SEL(sel, "adaptivity_idle_probability = %u\n", regsty->adaptivity_idle_probability);
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfgvendor.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfgvendor.c
new file mode 100644
index 000000000000..a0602db749ca
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfgvendor.c
@@ -0,0 +1,2317 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+
+#ifdef CONFIG_IOCTL_CFG80211
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) || defined(RTW_VENDOR_EXT_SUPPORT)
+
+/*
+#include <linux/kernel.h>
+#include <linux/if_arp.h>
+#include <asm/uaccess.h>
+
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/netdevice.h>
+#include <linux/sched.h>
+#include <linux/etherdevice.h>
+#include <linux/wireless.h>
+#include <linux/ieee80211.h>
+#include <linux/wait.h>
+#include <net/cfg80211.h>
+*/
+
+#include <net/rtnetlink.h>
+
+#ifndef MIN
+#define MIN(x,y) (((x) < (y)) ? (x) : (y))
+#endif
+
+#ifdef DBG_MEM_ALLOC
+extern bool match_mstat_sniff_rules(const enum mstat_f flags, const size_t size);
+struct sk_buff *dbg_rtw_cfg80211_vendor_event_alloc(struct wiphy *wiphy, struct wireless_dev *wdev, int len, int event_id, gfp_t gfp
+		, const enum mstat_f flags, const char *func, const int line)
+{
+	struct sk_buff *skb;
+	unsigned int truesize = 0;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0))
+	skb = cfg80211_vendor_event_alloc(wiphy, len, event_id, gfp);
+#else
+	skb = cfg80211_vendor_event_alloc(wiphy, wdev, len, event_id, gfp);
+#endif
+
+	if (skb)
+		truesize = skb->truesize;
+
+	if (!skb || truesize < len || match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d), skb:%p, truesize=%u\n", func, line, __FUNCTION__, len, skb, truesize);
+
+	rtw_mstat_update(
+		flags
+		, skb ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, truesize
+	);
+
+	return skb;
+}
+
+void dbg_rtw_cfg80211_vendor_event(struct sk_buff *skb, gfp_t gfp
+		   , const enum mstat_f flags, const char *func, const int line)
+{
+	unsigned int truesize = skb->truesize;
+
+	if (match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s, truesize=%u\n", func, line, __FUNCTION__, truesize);
+
+	cfg80211_vendor_event(skb, gfp);
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, truesize
+	);
+}
+
+struct sk_buff *dbg_rtw_cfg80211_vendor_cmd_alloc_reply_skb(struct wiphy *wiphy, int len
+		, const enum mstat_f flags, const char *func, const int line)
+{
+	struct sk_buff *skb;
+	unsigned int truesize = 0;
+
+	skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy, len);
+
+	if (skb)
+		truesize = skb->truesize;
+
+	if (!skb || truesize < len || match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d), skb:%p, truesize=%u\n", func, line, __FUNCTION__, len, skb, truesize);
+
+	rtw_mstat_update(
+		flags
+		, skb ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, truesize
+	);
+
+	return skb;
+}
+
+int dbg_rtw_cfg80211_vendor_cmd_reply(struct sk_buff *skb
+	      , const enum mstat_f flags, const char *func, const int line)
+{
+	unsigned int truesize = skb->truesize;
+	int ret;
+
+	if (match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s, truesize=%u\n", func, line, __FUNCTION__, truesize);
+
+	ret = cfg80211_vendor_cmd_reply(skb);
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, truesize
+	);
+
+	return ret;
+}
+
+#define rtw_cfg80211_vendor_event_alloc(wiphy, wdev, len, event_id, gfp) \
+	dbg_rtw_cfg80211_vendor_event_alloc(wiphy, wdev, len, event_id, gfp, MSTAT_FUNC_CFG_VENDOR | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+
+#define rtw_cfg80211_vendor_event(skb, gfp) \
+	dbg_rtw_cfg80211_vendor_event(skb, gfp, MSTAT_FUNC_CFG_VENDOR | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+
+#define rtw_cfg80211_vendor_cmd_alloc_reply_skb(wiphy, len) \
+	dbg_rtw_cfg80211_vendor_cmd_alloc_reply_skb(wiphy, len, MSTAT_FUNC_CFG_VENDOR | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+
+#define rtw_cfg80211_vendor_cmd_reply(skb) \
+	dbg_rtw_cfg80211_vendor_cmd_reply(skb, MSTAT_FUNC_CFG_VENDOR | MSTAT_TYPE_SKB, __FUNCTION__, __LINE__)
+#else
+
+struct sk_buff *rtw_cfg80211_vendor_event_alloc(
+	struct wiphy *wiphy, struct wireless_dev *wdev, int len, int event_id, gfp_t gfp)
+{
+	struct sk_buff *skb;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0))
+	skb = cfg80211_vendor_event_alloc(wiphy, len, event_id, gfp);
+#else
+	skb = cfg80211_vendor_event_alloc(wiphy, wdev, len, event_id, gfp);
+#endif
+	return skb;
+}
+
+#define rtw_cfg80211_vendor_event(skb, gfp) \
+	cfg80211_vendor_event(skb, gfp)
+
+#define rtw_cfg80211_vendor_cmd_alloc_reply_skb(wiphy, len) \
+	cfg80211_vendor_cmd_alloc_reply_skb(wiphy, len)
+
+#define rtw_cfg80211_vendor_cmd_reply(skb) \
+	cfg80211_vendor_cmd_reply(skb)
+#endif /* DBG_MEM_ALLOC */
+
+/*
+ * This API is to be used for asynchronous vendor events. This
+ * shouldn't be used in response to a vendor command from its
+ * do_it handler context (instead rtw_cfgvendor_send_cmd_reply should
+ * be used).
+ */
+int rtw_cfgvendor_send_async_event(struct wiphy *wiphy,
+	   struct net_device *dev, int event_id, const void  *data, int len)
+{
+	int kflags;
+	struct sk_buff *skb;
+
+	kflags = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
+
+	/* Alloc the SKB for vendor_event */
+	skb = rtw_cfg80211_vendor_event_alloc(wiphy, ndev_to_wdev(dev), len, event_id, kflags);
+	if (!skb) {
+		RTW_ERR(FUNC_NDEV_FMT" skb alloc failed", FUNC_NDEV_ARG(dev));
+		return -ENOMEM;
+	}
+
+	/* Push the data to the skb */
+	nla_put_nohdr(skb, len, data);
+
+	rtw_cfg80211_vendor_event(skb, kflags);
+
+	return 0;
+}
+
+static int rtw_cfgvendor_send_cmd_reply(struct wiphy *wiphy,
+			struct net_device *dev, const void  *data, int len)
+{
+	struct sk_buff *skb;
+
+	/* Alloc the SKB for vendor_event */
+	skb = rtw_cfg80211_vendor_cmd_alloc_reply_skb(wiphy, len);
+	if (unlikely(!skb)) {
+		RTW_ERR(FUNC_NDEV_FMT" skb alloc failed", FUNC_NDEV_ARG(dev));
+		return -ENOMEM;
+	}
+
+	/* Push the data to the skb */
+	nla_put_nohdr(skb, len, data);
+
+	return rtw_cfg80211_vendor_cmd_reply(skb);
+}
+
+/* Feature enums */
+#define WIFI_FEATURE_INFRA              (u64)0x1      // Basic infrastructure mode
+#define WIFI_FEATURE_INFRA_5G           (u64)0x2      // Support for 5 GHz Band
+#define WIFI_FEATURE_HOTSPOT            (u64)0x4      // Support for GAS/ANQP
+#define WIFI_FEATURE_P2P                (u64)0x8      // Wifi-Direct
+#define WIFI_FEATURE_SOFT_AP            (u64)0x10      // Soft AP
+#define WIFI_FEATURE_GSCAN              (u64)0x20      // Google-Scan APIs
+#define WIFI_FEATURE_NAN                (u64)0x40      // Neighbor Awareness Networking
+#define WIFI_FEATURE_D2D_RTT            (u64)0x80      // Device-to-device RTT
+#define WIFI_FEATURE_D2AP_RTT           (u64)0x100      // Device-to-AP RTT
+#define WIFI_FEATURE_BATCH_SCAN         (u64)0x200      // Batched Scan (legacy)
+#define WIFI_FEATURE_PNO                (u64)0x400      // Preferred network offload
+#define WIFI_FEATURE_ADDITIONAL_STA     (u64)0x800      // Support for two STAs
+#define WIFI_FEATURE_TDLS               (u64)0x1000      // Tunnel directed link setup
+#define WIFI_FEATURE_TDLS_OFFCHANNEL    (u64)0x2000      // Support for TDLS off channel
+#define WIFI_FEATURE_EPR                (u64)0x4000      // Enhanced power reporting
+#define WIFI_FEATURE_AP_STA             (u64)0x8000      // Support for AP STA Concurrency
+#define WIFI_FEATURE_LINK_LAYER_STATS   (u64)0x10000     // Link layer stats collection
+#define WIFI_FEATURE_LOGGER             (u64)0x20000     // WiFi Logger
+#define WIFI_FEATURE_HAL_EPNO           (u64)0x40000     // WiFi PNO enhanced
+#define WIFI_FEATURE_RSSI_MONITOR       (u64)0x80000     // RSSI Monitor
+#define WIFI_FEATURE_MKEEP_ALIVE        (u64)0x100000    // WiFi mkeep_alive
+#define WIFI_FEATURE_CONFIG_NDO         (u64)0x200000    // ND offload configure
+#define WIFI_FEATURE_TX_TRANSMIT_POWER  (u64)0x400000    // Capture Tx transmit power levels
+#define WIFI_FEATURE_CONTROL_ROAMING    (u64)0x800000    // Enable/Disable firmware roaming
+#define WIFI_FEATURE_IE_WHITELIST       (u64)0x1000000   // Support Probe IE white listing
+#define WIFI_FEATURE_SCAN_RAND          (u64)0x2000000   // Support MAC & Probe Sequence Number randomization
+#define WIFI_FEATURE_SET_TX_POWER_LIMIT (u64)0x4000000   // Support Tx Power Limit setting
+#define WIFI_FEATURE_USE_BODY_HEAD_SAR  (u64)0x8000000   // Support Using Body/Head Proximity for SAR
+#define WIFI_FEATURE_DYNAMIC_SET_MAC    (u64)0x10000000  // Support changing MAC address without iface reset(down and up)
+#define WIFI_FEATURE_SET_LATENCY_MODE   (u64)0x40000000  // Support Latency mode setting
+#define WIFI_FEATURE_P2P_RAND_MAC       (u64)0x80000000  // Support P2P MAC randomization
+#define WIFI_FEATURE_INFRA_60G          (u64)0x100000000 // Support for 60GHz Band
+// Add more features here
+
+#define MAX_FEATURE_SET_CONCURRRENT_GROUPS  3
+
+u64 rtw_dev_get_feature_set(struct net_device *dev)
+{
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	u64 feature_set = 0;
+
+	feature_set |= WIFI_FEATURE_INFRA;
+
+#if CONFIG_IEEE80211_BAND_5GHZ
+	if (is_supported_5g(adapter_to_regsty(adapter)->band_type))
+		feature_set |= WIFI_FEATURE_INFRA_5G;
+#endif
+
+	feature_set |= WIFI_FEATURE_P2P;
+	feature_set |= WIFI_FEATURE_SOFT_AP;
+
+	feature_set |= WIFI_FEATURE_ADDITIONAL_STA;
+#ifdef CONFIG_RTW_CFGVENDOR_LLSTATS
+	feature_set |= WIFI_FEATURE_LINK_LAYER_STATS;
+#endif /* CONFIG_RTW_CFGVENDOR_LLSTATS */
+
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+        feature_set |= WIFI_FEATURE_RSSI_MONITOR;
+#endif
+
+#ifdef CONFIG_RTW_CFGVENDOR_WIFI_LOGGER
+	feature_set |= WIFI_FEATURE_LOGGER;
+#endif
+
+#ifdef CONFIG_RTW_WIFI_HAL
+	feature_set |= WIFI_FEATURE_CONFIG_NDO;
+	feature_set |= WIFI_FEATURE_SCAN_RAND;
+ 	feature_set |= WIFI_FEATURE_DYNAMIC_SET_MAC;
+#endif
+
+	return feature_set;
+}
+
+int *rtw_dev_get_feature_set_matrix(struct net_device *dev, int *num)
+{
+	int feature_set_full, mem_needed;
+	int *ret;
+
+	*num = 0;
+	mem_needed = sizeof(int) * MAX_FEATURE_SET_CONCURRRENT_GROUPS;
+	ret = (int *)rtw_malloc(mem_needed);
+
+	if (!ret) {
+		RTW_ERR(FUNC_NDEV_FMT" failed to allocate %d bytes\n"
+			, FUNC_NDEV_ARG(dev), mem_needed);
+		return ret;
+	}
+
+	feature_set_full = rtw_dev_get_feature_set(dev);
+
+	ret[0] = (feature_set_full & WIFI_FEATURE_INFRA) |
+		 (feature_set_full & WIFI_FEATURE_INFRA_5G) |
+		 (feature_set_full & WIFI_FEATURE_NAN) |
+		 (feature_set_full & WIFI_FEATURE_D2D_RTT) |
+		 (feature_set_full & WIFI_FEATURE_D2AP_RTT) |
+		 (feature_set_full & WIFI_FEATURE_PNO) |
+		 (feature_set_full & WIFI_FEATURE_BATCH_SCAN) |
+		 (feature_set_full & WIFI_FEATURE_GSCAN) |
+		 (feature_set_full & WIFI_FEATURE_HOTSPOT) |
+		 (feature_set_full & WIFI_FEATURE_ADDITIONAL_STA) |
+		 (feature_set_full & WIFI_FEATURE_EPR);
+
+	ret[1] = (feature_set_full & WIFI_FEATURE_INFRA) |
+		 (feature_set_full & WIFI_FEATURE_INFRA_5G) |
+		 /* Not yet verified NAN with P2P */
+		 /* (feature_set_full & WIFI_FEATURE_NAN) | */
+		 (feature_set_full & WIFI_FEATURE_P2P) |
+		 (feature_set_full & WIFI_FEATURE_D2AP_RTT) |
+		 (feature_set_full & WIFI_FEATURE_D2D_RTT) |
+		 (feature_set_full & WIFI_FEATURE_EPR);
+
+	ret[2] = (feature_set_full & WIFI_FEATURE_INFRA) |
+		 (feature_set_full & WIFI_FEATURE_INFRA_5G) |
+		 (feature_set_full & WIFI_FEATURE_NAN) |
+		 (feature_set_full & WIFI_FEATURE_D2D_RTT) |
+		 (feature_set_full & WIFI_FEATURE_D2AP_RTT) |
+		 (feature_set_full & WIFI_FEATURE_TDLS) |
+		 (feature_set_full & WIFI_FEATURE_TDLS_OFFCHANNEL) |
+		 (feature_set_full & WIFI_FEATURE_EPR);
+	*num = MAX_FEATURE_SET_CONCURRRENT_GROUPS;
+
+	return ret;
+}
+
+static int rtw_cfgvendor_get_feature_set(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	u64 reply;
+
+	reply = rtw_dev_get_feature_set(wdev_to_ndev(wdev));
+
+	err =  rtw_cfgvendor_send_cmd_reply(wiphy, wdev_to_ndev(wdev), &reply, sizeof(u64));
+
+	if (unlikely(err))
+		RTW_ERR(FUNC_NDEV_FMT" Vendor Command reply failed ret:%d\n"
+			, FUNC_NDEV_ARG(wdev_to_ndev(wdev)), err);
+
+	return err;
+}
+
+static int rtw_cfgvendor_get_feature_set_matrix(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct sk_buff *skb;
+	int *reply;
+	int num, mem_needed, i;
+
+	reply = rtw_dev_get_feature_set_matrix(wdev_to_ndev(wdev), &num);
+
+	if (!reply) {
+		RTW_ERR(FUNC_NDEV_FMT" Could not get feature list matrix\n"
+			, FUNC_NDEV_ARG(wdev_to_ndev(wdev)));
+		err = -EINVAL;
+		return err;
+	}
+
+	mem_needed = VENDOR_REPLY_OVERHEAD + (ATTRIBUTE_U32_LEN * num) +
+		     ATTRIBUTE_U32_LEN;
+
+	/* Alloc the SKB for vendor_event */
+	skb = rtw_cfg80211_vendor_cmd_alloc_reply_skb(wiphy, mem_needed);
+	if (unlikely(!skb)) {
+		RTW_ERR(FUNC_NDEV_FMT" skb alloc failed", FUNC_NDEV_ARG(wdev_to_ndev(wdev)));
+		err = -ENOMEM;
+		goto exit;
+	}
+
+	nla_put_u32(skb, ANDR_WIFI_ATTRIBUTE_NUM_FEATURE_SET, num);
+	for (i = 0; i < num; i++)
+		nla_put_u32(skb, ANDR_WIFI_ATTRIBUTE_FEATURE_SET, reply[i]);
+
+	err =  rtw_cfg80211_vendor_cmd_reply(skb);
+
+	if (unlikely(err))
+		RTW_ERR(FUNC_NDEV_FMT" Vendor Command reply failed ret:%d\n"
+			, FUNC_NDEV_ARG(wdev_to_ndev(wdev)), err);
+exit:
+	rtw_mfree((u8 *)reply, sizeof(int) * num);
+	return err;
+}
+
+#if defined(GSCAN_SUPPORT) && 0
+int rtw_cfgvendor_send_hotlist_event(struct wiphy *wiphy,
+	struct net_device *dev, void  *data, int len, rtw_vendor_event_t event)
+{
+	u16 kflags;
+	const void *ptr;
+	struct sk_buff *skb;
+	int malloc_len, total, iter_cnt_to_send, cnt;
+	gscan_results_cache_t *cache = (gscan_results_cache_t *)data;
+
+	total = len / sizeof(wifi_gscan_result_t);
+	while (total > 0) {
+		malloc_len = (total * sizeof(wifi_gscan_result_t)) + VENDOR_DATA_OVERHEAD;
+		if (malloc_len > NLMSG_DEFAULT_SIZE)
+			malloc_len = NLMSG_DEFAULT_SIZE;
+		iter_cnt_to_send =
+			(malloc_len - VENDOR_DATA_OVERHEAD) / sizeof(wifi_gscan_result_t);
+		total = total - iter_cnt_to_send;
+
+		kflags = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
+
+		/* Alloc the SKB for vendor_event */
+		skb = rtw_cfg80211_vendor_event_alloc(wiphy, ndev_to_wdev(dev), malloc_len, event, kflags);
+		if (!skb) {
+			WL_ERR(("skb alloc failed"));
+			return -ENOMEM;
+		}
+
+		while (cache && iter_cnt_to_send) {
+			ptr = (const void *) &cache->results[cache->tot_consumed];
+
+			if (iter_cnt_to_send < (cache->tot_count - cache->tot_consumed))
+				cnt = iter_cnt_to_send;
+			else
+				cnt = (cache->tot_count - cache->tot_consumed);
+
+			iter_cnt_to_send -= cnt;
+			cache->tot_consumed += cnt;
+			/* Push the data to the skb */
+			nla_append(skb, cnt * sizeof(wifi_gscan_result_t), ptr);
+			if (cache->tot_consumed == cache->tot_count)
+				cache = cache->next;
+
+		}
+
+		rtw_cfg80211_vendor_event(skb, kflags);
+	}
+
+	return 0;
+}
+
+
+static int rtw_cfgvendor_gscan_get_capabilities(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	dhd_pno_gscan_capabilities_t *reply = NULL;
+	uint32 reply_len = 0;
+
+
+	reply = dhd_dev_pno_get_gscan(bcmcfg_to_prmry_ndev(cfg),
+			      DHD_PNO_GET_CAPABILITIES, NULL, &reply_len);
+	if (!reply) {
+		WL_ERR(("Could not get capabilities\n"));
+		err = -EINVAL;
+		return err;
+	}
+
+	err =  rtw_cfgvendor_send_cmd_reply(wiphy, bcmcfg_to_prmry_ndev(cfg),
+					    reply, reply_len);
+
+	if (unlikely(err))
+		WL_ERR(("Vendor Command reply failed ret:%d\n", err));
+
+	kfree(reply);
+	return err;
+}
+
+static int rtw_cfgvendor_gscan_get_channel_list(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0, type, band;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	uint16 *reply = NULL;
+	uint32 reply_len = 0, num_channels, mem_needed;
+	struct sk_buff *skb;
+
+	type = nla_type(data);
+
+	if (type == GSCAN_ATTRIBUTE_BAND)
+		band = nla_get_u32(data);
+	else
+		return -1;
+
+	reply = dhd_dev_pno_get_gscan(bcmcfg_to_prmry_ndev(cfg),
+			      DHD_PNO_GET_CHANNEL_LIST, &band, &reply_len);
+
+	if (!reply) {
+		WL_ERR(("Could not get channel list\n"));
+		err = -EINVAL;
+		return err;
+	}
+	num_channels =  reply_len / sizeof(uint32);
+	mem_needed = reply_len + VENDOR_REPLY_OVERHEAD + (ATTRIBUTE_U32_LEN * 2);
+
+	/* Alloc the SKB for vendor_event */
+	skb = rtw_cfg80211_vendor_cmd_alloc_reply_skb(wiphy, mem_needed);
+	if (unlikely(!skb)) {
+		WL_ERR(("skb alloc failed"));
+		err = -ENOMEM;
+		goto exit;
+	}
+
+	nla_put_u32(skb, GSCAN_ATTRIBUTE_NUM_CHANNELS, num_channels);
+	nla_put(skb, GSCAN_ATTRIBUTE_CHANNEL_LIST, reply_len, reply);
+
+	err =  rtw_cfg80211_vendor_cmd_reply(skb);
+
+	if (unlikely(err))
+		WL_ERR(("Vendor Command reply failed ret:%d\n", err));
+exit:
+	kfree(reply);
+	return err;
+}
+
+static int rtw_cfgvendor_gscan_get_batch_results(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	gscan_results_cache_t *results, *iter;
+	uint32 reply_len, complete = 0, num_results_iter;
+	int32 mem_needed;
+	wifi_gscan_result_t *ptr;
+	uint16 num_scan_ids, num_results;
+	struct sk_buff *skb;
+	struct nlattr *scan_hdr;
+
+	dhd_dev_wait_batch_results_complete(bcmcfg_to_prmry_ndev(cfg));
+	dhd_dev_pno_lock_access_batch_results(bcmcfg_to_prmry_ndev(cfg));
+	results = dhd_dev_pno_get_gscan(bcmcfg_to_prmry_ndev(cfg),
+				DHD_PNO_GET_BATCH_RESULTS, NULL, &reply_len);
+
+	if (!results) {
+		WL_ERR(("No results to send %d\n", err));
+		err =  rtw_cfgvendor_send_cmd_reply(wiphy, bcmcfg_to_prmry_ndev(cfg),
+						    results, 0);
+
+		if (unlikely(err))
+			WL_ERR(("Vendor Command reply failed ret:%d\n", err));
+		dhd_dev_pno_unlock_access_batch_results(bcmcfg_to_prmry_ndev(cfg));
+		return err;
+	}
+	num_scan_ids = reply_len & 0xFFFF;
+	num_results = (reply_len & 0xFFFF0000) >> 16;
+	mem_needed = (num_results * sizeof(wifi_gscan_result_t)) +
+		     (num_scan_ids * GSCAN_BATCH_RESULT_HDR_LEN) +
+		     VENDOR_REPLY_OVERHEAD + SCAN_RESULTS_COMPLETE_FLAG_LEN;
+
+	if (mem_needed > (int32)NLMSG_DEFAULT_SIZE) {
+		mem_needed = (int32)NLMSG_DEFAULT_SIZE;
+		complete = 0;
+	} else
+		complete = 1;
+
+	WL_TRACE(("complete %d mem_needed %d max_mem %d\n", complete, mem_needed,
+		  (int)NLMSG_DEFAULT_SIZE));
+	/* Alloc the SKB for vendor_event */
+	skb = rtw_cfg80211_vendor_cmd_alloc_reply_skb(wiphy, mem_needed);
+	if (unlikely(!skb)) {
+		WL_ERR(("skb alloc failed"));
+		dhd_dev_pno_unlock_access_batch_results(bcmcfg_to_prmry_ndev(cfg));
+		return -ENOMEM;
+	}
+	iter = results;
+
+	nla_put_u32(skb, GSCAN_ATTRIBUTE_SCAN_RESULTS_COMPLETE, complete);
+
+	mem_needed = mem_needed - (SCAN_RESULTS_COMPLETE_FLAG_LEN + VENDOR_REPLY_OVERHEAD);
+
+	while (iter && ((mem_needed - GSCAN_BATCH_RESULT_HDR_LEN)  > 0)) {
+		scan_hdr = nla_nest_start(skb, GSCAN_ATTRIBUTE_SCAN_RESULTS);
+		nla_put_u32(skb, GSCAN_ATTRIBUTE_SCAN_ID, iter->scan_id);
+		nla_put_u8(skb, GSCAN_ATTRIBUTE_SCAN_FLAGS, iter->flag);
+		num_results_iter =
+			(mem_needed - GSCAN_BATCH_RESULT_HDR_LEN) / sizeof(wifi_gscan_result_t);
+
+		if ((iter->tot_count - iter->tot_consumed) < num_results_iter)
+			num_results_iter = iter->tot_count - iter->tot_consumed;
+
+		nla_put_u32(skb, GSCAN_ATTRIBUTE_NUM_OF_RESULTS, num_results_iter);
+		if (num_results_iter) {
+			ptr = &iter->results[iter->tot_consumed];
+			iter->tot_consumed += num_results_iter;
+			nla_put(skb, GSCAN_ATTRIBUTE_SCAN_RESULTS,
+				num_results_iter * sizeof(wifi_gscan_result_t), ptr);
+		}
+		nla_nest_end(skb, scan_hdr);
+		mem_needed -= GSCAN_BATCH_RESULT_HDR_LEN +
+			      (num_results_iter * sizeof(wifi_gscan_result_t));
+		iter = iter->next;
+	}
+
+	dhd_dev_gscan_batch_cache_cleanup(bcmcfg_to_prmry_ndev(cfg));
+	dhd_dev_pno_unlock_access_batch_results(bcmcfg_to_prmry_ndev(cfg));
+
+	return rtw_cfg80211_vendor_cmd_reply(skb);
+}
+
+static int rtw_cfgvendor_initiate_gscan(struct wiphy *wiphy,
+		       struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	int type, tmp = len;
+	int run = 0xFF;
+	int flush = 0;
+	const struct nlattr *iter;
+
+	nla_for_each_attr(iter, data, len, tmp) {
+		type = nla_type(iter);
+		if (type == GSCAN_ATTRIBUTE_ENABLE_FEATURE)
+			run = nla_get_u32(iter);
+		else if (type == GSCAN_ATTRIBUTE_FLUSH_FEATURE)
+			flush = nla_get_u32(iter);
+	}
+
+	if (run != 0xFF) {
+		err = dhd_dev_pno_run_gscan(bcmcfg_to_prmry_ndev(cfg), run, flush);
+
+		if (unlikely(err))
+			WL_ERR(("Could not run gscan:%d\n", err));
+		return err;
+	} else
+		return -1;
+
+
+}
+
+static int rtw_cfgvendor_enable_full_scan_result(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	int type;
+	bool real_time = FALSE;
+
+	type = nla_type(data);
+
+	if (type == GSCAN_ATTRIBUTE_ENABLE_FULL_SCAN_RESULTS) {
+		real_time = nla_get_u32(data);
+
+		err = dhd_dev_pno_enable_full_scan_result(bcmcfg_to_prmry_ndev(cfg), real_time);
+
+		if (unlikely(err))
+			WL_ERR(("Could not run gscan:%d\n", err));
+
+	} else
+		err = -1;
+
+	return err;
+}
+
+static int rtw_cfgvendor_set_scan_cfg(struct wiphy *wiphy,
+		     struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	gscan_scan_params_t *scan_param;
+	int j = 0;
+	int type, tmp, tmp1, tmp2, k = 0;
+	const struct nlattr *iter, *iter1, *iter2;
+	struct dhd_pno_gscan_channel_bucket  *ch_bucket;
+
+	scan_param = kzalloc(sizeof(gscan_scan_params_t), GFP_KERNEL);
+	if (!scan_param) {
+		WL_ERR(("Could not set GSCAN scan cfg, mem alloc failure\n"));
+		err = -EINVAL;
+		return err;
+
+	}
+
+	scan_param->scan_fr = PNO_SCAN_MIN_FW_SEC;
+	nla_for_each_attr(iter, data, len, tmp) {
+		type = nla_type(iter);
+
+		if (j >= GSCAN_MAX_CH_BUCKETS)
+			break;
+
+		switch (type) {
+		case GSCAN_ATTRIBUTE_BASE_PERIOD:
+			scan_param->scan_fr = nla_get_u32(iter) / 1000;
+			break;
+		case GSCAN_ATTRIBUTE_NUM_BUCKETS:
+			scan_param->nchannel_buckets = nla_get_u32(iter);
+			break;
+		case GSCAN_ATTRIBUTE_CH_BUCKET_1:
+		case GSCAN_ATTRIBUTE_CH_BUCKET_2:
+		case GSCAN_ATTRIBUTE_CH_BUCKET_3:
+		case GSCAN_ATTRIBUTE_CH_BUCKET_4:
+		case GSCAN_ATTRIBUTE_CH_BUCKET_5:
+		case GSCAN_ATTRIBUTE_CH_BUCKET_6:
+		case GSCAN_ATTRIBUTE_CH_BUCKET_7:
+			nla_for_each_nested(iter1, iter, tmp1) {
+				type = nla_type(iter1);
+				ch_bucket =
+					scan_param->channel_bucket;
+
+				switch (type) {
+				case GSCAN_ATTRIBUTE_BUCKET_ID:
+					break;
+				case GSCAN_ATTRIBUTE_BUCKET_PERIOD:
+					ch_bucket[j].bucket_freq_multiple =
+						nla_get_u32(iter1) / 1000;
+					break;
+				case GSCAN_ATTRIBUTE_BUCKET_NUM_CHANNELS:
+					ch_bucket[j].num_channels =
+						nla_get_u32(iter1);
+					break;
+				case GSCAN_ATTRIBUTE_BUCKET_CHANNELS:
+					nla_for_each_nested(iter2, iter1, tmp2) {
+						if (k >= PFN_SWC_RSSI_WINDOW_MAX)
+							break;
+						ch_bucket[j].chan_list[k] =
+							nla_get_u32(iter2);
+						k++;
+					}
+					k = 0;
+					break;
+				case GSCAN_ATTRIBUTE_BUCKETS_BAND:
+					ch_bucket[j].band = (uint16)
+							    nla_get_u32(iter1);
+					break;
+				case GSCAN_ATTRIBUTE_REPORT_EVENTS:
+					ch_bucket[j].report_flag = (uint8)
+							   nla_get_u32(iter1);
+					break;
+				}
+			}
+			j++;
+			break;
+		}
+	}
+
+	if (dhd_dev_pno_set_cfg_gscan(bcmcfg_to_prmry_ndev(cfg),
+				      DHD_PNO_SCAN_CFG_ID, scan_param, 0) < 0) {
+		WL_ERR(("Could not set GSCAN scan cfg\n"));
+		err = -EINVAL;
+	}
+
+	kfree(scan_param);
+	return err;
+
+}
+
+static int rtw_cfgvendor_hotlist_cfg(struct wiphy *wiphy,
+		    struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	gscan_hotlist_scan_params_t *hotlist_params;
+	int tmp, tmp1, tmp2, type, j = 0, dummy;
+	const struct nlattr *outer, *inner, *iter;
+	uint8 flush = 0;
+	struct bssid_t *pbssid;
+
+	hotlist_params = (gscan_hotlist_scan_params_t *)kzalloc(len, GFP_KERNEL);
+	if (!hotlist_params) {
+		WL_ERR(("Cannot Malloc mem to parse config commands size - %d bytes\n", len));
+		return -1;
+	}
+
+	hotlist_params->lost_ap_window = GSCAN_LOST_AP_WINDOW_DEFAULT;
+
+	nla_for_each_attr(iter, data, len, tmp2) {
+		type = nla_type(iter);
+		switch (type) {
+		case GSCAN_ATTRIBUTE_HOTLIST_BSSIDS:
+			pbssid = hotlist_params->bssid;
+			nla_for_each_nested(outer, iter, tmp) {
+				nla_for_each_nested(inner, outer, tmp1) {
+					type = nla_type(inner);
+
+					switch (type) {
+					case GSCAN_ATTRIBUTE_BSSID:
+						_rtw_memcpy(&(pbssid[j].macaddr),
+						       nla_data(inner), ETHER_ADDR_LEN);
+						break;
+					case GSCAN_ATTRIBUTE_RSSI_LOW:
+						pbssid[j].rssi_reporting_threshold =
+							(int8) nla_get_u8(inner);
+						break;
+					case GSCAN_ATTRIBUTE_RSSI_HIGH:
+						dummy = (int8) nla_get_u8(inner);
+						break;
+					}
+				}
+				j++;
+			}
+			hotlist_params->nbssid = j;
+			break;
+		case GSCAN_ATTRIBUTE_HOTLIST_FLUSH:
+			flush = nla_get_u8(iter);
+			break;
+		case GSCAN_ATTRIBUTE_LOST_AP_SAMPLE_SIZE:
+			hotlist_params->lost_ap_window = nla_get_u32(iter);
+			break;
+		}
+
+	}
+
+	if (dhd_dev_pno_set_cfg_gscan(bcmcfg_to_prmry_ndev(cfg),
+		DHD_PNO_GEOFENCE_SCAN_CFG_ID, hotlist_params, flush) < 0) {
+		WL_ERR(("Could not set GSCAN HOTLIST cfg\n"));
+		err = -EINVAL;
+		goto exit;
+	}
+exit:
+	kfree(hotlist_params);
+	return err;
+}
+static int rtw_cfgvendor_set_batch_scan_cfg(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0, tmp, type;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	gscan_batch_params_t batch_param;
+	const struct nlattr *iter;
+
+	batch_param.mscan = batch_param.bestn = 0;
+	batch_param.buffer_threshold = GSCAN_BATCH_NO_THR_SET;
+
+	nla_for_each_attr(iter, data, len, tmp) {
+		type = nla_type(iter);
+
+		switch (type) {
+		case GSCAN_ATTRIBUTE_NUM_AP_PER_SCAN:
+			batch_param.bestn = nla_get_u32(iter);
+			break;
+		case GSCAN_ATTRIBUTE_NUM_SCANS_TO_CACHE:
+			batch_param.mscan = nla_get_u32(iter);
+			break;
+		case GSCAN_ATTRIBUTE_REPORT_THRESHOLD:
+			batch_param.buffer_threshold = nla_get_u32(iter);
+			break;
+		}
+	}
+
+	if (dhd_dev_pno_set_cfg_gscan(bcmcfg_to_prmry_ndev(cfg),
+			      DHD_PNO_BATCH_SCAN_CFG_ID, &batch_param, 0) < 0) {
+		WL_ERR(("Could not set batch cfg\n"));
+		err = -EINVAL;
+		return err;
+	}
+
+	return err;
+}
+
+static int rtw_cfgvendor_significant_change_cfg(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	gscan_swc_params_t *significant_params;
+	int tmp, tmp1, tmp2, type, j = 0;
+	const struct nlattr *outer, *inner, *iter;
+	uint8 flush = 0;
+	wl_pfn_significant_bssid_t *pbssid;
+
+	significant_params = (gscan_swc_params_t *) kzalloc(len, GFP_KERNEL);
+	if (!significant_params) {
+		WL_ERR(("Cannot Malloc mem to parse config commands size - %d bytes\n", len));
+		return -1;
+	}
+
+
+	nla_for_each_attr(iter, data, len, tmp2) {
+		type = nla_type(iter);
+
+		switch (type) {
+		case GSCAN_ATTRIBUTE_SIGNIFICANT_CHANGE_FLUSH:
+			flush = nla_get_u8(iter);
+			break;
+		case GSCAN_ATTRIBUTE_RSSI_SAMPLE_SIZE:
+			significant_params->rssi_window = nla_get_u16(iter);
+			break;
+		case GSCAN_ATTRIBUTE_LOST_AP_SAMPLE_SIZE:
+			significant_params->lost_ap_window = nla_get_u16(iter);
+			break;
+		case GSCAN_ATTRIBUTE_MIN_BREACHING:
+			significant_params->swc_threshold = nla_get_u16(iter);
+			break;
+		case GSCAN_ATTRIBUTE_SIGNIFICANT_CHANGE_BSSIDS:
+			pbssid = significant_params->bssid_elem_list;
+			nla_for_each_nested(outer, iter, tmp) {
+				nla_for_each_nested(inner, outer, tmp1) {
+					switch (nla_type(inner)) {
+					case GSCAN_ATTRIBUTE_BSSID:
+						_rtw_memcpy(&(pbssid[j].macaddr),
+						       nla_data(inner),
+						       ETHER_ADDR_LEN);
+						break;
+					case GSCAN_ATTRIBUTE_RSSI_HIGH:
+						pbssid[j].rssi_high_threshold =
+							(int8) nla_get_u8(inner);
+						break;
+					case GSCAN_ATTRIBUTE_RSSI_LOW:
+						pbssid[j].rssi_low_threshold =
+							(int8) nla_get_u8(inner);
+						break;
+					}
+				}
+				j++;
+			}
+			break;
+		}
+	}
+	significant_params->nbssid = j;
+
+	if (dhd_dev_pno_set_cfg_gscan(bcmcfg_to_prmry_ndev(cfg),
+		DHD_PNO_SIGNIFICANT_SCAN_CFG_ID, significant_params, flush) < 0) {
+		WL_ERR(("Could not set GSCAN significant cfg\n"));
+		err = -EINVAL;
+		goto exit;
+	}
+exit:
+	kfree(significant_params);
+	return err;
+}
+#endif /* GSCAN_SUPPORT */
+
+#if defined(RTT_SUPPORT) && 0
+void rtw_cfgvendor_rtt_evt(void *ctx, void *rtt_data)
+{
+	struct wireless_dev *wdev = (struct wireless_dev *)ctx;
+	struct wiphy *wiphy;
+	struct sk_buff *skb;
+	uint32 tot_len = NLMSG_DEFAULT_SIZE, entry_len = 0;
+	gfp_t kflags;
+	rtt_report_t *rtt_report = NULL;
+	rtt_result_t *rtt_result = NULL;
+	struct list_head *rtt_list;
+	wiphy = wdev->wiphy;
+
+	WL_DBG(("In\n"));
+	/* Push the data to the skb */
+	if (!rtt_data) {
+		WL_ERR(("rtt_data is NULL\n"));
+		goto exit;
+	}
+	rtt_list = (struct list_head *)rtt_data;
+	kflags = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
+	/* Alloc the SKB for vendor_event */
+	skb = rtw_cfg80211_vendor_event_alloc(wiphy, wdev, tot_len, GOOGLE_RTT_COMPLETE_EVENT, kflags);
+	if (!skb) {
+		WL_ERR(("skb alloc failed"));
+		goto exit;
+	}
+	/* fill in the rtt results on each entry */
+	list_for_each_entry(rtt_result, rtt_list, list) {
+		entry_len = 0;
+		if (rtt_result->TOF_type == TOF_TYPE_ONE_WAY) {
+			entry_len = sizeof(rtt_report_t);
+			rtt_report = kzalloc(entry_len, kflags);
+			if (!rtt_report) {
+				WL_ERR(("rtt_report alloc failed"));
+				goto exit;
+			}
+			rtt_report->addr = rtt_result->peer_mac;
+			rtt_report->num_measurement = 1; /* ONE SHOT */
+			rtt_report->status = rtt_result->err_code;
+			rtt_report->type = (rtt_result->TOF_type == TOF_TYPE_ONE_WAY) ? RTT_ONE_WAY : RTT_TWO_WAY;
+			rtt_report->peer = rtt_result->target_info->peer;
+			rtt_report->channel = rtt_result->target_info->channel;
+			rtt_report->rssi = rtt_result->avg_rssi;
+			/* tx_rate */
+			rtt_report->tx_rate = rtt_result->tx_rate;
+			/* RTT */
+			rtt_report->rtt = rtt_result->meanrtt;
+			rtt_report->rtt_sd = rtt_result->sdrtt;
+			/* convert to centi meter */
+			if (rtt_result->distance != 0xffffffff)
+				rtt_report->distance = (rtt_result->distance >> 2) * 25;
+			else /* invalid distance */
+				rtt_report->distance = -1;
+
+			rtt_report->ts = rtt_result->ts;
+			nla_append(skb, entry_len, rtt_report);
+			kfree(rtt_report);
+		}
+	}
+	rtw_cfg80211_vendor_event(skb, kflags);
+exit:
+	return;
+}
+
+static int rtw_cfgvendor_rtt_set_config(struct wiphy *wiphy, struct wireless_dev *wdev,
+				       const void *data, int len)
+{
+	int err = 0, rem, rem1, rem2, type;
+	rtt_config_params_t rtt_param;
+	rtt_target_info_t *rtt_target = NULL;
+	const struct nlattr *iter, *iter1, *iter2;
+	int8 eabuf[ETHER_ADDR_STR_LEN];
+	int8 chanbuf[CHANSPEC_STR_LEN];
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+
+	WL_DBG(("In\n"));
+	err = dhd_dev_rtt_register_noti_callback(wdev->netdev, wdev, wl_cfgvendor_rtt_evt);
+	if (err < 0) {
+		WL_ERR(("failed to register rtt_noti_callback\n"));
+		goto exit;
+	}
+	memset(&rtt_param, 0, sizeof(rtt_param));
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+		switch (type) {
+		case RTT_ATTRIBUTE_TARGET_CNT:
+			rtt_param.rtt_target_cnt = nla_get_u8(iter);
+			if (rtt_param.rtt_target_cnt > RTT_MAX_TARGET_CNT) {
+				WL_ERR(("exceed max target count : %d\n",
+					rtt_param.rtt_target_cnt));
+				err = BCME_RANGE;
+			}
+			break;
+		case RTT_ATTRIBUTE_TARGET_INFO:
+			rtt_target = rtt_param.target_info;
+			nla_for_each_nested(iter1, iter, rem1) {
+				nla_for_each_nested(iter2, iter1, rem2) {
+					type = nla_type(iter2);
+					switch (type) {
+					case RTT_ATTRIBUTE_TARGET_MAC:
+						_rtw_memcpy(&rtt_target->addr, nla_data(iter2), ETHER_ADDR_LEN);
+						break;
+					case RTT_ATTRIBUTE_TARGET_TYPE:
+						rtt_target->type = nla_get_u8(iter2);
+						break;
+					case RTT_ATTRIBUTE_TARGET_PEER:
+						rtt_target->peer = nla_get_u8(iter2);
+						break;
+					case RTT_ATTRIBUTE_TARGET_CHAN:
+						_rtw_memcpy(&rtt_target->channel, nla_data(iter2),
+						       sizeof(rtt_target->channel));
+						break;
+					case RTT_ATTRIBUTE_TARGET_MODE:
+						rtt_target->continuous = nla_get_u8(iter2);
+						break;
+					case RTT_ATTRIBUTE_TARGET_INTERVAL:
+						rtt_target->interval = nla_get_u32(iter2);
+						break;
+					case RTT_ATTRIBUTE_TARGET_NUM_MEASUREMENT:
+						rtt_target->measure_cnt = nla_get_u32(iter2);
+						break;
+					case RTT_ATTRIBUTE_TARGET_NUM_PKT:
+						rtt_target->ftm_cnt = nla_get_u32(iter2);
+						break;
+					case RTT_ATTRIBUTE_TARGET_NUM_RETRY:
+						rtt_target->retry_cnt = nla_get_u32(iter2);
+					}
+				}
+				/* convert to chanspec value */
+				rtt_target->chanspec = dhd_rtt_convert_to_chspec(rtt_target->channel);
+				if (rtt_target->chanspec == 0) {
+					WL_ERR(("Channel is not valid\n"));
+					goto exit;
+				}
+				WL_INFORM(("Target addr %s, Channel : %s for RTT\n",
+					bcm_ether_ntoa((const struct ether_addr *)&rtt_target->addr, eabuf),
+					wf_chspec_ntoa(rtt_target->chanspec, chanbuf)));
+				rtt_target++;
+			}
+			break;
+		}
+	}
+	WL_DBG(("leave :target_cnt : %d\n", rtt_param.rtt_target_cnt));
+	if (dhd_dev_rtt_set_cfg(bcmcfg_to_prmry_ndev(cfg), &rtt_param) < 0) {
+		WL_ERR(("Could not set RTT configuration\n"));
+		err = -EINVAL;
+	}
+exit:
+	return err;
+}
+
+static int rtw_cfgvendor_rtt_cancel_config(struct wiphy *wiphy, struct wireless_dev *wdev,
+		const void *data, int len)
+{
+	int err = 0, rem, type, target_cnt = 0;
+	const struct nlattr *iter;
+	struct ether_addr *mac_list = NULL, *mac_addr = NULL;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+		switch (type) {
+		case RTT_ATTRIBUTE_TARGET_CNT:
+			target_cnt = nla_get_u8(iter);
+			mac_list = (struct ether_addr *)kzalloc(target_cnt * ETHER_ADDR_LEN , GFP_KERNEL);
+			if (mac_list == NULL) {
+				WL_ERR(("failed to allocate mem for mac list\n"));
+				goto exit;
+			}
+			mac_addr = &mac_list[0];
+			break;
+		case RTT_ATTRIBUTE_TARGET_MAC:
+			if (mac_addr)
+				_rtw_memcpy(mac_addr++, nla_data(iter), ETHER_ADDR_LEN);
+			else {
+				WL_ERR(("mac_list is NULL\n"));
+				goto exit;
+			}
+			break;
+		}
+		if (dhd_dev_rtt_cancel_cfg(bcmcfg_to_prmry_ndev(cfg), mac_list, target_cnt) < 0) {
+			WL_ERR(("Could not cancel RTT configuration\n"));
+			err = -EINVAL;
+			goto exit;
+		}
+	}
+exit:
+	if (mac_list)
+		kfree(mac_list);
+	return err;
+}
+static int rtw_cfgvendor_rtt_get_capability(struct wiphy *wiphy, struct wireless_dev *wdev,
+		const void *data, int len)
+{
+	int err = 0;
+	struct bcm_cfg80211 *cfg = wiphy_priv(wiphy);
+	rtt_capabilities_t capability;
+
+	err = dhd_dev_rtt_capability(bcmcfg_to_prmry_ndev(cfg), &capability);
+	if (unlikely(err)) {
+		WL_ERR(("Vendor Command reply failed ret:%d\n", err));
+		goto exit;
+	}
+	err =  rtw_cfgvendor_send_cmd_reply(wiphy, bcmcfg_to_prmry_ndev(cfg),
+					    &capability, sizeof(capability));
+
+	if (unlikely(err))
+		WL_ERR(("Vendor Command reply failed ret:%d\n", err));
+exit:
+	return err;
+}
+
+#endif /* RTT_SUPPORT */
+
+#ifdef CONFIG_RTW_CFGVENDOR_LLSTATS
+enum {
+    LSTATS_SUBCMD_GET_INFO = ANDROID_NL80211_SUBCMD_LSTATS_RANGE_START,
+	LSTATS_SUBCMD_SET_INFO,
+	LSTATS_SUBCMD_CLEAR_INFO,
+};
+static void LinkLayerStats(_adapter *padapter)
+{
+	struct xmit_priv		*pxmitpriv = &(padapter->xmitpriv);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct dvobj_priv	*pdvobjpriv = adapter_to_dvobj(padapter);
+	u32 ps_time, trx_total_time;
+	u64 tx_bytes, rx_bytes, trx_total_bytes = 0;
+	u64 tmp = 0;
+
+	RTW_DBG("%s adapter type : %u\n", __func__, padapter->adapter_type);
+
+	tx_bytes = 0;
+	rx_bytes = 0;
+	ps_time = 0;
+	trx_total_time = 0;
+
+	if ( padapter->netif_up == _TRUE ) {
+
+		pwrpriv->on_time = rtw_get_passing_time_ms(pwrpriv->radio_on_start_time);
+
+		ps_time = pwrpriv->pwr_saving_time;
+
+		/* Deviation caused by caculation start time */
+		if ( ps_time > pwrpriv->on_time )
+			ps_time = pwrpriv->on_time;
+
+		tx_bytes = pdvobjpriv->traffic_stat.last_tx_bytes;
+		rx_bytes = pdvobjpriv->traffic_stat.last_rx_bytes;
+		trx_total_bytes = tx_bytes + rx_bytes;
+
+		trx_total_time = pwrpriv->on_time - ps_time;
+
+		if ( trx_total_bytes == 0) {
+			pwrpriv->tx_time = 0;
+			pwrpriv->rx_time = 0;
+		} else {
+
+			/* tx_time = (trx_total_time * tx_total_bytes) / trx_total_bytes; */
+			/* rx_time = (trx_total_time * rx_total_bytes) / trx_total_bytes; */
+
+			tmp = (tx_bytes * trx_total_time);
+			tmp = rtw_division64(tmp, trx_total_bytes);
+			pwrpriv->tx_time = tmp;
+
+			tmp = (rx_bytes * trx_total_time);
+			tmp = rtw_division64(tmp, trx_total_bytes);
+			pwrpriv->rx_time = tmp;
+
+		}
+
+	}
+	else {
+			pwrpriv->on_time = 0;
+			pwrpriv->tx_time = 0;
+			pwrpriv->rx_time = 0;
+	}
+
+#ifdef CONFIG_RTW_WIFI_HAL_DEBUG
+	RTW_INFO("- tx_bytes : %llu rx_bytes : %llu total bytes : %llu\n", tx_bytes, rx_bytes, trx_total_bytes);
+	RTW_INFO("- netif_up = %s, on_time : %u ms\n", padapter->netif_up ? "1":"0", pwrpriv->on_time);
+	RTW_INFO("- pwr_saving_time : %u (%u) ms\n", pwrpriv->pwr_saving_time, ps_time);
+	RTW_INFO("- trx_total_time : %u ms\n", trx_total_time);
+	RTW_INFO("- tx_time : %u ms\n", pwrpriv->tx_time);
+	RTW_INFO("- rx_time : %u ms\n", pwrpriv->rx_time);
+#endif /* CONFIG_RTW_WIFI_HAL_DEBUG */
+
+}
+
+#define DUMMY_TIME_STATICS 99
+static int rtw_cfgvendor_lstats_get_info(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	_adapter *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	wifi_radio_stat_internal *radio;
+	wifi_iface_stat *iface;
+	char *output;
+
+	output = rtw_malloc(sizeof(wifi_radio_stat_internal) + sizeof(wifi_iface_stat));
+	if (output == NULL) {
+		RTW_DBG("Allocate lstats info buffer fail!\n");
+	}
+
+	radio = (wifi_radio_stat_internal *)output;
+
+	radio->num_channels = 0;
+	radio->radio = 1;
+
+	/* to get on_time, tx_time, rx_time */
+	LinkLayerStats(padapter);
+
+	radio->on_time = pwrpriv->on_time;
+	radio->tx_time = pwrpriv->tx_time;
+	radio->rx_time = pwrpriv->rx_time;
+	radio->on_time_scan = 0;
+	radio->on_time_nbd = 0;
+	radio->on_time_gscan = 0;
+	radio->on_time_pno_scan = 0;
+	radio->on_time_hs20 = 0;
+	#ifdef CONFIG_RTW_WIFI_HAL_DEBUG
+	RTW_INFO("==== %s ====\n", __func__);
+	RTW_INFO("radio->radio : %d\n", (radio->radio));
+	RTW_INFO("pwrpriv->on_time : %u ms\n", (pwrpriv->on_time));
+	RTW_INFO("pwrpriv->tx_time :  %u ms\n", (pwrpriv->tx_time));
+	RTW_INFO("pwrpriv->rx_time :  %u ms\n", (pwrpriv->rx_time));
+	RTW_INFO("radio->on_time :  %u ms\n", (radio->on_time));
+	RTW_INFO("radio->tx_time :  %u ms\n", (radio->tx_time));
+	RTW_INFO("radio->rx_time :  %u ms\n", (radio->rx_time));
+	#endif /* CONFIG_RTW_WIFI_HAL_DEBUG */
+
+	RTW_DBG(FUNC_NDEV_FMT" %s\n", FUNC_NDEV_ARG(wdev_to_ndev(wdev)), (char*)data);
+	err =  rtw_cfgvendor_send_cmd_reply(wiphy, wdev_to_ndev(wdev),
+		output, sizeof(wifi_iface_stat) + sizeof(wifi_radio_stat_internal));
+	if (unlikely(err))
+		RTW_ERR(FUNC_NDEV_FMT"Vendor Command reply failed ret:%d \n"
+			, FUNC_NDEV_ARG(wdev_to_ndev(wdev)), err);
+	rtw_mfree(output, sizeof(wifi_iface_stat) + sizeof(wifi_radio_stat_internal));
+	return err;
+}
+static int rtw_cfgvendor_lstats_set_info(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	RTW_INFO("%s\n", __func__);
+	return err;
+}
+static int rtw_cfgvendor_lstats_clear_info(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	RTW_INFO("%s\n", __func__);
+	return err;
+}
+#endif /* CONFIG_RTW_CFGVENDOR_LLSTATS */
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+static int rtw_cfgvendor_set_rssi_monitor(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+        _adapter *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+        struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	int err = 0, rem, type;
+        const struct nlattr *iter;
+
+        RTW_DBG(FUNC_NDEV_FMT" %s\n", FUNC_NDEV_ARG(wdev_to_ndev(wdev)), (char*)data);
+
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+
+		switch (type) {
+        		case RSSI_MONITOR_ATTRIBUTE_MAX_RSSI:
+                                pwdev_priv->rssi_monitor_max = (s8)nla_get_u32(iter);;
+	        		break;
+		        case RSSI_MONITOR_ATTRIBUTE_MIN_RSSI:
+                                pwdev_priv->rssi_monitor_min = (s8)nla_get_u32(iter);
+			        break;
+        		case RSSI_MONITOR_ATTRIBUTE_START:
+                                pwdev_priv->rssi_monitor_enable = (u8)nla_get_u32(iter);
+	        		break;
+		}
+	}
+
+	return err;
+}
+
+void rtw_cfgvendor_rssi_monitor_evt(_adapter *padapter) {
+	struct wireless_dev *wdev =  padapter->rtw_wdev;
+	struct wiphy *wiphy= wdev->wiphy;
+        struct recv_info *precvinfo = &padapter->recvinfo;
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	wlan_network	*pcur_network = &pmlmepriv->dev_cur_network;
+        struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(padapter);
+	struct sk_buff *skb;
+	u32 tot_len = NLMSG_DEFAULT_SIZE;
+	gfp_t kflags;
+        rssi_monitor_evt data ;
+        s8 rssi = precvinfo->rssi;
+
+        if (pwdev_priv->rssi_monitor_enable == 0 || check_fwstate(pmlmepriv, WIFI_ASOC_STATE) != _TRUE)
+                return;
+
+        if (rssi < pwdev_priv->rssi_monitor_max || rssi > pwdev_priv->rssi_monitor_min)
+                return;
+
+	kflags = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
+
+	/* Alloc the SKB for vendor_event */
+	skb = rtw_cfg80211_vendor_event_alloc(wiphy, wdev, tot_len, GOOGLE_RSSI_MONITOR_EVENT, kflags);
+	if (!skb) {
+		goto exit;
+	}
+
+        _rtw_memset(&data, 0, sizeof(data));
+
+        data.version = RSSI_MONITOR_EVT_VERSION;
+        data.cur_rssi = rssi;
+        _rtw_memcpy(data.BSSID, pcur_network->network.MacAddress, sizeof(mac_addr));
+
+        nla_append(skb, sizeof(data), &data);
+
+	rtw_cfg80211_vendor_event(skb, kflags);
+exit:
+	return;
+}
+#endif /* CONFIG_RTW_CFGVENDOR_RSSIMONITR */
+
+#ifdef CONFIG_RTW_CFGVENDOR_WIFI_LOGGER
+static int rtw_cfgvendor_logger_start_logging(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int ret = 0, rem, type;
+	char ring_name[32] = {0};
+	int log_level = 0, flags = 0, time_intval = 0, threshold = 0;
+	const struct nlattr *iter;
+
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+		switch (type) {
+			case LOGGER_ATTRIBUTE_RING_NAME:
+				strncpy(ring_name, nla_data(iter),
+					MIN(sizeof(ring_name) -1, nla_len(iter)));
+				break;
+			case LOGGER_ATTRIBUTE_LOG_LEVEL:
+				log_level = nla_get_u32(iter);
+				break;
+			case LOGGER_ATTRIBUTE_RING_FLAGS:
+				flags = nla_get_u32(iter);
+				break;
+			case LOGGER_ATTRIBUTE_LOG_TIME_INTVAL:
+				time_intval = nla_get_u32(iter);
+				break;
+			case LOGGER_ATTRIBUTE_LOG_MIN_DATA_SIZE:
+				threshold = nla_get_u32(iter);
+				break;
+			default:
+				RTW_ERR("Unknown type: %d\n", type);
+				ret = WIFI_ERROR_INVALID_ARGS;
+				goto exit;
+		}
+	}
+
+exit:
+	return ret;
+}
+static int rtw_cfgvendor_logger_get_feature(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void *data, int len)
+{
+	int err = 0;
+	u32 supported_features = 0;
+
+	err =  rtw_cfgvendor_send_cmd_reply(wiphy, wdev_to_ndev(wdev), &supported_features, sizeof(supported_features));
+
+	if (unlikely(err))
+		RTW_ERR(FUNC_NDEV_FMT" Vendor Command reply failed ret:%d\n"
+			, FUNC_NDEV_ARG(wdev_to_ndev(wdev)), err);
+
+	return err;
+}
+static int rtw_cfgvendor_logger_get_version(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void *data, int len)
+{
+	_adapter *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+	int ret = 0, rem, type;
+	int buf_len = 1024;
+	char *buf_ptr;
+	const struct nlattr *iter;
+	gfp_t kflags;
+
+	kflags = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
+	buf_ptr = kzalloc(buf_len, kflags);
+	if (!buf_ptr) {
+		RTW_ERR("failed to allocate the buffer for version n");
+		ret = -ENOMEM;
+		goto exit;
+	}
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+		switch (type) {
+			case LOGGER_ATTRIBUTE_GET_DRIVER:
+				_rtw_memcpy(buf_ptr, DRIVERVERSION, strlen(DRIVERVERSION)+1);
+				break;
+			case LOGGER_ATTRIBUTE_GET_FW:
+				rtw_phl_get_fw_ver(GET_PHL_INFO(adapter_to_dvobj(padapter)), buf_ptr, buf_len);
+				break;
+			default:
+				RTW_ERR("Unknown type: %d\n", type);
+				ret = -EINVAL;
+				goto exit;
+		}
+	}
+	if (ret < 0) {
+		RTW_ERR("failed to get the version %d\n", ret);
+		goto exit;
+	}
+
+
+	ret =  rtw_cfgvendor_send_cmd_reply(wiphy, wdev_to_ndev(wdev), buf_ptr, strlen(buf_ptr));
+exit:
+	kfree(buf_ptr);
+	return ret;
+}
+
+static int rtw_cfgvendor_logger_get_ring_status(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int ret = 0;
+	int ring_id;
+	char ring_buf_name[] = "RTW_RING_BUFFER";
+
+	struct sk_buff *skb;
+	wifi_ring_buffer_status ring_status;
+
+
+	_rtw_memcpy(ring_status.name, ring_buf_name, strlen(ring_buf_name)+1);
+	ring_status.ring_id = 1;
+	/* Alloc the SKB for vendor_event */
+	skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy,
+		sizeof(wifi_ring_buffer_status));
+	if (!skb) {
+		RTW_ERR("skb allocation is failed\n");
+		ret = FAIL;
+		goto exit;
+	}
+
+	nla_put_u32(skb, LOGGER_ATTRIBUTE_RING_NUM, 1);
+	nla_put(skb, LOGGER_ATTRIBUTE_RING_STATUS, sizeof(wifi_ring_buffer_status),
+				&ring_status);
+	ret = cfg80211_vendor_cmd_reply(skb);
+
+	if (ret) {
+		RTW_ERR("Vendor Command reply failed ret:%d \n", ret);
+	}
+exit:
+	return ret;
+}
+
+static int rtw_cfgvendor_logger_get_ring_data(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int ret = 0, rem, type;
+	char ring_name[32] = {0};
+	const struct nlattr *iter;
+
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+		switch (type) {
+			case LOGGER_ATTRIBUTE_RING_NAME:
+				strncpy(ring_name, nla_data(iter),
+					MIN(sizeof(ring_name) -1, nla_len(iter)));
+				RTW_INFO(" %s LOGGER_ATTRIBUTE_RING_NAME : %s\n", __func__, ring_name);
+				break;
+			default:
+				RTW_ERR("Unknown type: %d\n", type);
+				return ret;
+		}
+	}
+
+
+	return ret;
+}
+
+static int rtw_cfgvendor_logger_get_firmware_memory_dump(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int ret = WIFI_ERROR_NOT_SUPPORTED;
+
+	return ret;
+}
+
+static int rtw_cfgvendor_logger_start_pkt_fate_monitoring(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int ret = WIFI_SUCCESS;
+
+	return ret;
+}
+
+static int rtw_cfgvendor_logger_get_tx_pkt_fates(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int ret = WIFI_SUCCESS;
+
+	return ret;
+}
+
+static int rtw_cfgvendor_logger_get_rx_pkt_fates(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+	int ret = WIFI_SUCCESS;
+
+	return ret;
+}
+
+#endif /* CONFIG_RTW_CFGVENDOR_WIFI_LOGGER */
+
+#ifdef CONFIG_RTW_WIFI_HAL
+#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+
+#ifndef ETHER_ISMULTI
+#define ETHER_ISMULTI(ea) (((const u8 *)(ea))[0] & 1)
+#endif
+
+
+static u8 null_addr[ETH_ALEN] = {0};
+static void rtw_hal_random_gen_mac_addr(u8 *mac_addr)
+{
+	do {
+		get_random_bytes(&mac_addr[3], ETH_ALEN-3);
+		if (_rtw_memcmp(mac_addr, null_addr, ETH_ALEN) != _TRUE)
+			break;
+	} while(1);
+}
+
+void rtw_hal_pno_random_gen_mac_addr(_adapter *adapter)
+{
+	u8 mac_addr[ETH_ALEN];
+	struct rtw_wdev_priv *pwdev_priv = adapter_wdev_data(adapter);
+
+	_rtw_memcpy(mac_addr, pwdev_priv->pno_mac_addr, ETH_ALEN);
+	if (mac_addr[0] == 0xFF) return;
+	rtw_hal_random_gen_mac_addr(mac_addr);
+	_rtw_memcpy(pwdev_priv->pno_mac_addr, mac_addr, ETH_ALEN);
+#ifdef CONFIG_RTW_DEBUG
+	print_hex_dump(KERN_DEBUG, "pno_mac_addr: ",
+		       DUMP_PREFIX_OFFSET, 16, 1, pwdev_priv->pno_mac_addr,
+		       ETH_ALEN, 1);
+#endif
+}
+
+static int rtw_cfgvendor_set_rand_mac_oui(struct wiphy *wiphy,
+		struct wireless_dev *wdev, const void  *data, int len)
+{
+	int err = 0;
+	_adapter *adapter;
+	void *devaddr;
+	struct net_device *netdev;
+	int type, mac_len;
+	u8 pno_random_mac_oui[3];
+	u8 mac_addr[ETH_ALEN] = {0};
+	struct pwrctrl_priv *pwrctl;
+	struct rtw_wdev_priv *pwdev_priv;
+
+	type = nla_type(data);
+	mac_len = nla_len(data);
+	if (mac_len != 3) {
+		RTW_ERR("%s oui len error %d != 3\n", __func__, mac_len);
+		return -1;
+	}
+
+	if (type == ANDR_WIFI_ATTRIBUTE_RANDOM_MAC_OUI) {
+		_rtw_memcpy(pno_random_mac_oui, nla_data(data), 3);
+		print_hex_dump(KERN_DEBUG, "pno_random_mac_oui: ",
+			       DUMP_PREFIX_OFFSET, 16, 1, pno_random_mac_oui,
+			       3, 1);
+
+		if (ETHER_ISMULTI(pno_random_mac_oui)) {
+			pr_err("%s: oui is multicast address\n", __func__);
+			return -1;
+		}
+
+		adapter = wiphy_to_adapter(wiphy);
+		if (adapter == NULL) {
+			pr_err("%s: wiphy_to_adapter == NULL\n", __func__);
+			return -1;
+		}
+
+		pwdev_priv = adapter_wdev_data(adapter);
+
+		_rtw_memcpy(mac_addr, pno_random_mac_oui, 3);
+		rtw_hal_random_gen_mac_addr(mac_addr);
+		_rtw_memcpy(pwdev_priv->pno_mac_addr, mac_addr, ETH_ALEN);
+#ifdef CONFIG_RTW_DEBUG
+		print_hex_dump(KERN_DEBUG, "pno_mac_addr: ",
+			       DUMP_PREFIX_OFFSET, 16, 1, pwdev_priv->pno_mac_addr,
+			       ETH_ALEN, 1);
+#endif
+	} else {
+		RTW_ERR("%s oui type error %x != 0x2\n", __func__, type);
+		err = -1;
+	}
+
+
+	return err;
+}
+
+#endif
+
+#ifdef CONFIG_RTW_CFGVENDOR_WIFI_OFFLOAD
+static int rtw_cfgvendor_start_mkeep_alive(struct wiphy *wiphy, struct wireless_dev *wdev,
+	const void *data, int len)
+{
+	int ret = WIFI_SUCCESS;
+
+	RTW_INFO("%s : TODO\n", __func__);
+
+	return ret;
+}
+
+static int rtw_cfgvendor_stop_mkeep_alive(struct wiphy *wiphy, struct wireless_dev *wdev,
+	const void *data, int len)
+{
+	int ret = WIFI_SUCCESS;
+
+	RTW_INFO("%s : TODO\n", __func__);
+
+	return ret;
+}
+#endif
+
+static int rtw_cfgvendor_set_nodfs_flag(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void *data, int len)
+{
+	int err = 0;
+	int type;
+	u32 nodfs = 0;
+	_adapter *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+
+	RTW_DBG(FUNC_NDEV_FMT" %s\n", FUNC_NDEV_ARG(wdev_to_ndev(wdev)), (char*)data);
+
+	type = nla_type(data);
+	if (type == ANDR_WIFI_ATTRIBUTE_NODFS_SET) {
+		nodfs = nla_get_u32(data);
+		adapter_to_dvobj(padapter)->nodfs = nodfs;
+	} else {
+		err = -EINVAL;
+	}
+
+	RTW_INFO("%s nodfs=%d, err=%d\n", __func__, nodfs, err);
+
+	return err;
+}
+
+static int rtw_cfgvendor_set_country(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+#define CNTRY_BUF_SZ	4	/* Country string is 3 bytes + NUL */
+	int err = 0, rem, type;
+	char country_code[CNTRY_BUF_SZ] = {0};
+	const struct nlattr *iter;
+	_adapter *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+
+	RTW_DBG(FUNC_NDEV_FMT" %s\n", FUNC_NDEV_ARG(wdev_to_ndev(wdev)), (char*)data);
+
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+		switch (type) {
+			case ANDR_WIFI_ATTRIBUTE_COUNTRY:
+				_rtw_memcpy(country_code, nla_data(iter),
+					MIN(nla_len(iter), CNTRY_BUF_SZ));
+				break;
+			default:
+				RTW_ERR("Unknown type: %d\n", type);
+				return -EINVAL;
+		}
+	}
+
+	RTW_INFO("%s country_code:\"%c%c\" \n", __func__, country_code[0], country_code[1]);
+
+	rtw_set_country(padapter, country_code, RTW_ENV_NUM, RTW_REGD_SET_BY_USER);
+
+	return err;
+}
+
+static int rtw_cfgvendor_set_nd_offload(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void *data, int len)
+{
+	int err = 0;
+	int type;
+	u8 nd_en = 0;
+	_adapter *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+
+	RTW_DBG(FUNC_NDEV_FMT" %s\n", FUNC_NDEV_ARG(wdev_to_ndev(wdev)), (char*)data);
+
+	type = nla_type(data);
+	if (type == ANDR_WIFI_ATTRIBUTE_ND_OFFLOAD_VALUE) {
+		nd_en = nla_get_u8(data);
+		/* ND has been enabled when wow is enabled */
+	} else {
+		err = -EINVAL;
+	}
+
+	RTW_INFO("%s nd_en=%d, err=%d\n", __func__, nd_en, err);
+
+	return err;
+}
+#ifdef CONFIG_WOW_APF
+/*
+ * https://android.googlesource.com/platform/hardware/interfaces/+/refs/heads/android15-release/wifi/legacy_headers/include/hardware_legacy/wifi_hal.h
+ */
+static int rtw_cfgvendor_get_capabilities(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void  *data, int len)
+{
+		int ret = 0;
+		u32 version = CONFIG_APF_VERSION, maxlen = CONFIG_APF_RAM_SIZE;
+		struct sk_buff *skb;
+
+		RTW_INFO("%s()\n", __func__);
+
+		/* Alloc the SKB for vendor_event */
+		skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy, 2 * sizeof(u32));
+		if (!skb) {
+			RTW_ERR("skb allocation is failed\n");
+			ret = FAIL;
+			goto exit;
+		}
+
+		nla_put_u32(skb, APF_ATTRIBUTE_VERSION, version);
+		nla_put_u32(skb, APF_ATTRIBUTE_MAX_LEN, maxlen);
+
+		ret = cfg80211_vendor_cmd_reply(skb);
+
+		if (ret) {
+			RTW_ERR("Vendor Command reply failed ret:%d \n", ret);
+		}
+	exit:
+		return ret;
+
+}
+
+static int rtw_cfgvendor_set_packet_filter(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void *data, int len)
+{
+	struct _ADAPTER *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_apf_info *info = &wowpriv->apf_info;
+	int err = 0, rem, type;
+	u16 apf_prog_len, i;
+	u8 *apf_prog;
+	u8 index;
+	const struct nlattr *iter;
+
+	RTW_INFO("%s()\n", __func__);
+
+	nla_for_each_attr(iter, data, len, rem) {
+		type = nla_type(iter);
+		switch (type) {
+			case APF_ATTRIBUTE_PROGRAM_LEN:
+				apf_prog_len =  nla_get_u32(iter);
+				break;
+			case APF_ATTRIBUTE_PROGRAM:
+#ifdef CONFIG_WOW_APF_DBG
+				RTW_INFO("[APFDBG] %s() APF_ATTRIBUTE_PROGRAM, len(%u)\n", __func__, apf_prog_len);
+				RTW_INFO_DUMP(NULL, nla_data(iter), nla_len(iter));
+#endif
+				apf_prog = rtw_vmalloc(apf_prog_len);
+				if (apf_prog != NULL) {
+					_rtw_memcpy(apf_prog, nla_data(iter), apf_prog_len);
+				} else {
+					RTW_ERR("Cannot alloc memory for apf prog\n");
+					err = -EINVAL;
+					goto exit;
+				}
+				break;
+			default:
+				RTW_ERR("Unknown type: %d\n", type);
+				err = -EINVAL;
+				goto exit;
+		}
+	}
+	/* process apf prog */
+	if (apf_prog_len > MAX_APF_PROG_SIZE) {
+		err = -EINVAL;
+		goto exit;
+	}
+
+	RTW_INFO("Set apf prog\n");
+	rtw_wow_set_apf(padapter, apf_prog, apf_prog_len);
+
+exit:
+	if (apf_prog)
+		rtw_vmfree(apf_prog, apf_prog_len);
+	return err;
+}
+
+static int rtw_cfgvendor_get_packet_filter(struct wiphy *wiphy,
+	struct wireless_dev *wdev, const void *data, int len)
+{
+	struct _ADAPTER *padapter = GET_PRIMARY_ADAPTER(wiphy_to_adapter(wiphy));
+	struct wow_priv *wowpriv = adapter_to_wowlan(padapter);
+	struct rtw_apf_info *info = &wowpriv->apf_info;
+	int ret = 0;
+	struct sk_buff *skb;
+
+	RTW_INFO("%s()\n", __func__);
+	/* Alloc the SKB for vendor_event */
+	skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy, MAX_APF_PROG_SIZE + sizeof(u32));
+	if (!skb) {
+		RTW_ERR("skb allocation is failed\n");
+		ret = FAIL;
+		goto exit;
+	}
+	nla_put_u32(skb, APF_ATTRIBUTE_PROGRAM_LEN, MAX_APF_PROG_SIZE);
+	nla_put(skb, APF_ATTRIBUTE_PROGRAM, MAX_APF_PROG_SIZE, info->apf_prog);
+exit:
+	return ret;
+}
+
+#endif /* CONFIG_WOW_APF */
+#endif /* CONFIG_RTW_WIFI_HAL */
+
+static const struct wiphy_vendor_command rtw_vendor_cmds[] = {
+#if defined(GSCAN_SUPPORT) && 0
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_GET_CAPABILITIES
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_gscan_get_capabilities,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_SET_CONFIG
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_set_scan_cfg,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_SET_SCAN_CONFIG
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_set_batch_scan_cfg,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_ENABLE_GSCAN
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_initiate_gscan,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_ENABLE_FULL_SCAN_RESULTS
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_enable_full_scan_result,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_SET_HOTLIST
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_hotlist_cfg,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_SET_SIGNIFICANT_CHANGE_CONFIG
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_significant_change_cfg,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_GET_SCAN_RESULTS
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_gscan_get_batch_results,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = GSCAN_SUBCMD_GET_CHANNEL_LIST
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_gscan_get_channel_list,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#endif /* GSCAN_SUPPORT */
+#if defined(RTT_SUPPORT) && 0
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = RTT_SUBCMD_SET_CONFIG
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_rtt_set_config,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = RTT_SUBCMD_CANCEL_CONFIG
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_rtt_cancel_config,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = RTT_SUBCMD_GETCAPABILITY
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_rtt_get_capability,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#endif /* RTT_SUPPORT */
+#ifdef CONFIG_RTW_CFGVENDOR_LLSTATS
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LSTATS_SUBCMD_GET_INFO
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_lstats_get_info,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LSTATS_SUBCMD_SET_INFO
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_lstats_set_info,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LSTATS_SUBCMD_CLEAR_INFO
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_lstats_clear_info,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#endif /* CONFIG_RTW_CFGVENDOR_LLSTATS */
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+        {
+                {
+                        .vendor_id = OUI_GOOGLE,
+                        .subcmd = WIFI_SUBCMD_SET_RSSI_MONITOR
+                },
+                .flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+                .doit = rtw_cfgvendor_set_rssi_monitor,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#endif /* CONFIG_RTW_CFGVENDOR_RSSIMONITOR */
+#ifdef CONFIG_RTW_CFGVENDOR_WIFI_LOGGER
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_START_LOGGING
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_start_logging,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_GET_FEATURE
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_get_feature,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_GET_VER
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_get_version,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_GET_RING_STATUS
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_get_ring_status,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_GET_RING_DATA
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_get_ring_data,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy= VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_TRIGGER_MEM_DUMP
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_get_firmware_memory_dump,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_START_PKT_FATE_MONITORING
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_start_pkt_fate_monitoring,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_GET_TX_PKT_FATES
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_get_tx_pkt_fates,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = LOGGER_GET_RX_PKT_FATES
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_logger_get_rx_pkt_fates,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#endif /* CONFIG_RTW_CFGVENDOR_WIFI_LOGGER */
+#ifdef CONFIG_RTW_WIFI_HAL
+#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_SUBCMD_SET_PNO_RANDOM_MAC_OUI
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_set_rand_mac_oui,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#endif
+#ifdef CONFIG_RTW_CFGVENDOR_WIFI_OFFLOAD
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_OFFLOAD_SUBCMD_START_MKEEP_ALIVE
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_start_mkeep_alive,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_OFFLOAD_SUBCMD_STOP_MKEEP_ALIVE
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_stop_mkeep_alive,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#endif
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_SUBCMD_NODFS_SET
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_set_nodfs_flag,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_SUBCMD_SET_COUNTRY_CODE
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_set_country,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_SUBCMD_CONFIG_ND_OFFLOAD
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_set_nd_offload,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+#ifdef CONFIG_WOW_APF
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = APF_SUBCMD_GET_CAPABILITIES
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_get_capabilities,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+			.policy = VENDOR_CMD_RAW_DATA,
+#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = APF_SUBCMD_SET_FILTER
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_set_packet_filter,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = APF_SUBCMD_GET_FILTER
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_get_packet_filter,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+#endif
+	},
+#endif /* CONFIG_WOW_APF */
+#endif /* CONFIG_RTW_WIFI_HAL */
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_SUBCMD_GET_FEATURE_SET
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_get_feature_set,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	},
+	{
+		{
+			.vendor_id = OUI_GOOGLE,
+			.subcmd = WIFI_SUBCMD_GET_FEATURE_SET_MATRIX
+		},
+		.flags = WIPHY_VENDOR_CMD_NEED_WDEV | WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = rtw_cfgvendor_get_feature_set_matrix,
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0))
+		.policy = VENDOR_CMD_RAW_DATA,
+		#endif
+	}
+};
+
+static const struct  nl80211_vendor_cmd_info rtw_vendor_events[] = {
+#if defined(GSCAN_SUPPORT) && 0
+	{ OUI_GOOGLE, GSCAN_EVENT_SIGNIFICANT_CHANGE_RESULTS },
+	{ OUI_GOOGLE, GSCAN_EVENT_HOTLIST_RESULTS_FOUND },
+	{ OUI_GOOGLE, GSCAN_EVENT_SCAN_RESULTS_AVAILABLE },
+	{ OUI_GOOGLE, GSCAN_EVENT_FULL_SCAN_RESULTS },
+#endif /* GSCAN_SUPPORT */
+#if defined(RTT_SUPPORT) && 0
+	{ OUI_GOOGLE, RTT_EVENT_COMPLETE },
+#endif /* RTT_SUPPORT */
+
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+	{ OUI_GOOGLE, GOOGLE_RSSI_MONITOR_EVENT },
+#endif /* RTW_CFGVENDOR_RSSIMONITR */
+
+#if defined(GSCAN_SUPPORT) && 0
+	{ OUI_GOOGLE, GSCAN_EVENT_COMPLETE_SCAN },
+	{ OUI_GOOGLE, GSCAN_EVENT_HOTLIST_RESULTS_LOST }
+#endif /* GSCAN_SUPPORT */
+};
+
+int rtw_cfgvendor_attach(struct wiphy *wiphy)
+{
+
+	RTW_INFO("Register RTW cfg80211 vendor cmd(0x%x) interface\n", NL80211_CMD_VENDOR);
+
+	wiphy->vendor_commands	= rtw_vendor_cmds;
+	wiphy->n_vendor_commands = ARRAY_SIZE(rtw_vendor_cmds);
+	wiphy->vendor_events	= rtw_vendor_events;
+	wiphy->n_vendor_events	= ARRAY_SIZE(rtw_vendor_events);
+
+	return 0;
+}
+
+int rtw_cfgvendor_detach(struct wiphy *wiphy)
+{
+	RTW_INFO("Vendor: Unregister RTW cfg80211 vendor interface\n");
+
+	wiphy->vendor_commands  = NULL;
+	wiphy->vendor_events    = NULL;
+	wiphy->n_vendor_commands = 0;
+	wiphy->n_vendor_events  = 0;
+
+	return 0;
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) || defined(RTW_VENDOR_EXT_SUPPORT) */
+
+#endif /* CONFIG_IOCTL_CFG80211 */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfgvendor.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfgvendor.h
new file mode 100644
index 000000000000..db305a0f0b91
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_cfgvendor.h
@@ -0,0 +1,643 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef _RTW_CFGVENDOR_H_
+#define _RTW_CFGVENDOR_H_
+
+#define OUI_GOOGLE  0x001A11
+#define ATTRIBUTE_U32_LEN                  (NLA_HDRLEN  + 4)
+#define VENDOR_ID_OVERHEAD                 ATTRIBUTE_U32_LEN
+#define VENDOR_SUBCMD_OVERHEAD             ATTRIBUTE_U32_LEN
+#define VENDOR_DATA_OVERHEAD               (NLA_HDRLEN)
+
+#define SCAN_RESULTS_COMPLETE_FLAG_LEN       ATTRIBUTE_U32_LEN
+#define SCAN_INDEX_HDR_LEN                   (NLA_HDRLEN)
+#define SCAN_ID_HDR_LEN                      ATTRIBUTE_U32_LEN
+#define SCAN_FLAGS_HDR_LEN                   ATTRIBUTE_U32_LEN
+#define GSCAN_NUM_RESULTS_HDR_LEN            ATTRIBUTE_U32_LEN
+#define GSCAN_RESULTS_HDR_LEN                (NLA_HDRLEN)
+#define GSCAN_BATCH_RESULT_HDR_LEN  (SCAN_INDEX_HDR_LEN + SCAN_ID_HDR_LEN + \
+				     SCAN_FLAGS_HDR_LEN + \
+				     GSCAN_NUM_RESULTS_HDR_LEN + \
+				     GSCAN_RESULTS_HDR_LEN)
+
+#define VENDOR_REPLY_OVERHEAD       (VENDOR_ID_OVERHEAD + \
+				     VENDOR_SUBCMD_OVERHEAD + \
+				     VENDOR_DATA_OVERHEAD)
+typedef enum {
+    /* don't use 0 as a valid subcommand */
+    VENDOR_NL80211_SUBCMD_UNSPECIFIED,
+
+    /* define all vendor startup commands between 0x0 and 0x0FFF */
+    VENDOR_NL80211_SUBCMD_RANGE_START = 0x0001,
+    VENDOR_NL80211_SUBCMD_RANGE_END   = 0x0FFF,
+
+    /* define all GScan related commands between 0x1000 and 0x10FF */
+    ANDROID_NL80211_SUBCMD_GSCAN_RANGE_START = 0x1000,
+    ANDROID_NL80211_SUBCMD_GSCAN_RANGE_END   = 0x10FF,
+
+    /* define all NearbyDiscovery related commands between 0x1100 and 0x11FF */
+    ANDROID_NL80211_SUBCMD_NBD_RANGE_START = 0x1100,
+    ANDROID_NL80211_SUBCMD_NBD_RANGE_END   = 0x11FF,
+
+    /* define all RTT related commands between 0x1100 and 0x11FF */
+    ANDROID_NL80211_SUBCMD_RTT_RANGE_START = 0x1100,
+    ANDROID_NL80211_SUBCMD_RTT_RANGE_END   = 0x11FF,
+
+    ANDROID_NL80211_SUBCMD_LSTATS_RANGE_START = 0x1200,
+    ANDROID_NL80211_SUBCMD_LSTATS_RANGE_END   = 0x12FF,
+
+    /* define all Logger related commands between 0x1400 and 0x14FF */
+    ANDROID_NL80211_SUBCMD_DEBUG_RANGE_START = 0x1400,
+    ANDROID_NL80211_SUBCMD_DEBUG_RANGE_END   = 0x14FF,
+
+    /* define all wifi offload related commands between 0x1600 and 0x16FF */
+    ANDROID_NL80211_SUBCMD_WIFI_OFFLOAD_RANGE_START = 0x1600,
+    ANDROID_NL80211_SUBCMD_WIFI_OFFLOAD_RANGE_END   = 0x16FF,
+
+    /* define all NAN related commands between 0x1700 and 0x17FF */
+    ANDROID_NL80211_SUBCMD_NAN_RANGE_START = 0x1700,
+    ANDROID_NL80211_SUBCMD_NAN_RANGE_END   = 0x17FF,
+
+    /* define all Android Packet Filter related commands between 0x1800 and 0x18FF */
+    ANDROID_NL80211_SUBCMD_PKT_FILTER_RANGE_START = 0x1800,
+    ANDROID_NL80211_SUBCMD_PKT_FILTER_RANGE_END   = 0x18FF,
+
+    /* This is reserved for future usage */
+
+} ANDROID_VENDOR_SUB_COMMAND;
+
+enum rtw_vendor_subcmd {
+    GSCAN_SUBCMD_GET_CAPABILITIES = ANDROID_NL80211_SUBCMD_GSCAN_RANGE_START,
+
+    GSCAN_SUBCMD_SET_CONFIG,                            /* 0x1001 */
+
+    GSCAN_SUBCMD_SET_SCAN_CONFIG,                       /* 0x1002 */
+    GSCAN_SUBCMD_ENABLE_GSCAN,                          /* 0x1003 */
+    GSCAN_SUBCMD_GET_SCAN_RESULTS,                      /* 0x1004 */
+    GSCAN_SUBCMD_SCAN_RESULTS,                          /* 0x1005 */
+
+    GSCAN_SUBCMD_SET_HOTLIST,                           /* 0x1006 */
+
+    GSCAN_SUBCMD_SET_SIGNIFICANT_CHANGE_CONFIG,         /* 0x1007 */
+    GSCAN_SUBCMD_ENABLE_FULL_SCAN_RESULTS,              /* 0x1008 */
+    GSCAN_SUBCMD_GET_CHANNEL_LIST,                       /* 0x1009 */
+
+    WIFI_SUBCMD_GET_FEATURE_SET,                         /* 0x100A */
+    WIFI_SUBCMD_GET_FEATURE_SET_MATRIX,                  /* 0x100B */
+    WIFI_SUBCMD_SET_PNO_RANDOM_MAC_OUI,                  /* 0x100C */
+    WIFI_SUBCMD_NODFS_SET,                               /* 0x100D */
+    WIFI_SUBCMD_SET_COUNTRY_CODE,                             /* 0x100E */
+    /* Add more sub commands here */
+    GSCAN_SUBCMD_SET_EPNO_SSID,                          /* 0x100F */
+
+    WIFI_SUBCMD_SET_SSID_WHITE_LIST,                    /* 0x1010 */
+    WIFI_SUBCMD_SET_ROAM_PARAMS,                        /* 0x1011 */
+    WIFI_SUBCMD_ENABLE_LAZY_ROAM,                       /* 0x1012 */
+    WIFI_SUBCMD_SET_BSSID_PREF,                         /* 0x1013 */
+    WIFI_SUBCMD_SET_BSSID_BLACKLIST,                     /* 0x1014 */
+
+    GSCAN_SUBCMD_ANQPO_CONFIG,                          /* 0x1015 */
+    WIFI_SUBCMD_SET_RSSI_MONITOR,                       /* 0x1016 */
+    WIFI_SUBCMD_CONFIG_ND_OFFLOAD,                      /* 0x1017 */
+    /* Add more sub commands here */
+
+    GSCAN_SUBCMD_MAX,
+
+	RTT_SUBCMD_SET_CONFIG = ANDROID_NL80211_SUBCMD_RTT_RANGE_START,
+	RTT_SUBCMD_CANCEL_CONFIG,
+	RTT_SUBCMD_GETCAPABILITY,
+
+    APF_SUBCMD_GET_CAPABILITIES = ANDROID_NL80211_SUBCMD_PKT_FILTER_RANGE_START,
+    APF_SUBCMD_SET_FILTER,
+    APF_SUBCMD_GET_FILTER,
+
+    LOGGER_START_LOGGING = ANDROID_NL80211_SUBCMD_DEBUG_RANGE_START,
+    LOGGER_TRIGGER_MEM_DUMP,
+    LOGGER_GET_MEM_DUMP,
+    LOGGER_GET_VER,
+    LOGGER_GET_RING_STATUS,
+    LOGGER_GET_RING_DATA,
+    LOGGER_GET_FEATURE,
+    LOGGER_RESET_LOGGING,
+    LOGGER_TRIGGER_DRIVER_MEM_DUMP,
+    LOGGER_GET_DRIVER_MEM_DUMP,
+    LOGGER_START_PKT_FATE_MONITORING,
+    LOGGER_GET_TX_PKT_FATES,
+    LOGGER_GET_RX_PKT_FATES,
+
+	WIFI_OFFLOAD_SUBCMD_START_MKEEP_ALIVE = ANDROID_NL80211_SUBCMD_WIFI_OFFLOAD_RANGE_START,
+	WIFI_OFFLOAD_SUBCMD_STOP_MKEEP_ALIVE,
+
+	VENDOR_SUBCMD_MAX
+};
+
+enum gscan_attributes {
+	GSCAN_ATTRIBUTE_NUM_BUCKETS = 10,
+	GSCAN_ATTRIBUTE_BASE_PERIOD,
+	GSCAN_ATTRIBUTE_BUCKETS_BAND,
+	GSCAN_ATTRIBUTE_BUCKET_ID,
+	GSCAN_ATTRIBUTE_BUCKET_PERIOD,
+	GSCAN_ATTRIBUTE_BUCKET_NUM_CHANNELS,
+	GSCAN_ATTRIBUTE_BUCKET_CHANNELS,
+	GSCAN_ATTRIBUTE_NUM_AP_PER_SCAN,
+	GSCAN_ATTRIBUTE_REPORT_THRESHOLD,
+	GSCAN_ATTRIBUTE_NUM_SCANS_TO_CACHE,
+	GSCAN_ATTRIBUTE_BAND = GSCAN_ATTRIBUTE_BUCKETS_BAND,
+
+	GSCAN_ATTRIBUTE_ENABLE_FEATURE = 20,
+	GSCAN_ATTRIBUTE_SCAN_RESULTS_COMPLETE,
+	GSCAN_ATTRIBUTE_FLUSH_FEATURE,
+	GSCAN_ATTRIBUTE_ENABLE_FULL_SCAN_RESULTS,
+	GSCAN_ATTRIBUTE_REPORT_EVENTS,
+	/* remaining reserved for additional attributes */
+	GSCAN_ATTRIBUTE_NUM_OF_RESULTS = 30,
+	GSCAN_ATTRIBUTE_FLUSH_RESULTS,
+	GSCAN_ATTRIBUTE_SCAN_RESULTS,                       /* flat array of wifi_scan_result */
+	GSCAN_ATTRIBUTE_SCAN_ID,                            /* indicates scan number */
+	GSCAN_ATTRIBUTE_SCAN_FLAGS,                         /* indicates if scan was aborted */
+	GSCAN_ATTRIBUTE_AP_FLAGS,                           /* flags on significant change event */
+	GSCAN_ATTRIBUTE_NUM_CHANNELS,
+	GSCAN_ATTRIBUTE_CHANNEL_LIST,
+
+	/* remaining reserved for additional attributes */
+
+	GSCAN_ATTRIBUTE_SSID = 40,
+	GSCAN_ATTRIBUTE_BSSID,
+	GSCAN_ATTRIBUTE_CHANNEL,
+	GSCAN_ATTRIBUTE_RSSI,
+	GSCAN_ATTRIBUTE_TIMESTAMP,
+	GSCAN_ATTRIBUTE_RTT,
+	GSCAN_ATTRIBUTE_RTTSD,
+
+	/* remaining reserved for additional attributes */
+
+	GSCAN_ATTRIBUTE_HOTLIST_BSSIDS = 50,
+	GSCAN_ATTRIBUTE_RSSI_LOW,
+	GSCAN_ATTRIBUTE_RSSI_HIGH,
+	GSCAN_ATTRIBUTE_HOSTLIST_BSSID_ELEM,
+	GSCAN_ATTRIBUTE_HOTLIST_FLUSH,
+
+	/* remaining reserved for additional attributes */
+	GSCAN_ATTRIBUTE_RSSI_SAMPLE_SIZE = 60,
+	GSCAN_ATTRIBUTE_LOST_AP_SAMPLE_SIZE,
+	GSCAN_ATTRIBUTE_MIN_BREACHING,
+	GSCAN_ATTRIBUTE_SIGNIFICANT_CHANGE_BSSIDS,
+	GSCAN_ATTRIBUTE_SIGNIFICANT_CHANGE_FLUSH,
+	GSCAN_ATTRIBUTE_MAX
+};
+
+enum gscan_bucket_attributes {
+	GSCAN_ATTRIBUTE_CH_BUCKET_1,
+	GSCAN_ATTRIBUTE_CH_BUCKET_2,
+	GSCAN_ATTRIBUTE_CH_BUCKET_3,
+	GSCAN_ATTRIBUTE_CH_BUCKET_4,
+	GSCAN_ATTRIBUTE_CH_BUCKET_5,
+	GSCAN_ATTRIBUTE_CH_BUCKET_6,
+	GSCAN_ATTRIBUTE_CH_BUCKET_7
+};
+
+enum gscan_ch_attributes {
+	GSCAN_ATTRIBUTE_CH_ID_1,
+	GSCAN_ATTRIBUTE_CH_ID_2,
+	GSCAN_ATTRIBUTE_CH_ID_3,
+	GSCAN_ATTRIBUTE_CH_ID_4,
+	GSCAN_ATTRIBUTE_CH_ID_5,
+	GSCAN_ATTRIBUTE_CH_ID_6,
+	GSCAN_ATTRIBUTE_CH_ID_7
+};
+
+enum wifi_rssi_monitor_attr {
+        RSSI_MONITOR_ATTRIBUTE_MAX_RSSI,
+        RSSI_MONITOR_ATTRIBUTE_MIN_RSSI,
+        RSSI_MONITOR_ATTRIBUTE_START,
+};
+
+#ifdef CONFIG_WOW_APF
+enum wifi_apf_attr {
+    APF_ATTRIBUTE_VERSION,
+    APF_ATTRIBUTE_MAX_LEN,
+    APF_ATTRIBUTE_PROGRAM,
+    APF_ATTRIBUTE_PROGRAM_LEN
+};
+#endif /* CONFIG_WOW_APF */
+
+enum rtt_attributes {
+	RTT_ATTRIBUTE_TARGET_CNT,
+	RTT_ATTRIBUTE_TARGET_INFO,
+	RTT_ATTRIBUTE_TARGET_MAC,
+	RTT_ATTRIBUTE_TARGET_TYPE,
+	RTT_ATTRIBUTE_TARGET_PEER,
+	RTT_ATTRIBUTE_TARGET_CHAN,
+	RTT_ATTRIBUTE_TARGET_MODE,
+	RTT_ATTRIBUTE_TARGET_INTERVAL,
+	RTT_ATTRIBUTE_TARGET_NUM_MEASUREMENT,
+	RTT_ATTRIBUTE_TARGET_NUM_PKT,
+	RTT_ATTRIBUTE_TARGET_NUM_RETRY
+};
+
+enum logger_attributes {
+	LOGGER_ATTRIBUTE_GET_DRIVER,
+	LOGGER_ATTRIBUTE_GET_FW,
+	LOGGER_ATTRIBUTE_RING_ID,
+	LOGGER_ATTRIBUTE_RING_NAME,
+	LOGGER_ATTRIBUTE_RING_FLAGS,
+	LOGGER_ATTRIBUTE_LOG_LEVEL,
+	LOGGER_ATTRIBUTE_LOG_TIME_INTVAL,
+	LOGGER_ATTRIBUTE_LOG_MIN_DATA_SIZE,
+	LOGGER_ATTRIBUTE_FW_DUMP_LEN,
+	LOGGER_ATTRIBUTE_FW_DUMP_DATA,
+	LOGGERG_ATTRIBUTE_RING_DATA,
+	LOGGER_ATTRIBUTE_RING_STATUS,
+	LOGGER_ATTRIBUTE_RING_NUM
+};
+typedef enum rtw_vendor_event {
+    RTK_RESERVED1,
+    RTK_RESERVED2,
+    GSCAN_EVENT_SIGNIFICANT_CHANGE_RESULTS ,
+    GSCAN_EVENT_HOTLIST_RESULTS_FOUND,
+    GSCAN_EVENT_SCAN_RESULTS_AVAILABLE,
+    GSCAN_EVENT_FULL_SCAN_RESULTS,
+    RTT_EVENT_COMPLETE,
+    GSCAN_EVENT_COMPLETE_SCAN,
+    GSCAN_EVENT_HOTLIST_RESULTS_LOST,
+    GSCAN_EVENT_EPNO_EVENT,
+    GOOGLE_DEBUG_RING_EVENT,
+    GOOGLE_DEBUG_MEM_DUMP_EVENT,
+    GSCAN_EVENT_ANQPO_HOTSPOT_MATCH,
+    GOOGLE_RSSI_MONITOR_EVENT
+} rtw_vendor_event_t;
+
+enum andr_wifi_feature_set_attr {
+	ANDR_WIFI_ATTRIBUTE_NUM_FEATURE_SET,
+	ANDR_WIFI_ATTRIBUTE_FEATURE_SET,
+	ANDR_WIFI_ATTRIBUTE_RANDOM_MAC_OUI,
+	ANDR_WIFI_ATTRIBUTE_NODFS_SET,
+	ANDR_WIFI_ATTRIBUTE_COUNTRY,
+	ANDR_WIFI_ATTRIBUTE_ND_OFFLOAD_VALUE
+	// Add more attribute here
+};
+
+typedef enum rtw_vendor_gscan_attribute {
+	ATTR_START_GSCAN,
+	ATTR_STOP_GSCAN,
+	ATTR_SET_SCAN_BATCH_CFG_ID, /* set batch scan params */
+	ATTR_SET_SCAN_GEOFENCE_CFG_ID, /* set list of bssids to track */
+	ATTR_SET_SCAN_SIGNIFICANT_CFG_ID, /* set list of bssids, rssi threshold etc.. */
+	ATTR_SET_SCAN_CFG_ID, /* set common scan config params here */
+	ATTR_GET_GSCAN_CAPABILITIES_ID,
+	/* Add more sub commands here */
+	ATTR_GSCAN_MAX
+} rtw_vendor_gscan_attribute_t;
+
+typedef enum gscan_batch_attribute {
+	ATTR_GSCAN_BATCH_BESTN,
+	ATTR_GSCAN_BATCH_MSCAN,
+	ATTR_GSCAN_BATCH_BUFFER_THRESHOLD
+} gscan_batch_attribute_t;
+
+typedef enum gscan_geofence_attribute {
+	ATTR_GSCAN_NUM_HOTLIST_BSSID,
+	ATTR_GSCAN_HOTLIST_BSSID
+} gscan_geofence_attribute_t;
+
+typedef enum gscan_complete_event {
+	WIFI_SCAN_BUFFER_FULL,
+	WIFI_SCAN_COMPLETE
+} gscan_complete_event_t;
+/* wifi_hal.h */
+/* WiFi Common definitions */
+typedef unsigned char byte;
+typedef int wifi_request_id;
+typedef int wifi_channel;                       // indicates channel frequency in MHz
+typedef int wifi_rssi;
+typedef byte mac_addr[6];
+typedef byte oui[3];
+typedef int64_t wifi_timestamp;                 // In microseconds (us)
+typedef int64_t wifi_timespan;                  // In picoseconds  (ps)
+
+struct wifi_info;
+struct wifi_interface_info;
+typedef struct wifi_info *wifi_handle;
+typedef struct wifi_interface_info *wifi_interface_handle;
+
+/* channel operating width */
+typedef enum {
+    WIFI_CHAN_WIDTH_20    = 0,
+    WIFI_CHAN_WIDTH_40    = 1,
+    WIFI_CHAN_WIDTH_80    = 2,
+    WIFI_CHAN_WIDTH_160   = 3,
+    WIFI_CHAN_WIDTH_80P80 = 4,
+    WIFI_CHAN_WIDTH_5     = 5,
+    WIFI_CHAN_WIDTH_10    = 6,
+    WIFI_CHAN_WIDTH_INVALID = -1
+} wifi_channel_width;
+
+typedef int wifi_radio;
+
+typedef struct {
+    wifi_channel_width width;
+    int center_frequency0;
+    int center_frequency1;
+    int primary_frequency;
+} wifi_channel_spec;
+
+typedef enum {
+    WIFI_SUCCESS = 0,
+    WIFI_ERROR_NONE = 0,
+    WIFI_ERROR_UNKNOWN = -1,
+    WIFI_ERROR_UNINITIALIZED = -2,
+    WIFI_ERROR_NOT_SUPPORTED = -3,
+    WIFI_ERROR_NOT_AVAILABLE = -4,              // Not available right now, but try later
+    WIFI_ERROR_INVALID_ARGS = -5,
+    WIFI_ERROR_INVALID_REQUEST_ID = -6,
+    WIFI_ERROR_TIMED_OUT = -7,
+    WIFI_ERROR_TOO_MANY_REQUESTS = -8,          // Too many instances of this request
+    WIFI_ERROR_OUT_OF_MEMORY = -9,
+    WIFI_ERROR_BUSY = -10,
+} wifi_error;
+
+typedef int wifi_ring_buffer_id;
+/* ring buffer params */
+/**
+ * written_bytes and read_bytes implement a producer consumer API
+ *     hence written_bytes >= read_bytes
+ * a modulo arithmetic of the buffer size has to be applied to those counters:
+ * actual offset into ring buffer = written_bytes % ring_buffer_byte_size
+ *
+ */
+typedef struct {
+    u8 name[32];
+    u32 flags;
+    wifi_ring_buffer_id ring_id; // unique integer representing the ring
+    u32 ring_buffer_byte_size;   // total memory size allocated for the buffer
+    u32 verbose_level;           // verbose level for ring buffer
+    u32 written_bytes;           // number of bytes that was written to the buffer by driver,
+                                 // monotonously increasing integer
+    u32 read_bytes;              // number of bytes that was read from the buffer by user land,
+                                 // monotonously increasing integer
+    u32 written_records;         // number of records that was written to the buffer by driver,
+                                 // monotonously increasing integer
+} wifi_ring_buffer_status;
+
+#ifdef CONFIG_RTW_CFGVENDOR_LLSTATS
+#define STATS_MAJOR_VERSION      1
+#define STATS_MINOR_VERSION      0
+#define STATS_MICRO_VERSION      0
+
+typedef enum {
+    WIFI_DISCONNECTED = 0,
+    WIFI_AUTHENTICATING = 1,
+    WIFI_ASSOCIATING = 2,
+    WIFI_ASSOCIATED = 3,
+    WIFI_EAPOL_STARTED = 4,   // if done by firmware/driver
+    WIFI_EAPOL_COMPLETED = 5, // if done by firmware/driver
+} wifi_connection_state;
+
+typedef enum {
+    WIFI_ROAMING_IDLE = 0,
+    WIFI_ROAMING_ACTIVE = 1,
+} wifi_roam_state;
+
+typedef enum {
+    WIFI_INTERFACE_STA = 0,
+    WIFI_INTERFACE_SOFTAP = 1,
+    WIFI_INTERFACE_IBSS = 2,
+    WIFI_INTERFACE_P2P_CLIENT = 3,
+    WIFI_INTERFACE_P2P_GO = 4,
+    WIFI_INTERFACE_NAN = 5,
+    WIFI_INTERFACE_MESH = 6,
+    WIFI_INTERFACE_UNKNOWN = -1
+ } wifi_interface_mode;
+
+#define WIFI_CAPABILITY_QOS          0x00000001     // set for QOS association
+#define WIFI_CAPABILITY_PROTECTED    0x00000002     // set for protected association (802.11 beacon frame control protected bit set)
+#define WIFI_CAPABILITY_INTERWORKING 0x00000004     // set if 802.11 Extended Capabilities element interworking bit is set
+#define WIFI_CAPABILITY_HS20         0x00000008     // set for HS20 association
+#define WIFI_CAPABILITY_SSID_UTF8    0x00000010     // set is 802.11 Extended Capabilities element UTF-8 SSID bit is set
+#define WIFI_CAPABILITY_COUNTRY      0x00000020     // set is 802.11 Country Element is present
+
+typedef struct {
+   wifi_interface_mode mode;     // interface mode
+   u8 mac_addr[6];               // interface mac address (self)
+   wifi_connection_state state;  // connection state (valid for STA, CLI only)
+   wifi_roam_state roaming;      // roaming state
+   u32 capabilities;             // WIFI_CAPABILITY_XXX (self)
+   u8 ssid[33];                  // null terminated SSID
+   u8 bssid[6];                  // bssid
+   u8 ap_country_str[3];         // country string advertised by AP
+   u8 country_str[3];            // country string for this association
+} wifi_interface_link_layer_info;
+
+/* channel information */
+typedef struct {
+   wifi_channel_width width;   // channel width (20, 40, 80, 80+80, 160)
+   wifi_channel center_freq;   // primary 20 MHz channel
+   wifi_channel center_freq0;  // center frequency (MHz) first segment
+   wifi_channel center_freq1;  // center frequency (MHz) second segment
+} wifi_channel_info;
+
+/* wifi rate */
+typedef struct {
+   u32 preamble   :3;   // 0: OFDM, 1:CCK, 2:HT 3:VHT 4..7 reserved
+   u32 nss        :2;   // 0:1x1, 1:2x2, 3:3x3, 4:4x4
+   u32 bw         :3;   // 0:20MHz, 1:40Mhz, 2:80Mhz, 3:160Mhz
+   u32 rateMcsIdx :8;   // OFDM/CCK rate code would be as per ieee std in the units of 0.5mbps
+                        // HT/VHT it would be mcs index
+   u32 reserved  :16;   // reserved
+   u32 bitrate;         // units of 100 Kbps
+} wifi_rate;
+
+/* channel statistics */
+typedef struct {
+   wifi_channel_info channel;  // channel
+   u32 on_time;                // msecs the radio is awake (32 bits number accruing over time)
+   u32 cca_busy_time;          // msecs the CCA register is busy (32 bits number accruing over time)
+} wifi_channel_stat;
+
+// Max number of tx power levels. The actual number vary per device and is specified by |num_tx_levels|
+#define RADIO_STAT_MAX_TX_LEVELS 256
+
+/* Internal radio statistics structure in the driver */
+typedef struct {
+   wifi_radio radio;                      // wifi radio (if multiple radio supported)
+   u32 on_time;                           // msecs the radio is awake (32 bits number accruing over time)
+   u32 tx_time;                           // msecs the radio is transmitting (32 bits number accruing over time)
+   u32 rx_time;                           // msecs the radio is in active receive (32 bits number accruing over time)
+   u32 on_time_scan;                      // msecs the radio is awake due to all scan (32 bits number accruing over time)
+   u32 on_time_nbd;                       // msecs the radio is awake due to NAN (32 bits number accruing over time)
+   u32 on_time_gscan;                     // msecs the radio is awake due to G?scan (32 bits number accruing over time)
+   u32 on_time_roam_scan;                 // msecs the radio is awake due to roam?scan (32 bits number accruing over time)
+   u32 on_time_pno_scan;                  // msecs the radio is awake due to PNO scan (32 bits number accruing over time)
+   u32 on_time_hs20;                      // msecs the radio is awake due to HS2.0 scans and GAS exchange (32 bits number accruing over time)
+   u32 num_channels;                      // number of channels
+   wifi_channel_stat channels[];          // channel statistics
+} wifi_radio_stat_internal;
+
+/**
+ * Packet statistics reporting by firmware is performed on MPDU basi (i.e. counters increase by 1 for each MPDU)
+ * As well, "data packet" in associated comments, shall be interpreted as 802.11 data packet,
+ * that is, 802.11 frame control subtype == 2 and excluding management and control frames.
+ *
+ * As an example, in the case of transmission of an MSDU fragmented in 16 MPDUs which are transmitted
+ * OTA in a 16 units long a-mpdu, for which a block ack is received with 5 bits set:
+ *          tx_mpdu : shall increase by 5
+ *          retries : shall increase by 16
+ *          tx_ampdu : shall increase by 1
+ * data packet counters shall not increase regardless of the number of BAR potentially sent by device for this a-mpdu
+ * data packet counters shall not increase regardless of the number of BA received by device for this a-mpdu
+ *
+ * For each subsequent retransmission of the 11 remaining non ACK'ed mpdus
+ * (regardless of the fact that they are transmitted in a-mpdu or not)
+ *          retries : shall increase by 1
+ *
+ * If no subsequent BA or ACK are received from AP, until packet lifetime expires for those 11 packet that were not ACK'ed
+ *          mpdu_lost : shall increase by 11
+ */
+
+/* per rate statistics */
+typedef struct {
+   wifi_rate rate;     // rate information
+   u32 tx_mpdu;        // number of successfully transmitted data pkts (ACK rcvd)
+   u32 rx_mpdu;        // number of received data pkts
+   u32 mpdu_lost;      // number of data packet losses (no ACK)
+   u32 retries;        // total number of data pkt retries
+   u32 retries_short;  // number of short data pkt retries
+   u32 retries_long;   // number of long data pkt retries
+} wifi_rate_stat;
+
+/* access categories */
+typedef enum {
+   WIFI_AC_VO  = 0,
+   WIFI_AC_VI  = 1,
+   WIFI_AC_BE  = 2,
+   WIFI_AC_BK  = 3,
+   WIFI_AC_MAX = 4,
+} wifi_traffic_ac;
+
+/* wifi peer type */
+typedef enum
+{
+   WIFI_PEER_STA,
+   WIFI_PEER_AP,
+   WIFI_PEER_P2P_GO,
+   WIFI_PEER_P2P_CLIENT,
+   WIFI_PEER_NAN,
+   WIFI_PEER_TDLS,
+   WIFI_PEER_INVALID,
+} wifi_peer_type;
+
+/* per peer statistics */
+typedef struct {
+   wifi_peer_type type;           // peer type (AP, TDLS, GO etc.)
+   u8 peer_mac_address[6];        // mac address
+   u32 capabilities;              // peer WIFI_CAPABILITY_XXX
+   u32 num_rate;                  // number of rates
+   wifi_rate_stat rate_stats[];   // per rate statistics, number of entries  = num_rate
+} wifi_peer_info;
+
+/* Per access category statistics */
+typedef struct {
+   wifi_traffic_ac ac;             // access category (VI, VO, BE, BK)
+   u32 tx_mpdu;                    // number of successfully transmitted unicast data pkts (ACK rcvd)
+   u32 rx_mpdu;                    // number of received unicast data packets
+   u32 tx_mcast;                   // number of succesfully transmitted multicast data packets
+                                   // STA case: implies ACK received from AP for the unicast packet in which mcast pkt was sent
+   u32 rx_mcast;                   // number of received multicast data packets
+   u32 rx_ampdu;                   // number of received unicast a-mpdus; support of this counter is optional
+   u32 tx_ampdu;                   // number of transmitted unicast a-mpdus; support of this counter is optional
+   u32 mpdu_lost;                  // number of data pkt losses (no ACK)
+   u32 retries;                    // total number of data pkt retries
+   u32 retries_short;              // number of short data pkt retries
+   u32 retries_long;               // number of long data pkt retries
+   u32 contention_time_min;        // data pkt min contention time (usecs)
+   u32 contention_time_max;        // data pkt max contention time (usecs)
+   u32 contention_time_avg;        // data pkt avg contention time (usecs)
+   u32 contention_num_samples;     // num of data pkts used for contention statistics
+} wifi_wmm_ac_stat;
+
+/* interface statistics */
+typedef struct {
+   wifi_interface_handle iface;          // wifi interface
+   wifi_interface_link_layer_info info;  // current state of the interface
+   u32 beacon_rx;                        // access point beacon received count from connected AP
+   u64 average_tsf_offset;               // average beacon offset encountered (beacon_TSF - TBTT)
+                                         // The average_tsf_offset field is used so as to calculate the
+                                         // typical beacon contention time on the channel as well may be
+                                         // used to debug beacon synchronization and related power consumption issue
+   u32 leaky_ap_detected;                // indicate that this AP typically leaks packets beyond the driver guard time.
+   u32 leaky_ap_avg_num_frames_leaked;  // average number of frame leaked by AP after frame with PM bit set was ACK'ed by AP
+   u32 leaky_ap_guard_time;              // guard time currently in force (when implementing IEEE power management based on
+                                         // frame control PM bit), How long driver waits before shutting down the radio and
+                                         // after receiving an ACK for a data frame with PM bit set)
+   u32 mgmt_rx;                          // access point mgmt frames received count from connected AP (including Beacon)
+   u32 mgmt_action_rx;                   // action frames received count
+   u32 mgmt_action_tx;                   // action frames transmit count
+   wifi_rssi rssi_mgmt;                  // access Point Beacon and Management frames RSSI (averaged)
+   wifi_rssi rssi_data;                  // access Point Data Frames RSSI (averaged) from connected AP
+   wifi_rssi rssi_ack;                   // access Point ACK RSSI (averaged) from connected AP
+   wifi_wmm_ac_stat ac[WIFI_AC_MAX];     // per ac data packet statistics
+   u32 num_peers;                        // number of peers
+   wifi_peer_info peer_info[];           // per peer statistics
+} wifi_iface_stat;
+
+/* configuration params */
+typedef struct {
+   u32 mpdu_size_threshold;             // threshold to classify the pkts as short or long
+                                        // packet size < mpdu_size_threshold => short
+   u32 aggressive_statistics_gathering; // set for field debug mode. Driver should collect all statistics regardless of performance impact.
+} wifi_link_layer_params;
+
+/* wifi statistics bitmap  */
+#define WIFI_STATS_RADIO              0x00000001      // all radio statistics
+#define WIFI_STATS_RADIO_CCA          0x00000002      // cca_busy_time (within radio statistics)
+#define WIFI_STATS_RADIO_CHANNELS     0x00000004      // all channel statistics (within radio statistics)
+#define WIFI_STATS_RADIO_SCAN         0x00000008      // all scan statistics (within radio statistics)
+#define WIFI_STATS_IFACE              0x00000010      // all interface statistics
+#define WIFI_STATS_IFACE_TXRATE       0x00000020      // all tx rate statistics (within interface statistics)
+#define WIFI_STATS_IFACE_AC           0x00000040      // all ac statistics (within interface statistics)
+#define WIFI_STATS_IFACE_CONTENTION   0x00000080      // all contention (min, max, avg) statistics (within ac statisctics)
+
+#endif /* CONFIG_RTW_CFGVENDOR_LLSTATS */
+
+#define RSSI_MONITOR_EVT_VERSION   1
+typedef struct {
+    u8 version;
+    s8 cur_rssi;
+    mac_addr BSSID;
+} rssi_monitor_evt;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) || defined(RTW_VENDOR_EXT_SUPPORT)
+extern int rtw_cfgvendor_attach(struct wiphy *wiphy);
+extern int rtw_cfgvendor_detach(struct wiphy *wiphy);
+extern int rtw_cfgvendor_send_async_event(struct wiphy *wiphy,
+	struct net_device *dev, int event_id, const void  *data, int len);
+#if defined(GSCAN_SUPPORT) && 0
+extern int rtw_cfgvendor_send_hotlist_event(struct wiphy *wiphy,
+	struct net_device *dev, void  *data, int len, rtw_vendor_event_t event);
+#endif
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)) || defined(RTW_VENDOR_EXT_SUPPORT) */
+
+#ifdef CONFIG_RTW_CFGVENDOR_RSSIMONITOR
+void rtw_cfgvendor_rssi_monitor_evt(_adapter *padapter);
+#endif
+
+#ifdef CONFIG_RTW_CFGVENDOR_RANDOM_MAC_OUI
+void rtw_hal_pno_random_gen_mac_addr(_adapter *adapter);
+void rtw_hal_set_hw_mac_addr(_adapter *adapter, u8 *mac_addr);
+#endif
+
+
+#endif /* _RTW_CFGVENDOR_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_proc.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_proc.c
new file mode 100644
index 000000000000..528d160d69b2
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_proc.c
@@ -0,0 +1,6666 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <linux/ctype.h>	/* tolower() */
+#include <drv_types.h>
+#include "rtw_proc.h"
+#include <rtw_btc.h>
+#include "_hal_rate.h"
+
+#ifdef CONFIG_PROC_DEBUG
+
+static struct proc_dir_entry *rtw_proc = NULL;
+
+inline struct proc_dir_entry *get_rtw_drv_proc(void)
+{
+	return rtw_proc;
+}
+
+#define RTW_PROC_NAME DRV_NAME
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0))
+#define file_inode(file) ((file)->f_dentry->d_inode)
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0))
+#define PDE_DATA(inode) PDE((inode))->data
+#define proc_get_parent_data(inode) PDE((inode))->parent->data
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0))
+#define PDE_DATA pde_data
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24))
+#define get_proc_net proc_net
+#else
+#define get_proc_net init_net.proc_net
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0))
+int single_open_size(struct file *file, int (*show)(struct seq_file *, void *),
+		void *data, size_t size)
+{
+	char *buf = kmalloc(size, GFP_KERNEL);
+	int ret;
+	if (!buf)
+		return -ENOMEM;
+	ret = single_open(file, show, data);
+	if (ret) {
+		kfree(buf);
+		return ret;
+	}
+	((struct seq_file *)file->private_data)->buf = buf;
+	((struct seq_file *)file->private_data)->size = size;
+	return 0;
+}
+#endif
+
+inline struct proc_dir_entry *rtw_proc_create_dir(const char *name, struct proc_dir_entry *parent, void *data)
+{
+	struct proc_dir_entry *entry;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+	entry = proc_mkdir_data(name, S_IRUGO | S_IXUGO, parent, data);
+#else
+	/* entry = proc_mkdir_mode(name, S_IRUGO|S_IXUGO, parent); */
+	entry = proc_mkdir(name, parent);
+	if (entry)
+		entry->data = data;
+#endif
+
+	return entry;
+}
+
+inline struct proc_dir_entry *rtw_proc_create_entry(const char *name, struct proc_dir_entry *parent,
+	const struct rtw_proc_ops *fops, void * data)
+{
+	struct proc_dir_entry *entry;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26))
+	entry = proc_create_data(name,  S_IFREG | S_IRUGO | S_IWUGO, parent, fops, data);
+#else
+	entry = create_proc_entry(name, S_IFREG | S_IRUGO | S_IWUGO, parent);
+	if (entry) {
+		entry->data = data;
+		entry->proc_fops = fops;
+	}
+#endif
+
+	return entry;
+}
+
+static int proc_get_dummy(struct seq_file *m, void *v)
+{
+	return 0;
+}
+
+static int proc_get_drv_version(struct seq_file *m, void *v)
+{
+	dump_drv_version(m);
+	return 0;
+}
+
+static int proc_get_log_level(struct seq_file *m, void *v)
+{
+	dump_log_level(m);
+	return 0;
+}
+
+static int proc_get_drv_cfg(struct seq_file *m, void *v)
+{
+	dump_drv_cfg(m);
+	return 0;
+}
+
+static ssize_t proc_set_log_level(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	int log_level;
+
+	if (count < 1)
+		return -EINVAL;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+#ifdef CONFIG_RTW_DEBUG
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d ", &log_level);
+
+		if (num == 1 &&
+		    log_level >= _DRV_NONE_ && log_level <= _DRV_MAX_) {
+			rtw_drv_log_level = log_level;
+			printk("rtw_drv_log_level:%d\n", rtw_drv_log_level);
+		}
+	} else
+		return -EFAULT;
+#else
+	printk("CONFIG_RTW_DEBUG is disabled\n");
+#endif
+
+	return count;
+}
+
+#ifdef DBG_MEM_ALLOC
+static int proc_get_mstat(struct seq_file *m, void *v)
+{
+	rtw_mstat_dump(m);
+	return 0;
+}
+#endif /* DBG_MEM_ALLOC */
+
+static bool regd_info;
+static int proc_get_country_chplan_map(struct seq_file *m, void *v)
+{
+	dump_country_chplan_map(m, regd_info);
+	return 0;
+}
+
+static ssize_t proc_set_country_chplan_map(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	int regd_info_val;
+
+	if (count < 1)
+		return -EINVAL;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%d", &regd_info_val);
+
+		if (num >= 1)
+			regd_info = regd_info_val ? 1 : 0;
+	} else
+		return -EFAULT;
+
+	return count;
+}
+
+static int proc_get_country_list(struct seq_file *m, void *v)
+{
+	dump_country_list(m);
+	return 0;
+}
+
+static int proc_get_chplan_id_list(struct seq_file *m, void *v)
+{
+	dump_chplan_id_list(m);
+	return 0;
+}
+
+static int proc_get_chplan_country_list(struct seq_file *m, void *v)
+{
+	dump_chplan_country_list(m);
+	return 0;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static int proc_get_chplan_6g_id_list(struct seq_file *m, void *v)
+{
+	dump_chplan_6g_id_list(m);
+	return 0;
+}
+
+static int proc_get_chplan_6g_country_list(struct seq_file *m, void *v)
+{
+	dump_chplan_6g_country_list(m);
+	return 0;
+}
+#endif
+
+#ifdef CONFIG_RTW_DEBUG
+static int proc_get_chplan_test(struct seq_file *m, void *v)
+{
+	dump_chplan_test(m);
+	return 0;
+}
+#endif
+
+static int proc_get_chplan_ver(struct seq_file *m, void *v)
+{
+	dump_chplan_ver(m);
+	return 0;
+}
+
+static int proc_get_global_op_class(struct seq_file *m, void *v)
+{
+	dump_global_op_class(m);
+	return 0;
+}
+
+extern void rtw_hal_get_version(char *str, u32 len);
+
+static int proc_get_hal_info(struct seq_file *m, void *v)
+{
+	char ver[30] = {0};
+
+	rtw_hal_get_version(ver, 30);
+
+	RTW_PRINT_SEL(m, "version: %s\n", ver);
+
+	return 0;
+}
+
+
+/*
+* rtw_drv_proc:
+* init/deinit when register/unregister driver
+*/
+const struct rtw_proc_hdl drv_proc_hdls[] = {
+	RTW_PROC_HDL_SSEQ("ver_info", proc_get_drv_version, NULL),
+	RTW_PROC_HDL_SSEQ("log_level", proc_get_log_level, proc_set_log_level),
+	RTW_PROC_HDL_SSEQ("drv_cfg", proc_get_drv_cfg, NULL),
+#ifdef DBG_MEM_ALLOC
+	RTW_PROC_HDL_SSEQ("mstat", proc_get_mstat, NULL),
+#endif /* DBG_MEM_ALLOC */
+	RTW_PROC_HDL_SSEQ("country_chplan_map", proc_get_country_chplan_map, proc_set_country_chplan_map),
+	RTW_PROC_HDL_SSEQ("country_list", proc_get_country_list, NULL),
+	RTW_PROC_HDL_SSEQ("chplan_id_list", proc_get_chplan_id_list, NULL),
+	RTW_PROC_HDL_SSEQ("chplan_country_list", proc_get_chplan_country_list, NULL),
+#if CONFIG_IEEE80211_BAND_6GHZ
+	RTW_PROC_HDL_SSEQ("chplan_6g_id_list", proc_get_chplan_6g_id_list, NULL),
+	RTW_PROC_HDL_SSEQ("chplan_6g_country_list", proc_get_chplan_6g_country_list, NULL),
+#endif
+#ifdef CONFIG_RTW_DEBUG
+	RTW_PROC_HDL_SSEQ("chplan_test", proc_get_chplan_test, NULL),
+#endif
+	RTW_PROC_HDL_SSEQ("chplan_ver", proc_get_chplan_ver, NULL),
+	RTW_PROC_HDL_SSEQ("global_op_class", proc_get_global_op_class, NULL),
+	RTW_PROC_HDL_SSEQ("hal_info", proc_get_hal_info, NULL),
+#ifdef CONFIG_RTKM
+	RTW_PROC_HDL_SSEQ("rtkm", proc_get_rtkm_info, proc_set_rtkm_info),
+#endif /* CONFIG_RTKM */
+};
+
+const int drv_proc_hdls_num = sizeof(drv_proc_hdls) / sizeof(struct rtw_proc_hdl);
+
+static int rtw_drv_proc_open(struct inode *inode, struct file *file)
+{
+	/* struct net_device *dev = proc_get_parent_data(inode); */
+	ssize_t index = (ssize_t)PDE_DATA(inode);
+	const struct rtw_proc_hdl *hdl = drv_proc_hdls + index;
+	void *private = NULL;
+
+	if (hdl->type == RTW_PROC_HDL_TYPE_SEQ) {
+		int res = seq_open(file, hdl->u.seq_op);
+
+		if (res == 0)
+			((struct seq_file *)file->private_data)->private = private;
+
+		return res;
+	} else if (hdl->type == RTW_PROC_HDL_TYPE_SSEQ) {
+		int (*show)(struct seq_file *, void *) = hdl->u.show ? hdl->u.show : proc_get_dummy;
+
+		return single_open(file, show, private);
+	} else if (hdl->type == RTW_PROC_HDL_TYPE_SZSEQ) {
+		int (*show)(struct seq_file *, void *) = hdl->u.sz.show ? hdl->u.sz.show : proc_get_dummy;
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+		return single_open_size(file, show, private, hdl->u.sz.size);
+		#else
+		return single_open(file, show, private);
+		#endif
+	} else {
+		return -EROFS;
+	}
+}
+
+static ssize_t rtw_drv_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos)
+{
+	ssize_t index = (ssize_t)PDE_DATA(file_inode(file));
+	const struct rtw_proc_hdl *hdl = drv_proc_hdls + index;
+	ssize_t (*write)(struct file *, const char __user *, size_t, loff_t *, void *) = hdl->write;
+
+	if (write)
+		return write(file, buffer, count, pos, NULL);
+
+	return -EROFS;
+}
+
+static const struct rtw_proc_ops rtw_drv_proc_seq_fops = {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+	.proc_open = rtw_drv_proc_open,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+	.proc_release = seq_release,
+	.proc_write = rtw_drv_proc_write,
+#else
+	.owner = THIS_MODULE,
+	.open = rtw_drv_proc_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = seq_release,
+	.write = rtw_drv_proc_write,
+#endif
+};
+
+static const struct rtw_proc_ops rtw_drv_proc_sseq_fops = {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+	.proc_open = rtw_drv_proc_open,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+	.proc_release = single_release,
+	.proc_write = rtw_drv_proc_write,
+#else
+	.owner = THIS_MODULE,
+	.open = rtw_drv_proc_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = rtw_drv_proc_write,
+#endif
+};
+
+int rtw_drv_proc_init(void)
+{
+	int ret = _FAIL;
+	ssize_t i;
+	struct proc_dir_entry *entry = NULL;
+
+	if (rtw_proc != NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	rtw_proc = rtw_proc_create_dir(RTW_PROC_NAME, get_proc_net, NULL);
+
+	if (rtw_proc == NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	for (i = 0; i < drv_proc_hdls_num; i++) {
+		if (drv_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+			entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_seq_fops, (void *)i);
+		else if (drv_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ ||
+			drv_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SZSEQ)
+			entry = rtw_proc_create_entry(drv_proc_hdls[i].name, rtw_proc, &rtw_drv_proc_sseq_fops, (void *)i);
+		else
+			entry = NULL;
+
+		if (!entry) {
+			rtw_warn_on(1);
+			goto exit;
+		}
+	}
+
+	ret = _SUCCESS;
+
+exit:
+	return ret;
+}
+
+void rtw_drv_proc_deinit(void)
+{
+	int i;
+
+	if (rtw_proc == NULL)
+		return;
+
+	for (i = 0; i < drv_proc_hdls_num; i++)
+		remove_proc_entry(drv_proc_hdls[i].name, rtw_proc);
+
+	remove_proc_entry(RTW_PROC_NAME, get_proc_net);
+	rtw_proc = NULL;
+}
+
+#ifndef RTW_SEQ_FILE_TEST
+#define RTW_SEQ_FILE_TEST 0
+#endif
+
+#if RTW_SEQ_FILE_TEST
+#define RTW_SEQ_FILE_TEST_SHOW_LIMIT 300
+static void *proc_start_seq_file_test(struct seq_file *m, loff_t *pos)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+	if (*pos >= RTW_SEQ_FILE_TEST_SHOW_LIMIT) {
+		RTW_PRINT(FUNC_ADPT_FMT" pos:%llu, out of range return\n", FUNC_ADPT_ARG(adapter), *pos);
+		return NULL;
+	}
+
+	RTW_PRINT(FUNC_ADPT_FMT" return pos:%lld\n", FUNC_ADPT_ARG(adapter), *pos);
+	return pos;
+}
+void proc_stop_seq_file_test(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT(FUNC_ADPT_FMT"\n", FUNC_ADPT_ARG(adapter));
+}
+
+void *proc_next_seq_file_test(struct seq_file *m, void *v, loff_t *pos)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	(*pos)++;
+	if (*pos >= RTW_SEQ_FILE_TEST_SHOW_LIMIT) {
+		RTW_PRINT(FUNC_ADPT_FMT" pos:%lld, out of range return\n", FUNC_ADPT_ARG(adapter), *pos);
+		return NULL;
+	}
+
+	RTW_PRINT(FUNC_ADPT_FMT" return pos:%lld\n", FUNC_ADPT_ARG(adapter), *pos);
+	return pos;
+}
+
+static int proc_get_seq_file_test(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	u32 pos = *((loff_t *)(v));
+	RTW_PRINT(FUNC_ADPT_FMT" pos:%d\n", FUNC_ADPT_ARG(adapter), pos);
+	RTW_PRINT_SEL(m, FUNC_ADPT_FMT" pos:%d\n", FUNC_ADPT_ARG(adapter), pos);
+	return 0;
+}
+
+struct seq_operations seq_file_test = {
+	.start = proc_start_seq_file_test,
+	.stop  = proc_stop_seq_file_test,
+	.next  = proc_next_seq_file_test,
+	.show  = proc_get_seq_file_test,
+};
+#endif /* RTW_SEQ_FILE_TEST */
+
+#ifdef CONFIG_SDIO_HCI
+#include "rtw_sdio.h"
+#define CCCR_LEN	0x100
+static void sd_f0_reg_dump(void *sel, struct _ADAPTER *a)
+{
+	struct dvobj_priv *d = adapter_to_dvobj(a);
+	u8 *cccr = NULL, *ptr, val;
+	u8 ret;
+	int i;
+
+
+	cccr = rtw_vmalloc(CCCR_LEN);
+	if (cccr) {
+		ret = rtw_sdio_f0_read(d, 0, cccr, CCCR_LEN);
+		if (ret == _FAIL) {
+			RTW_PRINT_SEL(sel, "Fail to read F0 register!");
+			goto exit;
+		}
+	} else {
+		RTW_WARN(FUNC_ADPT_FMT ": alloc memory FAIL!\n",
+			 FUNC_ADPT_ARG(a));
+		ptr = &val;
+	}
+
+	for (i = 0x0; i < CCCR_LEN; i++) {
+		if (cccr) {
+			ptr = &cccr[i];
+		} else {
+			ret = rtw_sdio_f0_read(d, i, &val, 1);
+			if (ret == _FAIL) {
+				RTW_ERR(FUNC_ADPT_FMT ": Fail to read F0 "
+					"register 0x%02x!\n",
+					FUNC_ADPT_ARG(a), i);
+				RTW_PRINT_SEL(sel, "... error!\n");
+				goto exit;
+			}
+		}
+
+		if (i % 16 == 0)
+			RTW_PRINT_SEL(sel, "0x%02x ", i);
+
+		_RTW_PRINT_SEL(sel, "%02x ", *ptr);
+
+		if (i % 16 == 15)
+			_RTW_PRINT_SEL(sel, "\n");
+		else if (i % 8 == 7)
+			_RTW_PRINT_SEL(sel, "\t");
+	}
+
+exit:
+	if (cccr)
+		rtw_vmfree(cccr, CCCR_LEN);
+}
+
+static int proc_get_sd_f0_reg_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+//	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+
+//	rtw_hal_sd_f0_reg_dump(m, GET_PHL_INFO(d));
+	sd_f0_reg_dump(m, adapter);
+
+	return 0;
+}
+
+static int proc_get_sdio_local_reg_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+
+	rtw_hal_sdio_local_reg_dump(m, GET_PHL_INFO(d));
+
+	return 0;
+}
+static int proc_get_sdio_card_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_sdio_card_info(m, adapter_to_dvobj(adapter));
+
+	return 0;
+}
+
+#ifdef DBG_SDIO
+static int proc_get_sdio_dbg(struct seq_file *m, void *v)
+{
+	struct net_device *dev;
+	_adapter *a;
+	struct dvobj_priv *d;
+	struct sdio_data *sdio;
+
+
+	dev = m->private;
+	a = (_adapter *)rtw_netdev_priv(dev);
+	d = adapter_to_dvobj(a);
+	sdio = dvobj_to_sdio(d);
+
+	dump_sdio_card_info(m, d);
+
+	RTW_PRINT_SEL(m, "CMD52 error cnt: %d\n", sdio->cmd52_err_cnt);
+	RTW_PRINT_SEL(m, "CMD53 error cnt: %d\n", sdio->cmd53_err_cnt);
+
+#if (DBG_SDIO >= 3)
+	RTW_PRINT_SEL(m, "dbg: %s\n", sdio->dbg_enable?"enable":"disable");
+	RTW_PRINT_SEL(m, "err_stop: %s\n", sdio->err_stop?"enable":"disable");
+	RTW_PRINT_SEL(m, "err_test: %s\n", sdio->err_test?"enable":"disable");
+	RTW_PRINT_SEL(m, "err_test_triggered: %s\n",
+		      sdio->err_test_triggered?"yes":"no");
+#endif /* DBG_SDIO >= 3 */
+
+#if (DBG_SDIO >= 2)
+	RTW_PRINT_SEL(m, "I/O error dump mark: %d\n", sdio->reg_dump_mark);
+	if (sdio->reg_dump_mark) {
+		if (sdio->dbg_msg)
+			RTW_PRINT_SEL(m, "debug messages: %s\n", sdio->dbg_msg);
+		if (sdio->reg_mac)
+			RTW_BUF_DUMP_SEL(_DRV_ALWAYS_, m, "MAC register:",
+					 _TRUE, sdio->reg_mac, 0x800);
+		if (sdio->reg_mac_ext)
+			RTW_BUF_DUMP_SEL(_DRV_ALWAYS_, m, "MAC EXT register:",
+					 _TRUE, sdio->reg_mac_ext, 0x800);
+		if (sdio->reg_local)
+			RTW_BUF_DUMP_SEL(_DRV_ALWAYS_, m, "SDIO Local register:",
+					 _TRUE, sdio->reg_local, 0x100);
+		if (sdio->reg_cia)
+			RTW_BUF_DUMP_SEL(_DRV_ALWAYS_, m, "SDIO CIA register:",
+					 _TRUE, sdio->reg_cia, 0x200);
+	}
+#endif /* DBG_SDIO >= 2 */
+
+	return 0;
+}
+
+#if (DBG_SDIO >= 2)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+#define strnicmp	strncasecmp
+#endif /* Linux kernel >= 4.0.0 */
+void rtw_sdio_dbg_reg_free(struct dvobj_priv *d);
+#endif /* DBG_SDIO >= 2 */
+
+ssize_t proc_set_sdio_dbg(struct file *file, const char __user *buffer,
+			  size_t count, loff_t *pos, void *data)
+{
+#if (DBG_SDIO >= 2)
+	struct net_device *dev = data;
+	struct dvobj_priv *d;
+	_adapter *a;
+	struct sdio_data *sdio;
+	char tmp[32], cmd[32] = {0};
+	int num;
+
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	a = (_adapter *)rtw_netdev_priv(dev);
+	d = adapter_to_dvobj(a);
+	sdio = dvobj_to_sdio(d);
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num = sscanf(tmp, "%s", cmd);
+
+		if (num >= 1) {
+			if (strnicmp(cmd, "reg_reset", 10) == 0) {
+				sdio->reg_dump_mark = 0;
+				goto exit;
+			}
+			if (strnicmp(cmd, "reg_free", 9) == 0) {
+				rtw_sdio_dbg_reg_free(d);
+				sdio->reg_dump_mark = 0;
+				goto exit;
+			}
+#if (DBG_SDIO >= 3)
+			if (strnicmp(cmd, "dbg_enable", 11) == 0) {
+				sdio->dbg_enable = 1;
+				goto exit;
+			}
+			if (strnicmp(cmd, "dbg_disable", 12) == 0) {
+				sdio->dbg_enable = 0;
+				goto exit;
+			}
+			if (strnicmp(cmd, "err_stop", 9) == 0) {
+				sdio->err_stop = 1;
+				goto exit;
+			}
+			if (strnicmp(cmd, "err_stop_disable", 16) == 0) {
+				sdio->err_stop = 0;
+				goto exit;
+			}
+			if (strnicmp(cmd, "err_test", 9) == 0) {
+				sdio->err_test_triggered = 0;
+				sdio->err_test = 1;
+				goto exit;
+			}
+#endif /* DBG_SDIO >= 3 */
+		}
+
+		return -EINVAL;
+	}
+
+exit:
+#endif /* DBG_SDIO >= 2 */
+	return count;
+}
+#endif /* DBG_SDIO */
+#endif /* CONFIG_SDIO_HCI */
+
+#ifdef CONFIG_RTW_IO_RECORDS
+#include <rtw_io_records.h>
+
+static bool io_records_ori_enabled;
+
+static void *proc_start_io_records(struct seq_file *m, loff_t *pos)
+{
+	size_t seq = *pos;
+
+	if (seq == 0) {
+		io_records_ori_enabled = rtw_io_records_enabled();
+		if (io_records_ori_enabled) {
+			struct net_device *dev = m->private;
+			_adapter *adapter = rtw_netdev_priv(dev);
+
+			/* ori enabled, claim and disable */
+			rtw_io_records_claim_and_enable(adapter_to_dvobj(adapter), false);
+		}
+	}
+
+	if (!rtw_io_record_valid(seq))
+		return NULL;
+	return pos;
+}
+
+static void proc_stop_io_records(struct seq_file *m, void *v)
+{
+	if (!v) {
+		if (io_records_ori_enabled && !rtw_io_records_enabled()) {
+			struct net_device *dev = m->private;
+			_adapter *adapter = rtw_netdev_priv(dev);
+
+			/* ori enabled and disabled now, claim and enable */
+			rtw_io_records_claim_and_enable(adapter_to_dvobj(adapter), true);
+		}
+	}
+}
+
+static void *proc_next_io_records(struct seq_file *m, void *v, loff_t *pos)
+{
+	size_t seq = ++(*pos);
+
+	if (!rtw_io_record_valid(seq))
+		return NULL;
+
+	return pos;
+}
+
+static int proc_get_io_records(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	size_t seq = *((loff_t *)(v));
+	bool tab = false;
+
+	if (seq == 0)
+		rtw_io_records_dump_title(m, tab);
+	rtw_io_records_dump_value_by_seq(m, tab, seq);
+
+	return 0;
+}
+
+static int proc_get_io_records_tab(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	size_t seq = *((loff_t *)(v));
+	bool tab = true;
+
+	if (seq == 0)
+		rtw_io_records_dump_title(m, tab);
+	rtw_io_records_dump_value_by_seq(m, tab, seq);
+
+	return 0;
+}
+
+static struct seq_operations seq_ops_io_records = {
+	.start = proc_start_io_records,
+	.stop  = proc_stop_io_records,
+	.next  = proc_next_io_records,
+	.show  = proc_get_io_records,
+};
+
+static struct seq_operations seq_ops_io_records_tab = {
+	.start = proc_start_io_records,
+	.stop  = proc_stop_io_records,
+	.next  = proc_next_io_records,
+	.show  = proc_get_io_records_tab,
+};
+
+static ssize_t proc_set_io_records(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	int enable;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%d", &enable);
+
+		if (num >= 1) {
+			struct net_device *dev = data;
+			_adapter *adapter = rtw_netdev_priv(dev);
+			struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+			if (rtw_io_records_enabled() && !enable) {
+				/* on -> off */
+				rtw_io_records_claim_and_enable(dvobj, false);
+			} else if (!rtw_io_records_enabled() && enable) {
+				/* off -> on */
+				rtw_io_records_clear();
+				rtw_io_records_claim_and_enable(dvobj, true);
+			}
+		}
+	}
+
+	return count;
+}
+
+static int proc_get_io_records_enabled(struct seq_file *m, void *v)
+{
+	RTW_PRINT_SEL(m, "%d\n", rtw_io_records_enabled());
+
+	return 0;
+}
+
+static int proc_get_io_records_summary(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	bool ori_en = rtw_io_records_enabled();
+
+	if (ori_en)
+		rtw_io_records_claim_and_enable(dvobj, false);
+
+	rtw_io_records_summary_dump(m);
+
+	if (ori_en) {
+		rtw_io_records_clear();
+		rtw_io_records_claim_and_enable(dvobj, true);
+	}
+
+	return 0;
+}
+#endif /* CONFIG_RTW_IO_RECORDS */
+
+static int proc_get_mac_reg_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	rtw_phl_mac_reg_dump(m, GET_PHL_INFO(dvobj));
+
+	return 0;
+}
+
+static int proc_get_bb_reg_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	rtw_phl_bb_reg_dump(m, GET_PHL_INFO(dvobj));
+
+	return 0;
+}
+
+static int proc_get_bb_reg_dump_ex(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	rtw_phl_bb_reg_dump_ex(m, GET_PHL_INFO(dvobj));
+
+	return 0;
+}
+
+static int proc_get_rf_reg_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	rtw_phl_rf_reg_dump(m, GET_PHL_INFO(dvobj));
+
+	return 0;
+}
+
+#if 0
+int proc_get_led_config(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_led_config(m, adapter);
+
+	return 0;
+}
+
+ssize_t proc_set_led_config(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	char tmp[32];
+	u8 strategy;
+	u8 iface_en_mask;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hhx", &strategy, &iface_en_mask);
+
+		if (num >= 1)
+			rtw_led_set_strategy(adapter, strategy);
+		if (num >= 2)
+			rtw_led_set_iface_en_mask(adapter, iface_en_mask);
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_LED */
+
+#ifdef CONFIG_AP_MODE
+int proc_get_aid_status(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_aid_status(m, adapter);
+
+	return 0;
+}
+
+ssize_t proc_set_aid_status(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *stapriv = &adapter->stapriv;
+
+	char tmp[32];
+	u8 rr;
+	u16 started_aid;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hu", &rr, &started_aid);
+
+		if (num >= 1)
+			stapriv->rr_aid = rr ? 1 : 0;
+		if (num >= 2) {
+			started_aid = started_aid % (stapriv->max_aid + 1);
+			stapriv->started_aid = started_aid ? started_aid : 1;
+		}
+	}
+
+	return count;
+}
+
+#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+static int proc_get_ap_b2u_flags(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	if (MLME_IS_AP(adapter))
+		dump_ap_b2u_flags(m, adapter);
+
+	return 0;
+}
+
+static ssize_t proc_set_ap_b2u_flags(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u8 src, fwd;
+		int num = sscanf(tmp, "%hhx %hhx", &src, &fwd);
+
+		if (num >= 1)
+			adapter->b2u_flags_ap_src = src;
+		if (num >= 2)
+			adapter->b2u_flags_ap_fwd = fwd;
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_AP_DATA_BMC_TO_UC */
+#endif /* CONFIG_AP_MODE */
+
+static int proc_get_dump_tx_rate_bmp(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_tx_rate_bmp(m, adapter_to_dvobj(adapter));
+
+	return 0;
+}
+
+static int proc_get_dump_adapters_status(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_adapters_status(m, adapter_to_dvobj(adapter));
+
+	return 0;
+}
+
+#ifdef CONFIG_RTW_CUSTOMER_STR
+static int proc_get_customer_str(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	u8 cstr[RTW_CUSTOMER_STR_LEN];
+
+	if (rtw_hal_customer_str_read(adapter, cstr) != _SUCCESS)
+		goto exit;
+
+	RTW_PRINT_SEL(m, RTW_CUSTOMER_STR_FMT"\n", RTW_CUSTOMER_STR_ARG(cstr));
+
+exit:
+	return 0;
+}
+#endif /* CONFIG_RTW_CUSTOMER_STR */
+
+#ifdef CONFIG_SCAN_BACKOP
+static int proc_get_backop_flags_sta(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	RTW_PRINT_SEL(m, "0x%02x\n", mlmeext_scan_backop_flags_sta(mlmeext));
+
+	return 0;
+}
+
+static ssize_t proc_set_backop_flags_sta(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	char tmp[32];
+	u8 flags;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx", &flags);
+
+		if (num == 1)
+			mlmeext_assign_scan_backop_flags_sta(mlmeext, flags);
+	}
+
+	return count;
+}
+
+#ifdef CONFIG_AP_MODE
+static int proc_get_backop_flags_ap(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	RTW_PRINT_SEL(m, "0x%02x\n", mlmeext_scan_backop_flags_ap(mlmeext));
+
+	return 0;
+}
+
+static ssize_t proc_set_backop_flags_ap(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	char tmp[32];
+	u8 flags;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx", &flags);
+
+		if (num == 1)
+			mlmeext_assign_scan_backop_flags_ap(mlmeext, flags);
+	}
+
+	return count;
+}
+#endif /* CONFIG_AP_MODE */
+
+#ifdef CONFIG_RTW_MESH
+static int proc_get_backop_flags_mesh(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	RTW_PRINT_SEL(m, "0x%02x\n", mlmeext_scan_backop_flags_mesh(mlmeext));
+
+	return 0;
+}
+
+static ssize_t proc_set_backop_flags_mesh(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+
+	char tmp[32];
+	u8 flags;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhx", &flags);
+
+		if (num == 1)
+			mlmeext_assign_scan_backop_flags_mesh(mlmeext, flags);
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_MESH */
+
+#endif /* CONFIG_SCAN_BACKOP */
+
+/* gpio setting */
+#ifdef CONFIG_GPIO_API
+static ssize_t proc_set_config_gpio(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32] = {0};
+	int num = 0, gpio_pin = 0, gpio_mode = 0; /* gpio_mode:0 input  1:output; */
+
+	if (count < 2)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num	= sscanf(tmp, "%d %d", &gpio_pin, &gpio_mode);
+		RTW_INFO("num=%d gpio_pin=%d mode=%d\n", num, gpio_pin, gpio_mode);
+		padapter->pre_gpio_pin = gpio_pin;
+
+		if (gpio_mode == 0 || gpio_mode == 1)
+			rtw_hal_config_gpio(padapter, gpio_pin, gpio_mode);
+	}
+	return count;
+
+}
+static ssize_t proc_set_gpio_output_value(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32] = {0};
+	int num = 0, gpio_pin = 0, pin_mode = 0; /* pin_mode: 1 high         0:low */
+
+	if (count < 2)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num	= sscanf(tmp, "%d %d", &gpio_pin, &pin_mode);
+		RTW_INFO("num=%d gpio_pin=%d pin_high=%d\n", num, gpio_pin, pin_mode);
+		padapter->pre_gpio_pin = gpio_pin;
+
+		if (pin_mode == 0 || pin_mode == 1)
+			rtw_hal_set_gpio_output_value(padapter, gpio_pin, pin_mode);
+	}
+	return count;
+}
+static int proc_get_gpio(struct seq_file *m, void *v)
+{
+	u8 gpioreturnvalue = 0;
+	struct net_device *dev = m->private;
+
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	if (!padapter)
+		return -EFAULT;
+	gpioreturnvalue = rtw_hal_get_gpio(padapter, padapter->pre_gpio_pin);
+	RTW_PRINT_SEL(m, "get_gpio %d:%d\n", padapter->pre_gpio_pin, gpioreturnvalue);
+
+	return 0;
+
+}
+static ssize_t proc_set_gpio(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32] = {0};
+	int num = 0, gpio_pin = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num	= sscanf(tmp, "%d", &gpio_pin);
+		RTW_INFO("num=%d gpio_pin=%d\n", num, gpio_pin);
+		padapter->pre_gpio_pin = gpio_pin;
+
+	}
+	return count;
+}
+#endif
+
+static ssize_t proc_set_rx_info_msg(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct recv_info *recvinfo;
+	char tmp[32] = {0};
+	int phy_info_flag = 0;
+
+	if (!padapter)
+		return -EFAULT;
+
+	recvinfo = &padapter->recvinfo;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%d", &phy_info_flag);
+
+		if (num == 1)
+			recvinfo->store_law_data_flag = (boolean) phy_info_flag;
+	}
+	return count;
+}
+static int proc_get_rx_info_msg(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+#ifdef PRIVATE_R
+	_list *plist, *phead;
+	struct sta_info *psta = NULL;
+	struct stainfo_stats *pstats = NULL;
+	struct sta_priv	*pstapriv = &(padapter->stapriv);
+	u8 sta_mac[NUM_STA][ETH_ALEN] = {{0}};
+	uint mac_id[NUM_STA];
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	int i, macid_rec_idx = 0;
+	char mac_addr_str[MAC_FMT_LEN];
+#endif
+#ifdef DBG_RX_DFRAME_RAW_DATA
+	rtw_dump_rx_dframe_info(padapter, m);
+#endif
+#ifdef PRIVATE_R
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+			if (psta && psta->phl_sta
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, 6) !=  _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, 6) != _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, psta->padapter_link->mac_addr, 6) != _TRUE)) {
+				_rtw_memcpy(&sta_mac[macid_rec_idx][0], psta->phl_sta->mac_addr, ETH_ALEN);
+				mac_id[macid_rec_idx] = psta->phl_sta->macid;
+				macid_rec_idx++;
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < macid_rec_idx; i++) {
+		psta = rtw_get_stainfo(pstapriv, &sta_mac[i][0]);
+
+		if (psta) {
+			rtw_get_sta_tx_stat(padapter, psta);
+			pstats = &psta->sta_stats;
+
+			RTW_PRINT_SEL(m, "MAC: %s sent: %u fail: %u retry: %u\n",
+				get_macaddr_str(mac_addr_str, m, &sta_mac[i][0]),
+				pstats->tx_ok_cnt, pstats->tx_fail_cnt, pstats->tx_retry_cnt);
+			RTW_PRINT_SEL(m, "total_tx_retry: %llu, rx_retry: %u\n", pstats->total_tx_retry_cnt,
+						ATOMIC_READ((ATOMIC_T *)&pstats->rx_retry_cnt));
+			ATOMIC_SET((ATOMIC_T *)&pstats->rx_retry_cnt, 0);
+		} else {
+			RTW_PRINT_SEL(m, "STA is gone\n");
+		}
+	}
+#endif
+	return 0;
+}
+static int proc_get_tx_info_msg(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_info *psta;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	int i;
+	_list	*plist, *phead;
+	u16 current_rate_id = 0, current_sgi = 0;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	char *BW, *status;
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+
+	if (MLME_IS_STA(padapter))
+		status = "station mode";
+	else if (MLME_IS_AP(padapter))
+		status = "AP mode";
+	else if (MLME_IS_MESH(padapter))
+		status = "mesh mode";
+	else
+		status = " ";
+	_RTW_PRINT_SEL(m, "status=%s\n", status);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+
+			plist = get_next(plist);
+
+			if ((_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, ETH_ALEN)  !=  _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, ETH_ALEN) != _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, psta->padapter_link->mac_addr, ETH_ALEN) != _TRUE)) {
+
+				switch (psta->phl_sta->chandef.bw) {
+
+				case CHANNEL_WIDTH_20:
+					BW = "20M";
+					break;
+
+				case CHANNEL_WIDTH_40:
+					BW = "40M";
+					break;
+
+				case CHANNEL_WIDTH_80:
+					BW = "80M";
+					break;
+
+				case CHANNEL_WIDTH_160:
+					BW = "160M";
+					break;
+
+				default:
+					BW = "";
+					break;
+				}
+				current_rate_id = rtw_get_current_tx_rate(adapter, psta);
+				current_sgi = rtw_get_current_tx_sgi(adapter, psta);
+
+				RTW_PRINT_SEL(m, "==============================\n");
+				_RTW_PRINT_SEL(m, "macaddr=%s\n", get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr));
+				_RTW_PRINT_SEL(m, "Tx_Data_Rate=0x%x\n", current_rate_id);
+				_RTW_PRINT_SEL(m, "BW=%s,sgi=%u\n", BW, current_sgi);
+
+			}
+		}
+	}
+
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	return 0;
+
+}
+
+static int proc_get_false_alarm_accumulated(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(padapter);
+	struct rtw_wifi_role_link_t *wrlink = alink->wrlink;
+	enum phl_band_idx hw_band = wrlink->hw_band;
+
+	_RTW_PRINT_SEL(m , "Accumulated False Alarm:%u\n",
+		       ATOMIC_READ((ATOMIC_T *)&dvobj->fa_cnt_acc[hw_band]));
+
+	return 0;
+}
+
+static ssize_t proc_set_false_alarm_accumulated(struct file *file,
+						const char __user *buffer,
+						size_t count, loff_t *pos,
+						void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *alink = GET_PRIMARY_LINK(padapter);
+	struct rtw_wifi_role_link_t *wrlink = alink->wrlink;
+	enum phl_band_idx hw_band = wrlink->hw_band;
+	char tmp[32];
+	u32 false_clr;
+
+	if (count < 1)
+		return -EINVAL;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u32 num = sscanf(tmp, "%u ", &false_clr);
+		ATOMIC_SET((ATOMIC_T *)&dvobj->fa_cnt_acc[hw_band],
+			   (int)false_clr);
+	} else {
+		return -EFAULT;
+	}
+
+	return count;
+}
+
+#ifdef PRIVATE_R
+static int proc_get_roku_trx_info_msg(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(adapter);
+	struct dvobj_priv *devob = adapter_to_dvobj(adapter);
+	_list	*plist, *phead;
+	struct sta_info *psta = NULL;
+	u8 sta_mac[NUM_STA][ETH_ALEN] = {{0}};
+	uint mac_id[NUM_STA];
+	struct stainfo_stats *pstats = NULL;
+	struct sta_priv *pstapriv = &(adapter->stapriv);
+	struct sta_priv *pstapriv_primary = &(GET_PRIMARY_ADAPTER(adapter))->stapriv;
+	u32 i, j, macid_rec_idx = 0;
+	u8 bc_addr[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct xmit_priv *pxmitpriv = &(adapter->xmitpriv);
+	struct recv_info *recvinfo = &(adapter->recvinfo);
+	struct sta_recv_dframe_info *psta_dframe_info;
+	char cnt_str[168] = {0};
+	char tmp_str[21] = {0};
+	u8 rx_nss_num = get_phy_rx_nss(adapter, padapter_link);
+	char *BW;
+	u8 isCCKrate = 0, rf_path = 0;
+	u16 current_rate_id = 0;
+	u32 tx_retry_cnt[PHL_AC_QUEUE_TOTAL] = {0};
+	u32 tx_fail_cnt[PHL_AC_QUEUE_TOTAL] = {0};
+	u32 tx_ok_cnt[PHL_AC_QUEUE_TOTAL] = {0};
+	char mac_addr_str[MAC_FMT_LEN];
+
+	psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
+	if (psta) {
+		RTW_PRINT_SEL(m, "Interface Name: %s\n", ADPT_ARG(adapter));
+
+		RTW_PRINT_SEL(m, "===== TX and RX Count Statistics =====\n");
+		RTW_PRINT_SEL(m , "TX packets:%llu\n", pxmitpriv->tx_pkts);
+		RTW_PRINT_SEL(m , "RX packets:%llu\n", recvinfo->rx_pkts);
+
+		RTW_PRINT_SEL(m , "RX packets in the last query period:%llu\n", recvinfo->last_rx_uc_data);
+		_rtw_spinlock_bh(&pstapriv_primary->last_rx_uc_data_lock);
+		if (recvinfo->last_rx_uc_data)
+			recvinfo->last_rx_uc_data_rate_avg = rtw_division64(recvinfo->last_rx_uc_data_rate_avg, recvinfo->last_rx_uc_data);
+		RTW_PRINT_SEL(m , "Average RX data rate in the last query period:%llu\n", recvinfo->last_rx_uc_data_rate_avg);
+		/* These variables should be reset after reading */
+		recvinfo->last_rx_uc_data = 0;
+		recvinfo->last_rx_uc_data_rate_avg = 0;
+		_rtw_spinunlock_bh(&pstapriv_primary->last_rx_uc_data_lock);
+
+		RTW_PRINT_SEL(m, "===== TX Drop Statistics =====\n");
+		RTW_PRINT_SEL(m, "[Driver] BK packets drop count: %llu\n", pxmitpriv->tx_bk_drop_cnt);
+		RTW_PRINT_SEL(m, "[Driver] BE packets drop count: %llu\n", pxmitpriv->tx_be_drop_cnt);
+		RTW_PRINT_SEL(m, "[Driver] VI packets drop count: %llu\n", pxmitpriv->tx_vi_drop_cnt);
+		RTW_PRINT_SEL(m, "[Driver] VO packets drop count: %llu\n", pxmitpriv->tx_vo_drop_cnt);
+
+		RTW_PRINT_SEL(m, "===== RX VO packets Statistics =====\n");
+
+		for(i = DESC_RATE1M ; i <= DESC_RATE11M ; i++) {
+			snprintf(tmp_str, sizeof(tmp_str), "%u ", recvinfo->rx_vo_pkt_count_per_data_rate[i]);
+			strncat(cnt_str, tmp_str, strlen(tmp_str));
+		}
+		RTW_PRINT_SEL(m, "CCK (1M, 2M, 5.5M, 11M): [ %s]\n", cnt_str);
+		_rtw_memset(&cnt_str, '\0', sizeof(cnt_str));
+
+		for(i = DESC_RATE6M ; i <= DESC_RATE54M ; i++) {
+			snprintf(tmp_str, sizeof(tmp_str), "%u ", recvinfo->rx_vo_pkt_count_per_data_rate[i]);
+			strncat(cnt_str, tmp_str, strlen(tmp_str));
+		}
+		RTW_PRINT_SEL(m, "OFDM (6M, 9M, 12M, 18M, 24M, 36M, 48M, 54M): [ %s]\n", cnt_str);
+		_rtw_memset(&cnt_str, '\0', sizeof(cnt_str));
+
+		for(i = 0 ; i < rx_nss_num ; i++) {
+			for(j = DESC_RATEMCS0 ; j <= DESC_RATEMCS7 ; j++) {
+				snprintf(tmp_str, sizeof(tmp_str), "%u ", recvinfo->rx_vo_pkt_count_per_data_rate[j + (i<<3)]);
+				strncat(cnt_str, tmp_str, strlen(tmp_str));
+			}
+			RTW_PRINT_SEL(m, "HT_MCS (%d-%d): [ %s]\n", (i<<3), (i<<3) + 7, cnt_str);
+			_rtw_memset(&cnt_str, '\0', sizeof(cnt_str));
+		}
+
+		if (rx_nss_num == 1)
+			RTW_PRINT_SEL(m, "HT_MCS (8-15): [ 0 0 0 0 0 0 0 0 ]\n");
+
+		for(i = 0 ; i < rx_nss_num ; i++) {
+			for(j = DESC_RATEVHTSS1MCS0 ; j <= DESC_RATEVHTSS1MCS9 ; j++) {
+				snprintf(tmp_str, sizeof(tmp_str), "%u ", recvinfo->rx_vo_pkt_count_per_data_rate[j + (i * 10)]);
+				strncat(cnt_str, tmp_str, strlen(tmp_str));
+			}
+			RTW_PRINT_SEL(m, "VHT_%dSS_MCS (0-9): [ %s]\n", (i + 1), cnt_str);
+			_rtw_memset(&cnt_str, '\0', sizeof(cnt_str));
+		}
+
+		if (rx_nss_num == 1)
+			RTW_PRINT_SEL(m, "VHT_2SS_MCS (0-9): [ 0 0 0 0 0 0 0 0 0 0 ]\n");
+
+		for (i = 0; i < rx_nss_num; i++) {
+			for (j = DESC_RATEHESS1MCS0; j <= DESC_RATEHESS1MCS11; j++) {
+				snprintf(tmp_str, sizeof(tmp_str), "%u ", recvinfo->rx_vo_pkt_count_per_data_rate[j + (i * 12)]);
+				strncat(cnt_str, tmp_str, strlen(tmp_str));
+			}
+			RTW_PRINT_SEL(m, "HE_%dSS_MCS (0-11): [ %s]\n", (i + 1), cnt_str);
+			_rtw_memset(&cnt_str, '\0', sizeof(cnt_str));
+		}
+
+		if (rx_nss_num == 1)
+			RTW_PRINT_SEL(m, "HE_2SS_MCS (0-11): [ 0 0 0 0 0 0 0 0 0 0 0 0 ]\n");
+
+		RTW_PRINT_SEL(m, "RX counter for VO packets with retry bit set: %llu\n\n", recvinfo->rx_vo_pkt_retry_count);
+	}
+
+	_rtw_spinlock_bh(&pstapriv->sta_hash_lock);
+	for (i = 0; i < NUM_STA; i++) {
+		phead = &(pstapriv->sta_hash[i]);
+		plist = get_next(phead);
+		while ((rtw_end_of_queue_search(phead, plist)) == _FALSE) {
+			psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
+			plist = get_next(plist);
+			if ((_rtw_memcmp(psta->phl_sta->mac_addr, bc_addr, 6) !=  _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, null_addr, 6) != _TRUE)
+				&& (_rtw_memcmp(psta->phl_sta->mac_addr, psta->padapter_link->mac_addr, 6) != _TRUE)) {
+				_rtw_memcpy(&sta_mac[macid_rec_idx][0], psta->phl_sta->mac_addr, ETH_ALEN);
+				mac_id[macid_rec_idx] = psta->phl_sta->macid;
+				macid_rec_idx++;
+			}
+		}
+	}
+	_rtw_spinunlock_bh(&pstapriv->sta_hash_lock);
+
+	for (i = 0; i < macid_rec_idx; i++) {
+		psta = rtw_get_stainfo(pstapriv, &sta_mac[i][0]);
+
+		if (psta) {
+			RTW_PRINT_SEL(m, "========= STA(%s) =========\n", get_macaddr_str(mac_addr_str, m, psta->phl_sta->mac_addr));
+			RTW_PRINT_SEL(m, "oper_ch=%d, oper_bw=%d, oper_ch_offet=%d\n",
+						rtw_get_oper_ch(adapter, padapter_link),
+						rtw_get_oper_bw(adapter, padapter_link),
+						rtw_get_oper_choffset(adapter, padapter_link));
+#ifdef DBG_RX_DFRAME_RAW_DATA
+			psta_dframe_info = &psta->sta_dframe_info_bmc;
+			isCCKrate = (psta_dframe_info->sta_data_rate <= DESC_RATE11M) ? _TRUE : _FALSE;
+
+			switch (psta_dframe_info->sta_bw_mode) {
+				case CHANNEL_WIDTH_20:
+					BW = "20M";
+					break;
+				case CHANNEL_WIDTH_40:
+					BW = "40M";
+					break;
+				case CHANNEL_WIDTH_80:
+					BW = "80M";
+					break;
+				case CHANNEL_WIDTH_160:
+					BW = "160M";
+					break;
+				default:
+					BW = "";
+					break;
+			}
+
+			RTW_PRINT_SEL(m, "BW=%s, sgi =%d\n", BW, psta_dframe_info->sta_sgi);
+			RTW_PRINT_SEL(m, "Rx_Data_Rate = %s\n", HDATA_RATE(psta_dframe_info->sta_data_rate));
+			RTW_PRINT_SEL(m, "Rx_Data_Rate_Int = %d\n", (psta_dframe_info->sta_data_rate));
+
+			for (rf_path = 0; rf_path < GET_HAL_RFPATH_NUM(devob); rf_path++) {
+				if (!isCCKrate) {
+					RTW_PRINT_SEL(m , "RF_PATH_%d RSSI:%d(dBm)", rf_path, psta_dframe_info->sta_RxPwr[rf_path]);
+					RTW_PRINT_SEL(m , "\nrx_ofdm_snr:%d(dB)\n", psta_dframe_info->sta_ofdm_snr[rf_path]);
+				} else
+					RTW_PRINT_SEL(m , "RF_PATH_%d RSSI:%d(dBm)\n", rf_path,
+						rtw_phl_rssi_to_dbm(psta_dframe_info->sta_mimo_signal_strength[rf_path]));
+			}
+
+			if (GET_HAL_RFPATH_NUM(devob) == 1) {
+				if (!isCCKrate) {
+					RTW_PRINT_SEL(m , "RF_PATH_1 RSSI:%d(dBm)", 0);
+					RTW_PRINT_SEL(m , "\nrx_ofdm_snr:%d(dB)\n", 0);
+				} else
+					RTW_PRINT_SEL(m , "RF_PATH_1 RSSI:%d(dBm)\n", 0);
+			}
+#endif
+			RTW_PRINT_SEL(m , "Total False Alarm:%d / Cnt_CCA_all:%d\n",
+			rtw_phl_get_phy_stat_info(GET_PHL_INFO(devob), HW_BAND_0, STAT_INFO_FA_ALL),
+			rtw_phl_get_phy_stat_info(GET_PHL_INFO(devob), HW_BAND_0, STAT_INFO_CCA_ALL));
+
+			rtw_get_sta_tx_stat(adapter, psta);
+			pstats = &psta->sta_stats;
+
+			RTW_PRINT_SEL(m, "MAC: %s sent: %u fail: %u retry: %u\n",
+				get_macaddr_str(mac_addr_str, m, &sta_mac[i][0]),
+				pstats->tx_ok_cnt, pstats->tx_fail_cnt, pstats->tx_retry_cnt);
+			RTW_PRINT_SEL(m, "total_tx_retry: %llu, rx_retry: %u\n", pstats->total_tx_retry_cnt,
+						ATOMIC_READ((ATOMIC_T *)&pstats->rx_retry_cnt));
+						ATOMIC_SET((ATOMIC_T *)&pstats->rx_retry_cnt, 0);
+
+			RTW_PRINT_SEL(m, "[HW] BK packets drop count: %u\n", tx_fail_cnt[PHL_BE_QUEUE_SEL]);
+			RTW_PRINT_SEL(m, "[HW] BE packets drop count: %u\n", tx_fail_cnt[PHL_BK_QUEUE_SEL]);
+			RTW_PRINT_SEL(m, "[HW] VI packets drop count: %u\n", tx_fail_cnt[PHL_VI_QUEUE_SEL]);
+			RTW_PRINT_SEL(m, "[HW] VO packets drop count: %u\n", tx_fail_cnt[PHL_VO_QUEUE_SEL]);
+		} else {
+			RTW_PRINT_SEL(m, "STA is gone\n");
+		}
+	}
+
+	return 0;
+}
+#endif /* PRIVATE_R */
+
+
+static int proc_get_linked_info_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	if (padapter)
+		RTW_PRINT_SEL(m, "rtw_hal_linked_info_dump :%s\n", (padapter->bLinkInfoDump) ? "enable" : "disable");
+
+	return 0;
+}
+
+
+static ssize_t proc_set_linked_info_dump(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	char tmp[32] = {0};
+	int mode = 0, pre_mode = 0;
+	int num = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	pre_mode = padapter->bLinkInfoDump;
+	RTW_INFO("pre_mode=%d\n", pre_mode);
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num	= sscanf(tmp, "%d ", &mode);
+		RTW_INFO("num=%d mode=%d\n", num, mode);
+
+		if (num != 1) {
+			RTW_INFO("argument number is wrong\n");
+			return -EFAULT;
+		}
+
+		if (mode == 1 || (mode == 0 && pre_mode == 1)) /* not consider pwr_saving 0: */
+			padapter->bLinkInfoDump = mode;
+
+		else if ((mode == 2) || (mode == 0 && pre_mode == 2)) { /* consider power_saving */
+			/* RTW_INFO("rtw_hal_linked_info_dump =%s\n", (padapter->bLinkInfoDump)?"enable":"disable") */
+			rtw_hal_linked_info_dump(padapter, mode);
+		}
+	}
+	return count;
+}
+
+
+static int proc_get_sta_tp_dump(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (padapter)
+		RTW_PRINT_SEL(m, "sta_tp_dump :%s\n", (padapter->bsta_tp_dump) ? "enable" : "disable");
+
+	return 0;
+}
+
+static ssize_t proc_set_sta_tp_dump(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	char tmp[32] = {0};
+	int mode = 0;
+	int num = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num	= sscanf(tmp, "%d ", &mode);
+
+		if (num != 1) {
+			RTW_INFO("argument number is wrong\n");
+			return -EFAULT;
+		}
+		if (padapter)
+			padapter->bsta_tp_dump = mode;
+	}
+	return count;
+}
+
+static int proc_get_sta_tp_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (padapter)
+		rtw_sta_traffic_info(m, padapter);
+
+	return 0;
+}
+
+static int proc_get_rt_cap_ctrl(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+
+	RTW_PRINT_SEL(m, "[%d] MBO = %s\n", DIS_MBO,
+		      (padapter->dis_rt_cap & BIT(DIS_MBO))?"DIS":"EN");
+
+	RTW_PRINT_SEL(m, "Disabled runtime CAP = 0x%08x\n", padapter->dis_rt_cap);
+
+	return 0;
+}
+
+static ssize_t proc_set_rt_cap_ctrl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32] = {0};
+	int num = 0;
+	u32 param = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp))
+		return -EFAULT;
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		num = sscanf(tmp, "%x", &param);
+		padapter->dis_rt_cap = param;
+	}
+
+	return count;
+}
+
+static int proc_get_turboedca_ctrl(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (padapter->dis_turboedca == EN_TURBO)
+		RTW_PRINT_SEL(m, "Turbo-EDCA : %s\n", "Enable");
+	else if (padapter->dis_turboedca == DIS_TURBO)
+		RTW_PRINT_SEL(m, "Turbo-EDCA : %s\n", "Disable");
+	else
+		RTW_PRINT_SEL(m, "Turbo-EDCA mode=%d, edca_param_mode=0x%x\n",
+			      padapter->dis_turboedca, padapter->edca_param_mode);
+
+	_RTW_PRINT_SEL(m, "PARAM_BE:0x%x\n", padapter->edca_param_mode);
+
+	return 0;
+}
+
+static ssize_t proc_set_turboedca_ctrl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32] = {0};
+	int mode = 0, num = 0;
+	u32 param_mode = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp))
+		return -EFAULT;
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num = sscanf(tmp, "%d %x", &mode, &param_mode);
+
+		if (num < 1 || num > 2) {
+			RTW_INFO("argument number is wrong\n");
+			return -EFAULT;
+		}
+
+		/*  0: enable turboedca,
+			1: disable turboedca,
+			2: disable turboedca and setting EDCA parameter based on the input parameter
+			> 2 : currently reset to 0 */
+
+		if (mode > 2)
+			mode = 0;
+
+		padapter->dis_turboedca = mode;
+
+		padapter->edca_param_mode = 0; /* init. value */
+
+		RTW_INFO("dis_turboedca mode = 0x%x\n", padapter->dis_turboedca);
+
+		if (num == 2) {
+
+			padapter->edca_param_mode = param_mode;
+
+			RTW_INFO("param_mode = 0x%x\n", param_mode);
+		}
+
+	}
+	return count;
+}
+
+static int proc_get_mac_qinfo(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_hal_get_hwreg(adapter, HW_VAR_DUMP_MAC_QUEUE_INFO, (u8 *)m);
+
+	return 0;
+}
+
+int proc_get_wifi_spec(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	RTW_PRINT_SEL(m, "wifi_spec=%d\n", pregpriv->wifi_spec);
+	return 0;
+}
+
+static int proc_get_chan_plan(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_cur_chplan(m, adapter_to_rfctl(adapter));
+
+	return 0;
+}
+
+static ssize_t proc_set_chan_plan(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u16 chplan = RTW_CHPLAN_UNSPECIFIED;
+	u16 chplan_6g = RTW_CHPLAN_6G_UNSPECIFIED;
+	u8 env = RTW_ENV_NUM;
+
+	if (!padapter)
+		return -EFAULT;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hx %hx %hhu", &chplan, &chplan_6g, &env);
+		if (num < 1)
+			return count;
+
+		rtw_chplan_ioctl_input_mapping(&chplan, &chplan_6g);
+	}
+
+	rtw_set_channel_plan(padapter, chplan, chplan_6g, env, RTW_REGD_SET_BY_USER);
+
+	return count;
+}
+
+static int proc_get_country_code(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	dump_cur_country(m, rfctl);
+
+	return 0;
+}
+
+static ssize_t proc_set_country_code(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	char alpha2[2];
+	u8 env = RTW_ENV_NUM;
+	int num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	/* TODO: accept environment */
+	num = sscanf(tmp, "%c%c", &alpha2[0], &alpha2[1]);
+	if (num < 2)
+		return count;
+
+	rtw_set_country(padapter, alpha2, env, RTW_REGD_SET_BY_USER);
+
+exit:
+	return count;
+}
+
+#if CONFIG_IEEE80211_BAND_6GHZ
+static int proc_get_env(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_cur_env(m, adapter_to_rfctl(adapter));
+
+	return 0;
+}
+
+static ssize_t proc_set_env(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 env = RTW_ENV_NUM;
+
+	if (!padapter)
+		return -EFAULT;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhu", &env);
+
+		if (num < 1)
+			return count;
+	}
+
+	rtw_set_env(padapter, env, RTW_REGD_SET_BY_USER);
+
+	return count;
+}
+#endif /* CONFIG_IEEE80211_BAND_6GHZ */
+
+static int proc_get_chplan_confs(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	dump_cur_chplan_confs(m, rfctl);
+
+	return 0;
+}
+
+static ssize_t proc_set_chplan_confs(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	char *tmp;
+	struct chplan_confs confs;
+
+	if (!buffer || count < 1)
+		return -EFAULT;
+
+	tmp = rtw_vmalloc(count + 1);
+	if (!tmp)
+		return -ENOMEM;
+
+	if (copy_from_user(tmp, buffer, count))
+		goto exit;
+	tmp[count] = '\0';
+
+	if (rtw_parse_chplan_confs_cmd_str(&confs, tmp))
+		rtw_set_chplan_confs_cmd(adapter, RTW_CMDF_WAIT_ACK, &confs);
+
+exit:
+	rtw_vmfree(tmp, count + 1);
+
+	return count;
+}
+
+static int cap_spt_op_class_ch_detail = 0;
+
+static int proc_get_cap_spt_op_class_ch(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_cap_spt_op_class_ch(m , adapter_to_rfctl(adapter), cap_spt_op_class_ch_detail);
+	return 0;
+}
+
+static ssize_t proc_set_cap_spt_op_class_ch(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	sscanf(tmp, "%d", &cap_spt_op_class_ch_detail);
+
+exit:
+	return count;
+}
+
+static int reg_spt_op_class_ch_detail = 0;
+
+static int proc_get_reg_spt_op_class_ch(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_reg_spt_op_class_ch(m , adapter_to_rfctl(adapter), reg_spt_op_class_ch_detail);
+	return 0;
+}
+
+static ssize_t proc_set_reg_spt_op_class_ch(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	sscanf(tmp, "%d", &reg_spt_op_class_ch_detail);
+
+exit:
+	return count;
+}
+
+static int cur_spt_op_class_ch_detail = 0;
+
+static int proc_get_cur_spt_op_class_ch(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_cur_spt_op_class_ch(m , adapter_to_rfctl(adapter), cur_spt_op_class_ch_detail);
+	return 0;
+}
+
+static ssize_t proc_set_cur_spt_op_class_ch(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	sscanf(tmp, "%d", &cur_spt_op_class_ch_detail);
+
+exit:
+	return count;
+}
+
+#if CONFIG_RTW_MACADDR_ACL
+static int proc_get_macaddr_acl(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_macaddr_acl(m, adapter);
+	return 0;
+}
+
+ssize_t proc_set_macaddr_acl(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[17 * NUM_ACL + 32] = {0};
+	u8 period;
+	char cmd[32];
+	u8 mode;
+	u8 addr[ETH_ALEN];
+
+#define MAC_ACL_CMD_MODE	0
+#define MAC_ACL_CMD_ADD		1
+#define MAC_ACL_CMD_DEL		2
+#define MAC_ACL_CMD_CLR		3
+#define MAC_ACL_CMD_NUM		4
+
+	static const char * const mac_acl_cmd_str[] = {
+		"mode",
+		"add",
+		"del",
+		"clr",
+	};
+	u8 cmd_id = MAC_ACL_CMD_NUM;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		/*
+		* <period> mode <mode> <macaddr> [<macaddr>]
+		* <period> mode <mode>
+		* <period> add <macaddr> [<macaddr>]
+		* <period> del <macaddr> [<macaddr>]
+		* <period> clr
+		*/
+		char *c, *next;
+		int i;
+		u8 is_bcast;
+
+		next = tmp;
+		c = strsep(&next, " \t");
+		if (!c || sscanf(c, "%hhu", &period) != 1)
+			goto exit;
+
+		if (period >= RTW_ACL_PERIOD_NUM) {
+			RTW_WARN(FUNC_ADPT_FMT" invalid period:%u", FUNC_ADPT_ARG(adapter), period);
+			goto exit;
+		}
+
+		c = strsep(&next, " \t");
+		if (!c || sscanf(c, "%s", cmd) != 1)
+			goto exit;
+
+		for (i = 0; i < MAC_ACL_CMD_NUM; i++)
+			if (strcmp(mac_acl_cmd_str[i], cmd) == 0)
+				cmd_id = i;
+
+		switch (cmd_id) {
+		case MAC_ACL_CMD_MODE:
+			c = strsep(&next, " \t");
+			if (!c || sscanf(c, "%hhu", &mode) != 1)
+				goto exit;
+
+			if (mode >= RTW_ACL_MODE_MAX) {
+				RTW_WARN(FUNC_ADPT_FMT" invalid mode:%u", FUNC_ADPT_ARG(adapter), mode);
+				goto exit;
+			}
+			break;
+
+		case MAC_ACL_CMD_ADD:
+		case MAC_ACL_CMD_DEL:
+			break;
+
+		case MAC_ACL_CMD_CLR:
+			/* clear settings */
+			rtw_macaddr_acl_clear(adapter, period);
+			goto exit;
+
+		default:
+			RTW_WARN(FUNC_ADPT_FMT" invalid cmd:\"%s\"", FUNC_ADPT_ARG(adapter), cmd);
+			goto exit;
+		}
+
+		/* check for macaddr list */
+		c = strsep(&next, " \t");
+		if (!c && cmd_id == MAC_ACL_CMD_MODE) {
+			/* set mode only  */
+			rtw_set_macaddr_acl(adapter, period, mode);
+			goto exit;
+		}
+
+		if (cmd_id == MAC_ACL_CMD_MODE) {
+			/* set mode and entire macaddr list */
+			rtw_macaddr_acl_clear(adapter, period);
+			rtw_set_macaddr_acl(adapter, period, mode);
+		}
+
+		while (c != NULL) {
+			if (sscanf(c, MAC_SFMT, MAC_SARG(addr)) != 6)
+				break;
+
+			is_bcast = is_broadcast_mac_addr(addr);
+			if (is_bcast
+				|| rtw_check_invalid_mac_address(addr, 0) == _FALSE
+			) {
+				if (cmd_id == MAC_ACL_CMD_DEL) {
+					rtw_acl_remove_sta(adapter, period, addr);
+					if (is_bcast)
+						break;
+				 } else if (!is_bcast)
+					rtw_acl_add_sta(adapter, period, addr);
+			}
+
+			c = strsep(&next, " \t");
+		}
+	}
+
+exit:
+	return count;
+}
+#endif /* CONFIG_RTW_MACADDR_ACL */
+
+#if CONFIG_RTW_PRE_LINK_STA
+static int proc_get_pre_link_sta(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_pre_link_sta_ctl(m, &adapter->stapriv);
+	return 0;
+}
+
+ssize_t proc_set_pre_link_sta(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlme = &adapter->mlmepriv;
+	struct mlme_ext_priv *mlmeext = &adapter->mlmeextpriv;
+	char tmp[17 * RTW_PRE_LINK_STA_NUM + 32] = {0};
+	char arg0[16] = {0};
+	u8 addr[ETH_ALEN];
+
+#define PRE_LINK_STA_CMD_RESET	0
+#define PRE_LINK_STA_CMD_ADD	1
+#define PRE_LINK_STA_CMD_DEL	2
+#define PRE_LINK_STA_CMD_NUM	3
+
+	static const char * const pre_link_sta_cmd_str[] = {
+		"reset",
+		"add",
+		"del"
+	};
+	u8 cmd_id = PRE_LINK_STA_CMD_NUM;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		/* cmd [<macaddr>] */
+		char *c, *next;
+		int i;
+
+		next = tmp;
+		c = strsep(&next, " \t");
+
+		if (sscanf(c, "%s", arg0) != 1)
+			goto exit;
+
+		for (i = 0; i < PRE_LINK_STA_CMD_NUM; i++)
+			if (strcmp(pre_link_sta_cmd_str[i], arg0) == 0)
+				cmd_id = i;
+
+		switch (cmd_id) {
+		case PRE_LINK_STA_CMD_RESET:
+			rtw_pre_link_sta_ctl_reset(&adapter->stapriv);
+			goto exit;
+		case PRE_LINK_STA_CMD_ADD:
+		case PRE_LINK_STA_CMD_DEL:
+			break;
+		default:
+			goto exit;
+		}
+
+		/* macaddr list */
+		c = strsep(&next, " \t");
+		while (c != NULL) {
+			if (sscanf(c, MAC_SFMT, MAC_SARG(addr)) != 6)
+				break;
+
+			if (rtw_check_invalid_mac_address(addr, 0) == _FALSE) {
+				if (cmd_id == PRE_LINK_STA_CMD_ADD)
+					rtw_pre_link_sta_add(&adapter->stapriv, addr);
+				else
+					rtw_pre_link_sta_del(&adapter->stapriv, addr);
+			}
+
+			c = strsep(&next, " \t");
+		}
+	}
+
+exit:
+	return count;
+}
+#endif /* CONFIG_RTW_PRE_LINK_STA */
+
+static int proc_get_ch_sel_policy(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	RTW_PRINT_SEL(m, "%-16s\n", "within_same_band");
+
+	RTW_PRINT_SEL(m, "%16d\n", rfctl->ch_sel_within_same_band);
+
+	return 0;
+}
+
+static ssize_t proc_set_ch_sel_policy(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	char tmp[32];
+	u8 within_sb;
+	int num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%hhu", &within_sb);
+	if (num >=	1)
+		rfctl->ch_sel_within_same_band = within_sb ? 1 : 0;
+
+exit:
+	return count;
+}
+
+#ifdef CONFIG_80211D
+static int proc_get_country_ie_slave_records(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	dump_country_ie_slave_records(m, rfctl, 0);
+
+	return 0;
+}
+#endif /* CONFIG_80211D */
+
+#ifdef CONFIG_DFS_MASTER
+static int proc_get_dfs_test_case(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	RTW_PRINT_SEL(m, "%-24s %-19s\n", "radar_detect_trigger_non", "choose_dfs_ch_first");
+	RTW_PRINT_SEL(m, "%24hhu %19hhu\n"
+		, rfctl->dbg_dfs_radar_detect_trigger_non
+		, rfctl->dbg_dfs_choose_dfs_ch_first
+	);
+
+	return 0;
+}
+
+static ssize_t proc_set_dfs_test_case(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	char tmp[32];
+	u8 radar_detect_trigger_non;
+	u8 choose_dfs_ch_first;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhu %hhu", &radar_detect_trigger_non, &choose_dfs_ch_first);
+
+		if (num >= 1)
+			rfctl->dbg_dfs_radar_detect_trigger_non = radar_detect_trigger_non;
+		if (num >= 2)
+			rfctl->dbg_dfs_choose_dfs_ch_first = choose_dfs_ch_first;
+	}
+
+	return count;
+}
+
+ssize_t proc_set_update_non_ocp(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 band, ch, bw = CHANNEL_WIDTH_20, offset = CHAN_OFFSET_NO_EXT;
+	int ms = -1;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hhu %hhu %hhu %d", &band, &ch, &bw, &offset, &ms);
+
+		if (num < 1 || (bw != CHANNEL_WIDTH_20 && num < 4))
+			goto exit;
+
+		rtw_rfctl_force_update_non_ocp_ms(adapter_to_rfctl(adapter), band, ch, bw, offset, ms);
+	}
+
+exit:
+	return count;
+}
+
+ssize_t proc_set_radar_detect(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	char tmp[32];
+	u8 fake_radar_detect_cnt = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu", &fake_radar_detect_cnt);
+
+		if (num < 1)
+			goto exit;
+
+		rfctl->dbg_dfs_fake_radar_detect_cnt = fake_radar_detect_cnt;
+	}
+
+exit:
+	return count;
+}
+
+static int proc_get_dfs_ch_sel_e_flags(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	RTW_PRINT_SEL(m, "0x%02x\n", rfctl->dfs_ch_sel_e_flags);
+
+	return 0;
+}
+
+static ssize_t proc_set_dfs_ch_sel_e_flags(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	char tmp[32];
+	u8 e_flags;
+	int num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%hhx", &e_flags);
+	if (num != 1)
+		goto exit;
+
+	rfctl->dfs_ch_sel_e_flags = e_flags;
+
+exit:
+	return count;
+}
+
+static int proc_get_dfs_ch_sel_d_flags(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	RTW_PRINT_SEL(m, "0x%02x\n", rfctl->dfs_ch_sel_d_flags);
+
+	return 0;
+}
+
+static ssize_t proc_set_dfs_ch_sel_d_flags(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	char tmp[32];
+	u8 d_flags;
+	int num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%hhx", &d_flags);
+	if (num != 1)
+		goto exit;
+
+	rfctl->dfs_ch_sel_d_flags = d_flags;
+
+exit:
+	return count;
+}
+
+#if CONFIG_DFS_SLAVE_WITH_RADAR_DETECT
+static int proc_get_dfs_slave_with_rd(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+
+	RTW_PRINT_SEL(m, "%u\n", rfctl->dfs_slave_with_rd);
+
+	return 0;
+}
+
+static ssize_t proc_set_dfs_slave_with_rd(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(adapter);
+	char tmp[32];
+	u8 rd;
+	int num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%hhu", &rd);
+	if (num != 1)
+		goto exit;
+
+	rd = rd ? 1 : 0;
+
+	if (rfctl->dfs_slave_with_rd != rd) {
+		int i;
+
+		rfctl->dfs_slave_with_rd = rd;
+		for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+			rtw_dfs_rd_en_decision_cmd(adapter_to_dvobj(adapter), i);
+	}
+
+exit:
+	return count;
+}
+#endif /* CONFIG_DFS_SLAVE_WITH_RADAR_DETECT */
+#endif /* CONFIG_DFS_MASTER */
+
+#ifdef CONFIG_80211N_HT
+int proc_get_rx_ampdu_size_limit(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_regsty_rx_ampdu_size_limit(m, adapter);
+
+	return 0;
+}
+
+ssize_t proc_set_rx_ampdu_size_limit(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *regsty = adapter_to_regsty(adapter);
+	char tmp[32];
+	u8 nss;
+	u8 limit_by_bw[4] = {0xFF};
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int i;
+		int num = sscanf(tmp, "%hhu %hhu %hhu %hhu %hhu"
+			, &nss, &limit_by_bw[0], &limit_by_bw[1], &limit_by_bw[2], &limit_by_bw[3]);
+
+		if (num < 2)
+			goto exit;
+		if (nss == 0 || nss > 4)
+			goto exit;
+
+		for (i = 0; i < num - 1; i++)
+			regsty->rx_ampdu_sz_limit_by_nss_bw[nss - 1][i] = limit_by_bw[i];
+		rtw_rx_ampdu_apply(adapter);
+	}
+
+exit:
+	return count;
+}
+#endif /* CONFIG_80211N_HT */
+
+static int proc_get_rx_chk_limit(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "Rx chk limit : %d\n", rtw_get_rx_chk_limit(padapter));
+
+	return 0;
+}
+
+static ssize_t proc_set_rx_chk_limit(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char tmp[32];
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	int rx_chk_limit;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%d", &rx_chk_limit);
+
+		rtw_set_rx_chk_limit(padapter, rx_chk_limit);
+	}
+
+	return count;
+}
+
+static int proc_get_udpport(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct recv_info *precvinfo = &(padapter->recvinfo);
+
+	RTW_PRINT_SEL(m, "%d\n", precvinfo->sink_udpport);
+	return 0;
+}
+static ssize_t proc_set_udpport(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct recv_info *precvinfo;
+	int sink_udpport = 0;
+	char tmp[32];
+
+
+	if (!padapter)
+		return -EFAULT;
+
+	precvinfo = &padapter->recvinfo;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d", &sink_udpport);
+
+		if (num !=  1) {
+			RTW_INFO("invalid input parameter number!\n");
+			return count;
+		}
+
+	}
+	precvinfo->sink_udpport = sink_udpport;
+
+	return count;
+
+}
+
+#if 0
+static int proc_get_mi_ap_bc_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj);
+	u8 i;
+
+	for (i = 0; i < dvobj->iface_nums; i++)
+		RTW_PRINT_SEL(m, "iface_id:%d, mac_id && sec_cam_id = %d\n", i, macid_ctl->iface_bmc[i]);
+
+	return 0;
+}
+#endif
+#if 0
+static int proc_get_macid_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct macid_ctl_t *macid_ctl = dvobj_to_macidctl(dvobj);
+	u8 i;
+	u8 null_addr[ETH_ALEN] = {0};
+	u8 *macaddr;
+
+	RTW_PRINT_SEL(m, "max_num:%u\n", macid_ctl->num);
+	RTW_PRINT_SEL(m, "\n");
+
+	RTW_PRINT_SEL(m, "used:\n");
+	dump_macid_map(m, &macid_ctl->used, macid_ctl->num);
+	RTW_PRINT_SEL(m, "\n");
+
+	RTW_PRINT_SEL(m, "%-3s %-3s %-5s %-4s %-17s %-6s %-3s"
+		, "id", "bmc", "ifbmp", "ch_g", "macaddr", "bw", "vht");
+
+	if (get_phy_rx_nss(adapter, adapter_link) > 2)
+		_RTW_PRINT_SEL(m, " %-10s", "rate_bmp1");
+
+	_RTW_PRINT_SEL(m, " %-10s %s\n", "rate_bmp0", "status");
+
+	for (i = 0; i < macid_ctl->num; i++) {
+		if (rtw_macid_is_used(macid_ctl, i)
+			|| macid_ctl->h2c_msr[i]
+		) {
+			if (macid_ctl->sta[i])
+				macaddr = macid_ctl->sta[i]->phl_sta->mac_addr;
+			else
+				macaddr = null_addr;
+
+			RTW_PRINT_SEL(m, "%3u %3u  0x%02x %4d "MAC_FMT" %6s %3u"
+				, i
+				, rtw_macid_is_bmc(macid_ctl, i)
+				, rtw_macid_get_iface_bmp(macid_ctl, i)
+				, rtw_macid_get_ch_g(macid_ctl, i)
+				, MAC_ARG(macaddr)
+				, ch_width_str(macid_ctl->bw[i])
+				, macid_ctl->vht_en[i]
+			);
+
+			if (get_phy_rx_nss(adapter, adapter_link) > 2)
+				_RTW_PRINT_SEL(m, " 0x%08X", macid_ctl->rate_bmp1[i]);
+#ifdef DBG_MACID_MSR_INFO
+			_RTW_PRINT_SEL(m, " 0x%08X "H2C_MSR_FMT" %s\n"
+				, macid_ctl->rate_bmp0[i]
+				, H2C_MSR_ARG(&macid_ctl->h2c_msr[i])
+				, rtw_macid_is_used(macid_ctl, i) ? "" : "[unused]"
+			);
+#endif
+		}
+	}
+
+	return 0;
+}
+#endif
+#if 0
+static int proc_get_sec_cam(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+
+	RTW_PRINT_SEL(m, "sec_cap:0x%02x\n", cam_ctl->sec_cap);
+	RTW_PRINT_SEL(m, "flags:0x%08x\n", cam_ctl->flags);
+	RTW_PRINT_SEL(m, "\n");
+
+	RTW_PRINT_SEL(m, "max_num:%u\n", cam_ctl->num);
+	RTW_PRINT_SEL(m, "used:\n");
+	dump_sec_cam_map(m, &cam_ctl->used, cam_ctl->num);
+	RTW_PRINT_SEL(m, "\n");
+
+#if 0 /*GEORGIA_TODO_REDEFINE_IO*/
+	RTW_PRINT_SEL(m, "reg_scr:0x%04x\n", rtw_read16(adapter, 0x680));
+#endif
+	RTW_PRINT_SEL(m, "\n");
+
+	dump_sec_cam(m, adapter);
+
+	return 0;
+}
+
+static ssize_t proc_set_sec_cam(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct cam_ctl_t *cam_ctl = &dvobj->cam_ctl;
+	char tmp[32] = {0};
+	char cmd[4];
+	u8 id_1 = 0, id_2 = 0;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		/* c <id_1>: clear specific cam entry */
+		/* wfc <id_1>: write specific cam entry from cam cache */
+		/* sw <id_1> <id_2>: sec_cam 1/2 swap */
+
+		int num = sscanf(tmp, "%s %hhu %hhu", cmd, &id_1, &id_2);
+
+		if (num < 2)
+			return count;
+
+		if ((id_1 >= cam_ctl->num) || (id_2 >= cam_ctl->num)) {
+			RTW_ERR(FUNC_ADPT_FMT" invalid id_1:%u id_2:%u\n", FUNC_ADPT_ARG(adapter), id_1, id_2);
+			return count;
+		}
+
+		if (strcmp("c", cmd) == 0) {
+			_clear_cam_entry(adapter, id_1);
+			adapter->securitypriv.hw_decrypted = _FALSE; /* temporarily set this for TX path to use SW enc */
+		} else if (strcmp("wfc", cmd) == 0)
+			write_cam_from_cache(adapter, id_1);
+		else if (strcmp("sw", cmd) == 0)
+			rtw_sec_cam_swap(adapter, id_1, id_2);
+		else if (strcmp("cdk", cmd) == 0)
+			rtw_clean_dk_section(adapter);
+#ifdef DBG_SEC_CAM_MOVE
+		else if (strcmp("sgd", cmd) == 0)
+			rtw_hal_move_sta_gk_to_dk(adapter);
+		else if (strcmp("rsd", cmd) == 0)
+			rtw_hal_read_sta_dk_key(adapter, id_1);
+#endif
+	}
+
+	return count;
+}
+
+static int proc_get_sec_cam_cache(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_sec_cam_cache(m, adapter);
+	return 0;
+}
+#endif
+
+#ifdef CONFIG_DBG_AX_CAM
+
+static int proc_get_ax_address_cam(struct seq_file *m, void *v) {
+	struct net_device *dev = m->private;
+	struct _ADAPTER *adapter = (struct _ADAPTER *)rtw_netdev_priv(dev);
+
+	get_ax_address_cam(m, adapter);
+	return 0;
+}
+
+static int proc_get_ax_sec_cam(struct seq_file *m, void *v) {
+	struct net_device *dev = m->private;
+	struct _ADAPTER *adapter = (struct _ADAPTER *)rtw_netdev_priv(dev);
+
+	get_ax_sec_cam(m, adapter);
+	return 0;
+}
+
+static int proc_get_ax_valid_key(struct seq_file *m, void *v) {
+	struct net_device *dev = m->private;
+	struct _ADAPTER *adapter = (struct _ADAPTER *)rtw_netdev_priv(dev);
+
+	get_ax_valid_key(m, adapter);
+	return 0;
+}
+
+#endif
+
+static ssize_t proc_set_change_bss_chbw(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	int i;
+	char tmp[32];
+	s16 ch;
+	s8 bw = REQ_BW_NONE, offset = REQ_OFFSET_NONE, band = REQ_BAND_NONE;
+	u8 ifbmp = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hd %hhd %hhd %hhx", &ch, &bw, &offset, &ifbmp);
+
+		if (num < 1 || (bw != CHANNEL_WIDTH_20 && num < 3))
+			goto exit;
+
+		if (num < 4)
+			ifbmp = BIT(adapter->iface_id);
+		else
+			ifbmp &= (1 << dvobj->iface_nums) - 1;
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			if (!(ifbmp & BIT(i)) || !dvobj->padapters[i])
+				continue;
+
+			if (!CHK_MLME_STATE(dvobj->padapters[i], WIFI_AP_STATE | WIFI_MESH_STATE)
+				|| !MLME_IS_ASOC(dvobj->padapters[i]))
+				ifbmp &= ~BIT(i);
+		}
+
+		if (ifbmp)
+			rtw_change_bss_bchbw_cmd(adapter, RTW_CMDF_WAIT_ACK, ifbmp, 0, band, ch, bw, offset);
+	}
+
+exit:
+	return count;
+}
+
+static int proc_get_tx_bw_mode(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "0x%02x\n", adapter->driver_tx_bw_mode);
+	RTW_PRINT_SEL(m, "2.4G:%s\n", ch_width_str(ADAPTER_TX_BW_2G(adapter)));
+	RTW_PRINT_SEL(m, "5G:%s\n", ch_width_str(ADAPTER_TX_BW_5G(adapter)));
+	RTW_PRINT_SEL(m, "6G:%s\n", ch_width_str(ADAPTER_TX_BW_6G(adapter)));
+
+	return 0;
+}
+
+static void rtw_set_tx_bw_mode(_adapter *adapter, u16 bw_mode)
+{
+	struct _ADAPTER_LINK *alink;
+	struct link_mlme_ext_priv *mlmeext;
+	u8 i;
+	u8 update = 0;
+
+	for (i = 0; i < adapter->adapter_link_num; i++) {
+		alink = GET_LINK(adapter, i);
+		if (!LINK_MLME_IS_ASOC(alink))
+			continue;
+
+		mlmeext = &alink->mlmeextpriv;
+		switch (mlmeext->chandef.band) {
+		case BAND_ON_24G:
+			if (BW_MODE_2G(bw_mode) != ADAPTER_TX_BW_2G(adapter))
+				update |= BIT(mlmeext->chandef.band);
+			break;
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		case BAND_ON_5G:
+			if (BW_MODE_5G(bw_mode) != ADAPTER_TX_BW_5G(adapter))
+				update |= BIT(mlmeext->chandef.band);
+			break;
+		#endif
+		#if CONFIG_IEEE80211_BAND_6GHZ
+		case BAND_ON_6G:
+			if (BW_MODE_6G(bw_mode) != ADAPTER_TX_BW_6G(adapter))
+				update |= BIT(mlmeext->chandef.band);
+			break;
+		#endif
+		default:
+			break;
+		}
+	}
+
+	adapter->driver_tx_bw_mode = bw_mode;
+
+	if (update) {
+		_list *head, *list;
+		struct sta_info *sta = NULL;
+		struct sta_priv *stapriv = &adapter->stapriv;
+
+		_rtw_spinlock_bh(&stapriv->sta_hash_lock);
+		for (i = 0; i < NUM_STA; i++) {
+			head = &(stapriv->sta_hash[i]);
+			list = get_next(head);
+
+			while ((rtw_end_of_queue_search(head, list)) == _FALSE) {
+				sta = LIST_CONTAINOR(list, struct sta_info, hash_list);
+				list = get_next(list);
+
+				if (!(update & BIT(sta->phl_sta->chandef.band))
+					|| rtw_is_self_addr_stainfo(adapter, sta)
+					|| is_broadcast_mac_addr(sta->phl_sta->mac_addr))
+					continue;
+
+				rtw_sta_hal_ra_mask_update_cmd(adapter, sta, 0);
+			}
+		}
+		_rtw_spinunlock_bh(&stapriv->sta_hash_lock);
+	}
+}
+
+static ssize_t proc_set_tx_bw_mode(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u16 bw_mode;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hx", &bw_mode);
+
+		if (num < 1 || bw_mode == adapter->driver_tx_bw_mode)
+			goto exit;
+
+		rtw_set_tx_bw_mode(adapter, bw_mode);
+	}
+
+exit:
+	return count;
+}
+
+#ifdef CONFIG_ECSA_PHL
+static int proc_get_ecsa_allow_reason(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *a = (_adapter *)rtw_netdev_priv(dev);
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+
+	RTW_PRINT_SEL(m, "ECSA allow reason : 0x%02x\n", ecsa_info->ecsa_allow_case);
+	RTW_PRINT_SEL(m, "bit 0 : ECSA_START_MCC_24G_TO_24G\n");
+	RTW_PRINT_SEL(m, "bit 1 : ECSA_START_MCC_5G_TO_5G\n");
+	RTW_PRINT_SEL(m, "bit 2 : ECSA_START_MCC_24G_TO_5G\n");
+	RTW_PRINT_SEL(m, "bit 3 : ECSA_START_MCC_5G_TO_24G\n");
+	RTW_PRINT_SEL(m, "bit 4 : ECSA_START_CORE_TRIGGER\n");
+
+	return 0;
+}
+
+static ssize_t proc_set_ecsa_allow_reason(struct file *file,
+		const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *a = (_adapter *)rtw_netdev_priv(dev);
+	struct core_ecsa_info *ecsa_info = &(a->ecsa_info);
+	char tmp[12];
+	u8 reason;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhx", &reason);
+
+		if (num < 1 || reason >= ECSA_START_UNKNOWN)
+			goto exit;
+
+		ecsa_info->ecsa_allow_case = reason;
+	}
+exit:
+	return count;
+}
+
+#ifdef PRIVATE_R
+static ssize_t proc_set_p2p_park_ch(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *a = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(a);
+	char tmp[32];
+	u8 p2p_park_ch = 36;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhu", &p2p_park_ch);
+		if (num !=  1)
+			return count;
+	}
+
+	if (p2p_park_ch != 36 && p2p_park_ch != 149)
+		RTW_ERR("Only CH 36 or CH 149 is allowed\n");
+	else
+		rfctl->p2p_park_ch = p2p_park_ch;
+
+	return count;
+}
+
+static int proc_get_p2p_park_ch(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *a = (_adapter *)rtw_netdev_priv(dev);
+	struct rf_ctl_t *rfctl = adapter_to_rfctl(a);
+
+	RTW_PRINT_SEL(m, "p2p_park_ch: %u\n", rfctl->p2p_park_ch);
+
+	return 0;
+}
+#endif /* PRIVATE_R */
+#endif /* CONFIG_ECSA_PHL */
+
+static int proc_get_hal_txpwr_info(struct seq_file *m, void *v)
+{
+#ifdef CONFIG_TXPWR_PG_WITH_PWR_IDX
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *devob = adapter_to_dvobj(adapter);
+	HAL_DATA_TYPE *hal_data = GET_PHL_COM(devob);
+	struct hal_spec_t *hal_spec = GET_HAL_SPEC(devob);
+
+	if (hal_data->txpwr_pg_mode == TXPWR_PG_WITH_PWR_IDX) {
+		if (rtw_hw_is_band_support(devob, BAND_ON_24G))
+			dump_hal_txpwr_info_2g(m, adapter, hal_spec->rfpath_num_2g, hal_data->max_tx_cnt);
+
+		#if CONFIG_IEEE80211_BAND_5GHZ
+		if (rtw_hw_is_band_support(devob, BAND_ON_5G))
+			dump_hal_txpwr_info_5g(m, adapter, hal_spec->rfpath_num_5g, hal_data->max_tx_cnt);
+		#endif
+	}
+#endif
+
+	return 0;
+}
+
+static int proc_get_target_tx_power(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_hal_dump_target_tx_power(m, adapter);
+
+	return 0;
+}
+
+static int proc_get_tx_power_by_rate(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_txpwr_by_rate(m, adapter);
+
+	return 0;
+}
+
+#if CONFIG_TXPWR_LIMIT
+static int proc_get_tx_power_limit(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_txpwr_lmt(m, adapter);
+
+	return 0;
+}
+
+#ifdef CONFIG_80211AX_HE
+static int proc_get_tx_power_limit_ru(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_txpwr_lmt_ru(m, adapter);
+
+	return 0;
+}
+#endif
+#endif /* CONFIG_TXPWR_LIMIT */
+
+static int proc_get_tpc_settings(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_txpwr_tpc_settings(m, adapter_to_dvobj(adapter));
+
+	return 0;
+}
+
+static ssize_t proc_set_tpc_settings(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	struct rf_ctl_t *rfctl = dvobj_to_rfctl(dvobj);
+
+	char tmp[32] = {0};
+	u8 mode;
+	u16 m_constraint;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hu", &mode, &m_constraint);
+
+		if (num < 1)
+			return count;
+
+		if (mode >= TPC_MODE_INVALID)
+			return count;
+
+		if (mode == TPC_MODE_MANUAL && num >= 2)
+			rfctl->tpc_manual_constraint = rtw_min(m_constraint, TPC_MANUAL_CONSTRAINT_MAX);
+		rfctl->tpc_mode = mode;
+
+		if (rtw_hw_is_init_completed(dvobj))
+			rtw_run_in_thread_cmd_wait(adapter, (void *)rtw_update_txpwr_level_all_hwband, dvobj, 2000);
+	}
+
+	return count;
+}
+
+static int proc_get_tx_power_ext_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_tx_power_ext_info(m, adapter_to_dvobj(adapter));
+
+	return 0;
+}
+
+static ssize_t proc_set_tx_power_ext_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	char tmp[32] = {0};
+	char cmd[16] = {0};
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%s", cmd);
+
+		if (num < 1)
+			return count;
+
+#if 0 /* TODO */
+		#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+		phy_free_filebuf_mask(adapter, LOAD_BB_PG_PARA_FILE | LOAD_RF_TXPWR_LMT_PARA_FILE);
+		#endif
+
+		if (strcmp("default", cmd) == 0)
+			rtw_run_in_thread_cmd(adapter, ((void *)(phy_reload_default_tx_power_ext_info)), adapter);
+		else
+			rtw_run_in_thread_cmd(adapter, ((void *)(phy_reload_tx_power_ext_info)), adapter);
+
+		rtw_run_in_thread_cmd_wait(adapter, ((void *)(rtw_hal_update_txpwr_level)), adapter, 2000);
+#endif
+	}
+
+	return count;
+}
+
+#ifdef GEORGIA_TODO_TX_PWR
+static void *proc_start_tx_power_idx(struct seq_file *m, loff_t *pos)
+{
+	u8 path = ((*pos) & 0xFF00) >> 8;
+
+	if (path >= RF_PATH_MAX)
+		return NULL;
+
+	return pos;
+}
+static void proc_stop_tx_power_idx(struct seq_file *m, void *v)
+{
+}
+
+static void *proc_next_tx_power_idx(struct seq_file *m, void *v, loff_t *pos)
+{
+	u8 path = ((*pos) & 0xFF00) >> 8;
+	u8 rs = *pos & 0xFF;
+
+	rs++;
+	if (rs >= RATE_SECTION_NUM) {
+		rs = 0;
+		path++;
+	}
+
+	if (path >= RF_PATH_MAX)
+		return NULL;
+
+	*pos = (path << 8) | rs;
+
+	return pos;
+}
+
+static int proc_get_tx_power_idx(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	u32 pos = *((loff_t *)(v));
+	u8 path = (pos & 0xFF00) >> 8;
+	u8 rs = pos & 0xFF;
+
+	if (0)
+		RTW_INFO("%s path=%u, rs=%u\n", __func__, path, rs);
+
+	if (path == RF_PATH_A && rs == CCK)
+		dump_tx_power_idx_title(m, adapter);
+	dump_tx_power_idx_by_path_rs(m, adapter, path, rs);
+
+	return 0;
+}
+
+static struct seq_operations seq_ops_tx_power_idx = {
+	.start = proc_start_tx_power_idx,
+	.stop  = proc_stop_tx_power_idx,
+	.next  = proc_next_tx_power_idx,
+	.show  = proc_get_tx_power_idx,
+};
+
+static void *proc_start_txpwr_total_dbm(struct seq_file *m, loff_t *pos)
+{
+	u8 rs = *pos;
+
+	if (rs >= RATE_SECTION_NUM)
+		return NULL;
+
+	return pos;
+}
+
+static void proc_stop_txpwr_total_dbm(struct seq_file *m, void *v)
+{
+}
+
+static void *proc_next_txpwr_total_dbm(struct seq_file *m, void *v, loff_t *pos)
+{
+	u8 rs = *pos;
+
+	rs++;
+	if (rs >= RATE_SECTION_NUM)
+		return NULL;
+
+	*pos = rs;
+
+	return pos;
+}
+
+static int proc_get_txpwr_total_dbm(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	u32 pos = *((loff_t *)(v));
+	u8 rs = pos;
+
+	if (rs == CCK)
+		dump_txpwr_total_dbm_title(m, adapter);
+	dump_txpwr_total_dbm_by_rs(m, adapter, rs);
+
+	return 0;
+}
+
+static struct seq_operations seq_ops_txpwr_total_dbm = {
+	.start = proc_start_txpwr_total_dbm,
+	.stop  = proc_stop_txpwr_total_dbm,
+	.next  = proc_next_txpwr_total_dbm,
+	.show  = proc_get_txpwr_total_dbm,
+};
+#endif
+
+int proc_get_mac_addr(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_dump_macaddr(m, adapter);
+	return 0;
+}
+
+static int proc_get_skip_band(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	int bandskip;
+
+	bandskip = RTW_GET_SCAN_BAND_SKIP(adapter);
+	RTW_PRINT_SEL(m, "bandskip:0x%02x\n", bandskip);
+	return 0;
+}
+
+static ssize_t proc_set_skip_band(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[6];
+	u8 skip_band;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu", &skip_band);
+
+		if (num < 1)
+			return -EINVAL;
+
+		if (1 == skip_band)
+			RTW_SET_SCAN_BAND_SKIP(padapter, BAND_24G);
+		else if (2 == skip_band)
+			RTW_SET_SCAN_BAND_SKIP(padapter, BAND_5G);
+		else if (3 == skip_band)
+			RTW_CLR_SCAN_BAND_SKIP(padapter, BAND_24G);
+		else if (4 == skip_band)
+			RTW_CLR_SCAN_BAND_SKIP(padapter, BAND_5G);
+	}
+	return count;
+
+}
+
+#ifdef CONFIG_RTW_ACS
+static void rtw_acs_chan_info_dump(struct seq_file *m, _adapter *a)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+	struct rtw_acs_info_parm rpt;
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	int i;
+
+	RTW_PRINT_SEL(m, "%5s  %3s  %5s  %4s(%%)  %4s(%%)  %4s(%%)\n",
+						"Index", "CH", "Band", "NHM", "CLM", "tx");
+
+	for (i = 0; i < MAX_CHANNEL_NUM && chset->chs[i].ChannelNum != 0; i++){
+		_rtw_memset(&rpt, 0, sizeof(struct rtw_acs_info_parm));
+		rtw_acs_get_report(adapter, chset->chs[i].band, chset->chs[i].ChannelNum, &rpt);
+		RTW_PRINT_SEL(m, "%5d  %3d  %5s  %4d  %7d %4d\n",
+			i, chset->chs[i].ChannelNum ,band_str(chset->chs[i].band), rpt.rpt.nhm_ratio, rpt.rpt.clm_ratio, rpt.rpt.nhm_tx_ratio);
+	}
+}
+
+static int proc_get_chan_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_acs_chan_info_dump(m, adapter);
+	return 0;
+}
+
+#ifdef PRIVATE_R
+void rtw_acs_all_chan_info_dump(struct seq_file *m, _adapter *a)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *d = adapter_to_dvobj(adapter);
+	struct rtw_acs_info_parm rpt;
+	struct rtw_chset *chset = adapter_to_chset(adapter);
+	int i;
+
+	RTW_PRINT_SEL(m, "Channel Metrics Table\n");
+	RTW_PRINT_SEL(m, "---------------------\n");
+	RTW_PRINT_SEL(m, "%3s  %s(%%)  %s(%%)\n",
+						"Channel", "WiFi Util", "Interference Util");
+	RTW_PRINT_SEL(m, "-------  ------------  --------------------\n");
+
+	for (i = 0; i < MAX_CHANNEL_NUM && chset->chs[i].ChannelNum != 0; i++) {
+		_rtw_memset(&rpt, 0, sizeof(struct rtw_acs_info_parm));
+		rtw_acs_get_report(adapter, chset->chs[i].band, chset->chs[i].ChannelNum, &rpt);
+		RTW_PRINT_SEL(m, "%3d  %7d   %11d\n",
+						chset->chs[i].ChannelNum,
+						rpt.rpt.nhm_ratio,
+						rpt.rpt.clm_ratio);
+	}
+}
+
+static int proc_get_all_chan_load(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *a = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_acs_all_chan_info_dump(m, a);
+	return 0;
+}
+#endif /* PRIVATE_R */
+
+#ifdef WKARD_ACS
+static void rtw_acs_info_dump(struct seq_file *m, _adapter *a)
+{
+}
+static int proc_get_best_chan(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (IS_ACS_ENABLE(adapter))
+		rtw_acs_info_dump(m, adapter);
+	else
+		_RTW_PRINT_SEL(m,"ACS disabled\n");
+	return 0;
+}
+
+static ssize_t proc_set_acs(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+#ifdef CONFIG_RTW_ACS_DBG
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 acs_state = 0;
+	u16 scan_ch_ms= 0, acs_scan_ch_ms = 0;
+	u8 scan_type = SCAN_ACTIVE, igi= 0, bw = 0;
+	u8 acs_scan_type = SCAN_ACTIVE, acs_igi= 0, acs_bw = 0;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hhu %hu %hhx %hhu",
+			&acs_state, &scan_type, &scan_ch_ms, &igi, &bw);
+
+		if (num < 1)
+			return -EINVAL;
+
+		if (acs_state)
+			rtw_acs_start(padapter);
+		else
+			rtw_acs_stop(padapter);
+		num = num -1;
+
+		if(num) {
+			if (num-- > 0)
+				acs_scan_type = scan_type;
+			if (num-- > 0)
+				acs_scan_ch_ms = scan_ch_ms;
+			if (num-- > 0)
+				acs_igi = igi;
+			if (num-- > 0)
+				acs_bw = bw;
+			rtw_acs_adv_setting(padapter, acs_scan_type, acs_scan_ch_ms, acs_igi, acs_bw);
+		}
+	}
+#endif /*CONFIG_RTW_ACS_DBG*/
+	return count;
+}
+#endif /* WKARD_ACS */
+#endif /*CONFIG_RTW_ACS*/
+
+static int proc_get_env_rpt(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_dump_env_rpt(adapter, m);
+
+	return 0;
+}
+
+static int proc_get_hal_spec(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_hw_dump_hal_spec(m, adapter_to_dvobj(adapter));
+	return 0;
+}
+
+static int proc_get_hal_trx_mode(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	rtw_hal_dump_trx_mode(m, adapter);
+	return 0;
+}
+
+static int proc_get_phy_cap(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+#ifdef CONFIG_80211N_HT
+	rtw_dump_drv_phy_cap(m, adapter);
+	rtw_get_dft_phy_cap(m, adapter);
+#endif /* CONFIG_80211N_HT */
+	return 0;
+}
+
+static int proc_dump_rsvd_page(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_hal_dump_rsvd_page(m, adapter, adapter->rsvd_page_offset, adapter->rsvd_page_num);
+	return 0;
+}
+static ssize_t proc_set_rsvd_page_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 page_offset, page_num;
+
+	if (count < 2)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %hhu", &page_offset, &page_num);
+
+		if (num < 2)
+			return -EINVAL;
+		padapter->rsvd_page_offset = page_offset;
+		padapter->rsvd_page_num = page_num;
+	}
+	return count;
+}
+
+#ifdef CONFIG_SUPPORT_FIFO_DUMP
+static int proc_dump_fifo(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_dump_fifo(m, adapter, adapter->fifo_sel, adapter->fifo_addr, adapter->fifo_size);
+	return 0;
+}
+static ssize_t proc_set_fifo_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	u8 fifo_sel = 0;
+	u32 fifo_addr = 0;
+	u32 fifo_size = 0;
+
+	if (count < 3)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%hhu %x %d", &fifo_sel, &fifo_addr, &fifo_size);
+
+		if (num < 3)
+			return -EINVAL;
+
+		padapter->fifo_sel = fifo_sel;
+		padapter->fifo_addr = fifo_addr;
+		padapter->fifo_size = fifo_size;
+	}
+	return count;
+}
+#endif
+
+
+#ifdef CONFIG_WOWLAN
+int proc_dump_pattern_cam(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	int i;
+	#if 0 /* WOW_ToDo */
+	struct  rtl_wow_pattern context;
+
+	for (i = 0 ; i < pwrpriv->wowlan_pattern_idx; i++) {
+		rtw_wow_pattern_read_cam_ent(padapter, i, &context);
+		rtw_dump_wow_pattern(m, &context, i);
+	}
+	#endif
+	return 0;
+}
+#endif
+
+
+static int proc_get_napi_info(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregistrypriv = &adapter->registrypriv;
+	u8 napi = 0, gro = 0;
+	u32 weight = 0;
+	struct dvobj_priv *d;
+	d = adapter_to_dvobj(adapter);
+
+
+#ifdef CONFIG_RTW_NAPI
+	if (pregistrypriv->en_napi) {
+		napi = 1;
+		weight = RTL_NAPI_WEIGHT;
+	}
+
+#ifdef CONFIG_RTW_GRO
+	if (pregistrypriv->en_gro)
+		gro = 1;
+#endif /* CONFIG_RTW_GRO */
+#endif /* CONFIG_RTW_NAPI */
+
+	if (napi) {
+		RTW_PRINT_SEL(m, "NAPI enable, weight=%d\n", weight);
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+		RTW_PRINT_SEL(m, "Dynamaic NAPI mechanism is on, current NAPI %s\n",
+			      d->en_napi_dynamic ? "enable" : "disable");
+		RTW_PRINT_SEL(m, "Dynamaic NAPI info:\n"
+				 "\ttcp_rx_threshold = %d Mbps\n"
+				 "\tcur_rx_tp = %d Mbps\n",
+			      pregistrypriv->napi_threshold,
+			      d->traffic_stat.cur_rx_tp);
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+	} else {
+		RTW_PRINT_SEL(m, "NAPI disable\n");
+	}
+	RTW_PRINT_SEL(m, "GRO %s\n", gro?"enable":"disable");
+
+	return 0;
+
+}
+
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+static ssize_t proc_set_napi_th(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *registry = &adapter->registrypriv;
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	_adapter *iface = NULL;
+	char tmp[32] = {0};
+	int thrshld = 0;
+	int num = 0, i = 0;
+
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	RTW_INFO("%s: Last threshold = %d Mbps\n", __FUNCTION__, registry->napi_threshold);
+
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (iface) {
+			if (buffer && !copy_from_user(tmp, buffer, count)) {
+				registry = &iface->registrypriv;
+				num = sscanf(tmp, "%d", &thrshld);
+				if (num > 0) {
+					if (thrshld > 0)
+						registry->napi_threshold = thrshld;
+				}
+			}
+		}
+	}
+	RTW_INFO("%s: New threshold = %d Mbps\n", __FUNCTION__, registry->napi_threshold);
+	RTW_INFO("%s: Current RX throughput = %d Mbps\n",
+		 __FUNCTION__, adapter_to_dvobj(adapter)->traffic_stat.cur_rx_tp);
+
+	return count;
+}
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+
+
+ssize_t proc_set_dynamic_agg_enable(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+	int enable = 0, i = 0;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+		_adapter *iface = NULL;
+		int num = sscanf(tmp, "%d", &enable);
+
+		if (num !=  1) {
+			RTW_INFO("invalid parameter!\n");
+			return count;
+		}
+
+		RTW_INFO("dynamic_agg_enable:%d\n", enable);
+
+		for (i = 0; i < dvobj->iface_nums; i++) {
+			iface = dvobj->padapters[i];
+			if (iface)
+				iface->registrypriv.dynamic_agg_enable = enable;
+		}
+
+	}
+
+	return count;
+
+}
+
+static int proc_get_dynamic_agg_enable(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *pregistrypriv = &adapter->registrypriv;
+
+	RTW_PRINT_SEL(m, "dynamic_agg_enable:%d\n", pregistrypriv->dynamic_agg_enable);
+
+	return 0;
+}
+
+#ifdef CONFIG_RTW_WDS
+static int proc_get_wds_en(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	if (MLME_STATE(adapter) & (WIFI_AP_STATE | WIFI_STATION_STATE))
+		RTW_PRINT_SEL(m, "%d\n", adapter_use_wds(adapter));
+
+	return 0;
+}
+
+static ssize_t proc_set_wds_en(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (!(MLME_STATE(adapter) & (WIFI_AP_STATE | WIFI_STATION_STATE)))
+		return -EFAULT;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u8 enable;
+		int num = sscanf(tmp, "%hhu", &enable);
+
+		if (num >= 1)
+			adapter_set_use_wds(adapter, enable);
+	}
+
+	return count;
+}
+
+static ssize_t proc_set_sta_wds_en(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u8 enable;
+		u8 addr[ETH_ALEN];
+		struct sta_info *sta;
+		int num = sscanf(tmp, "%hhu "MAC_SFMT, &enable, MAC_SARG(addr));
+
+		if (num != 7)
+			return -EINVAL;
+
+		if (IS_MCAST(addr) || _rtw_memcmp(adapter_mac_addr(adapter), addr, ETH_ALEN))
+			return -EINVAL;
+
+		sta = rtw_get_stainfo(&adapter->stapriv, addr);
+		if (!sta)
+			return -EINVAL;
+
+		if (enable)
+			sta->flags |= WLAN_STA_WDS;
+		else
+			sta->flags &= ~WLAN_STA_WDS;
+	}
+
+	return count;
+}
+
+static int proc_get_wds_gptr(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	if (MLME_IS_STA(adapter) && MLME_IS_ASOC(adapter))
+		dump_wgptr(m, adapter);
+
+	return 0;
+}
+
+#ifdef CONFIG_AP_MODE
+static int proc_get_wds_path(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	if (MLME_IS_AP(adapter) && MLME_IS_ASOC(adapter))
+		dump_wpath(m, adapter);
+
+	return 0;
+}
+#endif /* CONFIG_AP_MODE */
+#endif /* CONFIG_RTW_WDS */
+
+#ifdef CONFIG_RTW_MULTI_AP
+static int proc_get_multi_ap_opmode(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+
+	if (MLME_STATE(adapter) & (WIFI_AP_STATE | WIFI_STATION_STATE))
+		RTW_PRINT_SEL(m, "0x%02x\n", adapter->multi_ap);
+
+	return 0;
+}
+
+static ssize_t proc_set_multi_ap_opmode(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (!(MLME_STATE(adapter) & (WIFI_AP_STATE | WIFI_STATION_STATE)))
+		return -EFAULT;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u8 mode;
+		int num = sscanf(tmp, "%hhx", &mode);
+
+		if (num >= 1) {
+			if (MLME_IS_AP(adapter))
+				adapter->multi_ap = mode & (MULTI_AP_FRONTHAUL_BSS | MULTI_AP_BACKHAUL_BSS);
+			else
+				adapter->multi_ap = mode & MULTI_AP_BACKHAUL_STA;
+			if (adapter->multi_ap & (MULTI_AP_BACKHAUL_BSS | MULTI_AP_BACKHAUL_STA))
+				adapter_set_use_wds(adapter, 1);
+		}
+	}
+
+	return count;
+}
+
+static int proc_get_unassoc_sta(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = GET_PRIMARY_ADAPTER(rtw_netdev_priv(dev));
+
+	dump_unassoc_sta(m, adapter);
+	return 0;
+}
+
+ssize_t proc_set_unassoc_sta(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = GET_PRIMARY_ADAPTER(rtw_netdev_priv(dev));
+	char tmp[17 * 10 + 32] = {0};
+	char cmd[32];
+	u8 mode;
+	u8 stype = 0;
+	u8 addr[ETH_ALEN];
+
+#define UNASOC_STA_CMD_MODE	0
+#define UNASOC_STA_CMD_ADD	1
+#define UNASOC_STA_CMD_DEL	2
+#define UNASOC_STA_CMD_CLR	3
+#define UNASOC_STA_CMD_UNINT	4
+#define UNASOC_STA_CMD_NUM	5
+
+	static const char * const unasoc_sta_cmd_str[] = {
+		"mode",
+		"add",
+		"del",
+		"clr",
+		"uninterest",
+	};
+	u8 cmd_id = UNASOC_STA_CMD_NUM;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		RTW_WARN(FUNC_ADPT_FMT" input string too long\n", FUNC_ADPT_ARG(adapter));
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		/*
+		* mode <stype>,<mode>
+		* add <macaddr> [<macaddr>]
+		* del <macaddr> [<macaddr>]
+		* clr
+		*/
+		char *c, *next;
+		int i;
+		u8 is_bcast;
+
+		next = tmp;
+		c = strsep(&next, " \t");
+		if (!c || sscanf(c, "%s", cmd) != 1)
+			goto exit;
+
+		for (i = 0; i < UNASOC_STA_CMD_NUM; i++)
+			if (strcmp(unasoc_sta_cmd_str[i], cmd) == 0)
+				cmd_id = i;
+
+		switch (cmd_id) {
+		case UNASOC_STA_CMD_MODE:
+			c = strsep(&next, " \t");
+			if (!c || sscanf(c, "%hhu,%hhu", &stype, &mode) != 2) {
+				RTW_WARN(FUNC_ADPT_FMT" invalid arguments of mode cmd\n", FUNC_ADPT_ARG(adapter));
+				goto exit;
+			}
+
+			if (stype >= UNASOC_STA_SRC_NUM) {
+				RTW_WARN(FUNC_ADPT_FMT" invalid stype:%u\n", FUNC_ADPT_ARG(adapter), stype);
+				goto exit;
+			}
+
+			if (mode >= UNASOC_STA_MODE_NUM) {
+				RTW_WARN(FUNC_ADPT_FMT" invalid mode:%u\n", FUNC_ADPT_ARG(adapter), mode);
+				goto exit;
+			}
+
+			rtw_unassoc_sta_set_mode(adapter, stype, mode);
+			break;
+
+		case UNASOC_STA_CMD_ADD:
+		case UNASOC_STA_CMD_DEL:
+		case UNASOC_STA_CMD_UNINT:
+			/* check for macaddr list */
+			c = strsep(&next, " \t");
+			while (c != NULL) {
+				if (sscanf(c, MAC_SFMT, MAC_SARG(addr)) != 6)
+					break;
+
+				is_bcast = is_broadcast_mac_addr(addr);
+				if (is_bcast
+					|| rtw_check_invalid_mac_address(addr, 0) == _FALSE
+				) {
+					if (cmd_id == UNASOC_STA_CMD_DEL) {
+						if (is_bcast) {
+							rtw_del_unassoc_sta_queue(adapter);
+							break;
+						} else
+							rtw_del_unassoc_sta(adapter, addr);
+					} else if (cmd_id == UNASOC_STA_CMD_UNINT) {
+						if (is_bcast) {
+							rtw_undo_all_interested_unassoc_sta(adapter);
+							break;
+						} else
+							rtw_undo_interested_unassoc_sta(adapter, addr);
+					} else if (!is_bcast)
+						rtw_add_interested_unassoc_sta(adapter, addr);
+				}
+
+				c = strsep(&next, " \t");
+			}
+			break;
+
+		case UNASOC_STA_CMD_CLR:
+			/* clear sta list */
+			rtw_del_unassoc_sta_queue(adapter);
+			goto exit;
+
+		default:
+			RTW_WARN(FUNC_ADPT_FMT" invalid cmd:\"%s\"\n", FUNC_ADPT_ARG(adapter), cmd);
+			goto exit;
+		}
+	}
+
+exit:
+	return count;
+}
+
+#ifdef CONFIG_IOCTL_CFG80211
+static u8 assoc_req_mac_addr[6];
+int proc_get_sta_assoc_req_frame_body(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_AP(adapter)) {
+		struct sta_info *psta;
+		u8 *passoc_req = NULL;
+		u32 assoc_req_len = 0;
+
+		psta = rtw_get_stainfo(&adapter->stapriv, assoc_req_mac_addr);
+		if (psta == NULL) {
+			RTW_PRINT(FUNC_ADPT_FMT" sta("MAC_FMT") not found\n",
+				  FUNC_ADPT_ARG(adapter), MAC_ARG(assoc_req_mac_addr));
+			return 0;
+		}
+
+		RTW_PRINT(FUNC_ADPT_FMT" sta("MAC_FMT") found\n",
+			  FUNC_ADPT_ARG(adapter), MAC_ARG(assoc_req_mac_addr));
+		_rtw_spinlock_bh(&psta->lock);
+		if (psta->passoc_req && psta->assoc_req_len > 0) {
+			passoc_req = rtw_zmalloc(psta->assoc_req_len);
+			if (passoc_req) {
+				assoc_req_len = psta->assoc_req_len;
+				_rtw_memcpy(passoc_req, psta->passoc_req, assoc_req_len);
+			}
+		}
+
+		_rtw_spinunlock_bh(&psta->lock);
+		if (passoc_req && assoc_req_len > IEEE80211_3ADDR_LEN) {
+			u8 *body = passoc_req + IEEE80211_3ADDR_LEN;
+			u32 body_len = assoc_req_len - IEEE80211_3ADDR_LEN;
+			u16 i;
+
+			for (i = 0; i < body_len; i++)
+				_RTW_PRINT_SEL(m, "%02X", body[i]);
+			_RTW_PRINT_SEL(m, "\n");
+		}
+
+		if (passoc_req && assoc_req_len > 0)
+			rtw_mfree(passoc_req, assoc_req_len);
+	}
+
+	return 0;
+}
+
+ssize_t proc_set_sta_assoc_req_frame_body(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[18] = {0};
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		if (sscanf(tmp, MAC_SFMT, MAC_SARG(assoc_req_mac_addr)) != 6) {
+			_rtw_memset(assoc_req_mac_addr, 0, 6);
+			RTW_PRINT(FUNC_ADPT_FMT" Invalid format\n",
+				  FUNC_ADPT_ARG(adapter));
+		}
+
+	}
+
+	return count;
+}
+#endif /* CONFIG_IOCTL_CFG80211 */
+
+static int proc_get_ch_util_threshold(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = GET_PRIMARY_ADAPTER(rtw_netdev_priv(dev));
+
+	RTW_PRINT_SEL(m, "%hhu\n", adapter->ch_util_threshold);
+
+	return 0;
+}
+
+static ssize_t proc_set_ch_util_threshold(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = GET_PRIMARY_ADAPTER(rtw_netdev_priv(dev));
+	char tmp[4];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		u8 threshold;
+		int num = sscanf(tmp, "%hhu", &threshold);
+
+		if (num == 1)
+			adapter->ch_util_threshold = threshold;
+	}
+
+	return count;
+}
+
+static int proc_get_ch_utilization(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	RTW_PRINT_SEL(m, "%hhu\n", rtw_get_ch_utilization(adapter));
+
+	return 0;
+}
+#endif /* CONFIG_RTW_MULTI_AP */
+
+#ifdef CONFIG_RTW_MESH
+static int proc_get_mesh_peer_sel_policy(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_mesh_peer_sel_policy(m, adapter);
+
+	return 0;
+}
+
+#if CONFIG_RTW_MESH_ACNODE_PREVENT
+static int proc_get_mesh_acnode_prevent(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		dump_mesh_acnode_prevent_settings(m, adapter);
+
+	return 0;
+}
+
+static ssize_t proc_set_mesh_acnode_prevent(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+		u8 enable;
+		u32 conf_timeout_ms;
+		u32 notify_timeout_ms;
+		int num = sscanf(tmp, "%hhu %u %u", &enable, &conf_timeout_ms, &notify_timeout_ms);
+
+		if (num >= 1)
+			peer_sel_policy->acnode_prevent = enable;
+		if (num >= 2)
+			peer_sel_policy->acnode_conf_timeout_ms = conf_timeout_ms;
+		if (num >= 3)
+			peer_sel_policy->acnode_notify_timeout_ms = notify_timeout_ms;
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_MESH_ACNODE_PREVENT */
+
+#if CONFIG_RTW_MESH_OFFCH_CAND
+static int proc_get_mesh_offch_cand(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		dump_mesh_offch_cand_settings(m, adapter);
+
+	return 0;
+}
+
+static ssize_t proc_set_mesh_offch_cand(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+		u8 enable;
+		u32 find_int_ms;
+		int num = sscanf(tmp, "%hhu %u", &enable, &find_int_ms);
+
+		if (num >= 1)
+			peer_sel_policy->offch_cand = enable;
+		if (num >= 2)
+			peer_sel_policy->offch_find_int_ms = find_int_ms;
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_MESH_OFFCH_CAND */
+
+#if CONFIG_RTW_MESH_PEER_BLACKLIST
+static int proc_get_mesh_peer_blacklist(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter)) {
+		dump_mesh_peer_blacklist_settings(m, adapter);
+		if (MLME_IS_ASOC(adapter))
+			dump_mesh_peer_blacklist(m, adapter);
+	}
+
+	return 0;
+}
+
+static ssize_t proc_set_mesh_peer_blacklist(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+		u32 conf_timeout_ms;
+		u32 blacklist_timeout_ms;
+		int num = sscanf(tmp, "%u %u", &conf_timeout_ms, &blacklist_timeout_ms);
+
+		if (num >= 1)
+			peer_sel_policy->peer_conf_timeout_ms = conf_timeout_ms;
+		if (num >= 2)
+			peer_sel_policy->peer_blacklist_timeout_ms = blacklist_timeout_ms;
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_MESH_PEER_BLACKLIST */
+
+#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+static int proc_get_mesh_cto_mgate_require(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		RTW_PRINT_SEL(m, "%u\n", adapter->mesh_cfg.peer_sel_policy.cto_mgate_require);
+
+	return 0;
+}
+
+static ssize_t proc_set_mesh_cto_mgate_require(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+		u8 require;
+		int num = sscanf(tmp, "%hhu", &require);
+
+		if (num >= 1) {
+			peer_sel_policy->cto_mgate_require = require;
+			#if CONFIG_RTW_MESH_CTO_MGATE_CARRIER
+			if (rtw_mesh_cto_mgate_required(adapter))
+				rtw_netif_carrier_off(adapter->pnetdev);
+			else
+				rtw_netif_carrier_on(adapter->pnetdev);
+			#endif
+		}
+	}
+
+	return count;
+}
+
+static int proc_get_mesh_cto_mgate_blacklist(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter)) {
+		dump_mesh_cto_mgate_blacklist_settings(m, adapter);
+		if (MLME_IS_ASOC(adapter))
+			dump_mesh_cto_mgate_blacklist(m, adapter);
+	}
+
+	return 0;
+}
+
+static ssize_t proc_set_mesh_cto_mgate_blacklist(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		struct mesh_peer_sel_policy *peer_sel_policy = &adapter->mesh_cfg.peer_sel_policy;
+		u32 conf_timeout_ms;
+		u32 blacklist_timeout_ms;
+		int num = sscanf(tmp, "%u %u", &conf_timeout_ms, &blacklist_timeout_ms);
+
+		if (num >= 1)
+			peer_sel_policy->cto_mgate_conf_timeout_ms = conf_timeout_ms;
+		if (num >= 2)
+			peer_sel_policy->cto_mgate_blacklist_timeout_ms = blacklist_timeout_ms;
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST */
+
+static int proc_get_mesh_networks(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	dump_mesh_networks(m, adapter);
+
+	return 0;
+}
+
+static int proc_get_mesh_plink_ctl(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		dump_mesh_plink_ctl(m, adapter);
+
+	return 0;
+}
+
+static int proc_get_mesh_mpath(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter))
+		dump_mpath(m, adapter);
+
+	return 0;
+}
+
+static int proc_get_mesh_mpp(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter) && MLME_IS_ASOC(adapter))
+		dump_mpp(m, adapter);
+
+	return 0;
+}
+
+static int proc_get_mesh_known_gates(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		dump_known_gates(m, adapter);
+
+	return 0;
+}
+
+#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+static int proc_get_mesh_b2u_flags(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		dump_mesh_b2u_flags(m, adapter);
+
+	return 0;
+}
+
+static ssize_t proc_set_mesh_b2u_flags(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+		u8 msrc, mfwd;
+		int num = sscanf(tmp, "%hhx %hhx", &msrc, &mfwd);
+
+		if (num >= 1)
+			mcfg->b2u_flags_msrc = msrc;
+		if (num >= 2)
+			mcfg->b2u_flags_mfwd = mfwd;
+	}
+
+	return count;
+}
+#endif /* CONFIG_RTW_MESH_DATA_BMC_TO_UC */
+
+static int proc_get_mesh_stats(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		dump_mesh_stats(m, adapter);
+
+	return 0;
+}
+
+static int proc_get_mesh_gate_timeout(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+
+	if (MLME_IS_MESH(adapter))
+		RTW_PRINT_SEL(m, "%u factor\n",
+			       adapter->mesh_cfg.path_gate_timeout_factor);
+
+	return 0;
+}
+
+static ssize_t proc_set_mesh_gate_timeout(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	char tmp[32];
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+		u32 timeout;
+		int num = sscanf(tmp, "%u", &timeout);
+
+		if (num < 1)
+			goto exit;
+
+		mcfg->path_gate_timeout_factor = timeout;
+	}
+
+exit:
+	return count;
+}
+
+static int proc_get_mesh_gate_state(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct rtw_mesh_cfg *mcfg = &adapter->mesh_cfg;
+	u8 cto_mgate = 0;
+
+	if (MLME_IS_MESH(adapter)) {
+		if (rtw_mesh_is_primary_gate(adapter))
+			RTW_PRINT_SEL(m, "PG\n");
+		else if (mcfg->dot11MeshGateAnnouncementProtocol)
+			RTW_PRINT_SEL(m, "G\n");
+		else if (rtw_mesh_gate_num(adapter))
+			RTW_PRINT_SEL(m, "C\n");
+		else
+			RTW_PRINT_SEL(m, "N\n");
+	}
+
+	return 0;
+}
+
+static int proc_get_peer_alive_based_preq(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter= (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv  *rp = &adapter->registrypriv;
+
+	RTW_PRINT_SEL(m, "peer_alive_based_preq = %u\n",
+		      rp->peer_alive_based_preq);
+
+	return 0;
+}
+
+static ssize_t
+proc_set_peer_alive_based_preq(struct file *file, const char __user *buffer,
+			       size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv  *rp = &adapter->registrypriv;
+	char tmp[8];
+	int num = 0;
+	u8 enable = 0;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%hhu", &enable);
+	if (num !=  1) {
+		RTW_ERR("%s: invalid parameter!\n", __FUNCTION__);
+		goto exit;
+	}
+
+	if (enable > 1) {
+		RTW_ERR("%s: invalid value!\n", __FUNCTION__);
+		goto exit;
+	}
+	rp->peer_alive_based_preq = enable;
+
+exit:
+	return count;
+}
+
+#endif /* CONFIG_RTW_MESH */
+
+#ifdef RTW_BUSY_DENY_SCAN
+static int proc_get_scan_interval_thr(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter= (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *rp = &adapter->registrypriv;
+
+
+	RTW_PRINT_SEL(m, "scan interval threshold = %u ms\n",
+		      rp->scan_interval_thr);
+
+	return 0;
+}
+
+static ssize_t proc_set_scan_interval_thr(struct file *file,
+				          const char __user *buffer,
+				          size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter= (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv *rp = &adapter->registrypriv;
+	char tmp[12];
+	int num = 0;
+	u32 thr = 0;
+
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%u", &thr);
+	if (num != 1) {
+		RTW_ERR("%s: invalid parameter!\n", __FUNCTION__);
+		goto exit;
+	}
+
+	rp->scan_interval_thr = thr;
+
+	RTW_PRINT("%s: scan interval threshold = %u ms\n",
+		  __FUNCTION__, rp->scan_interval_thr);
+
+exit:
+	return count;
+}
+
+#endif /* RTW_BUSY_DENY_SCAN */
+
+static int proc_get_scan_deny(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter= (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	RTW_PRINT_SEL(m, "scan_deny set by /proc is %s\n", (dvobj->scan_deny == _TRUE) ? "enable":"disable");
+	RTW_PRINT_SEL(m, "set_scan_deny set by driver is %s\n", (rtw_mi_is_scan_deny(adapter) != 0) ? "enable":"disable");
+
+	return 0;
+}
+
+static ssize_t proc_set_scan_deny(struct file *file, const char __user *buffer,
+				size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+	char tmp[8];
+	int num = 0;
+	int enable = 0;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (!buffer || copy_from_user(tmp, buffer, count))
+		goto exit;
+
+	num = sscanf(tmp, "%d", &enable);
+	if (num !=  1) {
+		RTW_ERR("%s: invalid parameter!\n", __FUNCTION__);
+		goto exit;
+	}
+
+	dvobj->scan_deny = enable ? _TRUE : _FALSE;
+
+	RTW_PRINT("%s: scan_deny is %s\n",
+		  __FUNCTION__, (dvobj->scan_deny == _TRUE) ? "enable":"disable");
+
+exit:
+	return count;
+}
+
+int proc_get_cur_beacon_keys(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	/* ToDo CONFIG_RTW_MLD: [currently primary link only] */
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+	struct link_mlme_priv *mlme = &adapter_link->mlmepriv;
+
+	rtw_dump_bcn_keys(m, &mlme->cur_beacon_keys);
+
+	return 0;
+}
+
+#if defined(PRIVATE_R) && defined(CONFIG_P2P)
+static int proc_get_go_hidden_ssid_mode(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv)
+		RTW_PRINT_SEL(m, "0x%lx\n", pregpriv->go_hidden_ssid_mode);
+
+	return 0;
+}
+
+static ssize_t proc_set_go_hidden_ssid_mode(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	char tmp[32];
+	int go_hidden_ssid_mode, feature_bit;
+
+	if (!padapter)
+		return -EFAULT;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d %d",&feature_bit,  &go_hidden_ssid_mode);
+
+		if (num < 1)
+			return -EINVAL;
+
+		/* Cancel hide_ssid_timer and reset flag when unhide SSID by proc */
+		if (!feature_bit && !go_hidden_ssid_mode) {
+			ATOMIC_SET(&pregpriv->set_hide_ssid_timer, 0);
+			_cancel_timer_ex(&pmlmeext->hide_ssid_timer);
+		}
+		if (go_hidden_ssid_mode == 1)
+			rtw_set_bit(feature_bit, &pregpriv->go_hidden_ssid_mode);
+		else if (!go_hidden_ssid_mode)
+			rtw_clear_bit(feature_bit, &pregpriv->go_hidden_ssid_mode);
+		else
+			return -EFAULT;
+
+		RTW_INFO("go_hidden_ssid_mode:0x%lx\n", pregpriv->go_hidden_ssid_mode);
+	} else
+		return -EFAULT;
+
+	issue_beacon(padapter, 0);
+
+	return count;
+}
+
+#endif
+
+static int proc_get_amsdu_mode(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv) {
+		if (pregpriv->amsdu_mode == RTW_AMSDU_MODE_NON_SPP)
+			RTW_PRINT_SEL(m, "amsdu mode: NON-SPP\n");
+		else if (pregpriv->amsdu_mode == RTW_AMSDU_MODE_SPP)
+			RTW_PRINT_SEL(m, "amsdu mode: SPP\n");
+		else if (pregpriv->amsdu_mode == RTW_AMSDU_MODE_ALL_DROP)
+			RTW_PRINT_SEL(m, "amsdu mode: ALL DROP\n");
+		else
+			RTW_PRINT_SEL(m, "unexpected amsdu mode\n");
+	}
+
+	return 0;
+}
+
+static ssize_t proc_set_amsdu_mode(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+	char tmp[32];
+	u32 mode;
+	u8 bw_2g;
+	u8 bw_5g;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		int num = sscanf(tmp, "%d", &mode);
+
+		if (mode == RTW_AMSDU_MODE_NON_SPP
+			|| mode == RTW_AMSDU_MODE_SPP
+			|| mode == RTW_AMSDU_MODE_ALL_DROP) {
+			pregpriv->amsdu_mode = mode;
+			RTW_INFO("amsdu mode=%u\n", mode);
+		} else {
+			RTW_INFO("set unexpected mode = %d, won't apply\n", mode);
+		}
+	}
+
+	return count;
+
+}
+
+#ifdef PRIVATE_R
+static int proc_get_vendor_ie_filter(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
+	int i, j;
+
+	if (!mlmepriv)
+		return -EFAULT;
+
+	RTW_PRINT_SEL(m, "enable: %u\n", mlmepriv->vendor_ie_filter_enable);
+	RTW_PRINT_SEL(m, "index\tcontent\n");
+
+	for (i = 0; i < MAX_VENDOR_IE_NUM; i++) {
+		RTW_PRINT_SEL(m, "%d\t", i);
+		if (mlmepriv->vendor_ie_len[i])
+			for (j = 0; j < mlmepriv->vendor_ie_len[i] + 2; j++)
+				RTW_PRINT_SEL(m, "%02x", mlmepriv->vendor_ie_filter[i][j]);
+		RTW_PRINT_SEL(m, "\n");
+	}
+
+	return 0;
+}
+
+static ssize_t proc_set_vendor_ie_filter(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = (_adapter *)rtw_netdev_priv(dev);
+	struct mlme_priv *mlmepriv = &adapter->mlmepriv;
+	char tmp[MAX_VENDOR_IE_PARAM_LEN];
+	u8 vendor_ie_num = 0;
+	u8 *vendor_ie;
+	u8 enable;
+	int num, i, len = 0;
+
+	vendor_ie = rtw_malloc(MAX_VENDOR_IE_LEN);
+
+	if (!vendor_ie || !adapter)
+		return -EFAULT;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		if (count > 2) {
+			num = sscanf(tmp, "%hhu %s", &vendor_ie_num, vendor_ie);
+
+			if (num < 1)
+				return -EINVAL;
+
+			if (vendor_ie_num > MAX_VENDOR_IE_NUM - 1)
+				return -EINVAL;
+
+			if (_rtw_memcmp(vendor_ie, "0", 1)) {
+				_rtw_memset(mlmepriv->vendor_ie_filter[vendor_ie_num] , 0 , MAX_VENDOR_IE_LEN);
+				mlmepriv->vendor_ie_len[vendor_ie_num] = 0;
+			}
+
+			for (i = 0; i < strlen(vendor_ie); i += 2) {
+				mlmepriv->vendor_ie_filter[vendor_ie_num][len] = key_2char2num(vendor_ie[i], vendor_ie[i+1]);
+				if (len == 1)
+					mlmepriv->vendor_ie_len[vendor_ie_num] = mlmepriv->vendor_ie_filter[vendor_ie_num][len];
+				len++;
+			}
+			rtw_mfree(vendor_ie, MAX_VENDOR_IE_LEN);
+		} else if (count == 2) {
+			num = sscanf(tmp, "%hhu", &mlmepriv->vendor_ie_filter_enable);
+		} else {
+			RTW_INFO("argument size is wrong\n");
+			return -EINVAL;
+		}
+	} else {
+		return -EFAULT;
+	}
+
+	return count;
+}
+
+int proc_get_dis_beacon_keys(struct seq_file *m, void *v)
+{
+	_list *plist = NULL, *phead = NULL;
+	struct net_device *dev = m->private;
+	struct dis_bcn_key *pdis_bcn_key = NULL;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	struct _ADAPTER_LINK *adapter_link = GET_PRIMARY_LINK(adapter);
+
+	RTW_PRINT_SEL(m, "dump disconnection bcn info\n");
+
+	_rtw_spinlock_bh(&adapter_link->mlmepriv.busy_dis_bcn_queue.lock);
+	phead = get_list_head(&adapter_link->mlmepriv.busy_dis_bcn_queue);
+	plist = get_next(phead);
+
+	while (rtw_end_of_queue_search(phead, plist) == _FALSE) {
+		pdis_bcn_key = LIST_CONTAINOR(plist, struct dis_bcn_key, list);
+		rtw_dump_bcn_keys(m, &pdis_bcn_key->bcn_content);
+		plist = get_next(plist);
+	}
+	_rtw_spinunlock_bh(&adapter_link->mlmepriv.busy_dis_bcn_queue.lock);
+
+	return 0;
+}
+#endif
+
+#ifdef PRIVATE_R
+static ssize_t proc_set_enable_dfs(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	char *alpha2;
+	char tmp[32];
+	u8 enable_dfs = 0;
+
+	if (count < 1) {
+		RTW_INFO("argument size is less than 1\n");
+		return -EFAULT;
+	}
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhx", &enable_dfs);
+
+		if (num >= 1) {
+			struct get_chplan_resp *chplan;
+
+			if (rtw_get_chplan_cmd(adapter, RTW_CMDF_WAIT_ACK, &chplan) == _SUCCESS) {
+				if (!!enable_dfs != !(chplan->confs.dis_ch_flags & RTW_CHF_DFS)) {
+					struct chplan_confs confs = {.set_types = BIT(CHPLAN_CONFS_DIS_CH_FLAGS)};
+
+					if (enable_dfs)
+						confs.dis_ch_flags = chplan->confs.dis_ch_flags & ~RTW_CHF_DFS;
+					else
+						confs.dis_ch_flags = chplan->confs.dis_ch_flags | RTW_CHF_DFS;
+
+					rtw_set_chplan_confs_cmd(adapter, RTW_CMDF_WAIT_ACK, &confs);
+				}
+				rtw_free_get_chplan_resp(chplan);
+			}
+		}
+	}
+
+	return count;
+}
+
+static int proc_get_enable_dfs(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	struct get_chplan_resp *chplan;
+
+	if (rtw_get_chplan_cmd(adapter, RTW_CMDF_WAIT_ACK, &chplan) == _FAIL)
+		goto exit;
+
+	RTW_PRINT_SEL(m, "dfs enabled: %d\n", !(chplan->confs.dis_ch_flags & RTW_CHF_DFS));
+
+	rtw_free_get_chplan_resp(chplan);
+
+exit:
+	return 0;
+}
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+static int proc_get_twt_oper(struct seq_file *m, void *v)
+{
+#if 0
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct registry_priv	*pregpriv = &padapter->registrypriv;
+
+	if (pregpriv) {
+		if (pregpriv->amsdu_mode == RTW_AMSDU_MODE_NON_SPP)
+			RTW_PRINT_SEL(m, "amsdu mode: NON-SPP\n");
+		else if (pregpriv->amsdu_mode == RTW_AMSDU_MODE_SPP)
+			RTW_PRINT_SEL(m, "amsdu mode: SPP\n");
+		else if (pregpriv->amsdu_mode == RTW_AMSDU_MODE_ALL_DROP)
+			RTW_PRINT_SEL(m, "amsdu mode: ALL DROP\n");
+		else
+			RTW_PRINT_SEL(m, "unexpected amsdu mode\n");
+	}
+#endif
+
+	return 0;
+}
+
+static ssize_t proc_set_twt_oper(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct _ADAPTER_LINK *padapter_link = GET_PRIMARY_LINK(padapter);
+	struct sta_priv *pstapriv = &padapter->stapriv;
+	struct link_mlme_ext_priv *pmlmeext = &padapter_link->mlmeextpriv;
+	struct link_mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	WLAN_BSSID_EX *cur_network = &(pmlmeinfo->network);
+	struct sta_info *psta = NULL;
+	char tmp[64], cmd[16] = {0};
+	u32 param_1 = 0, param_2 = 0, param_3 = 0, param_4 = 0, param_5 = 0, param_6 = 0;
+	int num;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	/* check dev cap support for TWT */
+	if ((!TWT_REQ_SUPP(padapter_link->wrlink->protocol_cap.twt)) &&
+	    (!TWT_RSP_SUPP(padapter_link->wrlink->protocol_cap.twt))) {
+		RTW_INFO("[TWT] %s: Dev cap does not support TWT\n", __func__);
+		return count;
+	}
+
+	/* limit to use in STA mode associated to 11AX AP currently */
+	if (!MLME_IS_STA(padapter)) {
+		RTW_INFO("[TWT] %s: Only support to use in STA mode\n", __func__);
+		return count;
+	}
+	if (!MLME_IS_ASOC(padapter)) {
+		RTW_INFO("[TWT] %s: Only support to use when assoc. to AP\n", __func__);
+		return count;
+	}
+	if (padapter_link->mlmepriv.hepriv.he_option == _FALSE) {
+		RTW_INFO("[TWT] %s: Only support to use in 802.11AX mode\n", __func__);
+		return count;
+	}
+
+	psta = rtw_get_stainfo(pstapriv, cur_network->MacAddress);
+	if (psta == NULL)
+		return count;
+	if (psta->phl_sta == NULL)
+		return count;
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+
+		num = sscanf(tmp, "%s %d %d %d %d %d %d", cmd, &param_1, &param_2,
+			     &param_3, &param_4, &param_5, &param_6);
+
+		if (num < 2)
+			return count;
+
+		/* limit to use in wifi test mode or TWT is enabled currently */
+		if (!padapter->registrypriv.wifi_spec && !padapter->registrypriv.twt_en && strcmp(cmd, "enable") != 0) {
+			RTW_INFO("[TWT] %s: TWT is disabled, only support to use in wifi test mode\n", __func__);
+			return count;
+		}
+
+		if (strcmp(cmd, "ctrl") == 0) {
+			rtw_core_twt_set_ctrl(padapter, padapter_link, param_1,
+					      param_2, param_3, 0, 0);
+		} else if (strcmp(cmd, "ireq") == 0) {
+			rtw_core_twt_set_ireq(padapter, padapter_link, 1, param_1,
+					      param_2, param_3, param_4, param_5,
+					      param_6);
+		} else if (strcmp(cmd, "breq") == 0) {
+			rtw_core_twt_set_breq(padapter, padapter_link, 1, param_1,
+					      param_2, param_3, param_4, param_5,
+					      param_6);
+		} else if (strcmp(cmd, "isetup") == 0) {
+			rtw_core_twt_set_isetup(padapter, padapter_link, psta,
+					      param_1, param_2, param_3, param_4);
+		} else if (strcmp(cmd, "bsetup") == 0) {
+			rtw_core_twt_set_bsetup(padapter, padapter_link, psta,
+					      param_1, param_2, param_3, param_4,
+					      param_5);
+		} else if (strcmp(cmd, "itrdwn") == 0) {
+			rtw_core_twt_set_iteardown(padapter, padapter_link, psta,
+					      param_1, param_2, 0);
+		} else if (strcmp(cmd, "btrdwn") == 0) {
+			rtw_core_twt_set_bteardown(padapter, padapter_link, psta,
+					      param_1, param_2, 0);
+		} else if (strcmp(cmd, "suspend") == 0) {
+			rtw_core_twt_set_info(padapter, padapter_link, psta,
+					      _TRUE, param_1);
+		} else if (strcmp(cmd, "resume") == 0) {
+			rtw_core_twt_set_info(padapter, padapter_link, psta,
+					      _FALSE, param_1);
+		} else if (strcmp(cmd, "enable") == 0) {
+			padapter->registrypriv.twt_en = (u8)param_1;
+		}
+	}
+
+	return count;
+}
+#endif
+#endif
+
+#ifdef CONFIG_USB_HCI
+
+static int proc_get_usb_mode(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	PUSB_DATA pusb_data = dvobj_to_usb(dvobj);
+
+	if (pusb_data->usb_speed == RTW_USB_SPEED_LOW)
+		RTW_PRINT_SEL(m, "current usb mode : usb 1.0\n");
+	else if (pusb_data->usb_speed == RTW_USB_SPEED_FULL)
+		RTW_PRINT_SEL(m, "current usb mode : usb 1.1\n");
+	else if (pusb_data->usb_speed == RTW_USB_SPEED_HIGH)
+		RTW_PRINT_SEL(m, "current usb mode : usb 2.0\n");
+	else if (pusb_data->usb_speed == RTW_USB_SPEED_SUPER)
+		RTW_PRINT_SEL(m, "current usb mode : usb 3.0\n");
+	else if (pusb_data->usb_speed == RTW_USB_SPEED_SUPER_10G)
+		RTW_PRINT_SEL(m, "current usb mode : usb 3.1\n");
+	else if (pusb_data->usb_speed == RTW_USB_SPEED_SUPER_20G)
+		RTW_PRINT_SEL(m, "current usb mode : usb 3.2\n");
+
+	return 0;
+}
+
+static ssize_t proc_set_usb_mode(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	struct net_device *dev = data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	PUSB_DATA pusb_data = dvobj_to_usb(dvobj);
+	struct registry_priv *registry_par = &padapter->registrypriv;
+	char tmp[32];
+	u8 usb_type;
+	u8 ret = _FALSE;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (count > sizeof(tmp)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	if (buffer && !copy_from_user(tmp, buffer, count)) {
+		int num = sscanf(tmp, "%hhu", &usb_type);
+
+		if (num != 1) {
+			RTW_ERR("%s: invalid parameter!\n", __FUNCTION__);
+			goto exit;
+		}
+		/*registry not allow driver to switch usb mode*/
+		registry_par->switch_usb_mode = usb_type;
+		if (registry_par->switch_usb_mode == 0)
+			goto exit;
+
+		#ifdef CONFIG_POWER_SAVE
+		if (rtw_hw_is_init_completed(dvobj))
+			rtw_phl_dbg_ps_op_mode(GET_PHL_INFO(dvobj), HW_BAND_0, PS_MODE_LPS, PS_OP_MODE_DISABLED);
+		#endif
+
+		if (registry_par->switch_usb_mode == 1 && pusb_data->usb_speed < RTW_USB_SPEED_SUPER)
+			ret = rtw_phl_cmd_force_usb_switch(dvobj->phl, RTW_USB_SPEED_SUPER, HW_BAND_0, PHL_CMD_DIRECTLY, 0);
+		else if (registry_par->switch_usb_mode == 2 && pusb_data->usb_speed >= RTW_USB_SPEED_SUPER)
+			ret = rtw_phl_cmd_force_usb_switch(dvobj->phl, RTW_USB_SPEED_HIGH, HW_BAND_0, PHL_CMD_DIRECTLY, 0);
+
+	}
+
+exit:
+	return count;
+}
+#endif
+
+/*
+* rtw_adapter_proc:
+* init/deinit when register/unregister net_device
+*/
+const struct rtw_proc_hdl adapter_proc_hdls[] = {
+#if RTW_SEQ_FILE_TEST
+	RTW_PROC_HDL_SEQ("seq_file_test", &seq_file_test, NULL),
+#endif
+	RTW_PROC_HDL_SSEQ("write_reg", NULL, proc_set_write_reg),
+	RTW_PROC_HDL_SSEQ("read_reg", proc_get_read_reg, proc_set_read_reg),
+
+	RTW_PROC_HDL_SSEQ("mac_dbg_status_dump", NULL, proc_set_mac_dbg_status_dump),
+
+	RTW_PROC_HDL_SSEQ("tx_rate_bmp", proc_get_dump_tx_rate_bmp, NULL),
+	RTW_PROC_HDL_SSEQ("adapters_status", proc_get_dump_adapters_status, NULL),
+#ifdef CONFIG_RTW_CUSTOMER_STR
+	RTW_PROC_HDL_SSEQ("customer_str", proc_get_customer_str, NULL),
+#endif
+	RTW_PROC_HDL_SSEQ("fwstate", proc_get_fwstate, NULL),
+	RTW_PROC_HDL_SSEQ("sec_info", proc_get_sec_info, NULL),
+	RTW_PROC_HDL_SSEQ("mlmext_state", proc_get_mlmext_state, NULL),
+	RTW_PROC_HDL_SSEQ("qos_option", proc_get_qos_option, NULL),
+	RTW_PROC_HDL_SSEQ("ht_option", proc_get_ht_option, NULL),
+	RTW_PROC_HDL_SSEQ("rf_info", proc_get_rf_info, NULL),
+	RTW_PROC_HDL_SSEQ("scan_param", proc_get_scan_param, proc_set_scan_param),
+	RTW_PROC_HDL_SSEQ("scan_abort", proc_get_scan_abort, proc_set_scan_abort),
+#ifdef CONFIG_SCAN_BACKOP
+	RTW_PROC_HDL_SSEQ("backop_flags_sta", proc_get_backop_flags_sta, proc_set_backop_flags_sta),
+	#ifdef CONFIG_AP_MODE
+	RTW_PROC_HDL_SSEQ("backop_flags_ap", proc_get_backop_flags_ap, proc_set_backop_flags_ap),
+	#endif
+	#ifdef CONFIG_RTW_MESH
+	RTW_PROC_HDL_SSEQ("backop_flags_mesh", proc_get_backop_flags_mesh, proc_set_backop_flags_mesh),
+	#endif
+#endif
+	RTW_PROC_HDL_SSEQ("survey_info", proc_get_survey_info, proc_set_survey_info),
+	RTW_PROC_HDL_SSEQ("ap_info", proc_get_ap_info, NULL),
+#ifdef PRIVATE_R
+	RTW_PROC_HDL_SSEQ("infra_ap", proc_get_infra_ap, NULL),
+#endif /* PRIVATE_R */
+	RTW_PROC_HDL_SSEQ("trx_info", proc_get_trx_info, proc_reset_trx_info),
+	RTW_PROC_HDL_SSEQ("tx_power_offset", proc_get_tx_power_offset, proc_set_tx_power_offset),
+	RTW_PROC_HDL_SSEQ("rate_ctl", proc_get_rate_ctl, proc_set_rate_ctl),
+	RTW_PROC_HDL_SSEQ("bw_ctl", proc_get_bw_ctl, proc_set_bw_ctl),
+	RTW_PROC_HDL_SSEQ("mac_qinfo", proc_get_mac_qinfo, NULL),
+	/*RTW_PROC_HDL_SSEQ("macid_info", proc_get_macid_info, NULL), */
+	/* RTW_PROC_HDL_SSEQ("bcmc_info", proc_get_mi_ap_bc_info, NULL), */
+	/*RTW_PROC_HDL_SSEQ("sec_cam", proc_get_sec_cam, proc_set_sec_cam),*/
+	/*RTW_PROC_HDL_SSEQ("sec_cam_cache", proc_get_sec_cam_cache, NULL),*/
+#ifdef CONFIG_DBG_AX_CAM
+	RTW_PROC_HDL_SSEQ("dump_ax_valid_key", proc_get_ax_valid_key, NULL),
+	RTW_PROC_HDL_SSEQ("dump_ax_address_cam", proc_get_ax_address_cam, NULL),
+	RTW_PROC_HDL_SSEQ("dump_ax_security_cam", proc_get_ax_sec_cam, NULL),
+#endif
+
+	RTW_PROC_HDL_SSEQ("ps_dbg_info", proc_get_ps_dbg_info, proc_set_ps_dbg_info),
+	RTW_PROC_HDL_SSEQ("wifi_spec", proc_get_wifi_spec, NULL),
+#ifdef CONFIG_LAYER2_ROAMING
+	RTW_PROC_HDL_SSEQ("roam_flags", proc_get_roam_flags, proc_set_roam_flags),
+	RTW_PROC_HDL_SSEQ("roam_param", proc_get_roam_param, proc_set_roam_param),
+	RTW_PROC_HDL_SSEQ("roam_tgt_addr", NULL, proc_set_roam_tgt_addr),
+#endif /* CONFIG_LAYER2_ROAMING */
+#ifdef CONFIG_RTW_MBO
+	RTW_PROC_HDL_SSEQ("non_pref_ch", rtw_mbo_proc_non_pref_chans_get, rtw_mbo_proc_non_pref_chans_set),
+	RTW_PROC_HDL_SSEQ("cell_data", rtw_mbo_proc_cell_data_get, rtw_mbo_proc_cell_data_set),
+	RTW_PROC_HDL_SSEQ("mbo_attr", rtw_mbo_proc_attr_get, rtw_mbo_proc_attr_set),
+#endif
+#ifdef CONFIG_RTW_80211R
+	RTW_PROC_HDL_SSEQ("ft_flags", rtw_ft_proc_flags_get, rtw_ft_proc_flags_set),
+#endif
+	RTW_PROC_HDL_SSEQ("defs_param", proc_get_defs_param, proc_set_defs_param),
+#ifdef CONFIG_SDIO_HCI
+	RTW_PROC_HDL_SSEQ("sd_f0_reg_dump", proc_get_sd_f0_reg_dump, NULL),
+	RTW_PROC_HDL_SSEQ("sdio_local_reg_dump", proc_get_sdio_local_reg_dump, NULL),
+	RTW_PROC_HDL_SSEQ("sdio_card_info", proc_get_sdio_card_info, NULL),
+#ifdef DBG_SDIO
+	RTW_PROC_HDL_SSEQ("sdio_dbg", proc_get_sdio_dbg, proc_set_sdio_dbg),
+#endif /* DBG_SDIO */
+#endif /* CONFIG_SDIO_HCI */
+
+#ifdef CONFIG_RTW_IO_RECORDS
+	RTW_PROC_HDL_SEQ("io_records", &seq_ops_io_records, proc_set_io_records),
+	RTW_PROC_HDL_SEQ("io_records_tab", &seq_ops_io_records_tab, proc_set_io_records),
+	RTW_PROC_HDL_SSEQ("io_records_enabled", proc_get_io_records_enabled, NULL),
+	RTW_PROC_HDL_SSEQ("io_records_summary", proc_get_io_records_summary, NULL),
+#endif
+
+	RTW_PROC_HDL_SSEQ("del_rx_ampdu_test_case", NULL, proc_set_del_rx_ampdu_test_case),
+	RTW_PROC_HDL_SSEQ("wait_hiq_empty", NULL, proc_set_wait_hiq_empty),
+	RTW_PROC_HDL_SSEQ("sta_linking_test", NULL, proc_set_sta_linking_test),
+#ifdef CONFIG_AP_MODE
+	RTW_PROC_HDL_SSEQ("ap_linking_test", NULL, proc_set_ap_linking_test),
+#endif
+
+	RTW_PROC_HDL_SSEQ("mac_reg_dump", proc_get_mac_reg_dump, NULL),
+	RTW_PROC_HDL_SSEQ("bb_reg_dump", proc_get_bb_reg_dump, NULL),
+	RTW_PROC_HDL_SSEQ("bb_reg_dump_ex", proc_get_bb_reg_dump_ex, NULL),
+	RTW_PROC_HDL_SSEQ("rf_reg_dump", proc_get_rf_reg_dump, NULL),
+
+#if 0
+	RTW_PROC_HDL_SSEQ("led_config", proc_get_led_config, proc_set_led_config),
+#endif
+
+#ifdef CONFIG_POWER_SAVE
+	RTW_PROC_HDL_SSEQ("ps_info", proc_get_ps_info, proc_set_ps_info),
+#if defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW)
+	RTW_PROC_HDL_SSEQ("ps_dtim", NULL, proc_set_ps_dtim),
+#endif /*defined(CONFIG_RTW_LPS) || defined(CONFIG_RTW_LPS_WOW) */
+#endif /* CONFIG_POWER_SAVE */
+
+#ifdef CONFIG_AP_MODE
+	RTW_PROC_HDL_SSEQ("aid_status", proc_get_aid_status, proc_set_aid_status),
+	RTW_PROC_HDL_SSEQ("sta_active_time", proc_get_sta_active_time, NULL),
+	RTW_PROC_HDL_SSEQ("all_sta_info", proc_get_all_sta_info, NULL),
+	RTW_PROC_HDL_SSEQ("bmc_tx_rate", proc_get_bmc_tx_rate, proc_set_bmc_tx_rate),
+	#if CONFIG_RTW_AP_DATA_BMC_TO_UC
+	RTW_PROC_HDL_SSEQ("ap_b2u_flags", proc_get_ap_b2u_flags, proc_set_ap_b2u_flags),
+	#endif
+#endif /* CONFIG_AP_MODE */
+
+#ifdef DBG_MEMORY_LEAK
+	RTW_PROC_HDL_SSEQ("_malloc_cnt", proc_get_malloc_cnt, NULL),
+#endif /* DBG_MEMORY_LEAK */
+
+#ifdef CONFIG_FIND_BEST_CHANNEL
+	RTW_PROC_HDL_SSEQ("best_channel", proc_get_best_channel, proc_set_best_channel),
+#endif
+
+	RTW_PROC_HDL_SSEQ("rx_signal", proc_get_rx_signal, proc_set_rx_signal),
+	RTW_PROC_HDL_SSEQ("rx_chk_limit", proc_get_rx_chk_limit, proc_set_rx_chk_limit),
+	RTW_PROC_HDL_SSEQ("hw_info", proc_get_hw_status, proc_set_hw_status),
+	RTW_PROC_HDL_SSEQ("mac_rptbuf", proc_get_mac_rptbuf, NULL),
+#ifdef CONFIG_80211N_HT
+	RTW_PROC_HDL_SSEQ("ht_enable", proc_get_ht_enable, proc_set_ht_enable),
+	RTW_PROC_HDL_SSEQ("bw_mode", proc_get_bw_mode, proc_set_bw_mode),
+	RTW_PROC_HDL_SSEQ("ampdu_enable", proc_get_ampdu_enable, proc_set_ampdu_enable),
+	RTW_PROC_HDL_SSEQ("rx_ampdu", proc_get_rx_ampdu, proc_set_rx_ampdu),
+	RTW_PROC_HDL_SSEQ("rx_ampdu_size_limit", proc_get_rx_ampdu_size_limit, proc_set_rx_ampdu_size_limit),
+	RTW_PROC_HDL_SSEQ("rx_ampdu_factor", proc_get_rx_ampdu_factor, proc_set_rx_ampdu_factor),
+	RTW_PROC_HDL_SSEQ("rx_ampdu_density", proc_get_rx_ampdu_density, proc_set_rx_ampdu_density),
+	RTW_PROC_HDL_SSEQ("tx_ampdu_density", proc_get_tx_ampdu_density, proc_set_tx_ampdu_density),
+	RTW_PROC_HDL_SSEQ("tx_ampdu_num", proc_get_tx_ampdu_num, proc_set_tx_ampdu_num),
+	RTW_PROC_HDL_SSEQ("tx_quick_addba_req", proc_get_tx_quick_addba_req, proc_set_tx_quick_addba_req),
+#ifdef CONFIG_TX_AMSDU
+	RTW_PROC_HDL_SSEQ("tx_amsdu", proc_get_tx_amsdu, proc_set_tx_amsdu),
+	RTW_PROC_HDL_SSEQ("tx_amsdu_rate", proc_get_tx_amsdu_rate, proc_set_tx_amsdu_rate),
+#endif
+	RTW_PROC_HDL_SSEQ("tx_max_agg_time", proc_get_tx_max_agg_time, proc_set_tx_max_agg_time),
+#endif /* CONFIG_80211N_HT */
+
+#ifdef CONFIG_80211AC_VHT
+	RTW_PROC_HDL_SSEQ("vht_24g_enable", proc_get_vht_24g_enable, proc_set_vht_24g_enable),
+#endif /* CONFIG_80211AC_VHT */
+
+	RTW_PROC_HDL_SSEQ("dynamic_rrsr", proc_get_dyn_rrsr, proc_set_dyn_rrsr),
+
+	/* RTW_PROC_HDL_SSEQ("path_rssi", proc_get_two_path_rssi, NULL),
+	* 	RTW_PROC_HDL_SSEQ("rssi_disp",proc_get_rssi_disp, proc_set_rssi_disp), */
+
+	RTW_PROC_HDL_SSEQ("trx_info_debug", proc_get_trx_info_debug, NULL),
+
+#ifdef CONFIG_HUAWEI_PROC
+	RTW_PROC_HDL_SSEQ("huawei_trx_info", proc_get_huawei_trx_info, NULL),
+#endif
+	RTW_PROC_HDL_SSEQ("rtw_hal_linked_info_dump", proc_get_linked_info_dump, proc_set_linked_info_dump),
+	RTW_PROC_HDL_SSEQ("sta_tp_dump", proc_get_sta_tp_dump, proc_set_sta_tp_dump),
+	RTW_PROC_HDL_SSEQ("sta_tp_info", proc_get_sta_tp_info, NULL),
+	RTW_PROC_HDL_SSEQ("dis_turboedca", proc_get_turboedca_ctrl, proc_set_turboedca_ctrl),
+	RTW_PROC_HDL_SSEQ("dis_rt_cap", proc_get_rt_cap_ctrl, proc_set_rt_cap_ctrl),
+	RTW_PROC_HDL_SSEQ("tx_info_msg", proc_get_tx_info_msg, NULL),
+	RTW_PROC_HDL_SSEQ("rx_info_msg", proc_get_rx_info_msg, proc_set_rx_info_msg),
+	RTW_PROC_HDL_SSEQ("false_alarm_accumulated",
+			  proc_get_false_alarm_accumulated,
+			  proc_set_false_alarm_accumulated),
+#ifdef PRIVATE_R
+	RTW_PROC_HDL_SSEQ("roku_trx_info_msg", proc_get_roku_trx_info_msg, NULL),
+#endif
+
+#ifdef CONFIG_GPIO_API
+	RTW_PROC_HDL_SSEQ("gpio_info", proc_get_gpio, proc_set_gpio),
+	RTW_PROC_HDL_SSEQ("gpio_set_output_value", NULL, proc_set_gpio_output_value),
+	RTW_PROC_HDL_SSEQ("gpio_set_direction", NULL, proc_set_config_gpio),
+#endif
+
+#ifdef CONFIG_DBG_COUNTER
+	RTW_PROC_HDL_SSEQ("rx_logs", proc_get_rx_logs, NULL),
+	RTW_PROC_HDL_SSEQ("tx_logs", proc_get_tx_logs, NULL),
+	RTW_PROC_HDL_SSEQ("int_logs", proc_get_int_logs, NULL),
+#endif
+
+#ifdef CONFIG_DBG_RF_CAL
+	RTW_PROC_HDL_SSEQ("iqk", proc_get_iqk_info, proc_set_iqk),
+	RTW_PROC_HDL_SSEQ("lck", proc_get_lck_info, proc_set_lck),
+#endif
+
+#ifdef CONFIG_PCI_HCI
+	RTW_PROC_HDL_SSEQ("rx_ring", proc_get_rx_ring, NULL),
+	RTW_PROC_HDL_SSEQ("tx_ring", proc_get_tx_ring, NULL),
+#ifdef DBG_TXBD_DESC_DUMP
+	RTW_PROC_HDL_SSEQ("tx_ring_ext", proc_get_tx_ring_ext, proc_set_tx_ring_ext),
+#endif
+	RTW_PROC_HDL_SSEQ("pci_aspm", proc_get_pci_aspm, NULL),
+
+	RTW_PROC_HDL_SSEQ("pci_conf_space", proc_get_pci_conf_space, proc_set_pci_conf_space),
+
+	RTW_PROC_HDL_SSEQ("pci_bridge_conf_space", proc_get_pci_bridge_conf_space, proc_set_pci_bridge_conf_space),
+
+#endif
+
+#ifdef CONFIG_WOWLAN
+	RTW_PROC_HDL_SSEQ("wow_enable", proc_get_wow_enable, proc_set_wow_enable),
+	RTW_PROC_HDL_SSEQ("wow_pattern_info", proc_get_pattern_info, proc_set_pattern_info),
+	RTW_PROC_HDL_SSEQ("wow_wakeup_event", proc_get_wakeup_event,
+			  proc_set_wakeup_event),
+	RTW_PROC_HDL_SSEQ("wowlan_last_wake_reason", proc_get_wakeup_reason, NULL),
+	RTW_PROC_HDL_SSEQ("wow_pattern_cam", proc_dump_pattern_cam, NULL),
+	RTW_PROC_HDL_SSEQ("wow_wake_indication", proc_get_wake_indication, NULL),
+#ifdef CONFIG_GPIO_WAKEUP
+	RTW_PROC_HDL_SSEQ("wowlan_gpio_info", proc_get_wowlan_gpio_info, proc_set_wowlan_gpio_info),
+	RTW_PROC_HDL_SSEQ("wow_gpio_duration", proc_get_wow_gpio_duration,
+			proc_set_wow_gpio_duration),
+#endif
+#ifdef CONFIG_WOW_PERIODIC_WAKE
+	RTW_PROC_HDL_SSEQ("wow_pw_info", proc_get_wow_pw_info, proc_set_wow_pw_info),
+#endif
+#ifdef CONFIG_MDNS_OFFLOAD
+	RTW_PROC_HDL_SSEQ("wow_mdns_resp", proc_get_wow_mdns_resp, proc_set_wow_mdns_resp),
+	RTW_PROC_HDL_SSEQ("wow_mdns_match_criteria", proc_get_wow_mdns_match_criteria, proc_set_wow_mdns_match_criteria),
+	RTW_PROC_HDL_SSEQ("wow_mdns_passthru_list", proc_get_wow_mdns_passthru_list, proc_set_wow_mdns_passthru_list),
+	RTW_PROC_HDL_SSEQ("wow_mdns_offload_state", proc_get_wow_mdns_offload_state, proc_set_wow_mdns_offload_state),
+	RTW_PROC_HDL_SSEQ("wow_mdns_passthru_behavior", proc_get_wow_mdns_passthru_behavior, proc_set_wow_mdns_passthru_behavior),
+#endif
+#ifdef CONFIG_WOW_APF
+	RTW_PROC_HDL_SSEQ("wow_apf", proc_get_wow_apf, proc_set_wow_apf),
+#endif
+#endif /* CONFIG_WOWLAN */
+
+
+#ifdef CONFIG_P2P_WOWLAN
+	RTW_PROC_HDL_SSEQ("p2p_wowlan_info", proc_get_p2p_wowlan_info, NULL),
+#endif
+	RTW_PROC_HDL_SSEQ("country_code", proc_get_country_code, proc_set_country_code),
+#if CONFIG_IEEE80211_BAND_6GHZ
+	RTW_PROC_HDL_SSEQ("env", proc_get_env, proc_set_env),
+#endif
+	RTW_PROC_HDL_SSEQ("chan_plan", proc_get_chan_plan, proc_set_chan_plan),
+	RTW_PROC_HDL_SSEQ("chplan_confs", proc_get_chplan_confs, proc_set_chplan_confs),
+	RTW_PROC_HDL_SSEQ("cap_spt_op_class_ch", proc_get_cap_spt_op_class_ch, proc_set_cap_spt_op_class_ch),
+	RTW_PROC_HDL_SSEQ("reg_spt_op_class_ch", proc_get_reg_spt_op_class_ch, proc_set_reg_spt_op_class_ch),
+	RTW_PROC_HDL_SSEQ("cur_spt_op_class_ch", proc_get_cur_spt_op_class_ch, proc_set_cur_spt_op_class_ch),
+#if CONFIG_RTW_MACADDR_ACL
+	RTW_PROC_HDL_SSEQ("macaddr_acl", proc_get_macaddr_acl, proc_set_macaddr_acl),
+#endif
+#if CONFIG_RTW_PRE_LINK_STA
+	RTW_PROC_HDL_SSEQ("pre_link_sta", proc_get_pre_link_sta, proc_set_pre_link_sta),
+#endif
+	RTW_PROC_HDL_SSEQ("ch_sel_policy", proc_get_ch_sel_policy, proc_set_ch_sel_policy),
+#ifdef CONFIG_80211D
+	RTW_PROC_HDL_SSEQ("country_ie_slave_records", proc_get_country_ie_slave_records, NULL),
+#endif
+#ifdef CONFIG_DFS_MASTER
+	RTW_PROC_HDL_SSEQ("dfs_test_case", proc_get_dfs_test_case, proc_set_dfs_test_case),
+	RTW_PROC_HDL_SSEQ("update_non_ocp", NULL, proc_set_update_non_ocp),
+	RTW_PROC_HDL_SSEQ("radar_detect", NULL, proc_set_radar_detect),
+	RTW_PROC_HDL_SSEQ("dfs_ch_sel_e_flags", proc_get_dfs_ch_sel_e_flags, proc_set_dfs_ch_sel_e_flags),
+	RTW_PROC_HDL_SSEQ("dfs_ch_sel_d_flags", proc_get_dfs_ch_sel_d_flags, proc_set_dfs_ch_sel_d_flags),
+	#if CONFIG_DFS_SLAVE_WITH_RADAR_DETECT
+	RTW_PROC_HDL_SSEQ("dfs_slave_with_rd", proc_get_dfs_slave_with_rd, proc_set_dfs_slave_with_rd),
+	#endif
+#endif
+#ifdef CONFIG_BCN_CNT_CONFIRM_HDL
+	RTW_PROC_HDL_SSEQ("new_bcn_max", proc_get_new_bcn_max, proc_set_new_bcn_max),
+#endif
+	RTW_PROC_HDL_SSEQ("sink_udpport", proc_get_udpport, proc_set_udpport),
+#ifdef DBG_RX_COUNTER_DUMP
+	RTW_PROC_HDL_SSEQ("dump_rx_cnt_mode", proc_get_rx_cnt_dump, proc_set_rx_cnt_dump),
+#endif
+	RTW_PROC_HDL_SSEQ("change_bss_chbw", NULL, proc_set_change_bss_chbw),
+	RTW_PROC_HDL_SSEQ("tx_bw_mode", proc_get_tx_bw_mode, proc_set_tx_bw_mode),
+
+#ifdef CONFIG_ECSA_PHL
+	RTW_PROC_HDL_SSEQ("ecsa_allow_reason", proc_get_ecsa_allow_reason, proc_set_ecsa_allow_reason),
+	#ifdef PRIVATE_R
+	RTW_PROC_HDL_SSEQ("p2p_park_ch", proc_get_p2p_park_ch, proc_set_p2p_park_ch),
+	#endif
+#endif
+
+	RTW_PROC_HDL_SSEQ("target_tx_power", proc_get_target_tx_power, NULL),
+	RTW_PROC_HDL_SSEQ("tx_power_by_rate", proc_get_tx_power_by_rate, NULL),
+#if CONFIG_TXPWR_LIMIT
+	RTW_PROC_HDL_SSEQ("tx_power_limit", proc_get_tx_power_limit, NULL),
+	#ifdef CONFIG_80211AX_HE
+	RTW_PROC_HDL_SSEQ("tx_power_limit_ru", proc_get_tx_power_limit_ru, NULL),
+	#endif
+#endif
+	RTW_PROC_HDL_SSEQ("tpc_settings", proc_get_tpc_settings, proc_set_tpc_settings),
+	RTW_PROC_HDL_SSEQ("tx_power_ext_info", proc_get_tx_power_ext_info, proc_set_tx_power_ext_info),
+#ifdef GEORGIA_TODO_TX_PWR
+	RTW_PROC_HDL_SEQ("tx_power_idx", &seq_ops_tx_power_idx, NULL),
+	RTW_PROC_HDL_SEQ("txpwr_total_dbm", &seq_ops_txpwr_total_dbm, NULL),
+#endif
+#ifdef CONFIG_TDLS
+	RTW_PROC_HDL_SSEQ("tdls_info", proc_get_tdls_info, NULL),
+	RTW_PROC_HDL_SSEQ("tdls_enable", proc_get_tdls_enable, proc_set_tdls_enable),
+#endif
+	RTW_PROC_HDL_SSEQ("monitor", proc_get_monitor, proc_set_monitor),
+
+#ifdef CONFIG_RTW_ACS
+	RTW_PROC_HDL_SSEQ("chan_info", proc_get_chan_info, NULL),
+#ifdef PRIVATE_R
+	RTW_PROC_HDL_SSEQ("all_chan_load", proc_get_all_chan_load, NULL),
+#endif
+#ifdef WKARD_ACS
+	RTW_PROC_HDL_SSEQ("acs", proc_get_best_chan, proc_set_acs),
+#endif
+#endif
+	RTW_PROC_HDL_SSEQ("env_info", proc_get_env_rpt, NULL),
+	RTW_PROC_HDL_SSEQ("hal_spec", proc_get_hal_spec, NULL),
+	RTW_PROC_HDL_SSEQ("hal_trx_mode", proc_get_hal_trx_mode, NULL),
+	RTW_PROC_HDL_SSEQ("hal_txpwr_info", proc_get_hal_txpwr_info, NULL),
+
+	RTW_PROC_HDL_SSEQ("efuse_map", proc_get_efuse_map, NULL),
+#ifdef CONFIG_IEEE80211W
+	RTW_PROC_HDL_SSEQ("11w_tx_sa_query", proc_get_tx_sa_query, proc_set_tx_sa_query),
+	RTW_PROC_HDL_SSEQ("11w_tx_deauth", proc_get_tx_deauth, proc_set_tx_deauth),
+	RTW_PROC_HDL_SSEQ("11w_tx_auth", proc_get_tx_auth, proc_set_tx_auth),
+#endif /* CONFIG_IEEE80211W */
+
+	RTW_PROC_HDL_SSEQ("mac_addr", proc_get_mac_addr, NULL),
+	RTW_PROC_HDL_SSEQ("skip_band", proc_get_skip_band, proc_set_skip_band),
+
+	RTW_PROC_HDL_SSEQ("rx_stat", proc_get_rx_stat, NULL),
+
+	RTW_PROC_HDL_SSEQ("tx_stat", proc_get_tx_stat, NULL),
+	RTW_PROC_HDL_SSEQ("sta_tx_stat", proc_get_sta_tx_stat, proc_set_sta_tx_stat),
+	/**** PHY Capability ****/
+	RTW_PROC_HDL_SSEQ("phy_cap", proc_get_phy_cap, NULL),
+#ifdef CONFIG_80211N_HT
+	RTW_PROC_HDL_SSEQ("stbc_cap", proc_get_stbc_cap, proc_set_stbc_cap),
+	RTW_PROC_HDL_SSEQ("ldpc_cap", proc_get_ldpc_cap, proc_set_ldpc_cap),
+#endif /* CONFIG_80211N_HT */
+#ifdef CONFIG_BEAMFORMING
+	RTW_PROC_HDL_SSEQ("txbf_cap", proc_get_txbf_cap, proc_set_txbf_cap),
+#endif
+
+	RTW_PROC_HDL_SSEQ("napi_info", proc_get_napi_info, NULL),
+#ifdef CONFIG_RTW_NAPI_DYNAMIC
+	RTW_PROC_HDL_SSEQ("napi_th", proc_get_napi_info, proc_set_napi_th),
+#endif /* CONFIG_RTW_NAPI_DYNAMIC */
+
+	RTW_PROC_HDL_SSEQ("rsvd_page", proc_dump_rsvd_page, proc_set_rsvd_page_info),
+
+#ifdef CONFIG_SUPPORT_FIFO_DUMP
+	RTW_PROC_HDL_SSEQ("fifo_dump", proc_dump_fifo, proc_set_fifo_info),
+#endif
+
+#ifdef CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST
+	RTW_PROC_HDL_SSEQ("ignore_go_and_low_rssi_in_scan",
+		proc_get_ignore_go_and_low_rssi_in_scan,
+		proc_set_ignore_go_and_low_rssi_in_scan),
+#endif /*CONFIG_IGNORE_GO_AND_LOW_RSSI_IN_SCAN_LIST*/
+
+#ifdef DBG_XMIT_BLOCK
+	RTW_PROC_HDL_SSEQ("xmit_block", proc_get_xmit_block, proc_set_xmit_block),
+#endif
+
+	RTW_PROC_HDL_SSEQ("ack_timeout", proc_get_ack_timeout, proc_set_ack_timeout),
+
+	RTW_PROC_HDL_SSEQ("dynamic_agg_enable", proc_get_dynamic_agg_enable, proc_set_dynamic_agg_enable),
+
+	RTW_PROC_HDL_SSEQ("fw_info", proc_get_fw_info, NULL),
+	RTW_PROC_HDL_SSEQ("fw_offload", proc_get_fw_offload, proc_set_fw_offload),
+
+#ifdef CONFIG_RTW_WDS
+	RTW_PROC_HDL_SSEQ("wds_en", proc_get_wds_en, proc_set_wds_en),
+	RTW_PROC_HDL_SSEQ("sta_wds_en", NULL, proc_set_sta_wds_en),
+	RTW_PROC_HDL_SSEQ("wds_gptr", proc_get_wds_gptr, NULL),
+	#ifdef CONFIG_AP_MODE
+	RTW_PROC_HDL_SSEQ("wds_path", proc_get_wds_path, NULL),
+	#endif
+#endif
+
+#ifdef CONFIG_RTW_MULTI_AP
+	RTW_PROC_HDL_SSEQ("multi_ap_opmode", proc_get_multi_ap_opmode, proc_set_multi_ap_opmode),
+	RTW_PROC_HDL_SSEQ("unassoc_sta", proc_get_unassoc_sta, proc_set_unassoc_sta),
+#ifdef CONFIG_IOCTL_CFG80211
+	RTW_PROC_HDL_SSEQ("sta_assoc_req_frame_body", proc_get_sta_assoc_req_frame_body, proc_set_sta_assoc_req_frame_body),
+#endif
+	RTW_PROC_HDL_SSEQ("ch_util_threshold", proc_get_ch_util_threshold, proc_set_ch_util_threshold),
+	RTW_PROC_HDL_SSEQ("ch_utilization", proc_get_ch_utilization, NULL),
+#endif
+
+#ifdef CONFIG_RTW_MESH
+	#if CONFIG_RTW_MESH_ACNODE_PREVENT
+	RTW_PROC_HDL_SSEQ("mesh_acnode_prevent", proc_get_mesh_acnode_prevent, proc_set_mesh_acnode_prevent),
+	#endif
+	#if CONFIG_RTW_MESH_OFFCH_CAND
+	RTW_PROC_HDL_SSEQ("mesh_offch_cand", proc_get_mesh_offch_cand, proc_set_mesh_offch_cand),
+	#endif
+	#if CONFIG_RTW_MESH_PEER_BLACKLIST
+	RTW_PROC_HDL_SSEQ("mesh_peer_blacklist", proc_get_mesh_peer_blacklist, proc_set_mesh_peer_blacklist),
+	#endif
+	#if CONFIG_RTW_MESH_CTO_MGATE_BLACKLIST
+	RTW_PROC_HDL_SSEQ("mesh_cto_mgate_require", proc_get_mesh_cto_mgate_require, proc_set_mesh_cto_mgate_require),
+	RTW_PROC_HDL_SSEQ("mesh_cto_mgate_blacklist", proc_get_mesh_cto_mgate_blacklist, proc_set_mesh_cto_mgate_blacklist),
+	#endif
+	RTW_PROC_HDL_SSEQ("mesh_peer_sel_policy", proc_get_mesh_peer_sel_policy, NULL),
+	RTW_PROC_HDL_SSEQ("mesh_networks", proc_get_mesh_networks, NULL),
+	RTW_PROC_HDL_SSEQ("mesh_plink_ctl", proc_get_mesh_plink_ctl, NULL),
+	RTW_PROC_HDL_SSEQ("mesh_mpath", proc_get_mesh_mpath, NULL),
+	RTW_PROC_HDL_SSEQ("mesh_mpp", proc_get_mesh_mpp, NULL),
+	RTW_PROC_HDL_SSEQ("mesh_known_gates", proc_get_mesh_known_gates, NULL),
+	#if CONFIG_RTW_MESH_DATA_BMC_TO_UC
+	RTW_PROC_HDL_SSEQ("mesh_b2u_flags", proc_get_mesh_b2u_flags, proc_set_mesh_b2u_flags),
+	#endif
+	RTW_PROC_HDL_SSEQ("mesh_stats", proc_get_mesh_stats, NULL),
+	RTW_PROC_HDL_SSEQ("mesh_gate_timeout_factor", proc_get_mesh_gate_timeout, proc_set_mesh_gate_timeout),
+	RTW_PROC_HDL_SSEQ("mesh_gate_state", proc_get_mesh_gate_state, NULL),
+	RTW_PROC_HDL_SSEQ("mesh_peer_alive_based_preq", proc_get_peer_alive_based_preq, proc_set_peer_alive_based_preq),
+#endif
+
+#ifdef CONFIG_SUPPORT_STATIC_SMPS
+	RTW_PROC_HDL_SSEQ("smps", proc_get_smps, proc_set_smps),
+#endif
+
+#ifdef RTW_BUSY_DENY_SCAN
+	RTW_PROC_HDL_SSEQ("scan_interval_thr", proc_get_scan_interval_thr, \
+			  proc_set_scan_interval_thr),
+#endif
+	RTW_PROC_HDL_SSEQ("scan_deny", proc_get_scan_deny, proc_set_scan_deny),
+
+#ifdef CONFIG_CTRL_TXSS_BY_TP
+	RTW_PROC_HDL_SSEQ("txss_tp", proc_get_txss_tp, proc_set_txss_tp),
+	#ifdef DBG_CTRL_TXSS
+	RTW_PROC_HDL_SSEQ("txss_ctrl", proc_get_txss_ctrl, proc_set_txss_ctrl),
+	#endif
+#endif
+
+	RTW_PROC_HDL_SSEQ("cur_beacon_keys", proc_get_cur_beacon_keys, NULL),
+
+	RTW_PROC_HDL_SSEQ("chan", proc_get_chan, proc_set_chan),
+
+	RTW_PROC_HDL_SSEQ("mr_test", proc_get_mr_test, proc_set_mr_test),
+
+	RTW_PROC_HDL_SSEQ("deny_legacy", proc_get_deny_legacy, proc_set_deny_legacy),
+#if defined(PRIVATE_R) && defined(CONFIG_P2P)
+	RTW_PROC_HDL_SSEQ("go_hidden_ssid_mode", proc_get_go_hidden_ssid_mode, proc_set_go_hidden_ssid_mode),
+#endif
+	RTW_PROC_HDL_SSEQ("rtw_amsdu_mode", proc_get_amsdu_mode, proc_set_amsdu_mode),
+#ifdef CONFIG_80211AX_HE
+	RTW_PROC_HDL_SSEQ("tx_ul_mu_disable", proc_get_tx_ul_mu_disable, proc_set_tx_ul_mu_disable),
+#endif
+#ifdef PRIVATE_R
+	RTW_PROC_HDL_SSEQ("vendor_ie_filter", proc_get_vendor_ie_filter, proc_set_vendor_ie_filter),
+	RTW_PROC_HDL_SSEQ("dis_beacon_keys", proc_get_dis_beacon_keys, NULL),
+#endif
+#ifdef RTW_DETECT_HANG
+	RTW_PROC_HDL_SSEQ("hang_info", proc_get_hang_info, proc_set_hang_info),
+#endif
+	RTW_PROC_HDL_SSEQ("disconnect_info", proc_get_disconnect_info,
+			  proc_set_disconnect_info),
+#ifdef PRIVATE_R
+	RTW_PROC_HDL_SSEQ("enable_dfs", proc_get_enable_dfs, proc_set_enable_dfs),
+#endif
+	RTW_PROC_HDL_SSEQ("vcs_registry", proc_get_vcs, proc_set_vcs),
+
+#ifdef CONFIG_RTW_CSI_CHANNEL_INFO
+	RTW_PROC_HDL_SSEQ("csi_ch_info", proc_get_csi_ch_info, proc_set_csi_ch_info),
+#endif
+
+#ifdef CONFIG_80211AX_HE
+#ifdef CONFIG_TWT
+	RTW_PROC_HDL_SSEQ("twt_oper", proc_get_twt_oper, proc_set_twt_oper),
+#endif
+#endif
+#ifdef CONFIG_SELF_DIAG_INFO
+	RTW_PROC_HDL_SSEQ("self_diag_info", proc_get_self_diag_info, proc_set_self_diag_info),
+#endif
+#ifdef CONFIG_USB_HCI
+	RTW_PROC_HDL_SSEQ("usb_mode", proc_get_usb_mode, proc_set_usb_mode),
+#endif
+	RTW_PROC_HDL_SSEQ("cur_tsf", proc_get_cur_tsf, NULL),
+	RTW_PROC_HDL_SSEQ("cur_tsf_cmd_enqueue", proc_get_cur_tsf_cmd_enqueue, proc_set_cur_tsf_cmd_enqueue),
+};
+
+const int adapter_proc_hdls_num = sizeof(adapter_proc_hdls) / sizeof(struct rtw_proc_hdl);
+
+static int rtw_adapter_proc_open(struct inode *inode, struct file *file)
+{
+	ssize_t index = (ssize_t)PDE_DATA(inode);
+	const struct rtw_proc_hdl *hdl = adapter_proc_hdls + index;
+	void *private = proc_get_parent_data(inode);
+
+	if (hdl->type == RTW_PROC_HDL_TYPE_SEQ) {
+		int res = seq_open(file, hdl->u.seq_op);
+
+		if (res == 0)
+			((struct seq_file *)file->private_data)->private = private;
+
+		return res;
+	} else if (hdl->type == RTW_PROC_HDL_TYPE_SSEQ) {
+		int (*show)(struct seq_file *, void *) = hdl->u.show ? hdl->u.show : proc_get_dummy;
+
+		return single_open(file, show, private);
+	} else if (hdl->type == RTW_PROC_HDL_TYPE_SZSEQ) {
+		int (*show)(struct seq_file *, void *) = hdl->u.sz.show ? hdl->u.sz.show : proc_get_dummy;
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+		return single_open_size(file, show, private, hdl->u.sz.size);
+		#else
+		return single_open(file, show, private);
+		#endif
+	} else {
+		return -EROFS;
+	}
+}
+
+static ssize_t rtw_adapter_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos)
+{
+	ssize_t index = (ssize_t)PDE_DATA(file_inode(file));
+	const struct rtw_proc_hdl *hdl = adapter_proc_hdls + index;
+	ssize_t (*write)(struct file *, const char __user *, size_t, loff_t *, void *) = hdl->write;
+
+	if (write)
+		return write(file, buffer, count, pos, ((struct seq_file *)file->private_data)->private);
+
+	return -EROFS;
+}
+
+static const struct rtw_proc_ops rtw_adapter_proc_seq_fops = {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+	.proc_open = rtw_adapter_proc_open,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+	.proc_release = seq_release,
+	.proc_write = rtw_adapter_proc_write,
+#else
+	.owner = THIS_MODULE,
+	.open = rtw_adapter_proc_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = seq_release,
+	.write = rtw_adapter_proc_write,
+#endif
+};
+
+static const struct rtw_proc_ops rtw_adapter_proc_sseq_fops = {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+	.proc_open = rtw_adapter_proc_open,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+	.proc_release = seq_release,
+	.proc_write = rtw_adapter_proc_write,
+#else
+	.owner = THIS_MODULE,
+	.open = rtw_adapter_proc_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = rtw_adapter_proc_write,
+#endif
+};
+
+static int proc_get_phy_adaptivity(struct seq_file *m, void *v)
+{
+	struct net_device *dev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(dev);
+
+	rtw_cfg_adaptivity_config_msg(m, padapter);
+
+	return 0;
+}
+
+static char *phydm_msg = NULL;
+#define PHYDM_MSG_LEN	80*24*4
+
+static int proc_get_phydm_cmd(struct seq_file *m, void *v)
+{
+	struct net_device *netdev = m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(netdev);
+
+	if (NULL == phydm_msg) {
+		_RTW_PRINT_SEL(m, "(Nothing to output)\n");
+		return 0;
+#if 0
+		phydm_msg = rtw_zmalloc(PHYDM_MSG_LEN);
+		if (NULL == phydm_msg)
+			return -ENOMEM;
+
+		phydm_cmd(phydm, NULL, 0, 0, phydm_msg, PHYDM_MSG_LEN);
+#endif
+	}
+
+	_RTW_PRINT_SEL(m, "%s\n", phydm_msg);
+
+	rtw_mfree(phydm_msg, PHYDM_MSG_LEN);
+	phydm_msg = NULL;
+
+	return 0;
+}
+
+static int proc_get_phl_cmd(struct seq_file *m, void *v)
+{
+	return proc_get_phydm_cmd(m, v);
+}
+
+static ssize_t proc_set_phydm_cmd(struct file *file, char *buffer, size_t count,
+				  loff_t *pos, void *data,
+				  enum rtw_proc_cmd_type type)
+{
+	struct net_device *netdev = (struct net_device *)data;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(netdev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct rtw_proc_cmd cmd;
+
+	if (count < 1)
+		return -EFAULT;
+
+	if (buffer) {
+		if (NULL == phydm_msg) {
+			phydm_msg = rtw_zmalloc(PHYDM_MSG_LEN);
+			if (NULL == phydm_msg)
+				return -ENOMEM;
+		} else
+			_rtw_memset(phydm_msg, 0, PHYDM_MSG_LEN);
+
+		cmd.in_type = RTW_ARG_TYPE_BUF;
+		cmd.in_cnt_len = count;
+		cmd.in.buf = buffer;
+
+		rtw_phl_proc_cmd(GET_PHL_INFO(dvobj), type, &cmd, phydm_msg, PHYDM_MSG_LEN);
+
+		if (strlen(phydm_msg) == 0) {
+			rtw_mfree(phydm_msg, PHYDM_MSG_LEN);
+			phydm_msg = NULL;
+		}
+	}
+
+	return count;
+}
+
+static ssize_t proc_set_phl_cmd(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char *buf, *p;
+	enum rtw_proc_cmd_type type;
+	ssize_t ret, len;
+	int type_len;
+
+	buf = _rtw_malloc(count + 1);
+	if (!buf)
+		return -ENOMEM;
+
+	if (copy_from_user(buf, buffer, count)) {
+		ret = -EFAULT;
+		goto err;
+	}
+
+	if (strncmp(buf, "bb ", 3) == 0) {
+		type = RTW_PROC_CMD_BB;
+		type_len = 3;
+	} else if (strncmp(buf, "rf ", 3) == 0) {
+		type = RTW_PROC_CMD_RF;
+		type_len = 3;
+	} else if (strncmp(buf, "mac ", 4) == 0) {
+		type = RTW_PROC_CMD_MAC;
+		type_len = 4;
+	} else if (strncmp(buf, "phl ", 4) == 0) {
+		type = RTW_PROC_CMD_PHL;
+		type_len = 4;
+	} else if (strncmp(buf, "core ", 5) == 0) {
+		printk("%s(%d)\n",__func__,__LINE__);
+		type = RTW_PROC_CMD_CORE;
+		type_len = 5;
+	} else if (strncmp(buf, "btc ", 4) == 0) {
+		printk("%s(%d)\n",__func__,__LINE__);
+		type = RTW_PROC_CMD_BTC;
+		type_len = 4;
+	} else if (strncmp(buf, "efuse ", 6) == 0) {
+		type = RTW_PROC_CMD_EFUSE;
+		type_len = 6;
+	} else {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	/* skip first type token, like 'bb ', 'rf ' */
+	p = buf + type_len;
+	len = count - type_len;
+
+	/* remove trailing newline character, because BB/RF parser uses
+	 * " ," as delimiter that leads the last token contains '\n' if
+	 * we use 'echo' command without '-n' argument.
+	 */
+	if (p[len - 1] == '\n')
+		len--;
+	p[len++] = '\0';
+
+	ret = proc_set_phydm_cmd(file, p, len, pos, data, type);
+
+err:
+	_rtw_mfree(buf, count + 1);
+
+	if (ret >= 0)
+		return count;
+
+	return ret;
+}
+
+/*
+* rtw_odm_proc:
+* init/deinit when register/unregister net_device, along with rtw_adapter_proc
+*/
+const struct rtw_proc_hdl odm_proc_hdls[] = {
+	RTW_PROC_HDL_SSEQ("adaptivity", proc_get_phy_adaptivity, NULL),
+	RTW_PROC_HDL_SZSEQ("phl_cmd", proc_get_phl_cmd, proc_set_phl_cmd, PHYDM_MSG_LEN),
+};
+
+const int odm_proc_hdls_num = sizeof(odm_proc_hdls) / sizeof(struct rtw_proc_hdl);
+
+static int rtw_odm_proc_open(struct inode *inode, struct file *file)
+{
+	ssize_t index = (ssize_t)PDE_DATA(inode);
+	const struct rtw_proc_hdl *hdl = odm_proc_hdls + index;
+	void *private = proc_get_parent_data(inode);
+
+	if (hdl->type == RTW_PROC_HDL_TYPE_SEQ) {
+		int res = seq_open(file, hdl->u.seq_op);
+
+		if (res == 0)
+			((struct seq_file *)file->private_data)->private = private;
+
+		return res;
+	} else if (hdl->type == RTW_PROC_HDL_TYPE_SSEQ) {
+		int (*show)(struct seq_file *, void *) = hdl->u.show ? hdl->u.show : proc_get_dummy;
+
+		return single_open(file, show, private);
+	} else if (hdl->type == RTW_PROC_HDL_TYPE_SZSEQ) {
+		int (*show)(struct seq_file *, void *) = hdl->u.sz.show ? hdl->u.sz.show : proc_get_dummy;
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0))
+		return single_open_size(file, show, private, hdl->u.sz.size);
+		#else
+		return single_open(file, show, private);
+		#endif
+	} else {
+		return -EROFS;
+	}
+}
+
+static ssize_t rtw_odm_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos)
+{
+	ssize_t index = (ssize_t)PDE_DATA(file_inode(file));
+	const struct rtw_proc_hdl *hdl = odm_proc_hdls + index;
+	ssize_t (*write)(struct file *, const char __user *, size_t, loff_t *, void *) = hdl->write;
+
+	if (write)
+		return write(file, buffer, count, pos, ((struct seq_file *)file->private_data)->private);
+
+	return -EROFS;
+}
+
+static const struct rtw_proc_ops rtw_odm_proc_seq_fops = {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+	.proc_open = rtw_odm_proc_open,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+	.proc_release = seq_release,
+	.proc_write = rtw_odm_proc_write,
+#else
+	.owner = THIS_MODULE,
+	.open = rtw_odm_proc_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = seq_release,
+	.write = rtw_odm_proc_write,
+#endif
+};
+
+static const struct rtw_proc_ops rtw_odm_proc_sseq_fops = {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+	.proc_open = rtw_odm_proc_open,
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+	.proc_release = single_release,
+	.proc_write = rtw_odm_proc_write,
+#else
+	.owner = THIS_MODULE,
+	.open = rtw_odm_proc_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = rtw_odm_proc_write,
+#endif
+};
+
+struct proc_dir_entry *rtw_odm_proc_init(struct net_device *dev)
+{
+	struct proc_dir_entry *dir_odm = NULL;
+	struct proc_dir_entry *entry = NULL;
+	_adapter	*adapter = rtw_netdev_priv(dev);
+	ssize_t i;
+
+	if (adapter->dir_dev == NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (adapter->dir_odm != NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	dir_odm = rtw_proc_create_dir("odm", adapter->dir_dev, dev);
+	if (dir_odm == NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	adapter->dir_odm = dir_odm;
+
+	for (i = 0; i < odm_proc_hdls_num; i++) {
+		if (odm_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+			entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_seq_fops, (void *)i);
+		else if (odm_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ ||
+			 odm_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SZSEQ)
+			entry = rtw_proc_create_entry(odm_proc_hdls[i].name, dir_odm, &rtw_odm_proc_sseq_fops, (void *)i);
+		else
+			entry = NULL;
+
+		if (!entry) {
+			rtw_warn_on(1);
+			goto exit;
+		}
+	}
+
+exit:
+	return dir_odm;
+}
+
+void rtw_odm_proc_deinit(_adapter  *adapter)
+{
+	struct proc_dir_entry *dir_odm = NULL;
+	int i;
+
+	dir_odm = adapter->dir_odm;
+
+	if (dir_odm == NULL) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	for (i = 0; i < odm_proc_hdls_num; i++)
+		remove_proc_entry(odm_proc_hdls[i].name, dir_odm);
+
+	remove_proc_entry("odm", adapter->dir_dev);
+
+	adapter->dir_odm = NULL;
+
+	if (phydm_msg) {
+		rtw_mfree(phydm_msg, PHYDM_MSG_LEN);
+		phydm_msg = NULL;
+	}
+}
+
+struct proc_dir_entry *rtw_adapter_proc_init(struct net_device *dev)
+{
+	struct proc_dir_entry *drv_proc = get_rtw_drv_proc();
+	struct proc_dir_entry *dir_dev = NULL;
+	struct proc_dir_entry *entry = NULL;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	ssize_t i;
+
+	if (drv_proc == NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (adapter->dir_dev != NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	dir_dev = rtw_proc_create_dir(dev->name, drv_proc, dev);
+	if (dir_dev == NULL) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	adapter->dir_dev = dir_dev;
+
+	for (i = 0; i < adapter_proc_hdls_num; i++) {
+		if (adapter_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SEQ)
+			entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_seq_fops, (void *)i);
+		else if (adapter_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SSEQ ||
+			 adapter_proc_hdls[i].type == RTW_PROC_HDL_TYPE_SZSEQ)
+			entry = rtw_proc_create_entry(adapter_proc_hdls[i].name, dir_dev, &rtw_adapter_proc_sseq_fops, (void *)i);
+		else
+			entry = NULL;
+
+		if (!entry) {
+			rtw_warn_on(1);
+			goto exit;
+		}
+	}
+
+	rtw_odm_proc_init(dev);
+exit:
+	return dir_dev;
+}
+
+void rtw_adapter_proc_deinit(struct net_device *dev)
+{
+	struct proc_dir_entry *drv_proc = get_rtw_drv_proc();
+	struct proc_dir_entry *dir_dev = NULL;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	int i;
+
+	dir_dev = adapter->dir_dev;
+
+	if (dir_dev == NULL) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	for (i = 0; i < adapter_proc_hdls_num; i++)
+		remove_proc_entry(adapter_proc_hdls[i].name, dir_dev);
+
+	rtw_odm_proc_deinit(adapter);
+	remove_proc_entry(dev->name, drv_proc);
+
+	adapter->dir_dev = NULL;
+}
+
+void rtw_adapter_proc_replace(struct net_device *dev)
+{
+	struct proc_dir_entry *drv_proc = get_rtw_drv_proc();
+	struct proc_dir_entry *dir_dev = NULL;
+	_adapter *adapter = rtw_netdev_priv(dev);
+	int i;
+
+	dir_dev = adapter->dir_dev;
+
+	if (dir_dev == NULL) {
+		rtw_warn_on(1);
+		return;
+	}
+
+	for (i = 0; i < adapter_proc_hdls_num; i++)
+		remove_proc_entry(adapter_proc_hdls[i].name, dir_dev);
+
+	rtw_odm_proc_deinit(adapter);
+
+	remove_proc_entry(adapter->old_ifname, drv_proc);
+
+	adapter->dir_dev = NULL;
+
+	rtw_adapter_proc_init(dev);
+
+}
+#ifdef CONFIG_SELF_DIAG_INFO
+static void _phl_cmd_set(char *buf, char* cmd_string, struct net_device *dev, enum rtw_proc_cmd_type type)
+{
+	u32 cmd_len = 0;
+
+	cmd_len = strlen(cmd_string);
+	_rtw_memset(buf, 0, 25);
+	_rtw_memcpy(buf, cmd_string,  cmd_len);
+	proc_set_phydm_cmd(NULL, buf, cmd_len, NULL, dev, type);
+}
+
+static int proc_get_self_diag_info(struct seq_file *m, void *v)
+{
+	char *buf = NULL;
+	char* cmd_string;
+	static systime last_time = 0;
+	struct net_device *netdev = (struct net_device *)m->private;
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(netdev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+#ifdef CONFIG_USB_HCI
+	int i;
+	u32 trx_submit = 0;
+	u32 trx_submit_fail = 0;
+	u32 trx_complete = 0;
+	u32 trx_complete_fail = 0;
+	u32 trx_last_submit = 0;
+	char trx_string[10] = {0};
+#endif
+
+	buf = _rtw_malloc(25);
+	if (buf) {
+
+		struct net_device *dev = m->private;
+
+		/* MAC */
+		RTW_PRINT_SEL(m, "\n==================================[MAC info]==================================\n");
+		cmd_string = "dbg_msg_en 1 0";
+		_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_MAC);
+		cmd_string = "self_diag 2";
+		_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_MAC);
+		proc_get_phl_cmd(m, v);
+		cmd_string = "dbg_msg_en 0 0";
+		_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_MAC);
+
+		/* BB */
+		RTW_PRINT_SEL(m, "=================================[BB info]==================================\n");
+		if (dvobj->bb_self_diag_en) {
+			cmd_string = "auto_dbg query 0";
+			_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_BB);
+			proc_get_phl_cmd(m, v);
+
+			cmd_string = "auto_dbg query 2";
+			_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_BB);
+			proc_get_phl_cmd(m, v);
+
+			cmd_string = "auto_dbg query 3";
+			_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_BB);
+			proc_get_phl_cmd(m, v);
+
+#ifdef CONFIG_DBCC_SUPPORT
+			if (rtw_phl_mr_is_db(dvobj->phl)) {
+				cmd_string = "1 auto_dbg query 0";
+				_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_BB);
+				proc_get_phl_cmd(m, v);
+
+				cmd_string = "1 auto_dbg query 2";
+				_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_BB);
+				proc_get_phl_cmd(m, v);
+
+				cmd_string = "1 auto_dbg query 3";
+				_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_BB);
+				proc_get_phl_cmd(m, v);
+			}
+#endif
+		} else {
+			RTW_PRINT_SEL(m, "\nPlease use \"echo bb 1 > /proc/net/xxxxx/wlanx/self_diag_info\" to enable bb self diag\n\n");
+		}
+		/* RF */
+		RTW_PRINT_SEL(m, "==================================[RF info]==================================\n");
+		cmd_string = "dz_dbg rfkinfo";
+		_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_RF);
+		proc_get_phl_cmd(m, v);
+
+		cmd_string = "dz_dbg rtinfo";
+		_phl_cmd_set(buf, cmd_string, dev, RTW_PROC_CMD_RF);
+		proc_get_phl_cmd(m, v);
+
+	}
+
+	_rtw_mfree(buf, 25);
+	buf = NULL;
+
+	RTW_PRINT_SEL(m,"==================================[driver info]==================================\n");
+
+	if (last_time != 0)
+		RTW_PRINT_SEL(m, "get passing time from last get self_diag_info (%d ms)\n\n", rtw_get_passing_time_ms(last_time));
+	last_time =  rtw_get_current_time();
+
+	RTW_PRINT_SEL(m, "driver warn on count = %d \n\n", rtw_warn_on_cnt);
+
+#ifdef CONFIG_USB_HCI
+	RTW_PRINT_SEL(m, "%s%17s, %s\n", "Driver RX:", "Last count", "Current count");
+	RTW_PRINT_SEL(m, "BULK_ID, Status, %s, %s, %s, %s, %s\n", "read_port ", "read_port", "read_port_complete", "read_port_fail", "read_port_complete_fail");
+	for(i = 0; i < RX_STATS_MAX_NUM; i++) {
+		trx_submit = dvobj->usb_data.trx_stats.rx_submit_cnt[i];
+		trx_submit_fail = dvobj->usb_data.trx_stats.rx_submit_fail[i];
+		trx_complete = dvobj->usb_data.trx_stats.rx_complete_cnt[i];
+		trx_complete_fail = dvobj->usb_data.trx_stats.rx_complete_fail[i];
+		trx_last_submit = dvobj->usb_data.trx_stats.last_rx_submit_cnt[i];
+		strcpy(trx_string, "-------");
+
+		if (trx_last_submit !=0) {
+			if (trx_last_submit == trx_submit)
+				strcpy(trx_string, "No RX");
+			else
+				strcpy(trx_string, "Have RX");
+		}
+
+		if (trx_submit)
+			RTW_PRINT_SEL(m, "(%2d,%7s,%u,%u,%u,%u,%u)\n", i, trx_string, trx_last_submit, trx_submit, trx_complete, trx_submit_fail, trx_complete_fail);
+		dvobj->usb_data.trx_stats.last_rx_submit_cnt[i] = trx_submit;
+	}
+	RTW_PRINT_SEL(m, "\n%s%17s, %s\n", "Driver TX:", "Last count", "Current count");
+	RTW_PRINT_SEL(m, "BULK_ID, Status, %s, %s, %s, %s, %s\n", "write_port", "write_port", "write_port_complete", "write_port_fail", "write_port_complete_fail");
+
+	for(i = 0; i < TX_STATS_MAX_NUM; i++) {
+		trx_submit = dvobj->usb_data.trx_stats.tx_submit_cnt[i];
+		trx_submit_fail = dvobj->usb_data.trx_stats.tx_submit_fail[i];
+		trx_complete = dvobj->usb_data.trx_stats.tx_complete_cnt[i];
+		trx_complete_fail = dvobj->usb_data.trx_stats.tx_complete_fail[i];
+		trx_last_submit = dvobj->usb_data.trx_stats.last_tx_submit_cnt[i];
+		strcpy(trx_string, "-------");
+
+		if (trx_last_submit !=0) {
+			if (trx_last_submit == trx_submit)
+				strcpy(trx_string, "No TX");
+			else
+				strcpy(trx_string, "Have TX");
+		}
+
+		if (trx_submit)
+			RTW_PRINT_SEL(m, "(%2d,%7s,%u,%u,%u,%u,%u)\n", i, trx_string, trx_last_submit, trx_submit, trx_complete, trx_submit_fail, trx_complete_fail);
+		dvobj->usb_data.trx_stats.last_tx_submit_cnt[i] = trx_submit;
+	}
+#endif
+	return 0;
+}
+
+static ssize_t proc_set_self_diag_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data)
+{
+	char *in_buf;
+	char type[32] = {0};
+	char val;
+	struct net_device *dev = data;
+	_adapter *adapter= (_adapter *)rtw_netdev_priv(dev);
+	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
+
+	if (count < 1)
+		return -EINVAL;
+
+	if (count > sizeof(type)) {
+		rtw_warn_on(1);
+		return -EFAULT;
+	}
+
+	in_buf = _rtw_zmalloc(count + 1);
+	if (!in_buf)
+		return -ENOMEM;
+	if (copy_from_user(in_buf, buffer, count)){
+		_rtw_mfree(in_buf, count + 1);
+		return -EFAULT;
+	}
+	if(sscanf(in_buf, "%s %c", type, &val) != 2){
+		_rtw_mfree(in_buf, count + 1);
+		return -EINVAL;
+	}
+
+	_rtw_mfree(in_buf, count + 1);
+
+	RTW_INFO("type : %s\n", type);
+
+	if (strcmp(type, "bb") == 0) {
+		char *buf = NULL;
+		u32 cmd_len = 0;
+		char* cmd_string;
+
+		buf = _rtw_malloc(25);
+		if (buf) {
+
+			struct net_device *dev = data;
+
+			/* BB */
+			cmd_string = "auto_dbg type 2 0";
+			cmd_len = strlen(cmd_string);
+			_rtw_memcpy(buf, cmd_string,  cmd_len);
+			_rtw_memcpy(buf+cmd_len-1, &val, 1);
+			proc_set_phydm_cmd(file, buf, cmd_len, pos, dev, RTW_PROC_CMD_BB);
+
+			cmd_string = "auto_dbg type 3 0";
+			cmd_len = strlen(cmd_string);
+			_rtw_memset(buf, 0, 25);
+			_rtw_memcpy(buf, cmd_string,  cmd_len);
+			_rtw_memcpy(buf+cmd_len-1, &val, 1);
+			proc_set_phydm_cmd(file, buf, cmd_len, pos, dev, RTW_PROC_CMD_BB);
+
+			if (val == '1')
+				dvobj->bb_self_diag_en = 1;
+			else
+				dvobj->bb_self_diag_en = 0;
+		}
+
+		_rtw_mfree(buf, 25);
+		buf = NULL;
+
+	}
+	else if (strcmp(type, "reset_drv_counter") == 0) {
+		if (val == '1') {
+			rtw_warn_on_cnt = 0;
+#ifdef CONFIG_USB_HCI
+			_rtw_memset(&(dvobj->usb_data.trx_stats), 0, sizeof(struct trx_stats));
+#endif
+		}
+		RTW_INFO("driver_warn_on_cnt : %u\n", rtw_warn_on_cnt);
+	}
+	return count;
+
+}
+#endif
+#endif /* CONFIG_PROC_DEBUG */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_proc.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_proc.h
new file mode 100644
index 000000000000..40540238a339
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_proc.h
@@ -0,0 +1,77 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_PROC_H__
+#define __RTW_PROC_H__
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+
+#define RTW_PROC_HDL_TYPE_SEQ	0
+#define RTW_PROC_HDL_TYPE_SSEQ	1
+#define RTW_PROC_HDL_TYPE_SZSEQ	2
+
+struct rtw_proc_hdl {
+	char *name;
+	u8 type;
+	union {
+		int (*show)(struct seq_file *, void *);
+		struct seq_operations *seq_op;
+		struct {
+			int (*show)(struct seq_file *, void *);
+			size_t size;
+		} sz;
+	} u;
+	ssize_t (*write)(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+};
+
+#define RTW_PROC_HDL_SEQ(_name, _seq_op, _write) \
+	{ .name = _name, .type = RTW_PROC_HDL_TYPE_SEQ, .u.seq_op = _seq_op, .write = _write}
+
+#define RTW_PROC_HDL_SSEQ(_name, _show, _write) \
+	{ .name = _name, .type = RTW_PROC_HDL_TYPE_SSEQ, .u.show = _show, .write = _write}
+
+#define RTW_PROC_HDL_SZSEQ(_name, _show, _write, _size) \
+	{ .name = _name, .type = RTW_PROC_HDL_TYPE_SZSEQ, .u.sz.show = _show, .write = _write, .u.sz.size = _size}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0))
+#define rtw_proc_ops proc_ops
+#else
+#define rtw_proc_ops file_operations
+#endif
+
+#ifdef CONFIG_PROC_DEBUG
+
+int rtw_drv_proc_init(void);
+void rtw_drv_proc_deinit(void);
+struct proc_dir_entry *rtw_adapter_proc_init(struct net_device *dev);
+void rtw_adapter_proc_deinit(struct net_device *dev);
+void rtw_adapter_proc_replace(struct net_device *dev);
+
+#ifdef CONFIG_SELF_DIAG_INFO
+static int proc_get_self_diag_info(struct seq_file *m, void *v);
+static ssize_t proc_set_self_diag_info(struct file *file, const char __user *buffer, size_t count, loff_t *pos, void *data);
+#endif
+
+#else /* !CONFIG_PROC_DEBUG */
+
+static inline int rtw_drv_proc_init(void) {return _FAIL;}
+#define rtw_drv_proc_deinit() do {} while (0)
+static inline struct proc_dir_entry *rtw_adapter_proc_init(struct net_device *dev) {return NULL;}
+#define rtw_adapter_proc_deinit(dev) do {} while (0)
+#define rtw_adapter_proc_replace(dev) do {} while (0)
+
+#endif /* !CONFIG_PROC_DEBUG */
+
+#endif /* __RTW_PROC_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_rhashtable.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_rhashtable.c
new file mode 100644
index 000000000000..24ba4deebfec
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_rhashtable.c
@@ -0,0 +1,76 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+
+#if defined(CONFIG_RTW_WDS) || defined(CONFIG_RTW_MESH) /* for now, only promised for kernel versions we support mesh */
+
+int rtw_rhashtable_walk_enter(rtw_rhashtable *ht, rtw_rhashtable_iter *iter)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0))
+	rhashtable_walk_enter((ht), (iter));
+	return 0;
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0))
+	return rhashtable_walk_init((ht), (iter), GFP_ATOMIC);
+#else
+	/* kernel >= 4.4.0 rhashtable_walk_init use GFP_KERNEL to alloc, spin_lock for assignment */
+	iter->ht = ht;
+	iter->p = NULL;
+	iter->slot = 0;
+	iter->skip = 0;
+
+	iter->walker = kmalloc(sizeof(*iter->walker), GFP_ATOMIC);
+	if (!iter->walker)
+		return -ENOMEM;
+
+	spin_lock(&ht->lock);
+	iter->walker->tbl =
+		rcu_dereference_protected(ht->tbl, lockdep_is_held(&ht->lock));
+	list_add(&iter->walker->list, &iter->walker->tbl->walkers);
+	spin_unlock(&ht->lock);
+
+	return 0;
+#endif
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25))
+static inline int is_vmalloc_addr(const void *x)
+{
+#ifdef CONFIG_MMU
+	unsigned long addr = (unsigned long)x;
+
+	return addr >= VMALLOC_START && addr < VMALLOC_END;
+#else
+	return 0;
+#endif
+}
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)) */
+
+void kvfree(const void *addr)
+{
+	if (is_vmalloc_addr(addr))
+		vfree(addr);
+	else
+		kfree(addr);
+}
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)) */
+
+#include "rhashtable.c"
+
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)) */
+
+#endif /* defined(CONFIG_RTW_WDS) || defined(CONFIG_RTW_MESH) */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_rhashtable.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_rhashtable.h
new file mode 100644
index 000000000000..f13bd06df8db
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/rtw_rhashtable.h
@@ -0,0 +1,67 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTW_RHASHTABLE_H__
+#define __RTW_RHASHTABLE_H__
+
+#if defined(CONFIG_RTW_WDS) || defined(CONFIG_RTW_MESH) /* for now, only promised for kernel versions we support mesh */
+
+/* directly reference rhashtable in kernel */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
+#include <linux/rhashtable.h>
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) */
+
+/* Use rhashtable from kernel 4.4 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0))
+#define NULLS_MARKER(value) (1UL | (((long)value) << 1))
+#endif
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0))
+static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags)
+{
+	if (size != 0 && n > ULONG_MAX / size)
+		return NULL;
+	return __kmalloc(n * size, flags);
+}
+#endif
+#include "rhashtable.h"
+#endif /* (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)) */
+
+typedef struct rhashtable rtw_rhashtable;
+typedef struct rhash_head rtw_rhash_head;
+typedef struct rhashtable_params rtw_rhashtable_params;
+
+#define rtw_rhashtable_init(ht, params) rhashtable_init(ht, params)
+
+typedef struct rhashtable_iter rtw_rhashtable_iter;
+
+int rtw_rhashtable_walk_enter(rtw_rhashtable *ht, rtw_rhashtable_iter *iter);
+#define rtw_rhashtable_walk_exit(iter) rhashtable_walk_exit(iter)
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 16, 0))
+#define rtw_rhashtable_walk_start(iter) rhashtable_walk_start_check(iter)
+#else
+#define rtw_rhashtable_walk_start(iter) rhashtable_walk_start(iter)
+#endif
+#define rtw_rhashtable_walk_next(iter) rhashtable_walk_next(iter)
+#define rtw_rhashtable_walk_stop(iter) rhashtable_walk_stop(iter)
+
+#define rtw_rhashtable_free_and_destroy(ht, free_fn, arg) rhashtable_free_and_destroy((ht), (free_fn), (arg))
+#define rtw_rhashtable_lookup_fast(ht, key, params) rhashtable_lookup_fast((ht), (key), (params))
+#define rtw_rhashtable_lookup_insert_fast(ht, obj, params) rhashtable_lookup_insert_fast((ht), (obj), (params))
+#define rtw_rhashtable_remove_fast(ht, obj, params) rhashtable_remove_fast((ht), (obj), (params))
+
+#endif /* defined(CONFIG_RTW_WDS) || defined(CONFIG_RTW_MESH) */
+
+#endif /* __RTW_RHASHTABLE_H__ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/sdio_intf.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/sdio_intf.c
new file mode 100644
index 000000000000..0f55d510e08d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/sdio_intf.c
@@ -0,0 +1,1108 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HCI_INTF_C_
+
+#include <drv_types.h>
+#include <rtw_io_records.h>
+#include <platform_ops.h>
+#include <linux/pm_wakeirq.h>
+extern struct rtw_intf_ops sdio_ops;
+
+#ifndef CONFIG_SDIO_HCI
+#error "CONFIG_SDIO_HCI shall be on!\n"
+#endif
+
+#ifndef dev_to_sdio_func
+#define dev_to_sdio_func(d)     container_of(d, struct sdio_func, dev)
+#endif
+
+static const struct sdio_device_id sdio_ids[] = {
+
+#ifdef CONFIG_RTL8852A
+	{SDIO_DEVICE(0x024c, 0x8852), .class = SDIO_CLASS_WLAN, .driver_data = RTL8852A},
+	{SDIO_DEVICE(0x024c, 0xa852), .class = SDIO_CLASS_WLAN, .driver_data = RTL8852A},
+#endif
+
+#ifdef CONFIG_RTL8852B
+	{SDIO_DEVICE(0x024c, 0xb852), .class = SDIO_CLASS_WLAN, .driver_data = RTL8852B},
+	{SDIO_DEVICE(0x024c, 0xb853), .class = SDIO_CLASS_WLAN, .driver_data = RTL8852B},
+#endif
+#ifdef CONFIG_RTL8852BP
+	{SDIO_DEVICE(0x024c, 0xA85C), .class = SDIO_CLASS_WLAN, .driver_data = RTL8852BP},
+#endif
+#ifdef CONFIG_RTL8852BT
+	{SDIO_DEVICE(0x024c, 0xB925), .class = SDIO_CLASS_WLAN, .driver_data = RTL8852BT},
+#endif
+#ifdef CONFIG_RTL8851B
+	{SDIO_DEVICE(0x024c, 0xB851), .class = SDIO_CLASS_WLAN, .driver_data = RTL8851B},
+#endif
+
+#if defined(RTW_ENABLE_WIFI_CONTROL_FUNC) /* temporarily add this to accept all sdio wlan id */
+	{ SDIO_DEVICE_CLASS(SDIO_CLASS_WLAN) },
+#endif
+	{ /* end: all zeroes */				},
+};
+
+MODULE_DEVICE_TABLE(sdio, sdio_ids);
+
+static int rtw_dev_probe(struct sdio_func *func, const struct sdio_device_id *id);
+static void rtw_dev_remove(struct sdio_func *func);
+static void rtw_dev_shutdown(struct device *dev);
+static int rtw_sdio_resume(struct device *dev);
+static int rtw_sdio_suspend(struct device *dev);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+static const struct dev_pm_ops rtw_sdio_pm_ops = {
+	.suspend = rtw_sdio_suspend,
+	.resume	= rtw_sdio_resume,
+};
+#endif
+
+struct sdio_drv_priv {
+	struct sdio_driver rtw_sdio_drv;
+	int drv_registered;
+};
+
+static struct sdio_drv_priv sdio_drvpriv = {
+	.rtw_sdio_drv.probe = rtw_dev_probe,
+	.rtw_sdio_drv.remove = rtw_dev_remove,
+	.rtw_sdio_drv.name = (char *)DRV_NAME,
+	.rtw_sdio_drv.id_table = sdio_ids,
+	.rtw_sdio_drv.drv = {
+		.shutdown = rtw_dev_shutdown,
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
+		.pm = &rtw_sdio_pm_ops,
+#endif
+	}
+};
+
+
+static void sd_sync_int_hdl(struct sdio_func *func)
+{
+	struct dvobj_priv *dvobj;
+
+	dvobj = sdio_get_drvdata(func);
+
+	if (RTW_CANNOT_RUN(dvobj))
+		return;
+
+	rtw_sdio_set_irq_thd(dvobj, current);
+	/*sd_int_hdl(dvobj_get_primary_adapter(psdpriv));*/
+	rtw_phl_interrupt_handler(dvobj->phl);
+	rtw_sdio_set_irq_thd(dvobj, NULL);
+}
+
+int rtw_sdio_alloc_irq(struct dvobj_priv *dvobj)
+{
+	struct sdio_data *psdio_data;
+	struct sdio_func *func;
+	int err;
+
+	psdio_data = dvobj_to_sdio(dvobj);
+	func = psdio_data->func;
+
+	sdio_claim_host(func);
+
+	err = sdio_claim_irq(func, &sd_sync_int_hdl);
+	if (err) {
+		dvobj->drv_dbg.dbg_sdio_alloc_irq_error_cnt++;
+		RTW_PRINT("%s: sdio_claim_irq FAIL(%d)!\n", __func__, err);
+	} else {
+		dvobj->drv_dbg.dbg_sdio_alloc_irq_cnt++;
+		psdio_data->irq_alloc = 1;
+	}
+
+	sdio_release_host(func);
+
+	return err ? _FAIL : _SUCCESS;
+}
+
+void rtw_sdio_free_irq(struct dvobj_priv *dvobj)
+{
+	struct sdio_data *psdio_data;
+	struct sdio_func *func;
+	int err;
+
+	psdio_data = dvobj_to_sdio(dvobj);
+	if (psdio_data && psdio_data->irq_alloc) {
+		func = psdio_data->func;
+
+		if (func) {
+			sdio_claim_host(func);
+			err = sdio_release_irq(func);
+			if (err) {
+				dvobj->drv_dbg.dbg_sdio_free_irq_error_cnt++;
+				RTW_ERR("%s: sdio_release_irq FAIL(%d)!\n", __func__, err);
+			} else
+				dvobj->drv_dbg.dbg_sdio_free_irq_cnt++;
+			sdio_release_host(func);
+		}
+		psdio_data->irq_alloc = 0;
+	}
+}
+
+#ifdef CONFIG_GPIO_WAKEUP
+extern unsigned int oob_irq;
+extern unsigned int oob_gpio;
+static irqreturn_t gpio_hostwakeup_irq_thread(int irq, void *data)
+{
+	_adapter *padapter = (_adapter *)data;
+	RTW_PRINT("gpio_hostwakeup_irq_thread\n");
+	/* Disable interrupt before calling handler */
+	/* disable_irq_nosync(oob_irq); */
+#ifdef CONFIG_PLATFORM_ARM_SUN6I
+	return 0;
+#else
+	return IRQ_HANDLED;
+#endif
+}
+
+static u8 gpio_hostwakeup_alloc_irq(_adapter *padapter)
+{
+	int err;
+	u32 status = 0;
+	struct device *dev = dvobj_to_dev(padapter->dvobj);
+
+	if (oob_irq == 0) {
+		RTW_INFO("oob_irq ZERO!\n");
+		return _FAIL;
+	}
+
+	RTW_INFO("%s : oob_irq = %d\n", __func__, oob_irq);
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32))
+	status = IRQF_NO_SUSPEND;
+#endif
+	status |= IRQF_ONESHOT;
+
+	err = request_threaded_irq(oob_irq, NULL, gpio_hostwakeup_irq_thread,
+		status, "rtw_wifi_gpio_wakeup", padapter);
+
+	if (err < 0) {
+		RTW_INFO("Oops: can't allocate gpio irq %d err:%d\n", oob_irq, err);
+		return _FALSE;
+	} else
+		RTW_INFO("allocate gpio irq %d ok\n", oob_irq);
+
+	device_init_wakeup(dev, true);
+	dev_pm_set_wake_irq(dev, oob_irq);
+#ifndef CONFIG_PLATFORM_ARM_SUN8I
+	enable_irq_wake(oob_irq);
+#endif
+	return _SUCCESS;
+}
+
+static void gpio_hostwakeup_free_irq(_adapter *padapter)
+{
+	struct device *dev = dvobj_to_dev(padapter->dvobj);
+
+	if (oob_irq == 0)
+		return;
+
+	dev_pm_clear_wake_irq(dev);
+	device_init_wakeup(dev, false);
+#ifndef CONFIG_PLATFORM_ARM_SUN8I
+	disable_irq_wake(oob_irq);
+#endif
+	free_irq(oob_irq, padapter);
+}
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+/*
+ * mmc_blksz_for_byte_mode() & mmc_card_broken_byte_mode_512() has been moved
+ * to drivers/mmc/core/card.h from include/linux/mmc/card.h since kernel v4.11.
+ */
+static inline int mmc_blksz_for_byte_mode(const struct mmc_card *c)
+{
+	return c->quirks & MMC_QUIRK_BLKSZ_FOR_BYTE_MODE;
+}
+
+static inline int mmc_card_broken_byte_mode_512(const struct mmc_card *c)
+{
+	return c->quirks & MMC_QUIRK_BROKEN_BYTE_MODE_512;
+}
+#endif /* kernel >= v4.11 */
+
+/*
+ * Calculate the maximum byte mode transfer size
+ */
+static inline unsigned int sdio_max_byte_size(struct sdio_func *func)
+{
+	unsigned mval =	func->card->host->max_blk_size;
+
+	if (mmc_blksz_for_byte_mode(func->card))
+		mval = min(mval, func->cur_blksize);
+	else
+		mval = min(mval, func->max_blksize);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
+	if (mmc_card_broken_byte_mode_512(func->card))
+		return min(mval, 511u);
+#endif /* kernel v3.3 */
+
+	return min(mval, 512u); /* maximum size for byte mode */
+}
+
+void dump_sdio_card_info(void *sel, struct dvobj_priv *dvobj)
+{
+	struct sdio_data *psdio_data = dvobj_to_sdio(dvobj);
+	struct mmc_card *card = psdio_data->card;
+	int i;
+
+	RTW_PRINT_SEL(sel, "== SDIO Card Info ==\n");
+	RTW_PRINT_SEL(sel, "  card: %p\n", card);
+	RTW_PRINT_SEL(sel, "  clock: %d Hz\n", psdio_data->clock);
+
+	RTW_PRINT_SEL(sel, "  timing spec: ");
+	switch (psdio_data->timing) {
+	case MMC_TIMING_LEGACY:
+		_RTW_PRINT_SEL(sel, "legacy");
+		break;
+	case MMC_TIMING_MMC_HS:
+		_RTW_PRINT_SEL(sel, "mmc high-speed");
+		break;
+	case MMC_TIMING_SD_HS:
+		_RTW_PRINT_SEL(sel, "sd high-speed");
+		break;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+	case MMC_TIMING_UHS_SDR12:
+		_RTW_PRINT_SEL(sel, "sd uhs SDR12");
+		break;
+	case MMC_TIMING_UHS_SDR25:
+		_RTW_PRINT_SEL(sel, "sd uhs SDR25");
+		break;
+	#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */
+
+	case MMC_TIMING_UHS_SDR50:
+		_RTW_PRINT_SEL(sel, "sd uhs SDR50");
+		break;
+
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+	case MMC_TIMING_MMC_DDR52:
+		_RTW_PRINT_SEL(sel, "mmc DDR52");
+		break;
+	#endif
+
+	case MMC_TIMING_UHS_SDR104:
+		_RTW_PRINT_SEL(sel, "sd uhs SDR104");
+		break;
+	case MMC_TIMING_UHS_DDR50:
+		_RTW_PRINT_SEL(sel, "sd uhs DDR50");
+		break;
+
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
+	case MMC_TIMING_MMC_HS200:
+		_RTW_PRINT_SEL(sel, "mmc HS200");
+		break;
+	#endif
+
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+	case MMC_TIMING_MMC_HS400:
+		_RTW_PRINT_SEL(sel, "mmc HS400");
+		break;
+	#endif
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) */
+	default:
+		_RTW_PRINT_SEL(sel, "unknown(%d)", psdio_data->timing);
+		break;
+	}
+	_RTW_PRINT_SEL(sel, "\n");
+
+	RTW_PRINT_SEL(sel, "  sd3_bus_mode: %s\n", (psdio_data->sd3_bus_mode) ? "TRUE" : "FALSE");
+
+	rtw_warn_on(card->sdio_funcs != rtw_sdio_get_num_of_func(dvobj));
+	RTW_PRINT_SEL(sel, "  func num: %u\n", card->sdio_funcs);
+	for (i = 0; card->sdio_func[i]; i++) {
+		RTW_PRINT_SEL(sel, "  func%u: %p%s\n"
+			, card->sdio_func[i]->num, card->sdio_func[i]
+			, psdio_data->func == card->sdio_func[i] ? " (*)" : "");
+	}
+
+	RTW_PRINT_SEL(sel, "  max_byte_size: %u\n", psdio_data->max_byte_size);
+
+	RTW_PRINT_SEL(sel, "================\n");
+}
+
+#define SDIO_CARD_INFO_DUMP(dvobj)	dump_sdio_card_info(RTW_DBGDUMP, dvobj)
+
+#ifdef DBG_SDIO
+#if (DBG_SDIO >= 2)
+void rtw_sdio_dbg_reg_free(struct dvobj_priv *d)
+{
+	struct sdio_data *sdio;
+	u8 *buf;
+	u32 size;
+
+
+	sdio = dvobj_to_sdio(d);
+
+	buf = sdio->dbg_msg;
+	size = sdio->dbg_msg_size;
+	if (buf){
+		sdio->dbg_msg = NULL;
+		sdio->dbg_msg_size = 0;
+		rtw_mfree(buf, size);
+	}
+
+	buf = sdio->reg_mac;
+	if (buf) {
+		sdio->reg_mac = NULL;
+		rtw_mfree(buf, 0x800);
+	}
+
+	buf = sdio->reg_mac_ext;
+	if (buf) {
+		sdio->reg_mac_ext = NULL;
+		rtw_mfree(buf, 0x800);
+	}
+
+	buf = sdio->reg_local;
+	if (buf) {
+		sdio->reg_local = NULL;
+		rtw_mfree(buf, 0x100);
+	}
+
+	buf = sdio->reg_cia;
+	if (buf) {
+		sdio->reg_cia = NULL;
+		rtw_mfree(buf, 0x200);
+	}
+}
+
+void rtw_sdio_dbg_reg_alloc(struct dvobj_priv *d)
+{
+	struct sdio_data *sdio;
+	u8 *buf;
+
+
+	sdio = dvobj_to_sdio(d);
+
+	buf = _rtw_zmalloc(0x800);
+	if (buf)
+		sdio->reg_mac = buf;
+
+	buf = _rtw_zmalloc(0x800);
+	if (buf)
+		sdio->reg_mac_ext = buf;
+
+	buf = _rtw_zmalloc(0x100);
+	if (buf)
+		sdio->reg_local = buf;
+
+	buf = _rtw_zmalloc(0x200);
+	if (buf)
+		sdio->reg_cia = buf;
+}
+#endif /* DBG_SDIO >= 2 */
+
+static void sdio_dbg_init(struct dvobj_priv *d)
+{
+	struct sdio_data *sdio;
+
+
+	sdio = dvobj_to_sdio(d);
+
+	sdio->cmd52_err_cnt = 0;
+	sdio->cmd53_err_cnt = 0;
+
+#if (DBG_SDIO >= 1)
+	sdio->reg_dump_mark = 0;
+#endif /* DBG_SDIO >= 1 */
+
+#if (DBG_SDIO >= 3)
+	sdio->dbg_enable = 0;
+	sdio->err_stop = 0;
+	sdio->err_test = 0;
+	sdio->err_test_triggered = 0;
+#endif /* DBG_SDIO >= 3 */
+}
+
+static void sdio_dbg_deinit(struct dvobj_priv *d)
+{
+#if (DBG_SDIO >= 2)
+	rtw_sdio_dbg_reg_free(d);
+#endif /* DBG_SDIO >= 2 */
+}
+#endif /* DBG_SDIO */
+
+u32 rtw_sdio_init(struct dvobj_priv *dvobj)
+{
+	struct sdio_data *psdio_data;
+	struct sdio_func *func;
+	int err;
+
+
+	psdio_data = dvobj_to_sdio(dvobj);
+	func = psdio_data->func;
+
+	/* 3 1. init SDIO bus */
+	sdio_claim_host(func);
+
+	err = sdio_enable_func(func);
+	if (err) {
+		dvobj->drv_dbg.dbg_sdio_init_error_cnt++;
+		RTW_PRINT("%s: sdio_enable_func FAIL(%d)!\n", __func__, err);
+		goto release;
+	}
+
+	err = sdio_set_block_size(func, 512);
+	if (err) {
+		dvobj->drv_dbg.dbg_sdio_init_error_cnt++;
+		RTW_PRINT("%s: sdio_set_block_size FAIL(%d)!\n", __func__, err);
+		goto release;
+	}
+	psdio_data->block_transfer_len = 512;
+	psdio_data->tx_block_mode = 1;
+	psdio_data->rx_block_mode = 1;
+
+	psdio_data->card = func->card;
+	psdio_data->timing = func->card->host->ios.timing;
+	psdio_data->clock = func->card->host->ios.clock;
+	psdio_data->func_number = func->card->sdio_funcs;
+
+	psdio_data->sd3_bus_mode = _FALSE;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
+	if (psdio_data->timing <= MMC_TIMING_UHS_DDR50
+		#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+		&& psdio_data->timing >= MMC_TIMING_UHS_SDR12
+		#else
+		&& psdio_data->timing >= MMC_TIMING_UHS_SDR50
+		#endif
+	)
+		psdio_data->sd3_bus_mode = _TRUE;
+#endif
+
+	psdio_data->max_byte_size = sdio_max_byte_size(func);
+
+#ifdef DBG_SDIO
+	sdio_dbg_init(dvobj);
+#endif /* DBG_SDIO */
+
+	SDIO_CARD_INFO_DUMP(dvobj);
+
+
+release:
+	sdio_release_host(func);
+
+	if (err)
+		return _FAIL;
+	return _SUCCESS;
+}
+
+void rtw_sdio_deinit(struct dvobj_priv *dvobj)
+{
+	struct sdio_func *func;
+	int err;
+
+	func = dvobj_to_sdio(dvobj)->func;
+
+	if (func) {
+		sdio_claim_host(func);
+		err = sdio_disable_func(func);
+		if (err) {
+			dvobj->drv_dbg.dbg_sdio_deinit_error_cnt++;
+			RTW_ERR("%s: sdio_disable_func(%d)\n", __func__, err);
+		}
+
+		sdio_release_host(func);
+	}
+
+#ifdef DBG_SDIO
+	sdio_dbg_deinit(dvobj);
+#endif /* DBG_SDIO */
+}
+
+u8 rtw_sdio_get_num_of_func(struct dvobj_priv *dvobj)
+{
+	return dvobj_to_sdio(dvobj)->func_number;
+}
+
+static struct dvobj_priv *sdio_dvobj_init(struct sdio_func *func,
+					const struct sdio_device_id *pdid)
+{
+	int status = _FAIL;
+	struct dvobj_priv *dvobj = NULL;
+	struct sdio_data *psdio;
+
+
+	dvobj = devobj_init();
+	if (dvobj == NULL)
+		goto exit;
+
+	sdio_set_drvdata(func, dvobj);
+
+	psdio = dvobj_to_sdio(dvobj);
+	psdio->func = func;
+
+	psdio->tmpbuf_sz = 32;
+	psdio->tmpbuf = rtw_malloc(psdio->tmpbuf_sz);
+	if (!psdio->tmpbuf) {
+		psdio->tmpbuf_sz = 0;
+		goto free_dvobj;
+	}
+
+	if (rtw_sdio_init(dvobj) != _SUCCESS) {
+		goto free_dvobj;
+	}
+
+	dvobj->interface_type = RTW_HCI_SDIO;
+	dvobj->ic_id = pdid->driver_data;
+	dvobj->intf_ops = &sdio_ops;
+
+	rtw_reset_continual_io_error(dvobj);
+	status = _SUCCESS;
+
+free_dvobj:
+	if (status != _SUCCESS && dvobj) {
+		sdio_set_drvdata(func, NULL);
+		devobj_deinit(dvobj);
+		dvobj = NULL;
+	}
+exit:
+	return dvobj;
+}
+
+static void sdio_dvobj_deinit(struct sdio_func *func)
+{
+	struct dvobj_priv *dvobj = sdio_get_drvdata(func);
+	struct sdio_data *sdio;
+
+
+	sdio_set_drvdata(func, NULL);
+	if (dvobj) {
+		rtw_sdio_deinit(dvobj);
+		rtw_sdio_free_irq(dvobj);
+
+		sdio = dvobj_to_sdio(dvobj);
+		if (sdio->tmpbuf_sz) {
+			rtw_mfree(sdio->tmpbuf, sdio->tmpbuf_sz);
+			sdio->tmpbuf_sz = 0;
+			sdio->tmpbuf = NULL;
+		}
+
+		devobj_deinit(dvobj);
+	}
+
+	return;
+}
+
+#ifdef RTW_SUPPORT_PLATFORM_SHUTDOWN
+_adapter * g_test_adapter = NULL;
+#endif /* RTW_SUPPORT_PLATFORM_SHUTDOWN */
+
+_adapter *rtw_sdio_primary_adapter_init(struct dvobj_priv *dvobj)
+{
+	int status = _FAIL;
+	_adapter *padapter = NULL;
+	u8 hw_mac_addr[ETH_ALEN] = {0};
+
+	padapter = (_adapter *)rtw_zvmalloc(sizeof(*padapter));
+	if (padapter == NULL)
+		goto exit;
+
+	/*registry_priv*/
+	if (rtw_load_registry(padapter) != _SUCCESS)
+		goto free_adapter;
+
+#ifdef RTW_SUPPORT_PLATFORM_SHUTDOWN
+	g_test_adapter = padapter;
+#endif /* RTW_SUPPORT_PLATFORM_SHUTDOWN */
+	padapter->dvobj = dvobj;
+
+	dvobj->padapters[dvobj->iface_nums++] = padapter;
+	padapter->iface_id = IFACE_ID0;
+
+	/* set adapter_type/iface type for primary padapter */
+	padapter->isprimary = _TRUE;
+	padapter->adapter_type = PRIMARY_ADAPTER;
+
+	/* 3 7. init driver common data */
+	if (rtw_init_drv_sw(padapter) == _FAIL) {
+		goto free_adapter;
+	}
+
+	/* get mac addr */
+	rtw_hw_get_mac_addr(dvobj, hw_mac_addr);
+
+	/* set mac addr */
+	rtw_macaddr_cfg(adapter_mac_addr(padapter), hw_mac_addr);
+
+	RTW_INFO("bDriverStopped:%s, bSurpriseRemoved:%s, netif_up:%d, hw_init_completed:%d\n"
+		, dev_is_drv_stopped(dvobj) ? "True" : "False"
+		, dev_is_surprise_removed(dvobj) ? "True" : "False"
+		, padapter->netif_up
+		, rtw_hw_get_init_completed(dvobj)
+	);
+
+	status = _SUCCESS;
+
+free_adapter:
+	if (status != _SUCCESS && padapter) {
+		rtw_vmfree((u8 *)padapter, sizeof(*padapter));
+		padapter = NULL;
+	}
+exit:
+	return padapter;
+}
+
+static void rtw_sdio_primary_adapter_deinit(_adapter *padapter)
+{
+
+#ifdef CONFIG_GPIO_WAKEUP
+#ifdef CONFIG_PLATFORM_ARM_SUN6I
+	sw_gpio_eint_set_enable(gpio_eint_wlan, 0);
+	sw_gpio_irq_free(eint_wlan_handle);
+#else
+	gpio_hostwakeup_free_irq(padapter);
+#endif
+#endif
+	rtw_free_drv_sw(padapter);
+
+	/* TODO: use rtw_os_ndevs_deinit instead at the first stage of driver's dev deinit function */
+	rtw_os_ndev_free(padapter);
+
+	rtw_vmfree((u8 *)padapter, sizeof(_adapter));
+#ifdef RTW_SUPPORT_PLATFORM_SHUTDOWN
+	g_test_adapter = NULL;
+#endif /* RTW_SUPPORT_PLATFORM_SHUTDOWN */
+}
+
+/*
+ * drv_init() - a device potentially for us
+ *
+ * notes: drv_init() is called when the bus driver has located a card for us to support.
+ *        We accept the new device by returning 0.
+ */
+static int rtw_dev_probe(
+	struct sdio_func *func,
+	const struct sdio_device_id *id)
+{
+	_adapter *padapter = NULL;
+	struct dvobj_priv *dvobj;
+
+	RTW_INFO("+%s\n", __func__);
+
+	dvobj = sdio_dvobj_init(func, id);
+	if (dvobj == NULL) {
+		RTW_ERR("dvobj == NULL\n");
+		goto exit;
+	}
+
+	if (devobj_trx_resource_init(dvobj) == _FAIL)
+		goto free_dvobj;
+
+	/*init hw - register and get chip-info and hw capability*/
+	if (rtw_hw_init(dvobj) == _FAIL) {
+		RTW_ERR("rtw_hw_init Failed!\n");
+		goto free_trx_reso;
+	}
+
+	padapter = rtw_sdio_primary_adapter_init(dvobj);
+	if (padapter == NULL) {
+		RTW_INFO("rtw_init_primary_adapter Failed!\n");
+		goto free_hw;
+	}
+
+#ifdef CONFIG_CONCURRENT_MODE
+	if (rtw_drv_add_vir_ifaces(dvobj) == _FAIL)
+		goto free_if_vir;
+#endif
+
+	if (rtw_adapter_link_init(dvobj) != _SUCCESS)
+		goto free_adapter_link;
+
+	/*init data of dvobj from registary and ic spec*/
+	if (devobj_data_init(dvobj) == _FAIL) {
+		RTW_ERR("devobj_data_init Failed!\n");
+		/*free self beacuse of the function donnot clean memory when fail*/
+		goto free_dvobj_data;
+	}
+
+	/* dev_alloc_name && register_netdev */
+	if (rtw_os_ndevs_init(dvobj) != _SUCCESS)
+		goto free_dvobj_data;
+
+	/* Update link_mlme_priv's ht/vht/he priv from padapter->mlmepriv */
+	rtw_init_link_capab(dvobj);
+
+#ifdef CONFIG_HOSTAPD_MLME
+	hostapd_mode_init(padapter);
+#endif
+
+	if (rtw_sdio_alloc_irq(dvobj) != _SUCCESS)
+		goto os_ndevs_deinit;
+
+#ifdef CONFIG_GPIO_WAKEUP
+	#ifdef CONFIG_PLATFORM_ARM_SUN6I
+	eint_wlan_handle = sw_gpio_irq_request(gpio_eint_wlan, TRIG_EDGE_NEGATIVE, (peint_handle)gpio_hostwakeup_irq_thread, NULL);
+	if (!eint_wlan_handle) {
+		RTW_INFO("%s: request irq failed\n", __func__);
+		goto os_ndevs_deinit;
+	}
+	#else
+	gpio_hostwakeup_alloc_irq(padapter);
+	#endif
+#endif/*CONFIG_GPIO_WAKEUP*/
+
+#ifdef CONFIG_GLOBAL_UI_PID
+	if (ui_pid[1] != 0) {
+		RTW_INFO("ui_pid[1]:%d\n", ui_pid[1]);
+		rtw_signal_process(ui_pid[1], SIGUSR2);
+	}
+#endif
+#ifdef CONFIG_RTW_CSI_NETLINK
+	rtw_csi_nl_init(dvobj);
+#endif
+#ifdef CONFIG_CSI_TIMER_POLLING
+	rtw_csi_poll_init(dvobj);
+#endif
+	RTW_INFO("-%s success\n", __func__);
+
+	return 0; /*_SUCCESS*/
+
+
+os_ndevs_deinit:
+	rtw_os_ndevs_deinit(dvobj);
+
+free_dvobj_data:
+	devobj_data_deinit(dvobj);
+
+free_adapter_link:
+	rtw_adapter_link_deinit(dvobj);
+
+#ifdef CONFIG_CONCURRENT_MODE
+free_if_vir:
+	rtw_drv_stop_vir_ifaces(dvobj);
+	rtw_drv_free_vir_ifaces(dvobj);
+#endif
+	if (padapter)
+		rtw_sdio_primary_adapter_deinit(padapter);
+free_hw:
+	rtw_hw_deinit(dvobj);
+
+free_trx_reso:
+	devobj_trx_resource_deinit(dvobj);
+
+free_dvobj:
+	sdio_dvobj_deinit(func);
+exit:
+	return -ENODEV;
+}
+
+static void rtw_dev_remove(struct sdio_func *func)
+{
+	struct dvobj_priv *dvobj = sdio_get_drvdata(func);
+	struct pwrctrl_priv *pwrctl = dvobj_to_pwrctl(dvobj);
+	_adapter *padapter = dvobj_get_primary_adapter(dvobj);
+
+	RTW_INFO("+%s\n", __func__);
+
+	dvobj->processing_dev_remove = _TRUE;
+
+#ifdef CONFIG_CSI_TIMER_POLLING
+	rtw_csi_poll_timer_cancel(dvobj);
+#endif
+#ifdef CONFIG_RTW_CSI_NETLINK
+	rtw_csi_nl_exit(dvobj);
+#endif
+
+	/* TODO: use rtw_os_ndevs_deinit instead at the first stage of driver's dev deinit function */
+	rtw_os_ndevs_unregister(dvobj);
+
+	if (!dev_is_surprise_removed(dvobj)) {
+		int err;
+
+		/* test surprise remove */
+		sdio_claim_host(func);
+		sdio_readb(func, 0, &err);
+		sdio_release_host(func);
+		if (err == -ENOMEDIUM) {
+			dev_set_surprise_removed(dvobj);
+			RTW_INFO("%s: device had been removed!\n", __func__);
+		}
+	}
+
+#if defined(CONFIG_HAS_EARLYSUSPEND) || defined(CONFIG_ANDROID_POWER)
+	rtw_unregister_early_suspend(pwrctl);
+#endif
+	dev_set_drv_stopped(adapter_to_dvobj(padapter));	/*for stop thread*/
+#if 0 /*#ifdef CONFIG_CORE_CMD_THREAD*/
+	rtw_stop_cmd_thread(padapter);
+#endif
+
+#ifdef CONFIG_CONCURRENT_MODE
+	rtw_drv_stop_vir_ifaces(dvobj);
+#endif
+	rtw_drv_stop_prim_iface(padapter);
+
+	rtw_hw_stop(dvobj);
+	dev_set_surprise_removed(dvobj);
+
+	rtw_adapter_link_deinit(dvobj);
+
+	rtw_sdio_primary_adapter_deinit(padapter);
+
+#ifdef CONFIG_CONCURRENT_MODE
+	rtw_drv_free_vir_ifaces(dvobj);
+#endif
+	rtw_hw_deinit(dvobj);
+	devobj_data_deinit(dvobj);
+	devobj_trx_resource_deinit(dvobj);
+	sdio_dvobj_deinit(func);
+
+	RTW_INFO("-%s done\n", __func__);
+}
+
+static void rtw_dev_shutdown(struct device *dev)
+{
+	struct sdio_func *func = dev_to_sdio_func(dev);
+
+	if (func == NULL)
+		return;
+
+	RTW_INFO("==> %s !\n", __func__);
+
+	rtw_dev_remove(func);
+
+	RTW_INFO("<== %s !\n", __func__);
+}
+
+static int rtw_sdio_suspend(struct device *dev)
+{
+	struct sdio_func *func = dev_to_sdio_func(dev);
+	struct dvobj_priv *psdpriv;
+	struct pwrctrl_priv *pwrpriv = NULL;
+	_adapter *padapter = NULL;
+	struct debug_priv *pdbgpriv = NULL;
+	int ret = 0;
+#ifdef CONFIG_RTW_SDIO_PM_KEEP_POWER
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34))
+	mmc_pm_flag_t pm_flag = 0;
+#endif
+#endif
+
+	if (dev == NULL)
+		goto exit;
+
+	psdpriv = sdio_get_drvdata(func);
+	if (psdpriv == NULL)
+		return ret;
+
+	pwrpriv = dvobj_to_pwrctl(psdpriv);
+	padapter = dvobj_get_primary_adapter(psdpriv);
+	pdbgpriv = &psdpriv->drv_dbg;
+	if (dev_is_drv_stopped(adapter_to_dvobj(padapter))) {
+		RTW_INFO("%s bDriverStopped == _TRUE\n", __func__);
+		rtw_sdio_deinit(adapter_to_dvobj(padapter));
+		#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+		rtw_sdio_free_irq(adapter_to_dvobj(padapter));
+		#endif
+		return ret;
+	}
+
+	if (pwrpriv->bInSuspend == _TRUE) {
+		RTW_INFO("%s bInSuspend = %d\n", __func__, pwrpriv->bInSuspend);
+		pdbgpriv->dbg_suspend_error_cnt++;
+		goto exit;
+	}
+
+	ret = rtw_suspend_common(padapter);
+
+#ifdef CONFIG_RTW_SDIO_PM_KEEP_POWER
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34))
+	/* Android 4.0 don't support WIFI close power */
+	/* or power down or clock will close after wifi resume, */
+	/* this is sprd's bug in Android 4.0, but sprd don't */
+	/* want to fix it. */
+	/* we have test power under 8723as, power consumption is ok */
+	pm_flag = sdio_get_host_pm_caps(func);
+	RTW_INFO("cmd: %s: suspend: PM flag = 0x%x\n", sdio_func_id(func), pm_flag);
+	if (!(pm_flag & MMC_PM_KEEP_POWER)) {
+		RTW_INFO("%s: cannot remain alive while host is suspended\n", sdio_func_id(func));
+		if (pdbgpriv)
+			pdbgpriv->dbg_suspend_error_cnt++;
+		return -ENOSYS;
+	} else {
+		RTW_INFO("cmd: suspend with MMC_PM_KEEP_POWER\n");
+		sdio_set_host_pm_flags(func, MMC_PM_KEEP_POWER);
+	}
+#endif
+#endif
+exit:
+	return ret;
+}
+static int rtw_resume_process(_adapter *padapter)
+{
+	struct pwrctrl_priv *pwrpriv = adapter_to_pwrctl(padapter);
+	struct dvobj_priv *psdpriv = padapter->dvobj;
+	struct debug_priv *pdbgpriv = &psdpriv->drv_dbg;
+
+	if (pwrpriv->bInSuspend == _FALSE) {
+		pdbgpriv->dbg_resume_error_cnt++;
+		RTW_INFO("%s bInSuspend = %d\n", __FUNCTION__, pwrpriv->bInSuspend);
+		if (dev_is_drv_stopped(adapter_to_dvobj(padapter))) {
+			/* interface init */
+			if (rtw_sdio_init(psdpriv) != _SUCCESS) {
+				RTW_WARN("%s rtw_sdio_init fail\n", __FUNCTION__);
+				return -1;
+			}
+			#if !(CONFIG_RTW_SDIO_KEEP_IRQ)
+			if (rtw_sdio_alloc_irq(psdpriv) != _SUCCESS) {
+				RTW_WARN("%s rtw_sdio_alloc_irq fail\n", __FUNCTION__);
+				return -1;
+			}
+			#endif/*CONFIG_RTW_SDIO_KEEP_IRQ*/
+			return 0;
+		}
+		return -1;
+	}
+
+	return rtw_resume_common(padapter);
+}
+
+static int rtw_sdio_resume(struct device *dev)
+{
+	struct sdio_func *func = dev_to_sdio_func(dev);
+	struct dvobj_priv *psdpriv = sdio_get_drvdata(func);
+	struct pwrctrl_priv *pwrpriv = dvobj_to_pwrctl(psdpriv);
+	_adapter *padapter = dvobj_get_primary_adapter(psdpriv);
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	int ret = 0;
+	struct debug_priv *pdbgpriv = &psdpriv->drv_dbg;
+
+	RTW_INFO("==> %s (%s:%d)\n", __FUNCTION__, current->comm, current->pid);
+
+	pdbgpriv->dbg_resume_cnt++;
+
+	if (pwrpriv->wowlan_mode || pwrpriv->wowlan_ap_mode) {
+		rtw_resume_lock_suspend();
+		ret = rtw_resume_process(padapter);
+		rtw_resume_unlock_suspend();
+	} else {
+#ifdef CONFIG_RESUME_IN_WORKQUEUE
+		rtw_resume_in_workqueue(pwrpriv);
+#else
+		if (rtw_is_earlysuspend_registered(pwrpriv)) {
+			/* jeff: bypass resume here, do in late_resume */
+			rtw_set_do_late_resume(pwrpriv, _TRUE);
+		} else {
+			rtw_resume_lock_suspend();
+			ret = rtw_resume_process(padapter);
+			rtw_resume_unlock_suspend();
+		}
+#endif
+	}
+	pmlmeext->last_scan_time = rtw_get_current_time();
+	RTW_INFO("<========  %s return %d\n", __FUNCTION__, ret);
+	return ret;
+
+}
+
+static int __init rtw_drv_entry(void)
+{
+	int ret = 0;
+
+	RTW_PRINT("module init start\n");
+	dump_drv_version(RTW_DBGDUMP);
+#ifdef BTCOEXVERSION
+	RTW_PRINT(DRV_NAME" BT-Coex version = %s\n", BTCOEXVERSION);
+#endif /* BTCOEXVERSION */
+
+#if (defined(CONFIG_RTKM) && defined(CONFIG_RTKM_BUILT_IN))
+	ret = rtkm_prealloc_init();
+	if (ret) {
+		RTW_INFO("%s: pre-allocate memory failed!!(%d)\n", __FUNCTION__,
+			 ret);
+		goto exit;
+	}
+#endif /* CONFIG_RTKM */
+
+	rtw_android_wifictrl_func_add();
+#ifdef CONFIG_GPIO_WAKEUP
+	platform_wifi_get_oob_irq(&oob_irq);
+#endif
+
+	ret = platform_wifi_power_on();
+	if (ret) {
+		RTW_INFO("%s: power on failed!!(%d)\n", __FUNCTION__, ret);
+		ret = -1;
+		goto exit;
+	}
+
+	sdio_drvpriv.drv_registered = _TRUE;
+	rtw_suspend_lock_init();
+	rtw_drv_proc_init();
+	rtw_nlrtw_init();
+	rtw_ndev_notifier_register();
+	rtw_inetaddr_notifier_register();
+	rtw_io_records_init();
+
+	ret = sdio_register_driver(&sdio_drvpriv.rtw_sdio_drv);
+	if (ret != 0) {
+		sdio_drvpriv.drv_registered = _FALSE;
+		rtw_suspend_lock_uninit();
+		rtw_drv_proc_deinit();
+		rtw_nlrtw_deinit();
+		rtw_ndev_notifier_unregister();
+		rtw_inetaddr_notifier_unregister();
+		rtw_io_records_deinit();
+		RTW_INFO("%s: register driver failed!!(%d)\n", __FUNCTION__, ret);
+		goto poweroff;
+	}
+
+	goto exit;
+
+poweroff:
+	platform_wifi_power_off();
+
+exit:
+	RTW_PRINT("module init ret=%d\n", ret);
+	return ret;
+}
+
+static void __exit rtw_drv_halt(void)
+{
+	RTW_PRINT("module exit start\n");
+
+	sdio_drvpriv.drv_registered = _FALSE;
+
+	sdio_unregister_driver(&sdio_drvpriv.rtw_sdio_drv);
+
+	rtw_android_wifictrl_func_del();
+
+	platform_wifi_power_off();
+
+	rtw_suspend_lock_uninit();
+	rtw_drv_proc_deinit();
+	rtw_nlrtw_deinit();
+	rtw_ndev_notifier_unregister();
+	rtw_inetaddr_notifier_unregister();
+	rtw_io_records_deinit();
+
+	RTW_PRINT("module exit success\n");
+
+	rtw_mstat_dump(RTW_DBGDUMP);
+
+#if (defined(CONFIG_RTKM) && defined(CONFIG_RTKM_BUILT_IN))
+	rtkm_prealloc_destroy();
+#elif (defined(CONFIG_RTKM) && defined(CONFIG_RTKM_STANDALONE))
+	rtkm_dump_mstatus(RTW_DBGDUMP);
+#endif /* CONFIG_RTKM */
+}
+
+module_init(rtw_drv_entry);
+module_exit(rtw_drv_halt);
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/sdio_ops_linux.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/sdio_ops_linux.c
new file mode 100644
index 000000000000..9282263dfafe
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/sdio_ops_linux.c
@@ -0,0 +1,1094 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _SDIO_OPS_LINUX_C_
+
+#include <drv_types.h>
+#include <rtw_io_records.h>
+
+static bool rtw_sdio_claim_host_needed(struct sdio_func *func)
+{
+	struct dvobj_priv *dvobj = sdio_get_drvdata(func);
+	struct sdio_data *sdio_data = dvobj_to_sdio(dvobj);
+
+	if (sdio_data->sys_sdio_irq_thd && sdio_data->sys_sdio_irq_thd == current)
+		return _FALSE;
+	return _TRUE;
+}
+
+#ifdef CONFIG_RTW_IO_RECORDS
+
+enum sdio_type {
+	SDIO_CMDT_F0_52_READ,
+	SDIO_CMDT_F0_52_WRITE,
+	SDIO_CMDT_52_READ,
+	SDIO_CMDT_52_WRITE,
+	SDIO_CMDT_53_READ,
+	SDIO_CMDT_53_WRITE,
+	SDIO_CMDT_NUM,
+};
+
+static const char *sdio_type_str[] = {
+	[SDIO_CMDT_F0_52_READ]	= "F052R",
+	[SDIO_CMDT_F0_52_WRITE]	= "F052W",
+	[SDIO_CMDT_52_READ]	= "52R",
+	[SDIO_CMDT_52_WRITE]	= "52W",
+	[SDIO_CMDT_53_READ]	= "53R",
+	[SDIO_CMDT_53_WRITE]	= "53W",
+};
+
+#define SDIO_R_TITLE_FMT "%-9s %-17s %-11s %-5s %-7s %-5s %-4s"
+#define SDIO_R_TITLE_TFMT "%s\t%s\t%s\t%s\t%s\t%s\t%s"
+#define SDIO_R_TITLE_ARG , "seq", "stime", "api_time", "type", "addr", "cnt", "err"
+#define SDIO_R_VALUE_FMT "%9zu %7lld.%09lld %lld.%09lld %-5s 0x%05x %5u %4d"
+#define SDIO_R_VALUE_TFMT "%zu\t%lld.%09lld\t%lld.%09lld\t%s\t0x%05x\t%u\t%d"
+#define SDIO_R_VALUE_ARG \
+		, seq \
+		, rtw_division64(rtw_sptime_to_ns(r->stime), 1000000000), rtw_modular64(rtw_sptime_to_ns(r->stime), 1000000000) \
+		, rtw_division64(rtw_sptime_diff_ns(r->stime, r->etime), 1000000000), rtw_modular64(rtw_sptime_diff_ns(r->stime, r->etime), 1000000000) \
+		, sdio_type_str[r->type] \
+		, r->addr, r->cnt, r->err
+
+#if DBG_RTW_IO_RECORD_DATA_LEN
+#define SDIO_R_TITLE_FMT_DATA " %-2s %-2s %-2s %-2s"
+#define SDIO_R_TITLE_TFMT_DATA "\t%s\t%s\t%s\t%s"
+#define SDIO_R_TITLE_ARG_DATA , "d0", "d1", "d2", "d3"
+#define SDIO_R_VALUE_FMT_DATA " %02x %02x %02x %02x"
+#define SDIO_R_VALUE_TFMT_DATA "\t%02x\t%02x\t%02x\t%02x"
+#define SDIO_R_VALUE_ARG_DATA  , r->cnt > 0 ? r->data[0] : 0, r->cnt > 1 ? r->data[1] : 0, r->cnt > 2 ? r->data[2] : 0, r->cnt > 3 ? r->data[3] : 0
+#else
+#define SDIO_R_TITLE_FMT_DATA ""
+#define SDIO_R_TITLE_TFMT_DATA ""
+#define SDIO_R_TITLE_ARG_DATA
+#define SDIO_R_VALUE_FMT_DATA ""
+#define SDIO_R_VALUE_TFMT_DATA ""
+#define SDIO_R_VALUE_ARG_DATA
+#endif
+
+#if DBG_RTW_IO_RECORD_TASK_INFO
+#define SDIO_R_TITLE_FMT_TASK_INFO " %-15s"
+#define SDIO_R_TITLE_TFMT_TASK_INFO "\t%s"
+#define SDIO_R_TITLE_ARG_TASK_INFO , "task_comm"
+#define SDIO_R_VALUE_FMT_TASK_INFO " %-15s"
+#define SDIO_R_VALUE_TFMT_TASK_INFO "\t%s"
+#define SDIO_R_VALUE_ARG_TASK_INFO , r->task_comm
+#else
+#define SDIO_R_TITLE_FMT_TASK_INFO ""
+#define SDIO_R_TITLE_TFMT_TASK_INFO ""
+#define SDIO_R_TITLE_ARG_TASK_INFO
+#define SDIO_R_VALUE_FMT_TASK_INFO ""
+#define SDIO_R_VALUE_TFMT_TASK_INFO ""
+#define SDIO_R_VALUE_ARG_TASK_INFO
+#endif
+
+#if DBG_RTW_IO_RECORD_CPU_INFO
+#define SDIO_R_TITLE_FMT_CPU_INFO " %-3s"
+#define SDIO_R_TITLE_TFMT_CPU_INFO "\t%s"
+#define SDIO_R_TITLE_ARG_CPU_INFO , "cpu"
+#define SDIO_R_VALUE_FMT_CPU_INFO " %-3d"
+#define SDIO_R_VALUE_TFMT_CPU_INFO "\t%d"
+#define SDIO_R_VALUE_ARG_CPU_INFO , r->cpu
+#else
+#define SDIO_R_TITLE_FMT_CPU_INFO ""
+#define SDIO_R_TITLE_TFMT_CPU_INFO ""
+#define SDIO_R_TITLE_ARG_CPU_INFO
+#define SDIO_R_VALUE_FMT_CPU_INFO ""
+#define SDIO_R_VALUE_TFMT_CPU_INFO ""
+#define SDIO_R_VALUE_ARG_CPU_INFO
+#endif
+
+#define SDIO_R_TITLE_FMT_REL " %-12s %-15s %-15s"
+#define SDIO_R_TITLE_TFMT_REL "\t%s\t%s\t%s"
+#define SDIO_R_TITLE_ARG_REL , "from_last_io", "from_last_rx_io", "from_last_tx_io"
+#define SDIO_R_VALUE_FMT_REL " %2lld.%09lld %5lld.%09lld %5lld.%09lld"
+#define SDIO_R_VALUE_TFMT_REL "\t%lld.%09lld\t%lld.%09lld\t%lld.%09lld"
+#define SDIO_R_VALUE_ARG_REL \
+		, last_io ? rtw_division64(rtw_sptime_diff_ns(last_io->etime, r->stime), 1000000000) : 0 \
+		, last_io ? rtw_modular64(rtw_sptime_diff_ns(last_io->etime, r->stime), 1000000000) : 0 \
+		, last_rx_io ? rtw_division64(rtw_sptime_diff_ns(last_rx_io->etime, r->stime), 1000000000) : 0 \
+		, last_rx_io ? rtw_modular64(rtw_sptime_diff_ns(last_rx_io->etime, r->stime), 1000000000) : 0 \
+		, last_tx_io ? rtw_division64(rtw_sptime_diff_ns(last_tx_io->etime, r->stime), 1000000000) : 0 \
+		, last_tx_io ? rtw_modular64(rtw_sptime_diff_ns(last_tx_io->etime, r->stime), 1000000000) : 0
+
+static void sdio_record_title_dump_tab(void *sel)
+{
+	RTW_PRINT_SEL(sel, SDIO_R_TITLE_TFMT
+		SDIO_R_TITLE_TFMT_DATA
+		SDIO_R_TITLE_TFMT_TASK_INFO
+		SDIO_R_TITLE_TFMT_CPU_INFO
+		SDIO_R_TITLE_TFMT_REL
+		"\n"
+		SDIO_R_TITLE_ARG
+		SDIO_R_TITLE_ARG_DATA
+		SDIO_R_TITLE_ARG_TASK_INFO
+		SDIO_R_TITLE_ARG_CPU_INFO
+		SDIO_R_TITLE_ARG_REL
+		);
+}
+
+static void sdio_record_value_dump_tab(void *sel, struct rtw_io_record *r, size_t seq
+	, struct rtw_io_record *last_io
+	, struct rtw_io_record *last_rx_io
+	, struct rtw_io_record *last_tx_io)
+{
+	RTW_PRINT_SEL(sel, SDIO_R_VALUE_TFMT
+		SDIO_R_VALUE_TFMT_DATA
+		SDIO_R_VALUE_TFMT_TASK_INFO
+		SDIO_R_VALUE_TFMT_CPU_INFO
+		SDIO_R_VALUE_TFMT_REL
+		"\n"
+		SDIO_R_VALUE_ARG
+		SDIO_R_VALUE_ARG_DATA
+		SDIO_R_VALUE_ARG_TASK_INFO
+		SDIO_R_VALUE_ARG_CPU_INFO
+		SDIO_R_VALUE_ARG_REL
+	);
+}
+
+
+static void sdio_record_title_dump(void *sel)
+{
+	RTW_PRINT_SEL(sel, SDIO_R_TITLE_FMT
+		SDIO_R_TITLE_FMT_DATA
+		SDIO_R_TITLE_FMT_TASK_INFO
+		SDIO_R_TITLE_FMT_CPU_INFO
+		SDIO_R_TITLE_FMT_REL
+		"\n"
+		SDIO_R_TITLE_ARG
+		SDIO_R_TITLE_ARG_DATA
+		SDIO_R_TITLE_ARG_TASK_INFO
+		SDIO_R_TITLE_ARG_CPU_INFO
+		SDIO_R_TITLE_ARG_REL
+		);
+}
+
+static void sdio_record_value_dump(void *sel, struct rtw_io_record *r, size_t seq
+	, struct rtw_io_record *last_io
+	, struct rtw_io_record *last_rx_io
+	, struct rtw_io_record *last_tx_io)
+{
+	RTW_PRINT_SEL(sel, SDIO_R_VALUE_FMT
+		SDIO_R_VALUE_FMT_DATA
+		SDIO_R_VALUE_FMT_TASK_INFO
+		SDIO_R_VALUE_FMT_CPU_INFO
+		SDIO_R_VALUE_FMT_REL
+		"\n"
+		SDIO_R_VALUE_ARG
+		SDIO_R_VALUE_ARG_DATA
+		SDIO_R_VALUE_ARG_TASK_INFO
+		SDIO_R_VALUE_ARG_CPU_INFO
+		SDIO_R_VALUE_ARG_REL
+	);
+}
+
+typedef void (*sdio_rec_title_dump)(void *);
+typedef void (*sdio_rec_value_dump)(void *, struct rtw_io_record *, size_t, struct rtw_io_record *, struct rtw_io_record *, struct rtw_io_record *);
+
+static bool rtw_sdio_record_is_himr(struct rtw_io_record *r)
+{
+	/* TODO: judge by HAL */
+	return r->addr == 0x01100;
+}
+
+static bool rtw_sdio_record_is_hisr(struct rtw_io_record *r)
+{
+	/* TODO: judge by HAL */
+	return r->addr == 0x01104;
+}
+
+static bool rtw_sdio_record_is_rx(struct rtw_io_record *r)
+{
+	/* TODO: judge by HAL */
+	return r->addr == 0x01f00;
+}
+
+static bool rtw_sdio_record_is_rx_aval(struct rtw_io_record *r)
+{
+	/* TODO: judge by HAL */
+	return r->addr == 0x01108;
+}
+
+static bool rtw_sdio_record_is_tx(struct rtw_io_record *r)
+{
+	/* TODO: judge by HAL */
+	return (r->addr & BIT16) && r->cnt > 4;
+}
+
+static bool rtw_sdio_record_is_tx_aval(struct rtw_io_record *r)
+{
+	/* TODO: judge by HAL */
+	return r->addr == 0x01110;
+}
+
+void rtw_sdio_records_dump_title(void *sel, bool tab)
+{
+	sdio_rec_title_dump title_dump = tab ? sdio_record_title_dump_tab : sdio_record_title_dump;
+
+	title_dump(sel);
+}
+
+void rtw_sdio_records_dump_value_by_seq(void *sel, bool tab, size_t seq)
+{
+	size_t oldest_pos = dbg_rtw_io_records.record[dbg_rtw_io_records.pos].valid ? dbg_rtw_io_records.pos : 0;
+	struct rtw_io_record *record = &dbg_rtw_io_records.record[(oldest_pos + seq) % dbg_rtw_io_records.record_num];
+
+	if (record->valid) {
+		sdio_rec_value_dump value_dump = tab ? sdio_record_value_dump_tab : sdio_record_value_dump;
+		bool rx_io = rtw_sdio_record_is_rx(record);
+		bool tx_io = rtw_sdio_record_is_tx(record);
+		struct rtw_io_record *last_io = NULL;
+		struct rtw_io_record *last_rx_io = NULL;
+		struct rtw_io_record *last_tx_io = NULL;
+
+		if (seq != 0) {
+			last_io = &dbg_rtw_io_records.record[(oldest_pos + seq - 1) % dbg_rtw_io_records.record_num];
+
+			if (rx_io || tx_io) {
+				struct rtw_io_record *r;
+				size_t i;
+
+				/* find last_rx_io or last_tx_io */
+				for (i = 1; i <= seq ; i++) {
+					r = &dbg_rtw_io_records.record[(oldest_pos + seq - i) % dbg_rtw_io_records.record_num];
+					if (rx_io && rtw_sdio_record_is_rx(r)) {
+						last_rx_io = r;
+						break;
+					}
+					if (tx_io && rtw_sdio_record_is_tx(r)) {
+						last_tx_io = r;
+						break;
+					}
+				}
+			}
+		}
+		value_dump(sel, record, seq, last_io, rx_io ? last_rx_io : NULL, tx_io ? last_tx_io : NULL);
+	}
+}
+
+void rtw_sdio_records_dump(void *sel, bool tab)
+{
+	struct rtw_io_record *record;
+	struct rtw_io_record *last_io = NULL;
+	struct rtw_io_record *last_rx_io = NULL;
+	struct rtw_io_record *last_tx_io = NULL;
+	sdio_rec_title_dump title_dump = tab ? sdio_record_title_dump_tab : sdio_record_title_dump;
+	sdio_rec_value_dump value_dump = tab ? sdio_record_value_dump_tab : sdio_record_value_dump;
+	bool rx_io, tx_io;
+	size_t oldest_pos = dbg_rtw_io_records.record[dbg_rtw_io_records.pos].valid ? dbg_rtw_io_records.pos : 0;
+	size_t i;
+
+	title_dump(sel);
+
+	for (i = 0; i < dbg_rtw_io_records.record_num; i++) {
+		record = &dbg_rtw_io_records.record[(oldest_pos + i) % dbg_rtw_io_records.record_num];
+		if (!record->valid)
+			break;
+		rx_io = rtw_sdio_record_is_rx(record);
+		tx_io = rtw_sdio_record_is_tx(record);
+		value_dump(sel, record, i, last_io, rx_io ? last_rx_io : NULL, tx_io ? last_tx_io : NULL);
+		last_io = record;
+		if (rx_io)
+			last_rx_io = record;
+		if (tx_io)
+			last_tx_io = record;
+	}
+}
+
+struct sdio_records_summary {
+	sysptime stime;
+	sysptime etime;
+
+	size_t io_num;
+	size_t himr_io_num;
+	size_t hisr_io_num;
+	size_t rx_io_num;
+	size_t rx_aval_io_num;
+	size_t tx_io_num;
+	size_t tx_aval_io_num;
+	size_t other_io_num;
+
+	sysptime io_time; /* the total IO time */
+	sysptime himr_io_time;
+	sysptime hisr_io_time;
+	sysptime rx_io_time;
+	sysptime rx_aval_io_time;
+	sysptime tx_io_time;
+	sysptime tx_aval_io_time;
+	sysptime other_io_time;
+
+	sysptime rx_io_int_total;
+	sysptime rx_io_int_min;
+	sysptime rx_io_int_max;
+	sysptime rx_io_int_avg;
+	sysptime tx_io_int_total;
+	sysptime tx_io_int_min;
+	sysptime tx_io_int_max;
+	sysptime tx_io_int_avg;
+
+	u64 rx_io_len_total;
+	u32 rx_io_len_min;
+	u32 rx_io_len_max;
+	u64 rx_io_len_avg;
+	u64 tx_io_len_total;
+	u32 tx_io_len_min;
+	u32 tx_io_len_max;
+	u64 tx_io_len_avg;
+};
+
+void rtw_sdio_records_summary_dump(void *sel)
+{
+	struct sdio_records_summary data;
+	struct rtw_io_record *record;
+	struct rtw_io_record *last_io = NULL;
+	struct rtw_io_record *last_rx_io = NULL;
+	struct rtw_io_record *last_tx_io = NULL;
+	bool himr_io, hisr_io, rx_io, rx_aval_io, tx_io, tx_aval_io;
+	size_t oldest_pos = dbg_rtw_io_records.record[dbg_rtw_io_records.pos].valid ? dbg_rtw_io_records.pos : 0;
+	size_t i;
+
+	_rtw_memset(&data, 0, sizeof(data));
+	data.io_time = rtw_sptime_zero();
+	data.himr_io_time = rtw_sptime_zero();
+	data.hisr_io_time = rtw_sptime_zero();
+	data.rx_io_time = rtw_sptime_zero();
+	data.tx_io_time = rtw_sptime_zero();
+	data.rx_aval_io_time = rtw_sptime_zero();
+	data.tx_aval_io_time = rtw_sptime_zero();
+	data.rx_io_int_total = rtw_sptime_zero();
+	data.tx_io_int_total = rtw_sptime_zero();
+
+	for (i = 0; i < dbg_rtw_io_records.record_num; i++) {
+		record = &dbg_rtw_io_records.record[(oldest_pos + i) % dbg_rtw_io_records.record_num];
+		if (!record->valid)
+			break;
+		himr_io = rtw_sdio_record_is_himr(record);
+		hisr_io = rtw_sdio_record_is_hisr(record);
+		rx_io = rtw_sdio_record_is_rx(record);
+		rx_aval_io = rtw_sdio_record_is_rx_aval(record);
+		tx_io = rtw_sdio_record_is_tx(record);
+		tx_aval_io = rtw_sdio_record_is_tx_aval(record);
+
+		if (i == 0)
+			data.stime = record->stime;
+		data.etime = record->etime;
+
+		data.io_num++;
+		data.io_time = rtw_sptime_add(data.io_time, rtw_sptime_sub(record->etime, record->stime));
+		last_io = record;
+		if (himr_io) {
+			data.himr_io_num++;
+			data.himr_io_time = rtw_sptime_add(data.himr_io_time, rtw_sptime_sub(record->etime, record->stime));
+		}
+		if (hisr_io) {
+			data.hisr_io_num++;
+			data.hisr_io_time = rtw_sptime_add(data.hisr_io_time, rtw_sptime_sub(record->etime, record->stime));
+		}
+		if (rx_io) {
+			data.rx_io_len_total += record->cnt;
+			if (data.rx_io_num == 0 || data.rx_io_len_min > record->cnt)
+				data.rx_io_len_min = record->cnt;
+			if (data.rx_io_num == 0 || data.rx_io_len_max < record->cnt)
+				data.rx_io_len_max = record->cnt;
+			data.rx_io_time = rtw_sptime_add(data.rx_io_time, rtw_sptime_sub(record->etime, record->stime));
+			if (last_rx_io) {
+				data.rx_io_int_total = rtw_sptime_add(data.rx_io_int_total, rtw_sptime_sub(record->stime, last_rx_io->etime));
+				if (data.rx_io_num == 1 || rtw_sptime_after(data.rx_io_int_min, rtw_sptime_sub(record->stime, last_rx_io->etime)))
+					data.rx_io_int_min = rtw_sptime_sub(record->stime, last_rx_io->etime);
+				if (data.rx_io_num == 1 || rtw_sptime_before(data.rx_io_int_max, rtw_sptime_sub(record->stime, last_rx_io->etime)))
+					data.rx_io_int_max = rtw_sptime_sub(record->stime, last_rx_io->etime);
+			}
+			data.rx_io_num++;
+			last_rx_io = record;
+		}
+		if (rx_aval_io) {
+			data.rx_aval_io_time = rtw_sptime_add(data.rx_aval_io_time, rtw_sptime_sub(record->etime, record->stime));
+			data.rx_aval_io_num++;
+		}
+		if (tx_io) {
+			data.tx_io_len_total += record->cnt;
+			if (data.tx_io_num == 0 || data.tx_io_len_min > record->cnt)
+				data.tx_io_len_min = record->cnt;
+			if (data.tx_io_num == 0 || data.tx_io_len_max < record->cnt)
+				data.tx_io_len_max = record->cnt;
+			data.tx_io_time = rtw_sptime_add(data.tx_io_time, rtw_sptime_sub(record->etime, record->stime));
+			if (last_tx_io) {
+				data.tx_io_int_total = rtw_sptime_add(data.tx_io_int_total, rtw_sptime_sub(record->stime, last_tx_io->etime));
+				if (data.tx_io_num == 1 || rtw_sptime_after(data.tx_io_int_min, rtw_sptime_sub(record->stime, last_tx_io->etime)))
+					data.tx_io_int_min = rtw_sptime_sub(record->stime, last_tx_io->etime);
+				if (data.tx_io_num == 1 || rtw_sptime_before(data.tx_io_int_max, rtw_sptime_sub(record->stime, last_tx_io->etime)))
+					data.tx_io_int_max = rtw_sptime_sub(record->stime, last_tx_io->etime);
+			}
+			data.tx_io_num++;
+			last_tx_io = record;
+		}
+		if (tx_aval_io) {
+			data.tx_aval_io_time = rtw_sptime_add(data.tx_aval_io_time, rtw_sptime_sub(record->etime, record->stime));
+			data.tx_aval_io_num++;
+		}
+	}
+
+	data.other_io_num = data.io_num;
+	data.other_io_num = data.other_io_num - data.himr_io_num;
+	data.other_io_num = data.other_io_num - data.hisr_io_num;
+	data.other_io_num = data.other_io_num - data.rx_io_num;
+	data.other_io_num = data.other_io_num - data.tx_io_num;
+	data.other_io_num = data.other_io_num - data.rx_aval_io_num;
+	data.other_io_num = data.other_io_num - data.tx_aval_io_num;
+
+	data.other_io_time = data.io_time;
+	data.other_io_time = rtw_sptime_sub(data.other_io_time, data.himr_io_time);
+	data.other_io_time = rtw_sptime_sub(data.other_io_time, data.hisr_io_time);
+	data.other_io_time = rtw_sptime_sub(data.other_io_time, data.rx_io_time);
+	data.other_io_time = rtw_sptime_sub(data.other_io_time, data.tx_io_time);
+	data.other_io_time = rtw_sptime_sub(data.other_io_time, data.rx_aval_io_time);
+	data.other_io_time = rtw_sptime_sub(data.other_io_time, data.tx_aval_io_time);
+
+	data.rx_io_int_avg = data.rx_io_num ? rtw_ns_to_sptime(rtw_division64(rtw_sptime_to_ns(data.rx_io_int_total), (data.rx_io_num - 1))) : rtw_sptime_zero();
+	data.tx_io_int_avg = data.tx_io_num ? rtw_ns_to_sptime(rtw_division64(rtw_sptime_to_ns(data.tx_io_int_total), (data.tx_io_num - 1))) : rtw_sptime_zero();
+
+	data.rx_io_len_avg = rtw_division64(data.rx_io_len_total * 1000, rtw_sptime_diff_ms(data.stime, data.etime));
+	data.tx_io_len_avg = rtw_division64(data.tx_io_len_total * 1000, rtw_sptime_diff_ms(data.stime, data.etime));
+
+#define SEC_FMT "%lld.%09lld"
+#define SEC_ARG(ns) rtw_division64(ns, 1000000000), rtw_modular64(ns, 1000000000)
+#define PCT2_FMT "%2u.%02u%%"
+#define PCT2_ARG(n, d) (u8)rtw_division64((n) * 100, d), (u8)rtw_modular64(rtw_division64((n) * 10000, d), 100)
+
+	RTW_PRINT_SEL(sel,
+		"         io_num: %zu\n"
+		"    himr_io_num: %zu\n"
+		"    hisr_io_num: %zu\n"
+		"      rx_io_num: %zu\n"
+		"      tx_io_num: %zu\n"
+		" rx_aval_io_num: %zu\n"
+		" tx_aval_io_num: %zu\n"
+		"   other_io_num: %zu\n"
+		"           time: "SEC_FMT"\n"
+		"        io_time: "SEC_FMT"\n"
+		"   himr_io_time: "SEC_FMT" ("PCT2_FMT")\n"
+		"   hisr_io_time: "SEC_FMT" ("PCT2_FMT")\n"
+		"     rx_io_time: "SEC_FMT" ("PCT2_FMT")\n"
+		"     tx_io_time: "SEC_FMT" ("PCT2_FMT")\n"
+		"rx_aval_io_time: "SEC_FMT" ("PCT2_FMT")\n"
+		"tx_aval_io_time: "SEC_FMT" ("PCT2_FMT")\n"
+		"  other_io_time: "SEC_FMT" ("PCT2_FMT")\n"
+		"rx_io_int total: "SEC_FMT"\n"
+		"            min: "SEC_FMT"\n"
+		"            max: "SEC_FMT"\n"
+		"            avg: "SEC_FMT"\n"
+		"tx_io_int total: "SEC_FMT"\n"
+		"            min: "SEC_FMT"\n"
+		"            max: "SEC_FMT"\n"
+		"            avg: "SEC_FMT"\n"
+		"rx_io_len total: %llu\n"
+		"            min: %u\n"
+		"            max: %u\n"
+		"            avg: %llu\n"
+		"tx_io_len total: %llu\n"
+		"            min: %u\n"
+		"            max: %u\n"
+		"            avg: %llu\n"
+		"          rx_tp: %8llu Kbps\n"
+		"          tx_tp: %8llu Kbps\n"
+		, data.io_num
+		, data.himr_io_num
+		, data.hisr_io_num
+		, data.rx_io_num
+		, data.tx_io_num
+		, data.rx_aval_io_num
+		, data.tx_aval_io_num
+		, data.other_io_num
+
+		, SEC_ARG(rtw_sptime_diff_ns(data.stime, data.etime))
+		, SEC_ARG(rtw_sptime_to_ns(data.io_time))
+		, SEC_ARG(rtw_sptime_to_ns(data.himr_io_time)), PCT2_ARG(rtw_sptime_to_ns(data.himr_io_time), rtw_sptime_to_ns(data.io_time))
+		, SEC_ARG(rtw_sptime_to_ns(data.hisr_io_time)), PCT2_ARG(rtw_sptime_to_ns(data.hisr_io_time), rtw_sptime_to_ns(data.io_time))
+		, SEC_ARG(rtw_sptime_to_ns(data.rx_io_time)), PCT2_ARG(rtw_sptime_to_ns(data.rx_io_time), rtw_sptime_to_ns(data.io_time))
+		, SEC_ARG(rtw_sptime_to_ns(data.tx_io_time)), PCT2_ARG(rtw_sptime_to_ns(data.tx_io_time), rtw_sptime_to_ns(data.io_time))
+		, SEC_ARG(rtw_sptime_to_ns(data.rx_aval_io_time)), PCT2_ARG(rtw_sptime_to_ns(data.rx_aval_io_time), rtw_sptime_to_ns(data.io_time))
+		, SEC_ARG(rtw_sptime_to_ns(data.tx_aval_io_time)), PCT2_ARG(rtw_sptime_to_ns(data.tx_aval_io_time), rtw_sptime_to_ns(data.io_time))
+		, SEC_ARG(rtw_sptime_to_ns(data.other_io_time)), PCT2_ARG(rtw_sptime_to_ns(data.other_io_time), rtw_sptime_to_ns(data.io_time))
+
+		, SEC_ARG(rtw_sptime_to_ns(data.rx_io_int_total))
+		, SEC_ARG(rtw_sptime_to_ns(data.rx_io_int_min))
+		, SEC_ARG(rtw_sptime_to_ns(data.rx_io_int_max))
+		, SEC_ARG(rtw_sptime_to_ns(data.rx_io_int_avg))
+
+		, SEC_ARG(rtw_sptime_to_ns(data.tx_io_int_total))
+		, SEC_ARG(rtw_sptime_to_ns(data.tx_io_int_min))
+		, SEC_ARG(rtw_sptime_to_ns(data.tx_io_int_max))
+		, SEC_ARG(rtw_sptime_to_ns(data.tx_io_int_avg))
+
+		, data.rx_io_len_total
+		, data.rx_io_len_min
+		, data.rx_io_len_max
+		, rtw_division64(data.rx_io_len_total, data.rx_io_num)
+
+		, data.tx_io_len_total
+		, data.tx_io_len_min
+		, data.tx_io_len_max
+		, rtw_division64(data.tx_io_len_total, data.tx_io_num)
+
+		, rtw_division64(data.rx_io_len_avg * 8, 1024)
+		, rtw_division64(data.tx_io_len_avg * 8, 1024)
+		);
+}
+
+void rtw_sdio_records_claim_and_enable(struct dvobj_priv *d, bool enable)
+{
+	struct sdio_func *func;
+	bool claim_needed;
+
+#if !CONFIG_RTW_IO_RECORDS_STATIC
+	if (!dbg_rtw_io_records.record) {
+		rtw_warn_on(1);
+		return;
+	}
+#endif
+
+	func = dvobj_to_sdio_func(d);
+	claim_needed = rtw_sdio_claim_host_needed(func);
+	if (claim_needed)
+		sdio_claim_host(func);
+
+	RTW_INFO("%s enable:%d\n", __func__, enable);
+	dbg_rtw_io_records.enable = enable;
+
+	if (claim_needed)
+		sdio_release_host(func);
+}
+#endif /* CONFIG_RTW_IO_RECORDS */
+
+/*#define RTW_SDIO_DUMP*/
+#ifdef RTW_SDIO_DUMP
+#define DUMP_LEN_LMT	0	/* buffer dump size limit */
+				/* unit: byte, 0 for no limit */
+#else
+#define DUMP_LEN_LMT	32
+#endif
+#define GET_DUMP_LEN(len)	(DUMP_LEN_LMT ? rtw_min(len, DUMP_LEN_LMT) : len)
+
+#ifdef DBG_SDIO
+#if (DBG_SDIO >= 1)
+static void sdio_dump_reg_by_cmd52(struct dvobj_priv *d,
+				   u32 addr, size_t len, u8 *buf)
+{
+	struct sdio_func *func;
+	size_t i;
+	u8 val;
+	u8 str[80], used = 0;
+	u8 read_twice = 0;
+	int error;
+
+
+	if (buf)
+		_rtw_memset(buf, 0xAE, len);
+	func = dvobj_to_sdio_func(d);
+	/*
+	 * When register is WLAN IOREG,
+	 * read twice to guarantee the result is correct.
+	 */
+	if (addr & 0x10000)
+		read_twice = 1;
+
+	_rtw_memset(str, 0, 80);
+	used = 0;
+	if (addr & 0xF) {
+		used += snprintf(str+used, (80-used), "0x%02x:\t", addr&~0xF);
+		used += snprintf(str+used, (80-used), "%*s", (addr&0xF)*5, "");
+	}
+	for (i = 0; i < len; i++, addr++) {
+		val = sdio_readb(func, addr, &error);
+		if (read_twice)
+			val = sdio_readb(func, addr, &error);
+		if (error)
+			break;
+
+		if (buf)
+			buf[i] = val;
+
+		if (!(addr & 0xF))
+			used += snprintf(str+used, (80-used), "0x%02x:\t", addr&~0xF);
+		used += snprintf(str+used, (80-used), "%02x ", val);
+		if (((i + 1) < len) && ((addr + 1) & 0xF) == 0) {
+			dev_err(&func->dev, "%s", str);
+			_rtw_memset(str, 0, 80);
+			used = 0;
+		}
+	}
+
+	if (used) {
+		dev_err(&func->dev, "%s", str);
+		_rtw_memset(str, 0, 80);
+		used = 0;
+	}
+
+	if (error)
+		dev_err(&func->dev, "rtw_sdio_dbg: READ 0x%02x FAIL!", addr);
+}
+
+static void sdio_dump_cia(struct dvobj_priv *d, u32 addr, size_t len, u8 *buf)
+{
+	struct sdio_func *func;
+	size_t i;
+	u8 val;
+	u8 str[80], used = 0;
+	int error;
+
+
+	if (buf)
+		_rtw_memset(buf, 0xAE, len);
+	func = dvobj_to_sdio_func(d);
+
+	_rtw_memset(str, 0, 80);
+	used = 0;
+	if (addr & 0xF) {
+		used += snprintf(str+used, (80-used), "0x%02x:\t", addr&~0xF);
+		used += snprintf(str+used, (80-used), "%*s", (addr&0xF)*5, "");
+	}
+	for (i = 0; i < len; i++, addr++) {
+		val = sdio_f0_readb(func, addr, &error);
+		if (error)
+			break;
+
+		if (buf)
+			buf[i] = val;
+
+		if (!(addr & 0xF))
+			used += snprintf(str+used, (80-used), "0x%02x:\t", addr&~0xF);
+		used += snprintf(str+used, (80-used), "%02x ", val);
+		if (((i + 1) < len) && ((addr + 1) & 0xF) == 0) {
+			dev_err(&func->dev, "%s", str);
+			_rtw_memset(str, 0, 80);
+			used = 0;
+		}
+	}
+
+	if (used) {
+		dev_err(&func->dev, "%s", str);
+		_rtw_memset(str, 0, 80);
+		used = 0;
+	}
+
+	if (error)
+		dev_err(&func->dev, "rtw_sdio_dbg: READ CIA 0x%02x FAIL!",
+			addr);
+}
+
+#if (DBG_SDIO >= 2)
+void rtw_sdio_dbg_reg_alloc(struct dvobj_priv *d);
+#endif /* DBG_SDIO >= 2 */
+
+/*
+ * Dump register when CMD53 fail
+ */
+static void sdio_dump_dbg_reg(struct dvobj_priv *d, u8 write,
+			      unsigned int addr, size_t len)
+{
+	struct sdio_data *sdio;
+	struct sdio_func *func;
+	u8 *buf = NULL;
+#if (DBG_SDIO >= 2)
+	u8 *msg;
+#endif /* DBG_SDIO >= 2 */
+
+
+	sdio = dvobj_to_sdio(d);
+	if (sdio->reg_dump_mark)
+		return;
+	func = dvobj_to_sdio_func(d);
+
+	sdio->reg_dump_mark = sdio->cmd53_err_cnt;
+
+#if (DBG_SDIO >= 2)
+	if (!sdio->dbg_msg) {
+		msg = rtw_zmalloc(80);
+		if (msg) {
+			sdio->dbg_msg = msg;
+			sdio->dbg_msg_size = 80;
+		}
+	}
+	if (sdio->dbg_msg_size) {
+		snprintf(sdio->dbg_msg, sdio->dbg_msg_size,
+			 "CMD53 %s 0x%05x, %zu bytes FAIL "
+			 "at err_cnt=%d",
+			 write?"WRITE":"READ",
+			 addr, len, sdio->reg_dump_mark);
+	}
+
+	rtw_sdio_dbg_reg_alloc(d);
+#endif /* DBG_SDIO >= 2 */
+
+	/* MAC register */
+	dev_err(&func->dev, "MAC register:");
+#if (DBG_SDIO >= 2)
+	buf = sdio->reg_mac;
+#endif /* DBG_SDIO >= 2 */
+	sdio_dump_reg_by_cmd52(d, 0x10000, 0x800, buf);
+	dev_err(&func->dev, "MAC Extend register:");
+#if (DBG_SDIO >= 2)
+	buf = sdio->reg_mac_ext;
+#endif /* DBG_SDIO >= 2 */
+	sdio_dump_reg_by_cmd52(d, 0x11000, 0x800, buf);
+
+	/* SDIO local register */
+	dev_err(&func->dev, "SDIO Local register:");
+#if (DBG_SDIO >= 2)
+	buf = sdio->reg_local;
+#endif /* DBG_SDIO >= 2 */
+	sdio_dump_reg_by_cmd52(d, 0x0, 0x100, buf);
+
+	/* F0 */
+	dev_err(&func->dev, "f0 register:");
+#if (DBG_SDIO >= 2)
+	buf = sdio->reg_cia;
+#endif /* DBG_SDIO >= 2 */
+	sdio_dump_cia(d, 0x0, 0x200, buf);
+}
+#endif /* DBG_SDIO >= 1 */
+#endif /* DBG_SDIO */
+
+/**
+ *	Returns driver error code,
+ *	0	no error
+ *	-1	Level 1 error, critical error and can't be recovered
+ *	-2	Level 2 error, normal error, retry to recover is possible
+ */
+static int linux_io_err_to_drv_err(int err)
+{
+	if (!err)
+		return 0;
+
+	/* critical error */
+	if ((err == -ESHUTDOWN) ||
+	    (err == -ENODEV) ||
+	    (err == -ENOMEDIUM))
+		return -1;
+
+	/* other error */
+	return -2;
+}
+
+/**
+ *	rtw_sdio_raw_read - Read from SDIO device
+ *	@d: driver object private data
+ *	@addr: address to read
+ *	@buf: buffer to store the data
+ *	@len: number of bytes to read
+ *	@fixed:
+ *
+ *	Reads from the address space of a SDIO device.
+ *	Return value indicates if the transfer succeeded or not.
+ */
+int __must_check rtw_sdio_raw_read(struct dvobj_priv *d, unsigned int addr,
+				   void *buf, size_t len, bool fixed)
+{
+	int error = -EPERM;
+	bool f0, cmd52;
+	struct sdio_func *func;
+	bool claim_needed;
+	u32 offset, i;
+	struct sdio_data *sdio;
+	u8 *tmpbuf = NULL;
+	DECLARE_DBG_RTW_IO_RECORD_P(r);
+
+	func = dvobj_to_sdio_func(d);
+	claim_needed = rtw_sdio_claim_host_needed(func);
+	f0 = RTW_SDIO_ADDR_F0_CHK(addr);
+	cmd52 = RTW_SDIO_ADDR_CMD52_CHK(addr);
+	sdio = dvobj_to_sdio(d);
+
+	/*
+	 * Mask addr to remove driver defined bit and
+	 * make sure addr is in valid range
+	 */
+	if (f0)
+		addr &= 0xFFF;
+	else
+		addr &= 0x1FFFF;
+
+#ifdef RTW_SDIO_DUMP
+	if (f0)
+		dev_dbg(&func->dev, "RTW_SDIO: READ F0\n");
+	else if (cmd52)
+		dev_dbg(&func->dev, "RTW_SDIO: READ use CMD52\n");
+	else
+		dev_dbg(&func->dev, "RTW_SDIO: READ use CMD53\n");
+
+	dev_dbg(&func->dev, "RTW_SDIO: READ from 0x%05x\n", addr);
+#endif /* RTW_SDIO_DUMP */
+
+	if (claim_needed)
+		sdio_claim_host(func);
+
+	if (f0) {
+		offset = addr;
+		for (i = 0; i < len; i++, offset++) {
+			dbg_rtw_io_record_get_and_advance(r);
+			dbg_rtw_io_record_fill_stime(r);
+			((u8 *)buf)[i] = sdio_f0_readb(func, offset, &error);
+			dbg_rtw_io_record_fill_1(r, SDIO_CMDT_F0_52_READ, offset, error, ((u8 *)buf) + i);
+			if (error)
+				break;
+#if 0
+			dev_info(&func->dev, "%s: sdio f0 read 52 addr 0x%x, byte 0x%02x\n",
+				 __func__, offset, ((u8 *)buf)[i]);
+#endif
+		}
+	} else {
+		if (cmd52) {
+#ifdef RTW_SDIO_IO_DBG
+			dev_info(&func->dev, "%s: sdio read 52 addr 0x%x, %zu bytes\n",
+				 __func__, addr, len);
+#endif
+			offset = addr;
+			for (i = 0; i < len; i++) {
+				dbg_rtw_io_record_get_and_advance(r);
+				dbg_rtw_io_record_fill_stime(r);
+				((u8 *)buf)[i] = sdio_readb(func, offset, &error);
+				dbg_rtw_io_record_fill_1(r, SDIO_CMDT_52_READ, offset, error, ((u8 *)buf) + i);
+				if (error)
+					break;
+#if 0
+				dev_info(&func->dev, "%s: sdio read 52 addr 0x%x, byte 0x%02x\n",
+					 __func__, offset, ((u8 *)buf)[i]);
+#endif
+				if (!fixed)
+					offset++;
+			}
+		} else {
+#ifdef RTW_SDIO_IO_DBG
+			dev_info(&func->dev, "%s: sdio read 53 addr 0x%x, %zu bytes\n",
+				 __func__, addr, len);
+#endif
+			if (len <= sdio->tmpbuf_sz) {
+				tmpbuf = buf;
+				buf = sdio->tmpbuf;
+			}
+			if (fixed) {
+				dbg_rtw_io_record_get_and_advance(r);
+				dbg_rtw_io_record_fill_stime(r);
+				error = sdio_readsb(func, buf, addr, len);
+				dbg_rtw_io_record_fill_n(r, SDIO_CMDT_53_READ, addr, len, error, buf);
+			} else {
+				dbg_rtw_io_record_get_and_advance(r);
+				dbg_rtw_io_record_fill_stime(r);
+				error = sdio_memcpy_fromio(func, buf, addr, len);
+				dbg_rtw_io_record_fill_n(r, SDIO_CMDT_53_READ, addr, len, error, buf);
+			}
+			if (!error && tmpbuf)
+				_rtw_memcpy(tmpbuf, buf, len);
+		}
+	}
+
+#ifdef DBG_SDIO
+#if (DBG_SDIO >= 3)
+	if (!error && !f0 && !cmd52
+	    && (sdio->dbg_enable
+		&& sdio->err_test && !sdio->err_test_triggered
+		&& ((addr & 0x10000)
+		    || (!(addr & 0xE000)
+			&& !((addr >= 0x40) && (addr < 0x48)))))) {
+		sdio->err_test_triggered = 1;
+		error = -ETIMEDOUT;
+		dev_warn(&func->dev, "Simulate error(%d) READ addr=0x%05x %zu bytes",
+			 error, addr, len);
+	}
+#endif /* DBG_SDIO >= 3 */
+
+	if (error) {
+		if (f0 || cmd52) {
+			sdio->cmd52_err_cnt++;
+		} else {
+			sdio->cmd53_err_cnt++;
+#if (DBG_SDIO >= 1)
+			sdio_dump_dbg_reg(d, 0, addr, len);
+#endif /* DBG_SDIO >= 1 */
+		}
+	}
+#endif /* DBG_SDIO */
+
+	if (claim_needed)
+		sdio_release_host(func);
+
+#ifdef RTW_SDIO_DUMP
+	print_hex_dump(KERN_DEBUG, "RTW_SDIO: READ ",
+		       DUMP_PREFIX_OFFSET, 16, 1,
+		       buf, GET_DUMP_LEN(len), false);
+#endif /* RTW_SDIO_DUMP */
+
+	if (WARN_ON(error)) {
+		dev_err(&func->dev, "%s: sdio read failed (%d)\n", __func__, error);
+#ifndef RTW_SDIO_DUMP
+		if (f0)
+			dev_err(&func->dev, "RTW_SDIO: READ F0\n");
+		if (cmd52)
+			dev_err(&func->dev, "RTW_SDIO: READ use CMD52\n");
+		else
+			dev_err(&func->dev, "RTW_SDIO: READ use CMD53\n");
+		dev_err(&func->dev, "RTW_SDIO: READ from 0x%05x, %zu bytes\n", addr, len);
+		print_hex_dump(KERN_ERR, "RTW_SDIO: READ ",
+			       DUMP_PREFIX_OFFSET, 16, 1,
+			       buf, GET_DUMP_LEN(len), false);
+#endif /* !RTW_SDIO_DUMP */
+	}
+
+	return linux_io_err_to_drv_err(error);
+}
+
+/**
+ *	rtw_sdio_raw_write - Write to SDIO device
+ *	@d: driver object private data
+ *	@addr: address to write
+ *	@buf: buffer that contains the data to write
+ *	@len: number of bytes to write
+ *	@fixed: address is fixed(FIFO) or incremented
+ *
+ *	Writes to the address space of a SDIO device.
+ *	Return value indicates if the transfer succeeded or not.
+ */
+int __must_check rtw_sdio_raw_write(struct dvobj_priv *d, unsigned int addr,
+				    void *buf, size_t len, bool fixed)
+{
+	int error = -EPERM;
+	bool f0, cmd52;
+	struct sdio_func *func;
+	bool claim_needed;
+	u32 offset, i;
+	struct sdio_data *sdio;
+	DECLARE_DBG_RTW_IO_RECORD_P(r);
+
+	func = dvobj_to_sdio_func(d);
+	claim_needed = rtw_sdio_claim_host_needed(func);
+	f0 = RTW_SDIO_ADDR_F0_CHK(addr);
+	cmd52 = RTW_SDIO_ADDR_CMD52_CHK(addr);
+	sdio = dvobj_to_sdio(d);
+
+	/*
+	 * Mask addr to remove driver defined bit and
+	 * make sure addr is in valid range
+	 */
+	if (f0)
+		addr &= 0xFFF;
+	else
+		addr &= 0x1FFFF;
+
+#ifdef RTW_SDIO_DUMP
+	if (f0)
+		dev_dbg(&func->dev, "RTW_SDIO: WRITE F0\n");
+	else if (cmd52)
+		dev_dbg(&func->dev, "RTW_SDIO: WRITE use CMD52\n");
+	else
+		dev_dbg(&func->dev, "RTW_SDIO: WRITE use CMD53\n");
+	dev_dbg(&func->dev, "RTW_SDIO: WRITE to 0x%05x\n", addr);
+	print_hex_dump(KERN_DEBUG, "RTW_SDIO: WRITE ",
+		       DUMP_PREFIX_OFFSET, 16, 1,
+		       buf, GET_DUMP_LEN(len), false);
+#endif /* RTW_SDIO_DUMP */
+
+	if (claim_needed)
+		sdio_claim_host(func);
+
+	if (f0) {
+		offset = addr;
+		for (i = 0; i < len; i++, offset++) {
+			dbg_rtw_io_record_get_and_advance(r);
+			dbg_rtw_io_record_fill_stime(r);
+			sdio_f0_writeb(func, ((u8 *)buf)[i], offset, &error);
+			dbg_rtw_io_record_fill_1(r, SDIO_CMDT_F0_52_WRITE, offset, error, ((u8 *)buf) + i);
+			if (error)
+				break;
+#if 0
+			dev_info(&func->dev, "%s: sdio f0 write 52 addr 0x%x, byte 0x%02x\n",
+				 __func__, offset, ((u8 *)buf)[i]);
+#endif
+		}
+	} else {
+		if (cmd52) {
+#ifdef RTW_SDIO_IO_DBG
+			dev_info(&func->dev, "%s: sdio write 52 addr 0x%x, %zu bytes\n",
+				 __func__, addr, len);
+#endif
+			offset = addr;
+			for (i = 0; i < len; i++) {
+				dbg_rtw_io_record_get_and_advance(r);
+				dbg_rtw_io_record_fill_stime(r);
+				sdio_writeb(func, ((u8 *)buf)[i], offset, &error);
+				dbg_rtw_io_record_fill_1(r, SDIO_CMDT_52_WRITE, offset, error, ((u8 *)buf) + i);
+				if (error)
+					break;
+#if 0
+				dev_info(&func->dev, "%s: sdio write 52 addr 0x%x, byte 0x%02x\n",
+					 __func__, offset, ((u8 *)buf)[i]);
+#endif
+				if (!fixed)
+					offset++;
+			}
+		} else {
+#ifdef RTW_SDIO_IO_DBG
+			dev_info(&func->dev, "%s: sdio write 53 addr 0x%x, %zu bytes\n",
+				 __func__, addr, len);
+#endif
+			if (len <= sdio->tmpbuf_sz) {
+				_rtw_memcpy(sdio->tmpbuf, buf, len);
+				buf = sdio->tmpbuf;
+			}
+			if (fixed) {
+				dbg_rtw_io_record_get_and_advance(r);
+				dbg_rtw_io_record_fill_stime(r);
+				error = sdio_writesb(func, addr, buf, len);
+				dbg_rtw_io_record_fill_n(r, SDIO_CMDT_53_WRITE, addr, len, error, buf);
+			} else {
+				dbg_rtw_io_record_get_and_advance(r);
+				dbg_rtw_io_record_fill_stime(r);
+				error = sdio_memcpy_toio(func, addr, buf, len);
+				dbg_rtw_io_record_fill_n(r, SDIO_CMDT_53_WRITE, addr, len, error, buf);
+			}
+		}
+	}
+
+#ifdef DBG_SDIO
+	if (error) {
+		if (f0 || cmd52) {
+			sdio->cmd52_err_cnt++;
+		} else {
+			sdio->cmd53_err_cnt++;
+#if (DBG_SDIO >= 1)
+			sdio_dump_dbg_reg(d, 1, addr, len);
+#endif /* DBG_SDIO >= 1 */
+		}
+	}
+#endif /* DBG_SDIO */
+
+	if (claim_needed)
+		sdio_release_host(func);
+
+	if (WARN_ON(error)) {
+		dev_err(&func->dev, "%s: sdio write failed (%d)\n", __func__, error);
+#ifndef RTW_SDIO_DUMP
+		if (f0)
+			dev_err(&func->dev, "RTW_SDIO: WRITE F0\n");
+		if (cmd52)
+			dev_err(&func->dev, "RTW_SDIO: WRITE use CMD52\n");
+		else
+			dev_err(&func->dev, "RTW_SDIO: WRITE use CMD53\n");
+		dev_err(&func->dev, "RTW_SDIO: WRITE to 0x%05x, %zu bytes\n", addr, len);
+		print_hex_dump(KERN_ERR, "RTW_SDIO: WRITE ",
+			       DUMP_PREFIX_OFFSET, 16, 1,
+			       buf, GET_DUMP_LEN(len), false);
+#endif /* !RTW_SDIO_DUMP */
+	}
+
+	return linux_io_err_to_drv_err(error);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/wifi_regd.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/wifi_regd.c
new file mode 100644
index 000000000000..bddb6f672aff
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/wifi_regd.c
@@ -0,0 +1,1406 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2010 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include <drv_types.h>
+
+#ifdef CONFIG_IOCTL_CFG80211
+#if !RTW_PER_ADAPTER_WIPHY
+void rtw_chset_hook_os_channels(struct rtw_chset *chset, void *os_ref)
+{
+	struct wiphy *wiphy = os_ref;
+	struct ieee80211_channel *ch;
+	unsigned int i;
+	u32 freq;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		freq = rtw_bch2freq(chset->chs[i].band, chset->chs[i].ChannelNum);
+		ch = ieee80211_get_channel(wiphy, freq);
+		if (!ch)
+			RTW_WARN(FUNC_WIPHY_FMT" can't get %s ch%u\n", FUNC_WIPHY_ARG(wiphy), band_str(chset->chs[i].band), chset->chs[i].ChannelNum);
+		chset->chs[i].os_chan = ch;
+	}
+}
+#endif
+
+#if CONFIG_RTW_CFG80211_CAC_EVENT
+static void rtw_regd_set_du_chdef(struct wiphy *wiphy)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	struct rtw_chset *chset = dvobj_to_chset(wiphy_to_dvobj(wiphy));
+	struct ieee80211_channel *ch = NULL;
+	int i;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		if (chset->chs[i].os_chan) {
+			ch = chset->chs[i].os_chan;
+			break;
+		}
+	}
+
+	if (ch) {
+		wiphy_data->du_chdef.chan = ch;
+		wiphy_data->du_chdef.width = NL80211_CHAN_WIDTH_20;
+		wiphy_data->du_chdef.center_freq1 = ch->center_freq;
+	}
+}
+
+static int rtw_du_wdev_ndev_open(struct net_device *ndev)
+{
+	return -EPERM;
+}
+
+static const struct net_device_ops rtw_du_wdev_ndev_ops = {
+	.ndo_open = rtw_du_wdev_ndev_open,
+};
+
+static struct wireless_dev *rtw_regd_alloc_du_wdev(struct wiphy *wiphy)
+{
+	struct rtw_netdev_priv_indicator *npi;
+	struct net_device *ndev = NULL;
+	struct wireless_dev *wdev = NULL;
+	int ret = 0;
+
+	ndev = alloc_etherdev(sizeof(struct rtw_netdev_priv_indicator));
+	if (!ndev) {
+		RTW_WARN(FUNC_WIPHY_FMT" allocate ndev fail\n", FUNC_WIPHY_ARG(wiphy));
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	snprintf(ndev->name, IFNAMSIZ, "%s-du", wiphy_name(wiphy));
+
+	ndev->priv_destructor = rtw_ndev_destructor;
+	ndev->netdev_ops = &rtw_du_wdev_ndev_ops;
+
+	npi = netdev_priv(ndev);
+	npi->priv = wiphy_to_adapter(wiphy);
+	npi->sizeof_priv = sizeof(_adapter);
+
+	wdev = rtw_zmalloc(sizeof(struct wireless_dev));
+	if (!wdev) {
+		RTW_WARN(FUNC_WIPHY_FMT" allocate wdev fail\n", FUNC_WIPHY_ARG(wiphy));
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	wdev->wiphy = wiphy;
+	wdev->netdev = ndev;
+	wdev->iftype = NL80211_IFTYPE_AP;
+	ndev->ieee80211_ptr = wdev;
+
+	ret = cfg80211_register_netdevice(ndev);
+
+out:
+	if (ret && wdev) {
+		rtw_mfree(wdev, sizeof(struct wireless_dev));
+		wdev = NULL;
+	}
+
+	if (ret && ndev)
+		free_netdev(ndev);
+
+	return wdev;
+}
+
+void rtw_regd_free_du_wdev(struct wiphy *wiphy)
+{
+	struct dvobj_priv *dvobj = wiphy_to_dvobj(wiphy);
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	u8 rtnl_lock_needed;
+
+	if (!wiphy_data->du_wdev)
+		goto exit;
+
+	rtnl_lock_needed = rtw_rtnl_lock_needed(dvobj);
+	if (rtnl_lock_needed)
+		rtnl_lock();
+
+	cfg80211_unregister_netdevice(wiphy_data->du_wdev->netdev);
+
+	if (rtnl_lock_needed)
+		rtnl_unlock();
+
+	wiphy_data->du_wdev = NULL;
+
+exit:
+	return;
+}
+
+static void rtw_regd_schedule_dfs_chan_update(struct wiphy *wiphy)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+
+	if (!wiphy_data->du_wdev) {
+		wiphy_data->du_wdev = rtw_regd_alloc_du_wdev(wiphy);
+		if (!wiphy_data->du_wdev)
+			return;
+		rtw_regd_set_du_chdef(wiphy);
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0))
+	cfg80211_ch_switch_notify(wiphy_data->du_wdev->netdev, &wiphy_data->du_chdef, 0);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0)) || defined(CONFIG_ACK_5_15_LTS_KERNEL)
+	/* ToDo CONFIG_RTW_MLD */
+	cfg80211_ch_switch_notify(wiphy_data->du_wdev->netdev, &wiphy_data->du_chdef, 0, 0);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2))
+	cfg80211_ch_switch_notify(wiphy_data->du_wdev->netdev, &wiphy_data->du_chdef, 0);
+#else
+	cfg80211_ch_switch_notify(wiphy_data->du_wdev->netdev, &wiphy_data->du_chdef);
+#endif
+}
+
+static void rtw_regd_override_dfs_state(struct wiphy *wiphy, struct get_chplan_resp *chplan, bool non_ocp_only)
+{
+	RT_CHANNEL_INFO *chinfo;
+	struct ieee80211_channel *chan;
+	int i;
+	u8 beaconing_hwband_bmp = 0;
+	bool update = false;
+
+	for (i = HW_BAND_0; i < HW_BAND_MAX; i++)
+		if (HWBAND_AP_NUM(wiphy_to_dvobj(wiphy), i) || HWBAND_MESH_NUM(wiphy_to_dvobj(wiphy), i))
+			beaconing_hwband_bmp |= BIT(i);
+
+	for (i = 0; i < chplan->chs_len; i++) {
+		if (chplan->chs[i].flags & RTW_CHF_DIS)
+			continue;
+		chinfo = &chplan->chs[i];
+		chan = chinfo->os_chan;
+		if (!chan)
+			continue;
+
+		if (CH_IS_NON_OCP(chinfo)
+			&& (chan->dfs_state != NL80211_DFS_UNAVAILABLE
+				|| chan->dfs_state_entered != chinfo->non_ocp_end_time - rtw_ms_to_systime(IEEE80211_DFS_MIN_NOP_TIME_MS))
+		) {
+			chan->dfs_state = NL80211_DFS_UNAVAILABLE;
+			chan->dfs_state_entered = chinfo->non_ocp_end_time
+				- rtw_ms_to_systime(IEEE80211_DFS_MIN_NOP_TIME_MS);
+			update = true;
+		}
+		if (non_ocp_only)
+			continue;
+
+		if (chinfo->cac_done && beaconing_hwband_bmp
+			&& chan->dfs_state != NL80211_DFS_AVAILABLE
+		) {
+			chan->dfs_state = NL80211_DFS_AVAILABLE;
+			update = true;
+		}
+	}
+
+	if (update) {
+		/* trigger cfg80211_sched_dfs_chan_update() */
+		rtw_regd_schedule_dfs_chan_update(wiphy);
+	}
+}
+#endif /* CONFIG_RTW_CFG80211_CAC_EVENT */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+bool rtw_regd_is_wiphy_self_managed(struct wiphy *wiphy)
+{
+	return rtw_rfctl_is_disable_sw_channel_plan(wiphy_to_dvobj(wiphy))
+		|| !REGSTY_REGD_SRC_FROM_OS(dvobj_to_regsty(wiphy_to_dvobj(wiphy)));
+}
+
+static struct ieee80211_reg_rule rtw_get_ieee80211_reg_rule(struct get_chplan_resp *chplan, enum band_type band, int last_end_freq, int start_freq, int end_freq, int next_start_freq, u32 flags)
+{
+	struct ieee80211_reg_rule rule = REG_RULE(
+		start_freq - 10, end_freq + 10, 20, 6, 20,
+		((flags & RTW_CHF_NO_IR) ? NL80211_RRF_NO_IR : 0)
+		| ((flags & RTW_CHF_DFS) ? NL80211_RRF_DFS : 0)
+	);
+
+	int regd_max_bw = 160;
+	int frange_max_bw = 160;
+
+	if (!(chplan->proto_en & CHPLAN_PROTO_EN_AC) || band == BAND_ON_24G)
+		regd_max_bw = 40;
+	/* TODO: !RFCTL_REG_EN_11HT(rfctl) limit to 20MHz  */
+
+	while ((end_freq - start_freq + 20) < frange_max_bw) {
+		frange_max_bw /= 2;
+		if (frange_max_bw == 20)
+			break;
+	}
+	rule.freq_range.max_bandwidth_khz = MHZ_TO_KHZ(rtw_min(regd_max_bw, frange_max_bw));
+
+	if (regd_max_bw > frange_max_bw
+		&& (rtw_freq_consecutive(last_end_freq, start_freq)
+			|| rtw_freq_consecutive(end_freq, next_start_freq)
+		)
+	)
+		rule.flags |= NL80211_RRF_AUTO_BW;
+
+	if (regd_max_bw < 40)
+		rule.flags |= NL80211_RRF_NO_HT40;
+	if (regd_max_bw < 80)
+		rule.flags |= NL80211_RRF_NO_80MHZ;
+	if (regd_max_bw < 160)
+		rule.flags |= NL80211_RRF_NO_160MHZ;
+
+	return rule;
+}
+
+static int rtw_build_wiphy_regd(struct wiphy *wiphy, struct get_chplan_resp *chplan, struct ieee80211_regdomain **regd)
+{
+	int i;
+	RT_CHANNEL_INFO *chinfo;
+	enum band_type start_band, band;
+	int last_end_freq, start_freq, end_freq, freq;
+	u32 start_flags, flags;
+	struct ieee80211_regdomain *r = NULL;
+	int rule_num = 0;
+	bool build = 0;
+
+	if (regd)
+		*regd = NULL;
+
+loop:
+	start_band = BAND_MAX;
+	last_end_freq = 0;
+	for (i = 0; i < chplan->chs_len; i++) {
+		if (chplan->chs[i].flags & RTW_CHF_DIS)
+			continue;
+		chinfo = &chplan->chs[i];
+		freq = rtw_bch2freq(chinfo->band, chinfo->ChannelNum);
+		if (!freq) {
+			RTW_WARN(FUNC_WIPHY_FMT" rtw_bch2freq(%s, %u) fail\n"
+				, FUNC_WIPHY_ARG(wiphy), band_str(chinfo->band), chinfo->ChannelNum);
+			continue;
+		}
+		band = chinfo->band;
+
+		flags = chinfo->flags & RTW_CHF_NO_IR;
+		#if CONFIG_RTW_CFG80211_CAC_EVENT
+		if (wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD)) {
+			if (chinfo->flags & RTW_CHF_DFS)
+				flags |= RTW_CHF_DFS;
+		}
+		#endif
+
+		if (start_band == BAND_MAX) {
+			start_band = band;
+			start_freq = end_freq = freq;
+			start_flags = flags;
+			continue;
+		}
+
+		if (start_band == band
+			&& start_flags == flags
+			&& rtw_freq_consecutive(end_freq, freq)
+		) {
+			end_freq = freq;
+			continue;
+		}
+
+		/* create rule */
+		if (build) {
+			RTW_DBG("add rule_%02d(%s, %d, %d, 0x%x)\n"
+				, r->n_reg_rules, band_str(start_band), start_freq, end_freq, start_flags);
+			r->reg_rules[r->n_reg_rules++] = rtw_get_ieee80211_reg_rule(chplan, start_band
+				, last_end_freq, start_freq, end_freq, freq, start_flags);
+		} else
+			rule_num++;
+
+		/* start a new rule */
+		start_band = band;
+		last_end_freq = end_freq;
+		start_freq = end_freq = freq;
+		start_flags = flags;
+	}
+
+	if (start_band != BAND_MAX) {
+		/* create rule */
+		if (build) {
+			RTW_DBG("add rule_%02d(%s, %d, %d, 0x%x)\n"
+				, r->n_reg_rules, band_str(start_band), start_freq, end_freq, start_flags);
+			r->reg_rules[r->n_reg_rules++] = rtw_get_ieee80211_reg_rule(chplan, start_band
+				, last_end_freq, start_freq, end_freq, 0, start_flags);
+		} else
+			rule_num++;
+	}
+
+	if (!build) {
+		/* switch to build phase */
+		build = 1;
+		if (!regd)
+			goto exit;
+
+		r = rtw_zmalloc(sizeof(**regd) + sizeof(struct ieee80211_reg_rule) * rule_num);
+		if (!r) {
+			rule_num = -1;
+			goto exit;
+		}
+
+		_rtw_memcpy(r->alpha2, chplan->alpha2, 2);
+		r->alpha2[2] = 0;
+		r->dfs_region = NL80211_DFS_UNSET;
+		goto loop;
+	}
+
+	*regd = r;
+
+exit:
+	return rule_num;
+}
+
+static void rtw_regd_disable_no_20mhz_chs(struct wiphy *wiphy)
+{
+	struct ieee80211_supported_band *sband;
+	struct ieee80211_channel *ch;
+	unsigned int i, j;
+
+	for (i = 0; i < NUM_NL80211_BANDS; i++) {
+		sband = wiphy->bands[i];
+		if (!sband)
+			continue;
+		for (j = 0; j < sband->n_channels; j++) {
+			ch = &sband->channels[j];
+			if (!ch)
+				continue;
+			if (ch->flags & IEEE80211_CHAN_NO_20MHZ) {
+				RTW_INFO(FUNC_WIPHY_FMT" disable band:%d ch:%u w/o 20MHz\n", FUNC_WIPHY_ARG(wiphy), ch->band, ch->hw_value);
+				ch->flags = IEEE80211_CHAN_DISABLED;
+			}
+		}
+	}
+}
+
+void rtw_update_wiphy_regd(struct wiphy *wiphy, struct get_chplan_resp *chplan, bool rtnl_lock_needed)
+{
+	struct ieee80211_regdomain *regd;
+	int ret;
+
+	ret = rtw_build_wiphy_regd(wiphy, chplan, &regd);
+	if (ret == -1) {
+		RTW_WARN(FUNC_WIPHY_FMT" rtw_build_wiphy_regd() fail\n", FUNC_WIPHY_ARG(wiphy));
+		return;
+	}
+
+	if (ret == 0) {
+		RTW_WARN(FUNC_WIPHY_FMT" rtw_build_wiphy_regd() builds empty regd, bypass regd setting\n", FUNC_WIPHY_ARG(wiphy));
+		goto free_regd;
+	}
+
+	if (rtnl_lock_needed)
+		rtnl_lock();
+
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0))
+	wiphy_lock(wiphy);
+	ret = regulatory_set_wiphy_regd_sync(wiphy, regd);
+	wiphy_unlock(wiphy);
+	#else
+	ret = regulatory_set_wiphy_regd_sync_rtnl(wiphy, regd);
+	#endif
+
+	rtw_regd_disable_no_20mhz_chs(wiphy);
+
+	#if CONFIG_RTW_CFG80211_CAC_EVENT
+	rtw_regd_override_dfs_state(wiphy, chplan, false);
+	#endif
+
+	if (rtnl_lock_needed)
+		rtnl_unlock();
+
+	if (ret != 0)
+		RTW_INFO(FUNC_WIPHY_FMT" regulatory_set_wiphy_regd_sync_rtnl return %d\n", FUNC_WIPHY_ARG(wiphy), ret);
+
+free_regd:
+	rtw_mfree(regd, sizeof(*regd) + sizeof(struct ieee80211_reg_rule) * regd->n_reg_rules);
+}
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)) */
+
+static void rtw_regd_overide_flags(struct wiphy *wiphy, struct get_chplan_resp *chplan, bool rtnl_lock_needed)
+{
+	RT_CHANNEL_INFO *chs = chplan->chs;
+	u8 chs_len = chplan->chs_len;
+	struct ieee80211_supported_band *sband;
+	struct ieee80211_channel *ch;
+	unsigned int i, j;
+
+	if (rtnl_lock_needed)
+		rtnl_lock();
+
+	/* all channels disable */
+	for (i = 0; i < NUM_NL80211_BANDS; i++) {
+		sband = wiphy->bands[i];
+		if (!sband)
+			continue;
+		for (j = 0; j < sband->n_channels; j++) {
+			ch = &sband->channels[j];
+			if (!ch)
+				continue;
+			ch->flags = IEEE80211_CHAN_DISABLED;
+		}
+	}
+
+	/* channels apply by channel plans. */
+	for (i = 0; i < chs_len; i++) {
+		if (chs[i].flags & RTW_CHF_DIS)
+			continue;
+		#if RTW_PER_ADAPTER_WIPHY
+		ch = ieee80211_get_channel(wiphy, rtw_bch2freq(chs[i].band, chs[i].ChannelNum));
+		#else
+		ch = chs[i].os_chan;
+		#endif
+		if (!ch)
+			continue;
+
+		/* enable */
+		ch->flags = 0;
+
+		if (chs[i].flags & RTW_CHF_DFS) {
+			/*
+			* before integrating with nl80211 flow
+			* bypass IEEE80211_CHAN_RADAR when configured with radar detection
+			* to prevent from hostapd blocking DFS channels
+			*/
+			#ifdef CONFIG_DFS_MASTER
+			if (chplan->dfs_domain == RTW_DFS_REGD_NONE)
+			#endif
+				ch->flags |= IEEE80211_CHAN_RADAR;
+		}
+
+		if (chs[i].flags & RTW_CHF_NO_IR) {
+			#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0))
+			ch->flags |= IEEE80211_CHAN_NO_IBSS | IEEE80211_CHAN_PASSIVE_SCAN;
+			#else
+			ch->flags |= IEEE80211_CHAN_NO_IR;
+			#endif
+		}
+	}
+
+	if (rtnl_lock_needed)
+		rtnl_unlock();
+}
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+static void rtw_regd_apply_dfs_flags(struct get_chplan_resp *chplan, bool rtnl_lock_needed)
+{
+	RT_CHANNEL_INFO *chs = chplan->chs;
+	u8 chs_len = chplan->chs_len;
+	unsigned int i;
+	struct ieee80211_channel *chan;
+
+	if (rtnl_lock_needed)
+		rtnl_lock();
+
+	/* channels apply by channel plans. */
+	for (i = 0; i < chs_len; i++) {
+		if (chs[i].flags & RTW_CHF_DIS)
+			continue;
+		chan = chs[i].os_chan;
+		if (!chan) {
+			rtw_warn_on(1);
+			continue;
+		}
+		if (chs[i].flags & RTW_CHF_DFS) {
+			/*
+			* before integrating with nl80211 flow
+			* clear IEEE80211_CHAN_RADAR when configured with radar detection
+			* to prevent from hostapd blocking DFS channels
+			*/
+			#ifdef CONFIG_DFS_MASTER
+			if (chplan->dfs_domain != RTW_DFS_REGD_NONE)
+				chan->flags &= ~IEEE80211_CHAN_RADAR;
+			#endif
+		}
+	}
+
+	if (rtnl_lock_needed)
+		rtnl_unlock();
+}
+
+static u8 wiphy_chan_get_rtw_ch_flags(struct ieee80211_channel *chan)
+{
+	u8 flags;
+
+	if (!chan || chan->flags & IEEE80211_CHAN_DISABLED)
+		return RTW_CHF_DIS;
+
+	flags = 0;
+
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0))
+	if (chan->flags & (IEEE80211_CHAN_NO_IBSS | IEEE80211_CHAN_PASSIVE_SCAN))
+	#else
+	if (chan->flags & IEEE80211_CHAN_NO_IR)
+	#endif
+		flags |= RTW_CHF_NO_IR;
+	if (chan->flags & IEEE80211_CHAN_RADAR)
+		flags |= RTW_CHF_DFS;
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27))
+	if (chan->flags & IEEE80211_CHAN_NO_HT40PLUS)
+		flags |= RTW_CHF_NO_HT40U;
+	if (chan->flags & IEEE80211_CHAN_NO_HT40MINUS)
+		flags |= RTW_CHF_NO_HT40L;
+	#endif
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
+	if (chan->flags & IEEE80211_CHAN_NO_80MHZ)
+		flags |= RTW_CHF_NO_80MHZ;
+	if (chan->flags & IEEE80211_CHAN_NO_160MHZ)
+		flags |= RTW_CHF_NO_160MHZ;
+	#endif
+
+	return flags;
+}
+
+static void rtw_chset_apply_wiphy_chans(struct rtw_chset *chset, u8 d_flags)
+{
+	RT_CHANNEL_INFO *chinfo;
+	u8 i;
+	u8 flags;
+
+	for (i = 0; i < chset->chs_len; i++) {
+		chinfo = &chset->chs[i];
+		if (chinfo->flags & RTW_CHF_DIS)
+			continue;
+		flags = wiphy_chan_get_rtw_ch_flags(chinfo->os_chan);
+		if ((flags & RTW_CHF_DIS) || (flags & d_flags))
+			chinfo->flags = RTW_CHF_DIS;
+		else
+			chinfo->flags |= flags;
+	}
+}
+
+void rtw_chset_apply_from_os(struct rtw_chset *chset, u8 d_flags)
+{
+	rtw_chset_apply_wiphy_chans(chset, d_flags);
+}
+
+enum country_ie_slave_en_mode rtw_os_get_cis_en_mode(_adapter *adapter)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0))
+	struct wiphy *wiphy = adapter_to_wiphy(adapter);
+
+	if (wiphy->regulatory_flags & REGULATORY_COUNTRY_IE_IGNORE)
+		return CISEM_DISABLE;
+#endif
+	return CISEM_ENABLE;
+}
+
+s16 rtw_os_get_total_txpwr_regd_lmt_mbm(_adapter *adapter, enum band_type band, u8 cch, enum channel_width bw)
+{
+	struct wiphy *wiphy = adapter_to_wiphy(adapter);
+	s16 mbm = UNSPECIFIED_MBM;
+	u8 *op_chs;
+	u8 op_ch_num;
+	u8 i;
+	u32 freq;
+	struct ieee80211_channel *ch;
+
+	if (!rtw_get_op_chs_by_bcch_bw(band, cch, bw, &op_chs, &op_ch_num))
+		goto exit;
+
+	for (i = 0; i < op_ch_num; i++) {
+		freq = rtw_bch2freq(band, op_chs[i]);
+		ch = ieee80211_get_channel(wiphy, freq);
+		if (!ch) {
+			rtw_warn_on(1);
+			continue;
+		}
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
+		mbm = rtw_min(mbm, ch->max_reg_power * MBM_PDBM);
+		#else
+		/* require max_power == 0 (therefore orig_mpwr set to 0) when wiphy registration */
+		mbm = rtw_min(mbm, ch->max_power * MBM_PDBM);
+		#endif
+	}
+
+exit:
+	return mbm;
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+static enum rtw_dfs_regd nl80211_dfs_regions_to_rtw_dfs_region(enum nl80211_dfs_regions region)
+{
+	switch (region) {
+	case NL80211_DFS_FCC:
+		return RTW_DFS_REGD_FCC;
+	case NL80211_DFS_ETSI:
+		return RTW_DFS_REGD_ETSI;
+	case NL80211_DFS_JP:
+		return RTW_DFS_REGD_MKK;
+	case NL80211_DFS_UNSET:
+	default:
+		return RTW_DFS_REGD_NONE;
+	}
+};
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)) */
+
+static enum rtw_env_t environment_cap_to_rtw_env(enum environment_cap env)
+{
+	switch (env) {
+	case ENVIRON_INDOOR:
+		return RTW_ENV_INDOOR;
+	case ENVIRON_OUTDOOR:
+		return RTW_ENV_OUTDOOR;
+	case ENVIRON_ANY:
+		return RTW_ENV_ANY;
+	}
+	rtw_warn_on(1);
+	return RTW_ENV_NUM;
+}
+#endif /* CONFIG_REGD_SRC_FROM_OS */
+
+static enum rtw_regd_inr nl80211_reg_initiator_to_rtw_regd_inr(enum nl80211_reg_initiator initiator)
+{
+	switch (initiator) {
+	case NL80211_REGDOM_SET_BY_DRIVER:
+		return RTW_REGD_SET_BY_DRIVER;
+	case NL80211_REGDOM_SET_BY_CORE:
+		return RTW_REGD_SET_BY_CORE;
+	case NL80211_REGDOM_SET_BY_USER:
+		return RTW_REGD_SET_BY_USER;
+	case NL80211_REGDOM_SET_BY_COUNTRY_IE:
+		return RTW_REGD_SET_BY_COUNTRY_IE;
+	}
+	rtw_warn_on(1);
+	return RTW_REGD_SET_BY_NUM;
+}
+
+#ifdef CONFIG_RTW_DEBUG
+static const char *nl80211_reg_initiator_str(enum nl80211_reg_initiator initiator)
+{
+	switch (initiator) {
+	case NL80211_REGDOM_SET_BY_DRIVER:
+		return "DRIVER";
+	case NL80211_REGDOM_SET_BY_CORE:
+		return "CORE";
+	case NL80211_REGDOM_SET_BY_USER:
+		return "USER";
+	case NL80211_REGDOM_SET_BY_COUNTRY_IE:
+		return "COUNTRY_IE";
+	}
+	rtw_warn_on(1);
+	return "UNKNOWN";
+}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+static const char *nl80211_user_reg_hint_type_str(enum nl80211_user_reg_hint_type type)
+{
+	switch (type) {
+	case NL80211_USER_REG_HINT_USER:
+		return "USER";
+	case NL80211_USER_REG_HINT_CELL_BASE:
+		return "CELL_BASE";
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0))
+	case NL80211_USER_REG_HINT_INDOOR:
+		return "INDOOR";
+	#endif
+	}
+	rtw_warn_on(1);
+	return "UNKNOWN";
+}
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+static const char *nl80211_dfs_regions_str(enum nl80211_dfs_regions region)
+{
+	switch (region) {
+	case NL80211_DFS_UNSET:
+		return "UNSET";
+	case NL80211_DFS_FCC:
+		return "FCC";
+	case NL80211_DFS_ETSI:
+		return "ETSI";
+	case NL80211_DFS_JP:
+		return "JP";
+	}
+	rtw_warn_on(1);
+	return "UNKNOWN";
+};
+#endif /* (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)) */
+
+static const char *environment_cap_str(enum environment_cap cap)
+{
+	switch (cap) {
+	case ENVIRON_ANY:
+		return "ANY";
+	case ENVIRON_INDOOR:
+		return "INDOOR";
+	case ENVIRON_OUTDOOR:
+		return "OUTDOOR";
+	}
+	rtw_warn_on(1);
+	return "UNKNOWN";
+}
+
+static void dump_requlatory_request(void *sel, struct regulatory_request *request)
+{
+	u8 alpha2_len;
+
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 16, 0))
+	alpha2_len = 3;
+	#else
+	alpha2_len = 2;
+	#endif
+
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0))
+	RTW_PRINT_SEL(sel, "initiator:%s, wiphy_idx:%d, type:%s\n"
+		, nl80211_reg_initiator_str(request->initiator)
+		, request->wiphy_idx
+		, nl80211_user_reg_hint_type_str(request->user_reg_hint_type));
+	#else
+	RTW_PRINT_SEL(sel, "initiator:%s, wiphy_idx:%d\n"
+		, nl80211_reg_initiator_str(request->initiator)
+		, request->wiphy_idx);
+	#endif
+
+	RTW_PRINT_SEL(sel, "alpha2:%.*s\n", alpha2_len, request->alpha2);
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+	RTW_PRINT_SEL(sel, "dfs_region:%s\n", nl80211_dfs_regions_str(request->dfs_region));
+	#endif
+
+	RTW_PRINT_SEL(sel, "intersect:%d\n", request->intersect);
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38))
+	RTW_PRINT_SEL(sel, "processed:%d\n", request->processed);
+	#endif
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36))
+	RTW_PRINT_SEL(sel, "country_ie_checksum:0x%08x\n", request->country_ie_checksum);
+	#endif
+
+	RTW_PRINT_SEL(sel, "country_ie_env:%s\n", environment_cap_str(request->country_ie_env));
+}
+#endif /* CONFIG_RTW_DEBUG */
+
+static void rtw_reg_notifier(struct wiphy *wiphy, struct regulatory_request *request)
+{
+	struct dvobj_priv *dvobj = wiphy_to_dvobj(wiphy);
+	struct registry_priv *regsty = dvobj_to_regsty(dvobj);
+	enum rtw_regd_inr inr;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	rtw_set_rtnl_lock_holder(dvobj, current);
+#endif
+
+#ifdef CONFIG_RTW_DEBUG
+	if (rtw_drv_log_level >= _DRV_INFO_) {
+		RTW_INFO(FUNC_WIPHY_FMT"\n", FUNC_WIPHY_ARG(wiphy));
+		dump_requlatory_request(RTW_DBGDUMP, request);
+	}
+#endif
+
+	inr = nl80211_reg_initiator_to_rtw_regd_inr(request->initiator);
+
+#ifdef CONFIG_REGD_SRC_FROM_OS
+	if (REGSTY_REGD_SRC_FROM_OS(regsty)) {
+		enum rtw_dfs_regd dfs_region =  RTW_DFS_REGD_NONE;
+		enum rtw_env_t env = RTW_ENV_NUM;
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0))
+		dfs_region = nl80211_dfs_regions_to_rtw_dfs_region(request->dfs_region);
+		#endif
+
+		if (request->initiator == NL80211_REGDOM_SET_BY_COUNTRY_IE)
+			env = environment_cap_to_rtw_env(request->country_ie_env);
+
+		/* trigger command to sync regulatory form OS */
+		rtw_sync_os_regd_cmd(wiphy_to_adapter(wiphy), RTW_CMDF_WAIT_ACK, request->alpha2, dfs_region, env, inr);
+	} else
+#endif
+	{
+		/* use alpha2 as input to select the corresponding channel plan settings defined by Realtek */
+		struct get_chplan_resp *chplan;
+
+		switch (request->initiator) {
+		case NL80211_REGDOM_SET_BY_USER:
+			rtw_set_country(wiphy_to_adapter(wiphy), request->alpha2, RTW_ENV_NUM, inr);
+			break;
+		case NL80211_REGDOM_SET_BY_DRIVER:
+		case NL80211_REGDOM_SET_BY_CORE:
+		case NL80211_REGDOM_SET_BY_COUNTRY_IE:
+		default:
+			#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+			rtw_warn_on(rtw_regd_is_wiphy_self_managed(wiphy));
+			#endif
+			if (rtw_get_chplan_cmd(wiphy_to_adapter(wiphy), RTW_CMDF_WAIT_ACK, &chplan) == _SUCCESS)
+				rtw_regd_change_complete_sync(wiphy, chplan, 0);
+			else
+				rtw_warn_on(1);
+			break;
+		}
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 11, 0))
+	rtw_set_rtnl_lock_holder(dvobj, NULL);
+#endif
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0))
+static int rtw_reg_notifier_return(struct wiphy *wiphy, struct regulatory_request *request)
+{
+	rtw_reg_notifier(wiphy, request);
+	return 0;
+}
+#endif
+
+void rtw_regd_change_complete_sync(struct wiphy *wiphy, struct get_chplan_resp *chplan, bool rtnl_lock_needed)
+{
+	if (chplan->regd_src == REGD_SRC_RTK_PRIV) {
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+		if (rtw_regd_is_wiphy_self_managed(wiphy))
+			rtw_update_wiphy_regd(wiphy, chplan, rtnl_lock_needed);
+		else
+		#endif
+			rtw_regd_overide_flags(wiphy, chplan, rtnl_lock_needed);
+	}
+#ifdef CONFIG_REGD_SRC_FROM_OS
+	else if (chplan->regd_src == REGD_SRC_OS)
+		rtw_regd_apply_dfs_flags(chplan, rtnl_lock_needed);
+#endif
+	else
+		rtw_warn_on(1);
+
+	rtw_free_get_chplan_resp(chplan);
+}
+
+struct async_regd_change_evt {
+	_list list; /* async_regd_change_list */
+	struct wiphy *wiphy;
+	struct get_chplan_resp *chplan;
+};
+
+static void async_regd_change_work_hdl(_workitem *work)
+{
+	struct rtw_wiphy_data *wiphy_data = container_of(work, struct rtw_wiphy_data, async_regd_change_work);
+	struct async_regd_change_evt *evt;
+	_list *list, *head = &wiphy_data->async_regd_change_list;
+
+	while (1) {
+		_rtw_mutex_lock_interruptible(&wiphy_data->async_regd_change_mutex);
+		list = rtw_is_list_empty(head) ? NULL : get_next(head);
+		if (list)
+			rtw_list_delete(list);
+		_rtw_mutex_unlock(&wiphy_data->async_regd_change_mutex);
+
+		if (!list)
+			break;
+
+		evt = LIST_CONTAINOR(list, struct async_regd_change_evt, list);
+		rtw_regd_change_complete_sync(evt->wiphy, evt->chplan, 1);
+		rtw_mfree(evt, sizeof(*evt));
+	}
+}
+
+int rtw_regd_change_complete_async(struct wiphy *wiphy, struct get_chplan_resp *chplan)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	struct async_regd_change_evt *evt;
+
+	evt = rtw_malloc(sizeof(*evt));
+	if (!evt) {
+		rtw_free_get_chplan_resp(chplan);
+		return _FAIL;
+	}
+
+	_rtw_init_listhead(&evt->list);
+	evt->wiphy = wiphy;
+	evt->chplan = chplan;
+
+	_rtw_mutex_lock_interruptible(&wiphy_data->async_regd_change_mutex);
+
+	rtw_list_insert_tail(&evt->list, &wiphy_data->async_regd_change_list);
+
+	_rtw_mutex_unlock(&wiphy_data->async_regd_change_mutex);
+
+	_set_workitem(&wiphy_data->async_regd_change_work);
+
+	return _SUCCESS;
+}
+
+static void rtw_regd_async_regd_change_list_free(struct wiphy *wiphy)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	struct async_regd_change_evt *evt;
+	struct get_chplan_resp *chplan;
+	_list *cur, *head;
+
+	_rtw_mutex_lock_interruptible(&wiphy_data->async_regd_change_mutex);
+
+	head = &wiphy_data->async_regd_change_list;
+	cur = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		evt = LIST_CONTAINOR(cur, struct async_regd_change_evt, list);
+		chplan = evt->chplan;
+		cur = get_next(cur);
+		rtw_list_delete(&evt->list);
+		rtw_free_get_chplan_resp(chplan);
+		rtw_mfree(evt, sizeof(*evt));
+	}
+
+	_rtw_mutex_unlock(&wiphy_data->async_regd_change_mutex);
+}
+
+#if CONFIG_RTW_CFG80211_CAC_EVENT
+static bool rtw_cfg80211_get_radar_event_chdef(struct wiphy *wiphy, struct cfg80211_chan_def *chdef
+	, u8 cch, enum channel_width bw, const char *caller)
+{
+	u8 *op_chs;
+	u8 op_ch_num;
+	u8 state;
+
+	/* only channel coverage matters, use first (lowest) op channel, and ht here */
+
+	if (!rtw_get_op_chs_by_bcch_bw(BAND_ON_5G, cch, bw, &op_chs, &op_ch_num)) {
+		RTW_WARN("%s invalid cch:%u bw:%s on %s\n", caller, cch, ch_width_str(bw), band_str(BAND_ON_5G));
+		return false;
+	}
+
+	state = rtw_bchbw_to_cfg80211_chan_def(wiphy, chdef, BAND_ON_5G
+		, op_chs[0], bw, bw == CHANNEL_WIDTH_20 ? CHAN_OFFSET_NO_EXT : CHAN_OFFSET_UPPER, 1);
+
+	return state == _SUCCESS ? true : false;
+
+}
+
+void rtw_cfg80211_radar_detected_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 cch, enum channel_width bw)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+	struct cfg80211_chan_def chdef;
+
+	if (!rtw_cfg80211_get_radar_event_chdef(wiphy, &chdef, cch, bw, __func__))
+		return;
+
+	cfg80211_radar_event(wiphy, &chdef, GFP_KERNEL);
+}
+
+struct async_cac_change_evt {
+	_list list; /* async_cac_change_list */
+	struct net_device *netdev;
+	struct cfg80211_chan_def chandef;
+	enum nl80211_radar_event event;
+};
+
+static void async_cac_change_work_hdl(_workitem *work)
+{
+	struct rtw_wiphy_data *wiphy_data = container_of(work, struct rtw_wiphy_data, async_cac_change_work);
+	struct async_cac_change_evt *evt;
+	_list *list, *head = &wiphy_data->async_cac_change_list;
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0))
+	unsigned int link_id = 0;
+	#endif
+
+	while (1) {
+		_rtw_mutex_lock_interruptible(&wiphy_data->async_cac_change_mutex);
+		list = rtw_is_list_empty(head) ? NULL : get_next(head);
+		if (list)
+			rtw_list_delete(list);
+		_rtw_mutex_unlock(&wiphy_data->async_cac_change_mutex);
+
+		if (!list)
+			break;
+
+		evt = LIST_CONTAINOR(list, struct async_cac_change_evt, list);
+
+		rtnl_lock();
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0))
+		cfg80211_cac_event(evt->netdev, &evt->chandef, evt->event, GFP_KERNEL, link_id);
+		#else
+		cfg80211_cac_event(evt->netdev, &evt->chandef, evt->event, GFP_KERNEL);
+		#endif
+
+		rtnl_unlock();
+
+		rtw_mfree(evt, sizeof(*evt));
+	}
+}
+
+static int cfg80211_cac_event_async(struct net_device *netdev
+	, const struct cfg80211_chan_def *chandef, enum nl80211_radar_event event)
+{
+	_adapter *adapter = rtw_netdev_priv(netdev);
+	struct wiphy *wiphy = adapter_to_wiphy(adapter);
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	struct async_cac_change_evt *evt;
+
+	evt = rtw_malloc(sizeof(*evt));
+	if (!evt)
+		return _FAIL;
+
+	_rtw_init_listhead(&evt->list);
+	evt->netdev = netdev;
+	_rtw_memcpy(&evt->chandef, chandef, sizeof(*chandef));
+	evt->event = event;
+
+	_rtw_mutex_lock_interruptible(&wiphy_data->async_cac_change_mutex);
+
+	rtw_list_insert_tail(&evt->list, &wiphy_data->async_cac_change_list);
+
+	_rtw_mutex_unlock(&wiphy_data->async_cac_change_mutex);
+
+	_set_workitem(&wiphy_data->async_cac_change_work);
+
+	return _SUCCESS;
+}
+
+static void rtw_regd_async_cac_change_list_free(struct wiphy *wiphy)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+	struct async_cac_change_evt *evt;
+	_list *cur, *head;
+
+	_rtw_mutex_lock_interruptible(&wiphy_data->async_cac_change_mutex);
+
+	head = &wiphy_data->async_cac_change_list;
+	cur = get_next(head);
+
+	while ((rtw_end_of_queue_search(head, cur)) == _FALSE) {
+		evt = LIST_CONTAINOR(cur, struct async_cac_change_evt, list);
+		cur = get_next(cur);
+		rtw_list_delete(&evt->list);
+		rtw_mfree(evt, sizeof(*evt));
+	}
+
+	_rtw_mutex_unlock(&wiphy_data->async_cac_change_mutex);
+}
+
+static void rtw_cfg80211_cac_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw, enum nl80211_radar_event event
+	, const char *caller)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+	struct cfg80211_chan_def chdef;
+	_adapter *iface;
+	int i;
+	bool async;
+	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0))
+	unsigned int link_id = 0;
+	#endif
+
+	if (!ifbmp) {
+		RTW_WARN(FUNC_HWBAND_FMT" ifbmp is empty", caller, HWBAND_ARG(band_idx));
+		return;
+	}
+
+	if (!rtw_cfg80211_get_radar_event_chdef(wiphy, &chdef, cch, bw, caller))
+		return;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !(ifbmp & BIT(iface->iface_id)))
+			continue;
+		if (!iface->rtw_wdev)
+			continue;
+#if defined(CONFIG_ACK_5_15_LTS_KERNEL) || (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 19, 2))
+		async = !((iface->rtw_wdev)->links[0].ap.chandef.chan);
+#else
+		async = !iface->rtw_wdev->chandef.chan;
+#endif
+		RTW_INFO(FUNC_ADPT_FMT" async:%d\n", caller, ADPT_ARG(iface), async);
+		if (async)
+			cfg80211_cac_event_async(iface->pnetdev, &chdef, event);
+		else
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0))
+			cfg80211_cac_event(iface->pnetdev, &chdef, event, GFP_KERNEL, link_id);
+		#else
+			cfg80211_cac_event(iface->pnetdev, &chdef, event, GFP_KERNEL);
+		#endif
+	}
+}
+
+void rtw_cfg80211_cac_started_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+
+	if (!wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD))
+		return;
+
+	if (ifbmp == 0xFF) {
+		ifbmp &= rtw_mi_get_ap_mesh_ifbmp_by_hwband(dvobj, band_idx);
+		if (!ifbmp)
+			return;
+	}
+
+	rtw_cfg80211_cac_event(rfctl, band_idx, ifbmp, cch, bw, NL80211_RADAR_CAC_STARTED, __func__);
+}
+
+void rtw_cfg80211_cac_finished_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+	_adapter *iface;
+	int i;
+	bool cac;
+
+	if (!wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD))
+		return;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !(ifbmp & BIT(iface->iface_id)))
+			continue;
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0))
+		cac = iface->rtw_wdev->links[0].cac_started;
+		#else
+		cac = iface->rtw_wdev->cac_started;
+		#endif
+
+		/* finish only for wdev with cac_started */
+		if (!iface->rtw_wdev || !cac)
+			ifbmp &= ~BIT(iface->iface_id);
+	}
+
+	rtw_cfg80211_cac_event(rfctl, band_idx, ifbmp, cch, bw, NL80211_RADAR_CAC_FINISHED, __func__);
+}
+
+void rtw_cfg80211_cac_aborted_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+	_adapter *iface;
+	int i;
+	bool cac;
+
+	if (!wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD))
+		return;
+
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !(ifbmp & BIT(iface->iface_id)))
+			continue;
+
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0))
+		cac = iface->rtw_wdev->links[0].cac_started;
+		#else
+		cac = iface->rtw_wdev->cac_started;
+		#endif
+		/* abort only for wdev with cac_started */
+		if (!iface->rtw_wdev || !cac)
+			ifbmp &= ~BIT(iface->iface_id);
+	}
+
+	rtw_cfg80211_cac_event(rfctl, band_idx, ifbmp, cch, bw, NL80211_RADAR_CAC_ABORTED, __func__);
+}
+
+static void rtw_cfg80211_cac_force_started_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	rtw_cfg80211_cac_event(rfctl, band_idx, ifbmp, cch, bw, NL80211_RADAR_CAC_STARTED, __func__);
+}
+
+static void rtw_cfg80211_cac_force_finished_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	rtw_cfg80211_cac_event(rfctl, band_idx, ifbmp, cch, bw, NL80211_RADAR_CAC_FINISHED, __func__);
+}
+
+void rtw_cfg80211_cac_force_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw)
+{
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+	struct rtw_chset *chset = &rfctl->chset;
+	_adapter *iface;
+	u8 i;
+	u8 *op_chs;
+	u8 op_ch_num;
+	RT_CHANNEL_INFO *chinfo;
+	struct ieee80211_channel *chan;
+	bool need_start = false;
+	u8 finished_ifbmp, started_ifbmp;
+	bool cac;
+
+	if (!wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD))
+		return;
+
+	if (ifbmp == 0xFF) {
+		ifbmp &= rtw_mi_get_ap_mesh_ifbmp_by_hwband(dvobj, band_idx);
+		if (!ifbmp)
+			return;
+	}
+
+	/*
+	* if coverage channels still USABLE and wdev is not cac_started
+	* send CAC_STARTED to set cac_started and
+	* send CAC_FINISHED to clear cac_started and make coverage channels AVAILABLE
+	*/
+
+	if (!rtw_get_op_chs_by_bcch_bw(BAND_ON_5G, cch, bw, &op_chs, &op_ch_num)) {
+		RTW_WARN("%s invalid cch:%u bw:%s on %s\n", __func__, cch, ch_width_str(bw), band_str(BAND_ON_5G));
+		return;
+	}
+
+	for (i = 0; i < op_ch_num; i++) {
+		chinfo = rtw_chset_get_chinfo_by_bch(chset, BAND_ON_5G, op_chs[i], true);
+		if (!chinfo) {
+			RTW_WARN("%s can't get chinfo by ch:%u on %s\n", __func__, op_chs[i], band_str(BAND_ON_5G));
+			continue;
+		}
+		if (!chinfo->os_chan) {
+			RTW_WARN("%s ch:%u on %s has no os_chan\n", __func__, op_chs[i], band_str(BAND_ON_5G));
+			continue;
+		}
+
+		chan = chinfo->os_chan;
+		if (chan->dfs_state == NL80211_DFS_UNAVAILABLE) {
+			RTW_WARN("%s ch:%u on %s is UNAVAILABLE(NON_OCP)\n", __func__, op_chs[i], band_str(BAND_ON_5G));
+			return;
+		}
+		if (chan->dfs_state == NL80211_DFS_USABLE) {
+			need_start = true;
+			break;
+		}
+	}
+
+	started_ifbmp = need_start ? ifbmp : 0;
+	finished_ifbmp = ifbmp;
+	for (i = 0; i < dvobj->iface_nums; i++) {
+		iface = dvobj->padapters[i];
+		if (!iface || !(ifbmp & BIT(iface->iface_id)))
+			continue;
+		if (!iface->rtw_wdev) {
+			finished_ifbmp &= ~BIT(iface->iface_id);
+			started_ifbmp &= ~BIT(iface->iface_id);
+			continue;
+		}
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0))
+		cac = iface->rtw_wdev->links[0].cac_started;
+		#else
+		cac = iface->rtw_wdev->cac_started;
+		#endif
+		if (need_start && cac)
+			started_ifbmp &= ~BIT(iface->iface_id);
+		else if (!need_start && !cac)
+			finished_ifbmp &= ~BIT(iface->iface_id);
+	}
+
+	/* send CAC_STARTED to set cac_started */
+	if (started_ifbmp)
+		rtw_cfg80211_cac_force_started_event(rfctl, band_idx, started_ifbmp, cch, bw);
+
+	/* send CAC_FINISHED to clear cac_started and make coverage channels AVAILABLE */
+	if (finished_ifbmp)
+		rtw_cfg80211_cac_force_finished_event(rfctl, band_idx, finished_ifbmp, cch, bw);
+}
+
+void rtw_cfg80211_nop_finished_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw)
+{
+	/* non occupancy is controlled by net/wireless/mlme.c */
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+
+	if (!wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD))
+		return;
+}
+
+void rtw_cfg80211_nop_started_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw, bool called_on_cmd_thd)
+{
+	/* non occupancy is controlled by net/wireless/mlme.c */
+	struct dvobj_priv *dvobj = rfctl_to_dvobj(rfctl);
+	struct wiphy *wiphy = dvobj_to_wiphy(dvobj); /* TODO: hwband specific wiphy mapping */
+	struct get_chplan_resp *chplan;
+	bool rtnl_lock_needed;
+
+	if (!wiphy_ext_feature_isset(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD))
+		return;
+
+	if (rtw_get_chplan_cmd(dvobj_get_primary_adapter(dvobj)
+		, called_on_cmd_thd ? RTW_CMDF_DIRECTLY : RTW_CMDF_WAIT_ACK, &chplan) == _FAIL)
+		return;
+
+	rtnl_lock_needed = rtw_rtnl_lock_needed(dvobj);
+	if (rtnl_lock_needed)
+		rtnl_lock();
+
+	rtw_regd_override_dfs_state(wiphy, chplan, true);
+	rtw_regd_schedule_dfs_chan_update(wiphy);
+
+	if (rtnl_lock_needed)
+		rtnl_unlock();
+
+	rtw_free_get_chplan_resp(chplan);
+}
+#endif /* CONFIG_RTW_CFG80211_CAC_EVENT  */
+
+int rtw_regd_init(struct wiphy *wiphy)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0))
+	wiphy->reg_notifier = rtw_reg_notifier_return;
+#else
+	wiphy->reg_notifier = rtw_reg_notifier;
+#endif
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0))
+	wiphy->flags &= ~WIPHY_FLAG_STRICT_REGULATORY;
+	wiphy->flags &= ~WIPHY_FLAG_DISABLE_BEACON_HINTS;
+#else
+	wiphy->regulatory_flags &= ~REGULATORY_STRICT_REG;
+	wiphy->regulatory_flags &= ~REGULATORY_DISABLE_BEACON_HINTS;
+	wiphy->regulatory_flags &= ~REGULATORY_COUNTRY_IE_IGNORE;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 39))
+	wiphy->regulatory_flags |= REGULATORY_IGNORE_STALE_KICKOFF;
+#endif
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0))
+	if (rtw_regd_is_wiphy_self_managed(wiphy))
+		wiphy->regulatory_flags |= REGULATORY_WIPHY_SELF_MANAGED;
+	else
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0))
+	if (!REGSTY_REGD_SRC_OS_11D(dvobj_to_regsty(wiphy_to_dvobj(wiphy))))
+		wiphy->regulatory_flags |= REGULATORY_COUNTRY_IE_IGNORE;
+#endif
+
+#if defined(CONFIG_DFS_MASTER) && (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0))
+	if (rtw_regd_is_wiphy_self_managed(wiphy)
+		&& wiphy->bands[NL80211_BAND_5GHZ])
+		wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD);
+#endif
+
+	_rtw_init_listhead(&wiphy_data->async_regd_change_list);
+	_rtw_mutex_init(&wiphy_data->async_regd_change_mutex);
+	_init_workitem(&wiphy_data->async_regd_change_work, async_regd_change_work_hdl, NULL);
+
+#if CONFIG_RTW_CFG80211_CAC_EVENT
+	_rtw_init_listhead(&wiphy_data->async_cac_change_list);
+	_rtw_mutex_init(&wiphy_data->async_cac_change_mutex);
+	_init_workitem(&wiphy_data->async_cac_change_work, async_cac_change_work_hdl, NULL);
+#endif
+
+	return 0;
+}
+
+void rtw_regd_deinit(struct wiphy *wiphy)
+{
+	struct rtw_wiphy_data *wiphy_data = rtw_wiphy_priv(wiphy);
+
+	_cancel_workitem_sync(&wiphy_data->async_regd_change_work);
+	rtw_regd_async_regd_change_list_free(wiphy);
+	_rtw_mutex_free(&wiphy_data->async_regd_change_mutex);
+
+#if CONFIG_RTW_CFG80211_CAC_EVENT
+	_cancel_workitem_sync(&wiphy_data->async_cac_change_work);
+	rtw_regd_async_cac_change_list_free(wiphy);
+	_rtw_mutex_free(&wiphy_data->async_cac_change_mutex);
+#endif
+}
+#endif /* CONFIG_IOCTL_CFG80211 */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/wifi_regd.h b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/wifi_regd.h
new file mode 100644
index 000000000000..be2a52b08166
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/wifi_regd.h
@@ -0,0 +1,69 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2010 - 2017 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef __WIFI_REGD_H__
+#define __WIFI_REGD_H__
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)) && (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31))
+#define IEEE80211_CHAN_NO_HT40PLUS IEEE80211_CHAN_NO_FAT_ABOVE
+#define IEEE80211_CHAN_NO_HT40MINUS IEEE80211_CHAN_NO_FAT_BELOW
+#define IEEE80211_CHAN_NO_HT40 (IEEE80211_CHAN_NO_HT40PLUS | IEEE80211_CHAN_NO_HT40MINUS)
+#endif
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
+#define IEEE80211_CHAN_IR_CONCURRENT IEEE80211_CHAN_GO_CONCURRENT
+#endif
+
+#if !RTW_PER_ADAPTER_WIPHY
+void rtw_chset_hook_os_channels(struct rtw_chset *chset, void *os_ref);
+#endif
+void rtw_regd_change_complete_sync(struct wiphy *wiphy, struct get_chplan_resp *chplan, bool rtnl_lock_needed);
+int rtw_regd_change_complete_async(struct wiphy *wiphy, struct get_chplan_resp *chplan);
+#ifdef CONFIG_REGD_SRC_FROM_OS
+void rtw_chset_apply_from_os(struct rtw_chset *chset, u8 d_flags);
+enum country_ie_slave_en_mode rtw_os_get_cis_en_mode(_adapter *adapter);
+s16 rtw_os_get_total_txpwr_regd_lmt_mbm(_adapter *adapter, enum band_type band, u8 cch, enum channel_width bw);
+#endif
+
+#if CONFIG_RTW_CFG80211_CAC_EVENT
+void rtw_cfg80211_radar_detected_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 cch, enum channel_width bw);
+void rtw_cfg80211_cac_started_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_cfg80211_cac_finished_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_cfg80211_cac_aborted_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_cfg80211_cac_force_finished(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 ifbmp, u8 cch, enum channel_width bw);
+void rtw_cfg80211_nop_finished_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw);
+void rtw_cfg80211_nop_started_event(struct rf_ctl_t *rfctl, u8 band_idx
+	, u8 band, u8 cch, enum channel_width bw, bool called_on_cmd_thd);
+void rtw_regd_free_du_wdev(struct wiphy *wiphy);
+#else
+#define rtw_cfg80211_radar_detected_event(rfctl, band_idx, cch, bw) do {} while (0)
+#define rtw_cfg80211_cac_started_event(rfctl, band_idx, ifbmp, cch, bw) do {} while (0)
+#define rtw_cfg80211_cac_finished_event(rfctl, band_idx, ifbmp, cch, bw) do {} while (0)
+#define rtw_cfg80211_cac_aborted_event(rfctl, band_idx, ifbmp, cch, bw) do {} while (0)
+#define rtw_cfg80211_cac_force_finished(rfctl, band_idx, ifbmp, cch, bw) do {} while (0)
+#define rtw_cfg80211_nop_finished_event(rfctl, band_idx, band, cch, bw) do {} while (0)
+#define rtw_cfg80211_nop_started_event(rfctl, band_idx, band, cch, bw, called_on_cmd_thd) do {} while (0)
+#define rtw_regd_free_du_wdev(wiphy) do {} while (0)
+#endif /* CONFIG_RTW_CFG80211_CAC_EVENT */
+
+int rtw_regd_init(struct wiphy *wiphy);
+void rtw_regd_deinit(struct wiphy *wiphy);
+
+#endif /* __WIFI_REGD_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/xmit_linux.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/xmit_linux.c
new file mode 100644
index 000000000000..a6105f5357da
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/linux/xmit_linux.c
@@ -0,0 +1,743 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _XMIT_OSDEP_C_
+
+#include <drv_types.h>
+
+#define DBG_DUMP_OS_QUEUE_CTL 0
+
+uint rtw_remainder_len(struct pkt_file *pfile)
+{
+	return pfile->buf_len - ((SIZE_PTR)(pfile->cur_addr) - (SIZE_PTR)(pfile->buf_start));
+}
+
+void _rtw_open_pktfile(struct sk_buff *pktptr, struct pkt_file *pfile)
+{
+
+	pfile->pkt = pktptr;
+	pfile->cur_addr = pfile->buf_start = pktptr->data;
+	pfile->pkt_len = pfile->buf_len = pktptr->len;
+
+	pfile->cur_buffer = pfile->buf_start ;
+
+}
+
+uint _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, uint rlen)
+{
+	uint	len = 0;
+
+
+	len =  rtw_remainder_len(pfile);
+	len = (rlen > len) ? len : rlen;
+
+	if (rmem)
+		skb_copy_bits(pfile->pkt, pfile->buf_len - pfile->pkt_len, rmem, len);
+
+	pfile->cur_addr += len;
+	pfile->pkt_len -= len;
+
+
+	return len;
+}
+
+sint rtw_endofpktfile(struct pkt_file *pfile)
+{
+
+	if (pfile->pkt_len == 0) {
+		return _TRUE;
+	}
+
+
+	return _FALSE;
+}
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+void rtw_set_tx_chksum_offload(struct sk_buff *pkt, struct pkt_attrib *pattrib)
+{
+	struct sk_buff *skb = (struct sk_buff *)pkt;
+	struct iphdr *iph = NULL;
+	struct ipv6hdr *i6ph = NULL;
+	struct udphdr *uh = NULL;
+	struct tcphdr *th = NULL;
+	u8 	protocol = 0xFF;
+
+	if (skb->protocol == htons(ETH_P_IP)) {
+		iph = (struct iphdr *)skb_network_header(skb);
+		protocol = iph->protocol;
+	} else if (skb->protocol == htons(ETH_P_IPV6)) {
+		i6ph = (struct ipv6hdr *)skb_network_header(skb);
+		protocol = i6ph->nexthdr;
+	} else
+		{}
+
+	/*	HW unable to compute CSUM if header & payload was be encrypted by SW(cause TXDMA error) */
+	if (pattrib->bswenc == _TRUE) {
+		if (skb->ip_summed == CHECKSUM_PARTIAL)
+			skb_checksum_help(skb);
+		return;
+	}
+
+	/*	For HW rule, clear ipv4_csum & UDP/TCP_csum if it is UDP/TCP packet	*/
+	switch (protocol) {
+	case IPPROTO_UDP:
+		uh = (struct udphdr *)skb_transport_header(skb);
+		uh->check = 0;
+		if (iph)
+			iph->check = 0;
+		pattrib->hw_csum = _TRUE;
+		break;
+	case IPPROTO_TCP:
+		th = (struct tcphdr *)skb_transport_header(skb);
+		th->check = 0;
+		if (iph)
+			iph->check = 0;
+		pattrib->hw_csum = _TRUE;
+		break;
+	default:
+		break;
+	}
+}
+#endif
+
+#if 0 /*CONFIG_CORE_XMITBUF*/
+int rtw_os_xmit_resource_alloc(_adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz, u8 flag)
+{
+	if (alloc_sz > 0) {
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_TX
+		struct dvobj_priv	*pdvobjpriv = adapter_to_dvobj(padapter);
+		struct usb_device	*pusbd = dvobj_to_usb(pdvobjpriv)->pusbdev;
+
+		pxmitbuf->pallocated_buf = rtw_usb_buffer_alloc(pusbd, (size_t)alloc_sz, &pxmitbuf->dma_transfer_addr);
+		pxmitbuf->pbuf = pxmitbuf->pallocated_buf;
+		if (pxmitbuf->pallocated_buf == NULL)
+			return _FAIL;
+#else /* CONFIG_USE_USB_BUFFER_ALLOC_TX */
+
+		pxmitbuf->pallocated_buf = rtw_zmalloc(alloc_sz);
+		if (pxmitbuf->pallocated_buf == NULL)
+			return _FAIL;
+
+		pxmitbuf->pbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitbuf->pallocated_buf), SZ_ALIGN_XMITFRAME_EXT);
+
+#endif /* CONFIG_USE_USB_BUFFER_ALLOC_TX */
+	}
+
+	if (flag) {
+#ifdef CONFIG_USB_HCI
+		int i;
+		for (i = 0; i < 8; i++) {
+			pxmitbuf->pxmit_urb[i] = usb_alloc_urb(0, GFP_KERNEL);
+			if (pxmitbuf->pxmit_urb[i] == NULL) {
+				RTW_INFO("pxmitbuf->pxmit_urb[i]==NULL");
+				return _FAIL;
+			}
+		}
+#endif
+	}
+
+	return _SUCCESS;
+}
+
+void rtw_os_xmit_resource_free(_adapter *padapter, struct xmit_buf *pxmitbuf, u32 free_sz, u8 flag)
+{
+	if (flag) {
+#ifdef CONFIG_USB_HCI
+		int i;
+
+		for (i = 0; i < 8; i++) {
+			if (pxmitbuf->pxmit_urb[i]) {
+				/* usb_kill_urb(pxmitbuf->pxmit_urb[i]); */
+				usb_free_urb(pxmitbuf->pxmit_urb[i]);
+			}
+		}
+#endif
+	}
+
+	if (free_sz > 0) {
+#ifdef CONFIG_USE_USB_BUFFER_ALLOC_TX
+		struct dvobj_priv	*pdvobjpriv = adapter_to_dvobj(padapter);
+		struct usb_device	*pusbd = dvobj_to_usb(pdvobjpriv)->pusbdev;
+
+		rtw_usb_buffer_free(pusbd, (size_t)free_sz, pxmitbuf->pallocated_buf, pxmitbuf->dma_transfer_addr);
+		pxmitbuf->pallocated_buf =  NULL;
+		pxmitbuf->dma_transfer_addr = 0;
+#else	/* CONFIG_USE_USB_BUFFER_ALLOC_TX */
+		if (pxmitbuf->pallocated_buf)
+			rtw_mfree(pxmitbuf->pallocated_buf, free_sz);
+#endif /* CONFIG_USE_USB_BUFFER_ALLOC_TX */
+	}
+}
+#else
+u8 rtw_os_xmit_resource_alloc(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	u32 alloc_sz = SZ_XMITFRAME_EXT + SZ_ALIGN_XMITFRAME_EXT;
+
+#if 0 /*def CONFIG_USE_USB_BUFFER_ALLOC_TX*/
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct usb_device *pusbd = dvobj_to_usb(dvobj)->pusbdev;
+
+	pxframe->prealloc_buf_addr = rtw_usb_buffer_alloc(pusbd, (size_t)alloc_sz, &pxframe->dma_transfer_addr);
+	if (pxframe->prealloc_buf_addr == NULL) {
+		RTW_ERR("%s prealloc_buf_addr failed\n", __func__);
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+	pxframe->buf_addr = pxframe->prealloc_buf_addr;
+#else
+	pxframe->prealloc_buf_addr = rtw_zmalloc(alloc_sz);
+	if (pxframe->prealloc_buf_addr == NULL) {
+		RTW_ERR("%s prealloc_buf_addr failed\n", __func__);
+		rtw_warn_on(1);
+		return _FAIL;
+	}
+	pxframe->buf_addr = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxframe->prealloc_buf_addr), SZ_ALIGN_XMITFRAME_EXT);
+#endif
+	return _SUCCESS;
+}
+
+void rtw_os_xmit_resource_free(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	u32 free_sz = SZ_XMITFRAME_EXT + SZ_ALIGN_XMITFRAME_EXT;
+
+#if 0 /*def CONFIG_USE_USB_BUFFER_ALLOC_TX*/
+	struct dvobj_priv *dvobj = adapter_to_dvobj(padapter);
+	struct usb_device *pusbd = dvobj_to_usb(dvobj)->pusbdev;
+
+	if (pxframe->prealloc_buf_addr) {
+		rtw_usb_buffer_free(pusbd, (size_t)free_sz, pxframe->prealloc_buf_addr, pxframe->dma_transfer_addr);
+		pxframe->prealloc_buf_addr = NULL;
+		pxframe->buf_addr = NULL;
+		pxframe->dma_transfer_addr = 0;
+	}
+#else
+	if (pxframe->prealloc_buf_addr) {
+		rtw_mfree(pxframe->prealloc_buf_addr, free_sz);
+		pxframe->prealloc_buf_addr = NULL;
+		pxframe->buf_addr = NULL;
+	}
+#endif
+}
+#endif
+
+void dump_os_queue(void *sel, _adapter *padapter)
+{
+	struct net_device *ndev = padapter->pnetdev;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		RTW_PRINT_SEL(sel, "os_queue[%d]:%s\n"
+			, i, __netif_subqueue_stopped(ndev, i) ? "stopped" : "waked");
+	}
+#else
+	RTW_PRINT_SEL(sel, "os_queue:%s\n"
+		      , netif_queue_stopped(ndev) ? "stopped" : "waked");
+#endif
+}
+
+#define WMM_XMIT_THRESHOLD	(NR_XMITFRAME*2/5)
+
+static inline bool rtw_os_need_wake_queue(_adapter *padapter, u16 os_qid)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	if (padapter->registrypriv.wifi_spec) {
+		if (pxmitpriv->hwxmits[os_qid].accnt < WMM_XMIT_THRESHOLD)
+			return _TRUE;
+	} else {
+		return _TRUE;
+	}
+	return _FALSE;
+#else
+	return _TRUE;
+#endif
+}
+
+static inline bool rtw_os_need_stop_queue(_adapter *padapter, u16 os_qid)
+{
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	if (padapter->registrypriv.wifi_spec) {
+		/* No free space for Tx, tx_worker is too slow */
+		if (pxmitpriv->hwxmits[os_qid].accnt > WMM_XMIT_THRESHOLD)
+			return _TRUE;
+	} else {
+		if (pxmitpriv->free_xmitframe_cnt <= 4)
+			return _TRUE;
+	}
+#else
+	if (pxmitpriv->free_xmitframe_cnt <= 4)
+		return _TRUE;
+#endif
+	return _FALSE;
+}
+
+void rtw_os_pkt_complete(_adapter *padapter, struct sk_buff *pkt)
+{
+	rtw_skb_free(pkt);
+}
+
+void rtw_os_xmit_complete(_adapter *padapter, struct xmit_frame *pxframe)
+{
+	if (pxframe->pkt)
+		rtw_os_pkt_complete(padapter, pxframe->pkt);
+
+	pxframe->pkt = NULL;
+}
+
+void rtw_os_xmit_schedule(_adapter *padapter)
+{
+#if 0 /*defined(CONFIG_SDIO_HCI) || defined(CONFIG_GSPI_HCI)*/
+	_adapter *pri_adapter;
+
+	if (!padapter)
+		return;
+	pri_adapter = GET_PRIMARY_ADAPTER(padapter);
+
+	if (_rtw_queue_empty(&padapter->xmitpriv.pending_xmitbuf_queue) == _FALSE)
+		_rtw_up_sema(&pri_adapter->xmitpriv.xmit_sema);
+
+
+#elif defined(CONFIG_PCI_HCI) || defined(CONFIG_USB_HCI)
+	struct xmit_priv *pxmitpriv;
+
+	if (!padapter)
+		return;
+
+	pxmitpriv = &padapter->xmitpriv;
+
+	_rtw_spinlock_bh(&pxmitpriv->lock);
+
+	if (rtw_txframes_pending(padapter))
+		rtw_tasklet_hi_schedule(&pxmitpriv->xmit_tasklet);
+
+	_rtw_spinunlock_bh(&pxmitpriv->lock);
+	
+	#if 0 /*defined(CONFIG_PCI_HCI) && defined(CONFIG_XMIT_THREAD_MODE)*/
+	if (_rtw_queue_empty(&padapter->xmitpriv.pending_xmitbuf_queue) == _FALSE)
+		_rtw_up_sema(&padapter->xmitpriv.xmit_sema);
+	#endif
+	
+
+#endif
+}
+
+void rtw_os_check_wakup_queue(_adapter *padapter, u16 os_qid)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	if (rtw_os_need_wake_queue(padapter, os_qid)) {
+		if (DBG_DUMP_OS_QUEUE_CTL)
+			RTW_INFO(FUNC_ADPT_FMT": netif_wake_subqueue[%d]\n", FUNC_ADPT_ARG(padapter), os_qid);
+		netif_wake_subqueue(padapter->pnetdev, os_qid);
+	}
+#else
+	if (rtw_os_need_wake_queue(padapter, 0)) {
+		if (DBG_DUMP_OS_QUEUE_CTL)
+			RTW_INFO(FUNC_ADPT_FMT": netif_wake_queue\n", FUNC_ADPT_ARG(padapter));
+		netif_wake_queue(padapter->pnetdev);
+	}
+#endif
+}
+
+bool rtw_os_check_stop_queue(_adapter *padapter, u16 os_qid)
+{
+	bool busy = _FALSE;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	if (rtw_os_need_stop_queue(padapter, os_qid)) {
+		if (DBG_DUMP_OS_QUEUE_CTL)
+			RTW_INFO(FUNC_ADPT_FMT": netif_stop_subqueue[%d]\n", FUNC_ADPT_ARG(padapter), os_qid);
+		netif_stop_subqueue(padapter->pnetdev, os_qid);
+		busy = _TRUE;
+	}
+#else
+	if (rtw_os_need_stop_queue(padapter, 0)) {
+		if (DBG_DUMP_OS_QUEUE_CTL)
+			RTW_INFO(FUNC_ADPT_FMT": netif_stop_queue\n", FUNC_ADPT_ARG(padapter));
+		rtw_netif_stop_queue(padapter->pnetdev);
+		busy = _TRUE;
+	}
+#endif
+	return busy;
+}
+
+void rtw_os_wake_queue_at_free_stainfo(_adapter *padapter, int *qcnt_freed)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		if (qcnt_freed[i] == 0)
+			continue;
+
+		if (rtw_os_need_wake_queue(padapter, i)) {
+			if (DBG_DUMP_OS_QUEUE_CTL)
+				RTW_INFO(FUNC_ADPT_FMT": netif_wake_subqueue[%d]\n", FUNC_ADPT_ARG(padapter), i);
+			netif_wake_subqueue(padapter->pnetdev, i);
+		}
+	}
+#else
+	if (qcnt_freed[0] || qcnt_freed[1] || qcnt_freed[2] || qcnt_freed[3]) {
+		if (rtw_os_need_wake_queue(padapter, 0)) {
+			if (DBG_DUMP_OS_QUEUE_CTL)
+				RTW_INFO(FUNC_ADPT_FMT": netif_wake_queue\n", FUNC_ADPT_ARG(padapter));
+			netif_wake_queue(padapter->pnetdev);
+		}
+	}
+#endif
+}
+
+int _rtw_xmit_entry(struct sk_buff *pkt, _nic_hdl pnetdev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	struct sk_buff *skb = pkt;
+	struct sk_buff *segs, *nskb;
+	netdev_features_t features = padapter->pnetdev->features;
+#endif
+	u16 os_qid = 0;
+	s32 res = 0;
+
+	if (padapter->registrypriv.mp_mode) {
+		RTW_INFO("MP_TX_DROP_OS_FRAME\n");
+		goto drop_packet;
+	}
+	DBG_COUNTER(padapter->tx_logs.os_tx);
+
+	if (rtw_if_up(padapter) == _FALSE) {
+		DBG_COUNTER(padapter->tx_logs.os_tx_err_up);
+		#ifdef DBG_TX_DROP_FRAME
+		RTW_INFO("DBG_TX_DROP_FRAME %s if_up fail\n", __FUNCTION__);
+		#endif
+		goto drop_packet;
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	os_qid = skb_get_queue_mapping(pkt);
+#endif
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	if (skb_shinfo(skb)->gso_size) {
+	/*	split a big(65k) skb into several small(1.5k) skbs */
+		features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
+		segs = skb_gso_segment(skb, features);
+		if (IS_ERR(segs) || !segs)
+			goto drop_packet;
+
+		do {
+			nskb = segs;
+			segs = segs->next;
+			nskb->next = NULL;
+			rtw_mstat_update( MSTAT_TYPE_SKB, MSTAT_ALLOC_SUCCESS, nskb->truesize);
+			res = rtw_xmit(padapter, &nskb, os_qid);
+			if (res < 0) {
+				#ifdef DBG_TX_DROP_FRAME
+				RTW_INFO("DBG_TX_DROP_FRAME %s rtw_xmit fail\n", __FUNCTION__);
+				#endif
+				pxmitpriv->tx_drop++;
+				rtw_os_pkt_complete(padapter, nskb);
+			}
+		} while (segs);
+		rtw_os_pkt_complete(padapter, skb);
+		goto exit;
+	}
+#endif
+
+	res = rtw_xmit(padapter, &pkt, os_qid);
+	if (res < 0) {
+		#ifdef DBG_TX_DROP_FRAME
+		RTW_INFO("DBG_TX_DROP_FRAME %s rtw_xmit fail\n", __FUNCTION__);
+		#endif
+		goto drop_packet;
+	}
+
+	goto exit;
+
+drop_packet:
+	pxmitpriv->tx_drop++;
+	rtw_os_pkt_complete(padapter, pkt);
+
+exit:
+
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0))
+/* copy from skbuff.c to be compatible with old kernel */
+static void kfree_skb_list(struct sk_buff *segs)
+{
+	while (segs) {
+		struct sk_buff *next = segs->next;
+
+		kfree_skb(segs);
+		segs = next;
+	}
+}
+#endif
+
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32))
+netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, _nic_hdl pnetdev)
+#else
+int rtw_xmit_entry(struct sk_buff *pkt, _nic_hdl pnetdev)
+#endif
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	int ret = 0;
+
+	if (pkt) {
+		if (check_fwstate(pmlmepriv, WIFI_MONITOR_STATE) == _TRUE) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24))
+			rtw_monitor_xmit_entry((struct sk_buff *)pkt, pnetdev);
+#endif
+		}
+		else {
+#ifdef CONFIG_RTW_NETIF_SG
+			/* After turning on SG, net stack may (0.0025%) TX
+			 * strange skb that is skb_has_frag_list() but linear
+			 * (i.e. skb_is_nonlinear() is false). This is out of
+			 * our expectation, so I free fragment list to be
+			 * compatible with our design.
+			 */
+			if (skb_has_frag_list(pkt)) {
+				if (!skb_is_nonlinear(pkt)) {
+					kfree_skb_list(skb_shinfo(pkt)->frag_list);
+					skb_shinfo(pkt)->frag_list = NULL;
+					RTW_DBG("%s:%d free frag list\n", __func__, __LINE__);
+				} else {
+					RTW_DBG("%s:%d nonlinear frag list\n", __func__, __LINE__);
+				}
+			}
+#endif
+			rtw_mstat_update(MSTAT_TYPE_SKB, MSTAT_ALLOC_SUCCESS, pkt->truesize);
+			ret = rtw_os_tx(pkt, pnetdev);
+		}
+
+	}
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32))
+	return (ret == 0) ? NETDEV_TX_OK : NETDEV_TX_BUSY;
+#else
+	return ret;
+#endif
+}
+
+
+
+#ifdef RTW_PHL_TX
+int rtw_os_is_adapter_ready(_adapter *padapter, struct sk_buff *pkt)
+{
+
+	if (padapter->registrypriv.mp_mode) {
+		RTW_INFO("MP_TX_DROP_OS_FRAME\n");
+		return _FALSE;
+	}
+
+	DBG_COUNTER(padapter->tx_logs.os_tx);
+
+	if (rtw_if_up(padapter) == _FALSE) {
+		PHLTX_LOG;
+		DBG_COUNTER(padapter->tx_logs.os_tx_err_up);
+	#ifdef DBG_TX_DROP_FRAME
+		RTW_INFO("DBG_TX_DROP_FRAME %s if_up fail\n", __FUNCTION__);
+	#endif
+		return _FALSE;
+	}
+
+	if (IS_CH_WAITING(adapter_to_rfctl(padapter))){
+		PHLTX_LOG;
+		return _FALSE;
+	}
+
+	if (rtw_linked_check(padapter) == _FALSE){
+		PHLTX_LOG;
+		return _FALSE;
+	}
+
+	return _TRUE;
+}
+
+int rtw_os_tx(struct sk_buff *pkt, _nic_hdl pnetdev)
+{
+	_adapter *padapter = (_adapter *)rtw_netdev_priv(pnetdev);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	u16 os_qid = 0;
+	s32 res = 0;
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	struct sk_buff *skb = pkt;
+	struct sk_buff *segs, *nskb;
+	netdev_features_t features = padapter->pnetdev->features;
+#endif
+#ifdef RTW_PHL_DBG_CMD
+	core_add_record(padapter, REC_TX_DATA, pkt);
+#endif
+
+	PHLTX_LOG;
+
+	if (pkt->len == 0)
+		return 0;
+
+	if ((rtw_os_is_adapter_ready(padapter, pkt) == _FALSE)
+#ifdef CONFIG_LAYER2_ROAMING
+		&& (!padapter->mlmepriv.roam_network)
+#endif
+	)
+		goto drop_packet;
+
+	if (adapter_to_pwrctl(padapter)->bInSuspend == _TRUE) {
+		RTW_INFO("[%s] Stop TX because bInSuspend has been carried! \n", __func__);
+		goto drop_packet;
+	}
+
+	PHLTX_LOG;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	os_qid = skb_get_queue_mapping(pkt);
+#endif
+
+#ifdef CONFIG_TCP_CSUM_OFFLOAD_TX
+	if (skb_shinfo(skb)->gso_size) {
+		/*	split a big(65k) skb into several small(1.5k) skbs */
+		features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
+		segs = skb_gso_segment(skb, features);
+		if (IS_ERR(segs) || !segs)
+			goto drop_packet;
+
+		do {
+			nskb = segs;
+			segs = segs->next;
+			nskb->next = NULL;
+			rtw_mstat_update( MSTAT_TYPE_SKB, MSTAT_ALLOC_SUCCESS, nskb->truesize);
+			res = rtw_core_tx(padapter, &nskb, NULL, os_qid);
+			if (res < 0) {
+				#ifdef DBG_TX_DROP_FRAME
+				RTW_INFO("DBG_TX_DROP_FRAME %s rtw_xmit fail\n", __FUNCTION__);
+				#endif
+				pxmitpriv->tx_drop++;
+				rtw_os_pkt_complete(padapter, nskb);
+			}
+		} while (segs);
+		rtw_os_pkt_complete(padapter, skb);
+		goto exit;
+	}
+#endif
+
+	PHLTX_LOG;
+	if (rtw_core_tx(padapter, &pkt, NULL, os_qid) == FAIL)
+		goto inc_drop_cnt;
+
+	PHLTX_LOG;
+
+	goto exit;
+
+drop_packet:
+	rtw_os_pkt_complete(padapter, pkt);
+
+inc_drop_cnt:
+	pxmitpriv->tx_drop++;
+
+exit:
+	return 0;
+}
+#endif
+
+#ifdef CONFIG_TX_AMSDU_SW_MODE
+static void ieee8023_header_to_rfc1042(struct sk_buff *skb, int pads)
+{
+	void *data;
+	int pad;
+	__be16 len;
+	const int headroom = SNAP_SIZE + 2 + pads;
+
+	if (!skb)
+		return;
+
+	if (skb_headroom(skb) < headroom) {
+		RTW_WARN("%s: headroom=%d isn't enough\n", __func__, skb_headroom(skb));
+		if (pskb_expand_head(skb, headroom, 0, GFP_ATOMIC)) {
+			RTW_ERR("%s: no headroom=%d for skb\n",
+				__func__, headroom);
+			return;
+		}
+	}
+
+	data = skb_push(skb, headroom);
+	memset(data, 0, pads);
+	data += pads;
+	memmove(data, data + SNAP_SIZE + 2, 2 * ETH_ALEN);
+	data += 2 * ETH_ALEN;
+	len = cpu_to_be16(skb->len - pads - 2 * ETH_ALEN - 2);
+	memcpy(data, &len, 2);
+	memcpy(data + 2, rtw_rfc1042_header, SNAP_SIZE);
+}
+
+void rtw_coalesce_tx_amsdu(_adapter *padapter, struct xmit_frame *pxframes[],
+			   int xf_nr, bool amsdu, u32 *pktlen)
+{
+	struct xmit_frame *head_xframe;
+	struct xmit_frame *pxframe;
+	struct sk_buff *skb;
+	struct sk_buff *head_skb;
+	struct sk_buff **frag_tail;
+	int pads;
+	int i;
+
+	/* prepare head xmitframe */
+	head_xframe = pxframes[0];
+	head_skb = head_xframe->pkt;
+
+	ieee8023_header_to_rfc1042(head_skb, 0);
+
+	frag_tail = &skb_shinfo(head_skb)->frag_list;
+	while (*frag_tail)
+		frag_tail = &(*frag_tail)->next;
+
+	for (i = 1; i < xf_nr; i++) {
+		pxframe = pxframes[i];
+		skb = pxframe->pkt;
+
+		if (head_skb->len & 0x03)
+			pads = 4 - (head_skb->len & 0x03);
+		else
+			pads = 0;
+
+		ieee8023_header_to_rfc1042(skb, pads);
+
+		/* free sk accounting to have TP like doing skb_linearize() */
+		if (skb->destructor)
+			skb_orphan(skb);
+
+		/* add this skb to head_skb */
+		head_skb->len += skb->len;
+		head_skb->data_len += skb->len;
+		*frag_tail = skb;
+		while (*frag_tail)
+			frag_tail = &(*frag_tail)->next;
+
+		/* free this xframe */
+		pxframe->pkt = NULL; /* head xframe own */
+		core_tx_free_xmitframe(padapter, pxframe);
+	}
+
+	/* total skb length (includes all fragments) */
+	*pktlen = head_skb->len;
+}
+#endif /* CONFIG_TX_AMSDU_SW_MODE */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/osdep_service.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/osdep_service.c
new file mode 100644
index 000000000000..416320ac5ced
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/osdep_service.c
@@ -0,0 +1,1214 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _OSDEP_SERVICE_C_
+
+#include <drv_types.h>
+
+#ifdef CONFIG_HWSIM
+#include "rtw_hwsim_intf.h"
+inline int _rtw_netif_rx(_nic_hdl ndev, struct sk_buff *skb)
+{
+	skb->dev = ndev;
+	rtw_hwsim_medium_pre_netif_rx(skb);
+	return netif_rx(skb);
+}
+#endif /* CONFIG_HWSIM */
+
+u32 rtw_atoi(u8 *s)
+{
+
+	int num = 0, flag = 0;
+	int i;
+	for (i = 0; i <= strlen(s); i++) {
+		if (s[i] >= '0' && s[i] <= '9')
+			num = num * 10 + s[i] - '0';
+		else if (s[0] == '-' && i == 0)
+			flag = 1;
+		else
+			break;
+	}
+
+	if (flag == 1)
+		num = num * -1;
+
+	return num;
+
+}
+
+#if defined(DBG_MEM_ALLOC)
+
+struct rtw_mem_stat {
+	ATOMIC_T alloc; /* the memory bytes we allocate currently */
+	ATOMIC_T peak; /* the peak memory bytes we allocate */
+	ATOMIC_T alloc_cnt; /* the alloc count for alloc currently */
+	ATOMIC_T alloc_err_cnt; /* the error times we fail to allocate memory */
+};
+
+struct rtw_mem_stat rtw_mem_type_stat[mstat_tf_idx(MSTAT_TYPE_MAX)];
+#ifdef RTW_MEM_FUNC_STAT
+struct rtw_mem_stat rtw_mem_func_stat[mstat_ff_idx(MSTAT_FUNC_MAX)];
+#endif
+
+char *MSTAT_TYPE_str[] = {
+	"VIR",
+	"PHY",
+	"SKB",
+	"USB",
+};
+
+#ifdef RTW_MEM_FUNC_STAT
+char *MSTAT_FUNC_str[] = {
+	"UNSP",
+	"IO",
+	"TXIO",
+	"RXIO",
+	"TX",
+	"RX",
+};
+#endif
+
+void rtw_mstat_dump(void *sel)
+{
+	int i;
+	int value_t[4][mstat_tf_idx(MSTAT_TYPE_MAX)];
+#ifdef RTW_MEM_FUNC_STAT
+	int value_f[4][mstat_ff_idx(MSTAT_FUNC_MAX)];
+#endif
+
+	for (i = 0; i < mstat_tf_idx(MSTAT_TYPE_MAX); i++) {
+		value_t[0][i] = ATOMIC_READ(&(rtw_mem_type_stat[i].alloc));
+		value_t[1][i] = ATOMIC_READ(&(rtw_mem_type_stat[i].peak));
+		value_t[2][i] = ATOMIC_READ(&(rtw_mem_type_stat[i].alloc_cnt));
+		value_t[3][i] = ATOMIC_READ(&(rtw_mem_type_stat[i].alloc_err_cnt));
+	}
+
+#ifdef RTW_MEM_FUNC_STAT
+	for (i = 0; i < mstat_ff_idx(MSTAT_FUNC_MAX); i++) {
+		value_f[0][i] = ATOMIC_READ(&(rtw_mem_func_stat[i].alloc));
+		value_f[1][i] = ATOMIC_READ(&(rtw_mem_func_stat[i].peak));
+		value_f[2][i] = ATOMIC_READ(&(rtw_mem_func_stat[i].alloc_cnt));
+		value_f[3][i] = ATOMIC_READ(&(rtw_mem_func_stat[i].alloc_err_cnt));
+	}
+#endif
+
+	RTW_PRINT_SEL(sel, "===================== MSTAT =====================\n");
+	RTW_PRINT_SEL(sel, "%4s %10s %10s %10s %10s\n", "TAG", "alloc", "peak", "aloc_cnt", "err_cnt");
+	RTW_PRINT_SEL(sel, "-------------------------------------------------\n");
+	for (i = 0; i < mstat_tf_idx(MSTAT_TYPE_MAX); i++)
+		RTW_PRINT_SEL(sel, "%4s %10d %10d %10d %10d\n", MSTAT_TYPE_str[i], value_t[0][i], value_t[1][i], value_t[2][i], value_t[3][i]);
+#ifdef RTW_MEM_FUNC_STAT
+	RTW_PRINT_SEL(sel, "-------------------------------------------------\n");
+	for (i = 0; i < mstat_ff_idx(MSTAT_FUNC_MAX); i++)
+		RTW_PRINT_SEL(sel, "%4s %10d %10d %10d %10d\n", MSTAT_FUNC_str[i], value_f[0][i], value_f[1][i], value_f[2][i], value_f[3][i]);
+#endif
+}
+
+void rtw_mstat_update(const enum mstat_f flags, const MSTAT_STATUS status, u32 sz)
+{
+	static systime update_time = 0;
+	int peak, alloc;
+	int i;
+
+	/* initialization */
+	if (!update_time) {
+		for (i = 0; i < mstat_tf_idx(MSTAT_TYPE_MAX); i++) {
+			ATOMIC_SET(&(rtw_mem_type_stat[i].alloc), 0);
+			ATOMIC_SET(&(rtw_mem_type_stat[i].peak), 0);
+			ATOMIC_SET(&(rtw_mem_type_stat[i].alloc_cnt), 0);
+			ATOMIC_SET(&(rtw_mem_type_stat[i].alloc_err_cnt), 0);
+		}
+		#ifdef RTW_MEM_FUNC_STAT
+		for (i = 0; i < mstat_ff_idx(MSTAT_FUNC_MAX); i++) {
+			ATOMIC_SET(&(rtw_mem_func_stat[i].alloc), 0);
+			ATOMIC_SET(&(rtw_mem_func_stat[i].peak), 0);
+			ATOMIC_SET(&(rtw_mem_func_stat[i].alloc_cnt), 0);
+			ATOMIC_SET(&(rtw_mem_func_stat[i].alloc_err_cnt), 0);
+		}
+		#endif
+	}
+
+	switch (status) {
+	case MSTAT_ALLOC_SUCCESS:
+		ATOMIC_INC(&(rtw_mem_type_stat[mstat_tf_idx(flags)].alloc_cnt));
+		alloc = ATOMIC_ADD_RETURN(&(rtw_mem_type_stat[mstat_tf_idx(flags)].alloc), sz);
+		peak = ATOMIC_READ(&(rtw_mem_type_stat[mstat_tf_idx(flags)].peak));
+		if (peak < alloc)
+			ATOMIC_SET(&(rtw_mem_type_stat[mstat_tf_idx(flags)].peak), alloc);
+
+		#ifdef RTW_MEM_FUNC_STAT
+		ATOMIC_INC(&(rtw_mem_func_stat[mstat_ff_idx(flags)].alloc_cnt));
+		alloc = ATOMIC_ADD_RETURN(&(rtw_mem_func_stat[mstat_ff_idx(flags)].alloc), sz);
+		peak = ATOMIC_READ(&(rtw_mem_func_stat[mstat_ff_idx(flags)].peak));
+		if (peak < alloc)
+			ATOMIC_SET(&(rtw_mem_func_stat[mstat_ff_idx(flags)].peak), alloc);
+		#endif
+		break;
+
+	case MSTAT_ALLOC_FAIL:
+		ATOMIC_INC(&(rtw_mem_type_stat[mstat_tf_idx(flags)].alloc_err_cnt));
+		#ifdef RTW_MEM_FUNC_STAT
+		ATOMIC_INC(&(rtw_mem_func_stat[mstat_ff_idx(flags)].alloc_err_cnt));
+		#endif
+		break;
+
+	case MSTAT_FREE:
+		ATOMIC_DEC(&(rtw_mem_type_stat[mstat_tf_idx(flags)].alloc_cnt));
+		ATOMIC_SUB(&(rtw_mem_type_stat[mstat_tf_idx(flags)].alloc), sz);
+		#ifdef RTW_MEM_FUNC_STAT
+		ATOMIC_DEC(&(rtw_mem_func_stat[mstat_ff_idx(flags)].alloc_cnt));
+		ATOMIC_SUB(&(rtw_mem_func_stat[mstat_ff_idx(flags)].alloc), sz);
+		#endif
+		break;
+	};
+
+	/* if (rtw_get_passing_time_ms(update_time) > 5000) { */
+	/*	rtw_mstat_dump(RTW_DBGDUMP); */
+	update_time = rtw_get_current_time();
+	/* } */
+}
+
+#ifndef SIZE_MAX
+#define SIZE_MAX (~(size_t)0)
+#endif
+
+struct mstat_sniff_rule {
+	enum mstat_f flags;
+	size_t lb;
+	size_t hb;
+};
+
+struct mstat_sniff_rule mstat_sniff_rules[] = {
+	/*{MSTAT_TYPE_VIR, 32, 32},*/
+};
+
+int mstat_sniff_rule_num = sizeof(mstat_sniff_rules) / sizeof(struct mstat_sniff_rule);
+
+bool match_mstat_sniff_rules(const enum mstat_f flags, const size_t size)
+{
+	int i;
+	for (i = 0; i < mstat_sniff_rule_num; i++) {
+		if (mstat_sniff_rules[i].flags == flags
+			&& mstat_sniff_rules[i].lb <= size
+			&& mstat_sniff_rules[i].hb >= size)
+			return _TRUE;
+	}
+
+	return _FALSE;
+}
+
+inline void *dbg_rtw_vmalloc(u32 sz, const enum mstat_f flags, const char *func, const int line)
+{
+	void *p;
+
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d)\n", func, line, __FUNCTION__, (sz));
+
+	p = _rtw_vmalloc((sz));
+
+	rtw_mstat_update(
+		flags
+		, p ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, sz
+	);
+
+	return p;
+}
+
+inline void *dbg_rtw_zvmalloc(u32 sz, const enum mstat_f flags, const char *func, const int line)
+{
+	void *p;
+
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d)\n", func, line, __FUNCTION__, (sz));
+
+	p = _rtw_zvmalloc((sz));
+
+	rtw_mstat_update(
+		flags
+		, p ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, sz
+	);
+
+	return p;
+}
+
+inline void dbg_rtw_vmfree(void *pbuf, u32 sz, const enum mstat_f flags, const char *func, const int line)
+{
+
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d)\n", func, line, __FUNCTION__, (sz));
+
+	_rtw_vmfree((pbuf), (sz));
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, sz
+	);
+}
+
+inline void *dbg_rtw_malloc(u32 sz, const enum mstat_f flags, const char *func, const int line)
+{
+	void *p;
+
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d)\n", func, line, __FUNCTION__, (sz));
+
+	p = _rtw_malloc((sz));
+
+	rtw_mstat_update(
+		flags
+		, p ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, sz
+	);
+
+	return p;
+}
+
+inline void *dbg_rtw_zmalloc(u32 sz, const enum mstat_f flags, const char *func, const int line)
+{
+	void *p;
+
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d)\n", func, line, __FUNCTION__, (sz));
+
+	p = _rtw_zmalloc((sz));
+
+	rtw_mstat_update(
+		flags
+		, p ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, sz
+	);
+
+	return p;
+}
+
+inline void dbg_rtw_mfree(void *pbuf, u32 sz, const enum mstat_f flags, const char *func, const int line)
+{
+	if (match_mstat_sniff_rules(flags, sz))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d)\n", func, line, __FUNCTION__, (sz));
+
+	_rtw_mfree((pbuf), (sz));
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, sz
+	);
+}
+
+inline void dbg_rtw_skb_mstat_aid(struct sk_buff *skb_head, const enum mstat_f flags, enum mstat_status status)
+{
+	unsigned int truesize = 0;
+	struct sk_buff *skb;
+
+	if (!skb_head)
+		return;
+
+	rtw_mstat_update(flags, status, skb_head->truesize);
+
+	skb_walk_frags(skb_head, skb)
+		rtw_mstat_update(flags, status, skb->truesize);
+}
+
+inline struct sk_buff *dbg_rtw_skb_alloc(unsigned int size, const enum mstat_f flags, const char *func, int line)
+{
+	struct sk_buff *skb;
+	unsigned int truesize = 0;
+
+	skb = _rtw_skb_alloc(size);
+
+	if (skb)
+		truesize = skb->truesize;
+
+	if (!skb || truesize < size || match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%d), skb:%p, truesize=%u\n", func, line, __FUNCTION__, size, skb, truesize);
+
+	rtw_mstat_update(
+		flags
+		, skb ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, truesize
+	);
+
+	return skb;
+}
+
+inline void dbg_rtw_skb_free(struct sk_buff *skb, const enum mstat_f flags, const char *func, int line)
+{
+	unsigned int truesize = skb->truesize;
+
+	if (match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s, truesize=%u\n", func, line, __FUNCTION__, truesize);
+
+	dbg_rtw_skb_mstat_aid(skb, flags, MSTAT_FREE);
+
+	_rtw_skb_free(skb);
+}
+
+inline struct sk_buff *dbg_rtw_skb_copy(const struct sk_buff *skb, const enum mstat_f flags, const char *func, const int line)
+{
+	struct sk_buff *skb_cp;
+	unsigned int truesize = skb->truesize;
+	unsigned int cp_truesize = 0;
+
+	skb_cp = _rtw_skb_copy(skb);
+	if (skb_cp)
+		cp_truesize = skb_cp->truesize;
+
+	if (!skb_cp || cp_truesize < truesize || match_mstat_sniff_rules(flags, cp_truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%u), skb_cp:%p, cp_truesize=%u\n", func, line, __FUNCTION__, truesize, skb_cp, cp_truesize);
+
+	rtw_mstat_update(
+		flags
+		, skb_cp ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, cp_truesize
+	);
+
+	return skb_cp;
+}
+
+inline struct sk_buff *dbg_rtw_skb_clone(struct sk_buff *skb, const enum mstat_f flags, const char *func, const int line)
+{
+	struct sk_buff *skb_cl;
+	unsigned int truesize = skb->truesize;
+	unsigned int cl_truesize = 0;
+
+	skb_cl = _rtw_skb_clone(skb);
+	if (skb_cl)
+		cl_truesize = skb_cl->truesize;
+
+	if (!skb_cl || cl_truesize < truesize || match_mstat_sniff_rules(flags, cl_truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%u), skb_cl:%p, cl_truesize=%u\n", func, line, __FUNCTION__, truesize, skb_cl, cl_truesize);
+
+	rtw_mstat_update(
+		flags
+		, skb_cl ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, cl_truesize
+	);
+
+	return skb_cl;
+}
+
+inline int dbg_rtw_skb_linearize(struct sk_buff *skb, const enum mstat_f flags, const char *func, int line)
+{
+	unsigned int truesize = 0;
+	int ret;
+
+	dbg_rtw_skb_mstat_aid(skb, flags, MSTAT_FREE);
+
+	ret = _rtw_skb_linearize(skb);
+
+	dbg_rtw_skb_mstat_aid(skb, flags, MSTAT_ALLOC_SUCCESS);
+
+	return ret;
+}
+
+inline int dbg_rtw_netif_rx(_nic_hdl ndev, struct sk_buff *skb, const enum mstat_f flags, const char *func, int line)
+{
+	int ret;
+	unsigned int truesize = skb->truesize;
+
+	if (match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s, truesize=%u\n", func, line, __FUNCTION__, truesize);
+
+	ret = _rtw_netif_rx(ndev, skb);
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, truesize
+	);
+
+	return ret;
+}
+
+#ifdef CONFIG_RTW_NAPI
+inline int dbg_rtw_netif_receive_skb(_nic_hdl ndev, struct sk_buff *skb, const enum mstat_f flags, const char *func, int line)
+{
+	int ret;
+	unsigned int truesize = skb->truesize;
+
+	if (match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s, truesize=%u\n", func, line, __FUNCTION__, truesize);
+
+	ret = _rtw_netif_receive_skb(ndev, skb);
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, truesize
+	);
+
+	return ret;
+}
+
+#ifdef CONFIG_RTW_GRO
+inline gro_result_t dbg_rtw_napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb, const enum mstat_f flags, const char *func, int line)
+{
+	int ret;
+	unsigned int truesize = skb->truesize;
+
+	if (match_mstat_sniff_rules(flags, truesize))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s, truesize=%u\n", func, line, __FUNCTION__, truesize);
+
+	ret = _rtw_napi_gro_receive(napi, skb);
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, truesize
+	);
+
+	return ret;
+}
+#endif /* CONFIG_RTW_GRO */
+#endif /* CONFIG_RTW_NAPI */
+
+inline void dbg_rtw_skb_queue_purge(struct sk_buff_head *list, enum mstat_f flags, const char *func, int line)
+{
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(list)) != NULL)
+		dbg_rtw_skb_free(skb, flags, func, line);
+}
+
+#ifdef CONFIG_USB_HCI
+inline void *dbg_rtw_usb_buffer_alloc(struct usb_device *dev, size_t size, dma_addr_t *dma, const enum mstat_f flags, const char *func, int line)
+{
+	void *p;
+
+	if (match_mstat_sniff_rules(flags, size))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%zu)\n", func, line, __FUNCTION__, size);
+
+	p = _rtw_usb_buffer_alloc(dev, size, dma);
+
+	rtw_mstat_update(
+		flags
+		, p ? MSTAT_ALLOC_SUCCESS : MSTAT_ALLOC_FAIL
+		, size
+	);
+
+	return p;
+}
+
+inline void dbg_rtw_usb_buffer_free(struct usb_device *dev, size_t size, void *addr, dma_addr_t dma, const enum mstat_f flags, const char *func, int line)
+{
+
+	if (match_mstat_sniff_rules(flags, size))
+		RTW_INFO("DBG_MEM_ALLOC %s:%d %s(%zu)\n", func, line, __FUNCTION__, size);
+
+	_rtw_usb_buffer_free(dev, size, addr, dma);
+
+	rtw_mstat_update(
+		flags
+		, MSTAT_FREE
+		, size
+	);
+}
+#endif /* CONFIG_USB_HCI */
+
+#endif /* defined(DBG_MEM_ALLOC) */
+
+void *rtw_malloc2d(int h, int w, size_t size)
+{
+	int j;
+
+	void **a = (void **) rtw_zmalloc(h * sizeof(void *) + h * w * size);
+	if (a == NULL) {
+		RTW_INFO("%s: alloc memory fail!\n", __FUNCTION__);
+		return NULL;
+	}
+
+	for (j = 0; j < h; j++)
+		a[j] = ((char *)(a + h)) + j * w * size;
+
+	return a;
+}
+
+void rtw_mfree2d(void *pbuf, int h, int w, int size)
+{
+	rtw_mfree((u8 *)pbuf, h * sizeof(void *) + w * h * size);
+}
+
+int _rtw_memcmp2(const void *dst, const void *src, u32 sz)
+{
+	const unsigned char *p1 = dst, *p2 = src;
+
+	if (sz == 0)
+		return 0;
+
+	while (*p1 == *p2) {
+		p1++;
+		p2++;
+		sz--;
+		if (sz == 0)
+			return 0;
+	}
+
+	return *p1 - *p2;
+}
+
+void _rtw_init_queue(_queue *pqueue)
+{
+	_rtw_init_listhead(&(pqueue->queue));
+	_rtw_spinlock_init(&(pqueue->lock));
+}
+
+void _rtw_deinit_queue(_queue *pqueue)
+{
+	_rtw_spinlock_free(&(pqueue->lock));
+}
+
+u32 _rtw_queue_empty(_queue	*pqueue)
+{
+	return rtw_is_list_empty(&(pqueue->queue));
+}
+
+
+u32 rtw_end_of_queue_search(_list *head, _list *plist)
+{
+	if (head == plist)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+/* the input parameter start use the same unit as returned by rtw_get_current_time */
+inline s32 _rtw_get_passing_time_ms(systime start)
+{
+	return _rtw_systime_to_ms(_rtw_get_current_time() - start);
+}
+
+inline s32 _rtw_get_remaining_time_ms(systime end)
+{
+	return _rtw_systime_to_ms(end - _rtw_get_current_time());
+}
+
+inline s32 _rtw_get_time_interval_ms(systime start, systime end)
+{
+	return _rtw_systime_to_ms(end - start);
+}
+
+bool rtw_macaddr_is_larger(const u8 *a, const u8 *b)
+{
+	u32 va, vb;
+
+	va = be32_to_cpu(*((u32 *)a));
+	vb = be32_to_cpu(*((u32 *)b));
+	if (va > vb)
+		return 1;
+	else if (va < vb)
+		return 0;
+
+	return be16_to_cpu(*((u16 *)(a + 4))) > be16_to_cpu(*((u16 *)(b + 4)));
+}
+
+
+/*
+* Test if the specifi @param path is a readable file with valid size.
+* If readable, @param sz is got
+* @param path the path of the file to test
+* @return _TRUE or _FALSE
+*/
+int rtw_readable_file_sz_chk(const char *path, u32 sz)
+{
+	u32 fsz;
+
+	if (rtw_is_file_readable_with_size(path, &fsz) == _FALSE)
+		return _FALSE;
+
+	if (fsz > sz)
+		return _FALSE;
+	
+	return _TRUE;
+}
+
+void rtw_buf_free(u8 **buf, u32 *buf_len)
+{
+	u32 ori_len;
+
+	if (!buf || !buf_len)
+		return;
+
+	ori_len = *buf_len;
+
+	if (*buf) {
+		u32 tmp_buf_len = *buf_len;
+		*buf_len = 0;
+		rtw_mfree(*buf, tmp_buf_len);
+		*buf = NULL;
+	}
+}
+
+void rtw_buf_update(u8 **buf, u32 *buf_len, const u8 *src, u32 src_len)
+{
+	u32 ori_len = 0, dup_len = 0;
+	u8 *ori = NULL;
+	u8 *dup = NULL;
+
+	if (!buf || !buf_len)
+		return;
+
+	if (!src || !src_len)
+		goto keep_ori;
+
+	/* duplicate src */
+	dup = rtw_malloc(src_len);
+	if (dup) {
+		dup_len = src_len;
+		_rtw_memcpy(dup, src, dup_len);
+	}
+
+keep_ori:
+	ori = *buf;
+	ori_len = *buf_len;
+
+	/* replace buf with dup */
+	*buf_len = 0;
+	*buf = dup;
+	*buf_len = dup_len;
+
+	/* free ori */
+	if (ori && ori_len > 0)
+		rtw_mfree(ori, ori_len);
+}
+
+
+/**
+ * rtw_cbuf_full - test if cbuf is full
+ * @cbuf: pointer of struct rtw_cbuf
+ *
+ * Returns: _TRUE if cbuf is full
+ */
+inline bool rtw_cbuf_full(struct rtw_cbuf *cbuf)
+{
+	return (cbuf->write == cbuf->read - 1) ? _TRUE : _FALSE;
+}
+
+/**
+ * rtw_cbuf_empty - test if cbuf is empty
+ * @cbuf: pointer of struct rtw_cbuf
+ *
+ * Returns: _TRUE if cbuf is empty
+ */
+inline bool rtw_cbuf_empty(struct rtw_cbuf *cbuf)
+{
+	return (cbuf->write == cbuf->read) ? _TRUE : _FALSE;
+}
+
+/**
+ * rtw_cbuf_push - push a pointer into cbuf
+ * @cbuf: pointer of struct rtw_cbuf
+ * @buf: pointer to push in
+ *
+ * Lock free operation, be careful of the use scheme
+ * Returns: _TRUE push success
+ */
+bool rtw_cbuf_push(struct rtw_cbuf *cbuf, void *buf)
+{
+	if (rtw_cbuf_full(cbuf))
+		return _FAIL;
+
+	if (0)
+		RTW_INFO("%s on %u\n", __func__, cbuf->write);
+	cbuf->bufs[cbuf->write] = buf;
+	cbuf->write = (cbuf->write + 1) % cbuf->size;
+
+	return _SUCCESS;
+}
+
+/**
+ * rtw_cbuf_pop - pop a pointer from cbuf
+ * @cbuf: pointer of struct rtw_cbuf
+ *
+ * Lock free operation, be careful of the use scheme
+ * Returns: pointer popped out
+ */
+void *rtw_cbuf_pop(struct rtw_cbuf *cbuf)
+{
+	void *buf;
+	if (rtw_cbuf_empty(cbuf))
+		return NULL;
+
+	if (0)
+		RTW_INFO("%s on %u\n", __func__, cbuf->read);
+	buf = cbuf->bufs[cbuf->read];
+	cbuf->read = (cbuf->read + 1) % cbuf->size;
+
+	return buf;
+}
+
+/**
+ * rtw_cbuf_alloc - allocte a rtw_cbuf with given size and do initialization
+ * @size: size of pointer
+ *
+ * Returns: pointer of srtuct rtw_cbuf, NULL for allocation failure
+ */
+struct rtw_cbuf *rtw_cbuf_alloc(u32 size)
+{
+	struct rtw_cbuf *cbuf;
+
+	cbuf = (struct rtw_cbuf *)rtw_malloc(sizeof(*cbuf) + sizeof(void *) * size);
+
+	if (cbuf) {
+		cbuf->write = cbuf->read = 0;
+		cbuf->size = size;
+	}
+
+	return cbuf;
+}
+
+/**
+ * rtw_cbuf_free - free the given rtw_cbuf
+ * @cbuf: pointer of struct rtw_cbuf to free
+ */
+void rtw_cbuf_free(struct rtw_cbuf *cbuf)
+{
+	rtw_mfree((u8 *)cbuf, sizeof(*cbuf) + sizeof(void *) * cbuf->size);
+}
+
+/**
+ * map_readN - read a range of map data
+ * @map: map to read
+ * @offset: start address to read
+ * @len: length to read
+ * @buf: pointer of buffer to store data read
+ *
+ * Returns: _SUCCESS or _FAIL
+ */
+int map_readN(const struct map_t *map, u16 offset, u16 len, u8 *buf)
+{
+	const struct map_seg_t *seg;
+	int ret = _FAIL;
+	int i;
+
+	if (len == 0) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	if (offset + len > map->len) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	_rtw_memset(buf, map->init_value, len);
+
+	for (i = 0; i < map->seg_num; i++) {
+		u8 *c_dst, *c_src;
+		u16 c_len;
+
+		seg = map->segs + i;
+		if (seg->sa + seg->len <= offset || seg->sa >= offset + len)
+			continue;
+
+		if (seg->sa >= offset) {
+			c_dst = buf + (seg->sa - offset);
+			c_src = seg->c;
+			if (seg->sa + seg->len <= offset + len)
+				c_len = seg->len;
+			else
+				c_len = offset + len - seg->sa;
+		} else {
+			c_dst = buf;
+			c_src = seg->c + (offset - seg->sa);
+			if (seg->sa + seg->len >= offset + len)
+				c_len = len;
+			else
+				c_len = seg->sa + seg->len - offset;
+		}
+			
+		_rtw_memcpy(c_dst, c_src, c_len);
+	}
+
+exit:
+	return ret;
+}
+
+/**
+ * map_read8 - read 1 byte of map data
+ * @map: map to read
+ * @offset: address to read
+ *
+ * Returns: value of data of specified offset. map.init_value if offset is out of range
+ */
+u8 map_read8(const struct map_t *map, u16 offset)
+{
+	const struct map_seg_t *seg;
+	u8 val = map->init_value;
+	int i;
+
+	if (offset + 1 > map->len) {
+		rtw_warn_on(1);
+		goto exit;
+	}
+
+	for (i = 0; i < map->seg_num; i++) {
+		seg = map->segs + i;
+		if (seg->sa + seg->len <= offset || seg->sa >= offset + 1)
+			continue;
+
+		val = *(seg->c + offset - seg->sa);
+		break;
+	}
+
+exit:
+	return val;
+}
+
+int rtw_blacklist_add(_queue *blist, const u8 *addr, u32 timeout_ms)
+{
+	struct blacklist_ent *ent;
+	_list *list, *head;
+	u8 exist = _FALSE, timeout = _FALSE;
+
+	_rtw_spinlock_bh(&blist->lock);
+
+	head = &blist->queue;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		ent = LIST_CONTAINOR(list, struct blacklist_ent, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(ent->addr, addr, ETH_ALEN) == _TRUE) {
+			exist = _TRUE;
+			if (rtw_time_after(rtw_get_current_time(), ent->exp_time))
+				timeout = _TRUE;
+			ent->exp_time = rtw_get_current_time()
+				+ rtw_ms_to_systime(timeout_ms);
+			break;
+		}
+
+		if (rtw_time_after(rtw_get_current_time(), ent->exp_time)) {
+			rtw_list_delete(&ent->list);
+			rtw_mfree(ent, sizeof(struct blacklist_ent));
+		}
+	}
+
+	if (exist == _FALSE) {
+		ent = rtw_malloc(sizeof(struct blacklist_ent));
+		if (ent) {
+			_rtw_memcpy(ent->addr, addr, ETH_ALEN);
+			ent->exp_time = rtw_get_current_time()
+				+ rtw_ms_to_systime(timeout_ms);
+			rtw_list_insert_tail(&ent->list, head);
+		}
+	}
+
+	_rtw_spinunlock_bh(&blist->lock);
+
+	return (exist == _TRUE && timeout == _FALSE) ? RTW_ALREADY : (ent ? _SUCCESS : _FAIL);
+}
+
+int rtw_blacklist_del(_queue *blist, const u8 *addr)
+{
+	struct blacklist_ent *ent = NULL;
+	_list *list, *head;
+	u8 exist = _FALSE;
+
+	_rtw_spinlock_bh(&blist->lock);
+	head = &blist->queue;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		ent = LIST_CONTAINOR(list, struct blacklist_ent, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(ent->addr, addr, ETH_ALEN) == _TRUE) {
+			rtw_list_delete(&ent->list);
+			rtw_mfree(ent, sizeof(struct blacklist_ent));
+			exist = _TRUE;
+			break;
+		}
+
+		if (rtw_time_after(rtw_get_current_time(), ent->exp_time)) {
+			rtw_list_delete(&ent->list);
+			rtw_mfree(ent, sizeof(struct blacklist_ent));
+		}
+	}
+
+	_rtw_spinunlock_bh(&blist->lock);
+
+	return exist == _TRUE ? _SUCCESS : RTW_ALREADY;
+}
+
+int rtw_blacklist_search(_queue *blist, const u8 *addr)
+{
+	struct blacklist_ent *ent = NULL;
+	_list *list, *head;
+	u8 exist = _FALSE;
+
+	_rtw_spinlock_bh(&blist->lock);
+	head = &blist->queue;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		ent = LIST_CONTAINOR(list, struct blacklist_ent, list);
+		list = get_next(list);
+
+		if (_rtw_memcmp(ent->addr, addr, ETH_ALEN) == _TRUE) {
+			if (rtw_time_after(rtw_get_current_time(), ent->exp_time)) {
+				rtw_list_delete(&ent->list);
+				rtw_mfree(ent, sizeof(struct blacklist_ent));
+			} else
+				exist = _TRUE;
+			break;
+		}
+
+		if (rtw_time_after(rtw_get_current_time(), ent->exp_time)) {
+			rtw_list_delete(&ent->list);
+			rtw_mfree(ent, sizeof(struct blacklist_ent));
+		}
+	}
+
+	_rtw_spinunlock_bh(&blist->lock);
+
+	return exist;
+}
+
+void rtw_blacklist_flush(_queue *blist)
+{
+	struct blacklist_ent *ent;
+	_list *list, *head;
+	_list tmp;
+
+	_rtw_init_listhead(&tmp);
+
+	_rtw_spinlock_bh(&blist->lock);
+	rtw_list_splice_init(&blist->queue, &tmp);
+	_rtw_spinunlock_bh(&blist->lock);
+
+	head = &tmp;
+	list = get_next(head);
+	while (rtw_end_of_queue_search(head, list) == _FALSE) {
+		ent = LIST_CONTAINOR(list, struct blacklist_ent, list);
+		list = get_next(list);
+		rtw_list_delete(&ent->list);
+		rtw_mfree(ent, sizeof(struct blacklist_ent));
+	}
+}
+
+void dump_blacklist(void *sel, _queue *blist, const char *title)
+{
+	struct blacklist_ent *ent = NULL;
+	_list *list, *head;
+	char mac_addr_str[MAC_FMT_LEN];
+
+	_rtw_spinlock_bh(&blist->lock);
+	head = &blist->queue;
+	list = get_next(head);
+
+	if (rtw_end_of_queue_search(head, list) == _FALSE) {
+		if (title)
+			RTW_PRINT_SEL(sel, "%s:\n", title);
+	
+		while (rtw_end_of_queue_search(head, list) == _FALSE) {
+			ent = LIST_CONTAINOR(list, struct blacklist_ent, list);
+			list = get_next(list);
+			if (rtw_time_after(rtw_get_current_time(), ent->exp_time))
+				RTW_PRINT_SEL(sel, "%s expired\n", get_macaddr_str(mac_addr_str, sel, ent->addr));
+			else
+				RTW_PRINT_SEL(sel, "%s %u\n", get_macaddr_str(mac_addr_str, sel, ent->addr)
+					, rtw_get_remaining_time_ms(ent->exp_time));
+		}
+
+	}
+	_rtw_spinunlock_bh(&blist->lock);
+}
+
+/**
+* is_null -
+*
+* Return	TRUE if c is null character
+*		FALSE otherwise.
+*/
+inline BOOLEAN is_null(char c)
+{
+	if (c == '\0')
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+inline BOOLEAN is_all_null(char *c, int len)
+{
+	for (; len > 0; len--)
+		if (c[len - 1] != '\0')
+			return _FALSE;
+
+	return _TRUE;
+}
+
+/**
+* is_eol -
+*
+* Return	TRUE if c is represent for EOL (end of line)
+*		FALSE otherwise.
+*/
+inline BOOLEAN is_eol(char c)
+{
+	if (c == '\r' || c == '\n')
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+/**
+* is_space -
+*
+* Return	TRUE if c is represent for space
+*		FALSE otherwise.
+*/
+inline BOOLEAN is_space(char c)
+{
+	if (c == ' ' || c == '\t')
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+/**
+* is_decimal -
+*
+* Return	TRUE if chTmp is represent for decimal digit
+*		FALSE otherwise.
+*/
+inline BOOLEAN is_decimal(char chTmp)
+{
+	if ((chTmp >= '0' && chTmp <= '9'))
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+/**
+* IsHexDigit -
+*
+* Return	TRUE if chTmp is represent for hex digit
+*		FALSE otherwise.
+*/
+inline BOOLEAN IsHexDigit(char chTmp)
+{
+	if ((chTmp >= '0' && chTmp <= '9') ||
+		(chTmp >= 'a' && chTmp <= 'f') ||
+		(chTmp >= 'A' && chTmp <= 'F'))
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+/**
+* is_alpha -
+*
+* Return	TRUE if chTmp is represent for alphabet
+*		FALSE otherwise.
+*/
+inline BOOLEAN is_alpha(char chTmp)
+{
+	if ((chTmp >= 'a' && chTmp <= 'z') ||
+		(chTmp >= 'A' && chTmp <= 'Z'))
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+inline char alpha_to_upper(char c)
+{
+	if ((c >= 'a' && c <= 'z'))
+		c = 'A' + (c - 'a');
+	return c;
+}
+
+int hex2num_i(char c)
+{
+	if (c >= '0' && c <= '9')
+		return c - '0';
+	if (c >= 'a' && c <= 'f')
+		return c - 'a' + 10;
+	if (c >= 'A' && c <= 'F')
+		return c - 'A' + 10;
+	return -1;
+}
+
+int hex2byte_i(const char *hex)
+{
+	int a, b;
+	a = hex2num_i(*hex++);
+	if (a < 0)
+		return -1;
+	b = hex2num_i(*hex++);
+	if (b < 0)
+		return -1;
+	return (a << 4) | b;
+}
+
+int hexstr2bin(const char *hex, u8 *buf, size_t len)
+{
+	size_t i;
+	int a;
+	const char *ipos = hex;
+	u8 *opos = buf;
+
+	for (i = 0; i < len; i++) {
+		a = hex2byte_i(ipos);
+		if (a < 0)
+			return -1;
+		*opos++ = a;
+		ipos += 2;
+	}
+	return 0;
+}
+
+void ustrs_add(char **ustrs, int *ustrs_len, const char *str)
+{
+	char *tmp_ustrs;
+	int tmp_ustrs_len;
+
+	if (!str || !strlen(str))
+		return;
+
+	tmp_ustrs = *ustrs;
+	tmp_ustrs_len = *ustrs_len;
+	if (tmp_ustrs) {
+		const char *pos;
+
+		/* search for same string */
+		for (pos = tmp_ustrs; pos < tmp_ustrs + tmp_ustrs_len; pos += strlen(pos) + 1) {
+			if (strcmp(pos, str) == 0)
+				return;
+		}
+
+		/* no match, realloc and add */
+		tmp_ustrs = rtw_malloc(tmp_ustrs_len + strlen(str) + 1);
+		if (!tmp_ustrs) {
+			rtw_warn_on(1);
+			return;
+		}
+		_rtw_memcpy((void *)tmp_ustrs, *ustrs, tmp_ustrs_len);
+		_rtw_memcpy((void *)(tmp_ustrs + tmp_ustrs_len), str, strlen(str) + 1);
+		rtw_mfree((void *)*ustrs, tmp_ustrs_len);
+		*ustrs = tmp_ustrs;
+		*ustrs_len += strlen(str) + 1;
+
+	} else {
+		tmp_ustrs = rtw_malloc(strlen(str) + 1);
+		if (!tmp_ustrs) {
+			rtw_warn_on(1);
+			return;
+		}
+		_rtw_memcpy((void *)tmp_ustrs, str, strlen(str) + 1);
+		*ustrs = tmp_ustrs;
+		*ustrs_len = strlen(str) + 1;
+	}
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/os_dep/osdep_service_linux.c b/drivers/net/wireless/realtek/rtl8852bs/os_dep/osdep_service_linux.c
new file mode 100644
index 000000000000..2af36ca70d27
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/os_dep/osdep_service_linux.c
@@ -0,0 +1,898 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _OSDEP_SERVICE_LINUX_C_
+#include <drv_types.h>
+
+#ifdef DBG_MEMORY_LEAK
+ATOMIC_T _malloc_cnt = ATOMIC_INIT(0);
+ATOMIC_T _malloc_size = ATOMIC_INIT(0);
+#endif /* DBG_MEMORY_LEAK */
+
+/*
+* Translate the OS dependent @param error_code to OS independent RTW_STATUS_CODE
+* @return: one of RTW_STATUS_CODE
+*/
+inline int RTW_STATUS_CODE(int error_code)
+{
+	if (error_code >= 0)
+		return _SUCCESS;
+
+	switch (error_code) {
+	/* case -ETIMEDOUT: */
+	/*	return RTW_STATUS_TIMEDOUT; */
+	default:
+		return _FAIL;
+	}
+}
+
+void _rtw_skb_queue_purge(struct sk_buff_head *list)
+{
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(list)) != NULL)
+		_rtw_skb_free(skb);
+}
+
+void _rtw_memcpy(void *dst, const void *src, u32 sz)
+{
+	memcpy(dst, src, sz);
+}
+
+inline void _rtw_memmove(void *dst, const void *src, u32 sz)
+{
+	memmove(dst, src, sz);
+}
+
+int _rtw_memcmp(const void *dst, const void *src, u32 sz)
+{
+	/* under Linux/GNU/GLibc, the return value of memcmp for two same mem. chunk is 0 */
+	if (!(memcmp(dst, src, sz)))
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+void _rtw_memset(void *pbuf, int c, u32 sz)
+{
+	memset(pbuf, c, sz);
+}
+
+void _rtw_init_listhead(_list *list)
+{
+	INIT_LIST_HEAD(list);
+}
+/*
+For the following list_xxx operations,
+caller must guarantee the atomic context.
+Otherwise, there will be racing condition.
+*/
+u32 rtw_is_list_empty(_list *phead)
+{
+	if (list_empty(phead))
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+void rtw_list_insert_head(_list *plist, _list *phead)
+{
+	list_add(plist, phead);
+}
+
+void rtw_list_insert_tail(_list *plist, _list *phead)
+{
+	list_add_tail(plist, phead);
+}
+
+inline void rtw_list_splice(_list *list, _list *head)
+{
+	list_splice(list, head);
+}
+
+inline void rtw_list_splice_init(_list *list, _list *head)
+{
+	list_splice_init(list, head);
+}
+
+inline void rtw_list_splice_tail(_list *list, _list *head)
+{
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27))
+	if (!list_empty(list))
+		__list_splice(list, head);
+	#else
+	list_splice_tail(list, head);
+	#endif
+}
+
+inline void rtw_hlist_head_init(rtw_hlist_head *h)
+{
+	INIT_HLIST_HEAD(h);
+}
+
+inline void rtw_hlist_add_head(rtw_hlist_node *n, rtw_hlist_head *h)
+{
+	hlist_add_head(n, h);
+}
+
+inline void rtw_hlist_del(rtw_hlist_node *n)
+{
+	hlist_del(n);
+}
+
+inline void rtw_hlist_add_head_rcu(rtw_hlist_node *n, rtw_hlist_head *h)
+{
+	hlist_add_head_rcu(n, h);
+}
+
+inline void rtw_hlist_del_rcu(rtw_hlist_node *n)
+{
+	hlist_del_rcu(n);
+}
+
+void rtw_init_timer(_timer *ptimer, void *pfunc, void *ctx)
+{
+	_init_timer(ptimer, pfunc, ctx);
+}
+
+systime _rtw_get_current_time(void)
+{
+	return jiffies;
+}
+
+inline u32 _rtw_systime_to_ms(systime stime)
+{
+	return jiffies_to_msecs(stime);
+}
+
+inline u32 _rtw_systime_to_us(systime stime)
+{
+	return jiffies_to_usecs(stime);
+}
+
+inline systime _rtw_ms_to_systime(u32 ms)
+{
+	return msecs_to_jiffies(ms);
+}
+
+inline systime _rtw_us_to_systime(u32 us)
+{
+	return usecs_to_jiffies(us);
+}
+
+inline bool _rtw_time_after(systime a, systime b)
+{
+	return time_after(a, b);
+}
+
+inline bool _rtw_time_after_eq(systime a, systime b)
+{
+	return time_after_eq(a, b);
+}
+
+void rtw_sleep_schedulable(int ms)
+{
+	u32 delta;
+
+	delta = (ms * HZ) / 1000; /* (ms) */
+	if (delta == 0) {
+		delta = 1;/* 1 ms */
+	}
+	set_current_state(TASK_INTERRUPTIBLE);
+        schedule_timeout(delta);
+	return;
+}
+
+void rtw_msleep_os(int ms)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36))
+	if (ms < 20) {
+		unsigned long us = ms * 1000UL;
+		usleep_range(us, us + 1000UL);
+	} else
+#endif
+		msleep((unsigned int)ms);
+
+}
+void rtw_usleep_os(int us)
+{
+	/* msleep((unsigned int)us); */
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36))
+	usleep_range(us, us + 1);
+#else
+	if (1 < (us / 1000))
+		msleep(1);
+	else
+		msleep((us / 1000) + 1);
+#endif
+}
+
+
+#ifdef DBG_DELAY_OS
+void _rtw_mdelay_os(int ms, const char *func, const int line)
+{
+	RTW_INFO("%s:%d %s(%d)\n", func, line, __FUNCTION__, ms);
+	mdelay((unsigned long)ms);
+}
+void _rtw_udelay_os(int us, const char *func, const int line)
+{
+	RTW_INFO("%s:%d %s(%d)\n", func, line, __FUNCTION__, us);
+	udelay((unsigned long)us);
+}
+#else
+void rtw_mdelay_os(int ms)
+{
+	mdelay((unsigned long)ms);
+}
+void rtw_udelay_os(int us)
+{
+	udelay((unsigned long)us);
+}
+#endif
+
+void rtw_yield_os(void)
+{
+	yield();
+}
+
+
+#define RTW_SUSPEND_LOCK_NAME "rtw_wifi"
+#define RTW_SUSPEND_TRAFFIC_LOCK_NAME "rtw_wifi_traffic"
+#define RTW_SUSPEND_RESUME_LOCK_NAME "rtw_wifi_resume"
+
+#ifdef CONFIG_WAKELOCK
+static struct wake_lock rtw_suspend_lock;
+static struct wake_lock rtw_suspend_traffic_lock;
+static struct wake_lock rtw_suspend_resume_lock;
+#elif defined(CONFIG_ANDROID_POWER)
+static android_suspend_lock_t rtw_suspend_lock = {
+	.name = RTW_SUSPEND_LOCK_NAME
+};
+static android_suspend_lock_t rtw_suspend_traffic_lock = {
+	.name = RTW_SUSPEND_TRAFFIC_LOCK_NAME
+};
+static android_suspend_lock_t rtw_suspend_resume_lock = {
+	.name = RTW_SUSPEND_RESUME_LOCK_NAME
+};
+#endif
+
+inline void rtw_suspend_lock_init(void)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_lock_init(&rtw_suspend_lock, WAKE_LOCK_SUSPEND, RTW_SUSPEND_LOCK_NAME);
+	wake_lock_init(&rtw_suspend_traffic_lock, WAKE_LOCK_SUSPEND, RTW_SUSPEND_TRAFFIC_LOCK_NAME);
+	wake_lock_init(&rtw_suspend_resume_lock, WAKE_LOCK_SUSPEND, RTW_SUSPEND_RESUME_LOCK_NAME);
+#elif defined(CONFIG_ANDROID_POWER)
+	android_init_suspend_lock(&rtw_suspend_lock);
+	android_init_suspend_lock(&rtw_suspend_traffic_lock);
+	android_init_suspend_lock(&rtw_suspend_resume_lock);
+#endif
+}
+
+inline void rtw_suspend_lock_uninit(void)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_lock_destroy(&rtw_suspend_lock);
+	wake_lock_destroy(&rtw_suspend_traffic_lock);
+	wake_lock_destroy(&rtw_suspend_resume_lock);
+#elif defined(CONFIG_ANDROID_POWER)
+	android_uninit_suspend_lock(&rtw_suspend_lock);
+	android_uninit_suspend_lock(&rtw_suspend_traffic_lock);
+	android_uninit_suspend_lock(&rtw_suspend_resume_lock);
+#endif
+}
+
+inline void rtw_lock_suspend(void)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_lock(&rtw_suspend_lock);
+#elif defined(CONFIG_ANDROID_POWER)
+	android_lock_suspend(&rtw_suspend_lock);
+#endif
+
+#if  defined(CONFIG_WAKELOCK) || defined(CONFIG_ANDROID_POWER)
+	/* RTW_INFO("####%s: suspend_lock_count:%d####\n", __FUNCTION__, rtw_suspend_lock.stat.count); */
+#endif
+}
+
+inline void rtw_unlock_suspend(void)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_unlock(&rtw_suspend_lock);
+#elif defined(CONFIG_ANDROID_POWER)
+	android_unlock_suspend(&rtw_suspend_lock);
+#endif
+
+#if  defined(CONFIG_WAKELOCK) || defined(CONFIG_ANDROID_POWER)
+	/* RTW_INFO("####%s: suspend_lock_count:%d####\n", __FUNCTION__, rtw_suspend_lock.stat.count); */
+#endif
+}
+
+inline void rtw_resume_lock_suspend(void)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_lock(&rtw_suspend_resume_lock);
+#elif defined(CONFIG_ANDROID_POWER)
+	android_lock_suspend(&rtw_suspend_resume_lock);
+#endif
+
+#if  defined(CONFIG_WAKELOCK) || defined(CONFIG_ANDROID_POWER)
+	/* RTW_INFO("####%s: suspend_lock_count:%d####\n", __FUNCTION__, rtw_suspend_lock.stat.count); */
+#endif
+}
+
+inline void rtw_resume_unlock_suspend(void)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_unlock(&rtw_suspend_resume_lock);
+#elif defined(CONFIG_ANDROID_POWER)
+	android_unlock_suspend(&rtw_suspend_resume_lock);
+#endif
+
+#if  defined(CONFIG_WAKELOCK) || defined(CONFIG_ANDROID_POWER)
+	/* RTW_INFO("####%s: suspend_lock_count:%d####\n", __FUNCTION__, rtw_suspend_lock.stat.count); */
+#endif
+}
+
+inline void rtw_lock_suspend_timeout(u32 timeout_ms)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_lock_timeout(&rtw_suspend_lock, rtw_ms_to_systime(timeout_ms));
+#elif defined(CONFIG_ANDROID_POWER)
+	android_lock_suspend_auto_expire(&rtw_suspend_lock, rtw_ms_to_systime(timeout_ms));
+#endif
+}
+
+inline void rtw_lock_traffic_suspend_timeout(u32 timeout_ms)
+{
+#ifdef CONFIG_WAKELOCK
+	wake_lock_timeout(&rtw_suspend_traffic_lock, rtw_ms_to_systime(timeout_ms));
+#elif defined(CONFIG_ANDROID_POWER)
+	android_lock_suspend_auto_expire(&rtw_suspend_traffic_lock, rtw_ms_to_systime(timeout_ms));
+#endif
+	/* RTW_INFO("traffic lock timeout:%d\n", timeout_ms); */
+}
+
+inline void rtw_set_bit(int nr, unsigned long *addr)
+{
+	set_bit(nr, addr);
+}
+
+inline void rtw_clear_bit(int nr, unsigned long *addr)
+{
+	clear_bit(nr, addr);
+}
+
+inline int rtw_test_and_clear_bit(int nr, unsigned long *addr)
+{
+	return test_and_clear_bit(nr, addr);
+}
+inline int rtw_test_and_set_bit(int nr, unsigned long *addr)
+{
+	return test_and_set_bit(nr, addr);
+}
+
+#if defined(CONFIG_RTW_ANDROID_GKI) && !defined(CONFIG_LOAD_FILE_BY_REQ_FW_API)
+#define CONFIG_LOAD_FILE_BY_REQ_FW_API
+#endif
+
+#ifdef CONFIG_LOAD_FILE_BY_REQ_FW_API
+#include <linux/firmware.h>
+
+static const char *get_file_name_from_path(const char *path)
+{
+	char *ret;
+	size_t path_len;
+
+	if (!path)
+		return NULL;
+
+	path_len = strlen(path);
+	if (path_len == 0)
+		return NULL;
+
+	ret = strrchr(path, '/');
+	if (ret && ret - path < path_len)
+		return ret + 1;
+	return NULL;
+}
+#endif /* CONFIG_LOAD_FILE_BY_REQ_FW_API */
+
+#if !defined(CONFIG_RTW_ANDROID_GKI)
+/*
+* Open a file with the specific @param path, @param flag, @param mode
+* @param fpp the pointer of struct file pointer to get struct file pointer while file opening is success
+* @param path the path of the file to open
+* @param flag file operation flags, please refer to linux document
+* @param mode please refer to linux document
+* @return Linux specific error code
+*/
+static int openFile(struct file **fpp, const char *path, int flag, int mode)
+{
+	struct file *fp;
+
+	fp = filp_open(path, flag, mode);
+	if (IS_ERR(fp)) {
+		*fpp = NULL;
+		return PTR_ERR(fp);
+	} else {
+		*fpp = fp;
+		return 0;
+	}
+}
+
+/*
+* Close the file with the specific @param fp
+* @param fp the pointer of struct file to close
+* @return always 0
+*/
+static int closeFile(struct file *fp)
+{
+	filp_close(fp, NULL);
+	return 0;
+}
+
+static int readFile(struct file *fp, char *buf, int len)
+{
+	int rlen = 0, sum = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+	if (!(fp->f_mode & FMODE_CAN_READ))
+#else
+	if (!fp->f_op || !fp->f_op->read)
+#endif
+		return -EPERM;
+
+	while (sum < len) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+		rlen = kernel_read(fp, buf + sum, len - sum, &fp->f_pos);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+		rlen = __vfs_read(fp, buf + sum, len - sum, &fp->f_pos);
+#else
+		rlen = fp->f_op->read(fp, buf + sum, len - sum, &fp->f_pos);
+#endif
+		if (rlen > 0)
+			sum += rlen;
+		else if (0 != rlen)
+			return rlen;
+		else
+			break;
+	}
+
+	return  sum;
+
+}
+
+static int writeFile(struct file *fp, char *buf, int len)
+{
+	int wlen = 0, sum = 0;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+	if (!(fp->f_mode & FMODE_CAN_WRITE))
+#else
+	if (!fp->f_op || !fp->f_op->write)
+#endif
+		return -EPERM;
+
+	while (sum < len) {
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0))
+		wlen = kernel_write(fp, buf + sum, len - sum, &fp->f_pos);
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0))
+		wlen = __vfs_write(fp, buf + sum, len - sum, &fp->f_pos);
+#else
+		wlen = fp->f_op->write(fp, buf + sum, len - sum, &fp->f_pos);
+#endif
+		if (wlen > 0)
+			sum += wlen;
+		else if (0 != wlen)
+			return wlen;
+		else
+			break;
+	}
+
+	return sum;
+
+}
+
+/*
+* Test if the specifi @param pathname is a direct and readable
+* If readable, @param sz is not used
+* @param pathname the name of the path to test
+* @return Linux specific error code
+*/
+static int isDirReadable(const char *pathname, u32 *sz)
+{
+	struct path path;
+	int error = 0;
+
+	return kern_path(pathname, LOOKUP_FOLLOW, &path);
+}
+#endif /* !defined(CONFIG_RTW_ANDROID_GKI)*/
+
+/*
+* Test if the specifi @param path is a file and readable
+* If readable, @param sz is got
+* @param path the path of the file to test
+* @return Linux specific error code
+*/
+static int isFileReadable(const char *path, u32 *sz)
+{
+#if defined(CONFIG_LOAD_FILE_BY_REQ_FW_API)
+	int ret = -EINVAL;
+	const struct firmware *fw = NULL;
+	const char *name;
+
+	if (path == NULL) {
+		RTW_ERR("%s() NULL pointer\n", __func__);
+		goto exit;
+	}
+
+	name = get_file_name_from_path(path);
+	if (name == NULL) {
+		RTW_ERR("%s() parsing file name fail\n", __func__);
+		goto exit;
+	}
+
+	/* request_firmware() will find file in /vendor/firmware but not in path */
+	ret = request_firmware(&fw, name, NULL);
+	if (ret != 0) {
+		RTW_ERR("%s() request_firmware file : %s, error : %d\n", __func__, name, ret);
+		goto exit;
+	}
+
+	if (sz)
+		*sz = (u32)fw->size;
+
+exit:
+	if (fw)
+		release_firmware(fw);
+
+	return ret;
+#else /* !defined(CONFIG_LOAD_FILE_BY_REQ_FW_API) */
+	struct file *fp;
+	int ret = 0;
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	mm_segment_t oldfs;
+	#endif
+	char buf;
+
+	fp = filp_open(path, O_RDONLY, 0);
+	if (IS_ERR(fp))
+		ret = PTR_ERR(fp);
+	else {
+		#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+		oldfs = get_fs();
+		#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0))
+		set_fs(KERNEL_DS);
+		#else
+		set_fs(get_ds());
+		#endif
+		#endif
+
+		if (1 != readFile(fp, &buf, 1))
+			ret = PTR_ERR(fp);
+
+		if (ret == 0 && sz) {
+			#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
+			*sz = i_size_read(fp->f_path.dentry->d_inode);
+			#else
+			*sz = i_size_read(fp->f_dentry->d_inode);
+			#endif
+		}
+
+		#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+		set_fs(oldfs);
+		#endif
+		filp_close(fp, NULL);
+	}
+	return ret;
+#endif /* defined(CONFIG_LOAD_FILE_BY_REQ_FW_API) */
+}
+
+/*
+* Open the file with @param path and retrive the file content into memory starting from @param buf for @param sz at most
+* @param path the path of the file to open and read
+* @param buf the starting address of the buffer to store file content
+* @param sz how many bytes to read at most
+* @return the byte we've read, or Linux specific error code
+*/
+static int retriveFromFile(const char *path, u8 *buf, u32 sz)
+{
+#if defined(CONFIG_LOAD_FILE_BY_REQ_FW_API)
+	int ret = -EINVAL;
+	const struct firmware *fw = NULL;
+	const char *name;
+
+	if (path == NULL || buf == NULL) {
+		RTW_ERR("%s() NULL pointer\n", __func__);
+		goto err;
+	}
+
+	name = get_file_name_from_path(path);
+	if (name == NULL) {
+		RTW_ERR("%s() parsing file name fail\n", __func__);
+		goto err;
+	}
+
+	/* request_firmware() will find file in /vendor/firmware but not in path */
+	ret = request_firmware(&fw, name, NULL);
+	if (ret == 0) {
+		RTW_INFO("%s() Success. retrieve file : %s, file size : %zu\n", __func__, name, fw->size);
+
+		if ((u32)fw->size <= sz) {
+			_rtw_memcpy(buf, fw->data, (u32)fw->size);
+			ret = (u32)fw->size;
+			goto exit;
+		} else {
+			RTW_ERR("%s() file size : %zu exceed buf size : %u\n", __func__, fw->size, sz);
+			ret = -EFBIG;
+			goto err;
+		}
+	} else {
+		RTW_ERR("%s() Fail. retrieve file : %s, error : %d\n", __func__, name, ret);
+		goto err;
+	}
+
+
+
+err:
+	RTW_ERR("%s() Fail. retrieve file : %s, error : %d\n", __func__, path, ret);
+exit:
+	if (fw)
+		release_firmware(fw);
+	return ret;
+#else /* !defined(CONFIG_LOAD_FILE_BY_REQ_FW_API) */
+	int ret = -1;
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	mm_segment_t oldfs;
+	#endif
+	struct file *fp;
+
+	if (path && buf) {
+		ret = openFile(&fp, path, O_RDONLY, 0);
+		if (0 == ret) {
+			RTW_INFO("%s openFile path:%s fp=%p\n", __FUNCTION__, path , fp);
+
+			#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+			oldfs = get_fs();
+			#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0))
+			set_fs(KERNEL_DS);
+			#else
+			set_fs(get_ds());
+			#endif
+			#endif
+
+			ret = readFile(fp, buf, sz);
+
+			#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+			set_fs(oldfs);
+			#endif
+			closeFile(fp);
+
+			RTW_INFO("%s readFile, ret:%d\n", __FUNCTION__, ret);
+
+		} else
+			RTW_INFO("%s openFile path:%s Fail, ret:%d\n", __FUNCTION__, path, ret);
+	} else {
+		RTW_INFO("%s NULL pointer\n", __FUNCTION__);
+		ret =  -EINVAL;
+	}
+	return ret;
+#endif /* defined(CONFIG_LOAD_FILE_BY_REQ_FW_API) */
+}
+
+#if !defined(CONFIG_RTW_ANDROID_GKI)
+/*
+* Open the file with @param path and wirte @param sz byte of data starting from @param buf into the file
+* @param path the path of the file to open and write
+* @param buf the starting address of the data to write into file
+* @param sz how many bytes to write at most
+* @return the byte we've written, or Linux specific error code
+*/
+static int storeToFile(const char *path, u8 *buf, u32 sz)
+{
+	int ret = 0;
+	#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+	mm_segment_t oldfs;
+	#endif
+	struct file *fp;
+
+	if (path && buf) {
+		ret = openFile(&fp, path, O_CREAT | O_WRONLY, 0666);
+		if (0 == ret) {
+			RTW_INFO("%s openFile path:%s fp=%p\n", __FUNCTION__, path , fp);
+
+			#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+			oldfs = get_fs();
+			#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0))
+			set_fs(KERNEL_DS);
+			#else
+			set_fs(get_ds());
+			#endif
+			#endif
+
+			ret = writeFile(fp, buf, sz);
+
+			#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0))
+			set_fs(oldfs);
+			#endif
+			closeFile(fp);
+
+			RTW_INFO("%s writeFile, ret:%d\n", __FUNCTION__, ret);
+
+		} else
+			RTW_INFO("%s openFile path:%s Fail, ret:%d\n", __FUNCTION__, path, ret);
+	} else {
+		RTW_INFO("%s NULL pointer\n", __FUNCTION__);
+		ret =  -EINVAL;
+	}
+	return ret;
+}
+
+/*
+* Test if the specifi @param path is a direct and readable
+* @param path the path of the direct to test
+* @return _TRUE or _FALSE
+*/
+int rtw_is_dir_readable(const char *path)
+{
+	if (isDirReadable(path, NULL) == 0)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+#endif /* !defined(CONFIG_RTW_ANDROID_GKI)*/
+
+/*
+* Test if the specifi @param path is a file and readable
+* @param path the path of the file to test
+* @return _TRUE or _FALSE
+*/
+int rtw_is_file_readable(const char *path)
+{
+	if (isFileReadable(path, NULL) == 0)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+/*
+* Test if the specifi @param path is a file and readable.
+* If readable, @param sz is got
+* @param path the path of the file to test
+* @return _TRUE or _FALSE
+*/
+int rtw_is_file_readable_with_size(const char *path, u32 *sz)
+{
+	if (isFileReadable(path, sz) == 0)
+		return _TRUE;
+	else
+		return _FALSE;
+}
+
+/*
+* Open the file with @param path and retrive the file content into memory starting from @param buf for @param sz at most
+* @param path the path of the file to open and read
+* @param buf the starting address of the buffer to store file content
+* @param sz how many bytes to read at most
+* @return the byte we've read
+*/
+int rtw_retrieve_from_file(const char *path, u8 *buf, u32 sz)
+{
+	int ret = retriveFromFile(path, buf, sz);
+	return ret >= 0 ? ret : 0;
+}
+
+#if !defined(CONFIG_RTW_ANDROID_GKI)
+/*
+* Open the file with @param path and wirte @param sz byte of data starting from @param buf into the file
+* @param path the path of the file to open and write
+* @param buf the starting address of the data to write into file
+* @param sz how many bytes to write at most
+* @return the byte we've written
+*/
+int rtw_store_to_file(const char *path, u8 *buf, u32 sz)
+{
+	int ret = storeToFile(path, buf, sz);
+	return ret >= 0 ? ret : 0;
+}
+#endif /* !defined(CONFIG_RTW_ANDROID_GKI) */
+
+struct net_device *rtw_alloc_etherdev_with_old_priv(int sizeof_priv, void *old_priv)
+{
+	struct net_device *pnetdev;
+	struct rtw_netdev_priv_indicator *pnpi;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	pnetdev = alloc_etherdev_mq(sizeof(struct rtw_netdev_priv_indicator), 4);
+#else
+	pnetdev = alloc_etherdev(sizeof(struct rtw_netdev_priv_indicator));
+#endif
+	if (!pnetdev)
+		goto RETURN;
+
+	pnpi = netdev_priv(pnetdev);
+	pnpi->priv = old_priv;
+	pnpi->sizeof_priv = sizeof_priv;
+
+RETURN:
+	return pnetdev;
+}
+
+struct net_device *rtw_alloc_etherdev(int sizeof_priv)
+{
+	struct net_device *pnetdev;
+	struct rtw_netdev_priv_indicator *pnpi;
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35))
+	pnetdev = alloc_etherdev_mq(sizeof(struct rtw_netdev_priv_indicator), 4);
+#else
+	pnetdev = alloc_etherdev(sizeof(struct rtw_netdev_priv_indicator));
+#endif
+	if (!pnetdev)
+		goto RETURN;
+
+	pnpi = netdev_priv(pnetdev);
+
+	pnpi->priv = rtw_zvmalloc(sizeof_priv);
+	if (!pnpi->priv) {
+		free_netdev(pnetdev);
+		pnetdev = NULL;
+		goto RETURN;
+	}
+
+	pnpi->sizeof_priv = sizeof_priv;
+RETURN:
+	return pnetdev;
+}
+
+void rtw_free_netdev(struct net_device *netdev)
+{
+	struct rtw_netdev_priv_indicator *pnpi;
+
+	if (!netdev)
+		goto RETURN;
+
+	pnpi = netdev_priv(netdev);
+
+	if (!pnpi->priv)
+		goto RETURN;
+
+	free_netdev(netdev);
+
+RETURN:
+	return;
+}
+
+inline u32 rtw_random32(void)
+{
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0))
+	return get_random_u32();
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
+	return prandom_u32();
+#elif (LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18))
+	u32 random_int;
+	get_random_bytes(&random_int , 4);
+	return random_int;
+#else
+	return random32();
+#endif
+}
+void rtw_wiphy_rfkill_set_hw_state(struct wiphy *wiphy, bool blocked)
+{
+	wiphy_rfkill_set_hw_state(wiphy, blocked);
+}
+
+s32 rtw_warn_on_cnt = 0;
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom.c b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom.c
new file mode 100644
index 000000000000..3e4ca2e98a83
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom.c
@@ -0,0 +1,594 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _PHL_CUSTOM_C_
+#include "../phl_headers.h"
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+#include "phl_custom_vr.h"
+#endif
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_FB
+#include "phl_custom_fb.h"
+#endif
+#include "phl_custom_antenna.h"
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+
+enum custom_status {
+	CUS_STAT_CMD_USED = BIT0,
+	CUS_STAT_RPT_USED = BIT1,
+	CUS_STAT_RPT_SENDING = BIT2,
+};
+
+struct phl_custom_ctx {
+	struct phl_info_t *phl_info;
+	u8 status; /* refer to enum custom_status*/
+	u8 cmd_buf[sizeof(struct rtw_custom_decrpt)];
+	u8 rpt_buf[sizeof(struct rtw_custom_decrpt)];
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+	struct _custom_vr_ctx vr_ctx;
+#endif
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_FB
+	struct _custom_fb_ctx fb_ctx;
+#endif
+};
+
+bool
+_custom_rpt_notify_check(void* priv, struct phl_msg* msg)
+{
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)priv;
+	bool ret = true;
+	if (!TEST_STATUS_FLAG(ctx->status, CUS_STAT_RPT_USED) ||
+	    TEST_STATUS_FLAG(ctx->status, CUS_STAT_RPT_SENDING)) {
+		ret = false;
+	}
+	return ret;
+}
+
+void
+_custom_rpt_notify_complete(void* priv, struct phl_msg* msg)
+{
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)priv;
+
+	CLEAR_STATUS_FLAG(ctx->status, CUS_STAT_RPT_USED);
+	CLEAR_STATUS_FLAG(ctx->status, CUS_STAT_RPT_SENDING);
+}
+
+void
+_custom_rpt_notify_start(void* priv,
+                         struct phl_msg* msg,
+                         struct phl_msg_attribute* attr)
+{
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)priv;
+	struct rtw_custom_decrpt *evt_rpt = (struct rtw_custom_decrpt *)ctx->rpt_buf;
+
+	SET_STATUS_FLAG(ctx->status, CUS_STAT_RPT_SENDING);
+
+	SET_MSG_MDL_ID_FIELD(msg->msg_id, PHL_MDL_CUSTOM);
+	SET_MSG_EVT_ID_FIELD(msg->msg_id, MSG_EVT_CUSTOM_CMD_DONE);
+
+	msg->inbuf = (u8*) evt_rpt;
+	msg->inlen = sizeof(struct rtw_custom_decrpt);
+
+	attr->completion.completion = _custom_rpt_notify_complete;
+	attr->completion.priv = ctx;
+}
+
+static void
+_indicate_custome_evt_rpt(struct phl_custom_ctx *ctx)
+{
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+
+	if (!_custom_rpt_notify_check((void*)ctx, &msg))
+		return;
+
+	_custom_rpt_notify_start((void*)ctx, &msg, &attr);
+
+	if (phl_msg_hub_send(ctx->phl_info,
+	                     &attr, &msg) != RTW_PHL_STATUS_SUCCESS) {
+		PHL_ERR("%s send msg fail\n", __func__);
+		_custom_rpt_notify_complete((void*)ctx, &msg);
+	}
+}
+enum rtw_phl_status
+_phl_custom_prepare_default_fail_rpt(struct phl_custom_ctx *ctx,
+                                     struct phl_msg* msg)
+{
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	u8 ret = MDL_RET_FAIL;
+
+	if (TEST_STATUS_FLAG(ctx->status, CUS_STAT_RPT_USED))
+		return RTW_PHL_STATUS_RESOURCE;
+
+	phl_custom_prepare_evt_rpt(ctx,
+                                   cmd->evt_id,
+                                   cmd->customer_id,
+                                   &ret,
+                                   1);
+	return RTW_PHL_STATUS_SUCCESS;
+}
+
+static enum phl_mdl_ret_code _phl_custom_hdl_fail_evt(
+    void *dispr, struct phl_custom_ctx *ctx, struct phl_msg *msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	struct rtw_custom_decrpt *cmd = NULL;
+
+	cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	switch (cmd->customer_id) {
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+	case CUS_ID_VR:
+		ret =
+		    phl_custom_hdl_vr_fail_evt(dispr, ctx, &(ctx->vr_ctx), msg);
+		break;
+#endif
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_FB
+	case CUS_ID_FB:
+		ret =
+		    phl_custom_hdl_fb_fail_evt(dispr, ctx, &(ctx->fb_ctx), msg);
+		break;
+#endif
+	default:
+		ret = MDL_RET_IGNORE;
+		break;
+	}
+	return ret;
+}
+
+static enum phl_mdl_ret_code _phl_custom_hdl_internal_evt(
+    void *dispr, struct phl_custom_ctx *ctx, struct phl_msg *msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct rtw_custom_decrpt *cmd = NULL;
+
+	cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	switch (cmd->customer_id) {
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+	case CUS_ID_VR:
+		ret = phl_custom_hdl_vr_evt(dispr, ctx, &(ctx->vr_ctx), msg);
+		break;
+#endif
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_FB
+	case CUS_ID_FB:
+		ret = phl_custom_hdl_fb_evt(dispr, ctx, &(ctx->fb_ctx), msg);
+		break;
+#endif
+	case CUS_ID_ANT_TOOL:
+		ret = phl_custom_antenna_cmd_hdlr(dispr, ctx, msg);
+		break;
+	default:
+		ret = MDL_RET_IGNORE;
+		break;
+	}
+	return ret;
+}
+static enum phl_mdl_ret_code
+_phl_custom_hdl_external_evt(void* dispr,
+                             struct phl_custom_ctx* ctx,
+                             struct phl_msg* msg)
+{
+	return MDL_RET_IGNORE;
+}
+
+void
+_phl_custom_cmd_completion(void* priv, struct phl_msg* msg)
+{
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)priv;
+	CLEAR_STATUS_FLAG(ctx->status, CUS_STAT_CMD_USED);
+}
+
+enum phl_mdl_ret_code
+_phl_custom_mdl_init(void* phl_info,
+                     void* dispr,
+                     void** priv)
+{
+	struct phl_info_t *phl = (struct phl_info_t *)phl_info;
+	struct phl_custom_ctx* ctx = NULL;
+	void *d = phl_to_drvpriv(phl);
+
+	FUNCIN();
+	if (priv == NULL)
+		return MDL_RET_FAIL;
+
+	(*priv) = NULL;
+	ctx = (struct phl_custom_ctx *)_os_mem_alloc(d, sizeof(struct phl_custom_ctx));
+	if (ctx == NULL) {
+		PHL_ERR(" %s, alloc fail\n",__FUNCTION__);
+		return MDL_RET_FAIL;
+	}
+	ctx->phl_info = phl_info;
+	(*priv) = (void*)ctx;
+	PHL_INFO(" %s, size phl_custom_ctx(%d)\n",
+			__FUNCTION__, (int)sizeof(struct phl_custom_ctx));
+	return MDL_RET_SUCCESS;
+}
+void
+_phl_custom_mdl_deinit(void* dispr, void* priv)
+{
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx*)priv;
+	void *d = phl_to_drvpriv(ctx->phl_info);
+
+	_os_mem_free(d, ctx, sizeof(struct phl_custom_ctx));
+	PHL_INFO(" %s\n", __FUNCTION__);
+}
+enum phl_mdl_ret_code
+_phl_custom_mdl_start(void* dispr, void* priv)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+
+	FUNCIN();
+	FUNCOUT();
+
+	ret = MDL_RET_SUCCESS;
+	return ret;
+}
+enum phl_mdl_ret_code
+_phl_custom_mdl_stop(void* dispr, void* priv)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+
+	FUNCIN();
+	FUNCOUT();
+
+	ret = MDL_RET_SUCCESS;
+	return ret;
+}
+enum phl_mdl_ret_code
+_phl_custom_mdl_msg_hdlr(void* dispr,
+                         void* priv,
+                         struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx*)priv;
+
+	FUNCIN();
+	if (IS_MSG_FAIL(msg->msg_id)) {
+		ret = _phl_custom_hdl_fail_evt(dispr, priv, msg);
+		_indicate_custome_evt_rpt(ctx);
+		return MDL_RET_IGNORE;
+	}
+
+	switch (MSG_MDL_ID_FIELD(msg->msg_id)) {
+		case PHL_MDL_CUSTOM:
+			ret = _phl_custom_hdl_internal_evt(dispr, priv, msg);
+			break;
+		default:
+			ret = _phl_custom_hdl_external_evt(dispr, priv, msg);
+			break;
+	}
+	_indicate_custome_evt_rpt(ctx);
+	FUNCOUT();
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_feature_set_hdlr(void* dispr,
+                         struct phl_custom_ctx *ctx,
+                         struct phl_module_op_info* info)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	struct rtw_custom_decrpt *cmd = NULL;
+
+	cmd = (struct rtw_custom_decrpt *)info->inbuf;
+
+	switch (cmd->customer_id) {
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+		case CUS_ID_VR:
+			ret = phl_custom_vr_feature_set_hdlr(dispr, ctx,
+			                                     &(ctx->vr_ctx), cmd);
+			break;
+#endif
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_FB
+		case CUS_ID_FB:
+			ret = phl_custom_fb_feature_set_hdlr(dispr, ctx,
+			                                     &(ctx->fb_ctx), cmd);
+			break;
+#endif
+		default:
+			ret = MDL_RET_IGNORE;
+			break;
+	}
+
+	return ret;
+}
+
+static enum phl_mdl_ret_code
+_phl_custom_mdl_set_info(void* dispr,
+                         void* priv,
+                         struct phl_module_op_info* info)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)priv;
+	void *d = phl_to_drvpriv(ctx->phl_info);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_custom_decrpt *cmd = NULL;
+	u8 idx = 0xff;
+
+	phl_dispr_get_idx(dispr, &idx);
+	FUNCIN();
+	switch (info->op_code) {
+		case BK_MODL_OP_INPUT_CMD:
+			if (TEST_STATUS_FLAG(ctx->status, CUS_STAT_CMD_USED) ||
+				info->inlen > sizeof(struct rtw_custom_decrpt)) {
+				PHL_ERR("%s buf len err or used\n", __func__);
+				break;
+			}
+			SET_STATUS_FLAG(ctx->status, CUS_STAT_CMD_USED);
+			_os_mem_cpy(d, ctx->cmd_buf, info->inbuf, info->inlen);
+			cmd = (struct rtw_custom_decrpt *)ctx->cmd_buf;
+			SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_CUSTOM);
+			SET_MSG_EVT_ID_FIELD(msg.msg_id, (u16)cmd->evt_id);
+			msg.inbuf = ctx->cmd_buf;
+			msg.inlen = info->inlen;
+			attr.completion.priv = priv;
+			attr.completion.completion = _phl_custom_cmd_completion;
+			msg.band_idx = idx;
+			if (phl_disp_eng_send_msg(ctx->phl_info,
+			                          &msg,
+			                          &attr,
+			                          NULL) == RTW_PHL_STATUS_SUCCESS) {
+				ret = MDL_RET_SUCCESS;
+			} else {
+				CLEAR_STATUS_FLAG(ctx->status, CUS_STAT_CMD_USED);
+				PHL_ERR("%s send msg fail\n", __func__);
+				ret = MDL_RET_FAIL;
+			}
+			break;
+		case BK_MODL_OP_CUS_FEATURE_SET:
+			ret = _phl_custom_feature_set_hdlr(dispr, ctx, info);
+			break;
+		default:
+			break;
+	}
+	FUNCOUT();
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_feature_query_hdlr(void* dispr,
+                         struct phl_custom_ctx *ctx,
+                         struct phl_module_op_info* info)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	struct rtw_custom_decrpt *cmd = NULL;
+
+	cmd = (struct rtw_custom_decrpt *)info->outbuf;
+
+	switch (cmd->customer_id) {
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+		case CUS_ID_VR:
+			ret = phl_custom_vr_feature_query_hdlr(dispr, ctx,
+			                                     &(ctx->vr_ctx), cmd);
+			break;
+#endif
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_FB
+		case CUS_ID_FB:
+			ret = phl_custom_fb_feature_query_hdlr(dispr, ctx,
+			                                     &(ctx->fb_ctx), cmd);
+			break;
+#endif
+		default:
+			ret = MDL_RET_IGNORE;
+			break;
+	}
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_mdl_query_info(void* dispr,
+                           void* priv,
+                           struct phl_module_op_info* info)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_IGNORE;
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)priv;
+	void *d = phl_to_drvpriv(ctx->phl_info);
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	struct rtw_custom_decrpt *cmd = NULL;
+	u8 idx = 0xff;
+
+	FUNCIN();
+	switch(info->op_code) {
+		case BK_MODL_OP_INPUT_CMD:
+			if (TEST_STATUS_FLAG(ctx->status, CUS_STAT_CMD_USED) ||
+				info->inlen > sizeof(struct rtw_custom_decrpt)) {
+				PHL_ERR("%s buf len err or used\n", __func__);
+				break;
+			}
+			SET_STATUS_FLAG(ctx->status, CUS_STAT_CMD_USED);
+			_os_mem_cpy(d, ctx->cmd_buf, info->inbuf, info->inlen);
+			_os_mem_cpy(d, ctx->rpt_buf, info->outbuf, info->outlen);
+			cmd = (struct rtw_custom_decrpt *)ctx->cmd_buf;
+			SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_CUSTOM);
+			SET_MSG_EVT_ID_FIELD(msg.msg_id, (u16)cmd->evt_id);
+			msg.inbuf = ctx->cmd_buf;
+			msg.inlen = info->inlen;
+			msg.outbuf = ctx->rpt_buf;
+			msg.outlen = info->outlen;
+			attr.completion.priv = priv;
+			attr.completion.completion = _phl_custom_cmd_completion;
+			msg.band_idx = idx;
+			if (phl_disp_eng_send_msg(ctx->phl_info,
+			                          &msg,
+			                          &attr,
+			                          NULL) == RTW_PHL_STATUS_SUCCESS) {
+				ret = MDL_RET_SUCCESS;
+			} else {
+				CLEAR_STATUS_FLAG(ctx->status, CUS_STAT_CMD_USED);
+				PHL_ERR("%s send msg fail\n", __func__);
+				ret = MDL_RET_FAIL;
+			}
+			break;
+		case BK_MODL_OP_CUS_FEATURE_QUERY:
+			ret = _phl_custom_feature_query_hdlr(dispr, ctx, info);
+			break;
+	default:
+		break;
+	}
+
+	FUNCOUT();
+	return ret;
+}
+
+enum rtw_phl_status
+phl_register_custom_module(struct phl_info_t *phl_info, u8 band_idx)
+{
+	enum rtw_phl_status phl_status = RTW_PHL_STATUS_FAILURE;
+	struct phl_bk_module_ops bk_ops = {0};
+
+	bk_ops.init = _phl_custom_mdl_init;
+	bk_ops.deinit = _phl_custom_mdl_deinit;
+	bk_ops.start = _phl_custom_mdl_start;
+	bk_ops.stop = _phl_custom_mdl_stop;
+	bk_ops.msg_hdlr = _phl_custom_mdl_msg_hdlr;
+	bk_ops.query_info = _phl_custom_mdl_query_info;
+	bk_ops.set_info = _phl_custom_mdl_set_info;
+
+	phl_status = phl_disp_eng_register_module(phl_info,
+	                                          band_idx,
+	                                          PHL_MDL_CUSTOM,
+	                                          &bk_ops);
+	if (RTW_PHL_STATUS_SUCCESS != phl_status)
+		PHL_ERR("%s register Custom module in cmd disp failed\n", __func__);
+
+	return phl_status;
+
+}
+
+enum rtw_phl_status
+phl_custom_prepare_evt_rpt(void *custom_ctx,
+                           u32 evt_id,
+                           u32 customer_id,
+                           u8 *rpt,
+                           u32 rpt_len)
+{
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)custom_ctx;
+	struct rtw_custom_decrpt *evt_rpt = (struct rtw_custom_decrpt *)ctx->rpt_buf;
+	void *d = phl_to_drvpriv(ctx->phl_info);
+	u8* val = evt_rpt->data;
+
+	if ((rpt_len > MAX_DATA_SIZE) ||
+	   TEST_STATUS_FLAG(ctx->status, CUS_STAT_RPT_USED))
+		return RTW_PHL_STATUS_RESOURCE;
+
+	SET_STATUS_FLAG(ctx->status, CUS_STAT_RPT_USED);
+	evt_rpt->evt_id = evt_id;
+	evt_rpt->customer_id = customer_id;
+	evt_rpt->len = rpt_len;
+	_os_mem_cpy(d, val, rpt, rpt_len);
+	return RTW_PHL_STATUS_SUCCESS;
+}
+
+enum rtw_phl_status
+rtw_phl_custom_prepare_set_info_evt(u32 evt_id,
+                                    u32 customer_id,
+                                    struct rtw_custom_decrpt *inbuf,
+                                    struct phl_module_op_info *op_info,
+                                    u32 data_len)
+{
+	if ((data_len > MAX_DATA_SIZE)) {
+		PHL_ERR("%s data_len exceeds MAX_DATA_SIZE\n", __func__);
+		return RTW_PHL_STATUS_RESOURCE;
+	}
+
+	inbuf->evt_id = evt_id;
+	inbuf->customer_id = customer_id;
+	inbuf->len = data_len;
+
+	op_info->op_code = BK_MODL_OP_CUS_FEATURE_SET;
+	op_info->inbuf = (u8*)inbuf;
+	op_info->inlen = sizeof(struct rtw_custom_decrpt);
+
+	return RTW_PHL_STATUS_SUCCESS;
+}
+
+enum rtw_phl_status
+rtw_phl_custom_prepare_query_info_evt(u32 evt_id,
+                                      u32 customer_id,
+                                      struct rtw_custom_decrpt *inbuf,
+                                      struct phl_module_op_info *op_info,
+                                      u32 data_len)
+{
+	if ((data_len > MAX_DATA_SIZE)) {
+		PHL_ERR("%s data_len exceeds MAX_DATA_SIZE\n", __func__);
+		return RTW_PHL_STATUS_RESOURCE;
+	}
+	inbuf->evt_id = evt_id;
+	inbuf->customer_id = customer_id;
+	inbuf->len = data_len;
+
+	op_info->op_code = BK_MODL_OP_CUS_FEATURE_QUERY;
+	op_info->outbuf = (u8*)inbuf;
+	op_info->outlen = sizeof(struct rtw_custom_decrpt);
+
+	return RTW_PHL_STATUS_SUCCESS;
+}
+
+struct phl_info_t*
+phl_custom_get_phl_info(void *custom_ctx)
+{
+	struct phl_custom_ctx *ctx = (struct phl_custom_ctx *)custom_ctx;
+
+	return ctx->phl_info;
+}
+
+/**
+ * phl_custom_init_role_link_cap
+ * 1. role cap customization
+ * input:
+ * @phl_info: (struct phl_info_t *)
+ * @hw_band:hw_band
+ * @role_link_cap: (struct role_link_cap_t)
+ * return: rtw_phl_status
+ */
+enum rtw_phl_status
+phl_custom_init_role_link_cap(struct phl_info_t *phl_info,
+                         u8 hw_band,
+                         struct role_link_cap_t *role_link_cap)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_FB
+	status = phl_custom_fb_init_role_link_cap(phl_info, hw_band, role_link_cap);
+#endif
+
+	return status;
+}
+
+/**
+ * phl_custom_init_protocol_cap
+ * 1. protocol cap customization
+ * input:
+ * @phl_info: (struct phl_info_t *)
+ * @hw_band:hw_band
+ * @rtype: role type
+ * @protocol_cap: (struct protocol_cap_t)
+ * return: rtw_phl_status
+ */
+enum rtw_phl_status phl_custom_init_protocol_cap(
+    struct phl_info_t *phl_info, u8 hw_band, enum role_type rtype,
+    struct protocol_cap_t *protocol_cap)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+	status = phl_custom_vr_init_protocol_cap(phl_info, hw_band, rtype,
+						 protocol_cap);
+#endif
+
+	return status;
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom.h b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom.h
new file mode 100644
index 000000000000..f66ebf0d5a2f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom.h
@@ -0,0 +1,63 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _PHL_CUSTOM_H_
+#define _PHL_CUSTOM_H_
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+enum rtw_phl_status
+phl_register_custom_module(struct phl_info_t *phl_info, u8 band_idx);
+
+enum rtw_phl_status
+phl_custom_prepare_evt_rpt(void *custom_ctx,
+                           u32 evt_id,
+                           u32 customer_id,
+                           u8 *rpt,
+                           u32 rpt_len);
+
+enum rtw_phl_status
+rtw_phl_custom_prepare_set_info_evt(u32 evt_id,
+                                    u32 customer_id,
+                                    struct rtw_custom_decrpt *inbuf,
+                                    struct phl_module_op_info *op_info,
+                                    u32 data_len);
+
+enum rtw_phl_status
+rtw_phl_custom_prepare_query_info_evt(u32 evt_id,
+                                      u32 customer_id,
+                                      struct rtw_custom_decrpt *inbuf,
+                                      struct phl_module_op_info *op_info,
+                                      u32 data_len);
+struct phl_info_t *
+phl_custom_get_phl_info(void *custom_ctx);
+
+enum rtw_phl_status
+phl_custom_init_role_link_cap(struct phl_info_t *phl_info,
+                         u8 hw_band,
+                         struct role_link_cap_t *role_link_cap);
+
+enum rtw_phl_status phl_custom_init_protocol_cap(
+    struct phl_info_t *phl_info, u8 hw_band, enum role_type rtype,
+    struct protocol_cap_t *protocol_cap);
+
+#else
+#define phl_register_custom_module(_phl_info, _band_idx) (RTW_PHL_STATUS_SUCCESS)
+#define phl_custom_init_role_link_cap(_phl_info, _hw_band, _role_link_cap) (RTW_PHL_STATUS_SUCCESS)
+#define phl_custom_init_protocol_cap(_phl_info, _hw_band, _rtype,              \
+				     _protocol_cap)                            \
+	(RTW_PHL_STATUS_SUCCESS)
+#endif
+
+#endif  /*_PHL_CUSTOMIZE_FEATURE_H_*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_antenna.c b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_antenna.c
new file mode 100644
index 000000000000..13a5b23ec409
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_antenna.c
@@ -0,0 +1,288 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _PHL_CUSTOMIZE_FEATURE_ANTENNA_C_
+#include "../phl_headers.h"
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_ANTENNA
+#include "phl_custom_antenna.h"
+
+
+void _phl_custom_ant_cmd_complete(void *priv, struct phl_msg *msg)
+{
+	struct rtw_custom_decrpt *c_cmd = NULL;
+
+	if ((msg->inbuf != NULL) && (msg->inlen !=0)) {
+		c_cmd = (struct rtw_custom_decrpt *)msg->inbuf;
+		PHL_INFO("[PHL][CUSTOM][ANT]---> %s, op code %d, free size %d \n", __func__, c_cmd->evt_id, msg->inlen);
+		_os_mem_free(priv, msg->inbuf, msg->inlen);
+	}
+}
+
+
+
+enum rtw_phl_status
+phl_custom_ant_cmd_enqueue(struct phl_info_t *phl_info,
+			   enum phl_band_idx band_idx,
+			   enum phl_msg_evt_id evt_id,
+			   u8 *cmd_buf,
+			   u32 cmd_len)
+{
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_CUSTOM);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, evt_id);
+
+	msg.inbuf = (u8 *)cmd_buf;
+	msg.inlen = cmd_len;
+	msg.band_idx = band_idx;
+	attr.completion.completion = _phl_custom_ant_cmd_complete;
+	attr.completion.priv = phl_info;
+
+
+	psts = phl_disp_eng_send_msg(phl_info, &msg, &attr, NULL);
+
+	if (psts != RTW_PHL_STATUS_SUCCESS) {
+		PHL_ERR("%s send msg failed\n", __func__);
+	}
+
+	return psts;
+}
+
+
+
+void rtw_phl_custom_antenna_frn_rst(struct phl_info_t *phl_info, enum phl_band_idx band_idx)
+{
+	void *drv = phl_to_drvpriv(phl_info);
+	u8 *cmd_buf = NULL;
+	u32 cmd_sz = sizeof(struct rtw_custom_decrpt);
+	struct rtw_custom_decrpt *c_cmd = NULL;
+
+	PHL_INFO("[PHL][CUSTOM][ANT] ---> %s\n", __func__);
+
+	cmd_buf = _os_mem_alloc(drv, cmd_sz);
+	PHL_INFO("[PHL][CUSTOM][ANT] allocate mem size %d \n", cmd_sz);
+
+	if (cmd_buf != NULL) {
+		c_cmd = ( struct rtw_custom_decrpt *)cmd_buf;
+		c_cmd->customer_id = CUS_ID_ANT_TOOL;
+		c_cmd->evt_id = (u8)RTW_PHL_CUSTOM_ANT_OP_FRN_CNT_RST;
+		c_cmd->type = CUSTOM_CORE;
+		c_cmd->len = 0;
+
+		PHL_INFO("[PHL][CUSTOM][ANT] send cmd MSG_EVT_ANT_TOOL_OP_HDLR\n");
+		if (RTW_PHL_STATUS_SUCCESS != phl_custom_ant_cmd_enqueue(
+					phl_info,
+					band_idx,
+					MSG_EVT_ANT_TOOL_OP_HDLR,
+					cmd_buf,
+					cmd_sz)) {
+			PHL_INFO("[PHL][CUSTOM][ANT] send cmd fail, free cmd buf memory \n");
+			_os_mem_free(drv, cmd_buf, cmd_sz);
+		}
+	}
+	PHL_INFO("[PHL][CUSTOM][ANT] <--- %s\n", __func__);
+}
+
+void rtw_phl_custom_antenna_fix_div_ant(struct phl_info_t *phl_info, enum phl_band_idx band_idx, u8 *ant_idx, u8 para_len)
+{
+	void *drv = phl_to_drvpriv(phl_info);
+	u8 *cmd_buf = NULL;
+	u32 cmd_sz = (para_len + sizeof(struct rtw_custom_decrpt));
+	struct rtw_custom_decrpt *c_cmd = NULL;
+
+	PHL_INFO("[PHL][CUSTOM][ANT] ---> %s\n", __func__);
+
+	cmd_buf = _os_mem_alloc(drv, cmd_sz);
+	PHL_INFO("[PHL][CUSTOM][ANT] allocate mem size %d \n", cmd_sz);
+
+	if (cmd_buf != NULL) {
+		c_cmd = ( struct rtw_custom_decrpt *)cmd_buf;
+		c_cmd->customer_id = CUS_ID_ANT_TOOL;
+		c_cmd->evt_id = (u8)RTW_PHL_CUSTOM_ANT_OP_FIX_DIV_ANT;
+		c_cmd->type = CUSTOM_CORE;
+		c_cmd->len = para_len;
+
+		if ((NULL != ant_idx) && (para_len > 0))
+			_os_mem_cpy(drv, (void *)(c_cmd + 1), ant_idx, para_len);
+
+		PHL_INFO("[PHL][CUSTOM][ANT] send cmd MSG_EVT_ANT_TOOL_OP_HDLR\n");
+		if (RTW_PHL_STATUS_SUCCESS != phl_custom_ant_cmd_enqueue(
+					phl_info,
+					band_idx,
+					MSG_EVT_ANT_TOOL_OP_HDLR,
+					cmd_buf,
+					cmd_sz)) {
+			PHL_INFO("[PHL][CUSTOM][ANT] send cmd fail, free cmd buf memory \n");
+			_os_mem_free(drv, cmd_buf, cmd_sz);
+		}
+	}
+	PHL_INFO("[PHL][CUSTOM][ANT] <--- %s\n", __func__);
+}
+
+void rtw_phl_custom_antenna_example_cmd(struct phl_info_t *phl_info, enum phl_band_idx band_idx, u8 *para, u8 para_len)
+{
+	void *drv = phl_to_drvpriv(phl_info);
+	u8 *cmd_buf = NULL;
+	u32 cmd_sz = (para_len + sizeof(struct rtw_custom_decrpt));
+	struct rtw_custom_decrpt *c_cmd = NULL;
+
+	PHL_INFO("[PHL][CUSTOM][ANT] ---> %s\n", __func__);
+
+	cmd_buf = _os_mem_alloc(drv, cmd_sz);
+	PHL_INFO("[PHL][CUSTOM][ANT] allocate mem size %d \n", cmd_sz);
+
+	if (cmd_buf != NULL) {
+
+		c_cmd = ( struct rtw_custom_decrpt *)cmd_buf;
+		c_cmd->customer_id = CUS_ID_ANT_TOOL;
+		c_cmd->evt_id = (u8)RTW_PHL_CUSTOM_ANT_OP_EXAMPLE;
+		c_cmd->type = CUSTOM_CORE;
+		c_cmd->len = para_len;
+		/* memory copy para to cmd buf, after rtw_custom_decrpt */
+		if ((NULL != para) && (para_len > 0))
+			_os_mem_cpy(drv, (void *)(c_cmd + 1), para, para_len);
+
+		PHL_INFO("[PHL][CUSTOM][ANT] send cmd RTW_PHL_CUSTOM_ANT_OP_MIN\n");
+		if (RTW_PHL_STATUS_SUCCESS != phl_custom_ant_cmd_enqueue(
+					phl_info,
+					band_idx,
+					MSG_EVT_ANT_TOOL_OP_HDLR,
+					cmd_buf,
+					cmd_sz)) {
+			PHL_INFO("[PHL][CUSTOM][ANT] send cmd fail, free cmd buf memory \n");
+			_os_mem_free(drv, cmd_buf, cmd_sz);
+		}
+	}
+	PHL_INFO("[PHL][CUSTOM][ANT] <--- %s\n", __func__);
+}
+
+
+enum phl_mdl_ret_code
+phl_custom_antenna_cmd_evt_post_hdlr(void* dispr,
+				     void* custom_ctx,
+				     struct phl_msg* msg)
+{
+	struct rtw_custom_decrpt *cmd = NULL;
+	enum phl_mdl_ret_code ret = MDL_RET_SUCCESS;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	enum rtw_hal_status hstate = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[PHL][CUSTOM][ANT] ---> %s\n", __func__);
+
+	if (msg->inlen < sizeof(struct rtw_custom_decrpt))
+		return MDL_RET_FAIL;
+
+
+	cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	PHL_INFO("[PHL][CUSTOM][ANT] op_code %d\n", cmd->evt_id);
+
+	switch (cmd->evt_id) {
+	case RTW_PHL_CUSTOM_ANT_OP_EXAMPLE:
+	{
+		u8 need_len = sizeof(u32);
+		u32 exp_para = 0;
+		PHL_INFO("[PHL][CUSTOM][ANT] example : cmd->len = %d\n", cmd->len);
+		if ((cmd->len >= need_len) &&
+			(msg->inlen >= (cmd->len + sizeof(struct rtw_custom_decrpt)))) {
+			exp_para = *((u32 *)(cmd + 1));
+			PHL_INFO("[PHL][CUSTOM][ANT] example : exp_para = %d\n", exp_para);
+		}
+	}
+	break;
+	case RTW_PHL_CUSTOM_ANT_OP_FRN_CNT_RST:
+	{
+		hstate = rtw_hal_reset_freerun_counter(phl->hal,  msg->band_idx);
+		if (hstate != RTW_HAL_STATUS_SUCCESS)
+			ret = MDL_RET_FAIL;
+	}
+	break;
+	case RTW_PHL_CUSTOM_ANT_OP_FIX_DIV_ANT:
+	{
+		u8 need_len = sizeof(u8);
+		u8 ant_idx = 0;
+		PHL_INFO("[PHL][CUSTOM][ANT] fix_div_ant : cmd->len = %d\n", cmd->len);
+		if ((cmd->len >= need_len) &&
+			(msg->inlen >= (cmd->len + sizeof(struct rtw_custom_decrpt)))) {
+			ant_idx = *((u8 *)(cmd + 1));
+			PHL_INFO("[PHL][CUSTOM][ANT] FIX_DIV_ANT : ant_idx = %d\n", ant_idx);
+			hstate = rtw_hal_antdiv_fix_ant(phl->hal, ant_idx);
+			if (hstate != RTW_HAL_STATUS_SUCCESS)
+				ret = MDL_RET_FAIL;
+		}
+	}
+	break;
+	default:
+		ret = MDL_RET_SUCCESS;
+	break;
+	}
+
+
+	PHL_INFO("[PHL][CUSTOM][ANT] <--- %s\n", __func__);
+	return ret;
+}
+
+enum phl_mdl_ret_code
+phl_custom_antenna_cmd_evt_pre_hdlr(void* dispr,
+				    void* custom_ctx,
+				    struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_SUCCESS;
+	struct rtw_custom_decrpt *cmd = NULL;
+
+	if (msg->inlen < sizeof(struct rtw_custom_decrpt))
+		return MDL_RET_FAIL;
+
+	cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+
+	switch (cmd->evt_id) {
+	case RTW_PHL_CUSTOM_ANT_OP_EXAMPLE:
+	case RTW_PHL_CUSTOM_ANT_OP_FRN_CNT_RST:
+	case RTW_PHL_CUSTOM_ANT_OP_FIX_DIV_ANT:
+		ret = MDL_RET_SUCCESS;
+	break;
+	default:
+		ret = MDL_RET_SUCCESS;
+	break;
+	}
+
+	return ret;
+}
+
+
+enum phl_mdl_ret_code
+phl_custom_antenna_cmd_hdlr(void* dispr,
+			    void* custom_ctx,
+			    struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_SUCCESS;
+
+	switch (MSG_EVT_ID_FIELD(msg->msg_id)) {
+	case MSG_EVT_ANT_TOOL_OP_HDLR:
+		if (IS_MSG_IN_PRE_PHASE(msg->msg_id))
+			ret = phl_custom_antenna_cmd_evt_pre_hdlr(dispr, custom_ctx, msg);
+		else
+			ret = phl_custom_antenna_cmd_evt_post_hdlr(dispr, custom_ctx, msg);
+	break;
+	default:
+		ret = MDL_RET_SUCCESS;
+	break;
+	}
+
+	return ret;
+}
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_antenna.h b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_antenna.h
new file mode 100644
index 000000000000..2bc6a916a0c1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_antenna.h
@@ -0,0 +1,44 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _PHL_CUSTOM_ANTENNA_H_
+#define _PHL_CUSTOM_ANTENNA_H_
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_ANTENNA
+
+enum rtw_phl_custom_ant_op {
+	RTW_PHL_CUSTOM_ANT_OP_EXAMPLE = 0,
+	RTW_PHL_CUSTOM_ANT_OP_FRN_CNT_RST = 1,
+	RTW_PHL_CUSTOM_ANT_OP_FIX_DIV_ANT = 2,
+	RTW_PHL_CUSTOM_ANT_OP_MAX,
+};
+
+void rtw_phl_custom_antenna_frn_rst(struct phl_info_t *phl_info, enum phl_band_idx band_idx);
+void rtw_phl_custom_antenna_example_cmd(struct phl_info_t *phl_info, enum phl_band_idx band_idx, u8 *para, u8 para_len);
+void rtw_phl_custom_antenna_fix_div_ant(struct phl_info_t *phl_info, enum phl_band_idx band_idx, u8 *ant_idx, u8 para_len);
+
+enum phl_mdl_ret_code
+phl_custom_antenna_cmd_hdlr(void* dispr,
+			    void* custom_ctx,
+			    struct phl_msg* msg);
+#else
+
+#define rtw_phl_custom_antenna_frn_rst(_phl_info, _band_idx)
+#define rtw_phl_custom_antenna_example_cmd(_phl_info, _band_idx, _para, _para_len)
+#define rtw_phl_custom_antenna_fix_div_ant(_phl_info, _band_idx, _ant_idx, _para_len)
+#define phl_custom_antenna_cmd_hdlr(_dispr, _custom_ctx, _msg) MDL_RET_SUCCESS
+
+
+#endif
+#endif
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_api.h b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_api.h
new file mode 100644
index 000000000000..706634a22eba
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_api.h
@@ -0,0 +1,34 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _PHL_CUSTOM_API_H_
+#define _PHL_CUSTOM_API_H_
+
+#include "phl_custom_antenna.h"
+
+enum rtw_phl_status
+rtw_phl_custom_prepare_set_info_evt(u32 evt_id,
+                                    u32 customer_id,
+                                    struct rtw_custom_decrpt *inbuf,
+                                    struct phl_module_op_info *op_info,
+                                    u32 data_len);
+
+enum rtw_phl_status
+rtw_phl_custom_prepare_query_info_evt(u32 evt_id,
+                                      u32 customer_id,
+                                      struct rtw_custom_decrpt *inbuf,
+                                      struct phl_module_op_info *op_info,
+                                      u32 data_len);
+
+#endif /*_PHL_CUSTOM_API_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_def.h b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_def.h
new file mode 100644
index 000000000000..c168544d0e95
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_def.h
@@ -0,0 +1,230 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _PHL_CUSTOM_DEF_H_
+#define _PHL_CUSTOM_DEF_H_
+
+#define PRIVATE_EVT_START (BIT15)
+#define MAX_DATA_SIZE	(1024)
+
+enum rtw_customer_feature_id {
+	CUS_ID_NONE = 0,
+	CUS_ID_FB = 1,
+	CUS_ID_XH = 2,
+	CUS_ID_VR = 3,
+
+	CUS_ID_ANT_TOOL = 4,
+	CUS_ID_HOSTAPD = 5,
+	CUS_ID_MAX
+};
+
+enum rtw_msg_custom_evt_id {
+	/* Reserved for Custom Feature
+	* be aware that the order of the following id are also used in
+	* upper layer application, thus, it shall not be changed to
+	* avoid mismatch error!!
+	*/
+	MSG_EVT_CUSTOME_NONE = PRIVATE_EVT_START,
+	MSG_EVT_CUSTOME_FEATURE_ENABLE = PRIVATE_EVT_START + 1,
+	MSG_EVT_CUSTOME_FEATURE_QUERY = PRIVATE_EVT_START + 2,
+	MSG_EVT_CUSTOME_TESTMODE_PARAM = PRIVATE_EVT_START + 3,
+	MSG_EVT_CUSTOME_SET_WIFI_ROLE = PRIVATE_EVT_START + 4,
+	MSG_EVT_EDCA_ADJUST = PRIVATE_EVT_START + 5,
+	MSG_EVT_EDCA_QUERY = PRIVATE_EVT_START + 6,
+	MSG_EVT_NAV_PADDING = PRIVATE_EVT_START + 7,
+	MSG_EVT_NAV_PADDING_QUERY = PRIVATE_EVT_START + 8,
+	MSG_EVT_CUSTOM_CMD_DONE = PRIVATE_EVT_START + 9,
+	MSG_EVT_AMPDU_CFG = PRIVATE_EVT_START + 10,
+	MSG_EVT_AMPDU_QUERY = PRIVATE_EVT_START + 11,
+	MSG_EVT_COLDBOOT_CALI = PRIVATE_EVT_START + 12,
+	MSG_EVT_PDTHR_CFG = PRIVATE_EVT_START + 13,
+	MSG_EVT_PDTHR_QUERY = PRIVATE_EVT_START + 14,
+	MSG_EVT_POP_CFG = PRIVATE_EVT_START + 15,
+	MSG_EVT_POP_QUERY = PRIVATE_EVT_START + 16,
+	MSG_EVT_STATS_RPT_CFG = PRIVATE_EVT_START + 17,
+	MSG_EVT_STATS_RPT_NOTIFY = PRIVATE_EVT_START + 18,
+	MSG_EVT_CHNL_SW = PRIVATE_EVT_START + 19,
+	MSG_EVT_STA_ASOC_NOTIFY = PRIVATE_EVT_START + 20,
+	MSG_EVT_BCN_OPT_IES_CFG = PRIVATE_EVT_START + 21,
+	MSG_EVT_BCN_VDR_IE_CFG = PRIVATE_EVT_START + 22,
+	MSG_EVT_SET_AP_START_CHNL = PRIVATE_EVT_START + 23,
+	MSG_EVT_SET_CUS_CHNL_LIST = PRIVATE_EVT_START + 24,
+	MSG_EVT_GET_CUS_CHNL_LIST = PRIVATE_EVT_START + 25,
+	MSG_EVT_ACS_GET_PREF_CHNL = PRIVATE_EVT_START + 26,
+	MSG_EVT_AMSDU_CFG = PRIVATE_EVT_START + 27,
+	MSG_EVT_AMSDU_QUERY = PRIVATE_EVT_START + 28,
+	MSG_EVT_SET_USB_SWITCH_MODE = PRIVATE_EVT_START + 29,
+	MSG_EVT_GET_USB_SWITCH_MODE = PRIVATE_EVT_START + 30,
+	MSG_EVT_SET_ROLE_CAP = PRIVATE_EVT_START + 31,
+	MSG_EVT_GET_ROLE_CAP = PRIVATE_EVT_START + 32,
+	MSG_EVT_SET_TX_RATE_MASKING = PRIVATE_EVT_START + 33,
+	MSG_EVT_GET_TX_RATE_MASKING = PRIVATE_EVT_START + 34,
+	MSG_EVT_SET_TX_RATE_RTY_TBL = PRIVATE_EVT_START + 35,
+	MSG_EVT_GET_TX_RATE_RTY_TBL = PRIVATE_EVT_START + 36,
+	MSG_EVT_TSF_CFG = PRIVATE_EVT_START + 37,
+	MSG_EVT_TSF_QUERY = PRIVATE_EVT_START + 38,
+	MSG_EVT_TSF_NOTIFY = PRIVATE_EVT_START + 39,
+	MSG_EVT_SET_MAX_SCAN_TIME = PRIVATE_EVT_START + 40,
+	MSG_EVT_SET_SCAN_ENABLE = PRIVATE_EVT_START + 41,
+	MSG_EVT_SET_SCAN_COMPLETE = PRIVATE_EVT_START + 42,
+	MSG_EVT_GET_MAX_SCAN_TIME = PRIVATE_EVT_START + 43,
+	MSG_EVT_HOSTQ_FLUSH_CFG = PRIVATE_EVT_START + 44,
+	MSG_EVT_HOSTQ_FLUSHED_NOTIFY = PRIVATE_EVT_START + 45,
+	MSG_EVT_CSA_RESET_ACS_PREF_CH = PRIVATE_EVT_START + 46,
+	MSG_EVT_ACS_SET_CH_TABLE_MASK = PRIVATE_EVT_START + 47,
+	MSG_EVT_LPS_DIG_CFG = PRIVATE_EVT_START + 48,
+	MSG_EVT_LPS_DIG_QUERY = PRIVATE_EVT_START + 49,
+	MSG_EVT_LPS_DIG_NOTIFY = PRIVATE_EVT_START + 50,
+	MSG_EVT_CSA_BCN_FILL_TIM = PRIVATE_EVT_START + 51,
+	MSG_EVT_GET_DRIVER_VERSION = PRIVATE_EVT_START + 52,
+	MSG_EVT_GET_PERF_METRICS = PRIVATE_EVT_START + 53,
+	MSG_EVT_SET_AP_HID_SSID = PRIVATE_EVT_START + 54,
+	MSG_EVT_GET_AP_HID_SSID = PRIVATE_EVT_START + 55,
+	MSG_EVT_CFG_START_AP_CAP = PRIVATE_EVT_START + 56,
+	MSG_EVT_RUN_PRE_RFK = PRIVATE_EVT_START + 57,
+	MSG_EVT_QUERY_PRE_RFK_STATUS = PRIVATE_EVT_START + 58,
+	MSG_EVT_CFG_REGULATORY_DOMAIN = PRIVATE_EVT_START + 59,
+	MSG_EVT_CFG_LED_LIGHTING = PRIVATE_EVT_START + 60,
+	MSD_EVT_SET_ECSA_START = PRIVATE_EVT_START + 61,
+	MSG_EVT_ANT_TOOL_OP_HDLR = PRIVATE_EVT_START + 62,
+	MSG_EVT_SET_SPATIAL_REUSE = PRIVATE_EVT_START + 63,
+	MSG_EVT_GET_SPATIAL_REUSE = PRIVATE_EVT_START + 64,
+	MSG_EVT_RF_SCRAMBLE = PRIVATE_EVT_START + 65,
+	MSG_EVT_SET_ANT_SWITCH = PRIVATE_EVT_START + 66,
+	MSG_EVT_SET_CSI = PRIVATE_EVT_START + 67,
+	MSG_EVT_GET_CSI = PRIVATE_EVT_START + 68,
+	MSG_EVT_SET_BSS_COLOR = PRIVATE_EVT_START + 69,
+	MSG_EVT_GET_BSS_COLOR = PRIVATE_EVT_START + 70,
+	MSG_EVT_PROC_USR_TX_RPT = PRIVATE_EVT_START + 71,
+};
+
+enum custom_type {
+	CUSTOM_CORE = 0,
+	CUSTOM_PHL = 1,
+	CUSTOM_MAX
+};
+
+enum rtw_custom_flags {
+	CUSTOM_FLAG_NONE = 0,
+	CUSTOM_FLAG_SET = BIT0,
+	CUSTOM_FLAG_GET = BIT1,
+	CUSTOM_FLAG_UNSOLICITED = BIT2,
+	CUSTOM_FLAG_FWD = BIT3
+};
+
+/*
+ * #pragma pack(1) would set the structure as 1-byte alignment .
+ */
+#pragma pack(1)
+struct rtw_custom_decrpt {
+	u32 evt_id;
+	u32 customer_id;
+	enum custom_type type;
+	u32 flags;
+	u32 len;
+	u8 data[MAX_DATA_SIZE];
+};
+
+struct rtw_custom_rpt {
+	u32 status;
+	union {
+		u32 value;
+		u8 addr[MAC_ALEN];
+	} u;
+};
+#pragma pack()
+
+enum rtw_phl_vr_test_mode
+{
+	VR_TEST_MODE_FAKE_TX_CYCLE = BIT0,
+	VR_TEST_MODE_USE_STA_MAC = BIT1,
+	VR_TEST_MODE_OP_CH_BY_PASS = BIT2,
+	VR_TEST_MODE_MAX = BIT16
+};
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+/*
+ * enum _usr_tx_stats_rpt_mode specify different mode for statistics rpt.
+ * @VR_STATS_RPT_DIS: disable
+ * @VR_STATS_RPT_PERIOD: Periodical statistics (use rpt_period_us to specify
+ *  the period)
+ * @VR_STATS_RPT_LAST_PKT: Frame interval statistics (use rpt_start to specify
+ *  first and last pkt)
+ */
+enum _usr_tx_stats_rpt_mode {
+	USR_TX_STATS_RPT_DIS = 0,
+	USR_TX_STATS_RPT_PERIOD,
+	USR_TX_STATS_RPT_LAST_PKT,
+	USR_TX_STATS_RPT_MAX,
+};
+
+/*
+ * struct rtw_phl_ch_stat_cnt is defined for querying chnl busy cnt info.
+ * see MAC_AX_HW_GET_CH_STAT_CNT for more info.
+ */
+struct rtw_phl_ch_stat_cnt {
+	u8 band;
+	u32 busy_cnt;
+	u32 idle_cnt;
+};
+
+struct rtw_phl_vr_frame_stats_rpt {
+	u32 rpt_mode;
+	u32 latency_first_pkt;
+	u32 latency_last_pkt;
+	u32 latency_first_2_last;
+	u32 tsf;
+	u32 rssi_dbm;
+	u32 snr_dbm;
+	u32 tx_rate_mode;
+	u32 tx_nss;
+	u32 tx_mcs;
+	u32 tx_per;
+	u32 tx_ok_cnt[4];
+	u32 tx_fail_cnt[4];
+	u32 tx_rty_fail_cnt[4];
+	u32 tx_ampdu_cnt_wo_last_pkt;
+	u32 tx_ampdu_len_wo_last_pkt;
+	u32 tx_rts_cnt;
+	u32 tx_rts_retry_cnt;
+	u32 rx_err_cnt;
+	u32 rx_clear_cycle;
+	u32 total_cycle;
+};
+
+/*
+ * struct rtw_phl_usr_tx_rpt_cfg is defined for statistic rpt configuration.
+ * FW will indicate stats rpt with C2H USR_TX_RPT_INFO.
+ * @mode: enum _usr_tx_stats_rpt_mode
+ * @rpt_start: set to 1 to notify the follwing pkt is the first pkt
+ *             set to 0 to notify the follwing pkt is the last pkt
+ * @macid: macid of the correspinding sta
+ * @rpt_period_us: specify the period in us for PHL_USR_TX_RPT_PERIOD mode only.
+ */
+struct rtw_phl_usr_tx_rpt_cfg {
+	enum _usr_tx_stats_rpt_mode mode;
+	u8 rpt_start;
+	u16 macid;
+	u8 band;
+	u8 port;
+	u32 rpt_period_us;
+};
+
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+
+struct rtw_phl_custom_ampdu_cfg {
+	u32 max_agg_time_32us;
+	u32 max_agg_num;
+};
+#endif /*_PHL_CUSTOM_DEF_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_usb.c b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_usb.c
new file mode 100644
index 000000000000..f476795f4dc8
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_usb.c
@@ -0,0 +1,65 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _PHL_CUSTOMIZE_FEATURE_USB_C_
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_USB
+#include "../phl_headers.h"
+#include "phl_custom_usb.h"
+
+enum rtw_phl_status phl_cmd_set_usb_support_ability(struct phl_info_t *phl_info,
+						u32 ability)
+{
+	if (rtw_hal_set_usb_support_ability(phl_info->hal, ability) == RTW_HAL_STATUS_SUCCESS) {
+		PHL_INFO("%s success \n", __func__);
+		return RTW_PHL_STATUS_SUCCESS;
+	} else {
+		return RTW_PHL_STATUS_FAILURE;
+	}
+}
+
+enum rtw_phl_status
+rtw_phl_custom_usb_set_support_ability(void *phl, u32 ability,
+				    enum phl_band_idx band_idx,
+				    enum phl_cmd_type cmd_type, u32 cmd_timeout)
+{
+	struct phl_info_t *phl_info = (struct phl_info_t *)phl;
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+
+#ifdef CONFIG_CMD_DISP
+	if (cmd_type == PHL_CMD_DIRECTLY) {
+		psts = phl_cmd_set_usb_support_ability(phl_info, ability);
+	} else {
+		psts = phl_cmd_enqueue(phl_info,
+					band_idx,
+					MSG_EVT_SET_USB_SW_ABILITY,
+					(u8*)&ability,
+					sizeof(u32),
+					NULL,
+					cmd_type,
+					cmd_timeout);
+	}
+
+	if (is_cmd_failure(psts))
+		/* Send cmd success, but wait cmd fail*/
+		psts = RTW_PHL_STATUS_FAILURE;
+	else if (psts != RTW_PHL_STATUS_SUCCESS)
+		/* Send cmd fail */
+		psts = RTW_PHL_STATUS_FAILURE;
+#else
+	psts = phl_cmd_set_usb_support_ability(phl_info, ability);
+#endif
+	return psts;
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_usb.h b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_usb.h
new file mode 100644
index 000000000000..0c0b5b207248
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_usb.h
@@ -0,0 +1,22 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _PHL_CUSTOM_USB_H_
+#define _PHL_CUSTOM_USB_H_
+
+enum rtw_phl_status
+rtw_phl_custom_usb_set_support_ability(void *phl, u32 ability,
+				    enum phl_band_idx band_idx,
+				    enum phl_cmd_type cmd_type, u32 cmd_timeout);
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr.c b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr.c
new file mode 100644
index 000000000000..2e9ad2bfb0e3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr.c
@@ -0,0 +1,1598 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _PHL_CUSTOMIZE_FEATURE_C_
+#include "../phl_headers.h"
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+#include "phl_custom_vr.h"
+#include "phl_custom_vr_csi.h"
+
+enum phl_mdl_ret_code
+_is_vr_mode_valid(void* custom_ctx,
+                  struct _custom_vr_ctx* vr_ctx,
+                  struct phl_msg* msg, u32 size)
+{
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	enum phl_mdl_ret_code ret = MDL_RET_SUCCESS;
+	if (!vr_ctx->init.enable || cmd->len < size || vr_ctx->init.wifi_role == NULL) {
+		PHL_INFO(" %s, evt_id(%d) not accepted\n",
+		         __FUNCTION__,
+		         MSG_EVT_ID_FIELD(msg->msg_id));
+		ret = MDL_RET_FAIL;
+		return ret;
+	}
+
+	return ret;
+}
+
+bool
+_feature_vr_enable_init_setting(void* custom_ctx,
+                             struct _custom_vr_ctx* vr_ctx)
+{
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_hal_com_t *hal_com = rtw_hal_get_halcom(phl->hal);
+
+	vr_ctx->init.phl = phl;
+#ifdef CONFIG_USB_HCI
+	/* To Do: to offload gpio number in halmac for different ICs */
+	rtw_hal_set_sw_gpio_mode(phl->phl_com, phl->hal
+		, RTW_AX_SW_IO_MODE_OUTPUT_OD, 12);
+
+	rtw_hal_set_sw_gpio_mode(phl->phl_com, phl->hal
+		, RTW_AX_SW_IO_MODE_OUTPUT_OD, 14);
+#endif
+	rtw_hal_auto_debug_en_phy_util(hal_com, true);
+
+	return true;
+}
+
+void
+_feature_vr_enable_deinit_setting(void* custom_ctx,
+                               struct _custom_vr_ctx* vr_ctx)
+{
+	/* TBD: merge codes from custom branch for the deinit setting*/
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_hal_com_t *hal_com = rtw_hal_get_halcom(phl->hal);
+
+	rtw_hal_auto_debug_en_phy_util(hal_com, false);
+
+	return;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_feature_enable(void* custom_ctx,
+                          struct _custom_vr_ctx* vr_ctx,
+                          struct phl_msg* msg)
+{
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+
+	if (cmd->len < sizeof(u32))
+		return MDL_RET_FAIL;
+
+	vr_ctx->init.enable = *(u32*)(cmd->data);
+	vr_ctx->init.test_mode = 0;
+	if (vr_ctx->init.enable){
+		_feature_vr_enable_init_setting(custom_ctx, vr_ctx);
+	} else {
+		_feature_vr_enable_deinit_setting(custom_ctx, vr_ctx);
+	}
+
+	PHL_INFO(" %s, vr feature enable(%d)\n", __FUNCTION__, vr_ctx->init.enable);
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&(vr_ctx->init.enable),
+	                           sizeof(u32));
+
+	return MDL_RET_SUCCESS;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_feature_query(void* custom_ctx,
+                             struct _custom_vr_ctx* vr_ctx,
+                             struct phl_msg* msg)
+{
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	if (cmd->len < sizeof(u32))
+		return ret;
+	PHL_INFO("%s, vr query feature enable(%d)\n",
+	         __FUNCTION__,
+	         vr_ctx->init.enable);
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&(vr_ctx->init.enable),
+	                           sizeof(u32));
+	ret = MDL_RET_SUCCESS;
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_testmode_param(void* custom_ctx,
+                              struct _custom_vr_ctx* vr_ctx,
+                              struct phl_msg* msg)
+{
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+
+	if (cmd->len < sizeof(u32))
+		return ret;
+	vr_ctx->init.test_mode = *(u32*)(cmd->data);
+	PHL_INFO("%s, test mode(0x%x)\n", __FUNCTION__,
+	         vr_ctx->init.test_mode);
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+
+	ret = MDL_RET_SUCCESS;
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_ampdu_cfg(void* custom_ctx,
+                         struct _custom_vr_ctx* vr_ctx,
+                         struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	void *d = phl_to_drvpriv(phl);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct rtw_phl_custom_ampdu_cfg custom_vr_ampdu_cfg = {0};
+	u32 size = sizeof(struct rtw_phl_custom_ampdu_cfg);
+	u8* val = cmd->data;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	u8 idx = 0;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	wrole = vr_ctx->init.wifi_role;
+	_os_mem_cpy(d, &custom_vr_ampdu_cfg, val, size);
+
+	PHL_INFO("%s, halsta(%d) ampdu dur(%d) num(%d)\n",
+	         __FUNCTION__,
+	         hal_status,
+	         custom_vr_ampdu_cfg.max_agg_time_32us,
+	         custom_vr_ampdu_cfg.max_agg_num);
+
+	for (idx = 0; idx < wrole->rlink_num; idx++) {
+		rlink = get_rlink(wrole, idx);
+
+		hal_status = rtw_hal_custom_cfg_tx_ampdu(phl->hal,
+		                                         rlink,
+		                                         &custom_vr_ampdu_cfg);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			ret = MDL_RET_FAIL;
+	}
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_ampdu_query(void* custom_ctx,
+                           struct _custom_vr_ctx* vr_ctx,
+                           struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct rtw_phl_custom_ampdu_cfg custom_vr_ampdu_cfg = {0};
+	u32 size = sizeof(struct rtw_phl_custom_ampdu_cfg);
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	u8 idx = 0;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	wrole = vr_ctx->init.wifi_role;
+
+	for (idx = 0; idx < wrole->rlink_num; idx++) {
+		rlink = get_rlink(wrole, idx);
+
+		hal_status = rtw_hal_get_ampdu_cfg(phl->hal,
+		                                   rlink,
+		                                   &custom_vr_ampdu_cfg);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			ret = MDL_RET_FAIL;
+	}
+
+	PHL_INFO(" %s, ampdu dur(%d) time(%d)\n",
+	         __FUNCTION__,
+	         custom_vr_ampdu_cfg.max_agg_time_32us,
+	         custom_vr_ampdu_cfg.max_agg_num);
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&custom_vr_ampdu_cfg,
+	                           sizeof(struct rtw_phl_custom_ampdu_cfg));
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_pdthr_cfg(void* custom_ctx,
+                         struct _custom_vr_ctx* vr_ctx,
+                         struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	int pd_thr = 0xff;
+	u32 size = sizeof(int);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	wrole = vr_ctx->init.wifi_role;
+	rlink = get_rlink(wrole, RTW_RLINK_PRIMARY);
+
+	pd_thr = *(int*)(cmd->data);
+	PHL_INFO("%s, pd_thr(%d)\n", __FUNCTION__, pd_thr);
+
+	hal_status = rtw_hal_set_pkt_detect_thold(phl->hal,
+						rlink->hw_band,
+						(u32)pd_thr);
+	PHL_INFO("%s, hal_status(%d)\n", __FUNCTION__, hal_status);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		ret = MDL_RET_FAIL;
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_pdthr_query(void* custom_ctx,
+                           struct _custom_vr_ctx* vr_ctx,
+                           struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	int pd_thr = 0xff;
+	u32 size = sizeof(int);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	wrole = vr_ctx->init.wifi_role;
+	rlink = get_rlink(wrole, RTW_RLINK_PRIMARY);
+
+	/*confirm whether pd thr is enabling or not*/
+	pd_thr = rtw_hal_query_pkt_detect_thold(phl->hal,
+	                                        true,
+	                                        rlink->hw_band);
+	if (pd_thr == 0) {
+		PHL_INFO("%s, disable! pd_thr(%d)\n", __FUNCTION__, pd_thr);
+	} else {
+		pd_thr = rtw_hal_query_pkt_detect_thold(phl->hal,
+		                                        false,
+		                                        rlink->hw_band);
+		PHL_INFO("%s, pd_thr(%d)\n", __FUNCTION__, pd_thr);
+	}
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&pd_thr,
+	                           sizeof(int));
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_pop_cfg(void* custom_ctx,
+                       struct _custom_vr_ctx* vr_ctx,
+                       struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	u32 size = sizeof(u32);
+	u32 pop_enable = 0xff;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	wrole = vr_ctx->init.wifi_role;
+	rlink = get_rlink(wrole, RTW_RLINK_PRIMARY);
+
+	pop_enable = *(u32*)(cmd->data);
+	PHL_INFO("%s, pop_enable(%d)\n", __FUNCTION__, pop_enable);
+	if (pop_enable != 0xff) {
+		hal_status = rtw_hal_set_pop_en(phl->hal,
+		                                (bool)pop_enable,
+		                                rlink->hw_band);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			ret = MDL_RET_FAIL;
+	}
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_pop_query(void* custom_ctx,
+                         struct _custom_vr_ctx* vr_ctx,
+                         struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	u32 size = sizeof(u32);
+	u32 pop_enable = 0xff;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS){
+		goto exit;
+	}
+
+	wrole = vr_ctx->init.wifi_role;
+	rlink = get_rlink(wrole, RTW_RLINK_PRIMARY);
+
+	pop_enable = rtw_hal_query_pop_en(phl->hal, rlink->hw_band);
+	PHL_INFO("%s, pop_en(%d)\n", __FUNCTION__, pop_enable);
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&pop_enable,
+	                            sizeof(u32));
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_set_tx_rate_masking(void* custom_ctx,
+                                   struct _custom_vr_ctx* vr_ctx,
+                                   struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	void *d = phl_to_drvpriv(phl);
+	struct rtw_phl_stainfo_t *sta = NULL;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	_os_spinlockfg sp_flags;
+	u64 tx_mask = (u64)(*(u32*)(cmd->data));
+	u64 tmp_mask = 0;
+	u8 idx = 0;
+
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, sizeof(u32));
+	if( ret != MDL_RET_SUCCESS){
+		goto exit;
+	}
+	wrole = vr_ctx->init.wifi_role;
+
+	/* Just only change the 1ss/2ss MCS rate*/
+	tx_mask = (tx_mask << 12);
+
+	/* search the stations in the queue*/
+
+	for (idx = 0; idx < wrole->rlink_num; idx++) {
+		rlink = get_rlink(wrole, idx);
+
+		_os_spinlock(d, &rlink->assoc_sta_queue.lock, _irq, &sp_flags);
+
+		phl_list_for_loop(sta, struct rtw_phl_stainfo_t,
+					&rlink->assoc_sta_queue.queue, list) {
+			if (sta == list_first_entry(&rlink->assoc_sta_queue.queue,
+						struct rtw_phl_stainfo_t, list))
+				continue;
+			if (sta) {
+				tmp_mask = sta->hal_sta->ra_info.cur_ra_mask & 0xfff;
+				tx_mask |= tmp_mask;
+				sta->hal_sta->ra_info.cur_ra_mask = tx_mask;
+				PHL_INFO("%s: tx mask(0x%016llx) macid(%d)\n",
+				         __FUNCTION__, tx_mask, sta->macid);
+			}
+		}
+		_os_spinunlock(d, &rlink->assoc_sta_queue.lock, _irq, &sp_flags);
+	}
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_get_tx_rate_masking(void* custom_ctx,
+                                   struct _custom_vr_ctx* vr_ctx,
+                                   struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	void *d = phl_to_drvpriv(phl);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct rtw_phl_stainfo_t *sta = NULL;
+	u32 tx_mask = 0;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	_os_spinlockfg sp_flags;
+	u8 idx = 0;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, sizeof(u32));
+	if( ret != MDL_RET_SUCCESS){
+		goto exit;
+	}
+
+	wrole = vr_ctx->init.wifi_role;
+
+	/** Search the stations in the queue
+	 *  We just need to query the first sta because all the masking
+	 *  setting are the same for all sta.
+	 */
+
+	for (idx = 0; idx < wrole->rlink_num; idx++) {
+		rlink = get_rlink(wrole, idx);
+
+		_os_spinlock(d, &rlink->assoc_sta_queue.lock, _irq, &sp_flags);
+		phl_list_for_loop(sta, struct rtw_phl_stainfo_t,
+					&rlink->assoc_sta_queue.queue, list) {
+			if (sta == list_first_entry(&rlink->assoc_sta_queue.queue,
+						struct rtw_phl_stainfo_t, list))
+				continue;
+			if (sta) {
+				tx_mask = (u32)(sta->hal_sta->ra_info.cur_ra_mask >> 12);
+				break;
+			}
+		}
+
+		_os_spinunlock(d, &rlink->assoc_sta_queue.lock, _irq, &sp_flags);
+	}
+
+	PHL_INFO("%s: tx mask(%x)\n", __FUNCTION__, tx_mask);
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&tx_mask,
+	                           sizeof(u32));
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_set_tx_rate_rty_tbl(void* custom_ctx,
+                                   struct _custom_vr_ctx* vr_ctx,
+                                   struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	void *d = phl_to_drvpriv(phl);
+	u8* val = (u8*)(cmd->data);
+	u8 i = 0;
+	u32 size = sizeof(struct _vr_tx_rty_param);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if( ret != MDL_RET_SUCCESS){
+		goto exit;
+	}
+
+	_os_mem_cpy(d, &vr_ctx->tx_rty_param, val, size);
+
+	PHL_INFO("%s retry setting %s\n",
+	          __func__,
+	          vr_ctx->tx_rty_param.enable ? "enable" : "disable");
+
+	for(i = 0; i < 24 ; i++){
+	PHL_INFO("%s retry setting index(%d) rty_count(%d)\n",
+	          __func__, i, vr_ctx->tx_rty_param.rty_rate_tbl[i]);
+	}
+
+	rtw_hal_set_tx_rate_rty_tbl(phl->hal,
+	                            (bool)vr_ctx->tx_rty_param.enable,
+	                            vr_ctx->tx_rty_param.rty_rate_tbl);
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_get_tx_rate_rty_tbl(void* custom_ctx,
+                                   struct _custom_vr_ctx* vr_ctx,
+                                   struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	u8 i = 0;
+	u32 size = sizeof(struct _vr_tx_rty_param);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if( ret != MDL_RET_SUCCESS){
+		goto exit;
+	}
+
+	PHL_INFO("%s retry setting %s\n",
+	          __func__,
+	          vr_ctx->tx_rty_param.enable ? "enable" : "disable");
+
+	for(i = 0; i < 24 ; i++){
+	PHL_INFO("%s retry setting index(%d) rty_count(%d)\n",
+	          __func__, i, vr_ctx->tx_rty_param.rty_rate_tbl[i]);
+	}
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&vr_ctx->tx_rty_param,
+	                           size);
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_edca_query(void* custom_ctx,
+                          struct _custom_vr_ctx* vr_ctx,
+                          struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct rtw_edca_param edca_param = {0};
+	struct rtw_wifi_role_t *wifi_role = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	u8 idx = 0;
+
+	edca_param.ac = 0xFF;
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, sizeof(u8));
+	if (ret != MDL_RET_SUCCESS)
+		goto exit;
+
+	wifi_role = vr_ctx->init.wifi_role;
+	edca_param.ac = *((u8*)(cmd->data));
+
+	for (idx = 0; idx < wifi_role->rlink_num; idx++) {
+		rlink = get_rlink(wifi_role, idx);
+
+		hal_status = rtw_hal_get_edca(phl->hal,
+		                              rlink,
+		                              &edca_param);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			ret = MDL_RET_FAIL;
+	}
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		edca_param.ac = 0xFF;
+
+	PHL_INFO("%s, custom_vr_edca_param rpt: ac(%d), param(0x%x)\n",
+	         __FUNCTION__,
+	         edca_param.ac,
+	         edca_param.param);
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&edca_param,
+	                           sizeof(struct rtw_edca_param));
+
+	return MDL_RET_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+bool
+_is_stats_rpt_cfg_needed(struct _vr_stats_rpt_param *cur,
+                         struct _vr_stats_rpt_cfg *target)
+{
+	if (cur->cfg.mode != target->mode)
+		return true;
+
+	if (cur->cfg.macid != target->macid)
+		return true;
+
+	if (cur->cfg.mode == USR_TX_STATS_RPT_PERIOD &&
+	    (cur->cfg.rpt_period_us != target->rpt_period_us))
+		return true;
+
+	return false;
+}
+
+bool
+_update_usr_tx_rpt_cfg(struct phl_info_t *phl,
+                       struct _custom_vr_ctx *vr_ctx,
+		       struct rtw_phl_stainfo_t *sta,
+                       enum _usr_tx_stats_rpt_mode mode,
+                       u16 macid,
+                       u32 rpt_period_us)
+{
+	struct rtw_phl_usr_tx_rpt_cfg usr_tx_rpt_cfg = {0};
+
+	usr_tx_rpt_cfg.mode = mode;
+	usr_tx_rpt_cfg.rpt_start = 0;
+	usr_tx_rpt_cfg.macid = macid;
+	usr_tx_rpt_cfg.band = sta->rlink->hw_band;
+	usr_tx_rpt_cfg.port = sta->rlink->hw_port;
+	usr_tx_rpt_cfg.rpt_period_us = rpt_period_us;
+
+	if (rtw_hal_set_usr_tx_rpt_cfg(phl->hal, &usr_tx_rpt_cfg))
+		return false;
+
+	/* update cur stats if succeed */
+	vr_ctx->stats_rpt.cfg.mode = mode;
+	vr_ctx->stats_rpt.cfg.macid = macid;
+	vr_ctx->stats_rpt.cfg.rpt_period_us = rpt_period_us;
+	return true;
+}
+
+void _reset_tx_stats_ring(void *drvpriv, struct _tx_stats_ring *ring)
+{
+	_os_spinlock(drvpriv, &ring->lock, _bh, NULL);
+	ring->r_idx = ring->w_idx = 0;
+	_os_mem_set(drvpriv,
+	            ring->entry,
+	            0,
+	            MAX_TX_STATS_RING_ENTRY_NUM * sizeof(struct _tx_stats_entry));
+	_os_spinunlock(drvpriv, &ring->lock, _bh, NULL);
+}
+
+bool
+_vr_stats_cfg(struct phl_info_t *phl,
+              struct _custom_vr_ctx *vr_ctx,
+              struct rtw_phl_stainfo_t *sta,
+              struct _vr_stats_rpt_cfg *cfg)
+{
+	bool ret = false;
+	void *drvpriv = phl_to_drvpriv(phl);
+	struct _tx_stats_ring *ring = &vr_ctx->stats_rpt.t_stats_ring;
+
+	if (cfg->mode >= USR_TX_STATS_RPT_MAX) {
+		PHL_INFO(" %s, invalid mode(%d)\n", __FUNCTION__, cfg->mode);
+		goto exit;
+	}
+
+	/* 1. Set to disable first or target is USR_TX_STATS_RPT_DIS*/
+	if (!_update_usr_tx_rpt_cfg(phl, vr_ctx, sta, USR_TX_STATS_RPT_DIS,
+	                            (u16)cfg->macid, 0)) {
+		PHL_INFO(" %s disable first fail\n", __FUNCTION__);
+		goto exit;
+	}
+	/*if target is USR_TX_STATS_RPT_DIS, just reset tx_stats */
+	if(cfg->mode == USR_TX_STATS_RPT_DIS) {
+		_reset_tx_stats_ring(drvpriv, ring);
+		ret = true;
+		goto exit;
+	}
+	/* 2. set_ch_busy_sta_cfg for channel busy information */
+	if (rtw_hal_set_ch_busy_stat_cfg(phl->hal, sta->rlink->hw_band)) {
+		PHL_INFO(" %s setup busy info fail\n", __FUNCTION__);
+		goto exit;
+	}
+	/* 3. Config stats mode */
+	if (_update_usr_tx_rpt_cfg(phl, vr_ctx, sta, cfg->mode, (u16)cfg->macid,
+	                           cfg->rpt_period_us)) {
+		_reset_tx_stats_ring(drvpriv, ring);
+		ret = true;
+	}
+exit:
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_stats_rpt_cfg(void *custom_ctx,
+                             struct _custom_vr_ctx *vr_ctx,
+                             struct phl_msg *msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct rtw_phl_stainfo_t *sta = NULL;
+	struct _vr_stats_rpt_param *stats_rpt = &(vr_ctx->stats_rpt);
+	struct _vr_stats_rpt_cfg *cfg = NULL;
+	u32 size = sizeof(struct _vr_stats_rpt_cfg);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS)
+		goto exit;
+
+	ret = MDL_RET_FAIL;
+	cfg = (struct _vr_stats_rpt_cfg *)(cmd->data);
+
+	/* Check whether the macid is existed */
+	sta = rtw_phl_get_stainfo_by_macid(phl, (u16)cfg->macid);
+	if (sta == NULL) {
+		PHL_INFO(" %s, invalid macid(%d).\n", __FUNCTION__, cfg->macid);
+		goto exit;
+	}
+
+	/* Check whether the configuration is changed. */
+	if (!_is_stats_rpt_cfg_needed(stats_rpt, cfg)) {
+		PHL_INFO(" %s, no need to config, mode(%d), macid(%d), period(%d).\n",
+		         __FUNCTION__,
+		         cfg->mode,
+		         cfg->macid,
+		         cfg->rpt_period_us);
+		ret = MDL_RET_SUCCESS;
+		goto exit;
+	}
+	if(_vr_stats_cfg(phl, vr_ctx, sta, cfg))
+		ret = MDL_RET_SUCCESS;
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8 *)&ret,
+	                           sizeof(u8));
+
+	return ret;
+}
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+
+enum phl_mdl_ret_code
+_phl_custom_vr_sr_cfg(void* custom_ctx,
+                       struct _custom_vr_ctx* vr_ctx,
+                       struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	u32 size = sizeof(u8);
+	u8 sr_enable = false;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	sr_enable = *(u8*)(cmd->data);
+	PHL_INFO("%s, sr_enable(%d)\n", __FUNCTION__, sr_enable);
+
+	hal_status = rtw_hal_set_spatial_reuse_en(phl->hal, (bool)sr_enable);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		ret = MDL_RET_FAIL;
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_sr_query(void* custom_ctx,
+                         struct _custom_vr_ctx* vr_ctx,
+                         struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	u32 size = sizeof(u8);
+	u8 sr_enable = 0xff;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS){
+		goto exit;
+	}
+
+	sr_enable = rtw_hal_is_spatial_reuse_en(phl->hal);
+	PHL_INFO("%s, sr_en(%d)\n", __FUNCTION__, sr_enable);
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           &sr_enable,
+	                            sizeof(u8));
+
+	return ret;
+}
+
+enum phl_mdl_ret_code _phl_custom_vr_rf_scramble(void *custom_ctx,
+						 struct _custom_vr_ctx *vr_ctx,
+						 struct phl_msg *msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	void *d = phl_to_drvpriv(phl);
+	struct rtw_custom_decrpt *cmd =
+	    (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct _vr_rf_scrmb *val = (struct _vr_rf_scrmb *)(cmd->data);
+	u32 size = sizeof(struct _vr_rf_scrmb);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		val->status = RTW_HAL_STATUS_FAILURE;
+		goto exit;
+	}
+
+	PHL_INFO("%s: scramble cmd(%s)\n", __func__,
+		 (val->is_set) ? "Set_scrmb" : "Get_scrmb");
+	if (val->is_set) {
+		hal_status = rtw_hal_set_usr_frame_to_act(
+		    phl->hal, val->mode, val->to_thr, val->trigger_cnt,
+		    val->sw_def_bmp);
+		val->status = (u8)hal_status;
+		if (hal_status == RTW_HAL_STATUS_SUCCESS)
+			_os_mem_cpy(d, &vr_ctx->cur_scrmb_param, val, size);
+	} else {
+		_os_mem_cpy(d, val, &vr_ctx->cur_scrmb_param, size);
+	}
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx, cmd->evt_id, cmd->customer_id,
+				   (u8 *)val, sizeof(struct _vr_rf_scrmb));
+
+	return ret;
+}
+
+struct custom_vr_perf_metrics_rlink {
+	u8 mac_addr[MAC_ALEN];
+	u8 rssi;
+	char tx_rate_str[32];
+	char rx_rate_str[32];
+	u8 clm_ratio;
+	u8 nhm_ratio;
+	u8 snr_avg;
+};
+
+struct custom_vr_perf_metrics {
+	u8 rlink_num;
+	struct custom_vr_perf_metrics_rlink perf_metrics_rlink[RTW_RLINK_MAX];
+};
+
+enum phl_mdl_ret_code
+_phl_custom_vr_perf_mertics_query(void* custom_ctx,
+                                 struct _custom_vr_ctx* vr_ctx,
+                                 struct phl_msg* msg) {
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	void *drv = phl_to_drvpriv(phl);
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct custom_vr_perf_metrics perf_metrics = {0};
+	struct custom_vr_perf_metrics_rlink *curr = NULL;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	struct rtw_phl_stainfo_t *n, *psta;
+	struct rtw_hal_com_t *hal_com = rtw_hal_get_halcom(phl->hal);
+	struct rtw_env_report env_rpt = {0};
+	u8 idx = 0;
+
+	PHL_INFO("Enter %s\n", __func__);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, sizeof(u8));
+	if (ret != MDL_RET_SUCCESS)
+		goto exit;
+
+	wrole = vr_ctx->init.wifi_role;
+	perf_metrics.rlink_num = wrole->rlink_num;
+	PHL_INFO("%s(): rlink_num = %d\n", __func__, perf_metrics.rlink_num);
+	for (idx = 0; idx < wrole->rlink_num; idx++) {
+		rlink = get_rlink(wrole, idx);
+		curr = &perf_metrics.perf_metrics_rlink[idx];
+		_os_spinlock(drv, &rlink->assoc_sta_queue.lock, _bh, NULL);
+		phl_list_for_loop_safe(psta, n, struct rtw_phl_stainfo_t,
+		       &rlink->assoc_sta_queue.queue, list) {
+			_os_mem_cpy(drv, curr->mac_addr, psta->mac_addr, MAC_ALEN);
+			PHL_INFO("mac addr %02x-%02x-%02x-%02x-%02x-%02x\n",
+			         curr->mac_addr[0], curr->mac_addr[1],
+			         curr->mac_addr[2], curr->mac_addr[3],
+			         curr->mac_addr[4], curr->mac_addr[5]);
+
+			PHL_INFO("WROLE-IDX:%d RLINK-IDX:%d wlan_mode:0x%02x, chan:%d, bw:%d, rlink_state:%s\n",
+			         psta->wrole->id,
+			         psta->rlink->id,
+			         psta->wmode,
+			         psta->chandef.chan,
+			         psta->chandef.bw,
+			         rlink->mstate?((rlink->mstate & MLME_LINKING)?"Linking":"Linked Up"):"No Link");
+
+			curr->rssi = psta->hal_sta->rssi_stat.ma_rssi;
+			PHL_INFO("[Stats] MA RSSI:%d(dBm)\n",
+			         curr->rssi - PHL_MAX_RSSI);
+
+			convert_tx_rate(psta->hal_sta->ra_info.rpt_rt_i.mode,
+			                 psta->hal_sta->ra_info.rpt_rt_i.mcs_ss_idx,
+			                 curr->tx_rate_str, 32);
+			PHL_INFO("[Stats] Tx Rate:%s\n", curr->tx_rate_str);
+
+			convert_rx_rate(psta->stats.rx_rate, curr->rx_rate_str,
+			                 32);
+			PHL_INFO("[Stats] Rx Rate:%s\n", curr->rx_rate_str);
+		}
+		_os_spinunlock(drv, &rlink->assoc_sta_queue.lock, _bh, NULL);
+
+		rtw_hal_env_rpt(hal_com, &env_rpt, rlink->hw_band);
+		curr->clm_ratio = env_rpt.clm_ratio;
+		curr->nhm_ratio = env_rpt.nhm_ratio;
+		PHL_INFO("[Stats] clm_ratio = %d\n", curr->clm_ratio);
+		PHL_INFO("[Stats] nhm_ratio = %d\n", curr->nhm_ratio);
+
+		rtw_hal_query_snr_avg(hal_com, &curr->snr_avg, rlink->hw_band);
+		PHL_INFO("[Stats] snr_avg = %d\n", curr->snr_avg);
+	}
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&perf_metrics,
+	                           sizeof(struct custom_vr_perf_metrics));
+
+	PHL_INFO("Leave %s\n", __func__);
+	return MDL_RET_SUCCESS;
+}
+
+enum phl_mdl_ret_code _phl_custom_vr_set_ant_switch(
+    void *custom_ctx, struct _custom_vr_ctx *vr_ctx, struct phl_msg *msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_custom_decrpt *cmd =
+	    (struct rtw_custom_decrpt *)(msg->inbuf);
+	u32 path = *(u32 *)(cmd->data);
+	u32 size = sizeof(u32);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	PHL_INFO("%s: set rf path(%d)\n", __func__, path);
+
+	switch (path) {
+	case RF_PATH_AC:
+		if ((rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 0, 12) !=
+		     RTW_HAL_STATUS_SUCCESS) ||
+		    (rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 0, 14) !=
+		     RTW_HAL_STATUS_SUCCESS))
+			ret = MDL_RET_FAIL;
+		PHL_INFO("%s: set path to AC\n", __func__);
+		break;
+	case RF_PATH_AD:
+		if ((rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 0, 12) !=
+		     RTW_HAL_STATUS_SUCCESS) ||
+		    (rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 1, 14) !=
+		     RTW_HAL_STATUS_SUCCESS))
+			ret = MDL_RET_FAIL;
+		PHL_INFO("%s: set path to AD\n", __func__);
+		break;
+	case RF_PATH_BC:
+		if ((rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 1, 12) !=
+		     RTW_HAL_STATUS_SUCCESS) ||
+		    (rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 0, 14) !=
+		     RTW_HAL_STATUS_SUCCESS))
+			ret = MDL_RET_FAIL;
+		PHL_INFO("%s: set path to BC\n", __func__);
+		break;
+	case RF_PATH_BD:
+		if ((rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 1, 12) !=
+		     RTW_HAL_STATUS_SUCCESS) ||
+		    (rtw_hal_sw_gpio_ctrl(phl->phl_com, phl->hal, 1, 14) !=
+		     RTW_HAL_STATUS_SUCCESS))
+			ret = MDL_RET_FAIL;
+		PHL_INFO("%s: set path to BD\n", __func__);
+		break;
+	default:
+		PHL_INFO("%s: unexpected rf path!\n", __func__);
+		break;
+	}
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx, cmd->evt_id, cmd->customer_id,
+				   (u8 *)&ret, sizeof(u32));
+
+	return ret;
+}
+
+#ifdef CONFIG_PHL_CHANNEL_INFO_VR
+enum phl_mdl_ret_code
+_phl_custom_vr_csi_cfg(void* custom_ctx,
+                       struct _custom_vr_ctx* vr_ctx,
+                       struct phl_msg* msg) {
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	u32 size = sizeof(u8);
+	u8 csi_enable = false;
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS) {
+		goto exit;
+	}
+
+	csi_enable = *(u8*)(cmd->data);
+	PHL_INFO("%s, csi_enable(%d)\n", __FUNCTION__, csi_enable);
+
+	if (csi_enable) {
+		struct rtw_wifi_role_t *wrole = NULL;
+		struct rtw_wifi_role_link_t *rlink = NULL;
+
+		wrole = vr_ctx->init.wifi_role;
+		rlink = get_rlink(wrole, RTW_RLINK_PRIMARY);
+		ret = rtw_phl_custom_csi_start(custom_ctx,
+		                               &vr_ctx->csi_ctrl,
+		                               rlink);
+	} else {
+		ret = rtw_phl_custom_csi_stop(custom_ctx, &vr_ctx->csi_ctrl);
+	}
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&ret,
+	                           sizeof(u8));
+
+	return ret;
+}
+
+/*
+ * custom_vr_csi - channel state information,
+ * @len: length of channel info buffer
+ * @buf: channel info buffer
+ *       buf size is limited by minumim of MAX_DATA_SIZE (VR command data size)
+ *       and CHAN_INFO_MAX_SIZE.
+ */
+struct custom_vr_csi {
+	u32 len;
+	u8 buf[MAX_DATA_SIZE - sizeof(u32)];
+	/* u8 buf[CHAN_INFO_MAX_SIZE]; */
+};
+
+enum phl_mdl_ret_code
+_phl_custom_vr_csi_query(void* custom_ctx,
+                         struct _custom_vr_ctx* vr_ctx,
+                         struct phl_msg* msg) {
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	struct rtw_custom_decrpt *cmd = (struct rtw_custom_decrpt *)(msg->inbuf);
+	struct custom_vr_csi vr_csi = {0};
+	u32 size = sizeof(struct custom_vr_csi);
+	u32 i = 0, print_len = 0;
+	u64 *buff_tmp = NULL;
+
+	PHL_INFO("%s\n", __func__);
+
+	ret = _is_vr_mode_valid(custom_ctx, vr_ctx, msg, size);
+	if (ret != MDL_RET_SUCCESS)
+		goto exit;
+
+	ret = rtw_phl_custom_csi_rslt_query(custom_ctx,
+	                                    (u8 *)vr_csi.buf,
+	                                    &vr_csi.len);
+
+	print_len = vr_csi.len >> 3;
+	if (vr_csi.len % 8)
+		print_len++;
+	buff_tmp = (u64 *)vr_csi.buf;
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "%s, CSI raw data: len = %d\n",
+	          __func__, vr_csi.len);
+	for (i = 0; i < print_len; i++)
+		PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[%02d]0x%016llx\n",
+		          i, buff_tmp[i]);
+
+exit:
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&vr_csi,
+	                           sizeof(struct custom_vr_csi));
+
+	return MDL_RET_SUCCESS;
+}
+#endif /* CONFIG_PHL_CHANNEL_INFO_VR */
+
+enum phl_mdl_ret_code _phl_custom_vr_set_bss_color(
+    void *custom_ctx, struct _custom_vr_ctx *vr_ctx, struct phl_msg *msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_SUCCESS;
+	struct rtw_custom_decrpt *cmd =
+	    (struct rtw_custom_decrpt *)(msg->inbuf);
+	u8 bss_color = 0;
+
+	bss_color = *(u8 *)(cmd->data);
+
+	/* check if bss color is valid or not */
+	if (bss_color >= 1 && bss_color <= 63) {
+		vr_ctx->bsscolor_param.bsscolor = bss_color;
+		vr_ctx->bsscolor_param.is_valid = true;
+	}
+
+	PHL_INFO("%s: bss color(0x%x) is_valid(%d)\n", __func__,
+		 vr_ctx->bsscolor_param.bsscolor, vr_ctx->bsscolor_param.is_valid);
+
+	phl_custom_prepare_evt_rpt(custom_ctx, cmd->evt_id, cmd->customer_id,
+				   (u8 *)&ret, sizeof(u8));
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+phl_custom_hdl_vr_evt(void* dispr,
+                      void* custom_ctx,
+                      struct _custom_vr_ctx* vr_ctx,
+                      struct phl_msg* msg)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+	u8 prephase = (IS_MSG_IN_PRE_PHASE(msg->msg_id)) ? (true) : (false);
+
+	if (prephase == true)
+		return MDL_RET_SUCCESS;
+
+	switch (MSG_EVT_ID_FIELD(msg->msg_id)) {
+		case MSG_EVT_CUSTOME_FEATURE_ENABLE:
+			ret = _phl_custom_vr_feature_enable(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_CUSTOME_FEATURE_QUERY:
+			ret = _phl_custom_vr_feature_query(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_CUSTOME_TESTMODE_PARAM:
+			ret = _phl_custom_vr_testmode_param(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_EDCA_QUERY:
+			ret = _phl_custom_vr_edca_query(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_AMPDU_CFG:
+			ret = _phl_custom_vr_ampdu_cfg(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_AMPDU_QUERY:
+			ret = _phl_custom_vr_ampdu_query(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_PDTHR_CFG:
+			ret = _phl_custom_vr_pdthr_cfg(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_PDTHR_QUERY:
+			ret = _phl_custom_vr_pdthr_query(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_POP_CFG:
+			ret = _phl_custom_vr_pop_cfg(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_POP_QUERY:
+			ret = _phl_custom_vr_pop_query(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_SET_TX_RATE_MASKING:
+			ret = _phl_custom_vr_set_tx_rate_masking(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_GET_TX_RATE_MASKING:
+			ret = _phl_custom_vr_get_tx_rate_masking(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_SET_TX_RATE_RTY_TBL:
+			ret = _phl_custom_vr_set_tx_rate_rty_tbl(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_GET_TX_RATE_RTY_TBL:
+			ret = _phl_custom_vr_get_tx_rate_rty_tbl(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_GET_PERF_METRICS:
+			ret = _phl_custom_vr_perf_mertics_query(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_SET_SPATIAL_REUSE:
+			ret = _phl_custom_vr_sr_cfg(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_GET_SPATIAL_REUSE:
+			ret = _phl_custom_vr_sr_query(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_RF_SCRAMBLE:
+			ret = _phl_custom_vr_rf_scramble(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_SET_ANT_SWITCH:
+			ret = _phl_custom_vr_set_ant_switch(custom_ctx, vr_ctx,
+							    msg);
+			break;
+#ifdef CONFIG_PHL_CHANNEL_INFO_VR
+		case MSG_EVT_SET_CSI:
+			ret = _phl_custom_vr_csi_cfg(custom_ctx, vr_ctx, msg);
+			break;
+		case MSG_EVT_GET_CSI:
+			ret = _phl_custom_vr_csi_query(custom_ctx, vr_ctx, msg);
+			break;
+#endif /* CONFIG_PHL_CHANNEL_INFO_VR */
+		case MSG_EVT_SET_BSS_COLOR:
+			ret = _phl_custom_vr_set_bss_color(custom_ctx, vr_ctx,
+							   msg);
+			break;
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+		case MSG_EVT_STATS_RPT_CFG:
+			ret = _phl_custom_vr_stats_rpt_cfg(custom_ctx, vr_ctx, msg);
+			break;
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+		default:
+			ret = MDL_RET_SUCCESS;
+			break;
+	}
+	PHL_INFO("%s, evt(%d), ret(%d)\n", __FUNCTION__,
+	         MSG_EVT_ID_FIELD(msg->msg_id),
+	         ret);
+	return ret;
+}
+enum phl_mdl_ret_code
+phl_custom_hdl_vr_fail_evt(void* dispr,
+                           void* custom_ctx,
+                           struct _custom_vr_ctx* vr_ctx,
+                           struct phl_msg* msg)
+{
+	return MDL_RET_IGNORE;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_set_wifi_role(void* dispr,
+                             void* custom_ctx,
+                             struct _custom_vr_ctx* vr_ctx,
+                             struct rtw_custom_decrpt *cmd)
+{
+	enum phl_mdl_ret_code status = MDL_RET_SUCCESS;
+	u32 size = sizeof(struct rtw_wifi_role_t *);
+
+	PHL_INFO("%s, start\n", __FUNCTION__);
+
+	if (cmd->len < size) {
+		PHL_INFO("%s: illegal info len\n", __FUNCTION__);
+		status = MDL_RET_FAIL;
+		return status;
+	}
+
+	vr_ctx->init.wifi_role = *(struct rtw_wifi_role_t **)(cmd->data);
+
+	phl_custom_prepare_evt_rpt(custom_ctx,
+	                           cmd->evt_id,
+	                           cmd->customer_id,
+	                           (u8*)&status,
+	                           sizeof(u8));
+
+	PHL_INFO("%s, status(%d).\n", __FUNCTION__, status);
+
+	return status;
+}
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+void _dump_frame_stats_rpt(struct rtw_phl_vr_frame_stats_rpt *cus_rpt)
+{
+	u8 q = 0;
+
+	PHL_INFO("%s, rpt_mode(%d), tsf(%d).\n", __FUNCTION__,
+		 cus_rpt->rpt_mode, cus_rpt->tsf);
+
+	for (q = PHL_BE_QUEUE_SEL; q < PHL_AC_QUEUE_TOTAL; q++) {
+		PHL_INFO(
+		    " %s, q(%d): (t_ok, t_rty_fail, t_fail) = (%d, %d, %d)\n",
+		    __FUNCTION__, q, cus_rpt->tx_ok_cnt[q],
+		    cus_rpt->tx_rty_fail_cnt[0], cus_rpt->tx_fail_cnt[q]);
+	}
+}
+
+bool
+_prepare_frame_stats_rpt(struct phl_info_t *phl,
+                         struct _vr_stats_rpt_param *stats_rpt,
+                         struct rtw_mac_usr_tx_rpt_info *c2h_rpt)
+{
+	bool is_rpt_ready = false;
+	u8 mode = (u8)c2h_rpt->rpt_mode;
+	u16 macid = (u16)c2h_rpt->macid;
+	void *drvpriv = phl_to_drvpriv(phl);
+	struct rtw_phl_stainfo_t *sta = rtw_phl_get_stainfo_by_macid(phl, macid);
+	struct _tx_stats_ring *ring = &stats_rpt->t_stats_ring;
+	struct rtw_phl_vr_frame_stats_rpt *cus_rpt = NULL;
+	u8 i;
+	u32 latency_first_pkt, latency_last_pkt, latency_first_2_last;
+
+	if (mode != stats_rpt->cfg.mode || macid != stats_rpt->cfg.macid || sta == NULL) {
+		PHL_INFO("%s, skip, c2h_mode(%d)!=cfg_mode(%d) or c2h_macid(%d)!=cfg_macid(%d).\n",
+	                 __FUNCTION__,
+			 mode,
+			 stats_rpt->cfg.mode,
+	                 macid,
+		         stats_rpt->cfg.macid);
+		return false;
+	}
+
+	latency_first_pkt =  phl_get_diff_time_us(c2h_rpt->freerun_cnt_first_in,
+	                                   c2h_rpt->freerun_cnt_first_out);
+	latency_last_pkt = phl_get_diff_time_us(c2h_rpt->freerun_cnt_last_in,
+	                                 c2h_rpt->freerun_cnt_last_out);
+	latency_first_2_last = phl_get_diff_time_us(c2h_rpt->freerun_cnt_first_in,
+	                                     c2h_rpt->freerun_cnt_last_out);
+
+	_os_spinlock(drvpriv, &ring->lock, _bh, NULL);
+
+	if (stats_rpt->cfg.mode == USR_TX_STATS_RPT_PERIOD)
+		ring->w_idx = (ring->w_idx + 1) % MAX_TX_STATS_RING_ENTRY_NUM;
+
+	/* select the first empty cus rpt */
+	i = ring->r_idx;
+	while (i != ring->w_idx) {
+		if (ring->entry[i].cus_rpt.rpt_mode == USR_TX_STATS_RPT_DIS)
+			break;
+		else if (ring->entry[i].total_pkt_cnt == ring->entry[i].recv_rpt_cnt &&
+		         !ring->entry[i].rpt_sent)
+			is_rpt_ready = true;
+		i = (i + 1) % MAX_TX_STATS_RING_ENTRY_NUM;
+	}
+
+	/* fill the cus rpt if index is valid */
+	if (i == ring->w_idx) {
+		PHL_WARN(" %s, skip this c2h rpt, ridx(%d), widx(%d)\n",
+		         __FUNCTION__,
+			 ring->r_idx,
+		         ring->w_idx);
+	} else {
+		PHL_INFO(" %s, recv c2h for ring entry idx(%d)\n", __FUNCTION__, i);
+
+		cus_rpt = &ring->entry[i].cus_rpt;
+		cus_rpt->rpt_mode = stats_rpt->cfg.mode;
+		cus_rpt->latency_first_pkt = latency_first_pkt;
+		cus_rpt->latency_last_pkt = latency_last_pkt;
+		cus_rpt->latency_first_2_last = latency_first_2_last;
+		cus_rpt->tsf = c2h_rpt->freerun_cnt_last_out ;
+		cus_rpt->rssi_dbm = PHL_MAX_RSSI - sta->hal_sta->rssi_stat.ma_rssi;
+		cus_rpt->snr_dbm = sta->hal_sta->rssi_stat.snr_ma >> 4;
+
+		cus_rpt->tx_rate_mode = sta->hal_sta->ra_info.rpt_rt_i.mode;
+		cus_rpt->tx_mcs = sta->hal_sta->ra_info.rpt_rt_i.mcs_idx;
+		cus_rpt->tx_nss =
+		    sta->hal_sta->ra_info.rpt_rt_i.ss + 1;
+		cus_rpt->tx_per = c2h_rpt->ra_ratio;
+		cus_rpt->tx_ampdu_cnt_wo_last_pkt = c2h_rpt->tx_ppdu_cnt_wo_last_pkt;
+		/* here we only cal avg mpdu cnt (per ampdu), the len of ampdu will be cal in core custom */
+		cus_rpt->tx_ampdu_len_wo_last_pkt =
+		        (c2h_rpt->tx_ppdu_cnt_wo_last_pkt)?
+		        (c2h_rpt->tx_mpdu_cnt_wo_last_pkt / c2h_rpt->tx_ppdu_cnt_wo_last_pkt):
+		        0;
+		cus_rpt->tx_rts_cnt = c2h_rpt->tx_rts_cnt;
+		cus_rpt->tx_rts_retry_cnt = c2h_rpt->tx_rts_retry_cnt;
+
+		cus_rpt->rx_err_cnt = c2h_rpt->rx_err_cnt;
+		cus_rpt->rx_clear_cycle = c2h_rpt->rx_clear_us;
+		cus_rpt->total_cycle = c2h_rpt->rx_clear_us + c2h_rpt->busy_us;
+
+		/* debug msg */
+		_dump_frame_stats_rpt(cus_rpt);
+
+		if (ring->entry[i].total_pkt_cnt <= ring->entry[i].recv_rpt_cnt) {
+			is_rpt_ready = true;
+		} else {
+			PHL_WARN(" %s, total_pkt_cnt(%d) != recv_rpt_cnt(%d)\n",
+			         __FUNCTION__,
+			         ring->entry[i].total_pkt_cnt,
+			         ring->entry[i].recv_rpt_cnt);
+		}
+	}
+	_os_spinunlock(drvpriv, &ring->lock, _bh, NULL);
+
+	return is_rpt_ready;
+}
+
+void
+_usr_tx_rpt_notify_complete(void *priv, struct phl_msg *msg)
+{
+	struct _custom_vr_ctx *vr_ctx = ( struct _custom_vr_ctx *)priv;
+	struct _tx_stats_ring *ring = &vr_ctx->stats_rpt.t_stats_ring;
+	void *drvpriv = phl_to_drvpriv(vr_ctx->init.phl);
+	u8 i;
+
+	_os_spinlock(drvpriv, &ring->lock, _bh, NULL);
+	i = ring->r_idx;
+	PHL_INFO(" %s, idx(%d)\n", __FUNCTION__, i);
+	_os_mem_set(drvpriv, ring->entry + i, 0, sizeof(struct _tx_stats_entry));
+	ring->r_idx = (i + 1) % MAX_TX_STATS_RING_ENTRY_NUM;
+	_os_spinunlock(drvpriv, &ring->lock, _bh, NULL);
+}
+
+enum phl_mdl_ret_code _phl_custom_vr_proc_usr_tx_rpt(
+    void *dispr, void *custom_ctx, struct _custom_vr_ctx *vr_ctx,
+    struct rtw_custom_decrpt *cmd)
+{
+	enum phl_mdl_ret_code status = MDL_RET_SUCCESS;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	void *drvpriv = phl_to_drvpriv(phl);
+	struct _tx_stats_ring *ring = &vr_ctx->stats_rpt.t_stats_ring;
+	struct rtw_mac_usr_tx_rpt_info *c;
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	u8 i;
+
+	if (!vr_ctx->init.enable ||
+	    vr_ctx->stats_rpt.cfg.mode == USR_TX_STATS_RPT_DIS ||
+	    cmd->len != sizeof(struct rtw_mac_usr_tx_rpt_info)) {
+		PHL_INFO("%s, skip, vr_mode(%d) or cur_cfg_mode(%d) or "
+			 "inlen(%d)!=%d.\n",
+			 __FUNCTION__, vr_ctx->init.enable,
+			 vr_ctx->stats_rpt.cfg.mode, cmd->len,
+			 sizeof(struct rtw_mac_usr_tx_rpt_info));
+		status = MDL_RET_IGNORE;
+		goto exit;
+	}
+
+	/* prepare report */
+	c = (struct rtw_mac_usr_tx_rpt_info *)cmd->data;
+	if (!_prepare_frame_stats_rpt(phl, &vr_ctx->stats_rpt, c)) {
+		status = MDL_RET_IGNORE;
+		goto exit;
+	}
+
+	/* send report to core layer */
+	_os_spinlock(drvpriv, &ring->lock, _bh, NULL);
+	i = ring->r_idx;
+	while (i != ring->w_idx) {
+		if (!ring->entry[i].rpt_sent &&
+		    ring->entry[i].cus_rpt.rpt_mode &&
+		    ring->entry[i].total_pkt_cnt <= ring->entry[i].recv_rpt_cnt) {
+			PHL_INFO(" %s, send report, idx(%d)\n", __FUNCTION__, i);
+
+			_os_mem_set(drvpriv, &msg, 0, sizeof(struct phl_msg));
+			_os_mem_set(drvpriv, &attr, 0, sizeof(struct phl_msg_attribute));
+
+			SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_CUSTOM);
+			SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_STATS_RPT_NOTIFY);
+			msg.inbuf = (u8*)(&ring->entry[i].cus_rpt);
+			msg.inlen = sizeof(struct rtw_phl_vr_frame_stats_rpt);
+			msg.rsvd[0].value = vr_ctx->stats_rpt.cfg.macid;
+			attr.completion.completion = _usr_tx_rpt_notify_complete;
+			attr.completion.priv = (void*)vr_ctx;
+
+			if (phl_msg_hub_send(phl, &attr, &msg) != RTW_PHL_STATUS_SUCCESS) {
+				PHL_ERR("%s send msg fail\n", __func__);
+				break;
+			}
+			ring->entry[i].rpt_sent = true;
+		}
+		i = (i + 1) % MAX_TX_STATS_RING_ENTRY_NUM;
+	}
+	_os_spinunlock(drvpriv, &ring->lock, _bh, NULL);
+exit:
+	return status;
+}
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+
+enum phl_mdl_ret_code
+phl_custom_vr_feature_set_hdlr(void* dispr,
+                               void* custom_ctx,
+                               struct _custom_vr_ctx* vr_ctx,
+                               struct rtw_custom_decrpt *cmd)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+
+	switch (cmd->evt_id) {
+		case MSG_EVT_CUSTOME_SET_WIFI_ROLE:
+			ret = _phl_custom_vr_set_wifi_role(dispr, custom_ctx, vr_ctx, cmd);
+			break;
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+		case MSG_EVT_PROC_USR_TX_RPT:
+			ret = _phl_custom_vr_proc_usr_tx_rpt(dispr, custom_ctx, vr_ctx, cmd);
+			break;
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+		default:
+			ret = MDL_RET_SUCCESS;
+			break;
+	}
+	PHL_INFO("%s, evt(%d), ret(%d)\n", __FUNCTION__, cmd->evt_id, ret);
+	return ret;
+}
+
+enum phl_mdl_ret_code
+_phl_custom_vr_get_bss_color(void* dispr,
+                         void* custom_ctx,
+                         struct _custom_vr_ctx* vr_ctx,
+                         struct rtw_custom_decrpt *cmd)
+{
+	enum phl_mdl_ret_code status = MDL_RET_SUCCESS;
+	struct phl_info_t *phl = phl_custom_get_phl_info(custom_ctx);
+	void *d = phl_to_drvpriv(phl);
+
+	if (cmd->len == 0)
+		return MDL_RET_FAIL;
+	_os_mem_cpy(d, cmd->data, &vr_ctx->bsscolor_param, cmd->len);
+
+	return status;
+}
+
+enum phl_mdl_ret_code
+phl_custom_vr_feature_query_hdlr(void* dispr,
+                                 void* custom_ctx,
+                                 struct _custom_vr_ctx* vr_ctx,
+                                 struct rtw_custom_decrpt *cmd)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+
+	switch (cmd->evt_id) {
+	case MSG_EVT_GET_BSS_COLOR:
+		ret = _phl_custom_vr_get_bss_color(dispr, custom_ctx, vr_ctx,
+						      cmd);
+		break;
+	default:
+		ret = MDL_RET_SUCCESS;
+		break;
+	}
+
+	PHL_INFO("%s, evt(%d), ret(%d)\n", __FUNCTION__, cmd->evt_id, ret);
+	return ret;
+}
+
+enum rtw_phl_status phl_custom_vr_init_protocol_cap(
+    struct phl_info_t *phl_info, u8 hw_band, enum role_type rtype,
+    struct protocol_cap_t *protocol_cap)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_SUCCESS;
+	void *d = phl_to_drvpriv(phl_info);
+	struct phl_module_op_info op_info = {0};
+	struct _vr_bss_color color_info = {0};
+	struct rtw_custom_decrpt cmd = {0};
+
+	/* Bss color customization */
+	rtw_phl_custom_prepare_query_info_evt(MSG_EVT_GET_BSS_COLOR,
+					      CUS_ID_VR, &cmd, &op_info,
+					      sizeof(struct _vr_bss_color));
+
+	if (phl_disp_eng_query_bk_module_info(phl_info, hw_band, PHL_MDL_CUSTOM,
+					      &op_info) == MDL_RET_SUCCESS) {
+
+		_os_mem_cpy(d, &color_info, cmd.data, cmd.len);
+
+		/* get custom info by request */
+
+		/* assign bss color with valid range 1 to 63 */
+		if (color_info.is_valid == true)
+			protocol_cap->bsscolor = color_info.bsscolor;
+
+		PHL_INFO(
+		    "%s, cap_info bss color(%d), protocol_cap bss_color(%d)\n",
+		    __func__, color_info.bsscolor, protocol_cap->bsscolor);
+	} else {
+		status = RTW_PHL_STATUS_FAILURE;
+	}
+	return status;
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr.h b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr.h
new file mode 100644
index 000000000000..dfd8868d2dbe
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr.h
@@ -0,0 +1,148 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _PHL_CUSTOM_VR_H_
+#define _PHL_CUSTOM_VR_H_
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+#include "phl_custom_vr_csi.h"
+
+#define MAX_TX_STATS_RING_ENTRY_NUM 10
+
+struct _vr_init_param {
+	u32 enable;
+	u32 test_mode;
+	struct rtw_wifi_role_t *wifi_role;
+        struct phl_info_t *phl;
+};
+
+struct _vr_tx_rty_param {
+	u32 enable;
+	u8 rty_rate_tbl[24]; /* 1SS MCS0 to 2SS MCS11 */
+};
+
+struct _vr_rf_scrmb {
+	u8 is_set;
+	u8 status;
+	enum rtw_mac_usr_frame_to_act_mode mode;
+	u32 to_thr;
+	u8 trigger_cnt;
+	u16 sw_def_bmp; /* enum mac_ax_wd_sw_def_bmp */
+};
+
+struct _vr_bss_color {
+	u8 bsscolor;
+	bool is_valid;
+};
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+/**
+ * this struct is the stats rpt config param shared with sdk.
+ * @mode: enum _vr_stats_rpt_mode
+ * @macid: client's macid
+ * @period: only use if mode is FB_STATS_RPT_PERIOD
+ */
+struct _vr_stats_rpt_cfg {
+    u32 mode;
+    u32 macid;
+    u32 rpt_period_us;
+};
+
+struct _tx_stats_entry {
+    u32 total_pkt_cnt;
+    u32 recv_rpt_cnt;
+    struct rtw_phl_vr_frame_stats_rpt cus_rpt;
+    bool rpt_sent;
+};
+
+struct _tx_stats_ring {
+    struct _tx_stats_entry entry[MAX_TX_STATS_RING_ENTRY_NUM];
+    _os_lock lock;
+    u8 r_idx;
+    u8 w_idx;
+};
+
+struct _vr_stats_rpt_param {
+    struct _vr_stats_rpt_cfg cfg;
+    /* tx stats */
+    struct _tx_stats_ring t_stats_ring;
+    u32 tx_pending_byte[4];
+    u32 tx_pending_cnt[4];
+    /* for general rpt */
+    _os_timer general_rpt_timer;
+    u32 last_busy_cnt_rst_time;
+    u32 last_tx_busy_cnt;
+    u32 last_rx_busy_cnt;
+};
+
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+
+struct _custom_vr_ctx {
+	struct _vr_init_param init;
+	struct _vr_tx_rty_param tx_rty_param;
+	struct _vr_rf_scrmb cur_scrmb_param;
+	struct _vr_bss_color bsscolor_param;
+#ifdef CONFIG_PHL_CHANNEL_INFO_VR
+	struct custom_csi_ctrl csi_ctrl;
+#endif /* CONFIG_PHL_CHANNEL_INFO_VR */
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+	struct _vr_stats_rpt_param stats_rpt;
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+};
+
+enum phl_mdl_ret_code
+phl_custom_hdl_vr_evt(void* dispr,
+                      void* custom_ctx,
+                      struct _custom_vr_ctx* vr_ctx,
+                      struct phl_msg* msg);
+enum phl_mdl_ret_code
+phl_custom_hdl_vr_fail_evt(void* dispr,
+                           void* custom_ctx,
+                           struct _custom_vr_ctx* vr_ctx,
+                           struct phl_msg* msg);
+
+enum phl_mdl_ret_code
+phl_custom_vr_feature_set_hdlr(void* dispr,
+                               void* custom_ctx,
+                               struct _custom_vr_ctx* vr_ctx,
+                               struct rtw_custom_decrpt *cmd);
+
+enum phl_mdl_ret_code
+phl_custom_vr_feature_query_hdlr(void* dispr,
+                                 void* custom_ctx,
+                                 struct _custom_vr_ctx* vr_ctx,
+                                 struct rtw_custom_decrpt *cmd);
+
+enum phl_mdl_ret_code
+phl_custom_vr_set_role_cap(void* dispr,
+                           void* custom_ctx,
+                           struct _custom_vr_ctx* vr_ctx,
+                           struct phl_module_op_info* info);
+
+enum rtw_phl_status phl_custom_vr_init_protocol_cap(
+    struct phl_info_t *phl_info, u8 hw_band, enum role_type rtype,
+    struct protocol_cap_t *protocol_cap);
+
+#else
+#define phl_custom_hdl_vr_evt(_dispr, _custom_ctx, _vr_ctx, _msg) (MDL_RET_IGNORE)
+#define phl_custom_hdl_vr_fail_evt(_dispr, _custom_ctx, _vr_ctx, _msg) (MDL_RET_IGNORE)
+#define phl_custom_vr_feature_set_hdlr(_dispr, _custom_ctx, _vr_ctx, _cmd) (MDL_RET_IGNORE)
+#define phl_custom_vr_feature_query_hdlr(_dispr, _custom_ctx, _vr_ctx, _cmd) (MDL_RET_IGNORE)
+#define phl_custom_vr_init_protocol_cap(_phl_info, _hw_band, _rtype,           \
+					_protocol_cap)                         \
+	(RTW_PHL_STATUS_SUCCESS)
+#endif
+
+#endif  /*_PHL_CUSTOMIZE_FEATURE_H_*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr_csi.c b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr_csi.c
new file mode 100644
index 000000000000..4fd11fa4c466
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr_csi.c
@@ -0,0 +1,258 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _PHL_CUSTOMIZE_FEATURE_C_
+#include "../phl_headers.h"
+#include "../phl_api.h"
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+#ifdef CONFIG_PHL_CHANNEL_INFO_VR
+#include "phl_custom_vr_csi.h"
+
+enum csi_status {
+	CSI_STATE_INIT = BIT0,
+	CSI_STATE_RUNNING = BIT1,
+};
+
+void _phl_custom_csi_init(struct custom_csi_ctrl *csi_ctrl)
+{
+	/* param */
+	csi_ctrl->param.light_mode = 1;
+	csi_ctrl->param.bitmap_1x1 = 1;
+	csi_ctrl->param.group_num = 0;
+	csi_ctrl->param.ap_role = true;
+	csi_ctrl->param.fb_en = false;
+	csi_ctrl->param.trig_mode = CHINFO_MODE_ACK;
+	csi_ctrl->param.fb_mode = 0;
+	csi_ctrl->param.trig_intv = 0;
+	csi_ctrl->param.rslt_qry_en = true;
+	csi_ctrl->param.rslt_qry_mode = 0;
+	/* info */
+	csi_ctrl->info.sta = NULL;
+}
+
+struct rtw_phl_stainfo_t *
+_ap_get_first_client_sta(void *phl,
+                         struct rtw_wifi_role_link_t *rlink,
+                         struct rtw_phl_stainfo_t *self)
+{
+	struct phl_info_t *phl_info = (struct phl_info_t *)phl;
+	void *drv = phl_to_drvpriv(phl_info);
+	struct rtw_phl_stainfo_t *n, *psta, *ret = NULL;
+
+	if (rlink->assoc_sta_queue.cnt > 1) {
+		_os_spinlock(drv, &rlink->assoc_sta_queue.lock, _bh, NULL);
+		phl_list_for_loop_safe(psta, n, struct rtw_phl_stainfo_t,
+			&rlink->assoc_sta_queue.queue, list) {
+		if (_os_mem_cmp(drv, self->mac_addr,
+					psta->mac_addr, MAC_ALEN) == 0) {
+				continue;
+			} else {
+				ret = psta;
+				break;
+			}
+		}
+		_os_spinunlock(drv, &rlink->assoc_sta_queue.lock, _bh, NULL);
+	}
+	return ret;
+}
+
+static u8 _is_sta_ready(void* custom_ctx,
+                        struct rtw_wifi_role_link_t *rlink,
+                        struct rtw_chinfo_action_parm *act_parm)
+{
+	struct phl_info_t *phl_info = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+
+	if (rlink->mstate != MLME_LINKED) {
+		PHL_ERR("%s, Not connected!\n", __func__);
+		return 0;
+	}
+
+	phl_sta = rtw_phl_get_stainfo_self(phl_info, rlink);
+	if (phl_sta == NULL || phl_sta->active != true) {
+		PHL_ERR("%s, phl_sta not available, return!\n", __func__);
+		return 0;
+	} else {
+		PHL_INFO("%s, macid(%d), peer mac addr %02X-%02X-%02X-%02X-%02X-%02X\n",
+			__func__, phl_sta->macid,
+			phl_sta->mac_addr[0], phl_sta->mac_addr[1],
+			phl_sta->mac_addr[2], phl_sta->mac_addr[3],
+			phl_sta->mac_addr[4], phl_sta->mac_addr[5]);
+		act_parm->sta = phl_sta;
+	}
+	return 1;
+}
+
+static u8 _is_ap_ready(void* custom_ctx,
+                       struct custom_csi_ctrl *csi_ctrl,
+                       struct rtw_wifi_role_link_t *rlink,
+                       struct rtw_chinfo_action_parm *act_parm)
+{
+	struct phl_info_t *phl_info = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+
+	if (rlink->mstate != MLME_LINKED) {
+		PHL_ERR("%s, Not connected!\n", __func__);
+		return 0;
+	} else {
+		phl_sta = rtw_phl_get_stainfo_self(phl_info, rlink);
+		if (phl_sta == NULL || phl_sta->active != true) {
+			PHL_ERR("%s, phl_sta not available\n", __func__);
+			return 0;
+		} else {
+			struct rtw_phl_stainfo_t *psta = NULL;
+
+			PHL_INFO("%s, self macid(%d), mac addr %02X-%02X-%02X-%02X-%02X-%02X\n",
+				__func__, phl_sta->macid,
+				phl_sta->mac_addr[0], phl_sta->mac_addr[1],
+				phl_sta->mac_addr[2], phl_sta->mac_addr[3],
+				phl_sta->mac_addr[4], phl_sta->mac_addr[5]);
+			psta = _ap_get_first_client_sta(phl_info, rlink, phl_sta);
+			if (psta != NULL) {
+				PHL_INFO("%s, found assoc sta, macid(%d), mac addr %02X-%02X-%02X-%02X-%02X-%02X\n",
+					__func__, psta->macid,
+					psta->mac_addr[0], psta->mac_addr[1],
+					psta->mac_addr[2], psta->mac_addr[3],
+					psta->mac_addr[4], psta->mac_addr[5]);
+				act_parm->sta = psta;
+			} else {
+				PHL_ERR("%s, assoc client not found.\n", __func__);
+				return 0;
+			}
+		}
+	}
+
+	return 1;
+}
+
+enum phl_mdl_ret_code
+rtw_phl_custom_csi_start(void* custom_ctx,
+                         struct custom_csi_ctrl *csi_ctrl,
+                         struct rtw_wifi_role_link_t *rlink)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_SUCCESS;
+	struct phl_info_t *phl_info = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_chinfo_action_parm act_parm = {0};
+	u8 ready = 0;
+
+	PHL_INFO("%s, ==>\n", __func__);
+
+	SET_STATUS_FLAG(csi_ctrl->status, CSI_STATE_INIT);
+
+	_phl_custom_csi_init(csi_ctrl);
+
+	if (csi_ctrl->param.ap_role)
+		ready = _is_ap_ready(custom_ctx, csi_ctrl, rlink, &act_parm);
+	else
+		ready = _is_sta_ready(custom_ctx, rlink, &act_parm);
+
+	if (ready != 1) {
+		ret = MDL_RET_FAIL;
+		return ret;
+	}
+
+	csi_ctrl->info.sta = act_parm.sta;
+
+	act_parm.act = CHINFO_ACT_CFG;
+	act_parm.mode = csi_ctrl->param.trig_mode;
+	if (csi_ctrl->param.light_mode == 1)
+		act_parm.enable_mode = CHINFO_EN_LIGHT_MODE;
+	else
+		act_parm.enable_mode = CHINFO_EN_RICH_MODE;
+	act_parm.accuracy = CHINFO_ACCU_1BYTE;
+	act_parm.group_num = csi_ctrl->param.group_num;
+	act_parm.enable = true;
+	if (csi_ctrl->param.trig_intv > 0)
+		act_parm.trig_period = csi_ctrl->param.trig_intv;
+	else
+		act_parm.trig_period = 30;
+	if (csi_ctrl->param.bitmap_1x1 == 1)
+		act_parm.ele_bitmap = 0x0101;
+	else
+		act_parm.ele_bitmap = 0x0303;
+	rtw_phl_cmd_cfg_chinfo(phl_info, &act_parm, PHL_CMD_NO_WAIT, 0);
+	act_parm.act = CHINFO_ACT_EN;
+	rtw_phl_cmd_cfg_chinfo(phl_info, &act_parm, PHL_CMD_NO_WAIT, 0);
+
+	SET_STATUS_FLAG(csi_ctrl->status, CSI_STATE_RUNNING);
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+rtw_phl_custom_csi_stop(void* custom_ctx, struct custom_csi_ctrl *csi_ctrl)
+{
+	enum phl_mdl_ret_code ret = MDL_RET_SUCCESS;
+	struct phl_info_t *phl_info = phl_custom_get_phl_info(custom_ctx);
+	struct rtw_chinfo_action_parm act_parm = {0};
+
+	PHL_INFO("%s, ==>\n", __func__);
+
+	if (!TEST_STATUS_FLAG(csi_ctrl->status, CSI_STATE_RUNNING)) {
+		ret = MDL_RET_FAIL;
+		return ret;
+	}
+
+	CLEAR_STATUS_FLAG(csi_ctrl->status, CSI_STATE_RUNNING);
+
+	PHL_INFO("\n* Stop CSI for %s side.\n",
+		(csi_ctrl->param.ap_role ? "AP" : "CLIENT"));
+
+	act_parm.sta = csi_ctrl->info.sta;
+
+	act_parm.act = CHINFO_ACT_CFG;
+	act_parm.mode = csi_ctrl->param.trig_mode;
+	if (csi_ctrl->param.light_mode == 1)
+		act_parm.enable_mode = CHINFO_EN_LIGHT_MODE;
+	else
+		act_parm.enable_mode = CHINFO_EN_RICH_MODE;
+	act_parm.accuracy = CHINFO_ACCU_1BYTE;
+	act_parm.group_num = csi_ctrl->param.group_num;
+	act_parm.enable = false; /* set FALSE to disable CSI */
+	if (csi_ctrl->param.trig_intv > 0)
+		act_parm.trig_period = csi_ctrl->param.trig_intv;
+	else
+		act_parm.trig_period = 30;
+	if (csi_ctrl->param.bitmap_1x1 == 1)
+		act_parm.ele_bitmap = 0x0101;
+	else
+		act_parm.ele_bitmap = 0x0303;
+	rtw_phl_cmd_cfg_chinfo(phl_info, &act_parm, PHL_CMD_NO_WAIT, 0);
+	act_parm.act = CHINFO_ACT_EN;
+	rtw_phl_cmd_cfg_chinfo(phl_info, &act_parm, PHL_CMD_NO_WAIT, 0);
+
+	CLEAR_STATUS_FLAG(csi_ctrl->status, CSI_STATE_INIT);
+
+	return ret;
+}
+
+enum phl_mdl_ret_code
+rtw_phl_custom_csi_rslt_query(void* custom_ctx, u8 *buf, u32 *len)
+{
+	struct phl_info_t *phl_info = phl_custom_get_phl_info(custom_ctx);
+	struct csi_header_t csi_hdr = {0};
+	enum phl_mdl_ret_code ret = MDL_RET_FAIL;
+
+	if (RTW_PHL_STATUS_SUCCESS == rtw_phl_query_chan_info(phl_info,
+		MAX_DATA_SIZE, buf, len, &csi_hdr))
+		ret = MDL_RET_SUCCESS;
+	else
+		PHL_ERR("%s, query chan info fail\n", __func__);
+
+	return ret;
+}
+
+#endif /* CONFIG_PHL_CHANNEL_INFO_VR */
+#endif /* CONFIG_PHL_CUSTOM_FEATURE_VR */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr_csi.h b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr_csi.h
new file mode 100644
index 000000000000..5afc2de315ba
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/custom/phl_custom_vr_csi.h
@@ -0,0 +1,57 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _PHL_CUSTOM_CSI_H_
+#define _PHL_CUSTOM_CSI_H_
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_VR
+#ifdef CONFIG_PHL_CHANNEL_INFO_VR
+
+struct custom_csi_param {
+	u8 ap_role;
+	u8 trig_mode;
+	u16 trig_intv;
+	u8 fb_en;
+	u8 fb_mode;
+	u8 rslt_qry_en;
+	u8 rslt_qry_mode;
+	u8 light_mode;
+	u8 bitmap_1x1;
+	u8 group_num;
+};
+
+struct custom_csi_info {
+	struct rtw_phl_stainfo_t *sta;
+};
+
+struct custom_csi_ctrl {
+	enum csi_status status;
+	struct custom_csi_param param;
+	struct custom_csi_info info;
+};
+
+enum phl_mdl_ret_code
+rtw_phl_custom_csi_start(void* custom_ctx,
+                         struct custom_csi_ctrl *csi_ctrl,
+                         struct rtw_wifi_role_link_t *rlink);
+enum phl_mdl_ret_code
+rtw_phl_custom_csi_stop(void* custom_ctx, struct custom_csi_ctrl *csi_ctrl);
+
+enum phl_mdl_ret_code
+rtw_phl_custom_csi_rslt_query(void* custom_ctx, u8 *buf, u32 *len);
+
+#endif /* CONFIG_PHL_CHANNEL_INFO_VR */
+#endif /* CONFIG_PHL_CUSTOM_FEATURE_VR */
+
+#endif /* #ifndef _PHL_CUSTOM_CSI_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc.mk b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc.mk
new file mode 100755
index 000000000000..918b129f5112
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc.mk
@@ -0,0 +1,60 @@
+# All needed files would be added to _HAL_INTFS_FILES, and it would include
+# hal_g6/btc and all related files in directory hal_g6/btc/.
+# Before include this makefile, be sure interface (CONFIG_*_HCI) and IC
+# (CONFIG_RTL*) setting are all ready!
+
+#HAL = hal_g6
+
+ifeq ($(CONFIG_PHL_ARCH), y)
+phl_path := phl/$(HAL)
+phl_path_d1 := $(src)/phl/$(HAL)
+else
+phl_path := $(HAL)
+phl_path_d1 := $(src)/$(HAL)
+endif
+
+# Base directory
+path_halbtc_d1 := $(phl_path)/btc
+
+halbtc-y += $(path_halbtc_d1)/hal_btc.o \
+			$(path_halbtc_d1)/halbtc_def.o \
+			$(path_halbtc_d1)/halbtc_action.o \
+			$(path_halbtc_d1)/halbtc_fw.o \
+			$(path_halbtc_d1)/halbtc_dbg_cmd.o
+
+ifeq ($(CONFIG_RTL8852A), y)
+ic := 8852a
+# Level 2 directory
+path_halbtc_8852a := $(path_halbtc_d1)/btc_$(ic)
+
+halbtc-y += $(path_halbtc_8852a)/btc_8852a.o
+
+endif
+
+# 8852B/8852BP/8852BT/8852BPT Support
+ifneq ($(filter y,$(CONFIG_RTL8852B) $(CONFIG_RTL8852BP) $(CONFIG_RTL8851B) $(CONFIG_RTL8852BT) $(CONFIG_RTL8852BPT)),)
+ic := 8852b
+# Level 2 directory
+path_halbtc_8852b := $(path_halbtc_d1)/btc_$(ic)
+
+halbtc-y += $(path_halbtc_8852b)/btc_8852b.o
+endif
+
+# 8852C/8842A Support
+ifneq ($(filter y,$(CONFIG_RTL8852C) $(CONFIG_RTL8842A)),)
+ic := 8852c
+# Level 2 directory
+path_halbtc_8852c := $(path_halbtc_d1)/btc_$(ic)
+
+halbtc-y += $(path_halbtc_8852c)/btc_8852c.o
+endif
+
+ifeq ($(CONFIG_RTL8852D), y)
+ic := 8852d
+# Level 2 directory
+path_halbtc_8852d := $(path_halbtc_d1)/btc_$(ic)
+
+halbtc-y += $(path_halbtc_8852d)/btc_8852d.o
+endif
+
+_BTC_FILES +=	$(halbtc-y)
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc_8852b/btc_8852b.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc_8852b/btc_8852b.c
new file mode 100644
index 000000000000..72d2c34cda60
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc_8852b/btc_8852b.c
@@ -0,0 +1,890 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _RTL8852B_BTC_C_
+
+#include "../../hal_headers_le.h"
+#include "../hal_btc.h"
+
+#ifdef CONFIG_BTCOEX
+#if defined (CONFIG_RTL8852B) || defined (CONFIG_RTL8852BP) || defined (CONFIG_RTL8851B) || defined (CONFIG_RTL8852BT) || defined (CONFIG_RTL8852BPT)
+
+#include "btc_8852b.h"
+
+/* WL rssi threshold in % (dbm = % - 110)
+ * array size limit by BTC_WL_RSSI_THMAX
+ * BT rssi threshold in % (dbm = % - 100)
+ * array size limit by BTC_BT_RSSI_THMAX
+ */
+
+u8 btc_8852b_wl_rssi_thres[BTC_WL_RSSI_THMAX] = {70, 60, 50, 40};
+u8 btc_8852b_bt_rssi_thres[BTC_BT_RSSI_THMAX] = {50, 40, 30, 20};
+
+static const struct btc_rf_cfg btc_8852b_rf_0[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},  {0x3f, 0x17},
+	{0x33, 0x2},    {0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x17}, {0xef, 0x0}};
+static const struct btc_rf_cfg btc_8852b_rf_1[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},  {0x3f, 0x5},
+	{0x33, 0x2},    {0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x5},  {0xef, 0x0}};
+static const struct btc_rf_cfg btc_8852bp_rf_0[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},  {0x3f, 0x17},
+	{0x33, 0x2}, 	{0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x17}, {0xef, 0x0},
+	{0xef, 0x4000}, {0x33, 0x7},  {0x3e, 0x0},  {0x3f, 0x700},{0x33, 0x6},
+	{0x3e, 0x0},    {0x3f, 0x700},{0x33, 0xf},  {0x3e, 0x0},  {0x3f, 0x700},
+	{0x33, 0xe},    {0x3e, 0x0},  {0x3f, 0x700},{0x33, 0x17}, {0x3e, 0x0},
+	{0x3f, 0x700}, 	{0x33, 0x16}, {0x3e, 0x0},  {0x3f, 0x700},{0xef, 0x0},
+	{0xee, 0x10}, 	{0x33, 0x0},  {0x3f, 0x3},  {0x33, 0x1},  {0x3f, 0x3},
+	{0x33, 0x2}, 	{0x3f, 0x3},  {0xee, 0x0}};
+static const struct btc_rf_cfg btc_8852bp_rf_1[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},    {0x3f, 0x5},
+	{0x33, 0x2}, 	{0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x5},    {0xef, 0x0},
+	{0xef, 0x4000}, {0x33, 0x7},  {0x3e, 0x0},  {0x3f, 0xa0700},{0x33, 0x6},
+	{0x3e, 0x0}, {0x3f, 0xa0700}, {0x33, 0xf},  {0x3e, 0x0},{0x3f, 0xa0700},
+	{0x33, 0xe},    {0x3e, 0x0},  {0x3f, 0xa0700},{0x33, 0x17}, {0x3e, 0x0},
+	{0x3f, 0xa0700},{0x33, 0x16}, {0x3e, 0x0},  {0x3f, 0xa0700},{0xef, 0x0},
+	{0xee, 0x10}, 	{0x33, 0x0},  {0x3f, 0x2},  {0x33, 0x1},    {0x3f, 0x2},
+	{0x33, 0x2}, 	{0x3f, 0x2},  {0xee, 0x0}};
+static const struct btc_rf_cfg btc_8851b_rf_0[] = {{0x2, 0x0}};
+static const struct btc_rf_cfg btc_8851b_rf_1[] = {{0x2, 0x1}};
+static const struct btc_rf_cfg btc_8852bt_rf_0[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},  {0x3f, 0x17},
+	{0x33, 0x2},    {0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x17}, {0xef, 0x0}};
+static const struct btc_rf_cfg btc_8852bt_rf_1[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},  {0x3f, 0x5},
+	{0x33, 0x2},    {0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x5},  {0xef, 0x0}};
+static const struct btc_rf_cfg btc_8852bpt_rf_0[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},  {0x3f, 0x17},
+	{0x33, 0x2},	{0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x17}, {0xef, 0x0},
+	{0xef, 0x4000}, {0x33, 0x7},  {0x3e, 0x0},  {0x3f, 0x700},{0x33, 0x6},
+	{0x3e, 0x0},	{0x3f, 0x700},{0x33, 0xf},  {0x3e, 0x0},  {0x3f, 0x700},
+	{0x33, 0xe},	{0x3e, 0x0},  {0x3f, 0x700},{0x33, 0x17}, {0x3e, 0x0},
+	{0x3f, 0x700},	{0x33, 0x16}, {0x3e, 0x0},  {0x3f, 0x700},{0xef, 0x0},
+	{0xee, 0x10},	{0x33, 0x0},  {0x3f, 0x3},  {0x33, 0x1},  {0x3f, 0x3},
+	{0x33, 0x2},	{0x3f, 0x3},  {0xee, 0x0}};
+static const struct btc_rf_cfg btc_8852bpt_rf_1[] = {
+	{0xef, 0x1000}, {0x33, 0x0},  {0x3f, 0x15}, {0x33, 0x1},    {0x3f, 0x5},
+	{0x33, 0x2},	{0x3f, 0x15}, {0x33, 0x3},  {0x3f, 0x5},    {0xef, 0x0},
+	{0xef, 0x4000}, {0x33, 0x7},  {0x3e, 0x0},  {0x3f, 0xa0700},{0x33, 0x6},
+	{0x3e, 0x0}, {0x3f, 0xa0700}, {0x33, 0xf},  {0x3e, 0x0},{0x3f, 0xa0700},
+	{0x33, 0xe},	{0x3e, 0x0},  {0x3f, 0xa0700},{0x33, 0x17}, {0x3e, 0x0},
+	{0x3f, 0xa0700},{0x33, 0x16}, {0x3e, 0x0},  {0x3f, 0xa0700},{0xef, 0x0},
+	{0xee, 0x10},	{0x33, 0x0},  {0x3f, 0x2},  {0x33, 0x1},    {0x3f, 0x2},
+	{0x33, 0x2},	{0x3f, 0x2},  {0xee, 0x0}};
+
+static struct btc_chip_ops btc_8852b_ops = {
+	_8852b_rfe_type,
+	_8852b_init_cfg,
+	_8852b_wl_tx_power,
+	_8852b_wl_rx_gain,
+	_8852b_wl_btg_standby,
+	_8852b_wl_req_mac,
+	_8852b_get_reg_status,
+	_8852b_bt_rssi
+};
+
+/* Set  WL/BT periodical moniter reg, Max size: CXMREG_MAX*/
+/*
+ * =========== CO-Rx AGC related monitor ===========
+ * Co-Rx condition : WL RSSI < -35dBm (BB defined)
+ * If WL RSSI < -35dBm --> 0x4aa4[18] = 0;
+ * 0x4aa4[18] = 0: Not to adjust WL Rx AGC by GNT_BT_Rx.
+ *		    To avoid WL Rx gain change.
+ * 0x4aa4[18] = 1: When WL RSSI too strong, out of Co-Rx range
+ *		    give up the Co-Rx WL enhancement.
+ * 0x476c[31:24] = 0x20 (dBm); LNA6 op1dB
+ * 0x4778[7:0]= 0x30 (dBm); TIA index0 LNA6 op1dB
+ */
+static struct fbtc_mreg btc_8852b_mon_reg[] = {
+	{REG_MAC, 4, 0xda24},
+	{REG_MAC, 4, 0xda28},
+	{REG_MAC, 4, 0xda2c},
+	{REG_MAC, 4, 0xda30},
+	{REG_MAC, 4, 0xda4c},
+	{REG_MAC, 4, 0xda10},
+	{REG_MAC, 4, 0xda20},
+	{REG_MAC, 4, 0xda34},
+	{REG_MAC, 4, 0xcef4},
+	{REG_MAC, 4, 0x8424},
+	{REG_MAC, 4, 0xd200},
+	{REG_MAC, 4, 0xd220},
+	{REG_BB, 4, 0x980},
+	{REG_BB, 4, 0x4aa4},
+	{REG_BB, 4, 0x4778},
+	{REG_BB, 4, 0x476c},
+	/*{REG_RF, 2, 0x2},*/ /* for RF, bytes->path, 1:A, 2:B... */
+	/*{REG_RF, 2, 0x18},*/
+	/*{REG_BT_RF, 2, 0x9},*/
+	/*{REG_BT_MODEM, 2, 0xa} */
+};
+static struct fbtc_mreg btc_8851b_mon_reg[] = {
+	{REG_MAC, 4, 0xda24},
+	{REG_MAC, 4, 0xda28},
+	{REG_MAC, 4, 0xda2c},
+	{REG_MAC, 4, 0xda30},
+	{REG_MAC, 4, 0xda4c},
+	{REG_MAC, 4, 0xda10},
+	{REG_MAC, 4, 0xda20},
+	{REG_MAC, 4, 0xda34},
+	{REG_MAC, 4, 0xcef4},
+	{REG_MAC, 4, 0x8424},
+	{REG_MAC, 4, 0xd200},
+	{REG_MAC, 4, 0xd220},
+	{REG_BB, 4, 0x980},
+	{REG_BB, 4, 0x4738}, /* path0 reg*/
+	{REG_BB, 4, 0x4688}, /* path0 reg*/
+	{REG_BB, 4, 0x4694}, /* path0 reg*/
+	/*{REG_RF, 2, 0x2},*/ /* for RF, bytes->path, 1:A, 2:B... */
+	/*{REG_RF, 2, 0x18},*/
+	/*{REG_BT_RF, 2, 0x9},*/
+	/*{REG_BT_MODEM, 2, 0xa} */
+};
+
+
+/* wl_tx_power: 255->original or BTC_WL_DEF_TX_PWR
+ *              else-> bit7:signed bit, ex: 13:13dBm, 0x85:-5dBm
+ * wl_rx_gain: 0->original, 1-> for Free-run, 2-> for BTG co-rx
+ * bt_tx_power: decrease power, 0->original, 5 -> decreas 5dB.
+ * bt_rx_gain: BT LNA constrain Level, 7->original
+ */
+
+struct btc_rf_trx_para btc_8852b_rf_ul[] = {
+	{15, 0, 0, 7}, /* 0 -> original */
+	{15, 2, 0, 7}, /* 1 -> for BT-connected ACI issue && BTG co-rx */
+	{15, 0, 0, 7}, /* 2 ->reserved for shared-antenna */
+	{15, 0, 0, 7}, /* 3- >reserved for shared-antenna */
+	{15, 0, 0, 7}, /* 4 ->reserved for shared-antenna */
+	{15, 1, 0, 7}, /* the below id is for non-shared-antenna free-run */
+	{ 6, 1, 0, 7},
+	{13, 1, 0, 7},
+	{13, 1, 0, 7}
+};
+
+struct btc_rf_trx_para btc_8852b_rf_dl[] = {
+	{15, 0, 0, 7}, /* 0 -> original */
+	{15, 2, 0, 7}, /* 1 -> reserved for shared-antenna */
+	{15, 0, 0, 7}, /* 2 ->reserved for shared-antenna */
+	{15, 0, 0, 7}, /* 3- >reserved for shared-antenna */
+	{15, 0, 0, 7}, /* 4 ->reserved for shared-antenna */
+	{15, 1, 0, 7}, /* the below id is for non-shared-antenna free-run */
+	{15, 1, 0, 7},
+	{15, 1, 0, 7},
+	{15, 1, 31, 7} /* For Amazon BIS */
+};
+
+const struct btc_chip chip_8852bpt = {
+	0x8852B, /* chip id */
+	0x0, /* chip HW feature/parameter, refer to enum btc_chip_feature */
+	0x7, /* desired bt_ver */
+	0x070e0000, /* desired wl_fw btc ver  */
+	0x1, /* scoreboard version */
+	0x1, /* mailbox version*/
+	BTC_COEX_RTK_MODE, /* pta_mode */
+	BTC_COEX_INNER, /* pta_direction */
+	6, /* afh_guard_ch */
+	btc_8852b_wl_rssi_thres, /* wl rssi threshold level */
+	btc_8852b_bt_rssi_thres, /* bt rssi threshold level */
+	(u8)2, /* rssi tolerance */
+	&btc_8852b_ops, /* chip-dependent function */
+	ARRAY_SIZE(btc_8852b_mon_reg),
+	btc_8852b_mon_reg, /* wl moniter register */
+	ARRAY_SIZE(btc_8852b_rf_ul),
+	btc_8852b_rf_ul,
+	ARRAY_SIZE(btc_8852b_rf_dl),
+	btc_8852b_rf_dl
+};
+
+const struct btc_chip chip_8852bt = {
+	0x8852B, /* chip id */
+	0x0, /* chip HW feature/parameter, refer to enum btc_chip_feature */
+	0x7, /* desired bt_ver */
+	0x07100000, /* desired wl_fw btc ver  */
+	0x1, /* scoreboard version */
+	0x1, /* mailbox version*/
+	BTC_COEX_RTK_MODE, /* pta_mode */
+	BTC_COEX_INNER, /* pta_direction */
+	6, /* afh_guard_ch */
+	btc_8852b_wl_rssi_thres, /* wl rssi threshold level */
+	btc_8852b_bt_rssi_thres, /* bt rssi threshold level */
+	(u8)2, /* rssi tolerance */
+	&btc_8852b_ops, /* chip-dependent function */
+	ARRAY_SIZE(btc_8852b_mon_reg),
+	btc_8852b_mon_reg, /* wl moniter register */
+	ARRAY_SIZE(btc_8852b_rf_ul),
+	btc_8852b_rf_ul,
+	ARRAY_SIZE(btc_8852b_rf_dl),
+	btc_8852b_rf_dl
+};
+
+const struct btc_chip chip_8852bp = {
+	0x8852B, /* chip id */
+	0x0, /* chip HW feature/parameter, refer to enum btc_chip_feature */
+	0x7, /* desired bt_ver */
+	0x070e0000, /* desired wl_fw btc ver  */
+	0x1, /* scoreboard version */
+	0x1, /* mailbox version*/
+	BTC_COEX_RTK_MODE, /* pta_mode */
+	BTC_COEX_INNER, /* pta_direction */
+	6, /* afh_guard_ch */
+	btc_8852b_wl_rssi_thres, /* wl rssi threshold level */
+	btc_8852b_bt_rssi_thres, /* bt rssi threshold level */
+	(u8)2, /* rssi tolerance */
+	&btc_8852b_ops, /* chip-dependent function */
+	ARRAY_SIZE(btc_8852b_mon_reg),
+	btc_8852b_mon_reg, /* wl moniter register */
+	ARRAY_SIZE(btc_8852b_rf_ul),
+	btc_8852b_rf_ul,
+	ARRAY_SIZE(btc_8852b_rf_dl),
+	btc_8852b_rf_dl
+};
+
+const struct btc_chip chip_8851b = {
+	0x8851B, /* chip id */
+	0x0, /* chip HW feature/parameter, refer to enum btc_chip_feature */
+	0x7, /* desired bt_ver */
+	0x07100000, /* desired wl_fw btc ver  */
+	0x1, /* scoreboard version */
+	0x1, /* mailbox version*/
+	BTC_COEX_RTK_MODE, /* pta_mode */
+	BTC_COEX_INNER, /* pta_direction */
+	6, /* afh_guard_ch */
+	btc_8852b_wl_rssi_thres, /* wl rssi threshold level */
+	btc_8852b_bt_rssi_thres, /* bt rssi threshold level */
+	(u8)2, /* rssi tolerance */
+	&btc_8852b_ops, /* chip-dependent function */
+	ARRAY_SIZE(btc_8851b_mon_reg),
+	btc_8851b_mon_reg, /* wl moniter register */
+	ARRAY_SIZE(btc_8852b_rf_ul),
+	btc_8852b_rf_ul,
+	ARRAY_SIZE(btc_8852b_rf_dl),
+	btc_8852b_rf_dl
+};
+
+const struct btc_chip chip_8852b = {
+	0x8852B, /* chip id */
+	0x0, /* chip HW feature/parameter, refer to enum btc_chip_feature */
+	0x7, /* desired bt_ver */
+	0x070f0000, /* desired wl_fw btc ver  */
+	0x1, /* scoreboard version */
+	0x1, /* mailbox version*/
+	BTC_COEX_RTK_MODE, /* pta_mode */
+	BTC_COEX_INNER, /* pta_direction */
+	6, /* afh_guard_ch */
+	btc_8852b_wl_rssi_thres, /* wl rssi threshold level */
+	btc_8852b_bt_rssi_thres, /* bt rssi threshold level */
+	(u8)2, /* rssi tolerance */
+	&btc_8852b_ops, /* chip-dependent function */
+	ARRAY_SIZE(btc_8852b_mon_reg),
+	btc_8852b_mon_reg, /* wl moniter register */
+	ARRAY_SIZE(btc_8852b_rf_ul),
+	btc_8852b_rf_ul,
+	ARRAY_SIZE(btc_8852b_rf_dl),
+	btc_8852b_rf_dl
+};
+
+u8 _8852b_get_rf_path(struct btc_t *btc, u8 antnum)
+{
+	u8 ret;
+
+	switch (antnum) {
+		default:
+		case 1:
+			ret = btc->mdinfo.ant.single_pos;
+			break;
+		case 2:
+			ret = RF_PATH_AB;
+			break;
+		case 3:
+			ret = RF_PATH_ABC;
+			break;
+	}
+	return ret;
+}
+
+void _8852b_rfe_type(struct btc_t *btc)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct rtw_phl_com_t *p = btc->phl;
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_module *module = &btc->mdinfo;
+	u8 tx_path_pos, rx_path_pos;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !! \n", __FUNCTION__);
+
+	/* get from final capability of device  */
+	module->rfe_type = p->dev_cap.rfe_type;
+	module->kt_ver = h->cv;
+	module->kt_ver_adie = h->acv;
+	module->bt_pos = BTC_BT_BTG;
+	module->switch_type = BTC_SWITCH_INTERNAL;
+	module->wa_type = 0;
+	dm->wl_trx_nss_en = 0;
+	module->wa_type |= BTC_WA_CO_RX;
+
+	module->ant.type = BTC_ANT_SHARED;
+	module->ant.num = 2;
+	if (btc->dm.ntfy_type != BTC_NTFY_CUSTOMERIZE)
+		module->ant.isolation = BTC_DEFAULT_ANISO;
+	module->ant.diversity = 0;
+	/* WL 1-stream+1-Ant is located at 0:s0(path-A) or 1:s1(path-B) */
+	module->ant.single_pos = RF_PATH_B;
+	module->ant.btg_pos = RF_PATH_B;
+	module->ant.stream_cnt = (p->phy_cap[0].txss << 4) + p->phy_cap[0].rxss;
+
+	if (module->rfe_type == 0) {
+		dm->error |= BTC_DMERR_RFE_TYPE0;
+		return;
+	}
+
+	cx->other.type = BTC_EXTSOC_FUNC_NONE;
+	cx->other.hw_coex = BTC_EXTSOC_INTF_NONE;
+	cx->other.ext_pta_type = BTC_COEX_RTK_MODE;
+	cx->other.ext_hpta_cfg = 0;
+	cx->other.ext_hmbx_cfg = 0;
+	cx->other.ext_swout_cfg = 0;
+	cx->other.ext_swin_cfg = 0;
+
+	switch (btc->hal->chip_id) {
+	case CHIP_WIFI6_8851B: /* 51B */
+		/* rfe_type 3*n+1: 1-Ant(shared),
+		 *          3*n+2: 2-Ant+Div(non-shared),
+		 *          3*n+3: 2-Ant+no-Div(non-shared)
+		 */
+		module->ant.num = (module->rfe_type % 3 == 1)? 1 : 2;
+		/* WL-1ss at S0, btg at s0 (On 1 WL RF) */
+		module->ant.single_pos = RF_PATH_A;
+		module->ant.btg_pos = RF_PATH_A;
+		module->wa_type |= BTC_WA_INIT_SCAN;
+
+		if (module->ant.num == 1) {
+			module->ant.type = BTC_ANT_SHARED;
+			module->bt_pos = BTC_BT_BTG;
+			module->wa_type |= BTC_WA_5G_HI_CH_RX;
+		} else { /* ant.num == 2 */
+			module->ant.type = BTC_ANT_DEDICATED;
+			module->bt_pos = BTC_BT_ALONE;
+			module->switch_type = BTC_SWITCH_EXTERNAL;
+			if (module->rfe_type % 3 == 2)
+				module->ant.diversity = 1;
+		}
+		break;
+	case CHIP_WIFI6_8852BT: /*52BT*/
+		module->ant.num = (module->rfe_type % 2)? 2 : 3;
+		module->wa_type |= BTC_WA_INIT_SCAN;
+
+		if (module->ant.num == 2) {
+			module->ant.type = BTC_ANT_SHARED;
+			module->bt_pos = BTC_BT_BTG;
+#ifdef PHL_PLATFORM_LINUX
+			module->wa_type |= BTC_WA_HFP_LAG;
+#endif
+			dm->wl_trx_nss_en = 1;
+		} else { /* ant.num == 3 */
+			module->ant.type = BTC_ANT_DEDICATED;
+			module->bt_pos = BTC_BT_ALONE;
+		}
+		break;
+	default: /* 52B, 52BP, 52BPT*/
+		/*rfe_type odd: 2-Ant(shared), even: 3-Ant(non-shared)*/
+		module->ant.num = (module->rfe_type % 2)? 2 : 3;
+
+		if (module->ant.num == 2) {
+			module->ant.type = BTC_ANT_SHARED;
+			module->bt_pos = BTC_BT_BTG;
+			if (p->phy_cap[0].txss == 1 &&
+			    p->phy_cap[0].rxss == 1) { /* 52B-VS */
+				module->ant.num = 1;
+				/* WL-1ss at S1, btg at s1*/
+				module->ant.single_pos = RF_PATH_B;
+				module->ant.btg_pos = RF_PATH_B;
+			} else {
+				dm->wl_trx_nss_en = 1;
+			}
+		} else { /* ant.num == 3 */
+			module->ant.type = BTC_ANT_DEDICATED;
+			module->bt_pos = BTC_BT_ALONE;
+		}
+		break;
+	}
+
+	dm->wl_trx_nss_en &= !!(GET_DEV_BTC_CAP(p).btc_deg_wifi_cap & BTC_DRG_WIFI_CAP_TRX1SS);
+	tx_path_pos = _8852b_get_rf_path(btc, btc->phl->phy_cap[0].tx_path_num);
+	rx_path_pos = _8852b_get_rf_path(btc, btc->phl->phy_cap[0].rx_path_num);
+	module->ant.path_pos = (tx_path_pos << 4) + rx_path_pos;
+	dm->wl_trx_nss.tx_path = tx_path_pos;
+	dm->wl_trx_nss.rx_path = rx_path_pos;
+#if 0
+	/* module->bt_solo info should be notified from driver */
+	module->bt_solo = BTC_EXT_SOC_8771;
+#endif
+	switch(module->bt_solo) {
+	default:
+		break;
+	case BTC_EXT_SOC_8771:
+		cx->other.type = BTC_EXTSOC_FUNC_THREAD;
+		cx->other.hw_coex = BTC_EXTSOC_INTF_SWIO;
+		/* input: ZB_Act to WL, GPIO5 as input + interrupt src*/
+		cx->other.ext_swin_cfg = BIT(6) | (BIT(6) << 16);
+		/* output: WL_Act to ZB */
+		cx->other.ext_swout_cfg = BIT(12);
+		break;
+	}
+}
+
+void _8852b_get_reg_status(struct btc_t *btc, u8 type, void *status)
+{
+	struct btc_module *md = &btc->mdinfo;
+	struct fbtc_mreg_val *pmreg = NULL;
+	u8 *val8 = NULL;
+	u32 *val32 = NULL;
+	u8 i = 0;
+	u32 reg_val;
+	u32 pre_agc_addr = R_BTC_BB_PRE_AGC_S1;
+
+	if (md->ant.btg_pos == RF_PATH_A)
+		pre_agc_addr = R_BTC_BB_PRE_AGC_S0;
+
+	switch (type) {
+	case BTC_CSTATUS_TXDIV_POS:
+		val8 = status;
+		if (md->switch_type == BTC_SWITCH_INTERNAL) {
+			*val8 = BTC_ANT_DIV_MAIN;
+			break;
+		}
+
+		break;
+	case BTC_CSTATUS_RXDIV_POS:
+		val8 = status;
+		if (md->switch_type == BTC_SWITCH_INTERNAL) {
+			*val8 = BTC_ANT_DIV_MAIN;
+			break;
+		}
+
+		break;
+	case BTC_CSTATUS_BB_GNT_MUX:
+		val32 = status;
+		reg_val = hal_read32(btc->hal, R_BTC_BB_BTG_RX | 0x10000);
+		*val32 = (reg_val & B_BTC_BB_GNT_MUX) == 0? 1 : 0;
+		break;
+	case BTC_CSTATUS_BB_GNT_MUX_MON:
+		val32 = status;
+		if (!btc->fwinfo.rpt_fbtc_mregval.cinfo.valid)
+			return;
+		pmreg = &btc->fwinfo.rpt_fbtc_mregval.finfo;
+		/*  Check BB reg 0x10980 setup from period reg-mon*/
+		for (i = 0; i < pmreg->reg_num; i++) {
+			if (btc->chip->mon_reg[i].type == REG_BB &&
+			    btc->chip->mon_reg[i].offset == R_BTC_BB_BTG_RX) {
+				reg_val = pmreg->mreg_val[i];
+			     	*val32 = (reg_val & B_BTC_BB_GNT_MUX) == 0? 1 : 0;
+				break;
+			} else if (i == pmreg->reg_num - 1) {
+				*val32 = BTC_BB_GNT_NOTFOUND;
+				return;
+			}
+		}
+		break;
+	case BTC_CSTATUS_BB_PRE_AGC:
+		val8 = status;
+		reg_val = hal_read32(btc->hal, pre_agc_addr | 0x10000);
+		reg_val &= B_BTC_BB_PRE_AGC_MASK;
+		*val8 = (reg_val == B_BTC_BB_PRE_AGC_VAL)? 1 : 0;
+		break;
+	case BTC_CSTATUS_BB_PRE_AGC_MON:
+		val32 = status;
+		if (!btc->fwinfo.rpt_fbtc_mregval.cinfo.valid)
+			return;
+		pmreg = &btc->fwinfo.rpt_fbtc_mregval.finfo;
+		/*  Check BB reg Pre-AGC setup from period reg-mon*/
+		for (i = 0; i < pmreg->reg_num; i++) {
+			if (btc->chip->mon_reg[i].type == REG_BB &&
+			    btc->chip->mon_reg[i].offset == pre_agc_addr) {
+				break; /* found */
+			} else if (i == pmreg->reg_num - 1) {
+				*val32 = BTC_BB_PRE_AGC_NOTFOUND;
+				return;
+			}
+		}
+
+		reg_val = pmreg->mreg_val[i] & B_BTC_BB_PRE_AGC_MASK;
+		*val32 = (reg_val == B_BTC_BB_PRE_AGC_VAL)? 1 : 0;
+		break;
+	}
+}
+
+void _8852b_wl_tx_power(struct btc_t *btc, u32 level)
+{
+	/*
+	 * =========== All-Time WL Tx power control ==========
+	 * (ex: all-time fix WL Tx 10dBm , don't care GNT _BT and GNT _LTE)
+	 * Turn off per-packet power control
+	 * 0xD220[1] = 0, 0xD220[2] = 0;
+	 *
+	 * disable using related power table
+	 * 0xd208[20] = 0, 0xd208[21] = 0, 0xd21c[17] = 0, 0xd20c[29] = 0;
+	 *
+	 * enable force tx power mode and value
+	 * 0xD200[9] = 1;
+	 * 0xD200[8:0] = 0x28; S(9,2): 1step= 0.25dB, i.e. 40*0.25 = 10 dBm
+	 * =========== per-packet Tx power control ===========
+	 * (ex: GNT_BT = 1 -> 5dBm,  GNT _BT = 0 -> 10dBm)
+	 * Turn on per-packet power control
+	 * 0xD220[1] = 1, 0xD220[2] = 0;
+	 * 0xD220[11:3] = 0x14; S(9,2): 1step = 0.25dB, i.e. 20*0.25 = 5 dBm
+	 *
+	 * disable using related power table
+	 * 0xd208[20] = 0, 0xd208[21] = 0, 0xd21c[17] = 0, 0xd20c[29] = 0;
+	 *
+	 * enable force tx power mode and value
+	 * 0xD200[9] = 1;
+	 * 0xD200[8:0] = 0x28; S(9,2): 1 step = 0.25dB, i.e. 40*0.25 = 10 dBm
+	 * ===================================================================
+	 * level define:
+	 *    if level = 255 -> back to original (btc don't control)
+	 *    else in dBm --> bit7->signed bit, ex: 0xa-> +10dBm, 0x85-> -5dBm
+	 * pwr_val define:
+	 *     bit15~0  --> All-time (GNT_BT = 0) Tx power control
+	 *     bit31~16 --> Per-Packet (GNT_BT = 1) Tx power control
+	 */
+	u32 pwr_val = bMASKDW, phi_idx = HW_PHY_0;
+	bool en = false;
+
+	if ((level & 0x7f) < BTC_WL_DEF_TX_PWR) { /* back to original */
+		pwr_val = (level & 0x3f) << 2; /* to fit s(9,2) format */
+
+		if (level & BIT(7)) /* negative value */
+			pwr_val |= BIT(8); /* to fit s(9,2) format */
+		pwr_val |= bMASKHW;
+		en = true;
+	}
+
+	if (btc->hal->dbcc_en)
+		phi_idx = btc->cx.wl.pta_req_mac;
+
+	rtw_hal_rf_wlan_tx_power_control(btc->hal, phi_idx, ALL_TIME_CTRL,
+					 pwr_val, en);
+}
+
+void _8852b_wl_rx_gain(struct btc_t *btc, u32 level)
+{
+	/* To improve BT ACI in co-rx
+	 * level=0 Default: TIA 1/0= (LNA2,TIAN6) = (7,1)/(5,1) = 21dB/12dB
+	 * level=1 Fix LNA2=5: TIA 1/0= (LNA2,TIAN6) = (5,0)/(5,1) = 18dB/12dB
+	 */
+	u32 mask = bMASKRF, n = 0, i = 0, val = 0;
+	u32 type = (btc->mdinfo.ant.btg_pos << 8) | RTW_MAC_RF_CMD_OFLD;
+	const struct btc_rf_cfg *rf = NULL;
+	bool en;
+
+	switch (level) {
+	case 0: /* original */
+	default:
+		btc->dm.wl_lna2 = 0;
+		break;
+	case 1: /* for FDD free-run */
+		btc->dm.wl_lna2 = 0;
+		break;
+	case 2: /* for BTG Co-Rx*/
+		btc->dm.wl_lna2 = 1;
+		break;
+	}
+
+	switch(btc->hal->chip_id) {
+	case CHIP_WIFI6_8852B:
+	default:
+		mask = bMASKRF;
+		if (btc->dm.wl_lna2 == 0) {
+			rf = btc_8852b_rf_0;
+			n = ARRAY_SIZE(btc_8852b_rf_0);
+		} else {
+			rf = btc_8852b_rf_1;
+			n = ARRAY_SIZE(btc_8852b_rf_1);
+		}
+		break;
+	case CHIP_WIFI6_8852BP:
+		mask = bMASKRF;
+		if (btc->dm.wl_lna2 == 0) {
+			rf = btc_8852bp_rf_0;
+			n = ARRAY_SIZE(btc_8852bp_rf_0);
+		} else {
+			rf = btc_8852bp_rf_1;
+			n = ARRAY_SIZE(btc_8852bp_rf_1);
+		}
+		break;;
+	case CHIP_WIFI6_8851B:
+		mask = 0x700; /* bit8~10 */
+		if (btc->dm.wl_lna2 == 0) {
+			rf = btc_8851b_rf_0;
+			n = ARRAY_SIZE(btc_8851b_rf_0);
+		} else {
+			rf = btc_8851b_rf_1;
+			n = ARRAY_SIZE(btc_8851b_rf_1);
+		}
+		break;
+	case CHIP_WIFI6_8852BT:
+		mask = bMASKRF;
+		if (btc->dm.wl_lna2 == 0) {
+			rf = btc_8852bt_rf_0;
+			n = ARRAY_SIZE(btc_8852bt_rf_0);
+		} else {
+			rf = btc_8852bt_rf_1;
+			n = ARRAY_SIZE(btc_8852bt_rf_1);
+		}
+		break;
+	case CHIP_WIFI6_8852BPT:
+		mask = bMASKRF;
+		if (btc->dm.wl_lna2 == 0) {
+			rf = btc_8852bpt_rf_0;
+			n = ARRAY_SIZE(btc_8852bpt_rf_0);
+		} else {
+			rf = btc_8852bpt_rf_1;
+			n = ARRAY_SIZE(btc_8852bpt_rf_1);
+		}
+		break;
+	}
+
+	while (1) {
+		en = (i == n-1) ? true : false;
+
+		val = rf->val;
+		/* bit[10] = 1 if non-shared-ant for 8851b */
+		if (btc->hal->chip_id == CHIP_WIFI6_8851B) {
+			if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED) {
+				//val |= 0x4;
+				rtw_hal_bb_npath_en_update(btc->hal, false);
+				/*set npath off until low RSSI TP degradation is solved*/
+			} else {
+				rtw_hal_bb_npath_en_update(btc->hal, false);
+			}
+		}
+
+		_btc_io_w(btc, type, rf->addr, mask, val, en);
+		i++;
+		if (i > n-1)
+			break;
+		rf++;
+	}
+}
+
+u8 _8852b_bt_rssi(struct btc_t *btc, u8 val)
+{
+	val = (val <= 127? 100 : (val >= 156? val - 156 : 0));
+	val = val + 6; /* compensate offset */
+
+	if (val > 100)
+		val = 100;
+
+	return (val);
+}
+
+void _8852b_wl_trx_mask(struct btc_t *btc, u32 type, u8 group, u32 val)
+{
+	if (btc->hal->chip_id == CHIP_WIFI6_8851B) {
+		if (group > BTC_BT_SS_GROUP)
+			group--; /* Tx-group=1, Rx-group=2 */
+
+		if (btc->mdinfo.ant.type == BTC_ANT_SHARED) /* 1-Ant */
+			group += 3;
+	}
+
+	_btc_io_w(btc, type, R_BTC_RF_LUT_WA, bMASKRF, group, false);
+	_btc_io_w(btc, type, R_BTC_RF_LUT_WD0, bMASKRF, val, false);
+}
+
+void _8852b_wl_btg_standby(struct btc_t *btc, u32 state)
+{
+	u32 data0, data1;
+	u32 type = (btc->mdinfo.ant.btg_pos << 8) | RTW_MAC_RF_CMD_OFLD;
+
+	switch(btc->hal->chip_id) {
+	default:
+	case CHIP_WIFI6_8852B:
+		data1 = 0x31;
+		data0 = (state == 1) ? 0x179 : 0x20;
+		break;
+	case CHIP_WIFI6_8852BP:
+		data1 = 0x620;
+		data0 = (state == 1) ? 0x179c : 0x208;
+		break;
+	case CHIP_WIFI6_8851B:
+		data1 = 0x110;
+		data0 = (state == 1) ? 0x179c : 0x208;
+		break;
+	case CHIP_WIFI6_8852BT:
+		data1 = 0x31;
+		data0 = (state == 1) ? 0x179 : 0x20;
+		break;
+	case CHIP_WIFI6_8852BPT:
+		data1 = 0x620;
+		data0 = (state == 1) ? 0x179c : 0x208;
+		break;
+	}
+	/* set WL standby = Rx for GNT_BT_Tx = 1->0 settle issue */
+	_btc_io_w(btc, type, R_BTC_RF_LUT_EN, bMASKRF, BIT(19), false);
+	_btc_io_w(btc, type, R_BTC_RF_LUT_WA, bMASKRF, 0x1, false);
+	_btc_io_w(btc, type, R_BTC_RF_LUT_WD1, bMASKRF, data1, false);
+	_btc_io_w(btc, type, R_BTC_RF_LUT_WD0, bMASKRF, data0, false);
+	_btc_io_w(btc, type, R_BTC_RF_LUT_EN, bMASKRF, 0x0, true);
+}
+
+void _8852b_wl_req_mac(struct btc_t *btc, u8 mac_id)
+{
+	u32 val1;
+
+	_read_cx_reg(btc, R_BTC_CFG, &val1);
+
+	if (mac_id == HW_PHY_0)
+		val1 = val1 & (~B_BTC_WL_SRC);
+	else
+		val1 = val1 | B_BTC_WL_SRC;
+
+	_write_cx_reg(btc, R_BTC_CFG, val1);
+}
+
+void _8852b_wl_pri(struct btc_t *btc, u8 map, bool state)
+{
+	u32 reg = 0, bitmap = 0;
+
+	switch (map) {
+	case BTC_PRI_MASK_TX_RESP:
+	default:
+		reg = R_BTC_BT_COEX_MSK_TABLE;
+		bitmap = B_BTC_PRI_MASK_TX_RESP_V1;
+		break;
+	case BTC_PRI_MASK_BEACON:
+		reg = R_BTC_WL_PRI_MSK;
+		bitmap = B_BTC_PTA_WL_PRI_MASK_BCNQ;
+		break;
+	case BTC_PRI_MASK_RX_CCK:
+		reg = R_BTC_BT_COEX_MSK_TABLE;
+		bitmap = B_BTC_PRI_MASK_RXCCK_V1;
+		break;
+	}
+
+	_btc_io_w(btc, RTW_MAC_MAC_CMD_OFLD, reg, bitmap, state, false);
+}
+
+void _8852b_init_ext_interface(struct btc_t *btc)
+{
+	struct btc_cx *cx = &btc->cx;
+
+	if (cx->other.hw_coex & BTC_EXTSOC_INTF_PTA) {
+		/* set GPIO as Ext-PTA interface wire */
+		hal_btc_fw_set_gpio(btc, CXDGPIO_EXT_HPTA,
+				    cx->other.ext_hpta_cfg);
+	}
+
+	/* set GPIO as Ext-mailbox interface wire */
+	if (cx->other.hw_coex & BTC_EXTSOC_INTF_MBX) {
+		hal_btc_fw_set_gpio(btc, CXDGPIO_EXT_HMBX,
+				    cx->other.ext_hmbx_cfg);
+	}
+
+	/* set GPIO as Ext-SWIO interface wire */
+	if (cx->other.hw_coex & BTC_EXTSOC_INTF_SWIO) {
+		hal_btc_fw_set_gpio(btc, CXDGPIO_EXT_SWOUT,
+				    cx->other.ext_swout_cfg);
+		hal_btc_fw_set_gpio(btc, CXDGPIO_EXT_SWIN,
+				    cx->other.ext_swin_cfg);
+	}
+}
+
+void _8852b_init_cfg(struct btc_t *btc)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_module *module = &btc->mdinfo;
+	struct btc_ant_info *ant = &module->ant;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_trx_nss_para *trx_nss = &dm->wl_trx_nss;
+	u32 path, type, path_min, path_max;
+
+	PHL_INFO("[BTC], %s !! \n", __FUNCTION__);
+
+	/* PTA init  */
+	rtw_hal_mac_coex_init(h, btc->chip->pta_mode, btc->chip->pta_direction);
+
+	/* set WL Tx response = Hi-Pri */
+	_8852b_wl_pri(btc, BTC_PRI_MASK_TX_RESP, true);
+
+	/* set WL Tx beacon = Hi-Pri */
+	_8852b_wl_pri(btc, BTC_PRI_MASK_BEACON, true);
+
+	/* for 1-Ant && 1-ss case: only 1-path */
+	if (ant->stream_cnt == 0x11) {
+		path_min = ant->single_pos;
+		path_max = path_min;
+	} else {
+		path_min = RF_PATH_A;
+		path_max = RF_PATH_B;
+	}
+
+	path = path_min;
+
+	while (1) {
+		type = (path << 8) | RTW_MAC_RF_CMD_OFLD;
+
+		/* set rf gnt-debug off when init*/
+		if (dm->btc_initing)
+			_btc_io_w(btc, type, R_BTC_RF_BTG_CTRL, bMASKRF, 0x0, false);
+
+		/* set DEBUG_LUT_RFMODE_MASK = 1 to start trx-mask-setup */
+		_btc_io_w(btc, type, R_BTC_RF_LUT_EN, bMASKRF, BIT(17), false);
+
+		/* For 1T2R/1T1R mechanism
+		 * BTG:     WL Tx = mask
+		 * Non-BTG: if GNT_WL = 0 && BT = Rx_group --> WL Tx = THRU
+		 */
+		if (module->bt_pos == BTC_BT_BTG && ant->btg_pos == path) {
+			if (trx_nss->tx_limit) {
+				_8852b_wl_trx_mask(btc, type, BTC_BT_SS_GROUP, 0x5dd);
+				_8852b_wl_trx_mask(btc, type, BTC_BT_TX_GROUP, 0x55d);
+				_8852b_wl_trx_mask(btc, type, BTC_BT_RX_GROUP, 0x5dd);
+			} else {
+				_8852b_wl_trx_mask(btc, type, BTC_BT_SS_GROUP, 0x5ff);
+				_8852b_wl_trx_mask(btc, type, BTC_BT_TX_GROUP, 0x55f);
+				_8852b_wl_trx_mask(btc, type, BTC_BT_RX_GROUP, 0x5df);
+			}
+		} else {
+			_8852b_wl_trx_mask(btc, type, BTC_BT_SS_GROUP, 0x5ff);
+			_8852b_wl_trx_mask(btc, type, BTC_BT_TX_GROUP, 0x5ff);
+			if (trx_nss->tx_limit)
+				_8852b_wl_trx_mask(btc, type, BTC_BT_RX_GROUP, 0x5ff);
+			else
+				_8852b_wl_trx_mask(btc, type, BTC_BT_RX_GROUP, 0x5df);
+		}
+
+		/* set DEBUG_LUT_RFMODE_MASK = 0 to stop trx-mask-setup */
+		_btc_io_w(btc, type, R_BTC_RF_LUT_EN, bMASKRF, 0, false);
+
+		path++;
+		if (path > path_max)
+			break;
+	}
+
+	if (btc->mdinfo.bt_solo != BTC_EXT_SOC_NONE) /* if connext ext-BTSOC */
+		_8852b_init_ext_interface(btc);
+
+	/* set PTA break table */
+	_btc_io_w(btc, RTW_MAC_MAC_CMD_OFLD, R_BTC_BREAK_TABLE, bMASKDW,
+		  0xf0ffffff, false);
+
+	/* enable BT counter 0xda40[16,2] = 2b'11 */
+	_btc_io_w(btc, RTW_MAC_MAC_CMD_OFLD, R_BTC_CSR_MODE, 0x10004,
+		  0x4001, true);
+}
+
+#endif /* CONFIG_RTL8852B */
+#endif
+
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc_8852b/btc_8852b.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc_8852b/btc_8852b.h
new file mode 100644
index 000000000000..7cf55f3b3868
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/btc_8852b/btc_8852b.h
@@ -0,0 +1,62 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016 - 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __RTL8852B_BTC_H__
+#define __RTL8852B_BTC_H__
+/* rtl8852b_btc.c */
+
+#define R_BTC_RF_BTG_CTRL 0x2
+#define R_BTC_RF_LUT_EN 0xEF
+#define R_BTC_RF_LUT_WA 0x33
+#define R_BTC_RF_LUT_WD0 0x3f
+#define R_BTC_RF_LUT_WD1 0x3e
+#define R_BTC_RF_LUT_WD2 0x3d
+
+#define R_BTC_BB_ANT_DIV_CTRL 0x1586c
+#define R_BTC_BB_BTG_RX 0x980
+#define R_BTC_BB_PRE_AGC_S1 0x476C
+#define R_BTC_BB_PRE_AGC_S0 0x4688
+
+#define R_BTC_CFG 0xDA00
+#define R_BTC_WL_PRI_MSK 0xDA10
+#define R_BTC_BREAK_TABLE 0xDA2C
+#define R_BTC_BT_COEX_MSK_TABLE 0xDA30
+#define R_BTC_CSR_MODE 0xDA40
+#define R_BTC_BT_STAST_HIGH 0xDA44
+#define R_BTC_BT_STAST_LOW 0xDA48
+
+#define B_BTC_PRI_MASK_TX_RESP_V1 BIT(3)
+#define B_BTC_PRI_MASK_RXCCK_V1 BIT(28)
+#define B_BTC_DBG_GNT_WL_BT BIT(27)
+#define B_BTC_BT_CNT_REST BIT(16)
+#define B_BTC_PTA_WL_PRI_MASK_BCNQ BIT(8)
+#define B_BTC_PTA_WL_PRI_MASK_MGQ BIT(4)
+#define B_BTC_BB_GNT_MUX 0x001e0000
+#define B_BTC_BB_PRE_AGC_MASK bMASKB3
+#define B_BTC_BB_PRE_AGC_VAL 0x80000000
+
+extern const struct btc_chip chip_8852b;
+extern const struct btc_chip chip_8852bp;
+extern const struct btc_chip chip_8851b;
+extern const struct btc_chip chip_8852bt;
+extern const struct btc_chip chip_8852bpt;
+void _8852b_rfe_type(struct btc_t *btc);
+void _8852b_init_cfg(struct btc_t *btc);
+void _8852b_wl_tx_power(struct btc_t *btc, u32 level);
+void _8852b_wl_rx_gain(struct btc_t *btc, u32 level);
+void _8852b_wl_btg_standby(struct btc_t *btc, u32 state);
+void _8852b_wl_req_mac(struct btc_t *btc, u8 mac_id);
+void _8852b_get_reg_status(struct btc_t *btc, u8 type, void *status);
+u8 _8852b_bt_rssi(struct btc_t *btc, u8 val);
+#endif /*__RTL8852B_PHY_H__*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/hal_btc.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/hal_btc.c
new file mode 100644
index 000000000000..b752210cf7c7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/hal_btc.c
@@ -0,0 +1,2893 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_BTC_C_
+#include "../hal_headers_le.h"
+#include "../hal_headers.h"
+#include "hal_btc.h"
+#include "halbtc_fw.h"
+#include "halbtc_def.h"
+#include "halbtc_action.h"
+
+#ifdef CONFIG_BTCOEX
+
+/* [31:24] -> main-version
+ * [23:16] -> sub-version
+ * [15:8]  -> Hot-Fix-version
+ * [7:0]   -> branch ID, ex: 0x00-> Main-Branch
+ * Modify bt_8852x.c chip_8852x member: btcx_desired, wlcx_desired if required
+ * btcx_desired: BT FW coex version -> main-version + 1 if update.
+ * wlcx_desired: WL FW coex version -> sub-version + 1 if update
+ */
+const u32 coex_ver = 0x07100013;
+
+static struct btc_ops _btc_ops = {
+	_ntfy_power_on,
+	_ntfy_power_off,
+	_ntfy_init_coex,
+	_ntfy_scan_start,
+	_ntfy_scan_finish,
+	_ntfy_switch_band,
+	_ntfy_specific_packet,
+	_ntfy_role_info,
+	_ntfy_radio_state,
+	_ntfy_customerize,
+	_ntfy_wl_rfk,
+	_ntfy_wl_sta,
+	_ntfy_fwinfo,
+	_ntfy_timer
+};
+
+#define _update_dbcc_band(phy_idx) \
+	btc->cx.wl.dbcc_info.real_band[phy_idx] =\
+	(btc->cx.wl.scan_info.phy_map & BIT(phy_idx) ?\
+	btc->cx.wl.dbcc_info.scan_band[phy_idx] :\
+	btc->cx.wl.dbcc_info.op_band[phy_idx])
+
+/* specific ap with tx null issue*/
+static const u8 btc_ap_sptn[][6] = {
+	{0x0c, 0x9d, 0x92, 0x01, 0x8a, 0x80},
+	{0x0c, 0x9d, 0x92, 0x02, 0x21, 0x74}
+};
+
+static const u8 btc_ap_tpl[][3] = {
+	{0x8C, 0x21, 0x0a}, {0x64, 0x66, 0xb3}, {0xbc, 0x46, 0x99}
+};
+
+static const u8 btc_ap_vw[][3] = {
+	{0x00, 0x14, 0xe8} };
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+/* fdd train control parameters  */
+static const struct btc_fddt_time_ctrl ft_tctrl_def = {20, 20, 40, 0};
+static const struct btc_fddt_break_check ft_bchk_def =
+						{0, 2, 80, 20, 8, 8, -85, 5};
+static const struct btc_fddt_fail_check ft_fchk_def = {0, 2, 115, 85};
+
+const struct btc_fddt_cell cell_ul_def[5][5] = { /* BT-RSSI 0~4, WL-RSSI 0~4 */
+	{{0,0,0,0},    {0,0,0,0},   {0,0,0,0},   {0,0,0,0},   {0,0,0,0}},
+	{{0,0,0,0},    {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{0,0,0,0},    {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{15,15,10,4}, {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{15,15,10,4}, {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}}
+};
+
+const struct btc_fddt_cell cell_dl_def[5][5] = { /* BT-RSSI 0~4, WL-RSSI 0~4 */
+	{{0,0,0,0},    {0,0,0,0},    {0,0,0,0},   {0,0,0,0},   {-5,5,15,6}},
+	{{10,15,20,5}, {10,15,20,5}, {5,15,20,5}, {0,10,20,5}, {-5,5,20,5}},
+	{{10,15,25,5}, {10,15,25,5}, {5,15,25,5}, {0,10,25,5}, {-5,5,25,5}},
+	{{10,15,30,4}, {10,15,30,4}, {5,15,30,5}, {0,10,30,5}, {-5,5,30,5}},
+	{{10,15,40,4}, {10,15,40,4}, {5,15,40,4}, {0,10,40,5}, {-5,5,40,5}}
+};
+
+static const struct btc_fddt_cell cell_ul_bt0[4][5] = {
+	{{0,0,0,0},   {0,0,0,0},   {0,0,0,0},   {-2,8,10,4}, {-5,5,10,4}},
+	{{0,0,0,0},   {0,0,0,0},   {0,0,0,0},   {-2,8,10,4}, {-5,5,10,4}},
+	{{0,0,0,0},   {0,0,0,0},   {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{0,0,0,0},   {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}} };
+
+static const struct btc_fddt_cell cell_ul_bt1[4][5] = {
+	{{0,0,0,0},   {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{0,0,0,0},   {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{5,15,10,4}, {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{5,15,10,4}, {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}} };
+
+static const struct btc_fddt_cell cell_ul_bt2[4][5] = {
+	{{15,15,10,4}, {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{15,15,10,4}, {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{5,15,10,4},  {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}},
+	{{5,15,10,4},  {5,15,10,4}, {3,13,10,4}, {-2,8,10,4}, {-5,5,10,4}} };
+
+static const struct btc_fddt_cell cell_dl_iso25[5][5] = {
+	{{0,0,0,0},    {0,0,0,0},    {0,0,0,0},    {0,0,0,0},   {0,10,15,6}},
+	{{10,15,20,5}, {10,15,20,5}, {5,15,20,5},  {0,10,20,5}, {-5,5,20,5}},
+	{{10,15,25,5}, {10,15,25,5}, {10,15,25,5}, {5,15,25,5}, {0,10,25,5}},
+	{{10,15,25,4}, {10,15,25,4}, {10,15,25,5}, {5,15,25,5}, {0,10,25,5}},
+	{{10,15,30,4}, {10,15,30,4}, {10,15,30,4}, {5,15,30,5}, {0,10,30,5}} };
+
+static const struct btc_fddt_cell cell_dl_iso30[5][5] = {
+	{{0,0,0,0},    {0,0,0,0},    {0,0,0,0},    {5,15,15,6},  {5,15,15,6}},
+	{{10,15,20,5}, {10,15,20,5}, {5,15,20,5},  {0,10,20,5},  {-5,5,20,5}},
+	{{10,15,25,5}, {10,15,25,5}, {10,15,25,5}, {10,15,25,5}, {5,15,25,5}},
+	{{10,15,25,5}, {10,15,25,5}, {10,15,25,5}, {5,15,25,5},  {5,15,25,5}},
+	{{10,15,25,4}, {10,15,25,4}, {10,15,25,4}, {10,15,25,5}, {5,15,25,5}} };
+
+static const struct btc_fddt_cell cell_dl_iso35[5][5] = {
+	{{0,0,0,0},    {0,0,0,0},    {5,15,15,7},  {5,15,15,7},  {5,15,15,7}},
+	{{10,15,15,5}, {10,15,15,5}, {5,15,15,5},  {0,10,15,5},  {-5,5,15,5}},
+	{{10,15,20,5}, {10,15,20,5}, {10,15,20,5}, {10,15,20,5}, {5,15,20,5}},
+	{{10,15,20,5}, {10,15,20,5}, {10,15,20,5}, {5,15,20,5},  {5,15,20,5}},
+	{{10,15,20,4}, {10,15,20,4}, {10,15,20,4}, {10,15,20,5}, {5,15,20,5}} };
+
+static const struct btc_fddt_cell cell_dl_iso40[5][5] = {
+	{{0,0,0,0},    {10,15,10,7}, {10,15,10,7}, {5,15,10,7},  {5,15,10,7}},
+	{{10,15,15,5}, {10,15,15,5}, {5,15,15,5},  {0,10,15,5},  {-5,5,15,5}},
+	{{10,15,20,5}, {10,15,20,5}, {10,15,20,5}, {10,15,20,5}, {5,15,20,5}},
+	{{10,15,20,5}, {10,15,20,5}, {10,15,20,5}, {5,15,20,5},  {5,15,20,5}},
+	{{10,15,20,4}, {10,15,20,4}, {10,15,20,4}, {10,15,20,5}, {5,15,20,5}} };
+#endif
+
+/******************************************************************************
+ *
+ * coex internal functions
+ *
+ *****************************************************************************/
+ static u8 _chk_freerun(struct btc_t *btc)
+{
+	struct btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED ||
+	    btc->mdinfo.ant.num < 3 ||
+	    bt_linfo->pan_desc.exist ||
+	    btc->cx.other.type != BTC_EXTSOC_FUNC_NONE)
+		return false; /* for dedicated antenna + no freerun */
+	else
+		return true; /* for dedicated antenna + ant_num = 3 : freerun */
+}
+
+static u8 _chk_fddtrain(struct btc_t *btc)
+{
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	struct btc_dm *dm = &btc->dm;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_link_info *bt_linfo = &bt->link_info;
+	struct btc_bt_smap *b = &bt_linfo->status.map;
+	u32 rsn = 0;
+
+	dm->fddt_info.nrsn_map = 0;
+
+	if (dm->fddt_info.type >= BTC_FDDT_TYPE_FIX_TDD)
+	    return BTC_FDDT_ENABLE;
+
+	if (dm->fddt_info.type == BTC_FDDT_TYPE_STOP)
+		rsn |= BIT(BTC_NFRSN_FORCE_STOP);
+
+	/* Check FDD-Train prerequisites */
+	if (btc->mdinfo.ant.type != BTC_ANT_DEDICATED)
+		rsn |= BIT(BTC_NFRSN_DEDICATED_ANT);
+
+	if (btc->mdinfo.ant.isolation < BTC_FDDTRAIN_ANTISO_MIN)
+		rsn |= BIT(BTC_NFRSN_ANT_ISO_LOW);
+
+	if (btc->mdinfo.ant.isolation >= BTC_FDDTRAIN_ANTISO_MAX)
+		rsn |= BIT(BTC_NFRSN_ANT_ISO_HI);
+
+	if (wl_rinfo->link_mode != BTC_WLINK_2G_STA)
+		rsn |= BIT(BTC_NFRSN_WL_2GSTA);
+
+	if (cx->state_map != BTC_WBUSY_BNOSCAN) {
+		if (wl->status.val & BTC_WL_LINK_SCAN)
+			rsn |= BIT(BTC_NFRSN_WL_NOSCAN);
+
+		if (!wl_smap->busy || wl_smap->lps || wl_smap->rf_off)
+		    	rsn |= BIT(BTC_NFRSN_WL_BUSY);
+
+		if (b->inq_pag)
+			rsn |= BIT(BTC_NFRSN_BT_NOINQ);
+	}
+
+	if (wl->rfk_info.state != BTC_WRFK_STOP)
+		rsn |= BIT(BTC_NFRSN_WL_NORFK);
+
+	if (!bt_linfo->a2dp_desc.exist) {
+		/* Todo: BTC_NFRSN_BT_A2DP_BUSY for a2dp idle case */
+		if (bt_linfo->a2dp_desc.active)
+			rsn |= BIT(BTC_NFRSN_BT_A2DP_BUSY);
+		else
+			rsn |= BIT(BTC_NFRSN_BT_PROFILE);
+	} else if (bt_linfo->pan_desc.exist || bt_linfo->a2dp_desc.sink ||
+		   (bt_linfo->multi_link.now && !bt_linfo->hid_desc.pair_cnt)) {
+		rsn |= BIT(BTC_NFRSN_BT_PROFILE);
+	}
+
+	/* Todo: for other not run cases */
+
+	dm->fddt_info.nrsn_map = rsn;
+#else
+	btc->dm.fddt_info.nrsn_map = BIT(BTC_NFRSN_SUPPORT);
+#endif
+
+	if (btc->dm.fddt_info.nrsn_map == 0)
+		return BTC_FDDT_ENABLE;
+	else
+		return BTC_FDDT_DISABLE;
+}
+
+static void _set_btc_timer(struct btc_t *btc, u16 tmr_id, u32 ms)
+{
+	struct btc_tmr *btmr = NULL;
+
+	if (tmr_id < BTC_TIMER_MAX) {
+		btmr = &btc->timer[tmr_id];
+		_os_set_timer(halcom_to_drvpriv(btc->hal), &btmr->tmr, ms);
+	}
+}
+
+static void _btmr_stop(struct btc_t *btc)
+{
+	struct btc_tmr *btmr = NULL;
+	u8 i = 0;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): stop btc timers!!\n",
+		  __func__);
+
+	btc->tmr_stop = true;
+
+	for (i = 0; i < BTC_TIMER_MAX; i++) {
+		btmr = &btc->timer[i];
+		_os_cancel_timer(halcom_to_drvpriv(btc->hal), &btmr->tmr);
+	}
+}
+
+static void _btmr_release(struct btc_t *btc)
+{
+	struct btc_tmr *btmr = NULL;
+	u8 i = 0;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): release btc timers!!\n", __func__);
+
+	for (i = 0; i < BTC_TIMER_MAX; i++) {
+		btmr = &btc->timer[i];
+		_os_release_timer(halcom_to_drvpriv(btc->hal), &btmr->tmr);
+	}
+}
+
+static void _btmr_start(struct btc_t *btc)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): start btc periodic timer!!\n", __func__);
+
+	btc->tmr_stop = false;
+	_set_btc_timer(btc, BTC_TIMER_PERIODIC, BTC_PERIODIC_TIME);
+}
+
+static void _btmr_cb(void *ctx)
+{
+	struct btc_tmr *btmr = NULL;
+	struct btc_t *btc = NULL;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(), ctx = 0x%p !!\n",
+		  __func__, ctx);
+
+	if (!ctx)
+		return;
+
+	btmr = (struct btc_tmr *)ctx;
+	btc = (struct btc_t *)btmr->btc;
+
+	if (!btc->tmr_init || btc->tmr_stop)
+		return;
+
+	_send_phl_evnt(btc, (u8 *)btmr, sizeof(struct btc_tmr),
+		       BTC_HMSG_TMR_EN);
+}
+
+static void _btmr_init(struct btc_t *btc)
+{
+	struct btc_tmr *btmr = NULL;
+	u8 i = 0;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): !!\n", __func__);
+
+	for (i = 0; i < BTC_TIMER_MAX; i++) {
+		btmr = &btc->timer[i];
+		btmr->btc = btc;
+		btmr->id = i;
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], init btc timer(%d) = 0x%p !!\n", i, btmr);
+		_os_init_timer(halcom_to_drvpriv(btc->hal), &btmr->tmr,
+			       _btmr_cb, btmr, "btc_tmr");
+	}
+	btc->tmr_init = true;
+	btc->tmr_stop = true;
+}
+
+static void _btmr_deinit(struct btc_t *btc)
+{
+	if (btc->tmr_init) {
+		_btmr_stop(btc);
+		_btmr_release(btc);
+		btc->tmr_init = false;
+	}
+}
+
+u8
+_send_fw_cmd(struct btc_t *btc, u8 h2c_func, u8 *param, u16 len)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct rtw_g6_h2c_hdr hdr = {0};
+
+	hdr.h2c_class = BTFC_SET;
+	hdr.h2c_func = h2c_func;
+	hdr.done_ack = 0;
+	hdr.content_len = len;
+
+	if (len > BTC_H2C_MAXLEN) {
+		PHL_INFO("[BTC], %s(): return by H2C-len %d > max-limit:%d!!\n",
+			  __func__, len, BTC_H2C_MAXLEN);
+		btc->fwinfo.cnt_h2c_fail++;
+		btc->dm.error |= BTC_DMERR_H2C_BUF_OVER;
+		return 0;
+	} else if (len > BTC_H2C_MAXLEN1) {
+		hdr.type = H2CB_TYPE_LONG_DATA;
+	} else {
+		hdr.type = H2CB_TYPE_DATA;
+	}
+
+	if (!wl->status.map.init_ok) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_ERR_,
+			  "[BTC], %s(): return by btc not init!!\n", __func__);
+		btc->fwinfo.cnt_h2c_fail++;
+		return 0;
+	}
+
+	if ((wl->status.map.rf_off_pre == 1 && wl->status.map.rf_off == 1) ||
+	    (wl->status.map.lps_pre == BTC_LPS_RF_OFF &&
+	     wl->status.map.lps == BTC_LPS_RF_OFF)) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_ERR_,
+			  "[BTC], %s(): return by wl off!!\n", __func__);
+		btc->fwinfo.cnt_h2c_fail++;
+		return 0;
+	}
+
+	if (rtw_hal_mac_send_h2c(h, &hdr, (u32 *)param) != 0) {
+		PHL_INFO("[BTC], %s(): return by rtw_hal_mac_send_h2c!!\n",
+			  __func__);
+		btc->fwinfo.cnt_h2c_fail++;
+		return 0;
+	}
+
+	btc->fwinfo.cnt_h2c++;
+	return 1;
+}
+
+u32 _read_scbd(struct btc_t *btc)
+{
+	const struct btc_chip *chip = btc->chip;
+	u32 scbd_val = 0;
+
+	if (!chip->scbd)
+		return 0;
+
+	btc->cx.cnt_bt[BTC_BCNT_SCBDREAD]++;
+	rtw_hal_mac_get_scoreboard(btc->hal, &scbd_val);
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], read scbd : 0x%08x \n", scbd_val);
+	return scbd_val;
+}
+
+void _write_scbd(struct btc_t *btc, u32 val, bool state)
+{
+	const struct btc_chip *chip = btc->chip;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u32 scbd_val = 0;
+	u8 force_exec = 0;
+
+	if (!chip->scbd)
+		return;
+
+	/* only use bit23~0 */
+	scbd_val = (state ? (wl->scbd | val) : (wl->scbd & (~val)));
+
+	if (val & BTC_WSCB_ACTIVE || val & BTC_WSCB_ON ||
+	    val & BTC_WSCB_RXGAIN || val & BTC_WSCB_COUNTRY_CODE ||
+	    val & BTC_WSCB_WLRFK)
+		force_exec = 1;
+
+	/* Just update wl->scbd and set wl->scbd_change,
+	 * moved "Write scoreboard I/O" to  _action_common()
+	 * _write_scbd will be executed if run_coex()
+	 */
+	if (scbd_val != wl->scbd || force_exec) {
+		wl->scbd = scbd_val;
+		wl->scbd_change = 1;
+	}
+}
+
+static u8
+_update_rssi_state(struct btc_t *btc, u8 pre_state, u8 rssi, u8 thresh)
+{
+	u8 next_state, tol = btc->chip->rssi_tol;
+
+	tol = tol + btc->dm.rssi_offset2;
+
+	if (pre_state == BTC_RSSI_ST_LOW ||
+	    pre_state == BTC_RSSI_ST_STAY_LOW) {
+		if (rssi >= (thresh + tol))
+			next_state = BTC_RSSI_ST_HIGH;
+		else
+			next_state = BTC_RSSI_ST_STAY_LOW;
+	} else {
+		if (rssi < thresh)
+			next_state = BTC_RSSI_ST_LOW;
+		else
+			next_state = BTC_RSSI_ST_STAY_HIGH;
+	}
+
+	return next_state;
+}
+
+static void _reset_btc_var(struct btc_t *btc, u8 type)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_bt_link_info *bt_linfo = &bt->link_info;
+	struct btc_wl_link_info *wl_linfo = wl->link_info;
+	u8 i;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s()\n", __func__);
+
+	/* Reset Coex variable */
+	if (type & BTC_RESET_CX)
+		hal_mem_set(h, cx, 0, sizeof(struct btc_cx));
+	if (type & BTC_RESET_BTINFO)
+		hal_mem_set(h, bt, 0, sizeof(struct btc_bt_info));
+
+	hal_mem_set(h, &wl->rf_para, 0, sizeof(struct btc_rf_para));
+	hal_mem_set(h, &bt->rf_para, 0, sizeof(struct btc_rf_para));
+
+	wl->rf_para.tx_pwr_freerun = BTC_WL_DEF_TX_PWR;
+	bt->rf_para.rx_gain_freerun = BTC_BT_RX_NORMAL_LVL;
+
+	if (type & BTC_RESET_CTRL)
+		hal_mem_set(h, &btc->ctrl, 0, sizeof(struct btc_ctrl));
+
+	/* Init Coex variables that are not zero */
+	if (type & BTC_RESET_DM) {
+		hal_mem_set(h, dm, 0, sizeof(struct btc_dm));
+		hal_mem_set(h, bt_linfo->rssi_state, 0, BTC_BT_RSSI_THMAX);
+
+		for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++)
+			hal_mem_set(h, wl_linfo[i].rssi_state, 0,
+				    BTC_WL_RSSI_THMAX);
+
+		/* set the slot_now table to original */
+		_tdma_cpy(&dm->tdma_now, &t_def[CXTD_OFF]);
+		_tdma_cpy(&dm->tdma, &t_def[CXTD_OFF]);
+		_slots_cpy(dm->slot_now, s_def);
+		_slots_cpy(dm->slot, s_def);
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+		hal_mem_cpy(h, &dm->fddt_info.train.t_ctrl,
+			    &ft_tctrl_def, sizeof(ft_tctrl_def));
+		hal_mem_cpy(h, &dm->fddt_info.train.b_chk,
+			    &ft_bchk_def, sizeof(ft_bchk_def));
+		hal_mem_cpy(h, &dm->fddt_info.train.f_chk,
+			    &ft_fchk_def, sizeof(ft_fchk_def));
+#endif
+
+		btc->policy_len = 0;
+		btc->bt_req_len[HW_PHY_0] = 0;
+		btc->bt_req_len[HW_PHY_1] = 0;
+		btc->hubmsg_cnt = 0;
+
+		dm->coex_info_map = BTC_COEX_INFO_ALL;
+		dm->wl_tx_limit.tx_time = BTC_MAX_TX_TIME_DEF;
+		dm->wl_tx_limit.tx_retry = BTC_MAX_TX_RETRY_DEF;
+		dm->fddt_info.type = BTC_FDDT_TYPE_AUTO;
+		dm->bt_slot_flood = BTC_B1_MAX;
+		dm->wl_btg_rx_rb = BTC_BB_GNT_NOTFOUND;
+		dm->wl_pre_agc_rb = BTC_BB_PRE_AGC_NOTFOUND;
+	}
+
+#ifdef BTC_AISO_SUPPORT
+	if (type & BTC_RESET_BT_PSD_DM) {
+		hal_mem_set(h, &btc->bt_psd_dm, 0,
+			    sizeof(struct btc_bt_psd_dm));
+		btc->bt_psd_dm.aiso_method = BTC_AISO_M_ALL;
+		btc->bt_psd_dm.aiso_method_final = BTC_AISO_M46_AVG;
+		btc->bt_psd_dm.wl_ch_last = 0xFF;
+	}
+#endif
+
+	if (type & BTC_RESET_MDINFO)
+		hal_mem_set(h, &btc->mdinfo, 0, sizeof(struct btc_module));
+}
+
+static bool _chk_wl_rfk_request(struct btc_t *btc)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_info *bt = &cx->bt;
+
+	_update_bt_scbd(btc, true);
+
+	cx->cnt_wl[BTC_WCNT_RFK_REQ]++;
+
+	if ((bt->rfk_info.map.run || bt->rfk_info.map.req) &&
+	    (!bt->rfk_info.map.timeout)) {
+		cx->cnt_wl[BTC_WCNT_RFK_REJECT]++;
+		return BTC_WRFK_REJECT;
+	} else {
+		cx->cnt_wl[BTC_WCNT_RFK_GO]++;
+		return BTC_WRFK_ALLOW;
+	}
+}
+
+void _set_fddt_cell_by_antiso(struct btc_t *btc)
+{
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_fddt_train_info *t = &dm->fddt_info.train;
+	u8 ant_iso_id = 0;
+	u32 sz = sizeof(struct btc_fddt_cell);
+	/* for uplink -> Ant_iso: 25/30/35/40 x WL-RSSI for BT-RSSI-level0 */
+
+	if (btc->mdinfo.ant.type != BTC_ANT_DEDICATED)
+		return;
+
+	if (btc->mdinfo.ant.isolation <= 20) {
+		hal_mem_cpy(h, t->cell_ul, cell_ul_def, 5 * 5 * sz);
+		hal_mem_cpy(h, t->cell_dl, cell_dl_def, 5 * 5 * sz);
+		_set_fddt_ctrl(btc, FC_EXEC);
+		return;
+	}
+
+	if (btc->mdinfo.ant.isolation <= 25)
+		ant_iso_id = 0;
+	else if (btc->mdinfo.ant.isolation <= 30)
+		ant_iso_id = 1;
+	else if (btc->mdinfo.ant.isolation <= 35)
+		ant_iso_id = 2;
+	else
+		ant_iso_id = 3;
+
+	/* set uplink cell if BT-RSSI-Level-0 for Ant_iso: 25/30/35/40 */
+	hal_mem_cpy(h, t->cell_ul[0], cell_ul_bt0[ant_iso_id], 5 * sz);
+	hal_mem_cpy(h, t->cell_ul[1], cell_ul_bt1[ant_iso_id], 5 * sz);
+	hal_mem_cpy(h, t->cell_ul[2], cell_ul_bt2[ant_iso_id], 5 * sz);
+
+	/* set downlink cell for Ant_iso: 25/30/35/40 */
+	switch(ant_iso_id) {
+	case 0:
+		hal_mem_cpy(h, t->cell_dl, cell_dl_iso25, 5 * 5 * sz);
+		break;
+	case 1:
+		hal_mem_cpy(h, t->cell_dl, cell_dl_iso30, 5 * 5 * sz);
+		break;
+	case 2:
+		hal_mem_cpy(h, t->cell_dl, cell_dl_iso35, 5 * 5 * sz);
+		break;
+	case 3:
+		hal_mem_cpy(h, t->cell_dl, cell_dl_iso40, 5 * 5 * sz);
+		break;
+	}
+	_set_fddt_ctrl(btc, FC_EXEC);
+#endif
+}
+
+void _set_init_info(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+
+	dm->init_info.wl_only = (u8)dm->wl_only;
+	dm->init_info.bt_only = (u8)dm->bt_only;
+	dm->init_info.wl_init_ok = (u8)wl->status.map.init_ok;
+	dm->init_info.dbcc_en = btc->hal->dbcc_en;
+	dm->init_info.cx_other = btc->cx.other.type;
+	dm->init_info.wl_guard_ch = btc->chip->afh_guard_ch;
+	dm->init_info.module = btc->mdinfo;
+
+	/* original tx power, no Tx power adjust */
+	hal_btc_fw_set_drv_info(btc, CXDRVINFO_INIT);
+	hal_btc_fw_set_drv_info(btc, CXDRVINFO_CTRL);
+
+	hal_btc_fw_set_drv_info(btc, CXDRVINFO_ROLE);
+
+	hal_btc_fw_set_slots(btc);
+	hal_btc_fw_set_monreg(btc);
+
+	_set_fddt_cell_by_antiso(btc);
+	/* original tx power, no Tx power adjust */
+	_set_wl_tx_power(btc, BTC_WL_DEF_TX_PWR);
+}
+
+void _update_role_link_mode(struct btc_t *btc, bool client_joined, u32 noa)
+{
+	struct btc_wl_role_info *wl_rinfo = &btc->cx.wl.role_info;
+	u32 wl_role = wl_rinfo->role_map;
+	u32 type = BTC_WLMROLE_NONE, dur = 0;
+
+	/* if no client_joined, don't care P2P-GO/AP role */
+	if (((wl_role & BIT(PHL_RTYPE_P2P_GO)) ||
+	      (wl_role & BIT(PHL_RTYPE_AP))) && !client_joined) {
+		if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC) {
+			wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+			wl_rinfo->connect_cnt--;
+		} else if (wl_rinfo->link_mode == BTC_WLINK_2G_GO ||
+			wl_rinfo->link_mode == BTC_WLINK_2G_AP) {
+			wl_rinfo->link_mode = BTC_WLINK_NOLINK;
+			wl_rinfo->connect_cnt--;
+		}
+	}
+
+	/* Identify 2-Role type  */
+	if (wl_rinfo->connect_cnt >= 2 &&
+	    (wl_rinfo->link_mode == BTC_WLINK_2G_SCC ||
+	     wl_rinfo->link_mode == BTC_WLINK_2G_MCC ||
+	     wl_rinfo->link_mode == BTC_WLINK_25G_MCC ||
+	     wl_rinfo->link_mode == BTC_WLINK_5G)) {
+		if ((wl_role & BIT(PHL_RTYPE_P2P_GO)) ||
+		     (wl_role & BIT(PHL_RTYPE_AP)))
+			type = noa? BTC_WLMROLE_STA_GO_NOA : BTC_WLMROLE_STA_GO;
+		else if (wl_role & BIT(PHL_RTYPE_P2P_GC))
+			type = noa? BTC_WLMROLE_STA_GC_NOA : BTC_WLMROLE_STA_GC;
+		else
+			type = BTC_WLMROLE_STA_STA;
+
+		dur = noa;
+	}
+
+	wl_rinfo->mrole_type = type;
+	wl_rinfo->mrole_noa_duration = dur;
+
+	PHL_INFO("[BTC], %s(): link_mode=%d, connect_cnt=%d, mtype=%d,"
+		 " client_joined=%d, noa=%d!!\n", __func__, wl_rinfo->link_mode,
+		 wl_rinfo->connect_cnt, type, client_joined, noa);
+}
+
+u8 _get_role_link_mode(struct btc_t *btc, u8 role)
+{
+	switch(role) {
+	case PHL_RTYPE_STATION:
+		return BTC_WLINK_2G_STA;
+	case PHL_RTYPE_P2P_GO:
+		return BTC_WLINK_2G_GO;
+	case PHL_RTYPE_P2P_GC:
+		return BTC_WLINK_2G_GC;
+	case PHL_RTYPE_AP:
+		return BTC_WLINK_2G_AP;
+	default:
+		return BTC_WLINK_OTHER;
+	}
+}
+
+void _set_wl_req_mac(struct btc_t *btc, u8 mac_id)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_chip_ops *ops = btc->chip->ops;
+
+	if (mac_id == wl->pta_req_mac || mac_id >= HW_PHY_MAX)
+		return;
+
+	if (ops && ops->wl_req_mac)
+		ops->wl_req_mac(btc, mac_id);
+
+	wl->pta_req_mac = mac_id;
+}
+
+static void _send_bt_country_code(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct rtw_regulation_info rg_info = {0};
+	u8 buf[3]={0};
+	/*use mailbox 0x11 to send country code*/
+
+	btc->dm.cnt_notify[BTC_NTFY_COUNTRYCODE]++;
+	hal_btc_notify_ps_tdma(btc, true);
+
+	if (wl->country_code[0] == 0 && wl->country_code[1] == 0) {
+		rtw_hal_query_regulation(btc->phl->phl_priv, &rg_info);
+		wl->country_code[0] = (u8)rg_info.country[0];
+		wl->country_code[1] = (u8)rg_info.country[1];
+	}
+
+	buf[0] = BTC_COUNTRY_CODE_OP; /*op_mode bit2 for country code*/
+	buf[1] = wl->country_code[0];
+	buf[2] = wl->country_code[1];
+	_send_fw_cmd(btc, SET_BT_WL_CH_INFO, buf, 3);
+	PHL_INFO("[BTC], %s(): send country code, 0x%02x%02x\n", __func__,
+		 buf[1], buf[2]);
+
+}
+
+
+static bool _chk_role_ch_group(struct btc_t *btc, struct rtw_chan_def r1,
+			       struct rtw_chan_def r2)
+{
+	if (r1.chan != r2.chan) { /* primary ch is different */
+		return false;
+	} else if (r1.bw == CHANNEL_WIDTH_40 && r2.bw == CHANNEL_WIDTH_40) {
+		if (r1.offset != r2.offset)
+			return false;
+	}
+
+	return true;  /* true-> the r1/r2 channel are grouped */
+}
+
+static u8 _chk_dbcc(struct btc_t *btc, struct rtw_chan_def* ch,
+		    u8* phy, u8* role, u8* dbcc_2g_phy)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	u8 j, k, dbcc_2g_cid = 0, dbcc_2g_cid2 = 0;
+	bool is_2g_ch_exist = false, is_multi_role_in_2g_phy = false;
+
+	/* find out the 2G-PHY by connect-id ->ch  */
+	for (j = 0; j < wl_rinfo->connect_cnt; j++) {
+		if ((ch+j)->center_ch <= 14) {
+			is_2g_ch_exist = true;
+			break;
+		}
+	}
+
+	/* If no any 2G-port exist, it's impossible because 5G-exclude */
+	if (!is_2g_ch_exist)
+		return BTC_WLINK_OTHER;
+
+	dbcc_2g_cid = j;
+	*dbcc_2g_phy = *(phy+dbcc_2g_cid);
+
+	/* connect_cnt <= 2 */
+	if (wl_rinfo->connect_cnt < BTC_TDMA_WLROLE_MAX)
+		 return (_get_role_link_mode(btc, (*role+dbcc_2g_cid)));
+
+	/* find the other-port in the 2G-PHY, ex: PHY-0:6G, PHY1: mcc/scc */
+	for (k = 0; k < wl_rinfo->connect_cnt; k++) {
+		if (k == dbcc_2g_cid)
+			continue;
+
+		if (*(phy+k) == *dbcc_2g_phy) {
+			is_multi_role_in_2g_phy = true;
+			dbcc_2g_cid2 = k;
+			break;
+		}
+	}
+
+	/* Single-role in 2G-PHY */
+	if (!is_multi_role_in_2g_phy)
+		return (_get_role_link_mode(btc, (*role+dbcc_2g_cid)));
+
+	/* 2-role in 2G-PHY */
+	if ((ch+dbcc_2g_cid2)->center_ch > 14)
+		return BTC_WLINK_25G_MCC;
+	else if (_chk_role_ch_group(btc, *(ch+dbcc_2g_cid), *(ch+dbcc_2g_cid2)))
+		return BTC_WLINK_2G_SCC;
+	else
+		return BTC_WLINK_2G_MCC;
+}
+
+static u8 _check_wl_ap_type(struct btc_t *btc, u8 *bssid, u8 type)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	u8 i = 0;
+	u8 is_match = 0;
+	u32 sz = 0;
+
+	switch (type) {
+	default:
+		break;
+	case BTC_SPAP_TPL:
+		for (i = 0; i < ARRAY_SIZE(btc_ap_tpl); i++) {
+			sz = sizeof(btc_ap_tpl[i]);
+			if (!hal_mem_cmp(h, btc_ap_tpl[i], &bssid[0], sz)) {
+				is_match = 1;
+				break;
+			}
+		}
+		break;
+	case BTC_SPAP_VW:
+		for (i = 0; i < ARRAY_SIZE(btc_ap_vw); i++) {
+			sz = sizeof(btc_ap_vw[i]);
+			if (!hal_mem_cmp(h, btc_ap_vw[i], &bssid[0], sz)) {
+				is_match = 1;
+				break;
+			}
+		}
+		break;
+	case BTC_SPAP_CMW:
+		if (bssid[0] == 0)
+			is_match = 1;
+		break;
+#if BTC_AP_BLIST
+	case BTC_SPAP_SPTN:
+		for (i = 0; i < ARRAY_SIZE(btc_ap_sptn); i++) {
+			sz = sizeof(btc_ap_sptn[i]);
+			if (!hal_mem_cmp(h, btc_ap_sptn[i], &bssid[0], sz)) {
+				is_match = 1;
+				break;
+			}
+		}
+		break;
+#endif
+	}
+
+	return is_match;
+}
+
+static void _update_wl_info(struct btc_t *btc, u8 rid, enum link_state reason)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_link_info *wl_linfo = wl->link_info;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
+	struct btc_wl_active_role *act_role = NULL;
+	struct rtw_hal_com_t *h = btc->hal;
+	struct rtw_chan_def cid_ch[BTC_WL_MAX_ROLE_NUMBER];
+
+	u8 i, j, cnt = 0, mac_id = HW_PHY_0;
+	u8 cnt_2g = 0, cnt_5g = 0, mode = BTC_WLINK_NOLINK;
+	u8 cid_phy[BTC_WL_MAX_ROLE_NUMBER] = {0};
+	u8 cid_role[BTC_WL_MAX_ROLE_NUMBER] = {0};
+	u8 client_cnt_last[BTC_WL_MAX_ROLE_NUMBER] = {0};
+	u8 dbcc_2g_phy = HW_PHY_0, phy_now = 0, phy_dbcc;
+	u8 cmw_ap_new = 0, buf[2] = {0};
+	u32 noa_duration = 0, sz = sizeof(struct rtw_chan_def), sz1;
+	bool b2g = false, b5g = false;
+	bool client_joined = false, client_inc_2g = false;
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++)
+		client_cnt_last[i] = wl_rinfo->active_role[i].client_cnt;
+
+	hal_mem_set(h, wl_rinfo, 0, sizeof(struct btc_wl_role_info));
+	hal_mem_set(h, wl_dinfo, 0, sizeof(struct btc_wl_dbcc_info));
+	hal_mem_set(h, cid_ch, 0, BTC_WL_MAX_ROLE_NUMBER * sz);
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++) {
+		/* check if role active? */
+		if (!wl_linfo[i].active || wl_linfo[i].phy >= HW_PHY_MAX)
+			continue;
+
+		/* Extract wl->link_info to wl->role_info
+		 * wl->link_info[i] --> i = role_index (0~4)
+		 * wl->link_info[i].pid --> pid = port_index
+		 * wl->role_info.active_role[i] --> i = role_id
+		 */
+		act_role = &wl_rinfo->active_role[i];
+		act_role->role = wl_linfo[i].role;
+
+		/* check if role connect? */
+		if (wl_linfo[i].connected == MLME_NO_LINK) {
+			act_role->connected = 0;
+			continue;
+		} else if (wl_linfo[i].connected == MLME_LINKING) {
+			continue;
+		}
+
+		/* wl_linfo[i].connected == MLME_LINKED */
+		act_role->connected = 1;
+		sz1 = sizeof(wl->bssid);
+		hal_mem_set(h, wl->bssid, 0, sz1);
+		if (rid == i && act_role->role == PHL_RTYPE_STATION) {
+			hal_mem_cpy(h, wl->bssid, wl_linfo->mac_addr, sz1);
+			/* reset special-AP flag if station mode linked */
+			btc->dm.leak_ap = 0;
+			wl->ap_type = 0;
+			
+			/* find matched bssid */
+			for (j = 1; j < BTC_SPAP_MAX; j++) {
+				if (_check_wl_ap_type(btc, wl->bssid, j)) {
+					wl->ap_type = j;
+					break;
+				}
+			}
+		}
+
+		cnt++;
+
+		act_role->pid = wl_linfo[i].pid;
+		act_role->phy = wl_linfo[i].phy;
+		act_role->band = wl_linfo[i].chdef.band;
+		act_role->ch = wl_linfo[i].chdef.center_ch;
+		act_role->bw = wl_linfo[i].chdef.bw;
+		act_role->noa = (u8)wl_linfo[i].noa;
+		act_role->noa_dur = (u8)(wl_linfo[i].noa_duration/1000);
+		hal_mem_cpy(h, &cid_ch[cnt-1], &wl_linfo[i].chdef, sz);
+		cid_phy[cnt-1] = wl_linfo[i].phy;
+		cid_role[cnt-1] = wl_linfo[i].role;
+		wl_rinfo->role_map |= BIT(wl_linfo[i].role);
+
+		if (rid == i)
+			phy_now = act_role->phy;
+
+		/* only if client connect for p2p-Go/AP */
+		if (wl_linfo[i].role == PHL_RTYPE_P2P_GO ||
+		     wl_linfo[i].role == PHL_RTYPE_AP) {
+			if (wl_linfo[i].client_cnt > 1)
+				client_joined = true;
+			if (client_cnt_last[i] < wl_linfo[i].client_cnt &&
+			    wl_linfo[i].chdef.band == BAND_ON_24G)
+				client_inc_2g = true;
+			act_role->client_cnt = (u8)wl_linfo[i].client_cnt;
+		} else {
+			act_role->client_cnt = 0;
+		}
+		/* only one noa-role exist */
+		if (act_role->noa && act_role->noa_dur > 0) {
+			noa_duration = act_role->noa_dur;
+		}
+
+		/* Check dbcc band, the last band may overwrite the former role,
+		 * This will be modified in the following code.
+		 */
+		if (h->dbcc_en) {
+			phy_dbcc = wl_linfo[i].phy;
+			wl_dinfo->role[phy_dbcc] |= BIT(wl_linfo[i].role);
+			wl_dinfo->op_band[phy_dbcc] = wl_linfo[i].chdef.band;
+		}
+
+		if (wl_linfo[i].chdef.band != BAND_ON_24G) {
+			cnt_5g++;
+			b5g = true;
+		} else {
+			if (((wl_linfo[i].role == PHL_RTYPE_P2P_GO ||
+			    wl_linfo[i].role == PHL_RTYPE_AP) &&
+			    client_joined) ||
+			    wl_linfo[i].role == PHL_RTYPE_P2P_GC)
+				wl_rinfo->p2p_2g = 1;
+
+			if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED &&
+			    (wl_linfo[i].mode & WLAN_MD_11AX) &&
+			    (wl->ap_type == BTC_SPAP_SPTN))
+				btc->mdinfo.wa_type |= BTC_WA_NULL_AP;
+			else
+				btc->mdinfo.wa_type &= ~ BTC_WA_NULL_AP;
+
+			if (wl_linfo[i].mode == WLAN_MD_11B ||
+			    wl_linfo[i].mode == WLAN_MD_11G ||
+			    wl_linfo[i].mode == WLAN_MD_11BG)
+				wl->legacy_mode = true;
+			else if (wl_linfo[i].mode & WLAN_MD_11AX)
+				wl->he_mode = true;
+
+			cnt_2g++;
+			b2g = true;
+		}
+
+		if (act_role->band == BAND_ON_5G && act_role->ch >= 100)
+			wl->is_5g_hi_channel = 1;
+		else
+			wl->is_5g_hi_channel = 0;
+	}
+
+	wl_rinfo->connect_cnt = cnt;
+	wl->client_cnt_inc_2g = (u8)client_inc_2g;
+
+	/* Be careful to change the following sequence!! */
+	if (cnt == 0) {
+		mode = BTC_WLINK_NOLINK;
+		wl_rinfo->role_map = BIT(PHL_RTYPE_NONE);
+	} else if (!b2g && b5g) {
+		mode = BTC_WLINK_5G;
+	} else if (wl_rinfo->role_map & BIT(PHL_RTYPE_NAN)) {
+		mode = BTC_WLINK_2G_NAN;
+	} else if (cnt > BTC_TDMA_WLROLE_MAX) {
+		mode = BTC_WLINK_OTHER;
+	} else if (h->dbcc_en) {
+		mode = _chk_dbcc(btc, cid_ch, cid_phy, cid_role, &dbcc_2g_phy);
+
+		/* correct 2G-located PHY band for gnt ctrl */
+		if (dbcc_2g_phy < HW_PHY_MAX)
+			wl_dinfo->op_band[dbcc_2g_phy] = BAND_ON_24G;
+	} else if (b2g && b5g && cnt == 2) {
+		mode = BTC_WLINK_25G_MCC;
+	} else if (!b5g && cnt == 2) { /* cnt_connect = 2 */
+		if (_chk_role_ch_group(btc, cid_ch[0], cid_ch[cnt - 1]))
+			mode = BTC_WLINK_2G_SCC;
+		else
+			mode = BTC_WLINK_2G_MCC;
+	} else if (!b5g && cnt == 1) { /* cnt_connect = 1 */
+		mode = _get_role_link_mode(btc, cid_role[0]);
+	}
+
+	wl_rinfo->link_mode = mode;
+	_update_role_link_mode(btc, client_joined, noa_duration);
+
+	if (reason == PHL_EN_DBCC_START ||
+	    reason == PHL_DIS_DBCC_START) {
+		wl->status.map.dbccing = 1;
+		_set_btc_timer(btc, BTC_TIMER_EVENT_1, BTC_SPKT_MAXT);
+	} else if (reason == PHL_EN_DBCC_DONE ||
+		   reason == PHL_DIS_DBCC_DONE) {
+		wl->status.map.dbccing = 0;
+	} else if (!h->dbcc_en ||
+		   (phy_now == dbcc_2g_phy && !wl->status.map.dbccing)) {
+		if (reason == PHL_LINK_STARTED ||
+		    reason == PHL_EN_LINK_START ||
+		    reason == PHL_DIS_LINK_START) {
+			wl->status.map.connecting = 1;
+			_set_btc_timer(btc, BTC_TIMER_WL_SPECPKT,BTC_SPKT_MAXT);
+		} else {
+			wl->status.map.connecting = 0;
+		}
+
+		if (reason == PHL_LINK_STOPPED ||
+		    reason == PHL_EN_LINK_DONE)
+			wl->status.map._4way = 0;
+	}
+
+	if (wl_rinfo->dbcc_en != (u32)h->dbcc_en) {
+		wl_rinfo->dbcc_chg = 1;
+		wl_rinfo->dbcc_en = (u32)h->dbcc_en;
+		btc->cx.cnt_wl[BTC_WCNT_DBCC_CHG]++;
+	}
+
+	if (h->dbcc_en) {
+		wl_rinfo->dbcc_2g_phy = (u32)dbcc_2g_phy;
+
+		if (dbcc_2g_phy == HW_PHY_1)
+			mac_id = HW_PHY_1;
+
+		_update_dbcc_band(HW_PHY_0);
+		_update_dbcc_band(HW_PHY_1);
+	}
+
+	cmw_ap_new = (wl->ap_type == BTC_SPAP_CMW) ? 1 : 0;
+	if (dm->cmw_ap != cmw_ap_new && btc->cx.other.type == BTC_EXTSOC_FUNC_THREAD) {
+		buf[0] = 0xf; /* for TIS issue */
+		buf[1] = cmw_ap_new;
+		if (_send_fw_cmd(btc, SET_H2C_TEST, buf, 2))
+			dm->cmw_ap = cmw_ap_new;
+	}
+	_set_wl_req_mac(btc, mac_id);
+	hal_btc_fw_set_drv_info(btc, CXDRVINFO_ROLE);
+}
+
+static u32 _get_wl_tp(struct btc_t *btc, u8 traffic_dir,
+		      struct rtw_stats *phl_stats)
+{
+	struct btc_traffic *wl_tra = &btc->cx.wl.traffic;
+	u32 now_time, last_time;
+	u32 wl_tp = 0;
+	u64 dt, dbyte, now_byte, last_byte;
+
+	if (traffic_dir == TRAFFIC_UL) {
+		now_time = phl_stats->last_tx_time_ms;
+		now_byte = phl_stats->tx_byte_total;
+		last_time = wl_tra->tx_time;
+		last_byte = wl_tra->tx_byte;
+		wl_tra->tx_time = now_time;
+		wl_tra->tx_byte = now_byte;
+		wl_tp = wl_tra->tx_tp;
+
+	} else {
+		now_time = phl_stats->last_rx_time_ms;
+		now_byte = phl_stats->rx_byte_total;
+		last_time = wl_tra->rx_time;
+		last_byte = wl_tra->rx_byte;
+		wl_tra->rx_time = now_time;
+		wl_tra->rx_byte = now_byte;
+		wl_tp = wl_tra->rx_tp;
+	}
+
+	if (now_time >= last_time)
+		dt = now_time - last_time;
+	else
+		dt = (0xffffffffffffffff - last_time) + now_time;
+
+	if (now_byte >= last_byte)
+		dbyte = now_byte - last_byte;
+	else
+		dbyte = (0xffffffffffffffff - last_byte) + now_byte;
+
+	if (dt > 0)
+		wl_tp = ((u32)_os_division64(dbyte, dt)/125);
+
+	return (wl_tp);
+}
+
+static u32 _update_wl_rx_err_ratio(struct btc_t *btc, u8 type)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	u32 rx_err = 0, rx_err_diff = 0, rx_ok = 0, rx_ok_diff = 0;
+	u32 err_ratio = 0, rx_cnt = 0;
+	u8 rx_ok_idx = BTC_WCNT_RX_OK_LAST, rx_err_idx = BTC_WCNT_RX_ERR_LAST;
+
+	/* Skip HW I/O during DBCC process/rf-off/LPS */
+	if (wl_smap->dbccing || wl_smap->rf_off || wl_smap->lps != BTC_LPS_OFF)
+		return 0;
+
+	if (!btc->dm.rx_err_rpt_en)
+		return 0;
+
+	if (type == 1) {
+		rx_ok_idx = BTC_WCNT_RX_OK_LAST2S;
+		rx_err_idx = BTC_WCNT_RX_ERR_LAST2S;
+	}
+
+	rtw_hal_mac_get_rx_cnt(h->hal_priv, wl->pta_req_mac, 0, &rx_ok);
+	rtw_hal_mac_get_rx_cnt(h->hal_priv, wl->pta_req_mac, 1, &rx_err);
+
+	rx_ok_diff = rx_ok - cx->cnt_wl[rx_ok_idx];
+	rx_err_diff = rx_err - cx->cnt_wl[rx_err_idx];
+	cx->cnt_wl[rx_ok_idx] = rx_ok;
+	cx->cnt_wl[rx_err_idx] = rx_err;
+	rx_cnt = rx_ok_diff + rx_err_diff;
+
+	if (rx_cnt != 0)
+		err_ratio = (rx_err_diff * 100) / rx_cnt;
+
+	if (type == 1)
+		cx->cnt_wl[BTC_WCNT_RX_LAST] = rx_cnt;
+
+	return (err_ratio);
+}
+
+static void _update_wl_client_status(struct btc_t *btc,
+				     struct btc_wl_link_info *linfo)
+{
+	struct btc_dm *dm = &btc->dm;
+
+	if (linfo->client_cnt == 0) {
+		dm->client_ps_tog1 = 0;
+		dm->client_ps_tog1_last = 0;
+		dm->client_ps_cnt = 0;
+		dm->client_ps_tdma_on = 0;
+		return;
+	}
+
+	if (dm->client_ps_tog1 != dm->client_ps_tog1_last) {
+		if (dm->client_ps_cnt < BTC_CLIENT_PS_MAX)
+			dm->client_ps_cnt++;
+		dm->client_ps_tog1_last = dm->client_ps_tog1;
+	} else {  /* no power-save toggle */
+		if (dm->client_ps_cnt > 0)
+			dm->client_ps_cnt--;
+	}
+
+	if (!dm->client_ps_tdma_on &&
+		dm->client_ps_cnt == BTC_CLIENT_PS_MAX) {
+		dm->client_ps_tdma_on = 1;
+	} else if (dm->client_ps_tdma_on &&
+		   dm->client_ps_cnt == 0) {
+		dm->client_ps_tdma_on = 0;
+	}
+}
+
+static void _update_btc_state_map(struct btc_t *btc)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_smap wl = cx->wl.status.map;
+	struct btc_bt_smap bt = cx->bt.link_info.status.map;
+
+	if (wl.connecting || wl._4way || wl.roaming || wl.dbccing) {
+		cx->state_map = BTC_WLINKING;
+	} else if (wl.scan) { /* wl scan */
+		if (bt.inq_pag)
+			cx->state_map = BTC_WSCAN_BSCAN;
+		else
+			cx->state_map = BTC_WSCAN_BNOSCAN;
+	} else if (wl.busy) { /* only busy */
+		if (bt.inq_pag)
+			cx->state_map = BTC_WBUSY_BSCAN;
+		else
+			cx->state_map = BTC_WBUSY_BNOSCAN;
+	} else { /* wl idle */
+		if (bt.inq_pag)
+			cx->state_map = BTC_WIDLE_BSCAN;
+		else
+			cx->state_map = BTC_WIDLE;
+	}
+}
+
+void _run_coex(struct btc_t *btc, const char *reason)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	u8 mode = wl_rinfo->link_mode;
+	u32 len = _os_strlen((u8 *)reason);
+
+	_rsn_cpy(dm->run_reason, (char*)reason, len);
+	_update_dm_step(btc, reason);
+	_update_btc_state_map(btc);
+
+	/* Be careful to change the following function sequence!! */
+	if (btc->ctrl.manual) {
+		PHL_INFO("[BTC], %s(): return for Manual CTRL!!\n", __func__);
+		return;
+	}
+
+	if (!wl_smap->init_ok) {
+		PHL_INFO("[BTC], %s(): return for WL init fail!!\n", __func__);
+		return;
+	}
+
+	if (dm->ntfy_type == BTC_NTFY_CUSTOMERIZE && dm->country_code_proc) {
+		goto exit0;
+	}
+
+	if (wl_smap->rf_off_pre == wl_smap->rf_off &&
+	    wl_smap->lps_pre == wl_smap->lps) {
+		if (dm->ntfy_type == BTC_NTFY_POWER_OFF ||
+		    dm->ntfy_type == BTC_NTFY_RADIO_STATE) {
+			PHL_INFO("[BTC], %s(): return"
+				 " WL rf off state no change\n", __func__);
+			return;
+		}
+
+		if (wl_smap->rf_off == 1 || wl_smap->lps == BTC_LPS_RF_OFF) {
+			PHL_INFO("[BTC], %s(): return"
+				 " for WL rf off state!!\n", __func__);
+			return;
+		}
+	}
+
+	dm->cnt_dm[BTC_DCNT_RUN]++;
+	dm->freerun = false;
+	dm->fddt_train = BTC_FDDT_DISABLE;
+	btc->ctrl.igno_bt = false;
+	bt->scan_rx_low_pri = false;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_ERR_,
+		  "[BTC], %s(): reason=%s, mode=%s(cnt=%d)\n", __func__, reason,
+		  id_to_str(BTC_STR_WLLINK, (u32)wl_rinfo->link_mode),
+		  dm->cnt_dm[BTC_DCNT_RUN]);
+
+	dm->fddt_train_chk = _chk_fddtrain(btc); /* check if meet fdd train  */
+	dm->freerun_chk = _chk_freerun(btc); /* check if meet freerun */
+
+	if (btc->ctrl.always_freerun) {
+		_action_freerun(btc);
+		btc->ctrl.igno_bt = true;
+	} else if (dm->wl_only) {
+		_action_wl_only(btc);
+		btc->ctrl.igno_bt = true;
+	} else if (wl_smap->rf_off || wl_smap->lps || dm->bt_only) {
+		_action_wl_off(btc);
+		btc->ctrl.igno_bt = true;
+	} else if (!(bt->run_patch_code && bt->enable.now) && !cx->other.type) {
+		_action_bt_off(btc);
+	} else if (dm->ntfy_type == BTC_NTFY_INIT_COEX) {
+		_action_wl_init(btc);
+	} else if (cx->bt.whql_test) {
+		_action_bt_whql(btc);
+	} else if (wl->rfk_info.state != BTC_WRFK_STOP) {
+		_action_wl_rfk(btc);
+	} else if (wl->status.val & BTC_WL_LINK_SCAN) {
+		bt->scan_rx_low_pri = true;
+		_action_wl_scan(btc);
+	} else {
+		switch (mode) {
+		case BTC_WLINK_NOLINK:
+			_action_wl_nc(btc);
+			break;
+		case BTC_WLINK_2G_STA:
+			if (wl_smap->traffic_dir & BIT(TRAFFIC_DL))
+				bt->scan_rx_low_pri = true;
+			_action_wl_2g_sta(btc);
+			break;
+		case BTC_WLINK_2G_AP:
+			bt->scan_rx_low_pri = true;
+			_action_wl_2g_ap(btc);
+			break;
+		case BTC_WLINK_2G_GO:
+			bt->scan_rx_low_pri = true;
+			_action_wl_2g_go(btc);
+			break;
+		case BTC_WLINK_2G_GC:
+			bt->scan_rx_low_pri = true;
+			_action_wl_2g_gc(btc);
+			break;
+		case BTC_WLINK_2G_SCC:
+			bt->scan_rx_low_pri = true;
+			_action_wl_2g_scc(btc);
+			break;
+		case BTC_WLINK_2G_MCC:
+			bt->scan_rx_low_pri = true;
+			_action_wl_2g_mcc(btc);
+			break;
+		case BTC_WLINK_25G_MCC:
+			bt->scan_rx_low_pri = true;
+			_action_wl_25g_mcc(btc);
+			break;
+		case BTC_WLINK_5G:
+			_action_wl_5g(btc);
+			break;
+		case BTC_WLINK_2G_NAN:
+			_action_wl_2g_nan(btc);
+			break;
+		default:
+			_action_wl_other(btc);
+			break;
+		}
+	}
+
+exit0:
+
+	_action_common(btc);
+}
+
+void _update_bt_scbd(struct btc_t *btc, bool only_update)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_wl_info *wl = &cx->wl;
+	u32 val, any_bt_connect;
+	u8 cc_now, cc_last, cc_btc;
+	bool status_change = false, bt_link_change = false;
+
+	if (!btc->chip->scbd)
+		return;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s()\n", __func__);
+	val = _read_scbd(btc);
+
+	if (val == 0xffffffff) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_ERR_,
+			  "[BTC], %s return by invalid scbd value\n", __func__);
+		return;
+	}
+
+	if (!(val & BTC_BSCB_ON))
+		bt->enable.now = 0;
+	else
+		bt->enable.now = 1;
+
+	if (bt->enable.now != bt->enable.last) {
+		status_change = true;
+		bt_link_change = true;
+		/* reset bt info if bt re-enable */
+		if (bt->enable.now && !bt->enable.last) {
+			_reset_btc_var(btc, BTC_RESET_BTINFO);
+			cx->cnt_bt[BTC_BCNT_REENABLE]++;
+			bt->enable.now = 1;
+			_write_scbd(btc, BTC_WSCB_COUNTRY_CODE, true);
+		}
+	}
+
+	/*send country code if bt scb[15]=1*/
+	cc_now = !!(val & BTC_BSCB_COUNTRY_CODE);
+	cc_last = !!(bt->scbd & BTC_BSCB_COUNTRY_CODE);
+	cc_btc = !!(wl->scbd & BTC_WSCB_COUNTRY_CODE);
+	if ((cc_now != cc_last) || (cc_btc && cc_now)) {
+		status_change = true;
+		if (cc_now) {
+			btc->dm.country_code_retry = 0;
+			_send_bt_country_code(btc);
+			_write_scbd(btc, BTC_WSCB_COUNTRY_CODE, false);
+			_set_btc_timer(btc, BTC_TIMER_COUNTRYCODE,
+			               BTC_COUNTRY_CODE_TIME);
+		} else {
+			hal_btc_notify_ps_tdma(btc, btc->hal->btc_ctrl.lps);
+			btc->dm.error &= ~BTC_DMERR_CCODE_BTNORS;
+		}
+	}
+
+	bt->enable.last = bt->enable.now;
+	bt->scbd = val;
+	bt->mbx_avl = !!(val & BTC_BSCB_ACT);
+
+	if (bt->whql_test != (u32)(!!(val & BTC_BSCB_WHQL)))
+		status_change = true;
+
+	bt->whql_test = !!(val & BTC_BSCB_WHQL);
+	bt->btg_type = (val & BTC_BSCB_BT_S1 ? BTC_BT_BTG: BTC_BT_ALONE);
+	bt->link_info.a2dp_desc.exist = !!(val & BTC_BSCB_A2DP_ACT);
+
+	bt->lna_constrain = 4 + !!(val & BTC_BSCB_BT_LNAB0) +
+			    (!!(val & BTC_BSCB_BT_LNAB1)) * 2;
+
+	/* if rfk run 1->0 */
+	if (bt->rfk_info.map.run && !(val & BTC_BSCB_RFK_RUN))
+		status_change = true;
+
+	bt->rfk_info.map.run  = !!(val & BTC_BSCB_RFK_RUN);
+	bt->rfk_info.map.req = !!(val & BTC_BSCB_RFK_REQ);
+	bt->hi_lna_rx = !!(val & BTC_BSCB_BT_HILNA);
+	any_bt_connect = !!(val & BTC_BSCB_BT_CONNECT);
+
+	/* if connect change */
+	if (bt->link_info.status.map.connect != any_bt_connect) {
+		status_change = true;
+		bt_link_change = true;
+	}
+
+	bt->link_info.status.map.connect = any_bt_connect;
+	bt->run_patch_code = !!(val & BTC_BSCB_PATCH_CODE);
+
+	if (bt_link_change) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s: bt status change!!\n", __func__);
+		_send_phl_evnt(btc, NULL, 0, BTC_HMSG_BT_LINK_CHG);
+	}
+
+	if (!only_update && status_change)
+		_run_coex(btc, __func__);
+}
+
+/******************************************************************************
+ *
+ * coexistence operations for external notifications
+ *
+ *****************************************************************************/
+static void _ntfy_power_on(struct btc_t *btc)
+{
+	/* no action for power on, beacuse power-on move halmac API
+	* the _ntfy_power_on = _ntfy_init_coex
+	*/
+}
+
+static void _ntfy_power_off(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+
+	PHL_INFO("[BTC], %s()\n", __func__);
+	btc->dm.cnt_notify[BTC_NTFY_POWER_OFF]++;
+	btc->dm.ntfy_type = BTC_NTFY_POWER_OFF;
+
+	wl->status.map.rf_off = 1;
+	wl->status.map.busy = 0;
+	wl->status.map.lps = BTC_LPS_OFF;
+	_write_scbd(btc, BTC_WSCB_ALL, false);
+	_set_fw_rpt(btc, RPT_EN_ALL, 0);
+
+	_run_coex(btc, __func__);
+	wl->status.map.rf_off_pre = wl->status.map.rf_off;
+}
+
+static void _ntfy_init_coex(struct btc_t *btc, u8 mode)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_chip_ops *ops = btc->chip->ops;
+	u32 val;
+
+	PHL_INFO("[BTC], %s(): mode=%d\n", __func__, mode);
+	dm->cnt_notify[BTC_NTFY_INIT_COEX]++;
+	dm->ntfy_type = BTC_NTFY_INIT_COEX;
+
+	if (mode == BTC_MODE_MECHANISM_INIT) {
+		_btmr_start(btc);
+		return;
+	}
+
+	wl->coex_mode = mode;
+	btc->ctrl.always_freerun = (mode == BTC_MODE_COTX? 1 : 0);
+	dm->wl_only = (mode == BTC_MODE_WL? 1 : 0);
+	dm->bt_only = (mode == BTC_MODE_BT? 1 : 0);
+	dm->vid = ((btc->phl->id.id) >> 8) & 0xff;
+	wl->status.map.rf_off = (mode == BTC_MODE_WLOFF? 1 : 0);
+
+	if (!wl->status.map.init_ok) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_ERR_,
+			  "[BTC], %s(): return for WL init fail!!\n", __func__);
+		dm->error |= BTC_DMERR_INIT;
+		return;
+	}
+
+	dm->btc_initing = 1;
+
+	/* Setup RF front end type from EFuse RFE type, MUST before init_cfg*/
+	if (ops && ops->set_rfe)
+		ops->set_rfe(btc);
+
+	if (ops && ops->init_cfg)
+		ops->init_cfg(btc);
+
+	_write_scbd(btc, BTC_WSCB_ACTIVE | BTC_WSCB_ON | BTC_WSCB_BTLOG, true);
+	_update_bt_scbd(btc, true);
+
+#if (BTC_WL_MAX_ROLE_NUMBER < MAX_WIFI_ROLE_NUMBER)
+	dm->error |= BTC_DMERR_ROLE_NUM_MISMATCH;
+	PHL_INFO("[BTC], %s(): ROLE_NUMBER conflict!!\n", __func__);
+#endif
+
+	/* check h2c/c2h structure size */
+	hal_btc_fw_chk_struct(btc);
+
+	/* check PTA control owner to avoid BT coex issue */
+	_read_cx_ctrl(btc, &val);
+	dm->pta_owner = val;
+	if (dm->pta_owner == BTC_CTRL_BY_WL &&
+	    ((btc->chip->hw & BTC_FEAT_PTA_ONOFF_CTRL) == 0)) {
+		PHL_INFO("[BTC], %s(): PTA owner warning!!\n", __func__);
+		dm->error |= BTC_DMERR_PTA_OWNER;
+	}
+
+	_set_init_info(btc);
+	_set_fw_rpt(btc, RPT_EN_MREG, 1);
+
+	_run_coex(btc, __func__);
+	dm->btc_initing = 0;
+}
+
+static void _ntfy_scan_start(struct btc_t *btc, u8 phy_idx, u8 band)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+
+	PHL_INFO("[BTC], %s(): phy_idx=%d, band=%d\n", __func__, phy_idx, band);
+	btc->dm.cnt_notify[BTC_NTFY_SCAN_START]++;
+	btc->dm.ntfy_type = BTC_NTFY_SCAN_START;
+
+	wl->scan_info.band[phy_idx] = band;
+	wl->scan_info.phy_map |= BIT(phy_idx);
+
+	if (wl_rinfo->dbcc_en) {
+		wl->dbcc_info.scan_band[phy_idx] = band;
+		_update_dbcc_band(phy_idx);
+
+		if (wl_rinfo->dbcc_2g_phy != phy_idx) /* return if not-2G PHY */
+			return;
+	}
+
+	wl->status.map.scan = true;
+
+	_run_coex(btc, __func__);
+}
+
+static void _ntfy_scan_finish(struct btc_t *btc, u8 phy_idx)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+
+	PHL_INFO("[BTC], %s(): phy_idx=%d\n", __func__, phy_idx);
+	btc->dm.cnt_notify[BTC_NTFY_SCAN_FINISH]++;
+	btc->dm.ntfy_type = BTC_NTFY_SCAN_FINISH;
+
+	wl->status.map.scan = false;
+	wl->scan_info.phy_map &= ~(u8)BIT(phy_idx);
+
+	if (wl_rinfo->dbcc_en)
+		_update_dbcc_band(phy_idx);
+
+	_run_coex(btc, __func__);
+}
+
+static void _ntfy_switch_band(struct btc_t *btc, u8 phy_idx, u8 band)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+
+	PHL_INFO("[BTC], %s(): phy_idx=%d, band=%d\n", __func__, phy_idx, band);
+	btc->dm.cnt_notify[BTC_NTFY_SWITCH_BAND]++;
+	btc->dm.ntfy_type = BTC_NTFY_SWITCH_BAND;
+
+#if 0
+	wl->scan_info.band[phy_idx] = band;
+	wl->scan_info.phy_map |= BIT(phy_idx);
+	hal_btc_fw_set_drv_info(btc, CXDRVINFO_SCAN);
+#endif
+
+	if (wl_rinfo->dbcc_en) {
+		/* wl->dbcc_info.scan_band[phy_idx] = band; */
+		_update_dbcc_band(phy_idx);
+	}
+	_run_coex(btc, __func__);
+}
+
+static void _ntfy_specific_packet(struct btc_t *btc, u8 pkt_type)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	u32 cnt;
+
+	if (btc->bt_req_en)
+		return;
+
+	switch (pkt_type) {
+	case PKT_EVT_DHCP:
+		cnt = ++cx->cnt_wl[BTC_WCNT_DHCP];
+		PHL_INFO("[BTC], %s(): pkt_type=%d, DHCP cnt=%d \n",
+			 __func__, pkt_type, cnt);
+		wl->status.map.connecting = 1;
+		_set_btc_timer(btc, BTC_TIMER_WL_SPECPKT, BTC_SPKT_MAXT);
+		break;
+	case PKT_EVT_EAPOL_START:
+		cnt = ++cx->cnt_wl[BTC_WCNT_EAPOL];
+		PHL_INFO("[BTC], %s(): pkt_type=%d, EAPOL_Start cnt=%d \n",
+			 __func__, pkt_type, cnt);
+		wl->status.map._4way = 1;
+		/* To avoid no EAPOL_END and lead to _4way always = 1  */
+		_set_btc_timer(btc, BTC_TIMER_WL_SPECPKT, BTC_SPKT_MAXT);
+		break;
+	case PKT_EVT_EAPOL_END:
+		cnt = ++cx->cnt_wl[BTC_WCNT_EAPOL];
+		PHL_INFO("[BTC], %s(): pkt_type=%d, EAPOL_End cnt=%d \n",
+			 __func__, pkt_type, cnt);
+		wl->status.map._4way = 0;
+		break;
+	default:
+	case PKT_EVT_ARP:
+		cnt = ++cx->cnt_wl[BTC_WCNT_ARP];
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): pkt_type=%d,"
+			  " ARP cnt=%d\n", __func__, pkt_type, cnt);
+		return;
+	}
+
+	btc->dm.cnt_notify[BTC_NTFY_SPECIAL_PACKET]++;
+	btc->dm.ntfy_type = BTC_NTFY_SPECIAL_PACKET;
+	_run_coex(btc, __func__);
+}
+
+void _update_dm_step(struct btc_t *btc, const char *strin)
+{
+	struct btc_dm *dm = &btc->dm;
+	u32 store_index = 0, len = _os_strlen((u8 *)strin);
+
+	dm->dm_step.cnt++;
+	if (dm->dm_step.cnt == 0)
+		dm->dm_step.cnt = 1;
+
+	store_index = ((dm->dm_step.cnt-1) % BTC_DM_MAXSTEP);
+	_rsn_cpy(dm->dm_step.step[store_index], (char*)strin, len);
+}
+
+static u8 _update_bt_rssi_level(struct btc_t *btc, u8 rssi)
+{
+	struct btc_bt_info *bt = &btc->cx.bt;
+	u8 *rssi_st = NULL;
+	u8 rssi_th = 0, j, rssi_level = 0;
+
+	/* for rssi locate in which {40, 36, 31, 28}
+	 * if rssi >= 40% (-60dBm) --> rssi_level = 4
+	 * if 36% <= rssi < 40%    --> rssi_level = 3
+	 * if 31% <= rssi < 36%    --> rssi_level = 2
+	 * if 28% <= rssi < 31%    --> rssi_level = 1
+	 * if rssi < 28%           --> rssi_level = 0
+	 */
+
+	/* check if rssi across bt_rssi_thres boundary */
+	for (j = 0; j < BTC_BT_RSSI_THMAX; j++) {
+		rssi_th = btc->chip->bt_rssi_thres[j];
+		rssi_st = &bt->link_info.rssi_state[j];
+
+		*rssi_st = _update_rssi_state(btc, *rssi_st, rssi, rssi_th);
+
+		if (BTC_RSSI_HIGH(*rssi_st)) {
+			rssi_level = BTC_BT_RSSI_THMAX - j;
+			break;
+		}
+	}
+
+	return rssi_level;
+}
+
+static void _reset_query_flag(struct btc_t *btc, u8 profile)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_a2dp_desc *a2dp = &bt->link_info.a2dp_desc;
+	struct btc_bt_hid_desc *hid = &bt->link_info.hid_desc;
+	struct btc_bt_pan_desc *pan = &bt->link_info.pan_desc;
+
+	bt->scan_info_update = 0;
+
+	if (profile & BTC_BT_A2DP) {
+		a2dp->handle_update = 0;
+		a2dp->devinfo_query = 0;
+		a2dp->vendor_id = 0;
+		a2dp->flush_time = 0;
+		a2dp->play_latency = 1;
+		hal_mem_set(h, a2dp->lists, 0, sizeof(a2dp->lists));
+		_set_btc_timer(btc, BTC_TIMER_BT_A2DPPLAY, BTC_REA2DP_MAXT);
+	}
+
+	if (profile & BTC_BT_HID) {
+		hid->handle_update = 0;
+		hid->devinfo_query = 0;
+		hal_mem_set(h, hid->lists, 0, sizeof(hid->lists));
+	}
+
+	if (profile & BTC_BT_PAN) {
+		pan->handle_update = 0;
+		pan->devinfo_query = 0;
+		hal_mem_set(h, pan->lists, 0, sizeof(pan->lists));
+	}
+}
+
+static void _update_bt_info(struct btc_t *btc, u8 *buf, u32 len)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_bt_hfp_desc *hfp = &b->hfp_desc;
+	struct btc_bt_hid_desc *hid = &b->hid_desc;
+	struct btc_bt_a2dp_desc *a2dp = &b->a2dp_desc;
+	struct btc_bt_pan_desc *pan = &b->pan_desc;
+	struct btc_bt_leaudio_desc *leaudio = &b->leaudio_desc;
+	union btc_btinfo btinfo;
+	bool bt_link_change = false;
+	u8 profile_chg = 0;
+
+	if (buf[BTC_BTINFO_L1] != BTC_BTINFO_LEN)
+		return;
+
+	/* return if bt info match last bt-info  */
+	if (!hal_mem_cmp(h, bt->raw_info, buf, BTC_BTINFO_MAX)) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s return by bt-info duplicate!\n", __func__);
+		cx->cnt_bt[BTC_BCNT_INFOSAME]++;
+		return;
+	}
+
+	hal_mem_cpy(h, bt->raw_info, buf, BTC_BTINFO_MAX);
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s: bt_info[2]=0x%02x\n", __func__, bt->raw_info[2]);
+
+	/* b->profile_cnt.last MUST here!! (used in _set_bt_afh_info()) */
+	b->profile_cnt.last = b->profile_cnt.now;
+	b->profile_cnt.now = 0;
+	hid->type = 0;
+
+	/* ======= parse raw info low-Byte2 ======= */
+	btinfo.val = bt->raw_info[BTC_BTINFO_L2];
+	if (b->status.map.connect != btinfo.lb2.connect)
+		bt_link_change = true;
+	b->status.map.connect = btinfo.lb2.connect;
+	b->status.map.sco_busy = btinfo.lb2.sco_busy;
+	b->status.map.acl_busy = btinfo.lb2.acl_busy;
+	b->status.map.inq_pag = btinfo.lb2.inq_pag;
+	bt->inq_pag.now = btinfo.lb2.inq_pag;
+	cx->cnt_bt[BTC_BCNT_INQPAG] += !!(bt->inq_pag.now && !bt->inq_pag.last);
+
+	hfp->exist = btinfo.lb2.hfp;
+	b->profile_cnt.now += (u8)hfp->exist;
+	hid->exist = btinfo.lb2.hid;
+	b->profile_cnt.now += (u8)hid->exist;
+	a2dp->exist = btinfo.lb2.a2dp;
+	b->profile_cnt.now += (u8)a2dp->exist;
+	pan->exist = btinfo.lb2.pan;
+	b->profile_cnt.now += (u8)pan->exist;
+	b->profile_cnt.now += (u8)leaudio->bis_cnt;
+	b->profile_cnt.now += (u8)leaudio->cis_cnt;
+	btc->dm.trx_info.bt_profile = (btinfo.val & 0xf0) >>4;
+
+	/* ======= parse raw info low-Byte3 ======= */
+	btinfo.val = bt->raw_info[BTC_BTINFO_L3];
+
+	cx->cnt_bt[BTC_BCNT_RETRY] += btinfo.lb3.retry;
+	b->cqddr = btinfo.lb3.cqddr;
+	cx->cnt_bt[BTC_BCNT_INQ] += !!(btinfo.lb3.inq && !bt->inq);
+	bt->inq = btinfo.lb3.inq;
+	cx->cnt_bt[BTC_BCNT_PAGE] += !!(btinfo.lb3.pag && !bt->pag);
+	bt->pag = btinfo.lb3.pag;
+
+	b->status.map.mesh_busy = btinfo.lb3.mesh_busy;
+	/* ======= parse raw info high-Byte0 ======= */
+	btinfo.val = bt->raw_info[BTC_BTINFO_H0];
+	/* raw val is dBm unit, translate from -100~ 0dBm to 0~100%*/
+	b->rssi = btc->chip->ops->bt_rssi(btc, btinfo.hb0.rssi);
+	bt->rssi_level = _update_bt_rssi_level(btc, (u8)b->rssi);
+	btc->dm.trx_info.bt_rssi = bt->rssi_level;
+
+	/* ======= parse raw info high-Byte1 ======= */
+	btinfo.val = bt->raw_info[BTC_BTINFO_H1];
+	if (b->status.map.ble_connect != btinfo.hb1.ble_connect)
+		bt_link_change = true;
+	b->status.map.ble_connect = btinfo.hb1.ble_connect;
+	if (btinfo.hb1.ble_connect) {
+		if (hid->exist)
+			hid->type |= BTC_HID_BLE;
+		else if (btinfo.hb1.voice)
+			hid->type |= BTC_HID_RCU_VOICE;
+		else
+			hid->type |= BTC_HID_RCU;
+	}
+
+	cx->cnt_bt[BTC_BCNT_REINIT] += !!(btinfo.hb1.reinit && !bt->reinit);
+	bt->reinit = btinfo.hb1.reinit;
+	cx->cnt_bt[BTC_BCNT_RELINK] += !!(btinfo.hb1.relink && !b->relink.now);
+	b->relink.now = btinfo.hb1.relink;
+	cx->cnt_bt[BTC_BCNT_IGNOWL] += !!(btinfo.hb1.igno_wl && !bt->igno_wl);
+	bt->igno_wl = btinfo.hb1.igno_wl;
+
+	bt->ble_scan_en = btinfo.hb1.ble_scan;
+
+	cx->cnt_bt[BTC_BCNT_ROLESW] += !!(btinfo.hb1.role_sw && !b->role_sw);
+	b->role_sw = btinfo.hb1.role_sw;
+
+	b->multi_link.now = btinfo.hb1.multi_link;
+	if (b->multi_link.now != b->multi_link.last)
+		bt_link_change = true;
+
+	/* ======= parse raw info high-Byte2 ======= */
+	btinfo.val = bt->raw_info[BTC_BTINFO_H2];
+	pan->active = !!btinfo.hb2.pan_active;
+
+	cx->cnt_bt[BTC_BCNT_AFH] += !!(btinfo.hb2.afh_update && !b->afh_update);
+	b->afh_update = btinfo.hb2.afh_update;
+	a2dp->active = btinfo.hb2.a2dp_active;
+	b->slave_role = btinfo.hb2.slave;
+	hid->slot_info = btinfo.hb2.hid_slot;
+
+	if (hid->pair_cnt != btinfo.hb2.hid_cnt)
+		bt_link_change = true;
+
+	hid->pair_cnt = btinfo.hb2.hid_cnt;
+	if (!b->status.map.ble_connect || hid->pair_cnt > 1)
+		hid->type |= (hid->slot_info == BTC_HID_218?
+			      BTC_HID_218 : BTC_HID_418);
+
+	/* ======= parse raw info high-Byte3 ======= */
+	btinfo.val = bt->raw_info[BTC_BTINFO_H3];
+	a2dp->bitpool = btinfo.hb3.a2dp_bitpool;
+
+	cx->cnt_bt[BTC_BCNT_RATECHG] += !!(b->tx_3M ^ (u32)btinfo.hb3.tx_3M);
+	b->tx_3M = (u32)btinfo.hb3.tx_3M;
+	a2dp->sink = btinfo.hb3.a2dp_sink;
+
+	if (bt_link_change) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s: bt link change!!\n", __func__);
+		_send_phl_evnt(btc, NULL, 0, BTC_HMSG_BT_LINK_CHG);
+	}
+
+	if (bt->igno_wl && !cx->wl.status.map.rf_off)
+		_set_bt_ignore_wl_act(btc, false);
+
+	/* reset after A2DP stop->play */
+	if (!a2dp->exist_last && a2dp->exist)
+		profile_chg |= BTC_BT_A2DP;
+
+	/* reset after PAN idle->busy */
+	if (!pan->exist_last && pan->exist)
+		profile_chg |= BTC_BT_PAN;
+
+	/* reset after HID pair-count change */
+	if (hid->pair_cnt_last != hid->pair_cnt)
+		profile_chg |= BTC_BT_HID;
+
+	/* "record the lateset state" for next comparison MUST after above */
+	b->relink.last = b->relink.now;
+	a2dp->exist_last = a2dp->exist;
+	pan->exist_last = pan->exist;
+	hid->exist_last = hid->exist;
+	hfp->exist_last = hfp->exist;
+	leaudio->exist_last = leaudio->exist;
+	leaudio->bis_cnt_last = leaudio->bis_cnt;
+	leaudio->cis_cnt_last = leaudio->cis_cnt;
+	hid->pair_cnt_last = hid->pair_cnt;
+	b->multi_link.last = b->multi_link.now;
+	bt->inq_pag.last = bt->inq_pag.now;
+
+	if (bt_link_change || profile_chg)
+		_reset_query_flag(btc, profile_chg);
+
+	if (!btc->ctrl.igno_bt)
+		_run_coex(btc, __func__);
+}
+
+static void _query_bt_dev_info(struct btc_t *btc, u8 sub_id, u8 handle)
+{
+	u8 buf[2] = {0};
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): query bt dev info\n", __func__);
+
+	buf[0] = sub_id;
+	buf[1] = handle;
+	_send_fw_cmd(btc, SET_BT_QUERY_DEV_INFO, buf, 2);
+}
+
+static void _query_bt_dev_list(struct btc_t *btc, u8 sub_id)
+{
+	u8 buf = 0;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): query bt dev list\n", __func__);
+
+	buf = sub_id;
+	_send_fw_cmd(btc, SET_BT_QUERY_DEV_LIST, &buf, 1);
+}
+
+static void _query_bt_dev(struct btc_t *btc)
+{
+	struct btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
+	struct btc_bt_hfp_desc *hfp = &bt_linfo->hfp_desc;
+	struct btc_bt_hid_desc *hid = &bt_linfo->hid_desc;
+	struct btc_bt_a2dp_desc *a2dp = &bt_linfo->a2dp_desc;
+	struct btc_bt_pan_desc *pan = &bt_linfo->pan_desc;
+	u8 i, h;
+
+	if (!(btc->cx.bt.enable.now && btc->cx.bt.run_patch_code))
+		return;
+
+	if (hfp->exist) {
+
+	}
+
+	if (hid->exist) {
+		if (!hid->handle_update) {
+			_query_bt_dev_list(btc, BTC_DEV_HID_LIST);
+		} else if (!hid->devinfo_query || hid->query_timer == 15) {
+			for (i = 0; i < hid->cnt; i++) {
+				h = hid->links[i].h;
+				_query_bt_dev_info(btc, BTC_DEV_HID_BINFO, h);
+				_query_bt_dev_info(btc, BTC_DEV_HID_VINFO, h);
+			}
+
+			hid->query_timer = 0;
+			hid->devinfo_query = 1;
+		}
+
+		if (hid->query_timer < 15)
+			hid->query_timer++;
+	}
+
+	if (a2dp->exist) {
+		if (!a2dp->handle_update) {
+			_query_bt_dev_list(btc, BTC_DEV_A2DP_LIST);
+		} else if (!a2dp->devinfo_query) {
+			for (i = 0; i < a2dp->cnt; i++) {
+				h = a2dp->links[i].h;
+				_query_bt_dev_info(btc, BTC_DEV_A2DP_BINFO, h);
+			}
+
+			a2dp->devinfo_query = 1;
+		}
+	}
+
+	if (pan->exist) {
+		if (!pan->handle_update) {
+			_query_bt_dev_list(btc, BTC_DEV_PAN_LIST);
+		} else if (!pan->devinfo_query) {
+			for (i = 0; i < pan->cnt; i++) {
+				h = pan->links[i].h;
+				_query_bt_dev_info(btc, BTC_DEV_PAN_BINFO, h);
+			}
+
+			pan->devinfo_query = 1;
+		}
+	}
+}
+
+static void _update_bt_dev_info(struct btc_t *btc, u8 *buf, u32 len)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_info *bt = &cx->bt;
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_bt_hid_desc *hid = &b->hid_desc;
+	struct btc_bt_a2dp_desc *a2dp = &b->a2dp_desc;
+	struct btc_bt_pan_desc *pan = &b->pan_desc;
+	struct btc_bt_mb_devinfo *src = (struct btc_bt_mb_devinfo *)buf;
+	u8 i, sz;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC] %s(): BT DEV INFO SubID=0x%x:"
+		  "[0x%x, 0x%x, 0x%x, 0x%x, 0x%x] \n", __func__,
+		  src->sub_id, src->data[0], src->data[1],
+		  src->data[2], src->data[3], src->data[4]);
+
+	switch (src->sub_id) {
+	case BTC_DEV_HID_LIST:
+		sz = sizeof(hid->lists);
+		if (hal_mem_cmp(h, hid->lists, src->data + 1, sz) == 0)
+			return;
+
+		hid->cnt = src->data[0];
+		if (hid->cnt > BTC_DEV_LINK_MAX)
+			hid->cnt = BTC_DEV_LINK_MAX;
+		hal_mem_cpy(h, hid->lists, src->data + 1, sizeof(hid->lists));
+		hal_mem_set(h, hid->links, 0, sizeof(hid->links));
+		for (i = 0; i < hid->cnt; i++)
+			hid->links[i].h = hid->lists[i];
+		hid->handle_update = 1;
+		break;
+	case BTC_DEV_A2DP_LIST:
+		sz = sizeof(a2dp->lists);
+		if (hal_mem_cmp(h, a2dp->lists, src->data+1, sz) == 0)
+			return;
+
+		a2dp->cnt = src->data[0];
+		if (a2dp->cnt > BTC_DEV_LINK_MAX)
+			a2dp->cnt = BTC_DEV_LINK_MAX;
+		hal_mem_cpy(h, a2dp->lists, src->data+1, sizeof(a2dp->lists));
+		hal_mem_set(h, a2dp->links, 0, sizeof(a2dp->links));
+
+		for (i = 0; i < a2dp->cnt; i++)
+			a2dp->links[i].h = a2dp->lists[i];
+		a2dp->handle_update = 1;
+		break;
+	case BTC_DEV_PAN_LIST:
+		sz = sizeof(pan->lists);
+		if (hal_mem_cmp(h, pan->lists, src->data+1, sz) == 0)
+			return;
+
+		pan->cnt = src->data[0];
+		if (pan->cnt > BTC_DEV_LINK_MAX)
+			pan->cnt = BTC_DEV_LINK_MAX;
+		hal_mem_cpy(h, pan->lists, src->data+1, sizeof(pan->lists));
+		hal_mem_set(h, pan->links, 0, sizeof(pan->links));
+
+		for (i = 0; i < pan->cnt; i++)
+			pan->links[i].h = pan->lists[i];
+		pan->handle_update = 1;
+		break;
+	case BTC_DEV_HID_VINFO:
+		for (i = 0; i < BTC_DEV_LINK_MAX; i++) {
+			if (src->data[0] != hid->links[i].h)
+				continue;
+
+			hid->links[i].vendor_id = src->data[1];
+			break;
+		}
+		break;
+	case BTC_DEV_HID_DNAME:
+		break;
+	case BTC_DEV_HID_BINFO:
+		for (i = 0; i < BTC_DEV_LINK_MAX; i++) {
+			if (src->data[0] != hid->links[i].h)
+				continue;
+
+			/*role: 1=Master, type: 0=Legacy, sniff: 0=active*/
+			hid->links[i].role = !!(src->data[1] & BIT(0));
+			hid->links[i].type = !!(src->data[1] & BIT(1));
+			hid->links[i].sniff = !!(src->data[1] & BIT(2));
+			hid->links[i].interval = src->data[2];
+			hid->links[i].window = src->data[3];
+			hid->links[i].update = 1;
+			break;
+		}
+		break;
+	case BTC_DEV_A2DP_BINFO:
+		for (i = 0; i < BTC_DEV_LINK_MAX; i++) {
+			if (src->data[0] != a2dp->links[i].h)
+				continue;
+
+			a2dp->links[i].role = !!(src->data[1] & BIT(0));
+			a2dp->links[i].type = !!(src->data[1] & BIT(1));
+			a2dp->links[i].sniff = !!(src->data[1] & BIT(2));
+			a2dp->links[i].update = 1;
+			break;
+		}
+		break;
+	case BTC_DEV_PAN_BINFO:
+		for (i = 0; i < BTC_DEV_LINK_MAX; i++) {
+			if (src->data[0] != pan->links[i].h)
+				continue;
+
+			pan->links[i].role = !!(src->data[1] & BIT(0));
+			pan->links[i].type = !!(src->data[1] & BIT(1));
+			pan->links[i].sniff = !!(src->data[1] & BIT(2));
+			pan->links[i].direct = !!(src->data[2] & BIT(0));
+			pan->links[i].update = 1;
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+static void _update_bt_leaudio_info(struct btc_t *btc, u8 *buf, u32 len)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_info *bt = &cx->bt;
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_bt_leaudio_desc *leaudio = &b->leaudio_desc;
+	struct btc_bt_mb_leaudio_info *src = (struct btc_bt_mb_leaudio_info *)buf;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC] %s(): BT LEAUDIO INFO :"
+		  "exist:%d, bis:%d, cis:%d, rssi:%d \n",
+		  __func__, src->exist, src->bis, src->cis, src->rssi-100);
+
+	if (!src->exist) {
+		hal_mem_set(h, leaudio, 0, sizeof(*leaudio));
+		return;
+	}
+
+	leaudio->exist = src->exist;
+	leaudio->bis_cnt = src->bis;
+	leaudio->cis_cnt = src->cis;
+	leaudio->rssi = src->rssi;
+}
+
+static void _update_zb_coex_tbl(struct btc_t *btc)
+{
+	u8 mode = btc->cx.wl.role_info.link_mode;
+	u32 zb_tbl0 = 0xda5a5a5a, zb_tbl1 = 0xda5a5a5a;
+
+	if (mode == BTC_WLINK_5G || btc->dm.freerun) {
+		zb_tbl0 = 0xffffffff;
+		zb_tbl1 = 0xffffffff;
+	} else if (mode == BTC_WLINK_25G_MCC) {
+		zb_tbl0 = 0xffffffff; /* for E5G slot */
+		zb_tbl1 = 0xda5a5a5a; /* for E2G slot */
+	}
+
+	_btc_io_w(btc, RTW_MAC_MAC_CMD_OFLD, R_BTC_ZB_COEX_TBL_0, bMASKDW,
+		  zb_tbl0, false);
+
+	_btc_io_w(btc, RTW_MAC_MAC_CMD_OFLD, R_BTC_ZB_COEX_TBL_1, bMASKDW,
+		  zb_tbl1, true);
+}
+
+
+static void _ntfy_role_info(struct btc_t *btc, u8 rid,
+			    struct btc_wl_link_info *info,
+			    enum link_state reason)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_link_info *wlinfo = NULL;
+	u8 link_mode_ori = wl->role_info.link_mode;
+	u8 pta_req_mac_ori = wl->pta_req_mac;
+
+	PHL_INFO("[BTC], %s(), role_id=%d, reason=%d\n", __func__, rid, reason);
+
+	dm->cnt_notify[BTC_NTFY_ROLE_INFO]++;
+	dm->ntfy_type = BTC_NTFY_ROLE_INFO;
+
+	if (rid >= BTC_WL_MAX_ROLE_NUMBER)
+		return;
+
+	wlinfo = &wl->link_info[rid];
+
+	hal_mem_cpy(h, wlinfo, info, sizeof(struct btc_wl_link_info));
+
+	if (reason == PHL_ClIENT_PS) {
+		wlinfo->client_ps = 1;
+		dm->client_ps_tog1++;
+		return;
+	} else if (reason == PHL_ClIENT_ACTIVE) {
+		wlinfo->client_ps = 0;
+		dm->client_ps_tog1++;
+		return;
+	}
+
+	/* refresh wifi info */
+	_update_wl_info(btc, rid, reason);
+
+	if (wl->role_info.link_mode != link_mode_ori) {
+		wl->role_info.link_mode_chg = 1;
+		/* work-around for HFP bit-4 define issue (coex-900) */
+		if (btc->mdinfo.wa_type & BTC_WA_HFP_ZB)
+			_update_zb_coex_tbl(btc);
+	}
+
+	if (wl->pta_req_mac != pta_req_mac_ori)
+		wl->pta_reg_mac_chg = 1;
+
+	if (reason == PHL_EN_DBCC_START || reason == PHL_DIS_DBCC_START)
+		btc->dm.tdma_instant_excute = 1;
+
+	_run_coex(btc, __func__);
+
+	wl->role_info.link_mode_chg = 0;
+	wl->pta_reg_mac_chg = 0;
+	wl->client_cnt_inc_2g = 0;
+}
+
+static void _ntfy_radio_state(struct btc_t *btc, u8 rf_state)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_chip_ops *ops = btc->chip->ops;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	u32 val = 0;
+	u8 fw_type = btc->phl->fw_info.fw_type;
+
+	PHL_INFO("[BTC], %s(): rf_state =%d\n", __func__, rf_state);
+	dm->cnt_notify[BTC_NTFY_RADIO_STATE]++;
+
+	switch(rf_state) {
+	case BTC_RFCTRL_WL_OFF: /* WL radio-off */
+		wl_smap->rf_off = 1;
+		wl_smap->lps = BTC_LPS_OFF;
+		wl_smap->busy = 0;
+		wl_smap->lps_exiting = 0;
+		break;
+	case BTC_RFCTRL_WL_ON: /* Exit LPS */
+	default:
+		wl_smap->rf_off = 0;
+		wl_smap->lps = BTC_LPS_OFF;
+		wl_smap->lps_exiting = 1;
+		break;
+	case BTC_RFCTRL_LPS_WL_ON: /* LPS-Protocol (RFon) */
+		wl_smap->rf_off = 0;
+		wl_smap->lps = BTC_LPS_RF_ON;
+		wl_smap->busy = 0;
+		wl_smap->lps_exiting = 0;
+		break;
+	case BTC_RFCTRL_FW_CTRL: /*  LPS-PG, LPS-CG */
+		wl_smap->rf_off = 0;
+		wl_smap->lps = BTC_LPS_RF_OFF;
+		wl_smap->busy = 0;
+		wl_smap->lps_exiting = 0;
+		break;
+	}
+
+	if (rf_state == BTC_RFCTRL_WL_ON ||
+	    rf_state == BTC_RFCTRL_RESUME_DL_FW) {
+		val = BTC_WSCB_ACTIVE | BTC_WSCB_ON | BTC_WSCB_BTLOG;
+		_write_scbd(btc, val, true);
+		_update_bt_scbd(btc, true);
+
+		if (ops && ops->init_cfg)
+			ops->init_cfg(btc);
+
+		if (rf_state == BTC_RFCTRL_RESUME_DL_FW) {
+			dm->cnt_notify[BTC_NTFY_RESUME_DL_FW]++;
+			/* These fw_types contain BTC config */
+			if (fw_type == RTW_FW_NIC ||
+			    fw_type == RTW_FW_NIC_CE ||
+			    fw_type == RTW_FW_SPIC)
+				_set_init_info(btc);
+		} else {
+			_set_btc_timer(btc, BTC_TIMER_WL_LPS_EXIT,
+				       BTC_WLPS_MAXT);
+		}
+		_set_fw_rpt(btc, RPT_EN_MREG, 1);
+	} else {
+		if (rf_state == BTC_RFCTRL_FW_CTRL)
+			_write_scbd(btc, BTC_WSCB_ACTIVE, false);
+		else if (rf_state == BTC_RFCTRL_WL_OFF)
+			_write_scbd(btc, BTC_WSCB_ALL, false);
+		else /* BTC_RFCTRL_LPS_WL_ON */
+			_write_scbd(btc, BTC_WSCB_ACTIVE, true);
+
+		if (rf_state == BTC_RFCTRL_LPS_WL_ON &&
+		    wl_smap->lps_pre != BTC_LPS_OFF)
+			_update_bt_scbd(btc, true);
+
+		_set_fw_rpt(btc, RPT_EN_ALL, 0);
+	}
+
+	dm->cnt_dm[BTC_DCNT_BTCNT_HANG] = 0;
+	dm->tdma_instant_excute = 1;
+	dm->ntfy_type = BTC_NTFY_RADIO_STATE;
+
+	_run_coex(btc, __func__);
+
+	wl_smap->rf_off_pre = wl_smap->rf_off;
+	wl_smap->lps_pre = wl_smap->lps;
+}
+
+static void _ntfy_customerize(struct btc_t *btc, u8 type, u16 len, u8 *buf)
+{
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_module *module = &btc->mdinfo;
+	struct btc_chip_ops *ops = btc->chip->ops;
+
+	if (!buf)
+		return;
+
+	PHL_INFO("[BTC], %s !! \n", __func__);
+	btc->dm.cnt_notify[BTC_NTFY_CUSTOMERIZE]++;
+	btc->dm.ntfy_type = BTC_NTFY_CUSTOMERIZE;
+
+	switch (type) {
+	case PHL_BTC_CNTFY_BTINFO:
+		if (len != 1)
+			return;
+		buf[0] = bt->raw_info[BTC_BTINFO_L2];
+		break;
+	case PHL_BTC_CNTFY_COUNTRY_CODE:
+		if (len != 2)
+			return;
+
+		hal_mem_cpy(h, wl->country_code, buf, sizeof(wl->country_code));
+		if (!(bt->run_patch_code && bt->enable.now))
+			return;
+
+		_write_scbd(btc, BTC_WSCB_COUNTRY_CODE, true);
+		hal_btc_notify_ps_tdma(btc, true);
+		_set_btc_timer(btc, BTC_TIMER_EVENT_2, BTC_FORCE_LEAVE_PS_TIME);
+		btc->dm.country_code_proc = 1;
+		_run_coex(btc, __func__);
+		break;
+	case PHL_BTC_CNTFY_EXT_CONFIG: /* send EXTSOC INFO to BTC */
+		PHL_INFO("[BTC], %s(): %02X %02X %02X %02X\n", __func__,
+				buf[0], buf[1], buf[2], buf[3]);
+		if (len == 2) {
+			module->bt_solo = buf[0];
+			if (buf[1] == 0xff) /*ant.isolation default 10*/
+				module->ant.isolation = BTC_DEFAULT_ANISO;
+			else
+				module->ant.isolation = buf[1];
+
+			if (ops && ops->set_rfe)
+				ops->set_rfe(btc);
+		}
+		break;
+	}
+}
+
+static u8 _ntfy_wl_rfk(struct btc_t *btc, u8 phy_path, u8 type, u8 state)
+{
+	struct btc_wl_rfk_info *rfk = &btc->cx.wl.rfk_info;
+	bool result = BTC_WRFK_REJECT;
+
+	rfk->type = type;
+	rfk->path_map = phy_path & BTC_RFK_PATH_MAP;
+	rfk->phy_map = (phy_path & BTC_RFK_PHY_MAP) >> 4;
+	rfk->band = (phy_path & BTC_RFK_BAND_MAP) >> 6;
+	state &= (BIT(0) | BIT(1));
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s()_1: phy=0x%x, path=0x%x, type=%d, state=%d\n",
+		  __func__, rfk->phy_map, rfk->path_map, type, state);
+
+	btc->dm.ntfy_type = BTC_NTFY_WL_RFK;
+
+	switch (state) {
+	case BTC_WRFK_START:
+		result = _chk_wl_rfk_request(btc);
+		rfk->state = (result? BTC_WRFK_START : BTC_WRFK_STOP);
+		btc->dm.cnt_notify[BTC_NTFY_WL_RFK]++;
+
+		if (result == BTC_WRFK_ALLOW) {
+			_write_scbd(btc, BTC_WSCB_WLRFK, true);
+			rfk->start_time = _os_get_cur_time_ms();
+			_run_coex(btc, __func__);
+			_set_btc_timer(btc, BTC_TIMER_WL_RFKTO, BTC_WRFK_MAXT);
+		}
+		break;
+	case BTC_WRFK_ONESHOT_START:
+	case BTC_WRFK_ONESHOT_STOP:
+		rfk->state = state;
+		if (type == BTC_RF_RXDCK)
+			_run_coex(btc, __func__);
+		else
+			return BTC_WRFK_ALLOW;
+		break;
+	case BTC_WRFK_STOP:
+		result = BTC_WRFK_ALLOW;
+		rfk->state = BTC_WRFK_STOP;
+		_write_scbd(btc, BTC_WSCB_WLRFK, false);
+		rfk->proc_time = phl_get_passing_time_ms(rfk->start_time);
+		_run_coex(btc, __func__);
+		break;
+	}
+
+	/* Only update coex for RFK START and STOP
+	 * because Start -> OneSHOT_START time is short
+	 */
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s()_2: rfk_cnt=%d, result=%d\n",
+		  __func__, btc->dm.cnt_notify[BTC_NTFY_WL_RFK], result);
+
+	return result;
+}
+
+static void _ntfy_wl_sta(struct btc_t *btc, struct rtw_stats *phl_stats,
+			 u8 ntfy_num, struct rtw_phl_stainfo_t *sta[],
+			 u8 reason)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_module *module = &btc->mdinfo;
+	struct btc_traffic *wl_tra = &wl->traffic;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	struct btc_wl_link_info *linfo = NULL;
+	struct rtw_phl_rainfo ra_info = {0};
+	u8 i, j;
+	u8 link_cnt = 0, rssi_map = 0, role_idx = 0, status_change = 0;
+	u8 rssi_map_all = 0, rssi_th = 0, rssi = 0;
+	u8 *r_st = NULL;
+	u16 last_rx_rate;
+	u32 pass_time = 0, busy = 0, dir = BIT(TRAFFIC_DL), client_tdma_last;
+
+	/* rssi_map = 4 bits for rssi locate in which {60, 50, 40, 30}
+	 * if rssi >= 60% (-50dBm) --> map = 4b'0000 --> rssi_level = 4
+	 * if 50% <= rssi < 60%    --> map = 4b'0001 --> rssi_level = 3
+	 * if 40% <= rssi < 50%    --> map = 4b'0011 --> rssi_level = 2
+	 * if 30% <= rssi < 40%    --> map = 4b'0111 --> rssi_level = 1
+	 * if rssi < 20%           --> map = 4b'1111 --> rssi_level = 0
+	 */
+
+	if (wl_smap->dbccing || wl_smap->rf_off || wl_smap->lps != BTC_LPS_OFF)
+		return;
+
+	dm->cnt_notify[BTC_NTFY_WL_STA]++;
+
+	for (i = 0; i < ntfy_num; i++) {
+		hal_mem_set(h, &ra_info, 0, sizeof(ra_info));
+		role_idx = sta[i]->wrole->id;
+
+		if (role_idx >= BTC_WL_MAX_ROLE_NUMBER)
+			continue;
+
+		linfo = &wl->link_info[role_idx];
+		linfo->mac_id = sta[i]->macid;
+
+		if (linfo->connected == MLME_NO_LINK) {
+			linfo->rx_rate_drop_cnt = 0;
+			continue;
+		}
+
+		link_cnt++;
+
+		/* ===== Refresh Tx/Rx Rate information (by role) ===== */
+		last_rx_rate = linfo->rx_rate;
+
+		if (RTW_HAL_STATUS_SUCCESS ==
+		    rtw_hal_bb_query_rainfo(h, sta[i]->hal_sta, &ra_info))
+			linfo->tx_rate = ra_info.rate;
+		else
+			linfo->tx_rate = RTW_DATA_RATE_MAX;
+
+		linfo->rx_rate = h->trx_stat.rx_rate_plurality;
+
+		/* check if rx rate drop for CCK-dead-lock  */
+		if (linfo->rx_rate <= RTW_DATA_RATE_CCK2 &&
+		    last_rx_rate > RTW_DATA_RATE_CCK2 &&
+		    phl_stats->rx_traffic.lvl > 0)
+			linfo->rx_rate_drop_cnt++;
+
+		if (linfo->role == PHL_RTYPE_STATION ||
+		    linfo->role == PHL_RTYPE_P2P_GC) {
+			dm->trx_info.tx_rate = linfo->tx_rate;
+			dm->trx_info.rx_rate = linfo->rx_rate;
+		} else if (linfo->role == PHL_RTYPE_AP ||
+			   linfo->role == PHL_RTYPE_P2P_GO) {
+			client_tdma_last = dm->client_ps_tdma_on;
+			_update_wl_client_status(btc, linfo);
+			if (client_tdma_last != dm->client_ps_tdma_on)
+				status_change = 1;
+		}
+
+		/* ===== Refresh RSSI information (by role) ===== */
+		linfo->rssi = sta[i]->hal_sta->rssi_stat.rssi >> 1;
+		rssi = linfo->rssi;
+		rssi_map = 0;
+
+		/* check if rssi across wl_rssi_thres boundary */
+		for (j = 0; j < BTC_WL_RSSI_THMAX; j++) {
+			if (module->ant.type == BTC_ANT_SHARED && j == 0)
+				rssi_th = BTC_WL_RSSI_MAX_BTG;
+			else
+				rssi_th = btc->chip->wl_rssi_thres[j];
+
+			r_st = &linfo->rssi_state[j];
+			*r_st = _update_rssi_state(btc, *r_st, rssi, rssi_th);
+
+			/* fill rssi bit map 0~3 if rssi < threshold  */
+			if (BTC_RSSI_LOW(*r_st))
+				rssi_map |= BIT(j);
+		}
+
+		if ((linfo->role == PHL_RTYPE_STATION ||
+		     linfo->role == PHL_RTYPE_P2P_GC) ||
+		     ((linfo->role == PHL_RTYPE_AP ||
+		      linfo->role == PHL_RTYPE_P2P_GO) &&
+		      linfo->client_cnt > 1)){
+			rssi_map_all |= rssi_map;
+		}
+
+		/* ===== Refresh Tx/Rx TP/Lvl (phl_stats = All-Roles) ===== */
+		if (link_cnt > 1) /* the TP/Lvl/sts data calcul by all-roles */
+			continue;
+
+		wl->busy_to_idle = 0;
+
+		/* _get_wl_tp Must before  wl_tra->tx_time = phl_stats->...  */
+		wl_tra->tx_tp = _get_wl_tp(btc, TRAFFIC_UL, phl_stats);
+		wl_tra->rx_tp = _get_wl_tp(btc, TRAFFIC_DL, phl_stats);
+		dm->trx_info.tx_tp = wl_tra->tx_tp;
+		dm->trx_info.rx_tp = wl_tra->rx_tp;
+
+		wl_tra->tx_lvl = phl_stats->tx_traffic.lvl;
+		wl_tra->rx_lvl = phl_stats->rx_traffic.lvl;
+		dm->trx_info.tx_lvl = (u8)wl_tra->tx_lvl;
+		dm->trx_info.rx_lvl = (u8)wl_tra->rx_lvl;
+
+		wl_tra->tx_sts = phl_stats->tx_traffic.sts;
+		wl_tra->rx_sts = phl_stats->rx_traffic.sts;
+
+		/* ===== Refresh Tx/Rx Busy/Dir (for All-Role) ===== */
+		pass_time = phl_get_passing_time_ms(wl->busy_t);
+
+		if ((wl_tra->tx_lvl != RTW_TFC_IDLE ||
+		     wl_tra->rx_lvl != RTW_TFC_IDLE) &&
+		     (phl_stats->tx_tp_kbits > BTC_WL_TP_BUSY_THRES ||
+		      phl_stats->rx_tp_kbits > BTC_WL_TP_BUSY_THRES)) {
+			/* record busy_t as the last busy time */
+			wl->busy_t = _os_get_cur_time_ms();
+
+			/* set busy once idle->busy immediately */
+			busy = 1;
+
+			if ((wl_tra->tx_lvl > wl_tra->rx_lvl) ||
+			     (wl_tra->tx_tp > BTC_WL_TX_TP_LOW_THRES &&
+			     wl_tra->rx_tp < BTC_WL_RX_TP_LOW_THRES))
+				dir = BIT(TRAFFIC_UL);
+			else
+				dir = BIT(TRAFFIC_DL);
+		} else if (pass_time > BTC_BUSY2IDLE_THRES) {
+			/*set idle if "busy -> idle" after BTC_BUSY2IDLE_THRES*/
+			busy = 0;
+			dir = BIT(TRAFFIC_DL);
+		} else { /* wiat for BTC_BUSY2IDLE_THRES to change busy/dir */
+			busy = wl_smap->busy;
+			dir = wl_smap->traffic_dir;
+			wl->busy_to_idle = (u8)busy;
+		}
+
+		if (wl_smap->busy != busy || wl_smap->traffic_dir != dir) {
+			wl_smap->busy = busy;
+			wl_smap->traffic_dir = dir;
+			status_change = 1;
+		}
+	}
+
+	dm->trx_info.rx_err_ratio = _update_wl_rx_err_ratio(btc, 0);
+
+	wl->rssi_level = 4;
+	for (j = BTC_WL_RSSI_THMAX; j > 0; j--) {
+		/* set RSSI level 4 ~ 0 if rssi bit map match */
+		if (rssi_map_all & BIT(j-1)) {
+			wl->rssi_level = BTC_WL_RSSI_THMAX - j;
+			break;
+		}
+	}
+
+	if (dm->trx_info.wl_rssi != wl->rssi_level) {
+		status_change = 1;
+		dm->trx_info.wl_rssi = wl->rssi_level;
+	}
+
+	/* Trigger coex-run if 0x10980 reg-value is diff with coex setup */
+	if ((dm->wl_btg_rx_rb != dm->wl_btg_rx &&
+	     dm->wl_btg_rx_rb != BTC_BB_GNT_NOTFOUND) ||
+	     (dm->wl_pre_agc_rb != dm->wl_pre_agc &&
+	      dm->wl_pre_agc_rb != BTC_BB_PRE_AGC_NOTFOUND))
+		status_change = 1;
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	if (module->ant.type == BTC_ANT_DEDICATED)
+		hal_btc_fw_set_drv_info(btc, CXDRVINFO_TRX);
+#endif
+
+	if (!status_change)
+		return;
+
+	_write_scbd(btc, BTC_WSCB_WLBUSY, (bool)!!busy);
+
+	if (wl_smap->busy && wl->rssi_level <= 2 &&
+	    wl->is_5g_hi_channel &&
+	    (btc->mdinfo.wa_type & BTC_WA_5G_HI_CH_RX))
+		_write_scbd(btc, BTC_WSCB_5GHICH, true);
+	else
+		_write_scbd(btc, BTC_WSCB_5GHICH, false);
+
+	dm->ntfy_type = BTC_NTFY_WL_STA;
+	_run_coex(btc, __func__);
+}
+
+static void _ntfy_fwinfo(struct btc_t *btc, u8 *buf, u32 len, u8 cls, u8 func)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	u32 i = 0;
+
+	if (!buf || !len)
+		return;
+
+	btc->dm.cnt_notify[BTC_NTFY_FWINFO]++;
+	btc->dm.ntfy_type = BTC_NTFY_FWINFO;
+	pfwinfo->cnt_c2h++;
+
+	if (cls == BTFC_FW_EVENT) {
+		switch (func) {
+		case BTF_EVNT_BUF_OVERFLOW:
+			pfwinfo->event[func]++;
+			break;
+		case BTF_EVNT_RPT:
+			pfwinfo->event[func]++;
+			hal_btc_fw_parse_rpt(btc, buf, len);
+			break;
+		case BTF_EVNT_BT_INFO:
+			btc->cx.cnt_bt[BTC_BCNT_INFOUPDATE]++;
+			_update_bt_info(btc, buf, len);
+			break;
+		case BTF_EVNT_BT_SCBD:
+			btc->cx.cnt_bt[BTC_BCNT_SCBDUPDATE]++;
+			_update_bt_scbd(btc, false);
+			break;
+		case BTF_EVNT_BT_PSD:
+			_bt_psd_update(btc, buf, len);
+			break;
+		case BTF_EVNT_BT_REG:
+			btc->dbg.rb_done = true;
+			btc->dbg.rb_val = ((buf[3] << 24) | (buf[2] << 16) |
+					   (buf[1] << 8) | (buf[0]));
+			break;
+		case BTF_EVNT_C2H_LOOPBACK:
+			btc->dbg.rb_done = true;
+			for (i = 0; i < len; i++) {
+				if (i >= sizeof(btc->dbg.lb_val))
+					break;
+				btc->dbg.lb_val[i] = buf[i];
+			}
+			break;
+		case BTF_EVNT_BT_DEV_INFO:
+			_update_bt_dev_info(btc, buf, len);
+			break;
+		case BTF_EVNT_BT_LEAUDIO_INFO:
+			btc->cx.cnt_bt[BTC_BCNT_LEAUDIO_INFOUPDATE]++;
+			_update_bt_leaudio_info(btc, buf, len);
+			break;
+		}
+	}
+}
+
+static void _ntfy_timer(struct btc_t *btc, u16 tmr_id)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_bt_a2dp_desc *a2dp = &cx->bt.link_info.a2dp_desc;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	bool is_sta_change = false;
+	u32 rpt_tick = BTC_RPT_PERIOD/BTC_PERIODIC_TIME;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+	          "[BTC], %s(): tmr_id =%d\n", __func__, tmr_id);
+
+	dm->cnt_notify[BTC_NTFY_TIMER]++;
+
+	switch (tmr_id) {
+	case BTC_TIMER_PERIODIC:
+		/* update per 100ms * 20 */
+		if (dm->cnt_notify[BTC_NTFY_TIMER] % rpt_tick == 0) {
+			_get_wl_nhm_dbm(btc);
+			if (wl_smap->rf_off == 0 && wl_smap->lps == BTC_LPS_RF_ON)
+				_get_wl_cn_report(btc);
+			_get_wl_evm_report(btc);
+
+			_query_bt_dev(btc);
+		}
+		/* start next periodic timer */
+		_set_btc_timer(btc, BTC_TIMER_PERIODIC, BTC_PERIODIC_TIME);
+		break;
+	case BTC_TIMER_WL_RFKTO:
+		if (wl->rfk_info.state == BTC_WRFK_STOP)
+			return;
+
+		cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]++;
+		dm->error |= BTC_DMERR_WL_RFK_TIMEOUT;
+		wl->rfk_info.state = BTC_WRFK_STOP;
+		is_sta_change = true;
+		break;
+	case BTC_TIMER_WL_SPECPKT:
+		if (wl->status.map._4way || wl->status.map.connecting) {
+			wl->status.map._4way = 0;
+			wl->status.map.connecting = 0;
+			is_sta_change = true;
+		}
+		break;
+	case BTC_TIMER_BT_A2DPPLAY:
+		a2dp->play_latency = 0;
+		break;
+	case BTC_TIMER_WL_LPS_EXIT:
+		if (wl->status.map.lps_exiting) {
+			_reset_query_flag(btc, BTC_BT_ALL);
+			wl->status.map.lps_exiting = 0;
+		}
+		break;
+	case BTC_TIMER_EVENT_1: /* dbccing */
+		if (wl->status.map.dbccing) {
+			wl->status.map.dbccing = 0;
+			is_sta_change = true;
+		}
+		break;
+	case BTC_TIMER_COUNTRYCODE:
+		if (bt->scbd & BTC_BSCB_COUNTRY_CODE &&
+		    dm->country_code_retry < BTC_COUNTRY_CODE_RETRY_LIMIT) {
+			_send_bt_country_code(btc);
+			_set_btc_timer(btc, BTC_TIMER_COUNTRYCODE,
+			               BTC_COUNTRY_CODE_TIME);
+			dm->country_code_retry++;
+		} else {
+			if (dm->country_code_retry == BTC_COUNTRY_CODE_RETRY_LIMIT)
+				dm->error |= BTC_DMERR_CCODE_BTNORS;
+			hal_btc_notify_ps_tdma(btc, btc->hal->btc_ctrl.lps);
+		}
+		break;
+	case BTC_TIMER_EVENT_2:
+		if (!(bt->scbd & BTC_BSCB_COUNTRY_CODE))
+			hal_btc_notify_ps_tdma(btc, btc->hal->btc_ctrl.lps);
+		break;
+	}
+
+	if (is_sta_change) {
+		dm->ntfy_type = BTC_NTFY_TIMER;
+		_run_coex(btc, __func__);
+	}
+}
+
+/******************************************************************************
+ *
+ * coexistence extern functions
+ *
+ *****************************************************************************/
+/*
+ * btc related sw initialization
+ */
+bool hal_btc_init(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u32 len;
+
+	switch (btc->hal->chip_id) {
+#ifdef BTC_8852A_SUPPORT
+	case CHIP_WIFI6_8852A:
+		PHL_INFO("[BTC], %s(): Init 8852A!!\n", __func__);
+		btc->chip = &chip_8852a;
+		break;
+#endif
+#if defined(BTC_8852B_SUPPORT) || defined(BTC_8852BP_SUPPORT) || defined(BTC_8851B_SUPPORT) || defined (BTC_8852BT_SUPPORT) || defined (BTC_8852BPT_SUPPORT)
+	case CHIP_WIFI6_8852B:
+		PHL_INFO("[BTC], %s(): Init 8852B!!\n", __func__);
+		btc->chip = &chip_8852b;
+		break;
+	case CHIP_WIFI6_8852BP:
+		PHL_INFO("[BTC], %s(): Init 8852BP!!\n", __func__);
+		btc->chip = &chip_8852bp;
+		break;
+	case CHIP_WIFI6_8851B:
+		PHL_INFO("[BTC], %s(): Init 8851B!!\n", __func__);
+		btc->chip = &chip_8851b;
+		break;
+	case CHIP_WIFI6_8852BT:
+		PHL_INFO("[BTC], %s(): Init 8852BT!!\n", __func__);
+		btc->chip = &chip_8852bt;
+		break;
+	case CHIP_WIFI6_8852BPT:
+		PHL_INFO("[BTC], %s(): Init 8852BPT!!\n", __func__);
+		btc->chip = &chip_8852bt;
+		break;
+#endif
+#ifdef BTC_8852C_SUPPORT
+	case CHIP_WIFI6_8852C:
+		PHL_INFO("[BTC], %s(): Init 8852C!!\n", __func__);
+		btc->chip = &chip_8852c;
+		break;
+#endif
+#ifdef BTC_8842A_SUPPORT
+	case CHIP_WIFI6_8842A:
+		PHL_INFO("[BTC], %s(): Init 8842A!!\n", __func__);
+		btc->chip = &chip_8842a;
+		break;
+#endif
+#ifdef BTC_8852D_SUPPORT
+	case CHIP_WIFI6_8852D:
+		PHL_INFO("[BTC], %s(): Init 8852D!!\n", __func__);
+		btc->chip = &chip_8852d;
+		break;
+#endif
+	default:
+		PHL_ERR("[BTC], %s(): no matched IC!!\n", __func__);
+		wl->status.map.init_ok = false;
+		return false;
+	}
+
+	_reset_btc_var(btc, BTC_RESET_ALL);
+	_btmr_init(btc);
+
+	len = _os_strlen((u8 *)"None") + 1;
+	_rsn_cpy(btc->dm.run_reason, "None", len);
+	_act_cpy(btc->dm.run_action, "None", len);
+
+	btc->hal->btc_vc.btc_ver = coex_ver;
+	btc->ops = &_btc_ops;
+	btc->mlen = BTC_MSG_MAXLEN;
+	btc->ctrl.igno_bt = true;
+	wl->status.map.init_ok = true;
+
+	return true;
+}
+
+void hal_btc_deinit(struct btc_t *btc)
+{
+	_btmr_deinit(btc);
+}
+
+bool hal_btc_check_io_ofld(struct btc_t *btc)
+{
+#ifdef BTC_CONFIG_FW_IO_OFLD_SUPPORT
+	return btc->phl->dev_cap.io_ofld;
+#else
+	return false;
+#endif
+}
+
+void _btc_io_w(struct btc_t *btc, u32 type, u32 addr, u32 mask,
+	       u32 value, bool trigger)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct rtw_mac_cmd cmd = {0};
+	u32 path = 0, reg_type = 0, ori_val = 0, i = 0, shift = 0;
+	u8 src_grp = 0;
+
+#ifdef BTC_CONFIG_FW_IO_OFLD_SUPPORT
+	if (hal_btc_check_io_ofld(btc)) {
+		/* 0: BB, 1:RF, 2:MAC */
+		src_grp = (u8)(type & 0x3);
+		if (src_grp == RTW_MAC_RF_CMD_OFLD)
+			cmd.rf_path = (u8)(type >> 8);
+
+		cmd.src = src_grp;
+		cmd.type = RTW_MAC_WRITE_OFLD;
+		cmd.lc = trigger;
+		cmd.offset = (u16)addr;
+		cmd.value = value;
+		cmd.mask = mask;
+		(void)rtw_hal_mac_add_cmd_ofld(h, &cmd);
+		return;
+	}
+#endif
+
+	path = (type & bMASKB1) >> 8;
+	reg_type = type & bMASKB0;
+	shift = 0;
+
+	switch(reg_type) {
+	case RTW_MAC_RF_CMD_OFLD:
+		_write_wl_rf_reg(btc, path, addr, mask, value);
+		break;
+	case RTW_MAC_MAC_CMD_OFLD:
+		if (mask == bMASKDW) {
+			_write_cx_reg(btc, addr, value);
+			break;
+		}
+
+		_read_cx_reg(btc, addr, &ori_val);
+		for (i = 0; i <= 31; i++) {
+			if ((mask >> i) & BIT(0)) {
+				shift = i;
+				break;
+			}
+		}
+
+		value = ((ori_val) & (~mask)) | (((value << shift)) & mask);
+		_write_cx_reg(btc, addr, value);
+		break;
+	default:
+		break;
+	}
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/hal_btc.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/hal_btc.h
new file mode 100644
index 000000000000..25b3f768d723
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/hal_btc.h
@@ -0,0 +1,1951 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_BTC_H_
+#define _HAL_BTC_H_
+
+#include "halbtc_fw.h"
+#include "halbtc_fwdef.h"
+#include "halbtc_dbg_cmd.h"
+
+#define BTC_BRANCH BTC_BRANCH_FORMAL
+
+#ifdef CONFIG_RTL8852A
+#define BTC_8852A_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8852B
+#define BTC_8852B_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8852BP
+#define BTC_8852B_SUPPORT
+#define BTC_8852BP_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8852BT
+#define BTC_8852B_SUPPORT
+#define BTC_8852BT_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8852BPT
+#define BTC_8852B_SUPPORT
+#define BTC_8852BPT_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8851B
+#define BTC_8852B_SUPPORT
+#define BTC_8851B_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8852C
+#define BTC_8852C_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8842A
+#define BTC_8842A_SUPPORT
+#endif
+
+#ifdef CONFIG_RTL8852D
+#define BTC_8852D_SUPPORT
+#endif
+
+/* this follwing define setup must sync with fw-btc */
+/* ============================================================= */
+#define BTC_AISO_SUPPORT
+//#define BTC_FDDT_TRAIN_SUPPORT /* this setup must sync with fw btc  */
+
+#ifdef CONFIG_PHL_IO_OFLD
+#define BTC_CONFIG_FW_IO_OFLD_SUPPORT
+#endif
+
+#if PLATFOM_IS_LITTLE_ENDIAN
+#define BTC_PLATFORM_BIG_ENDIAN 0
+#else
+#define BTC_PLATFORM_BIG_ENDIAN 1
+#endif
+
+//#define BTC_PAUTO_LESCAN
+#define LE_INIT_INTVL 160
+/* ============================================================= */
+
+#define BTC_PHY_MAX 2
+#define BTC_WL_TX_PWR_CTRL_OFLD
+
+#define BTC_FDDT_INFO_MAP_DEF 0x1f
+
+#define BTC_WL_RX_TP_LOW_THRES 10
+#define BTC_WL_TX_TP_LOW_THRES 1
+
+#define BTC_WL_TP_BUSY_THRES 1600 /* in k-bit/sec */
+
+#define BTC_SCBD_REWRITE_DELAY 1000
+#define BTC_MSG_MAXLEN 200
+#define BTC_H2C_MAXLEN 2020 /* limitby H2C hdr-type-20 0:36 ; 1:228 ; 2:2020 */
+#define BTC_H2C_MAXLEN1 228
+#define BTC_POLICY_MAXLEN 512
+#define BTC_HUBMSG_MAXLEN 512
+#define BTC_RPT_PERIOD 2000
+#define BTC_RSN_MAXLEN 32
+#define BTC_ACT_MAXLEN 32
+#define BTC_PERIODIC_TIME 100 /* 100ms, must be non zero and < 2000 */
+#define BTC_COUNTRY_CODE_TIME 2000 /*sned country code to BT every 2s until BT receive*/
+#define BTC_FORCE_LEAVE_PS_TIME 2000 /*force leave ps 2s at most when receive country code*/
+#define BTC_WRFK_MAXT 300 /* WL RFK 300ms */
+#define BTC_SPKT_MAXT 4000 /* special pkt unit: 4000ms */
+#define BTC_REA2DP_MAXT 16000 /* for A2DP resume, 16sec = 16000ms */
+#define BTC_WLPS_MAXT 6000
+#define BTC_RPT_HDR_SIZE 3
+#define BTC_BUSY2IDLE_THRES 10000 /* wait time for WL busy-to-idle, ms */
+#define BTC_WL_DEF_TX_PWR 15
+
+#define BTC_DM_MAXSTEP 30
+
+#define BTC_LEAK_AP_TH 10
+#define BTC_CYSTA_CHK_PERIOD 50
+#define BTC_SLOT_REQ_TH 2
+
+#define BTC_NULLTX_FAIL_TH 8
+#define BTC_NULLTX_CHK_PERIOD 100
+
+#define BTC_P2P_CYCLE 100
+#define BTC_NULL1_EBT_EARLY_T 10
+#define BTC_E2G_OFFSET 10;
+#define BTC_E2G_LIMIT_DEF 80
+#define BTC_E2G_LIMIT_MIN 40
+#define BTC_E2G_LIMIT_MAX 100
+
+#define B_BTC_DIS_BTC_CLK_G BIT(2)
+#define B_BTC_WL_SRC BIT(0)
+
+#define BTC_WL_LINK_SCAN 0x3e
+
+#define BTC_BB_GNT_FWCTRL 2
+#define BTC_BB_GNT_NOTFOUND 3
+#define BTC_BB_PRE_AGC_FWCTRL 2
+#define BTC_BB_PRE_AGC_NOTFOUND 3
+
+#define BTC_BB_TX_1SS_LIMIT 1 /* if halbb support rtw_hal_btc_cfg_tx_1ss */
+
+#define BTC_AP_BLIST 0 /* Sporton AP issue*/
+
+#define R_BTC_ZB_COEX_TBL_0 0xDA28
+#define R_BTC_ZB_COEX_TBL_1 0xDA2C
+#define R_BTC_ZB_BREAK_TBL  0xDA50
+
+#define BTC_COUNTRY_CODE_OP 4 /*for mailbox 0x11*/
+#define BTC_COUNTRY_CODE_RETRY_LIMIT 3
+
+struct btc_t;
+
+#define NM_EXEC false
+#define FC_EXEC true
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+#endif
+
+#ifndef BIT
+#define BIT(x) (1 << (x))
+#endif
+
+#define bMASKDW 0xffffffff
+#define bMASKHW 0xffff0000
+#define bMASKLW 0x0000ffff
+#define bMASKB0 0x000000ff
+#define bMASKB1 0x0000ff00
+#define bMASKB2 0x00ff0000
+#define bMASKB3 0xff000000
+#define bMASKRF 0x000fffff
+
+#define BTC_WL_RSSI_THMAX 4
+#define BTC_BT_RSSI_THMAX 4
+
+#define BTC_WL_RSSI_MAX_BTG 70 /* for BTG co-rx Hi-RSSI thres */
+
+#define BTC_TDMA_WLROLE_MAX 3
+#define BTC_FREERUN_ANTISO_MIN 30
+#define BTC_DEFAULT_ANISO 10
+#define BTC_FDDTRAIN_ANTISO_MIN 10
+#define BTC_FDDTRAIN_ANTISO_MAX 40
+#define BTC_BT_RX_NORMAL_LVL 7
+
+#define BTC_BTINFO_LEN 6
+#define BTC_B1_MAX 250  /* unit:ms, used for Max A2DP retry adjust */
+
+#define BTC_COEX_RTK_MODE 0
+#define BTC_COEX_CSR_MODE 1
+
+#define BTC_CHK_ERR_TH 3
+#define BTC_CHK_WLSLOT_DRIFT_MAX 15
+#define BTC_CHK_BTSLOT_DRIFT_MAX 15
+
+#define BTC_RFK_PATH_MAP 0xf
+#define BTC_RFK_PHY_MAP 0x30
+#define BTC_RFK_BAND_MAP 0xc0
+
+#define BTC_GNT_SET_SKIP 0xff
+
+#define BTC_CLIENT_PS_MAX 32 /* client TDMA-on if >= (unit: BTC_PERIODIC_TIME)*/
+
+/* mailbox 0x45 BT device info related */
+#define BTC_DEV_LINK_MAX 4
+
+#define _write_cx_reg(btc, offset, val) \
+	rtw_hal_mac_coex_reg_write(btc->hal, offset, val)
+
+#define _read_cx_reg(btc, offset, val) \
+	rtw_hal_mac_coex_reg_read(btc->hal, offset, val)
+
+#define _read_cx_ctrl(btc, val) \
+	rtw_hal_mac_get_coex_ctrl(btc->hal, val)
+
+#define _write_wl_rf_reg(btc, path, addr, mask, data) \
+	rtw_hal_write_rf_reg(btc->hal, path, addr, mask, data)
+
+#define _send_phl_evnt(btc, buf, len, vid) \
+	hal_btc_send_event(btc, btc->cx.wl.pta_req_mac, buf, len, vid)
+
+#define _update_poicy hal_btc_fw_set_policy
+#define _set_fw_rpt hal_btc_fw_set_rpt
+
+#define _rsn_cpy(dst, src, len) \
+	do { \
+		len = (len < BTC_RSN_MAXLEN) ? len : BTC_RSN_MAXLEN-1; \
+		hal_mem_set(btc->hal, dst, 0, BTC_RSN_MAXLEN); \
+		hal_mem_cpy(btc->hal, dst, src, len); \
+	} while (0)
+
+#define _act_cpy(dst, src, len) \
+	do { \
+		len = (len < BTC_ACT_MAXLEN) ? len : BTC_ACT_MAXLEN-1; \
+		hal_mem_set(btc->hal, dst, 0, BTC_ACT_MAXLEN); \
+		hal_mem_cpy(btc->hal, dst, src, len); \
+	} while (0)
+
+#define _tdma_cmp(dst, src) \
+	hal_mem_cmp(btc->hal, dst, src, sizeof(struct fbtc_tdma))
+#define _tdma_cpy(dst, src) \
+	hal_mem_cpy(btc->hal, dst, src, sizeof(struct fbtc_tdma))
+#define _tdma_set(tp,rxflc,txflc,wtg,lek,ext,flc_role) \
+	do { \
+		btc->dm.tdma.type = tp;\
+		btc->dm.tdma.rxflctrl = rxflc; \
+		btc->dm.tdma.txflctrl = txflc; \
+		btc->dm.tdma.rsvd = wtg; \
+		btc->dm.tdma.leak_n = lek; \
+		btc->dm.tdma.ext_ctrl = ext; \
+		btc->dm.tdma.rxflctrl_role = flc_role; \
+	} while (0)
+#define _tdma_set_rxflctrl(rxflc) btc->dm.tdma.rxflctrl = rxflc
+#define _tdma_set_txflctrl(txflc) btc->dm.tdma.txflctrl = txflc
+#define _tdma_set_flctrl_role(role) btc->dm.tdma.rxflctrl_role = role
+#define _tdma_set_lek(lek) btc->dm.tdma.leak_n = lek
+
+#define _tdma_set_instant() btc->dm.tdma.option_ctrl |= CXOPCTL_INST_EXEC
+#define _tdma_set_fddt_en() btc->dm.tdma.option_ctrl |= CXOPCTL_FDDT_ENABLE
+#define _tdma_set_fddt_renew() btc->dm.tdma.option_ctrl |= CXOPCTL_FDDT_RENEW
+#define _tdma_set_fddt_dbg() btc->dm.tdma.option_ctrl |= CXOPCTL_FDDT_DEBUG
+#define _tdma_set_le_init_end() btc->dm.tdma.option_ctrl |= CXOPCTL_LE_INIT_EXEC
+
+#define _slot_cmp(dst, src) \
+	hal_mem_cmp(btc->hal, dst, src, sizeof(struct fbtc_slot))
+#define _slot_cpy(dst, src) \
+	hal_mem_cpy(btc->hal, dst, src, sizeof(struct fbtc_slot))
+#define _slots_cpy(dst, src) \
+	hal_mem_cpy(btc->hal, dst, src, CXST_MAX*sizeof(struct fbtc_slot))
+#define _slot_set(sid,dura,tbl,type) \
+	do { \
+		btc->dm.slot[sid].dur = dura;\
+		btc->dm.slot[sid].cxtbl = tbl; \
+		btc->dm.slot[sid].cxtype = type; \
+	} while (0)
+#define _slot_set_dur(sid,dura) btc->dm.slot[sid].dur = dura
+#define _slot_set_tbl(sid,tbl) btc->dm.slot[sid].cxtbl = tbl
+#define _slot_set_type(sid,type) btc->dm.slot[sid].cxtype = type
+
+#define _fddt_cmp(dst, src) \
+	hal_mem_cmp(btc->hal, dst, src, sizeof(struct btc_fddt_train_info))
+#define _fddt_cpy(dst, src) \
+	hal_mem_cpy(btc->hal, dst, src, sizeof(struct btc_fddt_train_info))
+
+#define btc_swap2(a, b) \
+		do { u8 __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
+#define btc_swap4(a, b, c, d) \
+		do { btc_swap2(a, d); btc_swap2(b, c); } while (0)
+
+#define	BTC_RSSI_HIGH(_rssi_) \
+			((_rssi_ == BTC_RSSI_ST_HIGH || _rssi_ == BTC_RSSI_ST_STAY_HIGH)? 1:0)
+#define	BTC_RSSI_LOW(_rssi_) \
+			((_rssi_ == BTC_RSSI_ST_LOW || _rssi_ == BTC_RSSI_ST_STAY_LOW)? 1:0)
+#define	BTC_RSSI_CHANGE(_rssi_) \
+			((_rssi_ == BTC_RSSI_ST_LOW || _rssi_ == BTC_RSSI_ST_HIGH)? 1:0)
+
+#ifdef BTC_AISO_SUPPORT
+#define BT_PSD_RPT_TYPE_QUERY	0x0 /* report when WLAN query */
+#define BT_PSD_RPT_TYPE_CAL	0x1 /* when PSD scan idx~idx+4, idx+5~idx+9 */
+#define BT_PSD_RPT_TYPE_NOISE	0x2 /* report all channel */
+
+#define BT_PSD_SRC_RAW		0x0
+#define BT_PSD_SRC_POST_BB	0x1
+#define BT_PSD_SRC_POST_BB_FW	0x2
+
+#define BT_PSD_WAIT_CNT		200
+
+#define BT_PSD_TX_BUSY_CNT_MIN	3    /* data num per frequency point*/
+#define BT_PSD_VALID_CH_MIN	4    /* 1~5, 3/4 is more suitbale */
+#define BT_PSD_PRINT_PERIOD	10
+
+enum btc_aiso_method {
+	/* tx busy raw */
+	BTC_AISO_M0, /* m0: busy avg */
+	BTC_AISO_M1, /* m1: busy max */
+	BTC_AISO_M2, /* m2: busy remove max and min */
+	BTC_AISO_M3, /* m3: busy remove max */
+
+	/* tx busy filter */
+	BTC_AISO_M4, /* m4:filter m1 */
+
+	/* all data */
+	BTC_AISO_M5, /* m5: hold max value */
+	BTC_AISO_M6, /* m6: filter m5 */
+
+	/* avg */
+	BTC_AISO_M01_AVG, /* avg (m0 + m1) */
+	BTC_AISO_M12_AVG, /* avg (m1 + m2) */
+	BTC_AISO_M13_AVG, /* avg (m1 + m3) */
+
+	/* avg */
+	BTC_AISO_M46_AVG, /* avg (m4 + m6) */
+
+	BTC_AISO_M_MAX,
+	BTC_AISO_M_ALL = 0xFF,
+};
+
+struct btc_aiso_val {
+	u16 psd_rec_cnt;
+
+	s8 psd_max[10];
+	s8 psd_min[10];
+
+	s8 txbusy_psd_max[10];
+	s8 txbusy_psd_min[10];
+	s8 txbusy_psd_avg[10];
+	u8 txbusy_psd_cnt[10];
+	s16 txbusy_psd_sum[10];
+
+	s8 txidle_psd_max[10];
+	s8 txidle_psd_min[10];
+	s8 txidle_psd_avg[10];
+	u8 txidle_psd_cnt[10];
+	s16 txidle_psd_sum[10];
+
+	s8 wl_air_psd_avg;   /* Avg(Tx-power) in dBm per MHz */
+	s32 wl_air_psd_sum;  /* Sum(Tx-power) in dBm per MHz */
+
+	u8 last_1st_half[6];
+	u8 last_2nd_half[6];
+
+	s16 aiso_md[BTC_AISO_M_MAX];
+	s8 rx_psd[BTC_AISO_M_MAX];
+};
+
+struct btc_bt_psd_dm {
+	struct btc_aiso_val aiso_val;
+	u8 aiso_data_ok;
+	u8 aiso_db_cnt;
+	u8 aiso_cmd_cnt;
+	u8 aiso_db[16];
+	u8 aiso_sort_db[16];
+	u8 aiso_sort_avg;
+	u8 raw_info[8];
+
+	u8 aiso_method;
+	u8 aiso_method_final;
+	u8 rec_start;
+	u8 wl_tx_ss;
+	u8 wl_ch_last;
+	u16 wl_tx_rate;
+
+	bool en;
+	bool rec_time_out;
+};
+#endif
+
+enum btc_special_ap {
+	BTC_SPAP_UNKNOWN = 0,
+	BTC_SPAP_TPL = 1,
+	BTC_SPAP_VW = 2,
+	BTC_SPAP_SPTN = 3,
+	BTC_SPAP_CMW = 4, /* for tis test */
+	BTC_SPAP_MAX
+};
+
+enum btc_wl_gpio_debug {
+	BTC_DBG_GNT_BT = 0,
+	BTC_DBG_GNT_WL = 1,
+	/* The following signals should 0-1 tiggle by each function-call */
+	BTC_DBG_BCN_EARLY = 2,
+	BTC_DBG_WL_NULL0 = 3,
+	BTC_DBG_WL_NULL1 = 4,
+	BTC_DBG_WL_RXISR = 5,
+	BTC_DBG_TDMA_ENTRY = 6,
+	BTC_DBG_A2DP_EMPTY = 7,
+	BTC_DBG_BT_RETRY = 8,
+	/* The following signals should 0-1 tiggle by state L/H */
+	BTC_DBG_BT_RELINK = 9,
+	BTC_DBG_SLOT_WL = 10,
+	BTC_DBG_SLOT_BT = 11,
+	/* The following signals should 0-1 tiggle by external*/
+	BTC_DBG_WL_ERR = 12,
+	BTC_DBG_WL_OK = 13,
+	/* The following signals appear only 1-active at same time*/
+	BTC_DBG_SLOT_B2W = 14,
+	BTC_DBG_SLOT_W1 = 15,
+	BTC_DBG_SLOT_W2 = 16,
+	BTC_DBG_SLOT_W2B = 17,
+	BTC_DBG_SLOT_B1 = 18,
+	BTC_DBG_SLOT_B2 = 19,
+	BTC_DBG_SLOT_B3 = 20,
+	BTC_DBG_SLOT_B4 = 21,
+	BTC_DBG_SLOT_LK = 22,
+	BTC_DBG_SLOT_E2G = 23,
+	BTC_DBG_SLOT_E5G = 24,
+	BTC_DBG_SLOT_EBT = 25,
+	BTC_DBG_SLOT_WLK = 26,
+	BTC_DBG_SLOT_B1FDD = 27,
+	BTC_DBG_BT_CHANGE = 28,
+	/* The following signals should 0-1 tiggle by external*/
+	BTC_DBG_WL_CCA = 29,
+	BTC_DBG_BT_LEAUDIO = 30,
+	BTC_DBG_USER_DEF = 31 /* limit by BTC_DBG_MAX1-1  */
+};
+
+enum btc_bt_dev_info {
+	BTC_DEV_HID_LIST = 0x0,
+	BTC_DEV_HID_VINFO = 0x1,
+	BTC_DEV_HID_DNAME =  0x2,
+	BTC_DEV_HID_BINFO = 0x3,
+
+	BTC_DEV_A2DP_LIST = 0x10,
+	BTC_DEV_A2DP_BINFO = 0x11,
+
+	BTC_DEV_PAN_LIST = 0x20,
+	BTC_DEV_PAN_BINFO = 0x21
+};
+
+enum btc_coex_dir {
+	BTC_COEX_INNER = 0,
+	BTC_COEX_OUTPUT = 1,
+	BTC_COEX_INPUT = 2
+};
+
+enum btc_bt_rfk_state {
+	BTC_BRFK_STOP = 0,
+	BTC_BRFK_START = 1
+};
+
+enum btc_wl_rfk_state {
+	BTC_WRFK_STOP = 0,
+	BTC_WRFK_START = 1,
+	BTC_WRFK_ONESHOT_START = 2,
+	BTC_WRFK_ONESHOT_STOP = 3
+};
+
+enum btc_wl_rfk_result {
+	BTC_WRFK_REJECT = 0,
+	BTC_WRFK_ALLOW = 1
+};
+
+enum btc_lps_state {
+	BTC_LPS_OFF = 0,
+	BTC_LPS_RF_OFF = 1,
+	BTC_LPS_RF_ON = 2
+};
+
+enum btc_gnt_ctrl_type {
+	BTC_CTRL_BY_BT = 0,
+	BTC_CTRL_BY_WL
+};
+
+enum btc_3cx_type {
+	BTC_3CX_NONE = 0,
+	BTC_3CX_LTE = 1,
+	BTC_3CX_ZB = 2,
+	BTC_3CX_MAX
+};
+
+enum btc_wl_pri_mask_type {
+	BTC_PRI_MASK_RX_RESP = 0,
+	BTC_PRI_MASK_TX_RESP,
+	BTC_PRI_MASK_BEACON,
+	BTC_PRI_MASK_RX_CCK,
+	BTC_PRI_MASK_TX_CCK,
+	BTC_PRI_MASK_TX_TRIG,
+	BTC_PRI_MASK_MAX
+};
+
+enum btc_init_type {
+	BTC_INIT_NORMAL,
+	BTC_INIT_WLONLY,
+	BTC_INIT_WLOFF
+};
+
+enum btc_branch_type {
+	BTC_BRANCH_FORMAL = 0,
+	BTC_BRANCH_UPSTREAM,
+	BTC_BRANCH_HP,
+	BTC_BRANCH_LENOVO,
+	BTC_BRANCH_XBOX,
+	BTC_BRANCH_WBX,
+	BTC_BRANCH_ROKU,
+	BTC_BRANCH_LG,
+	BTC_BRANCH_SAMSUNG,
+	BTC_BRANCH_AMAZON,
+	BTC_BRANCH_MAX
+};
+
+enum btc_dbg_str_type {
+	BTC_STR_ROLE = 0,
+	BTC_STR_MROLE,
+	BTC_STR_REG,
+	BTC_STR_SLOT,
+	BTC_STR_TDMA,
+	BTC_STR_TRACE,
+	BTC_STR_BRANCH,
+	BTC_STR_RXFLCTRL,
+	BTC_STR_WLLINK,
+	BTC_STR_ANTPATH,
+	BTC_STR_GDBG,
+	BTC_STR_CHIPID,
+	BTC_STR_EVENT,
+	BTC_STR_WLMODE,
+	BTC_STR_WLBW,
+	BTC_STR_RFTYPE,
+	BTC_STR_POLICY,
+	BTC_STR_MSTATE,
+	BTC_STR_RATE,
+	BTC_STR_POLUT,
+	BTC_STR_BAND,
+	BTC_STR_CXSTATE,
+	BTC_STR_FDDT_TYPE,
+	BTC_STR_FDDT_STATE,
+	BTC_STR_FDDT_NORUN,
+	BTC_STR_FDDT_TSTATE,
+	BTC_STR_FDDT_TPHASE,
+	BTC_STR_FDDT_CHKMAP,
+	BTC_STR_DMERROR,
+	BTC_STR_FWERROR,
+	BTC_STR_RPTMATCH,
+	BTC_STR_H2CERROR,
+	BTC_STR_VENDORID,
+	BTC_STR_SPAP,
+	BTC_STR_ESOC,
+	BTC_STR_ESOC_INTF,
+	BTC_STR_ESOC_STATE,
+	BTC_STR_ESOC_FUNC,
+	BTC_STR_PATH,
+	BTC_STR_MAX
+};
+
+enum btc_bt_rf_group {
+	BTC_BT_SS_GROUP = 0x0,
+	BTC_BT_TX_GROUP = 0x2,
+	BTC_BT_RX_GROUP = 0x3,
+	BTC_BT_MAX_GROUP,
+};
+
+enum btc_rssi_st {
+	BTC_RSSI_ST_LOW = 0x0,
+	BTC_RSSI_ST_HIGH,
+	BTC_RSSI_ST_STAY_LOW,
+	BTC_RSSI_ST_STAY_HIGH,
+	BTC_RSSI_ST_MAX
+};
+
+enum btc_fddt_en {
+	BTC_FDDT_DISABLE,
+	BTC_FDDT_ENABLE,
+};
+
+enum btc_fddt_type {
+	BTC_FDDT_TYPE_STOP,
+	BTC_FDDT_TYPE_AUTO,
+	BTC_FDDT_TYPE_FIX_TDD,
+	BTC_FDDT_TYPE_FIX_FULL_FDD,
+	BTC_FDDT_MAX
+};
+
+enum btc_chip_feature {
+	BTC_FEAT_PTA_ONOFF_CTRL  = BIT(0), /* on/off ctrl by HW (not 0x73[2]) */
+	BTC_FEAT_NONBTG_GWL_THRU = BIT(1),  /* non-BTG GNT_WL!=0 if GNT_BT = 1*/
+	BTC_FEAT_WLAN_ACT_MUX = BIT(2)
+};
+
+enum btc_coex_info_map_en {
+	BTC_COEX_INFO_CX = BIT(0),
+	BTC_COEX_INFO_WL = BIT(1),
+	BTC_COEX_INFO_BT = BIT(2),
+	BTC_COEX_INFO_DM = BIT(3),
+	BTC_COEX_INFO_MREG = BIT(4),
+	BTC_COEX_INFO_SUMMARY = BIT(5),
+	BTC_COEX_INFO_ALL = 0xff
+};
+
+enum btc_vendor_id {
+	BTC_VID_NONE = 0,
+	BTC_VID_HP = 1,
+	BTC_VID_ASUS = 2,
+	BTC_VID_ACER = 3,
+	BTC_VID_LNV = 4,
+	BTC_VID_NEC = 5,
+	BTC_VID_AMD = 6,
+	BTC_VID_FUJITSU = 7,
+	BTC_VID_DELL = 8,
+};
+
+/* list all-chip scoreboard definition, remap at chip file */
+enum btc_w2b_scoreboard {
+	BTC_WSCB_ACTIVE	= BIT(0),
+	BTC_WSCB_ON = BIT(1),
+	BTC_WSCB_SCAN = BIT(2),
+	BTC_WSCB_UNDERTEST = BIT(3),
+	BTC_WSCB_RXGAIN = BIT(4), /* set BT LNA constrain for free-run */
+	BTC_WSCB_5GHICH = BIT(6),
+	BTC_WSCB_WLBUSY = BIT(7),
+	BTC_WSCB_EXTFEM = BIT(8),
+	BTC_WSCB_TDMA = BIT(9),
+	BTC_WSCB_FIX2M = BIT(10),
+	BTC_WSCB_WLRFK = BIT(11),
+	BTC_WSCB_RXSCAN_PRI = BIT(12), /* set BT Rx-Scan PTA pri */
+	BTC_WSCB_BT_HILNA = BIT(13), /* request BT use Hi-LNA table for BTG */
+	BTC_WSCB_BTLOG = BIT(14),  /* open BT log */
+	BTC_WSCB_COUNTRY_CODE = BIT(15), /*ready to send country code*/
+	BTC_WSCB_ALL = 0xffffff /* driver only use bit0~23 */
+};
+
+enum btc_b2w_scoreboard {
+	BTC_BSCB_ACT = BIT(0),
+	BTC_BSCB_ON = BIT(1),
+	BTC_BSCB_WHQL = BIT(2),
+	BTC_BSCB_BT_S1 = BIT(3),  /* 1-> BT at S1, 0-> BT at S0 */
+	BTC_BSCB_A2DP_ACT = BIT(4),
+	BTC_BSCB_RFK_RUN = BIT(5),
+	BTC_BSCB_RFK_REQ = BIT(6),
+	BTC_BSCB_LPS = BIT(7),
+	BTC_BSCB_BT_LNAB0 = BIT(8),
+	BTC_BSCB_BT_LNAB1 = BIT(10),
+	BTC_BSCB_WLRFK = BIT(11),
+	BTC_BSCB_BT_HILNA = BIT(13), /* reply if BT use Hi-LNA table for BTG */
+	BTC_BSCB_COUNTRY_CODE = BIT(15), /*ready to receive country code*/
+	BTC_BSCB_BT_CONNECT = BIT(16), /* If any BT connected */
+	BTC_BSCB_PATCH_CODE = BIT(30),  /* BT use 1: patch code 2:ROM code */
+	BTC_BSCB_ALL = 0x7fffffff
+};
+
+enum btc_dcnt_type {
+	BTC_DCNT_RUN = 0x0,
+	BTC_DCNT_CX_RUNINFO,
+	BTC_DCNT_RPT,
+	BTC_DCNT_RPT_HANG,
+	BTC_DCNT_CYCLE,
+	BTC_DCNT_CYCLE_HANG,
+	BTC_DCNT_W1,
+	BTC_DCNT_W1_HANG,
+	BTC_DCNT_B1,
+	BTC_DCNT_B1_HANG,
+	BTC_DCNT_TDMA_NONSYNC,
+	BTC_DCNT_SLOT_NONSYNC,
+	BTC_DCNT_BTCNT_HANG,
+	BTC_DCNT_BTTX_HANG,
+	BTC_DCNT_WL_SLOT_DRIFT,
+	BTC_DCNT_WL_STA_LAST,
+	BTC_DCNT_BT_SLOT_DRIFT,
+	BTC_DCNT_BT_SLOT_FLOOD,
+	BTC_DCNT_FDDT_TRIG,
+	BTC_DCNT_E2G,
+	BTC_DCNT_E2G_HANG,
+	BTC_DCNT_WL_FW_VER_MATCH,
+	BTC_DCNT_NULL_TX_FAIL,
+	BTC_DCNT_WL_STA_NTFY,
+	BTC_DCNT_MAX
+};
+
+enum btc_ntfy_type {
+	BTC_NTFY_POWER_ON = 0x0,
+	BTC_NTFY_POWER_OFF,
+	BTC_NTFY_INIT_COEX,
+	BTC_NTFY_SCAN_START,
+	BTC_NTFY_SCAN_FINISH,
+	BTC_NTFY_SPECIAL_PACKET,
+	BTC_NTFY_SWITCH_BAND,
+	BTC_NTFY_RFK_TIMEOUT,
+	BTC_NTFY_SHOW_COEX_INFO,
+	BTC_NTFY_ROLE_INFO,
+	BTC_NTFY_RADIO_STATE,
+	BTC_NTFY_CUSTOMERIZE,
+	BTC_NTFY_WL_RFK,
+	BTC_NTFY_WL_STA,
+	BTC_NTFY_WL_STA_LAST,
+	BTC_NTFY_FWINFO,
+	BTC_NTFY_TIMER,
+	BTC_NTFY_RESUME_DL_FW,
+	BTC_NTFY_COUNTRYCODE,
+	BTC_NTFY_MAX
+};
+
+enum btc_wl_state_cnt {
+	BTC_WCNT_SCANAP = 0x0,
+	BTC_WCNT_DHCP,
+	BTC_WCNT_EAPOL,
+	BTC_WCNT_ARP,
+	BTC_WCNT_SCBDUPDATE,
+	BTC_WCNT_RFK_REQ,
+	BTC_WCNT_RFK_GO,
+	BTC_WCNT_RFK_REJECT,
+	BTC_WCNT_RFK_TIMEOUT,
+	BTC_WCNT_CH_UPDATE,
+	BTC_WCNT_DBCC_ALL_2G,
+	BTC_WCNT_DBCC_CHG,
+	BTC_WCNT_RX_OK_LAST,
+	BTC_WCNT_RX_OK_LAST2S,
+	BTC_WCNT_RX_ERR_LAST,
+	BTC_WCNT_RX_ERR_LAST2S,
+	BTC_WCNT_RX_LAST,
+	BTC_WCNT_MAX
+};
+
+enum btc_bt_state_cnt {
+	BTC_BCNT_RETRY = 0x0,
+	BTC_BCNT_REINIT,
+	BTC_BCNT_REENABLE,
+	BTC_BCNT_SCBDREAD,
+	BTC_BCNT_RELINK,
+	BTC_BCNT_IGNOWL,
+	BTC_BCNT_INQPAG,
+	BTC_BCNT_INQ,
+	BTC_BCNT_PAGE,
+	BTC_BCNT_ROLESW,
+	BTC_BCNT_AFH,
+	BTC_BCNT_INFOUPDATE,
+	BTC_BCNT_LEAUDIO_INFOUPDATE,
+	BTC_BCNT_INFOSAME,
+	BTC_BCNT_SCBDUPDATE,
+	BTC_BCNT_HIPRI_TX,
+	BTC_BCNT_HIPRI_RX,
+	BTC_BCNT_LOPRI_TX,
+	BTC_BCNT_LOPRI_RX,
+	BTC_BCNT_POLUT_NOW,
+	BTC_BCNT_POLUT_DIFF,
+	BTC_BCNT_RATECHG,
+	BTC_BCNT_AFH_CONFLICT,
+	BTC_BCNT_AFH_LE_CONFLICT,
+	BTC_BCNT_AFH_UPDATE,
+	BTC_BCNT_AFH_LE_UPDATE,
+	BTC_BCNT_AFH_CHN,
+	BTC_BCNT_AFH_LE_CHN,
+	BTC_BCNT_MAX
+};
+
+enum btc_wl_link_mode {
+	BTC_WLINK_NOLINK = 0x0,
+	BTC_WLINK_2G_STA,
+	BTC_WLINK_2G_AP,
+	BTC_WLINK_2G_GO,
+	BTC_WLINK_2G_GC,
+	BTC_WLINK_2G_SCC,
+	BTC_WLINK_2G_MCC,
+	BTC_WLINK_25G_MCC,
+	BTC_WLINK_25G_DBCC,
+	BTC_WLINK_5G,
+	BTC_WLINK_2G_NAN,
+	BTC_WLINK_OTHER,
+	BTC_WLINK_MAX
+};
+
+enum btc_wl_mrole_type {
+	BTC_WLMROLE_NONE = 0x0,
+	BTC_WLMROLE_STA_GC,
+	BTC_WLMROLE_STA_GC_NOA,
+	BTC_WLMROLE_STA_GO,
+	BTC_WLMROLE_STA_GO_NOA,
+	BTC_WLMROLE_STA_STA,
+	BTC_WLMROLE_MAX
+};
+
+enum btc_btinfo_byte {
+	BTC_BTINFO_L0 = 0,
+	BTC_BTINFO_L1,
+	BTC_BTINFO_L2,
+	BTC_BTINFO_L3,
+	BTC_BTINFO_H0,
+	BTC_BTINFO_H1,
+	BTC_BTINFO_H2,
+	BTC_BTINFO_H3,
+	BTC_BTINFO_MAX
+};
+
+enum btc_dm_error {
+	BTC_DMERR_INIT = BIT(0),
+	BTC_DMERR_PTA_OWNER = BIT(1),
+	BTC_DMERR_WL_RFK_TIMEOUT = BIT(2),
+	BTC_DMERR_BT_RFK_TIMEOUT = BIT(3),
+
+	BTC_DMERR_WL_FW_HANG = BIT(4),
+	BTC_DMERR_CYCLE_HANG = BIT(5),
+	BTC_DMERR_W1_HANG = BIT(6),
+	BTC_DMERR_B1_HANG = BIT(7),
+
+	BTC_DMERR_TDMA_NO_SYNC = BIT(8),
+	BTC_DMERR_SLOT_NO_SYNC = BIT(9),
+	BTC_DMERR_WL_SLOT_DRIFT = BIT(10),
+	BTC_DMERR_BT_SLOT_DRIFT = BIT(11), /* for external slot control */
+
+	BTC_DMERR_ROLE_NUM_MISMATCH = BIT(12),
+	BTC_DMERR_NULL1_TX_LATE = BIT(13),
+	BTC_DMERR_AFH_CONFLICT = BIT(14),
+	BTC_DMERR_AFH_LE_CONFLICT = BIT(15),
+
+	BTC_DMERR_BT_SLOT_FLOOD = BIT(16),
+	BTC_DMERR_E2G_HANG = BIT(17),
+	BTC_DMERR_WL_VER_MISMATCH = BIT(18),
+	BTC_DMERR_BT_VER_MISMATCH = BIT(19),
+
+	BTC_DMERR_RFE_TYPE0 = BIT(20),
+	BTC_DMERR_H2C_BUF_OVER = BIT(21),
+	BTC_DMERR_BTTX_HANG = BIT(22), /* for SNR too low bug, BT has no Tx req*/
+	BTC_DMERR_WL_NO_STA_NTFY = BIT(23),
+	BTC_DMERR_H2C_BMAP_MISMATCH = BIT(24),
+	BTC_DMERR_C2H_BMAP_MISMATCH = BIT(25),
+	BTC_DMERR_H2C_STRUCT_INVALID = BIT(26),
+	BTC_DMERR_C2H_STRUCT_INVALID = BIT(27),
+	BTC_DMERR_CCODE_BTNORS = BIT(28),
+	BTC_DMERR_MAX = 29
+};
+
+enum btc_fw_btc_error {
+	BTC_FWERR_NULL_RESULT = BIT(0),
+	BTC_FWERR_NULL_EXCEPTION = BIT(1),
+	BTC_FWERR_B1_EXCEPTION = BIT(2),
+	BTC_FWERR_W1_EXCEPTION = BIT(3),
+
+	BTC_FWERR_B2_EXCEPTION = BIT(4),
+	BTC_FWERR_B3_EXCEPTION = BIT(5),
+	BTC_FWERR_B4_EXCEPTION = BIT(6),
+	BTC_FWERR_LPS_TIMER_START = BIT(7),
+
+	BTC_FWERR_EXAUTO_EXCEPTION = BIT(8),
+	BTC_FWERR_BUF_OVERFLOW = BIT(9),
+	BTC_FWERR_BRLY_EXCEPTION = BIT(10),
+	BTC_FWERR_SM = BIT(11),  /* un-known-event */
+
+	BTC_FWERR_B2_AFTER = BIT(12),
+	BTC_FWERR_LK_END = BIT(13),
+	BTC_FWERR_TMR_FAIL = BIT(14),
+	BTC_FWERR_H2C_SET_EXCEPTION = BIT(15),
+	BTC_FWERR_MAX = 16
+};
+
+enum btc_get_reg_status {
+	BTC_CSTATUS_TXDIV_POS = 0,
+	BTC_CSTATUS_RXDIV_POS = 1,
+	BTC_CSTATUS_BB_GNT_MUX = 2,
+	BTC_CSTATUS_BB_GNT_MUX_MON = 3,
+	BTC_CSTATUS_BB_PRE_AGC = 4,
+	BTC_CSTATUS_BB_PRE_AGC_MON = 5
+};
+
+enum halrf_rfk_type {
+	BTC_RF_IQK	= 0,
+	BTC_RF_LCK	= 1,
+	BTC_RF_DPK	= 2,
+	BTC_RF_TXGAPK	= 3,
+	BTC_RF_DACK	= 4,
+	BTC_RF_RXDCK	= 5,
+	BTC_RF_TSSI	= 6,
+	BTC_RF_CHLK	= 7
+};
+
+enum btc_ant_div_pos {
+	BTC_ANT_DIV_MAIN = 0,
+	BTC_ANT_DIV_AUX = 1
+};
+
+enum btc_fw_data_dir {
+	BTC_FW_H2C = 0,
+	BTC_FW_C2H = 1
+};
+
+enum btc_rinfo_lo_b2 {
+	BTC_RINFO_INQPAG = BIT(2),
+};
+
+enum btc_bt_hfp_type {
+	BTC_HFP_SCO = 0,
+	BTC_HFP_ESCO = 1,
+	BTC_HFP_ESCO_2SLOT = 2,
+};
+
+enum btc_bt_hid_type {
+	BTC_HID_218 = BIT(0),
+	BTC_HID_418 = BIT(1),
+	BTC_HID_BLE = BIT(2),
+	BTC_HID_RCU = BIT(3),
+	BTC_HID_RCU_VOICE = BIT(4),
+	BTC_HID_OTHER_LEGACY = BIT(5)
+};
+
+enum btc_bt_a2dp_type {
+	BTC_A2DP_LEGACY = 0,
+	BTC_A2DP_TWS_SNIFF = 1, /* Airpod */
+	BTC_A2DP_TWS_RELAY = 2, /* RTL8763B */
+};
+
+enum btc_bt_PAN_type {
+	BTC_BT_PAN_ONLY = 0,
+	BTC_BT_OPP_ONLY = 1
+};
+
+enum btc_bt_mailbox_id {
+	BTC_BTINFO_REPLY = 0x23,
+	BTC_BTINFO_AUTO = 0x27
+};
+
+enum btc_bt_profile_map {
+	BTC_BT_HFP = BIT(0),
+	BTC_BT_HID = BIT(1),
+	BTC_BT_A2DP = BIT(2),
+	BTC_BT_PAN = BIT(3),
+	BTC_BT_BIS = BIT(4),
+	BTC_BT_CIS = BIT(5),
+	BTC_PROFILE_MAX = 5,
+	BTC_BT_ALL = 0xf
+};
+
+enum btc_iqk_state {
+	BTC_BT_IQK_OK = 0,
+	BTC_BT_IQK_START = 1,
+	BTC_BT_IQK_STOP = 2,
+	BTC_BT_IQK_MAX
+};
+
+enum btc_ant_share_type {
+	BTC_ANT_SHARED = 0,
+	BTC_ANT_DEDICATED,
+	BTC_ANTTYPE_MAX
+};
+
+enum btc_extsoc_func { /* cx->other.type */
+	BTC_EXTSOC_FUNC_NONE = 0,
+	BTC_EXTSOC_FUNC_BT = BIT(0),
+	BTC_EXTSOC_FUNC_ZB = BIT(1),
+	BTC_EXTSOC_FUNC_THREAD = BIT(2),
+	BTC_EXTSOC_FUNC_24GPRO = BIT(3),
+	BTC_EXTSOC_FUNC_MAX
+};
+
+enum btc_extsoc_interface { /* cx->other.hw_coex */
+	BTC_EXTSOC_INTF_NONE = 0,
+	BTC_EXTSOC_INTF_PTA = BIT(0),
+	BTC_EXTSOC_INTF_MBX = BIT(1),
+	BTC_EXTSOC_INTF_SWIO = BIT(2),
+	BTC_EXTSOC_INTF_MAX
+};
+
+enum btc_bt_combo_type {
+	BTC_BT_ALONE = 0,
+	BTC_BT_BTG
+};
+
+enum btc_ant_switch_type {
+	BTC_SWITCH_INTERNAL = 0,  /* internal or none */
+	BTC_SWITCH_EXTERNAL
+};
+
+enum btc_reset_var_map {
+	BTC_RESET_CX = BIT(0),
+	BTC_RESET_DM = BIT(1),
+	BTC_RESET_CTRL = BIT(2),
+	BTC_RESET_CXDM = BIT(0) | BIT(1),
+	BTC_RESET_BTINFO = BIT(3),
+	BTC_RESET_MDINFO = BIT(4),
+	BTC_RESET_BT_PSD_DM = BIT(5),
+	BTC_RESET_ALL = 0xff
+};
+
+enum btc_gnt_state {
+	BTC_GNT_HW	= 0,
+	BTC_GNT_SW_LO,
+	BTC_GNT_SW_HI,
+	BTC_GNT_MAX
+};
+
+enum btc_wlact_state {
+	BTC_WLACT_HW	= 0,
+	BTC_WLACT_SW_LO,
+	BTC_WLACT_SW_HI,
+	BTC_WLACT_MAX
+};
+
+enum btc_wl_max_tx_time {
+	BTC_MAX_TX_TIME_L1 = 500,
+	BTC_MAX_TX_TIME_L2 = 1000,
+	BTC_MAX_TX_TIME_L3 = 2000,
+	BTC_MAX_TX_TIME_DEF = 5280
+};
+
+enum btc_wl_max_tx_retry {
+	BTC_MAX_TX_RETRY_L1 = 7,
+	BTC_MAX_TX_RETRY_L2 = 15,
+	BTC_MAX_TX_RETRY_DEF = 31
+};
+
+enum btc_nofddt_rsn {
+	BTC_NFRSN_SUPPORT = 0,
+	BTC_NFRSN_FORCE_STOP,
+	BTC_NFRSN_DEDICATED_ANT,
+	BTC_NFRSN_ANT_ISO_LOW,
+	BTC_NFRSN_ANT_ISO_HI,
+	BTC_NFRSN_WL_2GSTA,     /* if 2G STA only */
+	BTC_NFRSN_WL_BUSY,      /* if idle/LPS */
+	BTC_NFRSN_WL_NOSCAN,
+	BTC_NFRSN_WL_NORFK,
+	BTC_NFRSN_WB_RSSI,      /* if WL & BT RSSI to low */
+	BTC_NFRSN_BT_PROFILE,   /* if NOT A2DP or A2DP + HID/HFP */
+	BTC_NFRSN_BT_A2DP_BUSY, /* if A2DP idle */
+	BTC_NFRSN_BT_NOINQ,
+	BTC_NFRSN_COND_NUM,
+	BTC_NFRSN_NHM,
+	BTC_NFRSN_RETRY_PERIOD,/* if bt retry-period not reach */
+	BTC_NFRSN_MAX
+};
+
+enum btc_fddt_state {
+	BTC_FDDT_STATE_STOP,
+	BTC_FDDT_STATE_RUN,
+	BTC_FDDT_STATE_PENDING,
+	BTC_FDDT_STATE_DEBUG,
+	BTC_FDDT_STATE_MAX
+};
+
+enum btc_fddt_result {
+	BTC_FDDT_RESULT_UNFINISH,
+	BTC_FDDT_RESULT_OK,
+	BTC_FDDT_RESULT_FAIL,
+	BTC_FDDT_RESULT_MAX
+};
+
+enum btc_fddt_train_state {
+	CX_FDDT_STATE_STOP = 0,
+	CX_FDDT_STATE_RUN = 1,
+	CX_FDDT_STATE_OK = 2,
+	CX_FDDT_STATE_FAIL = 3,
+	CX_FDDT_STATE_NA = 4,  /* no train, always TDD for low-rssi case */
+	CX_FDDT_STATE_DEBUG = 5,
+	CX_FDDT_STATE_MAX
+};
+
+enum btc_fddt_train_phase {
+	CX_FDDT_PHASE_STOP = 0, /* for train-enable = 0  */
+	CX_FDDT_PHASE_DEBUG = 1,   /* for train-debug = 1 or TDMA-off  */
+	CX_FDDT_PHASE_TDD = 2,      /* TDD-phase  */
+	CX_FDDT_PHASE_FDD_A = 3,    /* FDD-phase, use A/B toggle for different step */
+	CX_FDDT_PHASE_FDD_B = 4,
+	CX_FDDT_PHASE_TDD_STAY = 5, /* always TDD, for TRAINFAIL/TRAINNA or CN/Low-TP */
+	CX_FDDT_PHASE_FDD_STAY = 6, /* for TRAINOK always FDD */
+	CX_FDDT_PHASE_TDD_FAIL = 7,
+	CX_FDDT_PHASE_MAX
+};
+
+enum btc_fddt_check_map {
+	CX_FDDT_CHKMAP_NOEMPTY = 0,
+	CX_FDDT_CHKMAP_TPRATIO = 1,
+	CX_FDDT_CHKMAP_TPLOW = 2,
+	CX_FDDT_CHKMAP_CN = 3,
+	CX_FDDT_CHKMAP_CELLCHG = 4,
+	CX_FDDT_CHKMAP_NHM = 5,
+	CX_FDDT_CHKMAP_MAX
+};
+
+enum btc_wa_type {
+	BTC_WA_5G_HI_CH_RX = BIT(0),
+	BTC_WA_NULL_AP = BIT(1),
+	BTC_WA_HFP_ZB = BIT(2),  /* HFP PTA req bit4 define issue */
+	BTC_WA_HFP_LAG = BIT(3),  /*52BT WL break BT Rx lag issue*/
+	BTC_WA_INIT_SCAN = BIT(4),  /*52A/C/D init scan move to wl slot WA*/
+	BTC_WA_CO_RX = BIT(5)  /* 52B high rate co-rx issue */
+};
+
+struct btc_byte_map_desc {
+	u16 map_u8;
+	u16 map_u16;
+	u16 map_u32;
+};
+
+struct btc_btinfo_lb2 {
+	u8 connect: 1;
+	u8 sco_busy: 1;
+	u8 inq_pag: 1;
+	u8 acl_busy: 1;
+	u8 hfp: 1;
+	u8 hid: 1;
+	u8 a2dp: 1;
+	u8 pan: 1;
+};
+
+struct btc_btinfo_lb3 {
+	u8 retry: 4;
+	u8 cqddr: 1;
+	u8 inq: 1;
+	u8 mesh_busy: 1;
+	u8 pag: 1;
+};
+
+struct btc_btinfo_hb0 {
+	u8 rssi;
+};
+
+struct btc_btinfo_hb1 {
+	u8 ble_connect: 1;
+	u8 reinit: 1;
+	u8 relink: 1;
+	u8 igno_wl: 1;
+	u8 voice: 1;
+	u8 ble_scan: 1;
+	u8 role_sw: 1;
+	u8 multi_link: 1;
+};
+
+struct btc_btinfo_hb2 {
+	u8 pan_active: 1;
+	u8 afh_update: 1;
+	u8 a2dp_active: 1;
+	u8 slave: 1;
+	u8 hid_slot: 2;
+	u8 hid_cnt: 2;
+};
+
+struct btc_btinfo_hb3 {
+	u8 a2dp_bitpool: 6;
+	u8 tx_3M: 1;
+	u8 a2dp_sink: 1;
+};
+
+union btc_btinfo {
+	u8 val;
+	struct btc_btinfo_lb2 lb2;
+	struct btc_btinfo_lb3 lb3;
+	struct btc_btinfo_hb0 hb0;
+	struct btc_btinfo_hb1 hb1;
+	struct btc_btinfo_hb2 hb2;
+	struct btc_btinfo_hb3 hb3;
+};
+
+struct btc_wl_tx_limit_para {
+	u8 en;
+	u8 tx_1ss;
+	u32 tx_time;	/* unit: us */
+	u16 tx_retry;
+};
+
+struct btc_wl_trx_nss_para {
+	u8 tx_limit;
+	u8 rx_limit;
+	u8 tx_ss;
+	u8 rx_ss;
+	u8 tx_path;
+	u8 rx_path;
+};
+
+struct btc_rf_trx_para {
+	u32 wl_tx_power; /* absolute Tx power (dBm), 1's complement -5->0x85 */
+	u32 wl_rx_gain;  /* rx gain table index (TBD.) */
+	u32 bt_tx_power; /* decrease Tx power (dB) */
+	u32 bt_rx_gain;  /* LNA constrain level */
+};
+
+struct btc_gnt_ctrl {
+	u8 gnt_bt_sw_en;
+	u8 gnt_bt;
+	u8 gnt_wl_sw_en;
+	u8 gnt_wl;
+};
+
+struct btc_wlact_ctrl {
+	u8 sw_en;
+	u8 sw_val;
+};
+
+struct btc_fw_dm {
+	u32 freerun: 1;
+	u32 wl_ps_ctrl: 2; /* 0: off, 1: force on, 2:forec off */
+	u32 wl_mimo_ps: 1;
+	u32 leak_ap: 1;
+	u32 igno_bt: 1;
+	u32 noisy_level: 3;
+	u32 set_ant_path: 16;
+	u32 rsvd: 7;
+};
+
+struct btc_cxr_result {
+	struct btc_fw_dm dm;
+	struct btc_rf_trx_para rf_trx_para;
+	u32 cx_state_map;
+	u32 policy_type;
+	u32 run_cnt;
+
+	char run_reason[BTC_RSN_MAXLEN];
+	char run_action[BTC_ACT_MAXLEN];
+};
+
+struct btc_bt_link_hid {
+	u8 h;  /* handle_id */
+	u8 vendor_id;
+	u8 update;
+	u8 interval;
+	u8 window;
+	bool type;
+	bool role;
+	bool sniff;
+};
+
+struct btc_bt_link_a2dp {
+	u8 h;  /* handle_id */
+	u8 vendor_id;
+	u8 update;
+	bool type;
+	bool role;
+	bool sniff;
+};
+
+struct btc_bt_link_pan {
+	u8 h;  /* handle_id */
+	u8 vendor_id;
+	u8 update;
+	bool type;
+	bool role;
+	bool sniff;
+	bool direct; /* 0:Rx, 1:Tx */
+};
+
+struct btc_bt_link_hfp {
+	u8 h;  /* handle_id */
+	u8 vendor_id;
+	u8 update;
+	bool type;
+	bool role;
+	bool sniff;
+};
+
+struct btc_bt_link_noprofile {
+	u8 h;  /* handle_id */
+	u8 vendor_id;
+	u8 update;
+	bool type;
+	bool role;
+	bool sniff;
+};
+
+struct btc_bt_hfp_desc {
+	u32 exist: 1;
+	u32 exist_last: 1;
+	u32 type: 2; /* refer to btc_bt_hfp_type */
+	u32 handle_update: 1;
+	u32 devinfo_query: 1;
+	u32 rsvd: 26;
+
+	u8 cnt;
+	u8 lists[BTC_DEV_LINK_MAX];
+	struct btc_bt_link_hfp links[BTC_DEV_LINK_MAX];
+};
+
+struct btc_bt_hid_desc {
+	u32 exist: 1;
+	u32 exist_last: 1;
+	u32 slot_info: 2;
+	u32 pair_cnt: 2;
+	u32 pair_cnt_last: 2;
+	u32 type: 8; /* refer to btc_bt_hid_type */
+	u32 handle_update: 1;
+	u32 devinfo_query: 1;
+	u32 query_timer: 4;
+	u32 rsvd: 10;
+
+	u8 cnt;
+	u8 lists[BTC_DEV_LINK_MAX];
+	struct btc_bt_link_hid links[BTC_DEV_LINK_MAX];
+};
+
+struct btc_bt_a2dp_desc {
+	u32 exist: 1;
+	u32 exist_last: 1;
+	u32 play_latency: 1;
+	u32 type: 3; /* refer to btc_bt_a2dp_type */
+	u32 active: 1;
+	u32 sink: 1;
+	u32 handle_update: 1;
+	u32 devinfo_query: 1;
+	u32 no_empty_streak_2s: 8;
+	u32 no_empty_streak_max: 8;
+	u32 rsvd: 6;
+
+	u8 bitpool;
+	u16 vendor_id;
+	u32 device_name;
+	u32 flush_time;
+
+	u8 cnt;
+	u8 lists[BTC_DEV_LINK_MAX];
+	struct btc_bt_link_a2dp links[BTC_DEV_LINK_MAX];
+};
+
+struct btc_bt_pan_desc {
+	u32 exist: 1;
+	u32 exist_last: 1;
+	u32 type: 1; /* refer to btc_bt_pan_type */
+	u32 active: 1;
+	u32 handle_update: 1;
+	u32 devinfo_query: 1;
+	u32 rsvd: 26;
+
+	u8 cnt;
+	u8 lists[BTC_DEV_LINK_MAX];
+	struct btc_bt_link_pan links[BTC_DEV_LINK_MAX];
+};
+
+struct btc_bt_leaudio_desc {
+	u32 exist: 1;
+	u32 exist_last: 1;
+	u32 bis_cnt: 2;
+	u32 cis_cnt: 2;
+	u32 rssi: 8;
+	u32 bis_cnt_last: 2;
+	u32 cis_cnt_last: 2;
+	u32 rsvd: 14;
+};
+
+struct btc_bt_noprofile_desc {
+	u8 no_profile_cnt;
+	u8 no_profile[BTC_DEV_LINK_MAX];
+	struct btc_bt_link_noprofile noprofiles[BTC_DEV_LINK_MAX];
+};
+
+struct btc_wl_nhm {
+	s8 instant_wl_nhm_dbm;
+	s8 instant_wl_nhm_per_mhz;
+	u16 valid_record_times;
+	s8 record_pwr[16];
+	u8 record_ratio[16];
+	s8 pwr; /* dbm_per_MHz  */
+	u8 ratio;
+	u8 current_status;
+	u8 refresh;
+	bool start_flag;
+	u8 last_ccx_rpt_stamp;
+	s8 pwr_max;
+	s8 pwr_min;
+};
+
+struct btc_bt_rfk_info {
+	u32 run: 1;
+	u32 req: 1;
+	u32 timeout: 1;
+	u32 rsvd: 29;
+};
+
+union btc_bt_rfk_info_map {
+	u32 val;
+	struct btc_bt_rfk_info map;
+};
+
+struct btc_bt_ver_info {
+	u32 fw_coex; /* match with which coex_ver */
+	u32 fw;
+};
+struct btc_bool_sta_chg { /* status change varible */
+	u32 now: 1;
+	u32 last: 1;
+	u32 remain: 1;
+	u32 srvd: 29;
+};
+
+struct btc_u8_sta_chg { /* status change varible */
+	u8 now;
+	u8 last;
+	u8 remain;
+	u8 rsvd;
+};
+
+struct btc_bit_remap {
+	u32 index;
+	u32 bpos;
+};
+
+struct btc_traffic {
+	enum rtw_tfc_lvl tx_lvl;
+	enum rtw_tfc_sts tx_sts;
+	enum rtw_tfc_lvl rx_lvl;
+	enum rtw_tfc_sts rx_sts;
+
+	u64 tx_byte;
+	u64 rx_byte;
+	u32 tx_time;
+	u32 rx_time;
+
+	u32 tx_tp;
+	u32 rx_tp;
+};
+
+struct btc_wl_scan_info {
+	u8 band[HW_PHY_MAX];
+	u8 phy_map;
+	u8 rsvd;
+};
+
+struct btc_wl_dbcc_info {
+	u8 op_band[HW_PHY_MAX]; /* op band in each phy */
+	u8 scan_band[HW_PHY_MAX]; /* scan band in  each phy */
+	u8 real_band[HW_PHY_MAX];
+	u16 role[HW_PHY_MAX]; /* role in each phy */
+};
+
+struct btc_wl_scc_ctrl {
+	u8 null_role1;
+	u8 null_role2;
+	u8 e2g_null; /* if tx null at E2G-slot before entering EBT-slot */
+	u8 rsvd;
+};
+
+struct btc_wl_link_info {
+	struct rtw_chan_def chdef;
+
+	u8 mode; /* wl mode: 11b, 11g, 11n... */
+	u8 ch;
+	u8 bw;
+	u8 band; /* enum  RF band: 2.4G/5G/6G */
+	u8 role; /*enum role_type */
+	u8 pid; /* MAC HW Port id: 0~4 */
+	u8 phy; /* PHY-0 or PHY-1*/
+	u8 dtim_period;
+	u8 rssi; /* 0%~110% (dBm = rssi -110) */
+	u8 tx_1ss_limit;
+
+	u8 busy;
+	u8 rssi_state[BTC_WL_RSSI_THMAX];
+	u8 mac_addr[6];
+	u8 tx_retry;
+
+	u16 mac_id; /* 0~63 */
+	u16 tx_rate;
+	u16 rx_rate;
+
+	u32 bcn_period;
+	u32 noa_duration; /* us */
+	u32 tx_time; /* the original max_tx_time */
+	u32 client_cnt; /* connected-client cnt for p2p-GO/soft-AP mode */
+	u32 rx_rate_drop_cnt;
+
+	u32 active: 1;
+	u32 noa: 1; /* for P2P */
+	u32 client_ps: 1; /* For AP mode, if client enter LPS */
+	u32 connected: 2;
+};
+
+struct btc_wl_rfk_info {
+	u32 state: 2;
+	u32 path_map: 4;
+	u32 phy_map: 2;
+	u32 band: 2; /*0:2g, 1:5g, 2:6g  */
+	u32 type: 8;
+	u32 rsvd: 14;
+
+	u32 start_time;
+	u32 proc_time;
+};
+
+struct btc_wl_ver_info {
+	u32 fw_coex; /* match with which coex_ver */
+	u32 fw;
+	u32 mac;
+	u32 bb;
+	u32 rf;
+};
+
+struct btc_wl_afh_info {
+	u8 en;
+	u8 ch;
+	u8 bw;
+	u8 rsvd;
+};
+
+struct btc_rf_cfg {
+	u32 addr;
+	u32 val;
+};
+
+struct btc_bt_smap {
+	u32 connect: 1;
+	u32 ble_connect: 1;
+	u32 acl_busy: 1;
+	u32 sco_busy: 1;
+	u32 mesh_busy: 1;
+	u32 inq_pag: 1;
+};
+
+union btc_bt_state_map {
+	u32 val;
+	struct btc_bt_smap map;
+};
+
+struct btc_bt_link_info {
+	struct btc_u8_sta_chg profile_cnt;
+	struct btc_bool_sta_chg multi_link;
+	struct btc_bool_sta_chg relink;
+	struct btc_bt_hfp_desc hfp_desc;
+	struct btc_bt_hid_desc hid_desc;
+	struct btc_bt_a2dp_desc a2dp_desc;
+	struct btc_bt_pan_desc pan_desc;
+	struct btc_bt_leaudio_desc leaudio_desc;
+	struct btc_bt_noprofile_desc noprofile_desc;
+	union btc_bt_state_map status;
+
+	u8 sut_pwr_level[BTC_PROFILE_MAX];
+	u8 golden_rx_shift[BTC_PROFILE_MAX];
+	u8 rssi_state[BTC_BT_RSSI_THMAX];
+	u8 afh_map[12];
+	u8 afh_map_le[5];
+
+	u32 role_sw: 1;
+	u32 slave_role: 1;
+	u32 afh_update: 1;
+	u32 cqddr: 1;
+	u32 rssi: 8;
+	u32 tx_3M: 1;
+	u32 rsvd: 19;
+};
+
+struct btc_3rdcx_info {
+	u8 type;
+	u8 hw_coex; /* Hard-Wire coex interface support */
+	u8 ext_pta_type; /* 0: RTK 4-wire mode, 1: 3-wire mode */
+	u8 ext_pta_req_exist;
+
+#if 0
+	u8 esoc_state;
+	u8 esoc_ch;
+	u8 esoc_bw;
+	u8 esoc_rssi;
+	u8 esoc_co_ch;
+	u8 esoc_type;
+#endif
+
+	u32 ext_hpta_cfg;
+	u32 ext_hmbx_cfg;
+	u32 ext_swout_cfg;
+	u32 ext_swin_cfg;
+};
+
+struct btc_rf_para {
+	u32 tx_pwr_freerun;
+	u32 rx_gain_freerun;
+	u32 tx_pwr_perpkt;
+	u32 rx_gain_perpkt;
+};
+
+struct btc_bt_info {
+	struct btc_bt_link_info link_info;
+	struct btc_bt_scan_info scan_info[CXSCAN_MAX1];
+	struct btc_bt_ver_info ver_info;
+	struct btc_bool_sta_chg enable;
+	struct btc_bool_sta_chg inq_pag;
+	struct btc_rf_para rf_para; /* for record */
+	union btc_bt_rfk_info_map rfk_info;
+
+	u8 raw_info[BTC_BTINFO_MAX]; /* raw bt info from mailbox */
+	u8 rssi_level;
+
+	u32 scbd; /* scoreboard value */
+	u32 feature;
+
+	u32 mbx_avl: 1; /* mailbox available */
+	u32 whql_test: 1;
+	u32 igno_wl: 1;
+	u32 reinit: 1;
+	u32 ble_scan_en: 1;
+	u32 btg_type: 1; /* BT located at BT only or BTG */
+	u32 inq: 1;
+	u32 pag: 1;
+
+	u32 run_patch_code: 1;
+	u32 hi_lna_rx: 1;
+	u32 scan_rx_low_pri: 1;
+	u32 lna_constrain: 3;
+	u32 scan_info_update: 1;
+	u32 fw_ver_mismatch: 1;
+	u32 rsvd: 24;
+};
+
+struct btc_bt_mb_devinfo {
+	u8 cmd;
+	u8 len;
+	u8 sub_id;
+	u8 data[5];
+};
+
+struct btc_bt_mb_leaudio_info {
+	u8 cmd;
+	u8 len;
+	u8 exist: 1;
+	u8 bis: 1;
+	u8 cis: 1;
+	u8 lb2rsvd: 5;
+	u8 rssi;
+	u32 hbrsvd;
+};
+
+struct btc_wl_smap {
+	u32 busy: 1;
+	u32 scan: 1;
+	u32 connecting: 1;
+	u32 roaming: 1;
+	u32 dbccing: 1;
+	u32 _4way: 1;
+	u32 rf_off: 1;
+	u32 lps: 2;  /* 1: refer to enum btc_lps_state  */
+	u32 ips: 1;
+	u32 init_ok: 1;
+	u32 traffic_dir : 2;
+	u32 rf_off_pre: 1;
+	u32 lps_pre: 2;
+	u32 lps_exiting: 1; /* Exit-LPS to rf_state = BTC_RFCTRL_WL_ON in 6s */
+};
+
+union btc_wl_state_map {
+	u32 val;
+	struct btc_wl_smap map;
+};
+
+struct btc_wl_info {
+	struct btc_wl_link_info link_info[BTC_WL_MAX_ROLE_NUMBER];
+	struct btc_wl_rfk_info  rfk_info;
+	struct btc_wl_ver_info  ver_info;
+	struct btc_wl_afh_info afh_info;
+	struct btc_wl_role_info role_info;
+	struct btc_wl_scan_info scan_info;
+	struct btc_wl_dbcc_info dbcc_info;
+	struct btc_bool_sta_chg cck_lock;
+	struct btc_rf_para rf_para; /* for record */
+	union btc_wl_state_map status;
+	struct btc_wl_nhm nhm;
+	struct btc_traffic traffic;
+
+	u8 iot_peer;
+	u8 rssi_level;/* the overall-role rssi-level 0~4, "low_rssi->lo_level"*/
+	u8 bssid[6];
+	u8 pta_req_mac;
+	u8 bt_polut_type[2]; /* BT polluted WL-Tx type for phy0/1  */
+	u8 cn_report;
+
+	u8 evm_1ss_rpt;
+	u8 evm_2ss_max_rpt;
+	u8 evm_2ss_min_rpt;
+
+	u8 ch_map[12];
+	u8 ch_map_le[5];
+
+	u8 scbd_change: 1;
+	u8 pta_reg_mac_chg: 1;
+	u8 fw_ver_mismatch: 1;
+	u8 is_5g_hi_channel: 1;
+	u8 busy_to_idle: 1;
+	u8 legacy_mode: 1;
+	u8 he_mode: 1;
+	u8 client_cnt_inc_2g: 1; /* client count increase for 2G AP/P2P */
+
+	u8 coex_mode;
+
+	u8 country_code[2];
+	u8 ap_type;
+
+	u32 scbd;
+	u32 busy_t; /* busy start time */
+};
+
+/* record the last 20 reason/action */
+struct btc_dm_step {
+	char step[BTC_DM_MAXSTEP][BTC_RSN_MAXLEN];
+	u32 cnt;
+};
+
+/* dynamic coex mechanism  */
+struct btc_dm {
+	struct fbtc_slot slot[CXST_MAX];
+	struct fbtc_slot slot_now[CXST_MAX];
+	struct fbtc_tdma tdma;
+	struct fbtc_tdma tdma_now;
+	struct btc_gnt_ctrl gnt_set[HW_PHY_MAX];
+	struct btc_gnt_ctrl gnt_val[HW_PHY_MAX];
+	struct btc_wlact_ctrl wlact_set[2]; /* BT 0/1 wlan_act setup */
+	struct btc_init_info init_info; /* pass to wl_fw if offload */
+	struct btc_rf_trx_para rf_trx_para;
+	struct btc_wl_tx_limit_para wl_tx_limit;
+	struct btc_wl_trx_nss_para wl_trx_nss;
+	struct btc_wl_scc_ctrl wl_scc;
+	struct btc_dm_step dm_step;
+
+	struct btc_fddt_info fddt_info;
+	struct btc_trx_info trx_info;
+
+	u32 error; /* halbtc dm error map, refer to "enum btc_dm_error" */
+
+	char run_reason[BTC_RSN_MAXLEN];
+	char run_action[BTC_ACT_MAXLEN];
+
+	u16 slot_dur[CXST_MAX]; /* for user-define slot duration */
+	u16 bt_slot_flood; /* time threshold for BTC_DCNT_BT_SLOT_FLOOD check */
+
+	u32 set_ant_path;
+	u32 cnt_dm[BTC_DCNT_MAX];
+	u32 cnt_notify[BTC_NTFY_MAX];
+
+	u32 wl_only: 1; /* drv->Fw if offload  */
+	u32 wl_fw_cx_offload: 1; /* BTC_CX_FW_OFFLOAD from FW code  */
+	u32 freerun: 1;
+	u32 fddt_train: 1;
+	u32 wl_ps_ctrl: 2; /* 0: off, 1: force on, 2:forec off */
+	u32 leak_ap: 1;
+	u32 noisy_level: 3;
+	u32 coex_info_map: 8;
+	u32 bt_only: 1; /* drv->Fw if offload  */
+	u32 wl_btg_rx: 2;  /* if wl rx shared with bt */
+	u32 trx_para_level: 8;
+	u32 wl_stb_chg: 1; /* if change s1 WL standby mode table = Rx  */
+	u32 pta_owner: 1; /* 0x73[2] BTC_CTRL_BY_WL, BTC_CTRL_BT*/
+	u32 tdma_instant_excute: 1;
+
+	u32 client_ps_tog1: 8; /* client active/LPS toggle-cnt in ntfy_role */
+	u32 client_ps_tog1_last: 8; /* last client_ps_tog1 in ntfy_role */
+	u32 client_ps_cnt: 8; /* client active/LPS accumulate-cnt in ntfy_sta*/
+	u32 client_ps_tdma_on: 1;
+	u32 rx_err_rpt_en: 1;
+	u32 wl_btg_rx_rb: 2;  /* 0x10980 reg state from reg-moniter read-back */
+	u32 wl_trx_nss_en: 1; /* xTxR switch to 1T1R */
+	u32 rsvd2: 3;
+
+	u8 wl_pre_agc: 2;
+	u8 wl_lna2: 1;
+	u8 freerun_chk: 1;
+	u8 fddt_train_chk: 1;
+	u8 slot_req_more: 1;
+	u8 wl_pre_agc_rb: 2;
+
+	u8 bt_select: 2; /* 0:s0, 1:s1, 2:s0 & s1, refer to enum btc_bt_index */
+	u8 btc_initing: 1;
+	u8 le_init_en: 1;
+	u8 country_code_proc: 1;
+	u8 country_code_retry: 2;
+	u8 cmw_ap: 1;
+
+	u8 vid;
+
+	u8 ntfy_type;
+
+	u8 rssi_offset2;
+
+	u32 e2g_slot_limit;
+	u32 e2g_slot_nulltx_time;
+};
+
+/* the wl/bt/zb cx instance */
+struct btc_cx {
+	struct btc_wl_info wl;
+	struct btc_bt_info bt;
+	struct btc_3rdcx_info other;
+	struct btc_rf_trx_para rf_para;
+	u32 state_map; /* wl/bt combined state map  */
+	u32 cnt_bt[BTC_BCNT_MAX];
+	u32 cnt_wl[BTC_WCNT_MAX];
+};
+
+struct btc_dbg {
+	/* cmd "rb" */
+	bool rb_done;
+	u32 rb_val;
+	u8 lb_val[8];  /* for H2C-C2H/mailbox-loopback test*/
+};
+
+struct btc_tmr {
+	void *btc;
+	u16 id;
+	_os_timer tmr;
+};
+
+struct btc_t {
+	struct rtw_phl_com_t *phl;
+	struct rtw_hal_com_t *hal;
+
+	struct btc_ops *ops;
+	const struct btc_chip *chip;
+
+	struct btc_cx cx;
+	struct btc_dm dm;
+	struct btc_ctrl ctrl;
+	struct btc_module mdinfo;
+	struct btf_fwinfo fwinfo;
+	struct btc_dbg dbg;
+
+#ifdef BTC_AISO_SUPPORT
+	struct btc_bt_psd_dm bt_psd_dm;
+#endif
+
+	/* btc timers */
+	bool tmr_init;
+	bool tmr_stop;
+	struct btc_tmr timer[BTC_TIMER_MAX];
+
+	char mbuf[BTC_MSG_MAXLEN]; /* msg buffer */
+	size_t mlen; /* max msg len */
+
+	u8 policy[BTC_POLICY_MAXLEN]; /* coex policy buffer */
+	u16 policy_len;
+	u16 policy_type;
+
+	u8 hubmsg[BTC_HUBMSG_MAXLEN]; /* phl hub msg */
+	u16 hubmsg_len;
+	u32 hubmsg_cnt;
+	u32 bt_req_len[HW_PHY_MAX]; /* request bt-slot in WL SCC/MCC +BT coex */
+	u8 bt_req_stbc; /*STBC flag*/
+	bool bt_req_en;
+
+	bool wlcli_h2c_cmd;  /* the H2C cmd is triggered by wlcli cmd */
+};
+
+struct btc_chip_ops {
+	void (*set_rfe)(struct btc_t *btc);
+	void (*init_cfg)(struct btc_t *btc);
+	void (*wl_tx_power)(struct btc_t *btc, u32 level);
+	void (*wl_rx_gain)(struct btc_t *btc, u32 level);
+	void (*wl_btg_standby)(struct btc_t *btc, u32 state);
+	void (*wl_req_mac)(struct btc_t *btc, u8 mac_id);
+	void (*get_reg_status)(struct btc_t *btc, u8 type, void *status);
+	u8 (*bt_rssi)(struct btc_t *btc, u8 val);
+};
+
+struct btc_ops {
+	void (*ntfy_power_on)(struct btc_t *btc);
+	void (*ntfy_power_off)(struct btc_t *btc);
+	void (*ntfy_init_coex)(struct btc_t *btc, u8 mode);
+	void (*ntfy_scan_start)(struct btc_t *btc, u8 band_idx, u8 band);
+	void (*ntfy_scan_finish)(struct btc_t *btc, u8 band_idx);
+	void (*ntfy_switch_band)(struct btc_t *btc, u8 band_idx, u8 band);
+	void (*ntfy_specific_packet)(struct btc_t *btc, u8 pkt_type);
+	void (*ntfy_role_info)(struct btc_t *btc, u8 rid,
+			      struct btc_wl_link_info *info,
+			      enum link_state reason);
+	void (*ntfy_radio_state)(struct btc_t *btc, u8 rf_state);
+	void (*ntfy_customerize)(struct btc_t *btc, u8 type, u16 len, u8 *buf);
+	u8  (*ntfy_wl_rfk)(struct btc_t *btc, u8 phy, u8 type, u8 state);
+	void (*ntfy_wl_sta)(struct btc_t *btc, struct rtw_stats *phl_stats,
+			   u8 ntfy_num, struct rtw_phl_stainfo_t *sta[],
+			   u8 reason);
+	void (*ntfy_fwinfo)(struct btc_t *btc, u8 *buf, u32 len, u8 cls,
+			   u8 func);
+	void (*ntfy_timer)(struct btc_t *btc, u16 tmr_id);
+};
+
+struct btc_chip {
+	u32 chip_id;
+	u32 hw;  /* chip HW feature/parameter */
+	u8 btcx_desired; /* bt fw desired coex version */
+	u32 wlcx_desired; /* wl fw desired coex version */
+	u8 scbd; /* scoreboard version, 0: not support*/
+	u8 mailbox; /* mailbox version, 0: not support */
+	u8 pta_mode;
+	u8 pta_direction;
+	u8 afh_guard_ch;
+	u8 *wl_rssi_thres; /* wl rssi thre level */
+	u8 *bt_rssi_thres; /* bt rssi thre level */
+	u8 rssi_tol; /* rssi tolerance */
+	struct btc_chip_ops *ops;
+	u8 mon_reg_num;
+	struct fbtc_mreg *mon_reg;
+	u8 rf_para_ulink_num;
+	struct btc_rf_trx_para *rf_para_ulink;
+	u8 rf_para_dlink_num;
+	struct btc_rf_trx_para *rf_para_dlink;
+};
+
+static void _ntfy_power_on(struct btc_t *btc);
+static void _ntfy_power_off(struct btc_t *btc);
+static void _ntfy_init_coex(struct btc_t *btc, u8 mode);
+static void _ntfy_scan_start(struct btc_t *btc, u8 phy_idx, u8 band);
+static void _ntfy_scan_finish(struct btc_t *btc, u8 pyh_idx);
+static void _ntfy_switch_band(struct btc_t *btc, u8 phy_idx, u8 band);
+static void _ntfy_specific_packet(struct btc_t *btc, u8 pkt_type);
+static void _ntfy_role_info(struct btc_t *btc, u8 rid,
+			    struct btc_wl_link_info *info,
+			    enum link_state reason);
+static void _ntfy_radio_state(struct btc_t *btc, u8 rf_state);
+static void _ntfy_customerize(struct btc_t *btc, u8 type, u16 len, u8 *buf);
+static u8 _ntfy_wl_rfk(struct btc_t *btc, u8 phy_path, u8 type, u8 state);
+static void _ntfy_wl_sta(struct btc_t *btc, struct rtw_stats *phl_stats,
+			u8 ntfy_num, struct rtw_phl_stainfo_t *sta[],
+			u8 reason);
+static void _ntfy_fwinfo(struct btc_t *btc, u8 *buf, u32 len, u8 cls, u8 func);
+static void _ntfy_timer(struct btc_t *btc, u16 tmr_id);
+
+#ifdef BTC_8852A_SUPPORT
+extern const struct btc_chip chip_8852a;
+#endif
+#ifdef BTC_8852B_SUPPORT
+extern const struct btc_chip chip_8852b;
+extern const struct btc_chip chip_8852bp;
+extern const struct btc_chip chip_8851b;
+extern const struct btc_chip chip_8852bt;
+extern const struct btc_chip chip_8852bpt;
+#endif
+#ifdef BTC_8852C_SUPPORT
+extern const struct btc_chip chip_8852c;
+#endif
+#ifdef BTC_8842A_SUPPORT
+extern const struct btc_chip chip_8842a;
+#endif
+#ifdef BTC_8852D_SUPPORT
+extern const struct btc_chip chip_8852d;
+#endif
+
+extern const u32 coex_ver;
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+extern const struct btc_fddt_cell cell_ul_def[5][5];
+extern const struct btc_fddt_cell cell_dl_def[5][5];
+#endif
+
+u8 _send_fw_cmd(struct btc_t *btc, u8 h2c_func, u8 *param, u16 len);
+void _update_bt_scbd(struct btc_t *btc, bool only_update);
+bool hal_btc_init(struct btc_t *btc);
+void hal_btc_deinit(struct btc_t *btc);
+u32 _read_scbd(struct btc_t *btc);
+void _write_scbd(struct btc_t *btc, u32 val, bool state);
+void _run_coex(struct btc_t *btc, const char *reason);
+void _set_init_info(struct btc_t *btc);
+void _update_dm_step(struct btc_t *btc, const char *strin);
+void hal_btc_send_event(struct btc_t *btc, enum phl_band_idx hw_band, u8 *buf,
+			u32 len, u16 ev_id);
+void _set_fddt_cell_by_antiso(struct btc_t *btc);
+bool hal_btc_check_io_ofld(struct btc_t *btc);
+void _btc_io_w (struct btc_t *btc, u32 type, u32 addr, u32 mask,
+		u32 value, bool trigger);
+
+#endif /*_HAL_BTC_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_action.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_action.c
new file mode 100644
index 000000000000..94f70abdf8fb
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_action.c
@@ -0,0 +1,2692 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_BTC_ACTION_C_
+#include "../hal_headers_le.h"
+#include "hal_btc.h"
+#include "halbtc_fw.h"
+#include "halbtc_fwdef.h"
+#include "halbtc_action.h"
+#include "halbtc_def.h"
+
+#ifdef CONFIG_BTCOEX
+
+#define _set_cx_ctrl(btc, val) rtw_hal_mac_set_coex_ctrl(btc->hal, val)
+
+/* tdma policy template */
+const struct fbtc_tdma t_def[] = {
+	{ CXTDMA_OFF,    CXFLC_OFF, CXTPS_OFF, 0, 0, 0, 0, 0}, /*CXTD_OFF*/
+	{ CXTDMA_OFF,    CXFLC_OFF, CXTPS_OFF, 0, 0, 1, 0, 0}, /*CXTD_B2*/
+	{ CXTDMA_OFF,    CXFLC_OFF, CXTPS_OFF, 0, 0, 2, 0, 0}, /*CXTD_OFF_EXT*/
+	{ CXTDMA_FIX,    CXFLC_OFF, CXTPS_OFF, 0, 0, 0, 0, 0}, /* CXTD_FIX */
+	{ CXTDMA_FIX,  CXFLC_NULLP,  CXTPS_ON, 0, 5, 0, 0, 0}, /* CXTD_PFIX */
+	{ CXTDMA_AUTO,   CXFLC_OFF, CXTPS_OFF, 0, 0, 0, 0, 0}, /* CXTD_AUTO */
+	{ CXTDMA_AUTO, CXFLC_NULLP,  CXTPS_ON, 0, 5, 0, 0, 0}, /* CXTD_PAUTO */
+	{CXTDMA_AUTO2,   CXFLC_OFF, CXTPS_OFF, 0, 0, 0, 0, 0}, /* CXTD_AUTO2 */
+	{CXTDMA_AUTO2, CXFLC_NULLP,  CXTPS_ON, 0, 5, 0, 0, 0} /* CXTD_PAUTO2 */
+};
+
+/* slot policy template */
+const struct fbtc_slot s_def[] = {
+	{100, SLOT_MIX, 0x55555555}, /* CXST_OFF */
+	{  5, SLOT_ISO, 0xea5a5a5a}, /* CXST_B2W */
+	{ 70, SLOT_ISO, 0xea5a5a5a}, /* CXST_W1 */
+	{ 15, SLOT_ISO, 0xea5a5a5a}, /* CXST_W2 */
+	{ 15, SLOT_ISO, 0xea5a5a5a}, /* CXST_W2B */
+	{250, SLOT_MIX, 0xe5555555}, /* CXST_B1 */
+	{  7, SLOT_ISO, 0xea5a5a5a}, /* CXST_B2 */
+	{  5, SLOT_MIX, 0xe5555555}, /* CXST_B3 */
+	{ 50, SLOT_MIX, 0xe5555555}, /* CXST_B4 */
+	{ 20, SLOT_ISO, 0xea5a5a5a}, /* CXST_LK */
+	{500, SLOT_MIX, 0x55555555}, /* CXST_BLK */
+	{  5, SLOT_MIX, 0xea5a5a5a}, /* CXST_E2G */
+	{  5, SLOT_ISO, 0xffffffff}, /* CXST_E5G */
+	{  5, SLOT_MIX, 0xea555555}, /* CXST_EBT */
+	{  5, SLOT_MIX, 0x55555555}, /* CXST_ENULL */
+	{250, SLOT_MIX, 0xea5a6a5a}, /* CXST_WLK */
+	{ 50, SLOT_ISO, 0xffffffff}, /* CXST_W1FDD */
+	{ 50, SLOT_ISO, 0xffffdfff}  /* CXST_B1FDD */
+};
+
+const u32 cxtbl[] = {
+	0xffffffff, /* 0 */
+	0xaaaaaaaa, /* 1 */
+	0xe5555555, /* 2 */
+	0xee555555, /* 3 */
+	0xd5555555, /* 4 */
+	0x5a5a5a5a, /* 5 */
+	0xfa5a5a5a, /* 6 */
+	0xda5a5a5a, /* 7 */
+	0xea5a5a5a, /* 8 */
+	0x6a5a5aaa, /* 9 */
+	0x6a5a6a5a, /* 10 */
+	0x6a5a6aaa, /* 11 */
+	0x6afa5afa, /* 12 */
+	0xaaaa5aaa, /* 13 */
+	0xaaffffaa, /* 14 */
+	0xaa5555aa, /* 15 */
+	0xfafafafa, /* 16 */
+	0xffffddff, /* 17 */
+	0xdaffdaff, /* 18 */
+	0xfafa5afa, /* 19 */
+	0xea6a6a6a, /* 20 */
+	0xea55556a, /* 21 */
+	0xaafafafa, /* 22 */
+	0xfafaaafa, /* 23 */
+	0xfafffaff, /* 24 */
+	0xea6a5a5a, /* 25 */
+	0xfaff5aff, /* 26 */
+	0xffffdfff, /* 27 */
+	0xe6555555  /* 28 */
+};
+
+void _set_bt_ignore_wl_act(struct btc_t *btc, u8 enable)
+{
+	u8 buf = 0;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): %s wlan_act\n",
+		  __func__, (enable? "ignore" : "do not ignore"));
+
+	buf = enable;
+	_send_fw_cmd(btc, SET_BT_IGNORE_WLAN_ACT, &buf, 1);
+}
+
+void _set_wl_tx_power(struct btc_t *btc, u32 level)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+
+	if ((btc->dm.rf_trx_para.wl_tx_power == level ||
+	     wl->status.map.dbccing) && !btc->wlcli_h2c_cmd)
+		return;
+
+	btc->dm.rf_trx_para.wl_tx_power = level;
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): level = %d\n",
+		  __func__, level);
+
+#ifdef BTC_WL_TX_PWR_CTRL_OFLD
+	hal_btc_fw_set_drv_info(btc, CXDRVINFO_TXPWR);
+#else
+	btc->chip->ops->wl_tx_power(btc, level);
+#endif
+}
+
+void _set_wl_rx_gain(struct btc_t *btc, u32 level)
+{
+	if ((btc->dm.rf_trx_para.wl_rx_gain == level) && !btc->wlcli_h2c_cmd)
+		return;
+
+	btc->dm.rf_trx_para.wl_rx_gain = level;
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): level = %d\n",
+		  __func__, level);
+
+	btc->chip->ops->wl_rx_gain(btc, level);
+}
+
+void _set_bt_tx_power(struct btc_t *btc, u32 level)
+{
+	u8 buf = 0;
+
+	if (btc->dm.rf_trx_para.bt_tx_power == level && !btc->wlcli_h2c_cmd)
+		return;
+
+	buf = level & bMASKB0;
+
+	if (_send_fw_cmd(btc, SET_BT_TX_PWR, &buf, 1)) {
+		btc->dm.rf_trx_para.bt_tx_power = level;
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			 "[BTC], %s(): level = %d\n",  __func__, level);
+	}
+}
+
+void _set_bt_rx_gain(struct btc_t *btc, u32 level)
+{
+	struct btc_bt_info *bt = &btc->cx.bt;
+	u8 buf = 0;
+
+	if ((btc->dm.rf_trx_para.bt_rx_gain == level ||
+	     level > BTC_BT_RX_NORMAL_LVL) &&
+	     (!btc->chip->scbd || bt->lna_constrain == level))
+		return;
+
+	buf = level & bMASKB0;
+
+	if (!_send_fw_cmd(btc, SET_BT_LNA_CONSTRAIN, &buf, 1)) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			 "[BTC], %s(): setup fail!!\n", __func__);
+		return;
+	}
+
+	btc->dm.rf_trx_para.bt_rx_gain = level;
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): level = %d\n",
+		  __func__, level);
+
+	_os_delay_us(btc->hal, BTC_SCBD_REWRITE_DELAY * 2);
+
+	if (buf == BTC_BT_RX_NORMAL_LVL)
+		_write_scbd(btc, BTC_WSCB_RXGAIN, false);
+	else
+		_write_scbd(btc, BTC_WSCB_RXGAIN, true);
+}
+
+static void _set_rf_trx_para(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_rf_trx_para para;
+	u32 level_id = 0;
+	u32 wl_stb_chg = 0;
+
+	/* decide trx_para_level */
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) {
+		/* fix LNA2 + TIA gain not change by GNT_BT */
+		if ((btc->dm.wl_btg_rx &&
+		     b->profile_cnt.now != 0) || dm->bt_only == 1)
+			dm->trx_para_level = 1; /* for better BT ACI issue */
+		else
+			dm->trx_para_level = 0;
+	} else { /* non-shared antenna  */
+		dm->trx_para_level = 5;
+		if (wl_rinfo->link_mode == BTC_WLINK_2G_STA &&
+		    b->leaudio_desc.bis_cnt)
+			dm->trx_para_level = 8;
+		/* modify trx_para if WK 2.4G-STA-DL + bt link */
+		if (dm->freerun && b->profile_cnt.now != 0 &&
+		    wl_rinfo->link_mode == BTC_WLINK_2G_STA &&
+		    wl_smap->traffic_dir & BIT(TRAFFIC_UL)) { /* uplink */
+			if (wl->rssi_level == 4 && bt->rssi_level > 2)
+				dm->trx_para_level = 6;
+			else if (wl->rssi_level == 3 && bt->rssi_level > 3)
+				dm->trx_para_level = 7;
+		}
+	}
+
+	level_id = dm->trx_para_level;
+
+	if (level_id >= btc->chip->rf_para_dlink_num ||
+	    level_id >= btc->chip->rf_para_ulink_num) {
+	    PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+	    	      "[BTC], %s(): invalid level_id:%d\n", __func__, level_id);
+	    return;
+	}
+
+	if (wl_smap->traffic_dir & BIT(TRAFFIC_UL))
+		para = btc->chip->rf_para_ulink[level_id];
+	else
+		para = btc->chip->rf_para_dlink[level_id];
+
+	/* Do not change if manual setup ever */
+	if (wl->rf_para.tx_pwr_freerun != BTC_WL_DEF_TX_PWR)
+		para.wl_tx_power = wl->rf_para.tx_pwr_freerun;
+
+	if (wl->rf_para.rx_gain_freerun != 0)
+		para.wl_rx_gain = wl->rf_para.rx_gain_freerun;
+
+	if (bt->rf_para.tx_pwr_freerun != 0)
+		para.bt_tx_power = bt->rf_para.tx_pwr_freerun;
+
+	if (bt->rf_para.rx_gain_freerun != BTC_BT_RX_NORMAL_LVL)
+		para.bt_rx_gain = bt->rf_para.rx_gain_freerun;
+
+	if (dm->fddt_train) {
+		_set_wl_rx_gain(btc, 1);
+		_write_scbd(btc, BTC_WSCB_RXGAIN, true);
+	} else {
+		_set_wl_tx_power(btc, para.wl_tx_power);
+		_set_wl_rx_gain(btc, para.wl_rx_gain);
+		_set_bt_tx_power(btc, para.bt_tx_power);
+		_set_bt_rx_gain(btc, para.bt_rx_gain);
+	}
+
+	if (!bt->enable.now || dm->wl_only || wl_smap->rf_off ||
+	    wl_smap->lps == BTC_LPS_RF_OFF ||
+	    wl_rinfo->link_mode == BTC_WLINK_5G ||
+	    wl_rinfo->link_mode == BTC_WLINK_NOLINK ||
+	    (wl_rinfo->dbcc_en && wl_rinfo->dbcc_2g_phy != HW_PHY_1))
+	    wl_stb_chg = 0;
+	else
+	    wl_stb_chg = 1;
+
+	if (wl_stb_chg != dm->wl_stb_chg) {
+		dm->wl_stb_chg = wl_stb_chg;
+		btc->chip->ops->wl_btg_standby(btc, dm->wl_stb_chg);
+	}
+}
+
+static void _set_wl_ch_map(struct btc_t *btc)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u16 ch_start = 0, ch_end = 0, i;
+	u8 b_group, b_pos; /* byte-group, bit-position */
+
+	hal_mem_set(h, wl->ch_map, 0x0, sizeof(wl->ch_map));
+	hal_mem_set(h, wl->ch_map_le, 0x0, sizeof(wl->ch_map_le));
+
+	if (!wl->afh_info.en || wl->afh_info.ch > 13)
+		return;
+
+	ch_start = wl->afh_info.ch * 5 + 2407 - wl->afh_info.bw/2;
+	ch_end = ch_start + wl->afh_info.bw;
+
+	for (i = ch_start; i < ch_end; i++) {
+		if (i < 2402 || i > 2480)
+			continue;
+		b_group = (i - 2402) / 8;
+		b_pos = 7 - ((i - 2402) % 8);
+		wl->ch_map[b_group] |= BIT(b_pos);
+
+		if (i % 2)  /* BLE ch = even-freq (ex:2402,2404) */
+			continue;
+
+		b_group = (i - 2402) / 16;
+		b_pos = 7 - (((i - 2402)/2) % 8);
+		wl->ch_map_le[b_group] |= BIT(b_pos);
+	}
+}
+
+static void _set_bt_afh_info(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_afh_info *wl_afh = &wl->afh_info;
+	struct btc_wl_active_role *act_role = NULL;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	/*struct btc_module *module = &btc->mdinfo;*/
+	u8 en = 0, i, ch = 0, bw = CHANNEL_WIDTH_MAX;
+	u8 buf[3] = {0};
+
+	if (btc->ctrl.manual || wl->status.map.scan)
+		return;
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++) {
+		if (wl->status.map.rf_off || bt->whql_test ||
+		    wl_rinfo->link_mode == BTC_WLINK_NOLINK ||
+		    wl_rinfo->link_mode == BTC_WLINK_5G)
+			break;
+
+		act_role = &wl_rinfo->active_role[i];
+
+		/* not care no-connected/non-2G-band role */
+		if (!act_role->connected ||
+		    act_role->band != BAND_ON_24G)
+			continue;
+
+		en = 1;
+		ch = act_role->ch;
+		bw = act_role->bw;
+
+		if (wl_rinfo->link_mode == BTC_WLINK_2G_MCC &&
+		    (act_role->role == PHL_RTYPE_AP ||
+		     act_role->role == PHL_RTYPE_P2P_GO ||
+		     act_role->role == PHL_RTYPE_P2P_GC)) {
+		/* for 2.4G MCC, take role = ap/go/gc*/
+			break;
+		} else if (wl_rinfo->link_mode != BTC_WLINK_2G_SCC ||
+			act_role->bw == CHANNEL_WIDTH_40) {
+			/* for 2.4G scc, take bw = 40M  */
+			break;
+		}
+	}
+
+	if (!en || ch > 14 || ch == 0)
+		bw = CHANNEL_WIDTH_MAX;
+
+	/* default AFH channel sapn = center-ch +- 6MHz  */
+	switch (bw) {
+	case CHANNEL_WIDTH_20:
+		if (!wl_smap->busy && btc->mdinfo.ant.num == 3)
+			bw = 10;
+		else if (btc->dm.freerun || btc->dm.fddt_train)
+			bw = 48;
+		else
+			bw = 20 + btc->chip->afh_guard_ch * 2;
+		break;
+	case CHANNEL_WIDTH_40:
+		if (!wl_smap->busy && btc->mdinfo.ant.num == 3)
+			bw = 10;
+		else if (btc->dm.freerun || btc->dm.fddt_train)
+			bw = 40 + btc->chip->afh_guard_ch * 2;
+		else
+			bw = 40;
+		break;
+	case CHANNEL_WIDTH_5:
+		bw = 5 + btc->chip->afh_guard_ch * 2;
+		break;
+	case CHANNEL_WIDTH_10:
+		bw = 10 + btc->chip->afh_guard_ch * 2;
+		break;
+	default:
+	case CHANNEL_WIDTH_MAX:
+		en = false; /* turn off AFH info if invalid BW */
+		bw = 0;
+		ch = 0;
+		break;
+	}
+
+	if (wl_afh->en == en &&
+	    wl_afh->ch == ch &&
+	    wl_afh->bw == bw &&
+	    b->profile_cnt.last == b->profile_cnt.now)
+		return;
+
+	buf[0] = en;
+	buf[1] = ch;
+	buf[2] = bw;
+
+	if (_send_fw_cmd(btc, SET_BT_WL_CH_INFO, buf, 3)) {
+		wl_afh->en = buf[0];
+		wl_afh->ch = buf[1];
+		wl_afh->bw = buf[2];
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s(): en=%d, ch=%d, bw=%d\n",
+			  __func__, en, ch, bw);
+
+		_set_wl_ch_map(btc);
+		btc->cx.cnt_wl[BTC_WCNT_CH_UPDATE]++;
+	}
+}
+
+static void _set_halbb_preagc_ctrl(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_chip_ops *ops = btc->chip->ops;
+	struct btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
+	u8 is_preagc = 0, val = 0;
+
+	if (btc->ctrl.manual)
+		return;
+
+	/* notify halbb ignore GNT_BT or not for WL BB Rx-AGC control */
+	if (wl_rinfo->link_mode == BTC_WLINK_25G_MCC)
+		is_preagc = 2; /* bb switch in WL FW by itself */
+	else if (!(bt->run_patch_code && bt->enable.now))
+		is_preagc = 0;
+	else if (wl_rinfo->link_mode == BTC_WLINK_5G) /* always 0 if 5G */
+		is_preagc = 0;
+	else if (wl_rinfo->link_mode == BTC_WLINK_NOLINK ||
+		 btc->cx.bt.link_info.profile_cnt.now == 0)
+		is_preagc = 0;
+	else if (dm->tdma_now.type != CXTDMA_OFF &&
+		 !bt_linfo->hfp_desc.exist &&
+		 !bt_linfo->hid_desc.exist &&
+		 dm->fddt_train == BTC_FDDT_DISABLE)
+		is_preagc = 0;
+	else if (wl_rinfo->dbcc_en && wl_rinfo->dbcc_2g_phy != HW_PHY_1)
+		is_preagc = 0;
+	else if (btc->mdinfo.ant.type == BTC_ANT_SHARED)
+		is_preagc = 0;
+	else
+		is_preagc = 1;
+
+	/* check pre-agc only if not-sync between wl_pre_agc_rb and wl_pre_agc*/
+	if (dm->wl_pre_agc_rb != dm->wl_pre_agc &&
+	    dm->wl_pre_agc_rb != BTC_BB_PRE_AGC_NOTFOUND &&
+	    (ops && ops->get_reg_status)) {
+		ops->get_reg_status(btc, BTC_CSTATUS_BB_PRE_AGC, (void*)&val);
+		dm->wl_pre_agc_rb = val;
+	}
+
+	if ((wl->coex_mode == BTC_MODE_NORMAL &&
+	    (dm->ntfy_type == BTC_NTFY_INIT_COEX ||
+	     dm->ntfy_type == BTC_NTFY_SWITCH_BAND ||
+	    dm->wl_pre_agc_rb != dm->wl_pre_agc)) ||
+	    is_preagc != dm->wl_pre_agc) {
+
+		dm->wl_pre_agc = is_preagc;
+
+		if (is_preagc > 1)
+			return;
+		rtw_hal_bb_ctrl_btc_preagc(btc->hal, dm->wl_pre_agc);
+	}
+}
+
+static void _set_halbb_btg_ctrl(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_chip_ops *ops = btc->chip->ops;
+	u32 is_btg = 0, val = 0;
+
+	/* rtw_hal_bb_ctrl_btg() control bb internal GNT_MUX:
+	 * if btc->dm.wl_btg_rx = 0 (gnt_wl always = 1, gnt_bt always = 0)
+	 * Lte_rx:    0x10980[17]=1, 0x10980[29]=0
+	 * Gnt_wl:    0x10980[18]=1, 0x10980[28]=1
+	 * Gnt_bt_tx: 0x10980[19]=1, 0x10980[27]=0
+	 * Gnt_bt:    0x10980[20]=1, 0x10980[26]=0   0x101e2250
+	 *
+	 * if btc->dm.wl_btg_rx = 1 (gnt from MAC)
+	 * Lte_rx:    0x10980[17]=0, 0x10980[29]=0
+	 * Gnt_wl:    0x10980[18]=0, 0x10980[28]=1
+	 * Gnt_bt_tx: 0x10980[19]=0, 0x10980[27]=0
+	 * Gnt_bt:    0x10980[20]=0, 0x10980[26]=0   0x10002250
+	 */
+
+	if (btc->ctrl.manual)
+		return;
+
+	/* notify halbb ignore GNT_BT or not for WL BB Rx-AGC control */
+	if (wl_rinfo->link_mode == BTC_WLINK_25G_MCC)
+		is_btg = 2; /* bb call ctrl_btg() in WL FW by slot */
+	else if (!(bt->run_patch_code && bt->enable.now))
+		is_btg = 0;
+	else if (wl_rinfo->link_mode == BTC_WLINK_5G) /* always 0 if 5G */
+		is_btg = 0;
+	else if (dm->freerun)
+		is_btg = 0;
+	else if (wl_rinfo->dbcc_en && wl_rinfo->dbcc_2g_phy != HW_PHY_1)
+		is_btg = 0;
+	else if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED)
+		is_btg = 0;
+	else
+		is_btg = 1;
+
+	/* check 0x10980 only if not-sync between wl_btg_rx_rb and wl_btg_rx */
+	if (dm->wl_btg_rx_rb != dm->wl_btg_rx &&
+	    dm->wl_btg_rx_rb != BTC_BB_GNT_NOTFOUND &&
+	    (ops && ops->get_reg_status)) {
+		ops->get_reg_status(btc, BTC_CSTATUS_BB_GNT_MUX, (void*)&val);
+		dm->wl_btg_rx_rb = val;
+	}
+
+	/* always run --> _ntfy_init_coex, _ntfy_switch_band, reg-RB no match */
+	if ((wl->coex_mode == BTC_MODE_NORMAL &&
+	    (dm->ntfy_type == BTC_NTFY_INIT_COEX ||
+	     dm->ntfy_type == BTC_NTFY_SWITCH_BAND ||
+	     dm->wl_btg_rx_rb != dm->wl_btg_rx)) ||
+	    is_btg != dm->wl_btg_rx) {
+
+		dm->wl_btg_rx = is_btg;
+
+		/* skip setup if btg_ctrl set by wl fw */
+		if (is_btg > 1)
+			return;
+
+		rtw_hal_bb_ctrl_btg(btc->hal, (bool)(!!is_btg));
+	}
+}
+
+static void _set_phl_bt_slot_req(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_link_info *b = &btc->cx.bt.link_info;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_bt_a2dp_desc a2dp = b->a2dp_desc;
+	struct btc_dm *dm = &btc->dm;
+	u8 len = 0;
+
+	/*  don't change bt slot req state during RFK for p2p/mcc case*/
+	if (dm->ntfy_type == BTC_NTFY_WL_RFK)
+		return;
+
+	/* enable bt-slot req if ext-slot-control  */
+	if (btc->dm.tdma_now.type == CXTDMA_OFF &&
+	    btc->dm.tdma_now.ext_ctrl == CXECTL_EXT) {
+		if (wl->status.val & BTC_WL_LINK_SCAN)
+			btc->bt_req_en = false;
+		else
+			btc->bt_req_en = true;
+	} else {
+		btc->bt_req_en = false;
+	}
+
+	if (!btc->bt_req_en) {
+		len = 0;
+	} else if (a2dp.exist || a2dp.active ||
+		   (b->profile_cnt.now == 1 && b->hid_desc.pair_cnt == 1 &&
+		    b->multi_link.now)) {
+		/*a2dp, hid + acl-idle  */
+		if (b->profile_cnt.now >= 2 || a2dp.vendor_id == 0x4c ||
+		    dm->slot_req_more == 1)
+			len = BTC_BSLOT_A2DP_HID;
+		else if (wl_rinfo->p2p_2g == 1)
+			len = BTC_BSLOT_A2DP_2;
+		else
+			len = BTC_BSLOT_A2DP;
+	} else if (b->pan_desc.exist || b->status.map.inq_pag) {
+		len = BTC_BSLOT_INQ;
+	} else {
+		len = BTC_BSLOT_IDLE;
+	}
+
+	if (len == btc->bt_req_len[wl->pta_req_mac] && !wl->pta_reg_mac_chg)
+		return;
+
+	btc->bt_req_len[wl->pta_req_mac] = len;
+	if (wl->pta_req_mac == HW_PHY_0)
+		btc->bt_req_len[HW_PHY_1] = 0;
+	else
+		btc->bt_req_len[HW_PHY_0] = 0;
+
+	_send_phl_evnt(btc, (u8*)&btc->bt_req_len[wl->pta_req_mac], 4,
+		       BTC_HMSG_SET_BT_REQ_SLOT);
+
+	PHL_INFO("[BTC], %s(): bt_req_len= PHY0:%d, PHY1:%d\n", __func__,
+		 btc->bt_req_len[HW_PHY_0], btc->bt_req_len[HW_PHY_1]);
+}
+
+static void _set_phl_stbc_req(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u8 req_stbc = 1;
+
+	if (btc->dm.freerun || btc->dm.wl_only || wl->role_info.dbcc_en)
+		req_stbc = 0;
+
+	if (req_stbc == btc->bt_req_stbc && !wl->pta_reg_mac_chg)
+		return;
+
+	btc->bt_req_stbc = req_stbc;
+	_send_phl_evnt(btc, (u8*)&btc->bt_req_stbc, 1,
+		       BTC_HMSG_SET_BT_REQ_STBC);
+
+	PHL_INFO("[BTC], %s(): bt_req_stbc=%d\n", __func__, btc->bt_req_stbc);
+}
+
+static void _set_bt_rx_agc(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	bool bt_hi_lna_rx = false;
+
+	if (wl_rinfo->dbcc_en && wl_rinfo->dbcc_2g_phy != HW_PHY_1)
+		bt_hi_lna_rx = false;
+	else if (wl_rinfo->link_mode != BTC_WLINK_NOLINK &&
+		 btc->dm.wl_btg_rx && btc->mdinfo.ant.type == BTC_ANT_SHARED)
+		bt_hi_lna_rx = true;
+
+	if (bt_hi_lna_rx == bt->hi_lna_rx)
+		return;
+
+	/* true: bt use Hi-LNA rx gain table (f/e/3/2) in -3x~-9xdBm for co-rx
+	 * false: bt use original rx gain table (f/b/7/3/2)
+	 */
+
+	_write_scbd(btc, BTC_WSCB_BT_HILNA, bt_hi_lna_rx);
+}
+
+static void _set_halmac_tx_limit(struct btc_t *btc)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_dm *dm = &btc->dm;
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_bt_hfp_desc *hfp = &b->hfp_desc;
+	struct btc_bt_hid_desc *hid = &b->hid_desc;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_link_info *plink = NULL;
+	u8 tx_1ss_limit = 0;
+	u8 mode = wl_rinfo->link_mode, i;
+	u8 enable = 0, id = 0, resume = 0;
+	u32 tx_time = BTC_MAX_TX_TIME_DEF;
+
+	if (btc->ctrl.manual || wl->status.map.dbccing)
+		return;
+
+	if (btc->dm.freerun || btc->ctrl.igno_bt || b->profile_cnt.now == 0 ||
+	    mode == BTC_WLINK_5G || mode == BTC_WLINK_NOLINK ||
+	    dm->wl_trx_nss.tx_limit) {
+		enable = 0;
+		tx_time = BTC_MAX_TX_TIME_DEF;
+		tx_1ss_limit = 0;
+	} else if (hfp->exist || hid->exist) {
+		enable = 1;
+		tx_time = BTC_MAX_TX_TIME_L3;
+	}
+
+	if (dm->wl_tx_limit.en == enable &&
+	    dm->wl_tx_limit.tx_time == tx_time &&
+	    !wl_rinfo->link_mode_chg)
+		return;
+
+	resume = !enable;
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++) {
+		plink = &wl->link_info[i];
+		id = (u8)plink->mac_id;
+
+		if (!plink->connected)
+			continue;
+
+		/* Skip non-2G-PHY setup for dbcc case */
+		if (wl_rinfo->dbcc_en &&
+		    (plink->phy != (u8)wl_rinfo->dbcc_2g_phy))
+			continue;
+
+		/* backup the original tx-limit-setup if tx-limit off */
+		if (!dm->wl_tx_limit.en)
+			rtw_hal_mac_get_tx_time(h, id, &plink->tx_time);
+
+		/* restore the original tx-limit-setup if no tx-limit */
+		if (resume)
+			tx_time = plink->tx_time;
+
+		rtw_hal_mac_set_tx_time(h, 1, resume, id, tx_time);
+	}
+
+
+	dm->wl_tx_limit.en = enable;
+	dm->wl_tx_limit.tx_time = tx_time;
+}
+
+static void _set_trx_nss(struct btc_t *btc)
+{
+	struct btc_wl_trx_nss_para *trx_nss = &btc->dm.wl_trx_nss;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_bt_a2dp_desc *a2dp = &b->a2dp_desc;
+	struct btc_bt_leaudio_desc *le_audio = &b->leaudio_desc;
+	struct btc_chip_ops *ops = btc->chip->ops;
+	u8 tx_en = 0, tx_re = 0, rx_en = 0, rx_re = 0;
+	u8 tx_limit = 0, rx_limit = 0;
+	u8 tx_ss, rx_ss, tx_path, rx_path;
+	u8 tx_ss_ori, rx_ss_ori, tx_path_ori, rx_path_ori;
+
+	/* tx/rx_limit = 1 ----> 1ss
+	 * tx/rx_limit = 0 ----> default
+	 */
+
+	if (btc->ctrl.manual ||
+	    wl->status.map.dbccing ||
+	    !dm->wl_trx_nss_en)
+		return;
+
+	if (wl_rinfo->link_mode == BTC_WLINK_NOLINK ||
+	    wl_rinfo->link_mode == BTC_WLINK_5G ||
+	    wl_rinfo->dbcc_en) {
+		tx_limit = 0;
+		rx_limit = 0;
+	} else if (le_audio->bis_cnt) {
+		tx_limit = 1;
+		rx_limit = 1;
+	} else if (le_audio->cis_cnt &&
+	           (btc->mdinfo.wa_type & BTC_WA_CO_RX)) {
+		if (wl->he_mode) {
+			tx_limit = 1;
+			rx_limit = 1;
+		}
+	} else if (a2dp->exist && a2dp->sink) {
+		tx_limit = 1;
+		rx_limit = 1;
+	} else if (wl_rinfo->link_mode == BTC_WLINK_2G_AP ||
+		   wl_rinfo->link_mode == BTC_WLINK_2G_GO) {
+		if (b->profile_cnt.now) {
+			tx_limit = 1;
+			rx_limit = 1;
+		}
+	}
+
+	if (trx_nss->tx_limit == tx_limit &&
+	    trx_nss->rx_limit == rx_limit &&
+	    !wl->client_cnt_inc_2g)
+		return;
+
+	tx_ss_ori = (btc->mdinfo.ant.stream_cnt & 0xf0) >> 4;
+	rx_ss_ori = btc->mdinfo.ant.stream_cnt & 0xf;
+	tx_path_ori = (btc->mdinfo.ant.path_pos & 0xf0) >> 4;
+	rx_path_ori = btc->mdinfo.ant.path_pos & 0xf;
+
+	if (tx_ss_ori == 2 && tx_limit) {
+		tx_en = 1;
+		tx_re = 0;
+		tx_ss = 1;
+		tx_path = RF_PATH_A;
+	} else {
+		tx_en = 0;
+		tx_re = 1;
+		tx_ss = tx_ss_ori;
+		tx_path = tx_path_ori;
+	}
+
+	if (rx_ss_ori == 2 && rx_limit) {
+		rx_en = 1;
+		rx_re = 0;
+		rx_ss = 1;
+		rx_path = RF_PATH_A;
+	} else {
+		rx_en = 0;
+		rx_re = 1;
+		rx_ss = rx_ss_ori;
+		rx_path = rx_path_ori;
+	}
+
+	rtw_hal_btc_cfg_1ss(btc->hal, btc->phl, BAND_ON_24G,
+			    tx_en, rx_en, tx_re, rx_re);
+	rtw_hal_btc_cfg_trx_path(btc->hal, tx_path, tx_ss, rx_path, rx_ss);
+
+	trx_nss->tx_limit = tx_limit;
+	trx_nss->rx_limit = rx_limit;
+	trx_nss->tx_ss = tx_ss;
+	trx_nss->rx_ss = rx_ss;
+	trx_nss->tx_path = tx_path;
+	trx_nss->rx_path = rx_path;
+
+	/* if enable MIMO-PS (to 1T1R), change ant shared-type */
+	if (dm->wl_trx_nss.tx_limit &&
+	    dm->wl_trx_nss.rx_limit)
+		btc->mdinfo.ant.type = BTC_ANT_DEDICATED;
+	else
+		btc->mdinfo.ant.type = BTC_ANT_SHARED;
+
+	if (ops && ops->init_cfg)
+		ops->init_cfg(btc);
+}
+
+void _set_fddt_ctrl(struct btc_t *btc, bool force_exec)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_fddt_train_info *train = &dm->fddt_info.train;
+	struct btc_fddt_train_info *train_now = &dm->fddt_info.train_now;
+
+	if (!force_exec && !_fddt_cmp(train, train_now)) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s(): fddt control no change!\n", __func__);
+		return;
+	}
+
+	hal_btc_fw_set_drv_info(btc, CXDRVINFO_FDDT);
+
+	_fddt_cpy(train_now, train);
+}
+
+static void _set_fddt_status(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	bool is_stop = false;
+
+	if (dm->fddt_train == BTC_FDDT_DISABLE) {
+		if ((dm->fddt_info.nrsn_map & BIT(BTC_NFRSN_FORCE_STOP)) ||
+		     dm->fddt_info.result == BTC_FDDT_RESULT_FAIL)
+		     is_stop = true;
+	}
+
+	/* fddt_train state change */
+	switch (dm->fddt_train) {
+	case BTC_FDDT_DISABLE: /* fddt_train enable->disable */
+		if (is_stop)
+			dm->fddt_info.state = BTC_FDDT_STATE_STOP;
+		else
+			dm->fddt_info.state = BTC_FDDT_STATE_PENDING;
+		break;
+	case BTC_FDDT_ENABLE:  /* fddt_train disable->enable */
+		if (dm->fddt_info.state == BTC_FDDT_STATE_STOP)
+			dm->cnt_dm[BTC_DCNT_FDDT_TRIG]++;
+
+		if (dm->fddt_info.type == BTC_FDDT_TYPE_AUTO)
+			dm->fddt_info.state = BTC_FDDT_STATE_RUN;
+		else
+			dm->fddt_info.state = BTC_FDDT_STATE_DEBUG;
+		break;
+	}
+}
+
+static void _set_policy(struct btc_t *btc, u16 policy_type, const char* action)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct fbtc_tdma *t = &dm->tdma;
+	struct fbtc_slot *s = dm->slot;
+	struct btc_wl_role_info *wl_rinfo = &btc->cx.wl.role_info;
+	struct btc_bt_link_info *b = &btc->cx.bt.link_info;
+	struct btc_bt_hid_desc *hid = &b->hid_desc;
+	struct btc_bt_hfp_desc *hfp = &b->hfp_desc;
+	u8 type, null_role = 0;
+	u16 dur_1 = 0, dur_2 = 0, offb_b2_dur = 10;
+	u32 tbl_w1, tbl_b1, tbl_b4;
+	bool mode = NM_EXEC;
+
+	_slot_cpy(&s[CXST_B2], &s_def[CXST_B2]);
+	type = (u8)((policy_type & bMASKB1) >> 8);
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) {
+		if (wl->status.map._4way)
+			tbl_w1 = cxtbl[1];
+		else if (hid->exist && hid->type == BTC_HID_218)
+			tbl_w1 = cxtbl[7]; /* Ack/BA no break bt Hi-Pri-rx */
+		else
+			tbl_w1 = cxtbl[8];
+
+		if (dm->leak_ap &&
+		    (type == BTC_CXP_PFIX || type == BTC_CXP_PAUTO2)) {
+			tbl_b1 = cxtbl[3];
+			tbl_b4 = cxtbl[3];
+		} else if (hid->exist && hid->type == BTC_HID_218) {
+			tbl_b1 = cxtbl[4]; /* Ack/BA no break bt Hi-Pri-rx */
+			tbl_b4 = cxtbl[4];
+		} else {
+			tbl_b1 = cxtbl[2];
+			tbl_b4 = cxtbl[2];
+		}
+	} else {
+		if (wl->legacy_mode)
+			tbl_w1 = cxtbl[8];
+		else if ((wl->status.map.traffic_dir & BIT(TRAFFIC_UL)) &&
+			 hid->exist)
+			tbl_w1 = cxtbl[19];
+		else
+			tbl_w1 = cxtbl[16];
+		tbl_b1 = cxtbl[17];
+		tbl_b4 = cxtbl[17];
+	}
+
+	switch(type) {
+	case BTC_CXP_USERDEF0:
+		mode = FC_EXEC;
+		_tdma_cpy(t, &t_def[CXTD_OFF]);
+		_slot_cpy(&s[CXST_OFF], &s_def[CXST_OFF]);
+		_slot_set_tbl(CXST_OFF, cxtbl[2]);
+		break;
+	case BTC_CXP_OFF: /* TDMA off */
+		_write_scbd(btc, BTC_WSCB_TDMA, false);
+		_tdma_cpy(t, &t_def[CXTD_OFF]);
+		_slot_cpy(&s[CXST_OFF], &s_def[CXST_OFF]);
+
+		switch (policy_type) {
+		case BTC_CXP_OFF_BT:
+			_slot_set_tbl(CXST_OFF, cxtbl[2]);
+			break;
+		case BTC_CXP_OFF_BT2:
+			_slot_set_tbl(CXST_OFF, cxtbl[28]);
+			break;
+		case BTC_CXP_OFF_WL:
+			_slot_set_tbl(CXST_OFF, cxtbl[1]);
+			break;
+		case BTC_CXP_OFF_WL2:
+			_slot_set_tbl(CXST_OFF, cxtbl[1]);
+			_slot_set_type(CXST_OFF, SLOT_ISO);
+			break;
+		case BTC_CXP_OFF_EQ0:
+			_slot_set_tbl(CXST_OFF, cxtbl[0]);
+			_slot_set_type(CXST_OFF, SLOT_ISO);
+			break;
+		case BTC_CXP_OFF_EQ1:
+			_slot_set_tbl(CXST_OFF, cxtbl[16]);
+			break;
+		case BTC_CXP_OFF_EQ2:
+			_slot_set_tbl(CXST_OFF, cxtbl[0]);
+			break;
+		case BTC_CXP_OFF_EQ3:
+			_slot_set_tbl(CXST_OFF, cxtbl[24]);
+			break;
+		case BTC_CXP_OFF_EQ4:
+			_slot_set_tbl(CXST_OFF, cxtbl[26]);
+			break;
+		case BTC_CXP_OFF_EQ5:
+			_slot_set_tbl(CXST_OFF, cxtbl[27]);
+			break;
+		case BTC_CXP_OFF_BWB0:
+			_slot_set_tbl(CXST_OFF, cxtbl[5]);
+			break;
+		case BTC_CXP_OFF_BWB1:
+			_slot_set_tbl(CXST_OFF, cxtbl[8]);
+			break;
+		case BTC_CXP_OFF_BWB2:
+			_slot_set_tbl(CXST_OFF, cxtbl[7]);
+			break;
+		case BTC_CXP_OFF_BWB3:
+			_slot_set_tbl(CXST_OFF, cxtbl[6]);
+			break;
+		}
+		break;
+	case BTC_CXP_OFFB: /* TDMA off + beacon protect */
+		_write_scbd(btc, BTC_WSCB_TDMA, false);
+		_tdma_cpy(t, &t_def[CXTD_OFF_B2]);
+		_slot_cpy(&s[CXST_OFF], &s_def[CXST_OFF]);
+		_slot_set(CXST_B2, offb_b2_dur, cxtbl[1], SLOT_ISO);
+
+		switch (policy_type) {
+		case BTC_CXP_OFFB_BWB0:
+			_slot_set_tbl(CXST_OFF, cxtbl[5]);
+			break;
+		case BTC_CXP_OFFB_BWB1:
+			_slot_set_tbl(CXST_OFF, cxtbl[8]);
+			break;
+		case BTC_CXP_OFFB_BWB2:
+			_slot_set_tbl(CXST_OFF, cxtbl[7]);
+			break;
+		case BTC_CXP_OFFB_BWB3:
+			_slot_set_tbl(CXST_OFF, cxtbl[6]);
+			break;
+		}
+		break;
+	case BTC_CXP_OFFE: /* TDMA off + beacon protect + Ext_control */
+		_write_scbd(btc, BTC_WSCB_TDMA, true);
+		_tdma_cpy(t, &t_def[CXTD_OFF_EXT]);
+
+		/* To avoid wl-s0 tx break by hid/hfp tx */
+		if (hid->exist || hfp->exist) {
+			if (dm->wl_trx_nss.tx_limit)
+				tbl_w1 = cxtbl[16];
+			else if (btc->mdinfo.ant.type == BTC_ANT_SHARED)
+				tbl_w1 = cxtbl[23];
+			else
+				tbl_w1 = cxtbl[16];
+		}
+		if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC &&
+		    dm->wl_scc.e2g_null) {
+			_tdma_set_rxflctrl(1);
+			_tdma_set_txflctrl(1);
+			/* Set Null-Tx time-tick by E2G duration */
+			dur_1 = (u16)dm->e2g_slot_nulltx_time;
+		}
+
+		/* To avoid A2DP pop by the long E2G duration isolated
+		 * Set E2G max-limit by EBT duration
+		 */
+		dur_2 = (u16)dm->e2g_slot_limit;
+
+		switch (policy_type) {
+		case BTC_CXP_OFFE_2GBWISOB: /* for normal-case */
+			_slot_set(CXST_E2G, dur_1, tbl_w1, SLOT_ISO);
+			_slot_cpy(&s[CXST_EBT], &s_def[CXST_EBT]);
+			_slot_set_dur(CXST_EBT, dur_2);
+			break;
+		case BTC_CXP_OFFE_2GISOB: /* for bt no-link */
+			_slot_set(CXST_E2G, dur_1, cxtbl[1], SLOT_ISO);
+			_slot_cpy(&s[CXST_EBT], &s_def[CXST_EBT]);
+			_slot_set_dur(CXST_EBT, dur_2);
+			break;
+		case BTC_CXP_OFFE_2GBWMIXB: /* for GC-only NOA < bt_slot_req */
+			_slot_set(CXST_E2G, dur_1, tbl_w1, SLOT_MIX);
+			_slot_cpy(&s[CXST_EBT], &s_def[CXST_EBT]);
+			break;
+		case BTC_CXP_OFFE_2GBWMIXB2: /* for GC+STA NOA < bt_slot_req */
+			_slot_set(CXST_E2G, dur_1, tbl_w1, SLOT_MIX);
+			_slot_cpy(&s[CXST_EBT], &s_def[CXST_EBT]);
+			break;
+		case BTC_CXP_OFFE_WL: /*for 4-way  */
+			_slot_set(CXST_E2G, dur_1, cxtbl[1], SLOT_MIX);
+			_slot_set(CXST_EBT, 0, cxtbl[1], SLOT_MIX);
+			break;
+		}
+
+		_slot_cpy(&s[CXST_E5G], &s_def[CXST_E5G]);
+		/* for p2p-on transient time */
+		_slot_cpy(&s[CXST_OFF], &s_def[CXST_OFF]);
+		break;
+	case BTC_CXP_FIX: /* TDMA Fix-Slot */
+		_write_scbd(btc, BTC_WSCB_TDMA, true);
+		_tdma_cpy(t, &t_def[CXTD_FIX]);
+
+		if (dm->fddt_train) {
+			tbl_w1 = cxtbl[0];
+			tbl_b1 = cxtbl[0];
+		}
+
+		switch (policy_type) {
+		case BTC_CXP_FIX_TD3030: /* W1:B1 = 30:30 */
+			_slot_set(CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD5050: /* W1:B1 = 50:50 */
+			_slot_set(CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 50, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD2030: /* W1:B1 = 20:30 */
+			_slot_set(CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD4010: /* W1:B1 = 40:10 */
+			_slot_set(CXST_W1, 40, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 10, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD4010ISO: /* W1:B1 = 40:10 for WL busy/scan */
+			_slot_set(CXST_W1, 40, cxtbl[1], SLOT_ISO);
+			_slot_set(CXST_B1, 10, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD4010ISO_DL: /* W1:B1 = 40:10 for WL DL/scan*/
+			_slot_set(CXST_W1, 40, cxtbl[25], SLOT_ISO);
+			_slot_set(CXST_B1, 10, cxtbl[25], SLOT_ISO);
+			break;
+		case BTC_CXP_FIX_TD7010ISO_UL: /* W1:B1 = 70:10 for WL UL/scan*/
+			_slot_set(CXST_W1, 70, cxtbl[8], SLOT_ISO);
+			_slot_set(CXST_B1, 10, cxtbl[3], SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD7010: /* W1:B1 = 70:10 */
+			_slot_set(CXST_W1, 70, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 10, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD2060: /* W1:B1 = 20:60 */
+			_slot_set(CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD3060: /* W1:B1 = 30:60 */
+			_slot_set(CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD2080: /* W1:B1 = 20:80 */
+			_slot_set(CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 80, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TDW1B1: /* W1:B1 = user-define */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		}
+		break;
+	case BTC_CXP_PFIX: /* PS-TDMA Fix-Slot */
+		_write_scbd(btc, BTC_WSCB_TDMA, true);
+		_tdma_cpy(t, &t_def[CXTD_PFIX]);
+
+		switch (policy_type) {
+		case BTC_CXP_PFIX_TD3030: /* W1:B1 = 30:30 */
+			_slot_set(CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD5050: /* W1:B1 = 50:50 */
+			_slot_set(CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 50, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD2030: /* W1:B1 = 20:30 */
+			_slot_set(CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD2060: /* W1:B1 = 20:60 */
+			_slot_set(CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD3070: /* W1:B1 = 30:60 */
+			_slot_set(CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD2080: /* W1:B1 = 20:80 */
+			_slot_set(CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, 80, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TDW1B1: /* W1:B1 = user-define */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		}
+		break;
+	case BTC_CXP_AUTO: /* TDMA Auto-Slot */
+		_write_scbd(btc, BTC_WSCB_TDMA, true);
+		_tdma_cpy(t, &t_def[CXTD_AUTO]);
+		_slot_cpy(&s[CXST_W2], &s_def[CXST_W2]);
+
+		if (dm->fddt_train) {
+			tbl_w1 = cxtbl[8];
+			tbl_b1 = cxtbl[0];
+		}
+
+		switch (policy_type) {
+		case BTC_CXP_AUTO_TD50B1: /* W1:B1 = 50:BTC_B1_MAX */
+			_slot_set(CXST_W1,  50, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO_TD60B1: /* W1:B1 = 60:BTC_B1_MAX */
+			_slot_set(CXST_W1,  60, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO_TD20B1:  /* W1:B1 = 20:BTC_B1_MAX */
+			_slot_set(CXST_W1,  20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO_TDW1B1: /* W1:B1 = user-define */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO_TDW1B1_CXTPS: /* W1:B1 = user-define */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			t->txflctrl= CXTPS_ON;
+			break;
+		case BTC_CXP_AUTO_TDW1B1_LESCAN:
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			_slot_set_dur(CXST_W2, dm->slot_dur[CXST_W2]);
+			_tdma_set_le_init_end();
+			break;
+		}
+		break;
+	case BTC_CXP_PAUTO: /* PS-TDMA Auto-Slot */
+		_write_scbd(btc, BTC_WSCB_TDMA, true);
+		_tdma_cpy(t, &t_def[CXTD_PAUTO]);
+		_slot_cpy(&s[CXST_W2], &s_def[CXST_W2]);
+
+		if (dm->fddt_train) {
+			tbl_w1 = cxtbl[8];
+			tbl_b1 = cxtbl[2];
+		}
+
+		switch (policy_type) {
+		case BTC_CXP_PAUTO_TD50B1: /* W1:B1 = 50:BTC_B1_MAX */
+			_slot_set(CXST_W1,  50, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_TD60B1: /* W1:B1 = 60:BTC_B1_MAX */
+			_slot_set(CXST_W1,  60, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_TD20B1:  /* W1:B1 = 20:BTC_B1_MAX */
+			_slot_set(CXST_W1,  20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_TDW1B1: /* W1:B1 = user-define */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_FDDT1: /* for FDD-Train Default */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_TDW1B1_LESCAN:
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			_slot_set_dur(CXST_W2, dm->slot_dur[CXST_W2]);
+			_tdma_set_le_init_end();
+			break;
+		}
+		break;
+	case BTC_CXP_AUTO2: /* TDMA Auto-Slot2 */
+		_write_scbd(btc, BTC_WSCB_TDMA, true);
+		_tdma_cpy(t, &t_def[CXTD_AUTO2]);
+
+		switch (policy_type) {
+		case BTC_CXP_AUTO2_TD3050: /* W1:B4 = 30:50 */
+			_slot_set(CXST_W1,  30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  50, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD3070: /* W1:B4 = 30:70 */
+			_slot_set(CXST_W1,  30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  70, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD5050: /* W1:B4 = 50:50 */
+			_slot_set(CXST_W1,  50, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  50, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD6060: /* W1:B4 = 60:60 */
+			_slot_set(CXST_W1,  60, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  60, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD2080: /* W1:B4 = 20:80 */
+			_slot_set(CXST_W1,  20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  80, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TDW1B4: /* W1:B1 = user-define */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4, dm->slot_dur[CXST_B4],
+				  tbl_b4, SLOT_MIX);
+			break;
+		}
+		break;
+	case BTC_CXP_PAUTO2: /* PS-TDMA Auto-Slot2 */
+		_write_scbd(btc, BTC_WSCB_TDMA, true);
+		_tdma_cpy(t, &t_def[CXTD_PAUTO2]);
+
+		switch (policy_type) {
+		case BTC_CXP_PAUTO2_TD3050: /* W1:B4 = 30:50 */
+			_slot_set(CXST_W1,  30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  50, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD3070: /* W1:B4 = 30:70 */
+			_slot_set(CXST_W1,  30, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, cxtbl[3], SLOT_MIX);
+			_slot_set(CXST_B4,  70, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD5050: /* W1:B4 = 50:50 */
+			_slot_set(CXST_W1,  50, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  50, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD6060: /* W1:B4 = 60:60 */
+			_slot_set(CXST_W1,  60, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  60, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD2080: /* W1:B4 = 20:80 */
+			_slot_set(CXST_W1,  20, tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4,  80, tbl_b4, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TDW1B4: /* W1:B1 = user-define */
+			_slot_set(CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			_slot_set(CXST_B4, dm->slot_dur[CXST_B4],
+				  tbl_b4, SLOT_MIX);
+			break;
+		}
+		break;
+	}
+
+	if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC && dm->tdma.rxflctrl) {
+		null_role = (dm->wl_scc.null_role1 & 0xf) +
+			    ((dm->wl_scc.null_role2 & 0xf) << 4);
+		_tdma_set_flctrl_role(null_role);
+	}
+
+	/* enter leak_slot after each null-1 */
+	if (dm->leak_ap && dm->tdma.leak_n > 1)
+		_tdma_set_lek(1);
+
+	if (dm->tdma_instant_excute ||
+	    (dm->error & BTC_DMERR_TDMA_NO_SYNC) ||
+	    (dm->error & BTC_DMERR_SLOT_NO_SYNC)) {
+		_tdma_set_instant();
+		mode = FC_EXEC;
+		PHL_INFO("[BTC], %s(): Set TDMA instant!!\n", __func__);
+	}
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	if (dm->fddt_train) {
+		_tdma_set_fddt_en();
+		
+		/* stop -> run/debug, different AP/BT  */
+		if (dm->fddt_info.state == BTC_FDDT_STATE_STOP ||
+		    hal_mem_cmp(btc->hal, dm->fddt_info.wl_iot, wl->bssid, 6) ||
+		    dm->fddt_info.bt_iot != b->a2dp_desc.vendor_id)
+			_tdma_set_fddt_renew();
+
+		if (dm->fddt_info.type > BTC_FDDT_TYPE_AUTO)
+			_tdma_set_fddt_dbg();
+
+		hal_mem_cpy(btc->hal, dm->fddt_info.wl_iot, wl->bssid, 6);
+		dm->fddt_info.bt_iot = b->a2dp_desc.vendor_id;
+	}
+#endif
+
+	if (btc->wlcli_h2c_cmd)
+		mode = FC_EXEC;
+
+	_update_poicy(btc, mode, policy_type, action);
+}
+
+void _set_gnt(struct btc_t *btc, u8 phy_map, u8 wl_state, u8 bt_state,
+	      u8 wlact_state)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_dm *dm = &btc->dm;
+	u8 bt_idx;
+	u8 val[12], i;
+	u8 sz1 = sizeof(dm->gnt_set);
+	u8 sz2 = sizeof(dm->wlact_set);
+
+	if (phy_map > BTC_PHY_ALL || ARRAY_SIZE(val) != (sz1 + sz2))
+		return;
+
+	bt_idx = dm->bt_select + 1;  /* translate index to bit-map */
+
+	for (i = 0; i < BTC_PHY_MAX; i++) {
+		if (!(phy_map & BIT(i)))
+			continue;
+
+		switch (wl_state) {
+		case BTC_GNT_HW:
+			dm->gnt_set[i].gnt_wl_sw_en = 0;
+			dm->gnt_set[i].gnt_wl = 0;
+			break;
+		case BTC_GNT_SW_LO:
+			dm->gnt_set[i].gnt_wl_sw_en = 1;
+			dm->gnt_set[i].gnt_wl = 0;
+			break;
+		case BTC_GNT_SW_HI:
+			dm->gnt_set[i].gnt_wl_sw_en = 1;
+			dm->gnt_set[i].gnt_wl = 1;
+			break;
+		case BTC_GNT_SET_SKIP:
+			break;
+		}
+
+		switch (bt_state) {
+		case BTC_GNT_HW:
+			dm->gnt_set[i].gnt_bt_sw_en = 0;
+			dm->gnt_set[i].gnt_bt = 0;
+			break;
+		case BTC_GNT_SW_LO:
+			dm->gnt_set[i].gnt_bt_sw_en = 1;
+			dm->gnt_set[i].gnt_bt = 0;
+			break;
+		case BTC_GNT_SW_HI:
+			dm->gnt_set[i].gnt_bt_sw_en = 1;
+			dm->gnt_set[i].gnt_bt = 1;
+			break;
+		case BTC_GNT_SET_SKIP:
+			break;
+		}
+	}
+
+	for (i = 0; i < 2; i++) {
+		if (!(btc->chip->hw & BTC_FEAT_WLAN_ACT_MUX))
+			break;
+
+		if (!(bt_idx & BIT(i)))
+			continue;
+
+		switch (wlact_state) {
+		case BTC_WLACT_HW:
+			dm->wlact_set[i].sw_en = 0;
+			dm->wlact_set[i].sw_val = 0;
+			break;
+		case BTC_WLACT_SW_LO:
+			dm->wlact_set[i].sw_en = 1;
+			dm->wlact_set[i].sw_val = 0;
+			break;
+		case BTC_WLACT_SW_HI:
+			dm->wlact_set[i].sw_en = 1;
+			dm->wlact_set[i].sw_val = 1;
+			break;
+		}
+	}
+
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		 "[BTC], %s(): phy_map=0x%x, gnt_wl:%d, gnt_bt:%d, wl_act:%d\n",
+		 __func__, phy_map, wl_state, bt_state, wlact_state);
+
+	hal_mem_cpy(h, &val[0], (u8*)dm->gnt_set, sz1);
+	hal_mem_cpy(h, &val[sz1], (u8*)dm->wlact_set, sz2);
+
+	rtw_hal_mac_set_grant_act(h, val);
+}
+
+static void _set_bt_plut(struct btc_t *btc, u8 val)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u8 band_id;
+
+	/* Set BT-polluted from PTA request-MAC */
+	band_id = wl->pta_req_mac;
+	if (wl->bt_polut_type[band_id] == val)
+		return;
+
+	wl->bt_polut_type[band_id] = val;
+	rtw_hal_mac_set_polluted(btc->hal, band_id, val, val);
+}
+
+static void _set_dbcc_ant_ctrl(struct btc_t *btc)
+{
+	struct btc_wl_dbcc_info *wl_dinfo = &btc->cx.wl.dbcc_info;
+	u8 gnt_wl_ctrl, plt_ctrl, i, b2g = 0;
+	bool is_dbcc_all_2g = true;
+
+	for (i = 0; i < HW_PHY_MAX; i++) {
+		if (wl_dinfo->real_band[i] != BAND_ON_24G) {
+			is_dbcc_all_2g = false;
+			break;
+		}
+	}
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): is_dbcc_all_2g=%d\n", __func__, is_dbcc_all_2g);
+
+	/* if dbcc 2-phy are 2G */
+	if (is_dbcc_all_2g) {
+		/* Set PHY-0 SW control  */
+		_set_gnt(btc, BIT(HW_PHY_0), BTC_GNT_SW_HI, BTC_GNT_SW_LO,
+			 BTC_WLACT_HW);
+
+		/* Set PHY-1(BTG) HW control */
+		_set_gnt(btc, BIT(HW_PHY_1), BTC_GNT_HW, BTC_GNT_HW,
+			 BTC_WLACT_HW);
+
+		_set_bt_plut(btc, BTC_PLT_GNT_WL);
+
+		btc->cx.cnt_wl[BTC_WCNT_DBCC_ALL_2G]++;
+		return;
+	}
+
+	for (i = 0; i < HW_PHY_MAX; i++) {
+		b2g = (wl_dinfo->real_band[i] == BAND_ON_24G);
+
+		gnt_wl_ctrl = (b2g? BTC_GNT_HW : BTC_GNT_SW_HI);
+		_set_gnt(btc, BIT(i), gnt_wl_ctrl, BTC_GNT_HW, BTC_WLACT_HW);
+
+		plt_ctrl = (b2g? BTC_PLT_BT : BTC_PLT_NONE);
+		if (b2g)
+			_set_bt_plut(btc, BTC_PLT_GNT_WL);
+	}
+}
+
+static void _set_ant(struct btc_t *btc, bool force_exec, u8 phy_map, u8 type)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	u32 ant_path_type;
+
+	ant_path_type = ((phy_map << 8) + type);
+
+	if (btc->wlcli_h2c_cmd ||
+	    dm->ntfy_type == BTC_NTFY_POWER_OFF ||
+	    dm->ntfy_type == BTC_NTFY_RADIO_STATE ||
+	    dm->ntfy_type == BTC_NTFY_INIT_COEX ||
+	    wl->role_info.dbcc_chg)
+		force_exec = FC_EXEC;
+
+	if (wl_rinfo->link_mode != BTC_WLINK_25G_MCC &&
+	    btc->dm.wl_btg_rx == 2)
+		force_exec = FC_EXEC;
+
+	if (!force_exec && (ant_path_type == dm->set_ant_path)) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s(): return by no change!!\n", __func__);
+		return;
+	} else if (bt->rfk_info.map.run) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s(): return by bt rfk!!\n", __func__);
+		return;
+	} else if (dm->ntfy_type != BTC_NTFY_WL_RFK &&
+		   wl->rfk_info.state != BTC_WRFK_STOP) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s(): return by wl rfk!!\n", __func__);
+		return;
+	}
+
+	dm->set_ant_path = ant_path_type;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): path=0x%x, set_type=%s\n", __func__,
+		  phy_map, id_to_str(BTC_STR_ANTPATH, dm->set_ant_path & 0xff));
+
+	switch (type){
+	case BTC_ANT_WPOWERON:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_BT);
+		break;
+	case BTC_ANT_WINIT:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		/* To avoid BT MP driver case (bt_enable but no mailbox) */
+		if (bt->enable.now && bt->run_patch_code) {
+			_set_gnt(btc, phy_map, BTC_GNT_SW_LO, BTC_GNT_SW_HI,
+				 BTC_WLACT_SW_LO);
+		} else {
+			_set_gnt(btc, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO,
+				 BTC_WLACT_SW_HI);
+		}
+		break;
+	case BTC_ANT_WONLY:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		_set_gnt(btc, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO,
+			 BTC_WLACT_SW_HI);
+		break;
+	case BTC_ANT_WOFF:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_BT);
+		break;
+	case BTC_ANT_W2G:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		if (wl_rinfo->dbcc_en) {
+			_set_dbcc_ant_ctrl(btc);
+			return;
+		}
+		_set_gnt(btc, phy_map, BTC_GNT_HW, BTC_GNT_HW, BTC_WLACT_HW);
+		break;
+	case BTC_ANT_W5G:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		_set_gnt(btc, phy_map, BTC_GNT_SW_HI, BTC_GNT_HW, BTC_WLACT_HW);
+		break;
+	case BTC_ANT_W25G:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		if (wl_rinfo->dbcc_en) {
+			_set_dbcc_ant_ctrl(btc);
+			return;
+		}
+		_set_gnt(btc, phy_map, BTC_GNT_HW, BTC_GNT_HW, BTC_WLACT_HW);
+		break;
+	case BTC_ANT_FREERUN:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		_set_gnt(btc, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_HI,
+			 BTC_WLACT_SW_LO);
+		break;
+	case BTC_ANT_FDDTRAIN:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		_set_gnt(btc, phy_map, BTC_GNT_SW_HI, BTC_GNT_HW, BTC_WLACT_HW);
+		break;
+	case BTC_ANT_WRFK:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		_set_gnt(btc, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO,
+			 BTC_WLACT_HW);
+		break;
+	case BTC_ANT_WRFK2:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_WL);
+		_set_gnt(btc, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO,
+			 BTC_WLACT_SW_HI); /* no BT-Tx */
+		break;
+	case BTC_ANT_BRFK:
+		_set_cx_ctrl(btc, BTC_CTRL_BY_BT);
+		_set_gnt(btc, phy_map, BTC_GNT_SW_LO, BTC_GNT_SW_HI,
+			 BTC_WLACT_SW_LO);
+		break;
+	}
+
+	_set_bt_plut(btc, BTC_PLT_GNT_WL);
+}
+
+void _action_wl_only(struct btc_t *btc)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !!\n", __func__);
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_WONLY);
+	_set_policy(btc, BTC_CXP_OFF_WL, __func__);
+}
+
+void _action_wl_init(struct btc_t *btc)
+{
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !!\n", __func__);
+
+	_set_ant(btc, FC_EXEC, BTC_PHY_ALL, BTC_ANT_WINIT);
+
+	_set_policy(btc, BTC_CXP_OFF_BT, __func__);
+}
+
+void _action_wl_off(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !!\n", __func__);
+
+	/* don't set PTA control if LPS-PG/LPS-CG */
+	if (wl->status.map.rf_off || btc->dm.bt_only) {
+		_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_WOFF);
+	} else if (wl->status.map.lps == BTC_LPS_RF_ON) {
+		if (wl->role_info.link_mode == BTC_WLINK_5G)
+			_set_ant(btc, FC_EXEC, BTC_PHY_ALL, BTC_ANT_W5G);
+		else
+			_set_ant(btc, FC_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	}
+
+	if (wl->role_info.link_mode == BTC_WLINK_5G) {
+		_set_policy(btc, BTC_CXP_OFF_EQ0, __func__);
+	} else if (wl->status.map.lps == BTC_LPS_RF_ON) {
+		if (btc->cx.bt.link_info.a2dp_desc.active)
+			_set_policy(btc, BTC_CXP_OFF_BT, __func__);
+		else
+			_set_policy(btc, BTC_CXP_OFF_BWB1, __func__);
+	} else {
+		_set_policy(btc, BTC_CXP_OFF_BT, __func__);
+	}
+}
+
+void _action_freerun(struct btc_t *btc)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !!\n", __func__);
+	btc->dm.freerun = true;
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, BTC_CXP_OFF_EQ0, __func__);
+}
+
+void _action_fddtrain(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s() type=%d !!\n", __func__, dm->fddt_info.type);
+
+	bt->scan_rx_low_pri = true;
+
+	switch (dm->fddt_info.type) {
+	case BTC_FDDT_TYPE_AUTO:
+		dm->slot_dur[CXST_W1] = 50;
+		dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+		policy_type = BTC_CXP_PAUTO_FDDT1;
+		break;
+	case BTC_FDDT_TYPE_FIX_TDD:
+		dm->slot_dur[CXST_W1] = 50;
+		dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+		policy_type = BTC_CXP_PAUTO_TDW1B1;
+		break;
+	case BTC_FDDT_TYPE_FIX_FULL_FDD:
+		dm->slot_dur[CXST_W1] = 50;
+		dm->slot_dur[CXST_B1] = 50;
+		policy_type = BTC_CXP_FIX_TDW1B1;
+		break;
+	}
+
+	dm->fddt_train = BTC_FDDT_ENABLE;
+
+	_set_fddt_ctrl(btc, NM_EXEC);
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+void _action_bt_whql(struct btc_t *btc)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !!\n", __func__);
+
+	_set_ant(btc, FC_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, BTC_CXP_OFF_BT, __func__);
+}
+
+void _action_bt_rfk(struct btc_t *btc)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !!\n", __func__);
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_BRFK);
+	_set_policy(btc, BTC_CXP_OFF_BT, __func__);
+}
+
+void _action_bt_off(struct btc_t *btc)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s !!\n", __func__);
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_WONLY);
+	_set_policy(btc, BTC_CXP_OFF_BT, __func__);
+}
+
+static void _action_bt_idle(struct btc_t *btc)
+{
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_link_info *b = &btc->cx.bt.link_info;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		switch(btc->cx.state_map) {
+		case BTC_WBUSY_BNOSCAN: /* wl-busy + bt idle*/
+		case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-idle */
+			if (b->status.map.connect) {
+				policy_type = BTC_CXP_FIX_TD4010;
+			} else if (wl->status.map.traffic_dir & BIT(TRAFFIC_DL)) {
+				policy_type = BTC_CXP_FIX_TD4010ISO_DL;
+			} else {
+				policy_type = BTC_CXP_FIX_TD7010ISO_UL;
+				bt->scan_rx_low_pri = true;
+			}
+			break;
+		case BTC_WBUSY_BSCAN: /*wl-busy + bt-inq */
+			if (bt->pag)
+				policy_type = BTC_CXP_PFIX_TD2080;
+			else
+				policy_type = BTC_CXP_PFIX_TD5050;
+			break;
+		case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq */
+		case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq */
+			if (bt->pag)
+				policy_type = BTC_CXP_FIX_TD2080;
+			else
+				policy_type = BTC_CXP_FIX_TD5050;
+			break;
+		case BTC_WLINKING: /* wl-connecting + bt-inq or bt-idle */
+			if (bt->pag)
+				policy_type = BTC_CXP_FIX_TD2080;
+			else
+				policy_type = BTC_CXP_FIX_TD7010;
+			break;
+		case BTC_WIDLE:       /* wl-idle + bt-idle */
+			if (b->status.map.connect)
+				policy_type = BTC_CXP_OFF_BWB1;
+			else
+				policy_type = BTC_CXP_FIX_TD4010;
+			break;
+		}
+	} else { /* dedicated-antenna */
+		if (btc->cx.other.type != BTC_EXTSOC_FUNC_NONE && wl->ap_type != BTC_SPAP_CMW) {
+			btc->dm.slot_dur[CXST_W1] = 50;
+			btc->dm.slot_dur[CXST_B1] = 5;
+			policy_type = BTC_CXP_PFIX_TDW1B1;
+		} else {
+			policy_type = BTC_CXP_OFF_EQ0;
+		}
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_hfp(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (btc->cx.wl.status.map._4way) {
+			policy_type = BTC_CXP_OFF_WL;
+		} else if (a2dp->active) {
+			btc->dm.slot_dur[CXST_W1] = 80;
+			btc->dm.slot_dur[CXST_B1] = 20;
+			policy_type = BTC_CXP_PFIX_TDW1B1;
+		} else if (wl->status.map.traffic_dir & BIT(TRAFFIC_UL)) {
+			btc->cx.bt.scan_rx_low_pri = true;
+			policy_type = BTC_CXP_OFF_BWB2;
+		} else if (btc->mdinfo.wa_type & BTC_WA_HFP_LAG) {
+			policy_type = BTC_CXP_OFF_BWB2;
+		}else {
+			policy_type = BTC_CXP_OFF_BWB1;
+		}
+	} else { /* dedicated-antenna */
+		if (wl->legacy_mode)
+			policy_type = BTC_CXP_OFF_BWB1;
+		else if (wl->status.map.traffic_dir & BIT(TRAFFIC_UL))
+			policy_type = BTC_CXP_OFF_EQ5;
+		else
+			policy_type = BTC_CXP_OFF_EQ2;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_hid(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_hid_desc *hid = &bt->link_info.hid_desc;
+	struct btc_bt_a2dp_desc *a2dp = &bt->link_info.a2dp_desc;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (wl->status.map._4way) {
+			policy_type = BTC_CXP_OFF_WL;
+		} else if (a2dp->active) {
+			btc->dm.slot_dur[CXST_W1] = 80;
+			btc->dm.slot_dur[CXST_B1] = 20;
+			policy_type = BTC_CXP_PFIX_TDW1B1;
+		} else if (wl->status.map.traffic_dir & BIT(TRAFFIC_UL)) {
+			btc->cx.bt.scan_rx_low_pri = true;
+			if (hid->type & BTC_HID_BLE)
+				policy_type = BTC_CXP_OFF_BWB0;
+			else
+				policy_type = BTC_CXP_OFF_BWB2;
+		} else if (hid->type == BTC_HID_218) {
+			bt->scan_rx_low_pri = true;
+			policy_type = BTC_CXP_OFF_BWB2;
+		} else if (btc->chip->hw & BTC_FEAT_NONBTG_GWL_THRU) {
+			policy_type = BTC_CXP_OFF_BWB3;
+		} else if (btc->mdinfo.wa_type & BTC_WA_HFP_LAG &&
+			   bt->link_info.hfp_desc.exist) {
+			policy_type = BTC_CXP_OFF_BWB2;
+		}else {
+			policy_type = BTC_CXP_OFF_BWB1;
+		}
+	} else { /* dedicated-antenna */
+		if (wl->legacy_mode)
+			policy_type = BTC_CXP_OFF_BWB1;
+		else if (wl->status.map.traffic_dir & BIT(TRAFFIC_UL))
+			policy_type = BTC_CXP_OFF_EQ4;
+		else
+			policy_type = BTC_CXP_OFF_EQ3;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_hid_hfp(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_hid_desc *hid = &bt->link_info.hid_desc;
+	struct btc_bt_a2dp_desc *a2dp = &bt->link_info.a2dp_desc;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (wl->status.map._4way) {
+			policy_type = BTC_CXP_OFF_WL;
+		} else if (a2dp->active) {
+			btc->dm.slot_dur[CXST_W1] = 80;
+			btc->dm.slot_dur[CXST_B1] = 20;
+			policy_type = BTC_CXP_PFIX_TDW1B1;
+		} else if (wl->status.map.traffic_dir & BIT(TRAFFIC_UL)) {
+			btc->cx.bt.scan_rx_low_pri = true;
+			if (hid->type & BTC_HID_BLE)
+				policy_type = BTC_CXP_OFFB_BWB0;
+			else
+				policy_type = BTC_CXP_OFFB_BWB2;
+		} else if (hid->type == BTC_HID_218) {
+			bt->scan_rx_low_pri = true;
+			policy_type = BTC_CXP_OFFB_BWB2;
+		} else if (btc->chip->hw & BTC_FEAT_NONBTG_GWL_THRU) {
+			policy_type = BTC_CXP_OFFB_BWB3;
+		} else if (btc->mdinfo.wa_type & BTC_WA_HFP_LAG &&
+			   bt->link_info.hfp_desc.exist) {
+			policy_type = BTC_CXP_OFFB_BWB2;
+		}else {
+			policy_type = BTC_CXP_OFFB_BWB1;
+		}
+	} else { /* dedicated-antenna */
+		if (wl->legacy_mode)
+			policy_type = BTC_CXP_OFFB_BWB1;
+		else if (wl->status.map.traffic_dir & BIT(TRAFFIC_UL))
+			policy_type = BTC_CXP_OFF_EQ4;
+		else
+			policy_type = BTC_CXP_OFF_EQ3;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+
+static void _action_bt_a2dp(struct btc_t *btc)
+{
+	struct btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
+	struct btc_bt_a2dp_desc a2dp = bt_linfo->a2dp_desc;
+	struct btc_dm *dm = &btc->dm;
+	u16 policy_type = BTC_CXP_OFF_BT, w1_slot_dur = 0;
+
+	if (bt_linfo->slave_role)
+		w1_slot_dur = 20;
+	else if (a2dp.vendor_id == 0x4c || dm->leak_ap)
+		w1_slot_dur = 40;
+	else if (dm->le_init_en)
+		w1_slot_dur = 60;
+	else
+		w1_slot_dur = 50;
+
+	switch(btc->cx.state_map) {
+	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-A2DP */
+		dm->slot_dur[CXST_W1] = w1_slot_dur;
+		dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+		// for sporton Tx null issue
+		if (btc->mdinfo.wa_type & BTC_WA_NULL_AP) {
+			policy_type = BTC_CXP_AUTO_TDW1B1_CXTPS;
+		} else if (dm->le_init_en) {
+			dm->slot_dur[CXST_W2] = 25;
+			policy_type = BTC_CXP_PAUTO_TDW1B1_LESCAN;
+		} else {
+			policy_type = BTC_CXP_PAUTO_TDW1B1;
+		}
+		break;
+	case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq + bt-A2DP */
+	case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP */
+		policy_type = BTC_CXP_PAUTO2_TD3070;
+		break;
+	case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq + bt-A2DP */
+		policy_type = BTC_CXP_AUTO2_TD3050;
+		break;
+	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-A2DP */
+	case BTC_WLINKING:      /* wl-connecting + bt-A2DP */
+		dm->slot_dur[CXST_W1] = w1_slot_dur;
+		dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+		if (dm->le_init_en) {
+			dm->slot_dur[CXST_W2] = 25;
+			policy_type = BTC_CXP_AUTO_TDW1B1_LESCAN;
+		} else {
+			policy_type = BTC_CXP_AUTO_TDW1B1;
+		}
+		break;
+	case BTC_WIDLE:  /* wl-idle + bt-A2DP */
+		policy_type = BTC_CXP_AUTO_TD20B1;
+		break;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_a2dpsink(struct btc_t *btc)
+{
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	switch(btc->cx.state_map) {
+	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-A2dp_Sink */
+		policy_type = BTC_CXP_PFIX_TD3070;
+		break;
+	case BTC_WBUSY_BSCAN:   /* wl-busy + bt-inq + bt-A2dp_Sink */
+		policy_type = BTC_CXP_PFIX_TD2060;
+		break;
+	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-A2dp_Sink */
+		policy_type = BTC_CXP_FIX_TD2030;
+		break;
+	case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq + bt-A2dp_Sink */
+		policy_type = BTC_CXP_FIX_TD2060;
+		break;
+	case BTC_WLINKING:    /* wl-connecting + bt-A2dp_Sink */
+		policy_type = BTC_CXP_FIX_TD3030;
+		break;
+	case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq + bt-A2dp_Sink */
+	case BTC_WIDLE:       /* wl-idle + bt-A2dp_Sink */
+		policy_type = BTC_CXP_FIX_TD2080;
+		break;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_bis(struct btc_t *btc)
+{
+	if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED)
+		_action_freerun(btc);
+}
+
+static void _action_bt_pan(struct btc_t *btc)
+{
+	struct btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	switch(btc->cx.state_map) {
+	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-PAN */
+		if (a2dp->active) {
+			btc->dm.slot_dur[CXST_W1] = 80;
+			btc->dm.slot_dur[CXST_B1] = 20;
+			policy_type = BTC_CXP_PFIX_TDW1B1;
+		} else {
+			policy_type = BTC_CXP_PFIX_TD5050;
+		}
+		break;
+	case BTC_WBUSY_BSCAN:   /* wl-busy + bt-inq + bt-PAN */
+		policy_type = BTC_CXP_PFIX_TD3070;
+		break;
+	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-PAN */
+		policy_type = BTC_CXP_FIX_TD3030;
+		break;
+	case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq + bt-PAN */
+		policy_type = BTC_CXP_FIX_TD3060;
+		break;
+	case BTC_WLINKING:    /* wl-connecting + bt-PAN */
+		policy_type = BTC_CXP_FIX_TD4010ISO;
+		break;
+	case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq + bt-pan */
+	case BTC_WIDLE:       /* wl-idle + bt-pan */
+		policy_type = BTC_CXP_PFIX_TD2080;
+		break;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_a2dp_hid(struct btc_t *btc)
+{
+	struct btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
+	struct btc_bt_a2dp_desc a2dp = bt_linfo->a2dp_desc;
+	struct btc_dm *dm = &btc->dm;
+	u16 policy_type = BTC_CXP_OFF_BT, w1_slot_dur = 0;
+
+	if (bt_linfo->slave_role)
+		w1_slot_dur = 20;
+	else if (a2dp.vendor_id == 0x4c || dm->leak_ap)
+		w1_slot_dur = 40;
+	else
+		w1_slot_dur = 50;
+
+	switch(btc->cx.state_map) {
+	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-A2DP + HID */
+	case BTC_WIDLE:  /* wl-idle + bt-A2DP + HID */
+		dm->slot_dur[CXST_W1] = w1_slot_dur;
+		dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+		policy_type = BTC_CXP_PAUTO_TDW1B1;
+		break;
+	case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq + bt-A2DP + HID */
+	case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP + HID */
+		policy_type = BTC_CXP_PAUTO2_TD3070;
+		break;
+	case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq + bt-A2DP + HID */
+		policy_type = BTC_CXP_AUTO2_TD3050;
+		break;
+	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-A2DP + HID */
+	case BTC_WLINKING: /* wl-connecting + bt-A2DP + HID */
+		dm->slot_dur[CXST_W1] = w1_slot_dur;
+		dm->slot_dur[CXST_B1] = BTC_B1_MAX;
+		policy_type = BTC_CXP_AUTO_TDW1B1;
+		break;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_a2dp_pan(struct btc_t *btc)
+{
+	struct btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_dm *dm = &btc->dm;
+	u16 policy_type = BTC_CXP_OFF_BT;
+	bool wl_cpt_test = false;
+
+	if (dm->vid == BTC_VID_LNV && (wl->ap_type & BIT(BTC_SPAP_TPL)) &&
+	    bt_linfo->slave_role && wl->rssi_level <= 2)
+		wl_cpt_test = true;
+
+	switch(btc->cx.state_map) {
+	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-A2DP+PAN */
+		if (wl_cpt_test)
+			policy_type = BTC_CXP_PAUTO2_TD5050;
+		else
+			policy_type = BTC_CXP_PAUTO2_TD3070;
+		break;
+	case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP+PAN */
+		if (wl_cpt_test)
+			policy_type = BTC_CXP_PAUTO2_TD5050;
+		else
+			policy_type = BTC_CXP_PAUTO2_TD3070;
+		break;
+	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-A2DP+PAN */
+		policy_type = BTC_CXP_AUTO2_TD5050;
+		break;
+	case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq + bt-A2DP+PAN */
+		policy_type = BTC_CXP_AUTO2_TD3070;
+		break;
+	case BTC_WLINKING:    /* wl-connecting + bt-A2DP+PAN */
+		policy_type = BTC_CXP_AUTO2_TD3050;
+		break;
+	case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq + bt-A2DP+PAN */
+	case BTC_WIDLE:       /* wl-idle + bt-A2DP+PAN */
+		policy_type = BTC_CXP_PAUTO2_TD2080;
+		break;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_pan_hid(struct btc_t *btc)
+{
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	switch(btc->cx.state_map) {
+	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-PAN+HID */
+		policy_type = BTC_CXP_PFIX_TD3030;
+		break;
+	case BTC_WBUSY_BSCAN:  /* wl-busy + bt-inq + bt-PAN+HID */
+		policy_type = BTC_CXP_PFIX_TD3070;
+		break;
+	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-PAN+HID */
+		policy_type = BTC_CXP_FIX_TD3030;
+		break;
+	case BTC_WSCAN_BSCAN: /* wl-scan + bt-inq + bt-PAN+HID */
+		policy_type = BTC_CXP_FIX_TD3060;
+		break;
+	case BTC_WLINKING:   /* wl-connecting + bt-PAN+HID */
+		policy_type = BTC_CXP_FIX_TD4010;
+		break;
+	case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq + bt-PAN+HID */
+	case BTC_WIDLE:       /* wl-idle + bt-PAN+HID */
+		policy_type = BTC_CXP_PFIX_TD2080;
+		break;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static void _action_bt_a2dp_pan_hid(struct btc_t *btc)
+{
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	switch(btc->cx.state_map) {
+	case BTC_WBUSY_BNOSCAN: /* wl-busy + bt-A2DP+PAN+HID */
+		policy_type = BTC_CXP_PAUTO2_TD3070;
+		break;
+	case BTC_WBUSY_BSCAN:   /* wl-busy + bt-inq + bt-A2DP+PAN+HID */
+		policy_type = BTC_CXP_PAUTO2_TD3070;
+		break;
+	case BTC_WSCAN_BSCAN:   /* wl-scan + bt-inq + bt-A2DP+PAN+HID */
+		policy_type = BTC_CXP_AUTO2_TD3070;
+		break;
+	case BTC_WSCAN_BNOSCAN: /* wl-scan + bt-A2DP+PAN+HID */
+	case BTC_WLINKING:      /* wl-connecting + bt-A2DP+PAN+HID */
+		policy_type = BTC_CXP_AUTO2_TD3050;
+		break;
+	case BTC_WIDLE_BSCAN: /* wl-idle + bt-inq + bt-A2DP+PAN+HID */
+	case BTC_WIDLE:       /* wl-idle + bt-A2DP+PAN+HID */
+		policy_type = BTC_CXP_PAUTO2_TD2080;
+		break;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+void _action_wl_5g(struct btc_t *btc)
+{
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W5G);
+	_set_policy(btc, BTC_CXP_OFF_EQ0, __func__);
+}
+
+void _action_wl_other(struct btc_t *btc)
+{
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED)
+		_set_policy(btc, BTC_CXP_OFFB_BWB1, __func__);
+	else
+		_set_policy(btc, BTC_CXP_OFF_EQ0, __func__);
+}
+
+void _action_wl_idle(struct btc_t *btc)
+{
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED)
+		_set_policy(btc, BTC_CXP_OFFB_BWB1, __func__);
+	else
+		_set_policy(btc, BTC_CXP_OFF_EQ0, __func__);
+}
+
+void _action_wl_nc(struct btc_t *btc)
+{
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED)
+		_set_policy(btc, BTC_CXP_OFF_BT2, __func__);
+	else
+		_set_policy(btc, BTC_CXP_OFF_EQ0, __func__);
+}
+
+void _action_wl_rfk(struct btc_t *btc)
+{
+	struct btc_wl_rfk_info rfk = btc->cx.wl.rfk_info;
+
+	if (rfk.state != BTC_WRFK_START && rfk.type != BTC_RF_RXDCK)
+		return;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): band=%d, type=%d, state=%d\n",
+		  __func__, rfk.band, rfk.type, rfk.state);
+
+	btc->dm.tdma_instant_excute = 1;
+
+	if ((rfk.type == BTC_RF_RXDCK && rfk.state == BTC_WRFK_ONESHOT_START) ||
+	    btc->mdinfo.ant.type == BTC_ANT_SHARED) {
+		_set_ant(btc, FC_EXEC, BTC_PHY_ALL, BTC_ANT_WRFK2);
+		_set_policy(btc, BTC_CXP_OFF_WL2, __func__);
+	} else {
+		_set_ant(btc, FC_EXEC, BTC_PHY_ALL, BTC_ANT_WRFK);
+		_set_policy(btc, BTC_CXP_OFF_WL, __func__);
+	}
+}
+
+void _action_common(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_dm *dm = &btc->dm;
+	u32 bt_rom_code_id = 0, bt_fw_ver = 0;
+
+#if BTC_BB_TX_1SS_LIMIT
+	_set_trx_nss(btc);
+#endif
+	_set_halbb_btg_ctrl(btc);
+	_set_halbb_preagc_ctrl(btc);
+	_set_halmac_tx_limit(btc);
+	_set_phl_bt_slot_req(btc);
+	_set_phl_stbc_req(btc); /* STBC flag*/
+	_set_bt_afh_info(btc);
+	_set_bt_rx_agc(btc); /* must call after _set_halbb_btg_ctrl() */
+	_set_rf_trx_para(btc);
+	_set_fddt_status(btc);
+
+	_write_scbd(btc, BTC_WSCB_RXSCAN_PRI, (bool)bt->scan_rx_low_pri);
+
+	if (wl->scbd_change) {
+		rtw_hal_mac_set_scoreboard(btc->hal, &wl->scbd);
+		/* Add delay to avoid BT FW loss information */
+		_os_delay_us(btc->hal, BTC_SCBD_REWRITE_DELAY);
+		wl->scbd_change = 0;
+
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], write scbd : 0x%08x \n", wl->scbd);
+		btc->cx.cnt_wl[BTC_WCNT_SCBDUPDATE]++;
+	}
+
+	bt_rom_code_id = (btc->chip->chip_id & 0xffff0) >> 4;
+	bt_fw_ver = bt->ver_info.fw & 0xffff;
+	if (bt->enable.now && (bt_fw_ver == 0 ||
+	    (bt_fw_ver == bt_rom_code_id &&
+	    bt->run_patch_code && btc->chip->scbd)))
+		_set_fw_rpt(btc, RPT_EN_BT_VER_INFO, 1);
+	else
+		_set_fw_rpt(btc, RPT_EN_BT_VER_INFO, 0);
+
+	dm->tdma_instant_excute = 0;
+	dm->country_code_proc = 0;
+}
+
+void _action_wl_2g_sta(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_link_info *bt_linfo = &bt->link_info;
+	struct btc_bt_hid_desc hid = bt_linfo->hid_desc;
+	struct btc_bt_a2dp_desc a2dp = bt_linfo->a2dp_desc;
+	struct btc_bt_pan_desc pan = bt_linfo->pan_desc;
+	u8 profile_map = 0;
+
+	if (dm->freerun_chk && !dm->fddt_train_chk) {
+		_action_freerun(btc);
+		return;
+	}
+
+	if (bt_linfo->hfp_desc.exist)
+		profile_map |= BTC_BT_HFP;
+
+	if (bt_linfo->hid_desc.exist)
+		profile_map |= BTC_BT_HID;
+
+	if (bt_linfo->a2dp_desc.exist)
+		profile_map |= BTC_BT_A2DP;
+
+	if (bt_linfo->pan_desc.exist)
+		profile_map |= BTC_BT_PAN;
+
+	if (bt_linfo->leaudio_desc.bis_cnt) {
+		profile_map |= BTC_BT_BIS;
+		_action_bt_bis(btc);
+		return;
+	}
+
+	switch (profile_map) {
+	case 0:
+		if (a2dp.active || pan.active)
+			_action_bt_pan(btc);
+		else
+			_action_bt_idle(btc);
+		break;
+	case BTC_BT_HFP:
+			_action_bt_hfp(btc);
+		break;
+	case BTC_BT_HID:
+			_action_bt_hid(btc);
+		break;
+	case BTC_BT_HFP | BTC_BT_HID:
+			_action_bt_hid_hfp(btc);
+		break;
+	case BTC_BT_A2DP:
+		if (dm->fddt_train_chk)
+			_action_fddtrain(btc);
+		else if (a2dp.sink)
+			_action_bt_a2dpsink(btc);
+		else if (bt_linfo->multi_link.now && !hid.pair_cnt)
+			 _action_bt_a2dp_pan(btc);
+		else
+			_action_bt_a2dp(btc);
+		break;
+	case BTC_BT_PAN:
+		_action_bt_pan(btc);
+		break;
+	case BTC_BT_A2DP | BTC_BT_HFP:
+	case BTC_BT_A2DP | BTC_BT_HID:
+	case BTC_BT_A2DP | BTC_BT_HFP | BTC_BT_HID:
+		if (dm->fddt_train_chk)
+			_action_fddtrain(btc);
+		else if (a2dp.sink)
+			_action_bt_a2dpsink(btc);
+		else if (pan.active)
+			_action_bt_a2dp_pan_hid(btc);
+		else
+			_action_bt_a2dp_hid(btc);
+		break;
+	case BTC_BT_A2DP | BTC_BT_PAN:
+		if (a2dp.sink)
+			_action_bt_a2dpsink(btc);
+		else
+			_action_bt_a2dp_pan(btc);
+		break;
+	case BTC_BT_PAN | BTC_BT_HFP:
+	case BTC_BT_PAN | BTC_BT_HID:
+	case BTC_BT_PAN | BTC_BT_HFP | BTC_BT_HID:
+		_action_bt_pan_hid(btc);
+		break;
+	case BTC_BT_A2DP | BTC_BT_PAN | BTC_BT_HID:
+	case BTC_BT_A2DP | BTC_BT_PAN | BTC_BT_HFP:
+	default:
+		if (a2dp.sink)
+			_action_bt_a2dpsink(btc);
+		else
+			_action_bt_a2dp_pan_hid(btc);
+		break;
+	}
+}
+
+void _action_wl_scan(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+
+	if (btc->hal->scanofld_en) {
+		_action_wl_25g_mcc(btc);
+	} else if (wl_rinfo->dbcc_en) {
+		if (wl_dinfo->real_band[HW_PHY_0] != BAND_ON_24G &&
+		    wl_dinfo->real_band[HW_PHY_1] != BAND_ON_24G)
+			_action_wl_5g(btc);
+		else
+			_action_wl_2g_sta(btc);
+	} else {
+		if (wl->scan_info.band[HW_PHY_0] != BAND_ON_24G)
+			_action_wl_5g(btc);
+		else
+			_action_wl_2g_sta(btc);
+	}
+}
+
+void _action_wl_25g_mcc(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+    struct btc_dm *dm = &btc->dm;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) {
+		if (wl->status.map._4way)
+			policy_type = BTC_CXP_OFFE_WL;
+		else if (wl->status.val & BTC_WL_LINK_SCAN)
+			policy_type = BTC_CXP_OFFE_2GBWMIXB;
+		else if (bt->link_info.status.map.connect == 0)
+			policy_type = BTC_CXP_OFFE_2GISOB;
+		else
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+	} else { /* dedicated-antenna */
+		policy_type = BTC_CXP_OFF_EQ0;
+	}
+
+	dm->e2g_slot_limit = BTC_E2G_LIMIT_DEF;
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W25G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+void _action_wl_2g_mcc(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_dm *dm = &btc->dm;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (wl->status.map._4way)
+			policy_type = BTC_CXP_OFFE_WL;
+		else if (bt->link_info.status.map.connect == 0)
+			policy_type = BTC_CXP_OFFE_2GISOB;
+		else
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+	} else { /* dedicated-antenna */
+		policy_type = BTC_CXP_OFF_EQ0;
+	}
+
+	dm->e2g_slot_limit = BTC_E2G_LIMIT_DEF;
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+void _action_wl_2g_scc(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	u16 policy_type = BTC_CXP_OFF_BT;
+	u32 dur = 0, tnull1 = 0;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED) {
+		_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+		_set_policy(btc, BTC_CXP_OFF_EQ0, __func__);
+		return;
+	}
+
+	/* In 2G-SCC + ext-slot mode, tx/rx flow ctrl is BTC's job.
+	 * But WL FW should notify BTC of E2G/EBT slot switch if Go-NOA is on
+	 * (From P2P-Power-Save function).
+	 */
+
+	/* shared-antenna */
+	switch(wl_rinfo->mrole_type) {
+	default:
+		break;
+	case BTC_WLMROLE_STA_GC:
+		dm->wl_scc.null_role1 = PHL_RTYPE_STATION;
+		dm->wl_scc.null_role2 = PHL_RTYPE_P2P_GC;
+		dm->wl_scc.e2g_null = 0; /* no ext-slot-control */
+		_action_wl_2g_sta(btc);
+		return;
+	case BTC_WLMROLE_STA_STA:
+		dm->wl_scc.null_role1 = PHL_RTYPE_STATION;
+		dm->wl_scc.null_role2 = PHL_RTYPE_STATION;
+		dm->wl_scc.e2g_null = 0; /* no ext-slot-control */
+		_action_wl_2g_sta(btc);
+		return;
+	case BTC_WLMROLE_STA_GC_NOA:
+	case BTC_WLMROLE_STA_GO:
+	case BTC_WLMROLE_STA_GO_NOA:
+		dm->wl_scc.null_role1 = PHL_RTYPE_STATION;
+		dm->wl_scc.null_role2 = PHL_RTYPE_NONE;
+		dur = wl_rinfo->mrole_noa_duration;
+
+		if (wl->status.map._4way) {
+			dm->wl_scc.e2g_null = 0;
+			policy_type = BTC_CXP_OFFE_WL;
+		} else if (bt->link_info.status.map.connect == 0) {
+			dm->wl_scc.e2g_null = 0;
+			policy_type = BTC_CXP_OFFE_2GISOB;
+		} else if (bt->link_info.a2dp_desc.exist ||
+ 			   bt->link_info.pan_desc.exist) {
+ 			dm->wl_scc.e2g_null = 1; /* tx null at E2G before EBT */
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+		} else {
+			dm->wl_scc.e2g_null = 0;
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+		}
+
+		break;
+	}
+
+	/* use E2G dur as timer to Tx-null-1 before EBT
+	 * E2G --> Null1-Tx --> 10ms --> EBT
+	 * tnull1 = from E2G to Tx-Null1
+	 */
+	if (dm->wl_scc.e2g_null &&
+	    dur <= BTC_P2P_CYCLE - BTC_NULL1_EBT_EARLY_T && dur > 0)
+	    tnull1 = BTC_P2P_CYCLE - BTC_NULL1_EBT_EARLY_T - dur;
+
+	if (tnull1 > BTC_NULL1_EBT_EARLY_T)
+		dm->e2g_slot_nulltx_time = tnull1;
+	else
+		dm->e2g_slot_nulltx_time = 0;
+
+	dm->e2g_slot_limit = BTC_E2G_LIMIT_DEF;
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+void _action_wl_2g_ap(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_a2dp_desc a2dp = bt->link_info.a2dp_desc;
+	struct btc_dm *dm = &btc->dm;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	/* Todo:if client issue Null-P, ap should follow Null/Null-P slot */
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (wl->status.map._4way) {
+			policy_type = BTC_CXP_OFFE_WL;
+		} else if (a2dp.exist && !a2dp.sink) {
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+		} else if (dm->client_ps_tdma_on == 1 && a2dp.sink) {
+			policy_type = BTC_CXP_OFF_BT2;
+		} else if (bt->link_info.status.map.connect == 0) {
+			if (wl->status.map.traffic_dir & BIT(TRAFFIC_DL)) {
+				policy_type = BTC_CXP_FIX_TD4010ISO_DL;
+			} else {
+				policy_type = BTC_CXP_FIX_TD7010ISO_UL;
+			}
+		} else {
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+		}
+	} else {/* dedicated-antenna */
+		policy_type = BTC_CXP_OFF_EQ0;
+	}
+
+	dm->e2g_slot_limit = BTC_E2G_LIMIT_DEF;
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+void _action_wl_2g_go(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_a2dp_desc a2dp = bt->link_info.a2dp_desc;
+	struct btc_dm *dm = &btc->dm;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (wl->status.map._4way)
+			policy_type = BTC_CXP_OFFE_WL;
+		else if (a2dp.exist && !a2dp.sink)
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+		else if (dm->client_ps_tdma_on == 1 && a2dp.sink)
+			policy_type = BTC_CXP_OFF_BT2;
+		else if (bt->link_info.status.map.connect == 0)
+			policy_type = BTC_CXP_OFFE_2GISOB;
+		else
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+	} else { /* dedicated-antenna */
+		policy_type = BTC_CXP_OFF_EQ0;
+	}
+
+	dm->e2g_slot_limit = BTC_E2G_LIMIT_DEF;
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+static u8 _get_wl_role_idx(struct btc_t *btc, u8 role, u8 *role_id)
+{
+	struct btc_wl_role_info *wl_rinfo = &btc->cx.wl.role_info;
+	u8 i;
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++) {
+		if (wl_rinfo->active_role[i].role != role)
+			continue;
+		
+		if (!wl_rinfo->dbcc_en ||
+		    wl_rinfo->active_role[i].phy == wl_rinfo->dbcc_2g_phy) {
+			*role_id = i;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+void _action_wl_2g_gc(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	u8 noa = 0, rid = 0;
+	u16 policy_type = BTC_CXP_OFF_BT;
+	u32 noa_dur = 0;
+
+	if (_get_wl_role_idx(btc, PHL_RTYPE_P2P_GC, &rid)) {
+		noa = wl_rinfo->active_role[rid].noa;
+		noa_dur = wl_rinfo->active_role[rid].noa_dur;
+	}
+
+	/* Check GC, GC+NOA */
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (!noa) { /* Gc without NOA */
+			_action_wl_2g_sta(btc);
+			return;
+		}
+
+		/* Gc with NOA */
+		if (wl->status.map._4way)
+			policy_type = BTC_CXP_OFFE_WL;
+		else if (bt->link_info.status.map.connect == 0)
+			policy_type = BTC_CXP_OFFE_2GISOB;
+		else
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+
+	} else {/* dedicated-antenna */
+		policy_type = BTC_CXP_OFF_EQ0;
+	}
+
+	/* Adjust E2G limit duration to avoid A2DP pop
+	 * If reach this duration from E2G start,
+	 * change slot-table form E2G to ENULL (allow BT)
+	 */
+	if (btc->bt_req_len[wl->pta_req_mac] != 0)
+		dm->e2g_slot_limit = (BTC_P2P_CYCLE * noa_dur) /
+				      btc->bt_req_len[wl->pta_req_mac];
+
+#if 0
+	dm->e2g_slot_limit += BTC_E2G_OFFSET;
+#endif
+
+	if (dm->e2g_slot_limit < BTC_E2G_LIMIT_MIN)
+		dm->e2g_slot_limit = BTC_E2G_LIMIT_MIN;
+	else if (dm->e2g_slot_limit > BTC_E2G_LIMIT_MAX)
+		dm->e2g_slot_limit = BTC_E2G_LIMIT_MAX;
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+void _action_wl_2g_nan(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	u16 policy_type = BTC_CXP_OFF_BT;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (wl->status.map._4way)
+			policy_type = BTC_CXP_OFFE_WL;
+		else if (bt->link_info.status.map.connect == 0)
+			policy_type = BTC_CXP_OFFE_2GISOB;
+		else
+			policy_type = BTC_CXP_OFFE_2GBWISOB;
+	} else { /* dedicated-antenna */
+		policy_type = BTC_CXP_OFF_EQ0;
+	}
+
+	_set_ant(btc, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(btc, policy_type, __func__);
+}
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_action.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_action.h
new file mode 100644
index 000000000000..b7990d0cdb79
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_action.h
@@ -0,0 +1,328 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __INC_BTC_ACTION_H__
+#define __INC_BTC_ACTION_H__
+
+enum btc_phy_map {
+	BTC_PHY_0 = BIT(0),
+	BTC_PHY_1 = BIT(1),
+	BTC_PHY_ALL = BIT(0) | BIT(1)
+};
+
+enum btc_bt_index {
+	BTC_BT_0 = 0x0,
+	BTC_BT_1 = 0x1,
+	BTC_ALL_BT = 0x2
+};
+
+enum btc_cx_state_map {
+	BTC_WIDLE = 0,
+	BTC_WBUSY_BNOSCAN,
+	BTC_WBUSY_BSCAN,
+	BTC_WSCAN_BNOSCAN,
+	BTC_WSCAN_BSCAN,
+	BTC_WLINKING,
+	BTC_WIDLE_BSCAN
+};
+
+enum btc_ant_phase {
+	BTC_ANT_WPOWERON = 0,
+	BTC_ANT_WINIT,
+	BTC_ANT_WONLY,
+	BTC_ANT_WOFF,
+	BTC_ANT_W2G,
+	BTC_ANT_W5G,
+	BTC_ANT_W25G,
+	BTC_ANT_FREERUN,
+	BTC_ANT_FDDTRAIN,
+	BTC_ANT_WRFK,
+	BTC_ANT_WRFK2,
+	BTC_ANT_BRFK,
+	BTC_ANT_MAX
+};
+
+enum btc_plt_map {
+	BTC_PLT_NONE = 0,
+	BTC_PLT_LTE_RX = BIT(0),
+	BTC_PLT_GNT_BT_TX = BIT(1),
+	BTC_PLT_GNT_BT_RX = BIT(2),
+	BTC_PLT_GNT_WL = BIT(3),
+	BTC_PLT_BT = BIT(1) | BIT(2),
+	BTC_PLT_ALL = 0xf
+};
+
+enum btc_cx_poicy_main_type {
+	BTC_CXP_OFF = 0,
+	BTC_CXP_OFFB,
+	BTC_CXP_OFFE,
+	BTC_CXP_FIX,
+	BTC_CXP_PFIX,
+	BTC_CXP_AUTO,
+	BTC_CXP_PAUTO,
+	BTC_CXP_AUTO2,
+	BTC_CXP_PAUTO2,
+	BTC_CXP_MANUAL,
+	BTC_CXP_USERDEF0,
+	BTC_CXP_MAIN_MAX
+};
+
+enum btc_bslot_length {
+	BTC_BSLOT_A2DP_HID = 60,
+	BTC_BSLOT_A2DP = 50,
+	BTC_BSLOT_A2DP_2 = 40,
+	BTC_BSLOT_INQ = 30,
+	BTC_BSLOT_IDLE = 20,
+};
+
+enum btc_cx_poicy_type {
+	/* TDMA off + pri: BT > WL */
+	BTC_CXP_OFF_BT = (BTC_CXP_OFF << 8) | 0,
+
+	/* TDMA off + pri: WL > BT */
+	BTC_CXP_OFF_WL = (BTC_CXP_OFF << 8) | 1,
+
+	/* TDMA off + pri: BT = WL */
+	BTC_CXP_OFF_EQ0 = (BTC_CXP_OFF << 8) | 2,
+
+	/* TDMA off + pri: BT = WL > BT_Lo */
+	BTC_CXP_OFF_EQ1 = (BTC_CXP_OFF << 8) | 3,
+
+	/* TDMA off + pri: WL = BT, BT_Rx > WL_Lo_Tx */
+	BTC_CXP_OFF_EQ2 = (BTC_CXP_OFF << 8) | 4,
+
+	/* TDMA off + pri: WL_Rx = BT, BT_HI > WL_Tx > BT_Lo */
+	BTC_CXP_OFF_EQ3 = (BTC_CXP_OFF << 8) | 5,
+
+	/* TDMA off + pri: WL_Rx = BT, BT_HI > WL_Tx > BT_Lo */
+	BTC_CXP_OFF_EQ4 = (BTC_CXP_OFF << 8) | 6,
+
+	/* TDMA off + pri: WL_Rx = BT, BT_HI > WL_Tx > BT_Lo */
+	BTC_CXP_OFF_EQ5 = (BTC_CXP_OFF << 8) | 7,
+
+	/* TDMA off + pri: BT_Hi > WL > BT_Lo */
+	BTC_CXP_OFF_BWB0 = (BTC_CXP_OFF << 8) | 8,
+
+	/* TDMA off + pri: BT_Hi_Tx = WL_Hi-Tx > BT_Hi_Rx, BT_Hi > WL > BT_Lo */
+	BTC_CXP_OFF_BWB1 = (BTC_CXP_OFF << 8) | 9,
+
+	/* TDMA off + pri: WL_Hi-Tx > BT, BT_Hi > other-WL > BT_Lo */
+	BTC_CXP_OFF_BWB2 = (BTC_CXP_OFF << 8) | 10,
+
+	/* TDMA off + pri: WL_Hi-Tx = BT */
+	BTC_CXP_OFF_BWB3 = (BTC_CXP_OFF << 8) | 11,
+
+	/* TDMA off + pri: WL > BT, Block-BT*/
+	BTC_CXP_OFF_WL2 = (BTC_CXP_OFF << 8) | 12,
+
+	/* TDMA off + pri: BT_Lo-Tx >WL_Hi-Tx = BT_Hi-Tx, BT > other-WL, */
+	BTC_CXP_OFF_BT2 = (BTC_CXP_OFF << 8) | 13,
+
+	/* TDMA off+Bcn-Protect + pri: BT_Hi > WL > BT_Lo*/
+	BTC_CXP_OFFB_BWB0 = (BTC_CXP_OFFB << 8) | 0,
+
+	/* TDMA off+Bcn-Protect + pri: BT_Hi_Tx = WL_Hi-Tx > BT_Hi_Rx, BT_Hi > WL > BT_Lo*/
+	BTC_CXP_OFFB_BWB1 = (BTC_CXP_OFFB << 8) | 1,
+
+	/* TDMA off+Bcn-Protect + pri: WL_Hi-Tx > BT, BT_Hi > other-WL > BT_Lo*/
+	BTC_CXP_OFFB_BWB2 = (BTC_CXP_OFFB << 8) | 2,
+
+	/* TDMA off+Bcn-Protect + pri: WL_Hi-Tx = BT*/
+	BTC_CXP_OFFB_BWB3 = (BTC_CXP_OFFB << 8) | 3,
+
+
+	/* TDMA off + Ext-Ctrl + pri: default */
+	BTC_CXP_OFFE_2GBWISOB = (BTC_CXP_OFFE << 8) | 0,
+
+	/* TDMA off + Ext-Ctrl + pri: E2G-slot block all BT */
+	BTC_CXP_OFFE_2GISOB = (BTC_CXP_OFFE << 8) | 1,
+
+	/* TDMA off + Ext-Ctrl + pri: E2G-slot WL > BT */
+	BTC_CXP_OFFE_2GBWMIXB = (BTC_CXP_OFFE << 8) | 2,
+
+	/* TDMA off + Ext-Ctrl + pri: E2G/EBT-slot WL > BT */
+	BTC_CXP_OFFE_WL = (BTC_CXP_OFFE << 8) | 3,
+
+	/* TDMA off + Ext-Ctrl + pri: default */
+	BTC_CXP_OFFE_2GBWMIXB2 = (BTC_CXP_OFFE << 8) | 4,
+
+	/* TDMA Fix slot-0: W1:B1 = 30:30 */
+	BTC_CXP_FIX_TD3030 = (BTC_CXP_FIX << 8) | 0,
+
+	/* TDMA Fix slot-1: W1:B1 = 50:50 */
+	BTC_CXP_FIX_TD5050 = (BTC_CXP_FIX << 8) | 1,
+
+	/* TDMA Fix slot-2: W1:B1 = 20:30 */
+	BTC_CXP_FIX_TD2030 = (BTC_CXP_FIX << 8) | 2,
+
+	/* TDMA Fix slot-3: W1:B1 = 40:10 */
+	BTC_CXP_FIX_TD4010 = (BTC_CXP_FIX << 8) | 3,
+
+	/* TDMA Fix slot-4: W1:B1 = 70:10 */
+	BTC_CXP_FIX_TD7010 = (BTC_CXP_FIX << 8) | 4,
+
+	/* TDMA Fix slot-5: W1:B1 = 20:60 */
+	BTC_CXP_FIX_TD2060 = (BTC_CXP_FIX << 8) | 5,
+
+	/* TDMA Fix slot-6: W1:B1 = 30:60 */
+	BTC_CXP_FIX_TD3060 = (BTC_CXP_FIX << 8) | 6,
+
+	/* TDMA Fix slot-7: W1:B1 = 20:80 */
+	BTC_CXP_FIX_TD2080 = (BTC_CXP_FIX << 8) | 7,
+
+	/* TDMA Fix slot-8: W1:B1 = user-define */
+	BTC_CXP_FIX_TDW1B1 = (BTC_CXP_FIX << 8) | 8,
+
+	/* TDMA Fix slot-9: W1:B1 = 40:10 */
+	BTC_CXP_FIX_TD4010ISO = (BTC_CXP_FIX << 8) | 9,
+
+	/* TDMA Fix slot-10: W1:B1 = 40:10 */
+	BTC_CXP_FIX_TD4010ISO_DL = (BTC_CXP_FIX << 8) | 10,
+
+	/* TDMA Fix slot-11: W1:B1 = 70:10 */
+	BTC_CXP_FIX_TD7010ISO_UL = (BTC_CXP_FIX << 8) | 11,
+
+	/* PS-TDMA Fix slot-0: W1:B1 = 30:30 */
+	BTC_CXP_PFIX_TD3030 = (BTC_CXP_PFIX << 8) | 0,
+
+	/* PS-TDMA Fix slot-1: W1:B1 = 50:50 */
+	BTC_CXP_PFIX_TD5050 = (BTC_CXP_PFIX << 8) | 1,
+
+	/* PS-TDMA Fix slot-2: W1:B1 = 20:30 */
+	BTC_CXP_PFIX_TD2030 = (BTC_CXP_PFIX << 8) | 2,
+
+	/* PS-TDMA Fix slot-3: W1:B1 = 20:60 */
+	BTC_CXP_PFIX_TD2060 = (BTC_CXP_PFIX << 8) | 3,
+
+	/* PS-TDMA Fix slot-4: W1:B1 = 30:70 */
+	BTC_CXP_PFIX_TD3070 = (BTC_CXP_PFIX << 8) | 4,
+
+	/* PS-TDMA Fix slot-5: W1:B1 = 20:80 */
+	BTC_CXP_PFIX_TD2080 = (BTC_CXP_PFIX << 8) | 5,
+
+	/* PS-TDMA Fix slot-6: W1:B1 = user-define */
+	BTC_CXP_PFIX_TDW1B1 = (BTC_CXP_PFIX << 8) | 6,
+
+	/* TDMA Auto slot-0: W1:B1 = 50:BTC_B1_MAX */
+	BTC_CXP_AUTO_TD50B1 = (BTC_CXP_AUTO << 8) | 0,
+
+	/* TDMA Auto slot-1: W1:B1 = 60:BTC_B1_MAX */
+	BTC_CXP_AUTO_TD60B1 = (BTC_CXP_AUTO << 8) | 1,
+
+	/* TDMA Auto slot-2: W1:B1 = 20:BTC_B1_MAX */
+	BTC_CXP_AUTO_TD20B1 = (BTC_CXP_AUTO << 8) | 2,
+
+	/* TDMA Auto slot-3: W1:B1 = user-define */
+	BTC_CXP_AUTO_TDW1B1 = (BTC_CXP_AUTO << 8) | 3,
+
+	/* TDMA Auto slot-4: W1:B1 = user-define */
+	BTC_CXP_AUTO_TDW1B1_CXTPS = (BTC_CXP_AUTO << 8) | 4,
+
+	/* TDMA Auto slot-5: W1:B1:B2 = user-define */
+	BTC_CXP_AUTO_TDW1B1_LESCAN = (BTC_CXP_AUTO << 8) | 5,
+
+	/* PS-TDMA Auto slot-0: W1:B1 = 50:BTC_B1_MAX */
+	BTC_CXP_PAUTO_TD50B1 = (BTC_CXP_PAUTO << 8) | 0,
+
+	/* PS-TDMA Auto slot-1: W1:B1 = 60:BTC_B1_MAX */
+	BTC_CXP_PAUTO_TD60B1 = (BTC_CXP_PAUTO << 8) | 1,
+
+	/* PS-TDMA Auto slot-2: W1:B1 = 20:BTC_B1_MAX */
+	BTC_CXP_PAUTO_TD20B1 = (BTC_CXP_PAUTO << 8) | 2,
+
+	/* PS-TDMA Auto slot-3: W1:B1 = user-define */
+	BTC_CXP_PAUTO_TDW1B1 = (BTC_CXP_PAUTO << 8) | 3,
+
+	/* PS-TDMA Auto slot-5: W1:B1 = user-define for FDD-Train default */
+	BTC_CXP_PAUTO_FDDT1 = (BTC_CXP_PAUTO << 8) | 4,
+
+	/* PS-TDMA Auto slot-6: W1:B1:B2 = user-define */
+	BTC_CXP_PAUTO_TDW1B1_LESCAN = (BTC_CXP_PAUTO << 8) | 5,
+
+	/* TDMA Auto slot2-0: W1:B4 = 30:50 */
+	BTC_CXP_AUTO2_TD3050 = (BTC_CXP_AUTO2 << 8) | 0,
+
+	/* TDMA Auto slot2-1: W1:B4 = 30:70 */
+	BTC_CXP_AUTO2_TD3070 = (BTC_CXP_AUTO2 << 8) | 1,
+
+	/* TDMA Auto slot2-2: W1:B4 = 50:50 */
+	BTC_CXP_AUTO2_TD5050 = (BTC_CXP_AUTO2 << 8) | 2,
+
+	/* TDMA Auto slot2-3: W1:B4 = 60:60 */
+	BTC_CXP_AUTO2_TD6060 = (BTC_CXP_AUTO2 << 8) | 3,
+
+	/* TDMA Auto slot2-4: W1:B4 = 20:80 */
+	BTC_CXP_AUTO2_TD2080 = (BTC_CXP_AUTO2 << 8) | 4,
+
+	/* TDMA Auto slot2-5: W1:B4 = user-define */
+	BTC_CXP_AUTO2_TDW1B4 = (BTC_CXP_AUTO2 << 8) | 5,
+
+	/* PS-TDMA Auto slot2-0: W1:B4 = 30:50 */
+	BTC_CXP_PAUTO2_TD3050 = (BTC_CXP_PAUTO2 << 8) | 0,
+
+	/* PS-TDMA Auto slot2-1: W1:B4 = 30:70 */
+	BTC_CXP_PAUTO2_TD3070 = (BTC_CXP_PAUTO2 << 8) | 1,
+
+	/* PS-TDMA Auto slot2-2: W1:B4 = 50:50 */
+	BTC_CXP_PAUTO2_TD5050 = (BTC_CXP_PAUTO2 << 8) | 2,
+
+	/* PS-TDMA Auto slot2-3: W1:B4 = 60:60 */
+	BTC_CXP_PAUTO2_TD6060 = (BTC_CXP_PAUTO2 << 8) | 3,
+
+	/* PS-TDMA Auto slot2-4: W1:B4 = 20:80 */
+	BTC_CXP_PAUTO2_TD2080 = (BTC_CXP_PAUTO2 << 8) | 4,
+
+	/* PS-TDMA Auto slot2-5: W1:B4 = user-define */
+	BTC_CXP_PAUTO2_TDW1B4 = (BTC_CXP_PAUTO2 << 8) | 5,
+
+	BTC_CXP_MAX = 0xffff
+};
+
+void _action_wl_25g_mcc(struct btc_t *btc);
+void _action_wl_2g_ap(struct btc_t *btc);
+void _action_wl_2g_go(struct btc_t *btc);
+void _action_wl_2g_gc(struct btc_t *btc);
+void _action_wl_2g_nan(struct btc_t *btc);
+void _action_wl_2g_scc(struct btc_t *btc);
+void _action_wl_2g_sta(struct btc_t *btc);
+void _action_wl_scan(struct btc_t *btc);
+void _action_wl_2g_mcc(struct btc_t *btc);
+void _action_wl_5g(struct btc_t *btc);
+void _action_wl_other(struct btc_t *btc);
+void _action_wl_idle(struct btc_t *btc);
+void _action_wl_nc(struct btc_t *btc);
+void _action_wl_rfk(struct btc_t *btc);
+void _action_wl_init(struct btc_t *btc);
+void _action_wl_off(struct btc_t *btc);
+void _action_freerun(struct btc_t *btc);
+void _action_wl_only(struct btc_t *btc);
+void _action_bt_off(struct btc_t *btc);
+void _action_bt_rfk(struct btc_t *btc);
+void _action_bt_whql(struct btc_t *btc);
+void _action_common(struct btc_t *btc);
+//extern const u32 cxtbl[];
+
+void _set_bt_ignore_wl_act(struct btc_t *btc, u8 enable);
+void _set_wl_tx_power(struct btc_t *btc, u32 level);
+void _set_wl_rx_gain(struct btc_t *btc, u32 level);
+void _set_bt_tx_power(struct btc_t *btc, u32 level);
+void _set_bt_rx_gain(struct btc_t *btc, u32 level);
+void _set_gnt(struct btc_t *btc, u8 phy_map, u8 wl_state, u8 bt_state,
+	      u8 wlact_state);
+
+void _set_fddt_ctrl(struct btc_t *btc, bool force_exec);
+
+extern const struct fbtc_tdma t_def[];
+extern const struct fbtc_slot s_def[];
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_dbg_cmd.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_dbg_cmd.c
new file mode 100644
index 000000000000..1fa4a8889675
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_dbg_cmd.c
@@ -0,0 +1,3861 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_BTC_DBG_CMD_C_
+#include "../hal_headers_le.h"
+#include "hal_btc.h"
+#include "halbtc_fw.h"
+#include "halbtc_fwdef.h"
+#include "halbtc_action.h"
+#include "halbtc_def.h"
+
+#ifdef CONFIG_BTCOEX
+
+#define	CLI_PRT(fmt, ...)\
+	do {							\
+		if (*used < out_len)			\
+			*used += _os_snprintf(output + *used,	\
+					      out_len - *used, fmt,\
+					      ##__VA_ARGS__);\
+	} while (0)
+
+struct halbtc_cmd_info {
+	char name[16];
+	u8 id;
+};
+
+enum HALBTC_CMD_ID {
+	HALBTC_HELP,
+	HALBTC_SHOW,
+	HALBTC_READ_BT,
+	HALBTC_WRITE_BT,
+	HALBTC_SET_COEX,
+	HALBTC_UPDATE_POLICY,
+	HALBTC_TDMA,
+	HALBTC_SLOT,
+	HALBTC_SIG_GDBG_EN,
+	HALBTC_SGPIO_MAP,
+	HALBTC_TRACE_STEP,
+	HALBTC_WL_TX_POWER,
+	HALBTC_WL_RX_LNA,
+	HALBTC_BT_AFH_MAP,
+	HALBTC_BT_TX_POWER,
+	HALBTC_BT_RX_LNA,
+	HALBTC_BT_IGNO_WLAN,
+	HALBTC_SET_GNT_WL,
+	HALBTC_SET_GNT_BT,
+	HALBTC_SET_BT_PSD,
+	HALBTC_GET_WL_NHM_DBM,
+	HALBTC_DBG,
+#ifdef BTC_AISO_SUPPORT
+	HALBTC_AISO,
+#endif
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	HALBTC_FT_SET,
+	HALBTC_FT_SHOW,
+	HALBTC_FT_CTRL
+#endif
+};
+
+enum btc_fddt_cell_type {
+	BTC_FDDT_CELL_RANGE_UL = BIT(TRAFFIC_UL),
+	BTC_FDDT_CELL_RANGE_DL = BIT(TRAFFIC_DL),
+	BTC_FDDT_CELL_STATE_UL = 0x80 + BIT(TRAFFIC_UL),
+	BTC_FDDT_CELL_STATE_DL = 0x80 + BIT(TRAFFIC_DL)
+};
+
+struct halbtc_cmd_info halbtc_cmd_i[] = {
+	{"-h", HALBTC_HELP},
+	{"show", HALBTC_SHOW},
+	{"rb", HALBTC_READ_BT},
+	{"wb", HALBTC_WRITE_BT},
+	{"mode", HALBTC_SET_COEX},
+	{"update", HALBTC_UPDATE_POLICY},
+	{"tdma", HALBTC_TDMA},
+	{"slot", HALBTC_SLOT},
+	{"sig", HALBTC_SIG_GDBG_EN},
+	{"gpio", HALBTC_SGPIO_MAP},
+	{"wpwr", HALBTC_WL_TX_POWER},
+	{"wlna", HALBTC_WL_RX_LNA},
+	{"afh", HALBTC_BT_AFH_MAP},
+	{"bpwr", HALBTC_BT_TX_POWER},
+	{"blna", HALBTC_BT_RX_LNA},
+	{"igwl", HALBTC_BT_IGNO_WLAN},
+	{"gwl", HALBTC_SET_GNT_WL},
+	{"gbt", HALBTC_SET_GNT_BT},
+	{"bpsd", HALBTC_SET_BT_PSD},
+	{"wnhm", HALBTC_GET_WL_NHM_DBM},
+	{"dbg", HALBTC_DBG},
+#ifdef BTC_AISO_SUPPORT
+	{"aiso", HALBTC_AISO},
+#endif
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	{"ftset", HALBTC_FT_SET},
+	{"ftshow", HALBTC_FT_SHOW},
+	{"ftctrl", HALBTC_FT_CTRL},
+#endif
+};
+
+#define _limit_val(val, max, min) \
+	val = (val > max ? max: (val < min ? min : val))
+
+static void _write_bt_reg(struct btc_t *btc, u8 reg_type, u16 addr, u32 val)
+{
+	u8 buf[4] = {0};
+
+	/* set write address */
+	buf[0] = reg_type;
+	buf[1] = addr & bMASKB0;
+	buf[2] = (addr & bMASKB1) >> 8;
+	_send_fw_cmd(btc, SET_BT_WREG_ADDR, buf, 3);
+
+	/* set write value */
+	buf[0] = val & bMASKB0;
+	buf[1] = (val & bMASKB1) >> 8;
+	buf[2] = (val & bMASKB2) >> 16;
+	buf[3] = (val & bMASKB3) >> 24;
+	_send_fw_cmd(btc, SET_BT_WREG_VAL, buf, 4);
+}
+
+static void _read_bt_reg(struct btc_t *btc, u8 reg_type, u16 addr)
+{
+	/* this function is only for API call.
+	 * If BTC should use hal_btc_fw_set_monreg to read bt reg.
+	 */
+	u8 buf[3] = {0};
+
+	/* set write address */
+	buf[0] = reg_type;
+	buf[1] = addr & bMASKB0;
+	buf[2] = (addr & bMASKB1) >> 8;
+	_send_fw_cmd(btc, SET_BT_RREG_ADDR, buf, 3);
+
+	/* To do wait FW event -> BTF_EVNT_BT_REG*/
+}
+
+static void _cmd_rb(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+		    u32 input_num, char *output, u32 out_len)
+{
+	u32 i = 0, type = 0, addr = 0;
+
+	if (input_num >= 3) {
+		_os_sscanf(input[1], "%d", &type);
+		_os_sscanf(input[2], "%x", &addr);
+	}
+
+	if (type > 4 || input_num < 3) {
+		CLI_PRT(" rb <type --- 0:rf, 1:modem, 2:bluewize, 3:vendor,"
+			" 4:LE> <addr:16bits> \n");
+		return;
+	}
+
+	btc->dbg.rb_done = false;
+	btc->dbg.rb_val = 0xffffffff;
+	_read_bt_reg(btc, (u8)(type), (u16)addr);
+	for (i = 0; i < 50; i++) {
+		if (!btc->dbg.rb_done)
+			hal_mdelay(btc->hal, 10);
+		else
+			break;
+	}
+
+	if (!btc->dbg.rb_done) {
+		CLI_PRT(" timeout !!\n");
+	} else {
+		CLI_PRT(" rb %d(0x%x), val=0x%x\n",
+			type, addr, btc->dbg.rb_val);
+	}
+}
+
+static void _show_cx_info(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			   u32 input_num, char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct rtw_phl_com_t *p = btc->phl;
+	struct btc_module *md = &btc->mdinfo;
+	u32 ver_main = 0, ver_sub = 0, ver_hotfix = 0, id_branch = 0;
+
+	if (!(dm->coex_info_map & BTC_COEX_INFO_CX))
+		return;
+
+	dm->cnt_notify[BTC_NTFY_SHOW_COEX_INFO]++;
+
+	CLI_PRT("\n\r========== [BTC COEX INFO (%s%s%s%s%s)] ==========",
+		id_to_str(BTC_STR_CHIPID, (u32)btc->hal->chip_id),
+		(btc->hal->chip_id == CHIP_WIFI6_8852B &&
+		md->ant.num == 1 ? "-VS" : ""),
+		(p->hci_type & RTW_HCI_PCIE ?  " PCIe" : ""),
+		(p->hci_type & RTW_HCI_USB ? " USB" : ""),
+		(p->hci_type & RTW_HCI_SDIO ? " SDIO" : ""));
+
+	ver_main = (coex_ver & bMASKB3) >> 24;
+	ver_sub = (coex_ver & bMASKB2) >> 16;
+	ver_hotfix = (coex_ver & bMASKB1) >> 8;
+	id_branch = coex_ver & bMASKB0;
+	CLI_PRT("\n\r %-15s : Coex:%d.%d.%d(branch:%s_%d), ", "[coex_version]",
+		ver_main, ver_sub, ver_hotfix,
+		id_to_str(BTC_STR_BRANCH, BTC_BRANCH), id_branch);
+
+	ver_main = (wl->ver_info.fw_coex & bMASKB3) >> 24;
+	ver_sub = (wl->ver_info.fw_coex & bMASKB2) >> 16;
+	ver_hotfix = (wl->ver_info.fw_coex & bMASKB1) >> 8;
+	id_branch = wl->ver_info.fw_coex & bMASKB0;
+
+	CLI_PRT("WL_FW_coex:%d.%d.%d(branch:%d)",
+		ver_main, ver_sub, ver_hotfix, id_branch);
+
+	ver_main = (btc->chip->wlcx_desired & bMASKB3) >> 24;
+	ver_sub = (btc->chip->wlcx_desired & bMASKB2) >> 16;
+	ver_hotfix = (btc->chip->wlcx_desired & bMASKB1) >> 8;
+
+	CLI_PRT("(%s, desired:%d.%d.%d), ",
+		(wl->fw_ver_mismatch ? "Mis-Match" : "Match"),
+		 ver_main, ver_sub, ver_hotfix);
+
+	CLI_PRT("BT_FW_coex:%d(%s, desired:%d)",
+		bt->ver_info.fw_coex,
+		(bt->fw_ver_mismatch ? "Mis-Match" : "Match"),
+		btc->chip->btcx_desired);
+
+	CLI_PRT("\n\r %-15s : WL_FW:%d.%d.%d.%d, BT_FW:0x%x(%s)",
+		"[sub_module]", (wl->ver_info.fw & bMASKB3) >> 24,
+		(wl->ver_info.fw & bMASKB2) >> 16,
+		(wl->ver_info.fw & bMASKB1) >> 8,
+		(wl->ver_info.fw & bMASKB0), bt->ver_info.fw,
+		(bt->run_patch_code ? "patch" : "ROM"));
+
+	CLI_PRT("\n\r %-15s : kt_ver:%x,"
+		" rfe_type:0x%x(ant_cnt:%d/ext_sw:%d/ant_div:%d), vendor:%s,"
+		 " hw_feat:0x%x, 3rd_coex:%d,"
+		 " endian:%s", "[hw_info]", md->kt_ver,
+		 md->rfe_type, md->ant.num, md->switch_type, md->ant.diversity,
+		id_to_str(BTC_STR_VENDORID, (u32)dm->vid),
+		btc->chip->hw, cx->other.type,
+		BTC_PLATFORM_BIG_ENDIAN ? "big" : "little");
+
+	if (md->ant.type == BTC_ANT_SHARED)
+		return;
+
+#ifdef BTC_AISO_SUPPORT
+		if (btc->bt_psd_dm.aiso_db_cnt > 0)
+			CLI_PRT(" ,ant_iso:%ddB(cal)", md->ant.isolation);
+		else
+#endif
+		CLI_PRT(" ,ant_iso:%ddB(def)", md->ant.isolation);
+
+	CLI_PRT("\n\r %-15s : soc:%s, func:%s, intf:%s, trx_req:%s",
+		"[ext_bt]",
+		id_to_str(BTC_STR_ESOC, md->bt_solo),
+		id_to_str(BTC_STR_ESOC_FUNC, cx->other.type),
+		id_to_str(BTC_STR_ESOC_INTF, cx->other.hw_coex),
+		(cx->other.ext_pta_req_exist? "Y" : "N"));
+}
+
+static void _show_wl_role_info(struct btc_t *btc, u32 *used,
+				char input[][MAX_ARGV], u32 input_num,
+				char *output, u32 out_len)
+{
+	struct btc_wl_link_info *plink = NULL;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_dbcc_info *wd = &wl->dbcc_info;
+	u8 i;
+	u32 sp_ap = wl->ap_type;
+
+	if (wl_rinfo->dbcc_en) {
+		CLI_PRT("\n\r %-15s : 2G_PHY:%s, chg_cnt:%d, all_2G_cnt:%d",
+			"[dbcc_info]",
+			(wl_rinfo->dbcc_2g_phy == HW_PHY_MAX ? "None" :
+			(wl_rinfo->dbcc_2g_phy == HW_PHY_0 ? "PHY0" : "PHY1")),
+			btc->cx.cnt_wl[BTC_WCNT_DBCC_CHG],
+			btc->cx.cnt_wl[BTC_WCNT_DBCC_ALL_2G]);
+
+		for (i = HW_PHY_0; i < HW_PHY_MAX; i++) {
+			CLI_PRT(", PHY-%d(op:%s/scan:%s/real:%s/role:0x%x)", i,
+				id_to_str(BTC_STR_BAND, (u32)wd->op_band[i]),
+				id_to_str(BTC_STR_BAND, (u32)wd->scan_band[i]),
+				id_to_str(BTC_STR_BAND, (u32)wd->real_band[i]),
+				wd->role[i]);
+		}
+	}
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++) {
+		plink = &wl->link_info[i];
+
+		if (!plink->active && plink->connected == MLME_NO_LINK)
+			continue;
+
+		CLI_PRT("\n\r [Role%d-HwB%d-p%d]  : %s(%s/%s/%s)",
+			i, plink->phy, plink->pid,
+			id_to_str(BTC_STR_ROLE, (u32)plink->role),
+			(plink->active? "Active" : "Doze"), 
+			id_to_str(BTC_STR_MSTATE, plink->connected),
+			id_to_str(BTC_STR_WLMODE, (u32)plink->mode));
+
+		if (plink->connected == MLME_NO_LINK)
+			continue;
+
+		CLI_PRT(", %s(ct:%d/pr:%d/bw:%s), ",
+			id_to_str(BTC_STR_BAND, plink->chdef.band),
+			plink->chdef.center_ch, plink->chdef.chan,
+			id_to_str(BTC_STR_WLBW, (u32)plink->chdef.bw));
+
+		if (plink->role == PHL_RTYPE_STATION && sp_ap)
+			CLI_PRT("AP:%s, ",
+				id_to_str(BTC_STR_SPAP, sp_ap));
+
+		if (!plink->active)
+			continue;
+
+		CLI_PRT("rssi:-%ddBm(%dp), rate(tx:%s/rx:%s), ",
+			110-plink->rssi, plink->rssi,
+			id_to_str(BTC_STR_RATE, (u32)plink->tx_rate),
+			id_to_str(BTC_STR_RATE, (u32)plink->rx_rate));
+
+		if (plink->role == PHL_RTYPE_AP ||
+		    plink->role == PHL_RTYPE_P2P_GO)
+			CLI_PRT("client(cnt:%d/Lps:%d/Tdma:%d)",
+				plink->client_cnt-1,
+				plink->client_ps,
+				btc->dm.client_ps_tdma_on);
+		else if (plink->role == PHL_RTYPE_STATION ||
+			 plink->role == PHL_RTYPE_P2P_GC)
+			CLI_PRT("mID:%d, bID:0x%02x%02x%02x%02x%02x%02x",
+				plink->mac_id,
+				plink->mac_addr[0], plink->mac_addr[1],
+				plink->mac_addr[2], plink->mac_addr[3],
+				plink->mac_addr[4], plink->mac_addr[5]);
+
+		if (plink->role == PHL_RTYPE_AP ||
+		    plink->role == PHL_RTYPE_P2P_GO ||
+		    plink->role == PHL_RTYPE_P2P_GC)
+			CLI_PRT(", noa(%s):%d.%03dms",
+				(plink->noa ? "on" : "off"),
+				plink->noa_duration / 1000,
+				plink->noa_duration % 1000);
+	}
+}
+
+static void _show_wl_info(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			  u32 input_num, char *output, u32 out_len)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct rtw_phl_com_t *p = btc->phl;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	struct btc_module *md = &btc->mdinfo;
+	struct btc_traffic *wl_tra = &btc->cx.wl.traffic;
+	struct btc_chip_ops *ops = btc->chip->ops;
+	char *s_tx = "", *s_rx = "", *s_1ant = "";
+	u8 pos = BTC_ANT_DIV_MAIN;
+
+	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_WL))
+		return;
+
+	CLI_PRT("\n\r========== [WL Status%s] ==========",
+		(p->drv_mode == RTW_DRV_MODE_MP ? "(MP-mode)" : ""));
+
+	CLI_PRT("\n\r %-15s : link_mode:%s(%s%s/%s(0x%x)), mrole:%s,"
+		" dbcc_en:%d, ", "[status]",
+		id_to_str(BTC_STR_WLLINK, (u32)wl_rinfo->link_mode),
+		(wl_smap->busy == 1 ? "busy" : "idle"),
+		(wl->busy_to_idle == 1? "->idle" : ""),
+		(wl_smap->traffic_dir & BIT(TRAFFIC_UL) ? "UL" : "DL"),
+		wl_smap->traffic_dir,
+		id_to_str(BTC_STR_MROLE, wl_rinfo->mrole_type),
+		wl->role_info.dbcc_en);
+
+	CLI_PRT("rf_off:%d, LPS:%s, nhm:%d(max:%d/min:%d), cn:%d",
+		wl_smap->rf_off,
+		(wl_smap->lps == BTC_LPS_OFF? "0":
+		(wl_smap->lps == BTC_LPS_RF_OFF? "1(RF-Off)":"1(RF-on)")),
+		wl->nhm.pwr, wl->nhm.pwr_max, wl->nhm.pwr_min, wl->cn_report);
+
+	CLI_PRT("\n\r %-15s : scan:%d(phy_map:0x%x),"
+		" connecting:%d, roam:%d, _4way:%d, dbccing:%d, cx_state:%s",
+		"[scan]", wl_smap->scan,
+		wl->scan_info.phy_map,
+		wl_smap->connecting, wl_smap->roaming, wl_smap->_4way,
+		wl_smap->dbccing, id_to_str(BTC_STR_CXSTATE, cx->state_map));
+
+	if (ops && ops->get_reg_status)
+		ops->get_reg_status(btc, BTC_CSTATUS_TXDIV_POS, (void*)&pos);
+	s_tx = pos ? "/pos:aux" : "/pos:main";
+
+	if (ops && ops->get_reg_status)
+		ops->get_reg_status(btc, BTC_CSTATUS_RXDIV_POS, (void*)&pos);
+	s_rx = pos ? "/pos:aux" : "/pos:main";
+
+	s_1ant = md->ant.single_pos? "(s1)" : "(s0)";
+
+	CLI_PRT("\n\r %-15s : tx[num:%d/ss:%d%s%s/path:%s], rx[num:%d/ss:%d%s%s/path:%s]",
+		"[ant_stream]",
+		p->phy_cap[0].tx_path_num, p->phy_cap[0].txss,
+		(p->phy_cap[0].txss != 1? "" : s_1ant), s_tx,
+		id_to_str(BTC_STR_PATH, (u32)(md->ant.path_pos & 0xf0)>> 4),
+		p->phy_cap[0].rx_path_num, p->phy_cap[0].rxss,
+		(p->phy_cap[0].rxss != 1? "" : s_1ant), s_rx,
+		id_to_str(BTC_STR_PATH, (u32)md->ant.path_pos & 0xf));
+
+	if (wl->role_info.dbcc_en)
+		CLI_PRT(", tx1[num:%d/ss:%d], rx1[num:%d/ss:%d]",
+			p->phy_cap[1].tx_path_num, p->phy_cap[1].txss,
+			p->phy_cap[1].rx_path_num, p->phy_cap[1].rxss);
+
+	CLI_PRT("\n\r %-15s : tx[tp:%dMbps/lvl:%d], rx[tp:%dMbps/lvl:%d], "
+		"evm(1ss:%d/2ss_max:%d/2ss_min:%d)]", "[trx_stat]",
+		wl_tra->tx_tp, wl_tra->tx_lvl, wl_tra->rx_tp, wl_tra->rx_lvl,
+		wl->evm_1ss_rpt, wl->evm_2ss_max_rpt, wl->evm_2ss_min_rpt);
+
+	_show_wl_role_info(btc, used, input, input_num, output, out_len);
+}
+
+static void _show_dm_step(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			  u32 input_num, char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	u32 n_begin = 0, n_end = 0, i = 0, cnt = 0;
+	u32 step_cnt = dm->dm_step.cnt;
+
+	if (step_cnt == 0)
+		return;
+
+	if (step_cnt <= BTC_DM_MAXSTEP)
+		n_begin = 1;
+	else
+		n_begin = step_cnt - BTC_DM_MAXSTEP + 1;
+
+	n_end = step_cnt;
+
+	if (n_begin > n_end)
+		return;
+
+	for (i = n_begin; i <= n_end; i++) {
+		if (cnt % 6 == 0)
+			CLI_PRT("\n\r %-15s : ", "[dm_steps]");
+
+		CLI_PRT("-> %s", dm->dm_step.step[(i-1) % BTC_DM_MAXSTEP]);
+		cnt++;
+	}
+}
+
+static void _show_bt_profile_info(struct btc_t *btc, u32 *used,
+				  char input[][MAX_ARGV], u32 input_num,
+				  char *output, u32 out_len)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_link_info *bt_linfo = &btc->cx.bt.link_info;
+	struct btc_bt_hfp_desc *hfp = &bt_linfo->hfp_desc;
+	struct btc_bt_hid_desc *hid = &bt_linfo->hid_desc;
+	struct btc_bt_a2dp_desc *a2dp = &bt_linfo->a2dp_desc;
+	struct btc_bt_pan_desc *pan = &bt_linfo->pan_desc;
+	struct btc_bt_leaudio_desc *leaudio = &bt_linfo->leaudio_desc;
+	u8 i;
+
+	if (hfp->exist) {
+#if 0
+		CLI_PRT("\n\r %-15s : type:%s", "[HFP]");
+#endif
+	}
+
+	if (hid->exist) {
+		CLI_PRT("\n\r %-15s : pair-cnt:%d", "[HID]", hid->cnt);
+
+		for (i = 0; i < hid->cnt; i++) {
+			if (!hid->links[i].update)
+				continue;
+
+			CLI_PRT(", Dev-0x%x[v-id:0x%x/%s/%s/slot:%d-%d]",
+				hid->links[i].h, hid->links[i].vendor_id,
+				(hid->links[i].type? "BLE" : "EDR"),
+				(hid->links[i].role? "Master" : "Slave"),
+				hid->links[i].window, hid->links[i].interval);
+		}
+	}
+
+	if (a2dp->exist) {
+		CLI_PRT("\n\r %-15s : pair-cnt:%d, vendor-id:0x%x,"
+			" flush-time:%d", "[A2DP]",
+			a2dp->cnt, a2dp->vendor_id, a2dp->flush_time);
+
+		for (i = 0; i < a2dp->cnt; i++) {
+			if (!a2dp->links[i].update)
+				continue;
+
+			CLI_PRT(", Dev-0x%x[%s/%s]",
+				a2dp->links[i].h,
+				(a2dp->links[i].type? "BLE" : "EDR"),
+				(a2dp->links[i].role? "Master" : "Slave"));
+		}
+	}
+
+	if (pan->exist) {
+		CLI_PRT("\n\r %-15s : pair-cnt:%d", "[PAN]", pan->cnt);
+
+		for (i = 0; i < pan->cnt; i++) {
+			if (!pan->links[i].update)
+				continue;
+
+			CLI_PRT(", Dev-0x%x[%s/%s/%s]",
+				pan->links[i].h,
+				(pan->links[i].type? "BLE" : "EDR"),
+				(pan->links[i].role? "Master" : "Slave"),
+				(pan->links[i].direct? "Tx" : "Rx"));
+		}
+	}
+
+	if (leaudio->exist) {
+		CLI_PRT("\n\r %-15s : rssi:%d, BIS:%d, CIS:%d (Update:%d)",
+			"[LEAUDIO]", leaudio->rssi - 100,
+			leaudio->bis_cnt, leaudio->cis_cnt,
+			cx->cnt_bt[BTC_BCNT_LEAUDIO_INFOUPDATE]);
+	}
+}
+
+static void _show_bt_info(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			  u32 input_num, char *output, u32 out_len)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_module *module = &btc->mdinfo;
+	struct btc_bt_link_info *bt_linfo = &bt->link_info;
+	u8 *afh = bt_linfo->afh_map;
+	u8 *afh_le = bt_linfo->afh_map_le;
+	u32 rpt_map = btc->fwinfo.rpt_en_map;
+
+	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_BT))
+		return;
+
+	CLI_PRT("%s", "\n\r========== [BT Status] ==========");
+
+	CLI_PRT("\n\r %-15s : enable:%d, btg:%d%s, connect:%d, ",
+		"[status]", bt->enable.now, bt->btg_type,
+		(bt->enable.now && (bt->btg_type != module->bt_pos)?
+		"(efuse-mismatch!!)" : ""), bt_linfo->status.map.connect);
+
+	CLI_PRT("rssi:%ddBm(lvl:%d), tx_rate:%dM, ",
+		bt_linfo->rssi-100, bt->rssi_level, (bt_linfo->tx_3M? 3 : 2));
+
+	CLI_PRT("igno_wl:%d, mailbox_avl:%d, rfk_state:0x%x",
+		bt->igno_wl, bt->mbx_avl, bt->rfk_info.val);
+
+	CLI_PRT("\n\r %-15s : profile:%s%s%s%s%s%s", "[profile]",
+		((bt_linfo->profile_cnt.now == 0) ? "None," : ""),
+		(bt_linfo->hfp_desc.exist? "HFP," : ""),
+		(bt_linfo->hid_desc.exist? "HID," : ""),
+		(bt_linfo->a2dp_desc.exist?
+		(bt_linfo->a2dp_desc.sink ? "A2DP_sink," :"A2DP,") : ""),
+		(bt_linfo->pan_desc.exist? "PAN," : ""),
+		(bt_linfo->leaudio_desc.exist? "LEAUDIO," : ""));
+
+	CLI_PRT(" multi-link:%d, role:%s, ble-connect:%d, CQDDR:%d,"
+		" A2DP_active:%d, PAN_active:%d",
+		bt_linfo->multi_link.now,
+		(bt_linfo->slave_role ? "Slave" : "Master"),
+		bt_linfo->status.map.ble_connect,
+		bt_linfo->cqddr,
+		bt_linfo->a2dp_desc.active,
+		bt_linfo->pan_desc.active);
+
+	CLI_PRT("\n\r %-15s : wl_ch_map[en:%d/ch:%d/bw:%d]", "[AFH]",
+		wl->afh_info.en, wl->afh_info.ch, wl->afh_info.bw);
+
+	CLI_PRT(", Legacy[%02x%02x_%02x%02x_%02x%02x_%02x%02x_%02x%02x]"
+		"(chs:%d/co_ch:%d/cnt:%d)",
+		afh[0], afh[1], afh[2], afh[3], afh[4],
+		afh[5], afh[6], afh[7], afh[8], afh[9],
+		cx->cnt_bt[BTC_BCNT_AFH_CHN],
+		cx->cnt_bt[BTC_BCNT_AFH_CONFLICT],
+		cx->cnt_bt[BTC_BCNT_AFH_UPDATE]);
+
+	if (bt_linfo->status.map.ble_connect || bt_linfo->leaudio_desc.exist) {
+		CLI_PRT(", LE[%02x%02x_%02x_%02x%02x](chs:%d/co_ch:%d/cnt:%d)",
+			afh_le[0], afh_le[1], afh_le[2], afh_le[3], afh_le[4],
+			cx->cnt_bt[BTC_BCNT_AFH_LE_CHN],
+			cx->cnt_bt[BTC_BCNT_AFH_LE_CONFLICT],
+			cx->cnt_bt[BTC_BCNT_AFH_LE_UPDATE]);
+	}
+
+	CLI_PRT("\n\r %-15s : retry:%d, relink:%d, rate_chg:%d,"
+		" reinit:%d, reenable:%d, ", "[stat_cnt]",
+		cx->cnt_bt[BTC_BCNT_RETRY],
+		cx->cnt_bt[BTC_BCNT_RELINK],
+		cx->cnt_bt[BTC_BCNT_RATECHG],
+		cx->cnt_bt[BTC_BCNT_REINIT],
+		cx->cnt_bt[BTC_BCNT_REENABLE]);
+
+	CLI_PRT("role-sw:%d, afh-sw:%d, inq:%d, pag:%d, igno_wl:%d",
+		cx->cnt_bt[BTC_BCNT_ROLESW],
+		cx->cnt_bt[BTC_BCNT_AFH],
+		cx->cnt_bt[BTC_BCNT_INQ],
+		cx->cnt_bt[BTC_BCNT_PAGE],
+		cx->cnt_bt[BTC_BCNT_IGNOWL]);
+
+	_show_bt_profile_info(btc, used, input, input_num, output, out_len);
+
+	CLI_PRT("\n\r %-15s : raw_data[%02x %02x %02x %02x %02x %02x]"
+		"(type:%s/cnt:%d/same:%d)", "[bt_info]",
+		bt->raw_info[2], bt->raw_info[3],
+		bt->raw_info[4], bt->raw_info[5],
+		bt->raw_info[6], bt->raw_info[7],
+		(bt->raw_info[0] == BTC_BTINFO_AUTO ? "auto" : "reply"),
+		cx->cnt_bt[BTC_BCNT_INFOUPDATE],
+		cx->cnt_bt[BTC_BCNT_INFOSAME]);
+
+	CLI_PRT("\n\r %-15s : Hi-rx=%d, Hi-tx=%d, Lo-rx=%d, Lo-tx=%d"
+		" (pollut_wl_tx_diff:%d/rx_scan_LP:%d), ", "[trx_req_cnt]",
+		cx->cnt_bt[BTC_BCNT_HIPRI_RX],
+		cx->cnt_bt[BTC_BCNT_HIPRI_TX],
+		cx->cnt_bt[BTC_BCNT_LOPRI_RX],
+		cx->cnt_bt[BTC_BCNT_LOPRI_TX],
+		cx->cnt_bt[BTC_BCNT_POLUT_DIFF],
+		bt->scan_rx_low_pri);
+
+	if (bt->run_patch_code && bt->enable.now)
+		rpt_map |= RPT_EN_BT_SCAN_INFO;
+	else
+		rpt_map &= ~RPT_EN_BT_SCAN_INFO;
+
+	CLI_PRT("(BG:%d-%d", bt->scan_info[CXSCAN_BG].win,
+		bt->scan_info[CXSCAN_BG].intvl);
+	CLI_PRT("/INIT:%d-%d", bt->scan_info[CXSCAN_INIT].win,
+		bt->scan_info[CXSCAN_INIT].intvl);
+	CLI_PRT("/LE:%d-%d)", bt->scan_info[CXSCAN_LE].win,
+		bt->scan_info[CXSCAN_LE].intvl);
+
+	if (bt_linfo->profile_cnt.now)
+		rpt_map |= RPT_EN_BT_AFH_MAP;
+	else
+		rpt_map &= ~RPT_EN_BT_AFH_MAP;
+
+	if (bt_linfo->status.map.ble_connect || bt_linfo->leaudio_desc.exist)
+		rpt_map |= RPT_EN_BT_AFH_MAP_LE;
+	else
+		rpt_map &= ~RPT_EN_BT_AFH_MAP_LE;
+
+	if (bt_linfo->a2dp_desc.exist &&
+	    (bt_linfo->a2dp_desc.flush_time == 0 ||
+	     bt_linfo->a2dp_desc.vendor_id == 0 ||
+	     bt_linfo->a2dp_desc.play_latency == 1))
+		rpt_map |= RPT_EN_BT_DEVICE_INFO;
+	else
+		rpt_map &= ~RPT_EN_BT_DEVICE_INFO;
+
+	_set_fw_rpt(btc, rpt_map, 2);
+}
+
+static void _show_fbtc_tdma(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_tdma *t = NULL;
+
+	pcinfo = &pfwinfo->rpt_fbtc_tdma.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	t = &pfwinfo->rpt_fbtc_tdma.finfo.tdma;
+
+	CLI_PRT("\n\r %-15s : policy:%s  [type:%s/rx_flow:%d/tx_flow:%d/",
+		"[tdma_policy]",
+		id_to_str(BTC_STR_POLICY, (u32)btc->policy_type),
+		id_to_str(BTC_STR_TDMA, (u32)t->type),
+		t->rxflctrl, t->txflctrl);
+
+	CLI_PRT("leak_n:%d,/ext_ctrl:%d/null_role:0x%x/opt:0x%x]",
+		t->leak_n, t->ext_ctrl, t->rxflctrl_role, t->option_ctrl);
+}
+
+static void _show_dm_info(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			  u32 input_num, char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_ant_info *ant = &btc->mdinfo.ant;
+	struct btc_module *md = &btc->mdinfo;
+#ifdef BTC_AISO_SUPPORT
+		struct btc_bt_psd_dm *bp = &btc->bt_psd_dm;
+		struct btc_aiso_val *av = &bp->aiso_val;
+		u8 cnt, aiso_cur, aiso_ori = ant->isolation;
+#endif
+
+	if (!(dm->coex_info_map & BTC_COEX_INFO_DM))
+		return;
+
+	CLI_PRT("\n\r========== [Mechanism Status %s] ==========",
+		(btc->ctrl.manual? "(Manual)":"(Auto)"));
+
+	CLI_PRT("\n\r %-15s : type:%s, reason:%s(), action:%s(),"
+		" ant_path:%s, run_cnt:%d", "[status]",
+		(ant->type == BTC_ANT_SHARED ? "shared" : "non-shared"),
+		dm->run_reason, dm->run_action,
+		id_to_str(BTC_STR_ANTPATH, dm->set_ant_path & 0xff),
+		dm->cnt_dm[BTC_DCNT_RUN]);
+
+	_show_fbtc_tdma(btc, used, input, input_num, output, out_len);
+
+	CLI_PRT("\n\r %-15s : wl_only:%d, bt_only:%d, igno_bt:%d,"
+		" free_run:%d, wl_ps_ctrl:%d, ", "[dm_flag]",
+		dm->wl_only, dm->bt_only, btc->ctrl.igno_bt,
+		dm->freerun, btc->hal->btc_ctrl.lps);
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	CLI_PRT("leak_ap:%d, fddt_train:%d", dm->leak_ap, dm->fddt_train);
+#else
+	CLI_PRT("leak_ap:%d, fddt_train:No-Support", dm->leak_ap);
+#endif
+
+	CLI_PRT("\n\r %-15s : wl_tx_limit[en:%d/max_t:%dus],"
+		" wl_trx_1ss[tx:%d/path:%s/rx:%d/path:%s],"
+		" bt_slot_req:[%d/%d], bt_stbc_req:%d, wa_type:0x%x,"
+		" ofld[scan:%d/",
+		"[dm_drv_ctrl]",
+		dm->wl_tx_limit.en, dm->wl_tx_limit.tx_time,
+		dm->wl_trx_nss.tx_ss,
+		id_to_str(BTC_STR_PATH, (u32)dm->wl_trx_nss.tx_path),
+		dm->wl_trx_nss.rx_ss,
+		id_to_str(BTC_STR_PATH, (u32)dm->wl_trx_nss.rx_path),
+		btc->bt_req_len[HW_PHY_0], btc->bt_req_len[HW_PHY_1],
+		btc->bt_req_stbc, md->wa_type, btc->hal->scanofld_en);
+
+#ifdef BTC_CONFIG_FW_IO_OFLD_SUPPORT
+	CLI_PRT("/io:%d", hal_btc_check_io_ofld(btc));
+#else
+	CLI_PRT("/io:0");
+#endif
+
+#ifdef BTC_OUTSRC_API_OFLD
+	CLI_PRT("/outsrc_api:1]");
+#else
+	CLI_PRT("/outsrc_api:0]");
+#endif
+
+	CLI_PRT("\n\r %-15s : wl[rssi_lvl:%d/para:%d/tx_pwr:%d/rx_lvl:%d"
+		"(pre_agc:%d/lna2:%d)/"
+		"btg_rx:%d/stb_chg:%d] ", "[dm_rf_ctrl]",
+		wl->rssi_level, dm->trx_para_level,
+		dm->rf_trx_para.wl_tx_power, dm->rf_trx_para.wl_rx_gain,
+		dm->wl_pre_agc, dm->wl_lna2,
+		dm->wl_btg_rx, dm->wl_stb_chg);
+
+	CLI_PRT("bt[tx_pwr_dec:%d/rx_lna:%d(%s-tbl)]",
+		dm->rf_trx_para.bt_tx_power, dm->rf_trx_para.bt_rx_gain,
+		(bt->hi_lna_rx? "Hi" : "Ori"));
+
+#ifdef BTC_AISO_SUPPORT
+	if (bp->en || bp->aiso_data_ok || bp->rec_time_out || bp->aiso_db_cnt) {
+		cnt = bp->aiso_db_cnt;
+		aiso_cur = (cnt > 0 ? bp->aiso_db[(cnt-1) & 0xf] : aiso_ori);
+		CLI_PRT("\n\r %-15s : cur_iso:%d dB(cnt:%d), en:%d, run:%d,"
+			" timeout:%d, data_ok:%d, psd_cnt:%d, exec_cmd_cnt:%d",
+			"[dm_ant_iso]", aiso_cur, cnt, bp->en, bp->rec_start,
+			bp->rec_time_out, bp->aiso_data_ok, av->psd_rec_cnt,
+			bp->aiso_cmd_cnt);
+	}
+#endif
+	_show_dm_step(btc, used, input, input_num, output, out_len);
+}
+
+static void _show_mreg(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			u32 input_num, char *output, u32 out_len)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_mreg_val *pmreg = NULL;
+	struct fbtc_gpio_dbg *gdbg = NULL;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_gnt_ctrl *gnt = NULL;
+	u8 i = 0, type = 0, cnt = 0;
+	u32 val, offset;
+
+	if (!(dm->coex_info_map & BTC_COEX_INFO_MREG))
+		return;
+
+	CLI_PRT("\n\r========== [HW Status] ==========");
+
+	CLI_PRT("\n\r %-15s : "
+		"WL->BT:0x%08x(cnt:%d), BT->WL:0x%08x(total:%d, bt_update:%d)",
+		"[scoreboard]", wl->scbd, cx->cnt_wl[BTC_WCNT_SCBDUPDATE],
+		bt->scbd, cx->cnt_bt[BTC_BCNT_SCBDREAD],
+		cx->cnt_bt[BTC_BCNT_SCBDUPDATE]);
+
+
+	CLI_PRT("\n\r %-15s : pta_owner:%s, pta_req_mac:MAC%d, ss_trxmask:0x%s,"
+		" polut_type:%s",
+		"[gnt_status]",
+		(btc->chip->hw & BTC_FEAT_PTA_ONOFF_CTRL ? "HW" :
+		(dm->pta_owner == BTC_CTRL_BY_WL? "WL" : "BT")),
+		wl->pta_req_mac,
+		(cx->other.type != BTC_EXTSOC_FUNC_NONE? "5DF" : "5FF"),
+		id_to_str(BTC_STR_POLUT, wl->bt_polut_type[wl->pta_req_mac]));
+
+	gnt = &dm->gnt_val[HW_PHY_0];
+
+	CLI_PRT(", phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d]",
+		(gnt->gnt_wl_sw_en? "SW" : "HW"), gnt->gnt_wl,
+		(gnt->gnt_bt_sw_en? "SW" : "HW"), gnt->gnt_bt);
+
+	if (wl_rinfo->dbcc_en) {
+		gnt = &dm->gnt_val[HW_PHY_1];
+		CLI_PRT(", phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]",
+			(gnt->gnt_wl_sw_en? "SW" : "HW"), gnt->gnt_wl,
+			(gnt->gnt_bt_sw_en? "SW" : "HW"), gnt->gnt_bt);
+	}
+
+	pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	pmreg = &pfwinfo->rpt_fbtc_mregval.finfo;
+
+	for (i = 0; i < pmreg->reg_num; i++) {
+		type = (u8)btc->chip->mon_reg[i].type;
+		offset = btc->chip->mon_reg[i].offset;
+		val = pmreg->mreg_val[i];
+
+		if (cnt % 6 == 0)
+			CLI_PRT("\n\r %-15s : %s_0x%x=0x%x", "[reg]",
+				id_to_str(BTC_STR_REG, (u32)type), offset, val);
+		else
+			CLI_PRT(", %s_0x%x=0x%x",
+				id_to_str(BTC_STR_REG, (u32)type), offset, val);
+		cnt++;
+	}
+
+	pcinfo = &pfwinfo->rpt_fbtc_gpio_dbg.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	gdbg = &pfwinfo->rpt_fbtc_gpio_dbg.finfo;
+	if (!gdbg->en_map)
+		return;
+
+	CLI_PRT("\n\r %-15s : enable_map:0x%08x", "[gpio_dbg]", gdbg->en_map);
+
+	for (i = 0; i < BTC_DBG_MAX1; i++) {
+		if (!(gdbg->en_map & BIT(i)))
+			continue;
+		CLI_PRT(", %s->GPIO%d",
+			id_to_str(BTC_STR_GDBG, (u32)i), gdbg->gpio_map[i]);
+	}
+}
+
+static void _show_summary(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			  u32 input_num, char *output, u32 out_len)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_rpt_ctrl *prptctrl = NULL;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &cx->wl;
+	u32 cnt_sum = 0;
+	u32 *cnt = btc->dm.cnt_notify;
+	u8 i;
+
+	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
+		return;
+	CLI_PRT("%s", "\n\r========== [Statistics] ==========");
+
+	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
+	if (pcinfo->valid && wl->status.map.lps != BTC_LPS_RF_OFF &&
+	    !wl->status.map.rf_off) {
+		prptctrl = &pfwinfo->rpt_ctrl.finfo;
+
+		CLI_PRT("\n\r %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d),"
+			" c2h_cnt=%d(fw_send:%d, len:%d, max:%d), ",
+			"[summary]", pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
+			prptctrl->rpt_info.cnt_h2c, pfwinfo->cnt_c2h,
+			prptctrl->rpt_info.cnt_c2h, prptctrl->rpt_info.len_c2h,
+			RTW_PHL_BTC_FWINFO_BUF);
+
+		CLI_PRT("rpt_cnt=%d(fw_send:%d), rpt_map=0x%x",
+			pfwinfo->event[BTF_EVNT_RPT], prptctrl->rpt_info.cnt,
+			prptctrl->rpt_info.en);
+
+		if (dm->error & BTC_DMERR_WL_FW_HANG)
+			CLI_PRT(" (WL FW Hang!!)");
+
+		CLI_PRT("\n\r %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			"[mailbox]", prptctrl->bt_mbx_info.cnt_send_ok,
+			prptctrl->bt_mbx_info.cnt_send_fail,
+			prptctrl->bt_mbx_info.cnt_recv);
+
+		CLI_PRT("A2DP_empty:%d(stop:%d/tx:%d/ack:%d/nack:%d)",
+			prptctrl->bt_mbx_info.a2dp.cnt_empty,
+			prptctrl->bt_mbx_info.a2dp.cnt_flowctrl,
+			prptctrl->bt_mbx_info.a2dp.cnt_tx,
+			prptctrl->bt_mbx_info.a2dp.cnt_ack,
+			prptctrl->bt_mbx_info.a2dp.cnt_nack);
+
+		CLI_PRT("\n\r %-15s :"
+			" wl_rfk[req:%d/go:%d/reject:%d/tout:%d/time:%dms]",
+			"[RFK/LPS]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			cx->cnt_wl[BTC_WCNT_RFK_GO],
+			cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT],
+			wl->rfk_info.proc_time);
+
+		CLI_PRT(", bt_rfk[req:%d]", prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]);
+
+		CLI_PRT(", AOAC[RF_on:%d/RF_off:%d]",
+			prptctrl->rpt_info.cnt_aoac_rf_on,
+			prptctrl->rpt_info.cnt_aoac_rf_off);
+	} else {
+		CLI_PRT("\n\r %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d"
+			" (lps=%d/rf_off=%d)", "[summary]",
+			pfwinfo->cnt_h2c, pfwinfo->cnt_h2c_fail,
+			pfwinfo->cnt_c2h,
+			wl->status.map.lps, wl->status.map.rf_off);
+	}
+
+	for (i = 0; i < BTC_NTFY_MAX; i++)
+		cnt_sum += dm->cnt_notify[i];
+
+	CLI_PRT("\n\r %-15s : total=%d, show_coex_info=%d,"
+		" power_on=%d, init_coex=%d, ", "[notify_cnt]",
+		cnt_sum, cnt[BTC_NTFY_SHOW_COEX_INFO],
+		cnt[BTC_NTFY_POWER_ON], cnt[BTC_NTFY_INIT_COEX]);
+
+	CLI_PRT("power_off=%d, radio_state=%d, role_info=%d,"
+		" wl_rfk=%d, wl_sta=%d",
+		cnt[BTC_NTFY_POWER_OFF], cnt[BTC_NTFY_RADIO_STATE],
+		cnt[BTC_NTFY_ROLE_INFO], cnt[BTC_NTFY_WL_RFK],
+		cnt[BTC_NTFY_WL_STA]);
+
+	CLI_PRT("\n\r %-15s : scan_start=%d, scan_finish=%d,"
+		" sw_band=%d, special_pkt=%d, ", "[notify_cnt]",
+		cnt[BTC_NTFY_SCAN_START], cnt[BTC_NTFY_SCAN_FINISH],
+		cnt[BTC_NTFY_SWITCH_BAND], cnt[BTC_NTFY_SPECIAL_PACKET]);
+
+	CLI_PRT("timer=%d, custm=%d, hub_msg=%d, chg_fw=%d, ctry_code=%d",
+		cnt[BTC_NTFY_TIMER], cnt[BTC_NTFY_CUSTOMERIZE],
+		btc->hubmsg_cnt, cnt[BTC_NTFY_RESUME_DL_FW],
+		cnt[BTC_NTFY_COUNTRYCODE]);
+}
+
+static void _show_fbtc_slots(struct btc_t *btc, u32 *used,
+			     char input[][MAX_ARGV], u32 input_num,
+			     char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct fbtc_slot s;
+	u8 i = 0;
+
+	for (i = 0; i < CXST_MAX; i++) {
+		s = dm->slot_now[i];
+		if (i % 5 == 0)
+			CLI_PRT("\n\r %-15s : %s[%d/%d/0x%x]", "[slot_list]",
+				id_to_str(BTC_STR_SLOT, (u32)i),
+				s.dur, s.cxtype, s.cxtbl);
+		else
+			CLI_PRT(", %s[%d/%d/0x%x]",
+				id_to_str(BTC_STR_SLOT, (u32)i),
+				s.dur, s.cxtype, s.cxtbl);
+	}
+}
+
+static void _show_fbtc_cysta(struct btc_t *btc, u32 *used,
+			     char input[][MAX_ARGV], u32 input_num,
+			     char *output, u32 out_len)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	struct btc_bt_a2dp_desc *a2dp = &bt->link_info.a2dp_desc;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_cysta *pcysta = NULL;
+	u8 i, cnt = 0, divide_cnt = 12;
+	u8 slot_pair;
+	u16 cycle, c_begin, c_end, s_id;
+
+	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo;
+	CLI_PRT("\n\r %-15s : cycle:%d", "[slot_stat]", pcysta->cycles);
+
+	for (i = 0; i < CXST_MAX; i++) {
+		if (!pcysta->slot_cnt[i])
+			continue;
+		CLI_PRT(", %s:%d",
+			id_to_str(BTC_STR_SLOT, (u32)i), pcysta->slot_cnt[i]);
+	}
+
+	if (dm->tdma_now.rxflctrl) {
+		CLI_PRT(", leak_rx:%d", pcysta->leak_slot.cnt_rximr);
+	}
+
+	if (pcysta->collision_cnt) {
+		CLI_PRT(", collision:%d", pcysta->collision_cnt);
+	}
+
+	if (pcysta->skip_cnt) {
+		CLI_PRT(", skip:%d", pcysta->skip_cnt);
+	}
+
+	CLI_PRT("\n\r %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]", "[cycle_stat]",
+		pcysta->cycle_time.tavg[CXT_WL],
+		pcysta->cycle_time.tavg[CXT_BT],
+		pcysta->leak_slot.tavg/1000, pcysta->leak_slot.tavg%1000);
+	CLI_PRT(", max_t[wl:%d/bt:%d(>%dms:%d)/lk:%d.%03d]",
+		pcysta->cycle_time.tmax[CXT_WL],
+		pcysta->cycle_time.tmax[CXT_BT],
+		dm->bt_slot_flood, dm->cnt_dm[BTC_DCNT_BT_SLOT_FLOOD],
+		pcysta->leak_slot.tamx/1000, pcysta->leak_slot.tamx%1000);
+	CLI_PRT(", bcn[all:%d/ok:%d/in_bt:%d/in_bt_ok:%d]",
+		pcysta->bcn_cnt[CXBCN_ALL], pcysta->bcn_cnt[CXBCN_ALL_OK],
+		pcysta->bcn_cnt[CXBCN_BT_SLOT], pcysta->bcn_cnt[CXBCN_BT_OK]);
+
+	if (a2dp->exist) {
+		CLI_PRT("\n\r %-15s : a2dp_ept:%d, a2dp_late:%d"
+			"(streak 2S:%d/max:%d)", "[a2dp_stat]",
+			pcysta->a2dp_ept.cnt, pcysta->a2dp_ept.cnt_timeout,
+			a2dp->no_empty_streak_2s, a2dp->no_empty_streak_max);
+
+		CLI_PRT(", avg_t:%d, max_t:%d",
+			pcysta->a2dp_ept.tavg, pcysta->a2dp_ept.tmax);
+	}
+
+	if (pcysta->cycles <= 1)
+		return;
+
+	/* 1 cycle = 1 wl-slot + 1 bt-slot */
+	slot_pair = BTC_CYCLE_SLOT_MAX/2;
+
+	if (pcysta->cycles <= slot_pair)
+		c_begin = 1;
+	else
+		c_begin = pcysta->cycles - slot_pair + 1;
+
+	c_end = pcysta->cycles;
+
+	if (a2dp->exist)
+		divide_cnt = 2;
+	else
+		divide_cnt = 6;
+
+	if (c_begin > c_end)
+		return;
+
+	for (cycle = c_begin; cycle <= c_end; cycle++) {
+		cnt++;
+		s_id = ((cycle-1) % slot_pair)*2;
+
+		if (cnt % divide_cnt == 1) {
+			if (dm->fddt_train == BTC_FDDT_ENABLE)
+				CLI_PRT("\n\r %-15s : ", "[slotT_permtan]");
+			if (a2dp->exist)
+				CLI_PRT("\n\r %-15s : ", "[slotT_wermtan]");
+			else
+				CLI_PRT("\n\r %-15s : ", "[slotT_rxerr]");
+		}
+
+		CLI_PRT("->b%d", pcysta->slot_step_time[s_id]);
+
+		if (a2dp->exist)
+			CLI_PRT("(%d/%d/%d/%dM/%d/%d/%d)",
+				pcysta->wl_rx_err_ratio[s_id],
+				pcysta->a2dp_trx[s_id].empty_cnt,
+				pcysta->a2dp_trx[s_id].retry_cnt,
+				(pcysta->a2dp_trx[s_id].tx_rate ? 3:2),
+				pcysta->a2dp_trx[s_id].tx_cnt,
+				pcysta->a2dp_trx[s_id].ack_cnt,
+				pcysta->a2dp_trx[s_id].nack_cnt);
+		else
+			CLI_PRT("(%d)", pcysta->wl_rx_err_ratio[s_id]);
+
+		CLI_PRT("->w%d", pcysta->slot_step_time[s_id+1]);
+
+		if (a2dp->exist)
+			CLI_PRT("(%d/%d/%d/%dM/%d/%d/%d)",
+				pcysta->wl_rx_err_ratio[s_id+1],
+				pcysta->a2dp_trx[s_id+1].empty_cnt,
+				pcysta->a2dp_trx[s_id+1].retry_cnt,
+				(pcysta->a2dp_trx[s_id+1].tx_rate ? 3:2),
+				pcysta->a2dp_trx[s_id+1].tx_cnt,
+				pcysta->a2dp_trx[s_id+1].ack_cnt,
+				pcysta->a2dp_trx[s_id+1].nack_cnt);
+		else
+			CLI_PRT("(%d)", pcysta->wl_rx_err_ratio[s_id+1]);
+	}
+}
+
+static void _show_fbtc_nullsta(struct btc_t *btc, u32 *used,
+				char input[][MAX_ARGV], u32 input_num,
+				char *output, u32 out_len)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_cynullsta *ns = NULL;
+	u8 i = 0;
+
+	if (!btc->dm.tdma_now.rxflctrl)
+		return;
+
+	pcinfo = &pfwinfo->rpt_fbtc_nullsta.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	ns = &pfwinfo->rpt_fbtc_nullsta.finfo;
+
+	for (i = CXNULL_0; i <= CXNULL_1; i++) {
+		CLI_PRT("\n\r %-15s : ", "[null_status]");
+		CLI_PRT("null-%d", i);
+		CLI_PRT("[Tx:%d/", ns->result[i][CXNULL_TX]);
+		CLI_PRT("ok:%d/", ns->result[i][CXNULL_OK]);
+		CLI_PRT("late:%d/", ns->result[i][CXNULL_LATE]);
+		CLI_PRT("fail:%d/", ns->result[i][CXNULL_FAIL]);
+		CLI_PRT("retry:%d/", ns->result[i][CXNULL_RETRY]);
+		CLI_PRT("avg_t:%d.%03d/", ns->tavg[i]/1000, ns->tavg[i]%1000);
+		CLI_PRT("max_t:%d.%03d]", ns->tmax[i]/1000, ns->tmax[i]%1000);
+	}
+}
+
+static void _show_error(struct btc_t *btc, u32 *used,
+			char input[][MAX_ARGV], u32 input_num,
+			char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	u32 fw_except_map = 0;
+	u8 fw_except_cnt = 0;
+	u8 valid = pfwinfo->rpt_fbtc_cysta.cinfo.valid;
+
+	if (valid) {
+		fw_except_map = pfwinfo->rpt_fbtc_cysta.finfo.except_map;
+		fw_except_cnt = pfwinfo->rpt_fbtc_cysta.finfo.except_cnt;
+	}
+
+	if (!(dm->error || fw_except_map || fw_except_cnt ||
+	    pfwinfo->len_mismch || pfwinfo->fver_mismch ||
+	    pfwinfo->err[BTFRE_EXCEPTION]))
+		return;
+
+	CLI_PRT("\n\r %-15s : dm_err:0x%x, fw_except:0x%x(cnt:%d), "
+		"fw_rpt[len_err:0x%x/ver_err:0x%x/except_cnt:%d]", "[ERROR]",
+		dm->error, fw_except_map, fw_except_cnt,
+		pfwinfo->len_mismch, pfwinfo->fver_mismch,
+		pfwinfo->err[BTFRE_EXCEPTION]);
+}
+
+#ifdef BTC_FW_STEP_DBG
+static void _show_fbtc_step(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_steps *pstep = NULL;
+	u8 type, val;
+	u16 diff_t;
+	u32 i, cnt = 0, n_begin = 0, n_end = 0, array_idx = 0;
+
+	PHL_INFO("[BTC],%s():rpt_en_map=0x%x\n", __func__, pfwinfo->rpt_en_map);
+
+	if ((pfwinfo->rpt_en_map & RPT_EN_FW_STEP_INFO) == 0)
+		return;
+
+	pcinfo = &pfwinfo->rpt_fbtc_step.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	pstep = &pfwinfo->rpt_fbtc_step.finfo;
+
+	if (pcinfo->req_fver != pstep->fver || !pstep->cnt)
+		return;
+
+	if (pstep->cnt <= FCXDEF_STEP)
+		n_begin = 1;
+	else
+		n_begin = pstep->cnt - FCXDEF_STEP + 1;
+
+	n_end = pstep->cnt;
+
+	if (n_begin > n_end)
+		return;
+
+	/* restore step info by using ring instead of FIFO */
+	for (i = n_begin; i <= n_end; i++) {
+		array_idx = (i - 1) % FCXDEF_STEP;
+		type = pstep->step[array_idx].type;
+		val = pstep->step[array_idx].val;
+		diff_t = (pstep->step[array_idx].difft_h8 << 8) +
+			  pstep->step[array_idx].difft_l8;
+
+		if (type == CXSTEP_NONE || type >= CXSTEP_MAX)
+			continue;
+
+		if (cnt % 10 == 0)
+			CLI_PRT("\n\r %-15s : ", "[fw_steps]");
+
+		CLI_PRT("-> %s(%d)",
+			(type == CXSTEP_SLOT?
+			id_to_str(BTC_STR_SLOT, (u32)val) :
+			id_to_str(BTC_STR_EVENT, (u32)val)), diff_t);
+		cnt++;
+	}
+}
+#endif
+
+static void _show_fw_dm_msg(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_DM))
+		return;
+
+	_show_fbtc_slots(btc, used, input, input_num, output, out_len);
+	_show_fbtc_cysta(btc, used, input, input_num, output, out_len);
+	_show_fbtc_nullsta(btc, used, input, input_num, output, out_len);
+#ifdef BTC_FW_STEP_DBG
+	_show_fbtc_step(btc, used, input, input_num, output, out_len);
+#endif
+}
+
+static void _cmd_show(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 out_len)
+{
+	u32 show_map = 0xff;
+
+	if (input_num > 1)
+		_os_sscanf(input[1], "%x", &show_map);
+
+	btc->dm.coex_info_map = show_map & 0xff;
+
+	_show_cx_info(btc, used, input, input_num, output, out_len);
+	_show_wl_info(btc, used, input, input_num, output, out_len);
+	_show_bt_info(btc, used, input, input_num, output, out_len);
+	_show_dm_info(btc, used, input, input_num, output, out_len);
+	_show_fw_dm_msg(btc, used, input, input_num, output, out_len);
+	_show_error(btc, used, input, input_num, output, out_len);
+	_show_mreg(btc, used, input, input_num, output, out_len);
+	_show_summary(btc, used, input, input_num, output, out_len);
+}
+
+static void _cmd_dbg(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+		     u32 input_num, char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	u8 buf[9] = {0};
+	u8 valid = pfwinfo->rpt_fbtc_cysta.cinfo.valid;
+	u32 len = 0, n;
+	u32 val = 0, type = 0, i, j;
+	u32 fw_err_map = 0;
+
+	/* input[0]->dbg, input[1]->type, input[2]->para1...
+	 * buf[0] = type, buf[1] = para1, buf[2] = para2
+	 */
+
+	if (input_num <= 1) {
+		CLI_PRT(" dbg <type> <para1:8bits> <para2:8bits>... \n");
+		CLI_PRT(" dbg 0: H2C-C2H loopback <para1~8: loopback data>\n");
+		CLI_PRT(" dbg 1: H2C-mailbox <para1~8: mailbox data>\n");
+		CLI_PRT(" dbg 2: bt slot-request as NOA <para1: slot(hex)>\n");
+		CLI_PRT(" dbg 3: enable fw_step debug <para1:8bits>\n");
+		CLI_PRT(" dbg 4: bt flood-threshold setup <para1:8bits>\n");
+		CLI_PRT(" dbg 5: dm error map decode\n");
+		CLI_PRT(" dbg 6: fw exception map decode\n");
+		CLI_PRT(" dbg 7: fw report length mismatch map decode\n");
+		CLI_PRT(" dbg 8: fw report version mismatch map decode\n");
+		CLI_PRT(" dbg 9: fddt enum map decode\n");
+		CLI_PRT(" dbg 10: MAC crc error report <para1: enable>\n");
+		CLI_PRT(" dbg 11: BB btg_rx_ctrl <para1: enable>\n");
+		CLI_PRT(" dbg 12: BB pre_agc_ctrl <para1: enable>\n");
+		return;
+	} else {
+		/* dbg cmd type */
+		_os_sscanf(input[1], "%d", &type);
+
+		/* error map debug command */
+		if ((type < 5 || type > 9) && input_num == 2) {
+			CLI_PRT(" para1 not found!!\n");
+			return;
+		}
+	}
+
+	/* dbg cmd para1~para8(max) */
+	for (n = 1; n <= input_num-1; n++) {
+		_os_sscanf(input[n], "%x", &val);
+		len++;
+		if(len > sizeof(buf)) {
+			CLI_PRT(" para length is too long!!\n");
+			return;
+		}
+		buf[len-1] = (u8)(val & 0xff);
+	}
+
+	switch (type) {
+	case 0: /* H2C-C2H loopback */
+		if (len <= 1) {
+			CLI_PRT(" no para found!!\n");
+			return;
+		}
+		_send_fw_cmd(btc, SET_H2C_TEST, buf, (u16)len);
+		btc->dbg.rb_done = false;
+		for (i = 0; i < 50; i++) {
+			if (!btc->dbg.rb_done) {
+				hal_mdelay(btc->hal, 10);
+				continue;
+			}
+
+			CLI_PRT(" H2C-C2H loopback data: [");
+			for (j = 0; j < len - 1; j++) {
+				CLI_PRT(" 0x%x", btc->dbg.lb_val[j]);
+			}
+			CLI_PRT("]\n");
+			return;
+		}
+		CLI_PRT(" H2C-C2H loopback timeout!!\n");
+		break;
+	case 1: /* H2C-mailbox, To DO: FW send mailbox*/
+		if (len != 9) {
+			CLI_PRT(" para count must be 8!!\n");
+			return;
+		}
+		_send_fw_cmd(btc, SET_H2C_TEST, buf, (u16)len);
+
+		if (buf[1] != 0x33)
+			return;
+
+		/* mailbox 0x33 loopback process */
+		btc->dbg.rb_done = false;
+		for (i = 0; i < 50; i++) {
+			if (!btc->dbg.rb_done) {
+				hal_mdelay(btc->hal, 10);
+				continue;
+			}
+
+			CLI_PRT(" Mailbox loopback data: [");
+			for (j = 0; j < len - 1; j++) {
+				CLI_PRT(" 0x%x", btc->dbg.lb_val[j]);
+			}
+			CLI_PRT("]\n");
+			return;
+		}
+		CLI_PRT(" Mailbox loopback timeout !!\n");
+		break;
+	case 2: /* bt slot request */
+		btc->bt_req_len[wl->pta_req_mac] = (u32)buf[1];
+		_send_phl_evnt(btc, (u8*)&btc->bt_req_len[wl->pta_req_mac], 4,
+			       BTC_HMSG_SET_BT_REQ_SLOT);
+		CLI_PRT(" bt-slot-req: mac%d=%d\n", wl->pta_req_mac, buf[1]);
+		break;
+	case 3: /* fw_step debug */
+#ifdef BTC_FW_STEP_DBG
+		if (buf[1] > 0)
+			_set_fw_rpt(btc, RPT_EN_FW_STEP_INFO, 1);
+		else
+			_set_fw_rpt(btc, RPT_EN_FW_STEP_INFO, 0);
+#endif
+		break;
+	case 4: /* bt slot flood-threshold setup */
+		dm->bt_slot_flood = (u16)buf[1];
+		CLI_PRT(" bt-slot-flood= %dms!!\n", dm->bt_slot_flood);
+		break;
+	case 5: /* dm error map */
+		if (!dm->error) {
+			CLI_PRT(" no DM Error!!\n");
+			return;
+		}
+
+		CLI_PRT(" DM Error includes:\n");
+		for (i = 0; i < BTC_DMERR_MAX; i++) {
+			if (!(btc->dm.error & BIT(i)))
+				continue;
+			CLI_PRT(" %s\n", id_to_str(BTC_STR_DMERROR, BIT(i)));
+		}
+		break;
+	case 6: /* fw exception map */
+		if (!valid) {
+			CLI_PRT(" no FW exception report!!\n");
+			return;
+		}
+
+		fw_err_map = pfwinfo->rpt_fbtc_cysta.finfo.except_map;
+
+		if (!fw_err_map) {
+			CLI_PRT(" no FW exception!\n");
+			return;
+		}
+
+		CLI_PRT(" FW exception includes:\n");
+		for (i = 0; i < BTC_FWERR_MAX; i++) {
+			if (!(fw_err_map & BIT(i)))
+				continue;
+			CLI_PRT(" %s\n", id_to_str(BTC_STR_FWERROR, BIT(i)));
+
+			if (i == 15)
+				CLI_PRT("  --> H2C-Error cmd:SET_%s\n",
+					id_to_str(BTC_STR_H2CERROR,
+						  (fw_err_map & bMASKB3) >>24));
+		}
+		break;
+	case 7: /* fw report length mismatch */
+		if (!pfwinfo->len_mismch) {
+			CLI_PRT(" no fw report length mismatch!!\n");
+			return;
+		}
+
+		CLI_PRT(" fw report length mismatch includes:\n");
+		for (i = 0; i < BTC_RPT_TYPE_MAX; i++) {
+			if (!(pfwinfo->len_mismch & BIT(i)))
+				continue;
+			CLI_PRT(" %s\n", id_to_str(BTC_STR_RPTMATCH, i));
+		}
+		break;
+	case 8: /* fw report version mismatch */
+		if (!pfwinfo->fver_mismch) {
+			CLI_PRT(" no fw report version mismatch!!\n");
+			return;
+		}
+
+		CLI_PRT(" fw report verison mismatch includes:\n");
+		for (i = 0; i < BTC_RPT_TYPE_MAX; i++) {
+			if (!(pfwinfo->fver_mismch & BIT(i)))
+				continue;
+			CLI_PRT(" %s\n", id_to_str(BTC_STR_RPTMATCH, i));
+		}
+		break;
+	case 9: /* fddt enum map */
+		CLI_PRT(" ==== fddt no-run reason enum ====\n");
+		for (i = 0; i < BTC_NFRSN_MAX; i++) {
+			CLI_PRT("%d:%s\n", i, id_to_str(BTC_STR_FDDT_NORUN, i));
+		}
+
+		CLI_PRT(" ==== fddt train state enum ====\n");
+		for (i = 0; i < CX_FDDT_STATE_MAX; i++) {
+			CLI_PRT("%d:%s\n", i, id_to_str(BTC_STR_FDDT_TSTATE, i));
+		}
+
+		CLI_PRT(" ==== fddt train phase enum ====\n");
+		for (i = 0; i < CX_FDDT_PHASE_MAX; i++) {
+			CLI_PRT("%d:%s\n", i, id_to_str(BTC_STR_FDDT_TPHASE, i));
+		}
+
+		CLI_PRT(" ==== fddt check map (train_result) enum ====\n");
+		for (i = 0; i < CX_FDDT_CHKMAP_MAX; i++) {
+			CLI_PRT("BIT%d:%s\n", i, id_to_str(BTC_STR_FDDT_CHKMAP, i));
+		}
+		break;
+	case 10:
+		btc->dm.rx_err_rpt_en = (buf[1] == 0 ? 0 : 1);
+		CLI_PRT("wl_rx_err_ratio enable = %d\n", buf[1]);
+		break;
+	case 11:
+		dm->wl_btg_rx = (u32)buf[1];
+		rtw_hal_bb_ctrl_btg(btc->hal, (bool)(!!dm->wl_btg_rx));
+		break;
+	case 12:
+		dm->wl_pre_agc = buf[1];
+		rtw_hal_bb_ctrl_btc_preagc(btc->hal, (bool)(!!dm->wl_pre_agc));
+		break;
+	}
+}
+
+static void _cmd_wb(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+		    u32 input_num, char *output, u32 out_len)
+{
+	u32 type = 0, addr = 0, val = 0;
+
+	if (input_num >= 4) {
+		_os_sscanf(input[1], "%d", &type);
+		_os_sscanf(input[2], "%x", &addr);
+		_os_sscanf(input[3], "%x", &val);
+	}
+
+	if (type > 4 || input_num < 4) {
+		CLI_PRT(" wb <type --- 0:rf, 1:modem, 2:bluewize, 3:vendor,"
+			" 4:LE> <addr:16bits> <val> \n");
+		return;
+	}
+
+	_write_bt_reg(btc, (u8)type, (u16)addr, val);
+
+	CLI_PRT(" wb type=%d, addr=0x%x, val=0x%x !! \n", type, addr, val);
+}
+
+static void _cmd_set_coex(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			  u32 input_num, char *output, u32 out_len)
+{
+	u32 mode = 0;
+
+	if (input_num < 2) {
+		CLI_PRT(" mode <val--- 0:original, 1:freeze coex,"
+			" 2:always-freerun, 3:always-WLonly,"
+			" 4:always-BTonly> \n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &mode);
+
+	switch(mode) {
+	case 0: /* original */
+		btc->ctrl.manual = 0;
+		btc->ctrl.always_freerun = 0;
+		btc->dm.wl_only = 0;
+		btc->dm.bt_only = 0;
+		CLI_PRT(" recovery original coex mechanism !! \n");
+		break;
+	case 1: /* freeze */
+		btc->ctrl.manual = 1;
+		CLI_PRT(" freeze coex mechanism !! \n");
+		break;
+	case 2: /* fix freerun */
+		btc->ctrl.always_freerun = 1;
+		CLI_PRT(" freerun coex mechanism !! \n");
+		break;
+	case 3: /* fix wl only */
+		btc->dm.wl_only = 1;
+		CLI_PRT(" always WL-only coex mechanism!! \n");
+		break;
+	case 4: /* fix bt only */
+		btc->dm.bt_only = 1;
+		CLI_PRT(" always BT-only coex mechanism!! \n");
+		break;
+	}
+
+	_set_init_info(btc);
+	_run_coex(btc, __func__);
+}
+
+static void _cmd_upd_policy(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	_update_poicy(btc, FC_EXEC, (u16)BTC_CXP_MANUAL<<8, __func__);
+
+	CLI_PRT(" Update tdma/slot policy !! \n");
+}
+
+static void _cmd_tdma(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 out_len)
+{
+	u32 para1 = 0, para2 = 0;
+	u8 buf[8] = {0};
+
+	if (input_num >= 3) {
+		_os_sscanf(input[1], "%x", &para1);
+		_os_sscanf(input[2], "%x", &para2);
+
+		buf[0] = (u8)((para1 & bMASKB3) >> 24);
+		buf[1] = (u8)((para1 & bMASKB2) >> 16);
+		buf[2] = (u8)((para1 & bMASKB1) >> 8);
+		buf[3] = (u8)(para1 & bMASKB0);
+
+		buf[4] = (u8)((para2 & bMASKB3) >> 24);
+		buf[5] = (u8)((para2 & bMASKB2) >> 16);
+		buf[6] = (u8)((para2 & bMASKB1) >> 8);
+		buf[7] = (u8)(para2 & bMASKB0);
+
+		if (buf[0] < CXTDMA_MAX) {
+			_tdma_cpy(&btc->dm.tdma, buf);
+			CLI_PRT("tdma-para1: type=%s,"
+				" rxflctrl=0x%x, txflctrl=%d, rsvd=%d\n",
+				id_to_str(BTC_STR_TDMA, (u32)buf[0]),
+				buf[1], buf[2], buf[3]);
+
+			CLI_PRT("tdma-para2: leak_n=%d,"
+				" ext_ctrl=%x, rxflctrl_role=0x%x,"
+				" option=0x%x\n",
+				buf[4], buf[5], buf[6], buf[7]);
+			return;
+		}
+	}
+
+	CLI_PRT("tdma type error!! (>= %d)\n", CXTDMA_MAX);
+	CLI_PRT(" tdma <para1:32bits> <para2:32bits>\n");
+	CLI_PRT(" <para1>\n");
+	CLI_PRT(" bit[31:24]= type 0:off, 1:fix, 2:auto, 3:auto2\n");
+	CLI_PRT(" bit[23:16]= rx_flow_ctrl 0:off, 1:null, 2:Qos-null, 3:cts\n");
+	CLI_PRT(" bit[15:8] = tx_flow_ctrl 0:off, 1:on\n");
+	CLI_PRT(" bit[7:0]  = resevred\n");
+	CLI_PRT(" <para2>\n");
+	CLI_PRT(" bit[31:24]= leak_n enter leak slot every n*w2b-slot\n");
+	CLI_PRT(" bit[23:16]= ext_ctrl 0:off, 1:bcn-early 2:ext\n");
+	CLI_PRT(" bit[15:8] = rxflctrl_role\n");
+	CLI_PRT(" bit[7:0]  = option\n");
+}
+
+static void _cmd_slot(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 out_len)
+{
+	u32 sid = 0, dur = 0, cx_tbl = 0, cx_type = 0, i;
+	u8 buf[9] = {0};
+
+	if (input_num >= 5) {
+		_os_sscanf(input[1], "%d", &sid);
+		_os_sscanf(input[2], "%d", &dur);
+		_os_sscanf(input[3], "%d", &cx_type);
+		_os_sscanf(input[4], "%x", &cx_tbl);
+
+		buf[0] = (u8) sid;
+		buf[1] = (u8)(dur & bMASKB0);
+		buf[2] = (u8)((dur & bMASKB1) >> 8);
+
+		buf[3] = (u8)(cx_type & bMASKB0);
+		buf[4] = (u8)((cx_type & bMASKB1) >> 8);
+
+		buf[5] = (u8)(cx_tbl & bMASKB0);
+		buf[6] = (u8)((cx_tbl & bMASKB1) >> 8);
+		buf[7] = (u8)((cx_tbl & bMASKB2) >> 16);
+		buf[8] = (u8)((cx_tbl & bMASKB3) >> 24);
+
+		if (buf[0] < CXST_MAX) {
+			_slot_cpy(&btc->dm.slot[buf[0]], &buf[1]);
+			CLI_PRT(" set slot para: slot:%s,"
+				" duration:%d, type=%s, table:%08x!\n",
+				id_to_str(BTC_STR_SLOT, sid), dur,
+				(cx_type? "iso" : "mix"), cx_tbl);
+			return;
+		}
+	}
+
+	CLI_PRT("slot id error!! (>= %d)\n", CXST_MAX);
+	CLI_PRT(" slot <slot_id:8bits> <slot_duration:16bits>"
+		" <type:16bits> <coex_table:32bits>\n");
+
+	CLI_PRT("      <slot_id> \n");
+
+	for (i = 0; i < CXST_MAX; i++) {
+		if ((i+1) % 5 == 1)
+			CLI_PRT("		  %d: %s",
+				i, id_to_str(BTC_STR_SLOT, i));
+		else
+			CLI_PRT(", %d: %s", i, id_to_str(BTC_STR_SLOT, i));
+
+		if (((i+1) % 5 == 0) || (i == CXST_MAX - 1))
+			CLI_PRT("\n");
+	}
+
+	CLI_PRT("      <slot_duration> unit: ms\n");
+	CLI_PRT("      <type> BT packet at WL slot 0: allowed, 1: isolated\n");
+	CLI_PRT("      <coex_table> 32bits coex table\n");
+}
+
+
+static void _cmd_sig_gdbg_en(struct btc_t *btc, u32 *used,
+			     char input[][MAX_ARGV], u32 input_num,
+			     char *output, u32 out_len)
+{
+	u32 map = 0, i;
+
+	if (input_num >= 2) {
+		_os_sscanf(input[1], "%x", &map);
+
+		hal_btc_fw_set_gpio(btc, CXDGPIO_EN_MAP, map);
+
+		CLI_PRT(" signal to gpio debug map = 0x%08x!!\n", map);
+		return;
+	}
+
+	CLI_PRT(" sig <enable_map:hex/32bits> ");
+	CLI_PRT("(Default Signal->GPIO: No-Assign->0, GNT_WL->2, GNT_BT->3, ");
+	CLI_PRT("WL_CCA->1, WL_ERR->6, WL_OK->7)\n");
+
+	for (i = 0; i < BTC_DBG_MAX1; i++) {
+		if ((i+1) % 8 == 1)
+			CLI_PRT("      %d: %s", i, id_to_str(BTC_STR_GDBG, i));
+		else
+			CLI_PRT(", %d: %s", i, id_to_str(BTC_STR_GDBG, i));
+
+		if (((i+1) % 8 == 0) || (i == BTC_DBG_MAX1 - 1))
+			CLI_PRT("\n");
+	}
+}
+
+static void _cmd_sgpio_map(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			   u32 input_num, char *output, u32 out_len)
+{
+	struct btc_cx *cx = &btc->cx;
+	u32 sig = 0, gpio = 0, used_gpio_map = 0;
+
+	if (input_num >= 3) {
+		_os_sscanf(input[1], "%d", &sig);
+		_os_sscanf(input[2], "%d", &gpio);
+	}
+
+	if ((sig >= BTC_DBG_MAX1 || gpio > 15) || input_num < 3) {
+		CLI_PRT(" gpio <sig:0~31> <gpio:0~15>\n");
+		return;
+	}
+
+	used_gpio_map = cx->other.ext_hpta_cfg | cx->other.ext_hmbx_cfg |
+			cx->other.ext_swout_cfg |
+			(cx->other.ext_swin_cfg & bMASKLW);
+	if (BIT(gpio) & used_gpio_map) {
+		CLI_PRT(" gpio setup conflict with ext-SOC interface!!\n");
+		return;
+	}
+
+	hal_btc_fw_set_gpio(btc, CXDGPIO_MUX_MAP, ((gpio << 8) + sig));
+
+	CLI_PRT(" signal-%s -> gpio-%d\n", id_to_str(BTC_STR_GDBG, sig), gpio);
+}
+
+static void _cmd_wl_tx_power(struct btc_t *btc, u32 *used,
+			     char input[][MAX_ARGV], u32 input_num,
+			     char *output, u32 out_len)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u32 pwr = 0;
+	u8 is_negative = 0;
+
+	if (input_num >= 2) {
+		_os_sscanf(input[1], "%d", &pwr);
+		if (pwr & BIT(31)) {
+			pwr = ~pwr + 1;
+			is_negative = 1;
+		}
+		pwr = pwr & bMASKB0;
+	}
+
+	if ((pwr != 255 && pwr > 20) || input_num < 2) {
+		CLI_PRT(" wpwr <wl_tx_power(dBm): -20dbm ~ +20dBm,"
+			" 255-> original tx power\n");
+		return;
+	}
+
+	if (is_negative)
+		pwr |= BIT(7);
+
+	wl->rf_para.tx_pwr_freerun = pwr;
+	_set_wl_tx_power(btc, pwr); /* pwr --> 1's complement */
+
+	if (pwr == 0xff)
+		CLI_PRT(" set wl tx power level to original!!\n");
+	else if (is_negative)
+		CLI_PRT(" set wl tx power level = -%d dBm!!\n", pwr & 0x7f);
+	else
+		CLI_PRT(" set wl tx power level = +%d dBm!!\n", pwr & 0x7f);
+}
+
+static void _cmd_wl_rx_lna(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			   u32 input_num, char *output, u32 out_len)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	u32 lna = 0;
+
+	if (input_num < 2) {
+		CLI_PRT(" wlna <wl_rx_level: 0~7> \n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &lna);
+
+	wl->rf_para.rx_gain_freerun = lna;
+	_set_wl_rx_gain(btc, lna);
+	CLI_PRT(" set wl rx level = %d!!\n", lna);
+}
+
+static void _cmd_bt_afh_map(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	struct btc_wl_afh_info *wl_afh = &btc->cx.wl.afh_info;
+	u32 en = 0, ch = 0, bw = 0;
+	u8 buf[3] = {0};
+
+	if (input_num < 4) {
+		CLI_PRT(" afh <en: 0/1> <ch: 0~255> <bw: 0~255>\n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &en);
+	_os_sscanf(input[2], "%d", &ch);
+	_os_sscanf(input[3], "%d", &bw);
+
+	buf[0] = (u8)(en & 0xff);
+	buf[1] = (u8)(ch & 0xff);
+	buf[2] = (u8)(bw & 0xff);
+
+	if (_send_fw_cmd(btc, SET_BT_WL_CH_INFO, buf, 3)) {
+		wl_afh->en = buf[0];
+		wl_afh->ch = buf[1];
+		wl_afh->bw = buf[2];
+		CLI_PRT(" set bt afh map: en=%d, ch=%d, map=%d\n", en, ch, bw);
+	} else {
+		CLI_PRT(" set bt afh map fail!!");
+	}
+}
+
+static void _cmd_bt_tx_power(struct btc_t *btc, u32 *used,
+			     char input[][MAX_ARGV], u32 input_num,
+			     char *output, u32 out_len)
+{
+	struct btc_bt_info *bt = &btc->cx.bt;
+	s32 pwr = 0;
+	u32 pwr2;
+
+	if (input_num < 2) {
+		CLI_PRT(" bpwr <decrease power: 0~255 > \n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &pwr);
+
+	pwr2 = pwr & bMASKB0;
+
+	bt->rf_para.tx_pwr_freerun = pwr2;
+	_set_bt_tx_power(btc, pwr2);
+	CLI_PRT(" decrease bt tx power level = %d!!\n", pwr2);
+}
+
+static void _cmd_bt_rx_lna(struct btc_t *btc, u32 *used, char input[][MAX_ARGV],
+			   u32 input_num, char *output, u32 out_len)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_bt_info *bt = &btc->cx.bt;
+	u32 lna = 0;
+	u8 buf;
+
+	if (input_num >= 2)
+		_os_sscanf(input[1], "%d", &lna);
+
+	if (input_num < 2 || lna > BTC_BT_RX_NORMAL_LVL) {
+		CLI_PRT(" blna <lna_constrain: 0~7> \n");
+		return;
+	}
+
+	buf = (u8)(lna & bMASKB0);
+
+	if (!_send_fw_cmd(btc, SET_BT_LNA_CONSTRAIN, &buf, 1)) {
+		CLI_PRT(" set bt rx lna constrain level fail!!\n");
+		return;
+	}
+
+	bt->rf_para.rx_gain_freerun = lna;
+	btc->dm.rf_trx_para.bt_rx_gain = lna;
+
+	_os_delay_us(btc->hal, BTC_SCBD_REWRITE_DELAY);
+
+	if (buf == BTC_BT_RX_NORMAL_LVL)
+		_write_scbd(btc, BTC_WSCB_RXGAIN, false);
+	else
+		_write_scbd(btc, BTC_WSCB_RXGAIN, true);
+
+	rtw_hal_mac_set_scoreboard(btc->hal, &wl->scbd);
+
+	CLI_PRT(" set bt rx lna constrain level = %d!!\n", buf);
+}
+
+static void _cmd_bt_igno_wl(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	u32 igno = 0;
+
+	if (input_num < 2) {
+		CLI_PRT(" igwl <0: don't ignore wlan, 1: ignore wlan > \n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &igno);
+
+	igno = igno & BIT(0);
+
+	_set_bt_ignore_wl_act(btc, (u8)igno);
+	CLI_PRT(" set bt ignore wlan = %d!!\n", igno);
+}
+
+static void _cmd_set_gnt_wl(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	u32 gwl = 0, phy_map = BTC_PHY_ALL, val = 0;
+
+	if (input_num != 2 && input_num != 3) {
+		CLI_PRT(" gwl <0:SW_0, 1:SW_1, 2:HW_PTA>"
+			" <b0->0:PHY-0/1:PHY-1, b1->0:bt0/1:bt1, none:All> \n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &gwl);
+
+	btc->dm.bt_select = BTC_ALL_BT;
+	if (input_num == 3) {
+		_os_sscanf(input[2], "%d", &val);
+		phy_map = ((val & BIT(0))? BTC_PHY_1 : BTC_PHY_0);
+		btc->dm.bt_select = ((val & BIT(1))? BTC_BT_1 : BTC_BT_0);
+	}
+
+	switch(gwl) {
+	case 0:
+		_set_gnt(btc, (u8)phy_map, BTC_GNT_SW_LO, BTC_GNT_SET_SKIP,
+			 BTC_WLACT_SW_LO);
+		CLI_PRT(" set gnt_wl = SW-0");
+		break;
+	case 1:
+		_set_gnt(btc, (u8)phy_map, BTC_GNT_SW_HI, BTC_GNT_SET_SKIP,
+			 BTC_WLACT_SW_HI);
+		CLI_PRT(" set gnt_wl = SW-1");
+		break;
+	case 2:
+		_set_gnt(btc, (u8)phy_map, BTC_GNT_HW, BTC_GNT_SET_SKIP,
+			 BTC_WLACT_HW);
+		CLI_PRT(" set gnt_wl = HW-PTA ctrl");
+		break;
+	}
+
+	CLI_PRT(" (phy_map=0x%x, bt_sel=0x%x)\n", phy_map, btc->dm.bt_select+1);
+}
+
+static void _cmd_set_gnt_bt(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	u32 gbt = 0, phy_map = BTC_PHY_ALL, val = 0;
+
+	if (input_num != 2 && input_num != 3) {
+		CLI_PRT(" gbt <0:SW_0, 1:SW_1, 2:HW_PTA>"
+			" <b0->0:PHY-0/1:PHY-1, b1->0:bt0/1:bt1, none:All> \n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &gbt);
+
+	btc->dm.bt_select = BTC_ALL_BT;
+	if (input_num == 3) {
+		_os_sscanf(input[2], "%d", &val);
+		phy_map = ((val & BIT(0))? BTC_PHY_1 : BTC_PHY_0);
+		btc->dm.bt_select = ((val & BIT(1))? BTC_BT_1 : BTC_BT_0);
+	}
+
+	switch(gbt) {
+	case 0:
+		_set_gnt(btc, (u8)phy_map, BTC_GNT_SET_SKIP, BTC_GNT_SW_LO,
+			 BTC_WLACT_SW_HI);
+		CLI_PRT(" set gnt_bt = SW-0");
+		break;
+	case 1:
+		_set_gnt(btc, (u8)phy_map, BTC_GNT_SET_SKIP, BTC_GNT_SW_HI,
+			 BTC_WLACT_SW_LO);
+		CLI_PRT(" set gnt_bt = SW-1");
+		break;
+	case 2:
+		_set_gnt(btc, (u8)phy_map, BTC_GNT_SET_SKIP, BTC_GNT_HW,
+			 BTC_WLACT_HW);
+		CLI_PRT(" set gnt_bt = HW-PTA ctrl");
+		break;
+	}
+
+	CLI_PRT(" (phy_map=0x%x, bt_sel=0x%x)\n", phy_map, btc->dm.bt_select+1);
+}
+
+static void _cmd_set_bt_psd(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	u32 idx = 0, type = 0;
+
+	if (input_num < 3) {
+		CLI_PRT(" bpsd <start_idex, dec-8bit>"
+			"<report_type, dec-8bit> \n");
+		return;
+	}
+
+	_os_sscanf(input[1], "%d", &idx);
+	_os_sscanf(input[2], "%d", &type);
+
+	_bt_psd_setup(btc, (u8)idx, (u8)type);
+}
+
+s8 _get_bw_att_db(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	s8 bw_att_db = 13;
+
+	if (wl->afh_info.bw >= CHANNEL_WIDTH_5)
+		return bw_att_db;
+
+	if (wl->afh_info.bw != CHANNEL_WIDTH_80_80)
+		bw_att_db += 3 * (s8)(wl->afh_info.bw - CHANNEL_WIDTH_20);
+	else
+		bw_att_db += 3 * 3;
+
+	return bw_att_db;
+}
+
+void _get_wl_nhm_dbm(struct btc_t *btc)
+{
+	struct btc_wl_nhm *wl_nhm = &btc->cx.wl.nhm;
+	s8 bw_att_db = 0, nhm_pwr_dbm = 0, pwr = 0, cur_pwr = 0;
+	u16 save_index = 0;
+	bool new_data_flag = false;
+	struct watchdog_nhm_report nhm_rpt = {0};
+
+	bw_att_db = _get_bw_att_db(btc);
+
+	rtw_hal_bb_nhm_mntr_result(btc->hal, &nhm_rpt, HW_PHY_0);
+
+	if (!nhm_rpt.ccx_rpt_result)
+		return;
+
+	nhm_pwr_dbm = nhm_rpt.nhm_pwr_dbm;
+
+	wl_nhm->instant_wl_nhm_dbm = nhm_pwr_dbm;
+
+	if (wl_nhm->start_flag == false) {
+		wl_nhm->start_flag = true;
+		pwr = nhm_pwr_dbm - bw_att_db;
+		new_data_flag = true;
+	} else {
+		pwr = wl_nhm->pwr;
+		if (wl_nhm->last_ccx_rpt_stamp != nhm_rpt.ccx_rpt_stamp) {
+			new_data_flag = true;
+			wl_nhm->current_status = 1; // new data
+		} else {
+			wl_nhm->current_status = 2; //duplicated data
+		}
+	}
+
+	if (!new_data_flag)
+		return;
+
+	wl_nhm->last_ccx_rpt_stamp = nhm_rpt.ccx_rpt_stamp;
+	cur_pwr = nhm_pwr_dbm - bw_att_db;
+	wl_nhm->instant_wl_nhm_per_mhz = cur_pwr;
+
+	/*  reset max/min by 100 record time */
+	if (wl_nhm->valid_record_times % 100 == 0) {
+		wl_nhm->pwr_max = -127;
+		wl_nhm->pwr_min = 0;
+	}
+
+	wl_nhm->valid_record_times += 1;
+
+	if (wl_nhm->valid_record_times == 0) {
+		wl_nhm->valid_record_times = 16;
+	}
+
+	wl_nhm->ratio = nhm_rpt.nhm_ratio;
+
+	if (cur_pwr < pwr) {
+		pwr = cur_pwr;
+		wl_nhm->refresh = true;
+	} else {
+		pwr = (pwr >> 1) + (cur_pwr >> 1);
+		wl_nhm->refresh = false;
+	}
+
+	save_index = ((wl_nhm->valid_record_times + 16 - 1) % 16) & 0xf;
+
+	wl_nhm->pwr = pwr;
+	wl_nhm->record_pwr[save_index] = pwr;
+	wl_nhm->record_ratio[save_index] = nhm_rpt.nhm_ratio;
+
+	if (wl_nhm->pwr > wl_nhm->pwr_max)
+		wl_nhm->pwr_max = wl_nhm->pwr;
+
+	if (wl_nhm->pwr < wl_nhm->pwr_min)
+		wl_nhm->pwr_min = wl_nhm->pwr;
+}
+
+static void _cmd_get_wl_nhm(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_wl_nhm *wl_nhm = &wl->nhm;
+	u16 start_index = 0, stop_index = 0, i;
+	u32 show_record_num = 16;
+
+	if (input_num >= 3) {
+		CLI_PRT(" wnhm <last_record_num, dec,1-16, default:16> \n");
+		return;
+	}
+
+	_get_wl_nhm_dbm(btc);
+
+	_os_sscanf(input[1], "%d", &show_record_num);
+
+	if (show_record_num >= 16)
+		show_record_num = 16;
+	else if (show_record_num == 0)
+		show_record_num = 1;
+
+	if (wl_nhm->valid_record_times == 0) {
+		CLI_PRT("wl nhm not ready\n");
+		return;
+	}
+
+	if (wl_nhm->current_status == 0) {
+		CLI_PRT("wl nhm failed this time\n");
+	}
+
+	CLI_PRT("nhm_psd = %d dBm/MHz, nhm_ratio = %d, status = %d, "
+		"valid_record_times = %d\n",
+		wl_nhm->pwr, wl_nhm->ratio,
+		wl_nhm->current_status, wl_nhm->valid_record_times);
+
+	stop_index = wl_nhm->valid_record_times;
+	if (wl_nhm->valid_record_times > (u16)show_record_num)
+		start_index = wl_nhm->valid_record_times - (u16)show_record_num;
+
+	for (i = start_index; i < stop_index; i++) {
+		if (i == start_index) {
+			CLI_PRT("record_pwr(old->new)= %4d",
+				wl_nhm->record_pwr[(i + 16) & 0xF]);
+		} else if (i == (stop_index - 1)) {
+			CLI_PRT(", %4d dBm/MHz\n",
+				wl_nhm->record_pwr[(i + 16) & 0xF]);
+		} else {
+			CLI_PRT(", %4d", wl_nhm->record_pwr[(i + 16) & 0xF]);
+		}
+	}
+
+	for (i = start_index; i < stop_index; i++) {
+		if (i == start_index) {
+			CLI_PRT("record_ratio(old->new)= %4d",
+				wl_nhm->record_ratio[(i + 16) & 0xF]);
+		} else if (i == (stop_index - 1)) {
+			CLI_PRT(", %4d percent\n",
+				wl_nhm->record_ratio[(i + 16) & 0xF]);
+		} else {
+			CLI_PRT(", %4d", wl_nhm->record_ratio[(i + 16) & 0xF]);
+		}
+	}
+}
+
+void _get_wl_cn_report(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	wl->cn_report = rtw_hal_ex_cn_report(btc->hal);
+}
+
+
+void _get_wl_evm_report(struct btc_t *btc)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	wl->evm_1ss_rpt = rtw_hal_ex_evm_1ss_report(btc->hal);
+	wl->evm_2ss_max_rpt = rtw_hal_ex_evm_max_report(btc->hal);
+	wl->evm_2ss_min_rpt = rtw_hal_ex_evm_min_report(btc->hal);
+}
+
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+static void _cmd_fddt_cell(struct btc_t *btc, u32 *used,
+			   char input[][MAX_ARGV], u32 type,
+			   char *output, u32 out_len)
+{
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_fddt_sta *pfddt = NULL;
+	u8 i, j, r_lvl, m_lvl = BTC_BT_RSSI_THMAX;
+	s8 rssi_th, rssi_th2, para1, para2, para3, para4;
+	struct btc_fddt_train_info *train_now = &btc->dm.fddt_info.train_now;
+	struct btc_fddt_cell cell[5][5];
+	struct fbtc_fddt_cell_status cell_c2h[5][5];
+
+	if (type == BTC_FDDT_CELL_STATE_UL || type == BTC_FDDT_CELL_STATE_DL) {
+		pcinfo = &pfwinfo->rpt_fbtc_fddt.cinfo;
+		if (!pcinfo->valid)
+			return;
+
+		pfddt = &pfwinfo->rpt_fbtc_fddt.finfo;
+	}
+
+	CLI_PRT("\n============ ");
+
+	switch (type) {
+	case BTC_FDDT_CELL_RANGE_UL:
+		CLI_PRT("Cell-Range-UL (WL_Tx_min/WL_Tx_max/BT_Tx_min/BT_Rx)");
+		hal_mem_cpy(btc->hal, cell, train_now->cell_ul, sizeof(cell));
+		break;
+	case BTC_FDDT_CELL_STATE_UL:
+		CLI_PRT("Cell-State-UL (WL_Tx/BT_Tx_Dec/BT_Rx/Cell_Status)");
+		hal_mem_cpy(btc->hal, cell_c2h, pfddt->fddt_cells[0],
+			    sizeof(cell_c2h));
+		break;
+	case BTC_FDDT_CELL_STATE_DL:
+		CLI_PRT("Cell-State-DL (WL_Tx/BT_Tx_Dec/BT_Rx/Cell_Status)");
+		hal_mem_cpy(btc->hal, cell_c2h, pfddt->fddt_cells[1],
+			    sizeof(cell_c2h));
+		break;
+	default:
+		CLI_PRT("Cell-Range-DL (WL_Tx_min/WL_Tx_max/BT_Tx_min/BT_Rx)");
+		hal_mem_cpy(btc->hal, cell, train_now->cell_dl, sizeof(cell));
+		break;
+	}
+
+	CLI_PRT(" ============\nWL_RSSI		 > %3d    ",
+		btc->chip->wl_rssi_thres[0] - 110);
+
+	for (i = 0; i < m_lvl; i++) {
+		rssi_th = btc->chip->wl_rssi_thres[i] - 110;
+		if (i == m_lvl - 1) {
+			CLI_PRT("   < %3d   \n", rssi_th);
+		} else {
+			rssi_th2 = btc->chip->wl_rssi_thres[i+1] - 110;
+			CLI_PRT(" %3d ~ %3d  ", rssi_th, rssi_th2);
+		}
+	}
+
+	CLI_PRT("BT_RSSI > %d:    ", btc->chip->bt_rssi_thres[0] - 100);
+
+	for (i = 0; i <= m_lvl; i++) {
+		if (type == BTC_FDDT_CELL_STATE_UL ||
+		    type == BTC_FDDT_CELL_STATE_DL) {
+			para1 = cell_c2h[m_lvl][m_lvl-i].wl_tx_pwr;
+			para2 = cell_c2h[m_lvl][m_lvl-i].bt_tx_pwr;
+			para3 = cell_c2h[m_lvl][m_lvl-i].bt_rx_gain;
+			para4 = cell_c2h[m_lvl][m_lvl-i].state_phase;
+		} else {
+			para1 = cell[m_lvl][m_lvl-i].wl_pwr_min;
+			para2 = cell[m_lvl][m_lvl-i].wl_pwr_max;
+			para3 = cell[m_lvl][m_lvl-i].bt_pwr_dec_max;
+			para4 = cell[m_lvl][m_lvl-i].bt_rx_gain;
+		}
+
+		CLI_PRT(" %2d/%2d/%2d/%2x", para1, para2, para3, para4);
+	}
+
+	CLI_PRT("\n");
+
+	for (j = 0; j < m_lvl; j++) {
+		rssi_th = btc->chip->bt_rssi_thres[j] - 100;
+		r_lvl = m_lvl - j - 1;
+
+		if (j == m_lvl - 1) {
+			CLI_PRT("BT_RSSI < %d:    ", rssi_th);
+		} else {
+			rssi_th2 = btc->chip->bt_rssi_thres[j+1] - 100;
+			CLI_PRT("BT_RSSI %d ~ %d:", rssi_th, rssi_th2);
+		}
+
+		for (i = 0; i <= 4; i++) {
+			if (type == BTC_FDDT_CELL_STATE_UL ||
+			    type == BTC_FDDT_CELL_STATE_DL) {
+				para1 = cell_c2h[r_lvl][m_lvl-i].wl_tx_pwr;
+				para2 = cell_c2h[r_lvl][m_lvl-i].bt_tx_pwr;
+				para3 = cell_c2h[r_lvl][m_lvl-i].bt_rx_gain;
+				para4 = cell_c2h[r_lvl][m_lvl-i].state_phase;
+			} else {
+				para1 = cell[r_lvl][m_lvl-i].wl_pwr_min;
+				para2 = cell[r_lvl][m_lvl-i].wl_pwr_max;
+				para3 = cell[r_lvl][m_lvl-i].bt_pwr_dec_max;
+				para4 = cell[r_lvl][m_lvl-i].bt_rx_gain;
+			}
+
+			CLI_PRT(" %2d/%2d/%2d/%2x", para1, para2, para3, para4);
+		}
+
+		CLI_PRT("\n");
+	}
+}
+
+static void _show_fddt_cycle(struct btc_t *btc, u32 *used,
+			     char input[][MAX_ARGV], u32 input_num,
+			     char *output, u32 out_len)
+{
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct fbtc_fddt_sta *pfddt = NULL;
+	struct fbtc_fddt_cycle_info c;
+	u8 cnt = 0;
+	u16 cycle, c_begin, c_end, s_id, cycle_total;
+
+	pcinfo = &pfwinfo->rpt_fbtc_fddt.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	pfddt = &pfwinfo->rpt_fbtc_fddt.finfo;
+	cycle_total = (pfddt->cycles_h8 << 8) + pfddt->cycles_l8;
+
+	if (cycle_total <= 1)
+		return;
+
+	if (cycle_total <= BTC_CYCLE_SLOT_MAX)
+		c_begin = 1;
+	else
+		c_begin = cycle_total - BTC_CYCLE_SLOT_MAX + 1;
+
+	c_end = cycle_total;
+
+	if (c_begin > c_end)
+		return;
+
+	CLI_PRT("\n============ Cycle ([RSSI]-[Train]"
+		 "-[TRx_Para]-[mW_KPI]-[mB_KPI])============\n");
+
+	for (cycle = c_begin; cycle <= c_end; cycle++) {
+		cnt++;
+		s_id = (cycle-1) % BTC_CYCLE_SLOT_MAX;
+		c = pfddt->fddt_trx[s_id];
+
+		CLI_PRT("No.%d", cycle);
+
+		CLI_PRT("([W%d/B%d/CN%d]-[Cy%d/St%d->S%d/P%d/R0x%x]-"
+			"[WT%d/WR%d/BT%d/BR%d]-[TP:%d]-[ne:%d])-->",
+			c.rssi & 0xf, (c.rssi & 0xf0) >> 4, c.cn,
+			c.phase_cycle, c.train_step,
+			(c.train_status & 0xf0) >> 4, c.train_status & 0xf,
+			c.train_result,
+			c.tx_power, wl->rf_para.rx_gain_freerun,
+			c.bt_tx_power, c.bt_rx_gain,
+			(c.tp_h8 << 8) + c.tp_l8,
+			c.no_empty_cnt);
+
+		if (cnt % 2 == 0)
+			CLI_PRT("\n");
+	}
+}
+
+static void _cmd_fddt_show(struct btc_t *btc, u32 *used,
+			   char input[][MAX_ARGV], u32 input_num,
+			   char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_module *module = &btc->mdinfo;
+	struct btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct btc_bt_link_info *bt_linfo = &bt->link_info;
+	struct btc_wl_link_info *plink = &wl->link_info[0];
+	struct btc_wl_smap *wl_smap = &wl->status.map;
+	struct fbtc_rpt_ctrl *finfo = &btc->fwinfo.rpt_ctrl.finfo;
+	struct btc_rpt_ctrl_a2dp_empty *a2dp = &finfo->bt_mbx_info.a2dp;
+	struct btc_fddt_info *fdinfo = &dm->fddt_info;
+	struct btc_fddt_train_info *train_now = &fdinfo->train_now;
+	struct btc_rpt_ctrl_a2dp_empty *a2dp_last = &fdinfo->bt_stat.a2dp_last;
+	u32 ratio_ack = 0, ratio_nack = 0, show_map = BTC_FDDT_INFO_MAP_DEF;
+	u32 n;
+
+	if (input_num > 1)
+		_os_sscanf(input[1], "%x", &show_map);
+
+	if (show_map & BIT(0)) {
+		CLI_PRT("============ %s()  ============\n", __func__);
+
+		CLI_PRT("enable=%d, count=%d, type=%s, state=%s, result=%d,"
+			" rsn_map=0x%x, tdma_policy=%s, aiso_sort_avg=%d\n",
+			dm->fddt_train, dm->cnt_dm[BTC_DCNT_FDDT_TRIG],
+			id_to_str(BTC_STR_FDDT_TYPE, (u32)fdinfo->type),
+			id_to_str(BTC_STR_FDDT_STATE, (u32)fdinfo->state),
+			fdinfo->result, fdinfo->nrsn_map,
+			id_to_str(BTC_STR_POLICY, (u32)btc->policy_type),
+			module->ant.isolation);
+
+		CLI_PRT("[Time_Ctrl] m_cycle=%d, w_cycle=%d, k_cycle=%d\n",
+			train_now->t_ctrl.m_cycle,
+			train_now->t_ctrl.w_cycle,
+			train_now->t_ctrl.k_cycle);
+
+		CLI_PRT("[Pass_Ctrl] map=0x%x, no_empty_cnt<%d, tp_ratio=%d, "
+			"kpibtr_ratio=%d\n",
+			train_now->f_chk.check_map,
+			train_now->f_chk.bt_no_empty_cnt,
+			train_now->f_chk.wl_tp_ratio,
+			train_now->f_chk.wl_kpibtr_ratio);
+
+		CLI_PRT("[Break_Ctrl] map=0x%x, no_empty_cnt>=%d, tp_ratio=%d,",
+			train_now->b_chk.check_map,
+			train_now->b_chk.bt_no_empty_cnt,
+			train_now->b_chk.wl_tp_ratio);
+
+		CLI_PRT(" tp_low=%d, cn=%d, cell_chg=%d, cn_limit=%d, "
+			"nhm_limit=%d\n",
+			train_now->b_chk.wl_tp_low_bound,
+			train_now->b_chk.cn,
+			train_now->b_chk.cell_chg,
+			train_now->b_chk.cn_limit,
+			train_now->b_chk.nhm_limit);
+	}
+
+	if (show_map & BIT(1))
+		_cmd_fddt_cell(btc, used, input, wl_smap->traffic_dir,
+			       output, out_len);
+
+	if (show_map & BIT(2)) {
+		CLI_PRT("\n============ WL ============\n");
+
+		CLI_PRT("link_mode=%s, rssi=-%ddBm(%d, level=%d), busy=%d,"
+			" nhm=%d, cn=%d\n",
+			id_to_str(BTC_STR_WLLINK, (u32)wl_rinfo->link_mode),
+			110-plink->rssi, plink->rssi, wl->rssi_level,
+			wl_smap->busy, wl->nhm.pwr, wl->cn_report);
+
+		CLI_PRT("evm(1ss:%d, 2ss_max:%d, 2ss_min:%d)\n",
+			wl->evm_1ss_rpt,
+			wl->evm_2ss_max_rpt, wl->evm_2ss_min_rpt);
+
+		CLI_PRT("dir=%s, tx[tp:%d Mbps/byte:%d/rate:%s/busy_level:%d],"
+			" rx[tp:%d Mbps/byte:%d/rate:%s/busy_level:%d]\n",
+			(wl_smap->traffic_dir & BIT(TRAFFIC_UL) ? "UL" : "DL"),
+			wl->traffic.tx_tp, (u32)wl->traffic.tx_byte,
+			id_to_str(BTC_STR_RATE, (u32)plink->tx_rate),
+			wl->traffic.tx_lvl,
+			wl->traffic.rx_tp, (u32)wl->traffic.rx_byte,
+			id_to_str(BTC_STR_RATE, (u32)plink->rx_rate),
+			wl->traffic.rx_lvl);
+	}
+
+	if (show_map & BIT(3)) {
+		CLI_PRT("\n============ BT ============\n");
+
+		CLI_PRT("state=0x%x, rssi=%ddBm(%d, level=%d), role:%s,"
+			" tx_rate:%dM, rate_chg:%d\n",
+			bt->raw_info[2], bt_linfo->rssi-100,
+			bt_linfo->rssi, bt->rssi_level,
+			(bt_linfo->slave_role ? "Slave" : "Master"),
+			(bt_linfo->tx_3M? 3 : 2),
+			cx->cnt_bt[BTC_BCNT_RATECHG]);
+
+		if (a2dp->cnt_tx != 0) {
+			ratio_ack = a2dp->cnt_ack * 100 / a2dp->cnt_tx;
+			ratio_nack = a2dp->cnt_nack * 100 / a2dp->cnt_tx;
+		}
+
+		CLI_PRT("Retry:%d, A2DP_empty:%d"
+			"[stop:%d/tx:%d/ack:%d(%d)/nack:%d(%d)]\n",
+			cx->cnt_bt[BTC_BCNT_RETRY],
+			a2dp->cnt_empty, a2dp->cnt_flowctrl, a2dp->cnt_tx,
+			a2dp->cnt_ack, ratio_ack, a2dp->cnt_nack, ratio_nack);
+
+		n = a2dp->cnt_tx - a2dp_last->cnt_tx;
+
+		if (n > 0) {
+			ratio_ack = (a2dp->cnt_ack - a2dp_last->cnt_ack) * 100;
+			ratio_ack /= n;
+			ratio_nack = (a2dp->cnt_nack - a2dp_last->cnt_nack)*100;
+			ratio_nack /= n;
+		}
+
+		n = cx->cnt_bt[BTC_BCNT_RETRY] - fdinfo->bt_stat.retry_last;
+
+		CLI_PRT("Retry_diff:%d, A2DP_empty_diff:%d"
+			"[stop:%d/tx:%d/ack:%d(%d)/nack:%d(%d)]\n",
+			n, a2dp->cnt_empty - a2dp_last->cnt_empty,
+			a2dp->cnt_flowctrl - a2dp_last->cnt_flowctrl,
+			a2dp->cnt_tx - a2dp_last->cnt_tx,
+			a2dp->cnt_ack - a2dp_last->cnt_ack, ratio_ack,
+			a2dp->cnt_nack - a2dp_last->cnt_nack, ratio_nack);
+
+		n = sizeof(struct btc_rpt_ctrl_a2dp_empty);
+		fdinfo->bt_stat.retry_last = cx->cnt_bt[BTC_BCNT_RETRY];
+		hal_mem_cpy(btc->hal, a2dp_last, a2dp, n);
+	}
+
+	if (show_map & BIT(4))
+		_show_fddt_cycle(btc, used, input, input_num, output, out_len);
+
+	n = wl_smap->traffic_dir | 0x80;
+	if (show_map & BIT(5))
+		_cmd_fddt_cell(btc, used, input, n, output, out_len);
+}
+
+static void _cmd_fddt_set(struct btc_t *btc, u32 *used,
+			  char input[][MAX_ARGV], u32 input_num,
+			  char *output, u32 out_len)
+{
+	u32 para = 0;
+
+	if (input_num > 1)
+		_os_sscanf(input[1], "%d", &para);
+
+	if (input_num <= 1 || para >= BTC_FDDT_MAX) {
+		CLI_PRT(" ftset <val--- 0:force-stop, 1:auto,"
+			" 2:fix-tdd-phase, 3:fix-fdd-phase> \n");
+		return;
+	}
+
+	btc->dm.fddt_info.type = (u8) para;
+
+	_run_coex(btc, __func__);
+	CLI_PRT(" FDD-Train para = %d!!\n", para);
+}
+
+static void _cmd_fddt_ctrl(struct btc_t *btc, u32 *used,
+			   char input[][MAX_ARGV], u32 input_num,
+			   char *output, u32 out_len)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btc_module *module = &btc->mdinfo;
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_fddt_train_info *t = &dm->fddt_info.train;
+	u32 type = 0, para = 0, para1 = 0;
+	u8 buf[7] = {0};
+	s8 wl_rssi_th, bt_rssi_th, rssi_tol;
+
+	_os_sscanf(input[1], "%d", &type);
+
+	if (input_num <= 1 || type > 6) {
+		CLI_PRT(" ftctrl <val--- 0:time_cycle, 1:break_check,"
+			" 2:fail_check, 3:cell_para, 4:cell_update, 5:cell_wl_rssi_update,"
+			" 6:cell_bt_rssi_update>\n");
+		return;
+	} else if (input_num <= 3) {
+		switch (type) {
+		case 0:
+			CLI_PRT(" time_cycle: ftctrl 0 "
+				"<1:m_cycle, 2:w_cycle, 3:k_cycle>\n");
+			return;
+		case 1:
+			CLI_PRT(" break_check: ftctrl 1 "
+				"<1:bt_no_ept_cnt, 2:wl_tp_ratio,"
+				" 3:wl_tp_low_bound, 4:cn, 5:cell_chg,"
+				" 6: cn_limit, 7: nhm_limit, 8: rssi_tol> \n");
+			return;
+		case 2:
+			CLI_PRT(" fail_check: ftctrl 2 "
+				"<1:bt_no_ept_cnt, 2:wl_tp_ratio,"
+				" 3:wl_kpibtr_ratio> \n");
+			return;
+		case 3:
+			CLI_PRT(" cell_para: ftctrl 3 "
+				"<cell_index:24bits> <cell_para:32bits> \n");
+			CLI_PRT(" <cell_index>\n");
+			CLI_PRT("   bit[23:16]= wl_dir 0:UL, 1:DL\n");
+			CLI_PRT("   bit[15:8] = bt_rssi 0~4 \n");
+			CLI_PRT("   bit[7:0]  = wl_rssi 0~4 \n");
+			CLI_PRT(" <cell_para>\n");
+			CLI_PRT("   bit[31:24]= wl_pwr_max 0~0xf \n");
+			CLI_PRT("   bit[23:16]= wl_pwr_min 0~0xf \n");
+			CLI_PRT("   bit[15:8] = bt_pwr_dec_max 0~0x28 \n");
+			CLI_PRT("   bit[7:0]  = bt_rx_gain 0x7~0x4 \n");
+			return;
+		case 4:
+			CLI_PRT(" cell_update: ftctrl 4 "
+				 "<0:reset to default, 1:update by abt_iso>\n");
+			return;
+		case 5:
+			CLI_PRT(" cell_wl_rssi_update: ftctrl 5 "
+					"<0:wl_rssi_thres[0], 1:wl_rssi_thres[1],"
+					"2:wl_rssi_thres[2], 3:wl_rssi_thres[3]>\n");
+			return;
+		case 6:
+			CLI_PRT(" cell_bt_rssi_update: ftctrl 6 "
+					"<0:bt_rssi_thres[0], 1:bt_rssi_thres[1],"
+					"2:bt_rssi_thres[2], 3:bt_rssi_thres[3]>\n");
+			return;
+		}
+	}
+
+	if (type != 3) {
+		_os_sscanf(input[2], "%d", &para);
+		_os_sscanf(input[3], "%d", &para1);
+	} else {
+		_os_sscanf(input[2], "%x", &para);
+		_os_sscanf(input[3], "%x", &para1);
+
+		/*buf[0] buf[1] buf[2] : UL/DL, bt_rssi->0~4, wl_rssi->0~4*/
+		buf[0] = (u8)((para & bMASKB2) >> 16);
+		buf[1] = (u8)((para & bMASKB1) >> 8);
+		buf[2] = (u8)(para & bMASKB0);
+
+		/*buf[3] buf[4] buf[5] buf[6] :
+		 * wl_pwr_max wl_pwr_min bt_pwr_dec_max bt_rx_gain
+		 */
+		buf[3] = (u8)((para1 & bMASKB3) >> 24);
+		buf[4] = (u8)((para1 & bMASKB2) >> 16);
+		buf[5] = (u8)((para1 & bMASKB1) >> 8);
+		buf[6] = (u8)(para1 & bMASKB0);
+	}
+
+	switch (type) {
+	case 0:
+		CLI_PRT(" FDDT Time_cycle setup\n");
+		if (para == 1) {
+			t->t_ctrl.m_cycle = (u8) para1;
+			CLI_PRT(" m_cycle=%d\n", para1);
+		} else if (para == 2) {
+			t->t_ctrl.w_cycle = (u8) para1;
+			CLI_PRT(" w_cycle=%d\n", para1);
+		} else {
+			t->t_ctrl.k_cycle = (u8) para1;
+			CLI_PRT(" k_cycle=%d\n", para1);
+		}
+		break;
+	case 1:
+		CLI_PRT(" FDDT Break_check setup\n");
+		if (para == 1) {
+			t->b_chk.bt_no_empty_cnt = (u8) para1;
+			CLI_PRT(" bt_no_ept_cnt=%d\n", para1);
+		} else if (para == 2) {
+			t->b_chk.wl_tp_ratio = (u8) para1;
+			CLI_PRT(" wl_tp_ratio=%d\n", para1);
+		} else if (para == 3) {
+			t->b_chk.wl_tp_low_bound = (u8) para1;
+			CLI_PRT(" wl_tp_low=%d\n", para1);
+		} else if (para == 4) {
+			t->b_chk.cn = (u8) para1;
+			CLI_PRT(" cn=%d\n", para1);
+		} else if (para == 5) {
+			t->b_chk.cell_chg = (u8) para1;
+			CLI_PRT(" cell_chg_cnt=%d\n", para1);
+		} else if (para == 6) {
+			t->b_chk.cn_limit = (u8) para1;
+			CLI_PRT(" cn_limit=%d\n", para1);
+		} else if (para == 7) {
+			t->b_chk.nhm_limit = (s8) para1;
+			CLI_PRT(" nhm_limit=%d\n", para1);
+		} else {
+			dm->rssi_offset2 = (u8) para1;
+			rssi_tol = btc->chip->rssi_tol + dm->rssi_offset2;
+			CLI_PRT(" rssi_tol=%d\n", rssi_tol);
+		}
+		break;
+	case 2:
+		CLI_PRT(" FDDT Fail_check setup\n");
+		if (para == 1) {
+			t->f_chk.bt_no_empty_cnt = (u8) para1;
+			CLI_PRT(" bt_no_ept_cnt=%d\n", para1);
+		} else if (para == 2) {
+			t->f_chk.wl_tp_ratio = (u8) para1;
+			CLI_PRT(" wl_tp_ratio=%d\n", para1);
+		} else {
+			t->f_chk.wl_kpibtr_ratio = (u8) para1;
+			CLI_PRT(" wl_kpibtr_ratio=%d\n", para1);
+		}
+		break;
+	case 3:
+		if (!buf[0]) {
+			t->cell_ul[buf[1]][buf[2]].wl_pwr_min = (u8) buf[3];
+			t->cell_ul[buf[1]][buf[2]].wl_pwr_max = (u8) buf[4];
+			t->cell_ul[buf[1]][buf[2]].bt_pwr_dec_max = (u8) buf[5];
+			t->cell_ul[buf[1]][buf[2]].bt_rx_gain = (u8) buf[6];
+			CLI_PRT(" FDDT Cell_para "
+				"[UL:%d] [bt:%d]/[wl:%d] = %d/%d/%d/%d!!\n",
+				buf[0], buf[1], buf[2], buf[3], buf[4],
+				buf[5], buf[6]);
+		} else {
+			t->cell_dl[buf[1]][buf[2]].wl_pwr_min = (u8) buf[3];
+			t->cell_dl[buf[1]][buf[2]].wl_pwr_max = (u8) buf[4];
+			t->cell_dl[buf[1]][buf[2]].bt_pwr_dec_max = (u8) buf[5];
+			t->cell_dl[buf[1]][buf[2]].bt_rx_gain = (u8) buf[6];
+			CLI_PRT(" FDDT Cell_para "
+				"[DL:%d] [bt:%d]/[wl:%d] = %d/%d/%d/%d!!\n",
+				buf[0], buf[1], buf[2], buf[3], buf[4],
+				buf[5], buf[6]);
+		}
+		break;
+	case 4:
+		if (para1 == 1) {
+			_set_fddt_cell_by_antiso(btc);
+			CLI_PRT(" FDDT Cell update by ant_iso = %d!\n",
+				module->ant.isolation);
+		} else {
+			hal_mem_cpy(h, t->cell_ul, cell_ul_def,
+				    sizeof(cell_ul_def));
+			hal_mem_cpy(h, t->cell_dl, cell_dl_def,
+				    sizeof(cell_dl_def));
+			CLI_PRT(" FDDT Cell reset to default!\n");
+		}
+		break;
+	case 5:
+		CLI_PRT(" FDDT cell_wl_rssi setup\n");
+		if (para == 0) {
+			btc->chip->wl_rssi_thres[0] = (u8) para1;
+			wl_rssi_th = btc->chip->wl_rssi_thres[0] - 110;
+			CLI_PRT(" rssi_thres[0]=%d, rssi_thres0=%ddBm\n", para1, wl_rssi_th);
+		} else if (para == 1) {
+			btc->chip->wl_rssi_thres[1] = (u8) para1;
+			wl_rssi_th = btc->chip->wl_rssi_thres[1] - 110;
+			CLI_PRT(" rssi_thres[1]=%d, rssi_thres1=%ddBm\n", para1, wl_rssi_th);
+		} else if (para == 2) {
+			btc->chip->wl_rssi_thres[2] = (u8) para1;
+			wl_rssi_th = btc->chip->wl_rssi_thres[2] - 110;
+			CLI_PRT(" rssi_thres[2]=%d, rssi_thres2=%ddBm\n", para1, wl_rssi_th);
+		} else {
+			btc->chip->wl_rssi_thres[3] = (u8) para1;
+			wl_rssi_th = btc->chip->wl_rssi_thres[3] - 110;
+			CLI_PRT(" rssi_thres[3]=%d, rssi_thres3=%ddBm\n", para1, wl_rssi_th);
+		}
+		break;
+	case 6:
+		CLI_PRT(" FDDT cell_bt_rssi setup\n");
+		if (para == 0) {
+			btc->chip->bt_rssi_thres[0] = (u8) para1;
+			bt_rssi_th = btc->chip->bt_rssi_thres[0] - 100;
+			CLI_PRT(" rssi_thres[0]=%d, rssi_thres0=%ddBm\n", para1, bt_rssi_th);
+		} else if (para == 1) {
+			btc->chip->bt_rssi_thres[1] = (u8) para1;
+			bt_rssi_th = btc->chip->bt_rssi_thres[1] - 100;
+			CLI_PRT(" rssi_thres[1]=%d, rssi_thres1=%ddBm\n", para1, bt_rssi_th);
+		} else if (para == 2) {
+			btc->chip->bt_rssi_thres[2] = (u8) para1;
+			bt_rssi_th = btc->chip->bt_rssi_thres[2] - 100;
+			CLI_PRT(" rssi_thres[2]=%d, rssi_thres2=%ddBm\n", para1, bt_rssi_th);
+		} else {
+			btc->chip->bt_rssi_thres[3] = (u8) para1;
+			bt_rssi_th = btc->chip->bt_rssi_thres[3] - 100;
+			CLI_PRT(" rssi_thres[3]=%d, rssi_thres3=%ddBm\n", para1, bt_rssi_th);
+		}
+		break;
+	default:
+		break;
+	}
+
+	_run_coex(btc, __func__);
+}
+#endif
+
+#ifdef BTC_AISO_SUPPORT
+static void _bt_psd_enable(struct btc_t *btc, bool enable)
+{
+	u8 buf[8] = {0};
+
+	buf[0] = 1;
+	buf[1] = 0x22;
+	buf[2] = 0x2;
+
+	if (enable)
+		buf[3] |= BIT(0);
+#if 0
+	if (fix_gain) {
+		buf[3] |= BIT(1);
+		buf[4] = (gain_idx & 0x3F);
+	}
+#endif
+
+	_send_fw_cmd(btc, SET_H2C_TEST, buf, 5);
+}
+
+void _bt_psd_setup(struct btc_t *btc, u8 start_idx, u8 auto_rpt_type)
+{
+	u8 buf[2] = {0};
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): set bt psd start_idx = %d, rpt_type = %d\n",
+		  __func__, start_idx, auto_rpt_type);
+
+	buf[0] = start_idx;
+	buf[1] = (auto_rpt_type & 0x3) | ((BT_PSD_SRC_RAW & 0x3) << 2);
+
+	_send_fw_cmd(btc, SET_BT_PSD_REPORT, buf, 2);
+}
+
+static bool _bt_psd_rpt_ctrl(struct btc_t *btc, bool en_psd)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	u8 cur_wl_ch = btc->cx.wl.afh_info.ch, start_index;
+	bool result = false;
+
+	if (en_psd && (cur_wl_ch >= 1 && cur_wl_ch <= 14) &&
+	    (!bt_psd_dm->aiso_data_ok && !bt_psd_dm->en)) {
+
+		if (cur_wl_ch < 14)
+			start_index = 3 + 5 * (cur_wl_ch - 1);
+		else
+			start_index = 75;
+
+		bt_psd_dm->en = true;
+		bt_psd_dm->rec_time_out = false;
+		result = true;
+
+		/* set query start index to BT */
+		_bt_psd_setup(btc, start_index, BT_PSD_RPT_TYPE_CAL);
+
+		_bt_psd_enable(btc, true);
+
+		PHL_INFO("[BTC], %s(): enable=%d: cur_wl_ch=%d, start_idx=%d\n",
+			 __func__, en_psd, cur_wl_ch, start_index);
+	} else if (!en_psd && bt_psd_dm->en) { /* disable */
+		_bt_psd_enable(btc, false);
+		bt_psd_dm->en = false;
+		result = true;
+
+		PHL_INFO("[BTC], %s(): disable!!\n", __func__);
+	}
+
+	return result;
+}
+
+static s8 _bt_psd_s16du8r(s16 sum, u8 num)
+{
+	s16 ans_int;
+	s16 ans_fra;
+
+	if (num == 0) {
+		PHL_INFO("[BTC], %s() Divider is zero\n", __func__);
+		return 1;
+	}
+
+	ans_int = (s16)(sum / num);
+	ans_fra = sum - ans_int * num;
+
+	if (ans_fra >= 0) {
+		if (ans_fra * 2 >= (s16)num)
+			ans_int++;
+	} else {
+		if (-ans_fra * 2 >= (s16)num)
+			ans_int--;
+	}
+	return (s8)ans_int;
+}
+
+static s16 _bt_psd_s32du16r(s32 sum, u16 num)
+{
+	s32 ans_int;
+	s32 ans_fra;
+
+	if (num == 0) {
+		PHL_INFO("[BTC], %s() Divider is zero\n", __func__);
+		return 1;
+	}
+
+	ans_int = (s32)(sum / num);
+	ans_fra = sum - ans_int * num;
+
+	if (ans_fra >= 0) {
+		if (ans_fra * 2 >= (s32)num)
+			ans_int++;
+	} else {
+		if (-ans_fra * 2 >= (s32)num)
+			ans_int--;
+	}
+	return (s16)ans_int;
+}
+
+static s8 _bt_psd_db_sum(s8 db_a, s8 db_b)
+{
+	s8 db_major;
+	s8 db_minor;
+	s8 db_sum;
+
+	if (db_a >= db_b) {
+		db_major = db_a;
+		db_minor = db_b;
+	} else {
+		db_major = db_b;
+		db_minor = db_a;
+	}
+
+	db_sum = db_major - db_minor; /* diff db */
+	if (db_sum <= 1) /* 0~1 */
+		db_sum = db_major + 3;
+	else if (db_major <= 3) /* 2~3 */
+		db_sum = db_major + 2;
+	else if (db_major <= 9) /* 4~9 */
+		db_sum = db_major + 1;
+	else /* >=10 */
+		db_sum = db_major;
+
+	return db_sum;
+}
+
+static u16 _bt_psd_get_tx_rate(struct btc_t *btc)
+{
+	struct btc_wl_link_info *plink = NULL;
+	u8 i;
+	bool is_found = false;
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++) {
+		plink = &btc->cx.wl.link_info[i];
+		if (plink->active && plink->connected != MLME_NO_LINK) {
+			is_found = true;
+			break;
+		}
+	}
+
+	if (!is_found)
+		PHL_INFO("[BTC], %s() wl no valid port!!!\n", __func__);
+
+	return (plink->tx_rate);
+}
+
+u8 _bt_psd_get_tx_ss_num(struct btc_t *btc)
+{
+	struct btc_wl_link_info *plink = NULL;
+	u8 ss = 0, i;
+	u16 tx_rate = RTW_DATA_RATE_MAX;
+	bool is_found = false;
+
+	for (i = 0; i < BTC_WL_MAX_ROLE_NUMBER; i++) {
+		plink = &btc->cx.wl.link_info[i];
+		if (plink->role == PHL_RTYPE_STATION &&
+		    plink->active && plink->connected != MLME_NO_LINK) {
+			is_found = true;
+			tx_rate = plink->tx_rate;
+			break;
+		}
+	}
+
+	if (is_found && tx_rate != RTW_DATA_RATE_MAX) {
+		if (tx_rate <= RTW_DATA_RATE_OFDM54)
+			ss = 1;
+		else if (tx_rate <= RTW_DATA_RATE_MCS31) /* bit[4:3] */
+			ss = ((tx_rate >> 3) & 0x3) + 1;
+		else /* bit[5:4] */
+			ss = ((tx_rate >> 4) & 0x3) + 1;
+	}
+
+	return ss;
+}
+
+s8 _bt_psd_get_tx_pwr_dbm(struct btc_t *btc, u8 rf_path)
+{
+	s16 wl_txp_dbm_tssi;
+	u32 reg_add;
+	u16 reg_val;
+
+	if (rf_path == RF_PATH_A)
+		reg_add = 0x1C44;
+	else
+		reg_add = 0x3C44;
+
+	reg_val = (u16)(rtw_hal_bb_read_cr(btc->hal, reg_add, 0x1FF) & 0x1FF);
+
+	wl_txp_dbm_tssi = 16 + (((s16)reg_val - 300) / 8);
+
+	return ((s8)wl_txp_dbm_tssi);
+}
+
+static s8 _bt_psd_get_tx_pwr(struct btc_t *btc)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	s8 wl_tx_power_dbm = 0, tx_power_a, tx_power_b;
+
+	bt_psd_dm->wl_tx_ss = _bt_psd_get_tx_ss_num(btc);
+	bt_psd_dm->wl_tx_rate = _bt_psd_get_tx_rate(btc);
+
+	tx_power_a = _bt_psd_get_tx_pwr_dbm(btc, RF_PATH_A);
+	tx_power_b = _bt_psd_get_tx_pwr_dbm(btc, RF_PATH_B);
+
+	if (bt_psd_dm->wl_tx_ss < 2)
+		wl_tx_power_dbm = tx_power_a;
+	else
+		wl_tx_power_dbm = _bt_psd_db_sum(tx_power_a, tx_power_b);
+
+	wl_tx_power_dbm -= _get_bw_att_db(btc); /* Tx power per MHz */
+
+	return (wl_tx_power_dbm);
+}
+
+static void _bt_psd_aiso_var_reset(struct btc_t *btc)
+{
+	struct rtw_hal_com_t *h = (struct rtw_hal_com_t *)btc->hal;
+	struct btc_aiso_val *av = &btc->bt_psd_dm.aiso_val;
+
+	PHL_INFO("[BTC], %s():\n", __func__);
+	hal_mem_set(h, av, 0, sizeof(struct btc_aiso_val));
+
+	hal_mem_set(h, av->psd_max, 0x80, sizeof(av->psd_max));
+	hal_mem_set(h, av->psd_min, 0x7F, sizeof(av->psd_min));
+
+	hal_mem_set(h, av->txbusy_psd_max, 0x80, sizeof(av->txbusy_psd_max));
+	hal_mem_set(h, av->txbusy_psd_min, 0x7F, sizeof(av->txbusy_psd_min));
+	hal_mem_set(h, av->txbusy_psd_avg, 0x80, sizeof(av->txbusy_psd_avg));
+
+	hal_mem_set(h, av->txidle_psd_max, 0x80, sizeof(av->txidle_psd_max));
+	hal_mem_set(h, av->txidle_psd_min, 0x7F, sizeof(av->txidle_psd_min));
+	hal_mem_set(h, av->txidle_psd_avg, 0x80, sizeof(av->txidle_psd_avg));
+
+	hal_mem_set(h, av->aiso_md, -128, sizeof(av->aiso_md));
+}
+
+static void _bt_psd_aiso_db_update(struct btc_t *btc)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	struct btc_aiso_val *av = &bt_psd_dm->aiso_val;
+	u8 aiso_method = bt_psd_dm->aiso_method;
+	u8 aiso_method_final = bt_psd_dm->aiso_method_final;
+	u8 idx_offset = 0, i, cnt;
+	s8 rx_psd_final = 0;
+	s16 aiso_db_final = -128, sum;
+
+	for (i = 0; i < 10; i++) {
+		cnt = av->txidle_psd_cnt[i];
+		sum = av->txidle_psd_sum[i];
+
+		if (cnt == 0) {
+			av->txidle_psd_avg[i] = -128;
+			continue;
+		}
+
+		av->txidle_psd_avg[i] = _bt_psd_s16du8r(sum, cnt);
+	}
+
+	if (aiso_method != BTC_AISO_M01_AVG && aiso_method != BTC_AISO_M0 &&
+	    aiso_method != BTC_AISO_M1 && aiso_method != BTC_AISO_M_ALL)
+		aiso_db_final = -128;
+
+	_limit_val(aiso_db_final, 127, -128);
+
+	if (aiso_method_final < BTC_AISO_M_MAX) {
+		aiso_db_final = av->aiso_md[aiso_method_final];
+		rx_psd_final = av->rx_psd[aiso_method_final];
+	}
+
+	PHL_INFO("[BTC], %s(): Goal %d tms m:%d aiso = %d (%d-%d) dB\n",
+		 __func__, av->psd_rec_cnt, aiso_method_final,
+		 aiso_db_final, av->wl_air_psd_avg, rx_psd_final);
+
+	/* Update ant iso to array: 1st_data -> index = 0  */
+	bt_psd_dm->aiso_db[bt_psd_dm->aiso_db_cnt & 0xF] = (s8)aiso_db_final;
+	bt_psd_dm->aiso_db_cnt++;
+
+	if (bt_psd_dm->aiso_db_cnt < 16)
+		idx_offset = 0;
+	else
+		idx_offset = bt_psd_dm->aiso_db_cnt - 16;
+
+	for (i = idx_offset; i < bt_psd_dm->aiso_db_cnt; i++) {
+		PHL_INFO("[BTC], %s(): %d times ant_iso = %d\n",
+			 __func__, i+1, bt_psd_dm->aiso_db[i & 0xF]);
+	}
+}
+
+static s16 _bt_psd_aiso_db_calc(struct btc_t *btc, s16 rx_psd_sum, u8 cnt)
+{
+	struct btc_aiso_val *av = &btc->bt_psd_dm.aiso_val;
+	s32 res_part = 0;
+	s16 aiso_db = -128;
+
+	if (av->psd_rec_cnt == 0)
+		return aiso_db;
+
+	res_part = av->wl_air_psd_sum - av->wl_air_psd_avg * av->psd_rec_cnt;
+	res_part = (res_part * cnt) / av->psd_rec_cnt;
+	res_part = rx_psd_sum - res_part;
+
+	_limit_val(res_part, 32767, -32768);
+
+	aiso_db = av->wl_air_psd_avg - _bt_psd_s16du8r((s16)res_part, cnt);
+
+	return aiso_db;
+}
+
+static void _bt_psd_group_calc(struct btc_t *btc, u8 md)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	struct btc_aiso_val *av = &bt_psd_dm->aiso_val;
+	s8 psd_max = 0, psd_min = 0;
+	s8 *psd_avg = NULL;
+	s16 rx_psd_sum = 0, psd_sum = 0, res_part = 0, val = 0;
+	u8 cnt = 0, i, psd_cnt = 0;
+
+	/* Excute for the following cases:
+	 * 1. aiso_method = BTC_AISO_M_ALL
+	 * 2. aiso_method = BTC_AISO_M4 && md = BTC_AISO_M1
+	 * 3. aiso_method = BTC_AISO_M6 && md = BTC_AISO_M5
+	 * 4. aiso_method = md
+	 */
+	if (bt_psd_dm->aiso_method != BTC_AISO_M_ALL &&
+	    (!(bt_psd_dm->aiso_method == BTC_AISO_M4 && md == BTC_AISO_M1)) &&
+	    (!(bt_psd_dm->aiso_method == BTC_AISO_M6 && md == BTC_AISO_M5)) &&
+	    bt_psd_dm->aiso_method != md)
+		return;
+
+	for (i = 0; i < 10; i++) {
+		psd_max = av->txbusy_psd_max[i];
+		psd_min = av->txbusy_psd_min[i];
+		psd_cnt = av->txbusy_psd_cnt[i];
+		psd_sum = av->txbusy_psd_sum[i];
+		psd_avg = &av->txbusy_psd_avg[i];
+
+		/* Skip if the freq-point not reach the threshold */
+		if (md <= BTC_AISO_M4 && psd_cnt < BT_PSD_TX_BUSY_CNT_MIN)
+			continue;
+
+		switch(md) {
+		case BTC_AISO_M0: /* m0: busy avg */
+			*psd_avg = _bt_psd_s16du8r(psd_sum, psd_cnt);
+			rx_psd_sum += *psd_avg;
+			cnt++;
+			break;
+		case BTC_AISO_M1: /* m1: busy max */
+			rx_psd_sum += psd_max;
+			cnt++;
+			break;
+		case BTC_AISO_M2: /* m2: busy remove max and min */
+			val = psd_sum - psd_max - psd_min;
+			*psd_avg = _bt_psd_s16du8r(val, psd_cnt - 2);
+			rx_psd_sum += *psd_avg;
+			cnt++;
+			break;
+		case BTC_AISO_M3: /* m3: busy remove max */
+			val = psd_sum - psd_max;
+			*psd_avg = _bt_psd_s16du8r(val, psd_cnt - 1);
+			rx_psd_sum += *psd_avg;
+			cnt++;
+			break;
+		case BTC_AISO_M4: /* m4:filter m1 */
+			if (psd_max >= av->rx_psd[BTC_AISO_M1])
+				rx_psd_sum += psd_max;
+			cnt++;
+			break;
+		case BTC_AISO_M5: /* m5: hold max value */
+			rx_psd_sum += psd_max;
+			cnt++;
+			break;
+		case BTC_AISO_M6: /* m6: filter m5 */
+			if (psd_max >= av->rx_psd[BTC_AISO_M5])
+				rx_psd_sum += psd_max;
+			cnt++;
+			break;
+		case BTC_AISO_M01_AVG: /* avg (m0 + m1) */
+			res_part = av->aiso_md[BTC_AISO_M0] +
+				   av->aiso_md[BTC_AISO_M1];
+			rx_psd_sum = av->rx_psd[BTC_AISO_M0] +
+				     av->rx_psd[BTC_AISO_M1];
+
+			cnt = 2;
+			break;
+		case BTC_AISO_M12_AVG: /* avg (m1 + m2) */
+			res_part = av->aiso_md[BTC_AISO_M1] +
+				   av->aiso_md[BTC_AISO_M2];
+			rx_psd_sum = av->rx_psd[BTC_AISO_M1] +
+				     av->rx_psd[BTC_AISO_M2];
+			cnt = 2;
+			break;
+		case BTC_AISO_M13_AVG: /* avg (m1 + m3) */
+			res_part = av->aiso_md[BTC_AISO_M1] +
+				   av->aiso_md[BTC_AISO_M3];
+			rx_psd_sum = av->rx_psd[BTC_AISO_M1] +
+				     av->rx_psd[BTC_AISO_M3];
+			cnt = 2;
+			break;
+		case BTC_AISO_M46_AVG: /* avg (m4 + m6) */
+			res_part = av->aiso_md[BTC_AISO_M4] +
+				   av->aiso_md[BTC_AISO_M6];
+			rx_psd_sum = av->rx_psd[BTC_AISO_M4] +
+				     av->rx_psd[BTC_AISO_M6];
+			cnt = 2;
+			break;
+		}
+	}
+
+	if (cnt == 0) {
+		PHL_INFO("[BTC], %s(): m:%d return by cnt = 0\n", __func__, md);
+		return;
+	}
+
+	av->rx_psd[md] = _bt_psd_s16du8r(rx_psd_sum, cnt);
+
+	if (md <= BTC_AISO_M6)
+		av->aiso_md[md] = _bt_psd_aiso_db_calc(btc, rx_psd_sum, cnt);
+	else
+		av->aiso_md[md] = _bt_psd_s16du8r(res_part, cnt);
+}
+
+static void _bt_psd_rec_refresh(struct btc_t *btc, bool is_2nd_half)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	struct btc_aiso_val *av = &bt_psd_dm->aiso_val;
+	u8 i = 0, rec_idx = 0;
+	u8 *half_data = NULL;
+	u8 wl_tx_busy_status = bt_psd_dm->raw_info[2];
+	s8 bt_psd_dbm_offset = 0; /* if lna-constrain 5 = 28*/
+	s8 cur_psd_rpt_dbm;
+
+	if (is_2nd_half)
+		half_data = &av->last_2nd_half[0];
+	else
+		half_data = &av->last_1st_half[0];
+
+	hal_mem_cpy(btc->hal, half_data, &bt_psd_dm->raw_info[2], 5);
+
+	for (i = 0; i < 5; i++) { /* psd_data_0 ~ psd_data_4 */
+		rec_idx = (is_2nd_half ? i + 5 : i);
+
+		cur_psd_rpt_dbm = bt_psd_dm->raw_info[i + 3];
+		cur_psd_rpt_dbm += bt_psd_dbm_offset;
+
+		if (cur_psd_rpt_dbm > av->psd_max[rec_idx])
+			av->psd_max[rec_idx] = cur_psd_rpt_dbm;
+
+		if (cur_psd_rpt_dbm < av->psd_min[rec_idx])
+			av->psd_min[rec_idx] = cur_psd_rpt_dbm;
+
+		if (wl_tx_busy_status & BIT(i)) {
+			if (cur_psd_rpt_dbm > av->txbusy_psd_max[rec_idx])
+				av->txbusy_psd_max[rec_idx] = cur_psd_rpt_dbm;
+
+			if (cur_psd_rpt_dbm < av->txbusy_psd_min[rec_idx])
+				av->txbusy_psd_min[rec_idx] = cur_psd_rpt_dbm;
+
+			/* avoid overflow */
+			if (av->txbusy_psd_cnt[rec_idx] < 200) {
+				av->txbusy_psd_sum[rec_idx] += cur_psd_rpt_dbm;
+				av->txbusy_psd_cnt[rec_idx]++;
+			}
+		} else {
+			if (cur_psd_rpt_dbm > av->txidle_psd_max[rec_idx])
+				av->txidle_psd_max[rec_idx] = cur_psd_rpt_dbm;
+
+			if (cur_psd_rpt_dbm < av->txidle_psd_min[rec_idx])
+				av->txidle_psd_min[rec_idx] = cur_psd_rpt_dbm;
+
+			/* avoid overflow */
+			if (av->txidle_psd_cnt[rec_idx] < 200) {
+				av->txidle_psd_sum[rec_idx] += cur_psd_rpt_dbm;
+				av->txidle_psd_cnt[rec_idx]++;
+			}
+		}
+	}
+}
+
+static bool _bt_psd_ready_check(struct btc_t *btc)
+{
+	struct btc_aiso_val *av = &btc->bt_psd_dm.aiso_val;
+
+	u8 half1_ch = 0, half2_ch = 0, i = 0;
+
+	/* calculate 1st_half_data_enough_flag */
+	for (i = 0; i < 5; i++)
+		if (av->txbusy_psd_cnt[i] >= BT_PSD_TX_BUSY_CNT_MIN)
+			half1_ch++;
+
+	/* calculate 2nd_half_data_enough_flag */
+	for (i = 5; i < 10; i++)
+		if (av->txbusy_psd_cnt[i] >= BT_PSD_TX_BUSY_CNT_MIN)
+			half2_ch++;
+
+	if (av->psd_rec_cnt % BT_PSD_PRINT_PERIOD == 1)
+		PHL_INFO("[BTC], %s(): TxBusy_PSD_CNT[ch]"
+			 " = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d\n", __func__,
+			 av->txbusy_psd_cnt[0], av->txbusy_psd_cnt[1],
+			 av->txbusy_psd_cnt[2], av->txbusy_psd_cnt[3],
+			 av->txbusy_psd_cnt[4], av->txbusy_psd_cnt[5],
+			 av->txbusy_psd_cnt[6], av->txbusy_psd_cnt[7],
+			 av->txbusy_psd_cnt[8], av->txbusy_psd_cnt[9]);
+
+	if (half1_ch >= BT_PSD_VALID_CH_MIN && half2_ch >= BT_PSD_VALID_CH_MIN)
+		return true;
+	else
+		return false;
+}
+
+/* post-process bt psd data and change flag to stop it */
+static void _bt_psd_process(struct btc_t *btc, bool is_reset)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	struct btc_aiso_val *av = &bt_psd_dm->aiso_val;
+
+	u8 i = 0, start_index, cur_wl_ch = btc->cx.wl.afh_info.ch;
+	u8 seq_idx = (bt_psd_dm->raw_info[1] >> 4) & 0xF;
+	u8 idx_1st_half = 0, idx_2nd_half = 0;
+	u16 cnt = 0;
+
+	/* bt_psd_dm->raw_info format:
+	 * (buf + 0) : malbox id
+	 * (buf + 1) :  [1:0] : data_src
+	 *		[3:2] : auto_rpt_type
+	 *		[7:4] : seq_idx
+	 * (buf + 2) : 	[4:0] : wl_s0_txbusy_mask
+	 *		[7:5] : reserved for wl_rxbusy
+	 * (buf + 3) : psd_data_0
+	 * (buf + 4) : psd_data_1
+	 * (buf + 5) : psd_data_2
+	 * (buf + 6) : psd_data_3
+	 * (buf + 7) : psd_data_4
+	 */
+
+	if (is_reset)
+		_bt_psd_aiso_var_reset(btc);
+
+	/* cur_wl_ch decide the psd-start index
+	 * ex: cur_wl_ch = 1 --> start index = 3 (2405MHz)
+	 * idx_1st_half = 3 ~ 7 (2405MHz~2409MHz)
+	 * idx_2nd_half = 13 ~ 17 (2415MHz~2419MHz)
+	 */
+	if (cur_wl_ch < 14)
+		start_index = 3 + 5 * (cur_wl_ch - 1);
+	else
+		start_index = 75;
+
+	idx_1st_half = start_index & 0xF;
+	idx_2nd_half = (start_index + 10) & 0xF;
+
+	if (seq_idx != idx_1st_half && seq_idx != idx_2nd_half) {
+		PHL_INFO("[BTC], %s(): wrong psd report index\n", __func__);
+		return;
+	} else {
+		/* refreh record  if data is NOT duplicate */
+		av->psd_rec_cnt++;
+		cnt = av->psd_rec_cnt;
+
+		PHL_INFO("[BTC], %s(): psd_rec_cnt = %d\n", __func__, cnt);
+
+		if (seq_idx == idx_1st_half)
+			_bt_psd_rec_refresh(btc, false);
+		else if (seq_idx == idx_2nd_half)
+			_bt_psd_rec_refresh(btc, true);
+	}
+
+	/* record wl tx psd */
+	av->wl_air_psd_sum += _bt_psd_get_tx_pwr(btc); /* dbm per MHz */
+
+	av->wl_air_psd_avg = (s8)_bt_psd_s32du16r(av->wl_air_psd_sum, cnt);
+
+	/* Check if data ready */
+	if (!_bt_psd_ready_check(btc)) {
+		if (cnt >= BT_PSD_WAIT_CNT) { /* timeout */
+			PHL_INFO("[BTC], %s(): wait rec timeout!\n", __func__);
+
+			bt_psd_dm->aiso_data_ok = false;
+			bt_psd_dm->rec_start = false;
+			bt_psd_dm->rec_time_out = true;
+			_bt_psd_rpt_ctrl(btc, false);
+		}
+
+		return;
+	}
+
+	/* calculate antenna isolation if data ready */
+	for (i = 0; i < BTC_AISO_M_MAX; i++) {
+
+		/* Must execute the other methos first for M4/M6 */
+		if (i == BTC_AISO_M4)
+			_bt_psd_group_calc(btc, BTC_AISO_M1);
+		else if (i == BTC_AISO_M6)
+			_bt_psd_group_calc(btc, BTC_AISO_M5);
+
+		_bt_psd_group_calc(btc, i);
+
+		PHL_INFO("[BTC], %s(): m%d = %d (%d-(%d)) dB\n", __func__,
+			 i, av->aiso_md[i], av->wl_air_psd_avg, av->rx_psd[i]);
+	}
+
+	bt_psd_dm->aiso_data_ok = true;
+	bt_psd_dm->rec_start = false;
+
+	_bt_psd_aiso_db_update(btc);
+
+	if (bt_psd_dm->aiso_db_cnt < bt_psd_dm->aiso_cmd_cnt) {
+		PHL_INFO("[BTC], %s(): aiso_db_cnt = %d, aiso_cmd_cnt = %d\n",
+			 __func__, bt_psd_dm->aiso_db_cnt,
+			 bt_psd_dm->aiso_cmd_cnt);
+
+		bt_psd_dm->aiso_data_ok = false;
+		bt_psd_dm->rec_start = false;
+		bt_psd_dm->rec_time_out = false;
+		return;
+	}
+
+	_bt_psd_rpt_ctrl(btc, false);
+
+	PHL_INFO("[BTC], %s(): Good-Ending!!\n", __func__);
+}
+
+/* post-process and change flag to stop it */
+void _bt_psd_update(struct btc_t *btc, u8 *buf, u32 len)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	u8 cur_wl_ch = btc->cx.wl.afh_info.ch;
+	bool reset_data_flag = false;
+
+	if (!bt_psd_dm->en) {
+		/* PHL_INFO("[BTC], %s(): drop by en = false\n", __func__); */
+		return;
+	}
+
+	if (!btc->dm.freerun) {
+		/* return; */
+	}
+
+	PHL_INFO("[BTC], %s(len=%d): seq_idx:0x%x/type:0x%x/src:0x%x/tx:0x%x"
+		 " [%x %x %x %x %x]\n", __func__, len,
+		 *(buf + 1) >> 4, (*(buf + 1) >> 2) & 0x3,
+		 *(buf + 1) & 0x3, *(buf + 2) & 0x1F,
+		 *(buf + 3), *(buf + 4), *(buf + 5), *(buf + 6), *(buf + 7));
+
+	hal_mem_cpy(h, bt_psd_dm->raw_info, buf, BTC_BTINFO_MAX);
+
+	/* before start */
+	if (!bt_psd_dm->rec_start && !bt_psd_dm->aiso_data_ok) {
+		bt_psd_dm->rec_start = true;
+		reset_data_flag = true;
+	} else if (bt_psd_dm->wl_ch_last != 0xFF &&
+		   (cur_wl_ch != bt_psd_dm->wl_ch_last)) {
+		reset_data_flag = true; /* rec_start & Not-Ready -> change ch */
+	}
+
+	if (bt_psd_dm->rec_start) {
+		_bt_psd_process(btc, reset_data_flag);
+		bt_psd_dm->wl_ch_last = cur_wl_ch;
+	}
+}
+
+static void _bt_psd_show_record(struct btc_t *btc, u32 *used,
+				char input[][MAX_ARGV], u32 input_num,
+				char *output, u32 out_len)
+{
+	struct btc_aiso_val *av = &btc->bt_psd_dm.aiso_val;
+
+	CLI_PRT("============ %s()  ============\n", __func__);
+
+	CLI_PRT(" PSD_MAX = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->psd_max[0], av->psd_max[1], av->psd_max[2], av->psd_max[3],
+		av->psd_max[4], av->psd_max[5], av->psd_max[6], av->psd_max[7],
+		av->psd_max[8], av->psd_max[9]);
+
+	CLI_PRT(" PSD_MIN = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->psd_min[0], av->psd_min[1], av->psd_min[2], av->psd_min[3],
+		av->psd_min[4], av->psd_min[5], av->psd_min[6], av->psd_min[7],
+		av->psd_min[8], av->psd_min[9]);
+
+	CLI_PRT(" TxBusy_PSD_MAX = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->txbusy_psd_max[0], av->txbusy_psd_max[1],
+		av->txbusy_psd_max[2], av->txbusy_psd_max[3],
+		av->txbusy_psd_max[4], av->txbusy_psd_max[5],
+		av->txbusy_psd_max[6], av->txbusy_psd_max[7],
+		av->txbusy_psd_max[8], av->txbusy_psd_max[9]);
+
+	CLI_PRT(" TxBusy_PSD_MIN = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->txbusy_psd_min[0], av->txbusy_psd_min[1],
+		av->txbusy_psd_min[2], av->txbusy_psd_min[3],
+		av->txbusy_psd_min[4], av->txbusy_psd_min[5],
+		av->txbusy_psd_min[6], av->txbusy_psd_min[7],
+		av->txbusy_psd_min[8], av->txbusy_psd_min[9]);
+
+	CLI_PRT(" TxBusy_PSD_AVG = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->txbusy_psd_avg[0], av->txbusy_psd_avg[1],
+		av->txbusy_psd_avg[2], av->txbusy_psd_avg[3],
+		av->txbusy_psd_avg[4], av->txbusy_psd_avg[5],
+		av->txbusy_psd_avg[6], av->txbusy_psd_avg[7],
+		av->txbusy_psd_avg[8], av->txbusy_psd_avg[9]);
+
+	CLI_PRT(" TxBusy_PSD_CNT = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d\n",
+		av->txbusy_psd_cnt[0], av->txbusy_psd_cnt[1],
+		av->txbusy_psd_cnt[2], av->txbusy_psd_cnt[3],
+		av->txbusy_psd_cnt[4], av->txbusy_psd_cnt[5],
+		av->txbusy_psd_cnt[6], av->txbusy_psd_cnt[7],
+		av->txbusy_psd_cnt[8], av->txbusy_psd_cnt[9]);
+
+	CLI_PRT(" TxIdle_PSD_MAX = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->txidle_psd_max[0], av->txidle_psd_max[1],
+		av->txidle_psd_max[2], av->txidle_psd_max[3],
+		av->txidle_psd_max[4], av->txidle_psd_max[5],
+		av->txidle_psd_max[6], av->txidle_psd_max[7],
+		av->txidle_psd_max[8], av->txidle_psd_max[9]);
+
+	CLI_PRT(" TxIdle_PSD_MIN = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->txidle_psd_min[0], av->txidle_psd_min[1],
+		av->txidle_psd_min[2], av->txidle_psd_min[3],
+		av->txidle_psd_min[4], av->txidle_psd_min[5],
+		av->txidle_psd_min[6], av->txidle_psd_min[7],
+		av->txidle_psd_min[8], av->txidle_psd_min[9]);
+
+	CLI_PRT(" TxIdle_PSD_AVG = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d dBm\n",
+		av->txidle_psd_avg[0], av->txidle_psd_avg[1],
+		av->txidle_psd_avg[2], av->txidle_psd_avg[3],
+		av->txidle_psd_avg[4], av->txidle_psd_avg[5],
+		av->txidle_psd_avg[6], av->txidle_psd_avg[7],
+		av->txidle_psd_avg[8], av->txidle_psd_avg[9]);
+
+	CLI_PRT(" TxIdle_PSD_CNT = %d,%d,%d,%d,%d,%d,%d,%d,%d,%d\n",
+		av->txidle_psd_cnt[0], av->txidle_psd_cnt[1],
+		av->txidle_psd_cnt[2], av->txidle_psd_cnt[3],
+		av->txidle_psd_cnt[4], av->txidle_psd_cnt[5],
+		av->txidle_psd_cnt[6], av->txidle_psd_cnt[7],
+		av->txidle_psd_cnt[8], av->txidle_psd_cnt[9]);
+}
+
+static void _bt_psd_aiso_sort(struct btc_t *btc, u32 *used,
+			      char *output, u32 out_len)
+{
+	struct btc_bt_psd_dm *bp = &btc->bt_psd_dm;
+	struct btc_module *module = &btc->mdinfo;
+	u8 i, rec_idx, cnt = bp->aiso_db_cnt, cnt1;
+	u8 max_aiso = 0, mid_cnt = 0, th = 3, min_lim = 0, max_lim = 0,
+	   avg_sort_cnt = 0;
+	u32 aiso_sort_tol = 0;
+
+	if (cnt > 16 || cnt == 0)
+		return;
+
+	mid_cnt = cnt % 2 ? cnt : cnt + 1;
+
+	if ((mid_cnt / 2) == 0)
+		return;
+
+	hal_mem_cpy(btc->hal, bp->aiso_sort_db, bp->aiso_db, 16);
+
+	//sort raw data of antenna-isolation results
+	for (i = 0; i < cnt-1; i++) {
+		cnt1 = (u8)(cnt - i - 1);
+		for (rec_idx = 0; rec_idx < cnt1; rec_idx++) {
+			if (bp->aiso_sort_db[rec_idx] <=
+			    bp->aiso_sort_db[rec_idx+1])
+			    continue;
+
+			max_aiso = bp->aiso_sort_db[rec_idx];
+			bp->aiso_sort_db[rec_idx] = bp->aiso_sort_db[rec_idx+1];
+			bp->aiso_sort_db[rec_idx+1] = max_aiso;
+		}
+	}
+
+	min_lim = (bp->aiso_sort_db[(mid_cnt/2)-1]) - th;
+	max_lim = (bp->aiso_sort_db[(mid_cnt/2)-1]) + th;
+	// list the sorting data of antenna-isolation results
+	for (rec_idx = 0; rec_idx < cnt; rec_idx++) {
+		CLI_PRT(" #%3d: ant_iso sort result = %3d dB\n",
+			rec_idx+1, bp->aiso_sort_db[rec_idx]);
+	}
+
+	// list mid data of sorting antenna-isolation results
+	CLI_PRT(" #%3d: ant_iso sort mid result = %3d dB\n",
+		(mid_cnt/2), bp->aiso_sort_db[(mid_cnt/2)-1]);
+
+	// calculate average of sorting antenna-isolation results
+	for (rec_idx = 0; rec_idx < cnt; rec_idx++) {
+		if (bp->aiso_sort_db[rec_idx] >= min_lim &&
+		    bp->aiso_sort_db[rec_idx] <= max_lim) {
+			avg_sort_cnt ++;
+			aiso_sort_tol += bp->aiso_sort_db[rec_idx];
+		}
+	}
+
+	//list average result
+	if (avg_sort_cnt != 0)
+		bp->aiso_sort_avg = aiso_sort_tol / avg_sort_cnt;
+	module->ant.isolation = bp->aiso_sort_avg;
+	CLI_PRT(" ant_iso sort avg result = %3d dB, avg_count = %3d\n",
+		bp->aiso_sort_avg, avg_sort_cnt);
+}
+
+static void _cmd_get_tx_pwr(struct btc_t *btc, u32 *used,
+			    char input[][MAX_ARGV], u32 input_num,
+			    char *output, u32 out_len)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	s8 txp_dbm = 127, txp_dbm_mhz = 127;
+
+	txp_dbm_mhz = _bt_psd_get_tx_pwr(btc);
+	txp_dbm = txp_dbm_mhz + _get_bw_att_db(btc);
+
+	CLI_PRT("tx_rate=%s, ss=%d, txp=%d dBm, txp_MHz=%d dBm/MHz\n",
+		id_to_str(BTC_STR_RATE, (u32)bt_psd_dm->wl_tx_rate),
+		bt_psd_dm->wl_tx_ss, txp_dbm, txp_dbm_mhz);
+}
+
+static void _cmd_trig_aiso(struct btc_t *btc, u32 *used,
+			   char input[][MAX_ARGV], u32 input_num,
+			   char *output, u32 out_len)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_bt_psd_dm *bp = &btc->bt_psd_dm;
+	u32 enable = 0, exec_cnt = 1;
+
+	if (input_num >= 3)
+		_os_sscanf(input[2], "%d", &enable);
+
+	if (input_num > 3)
+		_os_sscanf(input[3], "%d", &exec_cnt);
+
+	if (input_num < 3 || exec_cnt == 0 || exec_cnt > 16) {
+		CLI_PRT(" aiso 1 <0:force stop, 1:trig start>\n");
+		CLI_PRT(" aiso 1 1 <execute count:1~16>\n");
+		return;
+	}
+
+	if (enable && bp->en) {
+		CLI_PRT("ant isolation calculation is already running!!\n");
+		return;
+	}
+
+	if (enable) {
+		bp->aiso_data_ok = false;
+		bp->rec_start = false;
+		bp->aiso_cmd_cnt = (u8)exec_cnt;
+
+		if (bp->aiso_cmd_cnt > 1) {
+			bp->aiso_db_cnt = 0;
+			hal_mem_set(h, bp->aiso_db, 0, sizeof(bp->aiso_db));
+		}
+
+		_bt_psd_rpt_ctrl(btc, true);
+
+		CLI_PRT("trig ant isolation calculation (exec_cnt=%d)!!\n",
+			bp->aiso_cmd_cnt);
+
+	} else {
+		_bt_psd_rpt_ctrl(btc, false);
+		CLI_PRT("stop ant isolation calculation!!!!\n");
+	}
+
+	_run_coex(btc, __func__);
+}
+
+static void _cmd_get_aiso(struct btc_t *btc, u32 *used,
+			  char input[][MAX_ARGV], u32 input_num,
+			  char *output, u32 out_len)
+{
+	struct btc_bt_psd_dm *bt_psd_dm = &btc->bt_psd_dm;
+	struct btc_aiso_val *av = &bt_psd_dm->aiso_val;
+	u8 idx_offset, i, rec_idx, md, cnt;
+	u32 debug_level = 0;
+
+	if (input_num < 3) {
+		CLI_PRT(" aiso 0 <0:current result, 1:history result>\n");
+		return;
+	}
+
+	_os_sscanf(input[2], "%d", &debug_level);
+
+	if (!bt_psd_dm->aiso_data_ok) {
+		if (bt_psd_dm->rec_time_out)
+			CLI_PRT(" bt psd rpt record timeout!!\n");
+		else if (!bt_psd_dm->en)
+			CLI_PRT(" not start to enable bt psd rpt!!\n");
+		else
+			CLI_PRT(" ant isolation is under calculation!!\n");
+
+		return;
+	}
+
+	cnt = bt_psd_dm->aiso_db_cnt;
+
+	if (cnt == 0) {
+		CLI_PRT(" no ant_iso result!!\n");
+		return;
+	}
+
+	/* List the late antenna-isolation result */
+	if (debug_level == 0) {
+		rec_idx = (cnt - 1) & 0xF;
+		CLI_PRT(" current ant_iso result = %3d dB\n",
+			bt_psd_dm->aiso_db[rec_idx]);
+		return;
+	}
+
+	/* List the latest 16 antenna-isolation results */
+	idx_offset = (cnt < 16 ? 0 : cnt - 16);
+
+	for (i = idx_offset; i < cnt; i++) {
+		rec_idx = i & 0xF;
+		CLI_PRT(" #%3d: ant_iso result = %3d dB\n",
+			i+1, bt_psd_dm->aiso_db[rec_idx]);
+	}
+
+	_bt_psd_aiso_sort(btc, used, output, out_len);
+
+	md = bt_psd_dm->aiso_method_final;
+	CLI_PRT(" psd_rpt: rec_cnt=%d, method=%d, ant_iso_db=%d (%d-(%d))dB\n",
+		av->psd_rec_cnt, md, av->aiso_md[md],
+		av->wl_air_psd_avg, av->rx_psd[md]);
+
+	for (i = 0; i < BTC_AISO_M_MAX; i++) {
+
+		if (i != BTC_AISO_M4 && i != BTC_AISO_M5 &&
+		    i != BTC_AISO_M6 && i != BTC_AISO_M46_AVG)
+			continue;
+
+		CLI_PRT(" ant_iso_md (m:%d) = %d (%d - (%d))dB\n", i,
+			av->aiso_md[i], av->wl_air_psd_avg, av->rx_psd[i]);
+	}
+
+	_bt_psd_show_record(btc, used, input, input_num, output, out_len);
+}
+
+static void _cmd_ant_iso(struct btc_t *btc, u32 *used,
+			 char input[][MAX_ARGV], u32 input_num,
+			 char *output, u32 out_len)
+{
+	u32 cmd_set = 0;
+
+	if (input_num < 2)
+		goto help;
+
+	_os_sscanf(input[1], "%d", &cmd_set);
+
+	switch (cmd_set) {
+	case 0:
+		_cmd_get_aiso(btc, used, input, input_num, output, out_len);
+		break;
+	case 1:
+		_cmd_trig_aiso(btc, used, input, input_num, output, out_len);
+		break;
+	case 2:
+		_cmd_get_tx_pwr(btc, used, input, input_num, output, out_len);
+		break;
+	default:
+		goto help;
+	}
+
+	return;
+help:
+	CLI_PRT("aiso <0:show result 1:trig 2:show wl tx power>\n");
+}
+#else
+void _bt_psd_setup(struct btc_t *btc, u8 start_idx, u8 rpt_type)
+{
+	u8 buf[2] = {0};
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s(): set bt psd\n",
+		  __func__);
+
+	buf[0] = start_idx;
+	buf[1] = rpt_type;
+	_send_fw_cmd(btc, SET_BT_PSD_REPORT, buf, 2);
+}
+
+void _bt_psd_update(struct btc_t *btc, u8 *buf, u32 len)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s():\n", __func__);
+}
+#endif
+
+void halbtc_cmd_parser(struct btc_t *btc, char input[][MAX_ARGV],
+			u32 input_num, char *output, u32 out_len)
+{
+	u32 offset = 0;
+	u32 *used = &offset;
+	u8 id = 0;
+	u32 i;
+	u32 array_size = sizeof(halbtc_cmd_i) / sizeof(struct halbtc_cmd_info);
+
+	CLI_PRT("\n");
+
+	/* Parsing Cmd ID */
+	if (input_num) {
+		/* Avoid input number and debug cmd mismatch */
+		input_num--;
+		for (i = 0; i < array_size; i++) {
+			if (_os_strcmp(halbtc_cmd_i[i].name, input[0]) == 0) {
+				id = halbtc_cmd_i[i].id;
+				break;
+			}
+		}
+	}
+
+	btc->wlcli_h2c_cmd = true;
+
+	switch (id) {
+	case HALBTC_DBG:
+		_cmd_dbg(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SHOW:
+		_cmd_show(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_WRITE_BT:
+		_cmd_wb(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_READ_BT:
+		_cmd_rb(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SET_COEX:
+		_cmd_set_coex(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_UPDATE_POLICY:
+		_cmd_upd_policy(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_TDMA:
+		_cmd_tdma(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SLOT:
+		_cmd_slot(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SIG_GDBG_EN:
+		_cmd_sig_gdbg_en(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SGPIO_MAP:
+		_cmd_sgpio_map(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_WL_TX_POWER:
+		_cmd_wl_tx_power(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_WL_RX_LNA:
+		_cmd_wl_rx_lna(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_BT_AFH_MAP:
+		_cmd_bt_afh_map(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_BT_TX_POWER:
+		_cmd_bt_tx_power(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_BT_RX_LNA:
+		_cmd_bt_rx_lna(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_BT_IGNO_WLAN:
+		_cmd_bt_igno_wl(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SET_GNT_WL:
+		_cmd_set_gnt_wl(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SET_GNT_BT:
+		_cmd_set_gnt_bt(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_SET_BT_PSD:
+		_cmd_set_bt_psd(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_GET_WL_NHM_DBM:
+		_cmd_get_wl_nhm(btc, used, input, input_num, output, out_len);
+		break;
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	case HALBTC_FT_CTRL:
+		_cmd_fddt_ctrl(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_FT_SET:
+		_cmd_fddt_set(btc, used, input, input_num, output, out_len);
+		break;
+	case HALBTC_FT_SHOW:
+		_cmd_fddt_show(btc, used, input, input_num, output, out_len);
+		break;
+#endif
+#ifdef BTC_AISO_SUPPORT
+	case HALBTC_AISO:
+		_cmd_ant_iso(btc, used, input, input_num, output, out_len);
+		break;
+#endif
+	default:
+		CLI_PRT("BTC cmd ==>\n");
+
+		for (i = 0; i < array_size - 1; i++)
+			CLI_PRT(" %s\n", halbtc_cmd_i[i + 1].name);
+		break;
+	}
+
+	btc->wlcli_h2c_cmd = false;
+}
+
+s32 halbtc_cmd(struct btc_t *btc, char *input, char *output, u32 out_len)
+{
+	char *token;
+	u32 argc = 0;
+	char argv[MAX_ARGC][MAX_ARGV];
+
+	do {
+		token = _os_strsep(&input, ", ");
+		if (token) {
+			if (_os_strlen((u8 *)token) <= MAX_ARGV)
+				_os_strcpy(argv[argc], token);
+
+			argc++;
+		} else {
+			break;
+		}
+	} while (argc < MAX_ARGC);
+#if 0
+	if (argc == 1)
+		argv[0][_os_strlen((u8 *)argv[0]) - 1] = '\0';
+#endif
+
+	halbtc_cmd_parser(btc, argv, argc, output, out_len);
+	return 0;
+}
+
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_dbg_cmd.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_dbg_cmd.h
new file mode 100644
index 000000000000..758bdc90ef6a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_dbg_cmd.h
@@ -0,0 +1,26 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HALBTC_DBG_CMD_H_
+#define _HALBTC_DBG_CMD_H_
+
+void halbtc_cmd_parser(struct btc_t *btc, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 out_len);
+s32 halbtc_cmd(struct btc_t *btc, char *input, char *output, u32 out_len);
+void _get_wl_nhm_dbm(struct btc_t *btc);
+void _get_wl_cn_report(struct btc_t *btc);
+void _get_wl_evm_report(struct btc_t *btc);
+void _bt_psd_setup(struct btc_t *btc, u8 start_idx, u8 auto_rpt_type);
+void _bt_psd_update(struct btc_t *btc, u8 *buf, u32 len);
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_def.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_def.c
new file mode 100644
index 000000000000..cada9b261822
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_def.c
@@ -0,0 +1,861 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_BTC_DEF_C_
+#include "../hal_headers_le.h"
+#include "hal_btc.h"
+#include "halbtc_fw.h"
+#include "halbtc_action.h"
+
+#ifdef CONFIG_BTCOEX
+
+#define case_role(src) \
+	case PHL_RTYPE_##src: return #src
+#define case_mrole(src) \
+	case BTC_WLMROLE_##src: return #src
+#define case_slot(src) \
+	case CXST_##src: return #src
+#define case_reg(src) \
+	case REG_##src: return #src
+#define case_tdma(src) \
+	case CXTDMA_##src: return #src
+#define case_trace(src) \
+	case CXSTEP_##src: return #src
+#define case_branch(src) \
+	case BTC_BRANCH_##src: return #src
+#define case_rxflctrl_(src) \
+	case CXFLC_##src: return #src
+#define case_wllink(src) \
+	case BTC_WLINK_##src: return #src
+#define case_antpath(src) \
+	case BTC_ANT_##src: return #src
+#define case_gdbg(src) \
+	case BTC_DBG_##src: return #src
+#define case_chip(src) \
+	case CHIP_##src: return #src
+#define case_event(src) \
+	case CXEVNT_##src: return #src
+#define case_wlmode(src) \
+	case WLAN_MD_##src: return #src
+#define case_wlbw(src) \
+	case CHANNEL_WIDTH_##src: return #src
+#define case_cxp(src) \
+	case BTC_CXP_##src: return #src
+#define case_rftype(src) \
+	case RF_##src: return #src
+#define case_mstate(src) \
+	case MLME_##src: return #src
+#define case_rate(src) \
+	case RTW_DATA_RATE_##src: return #src
+#define case_polut(src) \
+	case BTC_PLT_##src: return #src
+#define case_band(src) \
+	case BAND_ON_##src: return #src
+#define case_cxstate(src) \
+	case BTC_##src: return #src
+#define case_fddt_state(src) \
+	case BTC_FDDT_STATE_##src: return #src
+#define case_fddt_type(src) \
+	case BTC_FDDT_TYPE_##src: return #src
+#define case_fddt_norun(src) \
+	case BTC_NFRSN_##src: return #src
+#define case_fddt_tstate(src) \
+	case CX_FDDT_STATE_##src: return #src
+#define case_fddt_tphase(src) \
+	case CX_FDDT_PHASE_##src: return #src
+#define case_fddt_chkmap(src) \
+	case CX_FDDT_CHKMAP_##src: return #src
+#define case_dmerr(src) \
+	case BTC_DMERR_##src: return #src
+#define case_fwerr(src) \
+	case BTC_FWERR_##src: return #src
+#define case_rptmatch(src) \
+	case BTC_RPT_TYPE_##src: return #src
+#define case_h2cerr(src) \
+	case SET_##src: return #src
+#define case_vendorid(src) \
+	case BTC_VID_##src: return #src
+#define case_spap(src) \
+	case BTC_SPAP_##src: return #src
+#define case_esoc(src) \
+	case BTC_EXT_SOC_##src: return #src
+#define case_esocintf(src) \
+	case BTC_EXTSOC_INTF_##src: return #src
+#define case_esocfunc(src) \
+	case BTC_EXTSOC_FUNC_##src: return #src
+#define case_path(src) \
+	case RF_PATH_##src: return #src
+
+const char *id_to_str(u8 type, u32 id)
+{
+	switch(type) {
+	case BTC_STR_ROLE:
+		switch (id) {
+		case_role(NONE);
+		case_role(STATION);
+		case_role(AP);
+		case_role(VAP);
+		case_role(ADHOC);
+		case_role(ADHOC_MASTER);
+		case_role(MESH);
+		case_role(MONITOR);
+		case_role(P2P_DEVICE);
+		case_role(P2P_GC);
+		case_role(P2P_GO);
+		case_role(NAN);
+		}
+		break;
+	case BTC_STR_MROLE:
+		switch (id) {
+		case_mrole(NONE);
+		case_mrole(STA_GC);
+		case_mrole(STA_GC_NOA);
+		case_mrole(STA_GO);
+		case_mrole(STA_GO_NOA);
+		case_mrole(STA_STA);
+		}
+		break;
+	case BTC_STR_SLOT:
+		switch (id) {
+		case_slot(OFF);
+		case_slot(B2W);
+		case_slot(W1);
+		case_slot(W2);
+		case_slot(W2B);
+		case_slot(B1);
+		case_slot(B2);
+		case_slot(B3);
+		case_slot(B4);
+		case_slot(LK);
+		case_slot(BLK);
+		case_slot(E2G);
+		case_slot(E5G);
+		case_slot(EBT);
+		case_slot(ENULL);
+		case_slot(WLK);
+		case_slot(W1FDD);
+		case_slot(B1FDD);
+		}
+		break;
+	case BTC_STR_REG:
+		switch (id) {
+		case_reg(MAC);
+		case_reg(BB);
+		case_reg(RF);
+		case_reg(BT_RF);
+		case_reg(BT_MODEM);
+		case_reg(BT_BLUEWIZE);
+		case_reg(BT_VENDOR);
+		case_reg(BT_LE);
+		}
+		break;
+	case BTC_STR_TDMA:
+		switch (id) {
+		case_tdma(OFF);
+		case_tdma(FIX);
+		case_tdma(AUTO);
+		case_tdma(AUTO2);
+		}
+		break;
+	case BTC_STR_TRACE:
+		switch (id) {
+		case_trace(NONE);
+		case_trace(EVNT);
+		case_trace(SLOT);
+		}
+		break;
+	case BTC_STR_BRANCH:
+		switch (id) {
+		case_branch(FORMAL);
+		case_branch(UPSTREAM);
+		case_branch(HP);
+		case_branch(LENOVO);
+		case_branch(XBOX);
+		case_branch(WBX);
+		case_branch(ROKU);
+		case_branch(LG);
+		case_branch(SAMSUNG);
+		case_branch(AMAZON);
+		}
+		break;
+	case BTC_STR_RXFLCTRL:
+		switch (id) {
+		case_rxflctrl_(OFF);
+		case_rxflctrl_(NULLP);
+		case_rxflctrl_(QOSNULL);
+		case_rxflctrl_(CTS);
+		}
+		break;
+	case BTC_STR_WLLINK:
+		switch (id) {
+		case_wllink(NOLINK);
+		case_wllink(2G_STA);
+		case_wllink(2G_GO);
+		case_wllink(2G_GC);
+		case_wllink(2G_AP);
+		case_wllink(2G_SCC);
+		case_wllink(2G_MCC);
+		case_wllink(25G_MCC);
+		case_wllink(25G_DBCC);
+		case_wllink(5G);
+		case_wllink(2G_NAN);
+		case_wllink(OTHER);
+		}
+		break;
+	case BTC_STR_ANTPATH:
+		switch (id) {
+		case_antpath(WPOWERON);
+		case_antpath(WINIT);
+		case_antpath(WONLY);
+		case_antpath(WOFF);
+		case_antpath(W2G);
+		case_antpath(W5G);
+		case_antpath(W25G);
+		case_antpath(FREERUN);
+		case_antpath(FDDTRAIN);
+		case_antpath(WRFK);
+		case_antpath(WRFK2);
+		case_antpath(BRFK);
+		}
+		break;
+	case BTC_STR_GDBG:
+		switch (id) {
+		case_gdbg(GNT_BT);
+		case_gdbg(GNT_WL);
+		case_gdbg(BCN_EARLY);
+		case_gdbg(WL_NULL0);
+		case_gdbg(WL_NULL1);
+		case_gdbg(WL_RXISR);
+		case_gdbg(TDMA_ENTRY);
+		case_gdbg(A2DP_EMPTY);
+		case_gdbg(BT_RETRY);
+		case_gdbg(BT_RELINK);
+		case_gdbg(SLOT_WL);
+		case_gdbg(SLOT_BT);
+		case_gdbg(WL_ERR);
+		case_gdbg(WL_OK);
+		case_gdbg(SLOT_B2W);
+		case_gdbg(SLOT_W1);
+		case_gdbg(SLOT_W2);
+		case_gdbg(SLOT_W2B);
+		case_gdbg(SLOT_B1);
+		case_gdbg(SLOT_B2);
+		case_gdbg(SLOT_B3);
+		case_gdbg(SLOT_B4);
+		case_gdbg(SLOT_LK);
+		case_gdbg(SLOT_E2G);
+		case_gdbg(SLOT_E5G);
+		case_gdbg(SLOT_EBT);
+		case_gdbg(SLOT_WLK);
+		case_gdbg(SLOT_B1FDD);
+		case_gdbg(BT_CHANGE);
+		case_gdbg(WL_CCA);
+		case_gdbg(BT_LEAUDIO);
+		case_gdbg(USER_DEF);
+		}
+		break;
+	case BTC_STR_CHIPID:
+		switch (id) {
+		case_chip(WIFI6_8852A);
+		case_chip(WIFI6_8852B);
+		case_chip(WIFI6_8852C);
+		case_chip(WIFI6_8842A);
+#ifdef BTC_8852D_SUPPORT
+		case_chip(WIFI6_8852D);
+#endif
+		case_chip(WIFI6_8852BP);
+		case_chip(WIFI6_8851B);
+#ifdef BTC_8852BT_SUPPORT
+		case_chip(WIFI6_8852BT);
+#endif
+#ifdef BTC_8852BT_SUPPORT
+		case_chip(WIFI6_8852BPT);
+#endif
+		}
+		break;
+	case BTC_STR_POLUT:
+		switch (id) {
+		case_polut(NONE);
+		case_polut(GNT_BT_TX);
+		case_polut(GNT_BT_RX);
+		case_polut(GNT_WL);
+		case_polut(BT);
+		case_polut(ALL);
+		}
+		break;
+	case BTC_STR_EVENT:
+		switch(id) {
+		case_event(TDMA_ENTRY);
+		case_event(WL_TMR);
+		case_event(B1_TMR);
+		case_event(B2_TMR);
+		case_event(B3_TMR);
+		case_event(B4_TMR);
+		case_event(W2B_TMR);
+		case_event(B2W_TMR);
+		case_event(BCN_EARLY);
+		case_event(A2DP_EMPTY);
+		case_event(LK_END);
+		case_event(RX_ISR);
+		case_event(RX_FC0);
+		case_event(RX_FC1);
+		case_event(BT_RELINK);
+		case_event(BT_RETRY);
+		case_event(E2G);
+		case_event(E5G);
+		case_event(EBT);
+		case_event(ENULL);
+		case_event(DRV_WLK);
+		case_event(BCN_OK);
+		case_event(BT_CHANGE);
+		case_event(EBT_EXTEND);
+		case_event(E2G_NULL1);
+		case_event(B1FDD_TMR);
+		case_event(LE_INIT_END);
+		}
+		break;
+	case BTC_STR_WLMODE:
+		switch(id) {
+		case_wlmode(INVALID);
+		case_wlmode(11B);
+		case_wlmode(11A);
+		case_wlmode(11G);
+		case_wlmode(11N);
+		case_wlmode(11AC);
+		case_wlmode(11AX);
+		case_wlmode(11BG);
+		case_wlmode(11GN);
+		case_wlmode(11AN);
+		case_wlmode(11BN);
+		case_wlmode(11BGN);
+		case_wlmode(11BGAC);
+		case_wlmode(11BGAX);
+		case_wlmode(11A_AC);
+		case_wlmode(11A_AX);
+		case_wlmode(11AGN);
+		case_wlmode(11ABGN);
+		case_wlmode(24G_MIX);
+		case_wlmode(5G_MIX);
+		case_wlmode(MAX);
+		}
+		break;
+	case BTC_STR_WLBW:
+		switch(id) {
+		case_wlbw(20);
+		case_wlbw(40);
+		case_wlbw(80);
+		case_wlbw(160);
+		case_wlbw(80_80);
+		case_wlbw(5);
+		case_wlbw(10);
+		case_wlbw(MAX);
+		}
+		break;
+	case BTC_STR_RFTYPE:
+		switch(id) {
+		case_rftype(1T1R);
+		case_rftype(1T2R);
+		case_rftype(2T3R);
+		case_rftype(2T4R);
+		case_rftype(3T3R);
+		case_rftype(3T4R);
+		case_rftype(4T4R);
+		}
+		break;
+	case BTC_STR_POLICY:
+		switch(id) {
+		case_cxp(OFF_BT);
+		case_cxp(OFF_WL);
+		case_cxp(OFF_EQ0);
+		case_cxp(OFF_EQ1);
+		case_cxp(OFF_EQ2);
+		case_cxp(OFF_EQ3);
+		case_cxp(OFF_EQ4);
+		case_cxp(OFF_EQ5);
+		case_cxp(OFF_BWB0);
+		case_cxp(OFF_BWB1);
+		case_cxp(OFF_BWB2);
+		case_cxp(OFF_BWB3);
+		case_cxp(OFF_WL2);
+		case_cxp(OFF_BT2);
+		case_cxp(OFFB_BWB0);
+		case_cxp(OFFB_BWB1);
+		case_cxp(OFFB_BWB2);
+		case_cxp(OFFB_BWB3);
+		case_cxp(OFFE_2GBWISOB);
+		case_cxp(OFFE_2GISOB);
+		case_cxp(OFFE_2GBWMIXB);
+		case_cxp(OFFE_2GBWMIXB2);
+		case_cxp(OFFE_WL);
+		case_cxp(FIX_TD3030);
+		case_cxp(FIX_TD5050);
+		case_cxp(FIX_TD2030);
+		case_cxp(FIX_TD4010);
+		case_cxp(FIX_TD7010);
+		case_cxp(FIX_TD2060);
+		case_cxp(FIX_TD3060);
+		case_cxp(FIX_TD2080);
+		case_cxp(FIX_TDW1B1);
+		case_cxp(FIX_TD4010ISO);
+		case_cxp(FIX_TD4010ISO_DL);
+		case_cxp(FIX_TD7010ISO_UL);
+		case_cxp(PFIX_TD3030);
+		case_cxp(PFIX_TD5050);
+		case_cxp(PFIX_TD2030);
+		case_cxp(PFIX_TD2060);
+		case_cxp(PFIX_TD3070);
+		case_cxp(PFIX_TD2080);
+		case_cxp(PFIX_TDW1B1);
+		case_cxp(AUTO_TD50B1);
+		case_cxp(AUTO_TD60B1);
+		case_cxp(AUTO_TD20B1);
+		case_cxp(AUTO_TDW1B1);
+		case_cxp(AUTO_TDW1B1_CXTPS);
+		case_cxp(PAUTO_TD50B1);
+		case_cxp(PAUTO_TD60B1);
+		case_cxp(PAUTO_TD20B1);
+		case_cxp(PAUTO_TDW1B1);
+		case_cxp(PAUTO_FDDT1);
+		case_cxp(AUTO2_TD3050);
+		case_cxp(AUTO2_TD3070);
+		case_cxp(AUTO2_TD5050);
+		case_cxp(AUTO2_TD6060);
+		case_cxp(AUTO2_TD2080);
+		case_cxp(AUTO2_TDW1B4);
+		case_cxp(PAUTO2_TD3050);
+		case_cxp(PAUTO2_TD3070);
+		case_cxp(PAUTO2_TD5050);
+		case_cxp(PAUTO2_TD6060);
+		case_cxp(PAUTO2_TD2080);
+		case_cxp(PAUTO2_TDW1B4);
+		}
+		break;
+	case BTC_STR_MSTATE:
+		switch(id) {
+		case_mstate(NO_LINK);
+		case_mstate(LINKING);
+		case_mstate(LINKED);
+		}
+		break;
+	case BTC_STR_RATE:
+		switch(id) {
+		case_rate(CCK1);
+		case_rate(CCK2);
+		case_rate(CCK5_5);
+		case_rate(CCK11);
+		case_rate(OFDM6);
+		case_rate(OFDM9);
+		case_rate(OFDM12);
+		case_rate(OFDM18);
+		case_rate(OFDM24);
+		case_rate(OFDM36);
+		case_rate(OFDM48);
+		case_rate(OFDM54);
+		case_rate(MCS0);
+		case_rate(MCS1);
+		case_rate(MCS2);
+		case_rate(MCS3);
+		case_rate(MCS4);
+		case_rate(MCS5);
+		case_rate(MCS6);
+		case_rate(MCS7);
+		case_rate(MCS8);
+		case_rate(MCS9);
+		case_rate(MCS10);
+		case_rate(MCS11);
+		case_rate(MCS12);
+		case_rate(MCS13);
+		case_rate(MCS14);
+		case_rate(MCS15);
+		case_rate(MCS16);
+		case_rate(MCS17);
+		case_rate(MCS18);
+		case_rate(MCS19);
+		case_rate(MCS20);
+		case_rate(MCS21);
+		case_rate(MCS22);
+		case_rate(MCS23);
+		case_rate(MCS24);
+		case_rate(MCS25);
+		case_rate(MCS26);
+		case_rate(MCS27);
+		case_rate(MCS28);
+		case_rate(MCS29);
+		case_rate(MCS30);
+		case_rate(MCS31);
+		case_rate(VHT_NSS1_MCS0);
+		case_rate(VHT_NSS1_MCS1);
+		case_rate(VHT_NSS1_MCS2);
+		case_rate(VHT_NSS1_MCS3);
+		case_rate(VHT_NSS1_MCS4);
+		case_rate(VHT_NSS1_MCS5);
+		case_rate(VHT_NSS1_MCS6);
+		case_rate(VHT_NSS1_MCS7);
+		case_rate(VHT_NSS1_MCS8);
+		case_rate(VHT_NSS1_MCS9);
+		case_rate(VHT_NSS2_MCS0);
+		case_rate(VHT_NSS2_MCS1);
+		case_rate(VHT_NSS2_MCS2);
+		case_rate(VHT_NSS2_MCS3);
+		case_rate(VHT_NSS2_MCS4);
+		case_rate(VHT_NSS2_MCS5);
+		case_rate(VHT_NSS2_MCS6);
+		case_rate(VHT_NSS2_MCS7);
+		case_rate(VHT_NSS2_MCS8);
+		case_rate(VHT_NSS2_MCS9);
+		case_rate(VHT_NSS3_MCS0);
+		case_rate(VHT_NSS3_MCS1);
+		case_rate(VHT_NSS3_MCS2);
+		case_rate(VHT_NSS3_MCS3);
+		case_rate(VHT_NSS3_MCS4);
+		case_rate(VHT_NSS3_MCS5);
+		case_rate(VHT_NSS3_MCS6);
+		case_rate(VHT_NSS3_MCS7);
+		case_rate(VHT_NSS3_MCS8);
+		case_rate(VHT_NSS3_MCS9);
+		case_rate(VHT_NSS4_MCS0);
+		case_rate(VHT_NSS4_MCS1);
+		case_rate(VHT_NSS4_MCS2);
+		case_rate(VHT_NSS4_MCS3);
+		case_rate(VHT_NSS4_MCS4);
+		case_rate(VHT_NSS4_MCS5);
+		case_rate(VHT_NSS4_MCS6);
+		case_rate(VHT_NSS4_MCS7);
+		case_rate(VHT_NSS4_MCS8);
+		case_rate(VHT_NSS4_MCS9);
+		case_rate(HE_NSS1_MCS0);
+		case_rate(HE_NSS1_MCS1);
+		case_rate(HE_NSS1_MCS2);
+		case_rate(HE_NSS1_MCS3);
+		case_rate(HE_NSS1_MCS4);
+		case_rate(HE_NSS1_MCS5);
+		case_rate(HE_NSS1_MCS6);
+		case_rate(HE_NSS1_MCS7);
+		case_rate(HE_NSS1_MCS8);
+		case_rate(HE_NSS1_MCS9);
+		case_rate(HE_NSS1_MCS10);
+		case_rate(HE_NSS1_MCS11);
+		case_rate(HE_NSS2_MCS0);
+		case_rate(HE_NSS2_MCS1);
+		case_rate(HE_NSS2_MCS2);
+		case_rate(HE_NSS2_MCS3);
+		case_rate(HE_NSS2_MCS4);
+		case_rate(HE_NSS2_MCS5);
+		case_rate(HE_NSS2_MCS6);
+		case_rate(HE_NSS2_MCS7);
+		case_rate(HE_NSS2_MCS8);
+		case_rate(HE_NSS2_MCS9);
+		case_rate(HE_NSS2_MCS10);
+		case_rate(HE_NSS2_MCS11);
+		case_rate(HE_NSS3_MCS0);
+		case_rate(HE_NSS3_MCS1);
+		case_rate(HE_NSS3_MCS2);
+		case_rate(HE_NSS3_MCS3);
+		case_rate(HE_NSS3_MCS4);
+		case_rate(HE_NSS3_MCS5);
+		case_rate(HE_NSS3_MCS6);
+		case_rate(HE_NSS3_MCS7);
+		case_rate(HE_NSS3_MCS8);
+		case_rate(HE_NSS3_MCS9);
+		case_rate(HE_NSS3_MCS10);
+		case_rate(HE_NSS3_MCS11);
+		case_rate(HE_NSS4_MCS0);
+		case_rate(HE_NSS4_MCS1);
+		case_rate(HE_NSS4_MCS2);
+		case_rate(HE_NSS4_MCS3);
+		case_rate(HE_NSS4_MCS4);
+		case_rate(HE_NSS4_MCS5);
+		case_rate(HE_NSS4_MCS6);
+		case_rate(HE_NSS4_MCS7);
+		case_rate(HE_NSS4_MCS8);
+		case_rate(HE_NSS4_MCS9);
+		case_rate(HE_NSS4_MCS10);
+		case_rate(HE_NSS4_MCS11);
+		case_rate(MAX);
+		}
+		break;
+	case BTC_STR_BAND:
+		switch(id) {
+		case BAND_ON_24G:
+			return "2.4G";
+		case_band(5G);
+		case_band(6G);
+		}
+		break;
+	case BTC_STR_CXSTATE:
+		switch(id) {
+		case_cxstate(WIDLE);
+		case_cxstate(WBUSY_BNOSCAN);
+		case_cxstate(WBUSY_BSCAN);
+		case_cxstate(WSCAN_BNOSCAN);
+		case_cxstate(WSCAN_BSCAN);
+		case_cxstate(WLINKING);
+		case_cxstate(WIDLE_BSCAN);
+		}
+		break;
+	case BTC_STR_FDDT_TYPE:
+		switch(id) {
+		case_fddt_type(STOP);
+		case_fddt_type(AUTO);
+		case_fddt_type(FIX_TDD);
+		case_fddt_type(FIX_FULL_FDD);
+		}
+		break;
+	case BTC_STR_FDDT_STATE:
+		switch(id) {
+		case_fddt_state(STOP);
+		case_fddt_state(RUN);
+		case_fddt_state(PENDING);
+		case_fddt_state(DEBUG);
+		}
+		break;
+	case BTC_STR_FDDT_NORUN:
+		switch(id) {
+		case_fddt_norun(SUPPORT);
+		case_fddt_norun(FORCE_STOP);
+		case_fddt_norun(DEDICATED_ANT);
+		case_fddt_norun(ANT_ISO_LOW);
+		case_fddt_norun(ANT_ISO_HI);
+		case_fddt_norun(WL_2GSTA);
+		case_fddt_norun(WL_BUSY);
+		case_fddt_norun(WL_NOSCAN);
+		case_fddt_norun(WL_NORFK);
+		case_fddt_norun(WB_RSSI);
+		case_fddt_norun(BT_PROFILE);
+		case_fddt_norun(BT_A2DP_BUSY);
+		case_fddt_norun(BT_NOINQ);
+		case_fddt_norun(COND_NUM);
+		case_fddt_norun(NHM);
+		case_fddt_norun(RETRY_PERIOD);
+		}
+		break;
+	case BTC_STR_FDDT_TSTATE:
+		switch(id) {
+		case_fddt_tstate(STOP);
+		case_fddt_tstate(RUN);
+		case_fddt_tstate(OK);
+		case_fddt_tstate(FAIL);
+		case_fddt_tstate(NA);
+		case_fddt_tstate(DEBUG);
+		case_fddt_tstate(MAX);
+		}
+		break;
+	case BTC_STR_FDDT_TPHASE:
+		switch(id) {
+		case_fddt_tphase(STOP);
+		case_fddt_tphase(DEBUG);
+		case_fddt_tphase(TDD);
+		case_fddt_tphase(FDD_A);
+		case_fddt_tphase(FDD_B);
+		case_fddt_tphase(TDD_STAY);
+		case_fddt_tphase(FDD_STAY);
+		case_fddt_tphase(TDD_FAIL);
+		case_fddt_tphase(MAX);
+		}
+		break;
+	case BTC_STR_FDDT_CHKMAP:
+		switch(id) {
+		case_fddt_chkmap(NOEMPTY);
+		case_fddt_chkmap(TPRATIO);
+		case_fddt_chkmap(TPLOW);
+		case_fddt_chkmap(CN);
+		case_fddt_chkmap(CELLCHG);
+		case_fddt_chkmap(NHM);
+		case_fddt_chkmap(MAX);
+		}
+		break;
+	case BTC_STR_DMERROR:
+		switch(id) {
+		case_dmerr(INIT);
+		case_dmerr(PTA_OWNER);
+		case_dmerr(WL_RFK_TIMEOUT);
+		case_dmerr(BT_RFK_TIMEOUT);
+		case_dmerr(WL_FW_HANG);
+		case_dmerr(CYCLE_HANG);
+		case_dmerr(W1_HANG);
+		case_dmerr(B1_HANG);
+		case_dmerr(TDMA_NO_SYNC);
+		case_dmerr(SLOT_NO_SYNC);
+		case_dmerr(WL_SLOT_DRIFT);
+		case_dmerr(BT_SLOT_DRIFT);
+		case_dmerr(ROLE_NUM_MISMATCH);
+		case_dmerr(NULL1_TX_LATE);
+		case_dmerr(AFH_CONFLICT);
+		case_dmerr(AFH_LE_CONFLICT);
+		case_dmerr(BT_SLOT_FLOOD);
+		case_dmerr(E2G_HANG);
+		case_dmerr(WL_VER_MISMATCH);
+		case_dmerr(BT_VER_MISMATCH);
+		case_dmerr(RFE_TYPE0);
+		case_dmerr(H2C_BUF_OVER);
+		case_dmerr(BTTX_HANG);
+		case_dmerr(WL_NO_STA_NTFY);
+		case_dmerr(H2C_BMAP_MISMATCH);
+		case_dmerr(C2H_BMAP_MISMATCH);
+		case_dmerr(H2C_STRUCT_INVALID);
+		case_dmerr(C2H_STRUCT_INVALID);
+		case_dmerr(CCODE_BTNORS);
+		}
+		break;
+	case BTC_STR_FWERROR:
+		switch(id) {
+		case_fwerr(NULL_RESULT);
+		case_fwerr(NULL_EXCEPTION);
+		case_fwerr(B1_EXCEPTION);
+		case_fwerr(W1_EXCEPTION);
+		case_fwerr(B2_EXCEPTION);
+		case_fwerr(B3_EXCEPTION);
+		case_fwerr(B4_EXCEPTION);
+		case_fwerr(LPS_TIMER_START);
+		case_fwerr(EXAUTO_EXCEPTION);
+		case_fwerr(BUF_OVERFLOW);
+		case_fwerr(BRLY_EXCEPTION);
+		case_fwerr(SM);
+		case_fwerr(B2_AFTER);
+		case_fwerr(LK_END);
+		case_fwerr(TMR_FAIL);
+		case_fwerr(H2C_SET_EXCEPTION);
+		}
+		break;
+	case BTC_STR_RPTMATCH:
+		switch(id) {
+		case_rptmatch(CTRL);
+		case_rptmatch(TDMA);
+		case_rptmatch(SLOT);
+		case_rptmatch(CYSTA);
+		case_rptmatch(STEP);
+		case_rptmatch(NULLSTA);
+		case_rptmatch(MREG);
+		case_rptmatch(GPIO_DBG);
+		case_rptmatch(BT_VER);
+		case_rptmatch(BT_SCAN);
+		case_rptmatch(BT_AFH);
+		case_rptmatch(BT_DEVICE);
+		case_rptmatch(TEST);
+		}
+		break;
+	case BTC_STR_H2CERROR:
+		switch(id) {
+		case_h2cerr(REPORT_EN);
+		case_h2cerr(SLOT_TABLE);
+		case_h2cerr(MREG_TABLE);
+		case_h2cerr(CX_POLICY);
+		case_h2cerr(GPIO_DBG);
+		case_h2cerr(DRV_INFO);
+		case_h2cerr(DRV_EVENT);
+		case_h2cerr(BT_WREG_ADDR);
+		case_h2cerr(BT_WREG_VAL);
+		case_h2cerr(BT_RREG_ADDR);
+		case_h2cerr(BT_WL_CH_INFO);
+		case_h2cerr(BT_INFO_REPORT);
+		case_h2cerr(BT_IGNORE_WLAN_ACT);
+		case_h2cerr(BT_TX_PWR);
+		case_h2cerr(BT_LNA_CONSTRAIN);
+		case_h2cerr(BT_QUERY_DEV_LIST);
+		case_h2cerr(BT_QUERY_DEV_INFO);
+		case_h2cerr(BT_PSD_REPORT);
+		case_h2cerr(H2C_TEST);
+		}
+		break;
+	case BTC_STR_VENDORID:
+		switch(id) {
+		case_vendorid(NONE);
+		case_vendorid(HP);
+		case_vendorid(ASUS);
+		case_vendorid(ACER);
+		case_vendorid(LNV);
+		case_vendorid(NEC);
+		case_vendorid(AMD);
+		case_vendorid(FUJITSU);
+		case_vendorid(DELL);
+		}
+		break;
+	case BTC_STR_SPAP:
+		switch(id) {
+		case_spap(TPL);
+		case_spap(VW);
+		case_spap(SPTN);
+		default:
+			return "x";
+		}
+		break;
+	case BTC_STR_ESOC:
+		switch(id) {
+		case_esoc(NONE);
+		case_esoc(8761);
+		case_esoc(8771);
+		case_esoc(SILAB_MG21);
+		case_esoc(NORDI_NRF52840);
+		}
+		break;
+	case BTC_STR_ESOC_INTF:
+		switch(id) {
+		case_esocintf(NONE);
+		case_esocintf(PTA);
+		case_esocintf(MBX);
+		case_esocintf(SWIO);
+		case (BTC_EXTSOC_INTF_PTA | BTC_EXTSOC_INTF_MBX):
+			return "PTA+MBX";
+		case (BTC_EXTSOC_INTF_SWIO | BTC_EXTSOC_INTF_MBX):
+			return "SWIO+MBX";
+		}
+		break;
+	case BTC_STR_ESOC_STATE:
+		switch(id) {
+		case 0:
+			return "Unknown";
+		case 1:
+			return "DETACHED";
+		case 2:
+			return "CHILD";
+		case 3:
+			return "ROUTER";
+		case 4:
+			return "LEADER";
+		}
+		break;
+	case BTC_STR_ESOC_FUNC:
+		switch(id) {
+		case_esocfunc(NONE);
+		case_esocfunc(BT);
+		case_esocfunc(ZB);
+		case_esocfunc(THREAD);
+		case_esocfunc(24GPRO);
+		}
+		break;
+	case BTC_STR_PATH:
+		switch(id) {
+		case_path(A);
+		case_path(B);
+		case_path(C);
+		case_path(D);
+		case_path(AB);
+		case_path(AC);
+		case_path(AD);
+		case_path(BC);
+		case_path(BD);
+		case_path(CD);
+		case_path(ABC);
+		case_path(ABD);
+		case_path(ACD);
+		case_path(BCD);
+		case_path(ABCD);
+		}
+		break;
+	}
+	return "Undefine";
+}
+
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_def.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_def.h
new file mode 100644
index 000000000000..72128432a449
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_def.h
@@ -0,0 +1,21 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __INC_BTC_DEF_H__
+#define __INC_BTC_DEF_H__
+
+
+const char *id_to_str(u8 type, u32 id);
+
+#endif
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fw.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fw.c
new file mode 100644
index 000000000000..fc664b78d05b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fw.c
@@ -0,0 +1,1344 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_BTC_FW_C_
+#include "../hal_headers_le.h"
+#include "hal_btc.h"
+#include "halbtc_fw.h"
+#include "halbtc_def.h"
+
+#ifdef CONFIG_BTCOEX
+
+const struct btc_byte_map_desc bmap_h2c[CXDRVINFO_MAX] = {
+	{ 24,  0,  0}, /* CXDRVINFO_INIT -> struct btc_init_info */
+	{ 76,  0,  6}, /* CXDRVINFO_ROLE -> struct btc_wl_role_info */
+	{  4,  0,  0}, /* CXDRVINFO_CTRL -> struct btc_ctrl */
+	{ 12,  2,  3}, /* CXDRVINFO_TRX -> struct btc_trx_info */
+	{  1,  0,  0}, /* CXDRVINFO_TXPWR -> u8 */
+	{216,  0,  0}, /* CXDRVINFO_FDDT -> struct btc_fddt_train_info */
+};
+
+const struct btc_byte_map_desc bmap_c2h[BTC_RPT_TYPE_MAX] = {
+	{ 12, 12, 11}, /* BTC_RPT_TYPE_CTRL -> struct fbtc_rpt_ctrl */
+	{ 12,  0,  0}, /* BTC_RPT_TYPE_TDMA -> struct fbtc_1tdma */
+	{  4, 72,  1}, /* BTC_RPT_TYPE_SLOT -> struct fbtc_slots */
+	{436, 82,  2}, /* BTC_RPT_TYPE_CYSTA -> struct fbtc_cysta */
+#ifdef BTC_FW_STEP_DBG
+	{204,  0,  1}, /* BTC_RPT_TYPE_STEP -> struct fbtc_steps */
+#else
+	{  4,  0,  1}, /* BTC_RPT_TYPE_STEP -> struct fbtc_steps */
+#endif
+	{  4,  0, 14}, /* BTC_RPT_TYPE_NULLSTA -> struct fbtc_cynullsta */
+	{780,  0,  0}, /* BTC_RPT_TYPE_FDDT -> struct fbtc_fddt_sta */
+	{  4,  0, 20}, /* BTC_RPT_TYPE_MREG -> struct fbtc_mreg_val */
+	{ 36,  0,  2}, /* BTC_RPT_TYPE_GPIO_DBG -> struct fbtc_gpio_dbg */
+	{  4,  0,  3}, /* BTC_RPT_TYPE_BT_VER -> struct fbtc_btver */
+	{  4,  6,  0}, /* BTC_RPT_TYPE_BT_SCAN -> struct fbtc_btscan */
+	{ 24,  0,  0}, /* BTC_RPT_TYPE_BT_AFH -> struct fbtc_btafh */
+	{  2,  1,  2}, /* BTC_RPT_TYPE_BT_DEVICE -> struct fbtc_btdevinfo */
+	{  8,  0,  0}, /* BTC_RPT_TYPE_TEST */
+};
+
+#if BTC_PLATFORM_BIG_ENDIAN
+void _endian_translate(struct btc_t *btc, u8 dir, u8* data)
+{
+	struct btc_dm *dm = &btc->dm;
+	u8 type;
+	u16 offset, len, i;
+	const struct btc_byte_map_desc *bmap = NULL;
+
+	if (!data)
+		return;
+
+	if (dir == BTC_FW_H2C) {
+		if (data[0] >= CXDRVINFO_MAX)
+			return;
+
+		type = data[0];
+		len = data[2];
+		offset = bmap[type].map_u8 + 2;
+		bmap = bmap_h2c;
+	} else {
+		if (data[0] >= BTC_RPT_TYPE_MAX)
+			return;
+
+		type = data[0];
+		len = (data[2] << 8) + data[1];
+		offset = bmap[type].map_u8 + 3;
+		bmap = bmap_c2h;
+	}
+
+	if (len != (bmap[type].map_u8 +
+		    bmap[type].map_u16 * 2 +
+		    bmap[type].map_u32 * 4)) {
+
+		if (dir == BTC_FW_H2C)
+			dm->error |= BTC_DMERR_H2C_BMAP_MISMATCH;
+		else
+			dm->error |= BTC_DMERR_C2H_BMAP_MISMATCH;
+		return;
+
+	}
+
+	if (!bmap[type].map_u16 && !bmap[type].map_u32) /* only u8 */
+		return;
+
+	if (bmap[type].map_u16 != 0) /* u16 byte-swap */
+		for (i = 0; i < bmap[type].map_u16; i++, offset += 2)
+			btc_swap2(data[offset], data[offset + 1]);
+
+	if (bmap[type].map_u32 != 0)  /* u32 byte-swap */
+		for (i = 0; i < bmap[type].map_u32; i++, offset += 4)
+			btc_swap4(data[offset], data[offset + 1],
+				  data[offset + 2], data[offset + 3]);
+}
+#endif
+
+void _chk_btc_err(struct btc_t *btc, u8 type, u32 val)
+{
+	struct btc_cx *cx = &btc->cx;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_bt_a2dp_desc *a2dp = &bt->link_info.a2dp_desc;
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct fbtc_cysta *pcysta = &pfwinfo->rpt_fbtc_cysta.finfo;
+	u32 *presult = &pfwinfo->rpt_fbtc_nullsta.finfo.result[CXNULL_1][0];
+	u32 empty_streak = 0, empty_streak_max = 0;
+	u32 slot_pair, c_begin, c_end, cycle, s_id;
+
+	if (wl->status.map.lps == BTC_LPS_RF_OFF ||
+	    wl->status.map.rf_off)
+		return;
+
+	switch (type) {
+	case BTC_DCNT_WL_FW_VER_MATCH:
+		if ((wl->ver_info.fw_coex & 0xffffff00) !=
+		     btc->chip->wlcx_desired) {
+			wl->fw_ver_mismatch = 1;
+			dm->error |= BTC_DMERR_WL_VER_MISMATCH;
+		} else {
+			wl->fw_ver_mismatch = 0;
+			dm->error &= ~BTC_DMERR_WL_VER_MISMATCH;
+		}
+		break;
+	case BTC_DCNT_NULL_TX_FAIL:
+		if (presult[CXNULL_LATE] * BTC_NULLTX_FAIL_TH >
+		    presult[CXNULL_OK])
+			dm->cnt_dm[BTC_DCNT_NULL_TX_FAIL]++;
+		else
+			dm->cnt_dm[BTC_DCNT_NULL_TX_FAIL] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_NULL_TX_FAIL] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_NULL1_TX_LATE;
+		else
+			dm->error &= ~BTC_DMERR_NULL1_TX_LATE;
+		break;
+	case BTC_DCNT_RPT_HANG:
+		if (dm->cnt_dm[BTC_DCNT_RPT] == val && pfwinfo->rpt_en_map)
+			dm->cnt_dm[BTC_DCNT_RPT_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_RPT_HANG] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_RPT_HANG] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_WL_FW_HANG;
+		else
+			dm->error &= ~BTC_DMERR_WL_FW_HANG;
+
+		dm->cnt_dm[BTC_DCNT_RPT] = val;
+		break;
+	case BTC_DCNT_BT_SLOT_FLOOD:
+		if (!pfwinfo->rpt_fbtc_cysta.cinfo.valid ||
+		    dm->cnt_dm[BTC_DCNT_CYCLE] == val)
+		    return;
+
+		if (val < BTC_CYCLE_SLOT_MAX || /* exclude old array data */
+		    (dm->tdma_now.type != CXTDMA_AUTO &&
+		     dm->tdma_now.type != CXTDMA_AUTO2 &&
+		     dm->tdma_now.ext_ctrl != CXECTL_EXT)) {
+			dm->error &= ~BTC_DMERR_BT_SLOT_FLOOD;
+			dm->cnt_dm[BTC_DCNT_BT_SLOT_FLOOD] = 0;
+			a2dp->no_empty_streak_2s = 0;
+			a2dp->no_empty_streak_max = 0;
+			return;
+		}
+
+		/* 1 cycle = 1 wl-slot + 1 bt-slot */
+		slot_pair = BTC_CYCLE_SLOT_MAX/2;
+
+		/* dm->cnt_dm[BTC_DCNT_CYCLE] --> last cycles */
+		if (val - dm->cnt_dm[BTC_DCNT_CYCLE] > slot_pair)
+			c_begin = val - slot_pair + 1;
+		else
+			c_begin = dm->cnt_dm[BTC_DCNT_CYCLE] + 1;
+
+		c_end = val;
+
+		for (cycle = c_begin; cycle <= c_end; cycle++) {
+			s_id = ((cycle-1) % slot_pair)*2;
+
+			if (pcysta->slot_step_time[s_id] >= dm->bt_slot_flood)
+				dm->cnt_dm[BTC_DCNT_BT_SLOT_FLOOD]++;
+
+			/* calculate no-A2DP-empty streak count */
+			if (pcysta->a2dp_trx[s_id].empty_cnt == 0 &&
+			    pcysta->a2dp_trx[s_id+1].empty_cnt == 0) {
+				empty_streak++;
+				if (empty_streak > empty_streak_max)
+					empty_streak_max = empty_streak;
+			} else {
+				empty_streak = 0;
+			}
+		}
+
+		a2dp->no_empty_streak_2s = empty_streak_max;
+
+		if (a2dp->no_empty_streak_2s > a2dp->no_empty_streak_max)
+			a2dp->no_empty_streak_max = a2dp->no_empty_streak_2s;
+
+		if (dm->cnt_dm[BTC_DCNT_BT_SLOT_FLOOD])
+			dm->error |= BTC_DMERR_BT_SLOT_FLOOD;
+		break;
+	case BTC_DCNT_CYCLE_HANG:
+		if (dm->cnt_dm[BTC_DCNT_CYCLE] == val &&
+		    (dm->tdma_now.type != CXTDMA_OFF ||
+		     dm->tdma_now.ext_ctrl == CXECTL_EXT))
+			dm->cnt_dm[BTC_DCNT_CYCLE_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_CYCLE_HANG] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_CYCLE_HANG])
+			dm->error |= BTC_DMERR_CYCLE_HANG;
+		else
+			dm->error &= ~BTC_DMERR_CYCLE_HANG;
+
+		dm->cnt_dm[BTC_DCNT_CYCLE] = val;
+		break;
+	case BTC_DCNT_W1_HANG:
+		if (dm->cnt_dm[BTC_DCNT_W1] == val &&
+		    dm->tdma_now.type != CXTDMA_OFF)
+			dm->cnt_dm[BTC_DCNT_W1_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_W1_HANG] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_W1_HANG])
+			dm->error |= BTC_DMERR_W1_HANG;
+		else
+			dm->error &= ~BTC_DMERR_W1_HANG;
+
+		dm->cnt_dm[BTC_DCNT_W1] = val;
+		break;
+	case BTC_DCNT_B1_HANG:
+		if (dm->cnt_dm[BTC_DCNT_B1] == val &&
+		    dm->tdma_now.type != CXTDMA_OFF)
+			dm->cnt_dm[BTC_DCNT_B1_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_B1_HANG] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_B1_HANG] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_B1_HANG;
+		else
+			dm->error &= ~BTC_DMERR_B1_HANG;
+
+		dm->cnt_dm[BTC_DCNT_B1] = val;
+		break;
+	case BTC_DCNT_E2G_HANG:
+		if (dm->cnt_dm[BTC_DCNT_E2G] == val &&
+		    dm->tdma_now.ext_ctrl == CXECTL_EXT)
+			dm->cnt_dm[BTC_DCNT_E2G_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_E2G_HANG] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_E2G_HANG] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_E2G_HANG;
+		else
+			dm->error &= ~BTC_DMERR_E2G_HANG;
+
+		dm->cnt_dm[BTC_DCNT_E2G] = val;
+		break;
+	case BTC_DCNT_TDMA_NONSYNC:
+		if (val != 0) /* if tdma not sync between drv/fw  */
+			dm->cnt_dm[BTC_DCNT_TDMA_NONSYNC]++;
+		else
+			dm->cnt_dm[BTC_DCNT_TDMA_NONSYNC] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_TDMA_NONSYNC] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_TDMA_NO_SYNC;
+		else
+			dm->error &= ~BTC_DMERR_TDMA_NO_SYNC;
+		break;
+	case BTC_DCNT_SLOT_NONSYNC:
+		if (val != 0) /* if slot not sync between drv/fw  */
+			dm->cnt_dm[BTC_DCNT_SLOT_NONSYNC]++;
+		else
+			dm->cnt_dm[BTC_DCNT_SLOT_NONSYNC] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_SLOT_NONSYNC] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_SLOT_NO_SYNC;
+		else
+			dm->error &= ~BTC_DMERR_SLOT_NO_SYNC;
+		break;
+	case BTC_DCNT_BTCNT_HANG:
+		val = cx->cnt_bt[BTC_BCNT_HIPRI_RX] +
+		      cx->cnt_bt[BTC_BCNT_HIPRI_TX] +
+		      cx->cnt_bt[BTC_BCNT_LOPRI_RX] +
+		      cx->cnt_bt[BTC_BCNT_LOPRI_TX];
+
+		if (val == 0)
+			dm->cnt_dm[BTC_DCNT_BTCNT_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_BTCNT_HANG] = 0;
+		break;
+	case BTC_DCNT_BTTX_HANG:
+		val = cx->cnt_bt[BTC_BCNT_LOPRI_TX];
+
+		if (val == 0 && bt->link_info.slave_role)
+			dm->cnt_dm[BTC_DCNT_BTTX_HANG]++;
+		else
+			dm->cnt_dm[BTC_DCNT_BTTX_HANG] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_BTTX_HANG] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_BTTX_HANG;
+		else
+			dm->error &= ~BTC_DMERR_BTTX_HANG;
+		break;
+	case BTC_DCNT_WL_SLOT_DRIFT:
+		if (val > dm->slot_now[CXST_W1].dur)
+			val = val - dm->slot_now[CXST_W1].dur;
+		else
+			val = dm->slot_now[CXST_W1].dur - val;
+
+		if (val >= BTC_CHK_WLSLOT_DRIFT_MAX)
+			dm->cnt_dm[BTC_DCNT_WL_SLOT_DRIFT]++;
+		else
+			dm->cnt_dm[BTC_DCNT_WL_SLOT_DRIFT] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_WL_SLOT_DRIFT] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_WL_SLOT_DRIFT;
+		else
+			dm->error &= ~BTC_DMERR_WL_SLOT_DRIFT;
+		break;
+	case BTC_DCNT_BT_SLOT_DRIFT:
+		if (!btc->bt_req_len[wl->pta_req_mac])
+			return;
+
+		if (val > btc->bt_req_len[wl->pta_req_mac])
+			val = val - btc->bt_req_len[wl->pta_req_mac];
+		else
+			val = btc->bt_req_len[wl->pta_req_mac] - val;
+
+		if (val >= BTC_CHK_BTSLOT_DRIFT_MAX)
+			dm->cnt_dm[BTC_DCNT_BT_SLOT_DRIFT]++;
+		else
+			dm->cnt_dm[BTC_DCNT_BT_SLOT_DRIFT] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_BT_SLOT_DRIFT] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_BT_SLOT_DRIFT;
+		else
+			dm->error &= ~BTC_DMERR_BT_SLOT_DRIFT;
+		break;
+	case BTC_DCNT_WL_STA_NTFY:
+		val = dm->cnt_notify[BTC_NTFY_WL_STA] -
+		      dm->cnt_notify[BTC_NTFY_WL_STA_LAST];
+
+		dm->cnt_notify[BTC_NTFY_WL_STA_LAST] =
+				dm->cnt_notify[BTC_NTFY_WL_STA];
+
+		if (val == 0 && wl->role_info.link_mode != BTC_WLINK_NOLINK)
+			dm->cnt_dm[BTC_DCNT_WL_STA_NTFY]++;
+		else
+			dm->cnt_dm[BTC_DCNT_WL_STA_NTFY] = 0;
+
+		if (dm->cnt_dm[BTC_DCNT_WL_STA_NTFY] >= BTC_CHK_ERR_TH)
+			dm->error |= BTC_DMERR_WL_NO_STA_NTFY;
+		else
+			dm->error &= ~BTC_DMERR_WL_NO_STA_NTFY;
+		break;
+	}
+}
+
+static void _update_bt_report(struct btc_t *btc, u8 rpt_type, u8* pfinfo)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct btc_bt_link_info *b = &bt->link_info;
+	struct btc_bt_a2dp_desc *a2dp = &b->a2dp_desc;
+
+	struct fbtc_btver* pver = (struct fbtc_btver*) pfinfo;
+	struct fbtc_btscan* pscan = (struct fbtc_btscan*) pfinfo;
+	struct fbtc_btafh* pafh = (struct fbtc_btafh*) pfinfo;
+	struct fbtc_btdevinfo* pdev = (struct fbtc_btdevinfo*) pfinfo;
+	u8 i, j, xmap, omap;
+	u32 cnt = 0, cnt_le = 0, cnt_ch = 0, cnt_ch_le = 0;
+	u16 win, intvl;
+
+	switch (rpt_type) {
+	case BTC_RPT_TYPE_BT_VER:
+		bt->ver_info.fw = pver->fw_ver;
+		bt->ver_info.fw_coex = (pver->coex_ver & bMASKB0);
+		bt->feature = pver->feature;
+
+		if (bt->ver_info.fw_coex != btc->chip->btcx_desired) {
+			bt->fw_ver_mismatch = 1;
+			dm->error |= BTC_DMERR_BT_VER_MISMATCH;
+		} else {
+			bt->fw_ver_mismatch = 0;
+			dm->error &= ~BTC_DMERR_BT_VER_MISMATCH;
+		}
+		break;
+	case BTC_RPT_TYPE_BT_SCAN:
+		for(i = CXSCAN_BG; i < CXSCAN_MAX1; i++) {
+			hal_mem_cpy(h, &bt->scan_info[i], &pscan->para[i], 4);
+			win = pscan->para[i].win;
+			intvl = pscan->para[i].intvl;
+#ifdef BTC_PAUTO_LESCAN
+			if (i == CXSCAN_INIT && (btc->mdinfo.wa_type & BTC_WA_INIT_SCAN)) {
+				if (intvl <= LE_INIT_INTVL && intvl != 0)
+					dm->le_init_en = 1;
+				else
+					dm->le_init_en = 0;
+			}
+#endif
+		}
+		break;
+	case BTC_RPT_TYPE_BT_AFH:
+		if (pafh->map_type & BTC_RPT_BT_AFH_LEGACY) {
+			hal_mem_cpy(h, &b->afh_map[0], pafh->afh_l, 4);
+			hal_mem_cpy(h, &b->afh_map[4], pafh->afh_m, 4);
+			hal_mem_cpy(h, &b->afh_map[8], pafh->afh_h, 2);
+			cx->cnt_bt[BTC_BCNT_AFH_UPDATE]++;
+			cx->cnt_bt[BTC_BCNT_AFH_CONFLICT] = 0;
+			dm->error &= ~BTC_DMERR_AFH_CONFLICT;
+		}
+
+		for(i = 0; i < 12; i++) {
+			if (!(pafh->map_type & BTC_RPT_BT_AFH_LEGACY) ||
+			    !b->status.map.connect)
+				break;
+
+			omap = b->afh_map[i];
+			/* conflict channel map*/
+			xmap = omap & wl->ch_map[i];
+
+			for (j = 0; j < 8; j++) {
+				if (omap & BIT(j))
+					cnt_ch++;
+				if ((xmap & BIT(j)) && wl->afh_info.en)
+					cnt++;
+			}
+
+			if (i == 11) {
+				cx->cnt_bt[BTC_BCNT_AFH_CHN] = cnt_ch;
+				cx->cnt_bt[BTC_BCNT_AFH_CONFLICT] = cnt;
+				if (cnt)
+					dm->error |= BTC_DMERR_AFH_CONFLICT;
+			}
+		}
+
+		if (pafh->map_type & BTC_RPT_BT_AFH_LE) {
+			hal_mem_cpy(h, &b->afh_map_le[0], pafh->afh_le_a, 4);
+			hal_mem_cpy(h, &b->afh_map_le[4], pafh->afh_le_b, 1);
+			cx->cnt_bt[BTC_BCNT_AFH_LE_UPDATE]++;
+			cx->cnt_bt[BTC_BCNT_AFH_LE_CONFLICT] = 0;
+			dm->error &= ~BTC_DMERR_AFH_LE_CONFLICT;
+		}
+
+		for(i = 0; i < 5; i++) {
+			if (!(pafh->map_type & BTC_RPT_BT_AFH_LE) ||
+			    (!b->status.map.ble_connect &&
+			     !b->leaudio_desc.exist))
+				break;
+
+			omap = b->afh_map_le[i];
+			/* conflict channel map*/
+			xmap = omap & wl->ch_map_le[i];
+
+			for (j = 0; j < 8; j++) {
+				if (omap & BIT(j))
+					cnt_ch_le++;
+				if ((xmap & BIT(j))  && wl->afh_info.en)
+					cnt_le++;
+			}
+
+			if (i == 4) {
+				cx->cnt_bt[BTC_BCNT_AFH_LE_CHN] = cnt_ch_le;
+				cx->cnt_bt[BTC_BCNT_AFH_LE_CONFLICT] = cnt_le;
+				if (cnt_le)
+					dm->error |= BTC_DMERR_AFH_LE_CONFLICT;
+			}
+		}
+		break;
+	case BTC_RPT_TYPE_BT_DEVICE:
+		a2dp->device_name = pdev->dev_name;
+		a2dp->vendor_id = pdev->vendor_id;
+		a2dp->flush_time = pdev->flush_time;
+		break;
+	default:
+		break;
+	}
+}
+
+static u32 _chk_btc_report(struct btc_t *btc, u8 *prptbuf, u32 index)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct btc_wl_role_info *wl_rinfo = &btc->cx.wl.role_info;
+	struct rtw_hal_com_t *hal = btc->hal;
+	struct btc_rpt_cmn_info *pcinfo = NULL;
+	struct btc_cx *cx = &btc->cx;
+	struct btc_wl_info *wl = &cx->wl;
+	struct btc_bt_info *bt = &cx->bt;
+	struct fbtc_rpt_ctrl *prpt = NULL;
+	struct fbtc_cysta *pcysta = NULL;
+	struct fbtc_cynullsta *pcynull = NULL;
+	struct fbtc_1tdma *ptdma = NULL;
+	struct fbtc_slot_u16 *pslot = NULL;
+	struct fbtc_testinfo *testinfo = NULL;
+	struct btc_chip_ops *ops = btc->chip->ops;
+	u8 rpt_type = 0;
+	u8 *rpt_content = NULL;
+	u8 *pfinfo = NULL;
+	u32 rpt_len = 0, i = 0;
+	u32 val1, val2;
+
+	if (!prptbuf) {
+		pfwinfo->err[BTFRE_INVALID_INPUT]++;
+		return 0;
+	}
+
+#if BTC_PLATFORM_BIG_ENDIAN
+	_endian_translate(btc, BTC_FW_C2H, prptbuf);
+#endif
+
+	rpt_type = prptbuf[index];
+	rpt_len = (prptbuf[index+2] << 8) + prptbuf[index+1];
+	rpt_content = &prptbuf[index+3];
+
+	switch (rpt_type) {
+	case BTC_RPT_TYPE_CTRL:
+		pcinfo = &pfwinfo->rpt_ctrl.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_ctrl.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_rpt_ctrl);
+		pcinfo->req_fver = FCX_VER_BTCRPT;
+		break;
+	case BTC_RPT_TYPE_TDMA:
+		pcinfo = &pfwinfo->rpt_fbtc_tdma.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_tdma.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_1tdma);
+		pcinfo->req_fver = FCX_VER_TDMA;
+		break;
+	case BTC_RPT_TYPE_SLOT:
+		pcinfo = &pfwinfo->rpt_fbtc_slots.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_slots.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_slots);
+		pcinfo->req_fver = FCX_VER_SLOT;
+		break;
+	case BTC_RPT_TYPE_CYSTA:
+		pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_cysta.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_cysta);
+		pcinfo->req_fver = FCX_VER_CYSTA;
+		break;
+#ifdef BTC_FDDT_TRAIN_SUPPORT
+	case BTC_RPT_TYPE_FDDT:
+		pcinfo = &pfwinfo->rpt_fbtc_fddt.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_fddt.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_fddt_sta);
+		pcinfo->req_fver = FCX_VER_FDDT;
+		break;
+#endif
+#ifdef BTC_FW_STEP_DBG
+	case BTC_RPT_TYPE_STEP:
+		pcinfo = &pfwinfo->rpt_fbtc_step.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_step.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_steps);
+		pcinfo->req_fver = FCX_VER_STEP;
+		break;
+#endif
+	case BTC_RPT_TYPE_NULLSTA:
+		pcinfo = &pfwinfo->rpt_fbtc_nullsta.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_nullsta.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_cynullsta);
+		pcinfo->req_fver = FCX_VER_NULLSTA;
+		break;
+	case BTC_RPT_TYPE_MREG:
+		pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_mregval.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_mreg_val);
+		pcinfo->req_fver = FCX_VER_MREG;
+		break;
+	case BTC_RPT_TYPE_GPIO_DBG:
+		pcinfo = &pfwinfo->rpt_fbtc_gpio_dbg.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_gpio_dbg.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_gpio_dbg);
+		pcinfo->req_fver = FCX_VER_GPIODBG;
+		break;
+	case BTC_RPT_TYPE_BT_VER:
+		pcinfo = &pfwinfo->rpt_fbtc_btver.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btver.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_btver);
+		pcinfo->req_fver = FCX_VER_BTVER;
+		break;
+	case BTC_RPT_TYPE_BT_SCAN:
+		pcinfo = &pfwinfo->rpt_fbtc_btscan.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btscan.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_btscan);
+		pcinfo->req_fver = FCX_VER_BTSCAN;
+		break;
+	case BTC_RPT_TYPE_BT_AFH:
+		pcinfo = &pfwinfo->rpt_fbtc_btafh.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btafh.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_btafh);
+		pcinfo->req_fver = FCX_VER_BTAFH;
+		break;
+	case BTC_RPT_TYPE_BT_DEVICE:
+		pcinfo = &pfwinfo->rpt_fbtc_btdev.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btdev.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_btdevinfo);
+		pcinfo->req_fver = FCX_VER_BTDEVINFO;
+		break;
+	case BTC_RPT_TYPE_TEST:
+		pcinfo = &pfwinfo->rpt_fbtc_testinfo.cinfo;
+		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_testinfo.finfo);
+		pcinfo->req_len = sizeof(struct fbtc_testinfo);
+		pcinfo->req_fver = FCX_VER_TEST;
+		break;
+	default:
+		pfwinfo->err[BTFRE_UNDEF_TYPE]++;
+		return 0;
+	}
+
+	pcinfo->rsp_fver = *rpt_content;
+	pcinfo->rx_len = rpt_len;
+	pcinfo->rx_cnt++;
+
+	if (rpt_len != pcinfo->req_len) {
+		pfwinfo->len_mismch |= BIT(rpt_type);
+		pcinfo->valid = 0;
+		return 0;
+	} else if (pcinfo->req_fver != pcinfo->rsp_fver) {
+		pfwinfo->fver_mismch |= BIT(rpt_type);
+		pcinfo->valid = 0;
+		return 0;
+	} else if (!pfinfo || !pcinfo->req_len) {
+		pfwinfo->err[BTFRE_EXCEPTION]++;
+		pcinfo->valid = 0;
+		return 0;
+	}
+
+	hal_mem_cpy(hal, (void *)pfinfo, (void *)rpt_content, pcinfo->req_len);
+	pcinfo->valid = 1;
+
+	switch (rpt_type) {
+	case BTC_RPT_TYPE_CTRL:
+		prpt = &pfwinfo->rpt_ctrl.finfo;
+		pfwinfo->rpt_en_map = prpt->rpt_info.en;
+		wl->ver_info.fw_coex = prpt->rpt_info.cx_ver;
+		wl->ver_info.fw = prpt->rpt_info.fw_ver;
+		cx->other.ext_pta_req_exist = prpt->ext_req_exist;
+
+		val1 = sizeof(struct btc_gnt_ctrl);
+		for (i = HW_PHY_0; i < HW_PHY_MAX; i++)
+			hal_mem_cpy(hal, (void *)&dm->gnt_val[i],
+				    (void *)&prpt->gnt_val[i][0], val1);
+
+		cx->cnt_bt[BTC_BCNT_HIPRI_TX] = prpt->bt_cnt[BTC_BCNT_HI_TX];
+		cx->cnt_bt[BTC_BCNT_HIPRI_RX] = prpt->bt_cnt[BTC_BCNT_HI_RX];
+		cx->cnt_bt[BTC_BCNT_LOPRI_TX] = prpt->bt_cnt[BTC_BCNT_LO_TX];
+		cx->cnt_bt[BTC_BCNT_LOPRI_RX] = prpt->bt_cnt[BTC_BCNT_LO_RX];
+
+		val1 = prpt->bt_cnt[BTC_BCNT_POLUT];
+		if (val1 > cx->cnt_bt[BTC_BCNT_POLUT_NOW]) /* check if increase */
+			val1 = val1 - cx->cnt_bt[BTC_BCNT_POLUT_NOW]; /* diff */
+
+		cx->cnt_bt[BTC_BCNT_POLUT_DIFF] = val1;
+		cx->cnt_bt[BTC_BCNT_POLUT_NOW] = prpt->bt_cnt[BTC_BCNT_POLUT];
+		dm->pta_owner = prpt->pta_owner;
+
+		val1 = pfwinfo->event[BTF_EVNT_RPT];
+		_chk_btc_err(btc, BTC_DCNT_BTCNT_HANG, 0);
+		_chk_btc_err(btc, BTC_DCNT_RPT_HANG, val1);
+		_chk_btc_err(btc, BTC_DCNT_WL_FW_VER_MATCH, 0);
+		_chk_btc_err(btc, BTC_DCNT_BTTX_HANG, 0);
+		_chk_btc_err(btc, BTC_DCNT_WL_STA_NTFY, 0);
+		break;
+	case BTC_RPT_TYPE_TDMA:
+		ptdma = &pfwinfo->rpt_fbtc_tdma.finfo;
+		_chk_btc_err(btc, BTC_DCNT_TDMA_NONSYNC,
+			     _tdma_cmp(&dm->tdma_now, &ptdma->tdma));
+
+		break;
+	case BTC_RPT_TYPE_SLOT:
+		pslot = pfwinfo->rpt_fbtc_slots.finfo.slot;
+		for (i = 0; i < CXST_MAX; i++) {
+			val2 = (pslot[i].cxtbl_h16 << 16) + pslot[i].cxtbl_l16;
+			if (dm->slot_now[i].dur != pslot[i].dur ||
+			    dm->slot_now[i].cxtype != pslot[i].cxtype ||
+			    dm->slot_now[i].cxtbl != val2) {
+				val1 = 1;
+			} else {
+				val1 = 0;
+			}
+
+			_chk_btc_err(btc, BTC_DCNT_SLOT_NONSYNC, val1);
+
+			if (dm->error & BTC_DMERR_SLOT_NO_SYNC)
+				break; /* exit if 1 slot no-sync error occur*/
+		}
+		break;
+	case BTC_RPT_TYPE_CYSTA:
+		if (dm->fddt_train == BTC_FDDT_ENABLE)
+			break;
+
+		pcysta = &pfwinfo->rpt_fbtc_cysta.finfo;
+
+		if (dm->tdma_now.type != CXTDMA_OFF) {
+			/* Check diff time between real WL slot and W1 slot */
+			_chk_btc_err(btc, BTC_DCNT_WL_SLOT_DRIFT,
+				     pcysta->cycle_time.tavg[CXT_WL]);
+
+			/* Check Leak-AP */
+			val1 = pcysta->leak_slot.cnt_rximr * BTC_LEAK_AP_TH;
+			val2 = pcysta->slot_cnt[CXST_LK];
+			if (val2 >= (BTC_CYSTA_CHK_PERIOD * 2) &&
+			    dm->tdma_now.rxflctrl && val1 > val2)
+				dm->leak_ap = 1;
+		} else if (dm->tdma_now.ext_ctrl == CXECTL_EXT &&
+			   wl_rinfo->link_mode != BTC_WLINK_2G_GC) {
+			/* Check diff between real BT slot and EBT/E5G slot */
+			_chk_btc_err(btc, BTC_DCNT_BT_SLOT_DRIFT,
+				     pcysta->cycle_time.tavg[CXT_BT]);
+
+			/* Check bt slot length for P2P mode*/
+			val1 = pcysta->a2dp_ept.cnt_timeout * BTC_SLOT_REQ_TH;
+			val2 = pcysta->a2dp_ept.cnt;
+			if (pcysta->cycles >= BTC_CYSTA_CHK_PERIOD &&
+			    val1 > val2)
+				dm->slot_req_more = 1;
+			else if (bt->link_info.status.map.connect == 0)
+				dm->slot_req_more = 0;
+		}
+
+		_chk_btc_err(btc, BTC_DCNT_E2G_HANG,pcysta->slot_cnt[CXST_E2G]);
+		_chk_btc_err(btc, BTC_DCNT_W1_HANG, pcysta->slot_cnt[CXST_W1]);
+		_chk_btc_err(btc, BTC_DCNT_B1_HANG, pcysta->slot_cnt[CXST_B1]);
+
+		/* "BT_SLOT_FLOOD" error-check MUST before "CYCLE_HANG" */
+		_chk_btc_err(btc, BTC_DCNT_BT_SLOT_FLOOD, (u32)pcysta->cycles);
+		_chk_btc_err(btc, BTC_DCNT_CYCLE_HANG, (u32)pcysta->cycles);
+		break;
+	case BTC_RPT_TYPE_NULLSTA:
+		pcynull = &pfwinfo->rpt_fbtc_nullsta.finfo;
+		if (dm->fddt_train == BTC_FDDT_ENABLE ||
+		    pcynull->result[CXNULL_1][CXNULL_TX] <
+		    BTC_NULLTX_CHK_PERIOD)
+			break;
+
+		_chk_btc_err(btc, BTC_DCNT_NULL_TX_FAIL, 0);
+		break;
+	case BTC_RPT_TYPE_MREG:
+		if (!ops || !ops->get_reg_status)
+			break;
+		ops->get_reg_status(btc, BTC_CSTATUS_BB_GNT_MUX_MON, (void*)&i);
+		if (dm->wl_btg_rx == BTC_BB_GNT_FWCTRL)
+			dm->wl_btg_rx_rb = BTC_BB_GNT_FWCTRL;
+		else
+			dm->wl_btg_rx_rb = i;
+
+		ops->get_reg_status(btc, BTC_CSTATUS_BB_PRE_AGC_MON, (void*)&i);
+		if (dm->wl_pre_agc == BTC_BB_PRE_AGC_FWCTRL)
+			dm->wl_pre_agc_rb = BTC_BB_PRE_AGC_FWCTRL;
+		else
+			dm->wl_pre_agc_rb = (u8)i;
+		break;
+	case BTC_RPT_TYPE_BT_VER:
+	case BTC_RPT_TYPE_BT_SCAN:
+	case BTC_RPT_TYPE_BT_AFH:
+	case BTC_RPT_TYPE_BT_DEVICE:
+		_update_bt_report(btc, rpt_type, pfinfo);
+		break;
+	case BTC_RPT_TYPE_TEST:
+		testinfo = &pfwinfo->rpt_fbtc_testinfo.finfo;
+		break;
+	}
+
+	return (rpt_len + BTC_RPT_HDR_SIZE);
+}
+
+static void _append_tdma(struct btc_t *btc, bool force_exec)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btf_tlv *tlv = NULL;
+	u16 len;
+
+	len = btc->policy_len;
+
+	if ((len + sizeof(struct fbtc_tdma) + 3) > BTC_POLICY_MAXLEN) {
+		PHL_INFO("[BTC], %s(): buff overflow!\n", __func__);
+		return;
+	}
+
+	if (!force_exec && !_tdma_cmp(&dm->tdma, &dm->tdma_now)) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s(): tdma no change!\n", __func__);
+		return;
+	}
+
+	tlv = (struct btf_tlv *)&btc->policy[len];
+	tlv->type = CXPOLICY_TDMA;
+	tlv->ver = FCX_VER_TDMA;
+	tlv->len = sizeof(struct fbtc_tdma);
+
+	_tdma_cpy(&tlv->val[0], &dm->tdma); /* put tdma-descriptor to h2c-buf */
+	len += (3 + (u16)tlv->len);
+	btc->policy_len = len; /* update total length */
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		 "[BTC], %s: len=%d, type:%d, rxfctrl=%d, txfctrl=%d, rsvd=%d, "
+		 "leak_n=%d, ext_ctrl=%d, rxflctrl_role=0x%x, opt_ctrl=0x%x\n",
+		 __func__, btc->policy_len,
+		 dm->tdma.type, dm->tdma.rxflctrl, dm->tdma.txflctrl,
+		 dm->tdma.rsvd, dm->tdma.leak_n, dm->tdma.ext_ctrl,
+		 dm->tdma.rxflctrl_role, dm->tdma.option_ctrl);
+}
+
+static void _append_slot(struct btc_t *btc, bool force_exec)
+{
+	struct btc_dm *dm = &btc->dm;
+	struct btf_tlv *tlv = NULL;
+	u8 i, cnt = 0;
+	u16 len;
+
+	for (i = 0; i < CXST_MAX; i++) {
+		if (!force_exec && !_slot_cmp(&dm->slot[i], &dm->slot_now[i]))
+			continue;
+
+		len = btc->policy_len;
+
+		if (cnt == 0) {
+			if ((len + 3) > BTC_POLICY_MAXLEN) {
+				PHL_INFO("[BTC], %s(): buff overflow!\n",
+					 __func__);
+				break;
+			}
+
+			tlv = (struct btf_tlv *)&btc->policy[len];
+			tlv->type = CXPOLICY_SLOT;
+			tlv->ver = FCX_VER_SLOT;
+			tlv->len = sizeof(struct fbtc_slot) + 1; /*u8 + 1-slot*/
+			len += 3;
+		}
+
+		if ((len + (u16)tlv->len) > BTC_POLICY_MAXLEN) {
+			PHL_INFO("[BTC], %s(): buff overflow!\n", __func__);
+			break;
+		}
+
+		btc->policy[len] = i; /* slot-id */
+
+		/* put tdma-descriptor to h2c-buf */
+		btc->policy[len+1] = (u8)(dm->slot[i].dur & bMASKB0);
+		btc->policy[len+2] = (u8)((dm->slot[i].dur & bMASKB1) >> 8);
+
+		btc->policy[len+3] = (u8)(dm->slot[i].cxtype & bMASKB0);
+		btc->policy[len+4] = (u8)((dm->slot[i].cxtype & bMASKB1) >> 8);
+
+		btc->policy[len+5] = (u8)(dm->slot[i].cxtbl & bMASKB0);
+		btc->policy[len+6] = (u8)((dm->slot[i].cxtbl & bMASKB1) >> 8);
+		btc->policy[len+7] = (u8)((dm->slot[i].cxtbl & bMASKB2) >> 16);
+		btc->policy[len+8] = (u8)((dm->slot[i].cxtbl & bMASKB3) >> 24);
+
+		len += tlv->len;
+
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			  "[BTC], %s: policy_len=%d, slot-%d:"
+			  " dur=%d, type=%d, table=0x%08x\n",
+			  __func__, btc->policy_len, i, dm->slot[i].dur,
+			  dm->slot[i].cxtype, dm->slot[i].cxtbl);
+		cnt++;
+		btc->policy_len = len; /* update total length */
+	}
+
+	if (cnt > 0)
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			 "[BTC], %s: slot update (cnt=%d, len=%d)!!\n",
+			 __func__, cnt, btc->policy_len);
+}
+
+/*
+ * extern functions
+ */
+
+void hal_btc_fw_chk_struct(struct btc_t *btc)
+{
+	struct btc_dm *dm = &btc->dm;
+	const struct btc_byte_map_desc *bmap = NULL;
+	u8 i = 0;
+	u16 sz = 0, map_sz = 0;
+
+	/* cehck h2c structure */
+	bmap = bmap_h2c;
+	for (i = 0; i < CXDRVINFO_MAX; i++) {
+
+		switch(i) {
+		case CXDRVINFO_INIT:
+			sz = sizeof(struct btc_init_info);
+			break;
+		case CXDRVINFO_ROLE:
+			sz = sizeof(struct btc_wl_role_info);
+			break;
+		case CXDRVINFO_CTRL:
+			sz = sizeof(struct btc_ctrl);
+			break;
+		case CXDRVINFO_TRX:
+			sz = sizeof(struct btc_trx_info);
+			break;
+		case CXDRVINFO_TXPWR:
+			sz = sizeof(u8);
+			break;
+		case CXDRVINFO_FDDT:
+			sz = sizeof(struct btc_fddt_train_info);
+			break;
+		}
+
+		map_sz = bmap[i].map_u8 + bmap[i].map_u16 * 2 +
+			 bmap[i].map_u32 * 4;
+
+		if (sz % 4 != 0 && i != CXDRVINFO_TXPWR)
+			dm->error |= BTC_DMERR_H2C_STRUCT_INVALID;
+
+		if (sz != map_sz)
+			dm->error |= BTC_DMERR_H2C_BMAP_MISMATCH;
+	}
+
+	/* cehck c2h structure */
+	bmap = bmap_c2h;
+	for (i = 0; i < BTC_RPT_TYPE_MAX; i++) {
+
+		switch(i) {
+		case BTC_RPT_TYPE_CTRL:
+			sz = sizeof(struct fbtc_rpt_ctrl);
+			break;
+		case BTC_RPT_TYPE_TDMA:
+			sz = sizeof(struct fbtc_1tdma);
+			break;
+		case BTC_RPT_TYPE_SLOT:
+			sz = sizeof(struct fbtc_slots);
+			break;
+		case BTC_RPT_TYPE_CYSTA:
+			sz = sizeof(struct fbtc_cysta);
+			break;
+		case BTC_RPT_TYPE_STEP:
+			sz = sizeof(struct fbtc_steps);
+			break;
+		case BTC_RPT_TYPE_NULLSTA:
+			sz = sizeof(struct fbtc_cynullsta);
+			break;
+		case BTC_RPT_TYPE_FDDT:
+			sz = sizeof(struct fbtc_fddt_sta);
+			break;
+		case BTC_RPT_TYPE_MREG:
+			sz = sizeof(struct fbtc_mreg_val);
+			break;
+		case BTC_RPT_TYPE_GPIO_DBG:
+			sz = sizeof(struct fbtc_gpio_dbg);
+			break;
+		case BTC_RPT_TYPE_BT_VER:
+			sz = sizeof(struct fbtc_btver);
+			break;
+		case BTC_RPT_TYPE_BT_SCAN:
+			sz = sizeof(struct fbtc_btscan);
+			break;
+		case BTC_RPT_TYPE_BT_AFH:
+			sz = sizeof(struct fbtc_btafh);
+			break;
+		case BTC_RPT_TYPE_BT_DEVICE:
+			sz = sizeof(struct fbtc_btdevinfo);
+			break;
+		case BTC_RPT_TYPE_TEST:
+			sz = sizeof(struct fbtc_testinfo);;
+			break;
+		}
+
+		map_sz = bmap[i].map_u8 + bmap[i].map_u16 * 2 +
+			 bmap[i].map_u32 * 4;
+
+		if (sz % 4 != 0)
+			dm->error |= BTC_DMERR_C2H_STRUCT_INVALID;
+
+		if (sz != map_sz)
+			dm->error |= BTC_DMERR_C2H_BMAP_MISMATCH;
+	}
+}
+
+void hal_btc_fw_set_rpt(struct btc_t *btc, u32 rpt_map, u32 rpt_state)
+{
+	struct btf_fwinfo* fwinfo = &btc->fwinfo;
+	struct btc_wl_smap *wl_smap = &btc->cx.wl.status.map;
+	u8 data[7] = {0};
+	u16 len = 0;
+	u32 val = 0;
+
+	if (rpt_state > 2)
+		return;
+
+	/* Skip enable report if enter IPS/LPS/RF-Off  */
+	if ((wl_smap->rf_off || wl_smap->lps != BTC_LPS_OFF) &&
+	     rpt_state != 0)
+	     return;
+
+	if (rpt_state == 2) /* for multi-bit operation */
+		val = rpt_map;
+	else if (rpt_state == 1)
+		val = fwinfo->rpt_en_map | rpt_map;
+	else if (rpt_state == 0)
+		val = fwinfo->rpt_en_map & (~rpt_map);
+
+	if ((val == fwinfo->rpt_en_map) && (!(rpt_map & RPT_EN_MREG)))
+		return;
+
+	data[0] = SET_REPORT_EN;  /* type */
+	data[1] = FCX_VER_BTCRPT; /* ver */
+	data[2] = 4;  /*  length */
+
+	/* report-enable data */
+	data[3] = (u8)(val & bMASKB0);
+	data[4] = (u8)((val & bMASKB1) >> 8);
+	data[5] = (u8)((val & bMASKB2) >> 16);
+	data[6] = (u8)((val & bMASKB3) >> 24);
+
+	len = data[2] + 3;
+	if (_send_fw_cmd(btc, SET_REPORT_EN, data, len))
+		fwinfo->rpt_en_map = val;
+}
+
+void hal_btc_fw_set_slots(struct btc_t *btc)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_dm *dm = &btc->dm;
+	struct btf_tlv *tlv = NULL;
+	u8 *ptr = NULL;
+	u8 i;
+	u16 len = 0, n = 0;
+
+	len = sizeof(btc->dm.slot) + sizeof(*tlv) - 1;
+	tlv = hal_mem_alloc(h, len);
+	if (!tlv)
+		return;
+
+	tlv->type = SET_SLOT_TABLE;
+	tlv->ver = FCX_VER_SLOT;
+	tlv->len = ARRAY_SIZE(dm->slot); /* total slot count: CXST_MAX*/
+	ptr = &tlv->val[0];
+
+	for (i = 0; i < tlv->len; i++) {
+		n = i * sizeof(struct fbtc_slot);
+		ptr[n+0] = (u8)(dm->slot[i].dur & bMASKB0);
+		ptr[n+1] = (u8)((dm->slot[i].dur & bMASKB1) >> 8);
+
+		ptr[n+2] = (u8)(dm->slot[i].cxtype & bMASKB0);
+		ptr[n+3] = (u8)((dm->slot[i].cxtype & bMASKB1) >> 8);
+
+		ptr[n+4] = (u8)(dm->slot[i].cxtbl & bMASKB0);
+		ptr[n+5] = (u8)((dm->slot[i].cxtbl & bMASKB1) >> 8);
+		ptr[n+6] = (u8)((dm->slot[i].cxtbl & bMASKB2) >> 16);
+		ptr[n+7] = (u8)((dm->slot[i].cxtbl & bMASKB3) >> 24);
+	}
+
+	_send_fw_cmd(btc, SET_SLOT_TABLE, (u8*)tlv, len);
+	hal_mem_free(h, (void*)tlv, len);
+}
+
+/* set RPT_EN_MREG = 0 to stop 2s monitor timer in WL FW,
+ * before SET_MREG_TABLE, and set RPT_EN_MREG = 1 after
+ * SET_MREG_TABLE
+ */
+void hal_btc_fw_set_monreg(struct btc_t *btc)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btf_tlv *tlv = NULL;
+	u8 *ptr = NULL;
+	u8 n, ulen, i;
+	u16 len = 0;
+
+	if (btc->chip->mon_reg_num > CXMREG_MAX) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_ERR_,
+			  "[BTC], mon reg count %d > %d\n",
+			  btc->chip->mon_reg_num , CXMREG_MAX);
+		return;
+	}
+
+	ulen = sizeof(struct fbtc_mreg);
+	len = (ulen * btc->chip->mon_reg_num) + sizeof(*tlv) - 1;
+	tlv = hal_mem_alloc(h, len);
+	if (!tlv)
+		return;
+
+	tlv->type = RPT_EN_MREG;
+	tlv->ver = FCX_VER_MREG;
+	tlv->len = btc->chip->mon_reg_num; /*total moniter register count */
+	ptr = &tlv->val[0];
+
+	for (i = 0; i < tlv->len; i++) {
+		n = i * ulen;
+		ptr[n+0] = (u8)(btc->chip->mon_reg[i].type & bMASKB0);
+		ptr[n+1] = (u8)((btc->chip->mon_reg[i].type & bMASKB1) >> 8);
+
+		ptr[n+2] = (u8)(btc->chip->mon_reg[i].bytes & bMASKB0);
+		ptr[n+3] = (u8)((btc->chip->mon_reg[i].bytes & bMASKB1) >> 8);
+
+		ptr[n+4] = (u8)(btc->chip->mon_reg[i].offset & bMASKB0);
+		ptr[n+5] = (u8)((btc->chip->mon_reg[i].offset & bMASKB1) >> 8);
+		ptr[n+6] = (u8)((btc->chip->mon_reg[i].offset & bMASKB2) >> 16);
+		ptr[n+7] = (u8)((btc->chip->mon_reg[i].offset & bMASKB3) >> 24);
+	}
+
+	_send_fw_cmd(btc, SET_MREG_TABLE, (u8 *)tlv, len);
+	hal_mem_free(h, (void *)tlv, len);
+}
+
+bool hal_btc_fw_set_policy(struct btc_t *btc, bool force_exec, u16 policy_type,
+		   const char* action)
+{
+	struct btc_dm *dm = &btc->dm;
+	u32 len = _os_strlen((u8 *)action);
+	u8 *btc_ctrl_lps = &btc->hal->btc_ctrl.lps;
+
+	_act_cpy(dm->run_action, (char*)action, len);
+	_update_dm_step(btc, action);
+	_update_dm_step(btc, id_to_str(BTC_STR_POLICY, (u32)policy_type));
+
+	btc->policy_len = 0; /* clear length before append */
+	btc->policy_type = policy_type;
+	_append_tdma(btc, force_exec);
+	_append_slot(btc, force_exec);
+
+	if (btc->policy_len == 0) {
+		return false;
+	} else if (btc->policy_len >= BTC_POLICY_MAXLEN) {
+		dm->error |= BTC_DMERR_H2C_BUF_OVER;
+		return false;
+	}
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		  "[BTC], %s(): action=%s -> policy type/len: 0x%04x/%d\n",
+		  __func__, action, policy_type, btc->policy_len);
+
+	*btc_ctrl_lps = (dm->tdma.rxflctrl == CXFLC_NULLP? 1 : 0);
+
+	if (*btc_ctrl_lps == 1)
+		hal_btc_notify_ps_tdma(btc, *btc_ctrl_lps);
+
+	if (_send_fw_cmd(btc, SET_CX_POLICY, btc->policy, btc->policy_len)) {
+		_tdma_cpy(&dm->tdma_now, &dm->tdma);
+		_slots_cpy(dm->slot_now, dm->slot);
+	}
+
+	if (*btc_ctrl_lps == 0)
+		hal_btc_notify_ps_tdma(btc, *btc_ctrl_lps);
+
+	return true;
+}
+
+void hal_btc_fw_set_gpio(struct btc_t *btc, u8 type, u32 val)
+{
+	u8 data[7] = {0};
+	u16 len = 0;
+
+	if (type >= CXDGPIO_MAX)
+		return;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "[BTC], %s()\n", __func__);
+
+	data[0] = type;  /* type */
+	data[1] = FCX_VER_GPIODBG; /* fver */
+
+	switch(type) {
+	case CXDGPIO_EN_MAP: /* GPIO debug signal en-map 0~31 */
+		data[2] = 4; /* data length */
+		/* Debbug siganl ID definition -> enum btc_wl_gpio_debug */
+		break;
+	case CXDGPIO_MUX_MAP: /* GPIO dbg: Signal to GPIO Mux */
+		data[2] = 2; /* data length */
+		/* [7:0]  -> GPIO 0~7 signal route */
+		/* [15:8] -> debug signal ID*/
+		break;
+	case CXDGPIO_EXT_HPTA: /* GPIO config for Ext HW-PTA */
+		data[2] = 2; /* data length */
+		/* [7:0]  -> GPIO 0~7 en-map*/
+		/* [15:8] -> GPIO 8~15 en-map*/
+		break;
+	case CXDGPIO_EXT_HMBX: /* GPIO config for Ext HW-mailbox */
+		data[2] = 2; /* data length */
+		/* [7:0]  -> GPIO 0~7 en-map*/
+		/* [15:8] -> GPIO 8~15 en-map*/
+		break;
+	case CXDGPIO_EXT_SWOUT: /* 1-GPIO config for Ext SW output (wlan_act) */
+		data[2] = 2; /* data length */
+		/* [7:0]   -> GPIO 7~0 output-en-map
+		 * [15:8]  -> GPIO 15~8 output-en-map
+		 */
+		break;
+	case CXDGPIO_EXT_SWIN: /* GPIO config for Ext SW input control */
+		data[2] = 4; /* data length */
+		/* [7:0]   -> GPIO 7~0 input-en-map
+		 * [15:8]  -> GPIO 15~8 input-en-map
+		 * only 1-GPIO can be assogned to interrupt source
+		 * [23:16]  -> GPIO 7~0 as interrupt source
+		 * [31:24]  -> GPIO 15~8 as interrupt source
+		 */
+		break;
+	default:
+		return;
+	}
+
+	data[3] = (u8)(val & bMASKB0);
+	data[4] = (u8)((val & bMASKB1) >> 8);
+	data[5] = (u8)((val & bMASKB2) >> 16);
+	data[6] = (u8)((val & bMASKB3) >> 24);
+
+	len = data[2] + 3;
+
+	_send_fw_cmd(btc, SET_GPIO_DBG, data, len);
+}
+
+void hal_btc_fw_set_drv_info(struct btc_t *btc, u8 type)
+{
+	struct rtw_hal_com_t *h = btc->hal;
+	struct btc_wl_info *wl = &btc->cx.wl;
+	struct btc_dm *dm = &btc->dm;
+	struct btc_rf_trx_para rf_para = dm->rf_trx_para;
+	u8 buf[256] = {0};
+	u8 sz = 0, n = 0;
+
+	if (type >= CXDRVINFO_MAX)
+		return;
+
+	switch (type) {
+	case CXDRVINFO_INIT:
+		n = sizeof(dm->init_info);
+		sz = n + 3;
+
+		if (sz > sizeof(buf))
+			return;
+
+		buf[0] = CXDRVINFO_INIT;
+		buf[1] = FCX_VER_INIT;
+		buf[2] = n;
+		hal_mem_cpy(h, (void *)&buf[3], &dm->init_info, n);
+		break;
+	case CXDRVINFO_ROLE:
+		n = sizeof(wl->role_info);
+		sz = n + 3;
+
+		if (sz > sizeof(buf))
+			return;
+
+		buf[0] = CXDRVINFO_ROLE;
+		buf[1] = FCX_VER_ROLE;
+		buf[2] = n;
+		hal_mem_cpy(h, (void *)&buf[3], &wl->role_info, n);
+		break;
+	case CXDRVINFO_CTRL:
+		n = sizeof(btc->ctrl);
+		sz = n + 3;
+
+		if (sz > sizeof(buf))
+			return;
+
+		buf[0] = CXDRVINFO_CTRL;
+		buf[1] = FCX_VER_CTRL;
+		buf[2] = n;
+		hal_mem_cpy(h, (void *)&buf[3], &btc->ctrl, n);
+		break;
+	case CXDRVINFO_TRX:
+		n = sizeof(dm->trx_info);
+		sz = n + 3;
+
+		if (sz > sizeof(buf))
+			return;
+
+		buf[0] = CXDRVINFO_TRX;
+		buf[1] = FCX_VER_TRX;
+		buf[2] = n;
+
+		dm->trx_info.tx_power = rf_para.wl_tx_power & 0xff;
+		dm->trx_info.rx_gain = rf_para.wl_rx_gain & 0xff;
+		dm->trx_info.bt_tx_power = rf_para.bt_tx_power & 0xff;
+		dm->trx_info.bt_rx_gain = rf_para.bt_rx_gain & 0xff;
+		dm->trx_info.cn = wl->cn_report;
+		dm->trx_info.nhm = wl->nhm.pwr;
+
+		hal_mem_cpy(h, (void *)&buf[3], &dm->trx_info, n);
+		break;
+	case CXDRVINFO_TXPWR:
+		n = 1;
+		sz = n + 3;
+
+		if (sz > sizeof(buf))
+			return;
+
+		buf[0] = CXDRVINFO_TXPWR;
+		buf[1] = FCX_VER_TRX;
+		buf[2] = n;
+		buf[3] = (u8)(dm->rf_trx_para.wl_tx_power & 0xff);
+		break;
+	case CXDRVINFO_FDDT:
+		n = sizeof(dm->fddt_info.train);
+		sz = n + 3;
+
+		if (sz > sizeof(buf))
+			return;
+
+		buf[0] = CXDRVINFO_FDDT;
+		buf[1] = FCX_VER_FDDT;
+		buf[2] = n;
+		hal_mem_cpy(h, (void *)&buf[3], &dm->fddt_info.train, n);
+		break;
+	default:
+		return;
+	}
+
+#if BTC_PLATFORM_BIG_ENDIAN
+	_endian_translate(btc, BTC_FW_H2C, buf);
+#endif
+	_send_fw_cmd(btc, SET_DRV_INFO, buf, sz);
+}
+
+void hal_btc_fw_parse_rpt(struct btc_t *btc, u8 *data, u32 len)
+{
+	u32 index = 0, rpt_len = 0;
+
+	if (!len || !data)
+		return;
+
+	while (1) {
+		if (index+2 >= RTW_PHL_BTC_FWINFO_BUF)
+			break;
+		/* At least 3 bytes: type(1) & len(2) */
+		rpt_len = (data[index+2] << 8) + data[index+1];
+		if ((index + rpt_len + BTC_RPT_HDR_SIZE) > len)
+			break;
+
+		rpt_len = _chk_btc_report(btc, data, index);
+		if (!rpt_len)
+			break;
+		index += rpt_len;
+	}
+}
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fw.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fw.h
new file mode 100644
index 000000000000..31fcec217d9d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fw.h
@@ -0,0 +1,298 @@
+#ifndef __INC_BTC_FW_H__
+#define __INC_BTC_FW_H__
+
+#include "halbtc_fwdef.h"
+
+#pragma pack(push)
+#pragma pack(1)
+
+#ifdef BTC_LE_INIT_TO_WL_SLOT
+#define LE_INIT_RATIO 2
+#endif
+
+struct btc_t;
+
+enum btf_reg_type {
+	REG_MAC = 0x0,
+	REG_BB = 0x1,
+	REG_RF = 0x2,
+	REG_BT_RF = 0x3,
+	REG_BT_MODEM = 0x4,
+	REG_BT_BLUEWIZE = 0x5,
+	REG_BT_VENDOR = 0x6,
+	REG_BT_LE = 0x7,
+	REG_MAX_TYPE
+};
+
+/*
+ * h2c_class : 0x10~0x17: BTC
+ * Please see : struct rtw_g6_h2c_hdr
+ */
+enum bt_h2c_class {
+	BTFC_SET = 0x10, /* 0x10~0x17 for BTC */
+	BTFC_GET = 0x11,
+	BTFC_FW_EVENT = 0x12,
+	BTFC_MAX
+};
+
+/* BTFC_SET class 0x10 ->function */
+enum btf_set {
+	SET_REPORT_EN = 0x0,
+	SET_SLOT_TABLE, /* Slot table  */
+	SET_MREG_TABLE, /* moniter register  */
+	SET_CX_POLICY,
+	SET_GPIO_DBG,
+	SET_DRV_INFO,
+	SET_DRV_EVENT,
+	SET_BT_WREG_ADDR,
+	SET_BT_WREG_VAL,
+	SET_BT_RREG_ADDR,
+	SET_BT_WL_CH_INFO,
+	SET_BT_INFO_REPORT,
+	SET_BT_IGNORE_WLAN_ACT,
+	SET_BT_TX_PWR,
+	SET_BT_LNA_CONSTRAIN,
+	SET_BT_QUERY_DEV_LIST,
+	SET_BT_QUERY_DEV_INFO,
+	SET_BT_QUERY_LEAUDIO_INFO,
+	SET_BT_PSD_REPORT,
+	SET_H2C_TEST,
+	SET_MAX1
+};
+
+/* SET_REPORT_EN function 0x0 -> TLV sub-function */
+enum btf_set_report_en {
+	RPT_EN_TDMA = BIT0,
+	RPT_EN_CYCLE = BIT1,
+	RPT_EN_MREG = BIT2,
+	RPT_EN_BT_VER_INFO = BIT3, /* FW, Coex, Feature*/
+	RPT_EN_BT_SCAN_INFO = BIT4,  /* BT scan parameter */
+	RPT_EN_BT_DEVICE_INFO = BIT5, /* vendor id, device name, flush-val*/
+	RPT_EN_BT_AFH_MAP = BIT6, /* AFH map H/M/L */
+	RPT_EN_BT_AFH_MAP_LE = BIT7, /* AFH map L/M */
+	RPT_EN_FW_STEP_INFO = BIT8,
+	RPT_EN_FW_TEST_INFO = BIT9,
+	RPT_EN_ALL = 0x1ff
+};
+
+/* SET_CX_POLICY function 0x3 -> TLV sub-function */
+enum btf_set_cx_policy {
+	CXPOLICY_TDMA = 0x0,
+	CXPOLICY_SLOT = 0x1,
+	CXPOLICY_MAX
+};
+
+/* SET_GPIO_DBG function 0x4 -> TLV sub-function */
+enum btc_gpio_dbg_type {
+	CXDGPIO_EN_MAP = 0x0,
+	CXDGPIO_MUX_MAP = 0x1,
+	CXDGPIO_EXT_HPTA = 0x2,
+	CXDGPIO_EXT_HMBX = 0x3,
+	CXDGPIO_EXT_SWOUT = 0x4,
+	CXDGPIO_EXT_SWIN = 0x5,
+	CXDGPIO_MAX
+};
+
+/* SET_DRV_INFO function 0x5 -> TLV sub-function */
+enum btc_drv_info_type {
+	CXDRVINFO_INIT = 0, /* wl_only, dbcc_en...*/
+	CXDRVINFO_ROLE, /* Role */
+	CXDRVINFO_CTRL, /* ctrl info */
+	CXDRVINFO_TRX,  /* WL traffic to WL fw */
+	CXDRVINFO_TXPWR, /* Set WL tx pwr in WL fw */
+	CXDRVINFO_FDDT, /* FDD train info  */
+	CXDRVINFO_MAX
+};
+
+/* SET_DRV_EVENT function 0x6 -> TLV sub-function */
+enum btc_drv_event_type {
+	CXDRVEVNT_1 = 0x0,
+	CXDRVEVNT_2 = 0x1,
+	CXDRVEVNT_MAX
+};
+
+/* BTFC_GET class 0x11 ->function */
+enum btf_get {
+	GET_BT_REG,
+	GET_MAX
+};
+
+/* BTFC_FW_EVENT class 0x12 ->function */
+enum btf_fw_event {
+	BTF_EVNT_RPT = 0,
+	BTF_EVNT_BT_INFO = 1,
+	BTF_EVNT_BT_SCBD = 2,
+	BTF_EVNT_BT_REG = 3,
+	BTF_EVNT_CX_RUNINFO = 4,
+	BTF_EVNT_BT_PSD = 5,
+	BTF_EVNT_BT_DEV_INFO = 6,
+	BTF_EVNT_BT_LEAUDIO_INFO = 7,
+	BTF_EVNT_BUF_OVERFLOW,
+	BTF_EVNT_C2H_LOOPBACK,
+	BTF_EVNT_MAX
+};
+
+/* BTF_EVNT_RPT function 0x0 -> TLV type */
+enum btf_fw_event_report {
+	BTC_RPT_TYPE_CTRL = 0x0,
+	BTC_RPT_TYPE_TDMA,
+	BTC_RPT_TYPE_SLOT,
+	BTC_RPT_TYPE_CYSTA,
+	BTC_RPT_TYPE_STEP,
+	BTC_RPT_TYPE_NULLSTA,
+	BTC_RPT_TYPE_FDDT,
+	BTC_RPT_TYPE_MREG,
+	BTC_RPT_TYPE_GPIO_DBG,
+	BTC_RPT_TYPE_BT_VER,
+	BTC_RPT_TYPE_BT_SCAN,
+	BTC_RPT_TYPE_BT_AFH,
+	BTC_RPT_TYPE_BT_DEVICE,
+	BTC_RPT_TYPE_TEST,
+	BTC_RPT_TYPE_MAX
+};
+
+enum btc_fbtc_invlaid_input {
+	BTFRE_INVALID_INPUT = 0x0, /* invalid input parameters */
+	BTFRE_UNDEF_TYPE,
+	BTFRE_EXCEPTION,
+	BTFRE_MAX
+};
+
+struct btf_tlv {
+	u8 type;
+	u8 ver;
+	u8 len;
+	u8 val[1];
+};
+
+struct btf_event {
+	u8 event;
+	u8 reserved;
+	u16 length;
+	u8 buf[1];
+};
+
+struct btc_rpt_cmn_info {
+	u32 rx_cnt;
+	u32 rx_len;
+	u32 req_len; /* expected rsp len */
+	u8 req_fver; /* expected rsp fver */
+	u8 rsp_fver; /* fver from fw */
+	u8 valid;
+};
+
+#pragma pack(pop)
+
+struct btc_report_ctrl_state {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_rpt_ctrl finfo; /* info from fw */
+};
+
+struct btc_fbtc_tdma {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_1tdma finfo; /* info from fw */
+};
+
+struct btc_fbtc_slots {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_slots finfo; /* info from fw */
+};
+
+struct btc_fbtc_cysta {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_cysta finfo; /* info from fw */
+};
+
+struct btc_fbtc_step {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_steps finfo; /* info from fw */
+};
+
+struct btc_fbtc_nullsta {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_cynullsta finfo; /* info from fw */
+};
+
+struct btc_fbtc_fddt {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_fddt_sta finfo; /* info from fw */
+};
+
+struct btc_fbtc_mreg {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_mreg_val finfo; /* info from fw */
+};
+
+struct btc_fbtc_gpio_dbg {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_gpio_dbg finfo; /* info from fw */
+};
+
+struct btc_fbtc_btver {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_btver finfo; /* info from fw */
+};
+
+struct btc_fbtc_btscan {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_btscan finfo; /* info from fw */
+};
+
+struct btc_fbtc_btafh {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_btafh finfo; /* info from fw */
+};
+
+struct btc_fbtc_btdev {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_btdevinfo finfo; /* info from fw */
+};
+
+struct btc_fbtc_testinfo {
+	struct btc_rpt_cmn_info cinfo; /* common info, by driver */
+	struct fbtc_testinfo finfo; /* info from fw */
+};
+
+struct btf_fwinfo {
+	u32 cnt_c2h; /* total c2h cnt */
+	u32 cnt_h2c; /* total h2c cnt */
+	u32 cnt_h2c_fail; /* total h2c fail cnt */
+	u32 event[BTF_EVNT_MAX]; /* event statistics */
+
+	u32 err[BTFRE_MAX]; /* counters for statistics */
+	u32 len_mismch; /* req/rsp length mis match */
+	u32 fver_mismch; /* req/rsp fver mis match */
+	u32 rpt_en_map;
+
+	struct btc_report_ctrl_state rpt_ctrl;
+	struct btc_fbtc_tdma rpt_fbtc_tdma;
+	struct btc_fbtc_slots rpt_fbtc_slots;
+	struct btc_fbtc_cysta rpt_fbtc_cysta;
+	struct btc_fbtc_step rpt_fbtc_step;
+	struct btc_fbtc_nullsta rpt_fbtc_nullsta;
+	struct btc_fbtc_fddt rpt_fbtc_fddt;
+	struct btc_fbtc_mreg rpt_fbtc_mregval;
+	struct btc_fbtc_gpio_dbg rpt_fbtc_gpio_dbg;
+	struct btc_fbtc_btver rpt_fbtc_btver;
+	struct btc_fbtc_btscan rpt_fbtc_btscan;
+	struct btc_fbtc_btafh rpt_fbtc_btafh;
+	struct btc_fbtc_btdev rpt_fbtc_btdev;
+	struct btc_fbtc_testinfo rpt_fbtc_testinfo;
+};
+
+/*
+ * extern functions
+ */
+
+void hal_btc_fw_chk_struct(struct btc_t *btc);
+void hal_btc_fw_parse_rpt(struct btc_t *btc, u8 *data, u32 len);
+void hal_btc_fw_set_rpt(struct btc_t *btc, u32 rpt_map, u32 rpt_state);
+void hal_btc_fw_set_slots(struct btc_t *btc);
+void hal_btc_fw_set_monreg(struct btc_t *btc);
+bool hal_btc_fw_set_policy(struct btc_t *btc, bool force_exec, u16 policy_type,
+			   const char* action);
+void hal_btc_fw_set_gpio(struct btc_t *btc, u8 type, u32 val);
+void hal_btc_fw_set_drv_info(struct btc_t *btc, u8 type);
+void hal_btc_notify_ps_tdma(struct btc_t *btc, bool tdma_start);
+
+#endif	/* __INC_BTC_FW_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fwdef.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fwdef.h
new file mode 100644
index 000000000000..a838f3b39bd3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/btc/halbtc_fwdef.h
@@ -0,0 +1,732 @@
+#ifndef __INC_BTC_FW_DEF_H__
+#define __INC_BTC_FW_DEF_H__
+
+#pragma pack(push)
+#pragma pack(1)
+/*
+ * shared FW Definition
+ */
+
+//#define BTC_FW_STEP_DBG
+
+#define CXMREG_MAX 20
+#define FCXDEF_STEP 50			/* MUST fw_step size*/
+#define BTC_CYCLE_SLOT_MAX 48		/* must be even number, non-zero */
+#define BTC_WL_MAX_ROLE_NUMBER 6	/* Must = MAX_WIFI_ROLE_NUMBER */
+#define BTC_DBG_MAX1 32
+
+#define FCX_VER_FDDT 7		/* H2c refer to "struct btc_fddt_train_info" */
+#define FCX_VER_TRX 7		/* H2c refer to "struct btc_trx_info" */
+#define FCX_VER_INIT 7		/* H2C refer to "struct btc_init_info" */
+#define FCX_VER_CTRL 7		/* H2C refer to "struct btc_ctrl" */
+#define FCX_VER_ROLE 7		/* H2C refer to "struct btc_wl_role_info" */
+
+#define FCX_VER_BTDEVINFO 7	/* C2H refer to "struct fbtc_btdevinfo" */
+#define FCX_VER_BTAFH 7		/* C2H refer to "struct fbtc_btafh" */
+#define FCX_VER_BTSCAN 7 	/* C2H refer to "struct fbtc_btscan" */
+#define FCX_VER_BTVER 7		/* C2H refer to "struct fbtc_btver */
+#define FCX_VER_NULLSTA 7 	/* C2H refer to "struct fbtc_cynullsta" */
+#define FCX_VER_CYSTA 7 	/* C2H refer to "struct fbtc_cysta" */
+#define FCX_VER_STEP 7		/* C2H refer to "struct fbtc_steps" */
+
+#define FCX_VER_TDMA 7		/* H2C/C2H refer to "struct fbtc_tdma" */
+#define FCX_VER_BTCRPT 8	/* H2C/C2H refer to "struct fbtc_rpt_ctrl" */
+#define FCX_VER_SLOT 7		/* H2C/C2H refer to "struct fbtc_slots" */
+#define FCX_VER_MREG 7		/* H2C/C2H refer to "struct fbtc_mreg_val */
+#define FCX_VER_GPIODBG 8 	/* H2C/C2H refer to "struct fbtc_gpio_dbg" */
+#define FCX_VER_TEST 1		/* H2C/C2H refer to "struct fbtc_testinfo" */
+
+enum btc_bt_sta_counter {
+	BTC_BCNT_RFK_REQ = 0,
+	BTC_BCNT_HI_TX = 1,
+	BTC_BCNT_HI_RX = 2,
+	BTC_BCNT_LO_TX = 3,
+	BTC_BCNT_LO_RX = 4,
+	BTC_BCNT_POLUT = 5,
+	BTC_BCNT_STA_MAX
+};
+
+/* ============== TDMA related ============== */
+enum fbtc_tdma_template {
+	CXTD_OFF = 0x0,
+	CXTD_OFF_B2,
+	CXTD_OFF_EXT,
+	CXTD_FIX,
+	CXTD_PFIX,
+	CXTD_AUTO,
+	CXTD_PAUTO,
+	CXTD_AUTO2,
+	CXTD_PAUTO2,
+	CXTD_MAX
+};
+
+enum fbtc_tdma_type {
+	CXTDMA_OFF = 0x0, /* tdma off */
+	CXTDMA_FIX = 0x1, /* fixed slot */
+	CXTDMA_AUTO = 0x2, /* auto slot */
+	CXTDMA_AUTO2 = 0x3, /* extended auto slot */
+	CXTDMA_MAX
+};
+
+enum fbtc_tdma_rx_flow_ctrl {
+	CXFLC_OFF = 0x0,  /* rx flow off */
+	CXFLC_NULLP = 0x1, /* Null/Null-P */
+	CXFLC_QOSNULL = 0x2, /* QOS Null/Null-P */
+	CXFLC_CTS = 0x3,  /* CTS to Self control */
+	CXFLC_MAX
+};
+
+enum fbtc_tdma_wlan_tx_pause {
+	CXTPS_OFF = 0x0,  /* no wl tx pause*/
+	CXTPS_ON = 0x1,
+	CXTPS_MAX
+};
+
+/* define if ext-ctrl-slot allowed while TDMA off */
+enum fbtc_ext_ctrl_type {
+	CXECTL_OFF = 0x0, /* tdma off */
+	CXECTL_B2 = 0x1, /* allow B2 (beacon-early) */
+	CXECTL_EXT = 0x2,
+	CXECTL_MAX
+};
+
+union fbtc_rxflct {
+	u8 val;
+	u8 type: 3;
+	u8 tgln_n: 5;
+};
+
+enum fbtc_option_ctrl_def {
+	CXOPCTL_INST_EXEC = BIT(0), /* tdma instant execute */
+	CXOPCTL_FDDT_ENABLE = BIT(1),/* info wl fw to enable fdd-train*/
+	CXOPCTL_FDDT_RENEW = BIT(2), /* info wl fw to update fddt_info.train */
+	CXOPCTL_FDDT_DEBUG = BIT(3), /* info wl fw to enter debug mode */
+	CXOPCTL_LE_INIT_EXEC = BIT(4) /*le_interval < 100ms*/
+};
+
+ /* ============== SLOT related ============== */
+ enum btc_slot_type { /* slot */
+	CXST_OFF = 0x0,
+	CXST_B2W = 0x1,
+	CXST_W1 = 0x2,
+	CXST_W2 = 0x3,
+	CXST_W2B = 0x4,
+	CXST_B1 = 0x5,
+	CXST_B2 = 0x6,
+	CXST_B3 = 0x7,
+	CXST_B4 = 0x8,
+	CXST_LK = 0x9,
+	CXST_BLK = 0xa,
+	CXST_E2G = 0xb, /* for ext-control-slot 2G*/
+	CXST_E5G = 0xc, /* for ext-control-slot 5G*/
+	CXST_EBT = 0xd, /* for ext-control-slot BT*/
+	CXST_ENULL = 0xe, /* for ext-control-slot Null*/
+	CXST_WLK = 0xf, /* for WL link slot */
+	CXST_W1FDD = 0x10,
+	CXST_B1FDD = 0x11,
+	CXST_MAX = 0x12 /* The max slot must be even*/
+};
+
+enum btc_evnt_type {
+	CXEVNT_TDMA_ENTRY = 0x0,
+	CXEVNT_WL_TMR,
+	CXEVNT_B1_TMR,
+	CXEVNT_B2_TMR,
+	CXEVNT_B3_TMR,
+	CXEVNT_B4_TMR,
+	CXEVNT_W2B_TMR,
+	CXEVNT_B2W_TMR,
+	CXEVNT_BCN_EARLY,
+	CXEVNT_A2DP_EMPTY,
+	CXEVNT_LK_END,
+	CXEVNT_RX_ISR,
+	CXEVNT_RX_FC0,
+	CXEVNT_RX_FC1,
+	CXEVNT_BT_RELINK,
+	CXEVNT_BT_RETRY,
+	CXEVNT_E2G,
+	CXEVNT_E5G,
+	CXEVNT_EBT,
+	CXEVNT_ENULL,
+	CXEVNT_DRV_WLK,
+	CXEVNT_BCN_OK,
+	CXEVNT_BT_CHANGE,
+	CXEVNT_EBT_EXTEND,
+	CXEVNT_E2G_NULL1,
+	CXEVNT_B1FDD_TMR,
+	CXEVNT_LE_INIT_END,
+	CXEVNT_MAX
+};
+
+enum btc_bcn_type {
+	CXBCN_ALL = 0x0,
+	CXBCN_ALL_OK,
+	CXBCN_BT_SLOT,
+	CXBCN_BT_OK,
+	CXBCN_MAX
+};
+
+enum btc_slot_iso_type {
+	SLOT_MIX = 0x0, /* accept BT Lower-Pri Tx/Rx request 0x778 = 1 */
+	SLOT_ISO = 0x1, /* no accept BT Lower-Pri Tx/Rx request 0x778 = d*/
+	CXSTYPE_MAX
+};
+
+enum btc_wbslot_type { /* TIME */
+	CXT_BT = 0x0,
+	CXT_WL = 0x1,
+	CXT_MAX
+};
+
+enum btc_flctrl_type { /* TIME-A2DP */
+	CXT_FLCTRL_OFF = 0x0,
+	CXT_FLCTRL_ON = 0x1,
+	CXT_FLCTRL_MAX
+};
+
+enum btc_step_type { /* STEP TYPE */
+	CXSTEP_NONE = 0x0,
+	CXSTEP_EVNT = 0x1,
+	CXSTEP_SLOT = 0x2,
+	CXSTEP_MAX
+};
+
+enum btc_null_state {
+	CXNULL_0 = 0,
+	CXNULL_1 = 1,
+	CXNULL_STATE_MAX = 2
+};
+
+enum btc_null_staus {
+	CXNULL_FAIL = 0,
+	CXNULL_OK = 1,
+	CXNULL_LATE = 2,
+	CXNULL_RETRY = 3,
+	CXNULL_TX = 4,
+	CXNULL_MAX = 5
+};
+
+enum btc_ble_scan_type {
+	CXSCAN_BG = 0,
+	CXSCAN_INIT,
+	CXSCAN_LE,
+	CXSCAN_MAX1
+};
+
+enum btc_bt_afh_type {
+	BTC_RPT_BT_AFH_LEGACY = 0x10,
+	BTC_RPT_BT_AFH_LE = 0x20
+};
+
+/* ========== H2C/C2H refer to "struct fbtc_tdma" ========== */
+struct fbtc_tdma {
+	u8 type; /* refer to fbtc_tdma_type*/
+	u8 rxflctrl; /* refer to fbtc_tdma_rx_flow_ctrl */
+	u8 txflctrl; /* If WL stop Tx while enter BT-slot */
+	u8 rsvd;
+
+	u8 leak_n; /* every leak_n cycle do leak detection */
+	u8 ext_ctrl; /* refer to fbtc_ext_ctrl_type*/
+	/* send rxflctrl to which role
+	 * enum role_type, default: 0 for single-role
+	 * if multi-role: [7:4] second-role, [3:0] fisrt-role
+	 */
+	u8 rxflctrl_role;
+	u8 option_ctrl; /*bit0: execute immediately, no tdma cycle waitting */
+};
+
+struct fbtc_1tdma {
+	u8 fver;
+	u8 rsvd1;
+	u8 rsvd2;
+	u8 rsvd3;
+
+	struct fbtc_tdma tdma;
+};
+/* ==========================================================================*/
+
+/* ========== H2C refer to "struct btc_wl_role_info" ========== */
+struct btc_wl_active_role { /* struct size must be n*4 bytes */
+	u8 connected;
+	u8 pid;
+	u8 phy;
+	u8 noa;
+
+	u8 band; /* enum band_type RF band: 2.4G/5G/6G */
+	u8 client_ps;
+	u8 bw; /* enum channel_width */
+	u8 role; /*enum role_type */
+
+	u8 ch;
+	u8 noa_dur; /* ms */
+	u8 client_cnt;
+	u8 rsvd2;
+};
+
+struct btc_wl_role_info { /* struct size must be n*4 bytes */
+	u8 connect_cnt;
+	u8 link_mode;
+	u8 link_mode_chg;
+	u8 p2p_2g;
+
+	struct btc_wl_active_role active_role[BTC_WL_MAX_ROLE_NUMBER];
+
+	u32 role_map;
+	u32 mrole_type; /* btc_wl_mrole_type */
+	u32 mrole_noa_duration; /* ms */
+	u32 dbcc_en;
+	u32 dbcc_chg;
+	u32 dbcc_2g_phy; /* which phy operate in 2G, HW_PHY_0 or HW_PHY_1 */
+};
+/* ==========================================================================*/
+
+/* ========== H2C refer to "struct btc_ctrl" ========== */
+struct btc_ctrl {
+	u8 manual;
+	u8 igno_bt;
+	u8 always_freerun;
+	u8 rsvd;
+};
+/* ==========================================================================*/
+
+/* ========== H2C/C2H refer to "struct fbtc_rpt_ctrl" ========== */
+struct btc_rpt_ctrl_info {
+	u16 cnt; /* fw report counter */
+	u16 cnt_c2h; /* fw send c2h counter  */
+
+	u16 cnt_h2c; /* fw recv h2c counter */
+	u16 len_c2h; /* The total length of the last C2H  */
+
+	u16 cnt_aoac_rf_on;  /* rf-on counter for aoac switch notify */
+	u16 cnt_aoac_rf_off; /* rf-off counter for aoac switch notify */
+
+	u32 cx_ver; /* match which driver's coex version */
+	u32 fw_ver;
+	u32 en; /* report map */
+};
+
+struct btc_rpt_ctrl_a2dp_empty {
+	u32 cnt_empty; /* a2dp empty count */
+	u32 cnt_flowctrl; /* a2dp empty flow control counter */
+	u32 cnt_tx;
+	u32 cnt_ack;
+	u32 cnt_nack;
+};
+
+struct btc_rpt_ctrl_bt_mailbox {
+	u32 cnt_send_ok; /* fw send mailbox ok counter */
+	u32 cnt_send_fail; /* fw send mailbox fail counter */
+	u32 cnt_recv; /* fw recv mailbox counter */
+	struct btc_rpt_ctrl_a2dp_empty a2dp;
+};
+
+struct fbtc_rpt_ctrl {
+	u8 fver;
+	u8 ext_req_exist;
+	u8 pta_owner;
+	u8 rsvd2;
+
+	u8 gnt_val[HW_PHY_MAX][4];
+	u16 bt_cnt[BTC_BCNT_STA_MAX];
+
+	struct btc_rpt_ctrl_info rpt_info;
+	struct btc_rpt_ctrl_bt_mailbox bt_mbx_info;
+};
+/* ==========================================================================*/
+
+/* ========== H2C refer to "struct btc_init_info" ========== */
+struct btc_ant_info {
+	u8 type;  /* shared, dedicated(non-shared) */
+	u8 num;   /* antenna count  */
+	u8 isolation;
+	u8 single_pos;/* wifi 1ss-1ant at 0:S0 or 1:S1 */
+
+	u8 diversity; /* only for wifi use 1-antenna */
+	u8 btg_pos; /* btg-circuit at 0:S0/1:S1/others:all */
+	u8 stream_cnt;  /* spatial_stream count: Tx[7:4], Rx[3:0] */
+	u8 path_pos;    /* path pos: Tx[7:4], Rx[3:0] */
+};
+
+struct btc_module {
+	u8 rfe_type;
+	u8 kt_ver;
+	u8 bt_solo;
+	u8 bt_pos; /* wl-end view: get from efuse, must compare bt.btg_type*/
+
+	u8 switch_type; /* WL/BT switch type: 0: internal, 1: external */
+	u8 wa_type; /* WA type: 0:none, 1: 51B 5G_Hi-Ch_Rx */
+	u8 kt_ver_adie;
+	u8 rsvd;
+
+	struct btc_ant_info ant;
+};
+
+struct btc_init_info {
+	u8 wl_guard_ch;
+	u8 wl_only;
+	u8 wl_init_ok;
+	u8 dbcc_en;
+
+	u8 cx_other;
+	u8 bt_only;
+	u8 rsvd1;
+	u8 rsvd2;
+
+	struct btc_module module;
+};
+/* ==========================================================================*/
+
+/* ========== H2C/C2H refer to "struct fbtc_gpio_dbg" ========== */
+struct fbtc_gpio_dbg {
+	u8 fver;
+	u8 rsvd0;
+	u8 rsvd1;
+	u8 rsvd2;
+
+	u8 gpio_map[BTC_DBG_MAX1]; /* debug signals to GPIO-Position mapping */
+
+	u32 en_map; /* which debug signal (see btc_wl_gpio_debug) is enable */
+	u32 pre_state; /* the debug signal is 1 or 0  */
+};
+/* ==========================================================================*/
+
+/* ========== H2C/C2H refer to "struct fbtc_mreg_val" ========== */
+struct fbtc_mreg_val {
+	u8 fver;
+	u8 reg_num;
+	u8 rsvd0;
+	u8 rsvd1;
+
+	u32 mreg_val[CXMREG_MAX];
+};
+
+struct fbtc_mreg {
+	u16 type;
+	u16 bytes;
+	u32 offset;
+};
+/* ==========================================================================*/
+
+/* ========== H2C/C2H refer to "struct fbtc_slots" ========== */
+struct fbtc_slot {
+	u16 dur; /* slot duration */
+	u16 cxtype;
+	u32 cxtbl;
+};
+
+struct fbtc_slot_u16 {
+	u16 dur; /* slot duration */
+	u16 cxtype;
+	u16 cxtbl_l16; /* coex table [15:0] */
+	u16 cxtbl_h16; /* coex table [31:16] */
+};
+
+struct fbtc_1slot {
+	u8 fver;
+	u8 sid; /* slot id */
+	u16 rsvd;
+	struct fbtc_slot slot;
+};
+
+struct fbtc_slots {
+	u8 fver;
+	u8 slot_cnt;
+	u8 rsvd0;
+	u8 rsvd1;
+
+	struct fbtc_slot_u16 slot[CXST_MAX];
+
+	u32 update_map;
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_steps" ========== */
+struct fbtc_step {
+	u8 type;
+	u8 val;
+	u8 difft_l8;
+	u8 difft_h8;
+};
+
+struct fbtc_steps {
+	u8 fver;
+	u8 en;
+	u8 rsvd0;
+	u8 rsvd1;
+
+#ifdef BTC_FW_STEP_DBG
+	struct fbtc_step step[FCXDEF_STEP];
+#endif
+
+	u32 cnt;
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_cysta" ========== */
+struct fbtc_sta_time_info {
+	u16 tavg[CXT_MAX]; /* avg wl/bt cycle time */
+	u16 tmax[CXT_MAX]; /* max wl/bt cycle time */
+};
+
+struct fbtc_cycle_a2dptrx_info {
+	u8 empty_cnt;
+	u8 retry_cnt;
+	u8 tx_rate;
+	u8 tx_cnt;
+
+	u8 ack_cnt;
+	u8 nack_cnt;
+	u8 no_empty_cnt;
+	u8 rsvd;
+};
+
+struct fbtc_sta_a2dpept_info {
+	u16 cnt; /* a2dp empty cnt */
+	u16 cnt_timeout; /* a2dp empty timeout cnt*/
+	u16 tavg; /* avg a2dp empty time */
+	u16 tmax; /* max a2dp empty time */
+};
+
+struct fbtc_sta_leak_info {
+	u16 tavg; /* avg leak-slot time */
+	u16 tamx; /* max leak-slot time */
+	u32 cnt_rximr; /* the rximr occur at leak slot  */
+};
+
+struct fbtc_cysta { /* statistics for cycles */
+	u8 fver;
+	u8 rsvd;
+	u8 collision_cnt; /* counter for event/timer occur at the same time */
+	u8 except_cnt;
+
+	u8 wl_rx_err_ratio[BTC_CYCLE_SLOT_MAX];
+
+	struct fbtc_cycle_a2dptrx_info a2dp_trx[BTC_CYCLE_SLOT_MAX];
+
+	u16 skip_cnt;
+	u16 cycles; /* total cycle number */
+
+	u16 slot_step_time[BTC_CYCLE_SLOT_MAX]; /* record the wl/bt slot time */
+	u16 slot_cnt[CXST_MAX]; /* slot count */
+	u16 bcn_cnt[CXBCN_MAX];
+
+	struct fbtc_sta_time_info cycle_time;
+	struct fbtc_sta_a2dpept_info a2dp_ept;
+	struct fbtc_sta_leak_info leak_slot;
+
+	u32 except_map;
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_cynullsta" ========== */
+struct fbtc_cynullsta { /* cycle null statistics */
+	u8 fver;
+	u8 rsvd0;
+	u8 rsvd1;
+	u8 rsvd2;
+
+	u32 tmax[CXNULL_STATE_MAX]; /* max_t for 0:null0/1:null1 */
+	u32 tavg[CXNULL_STATE_MAX]; /* avg_t for 0:null0/1:null1 */
+	/* result for null , 0:fail, 1:ok, 2:late, 3:retry */
+	u32 result[CXNULL_STATE_MAX][CXNULL_MAX];
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_btver" ========== */
+struct fbtc_btver {
+	u8 fver;
+	u8 rsvd0;
+	u8 rsvd1;
+	u8 rsvd2;
+
+	u32 coex_ver; /*bit[15:8]->shared, bit[7:0]->non-shared */
+	u32 fw_ver;
+	u32 feature;
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_btscan" ========== */
+struct btc_bt_scan_info {
+	u16 win;
+	u16 intvl;
+};
+
+struct fbtc_btscan {
+	u8 fver;
+	u8 type;
+	u8 rsvd0;
+	u8 rsvd1;
+
+	struct btc_bt_scan_info para[CXSCAN_MAX1];
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_btafh" ========== */
+struct fbtc_btafh {
+	u8 fver;
+	u8 map_type;
+	u8 rsvd0;
+	u8 rsvd1;
+
+	u8 afh_l[4]; /*bit0:2402, bit1: 2403.... bit31:2433 */
+	u8 afh_m[4]; /*bit0:2434, bit1: 2435.... bit31:2465 */
+	u8 afh_h[4]; /*bit0:2466, bit1:2467......bit14:2480 */
+
+	u8 afh_le_a[4];
+	u8 afh_le_b[4];
+};
+/* ==========================================================================*/
+
+/* ========== H2c refer to "struct btc_trx_info" ========== */
+struct btc_trx_info {
+	u8 tx_lvl;
+	u8 rx_lvl;
+	u8 wl_rssi;
+	u8 bt_rssi;
+
+	s8 tx_power; /* absolute Tx power (dBm), 0xff-> no BTC control */
+	s8 rx_gain;  /* rx gain table index (TBD.) */
+	s8 bt_tx_power; /* decrease Tx power (dB) */
+	s8 bt_rx_gain;  /* LNA constrain level */
+
+	u8 cn; /* condition_num */
+	s8 nhm;
+	u8 bt_profile;
+	u8 rsvd2;
+
+	u16 tx_rate;
+	u16 rx_rate;
+
+	u32 tx_tp;
+	u32 rx_tp;
+	u32 rx_err_ratio;
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_btdevinfo" ========== */
+struct fbtc_btdevinfo {
+	u8 fver;
+	u8 rsvd0;
+	u16 vendor_id;
+
+	u32 dev_name; /* only 24 bits valid */
+	u32 flush_time;
+};
+/* ==========================================================================*/
+
+/* ========== C2H refer to "struct fbtc_testinfo" ========== */
+struct fbtc_testinfo {
+	u8 fver;
+	u8 rsvd0;
+	u8 rsvd1;
+	u8 rsvd2;
+	u8 rsvd3;
+	u8 rsvd4;
+	u8 rsvd5;
+	u8 rsvd6;
+};
+/* ==========================================================================*/
+
+/* ========== H2c refer to "struct btc_fddt_train_info" ========== */
+struct fbtc_fddt_cycle_info {
+	s8 tx_power; /* absolute Tx power (dBm), 0xff-> no BTC control */
+	s8 bt_tx_power; /* decrease Tx power (dB) */
+	s8 bt_rx_gain;  /* LNA constrain level */
+	u8 no_empty_cnt;
+
+	u8 rssi; /* [7:4] -> bt_rssi_level, [3:0]-> wl_rssi_level */
+	u8 cn; /* condition_num */
+	u8 train_status; /* [7:4]-> state, [3:0]-> phase */
+	u8 train_result; /* response: break,pass..  */
+
+	u8 phase_cycle; /* phase-cycle */
+	u8 train_step;  /* train_step  */
+	u8 tp_l8;       /* TP[7:0]     */
+	u8 tp_h8;       /* TP[15:8]    */
+};
+
+struct fbtc_fddt_cell_status {
+	s8 wl_tx_pwr;
+	s8 bt_tx_pwr;
+	s8 bt_rx_gain;
+	u8 state_phase; /* train state(bit[7:4])_phase(bit[3:0]) */
+};
+
+struct fbtc_fddt_sta { /* statistics for cycles */
+	u8 fver;
+	u8 cycles_l8;
+	u8 cycles_h8;
+	u8 rsvd0;
+
+	struct fbtc_fddt_cell_status fddt_cells[2][5][5];
+	struct fbtc_fddt_cycle_info fddt_trx[BTC_CYCLE_SLOT_MAX];
+};
+
+struct btc_fddt_bt_stat {
+	struct btc_rpt_ctrl_a2dp_empty a2dp_last;
+	u32 retry_last;
+};
+
+struct btc_fddt_cell {
+	s8 wl_pwr_min;
+	s8 wl_pwr_max;
+	s8 bt_pwr_dec_max;
+	s8 bt_rx_gain;
+};
+
+struct btc_fddt_fail_check { /* for cell stay in training */
+	u8 check_map;         /* check pass condition if bit-map = 1 */
+	u8 bt_no_empty_cnt;   /* 0-fail if no bt-empty >= th in train_cycle */
+	u8 wl_tp_ratio;       /* 1-fail if wl tp rise ratio < th */
+	u8 wl_kpibtr_ratio;   /* 2-fail if phase_now_tp < phase_last_tp * kpibtr_ratio*/
+};
+
+struct btc_fddt_break_check { /* for cell stay in training or train-ok */
+	u8 check_map;            /* check break condition if bit-map = 1 */
+	u8 bt_no_empty_cnt;  /* 0-break if no empty count >= th  */
+	u8 wl_tp_ratio;      /* 1-break if wl tp ratio < th (%)*/
+	u8 wl_tp_low_bound;  /* 2-break if wl tp (in Mbps) < th */
+
+	u8 cn;               /* 3-break if (cn >= cn_limit) >= th cycle */
+	u8 cell_chg;         /* 4-break if non-matched-RSSI >= th cycle */
+	s8 nhm_limit;        /* 5-break if nhm >= th --> ill-condition*/
+	u8 cn_limit;         /* if condition number >= th --> ill-condition  */
+};
+
+struct btc_fddt_time_ctrl {
+	/* 1 TDD cycle = w1 + b1, FDD 1cycle = w1fdd-slot + b1fdd-slot */
+	u8 m_cycle; /* KPI Moving-Average-Cycle: 1~32 cycles */
+	u8 w_cycle; /* Start to calcul WKPI after this if train-phase change */
+	u8 k_cycle; /* Total kpi-estimate cycles for each training-step */
+	u8 rsvd;
+};
+
+struct btc_fddt_train_info {
+	struct btc_fddt_time_ctrl t_ctrl;
+	struct btc_fddt_break_check b_chk;
+	struct btc_fddt_fail_check f_chk;
+	struct btc_fddt_cell cell_ul[5][5];
+	struct btc_fddt_cell cell_dl[5][5];
+};
+
+struct btc_fddt_info {
+	u8 type;         /* refer to enum btc_fddt_type */
+	u8 result;       /* fw send fdd-training status by c2h  */
+	u8 state;        /* refer to enum btc_fddt_state */
+
+	u8 wl_iot[6];    /* wl bssid  */
+	u16 bt_iot;      /* bt vendor-id */
+
+	u32 nrsn_map;    /* the reason map for no-run fdd-traing */
+	struct btc_fddt_bt_stat bt_stat;  /* bt statistics */
+	struct btc_fddt_train_info train;
+	struct btc_fddt_train_info train_now;
+};
+/* ==========================================================================*/
+
+/*
+ * End of FW Definition
+ */
+#pragma pack(pop)
+
+#endif	/* __INC_BTC_FW_DEF_H__ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse.c
new file mode 100644
index 000000000000..18e45116bc35
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse.c
@@ -0,0 +1,1526 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+/*
+The purpose of hal_efuse.c
+
+Provide efuse operations.
+a. efuse init function
+b. efuse shadow map read/write/update
+c. efuse information query, map size/used bytes...
+
+*/
+
+#define _HAL_EFUSE_C_
+#include "../hal_headers.h"
+#include "hal_efuse.h"
+#include "hal_efuse_export.h"
+
+#define get_hci_to_a_die_offset(log_size, limit_log_size) (log_size - limit_log_size)
+#define get_a_die_start_offset(limit_log_size, a_die_size) (limit_log_size - a_die_size)
+
+/* WIFI EFUSE API */
+void efuse_shadow_read_one_byte(struct efuse_t *efuse, u16 offset, u8 *value)
+{
+	*value = efuse->shadow_map[offset];
+}
+
+void efuse_shadow_read_two_byte(struct efuse_t *efuse, u16 offset, u16 *value)
+{
+	*value = efuse->shadow_map[offset];
+	*value |= efuse->shadow_map[offset+1] << 8;
+}
+
+void efuse_shadow_read_four_byte(struct efuse_t *efuse, u16 offset, u32 *value)
+{
+	*value = efuse->shadow_map[offset];
+	*value |= efuse->shadow_map[offset+1] << 8;
+	*value |= efuse->shadow_map[offset+2] << 16;
+	*value |= efuse->shadow_map[offset+3] << 24;
+}
+
+void efuse_shadow_write_one_byte(struct efuse_t *efuse, u16 offset, u16 value)
+{
+	efuse->shadow_map[offset] = (u8)(value&0x00FF);
+}
+
+void efuse_shadow_write_two_byte(struct efuse_t *efuse, u16 offset, u16 value)
+{
+	efuse->shadow_map[offset] = (u8)(value&0x00FF);
+	efuse->shadow_map[offset+1] = (u8)((value&0xFF00) >> 8);
+}
+
+void efuse_shadow_write_four_byte(struct efuse_t *efuse, u16 offset, u32 value)
+{
+	efuse->shadow_map[offset] = (u8)(value&0x000000FF);
+	efuse->shadow_map[offset+1] = (u8)((value&0x0000FF00) >> 8);
+	efuse->shadow_map[offset+2] = (u8)((value&0x00FF0000) >> 16);
+	efuse->shadow_map[offset+3] = (u8)((value&0xFF000000) >> 24);
+}
+
+u32 efuse_check_autoload(struct efuse_t *efuse)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_mac_check_efuse_autoload(efuse->hal_com,
+												  &efuse->is_map_valid);
+	if (efuse->is_map_valid != true)
+		efuse->map_from_status = DEFAULT_MAP;
+
+	return hal_status;
+}
+
+u32 efuse_hidden_handle(struct efuse_t *efuse)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_mac_read_hidden_rpt(efuse->hal_com);
+
+	return hal_status;
+}
+
+enum rtw_hal_status efuse_set_hw_cap(struct efuse_t *efuse)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_com_t *hal_com = efuse->hal_com;
+	u8 pkg_type = 0xFF;
+	u8 rfe_type = 0xFF;
+	u8 xcap = 0xFF;
+	u8 domain = 0xFF;
+	u8 domain_6g = 0xFF;
+	u8 rf_board_opt = 0xFF;
+
+	status = rtw_efuse_get_info(efuse, EFUSE_INFO_RF_PKG_TYPE, &pkg_type,
+				    sizeof(pkg_type));
+
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("%s: Get pkg type fail! Status(%x)\n", __FUNCTION__, status);
+	}
+
+	status = rtw_efuse_get_info(efuse, EFUSE_INFO_RF_RFE, &rfe_type,
+				    sizeof(rfe_type));
+
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("%s: Get rfe type fail! Status(%x)\n", __FUNCTION__, status);
+	}
+
+	status = rtw_efuse_get_info(efuse, EFUSE_INFO_RF_XTAL, &xcap,
+				    sizeof(xcap));
+
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("%s: Get xcap fail! Status(%x)\n", __FUNCTION__, status);
+	}
+
+	status = rtw_efuse_get_info(efuse, EFUSE_INFO_RF_CHAN_PLAN, &domain,
+				    sizeof(domain));
+
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("%s: Get domain fail! Status(%x)\n", __FUNCTION__, status);
+	}
+
+	status = rtw_efuse_get_info(efuse, EFUSE_INFO_RF_CHAN_PLAN_6GHZ,
+					&domain_6g, sizeof(domain_6g));
+
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("%s: Get 6ghz domain fail! Status(%x)\n", __FUNCTION__, status);
+	}
+
+	status = rtw_efuse_get_info(efuse, EFUSE_INFO_RF_BOARD_OPTION, &rf_board_opt,
+				sizeof(rf_board_opt));
+
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("%s: Get domain fail! Status(%x)\n", __FUNCTION__, status);
+	}
+
+	hal_com->dev_hw_cap.pkg_type = pkg_type;
+	hal_com->dev_hw_cap.rfe_type = rfe_type;
+	hal_com->dev_hw_cap.xcap = xcap;
+	hal_com->dev_hw_cap.domain = domain;
+	hal_com->dev_hw_cap.domain_6g = domain_6g;
+	hal_com->dev_hw_cap.rf_board_opt = rf_board_opt;
+
+	return status;
+}
+
+/*
+ * This function is used for backward compatible by calling RF-API.
+ * ex:
+ *     In 1T2R setting, backward to 1T1R by checking rfe_type
+ *     RFE_43(efuse_x02CA=0x2B): 1T2R(Tx diversity + Rx MRC)
+ *     RFE_41(efuse_x02CA=0x29): 1T1R(Tx/Rx@pathB)
+ *
+ */
+void efuse_compatible_chk(struct efuse_t *efuse)
+{
+	rtw_hal_rf_rfe_ant_num_chk(efuse->hal_com);
+}
+
+enum rtw_hal_status rtw_efuse_logicmap_buf_load(void *efuse, u8* buf, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_read_log_efuse_map(efuse_info->hal_com,
+						buf,
+						is_limit);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_shadow_load(void *efuse, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_read_log_efuse_map(efuse_info->hal_com,
+											efuse_info->shadow_map,
+											is_limit);
+
+	if (efuse_info->is_map_valid == true && status == RTW_HAL_STATUS_SUCCESS)
+		efuse_info->map_from_status = HW_LOG_MAP;
+
+	return status;
+}
+
+bool rtw_efuse_chk_tssi_data_vaild(void *efuse) {
+	struct efuse_t *efuse_info = efuse;
+	u32 i =0 , countr =0;
+
+	/*path s0*/
+	for (i = 0x210 ; i <= 0x239; i++) {
+		if ((efuse_info->shadow_map[i]) != 0xff)
+			countr++;
+	}
+	/*path s1*/
+	for (i = 0x23a ; i <= 0x2dc; i++) {
+		if ((efuse_info->shadow_map[i]) != 0xff)
+			countr++;
+	}
+	PHL_INFO("%s , check Efuse countr = %d\n", __func__, countr);
+	if (countr > 40) {
+		PHL_INFO("%s check eFuse TSSI Data SUCCESS !!!\n", __func__);
+		return true;
+	}
+	PHL_INFO("%s eFuse TSSI Data all are 0xff, return STATUS_FAILURE !!!\n", __func__);
+	return false;
+}
+
+enum rtw_hal_status rtw_efuse_shadow_file_load(void *efuse, char *ic_name, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+#ifdef CONFIG_EFUSE_CONFIG_FILE
+	struct efuse_t *efuse_info = efuse;
+
+	if (efuse_info->is_map_valid != true || rtw_efuse_chk_tssi_data_vaild(efuse_info) != true) {
+
+		if (rtw_hal_efuse_shadow_file_load(efuse_info->hal_com ,
+						ic_name, is_limit) == RTW_HAL_STATUS_SUCCESS) {
+
+			if (rtw_efuse_chk_tssi_data_vaild(efuse_info) == true) {
+				efuse_info->is_map_valid = true;
+				PHL_INFO(" %s() hal_rf check file efuse is_map_valid.\n", __FUNCTION__);
+			} else {
+				status = RTW_HAL_STATUS_FAILURE;
+				PHL_WARN(" %s() efuse Power invalid !\n", __FUNCTION__);
+			}
+		} else {
+			PHL_WARN("%s: efuse shadow_file_load fail!\n", __FUNCTION__);
+			status = RTW_HAL_STATUS_FAILURE;
+		}
+	}
+#endif
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_shadow_update(void *efuse, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+	enum rtw_hal_status reload_status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	u32 map_size = 0, mask_size = 0;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false) {
+		PHL_WARN("%s: efuse map not load yet!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_UNINIT;
+		goto exit;
+	}
+
+	/* Load efuse mask file before PG */
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_MASK_FILE_LOADED) == false) {
+		PHL_WARN("%s: efuse mask not load yet!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+		goto exit;
+	}
+
+	/*
+	 * If driver does not load external map file,
+	 * assign the map version with the mask version for the version check.
+	 */
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_MAP_FILE_LOADED) == false) {
+		_os_mem_cpy(efuse_info->hal_com->drv_priv, efuse_info->map_version,
+					efuse_info->mask_version, efuse_info->version_len);
+	}
+
+	if(is_limit) {
+		map_size = efuse_info->limit_efuse_size;
+		mask_size = efuse_info->limit_mask_size;
+	}
+	else {
+		map_size = efuse_info->log_efuse_size;
+		mask_size = efuse_info->mask_size;
+	}
+
+	status = rtw_hal_mac_write_log_efuse_map(efuse_info->hal_com,
+							efuse_info->shadow_map,
+							map_size,
+							efuse_info->mask,
+							mask_size,
+							efuse_info->map_version,
+							efuse_info->mask_version,
+							efuse_info->version_len,
+							0,
+							is_limit,
+							efuse_info->efuse_a_die_size,
+							efuse_info->hci_to_a_die_offset,
+							efuse_info->a_die_start_offset);
+
+	if(status != RTW_HAL_STATUS_SUCCESS)
+		PHL_WARN("%s: PG Fail!\n", __FUNCTION__);
+
+	/* Reload shadow map after PG */
+	reload_status = rtw_hal_mac_read_log_efuse_map(efuse_info->hal_com,
+								efuse_info->shadow_map,
+								is_limit);
+	if(reload_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_WARN("%s: Reload shadow map Fail!\n", __FUNCTION__);
+
+	/*
+	 * Clear the load external map file flag and map version
+	 * after reloading the map.
+	 */
+	if(TEST_STATUS_FLAG(efuse_info->status,
+						EFUSE_STATUS_MAP_FILE_LOADED) == true) {
+		CLEAR_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_MAP_FILE_LOADED);
+	}
+
+	_os_mem_set(efuse_info->hal_com->drv_priv,
+				efuse_info->map_version,
+				0,
+				efuse_info->version_len);
+exit:
+	return status;
+}
+
+enum rtw_hal_status
+rtw_efuse_shadow_read(void *efuse, u8 byte_count, u16 offset, u32 *value,
+					  bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	u32	efuse_size = 0;
+
+	if(is_limit)
+		efuse_size = efuse_info->limit_efuse_size;
+	else
+		efuse_size = efuse_info->log_efuse_size;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false) {
+		PHL_WARN("%s: efuse map not load yet!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_UNINIT;
+		goto exit;
+	}
+
+	if((u32)(offset+byte_count) > efuse_size) {
+		PHL_WARN("%s: Invalid offset!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_IVALID_OFFSET;
+		goto exit;
+	}
+
+	if (efuse_info->efuse_a_die_size != 0 &&
+		offset >= efuse_info->a_die_start_offset &&
+		is_limit)
+		offset += (u16)efuse_info->hci_to_a_die_offset;
+
+	if (byte_count == 1)
+		efuse_shadow_read_one_byte(efuse_info, offset, (u8 *)value);
+	else if (byte_count == 2)
+		efuse_shadow_read_two_byte(efuse_info, offset, (u16 *)value);
+	else if (byte_count == 4)
+		efuse_shadow_read_four_byte(efuse_info, offset, (u32 *)value);
+
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return status;
+}
+
+enum rtw_hal_status
+rtw_efuse_shadow_write(void *efuse, u8 byte_count, u16 offset, u32 value,
+					   bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	u32	efuse_size = 0;
+
+	if(is_limit)
+		efuse_size = efuse_info->limit_efuse_size;
+	else
+		efuse_size = efuse_info->log_efuse_size;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false) {
+		PHL_WARN("%s: efuse map not load yet!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_UNINIT;
+		goto exit;
+	}
+
+	if((u32)(offset+byte_count) > efuse_size) {
+		PHL_WARN("%s: Invalid offset!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_IVALID_OFFSET;
+		goto exit;
+	}
+
+	if (efuse_info->efuse_a_die_size != 0 &&
+		offset >= efuse_info->a_die_start_offset &&
+		is_limit)
+		offset += (u16)efuse_info->hci_to_a_die_offset;
+
+	if (byte_count == 1)
+		efuse_shadow_write_one_byte(efuse_info, offset, (u8)value);
+	else if (byte_count == 2)
+		efuse_shadow_write_two_byte(efuse_info, offset, (u16)value);
+	else if (byte_count == 4)
+		efuse_shadow_write_four_byte(efuse_info, offset, (u32)value);
+
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return status;
+}
+
+
+/*
+ * This API is used for original mp dll command.
+ * usage = used percentage(1 Byte) + used bytes(2 Bytes)
+ */
+enum rtw_hal_status rtw_efuse_get_usage(void *efuse, u32 *usage)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	u32 avl_size = 0;
+	u16 used_byte = 0;
+	u32 total_size = 0;
+	u8 used_percentage = 0;
+
+	status = rtw_hal_mac_get_efuse_avl(efuse_info->hal_com, &avl_size);
+	status = rtw_hal_mac_get_efuse_size(efuse_info->hal_com, &total_size);
+
+	if(total_size != 0){
+		used_byte = (u16)(total_size - avl_size);
+		used_percentage = (u8)(used_byte*100/total_size);
+		*usage = (used_percentage<<16)|(used_byte);
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_shadow2buf(void *efuse, u8 *destbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct efuse_t *efuse_info = efuse;
+	u16 offset;
+
+	if (efuse_info->efuse_a_die_size != 0) {
+		for (offset = efuse_info->a_die_start_offset;
+			offset < (efuse_info->a_die_start_offset + (u32)efuse_info->efuse_a_die_size); offset++)
+			efuse_info->shadow_map[offset] =
+						efuse_info->shadow_map[offset + efuse_info->hci_to_a_die_offset];
+	}
+	_os_mem_cpy(efuse_info->hal_com->drv_priv, (void *)destbuf,
+				(void *)efuse_info->shadow_map , buflen);
+
+	return status;
+}
+
+enum rtw_hal_status
+efuse_map_buf2shadow(struct efuse_t *efuse, u8 *srcbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+	_os_mem_cpy(efuse->hal_com->drv_priv, (void *)efuse->shadow_map,
+				(void *)srcbuf, buflen);
+	SET_STATUS_FLAG(efuse->status, EFUSE_STATUS_MAP_FILE_LOADED);
+
+	return status;
+}
+
+enum rtw_hal_status
+efuse_file_map2version(struct efuse_t *efuse, u8 *srcbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+	_os_mem_cpy(efuse->hal_com->drv_priv, (void *)efuse->map_version,
+				(void *)srcbuf, buflen);
+	debug_dump_data(efuse->map_version, efuse->version_len,
+					"[HAL EFUSE] map version =");
+
+	return status;
+}
+
+enum rtw_hal_status
+efuse_file_mask2buf(struct efuse_t *efuse, u8 *srcbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_cpy(efuse->hal_com->drv_priv, (void *)efuse->mask, (void *)srcbuf,
+				buflen);
+	SET_STATUS_FLAG(efuse->status, EFUSE_STATUS_MASK_FILE_LOADED);
+	status = RTW_HAL_STATUS_SUCCESS;
+
+	debug_dump_data(efuse->mask, efuse->mask_size, "[HAL EFUSE] mask data =");
+
+	return status;
+}
+
+enum rtw_hal_status
+efuse_file_mask2version(struct efuse_t *efuse, u8 *srcbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	if (buflen <= efuse->version_len) {
+		_os_mem_cpy(efuse->hal_com->drv_priv, (void *)efuse->mask_version,
+					(void *)srcbuf, buflen);
+		debug_dump_data(efuse->mask_version, efuse->version_len,
+						"[HAL EFUSE] mask ver =");
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+
+static u16 efuse_file_open(void *d, char *filepath, u8 *buf, u32 buflen)
+{
+	char *ptmpbuf = NULL, *ptr;
+	u32 bufsize = 5120;
+	u8 val8 = 0;
+	u32 file_size = 0, i = 0, j= 0, tmp = 0;
+	int err = 0;
+
+	ptmpbuf = _os_mem_alloc(d, bufsize);
+	if (ptmpbuf == NULL) {
+		PHL_INFO("%s, alloc buf FAIL!!\n", __FUNCTION__);
+		return 0;
+	}
+
+	file_size = _os_read_file(filepath, (u8*)ptmpbuf, bufsize);
+	if (file_size > 0) {
+		if (file_size <= 90) {
+			_os_mem_free(d, (void*)ptmpbuf, bufsize);
+			PHL_INFO("%s, filepatch %s, size=%d not over 90, FAIL!!\n",
+								__FUNCTION__, filepath, file_size);
+			return 0;
+		}
+		i = 0;
+		j = 0;
+		ptr = ptmpbuf;
+		while ((j < buflen) && (i < file_size)) {
+			if (ptmpbuf[i] == '\0')
+				break;
+			ptr = _os_strpbrk((char *)&ptmpbuf[i], " \t\n\r");
+			if (ptr) {
+				if (ptr == &ptmpbuf[i]) {
+					i++;
+					continue;
+				}
+				/* Add string terminating null */
+				*ptr = 0;
+			} else {
+				ptr = &ptmpbuf[file_size-1];
+			}
+
+			err = _os_sscanf((char *)&ptmpbuf[i], "%x", &tmp);
+
+			if (err != 1) {
+				PHL_INFO("Something wrong to parse efuse file, string=%s\n", &ptmpbuf[i]);
+			} else {
+				val8 = (u8)tmp;
+				buf[j] = val8;
+				PHL_INFO("i=%d, j=%d, 0x%02x\n", i, j, buf[j]);
+				j++;
+			}
+			i = (u32)(ptr - ptmpbuf + 1);
+		}
+	}
+	if (ptmpbuf)
+		_os_mem_free(d, (void*)ptmpbuf, bufsize);
+	PHL_INFO("%s, filepatch %s, size=%d, done\n", __FUNCTION__, filepath, file_size);
+	return (u16)j;
+}
+
+enum rtw_hal_status
+rtw_efuse_file_map_load(void *efuse, char *file_path, u8 is_limit)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	void *d = efuse_info->hal_com->drv_priv;
+	u8 *mapbuf = NULL;
+	u16 data_len = 0;
+	u32 map_sz = 0, full_map_sz = 0;
+	u32 offset;
+
+	if (is_limit)
+		map_sz = efuse_info->limit_efuse_size;
+	else
+		map_sz = efuse_info->log_efuse_size;
+
+	if (map_sz == 0) {
+		map_sz = MAX_EFUSE_MAP_LEN;
+		full_map_sz = MAX_EFUSE_MAP_LEN + MAX_EFUSE_FILE_VERSION_LENGTH;
+	} else
+		full_map_sz = map_sz + efuse_info->version_len;
+
+	if (file_path && full_map_sz != 0) {
+		mapbuf= _os_mem_alloc(d, full_map_sz);
+		if (mapbuf == NULL) {
+			PHL_INFO("%s, alloc buf FAIL!!\n", __FUNCTION__);
+			goto exit;
+		}
+		_os_mem_set(d, mapbuf, 0xFF, full_map_sz);
+
+		data_len = efuse_file_open(d, file_path, mapbuf, full_map_sz);
+
+		if ((data_len > map_sz) && (data_len <= full_map_sz)) {
+			efuse_file_map2version(efuse_info, mapbuf + map_sz,
+								   efuse_info->version_len);
+
+			if (data_len > map_sz)
+				data_len -= efuse_info->version_len;
+
+			PHL_INFO("%s , File eFuse map to shadow len %d\n", __FUNCTION__, data_len);
+			hal_status = efuse_map_buf2shadow(efuse_info, mapbuf, data_len);
+
+			if (efuse_info->efuse_a_die_size != 0 && is_limit) {
+				for (offset = efuse_info->a_die_start_offset;
+					offset < (efuse_info->a_die_start_offset + (u32)efuse_info->efuse_a_die_size);
+					offset++)
+					efuse_info->shadow_map[offset + efuse_info->hci_to_a_die_offset] =
+						mapbuf[offset];
+			}
+
+			efuse_info->map_from_status = FILE_MAP;
+		} else {
+			PHL_INFO("Error No Map Version !, File Map Data Len %d not over 1536.\n", data_len);
+			goto exit;
+		}
+	}
+exit:
+	if (mapbuf)
+		_os_mem_free(d, (void*)mapbuf, full_map_sz);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_efuse_file_mask_load(void *efuse, char *file_path, u8 is_limit)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	void *d = efuse_info->hal_com->drv_priv;
+	u8 *maskbuf = NULL;
+	u16 data_len = 0;
+	u32 mask_sz = 0, full_mask_sz = 0;
+
+	if (is_limit)
+		mask_sz = efuse_info->limit_mask_size;
+	else
+		mask_sz = efuse_info->mask_size;
+
+	if (mask_sz == 0) {
+		mask_sz = EFUSE_MASK_FILE_LEN;
+		full_mask_sz = EFUSE_MASK_FILE_LEN + MAX_EFUSE_FILE_VERSION_LENGTH;
+	} else
+		full_mask_sz = mask_sz + efuse_info->version_len;
+
+	if (file_path) {
+		maskbuf= _os_mem_alloc(d, full_mask_sz);
+		if (maskbuf == NULL) {
+			PHL_INFO("%s, alloc buf FAIL!!\n", __FUNCTION__);
+			goto exit;
+		}
+		_os_mem_set(d, maskbuf, 0xFF, full_mask_sz);
+
+		data_len = efuse_file_open(d, file_path, maskbuf, full_mask_sz);
+
+		if ((data_len > mask_sz) && (data_len <= full_mask_sz)) {
+			efuse_file_mask2version(efuse_info, maskbuf + mask_sz,
+									efuse_info->version_len);
+
+			if (data_len > mask_sz)
+				data_len -= efuse_info->version_len;
+
+			PHL_INFO("Mask File data 2 buf len %d\n", data_len);
+			hal_status = efuse_file_mask2buf(efuse_info, maskbuf, data_len);
+
+		} else {
+			PHL_INFO("Error No Map Version !, File Map Data Len %d not over 102.\n", data_len);
+			goto exit;
+		}
+	}
+exit:
+	if (maskbuf)
+		_os_mem_free(d, (void*)maskbuf, full_mask_sz);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_efuse_get_logical_size(void *efuse, u32 *size,
+											   bool is_limited)
+{
+	struct efuse_t *efuse_info = efuse;
+
+	if(is_limited == true)
+		*size = efuse_info->limit_efuse_size;
+	else
+		*size = efuse_info->log_efuse_size;
+
+	PHL_INFO("%s: size = %d\n", __FUNCTION__, *size);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_efuse_get_size(void *efuse, u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_get_efuse_size(efuse_info->hal_com, size);
+
+	PHL_INFO("%s: size = %d\n", __FUNCTION__, *size);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_get_avl(void *efuse, u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_get_efuse_avl(efuse_info->hal_com, size);
+
+	/* [TEMP] Set avl size for testing */
+	/* size = 512; */
+	PHL_INFO("%s: size = %d\n", __FUNCTION__, *size);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_get_offset_mask(void *efuse, u16 offset, u8 *mask)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct efuse_t *efuse_info = efuse;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false)
+		return RTW_HAL_STATUS_EFUSE_UNINIT;
+
+	if(offset >= efuse_info->limit_mask_size)
+		return RTW_HAL_STATUS_EFUSE_IVALID_OFFSET;
+
+	*mask = efuse_info->mask[offset];
+	PHL_INFO("%s: offset = %x mask = %x\n", __FUNCTION__, offset, *mask);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_get_mask_buf(void *efuse, u8 *destbuf, u32 *buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	if (efuse_info->mask_size > 0) {
+		_os_mem_cpy(efuse_info->hal_com->drv_priv, (void *)destbuf,
+				(void *)efuse_info->mask , efuse_info->mask_size);
+		*buflen = efuse_info->mask_size;
+
+		PHL_INFO("%s: size = %d\n", __FUNCTION__, efuse_info->mask_size);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_get_shadowmap_from(void *efuse, u8 *val)
+{
+	struct efuse_t *efuse_info = efuse;
+
+	*val = efuse_info->map_from_status;
+
+	PHL_INFO("%s:get efuse %s\n", __FUNCTION__, SHADOWMAP_FROM2STR(*val));
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+char* rtw_efuse_get_shadowmap_from_to_str(void *efuse)
+{
+	struct efuse_t *efuse_info = efuse;
+
+	return SHADOWMAP_FROM2STR(efuse_info->map_from_status);
+}
+
+
+enum rtw_hal_status rtw_efuse_get_info(void *efuse,
+									   enum rtw_efuse_info info_type,
+									   void *value,
+									   u8 size)
+{
+	struct efuse_t *efuse_info = efuse;
+	struct rtw_hal_com_t *hal_com = efuse_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false)
+		return RTW_HAL_STATUS_EFUSE_UNINIT;
+
+	if(info_type <= EFUSE_INFO_MAC_MAX)
+		hal_status = rtw_hal_mac_get_efuse_info(hal_com,
+												efuse_info->shadow_map,
+												info_type,
+												value,
+												size,
+												efuse_info->is_map_valid);
+	else if (info_type <= EFUSE_INFO_BB_MAX)
+		hal_status = rtw_hal_bb_get_efuse_info(hal_com,
+											   efuse_info->shadow_map,
+											   info_type,
+											   value,
+											   size,
+											   efuse_info->is_map_valid);
+	else if (info_type <= EFUSE_INFO_RF_MAX)
+		hal_status = rtw_hal_rf_get_efuse_info(hal_com,
+											   efuse_info->shadow_map,
+											   info_type,
+											   value,
+											   size,
+											   efuse_info->is_map_valid);
+	else
+		hal_status = rtw_hal_btc_get_efuse_info(hal_com,
+												efuse_info->shadow_map,
+												info_type,
+												value,
+												size,
+												efuse_info->is_map_valid);
+
+	return hal_status;
+}
+
+void rtw_efuse_process(void *efuse, char *ic_name)
+{
+	struct efuse_t *efuse_info = (struct efuse_t *)efuse;
+
+	if(efuse_info == NULL) {
+		PHL_ERR("%s efuse_info is NULL\n",__FUNCTION__);
+		return;
+	}
+
+	efuse_check_autoload(efuse_info);
+	/* Load wifi full map to shadow map */
+	rtw_efuse_shadow_load(efuse_info, false);
+
+	SET_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS);
+
+	rtw_efuse_shadow_file_load(efuse_info, ic_name, true);
+
+	debug_dump_data(efuse_info->shadow_map, efuse_info->log_efuse_size,
+					"Logical EFUSE MAP:");
+	efuse_hidden_handle(efuse_info);
+
+	/*
+	 * We can set the hw cap after we got the shadow map.
+	 * The efuse get info API will check the efuse is processed or not.
+	 */
+	efuse_set_hw_cap(efuse_info);
+
+	efuse_compatible_chk(efuse_info);
+
+	if (RTW_DRV_MODE_EQC == efuse_info->phl_com->drv_mode) {
+		rtw_hal_rf_get_default_rfe_type(efuse_info->hal_com);
+		rtw_hal_rf_get_default_xtal(efuse_info->hal_com);
+		PHL_WARN("%s: Use default RFE type(0x%x) / XTAL(0x%x) configuration for EQC mode\n",
+			 __FUNCTION__,
+			 efuse_info->hal_com->dev_hw_cap.rfe_type,
+			 efuse_info->hal_com->dev_hw_cap.xcap);
+	}
+}
+
+bool rtw_efuse_is_processed(void *efuse)
+{
+	struct efuse_t *efuse_info = (struct efuse_t *)efuse;
+
+	if (TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == true)
+		return true;
+	else
+		return false;
+}
+
+u32 rtw_efuse_init(struct rtw_phl_com_t *phl_com,
+				   struct rtw_hal_com_t *hal_com, void **efuse)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = NULL;
+
+	efuse_info = _os_mem_alloc(hal_com->drv_priv, sizeof(struct efuse_t));
+
+	if(efuse_info == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		return hal_status;
+	}
+
+	/* Allocate shadow map memory */
+	hal_status = rtw_hal_mac_get_log_efuse_size(hal_com,
+						    &(efuse_info->log_efuse_size),
+						    false);
+
+	if(hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s Get full logical efuse map size fail!\n",__FUNCTION__);
+		goto error_efuse_shadow_init;
+	}
+
+	efuse_info->shadow_map = _os_mem_alloc(hal_com->drv_priv,
+					       efuse_info->log_efuse_size);
+
+	if(efuse_info->shadow_map == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		PHL_ERR("%s Allocate shadow efuse map fail!\n", __FUNCTION__);
+		goto error_efuse_shadow_init;
+	}
+
+	hal_status = rtw_hal_mac_get_log_efuse_size(hal_com,
+						    &(efuse_info->limit_efuse_size),
+						    true);
+
+	if(hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s Get limited logical efuse map size fail!\n", __FUNCTION__);
+		goto error_efuse_shadow_init;
+	}
+
+	hal_status = rtw_hal_mac_get_efuse_a_die_size(hal_com,
+						    &(efuse_info->efuse_a_die_size));
+
+	if(hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s Get efuse a die size fail!\n", __FUNCTION__);
+		goto error_efuse_shadow_init;
+	}
+
+	efuse_info->hci_to_a_die_offset = get_hci_to_a_die_offset(efuse_info->log_efuse_size,
+										efuse_info->limit_efuse_size);
+
+	efuse_info->a_die_start_offset = get_a_die_start_offset(efuse_info->limit_efuse_size,
+										efuse_info->efuse_a_die_size);
+
+	/* Allocate mask memory */
+	hal_status = rtw_hal_mac_get_efuse_mask_size(hal_com,
+						     &(efuse_info->mask_size),
+						     false);
+
+	if(hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s Get full efuse mask size fail!\n", __FUNCTION__);
+		goto error_efuse_mask_init;
+	}
+
+	efuse_info->mask = _os_mem_alloc(hal_com->drv_priv,
+					 efuse_info->mask_size);
+
+	if(efuse_info->mask == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		PHL_ERR("%s Allocate efuse mask fail!\n", __FUNCTION__);
+		goto error_efuse_mask_init;
+	}
+
+	hal_status = rtw_hal_mac_get_efuse_mask_size(hal_com,
+						     &(efuse_info->limit_mask_size),
+						     true);
+
+	if(hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s Get limited efuse mask size fail!\n", __FUNCTION__);
+		goto error_efuse_mask_init;
+	}
+
+	efuse_info->version_len = rtw_hal_mac_get_efuse_ver_len(hal_com);
+
+	efuse_info->map_version = _os_mem_alloc(hal_com->drv_priv,
+						efuse_info->version_len);
+
+	if(efuse_info->map_version == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		goto error_map_version_init;
+	}
+
+	efuse_info->mask_version = _os_mem_alloc(hal_com->drv_priv,
+						 efuse_info->version_len);
+	if(efuse_info->mask_version == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		goto error_mask_version_init;
+	}
+
+	/* Allocate bt shadow map memory */
+	hal_status = rtw_hal_mac_get_log_efuse_bt_size(hal_com,
+						&(efuse_info->bt_log_efuse_size));
+
+	if(hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s Get full logical efuse map size fail!\n",__FUNCTION__);
+		goto error_efuse_bt_shadow_init;
+	}
+
+	efuse_info->bt_shadow_map = _os_mem_alloc(hal_com->drv_priv,
+					       efuse_info->bt_log_efuse_size);
+
+	if(efuse_info->bt_shadow_map == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		PHL_ERR("%s Allocate shadow efuse map fail!\n", __FUNCTION__);
+		goto error_efuse_bt_shadow_init;
+	}
+
+	/* Allocate mask memory */
+	hal_status = rtw_hal_mac_get_efuse_bt_mask_size(hal_com,
+						     &(efuse_info->bt_mask_size));
+
+	if(hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s Get full efuse mask size fail!\n", __FUNCTION__);
+		goto error_efuse_bt_mask_init;
+	}
+
+	efuse_info->bt_mask = _os_mem_alloc(hal_com->drv_priv,
+					 efuse_info->bt_mask_size);
+
+	if(efuse_info->bt_mask == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		PHL_ERR("%s Allocate efuse mask fail!\n", __FUNCTION__);
+		goto error_efuse_bt_mask_init;
+	}
+
+
+	efuse_info->phl_com = phl_com;
+	efuse_info->hal_com = hal_com;
+
+	*efuse = efuse_info;
+
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+	return hal_status;
+
+error_efuse_bt_mask_init:
+	_os_mem_free(hal_com->drv_priv, efuse_info->bt_shadow_map,
+		     efuse_info->bt_log_efuse_size);
+
+error_efuse_bt_shadow_init:
+	_os_mem_free(hal_com->drv_priv, efuse_info->mask_version,
+		efuse_info->version_len);
+
+error_mask_version_init:
+	_os_mem_free(hal_com->drv_priv, efuse_info->map_version,
+		     efuse_info->version_len);
+
+error_map_version_init:
+	_os_mem_free(hal_com->drv_priv, efuse_info->mask,
+		     efuse_info->mask_size);
+
+error_efuse_mask_init:
+	_os_mem_free(hal_com->drv_priv, efuse_info->shadow_map,
+		     efuse_info->log_efuse_size);
+
+error_efuse_shadow_init:
+	_os_mem_free(hal_com->drv_priv, efuse_info, sizeof(struct efuse_t));
+
+	return hal_status;
+}
+
+void rtw_efuse_deinit(struct rtw_hal_com_t *hal_com, void *efuse)
+{
+	struct efuse_t *efuse_info = efuse;
+
+	if(efuse_info->bt_mask) {
+		_os_mem_free(hal_com->drv_priv, efuse_info->bt_mask,
+					 efuse_info->bt_mask_size);
+		efuse_info->bt_mask = NULL;
+	}
+
+	if(efuse_info->bt_shadow_map) {
+		_os_mem_free(hal_com->drv_priv, efuse_info->bt_shadow_map,
+					 efuse_info->bt_log_efuse_size);
+		efuse_info->bt_shadow_map = NULL;
+	}
+
+	if(efuse_info->mask) {
+		_os_mem_free(hal_com->drv_priv, efuse_info->mask,
+					 efuse_info->mask_size);
+		efuse_info->mask = NULL;
+	}
+	if(efuse_info->map_version) {
+		_os_mem_free(hal_com->drv_priv, efuse_info->map_version,
+					 efuse_info->version_len);
+		efuse_info->map_version = NULL;
+	}
+	if(efuse_info->mask_version) {
+		 _os_mem_free(hal_com->drv_priv, efuse_info->mask_version,
+		 				efuse_info->version_len);
+		 efuse_info->mask_version = NULL;
+	}
+	if(efuse_info->shadow_map) {
+		_os_mem_free(hal_com->drv_priv, efuse_info->shadow_map,
+					 efuse_info->log_efuse_size);
+		efuse_info->shadow_map = NULL;
+	}
+
+	_os_mem_free(hal_com->drv_priv, efuse_info, sizeof(struct efuse_t));
+	efuse_info = NULL;
+}
+
+/* BT EFUSE API */
+void efuse_bt_shadow_read_one_byte(struct efuse_t *efuse, u16 offset, u8 *value)
+{
+   *value = efuse->bt_shadow_map[offset];
+}
+
+void efuse_bt_shadow_read_two_byte(struct efuse_t *efuse, u16 offset, u16 *value)
+{
+   *value = efuse->bt_shadow_map[offset];
+   *value |= efuse->bt_shadow_map[offset+1] << 8;
+}
+
+void efuse_bt_shadow_read_four_byte(struct efuse_t *efuse, u16 offset, u32 *value)
+{
+   *value = efuse->bt_shadow_map[offset];
+   *value |= efuse->bt_shadow_map[offset+1] << 8;
+   *value |= efuse->bt_shadow_map[offset+2] << 16;
+   *value |= efuse->bt_shadow_map[offset+3] << 24;
+}
+
+void efuse_bt_shadow_write_one_byte(struct efuse_t *efuse, u16 offset, u16 value)
+{
+   efuse->bt_shadow_map[offset] = (u8)(value&0x00FF);
+}
+
+void efuse_bt_shadow_write_two_byte(struct efuse_t *efuse, u16 offset, u16 value)
+{
+   efuse->bt_shadow_map[offset] = (u8)(value&0x00FF);
+   efuse->bt_shadow_map[offset+1] = (u8)((value&0xFF00) >> 8);
+}
+
+void efuse_bt_shadow_write_four_byte(struct efuse_t *efuse, u16 offset, u32 value)
+{
+   efuse->bt_shadow_map[offset] = (u8)(value&0x000000FF);
+   efuse->bt_shadow_map[offset+1] = (u8)((value&0x0000FF00) >> 8);
+   efuse->bt_shadow_map[offset+2] = (u8)((value&0x00FF0000) >> 16);
+   efuse->bt_shadow_map[offset+3] = (u8)((value&0xFF000000) >> 24);
+}
+
+enum rtw_hal_status rtw_efuse_bt_shadow_load(void *efuse)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_read_log_efuse_bt_map(efuse_info->hal_com,
+						efuse_info->bt_shadow_map, efuse_info->bt_log_efuse_size);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_shadow_update(void *efuse)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+	enum rtw_hal_status reload_status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	u32 map_size = 0, mask_size = 0;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false) {
+		PHL_WARN("%s: efuse map not load yet!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_UNINIT;
+		goto exit;
+	}
+
+	/* Load efuse mask file before PG */
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_BT_MASK_FILE_LOADED) == false) {
+		PHL_WARN("%s: efuse mask not load yet!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+		goto exit;
+	}
+
+	map_size = efuse_info->bt_log_efuse_size;
+	mask_size = efuse_info->bt_mask_size;
+
+	status = rtw_hal_mac_write_log_efuse_bt_map(efuse_info->hal_com,
+							efuse_info->bt_shadow_map,
+							map_size,
+							efuse_info->bt_mask,
+							mask_size);
+
+	if(status != RTW_HAL_STATUS_SUCCESS)
+		PHL_WARN("%s: BT PG Fail!\n", __FUNCTION__);
+
+	/* Reload shadow map after PG */
+	reload_status = rtw_hal_mac_read_log_efuse_bt_map(efuse_info->hal_com,
+								efuse_info->bt_shadow_map, efuse_info->bt_log_efuse_size);
+	if(reload_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_WARN("%s: Reload bt shadow map Fail!\n", __FUNCTION__);
+
+	/*
+	 * Clear the load external map file flag and map version
+	 * after reloading the map.
+	 */
+	if(TEST_STATUS_FLAG(efuse_info->status,
+				EFUSE_STATUS_BT_MAP_FILE_LOADED) == true) {
+		CLEAR_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_BT_MAP_FILE_LOADED);
+	}
+
+exit:
+	return status;
+}
+
+enum rtw_hal_status
+rtw_efuse_bt_shadow_read(void *efuse, u8 byte_count, u16 offset, u32 *value)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	u32	efuse_bt_size = 0;
+
+	efuse_bt_size = efuse_info->bt_log_efuse_size;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false) {
+		PHL_WARN("%s: efuse map not load yet!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_UNINIT;
+		goto exit;
+	}
+
+	if((u32)(offset+byte_count) > efuse_bt_size) {
+		PHL_WARN("%s: Invalid offset!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_IVALID_OFFSET;
+		goto exit;
+	}
+
+	if (byte_count == 1)
+		efuse_bt_shadow_read_one_byte(efuse_info, offset, (u8 *)value);
+	else if (byte_count == 2)
+		efuse_bt_shadow_read_two_byte(efuse_info, offset, (u16 *)value);
+	else if (byte_count == 4)
+		efuse_bt_shadow_read_four_byte(efuse_info, offset, (u32 *)value);
+
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return status;
+}
+
+enum rtw_hal_status
+rtw_efuse_bt_shadow_write(void *efuse, u8 byte_count, u16 offset, u32 value)
+{
+  enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+  struct efuse_t *efuse_info = efuse;
+  u32 efuse_bt_size = 0;
+
+  efuse_bt_size = efuse_info->bt_log_efuse_size;
+
+  if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false) {
+	  PHL_WARN("%s: efuse map not load yet!\n", __FUNCTION__);
+	  status = RTW_HAL_STATUS_EFUSE_UNINIT;
+	  goto exit;
+  }
+
+  if((u32)(offset+byte_count) > efuse_bt_size) {
+	  PHL_WARN("%s: Invalid offset!\n", __FUNCTION__);
+	  status = RTW_HAL_STATUS_EFUSE_IVALID_OFFSET;
+	  goto exit;
+  }
+
+  if (byte_count == 1)
+	  efuse_bt_shadow_write_one_byte(efuse_info, offset, (u8)value);
+  else if (byte_count == 2)
+	  efuse_bt_shadow_write_two_byte(efuse_info, offset, (u16)value);
+  else if (byte_count == 4)
+	  efuse_bt_shadow_write_four_byte(efuse_info, offset, (u32)value);
+
+  status = RTW_HAL_STATUS_SUCCESS;
+exit:
+  return status;
+}
+
+/*
+ * This API is used for original mp dll command.
+ * usage = used percentage(1 Byte) + used bytes(2 Bytes)
+ */
+enum rtw_hal_status rtw_efuse_bt_get_usage(void *efuse, u32 *usage)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	u32 avl_size = 0;
+	u16 used_byte = 0;
+	u32 total_size = 0;
+	u8 used_percentage = 0;
+
+	status = rtw_hal_mac_get_efuse_bt_avl(efuse_info->hal_com, &avl_size);
+	status = rtw_hal_mac_get_efuse_bt_size(efuse_info->hal_com, &total_size);
+
+	if(total_size != 0){
+		used_byte = (u16)(total_size - avl_size);
+		used_percentage = (u8)(used_byte*100/total_size);
+		*usage = (used_percentage<<16)|(used_byte);
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_shadow2buf(void *efuse, u8 *destbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct efuse_t *efuse_info = efuse;
+
+	_os_mem_cpy(efuse_info->hal_com->drv_priv, (void *)destbuf,
+				(void *)efuse_info->bt_shadow_map , buflen);
+
+	return status;
+}
+
+enum rtw_hal_status
+efuse_bt_map_buf2shadow(struct efuse_t *efuse, u8 *srcbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_cpy(efuse->hal_com->drv_priv, (void *)efuse->bt_shadow_map,
+				(void *)srcbuf, buflen);
+	SET_STATUS_FLAG(efuse->status, EFUSE_STATUS_BT_MAP_FILE_LOADED);
+	status = RTW_HAL_STATUS_SUCCESS;
+
+	return status;
+}
+
+enum rtw_hal_status
+efuse_bt_file_mask2buf(struct efuse_t *efuse, u8 *srcbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_cpy(efuse->hal_com->drv_priv, (void *)efuse->bt_mask, (void *)srcbuf,
+				buflen);
+	SET_STATUS_FLAG(efuse->status, EFUSE_STATUS_BT_MASK_FILE_LOADED);
+	status = RTW_HAL_STATUS_SUCCESS;
+
+	debug_dump_data(efuse->bt_mask, efuse->bt_mask_size, "[HAL EFUSE] bt mask data =");
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_efuse_bt_file_map_load(void *efuse, char *file_path)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	void *d = efuse_info->hal_com->drv_priv;
+	u8 *mapbuf = NULL;
+	u16 data_len = 0;
+	u32 bt_map_sz = 0;
+
+	bt_map_sz = efuse_info->bt_log_efuse_size;
+
+	if (bt_map_sz == 0)
+		bt_map_sz = MAX_EFUSE_MAP_LEN;
+
+	if (file_path) {
+		mapbuf= _os_mem_alloc(d, bt_map_sz);
+		if (mapbuf == NULL) {
+			PHL_INFO("%s, alloc buf FAIL!!\n", __FUNCTION__);
+			goto exit;
+		}
+		_os_mem_set(d, mapbuf, 0xFF, bt_map_sz);
+
+		data_len = efuse_file_open(d, file_path, mapbuf, bt_map_sz);
+
+		if (data_len <= bt_map_sz && data_len > 0) {
+			PHL_INFO("%s , File eFuse bt map to shadow len %d\n", __FUNCTION__, data_len);
+			hal_status = efuse_bt_map_buf2shadow(efuse_info, mapbuf, data_len);
+
+		} else {
+			PHL_INFO("Error No bt Map Version !, File Map Data Len %d not over 1024.\n", data_len);
+			goto exit;
+		}
+	}
+exit:
+	if (mapbuf)
+		_os_mem_free(d, (void*)mapbuf, bt_map_sz);
+
+	return hal_status;
+}
+
+
+enum rtw_hal_status
+rtw_efuse_bt_file_mask_load(void *efuse, char *file_path)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+	void *d = efuse_info->hal_com->drv_priv;
+	u8 *maskbuf = NULL;
+	u16 data_len = 0;
+	u32 bt_mask_sz = 0;
+
+	bt_mask_sz = efuse_info->bt_mask_size;
+
+	if (bt_mask_sz == 0)
+		goto exit;
+
+	if (file_path) {
+		maskbuf= _os_mem_alloc(d, bt_mask_sz);
+		if (maskbuf == NULL) {
+			PHL_INFO("%s, alloc buf FAIL!!\n", __FUNCTION__);
+			goto exit;
+		}
+		_os_mem_set(d, maskbuf, 0xFF, bt_mask_sz);
+
+		data_len = efuse_file_open(d, file_path, maskbuf, bt_mask_sz);
+
+		if (data_len <= bt_mask_sz && data_len > 0) {
+			PHL_INFO("Mask File data 2 buf len %d\n", data_len);
+			hal_status = efuse_bt_file_mask2buf(efuse_info, maskbuf, data_len);
+
+		} else {
+			PHL_INFO("Error No Map Version !, File Map Data Len %d not over 96.\n", data_len);
+			goto exit;
+		}
+	}
+exit:
+	if (maskbuf)
+		_os_mem_free(d, (void*)maskbuf, bt_mask_sz);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_get_logical_size(void *efuse, u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_get_log_efuse_bt_size(efuse_info->hal_com, size);
+
+	PHL_INFO("%s: size = %d\n", __FUNCTION__, *size);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_get_size(void *efuse, u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_get_efuse_bt_size(efuse_info->hal_com, size);
+
+	PHL_INFO("%s: size = %d\n", __FUNCTION__, *size);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_get_avl(void *efuse, u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_get_efuse_bt_avl(efuse_info->hal_com, size);
+
+	PHL_INFO("%s: size = %d\n", __FUNCTION__, *size);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_get_offset_mask(void *efuse, u16 offset, u8 *mask)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	if(TEST_STATUS_FLAG(efuse_info->status, EFUSE_STATUS_PROCESS) == false)
+		return RTW_HAL_STATUS_EFUSE_UNINIT;
+
+	if(offset >= efuse_info->bt_mask_size)
+		return RTW_HAL_STATUS_EFUSE_IVALID_OFFSET;
+
+	*mask = efuse_info->bt_mask[offset];
+	PHL_INFO("%s: bt offset = %x mask = %x\n", __FUNCTION__, offset, *mask);
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_get_mask_buf(void *efuse, u8 *destbuf, u32 *buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	if (efuse_info->bt_mask_size > 0) {
+		_os_mem_cpy(efuse_info->hal_com->drv_priv, (void *)destbuf,
+				(void *)efuse_info->bt_mask , efuse_info->bt_mask_size);
+		*buflen = efuse_info->bt_mask_size;
+
+		PHL_INFO("%s: size = %d\n", __FUNCTION__, efuse_info->bt_mask_size);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_read_hidden(void *efuse, u32 addr, u32 size, u8 *val)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_read_efuse_bt_hidden(efuse_info->hal_com, addr, size, val);
+
+	if(status == RTW_HAL_STATUS_SUCCESS)
+	{
+		PHL_INFO("%s: bt hidden read ok!\n", __FUNCTION__);
+	}else{
+		PHL_INFO("%s: bt hidden read fail!\n", __FUNCTION__);
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_bt_write_hidden(void *efuse, u32 addr, u8 val)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	status = rtw_hal_mac_write_efuse_bt_hidden(efuse_info->hal_com, addr, val);
+
+	if(status == RTW_HAL_STATUS_SUCCESS)
+	{
+		PHL_INFO("%s: bt hidden PG ok!\n", __FUNCTION__);
+	}else{
+		PHL_INFO("%s: bt hidden PG fail!\n", __FUNCTION__);
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_read_phy_wifi(void *efuse, u32 addr, u32 size, u8 *data, u8 type)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct efuse_t *efuse_info = efuse;
+
+	if(type == HAL_MP_EFUSE_WIFI)
+		status = rtw_hal_mac_read_phy_efuse(efuse_info->hal_com, addr, size, data);
+	else if(type == HAL_MP_EFUSE_BT)
+		status = rtw_hal_mac_read_bt_phy_efuse(efuse_info->hal_com, addr, size, data);
+
+	if(status == RTW_HAL_STATUS_SUCCESS)
+	{
+		PHL_INFO("%s: real raw read ok!\n", __FUNCTION__);
+	}else{
+		PHL_INFO("%s: real raw read fail!\n", __FUNCTION__);
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_efuse_renew(void *efuse, u8 type)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct efuse_t *efuse_info = efuse;
+
+	if (type == HAL_MP_EFUSE_WIFI) {
+		rtw_hal_bb_get_efuse_init(efuse_info->hal_com);
+		rtw_hal_rf_get_efuse_ex(efuse_info->hal_com, HW_PHY_0);
+		PHL_INFO("%s: hal efuse renew done\n", __FUNCTION__);
+
+	} else if (type == HAL_MP_EFUSE_BT) {
+		PHL_INFO("%s: Not ready\n", __FUNCTION__);
+	}
+
+	return status;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse.h
new file mode 100644
index 000000000000..95155e56d5a7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse.h
@@ -0,0 +1,84 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_EFUSE_H_
+#define _HAL_EFUSE_H_
+
+#define MAX_EFUSE_FILE_VERSION_LENGTH 6
+#define EFUSE_MASK_FILE_LEN 96
+#define MAX_EFUSE_MAP_LEN 1536
+
+enum EFUSE_STATUS_FLAGS {
+	EFUSE_STATUS_PROCESS = BIT0,
+	EFUSE_STATUS_MAP_FILE_LOADED = BIT1,
+	EFUSE_STATUS_MASK_FILE_LOADED = BIT2,
+	EFUSE_STATUS_BT_MAP_FILE_LOADED = BIT3,
+	EFUSE_STATUS_BT_MASK_FILE_LOADED = BIT4
+};
+
+enum EFUSE_SHADOW_MAP_STATUS {
+	DEFAULT_MAP = 0,
+	HW_LOG_MAP = 1,
+	FILE_MAP = 2,
+	EFUSE_UNKNOWN,
+};
+
+#define SHADOWMAP_FROM2STR(status)\
+(status == DEFAULT_MAP) ? "DEFAULT" :\
+(status == HW_LOG_MAP) ? "HW_LOG_EFUSE" :\
+(status == FILE_MAP) ? "FILE_EFUSE" :\
+"UNknow"
+
+/*
+ *	@phl_com
+ *	@hal_com
+ *	@shadow_map: Buffer pointer for limited size logical map
+ *	@mask: Buffer pointer for limited size mask read from mask file
+ *	@map_version: Buffer pointer for map version read from map file
+ *	@mask_version: Buffer pointer for mask version read from mask file
+ *	@log_efuse_size: Limited logical map size
+ *	@mask_size: Limited mask size
+ *	@version_len: Length of verion field in map/mask
+ *	@status: Efuse status
+ *	@is_map_valid: Flag to check autoload status
+ *	@reserved
+ */
+struct efuse_t {
+	struct rtw_phl_com_t *phl_com;
+	struct rtw_hal_com_t *hal_com;
+	u8 *shadow_map;
+	u8 *mask;
+	u8 *map_version;
+	u8 *mask_version;
+	u32 log_efuse_size;
+	u32 mask_size;
+	u32 limit_efuse_size;
+	u32 limit_mask_size;
+	u8 efuse_a_die_size;
+	u32 hci_to_a_die_offset;
+	u32 a_die_start_offset;
+	u8 version_len;
+	u8 status;
+	u8 is_map_valid;
+	u8 reserved;
+	u8 map_from_status;
+	/* BT*/
+	u8 *bt_shadow_map;
+	u8 *bt_mask;
+	u32 bt_log_efuse_size;
+	u32 bt_efuse_size;
+	u32 bt_mask_size;
+};
+
+#endif /* _HAL_EFUSE_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse_export.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse_export.h
new file mode 100644
index 000000000000..569f4b1389d1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/efuse/hal_efuse_export.h
@@ -0,0 +1,147 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_EFUSE_EXPORT_H_
+#define _HAL_EFUSE_EXPORT_H_
+
+enum HAL_EFUSE_CMD_ID {
+	HAL_EFUSE_HELP,
+	HAL_EFUSE_WIFI_DUMP_MAP,
+	HAL_EFUSE_WIFI_LOGIC_DUMP_MAP
+};
+
+static const struct hal_cmd_info hal_efuse_cmd_i[] = {
+	{"-h", HAL_EFUSE_HELP},
+	{"dump_map", HAL_EFUSE_WIFI_DUMP_MAP},
+	{"dump_hw_map", HAL_EFUSE_WIFI_LOGIC_DUMP_MAP},
+};
+
+/* efuse exported API */
+u32
+rtw_efuse_init(struct rtw_phl_com_t *phl_com, struct rtw_hal_com_t *hal_com,
+			   void **efuse);
+
+void rtw_efuse_deinit(struct rtw_hal_com_t *hal_com, void *efuse);
+
+enum rtw_hal_status
+rtw_efuse_get_info(void *efuse, enum rtw_efuse_info info_type, void *value,
+				   u8 size);
+
+void rtw_efuse_process(void *efuse, char *ic_name);
+
+bool rtw_efuse_is_processed(void *efuse);
+
+enum rtw_hal_status
+rtw_efuse_logicmap_buf_load(void *efuse, u8* buf, bool is_limit);
+
+enum rtw_hal_status
+rtw_efuse_shadow_load(void *efuse, bool is_limit);
+
+enum rtw_hal_status
+rtw_efuse_shadow_read(void *efuse, u8 byte_count, u16 offset, u32 *value,
+					  bool is_limit);
+
+enum rtw_hal_status
+rtw_efuse_shadow_write(void *efuse, u8 byte_count, u16 offset, u32 value,
+					   bool is_limit);
+
+enum rtw_hal_status
+rtw_efuse_shadow_update(void *efuse, bool is_limit);
+
+enum rtw_hal_status
+rtw_efuse_shadow2buf(void *efuse, u8 *destbuf, u16 buflen);
+
+enum rtw_hal_status
+rtw_efuse_file_map_load(void *efuse, char *file_path, u8 is_limit);
+
+enum rtw_hal_status
+rtw_efuse_file_mask_load(void *efuse, char *file_path, u8 is_limit);
+
+enum rtw_hal_status
+rtw_efuse_get_usage(void *efuse, u32 *usage);
+
+enum rtw_hal_status
+rtw_efuse_get_logical_size(void *efuse, u32 *size, bool is_limited);
+
+enum rtw_hal_status
+rtw_efuse_get_size(void *efuse, u32 *size);
+
+enum rtw_hal_status
+rtw_efuse_get_avl(void *efuse, u32 *size);
+
+enum rtw_hal_status
+rtw_efuse_get_shadowmap_from(void *efuse, u8 *val);
+
+char*
+rtw_efuse_get_shadowmap_from_to_str(void *efuse);
+
+enum rtw_hal_status
+rtw_efuse_get_offset_mask(void *efuse, u16 offset, u8 *mask);
+
+enum rtw_hal_status
+rtw_efuse_get_mask_buf(void *efuse, u8 *destbuf, u32 *buflen);
+
+/* BT EFUSE */
+enum rtw_hal_status
+rtw_efuse_bt_shadow_load(void *efuse);
+
+enum rtw_hal_status
+rtw_efuse_bt_shadow_read(void *efuse, u8 byte_count, u16 offset, u32 *value);
+
+enum rtw_hal_status
+rtw_efuse_bt_shadow_write(void *efuse, u8 byte_count, u16 offset, u32 value);
+
+enum rtw_hal_status
+rtw_efuse_bt_shadow_update(void *efuse);
+
+enum rtw_hal_status
+rtw_efuse_bt_shadow2buf(void *efuse, u8 *destbuf, u16 buflen);
+
+enum rtw_hal_status
+rtw_efuse_bt_file_map_load(void *efuse, char *file_path);
+
+enum rtw_hal_status
+rtw_efuse_bt_file_mask_load(void *efuse, char *file_path);
+
+enum rtw_hal_status
+rtw_efuse_bt_get_usage(void *efuse, u32 *usage);
+
+enum rtw_hal_status
+rtw_efuse_bt_get_logical_size(void *efuse, u32 *size);
+
+enum rtw_hal_status
+rtw_efuse_bt_get_size(void *efuse, u32 *size);
+
+enum rtw_hal_status
+rtw_efuse_bt_get_avl(void *efuse, u32 *size);
+
+enum rtw_hal_status
+rtw_efuse_bt_get_offset_mask(void *efuse, u16 offset, u8 *mask);
+
+enum rtw_hal_status
+rtw_efuse_bt_get_mask_buf(void *efuse, u8 *destbuf, u32 *buflen);
+
+enum rtw_hal_status
+rtw_efuse_bt_read_hidden(void *efuse, u32 addr, u32 size, u8 *val);
+
+enum rtw_hal_status
+rtw_efuse_bt_write_hidden(void *efuse, u32 addr, u8 val);
+
+enum rtw_hal_status
+rtw_efuse_read_phy_wifi(void *efuse, u32 addr, u32 size, u8 *data, u8 type);
+
+enum rtw_hal_status
+rtw_efuse_renew(void *efuse, u8 type);
+
+#endif /* _HAL_EFUSE_EXPORT_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal.mk b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal.mk
new file mode 100755
index 000000000000..8660c88de635
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal.mk
@@ -0,0 +1,118 @@
+########### HAL PATH  #################################
+
+ifeq ($(CONFIG_PHL_ARCH), y)
+phl_path := phl/
+phl_path_d1 := $(src)/phl/$(HAL)
+else
+phl_path :=
+phl_path_d1 := $(src)/$(HAL)
+endif
+
+_HAL_FILES :=	$(phl_path)$(HAL)/hal_api_mac.o \
+			$(phl_path)$(HAL)/hal_api_bb.o \
+			$(phl_path)$(HAL)/hal_api_rf.o \
+			$(phl_path)$(HAL)/hal_api_btc.o \
+			$(phl_path)$(HAL)/hal_api_efuse.o \
+			$(phl_path)$(HAL)/hal_com_i.o \
+			$(phl_path)$(HAL)/hal_init.o \
+			$(phl_path)$(HAL)/hal_io.o \
+			$(phl_path)$(HAL)/hal_rx.o \
+			$(phl_path)$(HAL)/hal_tx.o \
+			$(phl_path)$(HAL)/hal_sta.o \
+			$(phl_path)$(HAL)/hal_cam.o \
+			$(phl_path)$(HAL)/hal_csi_buffer.o \
+			$(phl_path)$(HAL)/hal_beamform.o \
+			$(phl_path)$(HAL)/hal_sound.o \
+			$(phl_path)$(HAL)/hal_chan.o \
+			$(phl_path)$(HAL)/hal_str_proc.o \
+			$(phl_path)$(HAL)/hal_fw.o \
+			$(phl_path)$(HAL)/hal_cap.o \
+			$(phl_path)$(HAL)/hal_ser.o \
+			$(phl_path)$(HAL)/hal_ps.o \
+			$(phl_path)$(HAL)/hal_c2h.o \
+			$(phl_path)$(HAL)/hal_dbcc.o \
+			$(phl_path)$(HAL)/hal_chan_info.o \
+			$(phl_path)$(HAL)/hal_wow.o \
+			$(phl_path)$(HAL)/hal_ld_file.o \
+			$(phl_path)$(HAL)/hal_regulation.o \
+			$(phl_path)$(HAL)/hal_led.o \
+			$(phl_path)$(HAL)/hal_trx_mit.o \
+			$(phl_path)$(HAL)/hal_acs.o \
+			$(phl_path)$(HAL)/hal_mcc.o \
+			$(phl_path)$(HAL)/hal_api.o \
+			$(phl_path)$(HAL)/hal_twt.o \
+			$(phl_path)$(HAL)/hal_notify.o \
+			$(phl_path)$(HAL)/hal_p2pps.o \
+			$(phl_path)$(HAL)/hal_thermal.o \
+			$(phl_path)$(HAL)/hal_txpwr.o \
+			$(phl_path)$(HAL)/hal_ext_tx_pwr_lmt.o \
+			$(phl_path)$(HAL)/hal_dfs.o \
+			$(phl_path)$(HAL)/hal_scanofld.o
+
+ifeq ($(CONFIG_PCI_HCI), y)
+_HAL_FILES += $(phl_path)$(HAL)/hal_pci.o
+endif
+ifeq ($(CONFIG_USB_HCI), y)
+_HAL_FILES += $(phl_path)$(HAL)/hal_usb.o
+endif
+ifeq ($(CONFIG_SDIO_HCI), y)
+_HAL_FILES += $(phl_path)$(HAL)/hal_sdio.o
+endif
+########### hal test #################################
+ifeq ($(CONFIG_PHL_TEST_SUITE), y)
+_HAL_FILES += $(phl_path)$(HAL)/test/hal_test_module.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp_cal.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp_config.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp_efuse.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp_reg.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp_rx.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp_tx.o
+_HAL_FILES += $(phl_path)$(HAL)/test/mp/hal_test_mp_txpwr.o
+endif
+
+_HAL_FILES += $(phl_path)$(HAL)/efuse/hal_efuse.o
+include $(phl_path_d1)/mac/mac.mk
+
+ifeq ($(USE_TRUE_PHY), y)
+include $(phl_path_d1)/phy/bb/bb.mk
+include $(phl_path_d1)/phy/rf/rf.mk
+endif
+
+########### HALBTC #######################################
+include $(phl_path_d1)/btc/btc.mk
+########### HAL_RTL8852A #################################
+ifeq ($(CONFIG_RTL8852A), y)
+include $(phl_path_d1)/$(IC_NAME)/rtl8852a.mk
+endif
+########### HAL_RTL8852B #################################
+ifneq ($(filter y,$(CONFIG_RTL8852B) $(CONFIG_RTL8852BP) $(CONFIG_RTL8852BPT) $(CONFIG_RTL8852BT)),)
+include $(phl_path_d1)/rtl8852b/rtl8852b.mk
+endif
+########### HAL_RTL8852C #################################
+ifeq ($(CONFIG_RTL8852C), y)
+include $(phl_path_d1)/rtl8852c/rtl8852c.mk
+endif
+########### HAL_RTL8842A #################################
+ifeq ($(CONFIG_RTL8842A), y)
+include $(phl_path_d1)/rtl8842a/rtl8842a.mk
+endif
+########### HAL_RTL8852D #################################
+ifeq ($(CONFIG_RTL8852D), y)
+include $(phl_path_d1)/rtl8852d/rtl8852d.mk
+endif
+########### HAL_RTL8192XB #################################
+ifneq ($(filter y,$(CONFIG_RTL8832BR) $(CONFIG_RTL8192XB)),)
+include $(phl_path_d1)/rtl8192xb/rtl8192xb.mk
+endif
+########### HAL_RTL8982A #################################
+ifeq ($(CONFIG_RTL8982A), y)
+include $(phl_path_d1)/$(IC_NAME)/rtl8982a.mk
+endif
+########### HAL_RTL8851B #################################
+ifeq ($(CONFIG_RTL8851B), y)
+include $(phl_path_d1)/$(IC_NAME)/rtl8851b.mk
+endif
+
+OBJS += $(_HAL_FILES) $(_HAL_MAC_FILES) $(_BTC_FILES) $(_HAL_IC_FILES)
+OBJS += $(_HAL_BB_FILES) $(_HAL_RF_FILES) $(_HAL_EFUSE_FILES)
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_acs.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_acs.c
new file mode 100644
index 000000000000..50d0abf22d0b
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_acs.c
@@ -0,0 +1,33 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_ACS_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_RTW_ACS
+void rtw_hal_acs_mntr_trigger(void *hal, enum phl_band_idx band_idx, struct acs_mntr_parm *parm)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_bb_acs_mntr_trigger(hal_info, band_idx, parm);
+}
+
+enum rtw_hal_status rtw_hal_acs_mntr_result(void *hal,
+					enum phl_band_idx band_idx, struct acs_mntr_rpt *rpt)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_bb_acs_mntr_result(hal_info, band_idx, rpt);
+}
+#endif /* CONFIG_RTW_ACS */
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_acs.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_acs.h
new file mode 100644
index 000000000000..a80ecb4f24dc
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_acs.h
@@ -0,0 +1,22 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_ACS_H_
+#define _HAL_ACS_H_
+
+void rtw_hal_acs_mntr_trigger(void *hal, enum phl_band_idx band_idx, struct acs_mntr_parm *parm);
+enum rtw_hal_status rtw_hal_acs_mntr_result(void *hal,
+					enum phl_band_idx band_idx, struct acs_mntr_rpt *rpt);
+
+#endif
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api.c
new file mode 100644
index 000000000000..624be2ca2c48
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api.c
@@ -0,0 +1,721 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_API_C_
+#include "hal_headers.h"
+
+enum rtw_hal_status rtw_hal_get_tsf(void *hal, enum phl_band_idx band,
+					u8 port, u32 *tsf_h, u32 *tsf_l)
+{
+	return rtw_hal_mac_get_tsf(hal, band, port, tsf_h, tsf_l);
+}
+
+enum rtw_hal_status
+rtw_hal_tsf_sync(void *hal, u8 port_sync_from,
+					u8 port_sync_to, enum phl_band_idx band,
+					s32 sync_offset_tu, enum hal_tsf_sync_act act)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_tsf_sync(hal_info, port_sync_from,
+						port_sync_to, band, sync_offset_tu, act);
+}
+
+enum rtw_hal_status
+rtw_hal_config_rts_th(void *hal, u8 band_idx, u16 rts_time_th, u16 rts_len_th)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_set_hw_rts_th(hal_info, band_idx, rts_time_th, rts_len_th);
+}
+
+u32 rtw_hal_get_btc_req_slot(void *hal, enum phl_band_idx hw_band)
+{
+	return rtw_hal_btc_req_bt_slot_t(hal, hw_band);
+}
+
+/**
+ * This function is used to pause/unpause multiple macid
+ * @hal: see hal_info_t
+ * @macid_arr: macid array to be pause/unpause
+ * 1 means to be set (pause/unpause)
+ * 0 means don't care
+ * @macid_arr_sz: size of macid array
+ * @pause: all macid of this array to be paused/unpaused
+ *  1=paused,0=unpaused
+ */
+enum rtw_hal_status
+rtw_hal_set_macid_grp_pause(void *hal, u32 *macid_arr, u8 arr_size, bool pause)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return rtw_hal_mac_set_macid_grp_pause(hal_com, macid_arr, arr_size, pause);
+}
+
+/**
+ * This function is used to pause/unpause single macid
+ * @hal: see hal_info_t
+ * @macid: macid be pause/unpause
+ * @pause: all macid of this array to be paused/unpaused
+ *  1=paused,0=unpaused
+ */
+enum rtw_hal_status
+rtw_hal_set_macid_pause(void *hal, u16 macid, bool pause)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return rtw_hal_mac_set_macid_pause(hal_com, macid, pause);
+}
+
+/**
+ * This function is used to pause/unpause AC queue of a single macid
+ * @hal: see hal_info_t
+ * @macid: macid be pause/unpause
+ * @pause: paused/unpaused (1=paused,0=unpaused)
+ */
+enum rtw_hal_status rtw_hal_set_macid_pause_ac(void *hal, u16 macid, bool pause)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return rtw_hal_mac_set_macid_pause_sleep(hal_com, macid, pause, false);
+}
+
+enum rtw_hal_status
+rtw_hal_set_macid_pkt_drop(void *hal, u16 macid, u8 sel, u8 band, u8 port,
+                           u8 mbssid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return rtw_hal_mac_set_macid_pkt_drop(hal_com, macid, sel, band, port,
+	                                      mbssid);
+}
+
+enum rtw_hal_status rtw_hal_set_dfs_tb_ctrl(void *hal, u8 set)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_set_dfs_tb_ctrl(hal_info, set);
+}
+
+u32 rtw_hal_get_phy_stat_info(void *hal, enum phl_band_idx hw_band,
+			      enum phl_stat_info_query phy_stat)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_hal_stat_info *stat_info = &hal_com->band[hw_band].stat_info;
+
+	switch (phy_stat) {
+	case STAT_INFO_FA_ALL:
+		return stat_info->cnt_fail_all;
+	case STAT_INFO_CCA_ALL:
+		return stat_info->cnt_cca_all;
+	default:
+		return 0;
+	}
+}
+
+enum rtw_hal_status rtw_hal_get_rx_cnt_by_idx(void *hal,
+					      enum phl_band_idx hw_band,
+					      enum phl_rxcnt_idx idx,
+					      u16 *rx_cnt)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_get_rx_cnt_by_idx(hal_info, hw_band, idx, rx_cnt);
+}
+
+enum rtw_hal_status rtw_hal_set_reset_rx_cnt(void *hal,
+					     enum phl_band_idx hw_band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_set_reset_rx_cnt(hal_info, hw_band);
+}
+
+enum rtw_hal_status
+rtw_hal_set_dctrl_tbl_seq(void *hal,
+		struct rtw_phl_stainfo_t *sta, u32 seq)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_dctl_info dctrl = {0}, dctrl_mask = {0};
+
+	if (NULL == sta)
+		goto out;
+
+	dctrl.seq0 = seq;
+	dctrl.seq1 = seq;
+	dctrl.seq2 = seq;
+	dctrl.seq3 = seq;
+
+	dctrl_mask.seq0 = 0x0FFF;
+	dctrl_mask.seq1 = 0x0FFF;
+	dctrl_mask.seq2 = 0x0FFF;
+	dctrl_mask.seq3 = 0x0FFF;
+
+	sts = rtw_hal_dmc_tbl_cfg(hal_info, &dctrl ,&dctrl_mask, sta->macid);
+
+out:
+	return sts;
+}
+
+#ifdef CONFIG_PHL_DRV_HAS_NVM
+enum rtw_hal_status
+rtw_hal_extract_efuse_info(void *hal, u8 *efuse_map,
+                           enum rtw_efuse_info info_type,
+                           void *value,
+                           u8 size, u8 map_valid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_phl_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (info_type <= EFUSE_INFO_MAC_MAX)
+		hal_status = rtw_hal_mac_get_efuse_info(hal_info->hal_com,
+							efuse_map,
+							info_type,
+							value,
+							size,
+							1);
+	else if (info_type <= EFUSE_INFO_BB_MAX)
+		hal_status = rtw_hal_bb_get_efuse_info(hal_info->hal_com,
+		                                       efuse_map,
+		                                       info_type,
+		                                       value,
+		                                       size,
+		                                       1);
+	else if (info_type <= EFUSE_INFO_RF_MAX)
+		hal_status = rtw_hal_rf_get_efuse_info(hal_info->hal_com,
+		                                       efuse_map,
+		                                       info_type,
+		                                       value,
+		                                       size,
+		                                       1);
+	else if (info_type <= EFUSE_INFO_BTCOEX_MAX)
+		hal_status = rtw_hal_btc_get_efuse_info(hal_info->hal_com,
+							efuse_map,
+							info_type,
+							value,
+							size,
+							1);
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_get_efuse_size(void *hal, u32 *log_efuse_size, u32 *limit_efuse_size,
+                       u32 *mask_size, u32 *limit_mask_size)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	if (log_efuse_size)
+		hal_status = rtw_hal_mac_get_log_efuse_size(hal_info->hal_com,
+		                                            log_efuse_size,
+		                                            false);
+	if (limit_efuse_size && (hal_status == RTW_HAL_STATUS_SUCCESS))
+		hal_status = rtw_hal_mac_get_log_efuse_size(hal_info->hal_com,
+		                                            limit_efuse_size,
+		                                            true);
+	if (mask_size && (hal_status == RTW_HAL_STATUS_SUCCESS))
+		hal_status = rtw_hal_mac_get_efuse_mask_size(hal_info->hal_com,
+		                                             mask_size,
+		                                             false);
+	if (limit_mask_size  && (hal_status == RTW_HAL_STATUS_SUCCESS))
+		hal_status = rtw_hal_mac_get_efuse_mask_size(hal_info->hal_com,
+		                                             limit_mask_size,
+		                                             true);
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_nvm_apply_dev_cap(void *hal, struct rtw_phl_com_t *phl_com)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_sts;
+
+	hal_com->dev_hw_cap.pkg_type = phl_com->dev_sw_cap.pkg_type;
+	hal_com->dev_hw_cap.rfe_type = phl_com->dev_sw_cap.rfe_type;
+	hal_com->dev_hw_cap.xcap = phl_com->dev_sw_cap.xcap;
+	hal_com->dev_hw_cap.domain = phl_com->dev_sw_cap.domain;
+	hal_com->dev_hw_cap.domain_6g = phl_com->dev_sw_cap.domain_6g;
+
+	hal_sts = rtw_hal_mac_set_xcap(hal_com, 0, hal_com->dev_hw_cap.xcap);
+
+	if (hal_sts != RTW_HAL_STATUS_SUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	hal_sts = rtw_hal_mac_set_xcap(hal_com, 1, hal_com->dev_hw_cap.xcap);
+
+	#ifdef CONFIG_RTW_MULTI_DEV_MULTI_BAND
+	if (hal_sts == RTW_HAL_STATUS_SUCCESS) {
+		struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+		if (hal_ops->cfg_share_xstal != NULL) {
+			#ifdef CONFIG_SHARE_XSTAL
+			hal_sts = hal_ops->cfg_share_xstal(hal_info, phl_com,
+							    true);
+			#else
+			hal_sts = hal_ops->cfg_share_xstal(hal_info, phl_com,
+							    false);
+			#endif /* CONFIG_SHARE_XSTAL */
+		}
+	}
+	#endif /* CONFIG_RTW_MULTI_DEV_MULTI_BAND */
+
+	return hal_sts;
+}
+
+enum rtw_hal_status
+rtw_hal_flash_get_info(struct rtw_hal_com_t *hal_com,
+		       enum rtw_efuse_info info_type,
+		       void *value,
+		       u8 size)
+{
+	u32 ret = _os_nvm_get_info(hal_com->drv_priv, info_type, value, size);
+
+	return (ret == _SUCCESS) ? RTW_HAL_STATUS_SUCCESS : RTW_HAL_STATUS_FAILURE;
+}
+
+#endif /* CONFIG_PHL_DRV_HAS_NVM */
+
+enum rtw_hal_status
+rtw_hal_set_fw_ul_fixinfo(void *hal,
+			  struct rtw_phl_ax_ul_fixinfo *ul_fixinfo)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	hstatus = rtw_hal_mac_set_fw_ul_fixinfo(hal, ul_fixinfo);
+
+	return hstatus;
+}
+
+u16 rtw_hal_get_ampdu_num(void *hal, u8 band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_query_ampdu_num(hal_info, band);
+}
+
+enum rtw_chip_id rtw_hal_get_chip_id(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return hal_get_chip_id(hal_info->hal_com);
+}
+
+enum rtw_hal_status rtw_hal_set_append_fcs(void *hal, u8 enable)
+{
+	return rtw_hal_mac_set_append_fcs((struct hal_info_t *)hal, enable);
+}
+
+enum rtw_hal_status
+rtw_hal_pwr_switch_mac(void *hal, bool on)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	hstatus = rtw_hal_mac_pwr_switch(hal, on);
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_get_freerun_counter(void *hal, u8 band, u32 *freerun_h, u32 *freerun_l)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	status = rtw_hal_mac_get_freerun_cnt(hal_info->hal_com, band, freerun_h, freerun_l);
+
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_get_freerun_counter(): get failed\n");
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_reset_freerun_counter(void *hal, u8 band)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	status = rtw_hal_mac_reset_freerun_cnt(hal_info->hal_com, band);
+
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_reset_freerun_counter(): get failed\n");
+	}
+
+	return status;
+}
+
+u8 rtw_hal_query_group_cntry_num(void *hal, struct rtw_regu_policy *policy,
+	u8 group_id)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal || !policy)
+		return 0;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return 0;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_query_group_cntry_num)
+		return regu_ops->hal_query_group_cntry_num(policy, group_id);
+
+	return 0;
+}
+
+void rtw_hal_fill_group_cntry_list(void *hal,
+	struct rtw_regu_policy *policy, char *list,
+	u32 group_size, u8 group_id)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal || !policy)
+		return;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_fill_group_cntry_list)
+		regu_ops->hal_fill_group_cntry_list(policy, list,
+		group_size, group_id);
+}
+
+u8 rtw_hal_get_cntry_idx(void *hal, char *cntry)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return INVALID_CNTRY_IDX;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return INVALID_CNTRY_IDX;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_cntry_idx)
+		return regu_ops->hal_get_cntry_idx(cntry);
+
+	return INVALID_CNTRY_IDX;
+}
+
+u8 rtw_hal_get_cntry_tbl_size(void *hal)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return 0;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return 0;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_cntry_tbl_size)
+		return regu_ops->hal_get_cntry_tbl_size();
+
+	return 0;
+}
+
+void rtw_hal_qry_cntry_chnlplan(
+	void *hal, struct rtw_regulation_country_chplan *chplan,
+	char *country)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_qry_cntry_chnlplan)
+		regu_ops->hal_qry_cntry_chnlplan(chplan, country);
+}
+
+void rtw_hal_get_chplan_update_info(
+	void *hal, u8 group, u8 did,
+	void *info, enum band_type band)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_chplan_update_info)
+		regu_ops->hal_get_chplan_update_info(group, did, info, band);
+}
+
+u8 rtw_hal_get_chnlplan_ver(void *hal)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return INVALID_VER;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return INVALID_VER;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_chnlplan_ver)
+		return regu_ops->hal_get_chnlplan_ver();
+
+	return INVALID_VER;
+}
+
+u8 rtw_hal_get_country_ver(void *hal)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return INVALID_VER;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return INVALID_VER;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_country_ver)
+		return regu_ops->hal_get_country_ver();
+
+	return INVALID_VER;
+}
+
+void rtw_hal_get_chdef_6g(void * hal, u8 ch_idx,
+	struct chdef_6ghz *chdef)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_chdef_6g)
+		regu_ops->hal_get_chdef_6g(ch_idx, chdef);
+}
+
+u8 rtw_hal_get_domain_regulation(void *hal,
+	u8 domain, u8 band)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return REGULATION_MAX;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return REGULATION_MAX;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_domain_regulation)
+		return regu_ops->hal_get_domain_regulation(domain, band);
+
+	return REGULATION_MAX;
+}
+
+u8 rtw_hal_get_domain_idx(void *hal,
+	u8 domain, bool is_6g)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return INVALID_DOMAIN_INDEX;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return INVALID_DOMAIN_INDEX;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_domain_idx)
+		return regu_ops->hal_get_domain_idx(domain, is_6g);
+
+	return INVALID_DOMAIN_INDEX;
+}
+
+u8 rtw_hal_get_cat6g_by_country(
+	void *hal, char * country)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return 0;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return 0;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_cat6g_by_country)
+		return regu_ops->hal_get_cat6g_by_country(country);
+
+	return 0;
+}
+
+void rtw_hal_get_6g_regulatory_info(void *hal, u8 domain,
+	u8 *dm_code, u8 *regulation, u8 *ch_idx)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!hal)
+		return;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_6g_regulatory_info)
+		regu_ops->hal_get_6g_regulatory_info(domain,
+			dm_code, regulation, ch_idx);
+}
+
+bool rtw_hal_get_regu_func_cert_info(void *hal, char *country,
+	struct rtw_regu_func_cert_info *rg_cert)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct hal_ops_t *hal_ops = NULL;
+	struct hal_regu_ops *regu_ops = NULL;
+
+	if (!rg_cert)
+		return false;
+
+	/* Init rg_cert->valid as false */
+	rg_cert->valid = false;
+
+	if (!hal || !country)
+		return false;
+
+	hal_info = (struct hal_info_t *)hal;
+	hal_ops = hal_get_ops(hal_info);
+	if (!hal_ops)
+		return false;
+
+	regu_ops = hal_get_regu_ops(hal_ops);
+	if (regu_ops && regu_ops->hal_get_regu_func_cert_info) {
+		regu_ops->hal_get_regu_func_cert_info(country, rg_cert);
+		return rg_cert->valid;
+	} else {
+		return false;
+	}
+}
+
+#ifdef CONFIG_PHL_BCN_ERLY_RPT
+enum rtw_hal_status rtw_hal_set_bcn_early_rpt(void *hal, u8 band, u8 port, u8 en)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	hstatus = rtw_hal_mac_cfg_bcn_early_rpt(hal, band, port, en);
+
+	return hstatus;
+}
+#endif
+
+#ifdef CONFIG_PHL_DIAGNOSE
+enum rtw_hal_status rtw_hal_get_pmac_info(void *hal, u8 hw_band,
+		u8 *dbg_info, u32 len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum phl_phy_idx p_idx = HW_PHY_0;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	p_idx = rtw_hal_hw_band_to_phy_idx(hw_band);
+	hstatus = rtw_hal_bb_query_pmac_info(hal_info, p_idx, dbg_info, len);
+	PHL_INFO(
+	    "%s\n, hstatus(%d)\n", __func__, hstatus);
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_get_utility_info(void *hal, u8 hw_band,
+			u8 *dbg_info, u32 len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum phl_phy_idx p_idx = HW_PHY_0;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	p_idx = rtw_hal_hw_band_to_phy_idx(hw_band);
+	hstatus = rtw_hal_bb_query_utility_info(hal_info, p_idx, dbg_info, len);
+	PHL_INFO(
+	    "%s\n, hstatus(%d)\n", __func__, hstatus);
+
+	return hstatus;
+}
+#endif
+
+#ifdef CONFIG_PHL_CSUM_OFFLOAD_RX
+enum rtw_hal_status rtw_hal_chk_rx_tcpip_chksum_ofd(void *h,
+						    struct rtw_r_meta_data *mdata,
+                                                    u8 status)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+
+	return rtw_hal_mac_chk_rx_tcpip_chksum_ofd(hal, mdata, status);
+}
+#endif /* CONFIG_PHL_CSUM_OFFLOAD_RX */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api.h
new file mode 100644
index 000000000000..0e839a206b1f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api.h
@@ -0,0 +1,1515 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_H_
+#define _HAL_API_H_
+
+/******************* IO  APIs *******************/
+u8 rtw_hal_read8(void *h, u32 addr);
+u16 rtw_hal_read16(void *h, u32 addr);
+u32 rtw_hal_read32(void *h, u32 addr);
+void rtw_hal_write8(void *h, u32 addr, u8 val);
+void rtw_hal_write16(void *h, u32 addr, u16 val);
+void rtw_hal_write32(void *h, u32 addr, u32 val);
+
+u32 rtw_hal_read_macreg(void *h, u32 offset, u32 bit_mask);
+void rtw_hal_write_macreg(void *h,
+			u32 offset, u32 bit_mask, u32 data);
+u32 rtw_hal_read_bbreg(void *h, u32 offset, u32 bit_mask);
+void rtw_hal_write_bbreg(void *h,
+			u32 offset, u32 bit_mask, u32 data);
+u32 rtw_hal_read_rfreg(void *h,
+			enum rf_path path, u32 offset, u32 bit_mask);
+void rtw_hal_write_rfreg(void *h,
+			enum rf_path path, u32 offset, u32 bit_mask, u32 data);
+
+#ifdef PHL_PLATFORM_LINUX
+void rtw_hal_mac_reg_dump(void *sel, void *h);
+void rtw_hal_bb_reg_dump(void *sel, void *h);
+void rtw_hal_bb_reg_dump_ex(void *sel, void *h);
+void rtw_hal_rf_reg_dump(void *sel, void *h);
+#endif
+
+enum rtw_hal_status rtw_hal_set_resp_ack_chk_cca(void *hal, u8 band, u8 en);
+enum rtw_hal_status rtw_hal_sifs_chk_cca_en(void *hal, u8 band, u8 en);
+
+enum rtw_hal_status
+rtw_hal_get_addr_cam(void *h, u16 num, u8 *buf, u16 size);
+enum rtw_hal_status
+rtw_hal_get_sec_cam(void *h, u16 num, u8 *buf, u16 size);
+
+void rtw_hal_init_int_default_value(struct rtw_phl_com_t *phl_com, void *h, enum rtw_hal_int_set_opt opt);
+void rtw_hal_enable_interrupt(struct rtw_phl_com_t *phl_com, void *h);
+void rtw_hal_disable_interrupt_isr(struct rtw_phl_com_t *phl_com, void *h);
+void rtw_hal_disable_interrupt(struct rtw_phl_com_t *phl_com, void *h);
+void rtw_hal_config_interrupt(void *h, enum rtw_phl_config_int int_mode);
+bool rtw_hal_recognize_interrupt(void *h);
+bool rtw_hal_recognize_halt_c2h_interrupt(void *h);
+void rtw_hal_clear_interrupt(void *h);
+void rtw_hal_restore_interrupt(struct rtw_phl_com_t *phl_com, void *h);
+
+u32 rtw_hal_interrupt_handler(void *h);
+void rtw_hal_restore_rx_interrupt(void *h);
+#ifdef PHL_RXSC_ISR
+u16 rtw_hal_rpq_isr_check(void *hal, u8 dma_ch);
+#endif
+
+enum rtw_hal_status rtw_hal_get_pwr_state(void *h, enum rtw_mac_pwr_st *pwr_state);
+
+enum rtw_hal_status rtw_hal_init(void *drv_priv,
+	struct rtw_phl_com_t *phl_com, void **hal, enum rtl_ic_id chip_id);
+struct rtw_hal_com_t *rtw_hal_get_halcom(void *hal);
+void rtw_hal_deinit(struct rtw_phl_com_t *phl_com, void *hal);
+
+bool rtw_hal_is_inited(struct rtw_phl_com_t *phl_com, void *hal);
+
+u32 rtw_hal_hci_cfg(struct rtw_phl_com_t *phl_com, void *hal,
+					struct rtw_ic_info *ic_info);
+u32 rtw_hal_read_chip_info(struct rtw_phl_com_t *phl_com, void *hal);
+
+void rtw_hal_set_default_var(void *hal);
+
+u32 rtw_hal_var_init(struct rtw_phl_com_t *phl_com, void *hal);
+
+enum rtw_hal_status rtw_hal_ser_ctrl(void *hal, enum rtw_hal_ser_rsn rsn, bool en);
+
+u32
+rtw_hal_ser_get_error_status(void *hal, u32 *err);
+
+enum rtw_hal_status rtw_hal_ser_set_error_status(void *hal, u32 err);
+
+bool rtw_hal_ser_chk_ser_l1(void *hal);
+
+enum rtw_hal_status rtw_hal_trigger_cmac_err(void *hal);
+
+enum rtw_hal_status rtw_hal_trigger_dmac_err(void *hal);
+
+enum rtw_hal_status rtw_hal_lv1_rcvy(void *hal, u32 step);
+
+void
+rtw_hal_dump_fw_rsvd_ple(void *phl);
+void
+rtw_hal_ser_reset_wdt_intr(void *hal);
+
+void rtw_hal_ser_int_cfg(void *hal, struct rtw_phl_com_t *phl_com, enum RTW_PHL_SER_CFG_STEP step);
+
+
+enum rtw_hal_status rtw_hal_en_fw_log_comp(void *hal, u32 comp, bool en);
+enum rtw_hal_status rtw_hal_set_fw_log_lvl(void *hal, u32 lvl);
+void rtw_hal_fw_recover_log_cfg(void *hal);
+
+enum rtw_hal_status
+rtw_hal_download_fw(struct rtw_phl_com_t *phl_com, void *hal);
+
+enum rtw_hal_status
+rtw_hal_redownload_fw(struct rtw_phl_com_t *phl_com, void *hal);
+void rtw_hal_fw_dbg_dump(void *hal);
+
+enum rtw_hal_status
+rtw_hal_pg_redownload_fw(struct rtw_phl_com_t *phl_com, void *hal);
+
+#ifdef CONFIG_HAL_MAC_DBG
+enum rtw_fw_status rtw_hal_get_fw_status(void *h);
+#endif
+
+enum rtw_hal_status rtw_hal_preload(struct rtw_phl_com_t *phl_com, void *hal);
+enum rtw_hal_status rtw_hal_start(struct rtw_phl_com_t *phl_com, void *hal);
+void rtw_hal_stop(struct rtw_phl_com_t *phl_com, void *hal);
+enum rtw_hal_status rtw_hal_restart(struct rtw_phl_com_t *phl_com, void *hal);
+enum rtw_hal_status rtw_hal_hal_deinit(struct rtw_phl_com_t *phl_com, void *hal);
+
+enum rtw_hal_status rtw_hal_role_deinit(void *hal, struct rtw_wifi_role_link_t *rlink);
+
+#ifdef CONFIG_WOWLAN
+enum rtw_hal_status rtw_hal_get_wake_rsn(void *hal, enum rtw_mac_wow_wake_reason *wake_rsn, u8 *reset);
+enum rtw_hal_status rtw_hal_cfg_wow_sleep(void *hal, u8 sleep);
+enum rtw_hal_status rtw_hal_get_wow_aoac_rpt(void *hal, struct rtw_aoac_report *aoac_info, u8 rx_ready);
+enum rtw_hal_status rtw_hal_get_wow_fw_status(void *hal, u8 *status);
+enum rtw_hal_status rtw_hal_wow_cfg_txdma(void *hal, u8 state);
+
+enum rtw_hal_status rtw_hal_wow_init(struct rtw_phl_com_t *phl_com, void *hal, struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status rtw_hal_wow_deinit(struct rtw_phl_com_t *phl_com, void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_wow_func_en(struct rtw_phl_com_t *phl_com, void *hal, u16 macid,
+                    struct rtw_hal_wow_cfg *cfg);
+enum rtw_hal_status
+rtw_hal_wow_func_dis(struct rtw_phl_com_t *phl_com, void *hal, u16 macid,
+                     struct rtw_hal_wow_cfg *cfg);
+
+enum rtw_hal_status rtw_hal_wow_func_start(struct rtw_phl_com_t *phl_com, void *hal, u16 macid, struct rtw_hal_wow_cfg *cfg);
+enum rtw_hal_status rtw_hal_wow_func_stop(struct rtw_phl_com_t *phl_com, void *hal, u16 macid);
+
+enum rtw_hal_status rtw_hal_set_wowlan(struct rtw_phl_com_t *phl_com, void *hal,
+				       enum mac_ax_wow_ctrl ctrl);
+enum rtw_hal_status rtw_hal_wow_drop_tx(void *hal, u8 band);
+enum rtw_hal_status rtw_hal_wow_req_tri_evt(void *hal);
+enum rtw_hal_status rtw_hal_wow_req_diag_rpt(void *hal);
+enum rtw_hal_status
+rtw_hal_wow_cfg_nlo_chnl_list(void *hal, struct rtw_nlo_info *cfg);
+
+enum rtw_hal_status
+rtw_hal_wow_cfg_nlo(void *hal, enum SCAN_OFLD_OP op, u16 mac_id,
+                    u8 hw_band, u8 hw_port, struct rtw_nlo_info *cfg);
+
+enum rtw_hal_status rtw_hal_wow_dbg_dump(void *hal);
+
+#ifdef CONFIG_PHL_WOW_APF
+enum rtw_hal_status
+rtw_hal_wow_access_apf(void *hal, u16 mac_id, struct rtw_apf_info *cfg, u8 set);
+#endif
+
+#endif /* CONFIG_WOWLAN */
+
+enum rtw_hal_status rtw_hal_set_sw_gpio_mode(struct rtw_phl_com_t *phl_com, void *hal, enum rtw_gpio_mode mode, u8 gpio);
+
+enum rtw_hal_status rtw_hal_sw_gpio_ctrl(struct rtw_phl_com_t *phl_com,
+					 void *hal, u8 high, u8 gpio);
+
+int rtw_hal_find_ext_regd_num(struct rtw_para_pwrlmt_info_t *para_info, const char *regd_name);
+char *rtw_hal_get_ext_regd_name(struct rtw_para_pwrlmt_info_t *para_info, u8 idx);
+void
+ rtw_hal_dl_all_para_file(struct rtw_phl_com_t *phl_com, char *ic_name, void *hal);
+void rtw_hal_pwr_byrate_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_info_t *para_info, bool tmp_data_only);
+void rtw_hal_pwrlmt_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_pwrlmt_info_t *para_info, bool tmp_data_only);
+void rtw_hal_general_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_info_t *para_info);
+
+enum rtw_hal_status rtw_hal_trx_init(void *hal, u8 *txbd_buf, u8 *rxbd_buf);
+void rtw_hal_trx_deinit(void *hal);
+
+enum rtw_hal_status
+rtw_hal_role_cfg(void *hal,
+                 struct rtw_wifi_role_t *wrole,
+                 struct rtw_wifi_role_link_t *rlink);
+
+enum rtw_hal_status
+rtw_hal_role_cfg_ex(void *hal,
+                    struct rtw_wifi_role_link_t *rlink,
+                    enum pcfg_type type,
+                    void *param);
+
+enum rtw_hal_status
+rtw_hal_beacon_stop(void *hal,
+                    struct rtw_wifi_role_link_t *rlink,
+                    enum rlink_bcn_stop_rson reason,
+                    bool stop);
+
+enum rtw_hal_status
+rtw_hal_stainfo_init(void *hal, struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_stainfo_deinit(void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_set_dctrl_tbl_seciv(void *hal,
+                 struct rtw_phl_stainfo_t *sta, u64 iv);
+
+enum rtw_hal_status
+rtw_hal_add_sta_entry(void *hal, struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_del_sta_entry(void *hal, struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_restore_sta_entry(struct rtw_phl_com_t* phl_com, void *hal, struct rtw_phl_stainfo_t *sta,
+							bool is_connect);
+enum rtw_hal_status
+rtw_hal_update_sta_entry(struct rtw_phl_com_t* phl_com, void *hal, struct rtw_phl_stainfo_t *sta,
+							bool is_connect);
+enum rtw_hal_status
+rtw_hal_change_sta_entry(void *hal, struct rtw_phl_stainfo_t *sta,
+							enum phl_upd_mode mode);
+void
+rtw_hal_disconnect_drop_all_pkt(void *hal, struct rtw_phl_stainfo_t *sta);
+u8 rtw_hal_get_sta_rssi(struct rtw_phl_stainfo_t *sta);
+u8 rtw_hal_get_sta_rssi_bcn(struct rtw_phl_stainfo_t *sta);
+
+bool rtw_hal_is_sta_linked(void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_set_sta_rx_sts(struct rtw_phl_stainfo_t *sta, u8 rst,
+		       struct rtw_r_meta_data *meta);
+
+enum rtw_hal_status
+rtw_hal_query_rainfo(void *hal, struct rtw_hal_stainfo_t *hal_sta,
+		     struct rtw_phl_rainfo *phl_rainfo);
+
+enum rtw_hal_status
+rtw_hal_start_ba_session(void *hal, struct rtw_phl_stainfo_t *sta,
+                         u8 dialog_token, u16 timeout,
+                         u16 start_seq_num, u16 ba_policy,
+                         u16 tid, u16 buf_size);
+enum rtw_hal_status
+rtw_hal_stop_ba_session(void *hal, struct rtw_phl_stainfo_t *sta, u16 tid);
+
+/**
+ * rtw_hal_set_edca() - setup WMM EDCA parameter
+ * @hal:	struct hal_info_t *
+ * @wrole:	struct rtw_wifi_role_t *
+ * @ac:		Access Category, 0:BE, 1:BK, 2:VI, 3:VO
+ * @param:	AIFS:BIT[7:0], CWMIN:BIT[11:8], CWMAX:BIT[15:12],
+ *		TXOP:BIT[31:16]
+ *
+ * Setup WMM EDCA parameter set for sending packets.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status
+rtw_hal_set_edca(void *hal,
+                 struct rtw_wifi_role_link_t *rlink,
+                 u8 ac,
+                 u32 edca);
+
+enum rtw_hal_status
+rtw_hal_upd_ampdu_cctrl_info(void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_cfg_tx_ampdu(void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_cfg_tx_amsdu(void *hal,
+		     u8 macid,
+		     u8 enable,
+		     u8 amsdu_max_len,
+		     u8 qos_field_h,
+		     u8 qos_field_h_en,
+		     u8 mhdr_len,
+		     u8 vlan_tag_valid);
+
+/**
+ * rtw_hal_cfg_rsc() - set rsc config
+ * @hal:	struct hal_info_t *
+ * @wrole:	struct rtw_wifi_role_t *
+ * @rsc_cfg:    00: Primary 20Mhz BW(0x483[3:0] )
+ *                      01: Follow Rx packet BW mode
+ *                      10: Duplicate mode. (ofdm rate only, others bw20
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status
+rtw_hal_cfg_rsc(void *hal, struct rtw_phl_stainfo_t *sta, u8 rsc_cfg);
+
+enum rtw_hal_status
+rtw_hal_cfg_rrsr_ref_rate_sel(void *hal, struct rtw_phl_stainfo_t *sta, bool ref_rate_sel);
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+enum rtw_hal_status
+rtw_hal_custom_cfg_tx_ampdu(void *hal,
+                            struct rtw_wifi_role_link_t *rlink,
+                            struct rtw_phl_custom_ampdu_cfg *ampdu);
+
+enum rtw_hal_status
+rtw_hal_get_ampdu_cfg(void *hal,
+                      struct rtw_wifi_role_link_t *rlink,
+                      struct rtw_phl_custom_ampdu_cfg *cfg);
+
+enum rtw_hal_status
+rtw_hal_get_edca(void *hal,
+                 struct rtw_wifi_role_link_t *rlink,
+                 struct rtw_edca_param *edca_param);
+
+enum rtw_hal_status
+rtw_hal_set_usr_edca(void *hal,
+                     u32 idx,
+                     u32 enable,
+                     u32 band,
+                     u32 wmm,
+                     struct rtw_edca_param *aggr,
+                     struct rtw_edca_param *mod);
+
+enum rtw_hal_status
+rtw_hal_set_pop_en(void *hal, bool en, enum phl_phy_idx phy_idx);
+
+bool
+rtw_hal_query_pop_en(void *hal, enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_set_pkt_detect_thold(void *hal, enum phl_band_idx band_idx, u32 bound);
+
+u8
+rtw_hal_query_pkt_detect_thold(void *hal,
+                               bool get_en_info,
+                               enum phl_phy_idx phy_idx);
+
+void
+rtw_hal_set_tx_rate_rty_tbl(void *hal, bool en, u8 *rty_rate_tbl);
+
+enum rtw_hal_status
+rtw_hal_set_spatial_reuse_en(void *hal, bool en);
+
+bool
+rtw_hal_is_spatial_reuse_en(void *hal);
+
+enum rtw_hal_status rtw_hal_set_usr_frame_to_act(
+    void *hal, enum rtw_mac_usr_frame_to_act_mode mode, u32 to_thr,
+    u8 trigger_cnt, u16 sw_def_bmp);
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+enum rtw_hal_status
+rtw_hal_set_usr_tx_rpt_cfg(void *hal, struct rtw_phl_usr_tx_rpt_cfg *param);
+
+enum rtw_hal_status rtw_hal_set_ch_busy_stat_cfg(void *hal, u8 band);
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+#endif
+enum rtw_hal_status
+rtw_hal_thermal_protect_cfg_tx_duty(
+	void *hal,
+	u16 tx_duty_interval,
+	u8 ratio);
+
+enum rtw_hal_status
+rtw_hal_thermal_protect_stop_tx_duty(
+	void *hal);
+
+enum rtw_hal_status
+rtw_hal_set_key(void *hal, struct rtw_phl_stainfo_t *sta, u8 type, u8 ext_key, u8 spp,
+				u8 keyid, u8 keytype, u8 *keybuf);
+
+u32
+rtw_hal_search_key_idx(void *hal, struct rtw_phl_stainfo_t *sta,
+						u8 keyid, u8 keytype);
+/**
+ * rtw_hal_tx_chnl_mapping - query hw tx dma channel mapping to the sw xmit ring
+ * identified by macid, tid and band
+ * @hal: see struct hal_info_t
+ * @macid: input target macid is 0 ~ 127
+ * @cat: input target packet category, see enum rtw_phl_ring_cat
+ * @band: input target band, 0 for band 0 / 1 for band 1
+ *
+ * returns the mapping hw tx dma channel
+ */
+u8 rtw_hal_tx_chnl_mapping(void *hal, u16 macid,
+			   enum rtw_phl_ring_cat cat, u8 band);
+
+enum rtw_hal_status
+rtw_hal_enable_hwamsdu(void *hal,
+			u8 enable,
+			u8 max_num,
+			u8 en_single_amsdu,
+			u8 en_last_amsdu_padding);
+
+/**
+ * rtw_hal_convert_qsel_to_tid - convert qsel to tid value
+ * @hal: see struct hal_info_t
+ * @qsel: HW queue selection
+ * @tid_indic: tid indicator
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+u8 rtw_hal_convert_qsel_to_tid(void *hal, u8 qsel_id, u8 tid_indic);
+
+
+/**
+ * rtw_hal_tx_res_query - query current HW tx resource with specifc dma channel
+ * @hal: see struct hal_info_t
+ * @dma_ch: the target dma channel
+ * @host_idx: current host index of this channel
+ * @hw_idx: current hw index of this channel
+ *
+ * this function returns the number of available tx resource
+ * NOTE, input host_idx and hw_idx ptr shall NOT be NULL
+ */
+u16 rtw_hal_tx_res_query(void *hal, u8 dma_ch, u16 *host_idx, u16 *hw_idx);
+
+/**
+ * rtw_hal_rx_res_query - query current HW rx resource with specifc dma channel
+ * @hal: see struct hal_info_t
+ * @dma_ch: the target dma channel
+ * @host_idx: current host index of this channel
+ * @hw_idx: current hw index of this channel
+ *
+ * this function returns the number of available tx resource
+ * NOTE, input host_idx and hw_idx ptr shall NOT be NULL
+ */
+u16 rtw_hal_rx_res_query(void *hal, u8 dma_ch, u16 *host_idx, u16 *hw_idx);
+u16 rtw_hal_get_rxbd_num(void *hal, u8 dma_ch);
+u16 rtw_hal_get_rxbuf_num(void *hal, u8 dma_ch);
+u16 rtw_hal_get_rxbuf_size(void *hal, u8 dma_ch);
+
+enum phl_band_idx rtw_hal_query_txch_hwband(void *hal, u8 dma_ch);
+
+/**
+ * rtw_hal_query_txch_num - query total hw tx dma channels number
+ *
+ * returns the number of  hw tx dma channel
+ */
+u8 rtw_hal_query_txch_num(void *hal);
+
+/**
+ * rtw_hal_query_rxch_num - query total hw rx dma channels number
+ *
+ * returns the number of  hw rx dma channel
+ */
+u8 rtw_hal_query_rxch_num(void *hal);
+
+
+/**
+ * rtw_hal_update_wd_page - update wd page for xmit packet
+ * @hal: see struct hal_info_t
+ * @phl_pkt_req: packet xmit request from phl, see struct rtw_phl_pkt_req
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+enum rtw_hal_status rtw_hal_update_wd_page(void *hal, void *phl_pkt_req);
+
+enum rtw_hal_status
+rtw_hal_handle_rx_buffer(struct rtw_phl_com_t *phl_com, void* hal,
+				u8 *buf, u32 buf_size,
+				struct rtw_phl_rx_pkt *rxpkt);
+
+enum rtw_hal_status
+rtw_hal_query_info(void* hal, u8 info_id, void *value);
+
+#ifdef CONFIG_PHL_DFS
+bool rtw_hal_in_radar_domain(void *hal, enum band_type band, u8 ch
+	, enum channel_width bw, enum chan_offset offset);
+void rtw_hal_bb_dfs_change_domain(void *hal, enum band_type band, u8 ch
+	, enum channel_width bw, enum chan_offset offset);
+enum rtw_hal_status rtw_hal_radar_detect_cfg(void *hal, u8 band_idx, bool dfs_enable);
+#endif /*CONFIG_PHL_DFS*/
+
+enum rtw_hal_status
+rtw_hal_set_ch_bw(void *hal, u8 band_idx,
+		struct rtw_chan_def *chdef, enum rfk_tri_type rt_type, bool rd_enabled, bool frc_switch);
+u8 rtw_hal_get_cur_ch(void *hal, u8 band_idx);
+void rtw_hal_get_cur_chdef(void *hal, u8 band_idx,
+				struct rtw_chan_def *cur_chandef);
+
+void rtw_hal_sync_cur_ch(void *hal, u8 band_idx, struct rtw_chan_def chandef);
+
+u8 rtw_hal_get_fwcmd_queue_idx(void* hal);
+void rtw_hal_clear_rwptr(void *hal);
+void rtw_hal_cfg_txhci(void *hal, u8 en);
+void rtw_hal_cfg_rxhci(void *hal, u8 en);
+enum rtw_hal_status rtw_hal_chk_allq_empty(void *hal, u8 *empty);
+enum rtw_hal_status rtw_hal_set_power_limit(void *hal, u8 band_idx);
+s8 rtw_hal_get_power_limit(void *hal, enum phl_phy_idx phy,
+	u16 rate, u8 bandwidth, u8 beamforming, u8 tx_num, u8 channel);
+bool rtw_hal_get_ext_pwr_lmt_en(void *hal, u8 hw_band);
+s8 rtw_hal_get_power_by_rate_band(void *hal, u8 band_idx, u16 rate, u8 dcm, u8 offset, u32 band);
+s8 rtw_hal_get_power_limit_option(void *hal, u8 band_idx, u8 rf_path, u16 rate,
+	u8 bandwidth, u8 beamforming, u8 tx_num, u8 channel, u32 band, u8 reg);
+s8 rtw_hal_get_power_limit_ru_option(void *hal,
+	u8 band_idx, u8 rf_path, u16 rate, u8 bandwidth,
+	u8 tx_num, u8 channel, u32 band, u8 reg);
+u8 rtw_hal_get_tx_tbl_to_tx_pwr_times(void *hal);
+s8 rtw_hal_get_power_limit_value_ww(void *hal);
+s8 rtw_hal_get_power_limit_value_na(void *hal);
+u32 rtw_hal_get_regulation_max_num(void *hal, enum band_type band);
+void rtw_hal_set_ext_pwr_lmt_en(void *hal, u8 hw_band, bool enable);
+void rtw_hal_enable_ext_pwr_lmt(void *hal, u8 hw_band,
+		struct rtw_phl_ext_pwr_lmt_info *ext_pwr_lmt_info);
+
+u8 rtw_hal_get_tx_tbl_to_pwr_times(void *hal);
+void rtw_hal_set_tx_pwr_comp(void *hal, enum phl_phy_idx phy,
+		struct rtw_phl_regu_dyn_ant_gain *dyn_ag, enum band_type band,
+		const char *sregulation);
+
+#ifdef CONFIG_RTW_ACS
+void rtw_hal_acs_mntr_trigger(void *hal, enum phl_band_idx band_idx, struct acs_mntr_parm *parm);
+enum rtw_hal_status rtw_hal_acs_mntr_result(void *hal,
+					enum phl_band_idx band_idx, struct acs_mntr_rpt *rpt);
+#endif /* CONFIG_RTW_ACS */
+
+/*watchdog update env result*/
+void rtw_hal_env_rpt(struct rtw_hal_com_t *hal_com,
+                     struct rtw_env_report *env_rpt,
+                     u8 hw_band);
+
+
+enum rtw_hal_status
+rtw_hal_tbtt_tuning(void *hal, enum phl_band_idx band,
+			u8 port, u32 tbtt);
+
+bool rtw_hal_poll_txdma_idle(void *hal);
+
+void rtw_hal_auto_debug_en_phy_util(struct rtw_hal_com_t *hal_com, bool en);
+
+void rtw_hal_query_snr_avg(struct rtw_hal_com_t *hal_com,
+                           u8 *info,
+                           u8 hw_band);
+
+#ifdef CONFIG_PCI_HCI
+/**
+ * rtw_hal_update_txbd - update tx bd for xmit packet
+ * @hal: see struct hal_info_t
+ * @txbd: the target txbd to update
+ * @wd: buffer pointer of wd page to fill in txbd
+ *
+ * returns enum RTW_HAL_STATUS
+ * NOTE, this function is PCIe specific function
+ */
+enum rtw_hal_status
+rtw_hal_update_txbd(void *hal, void *txbd, void *wd, u8 dma_ch, u16 wd_num);
+
+/**
+ * rtw_hal_update_rxbd - update rx bd for recv packet
+ * @hal: see struct hal_info_t
+ * @rxbd: the target rxbd to update
+ * @wd: buffer pointer of wd page to fill in rxbd
+ *
+ * returns enum RTW_HAL_STATUS
+ * NOTE, this function is PCIe specific function
+ */
+enum rtw_hal_status
+rtw_hal_update_rxbd(void *hal, struct rx_base_desc *rxbd,
+					struct rtw_rx_buf *rxbuf, u8 ch_idx);
+
+/**
+ * rtw_hal_update_trigger_txstart - trigger hw to start tx
+ * @hal: see struct hal_info_t
+ * @txbd: the target txbd to update
+ * @dma_ch: the dma channel index of this txbd_ring
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+enum rtw_hal_status
+rtw_hal_trigger_txstart(void *hal, struct tx_base_desc *txbd, u8 dma_ch);
+
+/**
+ * rtw_hal_notify_rxdone - notify hw rx done
+ * @hal: see struct hal_info_t
+ * @rxbd: the target rxbd to update
+ * @dma_ch: the target rx channel
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+enum rtw_hal_status
+rtw_hal_notify_rxdone(void* hal, struct rx_base_desc *rxbd, u8 dma_ch);
+
+/**
+ * rtw_hal_handle_wp_rpt -parsing the wp recycle report packet
+ * @hal: see struct hal_info_t
+ * @rp: input, the pointer of target recycle report buffer to parse
+ * @len: input, the length of this report buffer
+ * @sw_retry: output, the packet mentioned in this report needs sw retry if sw_retry == 1
+ * @dma_ch: output, the tx dma channel of this packet mentioned in this report
+ * @wp_seq: output, the wp_seq of this packet mentioned in this report
+ * @macid: output, the mac_id of this packet mentioned in this report
+ * @ac_queue: output, the tid of this packet mentioned in this report
+ * @txsts: output, the tx status of this packet mentioned in this report
+ *
+ * returns the length of report buffer which has been parsed in this function
+ */
+u16 rtw_hal_handle_wp_rpt(void *hal, u8 *rp, u16 len, u8 *sw_retry, u8 *dma_ch,
+			  u16 *wp_seq, u8 *macid, u8 *ac_queue, u8 *txsts);
+
+u8 rtw_hal_check_rxrdy(struct rtw_phl_com_t *phl_com, void* hal,
+		       struct rtw_rx_buf *rxbuf, u8 dma_ch);
+u8 rtw_hal_handle_rxbd_info(void* hal, u8 *rxbuf, u16 *buf_size);
+enum rtw_hal_status rtw_hal_set_l2_leave(void *hal);
+void rtw_hal_clear_bdidx(void *hal);
+void rtw_hal_rst_bdram(void *hal);
+void rtw_hal_cfg_dma_io(void *hal, u8 en);
+
+#ifdef RTW_WKARD_DYNAMIC_PCIE_GEN
+void rtw_hal_pcie_gen_set(void *hal, enum rtw_pcie_gen gen);
+enum rtw_pcie_gen rtw_hal_pcie_gen_get(void *hal);
+#endif /* RTW_WKARD_DYNAMIC_PCIE_GEN */
+
+enum rtw_hal_status rtw_hal_ltr_sw_trigger(void *hal,
+	enum rtw_pcie_ltr_state state);
+enum rtw_hal_status rtw_hal_ltr_en_hw_mode(void *hal, bool hw_mode);
+#ifdef RTW_WKARD_DYNAMIC_LTR
+void rtw_hal_ltr_update_stats(void *hal, bool clear);
+bool rtw_hal_ltr_is_sw_ctrl(struct rtw_phl_com_t *phl_com, void *hal);
+bool rtw_hal_ltr_is_hw_ctrl(struct rtw_phl_com_t *phl_com, void *hal);
+#endif
+enum rtw_hal_status
+rtw_hal_pcie_cfg_get(void *hal, struct rtw_pcie_cfgspc_param *cfg);
+
+void rtw_hal_set_pcicfg(struct rtw_phl_com_t *phl_com, void *hal);
+#endif /*CONFIG_PCI_HCI*/
+
+#ifdef CONFIG_USB_HCI
+
+u8 rtw_hal_get_bulkout_id(void *hal, u8 dma_ch, u8 mode);
+
+u32 rtw_hal_get_wd_len(void *hal,
+				struct rtw_xmit_req *tx_req);
+
+enum rtw_hal_status
+	rtw_hal_usb_tx_agg_cfg(void *hal, u8* wd_buf, u8 agg_num);
+
+/**
+ * rtw_hal_usb_rx_agg_cfg - configure USB Rx aggregation setting
+ * @hal: see struct hal_info_t
+ * @mode: Parameter settings for Rx aggregation. see enum phl_usb_rx_agg_mode.
+ * @agg_mode: Mode of USB data transfer. see enum mac_ax_rx_agg_mode.
+ * @timeout: Maximum Rx aggregation time. In 32 ms.
+ * @size: Maximum Rx aggregation size. In 4096 bytes.
+ * @pkt_num: Maximum Rx aggregation packet number.
+ */
+enum rtw_hal_status
+	rtw_hal_usb_rx_agg_cfg(void *hal, u8 mode, u8 agg_mode,
+	u8 drv_define, u8 timeout, u8 size, u8 pkt_num);
+
+enum rtw_hal_status
+rtw_hal_usb_rx_agg_init(struct rtw_phl_com_t *phl_com, void *hal);
+
+u8 rtw_hal_get_max_bulkout_wd_num(void *hal);
+u16 rtw_hal_get_max_dma_txagg_msk(void *hal);
+u32 rtwl_hal_get_cur_usb_mode(void *h);
+u32 rtwl_hal_get_usb_support_ability(void *h);
+#ifdef CONFIG_PHL_CUSTOM_FEATURE_USB
+enum rtw_hal_status rtw_hal_set_usb_support_ability(void *h, u32 ability);
+#endif
+enum rtw_hal_status rtw_hal_get_usb_mode_status(void *h, u32 *status);
+enum rtw_hal_status rtw_hal_get_u3_perf_mode(void *h, u32 *perf_mode);
+
+enum rtw_hal_status rtw_hal_force_usb_switch(void *h, enum usb_type type);
+/**
+ * rtw_hal_handle_wp_rpt_usb -parsing the wp recycle report packet
+ * @hal: see struct hal_info_t
+ * @rp: input, the pointer of target recycle report buffer to parse
+ * @len: input, the length of this report buffer
+ * @macid: output, the mac_id of this packet mentioned in this report
+ * @ac_queue: output, the tid of this packet mentioned in this report
+ * @txsts: output, the tx status of this packet mentioned in this report
+ *
+ * returns the length of report buffer which has been parsed in this function
+ */
+u16 rtw_hal_handle_wp_rpt_usb(void *hal, u8 *rp, u16 len, u8 *macid, u8 *ac_queue,
+			u8 *txsts);
+
+#ifdef CONFIG_PHL_CSUM_OFFLOAD_RX
+enum rtw_hal_status rtw_hal_chk_rx_tcpip_chksum_ofd(void *h,
+						    struct rtw_r_meta_data *mdata,
+                                                    u8 status);
+#endif /* CONFIG_PHL_CSUM_OFFLOAD_RX */
+#endif
+
+#ifdef CONFIG_SDIO_HCI
+/**
+ * rtw_hal_sdio_tx_cfg - SDIO TX related setting
+ * @hal:	pointer of struct hal_info_t
+ *
+ * Configure setting for SDIO TX.
+ *
+ * No return value for this function.
+ */
+void rtw_hal_sdio_tx_cfg(void *hal);
+
+enum rtw_hal_status rtw_hal_sdio_tx(void *hal, u8 dma_ch, u8 *buf, u32 buf_len,
+				    u8 agg_count, u16 *pkt_len, u8 *wp_offset);
+void rtw_hal_sdio_rx_agg_cfg(void *hal, bool enable, u8 drv_define,
+			     u8 timeout, u8 size, u8 pkt_num);
+int rtw_hal_sdio_rx(void *hal, struct rtw_rx_buf *rxbuf);
+int rtw_hal_sdio_parse_rx(void *hal, struct rtw_rx_buf *rxbuf);
+/**
+ * rtw_hal_sdio_lps_flg - get lps status
+ * @hal:	pointer of struct hal_info_t
+ *
+ * Get driver lps status
+ *
+ * return lps flag state defined in enum rtw_hal_lps_flg_state
+ * RTW_HAL_LPS_FLG_STATE_LPS when driver set LPS-CG/LPS-PG enable;
+ * RTW_HAL_LPS_FLG_STATE_ACTIVE when driver is not in LPS-CG/LPS-PG
+ */
+enum rtw_hal_lps_flg_state rtw_hal_sdio_lps_flg(void *hal);
+#endif /* CONFIG_SDIO_HCI */
+
+enum rtw_hal_status rtw_hal_fill_wd(void *hal,
+				struct rtw_xmit_req *tx_req,
+				u8 *wd_buf, u32 *wd_len);
+
+/* HAL SOUND API */
+enum rtw_hal_status rtw_hal_snd_query_proc_sta_res(
+	void *hal, struct rtw_phl_stainfo_t *sta,
+	bool mu, enum channel_width bw, bool en_swap);
+
+enum rtw_hal_status
+rtw_hal_snd_release_proc_sta_res(void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_snd_proc_pre_cfg_sta(void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_snd_proc_pre_cfg(void *hal);
+
+void
+rtw_hal_snd_ndpa_sta_info_vht(struct rtw_phl_stainfo_t *psta_info,
+			      u32 *ndpa, u8 mu);
+
+void
+rtw_hal_snd_ndpa_sta_info_he(struct rtw_phl_stainfo_t *psta_info,
+			     u32 *ndpa, enum channel_width bw, u8 fb_type);
+
+enum rtw_hal_status
+rtw_hal_snd_proc_post_cfg_sta(void *hal,
+				struct rtw_phl_stainfo_t *sta, bool mu);
+
+enum rtw_hal_status
+rtw_hal_snd_proc_post_cfg_gid(void *hal, u8 gid, void *ba_info);
+
+enum rtw_hal_status
+rtw_hal_snd_proc_post_cfg(void *hal, bool he, bool mu, bool en_fixed_mode);
+
+enum rtw_hal_status
+rtw_hal_snd_mac_ctrl(void *hal, u8 band, u8 ctrl);
+
+enum rtw_hal_status
+rtw_hal_snd_chk_bf_res(void *hal, struct rtw_phl_stainfo_t *sta,
+		       bool mu, enum channel_width bw);
+
+void
+rtw_hal_snd_polling_snd_sts(void *hal, struct rtw_phl_stainfo_t *sta);
+
+/* fw sounding commmand ralated */
+void rtw_hal_snd_set_fw_cmd_dialogtkn(void *hal, u8 *buf, u8 he, u8 token);
+
+void rtw_hal_snd_vht_fwcmd_su(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u32 *npda_sta);
+
+void rtw_hal_snd_vht_fwcmd_mu_pri(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u8 sta_nr, u32 *ndpa_sta);
+
+void rtw_hal_snd_vht_fwcmd_mu_add_sta(void *hal, u8 *buf, u32 *ndpa_sta,
+				      struct rtw_phl_stainfo_t *sta,
+				      u8 ndpa_idx, u8 last);
+
+void rtw_hal_snd_ax_fwcmd_nontb(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u32 *npda_sta);
+
+void rtw_hal_snd_ax_fwcmd_tb_pri(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u8 sta_nr1, u8 sta_nr2);
+
+void rtw_hal_snd_ax_fwcmd_tb_add_sta(void *hal, u8 *buf, u32 *ndpa_sta,
+				     struct rtw_phl_stainfo_t *sta, u8 ru_idx,
+				     u8 ndpa_idx, u8 bfrp_idx, u8 bfrp_u_idx);
+
+u8 *rtw_hal_snd_prepare_snd_cmd(void *hal);
+enum rtw_hal_status rtw_hal_snd_release_snd_cmd(void *hal, u8 *buf);
+enum rtw_hal_status rtw_hal_snd_send_fw_cmd(void *hal, u8 *cmd);
+
+/*HAL CSI Buffer Mgnt APIs*/
+enum channel_width
+rtw_hal_get_csi_buf_bw(void *buf);
+
+bool
+rtw_hal_get_csi_buf_type(void *buf);
+
+/*HAL Beamform Mgnt APIs*/
+void rtw_hal_bf_dbg_dump_entry(void *entry);
+void rtw_hal_bf_dbg_dump_entry_all(void *hal);
+u8 rtw_hal_bf_get_sumu_idx(void *hal, void *entry);
+bool rtw_hal_bf_chk_bf_type(void *hal_info,
+			struct rtw_phl_stainfo_t *sta, bool mu);
+void rtw_hal_bf_preset_mu_ba_info(void *hal,
+			struct rtw_phl_stainfo_t *psta, void *hal_ba_info);
+void rtw_hal_bf_set_txmu_para(void *hal, u8 gid , u8 en,
+			      enum rtw_hal_protection_type rts_type,
+			      enum rtw_hal_ack_resp_type ack_type);
+enum rtw_hal_status
+rtw_hal_bf_set_fix_mode(void *hal, bool mu, bool he);
+enum rtw_hal_status
+rtw_hal_bf_get_entry_snd_sts(void *entry);
+
+void rtw_hal_beamform_set_vht_gid(void *hal, u8 band,
+				  struct rtw_phl_gid_pos_tbl *tbl);
+
+/******************************************************************************
+ *
+ * BTC APIs
+ *
+ *****************************************************************************/
+void rtw_hal_btc_scan_start_ntfy(void *hinfo, enum phl_phy_idx phy_idx,
+				  enum band_type band);
+void rtw_hal_btc_scan_finish_ntfy(void *hinfo, enum phl_phy_idx phy_idx);
+
+void rtw_hal_btc_update_role_info_ntfy(void *hinfo, u8 role_id,
+				       struct rtw_wifi_role_t *wrole,
+				       struct rtw_wifi_role_link_t *rlink,
+				       struct rtw_phl_stainfo_t *sta,
+				       enum link_state lstate);
+
+void rtw_hal_btc_bk_mdl_start_ntfy(void *hinfo);
+
+void
+rtw_hal_btc_ap_client_notify(void *hinfo,
+		struct rtw_wifi_role_link_t *rlink, enum link_state lstate);
+
+void rtw_hal_btc_packet_event_ntfy(void *hinfo, u8 pkt_evt_type);
+void rtw_hal_btc_radio_state_ntfy(void *hinfo, u8 rf_state);
+void rtw_hal_btc_customerize_ntfy(void *hinfo, u8 type, u16 len, u8 *buf);
+void rtw_hal_btc_wl_status_ntfy(void *hinfo, struct rtw_phl_com_t *phl_com, u8 ntfy_num,
+					struct rtw_phl_stainfo_t *sta[],
+									u8 reason);
+void rtw_hal_btc_timer(void *hinfo, void *timer);
+void rtw_hal_btc_fwinfo_ntfy(void *hinfo);
+
+enum rtw_hal_status rtw_hal_scan_set_rxfltr_by_mode(void *hinfo,
+	enum phl_phy_idx phy_idx, bool off_channel, u8 *mode);
+enum rtw_hal_status
+rtw_hal_enter_mon_mode(void *hinfo, enum phl_phy_idx phy_idx);
+enum rtw_hal_status
+rtw_hal_leave_mon_mode(void *hinfo, enum phl_phy_idx phy_idx);
+#ifdef CONFIG_FSM
+enum rtw_hal_status rtw_hal_scan_flush_queue(void *hinfo,
+	struct rtw_wifi_role_t *wrole);
+#endif
+enum rtw_hal_status rtw_hal_scan_pause_tx_fifo(void *hinfo,
+	u8 band_idx, bool off_ch);
+
+enum rtw_hal_status
+rtw_hal_dfs_pause_tx(void *hinfo, u8 band_idx, bool off_ch, enum tx_pause_rson rson);
+
+/*****************************************************************************/
+
+void rtw_hal_com_scan_set_tx_lifetime(void *hal, u8 band);
+void rtw_hal_com_scan_restore_tx_lifetime(void *hal, u8 band);
+enum rtw_hal_status
+rtw_hal_com_set_tx_lifetime(void *hal, u8 band, u8 en, u16 val);
+
+enum rtw_hal_status
+rtw_hal_com_set_power_offset(void *hal, u8 band, s8 ofst_mode, s8 ofst_bw);
+
+#ifdef RTW_PHL_BCN
+enum rtw_hal_status
+rtw_hal_add_beacon(struct rtw_phl_com_t *phl_com, void *hal, void *bcn_cmn);
+
+enum rtw_hal_status
+rtw_hal_update_beacon(struct rtw_phl_com_t *phl_com, void *hal, u8 bcn_id);
+
+enum rtw_hal_status
+rtw_hal_free_beacon(struct rtw_phl_com_t *phl_com, void *hal, u8 bcn_id);
+#ifdef CONFIG_RTW_DEBUG_BCN_TX
+enum rtw_hal_status
+rtw_hal_get_beacon_cnt(struct rtw_phl_com_t *phl_com, void *hal,
+		u8 bcn_id, struct rtw_bcn_stats **bcn_stats);
+#endif
+#endif
+
+enum rtw_hal_status
+rtw_hal_com_set_gt3(void *hal, u8 enable, u32 timeout);
+
+enum rtw_hal_status
+rtw_hal_set_dctrl_tbl_seq(void *hal,
+	struct rtw_phl_stainfo_t *sta, u32 seq);
+
+enum rtw_hal_status
+rtw_hal_proc_cmd(void *hal, char proc_cmd, struct rtw_proc_cmd *incmd,
+							char *output, u32 out_len);
+
+void rtw_hal_get_mac_version(char *ver_str, u16 len);
+void rtw_hal_get_fw_ver(void *hal, char *ver_str, u16 len);
+
+enum rtw_hal_status
+rtw_hal_antdiv_fix_ant(void *hal, u8 antIndex);
+
+enum rf_path
+rtw_hal_get_path_from_ant_num(void *hal, u8 antnum);
+
+enum rtw_hal_status
+rtw_hal_pause_tx_fifo_sw(void *hal, u8 band_idx,
+			bool tx_pause, enum tx_pause_rson rson);
+
+enum rtw_hal_status
+rtw_hal_tx_pause(struct rtw_hal_com_t *hal_com,
+			u8 band_idx, bool tx_pause, enum tx_pause_rson rson);
+
+enum rtw_hal_status rtw_hal_set_macid_pause(void *hinfo,
+                                            u16 macid,
+                                            bool pause);
+
+enum rtw_hal_status rtw_hal_set_macid_pause_ac(void *hinfo, u16 macid, bool pause);
+/**
+ * rtw_hal_set_rxfltr_opt_by_mode - Set rx filter option by scenario
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @mode:	scenario mode
+ *
+ * Set RX filter option setting by scenario.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_opt_by_mode(void *hal, u8 band,
+					       enum rtw_rx_fltr_opt_mode mode);
+
+/**
+ * rtw_hal_get_rxfltr_opt_mode - Get rx filter option
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ *
+ * Get RX filter mode
+ *
+ * Return rtw_rx_fltr_opt_mode
+ */
+enum rtw_rx_fltr_opt_mode rtw_hal_get_rxfltr_opt_mode(void *hal, u8 band);
+
+/**
+ * rtw_hal_set_rxfltr_type_by_mode - Set target for rx filter type by scenario
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @mode:	scenario mode
+ *
+ * Set target of RX filter type by scenario.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_type_by_mode(void *hal, u8 band,
+						    enum rtw_rxfltr_type_mode mode);
+
+/**
+ * rtw_hal_set_rxfltr_by_stype_ctrl - Set target for rx CTRL filter sub-type
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @cap_ctrl:	data structure to set target
+ *
+ * Set target for rx CTRL filter sub-type.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_by_stype_ctrl(void *hal, u8 band,
+						     struct rxfltr_cap_to_set_ctrl *cap_ctrl);
+
+/**
+ * rtw_hal_set_rxfltr_by_stype_mgnt - Set target for rx MGNT filter sub-type
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @cap_mgnt:	data structure to set target
+ *
+ * Set target for rx MGNT filter sub-type.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_by_stype_mgnt(void *hal, u8 band,
+						     struct rxfltr_cap_to_set_mgnt *cap_mgnt);
+
+/**
+ * rtw_hal_set_rxfltr_by_stype_data - Set target for rx DATA filter sub-type
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @cap_data:	data structure to set target
+ *
+ * Set target for rx DATA filter sub-type.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_by_stype_data(void *hal, u8 band,
+						     struct rxfltr_cap_to_set_data *cap_data);
+
+/**
+ * rtw_hal_set_rxfltr_by_stype_all - Set target for rx CTRL/MGNT/DATA filter sub-type
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @cap_all:	data structure to set target
+ *
+ * Set target for rx CTRL/MGNT/DATA filter filter sub-type.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_by_stype_all(void *hal, u8 band,
+						    struct rxfltr_type_cap_to_set *cap_all);
+/**
+ * rtw_hal_mac_acpt_crc_err_pkt - Accept CRC error packets or not
+ * @hal:	pointer of struct rtw_hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @enable:	0: deny, 1: accept
+ *
+ * Control accepting CRC error packets or not.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_acpt_crc_err_pkt(void *hal, u8 band, u8 enable);
+
+/**
+ * rtw_hal_mac_ppdu_stat_cfg - control mac ppdu sts settings
+ * @hal:	pointer of struct rtw_hal_info_t
+ * @phl_com:    pointer of struct rtw_phl_com_t
+ * @band:	0x0: band0, 0x1: band1
+ * @fltr:	bitmap:HAL_PPDU_HAS_CRC_OK/HAL_PPDU_HAS_A1M/...
+ *
+ * Control MAC layer ppdu status filter settings, such as CRC error pkt's ppdu sts.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_cfg_ppdu_sts_fltr(void *hal, struct rtw_phl_com_t *phl_com, u8 band, u8 fltr);
+
+/**
+ * rtw_hal_set_rxfltr_mpdu_size - Set max MPDU size
+ * @hal:	pointer of struct hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @size:	MPDU max size, unit: byte. 0 for no limit.
+ *
+ * MPDU size exceed max size would be dropped.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_mpdu_size(void *hal, u8 band, u16 size);
+
+/**
+ * rtw_hal_set_rxfltr_by_type - Set target for rx CTRL/MGNT/DATA filter type
+ * @hal:	pointer of struct rtw_hal_info_t
+ * @band:	0x0: band0, 0x1: band1
+ * @type:	enum rtw_packet_type
+ * @target:	enum rtw_rxfltr_target
+ *
+ * Set target for rx CTRL/MGNT/DATA filter type.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_set_rxfltr_by_type(void *hal, u8 band, enum rtw_packet_type type,
+					       enum rtw_rxfltr_target target);
+
+void rtw_hal_dbg_status_dump(void *hal, struct hal_mac_dbg_dump_cfg *cfg);
+
+#ifdef CONFIG_PHL_PKTOFLD
+/* Packet Offload APIs */
+enum rtw_hal_status rtw_hal_pkt_ofld(void *hal, u8 *id, u8 op,
+					u8 *pkt, u16 *len);
+enum rtw_hal_status rtw_hal_pkt_update_ids(void *hal,
+					struct pkt_ofld_entry *entry);
+#endif
+void rtw_hal_fw_cap_pre_config(struct rtw_phl_com_t *phl_com, void *hal);
+void rtw_hal_bus_cap_pre_config(struct rtw_phl_com_t *phl_com, void *hal);
+void rtw_hal_fw_final_cap_config(struct rtw_phl_com_t *phl_com, void *hal);
+void rtw_hal_final_cap_decision(struct rtw_phl_com_t *phl_com, void *hal);
+
+/* Power Save APIs */
+enum rtw_hal_status
+rtw_hal_ps_pwr_lvl_cfg(struct rtw_phl_com_t *phl_com, void *hal,
+			u32 req_pwr_lvl);
+enum rtw_hal_status
+rtw_hal_ps_lps_cfg(void *hal, struct rtw_hal_lps_info *lps_info);
+enum rtw_hal_status
+rtw_hal_ps_ips_cfg(void *hal, struct rtw_hal_ips_info *ips_info);
+enum rtw_hal_status
+rtw_hal_get_bf_proto_cap(struct rtw_phl_com_t *phl_com, void *hal,
+			 u8 band, struct protocol_cap_t *proto_cap);
+
+enum rtw_hal_status
+rtw_hal_get_stbc_proto_cap(struct rtw_phl_com_t *phl_com, void *hal,
+			 u8 band, struct protocol_cap_t *proto_cap);
+
+enum rtw_hal_status rtw_hal_watchdog(void *hal, struct rtw_phl_com_t *phl_com);
+enum rtw_hal_status rtw_hal_simple_watchdog(void *hal, u8 io_en);
+
+#ifdef CONFIG_POWER_SAVE
+void rtw_hal_ps_chk_hw_rf_state(struct rtw_phl_com_t *phl_com, void *hal);
+void rtw_hal_ps_notify_wake(void *hal);
+void rtw_hal_lps_pvb_wait_rx(void *hal, struct rtw_phl_lps_adv_cfg *cur,
+			u16 macid, bool pvb_wait_rx);
+
+void rtw_hal_lps_ext_trx_ntfy(void *hal, struct rtw_phl_lps_adv_cfg *cur,
+				u16 macid);
+enum rtw_hal_status
+rtw_hal_lps_bcn_tracking_cfg(void *hal,
+	struct rtw_bcn_tracking_cfg *cfg, struct rtw_bcn_tracking_cfg *cfg_cur);
+#endif /* CONFIG_POWER_SAVE */
+
+/******************************************************************************
+ *
+ * TWT APIs
+ *
+ *****************************************************************************/
+#ifdef CONFIG_PHL_TWT
+enum rtw_hal_status
+rtw_hal_twt_info_update(void *hal,
+                        struct rtw_phl_twt_info twt_info,
+                        struct rtw_wifi_role_link_t *rlink,
+                        u8 action);
+
+enum rtw_hal_status
+rtw_hal_twt_sta_update(void *hal, u8 macid, u8 twt_id, u8 action);
+
+enum rtw_hal_status
+rtw_hal_twt_sta_announce(void *hal, u8 macid);
+#endif /* CONFIG_PHL_TWT */
+
+enum rtw_hal_status
+rtw_hal_set_fw_ul_fixinfo(void *hal,
+			  struct  rtw_phl_ax_ul_fixinfo *tbl);
+/******************************************************************************
+ *
+ * BB APIs
+ *
+ *****************************************************************************/
+/*@--------------------------[Prptotype]-------------------------------------*/
+/**
+ * rtw_hal_bb_get_txsc input arguments:
+ * @hal_com: hal com info
+ * @pri_ch: Spec-defined primary channel index
+ * @central_ch: Spec-defined central channel index
+ * @cbw: Channel BW
+ * @dbw: Data BW
+ */
+u8 rtw_hal_bb_get_txsc(struct rtw_hal_com_t *hal_com, u8 pri_ch,
+		u8 central_ch, enum channel_width cbw, enum channel_width dbw);
+
+enum rtw_hal_status
+rtw_hal_bb_lps_info_update(void *hal, u16 macid);
+
+#ifdef CONFIG_DBCC_SUPPORT
+void rtw_hal_dbcc_hci_ctrl(void *hal, enum phl_band_idx band_idx, u8 pause);
+enum rtw_hal_status
+rtw_hal_dbcc_trx_ctrl(void *hal, struct rtw_phl_com_t *phl_com,
+				  enum phl_band_idx band_idx, bool pause);
+
+enum rtw_hal_status
+rtw_hal_dbcc_pre_cfg(void *hal, struct rtw_phl_com_t *phl_com, bool dbcc_en);
+
+enum rtw_hal_status
+rtw_hal_dbcc_cfg(void *hal, struct rtw_phl_com_t *phl_com, bool dbcc_en);
+
+enum rtw_hal_status
+rtw_hal_dbcc_reset_hal(void *hal);
+
+enum rtw_hal_status
+rtw_hal_dbcc_band_switch_hdl(void *hal, enum phl_band_idx band_idx);
+#endif
+
+/*
+ * API for config channel info CR
+ */
+#ifdef CONFIG_PHL_CHANNEL_INFO
+enum rtw_hal_status
+rtw_hal_cfg_chinfo(void *hal, struct rtw_chinfo_action_parm *act_param);
+
+enum rtw_hal_status
+rtw_hal_ch_info_en(void *hal, struct rtw_chinfo_action_parm *act_param);
+
+#ifdef CONFIG_PHL_CSI_FW_TX_OFLD
+enum rtw_hal_status
+rtw_hal_ch_info_pkt_ofld(void *hal, u16 macid, u8 en, u16 period, u8 retry_cnt,
+				u16 rate, u8 pkt_num, u8 *pkt_id);
+#endif
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+enum rtw_hal_status
+rtw_hal_get_efuse_info(void *hal, enum rtw_efuse_info info_type,
+		       void *value, u8 size);
+
+enum rtw_hal_status
+rtw_hal_cfg_trx_path(void *hal, enum rf_path tx, u8 tx_nss,
+		     enum rf_path rx, u8 rx_nss);
+
+enum rtw_hal_status
+rtw_hal_tsf_sync(void *hal, u8 wrole_sync_from, u8 wrole_sync_to,
+			enum phl_band_idx band, s32 sync_offset_tu,
+			enum hal_tsf_sync_act act);
+
+/**
+ * rtw_hal_fill_txdesc() - Fill hardware tx header
+ * @hal:	pointer of struct hal_info_t
+ * @treq:	the xmit request for this tx descriptor
+ * @wd_buf:	the wd buffer to fill
+ * @wd_len:	output, return the total length of filled wd
+ *
+ * Fill hardware tx header/tx descriptor/wifi descriptor
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when everything is ok.
+ */
+enum rtw_hal_status
+rtw_hal_fill_txdesc(void *hal,
+                    struct rtw_xmit_req *treq,
+                    u8 *wd_buf,
+                    u32 *wd_len);
+
+enum rtw_hal_status rtw_hal_poll_hw_tx_done(void *hal);
+enum rtw_hal_status rtw_hal_hw_tx_resume(void *hal);
+enum rtw_hal_status rtw_hal_poll_hw_rx_done(void *hal);
+enum rtw_hal_status rtw_hal_hw_rx_resume(void *hal);
+
+enum rtw_hal_status
+rtw_hal_get_hwseq(void *hal, u16 macid, u8 ref_sel, u8 ssn_sel, u16 *hw_seq);
+
+
+#ifdef RTW_WKARD_DYNAMIC_BFEE_CAP
+enum rtw_hal_status rtw_hal_bf_bfee_ctrl(void *hal, u8 band, bool ctrl);
+#endif
+
+
+enum rtw_hal_status
+rtw_hal_set_mu_edca(void *hal, u8 band, u8 ac,
+	u16 timer, u8 cw_min, u8 cw_max, u8 aifsn);
+enum rtw_hal_status
+rtw_hal_set_mu_edca_ctrl(void *hal, u8 band, u8 wmm, u8 set);
+
+enum rtw_hal_status rtw_hal_led_set_ctrl_mode(void *hal, enum rtw_led_id led_id,
+					      enum rtw_led_ctrl_mode ctrl_mode);
+enum rtw_hal_status rtw_hal_led_control(void *hal, enum rtw_led_id led_id,
+					u8 high);
+
+enum rtw_hal_status rtw_hal_pcie_trx_mit(void *hal, u32 tx_timer, u8 tx_counter,
+					 u32 rx_timer, u8 rx_counter);
+
+enum rtw_hal_status rtw_hal_get_tsf(void *hal, enum phl_band_idx band,
+				u8 port, u32 *tsf_h, u32 *tsf_l);
+
+u32 rtw_hal_get_btc_req_slot(void *hal, enum phl_band_idx hw_band);
+
+enum rtw_hal_status
+rtw_hal_set_macid_grp_pause(void *hal, u32 *macid_arr, u8 arr_size, bool pause);
+
+enum rtw_hal_status
+rtw_hal_set_macid_pkt_drop(void *hal, u16 macid, u8 sel, u8 band, u8 port,
+                           u8 mbssid);
+
+#ifdef CONFIG_MCC_SUPPORT
+enum rtw_hal_status rtw_hal_mcc_get_2ports_tsf(void *hal, u8 group,
+			u16 macid_x, u16 macid_y, u32 *tsf_x_h, u32 *tsf_x_l,
+			u32 *tsf_y_h, u32 *tsf_y_l);
+
+enum rtw_hal_status rtw_hal_notify_mcc_macid(void *hal,
+                                             struct rtw_phl_mcc_role *mrole,
+                                             enum rtw_phl_tdmra_wmode wmode);
+
+enum rtw_hal_status rtw_hal_mcc_update_macid_bitmap(void *hal, u8 group,
+			u16 macid, struct rtw_phl_mcc_macid_bitmap *info);
+
+enum rtw_hal_status rtw_hal_mcc_sync_enable(void *hal,
+					struct rtw_phl_mcc_en_info *info);
+
+enum rtw_hal_status rtw_hal_mcc_change_pattern(void *hal,
+				struct rtw_phl_mcc_en_info *ori_info,
+				struct rtw_phl_mcc_en_info *new_info,
+				struct rtw_phl_mcc_bt_info *new_bt_info);
+
+enum rtw_hal_status rtw_hal_mcc_reset(void *hal, u8 group,
+					enum rtw_phl_tdmra_wmode wmode);
+
+enum rtw_hal_status rtw_hal_mcc_disable(void *hal, u8 group, u16 macid,
+					enum rtw_phl_tdmra_wmode wmode);
+
+enum rtw_hal_status rtw_hal_mcc_enable(void *hal, struct rtw_phl_mcc_en_info *info,
+					struct rtw_phl_mcc_bt_info *bt_info,
+					enum rtw_phl_tdmra_wmode wmode);
+#endif /* CONFIG_MCC_SUPPORT */
+
+#ifdef CONFIG_PHL_P2PPS
+enum rtw_hal_status rtw_hal_noa_enable(void *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid);
+
+enum rtw_hal_status rtw_hal_noa_disable(void *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid);
+
+enum rtw_hal_status
+rtw_hal_tsf32_tog_enable(void *hal,
+                         struct rtw_wifi_role_link_t *rlink);
+
+enum rtw_hal_status
+rtw_hal_tsf32_tog_disable(void *hal,
+                         struct rtw_wifi_role_link_t *rlink);
+
+enum rtw_hal_status rtw_hal_get_tsf32_tog_rpt(void *hal,
+	struct rtw_phl_tsf32_tog_rpt *rpt);
+
+enum rtw_hal_status rtw_hal_noa_sta_macid_up(void *hal,
+			u16 wrole_macid, bool join, u8 *bitmap, u32 bitmap_len);
+
+#endif
+void rtw_hal_disconnect_notify(void *hal, struct rtw_chan_def *chandef);
+
+bool rtw_hal_check_ch_rfk(void *hal, struct rtw_chan_def *chandef);
+enum rtw_hal_status rtw_hal_ppdu_sts_cfg(void *hal, u8 band_idx, bool en);
+void rtw_hal_notification(void *hal, enum phl_msg_evt_id event, u8 hw_idx);
+void rtw_hal_notification_ex(void *hal, enum phl_msg_evt_id event,
+			     bool to_bb, bool to_mac, bool to_rf, u8 hw_idx);
+
+void rtw_hal_cmd_notification(void *hal,
+                   enum phl_msg_evt_id event,
+                   void *hal_cmd,
+                   u8 hw_idx);
+
+enum rtw_hal_status
+rtw_hal_config_rts_th(void *hal, u8 band_idx, u16 rts_time_th, u16 rts_len_th);
+
+enum rtw_hal_status
+rtw_hal_query_txsts_rpt(void *hal, u16 macid);
+
+enum rtw_hal_status rtw_hal_set_dfs_tb_ctrl(void *hal, u8 set);
+enum rtw_hal_status
+rtw_hal_thermal_protect_cfg_tx_ampdu(
+	void *hal,
+	struct rtw_phl_stainfo_t *sta,
+	u8 ratio);
+
+bool rtw_hal_check_thermal_protect(
+	struct rtw_phl_com_t *phl_com,
+	void *hal);
+
+enum rtw_hal_status
+rtw_hal_beamform_set_aid(void *hal, struct rtw_phl_stainfo_t *sta, u16 aid);
+
+#ifdef CONFIG_PHL_DRV_HAS_NVM
+enum rtw_hal_status
+rtw_hal_extract_efuse_info(void *hal, u8 *efuse_map,
+                           enum rtw_efuse_info info_type,
+                           void *value,
+                           u8 size, u8 map_valid);
+
+enum rtw_hal_status
+rtw_hal_get_efuse_size(void *phl, u32 *log_efuse_size, u32 *limit_efuse_size,
+                       u32 *mask_size, u32 *limit_mask_size);
+
+enum rtw_hal_status
+rtw_hal_nvm_apply_dev_cap(void *hal, struct rtw_phl_com_t *phl_com);
+
+#endif /* CONFIG_RTW_DRV_HAS_NVM */
+
+/******************************************************************************
+ *
+ * TX power APIs
+ *
+ *****************************************************************************/
+int rtw_hal_get_pw_lmt_regu_type_from_str(void *hal, enum band_type band, const char *str);
+const char *rtw_hal_get_pw_lmt_regu_str_from_type(void *hal, enum band_type band, u8 regu);
+
+u8 rtw_hal_get_pw_lmt_regu_type(void *hal, enum band_type band);
+const char *rtw_hal_get_pw_lmt_regu_type_str(void *hal, enum band_type band);
+
+bool rtw_hal_pw_lmt_regu_tbl_exist(void *hal, enum band_type band, u8 regu);
+u8 rtw_hal_ext_reg_codemap_search(void *hal, enum band_type band
+	, u16 domain_code, const char *country, const char **reg_name);
+
+bool rtw_hal_get_pwr_lmt_en(void *hal, u8 band_idx);
+
+void rtw_hal_auto_pw_lmt_regu(void *hal);
+void rtw_hal_force_pw_lmt_regu(void *hal,
+	u8 regu_2g[], u8 regu_2g_len, u8 regu_5g[], u8 regu_5g_len, u8 regu_6g[], u8 regu_6g_len);
+
+u16 rtw_hal_get_pwr_constraint(void *hal, u8 band_idx);
+enum rtw_hal_status rtw_hal_set_pwr_constraint(void *hal, u8 band_idx, u16 mb);
+
+enum rtw_hal_status rtw_hal_set_tx_power(void *hal, u8 band_idx,
+					enum phl_pwr_table pwr_table);
+
+enum rtw_hal_status rtw_hal_get_txinfo_power(
+	void *hal, s16 *txinfo_power_dbm);
+/*****************************************************************************/
+
+u32 rtw_hal_get_phy_stat_info(void *hal, enum phl_band_idx hw_band,
+			      enum phl_stat_info_query phy_stat);
+
+enum rtw_hal_status rtw_hal_get_rx_cnt_by_idx(void *hal,
+					      enum phl_band_idx hw_band,
+					      enum phl_rxcnt_idx idx,
+					      u16 *rx_cnt);
+enum rtw_hal_status rtw_hal_set_reset_rx_cnt(void *hal,
+					     enum phl_band_idx hw_band);
+#ifdef RTW_WKARD_AP_MP
+void rtw_hal_bb_rx_ndp_mp(void *hal);
+enum rtw_hal_status
+rtw_hal_mac_set_rxfltr_mp_mode(void *hal, u8 band, u16 size);
+#endif /*RTW_WKARD_AP_MP */
+
+enum rtw_hal_status rtw_hal_bb_dump_tpu_info (void *hal);
+
+u16 rtw_hal_get_ampdu_num(void *hal, u8 band);
+
+enum rtw_chip_id rtw_hal_get_chip_id(void *hal);
+
+#ifdef CONFIG_PHL_BCN_ERLY_RPT
+enum rtw_hal_status rtw_hal_set_bcn_early_rpt(void *hal, u8 band, u8 port, u8 en);
+#endif
+
+#ifdef DBG_DUMP_TX_COUNTER
+void rtw_hal_dump_tx_status(void *hal, enum phl_band_idx bidx);
+#endif
+u8 rtw_hal_query_group_cntry_num(void *hal,
+	struct rtw_regu_policy *policy, u8 group_id);
+
+void rtw_hal_fill_group_cntry_list(
+	void *hal, struct rtw_regu_policy *policy,
+	char* list, u32 group_size, u8 group_id);
+
+u8 rtw_hal_get_cntry_idx(void *hal, char *cntry);
+
+u8 rtw_hal_get_cntry_tbl_size(void *hal);
+
+void rtw_hal_qry_cntry_chnlplan(
+    void *hal, struct rtw_regulation_country_chplan *chplan,
+    char *country);
+
+u8 rtw_hal_get_domain_regulation(void *hal,
+	u8 domain, u8 band);
+
+u8 rtw_hal_get_domain_idx(void *hal,
+	u8 domain, bool is_6g);
+
+void rtw_hal_get_chplan_update_info(
+	void *hal, u8 group, u8 did,
+	void *info, enum band_type band);
+
+void rtw_hal_get_chdef_6g(void *hal, u8 ch_idx,
+	struct chdef_6ghz *chdef);
+
+void rtw_hal_get_6g_regulatory_info(void *hal, u8 domain,
+	u8 *dm_code, u8 *regulation, u8 *ch_idx);
+
+bool rtw_hal_get_regu_func_cert_info(void *hal, char *country,
+	struct rtw_regu_func_cert_info *rg_cert);
+
+u8 rtw_hal_get_cat6g_by_country(
+	void * hal, char *country);
+
+u8 rtw_hal_get_chnlplan_ver(void *hal);
+
+u8 rtw_hal_get_country_ver(void *hal);
+
+void rtw_hal_cfg_mac_dump_setting(void *hal, u8 cfg);
+
+enum rtw_hal_status rtw_hal_pwr_switch_mac(void *hal, bool on);
+
+enum rtw_hal_status rtw_hal_cfg_hw_cts2self(void *hal, u8 band_sel, u8 enable,
+					    u8 non_sec_thr, u8 sec_thr);
+bool
+rtw_hal_check_sta_has_busy_wp(struct rtw_phl_stainfo_t *sta);
+
+enum phl_band_idx rtw_hal_phy_idx_to_hw_band(enum phl_phy_idx p_idx);
+
+enum phl_phy_idx rtw_hal_hw_band_to_phy_idx(enum phl_band_idx band_idx);
+
+enum rtw_hal_status rtw_hal_set_append_fcs(void *hal, u8 enable);
+
+
+enum rtw_hal_status rtw_hal_get_freerun_counter(void *hal, u8 band, u32 *freerun_h, u32 *freerun_l);
+enum rtw_hal_status rtw_hal_reset_freerun_counter(void *hal, u8 band);
+
+
+#ifdef CONFIG_SMART_ANTENNA
+
+void rtw_hal_bb_get_antenna_info(void *hal, struct rtw_phl_smart_ant_info_t *antenna_info, bool reset_cnt);
+void rtw_hal_mac_get_rx_cnt_info(void *hal, u8 cur_phy_idx, u8 type_idx, u32 *ret_value);
+
+#endif
+
+#ifdef CONFIG_PHL_DIAGNOSE
+enum rtw_hal_status rtw_hal_rf_query_diag_err_code(
+	void *hal, u32 *err_code);
+enum rtw_hal_status rtw_hal_rf_query_diag_info_len(u32 *len);
+enum rtw_hal_status rtw_hal_rf_query_diag_info(void *hal,
+	struct rtw_phl_diag_rf_info *info);
+
+enum rtw_hal_status rtw_hal_bb_query_diag_info_len_ver(u32 *len, u8 *ver, u8 type);
+enum rtw_hal_status rtw_hal_get_pmac_info(void *hal, u8 hw_band,
+							u8 *dbg_info, u32 len);
+enum rtw_hal_status rtw_hal_get_utility_info(void *hal, u8 hw_band,
+							u8 *dbg_info, u32 len);
+#endif
+
+enum rtw_hal_status
+rtw_hal_update_tas_def_setting(void *hal, u32 tas_config);
+
+enum rtw_hal_status
+rtw_hal_tas_en(void *hal, u8 en);
+
+void rtw_hal_get_mac_sel_tx_status(void *hal, enum phl_band_idx bidx, void *out_tx_cnt);
+
+#endif /*_HAL_API_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_bb.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_bb.c
new file mode 100644
index 000000000000..2b0cff217eb1
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_bb.c
@@ -0,0 +1,3697 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_API_BB_C_
+#include "hal_headers.h"
+#ifdef USE_TRUE_PHY
+#include "phy/bb/halbb_features.h"
+#include "phy/bb/halbb_export_fun.h"
+
+//kevin-cmd
+#include "phy/bb/halbb_hw_cfg_ex.h"
+
+#ifdef CONFIG_PHL_DIAGNOSE
+void rtw_hal_bb_diagnostic_event(struct rtw_hal_com_t *hal, u8 type,
+		u8 level, u8 version, u8 *buf, u32 len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal->hal_priv;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+
+	rtw_phl_send_diag_hub_msg(phl_com, PHL_DIAG_EVT_BB,
+				       type, level, version, buf, len);
+}
+#endif
+
+void rtw_hal_bb_bb_reset_cmn(struct rtw_hal_com_t *hal, bool en, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal->hal_priv;
+
+	halbb_bb_reset_cmn(hal_info->bb, en, phy_idx);
+}
+
+void rtw_hal_bb_dfs_en(struct hal_info_t *hal_info, bool en)
+{
+	halbb_dfs_en(hal_info->bb, en);
+}
+
+void rtw_hal_bb_tssi_cont_en(struct hal_info_t *hal_info, bool en, enum rf_path path)
+{
+	halbb_tssi_cont_en(hal_info->bb, en, path);
+}
+
+void rtw_hal_bb_adc_en(struct hal_info_t *hal_info, bool en, enum phl_phy_idx phy_idx)
+{
+	halbb_adc_ctrl_en(hal_info->bb, en, phy_idx);
+}
+
+void rtw_hal_bb_reset_en(struct hal_info_t *hal_info, bool en, enum phl_phy_idx phy_idx)
+{
+	halbb_bb_reset_en(hal_info->bb, en, phy_idx);
+}
+
+bool rtw_hal_bb_proc_cmd(struct hal_info_t *hal_info, struct rtw_proc_cmd *incmd,
+						char *output, u32 out_len)
+{
+	if(incmd->in_type == RTW_ARG_TYPE_BUF)
+		halbb_cmd(hal_info->bb, incmd->in.buf, output, out_len);
+	else if(incmd->in_type == RTW_ARG_TYPE_ARRAY){
+		halbb_cmd_parser(hal_info->bb, incmd->in.vector,
+					incmd->in_cnt_len, output, out_len);
+	}
+
+	return true;
+}
+
+enum rtw_hal_status rtw_hal_bb_watchdog(struct hal_info_t *hal_info, u8 is_lps)
+{
+	enum bb_watchdog_mode_t mode = BB_WATCHDOG_NORMAL;
+
+	if (is_lps)
+		mode = BB_WATCHDOG_LOW_IO;
+
+	halbb_watchdog(hal_info->bb, mode, HW_PHY_0);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_simple_watchdog(struct hal_info_t *hal_info, u8 io_en)
+{
+	enum bb_watchdog_mode_t mode = BB_WATCHDOG_NON_IO;
+
+	if (io_en)
+		mode = BB_WATCHDOG_LOW_IO;
+
+	halbb_watchdog(hal_info->bb, mode, HW_PHY_0);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_reset(struct hal_info_t *hal_info)
+{
+	halbb_watchdog_reset(hal_info->bb);
+}
+
+void rtw_hal_bb_fw_edcca(struct hal_info_t *hal_info)
+{
+	PHL_INFO("[Cert], %s() ==> !! \n", __func__);
+
+	halbb_fw_edcca(hal_info->bb);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_dm_init(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	hsts = halbb_dm_init(hal_info->bb, HW_PHY_0);
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_BB_DM_INIT);
+#endif /* DBG_MONITOR_TIME */
+
+	return hsts;
+}
+
+void rtw_hal_bb_dm_deinit(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info)
+{
+	halbb_dm_deinit(phl_com, hal_info->bb);
+}
+
+bool rtw_hal_bb_query_cck_en(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+	bool cck_en = false;
+
+	cck_en = halbb_query_cck_en(hal->bb, phy_idx);
+	return cck_en;
+}
+
+enum rtw_hal_status rtw_hal_bb_ctrl_cck_en(struct rtw_hal_com_t *hal_com,
+					bool cca_en, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	halbb_ctrl_cck_en(hal->bb, cca_en, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_ctrl_rx_cca(struct rtw_hal_com_t *hal_com,
+	bool cca_en, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	halbb_ctrl_rx_cca(hal->bb, cca_en, phy_idx);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_DBCC_SUPPORT
+void
+rtw_hal_dbcc_init_bb_reg(struct hal_info_t *hal_info)
+{
+
+	halbb_init_bb_cr_per_phy(hal_info->bb, HW_PHY_1);
+	halbb_reset_bb(hal_info->bb);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_dbcc(struct hal_info_t *hal_info, bool dbcc_en)
+{
+	halbb_ctrl_dbcc(hal_info->bb, dbcc_en);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_dbcc_phy_map(struct hal_info_t *hal_info,
+					   enum phl_phy_idx phy_idx)
+{
+	if (halbb_cfg_dbcc_cck_phy_map(hal_info->bb, phy_idx))
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_dbcc(struct hal_info_t *hal_info,
+				struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct bb_dbcc_cfg_info cfg = {0};
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	cfg.dbcc_en = dbcc_en;
+	cfg.cck_phy_map = hal_dbcc_cck_phyidx_decision(hal_info, phl_com, dbcc_en);
+	if (halbb_cfg_dbcc(hal_info->bb, &cfg)) {
+		hsts = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_ERR("%s Fail \n", __FUNCTION__);
+	}
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_phy_dbcc_pre_cfg(struct hal_info_t *hal_info,
+			struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	if (dbcc_en)
+		rtw_hal_dbcc_init_bb_reg(hal_info);
+	else
+		rtw_hal_bb_cfg_dbcc_phy_map(hal_info, HW_PHY_0);
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+		return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_phy_dbcc_cfg(struct hal_info_t *hal_info,
+	struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	/* BB DBCC Settings */
+	hsts = rtw_hal_bb_cfg_dbcc(hal_info, phl_com, dbcc_en);
+	if (RTW_HAL_STATUS_SUCCESS != hsts) {
+		PHL_ERR("%s halbb cfg dbcc failed\n", __func__);
+		goto _err;
+	}
+	/* enable/disable rf */
+	hsts = rtw_hal_rf_ctrl_dbcc(hal_info->hal_com, dbcc_en);
+	if (RTW_HAL_STATUS_SUCCESS != hsts) {
+		PHL_ERR("%s halrf ctrl dbcc failed\n", __func__);
+		goto _err;
+	}
+
+_err:
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+u32 rtw_hal_bb_init(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status;
+
+	hal_status = halbb_buffer_init(phl_com, hal_com, &(hal_info->bb));
+
+	if ((hal_status != RTW_HAL_STATUS_SUCCESS) || (hal_info->bb == NULL))
+		PHL_ERR("[PHL] rtw_hal_bb_init failed status(%d), hal_info->bb(%p)\n",
+			hal_status, hal_info->bb);
+
+	return hal_status;
+}
+
+void rtw_hal_bb_deinit(struct rtw_phl_com_t *phl_com,
+						struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	halbb_buffer_deinit(phl_com, hal_com, hal_info->bb);
+}
+
+void rtw_hal_init_bb_early_init(struct hal_info_t *hal_info)
+{
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	halbb_early_init(hal_info->bb);
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_INIT_BB_REG1);
+#endif /* DBG_MONITOR_TIME */
+
+}
+
+void rtw_hal_init_bb_reg(struct hal_info_t *hal_info)
+{
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	halbb_init_reg(hal_info->bb);
+	halbb_reset_bb(hal_info->bb);
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_INIT_BB_REG2);
+#endif /* DBG_MONITOR_TIME */
+}
+
+u8 rtw_hal_ex_cn_report(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_ex_cn_report(hal->bb);
+}
+
+u8 rtw_hal_ex_evm_1ss_report(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_ex_evm_1ss_report(hal->bb);
+}
+
+u8 rtw_hal_ex_evm_max_report(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_ex_evm_max_report(hal->bb);
+}
+
+u8 rtw_hal_ex_evm_min_report(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_ex_evm_min_report(hal->bb);
+}
+
+u32 rtw_hal_read_rf_reg(struct rtw_hal_com_t *hal_com,
+			enum rf_path path, u32 addr, u32 mask)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_read_rf_reg(hal->bb, path, addr, mask);
+}
+
+bool rtw_hal_write_rf_reg(struct rtw_hal_com_t *hal_com,
+			  enum rf_path path, u32 addr, u32 mask, u32 data)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_write_rf_reg(hal->bb, path, addr, mask, data);
+}
+
+u32 rtw_hal_read_bb_reg(struct rtw_hal_com_t *hal_com, u32 addr, u32 mask)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_get_reg(hal->bb, addr, mask);
+}
+
+bool rtw_hal_write_bb_reg(struct rtw_hal_com_t *hal_com,
+			  u32 addr, u32 mask, u32 data)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	halbb_set_reg(hal->bb, addr, mask, data);
+	return true;
+}
+
+u32 rtw_hal_bb_read_cr(struct rtw_hal_com_t *hal_com, u32 addr, u32 mask)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_rf_get_bb_reg(hal->bb, addr, mask);
+}
+
+bool rtw_hal_bb_write_cr(struct rtw_hal_com_t *hal_com,
+			     u32 addr, u32 mask, u32 data)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halbb_rf_set_bb_reg(hal->bb, addr, mask, data);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_init(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (halbb_sta_info_init(hal_info->bb, sta))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("[HAL] halbb_stainfo_init failed status(%d)\n",
+			hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_deinit(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (halbb_sta_info_deinit(hal_info->bb, sta))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("[HAL] rtw_hal_bb_stainfo_deinit failed status(%d)\n",
+			hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_add(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (halbb_sta_info_add_entry(hal_info->bb, sta))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("[HAL] halbb_stainfo_init failed status(%d)\n",
+			hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_delete(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (halbb_sta_info_delete_entry(hal_info->bb, sta))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("[HAL] rtw_hal_bb_stainfo_deinit failed status(%d)\n",
+			hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_lps_info_update(void *hal, u16 macid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (halbb_lps_info(hal_info->bb, macid))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("[HAL] %s failed status(%d)\n",
+			__func__, hal_status);
+
+	return hal_status;
+}
+
+void rtw_hal_bb_media_status_update(struct hal_info_t *hal_info,
+			struct rtw_phl_stainfo_t *sta, bool is_connected)
+{
+	halbb_media_status_update(hal_info->bb, sta, is_connected);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_upt_ramask(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	sta->hal_sta->ra_info.ra_bw_mode = PHL_STA_TX_BW_MODE(sta);
+
+	if (!rtw_halbb_dft_mask(hal_info->bb, sta)) {
+		PHL_ERR("rtw_halbb_set_dft_mask failed\n");
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("sta's cur_ra_mask : 0x%016llx\n", sta->hal_sta->ra_info.cur_ra_mask);
+
+	if (sta->hal_sta->ra_info.ra_mask) {
+		sta->hal_sta->ra_info.cur_ra_mask &= sta->hal_sta->ra_info.ra_mask;
+		PHL_INFO("ra_mask : 0x%016llx, cur_ra_mask : 0x%016llx\n",
+			sta->hal_sta->ra_info.ra_mask, sta->hal_sta->ra_info.cur_ra_mask);
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ra_register(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	if (!rtw_halbb_raregistered(hal_info->bb, sta))
+		return RTW_HAL_STATUS_FAILURE;
+
+	sta->hal_sta->ra_info.ra_registered = true;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ra_deregister(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	if (!rtw_halbb_ra_deregistered(hal_info->bb, sta))
+		PHL_ERR("rtw_halbb_ra_deregistered failed\n");
+	sta->hal_sta->ra_info.ra_nss_limit = 0;
+	sta->hal_sta->ra_info.ra_registered = false;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ra_update(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_sts = RTW_HAL_STATUS_FAILURE;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	if (!sta->hal_sta->ra_info.ra_registered) {
+		PHL_WARN("%s mac-id:%d not register RA\n", __func__, sta->macid);
+		hal_sts = RTW_HAL_STATUS_SUCCESS;
+		goto exit;
+	}
+
+	sta->hal_sta->ra_info.ra_bw_mode = PHL_STA_TX_BW_MODE(sta);
+
+	if (sta->hal_sta->ra_info.ra_mask) {
+		sta->hal_sta->ra_info.cur_ra_mask &= sta->hal_sta->ra_info.ra_mask;
+		PHL_INFO("ra_mask : 0x%016llx, cur_ra_mask : 0x%016llx\n",
+			sta->hal_sta->ra_info.ra_mask, sta->hal_sta->ra_info.cur_ra_mask);
+	}
+
+	if (rtw_halbb_raupdate(hal_info->bb, sta) == true) {
+		hal_sts = RTW_HAL_STATUS_SUCCESS;
+		goto exit;
+	} else {
+		PHL_WARN("%s: macid:0x%x update fail\n", __func__, sta->macid);
+		hal_sts = RTW_HAL_STATUS_FAILURE;
+		goto exit;
+	}
+exit:
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hal_sts;
+}
+
+#ifdef CONFIG_BTCOEX
+/*
+ * Origin 2T2R
+ *  tx_en,  rx_en,  tx_res,  rx_res
+ *     1         1         0          0             2T2R >> 1T1R
+ *     0         0         1          1             1T1R >> 2T2R
+ *     1         0         0          0             2T2R >> 1T2R
+ *     0         0         1          0             1T2R >> 2T2R
+ */
+enum rtw_hal_status rtw_hal_btc_cfg_1ss(struct rtw_hal_com_t *hal_c,
+		struct rtw_phl_com_t *phl_c, enum band_type band,
+		bool tx_en, bool rx_en, bool tx_res, bool rx_res)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	enum rtw_phl_status psts = RTW_PHL_STATUS_FAILURE;
+	struct hal_info_t *hal_i = (struct hal_info_t *)hal_c->hal_priv;
+	void *drv = halcom_to_drvpriv(hal_c);
+	struct rtw_wifi_role_t *wr = NULL;
+	struct rtw_wifi_role_link_t *rlk = NULL;
+	struct phl_queue *sta_queue = NULL;
+	struct rtw_phl_stainfo_t *sta = NULL;
+	struct rtw_phl_stainfo_t **sta_todo = NULL;
+	size_t sta_todo_len = 0;
+	size_t i;
+	int sta_qlen = 0;
+	u8 ra_nss_lim, ridx = 0, lidx = 0, map_idx = 0;
+	u32 macid_map[2] = {0};
+	u16 bit_num = (u16)(sizeof(macid_map) * 8);
+	struct phl_msg msg = {0};
+	enum phl_msg_evt_id evt = MSG_EVT_NONE;
+
+	PHL_WARN("%s: Interferential ch-band(%d), tx_en(%d), rx_en(%d), tx_res(%d), rx_res(%d)\n",
+		__FUNCTION__, band, tx_en, rx_en, tx_res, rx_res);
+	if (!(GET_DEV_BTC_CAP(phl_c).btc_deg_wifi_cap & BTC_DRG_WIFI_CAP_TRX1SS)) {
+		PHL_ERR("%s: Don't support, btc_deg_wifi_cap(0x%x)\n",
+			__FUNCTION__, GET_DEV_BTC_CAP(phl_c).btc_deg_wifi_cap);
+		goto exit;
+	}
+	if ((tx_en && tx_res) || (rx_en && rx_res)) {
+		PHL_ERR("%s: error para\n", __FUNCTION__);
+		goto exit;
+	}
+	if (tx_en && !tx_res)
+		ra_nss_lim = 1;
+	else
+		ra_nss_lim = 0;
+	for (ridx = 0; ridx < MAX_WIFI_ROLE_NUMBER; ridx++) {
+		wr = &(phl_c->wifi_roles[ridx]);
+		if (false == wr->active || MLME_LINKED != wr->mstate)
+			continue;
+		if (!rtw_phl_role_is_ap_category(wr) &&
+		    !rtw_phl_role_is_client_category(wr))
+			continue;
+		for (lidx = 0; lidx < RTW_RLINK_MAX; lidx++) {
+			rlk = &(wr->rlink[lidx]);
+			if (MLME_LINKED != rlk->mstate)
+				continue;
+			if (band != rlk->chandef.band)
+				continue;
+			sta_queue = &rlk->assoc_sta_queue;
+			sta_todo = NULL;
+			sta_todo_len = 0;
+			/* collect the sta list */
+			_os_spinlock(drv, &sta_queue->lock, _bh, NULL);
+			sta_qlen = sta_queue->cnt;
+			sta_todo = (struct rtw_phl_stainfo_t **)
+				_os_kmem_alloc(drv, sta_qlen * sizeof(struct rtw_phl_stainfo_t *));
+			if (!sta_todo) {
+				PHL_ERR("%s: failed to alloc sta_todo\n", __FUNCTION__);
+				hsts = RTW_HAL_STATUS_RESOURCE;
+				_os_spinunlock(drv, &sta_queue->lock, _bh, NULL);
+				goto exit;
+			}
+			phl_list_for_loop(sta, struct rtw_phl_stainfo_t,
+			                  &sta_queue->queue, list) {
+				if (sta) {
+					sta_todo[sta_todo_len++] = sta;
+					if (sta->macid < bit_num) {
+						map_idx = (u8)(sta->macid / 32);
+						macid_map[map_idx] |=  BIT(sta->macid % 32);
+					} else {
+						PHL_ERR("%s: macid(%d) > macid_map\n",
+							__FUNCTION__, sta->macid);
+					}
+				}
+			}
+			_os_spinunlock(drv, &sta_queue->lock, _bh, NULL);
+			if (!tx_en && !tx_res)
+				goto _next_loop; /* no need to cfg tx */
+			for (i = 0; i < sta_todo_len; i++) {
+				sta = sta_todo[i];
+				sta->hal_sta->ra_info.ra_nss_limit = ra_nss_lim;
+				hsts = rtw_hal_bb_ra_update(hal_i, sta);
+				if (RTW_HAL_STATUS_SUCCESS != hsts) {
+					PHL_ERR("%s: macid(%d), Fail to cfg ra_nss_limit(%d)\n",
+					        __FUNCTION__, sta->macid, ra_nss_lim);
+				} else {
+					PHL_WARN("%s: macid(%d), succee to cfg ra_nss_limit(%d)\n",
+					         __FUNCTION__, sta->macid, ra_nss_lim);
+				}
+			}
+			_next_loop:
+			_os_kmem_free(drv, sta_todo, sta_qlen * sizeof(struct rtw_phl_stainfo_t *));
+		}
+	}
+	if (!rx_en && !rx_res)
+		goto exit; /* no need to cfg rx */
+	if (rx_en && !rx_res)
+		evt = MSG_EVT_ANN_RX1SS;
+	else if (!rx_en && rx_res)
+		evt = MSG_EVT_ANN_RX_MAXSS;
+	else
+		goto exit;
+	msg.rsvd[0].value = macid_map[0]; /* macid 0~31 */
+	msg.rsvd[1].value = macid_map[1]; /* macid 32~63 */
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_GENERAL);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, evt);
+	PHL_WARN("%s: macid map(0x%08x-%08x) need to announce rx nss to %d\n",
+		__FUNCTION__, (u32)msg.rsvd[1].value, (u32)msg.rsvd[0].value, evt);
+	psts = rtw_phl_msg_hub_hal_send(phl_c, NULL, &msg);
+	if (RTW_PHL_STATUS_SUCCESS != psts) {
+		PHL_ERR("%s: Send msg fail\n", __FUNCTION__);
+		hsts = RTW_HAL_STATUS_FAILURE;
+	} else {
+		PHL_WARN("%s: Send msg ok\n", __FUNCTION__);
+	}
+exit:
+	return hsts;
+}
+
+enum rtw_hal_status rtw_hal_btc_cfg_trx_path(struct rtw_hal_com_t *hal_c,
+		enum rf_path tx, u8 tx_nss, enum rf_path rx, u8 rx_nss)
+{
+	return rtw_hal_bb_trx_path_cfg(hal_c->hal_priv, tx, tx_nss, rx, rx_nss);
+}
+#endif /* CONFIG_BTCOEX */
+
+enum rtw_hal_status
+rtw_hal_bb_query_txsts_rpt(struct hal_info_t *hal_info,
+				u16 macid0, u16 macid1)
+{
+	if (!rtw_halbb_query_txsts(hal_info->bb, macid0, macid1))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#define INVALID_ARFR_IDX 0xFF
+#define rtw_halbb_get_arfr_idx(_bb, _sta)	INVALID_ARFR_IDX
+u8 rtw_hal_bb_get_arfr_idx(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta)
+{
+	return rtw_halbb_get_arfr_idx(hal_info->bb, sta);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid)
+{
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u8 hal_get_primary_channel_idx(u8 pri_ch,
+				u8 central_ch, enum channel_width bw,
+				enum chan_offset bw_offset)
+{
+	u8 idx = 255;
+
+	if(bw == CHANNEL_WIDTH_80) {
+		if(bw_offset == CHAN_OFFSET_NO_EXT ||
+		   bw_offset == CHAN_OFFSET_NO_DEF) {
+		   PHL_ERR("%s invalid bw offset\n",__FUNCTION__);
+		   return idx;
+		}
+		if (central_ch > pri_ch)
+			idx = (bw_offset == CHAN_OFFSET_UPPER) ? (4) : (2);
+		else
+			idx = (bw_offset == CHAN_OFFSET_UPPER) ? (1) : (3);
+	}
+	else if(bw == CHANNEL_WIDTH_40) {
+		if(bw_offset == CHAN_OFFSET_NO_EXT ||
+		   bw_offset == CHAN_OFFSET_NO_DEF) {
+		   PHL_ERR("%s invalid bw offset\n",__FUNCTION__);
+		   return idx;
+		}
+		idx = (bw_offset == CHAN_OFFSET_UPPER) ? (2) : (1);
+	}
+	else {
+		idx = 0;
+	}
+	PHL_INFO("Using SC index %u for P%u C%u B%u O%u\n",
+		 idx, pri_ch, central_ch, bw, bw_offset);
+	return idx;
+}
+
+enum rtw_hal_status rtw_hal_bb_set_ch_bw(struct hal_info_t *hal_info,
+					enum phl_phy_idx phy_idx,
+					u8 pri_ch,
+					u8 central_ch_seg0,
+					u8 central_ch_seg1,
+					enum band_type band,
+					enum channel_width bw)
+{
+	if(halbb_ctrl_bw_ch(hal_info->bb, pri_ch, central_ch_seg0,
+				central_ch_seg1, band, bw, phy_idx) == false)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+/**
+ * Early Weak Interference Rejection (EWIR)
+ * rtw_hal_bb_set_pop_en: configure bb feature about packet on packet.
+ * @hal_info: see struct hal_info_t
+ * @en: enable: true, disable: false
+ * @phy_idx: corresponding to the hw_band in wifi_role.
+ * returns enum rtw_hal_status
+ */
+enum rtw_hal_status rtw_hal_bb_set_pop_en(struct hal_info_t *hal_info,
+                                          bool en,
+                                          enum phl_phy_idx phy_idx)
+{
+	PHL_INFO(" %s, pop_enable(%d)\n", __FUNCTION__, en);
+	halbb_pop_en(hal_info->bb, en, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * Early Weak Interference Rejection (EWIR)
+ * rtw_hal_bb_query_pop_en: query the status of packet on packet operation.
+ * @hal_info: see struct hal_info_t
+ * @phy_idx: corresponding to the hw_band in wifi_role.
+ * returns true: running; false: not running.
+ */
+bool rtw_hal_bb_query_pop_en(struct hal_info_t *hal_info,
+                             enum phl_phy_idx phy_idx)
+{
+	return halbb_querry_pop_en(hal_info->bb, phy_idx);
+}
+
+/**
+ * rtw_hal_bb_set_pkt_detect_thold: configure packet threshold(dbm) detection
+ * It would pause DIG while pd threshold enabling(value > 0)
+ * @hal_info: see struct hal_info_t
+ * @bound: pd threshold value
+ * returns enum rtw_hal_status
+ */
+enum rtw_hal_status rtw_hal_bb_set_pkt_detect_thold(struct hal_info_t *hal_info,
+				enum phl_band_idx band_idx, u32 bound)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	enum halbb_pause_type pause_type = HALBB_PAUSE;
+	u8 pause_result = 0;
+	u32 val[5] = {0};
+	val[0] = bound;
+	val[1] = 0;
+	enum phl_phy_idx p_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+
+	if (bound == 0) {
+		pause_type = HALBB_RESUME;
+	} else if ((bound > 102) || (bound < 40)) {
+		PHL_INFO(" %s, invalid boundary!\n", __FUNCTION__);
+		return hal_status;
+	}
+	pause_result = halbb_pause_func(hal_info->bb,
+					F_DIG, pause_type,
+					HALBB_PAUSE_LV_2,
+					2,
+					val, p_idx);
+	if (pause_result == PAUSE_SUCCESS)
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	return hal_status;
+}
+
+/**
+ * rtw_hal_bb_query_pkt_detect_thold: query packet threshold(dbm) value.
+ * @hal_info: see struct hal_info_t
+ * @get_en_info: Caller can confirm the feature running or not with true value.
+ * 1. true: for query pd threshold detection enabling or not.
+ * 2. false:for query pd threshold value.
+ * @phy_idx: corresponding to the hw_band in wifi_role.
+ * returns pd threshold value
+ */
+u8 rtw_hal_bb_query_pkt_detect_thold(struct hal_info_t *hal_info,
+                                     bool get_en_info,
+                                     enum phl_phy_idx phy_idx)
+{
+	return halbb_querry_pd_lower_bound(hal_info->bb, get_en_info, phy_idx);
+}
+
+/**
+ * rtw_hal_bb_set_tx_rate_rty_tbl: set the tx retry count of tx MCS rate table.
+ * @hal_info: see struct hal_info_t
+ * @en: enable: true, disable: false(halbb default setting)
+ * @rty_rate_tbl: tx retry count of tx MCS rate table
+ * returns void
+ */
+void
+rtw_hal_bb_set_tx_rate_rty_tbl(struct hal_info_t *hal_info,
+                               bool en,
+                               u8 *rty_rate_tbl)
+{
+	halbb_ra_shift_darf_tc(hal_info->bb, en ,rty_rate_tbl);
+}
+
+/**
+ * Spatial Reuse for BSS Color
+ * rtw_hal_bb_set_spatial_reuse_en: configure bb feature about spatial reuse.
+ * @hal_info: see struct hal_info_t
+ * @en: enable: true, disable: false
+ * returns enum rtw_hal_status
+ */
+enum rtw_hal_status
+rtw_hal_bb_set_spatial_reuse_en(struct hal_info_t *hal_info, bool en)
+{
+	PHL_INFO(" %s, sr_enable(%d)\n", __FUNCTION__, en);
+	halbb_spatial_reuse_en(hal_info->bb, en);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * Spatial Reuse for BSS Color
+ * rtw_hal_bb_is_spatial_reuse_en: query the status of spatial reuse.
+ * @hal_info: see struct hal_info_t
+ * returns true: running; false: not running.
+ */
+bool
+rtw_hal_bb_is_spatial_reuse_en(struct hal_info_t *hal_info)
+{
+	return halbb_spatial_reuse_is_en(hal_info->bb);
+}
+#endif
+
+#ifdef CONFIG_PHL_DFS
+bool rtw_hal_in_radar_domain(void *hal, enum band_type band, u8 ch
+	, enum channel_width bw, enum chan_offset offset)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return band == BAND_ON_5G
+		/* BE CAREFUL: this function also change detecting parameters */
+		&& halbb_is_dfs_band(hal_info->bb, ch, bw)
+		;
+}
+
+void rtw_hal_bb_dfs_change_domain(void *hal, enum band_type band, u8 ch
+	, enum channel_width bw, enum chan_offset offset)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	halbb_dfs_change_dmn(hal_info->bb, ch, bw);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_dfs_rpt_cfg(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx, bool dfs_en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	if (dfs_en)
+		halbb_radar_detect_enable(hal_info->bb);
+	else
+		halbb_radar_detect_disable(hal_info->bb);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool
+rtw_hal_bb_radar_detect(struct hal_info_t *hal_info,
+				struct hal_dfs_rpt *hal_dfs)
+{
+
+	return halbb_radar_detect(hal_info->bb, hal_dfs);
+}
+#endif /*CONFIG_PHL_DFS*/
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+enum rtw_hal_status
+rtw_hal_bb_ch_info_parsing(struct hal_info_t *hal_info,
+	u8 *addr, struct rtw_r_meta_data *mdata,
+	u8 *rpt_buf, struct ch_rpt_hdr_info *ch_hdr_rpt,
+	struct phy_info_rpt *phy_rpt, struct ch_info_drv_rpt *drv)
+{
+	enum bb_ch_info_t chinfo_status = BB_CH_INFO_SUCCESS;
+	struct physts_rxd rxdesc = {0};
+	u32 len = mdata->pktlen;
+
+	rxdesc.data_rate = mdata->rx_rate;
+	chinfo_status = halbb_ch_info_parsing(hal_info->bb, addr, len, &rxdesc,rpt_buf
+			, (void*)ch_hdr_rpt, (void *)phy_rpt, (void *)drv);
+	if (chinfo_status == BB_CH_INFO_SUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else if (chinfo_status == BB_CH_INFO_LAST_SEG)
+		return RTW_HAL_STATUS_BB_CH_INFO_LAST_SEG;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+void
+_halchinfo2bbchinfo(struct chinfo_bbcr_cfg *cfg, struct bb_ch_info_cr_cfg_info *bb_cfg)
+{
+	bb_cfg->ch_i_data_src = cfg->ch_i_data_src;
+	bb_cfg->ch_i_cmprs = cfg->ch_i_cmprs;
+	bb_cfg->ch_i_grp_num = cfg->ch_i_grp_num_non_he;
+	bb_cfg->ch_i_grp_num_he = cfg->ch_i_grp_num_he;
+	bb_cfg->ch_i_blk_start_idx = cfg->ch_i_blk_start_idx;
+	bb_cfg->ch_i_blk_end_idx = cfg->ch_i_blk_end_idx;
+	bb_cfg->ch_i_ele_bitmap = cfg->ch_i_ele_bitmap;
+	switch(cfg->ch_i_type) {
+	case CH_INFO_LEGACY_CH:
+		bb_cfg->ch_i_type = BB_CH_LEGACY_CH;
+		break;
+	case CH_INFO_MIMO_CH:
+		bb_cfg->ch_i_type = BB_CH_MIMO_CH;
+		break;
+	default:
+		PHL_ERR("%s unknown ch_i_type=%d\n", __func__, cfg->ch_i_type);
+		bb_cfg->ch_i_type = BB_CH_LEGACY_CH;
+		break;
+	}
+	bb_cfg->ch_i_seg_len = cfg->ch_i_seg_len;
+
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - bb_cfg\n");
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_data_src:%d\n", bb_cfg->ch_i_data_src);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_cmprs:%d\n", bb_cfg->ch_i_cmprs);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_grp_num:%d\n", bb_cfg->ch_i_grp_num);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_grp_num_he:%d\n", bb_cfg->ch_i_grp_num_he);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_blk_start_idx:%d\n", bb_cfg->ch_i_blk_start_idx);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_blk_end_idx:%d\n", bb_cfg->ch_i_blk_end_idx);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_ele_bitmap:0x%02x\n", bb_cfg->ch_i_ele_bitmap);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_type:%d\n", bb_cfg->ch_i_type);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,"[CH INFO] - ch_i_seg_len:%d\n", bb_cfg->ch_i_seg_len);
+}
+
+void
+rtw_hal_bb_chan_info_cfg(struct hal_info_t *hal_info, struct chinfo_bbcr_cfg *cfg)
+{
+	struct bb_ch_info_cr_cfg_info bb_cfg;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &bb_cfg, 0, sizeof(struct bb_ch_info_cr_cfg_info));
+	_halchinfo2bbchinfo(cfg, &bb_cfg);
+	halbb_cfg_ch_info_cr(hal_info->bb, &bb_cfg);
+}
+
+void
+rtw_hal_bb_ch_info_status_en(struct hal_info_t *hal_info, bool en, enum phl_phy_idx phy_idx)
+{
+	PHL_INFO("[CH INFO] %s: en=%d\n", __func__, en);
+	halbb_ch_info_status_en(hal_info->bb, en, phy_idx);
+}
+
+void
+rtw_hal_bb_ch_info_physts_en(struct hal_info_t *hal_info, bool en,
+					enum phl_phy_idx phy_idx, enum wlan_mode wmode, bool ack_mode)
+{
+	u16 bitmap = 0;
+	if (!ack_mode) {
+		if (wmode & WLAN_MD_11AX)
+			bitmap = bitmap | BIT(HE_PKT);
+		else if (wmode & WLAN_MD_11AC)
+			bitmap = bitmap | BIT(VHT_PKT);
+		else if (wmode & WLAN_MD_11N)
+			bitmap = bitmap | BIT(HT_PKT);
+		else
+			bitmap = bitmap | BIT(LEGACY_OFDM_PKT);
+	} else {
+		bitmap = bitmap | BIT(LEGACY_OFDM_PKT);
+	}
+
+	PHL_INFO("[CH INFO] %s: en=%d, wmode=0x%x, bitmap=0x%02x\n",
+		__func__, en, wmode, bitmap);
+	halbb_ch_info_physts_en(hal_info->bb, en, bitmap, phy_idx);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ch_info_decision(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx,
+								enum channel_width bw, enum wlan_mode wmode,
+								bool *valid_ch_info_physts)
+{
+	 struct bb_ch_rpt_size_info exp_rpt_size = {0};
+	 enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+	halbb_ch_info_size_query(hal_info->bb, &exp_rpt_size, phy_idx);
+
+	if (bw > CHANNEL_WIDTH_160) {
+		PHL_ERR("%s unknown bw=%d\n", __func__, bw);
+		status = RTW_HAL_STATUS_FAILURE;
+		goto exit;
+	}
+
+	if (wmode & WLAN_MD_11AX)
+		*valid_ch_info_physts = exp_rpt_size.valid_size_physts_he[bw];
+	else if (wmode & WLAN_MD_11AC)
+		*valid_ch_info_physts = exp_rpt_size.valid_size_physts_vht[bw];
+	else if (wmode & WLAN_MD_11N)
+		*valid_ch_info_physts = exp_rpt_size.valid_size_physts_ht[bw];
+	else
+		*valid_ch_info_physts = exp_rpt_size.valid_size_physts_lgcy[bw];
+
+	PHL_INFO("[CH INFO] %s: bw=%d, wmode=0x%x, valid_ch_info_physts=%d\n",
+		__func__, bw, wmode, *valid_ch_info_physts);
+
+exit:
+	return status;
+}
+
+void rtw_hal_bb_ch_trig_select(struct hal_info_t *hal_info, u8 event)
+{
+	enum bb_ch_info_event_type event_type;
+
+	if (event == 0)
+		event_type = CH_RPT_TRIG_ONCE;
+	else if (event == 1)
+		event_type = CH_RPT_TRIG_ALWAYS;
+	else {
+		PHL_ERR("%s: unknown event(%d)\n", __func__, event);
+		event_type = CH_RPT_TRIG_ALWAYS;
+	}
+
+	halbb_ch_trig_select(hal_info->bb, event_type);
+}
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_btg(struct rtw_hal_com_t *hal_com, bool btg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("[HAL] call rtw_hal_bb_ctrl_btg !!!\n");
+
+	halbb_ctrl_btg(hal_info->bb, btg);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_btc_preagc(struct rtw_hal_com_t *hal_com, bool bt_en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("[HAL] call rtw_hal_bb_ctrl_btc_preagc !!!\n");
+
+	halbb_ctrl_btc_preagc(hal_info->bb, bt_en);
+
+	return status;
+}
+
+void
+rtw_hal_bb_npath_en_update(struct rtw_hal_com_t *hal_com, bool npath_en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("[HAL] call rtw_hal_bb_npath_en_update !!!\n");
+	halbb_npath_en_update(hal_info->bb, npath_en);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_rx_path(struct rtw_hal_com_t *hal_com, u8 rx_path, u8 phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("[HAL] call halbb_ctrl_rx_path !!!\n");
+
+	status = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, true);
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_reset PHY0 en - failed\n", __func__);
+		_os_warn_on(1);
+	}
+
+	if (hal_com->dbcc_en) {
+		status = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, true);
+		if(status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_reset PHY1 en - failed\n",
+			        __func__);
+			_os_warn_on(1);
+		}
+	}
+
+	if(halbb_ctrl_rx_path(hal_info->bb, rx_path, phy_idx)) {
+		halbb_ctrl_rf_mode_rx_path(hal_info->bb, rx_path);
+		status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	status = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, false);
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_reset PHY0 dis - failed\n", __func__);
+		_os_warn_on(1);
+	}
+
+	if (hal_com->dbcc_en) {
+		status = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, false);
+		if(status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_reset PHY1 dis - failed\n",
+			        __func__);
+			_os_warn_on(1);
+		}
+	}
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_tx_path(struct rtw_hal_com_t *hal_com, u8 tx_path, u8 phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("[HAL] call halbb_ctrl_tx_path !!!\n");
+
+        /* add for bb reset start */
+	status = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, true);
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_reset PHY0 en - failed\n", __func__);
+		_os_warn_on(1);
+	}
+
+	if (hal_com->dbcc_en) {
+		status = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, true);
+		if(status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_reset PHY1 en - failed\n",
+			        __func__);
+			_os_warn_on(1);
+		}
+	}
+
+	if(halbb_ctrl_tx_path_pmac(hal_info->bb, tx_path, phy_idx))
+		status = RTW_HAL_STATUS_SUCCESS;
+	else
+		status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, false);
+	if(status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_reset PHY0 dis - failed\n", __func__);
+		_os_warn_on(1);
+	}
+
+	if (hal_com->dbcc_en) {
+		status = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, false);
+		if(status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_reset PHY1 dis - failed\n",
+			        __func__);
+			_os_warn_on(1);
+		}
+	}
+
+	return status;
+}
+
+enum rtw_hal_status  rtw_hal_bb_get_rx_ok(struct hal_info_t *hal_info, u8 cur_phy_idx, u32 *rx_ok)
+{
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL BB API]%s\n", __FUNCTION__);
+
+	*rx_ok = halbb_mp_get_rx_crc_ok(hal_info->bb, cur_phy_idx);
+
+	return ret;
+}
+
+enum rtw_hal_status  rtw_hal_bb_get_rx_crc(struct hal_info_t *hal_info, u8 cur_phy_idx, u32 *rx_crc_err)
+{
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL BB API]%s\n", __FUNCTION__);
+
+	*rx_crc_err = halbb_mp_get_rx_crc_err(hal_info->bb, cur_phy_idx);
+
+	return ret;
+}
+
+enum rtw_hal_status  rtw_hal_bb_set_reset_cnt(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_INFO("[HAL] call halbb_mp_seset_cnt !!!\n");
+	halbb_mp_reset_cnt(hal_info->bb);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_power(struct rtw_hal_com_t *hal_com, s16 power_dbm,
+					 enum phl_phy_idx phy_idx)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("[MP HAL API] %s: call halbb_set_txpwr_dbm \n", __FUNCTION__);
+	PHL_INFO("[MP HAL API] %s: power_dbm = %d\n", __FUNCTION__, power_dbm);
+
+	if(halbb_set_txpwr_dbm(hal_info->bb, power_dbm, phy_idx))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	else
+		hal_status = RTW_HAL_STATUS_FAILURE;
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_get_power(struct rtw_hal_com_t *hal_com, s16 *power_dbm,
+					 enum phl_phy_idx phy_idx)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("[MP HAL API] %s: call halbb_get_txpwr_dbm \n", __FUNCTION__);
+	*power_dbm = halbb_get_txpwr_dbm(hal_info->bb, phy_idx);
+	PHL_INFO("[MP HAL API] %s: power_dbm = %d\n", __FUNCTION__, *power_dbm);
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pwr_index(void *hal, u16 pwr_idx, enum rf_path tx_path, bool is_cck)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_INFO("[MP HAL API] %s: call halbb_set_txpwr_idx\n", __FUNCTION__);
+
+	if (is_cck) {
+		if(halbb_set_cck_txpwr_idx(hal_info->bb, pwr_idx, tx_path))
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+		else
+			hal_status = RTW_HAL_STATUS_FAILURE;
+
+	} else {
+		if(halbb_set_ofdm_txpwr_idx(hal_info->bb, pwr_idx, tx_path))
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+		else
+			hal_status = RTW_HAL_STATUS_FAILURE;
+	}
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_pwr_index(void *hal, u16 *pwr_idx, enum rf_path tx_path, bool is_cck){
+	int ret = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_INFO("[MP HAL API] %s: call halbb_get_txpwr_idx\n", __FUNCTION__);
+
+	if (is_cck) {
+		PHL_INFO("[MP HAL API] call halbb_get_cck_txpwr_idx\n");
+		*pwr_idx = halbb_get_cck_txpwr_idx(hal_info->bb, tx_path);
+
+	} else {
+		PHL_INFO("[MP HAL API] call halbb_get_ofdm_txpwr_idx\n");
+		*pwr_idx = halbb_get_ofdm_txpwr_idx(hal_info->bb, tx_path);
+	}
+
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_bb_set_plcp_tx(struct rtw_hal_com_t *hal_com,
+						struct halbb_plcp_info *plcp_tx_struct,
+						struct usr_plcp_gen_in *plcp_usr_info,
+						enum phl_phy_idx plcp_phy_idx,
+						u8 *plcp_sts)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("%s\n", __FUNCTION__);
+
+	*plcp_sts = halbb_plcp_gen(hal_info->bb, (void*)plcp_tx_struct,(void*)plcp_usr_info,plcp_phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_cont_tx(struct rtw_hal_com_t *hal_com, u8 enable, u8 is_cck,
+							enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct halbb_pmac_info tx_info = {0};
+
+	PHL_INFO("%s: enable = %d is_cck = %d phy_idx = %d\n",
+			 __FUNCTION__, enable, is_cck, phy_idx);
+
+	tx_info.en_pmac_tx = enable;
+	tx_info.is_cck = is_cck;
+	tx_info.mode = CONT_TX;
+
+	halbb_set_pmac_tx(hal_info->bb, &tx_info, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_carrier_suppression_tx(struct rtw_hal_com_t *hal_com, u8 enable, u8 is_cck,
+							u16 tx_cnt, u16 period, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct halbb_pmac_info tx_info = {0};
+
+	PHL_INFO("%s: enable = %d is_cck = %d phy_idx = %d\n",
+			 __FUNCTION__, enable, is_cck, phy_idx);
+
+	tx_info.en_pmac_tx = enable;
+	tx_info.is_cck = is_cck;
+	tx_info.tx_cnt = tx_cnt;
+	tx_info.period = period;
+	tx_info.mode = CCK_CARRIER_SIPPRESSION_TX;
+
+	halbb_set_pmac_tx(hal_info->bb, &tx_info, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_packet_tx(struct rtw_hal_com_t *hal_com, u8 enable,
+							  u8 is_cck, u16 tx_cnt ,u16 period,
+							  u16 tx_time, u8 cck_lbk_en,
+							  enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct halbb_pmac_info tx_info = {0};
+
+	PHL_INFO("%s: enable = %d is_cck = %d phy_idx = %d\n",
+			 __FUNCTION__, enable, is_cck, phy_idx);
+	PHL_INFO("%s: tx_cnt = %d period = %d tx_time = %d\n",
+			 __FUNCTION__, tx_cnt, period, tx_time);
+
+	tx_info.en_pmac_tx = enable;
+	tx_info.is_cck = is_cck;
+	tx_info.mode = PKTS_TX;
+	tx_info.tx_cnt = tx_cnt;
+	tx_info.period = period;
+	tx_info.tx_time = tx_time;
+	tx_info.cck_lbk_en = cck_lbk_en;
+
+	halbb_set_pmac_tx(hal_info->bb, &tx_info, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_fw_trigger_tx(struct rtw_hal_com_t *hal_com, u8 enable,
+							  u8 is_cck, u16 tx_cnt, u8 tx_duty,
+							  enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct halbb_pmac_info tx_info = {0};
+
+	PHL_INFO("%s: enable = %d is_cck = %d phy_idx = %d\n",
+			 __FUNCTION__, enable, is_cck, phy_idx);
+	PHL_INFO("%s: tx_cnt = %d\n", __FUNCTION__, tx_cnt);
+	PHL_INFO("%s: tx_duty = %d\n", __FUNCTION__, tx_duty);
+
+	tx_info.en_pmac_tx = enable;
+	tx_info.is_cck = is_cck;
+	tx_info.mode = FW_TRIG_TX;
+	tx_info.tx_cnt = tx_cnt;
+	tx_info.duty_cycle = tx_duty;
+
+	halbb_set_pmac_tx(hal_info->bb, &tx_info, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_SNIFFER_SUPPORT
+void __dump_lsig(u32 *_lsig)
+{
+	struct phl_plcp_hdr_l_sig_info *lsig = (struct phl_plcp_hdr_l_sig_info *)_lsig;
+
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "--> %s \n", __func__);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "lsig->rate 0x%x\n", lsig->rate);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "lsig->len0_l 0x%x ; ", lsig->len0_l);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "lsig->len0_m 0x%x\n", lsig->len0_m);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "lsig->len1 0x%x\n", lsig->len1);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "lsig->parity_chk 0x%x\n", lsig->parity_chk);
+}
+
+void __dump_he_sig_a1(u32 *_siga1)
+{
+
+	struct phl_plcp_hdr_he_sig_a1_su_info *siga = (struct phl_plcp_hdr_he_sig_a1_su_info *)_siga1;
+
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "--> %s \n", __func__);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->fmt 0x%x\n", siga->fmt);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->beam_chg 0x%x\n", siga->beam_chg);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->ul_dl 0x%x\n", siga->ul_dl);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->he_mcs 0x%x\n", siga->he_mcs);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->dcm 0x%x\n", siga->dcm);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->bss_color 0x%x\n", siga->bss_color);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->spatial_reuse_l 0x%x ; ", siga->spatial_reuse_l);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->spatial_reuse_m 0x%x\n", siga->spatial_reuse_m);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->bw 0x%x\n", siga->bw);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->gi_ltf_size 0x%x\n", siga->gi_ltf_size);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->nsts_mid_period_l 0x%x ; ", siga->nsts_mid_period_l);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->nsts_mid_period_m 0x%x\n", siga->nsts_mid_period_m);
+}
+
+void __dump_he_sig_a2(u32 *_siga2)
+{
+
+	struct phl_plcp_hdr_he_sig_a2_su_info *siga = (struct phl_plcp_hdr_he_sig_a2_su_info *)_siga2;
+
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "--> %s \n", __func__);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->txop 0x%x\n", siga->txop);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->coding 0x%x\n", siga->coding);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->ldpc_ext_sym 0x%x\n", siga->ldpc_ext_sym);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->stbc 0x%x\n", siga->stbc);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->bf 0x%x\n", siga->bf);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->pre_fec_fac 0x%x\n", siga->pre_fec_fac);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->pe_dis 0x%x\n", siga->pe_dis);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_, "siga->doppler 0x%x\n", siga->doppler);
+}
+
+void _dump_bb_snif_info(struct rtw_phl_rx_pkt *phl_rx)
+{
+	struct rtw_phl_ppdu_phy_info *phy_info = &(phl_rx->r.phy_info);
+	struct phl_snif_info *snif_inf = &phy_info->snif_i;
+
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_,
+		  "snif_mode_en %d ; snif_rate_mode %d ; snif_rate %d ; snif_ie_bitmap_type %d\n",
+		  snif_inf->snif_mode_en ? 1 : 0,
+		  snif_inf->snif_rate_mode,
+		  snif_inf->snif_rate,
+		  snif_inf->snif_ie_bitmap_type);
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_INFO_,
+		  "snif_l_sig_len %d ; snif_sig_a1_len %d ; snif_sig_a2_len %d ; snif_sig_b_len %d\n",
+		  snif_inf->snif_l_sig_len,
+		  snif_inf->snif_sig_a1_len,
+		  snif_inf->snif_sig_a2_len,
+		  snif_inf->snif_sig_b_len);
+
+	switch (snif_inf->snif_rate_mode) {
+		case SNIF_I_RATE_MODE_LEGACY :
+			/* legacy */
+			if (snif_inf->snif_l_sig_len > 0) {
+				__dump_lsig(snif_inf->snif_l_sig);
+			}
+		break;
+		case SNIF_I_RATE_MODE_HT :
+			/* HT */
+			if (snif_inf->snif_l_sig_len > 0) {
+				__dump_lsig(snif_inf->snif_l_sig);
+			}
+		break;
+		case SNIF_I_RATE_MODE_VHT :
+			/* VHT */
+			if (snif_inf->snif_l_sig_len > 0) {
+				__dump_lsig(snif_inf->snif_l_sig);
+			}
+		break;
+		case SNIF_I_RATE_MODE_HE :
+			/* HE */
+			if (snif_inf->snif_l_sig_len > 0) {
+				__dump_lsig(snif_inf->snif_l_sig);
+			}
+			if(phl_rx->r.mdata.ppdu_type == RX_DESC_PPDU_T_HE_SU) {
+				if ((snif_inf->snif_sig_a1_len > 0) &&
+				    (snif_inf->snif_sig_a1 != NULL)) {
+					__dump_he_sig_a1(snif_inf->snif_sig_a1);
+				}
+				if ((snif_inf->snif_sig_a2_len > 0) &&
+				    (snif_inf->snif_sig_a2 != NULL)) {
+					__dump_he_sig_a2(snif_inf->snif_sig_a2);
+				}
+			}
+			/* TODO: SIG-B */
+		break;
+		default:
+		break;
+	}
+}
+#endif /* end of CONFIG_PHL_SNIFFER_SUPPORT */
+
+#ifdef  CONFIG_PHL_CHANNEL_INFO
+static void
+_get_ch_info_from_bb_rpt(struct rtw_phl_ppdu_phy_info *phy_info,
+							const struct physts_result bb_rpt)
+{
+	phy_info->n_rx = bb_rpt.n_rx;
+	phy_info->n_sts = bb_rpt.n_sts;
+	phy_info->evm_1_sts = bb_rpt.evm_1_sts;
+	phy_info->evm_2_sts = bb_rpt.evm_2_sts;
+	phy_info->avg_idle_noise_pwr = bb_rpt.avg_idle_noise_pwr;
+	phy_info->is_ch_info_len_valid = bb_rpt.is_ch_info_len_valid;
+	phy_info->ch_info_len = bb_rpt.ch_info_len;
+	phy_info->ch_info_addr = (u8 *)bb_rpt.ch_info_addr;
+}
+#endif
+
+enum rtw_hal_status
+rtw_hal_bb_switch_antenna(struct rtw_hal_com_t *hal_com, u8 ant_sw)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("[HAL] call halbb_set_ant !!!\n");
+	halbb_set_ant(hal_info->bb, ant_sw);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_antdiv_fix_ant(struct rtw_hal_com_t *hal_com, u8 antIndex)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	/* halbb_antdiv_fix_ant: 1=>MAIN_ANT, 2=>AUX_ANT, else=>AUTO_ANT */
+	if (antIndex == 0)
+		halbb_antdiv_fix_ant(hal_info->bb, 1);
+	else if (antIndex == 1)
+		halbb_antdiv_fix_ant(hal_info->bb, 2);
+	else
+		halbb_antdiv_fix_ant(hal_info->bb, 3);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_parse_phy_sts(void *hal,
+			 void *ppdu_sts,
+			 struct rtw_phl_rx_pkt *phl_rx,
+			 u8 is_su,
+			 bool sniffer_mode)
+{
+	enum rtw_hal_status hstutus = RTW_HAL_STATUS_SUCCESS;
+
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ppdu_sts *hal_ppdu = (struct hal_ppdu_sts *)ppdu_sts;
+	struct rtw_r_meta_data *mdata = &(phl_rx->r.mdata);
+	struct rtw_phl_ppdu_phy_info *phy_info = &(phl_rx->r.phy_info);
+	struct physts_rxd rxdesc = {0};
+	struct physts_result bb_rpt = {0};
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	struct rtw_phl_ppdu_sts_info *ppdu_info = &phl_com->ppdu_sts_info;
+	enum phl_band_idx band = (mdata->bb_sel > 0) ? HW_BAND_1 : HW_BAND_0;
+#ifdef CONFIG_PHL_WKARD_CHANNEL_INFO_ACK
+	struct rtw_chinfo_cur_parm *cur_parm = phl_com->cur_parm;
+#endif
+	u8 i = 0;
+	bool valid = false;
+#ifdef CONFIG_WKARD_CHECK_NULL_CSI_LEN
+	u32 chk_len = 0;
+	u32 gr_num = cur_parm->action_parm.group_num;
+
+	switch (gr_num) {
+	case CHINFO_GROUP_NUM_1:
+		chk_len = 208;
+		break;
+	case CHINFO_GROUP_NUM_2:
+		chk_len = 104;
+		break;
+	case CHINFO_GROUP_NUM_4:
+		chk_len = 56;
+		break;
+	case CHINFO_GROUP_NUM_16:
+		chk_len = 16;
+		break;
+	default:
+		PHL_ERR("%s: Unknown group num:%d\n", __func__, gr_num);
+		break;
+	}
+
+	if (cur_parm->action_parm.accuracy == CHINFO_ACCU_2BYTES) {
+		chk_len = chk_len << 1;
+	}
+#endif
+
+	rxdesc.data_rate = mdata->rx_rate;
+	rxdesc.gi_ltf = mdata->rx_gi_ltf;
+	rxdesc.is_su = is_su;
+	rxdesc.macid_su = (u8)mdata->macid;
+	rxdesc.user_num = hal_ppdu->usr_num;
+	rxdesc.is_to_self = (true == sniffer_mode) ? 1 : ((mdata->a1_match == true) ? 1 : 0);
+	if (RTW_IS_ASOC_REQ_PKT(ppdu_info->sts_ent[band][mdata->ppdu_cnt].frame_type))
+		rxdesc.is_to_self = true;
+
+	rxdesc.phy_idx = (mdata->bb_sel == 0) ? HW_PHY_0 : HW_PHY_1;
+
+	for (i = 0; i < rxdesc.user_num; i++) {
+		rxdesc.user_i[i].macid = (u8)hal_ppdu->usr[i].macid;
+		rxdesc.user_i[i].is_data = hal_ppdu->usr[i].has_data;
+		rxdesc.user_i[i].is_ctrl = hal_ppdu->usr[i].has_ctrl;
+		rxdesc.user_i[i].is_mgnt = hal_ppdu->usr[i].has_ctrl;
+		rxdesc.user_i[i].is_bcn = hal_ppdu->usr[i].has_bcn;
+	}
+
+	valid = halbb_physts_parsing(hal_info->bb, hal_ppdu->phy_st_ptr,
+			     (u16)hal_ppdu->phy_st_size,
+			     &rxdesc, &bb_rpt);
+	if (valid != true) {
+		PHL_TRACE(COMP_PHL_PSTS, _PHL_DEBUG_,
+						"halbb_physts_parsing Fail!\n");
+		hstutus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	phy_info->is_pkt_with_data = bb_rpt.is_pkt_with_data;
+	if ((bb_rpt.rssi_avg != 0) || (bb_rpt.physts_rpt_valid == 1)) {
+		phy_info->is_valid = true;
+		/* rssi from bb rpt, bit 0 is 0.0 ~ 0.9, removed it */
+		phy_info->rssi = (bb_rpt.rssi_avg >> 1);
+		phy_info->ch_idx = bb_rpt.ch_idx;
+		phy_info->rxsc = bb_rpt.rxsc;
+		phy_info->tx_bf = bb_rpt.is_bf;
+		for (i = 0; i < RTW_PHL_MAX_RF_PATH; i++) {
+			/* rssi from bb rpt, bit 0 is 0.0 ~ 0.9, removed it */
+			phy_info->rssi_path[i] = (bb_rpt.rssi[i] >> 1);
+			PHL_TRACE(COMP_PHL_PSTS, _PHL_INFO_,
+					"phy_info->rssi_path[%d] = %d\n",
+					i, phy_info->rssi_path[i]);
+		}
+		/* SNR from bb rpt */
+		phy_info->snr_fd_avg = bb_rpt.snr_fd_avg;
+		phy_info->snr_td_avg = bb_rpt.snr_td_avg;
+		for ( i = 0; i < RTW_PHL_MAX_RF_PATH; i++) {
+			phy_info->snr_fd[i] = bb_rpt.snr_fd[i];
+			phy_info->snr_td[i] = bb_rpt.snr_td[i];
+		}
+#ifdef RTW_WKARD_SINGLE_PATH_RSSI
+		if (RF_PATH_A == hal_info->hal_com->cur_rx_rfpath) {
+			phy_info->rssi = phy_info->rssi_path[0];
+		} else if (RF_PATH_B == hal_info->hal_com->cur_rx_rfpath) {
+			phy_info->rssi = phy_info->rssi_path[1];
+		} else if (RF_PATH_C == hal_info->hal_com->cur_rx_rfpath) {
+			phy_info->rssi = phy_info->rssi_path[2];
+		} else if (RF_PATH_D == hal_info->hal_com->cur_rx_rfpath) {
+			phy_info->rssi = phy_info->rssi_path[3];
+		}
+#endif
+		PHL_TRACE(COMP_PHL_PSTS, _PHL_INFO_,
+				"avg_rssi %d ; ch_idx %d\n",
+				phy_info->rssi, phy_info->ch_idx);
+
+#ifdef  CONFIG_PHL_CHANNEL_INFO
+		if (!ppdu_info->sts_ent[band][mdata->ppdu_cnt].bypass_csi
+			&& bb_rpt.ch_info_len != 0
+#ifdef CONFIG_WKARD_CHECK_NULL_CSI_LEN
+			&& bb_rpt.ch_info_len == chk_len
+#endif
+			&& bb_rpt.ch_info_addr != NULL
+			&& bb_rpt.is_ch_info_len_valid == true) {
+
+			#ifdef CONFIG_PHL_WKARD_CHANNEL_INFO_ACK
+			if (cur_parm->action_parm.mode == CHINFO_MODE_ACK) {
+				mdata->freerun_cnt = ppdu_info->sts_ent[band][mdata->ppdu_cnt].freerun_cnt;
+				hal_mem_cpy(hal_info->hal_com, mdata->ta,
+					ppdu_info->sts_ent[band][mdata->ppdu_cnt].src_mac_addr, MAC_ADDRESS_LENGTH);
+			}
+			#endif
+
+			_get_ch_info_from_bb_rpt(phy_info, bb_rpt);
+			rtw_hal_get_ch_info_physts(hal, mdata, hal_ppdu->usr, phy_info);
+		}
+#endif
+	}
+#ifdef CONFIG_PHL_SNIFFER_SUPPORT
+	PHL_TRACE(COMP_PHL_SNIFF, _PHL_DEBUG_,
+		  "bb_rpt.snif_rpt_valid %d \n",
+		  bb_rpt.snif_rpt_valid ? 1 : 0);
+	if (bb_rpt.snif_rpt_valid) {
+		/* copy bb_snif to phl_rx */
+		_os_mem_cpy(hal_info->hal_com->drv_priv,
+			    &(phy_info->snif_i),
+			    bb_rpt.bb_snif_i,
+			    sizeof(struct phl_snif_info));
+		phy_info->is_snif_i_vld = true;
+		/* _dump_bb_snif_info(phl_rx); */
+	}
+#endif
+	return hstutus;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_tx_ok(void *hal, u8 cur_phy_idx, u32 *tx_ok)
+{
+	/* struct hal_info_t *hal_info = (struct hal_info_t *)hal; */
+
+	PHL_INFO("[HAL] call get halbb_mp_get_tx_ok !!!\n");
+	/*tx_ok = halbb_mp_get_tx_ok(hal_info->bb, cur_phy_idx);*/
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_dump_tx_sts(struct hal_info_t *hal_info, bool en, u8 phy_idx)
+{
+	halbb_pmac_statistics_ex(hal_info->bb, en, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_txpwr_ref(struct hal_info_t *hal_info, u8 is_cck, u8 tx_path, s16 *txpwr_ref)
+{
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API]%s\n", __FUNCTION__);
+
+	if (is_cck)
+		*txpwr_ref = halbb_get_cck_ref_dbm(hal_info->bb, tx_path);
+	else
+		*txpwr_ref = halbb_get_ofdm_ref_dbm(hal_info->bb, tx_path);
+
+	PHL_INFO("[MP HAL API] %s: txpwr_ref = %x\n", __FUNCTION__, *txpwr_ref);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_rssi(struct hal_info_t *hal_info, enum rf_path rx_path, u8 *rssi)
+{
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API]%s\n", __FUNCTION__);
+
+	*rssi = halbb_mp_get_rssi(hal_info->bb, rx_path);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_rssi_ex(struct hal_info_t *hal_info, s32 *rssi_ex,
+						enum rf_path rx_path, u8 nss, u8 cur_phy_idx)
+{
+	struct rssi_physts  rssi_t;
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+	u8 i;
+
+	PHL_INFO("[MP HAL API]%s cur_phy(%d)\n", __FUNCTION__, cur_phy_idx);
+	rssi_t = halbb_get_mp_rssi_physts(hal_info->bb, rx_path, cur_phy_idx);
+
+	for(i = 0; i < nss; i++) {
+		*(rssi_ex + i) = rssi_t.rssi_seg[0].rssi[i];
+		PHL_INFO("[MP HAL API]%s nss(%d) rssi(%d)\n", __FUNCTION__, i,
+								*(rssi_ex + i));
+	}
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_rxevm(struct hal_info_t *hal_info, u8 user, u8 strm, u8 rxevm_table, u8 *rx_evm)
+{
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API]%s\n", __FUNCTION__);
+
+	*rx_evm = halbb_mp_get_rxevm(hal_info->bb,  user, strm, rxevm_table);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_bb_trigger_rxevm(struct hal_info_t *hal_info,
+                                             u32 *phy_user0_rxevm,
+                                             u32 *phy_user1_rxevm,
+                                             u32 *phy_user2_rxevm,
+                                             u32 *phy_user3_rxevm,
+                                             u8 rx_phy_idx)
+{
+
+	struct rxevm_physts evm;
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API]%s\n", __FUNCTION__);
+
+	evm = halbb_mp_get_rxevm_physts(hal_info->bb, rx_phy_idx);
+	*phy_user0_rxevm = evm.rxevm_seg[0].rxevm_user[0].rxevm_ss_3;
+	*phy_user0_rxevm = (*phy_user0_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[0].rxevm_ss_2;
+	*phy_user0_rxevm = (*phy_user0_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[0].rxevm_ss_1;
+	*phy_user0_rxevm = (*phy_user0_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[0].rxevm_ss_0;
+
+	*phy_user1_rxevm = evm.rxevm_seg[0].rxevm_user[1].rxevm_ss_3;
+	*phy_user1_rxevm = (*phy_user1_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[1].rxevm_ss_2;
+	*phy_user1_rxevm = (*phy_user1_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[1].rxevm_ss_1;
+	*phy_user1_rxevm = (*phy_user1_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[1].rxevm_ss_0;
+
+	*phy_user2_rxevm = evm.rxevm_seg[0].rxevm_user[2].rxevm_ss_3;
+	*phy_user2_rxevm = (*phy_user2_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[2].rxevm_ss_2;
+	*phy_user2_rxevm = (*phy_user2_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[2].rxevm_ss_1;
+	*phy_user2_rxevm = (*phy_user2_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[2].rxevm_ss_0;
+
+	*phy_user3_rxevm = evm.rxevm_seg[0].rxevm_user[3].rxevm_ss_3;
+	*phy_user3_rxevm = (*phy_user3_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[3].rxevm_ss_2;
+	*phy_user3_rxevm = (*phy_user3_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[3].rxevm_ss_1;
+	*phy_user3_rxevm = (*phy_user3_rxevm<<8)|evm.rxevm_seg[0].rxevm_user[3].rxevm_ss_0;
+
+	return ret;
+}
+
+/* mode: 0 = tmac, 1 = pmac */
+enum rtw_hal_status
+rtw_hal_bb_tx_mode_switch(struct rtw_hal_com_t *hal_com,
+						  enum phl_phy_idx phy_idx,
+						  u8 mode)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	if(mode == 0) {
+		PHL_INFO("%s: tmac mode\n", __FUNCTION__);
+		halbb_set_tmac_tx(hal_info->bb, phy_idx);
+	}
+	else {
+		PHL_INFO("%s: pmac mode\n", __FUNCTION__);
+	}
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_bb_set_txsc(struct hal_info_t *hal_info, u8 txsc,
+										enum phl_phy_idx phy_idx)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	PHL_INFO("[HAL] call halbb_set_txsc !!!\n");
+
+	status = halbb_set_txsc(hal_info->bb, txsc, phy_idx);
+
+	return status;
+}
+
+
+u8 rtw_hal_bb_get_txsc(struct rtw_hal_com_t *hal_com, u8 pri_ch,
+		u8 central_ch, enum channel_width cbw, enum channel_width dbw)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+
+	return halbb_get_txsc(hal_info->bb, pri_ch, central_ch, cbw, dbw);
+}
+
+u32 rtw_hal_bb_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg)
+{
+#ifdef RTW_WKARD_BB_C2H
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 cls = c2h->c2h_class;
+	u8 func = c2h->c2h_func;
+	u16 len = c2h->content_len;
+	u8 *buf = c2h->content;
+
+	return rtw_halbb_c2h_parsing(hal_info->bb, cls, func, (u8)len, buf);
+#else
+	return 0;
+#endif
+}
+
+u16 rtw_hal_bb_get_su_rx_rate(struct rtw_hal_com_t *hal_com,
+				enum phl_band_idx band_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+	struct bb_pkt_cnt_su_info rpt;
+	u16 max_num_tmp = 0;
+	u16 rx_rate = 0;
+	u16 i = 0;
+	u16 *pkt_cnt_tmp;
+	u8 rate_num_tmp;
+	u16 ofst_mode = 0;
+	u16 ofst_ss = 0;
+	u16 idx = 0;
+	bool is_ht_mode = false;
+	bool is_legacy_rate = true;
+	enum phl_phy_idx p_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+
+	halbb_get_rx_pkt_cnt_rpt_su(hal_info->bb, &rpt, p_idx);
+
+	/*Legacy rate*/
+	if (rpt.pkt_cnt_cck || rpt.pkt_cnt_ofdm) {
+		for (i = 0; i < 12; i++) {
+			if (rpt.pkt_cnt_legacy[i] >= max_num_tmp) {
+				max_num_tmp = rpt.pkt_cnt_legacy[i];
+				rx_rate = i;
+			}
+		}
+	}
+	if (rpt.pkt_cnt_t == 0) {
+		return rx_rate;
+	}
+	/*HT, VHT, HE*/
+	if (rpt.he_pkt_not_zero) {
+		pkt_cnt_tmp = rpt.pkt_cnt_he;
+		rate_num_tmp = HE_RATE_NUM;
+		ofst_mode = 0x180;
+	} else if (rpt.vht_pkt_not_zero) {
+		pkt_cnt_tmp = rpt.pkt_cnt_vht;
+		rate_num_tmp = VHT_RATE_NUM;
+		ofst_mode = 0x100;
+	} else if (rpt.ht_pkt_not_zero) {
+		pkt_cnt_tmp = rpt.pkt_cnt_ht;
+		rate_num_tmp = HT_RATE_NUM;
+		ofst_mode = 0x80;
+		is_ht_mode = true;
+	} else {
+		return rx_rate;
+	}
+	for (i = 0; i < rate_num_tmp; i++) {
+		if (pkt_cnt_tmp[i] >= max_num_tmp) {
+			max_num_tmp = pkt_cnt_tmp[i];
+			idx = i;
+			is_legacy_rate = false;
+		}
+	}
+	if (is_legacy_rate)
+		return rx_rate;
+	if (!is_ht_mode) {
+		ofst_ss = idx / HE_VHT_NUM_MCS;
+
+		if (ofst_ss >= 0) /*>=2SS*/
+			idx -= (ofst_ss * HE_VHT_NUM_MCS);
+	}
+	rx_rate = ofst_mode + (ofst_ss << 4) + idx;
+	return rx_rate;
+}
+
+
+enum rtw_hal_status
+rtw_hal_bb_get_txinfo_power(struct hal_info_t *hal_info,
+					s16 *txinfo_power_dbm)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s: call halbb_get_txinfo_txpwr_dbm \n",
+		__FUNCTION__);
+
+	*txinfo_power_dbm = halbb_get_txinfo_txpwr_dbm(hal_info->bb);
+
+	PHL_INFO("[MP HAL API] %s: txinfo_power_dbm = %d\n",
+		__FUNCTION__, *txinfo_power_dbm);
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_rf_mode(struct hal_info_t *hal_info,
+					enum phl_rf_mode rf_mode){
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s: call rtw_hal_bb_ctrl_rf_mode \n",
+		__FUNCTION__);
+
+	halbb_ctrl_rf_mode(hal_info->bb, rf_mode);
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_sta_id(struct hal_info_t *hal_info,
+	u16 staid, enum phl_phy_idx phy_idx)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	halbb_set_sta_id(hal_info->bb, staid, phy_idx);
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_bss_color(struct hal_info_t *hal_info,
+	u8 bsscolor, enum phl_phy_idx phy_idx)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	halbb_set_bss_color(hal_info->bb, bsscolor, phy_idx);
+
+	return ret;
+}
+
+/*
+ * Fill cmac table info of bb
+ * This caller only are allowed on rtw_hal_add_sta_entry, rtw_hal_change_sta_entry and rtw_hal_update_sta_entry.
+ */
+enum rtw_hal_status
+rtw_hal_bb_cfg_config_cmac_tbl(struct hal_info_t *hal_i,
+			struct rtw_phl_stainfo_t *phl_sta,
+			struct rtw_hal_mac_ax_cctl_info *cctrl,
+			struct rtw_hal_mac_ax_cctl_info *cctl_info_mask)
+{
+	return halbb_config_cmac_tbl(hal_i->bb, phl_sta, cctrl, cctl_info_mask);
+}
+
+#ifdef RTW_WKARD_DEF_CMACTBL_CFG
+enum rtw_hal_status
+rtw_hal_bb_trx_path_cfg(struct hal_info_t *hal_info,
+		enum rf_path tx, u8 tx_nss, enum rf_path rx, u8 rx_nss)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, true);
+	if(ret != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_reset PHY0 en - failed\n", __func__);
+		_os_warn_on(1);
+	}
+
+	if (hal_com->dbcc_en) {
+		ret = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, true);
+		if(ret != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_reset PHY1 en - failed\n",
+			        __func__);
+			_os_warn_on(1);
+		}
+	}
+
+	halbb_ctrl_trx_path(hal_info->bb, tx, tx_nss, rx, rx_nss);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		"tx_ant(%d) tx_nss(%d) rx_ant(%d) rx_nss(%d)\n",
+		tx, tx_nss, rx, rx_nss);
+
+	ret = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, false);
+	if(ret != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_reset PHY0 dis - failed\n", __func__);
+		_os_warn_on(1);
+	}
+
+	if (hal_com->dbcc_en) {
+		ret = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, false);
+		if(ret != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_reset PHY1 dis - failed\n",
+			        __func__);
+			_os_warn_on(1);
+		}
+	}
+
+	return ret;
+}
+
+u16 rtw_hal_bb_cfg_cmac_tx_ant(struct hal_info_t *hal_info,
+	enum rf_path tx_path)
+{
+	u16 ret;
+
+	ret = (BIT(RF_PATH_A) | BIT(RF_PATH_B))|(RF_PATH_A<<4)|(RF_PATH_B<<6);
+	ret = halbb_cfg_cmac_tx_ant(hal_info->bb, tx_path);
+
+	return ret;
+}
+#endif
+
+enum rtw_hal_status
+rtw_hal_bb_backup_info(struct rtw_hal_com_t *hal_com, u8 cur_phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s ==>start\n", __FUNCTION__);
+
+	halbb_backup_info(hal_info->bb, cur_phy_idx);
+
+	PHL_INFO("[MP HAL API] %s ==>end\n", __FUNCTION__);
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_restore_info(struct rtw_hal_com_t *hal_com, u8 cur_phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s ==>start\n", __FUNCTION__);
+
+	halbb_restore_info(hal_info->bb, cur_phy_idx);
+
+	PHL_INFO("[MP HAL API] %s ==>end\n", __FUNCTION__);
+
+	return ret;
+}
+
+void rtw_hal_bb_set_tx_pow_per_path_lmt(struct rtw_hal_com_t *hal_com,
+					s16 pwr_lmt_a, s16 pwr_lmt_b)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	halbb_set_tx_pow_per_path_lmt(hal_info->bb, pwr_lmt_a, pwr_lmt_b);
+}
+
+void rtw_hal_bb_set_tx_pow_ref(struct rtw_hal_com_t *hal_com,
+			       enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	halbb_set_tx_pow_ref(hal_info->bb, phy_idx);
+}
+
+#ifdef CONFIG_RTW_ACS
+void rtw_hal_bb_acs_mntr_trigger(struct hal_info_t *hal_info,
+				enum phl_band_idx band_idx, struct acs_mntr_parm *parm)
+{
+	struct ccx_para_info para = {0};
+	struct env_trig_rpt trig_rpt = {0};
+	enum phl_phy_idx p_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+
+	para.rac_lv = RAC_LV_3;
+	para.mntr_time = parm->mntr_time;
+	para.clm_app = CLM_ACS;
+	para.clm_input_opt = CLM_CCA_P20;
+	para.nhm_app = NHM_ACS;
+	para.nhm_incld_cca = parm->nhm_incld_cca ? NHM_INCLUDE_CCA : NHM_EXCLUDE_CCA;
+
+	halbb_env_mntr_trigger(hal_info->bb, &para, &trig_rpt, p_idx);
+}
+
+enum rtw_hal_status rtw_hal_bb_acs_mntr_result(struct hal_info_t *hal_info,
+					enum phl_band_idx band_idx, struct acs_mntr_rpt *rpt)
+{
+	u8 result = 0;
+	struct env_mntr_rpt mntr_rpt = {0};
+	enum phl_phy_idx p_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+
+	result = halbb_env_mntr_result(hal_info->bb, &mntr_rpt, p_idx);
+	if ((result & (CLM_SUCCESS | NHM_SUCCESS)) != (CLM_SUCCESS | NHM_SUCCESS)) {
+		return RTW_HAL_STATUS_FAILURE;
+	} else {
+		rpt->clm_ratio = mntr_rpt.clm_ratio;
+		rpt->nhm_pwr = mntr_rpt.nhm_pwr;
+		rpt->nhm_ratio = mntr_rpt.nhm_ratio;
+		rpt->nhm_tx_ratio = mntr_rpt.nhm_tx_ratio;
+		hal_mem_cpy(hal_info->hal_com, rpt->nhm_rpt, mntr_rpt.nhm_rpt, NHM_RPT_NUM);
+		return RTW_HAL_STATUS_SUCCESS;
+	}
+}
+#endif /* CONFIG_RTW_ACS */
+
+#ifdef RTW_WKARD_AP_MP
+void rtw_hal_bb_rx_ndp_mp(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *) hal;
+	struct bb_info *bb = hal_info->bb;
+
+	// r_en_sound_wo_ndp
+	halbb_set_reg(bb, 0xd7c, BIT(1), 1);
+}
+
+#endif /*WKARD_MP*/
+
+enum rtw_hal_status
+rtw_hal_bb_tssi_bb_reset(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	halbb_tssi_bb_reset(hal_info->bb);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#ifdef RTW_WKARD_DYNAMIC_BFEE_CAP
+void rtw_hal_bb_dcr_en(struct hal_info_t *hal_info, bool en)
+{
+	halbb_dcr_en(hal_info->bb, en);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_,
+		  "rtw_hal_bb_dcr_en : %d \n", (en ? 1 : 0));
+	 return;
+}
+
+bool rtw_hal_bb_csi_rsp(struct hal_info_t *hal_info)
+{
+	bool ret = true;
+	if (0 == halbb_dyn_csi_rsp_rlt_get(hal_info->bb)) {
+		ret = false;
+	}
+
+	if (0 == halbb_dcr_rssi_chk(hal_info->bb)){
+		ret = false;
+	}
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_,
+		  "rtw_hal_bb_csi_rsp : %d \n", (ret ? 1 : 0));
+	return ret;
+}
+#endif
+
+void rtw_hal_bb_nvar_src_sel(struct hal_info_t *hal_info, bool is_bf)
+{
+	halbb_nvar_src_sel(hal_info->bb, is_bf);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s, is_bf: %d \n", __func__, is_bf);
+}
+
+void rtw_hal_bb_notification(struct hal_info_t *hal_info,
+                             enum phl_msg_evt_id event,
+                             enum phl_phy_idx phy_idx)
+{
+	halbb_wifi_event_notify(hal_info->bb, event, phy_idx);
+}
+
+void rtw_hal_bb_cmd_notification(struct hal_info_t *hal_info,
+                             void *hal_cmd,
+                             enum phl_phy_idx phy_idx)
+{
+	halbb_bb_cmd_notify(hal_info->bb, hal_cmd, phy_idx);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_gain_offset(struct hal_info_t *hal_info, s8 rx_gain_offset,
+				enum rf_path rx_path, enum phl_phy_idx phy_idx, u8 iscck)
+{
+	if(iscck)
+		halbb_normal_efuse_verify_cck(hal_info->bb, rx_gain_offset, rx_path, phy_idx);
+	else
+		halbb_normal_efuse_verify(hal_info->bb, rx_gain_offset, rx_path, phy_idx);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_get_efuse_init(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	halbb_get_efuse_init(hal_info->bb);
+	PHL_INFO("[HAL API] %s ==>end\n", __FUNCTION__);
+}
+enum rtw_hal_status
+rtw_hal_bb_set_dpd_bypass(struct rtw_hal_com_t *hal_com, bool pdp_bypass,
+						enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	PHL_INFO("%s: pdp_bypass = %d phy_idx = %d\n",
+			 __FUNCTION__, pdp_bypass, phy_idx);
+
+	halbb_dpd_bypass(hal_info->bb, pdp_bypass, phy_idx);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+void rtw_hal_bb_gpio_setting(struct rtw_hal_com_t *hal_com, u8 gpio_idx, u8 path,
+			bool inv,  u8 src)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+
+	halbb_gpio_setting(hal_info->bb, gpio_idx, path, inv, src);
+}
+
+void rtw_hal_bb_gpio_setting_all(struct rtw_hal_com_t *hal_com, u8 rfe_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+
+	halbb_gpio_setting_all(hal_info->bb, rfe_idx);
+}
+
+bool rtw_hal_bb_check_tx_idle(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx)
+{
+	return halbb_chk_tx_idle(hal_info->bb, phy_idx);
+}
+
+static enum rtw_hal_status
+_cnvrt_rainfo_to_rate(enum rtw_rate_mode rate_mode, u8 mcs_ss_idx,
+		      enum rtw_data_rate *data_rate)
+{
+	enum rtw_hal_status hal_sts = RTW_HAL_STATUS_FAILURE;
+	u16 rate_idx = 0, ss = 0;
+
+	switch(rate_mode) {
+	case RTW_LEGACY_MODE:
+		rate_idx = (u16)(mcs_ss_idx & 0xf);
+		ss = 0;
+		hal_sts = RTW_HAL_STATUS_SUCCESS;
+		break;
+	case RTW_HT_MODE:
+		rate_idx = (u16)(mcs_ss_idx & 0x1f);
+		ss = 0;
+		hal_sts = RTW_HAL_STATUS_SUCCESS;
+		break;
+	case RTW_VHT_MODE:
+	case RTW_HE_MODE:
+		rate_idx = (u16)(mcs_ss_idx & 0xf);
+		ss = (mcs_ss_idx & 0x30) >> 4;
+		hal_sts = RTW_HAL_STATUS_SUCCESS;
+		break;
+	default:
+		PHL_TRACE(COMP_PHL_XMIT, _PHL_ERR_,
+			  "%s : incorrect rate mode(0x%x), fail to covert rate\n",
+			  __func__, rate_mode);
+		hal_sts = RTW_HAL_STATUS_FAILURE;
+		break;
+	}
+
+	if (RTW_HAL_STATUS_SUCCESS == hal_sts)
+		*data_rate = rate_idx | (ss << 4) | (rate_mode << 7);
+
+	return hal_sts;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ic_hw_setting_init(struct hal_info_t *hal_info)
+{
+	halbb_dm_init(hal_info->bb, HW_PHY_0);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_loop_bck_en(struct hal_info_t *hal_info, u8 enable, u8 is_dgt_mode,
+						enum rf_path tx_path, enum rf_path rx_path,
+		   				enum channel_width bw, enum phl_phy_idx phy_idx,
+						u8 is_cck)
+{
+	u8 status = false;
+
+	if (is_cck)
+		status = halbb_cfg_lbk_cck(hal_info->bb, enable, is_dgt_mode, tx_path, rx_path, bw, phy_idx);
+	else
+		status = halbb_cfg_lbk(hal_info->bb, enable, is_dgt_mode, tx_path, rx_path, bw, phy_idx);
+
+	if (status)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_tx_by_bt_link(struct hal_info_t *hal_info , u8 is_bt_link)
+{
+	halbb_mp_bt_cfg(hal_info->bb, is_bt_link);
+
+	return RTW_HAL_STATUS_SUCCESS;;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_query_rainfo(void *hal, struct rtw_hal_stainfo_t *hal_sta,
+			struct rtw_phl_rainfo *phl_rainfo)
+{
+	enum rtw_hal_status hal_sts = RTW_HAL_STATUS_FAILURE;
+	struct rtw_rate_info *rainfo = NULL;
+
+	do {
+		if (NULL == hal_sta) {
+			PHL_TRACE(COMP_PHL_XMIT, _PHL_ERR_,
+				  "%s : hal_sta is NULL\n",
+				  __func__);
+			break;
+		}
+
+		if (NULL == phl_rainfo) {
+			PHL_TRACE(COMP_PHL_XMIT, _PHL_ERR_,
+				  "%s : Input parameter is NULL\n",
+				  __func__);
+			break;
+		}
+
+		rainfo = &hal_sta->ra_info.rpt_rt_i;
+
+		phl_rainfo->gi_ltf = rainfo->gi_ltf;
+		phl_rainfo->is_actrl = rainfo->is_actrl;
+		phl_rainfo->en_stbc = rainfo->en_stbc;
+		if (HAL_RATE_BW_20 == rainfo->bw) {
+			phl_rainfo->bw = CHANNEL_WIDTH_20;
+		} else if (HAL_RATE_BW_40 == rainfo->bw) {
+			phl_rainfo->bw = CHANNEL_WIDTH_40;
+		} else if (HAL_RATE_BW_80 == rainfo->bw) {
+			phl_rainfo->bw = CHANNEL_WIDTH_80;
+		} else if (HAL_RATE_BW_160 == rainfo->bw) {
+			phl_rainfo->bw = CHANNEL_WIDTH_160;
+		} else {
+			PHL_TRACE(COMP_PHL_XMIT, _PHL_ERR_,
+				  "%s : incorrect bw(0x%x), fail to covert rate\n",
+				  __func__, rainfo->bw);
+			break;
+		}
+
+		if (RTW_HAL_STATUS_SUCCESS ==
+		    _cnvrt_rainfo_to_rate(rainfo->mode,
+					  rainfo->mcs_ss_idx,
+					  &phl_rainfo->rate)) {
+			hal_sts = RTW_HAL_STATUS_SUCCESS;
+			break;
+		} else {
+			break;
+		}
+	} while (false);
+
+	return hal_sts;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_dump_tpu_info (void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_tpu_info *tpu = &hal_com->band[0].rtw_tpu_i;
+	struct rtw_tpu_pwr_by_rate_info *by_rate = &tpu->rtw_tpu_pwr_by_rate_i;
+	u8 i;
+
+	PHL_PRINT("%-10s {%02d, %02d, %02d, %02d}\n", "[CCK]",
+				by_rate->pwr_by_rate_lgcy[0] >> 1, by_rate->pwr_by_rate_lgcy[1] >> 1,
+				by_rate->pwr_by_rate_lgcy[2] >> 1, by_rate->pwr_by_rate_lgcy[3] >> 1);
+	PHL_PRINT("%-10s {%02d, %02d, %02d, %02d, %02d, %02d, %02d, %02d}\n","[OFDM]",
+			by_rate->pwr_by_rate_lgcy[4] >> 1, by_rate->pwr_by_rate_lgcy[5] >> 1,
+			by_rate->pwr_by_rate_lgcy[6] >> 1, by_rate->pwr_by_rate_lgcy[7] >> 1,
+			by_rate->pwr_by_rate_lgcy[8] >> 1, by_rate->pwr_by_rate_lgcy[9] >> 1,
+			by_rate->pwr_by_rate_lgcy[10] >> 1, by_rate->pwr_by_rate_lgcy[11] >> 1);
+
+	for (i = 0; i < HAL_MAX_PATH; i++) {
+		PHL_PRINT("[%d]%-7s {%02d, %02d, %02d, %02d, %02d, %02d, %02d, %02d, %02d, %02d, %02d, %02d}\n",
+				i, "[MCS]",
+				by_rate->pwr_by_rate[i][0] >> 1, by_rate->pwr_by_rate[i][1] >> 1,
+				by_rate->pwr_by_rate[i][2] >> 1, by_rate->pwr_by_rate[i][3] >> 1,
+				by_rate->pwr_by_rate[i][4] >> 1, by_rate->pwr_by_rate[i][5] >> 1,
+				by_rate->pwr_by_rate[i][6] >> 1, by_rate->pwr_by_rate[i][7] >> 1,
+				by_rate->pwr_by_rate[i][8] >> 1, by_rate->pwr_by_rate[i][9] >> 1,
+				by_rate->pwr_by_rate[i][10] >> 1, by_rate->pwr_by_rate[i][11] >> 1);
+
+		PHL_PRINT("[%d]%-7s {%02d, %02d, %02d, %02d}\n",
+				i,"[DCM]",
+				by_rate->pwr_by_rate[i][12] >> 1, by_rate->pwr_by_rate[i][13] >> 1,
+				by_rate->pwr_by_rate[i][14] >> 1, by_rate->pwr_by_rate[i][15] >> 1);
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_nhm_mntr_result(struct rtw_hal_com_t *hal_com, void *rpt, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+	struct env_mntr_rpt mntr_rpt = {0};
+	struct watchdog_nhm_report *nhm_rpt = (struct watchdog_nhm_report *)rpt;
+
+	halbb_env_mntr_get_bg_result(hal_info->bb, &mntr_rpt, phy_idx);
+
+	nhm_rpt->ccx_rpt_stamp = mntr_rpt.ccx_rpt_stamp;
+	nhm_rpt->ccx_rpt_result = mntr_rpt.ccx_rpt_result;
+	nhm_rpt->nhm_pwr_dbm = (s8) ((s16) mntr_rpt.nhm_pwr - 110);
+	nhm_rpt->nhm_ratio = mntr_rpt.nhm_ratio;
+}
+
+void rtw_hal_bb_set_pow_patten_sharp(struct rtw_hal_com_t *hal_com, u8 channel, u8 is_cck, u8 sharp_id, enum phl_phy_idx phy_idx)
+{
+	u8 is_ofdm = (is_cck == 0)?1:0;
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+	struct rtw_tpu_info *tpu = &hal_info->hal_com->band[phy_idx].rtw_tpu_i;
+	tpu->tx_ptrn_shap_idx = sharp_id;
+
+	halbb_set_tx_pow_pattern_shap(hal_info->bb, channel, is_ofdm, phy_idx);
+}
+
+void rtw_hal_bb_env_rpt(struct rtw_hal_com_t *hal_com, struct rtw_env_report *env_rpt,
+		     enum phl_phy_idx phy_indx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+	struct env_mntr_rpt bg_rpt = {0};
+	u8 env_rpt_status = 0;
+
+	halbb_env_mntr_get_bg_result(hal_info->bb, &bg_rpt, phy_indx);
+	env_rpt_status = bg_rpt.ccx_rpt_result;
+	env_rpt_status = env_rpt_status & CCX_SUCCESS;
+
+	/*if ok update env rpt */
+	if (env_rpt_status == CCX_SUCCESS) {
+		env_rpt->clm_ratio = bg_rpt.clm_ratio;
+		env_rpt->nhm_ratio = bg_rpt.nhm_ratio;
+		env_rpt->nhm_tx_ratio = bg_rpt.nhm_tx_ratio;
+		env_rpt->nhm_cca_ratio = bg_rpt.nhm_cca_ratio;
+		env_rpt->nhm_idle_ratio = bg_rpt.nhm_idle_ratio;
+		env_rpt->nhm_tx_cnt = bg_rpt.nhm_tx_cnt;
+		env_rpt->nhm_cca_cnt = bg_rpt.nhm_cca_cnt;
+		env_rpt->nhm_idle_cnt = bg_rpt.nhm_idle_cnt;
+		env_rpt->nhm_pwr = bg_rpt.nhm_pwr;
+		env_rpt->rpt_status = 1;
+	} else {
+		env_rpt->rpt_status = 0;
+	}
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_tb_pwr_ofst(struct hal_info_t *hal_info,
+			s16 ofst, enum phl_phy_idx phy_idx)
+{
+	bool ret = false;
+
+	ret = halbb_set_pwr_ul_tb_ofst(hal_info->bb, ofst, phy_idx);
+	if (ret == true)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+u8 rtw_hal_bb_drv_info_dbm2rssi(struct hal_info_t *hal_info,
+				 u16 dbm, enum channel_width bw)
+{
+	return halbb_drv_info_rssi_parsing(hal_info->bb, dbm, bw);
+}
+
+#ifdef CONFIG_MCC_SUPPORT
+enum rtw_hal_status
+rtw_hal_bb_upd_mcc_macid(struct hal_info_t *hal_info,
+                         struct rtw_phl_mcc_role *mrole)
+{
+	bool ret = false;
+	struct bb_mcc_i mi = {0};
+
+	mi.type = mrole->wrole->type;
+	mi.self_macid = (u8)mrole->macid;
+	mi.chandef = mrole->chandef;
+	mi.macid_bitmap = mrole->used_macid.bitmap;
+	mi.macid_map_len = mrole->used_macid.len;
+
+	ret = halbb_upd_mcc_macid(hal_info->bb, &mi);
+
+	if (ret == true)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+void
+rtw_hal_bb_mcc_stop(struct hal_info_t *hal_info)
+{
+	halbb_mcc_stop(hal_info->bb);
+}
+
+enum rtw_hal_status
+rtw_hal_bb_mcc_start(struct hal_info_t *hal_info,
+                     struct rtw_phl_mcc_role *m_role1,
+                     struct rtw_phl_mcc_role *m_role2)
+{
+	bool ret = false;
+	struct bb_mcc_i mi_1 = {0}, mi_2 = {0};
+
+	mi_1.type = m_role1->wrole->type;
+	mi_1.self_macid = (u8)m_role1->macid;
+	mi_1.chandef = m_role1->chandef;
+	mi_1.macid_bitmap = m_role1->used_macid.bitmap;
+	mi_1.macid_map_len = m_role1->used_macid.len;
+
+	mi_2.type = m_role2->wrole->type;
+	mi_2.self_macid = (u8)m_role2->macid;
+	mi_2.chandef = m_role2->chandef;
+	mi_2.macid_bitmap = m_role2->used_macid.bitmap;
+	mi_2.macid_map_len = m_role2->used_macid.len;
+
+	ret = halbb_mcc_start(hal_info->bb, &mi_1, &mi_2);
+
+	if (ret == true)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+#endif
+
+bool
+rtw_hal_bb_adc_cfg(struct rtw_hal_com_t *hal_com,
+                   enum channel_width bw,
+                   enum rf_path path,
+                   enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)(hal_com->hal_priv);
+
+	PHL_INFO("%s: bw(%d) path(%d) phy_idx(%d)\n",
+			 __FUNCTION__, bw, path, phy_idx);
+	return halbb_adc_cfg(hal_info->bb, bw, path, phy_idx);
+}
+void rtw_hal_bb_snif_mode_ctrl(struct hal_info_t *hal_info, bool en)
+{
+	halbb_sniffer_mode_en(hal_info->bb, en);
+	return;
+}
+
+
+u8 rtw_hal_bb_decode_chidx(struct rtw_hal_com_t *hal_com, u8 chan_idx, enum band_type *band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	u8 ret_ch = 0;
+
+	halbb_ch_idx_decode(hal_info->bb, chan_idx,
+			    &ret_ch, band);
+
+	return ret_ch;
+}
+
+void rtw_hal_bb_pwr_ctrl_ability_set(struct hal_info_t *hal_info, bool enable)
+{
+	halbb_pwr_ctrl_ability_set(hal_info->bb, enable);
+}
+
+#ifdef CONFIG_SMART_ANTENNA
+void
+rtw_hal_bb_get_antenna_info(void *hal, struct rtw_phl_smart_ant_info_t *antenna_info, bool reset_cnt)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct halbb_statistic_exp_t statis_info = {0};
+	struct halbb_statistic_exp_t *p_statis_info = &statis_info;
+	struct env_mntr_rpt mntr_rpt = {0};
+	u16 pkt_cnt_ss = 0;
+	u16 ss_ofst = 0;
+	u16 i = 0, j = 0;
+
+	/* Enable statistic export */
+	halbb_statistic_exp_en(hal_info->bb, true);
+
+	/* Get halbb info */
+	halbb_statistic_exp(hal_info->bb , p_statis_info, 0);
+
+	/* Get env_mntr_rpt */
+	halbb_env_mntr_get_bg_result(hal_info->bb, &mntr_rpt, 0);
+
+	/* Fill in struct */
+	if (reset_cnt) {
+
+		/* tx_rate */
+		antenna_info->tx_rate = p_statis_info->tx_rate;
+
+		/* rx_rate*/
+		antenna_info->rx_rate = rtw_hal_bb_get_su_rx_rate(hal_info->hal_com, 0);
+
+		/* nhm_pwr */
+		antenna_info->nhm_pwr = mntr_rpt.nhm_pwr;
+
+		/* packet cnt */
+		antenna_info->pkt_cnt_cck = p_statis_info->bb_pkt_cnt_exp.pkt_cnt_cck;
+		antenna_info->pkt_cnt_ofdm = p_statis_info->bb_pkt_cnt_exp.pkt_cnt_ofdm;
+		antenna_info->pkt_cnt_1ss = p_statis_info->bb_pkt_cnt_exp.pkt_cnt_1ss;
+		antenna_info->pkt_cnt_2ss = p_statis_info->bb_pkt_cnt_exp.pkt_cnt_2ss;
+
+		/* rssi */
+		antenna_info->rssi_cck_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_cck_avg >> 1;
+		antenna_info->rssi_ofdm_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_ofdm[0] >> 1;
+		antenna_info->rssi_ofdm_avg[1] = p_statis_info->bb_rssi_su_avg_exp.rssi_ofdm[1] >> 1;
+
+		/* rssi 1ss/2ss for HT/VHT/HE */
+		if (p_statis_info->bb_pkt_cnt_exp.ht_pkt_not_zero) {
+			for (i = 0; i < 2; i++) {
+				ss_ofst = (i << 3);
+				for (j = 0; j < HT_NUM_MCS; j++) {
+					pkt_cnt_ss += p_statis_info->bb_pkt_cnt_exp.pkt_cnt_ht[ss_ofst + j];
+				}
+				if ( pkt_cnt_ss != 0 ) {
+					if ( i == 0 ) {
+						antenna_info->rssi_1ss_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[0] >> 1;
+						antenna_info->rssi_1ss_avg[1] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[1] >> 1;
+					} else {
+						antenna_info->rssi_2ss_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[0] >> 1;
+						antenna_info->rssi_2ss_avg[1] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[1] >> 1;
+					}
+				}
+			}
+			pkt_cnt_ss = 0;
+		}
+
+		if (p_statis_info->bb_pkt_cnt_exp.vht_pkt_not_zero) {
+			for (i = 0; i < 2; i++) {
+				ss_ofst = HE_VHT_NUM_MCS * i;
+				for (j = 0; j < HE_VHT_NUM_MCS; j++) {
+					pkt_cnt_ss += p_statis_info->bb_pkt_cnt_exp.pkt_cnt_vht[ss_ofst + j];
+				}
+				if ( pkt_cnt_ss != 0 ) {
+					if ( i == 0 ) {
+						antenna_info->rssi_1ss_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[0] >> 1;
+						antenna_info->rssi_1ss_avg[1] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[1] >> 1;
+					} else {
+						antenna_info->rssi_2ss_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[0] >> 1;
+						antenna_info->rssi_2ss_avg[1] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[1] >> 1;
+					}
+				}
+			}
+			pkt_cnt_ss = 0;
+		}
+
+		if (p_statis_info->bb_pkt_cnt_exp.he_pkt_not_zero) {
+			for (i = 0; i < 2; i++) {
+				ss_ofst = HE_VHT_NUM_MCS * i;
+				for (j = 0; j < HE_VHT_NUM_MCS; j++) {
+					pkt_cnt_ss += p_statis_info->bb_pkt_cnt_exp.pkt_cnt_he[ss_ofst + j];
+				}
+				if ( pkt_cnt_ss != 0 ) {
+					if ( i == 0 ) {
+						antenna_info->rssi_1ss_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[0] >> 1;
+						antenna_info->rssi_1ss_avg[1] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[1] >> 1;
+					} else {
+						antenna_info->rssi_2ss_avg[0] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[0] >> 1;
+						antenna_info->rssi_2ss_avg[1] = p_statis_info->bb_rssi_su_avg_exp.rssi_t[1] >> 1;
+					}
+				}
+			}
+			pkt_cnt_ss = 0;
+		}
+
+		/* legacy_pkt */
+		hal_mem_cpy(hal_info->hal_com, antenna_info->num_qry_legacy_pkt, p_statis_info->bb_pkt_cnt_exp.pkt_cnt_legacy, (LEGACY_RATE_NUM * sizeof(u16)));
+
+		/* ht_pkt */
+		antenna_info->ht_pkt_not_zero = p_statis_info->bb_pkt_cnt_exp.ht_pkt_not_zero;
+		hal_mem_cpy(hal_info->hal_com, antenna_info->num_qry_ht_pkt, p_statis_info->bb_pkt_cnt_exp.pkt_cnt_ht, (HT_RATE_NUM_2SS * sizeof(u16)));
+
+		/* vht_pkt */
+		antenna_info->vht_pkt_not_zero = p_statis_info->bb_pkt_cnt_exp.vht_pkt_not_zero;
+		hal_mem_cpy(hal_info->hal_com, antenna_info->num_qry_vht_pkt, p_statis_info->bb_pkt_cnt_exp.pkt_cnt_vht, (VHT_RATE_NUM_2SS * sizeof(u16)));
+
+		/* he_pkt */
+		antenna_info->he_pkt_not_zero = p_statis_info->bb_pkt_cnt_exp.he_pkt_not_zero;
+		hal_mem_cpy(hal_info->hal_com, antenna_info->num_qry_he_pkt, p_statis_info->bb_pkt_cnt_exp.pkt_cnt_he, (HE_RATE_NUM_2SS * sizeof(u16)));
+
+		/* bw_20 */
+		antenna_info->low_bw_20_occur = p_statis_info->bb_pkt_cnt_exp.sc20_occur;
+		hal_mem_cpy(hal_info->hal_com, antenna_info->num_qry_pkt_sc_20m, p_statis_info->bb_pkt_cnt_exp.pkt_cnt_sc20, (LOW_BW_RATE_NUM * sizeof(u16)));
+
+		/* bw_40 */
+		antenna_info->low_bw_40_occur = p_statis_info->bb_pkt_cnt_exp.sc40_occur;
+		hal_mem_cpy(hal_info->hal_com, antenna_info->num_qry_pkt_sc_40m, p_statis_info->bb_pkt_cnt_exp.pkt_cnt_sc40, (LOW_BW_RATE_NUM * sizeof(u16)));
+
+		/* bw_80 */
+		antenna_info->low_bw_80_occur = p_statis_info->bb_pkt_cnt_exp.sc80_occur;
+		hal_mem_cpy(hal_info->hal_com, antenna_info->num_qry_pkt_sc_80m, p_statis_info->bb_pkt_cnt_exp.pkt_cnt_sc80, (LOW_BW_RATE_NUM * sizeof(u16)));
+
+		/* EVM */
+		antenna_info->evm_1ss = p_statis_info->evm_1ss;
+		antenna_info->evm_min = p_statis_info->evm_min;
+		antenna_info->evm_max = p_statis_info->evm_max;
+
+		/* SNR */
+		antenna_info->snr_avg = p_statis_info->snr_avg;
+
+		/* Tx PER */
+		antenna_info->tx_per = p_statis_info->tx_per;
+
+		/* EDCCA */
+		antenna_info->edcca = p_statis_info->edcca_fb_pwdb;
+
+		/* Reset counter */
+		halbb_statistic_reset(hal_info->bb);
+
+	} else {
+		antenna_info->nhm_pwr = mntr_rpt.nhm_pwr;
+		antenna_info->tx_rate = p_statis_info->tx_rate;
+		antenna_info->rx_rate = rtw_hal_bb_get_su_rx_rate(hal_info->hal_com, 0);
+	}
+}
+
+#endif
+
+void rtw_hal_bb_auto_debug_en_phy_util(struct rtw_hal_com_t *hal_com, bool en)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	halbb_auto_debug_en(hal_info->bb, AUTO_DBG_PHY_UTILITY, en);
+}
+
+void rtw_hal_bb_query_snr_avg(struct rtw_hal_com_t *hal_com,
+                              u8 *info,
+                              enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct bb_bkp_phy_utility_info rpt = {0};
+
+	halbb_query_phy_utility_info(hal_info->bb, &rpt, phy_idx);
+	*info = rpt.bb_physts_avg_i.snr_avg;
+}
+
+#ifdef CONFIG_PHL_DIAGNOSE
+enum rtw_hal_status rtw_hal_bb_query_diag_info_len_ver(u32 *len, u8 *ver, u8 type)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	if (!len || !ver)
+		return RTW_HAL_STATUS_FAILURE;
+
+	*len = 0;
+	if (type & PHL_BB_PMAC) {
+		*len = sizeof(struct bb_bkp_pmac_info);
+		*ver = 1;
+	} else if (type & PHL_BB_UTILITY) {
+		*len = sizeof(struct bb_bkp_phy_utility_info);
+		*ver = 1;
+	} else {
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_bb_query_pmac_info(void *hal,
+				enum phl_phy_idx phy_idx, u8 *buf, u32 len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct bb_bkp_pmac_info *pmac_rpt = (struct bb_bkp_pmac_info *) buf;
+
+	if ((buf == NULL) || (len < sizeof(struct bb_bkp_pmac_info)))
+		return RTW_HAL_STATUS_FAILURE;
+
+	halbb_query_pmac_info(hal_info->bb, pmac_rpt, phy_idx);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_query_utility_info(void *hal,
+				enum phl_phy_idx phy_idx, u8 *buf, u32 len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct bb_bkp_phy_utility_info *utility_rpt =
+			(struct bb_bkp_phy_utility_info *) buf;
+
+	if ((buf == NULL) || (len < sizeof(struct bb_bkp_phy_utility_info)))
+		return RTW_HAL_STATUS_FAILURE;
+
+	halbb_query_phy_utility_info(hal_info->bb, utility_rpt, phy_idx);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+
+#else /*ifndef USE_TRUE_PHY*/
+
+#ifdef CONFIG_PHL_DIAGNOSE
+void rtw_hal_bb_diagnostic_event(struct rtw_hal_com_t *hal, u8 type,
+		u8 level, u8 version, u8 *buf, u32 len)
+{
+}
+#endif
+
+void rtw_hal_bb_bb_reset_cmn(struct rtw_hal_com_t *hal, bool en, enum phl_phy_idx phy_idx)
+{
+}
+
+void rtw_hal_bb_dfs_en(struct hal_info_t *hal_info, bool en)
+{
+}
+
+void rtw_hal_bb_tssi_cont_en(struct hal_info_t *hal_info, bool en, enum rf_path path)
+{
+}
+
+void rtw_hal_bb_adc_en(struct hal_info_t *hal_info, bool en, enum phl_phy_idx phy_idx)
+{
+}
+
+void rtw_hal_bb_reset_en(struct hal_info_t *hal_info, bool en, enum phl_phy_idx phy_idx)
+{
+}
+bool rtw_hal_bb_proc_cmd(struct hal_info_t *hal_info, struct rtw_proc_cmd *incmd,
+                         char *output, u32 out_len)
+{
+	return true;
+}
+
+enum rtw_hal_status rtw_hal_bb_watchdog(struct hal_info_t *hal_info, u8 is_lps)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_simple_watchdog(struct hal_info_t *hal_info, u8 io_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_reset(struct hal_info_t *hal_info)
+{
+}
+
+void rtw_hal_bb_fw_edcca(struct hal_info_t *hal_info)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_bb_dm_init(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_dm_deinit(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info)
+{
+}
+
+bool rtw_hal_bb_query_cck_en(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+	return true;
+}
+
+enum rtw_hal_status rtw_hal_bb_ctrl_cck_en(struct rtw_hal_com_t *hal_com,
+                                           bool cca_en, enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_ctrl_rx_cca(struct rtw_hal_com_t *hal_com,
+                                           bool cca_en, enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_DBCC_SUPPORT
+void rtw_hal_dbcc_init_bb_reg(struct hal_info_t *hal_info)
+{
+
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_dbcc(struct hal_info_t *hal_info, bool dbcc_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_dbcc_phy_map(struct hal_info_t *hal_info,
+                            enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_dbcc(struct hal_info_t *hal_info,
+                    struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+enum rtw_hal_status
+rtw_hal_phy_dbcc_cfg(struct hal_info_t *hal_info,
+                     struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_phy_dbcc_pre_cfg(struct hal_info_t *hal_info,
+                         struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32 rtw_hal_bb_init(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_deinit(struct rtw_phl_com_t *phl_com,
+                       struct hal_info_t *hal_info)
+{
+}
+
+void rtw_hal_init_bb_early_init(struct hal_info_t *hal_info)
+{
+}
+
+void rtw_hal_init_bb_reg(struct hal_info_t *hal_info)
+{
+}
+
+u8 rtw_hal_ex_cn_report(struct rtw_hal_com_t *hal_com)
+{
+	return 0;
+}
+
+u8 rtw_hal_ex_evm_1ss_report(struct rtw_hal_com_t *hal_com)
+{
+	return 0;
+}
+
+u8 rtw_hal_ex_evm_max_report(struct rtw_hal_com_t *hal_com)
+{
+	return 0;
+}
+
+u8 rtw_hal_ex_evm_min_report(struct rtw_hal_com_t *hal_com)
+{
+	return 0;
+}
+
+u32 rtw_hal_read_rf_reg(struct rtw_hal_com_t *hal_com,
+                        enum rf_path path, u32 addr, u32 mask)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool rtw_hal_write_rf_reg(struct rtw_hal_com_t *hal_com,
+                          enum rf_path path, u32 addr, u32 mask, u32 data)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32 rtw_hal_read_bb_reg(struct rtw_hal_com_t *hal_com,
+                        u32 addr, u32 mask)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool rtw_hal_write_bb_reg(struct rtw_hal_com_t *hal_com,
+                          u32 addr, u32 mask, u32 data)
+{
+	return true;
+}
+
+u32 rtw_hal_bb_read_cr(struct rtw_hal_com_t *hal_com, u32 addr, u32 mask)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool rtw_hal_bb_write_cr(struct rtw_hal_com_t *hal_com, u32 addr, u32 mask,
+                         u32 data)
+{
+	return true;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_init(struct hal_info_t *hal_info,
+                        struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_deinit(struct hal_info_t *hal_info,
+                          struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_add(struct hal_info_t *hal_info,
+                       struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_delete(struct hal_info_t *hal_info,
+                          struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_media_status_update(struct hal_info_t *hal_info,
+                                    struct rtw_phl_stainfo_t *sta, bool is_connected)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_bb_upt_ramask(struct hal_info_t *hal_info,
+                      struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ra_register(struct hal_info_t *hal_info,
+                       struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ra_deregister(struct hal_info_t *hal_info,
+                         struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ra_update(struct hal_info_t *hal_info,
+                     struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u8 rtw_hal_bb_get_arfr_idx(struct hal_info_t *hal_info,
+                           struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_get_efuse_info(struct rtw_hal_com_t *hal_com, u8 *efuse_map,
+                          enum rtw_efuse_info info_type, void *value,
+                          u8 size, u8 map_valid)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u8 hal_get_primary_channel_idx(u8 pri_ch, u8 central_ch, enum channel_width bw,
+                               enum chan_offset bw_offset)
+{
+	return 0;
+}
+
+enum rtw_hal_status rtw_hal_bb_set_ch_bw(struct hal_info_t *hal_info,
+                                         enum phl_phy_idx phy_idx,
+                                         u8 pri_ch,
+                                         u8 central_ch_seg0,
+                                         u8 central_ch_seg1,
+                                         enum band_type band,
+                                         enum channel_width bw)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_DFS
+bool rtw_hal_in_radar_domain(void *hal, enum band_type band, u8 ch
+                             , enum channel_width bw, enum chan_offset offset)
+{
+	return false;
+}
+
+void rtw_hal_bb_dfs_change_domain(void *hal, enum band_type band, u8 ch
+                                  , enum channel_width bw, enum chan_offset offset)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_bb_dfs_rpt_cfg(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx, bool dfs_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool
+rtw_hal_bb_radar_detect(struct hal_info_t *hal_info,
+                        struct hal_dfs_rpt *hal_dfs)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /*CONFIG_PHL_DFS*/
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+enum rtw_hal_status
+rtw_hal_bb_ch_info_parsing(struct hal_info_t *hal_info,
+                           u8 *addr, struct rtw_r_meta_data *mdata,
+                           u8 *rpt_buf, struct ch_rpt_hdr_info *ch_hdr_rpt,
+                           struct phy_info_rpt *phy_rpt, struct ch_info_drv_rpt *drv)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void
+rtw_hal_bb_chan_info_cfg(struct hal_info_t *hal_info, struct chinfo_bbcr_cfg *cfg)
+{
+}
+
+void
+rtw_hal_bb_ch_info_status_en(struct hal_info_t *hal_info, bool en, enum phl_phy_idx phy_idx)
+{
+}
+void
+rtw_hal_bb_ch_info_physts_en(struct hal_info_t *hal_info, bool en,
+                             enum phl_phy_idx phy_idx, enum wlan_mode wmode, bool ack_mode)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ch_info_decision(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx,
+                            enum channel_width bw, enum wlan_mode wmode,
+                            bool *valid_ch_info_physts)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_ch_trig_select(struct hal_info_t *hal_info, u8 event)
+{
+}
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_btg(struct rtw_hal_com_t *hal_com, bool btg)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_btc_preagc(struct rtw_hal_com_t *hal_com, bool bt_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void
+rtw_hal_bb_npath_en_update(struct rtw_hal_com_t *hal_com, bool npath_en)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_rx_path(struct rtw_hal_com_t *hal_com, u8 rx_path, u8 phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_tx_path(struct rtw_hal_com_t *hal_com, u8 tx_path, u8 phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status  rtw_hal_bb_get_rx_ok(struct hal_info_t *hal_info,
+                                          u8 cur_phy_idx, u32 *rx_ok)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status  rtw_hal_bb_get_rx_crc(struct hal_info_t *hal_info,
+                                           u8 cur_phy_idx, u32 *rx_crc_err)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status  rtw_hal_bb_set_reset_cnt(void *hal)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_power(struct rtw_hal_com_t *hal_com, s16 power_dbm,
+                     enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_get_power(struct rtw_hal_com_t *hal_com, s16 *power_dbm,
+                     enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pwr_index(void *hal, u16 pwr_idx, enum rf_path tx_path, bool is_cck)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_get_pwr_index(void *hal, u16 *pwr_idx,
+                         enum rf_path tx_path, bool is_cck)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+enum rtw_hal_status
+rtw_hal_bb_set_plcp_tx(struct rtw_hal_com_t *hal_com,
+                       struct halbb_plcp_info *plcp_tx_struct,
+                       struct usr_plcp_gen_in *plcp_usr_info,
+                       enum phl_phy_idx plcp_phy_idx,
+                       u8 *plcp_sts)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_cont_tx(struct rtw_hal_com_t *hal_com, u8 enable,
+                            u8 is_cck, enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_carrier_suppression_tx(struct rtw_hal_com_t *hal_com, u8 enable, u8 is_cck,
+                                           u16 tx_cnt, u16 period, enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_packet_tx(struct rtw_hal_com_t *hal_com, u8 enable,
+                              u8 is_cck, u16 tx_cnt ,u16 period, u16 tx_time, u8 cck_lbk_en,
+                              enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_fw_trigger_tx(struct rtw_hal_com_t *hal_com, u8 enable,
+                                  u8 is_cck, u16 tx_cnt, u8 tx_duty,
+                                  enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_switch_antenna(struct rtw_hal_com_t *hal_com, u8 ant_sw)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_antdiv_fix_ant(struct rtw_hal_com_t *hal_com, u8 antIndex)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_parse_phy_sts(void *hal,
+			 void *ppdu_sts,
+			 struct rtw_phl_rx_pkt *phl_rx,
+			 u8 is_su,
+			 bool sniffer_mode)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_tx_ok(void *hal, u8 cur_phy_idx, u32 *tx_ok)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_dump_tx_sts(struct hal_info_t *hal_info, bool en, u8 phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_get_txpwr_ref(struct hal_info_t *hal_info, u8 is_cck, u8 tx_path,
+                         s16 *txpwr_ref)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_rssi(struct hal_info_t *hal_info,
+                                        enum rf_path rx_path, u8 *rssi)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_rssi_ex(struct hal_info_t *hal_info, s32 *rssi_ex,
+                                           enum rf_path rx_path, u8 nss, u8 cur_phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_get_rxevm(struct hal_info_t *hal_info,
+                                         u8 user, u8 strm, u8 rxevm_table,
+                                         u8 *rx_evm)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_trigger_rxevm(struct hal_info_t *hal_info,
+                                             u32 *phy_user0_rxevm,
+                                             u32 *phy_user1_rxevm,
+                                             u32 *phy_user2_rxevm,
+                                             u32 *phy_user3_rxevm,
+                                             u8 rx_phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/* mode: 0 = tmac, 1 = pmac */
+enum rtw_hal_status
+rtw_hal_bb_tx_mode_switch(struct rtw_hal_com_t *hal_com,
+                          enum phl_phy_idx phy_idx, u8 mode)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_set_txsc(struct hal_info_t *hal_info, u8 txsc,
+                                        enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+u8 rtw_hal_bb_get_txsc(struct rtw_hal_com_t *hal_com, u8 pri_ch,
+                       u8 central_ch, enum channel_width cbw, enum channel_width dbw)
+{
+	return 0;
+}
+
+u32 rtw_hal_bb_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg)
+{
+	return 0;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_get_txinfo_power(struct hal_info_t *hal_info,
+                            s16 *txinfo_power_dbm)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_rf_mode(struct hal_info_t *hal_info,
+                        enum phl_rf_mode rf_mode){
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_sta_id(struct hal_info_t *hal_info,
+                      u16 staid, enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_bss_color(struct hal_info_t *hal_info,
+                         u8 bsscolor, enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef RTW_WKARD_DEF_CMACTBL_CFG
+enum rtw_hal_status
+rtw_hal_bb_trx_path_cfg(struct hal_info_t *hal_info,
+                        enum rf_path tx, u8 tx_nss, enum rf_path rx, u8 rx_nss)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u16 rtw_hal_bb_cfg_cmac_tx_ant(struct hal_info_t *hal_info,
+                               enum rf_path tx_path)
+{
+	return 0;
+}
+#endif
+enum rtw_hal_status
+rtw_hal_bb_cfg_config_cmac_tbl(struct hal_info_t *hal_i,
+                               struct rtw_phl_stainfo_t *phl_sta,
+                               struct rtw_hal_mac_ax_cctl_info *cctrl,
+                               struct rtw_hal_mac_ax_cctl_info *cctl_info_mask)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_backup_info(struct rtw_hal_com_t *hal_com, u8 cur_phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_restore_info(struct rtw_hal_com_t *hal_com, u8 cur_phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_bb_set_tx_pow_ref(struct rtw_hal_com_t *hal_com,
+                               enum phl_phy_idx phy_idx)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_bb_tssi_bb_reset(struct rtw_hal_com_t *hal_com)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_RTW_ACS
+void rtw_hal_bb_acs_mntr_trigger(struct hal_info_t *hal_info,
+                                 enum phl_band_idx band_idx, struct acs_mntr_parm *parm)
+{
+
+}
+
+enum rtw_hal_status rtw_hal_bb_acs_mntr_result(struct hal_info_t *hal_info,
+                                               enum phl_band_idx band_idx, struct acs_mntr_rpt *rpt)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+#endif /* CONFIG_RTW_ACS */
+#ifdef RTW_WKARD_DYNAMIC_BFEE_CAP
+void rtw_hal_bb_dcr_en(struct hal_info_t *hal_info, bool en)
+{
+	return;
+}
+
+bool rtw_hal_bb_csi_rsp(struct hal_info_t *hal_info)
+{
+	return true;
+}
+
+
+#endif
+
+void rtw_hal_bb_nvar_src_sel(struct hal_info_t *hal_info, bool is_bf)
+{
+	return;
+}
+
+void rtw_hal_bb_get_efuse_init(struct rtw_hal_com_t *hal_com)
+{
+
+}
+
+void rtw_hal_bb_notification(struct hal_info_t *hal_info,
+                             enum phl_msg_evt_id event,
+                             enum phl_phy_idx phy_idx)
+{
+
+}
+
+void rtw_hal_bb_cmd_notification(struct hal_info_t *hal_info,
+                                 void *hal_cmd,
+                                 enum phl_phy_idx phy_idx)
+{
+
+}
+
+enum rtw_hal_status
+rtw_hal_bb_set_dpd_bypass(struct rtw_hal_com_t *hal_com, bool pdp_bypass,
+                          enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_bb_set_gain_offset(struct hal_info_t *hal_info, s8 rx_gain_offset,
+                           enum rf_path rx_path, enum phl_phy_idx phy_idx, u8 iscck)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+bool rtw_hal_bb_check_tx_idle(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx)
+{
+	return false;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_ic_hw_setting_init(struct hal_info_t *hal_info)
+{
+
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_loop_bck_en(struct hal_info_t *hal_info, u8 enable, u8 is_dgt_mode,
+                       enum rf_path tx_path, enum rf_path rx_path,
+                       enum channel_width bw, enum phl_phy_idx phy_idx,
+                       u8 is_cck)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_tx_by_bt_link(struct hal_info_t *hal_info , u8 is_bt_link)
+{
+	return RTW_HAL_STATUS_SUCCESS;;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_query_rainfo(void *hal, struct rtw_hal_stainfo_t *hal_sta,
+                        struct rtw_phl_rainfo *phl_rainfo)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status rtw_hal_bb_dump_tpu_info (void *hal)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+void rtw_hal_bb_nhm_mntr_result(struct rtw_hal_com_t *hal_com, void *rpt, enum phl_phy_idx phy_idx)
+{
+
+}
+
+
+void rtw_hal_bb_set_pow_patten_sharp(struct rtw_hal_com_t *hal_com, u8 channel, u8 is_cck, u8 sharp_id, enum phl_phy_idx phy_idx)
+{
+
+}
+
+void rtw_hal_bb_env_rpt(struct rtw_hal_com_t *hal_com, struct rtw_env_report *env_rpt,
+                        enum phl_phy_idx phy_indx)
+{
+
+}
+
+
+enum rtw_hal_status
+rtw_hal_bb_set_tb_pwr_ofst(struct hal_info_t *hal_info,
+                           s16 ofst, enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_bb_query_txsts_rpt(struct hal_info_t *hal_info,
+                           u16 macid0, u16 macid1)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_bb_set_pop_en(struct hal_info_t *hal_info,
+                                          bool en,
+                                          enum phl_phy_idx phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool rtw_hal_bb_query_pop_en(struct hal_info_t *hal_info,
+                             enum phl_phy_idx phy_idx)
+{
+	return false;
+}
+
+enum rtw_hal_status rtw_hal_bb_set_pkt_detect_thold(struct hal_info_t *hal_info,
+                                                    enum phl_band_idx band_idx, u32 bound)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u8 rtw_hal_bb_query_pkt_detect_thold(struct hal_info_t *hal_info,
+                                     bool get_en_info,
+                                     enum phl_phy_idx phy_idx)
+{
+	return 0;
+}
+
+#ifdef RTW_WKARD_AP_MP
+void rtw_hal_bb_rx_ndp_mp(void *hal)
+{
+
+}
+#endif /*WKARD_MP*/
+
+u8 rtw_hal_bb_drv_info_dbm2rssi(struct hal_info_t *hal_info,
+                                u16 dbm, enum channel_width bw)
+{
+	return 0;
+}
+
+#ifdef CONFIG_MCC_SUPPORT
+enum rtw_hal_status
+rtw_hal_bb_upd_mcc_macid(struct hal_info_t *hal_info,
+                         struct rtw_phl_mcc_role *mrole)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void
+rtw_hal_bb_mcc_stop(struct hal_info_t *hal_info)
+{
+
+}
+
+enum rtw_hal_status
+rtw_hal_bb_mcc_start(struct hal_info_t *hal_info,
+                     struct rtw_phl_mcc_role *m_role1,
+                     struct rtw_phl_mcc_role *m_role2)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+
+void rtw_hal_bb_snif_mode_ctrl(struct hal_info_t *hal_info, bool en)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_bb_lps_info_update(void *hal, u16 macid)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u8 rtw_hal_bb_decode_chidx(struct rtw_hal_com_t *hal_com, u8 chan_idx, enum band_type *band)
+{
+	u8 ret_ch = 0;
+	return ret_ch;
+}
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+void
+rtw_hal_bb_set_tx_rate_rty_tbl(struct hal_info_t *hal_info,
+                               bool en,
+                               u8 *rty_rate_tbl)
+{
+}
+#endif
+
+void rtw_hal_bb_pwr_ctrl_ability_set(struct hal_info_t *hal_info, bool enable)
+{
+}
+
+void rtw_hal_bb_auto_debug_en_phy_util(struct rtw_hal_com_t *hal_com, bool en)
+{
+}
+
+void rtw_hal_bb_query_snr_avg(struct rtw_hal_com_t *hal_com,
+                              u8 *info,
+                              enum phl_phy_idx phy_idx)
+{
+}
+
+#endif /*ifdef USE_TRUE_PHY*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_bb.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_bb.h
new file mode 100644
index 000000000000..3de1a1298c7a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_bb.h
@@ -0,0 +1,375 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_BB_H_
+#define _HAL_API_BB_H_
+void rtw_hal_bb_bb_reset_cmn(struct rtw_hal_com_t *hal, bool en, enum phl_phy_idx phy_idx);
+void rtw_hal_bb_dfs_en(struct hal_info_t *hal_info, bool en);
+void rtw_hal_bb_tssi_cont_en(struct hal_info_t *hal_info, bool en, enum rf_path path);
+void rtw_hal_bb_adc_en(struct hal_info_t *hal_info,bool en, enum phl_phy_idx phy_idx);
+void rtw_hal_bb_reset_en(struct hal_info_t *hal_info,bool en, enum phl_phy_idx phy_idx);
+
+bool rtw_hal_bb_proc_cmd(struct hal_info_t *hal_info,
+					struct rtw_proc_cmd *incmd,
+					char *output, u32 out_len);
+
+enum rtw_hal_status rtw_hal_bb_watchdog(struct hal_info_t *hal_info, u8 is_lps);
+enum rtw_hal_status rtw_hal_bb_simple_watchdog(struct hal_info_t *hal_info, u8 io_en);
+
+void rtw_hal_bb_fw_edcca(struct hal_info_t *hal_info);
+
+void rtw_hal_bb_reset(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_bb_dm_init(struct hal_info_t *hal_info);
+
+void rtw_hal_bb_dm_deinit(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_bb_ctrl_rx_cca(struct rtw_hal_com_t *hal_com,
+	bool cca_en, enum phl_phy_idx phy_idx);
+
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status rtw_hal_bb_ctrl_dbcc(struct hal_info_t *hal_info, bool dbcc_en);
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_dbcc_phy_map(struct hal_info_t *hal_info,
+                            enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status rtw_hal_bb_cfg_dbcc(struct hal_info_t *hal_info,
+				struct rtw_phl_com_t *phl_com, bool dbcc_en);
+
+enum rtw_hal_status
+rtw_hal_phy_dbcc_pre_cfg(struct hal_info_t *hal_info,
+			struct rtw_phl_com_t *phl_com, bool dbcc_en);
+
+enum rtw_hal_status
+rtw_hal_phy_dbcc_cfg(struct hal_info_t *hal_info,
+			struct rtw_phl_com_t *phl_com, bool dbcc_en);
+
+void rtw_hal_dbcc_init_bb_reg(struct hal_info_t *hal_info);
+#endif /* CONFIG_DBCC_SUPPORT */
+
+u32 rtw_hal_bb_init(struct rtw_phl_com_t *phl_com,
+			struct hal_info_t *hal_info);
+
+void rtw_hal_bb_deinit(struct rtw_phl_com_t *phl_com,
+			struct hal_info_t *hal_info);
+
+void rtw_hal_init_bb_early_init(struct hal_info_t *hal_info);
+
+void rtw_hal_init_bb_reg(struct hal_info_t *hal_info);
+
+u8 rtw_hal_ex_cn_report(struct rtw_hal_com_t *hal_com);
+
+u8 rtw_hal_ex_evm_1ss_report(struct rtw_hal_com_t *hal_com);
+
+u8 rtw_hal_ex_evm_max_report(struct rtw_hal_com_t *hal_com);
+
+u8 rtw_hal_ex_evm_min_report(struct rtw_hal_com_t *hal_com);
+
+u32 rtw_hal_read_rf_reg(struct rtw_hal_com_t *hal_com,
+			enum rf_path path, u32 addr, u32 mask);
+
+bool rtw_hal_write_rf_reg(struct rtw_hal_com_t *hal_com,
+				enum rf_path path, u32 addr, u32 mask, u32 data);
+
+u32 rtw_hal_read_bb_reg(struct rtw_hal_com_t *hal_com,
+						u32 addr, u32 mask);
+
+bool rtw_hal_write_bb_reg(struct rtw_hal_com_t *hal_com,
+			  			  u32 addr, u32 mask, u32 data);
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_init(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_bb_stainfo_deinit(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_bb_stainfo_add(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_bb_stainfo_delete(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_bb_lps_info_update(void *hal, u16 macid);
+void rtw_hal_bb_media_status_update(struct hal_info_t *hal_info,
+			struct rtw_phl_stainfo_t *sta, bool is_connected);
+enum rtw_hal_status
+rtw_hal_bb_upt_ramask(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_bb_ra_update(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_bb_ra_register(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_bb_ra_deregister(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+u8
+rtw_hal_bb_get_arfr_idx(struct hal_info_t *hal_info,
+				struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_bb_set_ch_bw(struct hal_info_t *hal_info,
+					enum phl_phy_idx phy_idx,
+					u8 pri_ch,
+					u8 central_ch_seg0,
+		      		u8 central_ch_seg1,
+					enum band_type band,
+					enum channel_width bw);
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+enum rtw_hal_status
+rtw_hal_bb_set_pop_en(struct hal_info_t *hal_info,
+                      bool en,
+                      enum phl_phy_idx phy_idx);
+
+bool
+rtw_hal_bb_query_pop_en(struct hal_info_t *hal_info,
+                        enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_set_pkt_detect_thold(struct hal_info_t *hal_info,
+				enum phl_band_idx band_idx, u32 bound);
+
+u8
+rtw_hal_bb_query_pkt_detect_thold(struct hal_info_t *hal_info,
+                                  bool get_en_info,
+                                  enum phl_phy_idx phy_idx);
+
+void
+rtw_hal_bb_set_tx_rate_rty_tbl(struct hal_info_t *hal_info,
+                               bool en,
+                               u8 *rty_rate_tbl);
+
+enum rtw_hal_status
+rtw_hal_bb_set_spatial_reuse_en(struct hal_info_t *hal_info, bool en);
+
+bool
+rtw_hal_bb_is_spatial_reuse_en(struct hal_info_t *hal_info);
+#endif
+#ifdef CONFIG_RTW_ACS
+void rtw_hal_bb_acs_mntr_trigger(struct hal_info_t *hal_info,
+				enum phl_band_idx band_idx, struct acs_mntr_parm *parm);
+enum rtw_hal_status rtw_hal_bb_acs_mntr_result(struct hal_info_t *hal_info,
+					enum phl_band_idx band_idx, struct acs_mntr_rpt *rpt);
+#endif /* CONFIG_RTW_ACS */
+#ifdef CONFIG_PHL_DFS
+enum rtw_hal_status
+rtw_hal_bb_dfs_rpt_cfg(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx, bool dfs_en);
+bool
+rtw_hal_bb_radar_detect(struct hal_info_t *hal_info,
+			struct hal_dfs_rpt *hal_dfs);
+#endif
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+enum rtw_hal_status
+rtw_hal_bb_ch_info_parsing(struct hal_info_t *hal_info,
+	u8 *addr, struct rtw_r_meta_data *mdata,
+	u8 *rpt_buf, struct ch_rpt_hdr_info *ch_hdr_rpt,
+	struct phy_info_rpt *phy_rpt, struct ch_info_drv_rpt *drv);
+
+void
+rtw_hal_bb_chan_info_cfg(struct hal_info_t *hal_info, struct chinfo_bbcr_cfg *bbcr);
+
+void
+rtw_hal_bb_ch_info_status_en(struct hal_info_t *hal_info, bool en, enum phl_phy_idx phy_idx);
+
+void
+rtw_hal_bb_ch_info_physts_en(struct hal_info_t *hal_info, bool en,
+					enum phl_phy_idx phy_idx, enum wlan_mode wmode, bool ack_mode);
+
+enum rtw_hal_status
+rtw_hal_bb_ch_info_decision(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx,
+								enum channel_width bw, enum wlan_mode wmode,
+								bool *valid_ch_info_physts);
+
+void
+rtw_hal_bb_ch_info_get_physts(struct hal_info_t *hal_info,
+							u8 *rpt_buf, struct ch_rpt_hdr_info *ch_hdr_rpt,
+							struct phy_info_rpt *phy_rpt, struct ch_info_drv_rpt *drv);
+
+void
+rtw_hal_bb_ch_trig_select(struct hal_info_t *hal_info, u8 event);
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+enum rtw_hal_status rtw_hal_bb_get_rx_ok(struct hal_info_t *hal_info, u8 cur_phy_idx, u32 *rx_ok);
+
+enum rtw_hal_status rtw_hal_bb_get_rx_crc(struct hal_info_t *hal_info, u8 cur_phy_idx, u32 *rx_crc_err);
+
+enum rtw_hal_status rtw_hal_bb_set_reset_cnt(void *hal);
+
+enum rtw_hal_status rtw_hal_bb_set_pwr_index(void *hal, u16 pwr_idx, enum rf_path tx_path, bool is_cck);
+
+enum rtw_hal_status rtw_hal_bb_get_pwr_index(void *hal, u16 *pwr_idx, enum rf_path tx_path, bool is_cck);
+
+enum rtw_hal_status
+rtw_hal_bb_parse_phy_sts(void *hal,
+			 void *ppdu_sts,
+			 struct rtw_phl_rx_pkt *phl_rx,
+			 u8 is_su,
+			 bool sniffer_mode);
+
+enum rtw_hal_status rtw_hal_bb_get_tx_ok(void *hal, u8 cur_phy_idx, u32 *tx_ok);
+enum rtw_hal_status rtw_hal_bb_dump_tx_sts(struct hal_info_t *hal_info, bool en, u8 phy_idx);
+
+enum rtw_hal_status rtw_hal_bb_get_txpwr_ref(struct hal_info_t *hal_info, u8 is_cck, u8 tx_path, s16 *txpwr_ref);
+
+enum rtw_hal_status rtw_hal_bb_get_rssi(struct hal_info_t *hal_info, enum rf_path rx_path, u8 *rssi);
+
+enum rtw_hal_status rtw_hal_bb_get_rssi_ex(struct hal_info_t *hal_info, s32 *rssi_ex,
+						enum rf_path rx_path, u8 nss, u8 cur_phy_idx);
+
+enum rtw_hal_status rtw_hal_bb_get_rxevm(struct hal_info_t *hal_info, u8 user, u8 strm, u8 rxevm_table, u8 *rx_evm);
+
+enum rtw_hal_status rtw_hal_bb_trigger_rxevm(struct hal_info_t *hal_info,
+                                             u32 *phy_user0_rxevm,
+                                             u32 *phy_user1_rxevm,
+                                             u32 *phy_user2_rxevm,
+                                             u32 *phy_user3_rxevm,
+                                             u8 rx_phy_idx);
+
+enum rtw_hal_status rtw_hal_bb_set_txsc(struct hal_info_t *hal_info, u8 txsc,
+										enum phl_phy_idx phy_idx);
+u32 rtw_hal_bb_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg);
+
+
+enum rtw_hal_status
+rtw_hal_bb_get_txinfo_power(struct hal_info_t *hal_info,
+				s16	 *txinfo_power_dbm);
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_rf_mode(struct hal_info_t *hal_info,
+					enum phl_rf_mode rf_mode);
+
+enum rtw_hal_status
+rtw_hal_bb_set_sta_id(struct hal_info_t *hal_info,
+	u16 staid, enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_set_bss_color(struct hal_info_t *hal_info,
+	u8 bsscolor, enum phl_phy_idx phy_idx);
+
+#ifdef RTW_WKARD_DEF_CMACTBL_CFG
+enum rtw_hal_status
+rtw_hal_bb_trx_path_cfg(struct hal_info_t *hal_info,
+	enum rf_path tx, u8 tx_nss, enum rf_path rx, u8 rx_nss);
+
+u16 rtw_hal_bb_cfg_cmac_tx_ant(struct hal_info_t *hal_info,
+	enum rf_path tx_path);
+#endif
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_config_cmac_tbl(struct hal_info_t *hal_i,
+			struct rtw_phl_stainfo_t *phl_sta,
+			struct rtw_hal_mac_ax_cctl_info *cctrl,
+			struct rtw_hal_mac_ax_cctl_info *cctl_info_mask);
+
+#ifdef RTW_WKARD_DYNAMIC_BFEE_CAP
+void rtw_hal_bb_dcr_en(struct hal_info_t *hal_info, bool en);
+bool rtw_hal_bb_csi_rsp(struct hal_info_t *hal_info);
+#endif
+
+void rtw_hal_bb_nvar_src_sel(struct hal_info_t *hal_info, bool is_bf);
+
+void rtw_hal_bb_notification(struct hal_info_t *hal_info,
+                             enum phl_msg_evt_id event,
+                             enum phl_phy_idx phy_idx);
+
+void rtw_hal_bb_cmd_notification(struct hal_info_t *hal_info,
+                             void *hal_cmd,
+                             enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_set_gain_offset(struct hal_info_t *hal_info, s8 rx_gain_offset,
+				enum rf_path rx_path, enum phl_phy_idx phy_idx, u8 iscck);
+
+void rtw_hal_bb_get_efuse_init(struct rtw_hal_com_t *hal_com);
+
+enum rtw_hal_status
+rtw_hal_bb_ic_hw_setting_init(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_bb_loop_bck_en(struct hal_info_t *hal_info, u8 enable, u8 is_dgt_mode,
+						enum rf_path tx_path, enum rf_path rx_path,
+		   				enum channel_width bw, enum phl_phy_idx phy_idx,
+						u8 is_cck);
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_tx_by_bt_link(struct hal_info_t *hal_info , u8 is_bt_link);
+
+bool rtw_hal_bb_check_tx_idle(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_query_rainfo(void *hal, struct rtw_hal_stainfo_t *hal_sta,
+			struct rtw_phl_rainfo *phl_rainfo);
+
+enum rtw_hal_status
+rtw_hal_bb_query_txsts_rpt(struct hal_info_t *hal_info,
+				u16 macid0, u16 macid1);
+
+enum rtw_hal_status
+rtw_hal_bb_set_tb_pwr_ofst(struct hal_info_t *hal_info,
+			s16 ofst, enum phl_phy_idx phy_idx);
+
+u8 rtw_hal_bb_drv_info_dbm2rssi(struct hal_info_t *hal_info,
+				 u16 dbm, enum channel_width bw);
+
+#ifdef CONFIG_MCC_SUPPORT
+enum rtw_hal_status
+rtw_hal_bb_upd_mcc_macid(struct hal_info_t *hal_info,
+                         struct rtw_phl_mcc_role *mrole);
+
+void
+rtw_hal_bb_mcc_stop(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_bb_mcc_start(struct hal_info_t *hal_info,
+                     struct rtw_phl_mcc_role *m_role1,
+                     struct rtw_phl_mcc_role *m_role2);
+#endif
+
+u8 rtw_hal_bb_decode_chidx(struct rtw_hal_com_t *hal_com, u8 chan_idx, enum band_type *band);
+
+void rtw_hal_bb_snif_mode_ctrl(struct hal_info_t *hal_info, bool en);
+
+bool
+rtw_hal_bb_adc_cfg(struct rtw_hal_com_t *hal_com,
+                   enum channel_width bw,
+                   enum rf_path path,
+                   enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_antdiv_fix_ant(struct rtw_hal_com_t *hal_com, u8 antIndex);
+
+void
+rtw_hal_bb_pwr_ctrl_ability_set(struct hal_info_t *hal_info, bool enable);
+
+void rtw_hal_bb_auto_debug_en_phy_util(struct rtw_hal_com_t *hal_com, bool en);
+
+void rtw_hal_bb_query_snr_avg(struct rtw_hal_com_t *hal_com,
+                              u8 *info,
+                              enum phl_phy_idx phy_idx);
+
+#ifdef CONFIG_PHL_DIAGNOSE
+enum rtw_hal_status rtw_hal_bb_query_pmac_info(void *hal,
+				enum phl_phy_idx, u8 *buf, u32 len);
+enum rtw_hal_status rtw_hal_bb_query_utility_info(void *hal,
+				enum phl_phy_idx phy_idx, u8 *buf, u32 len);
+#endif
+
+
+#endif /*_HAL_API_BB_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_btc.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_btc.c
new file mode 100644
index 000000000000..aad43df5cd74
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_btc.c
@@ -0,0 +1,844 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_API_BTC_C_
+#include "hal_headers.h"
+#include "btc/hal_btc.h"
+
+#ifdef CONFIG_BTCOEX
+#define BTC_EXT_CFG_INFO_SZ 2
+/*******************************************
+ * C2H FW message
+ *******************************************/
+#define BTC_FWBUF_NUM 4
+struct fw_msg_entry {
+	_os_list list;
+	u8 c2h_class;
+	u8 c2h_func;
+	u16 len;
+	u8 buf[RTW_PHL_BTC_FWINFO_BUF];
+};
+
+static void _bt_msg_init(struct rtw_hal_com_t *hal_com,
+				struct hal_bt_msg *msg)
+{
+	void *d = halcom_to_drvpriv(hal_com);
+
+	_os_spinlock_init(d, &msg->lock);
+	_os_mem_set(d, &msg->latest[0], 0, RTW_BTC_OVERWRITE_BUF_LEN);
+	_os_mem_set(d, &msg->working[0], 0, RTW_BTC_OVERWRITE_BUF_LEN);
+	msg->len = 0;
+	msg->cnt = 0;
+}
+
+static void _bt_msg_deinit(struct rtw_hal_com_t *hal_com,
+				struct hal_bt_msg *msg)
+{
+	void *d = halcom_to_drvpriv(hal_com);
+
+	_os_spinlock_free(d, &msg->lock);
+}
+
+static void _msg_enq(struct rtw_hal_com_t *hal_com,
+			struct phl_queue *q, struct fw_msg_entry *entry)
+{
+	pq_push(halcom_to_drvpriv(hal_com), q, &entry->list, _tail, _bh);
+}
+
+static struct fw_msg_entry *_msg_deq(struct rtw_hal_com_t *hal_com,
+					struct phl_queue *q)
+{
+	struct fw_msg_entry *entry = NULL;
+	_os_list *list = NULL;
+
+	if (pq_pop(halcom_to_drvpriv(hal_com), q, &list, _first, _bh))
+		entry = (struct fw_msg_entry *)list;
+
+	return entry;
+}
+
+static bool _fw_msg_init(struct rtw_hal_com_t *hal_com)
+{
+	void *d = halcom_to_drvpriv(hal_com);
+	struct btc_fw_msg *fw_msg = &hal_com->btc_msg;
+	struct fw_msg_entry *entry = NULL;
+	u16 i = 0;
+
+	_os_spinlock_init(d, &fw_msg->lock);
+	fw_msg->fev_cnt = 0;
+
+	_bt_msg_init(hal_com, &fw_msg->btinfo);
+	_bt_msg_init(hal_com, &fw_msg->scbd);
+
+	pq_init(d, &fw_msg->idleq);
+	pq_init(d, &fw_msg->waitq);
+	for (i = 0; i < BTC_FWBUF_NUM; i++) {
+		entry = (struct fw_msg_entry *)_os_kmem_alloc(d,
+					sizeof(struct fw_msg_entry));
+		if (entry)
+			_msg_enq(hal_com, &fw_msg->idleq, entry);
+		else
+			return false;
+	}
+
+	return true;
+}
+
+static void _fw_msg_free(struct rtw_hal_com_t *hal_com)
+{
+	void *d = halcom_to_drvpriv(hal_com);
+	struct btc_fw_msg *fw_msg = &hal_com->btc_msg;
+	struct fw_msg_entry *entry = NULL;
+
+	_bt_msg_deinit(hal_com, &fw_msg->btinfo);
+	_bt_msg_deinit(hal_com, &fw_msg->scbd);
+
+	while (1) {
+		entry = _msg_deq(hal_com, &fw_msg->waitq);
+		if (entry)
+			_os_kmem_free(d, entry, sizeof(struct fw_msg_entry));
+		else
+			break;
+	}
+	pq_deinit(d, &fw_msg->waitq);
+
+	while (1) {
+		entry = _msg_deq(hal_com, &fw_msg->idleq);
+		if (entry)
+			_os_kmem_free(d, entry, sizeof(struct fw_msg_entry));
+		else
+			break;
+	}
+	pq_deinit(d, &fw_msg->idleq);
+
+	_os_spinlock_free(d, &fw_msg->lock);
+}
+
+static void _copy_btmsg(struct rtw_hal_com_t *hal_com,
+			struct hal_bt_msg *msg, u16 len, u8 *buf)
+{
+	void *d = halcom_to_drvpriv(hal_com);
+
+	if (len > RTW_BTC_OVERWRITE_BUF_LEN)
+		return;
+
+	_os_spinlock(d, &msg->lock, _bh, NULL);
+	msg->cnt++;
+	msg->len = len;
+	if (len <= RTW_BTC_OVERWRITE_BUF_LEN)
+		_os_mem_cpy(d, &msg->latest[0], buf, len);
+	_os_spinunlock(d, &msg->lock, _bh, NULL);
+}
+
+static bool _fw_evnt_enq(struct rtw_hal_com_t *hal_com,
+			u8 cls, u8 func, u16 len, u8 *buf)
+{
+	struct btc_fw_msg *fmsg = &hal_com->btc_msg;
+	void *d = halcom_to_drvpriv(hal_com);
+	struct fw_msg_entry *entry = NULL;
+
+	entry = _msg_deq(hal_com, &fmsg->idleq);
+	if (!entry)
+		return false;
+
+	entry->c2h_class = cls;
+	entry->c2h_func = func;
+	entry->len = len;
+	if (len <= RTW_PHL_BTC_FWINFO_BUF)
+		_os_mem_cpy(d, &entry->buf[0], buf, len);
+	_msg_enq(hal_com, &fmsg->waitq, entry);
+
+	return true;
+}
+
+enum rtw_hal_status rtw_hal_btc_init(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct btc_t *btc = NULL;
+	void *drv_priv = NULL;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "%s\n", __FUNCTION__);
+
+	drv_priv = halcom_to_drvpriv(hal_info->hal_com);
+	btc = _os_mem_alloc(drv_priv, sizeof(struct btc_t));
+	if (!btc) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		goto error_btc_init;
+	}
+
+	/* assign phl_com & hal_com */
+	btc->phl = phl_com;
+	btc->hal = hal_info->hal_com;
+
+	if (!hal_btc_init(btc)) {
+		hal_status = RTW_HAL_STATUS_BTC_INIT_FAILURE;
+		goto error_btc_init;
+	}
+
+	if (!_fw_msg_init(hal_info->hal_com)) {
+		hal_status = RTW_HAL_STATUS_BTC_INIT_FAILURE;
+		goto error_msg_init;
+	}
+
+
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+	hal_info->btc = btc;
+
+	return hal_status;
+
+error_msg_init:
+	_fw_msg_free(hal_info->hal_com);
+
+error_btc_init:
+	if (btc) {
+		_os_mem_free(drv_priv, (void *)btc, sizeof(struct btc_t));
+		hal_info->btc = NULL;
+	}
+
+	return hal_status;
+}
+
+void rtw_hal_btc_deinit(struct rtw_phl_com_t *phl_com,
+				struct hal_info_t *hal_info)
+{
+	struct btc_t *btc = hal_info->btc;
+	void *drv_priv = NULL;
+
+	_fw_msg_free(hal_info->hal_com);
+
+	drv_priv = halcom_to_drvpriv(hal_info->hal_com);
+	if (drv_priv && btc) {
+		hal_btc_deinit(btc);
+		_os_mem_free(drv_priv, (void *)btc, sizeof(struct btc_t));
+	}
+}
+
+/**********************/
+/* called by non-hal layers */
+/**********************/
+void rtw_hal_btc_update_role_info_ntfy(void *hinfo,  u8 role_id,
+				       struct rtw_wifi_role_t *wrole,
+				       struct rtw_wifi_role_link_t *rlink,
+				       struct rtw_phl_stainfo_t *sta,
+				       enum link_state lstate)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_wl_link_info r = {0};
+	struct btc_ops *ops = NULL;
+#ifdef CONFIG_PHL_P2PPS
+	u8 i =0;
+#endif /* CONFIG_PHL_P2PPS */
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (role_id >= MAX_WIFI_ROLE_NUMBER)
+		return;
+	PHL_TRACE(COMP_PHL_BTC, _PHL_INFO_, "%s: link_state(%s,%d), hw_band(%d), rid(%d), macid(%d)\n",
+		__FUNCTION__, rtw_phl_get_lstate_str(lstate), lstate,
+		rlink->hw_band, role_id, sta->macid);
+	if (wrole) {
+#ifdef CONFIG_PHL_P2PPS
+		r.noa = 0;
+		r.noa_duration = 0;
+		for (i = 0; i < MAX_NOA_DESC; i++) {
+			if (wrole->noa_desc[i].enable) {
+				r.noa = 1;
+				r.noa_duration = wrole->noa_desc[i].duration;
+				break;
+			}
+		}
+#endif /* CONFIG_PHL_P2PPS */
+		r.role = wrole->type;
+		r.phy = rlink->hw_band;
+		r.pid = rlink->hw_port;
+		r.active = wrole->active;
+		r.connected = rlink->mstate;
+		r.mode = rlink->cap.wmode;
+		r.client_cnt = rlink->assoc_sta_queue.cnt;
+		#ifdef RTW_PHL_BCN
+		r.bcn_period = rlink->bcn_cmn.bcn_interval;
+		r.dtim_period = rlink->dtim_period;
+		#endif
+		hal_mem_cpy(h->hal_com,
+		            &r.chdef,
+		            &rlink->chandef,
+		            sizeof(struct rtw_chan_def));
+	}
+
+	if (sta && rtw_phl_role_is_client_category(sta->wrole)) {/*associated node info??*/
+		r.mac_id = sta->macid;
+		r.mode = (u8)sta->wmode;
+		hal_mem_cpy(h->hal_com, r.mac_addr, sta->mac_addr, MAC_ALEN);
+	}
+	PHL_TRACE(COMP_PHL_BTC, _PHL_INFO_, "%s: rid(%d), phy(%d), mac_id(%d), client_cnt(%d)\n",
+			__FUNCTION__, role_id, r.phy, r.mac_id, r.client_cnt);
+	if (ops && ops->ntfy_role_info)
+		ops->ntfy_role_info(btc, role_id, &r, lstate);
+}
+
+void
+rtw_hal_btc_ap_client_notify(void *hinfo,
+		struct rtw_wifi_role_link_t *rlink, enum link_state lstate)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct rtw_phl_stainfo_t *sta = NULL;
+
+	sta = rtw_phl_get_stainfo_self(h->phl_com->phl_priv, rlink);
+	rtw_hal_btc_update_role_info_ntfy(h, rlink->wrole->id, rlink->wrole,
+					rlink, sta, lstate);
+}
+
+void rtw_hal_btc_power_on_ntfy(void *hinfo)
+{
+}
+
+void rtw_hal_btc_power_off_ntfy(void *hinfo)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_power_off)
+		ops->ntfy_power_off(btc);
+}
+
+static void _hal_btc_ext_cfg_info_ntfy(struct btc_t *btc)
+{
+	struct dev_cap_t *dev_cap = &btc->phl->dev_cap;
+	struct btc_ops *ops = btc->ops;
+	u8 info_buf[BTC_EXT_CFG_INFO_SZ] = {0};
+
+	if ((dev_cap->btc_esoc_type != BTC_EXT_SOC_NONE) || (dev_cap->btc_ant_iso_db != 0xFF)) {
+		if (ops && ops->ntfy_customerize) {
+			info_buf[0] = (u8)dev_cap->btc_esoc_type;
+			info_buf[1] = dev_cap->btc_ant_iso_db;
+			ops->ntfy_customerize(btc, PHL_BTC_CNTFY_EXT_CONFIG,
+					      BTC_EXT_CFG_INFO_SZ, info_buf);
+		}
+	}
+}
+
+void rtw_hal_btc_init_coex_cfg_ntfy(void *hinfo)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+	u8 mode;
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+	mode = btc->phl->dev_cap.btc_mode;
+
+	if (ops && ops->ntfy_init_coex)
+		ops->ntfy_init_coex(btc, mode);
+
+	_hal_btc_ext_cfg_info_ntfy(btc);
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(h->phl_com, &start_t, TIME_HAL_INIT_BTC);
+#endif /* DBG_MONITOR_TIME */
+
+}
+
+void rtw_hal_btc_bk_mdl_start_ntfy(void *hinfo)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_init_coex)
+		ops->ntfy_init_coex(btc, BTC_MODE_MECHANISM_INIT);
+}
+
+void rtw_hal_btc_redownload_fw_ntfy(void *hinfo)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_radio_state)
+		ops->ntfy_radio_state(btc, BTC_RFCTRL_RESUME_DL_FW);
+}
+
+void rtw_hal_btc_scan_start_ntfy(void *hinfo, enum phl_phy_idx phy_idx,
+				  enum band_type band)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_scan_start)
+		ops->ntfy_scan_start(btc, phy_idx, band);
+}
+
+void rtw_hal_btc_scan_finish_ntfy(void *hinfo, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_scan_finish)
+		ops->ntfy_scan_finish(btc, phy_idx);
+}
+
+void rtw_hal_btc_switch_band_ntfy(void *hinfo, enum phl_phy_idx phy_idx,
+				  enum band_type band)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_switch_band)
+		ops->ntfy_switch_band(btc, phy_idx, band);
+}
+
+void rtw_hal_btc_packet_event_ntfy(void *hinfo, u8 pkt_evt_type)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_specific_packet)
+		ops->ntfy_specific_packet(btc, pkt_evt_type);
+}
+
+void rtw_hal_btc_radio_state_ntfy(void *hinfo, u8 rf_state)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_radio_state)
+		ops->ntfy_radio_state(btc, rf_state);
+}
+
+void rtw_hal_btc_customerize_ntfy(void *hinfo, u8 type, u16 len, u8 *buf)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_customerize)
+		ops->ntfy_customerize(btc, type, len, buf);
+}
+
+u8 rtw_hal_btc_wl_rfk_ntfy(struct rtw_hal_com_t *hal_com, u8 phy_idx, u8 rfk_type, u8 rfk_process)
+{
+	struct hal_info_t *h = hal_com->hal_priv;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+	u8 val = 0;
+
+	if (!btc)
+		return val;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_wl_rfk)
+		val = ops->ntfy_wl_rfk(btc, phy_idx, rfk_type, rfk_process);
+
+	return val;
+}
+
+void rtw_hal_btc_wl_status_ntfy(void *hinfo, struct rtw_phl_com_t *phl_com, u8 ntfy_num,
+				struct rtw_phl_stainfo_t *sta[], u8 reason)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_ops *ops = NULL;
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (ops && ops->ntfy_wl_sta)
+		ops->ntfy_wl_sta(btc, &phl_com->phl_stats, ntfy_num, sta, reason);
+}
+
+void rtw_hal_btc_fwinfo_ntfy(void *hinfo)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct rtw_hal_com_t *hal_com = h->hal_com;
+	void *d = halcom_to_drvpriv(hal_com);
+	struct btc_fw_msg *fmsg = &hal_com->btc_msg;
+	struct fw_msg_entry *entry = NULL;
+	struct hal_bt_msg *bmsg = NULL;
+	struct btc_ops *ops = NULL;
+
+	_os_spinlock(d, &fmsg->lock, _bh, NULL);
+	fmsg->fev_cnt = 0;
+	_os_spinunlock(d, &fmsg->lock, _bh, NULL);
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (!ops || !ops->ntfy_fwinfo)
+		return;
+
+	/* bt score board notification */
+	while (1) {
+		bmsg = &fmsg->scbd;
+		_os_spinlock(d, &bmsg->lock, _bh, NULL);
+		if (bmsg->cnt) {
+			bmsg->cnt = 0;
+			_os_mem_cpy(d, &bmsg->working[0],
+				&bmsg->latest[0], bmsg->len);
+			_os_spinunlock(d, &bmsg->lock, _bh, NULL);
+			PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+				  "[BTC], scoreboard notify !! \n");
+			ops->ntfy_fwinfo(btc, &bmsg->working[0], bmsg->len,
+					BTC_CLASS_FEV, BTC_FEV_BT_SCBD);
+		} else {
+			_os_spinunlock(d, &bmsg->lock, _bh, NULL);
+			break;
+		}
+	}
+
+	/* bt info notification */
+	while (1) {
+		bmsg = &fmsg->btinfo;
+		_os_spinlock(d, &bmsg->lock, _bh, NULL);
+		if (bmsg->cnt) {
+			bmsg->cnt = 0;
+			_os_mem_cpy(d, &bmsg->working[0],
+				&bmsg->latest[0], bmsg->len);
+			_os_spinunlock(d, &bmsg->lock, _bh, NULL);
+			PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+				  "[BTC], bt info notify !! \n");
+			ops->ntfy_fwinfo(btc, &bmsg->working[0], bmsg->len,
+					BTC_CLASS_FEV, BTC_FEV_BT_INFO);
+		} else {
+			_os_spinunlock(d, &bmsg->lock, _bh, NULL);
+			break;
+		}
+	}
+
+	/* common btc fw events */
+	while (1) {
+		entry = _msg_deq(hal_com, &fmsg->waitq);
+		if (entry) {
+			PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+				  "[BTC], fw event notify !! \n");
+			ops->ntfy_fwinfo(btc, entry->buf, entry->len,
+					entry->c2h_class, entry->c2h_func);
+			_msg_enq(hal_com, &fmsg->idleq, entry);
+		}
+		else
+			break;
+	}
+}
+
+static struct rtw_phl_stainfo_t *
+_get_first_client_sta(
+	void *hinfo,
+	struct rtw_wifi_role_link_t *rlink,
+	struct rtw_phl_stainfo_t *self)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct rtw_phl_stainfo_t *n, *psta, *ret = NULL;
+	void *drv = hal_to_drvpriv(h);
+
+	if (rlink->assoc_sta_queue.cnt > 1) {
+		/* more than one sta, ex: softap & assoc sta */
+		_os_spinlock(drv, &rlink->assoc_sta_queue.lock, _bh, NULL);
+		phl_list_for_loop_safe(psta, n, struct rtw_phl_stainfo_t,
+			&rlink->assoc_sta_queue.queue, list) {
+			if (_os_mem_cmp(drv, self->mac_addr, psta->mac_addr, MAC_ALEN) == 0) {
+				/* skip softap self */
+				continue;
+			}
+			ret = psta;
+			break;
+		}
+		_os_spinunlock(drv, &rlink->assoc_sta_queue.lock, _bh, NULL);
+	}
+	return ret;
+}
+
+void rtw_hal_btc_timer(void *hinfo, void *timer)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+	struct btc_tmr *btmr = NULL;
+	struct btc_ops *ops = NULL;
+	u8 i = 0;
+	bool found = false;
+
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+		"[BTC], %s(), timer = 0x%p !!\n", __func__, timer);
+
+	if (!btc)
+		return;
+
+	ops = btc->ops;
+
+	if (!timer)
+		return;
+
+	btmr = (struct btc_tmr *)timer;
+
+	for (i = 0; i < BTC_TIMER_MAX; i++) {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			"[BTC], %s(): btmr = 0x%p, &btc->timer[%d] = 0x%p \n",
+				__func__, btmr, i, &btc->timer[i]);
+		if (btmr == &btc->timer[i]) {
+			found = true;
+			PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+				"[BTC], %s(): found timer %p, id = %d \n",
+				__func__, btmr, btmr->id);
+			break;
+		}
+	}
+
+	if (ops && found) {
+		if (btmr->id == BTC_TIMER_PERIODIC) {
+			struct rtw_wifi_role_t *wrole = NULL;
+			struct rtw_phl_mld_t *mld = NULL;
+			struct rtw_phl_stainfo_t *sta = NULL;
+			/* Support max link num is 3 for MLO */
+			struct rtw_phl_stainfo_t *wrole_sta[MAX_WIFI_ROLE_NUMBER * 3] = {0};
+			u8 ntfy_num = 0;
+			u8 idx = 0;
+
+			for (i = 0; i < MAX_WIFI_ROLE_NUMBER; i++) {
+				wrole = &(btc->phl->wifi_roles[i]);
+				if(wrole->mstate == MLME_LINKED) {
+					mld = rtw_phl_get_mld_self(btc->phl->phl_priv, wrole);
+
+					for (idx = 0; idx < wrole->rlink_num; idx++) {
+						sta = mld->phl_sta[idx];
+						if(sta != NULL) {
+							struct rtw_phl_stainfo_t *client = NULL;
+
+							client = _get_first_client_sta(hinfo,
+								&wrole->rlink[idx], sta);
+							wrole_sta[ntfy_num] =
+								((client != NULL) ? client : sta);
+							ntfy_num++;
+						}
+					}
+				}
+			}
+			rtw_hal_btc_wl_status_ntfy(hinfo, btc->phl, ntfy_num,
+					wrole_sta, PHL_BTC_NTFY_RSN_PERIOTIC);
+		}
+		PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_,
+			"[BTC], %s(): btmr->id = %d \n",
+			__func__, btmr->id);
+		if(ops->ntfy_timer)
+			ops->ntfy_timer(btc, btmr->id);
+	} else {
+		PHL_INFO("[BTC], %s(): not found, ops = 0x%p, found = %d \n",
+				__func__, ops, found);
+	}
+}
+
+u32 rtw_hal_btc_req_bt_slot_t(void *hinfo, enum phl_band_idx hw_band)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hinfo;
+	struct btc_t *btc = (struct btc_t *)h->btc;
+
+	if (btc)
+		return btc->bt_req_len[hw_band];
+	else
+		return 0;
+}
+
+/***********************/
+/* Called by BTC submodule */
+/***********************/
+void hal_btc_send_event(struct btc_t *btc, enum phl_band_idx hw_band,
+			u8 *buf, u32 len, u16 ev_id)
+{
+#ifdef CONFIG_PHL_CMD_BTC
+	if (!btc)
+		return;
+
+	rtw_phl_btc_send_cmd(btc->phl, hw_band, buf, len, ev_id);
+#endif
+}
+
+void hal_btc_notify_ps_tdma(struct btc_t *btc, bool tdma_start)
+{
+#ifdef CONFIG_POWER_SAVE
+	enum rtw_hal_status hstatus;
+
+	if (!btc)
+		return;
+
+	hstatus = rtw_hal_ps_pwr_req(btc->phl, HAL_BTC_PWR_REQ, (tdma_start ? true : false));
+	if (hstatus != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("[BTC], %s, TDMA (%d) request pwr fail, status: %d \n",
+			__func__, tdma_start, hstatus);
+	}
+#endif
+}
+
+bool rtw_hal_btc_proc_cmd(struct hal_info_t *hal_info, struct rtw_proc_cmd *incmd,
+						char *output, u32 out_len)
+{
+	if(incmd->in_type == RTW_ARG_TYPE_BUF)
+		halbtc_cmd(hal_info->btc, incmd->in.buf, output, out_len);
+	else if(incmd->in_type == RTW_ARG_TYPE_ARRAY){
+		halbtc_cmd_parser(hal_info->btc, incmd->in.vector,
+					incmd->in_cnt_len, output, out_len);
+	}
+
+	return true;
+}
+
+#else
+u32 rtw_hal_btc_req_bt_slot_t(void *hinfo, enum phl_band_idx hw_band)
+{
+	return 0;
+}
+
+static void _copy_btmsg(struct rtw_hal_com_t *hal_com,
+			struct hal_bt_msg *msg, u16 len, u8 *buf)
+{
+	return;
+}
+
+static bool _fw_evnt_enq(struct rtw_hal_com_t *hal_com,
+			u8 cls, u8 func, u16 len, u8 *buf)
+{
+	return true;
+}
+
+void rtw_hal_btc_scan_finish_ntfy(void *hinfo, enum phl_phy_idx phy_idx)
+{
+	return;
+}
+
+void rtw_hal_btc_scan_start_ntfy(void *hinfo, enum phl_phy_idx phy_idx,
+				  enum band_type band)
+{
+	return;
+}
+
+#endif
+
+enum rtw_hal_status
+rtw_hal_btc_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid)
+{
+	PHL_TRACE(COMP_PHL_BTC, _PHL_DEBUG_, "%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32 rtw_hal_btc_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg)
+{
+	struct hal_info_t *h = (struct hal_info_t *)hal;
+#ifdef CONFIG_PHL_CMD_BTC
+	struct btc_t *btc = (struct btc_t *)h->btc;
+#endif
+	struct rtw_hal_com_t *hal_com = h->hal_com;
+	struct btc_fw_msg *fmsg = &hal_com->btc_msg;
+	void *d = halcom_to_drvpriv(hal_com);
+	u8 cls = c2h->c2h_class;
+	u8 func = c2h->c2h_func;
+	u16 len = c2h->content_len;
+	u8 *buf = c2h->content;
+
+	if (len && len < RTW_PHL_BTC_FWINFO_BUF) {
+		if (cls == BTC_CLASS_FEV && func == BTC_FEV_BT_INFO)
+			_copy_btmsg(hal_com, &fmsg->btinfo, len, buf);
+		else if (cls == BTC_CLASS_FEV && func == BTC_FEV_BT_SCBD)
+			_copy_btmsg(hal_com, &fmsg->scbd, len, buf);
+		else
+			_fw_evnt_enq(hal_com, cls, func, len, buf);
+
+		_os_spinlock(d, &fmsg->lock, _bh, NULL);
+		if (fmsg->fev_cnt == 0) {
+			/* Only forward c2h content to btc */
+#ifdef CONFIG_PHL_CMD_BTC
+			if (!btc)
+				return RTW_HAL_STATUS_FAILURE;
+
+			if (rtw_phl_btc_send_cmd(btc->phl, HW_BAND_0, NULL, 0,
+						BTC_HMSG_FW_EV))
+				fmsg->fev_cnt++;
+#endif
+		}
+		_os_spinunlock(d, &fmsg->lock, _bh, NULL);
+	} else {
+		PHL_TRACE(COMP_PHL_BTC, _PHL_ERR_, "[BTC], %s(): Invalid c2h packet len : %d\n",
+			  __func__, len);
+	}
+
+	return 0;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_btc.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_btc.h
new file mode 100644
index 000000000000..b39ad9fce2df
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_btc.h
@@ -0,0 +1,43 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_BTC_H_
+#define _HAL_API_BTC_H_
+
+#ifdef CONFIG_BTCOEX
+enum rtw_hal_status
+rtw_hal_btc_init(struct rtw_phl_com_t *phl_com,
+			struct hal_info_t *hal_info);
+
+void rtw_hal_btc_deinit(struct rtw_phl_com_t *phl_com,
+				struct hal_info_t *hal_info);
+#else
+#define rtw_hal_btc_init(_phl_com_, _hal_info_) RTW_HAL_STATUS_SUCCESS
+#define rtw_hal_btc_deinit(_phl_com_, _hal_info_)
+#endif
+
+void rtw_hal_btc_switch_band_ntfy(void *hinfo, enum phl_phy_idx phy_idx,
+				  enum band_type band);
+void rtw_hal_btc_radio_state_ntfy(void *hinfo, u8 rf_state);
+
+u32 rtw_hal_btc_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg);
+void rtw_hal_btc_init_coex_cfg_ntfy(void *hinfo);
+
+void rtw_hal_btc_redownload_fw_ntfy(void *hinfo);
+
+bool rtw_hal_btc_proc_cmd(struct hal_info_t *hal_info, struct rtw_proc_cmd *incmd,
+						char *output, u32 out_len);
+u32 rtw_hal_btc_req_bt_slot_t(void *hinfo, enum phl_band_idx hw_band);
+
+#endif /*_HAL_API_BTC_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_drv.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_drv.h
new file mode 100644
index 000000000000..9d2998a5fb66
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_drv.h
@@ -0,0 +1,522 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_DRV_H_
+#define _HAL_API_DRV_H_
+
+#ifdef CONFIG_BTCOEX
+u32 rtw_hal_mac_coex_init(struct rtw_hal_com_t *hal_com, u8 pta_mode, u8 direction);
+u32 rtw_hal_mac_coex_reg_read(struct rtw_hal_com_t *hal_com, u32 offset, u32 *value);
+u32 rtw_hal_mac_coex_reg_write(struct rtw_hal_com_t *hal_com, u32 offset, u32 value);
+u32 rtw_hal_mac_set_scoreboard(struct rtw_hal_com_t *hal_com, u32 *value);
+u32 rtw_hal_mac_get_scoreboard(struct rtw_hal_com_t *hal_com, u32 *value);
+u32 rtw_hal_mac_set_grant_act(struct rtw_hal_com_t *hal_com, u8 *value);
+u32 rtw_hal_mac_set_grant(struct rtw_hal_com_t *hal_com, u8 *value);
+u32 rtw_hal_mac_get_grant(struct rtw_hal_com_t *hal_com, u8 *value);
+u32 rtw_hal_mac_set_polluted(struct rtw_hal_com_t *hal_com, u8 band, u8 tx_val, u8 rx_val);
+u32 rtw_hal_mac_get_bt_polt_cnt(struct rtw_hal_com_t *hal_com, u8 band, u16 *cnt);
+u32 rtw_hal_mac_set_coex_ctrl(struct rtw_hal_com_t *hal_com, u32 val);
+u32 rtw_hal_mac_get_coex_ctrl(struct rtw_hal_com_t *hal_com, u32 *val);
+
+void rtw_hal_btc_power_on_ntfy(void *hinfo);
+void rtw_hal_btc_power_off_ntfy(void *hinfo);
+u8 rtw_hal_btc_wl_rfk_ntfy(struct rtw_hal_com_t *hal_com, u8 phy_idx, u8 rfk_type, u8 rfk_process);
+
+enum rtw_hal_status rtw_hal_btc_cfg_1ss(struct rtw_hal_com_t *hal_c,
+		struct rtw_phl_com_t *phl_c, enum band_type band,
+		bool tx_en, bool rx_en, bool tx_res, bool rx_res);
+
+enum rtw_hal_status rtw_hal_btc_cfg_trx_path(struct rtw_hal_com_t *hal_c,
+		enum rf_path tx, u8 tx_nss, enum rf_path rx, u8 rx_nss);
+
+#else /* CONFIG_BTCOEX */
+#define rtw_hal_btc_wl_rfk_ntfy(hal_com, phy_idx, rfk_type, rfk_process) 0
+#endif /* CONFIG_BTCOEX */
+
+u32 rtw_hal_mac_set_tx_time(struct rtw_hal_com_t *hal_com, u8 is_btc,
+			    u8 is_resume, u8 macid, u32 tx_time);
+u32 rtw_hal_mac_get_tx_time(struct rtw_hal_com_t *hal_com, u8 macid, u32 *tx_time);
+u32 rtw_hal_mac_set_tx_retry_limit(struct rtw_hal_com_t *hal_com, u8 is_btc,
+				   u8 is_resume, u8 macid, u8 tx_retry);
+u32 rtw_hal_mac_set_rsc_cfg(struct rtw_hal_com_t *hal_com, u8 rrsr_cfg, u8 band);
+u32 rtw_hal_mac_set_rrsr_ref_rate_sel(struct rtw_hal_com_t *hal_com, bool ref_rate_sel, u8 band);
+u32 rtw_hal_mac_get_tx_retry_limit(struct rtw_hal_com_t *hal_com, u8 macid,
+				   u8 *tx_retry);
+
+u32 rtw_hal_mac_send_h2c(struct rtw_hal_com_t *hal_com,
+	struct rtw_g6_h2c_hdr *hdr, u32 *pvalue);
+
+u32 rtw_hal_read_rf_reg(struct rtw_hal_com_t *hal_com,
+					enum rf_path path, u32 addr, u32 mask);
+
+bool rtw_hal_write_rf_reg(struct rtw_hal_com_t *hal_com,
+					enum rf_path path, u32 addr, u32 mask, u32 data);
+
+u8 rtw_hal_ex_cn_report(struct rtw_hal_com_t *hal_com);
+
+u8 rtw_hal_ex_evm_1ss_report(struct rtw_hal_com_t *hal_com);
+
+u8 rtw_hal_ex_evm_max_report(struct rtw_hal_com_t *hal_com);
+
+u8 rtw_hal_ex_evm_min_report(struct rtw_hal_com_t *hal_com);
+
+u32 rtw_hal_bb_read_cr(struct rtw_hal_com_t *hal_com, u32 addr, u32 mask);
+
+bool rtw_hal_bb_write_cr(struct rtw_hal_com_t *hal_com, u32 addr, u32 mask,
+			 u32 data);
+
+#ifdef CONFIG_PHL_DFS
+bool rtw_hal_is_radar_detect_enabled(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx);
+bool rtw_hal_is_under_cac(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx);
+u32 rtw_hal_mac_cfg_dfs_rpt(struct rtw_hal_com_t *hal_com, struct hal_mac_dfs_rpt_cfg *conf);
+enum rtw_hal_status
+rtw_hal_bb_dfs_rpt_cfg(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx, bool dfs_en);
+#endif
+
+bool rtw_hal_txpwr_by_rate_store_from_external(struct rtw_hal_com_t *hal_com
+	, struct rtw_para_info_t *para_info);
+bool rtw_hal_txpwr_lmt_store_from_external(struct rtw_hal_com_t *hal_com
+	, struct rtw_para_pwrlmt_info_t *para_info);
+bool rtw_hal_txpwr_lmt_ru_store_from_external(struct rtw_hal_com_t *hal_com
+	, struct rtw_para_pwrlmt_info_t *para_info);
+
+u32 rtw_hal_mac_write_msk_pwr_reg(
+	struct rtw_hal_com_t *hal_com, u8 band, u32 offset, u32 mask, u32 val);
+
+u32 rtw_hal_mac_set_pwr_reg(struct rtw_hal_com_t *hal_com, u8 band, u32 offset, u32 val);
+
+u32 rtw_hal_mac_get_pwr_reg(struct rtw_hal_com_t *hal_com, u8 band, u32 offset, u32 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_get_log_efuse_size(struct rtw_hal_com_t *hal_com, u32 *val,
+						bool is_limited);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_a_die_size(struct rtw_hal_com_t *hal_com, u8 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_read_log_efuse_map(struct rtw_hal_com_t *hal_com, u8 *map,
+						bool is_limited);
+
+enum rtw_hal_status
+rtw_hal_mac_write_log_efuse_map(struct rtw_hal_com_t *hal_com,
+						u8 *map,
+						u32 map_size,
+						u8 *mask,
+						u32 mask_size,
+						u8 *map_version,
+						u8 *mask_version,
+						u8 version_length,
+						u8 part,
+						bool is_limited,
+						u8 a_die_size,
+						u32 hci_to_a_die_offset,
+						u32 a_die_start_offset);
+
+enum rtw_hal_status
+rtw_hal_mac_read_hidden_rpt(struct rtw_hal_com_t *hal_com);
+
+enum rtw_hal_status
+rtw_hal_mac_check_efuse_autoload(struct rtw_hal_com_t *hal_com, u8 *autoload);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_avl(struct rtw_hal_com_t *hal_com, u32 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_size(struct rtw_hal_com_t *hal_com, u32 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_mask_size(struct rtw_hal_com_t *hal_com, u32 *val,
+						bool is_limited);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid);
+
+enum rtw_hal_status
+rtw_hal_mac_read_phy_efuse(struct rtw_hal_com_t *hal_com,
+	u32 addr, u32 size, u8 *value);
+
+enum rtw_hal_status
+rtw_hal_mac_read_bt_phy_efuse(struct rtw_hal_com_t *hal_com,
+	u32 addr, u32 size, u8 *value);
+
+enum rtw_hal_status
+rtw_hal_mac_get_xcap(struct rtw_hal_com_t *hal_com, u8 sc_xo, u32 *value);
+
+enum rtw_hal_status
+rtw_hal_mac_set_xcap(struct rtw_hal_com_t *hal_com, u8 sc_xo, u32 value);
+
+enum rtw_hal_status
+rtw_hal_mac_get_xsi(struct rtw_hal_com_t *hal_com, u8 offset, u8 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_set_xsi(struct rtw_hal_com_t *hal_com, u8 offset, u8 val);
+
+enum rtw_hal_status
+rtw_hal_bb_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid);
+
+enum rtw_hal_status
+rtw_hal_rf_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid);
+
+void
+rtw_hal_rf_set_power_table_switch(struct rtw_hal_com_t *hal_com,
+				enum phl_phy_idx phy_idx,
+				u8 pwrbyrate_type, u8 pwrlmt_type);
+
+void rtw_hal_bb_set_tx_pow_per_path_lmt(struct rtw_hal_com_t *hal_com,
+					s16 pwr_lmt_a, s16 pwr_lmt_b);
+
+void
+rtw_hal_bb_set_tx_pow_ref(struct rtw_hal_com_t *hal_com,
+			  enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status rtw_hal_rf_read_pwr_table(
+	struct rtw_hal_com_t *hal_com, u8 rf_path, u16 rate,
+	u8 bandwidth, u8 channel, u8 offset, u8 dcm,
+	u8 beamforming, s16 *get_item);
+
+enum rtw_hal_status rtw_hal_rf_wlan_tx_power_control(
+	struct rtw_hal_com_t *hal_com,
+	enum phl_phy_idx phy, enum phl_pwr_ctrl pwr_ctrl_idx, u32 tx_power_val, bool enable);
+
+enum rtw_hal_status rtw_hal_rf_wl_tx_power_control(
+	struct rtw_hal_com_t *hal_com,
+	u32 tx_power_val);
+
+enum rtw_hal_status rtw_hal_rf_ctrl_dbcc(struct rtw_hal_com_t *hal_com,
+	bool dbcc_en);
+
+void rtw_hal_rf_tssi_scan_ch(struct rtw_hal_com_t *hal_com,
+		enum phl_phy_idx phy_idx, enum rf_path path);
+
+enum rtw_hal_status
+rtw_hal_btc_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid);
+
+enum rtw_hal_status rtw_hal_efuse_get_info(struct rtw_hal_com_t *hal_com,
+					enum rtw_efuse_info info_type, void *value, u8 size);
+enum rtw_hal_status rtw_hal_notify_switch_band(void *hinfo,
+			enum band_type band, enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status rtw_hal_reset(struct rtw_hal_com_t *hal_com,
+			enum phl_phy_idx phy_idx, u8 band_idx, bool reset);
+
+enum rtw_hal_status
+rtw_hal_pause_tx_fifo_sw(void *hal, u8 band_idx,
+			bool tx_pause, enum tx_pause_rson rson);
+
+enum rtw_hal_status
+rtw_hal_tx_pause(struct rtw_hal_com_t *hal_com,
+			u8 band_idx, bool tx_pause, enum tx_pause_rson rson);
+
+enum rtw_hal_status
+rtw_hal_mac_set_macid_pause(struct rtw_hal_com_t *hal_com,
+                            u16 macid, bool pause);
+
+enum rtw_hal_status
+rtw_hal_mac_set_macid_grp_pause(struct rtw_hal_com_t *hal_com,
+                            u32 *macid_arr, u8 macid_arr_sz, bool pause);
+
+enum rtw_hal_status
+rtw_hal_mac_set_macid_pkt_drop(struct rtw_hal_com_t *hal_com,
+                               u16 macid, u8 sel, u8 band, u8 port, u8 mbssid);
+enum rtw_hal_status
+rtw_hal_mac_data_drop_once(struct rtw_hal_com_t *hal_com,
+                                  struct rtw_phl_stainfo_t *sta);
+u32
+rtw_hal_mac_lamode_trig(struct rtw_hal_com_t *hal_com, u8 trig);
+
+enum rtw_hal_status
+rtw_hal_mac_lamode_cfg_buf(struct rtw_hal_com_t *hal_com, u8 buf_sel,
+				u32 *addr_start, u32 *addr_end);
+
+enum rtw_hal_status
+rtw_hal_mac_lamode_cfg(struct rtw_hal_com_t *hal_com, u8 func_en,
+				u8 restart_en, u8 timeout_en, u8 timeout_val,
+				u8 data_loss_imr, u8 la_tgr_tu_sel, u8 tgr_time_val);
+
+enum rtw_hal_status
+rtw_hal_mac_get_lamode_st(struct rtw_hal_com_t *hal_com, u8 *la_state,
+				u16 *la_finish_addr, bool *la_round_up,
+				bool *la_loss_data);
+
+void
+rtw_hal_mac_get_buffer_data(struct rtw_hal_com_t *hal_com, u32 strt_addr,
+				u8 *buf, u32 len, u32 dbg_path);
+
+enum rtw_hal_status
+rtw_hal_fw_log_cfg(void *halcom, u8 op, u8 type, u32 value);
+
+enum rtw_hal_status
+rtw_hal_mac_tx_path_map_cfg(struct rtw_hal_com_t *hal_com,
+                            struct hal_txmap_cfg *txmap_cfg);
+
+enum rtw_hal_status
+rtw_hal_mac_ctrl_ser(struct rtw_hal_com_t *hal_com,
+			enum rtw_hal_ser_rsn rsn, bool en);
+/* HALBB APIs for HW TX */
+
+enum rtw_hal_status
+rtw_hal_bb_set_plcp_tx(struct rtw_hal_com_t *hal_com,
+						struct halbb_plcp_info *plcp_tx_struct,
+						struct usr_plcp_gen_in *plcp_usr_info,
+						enum phl_phy_idx plcp_phy_idx,
+						u8 *plcp_sts);
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_cont_tx(struct rtw_hal_com_t *hal_com, u8 enable, u8 is_cck,
+							enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_carrier_suppression_tx(struct rtw_hal_com_t *hal_com, u8 enable, u8 is_cck,
+							u16 tx_cnt, u16 period, enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_packet_tx(struct rtw_hal_com_t *hal_com, u8 enable,
+							u8 is_cck, u16 tx_cnt ,u16 period, u16 tx_time, u8 cck_lbk_en,
+							enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_set_pmac_fw_trigger_tx(struct rtw_hal_com_t *hal_com, u8 enable,
+							u8 is_cck, u16 tx_cnt, u8 tx_duty,
+							enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_switch_antenna(struct rtw_hal_com_t *hal_com, u8 ant_sw);
+
+enum rtw_hal_status
+rtw_hal_bb_set_dpd_bypass(struct rtw_hal_com_t *hal_com, bool pdp_bypass,
+						enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_set_power(struct rtw_hal_com_t *hal_com, s16 power_dbm,
+					enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_get_power(struct rtw_hal_com_t *hal_com, s16 *power_dbm,
+					enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_rf_set_ch_bw(struct rtw_hal_com_t *hal_com,
+                     enum phl_phy_idx phy,
+                     u8 center_ch,
+                     enum band_type band,
+                     enum channel_width bw);
+
+enum rtw_hal_status
+rtw_hal_rf_chl_rfk_trigger(struct rtw_hal_com_t *hal_com,
+                           u8 phy_idx,
+                           enum rfk_tri_type rt_type);
+
+enum rtw_hal_status
+rtw_hal_rf_update_tas_def_setting(void *hal, u32 tas_config);
+
+enum rtw_hal_status
+rtw_hal_rf_tas_en(void *hal, u8 en);
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_btg(struct rtw_hal_com_t *hal_com, bool btg);
+
+enum rtw_hal_status
+rtw_hal_bb_ctrl_btc_preagc(struct rtw_hal_com_t *hal_com, bool bt_en);
+
+void
+rtw_hal_bb_npath_en_update(struct rtw_hal_com_t *hal_com, bool npath_en);
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_rx_path(struct rtw_hal_com_t *hal_com, u8 rx_path, u8 phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_cfg_tx_path(struct rtw_hal_com_t *hal_com, u8 tx_path, u8 phy_idx);
+
+/* mode: 0 = tmac, 1 = pmac */
+enum rtw_hal_status
+rtw_hal_bb_tx_mode_switch(struct rtw_hal_com_t *hal_com,
+						enum phl_phy_idx phy_idx,
+						u8 mode);
+/* HALBB APIs for HW TX END*/
+/*@--------------------------[Prptotype]-------------------------------------*/
+/**
+ * rtw_hal_bb_get_txsc input arguments:
+ * @hal_com: hal com info
+ * @pri_ch: Spec-defined primary channel index
+ * @central_ch: Spec-defined central channel index
+ * @cbw: Channel BW
+ * @dbw: Data BW
+ */
+u8 rtw_hal_bb_get_txsc(struct rtw_hal_com_t *hal_com, u8 pri_ch,
+		u8 central_ch, enum channel_width cbw, enum channel_width dbw);
+
+void rtw_hal_bb_nhm_mntr_result(struct rtw_hal_com_t *hal_com, void *rpt, enum phl_phy_idx phy_idx);
+
+bool rtw_hal_query_regulation(void *phl, struct rtw_regulation_info *info);
+
+enum rtw_hal_status
+rtw_hal_mac_set_tpu_mode(struct rtw_hal_com_t *hal_com,
+			 enum rtw_tpu_op_mode op_mode_new, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_rua_reg(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_reg(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_by_rate_reg(struct rtw_hal_com_t *hal_com, u8 band);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_bt_avl(struct rtw_hal_com_t *hal_com, u32 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_bt_size(struct rtw_hal_com_t *hal_com, u32 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_bt_mask_size(struct rtw_hal_com_t *hal_com, u32 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_get_log_efuse_bt_size(struct rtw_hal_com_t *hal_com, u32 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_read_log_efuse_bt_map(struct rtw_hal_com_t *hal_com, u8 *map, u32 size);
+
+enum rtw_hal_status
+rtw_hal_mac_write_log_efuse_bt_map(struct rtw_hal_com_t *hal_com,
+                            u8 *map,
+                            u32 map_size,
+                            u8 *mask,
+                            u32 mask_size);
+
+enum rtw_hal_status
+rtw_hal_bb_backup_info(struct rtw_hal_com_t *hal_com, u8 cur_phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_restore_info(struct rtw_hal_com_t *hal_com, u8 cur_phy_idx);
+
+enum rtw_hal_status
+rtw_hal_bb_tssi_bb_reset(struct rtw_hal_com_t *hal_com);
+
+enum rtw_hal_status
+rtw_hal_bb_query_rainfo(void *hal, struct rtw_hal_stainfo_t *hal_sta,
+			struct rtw_phl_rainfo *phl_rainfo);
+enum rtw_hal_status rtw_hal_bb_ctrl_rx_cca(struct rtw_hal_com_t *hal_com,
+	bool cca_en, enum phl_phy_idx phy_idx);
+bool rtw_hal_bb_query_cck_en(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status rtw_hal_bb_ctrl_cck_en(struct rtw_hal_com_t *hal_com,
+	bool cca_en, enum phl_phy_idx phy_idx);
+enum rtw_hal_status
+rtw_hal_mac_set_pwr_ofst_mode(struct rtw_hal_com_t *hal_com, u8 band, s8 pwr_ofst);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_ofst_mode(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_set_pwr_ofst_bw(struct rtw_hal_com_t *hal_com, u8 band, s8 pwr_ofst);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_ofst_bw(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_ref_reg(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_en(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_set_pwr_lmt_en_val(struct rtw_hal_com_t *hal_com, u8 band, bool en_val);
+bool
+rtw_hal_mac_get_pwr_lmt_en_val(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_rua_reg(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_reg(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_by_rate_reg(struct rtw_hal_com_t *hal_com, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_read_efuse_bt_hidden(struct rtw_hal_com_t *hal_com, u32 addr, u32 size,  u8 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_write_efuse_bt_hidden(struct rtw_hal_com_t *hal_com, u32 addr, u8 val);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_init_bf_role(struct rtw_hal_com_t *hal_com, u8 bf_role, u8 band);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_deinit_bfee(struct rtw_hal_com_t *hal_com, u8 band);
+
+void rtw_hal_bb_gpio_setting_all(struct rtw_hal_com_t *hal_com, u8 rfe_idx);
+
+void rtw_hal_bb_gpio_setting(struct rtw_hal_com_t *hal_com, u8 gpio_idx, u8 path,
+			bool inv, u8 src);
+
+enum rtw_hal_status
+rtw_hal_mac_set_gpio_func(struct rtw_hal_com_t *hal_com, u8 func, s8 gpio_cfg);
+
+enum rtw_phl_status
+rtw_hal_cmd_notify(struct rtw_phl_com_t *phl_com,
+                   enum phl_msg_evt_id event,
+                   void *hal_cmd,
+                   u8 hw_idx);
+
+#ifdef CONFIG_PHL_DIAGNOSE
+void rtw_hal_bb_diagnostic_event(struct rtw_hal_com_t *hal, u8 type,
+		u8 level, u8 version, u8 *buf, u32 len);
+
+void rtw_hal_rf_diagnostic_event(struct rtw_hal_com_t *hal, u8 type,
+		u8 level, u8 version, u8 *buf, u32 len);
+#endif
+
+#ifdef CONFIG_PHL_IO_OFLD
+enum rtw_hal_status rtw_hal_mac_add_cmd_ofld(struct rtw_hal_com_t *hal_com, struct rtw_mac_cmd *cmd);
+#endif
+
+void rtw_hal_bb_env_rpt(struct rtw_hal_com_t *hal_com, struct rtw_env_report *env_rpt,
+		     enum phl_phy_idx phy_indx);
+
+void rtw_hal_bb_set_pow_patten_sharp(struct rtw_hal_com_t *hal_com, u8 channel, u8 is_cck, u8 sharp_id, enum phl_phy_idx phy_idx);
+
+#if defined(CONFIG_PHL_DRV_HAS_NVM) || defined(RTW_FLASH_98D)
+/* For BB to get paramater values */
+enum rtw_hal_status
+rtw_hal_flash_get_info(struct rtw_hal_com_t *hal_com,
+		       enum rtw_efuse_info info_type,
+		       void *value,
+		       u8 size);
+#endif /* CONFIG_PHL_DRV_HAS_NVM || RTW_FLASH_98D */
+
+enum rtw_hal_status
+rtw_hal_rf_syn_config(struct rtw_hal_com_t *hal_com,
+                      u8 syn_id,
+                      enum phl_phy_idx phy_idx,
+                      u8 path,
+                      bool turn_on);
+void rtw_hal_rf_set_pwr_lmt_main_or_aux(struct rtw_hal_com_t *hal_com, u8 ant);
+
+bool
+rtw_hal_bb_adc_cfg(struct rtw_hal_com_t *hal_com,
+                   enum channel_width bw,
+                   enum rf_path path,
+                   enum phl_phy_idx phy_idx);
+
+void rtw_hal_rf_set_ant_main_or_aux(void *hal, enum rf_path path, bool main);
+
+struct halrf_fem_info rtw_hal_rf_efem_info(struct rtw_hal_com_t *hal_com);
+
+enum rtw_hal_status
+rtw_hal_mac_sr_update(struct rtw_hal_com_t *hal_com,
+                      void *sr_info,
+                      u8 hw_band);
+
+char *rtw_hal_get_ext_regd_name(struct rtw_para_pwrlmt_info_t *para_info, u8 idx);
+void rtw_hal_bb_bb_reset_cmn(struct rtw_hal_com_t *hal, bool en, enum phl_phy_idx phy_idx);
+
+#endif /*_HAL_API_DRV_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_efuse.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_efuse.c
new file mode 100644
index 000000000000..1889c73d9b16
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_efuse.c
@@ -0,0 +1,639 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_API_EFUSE_C_
+#include "hal_headers.h"
+#include "efuse/hal_efuse_export.h"
+
+/*WIFI Efuse*/
+enum rtw_hal_status
+rtw_hal_efuse_shadow_load(struct hal_info_t *hal_info, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_shadow_load(hal_info->efuse, is_limit);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_shadow_update(struct hal_info_t *hal_info, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+
+	status = rtw_efuse_shadow_update(hal_info->efuse, is_limit);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_shadow_read(struct hal_info_t *hal_info, u8 byte_count,
+						  u16 offset, u32 *value, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_shadow_read(hal_info->efuse, byte_count, offset, value,
+								   is_limit);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_shadow_write(struct hal_info_t *hal_info, u8 byte_count,
+						   u16 offset, u32 value, bool is_limit)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_shadow_write(hal_info->efuse, byte_count, offset, value,
+									is_limit);
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_shadow2buf(struct hal_info_t *hal_info, u8 *pbuf, u16 buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+	status = rtw_efuse_shadow2buf(hal_info->efuse, pbuf, buflen);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_file_map_load(
+	struct hal_info_t *hal_info, char *file_path, u8 is_limit)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_efuse_file_map_load(hal_info->efuse, file_path ,is_limit);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_file_mask_load(
+	struct hal_info_t *hal_info, char *file_path, u8 is_limit)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_efuse_file_mask_load(hal_info->efuse, file_path, is_limit);
+
+	return hal_status;
+}
+
+/* usage = used percentage(1 Byte) + used bytes(2 Bytes) */
+enum rtw_hal_status rtw_hal_efuse_get_usage(struct hal_info_t *hal_info,
+	u32 *usage)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_usage(hal_info->efuse, usage);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_get_logical_size(struct hal_info_t *hal_info,
+	u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_logical_size(hal_info->efuse, size, true);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_get_size(struct hal_info_t *hal_info,
+	u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_size(hal_info->efuse, size);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_get_avl(struct hal_info_t *hal_info,
+	u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_avl(hal_info->efuse, size);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_get_shadowmap_from(struct hal_info_t *hal_info,
+	u8 *val)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_shadowmap_from(hal_info->efuse, val);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_get_offset_mask(struct hal_info_t *hal_info,
+	u16 offset, u8 *mask)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_offset_mask(hal_info->efuse, offset, mask);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_get_mask_buf(struct hal_info_t *hal_info, 
+							u8 *destbuf, u32 *buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_mask_buf(hal_info->efuse, destbuf, buflen);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_get_info(struct rtw_hal_com_t *hal_com,
+		       enum rtw_efuse_info info_type,
+		       void *value,
+		       u8 size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	status = rtw_efuse_get_info(hal_info->efuse, info_type, value, size);
+
+	return status;
+}
+
+/* API export to PHL : rtw_hal_get_efuse_info */
+enum rtw_hal_status
+rtw_hal_get_efuse_info(void *hal,
+		       enum rtw_efuse_info info_type,
+		       void *value,
+		       u8 size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	status = rtw_efuse_get_info(hal_info->efuse, info_type, value, size);
+
+	return status;
+}
+
+void rtw_hal_efuse_process(struct rtw_phl_com_t *phl_com,
+                           struct hal_info_t *hal_info,
+                           char *ic_name
+)
+{
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+#endif /* DBG_MONITOR_TIME */
+
+	if (rtw_efuse_is_processed(hal_info->efuse) == true) {
+		PHL_INFO("%s EFUSE module is already initialized.\n", __FUNCTION__);
+		return;
+	}
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+#ifdef CONFIG_PHL_FW_DUMP_EFUSE
+	rtw_phl_fw_dump_efuse_precfg(phl_com);
+#endif
+
+	rtw_efuse_process(hal_info->efuse, ic_name);
+
+#ifdef CONFIG_PHL_FW_DUMP_EFUSE
+	rtw_phl_fw_dump_efuse_postcfg(phl_com);
+#endif
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_EFUSE_PROC);
+#endif /* DBG_MONITOR_TIME */
+}
+
+enum rtw_hal_status rtw_hal_efuse_init(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	status = rtw_efuse_init(phl_com, hal_com, &(hal_info->efuse));
+
+	return status;
+}
+
+void rtw_hal_efuse_deinit(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	rtw_efuse_deinit(hal_com, hal_info->efuse);
+}
+
+
+enum rtw_hal_status
+rtw_hal_efuse_bt_shadow_load(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_shadow_load(hal_info->efuse);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_bt_shadow_update(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+
+	status = rtw_efuse_bt_shadow_update(hal_info->efuse);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_bt_shadow_read(struct hal_info_t *hal_info, u8 byte_count,
+						  u16 offset, u32 *value)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_shadow_read(hal_info->efuse, byte_count, offset, value);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_efuse_bt_shadow_write(struct hal_info_t *hal_info, u8 byte_count,
+						   u16 offset, u32 value)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_shadow_write(hal_info->efuse, byte_count, offset, value);
+	return status;
+}
+
+
+enum rtw_hal_status
+rtw_hal_efuse_bt_shadow2buf(struct hal_info_t *hal_info, u8 *pbuf, u16 buflen)
+{
+   enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+   status = rtw_efuse_bt_shadow2buf(hal_info->efuse, pbuf, buflen);
+
+   return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_bt_file_map_load(
+	struct hal_info_t *hal_info, char *file_path)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_efuse_bt_file_map_load(hal_info->efuse, file_path);
+
+	return hal_status;
+}
+
+
+enum rtw_hal_status rtw_hal_efuse_bt_file_mask_load(
+	struct hal_info_t *hal_info, char *file_path)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_efuse_bt_file_mask_load(hal_info->efuse, file_path);
+
+	return hal_status;
+}
+
+/* usage = used percentage(1 Byte) + used bytes(2 Bytes) */
+enum rtw_hal_status rtw_hal_efuse_bt_get_usage(struct hal_info_t *hal_info,
+	u32 *usage)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_get_usage(hal_info->efuse, usage);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_logical_size(struct hal_info_t *hal_info,
+	u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_get_logical_size(hal_info->efuse, size);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_size(struct hal_info_t *hal_info,
+	u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_get_size(hal_info->efuse, size);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_avl(struct hal_info_t *hal_info,
+	u32 *size)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_get_avl(hal_info->efuse, size);
+
+	return status;
+}
+
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_offset_mask(struct hal_info_t *hal_info,
+	u16 offset, u8 *mask)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_get_offset_mask(hal_info->efuse, offset, mask);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_mask_buf(struct hal_info_t *hal_info,
+							u8 *destbuf, u32 *buflen)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_bt_get_mask_buf(hal_info->efuse, destbuf, buflen);
+
+	return status;
+}
+enum rtw_hal_status rtw_hal_efuse_bt_read_hidden(
+	struct hal_info_t *hal_info, u32 addr, u32 size, u8 *val)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_efuse_bt_read_hidden(hal_info->efuse, addr, size, val);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_bt_write_hidden(
+	struct hal_info_t *hal_info, u32 addr, u8 val)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_efuse_bt_write_hidden(hal_info->efuse, addr, val);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_read_phy_efuse(
+	struct hal_info_t *hal_info, u32 addr, u32 size, u8 *data, u8 type)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_read_phy_wifi(hal_info->efuse, addr, size, data, type);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_efuse_renew(
+	struct hal_info_t *hal_info, u8 type)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_efuse_renew(hal_info->efuse, type);
+
+	return status;
+}
+
+void hal_efuse_dump_wifi_map(
+	struct hal_info_t *hal_info,
+	u32 *used_len,
+	char *output,
+	u32 *out_len
+	)
+{
+	void *d = hal_info->hal_com->drv_priv;
+	u32 _used_len = *used_len;
+	u32 _out_len = *out_len;
+	u32 map_len = 0;
+	u32 i = 0, j = 0;
+	u8 *buf = NULL;
+
+	rtw_efuse_get_logical_size(hal_info->efuse, &map_len, true);
+
+	if(map_len == 0){
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "map length is 0!\n");
+		goto exit;
+	}
+
+	buf = _os_mem_alloc(d, map_len);
+
+	if(buf == NULL){
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "Allocate buffer fail!\n");
+		goto exit;
+	}
+	rtw_efuse_shadow2buf(hal_info->efuse, buf, (u16)map_len);
+
+	PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len, _out_len - _used_len,
+			 "EFUSE Wifi shadow map from %s\n\n",
+			 rtw_efuse_get_shadowmap_from_to_str(hal_info->efuse));
+
+	PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len, _out_len - _used_len,
+			 "%14s%-6s%-6s%-6s%-6s%-6s%-6s%-6s%-6s\n",
+			 " ", "00", "02", "04", "06", "08", "0A", "0C", "0E");
+	PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len, _out_len - _used_len,
+			 "%14s%s\n",
+			 " ", "================================================\n");
+	for(i = 0; i < map_len; i+=16){
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "0x%08X%4s", i, "");
+		for(j = 0; j<16;j++){
+			if((i+j) >= map_len)
+				break;
+			PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+					 _out_len - _used_len, "%02X ", buf[i+j]);
+		}
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "\n");
+	}
+	if(buf)
+		_os_mem_free(d, buf, map_len);
+exit:
+	*used_len = _used_len;
+	*out_len = _out_len;
+}
+
+void hal_efuse_dump_wifi_logic_map(
+	struct hal_info_t *hal_info,
+	u32 *used_len,
+	char *output,
+	u32 *out_len
+	)
+{
+	void *d = hal_info->hal_com->drv_priv;
+	u32 _used_len = *used_len;
+	u32 _out_len = *out_len;
+	u32 map_len = 0;
+	u32 i = 0, j = 0;
+	u8 *data = NULL;
+
+	rtw_efuse_get_logical_size(hal_info->efuse, &map_len, true);
+
+	if(map_len == 0){
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "map length is 0!\n");
+		goto exit;
+	}
+
+	data = _os_mem_alloc(d, map_len);
+
+	if(data == NULL){
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "Allocate buffer fail!\n");
+		goto exit;
+	}
+
+	rtw_efuse_logicmap_buf_load(hal_info->efuse, data, false);
+
+	PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len, _out_len - _used_len,
+			 "EFUSE Wifi map from HW\n");
+
+	PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len, _out_len - _used_len,
+			 "%14s%-6s%-6s%-6s%-6s%-6s%-6s%-6s%-6s\n",
+			 " ", "00", "02", "04", "06", "08", "0A", "0C", "0E");
+	PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len, _out_len - _used_len,
+			 "%14s%s\n",
+			 " ", "================================================\n");
+	for(i = 0; i < map_len; i+=16){
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "0x%08X%4s", i, "");
+		for(j = 0; j<16;j++){
+			if((i+j) >= map_len)
+				break;
+			PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+					 _out_len - _used_len, "%02X ", data[i+j]);
+		}
+		PHL_DBG_OUTBUF(_out_len, _used_len, output + _used_len,
+				 _out_len - _used_len, "\n");
+	}
+	if(data)
+		_os_mem_free(d, data, map_len);
+exit:
+	*used_len = _used_len;
+	*out_len = _out_len;
+}
+
+void hal_efuse_cmd_parser(
+	struct hal_info_t *hal_info,
+	char input[][MAX_ARGV],
+	u32 input_num,
+	char *output,
+	u32 out_len
+	)
+{
+	u32 hal_cmd_ary_size = sizeof(hal_efuse_cmd_i) / sizeof(struct hal_cmd_info);
+	u32 i = 0;
+	u8 id = 0;
+	u32 used = 0;
+
+	PHL_DBG_OUTBUF(out_len, used, output + used, out_len - used, "\n");
+
+	if (hal_cmd_ary_size == 0)
+		return;
+
+	/* Parsing Cmd ID */
+	if (input_num) {
+		for (i = 0; i < hal_cmd_ary_size; i++) {
+			if (_os_strcmp(hal_efuse_cmd_i[i].name, input[0]) == 0) {
+				id = hal_efuse_cmd_i[i].id;
+				PHL_INFO("enter EFUSE cmd %s\n", hal_efuse_cmd_i[i].name);
+				break;
+			}
+		}
+		if (i == hal_cmd_ary_size) {
+			PHL_DBG_OUTBUF(out_len, used, output + used,
+					 out_len - used,
+					 "EFUSE command not found!\n");
+			return;
+		}
+	}
+
+	switch (id) {
+		case HAL_EFUSE_HELP:
+			PHL_DBG_OUTBUF(out_len, used, output + used,
+					 out_len - used,
+					 "EFUSE cmd =>\n");
+			for (i = 0; i < hal_cmd_ary_size - 1; i++)
+				PHL_DBG_OUTBUF(out_len, used, output + used,
+						 out_len - used," %s\n", hal_efuse_cmd_i[i].name);
+			break;
+		case HAL_EFUSE_WIFI_DUMP_MAP:
+			hal_efuse_dump_wifi_map(hal_info, &used, output, &out_len);
+			break;
+		case HAL_EFUSE_WIFI_LOGIC_DUMP_MAP:
+			hal_efuse_dump_wifi_logic_map(hal_info, &used, output, &out_len);
+			break;
+		default:
+			PHL_DBG_OUTBUF(out_len, used, output + used,
+					 out_len - used,"Do not support this command\n");
+			break;
+	}
+}
+
+s32 hal_efuse_cmd(
+	struct hal_info_t *hal_info,
+	char *input,
+	char *output,
+	u32 out_len
+	)
+{
+	char *token;
+	u32 argc = 0;
+	char argv[MAX_ARGC][MAX_ARGV];
+
+	do {
+		token = _os_strsep(&input, ", ");
+		if (token) {
+			if (_os_strlen((u8*)token) <= MAX_ARGV)
+				_os_strcpy(argv[argc], token);
+
+			argc++;
+		} else {
+			break;
+		}
+	} while (argc < MAX_ARGC);
+
+	hal_efuse_cmd_parser(hal_info, argv, argc, output, out_len);
+
+	return 0;
+}
+
+bool rtw_hal_efuse_proc_cmd(
+	struct hal_info_t *hal_info,
+	struct rtw_proc_cmd *incmd,
+	char *output,
+	u32 out_len
+	)
+{
+	if(incmd->in_type == RTW_ARG_TYPE_BUF)
+		hal_efuse_cmd(hal_info, incmd->in.buf, output, out_len);
+	else if(incmd->in_type == RTW_ARG_TYPE_ARRAY){
+		hal_efuse_cmd_parser(hal_info, incmd->in.vector,
+					incmd->in_cnt_len, output, out_len);
+	}
+
+	return true;
+}
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_efuse.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_efuse.h
new file mode 100644
index 000000000000..7f893427f59c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_efuse.h
@@ -0,0 +1,117 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_EFUSE_H_
+#define _HAL_API_EFUSE_H_
+
+/* efuse exported API */
+/* WIFI EFUSE */
+enum rtw_hal_status rtw_hal_efuse_init(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info);
+void rtw_hal_efuse_deinit(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info);
+void rtw_hal_efuse_process(struct rtw_phl_com_t *phl_com,
+                           struct hal_info_t *hal_info,
+                           char *ic_name
+);
+enum rtw_hal_status
+rtw_hal_efuse_shadow_load(struct hal_info_t *hal_info, bool is_limit);
+enum rtw_hal_status
+rtw_hal_efuse_shadow_read(struct hal_info_t *hal_info, u8 byte_count,
+						  u16 offset, u32 *value, bool is_limit);
+enum rtw_hal_status
+rtw_hal_efuse_shadow_write(struct hal_info_t *hal_info, u8 byte_count,
+						   u16 offset, u32 value, bool is_limit);
+enum rtw_hal_status
+rtw_hal_efuse_shadow_update(struct hal_info_t *hal_info, bool is_limit);
+enum rtw_hal_status rtw_hal_efuse_shadow2buf(struct hal_info_t *hal_info,
+	u8 *pbuf, u16 buflen);
+enum rtw_hal_status rtw_hal_efuse_file_map_load(struct hal_info_t *hal_info,
+	char *file_path, u8 is_limit);
+enum rtw_hal_status rtw_hal_efuse_file_mask_load(struct hal_info_t *hal_info,
+	char *file_path, u8 is_limit);
+enum rtw_hal_status rtw_hal_efuse_get_usage(struct hal_info_t *hal_info,
+	u32 *usage);
+enum rtw_hal_status rtw_hal_efuse_get_logical_size(struct hal_info_t *hal_info,
+	u32 *size);
+enum rtw_hal_status rtw_hal_efuse_get_size(struct hal_info_t *hal_info,
+	u32 *size);
+enum rtw_hal_status rtw_hal_efuse_get_avl(struct hal_info_t *hal_info,
+	u32 *size);
+enum rtw_hal_status rtw_hal_efuse_get_shadowmap_from(struct hal_info_t *hal_info,
+	u8 *val);
+enum rtw_hal_status rtw_hal_efuse_get_offset_mask(struct hal_info_t *hal_info,
+	u16 offset, u8 *mask);
+enum rtw_hal_status rtw_hal_efuse_get_mask_buf(struct hal_info_t *hal_info,
+	u8 *mask, u32 *buflen);
+
+
+/* BT EFUSE */
+enum rtw_hal_status rtw_hal_efuse_bt_shadow_load(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_efuse_bt_shadow_read(struct hal_info_t *hal_info, u8 byte_count,
+						  u16 offset, u32 *value);
+
+enum rtw_hal_status rtw_hal_efuse_bt_shadow_write(struct hal_info_t *hal_info, u8 byte_count,
+						   u16 offset, u32 value);
+
+enum rtw_hal_status rtw_hal_efuse_bt_shadow_update(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_efuse_bt_shadow2buf(struct hal_info_t *hal_info,
+	u8 *pbuf, u16 buflen);
+
+enum rtw_hal_status rtw_hal_efuse_bt_file_map_load(
+	struct hal_info_t *hal_info, char *file_path);
+
+enum rtw_hal_status rtw_hal_efuse_bt_file_mask_load(
+	struct hal_info_t *hal_info, char *file_path);
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_usage(struct hal_info_t *hal_info,
+	u32 *usage);
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_logical_size(struct hal_info_t *hal_info,
+	u32 *size);
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_size(struct hal_info_t *hal_info,
+	u32 *size);
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_avl(struct hal_info_t *hal_info,
+	u32 *size);
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_offset_mask(struct hal_info_t *hal_info,
+	u16 offset, u8 *mask);
+
+enum rtw_hal_status rtw_hal_efuse_bt_get_mask_buf(struct hal_info_t *hal_info,
+	u8 *mask, u32 *buflen);
+
+enum rtw_hal_status rtw_hal_efuse_bt_read_hidden(
+	struct hal_info_t *hal_info, u32 addr, u32 size, u8 *val);
+
+enum rtw_hal_status rtw_hal_efuse_bt_write_hidden(
+	struct hal_info_t *hal_info, u32 addr, u8 val);
+
+enum rtw_hal_status rtw_hal_efuse_read_phy_efuse(
+	struct hal_info_t *hal_info, u32 addr, u32 size, u8 *data, u8 type);
+
+enum rtw_hal_status rtw_hal_efuse_renew(
+	struct hal_info_t *hal_info, u8 type);
+
+bool rtw_hal_efuse_proc_cmd(
+	struct hal_info_t *hal_info,
+	struct rtw_proc_cmd *incmd,
+	char *output,
+	u32 out_len
+	);
+
+#endif /* _HAL_API_EFUSE_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_mac.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_mac.c
new file mode 100644
index 000000000000..0618a20650b4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_mac.c
@@ -0,0 +1,12625 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_API_MAC_C_
+#include "hal_headers.h"
+#include "mac/mac_ax.h"
+#ifdef RTW_PHL_BCN
+#include "mac/mac_ax/fwcmd.h"
+#endif
+
+#define RTL8852A_FPGA_VERIFICATION 1
+
+u16 hal_mac_get_macid_num(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->hw_info->macid_num;
+}
+void hal_mac_get_hwinfo(struct hal_info_t *hal, struct hal_spec_t *hal_spec)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_hw_info *mac_info = mac->hw_info;
+	struct mac_ax_ops *ops = mac->ops;
+	/*struct mac_ax_hw_info* (*get_hw_info)(struct mac_ax_adapter *adapter);*/
+
+	mac_info = ops->get_hw_info(mac);
+
+	hal_spec->macid_num = mac_info->macid_num;
+
+	hal->hal_com->cv = mac_info->cv;
+	hal->hal_com->acv = mac_info->acv;
+	hal->hal_com->fv = mac_info->fv;
+	hal->hal_com->aid = mac_info->aid;
+
+	PHL_INFO("[MAC-INFO]- CV : %d\n", mac_info->cv);
+	PHL_INFO("[MAC-INFO]- ACV : %d\n", mac_info->acv);
+	PHL_INFO("[MAC-INFO]- FV : %d\n", mac_info->fv);
+	PHL_INFO("[MAC-INFO]- AID : %d\n", mac_info->aid);
+	PHL_INFO("[MAC-INFO]- tx_ch_num: %d\n", mac_info->tx_ch_num);
+
+	PHL_INFO("[MAC-INFO]- tx_data_ch_num: %d\n", mac_info->tx_data_ch_num);
+	PHL_INFO("[MAC-INFO]- wd_body_len: %d\n", mac_info->wd_body_len);
+	PHL_INFO("[MAC-INFO]- wd_info_len: %d\n", mac_info->wd_info_len);
+
+	PHL_INFO("[MAC-INFO]- pwr_seq_ver: %d\n", mac_info->pwr_seq_ver);
+	PHL_INFO("[MAC-INFO]- fifo_size: %d\n", mac_info->fifo_size);
+	PHL_INFO("[MAC-INFO]- macid_num: %d\n", mac_info->macid_num);
+	PHL_INFO("[MAC-INFO]- wl_efuse_size: %d\n", mac_info->wl_efuse_size);
+
+	PHL_INFO("[MAC-INFO]- efuse_size: %d\n", mac_info->efuse_size);
+	PHL_INFO("[MAC-INFO]- log_efuse_size: %d\n", mac_info->log_efuse_size);
+
+	PHL_INFO("[MAC-INFO]- bt_efuse_size: %d\n", mac_info->bt_efuse_size);
+	PHL_INFO("[MAC-INFO]- sec_ctrl_efuse_size: %d\n", mac_info->sec_ctrl_efuse_size);
+	PHL_INFO("[MAC-INFO]- sec_data_efuse_size: %d\n", mac_info->sec_data_efuse_size);
+
+}
+
+enum rtw_hal_status
+rtw_hal_mac_watchdog(struct hal_info_t *hal_info, struct rtw_phl_com_t *phl_com)
+{
+
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_wdt_param param = {0};
+	u32 ret = 0;
+#if 0
+	struct mac_ax_resp_chk_cca resp_chk;
+
+	if (phl_com->edcca_mode)
+	{
+		ret = mac->ops->sifs_chk_cca_en(mac, MAC_AX_BAND_0, 1);
+		if (ret != MACSUCCESS) {
+			PHL_ERR("[ERR]sifs_chk_cca_en(1) %d\n", ret);
+		}
+
+		resp_chk.band = MAC_AX_BAND_0;
+		resp_chk.resp_ack_chk_cca_en = 1;
+		ret = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_RESP_ACK_CHK_CCA, &resp_chk);
+
+		if (ret != MACSUCCESS) {
+			PHL_ERR("[ERR]mac_resp_chk_cca(1) %d\n", ret);
+		}
+	} else {
+		ret = mac->ops->sifs_chk_cca_en(mac, MAC_AX_BAND_0, 0);
+		if (ret != MACSUCCESS) {
+			PHL_ERR("[ERR]sifs_chk_cca_en(0) %d\n", ret);
+		}
+
+		resp_chk.band = MAC_AX_BAND_0;
+		resp_chk.resp_ack_chk_cca_en = 0;
+		ret = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_RESP_ACK_CHK_CCA, &resp_chk);
+
+		if (ret != MACSUCCESS) {
+			PHL_ERR("[ERR]mac_resp_chk_cca(0) %d\n", ret);
+		}
+	}
+#endif
+	param.drv_ctrl.autok_wdt_ctrl = MAC_AX_PCIE_DEFAULT;
+	param.drv_ctrl.tp_wdt_ctrl = MAC_AX_PCIE_DEFAULT;
+	param.tp.tx_tp = (u16)(phl_com->phl_stats.tx_tp_kbits >> 10);
+	param.tp.rx_tp = (u16)(phl_com->phl_stats.rx_tp_kbits >> 10);
+
+	ret = mac->ops->watchdog(mac, &param);
+
+	return (ret == MACSUCCESS) ?
+	       (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status
+hal_mac_set_chip_id(struct rtw_hal_com_t *hal_com,
+			struct mac_ax_adapter *mac)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	switch (mac->drv_info->sw_chip_id) {
+	case MAC_AX_CHIP_ID_8851B:
+		hal_com->chip_id = CHIP_WIFI6_8851B;
+		break;
+	case MAC_AX_CHIP_ID_8852A:
+		hal_com->chip_id = CHIP_WIFI6_8852A;
+		break;
+	case MAC_AX_CHIP_ID_8852B:
+		hal_com->chip_id = CHIP_WIFI6_8852B;
+		break;
+	case MAC_AX_CHIP_ID_8852BT:
+		hal_com->chip_id = CHIP_WIFI6_8852BT;
+		break;
+	case MAC_AX_CHIP_ID_8852C:
+		hal_com->chip_id = CHIP_WIFI6_8852C;
+		break;
+	case MAC_AX_CHIP_ID_8852D:
+		hal_com->chip_id = CHIP_WIFI6_8852D;
+		break;
+	case MAC_AX_CHIP_ID_8192XB:
+		hal_com->chip_id = CHIP_WIFI6_8192XB;
+		break;
+	default:
+		hal_com->chip_id = CHIP_WIFI6_MAX;
+		hal_status = RTW_HAL_STATUS_MAC_INIT_FAILURE;
+		break;
+	}
+	PHL_INFO("%s : hal_com->chip_id = %d\n", __FUNCTION__, hal_com->chip_id);
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_cfg_txdma(struct hal_info_t *hal, u8 state)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 ret = 0;
+
+	ret = hal_mac_ops->intf_ops->ctrl_txdma(mac, state);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_poll_txdma_idle(struct hal_info_t *hal)
+{
+
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 ret = 0;
+
+	ret = hal_mac_ops->intf_ops->poll_txdma_idle(mac);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS) : (RTW_HAL_STATUS_FAILURE);
+
+}
+
+#ifdef CONFIG_PCI_HCI
+enum rtw_hal_status rtw_hal_mac_set_pcicfg(struct hal_info_t *hal_info,
+					struct mac_ax_pcie_cfgspc_param *pci_cfgspc)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 ret = 0;
+
+	ret = mac->ops->set_hw_value(mac, MAC_AX_HW_PCIE_CFGSPC_SET, pci_cfgspc);
+	return (ret == 0) ? (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_ltr_sw_trigger(struct hal_info_t *hal_info, enum rtw_pcie_ltr_state state)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	enum mac_ax_pcie_ltr_sw_ctrl ctrl;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_FAILURE;
+	u32 ret = 0;
+
+	switch (state) {
+
+	case RTW_PCIE_LTR_SW_ACT:
+		ctrl = MAC_AX_PCIE_LTR_SW_ACT;
+		break;
+	case RTW_PCIE_LTR_SW_IDLE:
+		ctrl = MAC_AX_PCIE_LTR_SW_IDLE;
+		break;
+	default:
+		PHL_INFO("%s: state = %u, fail to trigger \n", __func__, state);
+		return RTW_HAL_STATUS_FAILURE;
+
+	}
+
+	ret = mac->ops->set_hw_value(mac, MAX_AX_HW_PCIE_LTR_SW_TRIGGER, &ctrl);
+
+	if (ret == 0)
+		hstats = RTW_HAL_STATUS_SUCCESS;
+ 	else
+		hstats = RTW_HAL_STATUS_FAILURE;
+
+	return hstats;
+}
+
+enum rtw_hal_status rtw_hal_mac_ltr_set_pcie(struct hal_info_t *hal_info,
+	enum rtw_pcie_bus_func_cap_t hw_ctrl,
+	u8 idle_ctrl, u32 idle_val, u8 act_ctrl, u32 act_val)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_pcie_ltr_param param = {0};
+	u32 ret = 0;
+	enum mac_ax_pcie_func_ctrl ctrl;
+
+	switch (hw_ctrl) {
+
+	case RTW_PCIE_BUS_FUNC_ENABLE:
+		ctrl = MAC_AX_PCIE_ENABLE;
+		break;
+	case RTW_PCIE_BUS_FUNC_DISABLE:
+		ctrl = MAC_AX_PCIE_DISABLE;
+		break;
+	case RTW_PCIE_BUS_FUNC_DEFAULT:
+		ctrl = MAC_AX_PCIE_DEFAULT;
+		break;
+	default:
+		ctrl = MAC_AX_PCIE_IGNORE;
+	}
+
+	param.write = 1;
+	param.read = 0;
+	param.ltr_ctrl = MAC_AX_PCIE_IGNORE;
+	param.ltr_hw_ctrl = ctrl;
+	param.ltr_spc_ctrl = MAC_AX_PCIE_LTR_SPC_IGNORE;
+	param.ltr_idle_timer_ctrl = MAC_AX_PCIE_LTR_IDLE_TIMER_IGNORE;
+	param.ltr_rx0_th_ctrl.ctrl = MAC_AX_PCIE_IGNORE;
+	param.ltr_rx0_th_ctrl.val = 0;
+	param.ltr_rx1_th_ctrl.ctrl = MAC_AX_PCIE_IGNORE;
+	param.ltr_rx1_th_ctrl.val = 0;
+
+	/* when ctrl is not equal to 0, setting ignore to make hw control the value */
+	if (idle_ctrl) {
+		param.ltr_idle_lat_ctrl.ctrl = MAC_AX_PCIE_ENABLE;
+		param.ltr_idle_lat_ctrl.val = idle_val;
+	} else {
+		param.ltr_idle_lat_ctrl.ctrl = MAC_AX_PCIE_IGNORE;
+		param.ltr_idle_lat_ctrl.val = 0;
+	}
+
+	if (act_ctrl) {
+		param.ltr_act_lat_ctrl.ctrl = MAC_AX_PCIE_ENABLE;
+		param.ltr_act_lat_ctrl.val = act_val;
+	} else {
+		param.ltr_act_lat_ctrl.ctrl = MAC_AX_PCIE_IGNORE;
+		param.ltr_act_lat_ctrl.val = 0;
+	}
+
+	PHL_INFO("%s: ltr_hw_ctrl %u \n", __func__, param.ltr_hw_ctrl);
+	PHL_INFO("%s: ltr_idle_lat_ctrl, ctrl %u, val %u\n", __func__,
+		param.ltr_idle_lat_ctrl.ctrl, param.ltr_idle_lat_ctrl.val);
+	PHL_INFO("%s: ltr_act_lat_ctrl, ctrl %u, val %u\n", __func__,
+		param.ltr_act_lat_ctrl.ctrl, param.ltr_act_lat_ctrl.val);
+
+
+	ret = mac->ops->intf_ops->ltr_set_pcie(mac, &param);
+	return (ret == 0) ? (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status hal_mac_set_l2_leave(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 ret = 0;
+	u8 set = true;
+
+	ret = mac->ops->set_hw_value(mac, MAX_AX_HW_PCIE_L2_LEAVE, &set);
+	return (ret == MACSUCCESS) ? (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_poll_rxdma_idle(struct hal_info_t *hal,
+					struct mac_ax_rxdma_ch_map *ch_map)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 ret = 0;
+
+	ret = hal_mac_ops->intf_ops->poll_rxdma_ch_idle(mac, ch_map);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_clr_bdidx(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 ret = 0;
+
+	ret = hal_mac_ops->intf_ops->clr_idx_all(mac);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_rst_bdram(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 ret = 0;
+	u8 val;
+
+	ret = hal_mac_ops->set_hw_value(mac, MAC_AX_HW_PCIE_RST_BDRAM, &val);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_cfg_dma_io(struct hal_info_t *hal, u8 en)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	enum mac_ax_func_sw ctrl;
+	u32 ret = 0;
+
+	if (en)
+		ctrl = MAC_AX_FUNC_EN;
+	else
+		ctrl = MAC_AX_FUNC_DIS;
+
+	ret = hal_mac_ops->intf_ops->ctrl_dma_io(mac, ctrl);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+void hal_txch_map_ignore(struct mac_ax_txdma_ch_map *txch_map)
+{
+	txch_map->ch0 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch1 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch2 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch3 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch4 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch5 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch6 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch7 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch8 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch9 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch10 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch11 = MAC_AX_PCIE_IGNORE;
+	txch_map->ch12 = MAC_AX_PCIE_IGNORE;
+}
+
+void hal_txch_map_dump(struct mac_ax_txdma_ch_map *txch_map)
+{
+	PHL_DBG("\ttxch_map:\n");
+	PHL_DBG("\tB0: ch0:%d, ch1:%d, ch2:%d, ch3:%d, ch8:%d, ch9:%d\n",
+		txch_map->ch0, txch_map->ch1, txch_map->ch2, txch_map->ch3,
+		txch_map->ch8, txch_map->ch9);
+	PHL_DBG("\tB1: ch4:%d, ch5:%d, ch6:%d, ch7:%d ch10:%d, ch11:%d\n",
+		txch_map->ch4, txch_map->ch5, txch_map->ch6, txch_map->ch7,
+		txch_map->ch10, txch_map->ch11);
+	//PHL_DBG("\t   ch12:%d\n", txch_map->ch12);
+	PHL_DBG("\n");
+}
+
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status
+rtw_hal_mac_dbcc_hci_ctrl(struct hal_info_t *hal_info, enum phl_band_idx band_idx, u8 pause)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_dbcc_hci_ctrl info = {0};
+	u32 ret = 0;
+
+	if(pause && (band_idx == HW_BAND_1)) {
+		trx_ops->query_txch_map(band_idx, &info.u.pcie_ctrl.clr_txch_map);
+	}
+	else {
+		hal_txch_map_ignore(&info.u.pcie_ctrl.clr_txch_map);
+	}
+
+	info.pause = pause;
+	info.band = band_idx; //don't care
+	hal_txch_map_dump(&info.u.pcie_ctrl.clr_txch_map);
+
+	ret = hal_mac_ops->intf_ops->dbcc_hci_ctrl(mac, &info);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+enum rtw_hal_status
+rtw_hal_mac_tx_res_query(struct hal_info_t *hal, u8 ch_idx, u16 *host_idx,
+                         u16 *hw_idx, u16 *avail_txbd)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+	u32 ret = 0;
+
+	ret = mac_intf_ops->get_avail_txbd(mac, ch_idx, host_idx, hw_idx,
+	                                   avail_txbd);
+
+	if (ret != MACSUCCESS && ret != MACNOTSUP)
+		PHL_ERR("%s: query tx res of ch %d fail!\n", __func__, ch_idx);
+
+	return (ret == MACSUCCESS) ?
+		(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_rx_res_query(struct hal_info_t *hal, u8 ch_idx, u16 *host_idx,
+                         u16 *hw_idx, u16 *avail_rxbd)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+	u32 ret = 0;
+
+	ret = mac_intf_ops->get_avail_rxbd(mac, ch_idx, host_idx, hw_idx,
+	                                   avail_rxbd);
+	return (ret == MACSUCCESS) ?
+		(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_trigger_txstart(struct hal_info_t *hal,
+                            struct tx_base_desc *txbd_ring, u8 ch_idx)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+	u32 ret = 0;
+
+	ret = mac_intf_ops->trigger_txdma(mac, txbd_ring, ch_idx);
+	return (ret == MACSUCCESS) ?
+		(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_notify_rxdone(struct hal_info_t *hal, struct rx_base_desc *rxbd,
+                          u8 ch_idx)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+	u32 ret = 0;
+
+	ret = mac_intf_ops->notify_rxdone(mac, rxbd, ch_idx);
+	return (ret == MACSUCCESS) ?
+		(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_set_hci_speed(struct hal_info_t *hal_info,
+					u8 speed)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+	u32 ret = 0;
+
+	ret = mac_intf_ops->set_pcie_speed(mac, speed);
+
+	return (ret == 0) ? (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_hci_speed(struct hal_info_t *hal_info, u8 *speed)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+	u32 ret = 0;
+
+	ret = mac_intf_ops->get_pcie_speed(mac, speed);
+
+	return (ret == 0) ? (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_max_hci_speed(struct hal_info_t *hal_info, u8 *speed)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	*speed = (u8)mac_intf_ops->get_pcie_sup_speed(mac);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_aspm_frontdoor_set(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u8 dummyval = 0;
+	u32 ret = 0;
+
+	ret = mac->ops->set_hw_value(mac, MAC_AX_HW_PCIE_ASPM_FRONTDOOR_SET,
+				     &dummyval);
+	return (ret == 0) ? (RTW_HAL_STATUS_SUCCESS) : (RTW_HAL_STATUS_FAILURE);
+}
+
+#endif
+
+#ifdef CONFIG_USB_HCI
+u8 hal_mac_get_bulkout_id(struct hal_info_t *hal, u8 dma_ch, u8 mode)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->ops->intf_ops->get_bulkout_id(mac, dma_ch, mode);
+}
+
+u32 hal_mac_usb_tx_agg_cfg(struct hal_info_t *hal, u8* wd_buf, u8 agg_num)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_usb_tx_agg_cfg agg = {0};
+
+	agg.pkt = wd_buf;
+	agg.agg_num = agg_num;
+
+	return mac->ops->intf_ops->usb_tx_agg_cfg(mac, &agg);
+}
+
+u32 hal_mac_usb_rx_agg_cfg(struct hal_info_t *hal, u8 agg_mode,
+	u8 drv_define, u8 timeout, u8 size, u8 pkt_num)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_rx_agg_cfg cfg = {0};
+
+	cfg.mode = agg_mode;
+	cfg.thold.drv_define = drv_define;
+	cfg.thold.timeout = timeout;
+	cfg.thold.size = size;
+	cfg.thold.pkt_num = pkt_num;
+
+	return mac->ops->intf_ops->usb_rx_agg_cfg(mac, &cfg);
+}
+
+enum rtw_hal_status hal_mac_force_usb_switch(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	if(mac->ops->intf_ops->u2u3_switch(mac) == MACSUCCESS) {
+		PHL_INFO("%s,success!\n", __func__);
+		return RTW_HAL_STATUS_SUCCESS;
+	}
+	else {
+		PHL_INFO("%s,fail!\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+}
+
+u32 hal_mac_get_cur_usb_mode(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	return mac->ops->intf_ops->get_usb_mode(mac);
+}
+u32 hal_mac_get_usb_support_ability(struct hal_info_t *hal)
+
+{
+
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->ops->intf_ops->get_usb_support_ability(mac);
+}
+u32 hal_mac_set_usb_support_ability(struct hal_info_t *hal, u32 ability)
+
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->ops->intf_ops->set_usb_support_ability(mac, ability);
+}
+u32 hal_mac_get_usb_mode_status(struct hal_info_t *hal, u32 *status)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->ops->intf_ops->get_usb_mode_status(mac, (enum usb_mode_status *)status);
+}
+u32 hal_mac_get_u3_perf_mode(struct hal_info_t *hal, u32 *perf_mode)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->ops->intf_ops->get_u3_perf_mode(mac, (enum mac_u3_perf_mode *)perf_mode);
+}
+u8 hal_mac_usb_get_max_bulkout_wd_num(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->usb_info.max_bulkout_wd_num;
+}
+
+u16 hal_mac_usb_get_max_dma_txagg_msk(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	return mac->usb_info.max_dma_txagg_msk;
+}
+
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status
+rtw_hal_mac_dbcc_hci_ctrl(struct hal_info_t *hal_info, enum phl_band_idx band_idx, u8 pause)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_dbcc_hci_ctrl info = {0};
+	u32 ret = 0;
+
+	info.band = band_idx;
+	info.pause = pause; /*1: flush mode enabled, 0: flush mode disabled */
+	//info.u.usb_ctrl.rsvd;
+
+	ret = hal_mac_ops->intf_ops->dbcc_hci_ctrl(mac, &info);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+enum rtw_hal_status rtw_hal_mac_set_hci_speed(struct hal_info_t *hal_info,
+					u8 speed)
+{
+	u32 ret = 0;
+
+	if(speed <= RTW_USB_SPEED_HIGH)
+		ret = rtw_hal_force_usb_switch(hal_info, USB_2_0);
+	else if(speed < RTW_USB_SPEED_MAX)
+		ret = rtw_hal_force_usb_switch(hal_info, USB_3_0);
+	PHL_INFO("%s (%d) !!\n", __FUNCTION__, speed);
+
+	return (ret == 0) ? (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_hci_speed(struct hal_info_t *hal_info, u8 *speed)
+{
+	u32 mode = 0;
+
+	mode = hal_mac_get_cur_usb_mode(hal_info);
+
+	switch(mode) {
+		case USB_1_1:
+			*speed = RTW_USB_SPEED_FULL;
+			break;
+		case USB_2_0:
+			*speed = RTW_USB_SPEED_HIGH;
+			break;
+		case USB_3_0:
+			*speed = RTW_USB_SPEED_SUPER;
+			break;
+		default:
+			*speed = RTW_USB_SPEED_UNKNOWN;
+			break;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_max_hci_speed(struct hal_info_t *hal_info, u8 *speed)
+{
+
+	PHL_INFO("%s hci not supported!!\n", __FUNCTION__);
+
+	return RTW_HAL_STATUS_FAILURE;
+
+}
+#endif
+
+#ifdef CONFIG_PHL_CSUM_OFFLOAD_RX
+static void hal_mac_parse_rxd_checksume(struct rtw_hal_com_t *hal,
+				 struct rtw_r_meta_data *meta, u8 status)
+{
+	if ((status == MAC_AX_CHKSUM_OFD_IPV4_TCP_OK) ||
+	    (status == MAC_AX_CHKSUM_OFD_IPV6_TCP_OK) ||
+	    (status == MAC_AX_CHKSUM_OFD_IPV4_UDP_OK) ||
+	    (status == MAC_AX_CHKSUM_OFD_IPV6_UDP_OK))
+		meta->chksum_status = 0;
+	else
+		meta->chksum_status = 1;
+}
+
+/**
+ * rtw_hal_mac_chk_rx_tcpip_chksum_ofd() - check whether rx checsksum offload succeed or not.
+ * @hal: see hal_info_t
+ * @mdata: according to the result of RX checksum offload, we will set mdata->chksum_status to
+ * 	   corresponding value.
+ * 	   mdata->chksum_status = 0: RX checksum offload succeed.
+ * 	   mdata->chksum_status = 1: RX checksum offload failed.
+ * @status: an one byte variable that appends on the end of packet.
+ * 	    indicating the result of RX checksum offload, in function mac_chk_rx_tcpip_chksum_ofd,
+ * 	    we will determine whether the offload succeeds via status.
+ *
+ * The function calls hal mac api to determine whether RX checksum offload succeed or not.
+ * If successfully offload, we will notify OS not to calculate checksum.
+ * In case of linux, setting ip_summed = CHECKSUM_UNNECESSARY in function rtw_os_recv_indicate_pkt
+ * indicates OS not to calculate checksum.
+ *
+ * Return:
+ * 0: RX checksum offload succeed.
+ * 1: RX checksum offload failed.
+ */
+enum rtw_hal_status rtw_hal_mac_chk_rx_tcpip_chksum_ofd(struct hal_info_t *hal,
+							struct rtw_r_meta_data *mdata,
+							u8 status)
+{
+	u32 result;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct rtw_hal_com_t *hal_com = hal->hal_com;
+
+	result =  mac->ops->chk_rx_tcpip_chksum_ofd(mac, status);
+	hal_mac_parse_rxd_checksume(hal_com, mdata, result);
+
+	return (!mdata->chksum_status) ? (RTW_HAL_STATUS_SUCCESS) : (RTW_HAL_STATUS_FAILURE);
+}
+#endif /* CONFIG_PHL_CSUM_OFFLOAD_RX */
+
+#ifdef CONFIG_SDIO_HCI
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status
+rtw_hal_mac_dbcc_hci_ctrl(struct hal_info_t *hal_info, enum phl_band_idx band_idx, u8 pause)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_dbcc_hci_ctrl info = {0};
+	u32 ret = 0;
+
+	info.band = band_idx;
+	info.pause = pause;
+	//info.u.sdio_ctrl.rsvd;
+
+	ret = hal_mac_ops->intf_ops->dbcc_hci_ctrl(mac, &info);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+u8 hal_mac_sdio_read8(struct rtw_hal_com_t *hal, u32 addr)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	return mac_intf_ops->reg_read8(mac, addr);
+}
+
+u16 hal_mac_sdio_read16(struct rtw_hal_com_t *hal, u32 addr)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	return mac_intf_ops->reg_read16(mac, addr);
+}
+
+u32 hal_mac_sdio_read32(struct rtw_hal_com_t *hal, u32 addr)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	return mac_intf_ops->reg_read32(mac, addr);
+}
+
+int hal_mac_sdio_write8(struct rtw_hal_com_t *hal, u32 addr, u8 value)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	mac_intf_ops->reg_write8(mac, addr, value);
+	return 0;
+}
+
+int hal_mac_sdio_write16(struct rtw_hal_com_t *hal, u32 addr, u16 value)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	mac_intf_ops->reg_write16(mac, addr, value);
+	return 0;
+}
+
+int hal_mac_sdio_write32(struct rtw_hal_com_t *hal, u32 addr, u32 value)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	mac_intf_ops->reg_write32(mac, addr, value);
+	return 0;
+}
+
+static void _read_register(struct rtw_hal_com_t *hal, u32 addr, u32 cnt, u8 *buf)
+{
+	u32 i, n;
+	u16 val16;
+	u32 val32;
+
+	i = addr & 0x3;
+	/* Handle address not start from 4 bytes alignment case */
+	if (i) {
+		val32 = cpu_to_le32(hal_read32(hal, addr & ~0x3));
+		n = 4 - i;
+		_os_mem_cpy(hal->drv_priv, buf, ((u8 *)&val32) + i, n);
+		i = n;
+		cnt -= n;
+	}
+
+	while (cnt) {
+		if (cnt >= 4)
+			n = 4;
+		else if (cnt >= 2)
+			n = 2;
+		else
+			n = 1;
+		cnt -= n;
+
+		switch (n) {
+		case 1:
+			buf[i] = hal_read8(hal, addr+i);
+			i++;
+			break;
+		case 2:
+			val16 = cpu_to_le16(hal_read16(hal, addr+i));
+			_os_mem_cpy(hal->drv_priv, &buf[i], &val16, 2);
+			i += 2;
+			break;
+		case 4:
+			val32 = cpu_to_le32(hal_read32(hal, addr+i));
+			_os_mem_cpy(hal->drv_priv, &buf[i], &val32, 4);
+			i += 4;
+			break;
+		}
+	}
+}
+
+static int _sdio_read_local(struct rtw_hal_com_t *hal, u32 addr, u32 cnt, u8 *buf)
+{
+	/*struct hal_info_t *hal_info = hal->hal_priv;*/
+	/*struct mac_ax_adapter *mac = hal_to_mac(hal_info);*/
+	/*struct mac_ax_ops *mac_api = mac->ops;*/
+	/*struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;*/
+
+	if (buf == NULL)
+		return -1;
+	/*GEORGIA_TODO_FIXIT_ASKFORMAC*/
+	/*mac_intf_ops->reg_readrn(mac, addr, cnt, buf);*/
+	return 0;
+}
+
+void hal_mac_sdio_read_mem(struct rtw_hal_com_t *hal, u32 addr, u32 cnt, u8 *pmem)
+{
+
+	if (pmem == NULL) {
+		PHL_ERR("pmem is NULL\n");
+		return;
+	}
+
+	if (addr & 0xFFFF0000) {
+		int err = 0;
+
+		err = _sdio_read_local(hal, addr, cnt, pmem);
+		if (!err)
+			return;
+	}
+	_read_register(hal, addr, cnt, pmem);
+}
+
+enum rtw_hal_status rtw_hal_mac_set_hci_speed(struct hal_info_t *hal_info,
+					u8 speed)
+{
+
+	return 0;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_hci_speed(struct hal_info_t *hal_info, u8 *speed)
+{
+
+	return 0;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_max_hci_speed(struct hal_info_t *hal_info, u8 *speed)
+{
+
+	PHL_INFO("%s hci not supported!!\n", __FUNCTION__);
+
+	return RTW_HAL_STATUS_FAILURE;
+
+}
+
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+u8 hal_mac_sdio_iread8(struct rtw_hal_com_t *hal, u32 addr)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	return mac_intf_ops->reg_read8(mac, addr);
+}
+
+u16 hal_mac_sdio_iread16(struct rtw_hal_com_t *hal, u32 addr)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	return mac_intf_ops->reg_read16(mac, addr);
+}
+
+u32 hal_mac_sdio_iread32(struct rtw_hal_com_t *hal, u32 addr)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	return 0;
+}
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#endif /* CONFIG_SDIO_HCI */
+
+
+#ifndef CONFIG_NEW_HALMAC_INTERFACE
+#ifdef CONFIG_SDIO_HCI
+static u8 hal_mac_sdio_cmd52_r8(void *h, u32 addr)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_sdio_cmd52_r8(hal->drv_priv, addr);
+}
+static u8 hal_mac_sdio_cmd53_r8(void *h, u32 addr)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_sdio_cmd53_r8(hal->drv_priv, addr);
+}
+static u16 hal_mac_sdio_cmd53_r16(void *h, u32 addr)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_sdio_cmd53_r16(hal->drv_priv, addr);
+}
+static u32 hal_mac_sdio_cmd53_r32(void *h, u32 addr)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_sdio_cmd53_r32(hal->drv_priv, addr);
+}
+static u8 hal_mac_sdio_cmd53_rn(void *h, u32 addr, u32 size, u8 *val)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_sdio_cmd53_rn(hal->drv_priv, addr, size,val);
+}
+static void hal_mac_sdio_cmd52_w8(void *h, u32 addr, u8 val)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_sdio_cmd52_w8(hal->drv_priv, addr, val);
+}
+static void hal_mac_sdio_cmd53_w8(void *h, u32 addr, u8 val)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_sdio_cmd53_w8(hal->drv_priv, addr, val);
+}
+static void hal_mac_sdio_cmd53_w16(void *h, u32 addr, u16 val)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_sdio_cmd53_w16(hal->drv_priv, addr, val);
+}
+static void hal_mac_sdio_cmd53_w32(void *h, u32 addr, u32 val)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_sdio_cmd53_w32(hal->drv_priv, addr, val);
+}
+static u8 hal_mac_sdio_cmd53_wn(void *h, u32 addr, u32 size, u8 *val)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_sdio_cmd53_wn(hal->drv_priv, addr, size, val);
+	return true;
+}
+static u8 hal_mac_sdio_cmd52_cia_r8(void *h, u32 addr)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_sdio_read_cia_r8(hal->drv_priv, addr);
+}
+#endif /*CONFIG_SDIO_HCI*/
+
+static u8 hal_mac_reg_r8(void *h, u32 addr)
+{
+	return hal_read8((struct rtw_hal_com_t *)h, addr);
+}
+static u16 hal_mac_reg_r16(void *h, u32 addr)
+{
+	return hal_read16((struct rtw_hal_com_t *)h, addr);
+}
+static u32 hal_mac_reg_r32(void *h, u32 addr)
+{
+	return hal_read32((struct rtw_hal_com_t *)h, addr);
+}
+static void hal_mac_reg_w8(void *h, u32 addr, u8 val)
+{
+	hal_write8((struct rtw_hal_com_t *)h, addr, val);
+}
+static void hal_mac_reg_w16(void *h, u32 addr, u16 val)
+{
+	hal_write16((struct rtw_hal_com_t *)h, addr, val);
+}
+static void hal_mac_reg_w32(void *h, u32 addr, u32 val)
+{
+	hal_write32((struct rtw_hal_com_t *)h, addr, val);
+}
+
+#ifdef CONFIG_PCI_HCI
+static enum rtw_hal_status hal_mac_get8_pci_cfg(void *h, u32 offset, void *buf)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return (_os_get_pci_cfg(hal->drv_priv, offset, buf, 1) ?
+	        RTW_HAL_STATUS_SUCCESS : RTW_HAL_STATUS_FAILURE);
+}
+static enum rtw_hal_status hal_mac_get16_pci_cfg(void *h, u32 offset, void *buf)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return (_os_get_pci_cfg(hal->drv_priv, offset, buf, 2) ?
+	        RTW_HAL_STATUS_SUCCESS : RTW_HAL_STATUS_FAILURE);
+}
+static enum rtw_hal_status hal_mac_get32_pci_cfg(void *h, u32 offset, void *buf)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return (_os_get_pci_cfg(hal->drv_priv, offset, buf, 4) ?
+	        RTW_HAL_STATUS_SUCCESS : RTW_HAL_STATUS_FAILURE);
+}
+static enum rtw_hal_status hal_mac_set8_pci_cfg(void *h, u32 offset, void *buf)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return (_os_set_pci_cfg(hal->drv_priv, offset, buf, 1) ?
+	        RTW_HAL_STATUS_SUCCESS : RTW_HAL_STATUS_FAILURE);
+}
+static enum rtw_hal_status hal_mac_set16_pci_cfg(void *h, u32 offset, void *buf)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return (_os_set_pci_cfg(hal->drv_priv, offset, buf, 2) ?
+	        RTW_HAL_STATUS_SUCCESS : RTW_HAL_STATUS_FAILURE);
+}
+static enum rtw_hal_status hal_mac_set32_pci_cfg(void *h, u32 offset, void *buf)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return (_os_set_pci_cfg(hal->drv_priv, offset, buf, 4) ?
+	        RTW_HAL_STATUS_SUCCESS : RTW_HAL_STATUS_FAILURE);
+}
+#endif /* #ifdef CONFIG_PCI_HCI */
+
+#ifdef DBG_HAL_MAC_MEM_MOINTOR
+static void hal_mac_mem_free(void *h, void *buf, u32 size)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	/*PHL_INFO("MAC free mem - :%d\n", size);*/
+	_os_atomic_sub(hal->drv_priv, &(hal->hal_mac_mem), size);
+	_os_mem_free(hal->drv_priv, buf, size);
+}
+static void* hal_mac_mem_alloc(void *h, u32 size)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	/*PHL_INFO("MAC Alloc mem - :%d\n", size);*/
+	_os_atomic_add_return(hal->drv_priv, &(hal->hal_mac_mem), size);
+
+	return _os_mem_alloc(hal->drv_priv, size);
+}
+#else
+static void hal_mac_mem_free(void *h, void *buf, u32 size)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_mem_free(hal->drv_priv, buf, size);
+}
+static void* hal_mac_mem_alloc(void *h, u32 size)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_mem_alloc(hal->drv_priv, size);
+}
+#endif /*DBG_HAL_MAC_MEM_MOINTOR*/
+static void hal_mac_mem_cpy(void *h, void *dest, void *src, u32 size)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_mem_cpy(hal->drv_priv, dest, src, size);
+}
+static void hal_mac_mem_set(void *h, void *addr, u8 val, u32 size)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_mem_set(hal->drv_priv, addr, val, size);
+
+}
+static s32 hal_mac_mem_cmp(void *h, void *ptr1, void *ptr2, u32 num)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	return _os_mem_cmp(hal->drv_priv, ptr1, ptr2, num);
+}
+static void hal_mac_udelay(void *h, u32 us)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_delay_us(hal->drv_priv, us);
+}
+static void hal_mac_mdelay(void *h, u32 ms)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_delay_ms(hal->drv_priv, ms);
+}
+static void hal_mac_usleep(void *h, u32 us)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_sleep_us(hal->drv_priv, us);
+}
+static void hal_mac_msleep(void *h, u32 ms)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_sleep_ms(hal->drv_priv, ms);
+}
+
+static u32 hal_mac_get_cur_time_ms(void)
+{
+	return _os_get_cur_time_ms();
+}
+
+static u32 hal_mac_get_cur_time_us(void)
+{
+	return _os_get_cur_time_us();
+}
+
+static u32 hal_mac_get_passing_time_ms(u32 start)
+{
+	return phl_get_passing_time_ms(start);
+}
+
+static u32 hal_mac_get_passing_time_us(u32 start)
+{
+	return phl_get_passing_time_us(start);
+}
+
+static void hal_mac_get_cur_raw_time(_os_raw_time *raw_time)
+{
+	*raw_time = _os_get_cur_raw_time();
+}
+
+static void hal_mac_mutex_init(void *h, mac_ax_mutex *mutex)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_mutex_init(hal->drv_priv, mutex);
+}
+static void hal_mac_mutex_deinit(void *h, mac_ax_mutex *mutex)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_mutex_deinit(hal->drv_priv, mutex);
+}
+static void hal_mac_mutex_lock(void *h, mac_ax_mutex *mutex)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_mutex_lock(hal->drv_priv, mutex);
+}
+static void hal_mac_mutex_unlock(void *h, mac_ax_mutex *mutex)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+	_os_mutex_unlock(hal->drv_priv, mutex);
+}
+
+static void hal_mac_event_notify(void *h,
+			enum mac_ax_feature mac_ft,
+			enum mac_ax_status stat, u8 *buf, u32 size)
+{
+	//struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)h;
+
+}
+
+struct rtw_h2c_pkt *hal_query_h2c_pkt(struct rtw_phl_com_t *phl_com,
+									  struct rtw_hal_com_t *hal_com,
+									  enum h2c_buf_class type)
+{
+	struct rtw_h2c_pkt *h2c_pkt = NULL;
+
+	enum rtw_h2c_pkt_type pkt_type = H2CB_TYPE_MAX;
+
+	switch (type) {
+	case H2CB_CLASS_CMD:
+		pkt_type = H2CB_TYPE_CMD;
+		break;
+	case H2CB_CLASS_DATA:
+		pkt_type = H2CB_TYPE_DATA;
+		break;
+	case H2CB_CLASS_LONG_DATA:
+		pkt_type = H2CB_TYPE_LONG_DATA;
+		break;
+	case H2CB_CLASS_LAST:
+		pkt_type = H2CB_TYPE_MAX;
+		break;
+	default:
+		PHL_ERR("%s: Unknown type(%d)\n", __func__, type);
+		break;
+	}
+
+	h2c_pkt = rtw_phl_query_h2c_pkt(phl_com, pkt_type);
+
+	if (!h2c_pkt)
+		PHL_ERR("hal_query_h2c_pkt fail (type %d).\n", type);
+
+	return h2c_pkt;
+}
+
+enum rtw_hal_status hal_pltfm_tx(struct rtw_phl_com_t *phl_com,
+								 struct rtw_hal_com_t *hal_com,
+								 struct rtw_h2c_pkt *pkt)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	if (rtw_phl_pltfm_tx(phl_com, pkt) == RTW_PHL_STATUS_SUCCESS)
+		hstatus = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("hal_pltfm_tx fail.\n");
+
+	return hstatus;
+}
+
+void hal_ser_l2_notify(void *phl, void *hal)
+{
+	struct rtw_phl_com_t *phl_com = (struct rtw_phl_com_t *)phl;
+
+	rtw_phl_ser_l2_notify(phl_com);
+}
+
+static s32 hal_mac_vsnprintf(void *drv_adapter, char *buf, size_t size, const char *fmt, _os_va_list args)
+{
+        s32 ret;
+
+        ret = _os_vsnprintf(buf, size, fmt, args);
+        if (ret >= 0)
+                ret = _os_strlen((u8 *)buf);
+
+        /* PHL_PRINT("[HAL_MMAC]%s", buf); */
+        return ret;
+}
+
+static s32 hal_mac_sprintf(void *drv_adapter, char *buf, size_t size, const char *fmt, ...)
+{
+	s32 ret;
+	_os_va_list args = {0};
+
+	_os_va_start(args, fmt);
+	ret = _os_vsnprintf(buf, size, fmt, args);
+	if (ret >= 0)
+		ret = _os_strlen((u8 *)buf);
+
+	/* PHL_PRINT("[HAL_MMAC]%s", buf); */
+	_os_va_end(args);
+	return ret;
+}
+
+static s32 hal_mac_strcmp(void *drv_adapter, const char *s1, const char *s2)
+{
+	return _os_strcmp(s1, s2);
+}
+
+static char* hal_mac_strsep(void *drv_adapter, char **s, const char *ct)
+{
+	return _os_strsep(s, ct);
+}
+
+static u32 hal_mac_strlen(void *drv_adapter, char *buf)
+{
+	return _os_strlen((u8 *)buf);
+}
+
+static char* hal_mac_strcpy(void *drv_adapter, char *dest, const char *src)
+{
+	return _os_strcpy(dest, src);
+}
+
+static char* hal_mac_strpbrk(void *drv_adapter, const char *cs, const char *ct)
+{
+	return _os_strpbrk(cs, ct);
+}
+
+static u32 hal_mac_strtoul(void *drv_adapter, const char *buf, u32 base)
+{
+#if 1
+	u32 ret = 0, val = 0;
+
+	if (base == 10)
+		ret = _os_sscanf(buf, "%d", &val);
+	else if (base == 16)
+		ret = _os_sscanf(buf, "%x", &val);
+	else
+		ret = _os_sscanf(buf, "%d", &val);
+
+	return val;
+#else
+	//stdlib strtoul
+#endif
+
+}
+
+static u8
+hal_mac_ld_fw_symbol(void *phl_com, void *hal_com, const char *name,
+	u8 **buf, u32 *buf_size)
+{
+	return rtw_hal_ld_fw_symbol((struct rtw_phl_com_t *)phl_com,
+		(struct rtw_hal_com_t *)hal_com, name, buf, buf_size);
+}
+
+static u8
+hal_mac_get_chip_id(void *hal_com)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)hal_com;
+
+	return hal->chip_id;
+}
+
+#define RTW_HALMAC_FAIL		0
+#define RTW_HALMAC_SUCCESS		1
+#define MSG_PREFIX			"[MAC]"
+
+void hal_mac_msg_print(void *p, u8 dbg_level, s8 *fmt, ...)
+{
+#if defined(PHL_PLATFORM_LINUX) || defined(PHL_PLATFORM_WINDOWS) || \
+	defined(PHL_PLATFORM_UEFI)
+#define MSG_LEN		100
+		va_list args;
+		char str[MSG_LEN] = {0};
+		int err;
+		u8 ret = RTW_HALMAC_SUCCESS;
+
+		str[0] = '\n';
+		va_start(args, fmt);
+		err = _os_vsnprintf(str, MSG_LEN, fmt, args);
+		va_end(args);
+
+		/* An output error is encountered */
+		if (err < 0)
+			return;
+		/* Output may be truncated due to size limit */
+		if ((err == (MSG_LEN - 1)) && (str[MSG_LEN - 2] != '\n'))
+			ret = RTW_HALMAC_FAIL;
+
+		PHL_TRACE(COMP_PHL_MAC, dbg_level, MSG_PREFIX " %s", str);
+#endif
+}
+
+#ifdef CONFIG_PHL_DIAGNOSE
+static void hal_mac_diagnostic_event(void *hal_com, u8 type,
+		u8 level, u8 version, u8 *buf, u32 len)
+{
+	struct rtw_hal_com_t *hal = (struct rtw_hal_com_t *)hal_com;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal->hal_priv;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct rtw_phl_com_t *phl_com = (struct rtw_phl_com_t *)mac->phl_adapter;
+
+	rtw_phl_send_diag_hub_msg(phl_com, PHL_DIAG_EVT_MAC,
+				       type, level, version, buf, len);
+}
+#endif
+
+#ifdef CONFIG_PHL_H2C_PKT_POOL_STATS_CHECK
+static struct phl_h2c_pkt_alloc_cnt *
+hal_mac_dump_h2c_pool_alloc_stats(struct rtw_phl_com_t *phl_com)
+{
+	return rtw_phl_dump_h2c_pool_alloc_stats(phl_com->phl_priv);
+}
+#endif
+
+struct mac_ax_pltfm_cb rtw_plt_cb = {0};
+void rtw_plt_cb_init(void)
+{
+	/* R/W register */
+#ifdef CONFIG_SDIO_HCI
+	rtw_plt_cb.sdio_cmd52_r8 = hal_mac_sdio_cmd52_r8;
+	rtw_plt_cb.sdio_cmd53_r8 = hal_mac_sdio_cmd53_r8;
+	rtw_plt_cb.sdio_cmd53_r16 = hal_mac_sdio_cmd53_r16;
+	rtw_plt_cb.sdio_cmd53_r32 = hal_mac_sdio_cmd53_r32;
+	rtw_plt_cb.sdio_cmd53_rn = hal_mac_sdio_cmd53_rn;
+	rtw_plt_cb.sdio_cmd52_w8 = hal_mac_sdio_cmd52_w8;
+	rtw_plt_cb.sdio_cmd53_w8 = hal_mac_sdio_cmd53_w8;
+	rtw_plt_cb.sdio_cmd53_w16 = hal_mac_sdio_cmd53_w16;
+	rtw_plt_cb.sdio_cmd53_w32 = hal_mac_sdio_cmd53_w32;
+	rtw_plt_cb.sdio_cmd53_wn = hal_mac_sdio_cmd53_wn;
+	rtw_plt_cb.sdio_cmd52_cia_r8 = hal_mac_sdio_cmd52_cia_r8;
+#endif /* CONFIG_SDIO_HCI */
+
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_PCI_HCI)
+	rtw_plt_cb.reg_r8 = hal_mac_reg_r8;
+	rtw_plt_cb.reg_r16 = hal_mac_reg_r16;
+	rtw_plt_cb.reg_r32 = hal_mac_reg_r32;
+	rtw_plt_cb.reg_w8 = hal_mac_reg_w8;
+	rtw_plt_cb.reg_w16 = hal_mac_reg_w16;
+	rtw_plt_cb.reg_w32 = hal_mac_reg_w32;
+#endif /* CONFIG_USB_HCI || CONFIG_PCI_HCI */
+
+#ifdef CONFIG_PCI_HCI
+	rtw_plt_cb.pcie_cfg_r8 = hal_mac_get8_pci_cfg;
+	rtw_plt_cb.pcie_cfg_r16 = hal_mac_get16_pci_cfg;
+	rtw_plt_cb.pcie_cfg_r32 = hal_mac_get32_pci_cfg;
+	rtw_plt_cb.pcie_cfg_w8 = hal_mac_set8_pci_cfg;
+	rtw_plt_cb.pcie_cfg_w16 = hal_mac_set16_pci_cfg;
+	rtw_plt_cb.pcie_cfg_w32 = hal_mac_set32_pci_cfg;
+#endif /* CONFIG_PCI_HCI */
+
+	/* Memory allocate */
+	rtw_plt_cb.rtl_free = hal_mac_mem_free;
+	rtw_plt_cb.rtl_malloc = hal_mac_mem_alloc;
+	rtw_plt_cb.rtl_memcpy = hal_mac_mem_cpy;
+	rtw_plt_cb.rtl_memset = hal_mac_mem_set;
+	rtw_plt_cb.rtl_memcmp = hal_mac_mem_cmp;
+	/* Delay */
+	rtw_plt_cb.rtl_delay_us = hal_mac_udelay;
+	rtw_plt_cb.rtl_delay_ms = hal_mac_mdelay;
+	/* Sleep */
+	rtw_plt_cb.rtl_sleep_us = hal_mac_usleep;
+	rtw_plt_cb.rtl_sleep_ms = hal_mac_msleep;
+
+	/* Process Synchronization */
+	rtw_plt_cb.rtl_mutex_init = hal_mac_mutex_init;
+	rtw_plt_cb.rtl_mutex_deinit = hal_mac_mutex_deinit;
+	rtw_plt_cb.rtl_mutex_lock = hal_mac_mutex_lock;
+	rtw_plt_cb.rtl_mutex_unlock = hal_mac_mutex_unlock;
+
+	rtw_plt_cb.msg_print = hal_mac_msg_print;
+	rtw_plt_cb.event_notify = hal_mac_event_notify;
+	rtw_plt_cb.ser_l2_notify = hal_ser_l2_notify;
+	rtw_plt_cb.ld_fw_symbol = hal_mac_ld_fw_symbol;
+	rtw_plt_cb.get_chip_id = hal_mac_get_chip_id;
+
+	/*.tx = ;	*/
+#if MAC_AX_PHL_H2C
+	rtw_plt_cb.tx = hal_pltfm_tx;
+	rtw_plt_cb.rtl_query_h2c = hal_query_h2c_pkt;
+#ifdef CONFIG_PHL_H2C_PKT_POOL_STATS_CHECK
+	rtw_plt_cb.dump_h2c_pool_alloc_stats = hal_mac_dump_h2c_pool_alloc_stats;
+#endif
+#endif
+
+	rtw_plt_cb.rtl_vsprintf = hal_mac_vsnprintf;
+	rtw_plt_cb.rtl_sprintf = hal_mac_sprintf;
+	rtw_plt_cb.rtl_strcmp = hal_mac_strcmp;
+	rtw_plt_cb.rtl_strsep = hal_mac_strsep;
+	rtw_plt_cb.rtl_strlen = hal_mac_strlen;
+	rtw_plt_cb.rtl_strcpy = hal_mac_strcpy;
+	rtw_plt_cb.rtl_strpbrk = hal_mac_strpbrk;
+	rtw_plt_cb.rtl_strtoul = hal_mac_strtoul;
+
+	rtw_plt_cb.rtl_get_current_times_ms = hal_mac_get_cur_time_ms;
+	rtw_plt_cb.rtl_get_current_times_us = hal_mac_get_cur_time_us;
+	rtw_plt_cb.rtl_get_passing_times_ms = hal_mac_get_passing_time_ms;
+	rtw_plt_cb.rtl_get_passing_times_us = hal_mac_get_passing_time_us;
+
+#ifdef CONFIG_PHL_DIAGNOSE
+	rtw_plt_cb.rtl_diagnostic_event = hal_mac_diagnostic_event;
+#endif
+}
+
+
+#endif /*CONFIG_NEW_HALMAC_INTERFACE*/
+
+/* halmac wrapper API for hal and proto type is at hal_api_mac.h */
+#define MAC_STATUS_MAX	MACSDIOSEQERR+1 /* Wrong interface */
+const char *const ma_status[] = {
+	"MAC Success",
+	"Callback of platform is null",
+	"Endian of platform error",
+	"Invalid base address",
+	"Leave suspend error",
+	"Pointer is null",
+	"Chip ID is undefined",
+	"Can not get MAC adapter",
+	"Unexpected structure alignment",
+	"Buffer space is not enough",
+	"Buffer size error",
+	"Invalid item",
+	"Polling timeout",
+	"Power switch fail",
+	"Work queue is busy",
+	"Failed compare result",
+	"Wrong interface",
+	"Incorrect FW bin file",
+	"Wrong FIFO configuration",
+	"Same MACID",
+	"MACID full",
+	"There is no FW",
+	"Process is busy",
+	"state machine error",
+	"switch efuse bank fail",
+	"read efuse fail",
+	"write efuse fail",
+	"efuse size error",
+	"eeprom parsing fail",
+	"compare efuse fail",
+	"secure on, no host indirect access",
+	"invalid tx dma channel",
+	"address cam update error",
+	"Power state error",
+	"SDIO Tx mix mode",
+	"SDIO Tx sequence error",
+};
+
+#define mac_sstr(status) (((status) >= MAC_STATUS_MAX) ? "unknown" : ma_status[(status)])
+
+/**
+ * rtw_hal_mac_get_version() - Get HALMAC version
+ * @ver_str:	string buffer for storing version string
+ * @len:	string buffer length
+ *
+ * HALMAC version format is "V[major].[prototype].[sub ver].[sub index]",
+ * ex: V0.16.1.0
+ *
+ */
+void rtw_hal_mac_get_version(char *ver_str, u16 len)
+{
+	_os_snprintf(ver_str, len, "V%u.%u.%u.%u",
+		     MAC_AX_MAJOR_VER, MAC_AX_PROTOTYPE_VER,
+		     MAC_AX_SUB_VER, MAC_AX_SUB_INDEX);
+}
+
+/**
+ * rtw_hal_mac_get_fw_ver() - Get Firmware version
+ * @hal_info:	struct hal_info_t *
+ * @ver_str:	string buffer for storing version string
+ * @len:	string buffer length
+ *
+ * Firmware version format is "V[major].[ver].[sub ver].[sub index]",
+ * ex: V0.5.0
+ *
+ */
+void rtw_hal_mac_get_fw_ver(struct hal_info_t *hal_info, char *ver_str, u16 len)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	_os_snprintf(ver_str, len, "V%u.%u.%u.%u",
+		     mac->fw_info.major_ver,mac->fw_info.minor_ver,
+		     mac->fw_info.sub_ver, mac->fw_info.sub_idx);
+}
+
+#define _IS_FW_READY(hal_info) \
+		(hal_to_mac(hal_info)->sm.fwdl == MAC_AX_FWDL_INIT_RDY)
+
+void hal_mac_print_fw_version(struct hal_info_t *hal_info)
+{
+	char ver[20] = {0};
+	rtw_hal_mac_get_fw_ver(hal_info, ver, 20);
+	PHL_PRINT("%s: FW version %s, %sReady\n", __func__, ver,
+		  _IS_FW_READY(hal_info) ? "" : "Not ");
+}
+
+u32 rtw_hal_mac_init(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_adapter *mac = NULL;
+	struct mac_ax_ops *mac_ops;
+	u32 status;
+
+#ifdef DBG_HAL_MAC_MEM_MOINTOR
+	_os_atomic_set(hal_com->drv_priv, &(hal_com->hal_mac_mem), 0);
+#endif
+	hal_info->mac = NULL;
+#ifdef CONFIG_NEW_HALMAC_INTERFACE
+	status = new_mac_ax_ops_init(phl_com, hal_com, &mac, &mac_ops);
+#else
+	{
+		enum mac_ax_intf intf = MAC_AX_INTF_INVALID;
+		if (phl_com->hci_type == RTW_HCI_PCIE)
+			intf = MAC_AX_INTF_PCIE;
+		else if (phl_com->hci_type ==  RTW_HCI_USB)
+			intf = MAC_AX_INTF_USB;
+		else if ((phl_com->hci_type ==  RTW_HCI_SDIO) ||
+			(phl_com->hci_type ==  RTW_HCI_GSPI))
+			intf = MAC_AX_INTF_SDIO;
+
+		rtw_plt_cb_init();
+		status = mac_ax_ops_init(hal_com,
+				&rtw_plt_cb, intf, &mac, &mac_ops);
+		#if MAC_AX_PHL_H2C
+		if (status == MACSUCCESS && mac != NULL)
+			mac_ax_phl_init(phl_com, mac);
+		#endif
+	}
+#endif
+
+	if (status != MACSUCCESS) {
+		PHL_ERR("%s: halmac_init_adapter fail!(status=%d-%s)\n",
+				__func__, status, mac_sstr(status));
+		hal_status = RTW_HAL_STATUS_MAC_INIT_FAILURE;
+		goto error_mac_init;
+	}
+
+	if (!mac) {
+		PHL_ERR("halmac alloc failed\n");
+		hal_status = RTW_HAL_STATUS_MAC_INIT_FAILURE;
+		/*_os_warn_on(1);*/
+		goto error_mac_init;
+	}
+
+	if (hal_com->chip_id == CHIP_WIFI6_MAX) {
+		hal_status = hal_mac_set_chip_id(hal_com, mac);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("halmac set chip id failed\n");
+			goto error_mac_init;
+		}
+	}
+
+	hal_com->cv = mac->hw_info->cv;
+	hal_info->mac = mac;
+
+	return hal_status;
+
+error_mac_init:
+	if (mac) {
+		mac_ax_ops_exit(mac);
+		hal_info->mac = NULL;
+	}
+	return hal_status;
+}
+
+u32 rtw_hal_mac_deinit(struct rtw_phl_com_t *phl_com,
+						struct hal_info_t *hal_info)
+{
+	#ifdef DBG_HAL_MAC_MEM_MOINTOR
+	struct rtw_hal_com_t *hal = hal_info->hal_com;
+	#endif
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	if (mac) {
+		if(mac_ax_ops_exit(mac) == MACSUCCESS) {
+			hal_status = RTW_HAL_STATUS_FAILURE;
+
+			hal_info->mac = NULL;
+		} else {
+			PHL_ERR("%s failed\n", __func__);
+		}
+	}
+	#ifdef DBG_HAL_MAC_MEM_MOINTOR
+	PHL_INFO("[PHL-MEM] %s HALMAC memory :%d\n", __func__,
+		_os_atomic_read(hal_to_drvpriv(hal_info), &(hal->hal_mac_mem)));
+	#endif
+
+	return hal_status;
+}
+
+
+#ifdef CONFIG_SDIO_HCI
+void rtw_hal_mac_sdio_cfg(struct rtw_phl_com_t *phl_com,
+			  struct hal_info_t *hal_info,
+			  struct rtw_ic_info *ic_info)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_sdio_info info = {0};
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &info, 0, sizeof(info));
+
+	info.sdio_4byte = (ic_info->sdio_info.io_align_sz == 4)
+		? MAC_AX_SDIO_4BYTE_MODE_RW : MAC_AX_SDIO_4BYTE_MODE_DISABLE;
+
+	if (ic_info->sdio_info.sd3_bus_mode)
+		info.spec_ver = MAC_AX_SDIO_SPEC_VER_3_00;
+	else
+		info.spec_ver = MAC_AX_SDIO_SPEC_VER_2_00;
+
+	info.block_size = ic_info->sdio_info.block_sz;
+
+	if (ic_info->sdio_info.tx_512_by_byte_mode)
+		info.opn_mode = MAC_AX_SDIO_OPN_MODE_BYTE;
+	else
+		info.opn_mode = MAC_AX_SDIO_OPN_MODE_BLOCK;
+
+	/*
+	 * MAC_AX_HW_SDIO_INFO always return MACSUCCESS,
+	 * so don't check return value here.
+	 */
+	mac->ops->set_hw_value(mac, MAC_AX_HW_SDIO_INFO, &info);
+}
+
+/**
+ * rtw_hal_mac_sdio_tx_cfg - SDIO TX related setting
+ * @hal:	pointer of struct rtw_hal_com_t
+ *
+ * Configure setting for SDIO TX.
+ * All tx related setting which need to be config after mac init(power on)
+ * should be included here.
+ *
+ * Config tx mode to DUMMY_AUTO to release (32K-4) bytes TX size limitation,
+ * but this mode only work on SDIO host supporting block mode.
+ *
+ * No return value for this function.
+ */
+void rtw_hal_mac_sdio_tx_cfg(struct rtw_hal_com_t *hal)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum mac_ax_sdio_tx_mode mode = MAC_AX_SDIO_TX_MODE_DUMMY_AUTO;
+	u32 err;
+
+
+	err = mac->ops->set_hw_value(mac, MAC_AX_HW_SDIO_TX_MODE, &mode);
+	if (err != MACSUCCESS)
+		PHL_ERR("%s: set TX mode(%u) FAIL!(%u)\n", __func__, mode, err);
+}
+
+/**
+ * rtw_hal_mac_sdio_rx_agg_cfg - SDIO RX aggregation setting
+ * @hal:	pointer of struct rtw_hal_com_t
+ * @enable:	enable function or not
+ * @drv_define:	use driver's parameters or not
+ * @timeout:	timeout threshold, unit 1us
+ * @size:	size threshold, unit 1KB
+ * @pkt_num:	packet number threshold
+ *
+ * Configure setting for SDIO RX aggregation.
+ *
+ * No return value for this function.
+ */
+void rtw_hal_mac_sdio_rx_agg_cfg(struct rtw_hal_com_t *hal, bool enable,
+				 u8 drv_define, u8 timeout, u8 size, u8 pkt_num)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_rx_agg_cfg cfg = {MAC_AX_RX_AGG_MODE_NONE};
+
+
+	if (enable) {
+		cfg.mode = MAC_AX_RX_AGG_MODE_DMA;
+		cfg.thold.drv_define = drv_define;
+		cfg.thold.timeout = timeout;
+		cfg.thold.size = size;
+		cfg.thold.pkt_num = pkt_num;
+	}
+
+	mac->ops->set_hw_value(mac, MAC_AX_HW_SDIO_RX_AGG, &cfg);
+}
+
+/* AX TX DESC */
+/* DWORD 0 ; Offset 00h */
+#define GET_TX_AX_DESC_CHANNEL_DMA(_wd)		LE_BITS_TO_4BYTE(_wd, 16, 4)
+#define GET_TX_AX_DESC_WP_OFFSET(_wd)		LE_BITS_TO_4BYTE(_wd, 24, 8)
+
+/* DWORD 2 ; Offset 08h */
+#define GET_TX_AX_DESC_PKT_LEN(_wd)		LE_BITS_TO_2BYTE(_wd+8, 0, 14)
+
+/**
+ * rtw_hal_mac_sdio_check_tx_allow - Check hardware resource enough to xmit
+ * @hal:	pointer of struct rtw_hal_com_t
+ * @dma_ch:	dma channel to xmit, should be the same as field "CHANNEL_DMA "in WD
+ * @buf:	data buffer to xmit
+ * @len:	data buffer length
+ * @agg_count:	how many packets aggregated in this buffer
+ * @pkt_len:	array to store each packet's lenght in this buffer,
+ *		pkt_len value should be the same as field "PKT_LEN" in WD
+ * @wp_offset:	array to store each packet's wp_offset in this buffer
+ *		wp_offset should be the same as field "WP_OFFSET" in WD
+ * @txaddr:	return SDIO TX address for this tx buffer, and only valid when
+ *		function's return value is true.
+ * @txlen:	return SDIO TX length for this tx buffer, and only valid when
+ *		function's return value is true. txlen would >= len and align
+ *		to particular size by IC.
+ *
+ * Check if hardware resource is enough to send this tx buffer and return
+ * SDIO TX address.
+ *
+ * Return true if hardware resource is enough, otherwise false.
+ */
+bool rtw_hal_mac_sdio_check_tx_allow(struct rtw_hal_com_t *hal, u8 dma_ch,
+				     u8 *buf, u32 len, u8 agg_count,
+				     u16 *pkt_len, u8 *wp_offset, u32 *txaddr,
+				     u32 *txlen)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_sdio_tx_info info = {0};
+	u32 err;
+	u16 pkt_len_tmp;
+	u8 wp_offset_tmp;
+	u8 wd_ch;
+
+
+	/* For H2C, pkt_len & wp_offset may be invalid */
+	if (!pkt_len && (agg_count == 1)) {
+		pkt_len_tmp = GET_TX_AX_DESC_PKT_LEN(buf);
+		pkt_len = &pkt_len_tmp;
+	}
+	if (!wp_offset && (agg_count == 1)) {
+		wp_offset_tmp = (u8)GET_TX_AX_DESC_WP_OFFSET(buf);
+		wp_offset = &wp_offset_tmp;
+	}
+
+	/* check tx wd */
+	wd_ch = (u8)GET_TX_AX_DESC_CHANNEL_DMA(buf);
+	if (wd_ch != dma_ch) {
+		PHL_ERR("%s: tx channel abnormal, %u(wd) != %u(input)\n",
+			__func__, wd_ch, dma_ch);
+		debug_dump_data(buf, len&0xFFFF0000?0xFFFF:(u16)len, "wd:");
+	}
+
+	info.total_size = len;
+	info.dma_txagg_num = agg_count;
+	info.ch_dma = dma_ch;
+	info.pkt_size = pkt_len;
+	info.wp_offset = wp_offset;
+	info.chk_cnt = 5;
+
+	err = mac->ops->intf_ops->tx_allow_sdio(mac, &info);
+	if (err != MACSUCCESS) {
+		PHL_DBG("%s: tx allow fail! (%d)\n", __func__, err);
+		return false;
+	}
+
+	err = mac->ops->intf_ops->tx_cmd_addr_sdio(mac, &info, txaddr);
+	if (err != MACSUCCESS) {
+		PHL_ERR("%s: get tx addr fail! (%d)\n", __func__, err);
+		return false;
+	}
+
+	/*
+	 * TX I/O size should align to 8 bytes on SDIO byte count mode,
+	 * or hardware would go wrong.
+	 *
+	 * TODO: alignment size would be get from HALMAC return.
+	 */
+	*txlen = _ALIGN(len, 8);
+
+	return true;
+}
+
+static enum rtw_rx_type _pkt_type_mac2phl(enum mac_ax_pkt_t mac_type)
+{
+	enum rtw_rx_type t = RTW_RX_TYPE_MAX;
+
+
+	switch (mac_type) {
+	case MAC_AX_PKT_DATA:
+		t = RTW_RX_TYPE_WIFI;
+		break;
+	case MAC_AX_PKT_MGNT:
+	case MAC_AX_PKT_CTRL:
+	case MAC_AX_PKT_8023:
+	case MAC_AX_PKT_H2C:
+	case MAC_AX_PKT_FWDL:
+		/* TODO: unknown */
+		PHL_WARN("%s: unhandled RX type(%d)\n", __func__, mac_type);
+		break;
+	case MAC_AX_PKT_C2H:
+		t = RTW_RX_TYPE_C2H;
+		break;
+	case MAC_AX_PKT_PPDU:
+		t = RTW_RX_TYPE_PPDU_STATUS;
+		break;
+	case MAC_AX_PKT_CH_INFO:
+		t = RTW_RX_TYPE_CHANNEL_INFO;
+		break;
+	case MAC_AX_PKT_DFS:
+		t = RTW_RX_TYPE_DFS_RPT;
+		break;
+	default:
+		PHL_WARN("%s: unknon RX type(%d)\n", __func__, mac_type);
+		break;
+	}
+
+	return t;
+}
+
+static void hal_mac_parse_rxpkt_info(struct mac_ax_rxpkt_info *info,
+			      struct rtw_r_meta_data *meta)
+{
+	meta->pktlen = (u16)info->pktsize;	/* DW0 [0:13] */
+	meta->shift = info->shift;		/* DW0 [14:15] */
+	meta->rpkt_type = _pkt_type_mac2phl(info->type); /* DW0 [24:27] */
+	meta->drv_info_size = info->drvsize;	/* DW0 [28:30] */
+	meta->long_rxd = (info->rxdlen == RXD_LONG_LEN) ? 1 : 0; /* DW0 [31:31] */
+
+	if (info->type == MAC_AX_PKT_DATA) {
+#if 0
+		meta->dma_ch;
+		meta->hal_port;
+		meta->ta[6]; /* Transmitter Address */
+#endif
+#if 0
+		meta->wl_hd_iv_len;	/* DW0 [16:21] */
+		meta->bb_sel;		/* DW0 [22:22] */
+		meta->mac_info_vld;	/* DW0 [23:23] */
+#endif
+#if 0
+		meta->long_rxd;		/* DW0 [31:31] */
+
+		meta->ppdu_type;		/* DW1 [0:3] */
+		meta->ppdu_cnt;		/* DW1 [4:6] */
+		meta->sr_en;		/* DW1 [7:7] */
+		meta->user_id;		/* DW1 [8:15] */
+		meta->rx_rate;		/* DW1 [16:24] */
+		meta->rx_gi_ltf;		/* DW1 [25:27] */
+		meta->non_srg_ppdu;	/* DW1 [28:28] */
+		meta->inter_ppdu;		/* DW1 [29:29] */
+		meta->bw;			/* DW1 [30:31] */
+
+		meta->freerun_cnt;	/* DW2 [0:31] */
+
+		meta->a1_match;		/* DW3 [0:0] */
+		meta->sw_dec;		/* DW3 [1:1] */
+		meta->hw_dec;		/* DW3 [2:2] */
+		meta->ampdu;		/* DW3 [3:3] */
+		meta->ampdu_end_pkt;	/* DW3 [4:4] */
+		meta->amsdu;		/* DW3 [5:5] */
+		meta->amsdu_cut;		/* DW3 [6:6] */
+		meta->last_msdu;		/* DW3 [7:7] */
+		meta->bypass;		/* DW3 [8:8] */
+#endif
+		meta->crc32 = info->u.data.crc_err;	/* DW3 [9:9] */
+		meta->icverr = info->u.data.icv_err;	/* DW3 [10:10] */
+#if 0
+		meta->magic_wake;		/* DW3 [11:11] */
+		meta->unicast_wake;	/* DW3 [12:12] */
+		meta->pattern_wake;	/* DW3 [13:13] */
+		meta->get_ch_info;		/* DW3 [14:15] */
+		meta->pattern_idx;		/* DW3 [16:20] */
+		meta->target_idc;		/* DW3 [21:23] */
+		meta->chksum_ofld_en;	/* DW3 [24:24] */
+		meta->with_llc;		/* DW3 [25:25] */
+		meta->rx_statistics;	/* DW3 [26:26] */
+
+		meta->frame_type;		/* DW4 [0:1] */
+		meta->mc;			/* DW4 [2:2] */
+		meta->bc;			/* DW4 [3:3] */
+		meta->more_data;		/* DW4 [4:4] */
+		meta->more_frag;		/* DW4 [5:5] */
+		meta->pwr_bit;		/* DW4 [6:6] */
+		meta->qos;			/* DW4 [7:7] */
+		meta->tid;			/* DW4 [8:11] */
+		meta->eosp;		/* DW4 [12:12] */
+		meta->htc;			/* DW4 [13:13] */
+		meta->q_null;		/* DW4 [14:14] */
+		meta->seq;		/* DW4 [16:27] */
+		meta->frag_num;		/* DW4 [28:31] */
+
+		meta->sec_cam_idx;		/* DW5 [0:7] */
+		meta->addr_cam;		/* DW5 [8:15] */
+		meta->macid;		/* DW5 [16:23] */
+		meta->rx_pl_id;		/* DW5 [24:27] */
+		meta->addr_cam_vld;	/* DW5 [28:28] */
+		meta->addr_fwd_en;		/* DW5 [29:29] */
+		meta->rx_pl_match;		/* DW5 [30:30] */
+
+		meta->mac_addr[6];		/* DW6 [0:31] DW7 [0:15] */
+		meta->smart_ant;		/* DW7 [16:16] */
+		meta->sec_type;		/* DW7 [17:20] */
+#endif
+	} else if (info->type == MAC_AX_PKT_PPDU) {
+		meta->mac_info_vld = info->u.ppdu.mac_info; /* DW0 [23:23] */
+	}
+}
+
+/* AX RX DESC */
+/* DWORD 0 ; Offset 00h */
+#define GET_RX_AX_DESC_PKT_LEN(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc, 0, 14)
+#define GET_RX_AX_DESC_SHIFT(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc, 14, 2)
+#define GET_RX_AX_DESC_HDR_IV_L(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc, 16, 6)
+#define GET_RX_AX_DESC_BB_SEL(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc, 22, 1)
+#define GET_RX_AX_DESC_MAC_INFO_VLD(__pRxStatusDesc) LE_BITS_TO_4BYTE( __pRxStatusDesc, 23, 1)
+#define GET_RX_AX_DESC_RPKT_TYPE(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc, 24, 4)
+#define GET_RX_AX_DESC_DRV_INFO_SIZE(__pRxStatusDesc) LE_BITS_TO_4BYTE( __pRxStatusDesc, 28, 3)
+#define GET_RX_AX_DESC_LONG_RXD(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc, 31, 1)
+
+/* DWORD 1 ; Offset 04h */
+#define GET_RX_AX_DESC_PPDU_TYPE(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 0, 4)
+#define GET_RX_AX_DESC_PPDU_CNT(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 4, 3)
+#define GET_RX_AX_DESC_SR_EN(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 7, 1)
+#define GET_RX_AX_DESC_USER_ID(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 8, 8)
+#define GET_RX_AX_DESC_RX_DATARATE(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 16, 9)
+#define GET_RX_AX_DESC_RX_GI_LTF(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 25, 3)
+#define GET_RX_AX_DESC_NON_SRG_PPDU(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 28, 1)
+#define GET_RX_AX_DESC_INTER_PPDU(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 29, 1)
+#define GET_RX_AX_DESC_BW(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+4, 30, 2)
+
+/* DWORD 2 ; Offset 08h */
+#define GET_RX_AX_DESC_FREERUN_CNT(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+8, 0, 32)
+
+/* DWORD 3 ; Offset 0ch */
+#define GET_RX_AX_DESC_A1_MATCH(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 0, 1)
+#define GET_RX_AX_DESC_SW_DEC(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 1, 1)
+#define GET_RX_AX_DESC_HW_DEC(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 2, 1)
+#define GET_RX_AX_DESC_AMPDU(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 3, 1)
+#define GET_RX_AX_DESC_AMPDU_EDN_PKT(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 4, 1)
+#define GET_RX_AX_DESC_AMSDU(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 5, 1)
+#define GET_RX_AX_DESC_AMSDU_CUT(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 6, 1)
+#define GET_RX_AX_DESC_LAST_MSDU(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 7, 1)
+#define GET_RX_AX_DESC_BYPASS(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 8, 1)
+#define GET_RX_AX_DESC_CRC32(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 9, 1)
+#define GET_RX_AX_DESC_ICVERR(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 10, 1)
+#define GET_RX_AX_DESC_MAGIC_WAKE(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 11, 1)
+#define GET_RX_AX_DESC_UNICAST_WAKE(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 12, 1)
+#define GET_RX_AX_DESC_PATTERN_WAKE(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 13, 1)
+
+#define GET_RX_AX_DESC_CH_INFO(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 14, 1)
+#define GET_RX_AX_DESC_STATISTICS(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 15, 1)
+#define GET_RX_AX_DESC_PATTERN_IDX(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 16, 5)
+#define GET_RX_AX_DESC_TARGET_IDC(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 21, 3)
+#define GET_RX_AX_DESC_CHKSUM_OFFLOAD(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 24, 1)
+#define GET_RX_AX_DESC_WITH_LLC(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+12, 25, 1)
+
+/* DWORD 4 ; Offset 10h */
+#define GET_RX_AX_DESC_TYPE(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 0, 2)
+#define GET_RX_AX_DESC_MC(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 2, 1)
+#define GET_RX_AX_DESC_BC(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 3, 1)
+#define GET_RX_AX_DESC_MD(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 4, 1)
+#define GET_RX_AX_DESC_MF(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 5, 1)
+#define GET_RX_AX_DESC_PWR(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 6, 1)
+#define GET_RX_AX_DESC_QOS(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 7, 1)
+#define GET_RX_AX_DESC_TID(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 8, 4)
+#define GET_RX_AX_DESC_EOSP(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 12, 1)
+#define GET_RX_AX_DESC_HTC(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 13, 1)
+#define GET_RX_AX_DESC_QNULL(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 14, 1)
+
+#define GET_RX_AX_DESC_SEQ(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 16, 12)
+#define GET_RX_AX_DESC_FRAG(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+16, 28, 4)
+
+/* DWORD 5 ; Offset 14h */
+#define GET_RX_AX_DESC_CAM_IDX(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+20, 0, 8)
+#define GET_RX_AX_DESC_ADDR_CAM(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+20, 8, 8)
+#define GET_RX_AX_DESC_MACID(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+20, 16, 8)
+#define GET_RX_AX_DESC_PL_ID(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+20, 24, 4)
+#define GET_RX_AX_DESC_CAM_VLD(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+20, 28, 1)
+#define GET_RX_AX_DESC_FWD_EN(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+20, 29, 1)
+#define GET_RX_AX_DESC_PL_MATCH(__pRxStatusDesc) LE_BITS_TO_4BYTE(__pRxStatusDesc+20, 30, 1)
+
+static void hal_mac_parse_rxd(struct rtw_hal_com_t *hal,
+			      u8 *rxd, struct rtw_r_meta_data *meta)
+{
+#if 0
+	meta->pktlen = GET_RX_AX_DESC_PKT_LEN(rxd);
+	meta->shift = GET_RX_AX_DESC_SHIFT(rxd);
+	meta->wl_hd_iv_len = GET_RX_AX_DESC_HDR_IV_L(rxd);
+	meta->bb_sel = GET_RX_AX_DESC_BB_SEL(rxd);
+	meta->mac_info_vld = GET_RX_AX_DESC_MAC_INFO_VLD(rxd);
+	meta->rpkt_type = GET_RX_AX_DESC_RPKT_TYPE(rxd);
+	meta->drv_info_size = GET_RX_AX_DESC_DRV_INFO_SIZE(rxd);
+	meta->long_rxd = GET_RX_AX_DESC_LONG_RXD(rxd);
+
+	meta->ppdu_type = GET_RX_AX_DESC_PPDU_TYPE(rxd);
+	meta->ppdu_cnt = GET_RX_AX_DESC_PPDU_CNT(rxd);
+	meta->sr_en = GET_RX_AX_DESC_SR_EN(rxd);
+	meta->user_id = GET_RX_AX_DESC_USER_ID(rxd);
+	meta->rx_rate = GET_RX_AX_DESC_RX_DATARATE(rxd);
+	meta->rx_gi_ltf = GET_RX_AX_DESC_RX_GI_LTF(rxd);
+	meta->non_srg_ppdu = GET_RX_AX_DESC_NON_SRG_PPDU(rxd);
+	meta->inter_ppdu = GET_RX_AX_DESC_INTER_PPDU(rxd);
+	meta->bw = GET_RX_AX_DESC_BW(rxd);
+
+	meta->freerun_cnt = GET_RX_AX_DESC_FREERUN_CNT(rxd);
+
+	meta->a1_match = GET_RX_AX_DESC_A1_MATCH(rxd);
+#endif
+	meta->sw_dec = GET_RX_AX_DESC_SW_DEC(rxd);
+	meta->hw_dec = GET_RX_AX_DESC_HW_DEC(rxd);
+#if 0
+	meta->ampdu = GET_RX_AX_DESC_AMPDU(rxd);
+	meta->ampdu_end_pkt = GET_RX_AX_DESC_AMPDU_EDN_PKT(rxd);
+	meta->amsdu = GET_RX_AX_DESC_AMSDU(rxd);
+#endif
+	meta->amsdu_cut = GET_RX_AX_DESC_AMSDU_CUT(rxd);
+	meta->last_msdu = GET_RX_AX_DESC_LAST_MSDU(rxd);
+#if 0
+	meta->bypass = GET_RX_AX_DESC_BYPASS(rxd);
+	meta->crc32 = GET_RX_AX_DESC_CRC32(rxd);
+	meta->icverr = GET_RX_AX_DESC_ICVERR(rxd);
+	meta->magic_wake = GET_RX_AX_DESC_MAGIC_WAKE(rxd);
+	meta->unicast_wake = GET_RX_AX_DESC_UNICAST_WAKE(rxd);
+	meta->pattern_wake = GET_RX_AX_DESC_PATTERN_WAKE(rxd);
+	meta->get_ch_info = GET_RX_AX_DESC_CH_INFO(rxd);
+	meta->rx_statistics = GET_RX_AX_DESC_STATISTICS(rxd);
+
+	meta->pattern_idx = GET_RX_AX_DESC_PATTERN_IDX(rxd);
+	meta->target_idc = GET_RX_AX_DESC_TARGET_IDC(rxd);
+	meta->chksum_ofld_en = GET_RX_AX_DESC_CHKSUM_OFFLOAD(rxd);
+	meta->with_llc = GET_RX_AX_DESC_WITH_LLC(rxd);
+#endif
+	meta->chksum_ofld_en = GET_RX_AX_DESC_CHKSUM_OFFLOAD(rxd);
+
+	if (meta->long_rxd == 1) {
+#if 0
+		meta->frame_type = GET_RX_AX_DESC_TYPE(rxd);
+#endif
+		meta->mc = GET_RX_AX_DESC_MC(rxd);
+		meta->bc = GET_RX_AX_DESC_BC(rxd);
+#if 0
+		meta->more_data = GET_RX_AX_DESC_MD(rxd);
+#endif
+		meta->more_frag = GET_RX_AX_DESC_MF(rxd);
+#if 0
+		meta->pwr_bit = GET_RX_AX_DESC_PWR(rxd);
+#endif
+		meta->qos = GET_RX_AX_DESC_QOS(rxd);
+
+		meta->tid = GET_RX_AX_DESC_TID(rxd);
+#if 0
+		meta->eosp = GET_RX_AX_DESC_EOSP(rxd);
+		meta->htc = GET_RX_AX_DESC_HTC(rxd);
+#endif
+		meta->q_null = GET_RX_AX_DESC_QNULL(rxd);
+		meta->seq = GET_RX_AX_DESC_SEQ(rxd);
+		meta->frag_num = GET_RX_AX_DESC_FRAG(rxd);
+
+#if 0
+		meta->sec_cam_idx = GET_RX_AX_DESC_CAM_IDX(rxd);
+		meta->addr_cam = GET_RX_AX_DESC_ADDR_CAM(rxd);
+
+		meta->macid = GET_RX_AX_DESC_MACID(rxd);
+		meta->rx_pl_id = GET_RX_AX_DESC_PL_ID(rxd);
+		meta->addr_cam_vld = GET_RX_AX_DESC_CAM_VLD(rxd);
+		meta->addr_fwd_en = GET_RX_AX_DESC_FWD_EN(rxd);
+		meta->rx_pl_match = GET_RX_AX_DESC_PL_MATCH(rxd);
+#endif
+
+		_os_mem_cpy(hal->drv_priv,
+			(void*)&meta->mac_addr, (void*)(rxd + 24), MAC_ALEN);
+	}
+}
+
+/**
+ * rtw_hal_mac_sdio_lps_flg - get lps status
+ * @hal:	struct rtw_hal_com_t *
+ * @flg:	show lps status,
+ * 		0: not in lps cg/pg status
+ * 		1: in lps cg/pg status
+ *
+ * Get driver lps status from HALMAC
+ *
+ */
+void rtw_hal_mac_sdio_lps_flg(struct rtw_hal_com_t *hal, u8 *flg)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+
+	ops->get_hw_value(mac, MAC_AX_HW_GET_SDIO_LPS_FLG, flg);
+}
+
+int rtw_hal_mac_sdio_parse_rx(struct rtw_hal_com_t *hal,
+			      struct rtw_rx_buf *rxbuf)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_rxpkt_info info = {0};
+	struct sdio_rx_pkt *pkt;
+	u8 *ptr;
+	u32 len;
+	int i;
+	u32 offset;
+	u32 err = MACSUCCESS;
+
+
+	rxbuf->agg_cnt = 0;
+	rxbuf->agg_start = 0;
+	ptr = rxbuf->ptr;
+	len = rxbuf->len;
+
+	for (i = 0; i < MAX_BUS_RX_AGG; i++) {
+		_os_mem_set(hal->drv_priv, &info, 0, sizeof(info));
+		err = mac->ops->parse_rxdesc(mac, &info, ptr, len);
+		if (err != MACSUCCESS) {
+			PHL_ERR("%s: agg_idx=%d, len=%u(%u), parse_rxdesc FAIL!(%u)\n",
+				__func__, i, len, rxbuf->len, err);
+			rxbuf->len -= len;
+			len = 0;
+			break;
+		}
+
+		pkt = &rxbuf->pkt[i];
+		_os_mem_set(hal->drv_priv, pkt, 0, sizeof(*pkt));
+
+		pkt->wd = ptr;
+		pkt->wd_len = (u8)info.rxdlen;
+		offset = info.rxdlen + (info.drvsize * 8) + (info.shift * 2);
+		pkt->pkt = ptr + offset;
+		pkt->pkt_len = (u16)info.pktsize;
+		hal_mac_parse_rxpkt_info(&info, &pkt->meta);
+		hal_mac_parse_rxd(hal, ptr, &pkt->meta);
+		_os_mem_cpy(hal->drv_priv, pkt->meta.ta, pkt->pkt + 10, 6);
+
+		offset += info.pktsize;
+
+		offset = _ALIGN(offset, 8);
+#ifdef CONFIG_PHL_CSUM_OFFLOAD_RX
+		if (pkt->meta.chksum_ofld_en) {
+			u8 status;
+			u32 result;
+
+			status = *(ptr + offset);
+			result = mac->ops->chk_rx_tcpip_chksum_ofd(mac, status);
+			hal_mac_parse_rxd_checksume(hal, &pkt->meta, result);
+			offset += 8;
+		}
+#endif /* CONFIG_PHL_CSUM_OFFLOAD_RX */
+		ptr += offset;
+		if (offset >= len) {
+			len = 0;
+			i++;
+			break;
+		}
+		len -= offset;
+	}
+
+	rxbuf->agg_cnt = (u8)i;
+	if (len) {
+		/* not finish yet */
+		rxbuf->next_ptr = ptr;
+		offset = (u32)(ptr - rxbuf->ptr);
+	} else {
+		rxbuf->next_ptr = NULL;
+		offset = rxbuf->len;
+	}
+
+	return offset;
+}
+
+int rtw_hal_mac_sdio_rx(struct rtw_hal_com_t *hal, struct rtw_rx_buf *rxbuf)
+{
+	struct hal_info_t *hal_info = hal->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	u32 len;
+
+
+	if (ops->get_hw_value(mac, MAC_AX_HW_GET_SDIO_RX_REQ_LEN,
+				&len) != MACSUCCESS) {
+		PHL_ERR("%s: Get SDIO Rx req len fail!\n", __func__);
+		return 0;
+	}
+
+	if (!len)
+		return 0;
+	if (len > rxbuf->buf_len) {
+		/* TODO: read and drop */
+		return 0;
+	}
+
+	if (_FAIL == hal_sdio_cmd53_r(hal, SDIO_CMD_ADDR_RXFF, len, rxbuf->buffer))
+		return 0;
+
+	rxbuf->used_len = len;
+	rxbuf->ptr = rxbuf->buffer;
+	rxbuf->len = len;
+
+#ifndef CONFIG_PHL_SDIO_READ_RXFF_IN_INT
+	len = rtw_hal_mac_sdio_parse_rx(hal, rxbuf);
+#endif
+
+	return len;
+}
+#endif /*CONFIG_SDIO_HCI*/
+
+enum rtw_hal_status rtw_hal_mac_get_pwr_state(struct hal_info_t *hal_info,
+				enum rtw_mac_pwr_st *pwr_state)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	enum mac_ax_mac_pwr_st st;
+
+	if (hal_mac_ops->get_hw_value(mac, MAC_AX_HW_GET_PWR_STATE, &st) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("%s : retrieve WL MAC state %u\n", __func__, st);
+
+	switch (st) {
+	case MAC_AX_MAC_OFF:
+		*pwr_state = RTW_MAC_PWR_OFF;
+		break;
+	case MAC_AX_MAC_ON:
+		*pwr_state = RTW_MAC_PWR_ON;
+		break;
+	case MAC_AX_MAC_LPS:
+		*pwr_state = RTW_MAC_PWR_LPS;
+		break;
+	default:
+		*pwr_state = RTW_MAC_PWR_NONE;
+		break;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/* halmac wrapper API for hal and proto type is at hal_api_mac.h */
+enum rtw_hal_status
+rtw_hal_mac_power_switch(struct rtw_phl_com_t *phl_com,
+			 struct hal_info_t *hal_info,
+			 u8 on_off)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	//u8 pwr_state = 0;
+
+	/*pwr_state = hal_mac_get_pwr_state(mac);
+	if(pwr_state != on_off)*/
+
+	if (mac->ops->pwr_switch(mac, on_off) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef DBG_PHL_MAC_REG_RW
+
+bool rtw_hal_mac_reg_chk(struct rtw_hal_com_t *hal_com, u32 addr)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	bool rst = true;
+	u32 mac_rst;
+
+	if (mac != NULL && mac->ops->io_chk_access) {
+		mac_rst = mac->ops->io_chk_access(mac, addr);
+		if (mac_rst != MACSUCCESS) {
+			rst = false;
+			PHL_ERR("%s failed - addr(0x%08x) is err code(%d)\n",
+				__func__, addr, mac_rst);
+			_os_warn_on(1);
+		}
+	}
+
+	return rst;
+}
+#endif
+
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status
+rtw_hal_mac_dbcc_trx_ctrl(struct hal_info_t *hal_info,
+				      enum phl_band_idx band_idx,
+				      bool pause)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum mac_ax_band mac_band;
+	u8 mac_pause;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	mac_band = (band_idx == HW_BAND_0) ? MAC_AX_BAND_0 : MAC_AX_BAND_1;
+	mac_pause = (pause == true) ? 1 : 0;
+
+	if (mac->ops->dbcc_trx_ctrl(mac, mac_band, mac_pause) != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s: mac_dbcc_trx_ctrl fail\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/* halmac wrapper API for hal and proto type is at hal_api_mac.h */
+enum rtw_hal_status rtw_hal_mac_dbcc_pre_cfg(struct rtw_phl_com_t *phl_com,
+					     struct hal_info_t *hal_info,
+					     u8 dbcc_en)
+{
+	enum rtw_hal_status h_sts = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_dbcc_cfg_info dbcc_info  = { 0 };
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	PHL_INFO("%s: dbcc_en(%d)\n", __func__, dbcc_en);
+
+	dbcc_info.trx_mode = (phl_com->dev_cap.tx_mu_ru == true) ? MAC_AX_TRX_SW_MODE
+								  : MAC_AX_TRX_HW_MODE;
+
+	dbcc_info.qta_mode = (dbcc_en == true) ? MAC_AX_QTA_DBCC
+						: MAC_AX_QTA_SCC;
+
+	dbcc_info.dbcc_en = dbcc_en;
+
+	if (mac->ops->dbcc_pre_cfg(mac, &dbcc_info) != MACSUCCESS) {
+		PHL_ERR("%s: mac dbcc_pre_cfg failed\n", __func__);
+		h_sts = RTW_HAL_STATUS_FAILURE;
+	}
+
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+
+	return h_sts;
+}
+
+enum rtw_hal_status rtw_hal_mac_dbcc_cfg(struct rtw_phl_com_t *phl_com,
+					 struct hal_info_t *hal_info,
+					 u8 dbcc_en)
+{
+	enum rtw_hal_status h_sts = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_dbcc_cfg_info dbcc_info  = { 0 };
+
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+
+	PHL_INFO("%s: dbcc_en(%d)\n", __func__, dbcc_en);
+
+	dbcc_info.trx_mode = (phl_com->dev_cap.tx_mu_ru == true) ? MAC_AX_TRX_SW_MODE
+								  : MAC_AX_TRX_HW_MODE;
+	dbcc_info.qta_mode = (dbcc_en == true) ? MAC_AX_QTA_DBCC
+						: MAC_AX_QTA_SCC;
+	dbcc_info.dbcc_en = dbcc_en;
+
+	if (mac->ops->dbcc_cfg(mac, &dbcc_info) != MACSUCCESS) {
+		PHL_ERR("%s: mac dbcc_pre_cfg failed\n", __func__);
+		h_sts = RTW_HAL_STATUS_FAILURE;
+	}
+
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return h_sts;
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+#ifdef CONFIG_BTCOEX
+u32 rtw_hal_mac_coex_init(struct rtw_hal_com_t *hal_com, u8 pta_mode, u8 direction)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_coex pta_para = {0};
+
+	pta_para.pta_mode = pta_mode;
+	pta_para.direction = direction;
+
+	return (ops->coex_init(mac, &pta_para));
+}
+
+u32 rtw_hal_mac_coex_reg_read(struct rtw_hal_com_t *hal_com, u32 offset, u32 *value)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+
+	/* valid offset -> 0xda00~0xdaff  */
+	offset = offset & 0xff;
+
+	return (ops->coex_read(mac, offset, value));
+}
+
+u32 rtw_hal_mac_coex_reg_write(struct rtw_hal_com_t *hal_com, u32 offset, u32 value)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+
+	/* valid offset -> 0xda00~0xdaff  */
+	offset = offset & 0xff;
+
+	return (ops->coex_write(mac, offset, value));
+}
+
+u32 rtw_hal_mac_set_scoreboard(struct rtw_hal_com_t *hal_com, u32 *value)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+
+	return (ops->set_hw_value(mac, MAC_AX_HW_SET_SCOREBOARD, value));
+}
+
+u32 rtw_hal_mac_get_scoreboard(struct rtw_hal_com_t *hal_com, u32 *value)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+
+	return (ops->get_hw_value(mac, MAC_AX_HW_GET_SCOREBOARD, value));
+}
+
+u32 rtw_hal_mac_set_grant_act(struct rtw_hal_com_t *hal_com, u8 *value)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_coex_gnt gnt_val = {0};
+
+	gnt_val.band0.gnt_bt_sw_en = value[0];
+	gnt_val.band0.gnt_bt = value[1];
+	gnt_val.band0.gnt_wl_sw_en = value[2];
+	gnt_val.band0.gnt_wl = value[3];
+
+	gnt_val.band1.gnt_bt_sw_en = value[4];
+	gnt_val.band1.gnt_bt = value[5];
+	gnt_val.band1.gnt_wl_sw_en = value[6];
+	gnt_val.band1.gnt_wl = value[7];
+
+	gnt_val.bt0.wlan_act_en = value[8];
+	gnt_val.bt0.wlan_act = value[9];
+	gnt_val.bt1.wlan_act_en = value[10];
+	gnt_val.bt1.wlan_act = value[11];
+
+	return (ops->set_hw_value(mac, MAC_AX_HW_SET_COEX_GNT, &gnt_val));
+}
+
+u32 rtw_hal_mac_set_grant(struct rtw_hal_com_t *hal_com, u8 *value)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_coex_gnt gnt_val = {0};
+
+	gnt_val.band0.gnt_bt_sw_en = value[0];
+	gnt_val.band0.gnt_bt = value[1];
+	gnt_val.band0.gnt_wl_sw_en = value[2];
+	gnt_val.band0.gnt_wl = value[3];
+
+	gnt_val.band1.gnt_bt_sw_en = value[4];
+	gnt_val.band1.gnt_bt = value[5];
+	gnt_val.band1.gnt_wl_sw_en = value[6];
+	gnt_val.band1.gnt_wl = value[7];
+
+	return (ops->set_hw_value(mac, MAC_AX_HW_SET_COEX_GNT, &gnt_val));
+}
+
+u32 rtw_hal_mac_get_grant(struct rtw_hal_com_t *hal_com, u8 *value)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+
+	return (ops->get_hw_value(mac, MAC_AX_HW_GET_COEX_GNT, value));
+}
+
+u32 rtw_hal_mac_set_polluted(struct rtw_hal_com_t *hal_com, u8 band, u8 tx_val, u8 rx_val)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_plt plt_val = {0};
+
+	plt_val.band = band;
+	plt_val.tx = tx_val;
+	plt_val.rx = rx_val;
+
+	return (ops->set_hw_value(mac, MAC_AX_HW_SET_POLLUTED, &plt_val));
+}
+
+u32 rtw_hal_mac_get_bt_polt_cnt(struct rtw_hal_com_t *hal_com, u8 band, u16 *cnt)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_bt_polt_cnt polt = {0};
+	u32 result;
+
+	polt.band = band;
+	polt.cnt = *cnt;
+	result = ops->get_hw_value(mac, MAC_AX_HW_GET_POLLUTED_CNT, &polt);
+	*cnt = polt.cnt;
+
+	return (result);
+}
+
+u32 rtw_hal_mac_set_coex_ctrl(struct rtw_hal_com_t *hal_com, u32 val)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+
+	return (ops->set_hw_value(mac, MAC_AX_HW_SET_COEX_CTRL, &val));
+}
+
+u32 rtw_hal_mac_get_coex_ctrl(struct rtw_hal_com_t *hal_com, u32* val)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+
+	return (ops->get_hw_value(mac, MAC_AX_HW_GET_COEX_CTRL, val));
+}
+#endif /* CONFIG_BTCOEX */
+
+u32 rtw_hal_mac_set_tx_time(struct rtw_hal_com_t *hal_com, u8 is_btc, u8 is_resume, u8 macid, u32 tx_time)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_max_tx_time max_tx_time = {0};
+	u32 result = 0xffffffff;
+
+	max_tx_time.macid = macid;
+	max_tx_time.is_cctrl = 1;
+	max_tx_time.max_tx_time = tx_time;
+
+	if (is_btc && !is_resume) { /* for btc control tx time case */
+		hal_com->btc_ctrl.tx_time = true;
+		result = ops->set_hw_value(mac, MAC_AX_HW_SET_MAX_TX_TIME, &max_tx_time);
+	} else if (is_btc && is_resume) { /* for btc release tx time case */
+		result = ops->set_hw_value(mac, MAC_AX_HW_SET_MAX_TX_TIME, &max_tx_time);
+		hal_com->btc_ctrl.tx_time = false;
+	} else { /* not btc control case */
+		if (hal_com->btc_ctrl.tx_time)
+			return result;
+		else
+			result = ops->set_hw_value(mac, MAC_AX_HW_SET_MAX_TX_TIME, &max_tx_time);
+	}
+
+	return result;
+}
+
+u32 rtw_hal_mac_get_tx_time(struct rtw_hal_com_t *hal_com, u8 macid, u32 *tx_time)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_max_tx_time max_tx_time = {0};
+	u32 result;
+
+	max_tx_time.macid = macid;
+	max_tx_time.is_cctrl = 1;
+	max_tx_time.max_tx_time = *tx_time;
+
+	result = ops->get_hw_value(mac, MAC_AX_HW_GET_MAX_TX_TIME, &max_tx_time);
+	*tx_time = max_tx_time.max_tx_time;
+
+	return (result);
+}
+
+u32 rtw_hal_mac_set_tx_retry_limit(struct rtw_hal_com_t *hal_com, u8 is_btc, u8 is_resume, u8 macid, u8 tx_retry)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_cctl_rty_lmt_cfg tx_retry_limit = {0};
+	u32 result = 0xffffffff;
+
+	tx_retry_limit.macid = macid;
+	tx_retry_limit.data_lmt_sel = 1;
+	tx_retry_limit.data_lmt_val = tx_retry;
+	tx_retry_limit.rts_lmt_sel = 0;
+	tx_retry_limit.rts_lmt_val = 0;
+
+	if (is_btc && !is_resume) { /* for btc control tx time case */
+		hal_com->btc_ctrl.tx_retry = true;
+		result = ops->set_hw_value(mac, MAC_AX_HW_SET_CCTL_RTY_LMT, &tx_retry_limit);
+	} else if (is_btc && is_resume) { /* for btc release tx time case */
+		result = ops->set_hw_value(mac, MAC_AX_HW_SET_CCTL_RTY_LMT, &tx_retry_limit);
+		hal_com->btc_ctrl.tx_retry = false;
+	} else { /* not btc control case */
+		if (hal_com->btc_ctrl.tx_retry)
+			return result;
+		else
+			result = ops->set_hw_value(mac, MAC_AX_HW_SET_CCTL_RTY_LMT, &tx_retry_limit);
+	}
+
+	return result;
+}
+
+u32 rtw_hal_mac_set_rsc_cfg(struct rtw_hal_com_t *hal_com,  u8 rsc_cfg, u8 band)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_rrsr_cfg rrsr_cfg = {0};
+	u32 result = 0xffffffff;
+
+	rrsr_cfg.band = band;
+
+	/* get origin rrsr config */
+	result = ops->get_hw_value(mac, MAC_AX_HW_GET_RRSR_CFG, &rrsr_cfg);
+	if (MACSUCCESS != result) {
+		PHL_ERR("%s :get rrsr cfg fail,  mac status %d.\n", __func__, result);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	/* set target rsc cfg */
+	rrsr_cfg.rsc = rsc_cfg;
+	result = ops->set_hw_value(mac, MAC_AX_HW_SET_RRSR_CFG, &rrsr_cfg);
+	if (MACSUCCESS != result) {
+		PHL_ERR("%s :set rrsr cfg fail,  mac status %d.\n", __func__, result);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32 rtw_hal_mac_set_rrsr_ref_rate_sel(struct rtw_hal_com_t *hal_com, bool ref_rate_sel, u8 band)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_rrsr_cfg rrsr_cfg = {0};
+	u32 result = 0xffffffff;
+
+	rrsr_cfg.band = band;
+
+	/* get origin rrsr config */
+	result = ops->get_hw_value(mac, MAC_AX_HW_GET_RRSR_CFG, &rrsr_cfg);
+	if (MACSUCCESS != result) {
+		PHL_ERR("%s :get rrsr cfg fail,  mac status %d.\n", __func__, result);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	/* set target ref_rate_sel cfg */
+	rrsr_cfg.ref_rate_sel = ref_rate_sel ? 1 : 0;
+	result = ops->set_hw_value(mac, MAC_AX_HW_SET_RRSR_CFG, &rrsr_cfg);
+	if (MACSUCCESS != result) {
+		PHL_ERR("%s :set rrsr cfg fail,  mac status %d.\n", __func__, result);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32 rtw_hal_mac_get_tx_retry_limit(struct rtw_hal_com_t *hal_com, u8 macid, u8 *tx_retry)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_rty_lmt tx_retry_limit = {0};
+	u32 result;
+
+	tx_retry_limit.macid = macid;
+	tx_retry_limit.tx_cnt = *tx_retry;
+	result = ops->get_hw_value(mac, MAC_AX_HW_GET_DATA_RTY_LMT, &tx_retry_limit);
+	*tx_retry = (u8)tx_retry_limit.tx_cnt;
+
+	return (result);
+}
+
+/* halmac wrapper API for hal and proto type is at hal_api_drv.h */
+u32 rtw_hal_mac_send_h2c(struct rtw_hal_com_t *hal_com,
+	struct rtw_g6_h2c_hdr *hdr, u32 *pvalue)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	u32 mac_err = 0;
+
+	mac_err = ops->outsrc_h2c_common(mac, hdr, pvalue);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+static enum mac_ax_net_type
+_rtype_to_mac_nettype(struct rtw_wifi_role_t *wifi_role,
+                      struct rtw_wifi_role_link_t *rlink)
+{
+	enum mac_ax_net_type net_type = MAC_AX_NET_TYPE_NO_LINK;
+
+	switch (wifi_role->type) {
+	case PHL_RTYPE_P2P_GC:
+	case PHL_RTYPE_STATION:
+	case PHL_RTYPE_TDLS:
+		net_type = (rlink->mstate == MLME_NO_LINK)
+			? MAC_AX_NET_TYPE_NO_LINK
+			: MAC_AX_NET_TYPE_INFRA;
+		break;
+	case PHL_RTYPE_MONITOR:
+	case PHL_RTYPE_P2P_DEVICE:
+		net_type = MAC_AX_NET_TYPE_NO_LINK;
+		break;
+	case PHL_RTYPE_AP:
+	case PHL_RTYPE_VAP:
+	case PHL_RTYPE_P2P_GO:
+	case PHL_RTYPE_MESH:
+		net_type = MAC_AX_NET_TYPE_AP;
+		break;
+	case PHL_RTYPE_ADHOC:
+	case PHL_RTYPE_ADHOC_MASTER:
+		net_type = MAC_AX_NET_TYPE_ADHOC;
+		break;
+	case PHL_RTYPE_NAN: /*TODO*/
+	default:
+		net_type = MAC_AX_NET_TYPE_NO_LINK;
+		break;
+	}
+
+	return net_type;
+}
+
+static void
+_hal_dump_port_init_info(struct mac_ax_port_init_para *para)
+{
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "%s(): port_idx(%d), band_idx(%d), net_type(%d), dtim_period(%d)\n",
+		__FUNCTION__, para->port_idx, para->band_idx, para->net_type, para->dtim_period);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "%s(): mbid_num(%d), bss_color(%d), bcn_interval(%d), hiq_win(%d)\n",
+		__FUNCTION__, para->mbid_num, para->bss_color, para->bcn_interval, para->hiq_win);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_port_init(struct hal_info_t *hal_info,
+                      struct rtw_wifi_role_t *wifi_role,
+                      struct rtw_wifi_role_link_t *rlink)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_port_init_para ppara = {0};
+	struct rtw_phl_stainfo_t *phl_sta;
+	u8 hw_port = rlink->hw_port;
+#ifdef RTW_PHL_BCN
+	u16 bcn_interval = (u16)rlink->bcn_cmn.bcn_interval;
+#endif
+	u8 bsscolor = rlink->protocol_cap.bsscolor;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	ppara.net_type = _rtype_to_mac_nettype(wifi_role, rlink);
+	ppara.band_idx = rlink->hw_band;
+	ppara.port_idx = hw_port;
+	/*#ifdef CONFIG_AP*/
+	ppara.hiq_win = rlink->hiq_win;
+	ppara.dtim_period = rlink->dtim_period;
+	ppara.mbid_num = rlink->mbid_num;/*max mbid number*/
+	/*#endif - CONFIG_AP*/
+
+	if (ppara.net_type == MAC_AX_NET_TYPE_INFRA) {
+		phl_sta = rtw_phl_get_stainfo_self(wifi_role->phl_com->phl_priv,
+		                                   rlink);
+
+		if (phl_sta->asoc_cap.bcn_interval)
+			ppara.bcn_interval = phl_sta->asoc_cap.bcn_interval;
+		else
+			ppara.bcn_interval = 100;
+		ppara.bss_color = phl_sta->asoc_cap.bsscolor;
+	} else if (ppara.net_type == MAC_AX_NET_TYPE_AP) {
+		#ifdef CONFIG_RTW_SUPPORT_MBSSID_VAP
+		if (hw_port == 0) {
+			PHL_INFO("Initializing port 0 with %u M-BSSID.\n", ppara.mbid_num);
+			ppara.mbid_num = MAX_MBSSID_NUMBER;
+		}
+		#endif /* CONFIG_RTW_SUPPORT_MBSSID_VAP */
+
+		#ifdef RTW_PHL_BCN
+		ppara.bcn_interval = bcn_interval;
+		#else
+		ppara.bcn_interval = 100;
+		#endif
+		ppara.bss_color = bsscolor;
+	} else if (ppara.net_type == MAC_AX_NET_TYPE_ADHOC) {
+		/* TODO */
+		ppara.bcn_interval = 100;
+	} else {
+		/* other net_type, i.e. MAC_AX_NO_LINK */
+		ppara.bcn_interval = 100;
+	}
+	_hal_dump_port_init_info(&ppara);
+	if (mac->ops->port_init(mac, &ppara) == MACSUCCESS) {
+		hsts = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_ERR("%s : mac_port_init fail\n", __FUNCTION__);
+	}
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+
+static void
+_hal_dump_port_cfg_info(enum mac_ax_port_cfg_type ptype,
+			struct mac_ax_port_cfg_para *para)
+{
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "%s():ptype(%d), mbssid_idx(%d), val(%d), port(%d), band(%d)\n",
+		__FUNCTION__, ptype, para->mbssid_idx, para->val, para->port,
+		para->band);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_port_cfg(struct hal_info_t *hal_info,
+                     struct rtw_wifi_role_link_t *rlink,
+                     enum pcfg_type type,
+                     void *param)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum mac_ax_port_cfg_type ptype = MAC_AX_PCFG_FUNC_SW;
+	struct mac_ax_port_cfg_para ppara = {0};
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	switch (type){
+	case PCFG_FUNC_SW :
+		ptype = MAC_AX_PCFG_FUNC_SW;
+		break;
+	case PCFG_TBTT_AGG :
+		ptype = MAC_AX_PCFG_TBTT_AGG;
+		break;
+	case PCFG_TBTT_SHIFT :
+		ptype = MAC_AX_PCFG_TBTT_SHIFT;
+		break;
+	case PCFG_HIQ_WIN :
+		ptype = MAC_AX_PCFG_HIQ_WIN;
+		break;
+	case PCFG_HIQ_DTIM :
+		ptype = MAC_AX_PCFG_HIQ_DTIM;
+		break;
+	case PCFG_BCN_INTERVAL :
+		ptype = MAC_AX_PCFG_BCN_INTV;
+		break;
+	case PCFG_BSS_CLR :
+		ptype = MAC_AX_PCFG_BSS_CLR;
+		break;
+	case PCFG_BCN_EN :
+		ptype = MAC_AX_PCFG_TX_SW;
+		break;
+	case PCFG_MBSSID_EN :
+		ptype = MAC_AX_PCFG_MBSSID_EN;
+		break;
+	case PCFG_BCN_DRP_ALL :
+		ptype = MAC_AX_PCFG_BCN_DRP_ALL;
+		break;
+	default :
+		PHL_ERR("Unknown port cfg type %d\n", type);
+		goto _exit;
+	}
+
+	ppara.band = rlink->hw_band;
+	ppara.port = rlink->hw_port;
+	#ifdef RTW_PHL_BCN/*#ifdef CONFIG_AP*/
+	ppara.mbssid_idx = rlink->hw_mbssid;
+	#endif
+	ppara.val = *(u32 *)param;
+	_hal_dump_port_cfg_info(ptype, &ppara);
+	if (mac->ops->port_cfg(mac, ptype, &ppara) == MACSUCCESS)
+		hsts = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("mac_port_cfg fail\n");
+_exit:
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_port_deinit(struct hal_info_t *hal_i,
+				struct rtw_wifi_role_link_t *rlink)
+{
+	u32 func_en = false;
+
+	return rtw_hal_mac_port_cfg(hal_i, rlink, PCFG_FUNC_SW, &func_en);
+}
+
+static enum mac_ax_wifi_role
+_rtype_to_mac_wifirole(struct rtw_wifi_role_t *wifi_role)
+{
+	enum mac_ax_wifi_role mac_wifi_role = MAC_AX_WIFI_ROLE_NONE;
+
+	switch (wifi_role->type) {
+	case PHL_RTYPE_NONE:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_NONE;
+		break;
+	case PHL_RTYPE_STATION:
+	case PHL_RTYPE_TDLS:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_STATION;
+		break;
+	case PHL_RTYPE_AP:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_AP;
+		break;
+	case PHL_RTYPE_VAP:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_VAP;
+		break;
+	case PHL_RTYPE_ADHOC:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_ADHOC;
+		break;
+	case PHL_RTYPE_ADHOC_MASTER:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_ADHOC_MASTER;
+		break;
+	case PHL_RTYPE_MESH:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_MESH;
+		break;
+	case PHL_RTYPE_MONITOR:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_MONITOR;
+		break;
+	case PHL_RTYPE_P2P_DEVICE:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_P2P_DEVICE;
+		break;
+	case PHL_RTYPE_P2P_GC:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_P2P_GC;
+		break;
+	case PHL_RTYPE_P2P_GO:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_P2P_GO;
+		break;
+	case PHL_RTYPE_NAN:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_NAN;
+		break;
+	default:
+		mac_wifi_role = MAC_AX_WIFI_ROLE_STATION;
+		break;
+	}
+
+	return mac_wifi_role;
+}
+
+static enum mac_ax_upd_mode
+	_hal_updmode_to_mac_upt_mode(enum phl_upd_mode mode)
+{
+	enum mac_ax_upd_mode upd_mode = MAC_AX_ROLE_INFO_CHANGE;
+
+	switch (mode) {
+	case PHL_UPD_ROLE_CREATE:
+		upd_mode = MAC_AX_ROLE_CREATE;
+		break;
+	case PHL_UPD_ROLE_REMOVE:
+		upd_mode = MAC_AX_ROLE_REMOVE;
+		break;
+	case PHL_UPD_ROLE_TYPE_CHANGE:
+		upd_mode = MAC_AX_ROLE_TYPE_CHANGE;
+		break;
+	case PHL_UPD_ROLE_INFO_CHANGE:
+	case PHL_UPD_STA_INFO_CHANGE:
+		upd_mode = MAC_AX_ROLE_INFO_CHANGE;
+		break;
+	case PHL_UPD_STA_CON_DISCONN:
+		upd_mode = MAC_AX_ROLE_CON_DISCONN;
+		break;
+	case PHL_UPD_ROLE_FW_RESTORE:
+		upd_mode = MAC_AX_ROLE_FW_RESTORE;
+		break;
+	case PHL_UPD_ROLE_BAND_SW:
+		upd_mode = MAC_AX_ROLE_BAND_SW;
+		break;
+	case PHL_UPD_ROLE_MAX:
+	default:
+		PHL_ERR("error upt mode %d\n", mode);
+		break;
+	}
+
+	return upd_mode;
+}
+
+static void _hal_stainfo_to_macrinfo(struct hal_info_t *hal_info,
+			struct rtw_phl_stainfo_t *sta,
+			struct mac_ax_role_info *rinfo,
+			enum phl_upd_mode mode,
+			bool is_connect)
+{
+	struct rtw_wifi_role_t *wifi_role = sta->wrole;
+	struct rtw_wifi_role_link_t *rlink = sta->rlink;
+	u8 *self_mac = rlink->mac_addr;
+	void *drv = hal_to_drvpriv(hal_info);
+	bool is_self = false;
+
+	rinfo->macid = (u8)sta->macid;
+	rinfo->band = rlink->hw_band;
+	rinfo->port = rlink->hw_port;
+	rinfo->wmm = rlink->hw_wmm;
+	rinfo->net_type = _rtype_to_mac_nettype(wifi_role, rlink);
+
+	#ifdef CONFIG_DBCC_SUPPORT
+	if (is_dbcc_sup(hal_info->phl_com))
+		rinfo->dbcc_role = 1;
+	#endif /* CONFIG_DBCC_SUPPORT */
+
+	rinfo->wifi_role = _rtype_to_mac_wifirole(wifi_role);
+	rinfo->bcn_hit_cond = sta->bcn_hit_cond;
+	rinfo->hit_rule = sta->hit_rule;
+	rinfo->tsf_sync = rlink->hw_port;
+	rinfo->aid = sta->aid;
+	rinfo->wapi = sta->wapi;
+	rinfo->sec_ent_mode = sta->sec_mode;
+	rinfo->upd_mode = _hal_updmode_to_mac_upt_mode(mode);
+	rinfo->opmode = (is_connect == true) ? MAC_AX_ROLE_CONNECT : MAC_AX_ROLE_DISCONN;
+	rinfo->dl_bw = sta->chandef.bw;
+
+	rinfo->wol_magic = sta->wol_magic;
+
+	if (rinfo->net_type == MAC_AX_NET_TYPE_AP) {
+		if (_os_mem_cmp(drv, self_mac, sta->mac_addr, MAC_ALEN) == 0)
+			is_self = true;
+
+		if(is_self == true) {
+			rinfo->self_role = MAC_AX_SELF_ROLE_AP;
+		} else {
+			rinfo->self_role = MAC_AX_SELF_ROLE_AP_CLIENT;
+			/* for ap client disconnect case,
+			need to set no-link only for MAC_AX_ROLE_CON_DISCONN mode */
+			if (is_connect == false && rinfo->upd_mode == MAC_AX_ROLE_CON_DISCONN)
+				rinfo->net_type = MAC_AX_NET_TYPE_NO_LINK;
+			/* only for client under AX SoftAP mode  */
+			if (sta->wmode & WLAN_MD_11AX)
+				rinfo->tf_mac_padding = sta->asoc_cap.trig_padding;
+		}
+	} else if (rinfo->net_type == MAC_AX_NET_TYPE_INFRA || rinfo->net_type == MAC_AX_NET_TYPE_NO_LINK) {
+		rinfo->self_role = MAC_AX_SELF_ROLE_CLIENT;
+	}
+
+	if ((sta->wmode & WLAN_MD_11AX) && (rlink->mstate == MLME_LINKED))
+	{
+		rinfo->trigger = sta->tf_trs;
+		rinfo->bss_color = sta->asoc_cap.bsscolor;
+		rinfo->is_hesta = 1;
+	}
+
+	if (sta->is_nontx) {
+		rinfo->addr_mask = MAC_AX_BYTE5 & MAC_AX_MSK_NONE;
+		rinfo->mask_sel = MAC_AX_BSSID_MSK;
+	}
+	//TODO
+	switch (rinfo->net_type) {
+	case MAC_AX_NET_TYPE_NO_LINK :
+		_os_mem_cpy(drv, rinfo->self_mac, self_mac, MAC_ALEN);
+		break;
+	case MAC_AX_NET_TYPE_ADHOC :
+		_os_mem_cpy(drv, rinfo->self_mac, self_mac, MAC_ALEN);
+		break;
+	case MAC_AX_NET_TYPE_INFRA :
+		rinfo->aid = sta->aid;
+		_os_mem_cpy(drv, rinfo->self_mac, self_mac, MAC_ALEN);
+		_os_mem_cpy(drv, rinfo->target_mac, sta->mac_addr, MAC_ALEN);
+		_os_mem_cpy(drv, rinfo->bssid, sta->mac_addr, MAC_ALEN);
+		break;
+	case MAC_AX_NET_TYPE_AP :
+		_os_mem_cpy(drv, rinfo->self_mac, self_mac, MAC_ALEN);
+		_os_mem_cpy(drv, rinfo->target_mac, sta->mac_addr, MAC_ALEN);
+		_os_mem_cpy(drv, rinfo->bssid, self_mac, MAC_ALEN);
+		break;
+	default :
+		break;
+	}
+}
+
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_add_entry(struct hal_info_t *hal_info,
+					struct rtw_phl_stainfo_t *sta)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_role_info mac_rinfo = {0};
+	u32 rst = 0;
+
+	_hal_stainfo_to_macrinfo(hal_info, sta, &mac_rinfo, PHL_UPD_ROLE_CREATE, false);
+
+	rst = mac->ops->add_role(mac, &mac_rinfo);
+
+	if ((rst == MACSUCCESS) || (rst == MACSAMACID))
+		return RTW_HAL_STATUS_SUCCESS;
+
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_change_entry(struct hal_info_t *hal_info,
+					struct rtw_phl_stainfo_t *sta,
+					enum phl_upd_mode mode,
+					bool is_connect)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_role_info mac_rinfo = {0};
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	_hal_stainfo_to_macrinfo(hal_info, sta, &mac_rinfo, mode, is_connect);
+
+	if (mac->ops->change_role(mac, &mac_rinfo) == MACSUCCESS) {
+		hsts = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s: mac_change_role fail\n",
+			__FUNCTION__);
+	}
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_set_aid(struct hal_info_t *hal_info,
+			     struct rtw_phl_stainfo_t *sta,
+			     u16 aid)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_role_info mac_rinfo = {0};
+
+	_hal_stainfo_to_macrinfo(hal_info, sta, &mac_rinfo, PHL_UPD_STA_INFO_CHANGE, true);
+
+	mac_rinfo.aid = aid;
+
+	if (mac->ops->change_role(mac, &mac_rinfo) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_del_entry(struct hal_info_t *hal_info,
+					struct rtw_phl_stainfo_t *sta)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->remove_role(mac, (u8)sta->macid) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_add_key(struct hal_info_t *hal_info, u8 macid, u8 type, u8 ext_key,
+					u8 spp,	u8 keyid, u8 keytype, u8 *keybuf)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	void *drv = hal_to_drvpriv(hal_info);
+	struct mac_ax_sec_cam_info sec_cam = {0};
+	u32 mac_err;
+
+	sec_cam.type = type;
+	sec_cam.ext_key = ext_key;
+	sec_cam.spp_mode = spp;
+	sec_cam.len = 20;
+	sec_cam.offset = 0;
+	_os_mem_set(drv, &sec_cam.key, 0, sizeof(sec_cam.key));
+
+	switch (type)
+	{
+		case RTW_ENC_WEP40:
+			_os_mem_cpy(drv, &sec_cam.key, keybuf, 5);
+			break;
+		case RTW_ENC_WEP104:
+			_os_mem_cpy(drv, &sec_cam.key, keybuf, 13);
+			break;
+		case RTW_ENC_TKIP:
+		case RTW_ENC_CCMP:
+		case RTW_ENC_CCMP256:
+		case RTW_ENC_GCMP:
+		case RTW_ENC_GCMP256:
+		case RTW_ENC_BIP_CCMP128:
+		case RTW_ENC_WAPI:
+		case RTW_ENC_GCMSMS4:
+			_os_mem_cpy(drv, &sec_cam.key, keybuf, 16);
+			break;
+		default:
+			break;
+	}
+
+	mac_err = mac->ops->sta_add_key(mac, &sec_cam, macid, keyid, keytype);
+
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_delete_key(struct hal_info_t *hal_info, u8 macid, u8 type,
+						u8 ext_key, u8 spp, u8 keyid, u8 keytype)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_err;
+
+	mac_err = mac->ops->sta_del_key(mac, macid, keyid, keytype);
+
+	if (mac_err != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, mac_err == MACKEYNOTEXT ? _PHL_INFO_ : _PHL_ERR_
+			, "%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_fw_ul_fixinfo(void *hal,
+			      struct rtw_phl_ax_ul_fixinfo *tbl)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (MACSUCCESS == mac->ops->upd_ul_fixinfo(mac, tbl))
+		return RTW_HAL_STATUS_SUCCESS;
+
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+u32
+rtw_hal_mac_search_key_idx(struct hal_info_t *hal_info, u8 macid,
+						u8 keyid, u8 keytype)
+{
+	u32 sec_cam_idx;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	sec_cam_idx = mac->ops->sta_search_key_idx(mac, macid, keyid, keytype);
+
+	return sec_cam_idx;
+}
+
+u32
+rtw_hal_mac_ser_reset_wdt_intr(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	u8 dummyval = 0;
+	u32 result = 0xffffffff;
+
+	result = ops->set_hw_value(mac, MAC_AX_HW_SET_WDT_ISR_RST, &dummyval);
+
+	return result;
+}
+
+bool rtw_hal_mac_ser_chk_ser_l1(struct hal_info_t *hal_info)
+{
+	u32 mac_err = 0;
+	u8 status = 0;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac_err = mac->ops->chk_err_status(mac, &status);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return false;
+	}
+
+	if (status == MAC_AX_L1_TRUE)
+		return true;
+
+	return false;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_ser_get_error_status(struct hal_info_t *hal_info, u32 *err)
+{
+	u32 mac_err = 0;
+	enum mac_ax_err_info err_info = 0;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac_err = mac->ops->get_err_status(mac, &err_info);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	*err = err_info;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_ser_set_error_status(struct hal_info_t *hal_info, enum RTW_PHL_SER_RCVY_STEP err)
+{
+	u32 mac_err = 0;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 err_info = 0;
+
+	if (err == RTW_PHL_SER_L1_RESET_START_DMAC) {
+		err_info = MAC_AX_ERR_L1_RESET_START_DMAC;
+	} else if (err == RTW_PHL_SER_L1_DISABLE_EN) {
+		err_info = MAC_AX_ERR_L1_DISABLE_EN;
+	} else if (err == RTW_PHL_SER_L1_RCVY_EN) {
+		err_info = MAC_AX_ERR_L1_RCVY_EN;
+	} else if (err == RTW_PHL_SER_L0_CFG_NOTIFY) {
+		err_info = MAC_AX_ERR_L0_CFG_NOTIFY;
+	} else if (err == RTW_PHL_SER_L0_CFG_DIS_NOTIFY) {
+		err_info = MAC_AX_ERR_L0_CFG_DIS_NOTIFY;
+	} else if (err == RTW_PHL_SER_L0_CFG_HANDSHAKE) {
+		err_info = MAC_AX_ERR_L0_CFG_HANDSHAKE;
+	} else if (err == RTW_PHL_SER_L0_RCVY_EN) {
+		err_info = MAC_AX_ERR_L0_RCVY_EN;
+	}
+	PHL_INFO("%s : error info to mac 0x%x.\n", __func__, err_info);
+
+	mac_err = mac->ops->set_err_status(mac, err_info);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_trigger_cmac_err(struct hal_info_t *hal_info)
+{
+	u32 mac_err = 0;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac_err = mac->ops->trigger_cmac_err(mac);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_trigger_dmac_err(struct hal_info_t *hal_info)
+{
+	u32 mac_err = 0;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac_err = mac->ops->trigger_dmac_err(mac);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_lv1_rcvy(struct hal_info_t *hal_info, enum rtw_phl_ser_lv1_recv_step step)
+{
+	u32 mac_err = 0;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac_err = mac->ops->lv1_rcvy(mac, (enum mac_ax_lv1_rcvy_step)step);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_ser_ctrl(struct hal_info_t *hal_info,
+					enum rtw_hal_ser_rsn rsn, bool en)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_err = 0;
+	enum mac_ax_func_sw cfg;
+	u32 start_t = 0;
+
+	start_t = _os_get_cur_time_us();
+
+	if (rsn == HAL_SER_RSN_WOW) {
+		cfg = (en == true) ? MAC_AX_FUNC_EN : MAC_AX_FUNC_DIS;
+	} else if (rsn == HAL_SER_RSN_RFK) {
+		cfg = (en == true) ?
+			MAC_AX_FUNC_WO_RECVY_EN : MAC_AX_FUNC_WO_RECVY_DIS;
+	} else {
+		PHL_ERR("%s: Unknown rsn(%d)\n", __func__, rsn);
+		goto exit;
+	}
+	PHL_INFO("%s: rsn(%d), en(%d), cfg(%d)\n", __func__, rsn, en, cfg);
+	mac_err = mac->ops->ser_ctrl(mac, cfg);
+
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		goto exit;
+	}
+	hsts = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("%s : %s ser success with %d us.\n", __func__,
+			(en ? "start" : "stop"), phl_get_passing_time_us(start_t));
+exit:
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_poll_hw_tx_done(struct hal_info_t *hal_info)
+{
+	PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "poll hw tx done is not supported now!\n");
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_hw_tx_resume(struct hal_info_t *hal_info)
+{
+	PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "resume hw tx is not supported now!\n");
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_poll_hw_rx_done(struct hal_info_t *hal_info)
+{
+	PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "poll hw rx done is not supported now!\n");
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_hw_rx_resume(struct hal_info_t *hal_info)
+{
+	PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "resume hw rx is not supported now!\n");
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+#define IS_ASCII(c) (((unsigned char)(c)) <= 0x7f)
+#define FW_PLE_SIZE 0x800
+#define FW_PLE_SEGMENT_SIZE 512 /*Because PHL_PRINT have prefix 5 bytes "PHL: "*/
+
+void
+_hal_fw_dbg_dump(struct hal_info_t *hal_info, u8 *buffer, u16 bufsize)
+{
+	char str[FW_PLE_SEGMENT_SIZE + 1] = {0};
+	u16 i = 0, ofst = 0;
+
+	for (ofst = 0; ofst < bufsize; ofst += FW_PLE_SEGMENT_SIZE) {
+ 		for (i = 0; i < FW_PLE_SEGMENT_SIZE; i++) {
+			if (buffer[i + ofst] == 0 || !IS_ASCII(buffer[i + ofst]))
+				break;
+			_os_snprintf(str + i, 2, "%c", buffer[i + ofst]);
+		}
+		PHL_PRINT("%s\n", str);
+		_os_mem_set(hal_to_drvpriv(hal_info), str, 0, sizeof(str));
+	}
+}
+
+enum rtw_hal_status
+rtw_hal_mac_dump_fw_rsvd_ple(struct hal_info_t *hal_info)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u8 *buffer = NULL;
+	u16 bufSize = FW_PLE_SIZE;
+
+	if (!IS_IO_DUMP_ALLOWED(hal_info->phl_com)) {
+		PHL_DBG("%s(): skip due to IS_IO_DUMP_ALLOWED(%d)\n",
+				  __func__,
+				  IS_IO_DUMP_ALLOWED(hal_info->phl_com));
+		return RTW_HAL_STATUS_SUCCESS;
+	}
+	if(mac->ops->dump_fw_rsvd_ple(mac, &buffer) != 0) {
+		PHL_ERR("%s fail!\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (buffer != NULL) {
+		PHL_PRINT("=======================\n");
+		PHL_PRINT("Start to dump fw rsvd ple :\n\n");
+		_hal_fw_dbg_dump(hal_info, buffer, bufSize);
+		PHL_PRINT("\n=======================\n");
+		_os_mem_free(hal_info->hal_com->drv_priv, buffer, bufSize);
+	} else {
+		PHL_ERR("dump_fw_rsvd_ple return invalid buffer!\n");
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
+
+
+/*
+ * halmac wrapper API for hal and proto type is at hal_api_mac.h
+ * init HW scope or start HW scope?
+ */
+enum rtw_hal_status
+rtw_hal_mac_init_mac(void *mac,struct hal_init_info_t *init_info)
+{
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct mac_ax_ops *hal_mac_ops = mac_info->ops;
+	struct mac_ax_trx_info trx_info = {0};
+
+	trx_info.trx_mode = init_info->trx_info.trx_mode;
+	trx_info.qta_mode = init_info->trx_info.qta_mode;
+
+	if (hal_mac_ops->sys_init(mac_info))
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	if (hal_mac_ops->trx_init(mac_info, &trx_info))
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_trx_init(void *mac, struct hal_init_info_t *init_info)
+{
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct mac_ax_ops *hal_mac_ops = mac_info->ops;
+	struct mac_ax_trx_info trx_info = {0};
+
+	trx_info.trx_mode = init_info->trx_info.trx_mode;
+	trx_info.qta_mode = init_info->trx_info.qta_mode;
+
+	if (hal_mac_ops->trx_init(mac_info, &trx_info))
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_loopback_set_trx_mode(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_trx_info trx_info;
+	struct mac_ax_host_rpr_cfg *rpr_cfg = (struct mac_ax_host_rpr_cfg *)hal_info->rpr_cfg;
+
+	rpr_cfg->tmr_def = 1;
+	rpr_cfg->agg_def = 1;
+	rpr_cfg->txok_en = MAC_AX_FUNC_DEF;
+	rpr_cfg->rty_lmt_en = MAC_AX_FUNC_DEF;
+	rpr_cfg->lft_drop_en = MAC_AX_FUNC_DEF;
+	rpr_cfg->macid_drop_en = MAC_AX_FUNC_DEF;
+	trx_info.rpr_cfg = rpr_cfg;
+
+	trx_info.trx_mode = MAC_AX_TRX_LOOPBACK;
+	trx_info.qta_mode = MAC_AX_QTA_SCC;
+
+	if (hal_mac_ops->trx_init(mac, &trx_info))
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_set_rxfltr_loopbk_mode(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_rx_fltr_ctrl_t ctrl = {0};
+	struct mac_ax_rx_fltr_ctrl_t mask = {0};
+	u32 err;
+
+	ctrl.sniffer_mode = 1;
+	ctrl.acpt_crc32_err_pkt = 1;
+	ctrl.acpt_ftm_req_pkt = 0;
+	/* unit of pkt_len_fltr is 512 bytes */
+	ctrl.pkt_len_fltr = 0x1c00 >> 9;
+	ctrl.pkt_len_fltr += (0x1c00 & 0x7F) ? 1 : 0;
+
+	mask.acpt_crc32_err_pkt = 1;
+	mask.acpt_ftm_req_pkt = 1;
+	mask.pkt_len_fltr = 0x3F;
+
+
+	err = mac->ops->set_rx_fltr_opt(mac, &ctrl, &mask, MAC_AX_BAND_0);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_fw_type
+rtw_hal_mac_get_fw_type(struct rtw_phl_com_t *phl_com)
+{
+	enum rtw_fw_type fw_type = phl_com->dev_cap.fw_cap.fw_type;
+	if (fw_type != RTW_FW_MAX) {
+		PHL_INFO("%s : Override phl fw_type to %d.\n", __func__, fw_type);
+		return fw_type;
+	}
+
+#ifdef PHL_FEATURE_NIC
+#if defined(MAC_FW_CATEGORY_NIC) || defined(MAC_FW_CATEGORY_NIC_BPLUS)
+	return RTW_FW_NIC;
+#else
+	return RTW_FW_NIC_CE;
+#endif /*MAC_FW_CATEGORY_NIC || MAC_FW_CATEGORY_NIC_BPLUS*/
+#elif defined(PHL_FEATURE_AP)
+	return RTW_FW_AP;
+#else
+	return RTW_FW_MAX;
+#endif
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_resp_ack_chk_cca(struct hal_info_t *hal_info, u8 band, u8 en)
+{
+	struct mac_ax_resp_chk_cca resp_chk = {0};
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_status = 0;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	resp_chk.band = band;
+	resp_chk.resp_ack_chk_cca_en = en;
+	mac_status = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_RESP_ACK_CHK_CCA, &resp_chk);
+
+	if (mac_status != MACSUCCESS) {
+		PHL_ERR("[rtw_hal_mac_set_resp_ack_chk_cca] : band(%d), en(%d), status(%d)\n",
+			mac_status,
+			band,
+			en);
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return hstatus;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_sifs_chk_cca_en(struct hal_info_t *hal_info, u8 band, u8 en)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_status = 0;
+
+	mac_status = mac->ops->sifs_chk_cca_en(mac, band, en);
+	if (mac_status != MACSUCCESS) {
+		PHL_ERR("sifs_chk_cca_en, band(%d), en(%d), status(%d)\n",
+			band,
+			en,
+			mac_status);
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_hal_init(struct rtw_phl_com_t *phl_com,
+		     struct hal_info_t *hal_info,
+		     struct hal_init_info_t *init_info)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_fw_info_t *fw_info = &phl_com->fw_info;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+	struct mac_ax_fwdl_info fwdl_info = {0};
+	u32 mac_status = 0;
+	enum rtw_fw_type wcpu_fw_type = rtw_hal_mac_get_fw_type(phl_com);
+#if defined(CONFIG_PHL_CSUM_OFFLOAD_RX) || defined(CONFIG_PHL_CSUM_OFFLOAD_TX)
+	u8 tx_chksum_offload = 0;
+	u8 rx_chksum_offload = 0;
+#endif
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	FUNCIN_WSTS(hstatus);
+
+	hstatus = hal_ops->hal_cfg_fw(phl_com, hal_info, init_info->ic_name, wcpu_fw_type);
+	if(RTW_HAL_STATUS_SUCCESS != hstatus) {
+		PHL_ERR("%s : Cfg FW Failed: %d!\n", __func__, hstatus);
+		return hstatus;
+	}
+	/* fwdl_info */
+	fwdl_info.fw_en = fw_info->fw_en;
+	fwdl_info.dlram_en = fw_info->dlram_en;
+	fwdl_info.dlrom_en = fw_info->dlrom_en;
+	fwdl_info.ram_buff = fw_info->ram_buff;
+	fwdl_info.ram_size = fw_info->ram_size;
+	fwdl_info.rom_buff = fw_info->rom_buff;
+	fwdl_info.rom_size = fw_info->rom_size;
+	fwdl_info.fw_cat = fw_info->fw_type;
+
+	if(fw_info->fw_src == RTW_FW_SRC_EXTNAL)
+		fwdl_info.fw_from_hdr = 0;
+	else
+		fwdl_info.fw_from_hdr = 1;
+
+	mac_status = mac->ops->hal_init(mac, &init_info->trx_info, &fwdl_info, &init_info->intf_info);
+
+	if (mac_status == MACSUCCESS) {
+		hstatus = RTW_HAL_STATUS_SUCCESS;
+		hal_mac_print_fw_version(hal_info);
+		PHL_INFO("%s : Get aid after fw download\n", __func__);
+		hal_mac_get_hwinfo(hal_info, &(phl_com->hal_spec));
+
+	} else {
+		hstatus = RTW_HAL_STATUS_MAC_INIT_FAILURE;
+		PHL_ERR("%s : mac_status %d!\n", __func__, mac_status);
+	}
+
+	if (rtw_hal_mac_get_wcpu_cap(phl_com, hal_info) !=
+	    RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s : can't get fw capability.\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+#ifdef CONFIG_PHL_CSUM_OFFLOAD_RX
+	rx_chksum_offload = 1;
+#endif
+#ifdef CONFIG_PHL_CSUM_OFFLOAD_TX
+	tx_chksum_offload = 1;
+#endif
+#if defined(CONFIG_PHL_CSUM_OFFLOAD_RX) || defined(CONFIG_PHL_CSUM_OFFLOAD_TX)
+	mac_status = mac->ops->tcpip_chksum_ofd(mac, tx_chksum_offload, rx_chksum_offload);
+	if (mac_status != MACSUCCESS)
+		PHL_ERR("%s : tcpip_chksum_ofd mac_status %d!!!!!!!\n", __func__, mac_status);
+#endif
+
+	FUNCOUT_WSTS(hstatus);
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_MAC_HAL_INIT);
+#endif /* DBG_MONITOR_TIME */
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_pwr_switch(struct hal_info_t *hal_info, bool on)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	if (hal_mac_ops->pwr_switch(mac, on) != MACSUCCESS) {
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_hal_fast_init(struct rtw_phl_com_t *phl_com,
+			  struct hal_info_t *hal_info,
+			  struct hal_init_info_t *init_info)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_fw_info_t *fw_info = &phl_com->fw_info;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+	struct mac_ax_fwdl_info fwdl_info = {0};
+	u32 mac_status = 0;
+	enum rtw_fw_type wcpu_fw_type = rtw_hal_mac_get_fw_type(phl_com);
+
+	FUNCIN_WSTS(hstatus);
+
+	hstatus = hal_ops->hal_cfg_fw(phl_com, hal_info, init_info->ic_name, wcpu_fw_type);
+	if(RTW_HAL_STATUS_SUCCESS != hstatus) {
+		PHL_ERR("%s : Cfg FW Failed: %d!\n", __func__, hstatus);
+		return hstatus;
+	}
+
+	/* fwdl_info */
+	fwdl_info.fw_en = fw_info->fw_en;
+	fwdl_info.dlram_en = fw_info->dlram_en;
+	fwdl_info.dlrom_en = fw_info->dlrom_en;
+	fwdl_info.ram_buff = fw_info->ram_buff;
+	fwdl_info.ram_size = fw_info->ram_size;
+	fwdl_info.rom_buff = fw_info->rom_buff;
+	fwdl_info.rom_size = fw_info->rom_size;
+	fwdl_info.fw_cat = fw_info->fw_type;
+
+	if(fw_info->fw_src == RTW_FW_SRC_EXTNAL)
+		fwdl_info.fw_from_hdr = 0;
+	else
+		fwdl_info.fw_from_hdr = 1;
+
+	mac_status = mac->ops->hal_fast_init(mac, &init_info->trx_info, &fwdl_info, &init_info->intf_info);
+	if (mac_status != MACSUCCESS) {
+		PHL_ERR("%s : mac_status %d!\n", __func__, mac_status);
+		return RTW_HAL_STATUS_HAL_INIT_FAILURE;
+	}
+
+	if (rtw_hal_mac_get_wcpu_cap(phl_com, hal_info) !=
+	    RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s : can't get fw capability.\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hal_info->hal_com->cv = mac->hw_info->cv;
+	hstatus = RTW_HAL_STATUS_SUCCESS;
+	FUNCOUT_WSTS(hstatus);
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_mac_hal_fast_deinit(struct rtw_phl_com_t *phl_com,
+						struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_status = 0;
+
+	mac_status = mac->ops->hal_fast_deinit(mac);
+	if (mac_status != MACSUCCESS) {
+		PHL_ERR("%s : mac_status %d!\n", __func__, mac_status);
+		return RTW_HAL_STATUS_HAL_INIT_FAILURE;
+	}
+
+	hstatus = RTW_HAL_STATUS_SUCCESS;
+	FUNCOUT_WSTS(hstatus);
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_hal_deinit(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_status = 0;
+
+	mac_status = mac->ops->hal_deinit(mac);
+
+	if (mac_status == MACSUCCESS)
+		hstatus = RTW_HAL_STATUS_SUCCESS;
+	else
+		PHL_ERR("%s : mac_status %d!\n", __func__, mac_status);
+
+	FUNCOUT_WSTS(hstatus);
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_chk_allq_empty(struct hal_info_t *hal_info, u8 *empty)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+
+	if (hal_mac_ops->chk_allq_empty(mac, empty))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+static void _dump_txq_status(struct mac_ax_tx_queue_empty *val)
+{
+	u8 i = 0;
+	u8 size = ARRAY_SIZE(val->macid_txq_empty);
+
+	for (i = 0; i < size; i++)
+		PHL_INFO("%s : macid_txq_empty[%d] %d.\n", __func__, i, val->macid_txq_empty[i]);
+
+	PHL_INFO("%s : band0_mgnt_empty %d.\n", __func__, val->band0_mgnt_empty);
+	PHL_INFO("%s : band1_mgnt_empty %d.\n", __func__, val->band1_mgnt_empty);
+	PHL_INFO("%s : fw_txq_empty %d.\n", __func__, val->fw_txq_empty);
+	PHL_INFO("%s : h2c_empty %d.\n", __func__, val->h2c_empty);
+	PHL_INFO("%s : others_empty %d.\n", __func__, val->others_empty);
+}
+
+#ifdef CONFIG_WOWLAN
+enum rtw_hal_status
+rtw_hal_mac_cfg_wow_sleep(struct hal_info_t *hal_info, u8 sleep)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+
+	if (hal_mac_ops->cfg_wow_sleep(mac,sleep))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_wow_fw_status(struct hal_info_t *hal_info, u8 *status, u8 func_en)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+
+	if (hal_mac_ops->get_wow_fw_status(mac,status, func_en))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_keep_alive(struct hal_info_t *hal_info, u16 macid, u8 en,
+								struct rtw_keep_alive_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_keep_alive_info info = {0};
+
+	if (en && cfg == NULL)
+		return RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &info, 0, sizeof(info));
+
+	if (en) {
+		info.keepalive_en = cfg->keep_alive_en;
+		info.period = cfg->keep_alive_period;
+		info.packet_id = cfg->keep_alive_pkt_id;
+	}
+
+	if (hal_mac_ops->cfg_keepalive(mac, (u8)macid, &info))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_disc_dec(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_disc_det_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_disconnect_det_info info = {0};
+
+	if (en && cfg == NULL)
+		return RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &info, 0, sizeof(info));
+
+	if (en) {
+		info.disconnect_detect_en = cfg->disc_det_en;
+		info.disconnect_en = (cfg->disc_wake_en == 1) ? 0 : 1;
+		info.check_period = cfg->check_period;
+		info.try_pkt_count = cfg->try_pkt_count;
+		info.tryok_bcnfail_count_en = cfg->cnt_bcn_lost_en;
+		info.tryok_bcnfail_count_limit = cfg->cnt_bcn_lost_limit;
+	}
+
+	if (hal_mac_ops->cfg_disconnect_det(mac, (u8)macid, &info))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_dev2hst_gpio(struct hal_info_t *hal_info, u8 en, struct rtw_wow_gpio_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct rtw_dev2hst_gpio_info info = {0};
+
+	if (en) {
+		if (!cfg)
+			return RTW_HAL_STATUS_FAILURE;
+
+		_os_mem_cpy(hal_to_drvpriv(hal_info), &info,
+			    &cfg->d2h_gpio_info,
+			    sizeof(struct rtw_dev2hst_gpio_info));
+	}
+
+	if (hal_mac_ops->cfg_dev2hst_gpio(mac, &info))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_wow_wake(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_wow_wake_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_wow_wake_info info = {0};
+	struct mac_ax_remotectrl_info_parm_ *content = NULL;
+	struct mac_ax_remotectrl_info_parm_ param = {0};
+
+	if (en && cfg == NULL)
+		return RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &info, 0, sizeof(info));
+
+	if (en) {
+		info.wow_en = cfg->wow_en;
+		info.drop_all_pkt = cfg->drop_all_pkt;
+		info.rx_parse_after_wake = cfg->rx_parse_after_wake;
+		info.pairwise_sec_algo = cfg->pairwise_sec_algo;
+		info.group_sec_algo = cfg->group_sec_algo;
+		/*
+		info.bip_sec_algo = cfg->bip_sec_algo;
+		*/
+		info.pattern_match_en = cfg->pattern_match_en;
+		info.magic_en = cfg->magic_pkt_en;
+		info.hw_unicast_en = cfg->hw_unicast_en;
+		info.fw_unicast_en = cfg->fw_unicast_en;
+		info.deauth_wakeup = cfg->deauth_wakeup;
+		info.rekey_wakeup = cfg->rekey_wakeup;
+		info.eap_wakeup = cfg->eap_wakeup;
+		info.all_data_wakeup = cfg->all_data_wakeup;
+
+		if (cfg->pairwise_sec_algo) {
+			param.validcheck = cfg->remote_wake_ctrl_info.valid_check;
+			param.symbolchecken = cfg->remote_wake_ctrl_info.symbol_check_en;
+			param.lastkeyid = cfg->remote_wake_ctrl_info.gtk_key_idx;
+			_os_mem_cpy(hal_to_drvpriv(hal_info), param.ptktxiv, cfg->remote_wake_ctrl_info.ptk_tx_iv, IV_LENGTH);
+			_os_mem_cpy(hal_to_drvpriv(hal_info), param.rxptkiv, cfg->remote_wake_ctrl_info.ptk_rx_iv, IV_LENGTH);
+			_os_mem_cpy(hal_to_drvpriv(hal_info), param.rxgtkiv_0, cfg->remote_wake_ctrl_info.gtk_rx_iv_idx0, IV_LENGTH);
+			_os_mem_cpy(hal_to_drvpriv(hal_info), param.rxgtkiv_1, cfg->remote_wake_ctrl_info.gtk_rx_iv_idx1, IV_LENGTH);
+			_os_mem_cpy(hal_to_drvpriv(hal_info), param.rxgtkiv_2, cfg->remote_wake_ctrl_info.gtk_rx_iv_idx2, IV_LENGTH);
+			_os_mem_cpy(hal_to_drvpriv(hal_info), param.rxgtkiv_3, cfg->remote_wake_ctrl_info.gtk_rx_iv_idx3, IV_LENGTH);
+			content = &param;
+		}
+	}
+
+	/* should pass cfg->remote_wake_ctrl_info to halmac */
+	if (hal_mac_ops->cfg_wow_wake(mac, (u8)macid, &info, content))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_wake_rsn(struct hal_info_t *hal_info, u8 *wake_rsn, u8 *reset)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+
+	if (hal_mac_ops->get_wow_wake_rsn(mac, wake_rsn, reset) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_ndp_ofld(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_ndp_ofld_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_ndp_ofld_info info = {0};
+	struct mac_ax_ndp_info_parm_ content[2] = {0};
+	void* drv_priv = hal_to_drvpriv(hal_info);
+	u8 idx = 0;
+
+	if (en && cfg == NULL)
+		return RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_set(drv_priv, &info, 0, sizeof(struct mac_ax_ndp_ofld_info));
+	_os_mem_set(drv_priv, content, 0, sizeof(struct mac_ax_ndp_info_parm_)*2);
+
+	if (en) {
+		info.ndp_en = cfg->ndp_en;
+		info.na_id = cfg->ndp_id;
+		for (idx = 0; idx < 2; idx++) {
+			content[idx].enable = cfg->ndp_ofld_content[idx].ndp_en;
+			_os_mem_cpy(drv_priv,content[idx].targetlinkaddress,
+				cfg->ndp_ofld_content[idx].mac_addr, MAC_ADDRESS_LENGTH);
+			content[idx].checkremoveip = cfg->ndp_ofld_content[idx].chk_remote_ip;
+			_os_mem_cpy(drv_priv,content[idx].remoteipv6address,
+				cfg->ndp_ofld_content[idx].remote_ipv6_addr, IPV6_ADDRESS_LENGTH);
+			content[idx].numberoftargetip = cfg->ndp_ofld_content[0].num_target_ip;
+			_os_mem_cpy(drv_priv,&(content[idx].targetip[0][0]),
+				&(cfg->ndp_ofld_content[idx].target_ipv6_addr[0][0]), IPV6_ADDRESS_LENGTH*2);
+		}
+	}
+
+	if (hal_mac_ops->cfg_ndp_ofld(mac, (u8)macid, &info, content))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_arp_ofld(struct hal_info_t *hal_info, u16 macid, u8 en,
+								struct rtw_arp_ofld_info *cfg)
+{
+
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_arp_ofld_info info = {0};
+
+	if (en && cfg == NULL)
+		return RTW_HAL_STATUS_FAILURE;
+
+	if (en) {
+		info.arp_en = cfg->arp_en;
+		info.arp_rsp_id = cfg->arp_rsp_id;
+		info.arp_action = cfg->arp_action;
+	}
+
+	if (hal_mac_ops->cfg_arp_ofld(mac, (u8)macid, &info, NULL))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_wow_cam(struct hal_info_t *hal_info, u16 macid, u8 en,
+							struct rtw_pattern_match_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct rtw_wowcam_upd_info *wowcam_info = NULL;
+	struct mac_ax_wowcam_upd_info info = {0};
+	void* drv_priv = hal_to_drvpriv(hal_info);
+	u8 i = 0;
+	u8 j = 0;
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s (en: %u) ==>\n", __func__, en);
+
+	if (en && cfg == NULL) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s() <== RTW_HAL_STATUS_FAILURE for input cfg == NULL\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	/* config each wow cam, no matter valid or not */
+	for (i = 0; i < MAX_WOW_CAM_NUM; ++i) {
+		_os_mem_set(drv_priv, &info, 0, sizeof(struct mac_ax_wowcam_upd_info));
+
+		if (en) {
+			wowcam_info = &(cfg->wowcam_info[i]);
+
+			info.idx = wowcam_info->wow_cam_idx;
+			info.r_w = wowcam_info->rw;
+			info.valid = wowcam_info->valid;
+
+			if (wowcam_info->valid != 0) {
+				info.wkfm1 = wowcam_info->wake_mask[0];
+				info.wkfm2 = wowcam_info->wake_mask[1];
+				info.wkfm3 = wowcam_info->wake_mask[2];
+				info.wkfm4 = wowcam_info->wake_mask[3];
+				info.crc = wowcam_info->match_crc;
+				info.negative_pattern_match = wowcam_info->is_negative_pattern_match;
+				info.skip_mac_hdr = wowcam_info->skip_mac_hdr;
+				info.uc = wowcam_info->uc;
+				info.mc = wowcam_info->mc;
+				info.bc = wowcam_info->bc;
+
+				PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "wow_cam [%u]:\n", info.idx);
+
+				for (j = 0; j < 4; ++j)
+					PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "- wake_mask[%u] = 0x%08x\n", j, wowcam_info->wake_mask[j]);
+				PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-      rw, crc = (%u, 0x%08x)\n", info.r_w, info.crc);
+				PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "- Neg_ptn_mtch = %u\n", info.negative_pattern_match);
+				PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-   SkipMacHdr = %u\n", info.skip_mac_hdr);
+				PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "- (UC, MC, BC) = (%u, %u, %u)\n", info.uc, info.mc, info.bc);
+			} else {
+				continue;
+			}
+		} else {
+			info.idx = i;
+			info.r_w = 1;
+			info.valid = 0;
+		}
+
+		if (hal_mac_ops->cfg_wowcam_upd(mac, &info)) {
+			PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s() <== RTW_HAL_STATUS_FAILURE for cfg wowcam(%u)\n", __func__, info.idx);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+	}
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s() <==\n", __func__);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+static u32 _hal_mac_recv_aoac_report(struct mac_ax_adapter *mac, struct mac_ax_aoac_report *buf, u8 rx_rdy)
+{
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 mac_status = 0;
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s(): request aoac report.\n", __func__);
+
+	mac_status = hal_mac_ops->request_aoac_report(mac, rx_rdy);
+
+	if (mac_status) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s() <== mac_status(%u) from request_aoac_report()\n", __func__, mac_status);
+		return mac_status;
+	}
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s(): read aoac report\n", __func__);
+
+	mac_status = hal_mac_ops->read_aoac_report(mac, buf, rx_rdy);
+
+	if (mac_status) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s() <== get mac_status(%u) from read_aoac_report()\n", __func__, mac_status);
+		return mac_status;
+	}
+
+	return mac_status;
+}
+
+#define KEY_ID_MASK 0x3
+#define KEY_ID_OFFSET 6
+
+enum rtw_hal_status _hal_mac_aoac_rpt_chk(struct rtw_aoac_report *aoac_info)
+{
+	u8 key_id_from_iv = 0;
+	u32 rx_iv = *((u32 *)aoac_info->ptk_rx_iv);
+
+	/* Case I. Aoac report is all zero in phase 0 */
+	if (rx_iv == 0) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s(): ptk_rx_iv is Zero, treating this case as error.\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	/* Case II. KEY_IDX does not match with GTK_RX_IV */
+	key_id_from_iv = (aoac_info->gtk_rx_iv[aoac_info->key_idx][3] >> KEY_ID_OFFSET) & KEY_ID_MASK;
+
+	if (key_id_from_iv != aoac_info->key_idx) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s(): Key_idx(%u) not match with the one(%u) parsed from GTK_RX_IV[%u]\n",
+						__func__, aoac_info->key_idx, key_id_from_iv, aoac_info->key_idx);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/*
+AOAC_RPT - PHASE 0:
+	Halmac will get aoac report through c2h reg.
+
+	(Cuz at this moment, the rx is still blocked by host.
+	The rx cannot be resume only if those IV be updated by aoac_rpt)
+
+	In PHASE 0, Fw will transfer some necessary info,
+	such as RX_IV, GTK_KEY_ID, GTK_RX_IV, Rekey_OK and iGTK_ipn.
+*/
+
+enum rtw_hal_status
+_hal_mac_read_aoac_rpt_phase_0(void* drv_priv, struct mac_ax_aoac_report *aoac_rpt_buf, struct rtw_aoac_report *aoac_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	aoac_info->rekey_ok = aoac_rpt_buf->rekey_ok;
+	aoac_info->key_idx = aoac_rpt_buf->key_idx;
+	_os_mem_cpy(drv_priv, aoac_info->ptk_rx_iv, aoac_rpt_buf->ptk_rx_iv, IV_LENGTH);
+
+	switch (aoac_info->key_idx) {
+	case 0:
+		_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[0], aoac_rpt_buf->gtk_rx_iv_0, IV_LENGTH);
+		break;
+	case 1:
+		_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[1], aoac_rpt_buf->gtk_rx_iv_1, IV_LENGTH);
+		break;
+	case 2:
+		_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[2], aoac_rpt_buf->gtk_rx_iv_2, IV_LENGTH);
+		break;
+	case 3:
+		_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[3], aoac_rpt_buf->gtk_rx_iv_3, IV_LENGTH);
+		break;
+	default:
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s(): Unknown gtk_key_idx(%u)\n", __func__, aoac_info->key_idx);
+		break;
+	}
+
+	_os_mem_cpy(drv_priv, aoac_info->igtk_ipn, aoac_rpt_buf->igtk_ipn, sizeof(aoac_rpt_buf->igtk_ipn));
+
+	hal_status = _hal_mac_aoac_rpt_chk(aoac_info);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "PHASE 0:\n");
+
+	debug_dump_data((u8 *)aoac_rpt_buf, sizeof(struct mac_ax_aoac_report), "aoac_report");
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-     rekey_ok = %u\n", aoac_info->rekey_ok);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-      key_idx = %u\n", aoac_info->key_idx);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-    ptk_rx_iv = 0x%08x%08x\n", *((u32*)(aoac_info->ptk_rx_iv)+1), *((u32*)(aoac_info->ptk_rx_iv)));
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "- gtk_rx_iv[%u] = 0x%08x%08x\n",
+					aoac_info->key_idx,
+					*((u32*)(aoac_info->gtk_rx_iv[aoac_info->key_idx])+1),
+					*((u32*)(aoac_info->gtk_rx_iv[aoac_info->key_idx])));
+
+	if (aoac_info->rekey_ok)
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-     igtk_ipn = 0x%08x%08x\n", *((u32*)(aoac_info->igtk_ipn)+1), *((u32*)(aoac_info->igtk_ipn)));
+
+	return hal_status;
+}
+
+/*
+AOAC_RPT - PHASE 1:
+	Halmac will get aoac report through c2h pkt.
+
+	(Cuz at this moment,
+	the host has resumed the rx, c2h pkt can be used here.)
+
+	In PHASE 1, Fw will transfer all info in aoac report.
+	Those entries got in phase 0 should remain the same value in phase 1.
+*/
+
+enum rtw_hal_status
+_hal_mac_read_aoac_rpt_phase_1(void* drv_priv, struct mac_ax_aoac_report *aoac_rpt_buf, struct rtw_aoac_report *aoac_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	u8 i = 0;
+
+	aoac_info->rpt_ver = aoac_rpt_buf->rpt_ver;
+	aoac_info->sec_type = aoac_rpt_buf->sec_type;
+	aoac_info->pattern_idx = aoac_rpt_buf->pattern_idx;
+
+	_os_mem_cpy(drv_priv, aoac_info->ptk_tx_iv, aoac_rpt_buf->ptk_tx_iv, IV_LENGTH);
+
+	for (i = 0; i < 4; ++i) {
+		switch (i) {
+		case 0:
+			_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[0], aoac_rpt_buf->gtk_rx_iv_0, IV_LENGTH);
+			break;
+		case 1:
+			_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[1], aoac_rpt_buf->gtk_rx_iv_1, IV_LENGTH);
+			break;
+		case 2:
+			_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[2], aoac_rpt_buf->gtk_rx_iv_2, IV_LENGTH);
+			break;
+		case 3:
+			_os_mem_cpy(drv_priv, aoac_info->gtk_rx_iv[3], aoac_rpt_buf->gtk_rx_iv_3, IV_LENGTH);
+			break;
+		default:
+			PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s(): Unknown gtk_key_idx(%u)\n", __func__, aoac_info->key_idx);
+			break;
+		}
+	}
+
+	_os_mem_cpy(drv_priv, aoac_info->gtk, aoac_rpt_buf->gtk, sizeof(aoac_rpt_buf->gtk));
+	_os_mem_cpy(drv_priv, aoac_info->eapol_key_replay_count, aoac_rpt_buf->eapol_key_replay_count, sizeof(aoac_rpt_buf->eapol_key_replay_count));
+	_os_mem_cpy(drv_priv, aoac_info->igtk_key_id, aoac_rpt_buf->igtk_key_id, sizeof(aoac_rpt_buf->igtk_key_id));
+	_os_mem_cpy(drv_priv, aoac_info->igtk_ipn, aoac_rpt_buf->igtk_ipn, sizeof(aoac_rpt_buf->igtk_ipn));
+	_os_mem_cpy(drv_priv, aoac_info->igtk, aoac_rpt_buf->igtk, sizeof(aoac_rpt_buf->igtk));
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "PHASE 1:\n");
+
+	debug_dump_data((u8 *)aoac_rpt_buf, sizeof(struct mac_ax_aoac_report), "aoac_report");
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-             rekey_ok = %u\n", aoac_info->rekey_ok);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-              key_idx = %u\n", aoac_info->key_idx);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-            ptk_rx_iv = 0x%08x%08x\n", *((u32*)(aoac_info->ptk_rx_iv)+1), *((u32*)(aoac_info->ptk_rx_iv)));
+	for(i = 0; i < 4; ++i) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-         gtk_rx_iv[%u] = 0x%08x%08x\n",
+					i,
+					*((u32*)(aoac_info->gtk_rx_iv[i])+1),
+					*((u32*)(aoac_info->gtk_rx_iv[i])));
+	}
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-            ptk_tx_iv = 0x%08x%08x\n", *((u32*)(aoac_info->ptk_tx_iv)+1), *((u32*)(aoac_info->ptk_tx_iv)));
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-              rpt_ver = %u\n", aoac_info->rpt_ver);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-             sec_type = %u\n", aoac_info->sec_type);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-          pattern_idx = %u\n", aoac_info->pattern_idx);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "- eapol_key_replay_cnt = 0x%08x%08x\n", *((u32*)(aoac_info->eapol_key_replay_count)+1), *((u32*)(aoac_info->eapol_key_replay_count)));
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-          igtk_key_id = 0x%08x%08x\n", *((u32*)(aoac_info->igtk_key_id)+1), *((u32*)(aoac_info->igtk_key_id)));
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "-             igtk_ipn = 0x%08x%08x\n", *((u32*)(aoac_info->igtk_ipn)+1), *((u32*)(aoac_info->igtk_ipn)));
+	debug_dump_data(aoac_info->gtk, sizeof(aoac_rpt_buf->gtk), "GTK:");
+	debug_dump_data(aoac_info->igtk, sizeof(aoac_rpt_buf->igtk), "iGTK:");
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_aoac_rpt(struct hal_info_t *hal_info, struct rtw_aoac_report *aoac_info, u8 rx_ready)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+#ifndef RTW_WKARD_WOW_SKIP_AOAC_RPT
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+#endif
+	struct mac_ax_aoac_report aoac_rpt_buf;
+	void* drv_priv = hal_to_drvpriv(hal_info);
+	u32 mac_status = 0;
+
+	if (aoac_info == NULL) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s() <== RTW_HAL_STATUS_FAILURE for input info == NULL\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+#ifdef RTW_WKARD_WOW_SKIP_AOAC_RPT
+	mac_status = MACPROCERR;
+#else
+	mac_status = _hal_mac_recv_aoac_report(mac, &aoac_rpt_buf, rx_ready);
+#endif
+
+	if (mac_status) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s() <== RTW_HAL_STATUS_FAILURE for _hal_mac_recv_aoac_report fail with mac_status(%u).\n",
+					__func__, mac_status);
+		hal_status = RTW_HAL_STATUS_FAILURE;
+	} else {
+		if (rx_ready == 0)
+			hal_status = _hal_mac_read_aoac_rpt_phase_0(drv_priv, &aoac_rpt_buf, aoac_info);
+		else
+			hal_status = _hal_mac_read_aoac_rpt_phase_1(drv_priv, &aoac_rpt_buf, aoac_info);
+	}
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_gtk_ofld(struct hal_info_t *hal_info, u16 macid, u8 en,
+								struct rtw_gtk_ofld_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_gtk_ofld_info info = {0};
+	struct mac_ax_gtk_info_parm_ param = {0};
+	void* drv_priv = hal_to_drvpriv(hal_info);
+
+	_os_mem_set(drv_priv, &info, 0, sizeof(struct mac_ax_gtk_ofld_info));
+	_os_mem_set(drv_priv, &param, 0, sizeof(struct mac_ax_gtk_info_parm_));
+
+	if (en && cfg == NULL) {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_WARNING_, "%s() <== RTW_HAL_STATUS_FAILURE for input cfg == NULL\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (en) {
+		info.gtk_en = cfg->gtk_en;
+		info.tkip_en = cfg->tkip_en;
+		info.ieee80211w_en = cfg->ieee80211w_en;
+		info.pairwise_wakeup = cfg->pairwise_wakeup;
+		info.gtk_rsp_id = cfg->gtk_rsp_id;
+
+		info.algo_akm_suit = cfg->akmtype_byte3;
+
+		if (info.gtk_en) {
+			_os_mem_cpy(drv_priv, param.kck, cfg->gtk_ofld_content.kck, cfg->gtk_ofld_content.kck_len);
+			_os_mem_cpy(drv_priv, param.kek, cfg->gtk_ofld_content.kek, cfg->gtk_ofld_content.kek_len);
+
+			if (info.tkip_en)
+				_os_mem_cpy(drv_priv, param.rxmickey, cfg->gtk_ofld_content.rxmickey, TKIP_MIC_KEY_LENGTH);
+		}
+
+		if (info.ieee80211w_en) {
+			info.bip_sec_algo = cfg->bip_sec_algo;
+			info.pmf_sa_query_id = cfg->sa_query_id;
+
+			_os_mem_cpy(drv_priv, param.igtk_keyid, cfg->gtk_ofld_content.igtk_keyid, cfg->gtk_ofld_content.igtk_len);
+			_os_mem_cpy(drv_priv, param.ipn, cfg->gtk_ofld_content.ipn, IGTK_PKT_NUM_LENGTH);
+			_os_mem_cpy(drv_priv, param.igtk, &(cfg->gtk_ofld_content.igtk[0]), cfg->gtk_ofld_content.igtk_len);
+
+			if (cfg->hw_11w_en == 0)
+				_os_mem_cpy(drv_priv, param.sk, cfg->gtk_ofld_content.psk, cfg->gtk_ofld_content.psk_len);
+		}
+	} else {
+		info.gtk_en = false;
+		info.ieee80211w_en = false;
+	}
+
+	if (hal_mac_ops->cfg_gtk_ofld(mac, (u8)macid, &info, &param))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_realwow(struct hal_info_t *hal_info, u16 macid, u8 en,
+								struct rtw_realwow_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_realwow_info info = {0};
+	struct mac_ax_realwowv2_info_parm_ param = {0};
+	void *drv_priv = hal_to_drvpriv(hal_info);
+
+	_os_mem_set(drv_priv, &info, 0, sizeof(struct mac_ax_realwow_info));
+	_os_mem_set(drv_priv, &param, 0, sizeof(struct mac_ax_realwowv2_info_parm_));
+
+	if (en) {
+		info.realwow_en = cfg->realwow_en;
+		info.auto_wakeup = cfg->auto_wakeup;
+		info.keepalive_id = cfg->keepalive_id;
+		info.wakeup_pattern_id = cfg->wakeup_pattern_id;
+		info.ack_pattern_id = cfg->ack_pattern_id;
+		if (info.realwow_en) {
+			param.interval = cfg->realwow_ofld_content.interval;
+			param.kapktsize = cfg->realwow_ofld_content.keep_alive_pkt_size;
+			param.acklostlimit = cfg->realwow_ofld_content.ack_lost_limit;
+			param.ackpatternsize = cfg->realwow_ofld_content.ack_ptrn_size;
+			param.wakeuppatternsize = cfg->realwow_ofld_content.wakeup_ptrn_size;
+			param.wakeupsecnum = cfg->realwow_ofld_content.wakeup_sec_num;
+		}
+	} else {
+		info.realwow_en = false;
+	}
+
+	if (hal_mac_ops->cfg_realwow(mac, (u8)macid, &info, &param))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_set_wowlan(struct hal_info_t *hal, enum mac_ax_wow_ctrl ctrl)
+{
+	u32 mac_err = 0;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	mac_err = mac->ops->intf_ops->set_wowlan(mac, ctrl);
+	if (mac_err != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+static bool _wow_is_txq_empty(struct mac_ax_tx_queue_empty *val)
+{
+	u8 i = 0;
+	u8 size = ARRAY_SIZE(val->macid_txq_empty);
+
+	if (!val->others_empty)
+		return false;
+
+	if (!val->band0_mgnt_empty)
+		return false;
+
+	if (!val->band1_mgnt_empty)
+		return false;
+
+	for (i = 0; i < size; i++) {
+		if (val->macid_txq_empty[i] != 0xFF)
+			return false;
+	}
+
+	return true;
+}
+
+#define MAX_WOW_POLLNG_TXQ_EMPTY_TIME 50000 /* us */
+#define MAX_WOW_CHK_TXQ_EMPTY_CNT 2 /* continously check ok should satisfied this value */
+enum rtw_hal_status rtw_hal_mac_wow_chk_txq_empty(struct hal_info_t *hal, u8 *empty)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_tx_queue_empty val = {0};
+	u32 start_t = _os_get_cur_time_us();
+	u8 chk_cnt = 0;
+
+	while (1) {
+
+		if (phl_get_passing_time_us(start_t) >= MAX_WOW_POLLNG_TXQ_EMPTY_TIME) {
+			_dump_txq_status(&val);
+			PHL_ERR("%s : reach maximum polling time.\n", __func__);
+			break;
+		}
+
+		for (chk_cnt = 0; chk_cnt < MAX_WOW_CHK_TXQ_EMPTY_CNT; chk_cnt++) {
+			if (mac->ops->is_txq_empty(mac, &val) != MACSUCCESS)
+				break;
+
+			if (!_wow_is_txq_empty(&val)) {
+				break;
+			} else {
+				*empty = 1;
+			}
+		}
+
+		if (*empty)
+			break;
+
+		_os_delay_us(hal_to_drvpriv(hal), 50);
+	}
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : polling empty %d with duration %d.\n",
+			  __func__, *empty, phl_get_passing_time_us(start_t));
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_mac_wow_wde_drop(struct hal_info_t *hal, u8 band)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_pkt_drop_info info = {0};
+	u32 mac_err = 0;
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s with band %d.\n", __func__, band);
+
+	info.sel = MAC_AX_PKT_DROP_SEL_BAND_ONCE;
+	info.band = band;
+
+	mac_err = mac->ops->pkt_drop(mac, &info);
+	if (mac_err != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_nlo(struct hal_info_t *hal, u16 macid, u8 en,
+		    struct rtw_nlo_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_nlo_info info = {0};
+	struct mac_ax_nlo_networklist_parm_ content = {0};
+	void *d = hal_to_drvpriv(hal);
+	u8 i = 0;
+	u32 mac_err;
+
+	info.nlo_en = en;
+
+	if (en) {
+		content.numofentries = cfg->num_of_networks;
+		_os_mem_cpy(d, content.ssidlen, cfg->ssidlen, content.numofentries);
+		_os_mem_cpy(d, content.ssid, cfg->ssid,
+			content.numofentries* MAX_SSID_LEN);
+		_os_mem_cpy(d, content.chipertype, cfg->chipertype,
+			content.numofentries);
+
+		info.compare_cipher_type = cfg->compare_cipher_type;
+
+		PHL_INFO("%s : compare_cipher_type = %u \n", __func__,
+			 info.compare_cipher_type);
+
+		PHL_INFO("%s : num of ssid %u \n", __func__, content.numofentries);
+
+		for (i = 0; i < content.numofentries; i++) {
+			PHL_INFO("%s : #%u ssid/len = %s/%u \n", __func__, i+1,
+				(char *)content.ssid[i], content.ssidlen[i]);
+		}
+	}
+
+	mac_err = mac->ops->cfg_nlo(mac, (u8)macid, &info, &content);
+
+	if (MACSUCCESS != mac_err) {
+		PHL_ERR("%s : failed, mac err (%u) \n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_periodic_wake(struct hal_info_t *hal, u16 macid, u8 en,
+			      struct rtw_periodic_wake_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_periodic_wake_info info = {0};
+
+	if (en) {
+		info.macid = macid;
+		info.enable = cfg->periodic_wake_en;
+		info.wake_period = cfg->wake_period;
+		info.wake_duration = cfg->wake_duration;
+	}
+
+	if (mac->ops->periodic_wake_cfg(mac, info) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_wow_req_tri_evt(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_err = 0;
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s\n", __func__);
+
+	mac_err = mac->ops->req_wow_tri_evt(mac);
+	if (mac_err != MACSUCCESS){
+		PHL_ERR("%s: failed, mac err (%u) \n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_wow_req_diag_rpt(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_err = 0;
+	struct mac_ax_sta_diag_err_rpt rpt = {0};
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s\n", __func__);
+
+	mac_err = mac->ops->req_wow_diag_rpt(mac, &rpt);
+	if (mac_err != MACSUCCESS){
+		PHL_ERR("%s: failed, mac err (%u) \n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	} else {
+		PHL_INFO("%s: rpt err 0x%x, wow_err 0x%x", __func__, rpt.err, rpt.wow_err);
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+enum rtw_hal_status
+rtw_hal_mac_wow_dbg_dump(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+
+	if (hal_mac_ops->wow_dbg_dump(mac))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+#if defined(CONFIG_PHL_MDNS_OFFLOAD) || defined(CONFIG_PHL_WOW_APF)
+static enum rtw_hal_status
+_hal_mac_cfg_check_proxy_done(struct hal_info_t *hal_info,
+			      u8 *fw_ret)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u16 loop_cnt = 0;
+
+	if (!mac)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	do {
+		if (hal_mac_ops->check_proxy_done(mac, fw_ret) == MACSUCCESS)
+			break;
+
+		_os_sleep_ms(hal_to_drvpriv(hal_info), POLLING_HALMAC_TIME);
+
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_PRINT("%s, check count = %d.\n", __func__, loop_cnt);
+		return RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_ERR("%s, polling timeout!!!\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+#ifdef CONFIG_PHL_MDNS_OFFLOAD
+enum rtw_hal_status
+rtw_hal_mac_cfg_mdns_ofld(struct hal_info_t *hal_info, u16 macid, u8 en,
+							struct rtw_mdns_ofld_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct rtw_hal_mac_mdns_ofld info = {0};
+	u8 ofld_idx;
+	u8 fw_ret;
+	u32 ret, mac_err;
+
+
+	if (en) {
+		info.ipv4_pktid = cfg->mdns_ipv4_header.ipv4_pktid;
+		info.ipv6_pktid = cfg->mdns_ipv6_header.ipv6_pktid;
+		info.mdns_offload_en = cfg->mdns_en;
+		info.offload_state = cfg->offload_state;
+		info.passthru_behavior = cfg->passthru_list.passthru_behavior;
+		info.macid = macid;
+		info.type = 0; /* reserved type */
+		/* mdns data response */
+		for (ofld_idx = 0; ofld_idx < MAX_MDNS_RESP_NUM; ofld_idx++) {
+			info.data_pktid[ofld_idx] = cfg->resp_entry[ofld_idx].data_pktid;
+			if (cfg->resp_entry[ofld_idx].content_len)
+				info.mdns_protocol_offset = info.mdns_protocol_offset | BIT(ofld_idx);
+		}
+		/* mdns passthru list */
+		for (ofld_idx = 0; ofld_idx < MAX_MDNS_PASSTHRU_NAME_NUM; ofld_idx++) {
+			info.pass_pktid[ofld_idx] = cfg->passthru_list.passthru_name[ofld_idx].pass_pktid;
+			if (cfg->passthru_list.passthru_name[ofld_idx].name_len)
+				info.passthru_list_offset = info.passthru_list_offset | BIT(ofld_idx);
+		}
+	}
+	else { // Disable
+		info.mdns_offload_en = 0;
+	}
+
+	mac_err = hal_mac_ops->mdns_ofld(mac, &info);
+
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : failed, mac err (%u) \n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	ret = _hal_mac_cfg_check_proxy_done(hal_info, &fw_ret);
+
+	if (ret != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s : check proxy done fail \n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /* CONFIG_PHL_MDNS_OFFLOAD */
+#ifdef CONFIG_PHL_WOW_APF
+enum rtw_hal_status
+rtw_hal_mac_cfg_apf_ofld(struct hal_info_t *hal_info, u16 macid, struct rtw_apf_info *cfg, u8 set)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct rtw_hal_mac_apf info = {0};
+	struct rtw_hal_mac_apf_report rpt_info = {0};
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	u8 i, fw_ret;
+	u32 mac_err;
+
+	/* mac_apf_ofld */
+	if (set == 1) {
+		info.macid = macid;
+		info.mac_pktid = cfg->mac_pktid;
+		for (i = 0; i < cfg->apf_prog_num; i++) {
+			/* mac->apf_info.ptr[i] for mac_apf_get_report */
+			mac->apf_info.ptr[i] = cfg->apf_prog;
+			info.program_pktid[i] = cfg->program_pktid;
+			info.ptr[i] = cfg->apf_prog;
+			info.prog_info[i].prog_len = cfg->apf_prog_len;
+			info.prog_info[i].data_len = cfg->apf_data_len;
+			info.program_mask |= BIT(i);
+
+			if (info.program_mask != 0) {
+				info.apf_en = 1;
+				info.apf_type = 0;
+			} else {
+				PHL_INFO("%s APF not enable\n", __func__);
+			}
+#ifdef CONFIG_PHL_WOW_APF_DBG
+		PHL_INFO("%s() set apf_prog(%u) prog_len(%u) data_len(%u)\n", __func__, i,
+			 info.prog_info[i].prog_len, info.prog_info[i].data_len);
+#endif
+		}
+		if (hal_mac_ops->apf_ofld) {
+			mac_err = hal_mac_ops->apf_ofld(mac, &info);
+		} else {
+			PHL_ERR("mac_apf_ofld is NULL\n");
+			return RTW_HAL_STATUS_MAC_API_FAILURE;
+		}
+
+		if (mac_err != MACSUCCESS) {
+			PHL_ERR("%s : apf_ofld failed, mac err (%u) \n", __func__, mac_err);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+
+		hstatus = _hal_mac_cfg_check_proxy_done(hal_info, &fw_ret);
+		if (hstatus != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s : check proxy done fail \n", __func__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+
+	} else { /* mac_apf_get_report */
+#ifdef CONFIG_PHL_WOW_APF_DBG
+		PHL_INFO("%s() get apf_prog(%u)\n", __func__, cfg->idx_apf_prog_ofld);
+#endif
+		rpt_info.program_num = cfg->idx_apf_prog_ofld;
+		if (hal_mac_ops->apf_get_report) {
+			mac_err = hal_mac_ops->apf_get_report(mac, &rpt_info);
+		} else {
+			PHL_ERR("mac_apf_get_report is NULL\n");
+			return RTW_HAL_STATUS_MAC_API_FAILURE;
+		}
+
+		if (mac_err != MACSUCCESS) {
+			PHL_ERR("%s : failed, mac err (%u) \n", __func__, mac_err);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+		hstatus = _hal_mac_cfg_check_proxy_done(hal_info, &fw_ret);
+	}
+
+	return hstatus;
+}
+
+#endif /* CONFIG_WOW_PHL_APF */
+#endif /* CONFIG_WOWLAN */
+
+#define MAX_POLLING_FW_STS_TIME 100 /* ms */
+
+#ifdef CONFIG_PHL_SCANOFLD
+#define SCAN_OFLD_PERIOD_UNIT 100 /* 100 ms */
+void
+rtw_hal_mac_scan_ofld_leave_cur_chnl(struct hal_info_t *hal, u8 hw_band,
+				     u8 ch_band, u8 ch)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	mac->ops->scanofld_hst_ctrl(mac, ch, ch_band, MAC_AX_SCAN_CTRL_NEXT_CH,
+				    hw_band);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld(struct hal_info_t *hal, u16 mac_id, u8 hw_band, u8 hw_port,
+                      struct scan_ofld_info *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_scanofld_param param = {0};
+	u32 mac_err = 0;
+
+	param.macid = mac_id;
+	param.port_id = hw_port;
+	param.band = hw_band;
+	param.operation = cfg->operation;
+	param.tsf_low = cfg->tsf_low;
+	param.tsf_high = cfg->tsf_high;
+	param.scan_type = cfg->mode;
+	param.norm_pd = cfg->period / SCAN_OFLD_PERIOD_UNIT;
+	param.norm_cy = cfg->cycle;
+	param.slow_pd = cfg->slow_period / SCAN_OFLD_PERIOD_UNIT;
+	param.c2h_end = cfg->end_notify;
+	if (cfg->tgt_mode) {
+		param.target_ch_mode = true;
+		param.target_ch_ch_band = cfg->tgt_chandef.band;
+		param.target_ch_bw = cfg->tgt_chandef.bw;
+		param.target_pri_ch = cfg->tgt_chandef.chan;
+		param.target_central_ch = cfg->tgt_chandef.center_ch;
+		PHL_INFO("%s : tgt_band/tgt_chnl/tgt_bw/tgt_center =\
+			%u/%u/%u/%u \n", __func__,
+			param.target_ch_ch_band, param.target_pri_ch,
+			param.target_ch_bw, param.target_central_ch);
+	} else {
+		param.target_ch_mode = false;
+	}
+
+	if (cfg->tsf_high != 0 || cfg->tsf_low != 0)
+		param.start_mode = 1;
+
+	PHL_INFO("%s : macid/port/band/operation/tsf_low/tsf_high =\
+		%u/%u/%u/%u/%u/%u \n", __func__,
+		param.macid, param.port_id, param.band, param.operation,
+		param.tsf_low, param.tsf_high);
+
+	PHL_INFO("%s : scan_type/norm_pd/norm_cy/slow_pd =\
+		%u/%u/%u/%u \n", __func__,
+		param.scan_type, param.norm_pd, param.norm_cy, param.slow_pd);
+
+	mac_err = mac->ops->scanofld(mac, &param);
+
+	if (MACSUCCESS != mac_err) {
+		PHL_ERR("%s : failed, mac error = %u \n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+
+}
+
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld_chlist_busy(struct hal_info_t *hal, u8 hw_band)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	if (mac->ops->scanofld_chlist_busy(mac, hw_band))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld_add_ch(struct hal_info_t *hal, u8 hw_band,
+                             struct scan_ofld_ch_info *cfg,
+                             bool ofld)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_scanofld_chinfo chinfo = {0};
+	u32 mac_err = 0, poll_cnt = 0, cur_time = _os_get_cur_time_ms();
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	void *drv_priv = hal_to_drvpriv(hal);
+
+	chinfo.central_ch = cfg->center_chan;
+	chinfo.pri_ch = cfg->chan;
+	chinfo.bw = cfg->bw;
+	chinfo.ch_band = cfg->band;
+	chinfo.period = cfg->period;
+	chinfo.tx_pkt = cfg->tx_pkt;
+	chinfo.pause_tx_data = cfg->tx_data_pause;
+	chinfo.c2h_notify_enterCH = cfg->enter_notify;
+	chinfo.tx_null = cfg->tx_null;
+	chinfo.probe_req_pkt_id = cfg->probe_req_id;
+	chinfo.ch_band = cfg->band;
+	chinfo.chkpt_timer = cfg->chkpt_time;
+
+	PHL_INFO("%s : band/central_ch//bw/period/tx_pkt/pause_tx_data/req_id/tx_null/chkpt =\
+		%u/%u/%u/%u/%u/%u/%u/%u/%u \n", __func__,
+		chinfo.ch_band, chinfo.central_ch, chinfo.bw,
+		chinfo.period, chinfo.tx_pkt, chinfo.pause_tx_data,
+		chinfo.probe_req_pkt_id, chinfo.tx_null, chinfo.chkpt_timer);
+
+	mac_err = mac->ops->add_scanofld_ch(mac, &chinfo, ofld, true, hw_band);
+
+	if (MACSUCCESS != mac_err) {
+		PHL_ERR("%s : failed, mac error = %u \n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (ofld == false)
+		return RTW_HAL_STATUS_SUCCESS;
+
+	/* polling fw status */
+	while (1) {
+
+		if (phl_get_passing_time_ms(cur_time) > MAX_POLLING_FW_STS_TIME) {
+			PHL_ERR("%s polling fw status timeout !!!\n", __func__);
+			hstatus = RTW_HAL_STATUS_FAILURE;
+			break;
+		}
+
+		if (mac->ops->scanofld_chlist_busy(mac, hw_band)) {
+			_os_delay_ms(drv_priv, 1);
+		} else {
+			PHL_INFO("%s : passing time %ums\n", __func__,
+				phl_get_passing_time_ms(cur_time));
+			break;
+		}
+
+		poll_cnt++;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld_fw_busy(struct hal_info_t *hal, u8 hw_band)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+
+	if (mac->ops->scanofld_fw_busy(mac, hw_band))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /* CONFIG_PHL_SCANOFLD */
+
+static enum rtw_hal_status
+hal_mac_read_efuse(struct mac_ax_adapter *mac, u32 addr, u32 size,
+						u8 *val, enum mac_ax_efuse_bank bank)
+{
+	if (mac->ops->read_efuse(mac, addr, size, val, bank) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+static enum rtw_hal_status
+hal_mac_write_efuse(struct mac_ax_adapter *mac, u32 addr, u8 val,
+						enum mac_ax_efuse_bank bank)
+{
+	if (mac->ops->write_efuse(mac, addr, val, bank) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_enable_cpu(struct hal_info_t *hal_info, u8 reason, u8 dlfw)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->enable_cpu(mac, reason, dlfw) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_disable_cpu(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->disable_cpu(mac) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_romdl(struct hal_info_t *hal_info, u8 *rom_buf, u32 rom_size)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 rom_addr = 0x18900000;
+
+	if (mac->ops->romdl(mac, rom_buf, rom_addr, rom_size) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_fwdl(struct hal_info_t *hal_info, u8 *fw_buf, u32 fw_size)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_err;
+
+	mac_err = mac->ops->fwdl(mac, fw_buf, fw_size);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hal_mac_print_fw_version(hal_info);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool
+rtw_hal_mac_fwredl_needed(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	return mac->ops->fwredl_needed(mac);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_fwredl(struct hal_info_t *hal_info, u8 *fw_buf, u32 fw_size)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_err;
+
+	mac_err = mac->ops->fwredl(mac, fw_buf, fw_size);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hal_mac_print_fw_version(hal_info);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_query_fw_buff(struct hal_info_t *hal_info, enum rtw_fw_type cat, u8 **fw, u32 *fw_len)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_status;
+
+	mac_status = mac->ops->query_fw_buff(mac, cat, fw, fw_len);
+	if (mac_status != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_enable_fw(struct hal_info_t *hal_info, u8 fw_type)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_err;
+
+	mac_err = mac->ops->enable_fw(mac, fw_type);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hal_mac_print_fw_version(hal_info);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32
+rtw_hal_mac_get_txdesc_len(void *mac, struct rtw_xmit_req *treq)
+{
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	u32 wd_len = 0;
+
+	wd_len = mac_info->ops->txdesc_len(
+		mac_info,
+		&treq->mdata);
+
+	return wd_len;
+}
+
+
+/*   */
+/**
+ * rtw_hal_mac_ax_fill_txdesc
+ * @mac: see struct mac_ax_adapter
+ * @treq: the xmit request for this tx descriptor
+ * @wd_buf: the wd buffer to fill
+ * @wd_len: output, return the total length of filled wd
+ *
+ * Note,halmac API for hal and proto type is at hal_api_mac.h
+ */
+enum rtw_hal_status
+rtw_hal_mac_fill_txdesc(void *mac,
+                        struct rtw_xmit_req *treq,
+                        u8 *wd_buf,
+                        u32 *wd_len)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+
+	*wd_len = mac_info->ops->txdesc_len(mac_info, &treq->mdata);
+
+	hal_status = mac_info->ops->build_txdesc(mac_info,
+	                                         &treq->mdata,
+	                                         wd_buf,
+	                                         *wd_len);
+
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_set_hw_ampdu_cfg
+ * @hal_info: see struct hal_info_t
+ * @band: target band this AMPDU going to send
+ * @max_agg_num: AMPDU maximum aggregation number
+ * @max_agg_time: AMPDU maximum aggregation time, in unit of 32 us
+ *
+ * Note,
+ * (1) halmac API for hal and proto type is at hal_api_mac.h
+ */
+enum rtw_hal_status
+rtw_hal_mac_set_hw_ampdu_cfg(struct hal_info_t *hal_info,
+                             u8 band,
+                             u16 max_agg_num, u8 max_agg_time)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_ampdu_cfg info = {0};
+	u32 mac_err;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &info, 0, sizeof(info));
+
+	info.band = band;
+	info.wdbk_mode = MAC_AX_WDBK_MODE_SINGLE_BK;
+	info.rty_bk_mode = MAC_AX_RTY_BK_MODE_AGG;
+	info.max_agg_num = max_agg_num;
+	info.max_agg_time_32us = max_agg_time;
+
+	mac_err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_AMPDU_CFG, &info);
+
+	if (mac_err != MACSUCCESS)
+		goto fail;
+
+	return RTW_HAL_STATUS_SUCCESS;
+fail:
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+/**
+ * rtw_hal_mac_enable_hwamsdu
+ * @hal_info: see struct hal_info_t
+ * @enable: enable HW AMSDU CR
+ * @max_num: set maximum AMSDU number
+ * @en_single_amsdu: Allow single AMSDU or not
+ * @en_last_amsdu_padding: enable AMSDU padding or not
+ *
+ * Note,
+ * (1) halmac API for hal and proto type is at hal_api_mac.h
+ */
+enum rtw_hal_status
+rtw_hal_mac_enable_hwamsdu(struct hal_info_t *hal_info,
+			   u8 enable,
+			   u8 max_num,
+			   u8 en_single_amsdu,
+			   u8 en_last_amsdu_padding)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 mac_err;
+	enum mac_ax_amsdu_pkt_num max_amsdu_num = MAC_AX_AMSDU_AGG_NUM_4;
+
+	switch(max_num) {
+	case 1:
+		max_amsdu_num = MAC_AX_AMSDU_AGG_NUM_1;
+		break;
+	case 2:
+		max_amsdu_num = MAC_AX_AMSDU_AGG_NUM_2;
+		break;
+	case 3:
+		max_amsdu_num = MAC_AX_AMSDU_AGG_NUM_3;
+		break;
+	case 4:
+		max_amsdu_num = MAC_AX_AMSDU_AGG_NUM_4;
+		break;
+	default:
+		PHL_ERR("%s max amsdu length value from 1 to 4, should not set(%d), default using value 4\n", __func__, max_num);
+		break;
+	}
+
+	mac_err = mac->ops->enable_hwamsdu(mac, enable, max_amsdu_num, en_single_amsdu, en_last_amsdu_padding);
+
+	if (mac_err != MACSUCCESS)
+		goto fail;
+
+	return RTW_HAL_STATUS_SUCCESS;
+fail:
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+/**
+ * rtw_hal_mac_hwamsdu_macid_en
+ * @hal_info: see struct hal_info_t
+ * @macid: macid
+ * @enable: enable HW AMSDU per macid
+ *
+ * Note,
+ * (1) halmac API for hal and proto type is at hal_api_mac.h
+ */
+enum rtw_hal_status
+rtw_hal_mac_hwamsdu_macid_en(struct hal_info_t *hal_info,
+			   u8 macid,
+			   u8 enable)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 mac_err;
+
+	mac_err = mac->ops->hwamsdu_macid_en(mac, macid, enable);
+
+	if (mac_err != MACSUCCESS)
+		goto fail;
+
+	return RTW_HAL_STATUS_SUCCESS;
+fail:
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+
+/**
+ * rtw_hal_mac_hwamsdu_max_len
+ * @hal_info: see struct hal_info_t
+ * @macid: macid
+ * @amsdu_max_len: max amsdu length
+ * +------------+-----------------------+
+ *|Bit map	|Max length(bytes)	|
+ *+-------------+-----------------------+
+ *|000b		|2048			|
+ *|001b		|3824			|
+ *|010b		|7920			|
+ *|011b		|11384			|
+ *|100b		|4024			|
+ *|101b		|5120			|
+ *|110b		|6656			|
+ *|111b		|10240			|
+ *+-------------+-----------------------+
+ *
+ * Note,
+ * (1) halmac API for hal and proto type is at hal_api_mac.h
+ */
+enum rtw_hal_status
+rtw_hal_mac_hwamsdu_max_len(struct hal_info_t *hal_info,
+			   u8 macid,
+			   u8 amsdu_max_len)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 mac_err;
+
+	mac_err = mac->ops->hwamsdu_max_len(mac, macid, amsdu_max_len);
+
+	if (mac_err != MACSUCCESS)
+		goto fail;
+
+	return RTW_HAL_STATUS_SUCCESS;
+fail:
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+/**
+ * rtw_hal_mac_hdr_conv_tx_macid_en
+ * @hal_info: see struct hal_info_t
+ * @qos_field_h: qos field need hw to fill
+ * @qos_field_h_en: enable qos field need hw to fill
+ * @mhdr_len: mac header length
+ * @vlan_tag_valid: vlan tag include or not
+ *
+ * Note,
+ * (1) halmac API for hal and proto type is at hal_api_mac.h
+ */
+enum rtw_hal_status
+rtw_hal_mac_hdr_conv_tx_macid_en(struct hal_info_t *hal_info,
+				 u8 macid,
+				 u8 qos_field_h,
+				 u8 qos_field_h_en,
+				 u8 mhdr_len,
+				 u8 vlan_tag_valid)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_hdr_conv_dctl_cfg hdr_conv_dctl_cfg;
+	u32 mac_err;
+
+	/* set mac_ax_hdr_conv_dctl_cfg*/
+	hdr_conv_dctl_cfg.qos_field_h = qos_field_h;
+	hdr_conv_dctl_cfg.qos_field_h_en = qos_field_h_en;
+	hdr_conv_dctl_cfg.mhdr_len = mhdr_len;
+	hdr_conv_dctl_cfg.vlan_tag_valid = vlan_tag_valid;
+
+	/* htc field need hw to fill or not */
+	/* Set htc_order = 0 as disable and htc_lb don't care */
+	hdr_conv_dctl_cfg.htc_order = 0;
+	hdr_conv_dctl_cfg.htc_lb = 0;
+
+	mac_err = mac->ops->hdr_conv_tx_macid_en(mac, macid, hdr_conv_dctl_cfg);
+
+	if (mac_err != MACSUCCESS)
+		goto fail;
+
+	return RTW_HAL_STATUS_SUCCESS;
+fail:
+	return RTW_HAL_STATUS_MAC_API_FAILURE;
+}
+
+/**
+ * the function to update DMAC control info by halmac api
+ * @hal_info: see struct hal_info_t
+ * @dctl_info: structure of dmac control information, define by halmac
+ * @macid: the macid corresponding to this cmac control info
+ *
+ * return RTW_HAL_STATUS_MAC_API_FAILURE if update fail
+ */
+enum rtw_hal_status rtw_hal_dmc_tbl_cfg(struct hal_info_t *hal_info,
+                                        struct mac_ax_dctl_info *dctl_info,
+                                        struct mac_ax_dctl_info *dctl_info_mask,
+                                        u16 macid)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 ret = 0;
+
+	ret = mac->ops->upd_dctl_info(mac, dctl_info, dctl_info_mask, (u8)macid, 1);
+
+	if (0 == ret) {
+		sts = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "mac_upd_dctl_info fail (0x%08X)\n",
+			  ret);
+		sts = RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+
+	return sts;
+}
+
+#ifdef DBG_DUMP_CMAC_CFG_INFO
+static void
+_hal_dump_cctrl_cfg_info(struct rtw_hal_mac_ax_cctl_info *cctrl,
+				struct rtw_hal_mac_ax_cctl_info *cctl_mask)
+{
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "%s(): start dump\n",
+		__func__);
+	/* Dump cctrl */
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: datarate(%d), force_txop(%d), data_bw(%d), data_gi_ltf(%d)\n",
+		cctrl->datarate, cctrl->force_txop,
+		cctrl->data_bw, cctrl->data_gi_ltf);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: darf_tc_index(%d), arfr_ctrl(%d), acq_rpt_en(%d), mgq_rpt_en(%d)\n",
+		cctrl->darf_tc_index, cctrl->arfr_ctrl,
+		cctrl->acq_rpt_en, cctrl->mgq_rpt_en);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: ulq_rpt_en(%d), twtq_rpt_en(%d), rsvd0(%d), disrtsfb(%d)\n",
+		cctrl->ulq_rpt_en, cctrl->twtq_rpt_en,
+		cctrl->rsvd0, cctrl->disrtsfb);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: disdatafb(%d), tryrate(%d), ampdu_density(%d), data_rty_lowest_rate(%d)\n",
+		cctrl->disdatafb, cctrl->tryrate,
+		cctrl->ampdu_density, cctrl->data_rty_lowest_rate);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: ampdu_time_sel(%d), ampdu_len_sel(%d), rts_txcnt_lmt_sel(%d), rts_txcnt_lmt(%d)\n",
+		cctrl->ampdu_time_sel, cctrl->ampdu_len_sel,
+		cctrl->rts_txcnt_lmt_sel, cctrl->rts_txcnt_lmt);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: rtsrate(%d), rsvd1(%d), vcs_stbc(%d), rts_rty_lowest_rate(%d)\n",
+		cctrl->rtsrate, cctrl->rsvd1,
+		cctrl->vcs_stbc, cctrl->rts_rty_lowest_rate);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: data_tx_cnt_lmt(%d), data_txcnt_lmt_sel(%d), max_agg_num_sel(%d), rts_en(%d)\n",
+		cctrl->data_tx_cnt_lmt, cctrl->data_txcnt_lmt_sel,
+		cctrl->max_agg_num_sel, cctrl->rts_en);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: cts2self_en(%d), cca_rts(%d), hw_rts_en(%d), rts_drop_data_mode(%d)\n",
+		cctrl->cts2self_en, cctrl->cca_rts,
+		cctrl->hw_rts_en, cctrl->rts_drop_data_mode);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: preld_en(%d), ampdu_max_len(%d), ul_mu_dis(%d), ampdu_max_time(%d)\n",
+		cctrl->preld_en, cctrl->ampdu_max_len,
+		cctrl->ul_mu_dis, cctrl->ampdu_max_time);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: max_agg_num(%d), ba_bmap(%d), rsvd3(%d), vo_lftime_sel(%d)\n",
+		cctrl->max_agg_num, cctrl->ba_bmap,
+		cctrl->rsvd3, cctrl->vo_lftime_sel);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: vi_lftime_sel(%d), be_lftime_sel(%d), bk_lftime_sel(%d), sectype(%d)\n",
+		cctrl->vi_lftime_sel, cctrl->be_lftime_sel,
+		cctrl->bk_lftime_sel, cctrl->sectype);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: multi_port_id(%d), bmc(%d), mbssid(%d), navusehdr(%d)\n",
+		cctrl->multi_port_id, cctrl->bmc,
+		cctrl->mbssid, cctrl->navusehdr);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: txpwr_mode(%d), data_dcm(%d), data_er(%d), data_ldpc(%d)\n",
+		cctrl->txpwr_mode, cctrl->data_dcm,
+		cctrl->data_er, cctrl->data_ldpc);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: data_stbc(%d), a_ctrl_bqr(%d), a_ctrl_uph(%d), a_ctrl_bsr(%d)\n",
+		cctrl->data_stbc, cctrl->a_ctrl_bqr,
+		cctrl->a_ctrl_uph, cctrl->a_ctrl_bsr);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: a_ctrl_cas(%d), data_bw_er(%d), lsig_txop_en(%d), rsvd4(%d)\n",
+		cctrl->a_ctrl_cas, cctrl->data_bw_er,
+		cctrl->lsig_txop_en, cctrl->rsvd4);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: ctrl_cnt_vld(%d), ctrl_cnt(%d), resp_ref_rate(%d), rsvd5(%d)\n",
+		cctrl->ctrl_cnt_vld, cctrl->ctrl_cnt,
+		cctrl->resp_ref_rate, cctrl->rsvd5);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: all_ack_support(%d), bsr_queue_size_format(%d), bsr_om_upd_en(%d), macid_fwd_idc(%d)\n",
+		cctrl->all_ack_support, cctrl->bsr_queue_size_format,
+		cctrl->bsr_om_upd_en, cctrl->macid_fwd_idc);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: ntx_path_en(%d), path_map_a(%d), path_map_b(%d), path_map_c(%d)\n",
+		cctrl->ntx_path_en, cctrl->path_map_a,
+		cctrl->path_map_b, cctrl->path_map_c);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: path_map_d(%d), antsel_a(%d), antsel_b(%d), antsel_c(%d)\n",
+		cctrl->path_map_d, cctrl->antsel_a,
+		cctrl->antsel_b, cctrl->antsel_c);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: antsel_d(%d), addr_cam_index(%d), paid(%d), uldl(%d)\n",
+		cctrl->antsel_d, cctrl->addr_cam_index,
+		cctrl->paid, cctrl->uldl);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: doppler_ctrl(%d), nominal_pkt_padding(%d), nominal_pkt_padding40(%d), txpwr_tolerence(%d)\n",
+		cctrl->doppler_ctrl, cctrl->nominal_pkt_padding,
+		cctrl->nominal_pkt_padding40, cctrl->txpwr_tolerence);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: nominal_pkt_padding80(%d), nc(%d), nr(%d)\n",
+		cctrl->nominal_pkt_padding80,
+		cctrl->nc, cctrl->nr);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: ng(%d), cb(%d), cs(%d), csi_txbf_en(%d)\n",
+		cctrl->ng, cctrl->cb, cctrl->cs, cctrl->csi_txbf_en);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: csi_stbc_en(%d), csi_ldpc_en(%d), csi_para_en(%d), csi_fix_rate(%d)\n",
+		cctrl->csi_stbc_en, cctrl->csi_ldpc_en,
+		cctrl->csi_para_en, cctrl->csi_fix_rate);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctrl: csi_gi_ltf(%d), nominal_pkt_padding160(%d), csi_bw(%d)\n",
+		cctrl->csi_gi_ltf, cctrl->nominal_pkt_padding160, cctrl->csi_bw);
+	/* Dump Mask */
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: datarate(%d), force_txop(%d), data_bw(%d), data_gi_ltf(%d)\n",
+		cctl_mask->datarate, cctl_mask->force_txop,
+		cctl_mask->data_bw, cctl_mask->data_gi_ltf);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: darf_tc_index(%d), arfr_ctrl(%d), acq_rpt_en(%d), mgq_rpt_en(%d)\n",
+		cctl_mask->darf_tc_index, cctl_mask->arfr_ctrl,
+		cctl_mask->acq_rpt_en, cctl_mask->mgq_rpt_en);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: ulq_rpt_en(%d), twtq_rpt_en(%d), rsvd0(%d), disrtsfb(%d)\n",
+		cctl_mask->ulq_rpt_en, cctl_mask->twtq_rpt_en,
+		cctl_mask->rsvd0, cctl_mask->disrtsfb);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: disdatafb(%d), tryrate(%d), ampdu_density(%d), data_rty_lowest_rate(%d)\n",
+		cctl_mask->disdatafb, cctl_mask->tryrate,
+		cctl_mask->ampdu_density, cctl_mask->data_rty_lowest_rate);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: ampdu_time_sel(%d), ampdu_len_sel(%d), rts_txcnt_lmt_sel(%d), rts_txcnt_lmt(%d)\n",
+		cctl_mask->ampdu_time_sel, cctl_mask->ampdu_len_sel,
+		cctl_mask->rts_txcnt_lmt_sel, cctl_mask->rts_txcnt_lmt);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: rtsrate(%d), rsvd1(%d), vcs_stbc(%d), rts_rty_lowest_rate(%d)\n",
+		cctl_mask->rtsrate, cctl_mask->rsvd1,
+		cctl_mask->vcs_stbc, cctl_mask->rts_rty_lowest_rate);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: data_tx_cnt_lmt(%d), data_txcnt_lmt_sel(%d), max_agg_num_sel(%d), rts_en(%d)\n",
+		cctl_mask->data_tx_cnt_lmt, cctl_mask->data_txcnt_lmt_sel,
+		cctl_mask->max_agg_num_sel, cctl_mask->rts_en);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: cts2self_en(%d), cca_rts(%d), hw_rts_en(%d), rts_drop_data_mode(%d)\n",
+		cctl_mask->cts2self_en, cctl_mask->cca_rts,
+		cctl_mask->hw_rts_en, cctl_mask->rts_drop_data_mode);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: preld_en(%d), ampdu_max_len(%d), ul_mu_dis(%d), ampdu_max_time(%d)\n",
+		cctl_mask->preld_en, cctl_mask->ampdu_max_len,
+		cctl_mask->ul_mu_dis, cctl_mask->ampdu_max_time);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: max_agg_num(%d), ba_bmap(%d), rsvd3(%d), vo_lftime_sel(%d)\n",
+		cctl_mask->max_agg_num, cctl_mask->ba_bmap,
+		cctl_mask->rsvd3, cctl_mask->vo_lftime_sel);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: vi_lftime_sel(%d), be_lftime_sel(%d), bk_lftime_sel(%d), sectype(%d)\n",
+		cctl_mask->vi_lftime_sel, cctl_mask->be_lftime_sel,
+		cctl_mask->bk_lftime_sel, cctl_mask->sectype);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: multi_port_id(%d), bmc(%d), mbssid(%d), navusehdr(%d)\n",
+		cctl_mask->multi_port_id, cctl_mask->bmc,
+		cctl_mask->mbssid, cctl_mask->navusehdr);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: txpwr_mode(%d), data_dcm(%d), data_er(%d), data_ldpc(%d)\n",
+		cctl_mask->txpwr_mode, cctl_mask->data_dcm,
+		cctl_mask->data_er, cctl_mask->data_ldpc);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: data_stbc(%d), a_ctrl_bqr(%d), a_ctrl_uph(%d), a_ctrl_bsr(%d)\n",
+		cctl_mask->data_stbc, cctl_mask->a_ctrl_bqr,
+		cctl_mask->a_ctrl_uph, cctl_mask->a_ctrl_bsr);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: a_ctrl_cas(%d), data_bw_er(%d), lsig_txop_en(%d), rsvd4(%d)\n",
+		cctl_mask->a_ctrl_cas, cctl_mask->data_bw_er,
+		cctl_mask->lsig_txop_en, cctl_mask->rsvd4);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: ctrl_cnt_vld(%d), ctrl_cnt(%d), resp_ref_rate(%d), rsvd5(%d)\n",
+		cctl_mask->ctrl_cnt_vld, cctl_mask->ctrl_cnt,
+		cctl_mask->resp_ref_rate, cctl_mask->rsvd5);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: all_ack_support(%d), bsr_queue_size_format(%d), bsr_om_upd_en(%d), macid_fwd_idc(%d)\n",
+		cctl_mask->all_ack_support, cctl_mask->bsr_queue_size_format,
+		cctl_mask->bsr_om_upd_en, cctl_mask->macid_fwd_idc);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: ntx_path_en(%d), path_map_a(%d), path_map_b(%d), path_map_c(%d)\n",
+		cctl_mask->ntx_path_en, cctl_mask->path_map_a,
+		cctl_mask->path_map_b, cctl_mask->path_map_c);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: path_map_d(%d), antsel_a(%d), antsel_b(%d), antsel_c(%d)\n",
+		cctl_mask->path_map_d, cctl_mask->antsel_a,
+		cctl_mask->antsel_b, cctl_mask->antsel_c);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: antsel_d(%d), addr_cam_index(%d), paid(%d), uldl(%d)\n",
+		cctl_mask->antsel_d, cctl_mask->addr_cam_index,
+		cctl_mask->paid, cctl_mask->uldl);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: doppler_ctrl(%d), nominal_pkt_padding(%d), nominal_pkt_padding40(%d), txpwr_tolerence(%d)\n",
+		cctl_mask->doppler_ctrl, cctl_mask->nominal_pkt_padding,
+		cctl_mask->nominal_pkt_padding40, cctl_mask->txpwr_tolerence);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: nominal_pkt_padding80(%d), nc(%d), nr(%d)\n",
+		cctl_mask->nominal_pkt_padding80,
+		cctl_mask->nc, cctl_mask->nr);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: ng(%d), cb(%d), cs(%d), csi_txbf_en(%d)\n",
+		cctl_mask->ng, cctl_mask->cb,
+		cctl_mask->cs, cctl_mask->csi_txbf_en);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: csi_stbc_en(%d), csi_ldpc_en(%d), csi_para_en(%d), csi_fix_rate(%d)\n",
+		cctl_mask->csi_stbc_en, cctl_mask->csi_ldpc_en,
+		cctl_mask->csi_para_en, cctl_mask->csi_fix_rate);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "cctl_mask: csi_gi_ltf(%d), nominal_pkt_padding160(%d), csi_bw(%d)\n",
+		cctl_mask->csi_gi_ltf, cctl_mask->nominal_pkt_padding160,
+		cctl_mask->csi_bw);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "%s(): end dump\n",
+		__func__);
+}
+#endif
+
+/**
+ * the function to update CMAC control info by halmac api
+ * @hal_info: see struct hal_info_t
+ * @cctl_info: structure of cmac control information, define by halmac
+ * @macid: the macid corresponding to this cmac control info
+ *
+ * return RTW_HAL_STATUS_MAC_API_FAILURE if update fail
+ */
+enum rtw_hal_status rtw_hal_cmc_tbl_cfg(struct hal_info_t *hal_info,
+				struct rtw_hal_mac_ax_cctl_info *cctl_info,
+				struct rtw_hal_mac_ax_cctl_info *cctl_info_mask,
+				u16 macid)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 ret = 0;
+
+	cctl_info_mask->addr_cam_index = 0;
+	#ifdef DBG_DUMP_CMAC_CFG_INFO
+	_hal_dump_cctrl_cfg_info(cctl_info, cctl_info_mask);
+	#endif
+	ret = mac->ops->upd_cctl_info(mac, cctl_info, cctl_info_mask, (u8)macid, 1);
+
+	if (MACSUCCESS == ret) {
+		sts = RTW_HAL_STATUS_SUCCESS;
+		PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "mac_upd_cctl_info ok, macid(0x%x)\n",
+			  macid);
+	} else {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "mac_upd_cctl_info fail (0x%08X), macid(0x%x)\n",
+			  ret, macid);
+		sts = RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+
+	return sts;
+}
+
+
+/**
+ * the function to update BA CAM entry by halmac api
+ * @hal_info: see struct hal_info_t
+ * @ba_cam: structure of ba cam entry, define by halmac
+ *
+ * return RTW_HAL_STATUS_MAC_API_FAILURE if update fail
+ */
+enum rtw_hal_status rtw_hal_bacam_cfg(struct hal_info_t *hal_info,
+				      struct mac_ax_bacam_info *ba_cam)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 ret = 0;
+
+	ret = mac->ops->bacam_info(mac, ba_cam);
+
+	if (0 == ret) {
+		sts = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "mac_bacam_info fail (0x%08X)\n",
+			  ret);
+		sts = RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+
+	return sts;
+}
+
+/**
+ * Update TX Path cfg in CMAC control info by halmac api
+ * @hal_com: see struct rtw_hal_com_t
+ * @txmap_cfg: structure of tx path related information
+ *
+ * return RTW_HAL_STATUS_MAC_API_FAILURE if update fail
+ */
+enum rtw_hal_status rtw_hal_mac_tx_path_map_cfg(struct rtw_hal_com_t *hal_com,
+                                                struct hal_txmap_cfg *txmap_cfg)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 ret = 0;
+
+	ret = mac->ops->tx_path_map_cfg(mac, txmap_cfg);
+
+	if (0 == ret) {
+		sts = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "tx_path_map_cfg fail (0x%08X)\n",
+			  ret);
+		sts = RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+
+	return sts;
+}
+
+
+/**
+ * rtw_hal_mac_set_bw() - Update channel and bandwdith related setting
+ * @hal_info:	struct hal_info_t*
+ * @band_idx:	0x0: band0, 0x1: band1
+ * @ch:		center channel
+ * @band:	band
+ * @bw:		bandwidth
+ *
+ * All channel and bandwidth related MAC setting would be done in
+ * this function.
+ * Following setting may be done in this functions:
+ *	a. Enable changing CCK data rate to OFDM 6M function
+ *	   to avoid BB/RF abnormal when channel is not 2.4G.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when operation success.
+ */
+enum rtw_hal_status rtw_hal_mac_set_bw(struct hal_info_t *hal_info,
+			u8 band_idx, u8 pri_ch,	u8 central_ch_seg0,
+			u8 central_ch_seg1,	enum band_type band,
+			enum channel_width bw)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_cfg_bw mac_bw = {0};
+	u32 ret = 0;
+
+	mac_bw.cbw = bw;
+	mac_bw.band = band_idx;
+	mac_bw.pri_ch = pri_ch;
+	mac_bw.central_ch = central_ch_seg0;
+	mac_bw.band_type = band;
+	ret = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_BW_CFG, &mac_bw);
+	return (ret == 0) ? (RTW_HAL_STATUS_SUCCESS): (RTW_HAL_STATUS_FAILURE);
+}
+
+#ifdef CONFIG_PHL_BEAMFORM
+/**
+ * rtw_hal_mac_ax_init_bf_role
+ * @bf_role: 0 = BFEE, 1 = BFER
+ * @band: 0 = BAND0, 1 = BAND1
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_init_bf_role(struct rtw_hal_com_t *hal_com, u8 bf_role, u8 band)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac_info = hal_to_mac(hal_info);
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	if (bf_role == HAL_BF_ROLE_BFEE) {
+		hal_status = mac_info->ops->init_snd_mee(
+				mac_info, band);
+	} else {
+		hal_status = mac_info->ops->init_snd_mer(
+				mac_info, band);
+	}
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hal_status;
+}
+
+
+/**
+ * rtw_hal_mac_ax_disable_bfee
+ * @band: 0 = BAND0, 1 = BAND1
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_deinit_bfee(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "--> %s : Warning BFee is going to deinit\n", __func__);
+
+	hal_status = mac_info->ops->deinit_mee(mac_info, band);
+
+	return hal_status;
+}
+
+
+/**
+ * rtw_hal_mac_ax_bfee_para_reg
+ * 	Set BFee capability with STA info by method : Control Register
+ * input:
+ * @sta: (struct rtw_phl_stainfo_t *)
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_bfee_para_reg(void *mac, struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct rtw_hal_com_t *hal_com = (struct rtw_hal_com_t *)mac_info->drv_adapter;
+	struct mac_reg_csi_para csi_para = {0};
+	struct rtw_wifi_role_link_t *rlink = sta->rlink;
+
+	_os_mem_set(hal_com->drv_priv, &csi_para, 0, sizeof(csi_para));
+
+	csi_para.band = rlink->hw_band;
+	csi_para.portsel = (rlink->hw_port == 0) ? 0 : 1;
+	csi_para.nc = (rlink->protocol_cap.max_nc > sta->asoc_cap.num_snd_dim) ?
+		       sta->asoc_cap.num_snd_dim :
+		       rlink->protocol_cap.max_nc;
+	csi_para.nr = (rlink->protocol_cap.bfme_sts >
+		       sta->asoc_cap.num_snd_dim) ?
+		       sta->asoc_cap.num_snd_dim :
+		       rlink->protocol_cap.bfme_sts;
+
+	/**
+	 * For HE/VHT, Ng = 0 can provide the most detail information.
+	 * Ng do not care bfer cap.
+	 **/
+	csi_para.ng = 0;
+	/**
+	 * for HE/VHT, Cb = 1 {6,4}/{9,7} can provide the most detail information
+	 * Cb do not care bfer cap, only care bfee self capabiltiy.
+	 **/
+	if (sta->wmode & WLAN_MD_11AX)
+		csi_para.cb = rlink->protocol_cap.cb_sz_su_fb;
+	else if (sta->wmode & WLAN_MD_11AC)
+		csi_para.cb = rlink->protocol_cap.ht_vht_cb;
+	else
+		csi_para.cb = 0;
+
+	csi_para.cs = 1; /* Carrier Sense */
+	if (sta->asoc_cap.ht_ldpc &&
+	    sta->asoc_cap.vht_ldpc &&
+	    sta->asoc_cap.he_ldpc)
+		csi_para.ldpc_en = 1;
+
+	if (sta->asoc_cap.stbc_ht_rx &&
+	    sta->asoc_cap.stbc_vht_rx &&
+	    sta->asoc_cap.stbc_he_rx)
+		csi_para.stbc_en = 1;
+
+	csi_para.bf_en = 0;
+
+	hal_status = mac_info->ops->set_csi_para_reg(mac_info, &csi_para);
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_ax_bfee_para_cctl
+ * 	Set BFee capability with STA info by method : CMAC Control Table
+ * input:
+ * @sta: (struct rtw_phl_stainfo_t *)
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_bfee_para_cctl(void *mac, struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct rtw_hal_com_t *hal_com =
+		(struct rtw_hal_com_t *)mac_info->drv_adapter;
+	struct mac_cctl_csi_para csi_para = {0};
+	struct rtw_wifi_role_link_t *rlink = sta->rlink;
+
+	_os_mem_set(hal_com->drv_priv, &csi_para, 0, sizeof(csi_para));
+
+	csi_para.macid = (u8)sta->macid;
+	csi_para.band = rlink->hw_band;
+	csi_para.nc = (rlink->protocol_cap.max_nc > sta->asoc_cap.num_snd_dim) ?
+		       sta->asoc_cap.num_snd_dim :
+		       rlink->protocol_cap.max_nc;
+	csi_para.nr = (rlink->protocol_cap.bfme_sts >
+		       sta->asoc_cap.num_snd_dim) ?
+		       sta->asoc_cap.num_snd_dim :
+		       rlink->protocol_cap.bfme_sts;
+
+	/**
+	 * For HE/VHT, Ng = 0 can provide the most detail information.
+	 * Ng do not care bfer cap.
+	 **/
+	csi_para.ng = 0;
+	/**
+	 * for HE/VHT, Cb = 1 {6,4}/{9,7} can provide the most detail information
+	 * Cb do not care bfer cap.
+	 **/
+	if (sta->wmode & WLAN_MD_11AX)
+		csi_para.cb = rlink->protocol_cap.cb_sz_su_fb;
+	else if (sta->wmode & WLAN_MD_11AC)
+		csi_para.cb = rlink->protocol_cap.ht_vht_cb;
+	else
+		csi_para.cb = 0;
+	csi_para.cs = 1;
+	csi_para.bf_en = 0;
+
+	if (sta->asoc_cap.stbc_ht_rx &&
+	    sta->asoc_cap.stbc_vht_rx &&
+	    sta->asoc_cap.stbc_he_rx)
+		csi_para.stbc_en = 1;
+
+	if (sta->asoc_cap.ht_ldpc &&
+	    sta->asoc_cap.vht_ldpc &&
+	    sta->asoc_cap.he_ldpc)
+		csi_para.ldpc_en = 1;
+	csi_para.rate = MAC_AX_OFDM54;
+	csi_para.gi_ltf = MAC_AX_SGI_4XHE08;
+	csi_para.gid_sel = 1;
+	csi_para.bw = MAC_AX_BW_20M;
+
+	hal_status = mac_info->ops->set_csi_para_cctl(mac_info, &csi_para);
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_ax_bfee_set_csi_rrsc(void *mac, u8 band, u32 rrsc)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+
+	hal_status = mac_info->ops->csi_rrsc(mac_info, band, rrsc);
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_ax_bfee_forced_csi_rate
+ * 	set bf report frame rate
+ * @mac:(struct mac_ax_adapter *)
+ * @ht_rate:
+ * @vht_rate:
+ * @he_rate:
+ */
+
+enum rtw_hal_status
+rtw_hal_mac_ax_bfee_forced_csi_rate(void *mac, struct rtw_phl_stainfo_t *sta,
+	u8 ht_rate, u8 vht_rate, u8 he_rate)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+
+	hal_status = mac_info->ops->csi_force_rate(mac_info,
+	                                           sta->rlink->hw_band,
+	                                           ht_rate,
+	                                           vht_rate,
+	                                           he_rate);
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_ax_set_bf_entry
+ * 	set HW BF entry for sounding and TxBF
+ * input :
+ * @band: BF Entry is band0 or band1;
+ * @macid: BF Entry's macid
+ * @bfee_idx: SU/MU HW Entry Index
+ * @txbf_idx: Tx BF CSI Entry Index
+ * @buffer_idx: CSI Buffer idx used by TxBF entry
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_set_bf_entry(void *mac, u8 band,
+		u8 macid, u8 bfee_idx, u8 txbf_idx, u16 buffer_idx)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	/* 1. CSI Buffer Idx */
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "set_csi_buffer_index : band 0x%x macid 0x%x txbf_idx 0x%x buffer_idx 0x%x\n",
+				       band, macid, txbf_idx, buffer_idx);
+	hal_status = mac_info->ops->set_csi_buffer_index(mac_info, band, macid,
+							 txbf_idx, buffer_idx);
+
+
+	/*TODO: this api might revised in the future */
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "set_snd_sts_index: band 0x%x macid 0x%x bf_idx 0x%x\n",
+				band, macid, bfee_idx);
+	hal_status = mac_info->ops->set_snd_sts_index(
+					mac_info, band, macid, bfee_idx);
+
+
+	return hal_status;
+}
+
+
+/**
+ * rtw_hal_mac_ax_get_snd_sts
+ * 	Get HW BF entry sounding status
+ * input :
+ * @band: BF Entry is band0 or band1;
+ * @bfee_idx: SU/MU HW Entry Index
+ * return
+ * @hal_status: enum rtw_hal_status
+ * 	RTW_HAL_STATUS_SUCCESS = Sounding Success
+ * 	RTW_HAL_STATUS_FAILURE = Sounding Fail
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_get_snd_sts(void *mac, u8 band, u8 bfee_idx)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	u32 sts = 0;
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "get_snd_sts_index: band 0x%x bf_idx 0x%x\n", band, bfee_idx);
+	/*TODO: This API shall modify to return sounding status instead of CR value*/
+	/*MAC Define : #define B_AX_MER_SU_BFMEE0_SND_STS BIT(9)*/
+	sts = mac_info->ops->get_snd_sts_index(mac_info, band, bfee_idx);
+	if (sts & B_AX_MER_SU_BFMEE0_SND_STS)
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_ax_hw_snd_control
+ * @band: 0 = BAND0, 1 = BAND1
+ * @hw_snd_ctrl: 0 = HW_SND_PAUSE 1 = HW_SND_RELEASE
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_hw_snd_control(
+	void *mac,
+	u8 band,
+	u8 hw_snd_ctrl)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+		  "mac_hw_snd_pause_release: band 0x%x hw_snd_ctrl 0x%x\n",
+		  band, hw_snd_ctrl);
+
+	hal_status = mac_info->ops->hw_snd_pause_release(
+				mac_info,
+				band,
+				hw_snd_ctrl);
+
+	return hal_status;
+}
+
+/* Tx Frame Exchange Related : MU */
+/**
+ * rtw_hal_mac_ax_mu_sta_upd
+ * @mac:  (struct mac_ax_adapter *)
+ * @macid: sta macid for configuration
+ * @bfmu_idx: 0~5, MU STA Index
+ * @prot_type: RTS/CTS type for the group : enum rtw_hal_protection_type
+ * @resp_type: Ack Policy for the group : enum rtw_hal_ack_resp_type
+ * @grp_bitmap: group bitmap for STA,
+ **/
+enum rtw_hal_status
+rtw_hal_mac_ax_mu_sta_upd(void *mac, u8 macid, u8 bfmu_idx,
+			enum rtw_hal_protection_type prot_type,
+			enum rtw_hal_ack_resp_type resp_type, u8 mugrp_bm)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct rtw_hal_com_t *hal_com =
+		(struct rtw_hal_com_t *)mac_info->drv_adapter;
+	struct mac_ax_mu_sta_upd sta_info = {0};
+	u8 i = 0;
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "==> rtw_hal_mac_ax_mu_sta_upd \n");
+	_os_mem_set(hal_com->drv_priv, &sta_info, 0, sizeof(sta_info));
+
+	sta_info.macid = macid;
+	sta_info.mu_idx = bfmu_idx;
+	for (i = 0; i < 5; i++) {
+		if (mugrp_bm & BIT(i)) {
+			sta_info.prot_rsp_type[i].u.feld_type.protect =
+				prot_type & 0xF;
+			sta_info.prot_rsp_type[i].u.feld_type.rsp =
+				resp_type & 0xF;
+		} else {
+			sta_info.prot_rsp_type[i].u.feld_type.protect = 0;
+			sta_info.prot_rsp_type[i].u.feld_type.rsp = 0;
+		}
+	}
+	sta_info.mugrp_bitmap = mugrp_bm & 0x1F;
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "sta_info.macid = 0x%x \n", sta_info.macid);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "sta_info.mu_idx = 0x%x \n", sta_info.mu_idx);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "sta_info.mugrp_bitmap = 0x%x \n", sta_info.mugrp_bitmap);
+
+	hal_status = mac_info->ops->mu_sta_upd(mac_info, &sta_info);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "<== rtw_hal_mac_ax_mu_sta_upd \n");
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_ax_mu_decision_para
+ * @mac:  (struct mac_ax_adapter *)
+ * @mu_thold:  MU MIMO pkt Threshold
+ * @bypass_thold: by pass mu_thold
+ * @bypass_tp: by pass MU TP > SU TP check.
+ **/
+enum rtw_hal_status
+rtw_hal_mac_ax_mu_decision_para(void *mac, u32 mu_thold,
+				bool bypass_thold, bool bypass_tp)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct rtw_hal_com_t *hal_com =
+		(struct rtw_hal_com_t *)mac_info->drv_adapter;
+	struct mac_ax_mudecision_para mu_d_para = {0};
+
+	_os_mem_set(hal_com->drv_priv, &mu_d_para, 0, sizeof(mu_d_para));
+
+	mu_d_para.mu_thold = mu_thold;
+	mu_d_para.bypass_thold = bypass_thold ? 1 : 0;
+	mu_d_para.bypass_tp = bypass_tp ? 1 : 0;
+
+	hal_status = mac_info->ops->upd_mudecision_para(mac_info, &mu_d_para);
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_ax_set_mu_fix_mode
+ * @mac:  (struct mac_ax_adapter *)
+ * @gid:  GID for STA X + STAY
+ * @prot_type: RTS/CTS type for the group : enum rtw_hal_protection_type
+ * @resp_type: Ack Policy for the group : enum rtw_hal_ack_resp_type
+ * @fix_mu: true = Fix FW decision = MU
+ * @he: true = Fix TX HE MU, false = Fix TX VHT MU;
+ * @fix_resp: fix frame exchange ack policy
+ * @fix_prot: fix frame exchange protection type
+ **/
+enum rtw_hal_status
+rtw_hal_mac_ax_set_mu_fix_mode(
+	void *mac, u8 gid, enum rtw_hal_protection_type prot_type,
+	enum rtw_hal_ack_resp_type resp_type,
+	bool fix_mu, bool he, bool fix_resp, bool fix_prot)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct rtw_hal_com_t *hal_com =
+		(struct rtw_hal_com_t *)mac_info->drv_adapter;
+	struct mac_ax_fixmode_para fix_info = {0};
+	PHL_INFO("===>rtw_hal_mac_ax_set_mu_fix_mode\n");
+
+	_os_mem_set(hal_com->drv_priv, &fix_info, 0, sizeof(fix_info));
+
+	fix_info.force_sumuru_en = fix_mu ? 1:0 ;
+	fix_info.forcemu = fix_mu ? 1:0 ;
+
+	if (fix_mu) {
+		if (!he){
+			fix_info.fix_fe_vhtmu_en = fix_prot ? 1 : 0;
+			fix_info.fix_frame_seq_vhtmu = fix_resp ? 1 : 0;
+			fix_info.prot_type_vhtmu = fix_prot ? prot_type : 5;/*default hw setting*/
+			fix_info.resp_type_vhtmu = fix_resp ? resp_type : 4;/*default hw setting*/
+			PHL_INFO("fix_info.prot_type_vhtmu = 0x%x\n", fix_info.prot_type_vhtmu);
+			PHL_INFO("fix_info.resp_type_vhtmu = 0x%x\n", fix_info.resp_type_vhtmu);
+
+		} else {
+			fix_info.fix_fe_hemu_en = fix_prot ? 1 : 0;;
+			fix_info.fix_frame_seq_hemu = fix_resp ? 1 : 0;
+			fix_info.prot_type_hemu = fix_prot ? prot_type : 5;/*default hw setting*/
+			fix_info.resp_type_hemu = fix_resp ? resp_type : 4;/*default hw setting*/
+			PHL_INFO("fix_info.prot_type_hemu = 0x%x\n", fix_info.prot_type_hemu);
+			PHL_INFO("fix_info.resp_type_hemu = 0x%x\n", fix_info.resp_type_hemu);
+		}
+
+		fix_info.mugrpid = gid;
+		PHL_INFO("fix_info.mugrpid = 0x%x\n", fix_info.mugrpid);
+	}
+
+	hal_status = mac_info->ops->set_fw_fixmode(mac_info, &fix_info);
+	PHL_INFO("<===rtw_hal_mac_ax_set_mu_fix_mode\n");
+	return hal_status;
+}
+#else
+
+enum rtw_hal_status
+rtw_hal_mac_ax_init_bf_role(struct rtw_hal_com_t *hal_com, u8 bf_role, u8 band)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_ax_deinit_bfee(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#endif
+
+
+void
+_hal_mac_fill_mu_sc_tbl_row(u32 *mac_score, void *hal_score)
+{
+	struct hal_mu_score_tbl_score *h_score =
+		(struct hal_mu_score_tbl_score *)hal_score;
+
+	*mac_score = (u32)h_score->score[0] |
+		     ((u32)h_score->score[1] << 8) | ((u32)h_score->valid << 10) |
+		     ((u32)h_score->macid << 11);
+
+}
+/**
+ * rtw_hal_mac_ax_set_mu_table_whole
+* @mac:  (struct mac_ax_adapter *)
+ *@hal_score_tbl:  struct hal_mu_score_tbl *
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_set_mu_table_whole(void *mac, void *hal_score_tbl)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac_info = (struct mac_ax_adapter *)mac;
+	struct rtw_hal_com_t *hal_com =
+		(struct rtw_hal_com_t *)mac_info->drv_adapter;
+	struct mac_mu_table mu_table = {0};
+	struct hal_mu_score_tbl *score_tbl = (struct hal_mu_score_tbl *)hal_score_tbl;
+	_os_mem_set(hal_com->drv_priv, &mu_table, 0, sizeof(mu_table));
+
+	/*TODO: halmac api shall refine!!!*/
+	mu_table.mu_score_tbl_ctrl = (score_tbl->mu_ctrl.mu_sc_thr) |
+				     (score_tbl->mu_ctrl.mu_opt << 2);
+
+	/*TODO: if next IC has more than 6 MU STAs!!! */
+	_hal_mac_fill_mu_sc_tbl_row(&mu_table.mu_score_tbl_0, &score_tbl->mu_score[0]);
+	_hal_mac_fill_mu_sc_tbl_row(&mu_table.mu_score_tbl_1, &score_tbl->mu_score[1]);
+	_hal_mac_fill_mu_sc_tbl_row(&mu_table.mu_score_tbl_2, &score_tbl->mu_score[2]);
+	_hal_mac_fill_mu_sc_tbl_row(&mu_table.mu_score_tbl_3, &score_tbl->mu_score[3]);
+	_hal_mac_fill_mu_sc_tbl_row(&mu_table.mu_score_tbl_4, &score_tbl->mu_score[4]);
+	_hal_mac_fill_mu_sc_tbl_row(&mu_table.mu_score_tbl_5, &score_tbl->mu_score[5]);
+
+	hal_status = mac_info->ops->set_mu_table(mac_info, &mu_table);
+
+	return hal_status;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_parse_c2h(void *hal, u8 *buf, u32 buf_len, void *c2h)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+
+	hal_status = mac->ops->process_c2h(mac, buf, buf_len, (u8 *)c2h);
+
+	return hal_status;
+}
+
+/**
+ * Required information in (hal_handle_rx_buffer_XXXXX case RX_DESC_PKT_TYPE_PPDU_STATUS),
+ * it cannot be used by core/phl/other hal module
+ **/
+void
+_hal_mac_ax_ppdu_sts_to_hal_ppdu_sts(
+	struct mac_ax_ppdu_rpt *mac_ppdu, void *hal_ppdu_sts)
+{
+	struct hal_ppdu_sts *hal_ppdu =
+			(struct hal_ppdu_sts *)hal_ppdu_sts;
+	u8 i = 0;
+	u32 j = 0;
+
+	hal_ppdu->rx_cnt_ptr = mac_ppdu->rx_cnt_ptr;
+	hal_ppdu->phy_st_ptr = mac_ppdu->phy_st_ptr;
+	hal_ppdu->plcp_ptr = mac_ppdu->plcp_ptr;
+	hal_ppdu->plcp_size = mac_ppdu->plcp_size;
+	hal_ppdu->phy_st_size = mac_ppdu->phy_st_size;
+	hal_ppdu->rx_cnt_size = mac_ppdu->rx_cnt_size;
+	hal_ppdu->usr_num = mac_ppdu->usr_num;
+	for(i = 0; i < mac_ppdu->usr_num; i++) {
+		if(1 == mac_ppdu->usr[i].vld) {
+			hal_ppdu->usr[i].vld = 1;
+			hal_ppdu->usr[i].macid = mac_ppdu->usr[i].macid;
+			hal_ppdu->usr[i].has_data = mac_ppdu->usr[i].has_data;
+			hal_ppdu->usr[i].has_ctrl = mac_ppdu->usr[i].has_ctrl;
+			hal_ppdu->usr[i].has_mgnt = mac_ppdu->usr[i].has_mgnt;
+			hal_ppdu->usr[i].has_bcn = mac_ppdu->usr[i].has_bcn;
+		}
+	}
+	/* process / decode rx cnt report */
+	/* TODO: Halmac api shall provid decoder */
+	if ((0 != hal_ppdu->rx_cnt_size) && (NULL != hal_ppdu->rx_cnt_ptr)) {
+		for(j = 0; (j < (hal_ppdu->rx_cnt_size/2)) &&
+		    (j < HAL_RXCNT_MAX); j++) {
+			hal_ppdu->rx_cnt.ppdu_cnt[j] =
+				((u16)*(hal_ppdu->rx_cnt_ptr + 2 * j));
+		}
+	}
+}
+
+/**
+ * if any information is required for other core/phl module,
+ * copy to rx meta data or hal_info from halmac ax ppdu status.
+ **/
+void
+_hal_mac_ax_ppdu_sts_to_hal_info(struct hal_info_t *hal_info,
+	struct mac_ax_ppdu_rpt *mac_ppdu, void *rx_mdata)
+{
+	/* struct rtw_r_meta_data *mdata =
+			(struct rtw_r_meta_data *)rx_mdata; */
+	return;
+}
+/**
+ * rtw_hal_mac_ax_parse_ppdu_sts
+ * @hal:(struct hal_info_t *)
+ * @mac_valid:if mac information invalid (from rx desc)
+ * @buf: pointer of ppdu status, point to header of mac_info
+ * @buf_l:ppdu status payload size
+ * @ppdu_sts: (struct hal_ppdu_sts *) for return value to hal
+ * @rx_mdata: (struct rtw_r_meta_data *) for saving ppdu status
+ */
+enum rtw_hal_status
+rtw_hal_mac_ax_parse_ppdu_sts(void *hal, u8 mac_valid, u8 *buf, u16 buf_l,
+			      void *ppdu_sts, void *rx_mdata)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_ppdu_rpt ppdu_rpt = {0};
+
+	PHL_DBG("%s\n", __FUNCTION__);
+
+	hal_status = mac->ops->parse_ppdu(hal_info->mac, buf, buf_l,
+					  mac_valid, &ppdu_rpt);
+
+	if (hal_status == RTW_HAL_STATUS_SUCCESS) {
+		_hal_mac_ax_ppdu_sts_to_hal_ppdu_sts(&ppdu_rpt, ppdu_sts);
+		_hal_mac_ax_ppdu_sts_to_hal_info(hal_info ,&ppdu_rpt,
+						 rx_mdata);
+	}
+
+	return hal_status;
+}
+
+
+/**
+ * the function to enable HW header conversion function
+ * @hal_info: see struct hal_info_t
+ * @en_hdr_conv: true to enable, false to disable
+ *
+ * return RTW_HAL_STATUS_MAC_API_FAILURE if update fail
+ */
+enum rtw_hal_status rtw_hal_hdr_conv_cfg(struct hal_info_t *hal_info,
+				      u8 en_hdr_conv)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 ret = 0;
+
+	ret = mac->ops->hdr_conv(mac, en_hdr_conv);
+
+	if (MACSUCCESS == ret) {
+		sts = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "hdr_conv fail (0x%08X)\n",
+			  ret);
+		sts = RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+
+	return sts;
+}
+
+
+#ifdef RTW_PHL_BCN //fill hal mac ops
+enum rtw_hal_status
+hal_mac_ax_config_beacon(struct hal_info_t *hal, struct rtw_bcn_entry *bcn_entry)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct rtw_bcn_info_cmn *bcn_cmn = bcn_entry->bcn_cmn;
+	struct rtw_bcn_info_hw *bcn_hw = &bcn_entry->bcn_hw;
+	enum mac_ax_port_cfg_type ptype;
+	struct mac_ax_port_cfg_para ppara = {0};
+
+	ppara.band = bcn_hw->band;
+	ppara.port = bcn_hw->port;
+	ppara.mbssid_idx = bcn_hw->mbssid;
+
+	ptype = MAC_AX_PCFG_BCN_INTV;
+	ppara.val = bcn_cmn->bcn_interval;
+	if (mac->ops->port_cfg(mac, ptype, &ppara) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	ptype = MAC_AX_PCFG_HIQ_DTIM;
+	ppara.val = bcn_cmn->bcn_dtim;
+	if (mac->ops->port_cfg(mac, ptype, &ppara) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+#ifdef CONFIG_RTW_DEBUG_BCN_TX
+	/* enable bcn report */
+	ptype = MAC_AX_PCFG_TX_RPT;
+	ppara.val = true;
+	if (mac->ops->port_cfg(mac, ptype, &ppara) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+#endif /* CONFIG_RTW_DEBUG_BCN_TX */
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+hal_mac_ax_send_beacon(struct hal_info_t *hal, struct rtw_bcn_entry *bcn_entry)
+{
+
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct rtw_bcn_info_cmn *bcn_cmn = bcn_entry->bcn_cmn;
+	struct rtw_bcn_info_hw *bcn_hw = &bcn_entry->bcn_hw;
+	struct mac_ax_bcn_info info = {0};
+
+	if(!mac->ops->send_bcn_h2c)
+		return RTW_HAL_STATUS_FAILURE;
+
+	info.pld_buf = bcn_cmn->bcn_buf;
+	info.pld_len = (u16)bcn_cmn->bcn_length;
+
+	info.band = bcn_hw->band;
+	info.port = bcn_hw->port;
+	info.mbssid = bcn_hw->mbssid;
+	info.grp_ie_ofst = (u8)bcn_cmn->ie_offset_tim;
+	info.macid = bcn_hw->mac_id;
+#ifdef CONFIG_PHL_DFS_SWITCH_CH_WITH_CSA
+	/* BIT0 => enable, BIT1 - BIT14 => csa ie countdown offset */
+	if (bcn_cmn->offset_csa_cntdown)
+		info.csa_ofst = (u16)((bcn_cmn->offset_csa_cntdown << 1) | BIT(0));
+#endif
+
+	if(bcn_cmn->bcn_offload & BIT(BCN_HW_TIM))
+		info.grp_ie_ofst |= BIT(7);
+
+	if(bcn_cmn->bcn_offload & BIT(BCN_HW_SEQ)){
+		info.ssn_sel = 1;
+		info.ssn_mode = 1;
+	}
+	else {
+		info.ssn_sel = 0;
+		info.ssn_mode = 0;
+	}
+
+	info.rate_sel = (u16)bcn_cmn->bcn_rate;
+
+	mac->ops->send_bcn_h2c(mac, &info);
+
+	return RTW_HAL_STATUS_SUCCESS;
+
+}
+#endif //RTW_PHL_BCN
+
+enum rtw_hal_status
+rtw_hal_mac_ppdu_stat_cfg(struct hal_info_t *hal_info,
+			  struct hal_ppdu_sts_cfg *cfg)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_phy_rpt_cfg mac_cfg = {0};
+
+	mac_cfg.en = cfg->ppdu_stat_en;
+	mac_cfg.type = MAC_AX_PPDU_STATUS;
+	/*cfg.dest = MAC_AX_PRPT_DEST_HOST;*/
+
+	mac_cfg.u.ppdu.band = cfg->band_idx;
+	if (cfg->ppdu_stat_en) {
+		if (cfg->appen_info&HAL_PPDU_MAC_INFO)
+			mac_cfg.u.ppdu.bmp_append_info |= MAC_AX_PPDU_MAC_INFO;
+		if (cfg->appen_info&HAL_PPDU_PLCP)
+			mac_cfg.u.ppdu.bmp_append_info |= MAC_AX_PPDU_PLCP;
+		if (cfg->appen_info&HAL_PPDU_RX_CNT)
+			mac_cfg.u.ppdu.bmp_append_info |= MAC_AX_PPDU_RX_CNT;
+
+		if (cfg->filter&HAL_PPDU_HAS_A1M)
+			mac_cfg.u.ppdu.bmp_filter |= MAC_AX_PPDU_HAS_A1M;
+		if (cfg->filter&HAL_PPDU_HAS_CRC_OK)
+			mac_cfg.u.ppdu.bmp_filter |= MAC_AX_PPDU_HAS_CRC_OK;
+		if (cfg->filter&HAL_PPDU_HAS_DMA_OK)
+			mac_cfg.u.ppdu.bmp_filter |= MAC_AX_PPDU_HAS_DMA_OK;
+
+		mac_cfg.u.ppdu.dup2fw_en = false;
+		mac_cfg.u.ppdu.dup2fw_len = 0;
+	}
+
+	if (mac->ops->cfg_phy_rpt(mac, &mac_cfg) != MACSUCCESS) {
+		PHL_ERR("%s fault\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_config_hw_mgnt_sec(struct hal_info_t *hal_info, u8 en)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+
+	hal_mac_ops->sta_hw_security_support(mac, SEC_UC_MGNT_ENC, en);
+	hal_mac_ops->sta_hw_security_support(mac, SEC_BMC_MGNT_ENC, en);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_get_append_fcs(struct hal_info_t *hal_info, u8 *val)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+
+	if (ops->get_hw_value(mac, MAC_AX_HW_GET_APP_FCS, val) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	else
+		return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_append_fcs(struct hal_info_t *hal_info, u8 enable)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_APP_FCS, &enable);
+	if (ret != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_get_acpt_icv_err(struct hal_info_t *hal_info, u8 *val)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+
+	if (ops->get_hw_value(mac, MAC_AX_HW_GET_RX_ICVERR, val) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	else
+		return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+void
+_phl_chinfo_2_mac_chainfo(struct mac_ax_phy_rpt_cfg *cfg,
+				enum phl_chinfo_mode mode, u8 filter, u8 sg_size)
+{
+	switch (mode) {
+	case CHINFO_MODE_ACK:
+		/* depand on ack_ch_info of TXD */
+		cfg->u.chif.trigger = MAC_AX_CH_INFO_ACK;
+		cfg->u.chif.bmp_filter = 0;
+		break;
+	case CHINFO_MODE_MACID:
+		cfg->u.chif.trigger = MAC_AX_CH_INFO_MACID;
+		cfg->u.chif.bmp_filter = MAC_AX_CH_INFO_DATA_FRM;
+		break;
+	case CHINFO_MODE_NDP:
+		cfg->u.chif.trigger = MAC_AX_CH_INFO_NDP;
+		break;
+	case CHINFO_MODE_MAX:
+		cfg->u.chif.trigger = MAC_AX_CH_INFO_MACID;
+		cfg->u.chif.bmp_filter = MAC_AX_CH_INFO_DATA_FRM;
+		PHL_INFO("%s unknown mode(%d)\n", __func__, mode);
+		break;
+	}
+	cfg->u.chif.seg_size = MAC_AX_CH_IFNO_SEG_512;
+}
+enum rtw_hal_status
+rtw_hal_mac_chan_info_cfg(struct hal_info_t *hal_info,
+				bool chinfo_en, u16 macid,
+				enum phl_chinfo_mode mode, u8 filter, u8 sg_size)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_phy_rpt_cfg cfg = {0};
+
+	cfg.en = chinfo_en;
+	cfg.type = MAC_AX_CH_INFO;
+	/*cfg.dest = MAC_AX_PRPT_DEST_HOST;*/
+	cfg.u.chif.macid = (u8)macid;
+
+	if (chinfo_en)
+		_phl_chinfo_2_mac_chainfo(&cfg, mode, filter, sg_size);
+
+	if (mac->ops->cfg_phy_rpt(mac, &cfg) != MACSUCCESS) {
+		PHL_ERR("%s fault\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_CSI_FW_TX_OFLD
+#define MAC_CFG_SENSING_CSI_TIMEOUT_MS 500
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_sensing_csi(struct hal_info_t *hal_info,
+							u16 macid, u8 en, u16 period, u8 retry_cnt,
+							u16 rate, u8 pkt_num, u8 *pkt_id)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct rtw_hal_mac_sensing_csi_param para = {0};
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	struct dev_cap_t *dev_cap = &phl_com->dev_cap;
+	u32 start_time = _os_get_cur_time_ms();
+
+	if (dev_cap->sensing_csi == false) {
+		PHL_ERR("%s not support sensing csi\n", __func__);
+		_os_warn_on(1);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	para.macid = (u8)macid;
+	para.en = en;
+	para.period = period;
+	para.retry_cnt = retry_cnt;
+	para.rate = rate;
+	para.pkt_num = pkt_num;
+	para.pkt_id = pkt_id;
+
+	if (mac->ops->cfg_sensing_csi(mac, &para) != MACSUCCESS) {
+		PHL_ERR("%s cfg_sensing_csi fail\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	while (mac->ops->chk_sensing_csi_done(mac, en) == MACPROCBUSY) {
+		if (phl_get_passing_time_ms(start_time) >= MAC_CFG_SENSING_CSI_TIMEOUT_MS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal_info), 10);
+	}
+	if (mac->ops->chk_sensing_csi_done(mac, en) != MACSUCCESS) {
+		PHL_ERR("%s chk_sensing_csi_done fail\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+void rtw_hal_mac_dbg_status_dump(struct hal_info_t *hal, struct hal_mac_dbg_dump_cfg *cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_dbgpkg pkg = {0};
+	struct mac_ax_dbgpkg_en pkg_en = {0};
+
+	pkg.ss_dbg_0 = cfg->ss_dbg_0;
+	pkg.ss_dbg_1 = cfg->ss_dbg_1;
+
+	pkg_en.ss_dbg = cfg->ss_dbg;
+	pkg_en.dle_dbg = cfg->dle_dbg;
+	pkg_en.dmac_dbg = cfg->dmac_dbg;
+	pkg_en.cmac_dbg = cfg->cmac_dbg;
+	pkg_en.mac_dbg_port = cfg->mac_dbg_port;
+	pkg_en.plersvd_dbg = cfg->plersvd_dbg;
+	pkg_en.tx_flow_dbg = cfg->tx_flow_dbg;
+
+	PHL_INFO("%s: ss_dbg_0 %d, ss_dbg_1 %d, ss_dbg %d\n", __func__, pkg.ss_dbg_0, pkg.ss_dbg_1, pkg_en.ss_dbg);
+	PHL_INFO("%s: dle_dbg %d, dmac_dbg %d, cmac_dbg %d\n", __func__, pkg_en.dle_dbg, pkg_en.dmac_dbg, pkg_en.cmac_dbg);
+	PHL_INFO("%s: mac_dbg_port %d, plersvd_dbg %d, tx_flow_dbg %d\n", __func__, pkg_en.mac_dbg_port, pkg_en.plersvd_dbg, pkg_en.tx_flow_dbg);
+
+#ifdef CONFIG_HAL_MAC_DBG
+	mac->ops->dbg_status_dump(mac, &pkg, &pkg_en);
+#else
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+#endif
+}
+
+#ifdef CONFIG_PHL_DFS
+u32 rtw_hal_mac_cfg_dfs_rpt(struct rtw_hal_com_t *hal_com, struct hal_mac_dfs_rpt_cfg *conf)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_phy_rpt_cfg cfg = {0};
+	u32 result;
+
+	cfg.en = conf->rpt_en;
+	cfg.type = MAC_AX_DFS;
+	/*cfg.dest = MAC_AX_PRPT_DEST_HOST;*/
+
+	if (conf->rpt_en) {
+		cfg.u.dfs.num_th = conf->rpt_num_th;
+		cfg.u.dfs.en_timeout = conf->rpt_en_to;
+		cfg.u.dfs.dfs_to = conf->rpt_to;
+	}
+
+	result = mac->ops->cfg_phy_rpt(mac, &cfg);
+	if (result != MACSUCCESS)
+		PHL_ERR("%s fault\n", __func__);
+
+	return result;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_dfs_rpt_cfg(struct hal_info_t *hal_info, struct hal_mac_dfs_rpt_cfg *conf)
+{
+	if (rtw_hal_mac_cfg_dfs_rpt(hal_info->hal_com, conf) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_parse_dfs(struct hal_info_t *hal_info,
+			u8 *buf, u32 buf_len, struct mac_ax_dfs_rpt *dfs_rpt)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+
+	if (mac->ops->parse_dfs(mac, buf, buf_len, dfs_rpt) != MACSUCCESS) {
+		PHL_ERR("%s fault\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /*CONFIG_PHL_DFS*/
+
+#ifdef CONFIG_PHL_PKTOFLD
+enum rtw_hal_status
+_hal_mac_get_pkt_ofld(struct hal_info_t *hal_info, u8 *id)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u8 *pkt_buf = NULL;
+	u16 pkt_len;
+
+	if(mac == NULL)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	if (mac->ops->pkt_ofld_packet(mac, &pkt_buf, &pkt_len, id) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	debug_dump_data((u8 *)pkt_buf, pkt_len, "pkt ofld");
+
+	_os_mem_free(hal_com->drv_priv, pkt_buf, pkt_len);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+_hal_mac_chk_pkt_ofld(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	void *d = hal_to_drvpriv(hal_info);
+	u16 loop_cnt = 0;
+
+	if(mac == NULL)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	do
+	{
+		if (mac->ops->check_fwofld_done(mac, 1) == MACSUCCESS)
+			break;
+
+		_os_sleep_ms(d, POLLING_HALMAC_TIME);
+
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if ( loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_PRINT("%s, check count = %d.\n", __func__, loop_cnt);
+		return RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_ERR("%s, polling timeout!!!\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+}
+
+enum rtw_hal_status
+_hal_mac_add_pkt_ofld(struct hal_info_t *hal_info, u8 *pkt, u16 len, u8 *id)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 status;
+
+	if(mac == NULL)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	PHL_PRINT("%s: len %d.\n", __func__, len);
+
+	status = mac->ops->add_pkt_ofld(mac, pkt, len, id);
+	if (status != MACSUCCESS) {
+		PHL_ERR("%s fault, status = %d.\n", __func__, status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	PHL_PRINT("%s: id %d.\n", __func__, *id);
+
+	status = _hal_mac_chk_pkt_ofld(hal_info);
+	return status;
+}
+
+enum rtw_hal_status
+_hal_mac_del_pkt_ofld(struct hal_info_t *hal_info, u8 *id)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 status;
+
+	if(mac == NULL)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	PHL_PRINT("%s: id %d.\n", __func__, *id);
+
+	status = mac->ops->del_pkt_ofld(mac, *id);
+	if (status != MACSUCCESS) {
+		PHL_ERR("%s fault, status = %d.\n", __func__, status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	status = _hal_mac_chk_pkt_ofld(hal_info);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+_hal_mac_read_pkt_ofld(struct hal_info_t *hal_info, u8 *id)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 status;
+
+	if(mac == NULL)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	PHL_PRINT("%s: id %d.\n", __func__, *id);
+
+	status = mac->ops->read_pkt_ofld(mac, *id);
+	if (status != MACSUCCESS) {
+		PHL_ERR("%s fault, status = %d.\n", __func__, status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	status = _hal_mac_chk_pkt_ofld(hal_info);
+	if (status != MACSUCCESS) {
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	status = _hal_mac_get_pkt_ofld(hal_info, id);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_pkt_ofld(struct hal_info_t *hal, u8 *id, u8 op,
+						u8 *pkt, u16 *len)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	switch(op) {
+	case PKT_OFLD_ADD:
+		status = _hal_mac_add_pkt_ofld(hal, pkt, *len, id);
+		break;
+
+	case PKT_OFLD_DEL:
+		status = _hal_mac_del_pkt_ofld(hal, id);
+		break;
+
+	case PKT_OFLD_READ:
+		status = _hal_mac_read_pkt_ofld(hal, id);
+		break;
+
+	default:
+		PHL_ERR("%s op(%d) not define.\n", __func__, op);
+		break;
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_mac_pkt_update_ids(struct hal_info_t *hal,
+						struct pkt_ofld_entry *entry)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal->mac;
+	struct mac_ax_general_pkt_ids mac_ids = {0};
+	u32 status;
+
+	mac_ids.macid = (u8)entry->macid;
+	mac_ids.probersp = entry->pkt_info[PKT_TYPE_PROBE_RSP][0].id;
+	mac_ids.pspoll = entry->pkt_info[PKT_TYPE_PS_POLL][0].id;
+	mac_ids.nulldata = entry->pkt_info[PKT_TYPE_NULL_DATA][0].id;
+	mac_ids.qosnull = entry->pkt_info[PKT_TYPE_QOS_NULL][0].id;
+	mac_ids.cts2self = entry->pkt_info[PKT_TYPE_CTS2SELF][0].id;
+
+	PHL_PRINT("macid %d, probersp %d, pspoll %d, nulldata %d, qosnull %d, cts2self %d.\n",
+		mac_ids.macid,
+		mac_ids.probersp,
+		mac_ids.pspoll,
+		mac_ids.nulldata,
+		mac_ids.qosnull,
+		mac_ids.cts2self);
+
+	status = mac->ops->general_pkt_ids(mac, &mac_ids);
+	if (status != MACSUCCESS) {
+		PHL_ERR("%s fault, status = %d.\n", __func__, status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_reset_pkt_ofld_state(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+
+	if (mac == NULL)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	if (mac->ops->reset_fwofld_state(mac, 1) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	else
+		return RTW_HAL_STATUS_SUCCESS;
+
+}
+#endif
+
+/* comment temporary and review it later */
+#if 0
+u4Byte hal_mac_ax_dbg_h2cpkt_lb(RT_HAL_MAC_INFO *hm_info, u32 size)
+{
+	PADAPTER adapter = hm_info->adapter;
+	u32 ret = 0;
+
+	u8 *h2cbuf = NULL;
+	FunctionIn(COMP_HAL_MAC_API);
+
+	//ret = hm_info->halmac_ax_ops->fwcmd_lb(hm_info->halmac_ax_apter,
+	//	100, 0);
+
+	PlatformAllocateMemory(adapter, (PVOID *)&h2cbuf, size);
+	PlatformZeroMemory(h2cbuf,size);
+
+	for (u4Byte tmpc = 0; tmpc < size - 32; tmpc++) {
+		h2cbuf[32 + tmpc] = (u8)tmpc & 0xFF;
+	}
+
+	hal_mac_ax_send_h2c_pkt(adapter, h2cbuf, size);
+
+	PlatformFreeMemory(h2cbuf, size);
+
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+
+
+u4Byte hal_mac_ax_update_table_dl_swru(RT_HAL_MAC_INFO *hm_info,
+	struct ofdma_dl_group *dl_group)
+{
+	PADAPTER adapter = hm_info->adapter;
+	u32 ret=0;
+	FunctionIn(COMP_HAL_MAC_API);
+
+	ret = hm_info->halmac_ax_ops->upd_dlru_grptbl(
+		hm_info->halmac_ax_apter,
+		&dl_group->dl_grp_table);
+
+
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+
+u4Byte hal_mac_ax_update_table_dl_swfix(RT_HAL_MAC_INFO *hm_info,
+	struct ofdma_dl_group *dl_group)
+{
+	PADAPTER adapter = hm_info->adapter;
+	u32 ret=0;
+	FunctionIn(COMP_HAL_MAC_API);
+
+	ret = hm_info->halmac_ax_ops->upd_dlru_fixtbl(
+		hm_info->halmac_ax_apter,
+		&dl_group->fixed_mode_group);
+
+
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+
+
+u4Byte hal_mac_ax_update_table_ul_rufix(RT_HAL_MAC_INFO *hm_info,
+	struct ofdma_ul_group *ul_group)
+{
+	PADAPTER adapter = hm_info->adapter;
+	u32 ret=0;
+	FunctionIn(COMP_HAL_MAC_API);
+
+	ret = hm_info->halmac_ax_ops->upd_ulru_fixtbl(
+		hm_info->halmac_ax_apter,
+		&ul_group->fixed_mode_group);
+
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+
+u4Byte hal_mac_ax_update_table_ul_ru_table(RT_HAL_MAC_INFO *hm_info,
+	struct ofdma_ul_group *ul_group)
+{
+	PADAPTER adapter = hm_info->adapter;
+	u32 ret=0;
+	FunctionIn(COMP_HAL_MAC_API);
+
+	ret = hm_info->halmac_ax_ops->upd_ulru_grptbl(
+		hm_info->halmac_ax_apter,
+		&ul_group->ul_grp_table);
+
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+
+
+u4Byte hal_mac_ax_update_table_ul_drvfix(RT_HAL_MAC_INFO *hm_info,
+	struct ofdma_ul_group *ul_group)
+{
+	PADAPTER adapter = hm_info->adapter;
+	u32 ret=0;
+	FunctionIn(COMP_HAL_MAC_API);
+
+	ret = hm_info->halmac_ax_ops->upd_ul_fixinfo(
+		hm_info->halmac_ax_apter,
+		&ul_group->drv_fixed_info);
+
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+
+
+u4Byte hal_mac_ax_issue_bsrp(RT_HAL_MAC_INFO *hm_info,
+	struct ofdma_ul_group *ul_group)
+{
+	PADAPTER adapter = hm_info->adapter;
+	u32 ret=0;
+	FunctionIn(COMP_HAL_MAC_API);
+
+	ret = hm_info->halmac_ax_ops->upd_ul_fixinfo(
+		hm_info->halmac_ax_apter,
+		&ul_group->drv_fixed_info
+		);
+
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+
+
+u4Byte hal_mac_ax_update_ru_sta(RT_HAL_MAC_INFO *hm_info,
+	u8 *ru_sta_info)
+{
+	/* upd_rusta_info */
+	struct mac_ax_bb_stainfo *bb_stainfo = (struct mac_ax_bb_stainfo *)ru_sta_info;
+
+	hm_info->halmac_ax_ops->upd_rusta_info(
+			hm_info->halmac_ax_apter,
+			bb_stainfo
+			);
+	return 0;
+}
+
+u4Byte hal_mac_ax_update_ba_info_table(RT_HAL_MAC_INFO *hm_info,
+	u8 ba_info)
+{
+	/* upd_ba_infotbl */
+	struct mac_ax_ba_infotbl *ba_info_tbl = (struct mac_ax_ba_infotbl *)ba_info;
+
+	hm_info->halmac_ax_ops->upd_ba_infotbl(
+				hm_info->halmac_ax_apter,
+				ba_info_tbl
+				);
+
+	return 0;
+}
+
+u4Byte hal_mac_ax_Test_H2C(RT_HAL_MAC_INFO *hm_info,
+	u32 length, u8 Burst)
+{
+
+	u32 ret =0;
+	FunctionIn(COMP_HAL_MAC_API);
+	ret = hm_info->halmac_ax_ops->fwcmd_lb(
+				hm_info->halmac_ax_apter,
+				length,
+				Burst
+				);
+
+
+	FunctionOut(COMP_HAL_MAC_API);
+
+	return ret;
+}
+
+u4Byte hal_mac_ax_compare_h2c_c2h(RT_HAL_MAC_INFO *hm_info,
+	u8 *buf, u32 len)
+{
+	u32 ret=0;
+	FunctionIn(COMP_HAL_MAC_API);
+	ret = hm_info->halmac_ax_ops->process_c2h(
+				hm_info->halmac_ax_apter,
+				buf,
+				len
+				);
+	FunctionOut(COMP_HAL_MAC_API);
+	return ret;
+}
+#endif
+//====================================================================
+
+/*
+ * halmac wrapper API for hal and proto type is at hal_api_mac.h
+ * Efuse part.
+ */
+enum rtw_hal_status
+rtw_hal_mac_get_log_efuse_size(struct rtw_hal_com_t *hal_com, u32 *val,
+							   bool is_limited)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	if(is_limited == true) {
+		if(mac->ops->get_hw_value(mac,
+				MAC_AX_HW_GET_LIMIT_LOG_EFUSE_SIZE, val) != MACSUCCESS){
+				PHL_ERR("%s: Get limited logical efuse size fail!\n",
+						__FUNCTION__);
+				return RTW_HAL_STATUS_FAILURE;
+		}
+	}
+	else {
+		if(mac->ops->get_hw_value(mac,
+				MAC_AX_HW_GET_LOGICAL_EFUSE_SIZE, val) != MACSUCCESS){
+				PHL_ERR("%s: Get logical efuse size fail!\n", __FUNCTION__);
+				return RTW_HAL_STATUS_FAILURE;
+		}
+	}
+	PHL_INFO("%s: Logical efuse size = %d!\n", __FUNCTION__, *val);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/*
+* halmac wrapper API for hal and proto type is at hal_api_mac.h
+* Efuse part.
+*/
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_a_die_size(struct rtw_hal_com_t *hal_com, u8 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if(mac->ops->get_hw_value(mac,
+			MAC_AX_HW_GET_DAV_LOG_EFUSE_SIZE, val) != MACSUCCESS) {
+			PHL_ERR("%s: Get a-die size fail!\n",
+					__FUNCTION__);
+			return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("%s: Efuse a die size = %d!\n", __FUNCTION__, *val);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_read_log_efuse_map(struct rtw_hal_com_t *hal_com, u8 *map,
+							   bool is_limited)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->dump_log_efuse(mac,
+			MAC_AX_EFUSE_PARSER_MAP,
+			#ifdef CONFIG_PHL_FW_DUMP_EFUSE
+			MAC_AX_EFUSE_R_AUTO,
+			#else
+			MAC_AX_EFUSE_R_DRV,
+			#endif
+			map,
+			is_limited
+			) != MACSUCCESS) {
+		PHL_INFO("%s: Dump logical efuse fail!\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("%s: Dump logical efuse ok!\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/*
+ * HALMAC PG EFUSE API put version length at the tail of map/mask buffer
+ */
+
+enum rtw_hal_status
+rtw_hal_mac_write_log_efuse_map(struct rtw_hal_com_t *hal_com,
+								u8 *map,
+								u32 map_size,
+								u8 *mask,
+								u32 mask_size,
+								u8 *map_version,
+								u8 *mask_version,
+								u8 version_length,
+								u8 part,
+								bool is_limited,
+								u8 a_die_size,
+								u32 hci_to_a_die_offset,
+								u32 a_die_start_offset)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_pg_efuse_info info = {0};
+	enum rtw_hal_status status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+	u8 *tmp_map = NULL;
+	u8 *tmp_mask = NULL;
+	u32 offset;
+
+	tmp_map = _os_mem_alloc(hal_com->drv_priv, (map_size + version_length));
+	if(tmp_map == NULL) {
+		PHL_WARN("%s: Allocate pg map buffer fail!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_RESOURCE;
+		goto err_mem_tmp_map;
+	}
+
+	tmp_mask = _os_mem_alloc(hal_com->drv_priv, (mask_size + version_length));
+	if(tmp_mask == NULL) {
+		PHL_WARN("%s: Allocate pg mask buffer fail!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_RESOURCE;
+		goto err_mem_tmp_mask;
+	}
+
+	/* Copy efuse map and map version to tmp_map buffer */
+	_os_mem_cpy(hal_com->drv_priv, tmp_map, map, map_size);
+	_os_mem_cpy(hal_com->drv_priv, tmp_map+map_size, map_version,
+				version_length);
+	/* Copy efuse mask and mask version to tmp_mask buffer */
+	_os_mem_cpy(hal_com->drv_priv, tmp_mask, mask, mask_size);
+	_os_mem_cpy(hal_com->drv_priv, tmp_mask+mask_size, mask_version,
+				version_length);
+#if 0 /* For debug usage */
+	debug_dump_data(map, (u16)map_size, "logical map:");
+	debug_dump_data(map_version, version_length, "logical map version:");
+	debug_dump_data(mask, (u16)mask_size, "mask:");
+	debug_dump_data(mask_version, version_length, "mask version:");
+	debug_dump_data(tmp_map, (u16)(map_size + version_length), "tmp_map:");
+	debug_dump_data(tmp_mask, (u16)(mask_size + version_length), "tmp_mask:");
+#endif
+	info.efuse_map = tmp_map;
+	info.efuse_map_size = map_size;
+	info.efuse_mask = tmp_mask;
+	info.efuse_mask_size= mask_size;
+
+	if (a_die_size != 0 && is_limited) {
+		for (offset = a_die_start_offset;
+			offset < (a_die_start_offset + (u32)a_die_size);
+			offset++)
+			info.efuse_map[offset] =
+				map[offset + hci_to_a_die_offset];
+	}
+
+	if (mac->ops->pg_efuse_by_map(mac,
+								  &info,
+								  MAC_AX_EFUSE_R_DRV,
+								  part,
+								  is_limited) != MACSUCCESS) {
+		PHL_INFO("%s: PG Fail!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+	}
+	else {
+		PHL_INFO("%s: PG ok!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+	_os_mem_free(hal_com->drv_priv, tmp_map, (map_size + version_length));
+	_os_mem_free(hal_com->drv_priv, tmp_mask, (mask_size + version_length));
+
+	return status;
+
+err_mem_tmp_mask:
+	_os_mem_free(hal_com->drv_priv, tmp_map, (map_size + version_length));
+
+err_mem_tmp_map:
+
+	return status;
+}
+
+#define MAC_HIDDEN_MAX_BW_NUM 9
+const u8 _mac_hidden_max_bw_to_hal_bw_cap[MAC_HIDDEN_MAX_BW_NUM] = {
+	0,
+	0,
+	(BW_CAP_5M),
+	(BW_CAP_10M | BW_CAP_5M),
+	(BW_CAP_20M | BW_CAP_10M | BW_CAP_5M),
+	(BW_CAP_40M | BW_CAP_20M | BW_CAP_10M | BW_CAP_5M),
+	(BW_CAP_80M | BW_CAP_40M | BW_CAP_20M | BW_CAP_10M | BW_CAP_5M),
+	(BW_CAP_160M | BW_CAP_80M | BW_CAP_40M | BW_CAP_20M | BW_CAP_10M | BW_CAP_5M),
+	(BW_CAP_320M | BW_CAP_160M | BW_CAP_80M | BW_CAP_40M | BW_CAP_20M | BW_CAP_10M | BW_CAP_5M),
+};
+#define mac_hidden_max_bw_to_hal_bw_cap(max_bw) (((max_bw) >= MAC_HIDDEN_MAX_BW_NUM) ? 0 : _mac_hidden_max_bw_to_hal_bw_cap[(max_bw)])
+
+#define MAC_HIDDEN_PROTOCOL_NUM 5
+const u8 _mac_hidden_proto_to_hal_proto_cap[MAC_HIDDEN_PROTOCOL_NUM] = {
+	0,
+	(WLAN_MD_11N | WLAN_MD_11G | WLAN_MD_11A | WLAN_MD_11B),
+	(WLAN_MD_11AC | WLAN_MD_11N | WLAN_MD_11G | WLAN_MD_11A | WLAN_MD_11B),
+	(WLAN_MD_11AX | WLAN_MD_11AC | WLAN_MD_11N | WLAN_MD_11G | WLAN_MD_11A | WLAN_MD_11B),
+	(WLAN_MD_11BE | WLAN_MD_11AX | WLAN_MD_11AC | WLAN_MD_11N | WLAN_MD_11G | WLAN_MD_11A | WLAN_MD_11B),
+};
+#define mac_hidden_proto_to_hal_proto_cap(proto) (((proto) >= MAC_HIDDEN_PROTOCOL_NUM) ? 0 : _mac_hidden_proto_to_hal_proto_cap[(proto)])
+
+
+enum rtw_hal_status
+rtw_hal_mac_read_hidden_rpt(struct rtw_hal_com_t *hal_com)
+{
+
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_defeature_value rpt = {0};
+	u32 err;
+
+	err = mac->ops->read_hidden_rpt(mac, &rpt);
+	if (err != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_TRIG, _PHL_INFO_, "err=0x%x\n", err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (rpt.tx_spatial_stream != 7 && rpt.tx_spatial_stream > 0) {
+		hal_com->phy_hw_cap[0].tx_num = rpt.tx_spatial_stream;
+		hal_com->phy_hw_cap[1].tx_num = rpt.tx_spatial_stream;
+		hal_com->phy_hw_cap[0].tx_path_num = rpt.tx_spatial_stream;
+		hal_com->phy_hw_cap[1].tx_path_num = rpt.tx_spatial_stream;
+		if (rpt.tx_path_num != 7 && rpt.tx_path_num > 0) {
+			hal_com->phy_hw_cap[0].tx_path_num = rpt.tx_path_num;
+			hal_com->phy_hw_cap[1].tx_path_num = rpt.tx_path_num;
+		}
+	}
+	if (rpt.rx_spatial_stream != 7 && rpt.rx_spatial_stream > 0) {
+		hal_com->phy_hw_cap[0].rx_num = rpt.rx_spatial_stream;
+		hal_com->phy_hw_cap[1].rx_num = rpt.rx_spatial_stream;
+		hal_com->phy_hw_cap[0].rx_path_num = rpt.rx_spatial_stream;
+		hal_com->phy_hw_cap[1].rx_path_num = rpt.rx_spatial_stream;
+		if (rpt.rx_path_num != 7 && rpt.rx_path_num > 0) {
+			hal_com->phy_hw_cap[0].rx_path_num = rpt.rx_path_num;
+			hal_com->phy_hw_cap[1].rx_path_num = rpt.rx_path_num;
+		}
+	}
+	if (rpt.hw_special_type > EFUSE_HW_STYPE_NONE &&
+	    rpt.hw_special_type < EFUSE_HW_STYPE_GENERAL)
+		hal_com->dev_hw_cap.hw_stype_cap = rpt.hw_special_type;
+
+	if (rpt.wl_func_support > EFUSE_WL_FUNC_NONE &&
+	    rpt.wl_func_support < EFUSE_WL_FUNC_GENERAL)
+		hal_com->dev_hw_cap.wl_func_cap = rpt.wl_func_support;
+
+	hal_com->dev_hw_cap.bw_sup &= mac_hidden_max_bw_to_hal_bw_cap(rpt.bandwidth);
+
+	hal_com->dev_hw_cap.band_sup &= (rpt.band_sel & 0x7);
+
+	hal_com->phy_hw_cap[0].proto_sup &= mac_hidden_proto_to_hal_proto_cap(rpt.protocol_80211);
+	hal_com->phy_hw_cap[1].proto_sup &= mac_hidden_proto_to_hal_proto_cap(rpt.protocol_80211);
+
+	hal_com->uuid = rpt.uuid;
+
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden tx_spatial_stream=%d hidden rx_spatial_stream=%d\n",
+		rpt.tx_spatial_stream, rpt.rx_spatial_stream);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden tx_path_num=%d hidden rx_path_num=%d\n",
+		rpt.tx_path_num, rpt.rx_path_num);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden bw=%d\n", rpt.bandwidth);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden band=%d\n", rpt.band_sel);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden protocol = %d\n",
+		rpt.protocol_80211);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden nic=%d\n", rpt.NIC_router);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden hw special type=%d\n",
+		rpt.hw_special_type);
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "hidden wl func=%d\n",
+		rpt.wl_func_support);
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_check_efuse_autoload(struct rtw_hal_com_t *hal_com, u8 *autoload)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->check_efuse_autoload(mac, autoload) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	if(*autoload == true)
+		PHL_INFO("%s: efuse auto load SUCCESS!\n", __FUNCTION__);
+	else
+		PHL_INFO("%s: efuse auto load FAIL!\n", __FUNCTION__);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_avl(struct rtw_hal_com_t *hal_com, u32 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->get_efuse_avl_size(mac, val) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_size(struct rtw_hal_com_t *hal_com, u32 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->get_hw_value(mac,
+		MAC_AX_HW_GET_EFUSE_SIZE, val) != MACSUCCESS){
+		PHL_ERR("%s: Get efuse size fail!\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	PHL_INFO("%s: Efuse size = %d!\n", __FUNCTION__, *val);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_mask_size(struct rtw_hal_com_t *hal_com, u32 *val,
+								bool is_limited)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if(is_limited == true) {
+		if(mac->ops->get_hw_value(mac,
+			MAC_AX_HW_GET_LIMIT_EFUSE_MASK_SIZE, val) != MACSUCCESS)
+			return RTW_HAL_STATUS_FAILURE;
+	}
+	else {
+		if(mac->ops->get_hw_value(mac,
+			MAC_AX_HW_GET_EFUSE_MASK_SIZE, val) != MACSUCCESS)
+			return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("%s: efuse mask size = %d\n", __FUNCTION__, *val);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	PHL_INFO("%s\n", __FUNCTION__);
+
+	if (mac->ops->get_efuse_info(mac, efuse_map, info_type, value, size,
+								 &map_valid) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_read_phy_efuse(struct rtw_hal_com_t *hal_com,
+	u32 addr, u32 size, u8 *value)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	PHL_INFO("%s\n", __FUNCTION__);
+
+	if (mac->ops->read_efuse(mac, addr, size, value,
+		MAC_AX_EFUSE_BANK_WIFI) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_read_bt_phy_efuse(struct rtw_hal_com_t *hal_com,
+	u32 addr, u32 size, u8 *value)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	PHL_INFO("%s\n", __FUNCTION__);
+
+	if (mac->ops->read_efuse(mac, addr, size, value,
+		MAC_AX_EFUSE_BANK_BT) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32 rtw_hal_mac_write_msk_pwr_reg(
+	struct rtw_hal_com_t *hal_com, u8 band, u32 offset, u32 mask, u32 val)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	u32 result = 0;
+
+	result = ops->write_msk_pwr_reg(mac, band, offset, mask, val);
+
+	if (result != MACSUCCESS)
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_,
+			"Write power register failure, status = %d\n", result);
+
+	return result;
+}
+
+u32 rtw_hal_mac_set_pwr_reg(struct rtw_hal_com_t *hal_com, u8 band, u32 offset, u32 val){
+
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	u32 result = 0;
+
+	result = ops->write_pwr_reg(mac, band, offset, val);
+
+	if (result != MACSUCCESS)
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "Set power register failure, status = %d\n", result);
+
+	return result;
+}
+
+u32 rtw_hal_mac_get_pwr_reg(struct rtw_hal_com_t *hal_com, u8 band, u32 offset, u32 *val){
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	u32 result = 0;
+
+	result = ops->read_pwr_reg(mac, band, offset, val);
+
+	if (result != MACSUCCESS)
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "Get power register failure, status = %d\n", result);
+
+	return result;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_xcap(struct rtw_hal_com_t *hal_com, u8 sc_xo, u32 *value){
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = ops->read_xcap_reg(mac, sc_xo, value);
+
+	if (ret != MACSUCCESS)
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "Get xacp failure, status = %d\n", ret);
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_xcap(struct rtw_hal_com_t *hal_com, u8 sc_xo, u32 value){
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = ops->write_xcap_reg(mac, sc_xo, value);
+
+	if (ret != MACSUCCESS)
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "Set xacp failure, status = %d\n", ret);
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_xsi(struct rtw_hal_com_t *hal_com, u8 offset, u8* val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	/* Because mac si function is not support mac ops*/
+	/* we call mac si function temporarily until mac team feedback.*/
+	if (mac_read_xtal_si(mac, offset, val) != MACSUCCESS) {
+		PHL_INFO("Get xsi failure, status = %s\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_xsi(struct rtw_hal_com_t *hal_com, u8 offset, u8 val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	/* Because mac si function is not support mac ops*/
+	/* we call mac si function temporarily until mac team feedback.*/
+	if (mac_write_xtal_si(mac, offset, val, 0xff) != MACSUCCESS) {
+		PHL_INFO("Set xsi failure, status = %s\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_fw_dbg_dump(struct hal_info_t *hal_info)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u8 *buffer = NULL;
+	u16 bufSize = FW_PLE_SIZE;
+	struct mac_ax_fwdbg_en en = {0};
+	u32 mac_err = 0;
+
+	if (!IS_IO_DUMP_ALLOWED(hal_info->phl_com)) {
+		PHL_DBG("%s(): skip due to IS_IO_DUMP_ALLOWED(%d)\n",
+				  __func__,
+				  IS_IO_DUMP_ALLOWED(hal_info->phl_com));
+		return RTW_HAL_STATUS_SUCCESS;
+	}
+	en.status_dbg = 1;
+	en.ps_dbg = 1;
+	en.rsv_ple_dbg = 0;
+
+	mac_err = mac->ops->fw_dbg_dump(mac, &buffer, &en);
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (en.rsv_ple_dbg && buffer) {
+		/*fw buffer is 2048, but Windows DbgPrint only 512 Bytes, so we split buffer to 4 segments*/
+		if (buffer != NULL) {
+			PHL_PRINT("=======================\n");
+			PHL_PRINT("Start to dump fw rsvd ple:\n\n");
+			_hal_fw_dbg_dump(hal_info, buffer, bufSize);
+			PHL_PRINT("\n=======================\n");
+		}
+	}
+	if (buffer != NULL)
+		_os_mem_free(hal_info->hal_com->drv_priv, buffer, bufSize);
+
+	return RTW_HAL_STATUS_SUCCESS;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+
+}
+
+#ifdef CONFIG_HAL_MAC_DBG
+enum rtw_fw_status rtw_hal_mac_get_fw_status(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 mac_fw_sts;
+
+	mac_fw_sts = hal_mac_ops->get_fw_status(mac);
+
+	switch (mac_fw_sts) {
+	case MACSUCCESS:
+		return RTW_FW_STATUS_OK;
+	case MACNOFW:
+		return RTW_FW_STATUS_NOFW;
+	case MACFWASSERT:
+		return RTW_FW_STATUS_ASSERT;
+	case MACFWEXCEP:
+		return RTW_FW_STATUS_EXCEP;
+	case MACFWRXI300:
+		return RTW_FW_STATUS_RXI300;
+	case MACFWPCHANG:
+		return RTW_FW_STATUS_HANG;
+	default:
+		return RTW_FW_STATUS_OK;
+	}
+}
+#endif
+
+#ifdef CONFIG_POWER_SAVE
+enum rtw_hal_status
+rtw_hal_mac_ps_notify_wake(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->ps_notify_wake(mac) == MACSUCCESS) {
+		return RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_WARN("%s: notify wake fail!\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+}
+
+#define PS_ADV_EXTRX_NUM 2
+enum rtw_hal_status
+rtw_hal_mac_lps_adv_setting(struct hal_info_t *hal,
+	struct rtw_phl_lps_adv_cfg *cur, u8 macid, enum ps_adv_action act)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ps_adv_parm parm = {0};
+	u32 ret = 0;
+
+	if (act == PS_ADV_EN_WAIT_PVB_RX) {
+		cur->pvb_wait_rx = true;
+	} else if (act == PS_ADV_DIS_WAIT_PVB_RX) {
+		cur->pvb_wait_rx = false;
+	} else if (act == PS_ADV_EN_TRX_EXT) {
+		cur->trx_ext_mode = true;
+		cur->ext_num = PS_ADV_EXTRX_NUM;
+	}
+	parm.macid = macid;
+	if (cur->pvb_wait_rx) {
+		parm.ensmartpsdtimrx = true;
+	}
+	if (cur->trx_ext_mode) {
+		parm.entrxextmode = true;
+		parm.extnum = cur->ext_num;
+	}
+	PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s: macid(%d), trxtimeouttimeset(%d), ensmartpsdtimrx(%d), entrxextmode(%d), trxtimeouttimeval(%d), extnum(%d)\n",
+		__FUNCTION__, parm.macid, parm.trxtimeouttimeset,
+		parm.ensmartpsdtimrx, parm.entrxextmode, parm.trxtimeouttimeval,
+		parm.extnum);
+	if (MACSUCCESS == (ret = mac->ops->cfg_ps_advance_parm(mac, &parm))) {
+		return RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_WARN("%s: Configure fail (ret = %d)!\n", __FUNCTION__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+}
+
+enum rtw_hal_status
+rtw_hal_mac_req_pwr_state(struct hal_info_t *hal_info, u8 pwr_state)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if(mac->ops->ps_pwr_state(mac, MAC_AX_PWR_STATE_ACT_REQ, pwr_state)
+			== MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_chk_pwr_state(struct hal_info_t *hal_info, u8 pwr_state, u32 *mac_sts)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	*mac_sts = mac->ops->ps_pwr_state(mac, MAC_AX_PWR_STATE_ACT_CHK, pwr_state);
+	if(*mac_sts == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_ips_cfg(struct hal_info_t *hal_info, u16 macid, bool enable)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->cfg_ips(mac, (u8)macid, enable) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_ips_chk_leave(struct hal_info_t *hal_info, u16 macid)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->chk_leave_ips(mac, (u8)macid) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_lps_cfg(struct hal_info_t *hal_info,
+			struct rtw_hal_lps_info *lps_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum mac_ax_ps_mode ax_ps_mode;
+	struct mac_ax_lps_info ax_lps_info = {0};
+
+	if (lps_info->en) {
+		ax_ps_mode = MAC_AX_PS_MODE_LEGACY;
+	} else {
+		ax_ps_mode = MAC_AX_PS_MODE_ACTIVE;
+	}
+
+	ax_lps_info.listen_bcn_mode = (enum mac_ax_listern_bcn_mode)lps_info->listen_bcn_mode;
+	ax_lps_info.awake_interval = lps_info->awake_interval;
+	ax_lps_info.smart_ps_mode = (enum mac_ax_smart_ps_mode)lps_info->smart_ps_mode;
+	ax_lps_info.bcnnohit_en = lps_info->bcnnohit_en;
+	ax_lps_info.dyn_tx_ant_num_en = lps_info->dyntxant_en;
+	ax_lps_info.max_tx_ant_num = lps_info->maxtxant;
+	ax_lps_info.lps_tx_ant_num = lps_info->lpstxant;
+
+	if (mac->ops->cfg_lps(mac, (u8)lps_info->macid, ax_ps_mode,
+		&ax_lps_info) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_lps_chk_leave(struct hal_info_t *hal_info, u16 macid, u32 *mac_sts)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	*mac_sts = mac->ops->chk_leave_lps(mac, (u8)macid);
+	if(*mac_sts  == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_lps_chk_access(struct hal_info_t *hal_info, u32 offset)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if(mac->ops->io_chk_access(mac, offset) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_lps_bcn_tracking_cfg(struct hal_info_t *hal,
+	struct rtw_bcn_tracking_cfg *cfg, struct rtw_bcn_tracking_cfg *cfg_cur)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_req_pwr_lvl_info pwr_lvl_info = {0};
+
+	pwr_lvl_info.ps_lvl = 0; /* default ps level */
+	pwr_lvl_info.bcn_to_val = cfg->bcn_timeout;
+	pwr_lvl_info.macid = cfg->macid;
+
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "%s: macid(%d), bcn_lvl(%d), ps_lvl(%d)\n",
+		__func__, pwr_lvl_info.macid, pwr_lvl_info.bcn_to_lvl,
+		pwr_lvl_info.ps_lvl);
+	if (mac->ops->req_pwr_lvl_cfg(mac, &pwr_lvl_info) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+#endif /* CONFIG_POWER_SAVE */
+
+enum rtw_hal_status
+rtw_hal_mac_tbtt_tuning_cfg(struct hal_info_t *hal, enum phl_band_idx band,
+				u8 port, u32 tbtt)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_tbtt_tuning_info tbtt_i = {0};
+
+	tbtt_i.band = band;
+	tbtt_i.port = port;
+	tbtt_i.shift_val = tbtt;
+
+	PHL_TRACE(COMP_PHL_MAC, _PHL_INFO_, "%s: band(%d), port(%d), shift_val(%d)\n",
+		__func__, tbtt_i.band, tbtt_i.port, tbtt_i.shift_val);
+	if (mac->ops->tbtt_tuning_cfg(mac, &tbtt_i) == MACSUCCESS) {
+		return RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s: Failed\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_tx_cnt(struct hal_info_t *hal, enum phl_band_idx bidx, u8 sel)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_tx_cnt tx_cnt = {0};
+	u32 ret = MACSUCCESS;
+
+	tx_cnt.band = bidx;
+	tx_cnt.sel = MAC_AX_TX_ALLTYPE;
+	ret = ops->get_hw_value(mac, MAC_AX_HW_GET_TX_CNT, (void *)&tx_cnt);
+	if (ret != MACSUCCESS) {
+		PHL_ERR("%s failed - ecode(%d)\n", __func__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("[MAC] TX CNT - band_%d\n", tx_cnt.band);
+	PHL_INFO("[MAC] TX CNT - LCCK : %d\n", tx_cnt.txcnt[MAC_AX_TX_LCCK]);
+	PHL_INFO("[MAC] TX CNT - SCCK : %d\n", tx_cnt.txcnt[MAC_AX_TX_SCCK]);
+	PHL_INFO("[MAC] TX CNT - OFDM : %d\n", tx_cnt.txcnt[MAC_AX_TX_OFDM]);
+	PHL_INFO("[MAC] TX CNT - HT : %d\n", tx_cnt.txcnt[MAC_AX_TX_HT]);
+	PHL_INFO("[MAC] TX CNT - HTGF : %d\n", tx_cnt.txcnt[MAC_AX_TX_HTGF]);
+	PHL_INFO("[MAC] TX CNT - VHTSU : %d\n", tx_cnt.txcnt[MAC_AX_TX_VHTSU]);
+	PHL_INFO("[MAC] TX CNT - VHTMU : %d\n", tx_cnt.txcnt[MAC_AX_TX_VHTMU]);
+	PHL_INFO("[MAC] TX CNT - HESU : %d\n", tx_cnt.txcnt[MAC_AX_TX_HESU]);
+	PHL_INFO("[MAC] TX CNT - HEERSU : %d\n", tx_cnt.txcnt[MAC_AX_TX_HEERSU]);
+	PHL_INFO("[MAC] TX CNT - HEMU : %d\n", tx_cnt.txcnt[MAC_AX_TX_HEMU]);
+	PHL_INFO("[MAC] TX CNT - HETB : %d\n", tx_cnt.txcnt[MAC_AX_TX_HETB]);
+
+	ret = ops->set_hw_value(mac, MAC_AX_HW_SET_CLR_TX_CNT, (void *)&tx_cnt);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_sel_tx_cnt(struct hal_info_t *hal, enum phl_band_idx bidx, void *out_tx_cnt)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_tx_cnt *tx_cnt = out_tx_cnt;
+	u32 ret = MACSUCCESS;
+
+	tx_cnt->band = bidx;
+	tx_cnt->sel = MAC_AX_TX_ALLTYPE;
+	ret = ops->get_hw_value(mac, MAC_AX_HW_GET_TX_CNT, (void *)tx_cnt);
+	if (ret != MACSUCCESS) {
+		PHL_ERR("%s failed - ecode(%d)\n", __func__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("[MAC] TX CNT - band_%d\n", tx_cnt->band);
+	PHL_INFO("[MAC] TX CNT - LCCK : %d\n", tx_cnt->txcnt[MAC_AX_TX_LCCK]);
+	PHL_INFO("[MAC] TX CNT - SCCK : %d\n", tx_cnt->txcnt[MAC_AX_TX_SCCK]);
+	PHL_INFO("[MAC] TX CNT - OFDM : %d\n", tx_cnt->txcnt[MAC_AX_TX_OFDM]);
+	PHL_INFO("[MAC] TX CNT - HT : %d\n", tx_cnt->txcnt[MAC_AX_TX_HT]);
+	PHL_INFO("[MAC] TX CNT - HTGF : %d\n", tx_cnt->txcnt[MAC_AX_TX_HTGF]);
+	PHL_INFO("[MAC] TX CNT - VHTSU : %d\n", tx_cnt->txcnt[MAC_AX_TX_VHTSU]);
+	PHL_INFO("[MAC] TX CNT - VHTMU : %d\n", tx_cnt->txcnt[MAC_AX_TX_VHTMU]);
+	PHL_INFO("[MAC] TX CNT - HESU : %d\n", tx_cnt->txcnt[MAC_AX_TX_HESU]);
+	PHL_INFO("[MAC] TX CNT - HEERSU : %d\n", tx_cnt->txcnt[MAC_AX_TX_HEERSU]);
+	PHL_INFO("[MAC] TX CNT - HEMU : %d\n", tx_cnt->txcnt[MAC_AX_TX_HEMU]);
+	PHL_INFO("[MAC] TX CNT - HETB : %d\n", tx_cnt->txcnt[MAC_AX_TX_HETB]);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_rx_cnt(struct hal_info_t *hal_info, u8 cur_phy_idx, u8 type_idx, u32 *ret_value)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	struct mac_ax_rx_cnt rx_cnt = {0};
+	u16 rx_cnt_buff[MAC_AX_RX_PPDU_MAX];
+	u32 rx_cnt_total=0;
+	u16 ppdu_idx;
+
+	rx_cnt.type = type_idx;
+	rx_cnt.op = MAC_AX_RXCNT_R;
+	rx_cnt.buf = rx_cnt_buff;
+	rx_cnt.band = cur_phy_idx;
+
+	ret = ops->rx_cnt(mac, &rx_cnt);
+
+	for(ppdu_idx=0;ppdu_idx<MAC_AX_RX_PPDU_MAX;ppdu_idx++)
+	{
+		rx_cnt_total+=rx_cnt_buff[ppdu_idx];
+	}
+
+	*ret_value = rx_cnt_total;
+
+	if (ret != RTW_HAL_STATUS_SUCCESS){
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "mac get rx counter fail, status=%d\n",ret);
+	}
+
+	return ret;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
+
+void
+rtw_hal_mac_get_rx_cnt_info(void *hal, u8 cur_phy_idx, u8 type_idx, u32 *ret_value){
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_mac_get_rx_cnt(hal_info, cur_phy_idx, type_idx, ret_value);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_rx_cnt_by_idx(struct hal_info_t *hal_info, u8 cur_phy_idx,
+			      u8 cnt_idx, u16 *ret_value)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_rx_cnt rx_cnt = {0};
+	u16 rx_cnt_buff[MAC_AX_RX_PPDU_MAX];
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	rx_cnt.type = MAC_AX_RX_IDX;
+	rx_cnt.op = MAC_AX_RXCNT_R;
+	rx_cnt.buf = rx_cnt_buff;
+	rx_cnt.band = cur_phy_idx;
+	rx_cnt.idx = cnt_idx;
+
+	ret = ops->rx_cnt(mac, &rx_cnt);
+
+	*ret_value = rx_cnt_buff[0];
+
+	if (ret != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_,
+			  "mac get rx counter idx=%d fail, status=%d\n",
+			  cnt_idx, ret);
+	}
+
+	return ret;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_reset_rx_cnt(struct hal_info_t *hal_info, u8 cur_phy_idx)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	struct mac_ax_rx_cnt rx_cnt = {0};
+	u16 rx_cnt_buff[MAC_AX_RX_PPDU_MAX];
+
+	rx_cnt.type = MAC_AX_RX_IDX;
+	rx_cnt.op = MAC_AX_RXCNT_RST_ALL;
+	rx_cnt.buf = rx_cnt_buff;
+	rx_cnt.band = cur_phy_idx;
+
+	ret = ops->rx_cnt(mac, &rx_cnt);
+
+	if (ret != RTW_HAL_STATUS_SUCCESS){
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "mac reset rx cnt fail, status=%d\n",ret);
+	}
+
+	return ret;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
+
+enum rtw_hal_status
+rtw_hal_mac_tx_idle_poll(struct rtw_hal_com_t *hal_com, u8 band_idx)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_tx_idle_poll_cfg cfg = {0};
+	u32 err = 0;
+
+
+	cfg.sel = MAC_AX_TX_IDLE_POLL_SEL_BAND;
+	cfg.band = band_idx;
+	err = mac->ops->tx_idle_poll(mac, &cfg);
+	if (err != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_,
+			  "Polling tx idle failure(%d)!\n", err);
+		return RTW_HAL_STATUS_TIMEOUT;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_sch_txen_cfg {
+	RTW_TXEN_BE0 = 1 << 0,
+	RTW_TXEN_BK0 = 1 << 1,
+	RTW_TXEN_VI0 = 1 << 2,
+	RTW_TXEN_VO0 = 1 << 3,
+	RTW_TXEN_BE1 = 1 << 4,
+	RTW_TXEN_BK1 = 1 << 5,
+	RTW_TXEN_VI1 = 1 << 6,
+	RTW_TXEN_VO1 = 1 << 7,
+	RTW_TXEN_MG0 = 1 << 8,
+	RTW_TXEN_MG1 = 1 << 9,
+	RTW_TXEN_MG2 = 1 << 10,
+	RTW_TXEN_HI = 1 << 11,
+	RTW_TXEN_BCN = 1 << 12,
+	RTW_TXEN_UL = 1 << 13,
+	RTW_TXEN_TWT0 = 1 << 14,
+	RTW_TXEN_TWT1 = 1 << 15,
+	RTW_TXEN_DRV_MASK = 0x19FF,
+	RTW_TXEN_ALL = 0xFFFF,
+};
+
+enum rtw_hal_status
+rtw_hal_mac_set_sch_tx_en(struct rtw_hal_com_t *hal_com, u8 band_idx,
+						u16 tx_en, u16 tx_en_mask)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_sch_tx_en_cfg cfg = {0};
+	enum rtw_hal_status ret;
+	u32 result = 0;
+
+	ret = RTW_HAL_STATUS_SUCCESS;
+	cfg.band = band_idx;
+
+	cfg.tx_en.be0 = ((tx_en & RTW_TXEN_BE0) ? 1 : 0);
+	cfg.tx_en.bk0 = ((tx_en & RTW_TXEN_BK0) ? 1 : 0);
+	cfg.tx_en.vi0 = ((tx_en & RTW_TXEN_VI0) ? 1 : 0);
+	cfg.tx_en.vo0 = ((tx_en & RTW_TXEN_VO0) ? 1 : 0);
+	cfg.tx_en.be1 = ((tx_en & RTW_TXEN_BE1) ? 1 : 0);
+	cfg.tx_en.bk1 = ((tx_en & RTW_TXEN_BK1) ? 1 : 0);
+	cfg.tx_en.vi1 = ((tx_en & RTW_TXEN_VI1) ? 1 : 0);
+	cfg.tx_en.vo1 = ((tx_en & RTW_TXEN_VO1) ? 1 : 0);
+	cfg.tx_en.mg0 = ((tx_en & RTW_TXEN_MG0) ? 1 : 0);
+	cfg.tx_en.mg1 = ((tx_en & RTW_TXEN_MG1) ? 1 : 0);
+	cfg.tx_en.mg2 = ((tx_en & RTW_TXEN_MG2) ? 1 : 0);
+	cfg.tx_en.hi = ((tx_en & RTW_TXEN_HI) ? 1 : 0);
+	cfg.tx_en.bcn = ((tx_en & RTW_TXEN_BCN) ? 1 : 0);
+	cfg.tx_en.ul = ((tx_en & RTW_TXEN_UL) ? 1 : 0);
+	cfg.tx_en.twt0 = ((tx_en & RTW_TXEN_TWT0) ? 1 : 0);
+	cfg.tx_en.twt1 = ((tx_en & RTW_TXEN_TWT1) ? 1 : 0);
+
+	cfg.tx_en_mask.be0 = ((tx_en_mask & RTW_TXEN_BE0) ? 1 : 0);
+	cfg.tx_en_mask.bk0 = ((tx_en_mask & RTW_TXEN_BK0) ? 1 : 0);
+	cfg.tx_en_mask.vi0 = ((tx_en_mask & RTW_TXEN_VI0) ? 1 : 0);
+	cfg.tx_en_mask.vo0 = ((tx_en_mask & RTW_TXEN_VO0) ? 1 : 0);
+	cfg.tx_en_mask.be1 = ((tx_en_mask & RTW_TXEN_BE1) ? 1 : 0);
+	cfg.tx_en_mask.bk1 = ((tx_en_mask & RTW_TXEN_BK1) ? 1 : 0);
+	cfg.tx_en_mask.vi1 = ((tx_en_mask & RTW_TXEN_VI1) ? 1 : 0);
+	cfg.tx_en_mask.vo1 = ((tx_en_mask & RTW_TXEN_VO1) ? 1 : 0);
+	cfg.tx_en_mask.mg0 = ((tx_en_mask & RTW_TXEN_MG0) ? 1 : 0);
+	cfg.tx_en_mask.mg1 = ((tx_en_mask & RTW_TXEN_MG1) ? 1 : 0);
+	cfg.tx_en_mask.mg2 = ((tx_en_mask & RTW_TXEN_MG2) ? 1 : 0);
+	cfg.tx_en_mask.hi = ((tx_en_mask & RTW_TXEN_HI) ? 1 : 0);
+	cfg.tx_en_mask.bcn = ((tx_en_mask & RTW_TXEN_BCN) ? 1 : 0);
+	cfg.tx_en_mask.ul = ((tx_en_mask & RTW_TXEN_UL) ? 1 : 0);
+	cfg.tx_en_mask.twt0 = ((tx_en_mask & RTW_TXEN_TWT0) ? 1 : 0);
+	cfg.tx_en_mask.twt1 = ((tx_en_mask & RTW_TXEN_TWT1) ? 1 : 0);
+
+	result = ops->set_hw_value(mac, MAC_AX_HW_SET_SCH_TXEN_CFG, &cfg);
+
+	if (result != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "Set Tx pause failure, status = %d\n", result);
+		ret = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_pause_tx_fifo_sw(void *hal, u8 band_idx,
+                         bool tx_pause, enum tx_pause_rson rson)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	u16 *tx_off;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	tx_off = &hal_com->band[band_idx].tx_pause[rson];
+	if (tx_pause == true) {
+		switch (rson) {
+		case PAUSE_RSON_NOR_SCAN:
+			*tx_off = (u16)~RTW_TXEN_MG0;
+			break;
+		case PAUSE_RSON_RFK:
+		case PAUSE_RSON_PSD:
+		case PAUSE_RSON_DFS:
+		case PAUSE_RSON_DFS_CAC:
+		case PAUSE_RSON_DBCC:
+		case PAUSE_RSON_RESET:
+			*tx_off = (u16)RTW_TXEN_ALL;
+			break;
+		case PAUSE_RSON_DFS_CSA:
+			*tx_off = (u16)~RTW_TXEN_BCN;
+			break;
+		case PAUSE_RSON_DFS_CSA_MG:
+			*tx_off = (u16)~(RTW_TXEN_BCN | RTW_TXEN_MG0);
+			break;
+		default:
+			PHL_ERR("Unknow pause reason:%d\n", rson);
+			goto _error;
+		}
+	} else {
+		*tx_off = 0;
+	}
+	hstatus = RTW_HAL_STATUS_SUCCESS;
+
+_error:
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_tx_pause(struct rtw_hal_com_t *hal_com,
+		 u8 band_idx, bool tx_pause, enum tx_pause_rson rson)
+{
+	u16 *tx_off;
+	enum tx_pause_rson i;
+	u16 tx_cfg = 0;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	hstatus = rtw_hal_pause_tx_fifo_sw(hal_com->hal_priv, band_idx,
+						tx_pause, rson);
+	if (hstatus != RTW_HAL_STATUS_SUCCESS)
+		goto _error;
+
+	tx_off = hal_com->band[band_idx].tx_pause;
+	for (i = 0; (i < PAUSE_RSON_MAX) && (tx_cfg != RTW_TXEN_ALL); i++)
+		if (tx_off[i])
+			tx_cfg |= tx_off[i];
+	/* tx_cfg is white list, but tx_pause of struct rtw_hw_band is black list */
+	tx_cfg = ~tx_cfg;
+	PHL_INFO("TX %sPause - Reason(%d) for band-%u, final tx_cfg(0x%04x)\n",
+		 tx_pause?"":"Un-", rson, band_idx, tx_cfg);
+
+	hstatus = rtw_hal_mac_set_sch_tx_en(hal_com, band_idx,
+					    tx_cfg, RTW_TXEN_ALL);
+	if (hstatus != RTW_HAL_STATUS_SUCCESS)
+		goto _error;
+
+#ifdef RTW_WKARD_TXPAUSE_BF_ISSUE_NULL
+	if ((rson == PAUSE_RSON_RFK || rson == PAUSE_RSON_NOR_SCAN) &&
+#else
+	if ((rson == PAUSE_RSON_RFK) &&
+#endif
+		tx_pause) {
+		hstatus = rtw_hal_mac_tx_idle_poll(hal_com, band_idx);
+		if (hstatus != RTW_HAL_STATUS_SUCCESS)
+			goto _error;
+	}
+
+_error:
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_macid_pause(struct rtw_hal_com_t *hal_com,
+                            u16 macid, bool pause)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_macid_pause_cfg cfg = {0};
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	cfg.macid = (u8)macid;
+	cfg.pause = pause;
+
+	PHL_INFO("%s macid:%d(%s) \n", __func__, macid, pause ? "pause":"unpause");
+	/*TODO - MAC_AX_HW_SET_MULTI_ID_PAUSE*/
+	if (ops->set_hw_value(mac, MAC_AX_HW_SET_ID_PAUSE, &cfg) != MACSUCCESS) {
+		PHL_ERR("%s failed\n", __func__);
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_macid_grp_pause(struct rtw_hal_com_t *hal_com,
+                             u32 *macid_arr, u8 macid_arr_sz, bool pause)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_macid_pause_grp cfg = {0};
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	u8 pause_grp_sz = 0, mac_grp_sz = 0, arr_idx = 0, bit_idx = 0;
+	u32 mac_ret = MACSUCCESS;
+
+	/* size check */
+	mac_grp_sz = sizeof(cfg.pause_grp) / sizeof(cfg.pause_grp[0]);
+	pause_grp_sz = MIN(mac_grp_sz, macid_arr_sz);
+
+	for (arr_idx = 0; arr_idx < pause_grp_sz; arr_idx++) {
+		for (bit_idx = 0; bit_idx < 32; bit_idx++) {
+			if (macid_arr[arr_idx] & BIT(bit_idx)) {
+				cfg.mask_grp[arr_idx] |= BIT(bit_idx);
+				if (pause)
+					cfg.pause_grp[arr_idx] |= BIT(bit_idx);
+				else
+					cfg.pause_grp[arr_idx] &= ~BIT(bit_idx);
+			}
+		}
+	}
+
+	PHL_INFO("%s cfg pause_grp:0x%x,0x%x,0x%x,0x%x, mask_grp:0x%x,0x%x,0x%x,0x%x(%s)\n", __func__,
+			cfg.pause_grp[0], cfg.pause_grp[1], cfg.pause_grp[2], cfg.pause_grp[3],
+			cfg.mask_grp[0], cfg.mask_grp[1], cfg.mask_grp[2], cfg.mask_grp[3],
+			pause ? "pause":"unpause");
+
+	mac_ret = ops->set_hw_value(mac, MAC_AX_HW_SET_MULTI_ID_PAUSE, &cfg);
+	if (mac_ret != MACSUCCESS) {
+		PHL_ERR("%s failed(mac ret:%d)\n", __func__, mac_ret);
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return hstatus;
+}
+
+/**
+ * rtw_hal_mac_set_macid_pause_sleep - Control MACID pause/sleep
+ * @hal_com:	pointer of struct rtw_hal_com_t
+ * @macid:	sta macid for configuration
+ * @pause:	false: unpause, true: pause
+ * @sleep:	false: unsleep, true: sleep
+ *
+ * Control sta to behave as the table below
+ * pause / sleep : ACQ can Tx / MGQ can Tx
+ * false / false :   true     /    true
+ * false / true  :   false    /    false
+ * true  / false :   false    /    true
+ * true  / true  :   false    /    false
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_mac_set_macid_pause_sleep(struct rtw_hal_com_t *hal_com,
+						      u16 macid, bool pause, bool sleep)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_macid_pause_sleep_cfg cfg = {0};
+	u32 mac_status;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	cfg.macid = (u8)macid;
+	cfg.pause = pause;
+	cfg.sleep = sleep;
+
+	PHL_INFO("%s macid:%d(%s/%s) \n", __func__, macid, pause ? "pause":"unpause", sleep ? "sleep":"unsleep");
+
+	mac_status = ops->set_hw_value(mac, MAC_AX_HW_SET_ID_PAUSE_SLEEP, &cfg);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s() : fail, status = %d.\n",
+			  __func__, mac_status);
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_macid_pkt_drop(struct rtw_hal_com_t *hal_com,
+                               u16 macid, u8 sel, u8 band, u8 port, u8 mbssid)
+{
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *ops = mac->ops;
+	struct mac_ax_pkt_drop_info cfg;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	cfg.sel = sel;
+	cfg.macid = (u8)macid;
+	cfg.port = port;
+	cfg.mbssid = mbssid;
+	cfg.band = band;
+
+	if(ops->pkt_drop(mac, &cfg) != MACSUCCESS) {
+		PHL_ERR("%s failed\n", __func__);
+		hstatus = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_data_drop_once(struct rtw_hal_com_t *hal_com,
+                                  struct rtw_phl_stainfo_t *sta)
+{
+	u16 macid = sta->macid;
+	u8 band = sta->rlink->hw_band;
+	u8 port = sta->rlink->hw_port;
+#ifdef RTW_PHL_BCN
+	u8 mbssid = sta->rlink->hw_mbssid;
+#else
+	u8 mbssid = 0;
+#endif
+	enum mac_ax_pkt_drop_sel sel = MAC_AX_PKT_DROP_SEL_INVALID;
+
+	sel = MAC_AX_PKT_DROP_SEL_MACID_BE_ONCE;
+	if (rtw_hal_mac_set_macid_pkt_drop(hal_com, macid, sel,  band, port,
+	                                   mbssid) != RTW_HAL_STATUS_SUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	sel = MAC_AX_PKT_DROP_SEL_MACID_BK_ONCE;
+	if (rtw_hal_mac_set_macid_pkt_drop(hal_com, macid, sel,  band, port,
+	                                   mbssid) != RTW_HAL_STATUS_SUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	sel = MAC_AX_PKT_DROP_SEL_MACID_VI_ONCE;
+	if (rtw_hal_mac_set_macid_pkt_drop(hal_com, macid, sel,  band, port,
+	                                   mbssid) != RTW_HAL_STATUS_SUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+	sel = MAC_AX_PKT_DROP_SEL_MACID_VO_ONCE;
+	if (rtw_hal_mac_set_macid_pkt_drop(hal_com, macid, sel,  band, port,
+	                                   mbssid) != RTW_HAL_STATUS_SUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+
+enum rtw_hal_status
+rtw_hal_mac_fw_log_cfg(struct rtw_hal_com_t *hal_com,
+			struct rtw_hal_fw_log_cfg *fl_cfg)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_fw_log log_cfg = {0};
+	u32 status;
+
+	if(mac == NULL)
+		return RTW_HAL_STATUS_MAC_INIT_FAILURE;
+
+	log_cfg.level = fl_cfg->level;
+	log_cfg.output = fl_cfg->output;
+	log_cfg.comp = fl_cfg->comp;
+	log_cfg.comp_ext = fl_cfg->comp_ext;
+
+	PHL_PRINT("%s: level %d, output 0x%08x, comp 0x%08x, comp ext 0x%08x.\n",
+			__func__,
+			log_cfg.level,
+			log_cfg.output,
+			log_cfg.comp,
+			log_cfg.comp_ext);
+
+	if(log_cfg.output == MAC_AX_FL_LV_UART)
+	{
+		mac->ops->pinmux_set_func(mac, MAC_AX_GPIO_UART_TX_GPIO5);
+		mac->ops->sel_uart_tx_pin(mac, MAC_AX_UART_TX_GPIO5);
+		mac->ops->pinmux_set_func(mac, MAC_AX_GPIO_UART_RX_GPIO6);
+		mac->ops->sel_uart_rx_pin(mac, MAC_AX_UART_RX_GPIO6);
+	}
+
+	status = mac->ops->fw_log_cfg(mac, &log_cfg);
+	if (status != MACSUCCESS) {
+		PHL_ERR("%s fault, status = %d.\n", __func__, status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+#else /* !CONFIG_HAL_MAC_DBG */
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
+
+u32
+rtw_hal_mac_lamode_trig(struct rtw_hal_com_t *hal_com, u8 trig)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 rpt = 0;
+
+	rpt = mac->ops->lamode_trigger(mac, trig);
+
+	return rpt;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_lamode_cfg_buf(struct rtw_hal_com_t *hal_com, u8 buf_sel,
+			   u32 *addr_start, u32 *addr_end)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_la_buf_param param = {0};
+
+	param.la_buf_sel = buf_sel;
+
+	mac->ops->lamode_buf_cfg(mac, &param);
+
+	*addr_start = param.start_addr;
+	*addr_end = param.end_addr;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_lamode_cfg(struct rtw_hal_com_t *hal_com, u8 func_en,
+		       u8 restart_en, u8 timeout_en, u8 timeout_val,
+		       u8 data_loss_imr, u8 la_tgr_tu_sel, u8 tgr_time_val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_la_cfg cfg = {0};
+
+	cfg.la_func_en = func_en;
+	cfg.la_restart_en = restart_en;
+	cfg.la_timeout_en = timeout_en;
+	cfg.la_timeout_val = timeout_val;
+	cfg.la_data_loss_imr = data_loss_imr;
+	cfg.la_tgr_tu_sel = la_tgr_tu_sel;
+	cfg.la_tgr_time_val = tgr_time_val;
+
+	mac->ops->lamode_cfg(mac, &cfg);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_lamode_st(struct rtw_hal_com_t *hal_com, u8 *la_state,
+			  u16 *la_finish_addr, bool *la_round_up,
+			  bool *la_loss_data)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_la_status info = {0};
+
+	info = mac->ops->get_lamode_st(mac);
+
+	*la_state = info.la_sw_fsmst;
+	*la_finish_addr = info.la_buf_wptr;
+	*la_round_up = info.la_buf_rndup_ind;
+	*la_loss_data = info.la_data_loss;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+static inline void _hal_set_dft_rxfltr(struct mac_ax_rx_fltr_ctrl_t *ctrl,
+					struct mac_ax_rx_fltr_ctrl_t *mask)
+{
+	/* filter packets by address */
+	ctrl->sniffer_mode = 0;
+	mask->sniffer_mode = 1;
+
+	/* check unicast */
+	ctrl->acpt_a1_match_pkt = 1;
+	mask->acpt_a1_match_pkt = 1;
+	ctrl->uc_pkt_chk_cam_match = 1;
+	mask->uc_pkt_chk_cam_match = 1;
+
+	/* check broadcast */
+	ctrl->acpt_bc_pkt = 1;
+	mask->acpt_bc_pkt = 1;
+	ctrl->bc_pkt_chk_cam_match = 1;
+	mask->bc_pkt_chk_cam_match = 1;
+
+	/* check multicast */
+	ctrl->acpt_mc_pkt = 1;
+	mask->acpt_mc_pkt = 1;
+	/* black list filter */
+	ctrl->mc_pkt_white_lst_mode = 0;
+	mask->mc_pkt_white_lst_mode = 1;
+
+	/* check beacon */
+	ctrl->bcn_chk_en = 1;
+	mask->bcn_chk_en = 1;
+	ctrl->bcn_chk_rule = 0; /* 2: A2&A3 match */
+	mask->bcn_chk_rule = 0x3;
+
+	/* misc */
+	ctrl->acpt_pwr_mngt_pkt = 1;
+	mask->acpt_pwr_mngt_pkt = 1;
+
+	ctrl->acpt_ftm_req_pkt = 1;
+	mask->acpt_ftm_req_pkt = 1;
+}
+/**
+ * rtw_hal_mac_set_rxfltr_opt_by_mode - Set rx filter option by scenario
+ * @hal_com:	pointer of struct rtw_hal_com_t
+ * @band:	0x0: band0, 0x1: band1
+ * @mode:	scenario mode
+ *
+ * Set RX filter setting by scenario.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status
+rtw_hal_mac_set_rxfltr_opt_by_mode(struct rtw_hal_com_t *hal_com,
+				   u8 band, enum rtw_rx_fltr_opt_mode mode)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_rx_fltr_ctrl_t ctrl = {0};
+	struct mac_ax_rx_fltr_ctrl_t mask = {0};
+	u32 err;
+
+	PHL_INFO("%s band(%d), mode(%d) \n", __func__, band, mode);
+	switch (mode) {
+	case RX_FLTR_OPT_MODE_RESTORE:
+		break;
+	case RX_FLTR_OPT_MODE_SNIFFER:
+		ctrl.sniffer_mode = 1;
+		mask.sniffer_mode = 1;
+		break;
+	case RX_FLTR_OPT_MODE_SCAN:
+		ctrl.acpt_a1_match_pkt = 1;
+		mask.acpt_a1_match_pkt = 1;
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.acpt_mc_pkt = 1;
+		mask.acpt_mc_pkt = 1;
+		ctrl.uc_pkt_chk_cam_match = 0;
+		mask.uc_pkt_chk_cam_match = 1;
+		ctrl.bc_pkt_chk_cam_match = 0;
+		mask.bc_pkt_chk_cam_match = 1;
+
+		/* Do NOT check B_AX_A_BCN_CHK_RULE
+		 * when receiving beacon and probe_response
+		 */
+		ctrl.bcn_chk_en = 0;
+		mask.bcn_chk_en = 1;
+		break;
+	case RX_FLTR_OPT_MODE_STA_NORMAL:
+		#if 1
+		_hal_set_dft_rxfltr(&ctrl, &mask);
+
+		#else
+		/* filter packets by address */
+		ctrl.sniffer_mode = 0;
+		mask.sniffer_mode = 1;
+		/* check unicast */
+		ctrl.acpt_a1_match_pkt = 1;
+		mask.acpt_a1_match_pkt = 1;
+		ctrl.uc_pkt_chk_cam_match = 1;
+		mask.uc_pkt_chk_cam_match = 1;
+		/* check broadcast */
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.bc_pkt_chk_cam_match = 1;
+		mask.bc_pkt_chk_cam_match = 1;
+
+		if ((hal_com->chip_id == CHIP_WIFI6_8852A)
+		    && (hal_com->cv == CAV)) {
+			/* don't check address cam for multicast, accept all */
+			ctrl.acpt_mc_pkt = 0;
+			mask.acpt_mc_pkt = 1;
+		} else {
+			/* check multicast */
+			ctrl.acpt_mc_pkt = 1;
+			mask.acpt_mc_pkt = 1;
+			/* black list filter */
+			ctrl.mc_pkt_white_lst_mode = 0;
+			mask.mc_pkt_white_lst_mode = 1;
+		}
+		/* check beacon */
+		ctrl.bcn_chk_en = 1;
+		mask.bcn_chk_en = 1;
+		ctrl.bcn_chk_rule = 2; /* 2: A2&A3 match */
+		mask.bcn_chk_rule = 0x3;
+		/* misc */
+		ctrl.acpt_pwr_mngt_pkt = 1;
+		mask.acpt_pwr_mngt_pkt = 1;
+		ctrl.acpt_ftm_req_pkt = 1;
+		mask.acpt_ftm_req_pkt = 1;
+		#endif
+		break;
+	case RX_FLTR_OPT_MODE_STA_LINKING:
+		#if 1
+		_hal_set_dft_rxfltr(&ctrl, &mask);
+		/* check broadcast */
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.bc_pkt_chk_cam_match = 0;
+		mask.bc_pkt_chk_cam_match = 1;
+
+		/* check beacon */
+		ctrl.bcn_chk_en = 0;
+		mask.bcn_chk_en = 1;
+
+		#else
+		/* filter packets by address */
+		ctrl.sniffer_mode = 0;
+		mask.sniffer_mode = 1;
+		/* check unicast */
+		ctrl.acpt_a1_match_pkt = 1;
+		mask.acpt_a1_match_pkt = 1;
+		ctrl.uc_pkt_chk_cam_match = 1;
+		mask.uc_pkt_chk_cam_match = 1;
+		/* check broadcast */
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.bc_pkt_chk_cam_match = 0;
+		mask.bc_pkt_chk_cam_match = 1;
+
+		if ((hal_com->chip_id == CHIP_WIFI6_8852A)
+		    && (hal_com->cv == CAV)) {
+			/* don't check address cam for multicast, accept all */
+			ctrl.acpt_mc_pkt = 0;
+			mask.acpt_mc_pkt = 1;
+		} else {
+			/* check multicast */
+			ctrl.acpt_mc_pkt = 1;
+			mask.acpt_mc_pkt = 1;
+			/* black list filter */
+			ctrl.mc_pkt_white_lst_mode = 0;
+			mask.mc_pkt_white_lst_mode = 1;
+		}
+		/* check beacon */
+		ctrl.bcn_chk_en = 0;
+		mask.bcn_chk_en = 1;
+		ctrl.bcn_chk_rule = 2; /* 2: A2&A3 match */
+		mask.bcn_chk_rule = 0x3;
+		/* misc */
+		ctrl.acpt_pwr_mngt_pkt = 1;
+		mask.acpt_pwr_mngt_pkt = 1;
+		ctrl.acpt_ftm_req_pkt = 1;
+		mask.acpt_ftm_req_pkt = 1;
+		#endif
+		break;
+
+	case RX_FLTR_OPT_MODE_LSN_DISCOV:
+		_hal_set_dft_rxfltr(&ctrl, &mask);
+
+		/* check unicast */
+		ctrl.acpt_a1_match_pkt = 1;
+		mask.acpt_a1_match_pkt = 1;
+		ctrl.uc_pkt_chk_cam_match = 0;
+		mask.uc_pkt_chk_cam_match = 1;
+
+		/* check broadcast (for probe req) */
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.bc_pkt_chk_cam_match = 0;
+		mask.bc_pkt_chk_cam_match = 1;
+
+		/* disable bcn ckeck */
+		ctrl.bcn_chk_en = 0;
+		mask.bcn_chk_en = 1;
+
+		break;
+	case RX_FLTR_OPT_MODE_AP_NORMAL:
+		#if 1
+		_hal_set_dft_rxfltr(&ctrl, &mask);
+
+		/* check unicast */
+		ctrl.acpt_a1_match_pkt = 1;
+		mask.acpt_a1_match_pkt = 1;
+		ctrl.uc_pkt_chk_cam_match = 0;
+		mask.uc_pkt_chk_cam_match = 1;
+
+		/* check broadcast (for probe req) */
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.bc_pkt_chk_cam_match = 0;
+		mask.bc_pkt_chk_cam_match = 1;
+
+		#else
+		/*
+		 * SNIFFER:	   	OFF
+		 * UC address CAM A1:	ON
+		 * UC addr CAM match:	OFF
+		 * BC accept:	     	ON
+		 * BC address CAM:    	OFF
+		 * Beacon check:    	OFF
+		 */
+		/* filter packets by address */
+		ctrl.sniffer_mode = 0;
+		mask.sniffer_mode = 1;
+		/* Unicast
+		 * Do not enable address CAM matching filtering but all A1
+		 * matched ones. AP should accept all UC requests from
+		 * unknown STAs.
+		 */
+		ctrl.acpt_a1_match_pkt = 1;
+		mask.acpt_a1_match_pkt = 1;
+		ctrl.uc_pkt_chk_cam_match = 0;
+		mask.uc_pkt_chk_cam_match = 1;
+
+		/* check broadcast (Probe req) */
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.bc_pkt_chk_cam_match = 0;
+		mask.bc_pkt_chk_cam_match = 1;
+
+		if ((hal_com->chip_id == CHIP_WIFI6_8852A)
+		    && (hal_com->cv == CAV)) {
+		    /* don't check address cam for multicast, accept all */
+		    ctrl.acpt_mc_pkt = 1;
+		    mask.acpt_mc_pkt = 1;
+		} else {
+		    /* check multicast */
+		    ctrl.acpt_mc_pkt = 1;
+		    mask.acpt_mc_pkt = 1;
+		    /* black list filter */
+		    ctrl.mc_pkt_white_lst_mode = 0;
+		    mask.mc_pkt_white_lst_mode = 1;
+		}
+
+		/* check beacon */
+		ctrl.bcn_chk_en = 1;
+		mask.bcn_chk_en = 1;
+		ctrl.bcn_chk_rule = 2; /* 2: A2&A3 match */
+		mask.bcn_chk_rule = 0x3;
+
+		/* bcn_chk_rule
+		0: A3 hit
+		1: A2 hit
+		2: A2 & A3 hit
+		3: A2 | A3 hit
+		*/
+		/* accept power management frame */
+		ctrl.acpt_pwr_mngt_pkt = 1;
+		mask.acpt_pwr_mngt_pkt = 1;
+		ctrl.acpt_ftm_req_pkt = 1;
+		mask.acpt_ftm_req_pkt = 1;
+		#endif
+		break;
+#ifdef CONFIG_PHL_TEST_MP
+	case RX_FLTR_OPT_MODE_MP:
+		ctrl.acpt_a1_match_pkt = 1;
+		mask.acpt_a1_match_pkt = 1;
+		ctrl.acpt_bc_pkt = 1;
+		mask.acpt_bc_pkt = 1;
+		ctrl.acpt_mc_pkt = 1;
+		mask.acpt_mc_pkt = 1;
+		ctrl.uc_pkt_chk_cam_match = 1;
+		mask.uc_pkt_chk_cam_match = 1;
+		ctrl.bc_pkt_chk_cam_match = 1;
+		mask.bc_pkt_chk_cam_match = 1;
+		/* check beacon */
+		ctrl.bcn_chk_en = 1;
+		mask.bcn_chk_en = 1;
+		ctrl.bcn_chk_rule = 2; /* 2: A2&A3 match */
+		mask.bcn_chk_rule = 3;
+		break;
+#endif
+	}
+
+	err = mac->ops->set_rx_fltr_opt(mac, &ctrl, &mask, band);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * rtw_hal_mac_set_rxfltr_acpt_crc_err - Accept CRC error packets or not
+ * @hal_com:	pointer of struct rtw_hal_com_t
+ * @band:	0x0: band0, 0x1: band1
+ * @enable:	0: deny, 1: accept
+ *
+ * Control accepting CRC error packets or not.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_mac_set_rxfltr_acpt_crc_err(
+		struct rtw_hal_com_t *hal_com, u8 band, u8 enable)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_rx_fltr_ctrl_t ctrl = {0};
+	struct mac_ax_rx_fltr_ctrl_t mask = {0};
+	u32 err;
+
+
+	if (enable)
+		ctrl.acpt_crc32_err_pkt = 1;
+	else
+		ctrl.acpt_crc32_err_pkt = 0;
+	mask.acpt_crc32_err_pkt = 1;
+
+	err = mac->ops->set_rx_fltr_opt(mac, &ctrl, &mask, band);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * rtw_hal_mac_set_rxfltr_mpdu_size - Set max MPDU size
+ * @hal_com:	pointer of struct rtw_hal_com_t
+ * @band:	0x0: band0, 0x1: band1
+ * @size:	MPDU max size, unit: byte. 0 for no limit.
+ *
+ * MPDU size exceed Max size would be dropped.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_mac_set_rxfltr_mpdu_size(
+		struct rtw_hal_com_t *hal_com, u8 band, u16 size)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_rx_fltr_ctrl_t ctrl = {0};
+	struct mac_ax_rx_fltr_ctrl_t mask = {0};
+	u32 err;
+
+
+	/* unit of pkt_len_fltr is 512 bytes */
+	ctrl.pkt_len_fltr = size >> 9;
+	ctrl.pkt_len_fltr += (size & 0x7F) ? 1 : 0;
+	mask.pkt_len_fltr = 0x3F;
+
+	err = mac->ops->set_rx_fltr_opt(mac, &ctrl, &mask, band);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef RTW_WKARD_AP_MP
+/**
+ * rtw_hal_mac_set_rxfltr_mp_mode - Set mp mode filter
+ * @hal_com:	pointer of struct rtw_hal_com_t
+ * @band:	0x0: band0, 0x1: band1
+ * @size:	MPDU max size, unit: byte. 0 for no limit.
+ *
+ * MP mode need set rx filter size
+ * acpt_crc32_err_pkt = 1
+ * acpt_ftm_req_pkt = 0
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_mac_set_rxfltr_mp_mode(void *hal, u8 band, u16 size)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_rx_fltr_ctrl_t ctrl = {0};
+	struct mac_ax_rx_fltr_ctrl_t mask = {0};
+	u32 err;
+
+	ctrl.acpt_crc32_err_pkt = 1;
+	ctrl.acpt_ftm_req_pkt = 0;
+	/* unit of pkt_len_fltr is 512 bytes */
+	ctrl.pkt_len_fltr = size >> 9;
+	ctrl.pkt_len_fltr += (size & 0x7F) ? 1 : 0;
+
+	mask.acpt_crc32_err_pkt = 1;
+	mask.acpt_ftm_req_pkt = 1;
+	mask.pkt_len_fltr = 0x3F;
+
+	err = mac->ops->set_rx_fltr_opt(mac, &ctrl, &mask, band);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+
+/**
+ * rtw_hal_mac_set_rxfltr_by_type - Filter RX frame by frame type
+ * @hal_com:	pointer of struct rtw_hal_com_t
+ * @band:	0x0: band0, 0x1: band1
+ * @type:	enum rtw_packet_type
+ * @target:	enum rtw_rxfltr_target
+ *
+ * Set RX filter setting by 802.11 frame type and frame would be dropped or
+ * forward to specific target.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok, otherwise fail.
+ */
+enum rtw_hal_status rtw_hal_mac_set_rxfltr_by_type(
+		struct rtw_hal_com_t *hal_com, u8 band, enum rtw_packet_type type,
+		enum rtw_rxfltr_target target)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum mac_ax_pkt_t ftype;
+	enum mac_ax_fwd_target fwd;
+	u32 err;
+
+
+	switch (type) {
+	case RTW_PHL_PKT_TYPE_MGNT:
+		ftype = MAC_AX_PKT_MGNT;
+		break;
+	case RTW_PHL_PKT_TYPE_CTRL:
+		ftype = MAC_AX_PKT_CTRL;
+		break;
+	case RTW_PHL_PKT_TYPE_DATA:
+		ftype = MAC_AX_PKT_DATA;
+		break;
+	default:
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	switch (target) {
+	case RXFLTR_TARGET_DROP:
+		fwd = MAC_AX_FWD_DONT_CARE;
+		break;
+	case RXFLTR_TARGET_TO_HOST:
+		fwd = MAC_AX_FWD_TO_HOST;
+		break;
+	case RXFLTR_TARGET_TO_WLCPU:
+		fwd = MAC_AX_FWD_TO_WLAN_CPU;
+		break;
+	default:
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	err = mac->ops->set_rx_fltr_typ_opt(mac, ftype, fwd, band);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * rtw_hal_mac_set_rxfltr_by_subtype - Filter RX frame by frame type & subtype
+ * @hal_com:	pointer of struct rtw_hal_com_t
+ * @band:	0x0: band0, 0x1: band1
+ * @type:	enum rtw_packet_type
+ * @subtype:	802.11 frame subtype, value range is 0x00~0xFF.
+ * @target:	enum rtw_rxfltr_target
+ *
+ * Set RX filter setting by 802.11 frame type and subtype, then frame would be
+ * dropped or forward to specific target.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok, otherwise fail.
+ */
+enum rtw_hal_status rtw_hal_mac_set_rxfltr_by_subtype(
+		struct rtw_hal_com_t *hal_com, u8 band, enum rtw_packet_type type,
+		u8 subtype, enum rtw_rxfltr_target target)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum mac_ax_pkt_t ftype;
+	enum mac_ax_fwd_target fwd;
+	u32 err;
+
+
+	switch (type) {
+	case RTW_PHL_PKT_TYPE_MGNT:
+		ftype = MAC_AX_PKT_MGNT;
+		break;
+	case RTW_PHL_PKT_TYPE_CTRL:
+		ftype = MAC_AX_PKT_CTRL;
+		break;
+	case RTW_PHL_PKT_TYPE_DATA:
+		ftype = MAC_AX_PKT_DATA;
+		break;
+	default:
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (subtype > 0xFF)
+		return RTW_HAL_STATUS_FAILURE;
+
+	switch (target) {
+	case RXFLTR_TARGET_DROP:
+		fwd = MAC_AX_FWD_DONT_CARE;
+		break;
+	case RXFLTR_TARGET_TO_HOST:
+		fwd = MAC_AX_FWD_TO_HOST;
+		break;
+	case RXFLTR_TARGET_TO_WLCPU:
+		fwd = MAC_AX_FWD_TO_WLAN_CPU;
+		break;
+	default:
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	err = mac->ops->set_rx_fltr_typstyp_opt(mac, ftype, subtype, fwd, band);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_enable_bb_rf(struct hal_info_t *hal_info, u8 enable)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+
+	mac->ops->set_hw_value(mac, MAC_AX_HW_EN_BB_RF, &enable);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void
+rtw_hal_mac_get_buffer_data(struct rtw_hal_com_t *hal_com, u32 strt_addr,
+			    u8 *buf, u32 len, u32 dbg_path)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac_mem_dump(mac, MAC_AX_MEM_SHARED_BUF, strt_addr, buf, len, dbg_path);
+#else
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+#endif
+}
+
+enum rtw_hal_status rtw_hal_mac_ctrl_ser(struct rtw_hal_com_t *hal_com,
+					enum rtw_hal_ser_rsn rsn, bool en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	return rtw_hal_mac_ser_ctrl(hal_info, rsn, en);
+}
+
+void
+rtl_hal_dump_sec_cam_tbl(struct rtw_hal_com_t *hal_com)
+{
+	u32 i = 0;
+	u32 sec_cam_tbl_sz = 128;
+	struct hal_info_t *hal = hal_com->hal_priv;
+	struct mac_ax_adapter *mac =  hal_to_mac(hal);
+	struct sec_cam_table_t *sec_cam_table = mac->hw_info->sec_cam_table;
+	struct sec_cam_entry_t *entry = NULL;
+
+	PHL_PRINT("===== HW Info Security CAM Table =====\n");
+	PHL_PRINT("entry  valid  mac_id  key_id  key_type\n");
+
+	if (sec_cam_table == NULL)
+		return;
+
+	for (i = 0; i < sec_cam_tbl_sz; i++) {
+		entry = sec_cam_table->sec_cam_entry[i];
+		if (entry == NULL)
+			continue;
+
+		if (!entry->valid)
+			continue;
+
+		PHL_PRINT("  %3d    %3d     %3d     %3d       %3d\n",
+			i, entry->valid, entry->mac_id, entry->key_id, entry->key_type);
+	}
+}
+
+#ifdef CONFIG_HAL_MAC_DBG
+void halmac_cmd_parser(struct hal_info_t *hal_info, char input[][MAX_ARGV],
+		      u32 input_num, char *output, u32 out_len)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac->ops->halmac_cmd_parser(mac, input, input_num, output, out_len);
+}
+
+
+s32 halmac_cmd(struct hal_info_t *hal_info, char *input, char *output, u32 out_len)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	return mac->ops->halmac_cmd(mac, input, output, out_len);
+}
+
+bool rtw_hal_mac_proc_cmd(struct hal_info_t *hal_info, struct rtw_proc_cmd *incmd,
+						char *output, u32 out_len)
+{
+	if(incmd->in_type == RTW_ARG_TYPE_BUF)
+		halmac_cmd(hal_info, incmd->in.buf, output, out_len);
+	else if(incmd->in_type == RTW_ARG_TYPE_ARRAY){
+		halmac_cmd_parser(hal_info, incmd->in.vector,
+					incmd->in_cnt_len, output, out_len);
+	}
+
+	return true;
+}
+#endif /* CONFIG_HAL_MAC_DBG */
+
+static enum mac_ax_cmac_path_sel _ac_drv2mac(u8 ac, u8 wmm)
+{
+	enum mac_ax_cmac_path_sel sel = MAC_AX_CMAC_PATH_SEL_INVALID;
+
+
+	switch (ac) {
+	case 0:
+		/* BE */
+		sel = MAC_AX_CMAC_PATH_SEL_BE0;
+		break;
+	case 1:
+		/* BK */
+		sel = MAC_AX_CMAC_PATH_SEL_BK0;
+		break;
+	case 2:
+		/* VI */
+		sel = MAC_AX_CMAC_PATH_SEL_VI0;
+		break;
+	case 3:
+		/* VO */
+		sel = MAC_AX_CMAC_PATH_SEL_VO0;
+		break;
+	}
+
+	if (sel != MAC_AX_CMAC_PATH_SEL_INVALID && wmm)
+		/* wmm == 1 */
+		sel += 4;
+
+	return sel;
+}
+
+/**
+ * rtw_hal_mac_set_edca() - setup WMM EDCA parameter
+ * @hal_com:	struct rtw_hal_com_t *
+ * @band:	0x0: band0, 0x1: band1
+ * @wmm:	hardware wmm index
+ * @ac:		Access Category, 0:BE, 1:BK, 2:VI, 3:VO
+ * @param:	AIFS:BIT[7:0], CWMIN:BIT[11:8], CWMAX:BIT[15:12],
+ *		TXOP:BIT[31:16]
+ *
+ * Setup WMM EDCA parameter set for specific AC.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status
+rtw_hal_mac_set_edca(struct rtw_hal_com_t *hal_com, u8 band, u8 wmm, u8 ac,
+		     u32 param)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_edca_param edca = {0};
+	u32 err = 0;
+
+
+	edca.band = band;
+	edca.path = _ac_drv2mac(ac, wmm);
+	edca.txop_32us = param  >> 16;
+	edca.ecw_max = (param >> 12) & 0xF;
+	edca.ecw_min = (param >> 8) & 0xF;
+	edca.aifs_us = param & 0xFF;
+
+	err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_EDCA_PARAM, &edca);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_edca(struct rtw_hal_com_t *hal_com,
+                     u8 band,
+                     u8 wmm,
+                     struct rtw_edca_param *edca_param)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_edca_param edca = {0};
+	u32 err = 0;
+
+	edca.band = band;
+	edca.path = _ac_drv2mac(edca_param->ac, wmm);
+	PHL_INFO("%s\n, band(%d) ac(%d) path(%d).\n", __FUNCTION__, edca.band,
+	          edca_param->ac, edca.path);
+
+	err = mac->ops->get_hw_value(mac, MAC_AX_HW_GET_EDCA_PARAM, &edca);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("%s\n, get edca success: txop_32us(0x%x) ecw_max(0x%x) ecw_min(0x%x) aifs_us(0x%x).\n",
+				__FUNCTION__, edca.txop_32us, edca.ecw_max, edca.ecw_min, edca.aifs_us);
+	edca_param->param = (edca.txop_32us << 16) +
+	                    ((edca.ecw_max & 0xF) << 12) +
+	                    ((edca.ecw_min & 0xF) << 8) +
+	                    edca.aifs_us;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_TWT
+static void
+_dump_mac_ax_twtact_para(struct mac_ax_twtact_para *para)
+{
+	PHL_TRACE(COMP_PHL_TWT, _PHL_INFO_, "%s: act(%d), macid(%d), id(%d)\n",
+		__FUNCTION__, para->act, para->macid, para->id);
+}
+
+static void
+_dump_mac_ax_twt_para(struct mac_ax_twt_para *para)
+{
+	PHL_TRACE(COMP_PHL_TWT, _PHL_INFO_, "%s: nego_tp(%d), act(%d), trig(%d), flow_tp(%d), proct(%d), flow_id(%d), id(%d)\n",
+		__FUNCTION__, para->nego_tp, para->act, para->trig,
+		para->flow_tp, para->proct, para->flow_id, para->id);
+	PHL_TRACE(COMP_PHL_TWT, _PHL_INFO_, "%s: wake_exp(%d), band(%d), port(%d), rsp_pm(%d), wake_unit(%d), impt(%d), twtulfixmode(%d)\n",
+		__FUNCTION__, para->wake_exp, para->band, para->port,
+		para->rsp_pm, para->wake_unit, para->impt, para->twtulfixmode);
+	PHL_TRACE(COMP_PHL_TWT, _PHL_INFO_, "%s: wake_man(%d), dur(%d), trgt_h(0x%08x), trgt_l(0x%08x)\n",
+		__FUNCTION__, para->wake_man, para->dur, para->trgt_h,
+		para->trgt_l);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_twt_info_update(void *hal,
+                            struct rtw_phl_twt_info twt_info,
+                            struct rtw_wifi_role_link_t *rlink,
+                            u8 action)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_twt_para mac_twt_para = {0};
+	u32 mac_status = MACSUCCESS;
+
+	switch (twt_info.nego_type)
+	{
+		case RTW_PHL_INDIV_TWT:
+			mac_twt_para.nego_tp = MAC_AX_TWT_NEGO_TP_IND;
+			mac_twt_para.flow_id = 0;
+			break;
+		case RTW_PHL_WAKE_TBTT_INR:
+			mac_twt_para.nego_tp = MAC_AX_TWT_NEGO_TP_WAKE;
+			mac_twt_para.flow_id = 0;
+			break;
+		case RTW_PHL_BCAST_TWT:
+		case RTW_PHL_MANAGE_BCAST_TWT:
+			mac_twt_para.nego_tp = MAC_AX_TWT_NEGO_TP_BRC;
+			mac_twt_para.flow_id = twt_info.bcast_twt_id;
+			break;
+		default:
+			PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s : Error TWT nego type %d\n", __func__, twt_info.nego_type);
+			return RTW_HAL_STATUS_FAILURE;
+	}
+	switch (action) {
+		case TWT_CFG_ADD:
+			mac_twt_para.act = MAC_AX_TWT_ACT_TP_ADD;
+			break;
+		case TWT_CFG_DELETE:
+			mac_twt_para.act = MAC_AX_TWT_ACT_TP_DEL;
+			break;
+		case TWT_CFG_MODIFY:
+			mac_twt_para.act = MAC_AX_TWT_ACT_TP_MOD;
+			break;
+		default:
+			PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s : Error TWT action %d\n", __func__, action);
+			return RTW_HAL_STATUS_FAILURE;
+	}
+	mac_twt_para.trig = twt_info.trigger;
+	mac_twt_para.flow_tp = twt_info.flow_type;
+	mac_twt_para.proct = twt_info.twt_protection;
+	mac_twt_para.id = twt_info.twt_id;
+	mac_twt_para.wake_exp = twt_info.twt_wake_int_exp;
+	mac_twt_para.rsp_pm = twt_info.responder_pm_mode;
+	mac_twt_para.wake_unit = twt_info.wake_dur_unit;
+	mac_twt_para.impt = twt_info.implicit_lastbcast;
+	mac_twt_para.wake_man = twt_info.twt_wake_int_mantissa;
+	mac_twt_para.dur = twt_info.nom_min_twt_wake_dur;
+	mac_twt_para.trgt_l = twt_info.target_wake_time_l;
+	mac_twt_para.trgt_h = twt_info.target_wake_time_h;
+
+	mac_twt_para.band = rlink->hw_band;
+	mac_twt_para.port = rlink->hw_port;
+	_dump_mac_ax_twt_para(&mac_twt_para);
+	/* HalMac API to setup/delete TWT config*/
+	mac_status = mac->ops->twt_info_upd_h2c(mac, &mac_twt_para);
+	if (MACSUCCESS != mac_status){
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s: mac_twt_info_upd_h2c fail(%d)\n",
+			__FUNCTION__, mac_status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_twt_sta_update(void *hal, u8 macid, u8 twt_id, u8 action)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_twtact_para mac_twtact_para = {0};
+	u32 mac_status = MACSUCCESS;
+
+	switch (action) {
+		case TWT_STA_ADD_MACID:
+			mac_twtact_para.act = MAC_AX_TWTACT_ACT_TP_ADD;
+			break;
+		case TWT_STA_DEL_MACID:
+			mac_twtact_para.act = MAC_AX_TWTACT_ACT_TP_DEL;
+			break;
+		case TWT_STA_TETMINATW_SP:
+			mac_twtact_para.act = MAC_AX_TWTACT_ACT_TP_TRMNT;
+			break;
+		case TWT_STA_SUSPEND_TWT:
+			mac_twtact_para.act = MAC_AX_TWTACT_ACT_TP_SUS;
+			break;
+		case TWT_STA_RESUME_TWT:
+			mac_twtact_para.act = MAC_AX_TWTACT_ACT_TP_RSUM;
+			break;
+		default:
+			PHL_ERR("%s : Error TWT action %d\n", __func__, action);
+			return RTW_HAL_STATUS_FAILURE;
+	}
+	mac_twtact_para.macid = macid;
+	mac_twtact_para.id = twt_id;
+	_dump_mac_ax_twtact_para(&mac_twtact_para);
+	/* Call HalMac API to setup/delete TWT STA config*/
+	mac_status = mac->ops->twt_act_h2c(mac, &mac_twtact_para);
+	if (MACSUCCESS != mac_status) {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s: mac_twt_act_h2c fail(%d)\n",
+			__FUNCTION__, mac_status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_twt_sta_announce(void *hal, u8 macid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_twtanno_para mac_twtanno_para = {0};
+	u32 mac_status = MACSUCCESS;
+
+	mac_twtanno_para.macid = macid;
+	mac_status = mac->ops->twt_anno_h2c(mac, &mac_twtanno_para);
+	if (MACSUCCESS != mac_status) {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "rtw_hal_mac_twt_sta_announce(): mac_twt_anno_h2c fail(%d)\n",
+			mac_status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /* CONFIG_PHL_TWT */
+
+
+enum rtw_hal_status
+rtw_hal_mac_get_hwseq(void *hal, u16 macid, u8 ref_sel, u8 ssn_sel, u16 *hw_seq)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_dctl_seq_cfg seq_info = {0};
+	u32 mac_status = MACSUCCESS;
+
+	mac_status = mac->ops->get_hwseq_cfg(mac, (u8)macid, ref_sel, &seq_info);
+	if (MACSUCCESS != mac_status) {
+		PHL_TRACE(COMP_PHL_XMIT, _PHL_DEBUG_, "rtw_hal_mac_get_hwseq(): get hw sequence fail!! (%d)\n",
+			mac_status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_TRACE(COMP_PHL_XMIT, _PHL_DEBUG_, "rtw_hal_mac_get_hwseq(): ref_sel = %d, ssn_sel = %d\n",
+			ref_sel,
+			ssn_sel);
+
+	/* return hw_seq */
+	switch (ssn_sel){
+	case 0:
+		*hw_seq = (u16)seq_info.seq0_val;
+		break;
+	case 1:
+		*hw_seq = (u16)seq_info.seq1_val;
+		break;
+	case 2:
+		*hw_seq = (u16)seq_info.seq2_val;
+		break;
+	case 3:
+		*hw_seq = (u16)seq_info.seq3_val;
+		break;
+	default:
+		PHL_TRACE(COMP_PHL_XMIT, _PHL_ERR_, "rtw_hal_mac_get_hwseq(): no ssn_sel = (%d)\n", ssn_sel);
+		break;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+/**
+ * rtw_hal_mac_get_ampdu_cfg() - get ampdu config
+ * @hal_com:    struct rtw_hal_com_t *
+ * @band:       0x0: band0, 0x1: band1
+ * @cfg:        struct hal_ax_ampdu_cfg *
+ * Get ampdu config.
+ * Return RTW_HAL_STATUS_SUCCESS when query is done.
+ */
+enum rtw_hal_status
+rtw_hal_mac_get_ampdu_cfg(struct rtw_hal_com_t *hal_com,
+                          u8 band,
+                          struct mac_ax_ampdu_cfg *cfg)
+{
+	/* To Do: need to refine after Mac api updating*/
+#if 0
+     struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+     struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+     u32 err = 0;
+     cfg->band = band;
+     err = mac->ops->get_hw_value(mac, MAC_AX_HW_GET_AMPDU_CFG, cfg);
+     if (err)
+	     return RTW_HAL_STATUS_FAILURE;
+#endif
+     return RTW_HAL_STATUS_SUCCESS;
+}
+
+/*
+ * rtw_hal_mac_set_rty_lmt() - setup retry limit parameter
+ * @hal_com:	struct rtw_hal_com_t *
+ * @macid:
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status
+rtw_hal_mac_set_rty_lmt(struct rtw_hal_com_t *hal_com, u8 macid,
+	u8 rts_lmt_sel, u8 rts_lmt_val, u8 data_lmt_sel, u8 data_lmt_val)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_cctl_rty_lmt_cfg cfg = {0};
+	u32 err = 0;
+
+
+	cfg.macid = macid;
+	cfg.data_lmt_sel = data_lmt_sel;
+	cfg.data_lmt_val = data_lmt_val;
+	cfg.rts_lmt_sel = rts_lmt_sel;
+	cfg.rts_lmt_val = rts_lmt_val;
+
+	err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_CCTL_RTY_LMT, &cfg);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * rtw_hal_mac_is_tx_mgnt_empty() - Get tx mgnt queue status
+ * @hal_info:	struct rtw_hal_info_t *
+ * @band:	0x0: band0, 0x1: band1
+ * @st:		queue status, 1 for empty and 0 for not empty.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status
+rtw_hal_mac_is_tx_mgnt_empty(struct hal_info_t *hal_info, u8 band, u8 *st)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_tx_queue_empty q_st = {{0}};
+	u32 err;
+
+
+	err = mac->ops->is_txq_empty(mac, &q_st);
+	if (err != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	if ((band == 0) && (q_st.band0_mgnt_empty == DLE_QUEUE_EMPTY))
+		*st = 1;
+	else if ((band == 1) && (q_st.band1_mgnt_empty == DLE_QUEUE_EMPTY))
+		*st = 1;
+	else
+		*st = 0;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+/* FW Sounding Command */
+/* 1. NDPA Content */
+void _hal_max_ax_snd_cmd_ndpa(struct mac_ax_ndpa_para *mac_ndpa,
+			      struct hal_ndpa_para *hal_ndpa,
+			      bool he, u8 sta_nr)
+{
+	u8 i = 0;
+	struct hal_he_ndpa_sta_info *hal_he_sta = NULL;
+	struct hal_vht_ndpa_sta_info *hal_vht_sta = NULL;
+
+	mac_ndpa->common.frame_ctl = hal_ndpa->common.frame_ctl;
+	mac_ndpa->common.duration = hal_ndpa->common.duration;
+	for (i = 0; i < MAC_ALEN; i++) {
+		mac_ndpa->common.addr1[i] = hal_ndpa->common.addr1[i];
+		mac_ndpa->common.addr2[i] = hal_ndpa->common.addr2[i];
+	}
+	mac_ndpa->snd_dialog.dialog = hal_ndpa->snd_dialog.token;
+	mac_ndpa->snd_dialog.he = hal_ndpa->snd_dialog.he;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "halmac he 0x%x sta_nr %d \n", he, sta_nr);
+	if (he) {
+		for (i = 0; (i < sta_nr)&&(i < HAL_MAX_HE_SND_STA_NUM); i++) {
+			hal_he_sta = (struct hal_he_ndpa_sta_info *)
+				      &hal_ndpa->ndpa_sta_info[i];
+			mac_ndpa->he_para.sta_info[i].aid = hal_he_sta->aid;
+			mac_ndpa->he_para.sta_info[i].bw = hal_he_sta->bw;
+			mac_ndpa->he_para.sta_info[i].fb_ng = hal_he_sta->fb_ng;
+			mac_ndpa->he_para.sta_info[i].disambiguation = 1;
+			mac_ndpa->he_para.sta_info[i].cb = hal_he_sta->cb;//cb nc
+			mac_ndpa->he_para.sta_info[i].nc = hal_he_sta->nc;//cb nc
+			PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "halmac cmd buf HE (%d) : aid 0x%x bw 0x%x fbng 0x%x cb 0x%x nc 0x%x\n",
+				i, mac_ndpa->he_para.sta_info[i].aid,
+				mac_ndpa->he_para.sta_info[i].bw,
+				mac_ndpa->he_para.sta_info[i].fb_ng,
+				mac_ndpa->he_para.sta_info[i].cb,
+				mac_ndpa->he_para.sta_info[i].nc);
+		}
+	} else {
+		for (i = 0; (i < sta_nr)&&(i < HAL_MAX_VHT_SND_STA_NUM); i++) {
+			hal_vht_sta = (struct hal_vht_ndpa_sta_info *)
+				      &hal_ndpa->ndpa_sta_info[i];
+			mac_ndpa->vht_para.sta_info[i].aid =
+					(u16)hal_vht_sta->aid12;
+			mac_ndpa->vht_para.sta_info[i].fb_type =
+					(u16)hal_vht_sta->feedback_type;
+			mac_ndpa->vht_para.sta_info[i].nc =
+					(u16)hal_vht_sta->nc;
+			PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "halmac cmd buf VHT(%d) : aid 0x%x fb 0x%x nc 0x%x\n",
+				i,
+				mac_ndpa->vht_para.sta_info[i].aid,
+				mac_ndpa->vht_para.sta_info[i].fb_type,
+				mac_ndpa->vht_para.sta_info[i].nc);
+		}
+	}
+}
+/* 2-1. BFRP Content - VHT */
+void _hal_max_ax_snd_cmd_bfrp_vht(struct mac_ax_bfrp_para *mac_bfrp,
+				 struct hal_bfrp_para *hal_bfrp, u8 bfrp_num)
+{
+	struct mac_ax_vht_bfrp_para *mac_vht_bfrp = NULL;
+	struct hal_bfrp_vht *hal_vht_bfrp = NULL;
+	u8 i = 0, j = 0;
+	for (i = 0; (i <= bfrp_num)&&(i < HAL_MAX_VHT_BFRP_NUM); i++){
+		mac_bfrp->hdr[i].frame_ctl = hal_bfrp->hdr[i].frame_ctl;
+		mac_bfrp->hdr[i].duration = hal_bfrp->hdr[i].duration;
+		for (j = 0; j < MAC_ALEN; j++) {
+			mac_bfrp->hdr[i].addr1[j] = hal_bfrp->hdr[i].addr1[j];
+			mac_bfrp->hdr[i].addr2[j] = hal_bfrp->hdr[i].addr2[j];
+		}
+		mac_vht_bfrp = &mac_bfrp->vht_para[i];
+		hal_vht_bfrp = &hal_bfrp->vht_para[i];
+		mac_vht_bfrp->retransmission_bitmap = hal_vht_bfrp->rexmit_bmp;
+	}
+}
+/* 2-2. BFRP Content - HE */
+void _hal_max_ax_snd_cmd_bfrp_he(struct mac_ax_bfrp_para *mac_bfrp,
+			      struct hal_bfrp_para *hal_bfrp,
+			      u8 num_1, u8 num_2)
+{
+	u8 i = 0;
+	struct mac_ax_he_bfrp_para *mac_he_bfrp = NULL;
+	struct hal_bfrp_he *hal_he_bfrp = NULL;
+
+	/* BFRP-0 */
+	if (num_1) {
+		mac_bfrp->hdr[0].frame_ctl = hal_bfrp->hdr[0].frame_ctl;
+		mac_bfrp->hdr[0].duration = hal_bfrp->hdr[0].duration;
+		for (i = 0; i < MAC_ALEN; i++) {
+			mac_bfrp->hdr[0].addr1[i] = hal_bfrp->hdr[0].addr1[i];
+			mac_bfrp->hdr[0].addr2[i] = hal_bfrp->hdr[0].addr2[i];
+		}
+		mac_he_bfrp = &mac_bfrp->he_para[0];
+		hal_he_bfrp = &hal_bfrp->he_para[0];
+		mac_he_bfrp->common.tgr_info = hal_he_bfrp->common.tgr_info;
+		mac_he_bfrp->common.ul_len = hal_he_bfrp->common.ul_len;
+		mac_he_bfrp->common.more_tf = hal_he_bfrp->common.more_tf;
+		mac_he_bfrp->common.cs_rqd = hal_he_bfrp->common.cs_rqd;
+		mac_he_bfrp->common.ul_bw = hal_he_bfrp->common.ul_bw;
+		mac_he_bfrp->common.gi_ltf = hal_he_bfrp->common.gi_ltf;
+		mac_he_bfrp->common.mimo_ltfmode =
+					hal_he_bfrp->common.mimo_ltfmode;
+		mac_he_bfrp->common.num_heltf = hal_he_bfrp->common.num_heltf;
+		mac_he_bfrp->common.ul_pktext = hal_he_bfrp->common.ul_pktext;
+		mac_he_bfrp->common.ul_stbc = hal_he_bfrp->common.ul_stbc;
+		mac_he_bfrp->common.ldpc_extra_sym =
+					hal_he_bfrp->common.ldpc_extra_sym;
+		mac_he_bfrp->common.dplr = hal_he_bfrp->common.dplr;
+		mac_he_bfrp->common.ap_tx_pwr = hal_he_bfrp->common.ap_tx_pwr;
+		mac_he_bfrp->common.ul_sr  = hal_he_bfrp->common.ul_sr;
+		mac_he_bfrp->common.ul_siga2_rsvd =
+					hal_he_bfrp->common.ul_siga2_rsvd;
+		for( i = 0; (i < num_1)&&(i < HAL_MAX_HE_BFRP_STA_NUM); i++) {
+			mac_he_bfrp->user[i].aid12 =
+					hal_he_bfrp->user[i].aid12;
+			mac_he_bfrp->user[i].fbseg_rexmit_bmp =
+					hal_he_bfrp->fbseg_rexmit_bmp[i];
+			mac_he_bfrp->user[i].ru_pos =
+					hal_he_bfrp->user[i].ru_pos;
+			mac_he_bfrp->user[i].ss_alloc =
+					hal_he_bfrp->user[i].ss_alloc;
+			mac_he_bfrp->user[i].ul_dcm =
+					hal_he_bfrp->user[i].ul_dcm;
+			mac_he_bfrp->user[i].ul_fec_code =
+					hal_he_bfrp->user[i].ul_fec_code;
+			mac_he_bfrp->user[i].ul_mcs =
+					hal_he_bfrp->user[i].ul_mcs;
+			mac_he_bfrp->user[i].ul_tgt_rssi =
+					hal_he_bfrp->user[i].ul_tgt_rssi;
+		}
+	}
+	/* BFRP - 1 */
+	if (num_2) {
+		mac_bfrp->hdr[1].frame_ctl = hal_bfrp->hdr[1].frame_ctl;
+		mac_bfrp->hdr[1].duration = hal_bfrp->hdr[1].duration;
+		for (i = 0; i < MAC_ALEN; i++) {
+			mac_bfrp->hdr[1].addr1[i] = hal_bfrp->hdr[1].addr1[i];
+			mac_bfrp->hdr[1].addr2[i] = hal_bfrp->hdr[1].addr2[i];
+		}
+		mac_he_bfrp = &mac_bfrp->he_para[1];
+		hal_he_bfrp = &hal_bfrp->he_para[1];
+		mac_he_bfrp->common.tgr_info = hal_he_bfrp->common.tgr_info;
+		mac_he_bfrp->common.ul_len = hal_he_bfrp->common.ul_len;
+		mac_he_bfrp->common.more_tf = hal_he_bfrp->common.more_tf;
+		mac_he_bfrp->common.cs_rqd = hal_he_bfrp->common.cs_rqd;
+		mac_he_bfrp->common.ul_bw = hal_he_bfrp->common.ul_bw;
+		mac_he_bfrp->common.gi_ltf = hal_he_bfrp->common.gi_ltf;
+		mac_he_bfrp->common.mimo_ltfmode =
+					hal_he_bfrp->common.mimo_ltfmode;
+		mac_he_bfrp->common.num_heltf = hal_he_bfrp->common.num_heltf;
+		mac_he_bfrp->common.ul_pktext = hal_he_bfrp->common.ul_pktext;
+		mac_he_bfrp->common.ul_stbc = hal_he_bfrp->common.ul_stbc;
+		mac_he_bfrp->common.ldpc_extra_sym =
+					hal_he_bfrp->common.ldpc_extra_sym;
+		mac_he_bfrp->common.dplr = hal_he_bfrp->common.dplr;
+		mac_he_bfrp->common.ap_tx_pwr = hal_he_bfrp->common.ap_tx_pwr;
+		mac_he_bfrp->common.ul_sr  = hal_he_bfrp->common.ul_sr;
+		mac_he_bfrp->common.ul_siga2_rsvd =
+					hal_he_bfrp->common.ul_siga2_rsvd;
+		for( i = 0; (i < num_2) && (i < HAL_MAX_HE_BFRP_STA_NUM); i++) {
+			mac_he_bfrp->user[i].aid12 =
+					hal_he_bfrp->user[i].aid12;
+			mac_he_bfrp->user[i].fbseg_rexmit_bmp =
+					hal_he_bfrp->fbseg_rexmit_bmp[i];
+			mac_he_bfrp->user[i].ru_pos =
+					hal_he_bfrp->user[i].ru_pos;
+			mac_he_bfrp->user[i].ss_alloc =
+					hal_he_bfrp->user[i].ss_alloc;
+			mac_he_bfrp->user[i].ul_dcm =
+					hal_he_bfrp->user[i].ul_dcm;
+			mac_he_bfrp->user[i].ul_fec_code =
+					hal_he_bfrp->user[i].ul_fec_code;
+			mac_he_bfrp->user[i].ul_mcs =
+					hal_he_bfrp->user[i].ul_mcs;
+			mac_he_bfrp->user[i].ul_tgt_rssi =
+					hal_he_bfrp->user[i].ul_tgt_rssi;
+		}
+	}
+}
+
+/* 2-3 he bfrp f2p cmd */
+void _hal_max_ax_snd_cmd_bfrp_he_f2p(struct mac_ax_snd_f2P *mac_bfrp_f2p,
+				     struct hal_bfrp_para *hal_bfrp,
+				     u8 num_1, u8 num_2)
+{
+	struct hal_bfrp_he *hal_he_bfrp = NULL;
+	if (num_1) {
+		hal_he_bfrp = &hal_bfrp->he_para[0];
+		mac_bfrp_f2p[0].csi_len_bfrp =
+			hal_he_bfrp->f2p_info.csi_len_bfrp;
+		mac_bfrp_f2p[0].tb_t_pe_bfrp =
+			hal_he_bfrp->f2p_info.tb_t_pe_bfrp;
+		mac_bfrp_f2p[0].tri_pad_bfrp =
+			hal_he_bfrp->f2p_info.tri_pad_bfrp;
+		mac_bfrp_f2p[0].ul_cqi_rpt_tri_bfrp =
+			hal_he_bfrp->f2p_info.ul_cqi_rpt_tri_bfrp;
+		mac_bfrp_f2p[0].rf_gain_idx_bfrp =
+			hal_he_bfrp->f2p_info.rf_gain_idx_bfrp;
+		mac_bfrp_f2p[0].fix_gain_en_bfrp =
+			hal_he_bfrp->f2p_info.fix_gain_en_bfrp;
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "==> _hal_max_ax_snd_cmd_bfrp_he_f2p[0] \n");
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[0].csi_len_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[0].csi_len_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[0].tb_t_pe_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[0].tb_t_pe_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[0].tri_pad_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[0].tri_pad_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[0].ul_cqi_rpt_tri_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[0].ul_cqi_rpt_tri_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[0].rf_gain_idx_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[0].rf_gain_idx_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[0].fix_gain_en_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[0].fix_gain_en_bfrp);
+	}
+	if (num_2) {
+		hal_he_bfrp = &hal_bfrp->he_para[1];
+		mac_bfrp_f2p[1].csi_len_bfrp =
+			hal_he_bfrp->f2p_info.csi_len_bfrp;
+		mac_bfrp_f2p[1].tb_t_pe_bfrp =
+			hal_he_bfrp->f2p_info.tb_t_pe_bfrp;
+		mac_bfrp_f2p[1].tri_pad_bfrp =
+			hal_he_bfrp->f2p_info.tri_pad_bfrp;
+		mac_bfrp_f2p[1].ul_cqi_rpt_tri_bfrp =
+			hal_he_bfrp->f2p_info.ul_cqi_rpt_tri_bfrp;
+		mac_bfrp_f2p[1].rf_gain_idx_bfrp =
+			hal_he_bfrp->f2p_info.rf_gain_idx_bfrp;
+		mac_bfrp_f2p[1].fix_gain_en_bfrp =
+			hal_he_bfrp->f2p_info.fix_gain_en_bfrp;
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "==> _hal_max_ax_snd_cmd_bfrp_he_f2p[1] \n");
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[1].csi_len_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[1].csi_len_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[1].tb_t_pe_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[1].tb_t_pe_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[1].tri_pad_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[1].tri_pad_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[1].ul_cqi_rpt_tri_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[1].ul_cqi_rpt_tri_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[1].rf_gain_idx_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[1].rf_gain_idx_bfrp);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_,
+			  "mac_bfrp_f2p[1].fix_gain_en_bfrp = 0x%x \n",
+			  mac_bfrp_f2p[1].fix_gain_en_bfrp);
+	}
+}
+
+
+/* 3. WD Content */
+void _hal_max_ax_snd_cmd_wd(struct mac_ax_snd_wd_para *mac_wd,
+			    struct hal_snd_wd_para *hal_wd)
+{
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "==> _hal_max_ax_snd_cmd_wd \n");
+	mac_wd->txpktsize = hal_wd->txpktsize;
+	mac_wd->ndpa_duration = hal_wd->ndpa_duration;
+	mac_wd->datarate = hal_wd->datarate;
+	mac_wd->macid = hal_wd->macid;
+	mac_wd->force_txop = hal_wd->force_txop;
+	mac_wd->data_bw = hal_wd->data_bw;
+	mac_wd->gi_ltf = hal_wd->gi_ltf;
+	mac_wd->data_er = hal_wd->data_er;
+	mac_wd->data_dcm = hal_wd->data_dcm;
+	mac_wd->data_stbc = hal_wd->data_stbc;
+	mac_wd->data_ldpc = hal_wd->data_ldpc;
+	mac_wd->data_bw_er = hal_wd->data_bw_er;
+	mac_wd->multiport_id = hal_wd->multiport_id;
+	mac_wd->mbssid = hal_wd->mbssid;
+
+	mac_wd->signaling_ta_pkt_sc = hal_wd->signaling_ta_pkt_sc;
+	mac_wd->sw_define = hal_wd->sw_define;
+	mac_wd->txpwr_ofset_type = hal_wd->txpwr_ofset_type;
+	mac_wd->lifetime_sel = hal_wd->lifetime_sel;
+	mac_wd->stf_mode = hal_wd->stf_mode;
+	mac_wd->disdatafb = hal_wd->disdatafb;
+	mac_wd->data_txcnt_lmt_sel = hal_wd->data_txcnt_lmt_sel;
+	mac_wd->data_txcnt_lmt = hal_wd->data_txcnt_lmt;
+	mac_wd->sifs_tx = hal_wd->sifs_tx;
+	mac_wd->snd_pkt_sel = hal_wd->snd_pkt_sel;
+	mac_wd->ndpa = hal_wd->ndpa;
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->txpktsize = 0x%x \n", mac_wd->txpktsize);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->ndpa_duration = 0x%x \n", mac_wd->ndpa_duration);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->datarate = 0x%x \n", mac_wd->datarate);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "mac_wd->macid = 0x%x \n", mac_wd->macid);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->force_txop = 0x%x \n", mac_wd->force_txop);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_bw = 0x%x \n", mac_wd->data_bw);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->gi_ltf = 0x%x \n", mac_wd->gi_ltf);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_er = 0x%x \n", mac_wd->data_er);
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_dcm = 0x%x \n", mac_wd->data_dcm);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_stbc = 0x%x \n", mac_wd->data_stbc);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_ldpc = 0x%x \n", mac_wd->data_ldpc);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_bw_er = 0x%x \n", mac_wd->data_bw_er);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->multiport_id = 0x%x \n", mac_wd->multiport_id);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->mbssid = 0x%x \n", mac_wd->mbssid);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->signaling_ta_pkt_sc = 0x%x \n", mac_wd->signaling_ta_pkt_sc);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->sw_define = 0x%x \n", mac_wd->sw_define);
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->txpwr_ofset_type = 0x%x \n", mac_wd->txpwr_ofset_type);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->lifetime_sel = 0x%x \n", mac_wd->lifetime_sel);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->stf_mode = 0x%x \n", mac_wd->stf_mode);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->disdatafb = 0x%x \n", mac_wd->disdatafb);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_txcnt_lmt_sel = 0x%x \n", mac_wd->data_txcnt_lmt_sel);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->data_txcnt_lmt = 0x%x \n", mac_wd->data_txcnt_lmt);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_DEBUG_, "mac_wd->sifs_tx = 0x%x \n", mac_wd->sifs_tx);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "mac_wd->snd_pkt_sel = 0x%x \n", mac_wd->snd_pkt_sel);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "mac_wd->ndpa = 0x%x \n", mac_wd->ndpa);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "<== _hal_max_ax_snd_cmd_wd \n");
+}
+
+
+enum rtw_hal_status
+hal_mac_ax_send_fw_snd(struct hal_info_t *hal_info,
+		       struct hal_ax_fwcmd_snd *hal_cmd)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct mac_ax_fwcmd_snd snd_cmd = {0};
+	u8 i = 0, he = 0, sta_nr = 0;
+
+	/* 1. COMMAND Common */
+	snd_cmd.frexgtype = hal_cmd->frame_ex_type;
+	snd_cmd.bfrp0_user_num = hal_cmd->bfrp0_sta_nr;
+	snd_cmd.bfrp1_user_num = hal_cmd->bfrp1_sta_nr;
+	for (i = 0; i < HAL_MAX_HE_SND_STA_NUM; i++) {
+		snd_cmd.macid[i] = (u8)hal_cmd->macid[i];
+	}
+
+	he = (hal_cmd->frame_ex_type >= HAL_FEXG_TYPE_AX_SU) ? 1 : 0;
+
+	if (hal_cmd->frame_ex_type == HAL_FEXG_TYPE_AX_SU) {
+		sta_nr = 1;
+	} else {
+		sta_nr = he ? (hal_cmd->bfrp0_sta_nr + hal_cmd->bfrp1_sta_nr) :
+			 (hal_cmd->frame_ex_type - HAL_FEXG_TYPE_AC_SU + 1);
+	}
+
+	/* 2. NDPA Content */
+	_hal_max_ax_snd_cmd_ndpa(&snd_cmd.pndpa, &hal_cmd->ndpa,
+			         he, sta_nr);
+	/* 3. BFRP Content */
+	if (he) {
+		_hal_max_ax_snd_cmd_bfrp_he(&snd_cmd.pbfrp, &hal_cmd->bfrp,
+				hal_cmd->bfrp0_sta_nr, hal_cmd->bfrp1_sta_nr);
+		_hal_max_ax_snd_cmd_bfrp_he_f2p(snd_cmd.f2p, &hal_cmd->bfrp,
+				hal_cmd->bfrp0_sta_nr, hal_cmd->bfrp1_sta_nr);
+	} else {
+		_hal_max_ax_snd_cmd_bfrp_vht(&snd_cmd.pbfrp, &hal_cmd->bfrp,
+					     (sta_nr - 1));
+	}
+	/* 4. WD Content */
+	/* 4-1 NDPA */
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "NDPA WD: \n");
+	_hal_max_ax_snd_cmd_wd(&snd_cmd.wd[0], &hal_cmd->wd[0]);
+	/* 4-2 NDP */
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "NDP WD: \n");
+	_hal_max_ax_snd_cmd_wd(&snd_cmd.wd[1], &hal_cmd->wd[1]);
+	/* 4-3 BFRP*/
+	if (he) {
+		if (hal_cmd->bfrp0_sta_nr) {
+			PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "HE BFRP-1 WD: \n");
+			_hal_max_ax_snd_cmd_wd(&snd_cmd.wd[2],
+					       &hal_cmd->wd[2]);
+		}
+		if (hal_cmd->bfrp1_sta_nr) {
+			PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "HE BFRP-2 WD: \n");
+			_hal_max_ax_snd_cmd_wd(&snd_cmd.wd[3],
+					       &hal_cmd->wd[3]);
+		}
+	} else {
+		for (i = 0; i < (sta_nr - 1)&&(i < HAL_MAX_VHT_BFRP_NUM);
+		     i++) {
+			PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "VHT BFRP %d WD: \n", i);
+			_hal_max_ax_snd_cmd_wd(&snd_cmd.wd[2 + i],
+					       &hal_cmd->wd[2 + i]);
+		}
+	}
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[fw_snd] snd_cmd.macid[0] = 0x%x \n", snd_cmd.macid[0]);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[fw_snd] snd_cmd.bfrp0_user_num = 0x%x \n", snd_cmd.bfrp0_user_num);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[fw_snd] snd_cmd.bfrp1_user_num = 0x%x \n", snd_cmd.bfrp1_user_num);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[fw_snd] snd_cmd.mode = 0x%x \n", snd_cmd.mode);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[fw_snd] snd_cmd.frexgtype = 0x%x \n", snd_cmd.frexgtype);
+	hstatus = mac->ops->set_snd_para(mac, &snd_cmd);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[fw_snd] hal_mac_ax_send_fw_snd hstatus = 0x%x\n", hstatus);
+
+	return hstatus;
+}
+
+
+
+enum rtw_hal_status
+rtw_hal_mac_tx_mode_sel(struct hal_info_t *hal_info, u8 fw_tx, u8 txop_wmm_en_bm)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+
+	struct mac_ax_mac_tx_mode_sel mode_sel = {0};
+
+	if (fw_tx)
+		mode_sel.sw_mode_band0_en = 1;
+	if (txop_wmm_en_bm & BIT(0))
+		mode_sel.txop_rot_wmm0_en = 1;
+	if (txop_wmm_en_bm & BIT(1))
+		mode_sel.txop_rot_wmm1_en = 1;
+	if (txop_wmm_en_bm & BIT(2))
+		mode_sel.txop_rot_wmm2_en = 1;
+	if (txop_wmm_en_bm & BIT(3))
+		mode_sel.txop_rot_wmm3_en = 1;
+
+	mac->ops->tx_mode_sel(mac, &mode_sel);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_RTW_DEBUG_BCN_TX
+void hal_mac_process_bcn_stats(void *hal, struct mac_ax_bcn_cnt *bcn_cnt)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct bcn_entry_pool *bcn_pool = &hal_com->bcn_pool;
+	struct rtw_bcn_entry *bcn_entry = NULL;
+	struct rtw_bcn_entry *tmp_entry, *type = NULL;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+
+	if(bcn_pool->bcn_num == 0)
+		return;
+
+	_os_spinlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	phl_list_for_loop_safe(tmp_entry, type,
+		struct rtw_bcn_entry, &bcn_pool->bcn_list, list)
+	{
+		if (tmp_entry->bcn_hw.port == bcn_cnt->port &&
+		   tmp_entry->bcn_hw.mbssid == bcn_cnt->mbssid &&
+		   tmp_entry->bcn_hw.band == bcn_cnt->band) {
+			bcn_entry = tmp_entry;
+			break;
+		}
+	}
+
+	if (bcn_entry) {
+		bool print = false;
+		u8 dbg_lvl = BCN_DBG_LVL;
+
+		bcn_entry->bcn_stats.bcn_ok += bcn_cnt->ok_cnt;
+		bcn_entry->bcn_stats.bcn_fail += 10 - bcn_cnt->ok_cnt;
+
+		/* record detailed C2H Beacon Report */
+		bcn_entry->bcn_stats.ok_cnt += bcn_cnt->ok_cnt;
+		bcn_entry->bcn_stats.cca_cnt += bcn_cnt->cca_cnt;
+		bcn_entry->bcn_stats.edcca_cnt += bcn_cnt->edcca_cnt;
+		bcn_entry->bcn_stats.nav_cnt += bcn_cnt->nav_cnt;
+		bcn_entry->bcn_stats.txon_cnt += bcn_cnt->txon_cnt;
+		bcn_entry->bcn_stats.mac_cnt += bcn_cnt->mac_cnt;
+		bcn_entry->bcn_stats.others_cnt += bcn_cnt->others_cnt;
+		bcn_entry->bcn_stats.lock_cnt += bcn_cnt->lock_cnt;
+		bcn_entry->bcn_stats.cmp_cnt += bcn_cnt->cmp_cnt;
+		bcn_entry->bcn_stats.invalid_cnt += bcn_cnt->invalid_cnt;
+		bcn_entry->bcn_stats.srchend_cnt += bcn_cnt->srchend_cnt;
+
+		if (phl_is_mp_mode(hal_info->phl_com)) {
+			goto _unlock;
+		}
+		if (dbg_lvl == BCN_DBG_LVL_TRACE) {
+			print = true;
+		} else if (dbg_lvl == BCN_DBG_LVL_WARN) {
+			if (bcn_entry->bcn_stats.bcn_fail >=
+			    bcn_entry->bcn_stats.bcn_ok)
+				print = true;
+		} else if (dbg_lvl == BCN_DBG_LVL_ERROR) {
+			if (bcn_entry->bcn_stats.bcn_ok == 0)
+				print = true;
+		}
+		if (print) {
+			bcn_entry->bcn_stats.dbg_bcn_tx_cnt++;
+			PHL_WARN("(Band:%u, Port:%u, Mbssid:%u)OK%u Fail %u C%u E%u N%u T%u M%u O%u L%u P%u I%u S%u\n",
+				hal_com->band[bcn_cnt->band].cur_chandef.band,
+				bcn_cnt->port, bcn_cnt->mbssid,
+				bcn_cnt->ok_cnt, bcn_entry->bcn_stats.bcn_fail,
+				bcn_cnt->cca_cnt,
+				bcn_cnt->edcca_cnt, bcn_cnt->nav_cnt,
+				bcn_cnt->txon_cnt, bcn_cnt->mac_cnt,
+				bcn_cnt->others_cnt, bcn_cnt->lock_cnt,
+				bcn_cnt->cmp_cnt, bcn_cnt->invalid_cnt,
+				bcn_cnt->srchend_cnt);
+		}
+	}
+_unlock:
+	_os_spinunlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	return;
+}
+#endif /* CONFIG_RTW_DEBUG_BCN_TX */
+
+enum rtw_hal_status rtw_hal_mac_usr_frame_to_act(
+    struct hal_info_t *hal_info, struct mac_ax_usr_frame_to_act_cfg *act_cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (MACSUCCESS != mac->ops->set_hw_value(
+			      mac, MAC_AX_HW_SET_USR_FRAME_TO_ACT_CFG, act_cfg))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+void _dump_c2h_usr_tx_rpt(struct rtw_mac_usr_tx_rpt_info *c2h_rpt)
+{
+	PHL_DBG("%s, first_in(%d), first_out(%d), last_in(%d), last_out(%d).\n",
+		__FUNCTION__, c2h_rpt->freerun_cnt_first_in,
+		c2h_rpt->freerun_cnt_first_out, c2h_rpt->freerun_cnt_last_in,
+		c2h_rpt->freerun_cnt_last_out);
+	PHL_DBG("%s, tx: ppdu cnt(%d, %d), mpdu cnt(%d, %d), mpdu ok "
+		"cnt(%d), max_ampdu_num(%d).\n",
+		__FUNCTION__, c2h_rpt->tx_ppdu_cnt,
+		c2h_rpt->tx_ppdu_cnt_wo_last_pkt, c2h_rpt->tx_mpdu_cnt,
+		c2h_rpt->tx_mpdu_cnt_wo_last_pkt, c2h_rpt->tx_mpdu_ok_cnt,
+		c2h_rpt->max_ampdu_num);
+	PHL_DBG("%s, tx: rts cnt(%d), rts rty cnt(%d), max_rts_cnt(%d).\n",
+		__FUNCTION__, c2h_rpt->tx_rts_cnt, c2h_rpt->tx_rts_retry_cnt,
+		c2h_rpt->max_rts_cnt);
+	PHL_DBG(
+	    "%s, rx_clear_us(%d), busy_us(%d), ra ratio(%d), rx_err_cnt(%d).\n",
+	    __FUNCTION__, c2h_rpt->rx_clear_us, c2h_rpt->busy_us,
+	    c2h_rpt->ra_ratio, c2h_rpt->rx_err_cnt);
+}
+void
+_hal_mac_process_usr_tx_rpt(void *hal, struct rtw_mac_usr_tx_rpt_info *tx_rpt)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_custom_decrpt des = {0};
+	struct phl_module_op_info op_info = {0};
+
+	/* debug msg */
+	_dump_c2h_usr_tx_rpt(tx_rpt);
+
+	_os_mem_cpy(hal_info->phl_com->phl_priv, des.data, (u8*)tx_rpt,
+	            sizeof(struct rtw_mac_usr_tx_rpt_info));
+
+	rtw_phl_custom_prepare_set_info_evt(MSG_EVT_PROC_USR_TX_RPT,
+	                                    CUS_ID_VR,
+	                                    &des,
+	                                    &op_info,
+	                                    sizeof(struct rtw_mac_usr_tx_rpt_info));
+
+	rtw_phl_set_bk_module_info(hal_info->phl_com->phl_priv,
+	                           0,
+	                           PHL_MDL_CUSTOM,
+	                           &op_info);
+
+}
+
+enum rtw_hal_status rtw_hal_mac_set_ch_busy_stat_cfg(
+    struct hal_info_t *hal_info, struct mac_ax_ch_busy_cnt_cfg *mac_param)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (MACSUCCESS != mac->ops->set_hw_value(mac,
+						 MAC_AX_HW_SET_CH_BUSY_STAT_CFG,
+						 (void *)mac_param)) {
+		PHL_ERR(" %s, set ch busy sta cfg fail.\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_set_usr_tx_rpt_cfg(
+    struct hal_info_t *hal_info, struct rtw_phl_usr_tx_rpt_cfg *param)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_usr_tx_rpt_cfg mac_param = {0};
+
+	mac_param.mode = param->mode;
+	mac_param.rpt_start = param->rpt_start;
+	mac_param.macid = (u8)param->macid;
+	mac_param.band = param->band;
+	mac_param.port = param->port;
+	mac_param.rpt_period_us = param->rpt_period_us;
+
+	PHL_INFO(" %s, mode(%d), rpt_start(%d), macid(%d), band(%d), port(%d), "
+		 "rpt_period_us(%d)\n",
+		 __FUNCTION__, mac_param.mode, mac_param.rpt_start,
+		 mac_param.macid, mac_param.band, mac_param.port,
+		 mac_param.rpt_period_us);
+
+	if (MACSUCCESS != mac->ops->set_hw_value(mac,
+						 MAC_AX_HW_SET_USR_TX_RPT_CFG,
+						 (void *)&mac_param)) {
+		PHL_ERR(" %s, set usr tx rpt fail.\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+
+/* parameter for different c2h events */
+struct c2h_event_rpt {
+	union {
+		struct mac_ax_ccxrpt ccx_rpt;
+		struct mac_ax_bcn_cnt bcn_cnt_rpt;
+		struct mac_ax_scanofld_rsp scanofld_rpt;
+		struct mac_ax_twtanno_c2hpara twt_anno_rpt;
+		struct mac_ax_bcn_erly_rpt bcn_erly_rpt;
+		struct rtw_mac_frame_to_act_rpt f2a_rpt;
+		struct rtw_mac_usr_tx_rpt_info usr_tx_rpt;
+	} u;
+};
+
+#ifdef CONFIG_PHL_SCANOFLD
+static void _hal_mac_scanofld_event_hdl(struct hal_info_t *hal_info,
+                                        struct mac_ax_scanofld_rsp *rpt)
+{
+	struct rtw_scanofld_rsp rsp = {0};
+	u16 evt_id = 0;
+
+	rsp.notify_reason = (u8)rpt->notify_reason;
+	rsp.actual_period = (u8)rpt->actual_period;
+	rsp.pri_ch = (u8)rpt->pri_ch;
+	rsp.status = (u8)rpt->status;
+	rsp.band = (u8)rpt->ch_band;
+	rsp.hw_band = (u8)rpt->band;
+	PHL_INFO("%s: notify_reason %u \n", __func__, rpt->notify_reason);
+
+	switch (rpt->notify_reason) {
+	case MAC_AX_SCAN_ENTER_CH_NOTIFY:
+		evt_id = MSG_EVT_SCANOFLD_SWITCH;
+		break;
+	case MAC_AX_SCAN_END_SCAN_NOTIFY:
+		evt_id = MSG_EVT_SCANOFLD_END;
+		break;
+	case MAC_AX_SCAN_CHKPT_TIMER:
+		evt_id = MSG_EVT_SCANOFLD_CHKPT_TIMER;
+		break;
+	default:
+		return;
+	}
+
+	rtw_phl_cmd_scan_send_msg(hal_info->phl_com->phl_priv, evt_id,
+				  rsp.hw_band, (u8 *)&rsp,
+				  sizeof(struct rtw_scanofld_rsp));
+}
+#endif
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+void
+_hal_mac_process_frame_to_act_rpt(void *hal, struct rtw_mac_frame_to_act_rpt *f2a_rpt)
+{
+
+	PHL_INFO("%s, mode=%d\n", __func__, f2a_rpt->mode);
+	PHL_INFO("%s, trigger_cnt=%d\n", __func__, f2a_rpt->trigger_cnt);
+	PHL_INFO("%s, sw_def=%d\n", __func__, f2a_rpt->sw_def);
+	PHL_INFO("%s, to_thr=%d\n", __func__, f2a_rpt->to_thr);
+	PHL_INFO("%s, cont_fail_cnt=%d\n", __func__, f2a_rpt->cont_fail_cnt);
+	PHL_INFO("%s, frame_fail_flg=%d\n", __func__, f2a_rpt->frame_fail_flg);
+	PHL_INFO("%s, start_pkt_time_us=%u\n", __func__, f2a_rpt->start_pkt_time_us);
+	PHL_INFO("%s, pkt_in=%u\n", __func__, f2a_rpt->pkt_in);
+	PHL_INFO("%s, queue_time=%u\n", __func__, f2a_rpt->queue_time);
+	PHL_INFO("%s, duration=%u\n", __func__, f2a_rpt->duration);
+	PHL_INFO("%s, c2h_enqueue_fail_cnt=%d\n", __func__, f2a_rpt->c2h_enqueue_fail_cnt);
+	PHL_INFO("%s, fw_action=%d\n", __func__, f2a_rpt->fw_action);
+
+
+}
+#endif /* CONFIG_PHL_CUSTOM_FEATURE */
+
+
+u32 rtw_hal_mac_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct c2h_event_rpt c2h_rpt = {0};
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+#ifdef RTW_WKARD_CCX_RPT_LIMIT_CTRL
+	u32 retry_limit = 32;
+#endif
+	u32 retid = 0;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &c2h_rpt, 0, sizeof(c2h_rpt));
+	hal_status = mac->ops->get_c2h_event(mac, c2h, (enum phl_msg_evt_id *)&retid, (u8 *)&c2h_rpt);
+	/*PHL_INFO("%s, hal_status=%d, retid=%d\n", __func__, hal_status, retid);*/
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		goto exit;
+
+
+	switch (retid) {
+
+	case MSG_EVT_CCX_REPORT_TX_OK:
+	case MSG_EVT_CCX_REPORT_TX_FAIL:
+		#if 0
+		PHL_INFO("%s, c2h_rpt.u.ccx_rpt.tx_state=%d\n", __func__, c2h_rpt.u.ccx_rpt.tx_state);
+		PHL_INFO("%s, c2h_rpt.u.ccx_rpt.sw_define=%d\n", __func__, c2h_rpt.u.ccx_rpt.sw_define);
+		PHL_INFO("%s, c2h_rpt.u.ccx_rpt.macid=%d\n", __func__,  c2h_rpt.u.ccx_rpt.macid);
+		PHL_INFO("%s, c2h_rpt.u.ccx_rpt.pkt_ok_num=%d\n", __func__, c2h_rpt.u.ccx_rpt.pkt_ok_num);
+		PHL_INFO("%s, c2h_rpt.u.ccx_rpt.data_txcnt=%d\n", __func__, c2h_rpt.u.ccx_rpt.data_txcnt);
+		#endif
+		#ifdef RTW_WKARD_CCX_RPT_LIMIT_CTRL
+		if (hal_info->hal_com->spe_pkt_cnt_lmt)
+			retry_limit = hal_info->hal_com->spe_pkt_cnt_lmt;
+
+		if (retid == MSG_EVT_CCX_REPORT_TX_FAIL &&
+			c2h_rpt.u.ccx_rpt.data_txcnt != retry_limit)
+			retid = 0;
+		#endif
+		c2h_msg->u.tx_rpt.macid = (u16)c2h_rpt.u.ccx_rpt.macid;
+		c2h_msg->u.tx_rpt.seq_no = (u8)c2h_rpt.u.ccx_rpt.sw_define;
+		break;
+	case MSG_EVT_SCANOFLD:
+		#ifdef CONFIG_PHL_SCANOFLD
+		_hal_mac_scanofld_event_hdl(hal_info,
+					    &(c2h_rpt.u.scanofld_rpt));
+		#endif
+		retid = 0;
+		break;
+	case MSG_EVT_BCN_CNT_RPT:
+		#ifdef CONFIG_RTW_DEBUG_BCN_TX
+		hal_mac_process_bcn_stats(hal, &c2h_rpt.u.bcn_cnt_rpt);
+		#endif /* CONFIG_RTW_DEBUG_BCN_TX */
+		break;
+#ifdef CONFIG_PHL_TWT
+	case MSG_EVT_TWT_WAIT_ANNOUNCE:
+		c2h_msg->u.twt_anno_rpt.wait_case = c2h_rpt.u.twt_anno_rpt.wait_case;
+		c2h_msg->u.twt_anno_rpt.macid[0] = (u8)c2h_rpt.u.twt_anno_rpt.macid0;
+		c2h_msg->u.twt_anno_rpt.macid[1] = (u8)c2h_rpt.u.twt_anno_rpt.macid1;
+		c2h_msg->u.twt_anno_rpt.macid[2] = (u8)c2h_rpt.u.twt_anno_rpt.macid2;
+		break;
+#endif
+#ifdef CONFIG_PHL_BCN_ERLY_RPT
+	case MSG_EVT_BCN_EARLY_REPORT:
+		c2h_msg->u.bcn_erly_rpt.port = c2h_rpt.u.bcn_erly_rpt.port;
+		c2h_msg->u.bcn_erly_rpt.band = c2h_rpt.u.bcn_erly_rpt.band;
+		break;
+#endif
+	case MSG_EVT_USR_FRAME_ACT_RPT:
+		#ifdef CONFIG_PHL_CUSTOM_FEATURE
+		_hal_mac_process_frame_to_act_rpt(hal, &c2h_rpt.u.f2a_rpt);
+		#endif
+		break;
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+	case MSG_EVT_USR_TX_RPT:
+		_hal_mac_process_usr_tx_rpt(hal, &c2h_rpt.u.usr_tx_rpt);
+		break;
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+	}
+exit:
+	return retid;
+}
+
+void rtw_hal_mac_upd_c2h(void *hal, struct rtw_c2h_info *c2h, u32 evt_id, struct c2h_evt_msg *c2h_msg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	void *drv = hal_to_drvpriv(hal_info);
+	u8 *buf = NULL;
+	u16 len = 0;
+
+	switch (evt_id)
+	{
+#ifdef CONFIG_PHL_DFS_SWITCH_CH_WITH_CSA
+	case MSG_EVT_CSA_COUNTDOWN_ZERO:
+		{
+			struct rtw_csa_cntdown_rpt *csa_rpt = NULL;
+
+			if (c2h->content_len == 1) {
+				len = sizeof(struct rtw_csa_cntdown_rpt);
+				csa_rpt = _os_kmem_alloc(drv, len);
+				if (csa_rpt == NULL)
+					break;
+
+				csa_rpt->port = c2h->content[0] & 0x7f;
+				csa_rpt->band = (c2h->content[0] >> 7) & 0x1;
+
+				//PHL_PRINT("content: %x, port: %u, band: %u\n", c2h->content[0], csa_rpt->port, csa_rpt->band);
+
+				buf = (u8 *)csa_rpt;
+			} else {
+				PHL_ERR("[%s %d] invalid c2h content_len: %u, evt_id: %u\n", __func__, __LINE__, c2h->content_len, evt_id);
+			}
+		}
+		break;
+#endif
+	case MSG_EVT_CCX_REPORT_TX_OK:
+	case MSG_EVT_CCX_REPORT_TX_FAIL:
+		{
+			struct rtw_tx_pkt_rpt *tx_rpt = NULL;
+
+			len = sizeof(struct rtw_tx_pkt_rpt);
+			tx_rpt = _os_kmem_alloc(drv, len);
+			if (tx_rpt == NULL)
+				break;
+
+			tx_rpt->macid = c2h_msg->u.tx_rpt.macid;
+			tx_rpt->seq_no = c2h_msg->u.tx_rpt.seq_no;
+
+			buf = (u8 *)tx_rpt;
+		}
+		break;
+#ifdef CONFIG_PHL_TWT
+	case MSG_EVT_TWT_WAIT_ANNOUNCE:
+		{
+			struct rtw_phl_twt_wait_anno_rpt *twt_anno_rpt = NULL;
+			u8 i;
+			len = sizeof(struct rtw_phl_twt_wait_anno_rpt);
+			twt_anno_rpt = _os_kmem_alloc(drv, len);
+			if (twt_anno_rpt == NULL)
+				break;
+			twt_anno_rpt->wait_case = c2h_msg->u.twt_anno_rpt.wait_case;
+			for (i = 0; i < RTW_PHL_TWT_WAIT_ANNO_STA_NUM; i++)
+				twt_anno_rpt->macid[i] = c2h_msg->u.twt_anno_rpt.macid[i];
+
+			buf = (u8 *)twt_anno_rpt;
+		}
+		break;
+#endif
+#ifdef CONFIG_PHL_BCN_ERLY_RPT
+	case MSG_EVT_BCN_EARLY_REPORT:
+		{
+			struct rtw_bcn_early_rpt *bcn_early_rpt = NULL;
+
+			len = sizeof(struct rtw_bcn_early_rpt);
+			bcn_early_rpt = _os_kmem_alloc(drv, len);
+			if (bcn_early_rpt == NULL)
+				break;
+
+			bcn_early_rpt->port = c2h_msg->u.bcn_erly_rpt.port;
+			bcn_early_rpt->band = c2h_msg->u.bcn_erly_rpt.band;
+			buf = (u8 *)bcn_early_rpt;
+		}
+		break;
+#endif
+	}
+
+	if (buf) {
+		c2h->content = buf;
+	} else if (len > 0) {
+		PHL_ERR("[%s %d] alloc buf fail, evt_id: %u\n", __func__, __LINE__, evt_id);
+		len = 0;
+	}
+
+	c2h->content_len = len;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_pwr_ofst_mode(struct rtw_hal_com_t *hal_com, u8 band, s8 pwr_ofst)
+{
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+	int i;
+
+	for (i=0; i<TPU_SIZE_MODE; i++)
+		tpu->pwr_ofst_mode[i] = pwr_ofst;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_ofst_mode(struct rtw_hal_com_t *hal_com, u8 band)
+{
+
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	mac->ops->write_pwr_ofst_mode(mac, band, tpu);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_pwr_ofst_bw(struct rtw_hal_com_t *hal_com, u8 band, s8 pwr_ofst)
+{
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+	int i;
+
+	for (i=0; i<TPU_SIZE_BW; i++)
+		tpu->pwr_ofst_bw[i] = pwr_ofst;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_ofst_bw(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	mac->ops->write_pwr_ofst_bw(mac, band, tpu);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_ref_reg(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	mac->ops->write_pwr_ref_reg(mac, band, tpu);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_en(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	mac->ops->write_pwr_limit_en(mac, band, tpu);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_pwr_lmt_en_val(struct rtw_hal_com_t *hal_com, u8 band, bool en_val)
+{
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	tpu->pwr_lmt_en = en_val;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool
+rtw_hal_mac_get_pwr_lmt_en_val(struct rtw_hal_com_t *hal_com, u8 band)
+{
+
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	return tpu->pwr_lmt_en;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_tpu_mode(struct rtw_hal_com_t *hal_com,
+			 enum rtw_tpu_op_mode op_mode_new, u8 band)
+{
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	if (op_mode_new == TPU_DBG_MODE) {
+		tpu->op_mode = TPU_DBG_MODE;
+		tpu->normal_mode_lock_en = true;
+	} else {
+		tpu->op_mode = TPU_NORMAL_MODE;
+		tpu->normal_mode_lock_en = false;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_rua_reg(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	mac->ops->write_pwr_limit_rua_reg(mac, band, tpu);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_limit_reg(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+	struct rtw_tpu_pwr_imt_info *lmt = &tpu->rtw_tpu_pwr_imt_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	mac->ops->write_pwr_limit_reg(mac, band, lmt);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_pwr_by_rate_reg(struct rtw_hal_com_t *hal_com, u8 band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct rtw_tpu_info *tpu = &hal_com->band[band].rtw_tpu_i;
+	struct rtw_tpu_pwr_by_rate_info *by_rate = &tpu->rtw_tpu_pwr_by_rate_i;
+
+	if (tpu->normal_mode_lock_en)
+		return RTW_HAL_STATUS_FAILURE;
+
+	mac->ops->write_pwr_by_rate_reg(mac, band, by_rate);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_log_efuse_bt_size(struct rtw_hal_com_t *hal_com, u32 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->get_hw_value(mac,
+		MAC_AX_HW_GET_BT_LOGICAL_EFUSE_SIZE, val) != MACSUCCESS){
+		PHL_ERR("%s: Get bt efuse size fail!\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	PHL_INFO("%s: BT Efuse log size = %d!\n", __FUNCTION__, *val);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_bt_mask_size(struct rtw_hal_com_t *hal_com, u32 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if(mac->ops->get_hw_value(mac,
+	MAC_AX_HW_GET_BT_EFUSE_MASK_SIZE, val) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("%s: bt efuse mask size = %d\n", __FUNCTION__, *val);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_log_efuse_bt_map(struct rtw_hal_com_t *hal_com,
+							u8 *map,
+							u32 map_size,
+							u8 *mask,
+							u32 mask_size)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_pg_efuse_info info = {0};
+	enum rtw_hal_status status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+	u8 *tmp_map = NULL;
+	u8 *tmp_mask = NULL;
+
+	tmp_map = _os_mem_alloc(hal_com->drv_priv, map_size);
+	if(tmp_map == NULL) {
+		PHL_WARN("%s: Allocate pg map buffer fail!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_RESOURCE;
+		goto err_mem_tmp_map;
+	}
+
+	tmp_mask = _os_mem_alloc(hal_com->drv_priv, mask_size);
+	if(tmp_mask == NULL) {
+		PHL_WARN("%s: Allocate pg mask buffer fail!\n", __FUNCTION__);
+		status = RTW_HAL_STATUS_RESOURCE;
+		goto err_mem_tmp_mask;
+	}
+
+	/* Copy efuse map to tmp_map buffer */
+	_os_mem_cpy(hal_com->drv_priv, tmp_map, map, map_size);
+
+	/* Copy efuse mask to tmp_mask buffer */
+	_os_mem_cpy(hal_com->drv_priv, tmp_mask, mask, mask_size);
+
+
+	info.efuse_map = tmp_map;
+	info.efuse_map_size = map_size;
+	info.efuse_mask = tmp_mask;
+	info.efuse_mask_size= mask_size;
+
+	if (mac->hw_info->chip_id == MAC_AX_CHIP_ID_8852BT) {
+		if (mac->ops->pg_efuse_by_block_bt(mac,
+						&info,
+						MAC_AX_EFUSE_R_DRV,
+						0x0)) {
+			PHL_INFO("%s: BT PG Fail!\n", __FUNCTION__);
+			status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+		} else {
+			PHL_INFO("%s: BT PG ok!\n", __FUNCTION__);
+			status = RTW_HAL_STATUS_SUCCESS;
+		}
+
+	} else {
+		if (mac->ops->pg_efuse_by_map_bt(mac,
+						&info,
+						MAC_AX_EFUSE_R_DRV) != MACSUCCESS) {
+			PHL_INFO("%s: BT PG Fail!\n", __FUNCTION__);
+			status = RTW_HAL_STATUS_EFUSE_PG_FAIL;
+		}
+		else {
+			PHL_INFO("%s: BT PG ok!\n", __FUNCTION__);
+			status = RTW_HAL_STATUS_SUCCESS;
+		}
+	}
+	_os_mem_free(hal_com->drv_priv, tmp_map, map_size);
+	_os_mem_free(hal_com->drv_priv, tmp_mask, mask_size);
+
+	return status;
+
+err_mem_tmp_mask:
+	_os_mem_free(hal_com->drv_priv, tmp_mask, mask_size);
+
+err_mem_tmp_map:
+	_os_mem_free(hal_com->drv_priv, tmp_map, map_size);
+
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_read_log_efuse_bt_map(struct rtw_hal_com_t *hal_com, u8 *map, u32 size)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (hal_com->chip_id == CHIP_WIFI6_8852BT) {
+		if (mac->ops->dump_log_block_bt(mac,
+						MAC_AX_EFUSE_PARSER_MAP,
+						MAC_AX_EFUSE_R_DRV,
+						map,
+						0x0,
+						size)!= MACSUCCESS) {
+			PHL_INFO("%s: Dump bt logical efuse fail!\n", __FUNCTION__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+	} else {
+		if (mac->ops->dump_log_efuse_bt(mac,
+				MAC_AX_EFUSE_PARSER_MAP,
+				MAC_AX_EFUSE_R_DRV,
+				map
+				) != MACSUCCESS) {
+			PHL_INFO("%s: Dump bt logical efuse fail!\n", __FUNCTION__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+	}
+
+	PHL_INFO("%s: Dump bt logical efuse ok!\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_bt_avl(struct rtw_hal_com_t *hal_com, u32 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->get_efuse_avl_size_bt(mac, val) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_efuse_bt_size(struct rtw_hal_com_t *hal_com, u32 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->get_hw_value(mac,
+		MAC_AX_HW_GET_BT_EFUSE_SIZE, val) != MACSUCCESS){
+		PHL_ERR("%s: Get efuse bt size fail!\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	PHL_INFO("%s: Efuse size = %d!\n", __FUNCTION__, *val);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_mu_edca(struct rtw_hal_com_t *hal_com, u8 band, u8 ac,
+	u16 timer, u8 cw_min, u8 cw_max, u8 aifs)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_muedca_param edca = {0};
+	u32 err = 0;
+
+	edca.band = band;
+	edca.ac = ac;
+	edca.aifs_us = aifs;
+	edca.ecw_min = cw_min;
+	edca.ecw_max = cw_max;
+	edca.muedca_timer_32us = timer;
+	err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_MUEDCA_PARAM, &edca);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_mu_edca_ctrl(struct rtw_hal_com_t *hal_com,
+	u8 band, u8 wmm, u8 set)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_muedca_cfg cfg = {0};
+	u32 err = 0;
+
+	cfg.band = band;
+	cfg.wmm_sel = wmm;
+	cfg.countdown_en = set;
+	cfg.tb_update_en = set;
+	err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_MUEDCA_CTRL, &cfg);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_led_set_ctrl_mode(struct hal_info_t *hal_info,
+						  enum mac_ax_led_mode mode,
+						  u8 led_id)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->set_led_mode(mac, mode, led_id) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_led_ctrl(struct hal_info_t *hal_info, u8 high,
+					 u8 led_id)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->led_ctrl(mac, high, led_id) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_sw_gpio_ctrl(struct hal_info_t *hal_info, u8 high,
+					 u8 gpio)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->sw_gpio_ctrl(mac, high, gpio) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_set_sw_gpio_mode(struct hal_info_t *hal_info, enum rtw_gpio_mode mode,
+					 u8 gpio)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->set_sw_gpio_mode(mac, mode, gpio) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_get_wl_dis_val(struct hal_info_t *hal_info, u8 *val)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->get_wl_dis_val(mac, val) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_pcie_trx_mit(struct hal_info_t *hal_info,
+			 struct mac_ax_pcie_trx_mitigation *mit_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (MACSUCCESS !=
+	    mac->ops->set_hw_value(mac, MAX_AX_HW_PCIE_MIT, mit_info))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_wcpu_cap(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	void *drv = hal_to_drvpriv(hal_info);
+	struct rtw_wcpu_cap_t *wcpu_cap = NULL;
+
+	if (mac->ops->get_wlanfw_cap(mac, &hal_info->hal_com->dev_hw_cap.wcpu_cap) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	_os_mem_cpy(drv, &phl_com->dev_cap.wcpu_cap,
+		    &hal_info->hal_com->dev_hw_cap.wcpu_cap,
+		    sizeof(struct rtw_wcpu_cap_t));
+
+	wcpu_cap = &hal_info->hal_com->dev_hw_cap.wcpu_cap;
+	PHL_INFO("FW Capability\n");
+	PHL_INFO("   io_offload: %d,\n", wcpu_cap->mac_ofld_cap.io_offload);
+	PHL_INFO("   chsw_offload: %d,\n", wcpu_cap->mac_ofld_cap.chsw_offload);
+	PHL_INFO("   lps_pg: %d,\n", wcpu_cap->mac_ofld_cap.lps_pg);
+	PHL_INFO("   twt_sta: %d,\n", wcpu_cap->mac_ofld_cap.twt_sta);
+	PHL_INFO("   nan: %d,\n", wcpu_cap->mac_ofld_cap.nan);
+	PHL_INFO("   ftm: %d,\n", wcpu_cap->mac_ofld_cap.ftm);
+	PHL_INFO("   scan_offload: %d,\n", wcpu_cap->mac_ofld_cap.scan_offload);
+	PHL_INFO("   sensing_csi: %d,\n", wcpu_cap->mac_ofld_cap.sensing_csi);
+	PHL_INFO("   [bb]: 0x%08x,\n", wcpu_cap->bb_ofld_cap[0]);
+	PHL_INFO("   [rf]: 0x%08x,\n", wcpu_cap->rf_ofld_cap[0]);
+	PHL_INFO("   [btc]: 0x%08x\n", wcpu_cap->btc_ofld_cap[0]);
+
+	#if 0
+	wcpu_cap = &phl_com->dev_cap.wcpu_cap;
+	PHL_INFO("FW Capability by phl_com->dev_cap.\n");
+	PHL_INFO("   io_offload: %d,\n", wcpu_cap->mac_ofld_cap.io_offload);
+	PHL_INFO("   chsw_offload: %d,\n", wcpu_cap->mac_ofld_cap.chsw_offload);
+	PHL_INFO("   lps_pg: %d,\n", wcpu_cap->mac_ofld_cap.lps_pg);
+	PHL_INFO("   twt_sta: %d,\n", wcpu_cap->mac_ofld_cap.twt_sta);
+	PHL_INFO("   nan: %d,\n", wcpu_cap->mac_ofld_cap.nan);
+	PHL_INFO("   ftm: %d,\n", wcpu_cap->mac_ofld_cap.ftm);
+	PHL_INFO("   scan_offload: %d,\n", wcpu_cap->mac_ofld_cap.scan_offload);
+	PHL_INFO("   sensing_csi: %d\n", wcpu_cap->mac_ofld_cap.sensing_csi);
+	PHL_INFO("   [bb]: 0x%08x,\n", wcpu_cap->bb_ofld_cap[0]);
+	PHL_INFO("   [rf]: 0x%08x,\n", wcpu_cap->rf_ofld_cap[0]);
+	PHL_INFO("   [btc]: 0x%08x\n", wcpu_cap->btc_ofld_cap[0]);
+	#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_read_efuse_bt_hidden(struct rtw_hal_com_t *hal_com, u32 addr, u32 size, u8 *val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (mac->ops->read_efuse(mac, addr, size, val, MAC_AX_EFUSE_BANK_BT) != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_write_efuse_bt_hidden(struct rtw_hal_com_t *hal_com, u32 addr, u8 val)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	u8 tmp_value;
+
+	if (mac->ops->read_efuse(mac, addr, 1, &tmp_value, MAC_AX_EFUSE_BANK_BT) != MACSUCCESS) {
+		PHL_INFO("%s read bt efuse hideen block fail.\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if(tmp_value != 0xFF) {
+		PHL_INFO("%s bt efuse hidden offset = 0x%x has value = 0x%x.\n", __FUNCTION__, addr, tmp_value);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (mac->ops->write_efuse(mac, addr, val, MAC_AX_EFUSE_BANK_BT) != MACSUCCESS) {
+		PHL_INFO("%s write bt efuse hideen block fail.\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_tsf_sync(struct hal_info_t *hal_info,
+						u8 from_port, u8 to_port, enum phl_band_idx band,
+						s32 sync_offset_tu, enum hal_tsf_sync_act action)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum mac_ax_tsf_sync_act mac_action = MAC_AX_TSF_SYNC_NOW_ONCE;
+	s32 sync_offset_unit = 0;/* for halmac API use, unit is 32us  */
+
+	switch (action){
+	case HAL_TSF_SYNC_NOW_ONCE :
+		mac_action = MAC_AX_TSF_SYNC_NOW_ONCE;
+		break;
+	case HAL_TSF_EN_SYNC_AUTO :
+		mac_action = MAC_AX_TSF_EN_SYNC_AUTO;
+		break;
+	case HAL_TSF_DIS_SYNC_AUTO :
+		mac_action = MAC_AX_TSF_DIS_SYNC_AUTO;
+		break;
+	default :
+		PHL_ERR("Unknown tsf sync action %d\n", action);
+		goto _error;
+	}
+
+	/* covert TU to unit(unit is 32us, 1TU=1024us=32*32us) */
+	sync_offset_unit = sync_offset_tu * 32;
+
+	if (mac->ops->tsf_sync(mac, from_port, to_port,
+		sync_offset_unit, mac_action) == MACSUCCESS)
+		return RTW_HAL_STATUS_SUCCESS;
+_error:
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+/**
+ * rtw_hal_mac_get_sec_cam() - get the security cam raw data from HW
+ * @hal_info:		struct hal_info_t *
+ * @num:		How many cam you wnat to dump from the first one.
+ * @buf:		ptr to buffer which store the content from HW.
+ *			If buf is NULL, use console as debug path.
+ * @size		Size of allocated memroy for @buf.
+ *			The size should be @num * size of security cam offset(0x20).
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when function successfully works,
+ * otherwise, return RTW_HAL_STATUS_FAILURE.
+ */
+enum rtw_hal_status
+rtw_hal_mac_get_sec_cam(struct hal_info_t *hal_info, u16 num, u8 *buf, u16 size)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	int i = 0;
+	/* ToDO: fix the magic number later */
+	u8 sec_cam_offset_sz = 0x20;
+
+	if (buf == NULL) {
+		/* to console */
+		for (i = 0; i < num; i++) {
+			PHL_INFO("======= SEC CAM (%d)DUMP  =======\n", i);
+			mac_mem_dump(mac, MAC_AX_MEM_SECURITY_CAM, i*sec_cam_offset_sz\
+				, NULL, sec_cam_offset_sz, 0);
+			PHL_INFO("\n");
+		}
+	} else {
+		/* to buffer */
+		if (size < sec_cam_offset_sz*num) {
+			PHL_ERR("%s buf size is not enough to dump security cam\n", __func__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+
+		for (i = 0; i < num; i++)
+			mac_mem_dump(mac, MAC_AX_MEM_SECURITY_CAM, i*sec_cam_offset_sz\
+				, buf + (i*sec_cam_offset_sz), sec_cam_offset_sz, 1);
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
+
+/**
+ * rtw_hal_mac_get_addr_cam() - get the address cam raw data from HW
+ * @hal_info:		struct hal_info_t *
+ * @num:		How many cam you wnat to dump from the first one.
+ * @buf:		ptr to buffer which store the content from HW.
+ *			If buf is NULL, use console as debug path.
+ * @size		Size of allocated memroy for @buf.
+ *			The size should be @num * size of Addr cam offset(0x40).
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when function successfully works,
+ * otherwise, return RTW_HAL_STATUS_FAILURE.
+ */
+enum rtw_hal_status
+rtw_hal_mac_get_addr_cam(struct hal_info_t *hal_info, u16 num, u8 *buf, u16 size)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	int i = 0;
+	/* ToDO: fix the magic number later */
+	u8 addr_cam_offset_sz = 0x40;
+
+	if (buf == NULL) {
+		/* to console */
+		for (i = 0; i < num; i++) {
+			PHL_INFO("======= ADDR CAM (%d)DUMP  =======\n", i);
+			mac_mem_dump(mac, MAC_AX_MEM_ADDR_CAM, i*addr_cam_offset_sz\
+				, NULL, addr_cam_offset_sz, 0);
+			PHL_INFO("\n");
+		}
+	} else {
+		/* to buffer */
+		if (size < addr_cam_offset_sz*num) {
+			PHL_ERR("%s buf size is not enough to dump addr cam\n", __func__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+
+		for (i = 0; i < num; i++)
+			mac_mem_dump(mac, MAC_AX_MEM_ADDR_CAM, i*addr_cam_offset_sz\
+				, buf + (i*addr_cam_offset_sz), addr_cam_offset_sz, 1);
+
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
+
+enum rtw_hal_status rtw_hal_mac_get_tsf(struct hal_info_t *hal,
+		enum phl_band_idx band, u8 port, u32 *tsf_h, u32 *tsf_l)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct mac_ax_port_tsf val = {0};
+
+	val.band = band;
+	val.port = port;
+	if (hal_mac_ops->get_hw_value(mac, MAC_AX_HW_GET_TSF, &val)) {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s: get failed\n",
+			  __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	*tsf_h = val.tsf_h;
+	*tsf_l = val.tsf_l;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * rtw_hal_mac_clear_rwptr - clear read/write pointer of HCI
+ * @hal:	pointer of struct hal_info_t
+ *
+ * Clear read/write pointer of HCI.
+ * For PCIE, the clr_hci_trx() would also reset bdram.
+ *
+ * Return RTW_HAL_STATUS_SUCCESS if clear read write pointer is ok,
+ * otherwise return RTW_HAL_STATUS_FAILURE.
+ */
+enum rtw_hal_status rtw_hal_mac_clear_rwptr(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	u32 ret = 0;
+
+	ret = hal_mac_ops->intf_ops->clr_hci_trx(mac);
+	if (ret != MACSUCCESS) {
+		PHL_ERR("%s: clear read/write pointer of HCI fail! (%d)\n", __func__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_cfg_txhci(struct hal_info_t *hal, u8 en)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	enum mac_ax_func_sw mac_en;
+	u32 ret = 0;
+
+	if (en)
+		mac_en = MAC_AX_FUNC_EN;
+	else
+		mac_en = MAC_AX_FUNC_DIS;
+
+	ret = hal_mac_ops->intf_ops->ctrl_txhci(mac, mac_en);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS):(RTW_HAL_STATUS_FAILURE);
+}
+
+enum rtw_hal_status rtw_hal_mac_cfg_rxhci(struct hal_info_t *hal, u8 en)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	enum mac_ax_func_sw mac_en;
+	u32 ret = 0;
+
+	if (en)
+		mac_en = MAC_AX_FUNC_EN;
+	else
+		mac_en = MAC_AX_FUNC_DIS;
+
+	ret = hal_mac_ops->intf_ops->ctrl_rxhci(mac, mac_en);
+
+	return (ret == MACSUCCESS) ?
+			(RTW_HAL_STATUS_SUCCESS):(RTW_HAL_STATUS_FAILURE);
+}
+
+#ifdef CONFIG_MCC_SUPPORT
+void _hal_mac_mcc_fill_role_info(struct rtw_phl_mcc_role *mcc_role,
+				struct mac_ax_mcc_role *info)
+{
+	struct rtw_phl_mcc_policy_info *policy = &mcc_role->policy;
+
+	if (mcc_role->bt_role) {
+		info->group = mcc_role->group;
+		info->btc_in_2g = true;
+		info->duration = policy->dur_info.dur;
+	} else {
+		info->macid = mcc_role->macid;
+		info->central_ch_seg0 = mcc_role->chandef->center_ch;
+		info->central_ch_seg1 = (u8)mcc_role->chandef->center_freq2;
+		info->primary_ch = mcc_role->chandef->chan;
+		info->bandwidth = mcc_role->chandef->bw;
+		info->ch_band_type = mcc_role->chandef->band;
+		info->group = mcc_role->group;
+		info->c2h_rpt = policy->c2h_rpt;
+		info->dis_tx_null = policy->dis_tx_null;
+		info->dis_sw_retry = policy->dis_sw_retry;
+		info->in_curr_ch = policy->in_curr_ch;
+		info->sw_retry_count = policy->sw_retry_count;
+		info->tx_null_early= policy->tx_null_early;
+		info->duration = policy->dur_info.dur;
+		info->courtesy_en = policy->courtesy_en;
+		info->courtesy_num = policy->courtesy_num;
+		info->courtesy_target = policy->courtesy_target;
+	}
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_role_info(): macid(%d), central_ch_seg0(%d), central_ch_seg1(%d)\n",
+		info->macid, info->central_ch_seg0, info->central_ch_seg1);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_role_info(): primary_ch(%d), bandwidth(%d), group(%d), c2h_rpt(%d)\n",
+		info->primary_ch, info->bandwidth, info->group, info->c2h_rpt);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_role_info(): dis_tx_null(%d), dis_sw_retry(%d), in_curr_ch(%d)\n",
+		info->dis_tx_null, info->dis_sw_retry, info->in_curr_ch);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_role_info(): sw_retry_count(%d), tx_null_early(%d), btc_in_2g(%d)\n",
+		info->sw_retry_count, info->tx_null_early, info->btc_in_2g);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_role_info(): duration(%d)\n",
+		info->duration);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_role_info(): courtesy_en(%d), courtesy_num(%d), courtesy_target(0x%x)\n",
+		info->courtesy_en, info->courtesy_num, info->courtesy_target);
+}
+
+void _hal_mac_mcc_fill_duration_info(struct rtw_phl_mcc_en_info *en_info,
+					struct rtw_phl_mcc_bt_info *bt_info,
+					struct mac_ax_mcc_duration_info *info)
+{
+	info->group = en_info->mcc_role[0].group;
+	info->btc_in_group = (bt_info->bt_dur > 0) ? 1 : 0;
+	info->start_macid = en_info->mcc_role[en_info->ref_role_idx].macid;
+	info->macid_x = en_info->mcc_role[0].macid;
+	info->macid_y = en_info->mcc_role[1].macid;
+	info->duration_x = en_info->mcc_role[0].policy.dur_info.dur;;
+	info->duration_y = en_info->mcc_role[1].policy.dur_info.dur;;
+	info->start_tsf_low = en_info->tsf_low;
+	info->start_tsf_high = en_info->tsf_high;
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_duration_info(): info->group(%d), btc_in_group(%d), start_macid(%d), macid_x(%d), macid_y(%d)\n",
+		info->group, info->btc_in_group, info->start_macid,
+		info->macid_x, info->macid_y);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_duration_info(): duration_x(%d), duration_y(%d), start_tsf(0x%08x %08x)\n",
+		info->duration_x, info->duration_y, info->start_tsf_high,
+		info->start_tsf_low);
+}
+
+void _hal_mac_mcc_fill_start_info(u8 group, u8 macid, u32 tsf_high, u32 tsf_low,
+			u8 btc_in_group, u8 old_group_action, u8 old_group,
+			struct mac_ax_mcc_start *info)
+{
+	info->group = group;
+	info->btc_in_group = btc_in_group;
+	info->old_group_action = old_group_action;
+	info->old_group = old_group;
+	info->macid = macid;
+	info->tsf_low = tsf_low;
+	info->tsf_high = tsf_high;
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_hal_mac_mcc_fill_start_info(): info->group(%d), btc_in_group(%d), old_group_action(%d), old_group(%d), macid(%d), Tsf(0x%08x %08x)\n",
+		info->group, info->btc_in_group, info->old_group_action,
+		info->old_group, info->macid, info->tsf_high, info->tsf_low);
+}
+
+enum rtw_hal_status rtw_hal_mac_add_mcc(struct hal_info_t *hal,
+					struct rtw_phl_mcc_role *mcc_role)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+	u16 loop_cnt = 0;
+	struct mac_ax_mcc_role info = {0};
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_add_mcc()\n");
+	if (mac == NULL)
+		goto exit;
+	_hal_mac_mcc_fill_role_info(mcc_role, &info);
+	mac_status = mac->ops->add_mcc(mac, &info);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_add_mcc(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	do {
+		if (mac->ops->check_add_mcc_done(mac, (u8)info.group) ==
+								MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mac_add_mcc(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_add_mcc(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_start_mcc(struct hal_info_t *hal,
+				u8 group, u8 macid, u32 tsf_high, u32 tsf_low,
+				u8 btc_in_group, u8 old_group_action, u8 old_group)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_mcc_start info = {0};
+	u32 mac_status = MACSUCCESS;
+	u16 loop_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_start_mcc(): group(%d), macid(%d) Start tsf(0x%08X %08X), btc_in_group(%d)\n",
+		group, macid, tsf_high, tsf_low, btc_in_group);
+	if (mac == NULL)
+		goto exit;
+	_hal_mac_mcc_fill_start_info(group, macid, tsf_high, tsf_low,
+					btc_in_group, old_group_action,
+					old_group, &info);
+	mac_status = mac->ops->start_mcc(mac, &info);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_start_mcc(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	do {
+		if (mac->ops->check_start_mcc_done(mac, group) ==
+								MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mac_start_mcc(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_start_mcc(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_stop_mcc(struct hal_info_t *hal, u8 group,
+					u8 macid)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+	u16 loop_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_stop_mcc(): group(%d), macid(%d)\n",
+		group, macid);
+	if (mac == NULL)
+		goto exit;
+	/*prev_groups always set to 1, driver stop all group pattern in the same hw band.*/
+	mac_status = mac->ops->stop_mcc(mac, group, macid, 1);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_stop_mcc(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	do {
+		if (mac->ops->check_stop_mcc_done(mac, group) ==
+								MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mac_stop_mcc(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_stop_mcc(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_reset_mcc_group(struct hal_info_t *hal, u8 group)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> %s(): group(%d)\n",
+		__func__, group);
+	if (mac == NULL)
+		goto exit;
+
+	mac_status = mac->ops->reset_mcc_group(mac, group);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_,
+			"%s(): reset group(%d) fail, status = %d.\n",
+			__func__, group, mac_status);
+		goto exit;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_,
+			"%s(): reset group(%d) ok.\n",
+			__func__, group);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_del_mcc_group(struct hal_info_t *hal, u8 group)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+	u16 loop_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_del_mcc_group(): group(%d)\n",
+		group);
+	if (mac == NULL)
+		goto exit;
+	/*prev_groups always set to 1, driver stop all group pattern in the same hw band.*/
+	mac_status = mac->ops->del_mcc_group(mac, group, 1);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_del_mcc_group(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+
+	do {
+		if (mac->ops->check_del_mcc_group_done(mac, group) ==
+								MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mac_del_mcc_group(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_del_mcc_group(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_mcc_request_tsf(struct hal_info_t *hal,
+					u8 group, u8 macid_x, u8 macid_y)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+	u16 loop_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_mcc_request_tsf(): group(%d), macid_x(%d), macid_y(%d)\n",
+		group, macid_x, macid_y);
+	if (mac == NULL)
+		goto exit;
+	mac_status = mac->ops->mcc_request_tsf(mac, group, macid_x, macid_y);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_mcc_request_tsf(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+
+	do {
+		if (mac->ops->check_mcc_request_tsf_done(mac, group) ==
+								MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mac_mcc_request_tsf(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_mcc_request_tsf(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_mcc_macid_bitmap(struct hal_info_t *hal,
+					u8 group, u8 macid, u8 *bitmap, u8 len)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+	u16 loop_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_mcc_macid_bitmap(): group(%d), macid(%d)\n",
+		group, macid);
+	if (mac == NULL)
+		goto exit;
+	mac_status = mac->ops->mcc_macid_bitmap(mac, group, macid, bitmap, len);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_mcc_macid_bitmap(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+
+	do {
+		if (mac->ops->check_mcc_macid_bitmap_done(mac, group) ==
+								MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mac_mcc_macid_bitmap(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_mcc_macid_bitmap(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_mcc_sync_enable(struct hal_info_t *hal,
+				u8 group, u8 source, u8 target, u8 offset)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+	u16 loop_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_mcc_sync_enable(): group(%d), source macid(%d), target macid(%d), offset(%d)\n",
+		group, source, target, offset);
+	if (mac == NULL)
+		goto exit;
+	mac_status = mac->ops->mcc_sync_enable(mac, group, source, target, offset);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "mac_mcc_sync_enable(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	do {
+		if (mac->ops->check_mcc_sync_enable_done(mac, group) ==
+								MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "mac_mcc_sync_enable(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "mac_mcc_sync_enable(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_set_duration(struct hal_info_t *hal,
+					struct rtw_phl_mcc_en_info *en_info,
+					struct rtw_phl_mcc_bt_info *bt_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_mcc_duration_info info = {0};
+	u32 mac_status;
+	u16 loop_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> rtw_hal_mac_set_duration()\n");
+	_hal_mac_mcc_fill_duration_info(en_info, bt_info, &info);
+	mac_status = mac->ops->mcc_set_duration(mac, &info);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_set_duration(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	do {
+		if (mac->ops->check_mcc_set_duration_done(mac, (u8)info.group)
+								== MACSUCCESS)
+			break;
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < POLLING_HALMAC_CNT);
+
+	if (loop_cnt < POLLING_HALMAC_CNT) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mac_set_duration(): polling ok, count(%d)\n",
+			  loop_cnt);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_set_duration(): polling timeout\n");
+	}
+exit:
+	return hal_status;
+
+}
+
+enum rtw_hal_status rtw_hal_mac_get_mcc_tsf_rpt(struct hal_info_t *hal,
+					u8 group, u32 *tsf_x_h, u32 *tsf_x_l,
+					u32 *tsf_y_h, u32 *tsf_y_l)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+
+	if (mac == NULL)
+		goto exit;
+	mac_status = mac->ops->get_mcc_tsf_rpt(mac, group, tsf_x_h, tsf_x_l,
+						tsf_y_h, tsf_y_l);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_get_mcc_tsf_rpt(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_get_mcc_status_rpt(struct hal_info_t *hal,
+				u8 group, u8 *status, u32 *tsf_h, u32 *tsf_l)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+
+	if (mac == NULL)
+		goto exit;
+	mac_status = mac->ops->get_mcc_status_rpt(mac, group, status, tsf_h,
+						tsf_l);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "mac_mcc_sync_enable(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_mac_get_mcc_group(struct hal_info_t *hal, u8 *group)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status;
+
+	if (mac == NULL)
+		goto exit;
+	mac_status = mac->ops->get_mcc_group(mac, group);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mac_get_mcc_group(): fault, status = %d.\n",
+			  mac_status);
+		goto exit;
+	}
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return hal_status;
+}
+#endif /* CONFIG_MCC_SUPPORT */
+
+#ifdef CONFIG_PHL_CHSWOFLD
+/**
+ * rtw_hal_mac_ch_switch_ofld() - Offload ch switching and rf reloading to fw
+ * @hal_info:	struct hal_info_t *
+ * @band_idx:	band idx 0~1, 0x0: band0, 0x1: band1
+ * @pri_ch:	primary channel
+ * @central_ch:	center channel
+ * @band:	enum band_type
+ * @bw:		enum channel_width
+ * @reload_rf:	reload RF params is needed or not
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when operation success.
+ */
+enum rtw_hal_status rtw_hal_mac_ch_switch_ofld(struct hal_info_t *hal_info,
+					       u8 band_idx, u8 pri_ch, u8 central_ch,
+					       enum band_type band, enum channel_width bw,
+					       bool reload_rf)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_ch_switch_parm chsw = {0};
+	struct mac_ax_ch_switch_rpt rpt = {0};
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 cur_time = _os_get_cur_time_ms();
+	u32 passing_time;
+	u32 mac_status;
+
+	if (mac == NULL)
+		goto exit;
+
+	chsw.pri_ch = pri_ch;
+	chsw.central_ch = central_ch;
+	chsw.bw = bw;
+	chsw.ch_band = band;
+	chsw.band = band_idx;
+	chsw.reload_rf = reload_rf;
+
+	mac_status = mac->ops->ch_switch_ofld(mac, chsw);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s: fault, status = %d.\n",
+			  __func__, mac_status);
+		goto exit;
+	}
+
+	while (1) {
+		mac_status = mac->ops->check_fwofld_done(mac, FW_OFLD_OP_CH_SWITCH);
+		passing_time = phl_get_passing_time_ms(cur_time);
+		if (mac_status == MACSUCCESS) {
+			PHL_TRACE(COMP_PHL_MAC, _PHL_DEBUG_, "%s: polling ok with duration %ums\n",
+				  __func__, passing_time);
+			break;
+		}
+
+		if (mac_status != MACPROCBUSY) {
+			PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_,
+				  "%s: operation %u failed in check_fwofld_done(), status = %d.\n",
+				  __func__, FW_OFLD_OP_CH_SWITCH, mac_status);
+			hal_status = RTW_HAL_STATUS_NOT_SUPPORT;
+			goto exit;
+		}
+
+		if (passing_time > MAX_POLLING_FW_STS_TIME) {
+			PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s: polling timeout (> %ums)!!!\n",
+				  __func__, MAX_POLLING_FW_STS_TIME);
+			hal_status = RTW_HAL_STATUS_TIMEOUT;
+			goto exit;
+		}
+
+		_os_delay_ms(hal_to_drvpriv(hal_info), 1);
+	}
+
+	mac_status = mac->ops->get_ch_switch_rpt(mac, &rpt);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s: get_ch_switch_rpt() fault, status = %d.\n",
+			  __func__, mac_status);
+		goto exit;
+	}
+	if (rpt.result != CHSW_OK)
+		PHL_TRACE(COMP_PHL_MAC, _PHL_WARNING_, "%s: chsw failed, result = %d.\n",
+			  __func__, rpt.result);
+	if (rpt.result == CHSW_OK || rpt.result == CHSW_STOPSER_FAIL_WARN
+	    || rpt.result == CHSW_RF_RELOAD_FAIL_WARN || rpt.result == CHSW_STARTSER_FAIL_WARN)
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+
+exit:
+	return hal_status;
+}
+#endif
+
+#ifdef CONFIG_PHL_BCN_ERLY_RPT
+/**
+ * rtw_hal_mac_cfg_bcn_early_rpt() - Set bcn early rpt
+ * @hal_info:	struct hal_info_t *
+ * @band:	band idx 0~1, 0x0: band0, 0x1: band1
+ * @port:	port idx 0~4
+ * @en:		0x0: disable rpt, 0x1: enable rpt
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when setting is ok.
+ */
+enum rtw_hal_status rtw_hal_mac_cfg_bcn_early_rpt(struct hal_info_t *hal_info,
+						  u8 band, u8 port, u8 en)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	u32 mac_status;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac_status = mac->ops->cfg_bcn_early_rpt(mac, band, port, en);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "rtw_hal_mac_cfg_bcn_early_rpt(en=%d): fault, status = %d.\n",
+			  en, mac_status);
+		goto exit;
+	}
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return hal_status;
+}
+#endif
+
+#ifdef CONFIG_PHL_P2PPS
+#define P2P_ACT_INIT 0
+#define P2P_ACT_UPDATE 1
+#define P2P_ACT_REMOVE 2
+#define P2P_ACT_TERMINATE 3
+
+#define P2P_TYPE_GO 0
+#define P2P_TYPE_GC 1
+
+void _hal_mac_dump_p2p_act_struct(struct mac_ax_p2p_act_info *mac_p2p_info)
+{
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): macid = %d\n",
+		mac_p2p_info->macid);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): noaid = %d\n",
+		mac_p2p_info->noaid);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): act = %d\n",
+		mac_p2p_info->act);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): type = %d\n",
+		mac_p2p_info->type);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): all_slep = %d\n",
+		mac_p2p_info->all_slep);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): srt = %x\n",
+		mac_p2p_info->srt);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): itvl = %d\n",
+		mac_p2p_info->itvl);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): dur = %d\n",
+		mac_p2p_info->dur);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): cnt = %d\n",
+		mac_p2p_info->cnt);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_dump_p2p_act_struct(): ctw = %d\n",
+		mac_p2p_info->ctw);
+}
+
+void _hal_mac_noa_fill_info(u8 action,
+	struct rtw_phl_noa_desc *desc,
+	u16 macid,
+	struct mac_ax_p2p_act_info *mac_p2p_info)
+{
+	struct rtw_wifi_role_t *wrole = desc->w_role;
+
+	mac_p2p_info->macid = (u8)macid;
+	mac_p2p_info->noaid = desc->noa_id;
+	mac_p2p_info->act = action;
+	if (wrole->type == PHL_RTYPE_AP)
+		mac_p2p_info->type = P2P_TYPE_GO;
+	else if (wrole->type == PHL_RTYPE_STATION)
+		mac_p2p_info->type = P2P_TYPE_GC;
+	mac_p2p_info->srt = desc->start_t_l;
+	mac_p2p_info->itvl = desc->interval;
+	mac_p2p_info->dur = desc->duration;
+	mac_p2p_info->cnt = desc->count;
+}
+
+enum rtw_hal_status _hal_mac_get_p2p_stat(struct hal_info_t *hal)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u16 loop_cnt;
+	u32 m_stat;
+
+	for (loop_cnt = 0; loop_cnt < POLLING_HALMAC_CNT; loop_cnt++) {
+		m_stat = mac->ops->get_p2p_stat(mac) ;
+		if (m_stat == MACSUCCESS) {
+			PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]_hal_mac_get_p2p_stat(): polling ok, count(%d)\n",
+				loop_cnt);
+			return RTW_HAL_STATUS_SUCCESS;
+		} else if (m_stat == MACP2PSTFAIL) {
+			PHL_TRACE(COMP_PHL_P2PPS, _PHL_ERR_, "[NoA]_hal_mac_get_p2p_stat(): polling error, count(%d)\n",
+				loop_cnt);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+		_os_sleep_ms(hal_to_drvpriv(hal), POLLING_HALMAC_TIME);
+	}
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_ERR_, "[NoA]_hal_mac_get_p2p_stat(): polling timeout!\n");
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+
+enum rtw_hal_status rtw_hal_mac_p2p_macid_ctrl(struct hal_info_t *hal,
+			bool join, u16 main_macid, u8 *bitmap, u32 bitmap_len)
+{
+	enum rtw_hal_status h_sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_p2p_macid_info macid_info = {0};
+	u32 m_stat;
+
+	macid_info.main_macid = (u8)main_macid;
+	macid_info.ctrl_type = join ? P2P_MACID_CTRL_JOIN : P2P_MACID_CTRL_CLR;
+	macid_info.bitmap = bitmap;
+	macid_info.bmap_len = bitmap_len;
+	m_stat = mac->ops->p2p_macid_ctrl_h2c(mac, &macid_info);
+	if (m_stat == MACSUCCESS) {
+		h_sts = _hal_mac_get_p2p_stat(hal);
+	} else {
+		PHL_TRACE(COMP_PHL_P2PPS, _PHL_ERR_, "[NoA]%s: mac_p2p_macid_ctrl_h2c Fault, status = %d\n",
+			__FUNCTION__, m_stat);
+		h_sts = RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]%s: h_stat(%d), main_macid = %d, ctrl_type(%d), bitmap(0x%x)\n",
+		__FUNCTION__, h_sts, macid_info.main_macid,
+		macid_info.ctrl_type, (u8)(macid_info.bitmap[0]));
+	return h_sts;
+}
+
+enum rtw_hal_status _hal_mac_set_p2p_act(struct hal_info_t *hal,
+	struct mac_ax_p2p_act_info *mac_p2p_info)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 m_stat;
+
+	m_stat = mac->ops->p2p_act_h2c(mac, mac_p2p_info);
+	if (m_stat != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_P2PPS, _PHL_ERR_, "[NoA]_hal_mac_set_noa_act(): Fault, status = %d, Action = %d\n",
+				m_stat, mac_p2p_info->act);
+		return RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+	h_stat = _hal_mac_get_p2p_stat(hal);
+	return h_stat;
+}
+
+enum rtw_hal_status rtw_hal_mac_noa_init(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_p2p_act_info mac_p2p_info = {0};
+
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_noa_init()\n");
+	_hal_mac_noa_fill_info(P2P_ACT_INIT, in_desc, macid, &mac_p2p_info);
+	_hal_mac_dump_p2p_act_struct(&mac_p2p_info);
+	h_stat = _hal_mac_set_p2p_act(hal, &mac_p2p_info);
+	return h_stat;
+}
+
+enum rtw_hal_status rtw_hal_mac_noa_update(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_p2p_act_info mac_p2p_info = {0};
+
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_noa_update()\n");
+	_hal_mac_noa_fill_info(P2P_ACT_UPDATE, in_desc, macid, &mac_p2p_info);
+	_hal_mac_dump_p2p_act_struct(&mac_p2p_info);
+	h_stat = _hal_mac_set_p2p_act(hal, &mac_p2p_info);
+	return h_stat;
+}
+
+enum rtw_hal_status rtw_hal_mac_noa_remove(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_p2p_act_info mac_p2p_info = {0};
+
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_noa_remove()\n");
+	_hal_mac_noa_fill_info(P2P_ACT_REMOVE, in_desc, macid, &mac_p2p_info);
+	_hal_mac_dump_p2p_act_struct(&mac_p2p_info);
+	h_stat = _hal_mac_set_p2p_act(hal, &mac_p2p_info);
+	return h_stat;
+
+}
+
+enum rtw_hal_status rtw_hal_mac_noa_terminate(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_p2p_act_info mac_p2p_info = {0};
+
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_noa_terminate()\n");
+	_hal_mac_noa_fill_info(P2P_ACT_TERMINATE, in_desc, macid, &mac_p2p_info);
+	_hal_mac_dump_p2p_act_struct(&mac_p2p_info);
+	h_stat = _hal_mac_set_p2p_act(hal, &mac_p2p_info);
+	return h_stat;
+
+}
+
+enum rtw_hal_status rtw_hal_mac_tsf32_tog_enable(struct hal_info_t *hal,
+	u8 hw_band,
+	u8 port,
+	u16 early)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_t32_togl_info mac_t32_tog_info = {0};
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 m_stat;
+
+	mac_t32_tog_info.band = hw_band;
+	mac_t32_tog_info.port = port;
+	mac_t32_tog_info.en = true;
+	mac_t32_tog_info.early = early;
+	m_stat = mac->ops->tsf32_togl_h2c(mac, &mac_t32_tog_info);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_tsf32_tog_enable() m_stat = %d\n",
+		m_stat);
+	if(m_stat == MACSUCCESS)
+		h_stat = RTW_HAL_STATUS_SUCCESS;
+	return h_stat;
+}
+
+enum rtw_hal_status rtw_hal_mac_tsf32_tog_disable(struct hal_info_t *hal,
+	u8 hw_band,
+	u8 port)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_t32_togl_info mac_t32_tog_info = {0};
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 m_stat;
+
+	mac_t32_tog_info.band = hw_band;
+	mac_t32_tog_info.port = port;
+	mac_t32_tog_info.en = false;
+	m_stat = mac->ops->tsf32_togl_h2c(mac, &mac_t32_tog_info);
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_tsf32_tog_disable() m_stat = %d\n",
+		m_stat);
+	if(m_stat == MACSUCCESS)
+		h_stat = RTW_HAL_STATUS_SUCCESS;
+	return h_stat;
+}
+
+enum rtw_hal_status rtw_hal_mac_get_tsf32_tog_rpt(struct hal_info_t *hal,
+	struct rtw_phl_tsf32_tog_rpt *rpt)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_t32_togl_rpt m_rpt = {0};
+	u32 m_stat = 0;
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_get_tsf32_tog_rpt()\n");
+	m_stat = mac->ops->get_t32_togl_rpt(mac, &m_rpt);
+	if (m_stat != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]rtw_hal_mac_get_tsf32_tog_rpt() MAC FAIL(%d)!\n",
+			m_stat);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	rpt->band = m_rpt.band;
+	rpt->port = m_rpt.port;
+	rpt->valid = m_rpt.valid;
+	rpt->early = m_rpt.early;
+	rpt->status = m_rpt.status;
+	rpt->tsf_l = m_rpt.tsf_l;
+	rpt->tsf_h = m_rpt.tsf_h;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+void rtw_hal_mac_notification(struct hal_info_t *hal_info,
+                              enum phl_msg_evt_id event,
+                              u8 band)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	mac->ops->event_notify(mac, event, band);
+#else
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+#endif
+}
+
+void rtw_hal_mac_cmd_notification(struct hal_info_t *hal_info,
+                              void *hal_cmd,
+                              u8 band)
+{
+	/*todo*/
+}
+
+enum rtw_hal_status
+rtw_hal_mac_trigger_fw_conflict(struct hal_info_t *hal_info, u32 addr, u8 vol)
+{
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 i = 0;
+	u32 convert_mask = 0xffffffff;
+	u32 val;
+
+	/* Switch voltage */
+	mac->ops->set_hw_value(mac, MAC_AX_HW_SET_CORE_SWR_VOLT, &vol);
+
+	/* Trigger method: H2C Halt */
+	hal_ops->write_macreg(hal_info, 0x168, convert_mask, 0x5dc0007);
+	hal_ops->write_macreg(hal_info, 0x160, convert_mask, 0x1);
+
+	/* loop read reg */
+	for(i = 0; i<1000; i++){
+		val = hal_ops->read_macreg(hal_info,
+					    addr,
+					    convert_mask);
+		PHL_INFO("%s: voltag %d count %d io_value = %x\n", __FUNCTION__,vol, i, val);
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_set_gpio_func(struct rtw_hal_com_t *hal_com, u8 func, s8 gpio_cfg){
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = ops->set_gpio_func(mac, func, gpio_cfg);
+
+	if (ret != MACSUCCESS)
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "Set GPIO failure, status = %d\n", ret);
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_bfee_set_vht_gid(struct hal_info_t *hal,
+			     u8 band, struct rtw_phl_gid_pos_tbl *tbl)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct mac_gid_pos gid_pos = {0};
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	u32 mac_status = 0;
+	u8 i = 0;
+
+	gid_pos.band = band;
+	/* For GID 0~31 */
+	for (i = 0; i < 4; i++) {
+		gid_pos.gid_tab[0] |= (tbl->gid_vld[i] << (i << 3));
+	}
+
+	for (i = 0; i < 8; i++) {
+		if (i < 4)
+			gid_pos.user_pos[0] |= (tbl->pos[i] << (i << 3));
+		else
+			gid_pos.user_pos[1] |= (tbl->pos[i] << ((i - 4) << 3));
+	}
+	/* For GID 32~64 */
+	for (i = 4; i < 8; i++) {
+		gid_pos.gid_tab[1] |= (tbl->gid_vld[i] << ((i - 4) << 3));
+	}
+	for (i = 8; i < 16; i++) {
+		if (i < 12)
+			gid_pos.user_pos[2] |= (tbl->pos[i] << ((i - 8) << 3));
+		else
+			gid_pos.user_pos[3] |= (tbl->pos[i] << ((i - 12) << 3));
+	}
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s() : Set VHT GID for band %d;\n",
+		  __func__, band);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s() : band %d ; gid tbl 0x%x 0x%x;\n",
+		  __func__, band,
+		  gid_pos.gid_tab[0], gid_pos.gid_tab[1]);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s() : user position 0x%x 0x%x 0x%x 0x%x;\n",
+		  __func__,
+		  gid_pos.user_pos[0], gid_pos.user_pos[1],
+		  gid_pos.user_pos[2], gid_pos.user_pos[3]);
+
+	mac_status = mac->ops->gidpos(mac, &gid_pos);
+	if (mac_status != MACSUCCESS) {
+		hal_status = RTW_HAL_STATUS_FAILURE;
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s() : fail, status = %d.\n",
+			  __func__, mac_status);
+	}
+
+	return hal_status;
+}
+
+/* val input unit : 256 us */
+enum rtw_hal_status
+rtw_hal_mac_set_tx_lifetime(struct hal_info_t *hal, enum phl_band_idx band,
+			    enum phl_lifetime_queue que, bool en, u16 val)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_lifetime_cfg cfg = {0};
+	u32 mac_status = 0;
+
+	if (HW_BAND_1 == band)
+		cfg.band = 1;
+	else
+		cfg.band = 0;
+
+	mac_status = mac->ops->get_hw_value(mac, MAC_AX_HW_GET_LIFETIME_CFG, (void *)&cfg);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s() : get fail, status = %d.\n",
+			  __func__, mac_status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (que == PHL_LIFETIME_MGQ) {
+		cfg.en.mgq_en = en;
+		cfg.val.mgq_val = val;
+	} else if (que == PHL_LIFETIME_ACQ) {
+		cfg.en.acq_en = en;
+		cfg.val.acq_val_1 = val;
+		cfg.val.acq_val_2 = val;
+		cfg.val.acq_val_3 = val;
+		cfg.val.acq_val_4 = val;
+	} else {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s() : incorrect queue.\n",
+			  __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	mac_status = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_LIFETIME_CFG, (void *)&cfg);
+
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s() : set fail, status = %d.\n",
+			  __func__, mac_status);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_gt3(struct hal_info_t *hal, u8 en, u32 timeout)
+{
+#ifdef CONFIG_VW_REFINE
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_gt3_cfg info = {0};
+	u32 mac_err = 0;
+
+	if (en) {
+		info.count_en = 1;	/* count mode */
+		info.gt3_en = 1;	/* enable */
+		info.timeout = (timeout & GT3_TIMEOUT_MASK);
+	}
+
+	mac_err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_GT3_TIMER, &info);
+
+	if (mac_err != MACSUCCESS)
+		return RTW_HAL_STATUS_FAILURE;
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_PHL_IO_OFLD
+enum rtw_hal_status rtw_hal_mac_add_cmd_ofld(struct rtw_hal_com_t *hal_com, struct rtw_mac_cmd *cmd)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 mac_status;
+
+	if (mac == NULL)
+		goto exit;
+	mac_status = mac->ops->add_cmd_ofld(mac, cmd);
+	if (mac_status != MACSUCCESS) {
+		PHL_TRACE(COMP_PHL_MAC, _PHL_ERR_, "%s(): fault, status = %d.\n",
+			 __func__, mac_status);
+		goto exit;
+	}
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return hal_status;
+}
+#endif /* CONFIG_PHL_IO_OFLD */
+
+enum rtw_hal_status rtw_hal_mac_set_hw_rts_th(struct hal_info_t *hal, u8 band,
+					      u16 time_th, u16 len_th)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_hw_rts_th hw_rts_th = {0};
+	u32 mac_err;
+
+	PHL_INFO("%s\n", __func__);
+	_os_mem_set(hal_to_drvpriv(hal), &hw_rts_th, 0, sizeof(hw_rts_th));
+
+	hw_rts_th.band = band;
+	hw_rts_th.time_th = time_th;
+	hw_rts_th.len_th = len_th;
+
+	mac_err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_HW_RTS_TH,
+					 &hw_rts_th);
+
+	if (mac_err != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, mac_err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u8 rtw_hal_mac_get_efuse_ver_len(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	return mac->hw_info->efuse_version_size;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_set_dfs_tb_ctrl(struct hal_info_t *hal_info, u8 set)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u8 dis_ru_26 = 0;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	if (set)
+		dis_ru_26 = 1;
+	ret = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_TX_RU26_TB, &dis_ru_26);
+	if (ret != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_RTW_MULTI_DEV_MULTI_BAND
+enum rtw_hal_status
+rtw_hal_mac_set_xtal_aac(struct rtw_hal_com_t *hal_com, u8 value)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = ops->set_hw_value(mac, MAC_AX_HW_SET_XTAL_AAC_MODE, &value);
+
+	if (ret != MACSUCCESS)
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "Set xtal aac failure, status = %d\n", ret);
+
+	return ret;
+}
+#endif /* CONFIG_RTW_MULTI_DEV_MULTI_BAND */
+
+enum rtw_hal_status
+rtw_hal_mac_patch_rx_rate(struct hal_info_t *hal_info, struct rtw_r_meta_data *mdata)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = mac->ops->patch_rx_rate(mac, mdata);
+	if (ret != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/**
+ * Control MAC Rx Pkt bring per packet DrvInfo.
+ **/
+enum rtw_hal_status
+rtw_hal_mac_drvinfo_cfg(struct hal_info_t *hal_info,
+			bool enable, enum phl_band_idx hw_band)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_per_pkt_phy_rpt cfg = {0};
+
+	cfg.en = enable;
+	cfg.band = hw_band;
+
+	ret = mac->ops->cfg_per_pkt_phy_rpt(mac, &cfg);
+
+	if (ret != MACSUCCESS) {
+		PHL_ERR("%s : mac status %d.\n", __func__, ret);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+/**
+ * rtw_hal_mac_set_tx_duty() - Set tx pause/un-pause interval
+ * @hal_info: struct hal_info_t*
+ * @pause_interval: tx pause interval (ms)
+ * @tx_interval: tx interval (ms)
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when operation success.
+ */
+enum rtw_hal_status
+rtw_hal_mac_set_tx_duty(struct hal_info_t *hal_info,
+			u16 pause_interval,
+			u16 tx_interval)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 mac_status = 0;
+
+	mac_status = mac->ops->tx_duty(mac, pause_interval, tx_interval);
+	if (mac_status != MACSUCCESS) {
+		hal_status = RTW_HAL_STATUS_FAILURE;
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s() : fail, status = %d.\n",
+			  __func__, mac_status);
+	}
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_mac_stop_tx_duty() - Stop previous tx duty config
+ * @hal_info: struct hal_info_t*
+ *
+ * Return RTW_HAL_STATUS_SUCCESS when operation success.
+ */
+enum rtw_hal_status
+rtw_hal_mac_stop_tx_duty(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	u32 mac_status = 0;
+
+	mac_status = mac->ops->tx_duty_stop(mac);
+	if (mac_status != MACSUCCESS) {
+		hal_status = RTW_HAL_STATUS_FAILURE;
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s() : fail, status = %d.\n",
+			  __func__, mac_status);
+	}
+
+	return hal_status;
+}
+
+/**
+ * query IC support AMPDU num
+ */
+u16 rtw_hal_mac_query_ampdu_num(struct hal_info_t *hal, u8 band)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_ops *mac_api = mac->ops;
+	struct mac_ax_intf_ops *mac_intf_ops = mac_api->intf_ops;
+
+	return (u16)mac_intf_ops->get_txagg_num(mac, band);
+}
+
+void
+rtw_hal_mac_init_txagg_num(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = (struct mac_ax_adapter *)hal_info->mac;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	hal_com->phy_hw_cap[0].txagg_num =  mac->ops->intf_ops->get_txagg_num(
+		mac,
+		HW_BAND_0);
+	hal_com->phy_hw_cap[1].txagg_num =  mac->ops->intf_ops->get_txagg_num(
+		mac,
+		HW_BAND_1);
+}
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_hw_cts2slef(struct hal_info_t *hal_info,
+			 struct mac_ax_cts2self_cfg *cts_cfg)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (MACSUCCESS !=
+	    mac->ops->set_hw_value(mac, MAC_AX_HW_SET_CTS2SELF, cts_cfg))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_usr_edca(struct hal_info_t *hal_info,
+                        u32 idx,
+                        u32 enable,
+                        u32 band,
+                        u32 wmm,
+                        struct rtw_edca_param *aggr,
+                        struct rtw_edca_param *mod)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u8 tmp_enable = 0;
+	struct mac_ax_usr_edca_param mac_param = {0};
+
+	mac_param.ac = aggr->ac;
+	mac_param.enable = (u8)enable;
+	mac_param.band = (u8)band;
+	mac_param.idx = idx;
+	mac_param.wmm = wmm;
+	mac_param.aggressive.aifs_us = (u8)(aggr->param & 0xff);
+	mac_param.aggressive.ecw_max = (u8)((aggr->param >> 12) & 0xf);
+	mac_param.aggressive.ecw_min = (u8)((aggr->param >> 8) & 0xf);
+	mac_param.aggressive.txop_32us = (u16)(aggr->param >> 16);
+	mac_param.moderate.aifs_us = (u8)(mod->param & 0xff);
+	mac_param.moderate.ecw_max = (u8)((mod->param >> 12) & 0xf);
+	mac_param.moderate.ecw_min = (u8)((mod->param >> 8) & 0xf);
+	mac_param.moderate.txop_32us = (u16)(mod->param >> 16);
+
+	tmp_enable = mac_param.enable;
+
+	PHL_INFO(" %s, ac(%d), enable(%d), band(%d), idx(%d), wmm(%d)\n",
+	         __FUNCTION__,
+	         mac_param.ac,
+	         mac_param.enable,
+	         mac_param.band,
+	         mac_param.idx,
+	         mac_param.wmm);
+	PHL_INFO(" %s, aggressive: aifs_us(%d), ecw_max(%d), ecw_min(%d), txop_32us(%d)\n",
+	         __FUNCTION__,
+	         mac_param.aggressive.aifs_us,
+	         mac_param.aggressive.ecw_max,
+	         mac_param.aggressive.ecw_min,
+	         mac_param.aggressive.txop_32us);
+	PHL_INFO(" %s, moderate: aifs_us(%d), ecw_max(%d), ecw_min(%d), txop_32us(%d)\n",
+	         __FUNCTION__,
+	         mac_param.moderate.aifs_us,
+	         mac_param.moderate.ecw_max,
+	         mac_param.moderate.ecw_min,
+	         mac_param.moderate.txop_32us);
+
+	if (mac_param.enable) {
+		mac_param.enable = 0x0;
+		PHL_INFO(" %s, set enable to 0 first.\n", __FUNCTION__);
+		if (mac->ops->set_hw_value(mac,
+		                           MAC_AX_HW_SET_USR_EDCA_PARAM,
+		                           (void*)&mac_param)) {
+			PHL_ERR(" %s, disable usr edca setting fail.\n", __FUNCTION__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+		mac_param.enable = tmp_enable;
+	}
+
+	if (MACSUCCESS != mac->ops->set_hw_value(mac,
+	                                         MAC_AX_HW_SET_USR_EDCA_PARAM,
+	                                         (void*)&mac_param)) {
+		PHL_ERR(" %s, set usr edca param fail.\n", __FUNCTION__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_txagg_timeout_thrhold(struct hal_info_t *hal_info)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_host_rpr_cfg rpr_cfg = {0};
+
+	rpr_cfg.txok_en = MAC_AX_FUNC_DEF;
+	rpr_cfg.rty_lmt_en = MAC_AX_FUNC_DEF;
+	rpr_cfg.lft_drop_en = MAC_AX_FUNC_DEF;
+	rpr_cfg.macid_drop_en = MAC_AX_FUNC_DEF;
+	rpr_cfg.tmr_def = 1;
+	rpr_cfg.agg_def = 1;
+	rpr_cfg.tmr = 0x10;
+
+	if (MACSUCCESS !=
+		mac->ops->set_hw_value(mac, MAC_AX_HW_SET_HOST_RPR, &rpr_cfg))
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_fw_general_io_test(struct hal_info_t *hal_info)
+{
+	/*struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+
+	if (MACSUCCESS !=
+		mac->ops->fw_general_io_test(mac))
+		return RTW_HAL_STATUS_FAILURE;
+
+	wait for halmac API update*/
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_mac_set_rsp_stat_rts_chk_en(struct hal_info_t *hal,
+							enum phl_band_idx band,
+							bool en)
+{
+	struct mac_ax_adapter *mac = hal_to_mac(hal);
+	struct mac_ax_set_resp_stat_rts_chk_cfg cfg = {0};
+	u32 err = 0;
+
+	cfg.band = band;
+	cfg.enable = en;
+
+	err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_RESP_STAT_RTS_CHK_EN,
+				     &cfg);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_get_freerun_cnt(struct rtw_hal_com_t *hal_com, enum phl_band_idx hw_band,
+			    u32 *freerun_cnt_h, u32 *freerun_cnt_l)
+{
+
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_freerun val;
+	u32 err = 0;
+
+	val.band = hw_band;
+
+	err = mac->ops->get_hw_value(mac, MAC_AX_HW_GET_FREERUN_CNT, &val);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	*freerun_cnt_h = val.freerun_h;
+	*freerun_cnt_l = val.freerun_l;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_mac_reset_freerun_cnt(struct rtw_hal_com_t *hal_com, enum phl_band_idx hw_band)
+{
+
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	u32 err = 0;
+
+	err = mac->ops->set_hw_value(mac, MAC_AX_HW_SET_FREERUN_RST, &hw_band);
+	if (err)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_sr_update(struct rtw_hal_com_t *hal_com,
+                      void *sr_info,
+                      u8 hw_band)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *ops = mac->ops;
+	u32 err = 0;
+
+	err = ops->sr_update(mac, (struct rtw_mac_ax_sr_info *)sr_info,
+	                     (enum mac_ax_band) hw_band);
+
+	if (err) {
+		PHL_INFO("%s: failure, status = %d\n", __func__, err);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_mac_set_aspm_test(struct hal_info_t *hal_info)
+{
+#ifdef CONFIG_HAL_MAC_DBG
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	u32 mac_err = MACSUCCESS;
+	u16 loop_cnt = 0;
+	u32 aspm_status_code = 0;
+	/* H2C */
+	 mac_err = mac->ops->test_l12(mac);
+	if (mac_err != MACSUCCESS) {
+		PHL_INFO("%s, test_l12 fail status(%d)\n", __func__, mac_err);
+		return RTW_HAL_STATUS_MAC_API_FAILURE;
+	}
+
+	/* wait for C2H*/
+	do {
+		 mac_err = mac->ops->get_test_l12_done(mac);
+		if (mac_err == MACSUCCESS)
+			break;
+		PHL_INFO("%s, get_test_l12_done status(%d)\n", __func__, mac_err);
+		_os_delay_ms(hal_to_drvpriv(hal_info), POLLING_HALMAC_TIME);
+		loop_cnt++;
+	} while (loop_cnt < 200);
+	if (loop_cnt < 200) {
+		PHL_INFO("%s, polling ok, count(%d)\n", __func__, loop_cnt);
+
+	} else {
+		PHL_INFO("%s, polling timeout\n", __func__);
+		hal_status = RTW_HAL_STATUS_FAILURE;
+		return hal_status;
+	}
+
+	/* check aspm result */
+	if(mac->ops->get_test_l12_rpt(mac , &aspm_status_code) == MACSUCCESS) {
+		if(aspm_status_code != 0) {
+			PHL_INFO("%s, rpt result failed! status(%d)\n", __func__, aspm_status_code);
+			hal_status = RTW_HAL_STATUS_FAILURE;
+		} else {
+			PHL_INFO("%s, rpt result succeed!\n", __func__);
+		}
+	} else {
+		PHL_INFO("%s, get l12 rpt fail\n", __func__);
+		hal_status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+#else /* !CONFIG_HAL_MAC_DBG */
+	PHL_WARN("%s: CONFIG_HAL_MAC_DBG not support!\n", __func__);
+
+	return RTW_HAL_STATUS_NOT_SUPPORT;
+#endif /* !CONFIG_HAL_MAC_DBG */
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_mac.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_mac.h
new file mode 100644
index 000000000000..7921973498bb
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_mac.h
@@ -0,0 +1,964 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_MAC_H_
+#define _HAL_API_MAC_H_
+#include "mac/mac_ax.h"
+
+#define POLLING_HALMAC_TIME 5
+#define POLLING_HALMAC_CNT 100
+
+#define hal_to_mac(_halinfo)	((struct mac_ax_adapter *)((_halinfo)->mac))
+
+#ifdef CONFIG_POWER_SAVE
+enum ps_adv_action {
+	PS_ADV_NONE = 0,
+	PS_ADV_EN_WAIT_PVB_RX,
+	PS_ADV_DIS_WAIT_PVB_RX,
+	PS_ADV_EN_TRX_EXT
+};
+
+#endif /* CONFIG_POWER_SAVE */
+
+
+#ifdef CONFIG_SDIO_HCI
+u8 hal_mac_sdio_read8(struct rtw_hal_com_t *hal, u32 addr);
+u16 hal_mac_sdio_read16(struct rtw_hal_com_t *hal, u32 addr);
+u32 hal_mac_sdio_read32(struct rtw_hal_com_t *hal, u32 addr);
+int hal_mac_sdio_write8(struct rtw_hal_com_t *hal, u32 addr, u8 value);
+int hal_mac_sdio_write16(struct rtw_hal_com_t *hal, u32 addr, u16 value);
+int hal_mac_sdio_write32(struct rtw_hal_com_t *hal, u32 addr, u32 value);
+void hal_mac_sdio_read_mem(struct rtw_hal_com_t *hal,
+					u32 addr, u32 cnt, u8 *pmem);
+
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+u8 hal_mac_sdio_iread8(struct rtw_hal_com_t *hal, u32 addr);
+u16 hal_mac_sdio_iread16(struct rtw_hal_com_t *hal, u32 addr);
+u32 hal_mac_sdio_iread32(struct rtw_hal_com_t *hal, u32 addr);
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+
+#endif /* CONFIG_SDIO_HCI */
+
+void rtw_hal_mac_get_version(char *ver_str, u16 len);
+void rtw_hal_mac_get_fw_ver(struct hal_info_t *hal_info, char *ver_str, u16 len);
+
+u16 hal_mac_get_macid_num(struct hal_info_t *hal);
+void hal_mac_get_hwinfo(struct hal_info_t *hal, struct hal_spec_t *hal_spec);
+
+#ifdef DBG_PHL_MAC_REG_RW
+bool rtw_hal_mac_reg_chk(struct rtw_hal_com_t *hal_com, u32 addr);
+#endif
+
+u32 rtw_hal_mac_init(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info);
+u32 rtw_hal_mac_deinit(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal_info);
+#ifdef CONFIG_HAL_MAC_DBG
+bool rtw_hal_mac_proc_cmd(struct hal_info_t *hal_info, struct rtw_proc_cmd *incmd,
+						char *output, u32 out_len);
+#endif
+enum rtw_hal_status rtw_hal_mac_get_hwseq(void *hal, u16 macid, u8 ref_sel, u8 ssn_sel,
+						u16 *hw_seq);
+
+enum rtw_hal_status rtw_hal_mac_get_pwr_state(struct hal_info_t *hal_info,
+						enum rtw_mac_pwr_st *pwr_state);
+
+enum rtw_hal_status rtw_hal_mac_power_switch(struct rtw_phl_com_t *phl_com,
+					     struct hal_info_t *hal_info,
+					     u8 on_off);
+
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status rtw_hal_mac_dbcc_pre_cfg(struct rtw_phl_com_t *phl_com,
+					     struct hal_info_t *hal_info,
+					     u8 dbcc_en);
+enum rtw_hal_status rtw_hal_mac_dbcc_cfg(struct rtw_phl_com_t *phl_com,
+					 struct hal_info_t *hal_info,
+					 u8 dbcc_en);
+enum rtw_hal_status rtw_hal_mac_dbcc_trx_ctrl(struct hal_info_t *hal_info,
+                                              enum phl_band_idx band_idx,
+                                              bool pause);
+enum rtw_hal_status rtw_hal_mac_dbcc_hci_ctrl(struct hal_info_t *hal_info,
+                                              enum phl_band_idx band_idx, u8 pause);
+#endif /* CONFIG_DBCC_SUPPORT */
+
+enum rtw_hal_status
+rtw_hal_mac_poll_hw_tx_done(struct hal_info_t *hal_info);
+enum rtw_hal_status
+rtw_hal_mac_hw_tx_resume(struct hal_info_t *hal_info);
+enum rtw_hal_status
+rtw_hal_mac_poll_hw_rx_done(struct hal_info_t *hal_info);
+enum rtw_hal_status
+rtw_hal_mac_hw_rx_resume(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_watchdog(struct hal_info_t *hal_info, struct rtw_phl_com_t *phl_com);
+
+enum rtw_hal_status rtw_hal_mac_cfg_txdma(struct hal_info_t *hal, u8 state);
+enum rtw_hal_status rtw_hal_mac_poll_txdma_idle(struct hal_info_t *hal);
+#ifdef CONFIG_PCI_HCI
+enum rtw_hal_status rtw_hal_mac_set_pcicfg(struct hal_info_t *hal_info,
+					struct mac_ax_pcie_cfgspc_param *pci_cfgspc);
+enum rtw_hal_status rtw_hal_mac_ltr_set_pcie(struct hal_info_t *hal_info,
+					enum rtw_pcie_bus_func_cap_t hw_ctrl,
+					u8 idle_ctrl, u32 idle_val, u8 act_ctrl, u32 act_val);
+
+enum rtw_hal_status rtw_hal_mac_ltr_sw_trigger(struct hal_info_t *hal_info, enum rtw_pcie_ltr_state state);
+enum rtw_hal_status hal_mac_set_l2_leave(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_mac_poll_rxdma_idle(struct hal_info_t *hal,
+					struct mac_ax_rxdma_ch_map *ch_map);
+
+enum rtw_hal_status rtw_hal_mac_clr_bdidx(struct hal_info_t *hal);
+
+enum rtw_hal_status rtw_hal_mac_rst_bdram(struct hal_info_t *hal);
+
+
+enum rtw_hal_status rtw_hal_mac_cfg_dma_io(struct hal_info_t *hal, u8 en);
+
+enum rtw_hal_status
+rtw_hal_mac_tx_res_query(struct hal_info_t *hal, u8 ch_idx, u16 *host_idx,
+                         u16 *hw_idx, u16 *avail_txbd);
+enum rtw_hal_status
+rtw_hal_mac_rx_res_query(struct hal_info_t *hal, u8 ch_idx, u16 *host_idx,
+                         u16 *hw_idx, u16 *avail_rxbd);
+enum rtw_hal_status
+rtw_hal_mac_trigger_txstart(struct hal_info_t *hal,
+                            struct tx_base_desc *txbd_ring, u8 ch_idx);
+enum rtw_hal_status
+rtw_hal_mac_notify_rxdone(struct hal_info_t *hal, struct rx_base_desc *rxbd,
+                          u8 ch_idx);
+enum rtw_hal_status rtw_hal_mac_aspm_frontdoor_set(struct hal_info_t *hal_info);
+#endif
+
+
+#ifdef CONFIG_USB_HCI
+u8 hal_mac_get_bulkout_id(struct hal_info_t *hal, u8 ch_dma, u8 mode);
+u32 hal_mac_usb_tx_agg_cfg(struct hal_info_t *hal, u8* wd_buf, u8 agg_num);
+u32 hal_mac_usb_rx_agg_cfg(struct hal_info_t *hal, u8 agg_mode,
+	u8 drv_define, u8 timeout, u8 size, u8 pkt_num);
+u8 hal_mac_usb_get_max_bulkout_wd_num(struct hal_info_t *hal);
+u16 hal_mac_usb_get_max_dma_txagg_msk(struct hal_info_t *hal);
+enum rtw_hal_status hal_mac_force_usb_switch(struct hal_info_t *hal);
+u32 hal_mac_get_cur_usb_mode(struct hal_info_t *hal);
+u32 hal_mac_get_usb_support_ability(struct hal_info_t *hal);
+u32 hal_mac_set_usb_support_ability(struct hal_info_t *hal, u32 ability);
+u32 hal_mac_get_usb_mode_status(struct hal_info_t *hal, u32 *status);
+u32 hal_mac_get_u3_perf_mode(struct hal_info_t *hal, u32 *perf_mode);
+#endif
+
+#ifdef CONFIG_PHL_CSUM_OFFLOAD_RX
+enum rtw_hal_status rtw_hal_mac_chk_rx_tcpip_chksum_ofd(struct hal_info_t *hal,
+							struct rtw_r_meta_data *mdata,
+							u8 status);
+#endif /* CONFIG_PHL_CSUM_OFFLOAD_RX */
+
+#ifdef CONFIG_SDIO_HCI
+void rtw_hal_mac_sdio_cfg(struct rtw_phl_com_t *phl_com,
+	struct hal_info_t *hal_info, struct rtw_ic_info *ic_info);
+void rtw_hal_mac_sdio_tx_cfg(struct rtw_hal_com_t *hal);
+void rtw_hal_mac_sdio_rx_agg_cfg(struct rtw_hal_com_t *hal, bool enable,
+				 u8 drv_define, u8 timeout, u8 size, u8 pkt_num);
+bool rtw_hal_mac_sdio_check_tx_allow(struct rtw_hal_com_t *hal, u8 dma_ch,
+				     u8 *buf, u32 len, u8 agg_count,
+				     u16 *pkt_len, u8 *wp_offset, u32 *txaddr,
+				     u32 *txlen);
+int rtw_hal_mac_sdio_parse_rx(struct rtw_hal_com_t *hal,
+			      struct rtw_rx_buf *rxbuf);
+int rtw_hal_mac_sdio_rx(struct rtw_hal_com_t *hal,
+			struct rtw_rx_buf *rxbuf);
+void rtw_hal_mac_sdio_lps_flg(struct rtw_hal_com_t *hal, u8 *flg);
+#endif
+
+struct hal_init_info_t {
+	struct mac_ax_trx_info trx_info;
+	struct mac_ax_intf_info intf_info;
+	char *ic_name;
+};
+enum rtw_hal_status
+rtw_hal_mac_init_mac(void *mac, struct hal_init_info_t *init_info);
+
+enum rtw_hal_status
+rtw_hal_mac_trx_init(void *mac, struct hal_init_info_t *init_info);
+
+enum rtw_hal_status
+rtw_hal_mac_loopback_set_trx_mode(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_set_hci_speed(struct hal_info_t *hal_info, u8 speed);
+
+enum rtw_hal_status
+rtw_hal_mac_get_hci_speed(struct hal_info_t *hal_info, u8 *speed);
+
+enum rtw_hal_status
+rtw_hal_mac_get_max_hci_speed(struct hal_info_t *hal_info, u8 *speed);
+
+enum rtw_hal_status rtw_hal_mac_set_rxfltr_loopbk_mode(void *hal);
+
+enum rtw_fw_type
+rtw_hal_mac_get_fw_type(struct rtw_phl_com_t *phl_com);
+
+enum rtw_hal_status
+rtw_hal_mac_hal_init(struct rtw_phl_com_t *phl_com,
+		     struct hal_info_t *hal_info,
+		     struct hal_init_info_t *init_info);
+
+enum rtw_hal_status
+rtw_hal_mac_set_resp_ack_chk_cca(struct hal_info_t *hal_info, u8 band, u8 en);
+
+enum rtw_hal_status
+rtw_hal_mac_sifs_chk_cca_en(struct hal_info_t *hal_info, u8 band, u8 en);
+
+enum rtw_hal_status
+rtw_hal_mac_hal_fast_init(struct rtw_phl_com_t *phl_com,
+			  struct hal_info_t *hal_info,
+			  struct hal_init_info_t *init_info);
+
+enum rtw_hal_status rtw_hal_mac_hal_fast_deinit(struct rtw_phl_com_t *phl_com,
+						struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_hal_deinit(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_mac_chk_allq_empty(struct hal_info_t *hal_info, u8 *empty);
+
+#ifdef CONFIG_WOWLAN
+enum rtw_hal_status
+rtw_hal_mac_cfg_wow_sleep(struct hal_info_t *hal_info, u8 sleep);
+enum rtw_hal_status
+rtw_hal_mac_get_wow_fw_status(struct hal_info_t *hal_info, u8 *status, u8 func_en);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_wow_wake(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_wow_wake_info *info);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_disc_dec(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_disc_det_info *info);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_dev2hst_gpio(struct hal_info_t *hal_info, u8 en, struct rtw_wow_gpio_info *cfg);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_keep_alive(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_keep_alive_info *info);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_ndp_ofld(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_ndp_ofld_info *cfg);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_arp_ofld(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_arp_ofld_info *cfg);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_gtk_ofld(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_gtk_ofld_info *cfg);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_realwow(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_realwow_info *cfg);
+
+enum rtw_hal_status rtw_hal_mac_get_wake_rsn(struct hal_info_t *hal_info, u8 *wake_rsn, u8 *reset);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_wow_cam(struct hal_info_t *hal_info, u16 macid, u8 en, struct rtw_pattern_match_info *cfg);
+
+enum rtw_hal_status
+rtw_hal_mac_get_aoac_rpt(struct hal_info_t *hal_info, struct rtw_aoac_report *aoac_info, u8 rx_ready);
+
+enum rtw_hal_status rtw_hal_mac_set_wowlan(struct hal_info_t *hal, enum mac_ax_wow_ctrl ctrl);
+enum rtw_hal_status rtw_hal_mac_wow_chk_txq_empty(struct hal_info_t *hal, u8 *empty);
+enum rtw_hal_status rtw_hal_mac_wow_wde_drop(struct hal_info_t *hal, u8 band);
+enum rtw_hal_status
+rtw_hal_mac_cfg_nlo(struct hal_info_t *hal, u16 macid, u8 en,
+		    struct rtw_nlo_info *cfg);
+enum rtw_hal_status
+rtw_hal_mac_cfg_periodic_wake(struct hal_info_t *hal, u16 macid, u8 en,
+			      struct rtw_periodic_wake_info *cfg);
+enum rtw_hal_status
+rtw_hal_mac_wow_dbg_dump(struct hal_info_t *hal_info);
+enum rtw_hal_status
+rtw_hal_mac_wow_req_tri_evt(struct hal_info_t *hal);
+enum rtw_hal_status
+rtw_hal_mac_wow_req_diag_rpt(struct hal_info_t *hal);
+
+#ifdef CONFIG_PHL_MDNS_OFFLOAD
+enum rtw_hal_status
+rtw_hal_mac_cfg_mdns_ofld(struct hal_info_t *hal_info, u16 macid, u8 en,
+			  struct rtw_mdns_ofld_info *cfg);
+#endif
+#ifdef CONFIG_PHL_WOW_APF
+enum rtw_hal_status
+rtw_hal_mac_cfg_apf_ofld(struct hal_info_t *hal_info, u16 macid, struct rtw_apf_info *cfg, u8 set);
+#endif
+#endif /* CONFIG_WOWLAN */
+
+#ifdef CONFIG_PHL_SCANOFLD
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld_add_ch(struct hal_info_t *hal, u8 hw_band,
+                             struct scan_ofld_ch_info *cfg, bool ofld);
+
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld_fw_busy(struct hal_info_t *hal, u8 hw_band);
+
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld_chlist_busy(struct hal_info_t *hal, u8 hw_band);
+
+enum rtw_hal_status
+rtw_hal_mac_scan_ofld(struct hal_info_t *hal, u16 mac_id, u8 hw_band, u8 hw_port,
+                      struct scan_ofld_info *cfg);
+
+u8 rtw_hal_mac_scan_ofld_trans_notify_rsn(u8 mac_rsn);
+
+void rtw_hal_mac_scan_ofld_leave_cur_chnl(struct hal_info_t *hal, u8 hw_band,
+					  u8 ch_band, u8 ch);
+#endif /* CONFIG_PHL_SCANOFLD */
+
+u32
+rtw_hal_mac_get_txdesc_len(void *mac, struct rtw_xmit_req *treq);
+
+enum rtw_hal_status
+rtw_hal_mac_fill_txdesc(void *mac,
+                        struct rtw_xmit_req *treq,
+                        u8 *wd_buf,
+                        u32 *wd_len);
+
+enum rtw_hal_status
+rtw_hal_mac_enable_cpu(struct hal_info_t *hal_info, u8 reason, u8 dlfw);
+
+enum rtw_hal_status
+rtw_hal_mac_disable_cpu(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_romdl(struct hal_info_t *hal_info, u8 *rom_buf, u32 rom_size);
+
+enum rtw_hal_status
+rtw_hal_mac_fwdl(struct hal_info_t *hal_info, u8 *fw_buf, u32 fw_size);
+
+bool
+rtw_hal_mac_fwredl_needed(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_fwredl(struct hal_info_t *hal_info, u8 *fw_buf, u32 fw_size);
+
+enum rtw_hal_status
+rtw_hal_mac_query_fw_buff(struct hal_info_t *hal_info, enum rtw_fw_type cat, u8 **fw, u32 *fw_len);
+
+enum rtw_hal_status
+rtw_hal_mac_enable_fw(struct hal_info_t *hal_info, u8 fw_type);
+
+enum rtw_hal_status
+rtw_hal_mac_set_hw_ampdu_cfg(struct hal_info_t *hal_info, u8 band,
+								  u16 max_agg_num, u8 max_agg_time);
+
+enum rtw_hal_status
+rtw_hal_mac_hwamsdu_macid_en(struct hal_info_t *hal_info,
+			   u8 macid,
+			   u8 enable);
+
+enum rtw_hal_status
+rtw_hal_mac_enable_hwamsdu(struct hal_info_t *hal_info,
+			   u8 enable,
+			   u8 max_num,
+			   u8 en_single_amsdu,
+			   u8 en_last_amsdu_padding);
+
+enum rtw_hal_status
+rtw_hal_mac_hwamsdu_max_len(struct hal_info_t *hal_info,
+			   u8 macid,
+			   u8 amsdu_max_len);
+
+enum rtw_hal_status
+rtw_hal_mac_hdr_conv_tx_macid_en(struct hal_info_t *hal_info,
+				 u8 macid,
+				 u8 qos_field_h,
+				 u8 qos_field_h_en,
+				 u8 mhdr_len,
+				 u8 vlan_tag_valid);
+
+enum rtw_hal_status rtw_hal_dmc_tbl_cfg(struct hal_info_t *hal_info,
+					struct mac_ax_dctl_info *dctl_info,
+					struct mac_ax_dctl_info *dctl_info_mask,
+					u16 macid);
+
+enum rtw_hal_status rtw_hal_cmc_tbl_cfg(struct hal_info_t *hal_info,
+				struct rtw_hal_mac_ax_cctl_info *cctl_info,
+				struct rtw_hal_mac_ax_cctl_info *cctl_info_mask,
+				u16 macid);
+
+enum rtw_hal_status rtw_hal_bacam_cfg(struct hal_info_t *hal_info,
+				      struct mac_ax_bacam_info *ba_cam);
+
+enum rtw_hal_status
+rtw_hal_mac_port_init(struct hal_info_t *hal_info,
+                      struct rtw_wifi_role_t *wifi_role,
+                      struct rtw_wifi_role_link_t *rlink);
+
+enum rtw_hal_status
+rtw_hal_mac_port_deinit(struct hal_info_t *hal_i,
+				struct rtw_wifi_role_link_t *rlink);
+
+enum rtw_hal_status
+rtw_hal_mac_port_cfg(struct hal_info_t *hal_info,
+                     struct rtw_wifi_role_link_t *rlink,
+                     enum pcfg_type type,
+                     void *param);
+
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_add_entry(struct hal_info_t *hal_info,
+					struct rtw_phl_stainfo_t *sta);
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_change_entry(struct hal_info_t *hal_info,
+					struct rtw_phl_stainfo_t *sta,
+					enum phl_upd_mode mode,
+					bool is_connect);
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_del_entry(struct hal_info_t *hal_info,
+					struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_mac_add_key(struct hal_info_t *hal_info, u8 macid, u8 type, u8 ext_key,
+					u8 spp,	u8 keyid, u8 keytype, u8 *keybuf);
+enum rtw_hal_status
+rtw_hal_mac_delete_key(struct hal_info_t *hal_info, u8 macid, u8 type,
+						u8 ext_key, u8 spp, u8 keyid, u8 keytype);
+
+u32
+rtw_hal_mac_search_key_idx(struct hal_info_t *hal_info, u8 macid, u8 keyid, u8 keytype);
+
+enum rtw_hal_status
+rtw_hal_mac_set_fw_ul_fixinfo(void *hal,
+			struct rtw_phl_ax_ul_fixinfo *tbl);
+
+u32
+rtw_hal_mac_ser_reset_wdt_intr(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_ser_get_error_status(struct hal_info_t *hal_info, u32 *err);
+
+bool rtw_hal_mac_ser_chk_ser_l1(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_ser_set_error_status(struct hal_info_t *hal_info, enum RTW_PHL_SER_RCVY_STEP err);
+
+enum rtw_hal_status
+rtw_hal_mac_trigger_cmac_err(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_trigger_dmac_err(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_lv1_rcvy(struct hal_info_t *hal_info, enum rtw_phl_ser_lv1_recv_step step);
+
+enum rtw_hal_status rtw_hal_mac_ser_ctrl(struct hal_info_t *hal_info,
+			enum rtw_hal_ser_rsn rsn, bool en);
+
+enum rtw_hal_status
+rtw_hal_mac_dump_fw_rsvd_ple(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_set_bw(struct hal_info_t *hal_info,	u8 band_idx, u8 pri_ch,
+		u8 central_ch_seg0, u8 central_ch_seg1,	enum band_type band, enum channel_width bw);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_set_bf_entry(void *mac, u8 band, u8 macid,
+			    u8 bfee_idx, u8 txbf_idx, u16 buffer_idx);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_get_snd_sts(void *mac, u8 band, u8 bfee_idx);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_bfee_para_reg(void *mac, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_bfee_para_cctl(void *mac, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_mac_bfee_set_vht_gid(struct hal_info_t *hal,
+			     u8 band, struct rtw_phl_gid_pos_tbl *tbl);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_hw_snd_control(void *mac, u8 band, u8 hw_snd_ctrl);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_mu_sta_upd(void *mac, u8 macid, u8 bfmu_idx,
+			enum rtw_hal_protection_type prot_type,
+			enum rtw_hal_ack_resp_type resp_type, u8 mugrp_bm);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_mu_decision_para(void *mac, u32 mu_thold,
+	bool bypass_thold, bool bypass_tp);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_set_mu_fix_mode(
+	void *mac, u8 gid, enum rtw_hal_protection_type prot_type,
+	enum rtw_hal_ack_resp_type resp_type,
+	bool fix_mu, bool he, bool fix_resp, bool fix_prot);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_set_mu_table_whole(void *mac, void *hal_score_tbl);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_snd_fx_cmd(void *mac, u8 *cmd_buf);
+
+enum rtw_hal_status
+rtw_hal_mac_parse_c2h(void *hal, u8 *buf, u32 buf_len, void *c2h);
+
+enum rtw_hal_status
+rtw_hal_mac_ax_parse_ppdu_sts(void *hal, u8 mac_valid, u8 *buf, u16 buf_l,
+							  void *ppdu_sts, void *rx_mdata);
+enum rtw_hal_status
+rtw_hal_hdr_conv_cfg(struct hal_info_t *hal_info, u8 en_hdr_conv);
+enum rtw_hal_status
+rtw_hal_mac_enable_bb_rf(struct hal_info_t *hal_info, u8 enable);
+
+#ifdef RTW_PHL_BCN
+enum rtw_hal_status
+hal_mac_ax_config_beacon(struct hal_info_t *hal, struct rtw_bcn_entry *bcn_entry);
+
+enum rtw_hal_status
+hal_mac_ax_send_beacon(struct hal_info_t *hal, struct rtw_bcn_entry *bcn_entry);
+#endif
+
+enum rtw_hal_status
+rtw_hal_mac_ppdu_stat_cfg(struct hal_info_t *hal_info,
+			  struct hal_ppdu_sts_cfg *cfg);
+
+enum rtw_hal_status rtw_hal_mac_config_hw_mgnt_sec( struct hal_info_t *hal_info, u8 en);
+enum rtw_hal_status rtw_hal_mac_get_append_fcs(struct hal_info_t *hal_info, u8 *val);
+enum rtw_hal_status rtw_hal_mac_set_append_fcs(struct hal_info_t *hal_info, u8 enable);
+enum rtw_hal_status rtw_hal_mac_get_acpt_icv_err(struct hal_info_t *hal_info, u8 *val);
+
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+enum rtw_hal_status
+rtw_hal_mac_chan_info_cfg(struct hal_info_t *hal_info,
+				bool chinfo_en, u16 macid,
+				enum phl_chinfo_mode mode, u8 filter, u8 sg_size);
+
+#ifdef CONFIG_PHL_CSI_FW_TX_OFLD
+enum rtw_hal_status
+rtw_hal_mac_cfg_sensing_csi(struct hal_info_t *hal_info,
+							u16 macid, u8 en, u16 period, u8 retry_cnt,
+							u16 rate, u8 pkt_num, u8 *pkt_id);
+#endif
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+void rtw_hal_mac_dbg_status_dump(struct hal_info_t *hal, struct hal_mac_dbg_dump_cfg *cfg);
+
+#ifdef CONFIG_PHL_DFS
+enum rtw_hal_status
+rtw_hal_mac_dfs_rpt_cfg(struct hal_info_t *hal_info, struct hal_mac_dfs_rpt_cfg *conf);
+enum rtw_hal_status
+rtw_hal_mac_parse_dfs(struct hal_info_t *hal_info,
+			u8 *buf, u32 buf_len, struct mac_ax_dfs_rpt *dfs_rpt);
+#endif /*CONFIG_PHL_DFS*/
+u32
+rtw_hal_mac_lamode_trig(struct rtw_hal_com_t *hal_com, u8 trig);
+
+enum rtw_hal_status
+rtw_hal_mac_lamode_cfg_buf(struct rtw_hal_com_t *hal_com, u8 buf_sel,
+			   u32 *addr_start, u32 *addr_end);
+
+enum rtw_hal_status
+rtw_hal_mac_lamode_cfg(struct rtw_hal_com_t *hal_com, u8 func_en,
+		       u8 restart_en, u8 timeout_en, u8 timeout_val,
+		       u8 data_loss_imr, u8 la_tgr_tu_sel, u8 tgr_time_val);
+enum rtw_hal_status
+rtw_hal_mac_get_lamode_st(struct rtw_hal_com_t *hal_com, u8 *la_state,
+			  u16 *la_finish_addr, bool *la_round_up,
+			  bool *la_loss_data);
+
+void
+rtl_hal_dump_sec_cam_tbl(struct rtw_hal_com_t *hal_com);
+
+enum rtw_hal_status
+rtw_hal_mac_set_rxfltr_opt_by_mode(struct rtw_hal_com_t *hal_com, u8 band,
+				   enum rtw_rx_fltr_opt_mode mode);
+enum rtw_hal_status
+rtw_hal_mac_set_rxfltr_acpt_crc_err(struct rtw_hal_com_t *hal_com,
+				    u8 band, u8 enable);
+enum rtw_hal_status
+rtw_hal_mac_set_rxfltr_mpdu_size(struct rtw_hal_com_t *hal_com,
+				 u8 band, u16 size);
+enum rtw_hal_status
+rtw_hal_mac_set_rxfltr_by_type(struct rtw_hal_com_t *hal_com, u8 band,
+			       enum rtw_packet_type type, enum rtw_rxfltr_target target);
+enum rtw_hal_status
+rtw_hal_mac_set_rxfltr_by_subtype(struct rtw_hal_com_t *hal_com, u8 band,
+				  enum rtw_packet_type type, u8 subtype, enum rtw_rxfltr_target target);
+
+enum rtw_hal_status
+rtw_hal_mac_fw_log_cfg(struct rtw_hal_com_t *hal_com,
+			struct rtw_hal_fw_log_cfg *fl_cfg);
+
+enum rtw_hal_status rtw_hal_set_macid_pause(void *hinfo,
+                                            u16 macid,
+                                            bool pause);
+
+enum rtw_hal_status rtw_hal_mac_set_macid_pause_sleep(struct rtw_hal_com_t *hal_com,
+						      u16 macid, bool pause, bool sleep);
+
+void
+rtw_hal_mac_get_buffer_data(struct rtw_hal_com_t *hal_com, u32 strt_addr,
+			    u8 *buf, u32 len, u32 dbg_path);
+
+enum rtw_hal_status rtw_hal_mac_pkt_ofld(struct hal_info_t *hal, u8 *id, u8 op,
+						u8 *pkt, u16 *len);
+enum rtw_hal_status rtw_hal_mac_pkt_update_ids(struct hal_info_t *hal,
+						struct pkt_ofld_entry *entry);
+
+enum rtw_hal_status rtw_hal_mac_reset_pkt_ofld_state(struct hal_info_t *hal_info);
+
+
+enum rtw_hal_status
+rtw_hal_mac_set_edca(struct rtw_hal_com_t *hal_com,
+                     u8 band,
+                     u8 wmm,
+                     u8 ac,
+                     u32 param);
+enum rtw_hal_status
+rtw_hal_mac_get_ampdu_cfg(struct rtw_hal_com_t *hal_com,
+                          u8 band,
+                          struct mac_ax_ampdu_cfg *cfg);
+enum rtw_hal_status
+rtw_hal_mac_set_rty_lmt(struct rtw_hal_com_t *hal_com, u8 macid,
+	u8 rts_lmt_sel, u8 rts_lmt_val, u8 data_lmt_sel, u8 data_lmt_val);
+enum rtw_hal_status
+rtw_hal_mac_is_tx_mgnt_empty(struct hal_info_t *hal_info, u8 band, u8 *st);
+
+enum rtw_hal_status
+rtw_hal_mac_fw_dbg_dump(struct hal_info_t *hal_info);
+#ifdef CONFIG_HAL_MAC_DBG
+enum rtw_fw_status
+rtw_hal_mac_get_fw_status(struct hal_info_t *hal_info);
+#endif
+
+#ifdef CONFIG_POWER_SAVE
+enum rtw_hal_status
+rtw_hal_mac_ps_notify_wake(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_lps_adv_setting(struct hal_info_t *hal,
+	struct rtw_phl_lps_adv_cfg *cur, u8 macid, enum ps_adv_action act);
+
+enum rtw_hal_status
+rtw_hal_mac_req_pwr_state(struct hal_info_t *hal_info, u8 pwr_state);
+enum rtw_hal_status
+rtw_hal_mac_chk_pwr_state(struct hal_info_t *hal_info, u8 pwr_state, u32 *mac_sts);
+enum rtw_hal_status
+rtw_hal_mac_lps_cfg(struct hal_info_t *hal_info,
+			struct rtw_hal_lps_info *lps_info);
+enum rtw_hal_status
+rtw_hal_mac_lps_chk_leave(struct hal_info_t *hal_info, u16 macid, u32 *mac_sts);
+enum rtw_hal_status
+rtw_hal_mac_ips_cfg(struct hal_info_t *hal_info, u16 macid, bool enable);
+enum rtw_hal_status
+rtw_hal_mac_ips_chk_leave(struct hal_info_t *hal_info, u16 macid);
+
+enum rtw_hal_status
+rtw_hal_mac_lps_chk_access(struct hal_info_t *hal_info, u32 offset);
+
+enum rtw_hal_status
+rtw_hal_mac_lps_bcn_tracking_cfg(struct hal_info_t *hal,
+	struct rtw_bcn_tracking_cfg *cfg, struct rtw_bcn_tracking_cfg *cfg_cur);
+
+#endif /* CONFIG_POWER_SAVE */
+
+enum rtw_hal_status
+rtw_hal_mac_tbtt_tuning_cfg(struct hal_info_t *hal, enum phl_band_idx band,
+				u8 port, u32 tbtt);
+
+enum rtw_hal_status
+hal_mac_ax_send_fw_snd(struct hal_info_t *hal_info,
+		       struct hal_ax_fwcmd_snd *hal_cmd);
+
+enum rtw_hal_status
+rtw_hal_mac_tx_mode_sel(struct hal_info_t *hal_info, u8 fw_tx, u8 txop_wmm_en_bm);
+enum rtw_hal_status
+rtw_hal_mac_get_tx_cnt(struct hal_info_t *hal, enum phl_band_idx bidx, u8 sel);
+
+enum rtw_hal_status
+rtw_hal_mac_get_sel_tx_cnt(struct hal_info_t *hal, enum phl_band_idx bidx, void *out_tx_cnt);
+
+enum rtw_hal_status
+rtw_hal_mac_get_rx_cnt(struct hal_info_t *hal_info, u8 cur_phy_idx, u8 type_idx, u32 *ret_value);
+
+enum rtw_hal_status
+rtw_hal_mac_get_rx_cnt_by_idx(struct hal_info_t *hal_info, u8 cur_phy_idx,
+			      u8 cnt_idx, u16 *ret_value);
+enum rtw_hal_status
+rtw_hal_mac_set_reset_rx_cnt(struct hal_info_t *hal_info, u8 cur_phy_idx);
+
+#ifdef CONFIG_PHL_TWT
+enum rtw_hal_status
+rtw_hal_mac_twt_info_update(void *hal,
+                            struct rtw_phl_twt_info twt_info,
+                            struct rtw_wifi_role_link_t *rlink,
+                            u8 action);
+
+enum rtw_hal_status
+rtw_hal_mac_twt_sta_update(void *hal, u8 macid, u8 twt_id, u8 action);
+
+enum rtw_hal_status
+rtw_hal_mac_twt_sta_announce(void *hal, u8 macid);
+#endif /* CONFIG_PHL_TWT */
+
+enum rtw_hal_status
+rtw_hal_mac_ax_bfee_set_csi_rrsc(void *mac, u8 band, u32 rrsc);
+
+u32 rtw_hal_mac_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg);
+void rtw_hal_mac_upd_c2h(void *hal, struct rtw_c2h_info *c2h, u32 evt_id, struct c2h_evt_msg *c2h_msg);
+
+enum rtw_hal_status
+rtw_hal_mac_set_mu_edca(struct rtw_hal_com_t *hal_com, u8 band, u8 ac,
+	u16 timer, u8 cw_min, u8 cw_max, u8 aifs);
+enum rtw_hal_status
+rtw_hal_mac_set_mu_edca_ctrl(struct rtw_hal_com_t *hal_com,
+	u8 band, u8 wmm, u8 set);
+
+enum rtw_hal_status rtw_hal_mac_led_set_ctrl_mode(struct hal_info_t *hal_info,
+						  enum mac_ax_led_mode mode,
+						  u8 led_id);
+enum rtw_hal_status rtw_hal_mac_led_ctrl(struct hal_info_t *hal_info, u8 high,
+					 u8 led_id);
+
+enum rtw_hal_status rtw_hal_mac_sw_gpio_ctrl(struct hal_info_t *hal_info, u8 high,
+					 u8 gpio);
+enum rtw_hal_status rtw_hal_mac_set_sw_gpio_mode(struct hal_info_t *hal_info, enum rtw_gpio_mode mode,
+					 u8 gpio);
+
+enum rtw_hal_status rtw_hal_mac_get_wl_dis_val(struct hal_info_t *hal_info, u8 *val);
+
+enum rtw_hal_status
+rtw_hal_mac_pcie_trx_mit(struct hal_info_t *hal_info,
+			 struct mac_ax_pcie_trx_mitigation *mit_info);
+
+enum rtw_hal_status
+rtw_hal_mac_tsf_sync(struct hal_info_t *hal_info,
+						u8 from_port, u8 to_port, enum phl_band_idx band,
+						s32 sync_offset_tu, enum hal_tsf_sync_act action);
+
+enum rtw_hal_status
+rtw_hal_mac_get_sec_cam(struct hal_info_t *hal_info, u16 num, u8 *buf, u16 size);
+enum rtw_hal_status
+rtw_hal_mac_get_addr_cam(struct hal_info_t *hal_info, u16 num, u8 *buf, u16 size);
+
+enum rtw_hal_status rtw_hal_mac_get_tsf(struct hal_info_t *hal,
+		enum phl_band_idx band, u8 port, u32 *tsf_h, u32 *tsf_l);
+
+enum rtw_hal_status rtw_hal_mac_clear_rwptr(struct hal_info_t *hal);
+
+enum rtw_hal_status rtw_hal_mac_cfg_txhci(struct hal_info_t *hal,u8 en);
+
+enum rtw_hal_status rtw_hal_mac_cfg_rxhci(struct hal_info_t *hal,u8 en);
+
+enum rtw_hal_status rtw_hal_mac_get_wcpu_cap(struct rtw_phl_com_t *phl_com,
+					     struct hal_info_t *hal_info);
+#ifdef CONFIG_MCC_SUPPORT
+enum rtw_hal_status rtw_hal_mac_add_mcc(struct hal_info_t *hal,
+					struct rtw_phl_mcc_role *mcc_role);
+
+enum rtw_hal_status rtw_hal_mac_start_mcc(struct hal_info_t *hal,
+		u8 group, u8 macid, u32 tsf_high, u32 tsf_low, u8 btc_in_group,
+		u8 old_group_action, u8 old_group);
+
+enum rtw_hal_status rtw_hal_mac_stop_mcc(struct hal_info_t *hal, u8 group,
+					u8 macid);
+
+enum rtw_hal_status rtw_hal_mac_reset_mcc_group(struct hal_info_t *hal, u8 group);
+
+enum rtw_hal_status rtw_hal_mac_del_mcc_group(struct hal_info_t *hal, u8 group);
+
+enum rtw_hal_status rtw_hal_mac_mcc_request_tsf(struct hal_info_t *hal,
+					u8 group, u8 macid_x, u8 macid_y);
+
+enum rtw_hal_status rtw_hal_mac_mcc_macid_bitmap(struct hal_info_t *hal,
+					u8 group, u8 macid, u8 *bitmap, u8 len);
+
+enum rtw_hal_status rtw_hal_mac_mcc_sync_enable(struct hal_info_t *hal,
+				u8 group, u8 source, u8 target, u8 offset);
+
+enum rtw_hal_status rtw_hal_mac_set_duration(struct hal_info_t *hal,
+					struct rtw_phl_mcc_en_info *en_info,
+					struct rtw_phl_mcc_bt_info *bt_info);
+
+enum rtw_hal_status rtw_hal_mac_get_mcc_tsf_rpt(struct hal_info_t *hal,
+					u8 group, u32 *tsf_x_h, u32 *tsf_x_l,
+					u32 *tsf_y_h, u32 *tsf_y_l);
+
+enum rtw_hal_status rtw_hal_mac_get_mcc_status_rpt(struct hal_info_t *hal,
+				u8 group, u8 *status, u32 *tsf_h, u32 *tsf_l);
+
+enum rtw_hal_status rtw_hal_mac_get_mcc_group(struct hal_info_t *hal, u8 *group);
+#endif /* CONFIG_MCC_SUPPORT */
+
+#ifdef CONFIG_PHL_CHSWOFLD
+enum rtw_hal_status rtw_hal_mac_ch_switch_ofld(struct hal_info_t *hal_info,
+					       u8 band_idx, u8 pri_ch, u8 central_ch,
+					       enum band_type band, enum channel_width bw,
+					       bool reload_rf);
+#endif
+
+#ifdef CONFIG_PHL_BCN_ERLY_RPT
+enum rtw_hal_status rtw_hal_mac_cfg_bcn_early_rpt(struct hal_info_t *hal_info,
+						  u8 band, u8 port, u8 en);
+#endif
+
+void rtw_hal_mac_notification(struct hal_info_t *hal_info,
+                              enum phl_msg_evt_id event,
+                              u8 band);
+
+void rtw_hal_mac_cmd_notification(struct hal_info_t *hal_info,
+                              void *hal_cmd,
+                              u8 band);
+
+enum rtw_hal_status rtw_hal_mac_set_hw_rts_th(struct hal_info_t *hal, u8 band,
+					      u16 time_th, u16 len_th);
+
+enum rtw_hal_status rtw_hal_mac_set_dfs_tb_ctrl(struct hal_info_t *hal, u8 set);
+
+enum rtw_hal_status
+rtw_hal_mac_trigger_fw_conflict(struct hal_info_t *hal_com, u32 addr, u8 vol);
+
+u8 rtw_hal_mac_get_efuse_ver_len(struct rtw_hal_com_t *hal_com);
+
+#ifdef CONFIG_PHL_P2PPS
+
+enum rtw_hal_status rtw_hal_mac_p2p_macid_ctrl(struct hal_info_t *hal,
+	bool join,
+	u16 main_macid,
+	u8 *bitmap,
+	u32 bitmap_len
+	);
+
+enum rtw_hal_status rtw_hal_mac_noa_init(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid);
+
+enum rtw_hal_status rtw_hal_mac_noa_update(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid);
+
+enum rtw_hal_status rtw_hal_mac_noa_remove(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid);
+
+enum rtw_hal_status rtw_hal_mac_noa_terminate(struct hal_info_t *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid);
+
+enum rtw_hal_status rtw_hal_mac_tsf32_tog_enable(struct hal_info_t *hal,
+	u8 hw_band,
+	u8 port,
+	u16 early);
+
+enum rtw_hal_status rtw_hal_mac_tsf32_tog_disable(struct hal_info_t *hal,
+	u8 hw_band,
+	u8 port);
+
+enum rtw_hal_status rtw_hal_mac_get_tsf32_tog_rpt(struct hal_info_t *hal,
+	struct rtw_phl_tsf32_tog_rpt *rpt);
+
+#endif
+enum rtw_hal_status
+rtw_hal_mac_addr_cam_set_aid(struct hal_info_t *hal_info,
+			     struct rtw_phl_stainfo_t *sta,
+			     u16 aid);
+
+enum rtw_hal_status
+rtw_hal_mac_set_tx_lifetime(struct hal_info_t *hal, enum phl_band_idx band,
+			    enum phl_lifetime_queue que, bool en, u16 val);
+
+enum rtw_hal_status
+rtw_hal_mac_set_gt3(struct hal_info_t *hal, u8 en, u32 timeout);
+
+#ifdef CONFIG_RTW_MULTI_DEV_MULTI_BAND
+enum rtw_hal_status
+rtw_hal_mac_set_xtal_aac(struct rtw_hal_com_t *hal_com, u8 value);
+#endif /* CONFIG_RTW_MULTI_DEV_MULTI_BAND */
+
+enum rtw_hal_status
+rtw_hal_mac_patch_rx_rate(struct hal_info_t *hal_info, struct rtw_r_meta_data *mdata);
+
+enum rtw_hal_status
+rtw_hal_mac_drvinfo_cfg(struct hal_info_t *hal_info,
+			bool enable, enum phl_band_idx hw_band);
+
+enum rtw_hal_status
+rtw_hal_mac_set_tx_duty(struct hal_info_t *hal_info,
+			u16 pause_interval,
+			u16 tx_interval);
+
+enum rtw_hal_status
+rtw_hal_mac_stop_tx_duty(struct hal_info_t *hal_info);
+u16 rtw_hal_mac_query_ampdu_num(struct hal_info_t *hal, u8 band);
+
+void rtw_hal_mac_init_txagg_num(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_mac_pwr_switch(struct hal_info_t *hal_info, bool on);
+
+enum rtw_hal_status
+rtw_hal_mac_get_freerun_cnt(struct rtw_hal_com_t *hal_com, enum phl_band_idx hw_band,
+			    u32 *freerun_cnt_h, u32 *freerun_cnt_l);
+
+enum rtw_hal_status
+rtw_hal_mac_reset_freerun_cnt(struct rtw_hal_com_t *hal_com, enum phl_band_idx hw_band);
+
+enum rtw_hal_status
+rtw_hal_mac_cfg_hw_cts2slef(struct hal_info_t *hal_info,
+			 struct mac_ax_cts2self_cfg *cts_cfg);
+
+enum rtw_hal_status
+rtw_hal_mac_set_usr_edca(struct hal_info_t *hal,
+                         u32 idx,
+                         u32 enable,
+                         u32 band,
+                         u32 wmm,
+                         struct rtw_edca_param *aggr,
+                         struct rtw_edca_param *mod);
+
+enum rtw_hal_status
+rtw_hal_mac_get_edca(struct rtw_hal_com_t *hal_com,
+                     u8 band,
+                     u8 wmm,
+                     struct rtw_edca_param *edca_param);
+
+enum rtw_hal_status
+rtw_hal_mac_txagg_timeout_thrhold(struct hal_info_t *hal_info);
+enum rtw_hal_status
+rtw_hal_mac_fw_general_io_test(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_mac_set_rsp_stat_rts_chk_en(struct hal_info_t *hal,
+							enum phl_band_idx band,
+							bool en);
+
+enum rtw_hal_status
+rtw_hal_mac_sr_update(struct rtw_hal_com_t *hal_com,
+                      void *sr_info,
+                      u8 hw_band);
+
+enum rtw_hal_status
+rtw_hal_mac_set_aspm_test(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_mac_usr_frame_to_act(
+    struct hal_info_t *hal_info, struct mac_ax_usr_frame_to_act_cfg *act_cfg);
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+enum rtw_hal_status
+rtw_hal_mac_set_usr_tx_rpt_cfg(void *hal, struct rtw_phl_usr_tx_rpt_cfg *param);
+
+enum rtw_hal_status
+rtw_hal_mac_set_ch_busy_stat_cfg(struct hal_info_t *hal_info,
+                                 struct mac_ax_ch_busy_cnt_cfg *mac_param);
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+#endif /*_HAL_API_MAC_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_rf.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_rf.c
new file mode 100644
index 000000000000..85a388a9d500
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_rf.c
@@ -0,0 +1,1929 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_API_RF_C_
+#include "hal_headers.h"
+#ifdef USE_TRUE_PHY
+#include "phy/rf/halrf_api.h"
+#include "phy/rf/halrf_export_fun.h"
+
+#ifdef CONFIG_PHL_DIAGNOSE
+void rtw_hal_rf_diagnostic_event(struct rtw_hal_com_t *hal, u8 type,
+		u8 level, u8 version, u8 *buf, u32 len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal->hal_priv;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+
+	rtw_phl_send_diag_hub_msg(phl_com, PHL_DIAG_EVT_RF,
+				       type, level, version, buf, len);
+}
+
+enum rtw_hal_status rtw_hal_rf_query_diag_err_code(
+	void *hal, u32 *err_code)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return halrf_query_rfdz_err_code(hal_info->rf, err_code);
+}
+
+enum rtw_hal_status rtw_hal_rf_query_diag_info_len(u32 *len)
+{
+	if (!len)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return halrf_query_rf_diag_buf_len(len);
+}
+
+enum rtw_hal_status rtw_hal_rf_query_diag_info(void *hal,
+	struct rtw_phl_diag_rf_info *info)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct halrf_outsrc_rpt_v1 *rpt = NULL;
+
+	if (!info)
+		return hal_status;
+
+	hal_status = halrf_query_rf_diag_buf(hal_info->rf,
+		&info->ver, info->buf, info->len);
+	if (hal_status == RTW_HAL_STATUS_SUCCESS) {
+		/* check framework version */
+		if (info->ver != 1) {
+			PHL_ERR("%s: fail, incompatible framework ver(%d)\n",
+				__func__, info->ver);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+		rpt = (struct halrf_outsrc_rpt_v1 *)info->buf;
+		/* copy err code for upper layer */
+		info->err_code.iqk = rpt->err.iqk;
+		info->err_code.dpk = rpt->err.dpk;
+		info->err_code.dack = rpt->err.dack;
+		info->err_code.rxdck = rpt->err.rxdck;
+		info->err_code.txgapk = rpt->err.txgapk;
+		info->err_code.tssi = rpt->err.tssi;
+		/* assign buf addr(point to raw data) & specify length/ver */
+		info->iqk_rpt.ver = rpt->iqk_rpt_ver;
+		info->iqk_rpt.buf = (u8 *)&rpt->iqk_rpt;
+		info->iqk_rpt.len = sizeof(struct halrf_ex_iqk_rpt_v1);
+		info->dpk_rpt.ver = rpt->dpk_rpt_ver;
+		info->dpk_rpt.buf = (u8 *)&rpt->dpk_rpt;
+		info->dpk_rpt.len = sizeof(struct halrf_ex_dpk_rpt_v1);
+		info->rt_rpt.ver = rpt->rt_rpt_ver;
+		info->rt_rpt.buf = (u8 *)&rpt->rt_rpt;
+		info->rt_rpt.len = sizeof(struct halrf_ex_rt_rpt_v1);
+	}
+
+	return hal_status;
+}
+#endif
+
+enum rtw_hal_status
+rtw_hal_rf_init(struct rtw_phl_com_t *phl_com,
+			struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = halrf_init(phl_com, hal_com, &(hal_info->rf));
+
+	if ((hal_status != RTW_HAL_STATUS_SUCCESS) ||
+		(hal_info->rf == NULL)) {
+		PHL_ERR("[PHL] rtw_halrf_init failed status(%d), hal_info->rf(%p)\n",
+			hal_status, hal_info->rf);
+	}
+
+	return hal_status;
+}
+
+void rtw_hal_rf_deinit(struct rtw_phl_com_t *phl_com,
+			struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	halrf_deinit(phl_com, hal_com, hal_info->rf);
+}
+
+void rtw_hal_init_rf_reg(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct rtw_phl_evt_ops *ops = &phl_com->evt_ops;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 phy_idx = 0;
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	halrf_config_rf_parameter(hal_info->rf, phy_idx);
+
+	if (ops->tx_power_tbl_loaded)
+		ops->tx_power_tbl_loaded(phl_com->drv_priv, true, true);
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_INIT_RF_REG);
+#endif /* DBG_MONITOR_TIME */
+
+}
+
+void rtw_hal_rf_ic_cfg_init(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	halrf_ic_cfg_init(hal_info->rf);
+}
+void rtw_hal_rf_dm_init(struct hal_info_t *hal_info)
+{
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	halrf_dm_init(hal_info->rf);
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_RF_DM_INIT);
+#endif /* DBG_MONITOR_TIME */
+}
+
+
+enum rtw_hal_status
+rtw_hal_rf_get_efuse_info(struct rtw_hal_com_t *hal_com,
+	u8 *efuse_map, enum rtw_efuse_info info_type, void *value,
+	u8 size, u8 map_valid)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	PHL_INFO("%s\n", __FUNCTION__);
+
+	if(halrf_get_efuse_info(hal_info->rf, efuse_map, info_type, value, size,
+							map_valid))
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+void
+rtw_hal_rf_set_power_table_switch(struct rtw_hal_com_t *hal_com,
+				enum phl_phy_idx phy_idx,
+				u8 pwrbyrate_type, u8 pwrlmt_type)
+{
+/*	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	halrf_set_power_table_switch(hal_info->rf, phy_idx,
+				pwrbyrate_type, pwrlmt_type);*/
+}
+
+int rtw_hal_rf_get_predef_pw_lmt_regu_type_from_str(
+	enum band_type band, const char *str)
+{
+	return halrf_get_predef_pw_lmt_regu_type_of_band_from_str(band, str);
+}
+
+const char * const *rtw_hal_rf_get_predef_pw_lmt_regu_type_str_array(
+	enum band_type band, u8 *num)
+{
+	return halrf_get_predef_pw_lmt_regu_type_of_band_str_array(band, num);
+}
+
+u8 rtw_hal_rf_get_pw_lmt_regu_type(struct hal_info_t *hal_info, enum band_type band)
+{
+	return halrf_get_regulation_info(hal_info->rf, band);
+}
+
+const char *rtw_hal_rf_get_pw_lmt_regu_type_str(struct hal_info_t *hal_info, enum band_type band)
+{
+	return halrf_get_pw_lmt_regu_type_str(hal_info->rf, band);
+}
+
+bool rtw_hal_rf_pw_lmt_regu_tbl_exist(struct hal_info_t *hal_info, enum band_type band, u8 regu)
+{
+	return halrf_reg_tbl_exist(hal_info->rf, band, regu);
+}
+
+void rtw_hal_rf_set_pw_lmt_regu_table_exist(struct hal_info_t *hal_info, u8 band, u8 reg)
+{
+	halrf_set_reg_tbl_exist(hal_info->rf, band, reg);
+}
+
+int rtw_hal_rf_file_regd_ext_search(struct hal_info_t *hal_info
+	, enum band_type band, u16 domain_code, const char *country)
+{
+	int aidx_match;
+
+	halrf_file_regd_ext_of_band_search(hal_info->rf
+		, band, domain_code, (char *)country, PWR_LMT_6G_MAX, &aidx_match);
+
+	return aidx_match;
+}
+
+void rtw_hal_rf_auto_pw_lmt_regu(struct hal_info_t *hal_info)
+{
+	halrf_force_regulation(hal_info->rf, false, NULL, 0, NULL, 0, NULL, 0);
+}
+
+void rtw_hal_rf_force_pw_lmt_regu(struct hal_info_t *hal_info,
+	u8 regu_2g[], u8 regu_2g_len, u8 regu_5g[], u8 regu_5g_len, u8 regu_6g[], u8 regu_6g_len)
+{
+	halrf_force_regulation(hal_info->rf, true
+		, regu_2g, regu_2g_len, regu_5g, regu_5g_len, regu_6g, regu_6g_len);
+}
+
+enum rtw_hal_status rtw_hal_rf_read_pwr_table(
+	struct rtw_hal_com_t *hal_com, u8 rf_path, u16 rate,
+	u8 bandwidth, u8 channel, u8 offset, u8 dcm,
+	u8 beamforming, s16 *get_item)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("[MP HAL API] %s \n", __FUNCTION__);
+
+	*get_item = halrf_get_power(hal_info->rf, rf_path, rate, dcm,offset, bandwidth, beamforming, channel);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_rf_wlan_tx_power_control(struct rtw_hal_com_t *hal_com,
+	enum phl_phy_idx phy, enum phl_pwr_ctrl pwr_ctrl_idx, u32 tx_power_val, bool enable)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("%s\n", __FUNCTION__);
+
+	halrf_wlan_tx_power_control(hal_info->rf, phy, pwr_ctrl_idx, tx_power_val, enable);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_wl_tx_power_control(struct rtw_hal_com_t *hal_com,
+	u32 tx_power_val)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	PHL_INFO("%s\n", __FUNCTION__);
+
+	halrf_wl_tx_power_control(hal_info->rf, tx_power_val);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_pwrtrack(struct hal_info_t *hal_info,
+                                            u8 *txpwr_track_status,
+                                            u8 phy_idx)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s , phy_idx = %d\n", __FUNCTION__, phy_idx);
+
+	*txpwr_track_status = halrf_get_power_track_phy(hal_info->rf, (enum phl_phy_idx)phy_idx);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_pwrtrack(struct hal_info_t *hal_info, u8 phy_idx, u8 txpwr_track_status)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s \n", __FUNCTION__);
+
+	halrf_set_power_track(hal_info->rf, phy_idx, txpwr_track_status);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_thermal(struct hal_info_t *hal_info, u8 rf_path, u8 *thermal)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s \n", __FUNCTION__);
+
+	*thermal = halrf_get_thermal(hal_info->rf, rf_path);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_tssi(struct hal_info_t *hal_info, u8 phy_idx, u8 rf_path, u32 tssi_de)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s \n", __FUNCTION__);
+
+	halrf_set_tssi_de_for_tx_verify(hal_info->rf, phy_idx, tssi_de, rf_path);
+
+	return ret;
+
+}
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_offset(struct hal_info_t *hal_info, u8 phy_idx, u32 tssi_de_offset, u8 rf_path)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s \n", __FUNCTION__);
+
+	halrf_set_tssi_de_offset(hal_info->rf, phy_idx, tssi_de_offset, rf_path);
+
+	return ret;
+
+}
+
+enum rtw_hal_status rtw_hal_rf_get_tssi(struct hal_info_t *hal_info, u8 phy_idx, u8 rf_path, u32 *tssi_de)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s \n", __FUNCTION__);
+
+	*tssi_de = halrf_get_tssi_de(hal_info->rf, phy_idx, rf_path);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_online_tssi_de(struct hal_info_t *hal_info, u8 phy_idx, u8 rf_path,s32 dbm, s32 pout, s32 *tssi_de)
+{
+	int ret = RTW_HAL_STATUS_SUCCESS;
+
+	PHL_INFO("[MP HAL API] %s \n", __FUNCTION__);
+
+	*tssi_de = halrf_get_online_tssi_de(hal_info->rf, phy_idx, rf_path, dbm, pout);
+
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_continuous_tx(struct hal_info_t *hal_info)
+{
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_singletone_tx(struct hal_info_t *hal_info,
+						u8 is_on,enum rf_path path)
+{
+	PHL_INFO("%s: enable = %d path = %x\n", __FUNCTION__, is_on, path);
+	halrf_lo_test(hal_info->rf, is_on, path);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_suppression_tx(struct hal_info_t *hal_info)
+{
+	PHL_INFO("%s\n", __FUNCTION__);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status rtw_hal_rf_ctrl_dbcc(struct rtw_hal_com_t *hal_com,
+	bool dbcc_en)
+{
+	/*need replace with rf api*/
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	if (halrf_set_dbcc(hal_info->rf, dbcc_en))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hal_status;
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+#define DBG_RFK_TIME
+enum rtw_hal_status rtw_hal_rf_chl_rfk_trigger(struct rtw_hal_com_t *hal_com,
+                                               u8 phy_idx,
+                                               enum rfk_tri_type rt_type)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	#ifdef DBG_RFK_TIME
+	u32 iqk_start = _os_get_cur_time_ms();
+	#endif
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+
+	PHL_INFO("[DBG_RFK]%s: phy_idx(%d), rt_type(%d)\n", __func__,
+		phy_idx, rt_type);
+
+	hal_status = halrf_chl_rfk_trigger(hal_info->rf, phy_idx, rt_type);
+
+	#ifdef DBG_RFK_TIME
+	PHL_INFO("[DBG_RFK]%s: RFK take %d (ms)\n", __func__,
+		phl_get_passing_time_ms(iqk_start));
+	#endif
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_HAL_RFK);
+#endif /* DBG_DBCC_MONITOR_TIME */
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_dack_trigger(struct hal_info_t *hal_info,
+						u8 force)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+#if 0
+	hal_status = halrf_dack_trigger(hal_info->rf, force);
+#else
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_iqk_trigger(struct hal_info_t *hal_info,
+						u8 phy_idx, u8 force)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = halrf_iqk_trigger(hal_info->rf, phy_idx, force);
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_lck_trigger(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+#if 0
+	hal_status = halrf_lck_trigger(hal_info->rf);
+#else
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_dpk_trigger(struct hal_info_t *hal_info,
+						u8 phy_idx, u8 force)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = halrf_dpk_trigger(hal_info->rf, phy_idx, force);
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_tssi_trigger(struct hal_info_t *hal_info,
+				u8 phy_idx)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+#if 0
+	hal_status = halrf_tssi_trigger(hal_info->rf, phy_idx);
+#else
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_gapk_trigger(struct hal_info_t *hal_info,
+						u8 phy_idx, u8 force)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+#if 0
+	hal_status = halrf_gapk_trigger(hal_info->rf, phy_idx, force);
+#else
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dack(struct hal_info_t *hal_info,
+				u8 enable)
+{
+#if 0
+	halrf_dack_onoff(hal_info->rf, enable);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_capability_iqk(struct hal_info_t *hal_info,
+				u8 enable)
+{
+#if 0
+	halrf_iqk_onoff(hal_info->rf, enable);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dpk(struct hal_info_t *hal_info,
+						u8 enable)
+{
+#if 0
+	halrf_dpk_onoff(hal_info->rf, enable);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dpk_track(struct hal_info_t *hal_info,
+						u8 enable)
+{
+#if 0
+	halrf_dpk_track_onoff(hal_info->rf, enable);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_tssi(struct hal_info_t *hal_info,
+				u8 enable)
+{
+#if 0
+	halrf_tssi_onoff(hal_info->rf, enable);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_capability_gapk(struct hal_info_t *hal_info,
+						u8 enable)
+{
+#if 0
+	halrf_gapk_onoff(hal_info->rf, enable);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dack(struct hal_info_t *hal_info,
+						u8 *enable)
+{
+#if 0
+	*enable = halrf_get_dack_onoff(hal_info->rf);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_iqk(struct hal_info_t *hal_info,
+				u8 *enable)
+{
+#if 0
+	*enable = halrf_get_iqk_onoff(hal_info->rf);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dpk(struct hal_info_t *hal_info,
+					u8 *enable)
+{
+#if 0
+	*enable = halrf_get_dpk_onoff(hal_info->rf);
+#endif
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dpk_track(struct hal_info_t *hal_info,
+				u8 *enable)
+{
+#if 0
+	*enable = halrf_get_dpk_track_onoff(hal_info->rf);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_tssi(struct hal_info_t *hal_info,
+				u8 *enable)
+{
+#if 0
+	*enable = halrf_get_tssi_onoff(hal_info->rf);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_gapk(struct hal_info_t *hal_info,
+					u8 *enable)
+{
+#if 0
+	*enable = halrf_get_gapk_onoff(hal_info->rf);
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_get_tssi_de_value(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+#if 0
+	hal_status = halrf_get_tssi_de_value(hal_info->rf);
+#else
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_de_tx_verify(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+#if 0
+	hal_status = halrf_set_tssi_de_tx_verify(hal_info->rf);
+#else
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_txpwr_final_abs(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+#if 0
+	hal_status = halrf_get_txpwr_final_abs(hal_info->rf);
+#else
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+bool
+rtw_hal_rf_proc_cmd(struct hal_info_t *hal_info,
+					struct rtw_proc_cmd *incmd,
+					char *output, u32 out_len)
+{
+
+	if(incmd->in_type == RTW_ARG_TYPE_BUF)
+		halrf_cmd(hal_info->rf, incmd->in.buf, output, out_len);
+
+	else if(incmd->in_type == RTW_ARG_TYPE_ARRAY){
+
+		halrf_cmd_parser(hal_info->rf, incmd->in.vector,
+					incmd->in_cnt_len, output, out_len);
+	}
+
+	return true;
+}
+
+enum rtw_hal_status rtw_hal_rf_watchdog(struct hal_info_t *hal_info)
+{
+
+	halrf_watchdog(hal_info->rf);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_power(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+					enum phl_pwr_table pwr_table)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (phy >= HW_PHY_MAX)
+		goto exit;
+
+	if (halrf_set_power(hal_info->rf, phy, pwr_table))
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_power_constraint(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+					u16 mb)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if (phy >= HW_PHY_MAX)
+		goto exit;
+
+	/* here we choose to have software configuration only */
+	halrf_set_power_constraint(hal_info->rf, phy, mb, false);
+
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_update_tas_def_setting(void *hal, u32 tas_config)
+{
+#ifdef CONFIG_TAS_SUPPORT
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	halrf_update_tas_def_setting(hal_info->rf, tas_config);
+#endif
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_tas_en(void *hal, u8 en)
+{
+#ifdef CONFIG_TAS_SUPPORT
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	if (en)
+		halrf_tas_start(hal_info->rf);
+	else
+		halrf_tas_stop(hal_info->rf);
+#endif
+	return RTW_HAL_STATUS_SUCCESS;
+}
+enum rtw_hal_status rtw_hal_rf_set_gain_offset(struct hal_info_t *hal_info, u8 cur_phy_idx,
+						s8 offset, u8 rf_path)
+{
+
+	enum rtw_hal_status ret=RTW_HAL_STATUS_SUCCESS;
+	PHL_INFO("[MP HAL API]%s\n", __FUNCTION__);
+
+	halrf_set_rx_gain_offset_for_rx_verify(hal_info->rf, cur_phy_idx, offset, rf_path);
+
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_rf_trigger_dpk_tracking(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = halrf_dpk_tracking(hal_info->rf);
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("[MP HAL API] %s failed status(%d)\n",__FUNCTION__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_get_default_rfe_type(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	hal_com->dev_hw_cap.rfe_type = halrf_get_default_rfe_type(hal_info->rf);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_get_default_xtal(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	hal_com->dev_hw_cap.xcap = halrf_get_default_xtal(hal_info->rf);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_avg(struct hal_info_t *hal_info, u8 cur_phy_idx,
+						s32 xdbm)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	halrf_set_tssi_avg_mp(hal_info->rf, cur_phy_idx, xdbm);
+
+	return hal_status;
+}
+
+void rtw_hal_rf_do_tssi_scan(struct hal_info_t *hal_info, u8 cur_phy_idx)
+{
+	halrf_do_tssi_scan(hal_info->rf, cur_phy_idx);
+}
+
+void
+rtw_hal_rf_update_ext_pwr_lmt_table(struct hal_info_t *hal_info,
+					      enum phl_phy_idx phy)
+{
+	halrf_power_limit_set_ext_pwr_limit_table(hal_info->rf, phy);
+	halrf_power_limit_set_ext_pwr_limit_ru_table(hal_info->rf, phy);
+}
+
+u8
+rtw_hal_rf_get_tx_tbl_to_pwr_times(struct hal_info_t *hal_info)
+{
+	return halrf_get_tx_tbl_to_tx_pwr_times(hal_info->rf);
+}
+
+void
+rtw_hal_rf_set_tx_pwr_comp(struct hal_info_t *hal_info,	enum phl_phy_idx phy,
+			   struct rtw_phl_regu_dyn_ant_gain *dyn_ag)
+{
+	halrf_set_dynamic_ant_gain(hal_info->rf, phy, dyn_ag);
+}
+
+enum rtw_hal_status
+rtw_hal_rf_config_radio_to_fw(struct hal_info_t *hal_info)
+{
+	halrf_config_radio_to_fw(hal_info->rf);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool
+rtw_hal_rf_check_efuse_data(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	return halrf_tssi_check_efuse_data(hal_info->rf, phy_idx);
+}
+
+void
+rtw_hal_rf_disconnect_notify(void *hal, struct rtw_chan_def *chandef)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	halrf_disconnect_notify(hal_info->rf, chandef);
+}
+
+bool
+rtw_hal_rf_check_mcc_ch(void *hal, struct rtw_chan_def *chandef)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	return halrf_check_mcc_ch(hal_info->rf, chandef);
+}
+
+void
+rtw_hal_rf_dpk_switch(void *hal, bool enable)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_rf_dpk_switch(): enable(%d)\n",
+		enable);
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_ser_ctrl(hal, HAL_SER_RSN_RFK, false))
+		return;
+	halrf_dpk_switch(hal_info->rf, enable);
+	rtw_hal_mac_ser_ctrl(hal, HAL_SER_RSN_RFK, true);
+}
+
+void
+rtw_hal_rf_tssi_config(void *hal, enum phl_phy_idx phy_idx, bool enable)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_rf_tssi_config(): enable(%d), phy_idx(%d)\n",
+		enable, phy_idx);
+	if (enable)
+		halrf_tssi_enable(hal_info->rf, phy_idx);
+	else
+		halrf_tssi_disable(hal_info->rf, phy_idx);
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_ch_bw(struct rtw_hal_com_t *hal_com,
+                     enum phl_phy_idx phy,
+                     u8 center_ch,
+                     enum band_type band,
+                     enum channel_width bw)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	halrf_ctrl_bw_ch(hal_info->rf, phy, center_ch, band, bw);
+
+	return hal_status;
+}
+
+void
+rtw_hal_rf_get_efuse_ex(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	halrf_tssi_get_efuse_ex(hal_info->rf, phy_idx);
+
+}
+
+
+/* PSD */
+enum rtw_hal_status rtw_hal_rf_psd_init(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					u8 path, u8 iq_path, u32 avg, u32 fft)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	halrf_psd_init(hal_info->rf, cur_phy_idx, path, iq_path, avg, fft);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_psd_restore(struct hal_info_t *hal_info, u8 cur_phy_idx)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	halrf_psd_restore(hal_info->rf, cur_phy_idx);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_psd_get_point_data(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					s32 point, u32 *value)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	*value = halrf_psd_get_point_data(hal_info->rf, cur_phy_idx, point);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_psd_query(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					u32 point, u32 start_point, u32 stop_point, u32 *outbuf)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	halrf_psd_query(hal_info->rf, cur_phy_idx, point, start_point, stop_point, outbuf);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_rf_bfer_cfg(struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	halrf_bf_config_rf(hal_info->rf);
+
+	return hal_status;
+}
+
+void rtw_hal_rf_rx_ant(struct hal_info_t *hal_info, /*enum halrf_ant*/u8 ant)
+{
+	halrf_2g_rxant(hal_info->rf, ant);
+}
+
+enum halrf_thermal_status
+rtw_hal_rf_get_ther_protected_threshold(
+	struct hal_info_t *hal_info
+)
+{
+	s8 val = 0;
+	enum halrf_thermal_status status = HALRF_THERMAL_STATUS_UNKNOWN;
+
+	val = halrf_get_ther_protected_threshold(hal_info->rf);
+
+	if(val == -1){
+		status = HALRF_THERMAL_STATUS_ABOVE_THRESHOLD;
+		PHL_TRACE(COMP_PHL_RF, _PHL_INFO_, "[HALRF] Thermal above threshold!\n");
+	}
+	else if(val == 0){
+		status = HALRF_THERMAL_STATUS_STAY_THRESHOLD;
+		PHL_TRACE(COMP_PHL_RF, _PHL_INFO_, "[HALRF] Thermal stay at threshold!\n");
+	}
+	else if(val == 1){
+		status = HALRF_THERMAL_STATUS_BELOW_THRESHOLD;
+		PHL_TRACE(COMP_PHL_RF, _PHL_INFO_, "[HALRF] Thermal below threshold!\n");
+	}
+	else{
+		status = HALRF_THERMAL_STATUS_UNKNOWN;
+		PHL_TRACE(COMP_PHL_RF, _PHL_WARNING_, "[HALRF] Thermal unknown status!\n");
+	}
+	return status;
+}
+
+void rtw_hal_rf_notification(struct hal_info_t *hal_info,
+			     enum phl_msg_evt_id event,
+			     enum phl_phy_idx phy_idx)
+{
+	halrf_wifi_event_notify(hal_info->rf, event, phy_idx);
+}
+
+void rtw_hal_rf_cmd_notification(struct hal_info_t *hal_info,
+                             void *hal_cmd,
+                             enum phl_phy_idx phy_idx)
+{
+	return;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_syn_config(struct rtw_hal_com_t *hal_com,
+                      u8 syn_id,
+                      enum phl_phy_idx phy_idx,
+                      u8 path,
+                      bool turn_on)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	switch(syn_id) {
+		case 1:
+			halrf_syn1_onoff(hal_info->rf, phy_idx, path, turn_on);
+			break;
+		default:
+			hal_status = RTW_HAL_STATUS_FAILURE;
+			PHL_ERR("%s unknown syn id(%d)\n", __func__, syn_id);
+			break;
+	}
+	return hal_status;
+}
+
+u32 rtw_hal_rf_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 cls = c2h->c2h_class;
+	u8 func = c2h->c2h_func;
+	u16 len = c2h->content_len;
+	u8 *buf = c2h->content;
+
+	return halrf_c2h_parsing(hal_info->rf, cls, func, len, buf);
+}
+
+void
+rtw_hal_rf_tssi_scan_ch(struct rtw_hal_com_t *hal_com,
+	enum phl_phy_idx phy_idx, enum rf_path path)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	halrf_tssi_scan_ch(hal_info->rf, path);
+
+	halrf_tssi_set_efuse_to_de(hal_info->rf, phy_idx);
+}
+
+void
+rtw_hal_rf_set_mp_regulation(struct hal_info_t *hal_info,
+		enum phl_phy_idx phy_idx, u8 regulation)
+{
+	halrf_set_mp_regulation(hal_info->rf, phy_idx, regulation);
+}
+
+
+void rtw_hal_rf_test_event_trigger(struct hal_info_t *hal_info,
+				enum phl_phy_idx phy_idx,
+				u8 event,
+				u8 func,
+				u32 *buf)
+{
+	*buf = halrf_test_event_trigger(hal_info->rf, phy_idx, event, func);
+}
+
+s8 rtw_hal_rf_get_power_limit(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u16 rate, u8 bandwidth,
+	u8 beamforming, u8 tx_num, u8 channel)
+{
+	return halrf_get_power_limit(hal_info->rf, phy, RF_PATH_A, rate,
+		bandwidth, beamforming, tx_num, channel);
+}
+
+s8 rtw_hal_rf_get_power_by_rate_band(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+	u16 rate, u8 dcm, u8 offset, u32 band)
+{
+	return halrf_get_power_by_rate_band(hal_info->rf, phy, rate, dcm, offset, band);
+}
+
+s8 rtw_hal_rf_get_power_limit_option(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u8 rf_path, u16 rate, u8 bandwidth, u8 beamforming,
+	u8 tx_num, u8 channel, u32 band, u8 reg)
+{
+	return halrf_get_power_limit_option(hal_info->rf, phy, rf_path, rate, bandwidth,
+					    beamforming, tx_num, channel, band, reg);
+}
+
+s8 rtw_hal_rf_get_power_limit_ru_option(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u8 rf_path, u16 rate, u8 bandwidth,
+	u8 tx_num, u8 channel, u32 band, u8 reg)
+{
+	return halrf_get_power_limit_ru_option(hal_info->rf, phy, rf_path, rate, bandwidth,
+		tx_num, channel, band, reg);
+}
+
+u8 rtw_hal_rf_get_tx_tbl_to_tx_pwr_times(struct hal_info_t *hal_info)
+{
+	return halrf_get_tx_tbl_to_tx_pwr_times(hal_info->rf);
+}
+
+s8 rtw_hal_rf_get_power_limit_value_ww(struct hal_info_t *hal_info)
+{
+	return halrf_get_power_limit_value_ww(hal_info->rf);
+}
+
+s8 rtw_hal_rf_get_power_limit_value_na(struct hal_info_t *hal_info)
+{
+	return halrf_get_power_limit_value_na(hal_info->rf);
+}
+
+u32 rtw_hal_rf_get_regulation_max_num(struct hal_info_t *hal_info, enum band_type band)
+{
+	return halrf_get_regulation_max_num(hal_info->rf, band);
+}
+
+void rtw_hal_rf_power_by_rate_store_to_array(struct hal_info_t *hal_info,
+	u32 band, u32 tx_num, u32 rate_id, u32 data)
+{
+	halrf_power_by_rate_store_to_array(hal_info->rf, band, tx_num, rate_id, data);
+}
+
+void rtw_hal_rf_power_limit_store_to_array(struct hal_info_t *hal_info,
+	u8 regulation, u8 band, u8 bandwidth, u8 rate, u8 tx_num, u8 beamforming, u8 chnl, s8 val)
+{
+	halrf_power_limit_store_to_array(hal_info->rf,
+		regulation, band, bandwidth, rate, tx_num, beamforming, chnl, val);
+}
+
+void rtw_hal_rf_power_limit_shape_store_to_array(struct hal_info_t *hal_info,
+	u8 regulation, u8 band, u8 bandwidth, u8 rate,u8 tx_num, u8 beamforming, u8 val)
+{
+	halrf_power_limit_shape_store_to_array(hal_info->rf,
+		regulation, band, bandwidth, rate, tx_num, beamforming, val);
+}
+
+void rtw_hal_rf_power_limit_ru_store_to_array(struct hal_info_t *hal_info,
+	u8 band, u8 bandwidth, u8 tx_num, u8 rate, u8 regulation, u8 chnl, s8 val)
+{
+	halrf_power_limit_ru_store_to_array(hal_info->rf,
+		band, bandwidth, tx_num, rate, regulation, chnl, val);
+}
+
+void rtw_hal_rf_power_limit_ru_shape_store_to_array(struct hal_info_t *hal_info,
+	u8 band, u8 bandwidth, u8 tx_num, u8 rate, u8 regulation, u8 val)
+{
+	halrf_power_limit_ru_shape_store_to_array(hal_info->rf,
+		band, bandwidth, tx_num, rate, regulation, val);
+}
+
+void rtw_hal_rf_clear_limit_table(struct hal_info_t *hal_info,
+	u8 band_bmp, enum phl_pwr_table pwr_table)
+{
+	halrf_config_limit_default_option(hal_info->rf, band_bmp, pwr_table);
+}
+
+void
+rtw_hal_rf_rfe_ant_num_chk(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+
+	halrf_rfe_ant_num_chk(hal_info->rf);
+}
+
+void
+rtw_hal_rf_set_pwr_lmt_main_or_aux(struct rtw_hal_com_t *hal_com, u8 ant)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	halrf_set_pwr_lmt_main_or_aux(hal_info->rf, ant);
+}
+
+void
+rtw_hal_rf_cfg_tx_by_bt_link(struct hal_info_t *hal_info , u8 is_bt_link)
+{
+	halrf_cfg_radio_b_w_bt_status(hal_info->rf, is_bt_link);
+}
+
+void rtw_hal_rf_set_ant_main_or_aux(void *hal, enum rf_path path, bool main)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *) hal;
+
+	PHL_INFO ("[HAL] %s: path=%d, main = %d\n", __func__, path, main);
+	halrf_set_ant_main_or_aux(hal_info->rf, path, main);
+}
+
+struct halrf_fem_info rtw_hal_rf_efem_info(struct rtw_hal_com_t *hal_com)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	return halrf_ex_efem_info(hal_info->rf);
+}
+
+enum rtw_hal_status
+rtw_hal_rf_ic_hw_setting_init(struct hal_info_t *hal_info)
+{
+	rtw_hal_rf_dm_init(hal_info);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_rf_fwredl_config(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx)
+{
+	PHL_INFO ("%s(): phy_idx=%d\n", __func__, phy_idx);
+
+	halrf_wowlan_config(hal_info->rf, phy_idx);
+}
+
+#ifdef CONFIG_PHL_RFK_FCS_SUPPPORT
+u8 rtw_hal_rf_fcs_support_num(struct hal_info_t *hal_info)
+{
+	return halrf_fcs_support_num(hal_info->rf);
+}
+#endif
+
+#else /*ifdef USE_TRUE_PHY*/
+#ifdef CONFIG_PHL_DIAGNOSE
+void rtw_hal_rf_diagnostic_event(struct rtw_hal_com_t *hal, u8 type,
+		u8 level, u8 version, u8 *buf, u32 len)
+{
+}
+
+enum rtw_hal_status rtw_hal_rf_query_diag_err_code(
+	void *hal, u32 *err_code)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status rtw_hal_rf_query_diag_info_len(u32 *len)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status rtw_hal_rf_query_diag_info(void *hal,
+	struct rtw_phl_diag_rf_info *info)
+{
+	return RTW_HAL_STATUS_FAILURE;
+}
+#endif
+
+enum rtw_hal_status
+rtw_hal_rf_init(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+void rtw_hal_rf_deinit(struct rtw_phl_com_t *phl_com,
+				struct hal_info_t *hal_info)
+{
+}
+
+void rtw_hal_init_rf_reg(struct rtw_phl_com_t *phl_com, void *hal)
+{
+}
+
+void rtw_hal_rf_dm_init(struct hal_info_t *hal_info)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_rf_get_efuse_info(struct rtw_hal_com_t *hal_com, u8 *efuse_map,
+				enum rtw_efuse_info info_type, void *value,
+				u8 size, u8 map_valid)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_read_pwr_table(struct rtw_hal_com_t *hal_com, u8 rf_path, u16 rate,
+				u8 bandwidth, u8 channel, u8 offset, u8 dcm,
+				u8 beamforming, s16 *get_item)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_wlan_tx_power_control(struct rtw_hal_com_t *hal_com,
+	enum phl_phy_idx phy, enum phl_pwr_ctrl pwr_ctrl_idx, u32 tx_power_val, bool enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_wl_tx_power_control(struct rtw_hal_com_t *hal_com,
+				u32 tx_power_val)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_pwrtrack(struct hal_info_t *hal_info,
+                                            u8 *txpwr_track_status,
+                                            u8 phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_pwrtrack(struct hal_info_t *hal_info,
+				u8 phy_idx, u8 txpwr_track_status)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_thermal(struct hal_info_t *hal_info,
+						u8 rf_path, u8 *thermal)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_get_default_xtal(struct rtw_hal_com_t *hal_com)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_get_default_rfe_type(struct rtw_hal_com_t *hal_com)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_tssi(struct hal_info_t *hal_info,
+				u8 phy_idx, u8 rf_path, u32 tssi_de)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+enum rtw_hal_status rtw_hal_rf_set_tssi_offset(struct hal_info_t *hal_info, u8 phy_idx, u32 tssi_de_offset, u8 rf_path)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+enum rtw_hal_status rtw_hal_rf_get_tssi(struct hal_info_t *hal_info,
+					u8 phy_idx, u8 rf_path, u32 *tssi_de)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_online_tssi_de(
+	struct hal_info_t *hal_info, u8 phy_idx, u8 rf_path,
+	s32 dbm, s32 pout, s32 *tssi_de)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_continuous_tx(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_singletone_tx(struct hal_info_t *hal_info,
+						u8 is_on,enum rf_path path)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_suppression_tx(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+#ifdef CONFIG_DBCC_SUPPORT
+enum rtw_hal_status rtw_hal_rf_ctrl_dbcc(struct rtw_hal_com_t *hal_com,
+	bool dbcc_en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+enum rtw_hal_status
+rtw_hal_rf_chl_rfk_trigger(struct rtw_hal_com_t *hal_com,
+                           u8 phy_idx,
+			   enum rfk_tri_type rt_type)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_dack_trigger(struct hal_info_t *hal_info,
+						u8 force)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_iqk_trigger(struct hal_info_t *hal_info,
+						u8 phy_idx, u8 force)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_lck_trigger(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_dpk_trigger(struct hal_info_t *hal_info,
+						u8 phy_idx, u8 force)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_tssi_trigger(struct hal_info_t *hal_info,
+						u8 phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_gapk_trigger(struct hal_info_t *hal_info,
+						u8 phy_idx, u8 force)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dack(struct hal_info_t *hal_info,
+						u8 enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_capability_iqk(struct hal_info_t *hal_info,
+						u8 enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dpk(struct hal_info_t *hal_info,
+						u8 enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_capability_dpk_track(struct hal_info_t *hal_info, u8 enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_rf_set_capability_tssi(struct hal_info_t *hal_info, u8 enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_capability_gapk(struct hal_info_t *hal_info,
+						u8 enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dack(struct hal_info_t *hal_info,
+						u8 *enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_iqk(struct hal_info_t *hal_info,
+						u8 *enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dpk(struct hal_info_t *hal_info,
+						u8 *enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_get_capability_dpk_track(struct hal_info_t *hal_info, u8 *enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_tssi(struct hal_info_t *hal_info,
+						u8 *enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_capability_gapk(struct hal_info_t *hal_info,
+						u8 *enable)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_get_tssi_de_value(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_de_tx_verify(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_get_txpwr_final_abs(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool rtw_hal_rf_proc_cmd(struct hal_info_t *hal_info,
+				struct rtw_proc_cmd *incmd,
+				char *output, u32 out_len)
+{
+	return true;
+}
+
+enum rtw_hal_status rtw_hal_rf_watchdog(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+enum rtw_hal_status
+rtw_hal_rf_set_power(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+					enum phl_pwr_table pwr_table)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_power_constraint(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+					u16 mb)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_update_tas_def_setting(void *hal, u32 tas_config)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_tas_en(void *hal, u8 en)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+enum rtw_hal_status rtw_hal_rf_set_gain_offset(struct hal_info_t *hal_info, u8 cur_phy_idx,
+						s8 offset, u8 rf_path)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_trigger_dpk_tracking(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_rf_do_tssi_scan(struct hal_info_t *hal_info, u8 cur_phy_idx)
+{
+
+}
+
+enum rtw_hal_status
+rtw_hal_rf_config_radio_to_fw(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_avg(struct hal_info_t *hal_info, u8 cur_phy_idx,
+						s32 xdbm)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool
+rtw_hal_rf_check_efuse_data(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+	return true;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_set_ch_bw(struct rtw_hal_com_t *hal_com,
+                     enum phl_phy_idx phy,
+                     u8 center_ch,
+                     enum band_type band,
+                     enum channel_width bw)
+
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/* PSD */
+enum rtw_hal_status rtw_hal_rf_psd_init(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					u8 path, u8 iq_path, u32 avg, u32 fft)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_psd_restore(struct hal_info_t *hal_info, u8 cur_phy_idx)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_psd_get_point_data(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					s32 point, u32 *value)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_rf_psd_query(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					u32 point, u32 start_point, u32 stop_point, u32 *outbuf)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void
+rtw_hal_rf_disconnect_notify(void *hal, struct rtw_chan_def *chandef)
+{
+}
+
+bool
+rtw_hal_rf_check_mcc_ch(void *hal, struct rtw_chan_def *chandef)
+{
+	return false;
+}
+
+void
+rtw_hal_rf_dpk_switch(void *hal, bool enable)
+{
+
+}
+
+void
+rtw_hal_rf_tssi_config(void *hal, enum phl_phy_idx phy_idx, bool enable)
+{
+
+}
+
+void
+rtw_hal_rf_get_efuse_ex(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+}
+
+void
+rtw_hal_rf_set_power_table_switch(struct rtw_hal_com_t *hal_com,
+				enum phl_phy_idx phy_idx,
+				u8 pwrbyrate_type, u8 pwrlmt_type)
+{
+	return;
+}
+
+int rtw_hal_rf_get_predef_pw_lmt_regu_type_from_str(
+	enum band_type band, const char *str)
+{
+	return -1;
+}
+
+const char * const *rtw_hal_rf_get_predef_pw_lmt_regu_type_str_array(
+	enum band_type band, u8 *num)
+{
+	return NULL;
+}
+
+u8 rtw_hal_rf_get_pw_lmt_regu_type(struct hal_info_t *hal_info, enum band_type band)
+{
+	return 0;
+}
+
+const char *rtw_hal_rf_get_pw_lmt_regu_type_str(struct hal_info_t *hal_info, enum band_type band)
+{
+	return NULL;
+}
+
+bool rtw_hal_rf_pw_lmt_regu_tbl_exist(struct hal_info_t *hal_info, enum band_type band, u8 regu)
+{
+	return false;
+}
+
+void rtw_hal_rf_set_pw_lmt_regu_table_exist(struct hal_info_t *hal_info, u8 band, u8 reg)
+{
+}
+
+int rtw_hal_rf_file_regd_ext_search(struct hal_info_t *hal_info
+	, enum band_type band, u16 domain_code, const char *country)
+{
+	return -1;
+}
+
+void rtw_hal_rf_auto_pw_lmt_regu(struct hal_info_t *hal_info)
+{
+}
+
+void rtw_hal_rf_force_pw_lmt_regu(struct hal_info_t *hal_info,
+	u8 regu_2g[], u8 regu_2g_len, u8 regu_5g[], u8 regu_5g_len, u8 regu_6g[], u8 regu_6g_len)
+{
+}
+
+enum rtw_hal_status rtw_hal_rf_bfer_cfg(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_rf_rx_ant(struct hal_info_t *hal_info, /*enum halrf_ant*/u8 ant)
+{
+
+}
+
+enum halrf_thermal_status
+rtw_hal_rf_get_ther_protected_threshold(
+	struct hal_info_t *hal_info
+)
+{
+	return HALRF_THERMAL_STATUS_BELOW_THRESHOLD;
+}
+
+void rtw_hal_rf_notification(struct hal_info_t *hal_info,
+                             enum phl_msg_evt_id event,
+                             enum phl_phy_idx phy_idx)
+{
+	return;
+}
+
+
+
+void rtw_hal_rf_cmd_notification(struct hal_info_t *hal_info,
+                             void *hal_cmd,
+                             enum phl_phy_idx phy_idx)
+{
+	return;
+}
+
+enum rtw_hal_status
+rtw_hal_rf_syn_config(struct rtw_hal_com_t *hal_com,
+                      u8 syn_id,
+                      enum phl_phy_idx phy_idx,
+                      u8 path,
+                      bool turn_on)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+u32 rtw_hal_rf_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg)
+{
+	return 0;
+}
+
+void
+rtw_hal_rf_tssi_scan_ch(struct rtw_hal_com_t *hal_com,
+	enum phl_phy_idx phy_idx, enum rf_path path)
+
+{
+	return;
+}
+
+void rtw_hal_rf_test_event_trigger(struct hal_info_t *hal_info,
+				enum phl_phy_idx phy_idx,
+				u8 event,
+				u8 func,
+				u32 *buf)
+{
+
+}
+
+
+s8 rtw_hal_rf_get_power_limit(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u16 rate, u8 bandwidth,
+	u8 beamforming, u8 tx_num, u8 channel)
+{
+	return 0;
+}
+
+s8 rtw_hal_rf_get_power_by_rate_band(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+	u16 rate, u8 dcm, u8 offset, u32 band)
+{
+	return 0;
+}
+
+s8 rtw_hal_rf_get_power_limit_option(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u8 rf_path, u16 rate, u8 bandwidth, u8 beamforming,
+	u8 tx_num, u8 channel, u32 band, u8 reg)
+{
+	return 0;
+}
+
+s8 rtw_hal_rf_get_power_limit_ru_option(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u8 rf_path, u16 rate, u8 bandwidth,
+	u8 tx_num, u8 channel, u32 band, u8 reg)
+{
+	return 0;
+}
+
+u8 rtw_hal_rf_get_tx_tbl_to_tx_pwr_times(struct hal_info_t *hal_info)
+{
+	return 0;
+}
+
+s8 rtw_hal_rf_get_power_limit_value_ww(struct hal_info_t *hal_info)
+{
+	return 0;
+}
+
+s8 rtw_hal_rf_get_power_limit_value_na(struct hal_info_t *hal_info)
+{
+	return 0;
+}
+
+u32 rtw_hal_rf_get_regulation_max_num(struct hal_info_t *hal_info, enum band_type band)
+{
+	return 0;
+}
+
+void rtw_hal_rf_power_by_rate_store_to_array(struct hal_info_t *hal_info,
+	u32 band, u32 tx_num, u32 rate_id, u32 data)
+{
+}
+
+void rtw_hal_rf_power_limit_store_to_array(struct hal_info_t *hal_info,
+	u8 regulation, u8 band, u8 bandwidth, u8 rate, u8 tx_num, u8 beamforming, u8 chnl, s8 val)
+{
+}
+
+void rtw_hal_rf_power_limit_shape_store_to_array(struct hal_info_t *hal_info,
+	u8 regulation, u8 band, u8 bandwidth, u8 rate, u8 tx_num, u8 beamforming, u8 val)
+{
+}
+
+void rtw_hal_rf_power_limit_ru_store_to_array(struct hal_info_t *hal_info,
+	u8 band, u8 bandwidth, u8 tx_num, u8 rate, u8 regulation, u8 chnl, s8 val)
+{
+}
+
+void rtw_hal_rf_power_limit_ru_shape_store_to_array(struct hal_info_t *hal_info,
+	u8 band, u8 bandwidth, u8 tx_num, u8 rate, u8 regulation, u8 val)
+{
+}
+
+void rtw_hal_rf_clear_limit_table(struct hal_info_t *hal_info,
+	u8 band_bmp, enum phl_pwr_table pwr_table)
+{
+}
+
+void
+rtw_hal_rf_rfe_ant_num_chk(struct rtw_hal_com_t *hal_com)
+{
+}
+
+void
+rtw_hal_rf_set_pwr_lmt_main_or_aux(struct rtw_hal_com_t *hal_com, u8 ant)
+{
+}
+
+void
+rtw_hal_rf_cfg_tx_by_bt_link(struct hal_info_t *hal_info , u8 is_bt_link)
+{
+
+}
+
+void
+rtw_hal_rf_update_ext_pwr_lmt_table(struct hal_info_t *hal_info,
+                                    enum phl_phy_idx phy)
+{
+}
+
+u8
+rtw_hal_rf_get_tx_tbl_to_pwr_times(struct hal_info_t *hal_info)
+{
+	return 4;
+}
+
+void
+rtw_hal_rf_set_tx_pwr_comp(struct hal_info_t *hal_info,	enum phl_phy_idx phy,
+                           struct rtw_phl_regu_dyn_ant_gain *dyn_ag)
+{
+}
+
+void
+rtw_hal_rf_set_mp_regulation(struct hal_info_t *hal_info,
+                             enum phl_phy_idx phy_idx, u8 regulation)
+{
+}
+
+enum rtw_hal_status
+rtw_hal_rf_ic_hw_setting_init(struct hal_info_t *hal_info)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+void rtw_hal_rf_fwredl_config(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx)
+{
+}
+
+#ifdef CONFIG_PHL_RFK_FCS_SUPPPORT
+u8 rtw_hal_rf_fcs_support_num(struct hal_info_t *hal_info)
+{
+	return 0;
+}
+#endif
+
+#endif /*ifdef USE_TRUE_PHY*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_rf.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_rf.h
new file mode 100644
index 000000000000..a0475bdc4b18
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_api_rf.h
@@ -0,0 +1,329 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_API_RF_H_
+#define _HAL_API_RF_H_
+
+enum halrf_thermal_status{
+	HALRF_THERMAL_STATUS_BELOW_THRESHOLD,
+	HALRF_THERMAL_STATUS_STAY_THRESHOLD,
+	HALRF_THERMAL_STATUS_ABOVE_THRESHOLD,
+	HALRF_THERMAL_STATUS_UNKNOWN
+};
+
+enum rtw_hal_status
+rtw_hal_rf_init(struct rtw_phl_com_t *phl_com,
+			struct hal_info_t *hal_info);
+
+void rtw_hal_rf_deinit(struct rtw_phl_com_t *phl_com,
+			struct hal_info_t *hal_info);
+
+void rtw_hal_init_rf_reg(struct rtw_phl_com_t *phl_com, void *hal);
+
+void rtw_hal_rf_ic_cfg_init(void *hal);
+
+enum rtw_hal_status rtw_hal_rf_get_pwrtrack(struct hal_info_t *hal_info, u8 *txpwr_track_status, u8 phy_idx);
+
+void rtw_hal_rf_dm_init(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_rf_set_pwrtrack(struct hal_info_t *hal_info, u8 phy_idx, u8 txpwr_track_status);
+
+void rtw_hal_rf_dm_init(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_rf_get_thermal(struct hal_info_t *hal_info, u8 rf_path, u8 *thermal);
+
+enum rtw_hal_status rtw_hal_rf_set_tssi(struct hal_info_t *hal_info, u8 phy_idx, u8 rf_path, u32 tssi_de);
+
+enum rtw_hal_status rtw_hal_rf_get_tssi(struct hal_info_t *hal_info, u8 phy_idx, u8 rf_path, u32 *tssi);
+
+enum rtw_hal_status rtw_hal_rf_get_online_tssi_de(struct hal_info_t *hal_info, u8 phy_idx, u8 rf_path,s32 dbm, s32 pout, s32 *tssi_de);
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_offset(struct hal_info_t *hal_info, u8 phy_idx, u32 tssi_de_offset, u8 rf_path);
+
+enum rtw_hal_status rtw_hal_rf_set_continuous_tx(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_rf_set_singletone_tx(struct hal_info_t *hal_info,
+							 u8 is_on,enum rf_path path);
+
+enum rtw_hal_status rtw_hal_rf_set_suppression_tx(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_rf_chl_rfk_trigger(struct rtw_hal_com_t *hal_com,
+                           u8 phy_idx,
+                           enum rfk_tri_type rt_type);
+
+enum rtw_hal_status rtw_hal_rf_dack_trigger(struct hal_info_t *hal_info,
+			   u8 force);
+
+
+enum rtw_hal_status rtw_hal_rf_iqk_trigger(struct hal_info_t *hal_info,
+			   u8 phy_idx, u8 force);
+
+
+enum rtw_hal_status rtw_hal_rf_lck_trigger(struct hal_info_t *hal_info);
+
+
+enum rtw_hal_status rtw_hal_rf_dpk_trigger(struct hal_info_t *hal_info,
+			   u8 phy_idx, u8 force);
+
+
+enum rtw_hal_status rtw_hal_rf_tssi_trigger(struct hal_info_t *hal_info,
+				u8 phy_idx);
+
+
+enum rtw_hal_status rtw_hal_rf_gapk_trigger(struct hal_info_t *hal_info,
+			   u8 phy_idx, u8 force);
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dack(struct hal_info_t *hal_info,
+				u8 enable);
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_iqk(struct hal_info_t *hal_info,
+				u8 enable);
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dpk(struct hal_info_t *hal_info,
+			   u8 enable);
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_dpk_track(struct hal_info_t *hal_info,
+			   u8 enable);
+
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_tssi(struct hal_info_t *hal_info,
+				u8 enable);
+
+
+enum rtw_hal_status rtw_hal_rf_set_capability_gapk(struct hal_info_t *hal_info,
+			   u8 enable);
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dack(struct hal_info_t *hal_info,
+				u8 *enable);
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_iqk(struct hal_info_t *hal_info,
+				u8 *enable);
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dpk(struct hal_info_t *hal_info,
+			   u8 *enable);
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_dpk_track(struct hal_info_t *hal_info,
+				u8 *enable);
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_tssi(struct hal_info_t *hal_info,
+				u8 *enable);
+
+
+enum rtw_hal_status rtw_hal_rf_get_capability_gapk(struct hal_info_t *hal_info,
+			   u8 *enable);
+
+enum rtw_hal_status rtw_hal_rf_get_tssi_de_value(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_de_tx_verify(struct hal_info_t *hal_info);
+
+enum rtw_hal_status rtw_hal_rf_get_txpwr_final_abs(struct hal_info_t *hal_info);
+
+int rtw_hal_rf_get_predef_pw_lmt_regu_type_from_str(
+	enum band_type band, const char *str);
+const char * const *rtw_hal_rf_get_predef_pw_lmt_regu_type_str_array(
+	enum band_type band, u8 *num);
+
+u8 rtw_hal_rf_get_pw_lmt_regu_type(struct hal_info_t *hal_info, enum band_type band);
+const char *rtw_hal_rf_get_pw_lmt_regu_type_str(struct hal_info_t *hal_info, enum band_type band);
+
+bool rtw_hal_rf_pw_lmt_regu_tbl_exist(struct hal_info_t *hal_info, enum band_type band, u8 regu);
+void rtw_hal_rf_set_pw_lmt_regu_table_exist(struct hal_info_t *hal_info, u8 band, u8 reg);
+int rtw_hal_rf_file_regd_ext_search(struct hal_info_t *hal_info, enum band_type band
+	, u16 domain_code, const char *country);
+
+void rtw_hal_rf_auto_pw_lmt_regu(struct hal_info_t *hal_info);
+void rtw_hal_rf_force_pw_lmt_regu(struct hal_info_t *hal_info,
+	u8 regu_2g[], u8 regu_2g_len, u8 regu_5g[], u8 regu_5g_len, u8 regu_6g[], u8 regu_6g_len);
+
+bool
+rtw_hal_rf_proc_cmd(struct hal_info_t *hal_info,
+					struct rtw_proc_cmd *incmd,
+					char *output, u32 out_len);
+
+enum rtw_hal_status
+rtw_hal_rf_recover(struct hal_info_t *hal_info, u32 offset, u32 val, enum rf_path path);
+
+enum rtw_hal_status rtw_hal_rf_watchdog(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_rf_set_power(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+				enum phl_pwr_table pwr_table);
+
+enum rtw_hal_status
+rtw_hal_rf_set_power_constraint(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+					u16 mb);
+
+enum rtw_hal_status rtw_hal_rf_set_gain_offset(struct hal_info_t *hal_info, u8 cur_phy_idx,
+						s8 offset, u8 rf_path);
+
+enum rtw_hal_status rtw_hal_rf_trigger_dpk_tracking(struct hal_info_t *hal_info);
+
+enum rtw_hal_status
+rtw_hal_rf_get_default_rfe_type(struct rtw_hal_com_t *hal_com);
+
+enum rtw_hal_status
+rtw_hal_rf_get_default_xtal(struct rtw_hal_com_t *hal_com);
+
+
+enum rtw_hal_status rtw_hal_rf_set_tssi_avg(struct hal_info_t *hal_info, u8 cur_phy_idx,
+						s32 xdbm);
+void
+rtw_hal_rf_do_tssi_scan(struct hal_info_t *hal_ionfo, u8 cur_phy_idx);
+
+void
+rtw_hal_rf_update_ext_pwr_lmt_table(struct hal_info_t *hal_info,
+					      enum phl_phy_idx phy);
+
+enum rtw_hal_status
+rtw_hal_rf_config_radio_to_fw(struct hal_info_t *hal_info);
+
+bool
+rtw_hal_rf_check_efuse_data(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx);
+
+void
+rtw_hal_rf_disconnect_notify(void *hal, struct rtw_chan_def *chandef);
+
+bool
+rtw_hal_rf_check_mcc_ch(void *hal, struct rtw_chan_def *chandef);
+
+void
+rtw_hal_rf_dpk_switch(void *hal, bool enable);
+
+void
+rtw_hal_rf_tssi_config(void *hal, enum phl_phy_idx phy_idx, bool enable);
+
+
+enum rtw_hal_status
+rtw_hal_rf_set_ch_bw(struct rtw_hal_com_t *hal_com,
+                     enum phl_phy_idx phy,
+                     u8 center_ch,
+                     enum band_type band,
+                     enum channel_width bw);
+
+void
+rtw_hal_rf_get_efuse_ex(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx);
+
+
+/* PSD */
+enum rtw_hal_status rtw_hal_rf_psd_init(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					u8 path, u8 iq_path, u32 avg, u32 fft);
+
+enum rtw_hal_status rtw_hal_rf_psd_restore(struct hal_info_t *hal_info, u8 cur_phy_idx);
+
+enum rtw_hal_status rtw_hal_rf_psd_get_point_data(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					s32 point, u32 *value);
+
+enum rtw_hal_status rtw_hal_rf_psd_query(struct hal_info_t *hal_info, u8 cur_phy_idx,
+					u32 point, u32 start_point, u32 stop_point, u32 *outbuf);
+
+enum rtw_hal_status rtw_hal_rf_bfer_cfg(struct hal_info_t *hal_info);
+
+void rtw_hal_rf_rx_ant(struct hal_info_t *hal_info, /*enum halrf_ant*/ u8 ant);
+
+enum halrf_thermal_status
+rtw_hal_rf_get_ther_protected_threshold(
+	struct hal_info_t *hal_info
+);
+
+void rtw_hal_rf_notification(struct hal_info_t *hal_info,
+			     enum phl_msg_evt_id event,
+			     enum phl_phy_idx phy_idx);
+
+void rtw_hal_rf_cmd_notification(struct hal_info_t *hal_info,
+                             void *hal_cmd,
+                             enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_rf_syn_config(struct rtw_hal_com_t *hal_com,
+                      u8 syn_id,
+                      enum phl_phy_idx phy_idx,
+                      u8 path,
+                      bool turn_on);
+void
+rtw_hal_rf_set_mp_regulation(struct hal_info_t *hal_info,
+		enum phl_phy_idx phy_idx, u8 regulation);
+
+void rtw_hal_rf_test_event_trigger(struct hal_info_t *hal_info,
+				enum phl_phy_idx phy_idx,
+				u8 event,
+				u8 func,
+				u32 *buf);
+
+void rtw_hal_rf_test_event_trigger(struct hal_info_t *hal_info,
+				enum phl_phy_idx phy_idx,
+				u8 event,
+				u8 func,
+				u32 *buf);
+
+s8 rtw_hal_rf_get_power_limit(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u16 rate, u8 bandwidth,
+	u8 beamforming, u8 tx_num, u8 channel);
+s8 rtw_hal_rf_get_power_by_rate_band(struct hal_info_t *hal_info, enum phl_phy_idx phy,
+	u16 rate, u8 dcm, u8 offset, u32 band);
+s8 rtw_hal_rf_get_power_limit_option(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u8 rf_path, u16 rate, u8 bandwidth, u8 beamforming,
+	u8 tx_num, u8 channel, u32 band, u8 reg);
+s8 rtw_hal_rf_get_power_limit_ru_option(struct hal_info_t *hal_info,
+	enum phl_phy_idx phy, u8 rf_path, u16 rate, u8 bandwidth,
+	u8 tx_num, u8 channel, u32 band, u8 reg);
+u8 rtw_hal_rf_get_tx_tbl_to_tx_pwr_times(struct hal_info_t *hal_info);
+s8 rtw_hal_rf_get_power_limit_value_ww(struct hal_info_t *hal_info);
+s8 rtw_hal_rf_get_power_limit_value_na(struct hal_info_t *hal_info);
+u32 rtw_hal_rf_get_regulation_max_num(struct hal_info_t *hal_info, enum band_type band);
+
+void rtw_hal_rf_power_by_rate_store_to_array(struct hal_info_t *hal_info,
+	u32 band, u32 tx_num, u32 rate_id, u32 data);
+void rtw_hal_rf_power_limit_store_to_array(struct hal_info_t *hal_info,
+	u8 regulation, u8 band, u8 bandwidth, u8 rate, u8 tx_num, u8 beamforming, u8 chnl, s8 val);
+void rtw_hal_rf_power_limit_shape_store_to_array(struct hal_info_t *hal_info,
+	u8 regulation, u8 band, u8 bandwidth, u8 rate, u8 tx_num, u8 beamforming, u8 val);
+void rtw_hal_rf_power_limit_ru_store_to_array(struct hal_info_t *hal_info,
+	u8 band, u8 bandwidth, u8 tx_num, u8 rate, u8 regulation, u8 chnl, s8 val);
+void rtw_hal_rf_power_limit_ru_shape_store_to_array(struct hal_info_t *hal_info,
+	u8 band, u8 bandwidth, u8 tx_num, u8 rate, u8 regulation, u8 val);
+void rtw_hal_rf_clear_limit_table(struct hal_info_t *hal_info,
+	u8 band_bmp, enum phl_pwr_table pwr_table);
+
+void rtw_hal_rf_rfe_ant_num_chk(struct rtw_hal_com_t *hal_com);
+
+u8 rtw_hal_rf_get_tx_tbl_to_pwr_times(struct hal_info_t *hal_info);
+void rtw_hal_rf_set_tx_pwr_comp(struct hal_info_t *hal_info,
+			enum phl_phy_idx phy,
+			struct rtw_phl_regu_dyn_ant_gain *dyn_ag);
+
+void
+rtw_hal_rf_cfg_tx_by_bt_link(struct hal_info_t *hal_info , u8 is_bt_link);
+
+u32 rtw_hal_rf_process_c2h(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg);
+
+enum rtw_hal_status
+rtw_hal_rf_ic_hw_setting_init(struct hal_info_t *hal_info);
+
+void rtw_hal_rf_fwredl_config(struct hal_info_t *hal_info, enum phl_phy_idx phy_idx);
+#ifdef CONFIG_PHL_RFK_FCS_SUPPPORT
+u8 rtw_hal_rf_fcs_support_num(struct hal_info_t *hal_info);
+#endif
+#endif /*_HAL_API_RF_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_beamform.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_beamform.c
new file mode 100644
index 000000000000..7fbe6138a887
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_beamform.c
@@ -0,0 +1,1195 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_BEAMFORM
+/**
+ * rtw_hal_bf_dbg_dump_entry
+ * @entry: hal_bf_entry for dump content
+ **/
+void rtw_hal_bf_dbg_dump_entry(void *entry)
+{
+	struct hal_bf_entry *bf_entry = (struct hal_bf_entry *)entry;
+	if (bf_entry == NULL) {
+		return;
+	}
+
+	PHL_INFO("-----DUMP BF ENTRY-----\n");
+	PHL_INFO("bf_entry->bf_idx 0x%x \n", bf_entry->bf_idx);
+	PHL_INFO("bf_entry->macid 0x%x \n", bf_entry->macid);
+	PHL_INFO("bf_entry->aid12 0x%x \n", bf_entry->aid12);
+	PHL_INFO("bf_entry->band 0x%x \n", bf_entry->band);
+	PHL_INFO("bf_entry->en_swap 0x%x \n", bf_entry->en_swap ? 1 : 0);
+	PHL_INFO("bf_entry->counter 0x%x \n", bf_entry->couter);
+	PHL_INFO("bf_entry->csi_buf 0x%x \n", bf_entry->csi_buf);
+	PHL_INFO("bf_entry->csi_buf_swap 0x%x \n", bf_entry->csi_buf_swap);
+	if (bf_entry->bfee != NULL) {
+		PHL_INFO("bf_entry->bfee->type 0x%x \n", bf_entry->bfee->type);
+	}
+}
+/**
+ * rtw_hal_bf_dbg_dump_entry_all
+ * @hal: hal_info_t
+ **/
+void rtw_hal_bf_dbg_dump_entry_all(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_entry *bf_entry;
+	_os_list *busy_list = &bf_obj->bf_busy_list;
+	_os_list *idle_list = &bf_obj->bf_idle_list;
+
+	PHL_INFO("-----DUMP BF BUSY LIST-----\n");
+	if(list_empty(busy_list)) {
+		PHL_INFO("BF Entry BUSY LIST is Empty\n");
+	} else {
+		phl_list_for_loop(bf_entry, struct hal_bf_entry, busy_list,
+					list) {
+			PHL_INFO("bf_entry->bf_idx 0x%x \n", bf_entry->bf_idx);
+			PHL_INFO("bf_entry->macid 0x%x \n", bf_entry->macid);
+			PHL_INFO("bf_entry->aid12 0x%x \n", bf_entry->aid12);
+			PHL_INFO("bf_entry->band 0x%x \n", bf_entry->band);
+			PHL_INFO("bf_entry->en_swap 0x%x \n", bf_entry->en_swap ? 1 : 0);
+			PHL_INFO("bf_entry->counter 0x%x \n", bf_entry->couter);
+			PHL_INFO("bf_entry->csi_buf 0x%x \n",
+					bf_entry->csi_buf);
+			PHL_INFO("bf_entry->csi_buf_swap 0x%x \n",
+					bf_entry->csi_buf_swap);
+			if (NULL!=bf_entry->bfee) {
+				PHL_INFO("bfee->idx 0x%x \n",
+					bf_entry->bfee->idx);
+				PHL_INFO("bfee->type 0x%x \n",
+					bf_entry->bfee->type);
+			}
+		}
+	}
+	PHL_INFO("-----DUMP BF IDLE LIST-----\n");
+	if(list_empty(idle_list)) {
+		PHL_INFO("BF Entry IDLE LIST is Empty\n");
+	} else {
+		phl_list_for_loop(bf_entry, struct hal_bf_entry, idle_list,
+					list) {
+			PHL_INFO("bf_entry->bf_idx 0x%x \n", bf_entry->bf_idx);
+			PHL_INFO("bf_entry->macid 0x%x \n", bf_entry->macid);
+			PHL_INFO("bf_entry->aid12 0x%x \n", bf_entry->aid12);
+			PHL_INFO("bf_entry->band 0x%x \n", bf_entry->band);
+			PHL_INFO("bf_entry->csi_buf 0x%x \n",
+					bf_entry->csi_buf);
+		}
+	}
+}
+
+void _reset_bf_entry(struct hal_bf_entry *bf_entry)
+{
+	bf_entry->macid = 0;
+	bf_entry->aid12 = 0;
+	bf_entry->csi_buf = 0;
+	bf_entry->bfee = NULL;
+	bf_entry->csi_buf_swap = 0;
+	bf_entry->couter = 0;
+}
+
+void _reset_sumu_entry(struct hal_sumu_entry *entry)
+{
+	entry->snd_sts = 0;
+	return;
+}
+
+/* START of tx bf entry */
+static struct hal_bf_entry *_query_idle_bf_entry(struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj)
+{
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *idle_list = &bf_obj->bf_idle_list;
+	struct hal_bf_entry *bf_entry = NULL;
+
+	_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+	if (true == list_empty(idle_list)) {
+		bf_entry = NULL;
+	} else {
+		bf_entry = list_first_entry(idle_list, struct hal_bf_entry,
+					list);
+		bf_obj->num_idle_bf_entry--;
+		list_del(&bf_entry->list);
+	}
+	_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+
+	return bf_entry;
+}
+
+static enum rtw_hal_status _enqueue_idle_bf_entry(
+				struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj,
+				struct hal_bf_entry *bf_entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *list = &bf_obj->bf_idle_list;
+
+	if (bf_entry != NULL) {
+		_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		list_add(&bf_entry->list, list);
+		bf_obj->num_idle_bf_entry++;
+		_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+
+static enum rtw_hal_status _enqueue_busy_bf_entry(
+				struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj,
+				struct hal_bf_entry *bf_entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *list = &bf_obj->bf_busy_list;
+
+	if (bf_entry != NULL) {
+		_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		list_add_tail(&bf_entry->list, list);
+		_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+/*su entry*/
+static struct hal_sumu_entry *_query_idle_su_entry(struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj)
+{
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *idle_list = &bf_obj->su_idle_list;
+	struct hal_sumu_entry *su_entry = NULL;
+
+	_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+	if (true == list_empty(idle_list)) {
+		su_entry = NULL;
+	} else {
+		su_entry = list_first_entry(idle_list, struct hal_sumu_entry,
+					list);
+		bf_obj->num_idle_su_entry--;
+		list_del(&su_entry->list);
+	}
+	_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+
+	return su_entry;
+}
+
+static enum rtw_hal_status _enqueue_idle_su_entry(
+				struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj,
+				struct hal_sumu_entry *su_entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *list = &bf_obj->su_idle_list;
+
+	if (su_entry != NULL) {
+		_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		list_add(&su_entry->list, list);
+		bf_obj->num_idle_su_entry++;
+		_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+
+static enum rtw_hal_status _enqueue_busy_su_entry(
+				struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj,
+				struct hal_sumu_entry *su_entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *list = &bf_obj->su_busy_list;
+
+	if (su_entry != NULL) {
+		_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		list_add_tail(&su_entry->list, list);
+		_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+
+
+/*mu entry*/
+static struct hal_sumu_entry *_query_idle_mu_entry(struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj)
+{
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *idle_list = &bf_obj->mu_idle_list;
+	struct hal_sumu_entry *mu_entry = NULL;
+
+	_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+	if (true == list_empty(idle_list)) {
+		mu_entry = NULL;
+	} else {
+		mu_entry = list_first_entry(idle_list, struct hal_sumu_entry,
+					list);
+		bf_obj->num_idle_mu_entry--;
+		list_del(&mu_entry->list);
+	}
+	_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+
+	return mu_entry;
+}
+
+static enum rtw_hal_status _enqueue_idle_mu_entry(
+				struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj,
+				struct hal_sumu_entry *mu_entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *list = &bf_obj->mu_idle_list;
+
+	if (mu_entry != NULL) {
+		_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		list_add(&mu_entry->list, list);
+		bf_obj->num_idle_mu_entry++;
+		_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+
+static enum rtw_hal_status _enqueue_busy_mu_entry(
+				struct hal_info_t *hal_info,
+				struct hal_bf_obj *bf_obj,
+				struct hal_sumu_entry *mu_entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	_os_list *list = &bf_obj->mu_busy_list;
+
+	if (mu_entry != NULL) {
+		_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		list_add_tail(&mu_entry->list, list);
+		_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return status;
+}
+
+/* hal bf init */
+enum rtw_hal_status _hal_bf_init_su_entry(
+				struct hal_info_t *hal_info,
+				u8 num)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_sumu_entry *su_entry = NULL;
+	u8 i;
+
+	do {
+		bf_obj->su_entry = _os_mem_alloc(drv_priv,
+					sizeof(*su_entry) * num);
+
+		if (NULL == bf_obj->su_entry) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		_os_mem_set(drv_priv, bf_obj->su_entry, 0,
+					sizeof(*su_entry) * num);
+
+		su_entry = bf_obj->su_entry;
+
+		for ( i = 0 ; i < num; i++) {
+			su_entry[i].idx = i;
+			su_entry[i].type = HAL_BFEE_SU;
+			_reset_sumu_entry(&su_entry[i]);
+			INIT_LIST_HEAD(&su_entry[i].list);
+			list_add_tail(&su_entry[i].list, &bf_obj->su_idle_list);
+			bf_obj->num_idle_su_entry++;
+		}
+
+	} while (0);
+	return status;
+}
+
+
+enum rtw_hal_status _hal_bf_init_mu_entry(
+	struct hal_info_t *hal_info,
+	u8 num)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_sumu_entry *mu_entry = NULL;
+	u8 i;
+
+	do {
+		bf_obj->mu_entry = _os_mem_alloc(drv_priv,
+					sizeof(*mu_entry) * num);
+
+		if (NULL == bf_obj->mu_entry) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		_os_mem_set(drv_priv, bf_obj->mu_entry, 0,
+					sizeof(*mu_entry) * num);
+
+		mu_entry = bf_obj->mu_entry;
+
+		for ( i = 0 ; i < num; i++) {
+			mu_entry[i].idx = i;
+			mu_entry[i].type = HAL_BFEE_MU;
+			_reset_sumu_entry(&mu_entry[i]);
+			INIT_LIST_HEAD(&mu_entry[i].list);
+			list_add_tail(&mu_entry[i].list, &bf_obj->mu_idle_list);
+			bf_obj->num_idle_mu_entry++;
+		}
+
+	} while (0);
+	return status;
+}
+
+
+enum rtw_hal_status _hal_bf_init_bf_entry(
+	struct hal_info_t *hal_info,
+	u8 num)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_entry *bf_entry = NULL;
+	u8 i;
+
+	do {
+		bf_obj->bf_entry = _os_mem_alloc(drv_priv,
+						sizeof(*bf_entry) * num);
+		if (NULL == bf_obj->bf_entry) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		_os_mem_set(drv_priv, bf_obj->bf_entry, 0,
+					sizeof(*bf_entry) * num);
+
+		bf_entry = bf_obj->bf_entry;
+
+		for ( i = 0 ; i < num; i++) {
+			bf_entry[i].bf_idx = i;
+			_reset_bf_entry(&bf_entry[i]);
+			INIT_LIST_HEAD(&bf_entry[i].list);
+			list_add_tail(&bf_entry[i].list,
+					&bf_obj->bf_idle_list);
+			bf_obj->num_idle_bf_entry++;
+		}
+
+	} while (0);
+	return status;
+}
+
+
+/* Start of HAL API for other HAL modules */
+/**
+ * hal_bf_init:
+ * 	initialize of beamform resource mgnt module
+ * input :
+ * @hal_info: (struct hal_info_t *)
+ * @bf_entry_nr:  Number of HW support TxBF Entry
+ * @su_entry_nr: Number of HW support BFee-SU Entry
+ * @mu_entry_nr: Number of HW support BFee-MU Entry
+ **/
+enum rtw_hal_status hal_bf_init(
+	struct hal_info_t *hal_info,
+	u8 bf_entry_nr,
+	u8 su_entry_nr,
+	u8 mu_entry_nr)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = NULL;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+
+	FUNCIN();
+	do {
+		bf_obj = _os_mem_alloc(drv_priv, sizeof(*bf_obj));
+
+		if (bf_obj == NULL) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		hal_com->bf_obj = bf_obj;
+
+		_os_spinlock_init(drv_priv, &bf_obj->bf_lock);
+
+		INIT_LIST_HEAD(&bf_obj->bf_idle_list);
+		INIT_LIST_HEAD(&bf_obj->bf_busy_list);
+
+		INIT_LIST_HEAD(&bf_obj->su_idle_list);
+		INIT_LIST_HEAD(&bf_obj->su_busy_list);
+
+		INIT_LIST_HEAD(&bf_obj->mu_idle_list);
+		INIT_LIST_HEAD(&bf_obj->mu_busy_list);
+
+		if (RTW_HAL_STATUS_SUCCESS !=
+			_hal_bf_init_bf_entry(hal_info, bf_entry_nr)) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		bf_obj->max_bf_entry_nr =  bf_entry_nr;
+
+		if (RTW_HAL_STATUS_SUCCESS !=
+			_hal_bf_init_su_entry(hal_info, su_entry_nr)) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		bf_obj->max_su_bfee_nr = su_entry_nr;
+
+		if (RTW_HAL_STATUS_SUCCESS !=
+			_hal_bf_init_mu_entry(hal_info, mu_entry_nr)) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		bf_obj->max_mu_bfee_nr =  mu_entry_nr;
+
+		bf_obj->self_bf_cap[0] = 0;
+		bf_obj->self_bf_cap[1] = 0;
+
+	} while (0);
+
+	if (RTW_HAL_STATUS_SUCCESS != status) {
+		hal_bf_deinit(hal_info);
+	}
+	FUNCOUT();
+
+	return status;
+}
+
+/**
+ * hal_bf_deinit
+ * 	deinitialize of beamform resource mgnt module
+ * input :
+ * @hal_info: (struct hal_info_t *)
+ **/
+void hal_bf_deinit(struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct hal_bf_entry *bf_entry = NULL;
+	struct hal_sumu_entry *mu_entry = NULL;
+	struct hal_sumu_entry *su_entry = NULL;
+
+	if (bf_obj != NULL) {
+		bf_entry = bf_obj->bf_entry;
+		mu_entry = bf_obj->mu_entry;
+		su_entry = bf_obj->su_entry;
+
+		if (bf_entry != NULL) {
+			_os_mem_free(drv_priv, bf_entry,
+				sizeof(*bf_entry) * bf_obj->max_bf_entry_nr);
+			bf_obj->bf_entry = NULL;
+		}
+		if (su_entry != NULL) {
+			_os_mem_free(drv_priv, su_entry,
+				sizeof(*su_entry) * bf_obj->max_su_bfee_nr);
+			bf_obj->su_entry = NULL;
+		}
+
+		if (mu_entry != NULL) {
+			_os_mem_free(drv_priv, mu_entry,
+				sizeof(*mu_entry) * bf_obj->max_mu_bfee_nr);
+			bf_obj->mu_entry = NULL;
+		}
+
+		_os_spinlock_free(drv_priv, &bf_obj->bf_lock);
+
+		/* bf obj need free as last */
+		_os_mem_free(drv_priv, bf_obj, sizeof(struct hal_bf_obj));
+		hal_com->bf_obj = NULL;
+	}
+}
+/**
+ * hal_bf_release_target_bf_entry
+ * 	Release the bf entry resource
+ * input
+ * @hal_info: (struct hal_info_t *)
+ * @entry:  hal_bf_entry to be released
+ **/
+enum rtw_hal_status hal_bf_release_target_bf_entry(
+				struct hal_info_t *hal_info,
+				void *entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_entry *bf_entry = (struct hal_bf_entry *)entry;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	u8 bfee_idx = 0;
+	u16 macid_rsvd = 0;
+	FUNCIN();
+	if (bf_obj != NULL && bf_entry != NULL) {
+		if (bf_entry->bfee != NULL) {
+
+			bfee_idx = bf_entry->bfee->idx +
+				((bf_entry->bfee->type == HAL_BFEE_MU) ?
+				  bf_obj->max_su_bfee_nr : 0);
+			/* Clear HW CR to avoid TxBF when sounding is abort */
+			rtw_hal_mac_ax_set_bf_entry(hal_info->mac,
+					bf_entry->band, (u8)macid_rsvd,
+					bfee_idx, bf_entry->bf_idx, 0);
+
+			_reset_sumu_entry(bf_entry->bfee);
+			_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+			list_del(&bf_entry->bfee->list);
+			_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+			if (bf_entry->bfee->type == HAL_BFEE_MU) {
+				_enqueue_idle_mu_entry(
+					hal_info, bf_obj, bf_entry->bfee);
+			} else {
+				_enqueue_idle_su_entry(
+					hal_info, bf_obj, bf_entry->bfee);
+			}
+		}
+		_reset_bf_entry(bf_entry);
+		_os_spinlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		list_del(&bf_entry->list);
+		_os_spinunlock(drv_priv, &bf_obj->bf_lock, _ps, NULL);
+		_enqueue_idle_bf_entry(hal_info, bf_obj, bf_entry);
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+	FUNCOUT();
+	return status;
+}
+
+/**
+ * hal_bf_query_idle_bf_entry
+ * 	Get available beamformee entry
+ * @hal_info: (struct hal_info_t *)
+ * @mu:  Is MU BFee ? 1 = MU / 0 = SU
+ * return :
+ * @hal_bf_entry:  hal_bf_entry pointer
+ **/
+struct hal_bf_entry *
+hal_bf_query_idle_bf_entry(
+	struct hal_info_t *hal_info,
+	bool mu)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_entry *bf_entry = NULL;
+	struct hal_sumu_entry *sumu_entry;
+
+	FUNCIN();
+
+	do {
+		if (bf_obj == NULL)
+			break;
+
+		bf_entry = _query_idle_bf_entry(hal_info, bf_obj);
+
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+			  "_query_idle_bf_entry bf_entry %p \n", bf_entry);
+
+		if (NULL == bf_entry)
+			break;
+
+		if (true == mu)
+			bf_entry->bfee = _query_idle_mu_entry(hal_info, bf_obj);
+		else
+			bf_entry->bfee = _query_idle_su_entry(hal_info, bf_obj);
+
+		if (NULL == bf_entry->bfee)
+			break;
+
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+			  "_query_idle_bf_entry sumu_entry %p \n", bf_entry->bfee);
+		sumu_entry = bf_entry->bfee;
+
+		status = _enqueue_busy_bf_entry(hal_info, bf_obj, bf_entry);
+		if (RTW_HAL_STATUS_SUCCESS != status) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "warring :hal_bf_entry add busy queue fail !!!\n");
+			break;
+		}
+
+		if (true == mu)
+			status = _enqueue_busy_mu_entry(
+					hal_info, bf_obj, sumu_entry);
+		else
+			status = _enqueue_busy_su_entry(
+					hal_info, bf_obj, sumu_entry);
+		if (RTW_HAL_STATUS_SUCCESS != status) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "warring :sumu_entry add busy queue fail !!!\n");
+			break;
+		}
+	} while (0);
+
+	if((status != RTW_HAL_STATUS_SUCCESS) && (bf_entry != NULL)) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "Get BF Entry Fail !!!\n");
+		hal_bf_release_target_bf_entry(hal_info, (void *)bf_entry);
+	}
+	FUNCOUT();
+	return bf_entry;
+}
+/**
+ * hal_bf_cfg_swbf_entry
+ * 	Fill BF Entry SW Content
+ * input:
+ * @sta: rtw_phl_stainfo_t
+ **/
+enum rtw_hal_status
+hal_bf_cfg_swbf_entry(struct rtw_phl_stainfo_t *sta, bool swap)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_bf_entry *bf_entry =
+		(struct hal_bf_entry *)sta->hal_sta->bf_entry;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "hal_bf_cfg_swbf_entry bf_entry->idx = 0x%x\n",
+		bf_entry->bf_idx);
+
+	bf_entry->macid = sta->macid;
+	bf_entry->aid12 = sta->aid;
+	bf_entry->band = sta->rlink->hw_band;
+	bf_entry->csi_buf = sta->hal_sta->bf_csi_buf;
+	if (swap) {
+		bf_entry->en_swap = true;
+		bf_entry->csi_buf_swap = sta->hal_sta->bf_csi_buf_swap;
+	}
+
+	return status;
+}
+
+/**
+ * hal_bf_set_entry_hwcfg
+ * 	Configure BF Entry HW Setting
+ * input
+ * @hal_info: (struct hal_info_t *)
+ * @entry: struct hal_bf_entry
+ **/
+enum rtw_hal_status hal_bf_set_entry_hwcfg(
+	struct hal_info_t *hal_info,
+	void *entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_entry *bf_entry = (struct hal_bf_entry *)entry;
+	u8 bfee_idx = 0;
+	u16 csi_buf = 0;
+
+	/*TODO: 8852A : SU : 0 ~ N-1;  MU : N ~ M*/
+	bfee_idx = bf_entry->bfee->idx +
+		((bf_entry->bfee->type == HAL_BFEE_MU) ?
+			bf_obj->max_su_bfee_nr : 0);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+		  "hal_bf_set_entry_hwcfg set bf sts idx = 0x%x \n", bfee_idx);
+
+	/*1. HALMAC Settings*/
+	if ((bf_entry->bfee->type == HAL_BFEE_MU) && bf_entry->en_swap) {
+		/*swap mode*/
+		if(0 == (bf_entry->couter % 2))
+			csi_buf = bf_entry->csi_buf&CSI_BUF_IDX_HW_MSK;
+		else
+			csi_buf = bf_entry->csi_buf_swap&CSI_BUF_IDX_HW_MSK;
+
+		status = rtw_hal_mac_ax_set_bf_entry(
+				hal_info->mac, bf_entry->band,
+				(u8)(bf_entry->macid&0xFF), bfee_idx,
+				bf_entry->bf_idx, csi_buf);
+
+		bf_entry->couter++;
+	} else {
+		status = rtw_hal_mac_ax_set_bf_entry(
+				hal_info->mac, bf_entry->band,
+				(u8)(bf_entry->macid&0xFF), bfee_idx,
+				bf_entry->bf_idx,
+				bf_entry->csi_buf&CSI_BUF_IDX_HW_MSK);
+	}
+	/*2.TODO: HALBB Settings if needed*/
+
+	return status;
+}
+
+/**
+ * hal_bf_update_entry_snd_sts
+ * 	Update BF Entry Sounding Status
+ * input
+ * @hal_info: (struct hal_info_t *)
+ * @entry: struct hal_bf_entry
+ **/
+void hal_bf_update_entry_snd_sts(struct hal_info_t *hal_info, void *entry)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_entry *bf_entry = (struct hal_bf_entry *)entry;
+	u8 bfee_idx = 0;
+	/*TODO: 8852A : SU : 0 ~ N-1;  MU : N ~ M*/
+	if (NULL == bf_entry) {
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[ERROR] hal_bf_update_entry_snd_sts BF_Entry = NULL\n");
+		return;
+	}
+	if (NULL == bf_entry->bfee) {
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+			  "[ERROR] rtw_hal_bf_get_entry_snd_sts bf_entry->macid = 0x%x\n", bf_entry->macid);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+			  "[ERROR] rtw_hal_bf_get_entry_snd_sts bf_entry->bfee = NULL\n");
+		return;
+	}
+	bfee_idx = bf_entry->bfee->idx +
+			((bf_entry->bfee->type == HAL_BFEE_MU) ?
+			bf_obj->max_su_bfee_nr : 0);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "hal_bf_update_entry_snd_sts idx = 0x%x \n", bfee_idx);
+
+	status = rtw_hal_mac_ax_get_snd_sts(hal_info->mac,
+							bf_entry->band, bfee_idx);
+
+	bf_entry->bfee->snd_sts = (status == RTW_HAL_STATUS_SUCCESS) ? 1 : 0;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "hal_bf_update_entry_snd_sts bf_entry->bfee->snd_sts = 0x%x \n",
+		  bf_entry->bfee->snd_sts);
+
+}
+
+enum rtw_hal_status
+hal_bf_hw_mac_deinit_bfee(struct hal_info_t *hal_info, u8 band)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+
+	do {
+		if (band >= 2) {
+			status = RTW_HAL_STATUS_FAILURE;
+			break;
+		}
+		if ((0 == (bf_obj->self_bf_cap[band]&BF_CAP_HE_BFEE)) &&
+		    (0 == (bf_obj->self_bf_cap[band]&BF_CAP_VHT_BFEE))) {
+			/* already disable */
+			break;
+		}
+		status = rtw_hal_mac_ax_deinit_bfee(hal_info->hal_com, band);
+
+		if (status != RTW_HAL_STATUS_SUCCESS)
+			break;
+
+		bf_obj->self_bf_cap[band] &= ~(BF_CAP_HE_BFEE|BF_CAP_VHT_BFEE);
+	} while (0);
+
+	return status;
+}
+
+/**
+ * rtw_hal_bf_get_entry_snd_sts
+ * 	Get BF Entry Sounding Status
+ * input
+ * @hal_info: (struct hal_info_t *)
+ * @entry: struct hal_bf_entry
+ * return
+ * 	rtw_hal_status : RTW_HAL_STATUS_FAILURE or RTW_HAL_STATUS_SUCCESS
+ **/
+enum rtw_hal_status
+rtw_hal_bf_get_entry_snd_sts(void *entry)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_bf_entry *bf_entry = (struct hal_bf_entry *)entry;
+	if (NULL == bf_entry) {
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "[ERROR] rtw_hal_bf_get_entry_snd_sts BF_Entry = NULL\n");
+		return hstatus;
+	}
+	if (NULL == bf_entry->bfee) {
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+			  "[ERROR] rtw_hal_bf_get_entry_snd_sts bf_entry->macid = 0x%x\n", bf_entry->macid);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+			  "[ERROR] rtw_hal_bf_get_entry_snd_sts bf_entry->bfee = NULL\n");
+		return hstatus;
+	}
+
+	if (1 == bf_entry->bfee->snd_sts)
+		hstatus = RTW_HAL_STATUS_SUCCESS;
+
+	return hstatus;
+}
+
+/**
+ * hal_bf_hw_mac_init_bfee
+ * 	Initialize BFee HW Settings
+ * input
+ * @hal_info: struct hal_info_t
+ * @band: Band 0 / Band 1
+ **/
+enum rtw_hal_status hal_bf_hw_mac_init_bfee(
+	struct hal_info_t *hal_info,
+	u8 band)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+
+	do {
+		if (band >= 2) {
+			status = RTW_HAL_STATUS_FAILURE;
+			break;
+		}
+		status = rtw_hal_mac_ax_init_bf_role(
+				hal_info->hal_com, HAL_BF_ROLE_BFEE, band);
+
+		if(status != RTW_HAL_STATUS_SUCCESS)
+			break;
+
+		/*TODO: Check Wireless Mode*/
+		bf_obj->self_bf_cap[band] |= BF_CAP_HE_BFEE|BF_CAP_VHT_BFEE;
+
+	} while (0);
+
+	return status;
+}
+#ifdef RTW_WKARD_DYNAMIC_BFEE_CAP
+enum rtw_hal_status
+rtw_hal_bf_bfee_ctrl(void *hal, u8 band, bool ctrl)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	bool cur_bfee_cap = false;
+
+	if ((bf_obj->self_bf_cap[band] & BF_CAP_HE_BFEE) ||
+	    (bf_obj->self_bf_cap[band] & BF_CAP_VHT_BFEE)) {
+		    cur_bfee_cap = true;
+	}
+
+	/* BB Workaround */
+	if ((ctrl != cur_bfee_cap) && (true == ctrl)) {
+		/* From Tx to Rx (need Enable BFee) */
+		rtw_hal_bb_dcr_en(hal_info, true);
+	} else if ((ctrl != cur_bfee_cap) && (false == ctrl)) {
+		rtw_hal_bb_dcr_en(hal_info, false);
+	}
+
+	if ((ctrl == true) &&
+	    (false == rtw_hal_bb_csi_rsp(hal_info))) {
+		ctrl = false;
+	}
+
+	if (ctrl != cur_bfee_cap) {
+		status = ((true == ctrl) ?
+			   hal_bf_hw_mac_init_bfee(hal_info, band) :
+			   hal_bf_hw_mac_deinit_bfee(hal_info, band));
+	}
+
+	return status;
+}
+#endif
+
+/**
+ * hal_bf_set_bfee_csi_para
+ * 	Set self bfee hw capability.
+ * input:
+ * @hal_info: struct hal_info_t *
+ * @cr_cctl: set bfee capabiliy method
+ * 		true = by register  (only support 2 sets, Port 0 / Port 1-4 )
+ * 		false = by cmac control table
+ * @sta: struct rtw_phl_stainfo_t * (self)
+ */
+enum rtw_hal_status hal_bf_set_bfee_csi_para(struct hal_info_t *hal_info,
+	bool cr_cctl, struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	u32 rrsc = BIT(HAL_BF_RRSC_6M) | BIT(HAL_BF_RRSC_24M);
+
+	if (false == cr_cctl) {
+		status = rtw_hal_mac_ax_bfee_para_reg(hal_info->mac, sta);
+	} else {
+		status = rtw_hal_mac_ax_bfee_para_cctl(hal_info->mac, sta);
+	}
+	/* mac/bb csi rate settings initialize */
+	if (RTW_HAL_STATUS_SUCCESS == status) {
+		/* Initialize CSI rate RA parameters */
+		sta->hal_sta->ra_info.fixed_csi_rate_en = false;
+		sta->hal_sta->ra_info.ra_csi_rate_en = true;
+		sta->hal_sta->ra_info.band_num = sta->rlink->hw_band;
+		if (sta->chandef.bw >= CHANNEL_WIDTH_80)
+			sta->hal_sta->ra_info.csi_rate.bw = HAL_RATE_BW_80;
+		else if (sta->chandef.bw == CHANNEL_WIDTH_40)
+			sta->hal_sta->ra_info.csi_rate.bw = HAL_RATE_BW_40;
+		else
+			sta->hal_sta->ra_info.csi_rate.bw = HAL_RATE_BW_20;
+		sta->hal_sta->ra_info.csi_rate.gi_ltf = RTW_GILTF_LGI_4XHE32;
+		sta->hal_sta->ra_info.csi_rate.mcs_ss_idx = 5;
+		if (sta->wmode & WLAN_MD_11N) {
+			rrsc |= (BIT(HAL_BF_RRSC_HT_MSC0) |
+				 BIT(HAL_BF_RRSC_HT_MSC3) |
+				 BIT(HAL_BF_RRSC_HT_MSC5));
+			sta->hal_sta->ra_info.csi_rate.mode = RTW_HT_MODE;
+		}
+		if (sta->wmode & WLAN_MD_11AC) {
+			rrsc |= (BIT(HAL_BF_RRSC_VHT_MSC0) |
+				 BIT(HAL_BF_RRSC_VHT_MSC3) |
+				 BIT(HAL_BF_RRSC_VHT_MSC5));
+			sta->hal_sta->ra_info.csi_rate.mode = RTW_VHT_MODE;
+		}
+		if (sta->wmode & WLAN_MD_11AX) {
+			rrsc |= (BIT(HAL_BF_RRSC_HE_MSC0) |
+				 BIT(HAL_BF_RRSC_HE_MSC3) |
+				 BIT(HAL_BF_RRSC_HE_MSC5));
+			sta->hal_sta->ra_info.csi_rate.mode = RTW_HE_MODE;
+		}
+		/* Initialize mac rrsc function */
+		rtw_hal_mac_ax_bfee_set_csi_rrsc(hal_info->mac,
+		                                 sta->rlink->hw_band,
+		                                 rrsc);
+
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, " set bfee csi rrsc =  0x%x\n", rrsc);
+	}
+
+	return status;
+}
+
+
+
+/**
+ * hal_bf_hw_mac_init_bfer
+ * 	Initialize BFer HW Settings
+ * @hal_info: struct hal_info_t
+ * @band: Band 0 / Band 1
+ **/
+enum rtw_hal_status hal_bf_hw_mac_init_bfer(
+	struct hal_info_t *hal_info,
+	u8 band)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+
+	do {
+		if (band >= 2) {
+			status = RTW_HAL_STATUS_FAILURE;
+			break;
+		}
+		status = rtw_hal_mac_ax_init_bf_role(hal_info->hal_com,
+						     HAL_BF_ROLE_BFER, band);
+		if(status != RTW_HAL_STATUS_SUCCESS)
+			break;
+		/*TODO: Check Wireless Mode*/
+		bf_obj->self_bf_cap[band] |= BF_CAP_HE_BFER|BF_CAP_VHT_BFER;
+
+	} while (0);
+
+	return status;
+}
+
+/**
+ * rtw_hal_bf_chk_bf_type
+ * 	Check the STA's BF Entry Type
+ * @hal_info: struct hal_info_t
+ * @sta: rtw_phl_stainfo_t *
+ * @mu: check condition MU or SU?
+ * return :
+ * @ret: true : bf entry type is same to flag(mu)
+ *		false : bf entry is not same to flag(mu) or bf entry is NULL
+ **/
+bool rtw_hal_bf_chk_bf_type(
+	void *hal_info,
+	struct rtw_phl_stainfo_t *sta,
+	bool mu)
+{
+	struct hal_bf_entry *bf_entry =
+		(struct hal_bf_entry *)sta->hal_sta->bf_entry;
+	bool ret = false;
+	do {
+		if (NULL == bf_entry)
+			break;
+
+		if (bf_entry->bfee == NULL)
+			break;
+
+		if (true == mu) {
+			if (HAL_BFEE_MU != bf_entry->bfee->type)
+				break;
+		} else {
+			if (HAL_BFEE_SU != bf_entry->bfee->type)
+				break;
+		}
+
+		ret = true;
+	} while(0);
+
+	return ret;
+}
+
+u8 rtw_hal_bf_get_sumu_idx(void *hal, void *entry)
+{
+	struct hal_bf_entry *bf_entry = (struct hal_bf_entry *)entry;
+	u8 ret = 0xFF;
+	if (bf_entry->bfee != NULL)
+		ret = bf_entry->bfee->idx;
+
+	return ret;
+}
+
+
+void
+rtw_hal_bf_preset_mu_ba_info(
+	void *hal,
+	struct rtw_phl_stainfo_t *psta,
+	void *hal_ba_info)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct rtw_hal_muba_info *ba_info =
+		(struct rtw_hal_muba_info *)hal_ba_info;
+
+	if (1 == bf_obj->fixed_para.fix_ba_info.fix_ba) {
+		/*Fixed BA info*/
+		_os_mem_cpy(drv_priv, ba_info, &bf_obj->fixed_para.fix_ba_info, sizeof(*ba_info));
+	} else {
+		/* TODO:  Config BA by STA Capabiliy or default value*/
+		ba_info->fix_ba = 0;
+		ba_info->ru_psd = 0;
+		ba_info->tf_rate = RTW_DATA_RATE_OFDM24;
+		ba_info->rf_gain_fix = 0;
+		ba_info->rf_gain_idx = 0;
+		ba_info->tb_ppdu_bw = (psta->chandef.bw == CHANNEL_WIDTH_80) ? 2 : 0;
+		ba_info->dcm = 0;
+		ba_info->ss = 0;
+		ba_info->mcs = 3;
+		ba_info->gi_ltf = 2;
+		ba_info->doppler = 0;
+		ba_info->stbc = 0;
+		ba_info->sta_coding = 0;
+		ba_info->tb_t_pe_nom = 0;
+		ba_info->pr20_bw_en = 0;
+		ba_info->ma_type = 0;
+	}
+}
+
+void
+rtw_hal_bf_set_txmu_para(void *hal, u8 gid , u8 en,
+	enum rtw_hal_protection_type rts_type,
+	enum rtw_hal_ack_resp_type ack_type)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_fixed_m_para *para = &bf_obj->fixed_para;
+
+	para->fix_m = (1 == en) ? HAL_BF_FIX_M_MU : HAL_BF_FIX_M_DISABLE;
+	para->gid = gid;
+	para->fix_resp = en;
+	para->fix_prot = en;
+	if (en) {
+		para->prot_type = rts_type;
+		para->resp_type = ack_type;
+	}
+}
+
+enum rtw_hal_status
+rtw_hal_bf_set_fix_mode(void *hal, bool mu, bool he)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_bf_obj *bf_obj = (struct hal_bf_obj *)hal_com->bf_obj;
+	struct hal_bf_fixed_m_para *para = &bf_obj->fixed_para;
+
+
+	if (mu) {
+		status = rtw_hal_mac_ax_set_mu_fix_mode(
+			hal_info->mac,
+			para->gid,
+			para->prot_type,
+			para->resp_type,
+			(para->fix_m == HAL_BF_FIX_M_MU) ? true : false,
+			he,
+			para->fix_resp,
+			para->fix_prot);
+	} else {
+		/*TODO: Force SU*/
+	}
+	return status;
+}
+
+enum rtw_hal_status
+hal_bf_set_mu_sta_fw(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_bf_entry *bf_entry =
+			(struct hal_bf_entry *)sta->hal_sta->bf_entry;
+	struct hal_sumu_entry *mu_entry = NULL;
+
+	do {
+		if (NULL == bf_entry)
+			break;
+
+		if (NULL == bf_entry->bfee)
+			break;
+
+		mu_entry = bf_entry->bfee;
+
+		if (HAL_BFEE_MU != mu_entry->type)
+			break;
+
+		status = rtw_hal_mac_ax_mu_sta_upd(
+				hal_info->mac,
+				(u8)(sta->macid&0xFF),
+				mu_entry->idx,
+				sta->hal_sta->prot_type,
+				sta->hal_sta->resp_type,
+				sta->hal_sta->mugrp_bmp);
+
+	} while (0);
+
+	return status;
+}
+
+void rtw_hal_beamform_set_vht_gid(void *hal, u8 band,
+				  struct rtw_phl_gid_pos_tbl *tbl)
+{
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "--> %s()\n", __func__);
+	if (RTW_HAL_STATUS_SUCCESS !=
+		rtw_hal_mac_bfee_set_vht_gid(hal, band, tbl)) {
+		PHL_INFO("%s : Error to set VHT GID Position to MAC !!! \n",
+			 __func__);
+	}
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "<-- %s()\n", __func__);
+}
+
+enum rtw_hal_status rtw_hal_beamform_set_aid(void *hal, struct rtw_phl_stainfo_t *sta, u16 aid)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "--> %s() : set aid = %d\n", __func__, aid);
+
+	/*TODO: halmac need provide api that only change AID */
+	status = rtw_hal_mac_addr_cam_set_aid(hal_info, sta, aid);
+
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_mac_addr_cam_change_entry failed\n");
+	}
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "<-- %s()\n", __func__);
+
+	return status;
+}
+#endif
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_beamform.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_beamform.h
new file mode 100644
index 000000000000..b3bb3d0f6972
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_beamform.h
@@ -0,0 +1,208 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __HAL_BEAMFORM_H__
+#define __HAL_BEAMFORM_H__
+
+#define BF_MU_IDX_INVALID 0xFF
+
+#define BF_CAP_HT_BFEE BIT(0)
+#define BF_CAP_HT_BFER BIT(1)
+#define BF_CAP_VHT_BFEE BIT(2)
+#define BF_CAP_VHT_BFER BIT(3)
+#define BF_CAP_HE_BFEE BIT(4)
+#define BF_CAP_HE_BFER BIT(5)
+
+#define IS_SUPPORT_ACT_AS_BFER(_cap) \
+	(_cap & BF_CAP_HE_BFER) || (_cap & BF_CAP_VHT_BFER) \
+	|| (_cap & BF_CAP_HT_BFER)
+
+#define IS_SUPPORT_ACT_AS_BFEE(_cap) \
+	(_cap & BF_CAP_HE_BFEE) || (_cap & BF_CAP_VHT_BFEE) \
+	|| (_cap & BF_CAP_HT_BFEE)
+
+#ifdef CONFIG_PHL_BEAMFORM
+//B[0:7] = {6M, 9M, 12M, 18M, 24M, 36M, 48M, 54M}
+//B[8:15] = HT MCS0~MCS7
+//B[16:23] = VHT 1SSMCS0~MCS7
+//B[24:31] = HE 1SSMCS0~MCS7
+enum hal_bf_rrsc_rata {
+	HAL_BF_RRSC_6M = 0,
+	HAL_BF_RRSC_9M = 1,
+	HAL_BF_RRSC_12M,
+	HAL_BF_RRSC_18M,
+	HAL_BF_RRSC_24M,
+	HAL_BF_RRSC_36M,
+	HAL_BF_RRSC_48M,
+	HAL_BF_RRSC_54M,
+	HAL_BF_RRSC_HT_MSC0,
+	HAL_BF_RRSC_HT_MSC1,
+	HAL_BF_RRSC_HT_MSC2,
+	HAL_BF_RRSC_HT_MSC3,
+	HAL_BF_RRSC_HT_MSC4,
+	HAL_BF_RRSC_HT_MSC5,
+	HAL_BF_RRSC_HT_MSC6,
+	HAL_BF_RRSC_HT_MSC7,
+	HAL_BF_RRSC_VHT_MSC0,
+	HAL_BF_RRSC_VHT_MSC1,
+	HAL_BF_RRSC_VHT_MSC2,
+	HAL_BF_RRSC_VHT_MSC3,
+	HAL_BF_RRSC_VHT_MSC4,
+	HAL_BF_RRSC_VHT_MSC5,
+	HAL_BF_RRSC_VHT_MSC6,
+	HAL_BF_RRSC_VHT_MSC7,
+	HAL_BF_RRSC_HE_MSC0,
+	HAL_BF_RRSC_HE_MSC1,
+	HAL_BF_RRSC_HE_MSC2,
+	HAL_BF_RRSC_HE_MSC3,
+	HAL_BF_RRSC_HE_MSC4,
+	HAL_BF_RRSC_HE_MSC5,
+	HAL_BF_RRSC_HE_MSC6,
+	HAL_BF_RRSC_HE_MSC7 = 31,
+	HAL_BF_RRSC_MAX = 32
+};
+
+enum hal_bf_role {
+	HAL_BF_ROLE_BFEE,
+	HAL_BF_ROLE_BFER
+};
+
+enum hal_bfee_type {
+	HAL_BFEE_SU,
+	HAL_BFEE_MU
+};
+
+struct hal_sumu_entry {
+	_os_list list;
+	enum hal_bfee_type type;
+	u8 idx;
+	u8 snd_sts; /*0: Fail or Not Sound, 1: Sound Success*/
+};
+struct hal_bf_entry {
+	_os_list list;
+	u8 bf_idx;
+	u16 macid;
+	u16 aid12;
+	u8 band;
+	u16 csi_buf;
+	/* csi_buf for swap mode */
+	bool en_swap;
+	u16 csi_buf_swap;
+	u8 couter; /*for swap*/
+	struct hal_sumu_entry *bfee;
+};
+
+enum hal_bf_forced_mode{
+	HAL_BF_FIX_M_DISABLE = 0,
+	HAL_BF_FIX_M_SU = 1,
+	HAL_BF_FIX_M_MU = 2
+};
+struct hal_bf_fixed_m_para {
+	enum hal_bf_forced_mode fix_m;
+	u8 gid;
+	u8 fix_resp;
+	u8 fix_prot;
+	enum rtw_hal_protection_type prot_type;
+	enum rtw_hal_ack_resp_type resp_type;
+	struct rtw_hal_muba_info fix_ba_info;
+};
+
+struct hal_bf_obj {
+	/*tx bf entry*/
+	u8 max_bf_entry_nr;
+	struct hal_bf_entry *bf_entry;
+	_os_list bf_idle_list;
+	_os_list bf_busy_list;
+	u8 num_idle_bf_entry;
+
+	/*su entry*/
+	u8 max_su_bfee_nr;
+	struct hal_sumu_entry *su_entry;
+	_os_list su_idle_list;
+	_os_list su_busy_list;
+	u8 num_idle_su_entry;
+
+	/*mu entry*/
+	u8 max_mu_bfee_nr;
+	struct hal_sumu_entry *mu_entry;
+	_os_list mu_idle_list;
+	_os_list mu_busy_list;
+	u8 num_idle_mu_entry;
+
+	_os_lock bf_lock;
+
+	u8 self_bf_cap[2];
+	/*for fixed mode*/
+	struct hal_bf_fixed_m_para fixed_para;
+};
+
+
+enum rtw_hal_status hal_bf_init(
+	struct hal_info_t *hal_info,
+	u8 bf_entry_nr,
+	u8 su_entry_nr,
+	u8 mu_entry_nr);
+
+void hal_bf_deinit(struct hal_info_t *hal_info);
+
+
+struct hal_bf_entry *
+hal_bf_query_idle_bf_entry(
+	struct hal_info_t *hal_info,
+	bool mu);
+
+
+enum rtw_hal_status
+hal_bf_release_target_bf_entry(
+	struct hal_info_t *hal_info,
+	void *entry);
+
+
+enum rtw_hal_status hal_bf_hw_mac_init_bfee(
+	struct hal_info_t *hal_info,
+	u8 band);
+
+enum rtw_hal_status hal_bf_hw_mac_init_bfer(
+	struct hal_info_t *hal_info,
+	u8 band);
+
+enum rtw_hal_status hal_bf_set_entry_hwcfg(
+	struct hal_info_t *hal_info, void *entry);
+
+void hal_bf_update_entry_snd_sts(struct hal_info_t *hal_info, void *entry);
+
+enum rtw_hal_status hal_bf_cfg_swbf_entry(struct rtw_phl_stainfo_t *sta,
+					  bool swap);
+
+enum rtw_hal_status
+hal_bf_set_mu_sta_fw(void *hal, struct rtw_phl_stainfo_t *sta);
+
+enum rtw_hal_status
+rtw_hal_bf_set_fix_mode(void *hal, bool mu, bool he);
+
+bool rtw_hal_bf_chk_bf_type(void *hal_info,
+	struct rtw_phl_stainfo_t *sta, bool mu);
+
+enum rtw_hal_status
+hal_bf_set_bfee_csi_para(struct hal_info_t *hal_info, bool cr_cctl,
+			 struct rtw_phl_stainfo_t *sta);
+
+#else
+#define hal_bf_init(_hal_info, _bf_entry_nr, _su_entry_nr, _mu_entry_nr) RTW_HAL_STATUS_SUCCESS
+#define hal_bf_deinit(_hal_info)
+#define hal_bf_hw_mac_init_bfee(_hal_info, _band) RTW_HAL_STATUS_SUCCESS
+#define hal_bf_hw_mac_init_bfer(_hal_info, _band) RTW_HAL_STATUS_SUCCESS
+#define hal_bf_set_bfee_csi_para(_hal_info, _cr_cctl, _sta) RTW_HAL_STATUS_SUCCESS
+#endif
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_c2h.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_c2h.c
new file mode 100644
index 000000000000..f415a3bf4f57
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_c2h.c
@@ -0,0 +1,89 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_C2H_C_
+#include "hal_headers.h"
+
+static void _hal_c2h_process_complete(void* priv, struct phl_msg* msg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)priv;
+	void *drv = hal_to_drvpriv(hal_info);
+
+	if (msg->inbuf && msg->inlen)
+		_os_kmem_free(drv, msg->inbuf, msg->inlen);
+}
+
+struct hal_c2h_hdl c2h_list[] =
+{
+	{C2H_CAT_OUTSRC, C2H_CLS_PHYDM_MIN, C2H_CLS_PHYDM_MAX,
+		rtw_hal_bb_process_c2h, NULL},
+	{C2H_CAT_OUTSRC, C2H_CLS_RF_MIN, C2H_CLS_RF_MAX,
+		rtw_hal_rf_process_c2h, NULL},
+	{C2H_CAT_OUTSRC, C2H_CLS_RF_MIN, C2H_CLS_RF_MAX,
+		rtw_hal_rf_process_c2h},
+	{C2H_CAT_OUTSRC, C2H_CLS_BTC_MIN, C2H_CLS_BTC_MAX,
+		rtw_hal_btc_process_c2h, NULL},
+	{C2H_CAT_MAC, C2H_CLS_MAC_MIN, C2H_CLS_MAC_MAX,
+		rtw_hal_mac_process_c2h, rtw_hal_mac_upd_c2h},
+	{C2H_CAT_MAX, C2H_CLS_MAX, C2H_CLS_MAX,
+		NULL, NULL}
+};
+
+void hal_c2h_post_process(void *phl, void *hal, void *c2h)
+{
+	struct rtw_phl_com_t *phl_com = (struct rtw_phl_com_t *)phl;
+	struct rtw_c2h_info *c = (struct rtw_c2h_info *)c2h;
+	struct c2h_evt_msg c2h_msg;
+	u8 i = 0;
+	u32 evt_id = 0;
+
+	for (i = 0; C2H_CAT_MAX != c2h_list[i].cat; i++) {
+		if (c->c2h_cat == c2h_list[i].cat) {
+			if (c->c2h_class >= c2h_list[i].cls_min &&
+				c->c2h_class <= c2h_list[i].cls_max) {
+				evt_id = c2h_list[i].c2h_hdl(hal, c2h, &c2h_msg);
+
+				if (c2h_list[i].c2h_buf_wb)
+					c2h_list[i].c2h_buf_wb(hal, c, evt_id, &c2h_msg);
+				else
+					c->content_len = 0;
+
+				break;
+			}
+		}
+	}
+
+	if (evt_id > 0) {
+		/* submit phl msg hub */
+		struct phl_msg msg = {0};
+		struct phl_msg_attribute attr = {0};
+
+		if (c->content_len > 0) {
+			msg.inbuf = c->content;
+			msg.inlen = c->content_len;
+			attr.completion.completion = _hal_c2h_process_complete;
+			attr.completion.priv = hal;
+		}
+
+		SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_RX);
+		SET_MSG_EVT_ID_FIELD(msg.msg_id, (u16)evt_id);
+		if (rtw_phl_msg_hub_hal_send(phl_com, &attr, &msg) !=
+			RTW_PHL_STATUS_SUCCESS) {
+			if (c->content_len > 0)
+				_hal_c2h_process_complete(hal, &msg);
+			PHL_ERR("%s rtw_phl_msg_hub_hal_send failed\n", __func__);
+		}
+	}
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_c2h.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_c2h.h
new file mode 100644
index 000000000000..6794b3da5840
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_c2h.h
@@ -0,0 +1,21 @@
+
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_C2H_H_
+#define _HAL_C2H_H_
+
+void hal_c2h_post_process(void *phl, void *hal, void *c2h);
+
+#endif /*_HAL_C2H_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cam.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cam.c
new file mode 100644
index 000000000000..6d3e98602b24
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cam.c
@@ -0,0 +1,72 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_CAM_C_
+#include "hal_headers.h"
+
+/**
+ * This function calls halmac api to configure security cam
+ * @hal: see struct hal_info_t
+ * @sta: sta info of specified address cam entry
+ * @type: encryption type of this key
+ * @ext_key: whether 256bit key or not
+ * @spp: spp mode
+ * @keyid: key index
+ * @keytype: specify unicast or multicast or bip key type
+ * @keybuf: key buffer length according to ext_key
+ * return enum RTW_HAL_STATUS
+ */
+enum rtw_hal_status
+rtw_hal_set_key(void *hal, struct rtw_phl_stainfo_t *sta, u8 type, u8 ext_key, u8 spp,
+				u8 keyid, u8 keytype, u8 *keybuf)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	u8 macid = (u8)sta->macid;
+
+	if (keybuf == NULL) {
+		/* delete key */
+		hal_status = rtw_hal_mac_delete_key(hal_info, macid, type, ext_key,
+					spp, keyid, keytype);
+	} else {
+		/* add key */
+		if (ext_key) {
+			hal_status = rtw_hal_mac_add_key(hal_info, macid, type, ext_key,
+					spp, keyid, keytype, keybuf);
+			hal_status = rtw_hal_mac_add_key(hal_info, macid, type, ext_key, spp,
+					keyid, keytype, (keybuf+16));
+		} else {
+			hal_status = rtw_hal_mac_add_key(hal_info, macid, type, 0,
+					spp, keyid, keytype, keybuf);
+		}
+	}
+
+	return hal_status;
+}
+
+u32
+rtw_hal_search_key_idx(void *hal, struct rtw_phl_stainfo_t *sta,
+						u8 keyid, u8 keytype)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u32 sec_cam_idx = 0;
+	u8 macid = (u8)sta->macid;
+
+	sec_cam_idx = rtw_hal_mac_search_key_idx(hal_info,
+						 macid,
+						 keyid,
+						 keytype);
+
+	return sec_cam_idx;
+}
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cap.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cap.c
new file mode 100644
index 000000000000..a17eee69b02e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cap.c
@@ -0,0 +1,833 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include "hal_headers.h"
+
+static void _hal_bus_cap_pre_decision(struct rtw_phl_com_t *phl_com,
+					 	void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct bus_sw_cap_t *bus_sw = &phl_com->bus_sw_cap;
+	struct bus_hw_cap_t *bus_hw = &hal_com->bus_hw_cap;
+	struct bus_cap_t *bus_cap = &hal_com->bus_cap;
+
+#ifdef CONFIG_PCI_HCI
+	bus_cap->l0s_ctrl = bus_sw->l0s_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->l0s_ctrl : bus_sw->l0s_ctrl;
+	bus_cap->l1_ctrl = bus_sw->l1_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->l1_ctrl : bus_sw->l1_ctrl;
+	bus_cap->l1ss_ctrl = bus_sw->l1ss_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->l1ss_ctrl : bus_sw->l1ss_ctrl;
+	bus_cap->wake_ctrl = bus_sw->wake_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->wake_ctrl : bus_sw->wake_ctrl;
+	bus_cap->crq_ctrl = bus_sw->crq_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->crq_ctrl : bus_sw->crq_ctrl;
+
+
+	bus_cap->l0sdly_ctrl = bus_sw->l0sdly_ctrl == RTW_PCIE_BUS_ASPM_DLY_IGNORE ?
+		bus_hw->l0sdly_ctrl : bus_sw->l0sdly_ctrl;
+	bus_cap->l1dly_ctrl = bus_sw->l1dly_ctrl == RTW_PCIE_BUS_ASPM_DLY_IGNORE ?
+		bus_hw->l1dly_ctrl : bus_sw->l1dly_ctrl;
+	bus_cap->clkdly_ctrl = bus_sw->clkdly_ctrl == RTW_PCIE_BUS_ASPM_DLY_IGNORE ?
+		bus_hw->clkdly_ctrl : bus_sw->clkdly_ctrl;
+
+	bus_cap->txbd_num = (bus_sw->txbd_num) ?
+		((bus_sw->txbd_num > bus_hw->max_txbd_num) ?
+		 bus_hw->max_txbd_num : bus_sw->txbd_num) :
+		bus_hw->max_txbd_num;
+	bus_cap->read_txbd_th = bus_cap->txbd_num >> bus_sw->read_txbd_lvl;
+
+
+	/* RXBD NUM */
+	bus_cap->rxbd_num = (bus_sw->rxbd_num) ?
+		((bus_sw->rxbd_num > bus_hw->max_rxbd_num) ?
+		 bus_hw->max_rxbd_num : bus_sw->rxbd_num) :
+		bus_hw->max_rxbd_num;
+	/* RPBD NUM */
+	bus_cap->rpbd_num = (bus_sw->rpbd_num) ?
+		((bus_sw->rpbd_num > bus_hw->max_rpbd_num) ?
+		 bus_hw->max_rpbd_num : bus_sw->rpbd_num) :
+		bus_hw->max_rpbd_num;
+	/* RXBUF NUM */
+	/* rx buffer number must be larger or equal to rx bd number */
+	bus_cap->rxbuf_num = (bus_sw->rxbuf_num < bus_cap->rxbd_num) ?
+		 (bus_cap->rxbd_num + RTW_RX_BUF_BD_MIN_DIFF) : bus_sw->rxbuf_num;
+	/* RPBUF NUM */
+	bus_cap->rpbuf_num = (bus_sw->rpbuf_num < bus_cap->rpbd_num) ?
+		 (bus_cap->rpbd_num + RTW_RX_BUF_BD_MIN_DIFF) : bus_sw->rpbuf_num;
+	/* RXBUF SIZE */
+	bus_cap->rxbuf_size = (bus_sw->rxbuf_size) ?
+		((bus_sw->rxbuf_size > bus_hw->max_rxbuf_size) ?
+		 bus_hw->max_rxbuf_size : bus_sw->rxbuf_size) :
+		bus_hw->max_rxbuf_size;
+	/* RPBUF SIZE */
+	bus_cap->rpbuf_size = (bus_sw->rpbuf_size) ?
+		((bus_sw->rpbuf_size > bus_hw->max_rpbuf_size) ?
+		 bus_hw->max_rpbuf_size : bus_sw->rpbuf_size) :
+		bus_hw->max_rpbuf_size;
+
+	bus_cap->ltr_act.ctrl = bus_sw->ltr_act.ctrl;
+	bus_cap->ltr_act.val = bus_sw->ltr_act.val;
+	bus_cap->ltr_idle.ctrl = bus_sw->ltr_idle.ctrl;
+	bus_cap->ltr_idle.val = bus_sw->ltr_idle.val;
+	bus_cap->ltr_init_state = bus_sw->ltr_init_state;
+	bus_cap->ltr_sw_ctrl = bus_hw->ltr_sw_ctrl ?
+		(bus_sw->ltr_sw_ctrl ? true : false) : false;
+	bus_cap->ltr_hw_ctrl = bus_hw->ltr_hw_ctrl ?
+		(bus_sw->ltr_hw_ctrl ? true : false) : false;
+
+#ifdef RTW_WKARD_DYNAMIC_PCIE_GEN
+	bus_cap->pcie_gen_dm_en = bus_sw->pcie_gen_dm_en;
+#endif
+
+#elif defined (CONFIG_USB_HCI)
+	bus_cap->tx_buf_size = bus_sw->tx_buf_size ?
+		bus_sw->tx_buf_size : bus_hw->tx_buf_size;
+	bus_cap->tx_buf_num = bus_sw->tx_buf_num ?
+		bus_sw->tx_buf_num : bus_hw->tx_buf_num;
+	bus_cap->tx_mgnt_buf_size = bus_sw->tx_mgnt_buf_size ?
+		bus_sw->tx_mgnt_buf_size : bus_hw->tx_mgnt_buf_size;
+	bus_cap->tx_mgnt_buf_num = bus_sw->tx_mgnt_buf_num ?
+		bus_sw->tx_mgnt_buf_num : bus_hw->tx_mgnt_buf_num;
+	bus_cap->tx_h2c_buf_num = bus_sw->tx_h2c_buf_num ?
+		bus_sw->tx_h2c_buf_num : bus_hw->tx_h2c_buf_num;
+	bus_cap->rx_buf_size = bus_sw->rx_buf_size ?
+		bus_sw->rx_buf_size : bus_hw->rx_buf_size;
+	bus_cap->rx_buf_align_size = bus_sw->rx_buf_align_size ?
+		bus_sw->rx_buf_align_size : bus_hw->rx_buf_align_size;
+	bus_cap->rx_buf_num = bus_sw->rx_buf_num ?
+		bus_sw->rx_buf_num : bus_hw->rx_buf_num;
+	bus_cap->in_token_num = bus_sw->in_token_num ?
+		bus_sw->in_token_num : bus_hw->in_token_num;
+#elif defined(CONFIG_SDIO_HCI)
+	bus_cap->tx_buf_size = bus_sw->tx_buf_size ?
+				bus_sw->tx_buf_size : bus_hw->tx_buf_size;
+	bus_cap->tx_buf_num = bus_sw->tx_buf_num ?
+				bus_sw->tx_buf_num : bus_hw->tx_buf_num;
+	bus_cap->tx_mgnt_buf_size = bus_sw->tx_mgnt_buf_size ?
+			bus_sw->tx_mgnt_buf_size : bus_hw->tx_mgnt_buf_size;
+	bus_cap->tx_mgnt_buf_num = bus_sw->tx_mgnt_buf_num ?
+			bus_sw->tx_mgnt_buf_num : bus_hw->tx_mgnt_buf_num;
+	bus_cap->rx_buf_size = bus_sw->rx_buf_size ?
+				bus_sw->rx_buf_size : bus_hw->rx_buf_size;
+	bus_cap->rx_buf_num = bus_sw->rx_buf_num ?
+				bus_sw->rx_buf_num : bus_hw->rx_buf_num;
+#endif
+}
+
+#ifdef CONFIG_PCI_HCI
+static void _hal_bus_final_cap_decision(struct rtw_phl_com_t *phl_com,
+			struct rtw_hal_com_t *hal_com)
+{
+	struct bus_sw_cap_t *bus_sw = &phl_com->bus_sw_cap;
+	struct bus_hw_cap_t *bus_hw = &hal_com->bus_hw_cap;
+	struct bus_cap_t *bus_cap = &hal_com->bus_cap;
+
+	bus_cap->l0s_ctrl = bus_sw->l0s_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->l0s_ctrl : bus_sw->l0s_ctrl;
+	bus_cap->l1_ctrl = bus_sw->l1_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->l1_ctrl : bus_sw->l1_ctrl;
+	bus_cap->l1ss_ctrl = bus_sw->l1ss_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->l1ss_ctrl : bus_sw->l1ss_ctrl;
+	bus_cap->wake_ctrl = bus_sw->wake_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->wake_ctrl : bus_sw->wake_ctrl;
+	bus_cap->crq_ctrl = bus_sw->crq_ctrl >= RTW_PCIE_BUS_FUNC_DEFAULT ?
+		bus_hw->crq_ctrl : bus_sw->crq_ctrl;
+
+	bus_cap->l0sdly_ctrl = bus_sw->l0sdly_ctrl == RTW_PCIE_BUS_ASPM_DLY_IGNORE ?
+		bus_hw->l0sdly_ctrl : bus_sw->l0sdly_ctrl;
+	bus_cap->l1dly_ctrl = bus_sw->l1dly_ctrl == RTW_PCIE_BUS_ASPM_DLY_IGNORE ?
+		bus_hw->l1dly_ctrl : bus_sw->l1dly_ctrl;
+	bus_cap->clkdly_ctrl = bus_sw->clkdly_ctrl == RTW_PCIE_BUS_ASPM_DLY_IGNORE ?
+		bus_hw->clkdly_ctrl : bus_sw->clkdly_ctrl;
+
+	bus_cap->txbd_num = (bus_sw->txbd_num) ?
+		((bus_sw->txbd_num > bus_hw->max_txbd_num) ?
+		 bus_hw->max_txbd_num : bus_sw->txbd_num) :
+		bus_hw->max_txbd_num;
+	bus_cap->read_txbd_th = bus_cap->txbd_num >> bus_sw->read_txbd_lvl;
+
+
+	/* RXBD NUM */
+	bus_cap->rxbd_num = (bus_sw->rxbd_num) ?
+		((bus_sw->rxbd_num > bus_hw->max_rxbd_num) ?
+		 bus_hw->max_rxbd_num : bus_sw->rxbd_num) :
+		bus_hw->max_rxbd_num;
+	/* RPBD NUM */
+	bus_cap->rpbd_num = (bus_sw->rpbd_num) ?
+		((bus_sw->rpbd_num > bus_hw->max_rpbd_num) ?
+		 bus_hw->max_rpbd_num : bus_sw->rpbd_num) :
+		bus_hw->max_rpbd_num;
+	/* RXBUF NUM */
+	/* rx buffer number must be larger or equal to rx bd number */
+	bus_cap->rxbuf_num = (bus_sw->rxbuf_num < bus_cap->rxbd_num) ?
+		 (bus_cap->rxbd_num + RTW_RX_BUF_BD_MIN_DIFF) : bus_sw->rxbuf_num;
+	/* RPBUF NUM */
+	bus_cap->rpbuf_num = (bus_sw->rpbuf_num < bus_cap->rpbd_num) ?
+		 (bus_cap->rpbd_num + RTW_RX_BUF_BD_MIN_DIFF) : bus_sw->rpbuf_num;
+	/* RXBUF SIZE */
+	bus_cap->rxbuf_size = (bus_sw->rxbuf_size) ?
+		((bus_sw->rxbuf_size > bus_hw->max_rxbuf_size) ?
+		 bus_hw->max_rxbuf_size : bus_sw->rxbuf_size) :
+		bus_hw->max_rxbuf_size;
+	/* RPBUF SIZE */
+	bus_cap->rpbuf_size = (bus_sw->rpbuf_size) ?
+		((bus_sw->rpbuf_size > bus_hw->max_rpbuf_size) ?
+		 bus_hw->max_rpbuf_size : bus_sw->rpbuf_size) :
+		bus_hw->max_rpbuf_size;
+
+	bus_cap->ltr_act.ctrl = bus_sw->ltr_act.ctrl;
+	bus_cap->ltr_act.val = bus_sw->ltr_act.val;
+	bus_cap->ltr_idle.ctrl = bus_sw->ltr_idle.ctrl;
+	bus_cap->ltr_idle.val = bus_sw->ltr_idle.val;
+	bus_cap->ltr_init_state = bus_sw->ltr_init_state;
+	bus_cap->ltr_sw_ctrl = bus_hw->ltr_sw_ctrl ?
+		(bus_sw->ltr_sw_ctrl ? true : false) : false;
+	bus_cap->ltr_hw_ctrl = bus_hw->ltr_hw_ctrl ?
+		(bus_sw->ltr_hw_ctrl ? true : false) : false;
+
+#ifdef RTW_WKARD_DYNAMIC_PCIE_GEN
+	bus_cap->pcie_gen_dm_en = bus_sw->pcie_gen_dm_en;
+#endif
+}
+#endif
+
+static void _hal_ps_final_cap_decision(struct rtw_phl_com_t *phl_com,
+			struct rtw_hal_com_t *hal_com)
+{
+	struct rtw_ps_cap_t *ps_cap = &phl_com->dev_cap.ps_cap;
+	struct rtw_ps_cap_t *ps_sw_cap = &phl_com->dev_sw_cap.ps_cap;
+	struct rtw_ps_cap_t *ps_hw_cap = &hal_com->dev_hw_cap.ps_cap;
+
+	/* sw */
+	ps_cap->init_rf_state = ps_sw_cap->init_rf_state;
+	ps_cap->init_rt_stop_rson = ps_sw_cap->init_rt_stop_rson;
+	ps_cap->leave_fail_act = ps_sw_cap->leave_fail_act;
+	ps_cap->ips_en = ps_sw_cap->ips_en;
+	ps_cap->ips_wow_en = ps_sw_cap->ips_wow_en;
+	ps_cap->lps_en = ps_sw_cap->lps_en;
+	ps_cap->lps_awake_interval = ps_sw_cap->lps_awake_interval;
+	ps_cap->lps_listen_bcn_mode = ps_sw_cap->lps_listen_bcn_mode;
+	ps_cap->lps_smart_ps_mode = ps_sw_cap->lps_smart_ps_mode;
+	ps_cap->lps_bcnnohit_en = ps_sw_cap->lps_bcnnohit_en;
+	ps_cap->lps_rssi_enter_threshold = ps_sw_cap->lps_rssi_enter_threshold;
+	ps_cap->lps_rssi_leave_threshold = ps_sw_cap->lps_rssi_leave_threshold;
+	ps_cap->lps_rssi_diff_threshold = ps_sw_cap->lps_rssi_diff_threshold;
+	ps_cap->defer_para.defer_rson= ps_sw_cap->defer_para.defer_rson;
+	ps_cap->defer_para.lps_ping_defer_time= ps_sw_cap->defer_para.lps_ping_defer_time;
+	ps_cap->defer_para.lps_dhcp_defer_time= ps_sw_cap->defer_para.lps_dhcp_defer_time;
+	ps_cap->lps_adv_cap = ps_sw_cap->lps_adv_cap;
+	ps_cap->lps_force_tx = ps_sw_cap->lps_force_tx;
+	ps_cap->lps_wow_en = ps_sw_cap->lps_wow_en;
+	ps_cap->lps_wow_awake_interval = ps_sw_cap->lps_wow_awake_interval;
+	ps_cap->lps_wow_listen_bcn_mode = ps_sw_cap->lps_wow_listen_bcn_mode;
+	ps_cap->lps_wow_smart_ps_mode = ps_sw_cap->lps_wow_smart_ps_mode;
+	ps_cap->lps_wow_bcnnohit_en = ps_sw_cap->lps_wow_bcnnohit_en;
+	ps_cap->lps_intensive_trx_lvl = ps_sw_cap->lps_intensive_trx_lvl;
+	ps_cap->lps_intensive_chk = ps_sw_cap->lps_intensive_chk;
+	/* hw */
+	ps_cap->ps_pause_tx = ps_hw_cap->ps_pause_tx;
+	/* sw & hw */
+	ps_cap->ips_cap = (ps_sw_cap->ips_cap & ps_hw_cap->ips_cap);
+	ps_cap->ips_wow_cap = (ps_sw_cap->ips_wow_cap & ps_hw_cap->ips_wow_cap);
+	ps_cap->lps_cap = (ps_sw_cap->lps_cap & ps_hw_cap->lps_cap);
+	ps_cap->lps_wow_cap = (ps_sw_cap->lps_wow_cap & ps_hw_cap->lps_wow_cap);
+	ps_cap->bcn_tracking = (ps_sw_cap->bcn_tracking & ps_hw_cap->bcn_tracking);
+	/* fw */
+	rtw_hal_ps_fw_cap_decision(phl_com, false);
+}
+
+#ifdef CONFIG_BTCOEX
+static void _hal_btc_final_cap_decision(struct rtw_phl_com_t *phl_com)
+{
+	struct btc_cap_info *btc_sw_cap = &GET_DEV_SW_BTC_CAP(phl_com);
+	struct btc_cap_info *btc_dev_cap = &GET_DEV_BTC_CAP(phl_com);
+
+	btc_dev_cap->btc_deg_wifi_cap = btc_sw_cap->btc_deg_wifi_cap;
+}
+#endif /* CONFIG_BTCOEX */
+
+static void _hal_edcca_final_cap_decision(struct rtw_phl_com_t *phl_com,
+			struct rtw_hal_com_t *hal_com)
+{
+	struct rtw_edcca_cap_t *edcca_cap = &phl_com->dev_cap.edcca_cap;
+	struct rtw_edcca_cap_t *edcca_sw_cap = &phl_com->dev_sw_cap.edcca_cap;
+	struct rtw_edcca_cap_t *edcca_hw_cap = &hal_com->dev_hw_cap.edcca_cap;
+
+	edcca_cap->edcca_adap_th_2g = (edcca_sw_cap->edcca_adap_th_2g)?
+					(edcca_sw_cap->edcca_adap_th_2g):
+					(edcca_hw_cap->edcca_adap_th_2g);
+	edcca_cap->edcca_adap_th_5g = (edcca_sw_cap->edcca_adap_th_5g)?
+					(edcca_sw_cap->edcca_adap_th_5g):
+					(edcca_hw_cap->edcca_adap_th_5g);
+
+	edcca_cap->edcca_carrier_sense_th = (edcca_sw_cap->edcca_carrier_sense_th)?
+						(edcca_sw_cap->edcca_carrier_sense_th):
+						(edcca_hw_cap->edcca_carrier_sense_th);
+
+	edcca_cap->edcca_cbp_th_6g = (edcca_sw_cap->edcca_cbp_th_6g)?
+					(edcca_sw_cap->edcca_cbp_th_6g):
+					(edcca_hw_cap->edcca_cbp_th_6g);
+}
+
+void rtw_hal_ps_fw_cap_decision(struct rtw_phl_com_t *phl_com, bool is_wow)
+{
+        struct rtw_ps_cap_t *ps_cap = &phl_com->dev_cap.ps_cap;
+        struct rtw_wcpu_mac_cap_t *ps_fw_cap = &phl_com->dev_cap.wcpu_cap.mac_ofld_cap;
+
+        if (!ps_fw_cap->lps_pg) {
+                PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+                          "[PS], %s(): The FW does not support LPS_PG.\n", __func__);
+
+                if (!is_wow) {
+                        ps_cap->ips_cap &= ~PS_CAP_PWR_GATED;
+                        ps_cap->lps_cap &= ~PS_CAP_PWR_GATED;
+                } else {
+                        ps_cap->ips_wow_cap &= ~PS_CAP_PWR_GATED;
+                        ps_cap->lps_wow_cap &= ~PS_CAP_PWR_GATED;
+                }
+        }
+}
+
+void rtw_hal_fw_cap_pre_config(struct rtw_phl_com_t *phl_com, void *hal)
+{
+
+	struct dev_cap_t *dev_cap = &phl_com->dev_cap;
+	struct dev_cap_t *dev_sw_cap = &phl_com->dev_sw_cap;
+
+	/* leaving for fw hw cap
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct dev_cap_t *dev_hw_cap = &hal_com->dev_hw_cap;
+	*/
+
+	dev_cap->fw_cap.fw_src = dev_sw_cap->fw_cap.fw_src;
+	dev_cap->fw_cap.fw_type = dev_sw_cap->fw_cap.fw_type;
+	dev_cap->fw_cap.dlram_en = dev_sw_cap->fw_cap.dlram_en;
+	dev_cap->fw_cap.dlrom_en = dev_sw_cap->fw_cap.dlrom_en;
+}
+
+void rtw_hal_bus_cap_pre_config(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	_hal_bus_cap_pre_decision(phl_com, hal);
+}
+
+
+
+void rtw_hal_fw_final_cap_config(struct rtw_phl_com_t *phl_com, void *hal)
+{
+
+	struct dev_cap_t *dev_cap = &phl_com->dev_cap;
+	struct dev_cap_t *dev_sw_cap = &phl_com->dev_sw_cap;
+
+	/* leaving for fw hw cap
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct dev_cap_t *dev_hw_cap = &hal_com->dev_hw_cap;
+	*/
+
+	dev_cap->fw_cap.fw_src = dev_sw_cap->fw_cap.fw_src;
+	dev_cap->fw_cap.dlram_en = dev_sw_cap->fw_cap.dlram_en;
+	dev_cap->fw_cap.dlrom_en = dev_sw_cap->fw_cap.dlrom_en;
+}
+
+void rtw_hal_final_cap_decision(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct phy_hw_cap_t *phy_hw = hal_com->phy_hw_cap;
+	struct phy_sw_cap_t *phy_sw = phl_com->phy_sw_cap;
+	struct phy_cap_t *phy_cap = phl_com->phy_cap;
+
+	struct dev_cap_t *dev_cap = &phl_com->dev_cap;
+	struct dev_cap_t *dev_sw_cap = &phl_com->dev_sw_cap;
+	struct dev_cap_t *dev_hw_cap = &hal_com->dev_hw_cap;
+	struct rtw_wcpu_mac_cap_t *mac_ofld_cap = NULL;
+
+	mac_ofld_cap = &dev_cap->wcpu_cap.mac_ofld_cap;
+#ifdef RTW_WKARD_PHY_CAP
+	phy_cap[0].proto_sup = phy_sw[0].proto_sup & phy_hw[0].proto_sup;
+	phy_cap[1].proto_sup = phy_sw[1].proto_sup & phy_hw[1].proto_sup;
+
+	phy_cap[0].txss = (phy_sw[0].txss)?((phy_sw[0].txss > phy_hw[0].tx_num)?
+						phy_hw[0].tx_num:phy_sw[0].txss):phy_hw[0].tx_num;
+	phy_cap[0].rxss = (phy_sw[0].rxss)?((phy_sw[0].rxss > phy_hw[0].rx_num)?
+						phy_hw[0].rx_num:phy_sw[0].rxss):phy_hw[0].rx_num;
+	phy_cap[0].tx_path_num = (phy_sw[0].tx_path_num)?((phy_sw[0].tx_path_num > phy_hw[0].tx_path_num)?
+						phy_hw[0].tx_path_num:phy_sw[0].tx_path_num):phy_hw[0].tx_path_num;
+	phy_cap[0].rx_path_num = (phy_sw[0].rx_path_num)?((phy_sw[0].rx_path_num > phy_hw[0].rx_path_num)?
+						phy_hw[0].rx_path_num:phy_sw[0].rx_path_num):phy_hw[0].rx_path_num;
+	phy_cap[1].txss = (phy_sw[1].txss)?((phy_sw[1].txss > phy_hw[1].tx_num)?
+						phy_hw[1].tx_num:phy_sw[1].txss):phy_hw[1].tx_num;
+	phy_cap[1].rxss = (phy_sw[1].rxss)?((phy_sw[1].rxss > phy_hw[1].rx_num)?
+						phy_hw[1].rx_num:phy_sw[1].rxss):phy_hw[1].rx_num;
+	phy_cap[1].tx_path_num = (phy_sw[1].tx_path_num)?((phy_sw[1].tx_path_num > phy_hw[1].tx_path_num)?
+						phy_hw[1].tx_path_num:phy_sw[1].tx_path_num):phy_hw[1].tx_path_num;
+	phy_cap[1].rx_path_num = (phy_sw[1].rx_path_num)?((phy_sw[1].rx_path_num > phy_hw[1].rx_path_num)?
+						phy_hw[1].rx_path_num:phy_sw[1].rx_path_num):phy_hw[1].rx_path_num;
+	phy_cap[0].hw_rts_time_th = (phy_sw[0].hw_rts_time_th)?
+			phy_sw[0].hw_rts_time_th:phy_hw[0].hw_rts_time_th;
+	phy_cap[1].hw_rts_time_th = (phy_sw[1].hw_rts_time_th)?
+			phy_sw[1].hw_rts_time_th:phy_hw[1].hw_rts_time_th;
+	phy_cap[0].hw_rts_len_th = (phy_sw[0].hw_rts_len_th)?
+			phy_sw[0].hw_rts_len_th:phy_hw[0].hw_rts_len_th;
+	phy_cap[1].hw_rts_len_th = (phy_sw[1].hw_rts_len_th)?
+			phy_sw[1].hw_rts_len_th:phy_hw[1].hw_rts_len_th;
+	/* fw */
+	rtw_hal_fw_final_cap_config(phl_com,hal);
+
+	phy_cap[0].txagg_num = (phy_sw[0].txagg_num)?((phy_sw[0].txagg_num > phy_hw[0].txagg_num)?
+						phy_hw[0].txagg_num:phy_sw[0].txagg_num):phy_hw[0].txagg_num;
+	phy_cap[1].txagg_num = (phy_sw[1].txagg_num)?((phy_sw[1].txagg_num > phy_hw[1].txagg_num)?
+						phy_hw[1].txagg_num:phy_sw[1].txagg_num):phy_hw[1].txagg_num;
+#endif
+
+	dev_cap->band_sup = (dev_hw_cap->band_sup & dev_sw_cap->band_sup);
+	phy_cap[0].band_sup = (dev_cap->band_sup & phy_sw[0].band_sup);
+	phy_cap[1].band_sup = (dev_cap->band_sup & phy_sw[1].band_sup);
+
+	dev_cap->bw_sup = (dev_hw_cap->bw_sup & dev_sw_cap->bw_sup);
+	phy_cap[0].bw_sup = (dev_cap->bw_sup & phy_sw[0].bw_sup);
+	phy_cap[1].bw_sup = (dev_cap->bw_sup & phy_sw[1].bw_sup);
+
+#ifdef RTW_WKARD_LAMODE
+	if (dev_sw_cap->la_mode && dev_hw_cap->la_mode)
+		dev_cap->la_mode = true;
+#endif
+
+	/* wow, to be refined along with hw cap */
+	dev_cap->wow_cap.magic_sup = dev_sw_cap->wow_cap.magic_sup;
+	dev_cap->wow_cap.pattern_sup = dev_sw_cap->wow_cap.pattern_sup;
+	dev_cap->wow_cap.arp_ofld_sup = dev_sw_cap->wow_cap.arp_ofld_sup;
+	dev_cap->wow_cap.ns_oflod_sup = dev_sw_cap->wow_cap.ns_oflod_sup;
+	dev_cap->wow_cap.gtk_ofld_sup = dev_sw_cap->wow_cap.gtk_ofld_sup;
+	dev_cap->wow_cap.ping_pattern_wake_sup = dev_sw_cap->wow_cap.ping_pattern_wake_sup;
+	dev_cap->wow_cap.nlo_sup = dev_sw_cap->wow_cap.nlo_sup;
+
+	if (dev_sw_cap->pkg_type != 0xFF)
+		dev_cap->pkg_type = dev_sw_cap->pkg_type;
+	else
+		dev_cap->pkg_type = dev_hw_cap->pkg_type;
+
+	if (dev_sw_cap->rfe_type != 0xFF)
+		dev_cap->rfe_type = dev_sw_cap->rfe_type;
+	else
+		dev_cap->rfe_type = dev_hw_cap->rfe_type;
+
+	dev_cap->bypass_rfe_chk = dev_sw_cap->bypass_rfe_chk;
+
+	hal_com->dbcc_en = false;/*init default value*/
+#ifdef CONFIG_DBCC_SUPPORT
+	if (dev_sw_cap->dbcc_sup && dev_hw_cap->dbcc_sup) {
+		dev_cap->dbcc_sup = true;
+		dev_cap->mcmb_skip_dbcc = dev_sw_cap->mcmb_skip_dbcc;
+		dev_cap->mcmb_itf_ctrl = dev_sw_cap->mcmb_itf_ctrl;
+
+		dev_cap->dbcc_force_rmap = dev_sw_cap->dbcc_force_rmap;
+#ifdef CONFIG_DBCC_FORCE
+		dev_cap->dbcc_force_mode = dev_sw_cap->dbcc_force_mode;
+		if (is_frc_dbcc_mode(phl_com)) {
+			dev_cap->dbcc_force_cck_phyidx = dev_sw_cap->dbcc_force_cck_phyidx;
+			hal_com->dbcc_en = true;
+			phl_com->mr_ctrl.is_db = true;
+		}
+#endif
+	}
+#endif
+
+#ifdef CONFIG_MCC_SUPPORT
+	if (dev_sw_cap->mcc_sup && dev_hw_cap->mcc_sup)
+		dev_cap->mcc_sup = true;
+#endif /*CONFIG_MCC_SUPPORT*/
+
+#ifdef CONFIG_PHL_NAN
+	if (dev_sw_cap->nan_sup && dev_hw_cap->nan_sup)
+		dev_cap->nan_sup = true;
+#endif /*CONFIG_MCC_SUPPORT*/
+
+	if (dev_sw_cap->hw_hdr_conv && dev_hw_cap->hw_hdr_conv)
+		dev_cap->hw_hdr_conv = true;
+
+	if (dev_sw_cap->hw_amsdu && dev_hw_cap->hw_amsdu)
+		dev_cap->hw_amsdu = true;
+
+	if (dev_sw_cap->tx_mu_ru && dev_hw_cap->tx_mu_ru)
+		dev_cap->tx_mu_ru = true;
+
+	dev_cap->xcap = dev_hw_cap->xcap;
+	dev_cap->domain = dev_hw_cap->domain;
+	dev_cap->domain_6g = dev_hw_cap->domain_6g;
+
+	/* btc related */
+	dev_cap->btc_mode = dev_sw_cap->btc_mode;
+	dev_cap->btc_esoc_type = dev_sw_cap->btc_esoc_type;
+	dev_cap->btc_ant_iso_db = dev_sw_cap->btc_ant_iso_db;
+#ifdef CONFIG_BTCOEX
+	_hal_btc_final_cap_decision(phl_com);
+#endif /* CONFIG_BTCOEX */
+
+#ifdef CONFIG_PCI_HCI
+	_hal_bus_final_cap_decision(phl_com, hal_com);
+#endif
+
+	if (dev_sw_cap->pwrbyrate_off != 0xFF)
+		dev_cap->pwrbyrate_off = dev_sw_cap->pwrbyrate_off;
+	else
+		dev_cap->pwrbyrate_off = PW_BY_RATE_ON;
+
+	if (dev_sw_cap->pwrlmt_type != 0xFF)
+		dev_cap->pwrlmt_type = dev_sw_cap->pwrlmt_type;
+	else
+		dev_cap->pwrlmt_type = PWLMT_BY_EFUSE;
+
+	if (dev_sw_cap->rf_board_opt != 0xFF)
+		dev_cap->rf_board_opt = dev_sw_cap->rf_board_opt;
+	else
+		dev_cap->rf_board_opt = dev_hw_cap->rf_board_opt;
+
+	if (RTW_HW_CAP_ULRU_AUTO == dev_sw_cap->sta_ulru) {
+		/* auto : use hw default setting */
+		dev_cap->sta_ulru = dev_hw_cap->sta_ulru;
+	} else {
+		/* forced use sw settings */
+		dev_cap->sta_ulru = dev_sw_cap->sta_ulru;
+	}
+	PHL_INFO("sta_ulru final : 0x%x ; sw 0x%x ; hw 0x%x\n",
+		 dev_cap->sta_ulru, dev_sw_cap->sta_ulru, dev_hw_cap->sta_ulru);
+
+#ifdef RTW_WKARD_BB_DISABLE_STA_2G40M_ULOFDMA
+	/* 2.4G 40MHz UL OFDAM : bb ask to disable */
+	if (RTW_HW_CAP_ULRU_AUTO == dev_sw_cap->sta_ulru_2g40mhz) {
+		/* auto : use hw default setting */
+		dev_cap->sta_ulru_2g40mhz = dev_hw_cap->sta_ulru_2g40mhz;
+	} else {
+		/* forced use sw settings */
+		dev_cap->sta_ulru_2g40mhz = dev_sw_cap->sta_ulru_2g40mhz;
+	}
+	PHL_INFO("sta_ulru_2g40mhz final : 0x%x ; sw 0x%x ; hw 0x%x\n",
+		 dev_cap->sta_ulru_2g40mhz, dev_sw_cap->sta_ulru_2g40mhz, dev_hw_cap->sta_ulru_2g40mhz);
+#endif
+
+	_hal_ps_final_cap_decision(phl_com, hal_com);
+
+	_hal_edcca_final_cap_decision(phl_com, hal_com);
+
+	if (dev_hw_cap->hw_stype_cap != EFUSE_HW_STYPE_GENERAL)
+		dev_cap->hw_stype_cap = dev_hw_cap->hw_stype_cap;
+	else
+		dev_cap->hw_stype_cap = EFUSE_HW_STYPE_NONE;
+
+	if (dev_hw_cap->wl_func_cap != EFUSE_WL_FUNC_GENERAL)
+		dev_cap->wl_func_cap = dev_hw_cap->wl_func_cap;
+	else
+		dev_cap->wl_func_cap = EFUSE_WL_FUNC_NONE;
+
+	/* get rfk cap from sw settings */
+	if (dev_sw_cap->rfk_cap != 0xFFFFFFFF)
+		dev_cap->rfk_cap = dev_sw_cap->rfk_cap;
+	else
+		dev_cap->rfk_cap = dev_hw_cap->rfk_cap;
+
+	dev_cap->rpq_agg_num = dev_sw_cap->rpq_agg_num ?
+		dev_sw_cap->rpq_agg_num : dev_hw_cap->rpq_agg_num;
+	dev_cap->rpq_tmr = dev_sw_cap->rpq_tmr ?
+		dev_sw_cap->rpq_tmr : dev_hw_cap->rpq_tmr;
+	/* MAC_AX_QTA_SCC_TURBO, decide by sw, need to be refined after we have hw cap */
+	dev_cap->quota_turbo = dev_sw_cap->quota_turbo;
+
+	dev_cap->sgi_160_sup = dev_hw_cap->sgi_160_sup;
+
+	/* secruity cap */
+	if (dev_sw_cap->sec_cap.hw_form_hdr && dev_hw_cap->sec_cap.hw_form_hdr)
+		dev_cap->sec_cap.hw_form_hdr = true;
+	if (dev_sw_cap->sec_cap.hw_tx_search_key && dev_hw_cap->sec_cap.hw_tx_search_key)
+		dev_cap->sec_cap.hw_tx_search_key = true;
+	if (dev_sw_cap->sec_cap.hw_sec_iv && dev_hw_cap->sec_cap.hw_sec_iv)
+		dev_cap->sec_cap.hw_sec_iv = true;
+
+	if (dev_sw_cap->max_link_num != 0) {
+		dev_cap->max_link_num = (dev_sw_cap->max_link_num < dev_hw_cap->max_link_num) ?
+		                        dev_sw_cap->max_link_num :
+		                        dev_hw_cap->max_link_num;
+	} else {
+		dev_cap->max_link_num = dev_hw_cap->max_link_num;
+	}
+
+	if (dev_sw_cap->max_mld_num != 0) {
+		dev_cap->max_mld_num = (dev_sw_cap->max_mld_num < dev_hw_cap->max_mld_num) ?
+		                       dev_sw_cap->max_mld_num :
+		                       dev_hw_cap->max_mld_num;
+	} else {
+		dev_cap->max_mld_num = dev_hw_cap->max_mld_num;
+	}
+
+	dev_cap->macid_num = MIN(dev_sw_cap->macid_num, dev_hw_cap->macid_num);
+
+	dev_cap->bfee_rx_ndp_sts = dev_hw_cap->bfee_rx_ndp_sts; /* direct follow hw cap */
+
+	if (RTW_DEV_CAP_DISABLE == dev_sw_cap->drv_info_sup) {
+		/* sw forced disable */
+		dev_cap->drv_info_sup = RTW_DEV_CAP_DISABLE;
+	} else {
+		dev_cap->drv_info_sup = dev_hw_cap->drv_info_sup;
+	}
+#ifdef CONFIG_PHL_THERMAL_PROTECT
+	dev_cap->min_tx_duty = dev_sw_cap->min_tx_duty;
+	dev_cap->thermal_threshold = dev_sw_cap->thermal_threshold;
+#endif
+	dev_cap->nb_config = dev_sw_cap->nb_config;
+
+#ifdef CONFIG_PHL_IO_OFLD
+	dev_cap->io_ofld = dev_sw_cap->io_ofld &
+	                   dev_hw_cap->wcpu_cap.mac_ofld_cap.io_offload;
+#endif
+
+#ifdef CONFIG_PHL_SCANOFLD
+	dev_cap->scan_ofld = dev_sw_cap->scan_ofld &
+			     dev_hw_cap->wcpu_cap.mac_ofld_cap.scan_offload;
+#endif
+#ifdef CONFIG_PHL_CHSWOFLD
+	dev_cap->chsw_ofld = dev_sw_cap->chsw_ofld &
+				dev_hw_cap->wcpu_cap.mac_ofld_cap.chsw_offload;
+#endif
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+	dev_cap->sensing_csi = dev_sw_cap->sensing_csi &
+			     dev_hw_cap->wcpu_cap.mac_ofld_cap.sensing_csi;
+#endif
+
+#ifdef CONFIG_PHL_FW_DUMP_EFUSE
+	dev_cap->efuse_dump_ofld = dev_sw_cap->efuse_dump_ofld &
+	                           dev_hw_cap->wcpu_cap.mac_ofld_cap.efuse_dump_offload;
+
+	dev_cap->adie_efuse_dump_ofld = dev_sw_cap->adie_efuse_dump_ofld &
+	                                dev_hw_cap->wcpu_cap.mac_ofld_cap.adie_efuse_dump_offload;
+#endif
+
+#ifdef CONFIG_PHL_TWT
+	if ((dev_sw_cap->twt_sup & RTW_PHL_TWT_REQ_SUP) &&
+	    (mac_ofld_cap->twt_sta)) {
+		dev_cap->twt_sup |= RTW_PHL_TWT_REQ_SUP;
+	}
+	if ((dev_sw_cap->twt_sup & RTW_PHL_TWT_RSP_SUP) &&
+	    (mac_ofld_cap->twt_ap)) {
+		dev_cap->twt_sup |= RTW_PHL_TWT_RSP_SUP;
+	}
+	if ((dev_sw_cap->twt_sup & RTW_PHL_TWT_BC_SUP) &&
+	    (dev_cap->twt_sup)) {
+		dev_cap->twt_sup |= RTW_PHL_TWT_BC_SUP;
+	}
+#endif /*CONFIG_PHL_TWT*/
+
+	if (dev_hw_cap->antdiv_sup)
+		dev_cap->antdiv_sup = dev_sw_cap->antdiv_sup;
+	else
+		dev_cap->antdiv_sup = false;
+
+	dev_cap->disable_dyn_txpwr = dev_sw_cap->disable_dyn_txpwr;
+
+#ifdef CONFIG_PHL_RFK_FCS_SUPPPORT
+	if (dev_sw_cap->rfk_fcs_sup && dev_hw_cap->rfk_fcs_sup) {
+		dev_cap->rfk_fcs_sup = true;
+	}
+	if (dev_sw_cap->rfk_fcs_num != 0) {
+		dev_cap->rfk_fcs_num = (dev_sw_cap->rfk_fcs_num < dev_hw_cap->rfk_fcs_num) ?
+		                       dev_sw_cap->rfk_fcs_num :
+		                       dev_hw_cap->rfk_fcs_num;
+	} else {
+		dev_cap->rfk_fcs_num = dev_hw_cap->rfk_fcs_num;
+	}
+#endif
+}
+
+/**
+ * rtw_hal_get_bf_proto_cap
+ * input:
+ * @phl: (struct phl_info_t *phl_info)
+ * @band:
+ * input_output:
+ * @proto_cap: (struct protocol_cap_t *proto_cap) the variable to save the bf_cap result.
+ * return:
+ * @rtw_hal_status:
+ */
+enum rtw_hal_status
+rtw_hal_get_bf_proto_cap(struct rtw_phl_com_t *phl_com, void *hal,
+			 u8 band, struct protocol_cap_t *proto_cap)
+{
+#ifdef RTW_WKARD_PHY_CAP
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct protocol_cap_t *hw_cap = NULL;
+	struct protocol_cap_t *sw_cap = NULL;
+
+	if (band >= HW_BAND_MAX) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "rtw_hal_get_bf_proto_cap : invalid band index \n");
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	if (proto_cap == NULL) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "rtw_hal_get_bf_proto_cap : input proto_cap is NULL \n");
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+
+	hw_cap = &hal_com->proto_hw_cap[band];
+	sw_cap = &phl_com->proto_sw_cap[band];
+
+	proto_cap->he_su_bfmr =
+		(sw_cap->he_su_bfmr && hw_cap->he_su_bfmr) ? 1 : 0;
+	proto_cap->he_su_bfme =
+		(sw_cap->he_su_bfme && hw_cap->he_su_bfme) ? 1 : 0;
+	proto_cap->he_mu_bfmr =
+		(sw_cap->he_mu_bfmr && hw_cap->he_mu_bfmr) ? 1 : 0;
+	proto_cap->he_mu_bfme =
+		(sw_cap->he_mu_bfme && hw_cap->he_mu_bfme) ? 1 : 0;
+	proto_cap->non_trig_cqi_fb =
+		(sw_cap->non_trig_cqi_fb && hw_cap->non_trig_cqi_fb) ? 1 : 0;
+	proto_cap->trig_cqi_fb =
+		(sw_cap->trig_cqi_fb && hw_cap->trig_cqi_fb) ? 1 : 0;
+
+	proto_cap->vht_su_bfme =
+		(sw_cap->vht_su_bfme && hw_cap->vht_su_bfme) ? 1 : 0;
+	proto_cap->vht_su_bfmr =
+		(sw_cap->vht_su_bfmr && hw_cap->vht_su_bfmr) ? 1 : 0;
+	proto_cap->vht_mu_bfme =
+		(sw_cap->vht_mu_bfme && hw_cap->vht_mu_bfme) ? 1 : 0;
+	proto_cap->vht_mu_bfmr =
+		(sw_cap->vht_mu_bfmr && hw_cap->vht_mu_bfmr) ? 1 : 0;
+
+	proto_cap->ht_su_bfme =
+		(sw_cap->ht_su_bfme && hw_cap->ht_su_bfme) ? 1 : 0;
+	proto_cap->ht_su_bfmr =
+		(sw_cap->ht_su_bfmr && hw_cap->ht_su_bfmr) ? 1 : 0;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->he_su_bfmr = 0x%x \n",
+		  proto_cap->he_su_bfmr);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->he_su_bfme = 0x%x \n",
+		  proto_cap->he_su_bfme);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->he_mu_bfmr = 0x%x \n",
+		  proto_cap->he_mu_bfmr);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->he_mu_bfme = 0x%x \n",
+		  proto_cap->he_mu_bfme);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->non_trig_cqi_fb = 0x%x \n",
+		  proto_cap->non_trig_cqi_fb);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->trig_cqi_fb = 0x%x \n",
+		  proto_cap->trig_cqi_fb);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->vht_su_bfme = 0x%x \n",
+		  proto_cap->vht_su_bfme);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->vht_su_bfmr = 0x%x \n",
+		  proto_cap->vht_su_bfmr);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->vht_mu_bfme = 0x%x \n",
+		  proto_cap->vht_mu_bfme);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->vht_mu_bfmr = 0x%x \n",
+		  proto_cap->vht_mu_bfmr);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->ht_su_bfme = 0x%x \n",
+		  proto_cap->ht_su_bfme);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "rtw_hal_get_bf_proto_cap : proto_cap->ht_su_bfmr = 0x%x \n",
+		  proto_cap->ht_su_bfmr);
+#endif
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+/**
+ * rtw_hal_get_stbc_proto_cap() - Get the intersection proto_cap of the proto_hw_cap and proto_sw_cap
+ * @phl_com: input, the pointer of phl_com (struct rtw_phl_com_t)
+ * @hal: input, the pointer of hal (struct hal_info_t)
+ * @band: input, the band index
+ * @proto_cap: input_output (struct protocol_cap_t)
+ * 	       The variable pointer to save the ouput proto_cap results.
+ * 	       Here, we update STBC cap into STBC fileds of proto_cap.
+ * Return: return the result of getting proto_cap (enum @rtw_hal_status)
+ */
+enum rtw_hal_status
+rtw_hal_get_stbc_proto_cap(struct rtw_phl_com_t *phl_com, void *hal,
+			 u8 band, struct protocol_cap_t *proto_cap)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct protocol_cap_t *hw_cap = NULL;
+	struct protocol_cap_t *sw_cap = NULL;
+
+	if (band >= HW_BAND_MAX) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "rtw_hal_get_stbc_proto_cap : invalid band index \n");
+		return RTW_HAL_STATUS_FAILURE;
+	}
+	if (proto_cap == NULL) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "rtw_hal_get_stbc_proto_cap : input proto_cap is NULL \n");
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hw_cap = &hal_com->proto_hw_cap[band];
+	sw_cap = &phl_com->proto_sw_cap[band];
+
+	proto_cap->stbc_ht_tx =
+		(sw_cap->stbc_ht_tx && hw_cap->stbc_ht_tx) ? 1 : 0;
+	proto_cap->stbc_vht_tx =
+		(sw_cap->stbc_vht_tx && hw_cap->stbc_vht_tx) ? 1 : 0;
+	proto_cap->stbc_he_tx =
+		(sw_cap->stbc_he_tx && hw_cap->stbc_he_tx) ? 1 : 0;
+	proto_cap->stbc_tx_greater_80mhz =
+		(sw_cap->stbc_tx_greater_80mhz && hw_cap->stbc_tx_greater_80mhz) ? 1 : 0;
+
+	proto_cap->stbc_ht_rx =
+		(sw_cap->stbc_ht_rx <= hw_cap->stbc_ht_rx) ?
+			sw_cap->stbc_ht_rx : hw_cap->stbc_ht_rx;
+	proto_cap->stbc_vht_rx =
+		(sw_cap->stbc_vht_rx <= hw_cap->stbc_vht_rx) ?
+			sw_cap->stbc_vht_rx : hw_cap->stbc_vht_rx;
+	proto_cap->stbc_he_rx =
+		(sw_cap->stbc_he_rx && hw_cap->stbc_he_rx) ? 1 : 0;
+	proto_cap->stbc_rx_greater_80mhz =
+		(sw_cap->stbc_rx_greater_80mhz && hw_cap->stbc_rx_greater_80mhz) ? 1 : 0;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_ht_tx = 0x%x \n",
+		  __func__, proto_cap->stbc_ht_tx);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_vht_tx = 0x%x \n",
+		  __func__, proto_cap->stbc_vht_tx);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_he_tx = 0x%x \n",
+		  __func__, proto_cap->stbc_he_tx);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_tx_greater_80mhz = 0x%x \n",
+		  __func__, proto_cap->stbc_tx_greater_80mhz);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_ht_rx = 0x%x \n",
+		  __func__, proto_cap->stbc_ht_rx);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_vht_rx = 0x%x \n",
+		  __func__, proto_cap->stbc_vht_rx);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_he_rx = 0x%x \n",
+		  __func__, proto_cap->stbc_he_rx);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+		  "%s : proto_cap->stbc_rx_greater_80mhz = 0x%x \n",
+		  __func__, proto_cap->stbc_rx_greater_80mhz);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cap.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cap.h
new file mode 100644
index 000000000000..cdbefdea4593
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_cap.h
@@ -0,0 +1,22 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_CAP_H_
+#define _HAL_CAP_H_
+
+void rtw_hal_ps_fw_cap_decision(struct rtw_phl_com_t *phl_com, bool is_wow);
+void rtw_hal_final_cap_decision(struct rtw_phl_com_t *phl_com, void *hal);
+
+#endif /* _HAL_CAP_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan.c
new file mode 100644
index 000000000000..b1d28be6c78d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan.c
@@ -0,0 +1,226 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_CHAN_C_
+#include "hal_headers.h"
+
+enum rtw_hal_status rtw_hal_set_ch_bw(void *hal, u8 band_idx,
+		struct rtw_chan_def *chdef, enum rfk_tri_type rt_type, bool rd_enabled, bool frc_switch)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_chan_def *cur_chdef = &(hal_com->band[band_idx].cur_chandef);
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	u8 center_ch = 0;
+	u8 central_ch_seg1 = 0;
+	enum band_type change_band;
+	enum phl_phy_idx phy_idx = HW_PHY_0;
+	enum channel_width tmp_bw = chdef->bw;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	bool is_reset = false;
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+#endif
+
+#ifdef CONFIG_PHL_NARROW_BW
+	struct dev_cap_t *dev_cap = &phl_com->dev_cap;
+
+	if (dev_cap->nb_config == CHANNEL_WIDTH_10)
+		tmp_bw = CHANNEL_WIDTH_10;
+	else if (dev_cap->nb_config == CHANNEL_WIDTH_5)
+		tmp_bw = CHANNEL_WIDTH_5;
+#endif /*CONFIG_PHL_NARROW_BW*/
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+
+
+	phy_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+
+	if (phy_idx >= HW_PHY_MAX) {
+		PHL_ERR("%s wrong phy_idx (%d)\n", __func__, phy_idx);
+		status = RTW_HAL_STATUS_FAILURE;
+		goto _exit;
+	}
+
+	if ((frc_switch) ||
+	    (chdef->band != cur_chdef->band) ||
+	    (chdef->chan != cur_chdef->chan) ||
+	    (tmp_bw != cur_chdef->bw) ||
+	    (chdef->offset != cur_chdef->offset)) {
+
+#ifdef CONFIG_PHL_CHSWOFLD
+		if (!phl_com->chsw_ofld_info.chsw_ofld_en)
+#endif
+		{
+			status = rtw_hal_reset(hal_com, phy_idx, band_idx, true);
+			if (status != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s rtw_hal_reset en - failed\n", __func__);
+				_os_warn_on(1);
+			}
+			is_reset = true;
+		}
+		/* if central channel changed, reset BB & MAC */
+		center_ch = rtw_phl_get_center_ch(chdef);
+		PHL_INFO("Using band:%d central channel %u for primary channel %u BW %u\n",
+		         chdef->band, center_ch, chdef->chan, chdef->bw);
+
+		change_band = chdef->band;
+
+#ifdef CONFIG_PHL_CHSWOFLD
+		if (phl_com->chsw_ofld_info.chsw_ofld_en) {
+			status = rtw_hal_mac_ch_switch_ofld(hal, band_idx, chdef->chan,
+						center_ch, change_band, chdef->bw, phl_com->chsw_ofld_info.rf_reload);
+			if (status != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s rtw_hal_mac_ch_switch_ofld - failed\n", __func__);
+				goto _exit;
+			}
+		} else
+#endif
+		{
+			status = rtw_hal_mac_set_bw(hal_info, band_idx, chdef->chan,
+							center_ch, central_ch_seg1, change_band,
+							tmp_bw);
+			if (status != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s rtw_hal_mac_set_bw - failed\n", __func__);
+				goto _exit;
+			}
+
+			if (tmp_bw == CHANNEL_WIDTH_80_80 && central_ch_seg1 == 0) {
+				PHL_ERR("%s mising info for 80+80M configuration\n", __func__);
+				status = RTW_HAL_STATUS_FAILURE;
+				goto _exit;
+			}
+			status = rtw_hal_bb_set_ch_bw(hal_info, phy_idx, chdef->chan,
+							center_ch, central_ch_seg1, change_band,
+							tmp_bw);
+			if (status != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s rtw_hal_bb_set_ch_bw - failed\n", __func__);
+				goto _exit;
+			}
+
+			status = rtw_hal_rf_set_ch_bw(hal_com,
+							phy_idx,
+							center_ch,
+							change_band,
+							tmp_bw);
+
+			if (status != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s rtw_hal_rf_set_ch_bw - failed\n", __func__);
+				goto _exit;
+			}
+		}
+
+		cur_chdef->chan = chdef->chan;
+		cur_chdef->bw = chdef->bw;
+		cur_chdef->offset = chdef->offset;
+		cur_chdef->center_ch = center_ch;
+
+		if ((frc_switch) || (cur_chdef->band != change_band)) {
+			cur_chdef->band = change_band;
+			rtw_hal_notify_switch_band(hal, change_band, phy_idx);
+		}
+
+		status = rtw_hal_rf_set_power(hal_info, phy_idx, PWR_LIMIT);
+
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_rf_set_power - failed\n", __func__);
+			goto _exit;
+		}
+
+		status = rtw_hal_rf_set_power(hal_info, phy_idx, PWR_LIMIT_RU);
+
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_rf_set_power - failed\n", __func__);
+			goto _exit;
+		}
+
+		PHL_INFO("%s phy_idx:%d, band:%d, ch:%d, bw:%d, offset:%d\n",
+			__func__, phy_idx, chdef->band, chdef->chan, tmp_bw, chdef->offset);
+
+#ifdef CONFIG_PHL_CHSWOFLD
+		if (!phl_com->chsw_ofld_info.chsw_ofld_en)
+#endif
+		{
+			status = rtw_hal_reset(hal_com, phy_idx, band_idx, false);
+			if (status != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s rtw_hal_reset dis- failed\n", __func__);
+				_os_warn_on(1);
+			}
+			is_reset = false;
+		}
+
+		/*PHL_DUMP_CHAN_DEF_EX(chandef);*/
+	}
+
+	status = rtw_hal_rf_chl_rfk_trigger(hal_com, phy_idx, rt_type);
+	if (status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("rtw_hal_rf_chl_rfk_trigger fail!\n");
+
+	PHL_INFO("%s: Switch chdef done, rt_type:%s, frc_switch(%d)\n", __func__,
+		(rt_type) ?"Y" : "N", frc_switch);
+_exit:
+	if (is_reset) {
+		PHL_INFO(
+		    "%s: unexpected error with reset, needs to dis-reset \n",
+		    __func__);
+		status = rtw_hal_reset(hal_com, phy_idx, band_idx, false);
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s rtw_hal_reset dis- failed\n", __func__);
+			_os_warn_on(1);
+		}
+	}
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_HAL_SET_CHAN);
+#endif /* DBG_MONITOR_TIME */
+	return status;
+}
+
+u8 rtw_hal_get_cur_ch(void *hal, u8 band_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_chan_def *chandef = &(hal_com->band[band_idx].cur_chandef);
+
+	return chandef->chan;
+}
+
+void rtw_hal_get_cur_chdef(void *hal, u8 band_idx,
+				struct rtw_chan_def *cur_chandef)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_chan_def *chandef = &(hal_com->band[band_idx].cur_chandef);
+	void *drv = hal_com->drv_priv;
+
+	_os_mem_cpy(drv, cur_chandef, chandef, sizeof(struct rtw_chan_def));
+}
+
+void rtw_hal_sync_cur_ch(void *hal, u8 band_idx, struct rtw_chan_def chandef)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_chan_def *cur_chandef = &(hal_com->band[band_idx].cur_chandef);
+
+	PHL_INFO("%s: Sync Ori-cur ch to src ch\n", __FUNCTION__);
+	PHL_INFO("%s: Src ch: Bnad(%d), ch(%d), bw(%d), ofst(%d), cntr_ch(%d)\n",
+		 __FUNCTION__, chandef.band, chandef.chan, chandef.bw, chandef.offset,
+		 chandef.center_ch);
+	PHL_INFO("%s: Ori-cur ch: Bnad(%d), ch(%d), bw(%d), ofst(%d), cntr_ch(%d)\n",
+		 __FUNCTION__, cur_chandef->band, cur_chandef->chan,
+		 cur_chandef->bw, cur_chandef->offset, cur_chandef->center_ch);
+	_os_mem_cpy(hal_to_drvpriv(hal_info), cur_chandef, &chandef,
+		    sizeof(struct rtw_chan_def));
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan.h
new file mode 100644
index 000000000000..e6ae425e1c27
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan.h
@@ -0,0 +1,35 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_CHAN_H_
+#define _HAL_CHAN_H_
+
+enum rtw_hal_status rtw_hal_set_ch_bw(void *hal, u8 band_idx,
+		struct rtw_chan_def *chdef, enum rfk_tri_type rt_type, bool rd_enabled, bool frc_switch);
+
+void rtw_hal_get_cur_chdef(void *hal, u8 band_idx,
+				struct rtw_chan_def *cur_chandef);
+
+#ifdef DBG_PHL_CHAN
+void phl_chan_dump_chandef(const char *caller, const int line, bool show_caller,
+						struct rtw_chan_def *chandef);
+#define PHL_DUMP_CHAN_DEF(_chandef) phl_chan_dump_chandef(__FUNCTION__, __LINE__, false, _chandef);
+#define PHL_DUMP_CHAN_DEF_EX(_chandef) phl_chan_dump_chandef(__FUNCTION__, __LINE__, true, _chandef);
+#else
+#define PHL_DUMP_CHAN_DEF(_chandef)
+#define PHL_DUMP_CHAN_DEF_EX(_chandef)
+#endif
+
+#endif /* _HAL_CHAN_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan_info.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan_info.c
new file mode 100644
index 000000000000..c6ed796335fd
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan_info.c
@@ -0,0 +1,652 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_CHAN_INFO_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+
+#define CSI_BLOCK_SZ 1280
+
+static enum rtw_hal_status
+_hal_chinfo_query_idle_csi_buf(struct hal_info_t *hal,
+				u8 chinfo_sz, u8 *buf_start, u8 *buf_end)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	#if 0
+	if (chinfo_sz > CSI_BLOCK_SZ*2) {
+		PHL_ERR("%s chinfo_sz(%d) over hw support(%d)\n",
+			__func__, chinfo_sz, CSI_BLOCK_SZ*2);
+		return hsts;
+	}
+	#endif
+
+	*buf_start = 1;
+	*buf_end = 2;
+
+	hsts = RTW_HAL_STATUS_SUCCESS;
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_cfg_chinfo(void *hal, struct rtw_chinfo_action_parm *act_param)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct chinfo_bbcr_cfg bbcr = {0};
+	/*enum phl_chinfo_mode ch_mode = act_param->mode;*/
+	enum wlan_mode wmode = act_param->sta->wmode;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	u8 enable = act_param->enable;
+	u8 hw_band = act_param->sta->rlink->hw_band;
+	u8 buf_start = 0;
+	u8 buf_end = 0;
+
+	if (enable) {
+		/*
+		 *Below is Example code for BBCR set to Legacy channel 80MHz
+		 *TODO 1: need to calculate channel raw data size to see if
+		 *	it is acceptable.
+		 *TODO 2: there should be 10 sets of pre-defined configuration for BBCR.
+		 *	need to decide which CR to use.
+		 */
+		if (hw_band == 0)
+			bbcr.ch_i_phy0_en = true;
+		else
+			bbcr.ch_i_phy1_en = true;
+
+		bbcr.ch_i_grp_num_he = act_param->group_num;
+		bbcr.ch_i_grp_num_non_he = act_param->group_num;
+		bbcr.ch_i_data_src = false; /*LS, channel estimation*/
+		switch (act_param->accuracy) {
+		case CHINFO_ACCU_1BYTE:
+			bbcr.ch_i_cmprs = false; /*8 bits*/
+			break;
+		case CHINFO_ACCU_2BYTES:
+			bbcr.ch_i_cmprs = true; /*16 bits*/
+			break;
+		default:
+			bbcr.ch_i_cmprs = false; /*8 bits*/
+			break;
+		}
+
+		if (act_param->ele_bitmap == 0) {
+			/* 2T2R */
+			bbcr.ch_i_ele_bitmap = 0x303;
+		} else {
+			bbcr.ch_i_ele_bitmap = act_param->ele_bitmap;
+		}
+
+		hal_status = _hal_chinfo_query_idle_csi_buf(hal_info, 0, &buf_start, &buf_end);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s: query_idle_csi_buf fail\n", __func__);
+			goto exit;
+		}
+		bbcr.ch_i_blk_start_idx = buf_start;
+		bbcr.ch_i_blk_end_idx = buf_end;
+
+		if (act_param->mode != CHINFO_MODE_ACK) {
+			if (wmode & WLAN_MD_11AX ||
+				wmode & WLAN_MD_11AC ||
+				wmode & WLAN_MD_11N)
+				 /* MIMO channel for non legacy mode */
+				bbcr.ch_i_type = CH_INFO_MIMO_CH;
+			else
+				/* legacy channel for legacy mode */
+				bbcr.ch_i_type = CH_INFO_LEGACY_CH;
+		} else {
+			/* use legacy mode becasue ACK is OFDM rate */
+			bbcr.ch_i_type = CH_INFO_LEGACY_CH;
+		}
+
+		bbcr.ch_i_seg_len = 0; /*12*/
+		rtw_hal_bb_chan_info_cfg(hal_info, &bbcr);
+	}
+
+exit:
+	return hal_status;
+}
+
+void
+_hal_fill_csi_header_remain(void* hal, struct csi_header_t *csi_header
+	, const struct rtw_r_meta_data *mdata)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	struct rtw_chinfo_cur_parm *cur_parm = phl_com->cur_parm;
+	struct rtw_chinfo_action_parm *act_parm = &cur_parm->action_parm;
+	/* struct rtw_hal_com_t *h = hal_info->hal_com; */
+
+	/* from mdata */
+	csi_header->hw_assigned_timestamp = mdata->freerun_cnt;
+	csi_header->rx_data_rate = mdata->rx_rate;
+	csi_header->bandwidth = mdata->bw;
+	csi_header->ch_matrix_report = mdata->get_ch_info;
+
+	/* TODO: from drv define, get value from other side.*/
+	if (act_parm->mode == CHINFO_MODE_ACK) {/* BW is 20M, and non-HT */
+		switch (act_parm->group_num) {
+		case CHINFO_GROUP_NUM_1:
+			csi_header->num_sub_carrier = 52;
+			break;
+		case CHINFO_GROUP_NUM_2:
+			csi_header->num_sub_carrier = 26;
+			break;
+		case CHINFO_GROUP_NUM_4:
+			csi_header->num_sub_carrier = 14;
+			break;
+		case CHINFO_GROUP_NUM_16:
+			csi_header->num_sub_carrier = 4;
+			break;
+		default:
+			csi_header->num_sub_carrier = 0;
+			PHL_ERR("%s: Unknown CSI group_num: %d\n", __func__, act_parm->group_num);
+			break;
+		}
+	} else
+		csi_header->num_sub_carrier = 0;
+
+	if (act_parm->accuracy == CHINFO_ACCU_1BYTE)
+		csi_header->num_bit_per_tone = 8<<1; /* 8 bits per Real and Imag part */
+	else if (act_parm->accuracy == CHINFO_ACCU_2BYTES)
+		csi_header->num_bit_per_tone = 16<<1; /* 16 bits per Real and Imag part */
+	else {
+		csi_header->num_bit_per_tone = 0;
+		PHL_ERR("%s: Unknown CSI accuracy: %d\n", __func__, act_parm->accuracy);
+	}
+
+	csi_header->channel = 0;
+
+	/* Others: mac addres not from TA ? */
+	/* _os_mem_cpy(halcom_to_drvpriv(h), &(csi_header->mac_addr[0]), &(mdata->ta[0]), MAC_ALEN); */
+}
+
+void
+_hal_fill_csi_header_phy_info(void* hal, struct csi_header_t *csi_header
+	, struct ch_rpt_hdr_info *ch_hdr_rpt, struct phy_info_rpt *phy_info)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *h = hal_info->hal_com;
+	void *drv = halcom_to_drvpriv(h);
+
+	/* Initialize csi header */
+	_os_mem_set(drv, csi_header, 0, sizeof(struct csi_header_t));
+	/* from phy_info_rpt */
+	/* shift 1 for remove decimal point */
+	csi_header->rssi[0] = phy_info->rssi[0] >> 1;
+	csi_header->rssi[1] = phy_info->rssi[1] >> 1;
+	csi_header->rxsc = phy_info->rxsc;
+	/* from ch_rpt_hdr_info */
+	csi_header->nc = ch_hdr_rpt->n_rx - 1;
+	csi_header->nr = ch_hdr_rpt->n_sts;
+	/* shift 1 for remove decimal point */
+	csi_header->avg_idle_noise_pwr = ch_hdr_rpt->avg_noise_pow >> 1;
+	csi_header->csi_data_length = (ch_hdr_rpt->total_len_m <<16 | ch_hdr_rpt->total_len_l)
+		- 16; /* header(8byte) + Phy_info_in_seg_0(8byte) */
+	csi_header->csi_valid = ch_hdr_rpt->set_valid;
+	/* shift 1 for remove decimal point */
+	csi_header->evm[0] = ch_hdr_rpt->sts0_evm >> 2;
+	csi_header->evm[1] = (phy_info->sts1_evm_m << 4 | phy_info->sts1_evm_l) >> 2;
+}
+
+static void _hal_fill_csi_header_phy_info_ppdu_sts(struct hal_info_t *hal_info,
+	const struct rtw_r_meta_data *mdata, const struct hal_ppdu_sts_usr *usr,
+	const struct rtw_phl_ppdu_phy_info *phy_info, struct csi_header_t *csi_header)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	void *drv = halcom_to_drvpriv(hal_com);
+
+	/* Initialize csi header */
+	_os_mem_set(drv, csi_header, 0, sizeof(struct csi_header_t));
+	/* already remove decimal point in rtw_hal_bb_parse_phy_sts */
+	_os_mem_cpy(drv, &(csi_header->rssi[0]), &(phy_info->rssi_path[0]), 2);
+	csi_header->nc = phy_info->n_rx - 1;
+	csi_header->nr = phy_info->n_sts;
+	/* shift 1 for remove decimal point */
+	csi_header->avg_idle_noise_pwr = phy_info->avg_idle_noise_pwr >> 1;
+	/* shift 2 for remove decimal point */
+	csi_header->evm[0] = phy_info->evm_1_sts >> 2;
+	csi_header->evm[1] = phy_info->evm_2_sts >> 2;
+	csi_header->csi_data_length = phy_info->ch_info_len;
+	csi_header->rxsc = phy_info->rxsc;
+	csi_header->csi_valid = phy_info->is_ch_info_len_valid;
+
+	_hal_fill_csi_header_remain(hal_info, csi_header, mdata);
+	if (usr[0].vld) {
+		struct rtw_phl_stainfo_t *sta = rtw_phl_get_stainfo_by_macid(phl_com->phl_priv, usr[0].macid);
+		if (sta != NULL)
+			_os_mem_cpy(drv, &(csi_header->mac_addr[0]), sta->mac_addr, MAC_ADDRESS_LENGTH);
+	}
+#ifdef CONFIG_PHL_WKARD_CHANNEL_INFO_ACK
+	else {
+		_os_mem_cpy(drv, &(csi_header->mac_addr[0]), mdata->ta, MAC_ADDRESS_LENGTH);
+	}
+#endif
+}
+
+#ifndef CONFIG_PHL_CHANNEL_INFO_DIRECT_INDICATE
+static void _hal_get_ch_info_physts(void *hal,
+	struct rtw_r_meta_data *mdata, struct hal_ppdu_sts_usr *usr,
+	struct rtw_phl_ppdu_phy_info *phy_info)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	struct csi_header_t *csi_header = NULL;
+	void *drv = hal_to_drvpriv(hal_info);
+	struct chan_info_t *chan_info_old = NULL;
+	u8* buf_addr;
+	u32 idle_num = CHAN_INFO_PKT_TOTAL;
+
+	if (CHAN_INFO_MAX_SIZE < phy_info->ch_info_len) {
+		PHL_ERR("chan info buffer smaller than ch info len\n");
+		return;
+	}
+
+	idle_num = rtw_phl_get_chaninfo_idle_number(drv, phl_com);
+
+	if (idle_num == CHAN_INFO_PKT_TOTAL)
+		phl_com->chan_info = rtw_phl_query_idle_chaninfo(drv, phl_com);
+
+	if (phl_com->chan_info == NULL) {
+		/*hstatus = RTW_HAL_STATUS_SUCCESS is expected*/
+		PHL_INFO("channel info packet not avaialbe due to no pakcet handle\n");
+		return;
+	}
+
+	buf_addr = phl_com->chan_info->chan_info_buffer;
+	_os_mem_cpy(drv, buf_addr, phy_info->ch_info_addr, phy_info->ch_info_len);
+	phl_com->chan_info->length = phy_info->ch_info_len;
+	csi_header = &phl_com->chan_info->csi_header;
+
+	_hal_fill_csi_header_phy_info_ppdu_sts(hal_info, mdata, usr, phy_info, csi_header);
+
+#ifdef CONFIG_PHL_CHANNEL_INFO_DBG
+	hal_print_csi_raw_data(phl_com->chan_info);
+#endif
+	/* push compelete channel info resourecs to busy queue */
+	chan_info_old = rtw_phl_recycle_busy_chaninfo(drv, phl_com, phl_com->chan_info);
+	if (chan_info_old)
+			rtw_phl_enqueue_idle_chaninfo(drv, phl_com, chan_info_old);
+	phl_com->chan_info = rtw_phl_query_idle_chaninfo(drv, phl_com);
+	if(phl_com->chan_info == NULL)
+		PHL_INFO("channel info packet not avaialbe after recycle\n");
+}
+#else
+static void _hal_get_ch_info_physts_dind(void *hal,
+	struct rtw_r_meta_data *mdata, struct hal_ppdu_sts_usr *usr,
+	struct rtw_phl_ppdu_phy_info *phy_info)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	void *drv = hal_to_drvpriv(hal_info);
+
+	_hal_fill_csi_header_phy_info_ppdu_sts(hal_info, mdata, usr, phy_info
+		, &phl_com->chan_info->csi_header);
+
+	if (phl_com->evt_ops.csi_indicate) {
+		struct csi_ind_t csi_ind = {
+			.drv_priv = drv,
+			.csi_header = &phl_com->chan_info->csi_header,
+			.csi_raw = phy_info->ch_info_addr,
+		};
+
+		phl_com->evt_ops.csi_indicate(&csi_ind);
+	}
+}
+#endif
+
+void rtw_hal_get_ch_info_physts(void *hal,
+	struct rtw_r_meta_data *mdata, struct hal_ppdu_sts_usr *usr,
+	struct rtw_phl_ppdu_phy_info *phy_info)
+{
+#ifndef CONFIG_PHL_CHANNEL_INFO_DIRECT_INDICATE
+	_hal_get_ch_info_physts(hal, mdata, usr, phy_info);
+#else
+	_hal_get_ch_info_physts_dind(hal, mdata, usr, phy_info);
+#endif
+}
+
+enum rtw_hal_status
+rtw_hal_ch_info_en(void *hal, struct rtw_chinfo_action_parm *act_param)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	enum phl_phy_idx hw_phy_idx = HW_PHY_0;
+	u8 filter = 0, sg_size = 0;
+	u8 phy_idx = act_param->sta->rlink->hw_band;
+	bool en = act_param->enable;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct rtw_phl_stainfo_t *sta = act_param->sta;
+	struct rtw_phl_stainfo_t *self_sta = rtw_phl_get_stainfo_self(hal_info->phl_com->phl_priv, sta->rlink);
+	u16 macid;
+	struct rtw_chinfo_cur_parm *cur_parm = phl_com->cur_parm;
+	bool valid_ch_info_physts = false;
+	u8 arr_idx = 0;
+	void *drv = hal_to_drvpriv(hal_info);
+
+	switch (phy_idx) {
+	case 0:
+		hw_phy_idx = HW_PHY_0;
+		break;
+	case 1:
+		hw_phy_idx = HW_PHY_1;
+		break;
+	default:
+		hw_phy_idx = HW_PHY_0;
+		break;
+	}
+
+	PHL_INFO("[CH INFO] %s: en=%d,num=%d\n",__func__, en, cur_parm->num);
+	if (en == true) {
+		/* first enable BB */
+		if (cur_parm->num == 0) {
+			hal_status = rtw_hal_bb_ch_info_decision(hal_info, hw_phy_idx,
+						act_param->sta->chandef.bw,
+						sta->wmode, &valid_ch_info_physts);
+
+			if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s ch info decision fail(%d)\n", __func__, hal_status);
+				goto exit;
+			}
+
+			if (act_param->enable_mode == CHINFO_EN_AUTO_MODE) {
+				if (valid_ch_info_physts == true)
+					act_param->enable_mode = CHINFO_EN_LIGHT_MODE;
+				else
+					act_param->enable_mode = CHINFO_EN_RICH_MODE;
+			}
+
+			if (act_param->enable_mode == CHINFO_EN_LIGHT_MODE) {
+				rtw_hal_bb_ch_trig_select(hal_info, 1);
+				if (act_param->mode == CHINFO_MODE_ACK)
+					rtw_hal_bb_ch_info_physts_en(hal_info, en, hw_phy_idx, sta->wmode, true);
+				else if (act_param->mode == CHINFO_MODE_MACID)
+					rtw_hal_bb_ch_info_physts_en(hal_info, en, hw_phy_idx, sta->wmode, false);
+				else
+					PHL_ERR("[CH INFO] not implement for mode(%d)\n", act_param->mode);
+			} else {
+				rtw_hal_bb_ch_info_status_en(hal_info, en, hw_phy_idx);
+			}
+
+			/* first enbale, memcpy the current parameter */
+			_os_mem_cpy(drv, &cur_parm->action_parm,
+						act_param, sizeof(struct rtw_chinfo_action_parm));
+		}
+
+		macid = cur_parm->action_parm.mode == CHINFO_MODE_ACK ? self_sta->macid : sta->macid;
+
+		/* set macid info to MAC cfg for more one client */
+		if (phl_com->cur_parm->action_parm.enable_mode == CHINFO_EN_RICH_MODE)
+			hal_status = rtw_hal_mac_chan_info_cfg(hal_info, en, macid
+				, cur_parm->action_parm.mode, filter, sg_size);
+
+		/* add macid to cur_parm */
+		arr_idx = (u8)_os_division64(macid, 8);
+		cur_parm->macid_bitmap[arr_idx] |= BIT(_os_modular64(macid, 8));
+		cur_parm->num++;
+		PHL_INFO("[CH INFO] num:%d,macid_bitmap[%d]:0x%02x\n",
+			cur_parm->num, arr_idx, cur_parm->macid_bitmap[arr_idx]);
+	} else {
+		/* disable BB for no client */
+		if (cur_parm->num == 0) {
+			PHL_ERR("[CHAN INFO] error cur_parm->num = %d\n", cur_parm->num);
+			goto exit;
+		}
+
+		if (cur_parm->num == 1) {
+			if (cur_parm->action_parm.enable_mode== CHINFO_EN_LIGHT_MODE) {
+				if (cur_parm->action_parm.mode == CHINFO_MODE_ACK)
+					rtw_hal_bb_ch_info_physts_en(hal_info, en, hw_phy_idx, sta->wmode, true);
+				else if (cur_parm->action_parm.mode == CHINFO_MODE_MACID)
+					rtw_hal_bb_ch_info_physts_en(hal_info, en, hw_phy_idx, sta->wmode, false);
+				else
+					PHL_ERR("[CH INFO] not implement for mode(%d)\n",
+								cur_parm->action_parm.mode);
+			} else {
+				rtw_hal_bb_ch_info_status_en(hal_info, en, hw_phy_idx);
+			}
+			cur_parm->action_parm.enable = false;
+		}
+
+		macid = cur_parm->action_parm.mode == CHINFO_MODE_ACK ? self_sta->macid : sta->macid;
+
+		/* remvoe macid info to MAC cfg for more one client */
+		if (phl_com->cur_parm->action_parm.enable_mode == CHINFO_EN_RICH_MODE)
+			hal_status = rtw_hal_mac_chan_info_cfg(hal_info, en, macid
+				, cur_parm->action_parm.mode, filter, sg_size);
+
+		/* remove macid to cur_parm */
+		arr_idx = (u8)_os_division64(macid, 8);
+		cur_parm->macid_bitmap[arr_idx] &= ~(BIT(_os_modular64(macid, 8)));
+		cur_parm->num--;
+		PHL_INFO("[CH INFO] num:%d,macid_bitmap[%d]:0x%02x\n",
+			cur_parm->num, arr_idx, cur_parm->macid_bitmap[arr_idx]);
+	}
+
+exit:
+	return hal_status;
+}
+
+#ifdef CONFIG_PHL_CSI_FW_TX_OFLD
+enum rtw_hal_status
+rtw_hal_ch_info_pkt_ofld(void *hal, u16 macid, u8 en, u16 period, u8 retry_cnt,
+				u16 rate, u8 pkt_num, u8 *pkt_id)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_cfg_sensing_csi(hal_info, macid, en, period, retry_cnt, rate, pkt_num, pkt_id);
+}
+#endif
+
+#ifdef CONFIG_PHL_CHANNEL_INFO_DBG
+void
+hal_print_csi_raw_data(struct chan_info_t *chan_info)
+{
+	u64 *buff_tmp = NULL;
+	u32 print_len = chan_info->csi_header.csi_data_length >> 3;
+	struct csi_header_t csi_header = chan_info->csi_header;
+	u32 i;
+
+	if (chan_info->chan_info_buffer == NULL
+		|| chan_info->length == 0
+		|| csi_header.csi_data_length == 0) {
+		PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] chan_info_buffer is NULL\n");
+		return;
+	}
+
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] CSI raw data: len=%d\n",
+											csi_header.csi_data_length);
+
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_,
+		"[CH INFO] - mac addr: %02x:%02x:%02x:%02x:%02x:%02x\n",
+	         csi_header.mac_addr[0], csi_header.mac_addr[1], csi_header.mac_addr[2],
+	         csi_header.mac_addr[3], csi_header.mac_addr[4], csi_header.mac_addr[5]);
+
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] rxsc=%d\n",
+											csi_header.rxsc);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] nc=%d\n",
+											csi_header.nc);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] nr=%d\n",
+											csi_header.nr);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] avg_idle_noise_pwr=%d\n",
+											csi_header.avg_idle_noise_pwr);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] csi_valid=%d\n",
+											csi_header.csi_valid);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] evm[0]=%d\n",
+											csi_header.evm[0]);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] evm[1]=%d\n",
+											csi_header.evm[1]);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] rssi[0]=%d%%\n",
+											csi_header.rssi[0]);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] rssi[1]=%d%%\n",
+											csi_header.rssi[1]);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] timestamp=%d\n",
+											csi_header.hw_assigned_timestamp);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] rx_data_rate=0x%02x\n",
+											csi_header.rx_data_rate);
+	PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[CH INFO] bandwidth=%d\n",
+											csi_header.bandwidth);
+
+	if (chan_info->csi_header.csi_data_length % 8)
+		print_len++;
+
+	buff_tmp = (u64 *)chan_info->chan_info_buffer;
+
+	for (i = 0; i < print_len; i++)
+		PHL_TRACE(COMP_PHL_CHINFO, _PHL_INFO_, "[%02d]0x%016llx\n", i, buff_tmp[i]);
+
+	PHL_DATA(COMP_PHL_CHINFO, _PHL_INFO_, "\n");
+}
+#endif
+#ifdef CONFIG_PHL_WKARD_CHANNEL_INFO_ACK
+u8 rtw_hal_ch_info_process_ack(struct rtw_r_meta_data *meta,
+									struct rtw_phl_ppdu_sts_ent *ppdu_sts_ent,
+									struct rtw_chinfo_cur_parm *cur_parm,
+									u16 macid)
+{
+	if (ppdu_sts_ent->frame_type == RTW_FRAME_TYPE_ACK
+		&& ((cur_parm->action_parm.chk_ack_rate == true && meta->rx_rate == cur_parm->rate) || cur_parm->action_parm.chk_ack_rate == false)
+	) {
+		int map_idx = (int)macid / 32;
+
+		/* check macid match or not */
+		if (cur_parm->macid_bitmap[map_idx] & BIT(macid % 32))
+			return true;
+		else
+			return false;
+	} else {
+		return false;
+	}
+}
+#endif
+
+#ifndef CONFIG_PHL_CHANNEL_INFO_DIRECT_INDICATE
+static void _hal_handle_ch_info_from_chan_sts(struct rtw_phl_com_t *phl_com,
+	struct hal_info_t *hal, struct rtw_pkt_buf_list *pkt, struct rtw_r_meta_data *mdata)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	void *drv = hal_to_drvpriv(hal);
+	u8* buf_addr;
+	struct ch_rpt_hdr_info ch_hdr_rpt = {0};
+	struct phy_info_rpt phy_rpt = {0};
+	struct ch_info_drv_rpt drv_rpt = {0};
+
+	u32 idle_num = CHAN_INFO_PKT_TOTAL;
+	struct chan_info_t *chan_info_old = NULL;
+
+	/* Channel Report */
+	/* TODO: need to discuss the final csi header format further.*/
+	idle_num = rtw_phl_get_chaninfo_idle_number(drv, phl_com);
+
+	if (idle_num == CHAN_INFO_PKT_TOTAL)
+		phl_com->chan_info = rtw_phl_query_idle_chaninfo(drv, phl_com);
+
+	if (phl_com->chan_info == NULL) {
+		/*hstatus = RTW_HAL_STATUS_SUCCESS is expected*/
+		PHL_INFO("channel info packet not avaialbe due to no pakcet handle\n");
+		return;
+	}
+
+	buf_addr = phl_com->chan_info->chan_info_buffer;
+	status = rtw_hal_bb_ch_info_parsing(hal, pkt->vir_addr, mdata,
+		buf_addr + phl_com->chan_info->length,
+		&ch_hdr_rpt, &phy_rpt, &drv_rpt);
+
+	if (status == RTW_HAL_STATUS_FAILURE) {
+		phl_com->chan_info->length = 0;
+		return;
+	}
+
+	phl_com->chan_info->length += drv_rpt.raw_data_len;
+	/* store phy info if seg#0 is success*/
+	if (drv_rpt.seg_idx_curr == 0)
+		_hal_fill_csi_header_phy_info(hal, &(phl_com->chan_info->csi_header),
+			&ch_hdr_rpt, &phy_rpt);
+	if (status == RTW_HAL_STATUS_BB_CH_INFO_LAST_SEG) {
+		/* Fill remain csi header to buffer  */
+		_hal_fill_csi_header_remain(hal,
+			&(phl_com->chan_info->csi_header), mdata);
+		#ifdef CONFIG_PHL_CHANNEL_INFO_DBG
+		hal_print_csi_raw_data(phl_com->chan_info);
+		#endif
+
+		/* push compelete channel info resourecs to busy queue */
+		chan_info_old = rtw_phl_recycle_busy_chaninfo(drv, phl_com, phl_com->chan_info);
+		if (chan_info_old)
+			rtw_phl_enqueue_idle_chaninfo(drv, phl_com, chan_info_old);
+		phl_com->chan_info = rtw_phl_query_idle_chaninfo(drv, phl_com);
+		if(phl_com->chan_info == NULL)
+			PHL_INFO("channel info packet not avaialbe after recycle\n");
+	}
+}
+#else
+static void _hal_handle_ch_info_from_chan_sts_dind(struct rtw_phl_com_t *phl_com,
+	struct hal_info_t *hal, struct rtw_pkt_buf_list *pkt, struct rtw_r_meta_data *mdata)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	void *drv = hal_to_drvpriv(hal);
+	u8* buf_addr;
+	struct ch_rpt_hdr_info ch_hdr_rpt = {0};
+	struct phy_info_rpt phy_rpt = {0};
+	struct ch_info_drv_rpt drv_rpt = {0};
+
+	buf_addr = phl_com->chan_info->chan_info_buffer;
+	status = rtw_hal_bb_ch_info_parsing(hal, pkt->vir_addr, mdata,
+		buf_addr + phl_com->chan_info->length,
+		&ch_hdr_rpt, &phy_rpt, &drv_rpt);
+
+	if (status == RTW_HAL_STATUS_FAILURE) {
+		phl_com->chan_info->length = 0;
+		return;
+	}
+
+	phl_com->chan_info->length += drv_rpt.raw_data_len;
+	/* store phy info if seg#0 is success*/
+	if (drv_rpt.seg_idx_curr == 0)
+		_hal_fill_csi_header_phy_info(hal, &(phl_com->chan_info->csi_header),
+			&ch_hdr_rpt, &phy_rpt);
+	if (status == RTW_HAL_STATUS_BB_CH_INFO_LAST_SEG) {
+		/* Fill remain csi header to buffer  */
+		_hal_fill_csi_header_remain(hal,
+			&(phl_com->chan_info->csi_header), mdata);
+		#ifdef CONFIG_PHL_CHANNEL_INFO_DBG
+		hal_print_csi_raw_data(phl_com->chan_info);
+		#endif
+
+		if (phl_com->evt_ops.csi_indicate) {
+			struct csi_ind_t csi_ind = {
+				.drv_priv = drv,
+				.csi_header = &phl_com->chan_info->csi_header,
+				.csi_raw = phl_com->chan_info->chan_info_buffer,
+			};
+
+			phl_com->evt_ops.csi_indicate(&csi_ind);
+		}
+
+		phl_chaninfo_pkt_init(drv, phl_com->chan_info);
+	}
+}
+#endif
+
+void hal_handle_ch_info_from_chan_sts(struct rtw_phl_com_t *phl_com,
+	struct hal_info_t *hal, struct rtw_pkt_buf_list *pkt, struct rtw_r_meta_data *mdata)
+{
+#ifndef CONFIG_PHL_CHANNEL_INFO_DIRECT_INDICATE
+	_hal_handle_ch_info_from_chan_sts(phl_com, hal, pkt, mdata);
+#else
+	_hal_handle_ch_info_from_chan_sts_dind(phl_com, hal, pkt, mdata);
+#endif
+}
+#endif /* CONFIG_PHL_CHANNEL_INFO */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan_info.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan_info.h
new file mode 100644
index 000000000000..aab4e59ebb20
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_chan_info.h
@@ -0,0 +1,49 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_CHAN_INFO_H_
+#define _HAL_CHAN_INFO_H_
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+
+void
+_hal_fill_csi_header_remain(void* hal, struct csi_header_t *csi_header
+	, const struct rtw_r_meta_data *mdata);
+
+void
+_hal_fill_csi_header_phy_info(void* hal, struct csi_header_t *csi_header
+	, struct ch_rpt_hdr_info *ch_hdr_rpt, struct phy_info_rpt *phy_info);
+
+void
+rtw_hal_get_ch_info_physts(void *hal,
+	struct rtw_r_meta_data *mdata, struct hal_ppdu_sts_usr *usr,
+	struct rtw_phl_ppdu_phy_info *phy_info);
+
+#ifdef CONFIG_PHL_WKARD_CHANNEL_INFO_ACK
+u8 rtw_hal_ch_info_process_ack(struct rtw_r_meta_data *meta,
+									struct rtw_phl_ppdu_sts_ent *ppdu_sts_ent,
+									struct rtw_chinfo_cur_parm *cur_parm,
+									u16 macid);
+#endif
+
+#ifdef CONFIG_PHL_CHANNEL_INFO_DBG
+void
+hal_print_csi_raw_data(struct chan_info_t *chan_info);
+#endif
+
+void hal_handle_ch_info_from_chan_sts(struct rtw_phl_com_t *phl_com,
+	struct hal_info_t *hal, struct rtw_pkt_buf_list *pkt, struct rtw_r_meta_data *mdata);
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+#endif /* _HAL_CHAN_INFO_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_com_i.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_com_i.c
new file mode 100644
index 000000000000..f67431a11905
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_com_i.c
@@ -0,0 +1,467 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2024 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_COM_I_C_
+#define _HAL_COM_I_C_
+#include "hal_headers.h"
+
+#define HAL_MAC_TX_LIFETIME_UNIT_US_SHT 8 /* 256 us */
+
+void
+rtw_hal_com_scan_set_tx_lifetime(void *hal, u8 band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	/* set mgnt queue pkt lifetime */
+#define HAL_COM_MGQ_TX_LIFETIME_MS 19
+	rtw_hal_mac_set_tx_lifetime(hal_info, band, PHL_LIFETIME_MGQ, true,
+		((HAL_COM_MGQ_TX_LIFETIME_MS * 1000) >> HAL_MAC_TX_LIFETIME_UNIT_US_SHT));
+}
+
+void rtw_hal_com_scan_restore_tx_lifetime(void *hal, u8 band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	/* reset data/mgnt queue pkt lifetime to 0 */
+	rtw_hal_mac_set_tx_lifetime(hal_info, band, PHL_LIFETIME_MGQ, false, 0);
+}
+
+/* set ac queue pkt lifetime, val is in unit of ms */
+enum rtw_hal_status
+rtw_hal_com_set_tx_lifetime(void *hal, u8 band, u8 en, u16 val)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	if (en)
+		return rtw_hal_mac_set_tx_lifetime(hal_info, band, PHL_LIFETIME_ACQ, true,
+					((val * 1000) >> HAL_MAC_TX_LIFETIME_UNIT_US_SHT));
+	else
+		return rtw_hal_mac_set_tx_lifetime(hal_info, band, PHL_LIFETIME_ACQ, false,
+					0);
+}
+
+/* set TPU ofst_mode & ofst_bw, step 0.5 dB */
+enum rtw_hal_status
+rtw_hal_com_set_power_offset(void *hal, u8 band, s8 ofst_mode, s8 ofst_bw)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts;
+
+	hsts = rtw_hal_mac_set_pwr_ofst_mode(hal_info->hal_com, band, ofst_mode);
+	if(hsts != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("%s set tpu pwr_ofst_mode - failed\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hsts = rtw_hal_mac_write_pwr_ofst_mode(hal_info->hal_com, band);
+	if(hsts != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("%s write tpu pwr_ofst_mode - failed\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hsts = rtw_hal_mac_set_pwr_ofst_bw(hal_info->hal_com, band, ofst_bw);
+	if(hsts != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("%s set tpu pwr_ofst_bw - failed\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hsts = rtw_hal_mac_write_pwr_ofst_bw(hal_info->hal_com, band);
+	if(hsts != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("%s write tpu pwr_ofst_bw - failed\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_com_set_gt3(void *hal, u8 en, u8 timeout)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_set_gt3(hal_info, en, timeout);
+}
+
+enum rtw_hal_status rtw_hal_scan_pause_tx_fifo(void *hinfo,
+	u8 band_idx, bool off_ch)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hinfo;
+
+
+	hal_status = rtw_hal_tx_pause(hal_info->hal_com, band_idx,
+		off_ch, PAUSE_RSON_NOR_SCAN);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_dfs_pause_tx(void *hinfo, u8 band_idx, bool off_ch, enum tx_pause_rson reason)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hinfo;
+
+	hal_status = rtw_hal_tx_pause(hal_info->hal_com, band_idx,
+				      off_ch, reason);
+
+	return hal_status;
+}
+
+#ifdef CONFIG_FSM
+enum rtw_hal_status rtw_hal_scan_flush_queue(void *hinfo,
+	struct rtw_wifi_role_t *wrole)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+
+enum rtw_hal_status rtw_hal_notify_switch_band(void *hinfo,
+			enum band_type band, enum phl_phy_idx phy_idx)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hinfo;
+
+	PHL_INFO("%s: band(%d), phy_idx(%d)\n", __FUNCTION__, band, phy_idx);
+#ifdef CONFIG_BTCOEX
+	rtw_hal_btc_switch_band_ntfy(hal_info, phy_idx, band);
+#endif
+
+	rtw_hal_bb_fw_edcca(hal_info);
+
+	rtw_hal_rf_set_power(hal_info, phy_idx, PWR_BY_RATE);
+
+	rtw_hal_rf_do_tssi_scan(hal_info, phy_idx);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_proc_cmd(void *hal, char proc_cmd, struct rtw_proc_cmd *incmd,
+							char *output, u32 out_len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	if(proc_cmd == RTW_PROC_CMD_BB){
+		if(rtw_hal_bb_proc_cmd(hal_info, incmd, output, out_len))
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+	}
+	if(proc_cmd == RTW_PROC_CMD_RF){
+		if(rtw_hal_rf_proc_cmd(hal_info, incmd, output, out_len))
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+	}
+	if(proc_cmd == RTW_PROC_CMD_MAC){
+#ifdef CONFIG_HAL_MAC_DBG
+		if(rtw_hal_mac_proc_cmd(hal_info, incmd, output, out_len))
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+#else
+		hal_status = RTW_HAL_STATUS_NOT_SUPPORT;
+#endif
+	}
+#ifdef CONFIG_BTCOEX
+	if(proc_cmd == RTW_PROC_CMD_BTC){
+		if(rtw_hal_btc_proc_cmd(hal_info, incmd, output, out_len))
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+	}
+#endif
+	if(proc_cmd == RTW_PROC_CMD_EFUSE){
+		if(rtw_hal_efuse_proc_cmd(hal_info, incmd, output, out_len))
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+	}
+	return hal_status;
+}
+
+void rtw_hal_get_mac_version(char *ver_str, u16 len)
+{
+	rtw_hal_mac_get_version(ver_str, len);
+}
+
+void rtw_hal_get_fw_ver(void *hal, char *ver_str, u16 len)
+{
+	rtw_hal_mac_get_fw_ver((struct hal_info_t *)hal, ver_str, len);
+}
+
+enum rtw_hal_status
+rtw_hal_antdiv_fix_ant(void *hal, u8 antIndex)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_bb_antdiv_fix_ant(hal_com, antIndex);
+	return hal_status;
+}
+
+enum rf_path rtw_hal_get_path_from_ant_num(void *hal, u8 antnum)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+	return hal_ops->get_path_from_ant_num(antnum);
+}
+
+enum rtw_hal_status
+rtw_hal_set_mu_edca(void *hal, u8 band, u8 ac,
+	u16 timer, u8 cw_min, u8 cw_max, u8 aifsn)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	u16 timer_32us;
+	u8 aifs_us;
+	u8 slot_time, sifs;
+
+	/* TODO: Get aSlotTime and aSIFS according to current PHY */
+	slot_time = 9;
+	sifs = 16;
+	timer_32us = (timer<<8);
+	aifs_us = (aifsn == 0)?0:(aifsn*slot_time + sifs);
+	hal_status = rtw_hal_mac_set_mu_edca(hal_info->hal_com,
+					band, ac, timer_32us, cw_min, cw_max, aifs_us);
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_set_mu_edca_ctrl(void *hal, u8 band, u8 wmm, u8 set)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_mac_set_mu_edca_ctrl(hal_info->hal_com,
+					band, wmm, set);
+	return hal_status;
+}
+
+/*init rx filter opt,type, max MPDU size, RTS threshold*/
+enum rtw_hal_status
+rtw_hal_hw_param_init(struct hal_info_t *hal_info, struct rtw_phl_com_t *phl_com,
+				enum phl_band_idx band_idx)
+{
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct phy_cap_t *phy_cap = phl_com->phy_cap;
+
+	hsts = rtw_hal_set_rxfltr_opt_by_mode(hal_info, band_idx, RX_FLTR_OPT_MODE_STA_NORMAL);
+	if (hsts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s set rx filter opt failed\n", __func__);
+		goto _exit;
+	}
+	hsts = rtw_hal_set_rxfltr_type_by_mode(hal_info, band_idx, RX_FLTR_TYPE_MODE_HAL_INIT);
+	if (hsts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s set rx filter type failed\n", __func__);
+		goto _exit;
+	}
+	hsts = rtw_hal_mac_set_rxfltr_mpdu_size(hal_info->hal_com, band_idx, 0x2c00);
+	if (hsts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s set mpdu size failed\n", __func__);
+		goto _exit;
+	}
+
+	hsts = rtw_hal_mac_set_hw_rts_th(hal_info, band_idx,
+				  phy_cap[band_idx].hw_rts_time_th,
+				  phy_cap[band_idx].hw_rts_len_th);
+	if (hsts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s set RTS threshold failed\n", __func__);
+		goto _exit;
+	}
+
+_exit:
+	return hsts;
+}
+
+enum rtw_hal_status rtw_hal_ppdu_sts_init(void *hal, struct hal_ppdu_sts_cfg *cfg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+
+	hal_status = hal_info->hal_ops.cfg_ppdu_sts(hal_info, cfg);
+
+	hal_com->band[cfg->band_idx].ppdu_sts_appen_info = cfg->appen_info;
+	hal_com->band[cfg->band_idx].ppdu_sts_filter = cfg->filter;
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("%s - failed\n", __func__);
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_ppdu_sts_cfg(void *hal, u8 band_idx, bool en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_ppdu_sts_cfg cfg = {0};
+
+	cfg.band_idx = band_idx;
+	cfg.ppdu_stat_en = en;
+	if(en) {
+		cfg.appen_info = hal_com->band[band_idx].ppdu_sts_appen_info;
+		cfg.filter = hal_com->band[band_idx].ppdu_sts_filter;
+	}
+
+	hal_status = hal_info->hal_ops.cfg_ppdu_sts(hal_info, &cfg);
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("%s (en %d) - failed\n", __func__, en);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_reset(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx, u8 band_idx, bool reset)
+{
+	struct hal_info_t *hal_info = hal_com->hal_priv;
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	PHL_INFO("%s: phy_idx(%d) band_idx(%d)\n", __FUNCTION__, phy_idx, band_idx);
+
+	if (reset){
+		/* disable contention */
+		status = rtw_hal_tx_pause(hal_com, band_idx, true, PAUSE_RSON_RESET);
+		if(status != RTW_HAL_STATUS_SUCCESS){
+			PHL_ERR("%s rtw_hal_tx_pause - failed\n", __func__);
+			return status;
+		}
+
+		/* disable ppdu_sts */
+		status = rtw_hal_ppdu_sts_cfg(hal_info, band_idx, false);
+		if(status != RTW_HAL_STATUS_SUCCESS){
+			PHL_ERR("%s rtw_hal_ppdu_sts_cfg - failed\n", __func__);
+			return status;
+		}
+
+		rtw_hal_bb_bb_reset_cmn(hal_com, true, phy_idx);
+	}else{
+		/*enable ppdu_sts*/
+		status = rtw_hal_ppdu_sts_cfg(hal_info, band_idx, true);
+
+		if(status != RTW_HAL_STATUS_SUCCESS){
+			PHL_ERR("%s rtw_hal_ppdu_sts_cfg - failed\n", __func__);
+			return status;
+		}
+
+		rtw_hal_bb_bb_reset_cmn(hal_com, false, phy_idx);
+
+		status = rtw_hal_tx_pause(hal_com, band_idx, false, PAUSE_RSON_RESET);
+		if(status != RTW_HAL_STATUS_SUCCESS){
+			PHL_ERR("%s rtw_hal_tx_pause - failed\n", __func__);
+			return status;
+		}
+	}
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return status;
+}
+
+void
+rtw_hal_disconnect_notify(void *hal, struct rtw_chan_def *chandef)
+{
+	rtw_hal_rf_disconnect_notify(hal, chandef);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_disconnect_notify(): chandef band(%d), chan(%d), bw(%d), offset(%d)\n",
+		chandef->band, chandef->chan, chandef->bw, chandef->offset);
+}
+
+bool rtw_hal_check_ch_rfk(void *hal, struct rtw_chan_def *chandef)
+{
+	bool check = false;
+	check = rtw_hal_rf_check_mcc_ch(hal, chandef);
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_check_ch_rfk(): check ok(%d), band(%d), chan(%d), bw(%d), offset(%d)\n",
+		check, chandef->band, chandef->chan, chandef->bw, chandef->offset);
+	return check;
+}
+
+void rtw_hal_env_rpt(struct rtw_hal_com_t *hal_com,
+                     struct rtw_env_report *env_rpt,
+                     u8 hw_band)
+{
+	enum phl_phy_idx p_idx = HW_PHY_0;
+
+	p_idx = rtw_hal_hw_band_to_phy_idx(hw_band);
+	rtw_hal_bb_env_rpt(hal_com, env_rpt, p_idx);
+}
+
+enum phl_band_idx rtw_hal_phy_idx_to_hw_band(enum phl_phy_idx p_idx)
+{
+	enum phl_band_idx band_idx = HW_BAND_MAX;
+
+	if (p_idx == HW_PHY_0)
+		band_idx = HW_BAND_0;
+	else if (p_idx == HW_PHY_1)
+		band_idx = HW_BAND_1;
+	else
+		PHL_ERR("%s: error phy_idx(%d)\n", __func__, p_idx);
+	return band_idx;
+}
+
+enum phl_phy_idx rtw_hal_hw_band_to_phy_idx(enum phl_band_idx band_idx)
+{
+	enum phl_phy_idx p_idx = HW_PHY_MAX;
+
+	if (band_idx == HW_BAND_0)
+		p_idx = HW_PHY_0;
+	else if (band_idx == HW_BAND_1)
+		p_idx = HW_PHY_1;
+	else
+		PHL_ERR("%s: error band_idx(%d)\n", __func__, band_idx);
+	return p_idx;
+}
+
+void
+rtw_hal_init_hw_band_info(void *hal, enum phl_band_idx band_idx)
+{
+	struct hal_info_t *hal_i = (struct hal_info_t *)hal;
+	struct rtw_hw_band *hw_band_i = &hal_i->hal_com->band[band_idx];
+
+	PHL_INFO("%s: band_idx(%d)\n", __FUNCTION__, band_idx);
+	_os_mem_set(hal_i->phl_com->drv_priv, hw_band_i,
+			0, sizeof(struct rtw_hw_band));
+	rtw_phl_init_chdef(hal_i->phl_com, &hw_band_i->cur_chandef);
+}
+#ifdef DBG_DUMP_TX_COUNTER
+void rtw_hal_dump_tx_status(void *hal, enum phl_band_idx bidx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_mac_get_tx_cnt(hal_info, bidx, 0);
+	rtw_hal_bb_dump_tx_sts(hal_info, true, rtw_hal_hw_band_to_phy_idx(bidx));
+
+}
+#endif
+
+void rtw_hal_get_mac_sel_tx_status(void *hal, enum phl_band_idx bidx, void *out_tx_cnt)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_mac_get_sel_tx_cnt(hal_info, bidx, out_tx_cnt);
+
+}
+
+enum rtw_hal_status
+rtw_hal_tbtt_tuning(void *hal, enum phl_band_idx band,
+			u8 port, u32 tbtt)
+{
+	return rtw_hal_mac_tbtt_tuning_cfg(hal, band, port, tbtt);
+}
+#endif /* _HAL_COM_I_C_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_com_i.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_com_i.h
new file mode 100644
index 000000000000..2eaaa9b26ac6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_com_i.h
@@ -0,0 +1,29 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_COM_I_H_
+#define _HAL_COM_I_H_
+
+enum rtw_hal_status
+rtw_hal_hw_param_init(struct hal_info_t *hal_info, struct rtw_phl_com_t *phl_com,
+				enum phl_band_idx band_idx);
+enum rtw_hal_status rtw_hal_ppdu_sts_cfg(void *hal, u8 band_idx, bool en);
+enum rtw_hal_status rtw_hal_ppdu_sts_init(void *hal, struct hal_ppdu_sts_cfg *cfg);
+
+enum phl_band_idx rtw_hal_phy_idx_to_hw_band(enum phl_phy_idx p_idx);
+
+enum phl_phy_idx rtw_hal_hw_band_to_phy_idx(enum phl_band_idx band_idx);
+
+void rtw_hal_init_hw_band_info(void *hal, enum phl_band_idx band_idx);
+#endif /* _HAL_COM_I_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_config.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_config.h
new file mode 100644
index 000000000000..5c4901f75785
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_config.h
@@ -0,0 +1,145 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_CONFIG_H_
+#define _HAL_CONFIG_H_
+#include "mac/chip_cfg.h"
+
+#ifdef CONFIG_PHL_TEST_MP
+	#define CONFIG_HAL_TEST_MP
+#endif
+
+#ifdef CONFIG_PHL_TEST_FPGA
+	#define CONFIG_HAL_TEST_FPGA
+#endif
+
+#define MAX_PATH_LEN			256
+
+#define MAX_LINES_HWCONFIG_TXT	2000
+#define MAX_BYTES_LINE_HWCONFIG_TXT 	256
+#define MAX_HWCONFIG_FILE_CONTENT	MAX_LINES_HWCONFIG_TXT * MAX_BYTES_LINE_HWCONFIG_TXT
+
+/* Disable HALBB Features */
+#ifndef CONFIG_PHL_DFS
+	#define DRV_BB_DFS_DISABLE
+#endif
+
+#ifdef CONFIG_PHL_THERMAL_PROTECT
+	#define CONFIG_HAL_THERMAL_PROTECT
+#endif
+
+#ifdef CONFIG_BB_TIMER_SUPPORT_DISABLE
+	#define DRV_BB_TIMER_SUPPORT_DISABLE
+#endif
+
+/* HALMAC Features */
+/*
+ * CONFIG_HAL_MAC_DBG - HALMAC debug function is supported or not
+ *
+ * HALMAC would check this flag to decide debug code would be included or not.
+ * If debug feature doesn't be needed for stable/release version, undef this
+ * flag could help to reduce code size.
+ */
+#ifndef CONFIG_CORE_DBG_NONE
+#define CONFIG_HAL_MAC_DBG
+#endif
+
+/* Compile flag for fw for different cut versions.
+* This should be wrapped with the compile flags from core, and need to be discussed
+*/
+#if !defined(CONFIG_FW_SPECIFY_FROM_CORE)
+#ifdef PHL_FEATURE_NIC
+#define MAC_FW_CATEGORY_NIC
+#endif /*PHL_FEATURE_NIC*/
+
+#ifdef PHL_FEATURE_AP
+#define MAC_FW_CATEGORY_AP
+#endif /*PHL_FEATURE_AP*/
+
+#ifdef CONFIG_WOWLAN
+#define MAC_FW_CATEGORY_WOWLAN
+#endif /*CONFIG_WOWLAN*/
+
+/*8851B*/
+#ifdef CONFIG_RTL8851B
+#ifndef PHL_PLATFORM_UEFI
+#define MAC_FW_8851B_U1
+#endif
+#define MAC_FW_8851B_U2
+#define MAC_FW_CATEGORY_NICCE
+#endif
+
+/*8852A*/
+#ifdef CONFIG_RTL8852A
+#define MAC_FW_8852A_U1
+#define MAC_FW_8852A_U2
+#define MAC_FW_8852A_U3
+/*CATEG*/
+/*#define MAC_FW_CATEGORY_VRAP*/ /*sd7 only*/
+#endif
+
+/*8852B*/
+#if defined(CONFIG_RTL8852B) || defined(CONFIG_RTL8852BP) || defined(CONFIG_RTL8852BPT) || defined(CONFIG_RTL8852BT)
+#define MAC_FW_8852B_U1
+#define MAC_FW_8852B_U2
+#define MAC_FW_8852B_U3
+/*CATEG ,8852B need to set CATEGORY_NIC_BPLUS 0*/
+#ifdef CONFIG_RTL8852B
+#define MAC_FW_CATEGORY_NICCE
+#define MAC_FW_CATEGORY_NIC_PLE
+#endif /*CONFIG_RTL8852B*/
+
+#ifdef CONFIG_RTL8852BP
+#define MAC_FW_CATEGORY_NIC_BPLUS
+#endif /*CONFIG_RTL8852BP*/
+
+#ifdef CONFIG_RTL8852BT
+#define MAC_FW_8852BT_U1
+#endif /*CONFIG_RTL8852BT*/
+
+#include "mac/fw_ax/rtl8852b/hal8852b_fw_cap.h"
+#include "rtl8852b/hal_feature_8852b.h"
+#endif  /*(CONFIG_RTL8852B) || defined(CONFIG_RTL8852BP)*/
+
+#ifdef CONFIG_RTL8852C
+#define MAC_FW_8852C_U1
+#define MAC_FW_8852C_U2
+/*CATEG*/
+/*#define MAC_FW_CATEGORY_VRAP*/ /*sd7 only*/
+#endif /*CONFIG_RTL8852C*/
+
+#ifdef CONFIG_RTL8842A
+#define MAC_FW_8842A_U1
+#endif /*CONFIG_RTL8842A*/
+
+#ifdef CONFIG_RTL8852D
+#define MAC_FW_8852D_U1
+#define MAC_FW_8852D_U2
+#endif /*CONFIG_RTL8852D*/
+
+#if defined(CONFIG_RTL8192XB) || defined(CONFIG_RTL8832BR)
+#define MAC_FW_8192XB_U1
+#endif
+#endif /*CONFIG_FW_SPECIFY_FROM_CORE*/
+
+#ifdef CONFIG_WPP
+#define DRV_BB_DBG_TRACE_DISABLE
+#endif
+/* HALRF Features */
+
+#ifdef RTW_WKARD_PHL_LPS_IQK_TWICE
+#define RTW_WKARD_HAL_LPS_IQK_TWICE
+#endif
+#include "hal_feature.h"
+#endif /*_HAL_CONFIG_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_csi_buffer.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_csi_buffer.c
new file mode 100644
index 000000000000..916fbc672e55
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_csi_buffer.c
@@ -0,0 +1,611 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include "hal_headers.h"
+
+/**
+ * __query_avl_buf_idx_20
+ * 	Get available sub 20MHz csi buffer
+ * input :
+ * @csi_buf: (struct hal_csi_buf *) 80MHz hal csi buffer for checking avaible
+ * return
+ * @sub_idx: (u8) csi buffer sub index
+ **/
+u8 __query_avl_buf_idx_20(struct hal_csi_buf *csi_buf)
+{
+	u8 sub_idx = CSI_BUF_SUB_IDX_NON;
+	do {
+		if (IS_SUB20_BUSY(csi_buf, 0) == CSI_BUF_STS_IDLE) {
+			SET_SUB20_BUSY(csi_buf, 0);
+			sub_idx = CSI_BUF_SUB_IDX_20_LL;
+			break;
+		}
+		if (IS_SUB20_BUSY(csi_buf, 1) == CSI_BUF_STS_IDLE) {
+			SET_SUB20_BUSY(csi_buf, 1);
+			sub_idx = CSI_BUF_SUB_IDX_20_LU;
+			break;
+		}
+		if (IS_SUB20_BUSY(csi_buf, 2) == CSI_BUF_STS_IDLE) {
+			SET_SUB20_BUSY(csi_buf, 2);
+			sub_idx = CSI_BUF_SUB_IDX_20_UL;
+			break;
+		}
+		if (IS_SUB20_BUSY(csi_buf, 3) == CSI_BUF_STS_IDLE) {
+			SET_SUB20_BUSY(csi_buf, 3);
+			sub_idx = CSI_BUF_SUB_IDX_20_UU;
+			break;
+		}
+	} while (0);
+
+	return sub_idx;
+}
+/**
+ * __query_avl_buf_idx_40
+ * 	Get available sub 40MHz csi buffer
+ * input :
+ * @csi_buf: (struct hal_csi_buf *) 80MHz hal csi buffer for checking avaible
+ * return
+ * @sub_idx: (u8) csi buffer sub index
+ **/
+u8 __query_avl_buf_idx_40(struct hal_csi_buf *csi_buf)
+{
+	u8 sub_idx = CSI_BUF_SUB_IDX_NON;
+	do {
+		if(IS_40L_BUSY(csi_buf) == CSI_BUF_STS_IDLE) {
+			SET_40L_BUSY(csi_buf);
+			sub_idx = CSI_BUF_SUB_IDX_40_L;
+			break;
+		}
+		if (IS_40U_BUSY(csi_buf) == CSI_BUF_STS_IDLE) {
+			SET_40U_BUSY(csi_buf);
+			sub_idx = CSI_BUF_SUB_IDX_40_U;
+			break;
+		}
+	} while (0);
+
+	return sub_idx;
+}
+/**
+ * __query_avl_buf_idx_80
+ * 	Get available sub 80MHz csi buffer
+ * input :
+ * @csi_buf: (struct hal_csi_buf *) 80MHz hal csi buffer for checking avaible
+ * return
+ * @sub_idx: (u8) csi buffer sub index
+ **/
+u8 __query_avl_buf_idx_80(struct hal_csi_buf *csi_buf)
+{
+	u8 sub_idx = CSI_BUF_SUB_IDX_NON;
+
+	if (csi_buf->sub_idx == CSI_BUF_STS_IDLE) {
+		sub_idx = CSI_BUF_SUB_IDX_FULL_BW;
+		csi_buf->sub_idx = 0xF;
+	}
+
+	return sub_idx;
+}
+
+/**
+ * __query_avl_buf_idx_160
+ * 	Get available 160MHz csi buffer
+ * input :
+ * @csi_buf: (struct hal_csi_buf *) 160MHz hal csi buffer for checking avaible
+ * return
+ * @sub_idx: (u8) csi buffer sub index
+ **/
+u8 __query_avl_buf_idx_160(struct hal_csi_buf *csi_buf)
+{
+	u8 sub_idx = CSI_BUF_SUB_IDX_NON;
+
+	if (csi_buf->sub_idx == CSI_BUF_STS_IDLE) {
+		sub_idx = CSI_BUF_SUB_IDX_FULL_BW;
+		csi_buf->sub_idx = 0xF;
+	}
+
+	return sub_idx;
+}
+
+/**
+ * _query_csi_buf_su
+ * 	Get available sub 40MHz csi buffer
+ * input :
+ * @hal_info: (struct hal_info_t *)
+ * @size:enum hal_csi_buf_size 20/40/80 MHz
+ * return
+ * @sub_id: (u8) available csi buffer sub index
+ * @csi_buf: (struct hal_csi_buf *) original hal csi buffer
+ **/
+struct hal_csi_buf *_query_csi_buf_su(
+	struct hal_info_t *hal_info,
+	enum hal_csi_buf_size size,
+	u8 *sub_id)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = (struct hal_csi_obj *)hal_com->csi_obj;
+	struct hal_csi_buf *csi_buf = NULL;
+	u8 i = 0;
+	u8 e_idx_su = csi_obj->max_csi_buf_nr_su;
+
+	*sub_id = CSI_BUF_SUB_IDX_NON;
+
+	switch (size) {
+	case HAL_CSI_BUF_SIZE_20:
+		for (i = 0; i < e_idx_su; i++) {
+			*sub_id = __query_avl_buf_idx_20(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	case HAL_CSI_BUF_SIZE_40:
+		for (i = 0; i < e_idx_su; i++) {
+			*sub_id = __query_avl_buf_idx_40(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	case HAL_CSI_BUF_SIZE_80:
+		for (i = 0; i < e_idx_su; i++) {
+			*sub_id = __query_avl_buf_idx_80(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	case HAL_CSI_BUF_SIZE_160:
+		for (i = 0; i < e_idx_su; i++) {
+			*sub_id = __query_avl_buf_idx_160(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	default:
+	break;
+	}
+	if (*sub_id == CSI_BUF_SUB_IDX_NON) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "warring :_query_csi_buf_su fail !!!\n");
+	}
+	return csi_buf;
+}
+
+/**
+ * _query_csi_buf_mu
+ * 	Get available sub 40MHz csi buffer
+ * input :
+ * @hal_info: (struct hal_info_t *)
+ * @size:enum hal_csi_buf_size 20/40/80 MHz
+ * return
+ * @sub_id: (u8) available csi buffer sub index
+ * @csi_buf: (struct hal_csi_buf *) original hal csi buffer
+ **/
+struct hal_csi_buf *_query_csi_buf_mu(
+	struct hal_info_t *hal_info,
+	enum hal_csi_buf_size size,
+	u8 *sub_id)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = (struct hal_csi_obj *)hal_com->csi_obj;
+	struct hal_csi_buf *csi_buf = NULL;
+	u8 i = 0;
+	u8 s_idx_mu = csi_obj->max_csi_buf_nr_su;
+	u8 e_idx_mu = csi_obj->max_csi_buf_nr;
+
+	*sub_id = CSI_BUF_SUB_IDX_NON;
+
+	switch (size) {
+	case HAL_CSI_BUF_SIZE_20:
+		for (i = s_idx_mu; i < e_idx_mu; i++) {
+			*sub_id = __query_avl_buf_idx_20(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	case HAL_CSI_BUF_SIZE_40:
+		for (i = s_idx_mu; i < e_idx_mu; i++) {
+			*sub_id = __query_avl_buf_idx_40(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	case HAL_CSI_BUF_SIZE_80:
+		for (i = s_idx_mu; i < e_idx_mu; i++) {
+			*sub_id = __query_avl_buf_idx_80(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	case HAL_CSI_BUF_SIZE_160:
+		for (i = s_idx_mu; i < e_idx_mu; i++) {
+			*sub_id = __query_avl_buf_idx_160(&csi_obj->csi_buf[i]);
+			if(*sub_id != CSI_BUF_SUB_IDX_NON) {
+				csi_buf = &csi_obj->csi_buf[i];
+				break;
+			}
+		}
+	break;
+	default:
+	break;
+	}
+
+	if (*sub_id == CSI_BUF_SUB_IDX_NON) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "warring :_query_csi_buf_mu fail !!!\n");
+	}
+	return csi_buf;
+}
+
+/**
+ * _bw2csi
+ * 	phl channel width to hal csi buffer size
+ * input :
+ * @bw: (struct hal_info_t *)
+ * return
+ * @ret: enum hal_csi_buf_size
+ **/
+enum hal_csi_buf_size
+_bw2csi(enum channel_width bw)
+{
+	enum hal_csi_buf_size ret = HAL_CSI_BUF_SIZE_NONE;
+	switch (bw){
+	case CHANNEL_WIDTH_20:
+		ret = HAL_CSI_BUF_SIZE_20;
+	break;
+	case CHANNEL_WIDTH_40:
+		ret = HAL_CSI_BUF_SIZE_40;
+	break;
+	case CHANNEL_WIDTH_80:
+		ret = HAL_CSI_BUF_SIZE_80;
+	break;
+	case CHANNEL_WIDTH_160:
+		ret = HAL_CSI_BUF_SIZE_160;
+	break;
+	default:
+	break;
+	}
+	return ret;
+}
+
+enum rtw_hal_status _hal_csi_init_buf(struct hal_info_t *hal_info, u8 num)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = hal_com->csi_obj;
+	struct hal_csi_buf *csi_buf = NULL;
+	u8 idx = 0;
+
+	do {
+		csi_obj->csi_buf = _os_mem_alloc(drv_priv,
+					sizeof(*csi_buf) * num);
+		_os_mem_set(drv_priv, csi_obj->csi_buf, 0,
+					sizeof(*csi_buf) * num);
+
+		if (NULL == csi_obj->csi_buf) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		csi_buf = csi_obj->csi_buf;
+		for (idx = 0; idx < num ; idx++) {
+			csi_buf[idx].idx = idx;
+			csi_buf[idx].sub_idx = 0;
+		}
+	} while (0);
+
+	return status;
+}
+
+
+/**
+ * hal_csi_init
+ * 	hal csi module initialize
+ * input :
+ * @hal_info: struct hal_info_t
+ * @su_buf_nr: Number of HW SU CSI Buffer for HAL
+ * @mu_buf_nr: Number of HW MU CSI Buffer for HAL
+ **/
+enum rtw_hal_status
+hal_csi_init(struct hal_info_t *hal_info, u8 su_buf_nr, u8 mu_buf_nr)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = NULL;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	u8 idx = 0;
+	FUNCIN();
+
+	do {
+		csi_obj = _os_mem_alloc(drv_priv, sizeof(*csi_obj));
+
+		if (csi_obj == NULL) {
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		hal_com->csi_obj = csi_obj;
+		_os_spinlock_init(drv_priv, &csi_obj->csi_lock);
+
+		if ((su_buf_nr + mu_buf_nr) == 0) {
+			PHL_ERR("hal_csi_init fail : csi buffer number = 0\n");
+			status = RTW_HAL_STATUS_RESOURCE;
+			break;
+		}
+		_hal_csi_init_buf(hal_info, mu_buf_nr + su_buf_nr);
+
+		csi_obj->max_csi_buf_nr_mu = mu_buf_nr;
+		csi_obj->max_csi_buf_nr_su = su_buf_nr;
+		csi_obj->max_csi_buf_nr = su_buf_nr + mu_buf_nr;
+		for (idx = su_buf_nr; idx < csi_obj->max_csi_buf_nr; idx++){
+			csi_obj->csi_buf[idx].type = HAL_CSI_BUF_TYPE_MU;
+		}
+
+	} while (0);
+
+	if (RTW_HAL_STATUS_SUCCESS != status) {
+		hal_csi_deinit(hal_info);
+	}
+	FUNCOUT();
+	return status;
+}
+/**
+ * hal_csi_deinit
+ * 	hal csi module deinitialize
+ * input :
+ * @hal_info: struct hal_info_t
+ **/
+void hal_csi_deinit(struct hal_info_t *hal_info)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = (struct hal_csi_obj *)hal_com->csi_obj;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+
+	if (csi_obj != NULL) {
+		if (csi_obj->csi_buf != NULL) {
+			_os_mem_free(drv_priv, csi_obj->csi_buf,
+			             sizeof(struct hal_csi_buf) * csi_obj->max_csi_buf_nr);
+			csi_obj->csi_buf = NULL;
+		}
+
+		_os_spinlock_free(drv_priv, &csi_obj->csi_lock);
+		/* bf obj need free as last */
+		_os_mem_free(drv_priv, csi_obj, sizeof(struct hal_csi_obj));
+		hal_com->csi_obj = NULL;
+	}
+}
+
+/**
+ * hal_csi_query_idle_csi_buf
+ * 	Get available CSI buffer
+ * input :
+ * @hal_info: struct hal_info_t
+ * @mu: Request CSI Type : 1 = MU / 0 = SU
+ * @bw: Request CSI BW : enum channel_width (20/40/80MHz)
+ * @buf: (struct hal_csi_buf *)input/return : memory for CSI buffer index+subindex
+ * 	 (the call shall provied 1byte memory to save csi index)
+ **/
+enum rtw_hal_status hal_csi_query_idle_csi_buf(
+	struct hal_info_t *hal_info,
+	u8 mu,
+	enum channel_width bw,
+	void *buf)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = (struct hal_csi_obj *)hal_com->csi_obj;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct hal_csi_buf *csi_buf = (struct hal_csi_buf *)buf;
+	struct hal_csi_buf *tmp_csi_buf = NULL;
+	u8 sub_idx = CSI_BUF_SUB_IDX_NON;
+	enum hal_csi_buf_size size = HAL_CSI_BUF_SIZE_NONE;
+	enum hal_csi_buf_type type = HAL_CSI_BUF_TYPE_SU;
+
+	FUNCIN();
+	if (csi_buf == NULL) {
+		return status;
+	}
+
+	size = _bw2csi(bw);
+	type = mu ? HAL_CSI_BUF_TYPE_MU : HAL_CSI_BUF_TYPE_SU;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "size = 0x%x , type = 0x%x \n",
+		  size, type);
+
+	switch (type) {
+	case HAL_CSI_BUF_TYPE_SU:
+	{
+		_os_spinlock(drv_priv, &csi_obj->csi_lock, _ps, NULL);
+		tmp_csi_buf = _query_csi_buf_su(hal_info, size, &sub_idx);
+		_os_spinunlock(drv_priv, &csi_obj->csi_lock, _ps, NULL);
+	}
+	break;
+	case HAL_CSI_BUF_TYPE_MU:
+	{
+		_os_spinlock(drv_priv, &csi_obj->csi_lock, _ps, NULL);
+		tmp_csi_buf = _query_csi_buf_mu(hal_info, size, &sub_idx);
+		_os_spinunlock(drv_priv, &csi_obj->csi_lock, _ps, NULL);
+	}
+	break;
+	default:
+	break;
+	}
+	if (tmp_csi_buf == NULL || sub_idx == CSI_BUF_SUB_IDX_NON) {
+		status = RTW_HAL_STATUS_RESOURCE;
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "hal_csi_query_idle_csi_buf Fail\n");
+	} else {
+		csi_buf->idx = tmp_csi_buf->idx;
+		csi_buf->sub_idx = sub_idx;
+		csi_buf->type = type;
+		if (bw == CHANNEL_WIDTH_160)
+			csi_buf->b160mhz = 1;
+		else
+			csi_buf->b160mhz = 0;
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	FUNCOUT();
+	return status;
+}
+
+/**
+ * hal_csi_release_csi_buf
+ * 	Release the CSI buffer
+ * input :
+ * @hal_info: struct hal_info_t
+ * @buf: (struct hal_csi_buf *)input/return : CSI buffer index to release
+ **/
+enum rtw_hal_status hal_csi_release_csi_buf(
+	struct hal_info_t *hal_info,
+	void *buf)
+{
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = (struct hal_csi_obj *)hal_com->csi_obj;
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_csi_buf *tmp_csi_buf = NULL;
+	struct hal_csi_buf *csi_buf = (struct hal_csi_buf *)buf;
+
+	if (csi_buf == NULL) {
+		return status;
+	}
+
+	_os_spinlock(drv_priv, &csi_obj->csi_lock, _ps, NULL);
+	if (csi_buf->idx < csi_obj->max_csi_buf_nr) {
+		tmp_csi_buf = &csi_obj->csi_buf[csi_buf->idx];
+
+
+		switch (csi_buf->sub_idx) {
+		case CSI_BUF_SUB_IDX_FULL_BW:
+			tmp_csi_buf->sub_idx = 0;
+		break;
+		case CSI_BUF_SUB_IDX_40_U:
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 3);
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 2);
+		break;
+		case CSI_BUF_SUB_IDX_40_L:
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 1);
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 0);
+		break;
+		case CSI_BUF_SUB_IDX_20_UU:
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 3);
+		break;
+		case CSI_BUF_SUB_IDX_20_UL:
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 2);
+		break;
+		case CSI_BUF_SUB_IDX_20_LU:
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 1);
+		break;
+		case CSI_BUF_SUB_IDX_20_LL:
+			CLEAR_CSI_STS_BIT(tmp_csi_buf, 0);
+		break;
+		default:
+		break;
+		}
+		csi_buf->idx = 0;
+		csi_buf->sub_idx = CSI_BUF_SUB_IDX_NON;
+		status = RTW_HAL_STATUS_SUCCESS;
+	}
+	_os_spinunlock(drv_priv, &csi_obj->csi_lock, _ps, NULL);
+	return status;
+}
+
+/**
+ * hal_is_csi_buf_valid : check the csi buffer index is valid or not.
+ * @hal_info: struct hal_info_t
+ * @buf: (struct hal_csi_buf *)input/return : CSI buffer index to release
+ **/
+u8 hal_is_csi_buf_valid(struct hal_info_t *hal_info, void *buf)
+{
+	struct hal_csi_buf *csi_buf = (struct hal_csi_buf *)buf;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_csi_obj *csi_obj = (struct hal_csi_obj *)hal_com->csi_obj;
+
+	u8 ret = _SUCCESS;
+	do {
+		if(csi_buf->idx >= csi_obj->max_csi_buf_nr){
+			ret = _FAIL;
+			break;
+		}
+		if(csi_buf->sub_idx == CSI_BUF_SUB_IDX_NON) {
+			ret = _FAIL;
+			break;
+		}
+	} while(0);
+
+	return ret;
+}
+
+/**
+ * rtw_hal_get_csi_buf_bw : Get the csi buf bandwidth;
+ * @buf: (struct hal_csi_buf *)
+ * return
+ * @ret: enum channel_width
+ **/
+enum channel_width
+rtw_hal_get_csi_buf_bw(void *buf)
+{
+	struct hal_csi_buf *csi_buf = (struct hal_csi_buf *)buf;
+	enum channel_width bw = CHANNEL_WIDTH_20;
+	switch (csi_buf->sub_idx) {
+	case CSI_BUF_SUB_IDX_FULL_BW:
+		if (csi_buf->b160mhz)
+			bw = CHANNEL_WIDTH_160;
+		else
+			bw = CHANNEL_WIDTH_80;
+	break;
+	case CSI_BUF_SUB_IDX_40_U:
+	case CSI_BUF_SUB_IDX_40_L:
+		bw = CHANNEL_WIDTH_40;
+	break;
+	case CSI_BUF_SUB_IDX_20_UU:
+	case CSI_BUF_SUB_IDX_20_UL:
+	case CSI_BUF_SUB_IDX_20_LU:
+	case CSI_BUF_SUB_IDX_20_LL:
+		bw = CHANNEL_WIDTH_20;
+	break;
+	case CSI_BUF_SUB_IDX_NON:
+	default:
+		bw = CHANNEL_WIDTH_MAX;
+		break;
+	}
+
+	return bw;
+}
+
+
+/**
+ * rtw_hal_get_csi_buf_type : check the csi buffer index is valid or not.
+ * @buf: (struct hal_csi_buf *)
+ * return:
+ * @ret: true = TYPE MU ; false = TYPE SU
+ **/
+bool
+rtw_hal_get_csi_buf_type(void *buf)
+{
+	struct hal_csi_buf *csi_buf = (struct hal_csi_buf *)buf;
+	bool ret = false;
+
+	if (csi_buf != NULL) {
+		ret = (csi_buf->type == HAL_CSI_BUF_TYPE_MU) ? true : false;
+	}
+
+	return ret;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_csi_buffer.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_csi_buffer.h
new file mode 100644
index 000000000000..4e550bb2099a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_csi_buffer.h
@@ -0,0 +1,107 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef __HAL_CSI_BUFFER_H__
+#define __HAL_CSI_BUFFER_H__
+
+#define CSI_BUF_IDX_HW_MSK 0x7FF
+
+
+#define CSI_BUF_SUB_IDX_FULL_BW 0b0
+#define CSI_BUF_SUB_IDX_40_U 0b100100
+#define CSI_BUF_SUB_IDX_40_L 0b101000
+#define CSI_BUF_SUB_IDX_20_UU 0b000100
+#define CSI_BUF_SUB_IDX_20_UL 0b001000
+#define CSI_BUF_SUB_IDX_20_LU 0b001100
+#define CSI_BUF_SUB_IDX_20_LL 0b010000
+/* following is used in csi buffer size is 160Mhz */
+#define CSI_BUF_SUB_IDX_160M_20_LL 0b010000
+#define CSI_BUF_SUB_IDX_160M_20_LL 0b010000
+#define CSI_BUF_SUB_IDX_160M_20_LL 0b010000
+#define CSI_BUF_SUB_IDX_160M_20_LL 0b010000
+#define CSI_BUF_SUB_IDX_160M_40_U 0b101100
+#define CSI_BUF_SUB_IDX_160M_40_L 0b110000
+#define CSI_BUF_SUB_IDX_160M_80_U 0b110100
+#define CSI_BUF_SUB_IDX_160M_80_L 0b111000
+
+#define CSI_BUF_SUB_IDX_NON 0b111111
+
+#define CSI_BUF_STS_BUSY 1
+#define CSI_BUF_STS_IDLE 0
+
+#define IS_SUB20_BUSY(_csi, _idx) \
+	(((_csi->sub_idx&BIT(_idx)) == BIT(_idx)) ? CSI_BUF_STS_BUSY : CSI_BUF_STS_IDLE)
+
+#define SET_SUB20_BUSY(_csi, _idx) _csi->sub_idx |= BIT(_idx)
+
+#define IS_40L_BUSY(_csi) \
+	(((_csi->sub_idx & (BIT(0)|BIT(1)))==(BIT(0)|BIT(1))) ? CSI_BUF_STS_BUSY : CSI_BUF_STS_IDLE)
+#define IS_40U_BUSY(_csi) \
+	(((_csi->sub_idx & (BIT(2)|BIT(3)))==(BIT(2)|BIT(3))) ? CSI_BUF_STS_BUSY : CSI_BUF_STS_IDLE)
+
+#define SET_40L_BUSY(_csi) _csi->sub_idx |= (BIT(0)|BIT(1))
+#define SET_40U_BUSY(_csi) _csi->sub_idx |= (BIT(2)|BIT(3))
+
+#define CLEAR_CSI_STS_BIT(_csi, _x) _csi->sub_idx &= ~(1 << _x)
+
+enum hal_csi_buf_type {
+	HAL_CSI_BUF_TYPE_SU = 0,
+	HAL_CSI_BUF_TYPE_MU = 1
+};
+
+enum hal_csi_buf_size {
+	HAL_CSI_BUF_SIZE_NONE = 0,
+	HAL_CSI_BUF_SIZE_20,
+	HAL_CSI_BUF_SIZE_40,
+	HAL_CSI_BUF_SIZE_80,
+	HAL_CSI_BUF_SIZE_160
+};
+
+struct hal_csi_buf {
+	u16 sub_idx:6;
+	u16 idx:5;
+	u16 type:1;
+	u16 b160mhz:1;
+	u16 rsvd:3;
+};
+
+struct hal_csi_obj {
+	u8 max_csi_buf_nr;
+	u8 max_csi_buf_nr_su;
+	u8 max_csi_buf_nr_mu;
+	struct hal_csi_buf *csi_buf;
+	_os_lock csi_lock;
+};
+
+enum rtw_hal_status 
+hal_csi_init(struct hal_info_t *hal_info, u8 su_buf_nr, u8 mu_buf_nr);
+
+void hal_csi_deinit(struct hal_info_t *hal_info);
+
+enum rtw_hal_status hal_csi_query_idle_csi_buf(
+	struct hal_info_t *hal_info, u8 mu, enum channel_width bw, void *buf);
+
+enum rtw_hal_status hal_csi_release_csi_buf(
+	struct hal_info_t *hal_info,
+	void *buf);
+
+u8 hal_is_csi_buf_valid(struct hal_info_t *hal_info, void *buf);
+
+enum channel_width
+rtw_hal_get_csi_buf_bw(void *buf);
+
+bool
+rtw_hal_get_csi_buf_type(void *buf);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_custom.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_custom.c
new file mode 100644
index 000000000000..eb6d9fe13d15
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_custom.c
@@ -0,0 +1,266 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_CUSTOM_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+enum rtw_hal_status
+rtw_hal_custom_cfg_tx_ampdu(void *hal,
+                            struct rtw_wifi_role_link_t *rlink,
+                            struct rtw_phl_custom_ampdu_cfg *ampdu)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	if (rlink == NULL || ampdu == NULL)
+		return hsts;
+
+	hsts = rtw_hal_mac_set_hw_ampdu_cfg(hal_info,
+	                                    rlink->hw_band,
+	                                    (u16)ampdu->max_agg_num,
+	                                    (u8)ampdu->max_agg_time_32us);
+
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_get_ampdu_cfg(void *hal,
+                      struct rtw_wifi_role_link_t *rlink,
+                      struct rtw_phl_custom_ampdu_cfg *cfg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_ampdu_cfg ampdu_cfg = {0};
+
+	if (rlink == NULL || cfg == NULL)
+		return hsts;
+
+	hsts = rtw_hal_mac_get_ampdu_cfg(hal_info->hal_com,
+	                                 rlink->hw_band,
+	                                 &ampdu_cfg);
+	if (hsts == RTW_HAL_STATUS_SUCCESS) {
+		cfg->max_agg_num = ampdu_cfg.max_agg_num;
+		cfg->max_agg_time_32us = ampdu_cfg.max_agg_time_32us;
+	}
+
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_get_edca(void *hal,
+                 struct rtw_wifi_role_link_t *rlink,
+                 struct rtw_edca_param *edca_param)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status;
+
+	hal_status = rtw_hal_mac_get_edca(hal_info->hal_com,
+	                                  rlink->hw_band,
+	                                  rlink->hw_wmm,
+	                                  edca_param);
+
+	PHL_INFO("%s\n, ac(%d), param(0x%x)\n, ret(%d)", __FUNCTION__, edca_param->ac,
+	         edca_param->param, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_set_usr_edca(void *hal,
+                     u32 idx,
+                     u32 enable,
+                     u32 band,
+                     u32 wmm,
+                     struct rtw_edca_param *aggr,
+                     struct rtw_edca_param *mod)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_set_usr_edca(hal_info, idx, enable, band, wmm, aggr, mod);
+}
+
+enum rtw_hal_status
+rtw_hal_set_pop_en(void *hal, bool en, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	return rtw_hal_bb_set_pop_en(hal_info, en, phy_idx);
+}
+
+bool
+rtw_hal_query_pop_en(void *hal, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	return rtw_hal_bb_query_pop_en(hal_info, phy_idx);
+}
+
+enum rtw_hal_status
+rtw_hal_set_pkt_detect_thold(void *hal, enum phl_band_idx band_idx, u32 bound)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	return rtw_hal_bb_set_pkt_detect_thold(hal_info, band_idx, bound);
+}
+
+u8
+rtw_hal_query_pkt_detect_thold(void *hal,
+                               bool get_en_info,
+                               enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	return rtw_hal_bb_query_pkt_detect_thold(hal_info, get_en_info, phy_idx);
+}
+
+void
+rtw_hal_set_tx_rate_rty_tbl(void *hal, bool en, u8 *rty_rate_tbl)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_bb_set_tx_rate_rty_tbl(hal_info, en, rty_rate_tbl);
+}
+
+enum rtw_hal_status
+rtw_hal_set_spatial_reuse_en(void *hal, bool en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	return rtw_hal_bb_set_spatial_reuse_en(hal_info, en);
+}
+
+bool
+rtw_hal_is_spatial_reuse_en(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	return rtw_hal_bb_is_spatial_reuse_en(hal_info);
+}
+
+enum rtw_hal_status rtw_hal_set_usr_frame_to_act(
+    void *hal, enum rtw_mac_usr_frame_to_act_mode mode, u32 to_thr,
+    u8 trigger_cnt, u16 sw_def_bmp)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_usr_frame_to_act_cfg act_cfg = {0};
+
+	act_cfg.mode = mode;
+	act_cfg.to_thr = to_thr;
+	act_cfg.trigger_cnt = trigger_cnt;
+	act_cfg.sw_def_bmp = sw_def_bmp;
+	PHL_INFO(
+	    "%s\n, mode(%d), to_thr(%d)\n, trigger_cnt(%d) sw_def_bmp(%d)\n",
+	    __func__, mode, to_thr, trigger_cnt, sw_def_bmp);
+
+	return rtw_hal_mac_usr_frame_to_act(hal_info, &act_cfg);
+}
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+enum rtw_hal_status
+rtw_hal_set_usr_tx_rpt_cfg(void *hal, struct rtw_phl_usr_tx_rpt_cfg *param)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_set_usr_tx_rpt_cfg(hal_info, param);
+}
+
+bool
+_set_ch_busy_rx_stat_cfg(struct hal_info_t *hal_info, u8 band)
+{
+	struct mac_ax_ch_busy_cnt_cfg mac_param = {0};
+	bool ret = true;
+
+	mac_param.band = band;
+	mac_param.ref.basic_nav = 1;
+	mac_param.ref.intra_nav = 1;
+	mac_param.ref.data_on = 1;
+	mac_param.ref.edcca_p20 = 1;
+	mac_param.ref.cca_p20 = 1;
+	mac_param.ref.cca_s20 = 1;
+	mac_param.ref.cca_s40 = 1;
+	mac_param.ref.cca_s80 = 1;
+	mac_param.ref.phy_txon = 1;
+
+	/* setup CNT_REF */
+	mac_param.cnt_ctrl = MAC_AX_CH_BUSY_CNT_CTRL_CNT_REF;
+	if (rtw_hal_mac_set_ch_busy_stat_cfg(hal_info, &mac_param)) {
+		ret = false;
+		PHL_INFO(" %s, fail band(%d) with CNT_REF\n",
+				__FUNCTION__, band);
+		goto exit;
+	}
+
+	mac_param.cnt_ctrl = MAC_AX_CH_BUSY_CNT_CTRL_CNT_EN;
+	/* setup CNT_EN */
+	if (rtw_hal_mac_set_ch_busy_stat_cfg(hal_info, &mac_param)) {
+		ret = false;
+		PHL_INFO(" %s, fail band(%d) with CNT_EN\n",
+				__FUNCTION__, band);
+		goto exit;
+	}
+exit:
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_set_ch_busy_stat_cfg(void *hal, u8 band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct mac_ax_ch_busy_cnt_cfg mac_param = {0};
+
+	PHL_INFO(" %s, band(%d)\n", __FUNCTION__, band);
+	mac_param.band = band;
+	mac_param.ref.basic_nav = 1;
+	mac_param.ref.intra_nav = 1;
+	mac_param.ref.data_on = 1;
+	mac_param.ref.edcca_p20 = 1;
+	mac_param.ref.cca_p20 = 1;
+	mac_param.ref.cca_s20 = 1;
+	mac_param.ref.cca_s40 = 1;
+	mac_param.ref.cca_s80 = 1;
+	mac_param.ref.phy_txon = 1;
+
+	/* setup CNT_REF */
+	mac_param.cnt_ctrl = MAC_AX_CH_BUSY_CNT_CTRL_CNT_REF;
+	if (rtw_hal_mac_set_ch_busy_stat_cfg(hal_info, &mac_param)) {
+		PHL_INFO(" %s, fail band(%d) with CNT_REF\n", __FUNCTION__,
+			 band);
+		hal_status = RTW_HAL_STATUS_FAILURE;
+		goto exit;
+	}
+
+	mac_param.cnt_ctrl = MAC_AX_CH_BUSY_CNT_CTRL_CNT_EN;
+	/* setup CNT_EN */
+	if (rtw_hal_mac_set_ch_busy_stat_cfg(hal_info, &mac_param)) {
+		PHL_INFO(" %s, fail band(%d) with CNT_EN\n", __FUNCTION__,
+			 band);
+		hal_status = RTW_HAL_STATUS_FAILURE;
+		goto exit;
+	}
+
+exit:
+	return hal_status;
+}
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+#endif
+
+void rtw_hal_auto_debug_en_phy_util(struct rtw_hal_com_t *hal_com, bool en)
+{
+	rtw_hal_bb_auto_debug_en_phy_util(hal_com, en);
+}
+
+void rtw_hal_query_snr_avg(struct rtw_hal_com_t *hal_com,
+                           u8 *info,
+                           u8 hw_band)
+{
+	enum phl_phy_idx p_idx = HW_PHY_0;
+
+	p_idx = rtw_hal_hw_band_to_phy_idx(hw_band);
+	rtw_hal_bb_query_snr_avg(hal_com, info, p_idx);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_custom.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_custom.h
new file mode 100644
index 000000000000..aea70ee338ad
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_custom.h
@@ -0,0 +1,79 @@
+
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_CUSTOM_H_
+#define _HAL_CUSTOM_H_
+
+#ifdef CONFIG_PHL_CUSTOM_FEATURE
+enum rtw_hal_status
+rtw_hal_custom_cfg_tx_ampdu(void *hal,
+                            struct rtw_wifi_role_link_t *rlink,
+                            struct rtw_phl_custom_ampdu_cfg *ampdu);
+
+enum rtw_hal_status
+rtw_hal_get_ampdu_cfg(void *hal,
+                      struct rtw_wifi_role_link_t *rlink,
+                      struct rtw_phl_custom_ampdu_cfg *cfg);
+
+enum rtw_hal_status
+rtw_hal_get_edca(void *hal,
+                 struct rtw_wifi_role_link_t *rlink,
+                 struct rtw_edca_param *edca_param);
+
+enum rtw_hal_status
+rtw_hal_set_usr_edca(void *hal,
+                     u32 idx,
+                     u32 enable,
+                     u32 band,
+                     u32 wmm,
+                     struct rtw_edca_param *aggr,
+                     struct rtw_edca_param *mod);
+
+enum rtw_hal_status
+rtw_hal_set_pop_en(void *hal, bool en, enum phl_phy_idx phy_idx);
+
+bool
+rtw_hal_query_pop_en(void *hal, enum phl_phy_idx phy_idx);
+
+enum rtw_hal_status
+rtw_hal_set_pkt_detect_thold(void *hal, enum phl_band_idx band_idx, u32 bound);
+
+u8
+rtw_hal_query_pkt_detect_thold(void *hal,
+                               bool get_en_info,
+                               enum phl_phy_idx phy_idx);
+
+void
+rtw_hal_set_tx_rate_rty_tbl(void *hal, bool en, u8 *rty_rate_tbl);
+
+enum rtw_hal_status
+rtw_hal_set_spatial_reuse_en(void *hal, bool en);
+
+bool
+rtw_hal_is_spatial_reuse_en(void *hal);
+
+enum rtw_hal_status rtw_hal_set_usr_frame_to_act(
+    void *hal, enum rtw_mac_usr_frame_to_act_mode mode, u32 to_thr,
+    u8 trigger_cnt, u16 sw_def_bmp);
+
+#ifdef CONFIG_PHL_CUSTOM_FRAME_STAT
+enum rtw_hal_status
+rtw_hal_set_usr_tx_rpt_cfg(void *hal, struct rtw_phl_usr_tx_rpt_cfg *param);
+
+enum rtw_hal_status rtw_hal_set_ch_busy_stat_cfg(void *hal, u8 band);
+#endif /*CONFIG_PHL_CUSTOM_FRAME_STAT*/
+#endif
+
+#endif /*_HAL_CUSTOM_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dbcc.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dbcc.c
new file mode 100644
index 000000000000..73a6e29d0ff9
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dbcc.c
@@ -0,0 +1,348 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_DBCC_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_DBCC_SUPPORT
+enum phl_phy_idx
+hal_dbcc_cck_phyidx_decision(struct hal_info_t *hal_info,
+					     struct rtw_phl_com_t *phl_com,
+					     bool dbcc_en)
+{
+	enum phl_phy_idx phy_idx = HW_PHY_0;
+	struct mr_ctl_t *mr_ctl = &phl_com->mr_ctrl;
+	struct hw_band_ctl_t *band0_ctrl = &(mr_ctl->band_ctrl[HW_BAND_0]);
+	/*struct hw_band_ctl_t *band1_ctrl = &(mr_ctl->band_ctrl[HW_BAND_1]);*/
+
+	if (dbcc_en == false)
+		goto _exit;
+
+	#ifdef CONFIG_DBCC_FORCE
+	if (is_frc_dbcc_mode(phl_com))
+		phy_idx = phl_com->dev_cap.dbcc_force_cck_phyidx;
+	else
+	#endif
+	if (phl_is_mp_mode(phl_com)) {
+        	phy_idx = (hal_info->hal_com->band[HW_BAND_0].cur_chandef.band ==
+		           BAND_ON_24G)
+		          ? HW_PHY_0 : HW_PHY_1;
+	} else {
+		phy_idx = (band0_ctrl->chctx_band_map & BIT(CC_BAND_24G)) ? HW_PHY_0 : HW_PHY_1;
+	}
+
+_exit:
+	mr_ctl->cck_phyidx = phy_idx;
+	PHL_INFO("%s => mr_ctl->cck_phyidx:%d\n", __func__, mr_ctl->cck_phyidx);
+	return phy_idx;
+}
+
+enum rtw_hal_status
+rtw_hal_dbcc_trx_ctrl(void *hal, struct rtw_phl_com_t *phl_com,
+				enum phl_band_idx band_idx, bool pause)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	hsts = rtw_hal_mac_dbcc_trx_ctrl(hal_info, band_idx, pause);
+	if (hsts != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("%s band_%d , pause:%d failed\n", __func__, band_idx, pause);
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_dbcc_pre_cfg(void *hal, struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct hal_ppdu_sts_cfg psts_cfg = {0};
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	if (!phl_com->dev_cap.dbcc_sup)
+		goto exit_func;
+
+	hal_com->dbcc_en = (dbcc_en) ? true : false;
+	if (dbcc_en) { /*dbcc enable*/
+		rtw_hal_init_hw_band_info(hal, HW_BAND_1);
+		hsts = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, true);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - EN hal reset - HW_PHY_0, HW_BAND_0 failed\n", __func__);
+			goto exit_func;
+		}
+
+		hsts = rtw_hal_mac_dbcc_pre_cfg(phl_com, hal_info, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - EN mac_dbcc_pre_cfg failed\n", __func__);
+			hal_com->dbcc_en = false;
+			goto exit_func;
+		}
+
+		psts_cfg.band_idx = HW_BAND_1;
+		psts_cfg.ppdu_stat_en = true;
+		psts_cfg.filter = HAL_PPDU_HAS_CRC_OK | HAL_PPDU_HAS_DMA_OK;
+		psts_cfg.appen_info = HAL_PPDU_MAC_INFO | HAL_PPDU_PLCP | HAL_PPDU_RX_CNT;
+
+		hsts = rtw_hal_ppdu_sts_init(hal, &psts_cfg);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			hal_com->dbcc_en = false;
+			PHL_ERR("%s - EN HW_BAND_1 ppdu_sts_init failed\n", __func__);
+			goto exit_func;
+		}
+		phl_com->ppdu_sts_info.en_ppdu_sts[HW_BAND_1] = true;
+
+		/* Enable DrvInfo for per pkt rssi */
+		if (RTW_DEV_CAP_ENABLE == phl_com->dev_cap.drv_info_sup) {
+			hsts = rtw_hal_mac_drvinfo_cfg(hal, true, HW_BAND_1);
+			if (hsts != RTW_HAL_STATUS_SUCCESS) {
+				hal_com->dbcc_en = false;
+				PHL_ERR("%s - EN HW_BAND_1 drvinfo failed\n", __func__);
+				goto exit_func;
+			}
+		}
+
+		rtw_hal_mac_dbcc_trx_ctrl(hal_info, HW_BAND_1, true);
+
+		hsts = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, true);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - EN hal reset - HW_PHY_1, HW_BAND_1 failed\n", __func__);
+			goto exit_func;
+		}
+
+		/*phy_dbcc_pre_cfg*/
+		hsts = rtw_hal_phy_dbcc_pre_cfg(hal_info, phl_com, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - EN phy_dbcc_pre_cfg failed\n", __func__);
+			hal_com->dbcc_en = false;
+			goto exit_func;
+		}
+	}
+	else { /*dbcc disable*/
+		hsts = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, true);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - DIS hal reset - HW_PHY_0, HW_BAND_0 failed\n", __func__);
+			goto exit_func;
+		}
+
+		hsts = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, true);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - DIS hal reset - HW_PHY_1, HW_BAND_1 failed\n", __func__);
+			goto exit_func;
+		}
+
+		hsts = rtw_hal_mac_dbcc_pre_cfg(phl_com, hal_info, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - DIS mac_dbcc_pre_cfg failed\n", __func__);
+			hal_com->dbcc_en = true;
+			goto exit_func;
+		}
+		/*phy_dbcc_pre_cfg*/
+		hsts = rtw_hal_phy_dbcc_pre_cfg(hal_info, phl_com, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - DIS phy_dbcc_pre_cfg failed\n", __func__);
+			hal_com->dbcc_en = true;
+			goto exit_func;
+		}
+	}
+
+	PHL_INFO("%s-%s done and success\n", __func__, (dbcc_en) ? "EN" : "DIS");
+
+exit_func:
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_dbcc_cfg(void *hal, struct rtw_phl_com_t *phl_com, bool dbcc_en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	struct hal_ppdu_sts_cfg psts_cfg = {0};
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	if (!phl_com->dev_cap.dbcc_sup)
+		goto exit_func;
+
+	if (dbcc_en) { /*dbcc enable*/
+		/*mac_dbcc_cfg*/
+		hsts = rtw_hal_mac_dbcc_cfg(phl_com, hal_info, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - EN mac_dbcc_cfg failed\n", __func__);
+			hal_com->dbcc_en = false;
+			goto exit_func;
+		}
+
+		/*bb,rf_dbcc_cfg*/
+		hsts = rtw_hal_phy_dbcc_cfg(hal_info, phl_com, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - EN phy_dbcc_cfg failed\n", __func__);
+			hal_com->dbcc_en = false;
+			goto exit_func;
+		}
+
+		/*init rx filter opt,type, max MPDU size, RTS threshold*/
+		hsts = rtw_hal_hw_param_init(hal_info, phl_com, HW_BAND_1);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - EN rx param init failed\n", __func__);
+			hal_com->dbcc_en = false;
+			goto exit_func;
+		}
+	}
+	else { /*dbcc disable*/
+		psts_cfg.band_idx = HW_BAND_1;
+		psts_cfg.ppdu_stat_en = false;
+
+		hsts = rtw_hal_ppdu_sts_init(hal, &psts_cfg);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			hal_com->dbcc_en = true;
+			PHL_ERR("%s - DIS HW_BAND_1 ppdu_sts_deinit failed\n", __func__);
+			goto exit_func;
+		}
+		phl_com->ppdu_sts_info.en_ppdu_sts[HW_BAND_1] = false;
+
+		/* Disable DrvInfo for per pkt rssi */
+		if (RTW_DEV_CAP_ENABLE == phl_com->dev_cap.drv_info_sup) {
+			hsts = rtw_hal_mac_drvinfo_cfg(hal, false, HW_BAND_1);
+			if (hsts != RTW_HAL_STATUS_SUCCESS) {
+				PHL_ERR("%s - DIS HW_BAND_1 drvinfo failed\n", __func__);
+				hal_com->dbcc_en = true;
+				goto exit_func;
+			}
+		}
+
+		/*mac_dbcc_cfg*/
+		hsts = rtw_hal_mac_dbcc_cfg(phl_com, hal_info, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - DIS HW_BAND_1 mac_dbcc_cfg failed\n", __func__);
+			hal_com->dbcc_en = true;
+			goto exit_func;
+		}
+
+		/*bb,rf_dbcc_cfg*/
+		hsts = rtw_hal_phy_dbcc_cfg(hal_info, phl_com, dbcc_en);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - DIS HW_BAND_1 phy_dbcc_cfg failed\n", __func__);
+			hal_com->dbcc_en = true;
+			goto exit_func;
+		}
+		hsts = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, false);
+		if (hsts != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s - DIS hal reset - HW_PHY_0, HW_BAND_0 failed\n", __func__);
+			goto exit_func;
+		}
+	}
+
+	PHL_INFO("%s-%s done and success\n", __func__, (dbcc_en) ? "EN" : "DIS");
+
+exit_func:
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_dbcc_reset_hal(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	hsts = rtw_hal_reset(hal_com, HW_PHY_0, HW_BAND_0, false);
+	if (hsts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s - hal reset - HW_PHY_0, HW_BAND_0 failed\n", __func__);
+		_os_warn_on(1);
+		goto exit_func;
+	}
+	hsts = rtw_hal_reset(hal_com, HW_PHY_1, HW_BAND_1, false);
+	if (hsts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s - hal reset - HW_PHY_1, HW_BAND_1 failed\n", __func__);
+		_os_warn_on(1);
+		goto exit_func;
+	}
+
+	hsts = RTW_HAL_STATUS_SUCCESS;
+
+exit_func:
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_dbcc_band_switch_hdl(void *hal, enum phl_band_idx band_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	PHL_INFO("[assoc_sta_cnt] all:%d, B0:%d, B1:%d\n",
+			hal_com->assoc_sta_cnt,
+			hal_com->band[HW_BAND_0].assoc_sta_cnt,
+			hal_com->band[HW_BAND_1].assoc_sta_cnt);
+
+	hal_com->band[HW_BAND_0].assoc_sta_cnt =
+		hal_com->band[HW_BAND_1].assoc_sta_cnt;
+	hal_com->band[HW_BAND_1].assoc_sta_cnt = 0;
+
+	PHL_INFO("[assoc_sta_cnt] all:%d, B0:%d, B1:%d\n",
+			hal_com->assoc_sta_cnt,
+			hal_com->band[HW_BAND_0].assoc_sta_cnt,
+			hal_com->band[HW_BAND_1].assoc_sta_cnt);
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+void rtw_hal_dbcc_hci_ctrl(void *hal, enum phl_band_idx band_idx, u8 pause)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	/* Stop DMA + Disable all of txch dma (Reset B1 TXBD idx to 0) */
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_dbcc_hci_ctrl(hal_info, band_idx, pause))
+		PHL_ERR("%s failure \n", __func__);
+}
+
+
+void
+hal_dbcc_cfg_phy_map(struct hal_info_t *hal, enum phl_band_idx band_idx)
+{
+	enum phl_phy_idx phy_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+	enum phl_phy_idx cck_phy_idx = HW_PHY_MAX;
+
+	if (phy_idx == HW_PHY_MAX) {
+		PHL_ERR("%s invalid phy idx\n", __func__);
+		return;
+	}
+	rtw_hal_reset(hal->hal_com, phy_idx, band_idx, true);
+
+	cck_phy_idx = hal_dbcc_cck_phyidx_decision(hal, hal->phl_com,
+	                                           hal->hal_com->dbcc_en);
+	PHL_INFO("%s: cck_phy_idx(%d)\n", __func__, cck_phy_idx);
+	rtw_hal_bb_cfg_dbcc_phy_map(hal, cck_phy_idx);
+
+	rtw_hal_reset(hal->hal_com, phy_idx, band_idx, false);
+}
+#endif /*CONFIG_DBCC_SUPPORT*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dbcc.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dbcc.h
new file mode 100644
index 000000000000..7a44fc96683c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dbcc.h
@@ -0,0 +1,27 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_DBCC_H_
+#define _HAL_DBCC_H_
+
+enum phl_phy_idx
+hal_dbcc_cck_phyidx_decision(struct hal_info_t *hal_info,
+					     struct rtw_phl_com_t *phl_com,
+					     bool dbcc_en);
+
+void
+hal_dbcc_cfg_phy_map(struct hal_info_t *hal, enum phl_band_idx band_idx);
+
+#endif /* _HAL_DBCC_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def.h
new file mode 100644
index 000000000000..4e2716d2196d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def.h
@@ -0,0 +1,1310 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_DEF_H_
+#define _HAL_DEF_H_
+
+#define halcom_to_drvpriv(_hcom)	(_hcom->drv_priv)
+#define hal_get_chip_id(_halcom) (_halcom->chip_id)
+
+#define MAX_WD_LEN		(48)
+#define MAX_WD_BODY_LEN (24)
+#define MAX_BAENTRY		16
+
+#define HAL_MAX_PATH HALBB_MAX_PATH
+
+enum hal_path {
+	PATH_NON = 0,
+	PATH_A = 0x00000001,
+	PATH_B = 0x00000002,
+	PATH_C = 0x00000004,
+	PATH_D = 0x00000008,
+
+	PATH_AB = (PATH_A | PATH_B),
+	PATH_AC = (PATH_A | PATH_C),
+	PATH_AD = (PATH_A | PATH_D),
+	PATH_BC = (PATH_B | PATH_C),
+	PATH_BD = (PATH_B | PATH_D),
+	PATH_CD = (PATH_C | PATH_D),
+
+	PATH_ABC = (PATH_A | PATH_B | PATH_C),
+	PATH_ABD = (PATH_A | PATH_B | PATH_D),
+	PATH_ACD = (PATH_A | PATH_C | PATH_D),
+	PATH_BCD = (PATH_B | PATH_C | PATH_D),
+
+	PATH_ABCD = (PATH_A | PATH_B | PATH_C | PATH_D),
+	PATH_AUTO = 0xff /*for auto path selection*/
+};
+
+enum HAL_CMD_ID {
+	HAL_HELP,
+	MAC_DD_DBG,
+	MAC_DUMP_SEC_CAM_TBL
+};
+
+struct hal_cmd_info {
+	char name[16];
+	u8 id;
+};
+
+static const struct hal_cmd_info hal_cmd_i[] = {
+	{"-h", HAL_HELP},
+	{"dd_dbg", MAC_DD_DBG},/*@do not move this element to other position*/
+	{"sec_cam_tbl", MAC_DUMP_SEC_CAM_TBL}
+};
+
+enum rtw_hal_ser_rsn {
+	HAL_SER_RSN_NONE = 0,
+	HAL_SER_RSN_WOW,
+	HAL_SER_RSN_RFK
+};
+
+enum rtw_hal_status {
+	RTW_HAL_STATUS_SUCCESS, /* 0 */
+	RTW_HAL_STATUS_FAILURE, /* 1 */
+	RTW_HAL_STATUS_RESOURCE, /* 2 */
+	RTW_HAL_STATUS_IO_INIT_FAILURE, /* 3 */
+	RTW_HAL_STATUS_MAC_INIT_FAILURE, /* 4 */
+	RTW_HAL_STATUS_BB_INIT_FAILURE, /* 5 */
+	RTW_HAL_STATUS_RF_INIT_FAILURE, /* 6 */
+	RTW_HAL_STATUS_BTC_INIT_FAILURE, /* 7 */
+	RTW_HAL_STATUS_HAL_INIT_FAILURE, /* 8 */
+	RTW_HAL_STATUS_EFUSE_UNINIT, /* 9 */
+	RTW_HAL_STATUS_EFUSE_IVALID_OFFSET, /* 10 */
+	RTW_HAL_STATUS_EFUSE_PG_FAIL, /* 11 */
+	RTW_HAL_STATUS_MAC_API_FAILURE, /* 12 */
+	RTW_HAL_STATUS_BB_CH_INFO_LAST_SEG, /*13*/
+	RTW_HAL_STATUS_UNKNOWN_RFE_TYPE, /* 14 */
+	RTW_HAL_STATUS_TIMEOUT, /* 15 */
+	RTW_HAL_STATUS_NOT_SUPPORT, /* 16 */
+};
+
+#define FW_FILE_NIC_POSTFIX ""
+#define FW_FILE_NIC_CE_POSTFIX "_ce"
+#define FW_FILE_WOWLAN_POSTFIX "_wowlan"
+#define FW_FILE_WOWLAN_CE_POSTFIX "_wowlan_ce"
+#define FW_FILE_SPIC_POSTFIX "_spic"
+#define FW_FILE_AP_POSTFIX "_ap"
+
+enum _rtw_hal_query_info {
+	RTW_HAL_RXDESC_SIZE,
+};
+
+enum rtw_h2c_pkt_type {
+	H2CB_TYPE_CMD = 0,
+	H2CB_TYPE_DATA = 1,
+	H2CB_TYPE_LONG_DATA = 2,
+	H2CB_TYPE_MAX = 0x3
+};
+
+enum tx_pause_rson {
+	PAUSE_RSON_NOR_SCAN, /*normal scan*/
+	PAUSE_RSON_UNSPEC_BY_MACID, /*P2P_SCAN*/
+	PAUSE_RSON_RFK,
+	PAUSE_RSON_PSD,
+	PAUSE_RSON_DFS,
+	PAUSE_RSON_DFS_CSA, /* allow beacon only */
+	PAUSE_RSON_DFS_CSA_MG, /* allow beacon and mgnt frame */
+	PAUSE_RSON_DFS_CAC,
+	PAUSE_RSON_DBCC,
+	PAUSE_RSON_RESET,
+	PAUSE_RSON_MAX
+};
+
+enum rtw_hal_config_int {
+	RTW_HAL_EN_DEFAULT_INT,
+	RTW_HAL_DIS_DEFAULT_INT,
+	RTW_HAL_STOP_RX_INT,
+	RTW_HAL_RESUME_RX_INT,
+	RTW_HAL_SER_HANDSHAKE_MODE,
+	RTW_HAL_EN_HCI_INT,
+	RTW_HAL_DIS_HCI_INT,
+	RTW_HAL_CONFIG_INT_MAX
+};
+
+enum hal_mp_efuse_type {
+	HAL_MP_EFUSE_WIFI = 0,
+	HAL_MP_EFUSE_BT,
+	HAL_MP_EFUSE_NONE,
+};
+
+struct rtw_g6_h2c_hdr {
+	u8 h2c_class; //0x0~0x7: Phydm; 0x8~0xF: RF; 0x10~0x17: BTC
+	u8 h2c_func;
+	u8 seq_valid:1;
+	u8 seq:3;
+	u8 seq_stop:1;
+	enum rtw_h2c_pkt_type type; //0:cmd ; 1:cmd+data ; 2:cmd+long data
+	u16 content_len:12;
+	u16 rec_ack:1; //Ack when receive H2C
+	u16 done_ack:1; //Ack when FW execute H2C cmd done
+	u16 rsvd2:2;
+};
+
+/**
+ * @c2h_cat: target category of this c2h / c2h ack
+ * @c2h_class: target class of this c2h / c2h ack
+ * @c2h_func: target function of this c2h / c2h ack
+ * @type_rec_ack: set 1 as a receive ack to this c2h
+ * @type_done_ack: set 1 as a done ack to this c2h
+ * @h2c_return: status code of done ack responding to h2c
+ *
+ */
+struct rtw_c2h_info {
+	u8 c2h_cat;
+	u8 c2h_class;
+	u8 c2h_func;
+	u8 type_rec_ack:1;
+	u8 type_done_ack:1;
+	u8 rsvd:6;
+	u8 h2c_return;/*H2C return value, 0 = success*/
+	u8 h2c_seq;
+	u16 content_len;
+	u8 *content;
+};
+
+
+#define RTW_BTC_OVERWRITE_BUF_LEN 10
+struct hal_bt_msg {
+	_os_lock lock;
+	u16 len;
+	u16 cnt; /* update cnt */
+	u8 latest[RTW_BTC_OVERWRITE_BUF_LEN];
+	u8 working[RTW_BTC_OVERWRITE_BUF_LEN];
+};
+
+struct btc_fw_msg {
+	_os_lock lock;
+	u32 fev_cnt; /* fw event cnt, need to be protected by lock */
+
+	struct hal_bt_msg btinfo;
+	struct hal_bt_msg scbd;
+
+	/* common fwinfo queue */
+	struct phl_queue idleq;
+	struct phl_queue waitq;
+};
+
+struct btc_ctrl_t {
+	u8 lps;
+	u8 tx_time;
+	u8 tx_retry;
+	u8 disable_rx_stbc;
+};
+
+/*except version*/
+struct ver_ctrl_t {
+	u64 mac_ver;
+	u64 bb_ver;
+	u64 rf_ver;
+	u64 btc_ver;
+	u64 fw_ver;
+};
+
+
+struct hal_mu_score_tbl_ctrl {
+	u32 mu_sc_thr:2;
+	u32 mu_opt:1;
+	u32 rsvd:29;
+};
+
+#define HAL_MAX_MU_SCORE_SIZE 8 /* Unit: Byte */
+struct hal_mu_score_tbl_score {
+	u8 valid;
+	u16 macid;
+	u8 score[HAL_MAX_MU_SCORE_SIZE]; /*by case: [1:0], [3:2], ..... */
+};
+
+#define HAL_MAX_MU_STA_NUM 6
+struct hal_mu_score_tbl {
+	struct hal_mu_score_tbl_ctrl mu_ctrl;
+	_os_lock lock;
+	u8 num_mu_sta; /*By IC, shall alway <= than HAL_MAX_MU_STA_NUM , 0 = tbl invalid */
+	u8 sz_mu_score; /*By IC, shall alway <= than HAL_MAX_MU_SCORE_SIZE , 0 = score invalid */
+	struct hal_mu_score_tbl_score mu_score[HAL_MAX_MU_STA_NUM]; /* mu_score[num_mu_sta] */
+};
+
+
+#define HAL_MAX_VHT_BFRP_NUM 3
+#define HAL_MAX_HE_BFRP_NUM 2
+#define HLA_MAX_BFRP_NUM ((HAL_MAX_VHT_BFRP_NUM > HAL_MAX_HE_BFRP_NUM) ?\
+			  HAL_MAX_VHT_BFRP_NUM : HAL_MAX_HE_BFRP_NUM)
+#define HAL_MAX_VHT_SND_STA_NUM HAL_MAX_VHT_BFRP_NUM + 1
+#define HAL_MAX_HE_BFRP_STA_NUM 4
+#define HAL_MAX_HE_SND_STA_NUM HAL_MAX_HE_BFRP_NUM * HAL_MAX_HE_BFRP_STA_NUM
+
+struct hal_frame_hdr {
+	u16 frame_ctl;
+	u16 duration;
+	u8 addr1[MAC_ALEN];
+	u8 addr2[MAC_ALEN];
+};
+struct hal_npda_dialog_token {
+	u8 rsvd:1;
+	u8 he:1;
+	u8 token:6;
+};
+
+#define HAL_NPDA_AC_SU 0
+#define HAL_NPDA_AC_MU 1
+#define HAL_NDPA_AX_FB_SU_NG_4 0
+#define HAL_NDPA_AX_FB_SU_NG_16 2
+#define HAL_NDPA_AX_FB_MU_NG_4 1
+#define HAL_NDPA_AX_FB_MU_NG_16 3
+#define HAL_NDPA_AX_FB_CQI 3
+#define HAL_NPDA_AX_CB_SU42_MU75 0
+#define HAL_NPDA_AX_CB_SU64_MU97 1
+
+struct hal_he_snd_f2p {
+	u16 csi_len_bfrp:12;
+	u16 tb_t_pe_bfrp:2;
+	u16 tri_pad_bfrp:2;
+
+	u16 ul_cqi_rpt_tri_bfrp:1;
+	u16 rf_gain_idx_bfrp:10;
+	u16 fix_gain_en_bfrp:1;
+	u16 rsvd:4;
+};
+
+struct hal_vht_ndpa_sta_info {
+	u32 aid12:12;
+	u32 feedback_type:1;
+	u32 nc:3;
+	u32 rsvd:16;
+};
+
+struct hal_he_ndpa_sta_info {
+	u32 aid:11;
+	u32 bw:14;
+	u32 fb_ng:2;
+	u32 disambiguation:1;
+	u32 cb:1;
+	u32 nc:3;
+};
+
+struct hal_ndpa_para {
+	struct hal_frame_hdr common;
+	struct hal_npda_dialog_token snd_dialog;
+	u32 ndpa_sta_info[HAL_MAX_HE_SND_STA_NUM];
+};
+
+struct hal_he_trg_frm_cmn {
+	u32 tgr_info: 4;
+	u32 ul_len: 12;
+	u32 more_tf: 1;
+	u32 cs_rqd: 1;
+	u32 ul_bw: 2;
+	u32 gi_ltf: 2;
+	u32 mimo_ltfmode: 1;
+	u32 num_heltf: 3;
+	u32 ul_pktext: 3;
+	u32 ul_stbc: 1;
+	u32 ldpc_extra_sym: 1;
+	u32 dplr: 1;
+	u32 ap_tx_pwr: 6;
+	u32 ul_sr: 16;
+	u32 ul_siga2_rsvd: 9;
+	u32 rsvd: 1;
+};
+
+struct hal_he_trg_fm_user {
+	u16 aid12;
+	u8 ru_pos;
+	u8 ul_fec_code;
+	u8 ul_mcs;
+	u8 ul_dcm;
+	u8 ss_alloc;
+	u8 ul_tgt_rssi;
+};
+
+struct hal_bfrp_he {
+	struct hal_he_trg_frm_cmn common;
+	struct hal_he_trg_fm_user user[HAL_MAX_HE_BFRP_STA_NUM];
+	u8 fbseg_rexmit_bmp[HAL_MAX_HE_BFRP_STA_NUM];/*BFRP only*/
+	struct hal_he_snd_f2p f2p_info;
+};
+
+
+struct hal_bfrp_vht {
+	u8 rexmit_bmp;
+};
+
+struct hal_bfrp_para {
+	struct hal_frame_hdr hdr[HLA_MAX_BFRP_NUM];
+	struct hal_bfrp_he he_para[HAL_MAX_HE_BFRP_NUM];
+	struct hal_bfrp_vht vht_para[HAL_MAX_VHT_BFRP_NUM];
+};
+
+struct hal_snd_wd_para {
+	u16 txpktsize;
+	u16 ndpa_duration;
+	u16 datarate:9;
+	u16 macid:7;
+	u8 force_txop:1;
+	u8 data_bw:2;
+	u8 gi_ltf:3;
+	u8 data_er:1;
+	u8 data_dcm:1;
+	u8 data_stbc:1;
+	u8 data_ldpc:1;
+	u8 data_bw_er:1;
+	u8 multiport_id:1;
+	u8 mbssid:4;
+	u8 signaling_ta_pkt_sc:4;
+	u8 sw_define:4;
+	u8 txpwr_ofset_type:3;
+	u8 lifetime_sel:3;
+	u8 stf_mode:1;
+	u8 disdatafb:1;
+	u8 data_txcnt_lmt_sel:1;
+	u8 data_txcnt_lmt:6;
+	u8 sifs_tx:1;
+	u8 snd_pkt_sel:3;
+	u8 ndpa:2;
+	u8 rsvd:3;
+};
+
+#define HAL_FEXG_TYPE_AC_SU 31
+#define HAL_FEXG_TYPE_AC_MU_1 32
+#define HAL_FEXG_TYPE_AC_MU_2 33
+#define HAL_FEXG_TYPE_AC_MU_3 34
+#define HAL_FEXG_TYPE_AX_SU 35
+#define HAL_FEXG_TYPE_AX_MU_1 36
+#define HAL_FEXG_TYPE_AX_MU_2 37
+
+struct hal_ax_fwcmd_snd {
+	u8 frame_ex_type;
+	u8 bfrp0_sta_nr;
+	u8 bfrp1_sta_nr;
+	u16 macid[8];
+	struct hal_ndpa_para ndpa;
+	struct hal_bfrp_para bfrp;
+	/**
+	 * HE: NDPA NDP HE_Trigger_BFRP (CSI) BFRP (CSI)
+	 * VHT: NDPA NDP (CSI) VHT_BFRP (CSI) BFRP (CSI) BFRP (CSI)
+	 **/
+	struct hal_snd_wd_para wd[HLA_MAX_BFRP_NUM + 2];
+};
+
+
+struct rtw_hal_com_t;
+struct hal_io_ops {
+	u8(*_read8)(struct rtw_hal_com_t *hal, u32 addr);
+	u16(*_read16)(struct rtw_hal_com_t *hal, u32 addr);
+	u32(*_read32)(struct rtw_hal_com_t *hal, u32 addr);
+	void (*_read_mem)(struct rtw_hal_com_t *hal, u32 addr, u32 cnt, u8 *pmem);
+
+	int (*_write8)(struct rtw_hal_com_t *hal, u32 addr, u8 val);
+	int (*_write16)(struct rtw_hal_com_t *hal, u32 addr, u16 val);
+	int (*_write32)(struct rtw_hal_com_t *hal, u32 addr, u32 val);
+	int (*_write_mem)(struct rtw_hal_com_t *hal, u32 addr, u32 length, u8 *pdata);
+
+#ifdef RTW_WKARD_BUS_WRITE
+	int (*_write_post_cfg)(struct rtw_hal_com_t *hal, u32 addr, u32 val);
+#endif
+
+#ifdef CONFIG_SDIO_HCI
+	u8(*_sd_f0_read8)(struct rtw_hal_com_t *hal, u32 addr);
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+	u8(*_sd_iread8)(struct rtw_hal_com_t *hal, u32 addr);
+	u16(*_sd_iread16)(struct rtw_hal_com_t *hal, u32 addr);
+	u32(*_sd_iread32)(struct rtw_hal_com_t *hal, u32 addr);
+	int (*_sd_iwrite8)(struct rtw_hal_com_t *hal, u32 addr, u8 val);
+	int (*_sd_iwrite16)(struct rtw_hal_com_t *hal, u32 addr, u16 val);
+	int (*_sd_iwrite32)(struct rtw_hal_com_t *hal, u32 addr, u32 val);
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#endif
+};
+
+struct hal_io_priv {
+	#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+	_os_mutex sd_indirect_access_mutex;
+	#endif
+	struct hal_io_ops io_ops;
+};
+
+enum pcfg_type {
+	PCFG_FUNC_SW,
+	PCFG_TBTT_AGG,
+	PCFG_TBTT_SHIFT,
+	PCFG_HIQ_WIN,
+	PCFG_HIQ_DTIM,
+	PCFG_HIQ_MAX,
+	PCFG_BCN_INTERVAL,	/* Beacon Interval */
+	PCFG_BSS_CLR,
+	PCFG_BCN_EN,
+	PCFG_MBSSID_EN,		/* M-BSSID ID enable */
+	PCFG_BCN_DRP_ALL
+};
+
+/*
+ * refers to _usb.h
+ * #define USB11                0x1
+ * #define USB2                 0x2
+ * #define USB3                 0x3
+ * */
+enum usb_type {
+	USB_1_1 = 1,
+	USB_2_0,
+	USB_3_0,
+};
+#ifdef RTW_PHL_BCN
+struct bcn_entry_pool {
+	u8 bcn_num;
+	_os_list bcn_list;
+	_os_lock bcn_lock;
+};
+#endif
+
+enum rtw_hal_int_set_opt {
+	INT_SET_OPT_HAL_INIT,
+	INT_SET_OPT_SER_START,
+	INT_SET_OPT_SER_DONE,
+	INT_SET_OPT_PS_START,
+	INT_SET_OPT_PS_STOP
+};
+
+struct hal_intr_mask_cfg {
+	u8 halt_c2h_en;
+	u8 wdt_en;
+};
+
+struct hal_halt_c2h_int {
+	/* halt c2h */
+	u32 intr;
+	u32 val_mask;
+	u32 val_default;
+};
+
+struct hal_watchdog_timer_int {
+	/* watchdog timer */
+	u32 intr;
+	u32 val_mask;
+	u32 val_default;
+};
+
+struct hal_int_array {
+	struct hal_halt_c2h_int halt_c2h_int;
+	struct hal_watchdog_timer_int watchdog_timer_int;
+};
+
+/* c2h event id for hal/phl layer */
+enum rtw_hal_c2h_ev {
+	HAL_C2H_EV_DO_NOTHING = 0,
+	HAL_C2H_EV_BB_MUGRP_DOWN = 1,/* BB Process C2H mu-score-tbl done */
+	HAL_C2H_EV_BTC_INFO = 2,	/* BTC event */
+	HAL_C2H_EV_BTC_SCBD = 3,	/* BTC event */
+	HAL_C2H_EV_MAC_TSF32_TOG = 4,	/* MAC event */
+	HAL_C2H_EV_MAX
+};
+
+/* ppdu status : per user info */
+struct hal_ppdu_sts_usr {
+	/* MAC */
+	u8 vld:1;
+	u8 has_data:1;
+	u8 has_ctrl:1;
+	u8 has_mgnt:1;
+	u8 has_bcn:1;
+	u16 macid;
+};
+
+enum hal_rxcnt_sel {
+	HAL_RXCNT_OFDM_OK = 0,
+	HAL_RXCNT_OFDM_FAIL = 1,
+	HAL_RXCNT_OFDM_FAM = 2,
+	HAL_RXCNT_CCK_OK = 3,
+	HAL_RXCNT_CCK_FAIL = 4,
+	HAL_RXCNT_CCK_FAM = 5,
+	HAL_RXCNT_HT_OK = 6,
+	HAL_RXCNT_HT_FAIL = 7,
+	HAL_RXCNT_HT_PPDU = 8,
+	HAL_RXCNT_HT_FAM = 9,
+	HAL_RXCNT_VHTSU_OK = 0xA,
+	HAL_RXCNT_VHTSU_FAIL = 0xB,
+	HAL_RXCNT_VHTSU_PPDU = 0xC,
+	HAL_RXCNT_VHTSU_FAM = 0xD,
+	HAL_RXCNT_VHTMU_OK = 0xE,
+	HAL_RXCNT_VHTMU_FAIL = 0xF,
+	HAL_RXCNT_VHTMU_PPDU = 0x10,
+	HAL_RXCNT_VHTMU_FAM = 0x11,
+	HAL_RXCNT_HESU_OK = 0x12,
+	HAL_RXCNT_HESU_FAIL = 0x13,
+	HAL_RXCNT_HESU_PPDU = 0x14,
+	HAL_RXCNT_HESU_FAM = 0x15,
+	HAL_RXCNT_HEMU_OK = 0x16,
+	HAL_RXCNT_HEMU_FAIL = 0x17,
+	HAL_RXCNT_HEMU_PPDU = 0x18,
+	HAL_RXCNT_HEMU_FAM = 0x19,
+	HAL_RXCNT_HETB_OK = 0x1A,
+	HAL_RXCNT_HETB_FAIL = 0x1B,
+	HAL_RXCNT_HETB_PPDU = 0x1C,
+	HAL_RXCNT_HETB_FAM = 0x1D,
+	HAL_RXCNT_INVD = 0x1E,
+	HAL_RXCNT_RECCA = 0x1F,
+	HAL_RXCNT_FULLDRP = 0x20,
+	HAL_RXCNT_FULLDRP_PKT = 0x21,
+	HAL_RXCNT_RXDMA = 0x22,
+	HAL_RXCNT_USER0 = 0x23,
+	HAL_RXCNT_USER1 = 0x24,
+	HAL_RXCNT_USER2 = 0x25,
+	HAL_RXCNT_USER3 = 0x26,
+	HAL_RXCNT_CONT_FCS = 0x27,
+	HAL_RXCNT_PKTFLTR_DRP = 0x28,
+	HAL_RXCNT_CSIPKT_DMA_OK = 0x29,
+	HAL_RXCNT_CSIPKT_DMA_DROP = 0x2A,
+	HAL_RXCNT_MAX
+};
+
+enum hal_rate_bw {
+	HAL_RATE_BW_20	= 0,
+	HAL_RATE_BW_40	= 1,
+	HAL_RATE_BW_80	= 2,
+	HAL_RATE_BW_160	= 3,
+};
+
+struct hal_ppdu_rx_cnt {
+	u16 ppdu_cnt[HAL_RXCNT_MAX];
+};
+
+/* ppdu sts mac bmp_append_info */
+#define HAL_PPDU_MAC_INFO BIT(1)
+#define HAL_PPDU_PLCP BIT(3)
+#define HAL_PPDU_RX_CNT BIT(2)
+/* ppdu sts mac bmp_filter */
+#define HAL_PPDU_HAS_A1M BIT(4)
+#define HAL_PPDU_HAS_CRC_OK BIT(5)
+#define HAL_PPDU_HAS_DMA_OK BIT(6)
+
+/* ppdu status (mac info + phy info) */
+struct hal_ppdu_sts {
+#define RTW_HAL_PPDU_STS_MAX_USR 4
+	/* MAC */
+	/* NOTE: (rx_cnt, plcp, phy_st)_ptr are only available before phl_recycle_rx_buf() */
+	u8 *rx_cnt_ptr;
+	u8 *plcp_ptr;
+	u8 *phy_st_ptr; /* for bb phy status */
+	u8 plcp_size;
+	u32 phy_st_size;/* for bb phy status */
+	u32 rx_cnt_size;
+	u8 usr_num;
+	struct hal_ppdu_sts_usr usr[RTW_HAL_PPDU_STS_MAX_USR];
+	struct hal_ppdu_rx_cnt rx_cnt;
+};
+
+#define HAL_RSSI_MAVG_NUM 16
+#define STA_UPDATE_MA_RSSI_FAST(_RSSI, _VAL) _RSSI = ((_RSSI * \
+					     (HAL_RSSI_MAVG_NUM - 1)) + _VAL) \
+					     / HAL_RSSI_MAVG_NUM
+
+struct rtw_cfo_info {
+	s32		cfo_tail;
+	s32		pre_cfo_avg;
+	s32		cfo_avg;
+	u16		cfo_cnt;
+	u32		tp;
+};
+
+
+struct rtw_rssi_info {
+	u8 rssi; /* u(8,1), hal-bb provide, read only : 0~110 (dBm = rssi -110) */
+	u16 rssi_ma; /* u(16,5),  hal-bb provide, read only : u16 U(12,4)*/
+	u16 rssi_ma_path[4];
+	u16 pkt_cnt_data;
+	u8 rssi_bcn; /* u(8,1), beacon RSSI, hal-bb provide, read only : 0~110 (dBm = rssi -110) */
+	u16 rssi_bcn_ma; /* u(16,5),  beacon RSSI, hal-bb provide, read only*/
+	u16 rssi_bcn_ma_path[4];
+	u16 pkt_cnt_bcn;
+	u8 ma_factor:4;
+	u8 ma_factor_bcn:4;
+	u8 rssi_ofdm; /* u(8,1),  hal-bb provide, read only : packet, for debug */
+	u8 rssi_cck; /* u(8,1),  hal-bb provide, read only : packet, for debug */
+	u8 assoc_rssi; /* phl_rx provide, read only */
+	/* phl_rx provide, read only : Moving Average RSSI information for the STA */
+	u8 ma_rssi; /* moving average : 0 ~ PHL_MAX_RSSI (dBm = rssi - PHL_MAX_RSSI) */
+	u8 ma_rssi_mgnt; /* moving average rssi for beacon/probe : 0 ~ PHL_MAX_RSSI (dBm = rssi - PHL_MAX_RSSI) */
+	u16 snr_ma; /* u(16,4), hal-bb provide, read only, SNR= snr_ma dBm*/
+	u16 snr_ma_path[4];
+};
+
+struct rtw_rate_info {
+ 	enum rtw_gi_ltf gi_ltf; /* 3bit GILTF */
+	enum rtw_rate_mode mode; /* 2bit 0:legacy, 1:HT, 2:VHT, 3:HE*/
+	enum hal_rate_bw bw; /*2bit 0:5M/10M/20M, 1:40M, 2:80M, 3:160M or 80+80*/
+	u8 mcs_ss_idx; /*HE: 3bit SS + 4bit MCS; non-HE: 5bit MCS/rate idx */
+	u8 mcs_idx;
+	u8 ss; /* 0: 1ss, 1:2ss, ... */
+	bool is_actrl; /* 0: don't append a-ctrl field; 1: append a-ctrl field */
+	bool en_stbc; /* 0: enable stbc; 1: disable stbc */
+};
+
+/* from cmn_sta_info */
+struct rtw_ra_sta_info {
+	/*u8 rate_id;			remove !!! use wmode in phl, [PHYDM] ratr_idx*/
+	/*u8 rssi_level;			[PHYDM]*/
+	/*u8 is_first_connect:1;		change connect flow, [PHYDM] CE: ra_rpt_linked, AP: H2C_rssi_rpt*/
+	/*u8 is_support_sgi:1;		mov to phl [driver]*/
+	/*u8 is_vht_enable:2;		mov to phl [driver]*/
+	/*u8 disable_ra:1;			mov to hal [driver]*/
+	/*u8 disable_pt:1;			remove for no PT, [driver] remove is_disable_power_training*/
+	/*u8 txrx_state:2;			///////////////need to check if needed, [PHYDM] 0: Tx, 1:Rx, 2:bi-direction*/
+	/*u8 is_noisy:1;			///////////////need to check if needed, [PHYDM]*/
+	u16 curr_tx_rate;			/*use struct bb_rate_info, [PHYDM] FW->Driver*/
+	enum channel_width ra_bw_mode;	/* [Driver] max bandwidth, for RA only */
+	enum channel_width curr_tx_bw;	/*bb_rate_info, [PHYDM] FW->Driver*/
+	/* u8 drv_ractrl; */
+
+	/* Ctrl */
+	u8 ra_nss_limit; /* 0: no limitation, otherwise, limit to tx nss pkt*/
+	bool dis_ra; /*move from rtw_hal_stainfo_t*/
+	bool ra_registered;/*move from rtw_hal_stainfo_t*/
+	u64 ra_mask;/*move from rtw_hal_stainfo_t*/ /*drv decide by specific req*/
+	u64 cur_ra_mask;/*move from rtw_hal_stainfo_t*/
+	/*halbb create, mod by driver and decide by rssi or other*/
+	u8 cal_giltf; /* 3bit gi_ltf ctrl by driver*/
+	bool fix_giltf_en; /*giltf from cal_giltf or halbb*/
+	bool fixed_rt_en;
+	struct rtw_rate_info fixed_rt_i;
+	u8 rainfo_cfg1; /* prepare for other control*/
+	u8 rainfo_cfg2; /* prepare for other control*/
+
+	/* Report */
+	struct rtw_rate_info rpt_rt_i;
+	u8 curr_retry_ratio;		/*[HALBB] FW->Driver*/
+
+	bool ra_csi_rate_en;
+	bool fixed_csi_rate_en;
+	u8 band_num;
+	struct rtw_rate_info csi_rate;
+
+	u8 avg_agg;
+	u32 tx_ok_cnt[4];
+	u32 tx_retry_cnt[4];
+	u32 tx_total_cnt;
+	/*u64 ramask;*/
+};
+
+struct rtw_mura_info {
+	/* Ctrl */
+	bool fixed_rt_en;
+	struct rtw_rate_info fixed_rt_i;
+	/* Report */
+	struct rtw_rate_info rpt_rt_i;
+	u8 curr_retry_ratio;		/*[PHYDM] FW->Driver*/
+};
+
+/* from cmn_sta_info */
+struct rtw_dtp_info {
+	u8 dyn_tx_power;	/*Dynamic Tx power offset*/
+	u8 last_tx_power;
+	u8 sta_tx_high_power_lvl:4;
+	u8 sta_last_dtp_lvl:4;
+};
+
+struct rtw_hal_muba_info {
+	u32 fix_ba:1;
+	u32 ru_psd:9;
+	u32 tf_rate:9;
+	u32 rf_gain_fix:1;
+	u32 rf_gain_idx:10;
+	u32 tb_ppdu_bw:2;
+	u8 dcm:1;
+	u8 ss:3;
+	u8 mcs:4;
+	u8 gi_ltf:3;
+	u8 doppler:1;
+	u8 stbc:1;
+	u8 sta_coding:1;
+	u8 tb_t_pe_nom:2;
+	u8 pr20_bw_en:1;
+	u8 ma_type: 1;
+	u8 rsvd1: 6;
+};
+
+enum rtw_hal_protection_type {
+	HAL_PROT_NO_PROETCT = 0,
+	HAL_PROT_PRIUSER_HW_RTS = 1,
+	HAL_PROT_RTS = 2,
+	HAL_PROT_CTS2SELF = 3,
+	HAL_PROT_MU_RTS = 4,
+	HAL_PROT_HW_DEFAULT_ = 5
+};
+
+enum rtw_hal_ack_resp_type {
+	HAL_ACK_N_MINUS_1USER_BA = 0,
+	HAL_ACK_N_USER_BA = 1,
+	HAL_ACK_MU_BAR = 2,
+	HAL_ACK_HTP_ACK = 3,
+	HAL_ACK_HW_DEFAULT = 4
+};
+
+struct rtw_wp_rpt_stats {
+	u32 busy_cnt;
+	u32 tx_ok_cnt;
+	u32 tx_fail_cnt;
+	u32 rty_fail_cnt;
+	u32 lifetime_drop_cnt;
+	u32 macid_drop_cnt;
+	u32 sw_drop_cnt;
+	u32 recycle_fail_cnt;
+	u32 delay_tx_ok_cnt;
+	u32 delay_rty_fail_cnt;
+	u32 delay_lifetime_drop_cnt;
+	u32 delay_macid_drop_cnt;
+};
+
+struct rtw_trx_stat {
+	u32 rx_ok_cnt;
+	u32 rx_err_cnt;
+	u16 rx_rate_plurality;
+	/* add lock for tx statistics */
+	_os_lock tx_sts_lock;
+	/* Below info is for release report*/
+	u32 tx_fail_cnt;
+	u32 tx_ok_cnt;
+#if defined(CONFIG_PHL_RELEASE_RPT_ENABLE) || defined(CONFIG_PCI_HCI)
+	struct rtw_wp_rpt_stats *wp_rpt_stats;
+#endif
+#ifdef CONFIG_PCI_HCI
+	u32 ltr_tx_dly_count;
+	u32 ltr_last_tx_dly_time;
+	u32 rx_rdu_cnt;
+#endif
+#ifdef CONFIG_VW_REFINE
+	u16 pretx_fail; /* pretx fail count */
+	u16 phltx_cnt;  /* phl_tx run count */
+
+	u32 vw_cnt_snd; /* total send count for veriwave data */
+	u32 vw_cnt_rev; /* total recv count for veriwave data */
+	u32 vw_cnt_err; /* total err  count for veriwave data */
+#endif
+	u16 rx_rate;
+	u8 rx_bw;
+	u8 rx_gi_ltf;
+};
+
+struct bacam_ctrl_t {
+	u8 used_map[MAX_BAENTRY];
+	u8 tid[MAX_BAENTRY];
+	u16 mac_id[MAX_BAENTRY];
+	u8 count;
+};
+
+struct rtw_hal_stainfo_t {
+	/* from cmn_sta_info */
+	u16	dm_ctrl;
+	/* struct su_ra_info */
+	struct rtw_rssi_info rssi_stat;
+	struct rtw_cfo_info cfo_stat;
+
+	/* Beamform Related */
+	u8 bf_cap;/* sta's beamform capability : ht/vht/he + bfee/bfer */
+	void *bf_entry;
+	u16 bf_csi_buf;
+	u16 bf_csi_buf_swap;/*used in mu swap mode*/
+	/* BFee capability */
+	u8 max_nc;
+	u8 nr;
+	u8 ng16;/* 0:non-support ; BIT0:support SU ; BIT1:support MU */
+	u8 cb_sz;/* 0:non-support ; BIT0:support (4,2) SU ; BIT1:support (7,5) MU */
+	u8 support_cqi_fb;
+	/*mu group*/
+	u8 mugrp_bmp;
+	u32 mu_score;
+	/*FW Frame Exchange : when STA is primary STA, prefer protect type and ack resp type in MU*/
+	enum rtw_hal_protection_type prot_type;
+	enum rtw_hal_ack_resp_type resp_type;
+
+	/* from cmn_sta_info */
+	struct rtw_ra_sta_info	ra_info;
+	/* from cmn_sta_info */
+	struct rtw_dtp_info dtp_stat;
+	struct rtw_trx_stat trx_stat;
+	void *bb_sta;
+};
+
+
+
+struct bus_hw_cap_t {
+#ifdef CONFIG_PCI_HCI
+	enum rtw_pcie_bus_func_cap_t l0s_ctrl;
+	enum rtw_pcie_bus_func_cap_t l1_ctrl;
+	enum rtw_pcie_bus_func_cap_t l1ss_ctrl;
+	enum rtw_pcie_bus_func_cap_t wake_ctrl;
+	enum rtw_pcie_bus_func_cap_t crq_ctrl;
+	u8 clkdly_ctrl;
+	u8 l0sdly_ctrl;
+	u8 l1dly_ctrl;
+	u8 ltr_sw_ctrl; /* whether ltr can be controlled by sw */
+	u8 ltr_hw_ctrl;
+
+#ifdef RTW_WKARD_DYNAMIC_PCIE_GEN
+	u8 pcie_gen_dm_en;
+#endif
+
+	u16 max_txbd_num;
+	u16 max_rxbd_num;
+	u16 max_rpbd_num;
+	u32 max_rxbuf_size;
+	u32 max_rpbuf_size;
+	u8 max_phyaddr_num;
+	u8 max_wd_page_size;
+	u8 txbd_len;
+	u8 rxbd_len;
+	u8 wdb_size;
+	u8 wdi_size;
+	u8 addr_info_size;
+	u8 seq_info_size;
+#elif defined (CONFIG_USB_HCI)
+	u32 tx_buf_size;
+	u32 tx_buf_num;
+	u32 tx_mgnt_buf_size;
+	u32 tx_mgnt_buf_num;
+	u32 tx_h2c_buf_num;
+	u32 rx_buf_size;
+	u32 rx_buf_align_size;
+	u32 rx_buf_num;
+	u32 in_token_num;
+#elif defined (CONFIG_SDIO_HCI)
+	u32 tx_buf_size;
+	u32 tx_buf_num;
+	u32 tx_mgnt_buf_size;
+	u32 tx_mgnt_buf_num;
+	u32 rx_buf_size;
+	u32 rx_buf_num;
+#else
+	u8 temp_for_struct_empty; /* for undefined interface */
+#endif
+};
+
+/* phy capability of phy */
+struct phy_hw_cap_t {
+	#ifdef RTW_WKARD_BTC_RFETYPE
+	u8 rfe_type;
+	#endif
+	u8 tx_num;
+	u8 rx_num;
+	u8 tx_path_num;
+	u8 rx_path_num;
+	u8 proto_sup;
+	u16 hw_rts_time_th;
+	u16 hw_rts_len_th;
+	u32 txagg_num;
+};
+
+
+/*PHYx + Sx*/
+enum phl_phy_idx {
+	HW_PHY_0,
+	HW_PHY_1,
+	HW_PHY_MAX
+};
+
+enum phl_pwr_table {
+	PWR_BY_RATE	= BIT0,
+	PWR_LIMIT	= BIT1,
+	PWR_LIMIT_RU	= BIT2
+};
+
+enum phl_rf_mode {
+    RF_MODE_NORMAL		= 0,
+    RF_MODE_SHUTDOWN	= 1,
+    RF_MODE_STANDBY		= 2,
+	RF_MODE_RX		= 3,
+	RF_MODE_TX		= 4,
+	RF_MODE_MAX
+};
+
+enum phl_pwr_ctrl {
+	ALL_TIME_CTRL = 0,
+        GNT_TIME_CTRL,
+        PWR_CTRL_MAX
+};
+
+/*--------------------------[Structure]-------------------------------------*/
+#if 0
+enum rtw_tpu_op_mode {
+	TPU_NORMAL_MODE		= 0,
+	TPU_DBG_MODE		= 1
+};
+
+struct rtw_tpu_pwr_by_rate_info { /*TX Power Unit (TPU)*/
+	s8 pwr_by_rate_lgcy[TPU_SIZE_PWR_TAB_lGCY];
+	s8 pwr_by_rate[HAL_MAX_PATH][TPU_SIZE_PWR_TAB];
+};
+
+struct rtw_tpu_pwr_imt_info { /*TX Power Unit (TPU)*/
+	s8 pwr_lmt_cck_20m[HAL_MAX_PATH][TPU_SIZE_BF];
+	s8 pwr_lmt_cck_40m[HAL_MAX_PATH][TPU_SIZE_BF];
+	s8 pwr_lmt_lgcy_20m[HAL_MAX_PATH][TPU_SIZE_BF]; /*ofdm*/
+	s8 pwr_lmt_20m[HAL_MAX_PATH][TPU_SIZE_BW20_SC][TPU_SIZE_BF];
+	s8 pwr_lmt_40m[HAL_MAX_PATH][TPU_SIZE_BW40_SC][TPU_SIZE_BF];
+	s8 pwr_lmt_80m[HAL_MAX_PATH][TPU_SIZE_BW80_SC][TPU_SIZE_BF];
+	s8 pwr_lmt_160m[HAL_MAX_PATH][TPU_SIZE_BF];
+	s8 pwr_lmt_40m_0p5[HAL_MAX_PATH][TPU_SIZE_BF];
+	s8 pwr_lmt_40m_2p5[HAL_MAX_PATH][TPU_SIZE_BF];
+};
+
+struct rtw_tpu_info { /*TX Power Unit (TPU)*/
+	enum rtw_tpu_op_mode op_mode; /*In debug mode, only debug tool control TPU APIs*/
+	bool normal_mode_lock_en;
+	s8 ofst_int; /*SW: S(8,3) -16 ~ +15.875 (dB)*/
+	u8 ofst_fraction; /*[0:3] * 0.125(dBm)*/
+	enum hal_path ref_pow_path; /*Select the path with larger pow as the re_ path*/
+	u8 path_pow_ofst_decrease; /* Select the path with lower pow and subtract pow_path_ofst_decrease from path_ref*/
+	u8 base_cw_0db; /*[63~39~15]: [+24~0~-24 dBm]*/
+	u16 tssi_16dBm_cw;
+	/*[Ref Pwr]*/
+	s16 ref_pow_ofdm; /*-> HW: s(9,2)*/
+	s16 ref_pow_cck; /*-> HW: s(9,2)*/
+	u16 ref_pow_ofdm_cw; /*BBCR 0x58E0[9:0]*/
+	u16 ref_pow_cck_cw; /*BBCR 0x58E0[21:12]*/
+	/*[Pwr Ofsset]*/ /*-> HW: s(7,1)*/
+	s8 pwr_ofst_mode[TPU_SIZE_MODE]; /*0~4: HE, VHT, HT, Legacy, CCK, */
+	s8 pwr_ofst_bw[TPU_SIZE_BW]; /*0~4: 80_80, 160, 80, 40, 20*/
+	/*[Pwr By rate]*/ /*-> HW: s(7,1)*/
+	struct rtw_tpu_pwr_by_rate_info rtw_tpu_pwr_by_rate_i;
+	/*[Pwr Limit]*/ /*-> HW: s(7,1)*/
+	struct rtw_tpu_pwr_imt_info rtw_tpu_pwr_imt_i;
+	/*[Pwr Limit RUA]*/ /*-> HW: s(7,1)*/
+	s8 pwr_lmt_ru[HAL_MAX_PATH][TPU_SIZE_RUA][TPU_SIZE_BW20_SC];
+	u16 pwr_lmt_ru_mem_size;
+	bool pwr_lmt_en;
+	bool ext_pwr_lmt_en;
+	struct rtw_phl_ext_pwr_lmt_info ext_pwr_lmt_i;
+	u8 tx_ptrn_shap_idx;
+	u8 tx_ptrn_shap_idx_cck;
+	u16 pwr_constraint_mb;
+};
+#endif
+struct rtw_hal_stat_info {
+	u32 cnt_fail_all;
+	u32 cnt_cck_fail;
+	u32 cnt_ofdm_fail;
+	u32 cnt_cca_all;
+	u32 cnt_ofdm_cca;
+	u32 cnt_cck_cca;
+	u32 cnt_crc32_error_all;
+	u32 cnt_he_crc32_error;
+	u32 cnt_vht_crc32_error;
+	u32 cnt_ht_crc32_error ;
+	u32 cnt_ofdm_crc32_error;
+	u32 cnt_cck_crc32_error;
+	u32 cnt_crc32_ok_all;
+	u32 cnt_he_crc32_ok;
+	u32 cnt_vht_crc32_ok;
+	u32 cnt_ht_crc32_ok;
+	u32 cnt_ofdm_crc32_ok;
+	u32 cnt_cck_crc32_ok;
+	u32 igi_fa_rssi;
+};
+
+struct rtw_hw_band {
+	struct rtw_chan_def cur_chandef;
+	u8 ppdu_sts_appen_info;
+	u8 ppdu_sts_filter;
+	struct rtw_tpu_info rtw_tpu_i; /*TX Power Unit (TPU)*/
+	union bb_tpu_all_info bb_tpu_all_i; /*TX Power Unit (TPU)*/
+	u16 tx_pause[PAUSE_RSON_MAX]; /* ref: enum rtw_sch_txen_cfg */
+	struct rtw_hal_stat_info stat_info;
+	u8 assoc_sta_cnt; /*number of associated nodes (sta or ap)*/
+	enum rtw_rx_fltr_opt_mode rx_fltr_opt_mode;
+};
+
+struct rtw_intr_t {
+	bool en;
+	u32 mask_dflt;
+	u32 mask;
+	u32 val;
+};
+
+struct rtw_hal_com_t {
+	enum rtw_chip_id chip_id;
+	enum rtw_cv cv;
+	enum rtw_cv acv;
+	enum rtw_fv fv;
+	u32 aid;
+
+	struct ver_ctrl_t mac_vc;
+	struct ver_ctrl_t bb_vc;
+	struct ver_ctrl_t rf_vc;
+	struct ver_ctrl_t btc_vc;
+	struct ver_ctrl_t fw_vc;
+
+	struct protocol_cap_t proto_hw_cap[MAX_BAND_NUM]; /* wifi protocol capability from EFUSE/halmac/halbb/halrf/... */
+	struct phy_hw_cap_t phy_hw_cap[MAX_BAND_NUM]; /* phy capability from EFUSE/halmac/halbb/halrf/... */
+	struct dev_cap_t dev_hw_cap;
+
+	struct bus_hw_cap_t bus_hw_cap; /* Bus HW capability */
+	struct bus_cap_t bus_cap; /* Final bus capability */
+	struct hal_io_priv iopriv;
+	#ifdef DBG_HAL_MAC_MEM_MOINTOR
+	_os_atomic hal_mac_mem;
+	#endif
+	#ifdef DBG_HAL_MEM_MOINTOR
+	_os_atomic hal_mem;
+	#endif
+	bool is_hal_init;
+	struct rtw_hw_band band[MAX_BAND_NUM];/*band0/band1 for DBCC*/
+
+	bool dbcc_en;
+	u8 assoc_sta_cnt; /*number of associated nodes (sta or ap)*/
+
+#ifdef RTW_WKARD_SINGLE_PATH_RSSI
+	enum rf_path cur_rx_rfpath;
+#endif
+#ifdef CONFIG_PCI_HCI /*TODO move to hal_info_t*/
+	/*interrupt*/
+	u32 int_array[4];
+	u32 int_mask[4];
+	u32 int_mask_default[4];
+
+	struct rtw_intr_t _intr_ind[4];
+	struct rtw_intr_t _intr[4];
+	bool in_ps_intr_cfg;
+#ifdef PHL_RXSC_ISR
+	u32 rx_int_array;
+#endif
+#endif /* CONFIG_PCI_HCI */
+#ifdef CONFIG_SDIO_HCI /*TODO move to hal_info_t*/
+	u32 block_sz;
+
+	/*interrupt*/
+	/*
+	 * Change type of int_mask from u32 to unsigned long for bit operation
+	 * API, ex: _os_test_and_clear_bit() and _os_test_and_set_bit() .
+	 */
+	unsigned long int_mask;
+	u32 int_mask_default;
+#endif /* CONFIG_SDIO_HCI */
+
+	struct hal_int_array intr;
+
+#ifdef RTW_PHL_BCN
+	struct bcn_entry_pool bcn_pool;
+#endif
+	struct rtw_trx_stat trx_stat;
+	void *hal_priv;/*pointer to hal_info*/
+	void *drv_priv; /*drv priv*/
+
+	void *csi_obj;
+	void *bf_obj;
+	void *snd_obj;
+
+	struct hal_mu_score_tbl bb_mu_score_tbl;
+	bool csi_para_ctrl_sel;
+
+	struct btc_ctrl_t btc_ctrl;
+	struct btc_fw_msg btc_msg;
+#ifdef RTW_WKARD_CCX_RPT_LIMIT_CTRL
+	u8 spe_pkt_cnt_lmt;
+#endif
+	u32 uuid;
+	u8 scanofld_en;
+	struct bacam_ctrl_t ba_ctl;
+};
+
+#define FL_CFG_OP_SET 0
+#define FL_CFG_OP_CLR 1
+#define FL_CFG_OP_INFO 2
+
+#define FL_CFG_TYPE_LEVEL 0
+#define FL_CFG_TYPE_OUTPUT 1
+#define FL_CFG_TYPE_COMP 2
+#define FL_CFG_TYPE_COMP_EXT 3
+
+#define FL_LV_OFF 0
+#define FL_LV_CRT 1
+#define FL_LV_SER 2
+#define FL_LV_WARN 3
+#define FL_LV_LOUD 4
+#define FL_LV_TR 5
+
+#define FL_OP_UART BIT0
+#define FL_OP_C2H BIT1
+#define FL_OP_SNI BIT2
+
+struct rtw_hal_fw_log_cfg {
+	u32 level;
+	u32 output;
+	u32 comp;
+	u32 comp_ext;
+};
+
+enum pkt_ofld_op_type {
+	PKT_OFLD_ADD = 0,
+	PKT_OFLD_DEL = 1,
+	PKT_OFLD_READ = 2,
+	PKT_OFLD_MAX
+};
+
+struct pkt_ofld_info {
+	struct list_head req_q;
+	u32 req_cnt;
+	u8 id;
+};
+
+/* Up to 8 add_pkt_ofld are allowed for the same type */
+#define PKT_OFLD_MAX_VAL 8
+struct pkt_ofld_entry {
+	struct list_head list;
+	u16 macid;
+	struct pkt_ofld_info pkt_info[PKT_OFLD_TYPE_MAX][PKT_OFLD_MAX_VAL];
+};
+
+enum rtw_c2h_cat {
+	C2H_CAT_TEST = 0,
+	C2H_CAT_MAC = 1,
+	C2H_CAT_OUTSRC = 2,
+	C2H_CAT_MAX = 3
+};
+
+enum rtw_c2h_clas {
+	C2H_CLS_PHYDM_MIN = 0,
+	C2H_CLS_PHYDM_MAX = 7,
+	C2H_CLS_RF_MIN = 8,
+	C2H_CLS_RF_MAX = 0xf,
+	C2H_CLS_BTC_MIN = 0x10,
+	C2H_CLS_BTC_MAX = 0x17,
+	C2H_CLS_MAX = 0x18
+};
+
+#define C2H_CLS_MAC_MIN 0x00
+#define C2H_CLS_MAC_MAX 0xFF
+
+enum rtw_hal_ps_pwr_req_src {
+	HAL_BB_PWR_REQ = 0,
+	HAL_RF_PWR_REQ = 1,
+	HAL_MAC_PWR_REQ = 2,
+	HAL_BTC_PWR_REQ = 3
+};
+
+struct rtw_hal_lps_info {
+	u8 en;
+	u16 macid;
+	enum rtw_lps_listen_bcn_mode listen_bcn_mode;
+	u8 awake_interval;
+	enum rtw_lps_smart_ps_mode smart_ps_mode;
+	u8 bcnnohit_en;
+	u8 dyntxant_en;
+	u8 maxtxant;
+	u8 lpstxant;
+};
+
+struct rtw_hal_ips_info {
+	u8 en;
+	u16 macid;
+};
+
+enum ps_pwr_state {
+	PS_PWR_STATE_ACTIVE = 0,
+	PS_PWR_STATE_BAND0_RFON = 1,
+	PS_PWR_STATE_BAND1_RFON = 2,
+	PS_PWR_STATE_BAND0_RFOFF = 3,
+	PS_PWR_STATE_BAND1_RFOFF = 4,
+	PS_PWR_STATE_CLK_GATED = 5,
+	PS_PWR_STATE_PWR_GATED = 6,
+	PS_PWR_STATE_MAX,
+};
+
+#ifdef CONFIG_PHL_DFS
+struct hal_mac_dfs_rpt_cfg {
+	bool rpt_en;
+	u8 rpt_num_th;
+	bool rpt_en_to;
+	u8 rpt_to;
+};
+
+struct hal_dfs_rpt {
+	u8 *dfs_ptr;
+	u16 dfs_num;
+	u8 phy_idx; /*phy0,phy1*/
+};
+#endif
+
+#ifdef CONFIG_WOWLAN
+struct rtw_hal_wow_cfg {
+	struct rtw_keep_alive_info *keep_alive_cfg;
+	struct rtw_disc_det_info *disc_det_cfg;
+	struct rtw_nlo_info *nlo_cfg;
+	struct rtw_arp_ofld_info *arp_ofld_cfg;
+	struct rtw_ndp_ofld_info *ndp_ofld_cfg;
+	struct rtw_gtk_ofld_info *gtk_ofld_cfg;
+	struct rtw_realwow_info *realwow_cfg;
+	struct rtw_wow_wake_info *wow_wake_cfg;
+	struct rtw_pattern_match_info *pattern_match_info;
+	struct rtw_wow_gpio_info *wow_gpio;
+	struct rtw_periodic_wake_info *periodic_wake_cfg;
+#ifdef CONFIG_PHL_MDNS_OFFLOAD
+	struct rtw_mdns_ofld_info *mdns_ofld_info;
+#endif
+#ifdef CONFIG_PHL_WOW_APF
+	struct rtw_apf_info *apf_info;
+#endif
+};
+#endif /* CONFIG_WOWLAN */
+
+
+enum hal_tsf_sync_act {
+	HAL_TSF_SYNC_NOW_ONCE = 0,
+	HAL_TSF_EN_SYNC_AUTO = 1,
+	HAL_TSF_DIS_SYNC_AUTO = 2,
+};
+
+struct watchdog_nhm_report {
+	u8 ccx_rpt_stamp;
+	u8 ccx_rpt_result;
+	s8 nhm_pwr_dbm;
+	u8 nhm_ratio;
+};
+
+#define GT3_TIMEOUT_MASK 0x0FFFFFFF
+struct hal_mac_dbg_dump_cfg {
+	u32 ss_dbg_0;
+	u32 ss_dbg_1;
+	u8 ss_dbg;
+	u8 dle_dbg;
+	u8 dmac_dbg;
+	u8 cmac_dbg;
+	u8 mac_dbg_port;
+	u8 plersvd_dbg;
+	u8 tx_flow_dbg;
+};
+
+/**
+ * @enum rtw_hal_lps_flg_state
+ * @brief rtw_hal_lps_flg_state
+ *
+ * @var rtw_hal_lps_flg_state::RTW_HAL_LPS_FLG_STATE_ACTIVE
+ * Active mode or lps rf off
+ * @var rtw_hal_lps_flg_state::RTW_HAL_LPS_FLG_STATE_LPS
+ * Enter lps cg or lps pg
+ *
+ */
+enum rtw_hal_lps_flg_state {
+	RTW_HAL_LPS_FLG_STATE_ACTIVE = 0,
+	RTW_HAL_LPS_FLG_STATE_LPS = 1,
+	RTW_HAL_LPS_FLG_STATE_MAX,
+};
+
+struct hal_ppdu_sts_cfg {
+	u8 band_idx;
+	bool ppdu_stat_en;
+	u8 appen_info;
+	u8 filter;
+	bool towcpu;
+	bool todcpu;
+};
+
+#endif /*_HAL_DEF_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_btc.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_btc.h
new file mode 100644
index 000000000000..93c1cc3cc42a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_btc.h
@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_DEF_BTC_H_
+#define _HAL_DEF_BTC_H_
+
+#endif /*_HAL_DEF_BTC_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_mac.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_mac.h
new file mode 100644
index 000000000000..778526353340
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_mac.h
@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_DEF_MAC_H_
+#define _HAL_DEF_MAC_H_
+
+#endif /*_HAL_DEF_MAC_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_phy.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_phy.h
new file mode 100644
index 000000000000..ec67ddad9864
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_def_phy.h
@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_DEF_PHY_H_
+#define _HAL_DEF_PHY_H_
+
+#endif /*_HAL_DEF_PHY_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dfs.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dfs.c
new file mode 100644
index 000000000000..deda5448852e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dfs.c
@@ -0,0 +1,46 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_DFS_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_DFS
+
+enum rtw_hal_status
+rtw_hal_radar_detect_cfg(void *hal, u8 band_idx, bool dfs_enable)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_INFO("====>%s dfs_en:%d ============\n", __func__, dfs_enable);
+
+	rtw_hal_bb_dfs_rpt_cfg(hal_info->hal_com, rtw_hal_hw_band_to_phy_idx(band_idx), dfs_enable);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool rtw_hal_is_radar_detect_enabled(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	return rtw_phl_is_radar_detect_enabled(hal_info->phl_com, rtw_hal_phy_idx_to_hw_band(phy_idx));
+}
+
+bool rtw_hal_is_under_cac(struct rtw_hal_com_t *hal_com, enum phl_phy_idx phy_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+
+	return rtw_phl_is_under_cac(hal_info->phl_com, rtw_hal_phy_idx_to_hw_band(phy_idx));
+}
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dfs.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dfs.h
new file mode 100644
index 000000000000..3fd96df8d9b5
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_dfs.h
@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_DFS_H_
+#define _HAL_DFS_H_
+
+#endif
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ext_tx_pwr_lmt.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ext_tx_pwr_lmt.c
new file mode 100644
index 000000000000..f40e5e858eac
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ext_tx_pwr_lmt.c
@@ -0,0 +1,103 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_EXT_TX_PWR_LMT_C_
+#include "hal_headers.h"
+
+enum rtw_hal_status
+rtw_hal_set_power_limit(void *hal, u8 band_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	enum phl_phy_idx phy_idx = HW_PHY_0;
+
+	if (band_idx == 1) {
+		if (hal_info->hal_com->dbcc_en) {
+			phy_idx = HW_PHY_1;
+		} else {
+			PHL_WARN("%s: band_idx(%d) dbcc_en(%d)\n", __func__,
+				 band_idx, hal_info->hal_com->dbcc_en);
+			return status;
+		}
+	}
+	status = rtw_hal_rf_set_power(hal_info, phy_idx, (PWR_LIMIT | PWR_LIMIT_RU));
+	return status;
+}
+
+s8 rtw_hal_get_power_limit(void *hal, enum phl_phy_idx phy,
+	u16 rate, u8 bandwidth, u8 beamforming, u8 tx_num, u8 channel)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_power_limit(hal_info, phy, rate, bandwidth,
+					  beamforming, tx_num, channel);
+}
+
+bool
+rtw_hal_get_ext_pwr_lmt_en(void *hal,
+		u8 hw_band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_tpu_info *tpu = &(hal_com->band[hw_band].rtw_tpu_i);
+
+	PHL_INFO("%s: hw_band(%d), en(%d)\n", __func__, hw_band, tpu->ext_pwr_lmt_en);
+
+	return tpu->ext_pwr_lmt_en;
+}
+
+void
+rtw_hal_set_ext_pwr_lmt_en(void *hal,
+		u8 hw_band, bool enable)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_tpu_info *tpu = &(hal_com->band[hw_band].rtw_tpu_i);
+
+	PHL_INFO("%s: hw_band(%d), en(%d)\n", __func__, hw_band, enable);
+
+	tpu->ext_pwr_lmt_en = enable;
+}
+
+void
+rtw_hal_enable_ext_pwr_lmt(void *hal, u8 hw_band,
+		struct rtw_phl_ext_pwr_lmt_info *ext_pwr_lmt_info)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct rtw_tpu_info *tpu = &(hal_com->band[hw_band].rtw_tpu_i);
+	enum phl_phy_idx phy_idx = HW_PHY_0;
+
+	PHL_INFO("%s\n", __func__);
+
+	if (hw_band == 1)
+		phy_idx = HW_PHY_1;
+
+
+	/* Fill external tx power limit into hal_com->band[hw_band].rtw_tpu_i.ext_pwr_lmt_i */
+	_os_mem_cpy(hal_com->drv_priv, &(tpu->ext_pwr_lmt_i),
+		    ext_pwr_lmt_info, sizeof(struct rtw_phl_ext_pwr_lmt_info));
+
+	/**
+	 * Update extension(or external) tx power limit to corresponding field of halrf power limit table (as below)
+	 *   2g=> rf->pwr_info.tx_pwr_limit_2g[PW_LMT_REGU_EXT_PWR][bw][rate][bf][ch][tx_num]
+	 *   5g=> rf->pwr_info.tx_pwr_limit_5g[PW_LMT_REGU_EXT_PWR][bw][rate][bf][ch][tx_num]
+	 *   6g=> rf->pwr_info.tx_pwr_limit_6g[PW_LMT_REGU_EXT_PWR][bw][rate][bf][ch][tx_num]
+	 */
+
+	/* enable external tx power limit mechanism */
+	tpu->ext_pwr_lmt_en = true;
+	rtw_hal_rf_update_ext_pwr_lmt_table(hal_info, phy_idx);
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_feature.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_feature.h
new file mode 100644
index 000000000000..adbf884306bf
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_feature.h
@@ -0,0 +1,69 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2022 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_FEATURE_H_
+#define _HAL_FEATURE_H_
+
+#if defined(PHL_PLATFORM_LINUX)
+/*
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_WIFI_SENSING_CSI
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_SCAN_OFFLOAD
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_WRITE_OFFLOAD
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_SWITCH_CHANNEL_OFFLOAD
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_PS_SUPPORT_CPU_PG
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_TWT_NIC
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_NAN
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_FTM
+			#error "fw feature unexpected"
+		#endif
+	#endif
+	#ifdef CONFIG_PHL_XXXX
+		#ifndef FW_CONFIG_RATE_ADAPTIVE
+			#error "fw feature unexpected"
+		#endif
+	#endif
+*/
+
+#endif
+
+#endif /*_HAL_FEATURE_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_fw.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_fw.c
new file mode 100644
index 000000000000..ad7dff60ad69
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_fw.c
@@ -0,0 +1,341 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_FW_C_
+#include "hal_headers.h"
+
+static void _hal_send_fwdl_hub_msg(struct rtw_phl_com_t *phl_com, u8 dl_ok)
+{
+	struct phl_msg msg = {0};
+	u16 evt_id = (dl_ok) ? MSG_EVT_FWDL_OK : MSG_EVT_FWDL_FAIL;
+
+	msg.inbuf = NULL;
+	msg.inlen = 0;
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_PHY_MGNT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, evt_id);
+	msg.band_idx = HW_BAND_0;
+	rtw_phl_msg_hub_hal_send(phl_com, NULL, &msg);
+}
+
+static void _hal_fw_log_set(struct rtw_hal_fw_log_cfg *fl_cfg, u8 type, u32 value)
+{
+	switch(type) {
+	case FL_CFG_TYPE_LEVEL:
+		fl_cfg->level = value;
+		break;
+
+	case FL_CFG_TYPE_OUTPUT:
+		fl_cfg->output |= value;
+		break;
+
+	case FL_CFG_TYPE_COMP:
+		fl_cfg->comp |= value;
+		break;
+
+	case FL_CFG_TYPE_COMP_EXT:
+		fl_cfg->comp_ext |= value;
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void _hal_fw_log_clr(struct rtw_hal_fw_log_cfg *fl_cfg, u8 type, u32 value)
+{
+	switch(type) {
+	case FL_CFG_TYPE_LEVEL:
+		break;
+
+	case FL_CFG_TYPE_OUTPUT:
+		fl_cfg->output &= (~value);
+		break;
+
+	case FL_CFG_TYPE_COMP:
+		fl_cfg->comp &= (~value);
+		break;
+
+	case FL_CFG_TYPE_COMP_EXT:
+		fl_cfg->comp_ext &= (~value);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void _hal_fw_log_info(struct rtw_hal_fw_log_cfg *fl_cfg)
+{
+	PHL_PRINT("%s: level %d, output 0x%08x, comp 0x%08x, comp ext 0x%08x.\n",
+			__func__,
+			fl_cfg->level,
+			fl_cfg->output,
+			fl_cfg->comp,
+			fl_cfg->comp_ext);
+}
+
+/*
+ * Only export for hal_submodule
+*/
+enum rtw_hal_status rtw_hal_fw_log_cfg(void *halcom, u8 op, u8 type, u32 value)
+{
+	struct rtw_hal_com_t *hal_com = (struct rtw_hal_com_t *)halcom;
+	static struct rtw_hal_fw_log_cfg fl_cfg = {0};
+	struct hal_info_t *hal_info = NULL;
+
+	switch(op) {
+	case FL_CFG_OP_SET:
+		if (type == FL_CFG_TYPE_LEVEL) {
+			hal_info = (struct hal_info_t *)hal_com->hal_priv;
+			if (hal_info && !IS_FW_LOG_DUMP_ALLOWED(hal_info->phl_com)) {
+				value = FL_LV_OFF;
+				PHL_INFO("%s(): force log level to (%d) due to IS_FW_LOG_DUMP_ALLOWED(%d)\n",
+						__func__,
+						value,
+						(IS_FW_LOG_DUMP_ALLOWED(hal_info->phl_com) > 0));
+			}
+		}
+		_hal_fw_log_set(&fl_cfg, type, value);
+		break;
+	case FL_CFG_OP_CLR:
+		_hal_fw_log_clr(&fl_cfg, type, value);
+		break;
+	case FL_CFG_OP_INFO:
+		_hal_fw_log_info(&fl_cfg);
+		break;
+	default:
+		break;
+	}
+
+	return rtw_hal_mac_fw_log_cfg(hal_com, &fl_cfg);
+}
+
+void rtw_hal_fw_recover_log_cfg(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	rtw_hal_fw_log_cfg(hal_com, FL_CFG_OP_INFO, 0, 0);
+}
+
+void hal_fw_en_basic_log(struct rtw_hal_com_t *hal_com,
+                         struct mac_ax_fw_log* fw_log_info)
+{
+	rtw_hal_fw_log_cfg(hal_com, FL_CFG_OP_SET, FL_CFG_TYPE_LEVEL,
+				fw_log_info->level);
+	rtw_hal_fw_log_cfg(hal_com, FL_CFG_OP_SET, FL_CFG_TYPE_OUTPUT,
+				fw_log_info->output);
+	rtw_hal_fw_log_cfg(hal_com, FL_CFG_OP_SET, FL_CFG_TYPE_COMP,
+				fw_log_info->comp);
+	rtw_hal_fw_log_cfg(hal_com, FL_CFG_OP_SET, FL_CFG_TYPE_COMP_EXT,
+				fw_log_info->comp_ext);
+}
+
+enum rtw_hal_status rtw_hal_en_fw_log_comp(void *hal, u32 comp, bool en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	if (en)
+		status = rtw_hal_fw_log_cfg(hal_info->hal_com, FL_CFG_OP_SET,
+						FL_CFG_TYPE_COMP, comp);
+	else
+		status = rtw_hal_fw_log_cfg(hal_info->hal_com, FL_CFG_OP_CLR,
+						FL_CFG_TYPE_COMP, comp);
+	PHL_INFO("rtw_hal_en_fw_log_comp(): status(%d), en(%d)\n", status, en);
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_set_fw_log_lvl(void *hal, u32 lvl)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_hal_fw_log_cfg(hal_info->hal_com, FL_CFG_OP_SET,
+				    FL_CFG_TYPE_LEVEL, lvl);
+
+	PHL_INFO("%s(): status(%d), level(%d)\n",
+			 __func__, status, lvl);
+	return status;
+}
+
+enum rtw_hal_status
+rtw_hal_download_fw(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_fw_info_t *fw_info = &phl_com->fw_info;
+
+	FUNCIN_WSTS(hal_status);
+
+	if (!fw_info->fw_en)
+		return hal_status;
+
+	if (fw_info->dlrom_en) {
+		hal_status = rtw_hal_mac_romdl(hal_info, fw_info->rom_buff,
+									   fw_info->rom_size);
+
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			return hal_status;
+	}
+
+	hal_status = rtw_hal_mac_disable_cpu(hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("disable cpu fail!\n");
+		return hal_status;
+	}
+
+	hal_status = rtw_hal_mac_enable_cpu(hal_info, 0, true);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("enable cpu fail!\n");
+		return hal_status;
+	}
+
+	if (fw_info->dlram_en) {
+		hal_status = rtw_hal_mac_fwdl(hal_info, fw_info->ram_buff,
+									  fw_info->ram_size);
+	}
+
+	_hal_send_fwdl_hub_msg(phl_com, (!hal_status) ? true : false);
+
+	FUNCOUT_WSTS(hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_redownload_fw(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_fw_info_t *fw_info = &phl_com->fw_info;
+	u32 start_t = 0;
+
+	FUNCIN_WSTS(hal_status);
+
+	/* Disable/Enable CPU is necessary first when FWDL from files */
+	if(fw_info->dlram_en && fw_info->fw_src == RTW_FW_SRC_EXTNAL) {
+
+		hal_status = rtw_hal_mac_disable_cpu(hal_info);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("disable cpu fail!\n");
+			return hal_status;
+		}
+
+		hal_status = rtw_hal_mac_enable_cpu(hal_info, 0, true);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("enable cpu fail!\n");
+			return hal_status;
+		}
+
+		hal_status = rtw_hal_mac_fwdl(hal_info, fw_info->ram_buff,
+					      fw_info->ram_size);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("rtw_hal_mac_fwdl fail!\n");
+			return hal_status;
+		}
+	}
+
+	if (fw_info->dlram_en && fw_info->fw_src != RTW_FW_SRC_EXTNAL) {
+		start_t = _os_get_cur_time_us();
+		hal_status = rtw_hal_mac_enable_fw(hal_info, fw_info->fw_type);
+		PHL_INFO("%s : fw_type: %d, FWDL Total Cost Time: %u us\n",
+			 __func__, fw_info->fw_type,
+			 phl_get_passing_time_us(start_t));
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("rtw_hal_mac_enable_fw fail!\n");
+			return hal_status;
+		}
+	}
+
+	if (rtw_hal_mac_get_wcpu_cap(phl_com, hal_info) != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s : can't get fw capability.\n", __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+#ifdef CONFIG_POWER_SAVE
+	if (fw_info->fw_type == RTW_FW_WOWLAN || fw_info->fw_type == RTW_FW_WOWLAN_CE)
+		rtw_hal_ps_fw_cap_decision(phl_com, true);
+	else
+		rtw_hal_ps_fw_cap_decision(phl_com, false);
+#endif /* CONFIG_POWER_SAVE */
+
+	rtw_hal_rf_config_radio_to_fw(hal_info);
+
+#ifdef CONFIG_PHL_PKTOFLD
+	/**
+	 * Should reset packet offload related information
+	 * 1. phl packet offload module
+	 * 2. halmac packet offload module
+	 */
+	rtw_phl_pkt_ofld_del_all_entry_req(phl_com);
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_reset_pkt_ofld_state(hal_info))
+		PHL_ERR("%s: reset pkt ofld state fail!\n", __func__);
+#endif
+	_hal_send_fwdl_hub_msg(phl_com, (!hal_status) ? true : false);
+	#ifdef CONFIG_BTCOEX
+	rtw_hal_btc_redownload_fw_ntfy(hal_info);
+	#endif /* CONFIG_BTCOEX */
+	FUNCOUT_WSTS(hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_pg_redownload_fw(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct rtw_fw_info_t *fw_info = &phl_com->fw_info;
+	u8 *fw_buff = NULL;
+	u32 fw_size = 0;
+
+	FUNCIN_WSTS(hal_status);
+
+	if (!rtw_hal_mac_fwredl_needed(hal_info))
+		goto out;
+
+	if (fw_info->fw_src == RTW_FW_SRC_EXTNAL) {
+		fw_buff = fw_info->ram_buff;
+		fw_size = fw_info->ram_size;
+	} else {
+		hal_status = rtw_hal_mac_query_fw_buff(hal_info,
+						fw_info->fw_type,
+						&fw_buff, &fw_size);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			return RTW_HAL_STATUS_FAILURE;
+	}
+
+	hal_status = rtw_hal_mac_fwredl(hal_info, fw_buff, fw_size);
+
+out:
+	FUNCOUT_WSTS(hal_status);
+
+	return hal_status;
+}
+
+void rtw_hal_fw_dbg_dump(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_mac_fw_dbg_dump(hal_info);
+}
+#ifdef CONFIG_HAL_MAC_DBG
+enum rtw_fw_status rtw_hal_get_fw_status(void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+
+	return rtw_hal_mac_get_fw_status(hal);
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_fw.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_fw.h
new file mode 100644
index 000000000000..5ebd7d093168
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_fw.h
@@ -0,0 +1,35 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_FW_H_
+#define _HAL_FW_H_
+
+enum rtw_hal_status
+rtw_hal_download_fw(struct rtw_phl_com_t *phl_com, void *hal);
+
+enum rtw_hal_status
+rtw_hal_redownload_fw(struct rtw_phl_com_t *phl_com, void *hal);
+
+enum rtw_hal_status
+rtw_hal_pg_redownload_fw(struct rtw_phl_com_t *phl_com, void *hal);
+
+void rtw_hal_fw_dbg_dump(void *hal);
+
+void rtw_hal_fw_recover_log_cfg(void *hal);
+
+void hal_fw_en_basic_log(struct rtw_hal_com_t *hal_com,
+                        struct mac_ax_fw_log* fw_log_info);
+
+#endif /* _HAL_FW_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_general_def.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_general_def.h
new file mode 100644
index 000000000000..dbb45550aa83
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_general_def.h
@@ -0,0 +1,491 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_GENERAL_DEF_H_
+#define _HAL_GENERAL_DEF_H_
+
+#define FL_COMP_VER BIT0
+#define FL_COMP_INIT BIT1
+#define FL_COMP_TASK BIT2
+#define FL_COMP_CNS BIT3
+#define FL_COMP_H2C BIT4
+#define FL_COMP_C2H BIT5
+#define FL_COMP_TX BIT6
+#define FL_COMP_RX BIT7
+#define FL_COMP_IPSEC BIT8
+#define FL_COMP_TIMER BIT9
+#define FL_COMP_DBGPKT BIT10
+#define FL_COMP_PS BIT11
+#define FL_COMP_BB BIT16
+#define FL_COMP_TWT BIT17
+#define FL_COMP_MCC BIT20
+#define FL_COMP_SCAN BIT28
+
+#define RTW_MAC_TBTT_AGG_DEF 1
+
+/* RXBUF should larger than RXBD, or it will cause RX RDU
+   the minimum number of RXBUF - RXBD is 32 */
+#define RTW_RX_BUF_BD_MIN_DIFF 32
+
+enum rtw_chip_id {
+	CHIP_WIFI6_8852A,
+	CHIP_WIFI6_8834A,
+	CHIP_WIFI6_8852B,
+	CHIP_WIFI6_8852C,
+	CHIP_WIFI6_8852D,
+	CHIP_WIFI6_8192XB,
+	CHIP_WIFI6_8832BR,
+	CHIP_WIFI6_8852BP,
+	CHIP_WIFI7_8922A,
+	CHIP_WIFI6_8851B,
+	CHIP_WIFI6_8852BT,
+	CHIP_WIFI6_8852BPT,
+	CHIP_WIFI6_8842A,
+	CHIP_WIFI6_MAX
+};
+
+enum rtw_efuse_info {
+	/* MAC Part */
+	EFUSE_INFO_MAC_ADDR,
+	EFUSE_INFO_MAC_PID,
+	EFUSE_INFO_MAC_DID,
+	EFUSE_INFO_MAC_VID,
+	EFUSE_INFO_MAC_SVID,
+	EFUSE_INFO_MAC_SMID,
+	EFUSE_INFO_MAC_MAX,
+	/* BB Part */
+	EFUSE_INFO_BB_ANTDIV,
+	EFUSE_INFO_BB_MAX,
+	/* RF Part */
+	EFUSE_INFO_RF_PKG_TYPE,
+	EFUSE_INFO_RF_PA,
+	EFUSE_INFO_RF_VALID_PATH,
+	EFUSE_INFO_RF_RFE,
+	EFUSE_INFO_RF_TXPWR,
+	EFUSE_INFO_RF_BOARD_OPTION,
+	EFUSE_INFO_RF_CHAN_PLAN,
+	EFUSE_INFO_RF_CHAN_PLAN_6GHZ,
+	EFUSE_INFO_RF_CHAN_PLAN_FORCE_HW,
+	EFUSE_INFO_RF_COUNTRY,
+	EFUSE_INFO_RF_THERMAL,
+	EFUSE_INFO_RF_2G_CCK_A_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_CCK_A_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_CCK_A_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_CCK_A_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_CCK_A_TSSI_DE_5,
+	EFUSE_INFO_RF_2G_CCK_A_TSSI_DE_6,
+	EFUSE_INFO_RF_2G_BW40M_A_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_BW40M_A_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_BW40M_A_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_BW40M_A_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_BW40M_A_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_1,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_2,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_3,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_4,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_6,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_7,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_8,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_9,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_10,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_11,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_12,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_13,
+	EFUSE_INFO_RF_5G_BW40M_A_TSSI_DE_14,
+	EFUSE_INFO_RF_5G_BW20M1S_BW40M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_OFDM1T_BW40M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW40M2S_BW40M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW20M2S_BW20M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW40M3S_BW40M2S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW20M3S_BW20M2S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW40M4S_BW40M3S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW20M4S_BW20M3S_A_DIFF,
+	EFUSE_INFO_RF_5G_OFDM2T_OFDM1T_A_DIFF,
+	EFUSE_INFO_RF_5G_OFDM3T_OFDM2T_A_DIFF,
+	EFUSE_INFO_RF_5G_OFDM4T_OFDM3T_A_DIFF,
+	EFUSE_INFO_RF_5G_BW80M1S_BW40M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW160M1S_BW80M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW80M2S_BW80M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW160M2S_BW160M1S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW80M3S_BW80M2S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW160M3S_BW160M2S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW80M4S_BW80M3S_A_DIFF,
+	EFUSE_INFO_RF_5G_BW160M4S_BW160M3S_A_DIFF,
+	EFUSE_INFO_RF_2G_CCK_B_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_CCK_B_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_CCK_B_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_CCK_B_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_CCK_B_TSSI_DE_5,
+	EFUSE_INFO_RF_2G_CCK_B_TSSI_DE_6,
+	EFUSE_INFO_RF_2G_BW40M_B_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_BW40M_B_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_BW40M_B_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_BW40M_B_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_BW40M_B_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_1,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_2,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_3,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_4,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_6,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_7,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_8,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_9,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_10,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_11,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_12,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_13,
+	EFUSE_INFO_RF_5G_BW40M_B_TSSI_DE_14,
+	EFUSE_INFO_RF_5G_BW20M1S_BW40M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_OFDM1T_BW40M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW40M2S_BW40M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW20M2S_BW20M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW40M3S_BW40M2S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW20M3S_BW20M2S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW40M4S_BW40M3S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW20M4S_BW20M3S_B_DIFF,
+	EFUSE_INFO_RF_5G_OFDM2T_OFDM1T_B_DIFF,
+	EFUSE_INFO_RF_5G_OFDM3T_OFDM2T_B_DIFF,
+	EFUSE_INFO_RF_5G_OFDM4T_OFDM3T_B_DIFF,
+	EFUSE_INFO_RF_5G_BW80M1S_BW40M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW160M1S_BW80M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW80M2S_BW80M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW160M2S_BW160M1S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW80M3S_BW80M2S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW160M3S_BW160M2S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW80M4S_BW80M3S_B_DIFF,
+	EFUSE_INFO_RF_5G_BW160M4S_BW160M3S_B_DIFF,
+	EFUSE_INFO_RF_2G_CCK_C_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_CCK_C_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_CCK_C_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_CCK_C_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_CCK_C_TSSI_DE_5,
+	EFUSE_INFO_RF_2G_CCK_C_TSSI_DE_6,
+	EFUSE_INFO_RF_2G_BW40M_C_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_BW40M_C_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_BW40M_C_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_BW40M_C_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_BW40M_C_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_1,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_2,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_3,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_4,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_6,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_7,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_8,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_9,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_10,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_11,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_12,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_13,
+	EFUSE_INFO_RF_5G_BW40M_C_TSSI_DE_14,
+	EFUSE_INFO_RF_5G_BW20M1S_BW40M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_OFDM1T_BW40M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW40M2S_BW40M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW20M2S_BW20M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW40M3S_BW40M2S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW20M3S_BW20M2S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW40M4S_BW40M3S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW20M4S_BW20M3S_C_DIFF,
+	EFUSE_INFO_RF_5G_OFDM2T_OFDM1T_C_DIFF,
+	EFUSE_INFO_RF_5G_OFDM3T_OFDM2T_C_DIFF,
+	EFUSE_INFO_RF_5G_OFDM4T_OFDM3T_C_DIFF,
+	EFUSE_INFO_RF_5G_BW80M1S_BW40M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW160M1S_BW80M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW80M2S_BW80M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW160M2S_BW160M1S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW80M3S_BW80M2S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW160M3S_BW160M2S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW80M4S_BW80M3S_C_DIFF,
+	EFUSE_INFO_RF_5G_BW160M4S_BW160M3S_C_DIFF,
+	EFUSE_INFO_RF_2G_CCK_D_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_CCK_D_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_CCK_D_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_CCK_D_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_CCK_D_TSSI_DE_5,
+	EFUSE_INFO_RF_2G_CCK_D_TSSI_DE_6,
+	EFUSE_INFO_RF_2G_BW40M_D_TSSI_DE_1,
+	EFUSE_INFO_RF_2G_BW40M_D_TSSI_DE_2,
+	EFUSE_INFO_RF_2G_BW40M_D_TSSI_DE_3,
+	EFUSE_INFO_RF_2G_BW40M_D_TSSI_DE_4,
+	EFUSE_INFO_RF_2G_BW40M_D_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_1,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_2,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_3,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_4,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_5,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_6,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_7,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_8,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_9,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_10,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_11,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_12,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_13,
+	EFUSE_INFO_RF_5G_BW40M_D_TSSI_DE_14,
+	EFUSE_INFO_RF_5G_BW20M1S_BW40M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_OFDM1T_BW40M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW40M2S_BW40M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW20M2S_BW20M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW40M3S_BW40M2S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW20M3S_BW20M2S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW40M4S_BW40M3S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW20M4S_BW20M3S_D_DIFF,
+	EFUSE_INFO_RF_5G_OFDM2T_OFDM1T_D_DIFF,
+	EFUSE_INFO_RF_5G_OFDM3T_OFDM2T_D_DIFF,
+	EFUSE_INFO_RF_5G_OFDM4T_OFDM3T_D_DIFF,
+	EFUSE_INFO_RF_5G_BW80M1S_BW40M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW160M1S_BW80M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW80M2S_BW80M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW160M2S_BW160M1S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW80M3S_BW80M2S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW160M3S_BW160M2S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW80M4S_BW80M3S_D_DIFF,
+	EFUSE_INFO_RF_5G_BW160M4S_BW160M3S_D_DIFF,
+	EFUSE_INFO_RF_THERMAL_A,
+	EFUSE_INFO_RF_THERMAL_B,
+	EFUSE_INFO_RF_THERMAL_C,
+	EFUSE_INFO_RF_THERMAL_D,
+	EFUSE_INFO_RF_XTAL,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_1,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_2,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_3,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_4,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_5,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_6,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_7,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_8,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_9,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_10,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_11,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_12,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_13,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_14,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_15,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_16,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_17,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_18,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_19,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_20,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_21,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_22,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_23,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_24,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_25,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_26,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_27,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_28,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_29,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_30,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_31,
+	EFUSE_INFO_RF_6G_BW40M_A_TSSI_DE_32,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_1,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_2,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_3,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_4,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_5,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_6,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_7,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_8,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_9,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_10,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_11,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_12,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_13,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_14,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_15,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_16,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_17,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_18,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_19,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_20,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_21,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_22,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_23,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_24,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_25,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_26,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_27,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_28,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_29,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_30,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_31,
+	EFUSE_INFO_RF_6G_BW40M_B_TSSI_DE_32,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_1,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_2,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_3,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_4,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_5,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_6,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_7,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_8,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_9,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_10,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_11,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_12,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_13,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_14,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_15,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_16,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_17,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_18,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_19,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_20,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_21,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_22,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_23,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_24,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_25,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_26,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_27,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_28,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_29,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_30,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_31,
+	EFUSE_INFO_RF_6G_BW40M_C_TSSI_DE_32,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_1,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_2,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_3,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_4,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_5,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_6,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_7,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_8,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_9,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_10,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_11,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_12,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_13,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_14,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_15,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_16,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_17,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_18,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_19,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_20,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_21,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_22,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_23,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_24,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_25,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_26,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_27,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_28,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_29,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_30,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_31,
+	EFUSE_INFO_RF_6G_BW40M_D_TSSI_DE_32,
+	/*RX Gain K*/
+	EFUSE_INFO_RF_RX_GAIN_K_A_2G_CCK,
+	EFUSE_INFO_RF_RX_GAIN_K_A_2G_OFMD,
+	EFUSE_INFO_RF_RX_GAIN_K_A_5GL,
+	EFUSE_INFO_RF_RX_GAIN_K_A_5GM,
+	EFUSE_INFO_RF_RX_GAIN_K_A_5GH,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GL1,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GL2,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GL3,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GM1,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GM2,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GM3,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GH1,
+	EFUSE_INFO_RF_RX_GAIN_K_A_6GH2,
+	/*RF BW Power Diff*/
+	EFUSE_INFO_RF_5G_BW160M_BW40M_DIFF,
+	EFUSE_INFO_RF_6G_BW160M_BW40M_DIFF,
+	EFUSE_INFO_RF_6G_BW320M_BW160M_DIFF,
+	EFUSE_INFO_RF_MAX,
+	/* BTCOEX Part */
+	EFUSE_INFO_BTCOEX_COEX,
+	EFUSE_INFO_BTCOEX_ANT_NUM,
+	EFUSE_INFO_BTCOEX_ANT_PATH,
+	EFUSE_INFO_BTCOEX_MAX,
+};
+
+enum rtw_cv {
+	CAV,
+	CBV,
+	CCV,
+	CDV,
+	CEV,
+	CFV,
+	CGV,
+	CTV,
+	CV_MAX,
+};
+
+enum rtw_fv {
+	FTV,
+	FUV,
+	FSV,
+	FGV,
+	FV_MAX
+};
+
+enum rtw_dv_sel {
+	DAV,
+	DDV,
+};
+
+enum hal_pwr_by_rate_setting {
+	PW_BY_RATE_ON = 0,
+	PW_BY_RATE_ALL_SAME = 1
+};
+
+enum hal_pwr_limit_type {
+	PWLMT_BY_EFUSE = 0,
+	PWLMT_DISABLE = 1,
+	PWBYRATE_AND_PWLMT = 2
+};
+
+enum wl_func {
+	EFUSE_WL_FUNC_NONE = 0,
+	EFUSE_WL_FUNC_DRAGON = 0xe,
+	EFUSE_WL_FUNC_GENERAL = 0xf
+};
+enum hw_stype{
+	EFUSE_HW_STYPE_NONE = 0x0,
+	EFUSE_HW_STYPE_GENERAL = 0xf
+};
+
+enum n_bb_path {
+	BB_0PATH  = 0,
+	BB_1PATH  = 1,
+	BB_2PATH  = 2,
+	BB_3PATH  = 3,
+};
+
+enum n_rf_entity {
+	RF_1ENTITY = 1,
+	RF_2ENTITY = 2
+};
+
+#define BB0_N_PATH_SFT 0
+#define BB1_N_PATH_SFT 4
+#define RF_N_ENTITY_SFT 12
+enum mlo_dbcc_mode_type { /* for HALBB/HALRF reference */
+	MLO_DBCC_NOT_SUPPORT = 0,
+	MLO_0_PLUS_2_1RF = (RF_1ENTITY << RF_N_ENTITY_SFT | BB_2PATH << BB1_N_PATH_SFT | BB_0PATH),
+	MLO_0_PLUS_2_2RF = (RF_2ENTITY << RF_N_ENTITY_SFT | BB_2PATH << BB1_N_PATH_SFT | BB_0PATH),
+	MLO_1_PLUS_1_1RF = (RF_1ENTITY << RF_N_ENTITY_SFT | BB_1PATH << BB1_N_PATH_SFT | BB_1PATH),
+	MLO_1_PLUS_1_2RF = (RF_2ENTITY << RF_N_ENTITY_SFT | BB_1PATH << BB1_N_PATH_SFT | BB_1PATH),
+	MLO_2_PLUS_0_1RF = (RF_1ENTITY << RF_N_ENTITY_SFT | BB_0PATH << BB1_N_PATH_SFT | BB_2PATH),
+	MLO_2_PLUS_0_2RF = (RF_2ENTITY << RF_N_ENTITY_SFT | BB_0PATH << BB1_N_PATH_SFT | BB_2PATH),
+	MLO_2_PLUS_2_2RF = (RF_2ENTITY << RF_N_ENTITY_SFT | BB_2PATH << BB1_N_PATH_SFT | BB_2PATH),
+	DBCC_LEGACY = 0xffffffff /* ex:8852C DBCC (2PHY in 1BB) x 1 */
+};
+
+#endif /* _HAL_GENERAL_DEF_H_*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_headers.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_headers.h
new file mode 100644
index 000000000000..222b0db7a690
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_headers.h
@@ -0,0 +1,107 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_HEADERS_H_
+#define _HAL_HEADERS_H_
+#include "../rtw_general_def.h"
+#include "../phl_list.h"
+#include "../phl_status.h"
+#include "../pltfm_ops.h"
+#include "../phl_config.h"
+#include "../phl_types.h"
+#include "../phl_util.h"
+#include "mac/mac_exp_def.h"
+#include "../phl_def.h"
+#include "../phl_debug.h"
+#include "../phl_trx_def.h"
+#include "../phl_wow_def.h"
+#include "../phl_p2pps_def.h"
+#include "../phl_btc_def.h"
+#include "../phl_test_def.h"
+#include "../phl_ser_def.h"
+#include "../phl_regulation_def.h"
+#include "../phl_led_def.h"
+#include "../phl_acs_def.h"
+#include "../phl_ext_tx_pwr_lmt_def.h"
+#include "../custom/phl_custom_def.h"
+#include "../custom/phl_custom_api.h"
+#include "../phl_version.h"
+#ifdef CONFIG_PCI_HCI
+#include "../hci/phl_trx_def_pcie.h"
+#endif
+#ifdef CONFIG_USB_HCI
+#include "../hci/phl_trx_def_usb.h"
+#endif
+#ifdef CONFIG_SDIO_HCI
+#include "../hci/phl_trx_def_sdio.h"
+#endif
+
+#include "../phl_diagnose.h"
+#include "hal_general_def.h"
+#include "hal_config.h"
+#include "phy/bb/halbb_bb_wrapper_outsrc.h"
+#include "hal_def.h"
+#include "phy/bb/halbb_outsrc_def.h"
+#include "phy/rf/halrf_outsrc_def.h"
+#include "mac/mac_outsrc_def.h"
+#include "../phl_api_drv.h"
+#include "hal_struct.h"
+#include "hal_io.h"
+
+#ifdef CONFIG_PCI_HCI
+#include "hal_pci.h"
+#endif
+#ifdef CONFIG_USB_HCI
+#include "hal_usb.h"
+#endif
+
+#include "hal_api_mac.h"
+#include "hal_api_bb.h"
+#include "hal_api_rf.h"
+#include "hal_api_btc.h"
+#include "hal_api_efuse.h"
+#include "hal_str_proc.h"
+
+#include "hal_tx.h"
+#include "hal_rx.h"
+#include "hal_sta.h"
+#include "hal_cap.h"
+#include "hal_chan.h"
+#ifdef CONFIG_PHL_CHANNEL_INFO
+#include "hal_chan_info.h"
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+#include "hal_fw.h"
+#include "hal_wow.h"
+
+#include "hal_csi_buffer.h"
+#include "hal_beamform.h"
+#include "hal_sound.h"
+#include "hal_ser.h"
+#include "hal_c2h.h"
+#include "hal_acs.h"
+#include "hal_mcc_def.h"
+#include "hal_ld_file.h"
+#include "hal_notify.h"
+#include "hal_ps.h"
+#include "hal_custom.h"
+#include "hal_txpwr.h"
+#include "hal_com_i.h"
+#include "hal_dfs.h"
+#ifdef CONFIG_DBCC_SUPPORT
+#include "hal_dbcc.h"
+#endif
+
+#define hal_to_drvpriv(_halinfo) (_halinfo->hal_com->drv_priv)
+
+#endif /*_HAL_HEADERS_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_headers_le.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_headers_le.h
new file mode 100644
index 000000000000..45ad924d34ce
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_headers_le.h
@@ -0,0 +1,352 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_HEADERS_LE_H_
+#define _HAL_HEADERS_LE_H_
+
+/* Common definition from PHL */
+#include "../rtw_general_def.h"
+#include "../phl_list.h"
+#include "../phl_status.h"
+#include "../pltfm_ops.h"
+#include "../phl_config.h"
+#include "../phl_types.h"
+#include "../phl_regulation_def.h"
+#include "../phl_txpwr.h"
+#include "../phl_version.h"
+/* Exported structure/definition from PHL */
+#include "../phl_util.h"
+#include "mac/mac_exp_def.h"
+#include "../phl_def.h"
+#include "../phl_trx_def.h"
+#include "../phl_wow_def.h"
+#include "../phl_btc_def.h"
+#include "../phl_test_def.h"
+#include "../phl_debug.h"
+#include "../phl_ext_tx_pwr_lmt_def.h"
+#ifdef CONFIG_PHL_CHANNEL_INFO
+#include "../phl_chan_info_def.h"
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+#ifdef CONFIG_PCI_HCI
+#include "../hci/phl_trx_def_pcie.h"
+#endif
+#ifdef CONFIG_USB_HCI
+#include "../hci/phl_trx_def_usb.h"
+#endif
+#ifdef CONFIG_SDIO_HCI
+#include "../hci/phl_trx_def_sdio.h"
+#endif
+
+/* Common definition from HAL*/
+#include "hal_general_def.h"
+/* Exported structure/definition from HAL */
+#include "hal_config.h"
+#include "phy/bb/halbb_bb_wrapper_outsrc.h"
+#include "hal_def.h"
+#include "phy/bb/halbb_outsrc_def.h"
+#include "phy/rf/halrf_outsrc_def.h"
+#include "mac/mac_outsrc_def.h"
+
+/*
+Exported hal API  from HAL
+Prevent HAL mac/bb/ rf/ btc cross referencing
+*/
+#include "hal_api_drv.h"
+
+#include "hal_io.h" /*exported hal_read, hal_write API from HAL */
+#include "hal_fw.h"
+
+
+#if 1
+static inline void hal_mdelay(struct rtw_hal_com_t *h, int ms)
+{
+	_os_delay_ms(halcom_to_drvpriv(h), ms);
+}
+static inline void hal_udelay(struct rtw_hal_com_t *h, int us)
+{
+	_os_delay_us(halcom_to_drvpriv(h), us);
+}
+
+static inline void hal_msleep(struct rtw_hal_com_t *h, int ms)
+{
+	_os_sleep_ms(halcom_to_drvpriv(h),ms);
+}
+
+static inline void hal_usleep(struct rtw_hal_com_t *h, int us)
+{
+	_os_sleep_us(halcom_to_drvpriv(h), us);
+}
+
+#ifdef DBG_HAL_MEM_MOINTOR
+static inline void *hal_mem_alloc(struct rtw_hal_com_t *h, u32 buf_sz)
+{
+	_os_atomic_sub(halcom_to_drvpriv(h), &(h->hal_mem), buf_sz);
+	return _os_mem_alloc(halcom_to_drvpriv(h), buf_sz);
+}
+
+static inline void hal_mem_free(struct rtw_hal_com_t *h, void *buf, u32 buf_sz)
+{
+	_os_atomic_add_return(halcom_to_drvpriv(h), &(h->hal_mem), buf_sz);
+	_os_mem_free(halcom_to_drvpriv(h), buf, buf_sz);
+}
+#else
+static inline void *hal_mem_alloc(struct rtw_hal_com_t *h, u32 buf_sz)
+{
+	return _os_mem_alloc(halcom_to_drvpriv(h), buf_sz);
+}
+
+static inline void hal_mem_free(struct rtw_hal_com_t *h, void *buf, u32 buf_sz)
+{
+	_os_mem_free(halcom_to_drvpriv(h), buf, buf_sz);
+}
+#endif /*DBG_HAL_MEM_MOINTOR*/
+static inline void hal_mem_set(struct rtw_hal_com_t *h, void *buf, s8 value, u32 size)
+{
+	_os_mem_set(halcom_to_drvpriv(h), buf, value, size);
+}
+
+static inline void hal_mem_cpy(struct rtw_hal_com_t *h, void *dest,
+						const void *src, u32 size)
+{
+	_os_mem_cpy(halcom_to_drvpriv(h), dest, src, size);
+}
+
+static inline int hal_mem_cmp(struct rtw_hal_com_t *h, const void *dest,
+						const void *src, u32 size)
+{
+	return _os_mem_cmp(halcom_to_drvpriv(h), dest, src, size);
+}
+static inline void hal_mutex_init(struct rtw_hal_com_t *h, _os_mutex *mutex)
+{
+	_os_mutex_init(halcom_to_drvpriv(h), mutex);
+}
+static inline void hal_mutex_deinit(struct rtw_hal_com_t *h, _os_mutex *mutex)
+{
+	_os_mutex_deinit(halcom_to_drvpriv(h), mutex);
+}
+static inline void hal_mutex_lock(struct rtw_hal_com_t *h, _os_mutex *mutex)
+{
+	_os_mutex_lock(halcom_to_drvpriv(h), mutex);
+}
+static inline void hal_mutex_unlock(struct rtw_hal_com_t *h, _os_mutex *mutex)
+{
+	_os_mutex_unlock(halcom_to_drvpriv(h), mutex);
+}
+static inline void hal_init_timer(struct rtw_hal_com_t *h, _os_timer *timer,
+			void (*call_back_func)(void* context),
+			void *context, const char *sz_id)
+{
+	_os_init_timer(halcom_to_drvpriv(h), timer, call_back_func, context, sz_id);
+}
+
+static inline void hal_set_timer(struct rtw_hal_com_t *h, _os_timer *timer,
+								u32 ms_delay)
+{
+	_os_set_timer(halcom_to_drvpriv(h), timer, ms_delay);
+}
+static inline void hal_cancel_timer(struct rtw_hal_com_t *h, _os_timer *timer)
+{
+	_os_cancel_timer(halcom_to_drvpriv(h), timer);
+}
+static inline void hal_release_timer(struct rtw_hal_com_t *h, _os_timer *timer)
+{
+	_os_release_timer(halcom_to_drvpriv(h), timer);
+}
+
+/*IO OPS*/
+#ifdef CONFIG_PCI_HCI
+static inline u8 hal_pci_read8(struct rtw_hal_com_t *h,u32 addr)
+{
+	return _os_read8_pcie(halcom_to_drvpriv(h), addr);
+}
+static inline u16 hal_pci_read16(struct rtw_hal_com_t *h,u32 addr)
+{
+	return _os_read16_pcie(halcom_to_drvpriv(h), addr);
+}
+static inline u32 hal_pci_read32(struct rtw_hal_com_t *h,u32 addr)
+{
+	return _os_read32_pcie(halcom_to_drvpriv(h), addr);
+}
+
+static inline int hal_pci_write8(struct rtw_hal_com_t *h, u32 addr, u8 val)
+{
+	return _os_write8_pcie(halcom_to_drvpriv(h), addr, val);
+}
+static inline int hal_pci_write16(struct rtw_hal_com_t *h, u32 addr, u16 val)
+{
+	return _os_write16_pcie(halcom_to_drvpriv(h), addr, val);
+}
+static inline int hal_pci_write32(struct rtw_hal_com_t *h, u32 addr, u32 val)
+{
+	return _os_write32_pcie(halcom_to_drvpriv(h), addr, val);
+}
+
+#endif
+#ifdef CONFIG_USB_HCI
+static inline int hal_usb_vendor_req(struct rtw_hal_com_t *h,
+			u8 request, u16 value, u16 index,
+			void *pdata, u16 len, u8 requesttype)
+{
+	return _os_usbctrl_vendorreq(halcom_to_drvpriv(h), request, value,
+					index, pdata, len, requesttype);
+}
+#endif /*CONFIG_USB_HCI*/
+
+
+#ifdef CONFIG_SDIO_HCI
+static inline u8 hal_sdio_cmd52_r8(struct rtw_hal_com_t *h, u32 offset)
+{
+	return _os_sdio_cmd52_r8(halcom_to_drvpriv(h), offset);
+}
+static inline u8 hal_sdio_cmd53_r8(struct rtw_hal_com_t *h, u32 offset)
+{
+	return _os_sdio_cmd53_r8(halcom_to_drvpriv(h), offset);
+}
+
+static inline u16 hal_sdio_cmd53_r16(struct rtw_hal_com_t *h, u32 offset)
+{
+	return _os_sdio_cmd53_r16(halcom_to_drvpriv(h), offset);
+}
+
+static inline u32 hal_sdio_cmd53_r32(struct rtw_hal_com_t *h, u32 offset)
+{
+	return _os_sdio_cmd53_r32(halcom_to_drvpriv(h), offset);
+}
+
+static inline u8 hal_sdio_cmd53_rn(struct rtw_hal_com_t *h,
+					u32 offset, u32 size, u8 *data)
+{
+	return _os_sdio_cmd53_rn(halcom_to_drvpriv(h), offset, size, data);
+}
+
+static inline u8 hal_sdio_cmd53_r(struct rtw_hal_com_t *h,
+					u32 offset, u32 size, u8 *val)
+{
+	size = _ALIGN(size, 4);
+	if (size > h->block_sz)
+		size = _ALIGN(size, h->block_sz);
+	return _os_sdio_cmd53_r(halcom_to_drvpriv(h), offset, size, val);
+}
+
+static inline void hal_sdio_cmd52_w8(struct rtw_hal_com_t *h,
+					u32 offset, u8 val)
+{
+	_os_sdio_cmd52_w8(halcom_to_drvpriv(h), offset, val);
+}
+
+static inline void hal_sdio_cmd53_w8(struct rtw_hal_com_t *h,
+					u32 offset, u8 val)
+{
+	_os_sdio_cmd53_w8(halcom_to_drvpriv(h), offset, val);
+}
+
+static inline void hal_sdio_cmd53_w16(struct rtw_hal_com_t *h,
+					u32 offset, u16 val)
+{
+	_os_sdio_cmd53_w16(halcom_to_drvpriv(h), offset, val);
+}
+
+static inline void hal_sdio_cmd53_w32(struct rtw_hal_com_t *h,
+					u32 offset, u32 val)
+{
+	_os_sdio_cmd53_w32(halcom_to_drvpriv(h), offset, val);
+}
+
+static inline void hal_sdio_cmd53_wn(struct rtw_hal_com_t *h,
+					u32 offset, u32 size, u8 *val)
+{
+	_os_sdio_cmd53_wn(halcom_to_drvpriv(h), offset, size, val);
+}
+
+static inline void hal_sdio_cmd53_w(struct rtw_hal_com_t *h,
+					u32 offset, u32 size, u8 *val)
+{
+	size = _ALIGN(size, 4);
+	if (size > h->block_sz)
+		size = _ALIGN(size, h->block_sz);
+
+	_os_sdio_cmd53_w(halcom_to_drvpriv(h), offset, size, val);
+}
+
+static inline u8 hal_sdio_f0_read(struct rtw_hal_com_t *h,
+					u32 addr, void *buf, u32 len)
+{
+
+	return _os_sdio_f0_read(halcom_to_drvpriv(h), addr, buf, len);
+}
+
+static inline u8 hal_sdio_read_cia_r8(struct rtw_hal_com_t *h, u32 addr)
+{
+	return _os_sdio_read_cia_r8(halcom_to_drvpriv(h), addr);
+}
+
+#endif /*CONFIG_SDIO_HCI*/
+
+#else
+/*please refer to hal_pltfm_ops.h*/
+#define hal_mdelay(h, ms)	_os_delay_ms(halcom_to_drvpriv(h), ms)
+#define hal_udelay(h, us)	_os_delay_us(halcom_to_drvpriv(h), us)
+#define hal_msleep(h, ms)	_os_sleep_ms(halcom_to_drvpriv(h),ms)
+#define hal_usleep(h, us)	_os_sleep_us(halcom_to_drvpriv(h), us)
+
+#define hal_mem_alloc(h, buf_sz) _os_mem_alloc(halcom_to_drvpriv(h), buf_sz)
+#define hal_mem_free(h, buf, buf_sz)	_os_mem_free(halcom_to_drvpriv(h), buf, buf_sz)
+#define hal_mem_set(h, buf, value, size)	_os_mem_set(halcom_to_drvpriv(h), buf, value, size)
+#define hal_mem_cpy(h, dest, src, size)	_os_mem_cpy(halcom_to_drvpriv(h), dest, src, size)
+#define hal_mem_cmp(h, dest, src, size)	_os_mem_cmp(halcom_to_drvpriv(h), dest, src, size)
+
+#define hal_mutex_init(h, mutex)	_os_mutex_init(halcom_to_drvpriv(h), mutex)
+#define hal_mutex_deinit(h, hmutex)	_os_mutex_deinit(halcom_to_drvpriv(h), mutex)
+#define hal_mutex_lock(h, mutex)	_os_mutex_lock(halcom_to_drvpriv(h), mutex)
+#define hal_mutex_unlock(h, mutex)	_os_mutex_unlock(halcom_to_drvpriv(h), mutex)
+
+#define hal_init_timer(h, timer, call_back_func, context, sz_id) \
+	_os_init_timer(halcom_to_drvpriv(h), timer, call_back_func, context, sz_id)
+#define hal_set_timer(h, timer, ms_delay)	_os_set_timer(halcom_to_drvpriv(h), timer, ms_delay)
+#define hal_cancel_timer(h, timer)		_os_cancel_timer(halcom_to_drvpriv(h), timer)
+#define hal_release_timer(h, timer)		_os_release_timer(halcom_to_drvpriv(h), timer)
+
+/*IO OPS*/
+#ifdef CONFIG_PCI_HCI
+#define hal_pci_read8(h, addr)		_os_read8_pcie(halcom_to_drvpriv(h), addr)
+#define hal_pci_read16(h, addr)		_os_read16_pcie(halcom_to_drvpriv(h), addr)
+#define hal_pci_read32(h, addr)		_os_read32_pcie(halcom_to_drvpriv(h), addr)
+#define hal_pci_write8(h, addr, val)	_os_write8_pcie(halcom_to_drvpriv(h), addr, val)
+#define hal_pci_write16(h, addr, val)	_os_write16_pcie(halcom_to_drvpriv(h), addr, val)
+#define hal_pci_write32(h, addr, val)	_os_write32_pcie(halcom_to_drvpriv(h), addr, val)
+#endif
+
+#ifdef CONFIG_USB_HCI
+#define hal_usb_vendor_req(h, request, value, index, pdata, len, requesttype) \
+	_os_usbctrl_vendorreq(halcom_to_drvpriv(h), request, value, index, pdata, len, requesttype)
+#endif /*CONFIG_USB_HCI*/
+
+#ifdef CONFIG_SDIO_HCI
+#define hal_sdio_cmd52_r8(h, offset)	_os_sdio_cmd52_r8(halcom_to_drvpriv(h), offset)
+#define hal_sdio_cmd53_r8(h, offset)	_os_sdio_cmd53_r8(halcom_to_drvpriv(h), offset)
+#define hal_sdio_cmd53_r16(h, offset)	_os_sdio_cmd53_r16(halcom_to_drvpriv(h), offset)
+#define hal_sdio_cmd53_r32(h, offset)	_os_sdio_cmd53_r32(halcom_to_drvpriv(h), offset)
+#define hal_sdio_cmd53_rn(h, offset, size, data) _os_sdio_cmd53_rn(halcom_to_drvpriv(h), offset, size, data)
+#define hal_sdio_cmd52_w8(h, offset, val)	_os_sdio_cmd52_w8(halcom_to_drvpriv(h), offset, val)
+#define hal_sdio_cmd53_w8(h, offset, val)	_os_sdio_cmd53_w8(halcom_to_drvpriv(h), offset, val)
+#define hal_sdio_cmd53_w16(h, offset, val)	_os_sdio_cmd53_w16(halcom_to_drvpriv(h), offset, val)
+#define hal_sdio_cmd53_w32(h, offset, val)	_os_sdio_cmd53_w32(halcom_to_drvpriv(h), offset, val)
+#define hal_sdio_f0_read(h, addr, buf, len)	_os_sdio_f0_read(halcom_to_drvpriv(h), addr, buf, len)
+#define hal_sdio_read_cia_r8(h, addr)		_os_sdio_read_cia_r8(halcom_to_drvpriv(h), addr)
+#endif /*CONFIG_SDIO_HCI*/
+#endif
+
+
+#endif /*_HAL_HEADERS_LE_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_init.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_init.c
new file mode 100644
index 000000000000..499decf76db5
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_init.c
@@ -0,0 +1,1826 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2023 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_INIT_C_
+#include "hal_headers.h"
+
+#if defined(CONFIG_PCI_HCI)
+#include "hal_pci.h"
+#elif defined(CONFIG_USB_HCI)
+#include "hal_usb.h"
+#elif defined(CONFIG_SDIO_HCI)
+#include "hal_sdio.h"
+#endif
+
+
+/******************* IO  APIs *******************/
+u8 rtw_hal_read8(void *h, u32 addr)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return hal_read8(hal_com, addr);
+}
+u16 rtw_hal_read16(void *h, u32 addr)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return hal_read16(hal_com, addr);
+}
+u32 rtw_hal_read32(void *h, u32 addr)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return hal_read32(hal_com, addr);
+}
+void rtw_hal_write8(void *h, u32 addr, u8 val)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	hal_write8(hal_com, addr, val);
+}
+void rtw_hal_write16(void *h, u32 addr, u16 val)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	hal_write16(hal_com, addr, val);
+}
+void rtw_hal_write32(void *h, u32 addr, u32 val)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	hal_write32(hal_com, addr, val);
+}
+
+u32 rtw_hal_read_macreg(void *h, u32 offset, u32 bit_mask)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	return hal_ops->read_macreg(hal, offset, bit_mask);
+}
+void rtw_hal_write_macreg(void *h,
+			u32 offset, u32 bit_mask, u32 data)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	hal_ops->write_macreg(hal, offset, bit_mask, data);
+}
+u32 rtw_hal_read_bbreg(void *h, u32 offset, u32 bit_mask)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	return hal_ops->read_bbreg(hal, offset, bit_mask);
+}
+void rtw_hal_write_bbreg(void *h,
+			u32 offset, u32 bit_mask, u32 data)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	hal_ops->write_bbreg(hal, offset, bit_mask, data);
+}
+
+u32 rtw_hal_read_rfreg(void *h,
+			enum rf_path path, u32 offset, u32 bit_mask)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	return hal_ops->read_rfreg(hal, path, offset, bit_mask);
+}
+
+void rtw_hal_write_rfreg(void *h,
+			enum rf_path path, u32 offset, u32 bit_mask, u32 data)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	hal_ops->write_rfreg(hal, path, offset, bit_mask, data);
+}
+
+
+#ifdef PHL_PLATFORM_LINUX
+void rtw_hal_mac_reg_dump(void *sel, void *h)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	rtw_hal_notification(hal_info, MSG_EVT_DBG_FULL_REG_DUMP, HW_BAND_MAX);
+}
+
+
+void rtw_hal_bb_reg_dump(void *sel, void *h)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal = hal_info->hal_com;
+	int i, j = 1;
+
+	RTW_PRINT_SEL(sel, "======= BB REG =======\n");
+	for (i = 0x800; i < 0x1000; i += 4) {
+		if (j % 4 == 1)
+			RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", hal_read32(hal, i));
+		if ((j++) % 4 == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+
+#if 0
+	for (i = 0x1800; i < 0x2000; i += 4) {
+		if (j % 4 == 1)
+			RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", hal_read32(hal, i));
+		if ((j++) % 4 == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+#endif
+
+#if 0
+	for (i = 0x2c00; i < 0x2c60; i += 4) {
+		if (j % 4 == 1)
+			RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", rtw_read32(adapter, i));
+		if ((j++) % 4 == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+
+	for (i = 0x2d00; i < 0x2df0; i += 4) {
+		if (j % 4 == 1)
+			RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", rtw_read32(adapter, i));
+		if ((j++) % 4 == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+
+	for (i = 0x4000; i < 0x4060; i += 4) {
+		if (j % 4 == 1)
+			RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", rtw_read32(adapter, i));
+		if ((j++) % 4 == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+
+	for (i = 0x4100; i < 0x4200; i += 4) {
+		if (j % 4 == 1)
+			RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", rtw_read32(adapter, i));
+		if ((j++) % 4 == 0)
+			_RTW_PRINT_SEL(sel, "\n");
+	}
+
+#endif
+
+}
+
+void rtw_hal_bb_reg_dump_ex(void *sel, void *h)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal = hal_info->hal_com;
+	int i;
+
+	RTW_PRINT_SEL(sel, "======= BB REG =======\n");
+	for (i = 0x800; i < 0x1000; i += 4) {
+		RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", hal_read32(hal, i));
+		_RTW_PRINT_SEL(sel, "\n");
+	}
+
+#if 0
+	for (i = 0x1800; i < 0x2000; i += 4) {
+		RTW_PRINT_SEL(sel, "0x%04x", i);
+		_RTW_PRINT_SEL(sel, " 0x%08x ", hal_read32(hal, i));
+		_RTW_PRINT_SEL(sel, "\n");
+	}
+#endif
+}
+
+
+void rtw_hal_rf_reg_dump(void *sel, void *h)
+{
+#ifdef RTW_WKARD_RF_CR_DUMP
+	int i, j = 1, path;
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct rtw_hal_com_t *hal = hal_info->hal_com;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	u32 value;
+	u8 path_nums;
+
+	path_nums = (phl_com->phy_cap[0].tx_path_num > phl_com->phy_cap[0].rx_path_num) ?
+	             phl_com->phy_cap[0].tx_path_num : phl_com->phy_cap[0].rx_path_num;
+
+	RTW_PRINT_SEL(sel, "======= RF REG =======\n");
+	for (path = 0; path < path_nums; path++) {
+		RTW_PRINT_SEL(sel, "RF_Path(%x)\n", path);
+		for (i = 0; i < 0x100; i++) {
+			value = rtw_hal_read_rfreg(hal_info, path, i, 0xfffff);
+			if (j % 4 == 1)
+				RTW_PRINT_SEL(sel, "0x%02x ", i);
+			_RTW_PRINT_SEL(sel, " 0x%08x ", value);
+			if ((j++) % 4 == 0)
+				_RTW_PRINT_SEL(sel, "\n");
+		}
+		j = 1;
+		#ifdef CONFIG_RTL8852B
+		for (i = 0x10000; i < 0x10100; i++) {
+			value = rtw_hal_read_rfreg(hal_info, path, i, 0xfffff);
+			if (j % 4 == 1)
+				RTW_PRINT_SEL(sel, "0x%02x ", i);
+			_RTW_PRINT_SEL(sel, " 0x%08x ", value);
+			if ((j++) % 4 == 0)
+				_RTW_PRINT_SEL(sel, "\n");
+		}
+		_RTW_PRINT_SEL(sel, "\n");
+		#endif
+	}
+#endif/*RTW_WKARD_RF_CR_DUMP*/
+
+}
+#endif
+
+/**
+ * rtw_hal_get_sec_cam() - get the security cam raw data from HW
+ * @h:		struct hal_info_t *
+ * @num:		How many cam you wnat to dump from the first one.
+ * @buf:		ptr to buffer which store the content from HW.
+ *			If buf is NULL, use console as debug path.
+ * @size		Size of allocated memroy for @buf.
+ *			The size should be @num * size of security cam offset(0x20).
+ *
+ * Return rtw_hal_mac_get_addr_cam's return value in enum rtw_hal_status type.
+ */
+enum rtw_hal_status
+rtw_hal_get_sec_cam(void *h, u16 num, u8 *buf, u16 size)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = rtw_hal_mac_get_sec_cam(hal_info, num, buf, size);
+
+	return ret;
+}
+
+/**
+ * rtw_hal_get_addr_cam() - get the address cam raw data from HW
+ * @h:		struct hal_info_t *
+ * @num:		How many cam you wnat to dump from the first one.
+ * @buf:		ptr to buffer which store the content from HW.
+ *			If buf is NULL, use console as debug path.
+ * @size		Size of allocated memroy for @buf.
+ *			The size should be @num * size of Addr cam offset(0x40).
+ *
+ * Return rtw_hal_mac_get_addr_cam's return value in enum rtw_hal_status type.
+ */
+enum rtw_hal_status
+rtw_hal_get_addr_cam(void *h, u16 num, u8 *buf, u16 size)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	ret = rtw_hal_mac_get_addr_cam(hal_info, num, buf, size);
+
+	return ret;
+}
+
+void rtw_hal_init_int_default_value(struct rtw_phl_com_t *phl_com, void *h,
+					enum rtw_hal_int_set_opt opt)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->init_int_default_value)
+		hal_ops->init_int_default_value(hal, opt);
+}
+
+void rtw_hal_enable_interrupt(struct rtw_phl_com_t *phl_com, void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->enable_interrupt)
+		hal_ops->enable_interrupt(hal);
+}
+
+void rtw_hal_disable_interrupt_isr(struct rtw_phl_com_t *phl_com, void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->disable_interrupt_isr)
+		hal_ops->disable_interrupt_isr(hal);
+}
+
+void rtw_hal_disable_interrupt(struct rtw_phl_com_t *phl_com, void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->disable_interrupt)
+		hal_ops->disable_interrupt(hal);
+}
+
+void rtw_hal_config_interrupt(void *h, enum rtw_phl_config_int int_mode)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->config_interrupt)
+		hal_ops->config_interrupt(hal, int_mode);
+}
+
+bool rtw_hal_recognize_interrupt(void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->recognize_interrupt)
+		return hal_ops->recognize_interrupt(hal);
+	else
+		return false;
+}
+
+bool rtw_hal_recognize_halt_c2h_interrupt(void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->recognize_halt_c2h_interrupt)
+		return hal_ops->recognize_halt_c2h_interrupt(hal);
+	else
+		return false;
+}
+
+void rtw_hal_clear_interrupt(void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->clear_interrupt)
+		hal_ops->clear_interrupt(hal);
+}
+
+u32 rtw_hal_interrupt_handler(void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+	u32 ret = 0;
+
+
+	if (hal_ops->interrupt_handler)
+		ret = hal_ops->interrupt_handler(hal);
+	else
+		PHL_DBG("hal_ops->interrupt_handler is NULL\n");
+
+	PHL_DBG("hal_ops->interrupt_handler ret=0x%x\n", ret);
+
+
+	return ret;
+}
+
+void rtw_hal_restore_interrupt(struct rtw_phl_com_t *phl_com, void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->restore_interrupt)
+		hal_ops->restore_interrupt(hal);
+}
+
+void rtw_hal_restore_rx_interrupt(void *h)
+{
+	struct hal_info_t *hal = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal);
+
+	if (hal_ops->restore_rx_interrupt)
+		hal_ops->restore_rx_interrupt(hal);
+	else
+		PHL_DBG("hal_ops->restore_rx_interrupt is NULL\n");
+}
+
+#ifdef PHL_RXSC_ISR
+u16 rtw_hal_rpq_isr_check(void *h, u8 dma_ch)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)h;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+	u32 ret = RTW_HAL_STATUS_FAILURE;
+
+	if (hal_ops->check_rpq_isr)
+		ret = hal_ops->check_rpq_isr(dma_ch, hal_info->hal_com->rx_int_array);
+
+	if(ret == RTW_HAL_STATUS_SUCCESS)
+		return false;
+	else
+		return true;
+}
+#endif
+
+static enum rtw_hal_status hal_ops_check(struct hal_info_t *hal)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_ops_t *ops = hal_get_ops(hal);
+	struct hal_trx_ops *trx_ops = hal_get_trx_ops(hal);
+
+	/***hal_ops initialize section ***/
+	if (!ops->init_hal_spec) {
+		hal_error_msg("init_hal_spec");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!ops->hal_get_efuse) {
+		hal_error_msg("hal_get_efuse");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!ops->read_chip_version) {
+		hal_error_msg("read_chip_version");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!ops->hal_fast_start) {
+		hal_error_msg("hal_fast_start");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!ops->hal_fast_stop) {
+		hal_error_msg("hal_fast_stop");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!ops->hal_init) {
+		hal_error_msg("hal_init");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->hal_deinit) {
+		hal_error_msg("hal_deinit");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!ops->hal_start) {
+		hal_error_msg("hal_start");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->hal_stop) {
+		hal_error_msg("hal_stop");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#ifdef CONFIG_WOWLAN
+	if (!ops->hal_wow_init) {
+		hal_error_msg("hal_wow_init");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->hal_wow_deinit) {
+		hal_error_msg("hal_wow_deinit");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#endif /* CONFIG_WOWLAN */
+	if (!ops->hal_mp_init) {
+		hal_error_msg("hal_mp_init");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->hal_mp_deinit) {
+		hal_error_msg("hal_mp_deinit");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!ops->hal_cfg_fw) {
+		hal_error_msg("hal_cfg_fw");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->init_default_value) {
+		hal_error_msg("init_default_value");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->init_int_default_value) {
+		hal_error_msg("init_int_default_value");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->hal_hci_configure) {
+		hal_error_msg("hal_hci_configure");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->read_macreg) {
+		hal_error_msg("read_macreg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->write_macreg) {
+		hal_error_msg("write_macreg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->read_bbreg) {
+		hal_error_msg("read_bbreg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->write_bbreg) {
+		hal_error_msg("write_bbreg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->read_rfreg) {
+		hal_error_msg("read_rfreg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->write_rfreg) {
+		hal_error_msg("write_rfreg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+#if defined(CONFIG_PCI_HCI)
+	if (!ops->disable_interrupt_isr) {
+		hal_error_msg("disable_interrupt_isr");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->hal_set_pcicfg) {
+		hal_error_msg("hal_set_pcicfg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#endif
+
+#if defined(CONFIG_PCI_HCI) || defined(CONFIG_SDIO_HCI)
+	if (!ops->enable_interrupt) {
+		hal_error_msg("enable_interrupt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->disable_interrupt) {
+		hal_error_msg("disable_interrupt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#ifdef CONFIG_SDIO_HCI
+	if (!ops->config_interrupt) {
+		hal_error_msg("config_interrupt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->recognize_halt_c2h_interrupt) {
+		hal_error_msg("recognize_halt_c2h_interrupt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#endif
+	if (!ops->recognize_interrupt) {
+		hal_error_msg("recognize_interrupt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->clear_interrupt) {
+		hal_error_msg("clear_interrupt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->interrupt_handler) {
+		hal_error_msg("interrupt_handler");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!ops->restore_interrupt) {
+		hal_error_msg("restore_interrupt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#ifdef PHL_RXSC_ISR
+	if (!ops->check_rpq_isr) {
+		hal_error_msg("check_rpq_isr");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#endif
+#endif /*defined(CONFIG_PCI_HCI) || defined(CONFIG_SDIO_HCI)*/
+
+
+	/***hal_trx_ops section ***/
+	if (!trx_ops->init) {
+		hal_error_msg("trx init");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->deinit) {
+		hal_error_msg("trx deinit");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->map_hw_tx_chnl) {
+		hal_error_msg("trx map_hw_tx_chnl");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->get_fwcmd_queue_idx) {
+		hal_error_msg("trx get_fwcmd_queue_idx");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#ifdef CONFIG_PCI_HCI
+	if (!trx_ops->query_tx_res) {
+		hal_error_msg("trx query_tx_res");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->query_rx_res) {
+		hal_error_msg("trx query_rx_res");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->get_rxbd_num) {
+		hal_error_msg("trx get_rxbd_num");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->get_rxbuf_num) {
+		hal_error_msg("trx get_rxbuf_num");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->qsel_to_tid) {
+		hal_error_msg("trx qsel_to_tid");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->query_txch_hwband) {
+		hal_error_msg("trx query_txch_hwband");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->query_txch_map) {
+		hal_error_msg("trx query_txch_map");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->query_txch_num) {
+		hal_error_msg("trx query_txch_num");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (!trx_ops->query_rxch_num) {
+		hal_error_msg("trx query_rxch_num");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->update_wd) {
+		hal_error_msg("trx update_wd");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->update_txbd) {
+		hal_error_msg("trx update_txbd");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->tx_start) {
+		hal_error_msg("trx tx_start");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->check_rxrdy) {
+		hal_error_msg("trx check_rxrdy");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->handle_rxbd_info) {
+		hal_error_msg("trx handle_rxbd_info");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->handle_rx_buffer) {
+		hal_error_msg("trx handle_rx_buffer");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->update_rxbd) {
+		hal_error_msg("trx update_rxbd");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->notify_rxdone) {
+		hal_error_msg("trx notify_rxdone");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->handle_wp_rpt) {
+		hal_error_msg("trx handle_wp_rpt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#endif /*CONFIG_PCIE_HCI*/
+
+#ifdef CONFIG_USB_HCI
+	if (!trx_ops->hal_get_wd_len) {
+		hal_error_msg("trx hal_get_wd_len");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->hal_fill_wd) {
+		hal_error_msg("trx hal_fill_wd");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->get_bulkout_id) {
+		hal_error_msg("trx get_bulkout_id");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->handle_rx_buffer) {
+		hal_error_msg("trx handle_rx_buffer");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->usb_tx_agg_cfg) {
+		hal_error_msg("trx usb_tx_agg_cfg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->usb_rx_agg_cfg) {
+		hal_error_msg("trx usb_rx_agg_cfg");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+	if (!trx_ops->handle_wp_rpt) {
+		hal_error_msg("trx handle_wp_rpt");
+		status = RTW_HAL_STATUS_FAILURE;
+	}
+#endif
+
+#ifdef CONFIG_SDIO_HCI
+
+#endif
+	return status;
+}
+static enum rtw_hal_status hal_set_ops(struct rtw_phl_com_t *phl_com,
+						struct hal_info_t *hal_info)
+{
+	#ifdef CONFIG_PCI_HCI
+	if (phl_get_hci_type(phl_com) == RTW_HCI_PCIE)
+		hal_set_ops_pci(phl_com, hal_info);
+	#endif
+
+	#ifdef CONFIG_USB_HCI
+	if (phl_get_hci_type(phl_com) == RTW_HCI_USB)
+		hal_set_ops_usb(phl_com, hal_info);
+	#endif
+
+	#ifdef CONFIG_SDIO_HCI
+	if (phl_get_hci_type(phl_com) == RTW_HCI_SDIO)
+		hal_set_ops_sdio(phl_com, hal_info);
+	#endif
+
+	return hal_ops_check(hal_info);
+}
+
+#ifdef RTW_PHL_BCN
+enum rtw_hal_status hal_bcn_init(struct hal_info_t *hal_info)
+{
+	struct bcn_entry_pool *bcn_pool = &hal_info->hal_com->bcn_pool;
+
+	hal_info->hal_com->bcn_pool.bcn_num = 0;
+	INIT_LIST_HEAD(&bcn_pool->bcn_list);
+	_os_spinlock_init(hal_to_drvpriv(hal_info), &hal_info->hal_com->bcn_pool.bcn_lock);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status hal_bcn_deinit(struct hal_info_t *hal_info)
+{
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct bcn_entry_pool *bcn_pool = &hal_info->hal_com->bcn_pool;
+	struct rtw_bcn_entry *tmp_entry, *type = NULL;
+
+	_os_spinlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	phl_list_for_loop_safe(tmp_entry, type,
+		struct rtw_bcn_entry, &bcn_pool->bcn_list, list)
+	{
+		list_del(&tmp_entry->list);
+		_os_mem_free(drv_priv, tmp_entry, sizeof(struct rtw_bcn_entry));
+	}
+
+	_os_spinunlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status hal_alloc_bcn_entry(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info,
+		struct rtw_bcn_entry **bcn_entry, struct rtw_bcn_info_cmn *bcn_cmn)
+{
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct bcn_entry_pool *bcn_pool = &hal_info->hal_com->bcn_pool;
+	struct rtw_bcn_entry *new_entry = _os_mem_alloc(drv_priv, sizeof(struct rtw_bcn_entry));
+
+	if(new_entry == NULL)
+		return RTW_HAL_STATUS_FAILURE;
+
+	_os_spinlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	list_add_tail(&new_entry->list, &bcn_pool->bcn_list);
+	bcn_pool->bcn_num ++;
+
+	_os_spinunlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	*bcn_entry = new_entry;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status hal_free_bcn_entry(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info,
+		u8 bcn_id)
+{
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct bcn_entry_pool *bcn_pool = &hal_info->hal_com->bcn_pool;
+	struct rtw_bcn_entry *tmp_entry, *type = NULL;
+	u8 is_found = 0;
+
+	_os_spinlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	phl_list_for_loop_safe(tmp_entry, type,
+		struct rtw_bcn_entry, &bcn_pool->bcn_list, list)
+	{
+		if(tmp_entry->bcn_cmn->bcn_id == bcn_id){
+			is_found = 1;
+			break;
+		}
+	}
+
+	_os_spinunlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	if(is_found){
+		list_del(&tmp_entry->list);
+		_os_mem_free(drv_priv, tmp_entry, sizeof(struct rtw_bcn_entry));
+		bcn_pool->bcn_num --;
+		return RTW_HAL_STATUS_SUCCESS;
+	}
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status hal_get_bcn_entry(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info,
+		struct rtw_bcn_entry **bcn_entry, u8 bcn_id)
+{
+	void *drv_priv = hal_to_drvpriv(hal_info);
+	struct bcn_entry_pool *bcn_pool = &hal_info->hal_com->bcn_pool;
+	struct rtw_bcn_entry *tmp_entry, *type = NULL;
+	u8 is_found = 0;
+
+	_os_spinlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	phl_list_for_loop_safe(tmp_entry, type,
+		struct rtw_bcn_entry, &bcn_pool->bcn_list, list)
+	{
+		if(tmp_entry->bcn_cmn->bcn_id == bcn_id){
+			is_found = 1;
+			break;
+		}
+	}
+
+	_os_spinunlock(drv_priv, &bcn_pool->bcn_lock, _bh, NULL);
+
+	if(is_found){
+		*bcn_entry = tmp_entry;
+		return RTW_HAL_STATUS_SUCCESS;
+	}
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+enum rtw_hal_status hal_update_bcn_entry(struct rtw_phl_com_t *phl_com,
+                                         struct hal_info_t *hal_info,
+                                         struct rtw_bcn_entry *bcn_entry,
+                                         struct rtw_bcn_info_cmn *bcn_cmn)
+{
+	struct rtw_phl_stainfo_t *phl_sta = NULL;
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	u8 ridx = 0;
+	u8 lidx = 0;
+
+	if (bcn_cmn != NULL) {
+		bcn_entry->bcn_cmn = bcn_cmn;
+		ridx = bcn_cmn->role_idx;
+		lidx = bcn_cmn->lidx;
+
+	} else {
+		ridx = bcn_entry->bcn_cmn->role_idx;
+		lidx = bcn_entry->bcn_cmn->lidx;
+	}
+
+	wrole = &phl_com->wifi_roles[ridx];
+	rlink = rtw_phl_get_rlink(wrole, lidx);
+	phl_sta = rtw_phl_get_stainfo_self(phl_com->phl_priv, rlink);
+
+	bcn_entry->bcn_hw.band = rlink->hw_band;
+	bcn_entry->bcn_hw.port = rlink->hw_port;
+	bcn_entry->bcn_hw.mbssid = rlink->hw_mbssid;
+	bcn_entry->bcn_hw.hiq_win = rlink->hiq_win;
+	bcn_entry->bcn_hw.bss_color = rlink->protocol_cap.bsscolor;
+	bcn_entry->bcn_hw.mac_id = (u8)phl_sta->macid;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_add_beacon(struct rtw_phl_com_t *phl_com, void *hal,
+		void *bcn_cmn)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+	struct rtw_bcn_entry *bcn_entry = NULL;
+
+	if(hal_alloc_bcn_entry(phl_com, hal_info, &bcn_entry, (struct rtw_bcn_info_cmn *)bcn_cmn) == RTW_HAL_STATUS_FAILURE)
+		return RTW_HAL_STATUS_FAILURE;
+
+	if(hal_update_bcn_entry(phl_com, hal_info, bcn_entry, (struct rtw_bcn_info_cmn *)bcn_cmn) == RTW_HAL_STATUS_FAILURE)
+		return RTW_HAL_STATUS_FAILURE;
+
+	if(hal_ops->cfg_bcn(phl_com, hal_info, bcn_entry) == RTW_HAL_STATUS_FAILURE)
+		return RTW_HAL_STATUS_FAILURE;
+
+	if(hal_ops->upt_bcn(phl_com, hal_info, bcn_entry) == RTW_HAL_STATUS_FAILURE)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_update_beacon(struct rtw_phl_com_t *phl_com, void *hal,
+		u8 bcn_id)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+	struct rtw_bcn_entry *bcn_entry = NULL;
+
+	if (hal_get_bcn_entry(phl_com, hal_info, &bcn_entry, bcn_id)
+	    == RTW_HAL_STATUS_FAILURE) {
+		PHL_ERR("No beacon entry %u to config!\n", bcn_id);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	if(hal_update_bcn_entry(phl_com, hal_info, bcn_entry, NULL) == RTW_HAL_STATUS_FAILURE)
+		return RTW_HAL_STATUS_FAILURE;
+
+	/* Configure beacon in case of beacon timing changed.
+	 * ToDo: Apply only when beacon interval and/or DTIM period
+	 * are changed.
+	 */
+	do {
+		#ifdef CONFIG_RTW_SUPPORT_MBSSID_VAP
+		/* Beacon timing is port-wide setting. Ignore VAP. */
+		struct rtw_wifi_role_t *wrole =
+			&phl_com->wifi_roles[bcn_entry->bcn_cmn->role_idx];
+		if (wrole->type == PHL_RTYPE_VAP)
+			break;
+		#endif /* CONFIG_RTW_SUPPORT_MBSSID_VAP */
+		if (hal_ops->cfg_bcn(phl_com, hal_info, bcn_entry)
+		    == RTW_HAL_STATUS_FAILURE) {
+			PHL_ERR("Failed to configure beacon entry %u!\n",
+				bcn_id);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+	} while (0);
+
+	if (hal_ops->upt_bcn(phl_com, hal_info, bcn_entry)
+	    == RTW_HAL_STATUS_FAILURE) {
+		PHL_ERR("Failed to update beacon entry %u!\n", bcn_id);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_free_beacon(struct rtw_phl_com_t *phl_com, void *hal,
+		u8 bcn_id)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	if(hal_free_bcn_entry(phl_com, hal_info, bcn_id) == RTW_HAL_STATUS_FAILURE)
+		return RTW_HAL_STATUS_FAILURE;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#ifdef CONFIG_RTW_DEBUG_BCN_TX
+enum rtw_hal_status rtw_hal_get_beacon_cnt(struct rtw_phl_com_t *phl_com, void *hal,
+			u8 bcn_id, struct rtw_bcn_stats **bcn_stats)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_bcn_entry *bcn_entry = NULL;
+
+	if (RTW_HAL_STATUS_FAILURE ==
+		hal_get_bcn_entry(phl_com, hal_info, &bcn_entry, bcn_id)) {
+		PHL_ERR("Failed to get beacon entry %u!\n", bcn_id);
+		bcn_stats = NULL;
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	*bcn_stats = &bcn_entry->bcn_stats;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+#endif
+
+#ifdef CONFIG_PHL_PKTOFLD
+enum rtw_hal_status rtw_hal_pkt_ofld(void *hal, u8 *id, u8 op,
+					u8 *pkt_buf, u16 *pkt_len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_pkt_ofld(hal_info, id, op, pkt_buf, pkt_len);
+}
+
+enum rtw_hal_status rtw_hal_pkt_update_ids(void *hal,
+						struct pkt_ofld_entry *entry)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_pkt_update_ids(hal_info, entry);
+}
+#endif
+
+enum rtw_hal_status rtw_hal_get_pwr_state(void *hal, enum rtw_mac_pwr_st *pwr_state)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_get_pwr_state(hal_info, pwr_state);
+}
+
+enum rtw_hal_status rtw_hal_init(void *drv_priv,
+	struct rtw_phl_com_t *phl_com, void **hal, enum rtl_ic_id ic_id)
+{
+	struct hal_info_t *hal_info = NULL;
+	struct rtw_hal_com_t *hal_com = NULL;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	void (*set_intf_ops)(struct rtw_hal_com_t *hal, struct hal_io_ops *ops) = NULL;
+	enum rtw_chip_id chip_id = CHIP_WIFI6_MAX;
+
+	if (ic_id == RTL8852A)
+		chip_id = CHIP_WIFI6_8852A;
+	else if(ic_id == RTL8834A)
+		chip_id = CHIP_WIFI6_8834A;
+	else if(ic_id == RTL8852B)
+		chip_id = CHIP_WIFI6_8852B;
+	else if(ic_id == RTL8852BP)
+		chip_id = CHIP_WIFI6_8852BP;
+	else if(ic_id == RTL8852BPT)
+		chip_id = CHIP_WIFI6_8852BPT;
+	else if(ic_id == RTL8852BT)
+		chip_id = CHIP_WIFI6_8852BT;
+	else if(ic_id == RTL8852C)
+		chip_id = CHIP_WIFI6_8852C;
+	else if(ic_id == RTL8842A)
+		chip_id = CHIP_WIFI6_8842A;
+	else if(ic_id == RTL8192XB)
+		chip_id = CHIP_WIFI6_8192XB;
+	else if(ic_id == RTL8832BR)
+		chip_id = CHIP_WIFI6_8832BR;
+	else if (ic_id == RTL8851B)
+		chip_id = CHIP_WIFI6_8851B;
+	else if (ic_id == RTL8852D)
+		chip_id = CHIP_WIFI6_8852D;
+	else
+		chip_id = CHIP_WIFI6_MAX;
+
+	hal_info = _os_mem_alloc(drv_priv, sizeof(struct hal_info_t));
+	if (hal_info == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		PHL_ERR("alloc hal_info failed\n");
+		goto error_hal_mem;
+	}
+	*hal = hal_info;
+
+	hal_com = _os_mem_alloc(drv_priv, sizeof(struct rtw_hal_com_t));
+	if (hal_com == NULL) {
+		hal_status = RTW_HAL_STATUS_RESOURCE;
+		PHL_ERR("alloc hal_com failed\n");
+		goto error_hal_com_mem;
+	}
+	hal_info->phl_com = phl_com;
+	hal_info->hal_com = hal_com;
+	hal_com->drv_priv = drv_priv;
+	hal_com->hal_priv = hal_info;
+	hal_com->chip_id = chip_id;
+	hal_com->dbcc_en = false;
+	#ifdef DBG_HAL_MEM_MOINTOR
+	_os_atomic_set(drv_priv, &(hal_com->hal_mem), 0);
+	#endif
+
+	/*set io_ops*/
+	#ifdef CONFIG_PCI_HCI
+	if (phl_com->hci_type == RTW_HCI_PCIE)
+		set_intf_ops = hal_pci_set_io_ops;
+	#endif
+
+	#ifdef CONFIG_USB_HCI
+	if (phl_com->hci_type == RTW_HCI_USB)
+		set_intf_ops = hal_usb_set_io_ops;
+	#endif
+
+	#ifdef CONFIG_SDIO_HCI
+	if (phl_com->hci_type == RTW_HCI_SDIO)
+		set_intf_ops = hal_sdio_set_io_ops;
+	#endif
+
+	hal_status = hal_init_io_priv(hal_info->hal_com, set_intf_ops);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("hal_init_io_priv failed\n");
+		goto error_io_priv;
+	}
+
+	hal_status = rtw_hal_mac_init(phl_com, hal_info);
+	if ((hal_status != RTW_HAL_STATUS_SUCCESS) || (hal_info->mac == NULL)) {
+		PHL_ERR("rtw_hal_mac_init failed\n");
+		goto error_mac_init;
+	}
+
+	/*set hal_ops and hal_hook_trx_ops*/
+	hal_status = hal_set_ops(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("hal_set_ops failed\n");
+		goto error_hal_ops;
+	}
+
+	hal_status = hal_info->hal_ops.hal_init(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("hal_ops.hal_init failed\n");
+		goto error_hal_init;
+	}
+
+	hal_status = rtw_hal_efuse_init(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_efuse_init failed\n");
+		goto error_efuse_init;
+	}
+
+	hal_status = rtw_hal_bb_init(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_bb_init failed\n");
+		goto error_bb_init;
+	}
+
+	hal_status = rtw_hal_rf_init(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_rf_init failed\n");
+		goto error_rf_init;
+	}
+
+	hal_status = rtw_hal_btc_init(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("rtw_hal_btc_init failed\n");
+		goto error_btc_init;
+	}
+
+#ifdef RTW_PHL_BCN
+	hal_status = hal_bcn_init(hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("hal_bcn_init failed\n");
+		goto error_bcn_init;
+	}
+#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+
+#ifdef RTW_PHL_BCN
+error_bcn_init:
+	rtw_hal_btc_deinit(phl_com, hal_info);
+#endif
+
+error_btc_init:
+	rtw_hal_rf_deinit(phl_com, hal_info);
+
+error_rf_init:
+	rtw_hal_bb_deinit(phl_com, hal_info);
+
+error_bb_init:
+	rtw_hal_efuse_deinit(phl_com, hal_info);
+
+error_efuse_init:
+	hal_info->hal_ops.hal_deinit(phl_com, hal_info);
+
+error_hal_init:
+error_hal_ops:
+	rtw_hal_mac_deinit(phl_com, hal_info);
+
+error_mac_init:
+	hal_deinit_io_priv(hal_com);
+
+error_io_priv:
+	if (hal_com) {
+		_os_mem_free(drv_priv, hal_com, sizeof(struct rtw_hal_com_t));
+		hal_com = NULL;
+	}
+
+error_hal_com_mem:
+	if (hal_info) {
+		_os_mem_free(drv_priv, hal_info, sizeof(struct hal_info_t));
+		*hal = hal_info = NULL;
+	}
+
+error_hal_mem:
+	return hal_status;
+}
+struct rtw_hal_com_t *rtw_hal_get_halcom(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return hal_info->hal_com;
+}
+
+void rtw_hal_deinit(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	void *drv_priv = NULL;
+
+	if(hal_info == NULL)
+		return;
+	drv_priv = hal_to_drvpriv(hal_info);
+
+	/* stop mechanism / disassociate hal ops */
+#ifdef RTW_PHL_BCN
+	hal_bcn_deinit(hal_info);
+#endif
+	rtw_hal_btc_deinit(phl_com, hal_info);
+	rtw_hal_rf_deinit(phl_com, hal_info);
+	rtw_hal_bb_deinit(phl_com, hal_info);
+	rtw_hal_efuse_deinit(phl_com, hal_info);
+	rtw_hal_mac_deinit(phl_com, hal_info);
+	hal_info->hal_ops.hal_deinit(phl_com, hal_info);
+	hal_deinit_io_priv(hal_info->hal_com);
+
+	#ifdef DBG_HAL_MEM_MOINTOR
+	PHL_INFO("[PHL-MEM] %s HAL memory :%d\n", __func__,
+	         _os_atomic_read(drv_priv, &(hal_info->hal_com->hal_mem)));
+	#endif
+
+	if(hal_info->hal_com->bf_obj)
+		hal_bf_deinit(hal_info);
+	if(hal_info->hal_com->csi_obj)
+		hal_csi_deinit(hal_info);
+	if(hal_info->hal_com->snd_obj)
+		hal_snd_obj_deinit(hal_info);
+	_os_mem_free(drv_priv,
+	             hal_info->hal_com, sizeof(struct rtw_hal_com_t));
+	hal_info->hal_com = NULL;
+
+	_os_mem_free(drv_priv,
+	             hal_info, sizeof(struct hal_info_t));
+	hal_info = NULL;
+}
+
+bool rtw_hal_is_inited(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return hal_info->hal_com->is_hal_init;
+}
+
+u32 rtw_hal_hci_cfg(struct rtw_phl_com_t *phl_com, void *hal,
+					struct rtw_ic_info *ic_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+#ifdef CONFIG_SDIO_HCI
+	hal_info->hal_com->block_sz = ic_info->sdio_info.block_sz;
+#endif
+	hal_ops->hal_hci_configure(phl_com, hal_info, ic_info);
+
+	return hal_status;
+}
+
+u32 rtw_hal_read_chip_info(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+	hal_ops->read_chip_version(phl_com, hal_info);
+
+	/*get mac,bb,rf capability*/
+	hal_ops->init_hal_spec(phl_com, hal_info);
+
+	return hal_status;
+}
+
+static void _hal_reset_chandef(struct rtw_hal_com_t *hal_com)
+{
+	struct rtw_chan_def *chandef = NULL;
+	u8 bid = 0;
+
+	for (bid = 0; bid < HW_BAND_MAX; bid++) {
+		chandef = &(hal_com->band[bid].cur_chandef);
+		chandef->bw = CHANNEL_WIDTH_MAX;
+		chandef->chan = 0;
+		chandef->offset = CHAN_OFFSET_NO_EXT;
+	}
+}
+
+void rtw_hal_set_default_var(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+	hal_info->hal_com->assoc_sta_cnt = 0;
+	hal_info->hal_com->band[HW_PHY_0].assoc_sta_cnt = 0;
+	hal_info->hal_com->band[HW_PHY_0].assoc_sta_cnt = 0;
+
+	_hal_reset_chandef(hal_info->hal_com);
+
+	hal_ops->init_default_value(hal_info);
+}
+
+u32 rtw_hal_var_init(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	hal_com->is_hal_init = false;
+	rtw_hal_set_default_var(hal_info);
+
+	/*csi init shall after read hw chip info*/
+	hal_status = hal_csi_init(
+			hal_info,
+			phl_com->hal_spec.max_csi_buf_su_nr,
+			phl_com->hal_spec.max_csi_buf_mu_nr);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("rtw_hal_csi_init failed\n");
+	}
+
+	/*bf init shall after read hw chip info*/
+	hal_status = hal_bf_init(
+			hal_info,
+			phl_com->hal_spec.max_bf_ent_nr,
+			phl_com->hal_spec.max_su_sta_nr,
+			phl_com->hal_spec.max_mu_sta_nr);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("rtw_hal_bf_init failed\n");
+	}
+
+	hal_status = hal_snd_obj_init(hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("hal_snd_obj_init failed\n");
+	}
+	return hal_status;
+}
+
+static int
+_hal_parse_macreg(void *drv_priv, u32 *pdest_buf, u8 *psrc_buf, u32 buflen)
+{
+	return 0;
+}
+
+static void _hal_send_hal_init_hub_msg(struct rtw_phl_com_t *phl_com, u8 init_ok)
+{
+	struct phl_msg msg = {0};
+	u16 evt_id = (init_ok) ? MSG_EVT_HAL_INIT_OK : MSG_EVT_HAL_INIT_FAIL;
+
+	msg.inbuf = NULL;
+	msg.inlen = 0;
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_PHY_MGNT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, evt_id);
+	msg.band_idx = HW_BAND_0;
+	rtw_phl_msg_hub_hal_send(phl_com, NULL, &msg);
+}
+
+enum rtw_hal_status rtw_hal_preload(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+#endif /* DBG_MONITOR_TIME */
+
+#ifdef FPGA_TEST
+	FUNCIN();
+#else /*!FPGA_TEST*/
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	FUNCIN();
+
+	hal_status = hal_ops->hal_fast_start(phl_com, hal_info);
+
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_HAL_FAST_START);
+#endif
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("%s: Fast start fail!\n", __func__);
+		goto exit;
+	}
+
+#ifdef DBG_MONITOR_TIME
+	start_t = 0;
+	PHL_FUN_MON_START(&start_t);
+#endif
+	hal_status = hal_ops->hal_get_efuse(phl_com, hal_info);
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_HAL_GET_EFUSE);
+#endif
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		goto exit;
+
+#endif /*FPGA_TEST*/
+
+#ifdef DBG_MONITOR_TIME
+	start_t = 0;
+	PHL_FUN_MON_START(&start_t);
+#endif
+	hal_status = hal_ops->hal_fast_stop(phl_com, hal_info);
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_HAL_FAST_STOP);
+#endif
+
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status hal_rfe_type_chk(struct rtw_phl_com_t *phl_com,
+				     struct hal_info_t *hal_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_UNKNOWN_RFE_TYPE;
+
+	if(phl_com->dev_cap.rfe_type != 0xff){
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	}
+	else {
+		if(phl_com->dev_cap.bypass_rfe_chk == true){
+			rtw_hal_rf_get_default_rfe_type(hal_info->hal_com);
+			rtw_hal_rf_get_default_xtal(hal_info->hal_com);
+			PHL_WARN("%s: Use default RFE type(0x%x) / XTAL(0x%x) configuration for empty EFUSE\n",
+				 __FUNCTION__,
+				 hal_info->hal_com->dev_hw_cap.rfe_type,
+				 hal_info->hal_com->dev_hw_cap.xcap);
+			hal_status = RTW_HAL_STATUS_SUCCESS;
+		}
+	}
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_start(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+#ifdef RTW_WKARD_DEF_CMACTBL_CFG
+	enum rf_path tx, rx;
+#endif
+#ifdef DBG_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_MONITOR_TIME */
+
+	hal_status = hal_rfe_type_chk(phl_com, hal_info);
+	if(hal_status != RTW_HAL_STATUS_SUCCESS){
+		PHL_ERR("%s: Unknown RFE type!!!\n", __func__);
+		goto exit;
+	}
+
+	hal_status = hal_ops->hal_start(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s: hal start fail, status %u\n", __func__, hal_status);
+		goto exit;
+	}
+
+	if (phl_com->edcca_mode != RTW_EDCCA_NORMAL) {
+		hal_status = rtw_hal_mac_sifs_chk_cca_en(hal, HW_BAND_0, true);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s: enable rtw_hal_mac_sifs_chk_cca_en failed!\n", __func__);
+		}
+
+		hal_status = rtw_hal_mac_set_resp_ack_chk_cca(hal, HW_BAND_0, true);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("%s: enable rtw_hal_mac_set_resp_ack_chk_cca failed!\n", __func__);
+		}
+	}
+
+	if (phl_com->rsp_static_rts_chk_off)
+		rtw_hal_mac_set_rsp_stat_rts_chk_en(hal, HW_BAND_0, false);
+
+	hal_status = RTW_HAL_STATUS_SUCCESS;
+	hal_info->hal_com->is_hal_init = true;
+	rtw_hal_set_default_var(hal_info);
+
+	rtw_hal_rf_set_power(hal_info, HW_PHY_0, PWR_BY_RATE);
+#ifdef RTW_WKARD_DEF_CMACTBL_CFG
+	tx = hal_ops->get_path_from_ant_num(phl_com->phy_cap[0].tx_path_num);
+	rx = hal_ops->get_path_from_ant_num(phl_com->phy_cap[0].rx_path_num);
+	rtw_hal_bb_trx_path_cfg(hal_info, tx, phl_com->phy_cap[0].txss,
+		rx, phl_com->phy_cap[0].rxss);
+#endif
+#ifdef RTW_WKARD_SINGLE_PATH_RSSI
+	hal_info->hal_com->cur_rx_rfpath =
+		hal_ops->get_path_from_ant_num(phl_com->phy_cap[0].rx_path_num);
+#endif
+	#ifdef CONFIG_BTCOEX
+	rtw_hal_btc_radio_state_ntfy(hal_info, BTC_RFCTRL_WL_ON);
+	#endif
+
+	rtw_hal_bb_pwr_ctrl_ability_set(hal_info, !phl_com->dev_cap.disable_dyn_txpwr);
+
+#ifdef RTW_WKARD_DYNAMIC_PCIE_GEN
+	hal_pcie_gen_set(hal_info, RTW_PCIE_GEN_2);
+#endif
+
+exit:
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	/* if need not keep para buf, phl_com->dev_sw_cap.bfree_para_info = true */
+	rtw_phl_init_free_para_buf(phl_com);
+#endif
+#ifdef DBG_MONITOR_TIME
+	PHL_FUNC_MON_END(phl_com, &start_t, TIME_HAL_START);
+#endif
+
+	return hal_status;
+}
+
+void rtw_hal_stop(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+	do {
+		if (true == TEST_STATUS_FLAG(phl_com->dev_state,
+		                             RTW_DEV_SURPRISE_REMOVAL)) {
+			PHL_WARN("%s(): Device has removed, skip HW stop functions!\n", __func__);
+			break;
+ 		}
+
+#ifdef RTW_WKARD_DYNAMIC_PCIE_GEN
+		hal_pcie_gen_set(hal_info, RTW_PCIE_GEN_2);
+#endif
+
+#ifdef CONFIG_BTCOEX
+		rtw_hal_btc_radio_state_ntfy(hal_info, BTC_RFCTRL_WL_OFF);
+#endif
+		hal_status = hal_ops->hal_stop(phl_com, hal_info);
+		hal_info->hal_com->is_hal_init = false;
+	} while (false);
+}
+
+enum rtw_hal_status rtw_hal_restart(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+	#ifdef CONFIG_BTCOEX
+	rtw_hal_btc_radio_state_ntfy(hal_info, BTC_RFCTRL_WL_OFF);
+	#endif
+
+	hal_status = hal_ops->hal_stop(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		return hal_status;
+
+	hal_status = hal_ops->hal_start(phl_com, hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		return hal_status;
+
+	#ifdef CONFIG_BTCOEX
+	rtw_hal_btc_radio_state_ntfy(hal_info, BTC_RFCTRL_WL_ON);
+	#endif
+
+	hal_info->hal_com->is_hal_init = true;
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_hal_deinit(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	hstatus = rtw_hal_mac_hal_deinit(phl_com, hal_info);
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_role_deinit(void *hal, struct rtw_wifi_role_link_t *rlink)
+{
+	/* hw port cfg - mac_port_deinit */
+	return rtw_hal_mac_port_deinit(hal, rlink);
+}
+
+enum rtw_hal_status
+rtw_hal_role_cfg(void *hal,
+                 struct rtw_wifi_role_t *wrole,
+                 struct rtw_wifi_role_link_t *rlink)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	#ifdef CONFIG_RTW_SUPPORT_MBSSID_VAP
+	if (wrole->type == PHL_RTYPE_VAP) {
+		u32	param = 1;
+
+		/* Configure port for M-BSSID */
+		PHL_INFO("Config role #%u rlink #%u of type %u. MBSSID: %u.\n",
+	          wrole->id, rlink->id, wrole->type, rlink->hw_mbssid);
+
+		return rtw_hal_mac_port_cfg(hal_info, rlink,
+					    PCFG_MBSSID_EN, &param);
+	}
+	#endif /* CONFIG_RTW_SUPPORT_MBSSID_VAP */
+
+	/*enable & configure mac hw-port*/
+	hal_status = rtw_hal_mac_port_init(hal_info, wrole, rlink);
+
+	/* init hal mac bfer function if wrole cap if any of bfer cap is true*/
+	if (rlink->protocol_cap.he_su_bfmr ||
+	    rlink->protocol_cap.he_mu_bfmr ||
+	    rlink->protocol_cap.vht_su_bfmr ||
+	    rlink->protocol_cap.vht_mu_bfmr ||
+	    rlink->protocol_cap.ht_su_bfmr) {
+		if ((rtw_phl_role_is_ap_category(wrole)) ||
+		    ((rtw_phl_role_is_client_category(wrole)) &&
+		     (MLME_LINKED == rlink->mstate))) {
+			hal_status = hal_bf_hw_mac_init_bfer(hal_info,
+							     rlink->hw_band);
+		}
+	}
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_role_cfg_ex(void *hal,
+                    struct rtw_wifi_role_link_t *rlink,
+                    enum pcfg_type type,
+                    void *param)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	hal_status = rtw_hal_mac_port_cfg(hal_info, rlink, type, param);
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_beacon_stop(void *hal,
+                    struct rtw_wifi_role_link_t *rlink,
+                    enum rlink_bcn_stop_rson reason,
+                    bool stop)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_SUCCESS;
+	u32 bcn_en;
+
+	if (reason >= RLINK_BCN_STOP_RSON_MAX) {
+		PHL_ERR("%s wr-%d, rlink:%d, stop:%d with unknown reason:%d\n", __func__,
+			rlink->wrole->id, rlink->id, stop, reason);
+		hsts = RTW_HAL_STATUS_FAILURE;
+		goto exit;
+	}
+
+	if (stop) {
+		if (rlink->bcn_stop & BIT(reason))
+			goto bcn_en_chk;
+		rlink->bcn_stop |= BIT(reason);
+	} else {
+		if (!(rlink->bcn_stop & BIT(reason)))
+			goto bcn_en_chk;
+		rlink->bcn_stop &= ~BIT(reason);
+	}
+
+	PHL_INFO("%s wr-%d, rlink:%d, reason:%d, stop:%d (bcn_stop:0x%02x)\n", __func__,
+		rlink->wrole->id, rlink->id, reason, stop, rlink->bcn_stop);
+
+bcn_en_chk:
+	bcn_en = rlink->bcn_stop ? 0 : 1;
+
+	if (((TEST_STATUS_FLAG(rlink->status, RLINK_STATUS_BCN_STOP)) && !bcn_en) ||
+           ((!TEST_STATUS_FLAG(rlink->status, RLINK_STATUS_BCN_STOP)) && bcn_en))
+		goto exit;
+
+	PHL_INFO("%s wr-%d, rlink:%d, bcn_en:%s\n", __func__,
+		rlink->wrole->id, rlink->id, (bcn_en) ? "E" : "D");
+	hsts = rtw_hal_mac_port_cfg(hal_info, rlink, PCFG_BCN_EN, &bcn_en);
+	if (hsts == RTW_HAL_STATUS_SUCCESS) {
+		if (!bcn_en)
+			SET_STATUS_FLAG(rlink->status, RLINK_STATUS_BCN_STOP);
+		else
+			CLEAR_STATUS_FLAG(rlink->status, RLINK_STATUS_BCN_STOP);
+	}
+
+exit:
+	return hsts;
+}
+
+enum rtw_hal_status
+hal_ver_check(struct rtw_hal_com_t *hal_com)
+{
+	if ((hal_com->mac_vc.mac_ver < hal_com->bb_vc.mac_ver) ||
+		(hal_com->mac_vc.mac_ver < hal_com->rf_vc.mac_ver) ||
+		(hal_com->mac_vc.mac_ver < hal_com->btc_vc.mac_ver) ||
+		(hal_com->mac_vc.mac_ver < hal_com->fw_vc.mac_ver))
+		return RTW_HAL_STATUS_FAILURE;
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+
+enum rtw_hal_status
+rtw_hal_watchdog(void *hal, struct rtw_phl_com_t *phl_com)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_bb_watchdog(hal_info, false);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_bb_watchdog fail (%x)\n", __FUNCTION__, hal_status);
+		goto exit;
+	}
+
+	hal_status = rtw_hal_rf_watchdog(hal_info);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_rf_watchdog fail (%x)\n", __FUNCTION__, hal_status);
+		goto exit;
+	}
+
+	hal_status = rtw_hal_mac_watchdog(hal_info, phl_com);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("%s rtw_hal_mac_watchdog fail (%x)\n", __FUNCTION__, hal_status);
+		goto exit;
+	}
+
+exit:
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_simple_watchdog(void *hal, u8 io_en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_bb_simple_watchdog(hal_info, io_en);
+	if(hal_status != RTW_HAL_STATUS_SUCCESS){
+		PHL_INFO("%s fail (%x)\n",
+			 __FUNCTION__, hal_status);
+		goto exit;
+	}
+
+exit:
+	return hal_status;
+}
+
+
+enum rtw_hal_status
+rtw_hal_cfg_trx_path(void *hal, enum rf_path tx, u8 tx_nss,
+		     enum rf_path rx, u8 rx_nss)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	u32 freerun_cnt_h = 0, freerun_cnt_l = 0;
+
+	if (tx < RF_PATH_AB) {
+		/* forced tx nss = 1*/
+		tx_nss = 1;
+	}
+	if (rx < RF_PATH_AB) {
+		/* forced rx nss = 1*/
+		rx_nss = 1;
+	}
+
+	hal_status = rtw_hal_bb_trx_path_cfg(
+			hal_info,
+			tx,
+			((tx_nss > phl_com->phy_cap[0].tx_path_num) ?
+			  phl_com->phy_cap[0].tx_path_num : tx_nss),
+			rx,
+			((rx_nss > phl_com->phy_cap[0].rx_path_num) ?
+			  phl_com->phy_cap[0].rx_path_num : rx_nss));
+#ifdef RTW_WKARD_SINGLE_PATH_RSSI
+	hal_info->hal_com->cur_rx_rfpath = rx;
+#endif
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		return hal_status;
+
+	hal_status = rtw_hal_mac_get_freerun_cnt(hal_info->hal_com, HW_BAND_0, &freerun_cnt_h, &freerun_cnt_l);
+
+	phl_com->rssi_stat.last_switch_rx_freerun = freerun_cnt_l;
+
+	PHL_TRACE(COMP_PHL_RECV, _PHL_INFO_, "%s : switch rx path freerun 0x%x\n", __func__, freerun_cnt_l);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_set_sw_gpio_mode(struct rtw_phl_com_t *phl_com, void *hal, enum rtw_gpio_mode mode, u8 gpio)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_mac_set_sw_gpio_mode(hal_info, mode, gpio);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : status(%u).\n", __func__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_sw_gpio_ctrl(struct rtw_phl_com_t *phl_com, void *hal, u8 high, u8 gpio)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_mac_sw_gpio_ctrl(hal_info, high, gpio);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : status(%u).\n", __func__, hal_status);
+
+	return hal_status;
+}
+
+void rtw_hal_dbg_status_dump(void *hal, struct hal_mac_dbg_dump_cfg *cfg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_mac_dbg_status_dump(hal_info, cfg);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_io.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_io.c
new file mode 100644
index 000000000000..544e333ea726
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_io.c
@@ -0,0 +1,799 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+/*
+The purpose of hal_io.c
+
+a. provides the API
+b. provides the protocol engine
+c. provides the software interface between caller and the hardware interface
+
+Compiler Flag Option:
+1. CONFIG_SDIO_HCI:
+    a. USE_SYNC_IRP:  Only sync operations are provided.
+    b. USE_ASYNC_IRP:Both sync/async operations are provided.
+
+2. CONFIG_USB_HCI:
+   a. USE_ASYNC_IRP: Both sync/async operations are provided.
+
+3. CONFIG_CFIO_HCI:
+   b. USE_SYNC_IRP: Only sync operations are provided.
+
+Only sync read/rtw_write_mem operations are provided.
+*/
+
+#define _HAL_IO_C_
+#include "hal_headers.h"
+
+u8 _hal_read8(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u8 r_val;
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	u8(*_read8)(struct rtw_hal_com_t *hal, u32 addr);
+
+	#ifdef DBG_PHL_MAC_REG_RW
+	if (rtw_hal_mac_reg_chk(hal, addr) == false) {
+		r_val = 0xEA;
+		return r_val;
+	}
+	#endif
+
+	_read8 = io_priv->io_ops._read8;
+	r_val = _read8(hal, addr);
+	return r_val;
+}
+
+u16 _hal_read16(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u16 r_val;
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	u16(*_read16)(struct rtw_hal_com_t *hal, u32 addr);
+
+	#ifdef DBG_PHL_MAC_REG_RW
+	if (rtw_hal_mac_reg_chk(hal, addr) == false) {
+		r_val = 0xEAEA;
+		return r_val;
+	}
+	#endif
+
+	_read16 = io_priv->io_ops._read16;
+	r_val = _read16(hal, addr);
+	return r_val;
+}
+
+u32 _hal_read32(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u32 r_val;
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	u32(*_read32)(struct rtw_hal_com_t *hal, u32 addr);
+
+	#ifdef DBG_PHL_MAC_REG_RW
+	if (rtw_hal_mac_reg_chk(hal, addr) == false) {
+		r_val = 0xEAEAEAEA;
+		return r_val;
+	}
+	#endif
+
+	_read32 = io_priv->io_ops._read32;
+	r_val = _read32(hal, addr);
+	return r_val;
+}
+
+int _hal_write8(struct rtw_hal_com_t *hal, u32 addr, u8 val)
+{
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	int (*_write8)(struct rtw_hal_com_t *hal, u32 addr, u8 val);
+	#ifdef RTW_WKARD_BUS_WRITE
+	int (*_write_post_cfg)(struct rtw_hal_com_t *hal, u32 addr,
+						   u32 value) = NULL;
+	#endif
+	int ret;
+
+	#ifdef DBG_PHL_MAC_REG_RW
+	if (rtw_hal_mac_reg_chk(hal, addr) == false)
+		return 0;
+	#endif
+
+	_write8 = io_priv->io_ops._write8;
+	ret = _write8(hal, addr, val);
+
+	#ifdef RTW_WKARD_BUS_WRITE
+	_write_post_cfg = io_priv->io_ops._write_post_cfg;
+	if(NULL != _write_post_cfg) {
+		ret = _write_post_cfg(hal, addr, val);
+	}
+	#endif
+	return ret;
+}
+int _hal_write16(struct rtw_hal_com_t *hal, u32 addr, u16 val)
+{
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	int (*_write16)(struct rtw_hal_com_t *hal, u32 addr, u16 val);
+	#ifdef RTW_WKARD_BUS_WRITE
+	int (*_write_post_cfg)(struct rtw_hal_com_t *hal, u32 addr,
+						   u32 value) = NULL;
+	#endif
+	int ret;
+
+	#ifdef DBG_PHL_MAC_REG_RW
+	if (rtw_hal_mac_reg_chk(hal, addr) == false)
+		return 0;
+	#endif
+
+	_write16 = io_priv->io_ops._write16;
+	ret = _write16(hal, addr, val);
+
+	#ifdef RTW_WKARD_BUS_WRITE
+	_write_post_cfg = io_priv->io_ops._write_post_cfg;
+	if(NULL != _write_post_cfg) {
+		ret = _write_post_cfg(hal, addr, val);
+	}
+	#endif
+	return ret;
+}
+int _hal_write32(struct rtw_hal_com_t *hal, u32 addr, u32 val)
+{
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	int (*_write32)(struct rtw_hal_com_t *hal, u32 addr, u32 val);
+	#ifdef RTW_WKARD_BUS_WRITE
+	int (*_write_post_cfg)(struct rtw_hal_com_t *hal, u32 addr,
+						   u32 value) = NULL;
+	#endif
+	int ret;
+
+	#ifdef DBG_PHL_MAC_REG_RW
+	if (rtw_hal_mac_reg_chk(hal, addr) == false)
+		return 0;
+	#endif
+
+	_write32 = io_priv->io_ops._write32;
+	ret = _write32(hal, addr, val);
+
+	#ifdef RTW_WKARD_BUS_WRITE
+	_write_post_cfg = io_priv->io_ops._write_post_cfg;
+	if(NULL != _write_post_cfg) {
+		ret = _write_post_cfg(hal, addr, val);
+	}
+	#endif
+	return ret;
+}
+
+#ifdef CONFIG_SDIO_HCI
+u8 _hal_sd_f0_read8(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u8 r_val = 0x00;
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	u8(*_sd_f0_read8)(struct rtw_hal_com_t *hal, u32 addr);
+
+	_sd_f0_read8 = io_priv->io_ops._sd_f0_read8;
+
+	if (_sd_f0_read8)
+		r_val = _sd_f0_read8(hal, addr);
+
+	return r_val;
+}
+
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+u8 _hal_sd_iread8(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u8 r_val = 0x00;
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	u8(*_sd_iread8)(struct rtw_hal_com_t *hal, u32 addr);
+
+	_sd_iread8 = io_priv->io_ops._sd_iread8;
+
+	if (_sd_iread8)
+		r_val = _sd_iread8(hal, addr);
+	return r_val;
+}
+
+u16 _hal_sd_iread16(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u16 r_val = 0x00;
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	u16(*_sd_iread16)(struct rtw_hal_com_t *hal, u32 addr);
+
+	_sd_iread16 = io_priv->io_ops._sd_iread16;
+
+	if (_sd_iread16)
+		r_val = _sd_iread16(hal, addr);
+	return r_val;
+}
+
+u32 _hal_sd_iread32(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u32 r_val = 0x00;
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	u32(*_sd_iread32)(struct rtw_hal_com_t *hal, u32 addr);
+
+	_sd_iread32 = io_priv->io_ops._sd_iread32;
+
+	if (_sd_iread32)
+		r_val = _sd_iread32(hal, addr);
+	return r_val;
+}
+
+int _hal_sd_iwrite8(struct rtw_hal_com_t *hal, u32 addr, u8 val)
+{
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	int (*_sd_iwrite8)(struct rtw_hal_com_t *hal, u32 addr, u8 val);
+	int ret = -1;
+
+	_sd_iwrite8 = io_priv->io_ops._sd_iwrite8;
+
+	if (_sd_iwrite8)
+		ret = _sd_iwrite8(hal, addr, val);
+
+	return ret;
+}
+
+int _hal_sd_iwrite16(struct rtw_hal_com_t *hal, u32 addr, u16 val)
+{
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	int (*_sd_iwrite16)(struct rtw_hal_com_t *hal, u32 addr, u16 val);
+	int ret = -1;
+
+	_sd_iwrite16 = io_priv->io_ops._sd_iwrite16;
+
+	if (_sd_iwrite16)
+		ret = _sd_iwrite16(hal, addr, val);
+	return ret;
+}
+int _hal_sd_iwrite32(struct rtw_hal_com_t *hal, u32 addr, u32 val)
+{
+	struct hal_io_priv *io_priv = &hal->iopriv;
+	int (*_sd_iwrite32)(struct rtw_hal_com_t *hal, u32 addr, u32 val);
+	int ret = -1;
+
+	_sd_iwrite32 = io_priv->io_ops._sd_iwrite32;
+
+	if (_sd_iwrite32)
+		ret = _sd_iwrite32(hal, addr, val);
+
+	return ret;
+}
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#endif /* CONFIG_SDIO_HCI */
+
+u32 hal_init_io_priv(struct rtw_hal_com_t *hal,
+	void (*set_intf_ops)(struct rtw_hal_com_t *hal, struct hal_io_ops *ops))
+{
+	struct hal_io_priv *iopriv = &hal->iopriv;
+
+	if (set_intf_ops == NULL)
+		return RTW_HAL_STATUS_IO_INIT_FAILURE;
+
+	#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+	_os_mutex_init(hal->drv_priv, &iopriv->sd_indirect_access_mutex);
+	#endif
+	set_intf_ops(hal, &iopriv->io_ops);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+u32 hal_deinit_io_priv(struct rtw_hal_com_t *hal)
+{
+	#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+	struct hal_io_priv *iopriv = &hal->iopriv;
+
+	_os_mutex_init(hal->drv_priv, &iopriv->sd_indirect_access_mutex);
+	#endif
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+/*******************common IO  APIs *******************/
+static inline u32 _bit_shift(u32 mask)
+{
+	u32 i;
+
+	for (i = 0; i <= 31; i++)
+		if (mask & BIT(i))
+			break;
+
+	return i;
+}
+
+u32 hal_read_macreg(struct hal_info_t *hal,
+		u32 offset, u32 bit_mask)
+{
+	u32 val = 0, val32, shift;
+
+	val32 = hal_read32(hal->hal_com, offset);
+	if (bit_mask != 0xFFFFFFFF) {
+		shift = _bit_shift(bit_mask);
+		val = (val32 & bit_mask) >> shift;
+		return val;
+	}
+	else {
+		return val32;
+	}
+}
+void hal_write_macreg(struct hal_info_t *hal,
+		u32 offset, u32 bit_mask, u32 data)
+{
+	u32 val32, shift;
+	
+	if (bit_mask != 0xFFFFFFFF) {
+		val32 = hal_read32(hal->hal_com, offset);
+		shift = _bit_shift(bit_mask);
+		data = ((val32 & (~bit_mask)) | ((data << shift) & bit_mask));
+	}
+	
+	hal_write32(hal->hal_com, offset, data);
+}
+
+u32 hal_read_bbreg(struct hal_info_t *hal,
+		u32 offset, u32 bit_mask)
+{
+	u32 val32 = 0;
+	val32 = rtw_hal_read_bb_reg(hal->hal_com, offset, bit_mask);
+	
+	return val32;
+}
+
+void hal_write_bbreg(struct hal_info_t *hal,
+		u32 offset, u32 bit_mask, u32 data)
+{
+	rtw_hal_write_bb_reg(hal->hal_com, offset, bit_mask, data);
+}
+
+u32 hal_read_rfreg(struct hal_info_t *hal,
+		enum rf_path path, u32 offset, u32 bit_mask)
+{
+	u32 val32 = 0;
+	val32 = rtw_hal_read_rf_reg(hal->hal_com, path, offset, bit_mask);
+#ifdef DBG_IO
+	if (match_rf_read_sniff_ranges(hal->hal_com, path, offset, bit_mask)) {
+		PHL_INFO("DBG_IO hal_read_rfreg(%u, 0x%04x, 0x%08x) read:0x%08x(0x%08x)\n"
+			, path, offset, bit_mask, (val32 << _bit_shift(bit_mask)), val32);
+	}
+#endif
+
+	return val32;
+}
+void hal_write_rfreg(struct hal_info_t *hal,
+		enum rf_path path, u32 offset, u32 bit_mask, u32 data)
+{
+	rtw_hal_write_rf_reg(hal->hal_com, path, offset, bit_mask, data);
+#ifdef DBG_IO
+	if (match_rf_write_sniff_ranges(hal->hal_com, path, offset, bit_mask)) {
+		PHL_INFO("DBG_IO hal_write_rfreg(%u, 0x%04x, 0x%08x) write:0x%08x(0x%08x)\n"
+			, path, offset, bit_mask, (data << _bit_shift(bit_mask)), data);
+	}
+#endif
+}
+
+static u32 _cal_bit_shift(u32 bit_mask)
+{
+	u32 i;
+
+	for (i = 0; i <= 31; i++) {
+		if (((bit_mask >> i) &	0x1) == 1)
+			break;
+	}
+
+	return (i);
+}
+
+void hal_write32_mask(struct rtw_hal_com_t *hal, u16 addr, u32 mask, u32 val)
+{
+	u32 original_value, new_value;
+	u32 bit_shift;
+
+	if (mask == 0xFFFFFFFF) {
+		hal_write32(hal, addr, val);
+	} else {
+		original_value = hal_read32(hal, addr);
+		bit_shift = _cal_bit_shift(mask);
+		new_value = (((original_value) & (~mask)) | ((val << bit_shift) & mask));
+		hal_write32(hal, addr, new_value);
+	}
+}
+
+
+#ifdef DBG_IO
+#define RTW_IO_SNIFF_TYPE_RANGE	0 /* specific address range is accessed */
+#define RTW_IO_SNIFF_TYPE_VALUE	1 /* value match for sniffed range */
+
+struct rtw_io_sniff_ent {
+	u8 chip;
+	u8 hci;
+	u32 addr;
+	u8 type;
+	union {
+		u32 end_addr;
+		struct {
+			u32 mask;
+			u32 val;
+			bool equal;
+		} vm; /* value match */
+	} u;
+	bool trace;
+	char *tag;
+};
+
+#define RTW_IO_SNIFF_RANGE_ENT(_chip, _hci, _addr, _end_addr, _trace, _tag) \
+	{.chip = _chip, .hci = _hci, .addr = _addr, .u.end_addr = _end_addr, .trace = _trace, .tag = _tag, .type = RTW_IO_SNIFF_TYPE_RANGE,}
+
+#define RTW_IO_SNIFF_VALUE_ENT(_chip, _hci, _addr, _mask, _val, _equal, _trace, _tag) \
+	{.chip = _chip, .hci = _hci, .addr = _addr, .u.vm.mask = _mask, .u.vm.val = _val, .u.vm.equal = _equal, .trace = _trace, .tag = _tag, .type = RTW_IO_SNIFF_TYPE_VALUE,}
+
+/* part or all sniffed range is enabled (not all 0) */
+#define RTW_IO_SNIFF_EN_ENT(_chip, _hci, _addr, _mask, _trace, _tag) \
+	{.chip = _chip, .hci = _hci, .addr = _addr, .u.vm.mask = _mask, .u.vm.val = 0, .u.vm.equal = 0, .trace = _trace, .tag = _tag, .type = RTW_IO_SNIFF_TYPE_VALUE,}
+
+/* part or all sniffed range is disabled (not all 1) */
+#define RTW_IO_SNIFF_DIS_ENT(_chip, _hci, _addr, _mask, _trace, _tag) \
+	{.chip = _chip, .hci = _hci, .addr = _addr, .u.vm.mask = _mask, .u.vm.val = 0xFFFFFFFF, .u.vm.equal = 0, .trace = _trace, .tag = _tag, .type = RTW_IO_SNIFF_TYPE_VALUE,}
+
+const struct rtw_io_sniff_ent read_sniff[] = {
+#ifdef DBG_IO_HCI_EN_CHK
+	RTW_IO_SNIFF_EN_ENT(CHIP_WIFI6_MAX, RTW_HCI_SDIO, 0x02, 0x1FC, 1, "SDIO 0x02[8:2] not all 0"),
+	RTW_IO_SNIFF_EN_ENT(CHIP_WIFI6_MAX, RTW_HCI_USB, 0x02, 0x1E0, 1, "USB 0x02[8:5] not all 0"),
+	RTW_IO_SNIFF_EN_ENT(CHIP_WIFI6_MAX, RTW_HCI_PCIE, 0x02, 0x01C, 1, "PCI 0x02[4:2] not all 0"),
+#endif
+#ifdef DBG_IO_SNIFF_EXAMPLE
+	RTW_IO_SNIFF_RANGE_ENT(CHIP_WIFI6_MAX, 0, 0x522, 0x522, 0, "read TXPAUSE"),
+	RTW_IO_SNIFF_DIS_ENT(CHIP_WIFI6_MAX, 0, 0x02, 0x3, 0, "0x02[1:0] not all 1"),
+#endif
+};
+
+const int read_sniff_num = sizeof(read_sniff) / sizeof(struct rtw_io_sniff_ent);
+
+const struct rtw_io_sniff_ent write_sniff[] = {
+#ifdef DBG_IO_HCI_EN_CHK
+	RTW_IO_SNIFF_EN_ENT(CHIP_WIFI6_MAX, RTW_HCI_SDIO, 0x02, 0x1FC, 1, "SDIO 0x02[8:2] not all 0"),
+	RTW_IO_SNIFF_EN_ENT(CHIP_WIFI6_MAX, RTW_HCI_SDIO, 0x02, 0x1E0, 1, "USB 0x02[8:5] not all 0"),
+	RTW_IO_SNIFF_EN_ENT(CHIP_WIFI6_MAX, RTW_HCI_SDIO, 0x02, 0x01C, 1, "PCI 0x02[4:2] not all 0"),
+#endif
+#ifdef DBG_IO_SNIFF_EXAMPLE
+	RTW_IO_SNIFF_RANGE_ENT(CHIP_WIFI6_MAX, 0, 0x522, 0x522, 0, "write TXPAUSE"),
+	RTW_IO_SNIFF_DIS_ENT(CHIP_WIFI6_MAX, 0, 0x02, 0x3, 0, "0x02[1:0] not all 1"),
+#endif
+};
+
+const int write_sniff_num = sizeof(write_sniff) / sizeof(struct rtw_io_sniff_ent);
+
+static bool match_io_sniff_ranges(struct rtw_hal_com_t *hal
+	, const struct rtw_io_sniff_ent *sniff, int i, u32 addr, u16 len)
+{
+
+	/* check if IO range after sniff end address */
+	if (addr > sniff->u.end_addr)
+		return 0;
+
+	return 1;
+}
+
+static bool match_io_sniff_value(struct rtw_hal_com_t *hal
+	, const struct rtw_io_sniff_ent *sniff, int i, u32 addr, u8 len, u32 val)
+{
+	u8 sniff_len;
+	s8 mask_shift;
+	u32 mask;
+	s8 value_shift;
+	u32 value;
+	bool ret = 0;
+
+	/* check if IO range after sniff end address */
+	sniff_len = 4;
+	while (!(sniff->u.vm.mask & (0xFF << ((sniff_len - 1) * 8)))) {
+		sniff_len--;
+		if (sniff_len == 0)
+			goto exit;
+	}
+	if (sniff->addr + sniff_len <= addr)
+		goto exit;
+
+	/* align to IO addr */
+	mask_shift = (sniff->addr - addr) * 8;
+	value_shift = mask_shift + bitshift(sniff->u.vm.mask);
+	if (mask_shift > 0)
+		mask = sniff->u.vm.mask << mask_shift;
+	else if (mask_shift < 0)
+		mask = sniff->u.vm.mask >> -mask_shift;
+	else
+		mask = sniff->u.vm.mask;
+
+	if (value_shift > 0)
+		value = sniff->u.vm.val << value_shift;
+	else if (mask_shift < 0)
+		value = sniff->u.vm.val >> -value_shift;
+	else
+		value = sniff->u.vm.val;
+
+	if ((sniff->u.vm.equal && (mask & val) == (mask & value))
+		|| (!sniff->u.vm.equal && (mask & val) != (mask & value))
+	) {
+		ret = 1;
+		if (0)
+			PHL_INFO(" addr:0x%x len:%u val:0x%x (i:%d sniff_len:%u m_shift:%d mask:0x%x v_shifd:%d value:0x%x equal:%d)\n"
+				, addr, len, val, i, sniff_len, mask_shift, mask, value_shift, value, sniff->u.vm.equal);
+	}
+
+exit:
+	return ret;
+}
+
+static bool match_io_sniff(struct rtw_hal_com_t *hal
+	, const struct rtw_io_sniff_ent *sniff, int i, u32 addr, u8 len, u32 val)
+{
+	bool ret = 0;
+
+	if (sniff->chip != CHIP_WIFI6_MAX
+		&& sniff->chip != hal_get_chip_id(hal))
+		goto exit;
+	/*
+	if (sniff->hci
+		&& !(sniff->hci & hal_get_hci_type(hal)))
+		goto exit;
+	*/
+	if (sniff->addr >= addr + len) /* IO range below sniff start address */
+		goto exit;
+
+	switch (sniff->type) {
+	case RTW_IO_SNIFF_TYPE_RANGE:
+		ret = match_io_sniff_ranges(hal, sniff, i, addr, len);
+		break;
+	case RTW_IO_SNIFF_TYPE_VALUE:
+		if (len == 1 || len == 2 || len == 4)
+			ret = match_io_sniff_value(hal, sniff, i, addr, len, val);
+		break;
+	default:
+		/*_os_warn_on(1);*/
+		break;
+	}
+
+exit:
+	return ret;
+}
+
+u32 match_read_sniff(struct rtw_hal_com_t *hal, u32 addr, u16 len, u32 val)
+{
+	int i;
+	bool trace = 0;
+	u32 match = 0;
+
+	for (i = 0; i < read_sniff_num; i++) {
+		if (match_io_sniff(hal, &read_sniff[i], i, addr, len, val)) {
+			match++;
+			trace |= read_sniff[i].trace;
+			if (read_sniff[i].tag)
+				PHL_INFO("DBG_IO TAG %s\n", read_sniff[i].tag);
+		}
+	}
+
+	/*_os_warn_on(trace);*/
+
+	return match;
+}
+
+u32 match_write_sniff(struct rtw_hal_com_t *hal, u32 addr, u16 len, u32 val)
+{
+	int i;
+	bool trace = 0;
+	u32 match = 0;
+
+	for (i = 0; i < write_sniff_num; i++) {
+		if (match_io_sniff(hal, &write_sniff[i], i, addr, len, val)) {
+			match++;
+			trace |= write_sniff[i].trace;
+			if (write_sniff[i].tag)
+				PHL_INFO("DBG_IO TAG %s\n", write_sniff[i].tag);
+		}
+	}
+
+	/*_os_warn_on(trace);*/
+
+	return match;
+}
+
+struct rf_sniff_ent {
+	u8 path;
+	u16 reg;
+	u32 mask;
+};
+
+struct rf_sniff_ent rf_read_sniff_ranges[] = {
+	/* example for all path addr 0x55 with all RF Reg mask */
+	/* {MAX_RF_PATH, 0x55, bRFRegOffsetMask}, */
+};
+
+struct rf_sniff_ent rf_write_sniff_ranges[] = {
+	/* example for all path addr 0x55 with all RF Reg mask */
+	/* {MAX_RF_PATH, 0x55, bRFRegOffsetMask}, */
+};
+
+int rf_read_sniff_num = sizeof(rf_read_sniff_ranges) / sizeof(struct rf_sniff_ent);
+int rf_write_sniff_num = sizeof(rf_write_sniff_ranges) / sizeof(struct rf_sniff_ent);
+
+bool match_rf_read_sniff_ranges(struct rtw_hal_com_t *hal, u8 path, u32 addr, u32 mask)
+{
+	int i;
+
+	for (i = 0; i < rf_read_sniff_num; i++) {
+		if (rf_read_sniff_ranges[i].path == MAX_RF_PATH ||
+			rf_read_sniff_ranges[i].path == path)
+			if ((addr == rf_read_sniff_ranges[i].reg) &&
+				(mask & rf_read_sniff_ranges[i].mask))
+				return true;
+	}
+
+	return false;
+}
+
+bool match_rf_write_sniff_ranges(struct rtw_hal_com_t *hal,
+					u8 path, u32 addr, u32 mask)
+{
+	int i;
+
+	for (i = 0; i < rf_write_sniff_num; i++) {
+		if (rf_write_sniff_ranges[i].path == MAX_RF_PATH ||
+			rf_write_sniff_ranges[i].path == path)
+			if ((addr == rf_write_sniff_ranges[i].reg) && 
+				(mask & rf_write_sniff_ranges[i].mask))
+				return true;
+	}
+
+	return false;
+}
+
+u8 dbg_hal_read8(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line)
+{
+	u8 val = _hal_read8(hal, addr);
+
+	if (match_read_sniff(hal, addr, 1, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_read8(0x%04x) return 0x%02x\n"
+			, caller, line, addr, val);
+	}
+
+	return val;
+}
+
+u16 dbg_hal_read16(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line)
+{
+	u16 val = _hal_read16(hal, addr);
+
+	if (match_read_sniff(hal, addr, 2, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_read16(0x%04x) return 0x%04x\n"
+			, caller, line, addr, val);
+	}
+
+	return val;
+}
+
+u32 dbg_hal_read32(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line)
+{
+	u32 val = _hal_read32(hal, addr);
+
+	if (match_read_sniff(hal, addr, 4, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_read32(0x%04x) return 0x%08x\n"
+			, caller, line, addr, val);
+	}
+
+	return val;
+}
+
+int dbg_hal_write8(struct rtw_hal_com_t *hal, u32 addr, u8 val, const char *caller, const int line)
+{
+	if (match_write_sniff(hal, addr, 1, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_write8(0x%04x, 0x%02x)\n"
+			, caller, line, addr, val);
+	}
+
+	return _hal_write8(hal, addr, val);
+}
+int dbg_hal_write16(struct rtw_hal_com_t *hal, u32 addr, u16 val, const char *caller, const int line)
+{
+	if (match_write_sniff(hal, addr, 2, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_write16(0x%04x, 0x%04x)\n"
+			, caller, line, addr, val);
+	}
+
+	return _hal_write16(hal, addr, val);
+}
+int dbg_hal_write32(struct rtw_hal_com_t *hal, u32 addr, u32 val, const char *caller, const int line)
+{
+	if (match_write_sniff(hal, addr, 4, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_write32(0x%04x, 0x%08x)\n"
+			, caller, line, addr, val);
+	}
+
+	return _hal_write32(hal, addr, val);
+}
+
+#ifdef CONFIG_SDIO_HCI
+u8 dbg_hal_sd_f0_read8(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line)
+{
+	u8 val = _hal_sd_f0_read8(hal, addr);
+
+#if 0
+	if (match_read_sniff(adapter, addr, 1, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_sd_f0_read8(0x%04x) return 0x%02x\n"
+			, caller, line, addr, val);
+	}
+#endif
+
+	return val;
+}
+
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+u8 dbg_hal_sd_iread8(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line)
+{
+	u8 val = hal_sd_iread8(hal, addr);
+
+	if (match_read_sniff(hal, addr, 1, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_sd_iread8(0x%04x) return 0x%02x\n"
+			, caller, line, addr, val);
+	}
+
+	return val;
+}
+
+u16 dbg_hal_sd_iread16(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line)
+{
+	u16 val = _hal_sd_iread16(hal, addr);
+
+	if (match_read_sniff(hal, addr, 2, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_sd_iread16(0x%04x) return 0x%04x\n"
+			, caller, line, addr, val);
+	}
+
+	return val;
+}
+
+u32 dbg_hal_sd_iread32(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line)
+{
+	u32 val = _hal_sd_iread32(hal, addr);
+
+	if (match_read_sniff(hal, addr, 4, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_sd_iread32(0x%04x) return 0x%08x\n"
+			, caller, line, addr, val);
+	}
+
+	return val;
+}
+
+int dbg_hal_sd_iwrite8(struct rtw_hal_com_t *hal, u32 addr, u8 val, const char *caller, const int line)
+{
+	if (match_write_sniff(hal, addr, 1, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_sd_iwrite8(0x%04x, 0x%02x)\n"
+			, caller, line, addr, val);
+	}
+
+	return _hal_sd_iwrite8(hal, addr, val);
+}
+int dbg_hal_sd_iwrite16(struct rtw_hal_com_t *hal, u32 addr, u16 val, const char *caller, const int line)
+{
+	if (match_write_sniff(hal, addr, 2, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_sd_iwrite16(0x%04x, 0x%04x)\n"
+			, caller, line, addr, val);
+	}
+
+	return _hal_sd_iwrite16(hal, addr, val);
+}
+int dbg_hal_sd_iwrite32(struct rtw_hal_com_t *hal, u32 addr, u32 val, const char *caller, const int line)
+{
+	if (match_write_sniff(hal, addr, 4, val)) {
+		PHL_INFO("DBG_IO %s:%d hal_sd_iwrite32(0x%04x, 0x%08x)\n"
+			, caller, line, addr, val);
+	}
+
+	return _hal_sd_iwrite32(hal, addr, val);
+}
+
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+
+#endif /* CONFIG_SDIO_HCI */
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_io.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_io.h
new file mode 100644
index 000000000000..e0f553376afd
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_io.h
@@ -0,0 +1,123 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_IO_H_
+#define _HAL_IO_H_
+
+/*
+Below is the data structure used by _io_handler
+*/
+
+extern u8 _hal_read8(struct rtw_hal_com_t *hal, u32 addr);
+extern u16 _hal_read16(struct rtw_hal_com_t *hal, u32 addr);
+extern u32 _hal_read32(struct rtw_hal_com_t *hal, u32 addr);
+extern void _hal_read_mem(struct rtw_hal_com_t *hal, u32 addr, u32 cnt, u8 *pmem);
+
+extern int _hal_write8(struct rtw_hal_com_t *hal, u32 addr, u8 val);
+extern int _hal_write16(struct rtw_hal_com_t *hal, u32 addr, u16 val);
+extern int _hal_write32(struct rtw_hal_com_t *hal, u32 addr, u32 val);
+
+
+#ifdef CONFIG_SDIO_HCI
+u8 _hal_sd_f0_read8(struct rtw_hal_com_t *hal, u32 addr);
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+u8 _hal_sd_iread8(struct rtw_hal_com_t *hal, u32 addr);
+u16 _hal_sd_iread16(struct rtw_hal_com_t *hal, u32 addr);
+u32 _hal_sd_iread32(struct rtw_hal_com_t *hal, u32 addr);
+int _hal_sd_iwrite8(struct rtw_hal_com_t *hal, u32 addr, u8 val);
+int _hal_sd_iwrite16(struct rtw_hal_com_t *hal, u32 addr, u16 val);
+int _hal_sd_iwrite32(struct rtw_hal_com_t *hal, u32 addr, u32 val);
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#endif /* CONFIG_SDIO_HCI */
+
+
+#ifdef DBG_IO
+u32 match_read_sniff(struct rtw_hal_com_t *hal, u32 addr, u16 len, u32 val);
+u32 match_write_sniff(struct rtw_hal_com_t *hal, u32 addr, u16 len, u32 val);
+bool match_rf_read_sniff_ranges(struct rtw_hal_com_t *hal, u8 path, u32 addr, u32 mask);
+bool match_rf_write_sniff_ranges(struct rtw_hal_com_t *hal, u8 path, u32 addr, u32 mask);
+
+extern u8 dbg_hal_read8(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line);
+extern u16 dbg_hal_read16(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line);
+extern u32 dbg_hal_read32(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line);
+
+extern int dbg_hal_write8(struct rtw_hal_com_t *hal, u32 addr, u8 val, const char *caller, const int line);
+extern int dbg_hal_write16(struct rtw_hal_com_t *hal, u32 addr, u16 val, const char *caller, const int line);
+extern int dbg_hal_write32(struct rtw_hal_com_t *hal, u32 addr, u32 val, const char *caller, const int line);
+extern int dbg_hal_writeN(struct rtw_hal_com_t *hal, u32 addr , u32 length , u8 *data, const char *caller, const int line);
+
+#ifdef CONFIG_SDIO_HCI
+u8 dbg_hal_sd_f0_read8(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line);
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+u8 dbg_hal_sd_iread8(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line);
+u16 dbg_hal_sd_iread16(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line);
+u32 dbg_hal_sd_iread32(struct rtw_hal_com_t *hal, u32 addr, const char *caller, const int line);
+int dbg_hal_sd_iwrite8(struct rtw_hal_com_t *hal, u32 addr, u8 val, const char *caller, const int line);
+int dbg_hal_sd_iwrite16(struct rtw_hal_com_t *hal, u32 addr, u16 val, const char *caller, const int line);
+int dbg_hal_sd_iwrite32(struct rtw_hal_com_t *hal, u32 addr, u32 val, const char *caller, const int line);
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#endif /* CONFIG_SDIO_HCI */
+
+#define hal_read8(hal, addr) dbg_hal_read8((hal), (addr), __FUNCTION__, __LINE__)
+#define hal_read16(hal, addr) dbg_hal_read16((hal), (addr), __FUNCTION__, __LINE__)
+#define hal_read32(hal, addr) dbg_hal_read32((hal), (addr), __FUNCTION__, __LINE__)
+
+#define  hal_write8(hal, addr, val) dbg_hal_write8((hal), (addr), (val), __FUNCTION__, __LINE__)
+#define  hal_write16(hal, addr, val) dbg_hal_write16((hal), (addr), (val), __FUNCTION__, __LINE__)
+#define  hal_write32(hal, addr, val) dbg_hal_write32((hal), (addr), (val), __FUNCTION__, __LINE__)
+
+
+#ifdef CONFIG_SDIO_HCI
+#define hal_sd_f0_read8(hal, addr) dbg_hal_sd_f0_read8((hal), (addr), __func__, __LINE__)
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+#define hal_sd_iread8(hal, addr) dbg_hal_sd_iread8((hal), (addr), __func__, __LINE__)
+#define hal_sd_iread16(hal, addr) dbg_hal_sd_iread16((hal), (addr), __func__, __LINE__)
+#define hal_sd_iread32(hal, addr) dbg_hal_sd_iread32((hal), (addr), __func__, __LINE__)
+#define hal_sd_iwrite8(hal, addr, val) dbg_hal_sd_iwrite8((hal), (addr), (val), __func__, __LINE__)
+#define hal_sd_iwrite16(hal, addr, val) dbg_hal_sd_iwrite16((hal), (addr), (val), __func__, __LINE__)
+#define hal_sd_iwrite32(hal, addr, val) dbg_hal_sd_iwrite32((hal), (addr), (val), __func__, __LINE__)
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#endif /* CONFIG_SDIO_HCI */
+
+#else /* DBG_IO */
+#define hal_read8(hal, addr) _hal_read8((hal), (addr))
+#define hal_read16(hal, addr) _hal_read16((hal), (addr))
+#define hal_read32(hal, addr) _hal_read32((hal), (addr))
+
+#define  hal_write8(hal, addr, val) _hal_write8((hal), (addr), (val))
+#define  hal_write16(hal, addr, val) _hal_write16((hal), (addr), (val))
+#define  hal_write32(hal, addr, val) _hal_write32((hal), (addr), (val))
+
+
+#ifdef CONFIG_SDIO_HCI
+#define hal_sd_f0_read8(hal, addr) _hal_sd_f0_read8((hal), (addr))
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+#define hal_sd_iread8(hal, addr) _hal_sd_iread8((hal), (addr))
+#define hal_sd_iread16(hal, addr) _hal_sd_iread16((hal), (addr))
+#define hal_sd_iread32(hal, addr) _hal_sd_iread32((hal), (addr))
+#define hal_sd_iwrite8(hal, addr, val) _hal_sd_iwrite8((hal), (addr), (val))
+#define hal_sd_iwrite16(hal, addr, val) _hal_sd_iwrite16((hal), (addr), (val))
+#define hal_sd_iwrite32(hal, addr, val) _hal_sd_iwrite32((hal), (addr), (val))
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#endif /* CONFIG_SDIO_HCI */
+
+#endif /* DBG_IO */
+
+void hal_write32_mask(struct rtw_hal_com_t *hal, u16 addr, u32 mask, u32 val);
+
+u32 hal_init_io_priv(struct rtw_hal_com_t *hal,
+void (*set_intf_ops)(struct rtw_hal_com_t *hal, struct hal_io_ops *ops));
+u32 hal_deinit_io_priv(struct rtw_hal_com_t *hal);
+
+#endif /* _RTW_IO_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ld_file.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ld_file.c
new file mode 100644
index 000000000000..883fd741b8ce
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ld_file.c
@@ -0,0 +1,2536 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_LD_FILE_C_
+#include "hal_headers.h"
+#include "hal_ld_file.h"
+
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+static enum rtw_hal_status
+_hal_general_para_alloc(struct rtw_phl_com_t *phl_com, void *para_info_t)
+{
+	struct rtw_para_info_t *para_info = para_info_t;
+
+	if (para_info->para_src == RTW_PARA_SRC_EXTNAL) {
+		u32 buf_sz = MAX_HWCONFIG_FILE_CONTENT;
+		void *drv = phlcom_to_drvpriv(phl_com);
+
+		if (!para_info->para_data)
+			para_info->para_data = _os_mem_alloc(drv, buf_sz * sizeof(u32));
+
+		if (!para_info->para_data) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_,
+				"%s:: allocmem fail\n", __FUNCTION__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+		return RTW_HAL_STATUS_SUCCESS;
+	} else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+static void
+_hal_general_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_info_t *para_info)
+{
+	u32 buf_sz = MAX_HWCONFIG_FILE_CONTENT;
+	void *drv = phlcom_to_drvpriv(phl_com);
+
+	if (para_info->para_data)
+		_os_mem_free(drv, para_info->para_data, buf_sz * sizeof(u32));
+
+	para_info->para_data = NULL;
+	para_info->para_data_len = 0;
+	para_info->loaded = false;
+}
+
+static int
+_hal_parse_phyreg(struct hal_info_t *hal_info, void *para_info_t, u8 *psrc_buf, u32 buflen)
+{
+	struct rtw_para_info_t *para_info = (struct rtw_para_info_t *)para_info_t;
+	u32 *pdest_buf = para_info->para_data;
+	char	*sz_line, *ptmp;
+	u32 dest_buf_idx = 0;
+	u32	u4bregoffset, u4bregvalue, u4bmove;
+
+	ptmp = (char *)psrc_buf;
+
+	for (sz_line = hal_getLinefrombuffer(ptmp); sz_line != NULL; sz_line = hal_getLinefrombuffer(ptmp)) {
+		if (!hal_is_comment_string(sz_line)) {
+			/* Get 1st hex value as register offset. */
+			if (hal_get_hexvalue_fromstring(sz_line, &u4bregoffset, &u4bmove)) {
+
+				pdest_buf[dest_buf_idx] = u4bregoffset;
+				dest_buf_idx++;
+
+				if (u4bregoffset == 0xffff) {
+					/* Ending. */
+					pdest_buf[dest_buf_idx] = 0xffff;
+					PHL_INFO(" dest_buf_len: %d \n", dest_buf_idx);
+					break;
+				}
+				/* Get 2nd hex value as register value. */
+				sz_line += u4bmove;
+				if (hal_get_hexvalue_fromstring(sz_line, &u4bregvalue, &u4bmove)) {
+					/* PHL_INFO("[BB-ADDR]%03lX=%08lX\n", u4bregoffset, u4bregvalue); */
+					pdest_buf[dest_buf_idx] = u4bregvalue;
+					dest_buf_idx++;
+				}
+			}
+		}
+	}
+#if 0
+	{ /*debug dump buf*/
+		u32 i = 0;
+		for (i = 0; i <= dest_buf_idx; i+=2)
+			PHL_INFO("[BB-ADDR] 0x%x :[0x%x]\n", pdest_buf[i], pdest_buf[i+1]);
+	}
+#endif
+	return dest_buf_idx;
+}
+
+static enum rtw_hal_status
+_hal_pwr_byrate_para_alloc(struct rtw_phl_com_t *phl_com, void *para_info_t)
+{
+	struct rtw_para_info_t *para_info = para_info_t;
+
+	if (para_info->para_src == RTW_PARA_SRC_EXTNAL)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+static void
+_hal_pwr_byrate_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_info_t *para_info, bool tmp_data_only)
+{
+	if (tmp_data_only)
+		return;
+
+	para_info->loaded = false;
+}
+
+static u32 phypg_combvalue_fromstring(		char *sz_line)
+{
+	u32	u4bmove;
+	u32 combvalue = 0;
+	u8	integer = 0, fraction = 0;
+	u8 txgi_pdbm = 4;
+
+	if (hal_get_fractionvalue_fromstring(sz_line, &integer, &fraction, &u4bmove))
+		sz_line += u4bmove;
+	else
+		goto exit;
+
+	combvalue |= ((integer * txgi_pdbm) + (fraction * txgi_pdbm / 100));
+
+	if (hal_get_fractionvalue_fromstring(sz_line, &integer, &fraction, &u4bmove))
+		sz_line += u4bmove;
+	else
+		goto exit;
+
+	combvalue <<= 8;
+	combvalue |= ((integer * txgi_pdbm) + (fraction * txgi_pdbm / 100));
+
+	if (hal_get_fractionvalue_fromstring(sz_line, &integer, &fraction, &u4bmove))
+		sz_line += u4bmove;
+	else
+		goto exit;
+
+	combvalue <<= 8;
+	combvalue |= ((integer * txgi_pdbm) + (fraction * txgi_pdbm / 100));
+
+	if (hal_get_fractionvalue_fromstring(sz_line, &integer, &fraction, &u4bmove))
+		sz_line += u4bmove;
+	else
+		goto exit;
+
+	combvalue <<= 8;
+	combvalue |= ((integer * txgi_pdbm) + (fraction * txgi_pdbm / 100));
+
+	/*PHL_DBG("[combvalue 4] 0x%x , hex = %x\n", combvalue , integer);*/
+exit:
+	return combvalue;
+}
+
+static void _hal_txpwr_by_rate_store_from_external_single(struct hal_info_t *hal_info
+	, struct hal_txpwr_byrate_t *data)
+{
+	rtw_hal_rf_power_by_rate_store_to_array(hal_info,
+		data->band, data->nss, data->rs, data->vals);
+}
+
+static int
+_hal_parse_txpwr_by_rate(struct hal_info_t *hal_info, void *para_info_t, u8 *psrc_buf, u32 buflen)
+{
+	char	*sz_line, *ptmp;
+	u8 band = 0, nss, rate_section;
+	u8 phy_reg_pg_version = 0;
+	u8 phy_reg_pg_value_type = 0;
+	u32 pwrhexval = 0;
+	u32 line_idx = 0;
+	u32 buf_idx = 0;
+	bool firstline = true;
+	struct hal_txpwr_byrate_t data;
+
+	ptmp = (char*)psrc_buf;
+	for (sz_line = hal_getLinefrombuffer(ptmp); sz_line != NULL; sz_line = hal_getLinefrombuffer(ptmp)) {
+		if (hal_is_allspace_tab(sz_line, sizeof(*sz_line)))
+			continue;
+
+		if (!hal_is_comment_string(sz_line)) {
+			if (firstline) {
+				if (_os_strncmp(sz_line, "#[v2]", 5) == 0
+					|| _os_strncmp(sz_line, "#[v3]", 5) == 0) {
+					phy_reg_pg_version = sz_line[3] - '0'; /* store pg version*/
+					PHL_INFO("phy_reg_pg_version %d\n", phy_reg_pg_version);
+				} else {
+					PHL_ERR("The format in PHY_REG_PG are invalid %s\n", sz_line);
+					goto exit;
+				}
+				if (_os_strncmp(sz_line + 5, "[Exact]#", 8) == 0) {
+					phy_reg_pg_value_type = 1; /* store PHY_REG_PG_EXACT_VALUE*/
+					firstline = false;
+					PHL_INFO("REG_PG_EXACT_VALUE\n");
+					continue;
+				} else {
+					PHL_ERR("The values in PHY_REG_PG are invalid %s\n", sz_line);
+					goto exit;
+				}
+			}
+			if (_os_strncmp(sz_line , "#[2.4G]#", 8) == 0) {
+				band = BAND_ON_24G;
+				continue;
+			}
+			if (_os_strncmp(sz_line , "#[5G]#", 6) == 0) {
+				band = BAND_ON_5G;
+				continue;
+			}
+			if (_os_strncmp(sz_line , "#[6G]#", 6) == 0) {
+				band = BAND_ON_6G;
+				continue;
+			}
+			if ((_os_strncmp(sz_line , "#[START]#", 9) == 0) || (_os_strncmp(sz_line , "#[END]#", 9) == 0) ) {
+					continue;
+			}
+			if (_os_strncmp(sz_line , "[Nss1]", 6) == 0) {
+				line_idx = 0;
+				pwrhexval = 0;
+
+				nss = PHYPG_1SS;
+				line_idx += 6;
+				PHL_INFO("[Line]%s , line_idx %d \n", sz_line, line_idx);
+
+				while (sz_line[line_idx] == ' ' || sz_line[line_idx] == '\0' || sz_line[line_idx] == '\t')
+					++line_idx;
+
+				sz_line += line_idx;
+				if (_os_strncmp(sz_line , "11M_1M", 6) == 0) {
+					rate_section = CCK_11M_1M;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "18M_6M", 6) == 0) {
+					rate_section = OFDM_18M_6M;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "54M_24M", 7) == 0) {
+					rate_section = OFDM_54M_24M;
+					sz_line += 7;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "MCS3_0", 6) == 0) {
+					rate_section = HE1SS_MCS3_0;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "MCS7_4", 6) == 0) {
+					rate_section = HE1SS_MCS7_4;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "MCS11_8", 6) == 0) {
+					rate_section = HE1SS_MCS11_8;
+					sz_line += 7;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "DCM4_0", 6) == 0) {
+					rate_section = HE1SS_DCM4_0;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else {
+					PHL_ERR("nss:%u, invalid rate section:%s\n", nss, sz_line);
+					goto exit;
+				}
+			} else if (_os_strncmp(sz_line , "[Nss2]", 6) == 0) {
+				line_idx = 0;
+				pwrhexval = 0;
+
+				nss = PHYPG_2SS;
+				PHL_INFO("[Line]%s , line_idx %d \n", sz_line, line_idx);
+
+				line_idx += 6;
+				while (sz_line[line_idx] == ' ' || sz_line[line_idx] == '\0' || sz_line[line_idx] == '\t')
+					++line_idx;
+
+				sz_line += line_idx;
+				if (_os_strncmp(sz_line , "MCS3_0", 6) == 0) {
+					rate_section = HE2SS_MCS3_0;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "MCS7_4", 6) == 0) {
+					rate_section = HE2SS_MCS7_4;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "MCS11_8", 6) == 0) {
+					rate_section = HE2SS_MCS11_8;
+					sz_line += 7;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else if (_os_strncmp(sz_line , "DCM4_0", 6) == 0) {
+					rate_section = HE2SS_DCM4_0;
+					sz_line += 6;
+					PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+					pwrhexval = phypg_combvalue_fromstring(sz_line);
+					PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+				} else {
+					PHL_ERR("nss:%u, invalid rate section:%s\n", nss, sz_line);
+					goto exit;
+				}
+			} else if (_os_strncmp(sz_line , "[Offset]", 8) == 0) {
+				line_idx = 0;
+				pwrhexval = 0;
+
+				nss = PHYPG_OFFSET;
+				PHL_INFO("[Line]%s , line_idx %d \n", sz_line, line_idx);
+
+				line_idx += 8;
+				while (sz_line[line_idx] == ' ' || sz_line[line_idx] == '\t' || sz_line[line_idx] == '\0')
+					++line_idx;
+
+				sz_line += line_idx;
+				if (band == BAND_ON_24G) {
+					if (_os_strncmp(sz_line , "AllRate2_1", 10) == 0) {
+						rate_section = Legacy_AllRate2_1;
+						sz_line += 10;
+						PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+						pwrhexval = phypg_combvalue_fromstring(sz_line);
+						PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+					} else if (_os_strncmp(sz_line , "AllRate2_2", 10) == 0) {
+						rate_section = HE_AllRate2_2;
+						sz_line += 10;
+						PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+						pwrhexval = phypg_combvalue_fromstring(sz_line);
+						PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+					} else {
+						PHL_ERR("nss:%u, invalid rate section:%s\n", nss, sz_line);
+						goto exit;
+					}
+				} else if (band == BAND_ON_5G) {
+					if (_os_strncmp(sz_line , "AllRate5_1", 10) == 0) {
+						rate_section = OFDM_AllRate5_1; /* 5G band all rate only OFDM upper */
+						sz_line += 10;
+						PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+						pwrhexval = phypg_combvalue_fromstring(sz_line);
+						PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+					} else {
+						PHL_ERR("nss:%u, invalid rate section:%s\n", nss, sz_line);
+						goto exit;
+					}
+				} else if (band == BAND_ON_6G) {
+					if (_os_strncmp(sz_line , "AllRate6_1", 10) == 0) {
+						rate_section = OFDM_AllRate6_1; /* 6G band all rate only OFDM upper */
+						sz_line += 10;
+						PHL_INFO("[Line]%s, line_idx %d \n", sz_line, line_idx);
+						pwrhexval = phypg_combvalue_fromstring(sz_line);
+						PHL_INFO("combine Value Get Pwr hex Val = %x\n", pwrhexval);
+					} else {
+						PHL_ERR("nss:%u, invalid rate section:%s\n", nss, sz_line);
+						goto exit;
+					}
+				} else {
+					PHL_ERR("nss:%u, invalid band:%u\n", nss, band);
+					goto exit;
+				}
+			} else {
+				PHL_ERR("invalid nss:%s\n", sz_line);
+				goto exit;
+			}
+
+			data.band = band;
+			data.nss = nss;
+			data.rs = rate_section;
+			data.vals = pwrhexval;
+
+			_hal_txpwr_by_rate_store_from_external_single(hal_info, &data);
+			buf_idx++;
+		}
+	}
+	return buf_idx;
+
+exit:
+	return 0;
+}
+
+static int
+_hal_parse_radio(struct hal_info_t *hal_info, void *para_info_t, u8 *psrc_buf, u32 buflen)
+{
+	struct rtw_para_info_t *para_info = (struct rtw_para_info_t *)para_info_t;
+	u32 *pdest_buf = para_info->para_data;
+	char	*sz_line, *ptmp;
+	u32 dest_buf_idx = 0;
+	u32 u4bregoffset, u4bregvalue, u4bmove;
+
+	ptmp = (char *)psrc_buf;
+
+	for (sz_line = hal_getLinefrombuffer(ptmp); sz_line != NULL; sz_line = hal_getLinefrombuffer(ptmp)) {
+		if (!hal_is_comment_string(sz_line)) {
+			/* Get 1st hex value as register offset. */
+			if (hal_get_hexvalue_fromstring(sz_line, &u4bregoffset, &u4bmove)) {
+
+				pdest_buf[dest_buf_idx] = u4bregoffset;
+				dest_buf_idx++;
+
+				if (u4bregoffset == 0xffff) {
+					/* Ending. */
+					pdest_buf[dest_buf_idx] = 0xffff;
+					PHL_INFO(" dest_buf_len: %d \n", dest_buf_idx);
+					break;
+				}
+				/* Get 2nd hex value as register value. */
+				sz_line += u4bmove;
+				if (hal_get_hexvalue_fromstring(sz_line, &u4bregvalue, &u4bmove)) {
+					/*PHL_INFO("[RF-ADDR]%03lx=%08lx\n", u4bregoffset, u4bregvalue);*/
+					pdest_buf[dest_buf_idx] = u4bregvalue;
+					dest_buf_idx++;
+				}
+			}
+		}
+	}
+#if 0
+	{ /*debug dump buf*/
+		u32 i = 0;
+		for (i = 0; i <= dest_buf_idx; i+=2)
+			PHL_INFO("[Radio-ADDR] 0x%x :[0x%x]\n", pdest_buf[i], pdest_buf[i+1]);
+	}
+#endif
+	return dest_buf_idx;
+}
+
+static char *
+hal_phy_get_ext_regd_name_pos(struct rtw_para_pwrlmt_info_t *para_info, u8 idx)
+{
+#ifdef PHL_DYNAMIC_ALLOC_EXT_REGD_NAME
+	if (para_info->band == BAND_ON_6G)
+		return (((struct ext_regd_6g *)(para_info->ext_regd_name)) + idx)->name;
+	else
+		return (((struct ext_regd *)(para_info->ext_regd_name)) + idx)->name;
+#else
+	return para_info->ext_regd_name[idx];
+#endif
+}
+
+static char *
+hal_phy_get_ext_regd_name(struct rtw_para_pwrlmt_info_t *para_info, u8 idx)
+{
+	if (idx > para_info->ext_regd_arridx)
+		return NULL;
+
+	return hal_phy_get_ext_regd_name_pos(para_info, idx);
+}
+
+static void
+hal_phy_set_ext_regd_name(struct rtw_para_pwrlmt_info_t *para_info, u8 idx,
+			const char *regd_name)
+{
+	char *pos;
+	u8 max_size = para_info->band == BAND_ON_6G ? regd_name_max_size_6g : regd_name_max_size;
+
+	if (idx >= max_size) {
+		_os_warn_on(1);
+		return;
+	}
+
+	pos = hal_phy_get_ext_regd_name_pos(para_info, idx);
+
+	if (regd_name) {
+		u8 max_len;
+
+		if (para_info->band == BAND_ON_6G)
+			max_len = regd_name_max_len_6g;
+		else 
+			max_len = regd_name_max_len;
+		_os_strncpy(pos, regd_name, max_len - 1);
+	} else
+		*pos = 0;
+}
+
+static int
+hal_phy_find_ext_regd_num(struct rtw_para_pwrlmt_info_t *para_info,
+			const char *regd_name)
+{
+	u16 i = 0;
+
+	for (i = 0; i <= para_info->ext_regd_arridx ; i++) {
+		if (_os_strcmp(regd_name, hal_phy_get_ext_regd_name(para_info, (u8)i)) == 0) {
+			return i;
+		}
+	}
+	return -1;
+}
+
+static void
+_hal_add_ext_reg_codemap(void *d,
+			struct rtw_para_pwrlmt_info_t *para_info,
+			const char *country,
+			u16 		domain,
+			const char *regd_name,
+			u32 		nlen)
+{
+	struct pwrlmt_ext_regd_map *pregd_codemap;
+	struct pwrlmt_ext_regd_map *ent;
+
+	if (!regd_name || !nlen) {
+		PHL_ERR("regd_name || nlen Null\n");
+		goto exit;
+	}
+
+	pregd_codemap = (struct pwrlmt_ext_regd_map *)para_info->ext_reg_codemap;
+	ent = &pregd_codemap[para_info->ext_reg_map_num];
+
+	if (!regd_name || _os_strlen((u8*)regd_name) >= regd_name_max_len) {
+		PHL_ERR("reg_name = [%s]\n", ent->reg_name);
+		goto exit;
+	}
+
+	_os_mem_cpy(d, ent->reg_name, (void*)regd_name, nlen);
+	PHL_INFO("store reg_name = [%s]\n", ent->reg_name);
+
+	if (domain != 0)
+		ent->domain = domain;
+	else
+		ent->domain = 0xffff;
+	PHL_INFO("Store DomainCode = [0x%x]\n", ent->domain);
+
+	if (country) {
+		_os_mem_cpy(d, ent->country, (void*)country, 2);
+		PHL_INFO("Store country = [%c%c]\n", ent->country[0], ent->country[1]);
+	} else
+		_os_mem_set(d, ent->country, '\0', 2);
+
+	para_info->ext_reg_map_num++;
+exit:
+	return;
+}
+
+/*
+* @@Ver=2.0
+* or
+* @@DomainCode=0x28, Regulation=C6
+* or
+* @@CountryCode=GB, Regulation=C7
+*/
+static u8
+parse_reg_exc_config(void *drv_priv,
+		char *sz_line ,
+		struct rtw_para_pwrlmt_info_t *para_info)
+{
+#define VER_PREFIX "Ver="
+#define DOMAIN_PREFIX "DomainCode=0x"
+#define COUNTRY_PREFIX "CountryCode="
+#define REG_PREFIX "Regulation="
+
+	const u8 ver_prefix_len = (u8)_os_strlen((u8*)VER_PREFIX);
+	const u8 domain_prefix_len = (u8)_os_strlen((u8*)DOMAIN_PREFIX);
+	const u8 country_prefix_len = (u8)_os_strlen((u8*)COUNTRY_PREFIX);
+	const u8 reg_prefix_len = (u8)_os_strlen((u8*)REG_PREFIX);
+	u32 i, i_val_s, i_val_e;
+	u32 j;
+	u16 domain = 0;
+	char *country = NULL;
+	u8 parse_reg = 0;
+
+	if (sz_line[0] != '@' || sz_line[1] != '@')
+		return false;
+
+	i = 2;
+	if (_os_strncmp(sz_line + i, VER_PREFIX, ver_prefix_len) == 0)
+		; /* nothing to do */
+	else if (_os_strncmp(sz_line + i, DOMAIN_PREFIX, domain_prefix_len) == 0) {
+		/* get string after domain prefix to ',' */
+		i += domain_prefix_len;
+		i_val_s = i;
+		while (sz_line[i] != ',') {
+			if (sz_line[i] == '\0')
+				return false;
+			i++;
+		}
+		i_val_e = i;
+
+		/* check if all hex */
+		for (j = i_val_s; j < i_val_e; j++)
+			if (hal_ishexdigit(sz_line[j]) == false)
+				return false;
+
+		/* get value from hex string */
+		if (_os_sscanf(sz_line + i_val_s, "%hx", &domain) != 1)
+			return false;
+
+		parse_reg = 1;
+	} else if (_os_strncmp(sz_line + i, COUNTRY_PREFIX, country_prefix_len) == 0) {
+		/* get string after country prefix to ',' */
+		i += country_prefix_len;
+		i_val_s = i;
+		while (sz_line[i] != ',') {
+			if (sz_line[i] == '\0')
+				return false;
+			i++;
+		}
+		i_val_e = i;
+
+		if (i_val_e - i_val_s != 2)
+			return false;
+
+		/* check if all alpha */
+		for (j = i_val_s; j < i_val_e; j++)
+			if (hal_is_alpha(sz_line[j]) == false)
+				return false;
+
+		country = sz_line + i_val_s;
+
+		parse_reg = 1;
+
+	} else
+		return false;
+
+	if (parse_reg) {
+		/* move to 'R' */
+		while (sz_line[i] != 'R') {
+			if (sz_line[i] == '\0')
+				return false;
+			i++;
+		}
+
+		/* check if matching regulation prefix */
+		if (_os_strncmp(sz_line + i, REG_PREFIX, reg_prefix_len) != 0)
+			return false;
+
+		/* get string after regulation prefix ending with space */
+		i += reg_prefix_len;
+		i_val_s = i;
+		while (sz_line[i] != ' ' && sz_line[i] != '\t' && sz_line[i] != '\0')
+			i++;
+
+		if (i == i_val_s)
+			return false;
+
+		_hal_add_ext_reg_codemap(drv_priv, para_info,
+					country, domain,
+					sz_line + i_val_s, i - i_val_s);
+	}
+
+	return true;
+}
+
+static enum rtw_hal_status
+_hal_pwrlmt_para_alloc(struct rtw_phl_com_t *phl_com, void *para_info_t)
+{
+	struct rtw_para_pwrlmt_info_t *para_info = para_info_t;
+
+	if (para_info->para_src == RTW_PARA_SRC_EXTNAL) {
+		u32 buf_sz = MAX_NUM_PWRLMT_EXT_REGD_MAP *
+			sizeof(struct pwrlmt_ext_regd_map);
+		void *drv = phlcom_to_drvpriv(phl_com);
+		u8 para_regd_str_arridx;
+		enum band_type band = para_info->band;
+		const char * const *_para_regd_str =
+			rtw_hal_rf_get_predef_pw_lmt_regu_type_str_array(
+				band, &para_regd_str_arridx);
+		u8 i = 0;
+
+		if(_para_regd_str == NULL)
+			return RTW_HAL_STATUS_FAILURE;
+
+#ifdef PHL_DYNAMIC_ALLOC_EXT_REGD_NAME
+		if (!para_info->ext_regd_name) {
+			if (band == BAND_ON_6G) {
+				para_info->ext_regd_name_size = sizeof(struct ext_regd_6g) * regd_name_max_size_6g;
+			} else {
+				para_info->ext_regd_name_size = sizeof(struct ext_regd) * regd_name_max_size;
+			}
+
+			para_info->ext_regd_name = _os_mem_alloc(drv, para_info->ext_regd_name_size);
+		}
+
+		if (!para_info->ext_regd_name) {
+			para_info->ext_regd_name_size = 0;
+			PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_,
+				"%s::ext_regd_name allocmem fail\n",__FUNCTION__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+#endif
+
+		if (!para_info->ext_reg_codemap)
+			para_info->ext_reg_codemap = _os_mem_alloc(drv, buf_sz);
+		if (!para_info->ext_reg_codemap) {
+#ifdef PHL_DYNAMIC_ALLOC_EXT_REGD_NAME
+			_os_mem_free(drv, para_info->ext_regd_name, para_info->ext_regd_name_size);
+			para_info->ext_regd_name = NULL;
+			para_info->ext_regd_name_size = 0;
+#endif
+			PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_,
+				"%s::ext_reg_codemap allocmem fail\n",__FUNCTION__);
+			return RTW_HAL_STATUS_FAILURE;
+		}
+
+		para_info->ext_regd_arridx = 0;
+		for (i = 0; i < para_regd_str_arridx ; i++) {
+			hal_phy_set_ext_regd_name(para_info, i, _para_regd_str[i]);
+			para_info->ext_regd_arridx++;
+			PHL_INFO(" prepare ext_regd_name[%d] = %s\n",
+				i , hal_phy_get_ext_regd_name(para_info, i));
+		}
+		para_info->ext_regd_arridx--; /* array idx = number - 1 */
+
+		return RTW_HAL_STATUS_SUCCESS;
+	} else
+		return RTW_HAL_STATUS_FAILURE;
+}
+
+static void
+_hal_pwrlmt_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_pwrlmt_info_t *para_info, bool tmp_data_only)
+{
+	u32 buf_sz = MAX_NUM_PWRLMT_EXT_REGD_MAP * sizeof(struct pwrlmt_ext_regd_map);
+	void *drv = phlcom_to_drvpriv(phl_com);
+
+	if (tmp_data_only)
+		return;
+
+	para_info->loaded = false;
+
+#ifdef PHL_DYNAMIC_ALLOC_EXT_REGD_NAME
+	if (para_info->ext_regd_name)
+		_os_mem_free(drv, para_info->ext_regd_name, para_info->ext_regd_name_size);
+	para_info->ext_regd_name = NULL;
+	para_info->ext_regd_name_size = 0;
+#endif
+	para_info->ext_regd_arridx = 0;
+
+	if (para_info->ext_reg_codemap)
+		_os_mem_free(drv, para_info->ext_reg_codemap, buf_sz);
+	para_info->ext_reg_codemap = NULL;
+	para_info->ext_reg_map_num = 0;
+}
+
+static void _hal_txpwr_lmt_store_from_external_single(struct hal_info_t *hal_info
+	, struct hal_txpwr_lmt_t *data)
+{
+	rtw_hal_rf_set_pw_lmt_regu_table_exist(hal_info,
+		data->band, data->reg);
+
+	rtw_hal_rf_power_limit_shape_store_to_array(hal_info,
+		data->reg, data->band, data->bw, data->rs,
+		data->ntx, data->bf, data->tx_shap_idx);
+
+	rtw_hal_rf_power_limit_store_to_array(hal_info,
+		data->reg, data->band, data->bw, data->rs,
+		data->ntx, data->bf, data->ch, data->val);
+}
+
+static void _hal_txpwr_lmt_ru_store_from_external_single(struct hal_info_t *hal_info
+	, struct hal_txpwr_lmt_ru_t *data)
+{
+	rtw_hal_rf_power_limit_ru_shape_store_to_array(hal_info,
+		data->band, data->rubw, data->ntx, data->rs,
+		data->reg, data->tx_shap_idx);
+
+	rtw_hal_rf_power_limit_ru_store_to_array(hal_info,
+		data->band, data->rubw, data->ntx, data->rs,
+		data->reg, data->ch, data->val);
+}
+
+static void hal_phy_store_tx_power_limit(struct hal_info_t *hal_info,
+		u8				reg_idx,
+		u8				*sband,
+		u8				*sbandwidth,
+		u8				*sratesection,
+		u8				*sbf,
+		u8				*sntx,
+		u8				*schannel,
+		u8				*spwrlimit,
+		u8				*spshape_idx)
+{
+	struct hal_txpwr_lmt_t data = {0};
+
+	if (hal_get_u1bint_fromstr_indec((char *)schannel, &data.ch) == false
+		|| hal_get_s1bint_fromstr_indec((char *)spwrlimit, &data.val) == false) {
+		PHL_INFO("Illegal index of power limit table [ch %s][val %s]\n",
+			 (char *)schannel, (char *)spwrlimit);
+		return;
+	}
+
+	if (spshape_idx != NULL) {
+		if (hal_get_u1bint_fromstr_indec((char *)spshape_idx, &data.tx_shap_idx) == false) {
+			PHL_INFO("Illegal index of pshape idx [val %s]\n", (char *)spshape_idx);
+			return;
+		}
+	}
+
+	if (_os_strncmp((char *)sratesection, (const char *)"CCK", 3) == 0)
+		data.rs = _PW_LMT_RS_CCK;
+	else if (_os_strncmp((char *)sratesection, (const char *)"OFDM", 4) == 0)
+		data.rs = _PW_LMT_RS_OFDM;
+	else if (_os_strncmp((char *)sratesection, (const char *)"HT", 2) == 0)
+		data.rs = _PW_LMT_RS_HT;
+	else if (_os_strncmp((char *)sratesection, (const char *)"VHT", 3) == 0)
+		data.rs = _PW_LMT_RS_VHT;
+	else if (_os_strncmp((char *)sratesection, (const char *)"HE", 2) == 0)
+		data.rs = _PW_LMT_RS_HE;
+	else {
+		PHL_INFO("Wrong rate section:%s\n", (char *)sratesection);
+		return;
+	}
+
+	if (_os_strncmp((char *)sntx, (const char *)"1T", 2) == 0)
+		data.ntx = _PW_LMT_PH_1T;
+	else if (_os_strncmp((char *)sntx, (const char *)"2T", 2) == 0)
+		data.ntx = _PW_LMT_PH_2T;
+	else if (_os_strncmp((char *)sntx, (const char *)"3T", 2) == 0)
+		data.ntx = _PW_LMT_PH_3T;
+	else if (_os_strncmp((char *)sntx, (const char *)"4T", 2) == 0)
+		data.ntx = _PW_LMT_PH_4T;
+	else {
+		PHL_INFO("Wrong tx num:%s\n", (char *)sntx);
+		return;
+	}
+
+	if (_os_strncmp((char *)sbandwidth, (const char *)"20M", 3) == 0)
+		data.bw = _PW_LMT_BW_20M;
+	else if (_os_strncmp((char *)sbandwidth, (const char *)"40M", 3) == 0)
+		data.bw = _PW_LMT_BW_40M;
+	else if (_os_strncmp((char *)sbandwidth, (const char *)"80M", 3) == 0)
+		data.bw = _PW_LMT_BW_80M;
+	else if (_os_strncmp((char *)sbandwidth, (const char *)"160M", 4) == 0)
+		data.bw = _PW_LMT_BW_160M;
+	else {
+		PHL_INFO("unknown bandwidth: %s\n", (char *)sbandwidth);
+		return;
+	}
+
+	if (_os_strncmp((char *)sband, (const char *)"2.4G", 4) == 0)
+		data.band = _PW_LMT_BAND_2_4G;
+	else if (_os_strncmp((char *)sband, (const char *)"5G", 2) == 0)
+		data.band = _PW_LMT_BAND_5G;
+	else if (_os_strncmp((char *)sband, (const char *)"6G", 2) == 0)
+		data.band = _PW_LMT_BAND_6G;
+	else {
+		PHL_INFO("unknown band: %s\n", (char *)sband);
+		return;
+	}
+
+	if (_os_strncmp((char *)sbf, (const char *)"Non-BF", 6) == 0 || _os_strncmp((char *)sbf, (const char *)"NA", 2) == 0)
+		data.bf = _PW_LMT_NONBF;
+	else if (_os_strncmp((char *)sbf, (const char *)"BF", 2) == 0)
+		data.bf = _PW_LMT_BF;
+	else {
+		PHL_INFO("unknown BF: %s\n", (char *)sbf);
+		return;
+	}
+
+	data.reg = reg_idx;
+
+	_hal_txpwr_lmt_store_from_external_single(hal_info, &data);
+
+	PHL_INFO("Store of power limit table [regulation %d][band %d][bw %d]"
+		"[rate section %d][ntx %d][BF %d][chnl %d][val %d] [sphape_idx %d]\n",
+		data.reg, data.band, data.bw , data.rs, data.ntx, data.bf,
+		data.ch, data.val, data.tx_shap_idx);
+}
+
+static int
+_hal_get_ext_regd_name_idx(void *d,
+	struct rtw_para_pwrlmt_info_t *para_info, const char *regd_name)
+{
+	int idx;
+
+	idx = rtw_hal_rf_get_predef_pw_lmt_regu_type_from_str(para_info->band, regd_name);
+	if (idx != -1)
+		return idx;
+
+	idx = hal_phy_find_ext_regd_num(para_info, regd_name);
+	if (idx == -1) {
+		u8 max_size;
+
+		idx = para_info->ext_regd_arridx + 1;
+		if (para_info->band == BAND_ON_6G)
+			max_size = regd_name_max_size_6g;
+		else
+			max_size = regd_name_max_size;
+
+		if (idx >= max_size) {
+			PHL_ERR("extra reg [%d] over size\n", idx);
+			return -1;
+		}
+
+		hal_phy_set_ext_regd_name(para_info, (u8)idx, regd_name);
+		para_info->ext_regd_arridx++;
+		PHL_INFO("extra reg [%d] = [%s]\n",
+			idx, hal_phy_get_ext_regd_name(para_info, (u8)idx));
+	}
+
+	return idx;
+}
+
+static int
+_hal_parse_txpwrlmt(struct hal_info_t *hal_info, void *para_info_t, u8 *psrc_buf, u32 buflen)
+{
+#define LD_STAGE_EXC_MAPPING	0
+#define LD_STAGE_TAB_DEFINE		1
+#define LD_STAGE_TAB_START		2
+#define LD_STAGE_COLUMN_DEFINE	3
+#define LD_STAGE_COLUMN_REG		4
+#define LD_STAGE_CH_ROW			5
+#define LD_STAGE_PSHAPE_NUM		6
+
+	struct rtw_para_pwrlmt_info_t *para_info = (struct rtw_para_pwrlmt_info_t *)para_info_t;
+	void *drv_priv;
+	char **regulation = NULL;
+	int *reg_idx = NULL;
+	char **pshape = NULL;
+	char	*sz_line = NULL, *ptmp = NULL;
+	char band[10], bandwidth[10], ratesection[10], ntx[10], colnumbuf[10], bf_type[10];
+	u8	colnum = 0;
+	u8	loadingstage = LD_STAGE_EXC_MAPPING;
+	u32 struct_idx = 0;
+	u32	i = 0, forcnt = 0;
+	u8 dbm_to_txgi;
+	s8 ww_value, na_value;
+
+	if (psrc_buf == NULL) {
+		PHL_INFO("%s, fail !!! NULL buf !!!\n", __func__);
+		return 0;
+	}
+
+	drv_priv = hal_to_drvpriv(hal_info);
+	dbm_to_txgi = rtw_hal_get_tx_tbl_to_tx_pwr_times(hal_info);
+	ww_value = rtw_hal_get_power_limit_value_ww(hal_info);
+	na_value = rtw_hal_get_power_limit_value_na(hal_info);
+
+	para_info->ext_reg_map_num = 0;
+
+	ptmp = (char *)psrc_buf;
+	for (sz_line = hal_getLinefrombuffer(ptmp); sz_line != NULL; sz_line = hal_getLinefrombuffer(ptmp)) {
+line_start:
+		if (hal_is_allspace_tab(sz_line, sizeof(*sz_line)))
+			continue;
+
+		if (hal_is_comment_string(sz_line))
+			continue;
+
+		if (loadingstage == LD_STAGE_EXC_MAPPING) {
+			if (sz_line[0] == '#' || sz_line[1] == '#') {
+				loadingstage = LD_STAGE_TAB_DEFINE;
+			} else {
+				if (parse_reg_exc_config(drv_priv, sz_line, para_info) == false) {
+					PHL_ERR("Fail to parse regulation exception ruls!\n");
+					goto exit;
+				}
+			continue;
+			}
+		}
+
+		if (loadingstage == LD_STAGE_TAB_DEFINE) {
+			/* read "## 2.4G, 20M, 1T, CCK" */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+				continue;
+
+			/* skip the space */
+			i = 2;
+			while (sz_line[i] == ' ' || sz_line[i] == '\t')
+				++i;
+
+			sz_line[--i] = ' '; /* return the space in front of the regulation info */
+
+			/* Parse the label of the table */
+			_os_mem_set(drv_priv, (void *) band, 0, 10);
+			_os_mem_set(drv_priv, (void *) bandwidth, 0, 10);
+			_os_mem_set(drv_priv, (void *) ntx, 0, 10);
+			_os_mem_set(drv_priv, (void *) ratesection, 0, 10);
+			_os_mem_set(drv_priv, (void *) bf_type, 0, 10);
+
+			if (!hal_parse_fiedstring(sz_line, &i, band, ' ', ',')) {
+				PHL_ERR("Fail to parse band!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, bandwidth, ' ', ',')) {
+				PHL_ERR("Fail to parse bandwidth!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, ntx, ' ', ',')) {
+				PHL_ERR("Fail to parse ntx!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, ratesection, ' ', ',')) {
+				PHL_ERR("Fail to parse rate!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, bf_type, ' ', '/')) {
+				PHL_ERR("Fail to parse BF!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+
+			loadingstage = LD_STAGE_TAB_START;
+		} else if (loadingstage == LD_STAGE_TAB_START) {
+			/* read "## START" */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+				continue;
+
+			/* skip the space */
+			i = 2;
+			while (sz_line[i] == ' ' || sz_line[i] == '\t')
+				++i;
+
+			if (_os_strncmp((char *)(sz_line + i), (const char *)"START", 5)) {
+				PHL_ERR("Missing \"##	START\" label\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			loadingstage = LD_STAGE_COLUMN_DEFINE;
+		} else if (loadingstage == LD_STAGE_COLUMN_DEFINE) {
+			/* read "## #5# " */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+
+			/* skip the space */
+			i = 2;
+			while (sz_line[i] == ' ' || sz_line[i] == '\t')
+				++i;
+
+			_os_mem_set(drv_priv, (void *) colnumbuf, 0, 10);
+			if (!hal_parse_fiedstring(sz_line, &i, colnumbuf, '#', '#')) {
+				PHL_ERR("Fail to parse column number!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_get_u1bint_fromstr_indec(colnumbuf, &colnum)) {
+				PHL_ERR("Column number \"%s\" is not unsigned decimal\n", colnumbuf);
+				struct_idx = 0;
+				goto exit;
+			}
+			if (colnum == 0) {
+				PHL_ERR("Column number is 0\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			PHL_INFO("[%s][%s][%s][%s][%s] column num:%d\n",
+				band, bandwidth, ratesection, ntx, bf_type, colnum);
+
+			regulation = (char **)_os_mem_alloc(drv_priv, sizeof(char *) * colnum);
+			if (!regulation) {
+				PHL_ERR("Regulation alloc fail\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			reg_idx = _os_mem_alloc(drv_priv, sizeof(*reg_idx) * colnum);
+			if (!reg_idx) {
+				PHL_ERR("reg_idx alloc fail\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			pshape = (char **)_os_mem_alloc(drv_priv, sizeof(char *) * colnum);
+			if (!pshape) {
+				PHL_ERR("Regulation alloc fail\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			loadingstage = LD_STAGE_COLUMN_REG;
+		} else if (loadingstage == LD_STAGE_COLUMN_REG) {
+			/* read "##		FCC ETSI	MKK IC	KCC" */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+				continue;
+
+			/* skip the space */
+			i = 2;
+			for (forcnt = 0; forcnt < colnum; ++forcnt) {
+				u32 i_ns;
+
+				/* skip the space */
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					i++;
+				i_ns = i;
+
+				while (sz_line[i] != ' ' && sz_line[i] != '\t' && sz_line[i] != '\0')
+					i++;
+
+				regulation[forcnt] = (char *)_os_mem_alloc(drv_priv, i - i_ns + 1);
+				if (!regulation[forcnt]) {
+					PHL_ERR("Regulation alloc fail\n");
+					struct_idx = 0;
+					goto exit;
+				}
+
+				_os_mem_cpy(drv_priv, regulation[forcnt], sz_line + i_ns, i - i_ns);
+				regulation[forcnt][i - i_ns] = '\0';
+
+				reg_idx[forcnt] = _hal_get_ext_regd_name_idx(drv_priv
+					, para_info, regulation[forcnt]);
+			}
+
+			if (1) {
+				PHL_INFO("column name:");
+				for (forcnt = 0; forcnt < colnum; ++forcnt)
+					PHL_INFO(" %s", regulation[forcnt]);
+				PHL_INFO("\n");
+			}
+
+			loadingstage = LD_STAGE_PSHAPE_NUM;
+		}else if (loadingstage == LD_STAGE_PSHAPE_NUM) {
+
+			if (sz_line[0] == '#' || sz_line[1] == '#')
+				continue;
+
+			if ((sz_line[0] != 'p' && sz_line[0] != 'P') ||
+				(sz_line[1] != 's' && sz_line[1] != 'S')) {
+				PHL_INFO("No PShape prefix: '%c','%c'(%d,%d), continue to CH ROW\n",
+					sz_line[0], sz_line[1], sz_line[0], sz_line[1]);
+				loadingstage = LD_STAGE_CH_ROW;
+				goto line_start;
+			}
+			PHL_INFO("1.sz_line[0 1] %c %c", sz_line[0], sz_line[1]);
+			i = 6;/* move to the  location behind 'e' */
+
+			for (forcnt = 0; forcnt < colnum; ++forcnt) {
+				u32	i_ns;
+
+				/* skip the space */
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					i++;
+				i_ns = i;
+				PHL_INFO("1.sz_line[%d] %c\n", i ,sz_line[i]);
+				while (sz_line[i] != ' ' && sz_line[i] != '\t' && sz_line[i] != '\0')
+					i++;
+
+				pshape[forcnt] = (char *)_os_mem_alloc(drv_priv, i - i_ns + 1);
+				if (!pshape[forcnt]) {
+					PHL_ERR("Regulation alloc fail, pshape [%d]\n", forcnt);
+					struct_idx = 0;
+					goto exit;
+				}
+				PHL_INFO("2.pshape [%d] = %s\n", forcnt, pshape[forcnt]);
+
+				_os_mem_cpy(drv_priv, pshape[forcnt], sz_line + i_ns, i - i_ns);
+				pshape[forcnt][i - i_ns] = '\0';
+				PHL_INFO(" forcnt %d shape idx: %s:", forcnt, pshape[forcnt]);
+			}
+
+			if (1) {
+				PHL_INFO("pshape idx:");
+				for (forcnt = 0; forcnt < colnum; ++forcnt)
+					PHL_INFO(" %s", pshape[forcnt]);
+				PHL_INFO("\n");
+			}
+
+			loadingstage = LD_STAGE_CH_ROW;
+		} else if (loadingstage == LD_STAGE_CH_ROW) {
+			char	channel[10] = {0}, powerlimit[10] = {0};
+			u8	cnt = 0;
+
+			/* the table ends */
+			if (sz_line[0] == '#' && sz_line[1] == '#') {
+				i = 2;
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					++i;
+
+				if (_os_strncmp((char *)(sz_line + i), (const char *)"END", 3) == 0) {
+					loadingstage = LD_STAGE_TAB_DEFINE;
+					if (regulation) {
+						for (forcnt = 0; forcnt < colnum; ++forcnt) {
+							if (regulation[forcnt]) {
+								_os_mem_free(drv_priv,
+									(u8 *)regulation[forcnt],
+									_os_strlen((u8 *)regulation[forcnt]) + 1);
+								regulation[forcnt] = NULL;
+							}
+						}
+						_os_mem_free(drv_priv, (u8 *)regulation, sizeof(char *) * colnum);
+						regulation = NULL;
+					}
+					if (reg_idx) {
+						_os_mem_free(drv_priv, reg_idx
+							, sizeof(*reg_idx) * colnum);
+						reg_idx = NULL;
+					}
+					if (pshape) {
+						for (forcnt = 0; forcnt < colnum; ++forcnt) {
+							if (pshape[forcnt]) {
+								_os_mem_free(drv_priv,
+									(u8 *)pshape[forcnt],
+									_os_strlen((u8 *)pshape[forcnt]) + 1);
+								pshape[forcnt] = NULL;
+							}
+						}
+						_os_mem_free(drv_priv, (u8 *)pshape, sizeof(char *) * colnum);
+						pshape = NULL;
+					}
+					colnum = 0;
+					continue;
+				} else {
+					PHL_ERR("Missing \"##	END\" label\n");
+					struct_idx = 0;
+					goto exit;
+				}
+			}
+
+			if ((sz_line[0] != 'c' && sz_line[0] != 'C') ||
+				(sz_line[1] != 'h' && sz_line[1] != 'H')) {
+				PHL_ERR("Wrong channel prefix: '%c','%c'(%d,%d)\n",
+					sz_line[0], sz_line[1], sz_line[0], sz_line[1]);
+				continue;
+			}
+			i = 2;/* move to the  location behind 'h' */
+
+			/* load the channel number */
+			cnt = 0;
+			while (sz_line[i] >= '0' && sz_line[i] <= '9') {
+				channel[cnt] = sz_line[i];
+				++cnt;
+				++i;
+			}
+			PHL_INFO("chnl %s!\n", channel);
+
+			for (forcnt = 0; forcnt < colnum; ++forcnt) {
+				/* skip the space between channel number and the power limit value */
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					++i;
+
+				/* load the power limit value */
+				_os_mem_set(drv_priv, (void *) powerlimit, 0, 10);
+
+				if (sz_line[i] == 'W' && sz_line[i + 1] == 'W') {
+					/*
+					* case "WW" assign special ww value
+					* means to get minimal limit in other regulations at same channel
+					*/
+					_os_snprintf(powerlimit, 10, "%d", ww_value);
+					i += 2;
+
+				} else if (sz_line[i] == 'N' && sz_line[i + 1] == 'A') {
+					/*
+					* case "NA" assign max txgi value
+					* means no limitation
+					*/
+					_os_snprintf(powerlimit, 10, "%d", na_value);
+					i += 2;
+
+				} else if ((sz_line[i] >= '0' && sz_line[i] <= '9') || sz_line[i] == '.'
+					|| sz_line[i] == '+' || sz_line[i] == '-') {
+					/* case of dBm value */
+					u8 integer = 0, fraction = 0, negative = 0;
+					u32 u4bmove;
+					s8 lmt = 0;
+
+					if (sz_line[i] == '+' || sz_line[i] == '-') {
+						if (sz_line[i] == '-')
+							negative = 1;
+						i++;
+					}
+
+					if (hal_get_fractionvalue_fromstring(&sz_line[i], &integer, &fraction, &u4bmove))
+						i += u4bmove;
+					else {
+						PHL_ERR("Limit \"%s\" is not valid decimal\n", &sz_line[i]);
+						struct_idx = 0;
+						goto exit;
+					}
+
+					/* transform to string of value in unit of txgi */
+					lmt = (integer * dbm_to_txgi) + ((u16)fraction * dbm_to_txgi / 100);
+					if (negative)
+						lmt = -lmt;
+					_os_snprintf(powerlimit, 10, "%d", lmt);
+
+					} else {
+						PHL_ERR("Wrong limit expression \"%c%c\"(%d, %d)\n"
+							, sz_line[i], sz_line[i + 1], sz_line[i], sz_line[i + 1]);
+						struct_idx = 0;
+						goto exit;
+					}
+
+					if (reg_idx[forcnt] < 0)
+						continue;
+
+					/* store the power limit value */
+					hal_phy_store_tx_power_limit(hal_info, (u8)reg_idx[forcnt],
+						(u8 *)band, (u8 *)bandwidth, (u8 *)ratesection,
+						(u8 *)bf_type, (u8 *)ntx, (u8 *)channel,
+						(u8 *)powerlimit, (u8 *)pshape[forcnt]);
+
+					struct_idx++;
+					}
+				}
+	}
+
+exit:
+	if (struct_idx == 0) {
+		/* clear corresponding table */
+		u8 band_bmp = para_info->band == BAND_ON_24G ? BAND_CAP_2G | BAND_CAP_5G : BAND_CAP_6G;
+
+		rtw_hal_rf_clear_limit_table(hal_info, band_bmp, PWR_LIMIT);
+		_hal_pwrlmt_para_free(hal_info->phl_com, para_info, false);
+	}
+
+	if (regulation) {
+		for (forcnt = 0; forcnt < colnum; ++forcnt) {
+			if (regulation[forcnt]) {
+				_os_mem_free(drv_priv,
+					(u8 *)regulation[forcnt],
+					_os_strlen((u8 *)regulation[forcnt]) + 1);
+				regulation[forcnt] = NULL;
+			}
+		}
+		_os_mem_free(drv_priv, (u8 *)regulation, sizeof(char *) * colnum);
+		regulation = NULL;
+	}
+	if (reg_idx) {
+		_os_mem_free(drv_priv, reg_idx, sizeof(*reg_idx) * colnum);
+		reg_idx = NULL;
+	}
+	if (pshape) {
+		for (forcnt = 0; forcnt < colnum; ++forcnt) {
+			if (pshape[forcnt]) {
+				_os_mem_free(drv_priv,
+					(u8 *)pshape[forcnt],
+					_os_strlen((u8 *)pshape[forcnt]) + 1);
+				pshape[forcnt] = NULL;
+			}
+		}
+		_os_mem_free(drv_priv, (u8 *)pshape, sizeof(char *) * colnum);
+		pshape = NULL;
+	}
+
+	return struct_idx;
+}
+
+static void hal_phy_store_tx_power_limit_ru(struct hal_info_t *hal_info,
+		u8				reg_idx,
+		u8				*sband,
+		u8				*sbandwidth,
+		u8				*sratesection,
+		u8				*sntx,
+		u8				*schannel,
+		u8				*spwrlimit,
+		u8				*spshape_idx)
+{
+	struct hal_txpwr_lmt_ru_t data = {0};
+
+	if (hal_get_u1bint_fromstr_indec((char *)schannel, &data.ch) == false
+		|| hal_get_s1bint_fromstr_indec((char *)spwrlimit, &data.val) == false) {
+		PHL_INFO("Illegal index of power limit table [ch %s][val %s]\n",
+			 (char *)schannel, (char *)spwrlimit);
+		return;
+	}
+
+	if (spshape_idx != NULL) {
+		if (hal_get_u1bint_fromstr_indec((char *)spshape_idx, &data.tx_shap_idx) == false) {
+			PHL_INFO("Illegal index of pshape idx [val %s]\n", (char *)spshape_idx);
+			return;
+		}
+	}
+
+	if (_os_strncmp((char *)sband, (const char *)"2.4G", 4) == 0)
+		data.band = _PW_LMT_BAND_2_4G;
+	else if (_os_strncmp((char *)sband, (const char *)"5G", 2) == 0)
+		data.band = _PW_LMT_BAND_5G;
+	else if (_os_strncmp((char *)sband, (const char *)"6G", 2) == 0)
+		data.band = _PW_LMT_BAND_6G;
+	else {
+		PHL_INFO("unknown band: %s\n", (char *)sband);
+		return;
+	}
+
+	if (_os_strncmp((char *)sbandwidth, (const char *)"RU26", 4) == 0)
+		data.rubw = _PW_LMT_RU_BW_RU26;
+	else if (_os_strncmp((char *)sbandwidth, (const char *)"RU52", 4) == 0)
+		data.rubw = _PW_LMT_RU_BW_RU52;
+	else if (_os_strncmp((char *)sbandwidth, (const char *)"RU106", 5) == 0)
+		data.rubw = _PW_LMT_RU_BW_RU106;
+	else {
+		PHL_INFO("unknown RU bandwidth: %s\n", (char *)sbandwidth);
+		return;
+	}
+
+	if (_os_strncmp((char *)sntx, (const char *)"1T", 2) == 0)
+		data.ntx = _PW_LMT_PH_1T;
+	else if (_os_strncmp((char *)sntx, (const char *)"2T", 2) == 0)
+		data.ntx = _PW_LMT_PH_2T;
+	else if (_os_strncmp((char *)sntx, (const char *)"3T", 2) == 0)
+		data.ntx = _PW_LMT_PH_3T;
+	else if (_os_strncmp((char *)sntx, (const char *)"4T", 2) == 0)
+		data.ntx = _PW_LMT_PH_4T;
+	else {
+		PHL_INFO("Wrong tx num:%s\n", (char *)sntx);
+		return;
+	}
+
+	if (_os_strncmp((char *)sratesection, (const char *)"HE", 2) == 0)
+		data.rs = _PW_LMT_RS_HE;
+	else {
+		PHL_INFO("Wrong RU rate section:%s\n", (char *)sratesection);
+		return;
+	}
+
+	data.reg = reg_idx;
+
+	_hal_txpwr_lmt_ru_store_from_external_single(hal_info, &data);
+
+	PHL_INFO("Store of power limit RU table [band %d][bw %d][ntx %d]"
+		"[rate section %d][regulation %d][chnl %d][val %d]  [sphape_idx %d]\n",
+		data.band, data.rubw, data.ntx, data.rs,
+		data.reg, data.ch, data.val, data.tx_shap_idx);
+}
+
+static int
+_hal_parse_txpwrlmt_ru(struct hal_info_t *hal_info, void *para_info_t, u8 *psrc_buf, u32 buflen)
+{
+#define LD_STAGE_EXC_MAPPING	0
+#define LD_STAGE_TAB_DEFINE		1
+#define LD_STAGE_TAB_START		2
+#define LD_STAGE_COLUMN_DEFINE	3
+#define LD_STAGE_COLUMN_REG		4
+#define LD_STAGE_CH_ROW			5
+#define LD_STAGE_PSHAPE_NUM		6
+
+	struct rtw_para_pwrlmt_info_t *para_info = (struct rtw_para_pwrlmt_info_t *)para_info_t;
+	void *drv_priv;
+	char **regulation = NULL;
+	int *reg_idx = NULL;
+	char **pshape = NULL;
+	char	*sz_line = NULL, *ptmp = NULL;
+	char band[10], bandwidth[10], ratesection[10], ntx[10], col_num_buf[10];
+	u8	col_num = 0;
+	u8	loadingstage = LD_STAGE_EXC_MAPPING;
+	u32 struct_idx = 0;
+	u32	i = 0, for_cnt = 0;
+	u8 dbm_to_txgi;
+	s8 ww_value, na_value;
+
+	if (psrc_buf == NULL) {
+		PHL_INFO("%s, fail !!! NULL buf !!!\n", __func__);
+		return 0;
+	}
+
+	drv_priv = hal_to_drvpriv(hal_info);
+	dbm_to_txgi = rtw_hal_get_tx_tbl_to_tx_pwr_times(hal_info);
+	ww_value = rtw_hal_get_power_limit_value_ww(hal_info);
+	na_value = rtw_hal_get_power_limit_value_na(hal_info);
+
+	para_info->ext_reg_map_num = 0;
+
+	ptmp = (char *)psrc_buf;
+	for (sz_line = hal_getLinefrombuffer(ptmp); sz_line != NULL; sz_line = hal_getLinefrombuffer(ptmp)) {
+line_start:
+		if (hal_is_allspace_tab(sz_line, sizeof(*sz_line)))
+			continue;
+
+		if (hal_is_comment_string(sz_line))
+			continue;
+
+		if (loadingstage == LD_STAGE_EXC_MAPPING) {
+				if (sz_line[0] == '#' || sz_line[1] == '#') {
+					loadingstage = LD_STAGE_TAB_DEFINE;
+			} else {
+				if (parse_reg_exc_config(drv_priv, sz_line, para_info) == false) {
+					PHL_ERR("Fail to parse regulation exception ruls!\n");
+					goto exit;
+				}
+				continue;
+			}
+		}
+
+		if (loadingstage == LD_STAGE_TAB_DEFINE) {
+			/* read "## 2.4G, RU26, 1T, HE //" */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+				continue;
+
+			/* skip the space */
+			i = 2;
+			while (sz_line[i] == ' ' || sz_line[i] == '\t')
+				++i;
+
+			sz_line[--i] = ' '; /* return the space in front of the regulation info */
+
+			/* Parse the label of the table */
+			_os_mem_set(drv_priv, (void *) band, 0, 10);
+			_os_mem_set(drv_priv, (void *) bandwidth, 0, 10);
+			_os_mem_set(drv_priv, (void *) ntx, 0, 10);
+			_os_mem_set(drv_priv, (void *) ratesection, 0, 10);
+
+			if (!hal_parse_fiedstring(sz_line, &i, band, ' ', ',')) {
+				PHL_ERR("Fail to parse band!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, bandwidth, ' ', ',')) {
+				PHL_ERR("Fail to parse bandwidth!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, ntx, ' ', ',')) {
+				PHL_ERR("Fail to parse ntx!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, ratesection, ' ', ' ')) {
+				PHL_ERR("Fail to parse rate!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+
+			loadingstage = LD_STAGE_TAB_START;
+		} else if (loadingstage == LD_STAGE_TAB_START) {
+			/* read "## START" */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+				continue;
+
+			/* skip the space */
+			i = 2;
+			while (sz_line[i] == ' ' || sz_line[i] == '\t')
+				++i;
+
+			if (_os_strncmp((char *)(sz_line + i), (const char *)"START", 5)) {
+				PHL_ERR("Missing \"##	START\" label\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			loadingstage = LD_STAGE_COLUMN_DEFINE;
+		} else if (loadingstage == LD_STAGE_COLUMN_DEFINE) {
+			/* read "## #5# " */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+
+			/* skip the space */
+			i = 2;
+			while (sz_line[i] == ' ' || sz_line[i] == '\t')
+				++i;
+
+			_os_mem_set(drv_priv, (void *) col_num_buf, 0, 10);
+			if (!hal_parse_fiedstring(sz_line, &i, col_num_buf, '#', '#')) {
+				PHL_ERR("Fail to parse column number!\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			if (!hal_get_u1bint_fromstr_indec(col_num_buf, &col_num)) {
+				PHL_ERR("Column number \"%s\" is not unsigned decimal\n", col_num_buf);
+				struct_idx = 0;
+				goto exit;
+			}
+			if (col_num == 0) {
+				PHL_ERR("Column number is 0\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			PHL_INFO("[%s][%s][%s][%s] column num:%d\n",
+					band, bandwidth, ratesection, ntx, col_num);
+
+			regulation = (char **)_os_mem_alloc(drv_priv, sizeof(char *) * col_num);
+			if (!regulation) {
+				PHL_ERR("Regulation alloc fail\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			reg_idx = _os_mem_alloc(drv_priv, sizeof(*reg_idx) * col_num);
+			if (!reg_idx) {
+				PHL_ERR("reg_idx alloc fail\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			pshape = (char **)_os_mem_alloc(drv_priv, sizeof(char *) * col_num);
+			if (!pshape) {
+				PHL_ERR("Regulation alloc fail\n");
+				struct_idx = 0;
+				goto exit;
+			}
+			loadingstage = LD_STAGE_COLUMN_REG;
+		}else if (loadingstage == LD_STAGE_COLUMN_REG) {
+			/* read "##		FCC ETSI	MKK IC	KCC" */
+			if (sz_line[0] != '#' || sz_line[1] != '#')
+				continue;
+
+			/* skip the space */
+			i = 2;
+			for (for_cnt = 0; for_cnt < col_num; ++for_cnt) {
+				u32 i_ns;
+
+				/* skip the space */
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					i++;
+				i_ns = i;
+
+				while (sz_line[i] != ' ' && sz_line[i] != '\t' && sz_line[i] != '\0')
+					i++;
+
+				regulation[for_cnt] = (char *)_os_mem_alloc(drv_priv, i - i_ns + 1);
+				if (!regulation[for_cnt]) {
+					PHL_ERR("Regulation alloc fail\n");
+					struct_idx = 0;
+					goto exit;
+				}
+
+				_os_mem_cpy(drv_priv, regulation[for_cnt], sz_line + i_ns, i - i_ns);
+				regulation[for_cnt][i - i_ns] = '\0';
+
+				reg_idx[for_cnt] = _hal_get_ext_regd_name_idx(drv_priv
+					, para_info, regulation[for_cnt]);
+			}
+
+			if (1) {
+				PHL_INFO("column name:");
+				for (for_cnt = 0; for_cnt < col_num; ++for_cnt)
+					PHL_INFO(" %s", regulation[for_cnt]);
+				PHL_INFO("\n");
+			}
+				loadingstage = LD_STAGE_PSHAPE_NUM;
+		}else if (loadingstage == LD_STAGE_PSHAPE_NUM) {
+
+			if (sz_line[0] == '#' || sz_line[1] == '#')
+				continue;
+
+			if ((sz_line[0] != 'p' && sz_line[0] != 'P') ||
+				(sz_line[1] != 's' && sz_line[1] != 'S')) {
+				PHL_INFO("No PShape prefix: '%c','%c'(%d,%d), continue to CH ROW\n",
+					sz_line[0], sz_line[1], sz_line[0], sz_line[1]);
+				loadingstage = LD_STAGE_CH_ROW;
+				goto line_start;
+			}
+			PHL_INFO("1.sz_line[0 1] %c %c", sz_line[0], sz_line[1]);
+			i = 6;/* move to the  location behind 'e' */
+
+			for (for_cnt = 0; for_cnt < col_num; ++for_cnt) {
+				u32	i_ns;
+
+				/* skip the space */
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					i++;
+				i_ns = i;
+				PHL_INFO("1.sz_line[%d] %c\n", i ,sz_line[i]);
+				while (sz_line[i] != ' ' && sz_line[i] != '\t' && sz_line[i] != '\0')
+					i++;
+
+				pshape[for_cnt] = (char *)_os_mem_alloc(drv_priv, i - i_ns + 1);
+				if (!pshape[for_cnt]) {
+					PHL_ERR("Regulation alloc fail, pshape [%d]\n", for_cnt);
+					goto exit;
+				}
+				PHL_INFO("2.pshape [%d] = %s\n", for_cnt, pshape[for_cnt]);
+
+				_os_mem_cpy(drv_priv, pshape[for_cnt], sz_line + i_ns, i - i_ns);
+				pshape[for_cnt][i - i_ns] = '\0';
+				PHL_INFO(" forcnt %d shape idx: %s:", for_cnt, pshape[for_cnt]);
+			}
+
+			if (1) {
+				PHL_INFO("pshape idx:");
+				for (for_cnt = 0; for_cnt < col_num; ++for_cnt)
+					PHL_INFO(" %s", pshape[for_cnt]);
+				PHL_INFO("\n");
+			}
+
+			loadingstage = LD_STAGE_CH_ROW;
+		} else if (loadingstage == LD_STAGE_CH_ROW) {
+			char	channel[10] = {0}, powerlimit[10] = {0};
+			u8	cnt = 0;
+
+			/* the table ends */
+			if (sz_line[0] == '#' && sz_line[1] == '#') {
+				i = 2;
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					++i;
+
+				if (_os_strncmp((char *)(sz_line + i), (const char *)"END", 3) == 0) {
+					loadingstage = LD_STAGE_TAB_DEFINE;
+					if (regulation) {
+						for (for_cnt = 0; for_cnt < col_num; ++for_cnt) {
+							if (regulation[for_cnt]) {
+								_os_mem_free(drv_priv,
+									(u8 *)regulation[for_cnt],
+									_os_strlen((u8 *)regulation[for_cnt]) + 1);
+								regulation[for_cnt] = NULL;
+							}
+						}
+						_os_mem_free(drv_priv, (u8 *)regulation, sizeof(char *) * col_num);
+						regulation = NULL;
+					}
+					if (reg_idx) {
+						_os_mem_free(drv_priv, reg_idx
+							, sizeof(*reg_idx) * col_num);
+						reg_idx = NULL;
+					}
+					if (pshape) {
+						for (for_cnt = 0; for_cnt < col_num; ++for_cnt) {
+							if (pshape[for_cnt]) {
+								_os_mem_free(drv_priv,
+									(u8 *)pshape[for_cnt],
+									_os_strlen((u8 *)pshape[for_cnt]) + 1);
+								pshape[for_cnt] = NULL;
+							}
+						}
+						_os_mem_free(drv_priv, (u8 *)pshape, sizeof(char *) * col_num);
+						pshape = NULL;
+					}
+					col_num = 0;
+					continue;
+				} else {
+					PHL_ERR("Missing \"##	END\" label\n");
+					struct_idx = 0;
+					goto exit;
+				}
+			}
+
+			if ((sz_line[0] != 'c' && sz_line[0] != 'C') ||
+				(sz_line[1] != 'h' && sz_line[1] != 'H')) {
+				PHL_ERR("Wrong channel prefix: '%c','%c'(%d,%d)\n",
+					sz_line[0], sz_line[1], sz_line[0], sz_line[1]);
+				continue;
+			}
+			i = 2;/* move to the  location behind 'h' */
+
+			/* load the channel number */
+			cnt = 0;
+			while (sz_line[i] >= '0' && sz_line[i] <= '9') {
+				channel[cnt] = sz_line[i];
+				++cnt;
+				++i;
+			}
+			PHL_INFO("chnl %s!\n", channel);
+
+			for (for_cnt = 0; for_cnt < col_num; ++for_cnt) {
+				/* skip the space between channel number and the power limit value */
+				while (sz_line[i] == ' ' || sz_line[i] == '\t')
+					++i;
+
+				/* load the power limit value */
+				_os_mem_set(drv_priv, (void *) powerlimit, 0, 10);
+
+				if (sz_line[i] == 'W' && sz_line[i + 1] == 'W') {
+					/*
+					* case "WW" assign special ww value
+					* means to get minimal limit in other regulations at same channel
+					*/
+					_os_snprintf(powerlimit, 10, "%d", ww_value);
+					i += 2;
+
+				} else if (sz_line[i] == 'N' && sz_line[i + 1] == 'A') {
+					/*
+					* case "NA" assign max txgi value
+					* means no limitation
+					*/
+					_os_snprintf(powerlimit, 10, "%d", na_value);
+					i += 2;
+
+				} else if ((sz_line[i] >= '0' && sz_line[i] <= '9') || sz_line[i] == '.'
+					|| sz_line[i] == '+' || sz_line[i] == '-') {
+					/* case of dBm value */
+					u8 integer = 0, fraction = 0, negative = 0;
+					u32 u4bmove;
+					s8 lmt = 0;
+
+					if (sz_line[i] == '+' || sz_line[i] == '-') {
+						if (sz_line[i] == '-')
+							negative = 1;
+						i++;
+					}
+
+					if (hal_get_fractionvalue_fromstring(&sz_line[i], &integer, &fraction, &u4bmove))
+						i += u4bmove;
+					else {
+						PHL_ERR("Limit \"%s\" is not valid decimal\n", &sz_line[i]);
+						struct_idx = 0;
+						goto exit;
+					}
+					/* transform to string of value in unit of txgi */
+					lmt = (integer * dbm_to_txgi) + ((u16)fraction * dbm_to_txgi / 100);
+					if (negative)
+						lmt = -lmt;
+					_os_snprintf(powerlimit, 10, "%d", lmt);
+
+					} else {
+						PHL_ERR("Wrong limit expression \"%c%c\"(%d, %d)\n"
+							, sz_line[i], sz_line[i + 1], sz_line[i], sz_line[i + 1]);
+						struct_idx = 0;
+						goto exit;
+					}
+
+					if (reg_idx[for_cnt] < 0)
+						continue;
+
+					/* store the power limit value */
+					hal_phy_store_tx_power_limit_ru(hal_info,
+						(u8)reg_idx[for_cnt], (u8 *)band,
+						(u8 *)bandwidth, (u8 *)ratesection,
+						(u8 *)ntx, (u8 *)channel, (u8 *)powerlimit,
+						(u8 *)pshape[for_cnt]);
+					PHL_INFO("array_tc_8852a_txpwr_lmt_ru[%d] \n", struct_idx);
+					struct_idx++;
+					}
+				}
+	}
+
+exit:
+	if (struct_idx == 0) {
+		/* clear corresponding table */
+		u8 band_bmp = para_info->band == BAND_ON_24G ? BAND_CAP_2G | BAND_CAP_5G : BAND_CAP_6G;
+
+		rtw_hal_rf_clear_limit_table(hal_info, band_bmp, PWR_LIMIT_RU);
+		_hal_pwrlmt_para_free(hal_info->phl_com, para_info, false);
+	}
+	
+	if (regulation) {
+		for (for_cnt = 0; for_cnt < col_num; ++for_cnt) {
+			if (regulation[for_cnt]) {
+				_os_mem_free(drv_priv,
+					(u8 *)regulation[for_cnt],
+					_os_strlen((u8 *)regulation[for_cnt]) + 1);
+				regulation[for_cnt] = NULL;
+			}
+		}
+		_os_mem_free(drv_priv, (u8 *)regulation, sizeof(char *) * col_num);
+		regulation = NULL;
+	}
+	if (reg_idx) {
+		_os_mem_free(drv_priv, reg_idx, sizeof(*reg_idx) * col_num);
+		reg_idx = NULL;
+	}
+	if (pshape) {
+		for (for_cnt = 0; for_cnt < col_num; ++for_cnt) {
+			if (pshape[for_cnt]) {
+				_os_mem_free(drv_priv,
+					(u8 *)pshape[for_cnt],
+					_os_strlen((u8 *)pshape[for_cnt]) + 1);
+				pshape[for_cnt] = NULL;
+			}
+		}
+		_os_mem_free(drv_priv, (u8 *)pshape, sizeof(char *) * col_num);
+		pshape = NULL;
+	}
+
+	return struct_idx;
+}
+
+static void hal_phy_store_tx_power_track(
+		char *band,
+		char *path,
+		char *sign,
+		char *channel,
+		char *rate,
+		char *data,
+		void *txpwr_track_table)
+{
+#define STR_EQUAL_6G(_band, _path, _sign, _rate, _chnl) \
+	((_os_strcmp(band, _band) == 0) && (_os_strcmp(path, _path) == 0) && (_os_strcmp(sign, _sign) == 0) &&\
+	 (_os_strcmp(rate, _rate) == 0) && (_os_strcmp(channel, _chnl) == 0)\
+	)
+#define STR_EQUAL_5G(_band, _path, _sign, _rate, _chnl) \
+	((_os_strcmp(band, _band) == 0) && (_os_strcmp(path, _path) == 0) && (_os_strcmp(sign, _sign) == 0) &&\
+	 (_os_strcmp(rate, _rate) == 0) && (_os_strcmp(channel, _chnl) == 0)\
+	)
+#define STR_EQUAL_2G(_band, _path, _sign, _rate) \
+	((_os_strcmp(band, _band) == 0) && (_os_strcmp(path, _path) == 0) && (_os_strcmp(sign, _sign) == 0) &&\
+	 (_os_strcmp(rate, _rate) == 0)\
+	)
+
+#define STORE_SWING_TABLE(_array, _iteratedIdx) \
+	do {	\
+	for (token = (char *)_os_strsep(&data, delim); token != (char *)NULL; token = (char *)_os_strsep(&data, delim)) {\
+		_os_sscanf(token, "%d", &idx);\
+		_array[_iteratedIdx++] = (s8)idx;\
+	} } while (0)\
+
+	struct hal_txpwr_track_t *prfcalibrateInfo = (struct hal_txpwr_track_t *)txpwr_track_table;
+
+	u32	j = 0;
+	char	*token;
+	char	delim[] = ",";
+	u32	idx = 0;
+
+	PHL_INFO("===>initDeltaSwingIndexTables():"\
+		"Band: %s;\nPath: %s;\nSign: %s;\nChannel: %s;\nRate: %s;\n, Data: %s;\n",
+	band, path, sign, channel, rate, data);
+
+	if (STR_EQUAL_2G("2G", "A", "+", "CCK"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2g_cck_a_p, j);
+	else if (STR_EQUAL_2G("2G", "A", "-", "CCK"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2g_cck_a_n, j);
+	else if (STR_EQUAL_2G("2G", "B", "+", "CCK"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2g_cck_b_p, j);
+	else if (STR_EQUAL_2G("2G", "B", "-", "CCK"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2g_cck_b_n, j);
+	else if (STR_EQUAL_2G("2G", "A", "+", "ALL"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2ga_p, j);
+	else if (STR_EQUAL_2G("2G", "A", "-", "ALL"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2ga_n, j);
+	else if (STR_EQUAL_2G("2G", "B", "+", "ALL"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2gb_p, j);
+	else if (STR_EQUAL_2G("2G", "B", "-", "ALL"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_2gb_n, j);
+	else if (STR_EQUAL_5G("5G", "A", "+", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_p[0], j);
+	else if (STR_EQUAL_5G("5G", "A", "-", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_n[0], j);
+	else if (STR_EQUAL_5G("5G", "B", "+", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_p[0], j);
+	else if (STR_EQUAL_5G("5G", "B", "-", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_n[0], j);
+#if (BAND_NUM > 1)
+	else if (STR_EQUAL_5G("5G", "A", "+", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_p[1], j);
+	else if (STR_EQUAL_5G("5G", "A", "-", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_n[1], j);
+	else if (STR_EQUAL_5G("5G", "B", "+", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_p[1], j);
+	else if (STR_EQUAL_5G("5G", "B", "-", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_n[1], j);
+#endif
+#if (BAND_NUM > 2)
+	else if (STR_EQUAL_5G("5G", "A", "+", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_p[2], j);
+	else if (STR_EQUAL_5G("5G", "A", "-", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_n[2], j);
+	else if (STR_EQUAL_5G("5G", "B", "+", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_p[2], j);
+	else if (STR_EQUAL_5G("5G", "B", "-", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_n[2], j);
+#endif
+#if (BAND_NUM > 3)
+	else if (STR_EQUAL_5G("5G", "A", "+", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_p[3], j);
+	else if (STR_EQUAL_5G("5G", "A", "-", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5ga_n[3], j);
+	else if (STR_EQUAL_5G("5G", "B", "+", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_p[3], j);
+	else if (STR_EQUAL_5G("5G", "B", "-", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_5gb_n[3], j);
+#endif
+	else if (STR_EQUAL_6G("6G", "A", "+", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_p[0], j);
+	else if (STR_EQUAL_6G("6G", "A", "-", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_n[0], j);
+	else if (STR_EQUAL_6G("6G", "B", "+", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_p[0], j);
+	else if (STR_EQUAL_6G("6G", "B", "-", "ALL", "0"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_n[0], j);
+	else if (STR_EQUAL_6G("6G", "A", "+", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_p[1], j);
+	else if (STR_EQUAL_6G("6G", "A", "-", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_n[1], j);
+	else if (STR_EQUAL_6G("6G", "B", "+", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_p[1], j);
+	else if (STR_EQUAL_6G("6G", "B", "-", "ALL", "1"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_n[1], j);
+	else if (STR_EQUAL_6G("6G", "A", "+", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_p[2], j);
+	else if (STR_EQUAL_6G("6G", "A", "-", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_n[2], j);
+	else if (STR_EQUAL_6G("6G", "B", "+", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_p[2], j);
+	else if (STR_EQUAL_6G("6G", "B", "-", "ALL", "2"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_n[2], j);
+	else if (STR_EQUAL_6G("6G", "A", "+", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_p[3], j);
+	else if (STR_EQUAL_6G("6G", "A", "-", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6ga_n[3], j);
+	else if (STR_EQUAL_6G("6G", "B", "+", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_p[3], j);
+	else if (STR_EQUAL_6G("6G", "B", "-", "ALL", "3"))
+		STORE_SWING_TABLE(prfcalibrateInfo->delta_swing_table_idx_6gb_n[3], j);
+	else
+		PHL_INFO("===>initDeltaSwingIndexTables(): The input is invalid!!\n");
+}
+
+static int
+_hal_parse_txpwrtrack(struct hal_info_t *hal_info, void *para_info_t, u8 *psrc_buf, u32 buflen)
+{
+	struct rtw_para_info_t *para_info = (struct rtw_para_info_t *)para_info_t;
+	u32 *pdest_buf = para_info->para_data;
+	char *sz_line = NULL, *ptmp = NULL;
+	u32	i = 0;
+	struct hal_txpwr_track_t *txpwr_track;
+
+	if (pdest_buf == NULL || psrc_buf == NULL) {
+		PHL_INFO("%s, fail !!! NULL buf !!!\n", __func__);
+		return 0;
+	}
+	/* Assing destination buffer to be Txpwr_track table format*/
+	txpwr_track = (struct hal_txpwr_track_t *)pdest_buf;
+
+	ptmp = (char *)psrc_buf;
+	for (sz_line = hal_getLinefrombuffer(ptmp); sz_line != NULL;
+								sz_line = hal_getLinefrombuffer(ptmp))
+	{
+		if (!hal_is_comment_string(sz_line)) {
+			char band[5] = "", path[5] = "", sign[5]  = "";
+			char chnl[5] = "", rate[10] = "";
+			char data[300] = ""; /* 100 is too small */
+
+			if (_os_strlen((u8 *)sz_line) < 10 || sz_line[0] != '[')
+				continue;
+
+			_os_strncpy(band, sz_line + 1, 2);
+			_os_strncpy(path, sz_line + 5, 1);
+			_os_strncpy(sign, sz_line + 8, 1);
+
+			i = 10; /* sz_line+10 */
+			if (!hal_parse_fiedstring(sz_line, &i, rate, '[', ']')) {
+				PHL_ERR("Fail to parse rate!\n");
+			}
+			if (!hal_parse_fiedstring(sz_line, &i, chnl, '[', ']')) {
+				if (!_os_strcmp("5G",band))
+					PHL_ERR("Fail to parse channel group!\n");
+			}
+			while ( i < _os_strlen((u8 *)sz_line) && '{' != sz_line[i])
+				i++;
+			if (!hal_parse_fiedstring(sz_line, &i, data, '{', '}')) {
+				PHL_ERR("Fail to parse data!\n");
+			}
+			hal_phy_store_tx_power_track(
+						band, path, sign,
+						chnl, rate, data,
+						(void*)txpwr_track);
+		}
+	}
+	return 1;
+}
+
+static void
+_hal_decrypt_para_file(
+	char *paraFile,
+	u32  buflen
+)
+{
+	u32	i = 0, j = 0;
+	u32	currentPos = 0;
+	/*External file size shall<=MAX_LINES_HWCONFIG_TXT*MAX_BYTES_LINE_HWCONFIG_TXT=512000(no more than 18 bits)*/
+	u8	map[95] = {0};
+	u8	currentChar = *paraFile;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "=====> %s()\n", __func__);
+	for (i = 0; i < 95; i++)
+		map[i] = (u8)(94 - i);
+	i = 0;
+	while (currentChar != '\0') {
+		currentChar = *(paraFile + currentPos);
+
+		if (currentPos >= buflen) {/*Error handling in case that the encrypted file doesn't have an EOF symbol*/
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "currentPos:%u EncChar:%c\n", currentPos, currentChar);
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s(): curPos reach buflen, forced break!\n", __func__);
+			break;
+		}
+		if (currentChar == '\n') {
+			i++;
+			j = 0;
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "\n");
+		} else {
+			currentChar -= (u8)((((i + j) * 3) % 128));
+
+			*(paraFile + currentPos) = map[currentChar - 32] + 32;/*Translate to the plain text*/
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%c", (map[currentChar - 32] + 32));
+
+			if (currentChar == '\0') {
+				PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "\n%s(): currChar Reach EOF!!\n", __func__);
+			}
+			j++;
+		}
+		currentPos++;
+	}
+	PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "<===== %s(): countLines:%u, curPos:%u\n", __func__, i, currentPos);
+}
+
+static void _hal_get_para_path(struct rtw_para_info_t *para_info, char *ic_name, const char *file_name
+	, bool use_sub_type, char *total_path)
+{
+	char hal_phy_folder[MAX_PATH_LEN] = {0};
+	char para_file_name[MAX_PATH_LEN] = {0};
+	u32 postfix_size = 0, i = 0, dot_pos = 0;
+	char *sp, *ext = NULL;
+
+	/* Step1: Determine parameter folder path. Use path in para_info if it is not empty. */
+	if (para_info->para_path[0] != 0) {
+		_os_snprintf(hal_phy_folder, MAX_PATH_LEN, "%s",
+					 para_info->para_path);
+	} else if (para_info->hal_phy_folder != NULL) {
+		_os_snprintf(hal_phy_folder, MAX_PATH_LEN, "%s",
+					 para_info->hal_phy_folder);
+	} else {
+		_os_snprintf(hal_phy_folder, MAX_PATH_LEN, "%s%s%s",
+				     HAL_FILE_CONFIG_PATH , ic_name, _os_path_sep);
+	}
+
+	/* Step2: Determine parameter file name */
+	_os_strncpy(para_file_name, file_name, _os_strlen((u8 *)file_name)+1);
+
+	/* Step3: Add postfix into original file name if it is specified by user */
+	postfix_size = _os_strlen((u8 *)para_info->postfix);
+
+	if (postfix_size != 0) {
+		/* find the position of latest dot char in file name */
+		sp = para_file_name;
+		for (i = 0, dot_pos = 0; i < _os_strlen((u8 *)file_name); i++) {
+			if (sp[i] == '.')
+				dot_pos = (u8)i;
+		}
+
+		/* Get file extension name from original file name string */
+		ext = (char *)file_name + dot_pos;
+
+		/* Attach postfix, extension name and null terminator */
+		_os_strncpy(sp + dot_pos, para_info->postfix, postfix_size);
+		_os_strncpy(sp + dot_pos + postfix_size, ext, _os_strlen((u8 *)ext));
+		*(sp + dot_pos + postfix_size + _os_strlen((u8 *)ext)) = '\0';
+	}
+
+	/* Step4: Generate final parameter file full path */
+	_os_snprintf(total_path, MAX_PATH_LEN, "%s%s", hal_phy_folder, para_file_name);
+
+}
+
+static void
+_hal_dl_para_file(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info,
+	void *para_info_t, char *ic_name,
+	enum rtw_hal_status (*alloc_fun)(struct rtw_phl_com_t *phl_com, void *para_info_t),
+	int (*parser_fun)(struct hal_info_t *hal_info, void *para_info_t, u8 *psrc_buf, u32 buflen),
+	const char *file_name)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	void *drv = phl_com->drv_priv;
+	u32 para_size = 0;
+	u8 *para_buf = NULL;
+	u32 para_buf_len = MAX_HWCONFIG_FILE_CONTENT;
+	bool dec_cf = phl_com->decrypt_cf;
+	struct rtw_para_info_t *para_info = (struct rtw_para_info_t *)para_info_t;
+
+	if (para_info->para_src == RTW_PARA_SRC_INTNAL)
+		goto exit;
+
+	if (!parser_fun) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s.parser_fun=NULL\n", file_name);
+		goto exit;
+	}
+
+	if (!alloc_fun) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s.alloc_fun=NULL\n", file_name);
+		goto exit;
+	}
+
+	if (para_info->loaded) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s. is loaded\n", file_name);
+		/* exit with success directly to avoid reload and process file content */
+		status = RTW_HAL_STATUS_SUCCESS;
+		goto exit;
+	}
+
+	/* prepare para_buf, para_size */
+	if (para_info->para_src == RTW_PARA_SRC_EXTNAL_BUF) {
+		if (!para_info->ext_para_file_buf) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s.ext_para_file_buf=NULL\n", file_name);
+			goto exit;
+		}
+		if (para_info->ext_para_file_buf_len == 0) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s.ext_para_file_buf_len=0\n", file_name);
+			goto exit;
+		}
+		para_buf = para_info->ext_para_file_buf;
+		para_size = para_info->ext_para_file_buf_len;
+
+	} else if (para_info->para_src == RTW_PARA_SRC_CUSTOM) {
+		if (!para_info->para_data) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s.para_data=NULL\n", file_name);
+			goto exit;
+		}
+		if (para_info->para_data_len == 0) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s.para_data_len=0\n", file_name);
+			goto exit;
+		}
+		para_buf_len = para_info->para_data_len;
+		para_buf = _os_mem_alloc(drv, para_buf_len);
+		if (!para_buf) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_WARNING_,
+				"%s.alloc para_buf with size %d fail\n", file_name, para_buf_len);
+			goto exit;
+		}
+
+		_os_mem_cpy(drv, para_buf, para_info->para_data, para_info->para_data_len);
+		_os_mem_set(drv, para_info->para_data, 0, para_info->para_data_len);
+		para_size = para_info->para_data_len;
+		para_info->para_data_len = 0;
+
+	} else if (para_info->para_src == RTW_PARA_SRC_EXTNAL) {
+		char total_path[MAX_PATH_LEN];
+
+		_hal_get_para_path(para_info, ic_name, file_name, false, total_path);
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s:: %s\n",__FUNCTION__, total_path);
+
+		if (_os_file_readable_supported()) {
+			if (!_os_file_readable(total_path, &para_buf_len) || para_buf_len == 0) {
+				PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+					"%s::%s not readable\n", __FUNCTION__, total_path);
+				goto exit;
+			}
+			para_buf_len++; /* all parsing function rely on _os_strsep(), need '\0' ending */
+		}
+
+		para_buf = _os_mem_alloc(drv, para_buf_len);
+		if (!para_buf) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_WARNING_,
+				"%s.alloc para_buf with size %d fail\n", file_name, para_buf_len);
+			goto exit;
+		}
+
+		para_size = _os_read_file(total_path, para_buf, para_buf_len);
+		if (para_size == 0) {
+			PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+				"%s._os_read_file fail\n", file_name);
+			goto exit;
+		}
+
+		if (dec_cf)
+			_hal_decrypt_para_file((char*)para_buf, para_size);
+	}
+
+	/* allocate para_info */
+	if (alloc_fun(phl_com, para_info) != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_WARNING_,
+			"%s.alloc_fun fail\n", file_name);
+		goto exit;
+	}
+
+	/* Parsing file content */
+	para_info->para_data_len = parser_fun(hal_info, para_info, para_buf, para_size);
+	if (para_info->para_data_len == 0) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+			"%s:: Failed to parser %s\n", __FUNCTION__, file_name);
+		goto exit;
+	}
+
+	status = RTW_HAL_STATUS_SUCCESS;
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s:: Download file ok.\n", file_name);
+
+exit:
+	if (para_info->para_src == RTW_PARA_SRC_EXTNAL
+		|| para_info->para_src == RTW_PARA_SRC_CUSTOM
+	) {
+		if (para_buf)
+			_os_mem_free(drv, para_buf, para_buf_len);
+	}
+
+	para_info->loaded = status == RTW_HAL_STATUS_SUCCESS;
+ 	if (status != RTW_HAL_STATUS_SUCCESS) {
+		para_info->para_src = RTW_PARA_SRC_INTNAL;
+		para_info->para_data_len = 0;
+	}
+}
+#endif /* CONFIG_LOAD_PHY_PARA_FROM_FILE */
+
+int rtw_hal_find_ext_regd_num(struct rtw_para_pwrlmt_info_t *para_info, const char *regd_name)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	return hal_phy_find_ext_regd_num(para_info, regd_name);
+#else
+	return -1;
+#endif
+}
+
+char *rtw_hal_get_ext_regd_name(struct rtw_para_pwrlmt_info_t *para_info, u8 idx)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	return hal_phy_get_ext_regd_name(para_info, idx);
+#else
+	return NULL;
+#endif
+}
+
+void
+rtw_hal_dl_all_para_file(struct rtw_phl_com_t *phl_com,
+							char *ic_name, void *hal)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	struct dev_cap_t *dev_cap = &phl_com->dev_cap;
+	struct phy_sw_cap_t *phy_sw_cap = NULL;
+	u8 idx=0;
+	u8 max_phy_num = 1;
+	bool dec_cf = phl_com->decrypt_cf;
+
+#ifdef CONFIG_DBCC_SUPPORT
+	if (phl_com->dev_cap.dbcc_sup == true)
+		max_phy_num = MAX_BAND_NUM;
+#endif
+
+	for (idx = 0; idx < max_phy_num; idx++) {
+
+		phy_sw_cap = &phl_com->phy_sw_cap[idx];
+
+		_hal_dl_para_file(phl_com, hal,
+				&phy_sw_cap->bb_phy_reg_info,
+				ic_name,
+				_hal_general_para_alloc,
+				_hal_parse_phyreg,
+				"PHY_REG.txt");
+
+		_hal_dl_para_file(phl_com, hal,
+				&phy_sw_cap->bb_phy_reg_gain_info,
+				ic_name,
+				_hal_general_para_alloc,
+				_hal_parse_phyreg,
+				"PHY_REG_GAIN.txt");
+
+		_hal_dl_para_file(phl_com, hal,
+				&phy_sw_cap->rf_radio_a_info,
+				ic_name,
+				_hal_general_para_alloc,
+				_hal_parse_radio,
+				"RadioA.txt");
+
+		_hal_dl_para_file(phl_com, hal,
+				&phy_sw_cap->rf_radio_b_info,
+				ic_name,
+				_hal_general_para_alloc,
+				_hal_parse_radio,
+				"RadioB.txt");
+
+		_hal_dl_para_file(phl_com, hal,
+				&phy_sw_cap->rf_txpwr_byrate_info,
+				ic_name,
+				_hal_pwr_byrate_para_alloc,
+				_hal_parse_txpwr_by_rate,
+				((dec_cf) ?
+				"TXPWR_ByRate_Enc.txt" : "TXPWR_ByRate.txt"));
+
+		_hal_dl_para_file(phl_com, hal,
+				&phy_sw_cap->rf_txpwrtrack_info,
+				ic_name,
+				_hal_general_para_alloc,
+				_hal_parse_txpwrtrack,
+				"TXPWR_TrackTSSI.txt");
+
+		if (dev_cap->band_sup & (BAND_CAP_2G | BAND_CAP_5G))
+			_hal_dl_para_file(phl_com, hal,
+					&phy_sw_cap->rf_txpwrlmt_info,
+					ic_name,
+					_hal_pwrlmt_para_alloc,
+					_hal_parse_txpwrlmt,
+					((dec_cf) ?
+					"TXPWR_LMT_Enc.txt" : "TXPWR_LMT.txt"));
+
+		if (dev_cap->band_sup & (BAND_CAP_2G | BAND_CAP_5G))
+			_hal_dl_para_file(phl_com, hal,
+					&phy_sw_cap->rf_txpwrlmt_ru_info,
+					ic_name,
+					_hal_pwrlmt_para_alloc,
+					_hal_parse_txpwrlmt_ru,
+					((dec_cf) ?
+					"TXPWR_LMT_RU_Enc.txt" : "TXPWR_LMT_RU.txt"));
+
+		if (dev_cap->band_sup & BAND_CAP_6G)
+			_hal_dl_para_file(phl_com, hal,
+					&phy_sw_cap->rf_txpwrlmt_6g_info,
+					ic_name,
+					_hal_pwrlmt_para_alloc,
+					_hal_parse_txpwrlmt,
+					((dec_cf) ?
+					"TXPWR_LMT_6G_Enc.txt" : "TXPWR_LMT_6G.txt"));
+
+		if (dev_cap->band_sup & BAND_CAP_6G)
+			_hal_dl_para_file(phl_com, hal,
+					&phy_sw_cap->rf_txpwrlmt_ru_6g_info,
+					ic_name,
+					_hal_pwrlmt_para_alloc,
+					_hal_parse_txpwrlmt_ru,
+					((dec_cf) ?
+					"TXPWR_LMT_RU_6G_Enc.txt" : "TXPWR_LMT_RU_6G.txt"));
+
+		phy_sw_cap->bfreed_para = false;
+	}
+#endif
+}
+
+void rtw_hal_pwr_byrate_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_info_t *para_info, bool tmp_data_only)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	_hal_pwr_byrate_para_free(phl_com, para_info, tmp_data_only);
+#endif
+}
+
+void rtw_hal_pwrlmt_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_pwrlmt_info_t *para_info, bool tmp_data_only)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	_hal_pwrlmt_para_free(phl_com, para_info, tmp_data_only);
+#endif
+}
+
+void rtw_hal_general_para_free(struct rtw_phl_com_t *phl_com
+	, struct rtw_para_info_t *para_info)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	_hal_general_para_free(phl_com, para_info);
+#endif
+}
+
+bool rtw_hal_txpwr_by_rate_store_from_external(struct rtw_hal_com_t *hal_com
+	, struct rtw_para_info_t *para_info)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	if (para_info->loaded)
+		return true;
+#endif
+	return false;
+}
+
+bool rtw_hal_txpwr_lmt_store_from_external(struct rtw_hal_com_t *hal_com
+	, struct rtw_para_pwrlmt_info_t *para_info)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	if (para_info->loaded)
+		return true;
+#endif
+	return false;
+}
+
+bool rtw_hal_txpwr_lmt_ru_store_from_external(struct rtw_hal_com_t *hal_com
+	, struct rtw_para_pwrlmt_info_t *para_info)
+{
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+	if (para_info->loaded)
+		return true;
+#endif
+	return false;
+}
+
+u8 rtw_hal_ld_fw_symbol(struct rtw_phl_com_t *phl_com,
+	struct rtw_hal_com_t *hal_com, const char *name, u8 **buf, u32 *buf_size)
+{
+	struct rtw_fw_info_t *fw_info = &phl_com->fw_info;
+	char path[256] = {0};
+	char *ic_name = NULL;
+	char *intf = NULL;
+	void *d = phlcom_to_drvpriv(phl_com);
+
+	switch (phl_com->hci_type) {
+	case RTW_HCI_PCIE:
+		intf = "e";
+		break;
+	case RTW_HCI_USB:
+		intf = "u";
+		break;
+	case RTW_HCI_SDIO:
+		intf = "s";
+		break;
+	default:
+		PHL_WARN("%s unknown hci type %u \n", __func__, phl_com->hci_type);
+		return (u8)RTW_HAL_STATUS_FAILURE;
+	}
+
+	switch (hal_com->chip_id) {
+	case CHIP_WIFI6_8852A:
+		ic_name = "rtl8852a";
+		break;
+	case CHIP_WIFI6_8852B:
+		ic_name = "rtl8852b";
+		break;
+	default:
+		PHL_WARN("%s unknown chip id %u \n", __func__, hal_com->chip_id);
+		return (u8)RTW_HAL_STATUS_FAILURE;
+	}
+
+	if (name == NULL || buf_size == NULL)
+		return (u8)RTW_HAL_STATUS_FAILURE;
+
+	if (!fw_info->sym_buf) {
+
+		fw_info->sym_buf = _os_mem_alloc(d, RTW_MAX_FW_SIZE);
+		if (!fw_info->sym_buf) {
+			PHL_WARN("%s : buf for fw symbol allocate fail!!\n", __func__);
+			return (u8)RTW_HAL_STATUS_FAILURE;
+		}
+	}
+	_os_snprintf(path, MAX_PATH_LEN, "%s%s%s%s%s", FW_FILE_CONFIG_PATH,
+		     ic_name, intf, _os_path_sep, name);
+
+	PHL_INFO("%s : %s\n", __func__, path);
+
+	fw_info->sym_buf_size = _os_read_file(path, fw_info->sym_buf,
+		RTW_MAX_FW_SIZE);
+	/* reading fw file failed */
+	if (0 == fw_info->sym_buf_size)
+		return (u8)RTW_HAL_STATUS_FAILURE;
+
+	*buf = fw_info->sym_buf;
+	*buf_size = fw_info->sym_buf_size;
+
+	return (u8)RTW_HAL_STATUS_SUCCESS;
+}
+
+u8 rtw_hal_efuse_shadow_file_load(void *hal, char *ic_name, bool is_limit)
+{
+	u8 status = true;
+#ifdef CONFIG_EFUSE_CONFIG_FILE
+	struct rtw_hal_com_t *hal_com = (struct rtw_hal_com_t *)hal;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal_com->hal_priv;
+	char *hal_phy_folder = HAL_FILE_CONFIG_PATH;
+	char file_path[256];
+
+	_os_snprintf(file_path, MAX_PATH_LEN, "%s%s%s%s",
+			hal_phy_folder , ic_name, _os_path_sep, "efuse.map");
+	PHL_INFO("%s::open map path: %s\n", __FUNCTION__, file_path);
+
+	status = rtw_hal_efuse_file_map_load(hal_info, file_path, is_limit);
+	if (status == RTW_HAL_STATUS_FAILURE) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_WARNING_,
+				"%s:: %s FAIL\n", __FUNCTION__, file_path);
+		status = false;
+		goto exit;
+	}
+
+	_os_snprintf(file_path, MAX_PATH_LEN, "%s%s%s%s",
+			hal_phy_folder , ic_name, _os_path_sep, "efuse.mask");
+	PHL_INFO("%s::open mask path: %s\n", __FUNCTION__, file_path);
+
+	status = rtw_hal_efuse_file_mask_load(hal_info, file_path, is_limit);
+	if (status == RTW_HAL_STATUS_FAILURE) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_WARNING_,
+				"%s:: %s FAIL\n", __FUNCTION__, file_path);
+		status = false;
+		goto exit;
+	}
+exit:
+#endif
+	return status;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ld_file.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ld_file.h
new file mode 100644
index 000000000000..cabc01d02461
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ld_file.h
@@ -0,0 +1,258 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+
+#ifndef _HAL_LD_FILE_H_
+#define _HAL_LD_FILE_H_
+
+enum PHY_PG_NSS_def {
+	PHYPG_1SS	= 0,
+	PHYPG_2SS	= 1,
+	PHYPG_NSS_MAX,
+
+	PHYPG_OFFSET	= 15,
+};
+
+enum PHY_PG_RATE_def {
+	CCK_11M_1M = 0,
+	OFDM_18M_6M = 1,
+	OFDM_54M_24M = 2,
+	HE1SS_MCS3_0 = 3,
+	HE1SS_MCS7_4 = 4,
+	HE1SS_MCS11_8 = 5,
+	HE1SS_DCM4_0 = 6,
+	HE2SS_MCS3_0 = 3,
+	HE2SS_MCS7_4 = 4,
+	HE2SS_MCS11_8 = 5,
+	HE2SS_DCM4_0 = 6,
+	Legacy_AllRate2_1 = 7,
+	HE_AllRate2_2 = 8,
+	OFDM_AllRate5_1 = 9,
+	OFDM_AllRate6_1 = 10,
+};
+
+struct hal_txpwr_byrate_t {
+	u32	band;	/* enum band_type */
+	u32	nss;	/* enum PHY_PG_NSS_def */
+	u32	rs;	/* enum PHY_PG_RATE_def */
+	u32	vals;
+};
+
+typedef struct hal_txpwr_lmt_t {
+	u8 band;
+	u8 bw;
+	u8 ntx;
+	u8 rs;
+	u8 bf;
+	u8 reg;
+	u8 ch;
+	s8 val;
+	u8 tx_shap_idx;
+} HAL_TXPWR_LMT_T , *PHAL_TXPWR_LMT_T;
+
+typedef struct hal_txpwr_lmt_ru_t {
+	u8 band;
+	u8 rubw;
+	u8 ntx;
+	u8 rs;
+	u8 reg;
+	u8 ch;
+	s8 val;
+	u8 tx_shap_idx;
+} Hal_Txpwr_lmt_Ru_t , *pHal_Txpwr_lmt_Ru_t;
+
+enum _halrf_tx_pw_lmt_ru_bandwidth_type {
+	_PW_LMT_RU_BW_RU26,
+	_PW_LMT_RU_BW_RU52,
+	_PW_LMT_RU_BW_RU106,
+	_PW_LMT_RU_BW_NULL
+};
+
+enum _halrf_pw_lmt_band_type {
+		_PW_LMT_BAND_2_4G = 0,
+		_PW_LMT_BAND_5G = 1,
+		_PW_LMT_BAND_6G = 2,
+		_PW_LMT_MAX_BAND = 3
+};
+
+enum _halrf_pw_lmt_bandwidth_type {
+		_PW_LMT_BW_20M = 0,
+		_PW_LMT_BW_40M = 1,
+		_PW_LMT_BW_80M = 2,
+		_PW_LMT_BW_160M = 3,
+		_PW_LMT_MAX_BANDWIDTH_NUM = 4
+};
+
+enum _halrf_pw_lmt_ratesection_type {
+		_PW_LMT_RS_CCK = 0,
+		_PW_LMT_RS_OFDM = 1,
+		_PW_LMT_RS_HT = 2,
+		_PW_LMT_RS_VHT = 3,
+		_PW_LMT_RS_HE = 4,
+		_PW_LMT_MAX_RS_NUM = 5
+};
+
+enum _halrf_pw_lmt_rfpath_type {
+		_PW_LMT_PH_1T = 0,
+		_PW_LMT_PH_2T = 1,
+		_PW_LMT_PH_3T = 2,
+		_PW_LMT_PH_4T = 3,
+		_PW_LMT_MAX_PH_NUM = 4
+};
+
+enum _halrf_pw_lmt_beamforming_type {
+		_PW_LMT_NONBF = 0,
+		_PW_LMT_BF = 1,
+		_PW_LMT_MAX_BF_NUM = 2
+};
+
+/*@--------------------------Define Parameters-------------------------------*/
+#define AVG_THERMAL_NUM 8
+#define MAX_RF_PATH 4
+#define DELTA_SWINGIDX_SIZE 30
+#define BAND_NUM 4
+#define BAND_NUM_6G 4
+#define DELTA_SWINTSSI_SIZE 61
+
+/*@---------------------------End Define Parameters---------------------------*/
+
+struct hal_txpwr_track_t {
+	/* u8 is_txpowertracking; */
+	u8 tx_powercount;
+	bool is_txpowertracking_init;
+	bool is_txpowertracking;
+	u8 txpowertrack_control; /* for mp mode, turn off txpwrtracking as default */
+	u8 tm_trigger;
+	u8 internal_pa_5g[2];	/* pathA / pathB */
+
+	u8 thermal_meter[2];	/* thermal_meter, index 0 for RFIC0, and 1 for RFIC1 */
+	u8 thermal_value;
+	u8 thermal_value_path[MAX_RF_PATH];
+	u8 thermal_value_lck;
+	u8 thermal_value_iqk;
+	s8 thermal_value_delta; /* delta of thermal_value and efuse thermal */
+	u8 thermal_value_avg[AVG_THERMAL_NUM];
+	u8 thermal_value_avg_path[MAX_RF_PATH][AVG_THERMAL_NUM];
+	u8 thermal_value_avg_index;
+	u8 thermal_value_avg_index_path[MAX_RF_PATH];
+	s8 power_index_offset_path[MAX_RF_PATH];
+
+	u8 thermal_value_rx_gain;
+	u8 thermal_value_crystal;
+	u8 thermal_value_dpk_store;
+	u8 thermal_value_dpk_track;
+	bool txpowertracking_in_progress;
+
+	bool is_reloadtxpowerindex;
+	u8 is_rf_pi_enable;
+	u32	txpowertracking_callback_cnt; /* cosa add for debug */
+
+	u8 is_cck_in_ch14;
+	u8 CCK_index;
+	u8 OFDM_index[MAX_RF_PATH];
+	s8 power_index_offset;
+	s8 delta_power_index;
+	s8 delta_power_index_path[MAX_RF_PATH];
+	s8 delta_power_index_last;
+	s8 delta_power_index_last_path[MAX_RF_PATH];
+	bool is_tx_power_changed;
+
+	/*struct iqk_matrix_regs_setting iqk_matrix_reg_setting[IQK_MATRIX_SETTINGS_NUM];*/
+	u8 delta_lck;
+	s8 delta_swing_table_idx_2g_cck_a_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2g_cck_a_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2g_cck_b_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2g_cck_b_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2g_cck_c_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2g_cck_c_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2g_cck_d_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2g_cck_d_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2ga_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2ga_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2gb_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2gb_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2gc_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2gc_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2gd_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2gd_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5ga_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5ga_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5gb_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5gb_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5gc_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5gc_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5gd_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_5gd_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6ga_p[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6ga_n[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6gb_p[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6gb_n[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6gc_p[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6gc_n[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6gd_p[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_6gd_n[BAND_NUM_6G][DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_tssi_table_2g_cck_a[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_2g_cck_b[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_2g_cck_c[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_2g_cck_d[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_2ga[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_2gb[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_2gc[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_2gd[DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_5ga[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_5gb[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_5gc[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_tssi_table_5gd[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_table_xtal_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_xtal_n[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2ga_p_8188e[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_idx_2ga_n_8188e[DELTA_SWINGIDX_SIZE];
+
+	u8 bb_swing_idx_ofdm[MAX_RF_PATH];
+	u8 bb_swing_idx_ofdm_current;
+	u8 bb_swing_idx_ofdm_base;
+	u8 bb_swing_idx_ofdm_base_path[MAX_RF_PATH];
+	bool bb_swing_flag_ofdm;
+	u8 bb_swing_idx_cck;
+	u8 bb_swing_idx_cck_current;
+	u8 bb_swing_idx_cck_base;
+	u8 default_ofdm_index;
+	u8 default_cck_index;
+	bool bb_swing_flag_cck;
+
+	s8 absolute_ofdm_swing_idx[MAX_RF_PATH];
+	s8 remnant_ofdm_swing_idx[MAX_RF_PATH];
+	s8 absolute_cck_swing_idx[MAX_RF_PATH];
+	s8 remnant_cck_swing_idx;
+	s8 modify_tx_agc_value;	/*Remnat compensate value at tx_agc */
+	bool modify_tx_agc_flag_path_a;
+	bool modify_tx_agc_flag_path_b;
+	bool modify_tx_agc_flag_path_c;
+	bool modify_tx_agc_flag_path_d;
+	bool modify_tx_agc_flag_path_a_cck;
+	bool modify_tx_agc_flag_path_b_cck;
+
+	s8 kfree_offset[MAX_RF_PATH];
+	/*Add by Yuchen for Kfree Phydm*/
+	u8 reg_rf_kfree_enable;	/*for registry*/
+	u8 rf_kfree_enable;		/*for efuse enable check*/
+	};
+
+int rtw_hal_find_ext_regd_num(struct rtw_para_pwrlmt_info_t *para_info, const char *regd_name);
+char *rtw_hal_get_ext_regd_name(struct rtw_para_pwrlmt_info_t *para_info, u8 idx);
+void rtw_hal_dl_all_para_file(struct rtw_phl_com_t *phl_com, char *ic_name, void *hal);
+u8 rtw_hal_efuse_shadow_file_load(void *hal, char *ic_name, bool is_limit);
+u8 rtw_hal_ld_fw_symbol(struct rtw_phl_com_t *phl_com,
+	struct rtw_hal_com_t *hal_com, const char *name, u8 **buf, u32 *buf_size);
+#endif /*_HAL_LD_FILE_H_*/
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_led.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_led.c
new file mode 100644
index 000000000000..ff0987ea441f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_led.c
@@ -0,0 +1,73 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_LED_C_
+#include "hal_headers.h"
+
+#define HAL_LED_GET_CTRL_MODE(m)                                               \
+	((m) == RTW_LED_CTRL_SW_PP_MODE                                        \
+	     ? MAC_AX_LED_MODE_SW_CTRL_PP                                      \
+	     : ((m) == RTW_LED_CTRL_SW_OD_MODE                                 \
+		    ? MAC_AX_LED_MODE_SW_CTRL_OD                               \
+		    : ((m) == RTW_LED_CTRL_HW_TX_MODE                          \
+			   ? MAC_AX_LED_MODE_TX_ON                             \
+			   : ((m) == RTW_LED_CTRL_HW_RX_MODE                   \
+				  ? MAC_AX_LED_MODE_RX_ON                      \
+				  : MAC_AX_LED_MODE_TRX_ON))))
+
+#define HAL_LED_GET_ID(i)                                                      \
+	((i) == RTW_LED_ID_0 ? 0 : ((i) == RTW_LED_ID_1 ? 1 : 0))
+
+enum rtw_hal_status rtw_hal_led_set_ctrl_mode(void *hal, enum rtw_led_id led_id,
+					      enum rtw_led_ctrl_mode ctrl_mode)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_TRACE(COMP_PHL_LED, _PHL_INFO_,
+		  "%s: led_id == %d, ctrl_mode == %d\n", __func__, led_id,
+		  ctrl_mode);
+
+	if (ctrl_mode == RTW_LED_CTRL_NOT_SUPPORT)
+		return RTW_HAL_STATUS_SUCCESS;
+
+	return rtw_hal_mac_led_set_ctrl_mode(
+	    hal_info, HAL_LED_GET_CTRL_MODE(ctrl_mode), HAL_LED_GET_ID(led_id));
+}
+
+enum rtw_hal_status rtw_hal_led_control(void *hal, enum rtw_led_id led_id,
+					u8 high)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	PHL_TRACE(COMP_PHL_LED, _PHL_INFO_, "===> hal_led_ctrl()\n");
+
+	if (high) {
+		PHL_TRACE(COMP_PHL_LED, _PHL_INFO_,
+			  "hal_led_ctrl() : LED HIGH\n");
+		status =
+		    rtw_hal_mac_led_ctrl(hal_info, 1, HAL_LED_GET_ID(led_id));
+	} else {
+		PHL_TRACE(COMP_PHL_LED, _PHL_INFO_,
+			  "hal_led_ctrl() : LED LOW\n");
+		status =
+		    rtw_hal_mac_led_ctrl(hal_info, 0, HAL_LED_GET_ID(led_id));
+	}
+
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_LED, _PHL_INFO_,
+			  "hal_led_ctrl() : rtw_hal_mac_led_ctrl() failed\n");
+	}
+
+	return status;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc.c
new file mode 100644
index 000000000000..0ad3225e0b73
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc.c
@@ -0,0 +1,476 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_MCC_C_
+#include "hal_headers.h"
+#include "hal_mcc.h"
+
+#ifdef CONFIG_MCC_SUPPORT
+#define _mcc_fill_slot_bt_coex(_dbg_hal_i, _en) ((struct rtw_phl_mcc_dbg_hal_info *)_dbg_hal_i)->btc_in_group = _en;
+
+void _mcc_update_slot_dbg_info(struct rtw_phl_mcc_dbg_hal_info *dbg_hal_i,
+				bool bt_role, u16 macid, u16 dur)
+{
+	struct rtw_phl_mcc_dbg_slot_info *dbg_slot_i = NULL;
+	u8 idx = 0;
+
+	for (idx = 0; idx < dbg_hal_i->slot_num; idx++) {
+		dbg_slot_i = &dbg_hal_i->dbg_slot_i[idx];
+		if (dbg_slot_i->bt_role) {
+			if (bt_role) {
+				dbg_slot_i->dur = dur;
+				break;
+			}
+		} else if (false == bt_role && dbg_slot_i->macid == macid) {
+			dbg_slot_i->dur = dur;
+			break;
+		}
+	}
+}
+
+void _mcc_update_dbg_info(struct rtw_phl_mcc_en_info *info,
+			struct rtw_phl_mcc_bt_info *bt_info)
+{
+	u8 idx = 0;
+
+	if (bt_info->bt_dur > 0) {
+		_mcc_fill_slot_bt_coex(&info->dbg_hal_i, true);
+	} else {
+		_mcc_fill_slot_bt_coex(&info->dbg_hal_i, false);
+	}
+	for (idx = 0; idx < info->mrole_num; idx++) {
+		_mcc_update_slot_dbg_info(&info->dbg_hal_i, false,
+					info->mcc_role[idx].macid,
+					info->mcc_role[idx].policy.dur_info.dur);
+	}
+}
+
+void _mcc_fill_slot_dbg_info(struct rtw_phl_mcc_dbg_hal_info *dbg_hal_i,
+				struct rtw_phl_mcc_role *mrole)
+{
+	struct rtw_phl_mcc_dbg_slot_info *dbg_slot_i = NULL;
+	if (dbg_hal_i->slot_num >= SLOT_NUM) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_fill_hal_dbg_info(): Fill fail, dbg_hal_i->slot_num(%d) >= SLOT_NUM(%d), please check code\n",
+			dbg_hal_i->slot_num, SLOT_NUM);
+		goto exit;
+	}
+	dbg_slot_i = &dbg_hal_i->dbg_slot_i[dbg_hal_i->slot_num];
+	dbg_slot_i->bt_role = mrole->bt_role ? true : false;
+	dbg_slot_i->dur = mrole->policy.dur_info.dur;
+	if (false == dbg_slot_i->bt_role) {
+		dbg_slot_i->ch = mrole->chandef->chan;
+		dbg_slot_i->macid = mrole->macid;
+	}
+	dbg_hal_i->slot_num++;
+exit:
+	return;
+}
+
+enum rtw_hal_status _mcc_add_bt_role(struct hal_info_t *hal, u8 group,
+			struct rtw_phl_mcc_slot_info *slot_i,
+			struct rtw_phl_mcc_dbg_hal_info *dbg_hal_i)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_phl_mcc_role mrole = {0};
+
+	mrole.policy.dur_info.dur = (u8)slot_i->slot;
+	mrole.bt_role = true;
+	mrole.group = group;
+	PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_add_bt_role(): add bt role\n");
+	_mcc_fill_slot_dbg_info(dbg_hal_i, &mrole);
+	status = rtw_hal_mac_add_mcc(hal, &mrole);
+	return status;
+}
+
+enum rtw_hal_status _mcc_add_wifi_role(struct hal_info_t *hal,
+				u8 group, struct rtw_phl_mcc_slot_info *slot_i,
+				struct rtw_phl_mcc_dbg_hal_info *dbg_hal_i)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_phl_mcc_role *mrole = slot_i->mrole;
+
+	if (NULL == mrole) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_add_wifi_role(): Fail, NULL == mrole, Please check code\n");
+		goto exit;
+	}
+	PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_add_wifi_role(): add wifi role\n");
+	mrole->group = group;
+	_mcc_fill_slot_dbg_info(dbg_hal_i, mrole);
+	status = rtw_hal_mac_add_mcc(hal, mrole);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_add_wifi_role(): Add mcc failed\n");
+		goto exit;
+	}
+	status = rtw_hal_mcc_update_macid_bitmap(hal, group,
+						(u8)mrole->macid,
+						&mrole->used_macid);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_add_wifi_role(): Add MACID bitmap failed\n");
+		goto exit;
+	}
+exit:
+	return status;
+}
+
+enum rtw_hal_status _mcc_fill_role_setting(struct hal_info_t *hal,
+			struct rtw_phl_mcc_en_info *info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_phl_mcc_slot_info *slot_i = info->m_pattern.slot_order;
+	u8 i = 0;
+
+	if (info->m_pattern.slot_num > SLOT_NUM) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_fill_role_setting(): Fial, slot_num(%d) > SLOT_NUM(%d), please check code\n",
+			info->m_pattern.slot_num, SLOT_NUM);
+		goto exit;
+	} else if (info->m_pattern.slot_num < MIN_TDMRA_SLOT_NUM) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_fill_role_setting(): Fial, slot_num(%d) < MIN_TDMRA_SLOT_NUM(%d), please check code\n",
+			info->m_pattern.slot_num, MIN_TDMRA_SLOT_NUM);
+		goto exit;
+	}
+	for (i = 0; i < info->m_pattern.slot_num; i++) {
+		if (!slot_i[i].bt_role) {
+			if (RTW_HAL_STATUS_SUCCESS!= _mcc_add_wifi_role(hal,
+				info->group, &slot_i[i], &info->dbg_hal_i)) {
+				PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_fill_role_setting(): Add wifi role failed\n");
+				goto exit;
+			}
+		} else {
+			if (RTW_HAL_STATUS_SUCCESS != _mcc_add_bt_role(hal,
+				info->group, &slot_i[i], &info->dbg_hal_i)) {
+				PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_fill_role_setting(): Add bt role failed\n");
+				goto exit;
+			}
+		}
+	}
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return status;
+}
+
+enum rtw_hal_status _mcc_replace_pattern(struct hal_info_t *hal,
+				struct rtw_phl_mcc_en_info *ori_info,
+				struct rtw_phl_mcc_en_info *new_info,
+				struct rtw_phl_mcc_bt_info *new_bt_info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_phl_mcc_role *ref_role =
+				&new_info->mcc_role[new_info->ref_role_idx];
+	bool btc_in_group = false;
+
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, ">>> _mcc_replace_pattern\n");
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_get_mcc_group(hal, &new_info->group)) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_replace_pattern(): Allocate group fail\n");
+		goto exit;
+	}
+	_os_mem_set(hal_to_drvpriv(hal), &new_info->dbg_hal_i, 0,
+			sizeof(struct rtw_phl_mcc_dbg_hal_info));
+	status = _mcc_fill_role_setting(hal, new_info);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_replace_pattern(): Fail fill role setting\n");
+		goto exit;
+	}
+	if (new_info->sync_tsf_info.sync_en) {
+		status = rtw_hal_mcc_sync_enable(hal, new_info);
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_replace_pattern(): Sync enable failed\n");
+			goto exit;
+		}
+	}
+	if (new_bt_info->bt_dur > 0) {
+		btc_in_group = true;
+		_mcc_fill_slot_bt_coex(&new_info->dbg_hal_i, true);
+	} else {
+		_mcc_fill_slot_bt_coex(&new_info->dbg_hal_i, false);
+	}
+	status = rtw_hal_mac_start_mcc(hal, new_info->group,
+					(u8)ref_role->macid, new_info->tsf_high,
+					new_info->tsf_low, btc_in_group, 1, ori_info->group);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_replace_pattern(): Start MCC failed\n");
+		goto exit;
+	}
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "<<< _mcc_replace_pattern(): Ststus(%d)\n",
+		status);
+	return status;
+}
+
+void
+_mcc_check_start_t(struct rtw_phl_mcc_en_info *ori_info,
+			struct rtw_phl_mcc_en_info *new_info)
+{
+	u64 new_tsf = 0, ori_tsf = 0;
+
+	new_tsf = new_info->tsf_high;
+	new_tsf = new_tsf << 32;
+	new_tsf |= new_info->tsf_low;
+	ori_tsf = ori_info->tsf_high;
+	ori_tsf = ori_tsf << 32;
+	ori_tsf |= ori_info->tsf_low;
+	if (new_tsf < ori_tsf) {
+		u64 cnt = 0;
+
+		PHL_TRACE(COMP_PHL_MCC, _PHL_WARNING_, "%s: new_tsf(0x%08x %08x) < ori_tsf(0x%08x %08x), we shall extend the new start tsf\n",
+			__FUNCTION__, new_info->tsf_high, new_info->tsf_low,
+			ori_info->tsf_high, ori_info->tsf_low);
+		cnt = _os_division64((ori_tsf - new_tsf),
+				     (new_info->mcc_intvl * TU));
+		new_tsf = new_tsf + ((cnt + 1) * (new_info->mcc_intvl * TU));
+		new_info->tsf_high = (u32)(new_tsf >> 32);
+		new_info->tsf_low = (u32)new_tsf;
+		PHL_TRACE(COMP_PHL_MCC, _PHL_WARNING_, "%s: extended new_tsf(0x%08x %08x)\n",
+			__FUNCTION__, new_info->tsf_high, new_info->tsf_low);
+	}
+}
+
+enum rtw_hal_status rtw_hal_mcc_get_2ports_tsf(void *hal, u8 group,
+			u16 macid_x, u16 macid_y, u32 *tsf_x_h, u32 *tsf_x_l,
+			u32 *tsf_y_h, u32 *tsf_y_l)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_hal_mac_mcc_request_tsf(hal, group, (u8)macid_x, (u8)macid_y);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_get_2ports_tsf(): Req tsf failed\n");
+		goto exit;
+	}
+	status = rtw_hal_mac_get_mcc_tsf_rpt(hal, group, tsf_x_h, tsf_x_l,
+						tsf_y_h, tsf_y_l);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_get_2ports_tsf(): Get tsf rpt failed\n");
+		goto exit;
+	}
+exit:
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mcc_get_2ports_tsf(): Ststus(%d)\n",
+		status);
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_notify_mcc_macid(void *hal,
+                                             struct rtw_phl_mcc_role *mrole,
+                                             enum rtw_phl_tdmra_wmode wmode)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+
+	if (wmode == RTW_PHL_TDMRA_AP_CLIENT_WMODE ||
+	    wmode == RTW_PHL_TDMRA_2CLIENTS_WMODE) {
+		if (sts != rtw_hal_bb_upd_mcc_macid(hal, mrole)) {
+			PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "%s: Notify BB MCC MACID bitmap failed\n",
+			          __FUNCTION__);
+			goto exit;
+
+		}
+	}
+	sts = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return sts;
+}
+
+enum rtw_hal_status rtw_hal_mcc_update_macid_bitmap(void *hal, u8 group,
+			u16 macid, struct rtw_phl_mcc_macid_bitmap *info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_hal_mac_mcc_macid_bitmap(hal, group, (u8)macid,
+						(u8 *)info->bitmap, info->len);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_update_macid_bitmap(): Add MACID bitmap failed\n");
+	}
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mcc_update_macid_bitmap(): Ststus(%d)\n",
+		status);
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_mcc_sync_enable(void *hal,
+					struct rtw_phl_mcc_en_info *info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_phl_mcc_role *ref_role = &info->mcc_role[info->ref_role_idx];
+
+	if (info->sync_tsf_info.sync_en) {
+		status = rtw_hal_mac_mcc_sync_enable(hal, ref_role->group,
+						(u8)info->sync_tsf_info.source,
+						(u8)info->sync_tsf_info.target,
+						(u8)info->sync_tsf_info.offset);
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_sync_enable(): Sync enable failed\n");
+			goto exit;
+		}
+	} else {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_sync_enable(): sync_en == false, please check code\n");
+		goto exit;
+	}
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	return status;
+}
+
+enum rtw_hal_status _mcc_set_duration(void *hal,
+					struct rtw_phl_mcc_en_info *info,
+					struct rtw_phl_mcc_bt_info *bt_info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_hal_mac_set_duration(hal, info, bt_info);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "_mcc_set_duration(): set duration failed\n");
+	} else {
+		_mcc_update_dbg_info(info, bt_info);
+	}
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "_mcc_set_duration(): Ststus(%d)\n",
+		status);
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_mcc_change_pattern(void *hal,
+				struct rtw_phl_mcc_en_info *ori_info,
+				struct rtw_phl_mcc_en_info *new_info,
+				struct rtw_phl_mcc_bt_info *new_bt_info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	_mcc_check_start_t(ori_info, new_info);
+	/* mcc_set_duration only can change 2 wifi slot,
+	otherwise we shall create new pattern and replace previous pattern*/
+	if (new_info->m_pattern.slot_num == 2 &&
+	    new_info->m_pattern.bt_slot_num == 0 &&
+	    ori_info->m_pattern.slot_num == 2 &&
+	    ori_info->m_pattern.bt_slot_num == 0) {
+		status = _mcc_set_duration(hal, new_info, new_bt_info);
+	} else {
+		status = _mcc_replace_pattern(hal, ori_info, new_info,
+						new_bt_info);
+	}
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mcc_change_pattern(): status(%d)\n",
+		status);
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_mcc_reset(void *hal, u8 group,
+					enum rtw_phl_tdmra_wmode wmode)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	status = rtw_hal_mac_reset_mcc_group(hal, group);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_,
+			"%s(): reset group(%d) failed\n",
+			__func__, group);
+		goto exit;
+	}
+	if (wmode == RTW_PHL_TDMRA_AP_CLIENT_WMODE ||
+		wmode == RTW_PHL_TDMRA_2CLIENTS_WMODE) {
+		rtw_hal_bb_mcc_stop(hal);
+	}
+
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "%s(): status(%d)\n",
+		__func__, status);
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_mcc_disable(void *hal, u8 group, u16 macid,
+					enum rtw_phl_tdmra_wmode wmode)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_i = (struct hal_info_t *)hal;
+	struct rtw_phl_stainfo_t *sta = NULL;
+
+	sta = rtw_phl_get_stainfo_by_macid(hal_i->phl_com->phl_priv, macid);
+	status = rtw_hal_mac_stop_mcc(hal, group, (u8)macid);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_disable(): Stop mcc failed\n");
+		goto exit;
+	}
+	status = rtw_hal_mac_del_mcc_group(hal, group);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_disable(): Delete group failed\n");
+		goto exit;
+	}
+	if (wmode == RTW_PHL_TDMRA_AP_CLIENT_WMODE ||
+		wmode == RTW_PHL_TDMRA_2CLIENTS_WMODE) {
+		rtw_hal_bb_mcc_stop(hal);
+	}
+
+#ifdef RTW_WKARD_HALRF_MCC
+	rtw_hal_rf_dpk_switch(hal, true);
+#endif /* RTW_WKARD_HALRF_MCC */
+	rtw_hal_notification(hal, MSG_EVT_MCC_STOP, sta->rlink->hw_band);
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mcc_disable(): Ststus(%d)\n",
+		status);
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_mcc_enable(void *hal, struct rtw_phl_mcc_en_info *info,
+					struct rtw_phl_mcc_bt_info *bt_info,
+					enum rtw_phl_tdmra_wmode wmode)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct rtw_phl_mcc_role *ref_role = &info->mcc_role[info->ref_role_idx];
+	bool btc_in_group = false;
+
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_get_mcc_group(hal, &info->group)) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_enable(): Allocate group fail\n");
+		goto exit;
+	}
+#ifdef RTW_WKARD_HALRF_MCC
+	rtw_hal_rf_dpk_switch(hal, false);
+#endif /* RTW_WKARD_HALRF_MCC */
+	if (RTW_HAL_STATUS_SUCCESS != _mcc_fill_role_setting(hal, info)) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_enable(): Fail fill role setting\n");
+		goto exit;
+	}
+/*fill start setting*/
+	if (info->sync_tsf_info.sync_en) {
+		status = rtw_hal_mcc_sync_enable(hal, info);
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_enable(): Sync enable failed\n");
+			goto exit;
+		}
+	}
+	if (bt_info->bt_dur > 0) {
+		btc_in_group = true;
+		_mcc_fill_slot_bt_coex(&info->dbg_hal_i, true);
+	} else {
+		_mcc_fill_slot_bt_coex(&info->dbg_hal_i, false);
+	}
+	status = rtw_hal_mac_start_mcc(hal, info->group,
+					(u8)ref_role->macid, info->tsf_high,
+					info->tsf_low, btc_in_group, 0, 0);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_enable(): Start MCC failed\n");
+		goto exit;
+	}
+
+	if (wmode == RTW_PHL_TDMRA_AP_CLIENT_WMODE ||
+		wmode == RTW_PHL_TDMRA_2CLIENTS_WMODE) {
+		status = rtw_hal_bb_mcc_start(hal, &info->mcc_role[0],
+		                              &info->mcc_role[1]);
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_MCC, _PHL_ERR_, "rtw_hal_mcc_enable(): Notify MCC start failed\n");
+		}
+	}
+	rtw_hal_notification(hal, MSG_EVT_MCC_START, ref_role->rlink->hw_band);
+	status = RTW_HAL_STATUS_SUCCESS;
+exit:
+	PHL_TRACE(COMP_PHL_MCC, _PHL_INFO_, "rtw_hal_mcc_enable(): Ststus(%d)\n",
+		status);
+	return status;
+}
+#endif /* CONFIG_MCC_SUPPORT */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc.h
new file mode 100644
index 000000000000..d7a5879c9f76
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc.h
@@ -0,0 +1,25 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_MCC_H_
+#define _HAL_MCC_H_
+/* MCC definition for private usage */
+#ifdef CONFIG_MCC_SUPPORT
+enum rtw_hal_status rtw_hal_mcc_update_macid_bitmap(void *hal, u8 group,
+			u16 macid, struct rtw_phl_mcc_macid_bitmap *info);
+
+enum rtw_hal_status rtw_hal_mcc_sync_enable(void *hal,
+					struct rtw_phl_mcc_en_info *info);
+#endif
+#endif /*_HAL_MCC_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc_def.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc_def.h
new file mode 100644
index 000000000000..45e3b6447a0d
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_mcc_def.h
@@ -0,0 +1,19 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_MCC_DEF_H_
+#define _HAL_MCC_DEF_H_
+/* MCC definition for public usage */
+
+#endif /*_HAL_MCC_DEF_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_notify.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_notify.c
new file mode 100644
index 000000000000..8f6b59a6c883
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_notify.c
@@ -0,0 +1,117 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_NOTIFY_C_
+#include "hal_headers.h"
+
+
+void rtw_hal_notification_ex(void *hal, enum phl_msg_evt_id event,
+			     bool to_bb, bool to_mac, bool to_rf, u8 hw_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 idx = 0;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s: event(%d), hw_idx(%d), mac(%d), bb(%d), rf(%d)\n",
+	          __func__, event, hw_idx,
+	          to_mac ? 1 : 0, to_bb ? 1 : 0, to_rf ? 1 : 0);
+
+	if (!hal_info->hal_com->is_hal_init) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s:hal is not started!\n",
+				__func__);
+		return;
+	}
+
+	if (hw_idx == HW_BAND_MAX) {
+		for (idx = 0; idx < hw_idx; idx++) {
+			if (true == to_bb)
+				rtw_hal_bb_notification(hal_info, event, idx);
+
+			if (true == to_mac)
+				rtw_hal_mac_notification(hal_info, event, idx);
+
+			if (true == to_rf)
+				rtw_hal_rf_notification(hal_info, event, idx);
+		}
+	} else {
+		if (true == to_bb)
+			rtw_hal_bb_notification(hal_info, event, hw_idx);
+
+		if (true == to_mac)
+			rtw_hal_mac_notification(hal_info, event, hw_idx);
+
+		if (true == to_rf)
+			rtw_hal_rf_notification(hal_info, event, hw_idx);
+	}
+}
+
+void rtw_hal_notification(void *hal, enum phl_msg_evt_id event, u8 hw_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 idx = 0;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s: event(%d), hw_idx(%d)\n",
+	          __func__, event, hw_idx);
+	if (!hal_info->hal_com->is_hal_init) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s:hal is not started!\n",
+				__func__);
+		return;
+	}
+
+	if (hw_idx == HW_BAND_MAX) {
+		for (idx = 0; idx < hw_idx; idx++) {
+			rtw_hal_bb_notification(hal_info, event, idx);
+			rtw_hal_mac_notification(hal_info, event, idx);
+			rtw_hal_rf_notification(hal_info, event, idx);
+		}
+	} else {
+		rtw_hal_bb_notification(hal_info, event, hw_idx);
+		rtw_hal_mac_notification(hal_info, event, hw_idx);
+		rtw_hal_rf_notification(hal_info, event, hw_idx);
+	}
+}
+
+
+void rtw_hal_cmd_notification(void *hal,
+                               enum phl_msg_evt_id event,
+                               void *hal_cmd,
+                               u8 hw_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	if (event == MSG_EVT_NOTIFY_BB)
+		rtw_hal_bb_cmd_notification(hal_info, hal_cmd, hw_idx);
+	else if (event == MSG_EVT_NOTIFY_RF)
+		rtw_hal_rf_cmd_notification(hal_info, hal_cmd, hw_idx);
+	else if (event == MSG_EVT_NOTIFY_MAC)
+		rtw_hal_mac_cmd_notification(hal_info, hal_cmd, hw_idx);
+	else
+		PHL_ERR("%s unexpect eve id = 0x%x\n", __func__, event);
+
+}
+
+enum rtw_phl_status
+rtw_hal_cmd_notify(struct rtw_phl_com_t *phl_com,
+                   enum phl_msg_evt_id event,
+                   void *hal_cmd,
+                   u8 hw_idx)
+{
+#ifdef CONFIG_CMD_DISP
+	return rtw_phl_cmd_notify(phl_com, event, hal_cmd, hw_idx);
+#else
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s: not support cmd notify\n",
+	          __func__);
+
+	return RTW_PHL_STATUS_FAILURE;
+#endif /* CONFIG_CMD_DISP */
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_notify.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_notify.h
new file mode 100644
index 000000000000..9555335dd02f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_notify.h
@@ -0,0 +1,38 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_NOTIFY_H_
+#define _HAL_NOTIFY_H_
+
+
+void rtw_hal_notification_ex(void *hal, enum phl_msg_evt_id event,
+			     bool to_bb, bool to_mac, bool to_rf, u8 hw_idx);
+
+void rtw_hal_notification(void *hal, enum phl_msg_evt_id event, u8 hw_idx);
+
+
+void rtw_hal_cmd_notification(void *hal,
+                   enum phl_msg_evt_id event,
+                   void *hal_cmd,
+                   u8 hw_idx);
+
+
+enum rtw_phl_status
+rtw_hal_cmd_notify(struct rtw_phl_com_t *phl_com,
+                   enum phl_msg_evt_id event,
+                   void *hal_cmd,
+                   u8 hw_idx);
+
+#endif /*_HAL_NOTIFY_H_*/
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_p2pps.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_p2pps.c
new file mode 100644
index 000000000000..b1adf1304cbd
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_p2pps.c
@@ -0,0 +1,105 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_P2PPS_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_P2PPS
+#define TSF32_TOG_EARLY_T 2000 /*2ms*/
+
+enum rtw_hal_status rtw_hal_noa_enable(void *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid)
+{
+	enum rtw_hal_status ret = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 cur_en_desc_num = noa_info->en_desc_num;
+
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]%s: macid(0x%x), cur_en_desc_num(%d)\n",
+		__func__, macid, cur_en_desc_num);
+	if (cur_en_desc_num == 0) {
+		ret = rtw_hal_mac_noa_init(hal_info, noa_info, in_desc, macid);
+	} else {
+		ret = rtw_hal_mac_noa_update(hal_info, noa_info, in_desc, macid);
+	}
+	return ret;
+}
+
+enum rtw_hal_status rtw_hal_noa_disable(void *hal,
+	struct rtw_phl_noa_info *noa_info,
+	struct rtw_phl_noa_desc *in_desc,
+	u16 macid)
+{
+	enum rtw_hal_status ret = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 cur_en_desc_num = noa_info->en_desc_num;
+
+	PHL_TRACE(COMP_PHL_P2PPS, _PHL_INFO_, "[NoA]%s(): cur_en_desc_num(%d)\n",
+		__func__, cur_en_desc_num);
+	if (cur_en_desc_num == 1) {
+		ret = rtw_hal_mac_noa_terminate(hal_info, noa_info, in_desc,
+						macid);
+	} else {
+		ret = rtw_hal_mac_noa_remove(hal_info, noa_info, in_desc, macid);
+	}
+	return ret;
+}
+
+enum rtw_hal_status
+rtw_hal_tsf32_tog_enable(void *hal,
+                         struct rtw_wifi_role_link_t *rlink)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = hal;
+
+	h_stat = rtw_hal_mac_tsf32_tog_enable(hal_info,
+	                                      rlink->hw_band,
+	                                      rlink->hw_port,
+	                                      TSF32_TOG_EARLY_T);
+
+	return h_stat;
+}
+
+enum rtw_hal_status
+rtw_hal_tsf32_tog_disable(void *hal,
+                         struct rtw_wifi_role_link_t *rlink)
+{
+	enum rtw_hal_status h_stat = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = hal;
+
+	h_stat = rtw_hal_mac_tsf32_tog_disable(hal_info,
+	                                       rlink->hw_band,
+	                                       rlink->hw_port);
+
+	return h_stat;
+}
+
+enum rtw_hal_status rtw_hal_get_tsf32_tog_rpt(void *hal,
+	struct rtw_phl_tsf32_tog_rpt *rpt)
+{
+	struct hal_info_t *hal_info = hal;
+
+	return rtw_hal_mac_get_tsf32_tog_rpt(hal_info, rpt);
+}
+
+enum rtw_hal_status rtw_hal_noa_sta_macid_up(void *hal,
+			u16 wrole_macid, bool join, u8 *bitmap, u32 bitmap_len)
+{
+	return rtw_hal_mac_p2p_macid_ctrl(hal, join, wrole_macid, bitmap,
+						bitmap_len);
+}
+
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ps.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ps.c
new file mode 100644
index 000000000000..a8ba83dcd3f5
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ps.c
@@ -0,0 +1,479 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_PS_C_
+#include "hal_headers.h"
+#ifdef CONFIG_POWER_SAVE
+#define case_pwr_state(src) \
+	case PS_PWR_STATE_##src: return #src
+const char *hal_ps_pwr_state_to_str(u8 pwr_state)
+{
+	switch (pwr_state) {
+	case_pwr_state(ACTIVE);
+	case_pwr_state(BAND0_RFON);
+	case_pwr_state(BAND1_RFON);
+	case_pwr_state(BAND0_RFOFF);
+	case_pwr_state(BAND1_RFOFF);
+	case_pwr_state(CLK_GATED);
+	case_pwr_state(PWR_GATED);
+	case_pwr_state(MAX);
+	default:
+		return "Undefined";
+	}
+}
+
+#ifndef PS_PROTOCAL_LEAVE_TOLERANCE
+#define PS_PROTOCAL_LEAVE_TOLERANCE 50 /* ms */
+#endif
+#ifndef PS_PROTOCAL_LEAVE_MIN_POLLING_CNT
+#define PS_PROTOCAL_LEAVE_MIN_POLLING_CNT 3
+#endif
+#ifndef PWR_LVL_CHANGE_TOLERANCE
+#define PWR_LVL_CHANGE_TOLERANCE 50 /* ms */
+#endif
+#ifndef PWR_LVL_CHANGE_MIN_POLLING_CNT
+#define PWR_LVL_CHANGE_MIN_POLLING_CNT 3
+#endif
+#ifndef MAX_CHK_PWR_STATE_CHANGE_CNT
+#define MAX_CHK_PWR_STATE_CHANGE_CNT 5
+#endif
+
+static enum rtw_hal_status
+_hal_ps_lps_chk_leave(struct hal_info_t *hal_info, u16 macid)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	u32 mac_sts = 0;
+	u32 priv_mac_sts = 0;
+	u32 start_time = _os_get_cur_time_ms();
+	u32 pass_time = 0;
+	u32 polling_cnt = 1;
+
+	do {
+		status = rtw_hal_mac_lps_chk_leave(hal_info, macid, &mac_sts);
+		pass_time = phl_get_passing_time_ms(start_time);
+		if (status == RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+				"[HALPS], %s(): pass time = %d ms, cnt = %u.\n",
+				__func__, pass_time, polling_cnt);
+			break;
+		}
+
+		if (mac_sts != priv_mac_sts) {
+			PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+				"[HALPS], %s(): mac status %u, pass time = %d ms, cnt = %u.\n",
+				__func__, mac_sts, pass_time, polling_cnt);
+			priv_mac_sts = mac_sts;
+		}
+
+		if (pass_time > PS_PROTOCAL_LEAVE_TOLERANCE &&
+			polling_cnt >= PS_PROTOCAL_LEAVE_MIN_POLLING_CNT)
+			break;
+
+		_os_sleep_us(hal_to_drvpriv(hal_info), 50);
+		polling_cnt++;
+	} while (1);
+
+#ifdef CONFIG_PHL_PS_FW_DBG
+	rtw_hal_fw_dbg_dump(hal_info);
+#endif
+	if (status != RTW_HAL_STATUS_SUCCESS)
+		PHL_TRACE(COMP_PHL_PS, _PHL_ERR_,
+			"[HALPS], %s(): polling timeout! (%u ms, %u cnt)\n",
+			__func__, pass_time, polling_cnt);
+
+	return status;
+}
+
+static enum rtw_hal_status
+_hal_ps_lps_cfg(struct hal_info_t *hal_info,
+			struct rtw_hal_lps_info *lps_info)
+{
+	PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+		"[HALPS], %s(): mode(%d), listen bcn mode(%d), awake interval(%d), smart_ps_mode(%d), bcnnohit(%d) dyntxant_en(%d) 1tx(%d).\n",
+		__func__, lps_info->en, lps_info->listen_bcn_mode,
+		lps_info->awake_interval, lps_info->smart_ps_mode,
+		lps_info->bcnnohit_en, lps_info->dyntxant_en, lps_info->lpstxant);
+
+	return rtw_hal_mac_lps_cfg(hal_info, lps_info);
+}
+
+static enum rtw_hal_status
+_hal_ps_pwr_state_chk(struct hal_info_t *hal_info, u8 req_pwr_state)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	u32 mac_sts = 0;
+	u32 priv_mac_sts = 0;
+	u32 start_time = _os_get_cur_time_ms();
+	u32 pass_time = 0;
+	u32 polling_cnt = 1;
+
+	do {
+		status = rtw_hal_mac_chk_pwr_state(hal_info, req_pwr_state, &mac_sts);
+		pass_time = phl_get_passing_time_ms(start_time);
+		if (status == RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+				"[HALPS], %s(): pass time = %d ms, cnt = %u.\n",
+				__func__, pass_time, polling_cnt);
+			break;
+		}
+
+		if (mac_sts != priv_mac_sts) {
+			PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+				"[HALPS], %s(): mac status %u, pass time = %d ms, cnt = %u.\n",
+				__func__, mac_sts, pass_time, polling_cnt);
+			priv_mac_sts = mac_sts;
+		}
+
+		if (pass_time > PWR_LVL_CHANGE_TOLERANCE
+			&& polling_cnt >= PWR_LVL_CHANGE_MIN_POLLING_CNT)
+			break;
+
+		_os_sleep_us(hal_to_drvpriv(hal_info), 50);
+		polling_cnt++;
+	} while (1);
+
+#ifdef CONFIG_PHL_PS_FW_DBG
+	rtw_hal_fw_dbg_dump(hal_info);
+#endif
+
+	if (status != RTW_HAL_STATUS_SUCCESS)
+		PHL_TRACE(COMP_PHL_PS, _PHL_ERR_,
+			"[HALPS], %s(): polling timeout! (%u ms, %u cnt)\n",
+			__func__, pass_time, polling_cnt);
+
+	return status;
+}
+
+static enum rtw_hal_status
+_hal_ps_cfg_pwr_state(struct hal_info_t *hal_info, u8 req_pwr_state)
+{
+#if (HAL_LPS_SKIP_HW_CFG == 0)
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	u8 cnt = 0;
+
+	for (cnt = 0; cnt < MAX_CHK_PWR_STATE_CHANGE_CNT; cnt++) {
+
+		/* rpwm */
+		status = rtw_hal_mac_req_pwr_state(hal_info, req_pwr_state);
+		if (status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_PS, _PHL_ERR_,
+				"[HALPS], %s(): set pwr state %s fail!\n",
+				__func__, hal_ps_pwr_state_to_str(req_pwr_state));
+			return RTW_HAL_STATUS_FAILURE;
+		}
+
+		/* cpwm */
+		status = _hal_ps_pwr_state_chk(hal_info, req_pwr_state);
+		if (status == RTW_HAL_STATUS_SUCCESS)
+			break;
+	}
+
+	if (cnt < MAX_CHK_PWR_STATE_CHANGE_CNT) {
+		PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+				"[HALPS], %s(): chk pwr state %s success, cnt %d.\n",
+				__func__, hal_ps_pwr_state_to_str(req_pwr_state), cnt);
+		return RTW_HAL_STATUS_SUCCESS;
+	} else {
+		PHL_TRACE(COMP_PHL_PS, _PHL_ERR_,
+				"[HALPS], %s(): chk pwr state %s fail!\n",
+				__func__, hal_ps_pwr_state_to_str(req_pwr_state));
+		return RTW_HAL_STATUS_FAILURE;
+	}
+#else
+	return RTW_HAL_STATUS_SUCCESS;
+#endif
+}
+
+static enum rtw_hal_status
+_hal_ps_cfg_pwr_on(struct rtw_phl_com_t *phl_com,
+	struct hal_info_t *hal_info)
+{
+	struct rtw_phl_evt_ops *ops = &phl_com->evt_ops;
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+
+	if (hal_info->hal_com->is_hal_init) {
+		#if (HAL_LPS_SKIP_HW_CFG == 0)
+		status = _hal_ps_cfg_pwr_state(hal_info, PS_PWR_STATE_ACTIVE);
+		#else
+		status = RTW_HAL_STATUS_SUCCESS;
+		#endif
+	} else {
+		#if (HAL_IPS_SKIP_HW_CFG == 0)
+		if (ops->set_rf_state(phlcom_to_drvpriv(phl_com), RTW_RF_ON) == true)
+			status = RTW_HAL_STATUS_SUCCESS;
+		else
+			status = RTW_HAL_STATUS_FAILURE;
+		#else
+			status = RTW_HAL_STATUS_SUCCESS;
+		#endif
+	}
+
+	return status;
+}
+
+static enum rtw_hal_status
+_hal_ps_cfg_pwr_off(struct rtw_phl_com_t *phl_com,
+	struct hal_info_t *hal_info)
+{
+	struct rtw_phl_evt_ops *ops = &phl_com->evt_ops;
+
+	#if (HAL_IPS_SKIP_HW_CFG == 0)
+	if (ops->set_rf_state(phlcom_to_drvpriv(phl_com), RTW_RF_OFF) == true)
+		return RTW_HAL_STATUS_SUCCESS;
+	else
+		return RTW_HAL_STATUS_FAILURE;
+	#else
+		return RTW_HAL_STATUS_SUCCESS;
+	#endif
+}
+
+static enum rtw_hal_status
+_hal_ps_pwr_lvl_cfg(struct rtw_phl_com_t *phl_com, struct hal_info_t *hal_info,
+			u8 req_pwr_lvl)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	switch (req_pwr_lvl) {
+	case PS_PWR_LVL_PWROFF:
+		status = _hal_ps_cfg_pwr_off(phl_com, hal_info);
+		break;
+	case PS_PWR_LVL_PWR_GATED:
+		status = _hal_ps_cfg_pwr_state(hal_info, PS_PWR_STATE_PWR_GATED);
+		break;
+	case PS_PWR_LVL_CLK_GATED:
+		status = _hal_ps_cfg_pwr_state(hal_info, PS_PWR_STATE_CLK_GATED);
+		break;
+	case PS_PWR_LVL_RF_OFF:
+		status = _hal_ps_cfg_pwr_state(hal_info, PS_PWR_STATE_BAND0_RFOFF);
+		break;
+	case PS_PWR_LVL_PWRON:
+		status = _hal_ps_cfg_pwr_on(phl_com, hal_info);
+		break;
+	default:
+		PHL_TRACE(COMP_PHL_PS, _PHL_WARNING_, "[HALPS], %s(): undefined pwr lvl!\n", __func__);
+		break;
+	}
+
+	return status;
+}
+
+void _hal_ps_proc_hw_rf_state_done(void* priv, struct phl_msg* msg)
+{
+	struct rtw_phl_com_t *phl_com = (struct rtw_phl_com_t *)priv;
+
+	if (msg->inbuf && msg->inlen) {
+		_os_kmem_free(phlcom_to_drvpriv(phl_com), msg->inbuf, msg->inlen);
+	}
+}
+
+/**
+ * configured requested power level
+ * return success if configure power level ok
+ * @phl_com: see rtw_phl_com_t
+ * @hal: see hal_info_t
+ * @req_pwr_lvl: target power level to configured
+ */
+enum rtw_hal_status
+rtw_hal_ps_pwr_lvl_cfg(struct rtw_phl_com_t *phl_com, void *hal,
+			u8 req_pwr_lvl)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	status = _hal_ps_pwr_lvl_cfg(phl_com, hal_info, req_pwr_lvl);
+	if (status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_PS, _PHL_ERR_, "[HALPS], %s(): pwr lvl cfg fail!\n", __func__);
+	}
+
+	return status;
+}
+
+/**
+ * configure the legacy power save (protocol)
+ * return configure lps fail or not
+ * @hal: see hal_info_t
+ * @lps: see rtw_hal_lps_info
+ */
+enum rtw_hal_status
+rtw_hal_ps_lps_cfg(void *hal, struct rtw_hal_lps_info *lps_info)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	status = _hal_ps_lps_cfg(hal_info, lps_info);
+
+	if (status == RTW_HAL_STATUS_SUCCESS) {
+		if (lps_info->en == false)
+			status = _hal_ps_lps_chk_leave(hal_info, lps_info->macid);
+	}
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_ps_pwr_req(struct rtw_phl_com_t *phl_com, u8 src, bool pwr_req)
+{
+	enum rtw_phl_status status = RTW_PHL_STATUS_FAILURE;
+
+	status = phl_ps_hal_pwr_req(phl_com, src, pwr_req);
+	if (status != RTW_PHL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_PS, _PHL_ERR_, "[HALPS], %s(): fail (pwr_req %d).\n", __func__, pwr_req);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+static enum rtw_hal_status
+_hal_ps_ips_chk_leave(struct hal_info_t *hal_info, u16 macid)
+{
+	enum rtw_hal_status status = RTW_HAL_STATUS_SUCCESS;
+	u32 start_time = _os_get_cur_time_ms();
+	u32 pass_time = 0;
+
+	do {
+		status = rtw_hal_mac_ips_chk_leave(hal_info, macid);
+		if (status == RTW_HAL_STATUS_SUCCESS) {
+			PHL_TRACE(COMP_PHL_PS, _PHL_INFO_,
+				"[HALPS], %s(): pass time = %d ms.\n",
+				__func__, pass_time);
+			break;
+		}
+
+		pass_time = phl_get_passing_time_ms(start_time);
+		if (pass_time > PS_PROTOCAL_LEAVE_TOLERANCE)
+			break;
+
+		_os_sleep_us(hal_to_drvpriv(hal_info), 50);
+
+	} while (1);
+
+#ifdef CONFIG_PHL_PS_FW_DBG
+	rtw_hal_fw_dbg_dump(hal_info);
+#endif
+	if (status != RTW_HAL_STATUS_SUCCESS)
+		PHL_TRACE(COMP_PHL_PS, _PHL_ERR_, "[HALPS], %s(): polling timeout!\n", __func__);
+
+	return status;
+}
+
+enum rtw_hal_status rtw_hal_ps_ips_cfg(void *hal,
+	struct rtw_hal_ips_info *ips_info)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	hstatus = rtw_hal_mac_ips_cfg(hal, ips_info->macid, ips_info->en);
+
+	if (hstatus == RTW_HAL_STATUS_SUCCESS && !ips_info->en)
+		hstatus = _hal_ps_ips_chk_leave(hal_info, ips_info->macid);
+
+	return hstatus;
+}
+
+#ifdef CONFIG_HW_RADIO_ONOFF_DETECT
+static void
+_hal_ps_ntfy_hw_rf_state(struct rtw_phl_com_t *phl_com,
+			enum rtw_rf_state rf_state)
+{
+	struct phl_msg msg = {0};
+	struct phl_msg_attribute attr = {0};
+	void *d = phlcom_to_drvpriv(phl_com);
+	enum rtw_rf_state *rf_ntfy = NULL;
+
+	rf_ntfy = (enum rtw_rf_state *)_os_kmem_alloc(d, sizeof(*rf_ntfy));
+	if (rf_ntfy == NULL) {
+		PHL_ERR("[HALPS], %s(): alloc for ntfy fail.\n", __func__);
+		return;
+	}
+
+	*rf_ntfy = rf_state;
+	msg.inbuf = (u8 *)rf_ntfy;
+	msg.inlen = sizeof(*rf_ntfy);
+
+	SET_MSG_MDL_ID_FIELD(msg.msg_id, PHL_MDL_POWER_MGNT);
+	SET_MSG_EVT_ID_FIELD(msg.msg_id, MSG_EVT_HW_RF_CHG);
+	attr.completion.completion = _hal_ps_proc_hw_rf_state_done;
+	attr.completion.priv = phl_com;
+	if (rtw_phl_msg_hub_hal_send(phl_com, &attr, &msg) !=
+		RTW_PHL_STATUS_SUCCESS) {
+		PHL_ERR("[HALPS], %s(): send msg failed\n", __func__);
+		_os_kmem_free(d, rf_ntfy, sizeof(*rf_ntfy));
+	}
+}
+
+void
+rtw_hal_ps_chk_hw_rf_state(struct rtw_phl_com_t *phl_com, void *hal)
+{
+#ifdef USE_TRUE_PHY
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	enum rtw_rf_state rf_state = RTW_RF_ON;
+	u8 val = 0;
+
+	hstatus = rtw_hal_mac_get_wl_dis_val(hal_info, &val);
+	if (hstatus != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_PS, _PHL_ERR_, "[HALPS], %s(): get wl dis val fail, status: %d\n",
+		          __func__, hstatus);
+		return;
+	}
+
+	/* get new rf state */
+	if (val == 1) {
+		rf_state = RTW_RF_ON;
+	} else if (val == 0) {
+		rf_state = RTW_RF_OFF;
+	} else {
+		PHL_TRACE(COMP_PHL_PS, _PHL_INFO_, "[HALPS], %s(): wl_dis is invalid value: %d\n",
+		          __func__, val);
+		return;
+	}
+
+	PHL_TRACE(COMP_PHL_PS, _PHL_INFO_, "[HALPS], %s(): rf state = %d\n",
+	          __func__, rf_state);
+	_hal_ps_ntfy_hw_rf_state(phl_com, rf_state);
+#endif /*USE_TRUE_PHY*/
+}
+#endif
+void rtw_hal_ps_notify_wake(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_mac_ps_notify_wake(hal_info);
+}
+
+enum rtw_hal_status
+rtw_hal_lps_bcn_tracking_cfg(void *hal,
+	struct rtw_bcn_tracking_cfg *cfg, struct rtw_bcn_tracking_cfg *cfg_cur)
+{
+	return rtw_hal_mac_lps_bcn_tracking_cfg(hal, cfg, cfg_cur);
+}
+
+void rtw_hal_lps_pvb_wait_rx(void *hal, struct rtw_phl_lps_adv_cfg *cur,
+			u16 macid, bool pvb_wait_rx)
+{
+	struct hal_info_t *hal_i = (struct hal_info_t *)hal;
+	enum ps_adv_action act = PS_ADV_NONE;
+
+	act = pvb_wait_rx ? PS_ADV_EN_WAIT_PVB_RX : PS_ADV_DIS_WAIT_PVB_RX;
+	rtw_hal_mac_lps_adv_setting(hal_i, cur, (u8)macid, act);
+}
+
+void rtw_hal_lps_ext_trx_ntfy(void *hal, struct rtw_phl_lps_adv_cfg *cur,
+				u16 macid)
+{
+	struct hal_info_t *hal_i = (struct hal_info_t *)hal;
+
+	rtw_hal_mac_lps_adv_setting(hal_i, cur, (u8)macid, PS_ADV_EN_TRX_EXT);
+}
+#endif /* CONFIG_POWER_SAVE */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ps.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ps.h
new file mode 100644
index 000000000000..905a1de2f608
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ps.h
@@ -0,0 +1,24 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_PS_H_
+#define _HAL_PS_H_
+#ifdef CONFIG_POWER_SAVE
+/* Can be used to verify sw flow without configure hw */
+#define HAL_LPS_SKIP_HW_CFG 0
+#define HAL_IPS_SKIP_HW_CFG 0
+
+enum rtw_hal_status rtw_hal_ps_pwr_req(struct rtw_phl_com_t *phl_com, u8 src, bool pwr_req);
+#endif
+#endif /*_HAL_PS_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_regulation.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_regulation.c
new file mode 100644
index 000000000000..67e76688f4da
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_regulation.c
@@ -0,0 +1,23 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_REGULATION_C_
+#include "hal_headers.h"
+
+bool rtw_hal_query_regulation(void *phl, struct rtw_regulation_info *info)
+{
+	return rtw_phl_query_regulation_info(phl, info);
+}
+
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_rx.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_rx.c
new file mode 100644
index 000000000000..01732497d927
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_rx.c
@@ -0,0 +1,1185 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_RX_C_
+#include "hal_headers.h"
+
+void rtw_hal_cfg_rxhci(void *hal, u8 en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s : enable %d.\n", __func__, en);
+
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_cfg_rxhci(hal_info, en))
+		PHL_ERR("%s failure \n", __func__);
+}
+
+enum rtw_hal_status
+rtw_hal_set_rxfltr_opt_by_mode(void *hal, u8 band, enum rtw_rx_fltr_opt_mode mode)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	enum rtw_rx_fltr_opt_mode set_mode = -1;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s : band(%d), mode(%d)\n",
+		__func__, band, mode);
+
+	/* Note: @hal_com->band[band].rx_fltr_opt_mode is used to recored any mode other than
+	 * sniffer mode, it effectively records the mode before entering monitor
+	 * mode and the subsequent modes set after entering monitor mode.
+	 */
+	if ((mode == RX_FLTR_OPT_MODE_SNIFFER && hal_info->monitor_mode[band]) ||
+	    (mode == RX_FLTR_OPT_MODE_RESTORE && !hal_info->monitor_mode[band]))
+		return RTW_HAL_STATUS_FAILURE;
+
+	if (hal_info->monitor_mode[band] && mode != RX_FLTR_OPT_MODE_RESTORE) {
+		hal_com->band[band].rx_fltr_opt_mode = mode;
+		return RTW_HAL_STATUS_SUCCESS;
+	}
+
+	set_mode = (mode == RX_FLTR_OPT_MODE_RESTORE) ?
+		hal_com->band[band].rx_fltr_opt_mode : mode;
+
+	hstatus = rtw_hal_mac_set_rxfltr_opt_by_mode(hal_com, band, set_mode);
+	if (hstatus != RTW_HAL_STATUS_SUCCESS)
+		return hstatus;
+
+	hal_info->monitor_mode[band] = (mode == RX_FLTR_OPT_MODE_SNIFFER);
+
+	/* Record @hal_com->band[band].rx_fltr_opt_mode only when the mode is not monitor and
+	 * restore, otherwise, it is kept intact.
+	 */
+	if (mode != RX_FLTR_OPT_MODE_SNIFFER &&
+	    mode != RX_FLTR_OPT_MODE_RESTORE)
+		hal_com->band[band].rx_fltr_opt_mode = mode;
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_rx_fltr_opt_mode rtw_hal_get_rxfltr_opt_mode(void *hal, u8 band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	return (hal_info->monitor_mode[band]) ? RX_FLTR_OPT_MODE_SNIFFER :
+		hal_com->band[band].rx_fltr_opt_mode;
+}
+
+enum rtw_hal_status rtw_hal_scan_set_rxfltr_by_mode(void *hinfo,
+	enum phl_phy_idx phy_idx, bool off_channel, u8 *mode)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+
+	if (off_channel) {
+		/* backup rx filter opt mode */
+		*mode = rtw_hal_get_rxfltr_opt_mode(hinfo, phy_idx);
+		hal_status = rtw_hal_set_rxfltr_opt_by_mode(hinfo,
+			phy_idx, RX_FLTR_OPT_MODE_SCAN);
+	} else {
+		/* restore rx filter opt mode */
+		hal_status = rtw_hal_set_rxfltr_opt_by_mode(hinfo,
+			phy_idx, *mode);
+	}
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_enter_mon_mode(void *hinfo, enum phl_phy_idx phy_idx)
+{
+	rtw_hal_bb_snif_mode_ctrl((struct hal_info_t *)hinfo, true);
+
+	return rtw_hal_set_rxfltr_opt_by_mode(hinfo, phy_idx, RX_FLTR_OPT_MODE_SNIFFER);
+}
+
+enum rtw_hal_status
+rtw_hal_leave_mon_mode(void *hinfo, enum phl_phy_idx phy_idx)
+{
+	rtw_hal_bb_snif_mode_ctrl((struct hal_info_t *)hinfo, false);
+
+	return rtw_hal_set_rxfltr_opt_by_mode(hinfo, phy_idx, RX_FLTR_OPT_MODE_RESTORE);
+}
+
+enum rtw_hal_status rtw_hal_acpt_crc_err_pkt(void *hal, u8 band, u8 enable)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+
+	return rtw_hal_mac_set_rxfltr_acpt_crc_err(hal_com, band, enable);
+}
+
+enum rtw_hal_status rtw_hal_cfg_ppdu_sts_fltr(void *hal, struct rtw_phl_com_t *phl_com, u8 band, u8 fltr)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_sts;
+	struct hal_ppdu_sts_cfg cfg;
+
+	hal_com->band[band].ppdu_sts_filter = fltr;
+
+	cfg.appen_info = hal_com->band[band].ppdu_sts_appen_info;
+	cfg.band_idx = band;
+	cfg.filter = hal_com->band[band].ppdu_sts_filter;
+	cfg.ppdu_stat_en = phl_com->ppdu_sts_info.en_ppdu_sts[band];
+	cfg.todcpu = false;
+	cfg.towcpu = false;
+
+
+	hal_sts = rtw_hal_mac_ppdu_stat_cfg(hal_info, &cfg);
+
+	return hal_sts;
+}
+
+enum rtw_hal_status rtw_hal_set_rxfltr_mpdu_size(void *hal, u8 band, u16 size)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+
+	return rtw_hal_mac_set_rxfltr_mpdu_size(hal_com, band, size);
+}
+
+static const char *_hal_rxfltr_type_str(u8 type)
+{
+	switch (type) {
+	case RTW_PHL_PKT_TYPE_CTRL:
+		return "CTRL";
+	case RTW_PHL_PKT_TYPE_MGNT:
+		return "MGNT";
+	case RTW_PHL_PKT_TYPE_DATA:
+		return "DATA";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *_hal_rxfltr_stype_str_ctrl(u8 stype)
+{
+	switch (stype) {
+	case RXFLTR_STYPE_CTRL_RSVD_0000:
+		return "RSVD_0000";
+	case RXFLTR_STYPE_CTRL_RSVD_0001:
+		return "RSVD_0001";
+	case RXFLTR_STYPE_TRIGGER:
+		return "TRIGGER";
+	case RXFLTR_STYPE_TACK:
+		return "TACK";
+	case RXFLTR_STYPE_BFRP:
+		return "BFRP";
+	case RXFLTR_STYPE_VHT_HE_NDPA:
+		return "VHT_HE_NDPA";
+	case RXFLTR_STYPE_CFE:
+		return "CFE";
+	case RXFLTR_STYPE_CW:
+		return "CW";
+	case RXFLTR_STYPE_BAR:
+		return "BAR";
+	case RXFLTR_STYPE_BA:
+		return "BA";
+	case RXFLTR_STYPE_PS_POLL:
+		return "PS_POLL";
+	case RXFLTR_STYPE_RTS:
+		return "RTS";
+	case RXFLTR_STYPE_CTS:
+		return "CTS";
+	case RXFLTR_STYPE_ACK:
+		return "ACK";
+	case RXFLTR_STYPE_CF_END:
+		return "CF_END";
+	case RXFLTR_STYPE_CF_END_ACK:
+		return "CF_END_ACK";
+	case RXFLTR_STYPE_CTRL_MAX:
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *_hal_rxfltr_stype_str_mgnt(u8 stype)
+{
+	switch (stype) {
+	case RXFLTR_STYPE_ASSOC_REQ:
+		return "ASSOC_REQ";
+	case RXFLTR_STYPE_ASSOC_RSP:
+		return "ASSOC_RSP";
+	case RXFLTR_STYPE_REASSOC_REQ:
+		return "REASSOC_REQ";
+	case RXFLTR_STYPE_REASSOC_RSP:
+		return "REASSOC_RSP";
+	case RXFLTR_STYPE_PROBE_REQ:
+		return "PROBE_REQ";
+	case RXFLTR_STYPE_PROBE_RSP:
+		return "PROBE_RSP";
+	case RXFLTR_STYPE_TIMING_ADV:
+		return "TIMING_ADV";
+	case RXFLTR_STYPE_MGNT_RSVD_0111:
+		return "RSVD_0111";
+	case RXFLTR_STYPE_BEACON:
+		return "BEACON";
+	case RXFLTR_STYPE_ATIM:
+		return "ATIM";
+	case RXFLTR_STYPE_DIS_ASSOC:
+		return "DIS_ASSOC";
+	case RXFLTR_STYPE_AUTH:
+		return "AUTH";
+	case RXFLTR_STYPE_DEAUTH:
+		return "DAUTH";
+	case RXFLTR_STYPE_ACTION:
+		return "ACTION";
+	case RXFLTR_STYPE_ACTION_NO_ACK:
+		return "ACTION_NO_ACK";
+	case RXFLTR_STYPE_MGNT_RSVD_1111:
+		return "MGNT_RSVD_1111";
+	case RXFLTR_STYPE_MGNT_MAX:
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *_hal_rxfltr_stype_str_data(u8 stype)
+{
+	switch (stype) {
+	case RXFLTR_STYPE_DATA_DATA:
+		return "DATA_DATA";
+	case RXFLTR_STYPE_DATA_CF_ACK:
+		return "DATA_CF_ACK";
+	case RXFLTR_STYPE_DATA_CF_POLL:
+		return "DATA_CF_POLL";
+	case RXFLTR_STYPE_DATA_CF_ACK_POLL:
+		return "DATA_CF_ACK_POLL";
+	case RXFLTR_STYPE_NULL_DATA:
+		return "NULL_DATA";
+	case RXFLTR_STYPE_CF_ACK:
+		return "CF_ACK";
+	case RXFLTR_STYPE_CF_POLL:
+		return "CF_POLL";
+	case RXFLTR_STYPE_CF_ACK_POLL:
+		return "CF_ACK_POLL";
+	case RXFLTR_STYPE_QOS_DATA:
+		return "QOS_DATA";
+	case RXFLTR_STYPE_QOS_DATA_CF_ACK:
+		return "QOS_DATA_CF_ACK";
+	case RXFLTR_STYPE_QOS_DATA_CF_POLL:
+		return "QOS_DATA_CF_POLL";
+	case RXFLTR_STYPE_QOS_DATA_CF_ACK_POLL:
+		return "QOS_DATA_CF_ACK_POLL";
+	case RXFLTR_STYPE_QOS_NULL_DATA:
+		return "QOS_NULL_DATA";
+	case RXFLTR_STYPE_DATA_RSVD_1101:
+		return "RSVD_1101";
+	case RXFLTR_STYPE_QOS_CF_POLL:
+		return "QOS_CF_POLL";
+	case RXFLTR_STYPE_QOS_CF_ACK_POLL:
+		return "QOS_CF_ACK_POLL";
+	case RXFLTR_STYPE_DATA_MAX:
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *_hal_rxfltr_target_str(u8 target)
+{
+	switch (target) {
+	case RXFLTR_TARGET_DROP:
+		return "DROP";
+	case RXFLTR_TARGET_TO_HOST:
+		return "TO_HOST";
+	case RXFLTR_TARGET_TO_WLCPU:
+		return "TO_WLCPU";
+	case RXFLTR_TARGET_MAX:
+	default:
+		return "UNKNOWN";
+	}
+}
+
+enum rtw_hal_status rtw_hal_set_rxfltr_by_type(void *hal, u8 band, enum rtw_packet_type type,
+					       enum rtw_rxfltr_target target)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_FAILURE;
+
+	hstats = rtw_hal_mac_set_rxfltr_by_type(hal_info->hal_com, band, type, target);
+
+	if (RTW_HAL_STATUS_SUCCESS == hstats)
+		PHL_TRACE(COMP_PHL_RECV, _PHL_INFO_, "Set band %u rxfltr type %s target %s\n",
+			band, _hal_rxfltr_type_str(type), _hal_rxfltr_target_str(target));
+	else
+		PHL_TRACE(COMP_PHL_RECV, _PHL_ERR_, "%s : set band %u type %s target %s fail, status(%u)\n",
+			__func__, band, _hal_rxfltr_type_str(type), _hal_rxfltr_target_str(target), hstats);
+
+	return hstats;
+}
+
+enum rtw_hal_status
+rtw_hal_set_rxfltr_by_stype_ctrl(void *hal, u8 band, struct rxfltr_cap_to_set_ctrl *cap_ctrl)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_SUCCESS;
+	enum rtw_packet_type type = RTW_PHL_PKT_TYPE_CTRL;
+	enum rtw_rxfltr_target target = RXFLTR_TARGET_MAX, trgt_chk = RXFLTR_TARGET_MAX;
+	bool type_set = true, stype_set = false;
+	u8 i = 0;
+
+	for (i = 0; i < RXFLTR_STYPE_CTRL_MAX; i++) {
+		if (cap_ctrl->stype[i].set == false) {
+			type_set = false;
+			break;
+		}
+		if ((trgt_chk != RXFLTR_TARGET_MAX) && (cap_ctrl->stype[i].target != trgt_chk)) {
+			type_set = false;
+			break;
+		}
+
+		trgt_chk = cap_ctrl->stype[i].target;
+	}
+
+	if (type_set == true)
+		return rtw_hal_set_rxfltr_by_type(hal, band, type, trgt_chk);
+
+	for (i = 0; i < RXFLTR_STYPE_CTRL_MAX; i++) {
+		stype_set = cap_ctrl->stype[i].set;
+		target = cap_ctrl->stype[i].target;
+		if (stype_set == true) {
+			hstats = rtw_hal_mac_set_rxfltr_by_subtype(hal_com, band, type, i, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats) {
+				PHL_TRACE(COMP_PHL_RECV, _PHL_ERR_, "%s: set band %u type %s sub-type %s target %s fail, status(%u)\n",
+					__func__, band, _hal_rxfltr_type_str(type), _hal_rxfltr_stype_str_ctrl(i),
+					_hal_rxfltr_target_str(target), hstats);
+				break;
+			}
+
+			PHL_TRACE(COMP_PHL_RECV, _PHL_INFO_, "Set band %u rxfltr type %s sub-type %s target %s\n",
+				band, _hal_rxfltr_type_str(type), _hal_rxfltr_stype_str_ctrl(i), _hal_rxfltr_target_str(target));
+		}
+	}
+
+	return hstats;
+}
+
+enum rtw_hal_status
+rtw_hal_set_rxfltr_by_stype_mgnt(void *hal, u8 band, struct rxfltr_cap_to_set_mgnt *cap_mgnt)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_SUCCESS;
+	enum rtw_packet_type type = RTW_PHL_PKT_TYPE_MGNT;
+	enum rtw_rxfltr_target target = RXFLTR_TARGET_MAX, trgt_chk = RXFLTR_TARGET_MAX;
+	bool type_set = true, stype_set = false;
+	u8 i = 0;
+
+	for (i = 0; i < RXFLTR_STYPE_MGNT_MAX; i++) {
+		if (cap_mgnt->stype[i].set == false) {
+			type_set = false;
+			break;
+		}
+		if ((trgt_chk != RXFLTR_TARGET_MAX) && (cap_mgnt->stype[i].target != trgt_chk)) {
+			type_set = false;
+			break;
+		}
+
+		trgt_chk = cap_mgnt->stype[i].target;
+	}
+
+	if (type_set == true)
+		return rtw_hal_set_rxfltr_by_type(hal, band, type, trgt_chk);
+
+	for (i = 0; i < RXFLTR_STYPE_MGNT_MAX; i ++) {
+		stype_set = cap_mgnt->stype[i].set;
+		target = cap_mgnt->stype[i].target;
+		if (stype_set == true) {
+			hstats = rtw_hal_mac_set_rxfltr_by_subtype(hal_com, band, type, i, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats) {
+				PHL_TRACE(COMP_PHL_RECV, _PHL_ERR_, "%s: set band %u type %s sub-type %s target %s fail, status(%u)\n",
+					__func__, band, _hal_rxfltr_type_str(type), _hal_rxfltr_stype_str_mgnt(i),
+					_hal_rxfltr_target_str(target), hstats);
+				break;
+			}
+
+			PHL_TRACE(COMP_PHL_RECV, _PHL_INFO_, "Set band %u rxfltr type %s sub-type %s target %s\n",
+				band, _hal_rxfltr_type_str(type), _hal_rxfltr_stype_str_mgnt(i), _hal_rxfltr_target_str(target));
+		}
+	}
+
+	return hstats;
+}
+
+enum rtw_hal_status
+rtw_hal_set_rxfltr_by_stype_data(void *hal, u8 band, struct rxfltr_cap_to_set_data *cap_data)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_SUCCESS;
+	enum rtw_packet_type type = RTW_PHL_PKT_TYPE_DATA;
+	enum rtw_rxfltr_target target = RXFLTR_TARGET_MAX, trgt_chk = RXFLTR_TARGET_MAX;
+	bool type_set = true, stype_set = false;
+	u8 i = 0;
+
+	for (i = 0; i < RXFLTR_STYPE_DATA_MAX; i++) {
+		if (cap_data->stype[i].set == false) {
+			type_set = false;
+			break;
+		}
+		if ((trgt_chk != RXFLTR_TARGET_MAX) && (cap_data->stype[i].target != trgt_chk)) {
+			type_set = false;
+			break;
+		}
+
+		trgt_chk = cap_data->stype[i].target;
+	}
+
+	if (type_set == true)
+		return rtw_hal_set_rxfltr_by_type(hal, band, type, trgt_chk);
+
+	for (i = 0; i < RXFLTR_STYPE_DATA_MAX; i++) {
+		stype_set = cap_data->stype[i].set;
+		target = cap_data->stype[i].target;
+		if (stype_set == true) {
+			hstats = rtw_hal_mac_set_rxfltr_by_subtype(hal_com, band, type, i, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats) {
+				PHL_TRACE(COMP_PHL_RECV, _PHL_ERR_, "%s: set band %u type %s sub-type %s target %s fail, status(%u)\n",
+					__func__, band, _hal_rxfltr_type_str(type), _hal_rxfltr_stype_str_data(i),
+					_hal_rxfltr_target_str(target), hstats);
+				break;
+			}
+
+			PHL_TRACE(COMP_PHL_RECV, _PHL_INFO_, "Set band %u rxfltr type %s sub-type %s target %s\n",
+				band, _hal_rxfltr_type_str(type), _hal_rxfltr_stype_str_data(i), _hal_rxfltr_target_str(target));
+		}
+	}
+
+	return hstats;
+}
+
+enum rtw_hal_status
+rtw_hal_set_rxfltr_by_stype_all(void *hal, u8 band, struct rxfltr_type_cap_to_set *cap_all)
+{
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_FAILURE;
+
+	do {
+		hstats = rtw_hal_set_rxfltr_by_stype_ctrl(hal, band, &(cap_all->rxfltr_ctrl));
+		if (RTW_HAL_STATUS_SUCCESS != hstats)
+			break;
+
+		hstats = rtw_hal_set_rxfltr_by_stype_mgnt(hal, band, &(cap_all->rxfltr_mgnt));
+		if (RTW_HAL_STATUS_SUCCESS != hstats)
+			break;
+
+		hstats = rtw_hal_set_rxfltr_by_stype_data(hal, band, &(cap_all->rxfltr_data));
+		if (RTW_HAL_STATUS_SUCCESS != hstats)
+			break;
+	} while (0);
+
+	return hstats;
+}
+
+enum rtw_hal_status rtw_hal_set_rxfltr_type_by_mode(void *hal, u8 band, enum rtw_rxfltr_type_mode mode)
+{
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_FAILURE;
+	struct rxfltr_cap_to_set_ctrl cap_ctrl = {0};
+	enum rtw_packet_type type = RTW_PHL_PKT_TYPE_MAX;
+	enum rtw_rxfltr_target target = RXFLTR_TARGET_MAX;
+
+	if (band >= MAX_BAND_NUM)
+		return RTW_HAL_STATUS_FAILURE;
+
+	if (mode >= RX_FLTR_TYPE_MODE_MAX)
+		return RTW_HAL_STATUS_FAILURE;
+
+	switch (mode) {
+	case RX_FLTR_TYPE_MODE_HAL_INIT:
+		do {
+			type = RTW_PHL_PKT_TYPE_CTRL;
+			target = RXFLTR_TARGET_DROP;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_MGNT;
+			target = RXFLTR_TARGET_TO_HOST;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_DATA;
+			target = RXFLTR_TARGET_TO_HOST;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+		} while (0);
+		break;
+	case RX_FLTR_TYPE_MODE_STA_WOW_INIT_PRE:
+		do {
+			type = RTW_PHL_PKT_TYPE_CTRL;
+			target = RXFLTR_TARGET_DROP;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_MGNT;
+			target = RXFLTR_TARGET_DROP;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_DATA;
+			target = RXFLTR_TARGET_DROP;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+		} while (0);
+		break;
+	case RX_FLTR_TYPE_MODE_STA_WOW_INIT_POST:
+		do {
+			cap_ctrl.stype[RXFLTR_STYPE_VHT_HE_NDPA].set = true;
+			cap_ctrl.stype[RXFLTR_STYPE_VHT_HE_NDPA].target = RXFLTR_TARGET_TO_HOST;
+			cap_ctrl.stype[RXFLTR_STYPE_BAR].set = true;
+			cap_ctrl.stype[RXFLTR_STYPE_BAR].target = RXFLTR_TARGET_TO_HOST;
+			hstats = rtw_hal_set_rxfltr_by_stype_ctrl(hal, band, &cap_ctrl);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_MGNT;
+			target = RXFLTR_TARGET_TO_HOST;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_DATA;
+			target = RXFLTR_TARGET_TO_HOST;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+		} while (0);
+		break;
+#ifdef CONFIG_PHL_TEST_MP
+	case RX_FLTR_TYPE_MODE_MP:
+		do {
+			struct rxfltr_cap_to_set_mgnt cap_mgnt = {0};
+
+			type = RTW_PHL_PKT_TYPE_CTRL;
+			target = RXFLTR_TARGET_DROP;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_MGNT;
+			target = RXFLTR_TARGET_DROP;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			type = RTW_PHL_PKT_TYPE_DATA;
+			target = RXFLTR_TARGET_TO_HOST;
+			hstats = rtw_hal_set_rxfltr_by_type(hal, band, type, target);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+
+			cap_mgnt.stype[RXFLTR_STYPE_BEACON].set = true;
+			cap_mgnt.stype[RXFLTR_STYPE_BEACON].target = RXFLTR_TARGET_TO_HOST;
+			hstats = rtw_hal_set_rxfltr_by_stype_mgnt(hal, band, &cap_mgnt);
+			if (RTW_HAL_STATUS_SUCCESS != hstats)
+				break;
+		} while (0);
+		break;
+#endif
+	default:
+		PHL_TRACE(COMP_PHL_RECV, _PHL_ERR_, "%s: does not handle rxfltr mode(%u)\n", __func__, mode);
+		break;
+	}
+
+	return hstats;
+}
+
+enum rtw_hal_status
+rtw_hal_poll_hw_rx_done(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_SUCCESS;
+
+	sts = rtw_hal_mac_poll_hw_rx_done(hal_info);
+
+	return sts;
+}
+
+enum rtw_hal_status
+rtw_hal_hw_rx_resume(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_SUCCESS;
+
+	sts = rtw_hal_mac_hw_rx_resume(hal_info);
+
+	return sts;
+}
+
+#ifdef CONFIG_PCI_HCI
+/**
+ * rtw_hal_rx_res_query - query current HW rx resource with specifc dma channel
+ * @hal: see struct hal_info_t
+ * @dma_ch: the target dma channel
+ * @host_idx: current host index of this channel
+ * @hw_idx: current hw index of this channel
+ *
+ * this function returns the number of available tx resource
+ * NOTE, input host_idx and hw_idx ptr shall NOT be NULL
+ */
+u16 rtw_hal_rx_res_query(void *hal, u8 dma_ch, u16 *host_idx, u16 *hw_idx)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u16 res_num = 0;
+
+	sts = rtw_hal_mac_rx_res_query(hal_info, dma_ch, host_idx, hw_idx,
+	                               &res_num);
+
+	if (sts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_RECV, _PHL_WARNING_, "%s : query rx resource fail!\n",
+		          __func__);
+	}
+
+	return res_num;
+}
+
+u16 rtw_hal_get_rxbd_num(void *hal, u8 dma_ch)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u16 res_num = 0;
+
+	res_num = trx_ops->get_rxbd_num(hal_info->hal_com, dma_ch);
+
+	return res_num;
+}
+
+u16 rtw_hal_get_rxbuf_num(void *hal, u8 dma_ch)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u16 res_num = 0;
+
+	res_num = trx_ops->get_rxbuf_num(hal_info->hal_com, dma_ch);
+
+	return res_num;
+}
+
+u16 rtw_hal_get_rxbuf_size(void *hal, u8 dma_ch)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u16 res_num = 0;
+
+	res_num = trx_ops->get_rxbuf_size(hal_info->hal_com, dma_ch);
+
+	return res_num;
+}
+
+/**
+ * rtw_hal_query_rxch_num - query total hw rx dma channels number
+ *
+ * returns the number of  hw rx dma channel
+ */
+u8 rtw_hal_query_rxch_num(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u8 ch_num = 0;
+
+	ch_num = trx_ops->query_rxch_num();
+
+	return ch_num;
+}
+
+u8 rtw_hal_check_rxrdy(struct rtw_phl_com_t *phl_com, void* hal,
+		       struct rtw_rx_buf *rxbuf, u8 dma_ch)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u8 res = 0;
+
+	res = trx_ops->check_rxrdy(phl_com, rxbuf, dma_ch);
+
+	return res;
+}
+
+u8 rtw_hal_handle_rxbd_info(void* hal, u8 *rxbuf, u16 *buf_size)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u8 res = 0;
+
+	res = trx_ops->handle_rxbd_info(hal_info, rxbuf, buf_size);
+
+	return res;
+}
+
+enum rtw_hal_status
+rtw_hal_update_rxbd(void *hal, struct rx_base_desc *rxbd,
+					struct rtw_rx_buf *rxbuf, u8 ch_idx)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->update_rxbd(hal_info, rxbd, rxbuf, ch_idx);
+
+	return hstatus;
+}
+
+
+enum rtw_hal_status
+rtw_hal_notify_rxdone(void* hal, struct rx_base_desc *rxbd, u8 dma_ch)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	sts = rtw_hal_mac_notify_rxdone(hal_info, rxbd, dma_ch);
+	if (sts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_RECV, _PHL_INFO_, "%s : notify rx done fail!\n",
+		          __func__);
+	}
+
+	return sts;
+}
+
+u16 rtw_hal_handle_wp_rpt(void *hal, u8 *rp, u16 len, u8 *sw_retry, u8 *dma_ch,
+			  u16 *wp_seq, u8 *macid, u8 *ac_queue, u8 *txsts)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u16 rsize = 0;
+
+	rsize = trx_ops->handle_wp_rpt(hal_info, rp, len, sw_retry, dma_ch,
+				       wp_seq, macid, ac_queue, txsts);
+	return rsize;
+}
+
+#endif /*CONFIG_PCI_HCI*/
+
+
+#ifdef CONFIG_USB_HCI
+enum rtw_hal_status
+rtw_hal_query_info(void* hal, u8 info_id, void *value)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->query_hal_info(hal_info, info_id, value);
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+hal_usb_rx_agg_cfg(struct hal_info_t *hal, u8 mode, u8 agg_mode,
+		   u8 drv_define, u8 timeout, u8 size, u8 pkt_num)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	switch (mode){
+	case PHL_RX_AGG_DISABLE:
+		hstatus = hal_mac_usb_rx_agg_cfg(hal, MAC_AX_RX_AGG_MODE_NONE,
+						 0, 0, 0, 0);
+		break;
+	case PHL_RX_AGG_DEFAULT:
+		hstatus = hal_mac_usb_rx_agg_cfg(hal, MAC_AX_RX_AGG_MODE_USB,
+						 0, 0, 0, 0);
+		break;
+	case PHL_RX_AGG_SMALL_PKT:
+		hstatus = hal_mac_usb_rx_agg_cfg(hal, MAC_AX_RX_AGG_MODE_USB,
+						 1, 0x01, 0x01, 0);
+		break;
+	case PHL_RX_AGG_USER_DEFINE:
+		hstatus = hal_mac_usb_rx_agg_cfg(hal, agg_mode, drv_define,
+						 timeout, size, pkt_num);
+		break;
+	default:
+		hstatus = RTW_HAL_STATUS_FAILURE;
+		break;
+	}
+	return hstatus;
+}
+
+enum rtw_hal_status
+	rtw_hal_usb_rx_agg_cfg(void *hal, u8 mode, u8 agg_mode,
+	u8 drv_define, u8 timeout, u8 size, u8 pkt_num)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->usb_rx_agg_cfg(hal, mode, agg_mode,
+		drv_define, timeout, size, pkt_num);
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_usb_rx_agg_init(struct rtw_phl_com_t *phl_com, void *hal)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct bus_sw_cap_t *bus_sw = &phl_com->bus_sw_cap;
+	struct bus_hw_cap_t *bus_hw = &hal_com->bus_hw_cap;
+
+	if (bus_sw->rx_buf_size == 0 ||
+	    bus_sw->rx_buf_size > bus_hw->rx_buf_size) {
+		hstatus = rtw_hal_usb_rx_agg_cfg(hal, PHL_RX_AGG_DEFAULT,
+						 0, 0, 0, 0, 0);
+	} else {
+		/*
+		 * agg_size unit is 1024 bytes. We make rx_buf_size 4096 bytes
+		 * more than agg_size to avoid the case that the data size of
+		 * USB Rx exceeds rx_buf_size.
+		 */
+		u8 agg_size = (u8)(bus_sw->rx_buf_size / 1024) - 4;
+
+		hstatus = rtw_hal_usb_rx_agg_cfg(hal, PHL_RX_AGG_USER_DEFINE,
+						 MAC_AX_RX_AGG_MODE_USB , 1, 32,
+						 agg_size, 0);
+	}
+
+	return hstatus;
+}
+
+u16 rtw_hal_handle_wp_rpt_usb(void *hal, u8 *rp, u16 len, u8 *macid, u8 *ac_queue,
+		u8 *txsts)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u16 rsize = 0;
+
+	rsize = trx_ops->handle_wp_rpt(hal_info, rp, len, macid, ac_queue, txsts);
+	return rsize;
+}
+
+#endif
+
+enum rtw_hal_status
+rtw_hal_handle_rx_buffer(struct rtw_phl_com_t *phl_com, void* hal,
+				u8 *buf, u32 buf_size,
+				struct rtw_phl_rx_pkt *rxpkt)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->handle_rx_buffer(phl_com, hal_info,
+					      buf, buf_size, rxpkt);
+
+	return hstatus;
+}
+
+#ifdef CONFIG_SDIO_HCI
+void rtw_hal_sdio_rx_agg_cfg(void *hal, bool enable, u8 drv_define,
+			     u8 timeout, u8 size, u8 pkt_num)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+
+	rtw_hal_mac_sdio_rx_agg_cfg(hal_info->hal_com, enable, drv_define,
+				    timeout, size, pkt_num);
+}
+
+int rtw_hal_sdio_rx(void *hal, struct rtw_rx_buf *rxbuf)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+
+	return rtw_hal_mac_sdio_rx(hal_info->hal_com, rxbuf);
+}
+
+int rtw_hal_sdio_parse_rx(void *hal, struct rtw_rx_buf *rxbuf)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+
+	return rtw_hal_mac_sdio_parse_rx(hal_info->hal_com, rxbuf);
+}
+
+/**
+ * rtw_hal_sdio_lps_flg - get lps status
+ * @hal:	pointer of struct hal_info_t
+ *
+ * Get driver lps status
+ *
+ * return lps flag state defined in enum rtw_hal_lps_flg_state
+ * RTW_HAL_LPS_FLG_STATE_LPS when driver set LPS-CG/LPS-PG enable;
+ * RTW_HAL_LPS_FLG_STATE_ACTIVE when driver is not in LPS-CG/LPS-PG
+ */
+enum rtw_hal_lps_flg_state
+rtw_hal_sdio_lps_flg(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 lps_flg = 0;
+	enum rtw_hal_lps_flg_state state = RTW_HAL_LPS_FLG_STATE_ACTIVE;
+
+	rtw_hal_mac_sdio_lps_flg(hal_info->hal_com, &lps_flg);
+
+	if (lps_flg == 1)
+		state = RTW_HAL_LPS_FLG_STATE_LPS;
+
+	return state;
+}
+#endif /* CONFIG_SDIO_HCI */
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+static void
+hal_rx_ppdu_sts_normal_data_csi_chk(struct rtw_phl_com_t *phl_com,
+	struct rtw_phl_ppdu_sts_ent *ppdu_sts_ent, void *hdr, struct rtw_r_meta_data *meta)
+{
+	struct rtw_chinfo_cur_parm *cur_parm = phl_com->cur_parm;
+	bool bypass_csi = true;
+
+	if (!cur_parm->action_parm.enable)
+		goto exit;
+
+	switch (cur_parm->action_parm.mode) {
+#ifdef CONFIG_PHL_WKARD_CHANNEL_INFO_ACK
+	case CHINFO_MODE_ACK:
+		if (!rtw_hal_ch_info_process_ack(meta, ppdu_sts_ent, cur_parm, meta->macid))
+			goto exit;
+
+		if (meta->macid != 0xff) {
+			struct rtw_phl_stainfo_t *sta = rtw_phl_get_stainfo_by_macid(phl_com->phl_priv, meta->macid);
+
+			_os_mem_cpy(phl_com->drv_priv, ppdu_sts_ent->src_mac_addr,
+				sta->mac_addr, MAC_ADDRESS_LENGTH);
+			ppdu_sts_ent->freerun_cnt = meta->freerun_cnt;
+		}
+
+		break;
+#endif
+	default:
+		break;
+	}
+
+	bypass_csi = false;
+
+exit:
+	ppdu_sts_ent->bypass_csi = bypass_csi;
+}
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+
+void
+hal_rx_ppdu_sts_normal_data(struct rtw_phl_com_t *phl_com,
+			    void *hdr,
+			    struct rtw_r_meta_data *meta)
+{
+	struct rtw_phl_ppdu_sts_info *ppdu_info = NULL;
+	enum phl_band_idx band = HW_BAND_0;
+
+	do {
+		if (NULL == meta)
+			break;
+		ppdu_info = &phl_com->ppdu_sts_info;
+		band = (meta->bb_sel > 0) ? HW_BAND_1 : HW_BAND_0;
+		if ((ppdu_info->cur_rx_ppdu_cnt[band] == meta->ppdu_cnt) &&
+		    (false == ppdu_info->latest_rx_is_psts[band])) {
+			/**
+			 * add condition to avoid check fail for ppdu cnt run around 0 -> 1 -> ... -> 0
+			 * example :
+			 * 	[frame_A(ppdu_cnt = 0)] -> [ppdu_sts(ppdu_cnt = 0)]
+			 *  	->[ppdu_sts(ppdu_cnt = 1)] -> [ppdu_sts(ppdu_cnt = 2)] ...
+			 * 	... ->[ppdu_sts(ppdu_cnt = 7)] -> [frame_B(ppdu_cnt = 0)] ...
+			 * 	Therefore, frame_B has same ppdu_cnt with frame_A.
+			 *	But they are different PPDU.
+			**/
+			break;
+		}
+		meta->ppdu_cnt_chg = true;
+		/* start of the PPDU */
+		ppdu_info->latest_rx_is_psts[band] = false;
+		ppdu_info->sts_ent[band][meta->ppdu_cnt].addr_cam_vld = meta->addr_cam_vld;
+		ppdu_info->sts_ent[band][meta->ppdu_cnt].frame_type = PHL_GET_80211_HDR_TYPE(hdr);
+		ppdu_info->sts_ent[band][meta->ppdu_cnt].crc32 = meta->crc32;
+		ppdu_info->sts_ent[band][meta->ppdu_cnt].rx_rate = meta->rx_rate;
+		ppdu_info->sts_ent[band][meta->ppdu_cnt].ppdu_type = meta->ppdu_type;
+		ppdu_info->sts_ent[band][meta->ppdu_cnt].pkt_freerun_cnt = meta->freerun_cnt;
+
+		if(RTW_IS_BEACON_OR_PROBE_RESP_PKT(ppdu_info->sts_ent[band][meta->ppdu_cnt].frame_type)) {
+			PHL_GET_80211_HDR_ADDRESS3(phl_com->drv_priv, hdr,
+				ppdu_info->sts_ent[band][meta->ppdu_cnt].src_mac_addr);
+		}
+		else if (meta->a1_match &&
+			  RTW_IS_ASOC_REQ_PKT(ppdu_info->sts_ent[band][meta->ppdu_cnt].frame_type)) {
+
+			PHL_GET_80211_HDR_ADDRESS2(phl_com->drv_priv, hdr,
+				ppdu_info->sts_ent[band][meta->ppdu_cnt].src_mac_addr);
+
+			#ifdef DBG_AP_CLIENT_ASSOC_RSSI
+			{
+				u8 *src = NULL;
+
+				src = ppdu_info->sts_ent[band][meta->ppdu_cnt].src_mac_addr;
+
+				PHL_INFO("%s [Rx-ASOC_REQ] - MAC-Addr:%02x-%02x-%02x-%02x-%02x-%02x, a1_match:%d ppdu_cnt:%d\n",
+					__func__,
+					src[0], src[1], src[2], src[3], src[4], src[5],
+					meta->a1_match,
+					meta->ppdu_cnt);
+			}
+			#endif
+		}
+		else {
+			_os_mem_cpy(phl_com->drv_priv,
+				ppdu_info->sts_ent[band][meta->ppdu_cnt].src_mac_addr,
+				meta->ta, MAC_ADDRESS_LENGTH);
+		}
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+		hal_rx_ppdu_sts_normal_data_csi_chk(phl_com, &ppdu_info->sts_ent[band][meta->ppdu_cnt], hdr, meta);
+#endif
+
+		ppdu_info->sts_ent[band][meta->ppdu_cnt].valid = false;
+		ppdu_info->cur_rx_ppdu_cnt[band] = meta->ppdu_cnt;
+		PHL_TRACE(COMP_PHL_PSTS, _PHL_INFO_,
+				"Start of the PPDU : band %d ; ppdu_cnt %d ; frame_type %d ; addr_cam_vld %d ; size %d ; rate 0x%x ; crc32 %d\n",
+				band,
+				ppdu_info->cur_rx_ppdu_cnt[band],
+				ppdu_info->sts_ent[band][meta->ppdu_cnt].frame_type,
+				ppdu_info->sts_ent[band][meta->ppdu_cnt].addr_cam_vld,
+				meta->pktlen,
+				meta->rx_rate,
+				meta->crc32);
+	} while (false);
+
+}
+
+void
+hal_rx_ppdu_sts(struct rtw_phl_com_t *phl_com,
+		struct rtw_phl_rx_pkt *phl_rx,
+		struct hal_ppdu_sts *ppdu_sts)
+{
+	struct rtw_phl_ppdu_sts_info *ppdu_info = NULL;
+	struct rtw_phl_rssi_stat *rssi_stat = NULL;
+	struct rtw_r_meta_data *meta = &(phl_rx->r.mdata);
+	struct rtw_phl_ppdu_phy_info *phy_info = &(phl_rx->r.phy_info);
+	u8 i = 0;
+	enum phl_band_idx band = HW_BAND_0;
+	struct rtw_phl_ppdu_sts_ent *sts_ent = NULL;
+
+	if ((NULL == phl_com) || (NULL == meta) || (NULL == ppdu_sts))
+		return;
+
+	ppdu_info = &phl_com->ppdu_sts_info;
+	rssi_stat = &phl_com->rssi_stat;
+	band = (meta->bb_sel > 0) ? HW_BAND_1 : HW_BAND_0;
+	ppdu_info->latest_rx_is_psts[band] = true;
+
+	if (0 == phy_info->is_valid)
+		return;
+
+	if (ppdu_info->cur_rx_ppdu_cnt[band] != meta->ppdu_cnt) {
+		PHL_TRACE(COMP_PHL_PSTS, _PHL_INFO_,
+			  "[WARNING] ppdu cnt mis-match (band %d ; cur : %d , rxmeta : %d)\n",
+			  band,
+			  ppdu_info->cur_rx_ppdu_cnt[band],
+			  meta->ppdu_cnt);
+	}
+	sts_ent = &(ppdu_info->sts_ent[band][meta->ppdu_cnt]);
+
+	if (meta->crc32 || sts_ent->crc32) {
+		UPDATE_MA_RSSI(rssi_stat, RTW_RSSI_UNKNOWN,
+			 phy_info->rssi);
+		if (phl_com->drv_mode != RTW_DRV_MODE_SNIFFER)
+			return;
+	}
+	if (sts_ent->rx_rate != meta->rx_rate) {
+		PHL_TRACE(COMP_PHL_PSTS, _PHL_INFO_,
+			  "[WARNING] PPDU STS rx rate mis-match\n");
+		UPDATE_MA_RSSI(rssi_stat, RTW_RSSI_UNKNOWN,
+			       phy_info->rssi);
+		return;
+	}
+	if (sts_ent->ppdu_type != meta->ppdu_type) {
+		PHL_TRACE(COMP_PHL_PSTS, _PHL_INFO_,
+			  "[WARNING] PPDU STS ppdu_type mis-match\n");
+		UPDATE_MA_RSSI(rssi_stat, RTW_RSSI_UNKNOWN,
+			       phy_info->rssi);
+		return;
+	}
+	if (sts_ent->valid == true) {
+		PHL_TRACE(COMP_PHL_PSTS, _PHL_INFO_,
+			  "[WARNING] PPDU STS is already updated, skip this ppdu status\n");
+		return;
+	}
+
+	/* update ppdu_info entry */
+	sts_ent->freerun_cnt = meta->freerun_cnt;
+	_os_mem_cpy(phl_com->drv_priv,
+		    &(sts_ent->phy_info),
+		    phy_info, sizeof(struct rtw_phl_ppdu_phy_info));
+
+	sts_ent->usr_num = ppdu_sts->usr_num;
+	for (i = 0; i < ppdu_sts->usr_num; i++) {
+		if (ppdu_sts->usr[i].vld) {
+			sts_ent->sta[i].macid =
+				ppdu_sts->usr[i].macid;
+			sts_ent->sta[i].vld = 1;
+		} else {
+			sts_ent->sta[i].vld = 0;
+		}
+	}
+	sts_ent->phl_done = false;
+	sts_ent->valid = true;
+
+	/* update rssi stat */
+	_os_spinlock(phl_com->drv_priv, &rssi_stat->lock, _bh, NULL);
+	rssi_stat->last_rx_freerun = sts_ent->pkt_freerun_cnt;
+	switch (sts_ent->frame_type &
+		(BIT(1) | BIT(0))) {
+		case RTW_FRAME_TYPE_MGNT :
+			if (sts_ent->addr_cam_vld) {
+				UPDATE_MA_RSSI(rssi_stat,
+					 (1 == meta->a1_match) ?
+					  RTW_RSSI_MGNT_ACAM_A1M :
+					  RTW_RSSI_MGNT_ACAM,
+					 phy_info->rssi);
+			} else {
+				UPDATE_MA_RSSI(rssi_stat, RTW_RSSI_MGNT_OTHER,
+					 phy_info->rssi);
+			}
+		break;
+		case RTW_FRAME_TYPE_CTRL :
+			if (sts_ent->addr_cam_vld) {
+				UPDATE_MA_RSSI(rssi_stat,
+					 (1 == meta->a1_match) ?
+					  RTW_RSSI_CTRL_ACAM_A1M :
+					  RTW_RSSI_CTRL_ACAM,
+					 phy_info->rssi);
+			} else {
+				UPDATE_MA_RSSI(rssi_stat, RTW_RSSI_CTRL_OTHER,
+					 phy_info->rssi);
+			}
+		break;
+		case RTW_FRAME_TYPE_DATA :
+			if (sts_ent->addr_cam_vld) {
+				UPDATE_MA_RSSI(rssi_stat,
+					 (1 == meta->a1_match) ?
+					  RTW_RSSI_DATA_ACAM_A1M :
+					  RTW_RSSI_DATA_ACAM,
+					 phy_info->rssi);
+			} else {
+				UPDATE_MA_RSSI(rssi_stat, RTW_RSSI_DATA_OTHER,
+					 phy_info->rssi);
+			}
+		break;
+		default:
+			UPDATE_MA_RSSI(rssi_stat, RTW_RSSI_UNKNOWN,
+				       phy_info->rssi);
+		break;
+	}
+	_os_spinunlock(phl_com->drv_priv, &rssi_stat->lock, _bh, NULL);
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_rx.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_rx.h
new file mode 100644
index 000000000000..a7c4e47d1265
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_rx.h
@@ -0,0 +1,48 @@
+
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_RX_H_
+#define _HAL_RX_H_
+
+void
+hal_rx_ppdu_sts_normal_data(struct rtw_phl_com_t *phl_com,
+			    void *hdr,
+			    struct rtw_r_meta_data *meta);
+
+void
+hal_rx_ppdu_sts(struct rtw_phl_com_t *phl_com,
+		struct rtw_phl_rx_pkt *phl_rx,
+		struct hal_ppdu_sts *ppdu_sts);
+
+#ifdef CONFIG_PCI_HCI
+/**
+ * rtw_hal_query_rxch_num - query total hw rx dma channels number
+ *
+ * returns the number of  hw rx dma channel
+ */
+u8 rtw_hal_query_rxch_num(void *hal);
+#endif
+
+enum rtw_hal_status
+hal_usb_rx_agg_cfg(struct hal_info_t *hal, u8 mode, u8 agg_mode, u8 drv_define,
+		   u8 timeout, u8 size, u8 pkt_num);
+
+enum rtw_hal_status rtw_hal_set_rxfltr_opt_by_mode(void *hal, u8 band,
+					       enum rtw_rx_fltr_opt_mode mode);
+
+enum rtw_hal_status rtw_hal_set_rxfltr_type_by_mode(void *hal, u8 band,
+						    enum rtw_rxfltr_type_mode mode);
+
+#endif /*_HAL_RX_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_scanofld.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_scanofld.c
new file mode 100644
index 000000000000..dbd5e5ab6da4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_scanofld.c
@@ -0,0 +1,60 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_SCANOFLD_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_SCANOFLD
+void
+rtw_hal_scan_ofld_leave_cur_chnl(void *hal, u8 hw_band, u8 ch_band, u8 ch)
+{
+	rtw_hal_mac_scan_ofld_leave_cur_chnl((struct hal_info_t *)hal, hw_band,
+					     ch_band, ch);
+}
+
+void rtw_hal_scan_ofld_cfg_en(void *hal, u8 en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_INFO("%s %u \n", __func__, en);
+
+	hal_info->hal_com->scanofld_en = en;
+}
+
+enum rtw_hal_status
+rtw_hal_scan_ofld_add_ch(void *hal, u8 hw_band, struct scan_ofld_ch_info *cfg,
+			 bool ofld)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_scan_ofld_add_ch(hal_info, hw_band, cfg, ofld);
+}
+
+enum rtw_hal_status
+rtw_hal_scan_ofld_fw_busy(void *hal, u8 hw_band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_scan_ofld_fw_busy(hal_info, hw_band);
+}
+
+enum rtw_hal_status
+rtw_hal_scan_ofld(void *hal, u16 mac_id, u8 hw_band, u8 hw_port,
+		  struct scan_ofld_info *cfg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_scan_ofld(hal_info, mac_id, hw_band, hw_port, cfg);
+}
+#endif /* CONFIG_PHL_SCANOFLD */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_scanofld.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_scanofld.h
new file mode 100644
index 000000000000..faaf4b8a44b6
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_scanofld.h
@@ -0,0 +1,32 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_SCANOFLD_H_
+#define _HAL_SCANOFLD_H_
+
+#ifdef CONFIG_PHL_SCANOFLD
+void rtw_hal_scan_ofld_leave_cur_chnl(void *hal, u8 hw_band, u8 ch_band, u8 ch);
+void rtw_hal_scan_ofld_cfg_en(void *hal, u8 en);
+enum rtw_hal_status
+rtw_hal_scan_ofld_add_ch(void *hal, u8 hw_band, struct scan_ofld_ch_info *cfg,
+			 bool ofld);
+enum rtw_hal_status
+rtw_hal_scan_ofld(void *hal, u16 mac_id, u8 hw_band, u8 hw_port,
+		  struct scan_ofld_info *cfg);
+enum rtw_hal_status
+rtw_hal_scan_ofld_fw_busy(void *hal, u8 hw_band);
+#endif /* CONFIG_PHL_SCANOFLD */
+
+#endif /* _HAL_SCANOFLD_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio.c
new file mode 100644
index 000000000000..b528eec059e8
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio.c
@@ -0,0 +1,71 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_SDIO_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_SDIO_HCI
+static u8 hal_sdio_f0_read8(struct rtw_hal_com_t *hal, u32 addr)
+{
+	u8 val = 0;
+	u8 ret;
+
+	ret = _os_sdio_f0_read(hal->drv_priv, addr, &val, 1);
+	if (_FAIL == ret)
+		PHL_ERR("%s: Read f0 register(0x%x) FAIL!\n",
+			__FUNCTION__, addr);
+
+	return val;
+}
+#ifdef RTW_WKARD_BUS_WRITE
+static int sdio_write_post_cfg(struct rtw_hal_com_t *hal, u32 addr, u32 val)
+{
+	struct hal_info_t	*hal_info = hal->hal_priv;
+	struct hal_ops_t	*hal_ops = hal_get_ops(hal_info);
+
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	
+	if(NULL != hal_ops->write_reg_post_cfg) {
+		hal_status = hal_ops->write_reg_post_cfg(hal_info, addr, val);
+	}
+
+	return hal_status;
+}
+#endif
+void hal_sdio_set_io_ops(struct rtw_hal_com_t *hal, struct hal_io_ops *pops)
+{
+	pops->_read8 = hal_mac_sdio_read8;
+	pops->_read16 = hal_mac_sdio_read16;
+	pops->_read32 = hal_mac_sdio_read32;
+	pops->_read_mem = hal_mac_sdio_read_mem;
+
+	pops->_write8 = hal_mac_sdio_write8;
+	pops->_write16 = hal_mac_sdio_write16;
+	pops->_write32 = hal_mac_sdio_write32;
+
+	pops->_sd_f0_read8 = hal_sdio_f0_read8;
+
+#ifdef CONFIG_SDIO_INDIRECT_ACCESS
+	pops->_sd_iread8 = hal_mac_sdio_iread8;
+	pops->_sd_iread16 = hal_mac_sdio_iread16;
+	pops->_sd_iread32 = hal_mac_sdio_iread32;
+	pops->_sd_iwrite8 = hal_mac_sdio_write8;
+	pops->_sd_iwrite16 = hal_mac_sdio_write16;
+	pops->_sd_iwrite32 = hal_mac_sdio_write32;
+#endif /* CONFIG_SDIO_INDIRECT_ACCESS */
+#ifdef RTW_WKARD_BUS_WRITE
+	pops->_write_post_cfg = &sdio_write_post_cfg;
+#endif
+}
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio.h
new file mode 100644
index 000000000000..6d004192981f
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio.h
@@ -0,0 +1,70 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_SDIO_H_
+#define _HAL_SDIO_H_
+
+#ifdef CONFIG_SDIO_HCI
+/* Follow mac team suggestion, default I/O fail return value is 0xFF */
+void hal_sdio_set_io_ops(struct rtw_hal_com_t *h, struct hal_io_ops *pops);
+
+#ifdef CONFIG_RTL8851B
+#include "rtl8851b/rtl8851b.h"
+#endif
+
+#ifdef CONFIG_RTL8852A
+#include "rtl8852a/rtl8852a.h"
+#endif
+
+#if defined(CONFIG_RTL8852B) || defined(CONFIG_RTL8852BP) || defined(CONFIG_RTL8852BT)
+#include "rtl8852b/rtl8852b.h"
+#endif
+
+#ifdef CONFIG_RTL8852C
+#include "rtl8852c/rtl8852c.h"
+#endif
+
+static inline void hal_set_ops_sdio(struct rtw_phl_com_t *phl_com,
+						struct hal_info_t *hal)
+{
+	#ifdef CONFIG_RTL8852A
+	if (hal_get_chip_id(hal->hal_com) == CHIP_WIFI6_8852A) {
+		hal_set_ops_8852as(phl_com, hal);
+		hal_hook_trx_ops_8852as(hal);
+	}
+	#endif
+	#ifdef CONFIG_RTL8852A
+	if (hal_get_chip_id(hal->hal_com) == CHIP_WIFI6_8834A){
+	}
+	#endif
+
+	#if defined(CONFIG_RTL8852B) || defined(CONFIG_RTL8852BP) || defined(CONFIG_RTL8852BT)
+	if (hal_get_chip_id(hal->hal_com) == CHIP_WIFI6_8852B ||
+	    hal_get_chip_id(hal->hal_com) == CHIP_WIFI6_8852BP ||
+		hal_get_chip_id(hal->hal_com) == CHIP_WIFI6_8852BT) {
+		hal_set_ops_8852bs(phl_com, hal);
+		hal_hook_trx_ops_8852bs(hal);
+	}
+	#endif
+	#ifdef CONFIG_RTL8851B
+	if (hal_get_chip_id(hal->hal_com) == CHIP_WIFI6_8851B) {
+		hal_set_ops_8851bs(phl_com, hal);
+		hal_hook_trx_ops_8851bs(hal);
+	}
+#endif
+
+}
+#endif /*CONFIG_SDIO_HCI*/
+
+#endif /* _HAL_SDIO_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio_coex.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio_coex.c
new file mode 100644
index 000000000000..1ec893ac7d70
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sdio_coex.c
@@ -0,0 +1,33 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2013 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110, USA
+ *
+ *
+ ******************************************************************************/
+
+
+#ifdef CONFIG_SDIO_MULTI_FUNCTION_COEX
+
+int rtw_sdio_multi_state = SDIO_MULTI_WIFI;
+EXPORT_SYMBOL(rtw_sdio_multi_state);
+
+bool ex_hal_sdio_multi_if_bus_available(_adapter *adapter)
+{
+	return rtw_sdio_multi_state == SDIO_MULTI_WIFI;
+}
+
+#endif  /* CONFIG_SDIO_MULTI_FUNCTION_COEX */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ser.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ser.c
new file mode 100644
index 000000000000..d885033e205c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ser.c
@@ -0,0 +1,181 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_SER_C_
+#include "hal_headers.h"
+
+enum rtw_hal_status rtw_hal_ser_ctrl(void *hal, enum rtw_hal_ser_rsn rsn, bool en)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	hstatus = rtw_hal_mac_ser_ctrl(hal_info, rsn, en);
+
+	return hstatus;
+}
+
+u32
+rtw_hal_ser_get_error_status(void *hal, u32 *err)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum RTW_PHL_SER_NOTIFY_EVENT notify = RTW_PHL_SER_L2_RESET;
+
+	rtw_hal_mac_ser_get_error_status(hal_info, err);
+
+	if (*err == MAC_AX_ERR_L1_PREERR_DMAC) {
+		notify = RTW_PHL_SER_PREPARE_DMAC;
+	} else if ((*err == MAC_AX_ERR_L1_ERR_DMAC) || (*err == MAC_AX_ERR_L0_PROMOTE_TO_L1)) {
+		notify = RTW_PHL_SER_PAUSE_TRX;
+	} else if (*err == MAC_AX_ERR_L1_RESET_DISABLE_DMAC_DONE) {
+		notify = RTW_PHL_SER_DO_RECOVERY;
+	} else if (*err == MAC_AX_ERR_L1_RESET_RECOVERY_DONE) {
+		notify = RTW_PHL_SER_READY;
+	} else if (*err < MAC_AX_ERR_L0_PROMOTE_TO_L1) {
+		notify = RTW_PHL_SER_L0_RESET;
+	} else if (*err == MAC_AX_DUMP_SHAREBUFF_INDICATOR) {
+		notify = RTW_PHL_SER_DUMP_FW_LOG;
+	} else if (*err == MAC_AX_ERR_L2_ERR_APB_SA_TO_HCI_WCMAC) {
+		notify = RTW_PHL_SER_LOG_ONLY;
+	} else if ((*err == MAC_AX_ERR_L1_PROMOTE_TO_L2) ||
+			  ((*err >= MAC_AX_ERR_L2_ERR_AH_DMA) && (*err <= MAC_AX_GET_ERR_MAX))) {
+		notify = RTW_PHL_SER_L2_RESET;
+	}
+
+	return notify;
+}
+
+enum rtw_hal_status rtw_hal_ser_set_error_status(void *hal, u32 err)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_ser_set_error_status(hal_info, err);
+}
+
+bool rtw_hal_ser_chk_ser_l1(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_ser_chk_ser_l1(hal_info);
+}
+
+enum rtw_hal_status rtw_hal_trigger_cmac_err(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_trigger_cmac_err(hal_info);
+}
+
+enum rtw_hal_status rtw_hal_trigger_dmac_err(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_trigger_dmac_err(hal_info);
+}
+
+enum rtw_hal_status rtw_hal_lv1_rcvy(void *hal, u32 step)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status status = RTW_HAL_STATUS_FAILURE;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "===> rtw_hal_lv1_rcvy step %d\n", step);
+	status = rtw_hal_mac_lv1_rcvy(hal_info, step);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "<=== rtw_hal_lv1_rcvy step %d, status 0x%x\n", step, status);
+
+	return status;
+}
+
+void rtw_hal_dump_fw_rsvd_ple(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u32 mac_err;
+	mac_err = rtw_hal_mac_dump_fw_rsvd_ple(hal_info);
+}
+
+void
+rtw_hal_ser_reset_wdt_intr(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u32 mac_err;
+	mac_err = rtw_hal_mac_ser_reset_wdt_intr(hal_info);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "rtw_hal_ser_reset_wdt_intr status 0x%x\n",mac_err);
+}
+
+void rtw_hal_ser_int_cfg(void *hal, struct rtw_phl_com_t *phl_com,
+						 enum RTW_PHL_SER_CFG_STEP step)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+#ifdef CONFIG_SYNC_INTERRUPT
+	struct rtw_phl_evt_ops *evt_ops = &phl_com->evt_ops;
+#endif /* CONFIG_SYNC_INTERRUPT */
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+	struct hal_spec_t *hal_spec = phl_get_ic_spec(phl_com);
+
+	/* check whether to config imr during ser */
+	if (!hal_spec->ser_cfg_int)
+		return;
+
+	switch (step) {
+	case RTW_PHL_SER_M1_PRE_CFG:
+		/**
+		 * 1. disable imr
+		 * 2. set imr used during ser
+		 */
+		#ifdef CONFIG_SYNC_INTERRUPT
+		evt_ops->set_interrupt_caps(phlcom_to_drvpriv(phl_com), false);
+		#else
+		if (hal_ops->disable_interrupt)
+			hal_ops->disable_interrupt(hal);
+		#endif /* CONFIG_SYNC_INTERRUPT */
+
+		if (hal_ops->init_int_default_value)
+			hal_ops->init_int_default_value(hal, INT_SET_OPT_SER_START);
+		break;
+	case RTW_PHL_SER_M1_POST_CFG:
+		/**
+		 * 1. enable interrupt
+		 */
+		#ifdef CONFIG_SYNC_INTERRUPT
+		evt_ops->set_interrupt_caps(phlcom_to_drvpriv(phl_com), true);
+		#else
+		if (hal_ops->enable_interrupt)
+			hal_ops->enable_interrupt(hal);
+		#endif /* CONFIG_SYNC_INTERRUPT */
+		break;
+	case RTW_PHL_SER_M5_CFG:
+		/**
+		 * 1. disable interrupt
+		 * 2. set imr used after ser
+		 * 3. enable interrupt
+		 */
+		#ifdef CONFIG_SYNC_INTERRUPT
+		evt_ops->set_interrupt_caps(phlcom_to_drvpriv(phl_com), false);
+		#else
+		if (hal_ops->disable_interrupt)
+			hal_ops->disable_interrupt(hal);
+		#endif /* CONFIG_SYNC_INTERRUPT */
+
+		if (hal_ops->init_int_default_value)
+			hal_ops->init_int_default_value(hal, INT_SET_OPT_SER_DONE);
+
+		#ifdef CONFIG_SYNC_INTERRUPT
+		evt_ops->set_interrupt_caps(phlcom_to_drvpriv(phl_com), true);
+		#else
+		if (hal_ops->enable_interrupt)
+			hal_ops->enable_interrupt(hal);
+		#endif /* CONFIG_SYNC_INTERRUPT */
+		break;
+	default:
+		PHL_ERR("%s(): unknown step!\n", __func__);
+	}
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ser.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ser.h
new file mode 100644
index 000000000000..3e16703fd0cf
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_ser.h
@@ -0,0 +1,18 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_SER_H_
+#define _HAL_SER_H_
+
+#endif /* _HAL_SER_H_ */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sound.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sound.c
new file mode 100644
index 000000000000..7095400528a7
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sound.c
@@ -0,0 +1,1048 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_SOUND_C_
+
+#include "hal_headers.h"
+#ifdef CONFIG_PHL_BEAMFORM
+#ifdef CONFIG_PHL_CMD_BF
+struct csi_rpt_na {
+	u8 nr;
+	u8 nc;
+	u8 na;
+};
+
+#define CSI_NA_MATRIX_SIZE 35
+static const struct csi_rpt_na csi_na[CSI_NA_MATRIX_SIZE] =
+{
+	{2, 1, 2}, {2, 2, 2},
+	{3, 1, 4}, {3, 2, 6}, {3, 3, 6},
+	{4, 1, 6}, {4, 2, 10}, {4, 3, 12}, {4, 4, 12},
+	{5, 1, 8}, {5, 2, 14}, {5, 3, 18}, {5, 4, 20}, {5, 5, 20},
+	{6, 1, 10}, {6, 2, 18}, {6, 3, 24}, {6, 4, 28}, {6, 5, 30},
+	{6, 6, 30},
+	{7, 1, 12}, {7, 2, 22}, {7, 3, 30}, {7, 4, 36},
+	{7, 5, 40}, {7, 6, 42}, {7, 7, 42},
+	{8, 1, 14}, {8, 2, 26}, {8, 3, 36}, {8, 4, 55},
+	{8, 5, 50}, {8, 6, 54}, {8, 7, 56}, {8, 8, 56}
+};
+
+
+u32 _cal_he_csi_size(u8 mu, enum channel_width bw, u8 nr, u8 nc, u8 ng, u8 cb)
+{
+	u8 na = 0;
+	u8 ns = 0;
+	u8 i = 0;
+	u32 csi_size = 0;
+	u8 cb_s = 0;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+		  "%s : mu(%d) ; bw(%d) ; nr(%d) ; nc(%d), ng(%d), cb(%d)\n",
+		  __func__, mu, bw, nr, nc, ng ,cb);
+	do {
+		if (CHANNEL_WIDTH_80 == bw)
+			ns = (ng == 4) ? 250 : 64;
+		else if(CHANNEL_WIDTH_40 == bw)
+			ns = (ng == 4) ? 122 : 32;
+		else if(CHANNEL_WIDTH_20 == bw)
+			ns = (ng == 4) ? 64 : 20;
+		else
+			break;
+
+		for (i = 0; i < CSI_NA_MATRIX_SIZE; i++) {
+			if ((nr == csi_na[i].nr) && (nc ==csi_na[i].nc)) {
+				na = csi_na[i].na;
+				break;
+			}
+		}
+		if (na == 0)
+			break;
+
+		if (cb) {
+			if(mu)
+				cb_s = 8;
+			else
+				cb_s = 5;
+		} else {
+			if(mu)
+				cb_s = 6;
+			else
+				cb_s = 3;
+		}
+	} while(0);
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+		  "%s : na %d ; ns %d ; cb_s %d",__func__, na, ns, cb_s);
+
+	if ((0 != na) && (0 != ns) && (0 != cb_s)) {
+		csi_size = ((8 * (u32)nc) +
+			    ((u32)na * (u32)cb_s * (u32)ns)) / 8;
+		if(mu)
+			csi_size += (4 * (u32)nc * (u32)ns) / 8;
+	}
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_,
+		  "%s : expected he csi report size = %d byte",
+		  __func__, csi_size);
+	return csi_size;
+}
+
+/*1. BF Resource Related*/
+u8 _get_bw_ru_end_idx(enum channel_width bw) {
+	u8 ru_end_idx = 0;
+
+	switch (bw) {
+	case CHANNEL_WIDTH_20:
+		ru_end_idx = HAL_NPDA_RU_IDX_END_20MHZ;
+	break;
+	case CHANNEL_WIDTH_40:
+		ru_end_idx = HAL_NPDA_RU_IDX_END_40MHZ;
+	break;
+	case CHANNEL_WIDTH_80:
+		ru_end_idx = HAL_NPDA_RU_IDX_END_80MHZ;
+	break;
+	case CHANNEL_WIDTH_160:
+	case CHANNEL_WIDTH_80_80:
+		ru_end_idx = HAL_NPDA_RU_IDX_END_160MHZ;
+	break;
+	default:
+	break;
+	}
+	return ru_end_idx;
+}
+
+void _hal_snd_set_default_var(struct hal_snd_obj *snd_obj)
+{
+	snd_obj->ndpa_xpara.bw = CHANNEL_WIDTH_20;
+	snd_obj->ndpa_xpara.rate = RTW_DATA_RATE_OFDM6;
+	snd_obj->ndpa_xpara.gi_ltf = RTW_GILTF_LGI_4XHE32;
+	snd_obj->ndpa_xpara.stbc = 0;
+	snd_obj->ndpa_xpara.ldpc = 0;
+
+	snd_obj->bfrp_xpara.bw = CHANNEL_WIDTH_20;
+	snd_obj->bfrp_xpara.rate = RTW_DATA_RATE_OFDM6;
+	snd_obj->bfrp_xpara.gi_ltf = RTW_GILTF_LGI_4XHE32;
+	snd_obj->bfrp_xpara.stbc = 0;
+	snd_obj->bfrp_xpara.ldpc = 0;
+}
+
+enum rtw_hal_status
+hal_snd_obj_init(void *hal)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	struct hal_snd_obj *snd_obj = NULL;
+
+	do {
+		hal_com->snd_obj = _os_mem_alloc(hal_to_drvpriv(hal_info),
+				    sizeof(struct hal_snd_obj));
+		if (NULL == hal_com->snd_obj) {
+			break;
+		}
+		snd_obj = hal_com->snd_obj;
+		/* preset hal sounding default values */
+ 		_hal_snd_set_default_var(snd_obj);
+
+		hstatus = RTW_HAL_STATUS_SUCCESS;
+	} while (0);
+	return hstatus;
+}
+
+
+enum rtw_hal_status
+hal_snd_obj_deinit(void *hal)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+
+	do {
+		if (hal_com->snd_obj == NULL)
+			break;
+		_os_mem_free(hal_to_drvpriv(hal_info), hal_com->snd_obj,
+			     sizeof(struct hal_snd_obj));
+		hal_com->snd_obj = NULL;
+	} while (0);
+
+	return hstatus;
+}
+
+/**
+ * rtw_hal_snd_release_proc_sta_res
+ * 	free the resource for a STA used in sounding process
+ * input:
+ * @hal: (struct hal_info_t *)
+ * @sta: (struct rtw_phl_stainfo_t *)
+ **/
+enum rtw_hal_status
+rtw_hal_snd_release_proc_sta_res(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_stainfo_t *hal_sta = sta->hal_sta;
+	do {
+		if (hal_sta->bf_entry != NULL) {
+			hal_status = hal_bf_release_target_bf_entry(
+					hal_info, hal_sta->bf_entry);
+		}
+
+		if (hal_is_csi_buf_valid(hal_info, &hal_sta->bf_csi_buf)) {
+			hal_status = hal_csi_release_csi_buf(
+					hal_info, &hal_sta->bf_csi_buf);
+		}
+		if (hal_is_csi_buf_valid(hal_info, &hal_sta->bf_csi_buf_swap)) {
+			hal_status = hal_csi_release_csi_buf(
+					hal_info, &hal_sta->bf_csi_buf_swap);
+		}
+	} while(0);
+
+	return hal_status;
+}
+
+
+/**
+ * rtw_hal_snd_query_proc_sta_res
+ * input:
+ * @hal: hal_info
+ * @sta: rtw_phl_stainfo_t
+ * @mu: is this MU-MIMO STA resource request
+ * @bw: enum channel_width sounding bandwidth
+ **/
+enum rtw_hal_status
+rtw_hal_snd_query_proc_sta_res(
+	void *hal,
+	struct rtw_phl_stainfo_t *sta,
+	bool mu,
+	enum channel_width bw,
+	bool en_swap)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_stainfo_t *hal_sta = sta->hal_sta;
+
+	do {
+		if (hal_sta->bf_entry != NULL) {
+			PHL_TRACE(COMP_PHL_SOUND,  _PHL_INFO_, "[WARNING] sta->bf_entry != NULL\n");
+			/* TODO:Shall Release First ?*/
+			/* rtw_hal_snd_release_proc_sta_res(hal, sta); */
+		}
+
+		hal_sta->bf_entry =
+			(void *)hal_bf_query_idle_bf_entry(hal_info, mu);
+
+		if (hal_sta->bf_entry == NULL)
+			break;
+
+		hal_status = hal_csi_query_idle_csi_buf(
+				hal_info, mu, bw, &hal_sta->bf_csi_buf);
+
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			break;
+
+		if (mu && en_swap) {
+			hal_status = hal_csi_query_idle_csi_buf(
+				hal_info, mu, bw, &hal_sta->bf_csi_buf_swap);
+		}
+		if (hal_status == RTW_HAL_STATUS_FAILURE) {
+			PHL_INFO("Cannot Enable Swap Mode! Because of the CSI resource is not enougth.\n");
+			break;
+		}
+
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "sta->bf_csi_buf 0x%x \n",
+				hal_sta->bf_csi_buf);
+		PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "sta->bf_csi_buf_swap 0x%x \n",
+				hal_sta->bf_csi_buf_swap);
+
+		/* Set STA-INFO info to BF Entry */
+		hal_status = hal_bf_cfg_swbf_entry(sta, en_swap);
+	} while (0);
+
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_INFO("rtw_hal_snd_query_proc_sta_res FAIL \n");
+		rtw_hal_snd_release_proc_sta_res(hal, sta);
+	}
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_snd_proc_pre_cfg_sta
+ * 	hw preconfiguration for a sounding sta
+ * input:
+ * @hal: hal_info
+ * @sta: (struct rtw_phl_stainfo_t *)
+ **/
+enum rtw_hal_status
+rtw_hal_snd_proc_pre_cfg_sta(
+	void *hal,
+	struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_stainfo_t *hal_sta = sta->hal_sta;
+
+	/* 1. MAC HW BF Entry Settings */
+	hal_status = hal_bf_set_entry_hwcfg(
+				hal_info, hal_sta->bf_entry);
+
+	/* 2. Add other HAL setting here */
+	/*TODO:*/
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_snd_proc_pre_cfg
+ * 	hw preconfiguration for a sounding sequence
+ * input:
+ * @hal: hal_info
+ **/
+enum rtw_hal_status
+rtw_hal_snd_proc_pre_cfg(void *hal)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	/* 1. RF BFer Settings */
+	hal_status = rtw_hal_rf_bfer_cfg(hal_info);
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_snd_proc_post_cfg
+ * 	hw/fw post configuration for a sounding event
+ * input:
+ * @hal: hal_info
+ **/
+enum rtw_hal_status
+rtw_hal_snd_proc_post_cfg(void *hal, bool he, bool mu, bool en_fixed_mode)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_mu_score_tbl *hal_score_table = &hal_info->hal_com->bb_mu_score_tbl;
+	if (mu) {
+		/*1. MU Score Board */
+		hal_status = rtw_hal_mac_ax_set_mu_table_whole(
+				hal_info->mac, hal_score_table);
+
+		/*2. (optional) MU Fixed Mode */
+		if (en_fixed_mode)
+			hal_status = rtw_hal_bf_set_fix_mode(hal, mu, he);
+	}
+
+	return hal_status;
+}
+
+
+/**
+ * rtw_hal_snd_proc_post_cfg_gid
+ * 	hw/fw post configuration for a vht/he gid
+ * input:
+ * @hal: hal_info
+ * @gid: wifi protolcol gid (PLCP) for configuration
+ * @ba_info: pointer of struct rtw_hal_muba_info
+ **/
+enum rtw_hal_status
+rtw_hal_snd_proc_post_cfg_gid(void *hal, u8 gid, void *ba_info)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	/*
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_hal_muba_info *hal_ba_info = (struct rtw_hal_muba_info *)ba_info;
+	*/
+
+	/*TODO:*/
+	/*1. MU BAR Table : Table ID = GID, GID = STA-x + sSTA-y*/
+	return hal_status;
+}
+
+/**
+ * rtw_hal_snd_proc_post_cfg_sta
+ * 	hw/fw post configuration for a single sounding sta
+ * input:
+ * @hal: hal_info
+ * @sta: (struct rtw_phl_stainfo_t *)
+ **/
+enum rtw_hal_status
+rtw_hal_snd_proc_post_cfg_sta(
+	void *hal,
+	struct rtw_phl_stainfo_t *sta,
+	bool mu)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	/* MAC/FW Settings */
+	if (mu) {
+		/*1. FW MU STA Update */
+		hal_status = hal_bf_set_mu_sta_fw(hal, sta);
+		/*2. MU Score Board */
+		/* mac->mac_set_mu_table_single_sta */
+	}
+
+	/* BB/FW Settings */
+	/*TODO:*/
+
+	/* 2. Add other setting here */
+	/*TODO:*/
+
+	return hal_status;
+}
+
+/**
+ * rtw_hal_snd_mac_ctrl
+ * 	control sounding process : pause or start.
+ * @hal: hal_info
+ * @band: band0 / band1
+ * @ctrl: 0 = pause souning / 1 = start sounding
+ **/
+enum rtw_hal_status
+rtw_hal_snd_mac_ctrl(void *hal, u8 band, u8 ctrl)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	hstatus = rtw_hal_mac_ax_hw_snd_control(hal_info->mac, band, ctrl);
+
+	return hstatus;
+}
+
+/**
+ * rtw_hal_snd_chk_bf_res
+ * 	check the sta's sounding resource is enough for sounding
+ * input :
+ * @hal: hal_info
+ * @sta: (struct rtw_hal_stainfo_t *)
+ * @mu: true = mu / false = su
+ * @bw: enum channel_width
+ * return :
+ * @hstatus: RTW_HAL_STATUS_FAILURE = need release and query bf entry and CSI buffer
+ *  	     RTW_HAL_STATUS_SUCCESS = STA's BF Entry and CSI Buffer is same to condition.
+ **/
+enum rtw_hal_status
+rtw_hal_snd_chk_bf_res(void *hal, struct rtw_phl_stainfo_t *sta,
+	bool mu, enum channel_width bw)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_stainfo_t *hal_sta = NULL;
+	do {
+		if (sta == NULL)
+			break;
+		hal_sta = sta->hal_sta;
+
+		if (false == rtw_hal_bf_chk_bf_type(hal, sta, mu))
+			break;
+		if (mu != rtw_hal_get_csi_buf_type(&hal_sta->bf_csi_buf))
+			break;
+		if (bw != rtw_hal_get_csi_buf_bw(&hal_sta->bf_csi_buf))
+			break;
+
+		hstatus = RTW_HAL_STATUS_SUCCESS;
+	} while (0);
+
+	return hstatus;
+}
+
+/**
+ * rtw_hal_snd_polling_snd_sts
+ * 	update the sta's sounding status into sta->hal_sta->bf_entry->bfee
+ * input :
+ * @hal: hal_info
+ * @sta: (struct rtw_hal_stainfo_t *)
+ **/
+void
+rtw_hal_snd_polling_snd_sts(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	do {
+		if (sta == NULL)
+			break;
+
+		hal_bf_update_entry_snd_sts(hal_info, sta->hal_sta->bf_entry);
+
+	} while (0);
+}
+
+/* SND H2C CMD related functions */
+void
+rtw_hal_snd_ndpa_sta_info_vht(struct rtw_phl_stainfo_t *psta_info,
+			      u32 *ndpa, u8 mu)
+{
+	struct hal_vht_ndpa_sta_info *ndpa_sta =
+		(struct hal_vht_ndpa_sta_info *)ndpa;
+	u8 num_snd_dim = psta_info->rlink->protocol_cap.num_snd_dim;
+
+	if (rtw_phl_role_is_client_category(psta_info->wrole))
+		ndpa_sta->aid12 = 0; /* Target is an AP, AID = 0 */
+	else
+		ndpa_sta->aid12 = psta_info->aid;
+	ndpa_sta->feedback_type	= (mu == 0) ? HAL_NPDA_AC_SU : HAL_NPDA_AC_MU;
+	/* Nc shall alwary <= Nr */
+	if (psta_info->asoc_cap.max_nc > num_snd_dim)
+		ndpa_sta->nc = num_snd_dim;
+	else
+		ndpa_sta->nc = psta_info->asoc_cap.max_nc;
+
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "vht ndpa_sta aid12 0x%x ; fb 0x%x ; nc 0x%x\n",
+		  ndpa_sta->aid12, ndpa_sta->feedback_type, ndpa_sta->nc);
+}
+
+void
+rtw_hal_snd_ndpa_sta_info_he(struct rtw_phl_stainfo_t *psta_info,
+			     u32 *ndpa, enum channel_width bw, u8 fb_type)
+{
+
+	struct hal_he_ndpa_sta_info *ndpa_sta =
+		(struct hal_he_ndpa_sta_info *)ndpa;
+	u16 ru_start = HAL_NPDA_RU_IDX_START;
+	u16 ru_end = _get_bw_ru_end_idx(bw);
+	u8 num_snd_dim_greater_80mhz =
+		psta_info->rlink->protocol_cap.num_snd_dim_greater_80mhz;
+	u8 num_snd_dim = psta_info->rlink->protocol_cap.num_snd_dim;
+
+	if (rtw_phl_role_is_client_category(psta_info->wrole))
+		ndpa_sta->aid = 0; /* Target is and AP, AID = 0 */
+	else
+		ndpa_sta->aid = (psta_info->aid&0x7FF);
+	ndpa_sta->bw = ((ru_start&0x7F) << 0) | ((ru_end&0x7F) << 7);
+	if (0 == fb_type) {
+		ndpa_sta->fb_ng = psta_info->asoc_cap.ng_16_su_fb ?
+			 HAL_NDPA_AX_FB_SU_NG_16 : HAL_NDPA_AX_FB_SU_NG_4;
+		ndpa_sta->cb = psta_info->asoc_cap.cb_sz_su_fb ?
+			HAL_NPDA_AX_CB_SU42_MU75 : HAL_NPDA_AX_CB_SU64_MU97;
+	} else if (1 == fb_type) {
+		ndpa_sta->fb_ng = psta_info->asoc_cap.ng_16_mu_fb ?
+			HAL_NDPA_AX_FB_MU_NG_16 : HAL_NDPA_AX_FB_MU_NG_4;
+		ndpa_sta->cb = psta_info->asoc_cap.cb_sz_mu_fb ?
+			HAL_NPDA_AX_CB_SU42_MU75 : HAL_NPDA_AX_CB_SU64_MU97;
+	} else {
+		ndpa_sta->fb_ng = HAL_NDPA_AX_FB_CQI;
+		ndpa_sta->cb = 1;
+	}
+	ndpa_sta->disambiguation = 1;
+	/* Nc shall alwary <= Nr */
+	if ((CHANNEL_WIDTH_160 == bw) || (CHANNEL_WIDTH_80_80 == bw)) {
+		if (psta_info->asoc_cap.max_nc > num_snd_dim_greater_80mhz)
+			ndpa_sta->nc = num_snd_dim_greater_80mhz;
+		else
+			ndpa_sta->nc = psta_info->asoc_cap.max_nc;
+	} else {
+		if (psta_info->asoc_cap.max_nc > num_snd_dim)
+			ndpa_sta->nc = num_snd_dim;
+		else
+			ndpa_sta->nc = psta_info->asoc_cap.max_nc;
+	}
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "HE NDPA : aid 0x%x fb_ng 0x%x cb 0x%x nc 0x%x \n",
+		  ndpa_sta->aid, ndpa_sta->fb_ng, ndpa_sta->cb, ndpa_sta->nc);
+}
+
+
+/**
+ * rtw_hal_snd_set_fw_cmd_dialogtkn()
+ * 	Set cmd dialog token value in NDPA
+ * input
+ * @he: is NDPA HE or not(VHT).
+ * @token: dialog token value.
+ **/
+void rtw_hal_snd_set_fw_cmd_dialogtkn(void *hal, u8 *buf, u8 he, u8 token)
+{
+	struct hal_ax_fwcmd_snd *cmd = (struct hal_ax_fwcmd_snd *)buf;
+
+	cmd->ndpa.snd_dialog.he = he;
+	cmd->ndpa.snd_dialog.token = token;
+}
+
+/**
+ * rtw_hal_snd_vht_fwcmd_su()
+ * 	Prepared VHT SU Sounding Fw Cmd.
+ * @hal:
+ * @buf: (struct hal_ax_fwcmd_snd *) cmd buffer pointer.
+ * @psta:  (struct rtw_phl_stainfo_t *) STA to be sounding.
+ * @npda_sta: NPDA sta_info value
+ **/
+void rtw_hal_snd_vht_fwcmd_su(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u32 *npda_sta)
+{
+	struct hal_ax_fwcmd_snd *cmd = (struct hal_ax_fwcmd_snd *)buf;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_snd_obj *snd_obj =
+		(struct hal_snd_obj *)hal_info->hal_com->snd_obj;
+	u8 i = 0;
+
+	cmd->frame_ex_type = HAL_FEXG_TYPE_AC_SU;
+	cmd->macid[0] = psta->macid;
+	cmd->ndpa.common.frame_ctl = HAL_SND_VHT_NDPA_FRM_CTRL; /*TODO*/
+	for (i = 0; i < 6;i++) {
+		cmd->ndpa.common.addr1[i] = psta->mac_addr[i]; /* NDPA-RA*/
+		cmd->ndpa.common.addr2[i] = psta->wrole->mac_addr[i]; /* NDPA-TA*/
+	}
+	cmd->ndpa.common.duration = 100;
+	cmd->ndpa.ndpa_sta_info[0] = *npda_sta;
+	/* NDPA WD */
+	cmd->wd[0].txpktsize = 21;/* 2 + 2 + 6 + 6 + 1 + 4, NO FCS */
+	cmd->wd[0].ndpa_duration = 100;
+	cmd->wd[0].disdatafb = 1;
+	cmd->wd[0].datarate = snd_obj->ndpa_xpara.rate;
+	cmd->wd[0].data_bw = snd_obj->ndpa_xpara.bw;
+	cmd->wd[0].macid = psta->macid;
+	cmd->wd[0].gi_ltf = snd_obj->ndpa_xpara.gi_ltf;
+	cmd->wd[0].data_stbc = snd_obj->ndpa_xpara.stbc;
+	cmd->wd[0].data_ldpc = snd_obj->ndpa_xpara.ldpc;
+	cmd->wd[0].sifs_tx = 1;
+	cmd->wd[0].snd_pkt_sel = HAL_SND_PKT_SEL_UST_NDPA;/* unicast ndpa */
+	cmd->wd[0].ndpa = HAL_SND_PKT_NDPA_VHT;
+	/* NDP WD */
+	cmd->wd[1].disdatafb = 1;
+
+	if (1 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS2_MCS0;
+	else if (2 ==psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS3_MCS0;
+	else if (3 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS4_MCS0;
+	else
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS2_MCS0;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "psta->asoc_cap.nss_rx = 0x%x\n", psta->asoc_cap.nss_rx);
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "cmd->wd[1].datarate = 0x%x\n", cmd->wd[1].datarate);
+	cmd->wd[1].data_bw = bw;
+	cmd->wd[1].macid = psta->macid;
+	cmd->wd[1].gi_ltf = RTW_GILTF_LGI_4XHE32;
+	cmd->wd[1].sifs_tx = 0;
+	cmd->wd[1].snd_pkt_sel = HAL_SND_PKT_SEL_LAST_NDP;
+	cmd->wd[1].ndpa = HAL_SND_PKT_NDPA_VHT;
+}
+
+void rtw_hal_snd_vht_fwcmd_mu_add_sta(void *hal, u8 *buf, u32 *ndpa_sta,
+				      struct rtw_phl_stainfo_t *sta,
+		  		      u8 ndpa_idx, u8 last)
+{
+	struct hal_ax_fwcmd_snd *cmd = (struct hal_ax_fwcmd_snd *)buf;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_snd_obj *snd_obj =
+		(struct hal_snd_obj *)hal_info->hal_com->snd_obj;
+	u8 i = 0;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "==> rtw_hal_snd_vht_fwcmd_mu_add_sta\n");
+
+	if (ndpa_idx >= HAL_MAX_VHT_SND_STA_NUM)
+		return;
+
+	cmd->macid[ndpa_idx] = sta->macid;
+	/* NDPA sta_info*/
+	cmd->ndpa.ndpa_sta_info[ndpa_idx] = *ndpa_sta;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "mac id 0x%x ; ndpa sta_info 0x%x\n",
+		  cmd->macid[ndpa_idx], cmd->ndpa.ndpa_sta_info[ndpa_idx]);
+
+	/* VHT BFRP */
+	cmd->bfrp.hdr[ndpa_idx - 1].frame_ctl = HAL_SND_VHT_BFRP_FRM_CTRL;
+	cmd->bfrp.hdr[ndpa_idx - 1].duration = 100;
+	for (i = 0; i < 6; i++) {
+		cmd->bfrp.hdr[ndpa_idx - 1].addr1[i] = sta->mac_addr[i]; /* NDPA-RA = Broadcast*/
+		cmd->bfrp.hdr[ndpa_idx - 1].addr2[i] = sta->wrole->mac_addr[i]; /* NDPA-TA*/
+	}
+	cmd->bfrp.vht_para[ndpa_idx - 1].rexmit_bmp = 0;
+	/*BFRP WD*/
+	cmd->wd[1 + ndpa_idx].txpktsize = 17;  /* 2 + 2 + 6 + 6 + 1 */
+	cmd->wd[1 + ndpa_idx].ndpa_duration = 100;
+	cmd->wd[1 + ndpa_idx].datarate = snd_obj->bfrp_xpara.rate;
+	cmd->wd[1 + ndpa_idx].data_ldpc = snd_obj->bfrp_xpara.ldpc;
+	cmd->wd[1 + ndpa_idx].data_stbc = snd_obj->bfrp_xpara.stbc;
+	cmd->wd[1 + ndpa_idx].macid = sta->macid;
+	cmd->wd[1 + ndpa_idx].data_bw = snd_obj->bfrp_xpara.bw;
+	cmd->wd[1 + ndpa_idx].gi_ltf = snd_obj->bfrp_xpara.gi_ltf;
+	cmd->wd[1 + ndpa_idx].disdatafb = 1;
+	if(last) {
+		cmd->wd[1 + ndpa_idx].sifs_tx = 0;
+		cmd->wd[1 + ndpa_idx].snd_pkt_sel = HAL_SND_PKT_SEL_LAST_BFRP;/* Final BFRP */
+	} else {
+		cmd->wd[1 + ndpa_idx].sifs_tx = 1;
+		cmd->wd[1 + ndpa_idx].snd_pkt_sel = HAL_SND_PKT_SEL_MID_BFRP;
+	}
+	cmd->wd[1 + ndpa_idx].ndpa = HAL_SND_PKT_NDPA_VHT;
+
+}
+
+void rtw_hal_snd_vht_fwcmd_mu_pri(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u8 sta_nr, u32 *ndpa_sta)
+{
+	struct hal_ax_fwcmd_snd *cmd = (struct hal_ax_fwcmd_snd *)buf;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_snd_obj *snd_obj =
+		(struct hal_snd_obj *)hal_info->hal_com->snd_obj;
+	u8 i = 0;
+
+	if (sta_nr == 4)
+		cmd->frame_ex_type = HAL_FEXG_TYPE_AC_MU_3;
+	else if(sta_nr == 3)
+		cmd->frame_ex_type = HAL_FEXG_TYPE_AC_MU_2;
+	else if(sta_nr == 2)
+		cmd->frame_ex_type = HAL_FEXG_TYPE_AC_MU_1;
+	else
+		PHL_INFO("rtw_hal_snd_vht_fwcmd_mu_pri : ERROR!!!!!!!!!!!!!!!");
+	cmd->macid[0] = psta->macid;
+
+	/* NDPA */
+	cmd->ndpa.common.frame_ctl = HAL_SND_VHT_NDPA_FRM_CTRL;
+	for (i = 0; i < 6;i++) {
+		/**
+		 * Addr1: In VHT Case, Fill User_0's MAC Address to match BF Entry,
+		 * HW will auto re-fill to Broadcast Address
+		 **/
+		cmd->ndpa.common.addr1[i] = psta->mac_addr[i];
+		cmd->ndpa.common.addr2[i] = psta->wrole->mac_addr[i]; /* NDPA-TA*/
+	}
+	cmd->ndpa.common.duration = 150;/*TODO:*/
+	/* primary sta ndpa sta_info*/
+	cmd->ndpa.ndpa_sta_info[0] = *ndpa_sta;
+
+	/* VHT BFRP ==> by STA */
+
+	/* NDPA WD */
+	cmd->wd[0].txpktsize = 17 + HAL_SND_VHT_NDPA_STA_SZ * sta_nr;/* 2 + 2 + 6 + 6 + 1 + 2, NO FCS */
+	cmd->wd[0].ndpa_duration = 150;
+	cmd->wd[0].disdatafb = 1;
+	cmd->wd[0].datarate = snd_obj->ndpa_xpara.rate;
+	cmd->wd[0].data_bw = snd_obj->ndpa_xpara.bw;
+	cmd->wd[0].data_ldpc = snd_obj->ndpa_xpara.ldpc;
+	cmd->wd[0].data_stbc = snd_obj->ndpa_xpara.stbc;
+	cmd->wd[0].macid = psta->macid;
+	cmd->wd[0].gi_ltf = snd_obj->ndpa_xpara.gi_ltf;
+	cmd->wd[0].sifs_tx = 1;
+	cmd->wd[0].snd_pkt_sel = HAL_SND_PKT_SEL_BST_NDPA;
+	cmd->wd[0].ndpa = HAL_SND_PKT_NDPA_VHT;
+	/* NDP WD */
+	cmd->wd[1].disdatafb = 1;
+
+	if (1 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS2_MCS0;
+	else if (2 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS3_MCS0;
+	else if (3 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS4_MCS0;
+	else
+		cmd->wd[1].datarate = RTW_DATA_RATE_VHT_NSS2_MCS0;
+	cmd->wd[1].data_bw = bw;
+	cmd->wd[1].macid = psta->macid;
+	cmd->wd[1].gi_ltf = RTW_GILTF_LGI_4XHE32;
+	cmd->wd[1].sifs_tx = 1;
+	cmd->wd[1].snd_pkt_sel = HAL_SND_PKT_SEL_MID_NDP;/*Last NDP*/
+	cmd->wd[1].ndpa = HAL_SND_PKT_NDPA_VHT;
+}
+
+/* HE FW Command */
+/**
+ * rtw_hal_snd_ax_fwcmd_su()
+ * 	fill the fw cmd for HE sounding type HE Non-TB : NDPA - NDP - CSI Report
+ *
+ **/
+void rtw_hal_snd_ax_fwcmd_nontb(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u32 *npda_sta)
+{
+	struct hal_ax_fwcmd_snd *cmd = (struct hal_ax_fwcmd_snd *)buf;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_snd_obj *snd_obj =
+		(struct hal_snd_obj *)hal_info->hal_com->snd_obj;
+	u8 i = 0;
+
+	cmd->frame_ex_type = HAL_FEXG_TYPE_AX_SU;
+	cmd->macid[0] = psta->macid;
+	cmd->ndpa.common.frame_ctl = HAL_SND_HE_NDPA_FRM_CTRL;
+	for (i = 0; i < 6;i++) {
+		cmd->ndpa.common.addr1[i] = psta->mac_addr[i]; /* NDPA-RA*/
+		cmd->ndpa.common.addr2[i] = psta->wrole->mac_addr[i]; /* NDPA-TA*/
+	}
+	cmd->ndpa.common.duration = 100;
+	cmd->ndpa.ndpa_sta_info[0] = *npda_sta;
+	/* NDPA WD */
+	cmd->wd[0].txpktsize = 21;/* 2 + 2 + 6 + 6 + 1 + 4, NO FCS */
+	cmd->wd[0].ndpa_duration = 100;
+	cmd->wd[0].disdatafb = 1;
+	cmd->wd[0].datarate = snd_obj->ndpa_xpara.rate;
+	cmd->wd[0].data_bw = snd_obj->ndpa_xpara.bw;
+	cmd->wd[0].data_stbc = snd_obj->ndpa_xpara.stbc;
+	cmd->wd[0].data_ldpc = snd_obj->ndpa_xpara.ldpc;
+	cmd->wd[0].macid = psta->macid;
+	cmd->wd[0].gi_ltf = snd_obj->ndpa_xpara.gi_ltf;
+	cmd->wd[0].sifs_tx = 1;
+	cmd->wd[0].snd_pkt_sel = HAL_SND_PKT_SEL_UST_NDPA;/*unicast ndpa*/
+	cmd->wd[0].ndpa = HAL_SND_PKT_NDPA_HE;
+	/* NDP WD */
+	cmd->wd[1].disdatafb = 1;
+
+	if (1 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS2_MCS0;
+	else if (2 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS3_MCS0;
+	else if (3 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS4_MCS0;
+	else
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS2_MCS0;
+	cmd->wd[1].data_bw = bw;
+	cmd->wd[1].macid = psta->macid;
+	cmd->wd[1].gi_ltf = RTW_GILTF_2XHE16; /* TODO: if psta->asoc_cap.ltf_gi support NDP 4XHE32;*/
+	cmd->wd[1].sifs_tx = 0;
+	cmd->wd[1].snd_pkt_sel = HAL_SND_PKT_SEL_LAST_NDP;
+	cmd->wd[1].ndpa = HAL_SND_PKT_NDPA_HE;
+}
+
+/**
+ * rtw_hal_snd_ax_fwcmd_tb_add_sta()
+ * 	fill fw cmd for HE sounding with TB case.
+ * 	shall call rtw_hal_snd_ax_fwcmd_tb_pri() first, and add STA by this api.
+ * 	Note that RU-Allocation shall dicide from phl caller,
+ * 	and shall avoid same with others
+ * function input:
+ * @hal:
+ * @buf: (struct hal_ax_fwcmd_snd *) command buf pointer.
+ * @ndpa_sta:
+ * @sta: (struct rtw_phl_stainfo_t *)
+ * @ru_idx: RU Allocation index;
+ * @ndpa_idx: 0-7 for 8852a
+ * @bfrp_idx: 0-1 for 8852a
+ * @bfrp_u_idx: 0-3 for 8852a
+ **/
+void rtw_hal_snd_ax_fwcmd_tb_add_sta(void *hal, u8 *buf, u32 *ndpa_sta,
+				     struct rtw_phl_stainfo_t *sta, u8 ru_idx,
+		  		     u8 ndpa_idx, u8 bfrp_idx, u8 bfrp_u_idx)
+{
+	struct hal_ax_fwcmd_snd *cmd = (struct hal_ax_fwcmd_snd *)buf;
+	struct hal_he_ndpa_sta_info *ndpa =
+		(struct hal_he_ndpa_sta_info *)ndpa_sta;
+	u8 mu = 0, ng = 4;
+	u32 rpt_size = 0;
+
+	if (ndpa_idx >= HAL_MAX_HE_SND_STA_NUM)
+		return;
+	if (bfrp_idx >= HAL_MAX_HE_BFRP_NUM)
+		return;
+	cmd->macid[ndpa_idx] = sta->macid;
+	/* NDPA sta_info*/
+	cmd->ndpa.ndpa_sta_info[ndpa_idx] = *ndpa_sta;
+	/* BFRP user_info */
+	cmd->bfrp.he_para[bfrp_idx].fbseg_rexmit_bmp[bfrp_u_idx] = 0;
+	cmd->bfrp.he_para[bfrp_idx].user[bfrp_u_idx].aid12 = sta->aid;
+	if (cmd->bfrp.he_para[bfrp_idx].common.ul_bw > CHANNEL_WIDTH_80) {
+		ru_idx = (ru_idx << 1) | BIT(0);
+	} else {
+		ru_idx = (ru_idx << 1) &(~BIT(0));
+	}
+	cmd->bfrp.he_para[bfrp_idx].user[bfrp_u_idx].ru_pos = ru_idx;
+
+	cmd->bfrp.he_para[bfrp_idx].user[bfrp_u_idx].ul_fec_code = 0;
+	cmd->bfrp.he_para[bfrp_idx].user[bfrp_u_idx].ul_mcs = 3;/* HE-MCS3, TODO: Default value of golden */
+	cmd->bfrp.he_para[bfrp_idx].user[bfrp_u_idx].ul_dcm = 0 ;
+	cmd->bfrp.he_para[bfrp_idx].user[bfrp_u_idx].ss_alloc = 0;/* 1SS, TODO: Default value of golden */
+	cmd->bfrp.he_para[bfrp_idx].user[bfrp_u_idx].ul_tgt_rssi =
+			(u8)(sta->hal_sta->rssi_stat.rssi >> 1);/*TODO: From Rx ST SU RSSI */
+	if (ndpa->fb_ng == HAL_NDPA_AX_FB_MU_NG_4) {
+		mu = 1;
+		ng = 4;
+	} else if (ndpa->fb_ng == HAL_NDPA_AX_FB_MU_NG_16) {
+		mu = 1;
+		ng = 16;
+	} else if (ndpa->fb_ng == HAL_NDPA_AX_FB_SU_NG_4) {
+		mu = 0;
+		ng = 4;
+	} else if (ndpa->fb_ng == HAL_NDPA_AX_FB_SU_NG_16) {
+		mu = 0;
+		ng = 4;
+	}
+	rpt_size = _cal_he_csi_size(mu, sta->chandef.bw,
+				    (sta->rlink->protocol_cap.num_snd_dim + 1),
+				    ((u8)ndpa->nc + 1), ng, (u8)ndpa->cb);
+	if (cmd->bfrp.he_para[bfrp_idx].f2p_info.csi_len_bfrp < (rpt_size/64))
+		cmd->bfrp.he_para[bfrp_idx].f2p_info.csi_len_bfrp =
+			(u16)(rpt_size / 64); /*unit 64 byte*/
+}
+
+void rtw_hal_snd_ax_fwcmd_tb_pri(void *hal, u8 *buf, enum channel_width bw,
+		    struct rtw_phl_stainfo_t *psta, u8 sta_nr1, u8 sta_nr2)
+{
+	struct hal_ax_fwcmd_snd *cmd = (struct hal_ax_fwcmd_snd *)buf;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_snd_obj *snd_obj =
+		(struct hal_snd_obj *)hal_info->hal_com->snd_obj;
+	u8 i = 0;
+	u8 sta_nr = sta_nr1 + sta_nr2;
+
+	if (sta_nr2 != 0)
+		cmd->frame_ex_type = HAL_FEXG_TYPE_AX_MU_2;
+	else
+		cmd->frame_ex_type = HAL_FEXG_TYPE_AX_MU_1;
+	cmd->bfrp0_sta_nr = sta_nr1;
+	cmd->bfrp1_sta_nr = sta_nr2;
+	cmd->macid[0] = psta->macid;
+
+	/* NDPA */
+	cmd->ndpa.common.frame_ctl = HAL_SND_HE_NDPA_FRM_CTRL; /*TODO:*/
+	for (i = 0; i < 6;i++) {
+		cmd->ndpa.common.addr1[i] = 0xFF; /* NDPA-RA = Broadcast*/
+		cmd->ndpa.common.addr2[i] = psta->wrole->mac_addr[i]; /* NDPA-TA*/
+		/* BFRP - 1*/
+		cmd->bfrp.hdr[0].addr1[i] = 0xFF;
+		cmd->bfrp.hdr[0].addr2[i] = psta->wrole->mac_addr[i];
+		if (sta_nr2 != 0) {
+			/* BFRP - 2*/
+			cmd->bfrp.hdr[1].addr1[i] = 0xFF;
+			cmd->bfrp.hdr[1].addr2[i] =  psta->wrole->mac_addr[i];
+		}
+	}
+	cmd->ndpa.common.duration = 150;/*TODO:*/
+	/*TODO: ndpa user info in other api */
+
+	/* BFRP #1 */
+	cmd->bfrp.hdr[0].frame_ctl = HAL_SND_HE_BFRP_FRM_CTRL; /* Trigger Frame */
+	cmd->bfrp.hdr[0].duration = 100;
+	cmd->bfrp.he_para[0].common.tgr_info = HAL_SND_TRIG_INFO_BFRP;/*BFRP*/
+	cmd->bfrp.he_para[0].common.ul_len = 0xFFF;/*TODO: LSIG Length : sw provide or fw calculate */
+	cmd->bfrp.he_para[0].common.more_tf = 0;
+	cmd->bfrp.he_para[0].common.cs_rqd = 1;
+	cmd->bfrp.he_para[0].common.ul_bw = bw;
+	cmd->bfrp.he_para[0].common.gi_ltf = RTW_TB_GILTF_4XHE32;/* 8852A Limitation of UL OFDMA */
+	cmd->bfrp.he_para[0].common.num_heltf = 0;/* TODO: Default value of golden */
+	cmd->bfrp.he_para[0].common.ul_pktext = 0;/* TODO: fw? hw? */
+	cmd->bfrp.he_para[0].common.ap_tx_pwr = 0x3C;/* TODO: Default value of golden */
+
+	/* F2P cmd parameters */
+	cmd->bfrp.he_para[0].f2p_info.tb_t_pe_bfrp = 2;
+	cmd->bfrp.he_para[0].f2p_info.tri_pad_bfrp = 2;
+	cmd->bfrp.he_para[0].f2p_info.ul_cqi_rpt_tri_bfrp = 0;
+	cmd->bfrp.he_para[0].f2p_info.rf_gain_idx_bfrp = 0;/* ?? */
+	cmd->bfrp.he_para[0].f2p_info.fix_gain_en_bfrp = 0;/* ?? */
+
+	if (sta_nr2) {
+		cmd->bfrp.hdr[1].frame_ctl = HAL_SND_HE_BFRP_FRM_CTRL; /* Trigger Frame */
+		cmd->bfrp.hdr[1].duration = 100;
+		cmd->bfrp.he_para[1].common.tgr_info = HAL_SND_TRIG_INFO_BFRP;/*BFRP*/
+		cmd->bfrp.he_para[1].common.ul_len = 0xFFF;/*TODO: LSIG Length : sw provide or fw calculate */
+		cmd->bfrp.he_para[1].common.more_tf = 0;
+		cmd->bfrp.he_para[1].common.cs_rqd = 1;
+		cmd->bfrp.he_para[1].common.ul_bw = bw;
+		cmd->bfrp.he_para[1].common.gi_ltf = RTW_TB_GILTF_4XHE32;/* 8852A Limitation of UL OFDMA */
+		cmd->bfrp.he_para[1].common.num_heltf = 0;/* TODO: Default value of golden */
+		cmd->bfrp.he_para[1].common.ul_pktext = 0;/* TODO: fw? hw? */
+		cmd->bfrp.he_para[1].common.ap_tx_pwr = 0x32;/* TODO: Default value of golden */
+
+		cmd->bfrp.he_para[1].f2p_info.tb_t_pe_bfrp = 2;
+		cmd->bfrp.he_para[1].f2p_info.tri_pad_bfrp = 2;
+		cmd->bfrp.he_para[1].f2p_info.ul_cqi_rpt_tri_bfrp = 0;
+		cmd->bfrp.he_para[1].f2p_info.rf_gain_idx_bfrp = 0;
+		cmd->bfrp.he_para[1].f2p_info.fix_gain_en_bfrp = 0;
+	}
+	/* NDPA WD */
+	cmd->wd[0].txpktsize = 17 + HAL_SND_HE_NDPA_STA_SZ * sta_nr;/* 2 + 2 + 6 + 6 + 1 + 4, NO FCS */
+	cmd->wd[0].ndpa_duration = 100;
+	cmd->wd[0].disdatafb = 1;
+	cmd->wd[0].datarate = snd_obj->ndpa_xpara.rate;
+	cmd->wd[0].data_bw = snd_obj->ndpa_xpara.bw;
+	cmd->wd[0].data_stbc = snd_obj->ndpa_xpara.stbc;
+	cmd->wd[0].data_ldpc = snd_obj->ndpa_xpara.ldpc;
+	cmd->wd[0].macid = psta->macid;
+	cmd->wd[0].gi_ltf = snd_obj->ndpa_xpara.gi_ltf;
+	cmd->wd[0].sifs_tx = 1;
+	cmd->wd[0].snd_pkt_sel = HAL_SND_PKT_SEL_BST_NDPA;
+	cmd->wd[0].ndpa = HAL_SND_PKT_NDPA_HE;
+	/* NDP WD */
+	cmd->wd[1].disdatafb = 1;
+
+	if (1 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS2_MCS0;
+	else if (2 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS3_MCS0;
+	else if(3 == psta->rlink->protocol_cap.num_snd_dim)
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS4_MCS0;
+	else
+		cmd->wd[1].datarate = RTW_DATA_RATE_HE_NSS2_MCS0;
+	cmd->wd[1].data_bw = bw;
+	cmd->wd[1].macid = psta->macid;
+	cmd->wd[1].gi_ltf = RTW_GILTF_2XHE16; /* TODO: if support 4x32 NDP */
+	cmd->wd[1].sifs_tx = 0;
+	cmd->wd[1].snd_pkt_sel = HAL_SND_PKT_SEL_MID_NDP;
+	cmd->wd[1].ndpa = HAL_SND_PKT_NDPA_HE;
+	/* BFRP #1 WD */
+	cmd->wd[2].txpktsize = 24 + HAL_SND_HE_BFRP_STA_SZ * sta_nr1;  /* 2 + 2 + 6 + 6 + 8 + 5*N */
+	cmd->wd[2].ndpa_duration = 100;
+	cmd->wd[2].datarate = snd_obj->bfrp_xpara.rate;
+	//cmd->wd[2].macid
+	cmd->wd[2].data_bw = snd_obj->bfrp_xpara.bw;
+	cmd->wd[2].data_stbc = snd_obj->bfrp_xpara.stbc;
+	cmd->wd[2].data_ldpc = snd_obj->bfrp_xpara.ldpc;
+	cmd->wd[2].gi_ltf = snd_obj->bfrp_xpara.gi_ltf;
+	cmd->wd[2].disdatafb = 1;
+	if(sta_nr2) {
+		cmd->wd[2].sifs_tx = 1;
+		cmd->wd[2].snd_pkt_sel = HAL_SND_PKT_SEL_MID_BFRP;
+	} else {
+		cmd->wd[2].sifs_tx = 0;/* Final BFRP */
+		cmd->wd[2].snd_pkt_sel = HAL_SND_PKT_SEL_LAST_BFRP;
+	}
+	cmd->wd[2].ndpa = HAL_SND_PKT_NDPA_HE;
+	if (sta_nr2) {
+		cmd->wd[3].txpktsize = 24 + HAL_SND_HE_BFRP_STA_SZ * sta_nr2;  /* 2 + 2 + 6 + 6 + 8 + 5*N */
+		cmd->wd[3].ndpa_duration = 100;
+		cmd->wd[3].datarate = snd_obj->bfrp_xpara.rate;
+		cmd->wd[3].data_bw = snd_obj->bfrp_xpara.bw;
+		cmd->wd[3].data_stbc = snd_obj->bfrp_xpara.stbc;
+		cmd->wd[3].data_ldpc = snd_obj->bfrp_xpara.ldpc;
+		cmd->wd[3].gi_ltf = snd_obj->bfrp_xpara.gi_ltf;
+		cmd->wd[3].disdatafb = 1;
+		cmd->wd[3].sifs_tx = 0;/* Final BFRP */
+		cmd->wd[3].snd_pkt_sel = HAL_SND_PKT_SEL_LAST_BFRP;
+		cmd->wd[3].ndpa = HAL_SND_PKT_NDPA_HE;
+	}
+}
+
+u8 *
+rtw_hal_snd_prepare_snd_cmd(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 *buf = NULL;
+
+	buf = _os_mem_alloc(hal_to_drvpriv(hal_info),
+			    sizeof(struct hal_ax_fwcmd_snd));
+
+	return buf;
+}
+
+
+enum rtw_hal_status
+rtw_hal_snd_release_snd_cmd(void *hal, u8 *buf)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	do {
+		if (buf == NULL)
+			break;
+		_os_mem_free(hal_to_drvpriv(hal_info), buf,
+			     sizeof(struct hal_ax_fwcmd_snd));
+	} while (0);
+
+	return hstatus;
+}
+
+
+enum rtw_hal_status
+rtw_hal_snd_send_fw_cmd(void *hal, u8 *cmd)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "==> rtw_hal_snd_send_fw_cmd \n");
+	hstatus = hal_mac_ax_send_fw_snd(hal_info,
+			(struct hal_ax_fwcmd_snd *)cmd);
+	/*TODO: Dump CMD content */
+	PHL_TRACE(COMP_PHL_SOUND, _PHL_INFO_, "<== rtw_hal_snd_send_fw_cmd \n");
+	return hstatus;
+}
+#endif
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sound.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sound.h
new file mode 100644
index 000000000000..7cc122a08667
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sound.h
@@ -0,0 +1,79 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_SOUND_H_
+#define _HAL_SOUND_H_
+
+#ifdef CONFIG_PHL_BEAMFORM
+#ifdef CONFIG_PHL_CMD_BF
+
+#define HAL_NPDA_RU_IDX_START 0
+#define HAL_NPDA_RU_IDX_END_20MHZ 8
+#define HAL_NPDA_RU_IDX_END_40MHZ 17
+#define HAL_NPDA_RU_IDX_END_80MHZ 36
+#define HAL_NPDA_RU_IDX_END_160MHZ 73
+
+#define HAL_SND_HE_BFRP_STA_SZ 5
+#define HAL_SND_HE_NDPA_STA_SZ 4
+#define HAL_SND_VHT_NDPA_STA_SZ 2
+
+#define HAL_SND_VHT_NDPA_FRM_CTRL 0x54
+#define HAL_SND_HE_NDPA_FRM_CTRL 0x54
+#define HAL_SND_VHT_BFRP_FRM_CTRL 0x44
+#define HAL_SND_HE_BFRP_FRM_CTRL 0x24 /* Trigger Frame */
+
+#define HAL_SND_TRIG_INFO_BFRP 0x1
+
+enum hal_snd_pkt_sel {
+	HAL_SND_PKT_SEL_UST_NDPA = 0,
+	HAL_SND_PKT_SEL_BST_NDPA =1,
+	HAL_SND_PKT_SEL_LAST_NDP = 2,
+	HAL_SND_PKT_SEL_MID_NDP = 3,
+	HAL_SND_PKT_SEL_MID_BFRP = 4,
+	HAL_SND_PKT_SEL_LAST_BFRP = 5,
+	HAL_SND_PKT_SEL_MAX
+};
+
+enum hal_snd_pkt_ndpa_type {
+	HAL_SND_PKT_NDPA_HT = 1,
+	HAL_SND_PKT_NDPA_VHT = 2,
+	HAL_SND_PKT_NDPA_HE = 3,
+	HAL_SND_PKT_NDPA_MAX
+};
+
+
+struct hal_snd_pkt_xmit_para {
+	enum channel_width bw;
+	enum rtw_data_rate rate;
+	enum rtw_gi_ltf gi_ltf;
+	u8 stbc;
+	u8 ldpc;
+};
+
+struct hal_snd_obj {
+	struct hal_snd_pkt_xmit_para ndpa_xpara;
+	struct hal_snd_pkt_xmit_para bfrp_xpara;
+};
+
+enum rtw_hal_status hal_snd_obj_init(void *hal);
+enum rtw_hal_status hal_snd_obj_deinit(void *hal);
+#else
+#define hal_snd_obj_init(_h) RTW_HAL_STATUS_SUCCESS
+#define hal_snd_obj_deinit(_h) RTW_HAL_STATUS_SUCCESS
+#endif
+#else
+#define hal_snd_obj_init(_h) RTW_HAL_STATUS_SUCCESS
+#define hal_snd_obj_deinit(_h) RTW_HAL_STATUS_SUCCESS
+#endif
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sta.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sta.c
new file mode 100644
index 000000000000..ac35c5d17a8e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sta.c
@@ -0,0 +1,1265 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_STA_C_
+#include "hal_headers.h"
+
+void
+_hal_sta_rssi_init(struct rtw_phl_stainfo_t *sta)
+{
+	sta->hal_sta->rssi_stat.assoc_rssi = 0;
+	sta->hal_sta->rssi_stat.ma_rssi = 0;
+}
+#ifdef CONFIG_PHL_BEAMFORM
+static enum rtw_hal_status
+_hal_bfee_init(struct hal_info_t *hal_info,
+	       struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	bool enable_bfee = false;
+
+	do {
+		if(NULL == sta)
+			break;
+
+		rlink = sta->rlink;
+		if (NULL == rlink)
+			break;
+
+		/*only init BFee when wrole cap's bfee and sta cap 's bfer matched */
+		if ((rlink->protocol_cap.he_su_bfme ||
+		     rlink->protocol_cap.he_mu_bfme) &&
+		    (sta->asoc_cap.he_su_bfmr || sta->asoc_cap.he_mu_bfmr)) {
+			enable_bfee = true;
+		}
+
+		if ((rlink->protocol_cap.vht_su_bfme ||
+		     rlink->protocol_cap.vht_mu_bfme) &&
+		    (sta->asoc_cap.vht_su_bfmr || sta->asoc_cap.vht_mu_bfmr)) {
+			enable_bfee = true;
+		}
+
+		if (rlink->protocol_cap.ht_su_bfme &&
+		    sta->asoc_cap.ht_su_bfmr)
+			enable_bfee = true;
+
+		if (true == enable_bfee) {
+			/* BFee Functions */
+			if (RTW_HAL_STATUS_SUCCESS !=
+				hal_bf_hw_mac_init_bfee(hal_info, rlink->hw_band))
+			{
+				PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+					  "%s : Init HW MAC BFee Fail\n",
+					  __func__);
+				break;
+			}
+			/* BFee CSI parameters*/
+			hal_info->hal_com->csi_para_ctrl_sel = false;
+			if (RTW_HAL_STATUS_SUCCESS !=
+				hal_bf_set_bfee_csi_para(hal_info,
+					hal_info->hal_com->csi_para_ctrl_sel,
+					sta)) {
+				PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+					  "%s : Set BFee CSI Para Fail\n",
+					  __func__);
+				break;
+			}
+#ifdef RTW_WKARD_DYNAMIC_BFEE_CAP
+			/* BB Workaround */
+			rtw_hal_bb_dcr_en(hal_info, true);
+#endif
+
+			PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_,
+				  "%s : Enable HW BFee Function Success\n",
+				  __func__);
+		}
+		rtw_hal_bb_nvar_src_sel(hal_info, enable_bfee);
+		hstatus = RTW_HAL_STATUS_SUCCESS;
+	} while (0);
+
+	return hstatus;
+}
+#else
+static enum rtw_hal_status
+_hal_bfee_init(struct hal_info_t *hal_info,
+	       struct rtw_phl_stainfo_t *sta)
+{
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif
+static enum rtw_hal_status
+_hal_set_default_cctrl_tbl(struct hal_info_t *hal_info,
+				 struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_mac_ax_cctl_info cctrl, cctl_mask;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &cctrl, 0,
+			sizeof(struct rtw_hal_mac_ax_cctl_info));
+	_os_mem_set(hal_to_drvpriv(hal_info), &cctl_mask, 0,
+			sizeof(struct rtw_hal_mac_ax_cctl_info));
+	if (NULL == sta)
+		goto out;
+	sts = rtw_hal_bb_cfg_config_cmac_tbl(hal_info, sta, &cctrl, &cctl_mask);
+	if (sts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s: fill cmac table of bb failed)\n",
+			__func__);
+		goto out;
+	}
+#ifdef CONFIG_PHL_DEFAULT_MGNT_Q_RPT_EN
+	cctrl.mgq_rpt_en = 1;
+	cctl_mask.mgq_rpt_en = 1;
+#endif
+	sts = rtw_hal_cmc_tbl_cfg(hal_info, &cctrl ,&cctl_mask, sta->macid);
+out:
+	return sts;
+}
+
+static enum rtw_hal_status
+_hal_update_cctrl_tbl(struct hal_info_t *hal_info,
+			    struct rtw_phl_stainfo_t *sta)
+{
+	struct rtw_wifi_role_t *wrole = NULL;
+	struct rtw_wifi_role_link_t *rlink = NULL;
+	struct role_link_cap_t *cap = NULL;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_mac_ax_cctl_info cctrl, cctl_info_mask;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	phl_fun_monitor_start(&start_t, true, __FUNCTION__);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	_os_mem_set(hal_to_drvpriv(hal_info), &cctrl, 0, sizeof(struct rtw_hal_mac_ax_cctl_info));
+	_os_mem_set(hal_to_drvpriv(hal_info), &cctl_info_mask, 0, sizeof(struct rtw_hal_mac_ax_cctl_info));
+
+	if (NULL == sta)
+		goto out;
+	wrole = sta->wrole;
+	rlink = sta->rlink;
+	cap = &rlink->cap;
+
+	sts = rtw_hal_bb_cfg_config_cmac_tbl(hal_info, sta, &cctrl,
+						&cctl_info_mask);
+	if (sts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_DBG, _PHL_ERR_, "%s: fill cmac table of bb failed)\n",
+			__func__);
+		goto out;
+	}
+	/*TODO - update cctrl tab from stainfo*/
+	cctrl.disrtsfb = 1;
+	cctl_info_mask.disrtsfb = 1;
+
+	cctrl.disdatafb = 1;
+	cctl_info_mask.disdatafb = 1;
+
+	/*
+	if (!cctrl.disdatafb)
+		cctrl.arfr_ctrl = rtw_hal_bb_get_arfr_idx(hal_info, sta);
+	*/
+
+	if (cap->rty_lmt_rts == 0xFF) {
+		cctrl.rts_txcnt_lmt_sel = 0;
+	} else {
+		cctrl.rts_txcnt_lmt_sel = 1;
+		cctrl.rts_txcnt_lmt = cap->rty_lmt_rts & 0xF;
+		cctl_info_mask.rts_txcnt_lmt = 0xF;
+	}
+	cctl_info_mask.rts_txcnt_lmt_sel = 1;
+	cctrl.rts_rty_lowest_rate = (sta->chandef.band == BAND_ON_24G) ? (RTW_DATA_RATE_CCK1) : (RTW_DATA_RATE_OFDM6);
+	cctl_info_mask.rts_rty_lowest_rate = 0xF;
+
+	if (cap->rty_lmt == 0xFF) {
+		cctrl.data_txcnt_lmt_sel = 0;
+	} else {
+		cctrl.data_txcnt_lmt_sel = 1;
+		cctrl.data_tx_cnt_lmt = cap->rty_lmt & 0x3F;
+		cctl_info_mask.data_tx_cnt_lmt = 0x3F;
+	}
+	cctl_info_mask.data_txcnt_lmt_sel = 1;
+
+	cctrl.max_agg_num_sel = 1;
+	cctl_info_mask.max_agg_num_sel = 1;
+
+	cctrl.max_agg_num = sta->asoc_cap.num_ampdu - 1;
+	cctl_info_mask.max_agg_num = 0xFF;
+
+	cctrl.ampdu_density = sta->asoc_cap.ampdu_density;
+	cctl_info_mask.ampdu_density = 0xF;
+
+	if (cctrl.max_agg_num <= 0x3F)
+		cctrl.ba_bmap = 0; /* 64 */
+	else if (cctrl.max_agg_num > 0x3F && cctrl.max_agg_num <= 0x7F)
+		cctrl.ba_bmap = 1; /* 128*/
+	else if ( cctrl.max_agg_num > 0x7F && cctrl.max_agg_num <= 0xFF)
+		cctrl.ba_bmap = 2; /* 256 */
+	cctl_info_mask.ba_bmap = 0x3;
+
+	if (rtw_phl_role_is_client_category(sta->wrole)) {
+		cctrl.uldl = 1;
+		cctl_info_mask.uldl = 1;
+	} else {
+		cctrl.uldl = 0;
+		cctl_info_mask.uldl = 1;
+	}
+
+	cctrl.multi_port_id = rlink->hw_port;
+	cctl_info_mask.multi_port_id = 0x7;
+
+	if (rtw_phl_role_is_ap_category(wrole)) {
+		cctrl.data_dcm = 0; /*(sta->asoc_cap.dcm_max_const_rx > 0)*/
+		cctl_info_mask.data_dcm = 1;
+	}
+
+	cctrl.rts_en = sta->rts_en;
+	cctrl.hw_rts_en = sta->hw_rts_en;
+	cctrl.cts2self_en = sta->cts2self;
+	cctrl.cca_rts = sta->rts_cca_mode;
+
+	cctl_info_mask.hw_rts_en = 0x1;
+	cctl_info_mask.rts_en = 0x1;
+	cctl_info_mask.cts2self_en = 0x1;
+	cctl_info_mask.cca_rts = 0x3;
+
+	if (sta->asoc_cap.pkt_padding == 3) {
+		/* follow PPE threshold */
+		u8 ppe16 = 0, ppe8 = 0;
+		u8 nss = sta->asoc_cap.nss_rx;
+
+		/* bw = 20MHz */
+		ppe16 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_20] & 0x7);
+		ppe8 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_20]>>3) & 0x7;
+
+		if ((ppe16 != 7) && (ppe8 == 7)) {
+			cctrl.nominal_pkt_padding = 2;
+			cctl_info_mask.nominal_pkt_padding = 0x3;
+		} else if (ppe8 != 7) {
+			cctrl.nominal_pkt_padding = 1;
+			cctl_info_mask.nominal_pkt_padding = 0x3;
+		} else {
+			cctrl.nominal_pkt_padding = 0;
+			cctl_info_mask.nominal_pkt_padding = 0x3;
+		}
+		/* bw = 40MHz */
+		ppe16 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_40] & 0x7);
+		ppe8 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_40]>>3) & 0x7;
+
+		if ((ppe16 != 7) && (ppe8 == 7)) {
+			cctrl.nominal_pkt_padding40 = 2;
+			cctl_info_mask.nominal_pkt_padding40 = 0x3;
+		} else if (ppe8 != 7) {
+			cctrl.nominal_pkt_padding40 = 1;
+			cctl_info_mask.nominal_pkt_padding40 = 0x3;
+		} else {
+			cctrl.nominal_pkt_padding40 = 0;
+			cctl_info_mask.nominal_pkt_padding40 = 0x3;
+		}
+		/* bw = 80MHz */
+		ppe16 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_80] & 0x7);
+		ppe8 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_80]>>3) & 0x7;
+
+		if ((ppe16 != 7) && (ppe8 == 7)) {
+			cctrl.nominal_pkt_padding80 = 2;
+			cctl_info_mask.nominal_pkt_padding80 = 0x3;
+		} else if (ppe8 != 7) {
+			cctrl.nominal_pkt_padding80 = 1;
+			cctl_info_mask.nominal_pkt_padding80 = 0x3;
+		} else {
+			cctrl.nominal_pkt_padding80 = 0;
+			cctl_info_mask.nominal_pkt_padding80 = 0x3;
+		}
+		/* bw = 160MHz */
+		ppe16 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_160] & 0x7);
+		ppe8 = (sta->asoc_cap.ppe_thr[nss - 1][CHANNEL_WIDTH_160]>>3) & 0x7;
+
+		if ((ppe16 != 7) && (ppe8 == 7)) {
+			cctrl.nominal_pkt_padding160 = 2;
+			cctl_info_mask.nominal_pkt_padding160 = 0x3;
+		} else if (ppe8 != 7) {
+			cctrl.nominal_pkt_padding160 = 1;
+			cctl_info_mask.nominal_pkt_padding160 = 0x3;
+		} else {
+			cctrl.nominal_pkt_padding160 = 0;
+			cctl_info_mask.nominal_pkt_padding160 = 0x3;
+		}
+	} else {
+		cctrl.nominal_pkt_padding = sta->asoc_cap.pkt_padding;
+		cctrl.nominal_pkt_padding40 = sta->asoc_cap.pkt_padding;
+		cctrl.nominal_pkt_padding80 = sta->asoc_cap.pkt_padding;
+		cctrl.nominal_pkt_padding160 = sta->asoc_cap.pkt_padding;
+		cctl_info_mask.nominal_pkt_padding = 0x3;
+		cctl_info_mask.nominal_pkt_padding40 = 0x3;
+		cctl_info_mask.nominal_pkt_padding80 = 0x3;
+		cctl_info_mask.nominal_pkt_padding160 = 0x3;
+	}
+
+	if (sta->wmode&WLAN_MD_11AX) {
+          	/**
+                 * bsr_queue_size_format:
+                 * 1: buffer status unit is 802.11, HE mode
+                 * 0: buffer status unit is 802.11, legacy mode
+                 **/
+		cctrl.bsr_queue_size_format = 1;
+		cctl_info_mask.bsr_queue_size_format = 1;
+
+		if (sta->asoc_cap.he_ldpc)
+			cctrl.data_ldpc = 1;
+	} else if (sta->wmode & WLAN_MD_11AC) {
+		if (sta->asoc_cap.vht_ldpc)
+			cctrl.data_ldpc = 1;
+	} else if (sta->wmode & WLAN_MD_11N) {
+		if (sta->asoc_cap.ht_ldpc)
+			cctrl.data_ldpc = 1;
+	}
+	cctl_info_mask.data_ldpc = 1;
+
+	if (sta->set_rts_init_rate > 0) {
+		cctrl.rtsrate = sta->set_rts_init_rate - 1;
+		cctl_info_mask.rtsrate = 0x1FF;
+	}
+
+	sts = rtw_hal_cmc_tbl_cfg(hal_info, &cctrl, &cctl_info_mask, sta->macid);
+
+out:
+#ifdef DBG_DBCC_MONITOR_TIME
+	phl_fun_monitor_end(hal_info->phl_com, &start_t, __FUNCTION__);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return sts;
+}
+
+enum rtw_hal_status
+rtw_hal_upd_ampdu_cctrl_info(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct rtw_hal_mac_ax_cctl_info cctrl, cctl_info_mask;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &cctrl, 0, sizeof(struct rtw_hal_mac_ax_cctl_info));
+	_os_mem_set(hal_to_drvpriv(hal_info), &cctl_info_mask, 0, sizeof(struct rtw_hal_mac_ax_cctl_info));
+
+	if (NULL == sta)
+		goto out;
+
+	cctrl.max_agg_num_sel = 1;
+	cctl_info_mask.max_agg_num_sel = 1;
+
+	cctrl.max_agg_num = sta->asoc_cap.num_ampdu - 1;
+	cctl_info_mask.max_agg_num = 0xFF;
+
+	if (cctrl.max_agg_num <= 0x3F)
+		cctrl.ba_bmap = 0; /* 64 */
+	else if (cctrl.max_agg_num > 0x3F && cctrl.max_agg_num <= 0x7F)
+		cctrl.ba_bmap = 1; /* 128*/
+	else if ( cctrl.max_agg_num > 0x7F && cctrl.max_agg_num <= 0xFF)
+		cctrl.ba_bmap = 2; /* 256 */
+	cctl_info_mask.ba_bmap = 0x3;
+
+	sts = rtw_hal_cmc_tbl_cfg(hal_info, &cctrl, &cctl_info_mask, sta->macid);
+
+out:
+	return sts;
+}
+
+enum rtw_hal_status
+rtw_hal_cfg_rsc(void *hal, struct rtw_phl_stainfo_t *sta, u8 rsc_cfg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_wifi_role_link_t *rlink = sta->rlink;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	hsts = rtw_hal_mac_set_rsc_cfg(hal_info->hal_com, rsc_cfg, rlink->hw_band);
+
+	return hsts;
+}
+
+
+
+enum rtw_hal_status
+rtw_hal_cfg_rrsr_ref_rate_sel(void *hal, struct rtw_phl_stainfo_t *sta, bool ref_rate_sel)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	hsts = rtw_hal_mac_set_rrsr_ref_rate_sel(hal_info->hal_com, ref_rate_sel, sta->rlink->hw_band);
+
+	return hsts;
+}
+
+
+enum rtw_hal_status
+rtw_hal_cfg_tx_ampdu(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	/* update ampdu configuration */
+	if (256 >= sta->asoc_cap.num_ampdu) {
+		hsts = rtw_hal_mac_set_hw_ampdu_cfg(hal_info,
+		                                    sta->rlink->hw_band,
+		                                    sta->asoc_cap.num_ampdu,
+		                                    0xA5);
+	} else {
+		PHL_WARN("%s not consider this case : num_ampdu(%u), please check \n",
+			__func__, sta->asoc_cap.num_ampdu);
+	}
+
+	if (RTW_HAL_STATUS_SUCCESS != hsts)
+		goto out;
+out:
+	return hsts;
+}
+
+
+static enum rtw_hal_status
+_hal_update_dctrl_tbl(struct hal_info_t *hal_info,
+		      struct rtw_phl_stainfo_t *sta)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_dctl_info dctrl, dctl_info_mask;
+#if defined(CONFIG_RTW_TX_HW_AMSDU_HW_MERGE_MODE) || defined(CONFIG_RTW_TX_HW_AMSDU_SW_MERGE_MODE)
+	u8 amsdu_max_len = 0;
+#endif
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	phl_fun_monitor_start(&start_t, true, __FUNCTION__);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	if (NULL == sta)
+		goto out;
+
+	_os_mem_set(hal_to_drvpriv(hal_info), &dctrl, 0, sizeof(struct mac_ax_dctl_info));
+	_os_mem_set(hal_to_drvpriv(hal_info), &dctl_info_mask, 0, sizeof(struct mac_ax_dctl_info));
+
+#ifdef CONFIG_RTW_TX_HW_HDR_CONV
+	/* mhdr_len: qos = 26, nonqos = 24 */
+	sts = rtw_hal_mac_hdr_conv_tx_macid_en(hal_info, sta->macid, 0, 1, 26, 0);
+	if (sts != RTW_HAL_STATUS_SUCCESS)
+		goto out;
+
+	dctrl.vlan_tag_sel = 0;	/* [TODO] process vlan_tag */
+	dctrl.htc_order = 0;	/* [TODO] process htc_order */
+
+	dctl_info_mask.vlan_tag_sel = 0x3;
+	dctl_info_mask.htc_order = 0x1;
+
+#if defined(CONFIG_RTW_TX_HW_AMSDU_HW_MERGE_MODE) || defined(CONFIG_RTW_TX_HW_AMSDU_SW_MERGE_MODE)
+	amsdu_max_len = (sta->asoc_cap.max_amsdu_len > 0) ? HWAMSDU_MAX_LEN_7920 : HWAMSDU_MAX_LEN_4024;
+
+	sts = rtw_hal_mac_hwamsdu_max_len(hal_info, sta->macid, amsdu_max_len);
+	if (sts != RTW_HAL_STATUS_SUCCESS)
+		goto out;
+
+	sts = rtw_hal_mac_hwamsdu_macid_en(hal_info, sta->macid, 1);
+	if (sts != RTW_HAL_STATUS_SUCCESS)
+		goto out;
+#endif
+#endif
+
+#if defined(CONFIG_PHL_CSUM_OFFLOAD_RX) || defined(CONFIG_PHL_CSUM_OFFLOAD_TX)
+	dctrl.chksum_offload_en = 1;
+	dctl_info_mask.chksum_offload_en = 1;
+	dctrl.with_llc = 1;
+	dctl_info_mask.with_llc = 1;
+#endif /*CONFIG_PHL_CSUM_OFFLOAD_RX*/
+
+	sts = rtw_hal_dmc_tbl_cfg(hal_info, &dctrl, &dctl_info_mask, sta->macid);
+
+out:
+#ifdef DBG_DBCC_MONITOR_TIME
+	phl_fun_monitor_end(hal_info->phl_com, &start_t, __FUNCTION__);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return sts;
+}
+
+enum rtw_hal_status
+_hal_update_ba_cam(struct hal_info_t *hal_info, u8 valid, u16 macid,
+                   u8 dialog_token, u16 timeout, u16 start_seq_num, u16 ba_policy,
+                   u16 tid, u16 buf_size, u8 camid, u8 band_idx, u8 uid, bool is_std)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	void *drv = hal_to_drvpriv(hal_info);
+	struct mac_ax_bacam_info ba_cam = {0};
+	FUNCIN();
+	_os_mem_set(drv, &ba_cam, 0, sizeof(ba_cam));
+	ba_cam.valid = valid;
+	ba_cam.init_req = 1;
+	ba_cam.entry_idx = camid;
+	ba_cam.entry_idx_v1 = camid;
+	ba_cam.tid = tid;
+	ba_cam.macid = (u8)macid;
+	ba_cam.std_entry_en = is_std;
+	/**
+	 * uid_value valid when std_entry_en = 0,
+	 * it indicate the mapping of Rx UID in MAC/PHY intf and this temp entry.
+	 **/
+	ba_cam.uid_value = uid;
+	ba_cam.band_sel = band_idx;
+	if (buf_size > 64)
+		ba_cam.bmap_size = 4;
+	else
+		ba_cam.bmap_size = 0;
+	ba_cam.ssn = start_seq_num;
+
+	PHL_INFO("[BACAM] ba_cam.valid = %d, ba_cam.init_req = %d, ba_cam.entry_idx = %d\n",
+			 ba_cam.valid,
+			 ba_cam.init_req,
+			 ba_cam.entry_idx);
+	PHL_INFO("[BACAM] ba_cam.tid = %d, ba_cam.macid = %d, ba_cam.bmap_size = %d\n",
+			 ba_cam.tid,
+			 ba_cam.macid,
+			 ba_cam.bmap_size);
+	PHL_INFO("[BACAM] ba_cam.ssn = 0x%X, ba_cam.entry_idx_v1 =%d\n",
+			 ba_cam.ssn,
+			 ba_cam.entry_idx_v1);
+	PHL_INFO("[BACAM] ba_cam.band_sel = %d, ba_cam.uid_value = %d, ba_cam.std_entry_en = %d\n",
+			 ba_cam.band_sel,
+			 ba_cam.uid_value,
+			 ba_cam.std_entry_en);
+
+	hal_status = rtw_hal_bacam_cfg(hal_info, &ba_cam);
+
+	if (RTW_HAL_STATUS_FAILURE == hal_status) {
+		PHL_WARN("rtw_hal_bacam_cfg fail 0x%08X\n", hal_status);
+
+	}
+
+	FUNCOUT();
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_start_ba_session(void *hal, struct rtw_phl_stainfo_t *sta,
+                         u8 dialog_token, u16 timeout,
+                         u16 start_seq_num, u16 ba_policy,
+                         u16 tid, u16 buf_size)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_spec_t *hal_spec = phl_get_ic_spec(phl_com);
+	u8 max_std_entry_num = hal_spec->max_std_entry_num;
+	u8 band_idx = sta->rlink->hw_band;
+	u8 entry_idx = 0;
+
+	/* Update existing standard entry */
+	for (entry_idx = 0; entry_idx < max_std_entry_num; entry_idx++) {
+		if (hal_com->ba_ctl.used_map[entry_idx]
+		    && sta->macid == hal_com->ba_ctl.mac_id[entry_idx]
+		    && (u8)tid == hal_com->ba_ctl.tid[entry_idx]) {
+			PHL_INFO("Update existing standard entry(macid=%d, tid=%d)\n",
+			         sta->macid, tid);
+			hal_status = _hal_update_ba_cam(hal_info, 1, sta->macid, dialog_token,
+			                                timeout, start_seq_num, ba_policy, tid,
+			                                buf_size, entry_idx, band_idx, 0, true);
+			goto out;
+		}
+	}
+
+	if (hal_com->ba_ctl.count == max_std_entry_num) {
+		/*
+		 * Transfer existing standard entry to temp entry
+		 * when TID is 0(AC_BE) or 5(AC_VI)
+		 */
+		if (tid == 0 || tid == 5) {
+			for (entry_idx = 0; entry_idx < max_std_entry_num; entry_idx++) {
+				if (hal_com->ba_ctl.tid[entry_idx] != 0
+				    && hal_com->ba_ctl.tid[entry_idx] != 5) {
+					PHL_INFO("Transfer existing standard entry to temp entry \
+					         (entry_idx=%d, macid=%d, tid=%d).\n",
+					         entry_idx, hal_com->ba_ctl.mac_id[entry_idx],
+					         hal_com->ba_ctl.tid[entry_idx]);
+					hal_status = _hal_update_ba_cam(hal_info, 0, 0, 0, 0,
+					                                0, 0, 0, 0, entry_idx, HW_BAND_0, 0, false);
+					if (hal_status == RTW_HAL_STATUS_SUCCESS) {
+						hal_com->ba_ctl.used_map[entry_idx] = 0;
+						hal_com->ba_ctl.tid[entry_idx] = 0xff;
+						hal_com->ba_ctl.mac_id[entry_idx] = 0xffff;
+						hal_com->ba_ctl.count--;
+						break;
+					}
+				}
+			}
+		} else {
+			PHL_INFO("No available standard entry for tid(%d)\n", tid);
+			goto out;
+		}
+	}
+
+	for (entry_idx = 0; entry_idx < max_std_entry_num; entry_idx++) {
+		if (!hal_com->ba_ctl.used_map[entry_idx])
+			break;
+	}
+	if (entry_idx == max_std_entry_num) {
+		PHL_WARN("No avail standard entry found but count is(%d)\n",
+			hal_com->ba_ctl.count);
+		goto out;
+	}
+	hal_status = _hal_update_ba_cam(hal_info, 1, sta->macid, dialog_token,
+	                                timeout, start_seq_num, ba_policy, tid,
+	                                buf_size, entry_idx, band_idx, 0, true);
+	if (RTW_HAL_STATUS_SUCCESS != hal_status)
+		goto out;
+
+	hal_com->ba_ctl.used_map[entry_idx] = 1;
+	hal_com->ba_ctl.tid[entry_idx] = (u8)tid;
+	hal_com->ba_ctl.mac_id[entry_idx] = sta->macid;
+	hal_com->ba_ctl.count++;
+out:
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_stop_ba_session(void *hal, struct rtw_phl_stainfo_t *sta, u16 tid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	struct rtw_hal_com_t *hal_com = hal_info->hal_com;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_spec_t *hal_spec = phl_get_ic_spec(phl_com);
+	u8 max_std_entry_num = hal_spec->max_std_entry_num;
+	u8 entry_idx = 0;
+
+	for (entry_idx = 0; entry_idx < max_std_entry_num; entry_idx++) {
+		if (hal_com->ba_ctl.used_map[entry_idx]
+		    && sta->macid == hal_com->ba_ctl.mac_id[entry_idx]
+		    && (u8)tid == hal_com->ba_ctl.tid[entry_idx]) {
+			hal_status = _hal_update_ba_cam(hal_info, 0, 0, 0, 0,
+			                                0, 0, 0, 0, entry_idx, HW_BAND_0, 0, false);
+			break;
+		}
+	}
+	if (RTW_HAL_STATUS_SUCCESS != hal_status)
+		goto out;
+	hal_com->ba_ctl.used_map[entry_idx] = 0;
+	hal_com->ba_ctl.tid[entry_idx] = 0xff;
+	hal_com->ba_ctl.mac_id[entry_idx] = 0xffff;
+	hal_com->ba_ctl.count--;
+out:
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_init_tmp_entry(void *hal, u8 tmp_entry_num, u8 std_entry_num)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	u8 entry_idx = std_entry_num;
+	u8 uid = 0, cnt = 0;
+	u8 b0_tmp_num = tmp_entry_num;
+
+#ifdef CONFIG_DBCC_SUPPORT
+	u8 b1_tmp_num = 0;
+
+	if (is_dbcc_sup(hal_info->phl_com)) {
+		b1_tmp_num = tmp_entry_num / 2;
+		b0_tmp_num = tmp_entry_num - b1_tmp_num;
+		if (b1_tmp_num == 0) {
+			PHL_ERR("%s: b1_tmp_num = 0, we can't init tmp entry for Band1. It may can't respond to AMPDU!\n",
+				__FUNCTION__);
+		}
+	}
+#endif /* CONFIG_DBCC_SUPPORT */
+
+	uid = 0;
+	for (cnt = 0; cnt < b0_tmp_num; cnt++) {
+		hal_status = _hal_update_ba_cam(hal_info, 1, 0, 0, 0, 0,
+		                                0, 0, 0, entry_idx, HW_BAND_0, uid, false);
+		if (RTW_HAL_STATUS_SUCCESS != hal_status) {
+			PHL_ERR("rtw_hal_init_tmp_entry fail!\n");
+			break;
+		}
+		entry_idx++;
+		uid++;
+	}
+
+#ifdef CONFIG_DBCC_SUPPORT
+	uid = 0;
+	for (cnt = 0; cnt < b1_tmp_num; cnt++) {
+		hal_status = _hal_update_ba_cam(hal_info, 1, 0, 0, 0, 0,
+		                                0, 0, 0, entry_idx, HW_BAND_1, uid, false);
+		if (RTW_HAL_STATUS_SUCCESS != hal_status) {
+			PHL_ERR("rtw_hal_init_tmp_entry fail!\n");
+			break;
+		}
+		entry_idx++;
+		uid++;
+	}
+#endif /* CONFIG_DBCC_SUPPORT */
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_stainfo_init(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	void *drv = hal_to_drvpriv(hal_info);
+
+	sta->hal_sta = _os_mem_alloc(drv, sizeof(struct rtw_hal_stainfo_t));
+	if (sta->hal_sta == NULL) {
+		PHL_ERR("alloc hal_sta failed\n");
+		goto error_exit;
+	}
+
+#if defined(CONFIG_PHL_RELEASE_RPT_ENABLE) || defined(CONFIG_PCI_HCI)
+	/* alloc trx_stats */
+	sta->hal_sta->trx_stat.wp_rpt_stats = _os_mem_alloc(drv,
+		sizeof(struct rtw_wp_rpt_stats) * RTW_MAX_WP_RPT_AC_NUM(hal_info));
+
+	if (sta->hal_sta->trx_stat.wp_rpt_stats == NULL) {
+		PHL_ERR("alloc wp_rpt_stats failed\n");
+		goto error_rpt_stats;
+	}
+#endif
+
+	hal_status = rtw_hal_bb_stainfo_init(hal_info, sta);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("alloc bb_stainfo failed\n");
+		goto error_hsta_mem;
+	}
+	/* Init lock for tx statistics */
+	_os_spinlock_init(drv, &sta->hal_sta->trx_stat.tx_sts_lock);
+	/* Init STA RSSI Statistics */
+	_hal_sta_rssi_init(sta);
+
+	return hal_status;
+error_hsta_mem :
+#if defined(CONFIG_PHL_RELEASE_RPT_ENABLE) || defined(CONFIG_PCI_HCI)
+	if (sta->hal_sta->trx_stat.wp_rpt_stats) {
+		_os_mem_free(drv, sta->hal_sta->trx_stat.wp_rpt_stats,
+			     sizeof(struct rtw_wp_rpt_stats) * RTW_MAX_WP_RPT_AC_NUM(hal_info));
+		sta->hal_sta->trx_stat.wp_rpt_stats = NULL;
+	}
+
+error_rpt_stats :
+#endif
+	if (sta->hal_sta) {
+		_os_mem_free(drv, sta->hal_sta,
+				sizeof(struct rtw_hal_stainfo_t));
+		sta->hal_sta = NULL;
+	}
+error_exit :
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_stainfo_deinit(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	void *drv = hal_to_drvpriv(hal_info);
+
+	if (sta->hal_sta) {
+		/* Free lock for tx statistics */
+		_os_spinlock_free(drv, &sta->hal_sta->trx_stat.tx_sts_lock);
+
+#if defined(CONFIG_PHL_RELEASE_RPT_ENABLE) || defined(CONFIG_PCI_HCI)
+		if (sta->hal_sta->trx_stat.wp_rpt_stats) {
+			_os_mem_free(drv, sta->hal_sta->trx_stat.wp_rpt_stats,
+				     sizeof(struct rtw_wp_rpt_stats) * RTW_MAX_WP_RPT_AC_NUM(hal_info));
+			sta->hal_sta->trx_stat.wp_rpt_stats = NULL;
+		}
+#endif
+		hal_status = rtw_hal_bb_stainfo_deinit(hal_info, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			PHL_ERR("bb_stainfo deinit failed\n");
+
+		_os_mem_free(drv, sta->hal_sta,
+				sizeof(struct rtw_hal_stainfo_t));
+		sta->hal_sta = NULL;
+	}
+
+	return hal_status;
+}
+static void _hal_sta_set_default_value(struct hal_info_t *hal_info,
+        struct rtw_phl_stainfo_t *sta)
+{
+	sta->hal_sta->ra_info.ra_registered = false;
+}
+
+enum rtw_hal_status
+rtw_hal_set_dctrl_tbl_seciv(void *hal,
+				struct rtw_phl_stainfo_t *sta, u64 iv)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct mac_ax_dctl_info dctrl = {0}, dctrl_mask = {0};
+
+	if (NULL == sta)
+		goto out;
+
+	dctrl.aes_iv_l = (u16)(iv & 0xffffL);
+	dctrl.aes_iv_h = (u32)(iv >> 16) & 0xffffffffL;
+
+	dctrl_mask.aes_iv_l = 0xffffL;
+	dctrl_mask.aes_iv_h = 0xffffffffL;
+
+	sts = rtw_hal_dmc_tbl_cfg(hal_info, &dctrl ,&dctrl_mask, sta->macid);
+
+out:
+	return sts;
+}
+
+enum rtw_hal_status
+rtw_hal_add_sta_entry(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	_hal_sta_set_default_value(hal_info, sta);
+
+	/*add mac address-cam*/
+	if (rtw_hal_mac_addr_cam_add_entry(hal_info, sta) !=
+					RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_mac_addr_cam_add_entry failed\n");
+		goto _exit;
+	}
+
+	/*update default cmac table*/
+	if (_hal_set_default_cctrl_tbl(hal_info, sta) !=
+					RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("_hal_set_default_cctrl_tbl failed\n");
+		/* goto _exit; */ /* shall be unmark after header FW is ready */
+	}
+
+	if (_hal_update_dctrl_tbl(hal_info, sta) !=
+					RTW_HAL_STATUS_SUCCESS) {
+		PHL_WARN("_hal_set_default_dctrl_tbl failed\n");
+		/* goto _exit; */
+	}
+
+	/*add bb stainfo*/
+	if (rtw_hal_bb_stainfo_add(hal_info, sta) !=
+					RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_bb_stainfo_add failed\n");
+		goto _err_bbsta_add;
+	}
+	return RTW_HAL_STATUS_SUCCESS;
+
+_err_bbsta_add:
+	rtw_hal_mac_addr_cam_del_entry(hal_info, sta);
+_exit:
+	return RTW_HAL_STATUS_FAILURE;
+}
+
+#define WAIT_DROP_PKT_TIMEOUT 300
+void
+rtw_hal_disconnect_drop_all_pkt(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_i = (struct hal_info_t *)hal;
+
+#ifdef CONFIG_PCI_HCI
+	struct rtw_wp_rpt_stats *rpt_stats =
+		(struct rtw_wp_rpt_stats *)sta->hal_sta->trx_stat.wp_rpt_stats;
+	u32 start_time = _os_get_cur_time_ms();
+	u16 cnt = 0;
+
+	while (true) {
+		cnt++;
+		rtw_hal_mac_data_drop_once(hal_i->hal_com, sta);
+		if (0 == rpt_stats->busy_cnt) {
+			PHL_INFO("%s: rpt_stats->busy_cnt == 0, loop cnt(%d)\n",
+				__func__, cnt);
+			break;
+		}
+		if (phl_get_passing_time_ms(start_time) > WAIT_DROP_PKT_TIMEOUT) {
+			PHL_ERR("%s: Fail, wait time > (%d), busy_cnt(%d)\n",
+				__func__, WAIT_DROP_PKT_TIMEOUT,
+				rpt_stats->busy_cnt);
+			break;
+		}
+		_os_sleep_ms(hal_to_drvpriv(hal_i), 5);
+	}
+#else
+	/* todo: check empty for usb, sdio */
+	rtw_hal_mac_data_drop_once(hal_i->hal_com, sta);
+#endif
+}
+
+enum rtw_hal_status
+rtw_hal_restore_sta_entry(struct rtw_phl_com_t* phl_com, void *hal,
+                          struct rtw_phl_stainfo_t *sta, bool is_connect)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	enum phl_upd_mode mode = PHL_UPD_STA_CON_DISCONN;
+
+	hal_status = rtw_hal_mac_addr_cam_change_entry(hal_info, sta, PHL_UPD_ROLE_FW_RESTORE, is_connect);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_mac_addr_cam_change_entry failed\n");
+		return hal_status;
+	}
+
+	/* update cmac table */
+	if (RTW_HAL_STATUS_SUCCESS != _hal_update_cctrl_tbl(hal_info, sta))
+		PHL_WARN("_hal_update_cctrl_tbl failed\n");
+
+	/*update dmac table*/
+	if (RTW_HAL_STATUS_SUCCESS != _hal_update_dctrl_tbl(hal_info, sta))
+		PHL_WARN("_hal_update_dctrl_tbl failed\n");
+
+	/*change mac address-cam & mac_h2c_join_info*/
+	hal_status = rtw_hal_mac_addr_cam_change_entry(hal_info, sta, mode, is_connect);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_mac_addr_cam_change_entry failed\n");
+	}
+
+	if (is_connect) {
+	#ifdef CONFIG_PHL_PKTOFLD
+		rtw_phl_pkt_ofld_null_request(phl_com, sta, NULL);
+	#endif
+		hal_status = rtw_hal_cfg_tx_ampdu(hal, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("rtw_hal_cfg_tx_ampdu failed\n");
+		}
+
+		if (RTW_HAL_STATUS_SUCCESS != _hal_bfee_init(hal_info, sta)) {
+			PHL_ERR("_hal_bfee_init Fail!\n");
+		}
+
+		if (sta->hal_sta->rssi_stat.assoc_rssi == 0
+                #ifdef CONFIG_PHL_TDLS
+			/* There is no association frame for TDLS connection */
+			&& sta->wrole->type != PHL_RTYPE_TDLS
+                #endif
+		) {
+			PHL_ERR("%s macid:%d assoc_rssi == 0\n", __func__, sta->macid);
+			_os_warn_on(1);
+		}
+
+		hal_status = rtw_hal_bb_upt_ramask(hal_info, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			PHL_ERR("rtw_hal_bb_upt_ramask failed\n");
+
+		hal_status = rtw_hal_bb_ra_register(hal_info, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			PHL_ERR("rtw_hal_bb_ra_register failed\n");
+
+		if (sta->wmode & WLAN_MD_11AX) {
+			rtw_hal_bb_set_sta_id(hal_info, sta->aid, sta->rlink->hw_band);
+			rtw_hal_bb_set_bss_color(hal_info, sta->asoc_cap.bsscolor,
+				sta->rlink->hw_band);
+			rtw_hal_bb_set_tb_pwr_ofst(hal_info, 0, sta->rlink->hw_band);
+		}
+		/* reset rssi stat value */
+		sta->hal_sta->rssi_stat.ma_rssi_mgnt = 0;
+	} else {
+		#ifdef CONFIG_PHL_PKTOFLD
+		rtw_phl_pkt_ofld_reset_entry(phl_com, sta->macid);
+		#endif
+		hal_status = rtw_hal_bb_ra_deregister(hal_info, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			PHL_ERR("rtw_hal_bb_ra_deregister failed\n");
+
+		rtw_hal_disconnect_drop_all_pkt(hal_info, sta);
+		/* reset drv rssi_stat */
+		_hal_sta_rssi_init(sta);
+		_hal_sta_set_default_value(hal_info, sta);
+	}
+	/* reset bb rssi_stat */
+	rtw_hal_bb_media_status_update(hal_info, sta, is_connect);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_update_sta_entry(struct rtw_phl_com_t* phl_com, void *hal,
+                         struct rtw_phl_stainfo_t *sta, bool is_connect)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	enum phl_upd_mode mode = PHL_UPD_STA_CON_DISCONN;
+
+	/* update cmac table */
+	if (RTW_HAL_STATUS_SUCCESS != _hal_update_cctrl_tbl(hal_info, sta))
+		PHL_WARN("_hal_update_cctrl_tbl failed\n");
+
+	/* update dmac table */
+	if (RTW_HAL_STATUS_SUCCESS != _hal_update_dctrl_tbl(hal_info, sta))
+		PHL_WARN("_hal_update_dctrl_tbl failed\n");
+
+	/* change mac address-cam & mac_h2c_join_info */
+	hal_status = rtw_hal_mac_addr_cam_change_entry(hal_info, sta, mode, is_connect);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+		PHL_ERR("rtw_hal_cfg_tx_ampdu failed\n");
+	}
+
+	if (is_connect) {
+		if (rtw_phl_role_is_client_category(sta->wrole)) {
+			rtw_hal_mac_tbtt_tuning_cfg(hal_info,
+						    sta->rlink->hw_band,
+						    sta->rlink->hw_port,
+						    0);
+		}
+	#ifdef CONFIG_PHL_PKTOFLD
+		rtw_phl_pkt_ofld_null_request(phl_com, sta, NULL);
+	#endif
+		hal_status = rtw_hal_cfg_tx_ampdu(hal, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS) {
+			PHL_ERR("rtw_hal_cfg_tx_ampdu failed\n");
+		}
+
+		if (RTW_HAL_STATUS_SUCCESS != _hal_bfee_init(hal_info, sta)) {
+			PHL_ERR("_hal_bfee_init Fail!\n");
+		}
+
+		if (sta->hal_sta->rssi_stat.assoc_rssi == 0
+                #ifdef CONFIG_PHL_TDLS
+			/* There is no association frame for TDLS connection */
+			&& sta->wrole->type != PHL_RTYPE_TDLS
+                #endif
+		) {
+			PHL_ERR("%s macid:%d assoc_rssi == 0\n", __func__, sta->macid);
+			_os_warn_on(1);
+		}
+
+		hal_status = rtw_hal_bb_upt_ramask(hal_info, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			PHL_ERR("rtw_hal_bb_upt_ramask failed\n");
+
+		hal_status = rtw_hal_bb_ra_register(hal_info, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			PHL_ERR("rtw_hal_bb_ra_register failed\n");
+		hal_info->hal_com->assoc_sta_cnt++;
+		hal_info->hal_com->band[sta->rlink->hw_band].assoc_sta_cnt++;
+
+		if (sta->wmode & WLAN_MD_11AX) {
+			rtw_hal_bb_set_sta_id(hal_info, sta->aid, sta->rlink->hw_band);
+			rtw_hal_bb_set_bss_color(hal_info, sta->asoc_cap.bsscolor,
+				sta->rlink->hw_band);
+			rtw_hal_bb_set_tb_pwr_ofst(hal_info, 0, sta->rlink->hw_band);
+		}
+		/* reset rssi stat value */
+		sta->hal_sta->rssi_stat.ma_rssi_mgnt = 0;
+	} else {
+		#ifdef CONFIG_PHL_PKTOFLD
+		rtw_phl_pkt_ofld_reset_entry(phl_com, sta->macid);
+		#endif
+		hal_status = rtw_hal_bb_ra_deregister(hal_info, sta);
+		if (hal_status != RTW_HAL_STATUS_SUCCESS)
+			PHL_ERR("rtw_hal_bb_ra_deregister failed\n");
+
+		hal_info->hal_com->assoc_sta_cnt--;
+		hal_info->hal_com->band[sta->rlink->hw_band].assoc_sta_cnt--;
+		rtw_hal_disconnect_drop_all_pkt(hal_info, sta);
+		/* reset drv rssi_stat */
+		_hal_sta_rssi_init(sta);
+		_hal_sta_set_default_value(hal_info, sta);
+	}
+	/* reset bb rssi_stat */
+	rtw_hal_bb_media_status_update(hal_info, sta, is_connect);
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s: hal_status(%d), macid(0x%X), is_connect(%d)\n",
+		__FUNCTION__, hal_status, sta->macid , is_connect);
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_change_sta_entry(void *hal, struct rtw_phl_stainfo_t *sta,
+						enum phl_upd_mode mode)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	bool is_connect = false;
+#ifdef DBG_DBCC_MONITOR_TIME
+	u32 start_t = 0;
+
+	PHL_FUN_MON_START(&start_t);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	PHL_TRACE(COMP_PHL_DBG, _PHL_INFO_, "%s: sta->macid(0x%X), mode(%d)\n",
+		__FUNCTION__, sta->macid , mode);
+	/* update cmac table */
+	if (RTW_HAL_STATUS_SUCCESS != _hal_update_cctrl_tbl(hal_info, sta))
+		PHL_WARN("_hal_update_cctrl_tbl failed\n");
+
+	/* update dmac table */
+	if (RTW_HAL_STATUS_SUCCESS != _hal_update_dctrl_tbl(hal_info, sta))
+		PHL_WARN("_hal_update_dctrl_tbl failed\n");
+
+	/* change mac address-cam & mac_h2c_join_info */
+	is_connect = (sta->rlink->mstate == MLME_LINKED) ? true : false;
+	hal_status = rtw_hal_mac_addr_cam_change_entry(hal_info, sta, mode, is_connect);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("rtw_hal_mac_addr_cam_change_entry failed\n");
+
+	hal_status = rtw_hal_bb_ra_update(hal_info, sta);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("rtw_hal_bb_ra_update failed\n");
+#ifdef DBG_DBCC_MONITOR_TIME
+	PHL_FUNC_MON_END(hal_info->phl_com, &start_t, TIME_PHL_MAX);
+#endif /* DBG_DBCC_MONITOR_TIME */
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_del_sta_entry(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	_hal_sta_set_default_value(hal_info, sta);
+
+	hal_status = rtw_hal_mac_addr_cam_del_entry(hal_info, sta);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("mac_addr_cam_del_entry failed\n");
+
+	hal_status = rtw_hal_bb_stainfo_delete(hal_info, sta);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		PHL_ERR("bb_stainfo deinit failed\n");
+
+	return hal_status;
+}
+
+u8 rtw_hal_get_sta_rssi(struct rtw_phl_stainfo_t *sta)
+{
+	u8 rssi = (sta->hal_sta->rssi_stat.rssi >> 1);
+
+	return rssi;
+}
+
+u8 rtw_hal_get_sta_rssi_bcn(struct rtw_phl_stainfo_t *sta)
+{
+	u8 rssi = (sta->hal_sta->rssi_stat.rssi_bcn >> 1);
+
+	return rssi;
+}
+
+bool rtw_hal_is_sta_linked(void *hal, struct rtw_phl_stainfo_t *sta)
+{
+	return (sta->hal_sta->ra_info.ra_registered == true) ? true : false;
+}
+
+enum rtw_hal_status
+rtw_hal_set_edca(void *hal,
+                 struct rtw_wifi_role_link_t *rlink,
+                 u8 ac,
+                 u32 edca)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status;
+
+	hal_status = rtw_hal_mac_set_edca(hal_info->hal_com, rlink->hw_band,
+					  rlink->hw_wmm, ac, edca);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_cfg_tx_amsdu(void *hal,
+		     u8 macid,
+		     u8 enable,
+		     u8 amsdu_max_len,
+		     u8 qos_field_h,
+		     u8 qos_field_h_en,
+		     u8 mhdr_len,
+		     u8 vlan_tag_valid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	/* set mac hdr length / QoS content */
+	hsts = rtw_hal_mac_hdr_conv_tx_macid_en(hal_info,
+						macid, qos_field_h,
+						qos_field_h_en,
+						mhdr_len,
+						vlan_tag_valid);
+	if (RTW_HAL_STATUS_SUCCESS != hsts) {
+		PHL_ERR("rtw_hal_mac_hdr_conv_tx_macid_en failed, hsts = %d\n", hsts);
+		goto out;
+	}
+
+	/* set HWAMSDU max length */
+	hsts = rtw_hal_mac_hwamsdu_max_len(hal_info, macid, amsdu_max_len);
+	if (RTW_HAL_STATUS_SUCCESS != hsts) {
+		PHL_ERR("rtw_hal_mac_hwamsdu_max_len failed, hsts = %d\n", hsts);
+		goto out;
+	}
+
+	/* enable HWAMSDU by macid */
+	hsts = rtw_hal_mac_hwamsdu_macid_en(hal_info, macid, enable);
+	if (RTW_HAL_STATUS_SUCCESS != hsts) {
+		PHL_ERR("rtw_hal_mac_hwamsdu_macid_en failed, hsts = %d\n", hsts);
+		goto out;
+	}
+
+	/* Could add more halmac ops and input parameter in this function */
+
+out:
+	return hsts;
+}
+
+
+enum rtw_hal_status
+rtw_hal_set_sta_rx_sts(struct rtw_phl_stainfo_t *sta, u8 rst,
+									struct rtw_r_meta_data *meta)
+{
+	if (rst) {
+		sta->hal_sta->trx_stat.rx_ok_cnt = 0;
+		sta->hal_sta->trx_stat.rx_err_cnt = 0;
+	} else {
+		if (meta->crc32 || meta->icverr)
+			sta->hal_sta->trx_stat.rx_err_cnt++;
+		else
+			sta->hal_sta->trx_stat.rx_ok_cnt++;
+
+		sta->hal_sta->trx_stat.rx_bw = meta->bw;
+		sta->hal_sta->trx_stat.rx_rate = meta->rx_rate;
+		sta->hal_sta->trx_stat.rx_gi_ltf = meta->rx_gi_ltf;
+	}
+	/* TODO: rx_rate_plurality */
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_query_rainfo(void *hal, struct rtw_hal_stainfo_t *hal_sta,
+		     struct rtw_phl_rainfo *phl_rainfo)
+{
+	enum rtw_hal_status hal_sts = RTW_HAL_STATUS_FAILURE;
+
+	hal_sts = rtw_hal_bb_query_rainfo(hal, hal_sta, phl_rainfo);
+
+
+	return hal_sts;
+}
+
+/**
+ * rtw_hal_query_txsts_rpt() - get txok and tx retry info
+ * @hal:		struct hal_info_t *
+ * @macid:		indicate the first macid that you want to query.
+ * Return rtw_hal_bb_query_txsts_rpt's return value in enum rtw_hal_status type.
+ */
+enum rtw_hal_status
+rtw_hal_query_txsts_rpt(void *hal, u16 macid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	/*get tx ok and tx retry statistics*/
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_bb_query_txsts_rpt(hal_info, macid, 0xFFFF))
+		return RTW_HAL_STATUS_FAILURE;
+	else
+		return RTW_HAL_STATUS_SUCCESS;
+}
+
+bool
+rtw_hal_check_sta_has_busy_wp(struct rtw_phl_stainfo_t *sta)
+{
+#ifdef CONFIG_PCI_HCI
+	struct rtw_wp_rpt_stats *rpt_stats = sta->hal_sta->trx_stat.wp_rpt_stats;
+	return (rpt_stats->busy_cnt) ? true : false;
+#else
+	/*
+	* TODO: USB/SDIO not implement busy wp cnt statistic yet,
+	* so return false to avoid affect core-layer logic.
+	*/
+	return false;
+#endif
+}
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sta.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sta.h
new file mode 100644
index 000000000000..72e0ae58d9d4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_sta.h
@@ -0,0 +1,25 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_STA_H_
+#define _HAL_STA_H_
+
+enum rtw_hal_status
+rtw_hal_restore_sta_entry(struct rtw_phl_com_t* phl_com, void *hal,
+                          struct rtw_phl_stainfo_t *sta, bool is_connect);
+
+enum rtw_hal_status
+rtw_hal_init_tmp_entry(void *hal, u8 tmp_entry_num, u8 std_entry_num);
+
+#endif  /*_HAL_STA_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_str_proc.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_str_proc.c
new file mode 100644
index 000000000000..65510aecc388
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_str_proc.c
@@ -0,0 +1,248 @@
+/******************************************************************************
+*
+* Copyright(c) 2019 Realtek Corporation.
+*
+* This program is free software; you can redistribute it and/or modify it
+* under the terms of version 2 of the GNU General Public License as
+* published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful, but WITHOUT
+* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+* more details.
+*
+*****************************************************************************/
+
+#include "hal_str_proc.h"
+
+
+bool hal_is_comment_string(
+		char			*szStr
+)
+{
+	if (*szStr == '/' && *(szStr + 1) == '/')
+		return true;
+	else
+		return false;
+}
+
+bool hal_get_fractionvalue_fromstring(
+		char			*szStr,
+		u8				*pInteger,
+		u8				*pFraction,
+		u32			*pu4bMove
+)
+{
+	char	*szScan = szStr;
+
+	/* Initialize output. */
+	*pu4bMove = 0;
+	*pInteger = 0;
+	*pFraction = 0;
+
+	/* Skip leading space. */
+	while (*szScan != '\0' &&	(*szScan == ' ' || *szScan == '\t')) {
+		++szScan;
+		++(*pu4bMove);
+	}
+
+	if (*szScan < '0' || *szScan > '9')
+		return false;
+
+	/* Parse each digit. */
+	do {
+		(*pInteger) *= 10;
+		*pInteger += (*szScan - '0');
+
+		++szScan;
+		++(*pu4bMove);
+
+		if (*szScan == '.') {
+			++szScan;
+			++(*pu4bMove);
+
+			if (*szScan < '0' || *szScan > '9')
+				return false;
+
+			*pFraction += (*szScan - '0') * 10;
+			++szScan;
+			++(*pu4bMove);
+
+			if (*szScan >= '0' && *szScan <= '9') {
+				*pFraction += *szScan - '0';
+				++szScan;
+				++(*pu4bMove);
+			}
+			return true;
+		}
+	} while (*szScan >= '0' && *szScan <= '9');
+
+	return true;
+}
+
+bool hal_is_alpha(char ch_tmp)
+{
+	if ((ch_tmp >= 'a' && ch_tmp <= 'z') ||
+		(ch_tmp >= 'A' && ch_tmp <= 'Z'))
+		return true;
+	else
+		return false;
+}
+
+bool hal_ishexdigit(char ch_tmp)
+{
+	if ((ch_tmp >= '0' && ch_tmp <= '9') ||
+		(ch_tmp >= 'a' && ch_tmp <= 'f') ||
+		(ch_tmp >= 'A' && ch_tmp <= 'F'))
+		return true;
+	else
+		return false;
+}
+
+bool hal_get_hexvalue_fromstring(
+		char			*szStr,
+		u32			*pu4bVal,
+		u32			*pu4bMove
+)
+{
+	char		*szScan = szStr;
+
+	/* Check input parameter. */
+	if (szStr == NULL || pu4bVal == NULL || pu4bMove == NULL) {
+		PHL_INFO("GetHexValueFromString(): Invalid inpur argumetns! szStr: %p, pu4bVal: %p, pu4bMove: %p\n", szStr, pu4bVal, pu4bMove);
+		return false;
+	}
+
+	/* Initialize output. */
+	*pu4bMove = 0;
+	*pu4bVal = 0;
+
+	/* Skip leading space. */
+	while (*szScan != '\0' &&
+		(*szScan == ' ' || *szScan == '\t')) {
+		szScan++;
+		(*pu4bMove)++;
+	}
+
+	/* Skip leading '0x' or '0X'. */
+	if (*szScan == '0' && (*(szScan + 1) == 'x' || *(szScan + 1) == 'X')) {
+		szScan += 2;
+		(*pu4bMove) += 2;
+	}
+
+	/* Check if szScan is now pointer to a character for hex digit, */
+	/* if not, it means this is not a valid hex number. */
+	if (!hal_ishexdigit(*szScan))
+		return false;
+
+	/* Parse each digit. */
+	do {
+		(*pu4bVal) <<= 4;
+		*pu4bVal += hal_mapchar_tohexdigit(*szScan);
+
+		szScan++;
+		(*pu4bMove)++;
+	} while (hal_ishexdigit(*szScan));
+
+	return true;
+}
+
+bool hal_is_allspace_tab(
+	char	*data,
+	u8	size
+)
+{
+	u8	cnt = 0, NumOfSpaceAndTab = 0;
+
+	while (size > cnt) {
+		if (data[cnt] == ' ' || data[cnt] == '\t' || data[cnt] == '\0')
+			++NumOfSpaceAndTab;
+
+		++cnt;
+	}
+
+	return size == NumOfSpaceAndTab;
+}
+
+u32 hal_mapchar_tohexdigit(
+			char		chTmp
+)
+{
+	if (chTmp >= '0' && chTmp <= '9')
+		return chTmp - '0';
+	else if (chTmp >= 'a' && chTmp <= 'f')
+		return 10 + (chTmp - 'a');
+	else if (chTmp >= 'A' && chTmp <= 'F')
+		return 10 + (chTmp - 'A');
+	else
+		return 0;
+}
+
+
+bool hal_parse_fiedstring(char	*in_str, u32	*start, char	*out_str, char lqualifier, char rqualifier)
+{
+	u32	i = 0, j = 0;
+	char	c = in_str[(*start)++];
+
+	if (c != lqualifier)
+		return false;
+
+	i = (*start);
+	c = in_str[(*start)++];
+
+	while (c != rqualifier && c != '\0')
+		c = in_str[(*start)++];
+
+	if (c == '\0')
+		return false;
+
+	j = (*start) - 2;
+	_os_strncpy((char *)out_str, (const char *)(in_str + i), j - i + 1);
+
+	return true;
+}
+
+
+bool hal_get_u1bint_fromstr_indec(char	*str, u8	*pint)
+{
+	u16 i = 0;
+	*pint = 0;
+
+	while (str[i] != '\0') {
+		if (str[i] >= '0' && str[i] <= '9') {
+			*pint *= 10;
+			*pint += (str[i] - '0');
+		} else
+			return false;
+		++i;
+	}
+
+	return true;
+}
+
+
+bool hal_get_s1bint_fromstr_indec(char	*str,	s8 *val)
+{
+	u8 negative = 0;
+	u16 i = 0;
+
+	*val = 0;
+
+	while (str[i] != '\0') {
+		if (i == 0 && (str[i] == '+' || str[i] == '-')) {
+			if (str[i] == '-')
+				negative = 1;
+		} else if (str[i] >= '0' && str[i] <= '9') {
+			*val *= 10;
+			*val += (str[i] - '0');
+		} else
+			return false;
+		++i;
+	}
+
+	if (negative)
+		*val = -*val;
+
+	return true;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_str_proc.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_str_proc.h
new file mode 100644
index 000000000000..3c67a72a862e
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_str_proc.h
@@ -0,0 +1,43 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#include "hal_headers.h"
+
+
+#define hal_getLinefrombuffer(buffer)	_os_strsep(&buffer, "\r\n")
+
+bool hal_is_comment_string(char *szStr);
+
+bool hal_is_alpha(char ch_tmp);
+
+bool hal_ishexdigit(char chTmp);
+
+bool hal_get_hexvalue_fromstring(char	*szStr, u32 *pu4bVal, u32 *pu4bMove);
+
+bool hal_get_fractionvalue_fromstring(
+		char			*szStr,
+		u8				*pInteger,
+		u8				*pFraction,
+		u32			*pu4bMove);
+
+bool hal_is_allspace_tab(char	*data, u8	size);
+
+u32 hal_mapchar_tohexdigit(char 		chTmp);
+
+bool hal_parse_fiedstring(char	*in_str, u32	*start, char	*out_str, char lqualifier, char rqualifier);
+
+bool hal_get_u1bint_fromstr_indec(char	*str, u8	*pint);
+
+bool hal_get_s1bint_fromstr_indec(char	*str,	s8 *val);
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_struct.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_struct.h
new file mode 100644
index 000000000000..8e3b74fdc2c3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_struct.h
@@ -0,0 +1,439 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_STRUCT_H_
+#define _HAL_STRUCT_H_
+
+
+struct hal_info_t;
+
+#define hal_get_trx_ops(_halinfo)	(_halinfo->trx_ops)
+/**
+ * struct hal_trx_ops - hw ic specific operations
+ *
+ * @init: the function for initializing IC specific data and hw configuration
+ * @deinit: the function for deinitializing IC specific data and hw configuration
+ * @query_tx_res: the function for querying hw tx resource
+ * @query_rx_res: the function for querying hw rx resource
+ * @map_hw_tx_chnl: the function for getting mapping hw tx channel
+ * @qsel_to_tid: the function for converting hw qsel to tid value
+ * @query_txch_num: the function for querying total hw tx dma channels number
+ * @query_rxch_num: the function for querying total hw rx dma channels number
+ * @update_wd: the function for updating wd page for xmit packet
+ * @update_txbd: the function for updating tx bd for xmit packet
+ * @tx_start: the function to trigger hw to start tx
+ * @get_fwcmd_queue_idx: the function to get fwcmd queue idx
+ * @check_rxrdy: the function check if hw rx buffer is ready to access
+ * @handle_rxbd_info: the function handling hw rxbd information
+ * @handle_rx_buffer: the function handling hw rx buffer
+ * @update_rxbd: the function for updating rx bd for recv packet
+ * @notify_rxdone: the function to notify hw rx done
+ * @handle_wp_rpt: the function parsing wp report content
+ * @query_txch_hwband: which hwband that txch belong to
+ * @query_txch_map: fill txch map by band_idx
+ */
+struct hal_trx_ops {
+	u8 (*map_hw_tx_chnl)(struct hal_info_t *hal, u16 macid, enum rtw_phl_ring_cat cat, u8 band);
+	u8 (*query_txch_num)(void);
+	u8 (*query_rxch_num)(void);
+	u32 (*hal_get_wd_len)(struct hal_info_t *hal, struct rtw_xmit_req *tx_req);
+
+#ifdef CONFIG_PCI_HCI
+	enum rtw_hal_status (*init)(struct hal_info_t *hal, u8 *txbd_buf, u8 *rxbd_buf);
+	void (*deinit)(struct hal_info_t *hal);
+
+	u16 (*query_tx_res)(struct rtw_hal_com_t *hal_com, u8 dma_ch,
+			    u16 *host_idx, u16 *hw_idx);
+	u16 (*query_rx_res)(struct rtw_hal_com_t *hal_com, u8 dma_ch,
+			    u16 *host_idx, u16 *hw_idx);
+	u16 (*get_rxbd_num)(struct rtw_hal_com_t *hal_com, u8 dma_ch);
+	u16 (*get_rxbuf_num)(struct rtw_hal_com_t *hal_com, u8 dma_ch);
+	u16 (*get_rxbuf_size)(struct rtw_hal_com_t *hal_com, u8 dma_ch);
+	void (*cfg_dma_io)(struct hal_info_t *hal, u8 en);
+	void (*cfg_txdma)(struct hal_info_t *hal, u8 en, u8 dma_ch);
+	void (*cfg_txhci)(struct hal_info_t *hal, u8 en);
+	void (*cfg_rxhci)(struct hal_info_t *hal, u8 en);
+	void (*clr_rwptr)(struct hal_info_t *hal);
+	void (*rst_bdram)(struct hal_info_t *hal);
+	void (*cfg_rsvd_ctrl)(struct hal_info_t *hal);
+	u8 (*qsel_to_tid)(struct hal_info_t *hal, u8 qsel_id, u8 tid_indic);
+
+	u8 (*query_txch_hwband)(u8 dma_ch);
+	void (*query_txch_map)(enum phl_band_idx band, void *ch_map);
+
+	enum rtw_hal_status
+		(*update_wd)(struct hal_info_t *hal, struct rtw_phl_pkt_req *req);
+	enum rtw_hal_status
+		(*update_txbd)(struct hal_info_t *hal,
+				struct tx_base_desc *txbd_ring,
+				struct rtw_wd_page *wd_page,
+				u8 ch_idx, u16 wd_num);
+	enum rtw_hal_status
+		(*tx_start)(struct hal_info_t *hal,
+				struct tx_base_desc *txbd, u8 dma_ch);
+
+	u8 (*get_fwcmd_queue_idx)(void);
+
+	u8 (*check_rxrdy)(struct rtw_phl_com_t *phl_com,
+			  struct rtw_rx_buf *rx_buf, u8 dma_ch);
+	enum rtw_hal_status
+		(*handle_rx_buffer)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal,
+					u8 *buf, u32 buf_size,
+					struct rtw_phl_rx_pkt *rxpkt);
+	u8 (*handle_rxbd_info)(struct hal_info_t *hal, u8 *rxbuf, u16 *buf_size);
+
+	enum rtw_hal_status
+		(*update_rxbd)(struct hal_info_t *hal,
+				struct rx_base_desc *rxbd,
+				struct rtw_rx_buf *rx_buf, u8 ch_idx);
+
+	enum rtw_hal_status
+		(*notify_rxdone)(struct hal_info_t *hal,
+				struct rx_base_desc *rxbd, u8 ch, u16 rxcnt);
+
+	u16 (*handle_wp_rpt)(struct hal_info_t *hal, u8 *rp, u16 len,
+			     u8 *sw_retry, u8 *dma_ch, u16 *wp_seq, u8 *mac_id,
+			     u8 *ac_queue, u8 *txsts);
+#endif /*CONFIG_PCI_HCI*/
+
+#ifdef CONFIG_USB_HCI
+	enum rtw_hal_status (*init)(struct hal_info_t *hal);
+	void (*deinit)(struct hal_info_t *hal);
+
+	enum rtw_hal_status
+	(*hal_fill_wd)(struct hal_info_t *hal, struct rtw_xmit_req *tx_req,
+				u8 *wd_buf, u32 *wd_len);
+	u8 (*get_bulkout_id)(struct hal_info_t *hal, u8 ch_dma, u8 mode);
+	enum rtw_hal_status
+		(*handle_rx_buffer)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal,
+					u8 *buf, u32 buf_size,
+					struct rtw_phl_rx_pkt *rxpkt);
+ 	enum rtw_hal_status
+		(*query_hal_info)(struct hal_info_t *hal, u8 info_id, void *value);
+	enum rtw_hal_status
+		(*usb_tx_agg_cfg)(struct hal_info_t *hal, u8* wd_buf, u8 agg_num);
+	enum rtw_hal_status
+		(*usb_rx_agg_cfg)(struct hal_info_t *hal, u8 mode, u8 agg_mode,
+			u8 drv_define, u8 timeout, u8 size, u8 pkt_num);
+	u8 (*get_fwcmd_queue_idx)(void);
+	u8 (*get_max_bulkout_wd_num)(struct hal_info_t *hal);
+	void (*cfg_dma_io)(struct hal_info_t *hal, u8 en);
+	void (*cfg_txdma)(struct hal_info_t *hal, u8 en, u8 dma_ch);
+	void (*cfg_txhci)(struct hal_info_t *hal, u8 en);
+	void (*cfg_rxhci)(struct hal_info_t *hal, u8 en);
+	void (*clr_rwptr)(struct hal_info_t *hal);
+	void (*rst_bdram)(struct hal_info_t *hal);
+	void (*cfg_rsvd_ctrl)(struct hal_info_t *hal);
+	u16 (*handle_wp_rpt)(struct hal_info_t *hal, u8 *rp, u16 len,
+			u8 *mac_id, u8 *ac_queue, u8 *txsts);
+#endif /*CONFIG_USB_HCI*/
+
+#ifdef CONFIG_SDIO_HCI
+	enum rtw_hal_status (*init)(struct hal_info_t *hal);
+	void (*deinit)(struct hal_info_t *hal);
+	u16 (*query_tx_res)(struct rtw_hal_com_t *hal_com, u8 dma_ch,
+			    u16 *host_idx, u16 *hw_idx);
+	u16 (*query_rx_res)(struct rtw_hal_com_t *hal_com, u8 dma_ch,
+			    u16 *host_idx, u16 *hw_idx);
+	u16 (*get_rxbd_num)(struct rtw_hal_com_t *hal_com, u8 dma_ch);
+	u16 (*get_rxbuf_num)(struct rtw_hal_com_t *hal_com, u8 dma_ch);
+	u16 (*get_rxbuf_size)(struct rtw_hal_com_t *hal_com, u8 dma_ch);
+	enum rtw_hal_status
+	(*hal_fill_wd)(struct hal_info_t *hal, struct rtw_xmit_req *tx_req,
+				u8 *wd_buf, u32 *wd_len);
+	u8 (*get_fwcmd_queue_idx)(void);
+	void (*cfg_dma_io)(struct hal_info_t *hal, u8 en);
+	void (*cfg_txdma)(struct hal_info_t *hal, u8 en, u8 dma_ch);
+	void (*cfg_txhci)(struct hal_info_t *hal, u8 en);
+	void (*cfg_rxhci)(struct hal_info_t *hal, u8 en);
+	void (*clr_rwptr)(struct hal_info_t *hal);
+	void (*rst_bdram)(struct hal_info_t *hal);
+	void (*cfg_rsvd_ctrl)(struct hal_info_t *hal);
+
+	enum rtw_hal_status(*handle_rx_buffer)(struct rtw_phl_com_t *phl_com,
+					       struct hal_info_t *hal,
+					       u8 *buf, u32 buf_size,
+					       struct rtw_phl_rx_pkt *rxpkt);
+#endif
+
+};
+
+#define hal_get_ops(_halinfo)	(&_halinfo->hal_ops)
+#define hal_get_regu_ops(_halops_) (&((_halops_)->regu_ops))
+
+struct hal_regu_ops {
+	u8 (*hal_query_group_cntry_num)(
+		struct rtw_regu_policy *policy, u8 group_id);
+
+	u8 (*hal_get_cntry_idx)(char *cntry);
+
+	u8 (*hal_get_cntry_tbl_size)(void);
+
+	u8 (*hal_get_chnlplan_ver)(void);
+
+	u8 (*hal_get_country_ver)(void);
+
+	u8 (*hal_get_domain_regulation)(u8 domain, u8 band);
+
+	u8 (*hal_get_domain_idx)(u8 domain, bool is_6g);
+
+	u8 (*hal_get_cat6g_by_country)(char *country);
+
+	void (*hal_get_6g_regulatory_info)(u8 domain, u8 *dm_code,
+		u8 *regulation, u8 *ch_idx);
+
+	void (*hal_qry_cntry_chnlplan)(
+		struct rtw_regulation_country_chplan *chplan, char *country);
+
+	void (*hal_get_chplan_update_info)(
+		u8 group, u8 did, void *info, enum band_type band);
+
+	void (*hal_fill_group_cntry_list)(
+		struct rtw_regu_policy *policy, char *list,
+		u32 group_size, u8 group_id);
+
+	void (*hal_get_chdef_6g)(
+		u8 ch_idx, struct chdef_6ghz *chdef);
+
+	void (*hal_get_regu_func_cert_info)(char *country,
+		struct rtw_regu_func_cert_info *rg_cert);
+};
+
+struct hal_ops_t {
+	/*** initialize section ***/
+	void (*read_chip_version)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal);
+	void (*init_hal_spec)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal);
+	void (*init_default_value)(struct hal_info_t *hal);
+	void (*init_int_default_value)(struct hal_info_t *hal, enum rtw_hal_int_set_opt opt);
+	u32 (*hal_hci_configure)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal,
+					struct rtw_ic_info *ic_info);
+
+	enum rtw_hal_status (*hal_get_efuse)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal);
+#ifdef CONFIG_PCI_HCI
+	enum rtw_hal_status (*hal_set_pcicfg)(struct hal_info_t *hal);
+#endif /* CONFIG_PCI_HCI */
+	enum rtw_hal_status (*hal_init)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal);
+	void (*hal_deinit)(struct rtw_phl_com_t *phl_com,
+			   struct hal_info_t *hal);
+	enum rtw_hal_status (*hal_start)(struct rtw_phl_com_t *phl_com,
+					 struct hal_info_t *hal);
+	enum rtw_hal_status (*hal_stop)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal);
+	enum rtw_hal_status (*hal_cfg_fw)(struct rtw_phl_com_t *phl_com,
+					  struct hal_info_t *hal,
+					  char *ic_name,
+					  enum rtw_fw_type fw_type);
+	enum rf_path (*get_path_from_ant_num)(u8 antnum);
+
+	enum rtw_hal_status (*hal_fast_start)(struct rtw_phl_com_t *phl_com,
+					     struct hal_info_t *hal);
+	enum rtw_hal_status (*hal_fast_stop)(struct rtw_phl_com_t *phl_com,
+					       struct hal_info_t *hal);
+#ifdef CONFIG_WOWLAN
+	enum rtw_hal_status (*hal_wow_init)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal, struct rtw_phl_stainfo_t *sta);
+	enum rtw_hal_status (*hal_wow_deinit)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal, struct rtw_phl_stainfo_t *sta);
+#endif /* CONFIG_WOWLAN */
+
+	/* MP */
+	enum rtw_hal_status (*hal_mp_init)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal);
+	enum rtw_hal_status (*hal_mp_deinit)(struct rtw_phl_com_t *phl_com,
+					struct hal_info_t *hal);
+	bool (*hal_mp_path_chk)(struct rtw_phl_com_t *phl_com,
+					u8 ant_tx,
+					u8 cur_phy);
+	/*IO ops*/
+	u32 (*read_macreg)(struct hal_info_t *hal,
+			u32 offset, u32 bit_mask);
+	void (*write_macreg)(struct hal_info_t *hal,
+			u32 offset, u32 bit_mask, u32 data);
+	u32 (*read_bbreg)(struct hal_info_t *hal,
+			u32 offset, u32 bit_mask);
+	void (*write_bbreg)(struct hal_info_t *hal,
+			u32 offset, u32 bit_mask, u32 data);
+	u32 (*read_rfreg)(struct hal_info_t *hal,
+			enum rf_path path, u32 offset, u32 bit_mask);
+	void (*write_rfreg)(struct hal_info_t *hal,
+			enum rf_path path, u32 offset, u32 bit_mask, u32 data);
+#ifdef RTW_WKARD_BUS_WRITE
+	enum rtw_hal_status (*write_reg_post_cfg)(struct hal_info_t *hal_info,
+						  u32 offset, u32 value);
+#endif
+
+	/*** interrupt hdl section ***/
+	void (*disable_interrupt_isr)(struct hal_info_t *hal);
+	void (*enable_interrupt)(struct hal_info_t *hal);
+	void (*disable_interrupt)(struct hal_info_t *hal);
+	void (*config_interrupt)(struct hal_info_t *hal, enum rtw_phl_config_int int_mode);
+	bool (*recognize_interrupt)(struct hal_info_t *hal);
+	bool (*recognize_halt_c2h_interrupt)(struct hal_info_t *hal);
+	void (*clear_interrupt)(struct hal_info_t *hal);
+	u32 (*interrupt_handler)(struct hal_info_t *hal);
+	void (*restore_interrupt)(struct hal_info_t *hal);
+	void (*restore_rx_interrupt)(struct hal_info_t *hal);
+#ifdef PHL_RXSC_ISR
+	enum rtw_hal_status (*check_rpq_isr)(u8 dma_ch, u32 rx_int_array);
+#endif
+
+	/* regu */
+	struct hal_regu_ops regu_ops;
+
+#ifdef RTW_PHL_BCN
+	enum rtw_hal_status (*cfg_bcn)(struct rtw_phl_com_t *phl_com,
+		struct hal_info_t *hal, struct rtw_bcn_entry *bcn_entry);
+	enum rtw_hal_status (*upt_bcn)(struct rtw_phl_com_t *phl_com,
+		struct hal_info_t *hal, struct rtw_bcn_entry *bcn_entry);
+#endif
+#ifdef CONFIG_PCI_HCI
+	enum rtw_hal_status (*get_pcicfg)(struct hal_info_t *hal,
+					  struct rtw_pcie_cfgspc_param *cfg);
+#endif
+
+#ifdef CONFIG_RTW_MULTI_DEV_MULTI_BAND
+	enum rtw_hal_status (*cfg_share_xstal)(struct hal_info_t *hal,
+					       struct rtw_phl_com_t *phl_com,
+					       bool is_share);
+#endif /* CONFIG_RTW_MULTI_DEV_MULTI_BAND */
+
+	enum rtw_hal_status (*cfg_ppdu_sts)(struct hal_info_t *hal,
+					    struct hal_ppdu_sts_cfg *cfg);
+};
+
+struct hal_info_t {
+	struct rtw_phl_com_t *phl_com;
+	struct rtw_hal_com_t *hal_com;
+	_os_atomic hal_mac_mem;
+
+	struct hal_trx_ops *trx_ops;
+	struct hal_ops_t hal_ops;
+#ifdef CONFIG_PCI_HCI
+	void *txch_map;
+#endif
+	void *rpr_cfg;
+
+	void *mac; /*halmac*/
+	void *bb;
+	void *rf;
+	void *btc;
+	void *efuse;
+	u8 monitor_mode[MAX_BAND_NUM]; /* default: 0 */
+};
+
+struct c2h_evt_msg {
+	union {
+		struct rtw_tx_pkt_rpt tx_rpt;
+		#ifdef CONFIG_PHL_TWT
+		struct rtw_phl_twt_wait_anno_rpt twt_anno_rpt;
+		#endif
+		#ifdef CONFIG_PHL_BCN_ERLY_RPT
+		struct rtw_bcn_early_rpt bcn_erly_rpt;
+		#endif
+	} u;
+};
+
+struct hal_c2h_hdl {
+	u8 cat;
+	u8 cls_min;
+	u8 cls_max;
+	u32 (*c2h_hdl)(void *hal, struct rtw_c2h_info *c2h, struct c2h_evt_msg *c2h_msg);
+	void (*c2h_buf_wb)(void *hal, struct rtw_c2h_info *c2h, u32 evt_id, struct c2h_evt_msg *c2h_msg);
+};
+
+
+#ifdef CONFIG_PHL_CHANNEL_INFO
+enum chinfo_ch_mode {
+	CH_INFO_LEGACY_CH = 0,
+	CH_INFO_MIMO_CH = 1,
+	CH_INFO_MAX,
+};
+
+struct chinfo_bbcr_cfg {
+	bool	ch_i_phy0_en;
+	bool	ch_i_phy1_en;
+	bool	ch_i_data_src;
+	bool	ch_i_cmprs;
+	u8	ch_i_grp_num_non_he;
+	u8	ch_i_grp_num_he;
+	u8	ch_i_blk_start_idx;
+	u8	ch_i_blk_end_idx;
+	u32	ch_i_ele_bitmap;
+	enum chinfo_ch_mode	ch_i_type;
+	u8	ch_i_seg_len;
+};
+
+/*sync from struct bb_ch_rpt_hdr_info */
+struct ch_rpt_hdr_info {
+	u16 total_len_l; /*header(16byte) + Raw data length(Unit: byte)*/
+	#if (PLATFOM_IS_LITTLE_ENDIAN)
+	u8 total_len_m:1;
+	u8 total_seg_num:7;
+	#else
+	u8 total_seg_num:7;
+	u8 total_len_m:1;
+	#endif
+
+	u8 avg_noise_pow;
+	#if (PLATFOM_IS_LITTLE_ENDIAN)
+	u8 is_pkt_end:1;
+	u8 set_valid:1;
+	u8 n_rx:3;
+	u8 n_sts:3;
+	#else
+	u8 n_sts:3;
+	u8 n_rx:3;
+	u8 set_valid:1;
+	u8 is_pkt_end:1;
+	#endif
+	u8 segment_size; /*unit (8Byte)*/
+	u8 sts0_evm;
+	u8 seq_num;
+};
+
+/*sync from struct bb_phy_info_rpt */
+struct phy_info_rpt {
+	u8	rssi[2];
+	u16	rsvd_0;
+	u8	rssi_avg;
+	#if (PLATFOM_IS_LITTLE_ENDIAN)
+	u8	rxsc:4;
+	u8	sts1_evm_l:4;
+	u8	sts1_evm_m:4;
+	u8	rsvd_1:4;
+	#else
+	u8	rsvd_1:4;
+	u8	sts1_evm_m:4;
+	u8	sts1_evm_l:4;
+	u8	rxsc:4;
+	#endif
+	u8	rsvd_2;
+};
+
+
+struct ch_info_drv_rpt {
+	u32 raw_data_len;
+	u8 seg_idx_curr;
+	bool get_ch_rpt_success;
+};
+
+#endif /* CONFIG_PHL_CHANNEL_INFO */
+#endif /*_HAL_STRUCT_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_thermal.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_thermal.c
new file mode 100644
index 000000000000..c4a1a7c18ce3
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_thermal.c
@@ -0,0 +1,130 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_THERMAL_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_THERMAL_PROTECT
+
+enum rtw_hal_status
+rtw_hal_thermal_protect_cfg_tx_ampdu(
+	void *hal,
+	struct rtw_phl_stainfo_t *sta,
+	u8 ratio)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	u8 tx_time = 0;
+	u16 num_ampdu = 0;
+
+	if (64 == sta->asoc_cap.num_ampdu)
+		tx_time = 0xA5;
+	else if (128 == sta->asoc_cap.num_ampdu)
+		tx_time = 0xA5;
+	else if (256 == sta->asoc_cap.num_ampdu)
+		tx_time = 0xA5;
+
+	if(sta->asoc_cap.num_ampdu_bk == 0)
+		sta->asoc_cap.num_ampdu_bk = sta->asoc_cap.num_ampdu;
+
+	sta->asoc_cap.num_ampdu = sta->asoc_cap.num_ampdu_bk * ratio / 100;
+	num_ampdu = sta->asoc_cap.num_ampdu;
+
+	if(num_ampdu == 0)
+		num_ampdu = 1;
+
+	hsts = rtw_hal_mac_set_hw_ampdu_cfg(hal_info, 0, num_ampdu, tx_time);
+
+	PHL_INFO("%s: bk_num_ampdu = %d num_ampdu = %d, tx_time = %x\n",
+		 __FUNCTION__, sta->asoc_cap.num_ampdu_bk, num_ampdu, tx_time);
+
+	if (RTW_HAL_STATUS_SUCCESS != hsts)
+		goto out;
+
+out:
+	return hsts;
+}
+
+bool rtw_hal_check_thermal_protect(
+	struct rtw_phl_com_t *phl_com,
+	void *hal
+)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum halrf_thermal_status status = HALRF_THERMAL_STATUS_BELOW_THRESHOLD;
+	bool action_changed = false;
+
+	status = rtw_hal_rf_get_ther_protected_threshold(hal_info);
+
+	PHL_INFO("%s: Cur action = %x\n", __FUNCTION__, phl_com->thermal_protect_action);
+	PHL_INFO("%s: status = %x\n", __FUNCTION__, status);
+
+	if (status == HALRF_THERMAL_STATUS_ABOVE_THRESHOLD) {
+		if (phl_com->thermal_protect_action < PHL_THERMAL_PROTECT_ACTION_LEVEL_MAX) {
+			if (phl_com->thermal_protect_action == PHL_THERMAL_PROTECT_ACTION_NONE)
+				phl_com->drv_mode = RTW_DRV_MODE_HIGH_THERMAL;
+			phl_com->thermal_protect_action++;
+			action_changed = true;
+		}
+	} else if (status == HALRF_THERMAL_STATUS_BELOW_THRESHOLD) {
+		if (phl_com->thermal_protect_action > PHL_THERMAL_PROTECT_ACTION_NONE) {
+			if (phl_com->thermal_protect_action == PHL_THERMAL_PROTECT_ACTION_LEVEL1)
+				phl_com->drv_mode = RTW_DRV_MODE_NORMAL;
+			phl_com->thermal_protect_action--;
+			action_changed = true;
+		}
+	} else {
+		/* Do nothing */
+	}
+	if(action_changed)
+		PHL_INFO("%s: Next action = %x\n", __FUNCTION__, phl_com->thermal_protect_action);
+	return action_changed;
+}
+
+#endif /* CONFIG_PHL_THERMAL_PROTECT */
+
+
+enum rtw_hal_status
+rtw_hal_thermal_protect_cfg_tx_duty(
+	void *hal,
+	u16 tx_duty_interval,
+	u8 ratio)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+	u16 pause_duration = 0, tx_duration = 0;
+
+	tx_duration = tx_duty_interval * ratio / 100;
+	pause_duration = tx_duty_interval - tx_duration;
+	PHL_INFO("%s: tx duty interval = %d tx duration = %d, pause duration = %d\n",
+		 __FUNCTION__, tx_duty_interval, tx_duration, pause_duration);
+
+	hsts = rtw_hal_mac_set_tx_duty(hal_info, pause_duration, tx_duration);
+
+	return hsts;
+}
+
+enum rtw_hal_status
+rtw_hal_thermal_protect_stop_tx_duty(
+	void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hsts = RTW_HAL_STATUS_FAILURE;
+
+	PHL_INFO("%s: Stop tx duty!\n", __FUNCTION__);
+
+	hsts = rtw_hal_mac_stop_tx_duty(hal_info);
+
+	return hsts;
+}
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_trx_mit.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_trx_mit.c
new file mode 100644
index 000000000000..1a831838c948
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_trx_mit.c
@@ -0,0 +1,118 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_TRX_MIT_C_
+#include "hal_headers.h"
+
+static void
+_hal_trx_mit_timer_convert(u32 timer, u8 *mul_ptr,
+			   enum mac_ax_trx_mitigation_timer_unit *unit_ptr)
+{
+	/* timer = mul * unit */
+
+	const enum mac_ax_trx_mitigation_timer_unit UNIT_ARR[] = {
+	    MAC_AX_MIT_64US, MAC_AX_MIT_128US, MAC_AX_MIT_256US,
+	    MAC_AX_MIT_512US};
+	const u8 UNIT_EXP_ARR[] = {6, 7, 8, 9}; /* 2^exp = unit */
+	const u8 UNIT_ARR_LEN = 4;
+
+	const u8 MUL_MAX = 0xff; /* 8 bits for mul */
+
+	u32 timer_ = 0;
+	u8 idx;
+
+	for (idx = 0; idx < UNIT_ARR_LEN; idx++) {
+
+		timer_ = timer >> UNIT_EXP_ARR[idx];
+
+		if (timer_ <= MUL_MAX)
+			break;
+	}
+
+	if (timer_ > MUL_MAX)
+		*mul_ptr = MUL_MAX;
+	else
+		*mul_ptr = (u8)timer_;
+
+	if (idx < UNIT_ARR_LEN)
+		*unit_ptr = UNIT_ARR[idx];
+	else
+		*unit_ptr = UNIT_ARR[UNIT_ARR_LEN - 1];
+}
+
+enum rtw_hal_status rtw_hal_pcie_trx_mit(void *hal, u32 tx_timer, u8 tx_counter,
+					 u32 rx_timer, u8 rx_counter)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	struct mac_ax_pcie_trx_mitigation mit_info;
+	struct mac_ax_txdma_ch_map txch_map;
+	struct mac_ax_rxdma_ch_map rxch_map;
+
+	/* tx */
+
+	txch_map.ch0 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch1 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch2 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch3 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch4 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch5 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch6 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch7 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch8 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch9 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch10 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch11 = MAC_AX_PCIE_DISABLE;
+	txch_map.ch12 = MAC_AX_PCIE_DISABLE;
+
+	mit_info.txch_map = &txch_map;
+	mit_info.tx_counter = tx_counter;
+
+	_hal_trx_mit_timer_convert(tx_timer, &(mit_info.tx_timer),
+				   &(mit_info.tx_timer_unit));
+
+	PHL_INFO(
+	    "%s :: mit_info.tx_timer == %d, mit_info.tx_timer_unit == %d\n",
+	    __func__, mit_info.tx_timer, mit_info.tx_timer_unit);
+
+	/* tx - END */
+
+	/* rx */
+
+	rxch_map.rxq = MAC_AX_PCIE_ENABLE;
+	rxch_map.rpq = MAC_AX_PCIE_ENABLE;
+
+	mit_info.rxch_map = &rxch_map;
+	mit_info.rx_counter = rx_counter;
+
+	_hal_trx_mit_timer_convert(rx_timer, &(mit_info.rx_timer),
+				   &(mit_info.rx_timer_unit));
+
+	PHL_INFO(
+	    "%s :: mit_info.rx_timer == %d, mit_info.rx_timer_unit == %d\n",
+	    __func__, mit_info.rx_timer, mit_info.rx_timer_unit);
+
+	/* rx - END */
+
+	if (rtw_hal_mac_pcie_trx_mit(hal_info, &mit_info) !=
+	    RTW_HAL_STATUS_SUCCESS) {
+
+		PHL_INFO(
+		    "%s :: failed to config pcie trx interrupt mitigation\n",
+		    __func__);
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_twt.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_twt.c
new file mode 100644
index 000000000000..b441413f8254
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_twt.c
@@ -0,0 +1,40 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_TWT_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_PHL_TWT
+enum rtw_hal_status
+rtw_hal_twt_info_update(void *hal,
+                        struct rtw_phl_twt_info twt_info,
+                        struct rtw_wifi_role_link_t *rlink,
+                        u8 action)
+{
+	return rtw_hal_mac_twt_info_update(hal, twt_info, rlink, action);
+}
+
+enum rtw_hal_status
+rtw_hal_twt_sta_update(void *hal, u8 macid, u8 twt_id, u8 action)
+{
+	return rtw_hal_mac_twt_sta_update(hal, macid, twt_id, action);
+}
+
+enum rtw_hal_status
+rtw_hal_twt_sta_announce(void *hal, u8 macid)
+{
+	return rtw_hal_mac_twt_sta_announce(hal, macid);
+}
+#endif /* CONFIG_PHL_TWT */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_tx.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_tx.c
new file mode 100644
index 000000000000..e670527f2315
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_tx.c
@@ -0,0 +1,517 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 - 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_TX_C_
+#include "hal_headers.h"
+/**
+ * this function will be used in read / write pointer mechanism and
+ * return the number of available read pointer
+ * @rptr: input, the read pointer
+ * @wptr: input, the write pointer
+ * @bndy: input, the boundary of read / write pointer mechanism
+ */
+u16 hal_calc_avail_rptr(u16 rptr, u16 wptr, u16 bndy)
+{
+	u16 avail_rptr = 0;
+
+	if (wptr >= rptr)
+		avail_rptr = wptr - rptr;
+	else if (rptr > wptr)
+		avail_rptr = wptr + (bndy - rptr);
+
+	return avail_rptr;
+}
+
+
+/**
+ * this function will be used in read / write pointer mechanism and
+ * return the number of available write pointer
+ * @rptr: input, the read pointer
+ * @wptr: input, the write pointer
+ * @bndy: input, the boundary of read / write pointer mechanism
+ */
+u16 hal_calc_avail_wptr(u16 rptr, u16 wptr, u16 bndy)
+{
+	u16 avail_wptr = 0;
+
+	if (rptr > wptr)
+		avail_wptr = rptr - wptr - 1;
+	else if (wptr >= rptr)
+		avail_wptr = rptr + (bndy - wptr) - 1;
+
+	return avail_wptr;
+}
+
+
+enum rtw_hal_status
+rtw_hal_enable_hwamsdu(void *hal,
+			   u8 enable,
+			   u8 max_num,
+			   u8 en_single_amsdu,
+			   u8 en_last_amsdu_padding)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_enable_hwamsdu(hal_info, enable, max_num, en_single_amsdu, en_last_amsdu_padding);
+}
+
+/**
+ * rtw_hal_tx_chnl_mapping - query hw tx dma channel mapping to the sw xmit ring
+ * identified by macid, tid and band
+ * @hal: see struct hal_info_t
+ * @macid: input target macid is 0 ~ 127
+ * @cat: input target packet category, see enum rtw_phl_ring_cat
+ * @band: input target band, 0 for band 0 / 1 for band 1
+ *
+ * returns the mapping hw tx dma channel
+ */
+u8 rtw_hal_tx_chnl_mapping(void *hal, u16 macid,
+			   enum rtw_phl_ring_cat cat, u8 band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u8 tx_chnl = 0;
+
+	tx_chnl = trx_ops->map_hw_tx_chnl(hal_info, macid, cat, band);
+
+	return tx_chnl;
+}
+
+/**
+ * rtw_hal_get_fwcmd_queue_idx - get idx of fwcmd queue
+ * @hal: see struct hal_info_t
+ *
+ * returns u8 idx of fwcmd queue
+ */
+u8 rtw_hal_get_fwcmd_queue_idx(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	return trx_ops->get_fwcmd_queue_idx();
+}
+
+void rtw_hal_clear_rwptr(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_clear_rwptr(hal_info))
+		PHL_ERR("%s failure \n", __func__);
+}
+
+void rtw_hal_cfg_txhci(void *hal, u8 en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s : enable %d.\n", __func__, en);
+
+	if (RTW_HAL_STATUS_SUCCESS != rtw_hal_mac_cfg_txhci(hal_info, en))
+		PHL_ERR("%s failure \n", __func__);
+}
+
+
+
+enum rtw_hal_status rtw_hal_chk_allq_empty(void *hal, u8 *empty)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	FUNCIN();
+
+	return rtw_hal_mac_chk_allq_empty(hal_info, empty);
+}
+
+enum rtw_hal_status rtw_hal_set_resp_ack_chk_cca(void *hal, u8 band, u8 en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	FUNCIN();
+
+	return rtw_hal_mac_set_resp_ack_chk_cca(hal_info, band, en);
+}
+
+enum rtw_hal_status rtw_hal_sifs_chk_cca_en(void *hal, u8 band, u8 en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	FUNCIN();
+
+	return rtw_hal_mac_sifs_chk_cca_en(hal_info, band, en);
+}
+
+
+enum rtw_hal_status
+rtw_hal_fill_txdesc(void *hal,
+                    struct rtw_xmit_req *treq,
+                    u8 *wd_buf,
+                    u32 *wd_len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_fill_txdesc(hal_info->mac, treq, wd_buf, wd_len);
+}
+
+
+enum rtw_hal_status
+rtw_hal_get_hwseq(void *hal, u16 macid, u8 ref_sel, u8 ssn_sel, u16 *hw_seq)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_SUCCESS;
+
+	sts = rtw_hal_mac_get_hwseq(hal, macid, ref_sel, ssn_sel, hw_seq);
+
+	return sts;
+}
+
+
+enum rtw_hal_status
+rtw_hal_poll_hw_tx_done(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_SUCCESS;
+
+	sts = rtw_hal_mac_poll_hw_tx_done(hal_info);
+
+	return sts;
+}
+
+enum rtw_hal_status
+rtw_hal_hw_tx_resume(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status sts = RTW_HAL_STATUS_SUCCESS;
+
+	sts = rtw_hal_mac_hw_tx_resume(hal_info);
+
+	return sts;
+}
+
+u8 rtw_hal_poll_txdma_idle(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status ret = RTW_HAL_STATUS_SUCCESS;
+
+	FUNCIN();
+
+	ret = rtw_hal_mac_poll_txdma_idle(hal_info);
+
+	return (ret == RTW_HAL_STATUS_SUCCESS) ? true : false;
+}
+
+#ifdef CONFIG_PCI_HCI
+/**
+ * rtw_hal_convert_qsel_to_tid - convert qsel to tid value
+ * @hal: see struct hal_info_t
+ * @qsel: HW queue selection
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+u8 rtw_hal_convert_qsel_to_tid(void *hal, u8 qsel_id, u8 tid_indic)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->qsel_to_tid(hal_info, qsel_id, tid_indic);
+
+	return hstatus;
+}
+/**
+ * rtw_hal_tx_res_query - query current HW tx resource with specifc dma channel
+ * @hal: see struct hal_info_t
+ * @dma_ch: the target dma channel
+ * @host_idx: current host index of this channel
+ * @hw_idx: current hw index of this channel
+ *
+ * this function returns the number of available tx resource
+ * NOTE, input host_idx and hw_idx ptr shall NOT be NULL
+ */
+u16 rtw_hal_tx_res_query(void *hal, u8 dma_ch, u16 *host_idx, u16 *hw_idx)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u16 res_num = 0;
+
+	sts = rtw_hal_mac_tx_res_query(hal_info, dma_ch, host_idx, hw_idx,
+	                               &res_num);
+
+	return res_num;
+}
+
+enum phl_band_idx rtw_hal_query_txch_hwband(void *hal, u8 dma_ch)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	enum phl_band_idx band_idx = HW_BAND_0;
+
+	band_idx = trx_ops->query_txch_hwband(dma_ch);
+
+	return band_idx;
+}
+
+/**
+ * rtw_hal_query_txch_num - query total hw tx dma channels number
+ *
+ * returns the number of  hw tx dma channel
+ */
+u8 rtw_hal_query_txch_num(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u8 ch_num = 0;
+
+	ch_num = trx_ops->query_txch_num();
+
+	return ch_num;
+}
+
+enum rtw_hal_status rtw_hal_trx_init(void *hal, u8 *txbd_buf, u8 *rxbd_buf)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->init(hal_info, txbd_buf, rxbd_buf);
+
+	return hstatus;
+}
+
+void rtw_hal_trx_deinit(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	trx_ops->deinit(hal_info);
+}
+
+/**
+ * rtw_hal_update_wd_page - update wd page for xmit packet
+ * @hal: see struct hal_info_t
+ * @phl_pkt_req: packet xmit request from phl, see struct rtw_phl_pkt_req
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+enum rtw_hal_status rtw_hal_update_wd_page(void *hal, void *phl_pkt_req)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->update_wd(hal_info, phl_pkt_req);
+
+	return hstatus;
+}
+
+/**
+ * rtw_hal_update_txbd - update tx bd for xmit packet
+ * @hal: see struct hal_info_t
+ * @wd: buffer pointer of wd page to fill in txbd
+ *
+ * returns enum RTW_HAL_STATUS
+ * NOTE, this function is PCIe specific function
+ */
+enum rtw_hal_status rtw_hal_update_txbd(void *hal, void *txbd, void *wd, u8 dma_ch, u16 wd_num)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->update_txbd(hal, txbd, wd, dma_ch, wd_num);
+
+	return hstatus;
+}
+
+
+/**
+ * rtw_hal_update_trigger_txstart - trigger hw to start tx
+ * @hal: see struct hal_info_t
+ * @txbd: the target txbd to update
+ * @dma_ch: the dma channel index of this txbd_ring
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+enum rtw_hal_status
+rtw_hal_trigger_txstart(void *hal, struct tx_base_desc *txbd, u8 dma_ch)
+{
+	enum rtw_hal_status sts = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	sts = rtw_hal_mac_trigger_txstart(hal_info, txbd, dma_ch);
+
+	if (sts != RTW_HAL_STATUS_SUCCESS) {
+		PHL_TRACE(COMP_PHL_XMIT, _PHL_INFO_, "%s : trigger tx start fail!\n",
+		          __func__);
+	}
+
+	return sts;
+}
+
+#endif /*CONFIG_PCI_HCI*/
+
+#ifdef CONFIG_USB_HCI
+enum rtw_hal_status rtw_hal_trx_init(void *hal)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->init(hal_info);
+
+	return hstatus;
+}
+
+void rtw_hal_trx_deinit(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	trx_ops->deinit(hal_info);
+}
+
+u8 rtw_hal_get_bulkout_id(void *hal, u8 dma_ch, u8 mode)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->get_bulkout_id(hal, dma_ch, mode);
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+	rtw_hal_usb_tx_agg_cfg(void *hal, u8* wd_buf, u8 agg_num)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	hstatus = trx_ops->usb_tx_agg_cfg(hal, wd_buf, agg_num);
+
+	return hstatus;
+}
+
+
+u8 rtw_hal_get_max_bulkout_wd_num(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+
+	return trx_ops->get_max_bulkout_wd_num(hal);
+}
+
+u16 rtw_hal_get_max_dma_txagg_msk(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return hal_mac_usb_get_max_dma_txagg_msk(hal_info);
+}
+
+u32 rtw_hal_get_wd_len(void *hal,
+				struct rtw_xmit_req *tx_req)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+	u32 wd_len = 0;
+
+	wd_len = trx_ops->hal_get_wd_len(hal_info, tx_req);
+
+	return wd_len;
+}
+
+#endif
+
+#ifdef CONFIG_SDIO_HCI
+void rtw_hal_sdio_tx_cfg(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+
+	rtw_hal_mac_sdio_tx_cfg(hal_info->hal_com);
+}
+
+enum rtw_hal_status rtw_hal_sdio_tx(void *hal, u8 dma_ch, u8 *buf, u32 buf_len,
+				    u8 agg_count, u16 *pkt_len, u8 *wp_offset)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u32 txaddr;
+	u32 txlen;
+	bool ready;
+
+
+	ready = rtw_hal_mac_sdio_check_tx_allow(hal_info->hal_com, dma_ch,
+						buf, buf_len, agg_count,
+						pkt_len, wp_offset, &txaddr,
+						&txlen);
+	if (!ready)
+		return RTW_HAL_STATUS_RESOURCE;
+
+	hal_sdio_cmd53_w(hal_info->hal_com, txaddr, txlen, buf);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+#endif /* CONFIG_SDIO_HCI */
+
+/**
+ * rtw_hal_fill_wd - fill wd-info and wd-boddy for xmit packet
+ * @hal: see struct hal_info_t
+ * @phl_pkt_req: packet xmit request from phl, see struct rtw_phl_pkt_req
+ *
+ * returns enum RTW_HAL_STATUS
+ */
+enum rtw_hal_status rtw_hal_fill_wd(void *hal,
+				struct rtw_xmit_req *tx_req,
+				u8 *wd_buf, u32 *wd_len)
+{
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_SUCCESS;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI)
+	struct hal_trx_ops *trx_ops = hal_info->trx_ops;
+#endif
+
+#ifdef RTW_WKARD_CCX_RPT_LIMIT_CTRL
+	if (tx_req->mdata.spe_rpt) {
+		if (tx_req->mdata.data_tx_cnt_lmt_en)
+			hal_info->hal_com->spe_pkt_cnt_lmt = tx_req->mdata.data_tx_cnt_lmt;
+	}
+#endif
+#if defined(CONFIG_USB_HCI) || defined(CONFIG_SDIO_HCI)
+	if (trx_ops->hal_fill_wd)
+		hstatus = trx_ops->hal_fill_wd(hal_info, tx_req, wd_buf, wd_len);
+	else
+#endif
+		hstatus = rtw_hal_mac_fill_txdesc(hal_info->mac, tx_req, wd_buf, wd_len);
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_cfg_hw_cts2self(void *hal, u8 band_sel, u8 enable,
+					    u8 non_sec_thr, u8 sec_thr)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_cts2self_cfg mac_hw_cts_cfg;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	if (enable)
+		mac_hw_cts_cfg.threshold_sel = MAC_AX_CTS2SELF_BOTH_THRESHOLD;
+	else
+		mac_hw_cts_cfg.threshold_sel = MAC_AX_CTS2SELF_DISABLE;
+
+	mac_hw_cts_cfg.band_sel = band_sel;
+	mac_hw_cts_cfg.non_sec_threshold = non_sec_thr;
+	mac_hw_cts_cfg.sec_threshold = sec_thr;
+
+	hstatus = rtw_hal_mac_cfg_hw_cts2slef(hal_info, &mac_hw_cts_cfg);
+
+	return hstatus;
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_tx.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_tx.h
new file mode 100644
index 000000000000..bc0391938469
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_tx.h
@@ -0,0 +1,45 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_TX_H_
+#define _HAL_TX_H_
+
+/**
+ * this function will be used in read / write pointer mechanism and
+ * return the number of available read pointer
+ * @rptr: input, the read pointer
+ * @wptr: input, the write pointer
+ * @bndy: input, the boundary of read / write pointer mechanism
+ */
+u16 hal_calc_avail_rptr(u16 rptr, u16 wptr, u16 bndy);
+
+/**
+ * this function will be used in read / write pointer mechanism and
+ * return the number of available write pointer
+ * @rptr: input, the read pointer
+ * @wptr: input, the write pointer
+ * @bndy: input, the boundary of read / write pointer mechanism
+ */
+u16 hal_calc_avail_wptr(u16 rptr, u16 wptr, u16 bndy);
+
+#ifdef CONFIG_PCI_HCI
+/**
+ * rtw_hal_query_txch_num - query total hw tx dma channels number
+ *
+ * returns the number of  hw tx dma channel
+ */
+u8 rtw_hal_query_txch_num(void *hal);
+#endif
+
+#endif /*_HAL_TX_H_*/
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_txpwr.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_txpwr.c
new file mode 100644
index 000000000000..261a0df0d88a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_txpwr.c
@@ -0,0 +1,368 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_TXPWR_C_
+#include "hal_headers.h"
+
+void rtw_hal_reset_txpwr_table(void *hal)
+{
+	struct hal_info_t *hal_info = hal;
+	struct rtw_phl_com_t *phl_com = hal_info->phl_com;
+	struct dev_cap_t *dev_cap = &phl_com->dev_cap;
+	struct phy_sw_cap_t *phy_sw_cap = &phl_com->phy_sw_cap[0];
+
+	struct txpwr_table_record {
+		struct rtw_para_pwrlmt_info_t *para_info;
+		u8 band_bmp;
+		enum phl_pwr_table pwr_table;
+	};
+
+	struct txpwr_table_record table[] = {
+		{&phy_sw_cap->rf_txpwrlmt_info, BAND_CAP_2G | BAND_CAP_5G, PWR_LIMIT},
+		{&phy_sw_cap->rf_txpwrlmt_ru_info, BAND_CAP_2G | BAND_CAP_5G, PWR_LIMIT_RU},
+		{&phy_sw_cap->rf_txpwrlmt_6g_info, BAND_CAP_6G, PWR_LIMIT},
+		{&phy_sw_cap->rf_txpwrlmt_ru_6g_info, BAND_CAP_6G, PWR_LIMIT_RU},
+		{NULL, 0, 0},
+	};
+	struct txpwr_table_record *r;
+
+	/* power limit tables */
+	for (r = table; r->para_info; r++) {
+		if (!(dev_cap->band_sup & r->band_bmp))
+			continue;
+#ifdef CONFIG_LOAD_PHY_PARA_FROM_FILE
+		if (r->para_info->para_src != RTW_PARA_SRC_INTNAL && r->para_info->loaded)
+			continue;
+#endif
+
+		rtw_hal_rf_clear_limit_table(hal_info, r->band_bmp, r->pwr_table);
+	}
+}
+
+int rtw_hal_get_pw_lmt_regu_type_from_str(void *hal, enum band_type band, const char *str)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	int regu;
+
+	regu = rtw_hal_rf_get_predef_pw_lmt_regu_type_from_str(band, str);
+	if (regu == -1) {
+		struct rtw_para_pwrlmt_info_t *para_info;
+
+		if (band == BAND_ON_6G)
+			para_info = &hal_info->phl_com->phy_sw_cap[HW_PHY_0].rf_txpwrlmt_6g_info;
+		else
+			para_info = &hal_info->phl_com->phy_sw_cap[HW_PHY_0].rf_txpwrlmt_info;
+
+		regu = rtw_hal_find_ext_regd_num(para_info, str);
+	}
+
+	return regu;
+}
+
+const char *rtw_hal_get_pw_lmt_regu_str_from_type(void *hal, enum band_type band, u8 regu)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 predef_str_num;
+	const char * const *predef_str_array =
+		rtw_hal_rf_get_predef_pw_lmt_regu_type_str_array(band, &predef_str_num);
+
+	if (predef_str_array == NULL || predef_str_num == 0)
+		return NULL;
+
+	if (regu >= predef_str_num) {
+		struct rtw_para_pwrlmt_info_t *para_info;
+
+		if (band == BAND_ON_6G)
+			para_info = &hal_info->phl_com->phy_sw_cap[HW_PHY_0].rf_txpwrlmt_6g_info;
+		else
+			para_info = &hal_info->phl_com->phy_sw_cap[HW_PHY_0].rf_txpwrlmt_info;
+
+		return rtw_hal_get_ext_regd_name(para_info, regu);
+	} else
+		return predef_str_array[regu];
+}
+
+u8 rtw_hal_get_pw_lmt_regu_type(void *hal, enum band_type band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_pw_lmt_regu_type(hal_info, band);
+}
+
+const char *rtw_hal_get_pw_lmt_regu_type_str(void *hal, enum band_type band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_pw_lmt_regu_type_str(hal_info, band);
+}
+
+bool rtw_hal_pw_lmt_regu_tbl_exist(void *hal, enum band_type band, u8 regu)
+{
+	struct hal_info_t *hal_info = hal;
+
+	return rtw_hal_rf_pw_lmt_regu_tbl_exist(hal_info, band, regu);
+}
+
+u8 rtw_hal_ext_reg_codemap_search(void *hal, enum band_type band
+	, u16 domain_code, const char *country, const char **reg_name)
+{
+	struct hal_info_t *hal_info = hal;
+	int aidx = rtw_hal_rf_file_regd_ext_search(hal_info, band, domain_code, country);
+
+	if (aidx != -1) {
+		struct rtw_para_pwrlmt_info_t *para_info;
+		struct pwrlmt_ext_regd_map *regd_codemap;
+		u8 ret = 0;
+
+		if (band == BAND_ON_6G)
+			para_info = &hal_info->phl_com->phy_sw_cap[HW_PHY_0].rf_txpwrlmt_6g_info;
+		else
+			para_info = &hal_info->phl_com->phy_sw_cap[HW_PHY_0].rf_txpwrlmt_info;
+		regd_codemap = (struct pwrlmt_ext_regd_map *)para_info->ext_reg_codemap;
+
+		if (regd_codemap && para_info->ext_reg_map_num > aidx) {
+			*reg_name = regd_codemap[aidx].reg_name;
+
+			if (regd_codemap[aidx].domain == domain_code)
+				ret |= RTW_PHL_EXT_REG_MATCH_DOMAIN;
+			if (country && regd_codemap[aidx].country[0] == country[0]
+				&& regd_codemap[aidx].country[1] == country[1])
+				ret |= RTW_PHL_EXT_REG_MATCH_COUNTRY;
+
+			_os_warn_on(ret == 0);
+			return ret;
+		}
+	}
+
+	return RTW_PHL_EXT_REG_MATCH_NONE;
+}
+
+bool rtw_hal_get_pwr_lmt_en(void *hal, u8 band_idx)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_mac_get_pwr_lmt_en_val(hal_info->hal_com, band_idx);
+}
+
+void rtw_hal_auto_pw_lmt_regu(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_rf_auto_pw_lmt_regu(hal_info);
+}
+
+void rtw_hal_force_pw_lmt_regu(void *hal,
+	u8 regu_2g[], u8 regu_2g_len, u8 regu_5g[], u8 regu_5g_len, u8 regu_6g[], u8 regu_6g_len)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	rtw_hal_rf_force_pw_lmt_regu(hal_info, regu_2g, regu_2g_len
+		, regu_5g, regu_5g_len, regu_6g, regu_6g_len);
+}
+
+u16 rtw_hal_get_pwr_constraint(void *hal, u8 band_idx)
+{
+	struct hal_info_t *hal_info = hal;
+	struct rtw_tpu_info *tpu;
+
+	if (band_idx >= MAX_BAND_NUM) {
+		_os_warn_on(1);
+		return 0;
+	}
+
+	tpu = &hal_info->hal_com->band[band_idx].rtw_tpu_i;
+	return tpu->pwr_constraint_mb;
+}
+
+enum rtw_hal_status rtw_hal_set_pwr_constraint(void *hal, u8 band_idx, u16 mb)
+{
+	struct hal_info_t *hal_info = hal;
+	struct rtw_tpu_info *tpu;
+
+	if (band_idx >= MAX_BAND_NUM)
+		return RTW_HAL_STATUS_FAILURE;
+
+	tpu = &hal_info->hal_com->band[band_idx].rtw_tpu_i;
+
+	if (tpu->pwr_constraint_mb != mb) {
+		enum phl_phy_idx phy_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+
+		/* software configuration only, no need to check for hwband ready */
+		if (rtw_hal_rf_set_power_constraint(hal_info, phy_idx, mb) ==  RTW_HAL_STATUS_SUCCESS) {
+			tpu->pwr_constraint_mb = mb;
+			return RTW_HAL_STATUS_SUCCESS;
+		}
+		return RTW_HAL_STATUS_FAILURE;
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_set_tx_power(void *hal, u8 band_idx,
+					enum phl_pwr_table pwr_table)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	if (hal_info->hal_com->dbcc_en || band_idx == HW_BAND_0) {
+		enum phl_phy_idx phy_idx = rtw_hal_hw_band_to_phy_idx(band_idx);
+
+		return rtw_hal_rf_set_power(hal_info, phy_idx, pwr_table);
+	}
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_update_tas_def_setting(void *hal, u32 tas_config)
+{
+	rtw_hal_rf_update_tas_def_setting(hal, tas_config);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_tas_en(void *hal, u8 en)
+{
+	rtw_hal_rf_tas_en(hal, en);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_get_txinfo_power(void *hal,
+					s16 *txinfo_power_dbm)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_SUCCESS;
+	s16 power_dbm = 0;
+
+	hal_status = rtw_hal_bb_get_txinfo_power(hal_info, &power_dbm);
+	*txinfo_power_dbm = power_dbm;
+
+	return hal_status;
+}
+
+u8 rtw_hal_get_tx_tbl_to_pwr_times(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_tx_tbl_to_pwr_times(hal_info);
+}
+
+void rtw_hal_set_tx_pwr_comp(void *hal, enum phl_phy_idx phy,
+			struct rtw_phl_regu_dyn_ant_gain *dyn_ag, enum band_type band,
+			const char *sregulation)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	int regulation = -1, i = 0;
+
+	regulation = rtw_hal_rf_get_predef_pw_lmt_regu_type_from_str(band,
+		sregulation);
+	PHL_INFO("regulation country: %s (%d)\n", (char *)sregulation, regulation);
+	if (regulation == -1) {
+		PHL_INFO("unknown regulation!!\n");
+		return;
+	}
+
+	if (band == BAND_ON_24G) {
+		dyn_ag->regu_country_2g = (u8)regulation;
+		PHL_INFO("2g regulation = %d\n", (u8)regulation);
+	} else if (band == BAND_ON_5G){
+		dyn_ag->regu_country_5g = (u8)regulation;
+		PHL_INFO("5g regulation = %d\n", (u8)regulation);
+	} else if (band == BAND_ON_6G){
+		dyn_ag->regu_country_6g = (u8)regulation;
+		PHL_INFO("6g regulation = %d\n", (u8)regulation);
+	} else {
+		PHL_INFO("Unknow Band!!\n");
+		return;
+	}
+
+	for (i = 0; i < RTK_DAG_ANT_CNT; i++) {
+		PHL_INFO("compensation RF_PATH_%d => regulation: %s (%d)  "\
+				  "2_4g=%d, "\
+				  "5g-1=%d, 5g-2=%d, 5g-3=%d, 5g-4=%d, "\
+				  "6g-1=%d, 6g-2=%d, 6g-3=%d, 6g-4=%d, 6g-5=%d, 6g-6=%d\n",
+				 i, (char *)sregulation, (u8)regulation,
+				 dyn_ag->ag_comp[i].ag_2g_comp,
+				 dyn_ag->ag_comp[i].ag_5g_comp[0],
+				 dyn_ag->ag_comp[i].ag_5g_comp[1],
+				 dyn_ag->ag_comp[i].ag_5g_comp[2],
+				 dyn_ag->ag_comp[i].ag_5g_comp[3],
+				 dyn_ag->ag_comp[i].ag_6g_comp[0],
+				 dyn_ag->ag_comp[i].ag_6g_comp[1],
+				 dyn_ag->ag_comp[i].ag_6g_comp[2],
+				 dyn_ag->ag_comp[i].ag_6g_comp[3],
+				 dyn_ag->ag_comp[i].ag_6g_comp[4],
+				 dyn_ag->ag_comp[i].ag_6g_comp[5]);
+	}
+
+	rtw_hal_rf_set_tx_pwr_comp(hal_info, phy, dyn_ag);
+}
+
+s8 rtw_hal_get_power_by_rate_band(void *hal, u8 band_idx, u16 rate, u8 dcm, u8 offset, u32 band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_power_by_rate_band(hal_info, band_idx, rate, dcm, offset, band);
+}
+
+s8 rtw_hal_get_power_limit_option(void *hal, u8 band_idx, u8 rf_path, u16 rate,
+	u8 bandwidth, u8 beamforming, u8 tx_num, u8 channel, u32 band, u8 reg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_power_limit_option(hal_info
+		, rtw_hal_hw_band_to_phy_idx(band_idx), rf_path, rate, bandwidth
+		, beamforming, tx_num, channel, band, reg);
+}
+
+s8 rtw_hal_get_power_limit_ru_option(void *hal,
+	u8 band_idx, u8 rf_path, u16 rate, u8 bandwidth,
+	u8 tx_num, u8 channel, u32 band, u8 reg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_power_limit_ru_option(hal_info
+		, rtw_hal_hw_band_to_phy_idx(band_idx), rf_path, rate, bandwidth
+		, tx_num, channel, band, reg);
+}
+
+u8 rtw_hal_get_tx_tbl_to_tx_pwr_times(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_tx_tbl_to_tx_pwr_times(hal_info);
+}
+
+s8 rtw_hal_get_power_limit_value_ww(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_power_limit_value_ww(hal_info);
+}
+
+s8 rtw_hal_get_power_limit_value_na(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_power_limit_value_na(hal_info);
+}
+
+u32 rtw_hal_get_regulation_max_num(void *hal, enum band_type band)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	return rtw_hal_rf_get_regulation_max_num(hal_info, band);
+}
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_txpwr.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_txpwr.h
new file mode 100644
index 000000000000..c913cd35f8e0
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_txpwr.h
@@ -0,0 +1,55 @@
+/******************************************************************************
+ *
+ * Copyright(c)2021 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_TXPWR_H_
+#define _HAL_TXPWR_H_
+
+void rtw_hal_reset_txpwr_table(void *hal);
+
+int rtw_hal_get_pw_lmt_regu_type_from_str(void *hal, enum band_type band, const char *str);
+const char *rtw_hal_get_pw_lmt_regu_str_from_type(void *hal, enum band_type band, u8 regu);
+
+u8 rtw_hal_get_pw_lmt_regu_type(void *hal, enum band_type band);
+const char *rtw_hal_get_pw_lmt_regu_type_str(void *hal, enum band_type band);
+
+bool rtw_hal_pw_lmt_regu_tbl_exist(void *hal, enum band_type band, u8 regu);
+u8 rtw_hal_ext_reg_codemap_search(void *hal, enum band_type band
+	, u16 domain_code, const char *country, const char **reg_name);
+
+bool rtw_hal_get_pwr_lmt_en(void *hal, u8 band_idx);
+
+void rtw_hal_auto_pw_lmt_regu(void *hal);
+void rtw_hal_force_pw_lmt_regu(void *hal,
+	u8 regu_2g[], u8 regu_2g_len, u8 regu_5g[], u8 regu_5g_len, u8 regu_6g[], u8 regu_6g_len);
+
+u16 rtw_hal_get_pwr_constraint(void *hal, u8 band_idx);
+enum rtw_hal_status rtw_hal_set_pwr_constraint(void *hal, u8 band_idx, u16 mb);
+
+enum rtw_hal_status rtw_hal_set_tx_power(void *hal, u8 band_idx,
+					enum phl_pwr_table pwr_table);
+
+enum rtw_hal_status rtw_hal_get_txinfo_power(void *hal,
+					s16 *txinfo_power_dbm);
+
+s8 rtw_hal_get_power_by_rate_band(void *hal, u8 band_idx, u16 rate, u8 dcm, u8 offset, u32 band);
+s8 rtw_hal_get_power_limit_option(void *hal, u8 band_idx, u8 rf_path, u16 rate,
+	u8 bandwidth, u8 beamforming, u8 tx_num, u8 channel, u32 band, u8 reg);
+s8 rtw_hal_get_power_limit_ru_option(void *hal,
+	u8 band_idx, u8 rf_path, u16 rate, u8 bandwidth,
+	u8 tx_num, u8 channel, u32 band, u8 reg);
+u8 rtw_hal_get_tx_tbl_to_tx_pwr_times(void *hal);
+s8 rtw_hal_get_power_limit_value_ww(void *hal);
+s8 rtw_hal_get_power_limit_value_na(void *hal);
+u32 rtw_hal_get_regulation_max_num(void *hal, enum band_type band);
+#endif
\ No newline at end of file
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_wow.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_wow.c
new file mode 100644
index 000000000000..2682043004ae
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_wow.c
@@ -0,0 +1,612 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#define _HAL_WOW_C_
+#include "hal_headers.h"
+
+#ifdef CONFIG_WOWLAN
+
+#define MAX_POLLING_WOW_FW_STS_CNT 1000 /* 50ms */
+
+static enum rtw_hal_status _hal_check_wow_fw_ready(void *hal, u8 func_en)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_FAILURE;
+	void* drv_priv = hal_to_drvpriv(hal_info);
+	u8 fw_status = 0;
+	u32 poll_cnt = 0;
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s : start polling... (func_en %d)\n", __func__, func_en);
+
+	/* polling fw status */
+	while (1) {
+
+		if (poll_cnt >= MAX_POLLING_WOW_FW_STS_CNT) {
+			PHL_ERR("%s polling fw status timeout !!!\n", __func__);
+			hstats = RTW_HAL_STATUS_FAILURE;
+			break;
+		}
+
+		hstats = rtw_hal_mac_get_wow_fw_status(hal_info, &fw_status,
+							func_en);
+
+		if (RTW_HAL_STATUS_SUCCESS != hstats) {
+
+			PHL_ERR("%s : status %u\n", __func__, fw_status);
+			break;
+		}
+
+		if (fw_status) {
+			hstats = RTW_HAL_STATUS_SUCCESS;
+
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_,
+				"%s : polling count %u\n", __func__, poll_cnt);
+			break;
+		} else {
+
+			_os_delay_us(drv_priv, 50);
+
+		}
+		poll_cnt++;
+
+	}
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s : finish polling... (func_en %d)\n", __func__, func_en);
+
+	return hstats;
+}
+
+enum rtw_hal_status rtw_hal_get_wake_rsn(void *hal, enum rtw_mac_wow_wake_reason *wake_rsn, u8 *reset)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_FAILURE;
+	u8 mac_rsn = 0;
+
+	hstats = rtw_hal_mac_get_wake_rsn(hal_info, &mac_rsn, reset);
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : wake rsn from mac in hex 0x%x (reset %d).\n",
+				__func__, mac_rsn, *reset);
+
+	if (RTW_HAL_STATUS_SUCCESS != hstats)
+		PHL_ERR("%s : rtw_hal_mac_get_wake_rsn failed.\n", __func__);
+	else
+		*wake_rsn = mac_rsn;
+
+	return hstats;
+}
+
+enum rtw_hal_status rtw_hal_cfg_wow_sleep(void *hal, u8 sleep)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstats = RTW_HAL_STATUS_FAILURE;
+
+	hstats = rtw_hal_mac_cfg_wow_sleep(hal_info, sleep);
+
+	if (RTW_HAL_STATUS_SUCCESS != hstats)
+		PHL_ERR("%s : sleep %u\n", __func__, sleep);
+
+	return hstats;
+}
+
+enum rtw_hal_status rtw_hal_get_wow_aoac_rpt(void *hal, struct rtw_aoac_report *aoac_info, u8 rx_ready)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	FUNCIN();
+
+	/* get aoac report */
+	hstatus = rtw_hal_mac_get_aoac_rpt(hal_info, aoac_info, rx_ready);
+
+	if (RTW_HAL_STATUS_SUCCESS != hstatus)
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s(): failed with status(%u)\n", __func__, hstatus);
+
+	FUNCOUT();
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_wow_cfg_txdma(void *hal, u8 state)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	PHL_TRACE(COMP_PHL_DBG, _PHL_DEBUG_, "%s : state %d.\n", __func__, state);
+
+	rtw_hal_mac_cfg_txdma(hal_info, state);
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status
+rtw_hal_wow_cfg_nlo(void *hal, enum SCAN_OFLD_OP op, u16 mac_id,
+                    u8 hw_band, u8 hw_port, struct rtw_nlo_info *cfg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	struct scan_ofld_info info = {0};
+	u64 tsf;
+	u32 tsf_h = 0, tsf_l = 0;
+
+	info.operation = op;
+
+	if (op == SCAN_OFLD_OP_RPT || op == SCAN_OFLD_OP_STOP)
+		goto end;
+
+	if (cfg->delay != 0) {
+		hstatus = rtw_hal_mac_get_tsf(hal_info, hw_band, hw_port, &tsf_h, &tsf_l);
+		if (hstatus == RTW_HAL_STATUS_SUCCESS) {
+			tsf = ((u64)tsf_h << 32) | tsf_l;
+			tsf += cfg->delay * 1000;
+			tsf_h = tsf >> 32;
+			tsf_l = (u32)tsf;
+			info.tsf_high = tsf_h;
+			info.tsf_low = tsf_l;
+		} else {
+			PHL_WARN("%s Getting TSF failed \n", __func__);
+		}
+
+	}
+	/* configure scan offload */
+	info.period = cfg->period;
+	info.slow_period = cfg->slow_period;
+	info.cycle = cfg->cycle;
+
+	if (info.period == 0 && info.slow_period == 0) {
+		info.mode = SCAN_OFLD_MD_ONCE;
+	} else if (info.period == 0) {
+		info.mode = SCAN_OFLD_MD_SEEMLESS;
+	} else if (info.slow_period == 0) {
+		info.mode = SCAN_OFLD_MD_PD;
+	} else {
+		info.mode = SCAN_OFLD_MD_PD_SLOW;
+	}
+
+end:
+#ifdef CONFIG_PHL_SCANOFLD
+	hstatus = rtw_hal_mac_scan_ofld(hal_info, (u8)mac_id,
+				hw_band, hw_port, &info);
+#endif
+	if (hstatus != RTW_HAL_STATUS_SUCCESS)
+		return hstatus;
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_wow_cfg_nlo_chnl_list(void *hal, struct rtw_nlo_info *cfg)
+{
+#ifdef CONFIG_PHL_SCANOFLD
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+#endif
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+	u8 i = 0;
+
+	if (cfg->channel_num == 0)
+		return RTW_HAL_STATUS_FAILURE;
+
+	for (i = 0; i < cfg->channel_num; i++) {
+		/* set channel to mac */
+		cfg->channel_list[i].probe_req_id = cfg->probe_req_id;
+#ifdef CONFIG_PHL_SCANOFLD
+		hstatus = rtw_hal_mac_scan_ofld_add_ch(hal_info, HW_BAND_0,
+						&cfg->channel_list[i],
+						i == (cfg->channel_num-1) ? true : false);
+#endif
+		if (hstatus != RTW_HAL_STATUS_SUCCESS)
+			return hstatus;
+	}
+
+	return hstatus;
+}
+
+#ifdef CONFIG_PHL_WOW_APF
+enum rtw_hal_status
+rtw_hal_wow_access_apf(void *hal, u16 mac_id, struct rtw_apf_info *cfg, u8 set) {
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	struct mac_ax_adapter *mac = hal_to_mac(hal_info);
+	struct mac_ax_ops *hal_mac_ops = mac->ops;
+	struct rtw_hal_mac_apf info = {0};
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	hstatus = rtw_hal_mac_cfg_apf_ofld(hal_info, mac_id, cfg, set);
+
+	return hstatus;
+}
+#endif /* CONFIG_PHL_WOW_APF */
+
+enum rtw_hal_status rtw_hal_wow_init(struct rtw_phl_com_t *phl_com, void *hal,
+                                     struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+	FUNCIN_WSTS(hal_status);
+
+	/* download wowlan fw and do related tasks needed after redownload fw */
+	hal_status = hal_ops->hal_wow_init(phl_com, hal_info, sta);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		return hal_status;
+
+	if (sta->rlink->mstate == MLME_LINKED)
+		rtw_hal_rf_fwredl_config(hal_info, rtw_hal_hw_band_to_phy_idx(sta->rlink->hw_band));
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s successfully done.\n", __func__);
+
+	FUNCOUT_WSTS(hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_wow_deinit(struct rtw_phl_com_t *phl_com, void *hal,
+                                       struct rtw_phl_stainfo_t *sta)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_ops_t *hal_ops = hal_get_ops(hal_info);
+
+	FUNCIN_WSTS(hal_status);
+
+	/* download wowlan fw and do related tasks needed after redownload fw */
+	hal_status = hal_ops->hal_wow_deinit(phl_com, hal_info, sta);
+	if (hal_status != RTW_HAL_STATUS_SUCCESS)
+		return hal_status;
+
+	/* recover fw log settings after wowlan resume redl fw */
+	rtw_hal_fw_recover_log_cfg(hal);
+
+	if (sta->rlink->mstate == MLME_LINKED)
+		rtw_hal_rf_fwredl_config(hal_info, rtw_hal_hw_band_to_phy_idx(sta->rlink->hw_band));
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s successfully done.\n", __func__);
+
+	FUNCOUT_WSTS(hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status
+rtw_hal_wow_func_en(struct rtw_phl_com_t *phl_com, void *hal, u16 macid,
+                    struct rtw_hal_wow_cfg *cfg)
+{
+
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	FUNCIN();
+
+	do {
+		/* config wow cam : pattern match */
+#ifndef RTW_WKARD_WOW_SKIP_WOW_CAM_CONFIG
+		hstatus = rtw_hal_mac_cfg_wow_cam(hal_info, macid, true,
+		                                  cfg->pattern_match_info);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			break;
+#endif
+		/* gtk offload */
+		if (cfg->gtk_ofld_cfg->gtk_en) {
+			hstatus = rtw_hal_mac_cfg_gtk_ofld(hal_info, macid, true,
+			                                   cfg->gtk_ofld_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+		/* arp offload */
+		if (cfg->arp_ofld_cfg->arp_en) {
+			hstatus = rtw_hal_mac_cfg_arp_ofld(hal_info, macid, true,
+			                                   cfg->arp_ofld_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+		/* ndp offload */
+		if (cfg->ndp_ofld_cfg->ndp_en) {
+			hstatus = rtw_hal_mac_cfg_ndp_ofld(hal_info, macid, true,
+			                                   cfg->ndp_ofld_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+		/* config keep alive */
+		if (cfg->keep_alive_cfg->keep_alive_en) {
+			hstatus = rtw_hal_mac_cfg_keep_alive(hal_info, macid, true,
+			                                     cfg->keep_alive_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+		/* config disconnect detection */
+		if (cfg->disc_det_cfg->disc_det_en) {
+			hstatus = rtw_hal_mac_cfg_disc_dec(hal_info, macid, true,
+			                                   cfg->disc_det_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+		/* realwow offload */
+		if (cfg->realwow_cfg->realwow_en) {
+			hstatus = rtw_hal_mac_cfg_realwow(hal_info, macid, true,
+			                                  cfg->realwow_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+		/* config gpio */
+		if (cfg->wow_gpio->d2h_gpio_info.dev2hst_gpio_en) {
+			hstatus = rtw_hal_mac_cfg_dev2hst_gpio(hal_info, true, cfg->wow_gpio);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+		/* network list offload */
+		if (cfg->nlo_cfg->nlo_en) {
+			hstatus = rtw_hal_mac_cfg_nlo(hal_info, macid, true, cfg->nlo_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+
+		/* config period wake */
+		if (cfg->periodic_wake_cfg->periodic_wake_en) {
+			hstatus = rtw_hal_mac_cfg_periodic_wake(hal_info, macid, true,
+								cfg->periodic_wake_cfg);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+#ifdef CONFIG_PHL_MDNS_OFFLOAD
+		/* mDNS offload */
+		if (cfg->mdns_ofld_info->mdns_en) {
+			hstatus = rtw_hal_mac_cfg_mdns_ofld(hal_info, macid, true,
+			                                    cfg->mdns_ofld_info);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+#endif
+#ifdef CONFIG_PHL_WOW_APF
+		if (cfg->apf_info->apf_en) {
+			hstatus = rtw_hal_mac_cfg_apf_ofld(hal_info, macid, cfg->apf_info, 1);
+			if (RTW_HAL_STATUS_SUCCESS != hstatus)
+				break;
+		}
+#endif
+	} while(0);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s status(%u).\n", __func__, hstatus);
+
+	FUNCOUT();
+
+	return hstatus;
+}
+
+enum rtw_hal_status
+rtw_hal_wow_func_dis(struct rtw_phl_com_t *phl_com, void *hal, u16 macid,
+                     struct rtw_hal_wow_cfg *cfg)
+{
+
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	FUNCIN();
+
+	/* cancel pattern match */
+#ifndef RTW_WKARD_WOW_SKIP_WOW_CAM_CONFIG
+	hstatus = rtw_hal_mac_cfg_wow_cam(hal_info, macid, false, NULL);
+	if (RTW_HAL_STATUS_SUCCESS != hstatus)
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_wow_cam failed \n");
+#endif
+	/* cancel gtk offload */
+	if (cfg->gtk_ofld_cfg->gtk_en) {
+		hstatus = rtw_hal_mac_cfg_gtk_ofld(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_gtk_ofld failed \n");
+	}
+	/* cancel arp offload */
+	if (cfg->arp_ofld_cfg->arp_en) {
+		hstatus = rtw_hal_mac_cfg_arp_ofld(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_arp_ofld failed \n");
+	}
+	/* cancel ndp offload */
+	if (cfg->ndp_ofld_cfg->ndp_en) {
+		hstatus = rtw_hal_mac_cfg_ndp_ofld(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_ndp_ofld failed \n");
+	}
+	/* disable keep alive */
+	if (cfg->keep_alive_cfg->keep_alive_en) {
+		hstatus = rtw_hal_mac_cfg_keep_alive(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_keep_alive failed \n");
+	}
+	/* disable disconect detection */
+	if (cfg->disc_det_cfg->disc_det_en) {
+		hstatus = rtw_hal_mac_cfg_disc_dec(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_disc_dec failed \n");
+	}
+	/* realwow offload */
+	if (cfg->realwow_cfg->realwow_en) {
+		hstatus = rtw_hal_mac_cfg_realwow(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_realwow_ofld failed \n");
+	}
+	/* config gpio */
+	if (cfg->wow_gpio->d2h_gpio_info.dev2hst_gpio_en) {
+		hstatus = rtw_hal_mac_cfg_dev2hst_gpio(hal_info, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_dev2hst_gpio failed \n");
+	}
+	/* nlo */
+	if (cfg->nlo_cfg->nlo_en) {
+		hstatus = rtw_hal_mac_cfg_nlo(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_nlo failed \n");
+	}
+	/* config period wake */
+	if (cfg->periodic_wake_cfg->periodic_wake_en) {
+		hstatus = rtw_hal_mac_cfg_periodic_wake(hal_info, macid, false, NULL);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_periodic_wake failed \n");
+	}
+
+#ifdef CONFIG_PHL_MDNS_OFFLOAD
+	/* mDNS offload */
+	if (cfg->mdns_ofld_info->mdns_en) {
+		hstatus = rtw_hal_mac_cfg_mdns_ofld(hal_info, macid, false,
+		                                    cfg->mdns_ofld_info);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_mdns_ofld failed \n");
+	}
+#endif
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s status(%u).\n", __func__, hstatus);
+
+	FUNCOUT();
+
+	return RTW_HAL_STATUS_SUCCESS;
+}
+
+enum rtw_hal_status rtw_hal_wow_func_start(struct rtw_phl_com_t *phl_com, void *hal, u16 macid, struct rtw_hal_wow_cfg *cfg)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	do {
+		hstatus = rtw_hal_mac_cfg_wow_wake(hal_info, macid, true, cfg->wow_wake_cfg);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			break;
+		/* poll fw status */
+		hstatus = _hal_check_wow_fw_ready(hal_info, 1);
+		if (RTW_HAL_STATUS_SUCCESS != hstatus)
+			break;
+	} while(0);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s status(%u).\n", __func__, hstatus);
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_wow_func_stop(struct rtw_phl_com_t *phl_com, void *hal, u16 macid)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hstatus = RTW_HAL_STATUS_FAILURE;
+
+	/* config wow ctrl */
+	hstatus = rtw_hal_mac_cfg_wow_wake(hal_info, macid, false, NULL);
+	if (RTW_HAL_STATUS_SUCCESS != hstatus)
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] rtw_hal_mac_cfg_wow_wake failed \n");
+
+	hstatus = _hal_check_wow_fw_ready(hal_info, 0);
+	if (RTW_HAL_STATUS_SUCCESS != hstatus)
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] _hal_poll_wow_fw_status failed \n");
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s status(%u).\n", __func__, hstatus);
+
+	return hstatus;
+}
+
+enum rtw_hal_status rtw_hal_set_wowlan(struct rtw_phl_com_t *phl_com, void *hal,
+				       enum mac_ax_wow_ctrl ctrl)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	hal_status = rtw_hal_mac_set_wowlan(hal_info, ctrl);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : status(%u).\n", __func__, hal_status);
+
+	return hal_status;
+}
+
+static enum rtw_hal_status _wow_chk_txq_empty(struct hal_info_t *hal_info, u8 *empty)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_mac_wow_chk_txq_empty(hal_info, empty);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : status(%u).\n", __func__, hal_status);
+
+	return hal_status;
+}
+
+static enum rtw_hal_status _wow_wde_drop(struct hal_info_t *hal_info, u8 band)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+
+	hal_status = rtw_hal_mac_wow_wde_drop(hal_info, band);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : status(%u).\n", __func__, hal_status);
+
+	return hal_status;
+}
+
+#define MAX_WOW_DROP_HWTX_TRYCNT 3
+enum rtw_hal_status rtw_hal_wow_drop_tx(void *hal, u8 band)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	u8 empty = 0;
+	u8 i = 0;
+
+	for (i = 0; i < MAX_WOW_DROP_HWTX_TRYCNT; i++) {
+
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s : chk/drop for the %d time\n", __func__, i + 1);
+
+		if (RTW_HAL_STATUS_SUCCESS != _wow_chk_txq_empty(hal_info, &empty))
+			PHL_WARN("[wow] _wow_chk_txq_empty failed.\n");
+
+		/* force drop wde if txq is not empty */
+		if (!empty)
+			_wow_wde_drop(hal_info, band);
+		else
+			break;
+	}
+
+	if (!empty) {
+		PHL_WARN("[wow] %s : chk/drop fail!\n", __func__);
+	} else {
+		PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "[wow] %s : chk/drop ok.\n", __func__);
+		hal_status = RTW_HAL_STATUS_SUCCESS;
+	}
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_wow_req_tri_evt(void *hal)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	hal_status = rtw_hal_mac_wow_req_tri_evt(hal_info);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : status(%u).\n", __func__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_wow_req_diag_rpt(void *hal)
+{
+	enum rtw_hal_status hal_status = RTW_HAL_STATUS_FAILURE;
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+
+	hal_status = rtw_hal_mac_wow_req_diag_rpt(hal_info);
+
+	PHL_TRACE(COMP_PHL_WOW, _PHL_INFO_, "%s : status(%u).\n", __func__, hal_status);
+
+	return hal_status;
+}
+
+enum rtw_hal_status rtw_hal_wow_dbg_dump(void *hal)
+{
+	struct hal_info_t *hal_info = (struct hal_info_t *)hal;
+	enum rtw_hal_status hal_stats = RTW_HAL_STATUS_FAILURE;
+
+	hal_stats = rtw_hal_mac_wow_dbg_dump(hal_info);
+
+	if (RTW_HAL_STATUS_SUCCESS != hal_stats)
+		PHL_ERR("%s : Fail.\n", __func__);
+
+	return hal_stats;
+}
+
+#endif /* CONFIG_WOWLAN */
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_wow.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_wow.h
new file mode 100644
index 000000000000..05e4934ae40c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/hal_wow.h
@@ -0,0 +1,23 @@
+/******************************************************************************
+ *
+ * Copyright(c)2019 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ *****************************************************************************/
+#ifndef _HAL_WOW_H_
+#define _HAL_WOW_H_
+
+#ifdef CONFIG_WOWLAN
+/* ... */
+#endif /* CONFIG_WOWLAN */
+
+#endif /* _HAL_WOW_H_ */
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/chip_cfg.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/chip_cfg.h
new file mode 100644
index 000000000000..54731591f647
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/chip_cfg.h
@@ -0,0 +1,196 @@
+/** @file */
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ ******************************************************************************/
+
+#ifndef _MAC_AX_CHIP_CFG_H_
+#define _MAC_AX_CHIP_CFG_H_
+
+#ifndef __cplusplus /* for win/linux driver */
+
+/* Modify MakeFile to reduce code size (chip & interface) */
+#ifdef CONFIG_RTL8852A
+#define MAC_AX_8852A_SUPPORT	1
+#define MAC_8852A_SUPPORT
+#else
+#define MAC_AX_8852A_SUPPORT	0
+#endif
+
+#if defined(CONFIG_RTL8852B) || defined(CONFIG_RTL8852BP)
+#define MAC_AX_8852B_SUPPORT	1
+#define MAC_8852B_SUPPORT
+#else
+#define MAC_AX_8852B_SUPPORT	0
+#endif
+
+#if defined(CONFIG_RTL8852C) || defined(CONFIG_RTL8832BRH) || defined(CONFIG_RTL8842A)
+#define MAC_AX_8852C_SUPPORT	1
+#define MAC_8852C_SUPPORT
+#else
+#define MAC_AX_8852C_SUPPORT	0
+#endif
+
+#if defined(CONFIG_RTL8192XB) || defined(CONFIG_RTL8832BR)
+#define MAC_AX_8192XB_SUPPORT  1
+#define MAC_AX_8832BR_SUPPORT  1
+#define MAC_8192XB_SUPPORT
+#else
+#define MAC_AX_8192XB_SUPPORT  0
+#define MAC_AX_8832BR_SUPPORT  0
+#endif
+
+#ifdef CONFIG_RTL8851B
+#define MAC_AX_8851B_SUPPORT	1
+#define MAC_8851B_SUPPORT
+#else
+#define MAC_AX_8851B_SUPPORT	0
+#endif
+
+#ifdef CONFIG_RTL8852D
+#define MAC_AX_8852D_SUPPORT	1
+#define MAC_8852D_SUPPORT
+#else
+#define MAC_AX_8852D_SUPPORT	0
+#endif
+
+#if defined(CONFIG_RTL8852BT) || defined(CONFIG_RTL8852BPT)
+#define MAC_AX_8852BT_SUPPORT	1
+#define MAC_8852BT_SUPPORT
+#else
+#define MAC_AX_8852BT_SUPPORT	0
+#endif
+
+#ifdef CONFIG_RTL1115E
+#define MAC_AX_1115E_SUPPORT	1
+#define MAC_BE_1115E_SUPPORT	1
+#define MAC_BE_8922A_SUPPORT	1
+#define MAC_BE_8952A_SUPPORT	1
+#define MAC_BE_8934A_SUPPORT	1
+
+#define MAC_1115E_SUPPORT
+#define MAC_8922A_SUPPORT
+#define MAC_8952A_SUPPORT
+#define MAC_8934A_SUPPORT
+#else
+#define MAC_AX_1115E_SUPPORT	0
+#define MAC_BE_1115E_SUPPORT	0
+#define MAC_BE_8922A_SUPPORT	0
+#define MAC_BE_8952A_SUPPORT	0
+#define MAC_BE_8934A_SUPPORT	0
+#endif
+
+
+/* Interface support */
+#ifdef CONFIG_SDIO_HCI
+#define MAC_AX_SDIO_SUPPORT	1
+#else
+#define MAC_AX_SDIO_SUPPORT	0
+#endif
+#ifdef CONFIG_USB_HCI
+#define MAC_AX_USB_SUPPORT	1
+#else
+#define MAC_AX_USB_SUPPORT	0
+#endif
+#ifdef CONFIG_PCI_HCI
+#define MAC_AX_PCIE_SUPPORT	1
+#else
+#define MAC_AX_PCIE_SUPPORT	0
+#endif
+
+#else /* for WD1 test program */
+
+/* Modify MakeFile to reduce code size (chip & interface) */
+
+#if WIFI_HAL_G6
+#define MAC_AX_8852A_SUPPORT	1
+#define MAC_AX_8852B_SUPPORT	1
+#define MAC_AX_8852C_SUPPORT	1
+#define MAC_AX_8192XB_SUPPORT	1
+#define MAC_AX_8832BR_SUPPORT	1
+#define MAC_AX_8851B_SUPPORT	1
+#define MAC_AX_8852D_SUPPORT	1
+#define MAC_AX_8852BT_SUPPORT	1
+//for FW_PUB
+#define MAC_8852A_SUPPORT
+#define MAC_8852B_SUPPORT
+#define MAC_8852C_SUPPORT
+#define MAC_8192XB_SUPPORT
+#define MAC_8851B_SUPPORT
+#define MAC_8852D_SUPPORT
+#define MAC_8852BT_SUPPORT
+#else
+#define MAC_AX_8852A_SUPPORT	0
+#define MAC_AX_8852B_SUPPORT	0
+#define MAC_AX_8852C_SUPPORT	0
+#define MAC_AX_8192XB_SUPPORT	0
+#define MAC_AX_8832BR_SUPPORT	0
+#define MAC_AX_8851B_SUPPORT	0
+#define MAC_AX_8852D_SUPPORT	0
+#define MAC_AX_8852BT_SUPPORT	0
+#endif
+
+#if WIFI_HAL_G7
+#define MAC_BE_1115E_SUPPORT	1
+#define MAC_BE_8922A_SUPPORT	1
+#define MAC_BE_8952A_SUPPORT	1
+//for FW_PUB
+#define MAC_1115E_SUPPORT
+#define MAC_8922A_SUPPORT
+#define MAC_8952A_SUPPORT
+#else
+#define MAC_BE_1115E_SUPPORT	0
+#define MAC_BE_8922A_SUPPORT	0
+#define MAC_BE_8952A_SUPPORT	0
+#endif
+
+#if MAC_AX_8852A_SUPPORT
+#define MAC_FW_8852A_U1		0
+#define MAC_FW_8852A_U2		1
+#define MAC_FW_8852A_U3		1
+#define MAC_FW_8852A_U4		0
+#endif
+
+#if MAC_AX_8852B_SUPPORT
+#define MAC_FW_8852B_U1		0
+#define MAC_FW_8852B_U2		1
+#endif
+
+#if MAC_AX_8852C_SUPPORT
+#define MAC_FW_8852C_U1		1
+#endif
+
+#if MAC_AX_8192XB_SUPPORT
+#define MAC_FW_8192XB_U1	1
+#endif
+
+#if MAC_AX_8851B_SUPPORT
+#define MAC_FW_8851B_U1		1
+#endif
+
+#if MAC_AX_8852BT_SUPPORT
+#define MAC_FW_8852BT_U1	1
+#endif
+
+#if MAC_AX_8852D_SUPPORT
+#define MAC_FW_8852D_U1		1
+#endif
+
+/* Interface support */
+#define MAC_AX_SDIO_SUPPORT	1
+#define MAC_AX_USB_SUPPORT	1
+#define MAC_AX_PCIE_SUPPORT	1
+
+#endif // #else /* for WD1 test program */
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/chip_cfg_drv.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/chip_cfg_drv.h
new file mode 100644
index 000000000000..c443ee8f4438
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/chip_cfg_drv.h
@@ -0,0 +1,79 @@
+/** @file */
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ ******************************************************************************/
+
+#ifndef _MAC_AX_CHIP_CFG_H_
+#define _MAC_AX_CHIP_CFG_H_
+
+/* Modify MakeFile to reduce code size (chip & interface) */
+#ifdef CONFIG_RTL8852A
+#define MAC_AX_8852A_SUPPORT	1
+#else
+#define MAC_AX_8852A_SUPPORT	0
+#endif
+
+#ifdef CONFIG_RTL8852B
+#define MAC_AX_8852B_SUPPORT	1
+#else
+#define MAC_AX_8852B_SUPPORT	0
+#endif
+
+#ifdef CONFIG_RTL8852C
+#define MAC_AX_8852C_SUPPORT	1
+#else
+#define MAC_AX_8852C_SUPPORT	0
+#endif
+
+#ifdef defined(CONFIG_RTL8192XB) || defined(CONFIG_RTL8832BR)
+#define MAC_AX_8192XB_SUPPORT	1
+#else
+#define MAC_AX_8192XB_SUPPORT	0
+#endif
+
+#ifdef CONFIG_RTL8851B
+#define MAC_AX_8851B_SUPPORT	1
+#else
+#define MAC_AX_8851B_SUPPORT	0
+#endif
+
+#ifdef CONFIG_RTL8852D
+#define MAC_AX_8852D_SUPPORT	1
+#else
+#define MAC_AX_8852D_SUPPORT	0
+#endif
+
+#ifdef CONFIG_RTL8852BT
+#define MAC_AX_8852BT_SUPPORT	1
+#else
+#define MAC_AX_8852BT_SUPPORT	0
+#endif
+/* Interface support */
+#ifdef CONFIG_SDIO_HCI
+#define MAC_AX_SDIO_SUPPORT	1
+#else
+#define MAC_AX_SDIO_SUPPORT	0
+#endif
+#ifdef CONFIG_USB_HCI
+#define MAC_AX_USB_SUPPORT	1
+#else
+#define MAC_AX_USB_SUPPORT	0
+#endif
+#ifdef CONFIG_PCI_HCI
+#define MAC_AX_PCIE_SUPPORT	1
+#else
+#define MAC_AX_PCIE_SUPPORT	0
+#endif
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/errors.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/errors.h
new file mode 100644
index 000000000000..9845376497af
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/errors.h
@@ -0,0 +1,266 @@
+/** @file */
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ ******************************************************************************/
+
+#ifndef _MAC_AX_ERR_H_
+#define _MAC_AX_ERR_H_
+
+#define MACSUCCESS	0  /* Success return value */
+#define MACPFCB		1  /* Callback of platform is null */
+#define MACPFED		2  /* Endian of platform error */
+#define MACBADDR	3  /* Invalid base address */
+#define MACLSUS		4  /* Leave suspend error */
+#define MACNPTR		5  /* Pointer is null */
+#define MACCHIPID	6  /* Chip ID is undefined */
+#define MACADAPTER	7  /* Can not get MAC adapter */
+#define MACSTCAL	8  /* Unexpected structure alignment */
+#define MACNOBUF	9  /* Buffer space is not enough */
+#define MACBUFSZ	10 /* Buffer size error */
+#define MACNOITEM	11 /* Invalid item */
+#define MACPOLLTO	12 /* Polling timeout */
+#define MACPWRSW	13 /* Power switch fail */
+#define MACBUFALLOC	14 /* Buffer allocation fail */
+#define MACWQBUSY	15 /* Work queue is busy */
+#define MACCMP		16 /* Failed compare result */
+#define MACINTF		17 /* Wrong interface */
+#define MACFWBIN	18 /* Incorrect FW bin file */
+#define MACFFCFG	19 /* Wrong FIFO configuration */
+#define MACSAMACID	20 /* Same MACID */
+#define MACMACIDFL	21 /* MACID full */
+#define MACNOFW		22 /* There is no FW */
+#define MACPROCBUSY	23 /* Process is busy */
+#define MACPROCERR	24 /* state machine error */
+#define MACEFUSEBANK	25 /* switch efuse bank fail */
+#define MACEFUSEREAD	26 /* read efuse fail */
+#define MACEFUSEWRITE	27 /* write efuse fail */
+#define MACEFUSESIZE	28 /* efuse size error */
+#define MACEFUSEPARSE	29 /* eeprom parsing fail */
+#define MACEFUSECMP	30 /* compare efuse fail */
+#define MACSECUREON	31 /* secure on, no host indirect access */
+#define MACTXCHDMA	32 /* invalid tx dma channel */
+#define MACADDRCAMUPDERR 33 /* address cam update error */
+#define MACPWRSTAT	34 /* Power state error */
+#define MACSDIOMIXMODE	35 /* SDIO Tx mix mode */
+#define MACSDIOSEQERR	36 /* SDIO Tx sequence error */
+#define MACHFCH2CQTA	37 /* HCI FC invalid H2C quota */
+#define MACHFCCH011QTA	38 /* HCI FC invalid CH0-11 quota */
+#define MACHFCCH011GRP	39 /* HCI FC invalid CH0-11 group */
+#define MACHFCPUBQTA	40 /* HCI FC invalid public quota */
+#define MACHFCPUBINFO	41 /* HCI FC public info error */
+#define MACRFPMCAM	42 /* RX forwarding PM CAM access fail */
+#define MACHFSWDENOTNUF	43 /* HCI FC WDE page not enough */
+#define MACHFSPLENOTNUF	44 /* HCI FC PLE page not enough */
+#define MACMEMRO	45 /* Address is not writable */
+#define MACFUNCINPUT	46 /* invalid function input */
+#define MACALRDYON	47 /* MAC has already powered on */
+#define MACADDRCAMFL	48 /* ADDRESS CAM full */
+#define MACBSSIDCAMFL	49 /* BSSID CAM full */
+#define MACGPIOUSED	50 /* GPIO is used */
+#define MACDLELINK	51 /* DLE link error */
+#define MACPOLLTXIDLE	52 /* polling Tx idle fail */
+#define MACPARSEERR	53 /* parse report err */
+#define MACROLEINITFL	54 /* Role API init fail or C2H notify role init fail */
+#define MACPORTCFGTYPE	55 /* Port cfg type error */
+#define MACPORTERR	56 /* Invalid Port number */
+#define MACWNGKEYTYPE	57 /* Sec cam wrong key type*/
+#define MACKEYNOTEXT	58 /* Delete key , key not exist*/
+#define MACSECCAMFL	59 /* SEC CAM full*/
+#define MACADDRCAMKEYFL	60 /* Addr CAM key full*/
+#define MACNOROLE	61 /* SEC no this role*/
+#define MACHWNOTEN	62 /* hw module not enable*/
+#define MACPTMTXFAIL	63 /* platform TX fail*/
+#define MACSSLINK	64 /* STA scheduler link error */
+#define MACDBGPORTSEL	65 /* Debug port sel error */
+#define MACDBGPORTDMP	66 /* Debug port dump error */
+#define MACCPWMSEQERR	67 /* CPWM sequence mismatch */
+#define MACCPWMSTATERR	68 /* CPWM state mismatch */
+#define MACCPUSTATE	69 /* Incorrect CPU state */
+#define MACPSSTATFAIL	70 /* protocol power state check tx pause fail */
+#define MACLV1STEPERR	71 /* lv1 rcvy step sel error */
+#define MACFWCHKSUM	72 /* FW checksum is incorrect */
+#define MACFWSECBOOT	73 /* FW security boot is failed */
+#define MACFWCUT	74 /* Mismatch chip and FW cut */
+#define MACSUBSPCERR	75 /* Beacon sub-space setting fail */
+#define MACLENCMP	76 /* Length is not match */
+#define MACCHKSUMEMPTY	77 /* Checksum report empty  */
+#define MACCHKSUMFAIL	78 /* Checksum report fail  */
+#define MACVERERR	79 /* Map and mask version mismatch */
+#define MACFWNONRDY	80 /* FW not ready h2c error*/
+#define MACGPIONUM	81 /* The gpio number is wrong */
+#define MACNOTSUP	82 /* The function is NOT supported */
+#define MACCSIBUFIDERR	83 /* CSI buffer index is NOT supported */
+#define MACSNDSTSIDERR	84 /* Sounding status ID is NOT supported */
+#define MACCCTLWRFAIL	85 /* control info wrrite fail */
+#define MACHWNOSUP	86 /* HW not support */
+#define MACUNDEFCH	87 /* Channel is undefined */
+#define MACHWERR	88 /* HW error */
+#define MACFWTESTFAIL	89 /* FW auto test fail */
+#define MACP2PSTFAIL	90 /* P2P state fail */
+#define MACFLASHFAIL	91 /* FW auto test fail */
+#define MACSETVALERR	92 /* Setting value error */
+#define MACIOERRPWR	93 /* IO not allow when power not on */
+#define MACIOERRSERL1	94 /* IO not allow when SER Lv1 */
+#define MACIOERRLPS	95 /* IO not allow when LPS */
+#define MACIOERRDMAC	96 /* IO not allow when dmac not en */
+#define MACIOERRCMAC0	97 /* IO not allow when cmac0 not en */
+#define MACIOERRCMAC1	98 /* IO not allow when cmac1 not en */
+#define MACIOERRBB0	99 /* IO not allow when bb0 not en */
+#define MACIOERRBB1	100 /* IO not allow when bb1 not en */
+#define MACIOERRRF	101 /* IO not allow when rf not en */
+#define MACIOERRIND	102 /* IO not allow when indirect access */
+#define MACIOERRRSVD	103 /* IO not allow if address is rsvd */
+#define MACC2HREGEMP	104 /* C2H reg empty */
+#define MACBADC2HREG	105 /* received unexpected c2hreg */
+#define MACFIOOFLD	106 /* IO offload fail */
+#define MACROLEALOCFL	107 /* C2H notify alloc role failed */
+#define MACROLEHWUPDFL	108 /* C2H notify addrcam upd failed*/
+#define MACSDIOTXMODE	109 /* SDIO Tx mode undefined*/
+#define MACSDIOOPNMODE	110 /* SDIO opn mode unknown*/
+#define MACFWSTATUSFAIL 111 /* fw status command fail */
+#define MACIOERRPLAT	112 /* IO not allow when platform not on */
+#define MACCPWMPWRSTATERR	113 /* CPWM power state mismatch */
+#define MACIOERRISH		114 /* IO not allow when io state hang */
+#define MACHWDMACERR	115 /* DMAC_ERR_ISR */
+#define MACHWCMAC0ERR	116 /* CMAC0_ERR_ISR */
+#define MACDRVRM	117 /* driver is removed unexpectedly */
+#define MACMCCGPFL	118 /* Get MCC Group index fail*/
+#define MACFWSTATEERR	119 /* fw state error */
+#define MACFWLOGINTERR 120 /*fw log parsing error*/
+#define MACSYSSTEERR	121 /* Whole System Power State error */
+#define MACHWCMAC1ERR	122 /* CMAC1_ERR_ISR */
+#define MACFWASSERT	123 /* FW Assertion error */
+#define MACFWEXCEP	124 /* FW Exception error */
+#define MACFWRXI300	125 /* FW RXI300 error */
+#define MACFWPCHANG	126 /* FW PC hang error */
+#define MACRXDMAHANG	127 /*USB RXDMA HANG */
+#define MACUSBRXHANG	128 /*USB RX HANG */
+#define MACCPWMINTFERR	129 /* CPWM interface error */
+#define MACUSBPAUSEERR	130 /* USB EP PAUSE error */
+#define MACARDYDONE	131 /* The flow is already done */
+#define MACPSSTATPWRBITFAIL	132 /* protocol power state check pwr bit fail */
+#define MACIOERRINSEC	133 /* Security ic not allow indirect access */
+#define MACIOTESTERR	140 /* general io test error */
+#define MACFWNOSUPPORT	145 /* FW no support */
+#define MACMEMLEAK	146 /* Memory Leak */
+#define MACENTERLPSFAIL	147 /* Enter LPS Fail*/
+#define MACENTERIPSFAIL	148 /* Enter IPS Fail*/
+#define MACFWCAPDRVERR	149 /* FW cap error: driver compile flag error*/
+#define MACWRONGPARA	150 /* Wrong parameter for the API*/
+#define MACPWRSTATEERR	151 /* MAC Power State Error*/
+#define MACHIDDENERR	152 /* Hidden valid error */
+
+/*MAC DBG Status Indication*/
+#define MACSCH_NONEMPTY	1 /* MAC Scheduler non empty */
+
+/* Debug Package Indication */
+/* STA Scheduler 0, indirect */
+#define SS_TX_LEN_BE		BIT(0)
+#define SS_TX_LEN_BK		BIT(1)
+#define SS_TX_LEN_VI		BIT(2)
+#define SS_TX_LEN_VO		BIT(3)
+#define SS_LINK_WMM0_BE		BIT(4)
+#define SS_LINK_WMM0_BK		(SS_LINK_WMM0_BE << 1)
+#define SS_LINK_WMM0_VI		(SS_LINK_WMM0_BE << 2)
+#define SS_LINK_WMM0_VO		(SS_LINK_WMM0_BE << 3)
+#define SS_LINK_WMM1_BE		(SS_LINK_WMM0_BE << 4)
+#define SS_LINK_WMM1_BK		(SS_LINK_WMM0_BE << 5)
+#define SS_LINK_WMM1_VI		(SS_LINK_WMM0_BE << 6)
+#define SS_LINK_WMM1_VO		(SS_LINK_WMM0_BE << 7)
+#define SS_LINK_WMM2_BE		(SS_LINK_WMM0_BE << 8)
+#define SS_LINK_WMM2_BK		(SS_LINK_WMM0_BE << 9)
+#define SS_LINK_WMM2_VI		(SS_LINK_WMM0_BE << 10)
+#define SS_LINK_WMM2_VO		(SS_LINK_WMM0_BE << 11)
+#define SS_LINK_WMM3_BE		(SS_LINK_WMM0_BE << 12)
+#define SS_LINK_WMM3_BK		(SS_LINK_WMM0_BE << 13)
+#define SS_LINK_WMM3_VI		(SS_LINK_WMM0_BE << 14)
+#define SS_LINK_WMM3_VO		(SS_LINK_WMM0_BE << 15)
+#define SS_LINK_UL		(SS_LINK_WMM0_BE << 16)
+#define SS_POLL_OWN_TX_LEN	BIT(24)
+#define SS_POLL_OWN_LINK	BIT(25)
+#define SS_POLL_STAT_TX_LEN	BIT(26)
+#define SS_POLL_STAT_LINK	BIT(27)
+/* STA Scheduler 1, direct */
+#define SS_TX_HW_LEN_UDN	BIT(0)
+#define SS_TX_SW_LEN_UDN	BIT(1)
+#define SS_TX_HW_LEN_OVF	BIT(2)
+#define SS_STAT_FWTX		BIT(8)
+#define SS_STAT_RPTA		BIT(9)
+#define SS_STAT_WDEA		BIT(10)
+#define SS_STAT_PLEA		BIT(11)
+#define SS_STAT_ULRU		BIT(12)
+#define SS_STAT_DLTX		BIT(13)
+
+#define PLTFM_MSG_ALWAYS(...)                                                  \
+	adapter->pltfm_cb->msg_print(adapter->drv_adapter, _PHL_ALWAYS_,  __VA_ARGS__)
+
+#if MAC_AX_DBG_MSG_EN
+
+#undef PLTFM_MSG_ALWAYS
+
+void mac_console_log(void *vadapter, s8 *prefix, s8 *fmt, ...);
+
+	#if (MAC_AX_MSG_LEVEL >= MAC_AX_MSG_LEVEL_ALWAYS)
+	#define PLTFM_MSG_ALWAYS(...)  \
+		adapter->pltfm_cb->msg_print(adapter->drv_adapter, _PHL_ALWAYS_, __VA_ARGS__)
+	#else
+	#define PLTFM_MSG_ALWAYS(...)	do {} while (0)
+	#endif
+
+	/* Enable debug msg depends on  HALMAC_MSG_LEVEL */
+	#if (MAC_AX_MSG_LEVEL >= MAC_AX_MSG_LEVEL_ERR)
+	#define PLTFM_MSG_ERR(...)  \
+		adapter->pltfm_cb->msg_print(adapter->drv_adapter, _PHL_ERR_, __VA_ARGS__)
+	#else
+	#define PLTFM_MSG_ERR(...)	do {} while (0)
+	#endif
+
+	#if (MAC_AX_MSG_LEVEL >= MAC_AX_MSG_LEVEL_WARNING)
+	#define PLTFM_MSG_WARN(...)  \
+		adapter->pltfm_cb->msg_print(adapter->drv_adapter, _PHL_WARNING_, __VA_ARGS__)
+	#else
+	#define PLTFM_MSG_WARN(...)	do {} while (0)
+	#endif
+
+	#if (MAC_AX_MSG_LEVEL >= MAC_AX_MSG_LEVEL_TRACE)
+	#define PLTFM_MSG_TRACE(...)  \
+		adapter->pltfm_cb->msg_print(adapter->drv_adapter, _PHL_DEBUG_, __VA_ARGS__)
+	#else
+	#define PLTFM_MSG_TRACE(...)	do {} while (0)
+	#endif
+	#if (MAC_AX_MSG_LEVEL >= MAC_AX_MSG_LEVEL_ERR)
+	#define PLTFM_MSG_BUFFER(...)   do {\
+		if (adapter->fw_dbgcmd.dbg_bg_log_on)			\
+			adapter->pltfm_cb->msg_print(adapter->drv_adapter, _PHL_ERR_,\
+						     __VA_ARGS__);\
+		mac_console_log(adapter, "", __VA_ARGS__);\
+	} while (0)
+	#else
+	#define PLTFM_MSG_BUFFER(...)   do {\
+		mac_console_log(adapter, "", __VA_ARGS__);\
+	} while (0)
+	#endif
+
+#else
+
+/* Disable debug msg  */
+#define PLTFM_MSG_ALWAYS(...)	do {} while (0)
+#define PLTFM_MSG_ERR(...)	do {} while (0)
+#define PLTFM_MSG_WARN(...)	do {} while (0)
+#define PLTFM_MSG_TRACE(...)	do {} while (0)
+#define PLTFM_MSG_BUFFER(...)	do {} while (0)
+
+#endif
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/feature_cfg.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/feature_cfg.h
new file mode 100644
index 000000000000..df561d9e296c
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/feature_cfg.h
@@ -0,0 +1,370 @@
+/** @file */
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ ******************************************************************************/
+
+#ifndef _MAC_AX_FEATURE_CFG_H_
+#define _MAC_AX_FEATURE_CFG_H_
+
+#ifndef __cplusplus /* for win/linux driver */
+
+/* [Note] Modify MakeFile to reduce code size */
+
+/* Debug package for debugging */
+#ifdef CONFIG_HAL_MAC_DBG
+#define MAC_AX_FEATURE_DBGPKG	1
+#else
+#define MAC_AX_FEATURE_DBGPKG	0
+#endif
+
+/* Error Flag for HW debugging */
+#if MAC_AX_FEATURE_DBGPKG
+#define MAC_AX_FEATURE_ERR_FLAG	1
+#else
+#define MAC_AX_FEATURE_ERR_FLAG	0
+#endif
+
+/* Hadrware verification module */
+#define MAC_AX_FEATURE_HV	0
+
+/* FW offload reg read/write  */
+#define MAC_AX_FW_REG_OFLD	0
+
+/* Temp code for FPGA verification*/
+#define MAC_AX_FPGA_TEST	0
+
+/* temp for h2c alloc move to phl layer*/
+#define MAC_AX_PHL_H2C		1
+
+/* For USB IO Offload Accelerate enable*/
+#if defined(CONFIG_FW_DBCC_OFLD_SUPPORT) || defined(CONFIG_PHL_IO_OFLD)
+#define MAC_USB_IO_ACC_ON	1
+#else
+#define MAC_USB_IO_ACC_ON	0
+#endif
+
+/* For workaround temp usage */
+#define MAC_AX_RTK_WARD		0
+
+/* For workaround sdio issue temp usage */
+#ifdef CONFIG_PHL_WKARD_TXEN_ISSUE
+#define MAC_AX_RTK_WARD_TXEN_DATA_PLE_FULL	1
+#else
+#define MAC_AX_RTK_WARD_TXEN_DATA_PLE_FULL	0
+#endif
+
+/* For USB IO Offload Accelerate*/
+#define MAC_USB_IO_ACC		0
+
+#define MAC_SELF_DIAG_INFO	1
+
+#ifdef PHL_FEATURE_NONE
+#define MAC_AX_CONSOLE_EN	0
+#elif MAC_SELF_DIAG_INFO
+#define MAC_AX_CONSOLE_EN	1
+#else
+#define MAC_AX_CONSOLE_EN	0
+#endif
+
+#if defined(PHL_PLATFORM_WINDOWS) && defined(CONFIG_PHL_DIAGNOSE)
+#define MAC_AX_PLDR_DIAGNOSE_EN	1
+#else
+#define MAC_AX_PLDR_DIAGNOSE_EN	0
+#endif
+
+#ifdef CONFIG_IOW_STATISTICS
+#define MAC_TXRPT_STATISTIC	1
+#endif
+
+/* ------- Begin of Feature Flag  ------- */
+#if defined(CONFIG_PHL_BCN_ERLY_RPT) || defined(CONFIG_SCANOFLD) || defined(CONFIG_PHL_CHSWOFLD) || defined(CONFIG_PHL_PKTOFLD) || MAC_USB_IO_ACC_ON
+#define MAC_FEAT_FWOFLD		1
+#else
+#define MAC_FEAT_FWOFLD		0
+#endif
+
+#define MAC_FEAT_NLO		1
+
+#ifdef CONFIG_PHL_P2PPS
+#define MAC_FEAT_P2P		1
+#else
+#define MAC_FEAT_P2P		0
+#endif
+
+#define MAC_FEAT_RX_BCN_OFLD	1
+
+#define MAC_FEAT_ULOFDMA	1
+
+#define MAC_FEAT_DLOFDMA	1
+
+#define MAC_FEAT_BFMER		1
+#ifdef MAC_FEAT_BFMER //& wait yllinzo define F2PCMD
+	#define MAC_FEAT_BFMER_F2PSND	1
+#endif
+
+#ifdef CONFIG_MCC_SUPPORT
+#define MAC_FEAT_MCC		1
+#else
+#define MAC_FEAT_MCC		0
+#endif
+
+#ifdef CONFIG_POWER_SAVE
+#define MAC_FEAT_LPS		1
+#else
+#define MAC_FEAT_LPS		0
+#endif
+
+#define MAC_FEAT_TWTAP		1 // TWT AP feature compile flag
+
+#define MAC_FEAT_PSAP		1 // AP power save compile flag
+
+#define MAC_FEAT_MUMIMO		1
+
+#define MAC_FEAT_HWSSN		1
+
+#define MAC_FEAT_HDR_CONV	1
+
+#define MAC_FEAT_HWAMSDU	1
+
+#define MAC_FEAT_NAN		1 // NAN feature compile flag
+
+#define MAC_FEAT_SECURE_BOOT 1
+
+#define MAC_FEAT_BFMEE		1 // sounding MEE compile flag
+
+//FTM compile flag
+#ifdef CONFIG_PHL_FTM
+#define MAC_FEAT_FTM		1
+#else
+#define MAC_FEAT_FTM		0
+#endif
+
+#define MAC_FEAT_TX_BCN 1
+#if MAC_FEAT_TX_BCN
+#define MAC_FEAT_MBSSID 1
+#define MAC_FEAT_TX_BCN_DM_TBTT_SHIFT 0
+#define MAC_FEAT_TX_BCN_DM_IGNORE_EDCCA 1
+#else
+#define MAC_FEAT_MBSSID 0
+#define MAC_FEAT_TX_BCN_DM_TBTT_SHIFT 0
+#define MAC_FEAT_TX_BCN_DM_IGNORE_EDCCA 0
+#endif
+
+#define MAC_FEAT_LAMODE		1 // for BB LA mode debug usage
+
+#define MAC_FEAT_F2PCMD		1
+
+#ifdef CONFIG_PHL_TWT
+#define MAC_FEAT_TWT_STA	1
+#else
+#define MAC_FEAT_TWT_STA	0
+#endif
+
+#define MAC_FEAT_PPDU_STS	1
+#if defined(CONFIG_PHL_CHANNEL_INFO) || defined(CONFIG_PHL_WKARD_CHANNEL_INFO_ACK) ||\
+defined(CONFIG_PHL_CHANNEL_INFO_DBG)
+#define MAC_FEAT_CH_INFO	1
+#else
+#define MAC_FEAT_CH_INFO	0
+#endif
+#define MAC_FEAT_DFS		1
+#if (MAC_FEAT_PPDU_STS || MAC_FEAT_CH_INFO || MAC_FEAT_DFS)
+#define MAC_FEAT_PHY_RPT	1
+#else
+#define MAC_FEAT_PHY_RPT	0
+#endif
+
+#if defined(CONFIG_WOWLAN)
+#define MAC_FEAT_WOWLAN		1
+#else
+#define MAC_FEAT_WOWLAN		0
+#endif
+
+#if defined(CONFIG_BCN_CNT)
+#define MAC_FEAT_BCN_CNT	1
+#else
+#define MAC_FEAT_BCN_CNT	0
+#endif
+
+//Basic efuse R/W won't include in MAC_FEAT_EFUSE
+#define MAC_FEAT_EFUSE		1
+#if MAC_FEAT_EFUSE
+#define MAC_FEAT_EFUSE_MP	1
+#define MAC_FEAT_EFUSE_HV	0
+#else
+#define MAC_FEAT_EFUSE_MP	0
+#define MAC_FEAT_EFUSE_HV	0
+#endif
+
+#define MAC_FEAT_COLLOCATION 1
+
+#ifdef CONFIG_DBCC_SUPPORT
+#define MAC_FEAT_DBCC		1
+#else
+#define MAC_FEAT_DBCC		0
+#endif
+
+/* COEX */
+#ifdef CONFIG_BTCOEX
+#define MAC_FEAT_COEX		1
+#else
+#define MAC_FEAT_COEX		0
+#endif
+
+/* For NINTENDO DRIVER*/
+#define NINTENDO_EN		0
+
+/* -------  End of Feature Flag  -------  */
+
+#else /* for WD1 test program */
+
+/* [Note] Modify MakeFile to reduce code size */
+
+/* Debug package for debugging */
+#define MAC_AX_FEATURE_DBGPKG	1
+
+/* Error Flag for HW debugging */
+#if MAC_AX_FEATURE_DBGPKG
+#define MAC_AX_FEATURE_ERR_FLAG	1
+#else
+#define MAC_AX_FEATURE_ERR_FLAG	0
+#endif
+
+/* Debug package for debugging */
+#define MAC_AX_FW_REG_OFLD	0
+
+/* Hadrware verification module */
+#define MAC_AX_FEATURE_HV	1
+
+#define MAC_AX_FPGA_TEST	1
+
+/* temp for h2c alloc move to phl layer */
+#define MAC_AX_PHL_H2C		0
+
+/* [HV] for SIC command generagor */
+#define MAC_AX_HV_SIC_GEN	0
+
+/* For workaround temp usage */
+#define MAC_AX_RTK_WARD		0
+
+/* For workaround sdio issue temp usage */
+#define MAC_AX_RTK_WARD_TXEN_DATA_PLE_FULL	0
+
+/* For USB IO Offload Accelerate*/
+#define MAC_USB_IO_ACC		0
+
+/* For USB IO Offload Accelerate enable*/
+#define MAC_USB_IO_ACC_ON	1
+
+#define MAC_SELF_DIAG_INFO	1
+
+#define MAC_AX_CONSOLE_EN	0
+
+#define MAC_AX_PLDR_DIAGNOSE_EN	0
+
+/* ------- Begin of Feature Flag  ------- */
+#define MAC_FEAT_FWOFLD		1
+
+#define MAC_FEAT_NLO		1
+
+#define MAC_FEAT_P2P		1 // P2P feature compile flag
+#define MAC_FEAT_RX_BCN_OFLD	1 // RX beacon offload feature compile flag
+
+#define MAC_FEAT_ULOFDMA	1
+
+#define MAC_FEAT_BFMER		1
+#ifdef MAC_FEAT_BFMER //& wait yllinzo define F2PCMD
+	#define MAC_FEAT_BFMER_F2PSND	1
+#endif
+
+#define MAC_FEAT_MCC		1
+
+#define MAC_FEAT_LPS		1
+
+#define MAC_FEAT_TWTAP		1 // TWT AP feature compile flag
+
+#define MAC_FEAT_PSAP		1 // AP power save compile flag
+
+#define MAC_FEAT_MUMIMO		1
+
+#define MAC_FEAT_HWSSN		1
+
+#define MAC_FEAT_HDR_CONV	1
+
+#define MAC_FEAT_HWAMSDU	1
+
+#define MAC_FEAT_SECURE_BOOT 1
+
+#define MAC_FEAT_BFMEE		1 // sounding MEE compile flag
+
+// FTM compile flag
+#ifdef CONFIG_PHL_FTM
+#define MAC_FEAT_FTM		1
+#else
+#define MAC_FEAT_FTM		0
+#endif
+
+#define MAC_FEAT_TX_BCN 1
+#if MAC_FEAT_TX_BCN
+#define MAC_FEAT_MBSSID 1
+#define MAC_FEAT_TX_BCN_DM_TBTT_SHIFT 0
+#define MAC_FEAT_TX_BCN_DM_IGNORE_EDCCA 1
+#else
+#define MAC_FEAT_MBSSID 0
+#define MAC_FEAT_TX_BCN_DM_TBTT_SHIFT 0
+#define MAC_FEAT_TX_BCN_DM_IGNORE_EDCCA 0
+#endif
+
+#define MAC_FEAT_LAMODE		1 // for BB LA mode debug usage
+
+#define MAC_FEAT_F2PCMD		1
+
+#define MAC_FEAT_TWT_STA	1
+
+#define MAC_FEAT_PPDU_STS	1
+#define MAC_FEAT_CH_INFO	1
+#define MAC_FEAT_DFS		1
+#if (MAC_FEAT_PPDU_STS || MAC_FEAT_CH_INFO || MAC_FEAT_DFS)
+#define MAC_FEAT_PHY_RPT	1
+#else
+#define MAC_FEAT_PHY_RPT	0
+#endif
+
+#define MAC_FEAT_WOWLAN		1
+
+//Basic efuse R/W won't include in MAC_FEAT_EFUSE
+#define MAC_FEAT_EFUSE		1
+#if MAC_FEAT_EFUSE
+#define MAC_FEAT_EFUSE_MP	1
+#define MAC_FEAT_EFUSE_HV	1
+#else
+#define MAC_FEAT_EFUSE_MP	0
+#define MAC_FEAT_EFUSE_HV	0
+#endif
+
+#define MAC_FEAT_COLLOCATION 1
+#define MAC_FEAT_DBCC		1 // DBCC
+
+#define MAC_FEAT_COEX		1 // COEX
+
+/* For NINTENDO DRIVER*/
+#define NINTENDO_EN		0
+/* -------  End of Feature Flag  -------  */
+
+
+#endif // #else /* for WD1 test program */
+
+
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/fwcmd_intf.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/fwcmd_intf.h
new file mode 100644
index 000000000000..16276ad5181a
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/fwcmd_intf.h
@@ -0,0 +1,15624 @@
+#ifndef _MAC_FWCMD_INTF_H_
+#define _MAC_FWCMD_INTF_H_
+
+#define FWCMD_VER 0x0
+
+// ID definition//
+
+/*----------------------------------- H2CREG ---------------------------------------*/
+#define FWCMD_H2CREG_FUNC_H2CREG_LB 0x0
+#define FWCMD_H2CREG_FUNC_CNSL_CMD 0x1
+#define FWCMD_H2CREG_FUNC_FWERR 0x2
+#define FWCMD_H2CREG_FUNC_HIDDEN_GET 0x3
+#define FWCMD_H2CREG_FUNC_GETPKT_INFORM 0x4
+#define FWCMD_H2CREG_FUNC_SCH_TX_EN 0x5
+#define FWCMD_H2CREG_FUNC_WOW_TRX_STOP 0x6
+#define FWCMD_H2CREG_FUNC_AOAC_RPT_1 0x7
+#define FWCMD_H2CREG_FUNC_AOAC_RPT_2 0x8
+#define FWCMD_H2CREG_FUNC_AOAC_RPT_3_REQ 0x9
+#define FWCMD_H2CREG_FUNC_WOW_CPUIO_RX_CTRL 0xA
+#define FWCMD_H2CREG_FUNC_SET_GPIO 0xC
+#define FWCMD_H2CREG_FUNC_FWDBG_ADDR_CFG 0xD
+
+/*----------------------------------- C2HREG ---------------------------------------*/
+#define FWCMD_C2HREG_FUNC_C2HREG_LB 0x0
+#define FWCMD_C2HREG_FUNC_ERR_RPT 0x1
+#define FWCMD_C2HREG_FUNC_ERR_MSG 0x2
+#define FWCMD_C2HREG_FUNC_EFUSE_HIDDEN 0x3
+#define FWCMD_C2HREG_FUNC_TX_PAUSE_RPT 0x4
+#define FWCMD_C2HREG_FUNC_AOAC_RPT_1 0x5
+#define FWCMD_C2HREG_FUNC_AOAC_RPT_2 0x6
+#define FWCMD_C2HREG_FUNC_AOAC_RPT_3 0x7
+#define FWCMD_C2HREG_FUNC_WOW_TRX_STOP 0x8
+#define FWCMD_C2HREG_FUNC_IO_OFLD_RESULT 0x9
+#define FWCMD_C2HREG_FUNC_WOW_CPUIO_RX_ACK 0xA
+#define FWCMD_C2HREG_FUNC_FWDBG_ADDR_STS 0xB
+#define FWCMD_C2HREG_FUNC_EFUSE_HIDDEN_PART1 0xC
+
+/*----------------------------------- H2CPKT - CAT(TEST) ---------------------------*/
+#define FWCMD_H2C_CAT_TEST 0x0
+#define FWCMD_H2C_CAT_MAC 0x1
+#define FWCMD_H2C_CAT_OUTSRC 0x2
+#define FWCMD_H2C_CL_CMD_PATH 0x0
+#define FWCMD_H2C_CL_SND_TEST 0x1
+#define FWCMD_H2C_CL_PLAT_AUTO_TEST 0x2
+#define FWCMD_H2C_CL_MAC_TEST 0x3
+#define FWCMD_H2C_CL_FW_AUTO_TEST 0x4
+#define FWCMD_H2C_CL_FW_STATUS_TEST 0x5
+#define FWCMD_H2C_CL_WOW_TEST 0x6
+
+// CLASS 0 - CMD_PATH
+#define FWCMD_H2C_FUNC_H2C_LB 0x0
+#define FWCMD_H2C_FUNC_LOG_TEST 0x1
+#define FWCMD_H2C_FUNC_SEND_NULL_TEST 0x2
+#define FWCMD_H2C_FUNC_L1_2_TEST 0x3
+
+// CLASS 1 - SND_Test
+
+// CLASS 2 - PLATFORM_AUTO_TEST
+#define FWCMD_H2C_FUNC_PLAT_SPIC_TEST 0x0
+#define FWCMD_H2C_FUNC_PLAT_CPU_PLATFORM 0x1
+#define FWCMD_H2C_FUNC_PLAT_EFUSE_CTRL 0x2
+#define FWCMD_H2C_FUNC_PLAT_IDDMA 0x3
+#define FWCMD_H2C_FUNC_PLAT_AXIDMA 0x4
+#define FWCMD_H2C_FUNC_PLAT_IPSEC 0x5
+#define FWCMD_H2C_FUNC_PLAT_UART 0x6
+#define FWCMD_H2C_FUNC_PLAT_HIOE 0x7
+#define FWCMD_H2C_FUNC_PLAT_WATCHDOG 0x8
+#define FWCMD_H2C_FUNC_PLAT_SECURITY 0x9
+#define FWCMD_H2C_FUNC_PLAT_FL_WRITE 0xa
+#define FWCMD_H2C_FUNC_PLAT_FL_ERASE 0xb
+#define FWCMD_H2C_FUNC_PL_FLASH_READ 0xc
+#define FWCMD_H2C_FUNC_PLAT_MODULE_MAX 0x10
+#define FWCMD_H2C_FUNC_PLAT_INVALID 0x11
+
+// CLASS 3 - MAC_TEST
+#define FWCMD_H2C_FUNC_LONG_RUN 0x0
+#define FWCMD_H2C_FUNC_CPUIO_TEST 0x1
+
+// CLASS 4 - FW_AUTO_TEST
+#define FWCMD_H2C_FUNC_LPS_TEST 0x0
+#define FWCMD_H2C_FUNC_LPS_ONOFF_TEST 0x1
+#define FWCMD_H2C_FUNC_CPUMGENQ_TEST 0x2
+#define FWCMD_H2C_FUNC_MPIO_TEST 0x3
+#define FWCMD_H2C_FUNC_SENDNULL_TEST 0x4
+#define FWCMD_H2C_FUNC_MPIOV2_TEST 0x5
+#define FWCMD_H2C_FUNC_MIO_TEST 0x6
+#define FWCMD_H2C_FUNC_ANYTESTBUSYCALL_TEST 0x7
+#define FWCMD_H2C_FUNC_ANYRW_TEST 0x8
+
+// CLASS 5 - FW_STATUS_TEST
+#define FWCMD_H2C_FUNC_FW_STATUS 0x0
+#define FWCMD_H2C_FUNC_TRIGGER_CPU_EXCEPTION 0x1
+#define FWCMD_H2C_FUNC_FW_PROFILING 0x2
+
+// CLASS 6 - WOW_TEST
+#define FWCMD_H2C_FUNC_WOW_AUTO_TEST 0x0
+
+/*----------------------------------- H2CPKT - CAT(MAC) ----------------------------*/
+#define FWCMD_H2C_CAT_TEST 0x0
+#define FWCMD_H2C_CAT_MAC 0x1
+#define FWCMD_H2C_CAT_OUTSRC 0x2
+#define FWCMD_H2C_CL_FW_INFO 0x0
+#define FWCMD_H2C_CL_WOW 0x1
+#define FWCMD_H2C_CL_PS 0x2
+#define FWCMD_H2C_CL_FWDL 0x3
+#define FWCMD_H2C_CL_TWT 0x4
+#define FWCMD_H2C_CL_FR_EXCHG 0x5
+#define FWCMD_H2C_CL_ADDR_CAM_UPDATE 0x6
+#define FWCMD_H2C_CL_BSSID_CAM_UPDATE 0x7
+#define FWCMD_H2C_CL_MEDIA_RPT 0x8
+#define FWCMD_H2C_CL_FW_OFLD 0x9
+#define FWCMD_H2C_CL_SEC_CAM 0xA
+#define FWCMD_H2C_CL_SOUND 0xB
+#define FWCMD_H2C_CL_BA_CAM 0xC
+#define FWCMD_H2C_CL_IE_CAM 0xD
+#define FWCMD_H2C_CL_MCC 0xE
+#define FWCMD_H2C_CL_SCSI 0xF
+#define FWCMD_H2C_CL_FLASH 0x10
+#define FWCMD_H2C_CL_FCS 0x11
+#define FWCMD_H2C_CL_MISC 0x12
+#define FWCMD_H2C_CL_MPORT 0x13
+#define FWCMD_H2C_CL_MLO 0x14
+#define FWCMD_H2C_CL_PROXY 0x15
+#define FWCMD_H2C_CL_NAN 0x16
+#define FWCMD_H2C_CL_FW_CMDREQ 0x17
+#define FWCMD_H2C_CL_MRC 0x18
+#define FWCMD_H2C_CL_BF_SENS 0x19
+#define FWCMD_H2C_CL_OFFLOAD_AGG 0x1A
+#define FWCMD_H2C_CL_ECO_PATCH 0x1B
+#define FWCMD_H2C_CL_SER 0x1C
+#define FWCMD_H2C_CL_NAT2P5 0x1D
+#define FWCMD_H2C_CL_FTM 0x1E
+#define FWCMD_H2C_CL_WOW_DIAG 0x1F
+#define FWCMD_H2C_CL_WOW_TRI_EVT 0x20
+#define FWCMD_H2C_CL_AP_SELF_DIAG 0x21
+#define FWCMD_H2C_CL_STA_DIAG 0x22
+#define FWCMD_H2C_CL_ADDBA_OFLD 0x23
+#define FWCMD_H2C_CL_SOFTAP 0x24
+
+// CLASS 0 - FW_INFO
+#define FWCMD_H2C_FUNC_LOG_CFG 0x0
+#define FWCMD_H2C_FUNC_GENERAL_PKT 0x1
+#define FWCMD_H2C_FUNC_C2H_RPT_CFG 0x2
+#define FWCMD_H2C_FUNC_FW_DBGREG_CFG 0x3
+#define FWCMD_H2C_FUNC_FW_REDL_CFG 0x4
+#define FWCMD_H2C_FUNC_FW_SELF_DX 0x5
+#define FWCMD_H2C_FUNC_FW_DBG_ADDR_CFG 0x6
+
+// CLASS 1 - WOW
+#define FWCMD_H2C_FUNC_KEEP_ALIVE 0x0
+#define FWCMD_H2C_FUNC_DISCONNECT_DETECT 0x1
+#define FWCMD_H2C_FUNC_WOW_GLOBAL 0x2
+#define FWCMD_H2C_FUNC_GTK_OFLD 0x3
+#define FWCMD_H2C_FUNC_ARP_OFLD 0x4
+#define FWCMD_H2C_FUNC_NDP_OFLD 0x5
+#define FWCMD_H2C_FUNC_REALWOW 0x6
+#define FWCMD_H2C_FUNC_NLO 0x7
+#define FWCMD_H2C_FUNC_WAKEUP_CTRL 0x8
+#define FWCMD_H2C_FUNC_NEGATIVE_PATTERN 0x9
+#define FWCMD_H2C_FUNC_DEV2HST_GPIO 0xA
+#define FWCMD_H2C_FUNC_HST2DEV_CTRL 0xB
+#define FWCMD_H2C_FUNC_WOW_CAM_UPD 0xC
+#define FWCMD_H2C_FUNC_AOAC_REPORT_REQ 0xD
+#define FWCMD_H2C_FUNC_WOW_STOP_FW_TRX 0xE
+#define FWCMD_H2C_FUNC_WOW_CSA 0xF
+#define FWCMD_H2C_FUNC_MAGIC_WAKER_FILTER 0x10
+#define FWCMD_H2C_FUNC_TCP_KEEPALIVE 0x11
+#define FWCMD_H2C_FUNC_WOW_PAYLOAD_CAM_UPD 0x12
+
+// CLASS 2 - PS
+#define FWCMD_H2C_FUNC_LPS_PARM 0x0
+#define FWCMD_H2C_FUNC_P2P_ACT 0x1
+#define FWCMD_H2C_FUNC_P2P_MACID_CTRL 0x2
+#define FWCMD_H2C_FUNC_IPS_CFG 0x3
+#define FWCMD_H2C_FUNC_PS_ADVANCE_PARM 0x4
+#define FWCMD_H2C_FUNC_PERIODIC_WAKE 0x5
+#define FWCMD_H2C_FUNC_BB_RF_PWR_ST 0x6
+#define FWCMD_H2C_FUNC_PS_POWER_LEVEL 0x7
+#define FWCMD_H2C_FUNC_LPS_OPTION_CFG 0x8
+#define FWCMD_H2C_FUNC_DOZE_TIME_STATS 0x9
+#define FWCMD_H2C_FUNC_TBTT_TUNING 0xA
+#define FWCMD_H2C_FUNC_BKP_CAM_CFG 0xB
+#define FWCMD_H2C_FUNC_SMPS_CFG 0xC
+#define FWCMD_H2C_FUNC_RTD3_CFG 0xD
+
+// CLASS 3 - FWDL
+#define FWCMD_H2C_FUNC_FWHDR_DL 0x0
+#define FWCMD_H2C_FUNC_FWHDR_REDL 0x1
+
+// CLASS 4 - TWT
+#define FWCMD_H2C_FUNC_TWT_ANNOUNCE_UPD 0x00
+#define FWCMD_H2C_FUNC_TWTINFO_UPD 0x1
+#define FWCMD_H2C_FUNC_TWT_STANSP_UPD 0x2
+
+// CLASS 5 - Frame Exchange
+#define FWCMD_H2C_FUNC_TBLUD 0x0
+#define FWCMD_H2C_FUNC_DCTLINFO_UD 0x1
+#define FWCMD_H2C_FUNC_CCTLINFO_UD 0x2
+#define FWCMD_H2C_FUNC_F2P_TEST 0x3
+#define FWCMD_H2C_FUNC_SHCUT_UPDATE 0x4
+#define FWCMD_H2C_FUNC_BCN_UPD 0x5
+#define FWCMD_H2C_FUNC_SS_ULSTA_UPD 0x6
+#define FWCMD_H2C_FUNC_F2PDBG_SET 0x7
+#define FWCMD_H2C_FUNC_WLANINFO_GET 0x8
+#define FWCMD_H2C_FUNC_DCTLINFO_UD_V1 0x9
+#define FWCMD_H2C_FUNC_CCTLINFO_UD_V1 0xa
+#define FWCMD_H2C_FUNC_FW_STS_PARA 0xb
+#define FWCMD_H2C_FUNC_DCTLINFO_UD_V2 0xc
+#define FWCMD_H2C_FUNC_BCN_UPD_BE 0xd
+#define FWCMD_H2C_FUNC_BCN_IGNORE_EDCCA 0xe
+#define FWCMD_H2C_FUNC_BCN_UPD_V1 0xf
+#define FWCMD_H2C_FUNC_DCTLINFO_UD_V3 0x10
+#define FWCMD_H2C_FUNC_CCTLINFO_UD_G7 0x11
+#define FWCMD_H2C_FUNC_SWTX_STACFG 0x12
+#define FWCMD_H2C_FUNC_TXMODE_SWITCH 0x13
+#define FWCMD_H2C_FUNC_BCN_DYNAMIC_MECH 0x14
+
+// CLASS 6 - Address CAM
+#define FWCMD_H2C_FUNC_ADDRCAM_INFO 0x0
+#define FWCMD_H2C_FUNC_ADDRCAM_INFO_V1 0x0
+
+// CLASS 8 - Media Status Report
+#define FWCMD_H2C_FUNC_JOININFO 0x0
+#define FWCMD_H2C_FUNC_JOININFO_V1 0x0
+#define FWCMD_H2C_FUNC_DL_GRP_UPD 0x1
+#define FWCMD_H2C_FUNC_UL_GRP_UPD 0x2
+#define FWCMD_H2C_FUNC_MU_STA_UPD 0x3
+#define FWCMD_H2C_FUNC_FWROLE_MAINTAIN 0x4
+#define FWCMD_H2C_FUNC_NOTIFY_DBCC 0x5
+#define FWCMD_H2C_FUNC_DL_GRP_UPD_V2 0x1
+#define FWCMD_H2C_FUNC_FW_FLOW_CTRL 0x6
+#define FWCMD_H2C_FUNC_TXRPT_FORWARD 0x7
+#define FWCMD_H2C_FUNC_MRU_GRP_UPD 0x8
+
+// CLASS 9 - FW_OFLD
+#define FWCMD_H2C_FUNC_DUMP_EFUSE 0x0
+#define FWCMD_H2C_FUNC_PACKET_OFLD 0x1
+#define FWCMD_H2C_FUNC_READ_OFLD 0x2
+#define FWCMD_H2C_FUNC_WRITE_OFLD 0x3
+#define FWCMD_H2C_FUNC_CONF_OFLD 0x4
+#define FWCMD_H2C_FUNC_SYS_INIT 0x5
+#define FWCMD_H2C_FUNC_TRX_INIT 0x6
+#define FWCMD_H2C_FUNC_INTF_INIT 0x7
+#define FWCMD_H2C_FUNC_MACID_PAUSE 0x8
+#define FWCMD_H2C_FUNC_RX_FWD 0x9
+#define FWCMD_H2C_FUNC_EN_MAC_HDR_CONV 0xB
+#define FWCMD_H2C_FUNC_SET_HWSEQ_REG 0xA
+#define FWCMD_H2C_FUNC_HWAMSDU_REG 0xC
+#define FWCMD_H2C_FUNC_AMSDU_CUT_REG 0xD
+#define FWCMD_H2C_FUNC_TCPIP_CHKSUM_OFFLOAD_REG 0xE
+#define FWCMD_H2C_FUNC_USR_EDCA 0xF
+#define FWCMD_H2C_FUNC_TSF32_TOGL 0x10
+#define FWCMD_H2C_FUNC_CMD_OFLD_REG 0x11
+#define FWCMD_H2C_FUNC_USR_TX_RPT 0x12
+#define FWCMD_H2C_FUNC_CMD_OFLD_PKT 0x13
+#define FWCMD_H2C_FUNC_OFLD_CFG 0x14
+#define FWCMD_H2C_FUNC_H2C_AGG 0x15
+#define FWCMD_H2C_FUNC_ADD_SCANOFLD_CH 0x16
+#define FWCMD_H2C_FUNC_SCANOFLD 0x17
+#define FWCMD_H2C_FUNC_TX_DUTY 0x18
+#define FWCMD_H2C_FUNC_DISABLE_RF 0x19
+#define FWCMD_H2C_FUNC_TX_ADDBA_PKT 0x1A
+#define FWCMD_H2C_FUNC_PKT_DROP 0x1B
+#define FWCMD_H2C_FUNC_CH_SWITCH 0x1C
+#define FWCMD_H2C_FUNC_BCNERLYRPT 0x1D
+#define FWCMD_H2C_FUNC_CFG_BCNFLTR 0x1E
+#define FWCMD_H2C_FUNC_OFLD_RSSI 0x1F
+#define FWCMD_H2C_FUNC_OFLD_TP 0x20
+#define FWCMD_H2C_FUNC_HOST_EFUSE_REC 0x21
+#define FWCMD_H2C_FUNC_SCANOFLD_DRV_CTRL 0x22
+#define FWCMD_H2C_FUNC_WIFI_SENSING_CSI 0x23
+#define FWCMD_H2C_FUNC_CMD_OFLD_V1_REG 0x24
+#define FWCMD_H2C_FUNC_CMD_OFLD_V1_PKT 0x25
+#define FWCMD_H2C_FUNC_DLE_QTA_CHANGE 0x26
+#define FWCMD_H2C_FUNC_SCH_TX_EN_PKT 0x27
+#define FWCMD_H2C_FUNC_MACID_PAUSE_SLEEP 0x28
+#define FWCMD_H2C_FUNC_BCN_OFLD 0x29
+#define FWCMD_H2C_FUNC_TBTT_SH_STAT_OFLD 0x2A
+#define FWCMD_H2C_FUNC_STA_CSA 0x2B
+#define FWCMD_H2C_FUNC_SCANOFLD_V2 0x2C
+#define FWCMD_H2C_FUNC_FRAME_TO_ACT 0x2D
+#define FWCMD_H2C_FUNC_DUMP_EFUSE_V2 0x2E
+#define FWCMD_H2C_FUNC_TXRPT_STATS_EN 0x2F
+#define FWCMD_H2C_FUNC_CPUMGPKT_TX 0x30
+#define FWCMD_H2C_FUNC_BCN_CNT_GPIO 0x31
+#define FWCMD_H2C_FUNC_RX_FWD_BE 0x32
+#define FWCMD_H2C_FUNC_TRX_TIME_STATS_EN 0x33
+#define FWCMD_H2C_FUNC_APF_OFLD_CTRL 0x36
+
+// CLASS 10 - SECCAM
+#define FWCMD_H2C_FUNC_SEC_CAM_INIT 0x0
+#define FWCMD_H2C_FUNC_SECCAM_INFO 0x1
+#define FWCMD_H2C_FUNC_SECCAM_INFO_V1 0x2
+
+// CLASS 11 - Sound
+#define FWCMD_H2C_FUNC_SET_SND_PARA 0x0
+#define FWCMD_H2C_FUNC_GET_CSI_BUF 0x1
+#define FWCMD_H2C_FUNC_SET_CSI_BUF 0x2
+#define FWCMD_H2C_FUNC_GET_SND_STS 0x3
+#define FWCMD_H2C_FUNC_SET_SND_STS 0x4
+#define FWCMD_H2C_FUNC_INIT_SND_MER 0x5
+#define FWCMD_H2C_FUNC_INIT_SND_MEE 0x6
+#define FWCMD_H2C_FUNC_CSI_FIX_RATE 0x7
+#define FWCMD_H2C_FUNC_CSI_RRSC 0x8
+#define FWCMD_H2C_FUNC_SET_MU_TABLE 0x9
+#define FWCMD_H2C_FUNC_SET_CSI_PARA_REG 0xA
+#define FWCMD_H2C_FUNC_HW_SND_PR 0xB
+#define FWCMD_H2C_FUNC_BYPASS_SND_STS 0xC
+#define FWCMD_H2C_FUNC_SET_SND_PARA_V1 0xD
+#define FWCMD_H2C_FUNC_SET_SND_PARA_V2 0xE
+
+// CLASS 12 - BACAM
+#define FWCMD_H2C_FUNC_BA_CAM 0x0
+#define FWCMD_H2C_FUNC_BA_CAM_V1 0x1
+#define FWCMD_H2C_FUNC_BA_CAM_INIT 0x2
+
+// CLASS 13 - IECAM
+#define FWCMD_H2C_FUNC_IE_CAM 0x0
+
+// CLASS 14 - MCC
+#define FWCMD_H2C_FUNC_ADD_MCC 0x0
+#define FWCMD_H2C_FUNC_START_MCC 0x1
+#define FWCMD_H2C_FUNC_STOP_MCC 0x2
+#define FWCMD_H2C_FUNC_DEL_MCC_GROUP 0x3
+#define FWCMD_H2C_FUNC_RESET_MCC_GROUP 0x4
+#define FWCMD_H2C_FUNC_MCC_REQ_TSF 0x5
+#define FWCMD_H2C_FUNC_MCC_MACID_BITMAP 0x6
+#define FWCMD_H2C_FUNC_MCC_SYNC 0x7
+#define FWCMD_H2C_FUNC_MCC_SET_DURATION 0x8
+#define FWCMD_H2C_FUNC_MCC_DBG_NOTIFY 0x9
+
+// CLASS 15 - SCSI
+#define FWCMD_H2C_FUNC_SCSI_TX 0x0
+#define FWCMD_H2C_FUNC_USB_SWITCH 0x1
+
+// CLASS 16 - FLASH
+#define FWCMD_H2C_FUNC_PLAT_FLASH_WRITE 0x0
+#define FWCMD_H2C_FUNC_PLAT_FLASH_ERASE 0x1
+#define FWCMD_H2C_FUNC_PLAT_FLASH_READ 0x2
+
+// CLASS 17 - FCS
+#define FWCMD_H2C_FUNC_FCS 0x0
+
+// CLASS 18 - MISC
+#define FWCMD_H2C_FUNC_CFG_WPS 0x0
+
+// CLASS 19 - MPORT
+#define FWCMD_H2C_FUNC_PORT_INIT 0x0
+#define FWCMD_H2C_FUNC_PORT_CFG 0x1
+
+// CLASS 20 - MLO
+#define FWCMD_H2C_FUNC_MLOTBL_CFG 0x0
+#define FWCMD_H2C_FUNC_MLO_STAFUNC_CFG 0x1
+#define FWCMD_H2C_FUNC_MLO_TID_TO_LINK_MAP 0x2
+#define FWCMD_H2C_FUNC_MLO_DM_CFG 0x3
+#define FWCMD_H2C_FUNC_EMLSR_STA_CFG 0x4
+#define FWCMD_H2C_FUNC_MCMLO_RELINK_DROP 0x5
+#define FWCMD_H2C_FUNC_MCMLO_SN_SYNC 0x6
+#define FWCMD_H2C_FUNC_MLO_RELINK 0x7
+#define FWCMD_H2C_FUNC_MLO_LINK_CFG 0x8
+#define FWCMD_H2C_FUNC_MLO_DM_DBG 0x9
+
+// CLASS 21 - PROXY
+#define FWCMD_H2C_FUNC_PROXY 0x0
+#define FWCMD_H2C_FUNC_MDNS 0x1
+#define FWCMD_H2C_FUNC_PTCL_PATTERN 0x2
+#define FWCMD_H2C_FUNC_SNMP 0x3
+#define FWCMD_H2C_FUNC_LLMNR 0x4
+#define FWCMD_H2C_FUNC_MDNS_OFLD 0x8
+#define FWCMD_H2C_FUNC_APF_SET 0x10
+#define FWCMD_H2C_FUNC_APF_GET 0x11
+
+// CLASS 22 - NAN
+#define FWCMD_H2C_FUNC_ACT_SCHEDULE_REQ 0x0
+#define FWCMD_H2C_FUNC_BCN_REQ 0x1
+#define FWCMD_H2C_FUNC_NAN_FUNC_CTRL 0x2
+#define FWCMD_H2C_FUNC_NAN_DE_INFO 0x3
+#define FWCMD_H2C_FUNC_NAN_JOIN_CLUSTER 0x4
+#define FWCMD_H2C_FUNC_PAUSE_FAW_TX 0x5
+#define FWCMD_H2C_FUNC_NAN_GET_CLUSTER_INFO 0x6
+#define FWCMD_H2C_FUNC_NAN_AVAIL_T_BITMAP 0x7
+
+// CLASS 23 - FW_CMDREQ
+#define FWCMD_H2C_FUNC_FW_CMDREQ 0x0
+
+// CLASS 24 - MRC
+#define FWCMD_H2C_FUNC_MRC_REQ_TSF 0x0
+#define FWCMD_H2C_FUNC_ADD_MRC 0x1
+#define FWCMD_H2C_FUNC_START_MRC 0x2
+#define FWCMD_H2C_FUNC_DEL_MRC 0x3
+#define FWCMD_H2C_FUNC_MRC_SYNC 0x4
+#define FWCMD_H2C_FUNC_MRC_UPD_DURATION 0x5
+#define FWCMD_H2C_FUNC_MRC_UPD_BITMAP 0x6
+#define FWCMD_H2C_FUNC_MRC_DBG_DUMP_CFG 0x7
+
+// CLASS 25 - BF_SENS
+#define FWCMD_H2C_FUNC_SENS_TBL_UPD 0x0
+#define FWCMD_H2C_FUNC_SENS_NTB_TX_OFLD 0x1
+#define FWCMD_H2C_FUNC_SENS_INVLD_SMR 0x2
+
+// CLASS 26 - OFFLOAD_AGG
+#define FWCMD_H2C_FUNC_SUB_OFFLOAD_AGG 0x0
+#define FWCMD_H2C_FUNC_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE 0x1
+#define FWCMD_H2C_FUNC_SUB_OFFLOAD_AGG_WITH_NUM 0x2
+
+// CLASS 27 - ECO_PATCH
+#define FWCMD_H2C_FUNC_OM_ADV_RPT 0x0
+
+// CLASS 28 - SER
+#define FWCMD_H2C_FUNC_SER_DBG_CASE_SET 0x0
+
+// CLASS 29 - NAT2.5
+#define FWCMD_H2C_FUNC_NAT2P5_UPD 0x0
+
+// CLASS 30 - FTM
+#define FWCMD_H2C_FUNC_AZ_EN 0x0
+#define FWCMD_H2C_FUNC_AZ_PROC 0x1
+#define FWCMD_H2C_FUNC_AZ_UPDATE 0x2
+#define FWCMD_H2C_FUNC_AZ_CANCEL 0x3
+
+// CLASS 31 - WOW_DIAG
+#define FWCMD_H2C_FUNC_WOW_DIAG_CTRL 0x0
+
+// CLASS 32 - WOW_TRI_EVT
+#define FWCMD_H2C_FUNC_WOW_REQ_RX_PKT 0x0
+#define FWCMD_H2C_FUNC_WOW_REQ_MEM 0x1
+#define FWCMD_H2C_FUNC_WOW_REQ_ROLE_INFO 0x2
+#define FWCMD_H2C_FUNC_WOW_REQ_BB_RF_REG 0x3
+
+// CLASS 33 - AP_SELF_DIAG
+#define FWCMD_H2C_FUNC_AP_SELF_DIAG 0x0
+
+// CLASS 34 - STA_DIAG
+#define FWCMD_H2C_FUNC_REQ_STA_DIAG_RPT 0x0
+
+// CLASS 35 - ADDBA_OFLD
+#define FWCMD_H2C_FUNC_ADDBA_OFLD_REQ 0x0
+
+// CLASS 36 - SoftAP
+#define FWCMD_H2C_FUNC_SAP_INFO 0x0
+
+/*----------------------------------- H2CPKT - CAT(Table) --------------------------*/
+
+/*----------------------------------- H2CPKT - CAT(OutSrc,Phydm) -------------------*/
+#define FWCMD_H2C_CAT_TEST 0x0
+#define FWCMD_H2C_CAT_MAC 0x1
+#define FWCMD_H2C_CAT_OUTSRC 0x2
+#define FWCMD_H2C_CL_RUA 0x0
+#define FWCMD_H2C_CL_RA 0x1
+
+// CLASS 0 (RUA)
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 1 (RA)
+#define FWCMD_H2C_FUNC_MACID_CFG 0x0
+#define FWCMD_H2C_FUNC_RSSI_RA_CONFIG 0x1
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 2
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 3
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 4
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 5
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 6
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 7
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+/*----------------------------------- H2CPKT - CAT(OutSrc, RF) ---------------------*/
+#define FWCMD_H2C_CAT_TEST 0x0
+#define FWCMD_H2C_CAT_MAC 0x1
+#define FWCMD_H2C_CAT_OUTSRC 0x2
+
+// CLASS 8
+#define FWCMD_H2C_FUNC_RADIO_INIT_0 0x0
+#define FWCMD_H2C_FUNC_RADIO_INIT_1 0x1
+#define FWCMD_H2C_FUNC_RADIO_INIT_2 0x2
+#define FWCMD_H2C_FUNC_RADIO_INIT_3 0x3
+#define FWCMD_H2C_FUNC_RADIO_INIT_4 0x4
+#define FWCMD_H2C_FUNC_RADIO_INIT_5 0x5
+#define FWCMD_H2C_FUNC_RADIO_INIT_6 0x6
+#define FWCMD_H2C_FUNC_RADIO_INIT_7 0x7
+#define FWCMD_H2C_FUNC_RADIO_INIT_8 0x8
+#define FWCMD_H2C_FUNC_RADIO_INIT_9 0x9
+
+// CLASS 9
+#define FWCMD_H2C_FUNC_DACK_BACKUP 0x0
+#define FWCMD_H2C_FUNC_DACK_RELOAD 0x1
+
+// CLASS 10
+#define FWCMD_H2C_FUNC_FCS_BACKUP_RFK 0x0
+#define FWCMD_H2C_FUNC_FCS_RELOAD_RFK 0x1
+#define FWCMD_H2C_FUNC_GET_MCCCH 0x2
+#define FWCMD_H2C_FUNC_FWIQK_TRIGGER 0x4
+#define FWCMD_H2C_FUNC_FW_SCAN_LOAD_PWR_TABLE 0x5
+#define FWCMD_H2C_FUNC_BCN_MOD 0x7
+#define FWCMD_H2C_FUNC_FT_RFQ 0x8
+#define FWCMD_H2C_FUNC_LPS_CFG 0x9
+#define FWCMD_H2C_FUNC_RF_REG_FCS 0xA
+#define FWCMD_H2C_FUNC_BB_REG_FCS 0xB
+#define FWCMD_H2C_FUNC_KIP_REG_FCS 0xC
+#define FWCMD_H2C_FUNC_MCC_CFG 0xD
+#define FWCMD_H2C_FUNC_RF_INIT_CFG 0xE
+
+// CLASS 11
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 12
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 13
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 14
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+// CLASS 15
+#define FWCMD_H2C_FUNC_EXAMPLE 0x0
+
+/*----------------------------------- H2CPKT - CAT(OutSrc, BTC) --------------------*/
+#define FWCMD_H2C_CAT_TEST 0x0
+#define FWCMD_H2C_CAT_MAC 0x1
+#define FWCMD_H2C_CAT_OUTSRC 0x2
+
+// CLASS 16
+#define FWCMD_H2C_FUNC_TDMA 0x0
+
+/*----------------------------------- C2HPKT - CAT(TEST) ---------------------------*/
+#define FWCMD_C2H_CAT_TEST 0x0
+#define FWCMD_C2H_CAT_MAC 0x1
+#define FWCMD_C2H_CAT_OUTSRC 0x2
+#define FWCMD_C2H_CL_CMD_PATH 0x0
+#define FWCMD_C2H_CL_PLAT_AUTO 0x2
+#define FWCMD_C2H_CL_FW_AUTO 0x3
+#define FWCMD_C2H_CL_FW_STATUS 0x4
+
+// CLASS 0 - CMD_PATH
+#define FWCMD_C2H_FUNC_C2H_LB 0x00
+#define FWCMD_C2H_FUNC_TEST_PHY_RPT 0x01
+#define FWCMD_C2H_FUNC_C2H_LOG_TEST 0x2
+#define FWCMD_C2H_FUNC_SEND_NULL_TEST 0x3
+#define FWCMD_C2H_FUNC_C2H_L1_2_TEST 0x4
+
+// CLASS 2 - PLATFORM_AUTO_TEST
+#define FWCMD_C2H_FUNC_SPIC 0x00
+#define FWCMD_C2H_FUNC_SPIC2 0x01
+
+// CLASS 3 - FW_AUTO_TEST
+#define FWCMD_C2H_FUNC_LPS_TEST 0x00
+#define FWCMD_C2H_FUNC_LPS_ONOFF_TEST 0x1
+#define FWCMD_C2H_FUNC_MPIO_TEST 0x2
+#define FWCMD_C2H_FUNC_SENDNULL_TEST 0x3
+#define FWCMD_C2H_FUNC_MPIOV2_TEST 0x4
+#define FWCMD_C2H_FUNC_MIO_TEST 0x5
+#define FWCMD_C2H_FUNC_ANYTESTBUSYCALL_TEST 0x6
+#define FWCMD_C2H_FUNC_ANYRW_TEST 0x7
+
+// CLASS 4 - FW_STATUS
+#define FWCMD_C2H_FUNC_FW_STATUS 0x00
+#define FWCMD_C2H_FUNC_TRIGGER_CPU_EXCEPTION 0x1
+#define FWCMD_C2H_FUNC_FW_PROFILING 0x2
+
+// CLASS 5 - MAC_TEST
+#define FWCMD_C2H_FUNC_CPUIO_TEST 0x00
+
+/*----------------------------------- C2HPKT - CAT(MAC) ----------------------------*/
+#define FWCMD_C2H_CAT_TEST 0x0
+#define FWCMD_C2H_CAT_MAC 0x1
+#define FWCMD_C2H_CAT_OUTSRC 0x2
+#define FWCMD_C2H_CL_FW_INFO 0x0
+#define FWCMD_C2H_CL_FW_OFLD 0x1
+#define FWCMD_C2H_CL_TWT 0x2
+#define FWCMD_C2H_CL_WOW 0x3
+#define FWCMD_C2H_CL_MCC 0x4
+#define FWCMD_C2H_CL_FW_DBG 0x5
+#define FWCMD_C2H_CL_FLASH 0x6
+#define FWCMD_C2H_CL_SCSI 0x7
+#define FWCMD_C2H_CL_FCS 0x8
+#define FWCMD_C2H_CL_MISC 0x9
+#define FWCMD_C2H_CL_MPORT 0xA
+#define FWCMD_C2H_CL_OFDMA_INFO 0xB
+#define FWCMD_C2H_CL_MLO 0xC
+#define FWCMD_C2H_CL_NAN 0xD
+#define FWCMD_C2H_CL_MRC 0xE
+#define FWCMD_C2H_CL_PS 0xF
+#define FWCMD_C2H_CL_DCTL_INFO 0x10
+#define FWCMD_C2H_CL_ECO_PATCH 0x11
+#define FWCMD_C2H_CL_WOW_DIAG 0x12
+#define FWCMD_C2H_CL_WOW_TRI_EVT 0x13
+#define FWCMD_C2H_CL_FTM_AZ 0x14
+#define FWCMD_C2H_CL_FTM_MC 0x15
+#define FWCMD_C2H_CL_AP_SELF_DIAG_RST 0x16
+#define FWCMD_C2H_CL_STA_DIAG 0x17
+#define FWCMD_C2H_CL_SOFTAP 0x18
+
+// CLASS 0 - FW_INFO
+#define FWCMD_C2H_FUNC_REC_ACK 0x00
+#define FWCMD_C2H_FUNC_DONE_ACK 0x01
+#define FWCMD_C2H_FUNC_C2H_LOG 0x02
+#define FWCMD_C2H_FUNC_BCN_CNT 0x03
+#define FWCMD_C2H_FUNC_BCN_CSAZERO 0x04
+#define FWCMD_C2H_FUNC_BCN_CRITICAL_RST 0x05
+#define FWCMD_C2H_FUNC_BCN_UPD_DONE 0x06
+#define FWCMD_C2H_FUNC_BCN_BC_CHG_ZERO 0x07
+#define FWCMD_C2H_FUNC_FW_SELF_DX_INFO 0x08
+
+// CLASS 1 - FW_OFLD
+#define FWCMD_C2H_FUNC_EFUSE_DUMP 0x00
+#define FWCMD_C2H_FUNC_READ_RSP 0x01
+#define FWCMD_C2H_FUNC_PKT_OFLD_RSP 0x02
+#define FWCMD_C2H_FUNC_BEACON_RESEND 0x3
+#define FWCMD_C2H_FUNC_MACID_PAUSE 0x4
+#define FWCMD_C2H_FUNC_FW_GETPKT_RPT 0x5
+#define FWCMD_C2H_FUNC_TSF32_TOGL_RPT 0x6
+#define FWCMD_C2H_FUNC_USR_TX_RPT_INFO 0x7
+#define FWCMD_C2H_FUNC_CMD_OFLD_RSP 0x8
+#define FWCMD_C2H_FUNC_SCANOFLD_RSP 0x9
+#define FWCMD_C2H_FUNC_TX_DUTY_RPT 0xA
+#define FWCMD_C2H_FUNC_CH_SWITCH_RPT 0xB
+#define FWCMD_C2H_FUNC_BCNERLYNTFY 0xC
+#define FWCMD_C2H_FUNC_BCNFLTR_RPT 0xD
+#define FWCMD_C2H_FUNC_WIFI_SENSING_CSI_TX_RESULT 0xE
+#define FWCMD_C2H_FUNC_TBTT_SH_STAT_RPT 0xF
+#define FWCMD_C2H_FUNC_BCN_OFLD_RPT 0x10
+#define FWCMD_C2H_FUNC_FRAME_TO_ACT_RPT 0x11
+#define FWCMD_C2H_FUNC_EFUSE_DUMP_V2 0x12
+#define FWCMD_C2H_FUNC_TXRPT_STATS 0x13
+#define FWCMD_C2H_FUNC_BCN_SYNC_RPT 0x14
+#define FWCMD_C2H_FUNC_TRX_TIME_STATS 0x15
+#define FWCMD_C2H_FUNC_APF_OFLD_RPT 0x16
+
+// CLASS 2 - TWT
+#define FWCMD_C2H_FUNC_WAIT_ANNOUNCE 0x00
+#define FWCMD_C2H_FUNC_STAT_RPT 0x01
+
+// CLASS 3 - WOW
+#define FWCMD_C2H_FUNC_AOAC_REPORT 0x00
+#define FWCMD_C2H_FUNC_READ_WOW_CAM 0x01
+#define FWCMD_C2H_FUNC_APF_RPT 0x02
+
+// CLASS 4 - MCC
+#define FWCMD_C2H_FUNC_MCC_RCV_ACK 0x00
+#define FWCMD_C2H_FUNC_MCC_REQ_ACK 0x01
+#define FWCMD_C2H_FUNC_MCC_TSF_RPT 0x02
+#define FWCMD_C2H_FUNC_MCC_STATUS_RPT 0x03
+
+// CLASS 5 - FW_DBG
+#define FWCMD_C2H_FUNC_RX_DBG 0x00
+
+// CLASS 6 - FLASH
+#define FWCMD_C2H_FUNC_PLAT_FLASH_WRITE 0x0
+#define FWCMD_C2H_FUNC_PLAT_FLASH_ERASE 0x1
+#define FWCMD_C2H_FUNC_PLAT_FLASH_READ 0x2
+
+// CLASS 7 - SCSI
+#define FWCMD_C2H_FUNC_SCSI_RX 0x00
+
+// CLASS 8 - FCS
+#define FWCMD_C2H_FUNC_FCS_RPT 0x00
+
+// CLASS 9 - MISC
+#define FWCMD_C2H_FUNC_WPS_RPT 0x0
+#define FWCMD_C2H_FUNC_CCXRPT 0x1
+#define FWCMD_C2H_FUNC_BF_SENS_FEEDBACK 0x4
+#define FWCMD_C2H_FUNC_RX_FORWARD 0x5
+
+// CLASS 10 - MPORT
+#define FWCMD_C2H_FUNC_PORT_INIT_STAT 0x0
+#define FWCMD_C2H_FUNC_PORT_CFG_STAT 0x1
+
+// CLASS 11 - OFDMA_INFO
+#define FWCMD_C2H_FUNC_OFDMA_STS 0x0
+#define FWCMD_C2H_FUNC_OFDMA_STS_V1 0x0
+
+// CLASS 12 - MLO
+#define FWCMD_C2H_FUNC_MLO_GET_TBL 0x0
+#define FWCMD_C2H_FUNC_EMLSR_TRANS_DONE 0x1
+#define FWCMD_C2H_FUNC_EMLSR_STA_CFG_DONE 0x2
+#define FWCMD_C2H_FUNC_MCMLO_RELINK_RPT 0x3
+#define FWCMD_C2H_FUNC_MCMLO_SN_SYNC_RPT 0x4
+#define FWCMD_C2H_FUNC_MLO_LINK_CFG_STAT 0x5
+#define FWCMD_C2H_FUNC_MLO_DM_DBG_DUMP 0x6
+#define FWCMD_C2H_FUNC_MLO_DM_STATUS_DUMP 0x7
+
+// CLASS 13 - NAN
+#define FWCMD_C2H_FUNC_ACT_SCHEDULE_REQ_ACK 0x0
+#define FWCMD_C2H_FUNC_BCN_REQ_ACK 0x1
+#define FWCMD_C2H_FUNC_NAN_FUNC_CTRL_ACK 0x2
+#define FWCMD_C2H_FUNC_NAN_DE_INFO_ACK 0x3
+#define FWCMD_C2H_FUNC_NAN_JOIN_CLUSTER_ACK 0x4
+#define FWCMD_C2H_FUNC_NAN_PAUSE_FAW_TX_ACK 0x5
+#define FWCMD_C2H_FUNC_NAN_INFO_NOTIFY_CLUSTER_INFO 0x6
+#define FWCMD_C2H_FUNC_NAN_INFO_NOTIFY_TSF_INFO 0x7
+#define FWCMD_C2H_FUNC_NAN_INFO_NOTIFY_CLUSTER_JOIN 0x8
+#define FWCMD_C2H_FUNC_NAN_DEINIT_RPT 0x9
+
+// CLASS 14 - MRC
+#define FWCMD_C2H_FUNC_MRC_TSF_RPT 0x0
+#define FWCMD_C2H_FUNC_MRC_STATUS_RPT 0x1
+#define FWCMD_C2H_FUNC_MRC_NOTIFY_EVT 0x2
+
+// CLASS 15 - PS
+#define FWCMD_C2H_FUNC_DOZE_TIME_RPT 0x0
+
+// CLASS 16 - OFFLOAD_AGG
+#define FWCMD_C2H_FUNC_DCTL_INFO_DWORD 0x0
+
+// CLASS 17 - ECO_PATCH
+#define FWCMD_C2H_FUNC_OM_ADV_RPT 0x0
+
+// CLASS 18 - WOW_DIAG
+#define FWCMD_C2H_FUNC_WOW_DIAG_RPT 0x00
+#define FWCMD_C2H_FUNC_WOW_DIAG_GTK 0x01
+#define FWCMD_C2H_FUNC_WOW_DIAG_AP_LOST 0x02
+#define FWCMD_C2H_FUNC_WOW_DIAG_SER 0x03
+#define FWCMD_C2H_FUNC_WOW_DIAG_RX_EVT 0x04
+#define FWCMD_C2H_FUNC_WOW_DIAG_KEEPALIVE 0x05
+#define FWCMD_C2H_FUNC_WOW_DIAG_ARP 0x06
+#define FWCMD_C2H_FUNC_WOW_DIAG_NS 0x07
+#define FWCMD_C2H_FUNC_WOW_DIAG_PMF 0x08
+#define FWCMD_C2H_FUNC_WOW_DIAG_PER_WAKE 0x09
+#define FWCMD_C2H_FUNC_WOW_DIAG_NLO 0x0A
+#define FWCMD_C2H_FUNC_WOW_DIAG_STA_CSA 0x0B
+#define FWCMD_C2H_FUNC_WOW_DIAG_TCP_KEEPALIVE 0x0C
+#define FWCMD_C2H_FUNC_WOW_DIAG_PROXY_OFLD 0x0D
+
+// CLASS 19 - WOW_TRI_EVT
+#define FWCMD_C2H_FUNC_WOW_DUMP_RX 0x00
+#define FWCMD_C2H_FUNC_WOW_DUMP_MEM 0x01
+#define FWCMD_C2H_FUNC_WOW_DUMP_ROLE 0x02
+#define FWCMD_C2H_FUNC_WOW_DUMP_BB_RF_REG 0x03
+
+// CLASS 20 - FTM_AZ
+#define FWCMD_C2H_FUNC_FTM_AZ_PKTID_CONTENT 0x00
+#define FWCMD_C2H_FUNC_FTM_AZ_PARA_DUMP 0x01
+#define FWCMD_C2H_FUNC_FTM_AZ_DBG_DUMP 0x02
+#define FWCMD_C2H_FUNC_FTM_AZ_RPT_HDL 0x03
+
+// CLASS 21 - FTM_MC
+#define FWCMD_C2H_FUNC_FTM_MC_PKTID_CONTENT 0x00
+
+// CLASS 22 - AP_SELF_DIAG_RST
+#define FWCMD_C2H_FUNC_AP_SELF_DIAG_RST 0x00
+
+// CLASS 23 - STA_DIAG
+#define FWCMD_C2H_FUNC_STA_DIAG_RPT 0x00
+#define FWCMD_C2H_FUNC_STA_DIAG_SCAN 0x01
+#define FWCMD_C2H_FUNC_STA_DIAG_CSA 0x02
+#define FWCMD_C2H_FUNC_STA_DIAG_PS 0x03
+#define FWCMD_C2H_FUNC_STA_DIAG_MCC 0x04
+#define FWCMD_C2H_FUNC_STA_DIAG_EFUSE 0x05
+#define FWCMD_C2H_FUNC_STA_DIAG_MRC 0x06
+
+// CLASS 24 - SoftAP
+#define FWCMD_C2H_FUNC_PWR_INT_NOTIFY 0x00
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, Phydm) ------------------*/
+#define FWCMD_C2H_CAT_TEST 0x0
+#define FWCMD_C2H_CAT_MAC 0x1
+#define FWCMD_C2H_CAT_OUTSRC 0x2
+#define FWCMD_C2H_CL_RUA 0x0
+#define FWCMD_C2H_CL_RA 0x1
+#define FWCMD_C2H_CL_HALBB 0x2
+#define FWCMD_C2H_CL_DBG 0x7
+
+// CLASS 0
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 1
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 2
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 3
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 4
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 5
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 6
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 7
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, RF) ---------------------*/
+#define FWCMD_C2H_CAT_TEST 0x0
+#define FWCMD_C2H_CAT_MAC 0x1
+#define FWCMD_C2H_CAT_OUTSRC 0x2
+
+// CLASS 9 - HALRF_C2H_REPORT
+#define FWCMD_C2H_FUNC_RF_REPORT 0x0
+
+// CLASS 10
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 11
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 12
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 13
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 14
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// CLASS 15
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, BTC) --------------------*/
+#define FWCMD_C2H_CAT_TEST 0x0
+#define FWCMD_C2H_CAT_MAC 0x1
+#define FWCMD_C2H_CAT_OUTSRC 0x2
+
+// CLASS 16
+#define FWCMD_C2H_FUNC_EXAMPLE 0x0
+
+// Bit definition//
+
+/*----------------------------------- H2CREG ---------------------------------------*/
+#define FWCMD_H2CREG_H2CREG_HDR_FUNC_SH 0
+#define FWCMD_H2CREG_H2CREG_HDR_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_H2CREG_HDR_ACK BIT(7)
+#define FWCMD_H2CREG_H2CREG_HDR_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_H2CREG_HDR_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_H2CREG_HDR_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_H2CREG_HDR_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_H2CREG_DWORD0_SH 0
+#define FWCMD_H2CREG_H2CREG_DWORD0_MSK 0xffffffff
+#define FWCMD_H2CREG_H2CREG_DWORD1_SH 0
+#define FWCMD_H2CREG_H2CREG_DWORD1_MSK 0xffffffff
+#define FWCMD_H2CREG_H2CREG_DWORD2_SH 0
+#define FWCMD_H2CREG_H2CREG_DWORD2_MSK 0xffffffff
+#define FWCMD_H2CREG_H2CREG_DWORD3_SH 0
+#define FWCMD_H2CREG_H2CREG_DWORD3_MSK 0xffffffff
+#define FWCMD_H2CREG_H2CREG_LB_FUNC_SH 0
+#define FWCMD_H2CREG_H2CREG_LB_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_H2CREG_LB_ACK BIT(7)
+#define FWCMD_H2CREG_H2CREG_LB_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_H2CREG_LB_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_H2CREG_LB_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_H2CREG_LB_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_H2CREG_LB_PAYLOAD0_SH 16
+#define FWCMD_H2CREG_H2CREG_LB_PAYLOAD0_MSK 0xffff
+#define FWCMD_H2CREG_H2CREG_LB_PAYLOAD1_SH 0
+#define FWCMD_H2CREG_H2CREG_LB_PAYLOAD1_MSK 0xffffffff
+#define FWCMD_H2CREG_CNSL_CMD_FUNC_SH 0
+#define FWCMD_H2CREG_CNSL_CMD_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_CNSL_CMD_ACK BIT(7)
+#define FWCMD_H2CREG_CNSL_CMD_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_CNSL_CMD_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_CNSL_CMD_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_CNSL_CMD_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_CNSL_CMD_CMD_ID_SH 16
+#define FWCMD_H2CREG_CNSL_CMD_CMD_ID_MSK 0xff
+#define FWCMD_H2CREG_CNSL_CMD_ARG_NUM_SH 24
+#define FWCMD_H2CREG_CNSL_CMD_ARG_NUM_MSK 0xf
+#define FWCMD_H2CREG_CNSL_CMD_ARG_UNIT_SH 28
+#define FWCMD_H2CREG_CNSL_CMD_ARG_UNIT_MSK 0x3
+#define FWCMD_H2CREG_FWERR_FUNC_SH 0
+#define FWCMD_H2CREG_FWERR_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_FWERR_ACK BIT(7)
+#define FWCMD_H2CREG_FWERR_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_FWERR_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_FWERR_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_FWERR_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_FWERR_OP_SH 16
+#define FWCMD_H2CREG_FWERR_OP_MSK 0xf
+#define FWCMD_H2CREG_FWERR_IDX_SH 0
+#define FWCMD_H2CREG_FWERR_IDX_MSK 0xffffffff
+#define FWCMD_H2CREG_HIDDEN_GET_FUNC_SH 0
+#define FWCMD_H2CREG_HIDDEN_GET_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_HIDDEN_GET_ACK BIT(7)
+#define FWCMD_H2CREG_HIDDEN_GET_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_HIDDEN_GET_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_HIDDEN_GET_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_HIDDEN_GET_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_HIDDEN_GET_PART_NUM_SH 16
+#define FWCMD_H2CREG_HIDDEN_GET_PART_NUM_MSK 0xff
+#define FWCMD_H2CREG_GETPKT_INFORM_FUNC_SH 0
+#define FWCMD_H2CREG_GETPKT_INFORM_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_GETPKT_INFORM_ACK BIT(7)
+#define FWCMD_H2CREG_GETPKT_INFORM_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_GETPKT_INFORM_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_GETPKT_INFORM_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_GETPKT_INFORM_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_GETPKT_INFORM_MACID_SH 16
+#define FWCMD_H2CREG_GETPKT_INFORM_MACID_MSK 0xff
+#define FWCMD_H2CREG_GETPKT_INFORM_PKTTYPE_SH 24
+#define FWCMD_H2CREG_GETPKT_INFORM_PKTTYPE_MSK 0xff
+#define FWCMD_H2CREG_GETPKT_INFORM_MACID_EXT_SH 0
+#define FWCMD_H2CREG_GETPKT_INFORM_MACID_EXT_MSK 0xff
+#define FWCMD_H2CREG_GETPKT_INFORM_BAND_SH 8
+#define FWCMD_H2CREG_GETPKT_INFORM_BAND_MSK 0xff
+#define FWCMD_H2CREG_SCH_TX_EN_FUNC_SH 0
+#define FWCMD_H2CREG_SCH_TX_EN_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_SCH_TX_EN_ACK BIT(7)
+#define FWCMD_H2CREG_SCH_TX_EN_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_SCH_TX_EN_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_SCH_TX_EN_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_SCH_TX_EN_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_BE0 BIT(16)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_BK0 BIT(17)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_VI0 BIT(18)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_VO0 BIT(19)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_BE1 BIT(20)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_BK1 BIT(21)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_VI1 BIT(22)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_VO1 BIT(23)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_MG0 BIT(24)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_MG1 BIT(25)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_MG2 BIT(26)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_HI BIT(27)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_BCN BIT(28)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_UL BIT(29)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_TWT0 BIT(30)
+#define FWCMD_H2CREG_SCH_TX_EN_TX_EN_TWT1 BIT(31)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_BE0 BIT(0)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_BK0 BIT(1)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_VI0 BIT(2)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_VO0 BIT(3)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_BE1 BIT(4)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_BK1 BIT(5)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_VI1 BIT(6)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_VO1 BIT(7)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_MG0 BIT(8)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_MG1 BIT(9)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_MG2 BIT(10)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_HI BIT(11)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_BCN BIT(12)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_UL BIT(13)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_TWT0 BIT(14)
+#define FWCMD_H2CREG_SCH_TX_EN_MASK_TWT1 BIT(15)
+#define FWCMD_H2CREG_SCH_TX_EN_BAND BIT(16)
+#define FWCMD_H2CREG_WOW_TRX_STOP_FUNC_SH 0
+#define FWCMD_H2CREG_WOW_TRX_STOP_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_WOW_TRX_STOP_ACK BIT(7)
+#define FWCMD_H2CREG_WOW_TRX_STOP_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_WOW_TRX_STOP_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_WOW_TRX_STOP_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_WOW_TRX_STOP_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_AOAC_RPT_1_FUNC_SH 0
+#define FWCMD_H2CREG_AOAC_RPT_1_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_AOAC_RPT_1_ACK BIT(7)
+#define FWCMD_H2CREG_AOAC_RPT_1_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_AOAC_RPT_1_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_AOAC_RPT_1_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_AOAC_RPT_1_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_AOAC_RPT_1_RPT_SH 16
+#define FWCMD_H2CREG_AOAC_RPT_1_RPT_MSK 0xffff
+#define FWCMD_H2CREG_AOAC_RPT_2_FUNC_SH 0
+#define FWCMD_H2CREG_AOAC_RPT_2_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_AOAC_RPT_2_ACK BIT(7)
+#define FWCMD_H2CREG_AOAC_RPT_2_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_AOAC_RPT_2_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_AOAC_RPT_2_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_AOAC_RPT_2_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_AOAC_RPT_2_RPT_SH 16
+#define FWCMD_H2CREG_AOAC_RPT_2_RPT_MSK 0xffff
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_FUNC_SH 0
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_ACK BIT(7)
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_RPT_SH 16
+#define FWCMD_H2CREG_AOAC_RPT_3_REQ_RPT_MSK 0xffff
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_FUNC_SH 0
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_ACK BIT(7)
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_FW_RX_EN_SH 16
+#define FWCMD_H2CREG_WOW_CPUIO_RX_CTRL_FW_RX_EN_MSK 0xff
+#define FWCMD_H2CREG_SET_GPIO_FUNC_SH 0
+#define FWCMD_H2CREG_SET_GPIO_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_SET_GPIO_ACK BIT(7)
+#define FWCMD_H2CREG_SET_GPIO_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_SET_GPIO_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_SET_GPIO_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_SET_GPIO_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_FUNC_SH 0
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_FUNC_MSK 0x7f
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_ACK BIT(7)
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_TOTAL_LEN_SH 8
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_SEQ_NUM_SH 12
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_SEQ_NUM_MSK 0xf
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_BACKTRACE_ADDR_SH 0
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_BACKTRACE_ADDR_MSK 0xffffffff
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_DBGREG_ADDR_SH 0
+#define FWCMD_H2CREG_FWDBG_ADDR_CFG_DBGREG_ADDR_MSK 0xffffffff
+
+/*----------------------------------- C2HREG ---------------------------------------*/
+#define FWCMD_C2HREG_C2HREG_HDR_FUNC_SH 0
+#define FWCMD_C2HREG_C2HREG_HDR_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_C2HREG_HDR_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_C2HREG_HDR_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_C2HREG_HDR_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_C2HREG_HDR_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_C2HREG_DWORD0_SH 0
+#define FWCMD_C2HREG_C2HREG_DWORD0_MSK 0xffffffff
+#define FWCMD_C2HREG_C2HREG_DWORD1_SH 0
+#define FWCMD_C2HREG_C2HREG_DWORD1_MSK 0xffffffff
+#define FWCMD_C2HREG_C2HREG_DWORD2_SH 0
+#define FWCMD_C2HREG_C2HREG_DWORD2_MSK 0xffffffff
+#define FWCMD_C2HREG_C2HREG_DWORD3_SH 0
+#define FWCMD_C2HREG_C2HREG_DWORD3_MSK 0xffffffff
+#define FWCMD_C2HREG_C2HREG_LB_FUNC_SH 0
+#define FWCMD_C2HREG_C2HREG_LB_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_C2HREG_LB_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_C2HREG_LB_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_C2HREG_LB_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_C2HREG_LB_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_C2HREG_LB_PAYLOAD0_SH 16
+#define FWCMD_C2HREG_C2HREG_LB_PAYLOAD0_MSK 0xffff
+#define FWCMD_C2HREG_C2HREG_LB_PAYLOAD1_SH 0
+#define FWCMD_C2HREG_C2HREG_LB_PAYLOAD1_MSK 0xffffffff
+#define FWCMD_C2HREG_ERR_RPT_FUNC_SH 0
+#define FWCMD_C2HREG_ERR_RPT_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_ERR_RPT_ACK BIT(7)
+#define FWCMD_C2HREG_ERR_RPT_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_ERR_RPT_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_ERR_RPT_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_ERR_RPT_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_ERR_RPT_IDX_SH 16
+#define FWCMD_C2HREG_ERR_RPT_IDX_MSK 0xf
+#define FWCMD_C2HREG_ERR_RPT_CODE_SH 0
+#define FWCMD_C2HREG_ERR_RPT_CODE_MSK 0xffffffff
+#define FWCMD_C2HREG_ERR_MSG_FUNC_SH 0
+#define FWCMD_C2HREG_ERR_MSG_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_ERR_MSG_ACK BIT(7)
+#define FWCMD_C2HREG_ERR_MSG_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_ERR_MSG_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_ERR_MSG_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_ERR_MSG_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_ERR_MSG_MSG_SH 16
+#define FWCMD_C2HREG_ERR_MSG_MSG_MSK 0xffff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_FUNC_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_EFUSE_HIDDEN_ACK BIT(7)
+#define FWCMD_C2HREG_EFUSE_HIDDEN_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_EFUSE_HIDDEN_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_EFUSE_HIDDEN_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_EFUSE_HIDDEN_RX_NSS_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_RX_NSS_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_BW_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_BW_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_TX_NSS_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_TX_NSS_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PROT80211_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PROT80211_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_NIC_ROUTER_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_NIC_ROUTER_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_WL_FUNC_SUPPORT_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_WL_FUNC_SUPPORT_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_HW_SPECIAL_TYPE_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_HW_SPECIAL_TYPE_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_0_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_0_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_1_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_1_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_2_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_2_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_3_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_UUID_BYTE_3_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_TX_PATH_NUM_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_TX_PATH_NUM_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_RX_PATH_NUM_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_RX_PATH_NUM_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_BAND_SEL_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_BAND_SEL_MSK 0xff
+#define FWCMD_C2HREG_TX_PAUSE_RPT_FUNC_SH 0
+#define FWCMD_C2HREG_TX_PAUSE_RPT_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_TX_PAUSE_RPT_ACK BIT(7)
+#define FWCMD_C2HREG_TX_PAUSE_RPT_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_TX_PAUSE_RPT_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_TX_PAUSE_RPT_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_TX_PAUSE_RPT_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_TX_PAUSE_RPT_RPT_SH 16
+#define FWCMD_C2HREG_TX_PAUSE_RPT_RPT_MSK 0xffff
+#define FWCMD_C2HREG_AOAC_RPT_1_FUNC_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_1_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_AOAC_RPT_1_ACK BIT(7)
+#define FWCMD_C2HREG_AOAC_RPT_1_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_1_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_1_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_AOAC_RPT_1_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_1_KEY_IDX_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_1_KEY_IDX_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_REKEY_OK_SH 24
+#define FWCMD_C2HREG_AOAC_RPT_1_REKEY_OK_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_0_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_0_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_1_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_1_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_2_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_2_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_3_SH 24
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_3_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_4_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_4_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_5_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_5_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_6_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_6_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_7_SH 24
+#define FWCMD_C2HREG_AOAC_RPT_1_IV_7_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_0_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_0_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_1_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_1_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_2_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_2_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_3_SH 24
+#define FWCMD_C2HREG_AOAC_RPT_1_PKT_IV_3_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_FUNC_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_2_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_AOAC_RPT_2_ACK BIT(7)
+#define FWCMD_C2HREG_AOAC_RPT_2_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_2_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_2_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_AOAC_RPT_2_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_4_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_4_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_5_SH 24
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_5_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_6_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_6_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_7_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_2_PKT_IV_7_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_0_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_0_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_1_SH 24
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_1_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_2_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_2_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_3_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_3_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_4_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_4_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_5_SH 24
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_5_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_6_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_6_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_7_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_2_IGTK_IPN_7_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_3_FUNC_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_3_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_AOAC_RPT_3_ACK BIT(7)
+#define FWCMD_C2HREG_AOAC_RPT_3_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_3_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_3_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_AOAC_RPT_3_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_3_RPT_SH 16
+#define FWCMD_C2HREG_AOAC_RPT_3_RPT_MSK 0xffff
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_PRI_CH_SH 0
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_PRI_CH_MSK 0xff
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_BW_SH 8
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_BW_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_CH_OFFSET_SH 12
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_CH_OFFSET_MSK 0xf
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_CHSW_FAILED BIT(16)
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_CH_BAND_SH 17
+#define FWCMD_C2HREG_AOAC_RPT_3_CSA_CH_BAND_MSK 0x3
+#define FWCMD_C2HREG_WOW_TRX_STOP_FUNC_SH 0
+#define FWCMD_C2HREG_WOW_TRX_STOP_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_WOW_TRX_STOP_ACK BIT(7)
+#define FWCMD_C2HREG_WOW_TRX_STOP_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_WOW_TRX_STOP_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_WOW_TRX_STOP_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_WOW_TRX_STOP_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_WOW_TRX_STOP_RPT_SH 16
+#define FWCMD_C2HREG_WOW_TRX_STOP_RPT_MSK 0xffff
+#define FWCMD_C2HREG_IO_OFLD_RESULT_FUNC_SH 0
+#define FWCMD_C2HREG_IO_OFLD_RESULT_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_IO_OFLD_RESULT_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_IO_OFLD_RESULT_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_IO_OFLD_RESULT_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_IO_OFLD_RESULT_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_IO_OFLD_RESULT_RET_SH 16
+#define FWCMD_C2HREG_IO_OFLD_RESULT_RET_MSK 0xff
+#define FWCMD_C2HREG_IO_OFLD_RESULT_CMD_NUM_SH 24
+#define FWCMD_C2HREG_IO_OFLD_RESULT_CMD_NUM_MSK 0xff
+#define FWCMD_C2HREG_IO_OFLD_RESULT_OFFSET_SH 0
+#define FWCMD_C2HREG_IO_OFLD_RESULT_OFFSET_MSK 0xffffffff
+#define FWCMD_C2HREG_IO_OFLD_RESULT_EXP_VAL_SH 0
+#define FWCMD_C2HREG_IO_OFLD_RESULT_EXP_VAL_MSK 0xffffffff
+#define FWCMD_C2HREG_IO_OFLD_RESULT_R_VAL_SH 0
+#define FWCMD_C2HREG_IO_OFLD_RESULT_R_VAL_MSK 0xffffffff
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_FUNC_SH 0
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_ACK BIT(7)
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_FW_RX_EN_SH 16
+#define FWCMD_C2HREG_WOW_CPUIO_RX_ACK_FW_RX_EN_MSK 0xff
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_FUNC_SH 0
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_ACK BIT(7)
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_RET_SH 16
+#define FWCMD_C2HREG_FWDBG_ADDR_STS_RET_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_FUNC_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_FUNC_MSK 0x7f
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_ACK BIT(7)
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_TOTAL_LEN_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_TOTAL_LEN_MSK 0xf
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_SEQ_NUM_SH 12
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_SEQ_NUM_MSK 0xf
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA0_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA0_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA1_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA1_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA2_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA2_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA3_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA3_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA4_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA4_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA5_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA5_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA6_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA6_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA7_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA7_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA8_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA8_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA9_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA9_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA10_SH 0
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA10_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA11_SH 8
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA11_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA12_SH 16
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA12_MSK 0xff
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA13_SH 24
+#define FWCMD_C2HREG_EFUSE_HIDDEN_PART1_DATA13_MSK 0xff
+
+/*----------------------------------- H2CPKT - CAT(TEST) ---------------------------*/
+#define H2C_HDR_CAT_SH 0
+#define H2C_HDR_CAT_MSK 0x3
+#define H2C_HDR_CLASS_SH 2
+#define H2C_HDR_CLASS_MSK 0x3f
+#define H2C_HDR_FUNC_SH 8
+#define H2C_HDR_FUNC_MSK 0xff
+#define H2C_HDR_DEL_TYPE_SH 16
+#define H2C_HDR_DEL_TYPE_MSK 0xf
+#define H2C_HDR_H2C_SEQ_SH 24
+#define H2C_HDR_H2C_SEQ_MSK 0xff
+#define H2C_HDR_TOTAL_LEN_SH 0
+#define H2C_HDR_TOTAL_LEN_MSK 0x3fff
+#define H2C_HDR_REC_ACK BIT(14)
+#define H2C_HDR_DONE_ACK BIT(15)
+#define H2C_HDR_SEQ_VALID BIT(16)
+#define H2C_HDR_SEQ_SH 17
+#define H2C_HDR_SEQ_MSK 0x7
+#define H2C_HDR_SEQ_STOP BIT(20)
+#define FWCMD_H2C_H2C_LB_PAYLOAD0_SH 0
+#define FWCMD_H2C_H2C_LB_PAYLOAD0_MSK 0xffffffff
+#define FWCMD_H2C_H2C_LB_PAYLOAD1_SH 0
+#define FWCMD_H2C_H2C_LB_PAYLOAD1_MSK 0xffffffff
+#define FWCMD_H2C_LOG_TEST_LEN_SH 0
+#define FWCMD_H2C_LOG_TEST_LEN_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD0_SH 0
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD0_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD1_SH 0
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD1_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD2_SH 0
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD2_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD3_SH 0
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD3_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD4_SH 0
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD4_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD5_SH 0
+#define FWCMD_H2C_PLAT_SPIC_TEST_PAYLOAD5_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FL_WRITE_ADDR_SH 0
+#define FWCMD_H2C_PLAT_FL_WRITE_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FL_WRITE_LENGTH_SH 0
+#define FWCMD_H2C_PLAT_FL_WRITE_LENGTH_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FL_ERASE_ADDR_SH 0
+#define FWCMD_H2C_PLAT_FL_ERASE_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FL_ERASE_LENGTH_SH 0
+#define FWCMD_H2C_PLAT_FL_ERASE_LENGTH_MSK 0xffffffff
+#define FWCMD_H2C_PL_FLASH_READ_ADDR_SH 0
+#define FWCMD_H2C_PL_FLASH_READ_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_PL_FLASH_READ_LENGTH_SH 0
+#define FWCMD_H2C_PL_FLASH_READ_LENGTH_MSK 0xffffffff
+#define FWCMD_H2C_LONG_RUN_PAYLOAD0_SH 0
+#define FWCMD_H2C_LONG_RUN_PAYLOAD0_MSK 0xffffffff
+#define FWCMD_H2C_LONG_RUN_PAYLOAD1_SH 0
+#define FWCMD_H2C_LONG_RUN_PAYLOAD1_MSK 0xffffffff
+#define FWCMD_H2C_LPS_TEST_COUNT_SH 0
+#define FWCMD_H2C_LPS_TEST_COUNT_MSK 0xffffffff
+#define FWCMD_H2C_LPS_ONOFF_TEST_TYPE_SH 0
+#define FWCMD_H2C_LPS_ONOFF_TEST_TYPE_MSK 0xffffffff
+#define FWCMD_H2C_LPS_ONOFF_TEST_PARA_SH 0
+#define FWCMD_H2C_LPS_ONOFF_TEST_PARA_MSK 0xffffffff
+#define FWCMD_H2C_CPUMGENQ_TEST_PKTLEN_SH 0
+#define FWCMD_H2C_CPUMGENQ_TEST_PKTLEN_MSK 0xffff
+#define FWCMD_H2C_CPUMGENQ_TEST_MACID_SH 16
+#define FWCMD_H2C_CPUMGENQ_TEST_MACID_MSK 0xffff
+#define FWCMD_H2C_MPIO_TEST_PKTLEN_SH 0
+#define FWCMD_H2C_MPIO_TEST_PKTLEN_MSK 0xffff
+#define FWCMD_H2C_MPIO_TEST_MACID_SH 16
+#define FWCMD_H2C_MPIO_TEST_MACID_MSK 0xff
+#define FWCMD_H2C_MPIOV2_TEST_TYPE_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_TYPE_MSK 0xffffffff
+#define FWCMD_H2C_MPIOV2_TEST_TEST_ADDR_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_TEST_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD0_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD0_MSK 0xffffffff
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD1_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD1_MSK 0xffffffff
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD2_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD2_MSK 0xffffffff
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD3_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD3_MSK 0xffffffff
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD4_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD4_MSK 0xffffffff
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD5_SH 0
+#define FWCMD_H2C_MPIOV2_TEST_PAYLOAD5_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_SEQNUM_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_SEQNUM_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_TEST_FUNC_MODE_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_TEST_FUNC_MODE_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_TEST_CNT_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_TEST_CNT_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_BUFFERSIZE_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_BUFFERSIZE_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT_NUM_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT_NUM_MSK 0xffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT_BUFFER_MODE_SH 16
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT_BUFFER_MODE_MSK 0xffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT0_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT0_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT1_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT1_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT2_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT2_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT3_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT3_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT4_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT4_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT5_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT5_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT6_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT6_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT7_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INPUT7_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_RETURN_VALUE_EXPECT_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_RETURN_VALUE_EXPECT_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_RETURN_VALUE_MASK_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_RETURN_VALUE_MASK_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_RETURN_VAL_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_RETURN_VAL_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_CPU_CYCLE_START_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_CPU_CYCLE_START_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_CPU_CYCLE_END_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_CPU_CYCLE_END_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INST_CNT_START_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INST_CNT_START_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INST_CNT_END_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_INST_CNT_END_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_FREERUN_CNT_START_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_FREERUN_CNT_START_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_FREERUN_CNT_END_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_FREERUN_CNT_END_MSK 0xffffffff
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_BUFFER_SH 0
+#define FWCMD_H2C_ANYTESTBUSYCALL_TEST_BUFFER_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_SEQNUM_SH 0
+#define FWCMD_H2C_ANYRW_TEST_SEQNUM_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_OP_MODE_SH 0
+#define FWCMD_H2C_ANYRW_TEST_OP_MODE_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_BUFFERSIZE_SH 0
+#define FWCMD_H2C_ANYRW_TEST_BUFFERSIZE_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_INPUT0_SH 0
+#define FWCMD_H2C_ANYRW_TEST_INPUT0_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_INPUT1_SH 0
+#define FWCMD_H2C_ANYRW_TEST_INPUT1_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_INPUT2_SH 0
+#define FWCMD_H2C_ANYRW_TEST_INPUT2_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_INPUT3_SH 0
+#define FWCMD_H2C_ANYRW_TEST_INPUT3_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_INPUT4_SH 0
+#define FWCMD_H2C_ANYRW_TEST_INPUT4_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_INPUT5_SH 0
+#define FWCMD_H2C_ANYRW_TEST_INPUT5_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_RETURN_VAL_SH 0
+#define FWCMD_H2C_ANYRW_TEST_RETURN_VAL_MSK 0xffffffff
+#define FWCMD_H2C_ANYRW_TEST_BUFFER_SH 0
+#define FWCMD_H2C_ANYRW_TEST_BUFFER_MSK 0xffffffff
+#define FWCMD_H2C_FW_STATUS_COUNT_SH 0
+#define FWCMD_H2C_FW_STATUS_COUNT_MSK 0xffffffff
+#define FWCMD_H2C_TRIGGER_CPU_EXCEPTION_TYPE_SH 0
+#define FWCMD_H2C_TRIGGER_CPU_EXCEPTION_TYPE_MSK 0xffffffff
+#define FWCMD_H2C_FW_PROFILING_TYPE_SH 0
+#define FWCMD_H2C_FW_PROFILING_TYPE_MSK 0xffffffff
+#define FWCMD_H2C_WOW_AUTO_TEST_RX_TEST BIT(0)
+#define FWCMD_H2C_WOW_AUTO_TEST_TX_TEST BIT(1)
+
+/*----------------------------------- H2CPKT - CAT(MAC) ----------------------------*/
+#define H2C_HDR_CAT_SH 0
+#define H2C_HDR_CAT_MSK 0x3
+#define H2C_HDR_CLASS_SH 2
+#define H2C_HDR_CLASS_MSK 0x3f
+#define H2C_HDR_FUNC_SH 8
+#define H2C_HDR_FUNC_MSK 0xff
+#define H2C_HDR_DEL_TYPE_SH 16
+#define H2C_HDR_DEL_TYPE_MSK 0xf
+#define H2C_HDR_H2C_SEQ_SH 24
+#define H2C_HDR_H2C_SEQ_MSK 0xff
+#define H2C_HDR_TOTAL_LEN_SH 0
+#define H2C_HDR_TOTAL_LEN_MSK 0x3fff
+#define H2C_HDR_REC_ACK BIT(14)
+#define H2C_HDR_DONE_ACK BIT(15)
+#define H2C_HDR_SEQ_VALID BIT(16)
+#define H2C_HDR_SEQ_SH 17
+#define H2C_HDR_SEQ_MSK 0x7
+#define H2C_HDR_SEQ_STOP BIT(20)
+#define FWCMD_H2C_LOG_CFG_DBG_LV_SH 0
+#define FWCMD_H2C_LOG_CFG_DBG_LV_MSK 0xff
+#define FWCMD_H2C_LOG_CFG_PATH_SH 8
+#define FWCMD_H2C_LOG_CFG_PATH_MSK 0xff
+#define FWCMD_H2C_LOG_CFG_COMP_SH 0
+#define FWCMD_H2C_LOG_CFG_COMP_MSK 0xffffffff
+#define FWCMD_H2C_LOG_CFG_COMP_EXT_SH 0
+#define FWCMD_H2C_LOG_CFG_COMP_EXT_MSK 0xffffffff
+#define FWCMD_H2C_GENERAL_PKT_MACID_SH 0
+#define FWCMD_H2C_GENERAL_PKT_MACID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_PROBRSP_ID_SH 8
+#define FWCMD_H2C_GENERAL_PKT_PROBRSP_ID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_PSPOLL_ID_SH 16
+#define FWCMD_H2C_GENERAL_PKT_PSPOLL_ID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_NULL_ID_SH 24
+#define FWCMD_H2C_GENERAL_PKT_NULL_ID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_QOS_NULL_ID_SH 0
+#define FWCMD_H2C_GENERAL_PKT_QOS_NULL_ID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_CTS2SELF_ID_SH 8
+#define FWCMD_H2C_GENERAL_PKT_CTS2SELF_ID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_PROBREQ_ID_SH 16
+#define FWCMD_H2C_GENERAL_PKT_PROBREQ_ID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_APCSA_ID_SH 24
+#define FWCMD_H2C_GENERAL_PKT_APCSA_ID_MSK 0xff
+#define FWCMD_H2C_GENERAL_PKT_MACID_EXT_SH 0
+#define FWCMD_H2C_GENERAL_PKT_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_C2H_RPT_CFG_TX_STATISTIC_EN BIT(0)
+#define FWCMD_H2C_C2H_RPT_CFG_RX_STATISTIC_EN BIT(1)
+#define FWCMD_H2C_C2H_RPT_CFG_BEACON_STATISTIC_EN BIT(2)
+#define FWCMD_H2C_C2H_RPT_CFG_DL_STATISTIC_EN BIT(3)
+#define FWCMD_H2C_C2H_RPT_CFG_RPT_INTERVAL_SH 0
+#define FWCMD_H2C_C2H_RPT_CFG_RPT_INTERVAL_MSK 0xffff
+#define FWCMD_H2C_FW_DBGREG_CFG_FW_DBG_LOCK BIT(0)
+#define FWCMD_H2C_FW_REDL_CFG_STOP_WATCHDOG BIT(0)
+#define FWCMD_H2C_FW_REDL_CFG_RESET_IDMEM_MODE BIT(1)
+#define FWCMD_H2C_FW_SELF_DX_FWDX_TIME_INTERVAL_SH 0
+#define FWCMD_H2C_FW_SELF_DX_FWDX_TIME_INTERVAL_MSK 0xffff
+#define FWCMD_H2C_FW_SELF_DX_FWDX_GET_ENABLE_ITEM BIT(16)
+#define FWCMD_H2C_FW_SELF_DX_FWDX_HEAP_REMAIN BIT(0)
+#define FWCMD_H2C_FW_SELF_DX_FWDX_IRQ_CNT BIT(1)
+#define FWCMD_H2C_FW_SELF_DX_FWDX_AXIDMA_DEBUG_INFO BIT(2)
+#define FWCMD_H2C_FW_SELF_DX_FWDX_CPUIO_DEBUG_INFO BIT(3)
+#define FWCMD_H2C_FW_DBG_ADDR_CFG_BACKTRACE_ADDR_SH 0
+#define FWCMD_H2C_FW_DBG_ADDR_CFG_BACKTRACE_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_FW_DBG_ADDR_CFG_DBGREG_ADDR_SH 0
+#define FWCMD_H2C_FW_DBG_ADDR_CFG_DBGREG_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_KEEP_ALIVE_KEEPALIVE_EN BIT(0)
+#define FWCMD_H2C_KEEP_ALIVE_TX_TYPE_SH 1
+#define FWCMD_H2C_KEEP_ALIVE_TX_TYPE_MSK 0x7f
+#define FWCMD_H2C_KEEP_ALIVE_PACKET_ID_SH 8
+#define FWCMD_H2C_KEEP_ALIVE_PACKET_ID_MSK 0xff
+#define FWCMD_H2C_KEEP_ALIVE_PERIOD_SH 16
+#define FWCMD_H2C_KEEP_ALIVE_PERIOD_MSK 0xff
+#define FWCMD_H2C_KEEP_ALIVE_MAC_ID_SH 24
+#define FWCMD_H2C_KEEP_ALIVE_MAC_ID_MSK 0xff
+#define FWCMD_H2C_DISCONNECT_DETECT_DISCONNECT_DETECT_EN BIT(0)
+#define FWCMD_H2C_DISCONNECT_DETECT_TRYOK_BCNFAIL_COUNT_EN BIT(1)
+#define FWCMD_H2C_DISCONNECT_DETECT_DISCONNECT_EN BIT(2)
+#define FWCMD_H2C_DISCONNECT_DETECT_MAC_ID_SH 8
+#define FWCMD_H2C_DISCONNECT_DETECT_MAC_ID_MSK 0xff
+#define FWCMD_H2C_DISCONNECT_DETECT_CHECK_PERIOD_SH 16
+#define FWCMD_H2C_DISCONNECT_DETECT_CHECK_PERIOD_MSK 0xff
+#define FWCMD_H2C_DISCONNECT_DETECT_TRY_PKT_COUNT_SH 24
+#define FWCMD_H2C_DISCONNECT_DETECT_TRY_PKT_COUNT_MSK 0xff
+#define FWCMD_H2C_DISCONNECT_DETECT_TRYOK_BCNFAIL_COUNT_LIMIT_SH 0
+#define FWCMD_H2C_DISCONNECT_DETECT_TRYOK_BCNFAIL_COUNT_LIMIT_MSK 0xff
+#define FWCMD_H2C_WOW_GLOBAL_WOW_EN BIT(0)
+#define FWCMD_H2C_WOW_GLOBAL_DROP_ALL_PKT BIT(1)
+#define FWCMD_H2C_WOW_GLOBAL_RX_PARSE_AFTER_WAKE BIT(2)
+#define FWCMD_H2C_WOW_GLOBAL_WAKE_BAR_PULLED BIT(3)
+#define FWCMD_H2C_WOW_GLOBAL_PMF_BIP_SEC_ALGO_SH 6
+#define FWCMD_H2C_WOW_GLOBAL_PMF_BIP_SEC_ALGO_MSK 0x3
+#define FWCMD_H2C_WOW_GLOBAL_MAC_ID_SH 8
+#define FWCMD_H2C_WOW_GLOBAL_MAC_ID_MSK 0xff
+#define FWCMD_H2C_WOW_GLOBAL_PAIRWISE_SEC_ALGO_SH 16
+#define FWCMD_H2C_WOW_GLOBAL_PAIRWISE_SEC_ALGO_MSK 0xff
+#define FWCMD_H2C_WOW_GLOBAL_GROUP_SEC_ALGO_SH 24
+#define FWCMD_H2C_WOW_GLOBAL_GROUP_SEC_ALGO_MSK 0xff
+#define FWCMD_H2C_WOW_GLOBAL_REMOTECTRL_INFO_CONTENT_SH 0
+#define FWCMD_H2C_WOW_GLOBAL_REMOTECTRL_INFO_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_GTK_OFLD_GTK_EN BIT(0)
+#define FWCMD_H2C_GTK_OFLD_TKIP_EN BIT(1)
+#define FWCMD_H2C_GTK_OFLD_IEEE80211W_EN BIT(2)
+#define FWCMD_H2C_GTK_OFLD_PAIRWISE_WAKEUP BIT(3)
+#define FWCMD_H2C_GTK_OFLD_NOREKEY_WAKEUP BIT(4)
+#define FWCMD_H2C_GTK_OFLD_BCN_PROT_EN BIT(5)
+#define FWCMD_H2C_GTK_OFLD_AOAC_REPORT_ID_SH 8
+#define FWCMD_H2C_GTK_OFLD_AOAC_REPORT_ID_MSK 0xff
+#define FWCMD_H2C_GTK_OFLD_MAC_ID_SH 16
+#define FWCMD_H2C_GTK_OFLD_MAC_ID_MSK 0xff
+#define FWCMD_H2C_GTK_OFLD_GTK_RSP_ID_SH 24
+#define FWCMD_H2C_GTK_OFLD_GTK_RSP_ID_MSK 0xff
+#define FWCMD_H2C_GTK_OFLD_PMF_SA_QUERY_ID_SH 0
+#define FWCMD_H2C_GTK_OFLD_PMF_SA_QUERY_ID_MSK 0xff
+#define FWCMD_H2C_GTK_OFLD_PMF_BIP_SEC_ALGO_SH 8
+#define FWCMD_H2C_GTK_OFLD_PMF_BIP_SEC_ALGO_MSK 0x3
+#define FWCMD_H2C_GTK_OFLD_ALGO_AKM_SUIT_SH 10
+#define FWCMD_H2C_GTK_OFLD_ALGO_AKM_SUIT_MSK 0xff
+#define FWCMD_H2C_GTK_OFLD_MAC_ID_EXT_SH 18
+#define FWCMD_H2C_GTK_OFLD_MAC_ID_EXT_MSK 0xff
+#define FWCMD_H2C_GTK_OFLD_MLD_LINK_ID_SH 26
+#define FWCMD_H2C_GTK_OFLD_MLD_LINK_ID_MSK 0xf
+#define FWCMD_H2C_GTK_OFLD_GTK_INFO_CONTENT_SH 0
+#define FWCMD_H2C_GTK_OFLD_GTK_INFO_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_ARP_OFLD_ARP_EN BIT(0)
+#define FWCMD_H2C_ARP_OFLD_ARP_ACTION BIT(1)
+#define FWCMD_H2C_ARP_OFLD_MAC_ID_SH 16
+#define FWCMD_H2C_ARP_OFLD_MAC_ID_MSK 0xff
+#define FWCMD_H2C_ARP_OFLD_ARP_RSP_ID_SH 24
+#define FWCMD_H2C_ARP_OFLD_ARP_RSP_ID_MSK 0xff
+#define FWCMD_H2C_ARP_OFLD_ARP_INFO_CONTENT_SH 0
+#define FWCMD_H2C_ARP_OFLD_ARP_INFO_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_NDP_OFLD_NDP_EN BIT(0)
+#define FWCMD_H2C_NDP_OFLD_MAC_ID_SH 16
+#define FWCMD_H2C_NDP_OFLD_MAC_ID_MSK 0xff
+#define FWCMD_H2C_NDP_OFLD_NA_ID_SH 24
+#define FWCMD_H2C_NDP_OFLD_NA_ID_MSK 0xff
+#define FWCMD_H2C_NDP_OFLD_NDP_INFO_CONTENT_SH 0
+#define FWCMD_H2C_NDP_OFLD_NDP_INFO_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_REALWOW_REALWOW_EN BIT(0)
+#define FWCMD_H2C_REALWOW_AUTO_WAKEUP BIT(1)
+#define FWCMD_H2C_REALWOW_MAC_ID_SH 24
+#define FWCMD_H2C_REALWOW_MAC_ID_MSK 0xff
+#define FWCMD_H2C_REALWOW_KEEPALIVE_ID_SH 0
+#define FWCMD_H2C_REALWOW_KEEPALIVE_ID_MSK 0xff
+#define FWCMD_H2C_REALWOW_WAKEUP_PATTERN_ID_SH 8
+#define FWCMD_H2C_REALWOW_WAKEUP_PATTERN_ID_MSK 0xff
+#define FWCMD_H2C_REALWOW_ACK_PATTERN_ID_SH 16
+#define FWCMD_H2C_REALWOW_ACK_PATTERN_ID_MSK 0xff
+#define FWCMD_H2C_REALWOW_REALWOW_INFO_CONTENT_SH 0
+#define FWCMD_H2C_REALWOW_REALWOW_INFO_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_NLO_NLO_EN BIT(0)
+#define FWCMD_H2C_NLO_NLO_32K_EN BIT(1)
+#define FWCMD_H2C_NLO_IGNORE_CIPHER_TYPE BIT(2)
+#define FWCMD_H2C_NLO_MAC_ID_SH 24
+#define FWCMD_H2C_NLO_MAC_ID_MSK 0xff
+#define FWCMD_H2C_NLO_NLO_NETWORKLISTINFO_CONTENT_SH 0
+#define FWCMD_H2C_NLO_NLO_NETWORKLISTINFO_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_WAKEUP_CTRL_PATTERN_MATCH_EN BIT(0)
+#define FWCMD_H2C_WAKEUP_CTRL_MAGIC_EN BIT(1)
+#define FWCMD_H2C_WAKEUP_CTRL_HW_UNICAST_EN BIT(2)
+#define FWCMD_H2C_WAKEUP_CTRL_FW_UNICAST_EN BIT(3)
+#define FWCMD_H2C_WAKEUP_CTRL_DEAUTH_WAKEUP BIT(4)
+#define FWCMD_H2C_WAKEUP_CTRL_REKEY_WAKEUP BIT(5)
+#define FWCMD_H2C_WAKEUP_CTRL_EAP_WAKEUP BIT(6)
+#define FWCMD_H2C_WAKEUP_CTRL_ALL_DATA_WAKEUP BIT(7)
+#define FWCMD_H2C_WAKEUP_CTRL_MAGIC_MLD_EN BIT(8)
+#define FWCMD_H2C_WAKEUP_CTRL_MAC_ID_SH 24
+#define FWCMD_H2C_WAKEUP_CTRL_MAC_ID_MSK 0xff
+#define FWCMD_H2C_NEGATIVE_PATTERN_NEGATIVE_PATTERN_EN BIT(0)
+#define FWCMD_H2C_NEGATIVE_PATTERN_PATTERN_COUNT_SH 20
+#define FWCMD_H2C_NEGATIVE_PATTERN_PATTERN_COUNT_MSK 0xf
+#define FWCMD_H2C_NEGATIVE_PATTERN_MAC_ID_SH 24
+#define FWCMD_H2C_NEGATIVE_PATTERN_MAC_ID_MSK 0xff
+#define FWCMD_H2C_NEGATIVE_PATTERN_PATTERN_CONTENT_SH 0
+#define FWCMD_H2C_NEGATIVE_PATTERN_PATTERN_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_DEV2HST_GPIO_DEV2HST_GPIO_EN BIT(0)
+#define FWCMD_H2C_DEV2HST_GPIO_DISABLE_INBAND BIT(1)
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_OUTPUT_INPUT BIT(2)
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_ACTIVE BIT(3)
+#define FWCMD_H2C_DEV2HST_GPIO_TOGGLE_PULSE BIT(4)
+#define FWCMD_H2C_DEV2HST_GPIO_DATA_PIN_WAKEUP BIT(5)
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_PULSE_NONSTOP BIT(6)
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_TIME_UNIT BIT(7)
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_NUM_SH 8
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_NUM_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_PULSE_DURATION_SH 16
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_PULSE_DURATION_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_PULSE_PERIOD_SH 24
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_PULSE_PERIOD_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_PULSE_COUNT_SH 0
+#define FWCMD_H2C_DEV2HST_GPIO_GPIO_PULSE_COUNT_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_NUM_EXTEND_RSN_SH 8
+#define FWCMD_H2C_DEV2HST_GPIO_NUM_EXTEND_RSN_MSK 0xf
+#define FWCMD_H2C_DEV2HST_GPIO_INDICATE_DURATION_SH 16
+#define FWCMD_H2C_DEV2HST_GPIO_INDICATE_DURATION_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_INDICATE_INTERMISSION_SH 24
+#define FWCMD_H2C_DEV2HST_GPIO_INDICATE_INTERMISSION_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_CUSTOMER_ID_SH 0
+#define FWCMD_H2C_DEV2HST_GPIO_CUSTOMER_ID_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_EN BIT(0)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_TOGGLE_PULSE BIT(1)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_PULSE_NONSTOP BIT(2)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_TIME_UNIT BIT(3)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_SH 0
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_PULSE_DURATION_SH 8
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_PULSE_DURATION_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_PULSE_PERIOD_SH 16
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_PULSE_PERIOD_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_PULSE_COUNT_SH 24
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_A_PULSE_COUNT_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_EN BIT(0)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_TOGGLE_PULSE BIT(1)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_PULSE_NONSTOP BIT(2)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_TIME_UNIT BIT(3)
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_SH 0
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_PULSE_DURATION_SH 8
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_PULSE_DURATION_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_PULSE_PERIOD_SH 16
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_PULSE_PERIOD_MSK 0xff
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_PULSE_COUNT_SH 24
+#define FWCMD_H2C_DEV2HST_GPIO_RSN_B_PULSE_COUNT_MSK 0xff
+#define FWCMD_H2C_HST2DEV_CTRL_DISABLE_UPHY BIT(0)
+#define FWCMD_H2C_HST2DEV_CTRL_HANDSHAKE_MODE_SH 1
+#define FWCMD_H2C_HST2DEV_CTRL_HANDSHAKE_MODE_MSK 0x7
+#define FWCMD_H2C_HST2DEV_CTRL_RISE_HST2DEV_DIS_UPHY BIT(8)
+#define FWCMD_H2C_HST2DEV_CTRL_UPHY_DIS_DELAY_UNIT BIT(9)
+#define FWCMD_H2C_HST2DEV_CTRL_PDN_AS_UPHY_DIS BIT(10)
+#define FWCMD_H2C_HST2DEV_CTRL_PDN_TO_ENABLE_UPHY BIT(11)
+#define FWCMD_H2C_HST2DEV_CTRL_HST2DEV_EN BIT(12)
+#define FWCMD_H2C_HST2DEV_CTRL_HST2DEV_GPIO_NUM_SH 16
+#define FWCMD_H2C_HST2DEV_CTRL_HST2DEV_GPIO_NUM_MSK 0xff
+#define FWCMD_H2C_HST2DEV_CTRL_UPHY_DIS_DELAY_COUNT_SH 24
+#define FWCMD_H2C_HST2DEV_CTRL_UPHY_DIS_DELAY_COUNT_MSK 0xff
+#define FWCMD_H2C_WOW_CAM_UPD_R_W BIT(0)
+#define FWCMD_H2C_WOW_CAM_UPD_IDX_SH 1
+#define FWCMD_H2C_WOW_CAM_UPD_IDX_MSK 0x7f
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM1_SH 0
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM1_MSK 0xffffffff
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM2_SH 0
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM2_MSK 0xffffffff
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM3_SH 0
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM3_MSK 0xffffffff
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM4_SH 0
+#define FWCMD_H2C_WOW_CAM_UPD_WKFM4_MSK 0xffffffff
+#define FWCMD_H2C_WOW_CAM_UPD_CRC_SH 0
+#define FWCMD_H2C_WOW_CAM_UPD_CRC_MSK 0xffff
+#define FWCMD_H2C_WOW_CAM_UPD_NEGATIVE_PATTERN_MATCH BIT(22)
+#define FWCMD_H2C_WOW_CAM_UPD_SKIP_MAC_HDR BIT(23)
+#define FWCMD_H2C_WOW_CAM_UPD_UC BIT(24)
+#define FWCMD_H2C_WOW_CAM_UPD_MC BIT(25)
+#define FWCMD_H2C_WOW_CAM_UPD_BC BIT(26)
+#define FWCMD_H2C_WOW_CAM_UPD_VALID BIT(31)
+#define FWCMD_H2C_AOAC_REPORT_REQ_RX_READY BIT(0)
+#define FWCMD_H2C_AOAC_REPORT_REQ_MACID_SH 8
+#define FWCMD_H2C_AOAC_REPORT_REQ_MACID_MSK 0xff
+#define FWCMD_H2C_AOAC_REPORT_REQ_MACID_EXT_SH 16
+#define FWCMD_H2C_AOAC_REPORT_REQ_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_WOW_CSA_WOW_CSA_EN BIT(0)
+#define FWCMD_H2C_WOW_CSA_SIZE_OF_CHINFO_SH 1
+#define FWCMD_H2C_WOW_CSA_SIZE_OF_CHINFO_MSK 0x7f
+#define FWCMD_H2C_WOW_CSA_NUM_OF_CHINFO_SH 8
+#define FWCMD_H2C_WOW_CSA_NUM_OF_CHINFO_MSK 0xff
+#define FWCMD_H2C_WOW_CSA_MACID_SH 16
+#define FWCMD_H2C_WOW_CSA_MACID_MSK 0xff
+#define FWCMD_H2C_WOW_CSA_MAX_AP_GONE_TIME_SH 24
+#define FWCMD_H2C_WOW_CSA_MAX_AP_GONE_TIME_MSK 0xff
+#define FWCMD_H2C_MAGIC_WAKER_FILTER_WAKER_NUM_SH 0
+#define FWCMD_H2C_MAGIC_WAKER_FILTER_WAKER_NUM_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_MACID_SH 0
+#define FWCMD_H2C_TCP_KEEPALIVE_MACID_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_PERIOD_SH 8
+#define FWCMD_H2C_TCP_KEEPALIVE_PERIOD_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_TX_PKTID_SH 16
+#define FWCMD_H2C_TCP_KEEPALIVE_TX_PKTID_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_ENABLE BIT(24)
+#define FWCMD_H2C_TCP_KEEPALIVE_RETRY_INTVL_SH 0
+#define FWCMD_H2C_TCP_KEEPALIVE_RETRY_INTVL_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_MAX_RETRY_CNT_SH 8
+#define FWCMD_H2C_TCP_KEEPALIVE_MAX_RETRY_CNT_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_IMMED_TX BIT(16)
+#define FWCMD_H2C_TCP_KEEPALIVE_MACID_EXT_SH 17
+#define FWCMD_H2C_TCP_KEEPALIVE_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_ACK_PKTID_SH 0
+#define FWCMD_H2C_TCP_KEEPALIVE_ACK_PKTID_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_RECV_KEEPALIVE_TIMEOUT_SH 8
+#define FWCMD_H2C_TCP_KEEPALIVE_RECV_KEEPALIVE_TIMEOUT_MSK 0xff
+#define FWCMD_H2C_TCP_KEEPALIVE_SEQ_INCREASE BIT(16)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_R_W BIT(0)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_IDX_SH 1
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_IDX_MSK 0x7f
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM1_SH 0
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM1_MSK 0xffffffff
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM2_SH 0
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM2_MSK 0xffffffff
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM3_SH 0
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM3_MSK 0xffffffff
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM4_SH 0
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_WKFM4_MSK 0xffffffff
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_UC BIT(0)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_MC BIT(1)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_BC BIT(2)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_SKIP_MAC_HDR BIT(7)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_CRC_SH 0
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_CRC_MSK 0xffff
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_NEGATIVE_PATTERN_MATCH BIT(0)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_VALID BIT(0)
+#define FWCMD_H2C_WOW_PAYLOAD_CAM_UPD_IS_WOW_PTN BIT(1)
+#define FWCMD_H2C_LPS_PARM_MACID_SH 0
+#define FWCMD_H2C_LPS_PARM_MACID_MSK 0xff
+#define FWCMD_H2C_LPS_PARM_PSMODE_SH 8
+#define FWCMD_H2C_LPS_PARM_PSMODE_MSK 0xff
+#define FWCMD_H2C_LPS_PARM_RLBM_SH 16
+#define FWCMD_H2C_LPS_PARM_RLBM_MSK 0xf
+#define FWCMD_H2C_LPS_PARM_SMARTPS_SH 20
+#define FWCMD_H2C_LPS_PARM_SMARTPS_MSK 0xf
+#define FWCMD_H2C_LPS_PARM_AWAKEINTERVAL_SH 24
+#define FWCMD_H2C_LPS_PARM_AWAKEINTERVAL_MSK 0xff
+#define FWCMD_H2C_LPS_PARM_VOUAPSD BIT(0)
+#define FWCMD_H2C_LPS_PARM_VIUAPSD BIT(1)
+#define FWCMD_H2C_LPS_PARM_BEUAPSD BIT(2)
+#define FWCMD_H2C_LPS_PARM_BKUAPSD BIT(3)
+#define FWCMD_H2C_LPS_PARM_LASTRPWM_SH 8
+#define FWCMD_H2C_LPS_PARM_LASTRPWM_MSK 0xff
+#define FWCMD_H2C_LPS_PARM_BCNNOHIT_EN BIT(16)
+#define FWCMD_H2C_LPS_PARM_NULLTYPE BIT(17)
+#define FWCMD_H2C_LPS_PARM_DYNTXANTNUM_EN BIT(18)
+#define FWCMD_H2C_LPS_PARM_MAXTXANTNUM BIT(19)
+#define FWCMD_H2C_LPS_PARM_LPSTXANTNUM BIT(20)
+#define FWCMD_H2C_LPS_PARM_MACID_EXT_SH 24
+#define FWCMD_H2C_LPS_PARM_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_P2P_ACT_MACID_SH 0
+#define FWCMD_H2C_P2P_ACT_MACID_MSK 0xff
+#define FWCMD_H2C_P2P_ACT_P2PID_SH 8
+#define FWCMD_H2C_P2P_ACT_P2PID_MSK 0xf
+#define FWCMD_H2C_P2P_ACT_NOAID_SH 12
+#define FWCMD_H2C_P2P_ACT_NOAID_MSK 0xf
+#define FWCMD_H2C_P2P_ACT_ACT_SH 16
+#define FWCMD_H2C_P2P_ACT_ACT_MSK 0xf
+#define FWCMD_H2C_P2P_ACT_TYPE BIT(20)
+#define FWCMD_H2C_P2P_ACT_ALL_SLEP BIT(21)
+#define FWCMD_H2C_P2P_ACT_MACID_EXT_SH 24
+#define FWCMD_H2C_P2P_ACT_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_P2P_ACT_SRT_SH 0
+#define FWCMD_H2C_P2P_ACT_SRT_MSK 0xffffffff
+#define FWCMD_H2C_P2P_ACT_ITVL_SH 0
+#define FWCMD_H2C_P2P_ACT_ITVL_MSK 0xffffffff
+#define FWCMD_H2C_P2P_ACT_DUR_SH 0
+#define FWCMD_H2C_P2P_ACT_DUR_MSK 0xffffffff
+#define FWCMD_H2C_P2P_ACT_CNT_SH 0
+#define FWCMD_H2C_P2P_ACT_CNT_MSK 0xff
+#define FWCMD_H2C_P2P_ACT_CTW_SH 8
+#define FWCMD_H2C_P2P_ACT_CTW_MSK 0xffff
+#define FWCMD_H2C_P2P_MACID_CTRL_P2PID_SH 0
+#define FWCMD_H2C_P2P_MACID_CTRL_P2PID_MSK 0xf
+#define FWCMD_H2C_P2P_MACID_CTRL_CTRL_TYPE_SH 4
+#define FWCMD_H2C_P2P_MACID_CTRL_CTRL_TYPE_MSK 0xf
+#define FWCMD_H2C_P2P_MACID_CTRL_MACID_SRT_SH 8
+#define FWCMD_H2C_P2P_MACID_CTRL_MACID_SRT_MSK 0xff
+#define FWCMD_H2C_P2P_MACID_CTRL_BMAP_LEN_SH 16
+#define FWCMD_H2C_P2P_MACID_CTRL_BMAP_LEN_MSK 0xff
+#define FWCMD_H2C_P2P_MACID_CTRL_MACID_SRT_EXT_SH 24
+#define FWCMD_H2C_P2P_MACID_CTRL_MACID_SRT_EXT_MSK 0xff
+#define FWCMD_H2C_P2P_MACID_CTRL_BMAP_SRT_SH 0
+#define FWCMD_H2C_P2P_MACID_CTRL_BMAP_SRT_MSK 0xffffffff
+#define FWCMD_H2C_IPS_CFG_MACID_SH 0
+#define FWCMD_H2C_IPS_CFG_MACID_MSK 0xff
+#define FWCMD_H2C_IPS_CFG_ENABLE BIT(8)
+#define FWCMD_H2C_IPS_CFG_MACID_EXT_SH 16
+#define FWCMD_H2C_IPS_CFG_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_PS_ADVANCE_PARM_MACID_SH 0
+#define FWCMD_H2C_PS_ADVANCE_PARM_MACID_MSK 0xff
+#define FWCMD_H2C_PS_ADVANCE_PARM_TRXTIMEOUTTIMESET_SH 8
+#define FWCMD_H2C_PS_ADVANCE_PARM_TRXTIMEOUTTIMESET_MSK 0x3
+#define FWCMD_H2C_PS_ADVANCE_PARM_ENSMARTPSDTIMRX BIT(10)
+#define FWCMD_H2C_PS_ADVANCE_PARM_TRXEXTMODE BIT(11)
+#define FWCMD_H2C_PS_ADVANCE_PARM_TRXTIMEOUTTIMEVAL_SH 0
+#define FWCMD_H2C_PS_ADVANCE_PARM_TRXTIMEOUTTIMEVAL_MSK 0xff
+#define FWCMD_H2C_PS_ADVANCE_PARM_EXTNUM_SH 8
+#define FWCMD_H2C_PS_ADVANCE_PARM_EXTNUM_MSK 0xff
+#define FWCMD_H2C_PS_ADVANCE_PARM_MACID_EXT_SH 16
+#define FWCMD_H2C_PS_ADVANCE_PARM_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_PERIODIC_WAKE_MACID_SH 0
+#define FWCMD_H2C_PERIODIC_WAKE_MACID_MSK 0xff
+#define FWCMD_H2C_PERIODIC_WAKE_ENABLE BIT(8)
+#define FWCMD_H2C_PERIODIC_WAKE_BAND BIT(9)
+#define FWCMD_H2C_PERIODIC_WAKE_PORT_SH 10
+#define FWCMD_H2C_PERIODIC_WAKE_PORT_MSK 0x7
+#define FWCMD_H2C_PERIODIC_WAKE_MACID_EXT_SH 13
+#define FWCMD_H2C_PERIODIC_WAKE_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_PERIODIC_WAKE_WAKE_PERIOD_SH 0
+#define FWCMD_H2C_PERIODIC_WAKE_WAKE_PERIOD_MSK 0xffffffff
+#define FWCMD_H2C_PERIODIC_WAKE_WAKE_DURATIOIN_SH 0
+#define FWCMD_H2C_PERIODIC_WAKE_WAKE_DURATIOIN_MSK 0xffffffff
+#define FWCMD_H2C_BB_RF_PWR_ST_BB_RF_PWR_REQ_ST_SH 0
+#define FWCMD_H2C_BB_RF_PWR_ST_BB_RF_PWR_REQ_ST_MSK 0xff
+#define FWCMD_H2C_PS_POWER_LEVEL_MACID_SH 0
+#define FWCMD_H2C_PS_POWER_LEVEL_MACID_MSK 0xff
+#define FWCMD_H2C_PS_POWER_LEVEL_BCN_TO_VAL_SH 8
+#define FWCMD_H2C_PS_POWER_LEVEL_BCN_TO_VAL_MSK 0xff
+#define FWCMD_H2C_PS_POWER_LEVEL_PS_LVL_SH 16
+#define FWCMD_H2C_PS_POWER_LEVEL_PS_LVL_MSK 0xf
+#define FWCMD_H2C_PS_POWER_LEVEL_TRX_LVL_SH 20
+#define FWCMD_H2C_PS_POWER_LEVEL_TRX_LVL_MSK 0xf
+#define FWCMD_H2C_PS_POWER_LEVEL_BCN_TO_LVL_SH 24
+#define FWCMD_H2C_PS_POWER_LEVEL_BCN_TO_LVL_MSK 0xf
+#define FWCMD_H2C_PS_POWER_LEVEL_DTIM_TO_VAL_SH 28
+#define FWCMD_H2C_PS_POWER_LEVEL_DTIM_TO_VAL_MSK 0xf
+#define FWCMD_H2C_PS_POWER_LEVEL_MACID_EXT_SH 0
+#define FWCMD_H2C_PS_POWER_LEVEL_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_LPS_OPTION_CFG_REQ_XTAL_OPTION BIT(0)
+#define FWCMD_H2C_DOZE_TIME_STATS_MACID_SH 0
+#define FWCMD_H2C_DOZE_TIME_STATS_MACID_MSK 0xff
+#define FWCMD_H2C_DOZE_TIME_STATS_RPT_PERIOD_SH 8
+#define FWCMD_H2C_DOZE_TIME_STATS_RPT_PERIOD_MSK 0xffff
+#define FWCMD_H2C_DOZE_TIME_STATS_ENABLE BIT(24)
+#define FWCMD_H2C_TBTT_TUNING_BAND_SH 0
+#define FWCMD_H2C_TBTT_TUNING_BAND_MSK 0xf
+#define FWCMD_H2C_TBTT_TUNING_PORT_SH 4
+#define FWCMD_H2C_TBTT_TUNING_PORT_MSK 0xf
+#define FWCMD_H2C_TBTT_TUNING_SHIFT_VAL_SH 0
+#define FWCMD_H2C_TBTT_TUNING_SHIFT_VAL_MSK 0xffffffff
+#define FWCMD_H2C_BKP_CAM_CFG_PL_CAM_NUM_SH 0
+#define FWCMD_H2C_BKP_CAM_CFG_PL_CAM_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_WOW_CAM_NUM_SH 8
+#define FWCMD_H2C_BKP_CAM_CFG_WOW_CAM_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_ADDR_CAM_NUM_SH 16
+#define FWCMD_H2C_BKP_CAM_CFG_ADDR_CAM_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_BSSID_CAM_NUM_SH 24
+#define FWCMD_H2C_BKP_CAM_CFG_BSSID_CAM_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_S_DCTRL_NUM_SH 0
+#define FWCMD_H2C_BKP_CAM_CFG_S_DCTRL_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_L_DCTRL_NUM_SH 8
+#define FWCMD_H2C_BKP_CAM_CFG_L_DCTRL_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_S_CCTRL_NUM_SH 16
+#define FWCMD_H2C_BKP_CAM_CFG_S_CCTRL_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_L_CCTRL_NUM_SH 24
+#define FWCMD_H2C_BKP_CAM_CFG_L_CCTRL_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_SEC_CAM_NUM_SH 0
+#define FWCMD_H2C_BKP_CAM_CFG_SEC_CAM_NUM_MSK 0xff
+#define FWCMD_H2C_BKP_CAM_CFG_BA_CAM_NUM_SH 8
+#define FWCMD_H2C_BKP_CAM_CFG_BA_CAM_NUM_MSK 0xff
+#define FWCMD_H2C_SMPS_CFG_MACID_SH 0
+#define FWCMD_H2C_SMPS_CFG_MACID_MSK 0xffff
+#define FWCMD_H2C_SMPS_CFG_SMPS_MODE_SH 16
+#define FWCMD_H2C_SMPS_CFG_SMPS_MODE_MSK 0x7
+#define FWCMD_H2C_RTD3_CFG_RTD3_EN BIT(0)
+#define FWCMD_H2C_RTD3_CFG_RTD3_KEEPALIVE_EN BIT(1)
+#define FWCMD_H2C_RTD3_CFG_RTD3_PVB_HIT_WAKE BIT(2)
+#define FWCMD_H2C_RTD3_CFG_RTD3_KA_PKT_TYPE BIT(4)
+#define FWCMD_H2C_RTD3_CFG_KEEPALIVE_INTERVAL_SH 8
+#define FWCMD_H2C_RTD3_CFG_KEEPALIVE_INTERVAL_MSK 0xff
+#define FWCMD_H2C_FWHDR_DL_FWHDR_SH 0
+#define FWCMD_H2C_FWHDR_DL_FWHDR_MSK 0xffffffff
+#define FWCMD_H2C_FWHDR_REDL_FWHDR_SH 0
+#define FWCMD_H2C_FWHDR_REDL_FWHDR_MSK 0xffffffff
+#define FWCMD_H2C_TWT_ANNOUNCE_UPD_MACID_SH 0
+#define FWCMD_H2C_TWT_ANNOUNCE_UPD_MACID_MSK 0xff
+#define FWCMD_H2C_TWTINFO_UPD_NEGOTYPE_SH 0
+#define FWCMD_H2C_TWTINFO_UPD_NEGOTYPE_MSK 0x3
+#define FWCMD_H2C_TWTINFO_UPD_TRIGGER BIT(2)
+#define FWCMD_H2C_TWTINFO_UPD_FLOWTYPE BIT(3)
+#define FWCMD_H2C_TWTINFO_UPD_IMPT BIT(4)
+#define FWCMD_H2C_TWTINFO_UPD_WAKEDURUNIT BIT(5)
+#define FWCMD_H2C_TWTINFO_UPD_RSPPM BIT(6)
+#define FWCMD_H2C_TWTINFO_UPD_FLOWID_SH 7
+#define FWCMD_H2C_TWTINFO_UPD_FLOWID_MSK 0x7
+#define FWCMD_H2C_TWTINFO_UPD_PROT BIT(10)
+#define FWCMD_H2C_TWTINFO_UPD_ACT_SH 11
+#define FWCMD_H2C_TWTINFO_UPD_ACT_MSK 0x7
+#define FWCMD_H2C_TWTINFO_UPD_ID_SH 14
+#define FWCMD_H2C_TWTINFO_UPD_ID_MSK 0x7
+#define FWCMD_H2C_TWTINFO_UPD_BAND BIT(17)
+#define FWCMD_H2C_TWTINFO_UPD_PORT_SH 18
+#define FWCMD_H2C_TWTINFO_UPD_PORT_MSK 0x7
+#define FWCMD_H2C_TWTINFO_UPD_TWT_TYPE_SH 21
+#define FWCMD_H2C_TWTINFO_UPD_TWT_TYPE_MSK 0x7
+#define FWCMD_H2C_TWTINFO_UPD_UNAVAILABILITY BIT(24)
+#define FWCMD_H2C_TWTINFO_UPD_NIC_ROLE BIT(25)
+#define FWCMD_H2C_TWTINFO_UPD_UNAVAIL_IND BIT(26)
+#define FWCMD_H2C_TWTINFO_UPD_WAKE_MAN_SH 0
+#define FWCMD_H2C_TWTINFO_UPD_WAKE_MAN_MSK 0xffff
+#define FWCMD_H2C_TWTINFO_UPD_WAKE_EXP_SH 16
+#define FWCMD_H2C_TWTINFO_UPD_WAKE_EXP_MSK 0x1f
+#define FWCMD_H2C_TWTINFO_UPD_ULFIXMODE_SH 21
+#define FWCMD_H2C_TWTINFO_UPD_ULFIXMODE_MSK 0x7
+#define FWCMD_H2C_TWTINFO_UPD_DUR_SH 24
+#define FWCMD_H2C_TWTINFO_UPD_DUR_MSK 0xff
+#define FWCMD_H2C_TWTINFO_UPD_TGT_L_SH 0
+#define FWCMD_H2C_TWTINFO_UPD_TGT_L_MSK 0xffffffff
+#define FWCMD_H2C_TWTINFO_UPD_TGT_H_SH 0
+#define FWCMD_H2C_TWTINFO_UPD_TGT_H_MSK 0xffffffff
+#define FWCMD_H2C_TWTINFO_UPD_PTT_BEF_SH 0
+#define FWCMD_H2C_TWTINFO_UPD_PTT_BEF_MSK 0xff
+#define FWCMD_H2C_TWTINFO_UPD_PTT_AFT_SH 8
+#define FWCMD_H2C_TWTINFO_UPD_PTT_AFT_MSK 0xff
+#define FWCMD_H2C_TWTINFO_UPD_DMA_CH_SH 16
+#define FWCMD_H2C_TWTINFO_UPD_DMA_CH_MSK 0xff
+#define FWCMD_H2C_TWTINFO_UPD_STRERLY_INTV_SH 24
+#define FWCMD_H2C_TWTINFO_UPD_STRERLY_INTV_MSK 0xf
+#define FWCMD_H2C_TWTINFO_UPD_ENDERLY_INTV_SH 28
+#define FWCMD_H2C_TWTINFO_UPD_ENDERLY_INTV_MSK 0xf
+#define FWCMD_H2C_TWT_STANSP_UPD_MACID_SH 0
+#define FWCMD_H2C_TWT_STANSP_UPD_MACID_MSK 0xff
+#define FWCMD_H2C_TWT_STANSP_UPD_ID_SH 8
+#define FWCMD_H2C_TWT_STANSP_UPD_ID_MSK 0x7
+#define FWCMD_H2C_TWT_STANSP_UPD_ACT_SH 11
+#define FWCMD_H2C_TWT_STANSP_UPD_ACT_MSK 0xf
+#define FWCMD_H2C_TWT_STANSP_UPD_BAND BIT(15)
+#define FWCMD_H2C_TWT_STANSP_UPD_UL_MAP_SH 16
+#define FWCMD_H2C_TWT_STANSP_UPD_UL_MAP_MSK 0xff
+#define FWCMD_H2C_TWT_STANSP_UPD_DL_MAP_SH 24
+#define FWCMD_H2C_TWT_STANSP_UPD_DL_MAP_MSK 0xff
+#define FWCMD_H2C_TBLUD_R_W BIT(0)
+#define FWCMD_H2C_TBLUD_MACID_GROUP_SH 1
+#define FWCMD_H2C_TBLUD_MACID_GROUP_MSK 0x7f
+#define FWCMD_H2C_TBLUD_OFFSET_SH 8
+#define FWCMD_H2C_TBLUD_OFFSET_MSK 0x1f
+#define FWCMD_H2C_TBLUD_LENGTH_SH 13
+#define FWCMD_H2C_TBLUD_LENGTH_MSK 0x3ff
+#define FWCMD_H2C_TBLUD_TYPE BIT(23)
+#define FWCMD_H2C_TBLUD_TABLE_CLASS_SH 24
+#define FWCMD_H2C_TBLUD_TABLE_CLASS_MSK 0xff
+#define FWCMD_H2C_DCTLINFO_UD_MACID_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_MACID_MSK 0x7f
+#define FWCMD_H2C_DCTLINFO_UD_OP BIT(7)
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL0_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL0_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL1_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL1_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL2_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL2_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL3_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL3_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL0_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL0_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL1_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL1_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL2_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL2_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL3_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_DCTRL3_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_MACID_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_MACID_MSK 0x7f
+#define FWCMD_H2C_CCTLINFO_UD_OP BIT(7)
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL0_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL0_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL1_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL1_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL2_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL2_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL3_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL3_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL4_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL4_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL5_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL5_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL6_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL6_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL7_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL7_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL0_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL0_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL1_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL1_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL2_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL2_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL3_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL3_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL4_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL4_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL5_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL5_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL6_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL6_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL7_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_CCTRL7_MSK_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D0_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D0_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D1_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D1_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D2_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D2_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D3_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D3_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D4_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D4_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D5_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D5_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D6_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D6_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D7_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D7_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D8_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D8_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D9_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D9_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D10_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D10_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D11_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D11_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D12_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D12_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D13_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D13_MSK 0xffffffff
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D14_SH 0
+#define FWCMD_H2C_SHCUT_UPDATE_SHCUT_UPDATE_D14_MSK 0xffffffff
+#define FWCMD_H2C_BCN_UPD_PORT_SH 0
+#define FWCMD_H2C_BCN_UPD_PORT_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_MBSSID_SH 8
+#define FWCMD_H2C_BCN_UPD_MBSSID_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_BAND_SH 16
+#define FWCMD_H2C_BCN_UPD_BAND_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_GRP_IE_OFST_SH 24
+#define FWCMD_H2C_BCN_UPD_GRP_IE_OFST_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_MACID_SH 0
+#define FWCMD_H2C_BCN_UPD_MACID_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_SSN_SEL_SH 8
+#define FWCMD_H2C_BCN_UPD_SSN_SEL_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_SSN_MODE_SH 10
+#define FWCMD_H2C_BCN_UPD_SSN_MODE_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_RATE_SH 12
+#define FWCMD_H2C_BCN_UPD_RATE_MSK 0x1ff
+#define FWCMD_H2C_BCN_UPD_TXPWR_SH 21
+#define FWCMD_H2C_BCN_UPD_TXPWR_MSK 0x7
+#define FWCMD_H2C_BCN_UPD_TXINFO_CTRL_EN BIT(0)
+#define FWCMD_H2C_BCN_UPD_NTX_PATH_EN_SH 1
+#define FWCMD_H2C_BCN_UPD_NTX_PATH_EN_MSK 0xf
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_A_SH 5
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_A_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_B_SH 7
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_B_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_C_SH 9
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_C_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_D_SH 11
+#define FWCMD_H2C_BCN_UPD_PATH_MAP_D_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_ANTSEL_A BIT(13)
+#define FWCMD_H2C_BCN_UPD_ANTSEL_B BIT(14)
+#define FWCMD_H2C_BCN_UPD_ANTSEL_C BIT(15)
+#define FWCMD_H2C_BCN_UPD_ANTSEL_D BIT(16)
+#define FWCMD_H2C_BCN_UPD_CSA_OFST_SH 17
+#define FWCMD_H2C_BCN_UPD_CSA_OFST_MSK 0x7fff
+#define FWCMD_H2C_SS_ULSTA_UPD_MODE_SH 0
+#define FWCMD_H2C_SS_ULSTA_UPD_MODE_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_BSR_TH_SEL BIT(8)
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U0_SH 0
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U0_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U1_SH 8
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U1_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U2_SH 16
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U2_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U3_SH 24
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U3_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_BSR_LEN_U0_SH 0
+#define FWCMD_H2C_SS_ULSTA_UPD_BSR_LEN_U0_MSK 0xffff
+#define FWCMD_H2C_SS_ULSTA_UPD_BSR_LEN_U1_SH 16
+#define FWCMD_H2C_SS_ULSTA_UPD_BSR_LEN_U1_MSK 0xffff
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U0_EXT_SH 0
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U0_EXT_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U1_EXT_SH 8
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U1_EXT_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U2_EXT_SH 16
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U2_EXT_MSK 0xff
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U3_EXT_SH 24
+#define FWCMD_H2C_SS_ULSTA_UPD_MACID_U3_EXT_MSK 0xff
+#define FWCMD_H2C_F2PDBG_SET_SET_TXDBG BIT(0)
+#define FWCMD_H2C_F2PDBG_SET_TXDBG_EN BIT(1)
+#define FWCMD_H2C_F2PDBG_SET_SET_TPPKTSIZE BIT(2)
+#define FWCMD_H2C_F2PDBG_SET_PKT_SIZE_SH 3
+#define FWCMD_H2C_F2PDBG_SET_PKT_SIZE_MSK 0xffff
+#define FWCMD_H2C_WLANINFO_GET_INFO_SEL_SH 0
+#define FWCMD_H2C_WLANINFO_GET_INFO_SEL_MSK 0xf
+#define FWCMD_H2C_WLANINFO_GET_ARGV0_SH 8
+#define FWCMD_H2C_WLANINFO_GET_ARGV0_MSK 0xff
+#define FWCMD_H2C_WLANINFO_GET_ARGV1_SH 16
+#define FWCMD_H2C_WLANINFO_GET_ARGV1_MSK 0xff
+#define FWCMD_H2C_WLANINFO_GET_ARGV2_SH 24
+#define FWCMD_H2C_WLANINFO_GET_ARGV2_MSK 0xff
+#define FWCMD_H2C_WLANINFO_GET_ARGV3_SH 0
+#define FWCMD_H2C_WLANINFO_GET_ARGV3_MSK 0xff
+#define FWCMD_H2C_WLANINFO_GET_ARGV4_SH 8
+#define FWCMD_H2C_WLANINFO_GET_ARGV4_MSK 0xff
+#define FWCMD_H2C_WLANINFO_GET_ARGV5_SH 16
+#define FWCMD_H2C_WLANINFO_GET_ARGV5_MSK 0xff
+#define FWCMD_H2C_WLANINFO_GET_ARGV6_SH 24
+#define FWCMD_H2C_WLANINFO_GET_ARGV6_MSK 0xff
+#define FWCMD_H2C_WLANINFO_GET_ARGV7_SH 0
+#define FWCMD_H2C_WLANINFO_GET_ARGV7_MSK 0xff
+#define FWCMD_H2C_DCTLINFO_UD_V1_MACID_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_MACID_MSK 0x7f
+#define FWCMD_H2C_DCTLINFO_UD_V1_OP BIT(7)
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL0_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL0_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL1_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL1_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL2_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL2_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL3_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL3_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL4_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL4_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL5_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL5_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL6_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL6_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL7_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL7_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL0_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL0_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL1_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL1_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL2_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL2_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL3_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL3_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL4_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL4_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL5_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL5_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL6_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL6_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL7_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V1_DCTRL7_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_MACID_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_MACID_MSK 0x7f
+#define FWCMD_H2C_CCTLINFO_UD_V1_OP BIT(7)
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL0_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL0_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL1_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL1_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL2_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL2_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL3_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL3_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL4_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL4_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL5_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL5_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL6_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL6_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL7_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL7_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL0_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL0_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL1_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL1_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL2_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL2_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL3_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL3_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL4_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL4_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL5_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL5_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL6_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL6_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL7_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_V1_CCTRL7_MSK_MSK 0xffffffff
+#define FWCMD_H2C_FW_STS_PARA_EN BIT(0)
+#define FWCMD_H2C_FW_STS_PARA_BAND BIT(1)
+#define FWCMD_H2C_FW_STS_PARA_INTVL_MS_SH 16
+#define FWCMD_H2C_FW_STS_PARA_INTVL_MS_MSK 0xffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_MACID_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_MACID_MSK 0x7f
+#define FWCMD_H2C_DCTLINFO_UD_V2_OP BIT(7)
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL0_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL0_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL1_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL1_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL2_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL2_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL3_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL3_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL4_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL4_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL5_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL5_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL6_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL6_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL7_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL7_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL8_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL8_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL9_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL9_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL10_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL10_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL11_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL11_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL12_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL12_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL13_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL13_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL14_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL14_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL15_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL15_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL0_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL0_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL1_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL1_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL2_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL2_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL3_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL3_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL4_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL4_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL5_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL5_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL6_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL6_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL7_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL7_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL8_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL8_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL9_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL9_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL10_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL10_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL11_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL11_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL12_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL12_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL13_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL13_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL14_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL14_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL15_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V2_DCTRL15_MSK_MSK 0xffffffff
+#define FWCMD_H2C_BCN_UPD_BE_PORT_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_PORT_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_BE_MBSSID_SH 8
+#define FWCMD_H2C_BCN_UPD_BE_MBSSID_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_BE_BAND_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_BAND_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_BE_GRP_IE_OFST_SH 24
+#define FWCMD_H2C_BCN_UPD_BE_GRP_IE_OFST_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_BE_MACID_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_MACID_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_BE_SSN_SEL_SH 8
+#define FWCMD_H2C_BCN_UPD_BE_SSN_SEL_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_BE_SSN_MODE_SH 10
+#define FWCMD_H2C_BCN_UPD_BE_SSN_MODE_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_BE_RATE_SH 12
+#define FWCMD_H2C_BCN_UPD_BE_RATE_MSK 0x1ff
+#define FWCMD_H2C_BCN_UPD_BE_TXPWR_SH 21
+#define FWCMD_H2C_BCN_UPD_BE_TXPWR_MSK 0x7
+#define FWCMD_H2C_BCN_UPD_BE_MACID_EXT_SH 24
+#define FWCMD_H2C_BCN_UPD_BE_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_BE_TXINFO_CTRL_EN BIT(0)
+#define FWCMD_H2C_BCN_UPD_BE_NTX_PATH_EN_SH 1
+#define FWCMD_H2C_BCN_UPD_BE_NTX_PATH_EN_MSK 0xf
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_A_SH 5
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_A_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_B_SH 7
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_B_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_C_SH 9
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_C_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_D_SH 11
+#define FWCMD_H2C_BCN_UPD_BE_PATH_MAP_D_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_BE_ANTSEL_A BIT(13)
+#define FWCMD_H2C_BCN_UPD_BE_ANTSEL_B BIT(14)
+#define FWCMD_H2C_BCN_UPD_BE_ANTSEL_C BIT(15)
+#define FWCMD_H2C_BCN_UPD_BE_ANTSEL_D BIT(16)
+#define FWCMD_H2C_BCN_UPD_BE_CSA_OFST_SH 17
+#define FWCMD_H2C_BCN_UPD_BE_CSA_OFST_MSK 0x7fff
+#define FWCMD_H2C_BCN_UPD_BE_MLIE_CSA_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_MLIE_CSA_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_CRITICAL_UPD_FLAG_OFST_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_CRITICAL_UPD_FLAG_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_VAP1_DTIM_CNT_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_VAP1_DTIM_CNT_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_VAP2_DTIM_CNT_OFST_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_VAP2_DTIM_CNT_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_VAP3_DTIM_CNT_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_VAP3_DTIM_CNT_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_VAP4_DTIM_CNT_OFST_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_VAP4_DTIM_CNT_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_VAP5_DTIM_CNT_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_VAP5_DTIM_CNT_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_VAP6_DTIM_CNT_OFST_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_VAP6_DTIM_CNT_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_VAP7_DTIM_CNT_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_VAP7_DTIM_CNT_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_ECSA_OFST_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_ECSA_OFST_MSK 0x7fff
+#define FWCMD_H2C_BCN_UPD_BE_PROTECTION_KEY_ID BIT(31)
+#define FWCMD_H2C_BCN_UPD_BE_SEC_ENABLE BIT(0)
+#define FWCMD_H2C_BCN_UPD_BE_PN_RESET BIT(1)
+#define FWCMD_H2C_BCN_UPD_BE_SEC_ALGO_SH 2
+#define FWCMD_H2C_BCN_UPD_BE_SEC_ALGO_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_BE_PN_HIGH_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_PN_HIGH_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_PN_LOW_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_PN_LOW_MSK 0xffffffff
+#define FWCMD_H2C_BCN_UPD_BE_MLIE_ECSA_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_MLIE_ECSA_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_MLIE1_CSA_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_MLIE1_CSA_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_MLIE1_ECSA_OFST_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_MLIE1_ECSA_OFST_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST0_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST0_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST1_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST1_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST2_SH 0
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST2_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST3_SH 16
+#define FWCMD_H2C_BCN_UPD_BE_TWT_OFST3_MSK 0xffff
+#define FWCMD_H2C_BCN_IGNORE_EDCCA_PORT_SH 0
+#define FWCMD_H2C_BCN_IGNORE_EDCCA_PORT_MSK 0xff
+#define FWCMD_H2C_BCN_IGNORE_EDCCA_MBSSID_SH 8
+#define FWCMD_H2C_BCN_IGNORE_EDCCA_MBSSID_MSK 0xff
+#define FWCMD_H2C_BCN_IGNORE_EDCCA_BAND_SH 16
+#define FWCMD_H2C_BCN_IGNORE_EDCCA_BAND_MSK 0xff
+#define FWCMD_H2C_BCN_IGNORE_EDCCA_IGNORE_EDCCA_EN BIT(24)
+#define FWCMD_H2C_BCN_UPD_V1_PORT_SH 0
+#define FWCMD_H2C_BCN_UPD_V1_PORT_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_V1_MBSSID_SH 8
+#define FWCMD_H2C_BCN_UPD_V1_MBSSID_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_V1_BAND_SH 16
+#define FWCMD_H2C_BCN_UPD_V1_BAND_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_V1_GRP_IE_OFST_SH 24
+#define FWCMD_H2C_BCN_UPD_V1_GRP_IE_OFST_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_V1_MACID_SH 0
+#define FWCMD_H2C_BCN_UPD_V1_MACID_MSK 0xff
+#define FWCMD_H2C_BCN_UPD_V1_SSN_SEL_SH 8
+#define FWCMD_H2C_BCN_UPD_V1_SSN_SEL_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_V1_SSN_MODE_SH 10
+#define FWCMD_H2C_BCN_UPD_V1_SSN_MODE_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_V1_RATE_SH 12
+#define FWCMD_H2C_BCN_UPD_V1_RATE_MSK 0x1ff
+#define FWCMD_H2C_BCN_UPD_V1_TXPWR_SH 21
+#define FWCMD_H2C_BCN_UPD_V1_TXPWR_MSK 0x7
+#define FWCMD_H2C_BCN_UPD_V1_ECSA_SUPPORT BIT(24)
+#define FWCMD_H2C_BCN_UPD_V1_PROTECTION_KEY_ID BIT(25)
+#define FWCMD_H2C_BCN_UPD_V1_SEC_ALGO_SH 26
+#define FWCMD_H2C_BCN_UPD_V1_SEC_ALGO_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_V1_SEC_ENABLE BIT(28)
+#define FWCMD_H2C_BCN_UPD_V1_PN_RESET BIT(29)
+#define FWCMD_H2C_BCN_UPD_V1_TXINFO_CTRL_EN BIT(0)
+#define FWCMD_H2C_BCN_UPD_V1_NTX_PATH_EN_SH 1
+#define FWCMD_H2C_BCN_UPD_V1_NTX_PATH_EN_MSK 0xf
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_A_SH 5
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_A_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_B_SH 7
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_B_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_C_SH 9
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_C_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_D_SH 11
+#define FWCMD_H2C_BCN_UPD_V1_PATH_MAP_D_MSK 0x3
+#define FWCMD_H2C_BCN_UPD_V1_ANTSEL_A BIT(13)
+#define FWCMD_H2C_BCN_UPD_V1_ANTSEL_B BIT(14)
+#define FWCMD_H2C_BCN_UPD_V1_ANTSEL_C BIT(15)
+#define FWCMD_H2C_BCN_UPD_V1_ANTSEL_D BIT(16)
+#define FWCMD_H2C_BCN_UPD_V1_CSA_OFST_SH 17
+#define FWCMD_H2C_BCN_UPD_V1_CSA_OFST_MSK 0x7fff
+#define FWCMD_H2C_BCN_UPD_V1_ECSA_OFST_SH 0
+#define FWCMD_H2C_BCN_UPD_V1_ECSA_OFST_MSK 0x7fff
+#define FWCMD_H2C_BCN_UPD_V1_BC_CHG_OFST_SH 15
+#define FWCMD_H2C_BCN_UPD_V1_BC_CHG_OFST_MSK 0x7fff
+#define FWCMD_H2C_BCN_UPD_V1_CSA_RESET BIT(30)
+#define FWCMD_H2C_BCN_UPD_V1_ECSA_RESET BIT(31)
+#define FWCMD_H2C_BCN_UPD_V1_PN_LOW_SH 0
+#define FWCMD_H2C_BCN_UPD_V1_PN_LOW_MSK 0xffffffff
+#define FWCMD_H2C_BCN_UPD_V1_PN_HIGH_SH 0
+#define FWCMD_H2C_BCN_UPD_V1_PN_HIGH_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST0_SH 0
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST0_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST1_SH 16
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST1_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST2_SH 0
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST2_MSK 0xffff
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST3_SH 16
+#define FWCMD_H2C_BCN_UPD_V1_TWT_OFST3_MSK 0xffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_MACID_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_MACID_MSK 0xffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_OP BIT(16)
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL0_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL0_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL1_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL1_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL2_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL2_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL3_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL3_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL4_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL4_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL5_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL5_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL6_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL6_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL7_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL7_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL8_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL8_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL9_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL9_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL10_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL10_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL11_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL11_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL12_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL12_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL13_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL13_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL14_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL14_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL15_VAL_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL15_VAL_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL0_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL0_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL1_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL1_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL2_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL2_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL3_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL3_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL4_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL4_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL5_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL5_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL6_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL6_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL7_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL7_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL8_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL8_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL9_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL9_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL10_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL10_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL11_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL11_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL12_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL12_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL13_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL13_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL14_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL14_MSK_MSK 0xffffffff
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL15_MSK_SH 0
+#define FWCMD_H2C_DCTLINFO_UD_V3_DCTRL15_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_MACID_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_MACID_MSK 0x7f
+#define FWCMD_H2C_CCTLINFO_UD_G7_OP BIT(7)
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL0_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL0_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL1_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL1_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL2_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL2_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL3_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL3_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL4_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL4_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL5_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL5_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL6_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL6_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL7_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL7_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL8_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL8_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL9_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL9_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL10_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL10_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL11_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL11_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL12_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL12_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL13_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL13_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL14_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL14_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL15_VAL_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL15_VAL_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL0_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL0_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL1_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL1_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL2_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL2_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL3_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL3_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL4_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL4_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL5_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL5_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL6_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL6_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL7_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL7_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL8_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL8_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL9_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL9_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL10_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL10_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL11_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL11_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL12_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL12_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL13_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL13_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL14_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL14_MSK_MSK 0xffffffff
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL15_MSK_SH 0
+#define FWCMD_H2C_CCTLINFO_UD_G7_CCTRL15_MSK_MSK 0xffffffff
+#define FWCMD_H2C_SWTX_STACFG_MACID_SH 0
+#define FWCMD_H2C_SWTX_STACFG_MACID_MSK 0x3ff
+#define FWCMD_H2C_SWTX_STACFG_RATE_SH 16
+#define FWCMD_H2C_SWTX_STACFG_RATE_MSK 0xfff
+#define FWCMD_H2C_TXMODE_SWITCH_MODE BIT(0)
+#define FWCMD_H2C_TXMODE_SWITCH_SW_MODE_MAX_CHAIN_NUM_SH 1
+#define FWCMD_H2C_TXMODE_SWITCH_SW_MODE_MAX_CHAIN_NUM_MSK 0x7
+#define FWCMD_H2C_TXMODE_SWITCH_DECISION_GAIN_SH 4
+#define FWCMD_H2C_TXMODE_SWITCH_DECISION_GAIN_MSK 0xff
+#define FWCMD_H2C_TXMODE_SWITCH_SWAP_PRIMARY_ENABLE BIT(12)
+#define FWCMD_H2C_BCN_DYNAMIC_MECH_BCN_DM_TBTT_SHFT_EN BIT(0)
+#define FWCMD_H2C_ADDRCAM_INFO_R_W_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_R_W_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_IDX_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_IDX_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_OFFSET_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_OFFSET_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_LEN_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_LEN_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_IDX_EXT_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_IDX_EXT_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_VALID BIT(0)
+#define FWCMD_H2C_ADDRCAM_INFO_NET_TYPE_SH 1
+#define FWCMD_H2C_ADDRCAM_INFO_NET_TYPE_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_BCN_HIT_COND_SH 3
+#define FWCMD_H2C_ADDRCAM_INFO_BCN_HIT_COND_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_HIT_RULE_SH 5
+#define FWCMD_H2C_ADDRCAM_INFO_HIT_RULE_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_BB_SEL BIT(7)
+#define FWCMD_H2C_ADDRCAM_INFO_ADDR_MASK_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_ADDR_MASK_MSK 0x3f
+#define FWCMD_H2C_ADDRCAM_INFO_MASK_SEL_SH 14
+#define FWCMD_H2C_ADDRCAM_INFO_MASK_SEL_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SMA_HASH_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_SMA_HASH_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TMA_HASH_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_TMA_HASH_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID_CAM_IDX_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID_CAM_IDX_MSK 0x3f
+#define FWCMD_H2C_ADDRCAM_INFO_IS_MUL_ENT BIT(8)
+#define FWCMD_H2C_ADDRCAM_INFO_SMA0_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_SMA0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SMA1_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_SMA1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SMA2_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_SMA2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SMA3_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_SMA3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SMA4_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_SMA4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SMA5_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_SMA5_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TMA0_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_TMA0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TMA1_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_TMA1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TMA2_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_TMA2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TMA3_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_TMA3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TMA4_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_TMA4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TMA5_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_TMA5_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_MACID_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_MACID_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_PORT_INT_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_PORT_INT_MSK 0x7
+#define FWCMD_H2C_ADDRCAM_INFO_TSF_SYNC_SH 11
+#define FWCMD_H2C_ADDRCAM_INFO_TSF_SYNC_MSK 0x7
+#define FWCMD_H2C_ADDRCAM_INFO_TF_TRS BIT(14)
+#define FWCMD_H2C_ADDRCAM_INFO_LSIG_TXOP BIT(15)
+#define FWCMD_H2C_ADDRCAM_INFO_MACID_EXT_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_TGT_IND_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_TGT_IND_MSK 0x7
+#define FWCMD_H2C_ADDRCAM_INFO_FRM_TGT_IND_SH 27
+#define FWCMD_H2C_ADDRCAM_INFO_FRM_TGT_IND_MSK 0x7
+#define FWCMD_H2C_ADDRCAM_INFO_TB_RANGING BIT(30)
+#define FWCMD_H2C_ADDRCAM_INFO_AID12_0_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_AID12_0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_AID12_1_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_AID12_1_MSK 0xf
+#define FWCMD_H2C_ADDRCAM_INFO_WOL_PATTERN BIT(12)
+#define FWCMD_H2C_ADDRCAM_INFO_WOL_UC BIT(13)
+#define FWCMD_H2C_ADDRCAM_INFO_WOL_MAGIC BIT(14)
+#define FWCMD_H2C_ADDRCAM_INFO_WAPI BIT(15)
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT_MODE_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT_MODE_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT0_KEYID_SH 18
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT0_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT1_KEYID_SH 20
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT1_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT2_KEYID_SH 22
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT2_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT3_KEYID_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT3_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT4_KEYID_SH 26
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT4_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT5_KEYID_SH 28
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT5_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT6_KEYID_SH 30
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT6_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT_VALID_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT_VALID_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT0_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT1_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT2_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT3_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT4_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT5_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT5_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT6_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_SEC_ENT6_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_B_IDX_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_B_IDX_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_B_OFFSET_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_B_OFFSET_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_B_LEN_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_B_LEN_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_B_VALID BIT(0)
+#define FWCMD_H2C_ADDRCAM_INFO_B_BB_SEL BIT(1)
+#define FWCMD_H2C_ADDRCAM_INFO_B_MSK_SH 2
+#define FWCMD_H2C_ADDRCAM_INFO_B_MSK_MSK 0x3f
+#define FWCMD_H2C_ADDRCAM_INFO_BSS_COLOR_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_BSS_COLOR_MSK 0x7f
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID0_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID1_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID2_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID3_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID4_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID5_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_BSSID5_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_R_W_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_R_W_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_IDX_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_IDX_MSK 0x3ff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_OFFSET_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_OFFSET_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_LEN_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_LEN_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_VALID BIT(0)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_NET_TYPE_SH 1
+#define FWCMD_H2C_ADDRCAM_INFO_V1_NET_TYPE_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BCN_HIT_COND_SH 3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BCN_HIT_COND_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_HIT_RULE_SH 5
+#define FWCMD_H2C_ADDRCAM_INFO_V1_HIT_RULE_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BB_SEL BIT(7)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_ADDR_MASK_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_ADDR_MASK_MSK 0x3f
+#define FWCMD_H2C_ADDRCAM_INFO_V1_MASK_SEL_SH 14
+#define FWCMD_H2C_ADDRCAM_INFO_V1_MASK_SEL_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA_HASH_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA_HASH_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA_HASH_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA_HASH_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID_CAM_IDX_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID_CAM_IDX_MSK 0x3f
+#define FWCMD_H2C_ADDRCAM_INFO_V1_IS_MUL_ENT BIT(8)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA0_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA1_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA2_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA3_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA4_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA5_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SMA5_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA0_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA1_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA2_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA3_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA4_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA5_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TMA5_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_MACID_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_MACID_MSK 0x3ff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_PORT_INT_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_PORT_INT_MSK 0x7
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TSF_SYNC_SH 19
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TSF_SYNC_MSK 0x7
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TF_TRS BIT(22)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_LSIG_TXOP BIT(23)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TB_RANGING BIT(24)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_TB_SENSING BIT(25)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SENS_EN BIT(26)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_AID12_0_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_AID12_0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_AID12_1_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_AID12_1_MSK 0xf
+#define FWCMD_H2C_ADDRCAM_INFO_V1_WOL_PATTERN BIT(12)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_WOL_UC BIT(13)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_WOL_MAGIC BIT(14)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_WAPI BIT(15)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT_MODE_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT_MODE_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT0_KEYID_SH 18
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT0_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT1_KEYID_SH 20
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT1_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT2_KEYID_SH 22
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT2_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT3_KEYID_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT3_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT4_KEYID_SH 26
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT4_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT5_KEYID_SH 28
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT5_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT6_KEYID_SH 30
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT6_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT_VALID_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT_VALID_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT0_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT1_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT2_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT3_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT4_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT5_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT5_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT6_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_SEC_ENT6_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_IDX_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_IDX_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_OFFSET_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_OFFSET_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_LEN_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_LEN_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_VALID BIT(0)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_BB_SEL BIT(1)
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_MSK_SH 2
+#define FWCMD_H2C_ADDRCAM_INFO_V1_B_MSK_MSK 0x3f
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSS_COLOR_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSS_COLOR_MSK 0x7f
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID0_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID0_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID1_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID1_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID2_SH 0
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID2_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID3_SH 8
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID3_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID4_SH 16
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID4_MSK 0xff
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID5_SH 24
+#define FWCMD_H2C_ADDRCAM_INFO_V1_BSSID5_MSK 0xff
+#define FWCMD_H2C_JOININFO_MACID_SH 0
+#define FWCMD_H2C_JOININFO_MACID_MSK 0xff
+#define FWCMD_H2C_JOININFO_OPMODE BIT(8)
+#define FWCMD_H2C_JOININFO_BAND BIT(9)
+#define FWCMD_H2C_JOININFO_WMM_SH 10
+#define FWCMD_H2C_JOININFO_WMM_MSK 0x3
+#define FWCMD_H2C_JOININFO_TRIGGER BIT(12)
+#define FWCMD_H2C_JOININFO_ISHESTA BIT(13)
+#define FWCMD_H2C_JOININFO_DL_BW_SH 14
+#define FWCMD_H2C_JOININFO_DL_BW_MSK 0x3
+#define FWCMD_H2C_JOININFO_TF_MAC_PADDING_SH 16
+#define FWCMD_H2C_JOININFO_TF_MAC_PADDING_MSK 0x3
+#define FWCMD_H2C_JOININFO_DL_T_PE_SH 18
+#define FWCMD_H2C_JOININFO_DL_T_PE_MSK 0x7
+#define FWCMD_H2C_JOININFO_PORT_ID_SH 21
+#define FWCMD_H2C_JOININFO_PORT_ID_MSK 0x7
+#define FWCMD_H2C_JOININFO_NET_TYPE_SH 24
+#define FWCMD_H2C_JOININFO_NET_TYPE_MSK 0x3
+#define FWCMD_H2C_JOININFO_WIFI_ROLE_SH 26
+#define FWCMD_H2C_JOININFO_WIFI_ROLE_MSK 0xf
+#define FWCMD_H2C_JOININFO_SELF_ROLE_SH 30
+#define FWCMD_H2C_JOININFO_SELF_ROLE_MSK 0x3
+#define FWCMD_H2C_JOININFO_STA_TYPE_SH 0
+#define FWCMD_H2C_JOININFO_STA_TYPE_MSK 0x7
+#define FWCMD_H2C_JOININFO_IS_MLD BIT(3)
+#define FWCMD_H2C_JOININFO_MAIN_MACID_SH 4
+#define FWCMD_H2C_JOININFO_MAIN_MACID_MSK 0xff
+#define FWCMD_H2C_JOININFO_MLO_MODE BIT(12)
+#define FWCMD_H2C_JOININFO_EMLSR_CAB BIT(13)
+#define FWCMD_H2C_JOININFO_NSTR_EN BIT(14)
+#define FWCMD_H2C_JOININFO_INIT_PWR_STATE BIT(15)
+#define FWCMD_H2C_JOININFO_EMLSR_PADDING_SH 16
+#define FWCMD_H2C_JOININFO_EMLSR_PADDING_MSK 0x7
+#define FWCMD_H2C_JOININFO_EMLSR_TRANS_DELAY_SH 19
+#define FWCMD_H2C_JOININFO_EMLSR_TRANS_DELAY_MSK 0x7
+#define FWCMD_H2C_JOININFO_IS_MLD_EXT BIT(22)
+#define FWCMD_H2C_JOININFO_MACID_EXT_SH 0
+#define FWCMD_H2C_JOININFO_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_JOININFO_MAIN_MACID_EXT_SH 8
+#define FWCMD_H2C_JOININFO_MAIN_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_JOININFO_V1_MACID_SH 0
+#define FWCMD_H2C_JOININFO_V1_MACID_MSK 0x3ff
+#define FWCMD_H2C_JOININFO_V1_OPMODE BIT(10)
+#define FWCMD_H2C_JOININFO_V1_BAND BIT(11)
+#define FWCMD_H2C_JOININFO_V1_WMM_SH 12
+#define FWCMD_H2C_JOININFO_V1_WMM_MSK 0x3
+#define FWCMD_H2C_JOININFO_V1_TRIGGER BIT(14)
+#define FWCMD_H2C_JOININFO_V1_ISHESTA BIT(15)
+#define FWCMD_H2C_JOININFO_V1_DL_BW_SH 16
+#define FWCMD_H2C_JOININFO_V1_DL_BW_MSK 0x7
+#define FWCMD_H2C_JOININFO_V1_TF_MAC_PADDING_SH 19
+#define FWCMD_H2C_JOININFO_V1_TF_MAC_PADDING_MSK 0x3
+#define FWCMD_H2C_JOININFO_V1_DL_T_PE_SH 21
+#define FWCMD_H2C_JOININFO_V1_DL_T_PE_MSK 0x7
+#define FWCMD_H2C_JOININFO_V1_PORT_ID_SH 24
+#define FWCMD_H2C_JOININFO_V1_PORT_ID_MSK 0x7
+#define FWCMD_H2C_JOININFO_V1_NET_TYPE_SH 27
+#define FWCMD_H2C_JOININFO_V1_NET_TYPE_MSK 0x3
+#define FWCMD_H2C_JOININFO_V1_WIFI_ROLE_SH 0
+#define FWCMD_H2C_JOININFO_V1_WIFI_ROLE_MSK 0xf
+#define FWCMD_H2C_JOININFO_V1_SELF_ROLE_SH 4
+#define FWCMD_H2C_JOININFO_V1_SELF_ROLE_MSK 0x3
+#define FWCMD_H2C_JOININFO_V1_STA_TYPE_SH 6
+#define FWCMD_H2C_JOININFO_V1_STA_TYPE_MSK 0x7
+#define FWCMD_H2C_JOININFO_V1_IS_MLD BIT(9)
+#define FWCMD_H2C_JOININFO_V1_MAIN_MACID_SH 10
+#define FWCMD_H2C_JOININFO_V1_MAIN_MACID_MSK 0xff
+#define FWCMD_H2C_JOININFO_V1_MLO_MODE BIT(18)
+#define FWCMD_H2C_JOININFO_V1_EMLSR_CAB BIT(19)
+#define FWCMD_H2C_JOININFO_V1_NSTR_EN BIT(20)
+#define FWCMD_H2C_JOININFO_V1_INIT_PWR_STATE BIT(21)
+#define FWCMD_H2C_JOININFO_V1_EMLSR_PADDING_SH 22
+#define FWCMD_H2C_JOININFO_V1_EMLSR_PADDING_MSK 0x7
+#define FWCMD_H2C_JOININFO_V1_EMLSR_TRANS_DELAY_SH 25
+#define FWCMD_H2C_JOININFO_V1_EMLSR_TRANS_DELAY_MSK 0x7
+#define FWCMD_H2C_DL_GRP_UPD_GRP_VALID BIT(0)
+#define FWCMD_H2C_DL_GRP_UPD_GRP_ID_SH 1
+#define FWCMD_H2C_DL_GRP_UPD_GRP_ID_MSK 0x1f
+#define FWCMD_H2C_DL_GRP_UPD_IS_HWGRP BIT(6)
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U0_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U0_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U1_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U1_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U2_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U2_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U3_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U3_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U4_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U4_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U5_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U5_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U6_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U6_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U7_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_MACID_U7_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U0_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U0_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U1_SH 12
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U1_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U2_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U2_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U3_SH 20
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U3_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U4_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U4_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U5_SH 28
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U5_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U6_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U6_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U7_SH 4
+#define FWCMD_H2C_DL_GRP_UPD_AC_BITMAP_U7_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_NEXT_PROTECTTYPE_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_NEXT_PROTECTTYPE_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_NEXT_RSPTYPE_SH 12
+#define FWCMD_H2C_DL_GRP_UPD_NEXT_RSPTYPE_MSK 0xf
+#define FWCMD_H2C_DL_GRP_UPD_TRIGGER_LEN_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_TRIGGER_LEN_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_TRIGGER_UL_THRESHOLD_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_TRIGGER_UL_THRESHOLD_MSK 0xff
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U0_SH 0
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U0_MSK 0xff
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U1_SH 8
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U1_MSK 0xff
+#define FWCMD_H2C_UL_GRP_UPD_GRP_BITMAP_SH 16
+#define FWCMD_H2C_UL_GRP_UPD_GRP_BITMAP_MSK 0xffff
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U0_EXT_SH 0
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U0_EXT_MSK 0xff
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U1_EXT_SH 8
+#define FWCMD_H2C_UL_GRP_UPD_MACID_U1_EXT_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_MACID_SH 0
+#define FWCMD_H2C_MU_STA_UPD_MACID_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_MU_IDX_SH 8
+#define FWCMD_H2C_MU_STA_UPD_MU_IDX_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_0_SH 16
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_0_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_1_SH 24
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_1_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_2_SH 0
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_2_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_3_SH 8
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_3_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_4_SH 16
+#define FWCMD_H2C_MU_STA_UPD_PROT_RSP_TYPE_4_MSK 0xff
+#define FWCMD_H2C_MU_STA_UPD_MUGRP_BITMAP_SH 24
+#define FWCMD_H2C_MU_STA_UPD_MUGRP_BITMAP_MSK 0x1f
+#define FWCMD_H2C_MU_STA_UPD_DIS_256Q BIT(29)
+#define FWCMD_H2C_MU_STA_UPD_DIS_1024Q BIT(30)
+#define FWCMD_H2C_MU_STA_UPD_ACTION BIT(31)
+#define FWCMD_H2C_MU_STA_UPD_MACID_EXT_SH 0
+#define FWCMD_H2C_MU_STA_UPD_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_FWROLE_MAINTAIN_MACID_SH 0
+#define FWCMD_H2C_FWROLE_MAINTAIN_MACID_MSK 0xff
+#define FWCMD_H2C_FWROLE_MAINTAIN_SELF_ROLE_SH 8
+#define FWCMD_H2C_FWROLE_MAINTAIN_SELF_ROLE_MSK 0x3
+#define FWCMD_H2C_FWROLE_MAINTAIN_UPD_MODE_SH 10
+#define FWCMD_H2C_FWROLE_MAINTAIN_UPD_MODE_MSK 0x7
+#define FWCMD_H2C_FWROLE_MAINTAIN_WIFI_ROLE_SH 13
+#define FWCMD_H2C_FWROLE_MAINTAIN_WIFI_ROLE_MSK 0xf
+#define FWCMD_H2C_FWROLE_MAINTAIN_BAND_SH 17
+#define FWCMD_H2C_FWROLE_MAINTAIN_BAND_MSK 0x3
+#define FWCMD_H2C_FWROLE_MAINTAIN_PORT_SH 19
+#define FWCMD_H2C_FWROLE_MAINTAIN_PORT_MSK 0x7
+#define FWCMD_H2C_FWROLE_MAINTAIN_MACID_EXT_SH 24
+#define FWCMD_H2C_FWROLE_MAINTAIN_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_NOTIFY_DBCC_EN BIT(0)
+#define FWCMD_H2C_DL_GRP_UPD_V2_GRP_VALID_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_GRP_VALID_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_GRP_ID_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_GRP_ID_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_IS_HWGRP_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_IS_HWGRP_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MRU_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_MRU_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U0_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U0_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U1_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U1_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U2_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U2_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U3_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U3_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U4_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U4_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U5_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U5_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U6_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U6_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U7_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_MACID_U7_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U0_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U0_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U1_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U1_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U2_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U2_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U3_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U3_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U4_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U4_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U5_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U5_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U6_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U6_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U7_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_AC_BITMAP_U7_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_NEXT_PROTECTTYPE_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_NEXT_PROTECTTYPE_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_NEXT_RSPTYPE_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_NEXT_RSPTYPE_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_W_IDX_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_W_IDX_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_BAND BIT(24)
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U0_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U0_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U1_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U1_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U2_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U2_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U3_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U3_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U4_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U4_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U5_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U5_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U6_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U6_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U7_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_DCM_U7_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U0_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U0_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U1_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U1_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U2_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U2_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U3_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U3_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U4_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U4_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U5_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U5_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U6_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U6_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U7_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_SS_U7_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U0_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U0_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U1_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U1_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U2_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U2_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U3_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U3_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U4_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U4_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U5_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U5_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U6_SH 16
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U6_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U7_SH 24
+#define FWCMD_H2C_DL_GRP_UPD_V2_RATE_IDX_U7_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_TRIGGER_LEN_SH 0
+#define FWCMD_H2C_DL_GRP_UPD_V2_TRIGGER_LEN_MSK 0xff
+#define FWCMD_H2C_DL_GRP_UPD_V2_TRIGGER_UL_THRESHOLD_SH 8
+#define FWCMD_H2C_DL_GRP_UPD_V2_TRIGGER_UL_THRESHOLD_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FW_FLOW_CONTROL_EN BIT(0)
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_NUMS_SH 1
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_NUMS_MSK 0xf
+#define FWCMD_H2C_FW_FLOW_CTRL_NORMAL_FLOW_WMM BIT(5)
+#define FWCMD_H2C_FW_FLOW_CTRL_NORMAL_FLOW_AC_SH 6
+#define FWCMD_H2C_FW_FLOW_CTRL_NORMAL_FLOW_AC_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_LATENCY_FLOW_WMM BIT(8)
+#define FWCMD_H2C_FW_FLOW_CTRL_LATENCY_FLOW_AC_SH 9
+#define FWCMD_H2C_FW_FLOW_CTRL_LATENCY_FLOW_AC_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_THROUGHPUT_FLOW_WMM BIT(11)
+#define FWCMD_H2C_FW_FLOW_CTRL_THROUGHPUT_FLOW_AC_SH 12
+#define FWCMD_H2C_FW_FLOW_CTRL_THROUGHPUT_FLOW_AC_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_HYBRID_FLOW_WMM BIT(14)
+#define FWCMD_H2C_FW_FLOW_CTRL_HYBRID_FLOW_AC_SH 15
+#define FWCMD_H2C_FW_FLOW_CTRL_HYBRID_FLOW_AC_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_NORMAL_FLOW_FORWARD_EN BIT(17)
+#define FWCMD_H2C_FW_FLOW_CTRL_LATENCY_FLOW_FORWARD_EN BIT(18)
+#define FWCMD_H2C_FW_FLOW_CTRL_THROUGHPUT_FLOW_FORWARD_EN BIT(19)
+#define FWCMD_H2C_FW_FLOW_CTRL_HYBRID_FLOW_FORWARD_EN BIT(20)
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U0_TO_U7_IS_MAIN_MACID_BM_SH 24
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U0_TO_U7_IS_MAIN_MACID_BM_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U0_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U0_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U0_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U0_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U0_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U0_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U0_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U0_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U0_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U0_MSK 0x3f
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U1_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U1_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U1_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U1_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U1_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U1_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U1_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U1_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U1_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U1_MSK 0x3f
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U2_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U2_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U2_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U2_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U2_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U2_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U2_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U2_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U2_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U2_MSK 0x3f
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U3_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U3_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U3_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U3_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U3_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U3_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U3_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U3_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U3_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U3_MSK 0x3f
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U4_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U4_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U4_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U4_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U4_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U4_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U4_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U4_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U4_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U4_MSK 0x3f
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U5_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U5_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U5_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U5_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U5_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U5_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U5_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U5_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U5_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U5_MSK 0x3f
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U6_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U6_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U6_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U6_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U6_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U6_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U6_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U6_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U6_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U6_MSK 0x3f
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U7_SH 0
+#define FWCMD_H2C_FW_FLOW_CTRL_MACID_U7_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U7_SH 8
+#define FWCMD_H2C_FW_FLOW_CTRL_FLOW_TYPE_U7_MSK 0x3
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U7_SH 10
+#define FWCMD_H2C_FW_FLOW_CTRL_MAX_TX_INTERVAL_U7_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U7_SH 18
+#define FWCMD_H2C_FW_FLOW_CTRL_MIN_TX_INTERVAL_U7_MSK 0xff
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U7_SH 26
+#define FWCMD_H2C_FW_FLOW_CTRL_TARGET_THROUGHPUT_U7_MSK 0x3f
+#define FWCMD_H2C_TXRPT_FORWARD_EN BIT(0)
+#define FWCMD_H2C_MRU_GRP_UPD_UL_DL BIT(0)
+#define FWCMD_H2C_MRU_GRP_UPD_GRP_VALID BIT(1)
+#define FWCMD_H2C_MRU_GRP_UPD_IS_HWGRP BIT(2)
+#define FWCMD_H2C_MRU_GRP_UPD_MRU BIT(3)
+#define FWCMD_H2C_MRU_GRP_UPD_BAND_SH 4
+#define FWCMD_H2C_MRU_GRP_UPD_BAND_MSK 0x3
+#define FWCMD_H2C_MRU_GRP_UPD_STA_CNT_SH 8
+#define FWCMD_H2C_MRU_GRP_UPD_STA_CNT_MSK 0xff
+#define FWCMD_H2C_MRU_GRP_UPD_GPR_ID_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_GPR_ID_MSK 0xff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U0_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U0_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U1_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U1_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U2_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U2_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U3_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U3_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U4_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U4_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U5_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U5_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U6_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U6_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U7_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U7_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U8_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U8_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U9_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U9_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U10_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U10_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U11_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U11_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U12_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U12_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U13_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U13_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U14_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U14_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U15_SH 16
+#define FWCMD_H2C_MRU_GRP_UPD_MACID_U15_MSK 0xffff
+#define FWCMD_H2C_MRU_GRP_UPD_U0_BSR_SEL_BE BIT(0)
+#define FWCMD_H2C_MRU_GRP_UPD_U1_BSR_SEL_BE BIT(1)
+#define FWCMD_H2C_MRU_GRP_UPD_U2_BSR_SEL_BE BIT(2)
+#define FWCMD_H2C_MRU_GRP_UPD_U3_BSR_SEL_BE BIT(3)
+#define FWCMD_H2C_MRU_GRP_UPD_U4_BSR_SEL_BE BIT(4)
+#define FWCMD_H2C_MRU_GRP_UPD_U5_BSR_SEL_BE BIT(5)
+#define FWCMD_H2C_MRU_GRP_UPD_U6_BSR_SEL_BE BIT(6)
+#define FWCMD_H2C_MRU_GRP_UPD_U7_BSR_SEL_BE BIT(7)
+#define FWCMD_H2C_MRU_GRP_UPD_U8_BSR_SEL_BE BIT(8)
+#define FWCMD_H2C_MRU_GRP_UPD_U9_BSR_SEL_BE BIT(9)
+#define FWCMD_H2C_MRU_GRP_UPD_U10_BSR_SEL_BE BIT(10)
+#define FWCMD_H2C_MRU_GRP_UPD_U11_BSR_SEL_BE BIT(11)
+#define FWCMD_H2C_MRU_GRP_UPD_U12_BSR_SEL_BE BIT(12)
+#define FWCMD_H2C_MRU_GRP_UPD_U13_BSR_SEL_BE BIT(13)
+#define FWCMD_H2C_MRU_GRP_UPD_U14_BSR_SEL_BE BIT(14)
+#define FWCMD_H2C_MRU_GRP_UPD_U15_BSR_SEL_BE BIT(15)
+#define FWCMD_H2C_MRU_GRP_UPD_U0_BSR_SEL_BK BIT(16)
+#define FWCMD_H2C_MRU_GRP_UPD_U1_BSR_SEL_BK BIT(17)
+#define FWCMD_H2C_MRU_GRP_UPD_U2_BSR_SEL_BK BIT(18)
+#define FWCMD_H2C_MRU_GRP_UPD_U3_BSR_SEL_BK BIT(19)
+#define FWCMD_H2C_MRU_GRP_UPD_U4_BSR_SEL_BK BIT(20)
+#define FWCMD_H2C_MRU_GRP_UPD_U5_BSR_SEL_BK BIT(21)
+#define FWCMD_H2C_MRU_GRP_UPD_U6_BSR_SEL_BK BIT(22)
+#define FWCMD_H2C_MRU_GRP_UPD_U7_BSR_SEL_BK BIT(23)
+#define FWCMD_H2C_MRU_GRP_UPD_U8_BSR_SEL_BK BIT(24)
+#define FWCMD_H2C_MRU_GRP_UPD_U9_BSR_SEL_BK BIT(25)
+#define FWCMD_H2C_MRU_GRP_UPD_U10_BSR_SEL_BK BIT(26)
+#define FWCMD_H2C_MRU_GRP_UPD_U11_BSR_SEL_BK BIT(27)
+#define FWCMD_H2C_MRU_GRP_UPD_U12_BSR_SEL_BK BIT(28)
+#define FWCMD_H2C_MRU_GRP_UPD_U13_BSR_SEL_BK BIT(29)
+#define FWCMD_H2C_MRU_GRP_UPD_U14_BSR_SEL_BK BIT(30)
+#define FWCMD_H2C_MRU_GRP_UPD_U15_BSR_SEL_BK BIT(31)
+#define FWCMD_H2C_MRU_GRP_UPD_U0_BSR_SEL_VI BIT(0)
+#define FWCMD_H2C_MRU_GRP_UPD_U1_BSR_SEL_VI BIT(1)
+#define FWCMD_H2C_MRU_GRP_UPD_U2_BSR_SEL_VI BIT(2)
+#define FWCMD_H2C_MRU_GRP_UPD_U3_BSR_SEL_VI BIT(3)
+#define FWCMD_H2C_MRU_GRP_UPD_U4_BSR_SEL_VI BIT(4)
+#define FWCMD_H2C_MRU_GRP_UPD_U5_BSR_SEL_VI BIT(5)
+#define FWCMD_H2C_MRU_GRP_UPD_U6_BSR_SEL_VI BIT(6)
+#define FWCMD_H2C_MRU_GRP_UPD_U7_BSR_SEL_VI BIT(7)
+#define FWCMD_H2C_MRU_GRP_UPD_U8_BSR_SEL_VI BIT(8)
+#define FWCMD_H2C_MRU_GRP_UPD_U9_BSR_SEL_VI BIT(9)
+#define FWCMD_H2C_MRU_GRP_UPD_U10_BSR_SEL_VI BIT(10)
+#define FWCMD_H2C_MRU_GRP_UPD_U11_BSR_SEL_VI BIT(11)
+#define FWCMD_H2C_MRU_GRP_UPD_U12_BSR_SEL_VI BIT(12)
+#define FWCMD_H2C_MRU_GRP_UPD_U13_BSR_SEL_VI BIT(13)
+#define FWCMD_H2C_MRU_GRP_UPD_U14_BSR_SEL_VI BIT(14)
+#define FWCMD_H2C_MRU_GRP_UPD_U15_BSR_SEL_VI BIT(15)
+#define FWCMD_H2C_MRU_GRP_UPD_U0_BSR_SEL_VO BIT(16)
+#define FWCMD_H2C_MRU_GRP_UPD_U1_BSR_SEL_VO BIT(17)
+#define FWCMD_H2C_MRU_GRP_UPD_U2_BSR_SEL_VO BIT(18)
+#define FWCMD_H2C_MRU_GRP_UPD_U3_BSR_SEL_VO BIT(19)
+#define FWCMD_H2C_MRU_GRP_UPD_U4_BSR_SEL_VO BIT(20)
+#define FWCMD_H2C_MRU_GRP_UPD_U5_BSR_SEL_VO BIT(21)
+#define FWCMD_H2C_MRU_GRP_UPD_U6_BSR_SEL_VO BIT(22)
+#define FWCMD_H2C_MRU_GRP_UPD_U7_BSR_SEL_VO BIT(23)
+#define FWCMD_H2C_MRU_GRP_UPD_U8_BSR_SEL_VO BIT(24)
+#define FWCMD_H2C_MRU_GRP_UPD_U9_BSR_SEL_VO BIT(25)
+#define FWCMD_H2C_MRU_GRP_UPD_U10_BSR_SEL_VO BIT(26)
+#define FWCMD_H2C_MRU_GRP_UPD_U11_BSR_SEL_VO BIT(27)
+#define FWCMD_H2C_MRU_GRP_UPD_U12_BSR_SEL_VO BIT(28)
+#define FWCMD_H2C_MRU_GRP_UPD_U13_BSR_SEL_VO BIT(29)
+#define FWCMD_H2C_MRU_GRP_UPD_U14_BSR_SEL_VO BIT(30)
+#define FWCMD_H2C_MRU_GRP_UPD_U15_BSR_SEL_VO BIT(31)
+#define FWCMD_H2C_MRU_GRP_UPD_TRIGGER_LEN_SH 0
+#define FWCMD_H2C_MRU_GRP_UPD_TRIGGER_LEN_MSK 0xff
+#define FWCMD_H2C_MRU_GRP_UPD_TRIGGER_UL_THRESHOLD_SH 8
+#define FWCMD_H2C_MRU_GRP_UPD_TRIGGER_UL_THRESHOLD_MSK 0xff
+#define FWCMD_H2C_DUMP_EFUSE_DUMP_SIZE_SH 0
+#define FWCMD_H2C_DUMP_EFUSE_DUMP_SIZE_MSK 0x7ff
+#define FWCMD_H2C_DUMP_EFUSE_IS_HIDDEN BIT(11)
+#define FWCMD_H2C_DUMP_EFUSE_IS_DAV BIT(12)
+#define FWCMD_H2C_PACKET_OFLD_PKT_IDX_SH 0
+#define FWCMD_H2C_PACKET_OFLD_PKT_IDX_MSK 0xff
+#define FWCMD_H2C_PACKET_OFLD_PKT_OP_SH 8
+#define FWCMD_H2C_PACKET_OFLD_PKT_OP_MSK 0x7
+#define FWCMD_H2C_PACKET_OFLD_PKT_LENGTH_SH 16
+#define FWCMD_H2C_PACKET_OFLD_PKT_LENGTH_MSK 0xffff
+#define FWCMD_H2C_PACKET_OFLD_PKT_START_SH 0
+#define FWCMD_H2C_PACKET_OFLD_PKT_START_MSK 0xffffffff
+#define FWCMD_H2C_READ_OFLD_VALUE_LEN_SH 0
+#define FWCMD_H2C_READ_OFLD_VALUE_LEN_MSK 0x7ff
+#define FWCMD_H2C_READ_OFLD_LS BIT(15)
+#define FWCMD_H2C_READ_OFLD_OFLD_ID_SH 16
+#define FWCMD_H2C_READ_OFLD_OFLD_ID_MSK 0xff
+#define FWCMD_H2C_READ_OFLD_ENTRY_NUM_SH 24
+#define FWCMD_H2C_READ_OFLD_ENTRY_NUM_MSK 0xff
+#define FWCMD_H2C_READ_OFLD_OFFSET_SH 0
+#define FWCMD_H2C_READ_OFLD_OFFSET_MSK 0xffff
+#define FWCMD_H2C_WRITE_OFLD_VALUE_LEN_SH 0
+#define FWCMD_H2C_WRITE_OFLD_VALUE_LEN_MSK 0x7ff
+#define FWCMD_H2C_WRITE_OFLD_POLLING BIT(13)
+#define FWCMD_H2C_WRITE_OFLD_MASK_EN BIT(14)
+#define FWCMD_H2C_WRITE_OFLD_LS BIT(15)
+#define FWCMD_H2C_WRITE_OFLD_OFLD_ID_SH 16
+#define FWCMD_H2C_WRITE_OFLD_OFLD_ID_MSK 0xff
+#define FWCMD_H2C_WRITE_OFLD_ENTRY_NUM_SH 24
+#define FWCMD_H2C_WRITE_OFLD_ENTRY_NUM_MSK 0xff
+#define FWCMD_H2C_WRITE_OFLD_OFFSET_SH 0
+#define FWCMD_H2C_WRITE_OFLD_OFFSET_MSK 0xffff
+#define FWCMD_H2C_WRITE_OFLD_START_SH 0
+#define FWCMD_H2C_WRITE_OFLD_START_MSK 0xffffffff
+#define FWCMD_H2C_CONF_OFLD_PATTERN_COUNT_SH 0
+#define FWCMD_H2C_CONF_OFLD_PATTERN_COUNT_MSK 0xffff
+#define FWCMD_H2C_CONF_OFLD_START_SH 0
+#define FWCMD_H2C_CONF_OFLD_START_MSK 0xffffffff
+#define FWCMD_H2C_TRX_INIT_TRX_MODE_SH 0
+#define FWCMD_H2C_TRX_INIT_TRX_MODE_MSK 0xff
+#define FWCMD_H2C_TRX_INIT_QTA_MODE_SH 8
+#define FWCMD_H2C_TRX_INIT_QTA_MODE_MSK 0x1ff
+#define FWCMD_H2C_INTF_INIT_TXBD_TRUNC_MODE BIT(0)
+#define FWCMD_H2C_INTF_INIT_RXBD_TRUNC_MODE BIT(1)
+#define FWCMD_H2C_INTF_INIT_RXBD_MODE BIT(2)
+#define FWCMD_H2C_INTF_INIT_TAG_MODE BIT(3)
+#define FWCMD_H2C_INTF_INIT_TX_BURST_MODE_SH 4
+#define FWCMD_H2C_INTF_INIT_TX_BURST_MODE_MSK 0xf
+#define FWCMD_H2C_INTF_INIT_RX_BURST_MODE_SH 8
+#define FWCMD_H2C_INTF_INIT_RX_BURST_MODE_MSK 0xf
+#define FWCMD_H2C_INTF_INIT_WD_DMA_IDLE_INTVL_SH 12
+#define FWCMD_H2C_INTF_INIT_WD_DMA_IDLE_INTVL_MSK 0xf
+#define FWCMD_H2C_INTF_INIT_WD_DMA_ACT_INTVL_SH 16
+#define FWCMD_H2C_INTF_INIT_WD_DMA_ACT_INTVL_MSK 0xf
+#define FWCMD_H2C_INTF_INIT_SKIP_ALL_SH 24
+#define FWCMD_H2C_INTF_INIT_SKIP_ALL_MSK 0xff
+#define FWCMD_H2C_INTF_INIT_RX_SEP_APPEND_LEN_SH 0
+#define FWCMD_H2C_INTF_INIT_RX_SEP_APPEND_LEN_MSK 0xffff
+#define FWCMD_H2C_INTF_INIT_TXBD_BUF_SH 16
+#define FWCMD_H2C_INTF_INIT_TXBD_BUF_MSK 0xff
+#define FWCMD_H2C_INTF_INIT_RXBD_BUF_SH 24
+#define FWCMD_H2C_INTF_INIT_RXBD_BUF_MSK 0xff
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_1_SH 0
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_1_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_2_SH 0
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_2_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_3_SH 0
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_3_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_4_SH 0
+#define FWCMD_H2C_MACID_PAUSE_PAUSE_GRP_4_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_1_SH 0
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_1_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_2_SH 0
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_2_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_3_SH 0
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_3_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_4_SH 0
+#define FWCMD_H2C_MACID_PAUSE_MASK_GRP_4_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_TYPE_SH 0
+#define FWCMD_H2C_RX_FWD_TYPE_MSK 0xff
+#define FWCMD_H2C_RX_FWD_FRAME_SH 8
+#define FWCMD_H2C_RX_FWD_FRAME_MSK 0xff
+#define FWCMD_H2C_RX_FWD_FWD_TG_SH 16
+#define FWCMD_H2C_RX_FWD_FWD_TG_MSK 0xff
+#define FWCMD_H2C_RX_FWD_AF_UD_INDEX_SH 0
+#define FWCMD_H2C_RX_FWD_AF_UD_INDEX_MSK 0xff
+#define FWCMD_H2C_RX_FWD_AF_UD_FWD_TG_SH 8
+#define FWCMD_H2C_RX_FWD_AF_UD_FWD_TG_MSK 0xff
+#define FWCMD_H2C_RX_FWD_AF_UD_CATEGORY_SH 16
+#define FWCMD_H2C_RX_FWD_AF_UD_CATEGORY_MSK 0xff
+#define FWCMD_H2C_RX_FWD_AF_UD_ACTION_FIELD_SH 24
+#define FWCMD_H2C_RX_FWD_AF_UD_ACTION_FIELD_MSK 0xff
+#define FWCMD_H2C_RX_FWD_PM_CAM_VALID BIT(0)
+#define FWCMD_H2C_RX_FWD_PM_CAM_TYPE_SH 1
+#define FWCMD_H2C_RX_FWD_PM_CAM_TYPE_MSK 0x3
+#define FWCMD_H2C_RX_FWD_PM_CAM_SUBTYPE_SH 3
+#define FWCMD_H2C_RX_FWD_PM_CAM_SUBTYPE_MSK 0xf
+#define FWCMD_H2C_RX_FWD_PM_CAM_SKIP_MAC_IV_HDR BIT(7)
+#define FWCMD_H2C_RX_FWD_PM_CAM_TARGET_IND_SH 8
+#define FWCMD_H2C_RX_FWD_PM_CAM_TARGET_IND_MSK 0x7
+#define FWCMD_H2C_RX_FWD_PM_CAM_INDEX_SH 12
+#define FWCMD_H2C_RX_FWD_PM_CAM_INDEX_MSK 0xf
+#define FWCMD_H2C_RX_FWD_PM_CAM_CRC16_SH 16
+#define FWCMD_H2C_RX_FWD_PM_CAM_CRC16_MSK 0xffff
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK0_SH 0
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK0_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK1_SH 0
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK1_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK2_SH 0
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK2_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK3_SH 0
+#define FWCMD_H2C_RX_FWD_PM_CAM_PLD_MASK3_MSK 0xffffffff
+#define FWCMD_H2C_EN_MAC_HDR_CONV_ENABLE BIT(0)
+#define FWCMD_H2C_SET_HWSEQ_REG_REG_IDX_SH 0
+#define FWCMD_H2C_SET_HWSEQ_REG_REG_IDX_MSK 0x3
+#define FWCMD_H2C_SET_HWSEQ_REG_SEQ_VAL_SH 2
+#define FWCMD_H2C_SET_HWSEQ_REG_SEQ_VAL_MSK 0xfff
+#define FWCMD_H2C_HWAMSDU_REG_ENABLE BIT(0)
+#define FWCMD_H2C_HWAMSDU_REG_MAX_AMSDU_SUBFRAME_NUMBER_SH 1
+#define FWCMD_H2C_HWAMSDU_REG_MAX_AMSDU_SUBFRAME_NUMBER_MSK 0x3
+#define FWCMD_H2C_HWAMSDU_REG_ENABLE_SINGLE_AMSDU BIT(3)
+#define FWCMD_H2C_HWAMSDU_REG_ENABLE_LAST_AMSDU_PADDING BIT(4)
+#define FWCMD_H2C_AMSDU_CUT_REG_ENABLE BIT(0)
+#define FWCMD_H2C_AMSDU_CUT_REG_LOW_THRESHOLD_SH 1
+#define FWCMD_H2C_AMSDU_CUT_REG_LOW_THRESHOLD_MSK 0x1ff
+#define FWCMD_H2C_AMSDU_CUT_REG_HIGH_THRESHOLD_SH 10
+#define FWCMD_H2C_AMSDU_CUT_REG_HIGH_THRESHOLD_MSK 0xffff
+#define FWCMD_H2C_AMSDU_CUT_REG_EXTRA_SHIFT_SH 26
+#define FWCMD_H2C_AMSDU_CUT_REG_EXTRA_SHIFT_MSK 0x3
+#define FWCMD_H2C_TCPIP_CHKSUM_OFFLOAD_REG_ENABLE_TX_TCPIP_CHECKSUM_OFFLOAD BIT(0)
+#define FWCMD_H2C_TCPIP_CHKSUM_OFFLOAD_REG_ENABLE_RX_TCPIP_CHECKSUM_OFFLOAD BIT(1)
+#define FWCMD_H2C_USR_EDCA_PARAM_SEL_SH 0
+#define FWCMD_H2C_USR_EDCA_PARAM_SEL_MSK 0x3
+#define FWCMD_H2C_USR_EDCA_ENABLE BIT(2)
+#define FWCMD_H2C_USR_EDCA_BAND BIT(3)
+#define FWCMD_H2C_USR_EDCA_WMM BIT(4)
+#define FWCMD_H2C_USR_EDCA_AC_SH 5
+#define FWCMD_H2C_USR_EDCA_AC_MSK 0x3
+#define FWCMD_H2C_USR_EDCA_AGGRESSIVE_SH 0
+#define FWCMD_H2C_USR_EDCA_AGGRESSIVE_MSK 0xffffffff
+#define FWCMD_H2C_USR_EDCA_MODERATE_SH 0
+#define FWCMD_H2C_USR_EDCA_MODERATE_MSK 0xffffffff
+#define FWCMD_H2C_TSF32_TOGL_BAND BIT(0)
+#define FWCMD_H2C_TSF32_TOGL_EN BIT(1)
+#define FWCMD_H2C_TSF32_TOGL_PORT_SH 2
+#define FWCMD_H2C_TSF32_TOGL_PORT_MSK 0x7
+#define FWCMD_H2C_TSF32_TOGL_EARLY_SH 16
+#define FWCMD_H2C_TSF32_TOGL_EARLY_MSK 0xffff
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D0_SH 0
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D0_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D1_SH 0
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D1_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D2_SH 0
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D2_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D3_SH 0
+#define FWCMD_H2C_CMD_OFLD_REG_CMD_OFLD_D3_MSK 0xffffffff
+#define FWCMD_H2C_USR_TX_RPT_MODE_SH 0
+#define FWCMD_H2C_USR_TX_RPT_MODE_MSK 0x7
+#define FWCMD_H2C_USR_TX_RPT_RTP_START BIT(3)
+#define FWCMD_H2C_USR_TX_RPT_MACID_SH 0
+#define FWCMD_H2C_USR_TX_RPT_MACID_MSK 0xff
+#define FWCMD_H2C_USR_TX_RPT_AC_SH 8
+#define FWCMD_H2C_USR_TX_RPT_AC_MSK 0x3
+#define FWCMD_H2C_USR_TX_RPT_BAND BIT(10)
+#define FWCMD_H2C_USR_TX_RPT_PORT_SH 11
+#define FWCMD_H2C_USR_TX_RPT_PORT_MSK 0x7
+#define FWCMD_H2C_USR_TX_RPT_MACID_EXT_SH 24
+#define FWCMD_H2C_USR_TX_RPT_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_USR_TX_RPT_RPT_PERIOD_SH 0
+#define FWCMD_H2C_USR_TX_RPT_RPT_PERIOD_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D0_SH 0
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D0_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D1_SH 0
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D1_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D2_SH 0
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D2_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D3_SH 0
+#define FWCMD_H2C_CMD_OFLD_PKT_CMD_OFLD_D3_MSK 0xffffffff
+#define FWCMD_H2C_OFLD_CFG_MODE_SH 0
+#define FWCMD_H2C_OFLD_CFG_MODE_MSK 0x7
+#define FWCMD_H2C_OFLD_CFG_USR_TXOP_BE BIT(3)
+#define FWCMD_H2C_OFLD_CFG_USR_TXOP_BE_VAL_SH 0
+#define FWCMD_H2C_OFLD_CFG_USR_TXOP_BE_VAL_MSK 0xffff
+#define FWCMD_H2C_H2C_AGG_PLD_SH 0
+#define FWCMD_H2C_H2C_AGG_PLD_MSK 0xffffffff
+#define FWCMD_H2C_ADD_SCANOFLD_CH_NUM_OF_CH_SH 0
+#define FWCMD_H2C_ADD_SCANOFLD_CH_NUM_OF_CH_MSK 0xff
+#define FWCMD_H2C_ADD_SCANOFLD_CH_SIZE_OF_CHINFO_SH 8
+#define FWCMD_H2C_ADD_SCANOFLD_CH_SIZE_OF_CHINFO_MSK 0xff
+#define FWCMD_H2C_ADD_SCANOFLD_CH_BAND BIT(16)
+#define FWCMD_H2C_ADD_SCANOFLD_CH_APPEND BIT(17)
+#define FWCMD_H2C_SCANOFLD_MACID_SH 0
+#define FWCMD_H2C_SCANOFLD_MACID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_NORM_CY_SH 8
+#define FWCMD_H2C_SCANOFLD_NORM_CY_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_PORT_ID_SH 16
+#define FWCMD_H2C_SCANOFLD_PORT_ID_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_BAND BIT(19)
+#define FWCMD_H2C_SCANOFLD_OPERATION_SH 20
+#define FWCMD_H2C_SCANOFLD_OPERATION_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_CH_BAND_SH 22
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_CH_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_C2H_NOTIFY_END BIT(0)
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_MODE BIT(1)
+#define FWCMD_H2C_SCANOFLD_START_MODE BIT(2)
+#define FWCMD_H2C_SCANOFLD_SCAN_TYPE_SH 3
+#define FWCMD_H2C_SCANOFLD_SCAN_TYPE_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_BW_SH 5
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_BW_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_TARGET_PRI_CH_SH 8
+#define FWCMD_H2C_SCANOFLD_TARGET_PRI_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_TARGET_CENTRAL_CH_SH 16
+#define FWCMD_H2C_SCANOFLD_TARGET_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_TARGET_NUM_PKTS_SH 24
+#define FWCMD_H2C_SCANOFLD_TARGET_NUM_PKTS_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_NORM_PD_SH 0
+#define FWCMD_H2C_SCANOFLD_NORM_PD_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_SLOW_PD_SH 16
+#define FWCMD_H2C_SCANOFLD_SLOW_PD_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_TSF_HIGH_SH 0
+#define FWCMD_H2C_SCANOFLD_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_H2C_SCANOFLD_TSF_LOW_SH 0
+#define FWCMD_H2C_SCANOFLD_TSF_LOW_MSK 0xffffffff
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT0_SH 0
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT0_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT1_SH 8
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT1_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT2_SH 16
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT2_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT3_SH 24
+#define FWCMD_H2C_SCANOFLD_TARGET_CH_PKT3_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_NUMSSID_SH 0
+#define FWCMD_H2C_SCANOFLD_NUMSSID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_NUMSHORTSSID_SH 8
+#define FWCMD_H2C_SCANOFLD_NUMSHORTSSID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_NUMBSSID_SH 16
+#define FWCMD_H2C_SCANOFLD_NUMBSSID_MSK 0xff
+#define FWCMD_H2C_TX_DUTY_PAUSE_INTVL_SH 0
+#define FWCMD_H2C_TX_DUTY_PAUSE_INTVL_MSK 0xffff
+#define FWCMD_H2C_TX_DUTY_TX_INTVL_SH 16
+#define FWCMD_H2C_TX_DUTY_TX_INTVL_MSK 0xffff
+#define FWCMD_H2C_TX_DUTY_STOP BIT(0)
+#define FWCMD_H2C_DISABLE_RF_FUNC_SH 0
+#define FWCMD_H2C_DISABLE_RF_FUNC_MSK 0xf
+#define FWCMD_H2C_DISABLE_RF_NET_TYPE_SH 4
+#define FWCMD_H2C_DISABLE_RF_NET_TYPE_MSK 0x3
+#define FWCMD_H2C_TX_ADDBA_PKT_PKT_ID_SH 0
+#define FWCMD_H2C_TX_ADDBA_PKT_PKT_ID_MSK 0xff
+#define FWCMD_H2C_TX_ADDBA_PKT_PORT_SH 8
+#define FWCMD_H2C_TX_ADDBA_PKT_PORT_MSK 0x7
+#define FWCMD_H2C_TX_ADDBA_PKT_BAND BIT(11)
+#define FWCMD_H2C_PKT_DROP_DROP_SEL_SH 0
+#define FWCMD_H2C_PKT_DROP_DROP_SEL_MSK 0xff
+#define FWCMD_H2C_PKT_DROP_MACID_SH 8
+#define FWCMD_H2C_PKT_DROP_MACID_MSK 0xff
+#define FWCMD_H2C_PKT_DROP_BAND_SH 16
+#define FWCMD_H2C_PKT_DROP_BAND_MSK 0xff
+#define FWCMD_H2C_PKT_DROP_PORT_SH 24
+#define FWCMD_H2C_PKT_DROP_PORT_MSK 0xff
+#define FWCMD_H2C_PKT_DROP_MBSSID_SH 0
+#define FWCMD_H2C_PKT_DROP_MBSSID_MSK 0xff
+#define FWCMD_H2C_PKT_DROP_ROLE_A_INFO_TF_TRS_SH 8
+#define FWCMD_H2C_PKT_DROP_ROLE_A_INFO_TF_TRS_MSK 0xff
+#define FWCMD_H2C_PKT_DROP_MACID_EXT_SH 16
+#define FWCMD_H2C_PKT_DROP_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_0_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_0_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_1_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_1_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_2_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_2_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_3_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_3_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_4_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_4_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_5_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_5_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_6_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_6_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_7_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_7_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_8_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_8_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_9_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_9_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_10_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_10_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_11_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_11_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_12_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_12_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_13_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_13_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_14_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_14_MSK 0xffffffff
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_15_SH 0
+#define FWCMD_H2C_PKT_DROP_MACID_BAND_SEL_15_MSK 0xffffffff
+#define FWCMD_H2C_CH_SWITCH_PRI_CH_SH 0
+#define FWCMD_H2C_CH_SWITCH_PRI_CH_MSK 0xff
+#define FWCMD_H2C_CH_SWITCH_CENTRAL_CH_SH 8
+#define FWCMD_H2C_CH_SWITCH_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_CH_SWITCH_BW_SH 16
+#define FWCMD_H2C_CH_SWITCH_BW_MSK 0x7
+#define FWCMD_H2C_CH_SWITCH_CH_BAND_SH 19
+#define FWCMD_H2C_CH_SWITCH_CH_BAND_MSK 0x3
+#define FWCMD_H2C_CH_SWITCH_BAND BIT(21)
+#define FWCMD_H2C_CH_SWITCH_RELOAD_RF BIT(22)
+#define FWCMD_H2C_BCNERLYRPT_PORT_SH 0
+#define FWCMD_H2C_BCNERLYRPT_PORT_MSK 0x7
+#define FWCMD_H2C_BCNERLYRPT_BAND BIT(3)
+#define FWCMD_H2C_BCNERLYRPT_CONTROL BIT(4)
+#define FWCMD_H2C_BCNERLYRPT_RPT_INTVL_SH 5
+#define FWCMD_H2C_BCNERLYRPT_RPT_INTVL_MSK 0xff
+#define FWCMD_H2C_CFG_BCNFLTR_MON_RSSI BIT(0)
+#define FWCMD_H2C_CFG_BCNFLTR_MON_BCN BIT(1)
+#define FWCMD_H2C_CFG_BCNFLTR_MON_TP BIT(2)
+#define FWCMD_H2C_CFG_BCNFLTR_TP_THLD_SH 3
+#define FWCMD_H2C_CFG_BCNFLTR_TP_THLD_MSK 0x3
+#define FWCMD_H2C_CFG_BCNFLTR_BCN_LOSS_CNT_SH 8
+#define FWCMD_H2C_CFG_BCNFLTR_BCN_LOSS_CNT_MSK 0xf
+#define FWCMD_H2C_CFG_BCNFLTR_RSSI_HYS_SH 12
+#define FWCMD_H2C_CFG_BCNFLTR_RSSI_HYS_MSK 0xf
+#define FWCMD_H2C_CFG_BCNFLTR_RSSI_THLD_SH 16
+#define FWCMD_H2C_CFG_BCNFLTR_RSSI_THLD_MSK 0xff
+#define FWCMD_H2C_CFG_BCNFLTR_MACID_SH 24
+#define FWCMD_H2C_CFG_BCNFLTR_MACID_MSK 0xff
+#define FWCMD_H2C_CFG_BCNFLTR_MACID_EXT_SH 0
+#define FWCMD_H2C_CFG_BCNFLTR_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_OFLD_RSSI_MACID_SH 0
+#define FWCMD_H2C_OFLD_RSSI_MACID_MSK 0xff
+#define FWCMD_H2C_OFLD_RSSI_NUM_RSSI_SH 8
+#define FWCMD_H2C_OFLD_RSSI_NUM_RSSI_MSK 0xff
+#define FWCMD_H2C_OFLD_RSSI_MACID_EXT_SH 16
+#define FWCMD_H2C_OFLD_RSSI_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_OFLD_TP_MACID_SH 0
+#define FWCMD_H2C_OFLD_TP_MACID_MSK 0xff
+#define FWCMD_H2C_OFLD_TP_TXTP_SH 8
+#define FWCMD_H2C_OFLD_TP_TXTP_MSK 0x3ff
+#define FWCMD_H2C_OFLD_TP_RXTP_SH 18
+#define FWCMD_H2C_OFLD_TP_RXTP_MSK 0x3ff
+#define FWCMD_H2C_OFLD_TP_MACID_EXT_SH 0
+#define FWCMD_H2C_OFLD_TP_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_HOST_EFUSE_REC_HOST_ID_SH 0
+#define FWCMD_H2C_HOST_EFUSE_REC_HOST_ID_MSK 0xffffffff
+#define FWCMD_H2C_HOST_EFUSE_REC_EFUSE_VAL_SH 0
+#define FWCMD_H2C_HOST_EFUSE_REC_EFUSE_VAL_MSK 0xffffffff
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_PRI_CH_SH 0
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_PRI_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_CH_BAND_SH 8
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_CH_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_CMD_SH 10
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_CMD_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_BAND BIT(13)
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_NUM_TBL_SH 16
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_NUM_TBL_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_SIZEOF_TBL_SH 24
+#define FWCMD_H2C_SCANOFLD_DRV_CTRL_SIZEOF_TBL_MSK 0xff
+#define FWCMD_H2C_WIFI_SENSING_CSI_MACID_SH 0
+#define FWCMD_H2C_WIFI_SENSING_CSI_MACID_MSK 0xff
+#define FWCMD_H2C_WIFI_SENSING_CSI_EN BIT(8)
+#define FWCMD_H2C_WIFI_SENSING_CSI_PERIOD_SH 16
+#define FWCMD_H2C_WIFI_SENSING_CSI_PERIOD_MSK 0xffff
+#define FWCMD_H2C_WIFI_SENSING_CSI_RETRY_CNT_SH 0
+#define FWCMD_H2C_WIFI_SENSING_CSI_RETRY_CNT_MSK 0xff
+#define FWCMD_H2C_WIFI_SENSING_CSI_RATE_SH 8
+#define FWCMD_H2C_WIFI_SENSING_CSI_RATE_MSK 0xffff
+#define FWCMD_H2C_WIFI_SENSING_CSI_PKT_NUM_SH 24
+#define FWCMD_H2C_WIFI_SENSING_CSI_PKT_NUM_MSK 0xff
+#define FWCMD_H2C_WIFI_SENSING_CSI_MACID_EXT_SH 0
+#define FWCMD_H2C_WIFI_SENSING_CSI_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D0_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D0_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D1_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D1_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D2_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D2_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D3_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_REG_CMD_OFLD_D3_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D0_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D0_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D1_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D1_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D2_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D2_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D3_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_PKT_CMD_OFLD_D3_MSK 0xffffffff
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_BE0 BIT(0)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_BK0 BIT(1)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_VI0 BIT(2)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_VO0 BIT(3)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_BE1 BIT(4)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_BK1 BIT(5)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_VI1 BIT(6)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_VO1 BIT(7)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_MG0 BIT(8)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_MG1 BIT(9)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_MG2 BIT(10)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_HI BIT(11)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_BCN BIT(12)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_UL BIT(13)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_TWT0 BIT(14)
+#define FWCMD_H2C_SCH_TX_EN_PKT_TX_EN_TWT1 BIT(15)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_BE0 BIT(0)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_BK0 BIT(1)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_VI0 BIT(2)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_VO0 BIT(3)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_BE1 BIT(4)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_BK1 BIT(5)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_VI1 BIT(6)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_VO1 BIT(7)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_MG0 BIT(8)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_MG1 BIT(9)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_MG2 BIT(10)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_HI BIT(11)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_BCN BIT(12)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_UL BIT(13)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_TWT0 BIT(14)
+#define FWCMD_H2C_SCH_TX_EN_PKT_MASK_TWT1 BIT(15)
+#define FWCMD_H2C_SCH_TX_EN_PKT_BAND BIT(16)
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_1_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_1_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_2_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_2_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_3_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_3_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_4_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_4_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_1_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_1_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_2_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_2_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_3_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_3_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_4_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_4_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_1_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_1_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_2_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_2_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_3_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_3_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_4_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_4_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_1_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_1_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_2_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_2_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_3_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_3_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_4_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_4_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_5_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_5_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_6_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_6_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_7_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_7_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_8_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_8_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_5_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_5_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_6_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_6_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_7_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_7_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_8_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_8_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_5_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_5_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_6_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_6_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_7_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_7_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_8_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_8_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_5_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_5_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_6_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_6_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_7_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_7_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_8_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_8_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_9_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_9_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_10_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_10_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_11_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_11_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_12_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_12_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_9_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_9_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_10_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_10_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_11_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_11_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_12_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_12_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_9_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_9_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_10_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_10_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_11_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_11_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_12_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_12_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_9_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_9_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_10_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_10_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_11_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_11_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_12_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_12_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_13_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_13_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_14_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_14_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_15_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_15_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_16_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_EN_16_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_13_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_13_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_14_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_14_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_15_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_15_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_16_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_PAUSE_GRP_MASK_16_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_13_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_13_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_14_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_14_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_15_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_15_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_16_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_EN_16_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_13_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_13_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_14_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_14_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_15_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_15_MSK 0xffffffff
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_16_SH 0
+#define FWCMD_H2C_MACID_PAUSE_SLEEP_SLEEP_GRP_MASK_16_MSK 0xffffffff
+#define FWCMD_H2C_BCN_OFLD_BAND BIT(0)
+#define FWCMD_H2C_BCN_OFLD_PORT_SH 1
+#define FWCMD_H2C_BCN_OFLD_PORT_MSK 0x7
+#define FWCMD_H2C_BCN_OFLD_TYPE_SH 4
+#define FWCMD_H2C_BCN_OFLD_TYPE_MSK 0x7
+#define FWCMD_H2C_BCN_OFLD_RST_CAM BIT(7)
+#define FWCMD_H2C_BCN_OFLD_HIT_FWD_EN BIT(8)
+#define FWCMD_H2C_BCN_OFLD_MISS_FWD_EN BIT(9)
+#define FWCMD_H2C_BCN_OFLD_ERR_FWD_EN BIT(10)
+#define FWCMD_H2C_BCN_OFLD_CAM_NUM_SH 11
+#define FWCMD_H2C_BCN_OFLD_CAM_NUM_MSK 0x1f
+#define FWCMD_H2C_BCN_OFLD_HIT_FWD_SH 16
+#define FWCMD_H2C_BCN_OFLD_HIT_FWD_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_MISS_FWD_SH 24
+#define FWCMD_H2C_BCN_OFLD_MISS_FWD_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_ERR_FWD_SH 0
+#define FWCMD_H2C_BCN_OFLD_ERR_FWD_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_RPT_PERIOD_SH 8
+#define FWCMD_H2C_BCN_OFLD_RPT_PERIOD_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_RSSI_PERIOD_SH 16
+#define FWCMD_H2C_BCN_OFLD_RSSI_PERIOD_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_RSSI_MAX_NUM_SH 24
+#define FWCMD_H2C_BCN_OFLD_RSSI_MAX_NUM_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_DETN_PERIOD_SH 0
+#define FWCMD_H2C_BCN_OFLD_DETN_PERIOD_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_DETN_MIN_BCN_CNT_SH 8
+#define FWCMD_H2C_BCN_OFLD_DETN_MIN_BCN_CNT_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_DETN_RSSI_LOW_BND_SH 16
+#define FWCMD_H2C_BCN_OFLD_DETN_RSSI_LOW_BND_MSK 0xff
+#define FWCMD_H2C_BCN_OFLD_DETN_RSSI_HIGH_BND_SH 24
+#define FWCMD_H2C_BCN_OFLD_DETN_RSSI_HIGH_BND_MSK 0xff
+#define FWCMD_H2C_TBTT_SH_STAT_OFLD_BAND_SH 0
+#define FWCMD_H2C_TBTT_SH_STAT_OFLD_BAND_MSK 0xff
+#define FWCMD_H2C_TBTT_SH_STAT_OFLD_PORT_SH 8
+#define FWCMD_H2C_TBTT_SH_STAT_OFLD_PORT_MSK 0xff
+#define FWCMD_H2C_TBTT_SH_STAT_OFLD_NUM_INTV_SH 16
+#define FWCMD_H2C_TBTT_SH_STAT_OFLD_NUM_INTV_MSK 0xff
+#define FWCMD_H2C_TBTT_SH_STAT_OFLD_ENABLE BIT(24)
+#define FWCMD_H2C_STA_CSA_STA_CSA_EN BIT(0)
+#define FWCMD_H2C_STA_CSA_SIZE_OF_CHINFO_SH 1
+#define FWCMD_H2C_STA_CSA_SIZE_OF_CHINFO_MSK 0x7f
+#define FWCMD_H2C_STA_CSA_NUM_OF_CHINFO_SH 8
+#define FWCMD_H2C_STA_CSA_NUM_OF_CHINFO_MSK 0xff
+#define FWCMD_H2C_STA_CSA_MACID_SH 16
+#define FWCMD_H2C_STA_CSA_MACID_MSK 0xff
+#define FWCMD_H2C_STA_CSA_MAX_AP_GONE_TIME_SH 24
+#define FWCMD_H2C_STA_CSA_MAX_AP_GONE_TIME_MSK 0xff
+#define FWCMD_H2C_STA_CSA_MACID_EXT_SH 0
+#define FWCMD_H2C_STA_CSA_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_OPERATION_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_OPERATION_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_V2_SCAN_MODE_SH 2
+#define FWCMD_H2C_SCANOFLD_V2_SCAN_MODE_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_V2_REPEAT_TYPE_SH 4
+#define FWCMD_H2C_SCANOFLD_V2_REPEAT_TYPE_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_V2_NOTIFY_END BIT(6)
+#define FWCMD_H2C_SCANOFLD_V2_GEN_LEARNED_CH BIT(7)
+#define FWCMD_H2C_SCANOFLD_V2_MACID_SH 8
+#define FWCMD_H2C_SCANOFLD_V2_MACID_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_V2_PORT_ID_SH 24
+#define FWCMD_H2C_SCANOFLD_V2_PORT_ID_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_V2_BAND_SH 27
+#define FWCMD_H2C_SCANOFLD_V2_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_WITH_RATE BIT(29)
+#define FWCMD_H2C_SCANOFLD_V2_NUMMACCROLE_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_NUMMACCROLE_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_NUMOP_SH 8
+#define FWCMD_H2C_SCANOFLD_V2_NUMOP_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_NORM_PD_SH 16
+#define FWCMD_H2C_SCANOFLD_V2_NORM_PD_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_V2_SLOW_PD_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_SLOW_PD_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_V2_NORM_CY_SH 16
+#define FWCMD_H2C_SCANOFLD_V2_NORM_CY_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_OPCH_END_SH 24
+#define FWCMD_H2C_SCANOFLD_V2_OPCH_END_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_NUMSSID_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_NUMSSID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_NUMSHORTSSID_SH 8
+#define FWCMD_H2C_SCANOFLD_V2_NUMSHORTSSID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_NUMBSSID_SH 16
+#define FWCMD_H2C_SCANOFLD_V2_NUMBSSID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_ID_24G_SH 24
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_ID_24G_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_ID_5G_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_ID_5G_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_ID_6G_SH 8
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_ID_6G_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_SEC_DELAY_START_SH 16
+#define FWCMD_H2C_SCANOFLD_V2_SEC_DELAY_START_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_V2_SCAN_MLO_MODE_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_SCAN_MLO_MODE_MSK 0xffffffff
+#define FWCMD_H2C_SCANOFLD_V2_PROHIB_6G_20M_0_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_PROHIB_6G_20M_0_MSK 0xffffffff
+#define FWCMD_H2C_SCANOFLD_V2_PROHIB_6G_20M_1_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_PROHIB_6G_20M_1_MSK 0xffffffff
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_RATE_24G_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_RATE_24G_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_RATE_5G_SH 8
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_RATE_5G_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_RATE_6G_SH 16
+#define FWCMD_H2C_SCANOFLD_V2_PROBE_RATE_6G_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_SIZEOF_CFG_SH 0
+#define FWCMD_H2C_SCANOFLD_V2_SIZEOF_CFG_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_SIZEOF_MACC_ROLE_SH 8
+#define FWCMD_H2C_SCANOFLD_V2_SIZEOF_MACC_ROLE_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_V2_SIZEOF_OP_SH 16
+#define FWCMD_H2C_SCANOFLD_V2_SIZEOF_OP_MSK 0xff
+#define FWCMD_H2C_FRAME_TO_ACT_MODE_SH 0
+#define FWCMD_H2C_FRAME_TO_ACT_MODE_MSK 0xf
+#define FWCMD_H2C_FRAME_TO_ACT_TRIGGER_CNT_SH 8
+#define FWCMD_H2C_FRAME_TO_ACT_TRIGGER_CNT_MSK 0xff
+#define FWCMD_H2C_FRAME_TO_ACT_SW_DEF_SH 16
+#define FWCMD_H2C_FRAME_TO_ACT_SW_DEF_MSK 0xffff
+#define FWCMD_H2C_FRAME_TO_ACT_TO_THR_SH 0
+#define FWCMD_H2C_FRAME_TO_ACT_TO_THR_MSK 0xffffffff
+#define FWCMD_H2C_DUMP_EFUSE_V2_DUMP_SIZE_SH 0
+#define FWCMD_H2C_DUMP_EFUSE_V2_DUMP_SIZE_MSK 0xffff
+#define FWCMD_H2C_DUMP_EFUSE_V2_IS_HIDDEN BIT(16)
+#define FWCMD_H2C_DUMP_EFUSE_V2_IS_DAV BIT(17)
+#define FWCMD_H2C_DUMP_EFUSE_V2_END_BREAK BIT(18)
+#define FWCMD_H2C_DUMP_EFUSE_V2_DAV_CHIP_ID BIT(19)
+#define FWCMD_H2C_DUMP_EFUSE_V2_START_OFFSET_SH 0
+#define FWCMD_H2C_DUMP_EFUSE_V2_START_OFFSET_MSK 0xffff
+#define FWCMD_H2C_TXRPT_STATS_EN_MACID_NUM_SH 0
+#define FWCMD_H2C_TXRPT_STATS_EN_MACID_NUM_MSK 0xff
+#define FWCMD_H2C_TXRPT_STATS_EN_STATS_EN BIT(8)
+#define FWCMD_H2C_TXRPT_STATS_EN_ONLY_MAIN_MACID BIT(9)
+#define FWCMD_H2C_TXRPT_STATS_EN_C2H_MODE BIT(10)
+#define FWCMD_H2C_TXRPT_STATS_EN_DURATION_SH 16
+#define FWCMD_H2C_TXRPT_STATS_EN_DURATION_MSK 0x3ff
+#define FWCMD_H2C_TXRPT_STATS_EN_MACID_X_SH 0
+#define FWCMD_H2C_TXRPT_STATS_EN_MACID_X_MSK 0xffff
+#define FWCMD_H2C_TXRPT_STATS_EN_MACID_Y_SH 16
+#define FWCMD_H2C_TXRPT_STATS_EN_MACID_Y_MSK 0xffff
+#define FWCMD_H2C_CPUMGPKT_TX_PKT_LEN_SH 0
+#define FWCMD_H2C_CPUMGPKT_TX_PKT_LEN_MSK 0xffff
+#define FWCMD_H2C_CPUMGPKT_TX_MACID_SH 16
+#define FWCMD_H2C_CPUMGPKT_TX_MACID_MSK 0xffff
+#define FWCMD_H2C_CPUMGPKT_TX_TX_NUM_SH 0
+#define FWCMD_H2C_CPUMGPKT_TX_TX_NUM_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_BCN_EN BIT(0)
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_OUTPUT_INTPUT BIT(1)
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_ACTIVE BIT(2)
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_TOGGLE_PULSE BIT(3)
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_TIME_UNIT BIT(4)
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_DEBUG BIT(5)
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_GPIO_NUM_SH 8
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_GPIO_NUM_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_MACID_SH 16
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_MACID_MSK 0xffff
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_PULSE_CNT_SH 0
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_PULSE_CNT_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_PULSE_DURATION_SH 8
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_PULSE_DURATION_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_PULSE_PERIOD_SH 16
+#define FWCMD_H2C_BCN_CNT_GPIO_RX_PULSE_PERIOD_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_BCN_EN BIT(0)
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_OUTPUT_INTPUT BIT(1)
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_ACTIVE BIT(2)
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_TOGGLE_PULSE BIT(3)
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_TIME_UNIT BIT(4)
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_DEBUG BIT(5)
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_GPIO_NUM_SH 8
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_GPIO_NUM_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_MACID_SH 16
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_MACID_MSK 0xffff
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_PULSE_CNT_SH 0
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_PULSE_CNT_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_PULSE_DURATION_SH 8
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_PULSE_DURATION_MSK 0xff
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_PULSE_PERIOD_SH 16
+#define FWCMD_H2C_BCN_CNT_GPIO_TX_PULSE_PERIOD_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_TYPE_SH 0
+#define FWCMD_H2C_RX_FWD_BE_TYPE_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_FRAME_SH 8
+#define FWCMD_H2C_RX_FWD_BE_FRAME_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_FWD_TG_SH 16
+#define FWCMD_H2C_RX_FWD_BE_FWD_TG_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_INDEX_SH 0
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_INDEX_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_FWD_TG_SH 8
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_FWD_TG_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_CATEGORY_SH 16
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_CATEGORY_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_ACTION_FIELD_SH 24
+#define FWCMD_H2C_RX_FWD_BE_AF_UD_ACTION_FIELD_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_VALID BIT(0)
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_TYPE_SH 1
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_TYPE_MSK 0x3
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_SUBTYPE_SH 3
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_SUBTYPE_MSK 0xf
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_SKIP_MAC_IV_HDR BIT(7)
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_TARGET_IND_V1_SH 8
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_TARGET_IND_V1_MSK 0xff
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_CRC16_SH 16
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_CRC16_MSK 0xffff
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK0_SH 0
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK0_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK1_SH 0
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK1_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK2_SH 0
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK2_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK3_SH 0
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_PLD_MASK3_MSK 0xffffffff
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_INDEX_V1_SH 0
+#define FWCMD_H2C_RX_FWD_BE_PM_CAM_INDEX_V1_MSK 0x1f
+#define FWCMD_H2C_RX_FWD_BE_IGNORE_SUBTYPE BIT(8)
+#define FWCMD_H2C_TRX_TIME_STATS_EN_BAND_BITMAP_SH 0
+#define FWCMD_H2C_TRX_TIME_STATS_EN_BAND_BITMAP_MSK 0xf
+#define FWCMD_H2C_TRX_TIME_STATS_EN_STATS_EN BIT(7)
+#define FWCMD_H2C_TRX_TIME_STATS_EN_DURATION_SH 16
+#define FWCMD_H2C_TRX_TIME_STATS_EN_DURATION_MSK 0xffff
+#define FWCMD_H2C_APF_OFLD_CTRL_MACID_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_MACID_MSK 0xffff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_EN_SH 16
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_EN_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_PROG_FRAG_NUM_SH 24
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_PROG_FRAG_NUM_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_MACPKTID_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_MACPKTID_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID1_SH 8
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID1_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID2_SH 16
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID2_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID3_SH 24
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID3_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID4_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID4_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID5_SH 8
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID5_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID6_SH 16
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID6_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID7_SH 24
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_OFLD_PKTID7_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_RAM_SIZE_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_RAM_SIZE_MSK 0xffff
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_PROG_SIZE_SH 16
+#define FWCMD_H2C_APF_OFLD_CTRL_APF_PROG_SIZE_MSK 0xffff
+#define FWCMD_H2C_APF_OFLD_CTRL_VERSION1_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_VERSION1_MSK 0xffffffff
+#define FWCMD_H2C_APF_OFLD_CTRL_VERSION2_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_VERSION2_MSK 0xffffffff
+#define FWCMD_H2C_APF_OFLD_CTRL_HW_AES_IV_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_HW_AES_IV_MSK 0xff
+#define FWCMD_H2C_APF_OFLD_CTRL_MULTICAST_MASK_SH 0
+#define FWCMD_H2C_APF_OFLD_CTRL_MULTICAST_MASK_MSK 0xffff
+#define FWCMD_H2C_SECCAM_INFO_IDX_SH 0
+#define FWCMD_H2C_SECCAM_INFO_IDX_MSK 0xff
+#define FWCMD_H2C_SECCAM_INFO_OFFSET_SH 8
+#define FWCMD_H2C_SECCAM_INFO_OFFSET_MSK 0xff
+#define FWCMD_H2C_SECCAM_INFO_LEN_SH 16
+#define FWCMD_H2C_SECCAM_INFO_LEN_MSK 0xff
+#define FWCMD_H2C_SECCAM_INFO_TYPE_SH 0
+#define FWCMD_H2C_SECCAM_INFO_TYPE_MSK 0xf
+#define FWCMD_H2C_SECCAM_INFO_EXT_KEY BIT(4)
+#define FWCMD_H2C_SECCAM_INFO_SPP_MODE BIT(5)
+#define FWCMD_H2C_SECCAM_INFO_KEY_0_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_0_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_KEY_1_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_1_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_KEY_2_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_2_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_KEY_3_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_3_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_KEY_4_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_4_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_KEY_5_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_5_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_KEY_6_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_6_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_KEY_7_SH 0
+#define FWCMD_H2C_SECCAM_INFO_KEY_7_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_IDX_V1_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_IDX_V1_MSK 0xffff
+#define FWCMD_H2C_SECCAM_INFO_V1_OFFSET_V1_SH 16
+#define FWCMD_H2C_SECCAM_INFO_V1_OFFSET_V1_MSK 0xff
+#define FWCMD_H2C_SECCAM_INFO_V1_LEN_V1_SH 24
+#define FWCMD_H2C_SECCAM_INFO_V1_LEN_V1_MSK 0xff
+#define FWCMD_H2C_SECCAM_INFO_V1_TYPE_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_TYPE_MSK 0xf
+#define FWCMD_H2C_SECCAM_INFO_V1_EXT_KEY BIT(4)
+#define FWCMD_H2C_SECCAM_INFO_V1_SPP_MODE BIT(5)
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_0_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_0_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_1_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_1_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_2_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_2_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_3_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_3_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_4_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_4_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_5_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_5_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_6_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_6_MSK 0xffffffff
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_7_SH 0
+#define FWCMD_H2C_SECCAM_INFO_V1_KEY_7_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_FREXCH_TYPE_SH 0
+#define FWCMD_H2C_SET_SND_PARA_FREXCH_TYPE_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_MODE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_MODE_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_USER_NUM_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_USER_NUM_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_USER_NUM_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_USER_NUM_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_MACID0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_MACID0_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_MACID1_SH 8
+#define FWCMD_H2C_SET_SND_PARA_MACID1_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_MACID2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_MACID2_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_MACID3_SH 24
+#define FWCMD_H2C_SET_SND_PARA_MACID3_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_MACID4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_MACID4_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_MACID5_SH 8
+#define FWCMD_H2C_SET_SND_PARA_MACID5_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_MACID6_SH 16
+#define FWCMD_H2C_SET_SND_PARA_MACID6_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_MACID7_SH 24
+#define FWCMD_H2C_SET_SND_PARA_MACID7_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_NDPA_FRAME_CTRL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_NDPA_FRAME_CTRL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_NDPA_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_NDPA_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_NDPA_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_NDPA_SND_DLG_HE BIT(0)
+#define FWCMD_H2C_SET_SND_PARA_NDPA_SND_DLG_DIALOG_SH 1
+#define FWCMD_H2C_SET_SND_PARA_NDPA_SND_DLG_DIALOG_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HT_NDPA_A3_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HT_NDPA_A3_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_HT_NDPA_A3_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HT_NDPA_A3_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_HT_SEQ_CONTROL_SH 16
+#define FWCMD_H2C_SET_SND_PARA_HT_SEQ_CONTROL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA0_FEEDBACK_TYPE BIT(12)
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA0_NC_SH 13
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA0_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA1_AID12_SH 16
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA1_FEEDBACK_TYPE BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA1_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA1_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA2_FEEDBACK_TYPE BIT(12)
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA2_NC_SH 13
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA2_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA3_AID12_SH 16
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA3_FEEDBACK_TYPE BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA3_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_VHT_STA3_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA0_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA1_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA2_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA3_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA4_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA5_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA6_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_HE_STA7_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP0_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP1_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_BFRP2_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP0_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_HE_BFRP1_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_VHT_BFRP0_FB_REXMIT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_VHT_BFRP0_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_VHT_BFRP1_FB_REXMIT_SH 8
+#define FWCMD_H2C_SET_SND_PARA_VHT_BFRP1_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_VHT_BFRP2_FB_REXMIT_SH 16
+#define FWCMD_H2C_SET_SND_PARA_VHT_BFRP2_FB_REXMIT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD0_SH 16
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD0_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD0_SH 9
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD0_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_WD_FORCE_TXOP_WD0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD0_SH 19
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_ER_WD0 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_DCM_WD0 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_STBC_WD0 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_LDPC_WD0 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_ER_WD0 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_WD_MULTIPORT_ID_WD0 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD0_SH 28
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD0_SH 4
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD0_SH 8
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD0_SH 11
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_STF_MODE_WD0 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_WD_DISDATAFB_WD0 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_SEL_WD0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD0_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_WD_SIFS_TX_WD0 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD0_SH 24
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD0_SH 27
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD1_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD1_SH 9
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD1_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_WD_FORCE_TXOP_WD1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD1_SH 19
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_ER_WD1 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_DCM_WD1 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_STBC_WD1 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_LDPC_WD1 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_ER_WD1 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_WD_MULTIPORT_ID_WD1 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD1_SH 28
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD1_SH 4
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD1_SH 8
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD1_SH 11
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_STF_MODE_WD1 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_WD_DISDATAFB_WD1 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_SEL_WD1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD1_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_WD_SIFS_TX_WD1 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD1_SH 24
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD1_SH 27
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD2_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD2_SH 9
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD2_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_WD_FORCE_TXOP_WD2 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD2_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD2_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD2_SH 19
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_ER_WD2 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_DCM_WD2 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_STBC_WD2 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_LDPC_WD2 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_ER_WD2 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_WD_MULTIPORT_ID_WD2 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD2_SH 28
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD2_SH 4
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD2_SH 8
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD2_SH 11
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_STF_MODE_WD2 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_WD_DISDATAFB_WD2 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_SEL_WD2 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD2_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD2_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_WD_SIFS_TX_WD2 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD2_SH 24
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD2_SH 27
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD2_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD3_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD3_SH 9
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD3_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_WD_FORCE_TXOP_WD3 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD3_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD3_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD3_SH 19
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_ER_WD3 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_DCM_WD3 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_STBC_WD3 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_LDPC_WD3 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_ER_WD3 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_WD_MULTIPORT_ID_WD3 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD3_SH 28
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD3_SH 4
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD3_SH 8
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD3_SH 11
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_STF_MODE_WD3 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_WD_DISDATAFB_WD3 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_SEL_WD3 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD3_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD3_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_WD_SIFS_TX_WD3 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD3_SH 24
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD3_SH 27
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD3_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPKTSIZE_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD4_SH 16
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_DURATION_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_DATARATE_WD4_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD4_SH 9
+#define FWCMD_H2C_SET_SND_PARA_WD_MACID_WD4_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_WD_FORCE_TXOP_WD4 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD4_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_WD4_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD4_SH 19
+#define FWCMD_H2C_SET_SND_PARA_WD_GI_LTF_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_ER_WD4 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_DCM_WD4 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_STBC_WD4 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_LDPC_WD4 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_BW_ER_WD4 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_WD_MULTIPORT_ID_WD4 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD4_SH 28
+#define FWCMD_H2C_SET_SND_PARA_WD_MBSSID_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_WD_SIGNALING_TA_PKT_SC_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD4_SH 4
+#define FWCMD_H2C_SET_SND_PARA_WD_SW_DEFINE_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD4_SH 8
+#define FWCMD_H2C_SET_SND_PARA_WD_TXPWR_OFSET_TYPE_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD4_SH 11
+#define FWCMD_H2C_SET_SND_PARA_WD_LIFETIME_SEL_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_STF_MODE_WD4 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_WD_DISDATAFB_WD4 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_SEL_WD4 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD4_SH 17
+#define FWCMD_H2C_SET_SND_PARA_WD_DATA_TXCNT_LMT_WD4_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_WD_SIFS_TX_WD4 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD4_SH 24
+#define FWCMD_H2C_SET_SND_PARA_WD_SND_PKT_SEL_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD4_SH 27
+#define FWCMD_H2C_SET_SND_PARA_WD_NDPA_WD4_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_CSI_LEN_BFRP0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_CSI_LEN_BFRP0_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_TB_T_PE_BFRP0_SH 12
+#define FWCMD_H2C_SET_SND_PARA_TB_T_PE_BFRP0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_TRI_PAD_BFRP0_SH 14
+#define FWCMD_H2C_SET_SND_PARA_TRI_PAD_BFRP0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_UL_CQI_RPT_TRI_BFRP0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_RF_GAIN_IDX_BFRP0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_RF_GAIN_IDX_BFRP0_MSK 0x3ff
+#define FWCMD_H2C_SET_SND_PARA_FIX_GAIN_EN_BFRP0 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_CSI_LEN_BFRP1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_CSI_LEN_BFRP1_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_TB_T_PE_BFRP1_SH 12
+#define FWCMD_H2C_SET_SND_PARA_TB_T_PE_BFRP1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_TRI_PAD_BFRP1_SH 14
+#define FWCMD_H2C_SET_SND_PARA_TRI_PAD_BFRP1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_UL_CQI_RPT_TRI_BFRP1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_RF_GAIN_IDX_BFRP1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_RF_GAIN_IDX_BFRP1_MSK 0x3ff
+#define FWCMD_H2C_SET_SND_PARA_FIX_GAIN_EN_BFRP1 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_F2P_TYPE_SH 0
+#define FWCMD_H2C_SET_SND_PARA_F2P_TYPE_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_F2P_INDEX_SH 8
+#define FWCMD_H2C_SET_SND_PARA_F2P_INDEX_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_F2P_PERIOD_SH 16
+#define FWCMD_H2C_SET_SND_PARA_F2P_PERIOD_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_F2P_UPDCNT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_F2P_UPDCNT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_CR_IDX_SH 8
+#define FWCMD_H2C_SET_SND_PARA_CR_IDX_MSK 0x3fffff
+#define FWCMD_H2C_SET_SND_PARA_SOUNDING_EN BIT(31)
+#define FWCMD_H2C_GET_CSI_BUF_BAND_SH 0
+#define FWCMD_H2C_GET_CSI_BUF_BAND_MSK 0xff
+#define FWCMD_H2C_GET_CSI_BUF_CSI_BUF_ID_SH 8
+#define FWCMD_H2C_GET_CSI_BUF_CSI_BUF_ID_MSK 0xff
+#define FWCMD_H2C_SET_CSI_BUF_BAND_SH 0
+#define FWCMD_H2C_SET_CSI_BUF_BAND_MSK 0xff
+#define FWCMD_H2C_SET_CSI_BUF_MACID_SH 8
+#define FWCMD_H2C_SET_CSI_BUF_MACID_MSK 0xff
+#define FWCMD_H2C_SET_CSI_BUF_CSI_BUF_ID_SH 16
+#define FWCMD_H2C_SET_CSI_BUF_CSI_BUF_ID_MSK 0xffff
+#define FWCMD_H2C_SET_CSI_BUF_BUF_IDX_SH 0
+#define FWCMD_H2C_SET_CSI_BUF_BUF_IDX_MSK 0xffff
+#define FWCMD_H2C_GET_SND_STS_BAND_SH 0
+#define FWCMD_H2C_GET_SND_STS_BAND_MSK 0xff
+#define FWCMD_H2C_GET_SND_STS_INDEX_SH 8
+#define FWCMD_H2C_GET_SND_STS_INDEX_MSK 0xff
+#define FWCMD_H2C_SET_SND_STS_BAND_SH 0
+#define FWCMD_H2C_SET_SND_STS_BAND_MSK 0xff
+#define FWCMD_H2C_SET_SND_STS_MACID_SH 8
+#define FWCMD_H2C_SET_SND_STS_MACID_MSK 0xff
+#define FWCMD_H2C_SET_SND_STS_INDEX_SH 16
+#define FWCMD_H2C_SET_SND_STS_INDEX_MSK 0xff
+#define FWCMD_H2C_INIT_SND_MER_BAND_SH 0
+#define FWCMD_H2C_INIT_SND_MER_BAND_MSK 0xff
+#define FWCMD_H2C_INIT_SND_MEE_BAND_SH 0
+#define FWCMD_H2C_INIT_SND_MEE_BAND_MSK 0xff
+#define FWCMD_H2C_CSI_FIX_RATE_BAND_SH 0
+#define FWCMD_H2C_CSI_FIX_RATE_BAND_MSK 0xff
+#define FWCMD_H2C_CSI_FIX_RATE_HT_RATE_SH 8
+#define FWCMD_H2C_CSI_FIX_RATE_HT_RATE_MSK 0xff
+#define FWCMD_H2C_CSI_FIX_RATE_VHT_RATE_SH 16
+#define FWCMD_H2C_CSI_FIX_RATE_VHT_RATE_MSK 0xff
+#define FWCMD_H2C_CSI_FIX_RATE_HE_RATE_SH 24
+#define FWCMD_H2C_CSI_FIX_RATE_HE_RATE_MSK 0xff
+#define FWCMD_H2C_CSI_RRSC_BAND_SH 0
+#define FWCMD_H2C_CSI_RRSC_BAND_MSK 0xff
+#define FWCMD_H2C_CSI_RRSC_RRSC_SH 0
+#define FWCMD_H2C_CSI_RRSC_RRSC_MSK 0xffffffff
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_CTRL_SH 0
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_CTRL_MSK 0xffffffff
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_0_SH 0
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_0_MSK 0xffffffff
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_1_SH 0
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_1_MSK 0xffffffff
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_2_SH 0
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_2_MSK 0xffffffff
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_3_SH 0
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_3_MSK 0xffffffff
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_4_SH 0
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_4_MSK 0xffffffff
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_5_SH 0
+#define FWCMD_H2C_SET_MU_TABLE_MU_TBL_5_MSK 0xffffffff
+#define FWCMD_H2C_SET_CSI_PARA_REG_BAND BIT(0)
+#define FWCMD_H2C_SET_CSI_PARA_REG_PORT_SEL BIT(1)
+#define FWCMD_H2C_SET_CSI_PARA_REG_NC_SH 2
+#define FWCMD_H2C_SET_CSI_PARA_REG_NC_MSK 0x7
+#define FWCMD_H2C_SET_CSI_PARA_REG_NR_SH 5
+#define FWCMD_H2C_SET_CSI_PARA_REG_NR_MSK 0x7
+#define FWCMD_H2C_SET_CSI_PARA_REG_NG_SH 8
+#define FWCMD_H2C_SET_CSI_PARA_REG_NG_MSK 0x3
+#define FWCMD_H2C_SET_CSI_PARA_REG_CB_SH 10
+#define FWCMD_H2C_SET_CSI_PARA_REG_CB_MSK 0x3
+#define FWCMD_H2C_SET_CSI_PARA_REG_CS_SH 12
+#define FWCMD_H2C_SET_CSI_PARA_REG_CS_MSK 0x3
+#define FWCMD_H2C_SET_CSI_PARA_REG_LDPC_EN BIT(14)
+#define FWCMD_H2C_SET_CSI_PARA_REG_STBC_EN BIT(15)
+#define FWCMD_H2C_SET_CSI_PARA_REG_BF_EN BIT(16)
+#define FWCMD_H2C_HW_SND_PR_BAND_SH 0
+#define FWCMD_H2C_HW_SND_PR_BAND_MSK 0xff
+#define FWCMD_H2C_HW_SND_PR_PR_SH 8
+#define FWCMD_H2C_HW_SND_PR_PR_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_FREXCH_TYPE_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_FREXCH_TYPE_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_MODE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V1_MODE_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_USER_NUM_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_USER_NUM_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_USER_NUM_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_USER_NUM_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID0_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID1_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID1_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID2_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID3_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID3_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID4_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID5_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID5_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID6_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID6_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID7_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID7_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID8_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID8_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID9_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID9_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID10_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID10_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID11_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID11_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID12_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID13_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID13_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID14_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID14_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID15_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_MACID15_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_FRAME_CTRL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_FRAME_CTRL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_SND_DLG_HE BIT(0)
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_SND_DLG_DIALOG_SH 1
+#define FWCMD_H2C_SET_SND_PARA_V1_NDPA_SND_DLG_DIALOG_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HT_NDPA_A3_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HT_NDPA_A3_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_HT_NDPA_A3_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HT_NDPA_A3_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_HT_SEQ_CONTROL_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_HT_SEQ_CONTROL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA0_FEEDBACK_TYPE BIT(12)
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA0_NC_SH 13
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA0_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA1_AID12_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA1_FEEDBACK_TYPE BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA1_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA1_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA2_FEEDBACK_TYPE BIT(12)
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA2_NC_SH 13
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA2_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA3_AID12_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA3_FEEDBACK_TYPE BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA3_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_STA3_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA0_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA1_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA2_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA3_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA4_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA5_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA6_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA7_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA8_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA9_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA10_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA11_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA12_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA13_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA14_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_STA15_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP0_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP1_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_BFRP2_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U4_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U5_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U6_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP0_U7_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U4_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U5_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U6_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_HE_BFRP1_U7_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_BFRP0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_BFRP0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_BFRP1_FEEDBACKSEG_REXMIT_BITMAP_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_BFRP1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_BFRP2_FEEDBACKSEG_REXMIT_BITMAP_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_VHT_BFRP2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD0_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD0_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD0_SH 9
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD0_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_FORCE_TXOP_WD0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD0_SH 19
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_ER_WD0 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_DCM_WD0 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_STBC_WD0 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_LDPC_WD0 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_ER_WD0 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MULTIPORT_ID_WD0 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD0_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD0_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD0_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD0_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_STF_MODE_WD0 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DISDATAFB_WD0 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_SEL_WD0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD0_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIFS_TX_WD0 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD0_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD0_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_CQI_WD0 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD1_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD1_SH 9
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD1_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_FORCE_TXOP_WD1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD1_SH 19
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_ER_WD1 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_DCM_WD1 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_STBC_WD1 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_LDPC_WD1 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_ER_WD1 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MULTIPORT_ID_WD1 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD1_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD1_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD1_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD1_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_STF_MODE_WD1 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DISDATAFB_WD1 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_SEL_WD1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD1_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIFS_TX_WD1 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD1_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD1_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_CQI_WD1 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD2_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD2_SH 9
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD2_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_FORCE_TXOP_WD2 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD2_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD2_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD2_SH 19
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_ER_WD2 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_DCM_WD2 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_STBC_WD2 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_LDPC_WD2 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_ER_WD2 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MULTIPORT_ID_WD2 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD2_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD2_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD2_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD2_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_STF_MODE_WD2 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DISDATAFB_WD2 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_SEL_WD2 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD2_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD2_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIFS_TX_WD2 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD2_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD2_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD2_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_CQI_WD2 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD3_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD3_SH 9
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD3_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_FORCE_TXOP_WD3 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD3_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD3_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD3_SH 19
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_ER_WD3 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_DCM_WD3 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_STBC_WD3 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_LDPC_WD3 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_ER_WD3 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MULTIPORT_ID_WD3 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD3_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD3_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD3_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD3_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_STF_MODE_WD3 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DISDATAFB_WD3 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_SEL_WD3 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD3_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD3_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIFS_TX_WD3 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD3_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD3_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD3_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_CQI_WD3 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPKTSIZE_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD4_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_DURATION_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATARATE_WD4_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD4_SH 9
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MACID_WD4_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_FORCE_TXOP_WD4 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD4_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_WD4_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD4_SH 19
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_GI_LTF_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_ER_WD4 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_DCM_WD4 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_STBC_WD4 BIT(24)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_LDPC_WD4 BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_BW_ER_WD4 BIT(26)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MULTIPORT_ID_WD4 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD4_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_MBSSID_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIGNALING_TA_PKT_SC_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD4_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SW_DEFINE_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD4_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_TXPWR_OFSET_TYPE_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD4_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_LIFETIME_SEL_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_STF_MODE_WD4 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DISDATAFB_WD4 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_SEL_WD4 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD4_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_DATA_TXCNT_LMT_WD4_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SIFS_TX_WD4 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD4_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_SND_PKT_SEL_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD4_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_NDPA_WD4_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_WD_CQI_WD4 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V1_CSI_LEN_BFRP0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_CSI_LEN_BFRP0_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_TB_T_PE_BFRP0_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_TB_T_PE_BFRP0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_TRI_PAD_BFRP0_SH 14
+#define FWCMD_H2C_SET_SND_PARA_V1_TRI_PAD_BFRP0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_UL_CQI_RPT_TRI_BFRP0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_RF_GAIN_IDX_BFRP0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_RF_GAIN_IDX_BFRP0_MSK 0x3ff
+#define FWCMD_H2C_SET_SND_PARA_V1_FIX_GAIN_EN_BFRP0 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_UL_LEN_REF_BFRP0_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V1_UL_LEN_REF_BFRP0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_CSI_LEN_BFRP1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_CSI_LEN_BFRP1_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V1_TB_T_PE_BFRP1_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V1_TB_T_PE_BFRP1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_TRI_PAD_BFRP1_SH 14
+#define FWCMD_H2C_SET_SND_PARA_V1_TRI_PAD_BFRP1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V1_UL_CQI_RPT_TRI_BFRP1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V1_RF_GAIN_IDX_BFRP1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V1_RF_GAIN_IDX_BFRP1_MSK 0x3ff
+#define FWCMD_H2C_SET_SND_PARA_V1_FIX_GAIN_EN_BFRP1 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V1_UL_LEN_REF_BFRP1_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V1_UL_LEN_REF_BFRP1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_TYPE_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_TYPE_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_INDEX_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_INDEX_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_PERIOD_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_PERIOD_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_UPDCNT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V1_F2P_UPDCNT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V1_CR_IDX_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V1_CR_IDX_MSK 0x3fffff
+#define FWCMD_H2C_SET_SND_PARA_V1_SOUNDING_EN BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_FREXCH_TYPE_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_FREXCH_TYPE_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_MODE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V2_MODE_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_USER_NUM_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_USER_NUM_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_USER_NUM_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_USER_NUM_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_RETRY_NUM_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_RETRY_NUM_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_BAND BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_PROTECTION_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_PROTECTION_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID5_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID5_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID6_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID6_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID7_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID7_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID8_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID8_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID9_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID9_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID10_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID10_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID11_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID11_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID12_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID13_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID13_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID14_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID14_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID15_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_MACID15_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_FRAME_CTRL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_FRAME_CTRL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_SND_DLG_RANGING BIT(0)
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_SND_DLG_HE BIT(1)
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_SND_DLG_DIALOG_SH 2
+#define FWCMD_H2C_SET_SND_PARA_V2_NDPA_SND_DLG_DIALOG_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_NDPA_A3_L_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_NDPA_A3_L_MSK 0xffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_NDPA_A3_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_NDPA_A3_H_MSK 0xffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_SEQ_CONTROL_L_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_SEQ_CONTROL_L_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_SEQ_CONTROL_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_SEQ_CONTROL_H_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HTC_L_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HTC_L_MSK 0xffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_HTC_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HTC_H_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_OUI1_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_OUI1_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_OUI2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_OUI2_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_OUI3_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_HT_OUI3_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA0_FEEDBACK_TYPE BIT(12)
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA0_NC_SH 13
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA0_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA1_AID12_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA1_FEEDBACK_TYPE BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA1_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA1_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA2_FEEDBACK_TYPE BIT(12)
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA2_NC_SH 13
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA2_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA3_AID12_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA3_FEEDBACK_TYPE BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA3_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_STA3_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA0_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA1_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA2_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA3_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA4_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA5_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA6_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA7_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA8_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA9_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA10_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA11_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA12_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA13_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA14_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_BW_MSK 0x3fff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_NC_SH 29
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_STA15_NC_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA0_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA1_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA2_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA3_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA4_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA5_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA6_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA7_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA8_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA9_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA10_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA11_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA12_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA13_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA14_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_AID11_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_AID11_MSK 0x7ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_BW_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_BW_MSK 0x1ff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_NC_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_NC_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_FB_NG_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_FB_NG_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_DISAMBIGUATION BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_STA15_CB BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP0_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP1_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_FRAME_CTL_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_FRAME_CTL_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_DURATION_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_DURATION_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A1_L_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A1_L_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A1_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A1_H_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A2_L_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A2_L_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A2_H_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_BFRP2_A2_H_MSK 0xffffffff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U4_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U5_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U6_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP0_U7_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U4_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U5_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U6_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_UL_DCM BIT(25)
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_HE_BFRP1_U7_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_BFRP0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_BFRP0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_BFRP1_FEEDBACKSEG_REXMIT_BITMAP_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_BFRP1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_BFRP2_FEEDBACKSEG_REXMIT_BITMAP_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_VHT_BFRP2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_HE_EHT_P160 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_SPECIAL_PRESENT BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_SPATIAL_REUSE2_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_UL_SPATIAL_REUSE2_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_USIG_DISRGRD_AND_VLD_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_USIG_DISRGRD_AND_VLD_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_PHY_VER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_PHY_VER_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_ULBW_EXT_SH 3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_ULBW_EXT_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U0_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U1_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U2_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U3_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U4_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U5_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U6_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP0_U7_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_TRIGGER_INFO_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_TRIGGER_INFO_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_LENGTH_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_LENGTH_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_MORE_TF BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_CS_REQUIRED BIT(17)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_BW_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_BW_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_GI_LTF_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_GI_LTF_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_MU_MIMO_LTF_MODE BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_NUM_OF_HE_LTF_SH 23
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_NUM_OF_HE_LTF_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_PKTEXT_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_PKTEXT_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_STBC BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_LDPC_EXTRA_SYMBOL BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_DOPPLER BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_AP_TX_POWER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_AP_TX_POWER_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_SPATIAL_REUSE_SH 6
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_SPATIAL_REUSE_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_HE_EHT_P160 BIT(22)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_SPECIAL_PRESENT BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_SPATIAL_REUSE2_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_UL_SPATIAL_REUSE2_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_USIG_DISRGRD_AND_VLD_SH 20
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_USIG_DISRGRD_AND_VLD_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_PHY_VER_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_PHY_VER_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_ULBW_EXT_SH 3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_ULBW_EXT_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U0_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U1_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U2_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U3_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U4_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U5_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U6_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_AID12_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_AID12_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_RU_POS_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_RU_POS_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_UL_FEC_CODE BIT(20)
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_UL_MCS_SH 21
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_UL_MCS_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_SS_ALLOC_SH 26
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_SS_ALLOC_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_FEEDBACKSEG_REXMIT_BITMAP_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_FEEDBACKSEG_REXMIT_BITMAP_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_UL_TGT_RSSI_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_UL_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_EHT_BFRP1_U7_PS160 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD0_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD0_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD0_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD0_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD0_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD0_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD0_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_STF_MODE_WD0 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DISDATAFB_WD0 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_SEL_WD0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD0_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIFS_TX_WD0 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD0_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD0_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_CQI_SND_WD0 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_LDPC_WD0 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD0_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD0_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_FORCE_TXOP_WD0 BIT(21)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD0_SH 22
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD0_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD0_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_ER_WD0 BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_DCM_WD0 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_STBC_WD0 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_ER_WD0 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD1_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD1_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD1_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD1_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD1_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_STF_MODE_WD1 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DISDATAFB_WD1 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_SEL_WD1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD1_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIFS_TX_WD1 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD1_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD1_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_CQI_SND_WD1 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_LDPC_WD1 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD1_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_FORCE_TXOP_WD1 BIT(21)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD1_SH 22
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD1_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD1_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_ER_WD1 BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_DCM_WD1 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_STBC_WD1 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_ER_WD1 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD2_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD2_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD2_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD2_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD2_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD2_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_STF_MODE_WD2 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DISDATAFB_WD2 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_SEL_WD2 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD2_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD2_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIFS_TX_WD2 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD2_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD2_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_CQI_SND_WD2 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_LDPC_WD2 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD2_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD2_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD2_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_FORCE_TXOP_WD2 BIT(21)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD2_SH 22
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD2_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD2_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_ER_WD2 BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_DCM_WD2 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_STBC_WD2 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_ER_WD2 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD3_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD3_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD3_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD3_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD3_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD3_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_STF_MODE_WD3 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DISDATAFB_WD3 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_SEL_WD3 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD3_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD3_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIFS_TX_WD3 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD3_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD3_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_CQI_SND_WD3 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_LDPC_WD3 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD3_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD3_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD3_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_FORCE_TXOP_WD3 BIT(21)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD3_SH 22
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD3_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD3_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_ER_WD3 BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_DCM_WD3 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_STBC_WD3 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_ER_WD3 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPKTSIZE_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD4_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_DURATION_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MACID_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD4_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATARATE_WD4_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD4_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MBSSID_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIGNALING_TA_PKT_SC_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD4_SH 4
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SW_DEFINE_WD4_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD4_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_TXPWR_OFSET_TYPE_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD4_SH 11
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_LIFETIME_SEL_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_STF_MODE_WD4 BIT(14)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DISDATAFB_WD4 BIT(15)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_SEL_WD4 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD4_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_TXCNT_LMT_WD4_MSK 0x3f
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SIFS_TX_WD4 BIT(23)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD4_SH 24
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_SND_PKT_SEL_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD4_SH 27
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_NDPA_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_CQI_SND_WD4 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_LDPC_WD4 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_PATTERN_WD4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD4_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_PUNC_MODE_WD4_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD4_SH 18
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_GI_LTF_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_FORCE_TXOP_WD4 BIT(21)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD4_SH 22
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_MULTIPORT_ID_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD4_SH 25
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_WD4_MSK 0x7
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_ER_WD4 BIT(28)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_DCM_WD4 BIT(29)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_STBC_WD4 BIT(30)
+#define FWCMD_H2C_SET_SND_PARA_V2_WD_DATA_BW_ER_WD4 BIT(31)
+#define FWCMD_H2C_SET_SND_PARA_V2_CSI_LEN_BFRP0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_CSI_LEN_BFRP0_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_TB_T_PE_BFRP0_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_TB_T_PE_BFRP0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_TRI_PAD_BFRP0_SH 14
+#define FWCMD_H2C_SET_SND_PARA_V2_TRI_PAD_BFRP0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_CQI_RPT_TRI_BFRP0 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_RF_GAIN_IDX_BFRP0_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V2_RF_GAIN_IDX_BFRP0_MSK 0x3ff
+#define FWCMD_H2C_SET_SND_PARA_V2_FIX_GAIN_EN_BFRP0 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_LEN_REF_BFRP0_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_LEN_REF_BFRP0_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_ELNA_IDX_BFRP0 BIT(0)
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_PRI_EXP_RSSI_DBM_BFRP0_SH 1
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_PRI_EXP_RSSI_DBM_BFRP0_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_CSI_RPT_TRI_BFRP0 BIT(8)
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_APEP_UNIT_BFRP0_SH 9
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_APEP_UNIT_BFRP0_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_CSI_INT_EN_BFRP0 BIT(11)
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U5_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U5_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U6_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U6_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U7_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP0_U7_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_CSI_LEN_BFRP1_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_CSI_LEN_BFRP1_MSK 0xfff
+#define FWCMD_H2C_SET_SND_PARA_V2_TB_T_PE_BFRP1_SH 12
+#define FWCMD_H2C_SET_SND_PARA_V2_TB_T_PE_BFRP1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_TRI_PAD_BFRP1_SH 14
+#define FWCMD_H2C_SET_SND_PARA_V2_TRI_PAD_BFRP1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_CQI_RPT_TRI_BFRP1 BIT(16)
+#define FWCMD_H2C_SET_SND_PARA_V2_RF_GAIN_IDX_BFRP1_SH 17
+#define FWCMD_H2C_SET_SND_PARA_V2_RF_GAIN_IDX_BFRP1_MSK 0x3ff
+#define FWCMD_H2C_SET_SND_PARA_V2_FIX_GAIN_EN_BFRP1 BIT(27)
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_LEN_REF_BFRP1_SH 28
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_LEN_REF_BFRP1_MSK 0xf
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_ELNA_IDX_BFRP1 BIT(0)
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_PRI_EXP_RSSI_DBM_BFRP1_SH 1
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_PRI_EXP_RSSI_DBM_BFRP1_MSK 0x7f
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_CSI_RPT_TRI_BFRP1 BIT(8)
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_APEP_UNIT_BFRP1_SH 9
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_APEP_UNIT_BFRP1_MSK 0x3
+#define FWCMD_H2C_SET_SND_PARA_V2_UL_CSI_INT_EN_BFRP1 BIT(11)
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U0_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U0_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U1_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U1_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U2_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U2_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U3_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U3_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U4_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U4_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U5_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U5_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U6_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U6_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U7_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_SND_BITMAP_BFRP1_U7_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_TYPE_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_TYPE_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_INDEX_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_INDEX_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_PERIOD_SH 16
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_PERIOD_MSK 0xffff
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_UPDCNT_SH 0
+#define FWCMD_H2C_SET_SND_PARA_V2_F2P_UPDCNT_MSK 0xff
+#define FWCMD_H2C_SET_SND_PARA_V2_CR_IDX_SH 8
+#define FWCMD_H2C_SET_SND_PARA_V2_CR_IDX_MSK 0x3fffff
+#define FWCMD_H2C_BA_CAM_VALID BIT(0)
+#define FWCMD_H2C_BA_CAM_INIT_REQ BIT(1)
+#define FWCMD_H2C_BA_CAM_ENTRY_IDX_SH 2
+#define FWCMD_H2C_BA_CAM_ENTRY_IDX_MSK 0x3
+#define FWCMD_H2C_BA_CAM_TID_SH 4
+#define FWCMD_H2C_BA_CAM_TID_MSK 0xf
+#define FWCMD_H2C_BA_CAM_MACID_SH 8
+#define FWCMD_H2C_BA_CAM_MACID_MSK 0xff
+#define FWCMD_H2C_BA_CAM_BMAP_SIZE_SH 16
+#define FWCMD_H2C_BA_CAM_BMAP_SIZE_MSK 0xf
+#define FWCMD_H2C_BA_CAM_SSN_SH 20
+#define FWCMD_H2C_BA_CAM_SSN_MSK 0xfff
+#define FWCMD_H2C_BA_CAM_UID_VALUE_SH 0
+#define FWCMD_H2C_BA_CAM_UID_VALUE_MSK 0xff
+#define FWCMD_H2C_BA_CAM_STD_ENTRY_EN BIT(8)
+#define FWCMD_H2C_BA_CAM_BAND_SEL BIT(9)
+#define FWCMD_H2C_BA_CAM_ENTRY_IDX_V1_SH 28
+#define FWCMD_H2C_BA_CAM_ENTRY_IDX_V1_MSK 0xf
+#define FWCMD_H2C_BA_CAM_V1_VALID BIT(0)
+#define FWCMD_H2C_BA_CAM_V1_INIT_REQ BIT(1)
+#define FWCMD_H2C_BA_CAM_V1_TID_SH 4
+#define FWCMD_H2C_BA_CAM_V1_TID_MSK 0xf
+#define FWCMD_H2C_BA_CAM_V1_MACID_SH 8
+#define FWCMD_H2C_BA_CAM_V1_MACID_MSK 0xff
+#define FWCMD_H2C_BA_CAM_V1_BMAP_SIZE_SH 16
+#define FWCMD_H2C_BA_CAM_V1_BMAP_SIZE_MSK 0xf
+#define FWCMD_H2C_BA_CAM_V1_SSN_SH 20
+#define FWCMD_H2C_BA_CAM_V1_SSN_MSK 0xfff
+#define FWCMD_H2C_BA_CAM_V1_UID_VALUE_SH 0
+#define FWCMD_H2C_BA_CAM_V1_UID_VALUE_MSK 0xff
+#define FWCMD_H2C_BA_CAM_V1_STD_ENTRY_EN BIT(8)
+#define FWCMD_H2C_BA_CAM_V1_BAND_SEL BIT(9)
+#define FWCMD_H2C_BA_CAM_V1_MLD_EN BIT(10)
+#define FWCMD_H2C_BA_CAM_V1_ENTRY_IDX_SH 24
+#define FWCMD_H2C_BA_CAM_V1_ENTRY_IDX_MSK 0xff
+#define FWCMD_H2C_BA_CAM_INIT_USERS_SH 0
+#define FWCMD_H2C_BA_CAM_INIT_USERS_MSK 0xff
+#define FWCMD_H2C_BA_CAM_INIT_OFFSET_SH 12
+#define FWCMD_H2C_BA_CAM_INIT_OFFSET_MSK 0xff
+#define FWCMD_H2C_BA_CAM_INIT_BAND_SEL BIT(24)
+#define FWCMD_H2C_IE_CAM_BAND BIT(0)
+#define FWCMD_H2C_IE_CAM_PORT_SH 1
+#define FWCMD_H2C_IE_CAM_PORT_MSK 0x7
+#define FWCMD_H2C_IE_CAM_CAM_EN BIT(4)
+#define FWCMD_H2C_IE_CAM_HIT_FRWD_EN BIT(5)
+#define FWCMD_H2C_IE_CAM_HIT_FRWD_SH 6
+#define FWCMD_H2C_IE_CAM_HIT_FRWD_MSK 0x3
+#define FWCMD_H2C_IE_CAM_MISS_FRWD_EN BIT(8)
+#define FWCMD_H2C_IE_CAM_MISS_FRWD_SH 9
+#define FWCMD_H2C_IE_CAM_MISS_FRWD_MSK 0x3
+#define FWCMD_H2C_IE_CAM_UPD_NUM_SH 11
+#define FWCMD_H2C_IE_CAM_UPD_NUM_MSK 0x1f
+#define FWCMD_H2C_IE_CAM_RST BIT(16)
+#define FWCMD_H2C_IE_CAM_IDX_SH 0
+#define FWCMD_H2C_IE_CAM_IDX_MSK 0xff
+#define FWCMD_H2C_IE_CAM_CAM_INFO0_SH 8
+#define FWCMD_H2C_IE_CAM_CAM_INFO0_MSK 0xffffff
+#define FWCMD_H2C_IE_CAM_CAM_INFO1_SH 0
+#define FWCMD_H2C_IE_CAM_CAM_INFO1_MSK 0xffffffff
+#define FWCMD_H2C_ADD_MCC_MACID_SH 0
+#define FWCMD_H2C_ADD_MCC_MACID_MSK 0xff
+#define FWCMD_H2C_ADD_MCC_CENTRAL_CH_SEG0_SH 8
+#define FWCMD_H2C_ADD_MCC_CENTRAL_CH_SEG0_MSK 0xff
+#define FWCMD_H2C_ADD_MCC_CENTRAL_CH_SEG1_SH 16
+#define FWCMD_H2C_ADD_MCC_CENTRAL_CH_SEG1_MSK 0xff
+#define FWCMD_H2C_ADD_MCC_PRIMARY_CH_SH 24
+#define FWCMD_H2C_ADD_MCC_PRIMARY_CH_MSK 0xff
+#define FWCMD_H2C_ADD_MCC_BANDWIDTH_SH 0
+#define FWCMD_H2C_ADD_MCC_BANDWIDTH_MSK 0xf
+#define FWCMD_H2C_ADD_MCC_GROUP_SH 4
+#define FWCMD_H2C_ADD_MCC_GROUP_MSK 0x3
+#define FWCMD_H2C_ADD_MCC_C2H_RPT_SH 6
+#define FWCMD_H2C_ADD_MCC_C2H_RPT_MSK 0x3
+#define FWCMD_H2C_ADD_MCC_DIS_TX_NULL BIT(8)
+#define FWCMD_H2C_ADD_MCC_DIS_SW_RETRY BIT(9)
+#define FWCMD_H2C_ADD_MCC_IN_CURR_CH BIT(10)
+#define FWCMD_H2C_ADD_MCC_SW_RETRY_COUNT_SH 11
+#define FWCMD_H2C_ADD_MCC_SW_RETRY_COUNT_MSK 0x7
+#define FWCMD_H2C_ADD_MCC_TX_NULL_EARLY_SH 14
+#define FWCMD_H2C_ADD_MCC_TX_NULL_EARLY_MSK 0xf
+#define FWCMD_H2C_ADD_MCC_BTC_IN_2G BIT(18)
+#define FWCMD_H2C_ADD_MCC_PTA_EN BIT(19)
+#define FWCMD_H2C_ADD_MCC_RFK_BY_PASS BIT(20)
+#define FWCMD_H2C_ADD_MCC_CH_BAND_TYPE_SH 21
+#define FWCMD_H2C_ADD_MCC_CH_BAND_TYPE_MSK 0x3
+#define FWCMD_H2C_ADD_MCC_DURATION_SH 0
+#define FWCMD_H2C_ADD_MCC_DURATION_MSK 0xffffffff
+#define FWCMD_H2C_ADD_MCC_COURTESY_EN BIT(0)
+#define FWCMD_H2C_ADD_MCC_COURTESY_NUM_SH 8
+#define FWCMD_H2C_ADD_MCC_COURTESY_NUM_MSK 0xff
+#define FWCMD_H2C_ADD_MCC_COURTESY_TARGET_SH 16
+#define FWCMD_H2C_ADD_MCC_COURTESY_TARGET_MSK 0xff
+#define FWCMD_H2C_START_MCC_GROUP_SH 0
+#define FWCMD_H2C_START_MCC_GROUP_MSK 0x3
+#define FWCMD_H2C_START_MCC_BTC_IN_GROUP BIT(2)
+#define FWCMD_H2C_START_MCC_OLD_GROUP_ACTION_SH 3
+#define FWCMD_H2C_START_MCC_OLD_GROUP_ACTION_MSK 0x3
+#define FWCMD_H2C_START_MCC_OLD_GROUP_SH 5
+#define FWCMD_H2C_START_MCC_OLD_GROUP_MSK 0x3
+#define FWCMD_H2C_START_MCC_START_TSF_OPT BIT(7)
+#define FWCMD_H2C_START_MCC_NOTIFY_CNT_SH 16
+#define FWCMD_H2C_START_MCC_NOTIFY_CNT_MSK 0x7
+#define FWCMD_H2C_START_MCC_NOTIFY_RXDBG_EN BIT(21)
+#define FWCMD_H2C_START_MCC_MACID_SH 24
+#define FWCMD_H2C_START_MCC_MACID_MSK 0xff
+#define FWCMD_H2C_START_MCC_TSF_LOW_SH 0
+#define FWCMD_H2C_START_MCC_TSF_LOW_MSK 0xffffffff
+#define FWCMD_H2C_START_MCC_TSF_HIGH_SH 0
+#define FWCMD_H2C_START_MCC_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_H2C_STOP_MCC_MACID_SH 0
+#define FWCMD_H2C_STOP_MCC_MACID_MSK 0xff
+#define FWCMD_H2C_STOP_MCC_GROUP_SH 8
+#define FWCMD_H2C_STOP_MCC_GROUP_MSK 0x3
+#define FWCMD_H2C_STOP_MCC_PREV_GROUPS BIT(10)
+#define FWCMD_H2C_DEL_MCC_GROUP_GROUP_SH 0
+#define FWCMD_H2C_DEL_MCC_GROUP_GROUP_MSK 0x3
+#define FWCMD_H2C_DEL_MCC_GROUP_PREV_GROUPS BIT(2)
+#define FWCMD_H2C_RESET_MCC_GROUP_GROUP_SH 0
+#define FWCMD_H2C_RESET_MCC_GROUP_GROUP_MSK 0x3
+#define FWCMD_H2C_MCC_REQ_TSF_GROUP_SH 0
+#define FWCMD_H2C_MCC_REQ_TSF_GROUP_MSK 0x3
+#define FWCMD_H2C_MCC_REQ_TSF_MACID_X_SH 8
+#define FWCMD_H2C_MCC_REQ_TSF_MACID_X_MSK 0xff
+#define FWCMD_H2C_MCC_REQ_TSF_MACID_Y_SH 16
+#define FWCMD_H2C_MCC_REQ_TSF_MACID_Y_MSK 0xff
+#define FWCMD_H2C_MCC_MACID_BITMAP_GROUP_SH 0
+#define FWCMD_H2C_MCC_MACID_BITMAP_GROUP_MSK 0x3
+#define FWCMD_H2C_MCC_MACID_BITMAP_MACID_SH 8
+#define FWCMD_H2C_MCC_MACID_BITMAP_MACID_MSK 0xff
+#define FWCMD_H2C_MCC_MACID_BITMAP_BITMAP_LENGTH_SH 16
+#define FWCMD_H2C_MCC_MACID_BITMAP_BITMAP_LENGTH_MSK 0xff
+#define FWCMD_H2C_MCC_MACID_BITMAP_BITMAP_SH 0
+#define FWCMD_H2C_MCC_MACID_BITMAP_BITMAP_MSK 0xffffffff
+#define FWCMD_H2C_MCC_SYNC_GROUP_SH 0
+#define FWCMD_H2C_MCC_SYNC_GROUP_MSK 0x3
+#define FWCMD_H2C_MCC_SYNC_MACID_SOURCE_SH 8
+#define FWCMD_H2C_MCC_SYNC_MACID_SOURCE_MSK 0xff
+#define FWCMD_H2C_MCC_SYNC_MACID_TARGET_SH 16
+#define FWCMD_H2C_MCC_SYNC_MACID_TARGET_MSK 0xff
+#define FWCMD_H2C_MCC_SYNC_SYNC_OFFSET_SH 24
+#define FWCMD_H2C_MCC_SYNC_SYNC_OFFSET_MSK 0xff
+#define FWCMD_H2C_MCC_SET_DURATION_GROUP_SH 0
+#define FWCMD_H2C_MCC_SET_DURATION_GROUP_MSK 0x3
+#define FWCMD_H2C_MCC_SET_DURATION_BTC_IN_GROUP BIT(2)
+#define FWCMD_H2C_MCC_SET_DURATION_START_MACID_SH 8
+#define FWCMD_H2C_MCC_SET_DURATION_START_MACID_MSK 0xff
+#define FWCMD_H2C_MCC_SET_DURATION_MACID_X_SH 16
+#define FWCMD_H2C_MCC_SET_DURATION_MACID_X_MSK 0xff
+#define FWCMD_H2C_MCC_SET_DURATION_MACID_Y_SH 24
+#define FWCMD_H2C_MCC_SET_DURATION_MACID_Y_MSK 0xff
+#define FWCMD_H2C_MCC_SET_DURATION_START_TSF_LOW_SH 0
+#define FWCMD_H2C_MCC_SET_DURATION_START_TSF_LOW_MSK 0xffffffff
+#define FWCMD_H2C_MCC_SET_DURATION_START_TSF_HIGH_SH 0
+#define FWCMD_H2C_MCC_SET_DURATION_START_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_H2C_MCC_SET_DURATION_DURATION_X_SH 0
+#define FWCMD_H2C_MCC_SET_DURATION_DURATION_X_MSK 0xffffffff
+#define FWCMD_H2C_MCC_SET_DURATION_DURATION_Y_SH 0
+#define FWCMD_H2C_MCC_SET_DURATION_DURATION_Y_MSK 0xffffffff
+#define FWCMD_H2C_MCC_DBG_NOTIFY_NOTIFY_RXDBG_EN BIT(0)
+#define FWCMD_H2C_MCC_DBG_NOTIFY_NOTIFY_CNT_SH 8
+#define FWCMD_H2C_MCC_DBG_NOTIFY_NOTIFY_CNT_MSK 0x7
+#define FWCMD_H2C_PLAT_FLASH_WRITE_ADDR_SH 0
+#define FWCMD_H2C_PLAT_FLASH_WRITE_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FLASH_WRITE_LENGTH_SH 0
+#define FWCMD_H2C_PLAT_FLASH_WRITE_LENGTH_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FLASH_WRITE_PAYLOAD0_SH 0
+#define FWCMD_H2C_PLAT_FLASH_WRITE_PAYLOAD0_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FLASH_ERASE_ADDR_SH 0
+#define FWCMD_H2C_PLAT_FLASH_ERASE_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FLASH_ERASE_LENGTH_SH 0
+#define FWCMD_H2C_PLAT_FLASH_ERASE_LENGTH_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FLASH_READ_ADDR_SH 0
+#define FWCMD_H2C_PLAT_FLASH_READ_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_FLASH_READ_LENGTH_SH 0
+#define FWCMD_H2C_PLAT_FLASH_READ_LENGTH_MSK 0xffffffff
+#define FWCMD_H2C_FCS_AP_PORT_ID_SH 0
+#define FWCMD_H2C_FCS_AP_PORT_ID_MSK 0xf
+#define FWCMD_H2C_FCS_CH_IDX_SH 4
+#define FWCMD_H2C_FCS_CH_IDX_MSK 0xf
+#define FWCMD_H2C_FCS_THERMAL_IDX_SH 8
+#define FWCMD_H2C_FCS_THERMAL_IDX_MSK 0xf
+#define FWCMD_H2C_FCS_PAUSE_REL_MODE_SH 12
+#define FWCMD_H2C_FCS_PAUSE_REL_MODE_MSK 0xf
+#define FWCMD_H2C_FCS_CON_STA_NUM_SH 16
+#define FWCMD_H2C_FCS_CON_STA_NUM_MSK 0xff
+#define FWCMD_H2C_FCS_BAND BIT(24)
+#define FWCMD_H2C_FCS_BANDWIDTH_SH 25
+#define FWCMD_H2C_FCS_BANDWIDTH_MSK 0x3
+#define FWCMD_H2C_FCS_CH_BAND_SH 27
+#define FWCMD_H2C_FCS_CH_BAND_MSK 0x3
+#define FWCMD_H2C_FCS_PRI_CH_SH 0
+#define FWCMD_H2C_FCS_PRI_CH_MSK 0xff
+#define FWCMD_H2C_FCS_CENTRAL_CH_SH 8
+#define FWCMD_H2C_FCS_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_FCS_CSA_RETRY_SH 16
+#define FWCMD_H2C_FCS_CSA_RETRY_MSK 0xf
+#define FWCMD_H2C_FCS_REL_PAUSE_TSFL_SH 0
+#define FWCMD_H2C_FCS_REL_PAUSE_TSFL_MSK 0xffffffff
+#define FWCMD_H2C_FCS_REL_PAUSE_TSFH_SH 0
+#define FWCMD_H2C_FCS_REL_PAUSE_TSFH_MSK 0xffffffff
+#define FWCMD_H2C_FCS_REL_PAUSE_DELAY_TIME_SH 0
+#define FWCMD_H2C_FCS_REL_PAUSE_DELAY_TIME_MSK 0xffffffff
+#define FWCMD_H2C_FCS_CSA_PKT_ID0_SH 0
+#define FWCMD_H2C_FCS_CSA_PKT_ID0_MSK 0xff
+#define FWCMD_H2C_FCS_CSA_PKT_ID1_SH 8
+#define FWCMD_H2C_FCS_CSA_PKT_ID1_MSK 0xff
+#define FWCMD_H2C_FCS_CSA_PKT_ID2_SH 16
+#define FWCMD_H2C_FCS_CSA_PKT_ID2_MSK 0xff
+#define FWCMD_H2C_FCS_CSA_PKT_ID3_SH 24
+#define FWCMD_H2C_FCS_CSA_PKT_ID3_MSK 0xff
+#define FWCMD_H2C_CFG_WPS_EN BIT(0)
+#define FWCMD_H2C_CFG_WPS_GPIO_SH 8
+#define FWCMD_H2C_CFG_WPS_GPIO_MSK 0xff
+#define FWCMD_H2C_CFG_WPS_INTL_SH 16
+#define FWCMD_H2C_CFG_WPS_INTL_MSK 0xff
+#define FWCMD_H2C_PORT_INIT_BAND BIT(0)
+#define FWCMD_H2C_PORT_INIT_PORT_SH 1
+#define FWCMD_H2C_PORT_INIT_PORT_MSK 0x7
+#define FWCMD_H2C_PORT_INIT_NET_TYPE_SH 4
+#define FWCMD_H2C_PORT_INIT_NET_TYPE_MSK 0x7
+#define FWCMD_H2C_PORT_INIT_DTIM_PRD_SH 8
+#define FWCMD_H2C_PORT_INIT_DTIM_PRD_MSK 0xff
+#define FWCMD_H2C_PORT_INIT_MBSSID_NUM_SH 16
+#define FWCMD_H2C_PORT_INIT_MBSSID_NUM_MSK 0xff
+#define FWCMD_H2C_PORT_INIT_BSS_COLOR_SH 24
+#define FWCMD_H2C_PORT_INIT_BSS_COLOR_MSK 0xff
+#define FWCMD_H2C_PORT_INIT_BCN_ITVL_SH 0
+#define FWCMD_H2C_PORT_INIT_BCN_ITVL_MSK 0xffff
+#define FWCMD_H2C_PORT_INIT_HIQ_WND_SH 16
+#define FWCMD_H2C_PORT_INIT_HIQ_WND_MSK 0xff
+#define FWCMD_H2C_PORT_CFG_BAND BIT(0)
+#define FWCMD_H2C_PORT_CFG_PORT_SH 1
+#define FWCMD_H2C_PORT_CFG_PORT_MSK 0x7
+#define FWCMD_H2C_PORT_CFG_MBSSID_IDX_SH 8
+#define FWCMD_H2C_PORT_CFG_MBSSID_IDX_MSK 0xff
+#define FWCMD_H2C_PORT_CFG_TYPE_SH 16
+#define FWCMD_H2C_PORT_CFG_TYPE_MSK 0xff
+#define FWCMD_H2C_PORT_CFG_VAL_SH 0
+#define FWCMD_H2C_PORT_CFG_VAL_MSK 0xffffffff
+#define FWCMD_H2C_MLOTBL_CFG_MACID_SH 0
+#define FWCMD_H2C_MLOTBL_CFG_MACID_MSK 0xffff
+#define FWCMD_H2C_MLOTBL_CFG_OPERATION BIT(16)
+#define FWCMD_H2C_MLOTBL_CFG_GET_DEF_VAL BIT(17)
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B0_SH 0
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B0_SH 4
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B1_SH 8
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B1_SH 12
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B0_SH 16
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B0_SH 20
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B1_SH 24
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B1_SH 28
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B0_SH 0
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B0_SH 4
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B1_SH 8
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B1_SH 12
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B0_SH 16
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B0_SH 20
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B0_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B1_SH 24
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B1_SH 28
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B1_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_BA_LIMIT_SH 0
+#define FWCMD_H2C_MLOTBL_CFG_BE_BA_LIMIT_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_BA_LIMIT_SH 4
+#define FWCMD_H2C_MLOTBL_CFG_BK_BA_LIMIT_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_BA_LIMIT_SH 8
+#define FWCMD_H2C_MLOTBL_CFG_VI_BA_LIMIT_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_BA_LIMIT_SH 12
+#define FWCMD_H2C_MLOTBL_CFG_VO_BA_LIMIT_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_FIX_TO_B0 BIT(16)
+#define FWCMD_H2C_MLOTBL_CFG_FIX_TO_B1 BIT(17)
+#define FWCMD_H2C_MLOTBL_CFG_WAIT2C_EMPTY BIT(18)
+#define FWCMD_H2C_MLOTBL_CFG_B0_CHK BIT(20)
+#define FWCMD_H2C_MLOTBL_CFG_B1_CHK BIT(21)
+#define FWCMD_H2C_MLOTBL_CFG_EXT0_CHK BIT(22)
+#define FWCMD_H2C_MLOTBL_CFG_EXT1_CHK BIT(23)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_BE BIT(24)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_BK BIT(25)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_VI BIT(26)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_VO BIT(27)
+#define FWCMD_H2C_MLOTBL_CFG_PRI_BAND BIT(28)
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B0_MASK_SH 0
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B0_MASK_SH 4
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B1_MASK_SH 8
+#define FWCMD_H2C_MLOTBL_CFG_BE_MIN_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B1_MASK_SH 12
+#define FWCMD_H2C_MLOTBL_CFG_BE_MAX_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B0_MASK_SH 16
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B0_MASK_SH 20
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B1_MASK_SH 24
+#define FWCMD_H2C_MLOTBL_CFG_BK_MIN_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B1_MASK_SH 28
+#define FWCMD_H2C_MLOTBL_CFG_BK_MAX_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B0_MASK_SH 0
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B0_MASK_SH 4
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B1_MASK_SH 8
+#define FWCMD_H2C_MLOTBL_CFG_VI_MIN_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B1_MASK_SH 12
+#define FWCMD_H2C_MLOTBL_CFG_VI_MAX_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B0_MASK_SH 16
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B0_MASK_SH 20
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B0_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B1_MASK_SH 24
+#define FWCMD_H2C_MLOTBL_CFG_VO_MIN_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B1_MASK_SH 28
+#define FWCMD_H2C_MLOTBL_CFG_VO_MAX_B1_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BE_BA_LIMIT_MASK_SH 0
+#define FWCMD_H2C_MLOTBL_CFG_BE_BA_LIMIT_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_BK_BA_LIMIT_MASK_SH 4
+#define FWCMD_H2C_MLOTBL_CFG_BK_BA_LIMIT_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VI_BA_LIMIT_MASK_SH 8
+#define FWCMD_H2C_MLOTBL_CFG_VI_BA_LIMIT_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_VO_BA_LIMIT_MASK_SH 12
+#define FWCMD_H2C_MLOTBL_CFG_VO_BA_LIMIT_MASK_MSK 0xf
+#define FWCMD_H2C_MLOTBL_CFG_FIX_TO_B0_MASK BIT(16)
+#define FWCMD_H2C_MLOTBL_CFG_FIX_TO_B1_MASK BIT(17)
+#define FWCMD_H2C_MLOTBL_CFG_WAIT2C_EMPTY_MASK BIT(18)
+#define FWCMD_H2C_MLOTBL_CFG_B0_CHK_MASK BIT(20)
+#define FWCMD_H2C_MLOTBL_CFG_B1_CHK_MASK BIT(21)
+#define FWCMD_H2C_MLOTBL_CFG_EXT0_CHK_MASK BIT(22)
+#define FWCMD_H2C_MLOTBL_CFG_EXT1_CHK_MASK BIT(23)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_BE_MASK BIT(24)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_BK_MASK BIT(25)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_VI_MASK BIT(26)
+#define FWCMD_H2C_MLOTBL_CFG_WD_DROP_VO_MASK BIT(27)
+#define FWCMD_H2C_MLOTBL_CFG_PRI_BAND_MASK BIT(28)
+#define FWCMD_H2C_MLO_STAFUNC_CFG_MACID_SH 0
+#define FWCMD_H2C_MLO_STAFUNC_CFG_MACID_MSK 0xff
+#define FWCMD_H2C_MLO_STAFUNC_CFG_CFG_EMLSR BIT(8)
+#define FWCMD_H2C_MLO_STAFUNC_CFG_EMLSR_EN BIT(9)
+#define FWCMD_H2C_MLO_STAFUNC_CFG_MACID_EXT_SH 10
+#define FWCMD_H2C_MLO_STAFUNC_CFG_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_MACID_NUM_SH 0
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_MACID_NUM_MSK 0xffff
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_MACID_X_SH 0
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_MACID_X_MSK 0xffff
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_TID_TO_LINK_MAP_X_SH 16
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_TID_TO_LINK_MAP_X_MSK 0xff
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_MACID_Y_SH 0
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_MACID_Y_MSK 0xffff
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_TID_TO_LINK_MAP_Y_SH 16
+#define FWCMD_H2C_MLO_TID_TO_LINK_MAP_TID_TO_LINK_MAP_Y_MSK 0xff
+#define FWCMD_H2C_MLO_DM_CFG_ENABLE BIT(0)
+#define FWCMD_H2C_MLO_DM_CFG_RELINK_SH 1
+#define FWCMD_H2C_MLO_DM_CFG_RELINK_MSK 0x3
+#define FWCMD_H2C_MLO_DM_CFG_INTERVAL_SH 3
+#define FWCMD_H2C_MLO_DM_CFG_INTERVAL_MSK 0x1f
+#define FWCMD_H2C_MLO_DM_CFG_STATUS BIT(8)
+#define FWCMD_H2C_MLO_DM_CFG_ENABLE_MASK BIT(0)
+#define FWCMD_H2C_MLO_DM_CFG_RELINK_MASK_SH 1
+#define FWCMD_H2C_MLO_DM_CFG_RELINK_MASK_MSK 0x3
+#define FWCMD_H2C_MLO_DM_CFG_INTERVAL_MASK_SH 3
+#define FWCMD_H2C_MLO_DM_CFG_INTERVAL_MASK_MSK 0x1f
+#define FWCMD_H2C_MLO_DM_CFG_STATUS_MASK BIT(8)
+#define FWCMD_H2C_EMLSR_STA_CFG_MACID_SH 0
+#define FWCMD_H2C_EMLSR_STA_CFG_MACID_MSK 0xffff
+#define FWCMD_H2C_EMLSR_STA_CFG_ENABLE BIT(16)
+#define FWCMD_H2C_EMLSR_STA_CFG_PAUSE_SH 17
+#define FWCMD_H2C_EMLSR_STA_CFG_PAUSE_MSK 0x3
+#define FWCMD_H2C_EMLSR_STA_CFG_NO_ACT BIT(19)
+#define FWCMD_H2C_EMLSR_STA_CFG_PKTID_SH 20
+#define FWCMD_H2C_EMLSR_STA_CFG_PKTID_MSK 0xfff
+#define FWCMD_H2C_EMLSR_STA_CFG_TIMEOUT_SH 0
+#define FWCMD_H2C_EMLSR_STA_CFG_TIMEOUT_MSK 0xf
+#define FWCMD_H2C_EMLSR_STA_CFG_CTRL_SH 16
+#define FWCMD_H2C_EMLSR_STA_CFG_CTRL_MSK 0xf
+#define FWCMD_H2C_EMLSR_STA_CFG_SEC_TYPE_SH 20
+#define FWCMD_H2C_EMLSR_STA_CFG_SEC_TYPE_MSK 0xf
+#define FWCMD_H2C_EMLSR_STA_CFG_HW_SEC_IV BIT(24)
+#define FWCMD_H2C_EMLSR_STA_CFG_SEC_IV_L_SH 16
+#define FWCMD_H2C_EMLSR_STA_CFG_SEC_IV_L_MSK 0xffff
+#define FWCMD_H2C_EMLSR_STA_CFG_SEC_IV_H_SH 0
+#define FWCMD_H2C_EMLSR_STA_CFG_SEC_IV_H_MSK 0xffffffff
+#define FWCMD_H2C_MCMLO_RELINK_DROP_MACID_SH 0
+#define FWCMD_H2C_MCMLO_RELINK_DROP_MACID_MSK 0xffff
+#define FWCMD_H2C_MCMLO_RELINK_DROP_AC_BMP_SH 16
+#define FWCMD_H2C_MCMLO_RELINK_DROP_AC_BMP_MSK 0xf
+#define FWCMD_H2C_MCMLO_RELINK_DROP_BAND BIT(20)
+#define FWCMD_H2C_MCMLO_RELINK_DROP_OP BIT(21)
+#define FWCMD_H2C_MCMLO_SN_SYNC_OP_SH 0
+#define FWCMD_H2C_MCMLO_SN_SYNC_OP_MSK 0x3
+#define FWCMD_H2C_MCMLO_SN_SYNC_SRC_LINK_IDX_SH 2
+#define FWCMD_H2C_MCMLO_SN_SYNC_SRC_LINK_IDX_MSK 0x3
+#define FWCMD_H2C_MCMLO_SN_SYNC_MACID_START_SH 0
+#define FWCMD_H2C_MCMLO_SN_SYNC_MACID_START_MSK 0xffff
+#define FWCMD_H2C_MCMLO_SN_SYNC_MACID_NUM_SH 16
+#define FWCMD_H2C_MCMLO_SN_SYNC_MACID_NUM_MSK 0xffff
+#define FWCMD_H2C_MLO_RELINK_MACID_SH 0
+#define FWCMD_H2C_MLO_RELINK_MACID_MSK 0xffff
+#define FWCMD_H2C_MLO_RELINK_AC_BMP_SH 16
+#define FWCMD_H2C_MLO_RELINK_AC_BMP_MSK 0xf
+#define FWCMD_H2C_MLO_RELINK_BAND BIT(20)
+#define FWCMD_H2C_MLO_LINK_CFG_MACID_SH 0
+#define FWCMD_H2C_MLO_LINK_CFG_MACID_MSK 0xffff
+#define FWCMD_H2C_MLO_LINK_CFG_OPTION_SH 16
+#define FWCMD_H2C_MLO_LINK_CFG_OPTION_MSK 0xf
+#define FWCMD_H2C_MLO_DM_DBG_MAIN_MACID_SH 0
+#define FWCMD_H2C_MLO_DM_DBG_MAIN_MACID_MSK 0xffff
+#define FWCMD_H2C_MLO_DM_DBG_MLO_DM_DBG_ID_SH 16
+#define FWCMD_H2C_MLO_DM_DBG_MLO_DM_DBG_ID_MSK 0xf
+#define FWCMD_H2C_MLO_DM_DBG_ACTION_SH 20
+#define FWCMD_H2C_MLO_DM_DBG_ACTION_MSK 0xf
+#define FWCMD_H2C_PROXY_PROXY_EN BIT(0)
+#define FWCMD_H2C_PROXY_ARP_RSP BIT(1)
+#define FWCMD_H2C_PROXY_NS_RSP BIT(2)
+#define FWCMD_H2C_PROXY_ICMP_V4_RSP BIT(3)
+#define FWCMD_H2C_PROXY_ICMP_V6_RSP BIT(4)
+#define FWCMD_H2C_PROXY_NETBIOS_RSP BIT(5)
+#define FWCMD_H2C_PROXY_LLMNR_V4_RSP BIT(6)
+#define FWCMD_H2C_PROXY_LLMNR_V6_RSP BIT(7)
+#define FWCMD_H2C_PROXY_SNMP_V4_RSP BIT(8)
+#define FWCMD_H2C_PROXY_SNMP_V6_RSP BIT(9)
+#define FWCMD_H2C_PROXY_SNMP_V4_WAKE BIT(10)
+#define FWCMD_H2C_PROXY_SNMP_V6_WAKE BIT(11)
+#define FWCMD_H2C_PROXY_SSDP_V4_WAKE BIT(12)
+#define FWCMD_H2C_PROXY_SSDP_V6_WAKE BIT(13)
+#define FWCMD_H2C_PROXY_WSD_V4_WAKE BIT(14)
+#define FWCMD_H2C_PROXY_WSD_V6_WAKE BIT(15)
+#define FWCMD_H2C_PROXY_SLP_V4_WAKE BIT(16)
+#define FWCMD_H2C_PROXY_SLP_V6_WAKE BIT(17)
+#define FWCMD_H2C_PROXY_MDNS_V4_RSP BIT(18)
+#define FWCMD_H2C_PROXY_MDNS_V6_RSP BIT(19)
+#define FWCMD_H2C_PROXY_TARGET_MAC_WAKE BIT(20)
+#define FWCMD_H2C_PROXY_LLTD_WAKE BIT(21)
+#define FWCMD_H2C_PROXY_MDNS_V4_WAKE BIT(22)
+#define FWCMD_H2C_PROXY_MDNS_V6_WAKE BIT(23)
+#define FWCMD_H2C_PROXY_V4ADD0_SH 0
+#define FWCMD_H2C_PROXY_V4ADD0_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V4ADD1_SH 0
+#define FWCMD_H2C_PROXY_V4ADD1_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V4ADD2_SH 0
+#define FWCMD_H2C_PROXY_V4ADD2_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V4ADD3_SH 0
+#define FWCMD_H2C_PROXY_V4ADD3_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD0_0_SH 0
+#define FWCMD_H2C_PROXY_V6ADD0_0_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD0_1_SH 0
+#define FWCMD_H2C_PROXY_V6ADD0_1_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD0_2_SH 0
+#define FWCMD_H2C_PROXY_V6ADD0_2_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD0_3_SH 0
+#define FWCMD_H2C_PROXY_V6ADD0_3_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD1_0_SH 0
+#define FWCMD_H2C_PROXY_V6ADD1_0_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD1_1_SH 0
+#define FWCMD_H2C_PROXY_V6ADD1_1_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD1_2_SH 0
+#define FWCMD_H2C_PROXY_V6ADD1_2_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD1_3_SH 0
+#define FWCMD_H2C_PROXY_V6ADD1_3_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD2_0_SH 0
+#define FWCMD_H2C_PROXY_V6ADD2_0_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD2_1_SH 0
+#define FWCMD_H2C_PROXY_V6ADD2_1_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD2_2_SH 0
+#define FWCMD_H2C_PROXY_V6ADD2_2_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD2_3_SH 0
+#define FWCMD_H2C_PROXY_V6ADD2_3_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD3_0_SH 0
+#define FWCMD_H2C_PROXY_V6ADD3_0_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD3_1_SH 0
+#define FWCMD_H2C_PROXY_V6ADD3_1_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD3_2_SH 0
+#define FWCMD_H2C_PROXY_V6ADD3_2_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_V6ADD3_3_SH 0
+#define FWCMD_H2C_PROXY_V6ADD3_3_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_V4PKTID_SH 0
+#define FWCMD_H2C_MDNS_V4PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_V6PKTID_SH 8
+#define FWCMD_H2C_MDNS_V6PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_NUMSUPPSERV_SH 16
+#define FWCMD_H2C_MDNS_NUMSUPPSERV_MSK 0xff
+#define FWCMD_H2C_MDNS_NUMMACHNAME_SH 24
+#define FWCMD_H2C_MDNS_NUMMACHNAME_MSK 0xff
+#define FWCMD_H2C_MDNS_MACID_SH 0
+#define FWCMD_H2C_MDNS_MACID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV0PKTID_SH 8
+#define FWCMD_H2C_MDNS_SERV0PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV1PKTID_SH 16
+#define FWCMD_H2C_MDNS_SERV1PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV2PKTID_SH 24
+#define FWCMD_H2C_MDNS_SERV2PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV3PKTID_SH 0
+#define FWCMD_H2C_MDNS_SERV3PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV4PKTID_SH 8
+#define FWCMD_H2C_MDNS_SERV4PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV5PKTID_SH 16
+#define FWCMD_H2C_MDNS_SERV5PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV6PKTID_SH 24
+#define FWCMD_H2C_MDNS_SERV6PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV7PKTID_SH 0
+#define FWCMD_H2C_MDNS_SERV7PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV8PKTID_SH 8
+#define FWCMD_H2C_MDNS_SERV8PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_SERV9PKTID_SH 16
+#define FWCMD_H2C_MDNS_SERV9PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_MACID_EXT_SH 24
+#define FWCMD_H2C_MDNS_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_MDNS_MACHINE0_LEN_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_LEN_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_0_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_0_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_1_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_1_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_2_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_2_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_3_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_3_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_4_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_4_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_5_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_5_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_6_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_6_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_7_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_7_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_8_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_8_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_9_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_9_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_10_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_10_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_11_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_11_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_12_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_12_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_13_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_13_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_14_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_14_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE0_15_SH 0
+#define FWCMD_H2C_MDNS_MACHINE0_15_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_LEN_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_LEN_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_0_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_0_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_1_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_1_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_2_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_2_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_3_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_3_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_4_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_4_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_5_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_5_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_6_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_6_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_7_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_7_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_8_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_8_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_9_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_9_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_10_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_10_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_11_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_11_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_12_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_12_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_13_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_13_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_14_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_14_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE1_15_SH 0
+#define FWCMD_H2C_MDNS_MACHINE1_15_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_LEN_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_LEN_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_0_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_0_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_1_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_1_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_2_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_2_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_3_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_3_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_4_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_4_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_5_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_5_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_6_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_6_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_7_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_7_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_8_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_8_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_9_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_9_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_10_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_10_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_11_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_11_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_12_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_12_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_13_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_13_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_14_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_14_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_MACHINE2_15_SH 0
+#define FWCMD_H2C_MDNS_MACHINE2_15_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_A0_SH 0
+#define FWCMD_H2C_MDNS_A0_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_A1_SH 0
+#define FWCMD_H2C_MDNS_A1_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_A2_SH 0
+#define FWCMD_H2C_MDNS_A2_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_A3_SH 0
+#define FWCMD_H2C_MDNS_A3_MSK 0xffff
+#define FWCMD_H2C_MDNS_AAAA0_SH 16
+#define FWCMD_H2C_MDNS_AAAA0_MSK 0xffff
+#define FWCMD_H2C_MDNS_AAAA1_SH 0
+#define FWCMD_H2C_MDNS_AAAA1_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_AAAA2_SH 0
+#define FWCMD_H2C_MDNS_AAAA2_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_AAAA3_SH 0
+#define FWCMD_H2C_MDNS_AAAA3_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_AAAA4_SH 0
+#define FWCMD_H2C_MDNS_AAAA4_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_AAAA5_SH 0
+#define FWCMD_H2C_MDNS_AAAA5_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_AAAA7_SH 0
+#define FWCMD_H2C_MDNS_AAAA7_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR0_SH 0
+#define FWCMD_H2C_MDNS_PTR0_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR1_SH 0
+#define FWCMD_H2C_MDNS_PTR1_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR2_SH 0
+#define FWCMD_H2C_MDNS_PTR2_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR3_SH 0
+#define FWCMD_H2C_MDNS_PTR3_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR4_SH 0
+#define FWCMD_H2C_MDNS_PTR4_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR5_SH 0
+#define FWCMD_H2C_MDNS_PTR5_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR6_SH 0
+#define FWCMD_H2C_MDNS_PTR6_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR7_SH 0
+#define FWCMD_H2C_MDNS_PTR7_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR8_SH 0
+#define FWCMD_H2C_MDNS_PTR8_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR9_SH 0
+#define FWCMD_H2C_MDNS_PTR9_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR10_SH 0
+#define FWCMD_H2C_MDNS_PTR10_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR11_SH 0
+#define FWCMD_H2C_MDNS_PTR11_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR12_SH 0
+#define FWCMD_H2C_MDNS_PTR12_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR13_SH 0
+#define FWCMD_H2C_MDNS_PTR13_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR14_SH 0
+#define FWCMD_H2C_MDNS_PTR14_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR15_SH 0
+#define FWCMD_H2C_MDNS_PTR15_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR16_SH 0
+#define FWCMD_H2C_MDNS_PTR16_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR17_SH 0
+#define FWCMD_H2C_MDNS_PTR17_MSK 0xffffffff
+#define FWCMD_H2C_MDNS_PTR18_SH 0
+#define FWCMD_H2C_MDNS_PTR18_MSK 0xffffffff
+#define FWCMD_H2C_PTCL_PATTERN_MACID_SH 0
+#define FWCMD_H2C_PTCL_PATTERN_MACID_MSK 0xff
+#define FWCMD_H2C_PTCL_PATTERN_PTCL_SH 8
+#define FWCMD_H2C_PTCL_PATTERN_PTCL_MSK 0x7
+#define FWCMD_H2C_PTCL_PATTERN_NUM_PATTERN_SH 11
+#define FWCMD_H2C_PTCL_PATTERN_NUM_PATTERN_MSK 0x7
+#define FWCMD_H2C_PTCL_PATTERN_MACID_EXT_SH 14
+#define FWCMD_H2C_PTCL_PATTERN_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_SNMP_IPV4_PKTID_SH 0
+#define FWCMD_H2C_SNMP_IPV4_PKTID_MSK 0xff
+#define FWCMD_H2C_SNMP_IPV6_PKTID_SH 8
+#define FWCMD_H2C_SNMP_IPV6_PKTID_MSK 0xff
+#define FWCMD_H2C_SNMP_COMMUNITY0_LEN_SH 16
+#define FWCMD_H2C_SNMP_COMMUNITY0_LEN_MSK 0xff
+#define FWCMD_H2C_SNMP_COMMUNITY1_LEN_SH 24
+#define FWCMD_H2C_SNMP_COMMUNITY1_LEN_MSK 0xff
+#define FWCMD_H2C_SNMP_COMMUNITY0_0_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY0_0_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_COMMUNITY0_1_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY0_1_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_COMMUNITY0_2_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY0_2_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_COMMUNITY0_3_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY0_3_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_COMMUNITY1_0_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY1_0_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_COMMUNITY1_1_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY1_1_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_COMMUNITY1_2_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY1_2_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_COMMUNITY1_3_SH 0
+#define FWCMD_H2C_SNMP_COMMUNITY1_3_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_DEVICE_STATUS_SH 0
+#define FWCMD_H2C_SNMP_DEVICE_STATUS_MSK 0xff
+#define FWCMD_H2C_SNMP_PRINTER_STATUS_SH 8
+#define FWCMD_H2C_SNMP_PRINTER_STATUS_MSK 0xff
+#define FWCMD_H2C_SNMP_PRINTER_ERR_STATE_SH 16
+#define FWCMD_H2C_SNMP_PRINTER_ERR_STATE_MSK 0xffff
+#define FWCMD_H2C_SNMP_MACID_SH 0
+#define FWCMD_H2C_SNMP_MACID_MSK 0xff
+#define FWCMD_H2C_SNMP_SYS_DESCR_LEN_SH 8
+#define FWCMD_H2C_SNMP_SYS_DESCR_LEN_MSK 0xff
+#define FWCMD_H2C_SNMP_OBJ_ID_LEN_SH 16
+#define FWCMD_H2C_SNMP_OBJ_ID_LEN_MSK 0xff
+#define FWCMD_H2C_SNMP_NUM_ENTERPRISE_MIB_SH 24
+#define FWCMD_H2C_SNMP_NUM_ENTERPRISE_MIB_MSK 0xff
+#define FWCMD_H2C_SNMP_SYS_DESCR0_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR0_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_SYS_DESCR1_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR1_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_SYS_DESCR2_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR2_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_SYS_DESCR3_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR3_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_SYS_DESCR4_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR4_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_SYS_DESCR5_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR5_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_SYS_DESCR6_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR6_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_SYS_DESCR7_SH 0
+#define FWCMD_H2C_SNMP_SYS_DESCR7_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_ENTERPRISE_ID_LEN_SH 0
+#define FWCMD_H2C_SNMP_ENTERPRISE_ID_LEN_MSK 0xff
+#define FWCMD_H2C_SNMP_ENTERPRISE_ID0_SH 8
+#define FWCMD_H2C_SNMP_ENTERPRISE_ID0_MSK 0xffffff
+#define FWCMD_H2C_SNMP_OBJ_ID0_SH 0
+#define FWCMD_H2C_SNMP_OBJ_ID0_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_OBJ_ID1_SH 0
+#define FWCMD_H2C_SNMP_OBJ_ID1_MSK 0xffffffff
+#define FWCMD_H2C_SNMP_MACID_EXT_SH 0
+#define FWCMD_H2C_SNMP_MACID_EXT_MSK 0xff
+#define FWCMD_H2C_LLMNR_MACID_SH 0
+#define FWCMD_H2C_LLMNR_MACID_MSK 0xffff
+#define FWCMD_H2C_LLMNR_MACHINE0_LEN_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_LEN_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_0_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_0_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_1_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_1_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_2_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_2_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_3_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_3_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_4_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_4_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_5_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_5_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_6_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_6_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_7_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_7_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_8_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_8_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_9_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_9_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_10_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_10_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_11_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_11_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_12_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_12_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_13_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_13_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_14_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_14_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE0_15_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE0_15_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_LEN_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_LEN_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_0_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_0_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_1_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_1_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_2_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_2_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_3_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_3_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_4_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_4_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_5_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_5_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_6_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_6_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_7_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_7_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_8_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_8_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_9_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_9_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_10_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_10_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_11_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_11_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_12_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_12_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_13_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_13_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_14_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_14_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE1_15_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE1_15_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_LEN_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_LEN_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_0_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_0_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_1_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_1_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_2_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_2_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_3_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_3_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_4_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_4_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_5_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_5_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_6_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_6_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_7_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_7_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_8_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_8_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_9_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_9_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_10_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_10_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_11_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_11_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_12_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_12_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_13_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_13_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_14_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_14_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_MACHINE2_15_SH 0
+#define FWCMD_H2C_LLMNR_MACHINE2_15_MSK 0xffffffff
+#define FWCMD_H2C_LLMNR_V4PKTID_SH 0
+#define FWCMD_H2C_LLMNR_V4PKTID_MSK 0xff
+#define FWCMD_H2C_LLMNR_V6PKTID_SH 8
+#define FWCMD_H2C_LLMNR_V6PKTID_MSK 0xff
+#define FWCMD_H2C_LLMNR_NUM_MACHINE_NAMES_SH 16
+#define FWCMD_H2C_LLMNR_NUM_MACHINE_NAMES_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_V4PKTID_SH 0
+#define FWCMD_H2C_MDNS_OFLD_V4PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_V6PKTID_SH 8
+#define FWCMD_H2C_MDNS_OFLD_V6PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_MDNS_PROTOCOL_DATA_PKTID_SH 16
+#define FWCMD_H2C_MDNS_OFLD_MDNS_PROTOCOL_DATA_PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_LIST_PKTID_SH 24
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_LIST_PKTID_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_MDNS_OFFLOAD_EN_SH 0
+#define FWCMD_H2C_MDNS_OFLD_MDNS_OFFLOAD_EN_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_OFFLOAD_STATE_SH 8
+#define FWCMD_H2C_MDNS_OFLD_OFFLOAD_STATE_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_BEHAVIOR_SH 16
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_BEHAVIOR_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_MACID_SH 24
+#define FWCMD_H2C_MDNS_OFLD_MACID_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_0_SH 0
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_0_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_1_SH 8
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_1_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_2_SH 16
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_2_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_3_SH 24
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_3_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_4_SH 0
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_4_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_5_SH 8
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_5_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_6_SH 16
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_6_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_7_SH 24
+#define FWCMD_H2C_MDNS_OFLD_DATA_PKTID_7_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_0_SH 0
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_0_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_1_SH 8
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_1_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_2_SH 16
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_2_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_3_SH 24
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_3_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_4_SH 0
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_4_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_5_SH 8
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_5_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_6_SH 16
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_6_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_7_SH 24
+#define FWCMD_H2C_MDNS_OFLD_PASSTHRU_NAME_PKTID_7_MSK 0xff
+#define FWCMD_H2C_MDNS_OFLD_MACID_EXT_SH 0
+#define FWCMD_H2C_MDNS_OFLD_MACID_EXT_MSK 0xffff
+#define FWCMD_H2C_MDNS_OFLD_TYPE_SH 16
+#define FWCMD_H2C_MDNS_OFLD_TYPE_MSK 0xff
+#define FWCMD_H2C_APF_SET_MACID_SH 0
+#define FWCMD_H2C_APF_SET_MACID_MSK 0xffff
+#define FWCMD_H2C_APF_SET_MACPKTID_SH 0
+#define FWCMD_H2C_APF_SET_MACPKTID_MSK 0xff
+#define FWCMD_H2C_APF_SET_APF_EN_SH 0
+#define FWCMD_H2C_APF_SET_APF_EN_MSK 0xff
+#define FWCMD_H2C_APF_SET_PACKET_MASK_SH 16
+#define FWCMD_H2C_APF_SET_PACKET_MASK_MSK 0xffff
+#define FWCMD_H2C_APF_SET_VERSION1_SH 0
+#define FWCMD_H2C_APF_SET_VERSION1_MSK 0xffffffff
+#define FWCMD_H2C_APF_SET_VERSION2_SH 0
+#define FWCMD_H2C_APF_SET_VERSION2_MSK 0xffffffff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID0_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID0_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID1_SH 8
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID1_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID2_SH 16
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID2_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID3_SH 24
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID3_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID4_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID4_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID5_SH 8
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID5_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID6_SH 16
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID6_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID7_SH 24
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID7_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID8_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID8_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID9_SH 8
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID9_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID10_SH 16
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID10_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID11_SH 24
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID11_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID12_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID12_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID13_SH 8
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID13_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID14_SH 16
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID14_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID15_SH 24
+#define FWCMD_H2C_APF_SET_PROGRAM_PKTID15_MSK 0xff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN0_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN0_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN0_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN0_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN1_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN1_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN1_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN1_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN2_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN2_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN2_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN2_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN3_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN3_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN3_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN3_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN4_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN4_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN4_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN4_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN5_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN5_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN5_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN5_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN6_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN6_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN6_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN6_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN7_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN7_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN7_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN7_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN8_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN8_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN8_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN8_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN9_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN9_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN9_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN9_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN10_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN10_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN10_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN10_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN11_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN11_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN11_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN11_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN12_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN12_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN12_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN12_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN13_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN13_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN13_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN13_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN14_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN14_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN14_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN14_MSK 0xffff
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN15_SH 0
+#define FWCMD_H2C_APF_SET_PROGRAM_LEN15_MSK 0xffff
+#define FWCMD_H2C_APF_SET_DATA_LEN15_SH 16
+#define FWCMD_H2C_APF_SET_DATA_LEN15_MSK 0xffff
+#define FWCMD_H2C_APF_GET_PROGRAM_NUM_SH 0
+#define FWCMD_H2C_APF_GET_PROGRAM_NUM_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_MODULE_ID_SH 0
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_MODULE_ID_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_PRIORITY_SH 8
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_PRIORITY_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_OPTIONS_SH 16
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_OPTIONS_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_FAW_EN BIT(24)
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_START_TIME_SH 0
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_START_TIME_MSK 0xffffffff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_DURATION_SH 0
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_DURATION_MSK 0xffffffff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_PERIOD_SH 0
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_PERIOD_MSK 0xffffffff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_TSF_IDX_SH 0
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_TSF_IDX_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_CHANNEL_SH 8
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_CHANNEL_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_BW_SH 16
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_BW_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_PRIMARY_CH_SH 24
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_PRIMARY_CH_MSK 0xff
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_CH_BAND_TYPE_SH 0
+#define FWCMD_H2C_ACT_SCHEDULE_REQ_CH_BAND_TYPE_MSK 0x3
+#define FWCMD_H2C_BCN_REQ_MODULE_ID_SH 0
+#define FWCMD_H2C_BCN_REQ_MODULE_ID_MSK 0xff
+#define FWCMD_H2C_BCN_REQ_BCN_INTVL_MS_SH 16
+#define FWCMD_H2C_BCN_REQ_BCN_INTVL_MS_MSK 0xff
+#define FWCMD_H2C_BCN_REQ_PRIORITY_SH 24
+#define FWCMD_H2C_BCN_REQ_PRIORITY_MSK 0xff
+#define FWCMD_H2C_BCN_REQ_BCN_OFFSET_US_SH 0
+#define FWCMD_H2C_BCN_REQ_BCN_OFFSET_US_MSK 0xffffffff
+#define FWCMD_H2C_BCN_REQ_CUR_TBTT_SH 0
+#define FWCMD_H2C_BCN_REQ_CUR_TBTT_MSK 0xffffffff
+#define FWCMD_H2C_BCN_REQ_CUR_TBTT_FR_SH 0
+#define FWCMD_H2C_BCN_REQ_CUR_TBTT_FR_MSK 0xffffffff
+#define FWCMD_H2C_BCN_REQ_PROHIBIT_BEFORE_MS_SH 0
+#define FWCMD_H2C_BCN_REQ_PROHIBIT_BEFORE_MS_MSK 0xff
+#define FWCMD_H2C_BCN_REQ_PROHIBIT_AFTER_MS_SH 8
+#define FWCMD_H2C_BCN_REQ_PROHIBIT_AFTER_MS_MSK 0xff
+#define FWCMD_H2C_BCN_REQ_PORT_IDX_SH 16
+#define FWCMD_H2C_BCN_REQ_PORT_IDX_MSK 0xff
+#define FWCMD_H2C_BCN_REQ_OPTIONS_SH 24
+#define FWCMD_H2C_BCN_REQ_OPTIONS_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_PORT_IDX_SH 0
+#define FWCMD_H2C_NAN_FUNC_CTRL_PORT_IDX_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_MASTER_PREF_SH 16
+#define FWCMD_H2C_NAN_FUNC_CTRL_MASTER_PREF_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_RANDOM_FACTOR_SH 24
+#define FWCMD_H2C_NAN_FUNC_CTRL_RANDOM_FACTOR_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_OP_CH_24G_SH 0
+#define FWCMD_H2C_NAN_FUNC_CTRL_OP_CH_24G_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_OP_CH_5G_SH 8
+#define FWCMD_H2C_NAN_FUNC_CTRL_OP_CH_5G_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_OPTIONS_SH 16
+#define FWCMD_H2C_NAN_FUNC_CTRL_OPTIONS_MSK 0xffff
+#define FWCMD_H2C_NAN_FUNC_CTRL_TIME_INDICATE_PERIOD_SH 0
+#define FWCMD_H2C_NAN_FUNC_CTRL_TIME_INDICATE_PERIOD_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID0_SH 8
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID0_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID1_SH 16
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID1_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID2_SH 24
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID2_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID3_SH 0
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID3_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID4_SH 8
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID4_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID5_SH 16
+#define FWCMD_H2C_NAN_FUNC_CTRL_NAN_CLUSTER_ID5_MSK 0xff
+#define FWCMD_H2C_NAN_FUNC_CTRL_PARA_OPTIONS_SH 0
+#define FWCMD_H2C_NAN_FUNC_CTRL_PARA_OPTIONS_MSK 0xffff
+#define FWCMD_H2C_NAN_FUNC_CTRL_MAC_ID_BCN_SH 0
+#define FWCMD_H2C_NAN_FUNC_CTRL_MAC_ID_BCN_MSK 0xffff
+#define FWCMD_H2C_NAN_FUNC_CTRL_MAC_ID_MGN_SH 16
+#define FWCMD_H2C_NAN_FUNC_CTRL_MAC_ID_MGN_MSK 0xffff
+#define FWCMD_H2C_NAN_DE_INFO_STATUS_SH 0
+#define FWCMD_H2C_NAN_DE_INFO_STATUS_MSK 0xff
+#define FWCMD_H2C_NAN_DE_INFO_LOC_BCAST_SDF_SH 8
+#define FWCMD_H2C_NAN_DE_INFO_LOC_BCAST_SDF_MSK 0xff
+#define FWCMD_H2C_NAN_JOIN_CLUSTER_IS_ALLOW_SH 0
+#define FWCMD_H2C_NAN_JOIN_CLUSTER_IS_ALLOW_MSK 0xff
+#define FWCMD_H2C_PAUSE_FAW_TX_ID_MAP_SH 0
+#define FWCMD_H2C_PAUSE_FAW_TX_ID_MAP_MSK 0xffffffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_MODULE_ID_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_MODULE_ID_MSK 0xff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_OPTION_SH 8
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_OPTION_MSK 0xff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_START_OFFSET_16TU_SH 16
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_START_OFFSET_16TU_MSK 0xffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_PERIOD_TU_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_PERIOD_TU_MSK 0xffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_BIT_DURATION_TU_SH 16
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_BIT_DURATION_TU_MSK 0xff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_LEN_SH 24
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_LEN_MSK 0xff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_0_3_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_0_3_MSK 0xffffffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_4_7_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_4_7_MSK 0xffffffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_8_11_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_8_11_MSK 0xffffffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_12_15_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_TIME_BITMAP_12_15_MSK 0xffffffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_CHANNEL_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_CHANNEL_MSK 0xff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_BW_SH 8
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_BW_MSK 0xff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_PRIMARY_CH_SH 16
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_PRIMARY_CH_MSK 0xff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_CH_BAND_TYPE_SH 24
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_CH_BAND_TYPE_MSK 0x3
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_MAC_ID_SH 0
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_MAC_ID_MSK 0xffff
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_PRIORITY_SH 16
+#define FWCMD_H2C_NAN_AVAIL_T_BITMAP_PRIORITY_MSK 0xff
+#define FWCMD_H2C_FW_CMDREQ_STR_SH 0
+#define FWCMD_H2C_FW_CMDREQ_STR_MSK 0xffffffff
+#define FWCMD_H2C_MRC_REQ_TSF_REQ_TSF_NUM_SH 0
+#define FWCMD_H2C_MRC_REQ_TSF_REQ_TSF_NUM_MSK 0xff
+#define FWCMD_H2C_MRC_REQ_TSF_BAND_SH 8
+#define FWCMD_H2C_MRC_REQ_TSF_BAND_MSK 0xf
+#define FWCMD_H2C_MRC_REQ_TSF_PORT_SH 12
+#define FWCMD_H2C_MRC_REQ_TSF_PORT_MSK 0xf
+#define FWCMD_H2C_ADD_MRC_SCH_IDX_SH 0
+#define FWCMD_H2C_ADD_MRC_SCH_IDX_MSK 0xf
+#define FWCMD_H2C_ADD_MRC_SCH_TYPE_SH 4
+#define FWCMD_H2C_ADD_MRC_SCH_TYPE_MSK 0xf
+#define FWCMD_H2C_ADD_MRC_SLOT_NUM_SH 8
+#define FWCMD_H2C_ADD_MRC_SLOT_NUM_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_IS_BTC_SCH BIT(16)
+#define FWCMD_H2C_ADD_MRC_IS_EMLSR BIT(17)
+#define FWCMD_H2C_ADD_MRC_DURATION_SH 0
+#define FWCMD_H2C_ADD_MRC_DURATION_MSK 0xffff
+#define FWCMD_H2C_ADD_MRC_COURTESY_EN BIT(17)
+#define FWCMD_H2C_ADD_MRC_ROLE_NUM_SH 24
+#define FWCMD_H2C_ADD_MRC_ROLE_NUM_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_COURTESY_PERIOD_SH 0
+#define FWCMD_H2C_ADD_MRC_COURTESY_PERIOD_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_COURTESY_TARGET_SH 8
+#define FWCMD_H2C_ADD_MRC_COURTESY_TARGET_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_MACID_SH 0
+#define FWCMD_H2C_ADD_MRC_MACID_MSK 0xffff
+#define FWCMD_H2C_ADD_MRC_ROLE_TYPE_SH 16
+#define FWCMD_H2C_ADD_MRC_ROLE_TYPE_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_IS_MASTER BIT(24)
+#define FWCMD_H2C_ADD_MRC_IS_ALT_ROLE BIT(25)
+#define FWCMD_H2C_ADD_MRC_TX_NULL_EN BIT(26)
+#define FWCMD_H2C_ADD_MRC_ROLE_ALT_EN BIT(27)
+#define FWCMD_H2C_ADD_MRC_CENTRAL_CH_SEG_SH 0
+#define FWCMD_H2C_ADD_MRC_CENTRAL_CH_SEG_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_PRI_CH_SH 8
+#define FWCMD_H2C_ADD_MRC_PRI_CH_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_BW_SH 16
+#define FWCMD_H2C_ADD_MRC_BW_MSK 0xf
+#define FWCMD_H2C_ADD_MRC_CH_BAND_TYPE_SH 20
+#define FWCMD_H2C_ADD_MRC_CH_BAND_TYPE_MSK 0x3
+#define FWCMD_H2C_ADD_MRC_RFK_BY_PASS BIT(22)
+#define FWCMD_H2C_ADD_MRC_IS_BTC BIT(23)
+#define FWCMD_H2C_ADD_MRC_NULL_EARLY_SH 24
+#define FWCMD_H2C_ADD_MRC_NULL_EARLY_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_ALT_PERIOD_SH 0
+#define FWCMD_H2C_ADD_MRC_ALT_PERIOD_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_ALT_ROLE_TYPE_SH 8
+#define FWCMD_H2C_ADD_MRC_ALT_ROLE_TYPE_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_ALT_ROLE_MACID_SH 16
+#define FWCMD_H2C_ADD_MRC_ALT_ROLE_MACID_MSK 0xff
+#define FWCMD_H2C_ADD_MRC_MACID_MAIN_BM_SH 0
+#define FWCMD_H2C_ADD_MRC_MACID_MAIN_BM_MSK 0xffffffff
+#define FWCMD_H2C_ADD_MRC_MACID_PAIRED_BM_SH 0
+#define FWCMD_H2C_ADD_MRC_MACID_PAIRED_BM_MSK 0xffffffff
+#define FWCMD_H2C_START_MRC_SCH_IDX_SH 0
+#define FWCMD_H2C_START_MRC_SCH_IDX_MSK 0xf
+#define FWCMD_H2C_START_MRC_OLD_SCH_IDX_SH 4
+#define FWCMD_H2C_START_MRC_OLD_SCH_IDX_MSK 0xf
+#define FWCMD_H2C_START_MRC_ACTION_SH 8
+#define FWCMD_H2C_START_MRC_ACTION_MSK 0xff
+#define FWCMD_H2C_START_MRC_START_TSF_LOW_SH 0
+#define FWCMD_H2C_START_MRC_START_TSF_LOW_MSK 0xffffffff
+#define FWCMD_H2C_START_MRC_START_TSF_HIGH_SH 0
+#define FWCMD_H2C_START_MRC_START_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_H2C_DEL_MRC_SCH_IDX_SH 0
+#define FWCMD_H2C_DEL_MRC_SCH_IDX_MSK 0xf
+#define FWCMD_H2C_DEL_MRC_DEL_ALL BIT(4)
+#define FWCMD_H2C_DEL_MRC_STOP_ONLY BIT(5)
+#define FWCMD_H2C_DEL_MRC_SPECIFIC_ROLE_EN BIT(6)
+#define FWCMD_H2C_DEL_MRC_STOP_SLOT_IDX_SH 8
+#define FWCMD_H2C_DEL_MRC_STOP_SLOT_IDX_MSK 0xff
+#define FWCMD_H2C_DEL_MRC_SPECIFIC_ROLE_MACID_SH 16
+#define FWCMD_H2C_DEL_MRC_SPECIFIC_ROLE_MACID_MSK 0xffff
+#define FWCMD_H2C_MRC_SYNC_SYNC_EN BIT(0)
+#define FWCMD_H2C_MRC_SYNC_SRC_PORT_SH 8
+#define FWCMD_H2C_MRC_SYNC_SRC_PORT_MSK 0xf
+#define FWCMD_H2C_MRC_SYNC_SRC_BAND_SH 12
+#define FWCMD_H2C_MRC_SYNC_SRC_BAND_MSK 0xf
+#define FWCMD_H2C_MRC_SYNC_DEST_PORT_SH 16
+#define FWCMD_H2C_MRC_SYNC_DEST_PORT_MSK 0xf
+#define FWCMD_H2C_MRC_SYNC_DEST_BAND_SH 20
+#define FWCMD_H2C_MRC_SYNC_DEST_BAND_MSK 0xf
+#define FWCMD_H2C_MRC_SYNC_OFFSET_SH 0
+#define FWCMD_H2C_MRC_SYNC_OFFSET_MSK 0xffff
+#define FWCMD_H2C_MRC_UPD_DURATION_SCH_IDX_SH 0
+#define FWCMD_H2C_MRC_UPD_DURATION_SCH_IDX_MSK 0xf
+#define FWCMD_H2C_MRC_UPD_DURATION_SLOT_NUM_SH 8
+#define FWCMD_H2C_MRC_UPD_DURATION_SLOT_NUM_MSK 0xff
+#define FWCMD_H2C_MRC_UPD_DURATION_IS_BTC_SCH BIT(16)
+#define FWCMD_H2C_MRC_UPD_DURATION_START_TSF_LOW_SH 0
+#define FWCMD_H2C_MRC_UPD_DURATION_START_TSF_LOW_MSK 0xffffffff
+#define FWCMD_H2C_MRC_UPD_DURATION_START_TSF_HIGH_SH 0
+#define FWCMD_H2C_MRC_UPD_DURATION_START_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_H2C_MRC_UPD_DURATION_SLOT_IDX_SH 0
+#define FWCMD_H2C_MRC_UPD_DURATION_SLOT_IDX_MSK 0xff
+#define FWCMD_H2C_MRC_UPD_DURATION_DURATION_SH 16
+#define FWCMD_H2C_MRC_UPD_DURATION_DURATION_MSK 0xffff
+#define FWCMD_H2C_MRC_UPD_BITMAP_SCH_IDX_SH 0
+#define FWCMD_H2C_MRC_UPD_BITMAP_SCH_IDX_MSK 0xf
+#define FWCMD_H2C_MRC_UPD_BITMAP_ACTION BIT(4)
+#define FWCMD_H2C_MRC_UPD_BITMAP_MACID_SH 16
+#define FWCMD_H2C_MRC_UPD_BITMAP_MACID_MSK 0xffff
+#define FWCMD_H2C_MRC_UPD_BITMAP_CLIENT_MACID_SH 0
+#define FWCMD_H2C_MRC_UPD_BITMAP_CLIENT_MACID_MSK 0xffff
+#define FWCMD_H2C_MRC_DBG_DUMP_CFG_FW_DBG_CFG_SH 0
+#define FWCMD_H2C_MRC_DBG_DUMP_CFG_FW_DBG_CFG_MSK 0xf
+#define FWCMD_H2C_MRC_DBG_DUMP_CFG_DBG_LVL_SH 6
+#define FWCMD_H2C_MRC_DBG_DUMP_CFG_DBG_LVL_MSK 0x3
+#define FWCMD_H2C_SENS_TBL_UPD_VALID BIT(0)
+#define FWCMD_H2C_SENS_TBL_UPD_RPT_EN BIT(1)
+#define FWCMD_H2C_SENS_TBL_UPD_RPT_DELAY BIT(2)
+#define FWCMD_H2C_SENS_TBL_UPD_RPT_TS BIT(3)
+#define FWCMD_H2C_SENS_TBL_UPD_SENS_ROLE_RX BIT(4)
+#define FWCMD_H2C_SENS_TBL_UPD_SENS_ROLE_TX BIT(5)
+#define FWCMD_H2C_SENS_TBL_UPD_INST_TYPE_SH 6
+#define FWCMD_H2C_SENS_TBL_UPD_INST_TYPE_MSK 0x3
+#define FWCMD_H2C_SENS_TBL_UPD_WIENER_COEF_TAU_SH 8
+#define FWCMD_H2C_SENS_TBL_UPD_WIENER_COEF_TAU_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_MACID_SH 12
+#define FWCMD_H2C_SENS_TBL_UPD_MACID_MSK 0x3ff
+#define FWCMD_H2C_SENS_TBL_UPD_MSR_SETUP_ID_SH 22
+#define FWCMD_H2C_SENS_TBL_UPD_MSR_SETUP_ID_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_N_RX_SH 25
+#define FWCMD_H2C_SENS_TBL_UPD_N_RX_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_I_NG BIT(28)
+#define FWCMD_H2C_SENS_TBL_UPD_N_B BIT(29)
+#define FWCMD_H2C_SENS_TBL_UPD_SR2SR_EN BIT(30)
+#define FWCMD_H2C_SENS_TBL_UPD_STBC_EN BIT(31)
+#define FWCMD_H2C_SENS_TBL_UPD_SMR_RATE_SH 0
+#define FWCMD_H2C_SENS_TBL_UPD_SMR_RATE_MSK 0xfff
+#define FWCMD_H2C_SENS_TBL_UPD_RPT_TO_HOST_EN BIT(12)
+#define FWCMD_H2C_SENS_TBL_UPD_HT_CTRL_EN BIT(13)
+#define FWCMD_H2C_SENS_TBL_UPD_USE_INVLD BIT(14)
+#define FWCMD_H2C_SENS_TBL_UPD_SMR_LOCK BIT(15)
+#define FWCMD_H2C_SENS_TBL_UPD_SMR_HOST_RPT_PLDID_SH 16
+#define FWCMD_H2C_SENS_TBL_UPD_SMR_HOST_RPT_PLDID_MSK 0xff
+#define FWCMD_H2C_SENS_TBL_UPD_WIENER_COEF_SNR_SH 24
+#define FWCMD_H2C_SENS_TBL_UPD_WIENER_COEF_SNR_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_POLL_REQUIRE BIT(27)
+#define FWCMD_H2C_SENS_TBL_UPD_INVLD_BUF_PLDID_SH 28
+#define FWCMD_H2C_SENS_TBL_UPD_INVLD_BUF_PLDID_MSK 0xf
+#define FWCMD_H2C_SENS_TBL_UPD_MAX_TX_NSTS_SH 0
+#define FWCMD_H2C_SENS_TBL_UPD_MAX_TX_NSTS_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_MAX_RX_NSTS_SH 3
+#define FWCMD_H2C_SENS_TBL_UPD_MAX_RX_NSTS_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_MIN_TIME_BTW_MSR_SH 6
+#define FWCMD_H2C_SENS_TBL_UPD_MIN_TIME_BTW_MSR_MSK 0x7fffff
+#define FWCMD_H2C_SENS_TBL_UPD_BW_SH 29
+#define FWCMD_H2C_SENS_TBL_UPD_BW_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_BAND BIT(0)
+#define FWCMD_H2C_SENS_TBL_UPD_ENTRY_IDX_SH 1
+#define FWCMD_H2C_SENS_TBL_UPD_ENTRY_IDX_MSK 0x7
+#define FWCMD_H2C_SENS_TBL_UPD_ENTRY_IDX_EN BIT(4)
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_MACID_SH 0
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_MACID_MSK 0x3ff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SMS_ID_SH 10
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SMS_ID_MSK 0x7
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_ENTRY_IDX_SH 13
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_ENTRY_IDX_MSK 0x7
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_ENABLE BIT(16)
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_BW_SIG_EN BIT(17)
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_TXBF_EN BIT(18)
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SEC_EN BIT(19)
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_RATE_SH 20
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_RATE_MSK 0xfff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_MIN_MSR_INTVAL_SH 0
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_MIN_MSR_INTVAL_MSK 0x7fffff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SENS_BW_SH 23
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SENS_BW_MSK 0x7
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SENS_TX_NSTS_SH 26
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SENS_TX_NSTS_MSK 0x7
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SENS_RX_NSTS_SH 29
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_SENS_RX_NSTS_MSK 0x7
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_1_SH 0
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_1_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_2_SH 8
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_2_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_3_SH 16
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_3_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_4_SH 24
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_4_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_5_SH 0
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_5_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_6_SH 8
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A1_ADDR_6_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_I2R_NDP_PWR_SH 16
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_I2R_NDP_PWR_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_R2I_TAR_RSSI_SH 24
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_R2I_TAR_RSSI_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_1_SH 0
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_1_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_2_SH 8
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_2_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_3_SH 16
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_3_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_4_SH 24
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_4_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_5_SH 0
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_5_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_6_SH 8
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_A2_ADDR_6_MSK 0xff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_AID11_SH 16
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_AID11_MSK 0x7ff
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_BAND BIT(27)
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_PORT_SH 28
+#define FWCMD_H2C_SENS_NTB_TX_OFLD_PORT_MSK 0xf
+#define FWCMD_H2C_SENS_INVLD_SMR_BAND_SEL BIT(0)
+#define FWCMD_H2C_SENS_INVLD_SMR_ENTRY_IDX_SH 1
+#define FWCMD_H2C_SENS_INVLD_SMR_ENTRY_IDX_MSK 0x7
+#define FWCMD_H2C_SENS_INVLD_SMR_MACID_SH 4
+#define FWCMD_H2C_SENS_INVLD_SMR_MACID_MSK 0x3ff
+#define FWCMD_H2C_SENS_INVLD_SMR_SMEID_SH 14
+#define FWCMD_H2C_SENS_INVLD_SMR_SMEID_MSK 0x3f
+#define FWCMD_H2C_SENS_INVLD_SMR_SMSID_SH 20
+#define FWCMD_H2C_SENS_INVLD_SMR_SMSID_MSK 0x7
+#define FWCMD_H2C_SENS_INVLD_SMR_USE_PKTID BIT(23)
+#define FWCMD_H2C_SENS_INVLD_SMR_SEC_TYPE_SH 24
+#define FWCMD_H2C_SENS_INVLD_SMR_SEC_TYPE_MSK 0xf
+#define FWCMD_H2C_SENS_INVLD_SMR_TX_STA_ID_SH 0
+#define FWCMD_H2C_SENS_INVLD_SMR_TX_STA_ID_MSK 0xfff
+#define FWCMD_H2C_SENS_INVLD_SMR_RX_STA_ID_SH 12
+#define FWCMD_H2C_SENS_INVLD_SMR_RX_STA_ID_MSK 0xfff
+#define FWCMD_H2C_SENS_INVLD_SMR_PKTID_SH 24
+#define FWCMD_H2C_SENS_INVLD_SMR_PKTID_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_1_SH 0
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_1_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_2_SH 8
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_2_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_3_SH 16
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_3_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_4_SH 24
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_4_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_5_SH 0
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_5_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_6_SH 8
+#define FWCMD_H2C_SENS_INVLD_SMR_A1_ADDR_6_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_1_SH 16
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_1_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_2_SH 24
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_2_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_3_SH 0
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_3_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_4_SH 8
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_4_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_5_SH 16
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_5_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_6_SH 24
+#define FWCMD_H2C_SENS_INVLD_SMR_A2_ADDR_6_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_1_SH 0
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_1_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_2_SH 8
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_2_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_3_SH 16
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_3_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_4_SH 24
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_4_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_5_SH 0
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_5_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_6_SH 8
+#define FWCMD_H2C_SENS_INVLD_SMR_A3_ADDR_6_MSK 0xff
+#define FWCMD_H2C_SENS_INVLD_SMR_PLE_PG_START_OFST_SH 0
+#define FWCMD_H2C_SENS_INVLD_SMR_PLE_PG_START_OFST_MSK 0xffffffff
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_SUB_CONTENT_LEN_SH 0
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_SUB_CONTENT_LEN_MSK 0xfffff
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_SUB_CONTENT_TYPE_SH 20
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_SUB_CONTENT_TYPE_MSK 0xfff
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_MACID_SH 0
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_MACID_MSK 0x7f
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_OP BIT(7)
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_NUM_SH 8
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_NUM_MSK 0xffffff
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_VAL_SH 0
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_VAL_MSK 0xff
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_MASK_SH 8
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_MASK_MSK 0xff
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_SHIFT_SH 16
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_DCTL_INFO_BYTE_SHIFT_MSK 0xff
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_WITH_NUM_NUM_SH 0
+#define FWCMD_H2C_SUB_OFFLOAD_AGG_WITH_NUM_NUM_MSK 0xffff
+#define FWCMD_H2C_OM_ADV_RPT_MACID_SH 0
+#define FWCMD_H2C_OM_ADV_RPT_MACID_MSK 0xffff
+#define FWCMD_H2C_OM_ADV_RPT_TYPE_SH 16
+#define FWCMD_H2C_OM_ADV_RPT_TYPE_MSK 0xf
+#define FWCMD_H2C_OM_ADV_RPT_ENABLE BIT(24)
+#define FWCMD_H2C_SER_DBG_CASE_SET_CMAC_CASE_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_CMAC_CASE_MSK 0xffffffff
+#define FWCMD_H2C_SER_DBG_CASE_SET_DMAC_CASE_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_DMAC_CASE_MSK 0xffffffff
+#define FWCMD_H2C_SER_DBG_CASE_SET_ADDRESS_0_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_ADDRESS_0_MSK 0xffffffff
+#define FWCMD_H2C_SER_DBG_CASE_SET_SUBMODULE_FLAG_0_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_SUBMODULE_FLAG_0_MSK 0xffffffff
+#define FWCMD_H2C_SER_DBG_CASE_SET_ADDRESS_1_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_ADDRESS_1_MSK 0xffffffff
+#define FWCMD_H2C_SER_DBG_CASE_SET_SUBMODULE_FLAG_1_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_SUBMODULE_FLAG_1_MSK 0xffffffff
+#define FWCMD_H2C_SER_DBG_CASE_SET_ADDRESS_2_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_ADDRESS_2_MSK 0xffffffff
+#define FWCMD_H2C_SER_DBG_CASE_SET_SUBMODULE_FLAG_2_SH 0
+#define FWCMD_H2C_SER_DBG_CASE_SET_SUBMODULE_FLAG_2_MSK 0xffffffff
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_IDX_SH 0
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_IDX_MSK 0xf
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_EN BIT(31)
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_IP_ADDRESS_SH 0
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_IP_ADDRESS_MSK 0xffffffff
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_MAC_LOW_ADDRESS_SH 0
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_MAC_LOW_ADDRESS_MSK 0xffffffff
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_MAC_HIGH_ADDRESS_SH 0
+#define FWCMD_H2C_NAT2P5_UPD_NAT2P5_MAC_HIGH_ADDRESS_MSK 0xffff
+#define FWCMD_H2C_AZ_EN_MAC_ID_SH 0
+#define FWCMD_H2C_AZ_EN_MAC_ID_MSK 0xffff
+#define FWCMD_H2C_AZ_EN_INITIALSTA_SH 16
+#define FWCMD_H2C_AZ_EN_INITIALSTA_MSK 0xff
+#define FWCMD_H2C_AZ_EN_NONTB_SH 24
+#define FWCMD_H2C_AZ_EN_NONTB_MSK 0xff
+#define FWCMD_H2C_AZ_EN_BILMR_SH 0
+#define FWCMD_H2C_AZ_EN_BILMR_MSK 0xff
+#define FWCMD_H2C_AZ_EN_DELAYLMR_SH 8
+#define FWCMD_H2C_AZ_EN_DELAYLMR_MSK 0xff
+#define FWCMD_H2C_AZ_EN_TERMINATION_SH 16
+#define FWCMD_H2C_AZ_EN_TERMINATION_MSK 0xff
+#define FWCMD_H2C_AZ_EN_SECEN_SH 24
+#define FWCMD_H2C_AZ_EN_SECEN_MSK 0xff
+#define FWCMD_H2C_AZ_EN_ENABLE_SH 0
+#define FWCMD_H2C_AZ_EN_ENABLE_MSK 0xff
+#define FWCMD_H2C_AZ_EN_PWR_CTRL_EN_SH 8
+#define FWCMD_H2C_AZ_EN_PWR_CTRL_EN_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_ASAP_SH 0
+#define FWCMD_H2C_AZ_PROC_ASAP_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_PKTID_IFTMR_SH 8
+#define FWCMD_H2C_AZ_PROC_PKTID_IFTMR_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_PKTID_FTMR_SH 16
+#define FWCMD_H2C_AZ_PROC_PKTID_FTMR_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_TSF_TIMER_OFFSET_SH 24
+#define FWCMD_H2C_AZ_PROC_TSF_TIMER_OFFSET_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_PARTIAL_TSF_TIMER_SH 0
+#define FWCMD_H2C_AZ_PROC_PARTIAL_TSF_TIMER_MSK 0xffff
+#define FWCMD_H2C_AZ_PROC_RSP_CH_SH 16
+#define FWCMD_H2C_AZ_PROC_RSP_CH_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_MODE_SH 24
+#define FWCMD_H2C_AZ_PROC_MODE_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_FIRST_PSTIMER_HIGH_SH 0
+#define FWCMD_H2C_AZ_PROC_FIRST_PSTIMER_HIGH_MSK 0xffffffff
+#define FWCMD_H2C_AZ_PROC_FIRST_PSTIMER_LOW_SH 0
+#define FWCMD_H2C_AZ_PROC_FIRST_PSTIMER_LOW_MSK 0xffffffff
+#define FWCMD_H2C_AZ_PROC_ITVL_SH 0
+#define FWCMD_H2C_AZ_PROC_ITVL_MSK 0xffffffff
+#define FWCMD_H2C_AZ_PROC_BURST_CNT_SH 0
+#define FWCMD_H2C_AZ_PROC_BURST_CNT_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_PTK_KEYID_SH 8
+#define FWCMD_H2C_AZ_PROC_PTK_KEYID_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_AID_SH 0
+#define FWCMD_H2C_AZ_PROC_AID_MSK 0xffff
+#define FWCMD_H2C_AZ_PROC_MACID_SH 16
+#define FWCMD_H2C_AZ_PROC_MACID_MSK 0xffff
+#define FWCMD_H2C_AZ_PROC_BILMR_SH 0
+#define FWCMD_H2C_AZ_PROC_BILMR_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_DELAYLMR_SH 8
+#define FWCMD_H2C_AZ_PROC_DELAYLMR_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_TERMINATION_SH 16
+#define FWCMD_H2C_AZ_PROC_TERMINATION_MSK 0xff
+#define FWCMD_H2C_AZ_PROC_SECEN_SH 24
+#define FWCMD_H2C_AZ_PROC_SECEN_MSK 0xff
+#define FWCMD_H2C_AZ_UPDATE_UPD_PSTIMER_HIGH_SH 0
+#define FWCMD_H2C_AZ_UPDATE_UPD_PSTIMER_HIGH_MSK 0xffffffff
+#define FWCMD_H2C_AZ_UPDATE_UPD_PSTIMER_LOW_SH 0
+#define FWCMD_H2C_AZ_UPDATE_UPD_PSTIMER_LOW_MSK 0xffffffff
+#define FWCMD_H2C_AZ_UPDATE_ITVL_SH 0
+#define FWCMD_H2C_AZ_UPDATE_ITVL_MSK 0xffffffff
+#define FWCMD_H2C_AZ_UPDATE_UPD_BURST_CNT_SH 0
+#define FWCMD_H2C_AZ_UPDATE_UPD_BURST_CNT_MSK 0xff
+#define FWCMD_H2C_AZ_UPDATE_TERMINATION_SH 8
+#define FWCMD_H2C_AZ_UPDATE_TERMINATION_MSK 0xff
+#define FWCMD_H2C_AZ_UPDATE_BW_SH 16
+#define FWCMD_H2C_AZ_UPDATE_BW_MSK 0xff
+#define FWCMD_H2C_AZ_UPDATE_MACID_SH 24
+#define FWCMD_H2C_AZ_UPDATE_MACID_MSK 0xff
+#define FWCMD_H2C_AZ_CANCEL_PRI_CH_SH 0
+#define FWCMD_H2C_AZ_CANCEL_PRI_CH_MSK 0xff
+#define FWCMD_H2C_AZ_CANCEL_CENTRAL_CH_SH 8
+#define FWCMD_H2C_AZ_CANCEL_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_AZ_CANCEL_BW_SH 16
+#define FWCMD_H2C_AZ_CANCEL_BW_MSK 0xff
+#define FWCMD_H2C_AZ_CANCEL_CH_BAND_SH 24
+#define FWCMD_H2C_AZ_CANCEL_CH_BAND_MSK 0xff
+#define FWCMD_H2C_AZ_CANCEL_BAND_SH 0
+#define FWCMD_H2C_AZ_CANCEL_BAND_MSK 0xff
+#define FWCMD_H2C_AZ_CANCEL_RELOAD_RF_SH 8
+#define FWCMD_H2C_AZ_CANCEL_RELOAD_RF_MSK 0xff
+#define FWCMD_H2C_WOW_DIAG_CTRL_STOP BIT(0)
+#define FWCMD_H2C_WOW_REQ_RX_PKT_PKT_NUM_SH 0
+#define FWCMD_H2C_WOW_REQ_RX_PKT_PKT_NUM_MSK 0xff
+#define FWCMD_H2C_WOW_REQ_RX_PKT_PLD_SIZE_SH 8
+#define FWCMD_H2C_WOW_REQ_RX_PKT_PLD_SIZE_MSK 0xff
+#define FWCMD_H2C_WOW_REQ_RX_PKT_PERIOD_SH 16
+#define FWCMD_H2C_WOW_REQ_RX_PKT_PERIOD_MSK 0xffff
+#define FWCMD_H2C_WOW_REQ_MEM_HEAP_INFO BIT(0)
+#define FWCMD_H2C_WOW_REQ_MEM_MEM_INFO BIT(1)
+#define FWCMD_H2C_WOW_REQ_MEM_WOW_START BIT(2)
+#define FWCMD_H2C_WOW_REQ_MEM_WOW_END BIT(3)
+#define FWCMD_H2C_WOW_REQ_BB_RF_REG_BB BIT(0)
+#define FWCMD_H2C_WOW_REQ_BB_RF_REG_RF BIT(1)
+#define FWCMD_H2C_AP_SELF_DIAG_FEATURE_SH 0
+#define FWCMD_H2C_AP_SELF_DIAG_FEATURE_MSK 0xff
+#define FWCMD_H2C_AP_SELF_DIAG_MODE_SH 8
+#define FWCMD_H2C_AP_SELF_DIAG_MODE_MSK 0xff
+#define FWCMD_H2C_REQ_STA_DIAG_RPT_REQ_WOW_RPT BIT(0)
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_SSN_MODE_SH 0
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_SSN_MODE_MSK 0x3
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_SSN_INDEX_SH 2
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_SSN_INDEX_MSK 0x7
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_MACID_SH 5
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_MACID_MSK 0xff
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_SW_SEQ_NUM_SH 20
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_OFLD_SW_SEQ_NUM_MSK 0xfff
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SSN_MODE_SH 0
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SSN_MODE_MSK 0x3
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SSN_INDEX_SH 2
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SSN_INDEX_MSK 0x7
+#define FWCMD_H2C_ADDBA_OFLD_REQ_MACID_SH 5
+#define FWCMD_H2C_ADDBA_OFLD_REQ_MACID_MSK 0xff
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SW_SEQ_NUM_SH 20
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SW_SEQ_NUM_MSK 0xfff
+#define FWCMD_H2C_ADDBA_OFLD_REQ_BAND BIT(0)
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SEC_TYPE_SH 1
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SEC_TYPE_MSK 0xf
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SEC_CAM_IDX_SH 5
+#define FWCMD_H2C_ADDBA_OFLD_REQ_SEC_CAM_IDX_MSK 0x3ff
+#define FWCMD_H2C_ADDBA_OFLD_REQ_FORCE_KEY_EN BIT(15)
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_REQ_START_SH 0
+#define FWCMD_H2C_ADDBA_OFLD_REQ_ADDBA_REQ_START_MSK 0xffffffff
+#define FWCMD_H2C_SAP_INFO_SAP_PWR_INT_EN BIT(0)
+
+/*----------------------------------- H2CPKT - CAT(Table) --------------------------*/
+#define FWCMD_H2C_MUDE_PARA_TBLUD_TBLUD_SH 0
+#define FWCMD_H2C_MUDE_PARA_TBLUD_TBLUD_MSK 0xffffffff
+#define FWCMD_H2C_MUDE_PARA_TBLUD_MUDECISION_PARA0_SH 0
+#define FWCMD_H2C_MUDE_PARA_TBLUD_MUDECISION_PARA0_MSK 0xffffffff
+#define FWCMD_H2C_MUDE_PARA_TBLUD_MUDECISION_PARA1_SH 0
+#define FWCMD_H2C_MUDE_PARA_TBLUD_MUDECISION_PARA1_MSK 0xffffffff
+#define FWCMD_H2C_MUDECISION_PARA_MUINFO_THOLD_SH 0
+#define FWCMD_H2C_MUDECISION_PARA_MUINFO_THOLD_MSK 0x3fffffff
+#define FWCMD_H2C_MUDECISION_PARA_BYPASS_THOLD BIT(30)
+#define FWCMD_H2C_MUDECISION_PARA_BYPASS_TP BIT(31)
+#define FWCMD_H2C_MUDECISION_PARA_INIT_RATE_SH 0
+#define FWCMD_H2C_MUDECISION_PARA_INIT_RATE_MSK 0xf
+#define FWCMD_H2C_MUDECISION_PARA_RETRY_TH_SH 4
+#define FWCMD_H2C_MUDECISION_PARA_RETRY_TH_MSK 0x7
+#define FWCMD_H2C_MU_FIXINFO_MUINFO_THOLDEN BIT(0)
+#define FWCMD_H2C_MU_FIXINFO_MUINFO_THOLD_SH 1
+#define FWCMD_H2C_MU_FIXINFO_MUINFO_THOLD_MSK 0x3fffffff
+#define FWCMD_H2C_MU_FIXINFO_BYPASS_THOLD BIT(31)
+#define FWCMD_H2C_MU_FIXINFO_FIXMU_EN BIT(0)
+#define FWCMD_H2C_MU_FIXINFO_FIXMU_ID_SH 1
+#define FWCMD_H2C_MU_FIXINFO_FIXMU_ID_MSK 0x7f
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_TBLUD_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_TBLUD_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D2_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D2_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D3_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D3_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D4_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D4_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D5_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D5_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D6_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D6_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D7_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D7_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D8_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D8_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D9_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D9_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D10_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D10_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D11_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D11_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D12_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D12_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D13_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D13_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D14_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D14_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D15_SH 0
+#define FWCMD_H2C_UL_FIXINFO_TBLUD_UL_FIXINFO_D15_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_CFG_MODE_SH 0
+#define FWCMD_H2C_UL_FIXINFO_CFG_MODE_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_CFG_INTERVAL_SH 2
+#define FWCMD_H2C_UL_FIXINFO_CFG_INTERVAL_MSK 0x3f
+#define FWCMD_H2C_UL_FIXINFO_CFG_BSR_THOLD_SH 8
+#define FWCMD_H2C_UL_FIXINFO_CFG_BSR_THOLD_MSK 0xff
+#define FWCMD_H2C_UL_FIXINFO_CFG_STOREMODE_SH 16
+#define FWCMD_H2C_UL_FIXINFO_CFG_STOREMODE_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_NDPA_DUR_SH 0
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_NDPA_DUR_MSK 0xffff
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_TF_TYPE_SH 16
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_TF_TYPE_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_SIGEN BIT(19)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_SIGSC_SH 20
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_SIGSC_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_MURTS BIT(24)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_NDPA_SH 25
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_NDPA_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_SNDPKT_SH 27
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_SNDPKT_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_GI_LTF_SH 29
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_GI_LTF_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_DATART_SH 0
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_DATART_MSK 0x1ff
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_DATAER BIT(9)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_DATABW_SH 10
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_DATABW_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_STBC_SH 12
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_STBC_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_LDPC BIT(14)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_DATADCM BIT(15)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_APEPLEN_SH 16
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_APEPLEN_MSK 0xfff
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_MORETF BIT(28)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_DATA_VWER BIT(29)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_ISTWT BIT(30)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_UL_LOGO_TEST BIT(31)
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_MULTIPORT_SH 0
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_MULTIPORT_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_MBSSID_SH 3
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_MBSSID_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_TXPWR_MODE_SH 7
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_TXPWR_MODE_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_ULFIX_USAGE_SH 10
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_ULFIX_USAGE_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_TWTGRP_STANUM_SEL_SH 13
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_TWTGRP_STANUM_SEL_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_STORE_IDX_SH 15
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_STORE_IDX_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_ULINFO_NO_NEXT_SS BIT(19)
+#define FWCMD_H2C_UL_FIXINFO_UL_FIXINFO_STA_INFO_0_1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_UL_FIXINFO_STA_INFO_0_1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_UL_FIXINFO_STA_INFO_2_3_SH 0
+#define FWCMD_H2C_UL_FIXINFO_UL_FIXINFO_STA_INFO_2_3_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_RU2SU BIT(0)
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_PPDU_BW_SH 1
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_PPDU_BW_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_GI_LTF_SH 3
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_GI_LTF_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_STBC BIT(6)
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_DOPPLER BIT(7)
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_LTF_MA_SH 8
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_LTF_MA_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_STANUM_SH 11
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_STANUM_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_RFGFIX BIT(16)
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_RFGIDX_SH 17
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_RFGIDX_MSK 0x3ff
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_TB_NOM_SH 27
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_TB_NOM_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_GRP_MODE BIT(0)
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_GRP_ID_SH 1
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_GRP_ID_MSK 0x3f
+#define FWCMD_H2C_UL_FIXINFO_ULRUA_FIX_MODE BIT(7)
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U0_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U0_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U0_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U0_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U1_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U1_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U1_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U1_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U2_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U2_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U2_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U2_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U3_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U3_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U3_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_ENT_U3_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_MACID_0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_MACID_0_MSK 0xff
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_PREF_AC_0_SH 8
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_PREF_AC_0_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_MACID_1_SH 16
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_MACID_1_MSK 0xff
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_PREF_AC_1_SH 24
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_PREF_AC_1_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_DROP BIT(0)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_TGT_RSSI_SH 1
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_MAC_ID_SH 8
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_MAC_ID_MSK 0xff
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_RU_POS_SH 16
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_RU_POS_MSK 0xff
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_CODE BIT(24)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_VIP BIT(25)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_BSRLEN_SH 0
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_BSRLEN_MSK 0x7fff
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_DCM BIT(16)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_SS_SH 17
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_SS_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_MCS_SH 20
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_MCS_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_RT_TBLCOL_SH 24
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_RT_TBLCOL_MSK 0x3f
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_PRTL_ALLOC BIT(30)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_RATE_CHG BIT(31)
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_MODE_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_MODE_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_INTERVAL_SH 2
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_INTERVAL_MSK 0x3f
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_BSR_THOLD_SH 8
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_BSR_THOLD_MSK 0xff
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_STOREMODE_SH 16
+#define FWCMD_H2C_UL_FIXINFO_V2_CFG_STOREMODE_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_NDPA_DUR_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_NDPA_DUR_MSK 0xffff
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_TF_TYPE_SH 16
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_TF_TYPE_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_SIGEN BIT(19)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_SIGSC_SH 20
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_SIGSC_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_MURTS BIT(24)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_NDPA_SH 25
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_NDPA_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_SNDPKT_SH 27
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_SNDPKT_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_GI_LTF_SH 29
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_GI_LTF_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_DATART_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_DATART_MSK 0xfff
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_DATAER BIT(12)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_DATABW_SH 13
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_DATABW_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_STBC_SH 15
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_STBC_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_LDPC BIT(17)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_DATADCM BIT(18)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_APEPLEN_SH 19
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_APEPLEN_MSK 0xfff
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_MORETF BIT(31)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_MULTIPORT_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_MULTIPORT_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_MBSSID_SH 3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_MBSSID_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_TXPWR_MODE_SH 7
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_TXPWR_MODE_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_ULFIX_USAGE_SH 10
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_ULFIX_USAGE_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_TWTGRP_STANUM_SEL_SH 13
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_TWTGRP_STANUM_SEL_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_STORE_IDX_SH 15
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_STORE_IDX_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_BAND BIT(19)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_NO_NEXT_SS BIT(20)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_DATA_VWER BIT(21)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_ISTWT BIT(22)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULINFO_UL_LOGO_TEST BIT(23)
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_0_3_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_0_3_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_4_7_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_4_7_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_8_11_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_8_11_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_12_15_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_UL_FIXINFO_STA_INFO_V2_12_15_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_RU2SU BIT(0)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_TX_MODE_SH 1
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_TX_MODE_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_GRP_MODE BIT(3)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_BAND BIT(4)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_PPDU_BW_SH 5
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_PPDU_BW_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_STANUM_SH 8
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_STANUM_MSK 0x1f
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_BW_EXT_SH 13
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_BW_EXT_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_GI_LTF_SH 16
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_GI_LTF_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_FIX_MODE BIT(19)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_RFGFIX BIT(20)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_TFRPT_FMT BIT(21)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_GRP_ID_SH 24
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_GRP_ID_MSK 0x3f
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_RFGIDX_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_RFGIDX_MSK 0x3ff
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_STBC BIT(16)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_DOPPLER BIT(17)
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_TB_NOM_SH 18
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_TB_NOM_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_LTF_MA_SH 20
+#define FWCMD_H2C_UL_FIXINFO_V2_ULRUA_LTF_MA_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U0_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U0_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U0_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U0_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U1_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U1_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U1_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U1_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U2_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U2_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U2_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U2_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U3_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U3_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U3_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U3_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U4_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U4_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U4_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U4_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U5_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U5_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U5_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U5_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U6_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U6_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U6_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U6_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U7_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U7_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U7_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U7_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U8_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U8_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U8_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U8_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U9_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U9_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U9_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U9_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U10_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U10_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U10_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U10_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U11_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U11_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U11_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U11_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U12_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U12_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U12_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U12_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U13_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U13_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U13_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U13_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U14_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U14_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U14_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U14_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U15_D0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U15_D0_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U15_D1_SH 0
+#define FWCMD_H2C_UL_FIXINFO_V2_STA_ENT_V2_U15_D1_MSK 0xffffffff
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_0_SH 0
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_0_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_1_SH 8
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_1_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_2_SH 16
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_2_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_3_SH 24
+#define FWCMD_H2C_UL_FIXINFO_STA_INFO_V2_PREF_AC_3_MSK 0x3
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_MAC_ID_SH 0
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_MAC_ID_MSK 0xffff
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_RU_POS_SH 16
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_RU_POS_MSK 0xff
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_PS160 BIT(24)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_CODE BIT(25)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_RATE_MODE_SH 27
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_RATE_MODE_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_BSRLEN_SH 0
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_BSRLEN_MSK 0x7fff
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_DCM BIT(16)
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_SS_SH 17
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_SS_MSK 0x7
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_MCS_SH 20
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_MCS_MSK 0xf
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_TGT_RSSI_SH 24
+#define FWCMD_H2C_UL_FIXINFO_UL_RUA_STA_ENT_V2_TGT_RSSI_MSK 0x7f
+#define FWCMD_H2C_ADDR_CAM_VALID BIT(0)
+#define FWCMD_H2C_ADDR_CAM_NET_TYPE_SH 1
+#define FWCMD_H2C_ADDR_CAM_NET_TYPE_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_BCN_HIT_COND_SH 3
+#define FWCMD_H2C_ADDR_CAM_BCN_HIT_COND_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_HIT_RULE_SH 5
+#define FWCMD_H2C_ADDR_CAM_HIT_RULE_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_BB_SEL BIT(7)
+#define FWCMD_H2C_ADDR_CAM_ADDR_MASK_SH 8
+#define FWCMD_H2C_ADDR_CAM_ADDR_MASK_MSK 0x3f
+#define FWCMD_H2C_ADDR_CAM_MASK_SEL_SH 14
+#define FWCMD_H2C_ADDR_CAM_MASK_SEL_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SMA_HASH_SH 16
+#define FWCMD_H2C_ADDR_CAM_SMA_HASH_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_TMA_HASH_SH 24
+#define FWCMD_H2C_ADDR_CAM_TMA_HASH_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_BSSID_CAM_IDX_SH 0
+#define FWCMD_H2C_ADDR_CAM_BSSID_CAM_IDX_MSK 0x3f
+#define FWCMD_H2C_ADDR_CAM_SMA0_SH 0
+#define FWCMD_H2C_ADDR_CAM_SMA0_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SMA1_SH 8
+#define FWCMD_H2C_ADDR_CAM_SMA1_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SMA2_SH 16
+#define FWCMD_H2C_ADDR_CAM_SMA2_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SMA3_SH 24
+#define FWCMD_H2C_ADDR_CAM_SMA3_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SMA4_SH 0
+#define FWCMD_H2C_ADDR_CAM_SMA4_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SMA5_SH 8
+#define FWCMD_H2C_ADDR_CAM_SMA5_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_TMA0_SH 16
+#define FWCMD_H2C_ADDR_CAM_TMA0_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_TMA1_SH 24
+#define FWCMD_H2C_ADDR_CAM_TMA1_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_TMA2_SH 0
+#define FWCMD_H2C_ADDR_CAM_TMA2_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_TMA3_SH 8
+#define FWCMD_H2C_ADDR_CAM_TMA3_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_TMA4_SH 16
+#define FWCMD_H2C_ADDR_CAM_TMA4_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_TMA5_SH 24
+#define FWCMD_H2C_ADDR_CAM_TMA5_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_MACID_SH 0
+#define FWCMD_H2C_ADDR_CAM_MACID_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_PORT_INT_SH 8
+#define FWCMD_H2C_ADDR_CAM_PORT_INT_MSK 0x7
+#define FWCMD_H2C_ADDR_CAM_TSF_SYNC_SH 11
+#define FWCMD_H2C_ADDR_CAM_TSF_SYNC_MSK 0x7
+#define FWCMD_H2C_ADDR_CAM_TF_TRS BIT(14)
+#define FWCMD_H2C_ADDR_CAM_LSIG_TXOP BIT(15)
+#define FWCMD_H2C_ADDR_CAM_TGT_IND_SH 24
+#define FWCMD_H2C_ADDR_CAM_TGT_IND_MSK 0x7
+#define FWCMD_H2C_ADDR_CAM_FRM_TGT_IND_SH 27
+#define FWCMD_H2C_ADDR_CAM_FRM_TGT_IND_MSK 0x7
+#define FWCMD_H2C_ADDR_CAM_AID12_0_SH 0
+#define FWCMD_H2C_ADDR_CAM_AID12_0_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_AID12_1_SH 8
+#define FWCMD_H2C_ADDR_CAM_AID12_1_MSK 0xf
+#define FWCMD_H2C_ADDR_CAM_WOL_PATTERN BIT(12)
+#define FWCMD_H2C_ADDR_CAM_WOL_UC BIT(13)
+#define FWCMD_H2C_ADDR_CAM_WOL_MAGIC BIT(14)
+#define FWCMD_H2C_ADDR_CAM_WAPI BIT(15)
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT_MODE_SH 16
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT_MODE_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT0_KEYID_SH 18
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT0_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT1_KEYID_SH 20
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT1_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT2_KEYID_SH 22
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT2_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT3_KEYID_SH 24
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT3_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT4_KEYID_SH 26
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT4_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT5_KEYID_SH 28
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT5_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT6_KEYID_SH 30
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT6_KEYID_MSK 0x3
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT_VALID_SH 0
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT_VALID_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT0_SH 8
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT0_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT1_SH 16
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT1_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT2_SH 24
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT2_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT3_SH 0
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT3_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT4_SH 8
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT4_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT5_SH 16
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT5_MSK 0xff
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT6_SH 24
+#define FWCMD_H2C_ADDR_CAM_SEC_ENT6_MSK 0xff
+#define FWCMD_H2C_BSSID_CAM_BSSID_VALID BIT(0)
+#define FWCMD_H2C_BSSID_CAM_BB_SEL BIT(1)
+#define FWCMD_H2C_BSSID_CAM_MSK_SH 2
+#define FWCMD_H2C_BSSID_CAM_MSK_MSK 0x3f
+#define FWCMD_H2C_BSSID_CAM_BSS_COLOR_SH 8
+#define FWCMD_H2C_BSSID_CAM_BSS_COLOR_MSK 0x7f
+#define FWCMD_H2C_BSSID_CAM_BSSID0_SH 16
+#define FWCMD_H2C_BSSID_CAM_BSSID0_MSK 0xff
+#define FWCMD_H2C_BSSID_CAM_BSSID1_SH 24
+#define FWCMD_H2C_BSSID_CAM_BSSID1_MSK 0xff
+#define FWCMD_H2C_BSSID_CAM_BSSID2_SH 0
+#define FWCMD_H2C_BSSID_CAM_BSSID2_MSK 0xff
+#define FWCMD_H2C_BSSID_CAM_BSSID3_SH 8
+#define FWCMD_H2C_BSSID_CAM_BSSID3_MSK 0xff
+#define FWCMD_H2C_BSSID_CAM_BSSID4_SH 16
+#define FWCMD_H2C_BSSID_CAM_BSSID4_MSK 0xff
+#define FWCMD_H2C_BSSID_CAM_BSSID5_SH 24
+#define FWCMD_H2C_BSSID_CAM_BSSID5_MSK 0xff
+#define FWCMD_H2C_SEC_CAM_SEC_TYPE_SH 0
+#define FWCMD_H2C_SEC_CAM_SEC_TYPE_MSK 0xf
+#define FWCMD_H2C_SEC_CAM_EXT_KEY BIT(4)
+#define FWCMD_H2C_SEC_CAM_SPP_MODE BIT(5)
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_TBLUD_SH 0
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_TBLUD_MSK 0xffffffff
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_FIXMODE_PARA_D0_SH 0
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_FIXMODE_PARA_D0_MSK 0xffffffff
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_FIXMODE_PARA_D1_SH 0
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_FIXMODE_PARA_D1_MSK 0xffffffff
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_FIXMODE_PARA_D2_SH 0
+#define FWCMD_H2C_FIXMODE_PARA_TBLUD_FIXMODE_PARA_D2_MSK 0xffffffff
+#define FWCMD_H2C_FIXMODE_PARA_FORCE_SUMURU_EN BIT(0)
+#define FWCMD_H2C_FIXMODE_PARA_FORCESU BIT(1)
+#define FWCMD_H2C_FIXMODE_PARA_FORCEMU BIT(2)
+#define FWCMD_H2C_FIXMODE_PARA_FORCERU BIT(3)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FE_SU_EN BIT(4)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FE_VHTMU_EN BIT(5)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FE_HEMU_EN BIT(6)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FE_HERU_EN BIT(7)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FE_UL_EN BIT(8)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FRAME_SEQ_SU BIT(9)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FRAME_SEQ_VHTMU BIT(10)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FRAME_SEQ_HEMU BIT(11)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FRAME_SEQ_HERU BIT(12)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FRAME_SEQ_UL BIT(13)
+#define FWCMD_H2C_FIXMODE_PARA_IS_DLRUHWGRP BIT(14)
+#define FWCMD_H2C_FIXMODE_PARA_IS_ULRUHWGRP BIT(15)
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_SU_SH 16
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_SU_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_VHTMU_SH 20
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_VHTMU_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_RESP_TYPE_VHTMU_SH 24
+#define FWCMD_H2C_FIXMODE_PARA_RESP_TYPE_VHTMU_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_HEMU_SH 28
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_HEMU_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_RESP_TYPE_HEMU_SH 0
+#define FWCMD_H2C_FIXMODE_PARA_RESP_TYPE_HEMU_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_HERU_SH 4
+#define FWCMD_H2C_FIXMODE_PARA_PROT_TYPE_HERU_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_RESP_TYPE_HERU_SH 8
+#define FWCMD_H2C_FIXMODE_PARA_RESP_TYPE_HERU_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_UL_PROT_TYPE_SH 12
+#define FWCMD_H2C_FIXMODE_PARA_UL_PROT_TYPE_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_RUGRPID_SH 16
+#define FWCMD_H2C_FIXMODE_PARA_RUGRPID_MSK 0x1f
+#define FWCMD_H2C_FIXMODE_PARA_MUGRPID_SH 21
+#define FWCMD_H2C_FIXMODE_PARA_MUGRPID_MSK 0x1f
+#define FWCMD_H2C_FIXMODE_PARA_ULGRPID_SH 26
+#define FWCMD_H2C_FIXMODE_PARA_ULGRPID_MSK 0x1f
+#define FWCMD_H2C_FIXMODE_PARA_BAND BIT(31)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_TXCMDNUM_EN BIT(0)
+#define FWCMD_H2C_FIXMODE_PARA_FORCE_TO_ONE BIT(1)
+#define FWCMD_H2C_FIXMODE_PARA_SET_ULMODE BIT(2)
+#define FWCMD_H2C_FIXMODE_PARA_ULMODE_SH 3
+#define FWCMD_H2C_FIXMODE_PARA_ULMODE_MSK 0x3
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FE_UL_MU_EN BIT(5)
+#define FWCMD_H2C_FIXMODE_PARA_FIX_FRAME_SEQ_UL_MU BIT(6)
+#define FWCMD_H2C_FIXMODE_PARA_IS_ULMUHWGRP BIT(7)
+#define FWCMD_H2C_FIXMODE_PARA_UL_MU_PROT_TYPE_SH 8
+#define FWCMD_H2C_FIXMODE_PARA_UL_MU_PROT_TYPE_MSK 0xf
+#define FWCMD_H2C_FIXMODE_PARA_MACID_EXT_SH 12
+#define FWCMD_H2C_FIXMODE_PARA_MACID_EXT_MSK 0xfff
+#define FWCMD_H2C_WRITE_OFLD_REQ_VALUE_LEN_SH 0
+#define FWCMD_H2C_WRITE_OFLD_REQ_VALUE_LEN_MSK 0x7ff
+#define FWCMD_H2C_WRITE_OFLD_REQ_POLLING BIT(13)
+#define FWCMD_H2C_WRITE_OFLD_REQ_MASK_EN BIT(14)
+#define FWCMD_H2C_WRITE_OFLD_REQ_LS BIT(15)
+#define FWCMD_H2C_WRITE_OFLD_REQ_OFLD_ID_SH 16
+#define FWCMD_H2C_WRITE_OFLD_REQ_OFLD_ID_MSK 0xff
+#define FWCMD_H2C_WRITE_OFLD_REQ_ENTRY_NUM_SH 24
+#define FWCMD_H2C_WRITE_OFLD_REQ_ENTRY_NUM_MSK 0xff
+#define FWCMD_H2C_WRITE_OFLD_REQ_OFFSET_SH 0
+#define FWCMD_H2C_WRITE_OFLD_REQ_OFFSET_MSK 0xffff
+#define FWCMD_H2C_READ_OFLD_REQ_VALUE_LEN_SH 0
+#define FWCMD_H2C_READ_OFLD_REQ_VALUE_LEN_MSK 0x7ff
+#define FWCMD_H2C_READ_OFLD_REQ_LS BIT(15)
+#define FWCMD_H2C_READ_OFLD_REQ_OFLD_ID_SH 16
+#define FWCMD_H2C_READ_OFLD_REQ_OFLD_ID_MSK 0xff
+#define FWCMD_H2C_READ_OFLD_REQ_ENTRY_NUM_SH 24
+#define FWCMD_H2C_READ_OFLD_REQ_ENTRY_NUM_MSK 0xff
+#define FWCMD_H2C_READ_OFLD_REQ_OFFSET_SH 0
+#define FWCMD_H2C_READ_OFLD_REQ_OFFSET_MSK 0xffff
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_DEVICE_SH 0
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_DEVICE_MSK 0xff
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_HIOE_OP_SH 0
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_HIOE_OP_MSK 0xff
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_INST_TYPE_SH 8
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_INST_TYPE_MSK 0xff
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_DATA_MODE_SH 24
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_DATA_MODE_MSK 0xff
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_REGISTER_ADDR_SH 0
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_REGISTER_ADDR_MSK 0xffffffff
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_BYTE_DATA_H_SH 0
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_BYTE_DATA_H_MSK 0xffff
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_BYTE_DATA_L_SH 16
+#define FWCMD_H2C_CONF_OFLD_REQ_CMD_BYTE_DATA_L_MSK 0xffff
+#define FWCMD_H2C_DCTRL_QOS_FIELD_H_SH 0
+#define FWCMD_H2C_DCTRL_QOS_FIELD_H_MSK 0xff
+#define FWCMD_H2C_DCTRL_HW_EXSEQ_MACID_SH 8
+#define FWCMD_H2C_DCTRL_HW_EXSEQ_MACID_MSK 0x7f
+#define FWCMD_H2C_DCTRL_QOS_FIELD_H_EN BIT(15)
+#define FWCMD_H2C_DCTRL_AES_IV_L_SH 16
+#define FWCMD_H2C_DCTRL_AES_IV_L_MSK 0xffff
+#define FWCMD_H2C_DCTRL_AES_IV_H_SH 0
+#define FWCMD_H2C_DCTRL_AES_IV_H_MSK 0xffffffff
+#define FWCMD_H2C_DCTRL_SEQ0_SH 0
+#define FWCMD_H2C_DCTRL_SEQ0_MSK 0xfff
+#define FWCMD_H2C_DCTRL_SEQ1_SH 12
+#define FWCMD_H2C_DCTRL_SEQ1_MSK 0xfff
+#define FWCMD_H2C_DCTRL_AMSDU_MAX_LEN_SH 24
+#define FWCMD_H2C_DCTRL_AMSDU_MAX_LEN_MSK 0x7
+#define FWCMD_H2C_DCTRL_STA_AMSDU_EN BIT(27)
+#define FWCMD_H2C_DCTRL_CHKSUM_OFLD_EN BIT(28)
+#define FWCMD_H2C_DCTRL_WITH_LLC BIT(29)
+#define FWCMD_H2C_DCTRL_SEC_HW_ENC BIT(31)
+#define FWCMD_H2C_DCTRL_SEQ2_SH 0
+#define FWCMD_H2C_DCTRL_SEQ2_MSK 0xfff
+#define FWCMD_H2C_DCTRL_SEQ3_SH 12
+#define FWCMD_H2C_DCTRL_SEQ3_MSK 0xfff
+#define FWCMD_H2C_DCTRL_SEC_CAM_IDX_SH 24
+#define FWCMD_H2C_DCTRL_SEC_CAM_IDX_MSK 0xff
+#define FWCMD_H2C_CCTRL_DATARATE_SH 0
+#define FWCMD_H2C_CCTRL_DATARATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_FORCE_TXOP BIT(9)
+#define FWCMD_H2C_CCTRL_DATA_BW_SH 10
+#define FWCMD_H2C_CCTRL_DATA_BW_MSK 0x3
+#define FWCMD_H2C_CCTRL_DATA_GI_LTF_SH 12
+#define FWCMD_H2C_CCTRL_DATA_GI_LTF_MSK 0x7
+#define FWCMD_H2C_CCTRL_DARF_TC_INDEX BIT(15)
+#define FWCMD_H2C_CCTRL_ARFR_CTRL_SH 16
+#define FWCMD_H2C_CCTRL_ARFR_CTRL_MSK 0xf
+#define FWCMD_H2C_CCTRL_ACQ_RPT_EN BIT(20)
+#define FWCMD_H2C_CCTRL_MGQ_RPT_EN BIT(21)
+#define FWCMD_H2C_CCTRL_ULQ_RPT_EN BIT(22)
+#define FWCMD_H2C_CCTRL_TWTQ_RPT_EN BIT(23)
+#define FWCMD_H2C_CCTRL_DISRTSFB BIT(25)
+#define FWCMD_H2C_CCTRL_DISDATAFB BIT(26)
+#define FWCMD_H2C_CCTRL_TRYRATE BIT(27)
+#define FWCMD_H2C_CCTRL_AMPDU_DENSITY_SH 28
+#define FWCMD_H2C_CCTRL_AMPDU_DENSITY_MSK 0xf
+#define FWCMD_H2C_CCTRL_DATA_RTY_LOWEST_RATE_SH 0
+#define FWCMD_H2C_CCTRL_DATA_RTY_LOWEST_RATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_AMPDU_TIME_SEL BIT(9)
+#define FWCMD_H2C_CCTRL_AMPDU_LEN_SEL BIT(10)
+#define FWCMD_H2C_CCTRL_RTS_TXCNT_LMT_SEL BIT(11)
+#define FWCMD_H2C_CCTRL_RTS_TXCNT_LMT_SH 12
+#define FWCMD_H2C_CCTRL_RTS_TXCNT_LMT_MSK 0xf
+#define FWCMD_H2C_CCTRL_RTSRATE_SH 16
+#define FWCMD_H2C_CCTRL_RTSRATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_VCS_STBC BIT(27)
+#define FWCMD_H2C_CCTRL_RTS_RTY_LOWEST_RATE_SH 28
+#define FWCMD_H2C_CCTRL_RTS_RTY_LOWEST_RATE_MSK 0xf
+#define FWCMD_H2C_CCTRL_DATA_TX_CNT_LMT_SH 0
+#define FWCMD_H2C_CCTRL_DATA_TX_CNT_LMT_MSK 0x3f
+#define FWCMD_H2C_CCTRL_DATA_TXCNT_LMT_SEL BIT(6)
+#define FWCMD_H2C_CCTRL_MAX_AGG_NUM_SEL BIT(7)
+#define FWCMD_H2C_CCTRL_RTS_EN BIT(8)
+#define FWCMD_H2C_CCTRL_CTS2SELF_EN BIT(9)
+#define FWCMD_H2C_CCTRL_CCA_RTS_SH 10
+#define FWCMD_H2C_CCTRL_CCA_RTS_MSK 0x3
+#define FWCMD_H2C_CCTRL_HW_RTS_EN BIT(12)
+#define FWCMD_H2C_CCTRL_RTS_DROP_DATA_MODE_SH 13
+#define FWCMD_H2C_CCTRL_RTS_DROP_DATA_MODE_MSK 0x3
+#define FWCMD_H2C_CCTRL_PRELD_EN BIT(15)
+#define FWCMD_H2C_CCTRL_AMPDU_MAX_LEN_SH 16
+#define FWCMD_H2C_CCTRL_AMPDU_MAX_LEN_MSK 0x7ff
+#define FWCMD_H2C_CCTRL_UL_MU_DIS BIT(27)
+#define FWCMD_H2C_CCTRL_AMPDU_MAX_TIME_SH 28
+#define FWCMD_H2C_CCTRL_AMPDU_MAX_TIME_MSK 0xf
+#define FWCMD_H2C_CCTRL_MAX_AGG_NUM_SH 0
+#define FWCMD_H2C_CCTRL_MAX_AGG_NUM_MSK 0xff
+#define FWCMD_H2C_CCTRL_BA_BMAP_SH 8
+#define FWCMD_H2C_CCTRL_BA_BMAP_MSK 0x3
+#define FWCMD_H2C_CCTRL_VO_LFTIME_SEL_SH 16
+#define FWCMD_H2C_CCTRL_VO_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_VI_LFTIME_SEL_SH 19
+#define FWCMD_H2C_CCTRL_VI_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_BE_LFTIME_SEL_SH 22
+#define FWCMD_H2C_CCTRL_BE_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_BK_LFTIME_SEL_SH 25
+#define FWCMD_H2C_CCTRL_BK_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_SECTYPE_SH 28
+#define FWCMD_H2C_CCTRL_SECTYPE_MSK 0xf
+#define FWCMD_H2C_CCTRL_MULTI_PORT_ID_SH 0
+#define FWCMD_H2C_CCTRL_MULTI_PORT_ID_MSK 0x7
+#define FWCMD_H2C_CCTRL_BMC BIT(3)
+#define FWCMD_H2C_CCTRL_MBSSID_SH 4
+#define FWCMD_H2C_CCTRL_MBSSID_MSK 0xf
+#define FWCMD_H2C_CCTRL_NAVUSEHDR BIT(8)
+#define FWCMD_H2C_CCTRL_TXPWR_MODE_SH 9
+#define FWCMD_H2C_CCTRL_TXPWR_MODE_MSK 0x7
+#define FWCMD_H2C_CCTRL_DATA_DCM BIT(12)
+#define FWCMD_H2C_CCTRL_DATA_ER BIT(13)
+#define FWCMD_H2C_CCTRL_DATA_LDPC BIT(14)
+#define FWCMD_H2C_CCTRL_DATA_STBC BIT(15)
+#define FWCMD_H2C_CCTRL_A_CTRL_BQR BIT(16)
+#define FWCMD_H2C_CCTRL_A_CTRL_UPH BIT(17)
+#define FWCMD_H2C_CCTRL_A_CTRL_BSR BIT(18)
+#define FWCMD_H2C_CCTRL_A_CTRL_CAS BIT(19)
+#define FWCMD_H2C_CCTRL_DATA_BW_ER BIT(20)
+#define FWCMD_H2C_CCTRL_LSIG_TXOP_EN BIT(21)
+#define FWCMD_H2C_CCTRL_CTRL_CNT_VLD BIT(27)
+#define FWCMD_H2C_CCTRL_CTRL_CNT_SH 28
+#define FWCMD_H2C_CCTRL_CTRL_CNT_MSK 0xf
+#define FWCMD_H2C_CCTRL_RESP_REF_RATE_SH 0
+#define FWCMD_H2C_CCTRL_RESP_REF_RATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_ALL_ACK_SUPPORT BIT(12)
+#define FWCMD_H2C_CCTRL_BSR_QUEUE_SIZE_FORMAT BIT(13)
+#define FWCMD_H2C_CCTRL_NTX_PATH_EN_SH 16
+#define FWCMD_H2C_CCTRL_NTX_PATH_EN_MSK 0xf
+#define FWCMD_H2C_CCTRL_PATH_MAP_A_SH 20
+#define FWCMD_H2C_CCTRL_PATH_MAP_A_MSK 0x3
+#define FWCMD_H2C_CCTRL_PATH_MAP_B_SH 22
+#define FWCMD_H2C_CCTRL_PATH_MAP_B_MSK 0x3
+#define FWCMD_H2C_CCTRL_PATH_MAP_C_SH 24
+#define FWCMD_H2C_CCTRL_PATH_MAP_C_MSK 0x3
+#define FWCMD_H2C_CCTRL_PATH_MAP_D_SH 26
+#define FWCMD_H2C_CCTRL_PATH_MAP_D_MSK 0x3
+#define FWCMD_H2C_CCTRL_ANTSEL_A BIT(28)
+#define FWCMD_H2C_CCTRL_ANTSEL_B BIT(29)
+#define FWCMD_H2C_CCTRL_ANTSEL_C BIT(30)
+#define FWCMD_H2C_CCTRL_ANTSEL_D BIT(31)
+#define FWCMD_H2C_CCTRL_ADDR_CAM_INDEX_SH 0
+#define FWCMD_H2C_CCTRL_ADDR_CAM_INDEX_MSK 0xff
+#define FWCMD_H2C_CCTRL_PAID_SH 8
+#define FWCMD_H2C_CCTRL_PAID_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_ULDL BIT(17)
+#define FWCMD_H2C_CCTRL_DOPPLER_CTRL_SH 18
+#define FWCMD_H2C_CCTRL_DOPPLER_CTRL_MSK 0x3
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING_SH 20
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING_MSK 0x3
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING40_SH 22
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING40_MSK 0x3
+#define FWCMD_H2C_CCTRL_TXPWR_TOLERENCE_SH 24
+#define FWCMD_H2C_CCTRL_TXPWR_TOLERENCE_MSK 0x3f
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING80_SH 30
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING80_MSK 0x3
+#define FWCMD_H2C_CCTRL_NC_SH 0
+#define FWCMD_H2C_CCTRL_NC_MSK 0x7
+#define FWCMD_H2C_CCTRL_NR_SH 3
+#define FWCMD_H2C_CCTRL_NR_MSK 0x7
+#define FWCMD_H2C_CCTRL_NG_SH 6
+#define FWCMD_H2C_CCTRL_NG_MSK 0x3
+#define FWCMD_H2C_CCTRL_CB_SH 8
+#define FWCMD_H2C_CCTRL_CB_MSK 0x3
+#define FWCMD_H2C_CCTRL_CS_SH 10
+#define FWCMD_H2C_CCTRL_CS_MSK 0x3
+#define FWCMD_H2C_CCTRL_CSI_TXBF_EN BIT(12)
+#define FWCMD_H2C_CCTRL_CSI_STBC_EN BIT(13)
+#define FWCMD_H2C_CCTRL_CSI_LDPC_EN BIT(14)
+#define FWCMD_H2C_CCTRL_CSI_PARA_EN BIT(15)
+#define FWCMD_H2C_CCTRL_CSI_FIX_RATE_SH 16
+#define FWCMD_H2C_CCTRL_CSI_FIX_RATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_CSI_GI_LTF_SH 25
+#define FWCMD_H2C_CCTRL_CSI_GI_LTF_MSK 0x7
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING160_SH 28
+#define FWCMD_H2C_CCTRL_NOMINAL_PKT_PADDING160_MSK 0x3
+#define FWCMD_H2C_CCTRL_CSI_BW_SH 30
+#define FWCMD_H2C_CCTRL_CSI_BW_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_QOS_FIELD_H_SH 0
+#define FWCMD_H2C_DCTRL_V1_QOS_FIELD_H_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_HW_EXSEQ_MACID_SH 8
+#define FWCMD_H2C_DCTRL_V1_HW_EXSEQ_MACID_MSK 0x7f
+#define FWCMD_H2C_DCTRL_V1_QOS_DATA BIT(15)
+#define FWCMD_H2C_DCTRL_V1_AES_IV_L_SH 16
+#define FWCMD_H2C_DCTRL_V1_AES_IV_L_MSK 0xffff
+#define FWCMD_H2C_DCTRL_V1_AES_IV_H_SH 0
+#define FWCMD_H2C_DCTRL_V1_AES_IV_H_MSK 0xffffffff
+#define FWCMD_H2C_DCTRL_V1_SEQ0_SH 0
+#define FWCMD_H2C_DCTRL_V1_SEQ0_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V1_SEQ1_SH 12
+#define FWCMD_H2C_DCTRL_V1_SEQ1_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V1_AMSDU_MAX_LEN_SH 24
+#define FWCMD_H2C_DCTRL_V1_AMSDU_MAX_LEN_MSK 0x7
+#define FWCMD_H2C_DCTRL_V1_STA_AMSDU_EN BIT(27)
+#define FWCMD_H2C_DCTRL_V1_CHKSUM_OFLD_EN BIT(28)
+#define FWCMD_H2C_DCTRL_V1_WITH_LLC BIT(29)
+#define FWCMD_H2C_DCTRL_V1_SEQ2_SH 0
+#define FWCMD_H2C_DCTRL_V1_SEQ2_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V1_SEQ3_SH 12
+#define FWCMD_H2C_DCTRL_V1_SEQ3_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V1_TGT_IND_SH 24
+#define FWCMD_H2C_DCTRL_V1_TGT_IND_MSK 0xf
+#define FWCMD_H2C_DCTRL_V1_TGT_IND_EN BIT(28)
+#define FWCMD_H2C_DCTRL_V1_HTC_LB_SH 29
+#define FWCMD_H2C_DCTRL_V1_HTC_LB_MSK 0x7
+#define FWCMD_H2C_DCTRL_V1_MHDR_LEN_SH 0
+#define FWCMD_H2C_DCTRL_V1_MHDR_LEN_MSK 0x1f
+#define FWCMD_H2C_DCTRL_V1_VLAN_TAG_VALID BIT(5)
+#define FWCMD_H2C_DCTRL_V1_VLAN_TAG_SEL_SH 6
+#define FWCMD_H2C_DCTRL_V1_VLAN_TAG_SEL_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_HTC_ORDER BIT(8)
+#define FWCMD_H2C_DCTRL_V1_SEC_KEY_ID_SH 9
+#define FWCMD_H2C_DCTRL_V1_SEC_KEY_ID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_RVSD_SH 11
+#define FWCMD_H2C_DCTRL_V1_RVSD_MSK 0xf
+#define FWCMD_H2C_DCTRL_V1_WAPI BIT(15)
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT_MODE_SH 16
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT_MODE_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT0_KEYID_SH 18
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT0_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT1_KEYID_SH 20
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT1_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT2_KEYID_SH 22
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT2_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT3_KEYID_SH 24
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT3_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT4_KEYID_SH 26
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT4_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT5_KEYID_SH 28
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT5_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT6_KEYID_SH 30
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT6_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT_VALID_SH 0
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT_VALID_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT0_SH 8
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT0_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT1_SH 16
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT1_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT2_SH 24
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT2_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT3_SH 0
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT3_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT4_SH 8
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT4_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT5_SH 16
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT5_MSK 0xff
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT6_SH 24
+#define FWCMD_H2C_DCTRL_V1_SEC_ENT6_MSK 0xff
+#define FWCMD_H2C_CCTRL_V1_DATARATE_SH 0
+#define FWCMD_H2C_CCTRL_V1_DATARATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_V1_FORCE_TXOP BIT(9)
+#define FWCMD_H2C_CCTRL_V1_DATA_BW_SH 10
+#define FWCMD_H2C_CCTRL_V1_DATA_BW_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_DATA_GI_LTF_SH 12
+#define FWCMD_H2C_CCTRL_V1_DATA_GI_LTF_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_DARF_TC_INDEX BIT(15)
+#define FWCMD_H2C_CCTRL_V1_ARFR_CTRL_SH 16
+#define FWCMD_H2C_CCTRL_V1_ARFR_CTRL_MSK 0xf
+#define FWCMD_H2C_CCTRL_V1_ACQ_RPT_EN BIT(20)
+#define FWCMD_H2C_CCTRL_V1_MGQ_RPT_EN BIT(21)
+#define FWCMD_H2C_CCTRL_V1_ULQ_RPT_EN BIT(22)
+#define FWCMD_H2C_CCTRL_V1_TWTQ_RPT_EN BIT(23)
+#define FWCMD_H2C_CCTRL_V1_DISRTSFB BIT(25)
+#define FWCMD_H2C_CCTRL_V1_DISDATAFB BIT(26)
+#define FWCMD_H2C_CCTRL_V1_TRYRATE BIT(27)
+#define FWCMD_H2C_CCTRL_V1_AMPDU_DENSITY_SH 28
+#define FWCMD_H2C_CCTRL_V1_AMPDU_DENSITY_MSK 0xf
+#define FWCMD_H2C_CCTRL_V1_DATA_RTY_LOWEST_RATE_SH 0
+#define FWCMD_H2C_CCTRL_V1_DATA_RTY_LOWEST_RATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_V1_AMPDU_TIME_SEL BIT(9)
+#define FWCMD_H2C_CCTRL_V1_AMPDU_LEN_SEL BIT(10)
+#define FWCMD_H2C_CCTRL_V1_RTS_TXCNT_LMT_SEL BIT(11)
+#define FWCMD_H2C_CCTRL_V1_RTS_TXCNT_LMT_SH 12
+#define FWCMD_H2C_CCTRL_V1_RTS_TXCNT_LMT_MSK 0xf
+#define FWCMD_H2C_CCTRL_V1_RTSRATE_SH 16
+#define FWCMD_H2C_CCTRL_V1_RTSRATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_V1_VCS_STBC BIT(27)
+#define FWCMD_H2C_CCTRL_V1_RTS_RTY_LOWEST_RATE_SH 28
+#define FWCMD_H2C_CCTRL_V1_RTS_RTY_LOWEST_RATE_MSK 0xf
+#define FWCMD_H2C_CCTRL_V1_DATA_TX_CNT_LMT_SH 0
+#define FWCMD_H2C_CCTRL_V1_DATA_TX_CNT_LMT_MSK 0x3f
+#define FWCMD_H2C_CCTRL_V1_DATA_TXCNT_LMT_SEL BIT(6)
+#define FWCMD_H2C_CCTRL_V1_MAX_AGG_NUM_SEL BIT(7)
+#define FWCMD_H2C_CCTRL_V1_RTS_EN BIT(8)
+#define FWCMD_H2C_CCTRL_V1_CTS2SELF_EN BIT(9)
+#define FWCMD_H2C_CCTRL_V1_CCA_RTS_SH 10
+#define FWCMD_H2C_CCTRL_V1_CCA_RTS_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_HW_RTS_EN BIT(12)
+#define FWCMD_H2C_CCTRL_V1_RTS_DROP_DATA_MODE_SH 13
+#define FWCMD_H2C_CCTRL_V1_RTS_DROP_DATA_MODE_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_PRELD_EN BIT(15)
+#define FWCMD_H2C_CCTRL_V1_AMPDU_MAX_LEN_SH 16
+#define FWCMD_H2C_CCTRL_V1_AMPDU_MAX_LEN_MSK 0x7ff
+#define FWCMD_H2C_CCTRL_V1_UL_MU_DIS BIT(27)
+#define FWCMD_H2C_CCTRL_V1_AMPDU_MAX_TIME_SH 28
+#define FWCMD_H2C_CCTRL_V1_AMPDU_MAX_TIME_MSK 0xf
+#define FWCMD_H2C_CCTRL_V1_MAX_AGG_NUM_SH 0
+#define FWCMD_H2C_CCTRL_V1_MAX_AGG_NUM_MSK 0xff
+#define FWCMD_H2C_CCTRL_V1_BA_BMAP_SH 8
+#define FWCMD_H2C_CCTRL_V1_BA_BMAP_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_VO_LFTIME_SEL_SH 16
+#define FWCMD_H2C_CCTRL_V1_VO_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_VI_LFTIME_SEL_SH 19
+#define FWCMD_H2C_CCTRL_V1_VI_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_BE_LFTIME_SEL_SH 22
+#define FWCMD_H2C_CCTRL_V1_BE_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_BK_LFTIME_SEL_SH 25
+#define FWCMD_H2C_CCTRL_V1_BK_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_MULTI_PORT_ID_SH 0
+#define FWCMD_H2C_CCTRL_V1_MULTI_PORT_ID_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_BMC BIT(3)
+#define FWCMD_H2C_CCTRL_V1_MBSSID_SH 4
+#define FWCMD_H2C_CCTRL_V1_MBSSID_MSK 0xf
+#define FWCMD_H2C_CCTRL_V1_DATA_DCM BIT(12)
+#define FWCMD_H2C_CCTRL_V1_DATA_ER BIT(13)
+#define FWCMD_H2C_CCTRL_V1_DATA_LDPC BIT(14)
+#define FWCMD_H2C_CCTRL_V1_DATA_STBC BIT(15)
+#define FWCMD_H2C_CCTRL_V1_A_CTRL_BQR BIT(16)
+#define FWCMD_H2C_CCTRL_V1_A_CTRL_UPH BIT(17)
+#define FWCMD_H2C_CCTRL_V1_A_CTRL_BSR BIT(18)
+#define FWCMD_H2C_CCTRL_V1_A_CTRL_CAS BIT(19)
+#define FWCMD_H2C_CCTRL_V1_DATA_BW_ER BIT(20)
+#define FWCMD_H2C_CCTRL_V1_LSIG_TXOP_EN BIT(21)
+#define FWCMD_H2C_CCTRL_V1_CTRL_CNT_VLD BIT(27)
+#define FWCMD_H2C_CCTRL_V1_CTRL_CNT_SH 28
+#define FWCMD_H2C_CCTRL_V1_CTRL_CNT_MSK 0xf
+#define FWCMD_H2C_CCTRL_V1_RESP_REF_RATE_SH 0
+#define FWCMD_H2C_CCTRL_V1_RESP_REF_RATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_V1_ALL_ACK_SUPPORT BIT(12)
+#define FWCMD_H2C_CCTRL_V1_BSR_QUEUE_SIZE_FORMAT BIT(13)
+#define FWCMD_H2C_CCTRL_V1_BSR_OM_UPD_EN BIT(14)
+#define FWCMD_H2C_CCTRL_V1_MACID_FWD_IDC BIT(15)
+#define FWCMD_H2C_CCTRL_V1_ADDR_CAM_INDEX_SH 24
+#define FWCMD_H2C_CCTRL_V1_ADDR_CAM_INDEX_MSK 0xff
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING20_SH 0
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING20_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING40_SH 2
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING40_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING80_SH 4
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING80_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING160_SH 6
+#define FWCMD_H2C_CCTRL_V1_NOMINAL_PKT_PADDING160_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_PAID_SH 8
+#define FWCMD_H2C_CCTRL_V1_PAID_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_V1_ULDL BIT(17)
+#define FWCMD_H2C_CCTRL_V1_DOPPLER_CTRL_SH 18
+#define FWCMD_H2C_CCTRL_V1_DOPPLER_CTRL_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_TXPWR_TOLERENCE_SH 24
+#define FWCMD_H2C_CCTRL_V1_TXPWR_TOLERENCE_MSK 0x3f
+#define FWCMD_H2C_CCTRL_V1_NC_SH 0
+#define FWCMD_H2C_CCTRL_V1_NC_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_NR_SH 3
+#define FWCMD_H2C_CCTRL_V1_NR_MSK 0x7
+#define FWCMD_H2C_CCTRL_V1_NG_SH 6
+#define FWCMD_H2C_CCTRL_V1_NG_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_CB_SH 8
+#define FWCMD_H2C_CCTRL_V1_CB_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_CS_SH 10
+#define FWCMD_H2C_CCTRL_V1_CS_MSK 0x3
+#define FWCMD_H2C_CCTRL_V1_CSI_TXBF_EN BIT(12)
+#define FWCMD_H2C_CCTRL_V1_CSI_STBC_EN BIT(13)
+#define FWCMD_H2C_CCTRL_V1_CSI_LDPC_EN BIT(14)
+#define FWCMD_H2C_CCTRL_V1_CSI_PARA_EN BIT(15)
+#define FWCMD_H2C_CCTRL_V1_CSI_FIX_RATE_SH 16
+#define FWCMD_H2C_CCTRL_V1_CSI_FIX_RATE_MSK 0x1ff
+#define FWCMD_H2C_CCTRL_V1_CSI_BW_SH 25
+#define FWCMD_H2C_CCTRL_V1_CSI_BW_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_QOS_FIELD_H_SH 0
+#define FWCMD_H2C_DCTRL_V2_QOS_FIELD_H_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_HW_EXSEQ_MACID_SH 8
+#define FWCMD_H2C_DCTRL_V2_HW_EXSEQ_MACID_MSK 0x7f
+#define FWCMD_H2C_DCTRL_V2_QOS_DATA BIT(15)
+#define FWCMD_H2C_DCTRL_V2_AES_IV_L_SH 16
+#define FWCMD_H2C_DCTRL_V2_AES_IV_L_MSK 0xffff
+#define FWCMD_H2C_DCTRL_V2_AES_IV_H_SH 0
+#define FWCMD_H2C_DCTRL_V2_AES_IV_H_MSK 0xffffffff
+#define FWCMD_H2C_DCTRL_V2_SEQ0_SH 0
+#define FWCMD_H2C_DCTRL_V2_SEQ0_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V2_SEQ1_SH 12
+#define FWCMD_H2C_DCTRL_V2_SEQ1_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V2_AMSDU_MAX_LEN_SH 24
+#define FWCMD_H2C_DCTRL_V2_AMSDU_MAX_LEN_MSK 0x7
+#define FWCMD_H2C_DCTRL_V2_STA_AMSDU_EN BIT(27)
+#define FWCMD_H2C_DCTRL_V2_CHKSUM_OFLD_EN BIT(28)
+#define FWCMD_H2C_DCTRL_V2_WITH_LLC BIT(29)
+#define FWCMD_H2C_DCTRL_V2_NAT25_EN BIT(30)
+#define FWCMD_H2C_DCTRL_V2_IS_MLD BIT(31)
+#define FWCMD_H2C_DCTRL_V2_SEQ2_SH 0
+#define FWCMD_H2C_DCTRL_V2_SEQ2_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V2_SEQ3_SH 12
+#define FWCMD_H2C_DCTRL_V2_SEQ3_MSK 0xfff
+#define FWCMD_H2C_DCTRL_V2_TGT_IND_SH 24
+#define FWCMD_H2C_DCTRL_V2_TGT_IND_MSK 0xf
+#define FWCMD_H2C_DCTRL_V2_TGT_IND_EN BIT(28)
+#define FWCMD_H2C_DCTRL_V2_HTC_LB_SH 29
+#define FWCMD_H2C_DCTRL_V2_HTC_LB_MSK 0x7
+#define FWCMD_H2C_DCTRL_V2_VLAN_TAG_SEL_SH 5
+#define FWCMD_H2C_DCTRL_V2_VLAN_TAG_SEL_MSK 0x7
+#define FWCMD_H2C_DCTRL_V2_HTC_ORDER BIT(8)
+#define FWCMD_H2C_DCTRL_V2_SEC_KEY_ID_SH 9
+#define FWCMD_H2C_DCTRL_V2_SEC_KEY_ID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_VLAN_RX_DYNAMIC_PCP_EN BIT(11)
+#define FWCMD_H2C_DCTRL_V2_VLAN_RX_PKT_DROP BIT(12)
+#define FWCMD_H2C_DCTRL_V2_VLAN_RX_VALID BIT(13)
+#define FWCMD_H2C_DCTRL_V2_VLAN_TX_VALID BIT(14)
+#define FWCMD_H2C_DCTRL_V2_WAPI BIT(15)
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT_MODE_SH 16
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT_MODE_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT0_KEYID_SH 18
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT0_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT1_KEYID_SH 20
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT1_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT2_KEYID_SH 22
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT2_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT3_KEYID_SH 24
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT3_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT4_KEYID_SH 26
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT4_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT5_KEYID_SH 28
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT5_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT6_KEYID_SH 30
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT6_KEYID_MSK 0x3
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT_VALID_SH 0
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT_VALID_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT0_SH 8
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT0_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT1_SH 16
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT1_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT2_SH 24
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT2_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT3_SH 0
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT3_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT4_SH 8
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT4_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT5_SH 16
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT5_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT6_SH 24
+#define FWCMD_H2C_DCTRL_V2_SEC_ENT6_MSK 0xff
+#define FWCMD_H2C_DCTRL_V2_MLD_SMA_L_SH 0
+#define FWCMD_H2C_DCTRL_V2_MLD_SMA_L_MSK 0xffffffff
+#define FWCMD_H2C_DCTRL_V2_MLD_SMA_H_SH 0
+#define FWCMD_H2C_DCTRL_V2_MLD_SMA_H_MSK 0xffff
+#define FWCMD_H2C_DCTRL_V2_MLD_TMA_L_SH 16
+#define FWCMD_H2C_DCTRL_V2_MLD_TMA_L_MSK 0xffff
+#define FWCMD_H2C_DCTRL_V2_MLD_TMA_H_SH 0
+#define FWCMD_H2C_DCTRL_V2_MLD_TMA_H_MSK 0xffffffff
+#define FWCMD_H2C_DCTRL_V2_MLD_BSSID_MAC_L_SH 0
+#define FWCMD_H2C_DCTRL_V2_MLD_BSSID_MAC_L_MSK 0xffffffff
+#define FWCMD_H2C_DCTRL_V2_MLD_BSSID_MAC_H_SH 0
+#define FWCMD_H2C_DCTRL_V2_MLD_BSSID_MAC_H_MSK 0xffff
+#define FWCMD_H2C_CCTRL_V2_DATARATE_SH 0
+#define FWCMD_H2C_CCTRL_V2_DATARATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_V2_DATA_GI_LTF_SH 12
+#define FWCMD_H2C_CCTRL_V2_DATA_GI_LTF_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_TRYRATE BIT(15)
+#define FWCMD_H2C_CCTRL_V2_ARFR_CTRL_SH 16
+#define FWCMD_H2C_CCTRL_V2_ARFR_CTRL_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_DIS_HE1SS_STBC BIT(18)
+#define FWCMD_H2C_CCTRL_V2_ACQ_RPT_EN BIT(20)
+#define FWCMD_H2C_CCTRL_V2_MGQ_RPT_EN BIT(21)
+#define FWCMD_H2C_CCTRL_V2_ULQ_RPT_EN BIT(22)
+#define FWCMD_H2C_CCTRL_V2_TWTQ_RPT_EN BIT(23)
+#define FWCMD_H2C_CCTRL_V2_FORCE_TXOP BIT(24)
+#define FWCMD_H2C_CCTRL_V2_DISRTSFB BIT(25)
+#define FWCMD_H2C_CCTRL_V2_DISDATAFB BIT(26)
+#define FWCMD_H2C_CCTRL_V2_NSTR_EN BIT(27)
+#define FWCMD_H2C_CCTRL_V2_AMPDU_DENSITY_SH 28
+#define FWCMD_H2C_CCTRL_V2_AMPDU_DENSITY_MSK 0xf
+#define FWCMD_H2C_CCTRL_V2_DATA_RTY_LOWEST_RATE_SH 0
+#define FWCMD_H2C_CCTRL_V2_DATA_RTY_LOWEST_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_V2_RTS_TXCNT_LMT_SH 12
+#define FWCMD_H2C_CCTRL_V2_RTS_TXCNT_LMT_MSK 0xf
+#define FWCMD_H2C_CCTRL_V2_RTSRATE_SH 16
+#define FWCMD_H2C_CCTRL_V2_RTSRATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_V2_RTS_RTY_LOWEST_RATE_SH 28
+#define FWCMD_H2C_CCTRL_V2_RTS_RTY_LOWEST_RATE_MSK 0xf
+#define FWCMD_H2C_CCTRL_V2_DATA_TX_CNT_LMT_SH 0
+#define FWCMD_H2C_CCTRL_V2_DATA_TX_CNT_LMT_MSK 0x3f
+#define FWCMD_H2C_CCTRL_V2_DATA_TXCNT_LMT_SEL BIT(6)
+#define FWCMD_H2C_CCTRL_V2_MAX_AGG_NUM_SEL BIT(7)
+#define FWCMD_H2C_CCTRL_V2_RTS_EN BIT(8)
+#define FWCMD_H2C_CCTRL_V2_CTS2SELF_EN BIT(9)
+#define FWCMD_H2C_CCTRL_V2_CCA_RTS_SH 10
+#define FWCMD_H2C_CCTRL_V2_CCA_RTS_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_HW_RTS_EN BIT(12)
+#define FWCMD_H2C_CCTRL_V2_RTS_DROP_DATA_MODE_SH 13
+#define FWCMD_H2C_CCTRL_V2_RTS_DROP_DATA_MODE_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_PRELD_EN BIT(15)
+#define FWCMD_H2C_CCTRL_V2_AMPDU_MAX_LEN_SH 16
+#define FWCMD_H2C_CCTRL_V2_AMPDU_MAX_LEN_MSK 0x7ff
+#define FWCMD_H2C_CCTRL_V2_UL_MU_DIS BIT(27)
+#define FWCMD_H2C_CCTRL_V2_AMPDU_MAX_TIME_SH 28
+#define FWCMD_H2C_CCTRL_V2_AMPDU_MAX_TIME_MSK 0xf
+#define FWCMD_H2C_CCTRL_V2_MAX_AGG_NUM_SH 0
+#define FWCMD_H2C_CCTRL_V2_MAX_AGG_NUM_MSK 0xff
+#define FWCMD_H2C_CCTRL_V2_DATA_BW_SH 8
+#define FWCMD_H2C_CCTRL_V2_DATA_BW_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_DATA_BW_ER BIT(11)
+#define FWCMD_H2C_CCTRL_V2_BA_BMAP_SH 12
+#define FWCMD_H2C_CCTRL_V2_BA_BMAP_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_VCS_STBC BIT(15)
+#define FWCMD_H2C_CCTRL_V2_VO_LFTIME_SEL_SH 16
+#define FWCMD_H2C_CCTRL_V2_VO_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_VI_LFTIME_SEL_SH 19
+#define FWCMD_H2C_CCTRL_V2_VI_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_BE_LFTIME_SEL_SH 22
+#define FWCMD_H2C_CCTRL_V2_BE_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_BK_LFTIME_SEL_SH 25
+#define FWCMD_H2C_CCTRL_V2_BK_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_AMPDU_TIME_SEL BIT(28)
+#define FWCMD_H2C_CCTRL_V2_AMPDU_LEN_SEL BIT(29)
+#define FWCMD_H2C_CCTRL_V2_RTS_TXCNT_LMT_SEL BIT(30)
+#define FWCMD_H2C_CCTRL_V2_LSIG_TXOP_EN BIT(31)
+#define FWCMD_H2C_CCTRL_V2_MULTI_PORT_ID_SH 0
+#define FWCMD_H2C_CCTRL_V2_MULTI_PORT_ID_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_BYPASS_PUNC BIT(3)
+#define FWCMD_H2C_CCTRL_V2_MBSSID_SH 4
+#define FWCMD_H2C_CCTRL_V2_MBSSID_MSK 0xf
+#define FWCMD_H2C_CCTRL_V2_DATA_DCM BIT(8)
+#define FWCMD_H2C_CCTRL_V2_DATA_ER BIT(9)
+#define FWCMD_H2C_CCTRL_V2_DATA_LDPC BIT(10)
+#define FWCMD_H2C_CCTRL_V2_DATA_STBC BIT(11)
+#define FWCMD_H2C_CCTRL_V2_A_CTRL_BQR BIT(12)
+#define FWCMD_H2C_CCTRL_V2_A_CTRL_BSR BIT(14)
+#define FWCMD_H2C_CCTRL_V2_A_CTRL_CAS BIT(15)
+#define FWCMD_H2C_CCTRL_V2_ACT_SUBCH_CBW_SH 16
+#define FWCMD_H2C_CCTRL_V2_ACT_SUBCH_CBW_MSK 0xffff
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING0_SH 0
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING0_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING1_SH 2
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING1_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING2_SH 4
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING2_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING3_SH 6
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING3_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING4_SH 8
+#define FWCMD_H2C_CCTRL_V2_NOMINAL_PKT_PADDING4_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_DOPPLER_CTRL_SH 10
+#define FWCMD_H2C_CCTRL_V2_DOPPLER_CTRL_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_ALL_ACK_SUPPORT BIT(12)
+#define FWCMD_H2C_CCTRL_V2_BSR_QUEUE_SIZE_FORMAT BIT(13)
+#define FWCMD_H2C_CCTRL_V2_BSR_OM_UPD_EN BIT(14)
+#define FWCMD_H2C_CCTRL_V2_MACID_FWD_IDC BIT(15)
+#define FWCMD_H2C_CCTRL_V2_FIX_UL_ADDRCAM_IDX BIT(18)
+#define FWCMD_H2C_CCTRL_V2_CTRL_CNT_VLD BIT(19)
+#define FWCMD_H2C_CCTRL_V2_CTRL_CNT_SH 20
+#define FWCMD_H2C_CCTRL_V2_CTRL_CNT_MSK 0xf
+#define FWCMD_H2C_CCTRL_V2_ADDR_CAM_INDEX_SH 24
+#define FWCMD_H2C_CCTRL_V2_ADDR_CAM_INDEX_MSK 0xff
+#define FWCMD_H2C_CCTRL_V2_AID12_PAID_SH 0
+#define FWCMD_H2C_CCTRL_V2_AID12_PAID_MSK 0xfff
+#define FWCMD_H2C_CCTRL_V2_RESP_REF_RATE_SH 12
+#define FWCMD_H2C_CCTRL_V2_RESP_REF_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_V2_TXPWR_TOLERENCE_SH 24
+#define FWCMD_H2C_CCTRL_V2_TXPWR_TOLERENCE_MSK 0x3f
+#define FWCMD_H2C_CCTRL_V2_ULDL BIT(31)
+#define FWCMD_H2C_CCTRL_V2_NC_SH 0
+#define FWCMD_H2C_CCTRL_V2_NC_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_NR_SH 3
+#define FWCMD_H2C_CCTRL_V2_NR_MSK 0x7
+#define FWCMD_H2C_CCTRL_V2_NG_SH 6
+#define FWCMD_H2C_CCTRL_V2_NG_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_CB_SH 8
+#define FWCMD_H2C_CCTRL_V2_CB_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_CS_SH 10
+#define FWCMD_H2C_CCTRL_V2_CS_MSK 0x3
+#define FWCMD_H2C_CCTRL_V2_CSI_STBC_EN BIT(13)
+#define FWCMD_H2C_CCTRL_V2_CSI_LDPC_EN BIT(14)
+#define FWCMD_H2C_CCTRL_V2_CSI_PARA_EN BIT(15)
+#define FWCMD_H2C_CCTRL_V2_CSI_FIX_RATE_SH 16
+#define FWCMD_H2C_CCTRL_V2_CSI_FIX_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_V2_CSI_BW_SH 29
+#define FWCMD_H2C_CCTRL_V2_CSI_BW_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_DATARATE_SH 0
+#define FWCMD_H2C_CCTRL_G7_DATARATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_DATA_GI_LTF_SH 12
+#define FWCMD_H2C_CCTRL_G7_DATA_GI_LTF_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_TRYRATE BIT(15)
+#define FWCMD_H2C_CCTRL_G7_ARFR_CTRL_SH 16
+#define FWCMD_H2C_CCTRL_G7_ARFR_CTRL_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_DIS_HE1SS_STBC BIT(18)
+#define FWCMD_H2C_CCTRL_G7_ACQ_RPT_EN BIT(20)
+#define FWCMD_H2C_CCTRL_G7_MGQ_RPT_EN BIT(21)
+#define FWCMD_H2C_CCTRL_G7_ULQ_RPT_EN BIT(22)
+#define FWCMD_H2C_CCTRL_G7_TWTQ_RPT_EN BIT(23)
+#define FWCMD_H2C_CCTRL_G7_FORCE_TXOP BIT(24)
+#define FWCMD_H2C_CCTRL_G7_DISRTSFB BIT(25)
+#define FWCMD_H2C_CCTRL_G7_DISDATAFB BIT(26)
+#define FWCMD_H2C_CCTRL_G7_NSTR_EN BIT(27)
+#define FWCMD_H2C_CCTRL_G7_AMPDU_DENSITY_SH 28
+#define FWCMD_H2C_CCTRL_G7_AMPDU_DENSITY_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_DATA_RTY_LOWEST_RATE_SH 0
+#define FWCMD_H2C_CCTRL_G7_DATA_RTY_LOWEST_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_RTS_TXCNT_LMT_SH 12
+#define FWCMD_H2C_CCTRL_G7_RTS_TXCNT_LMT_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_RTSRATE_SH 16
+#define FWCMD_H2C_CCTRL_G7_RTSRATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_RTS_RTY_LOWEST_RATE_SH 28
+#define FWCMD_H2C_CCTRL_G7_RTS_RTY_LOWEST_RATE_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_DATA_TX_CNT_LMT_SH 0
+#define FWCMD_H2C_CCTRL_G7_DATA_TX_CNT_LMT_MSK 0x3f
+#define FWCMD_H2C_CCTRL_G7_DATA_TXCNT_LMT_SEL BIT(6)
+#define FWCMD_H2C_CCTRL_G7_MAX_AGG_NUM_SEL BIT(7)
+#define FWCMD_H2C_CCTRL_G7_RTS_EN BIT(8)
+#define FWCMD_H2C_CCTRL_G7_CTS2SELF_EN BIT(9)
+#define FWCMD_H2C_CCTRL_G7_CCA_RTS_SH 10
+#define FWCMD_H2C_CCTRL_G7_CCA_RTS_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_HW_RTS_EN BIT(12)
+#define FWCMD_H2C_CCTRL_G7_RTS_DROP_DATA_MODE_SH 13
+#define FWCMD_H2C_CCTRL_G7_RTS_DROP_DATA_MODE_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_PRELD_EN BIT(15)
+#define FWCMD_H2C_CCTRL_G7_AMPDU_MAX_LEN_SH 16
+#define FWCMD_H2C_CCTRL_G7_AMPDU_MAX_LEN_MSK 0x7ff
+#define FWCMD_H2C_CCTRL_G7_UL_MU_DIS BIT(27)
+#define FWCMD_H2C_CCTRL_G7_AMPDU_MAX_TIME_SH 28
+#define FWCMD_H2C_CCTRL_G7_AMPDU_MAX_TIME_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_MAX_AGG_NUM_SH 0
+#define FWCMD_H2C_CCTRL_G7_MAX_AGG_NUM_MSK 0xff
+#define FWCMD_H2C_CCTRL_G7_DATA_BW_SH 8
+#define FWCMD_H2C_CCTRL_G7_DATA_BW_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_DATA_BW_ER BIT(11)
+#define FWCMD_H2C_CCTRL_G7_BA_BMAP_SH 12
+#define FWCMD_H2C_CCTRL_G7_BA_BMAP_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_VCS_STBC BIT(15)
+#define FWCMD_H2C_CCTRL_G7_VO_LFTIME_SEL_SH 16
+#define FWCMD_H2C_CCTRL_G7_VO_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_VI_LFTIME_SEL_SH 19
+#define FWCMD_H2C_CCTRL_G7_VI_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_BE_LFTIME_SEL_SH 22
+#define FWCMD_H2C_CCTRL_G7_BE_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_BK_LFTIME_SEL_SH 25
+#define FWCMD_H2C_CCTRL_G7_BK_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_AMPDU_TIME_SEL BIT(28)
+#define FWCMD_H2C_CCTRL_G7_AMPDU_LEN_SEL BIT(29)
+#define FWCMD_H2C_CCTRL_G7_RTS_TXCNT_LMT_SEL BIT(30)
+#define FWCMD_H2C_CCTRL_G7_LSIG_TXOP_EN BIT(31)
+#define FWCMD_H2C_CCTRL_G7_MULTI_PORT_ID_SH 0
+#define FWCMD_H2C_CCTRL_G7_MULTI_PORT_ID_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_BYPASS_PUNC BIT(3)
+#define FWCMD_H2C_CCTRL_G7_MBSSID_SH 4
+#define FWCMD_H2C_CCTRL_G7_MBSSID_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_DATA_DCM BIT(8)
+#define FWCMD_H2C_CCTRL_G7_DATA_ER BIT(9)
+#define FWCMD_H2C_CCTRL_G7_DATA_LDPC BIT(10)
+#define FWCMD_H2C_CCTRL_G7_DATA_STBC BIT(11)
+#define FWCMD_H2C_CCTRL_G7_A_CTRL_BQR BIT(12)
+#define FWCMD_H2C_CCTRL_G7_A_CTRL_BSR BIT(14)
+#define FWCMD_H2C_CCTRL_G7_A_CTRL_CAS BIT(15)
+#define FWCMD_H2C_CCTRL_G7_ACT_SUBCH_CBW_SH 16
+#define FWCMD_H2C_CCTRL_G7_ACT_SUBCH_CBW_MSK 0xffff
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING0_SH 0
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING0_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING1_SH 2
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING1_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING2_SH 4
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING2_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING3_SH 6
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING3_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING4_SH 8
+#define FWCMD_H2C_CCTRL_G7_NOMINAL_PKT_PADDING4_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_SR_RATE_SH 10
+#define FWCMD_H2C_CCTRL_G7_SR_RATE_MSK 0x1f
+#define FWCMD_H2C_CCTRL_G7_TID_DISABLE_SH 16
+#define FWCMD_H2C_CCTRL_G7_TID_DISABLE_MSK 0xff
+#define FWCMD_H2C_CCTRL_G7_ADDR_CAM_INDEX_SH 24
+#define FWCMD_H2C_CCTRL_G7_ADDR_CAM_INDEX_MSK 0xff
+#define FWCMD_H2C_CCTRL_G7_AID12_PAID_SH 0
+#define FWCMD_H2C_CCTRL_G7_AID12_PAID_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_RESP_REF_RATE_SH 12
+#define FWCMD_H2C_CCTRL_G7_RESP_REF_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_ULDL BIT(31)
+#define FWCMD_H2C_CCTRL_G7_NC_SH 0
+#define FWCMD_H2C_CCTRL_G7_NC_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_NR_SH 3
+#define FWCMD_H2C_CCTRL_G7_NR_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_NG_SH 6
+#define FWCMD_H2C_CCTRL_G7_NG_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_CB_SH 8
+#define FWCMD_H2C_CCTRL_G7_CB_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_CS_SH 10
+#define FWCMD_H2C_CCTRL_G7_CS_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_CSI_STBC_EN BIT(13)
+#define FWCMD_H2C_CCTRL_G7_CSI_LDPC_EN BIT(14)
+#define FWCMD_H2C_CCTRL_G7_CSI_PARA_EN BIT(15)
+#define FWCMD_H2C_CCTRL_G7_CSI_FIX_RATE_SH 16
+#define FWCMD_H2C_CCTRL_G7_CSI_FIX_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_CSI_BW_SH 29
+#define FWCMD_H2C_CCTRL_G7_CSI_BW_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_ALL_ACK_SUPPORT BIT(0)
+#define FWCMD_H2C_CCTRL_G7_BSR_QUEUE_SIZE_FORMAT BIT(1)
+#define FWCMD_H2C_CCTRL_G7_BSR_OM_UPD_EN BIT(2)
+#define FWCMD_H2C_CCTRL_G7_MACID_FWD_IDC BIT(3)
+#define FWCMD_H2C_CCTRL_G7_AZ_SEC_EN BIT(4)
+#define FWCMD_H2C_CCTRL_G7_CSI_SEC_EN BIT(5)
+#define FWCMD_H2C_CCTRL_G7_FIX_UL_ADDRCAM_IDX BIT(6)
+#define FWCMD_H2C_CCTRL_G7_CTRL_CNT_VLD BIT(7)
+#define FWCMD_H2C_CCTRL_G7_CTRL_CNT_SH 8
+#define FWCMD_H2C_CCTRL_G7_CTRL_CNT_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_RESP_SEC_TYPE_SH 12
+#define FWCMD_H2C_CCTRL_G7_RESP_SEC_TYPE_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_VO_CURR_RATE_SH 0
+#define FWCMD_H2C_CCTRL_G7_VO_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_VI_CURR_RATE_SH 12
+#define FWCMD_H2C_CCTRL_G7_VI_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_BE_CURR_RATE_L_SH 24
+#define FWCMD_H2C_CCTRL_G7_BE_CURR_RATE_L_MSK 0xff
+#define FWCMD_H2C_CCTRL_G7_BE_CURR_RATE_H_SH 0
+#define FWCMD_H2C_CCTRL_G7_BE_CURR_RATE_H_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_BK_CURR_RATE_SH 4
+#define FWCMD_H2C_CCTRL_G7_BK_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_MGNT_CURR_RATE_SH 16
+#define FWCMD_H2C_CCTRL_G7_MGNT_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_MACID_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_MACID_MSK 0x3ff
+#define FWCMD_H2C_CCTRL_G7_V1_OP BIT(10)
+#define FWCMD_H2C_CCTRL_G7_V1_DATARATE_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_DATARATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_GI_LTF_SH 12
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_GI_LTF_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_TRYRATE BIT(15)
+#define FWCMD_H2C_CCTRL_G7_V1_ARFR_CTRL_SH 16
+#define FWCMD_H2C_CCTRL_G7_V1_ARFR_CTRL_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_DIS_HE1SS_STBC BIT(18)
+#define FWCMD_H2C_CCTRL_G7_V1_ACQ_RPT_EN BIT(20)
+#define FWCMD_H2C_CCTRL_G7_V1_MGQ_RPT_EN BIT(21)
+#define FWCMD_H2C_CCTRL_G7_V1_ULQ_RPT_EN BIT(22)
+#define FWCMD_H2C_CCTRL_G7_V1_TWTQ_RPT_EN BIT(23)
+#define FWCMD_H2C_CCTRL_G7_V1_FORCE_TXOP BIT(24)
+#define FWCMD_H2C_CCTRL_G7_V1_DISRTSFB BIT(25)
+#define FWCMD_H2C_CCTRL_G7_V1_DISDATAFB BIT(26)
+#define FWCMD_H2C_CCTRL_G7_V1_NSTR_EN BIT(27)
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_DENSITY_SH 28
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_DENSITY_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_RTY_LOWEST_RATE_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_RTY_LOWEST_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_TXCNT_LMT_SH 12
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_TXCNT_LMT_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_RTSRATE_SH 16
+#define FWCMD_H2C_CCTRL_G7_V1_RTSRATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_RTY_LOWEST_RATE_SH 28
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_RTY_LOWEST_RATE_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_TX_CNT_LMT_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_TX_CNT_LMT_MSK 0x3f
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_TXCNT_LMT_SEL BIT(6)
+#define FWCMD_H2C_CCTRL_G7_V1_MAX_AGG_NUM_SEL BIT(7)
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_EN BIT(8)
+#define FWCMD_H2C_CCTRL_G7_V1_CTS2SELF_EN BIT(9)
+#define FWCMD_H2C_CCTRL_G7_V1_CCA_RTS_SH 10
+#define FWCMD_H2C_CCTRL_G7_V1_CCA_RTS_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_HW_RTS_EN BIT(12)
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_DROP_DATA_MODE_SH 13
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_DROP_DATA_MODE_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_PRELD_EN BIT(15)
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_MAX_LEN_SH 16
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_MAX_LEN_MSK 0x7ff
+#define FWCMD_H2C_CCTRL_G7_V1_UL_MU_DIS BIT(27)
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_MAX_TIME_SH 28
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_MAX_TIME_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_MAX_AGG_NUM_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_MAX_AGG_NUM_MSK 0xff
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_BW_SH 8
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_BW_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_BW_ER BIT(11)
+#define FWCMD_H2C_CCTRL_G7_V1_BA_BMAP_SH 12
+#define FWCMD_H2C_CCTRL_G7_V1_BA_BMAP_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_VCS_STBC BIT(15)
+#define FWCMD_H2C_CCTRL_G7_V1_VO_LFTIME_SEL_SH 16
+#define FWCMD_H2C_CCTRL_G7_V1_VO_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_VI_LFTIME_SEL_SH 19
+#define FWCMD_H2C_CCTRL_G7_V1_VI_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_BE_LFTIME_SEL_SH 22
+#define FWCMD_H2C_CCTRL_G7_V1_BE_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_BK_LFTIME_SEL_SH 25
+#define FWCMD_H2C_CCTRL_G7_V1_BK_LFTIME_SEL_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_TIME_SEL BIT(28)
+#define FWCMD_H2C_CCTRL_G7_V1_AMPDU_LEN_SEL BIT(29)
+#define FWCMD_H2C_CCTRL_G7_V1_RTS_TXCNT_LMT_SEL BIT(30)
+#define FWCMD_H2C_CCTRL_G7_V1_LSIG_TXOP_EN BIT(31)
+#define FWCMD_H2C_CCTRL_G7_V1_MULTI_PORT_ID_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_MULTI_PORT_ID_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_BYPASS_PUNC BIT(3)
+#define FWCMD_H2C_CCTRL_G7_V1_MBSSID_SH 4
+#define FWCMD_H2C_CCTRL_G7_V1_MBSSID_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_TID_DISABLE_SH 8
+#define FWCMD_H2C_CCTRL_G7_V1_TID_DISABLE_MSK 0xff
+#define FWCMD_H2C_CCTRL_G7_V1_ACT_SUBCH_CBW_SH 16
+#define FWCMD_H2C_CCTRL_G7_V1_ACT_SUBCH_CBW_MSK 0xffff
+#define FWCMD_H2C_CCTRL_G7_V1_ADDR_CAM_INDEX_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_ADDR_CAM_INDEX_MSK 0x3ff
+#define FWCMD_H2C_CCTRL_G7_V1_A_CTRL_BQR BIT(15)
+#define FWCMD_H2C_CCTRL_G7_V1_A_CTRL_BSR BIT(16)
+#define FWCMD_H2C_CCTRL_G7_V1_A_CTRL_CAS BIT(17)
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_ER BIT(18)
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_DCM BIT(19)
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_LDPC BIT(20)
+#define FWCMD_H2C_CCTRL_G7_V1_DATA_STBC BIT(21)
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING0_SH 22
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING0_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING1_SH 24
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING1_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING2_SH 26
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING2_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING3_SH 28
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING3_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING4_SH 30
+#define FWCMD_H2C_CCTRL_G7_V1_NOMINAL_PKT_PADDING4_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_AID12_PAID_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_AID12_PAID_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_RESP_REF_RATE_SH 12
+#define FWCMD_H2C_CCTRL_G7_V1_RESP_REF_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_ULDL BIT(31)
+#define FWCMD_H2C_CCTRL_G7_V1_NC_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_NC_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_NR_SH 3
+#define FWCMD_H2C_CCTRL_G7_V1_NR_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_NG_SH 6
+#define FWCMD_H2C_CCTRL_G7_V1_NG_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_CB_SH 8
+#define FWCMD_H2C_CCTRL_G7_V1_CB_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_CS_SH 10
+#define FWCMD_H2C_CCTRL_G7_V1_CS_MSK 0x3
+#define FWCMD_H2C_CCTRL_G7_V1_CSI_STBC_EN BIT(13)
+#define FWCMD_H2C_CCTRL_G7_V1_CSI_LDPC_EN BIT(14)
+#define FWCMD_H2C_CCTRL_G7_V1_CSI_PARA_EN BIT(15)
+#define FWCMD_H2C_CCTRL_G7_V1_CSI_FIX_RATE_SH 16
+#define FWCMD_H2C_CCTRL_G7_V1_CSI_FIX_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_CSI_BW_SH 29
+#define FWCMD_H2C_CCTRL_G7_V1_CSI_BW_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_ALL_ACK_SUPPORT BIT(0)
+#define FWCMD_H2C_CCTRL_G7_V1_BSR_QUEUE_SIZE_FORMAT BIT(1)
+#define FWCMD_H2C_CCTRL_G7_V1_BSR_OM_UPD_EN BIT(2)
+#define FWCMD_H2C_CCTRL_G7_V1_MACID_FWD_IDC BIT(3)
+#define FWCMD_H2C_CCTRL_G7_V1_AZ_SEC_EN BIT(4)
+#define FWCMD_H2C_CCTRL_G7_V1_BF_SEC_EN BIT(5)
+#define FWCMD_H2C_CCTRL_G7_V1_FIX_UL_ADDRCAM_IDX BIT(6)
+#define FWCMD_H2C_CCTRL_G7_V1_CTRL_CNT_VLD BIT(7)
+#define FWCMD_H2C_CCTRL_G7_V1_CTRL_CNT_SH 8
+#define FWCMD_H2C_CCTRL_G7_V1_CTRL_CNT_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_RESP_SEC_TYPE_SH 12
+#define FWCMD_H2C_CCTRL_G7_V1_RESP_SEC_TYPE_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_EMLSR_TRANS_DLY_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_EMLSR_TRANS_DLY_MSK 0x7
+#define FWCMD_H2C_CCTRL_G7_V1_SR_MCS_SU_SH 3
+#define FWCMD_H2C_CCTRL_G7_V1_SR_MCS_SU_MSK 0x1f
+#define FWCMD_H2C_CCTRL_G7_V1_SR_MCS_RU_SH 8
+#define FWCMD_H2C_CCTRL_G7_V1_SR_MCS_RU_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_VO_CURR_RATE_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_VO_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_VI_CURR_RATE_SH 12
+#define FWCMD_H2C_CCTRL_G7_V1_VI_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_BE_CURR_RATE_L_SH 24
+#define FWCMD_H2C_CCTRL_G7_V1_BE_CURR_RATE_L_MSK 0xff
+#define FWCMD_H2C_CCTRL_G7_V1_BE_CURR_RATE_H_SH 0
+#define FWCMD_H2C_CCTRL_G7_V1_BE_CURR_RATE_H_MSK 0xf
+#define FWCMD_H2C_CCTRL_G7_V1_BK_CURR_RATE_SH 4
+#define FWCMD_H2C_CCTRL_G7_V1_BK_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_CCTRL_G7_V1_MGNT_CURR_RATE_SH 16
+#define FWCMD_H2C_CCTRL_G7_V1_MGNT_CURR_RATE_MSK 0xfff
+#define FWCMD_H2C_IE_CAM_INFO_TYPE_SH 0
+#define FWCMD_H2C_IE_CAM_INFO_TYPE_MSK 0xff
+#define FWCMD_H2C_IE_CAM_INFO_DATA0_SH 8
+#define FWCMD_H2C_IE_CAM_INFO_DATA0_MSK 0xff
+#define FWCMD_H2C_IE_CAM_INFO_DATA1_SH 16
+#define FWCMD_H2C_IE_CAM_INFO_DATA1_MSK 0xff
+#define FWCMD_H2C_IE_CAM_INFO_DATA2_SH 24
+#define FWCMD_H2C_IE_CAM_INFO_DATA2_MSK 0xff
+#define FWCMD_H2C_IE_CAM_INFO_DATA3_SH 0
+#define FWCMD_H2C_IE_CAM_INFO_DATA3_MSK 0xff
+#define FWCMD_H2C_PLAT_AUTO_TEST_D0_SH 0
+#define FWCMD_H2C_PLAT_AUTO_TEST_D0_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_AUTO_TEST_D1_SH 0
+#define FWCMD_H2C_PLAT_AUTO_TEST_D1_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_AUTO_TEST_D2_SH 0
+#define FWCMD_H2C_PLAT_AUTO_TEST_D2_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_AUTO_TEST_D3_SH 0
+#define FWCMD_H2C_PLAT_AUTO_TEST_D3_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_AUTO_TEST_D4_SH 0
+#define FWCMD_H2C_PLAT_AUTO_TEST_D4_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_AUTO_TEST_D5_SH 0
+#define FWCMD_H2C_PLAT_AUTO_TEST_D5_MSK 0xffffffff
+#define FWCMD_H2C_PLAT_AUTO_TEST_D6_SH 0
+#define FWCMD_H2C_PLAT_AUTO_TEST_D6_MSK 0xffffffff
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_FUNC_SH 0
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_FUNC_MSK 0x7f
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_ACK BIT(7)
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_TOTAL_LEN_SH 8
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_TOTAL_LEN_MSK 0xf
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_SEQ_NUM_SH 12
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_SEQ_NUM_MSK 0xf
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_TX_EN_SH 16
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_TX_EN_MSK 0xffff
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_MASK_SH 0
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_MASK_MSK 0xffff
+#define FWCMD_H2C_H2CREG_SCH_TX_PAUSE_BAND BIT(16)
+#define FWCMD_H2C_CMD_OFLD_SRC_SH 0
+#define FWCMD_H2C_CMD_OFLD_SRC_MSK 0x3
+#define FWCMD_H2C_CMD_OFLD_TYPE_SH 2
+#define FWCMD_H2C_CMD_OFLD_TYPE_MSK 0x3
+#define FWCMD_H2C_CMD_OFLD_LC BIT(4)
+#define FWCMD_H2C_CMD_OFLD_PATH_SH 5
+#define FWCMD_H2C_CMD_OFLD_PATH_MSK 0x3
+#define FWCMD_H2C_CMD_OFLD_CMD_NUM_SH 8
+#define FWCMD_H2C_CMD_OFLD_CMD_NUM_MSK 0x7f
+#define FWCMD_H2C_CMD_OFLD_OFFSET_SH 16
+#define FWCMD_H2C_CMD_OFLD_OFFSET_MSK 0xffff
+#define FWCMD_H2C_CMD_OFLD_ID_SH 0
+#define FWCMD_H2C_CMD_OFLD_ID_MSK 0xffff
+#define FWCMD_H2C_CMD_OFLD_BASE_OFFSET_SH 16
+#define FWCMD_H2C_CMD_OFLD_BASE_OFFSET_MSK 0xffff
+#define FWCMD_H2C_CMD_OFLD_VALUE_SH 0
+#define FWCMD_H2C_CMD_OFLD_VALUE_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_MASK_SH 0
+#define FWCMD_H2C_CMD_OFLD_MASK_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_TYPE_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_TYPE_MSK 0x7
+#define FWCMD_H2C_CMD_OFLD_V1_LC BIT(3)
+#define FWCMD_H2C_CMD_OFLD_V1_OFFSET0_SH 8
+#define FWCMD_H2C_CMD_OFLD_V1_OFFSET0_MSK 0xffffff
+#define FWCMD_H2C_CMD_OFLD_V1_OFFSET1_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_OFFSET1_MSK 0xffffff
+#define FWCMD_H2C_CMD_OFLD_V1_CMD_NUM_SH 24
+#define FWCMD_H2C_CMD_OFLD_V1_CMD_NUM_MSK 0xff
+#define FWCMD_H2C_CMD_OFLD_V1_MASK0_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_MASK0_MSK 0xffffffff
+#define FWCMD_H2C_CMD_OFLD_V1_VALUE_SH 0
+#define FWCMD_H2C_CMD_OFLD_V1_VALUE_MSK 0xffffffff
+#define FWCMD_H2C_H2CPKT_SCH_TX_PAUSE_TX_EN_SH 0
+#define FWCMD_H2C_H2CPKT_SCH_TX_PAUSE_TX_EN_MSK 0xffff
+#define FWCMD_H2C_H2CPKT_SCH_TX_PAUSE_MASK_SH 0
+#define FWCMD_H2C_H2CPKT_SCH_TX_PAUSE_MASK_MSK 0xffff
+#define FWCMD_H2C_H2CPKT_SCH_TX_PAUSE_BAND BIT(16)
+#define FWCMD_H2C_WOW_CSA_CHINFO_PRI_CH_SH 0
+#define FWCMD_H2C_WOW_CSA_CHINFO_PRI_CH_MSK 0xff
+#define FWCMD_H2C_WOW_CSA_CHINFO_SUPPORTED_BW_SH 8
+#define FWCMD_H2C_WOW_CSA_CHINFO_SUPPORTED_BW_MSK 0xff
+#define FWCMD_H2C_WOW_CSA_CHINFO_CH_BAND_SH 16
+#define FWCMD_H2C_WOW_CSA_CHINFO_CH_BAND_MSK 0x3
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF0_SH 0
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF0_MSK 0xffffffff
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF1_SH 0
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF1_MSK 0xffffffff
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF2_SH 0
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF2_MSK 0xffffffff
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF3_SH 0
+#define FWCMD_H2C_WOW_CSA_CHINFO_RF3_MSK 0xffffffff
+#define FWCMD_H2C_BA_CAM_G7_VALID BIT(0)
+#define FWCMD_H2C_BA_CAM_G7_INIT_REQ BIT(1)
+#define FWCMD_H2C_BA_CAM_G7_TID_SH 4
+#define FWCMD_H2C_BA_CAM_G7_TID_MSK 0xf
+#define FWCMD_H2C_BA_CAM_G7_STD_ENTRY_EN BIT(8)
+#define FWCMD_H2C_BA_CAM_G7_BAND_SEL BIT(9)
+#define FWCMD_H2C_BA_CAM_G7_MLD_EN BIT(10)
+#define FWCMD_H2C_BA_CAM_G7_BMAP_SIZE_SH 16
+#define FWCMD_H2C_BA_CAM_G7_BMAP_SIZE_MSK 0xf
+#define FWCMD_H2C_BA_CAM_G7_SSN_SH 20
+#define FWCMD_H2C_BA_CAM_G7_SSN_MSK 0xfff
+#define FWCMD_H2C_BA_CAM_G7_UID_VALUE_SH 0
+#define FWCMD_H2C_BA_CAM_G7_UID_VALUE_MSK 0xff
+#define FWCMD_H2C_BA_CAM_G7_MACID_SH 8
+#define FWCMD_H2C_BA_CAM_G7_MACID_MSK 0x3ff
+#define FWCMD_H2C_BA_CAM_G7_ENTRY_IDX_SH 24
+#define FWCMD_H2C_BA_CAM_G7_ENTRY_IDX_MSK 0xff
+#define FWCMD_H2C_STA_CSA_CHINFO_PRI_CH_SH 0
+#define FWCMD_H2C_STA_CSA_CHINFO_PRI_CH_MSK 0xff
+#define FWCMD_H2C_STA_CSA_CHINFO_SUPPORTED_BW_SH 8
+#define FWCMD_H2C_STA_CSA_CHINFO_SUPPORTED_BW_MSK 0xff
+#define FWCMD_H2C_STA_CSA_CHINFO_CH_BAND_SH 16
+#define FWCMD_H2C_STA_CSA_CHINFO_CH_BAND_MSK 0x3
+#define FWCMD_H2C_STA_CSA_CHINFO_RF0_SH 0
+#define FWCMD_H2C_STA_CSA_CHINFO_RF0_MSK 0xffffffff
+#define FWCMD_H2C_STA_CSA_CHINFO_RF1_SH 0
+#define FWCMD_H2C_STA_CSA_CHINFO_RF1_MSK 0xffffffff
+#define FWCMD_H2C_STA_CSA_CHINFO_RF2_SH 0
+#define FWCMD_H2C_STA_CSA_CHINFO_RF2_MSK 0xffffffff
+#define FWCMD_H2C_STA_CSA_CHINFO_RF3_SH 0
+#define FWCMD_H2C_STA_CSA_CHINFO_RF3_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID_LEN_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID_LEN_MSK 0xff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP_TYPE_SH 8
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP_TYPE_MSK 0xff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP_LEN_SH 16
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP_LEN_MSK 0xff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID0_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID0_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID1_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID1_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID2_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID2_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID3_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_OID3_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP0_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP0_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP1_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP1_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP2_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP2_MSK 0xffffffff
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP3_SH 0
+#define FWCMD_H2C_PROXY_SNMP_ENTERPRISE_RSP3_MSK 0xffffffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_PERIOD_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_PERIOD_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_DWELL_TIME_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_DWELL_TIME_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_CENTRAL_CH_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_PRI_CH_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_PRI_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_BW_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_BW_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_NOTIFY_DWELL BIT(3)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_NOTIFY_PRETX BIT(4)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_NOTIFY_POSTTX BIT(5)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_NOTIFY_ENTERCH BIT(6)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_NOTIFY_LEAVECH BIT(7)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_NUM_TX_PKT_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_NUM_TX_PKT_MSK 0xf
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT BIT(12)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_PAUSE_TX_DATA BIT(13)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_CH_BAND_SH 14
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_CH_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_PROBE_REQ_PKT_ID_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_PROBE_REQ_PKT_ID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_DFS_CH BIT(24)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_NULL BIT(25)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_RAND_SEQ_NUM BIT(26)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_CFG_TX_PWR BIT(27)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PROBE_REQ BIT(28)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID0_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID0_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID1_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID1_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID2_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID2_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID3_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID3_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID4_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID4_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID5_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID5_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID6_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID6_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID7_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ID7_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PWR_IDX_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PWR_IDX_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_CHKPT_TIMER_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_CHKPT_TIMER_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ACKING_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_TX_PKT_ACKING_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_SSIDS_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_SSIDS_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_TPLT_ID_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_TPLT_ID_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_DRIVER_DEF_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_DRIVER_DEF_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_SHORTSSIDS_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_SHORTSSIDS_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_BSSIDS_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V1_FW_PROBE0_BSSIDS_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PERIOD_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PERIOD_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_DWELL_TIME_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_DWELL_TIME_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_CENTRAL_CH_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PRI_CH_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PRI_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_BW_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_BW_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_CH_BAND_SH 3
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_CH_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_DFS_CH BIT(5)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PAUSE_TX_DATA BIT(6)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_NULL BIT(7)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_RAND_SEQ_NUM BIT(8)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_NOTIFY_DWELL BIT(9)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_NOTIFY_PRETX BIT(10)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_NOTIFY_POSTTX BIT(11)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_NOTIFY_ENTERCH BIT(12)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_NOTIFY_LEAVECH BIT(13)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PROBE BIT(14)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_EARLY_LEAVE_CRIT_SH 15
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_EARLY_LEAVE_CRIT_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_CHKPT_REPEAT_PROBE BIT(18)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PREFERRED BIT(19)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_PARAM_FOR_LEARNED_CH BIT(20)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TYPE_SH 21
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TYPE_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_NOTIFY_CHKPT BIT(23)
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_CHKPT_TIMER_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_CHKPT_TIMER_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_EARLY_LEAVE_TIME_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_EARLY_LEAVE_TIME_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_EARLY_LEAVE_TH_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_EARLY_LEAVE_TH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_CTRL_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_CTRL_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID0_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID0_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID1_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID1_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID2_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID2_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID3_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID3_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID4_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID4_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID5_SH 8
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID5_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID6_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID6_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID7_SH 24
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_TX_PKT_ID7_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_DRIVER_DEF_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_DRIVER_DEF_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_FW_PROBE0_SSIDS_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_FW_PROBE0_SSIDS_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_FW_PROBE0_SHORTSSIDS_SH 0
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_FW_PROBE0_SHORTSSIDS_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_FW_PROBE0_BSSIDS_SH 16
+#define FWCMD_H2C_SCANOFLD_CHINFO_V2_FW_PROBE0_BSSIDS_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_BAND_SH 0
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_PORT_ID_SH 2
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_PORT_ID_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_MACID_SH 8
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_MACID_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_OPCH_END_SH 24
+#define FWCMD_H2C_SCANOFLD_MACC_ROLE_OPCH_END_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_MACID_SH 0
+#define FWCMD_H2C_SCANOFLD_OPCH_MACID_MSK 0xffff
+#define FWCMD_H2C_SCANOFLD_OPCH_BAND_SH 16
+#define FWCMD_H2C_SCANOFLD_OPCH_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_OPCH_PORT_ID_SH 18
+#define FWCMD_H2C_SCANOFLD_OPCH_PORT_ID_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_OPCH_POLICY_SH 21
+#define FWCMD_H2C_SCANOFLD_OPCH_POLICY_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_OPCH_TXNULL BIT(23)
+#define FWCMD_H2C_SCANOFLD_OPCH_POLICY_VAL_SH 24
+#define FWCMD_H2C_SCANOFLD_OPCH_POLICY_VAL_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_DURATION_SH 0
+#define FWCMD_H2C_SCANOFLD_OPCH_DURATION_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_CH_BAND_SH 8
+#define FWCMD_H2C_SCANOFLD_OPCH_CH_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_OPCH_BW_SH 10
+#define FWCMD_H2C_SCANOFLD_OPCH_BW_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_OPCH_NOTIFY_ENTER BIT(13)
+#define FWCMD_H2C_SCANOFLD_OPCH_NOTIFY_LEAVE BIT(14)
+#define FWCMD_H2C_SCANOFLD_OPCH_NULL_TYPE BIT(15)
+#define FWCMD_H2C_SCANOFLD_OPCH_PRI_CH_SH 16
+#define FWCMD_H2C_SCANOFLD_OPCH_PRI_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_CENTRAL_CH_SH 24
+#define FWCMD_H2C_SCANOFLD_OPCH_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_PKTS_CTRL_SH 0
+#define FWCMD_H2C_SCANOFLD_OPCH_PKTS_CTRL_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_DRIVER_DEF_SH 8
+#define FWCMD_H2C_SCANOFLD_OPCH_DRIVER_DEF_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_SS_SH 16
+#define FWCMD_H2C_SCANOFLD_OPCH_SS_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT0_SH 0
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT0_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT1_SH 8
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT1_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT2_SH 16
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT2_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT3_SH 24
+#define FWCMD_H2C_SCANOFLD_OPCH_PKT3_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHRPT_PRI_CH_SH 0
+#define FWCMD_H2C_SCANOFLD_CHRPT_PRI_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHRPT_RX_CNT_SH 8
+#define FWCMD_H2C_SCANOFLD_CHRPT_RX_CNT_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHRPT_TX_FAIL_SH 16
+#define FWCMD_H2C_SCANOFLD_CHRPT_TX_FAIL_MSK 0x7f
+#define FWCMD_H2C_SCANOFLD_CHRPT_PARSED BIT(23)
+#define FWCMD_H2C_SCANOFLD_CHRPT_CH_BAND_SH 24
+#define FWCMD_H2C_SCANOFLD_CHRPT_CH_BAND_MSK 0x3
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL0_TX_SH 0
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL0_TX_MSK 0x1f
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL1_TX_SH 5
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL1_TX_MSK 0x1f
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL0_TX_FAIL_SH 10
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL0_TX_FAIL_MSK 0x1f
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL1_TX_FAIL_SH 15
+#define FWCMD_H2C_SCANOFLD_CHRPT_NULL1_TX_FAIL_MSK 0x1f
+#define FWCMD_H2C_SCANOFLD_CHRPT_BW_SH 20
+#define FWCMD_H2C_SCANOFLD_CHRPT_BW_MSK 0x7
+#define FWCMD_H2C_SCANOFLD_CHRPT_CENTRAL_CH_SH 24
+#define FWCMD_H2C_SCANOFLD_CHRPT_CENTRAL_CH_MSK 0xff
+#define FWCMD_H2C_SCANOFLD_CHRPT_MACID_SH 0
+#define FWCMD_H2C_SCANOFLD_CHRPT_MACID_MSK 0xffff
+
+/*----------------------------------- H2CPKT - CAT(OutSrc,Phydm) -------------------*/
+#define H2C_HDR_CAT_SH 0
+#define H2C_HDR_CAT_MSK 0x3
+#define H2C_HDR_CLASS_SH 2
+#define H2C_HDR_CLASS_MSK 0x3f
+#define H2C_HDR_FUNC_SH 8
+#define H2C_HDR_FUNC_MSK 0xff
+#define H2C_HDR_DEL_TYPE_SH 16
+#define H2C_HDR_DEL_TYPE_MSK 0xf
+#define H2C_HDR_H2C_SEQ_SH 24
+#define H2C_HDR_H2C_SEQ_MSK 0xff
+#define H2C_HDR_TOTAL_LEN_SH 0
+#define H2C_HDR_TOTAL_LEN_MSK 0x3fff
+#define H2C_HDR_REC_ACK BIT(14)
+#define H2C_HDR_DONE_ACK BIT(15)
+#define H2C_HDR_SEQ_VALID BIT(16)
+#define H2C_HDR_SEQ_SH 17
+#define H2C_HDR_SEQ_MSK 0x7
+#define H2C_HDR_SEQ_STOP BIT(20)
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_MACID_CFG_IS_DISRA BIT(0)
+#define FWCMD_H2C_MACID_CFG_MODE_CTRL_SH 1
+#define FWCMD_H2C_MACID_CFG_MODE_CTRL_MSK 0x1f
+#define FWCMD_H2C_MACID_CFG_BW_CAP_SH 6
+#define FWCMD_H2C_MACID_CFG_BW_CAP_MSK 0x3
+#define FWCMD_H2C_MACID_CFG_MACID_SH 8
+#define FWCMD_H2C_MACID_CFG_MACID_MSK 0xff
+#define FWCMD_H2C_MACID_CFG_DCM_CAP BIT(16)
+#define FWCMD_H2C_MACID_CFG_ER_CAP BIT(17)
+#define FWCMD_H2C_MACID_CFG_INIT_RATE_LV_SH 18
+#define FWCMD_H2C_MACID_CFG_INIT_RATE_LV_MSK 0x3
+#define FWCMD_H2C_MACID_CFG_BW_NOUPDATE BIT(20)
+#define FWCMD_H2C_MACID_CFG_EN_SGI BIT(21)
+#define FWCMD_H2C_MACID_CFG_LDPC_CAP BIT(22)
+#define FWCMD_H2C_MACID_CFG_STBC_CAP BIT(23)
+#define FWCMD_H2C_MACID_CFG_SS_NUM_SH 24
+#define FWCMD_H2C_MACID_CFG_SS_NUM_MSK 0x7
+#define FWCMD_H2C_MACID_CFG_GI_LTF_SH 27
+#define FWCMD_H2C_MACID_CFG_GI_LTF_MSK 0x7
+#define FWCMD_H2C_MACID_CFG_RA_MASK_LOW_SH 0
+#define FWCMD_H2C_MACID_CFG_RA_MASK_LOW_MSK 0xffffffff
+#define FWCMD_H2C_MACID_CFG_RA_MASK_HI_SH 0
+#define FWCMD_H2C_MACID_CFG_RA_MASK_HI_MSK 0xfffffff
+#define FWCMD_H2C_RSSI_RA_CONFIG_IS_DISRA BIT(0)
+#define FWCMD_H2C_RSSI_RA_CONFIG_RA_MASK_LOW_SH 0
+#define FWCMD_H2C_RSSI_RA_CONFIG_RA_MASK_LOW_MSK 0xffffffff
+#define FWCMD_H2C_RSSI_RA_CONFIG_RA_MASK_HI_SH 0
+#define FWCMD_H2C_RSSI_RA_CONFIG_RA_MASK_HI_MSK 0xffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+
+/*----------------------------------- H2CPKT - CAT(OutSrc, RF) ---------------------*/
+#define H2C_HDR_CAT_SH 0
+#define H2C_HDR_CAT_MSK 0x3
+#define H2C_HDR_CLASS_SH 2
+#define H2C_HDR_CLASS_MSK 0x3f
+#define H2C_HDR_FUNC_SH 8
+#define H2C_HDR_FUNC_MSK 0xff
+#define H2C_HDR_DEL_TYPE_SH 16
+#define H2C_HDR_DEL_TYPE_MSK 0xf
+#define H2C_HDR_H2C_SEQ_SH 24
+#define H2C_HDR_H2C_SEQ_MSK 0xff
+#define H2C_HDR_TOTAL_LEN_SH 0
+#define H2C_HDR_TOTAL_LEN_MSK 0x3fff
+#define H2C_HDR_REC_ACK BIT(14)
+#define H2C_HDR_DONE_ACK BIT(15)
+#define H2C_HDR_SEQ_VALID BIT(16)
+#define H2C_HDR_SEQ_SH 17
+#define H2C_HDR_SEQ_MSK 0x7
+#define H2C_HDR_SEQ_STOP BIT(20)
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_H2C_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+
+/*----------------------------------- H2CPKT - CAT(OutSrc, BTC) --------------------*/
+#define H2C_HDR_CAT_SH 0
+#define H2C_HDR_CAT_MSK 0x3
+#define H2C_HDR_CLASS_SH 2
+#define H2C_HDR_CLASS_MSK 0x3f
+#define H2C_HDR_FUNC_SH 8
+#define H2C_HDR_FUNC_MSK 0xff
+#define H2C_HDR_DEL_TYPE_SH 16
+#define H2C_HDR_DEL_TYPE_MSK 0xf
+#define H2C_HDR_H2C_SEQ_SH 24
+#define H2C_HDR_H2C_SEQ_MSK 0xff
+#define H2C_HDR_TOTAL_LEN_SH 0
+#define H2C_HDR_TOTAL_LEN_MSK 0x3fff
+#define H2C_HDR_REC_ACK BIT(14)
+#define H2C_HDR_DONE_ACK BIT(15)
+#define H2C_HDR_SEQ_VALID BIT(16)
+#define H2C_HDR_SEQ_SH 17
+#define H2C_HDR_SEQ_MSK 0x7
+#define H2C_HDR_SEQ_STOP BIT(20)
+#define FWCMD_H2C_TDMA_EN BIT(0)
+#define FWCMD_H2C_TDMA_PRO_PS BIT(1)
+#define FWCMD_H2C_TDMA_NULL BIT(2)
+#define FWCMD_H2C_TDMA_QOS_NULL BIT(3)
+#define FWCMD_H2C_TDMA_WL_SLOT_SH 8
+#define FWCMD_H2C_TDMA_WL_SLOT_MSK 0xff
+
+/*----------------------------------- C2HPKT - CAT(TEST) ---------------------------*/
+#define C2H_HDR_CAT_SH 0
+#define C2H_HDR_CAT_MSK 0x3
+#define C2H_HDR_CLASS_SH 2
+#define C2H_HDR_CLASS_MSK 0x3f
+#define C2H_HDR_FUNC_SH 8
+#define C2H_HDR_FUNC_MSK 0xff
+#define C2H_HDR_DEL_TYPE_SH 16
+#define C2H_HDR_DEL_TYPE_MSK 0xf
+#define C2H_HDR_C2H_SEQ_SH 24
+#define C2H_HDR_C2H_SEQ_MSK 0xff
+#define C2H_HDR_TOTAL_LEN_SH 0
+#define C2H_HDR_TOTAL_LEN_MSK 0x3fff
+#define FWCMD_C2H_C2H_LB_PAYLOAD_SH 0
+#define FWCMD_C2H_C2H_LB_PAYLOAD_MSK 0xffffffff
+#define FWCMD_C2H_TEST_PHY_RPT_PAYLOAD_SH 0
+#define FWCMD_C2H_TEST_PHY_RPT_PAYLOAD_MSK 0xffffffff
+#define FWCMD_C2H_C2H_LOG_TEST_PAYLOAD0_SH 0
+#define FWCMD_C2H_C2H_LOG_TEST_PAYLOAD0_MSK 0xffffffff
+#define FWCMD_C2H_SEND_NULL_TEST_SEND_NUM_SH 0
+#define FWCMD_C2H_SEND_NULL_TEST_SEND_NUM_MSK 0xffff
+#define FWCMD_C2H_SEND_NULL_TEST_RPT_NUM_SH 16
+#define FWCMD_C2H_SEND_NULL_TEST_RPT_NUM_MSK 0xffff
+#define FWCMD_C2H_C2H_L1_2_TEST_RESULT_SH 0
+#define FWCMD_C2H_C2H_L1_2_TEST_RESULT_MSK 0xffffffff
+#define FWCMD_C2H_SPIC_PAYLOAD0_SH 0
+#define FWCMD_C2H_SPIC_PAYLOAD0_MSK 0xff
+#define FWCMD_C2H_SPIC_PAYLOAD1_SH 8
+#define FWCMD_C2H_SPIC_PAYLOAD1_MSK 0xff
+#define FWCMD_C2H_SPIC_PAYLOAD2_SH 16
+#define FWCMD_C2H_SPIC_PAYLOAD2_MSK 0xff
+#define FWCMD_C2H_SPIC_PAYLOAD3_SH 24
+#define FWCMD_C2H_SPIC_PAYLOAD3_MSK 0xff
+#define FWCMD_C2H_SPIC_PAYLOAD4_SH 0
+#define FWCMD_C2H_SPIC_PAYLOAD4_MSK 0xff
+#define FWCMD_C2H_SPIC_PAYLOAD5_SH 8
+#define FWCMD_C2H_SPIC_PAYLOAD5_MSK 0xff
+#define FWCMD_C2H_SPIC_PAYLOAD6_SH 16
+#define FWCMD_C2H_SPIC_PAYLOAD6_MSK 0xff
+#define FWCMD_C2H_SPIC_PAYLOAD7_SH 24
+#define FWCMD_C2H_SPIC_PAYLOAD7_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD0_SH 0
+#define FWCMD_C2H_SPIC2_PAYLOAD0_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD1_SH 8
+#define FWCMD_C2H_SPIC2_PAYLOAD1_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD2_SH 16
+#define FWCMD_C2H_SPIC2_PAYLOAD2_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD3_SH 24
+#define FWCMD_C2H_SPIC2_PAYLOAD3_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD4_SH 0
+#define FWCMD_C2H_SPIC2_PAYLOAD4_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD5_SH 8
+#define FWCMD_C2H_SPIC2_PAYLOAD5_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD6_SH 16
+#define FWCMD_C2H_SPIC2_PAYLOAD6_MSK 0xff
+#define FWCMD_C2H_SPIC2_PAYLOAD7_SH 24
+#define FWCMD_C2H_SPIC2_PAYLOAD7_MSK 0xff
+#define FWCMD_C2H_LPS_ONOFF_TEST_TYPE_SH 0
+#define FWCMD_C2H_LPS_ONOFF_TEST_TYPE_MSK 0xffffffff
+#define FWCMD_C2H_LPS_ONOFF_TEST_RESULT_SH 0
+#define FWCMD_C2H_LPS_ONOFF_TEST_RESULT_MSK 0xffffffff
+#define FWCMD_C2H_MPIO_TEST_TYPE_SH 0
+#define FWCMD_C2H_MPIO_TEST_TYPE_MSK 0xffffffff
+#define FWCMD_C2H_MPIO_TEST_RESULT_SH 0
+#define FWCMD_C2H_MPIO_TEST_RESULT_MSK 0xffffffff
+#define FWCMD_C2H_SENDNULL_TEST_SEND_NUM_SH 0
+#define FWCMD_C2H_SENDNULL_TEST_SEND_NUM_MSK 0xffff
+#define FWCMD_C2H_SENDNULL_TEST_RPT_NUM_SH 16
+#define FWCMD_C2H_SENDNULL_TEST_RPT_NUM_MSK 0xffff
+#define FWCMD_C2H_MPIOV2_TEST_TYPE_SH 0
+#define FWCMD_C2H_MPIOV2_TEST_TYPE_MSK 0xffffffff
+#define FWCMD_C2H_MPIOV2_TEST_RESULT_SH 0
+#define FWCMD_C2H_MPIOV2_TEST_RESULT_MSK 0xffffffff
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD0_SH 0
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD0_MSK 0xffffffff
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD1_SH 0
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD1_MSK 0xffffffff
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD2_SH 0
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD2_MSK 0xffffffff
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD3_SH 0
+#define FWCMD_C2H_MPIOV2_TEST_PAYLOAD3_MSK 0xffffffff
+#define FWCMD_C2H_MIO_TEST_RESULT_SH 0
+#define FWCMD_C2H_MIO_TEST_RESULT_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_SEQNUM_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_SEQNUM_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_TEST_FUNC_MODE_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_TEST_FUNC_MODE_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_TEST_CNT_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_TEST_CNT_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_BUFFERSIZE_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_BUFFERSIZE_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT_NUM_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT_NUM_MSK 0xffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT_BUFFER_MODE_SH 16
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT_BUFFER_MODE_MSK 0xffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT0_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT0_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT1_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT1_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT2_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT2_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT3_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT3_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT4_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT4_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT5_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT5_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT6_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT6_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT7_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INPUT7_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_RETURN_VALUE_EXPECT_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_RETURN_VALUE_EXPECT_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_RETURN_VALUE_MASK_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_RETURN_VALUE_MASK_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_RETURN_VAL_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_RETURN_VAL_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_CPU_CYCLE_START_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_CPU_CYCLE_START_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_CPU_CYCLE_END_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_CPU_CYCLE_END_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INST_CNT_START_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INST_CNT_START_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INST_CNT_END_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_INST_CNT_END_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_FREERUN_CNT_START_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_FREERUN_CNT_START_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_FREERUN_CNT_END_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_FREERUN_CNT_END_MSK 0xffffffff
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_BUFFER_SH 0
+#define FWCMD_C2H_ANYTESTBUSYCALL_TEST_BUFFER_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_SEQNUM_SH 0
+#define FWCMD_C2H_ANYRW_TEST_SEQNUM_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_OP_MODE_SH 0
+#define FWCMD_C2H_ANYRW_TEST_OP_MODE_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_BUFFERSIZE_SH 0
+#define FWCMD_C2H_ANYRW_TEST_BUFFERSIZE_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_INPUT0_SH 0
+#define FWCMD_C2H_ANYRW_TEST_INPUT0_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_INPUT1_SH 0
+#define FWCMD_C2H_ANYRW_TEST_INPUT1_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_INPUT2_SH 0
+#define FWCMD_C2H_ANYRW_TEST_INPUT2_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_INPUT3_SH 0
+#define FWCMD_C2H_ANYRW_TEST_INPUT3_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_INPUT4_SH 0
+#define FWCMD_C2H_ANYRW_TEST_INPUT4_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_INPUT5_SH 0
+#define FWCMD_C2H_ANYRW_TEST_INPUT5_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_RETURN_VAL_SH 0
+#define FWCMD_C2H_ANYRW_TEST_RETURN_VAL_MSK 0xffffffff
+#define FWCMD_C2H_ANYRW_TEST_BUFFER_SH 0
+#define FWCMD_C2H_ANYRW_TEST_BUFFER_MSK 0xffffffff
+#define FWCMD_C2H_CPUIO_TEST_RESULT_SH 0
+#define FWCMD_C2H_CPUIO_TEST_RESULT_MSK 0xffffffff
+
+/*----------------------------------- C2HPKT - CAT(MAC) ----------------------------*/
+#define C2H_HDR_CAT_SH 0
+#define C2H_HDR_CAT_MSK 0x3
+#define C2H_HDR_CLASS_SH 2
+#define C2H_HDR_CLASS_MSK 0x3f
+#define C2H_HDR_FUNC_SH 8
+#define C2H_HDR_FUNC_MSK 0xff
+#define C2H_HDR_DEL_TYPE_SH 16
+#define C2H_HDR_DEL_TYPE_MSK 0xf
+#define C2H_HDR_C2H_SEQ_SH 24
+#define C2H_HDR_C2H_SEQ_MSK 0xff
+#define C2H_HDR_TOTAL_LEN_SH 0
+#define C2H_HDR_TOTAL_LEN_MSK 0x3fff
+#define FWCMD_C2H_REC_ACK_CAT_SH 0
+#define FWCMD_C2H_REC_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_REC_ACK_CLASS_SH 2
+#define FWCMD_C2H_REC_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_REC_ACK_FUNC_SH 8
+#define FWCMD_C2H_REC_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_REC_ACK_H2C_SEQ_SH 16
+#define FWCMD_C2H_REC_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_DONE_ACK_CAT_SH 0
+#define FWCMD_C2H_DONE_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_DONE_ACK_CLASS_SH 2
+#define FWCMD_C2H_DONE_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_DONE_ACK_FUNC_SH 8
+#define FWCMD_C2H_DONE_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_DONE_ACK_H2C_RETURN_SH 16
+#define FWCMD_C2H_DONE_ACK_H2C_RETURN_MSK 0xff
+#define FWCMD_C2H_DONE_ACK_H2C_SEQ_SH 24
+#define FWCMD_C2H_DONE_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_C2H_LOG_LOG_MSG_SH 0
+#define FWCMD_C2H_C2H_LOG_LOG_MSG_MSK 0xffffffff
+#define FWCMD_C2H_BCN_CNT_PORT_MBSSID_IDX_SH 0
+#define FWCMD_C2H_BCN_CNT_PORT_MBSSID_IDX_MSK 0x7f
+#define FWCMD_C2H_BCN_CNT_BAND_IDX BIT(7)
+#define FWCMD_C2H_BCN_CNT_CCA_FAIL_CNT_SH 8
+#define FWCMD_C2H_BCN_CNT_CCA_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_EDCCA_FAIL_CNT_SH 12
+#define FWCMD_C2H_BCN_CNT_EDCCA_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_NAV_FAIL_CNT_SH 16
+#define FWCMD_C2H_BCN_CNT_NAV_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_TXON_FAIL_CNT_SH 20
+#define FWCMD_C2H_BCN_CNT_TXON_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_MAC_FAIL_CNT_SH 24
+#define FWCMD_C2H_BCN_CNT_MAC_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_OTHERS_FAIL_CNT_SH 28
+#define FWCMD_C2H_BCN_CNT_OTHERS_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_LOCK_FAIL_CNT_SH 0
+#define FWCMD_C2H_BCN_CNT_LOCK_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_CMP_FAIL_CNT_SH 4
+#define FWCMD_C2H_BCN_CNT_CMP_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_INVALID_FAIL_CNT_SH 8
+#define FWCMD_C2H_BCN_CNT_INVALID_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_SRCHEND_FAIL_CNT_SH 12
+#define FWCMD_C2H_BCN_CNT_SRCHEND_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CNT_OK_CNT_SH 16
+#define FWCMD_C2H_BCN_CNT_OK_CNT_MSK 0xf
+#define FWCMD_C2H_BCN_CSAZERO_PORT_MBSSID_IDX_SH 0
+#define FWCMD_C2H_BCN_CSAZERO_PORT_MBSSID_IDX_MSK 0x7f
+#define FWCMD_C2H_BCN_CSAZERO_BAND_IDX BIT(7)
+#define FWCMD_C2H_BCN_CRITICAL_RST_PORT_MBSSID_IDX_SH 0
+#define FWCMD_C2H_BCN_CRITICAL_RST_PORT_MBSSID_IDX_MSK 0x7f
+#define FWCMD_C2H_BCN_CRITICAL_RST_BAND_IDX BIT(7)
+#define FWCMD_C2H_BCN_UPD_DONE_PORT_SH 0
+#define FWCMD_C2H_BCN_UPD_DONE_PORT_MSK 0x7
+#define FWCMD_C2H_BCN_UPD_DONE_MBSSID_SH 3
+#define FWCMD_C2H_BCN_UPD_DONE_MBSSID_MSK 0xf
+#define FWCMD_C2H_BCN_UPD_DONE_BAND_IDX BIT(7)
+#define FWCMD_C2H_BCN_BC_CHG_ZERO_PORT_MBSSID_IDX_SH 0
+#define FWCMD_C2H_BCN_BC_CHG_ZERO_PORT_MBSSID_IDX_MSK 0x7f
+#define FWCMD_C2H_BCN_BC_CHG_ZERO_BAND_IDX BIT(7)
+#define FWCMD_C2H_FW_SELF_DX_INFO_FWDX_TIME_INTERVAL_SH 0
+#define FWCMD_C2H_FW_SELF_DX_INFO_FWDX_TIME_INTERVAL_MSK 0xffff
+#define FWCMD_C2H_FW_SELF_DX_INFO_FWDX_HEAP_REMAIN BIT(0)
+#define FWCMD_C2H_FW_SELF_DX_INFO_FWDX_IRQ_CNT BIT(1)
+#define FWCMD_C2H_FW_SELF_DX_INFO_FWDX_AXIDMA_DEBUG_INFO BIT(2)
+#define FWCMD_C2H_FW_SELF_DX_INFO_FWDX_CPUIO_DEBUG_INFO BIT(3)
+#define FWCMD_C2H_EFUSE_DUMP_UUID0_SH 16
+#define FWCMD_C2H_EFUSE_DUMP_UUID0_MSK 0xff
+#define FWCMD_C2H_EFUSE_DUMP_UUID2_SH 24
+#define FWCMD_C2H_EFUSE_DUMP_UUID2_MSK 0xff
+#define FWCMD_C2H_EFUSE_DUMP_RX_SPATIAL_STREAM_REMARK_SH 24
+#define FWCMD_C2H_EFUSE_DUMP_RX_SPATIAL_STREAM_REMARK_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_RX_SPATIAL_STREAM_SH 28
+#define FWCMD_C2H_EFUSE_DUMP_RX_SPATIAL_STREAM_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_HCI_TYPE_SH 0
+#define FWCMD_C2H_EFUSE_DUMP_HCI_TYPE_MSK 0xf
+#define FWCMD_C2H_EFUSE_DUMP_BANDWIDTH_SH 8
+#define FWCMD_C2H_EFUSE_DUMP_BANDWIDTH_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_TX_SPATIAL_STREAM_SH 12
+#define FWCMD_C2H_EFUSE_DUMP_TX_SPATIAL_STREAM_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_ANT_TX_NUM_SH 16
+#define FWCMD_C2H_EFUSE_DUMP_ANT_TX_NUM_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_ANT_RX_NUM_SH 20
+#define FWCMD_C2H_EFUSE_DUMP_ANT_RX_NUM_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_CP_TEST_PATTEN0_SH 24
+#define FWCMD_C2H_EFUSE_DUMP_CP_TEST_PATTEN0_MSK 0xff
+#define FWCMD_C2H_EFUSE_DUMP_CP_TEST_PATTEN1_SH 0
+#define FWCMD_C2H_EFUSE_DUMP_CP_TEST_PATTEN1_MSK 0xff
+#define FWCMD_C2H_EFUSE_DUMP_PROTOCOL_REMARK_80211_SH 8
+#define FWCMD_C2H_EFUSE_DUMP_PROTOCOL_REMARK_80211_MSK 0x3
+#define FWCMD_C2H_EFUSE_DUMP_PROTOCOL_80211_SH 10
+#define FWCMD_C2H_EFUSE_DUMP_PROTOCOL_80211_MSK 0x3
+#define FWCMD_C2H_EFUSE_DUMP_NIC_ROUTER_REMARK_SH 12
+#define FWCMD_C2H_EFUSE_DUMP_NIC_ROUTER_REMARK_MSK 0x3
+#define FWCMD_C2H_EFUSE_DUMP_NIC_ROUTER_SH 14
+#define FWCMD_C2H_EFUSE_DUMP_NIC_ROUTER_MSK 0x3
+#define FWCMD_C2H_EFUSE_DUMP_ANT_TX_NUM_REMARK_SH 16
+#define FWCMD_C2H_EFUSE_DUMP_ANT_TX_NUM_REMARK_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_ANT_RX_NUM_REMARK_SH 20
+#define FWCMD_C2H_EFUSE_DUMP_ANT_RX_NUM_REMARK_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_BANDWIDTH_REMARK_SH 24
+#define FWCMD_C2H_EFUSE_DUMP_BANDWIDTH_REMARK_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_TX_SPATIAL_STREAM_REMARK_SH 28
+#define FWCMD_C2H_EFUSE_DUMP_TX_SPATIAL_STREAM_REMARK_MSK 0x7
+#define FWCMD_C2H_EFUSE_DUMP_WL_FUNCTION_SUPPORT_SH 0
+#define FWCMD_C2H_EFUSE_DUMP_WL_FUNCTION_SUPPORT_MSK 0xf
+#define FWCMD_C2H_EFUSE_DUMP_HW_SPECIAL_TYPE_SH 4
+#define FWCMD_C2H_EFUSE_DUMP_HW_SPECIAL_TYPE_MSK 0xf
+#define FWCMD_C2H_EFUSE_DUMP_UUID3_SH 0
+#define FWCMD_C2H_EFUSE_DUMP_UUID3_MSK 0xff
+#define FWCMD_C2H_EFUSE_DUMP_UUID1_SH 8
+#define FWCMD_C2H_EFUSE_DUMP_UUID1_MSK 0xff
+#define FWCMD_C2H_EFUSE_DUMP_PFID_SH 16
+#define FWCMD_C2H_EFUSE_DUMP_PFID_MSK 0xff
+#define FWCMD_C2H_READ_RSP_VALUE_LEN_SH 0
+#define FWCMD_C2H_READ_RSP_VALUE_LEN_MSK 0x7ff
+#define FWCMD_C2H_READ_RSP_LS BIT(15)
+#define FWCMD_C2H_READ_RSP_OFLD_ID_SH 16
+#define FWCMD_C2H_READ_RSP_OFLD_ID_MSK 0xff
+#define FWCMD_C2H_READ_RSP_ENTRY_NUM_SH 24
+#define FWCMD_C2H_READ_RSP_ENTRY_NUM_MSK 0xff
+#define FWCMD_C2H_READ_RSP_OFFSET_SH 0
+#define FWCMD_C2H_READ_RSP_OFFSET_MSK 0xffff
+#define FWCMD_C2H_READ_RSP_START_SH 0
+#define FWCMD_C2H_READ_RSP_START_MSK 0xffffffff
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_ID_SH 0
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_ID_MSK 0xff
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_OP_SH 8
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_OP_MSK 0x7
+#define FWCMD_C2H_PKT_OFLD_RSP_NUM_TIMING_SH 11
+#define FWCMD_C2H_PKT_OFLD_RSP_NUM_TIMING_MSK 0x3
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_LENGTH_SH 16
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_LENGTH_MSK 0xffff
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_CONTENT_SH 0
+#define FWCMD_C2H_PKT_OFLD_RSP_PKT_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_1_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_1_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_2_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_2_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_3_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_3_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_4_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_4_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_5_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_5_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_6_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_6_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_7_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_7_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_8_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_8_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_9_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_9_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_10_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_10_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_11_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_11_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_12_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_12_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_13_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_13_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_14_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_14_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_15_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_15_MSK 0xffffffff
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_16_SH 0
+#define FWCMD_C2H_MACID_PAUSE_STATUS_GRP_16_MSK 0xffffffff
+#define FWCMD_C2H_FW_GETPKT_RPT_MACID_SH 0
+#define FWCMD_C2H_FW_GETPKT_RPT_MACID_MSK 0xff
+#define FWCMD_C2H_FW_GETPKT_RPT_PKT_TYPE_SH 8
+#define FWCMD_C2H_FW_GETPKT_RPT_PKT_TYPE_MSK 0xf
+#define FWCMD_C2H_FW_GETPKT_RPT_MACID_EXT_SH 16
+#define FWCMD_C2H_FW_GETPKT_RPT_MACID_EXT_MSK 0xff
+#define FWCMD_C2H_TSF32_TOGL_RPT_BAND BIT(0)
+#define FWCMD_C2H_TSF32_TOGL_RPT_PORT_SH 1
+#define FWCMD_C2H_TSF32_TOGL_RPT_PORT_MSK 0x7
+#define FWCMD_C2H_TSF32_TOGL_RPT_STATUS_SH 4
+#define FWCMD_C2H_TSF32_TOGL_RPT_STATUS_MSK 0xfff
+#define FWCMD_C2H_TSF32_TOGL_RPT_EARLY_SH 16
+#define FWCMD_C2H_TSF32_TOGL_RPT_EARLY_MSK 0xffff
+#define FWCMD_C2H_TSF32_TOGL_RPT_TSF_L_SH 0
+#define FWCMD_C2H_TSF32_TOGL_RPT_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_TSF32_TOGL_RPT_TSF_H_SH 0
+#define FWCMD_C2H_TSF32_TOGL_RPT_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_RPT_MODE_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_RPT_MODE_MSK 0x7
+#define FWCMD_C2H_USR_TX_RPT_INFO_MACID_SH 8
+#define FWCMD_C2H_USR_TX_RPT_INFO_MACID_MSK 0xff
+#define FWCMD_C2H_USR_TX_RPT_INFO_AC_SH 16
+#define FWCMD_C2H_USR_TX_RPT_INFO_AC_MSK 0x3
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_BE_1K_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_BE_1K_MSK 0xffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_BK_1K_SH 16
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_BK_1K_MSK 0xffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_VI_1K_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_VI_1K_MSK 0xffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_VO_1K_SH 16
+#define FWCMD_C2H_USR_TX_RPT_INFO_PENDING_VO_1K_MSK 0xffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_FIRST_IN_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_FIRST_IN_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_FIRST_OUT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_FIRST_OUT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_LAST_OUT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_LAST_OUT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_BE_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_BE_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_BK_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_BK_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_VI_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_VI_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_VO_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_DROP_NUM_VO_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_BE_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_BE_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_BK_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_BK_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_VI_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_VI_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_VO_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_OK_NUM_VO_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_RX_CLEAR_US_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_RX_CLEAR_US_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_BUSY_US_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_BUSY_US_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_LAST_IN_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_FREERUN_CNT_LAST_IN_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_PPDU_CNT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_PPDU_CNT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_PPDU_CNT_WO_LAST_PKT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_PPDU_CNT_WO_LAST_PKT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_MPDU_CNT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_MPDU_CNT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_MPDU_CNT_WO_LAST_PKT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_MPDU_CNT_WO_LAST_PKT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_RTS_CNT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_RTS_CNT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_RTS_RETRY_CNT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_RTS_RETRY_CNT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_MPDU_OK_CNT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_TX_MPDU_OK_CNT_MSK 0xffffffff
+#define FWCMD_C2H_USR_TX_RPT_INFO_RA_RATIO_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_RA_RATIO_MSK 0xff
+#define FWCMD_C2H_USR_TX_RPT_INFO_MAX_RTS_CNT_SH 8
+#define FWCMD_C2H_USR_TX_RPT_INFO_MAX_RTS_CNT_MSK 0xff
+#define FWCMD_C2H_USR_TX_RPT_INFO_MAX_AMPDU_NUM_SH 16
+#define FWCMD_C2H_USR_TX_RPT_INFO_MAX_AMPDU_NUM_MSK 0xff
+#define FWCMD_C2H_USR_TX_RPT_INFO_MACID_EXT_SH 24
+#define FWCMD_C2H_USR_TX_RPT_INFO_MACID_EXT_MSK 0xff
+#define FWCMD_C2H_USR_TX_RPT_INFO_RX_ERR_CNT_SH 0
+#define FWCMD_C2H_USR_TX_RPT_INFO_RX_ERR_CNT_MSK 0xffffffff
+#define FWCMD_C2H_CMD_OFLD_RSP_RESULT BIT(0)
+#define FWCMD_C2H_CMD_OFLD_RSP_CMD_NUM_SH 8
+#define FWCMD_C2H_CMD_OFLD_RSP_CMD_NUM_MSK 0xff
+#define FWCMD_C2H_CMD_OFLD_RSP_OFFSET_SH 0
+#define FWCMD_C2H_CMD_OFLD_RSP_OFFSET_MSK 0xffffffff
+#define FWCMD_C2H_CMD_OFLD_RSP_EXP_VAL_SH 0
+#define FWCMD_C2H_CMD_OFLD_RSP_EXP_VAL_MSK 0xffffffff
+#define FWCMD_C2H_CMD_OFLD_RSP_R_VAL_SH 0
+#define FWCMD_C2H_CMD_OFLD_RSP_R_VAL_MSK 0xffffffff
+#define FWCMD_C2H_SCANOFLD_RSP_PRI_CH_SH 0
+#define FWCMD_C2H_SCANOFLD_RSP_PRI_CH_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_SCANNED_ROUND_SH 8
+#define FWCMD_C2H_SCANOFLD_RSP_SCANNED_ROUND_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_NOTIFY_REASON_SH 16
+#define FWCMD_C2H_SCANOFLD_RSP_NOTIFY_REASON_MSK 0xf
+#define FWCMD_C2H_SCANOFLD_RSP_STATUS_SH 20
+#define FWCMD_C2H_SCANOFLD_RSP_STATUS_MSK 0xf
+#define FWCMD_C2H_SCANOFLD_RSP_ACTUAL_PERIOD_SH 24
+#define FWCMD_C2H_SCANOFLD_RSP_ACTUAL_PERIOD_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_SPENT_LOW_SH 0
+#define FWCMD_C2H_SCANOFLD_RSP_SPENT_LOW_MSK 0xffffffff
+#define FWCMD_C2H_SCANOFLD_RSP_SPENT_HIGH_SH 0
+#define FWCMD_C2H_SCANOFLD_RSP_SPENT_HIGH_MSK 0xffffffff
+#define FWCMD_C2H_SCANOFLD_RSP_TX_FAIL_CNT_SH 0
+#define FWCMD_C2H_SCANOFLD_RSP_TX_FAIL_CNT_MSK 0xf
+#define FWCMD_C2H_SCANOFLD_RSP_AIR_DENSITY_SH 4
+#define FWCMD_C2H_SCANOFLD_RSP_AIR_DENSITY_MSK 0xf
+#define FWCMD_C2H_SCANOFLD_RSP_NUM_CH_RPT_SH 8
+#define FWCMD_C2H_SCANOFLD_RSP_NUM_CH_RPT_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_CH_RPT_SIZE_SH 16
+#define FWCMD_C2H_SCANOFLD_RSP_CH_RPT_SIZE_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_CH_BAND_SH 24
+#define FWCMD_C2H_SCANOFLD_RSP_CH_BAND_MSK 0x3
+#define FWCMD_C2H_SCANOFLD_RSP_BAND BIT(26)
+#define FWCMD_C2H_SCANOFLD_RSP_BW_SH 27
+#define FWCMD_C2H_SCANOFLD_RSP_BW_MSK 0x3
+#define FWCMD_C2H_SCANOFLD_RSP_ACTIVE_CH BIT(29)
+#define FWCMD_C2H_SCANOFLD_RSP_DRIVER_DEF_SH 0
+#define FWCMD_C2H_SCANOFLD_RSP_DRIVER_DEF_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_EXPECT_PERIOD_SH 8
+#define FWCMD_C2H_SCANOFLD_RSP_EXPECT_PERIOD_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_FW_DEF_SH 16
+#define FWCMD_C2H_SCANOFLD_RSP_FW_DEF_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_CENTRAL_CH_SH 24
+#define FWCMD_C2H_SCANOFLD_RSP_CENTRAL_CH_MSK 0xff
+#define FWCMD_C2H_SCANOFLD_RSP_REPORTING_TSF_SH 0
+#define FWCMD_C2H_SCANOFLD_RSP_REPORTING_TSF_MSK 0xffffffff
+#define FWCMD_C2H_TX_DUTY_RPT_TIMER_ERR_SH 0
+#define FWCMD_C2H_TX_DUTY_RPT_TIMER_ERR_MSK 0x7
+#define FWCMD_C2H_CH_SWITCH_RPT_RESULT_SH 0
+#define FWCMD_C2H_CH_SWITCH_RPT_RESULT_MSK 0x7
+#define FWCMD_C2H_BCNERLYNTFY_BAND BIT(0)
+#define FWCMD_C2H_BCNERLYNTFY_PORT_SH 1
+#define FWCMD_C2H_BCNERLYNTFY_PORT_MSK 0x7
+#define FWCMD_C2H_BCNERLYNTFY_FREERUN_CNT_LOW_SH 0
+#define FWCMD_C2H_BCNERLYNTFY_FREERUN_CNT_LOW_MSK 0xffffffff
+#define FWCMD_C2H_BCNERLYNTFY_FREERUN_CNT_HIGH_SH 0
+#define FWCMD_C2H_BCNERLYNTFY_FREERUN_CNT_HIGH_MSK 0xffffffff
+#define FWCMD_C2H_BCNERLYNTFY_TSF_LOW_SH 0
+#define FWCMD_C2H_BCNERLYNTFY_TSF_LOW_MSK 0xffffffff
+#define FWCMD_C2H_BCNERLYNTFY_TSF_HIGH_SH 0
+#define FWCMD_C2H_BCNERLYNTFY_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_C2H_BCNFLTR_RPT_MACID_SH 0
+#define FWCMD_C2H_BCNFLTR_RPT_MACID_MSK 0xff
+#define FWCMD_C2H_BCNFLTR_RPT_TYPE_SH 8
+#define FWCMD_C2H_BCNFLTR_RPT_TYPE_MSK 0x3
+#define FWCMD_C2H_BCNFLTR_RPT_RSSI_EVT_SH 10
+#define FWCMD_C2H_BCNFLTR_RPT_RSSI_EVT_MSK 0x3
+#define FWCMD_C2H_BCNFLTR_RPT_RSSI_MA_SH 16
+#define FWCMD_C2H_BCNFLTR_RPT_RSSI_MA_MSK 0xff
+#define FWCMD_C2H_BCNFLTR_RPT_MACID_EXT_SH 24
+#define FWCMD_C2H_BCNFLTR_RPT_MACID_EXT_MSK 0xff
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_MACID_SH 0
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_MACID_MSK 0xff
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_NUM_SH 8
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_NUM_MSK 0xff
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_SEQ_NUM_SH 16
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_SEQ_NUM_MSK 0xff
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_MACID_EXT_SH 24
+#define FWCMD_C2H_WIFI_SENSING_CSI_TX_RESULT_MACID_EXT_MSK 0xff
+#define FWCMD_C2H_TBTT_SH_STAT_RPT_BAND_SH 0
+#define FWCMD_C2H_TBTT_SH_STAT_RPT_BAND_MSK 0xff
+#define FWCMD_C2H_TBTT_SH_STAT_RPT_PORT_SH 8
+#define FWCMD_C2H_TBTT_SH_STAT_RPT_PORT_MSK 0xff
+#define FWCMD_C2H_TBTT_SH_STAT_RPT_NUM_TSF_SH 16
+#define FWCMD_C2H_TBTT_SH_STAT_RPT_NUM_TSF_MSK 0xff
+#define FWCMD_C2H_BCN_OFLD_RPT_BAND BIT(0)
+#define FWCMD_C2H_BCN_OFLD_RPT_PORT_SH 1
+#define FWCMD_C2H_BCN_OFLD_RPT_PORT_MSK 0x7
+#define FWCMD_C2H_BCN_OFLD_RPT_RSSI_NUM_SH 4
+#define FWCMD_C2H_BCN_OFLD_RPT_RSSI_NUM_MSK 0xff
+#define FWCMD_C2H_BCN_OFLD_RPT_TIMESTAMP_NUM_SH 12
+#define FWCMD_C2H_BCN_OFLD_RPT_TIMESTAMP_NUM_MSK 0xff
+#define FWCMD_C2H_BCN_OFLD_RPT_RSSI_DETECTED BIT(20)
+#define FWCMD_C2H_BCN_OFLD_RPT_TBTT_NUM_SH 0
+#define FWCMD_C2H_BCN_OFLD_RPT_TBTT_NUM_MSK 0xff
+#define FWCMD_C2H_BCN_OFLD_RPT_RXBCNOK_NUM_SH 8
+#define FWCMD_C2H_BCN_OFLD_RPT_RXBCNOK_NUM_MSK 0xff
+#define FWCMD_C2H_BCN_OFLD_RPT_IEHIT_NUM_SH 16
+#define FWCMD_C2H_BCN_OFLD_RPT_IEHIT_NUM_MSK 0xff
+#define FWCMD_C2H_BCN_OFLD_RPT_IEMISS_NUM_SH 24
+#define FWCMD_C2H_BCN_OFLD_RPT_IEMISS_NUM_MSK 0xff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_MODE_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_MODE_MSK 0xf
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_TRIGGER_CNT_SH 8
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_TRIGGER_CNT_MSK 0xff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_SW_DEF_SH 16
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_SW_DEF_MSK 0xffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_TO_THR_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_TO_THR_MSK 0xffffffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_CONT_FAIL_CNT_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_CONT_FAIL_CNT_MSK 0xffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_RESET BIT(16)
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_FRAME_FAIL_FLG BIT(17)
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_START_PKT_TIME_US_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_START_PKT_TIME_US_MSK 0xffffffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_FREERUN_NOW_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_FREERUN_NOW_MSK 0xffffffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_PKT_IN_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_PKT_IN_MSK 0xffffffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_QUEUE_TIME_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_QUEUE_TIME_MSK 0xffffffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_DURATION_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_DURATION_MSK 0xffffffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_C2H_ENQUEUE_FAIL_CNT_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_C2H_ENQUEUE_FAIL_CNT_MSK 0xffffffff
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_FW_ACTION_SH 0
+#define FWCMD_C2H_FRAME_TO_ACT_RPT_FW_ACTION_MSK 0xff
+#define FWCMD_C2H_EFUSE_DUMP_V2_CONTENT_SIZE_SH 0
+#define FWCMD_C2H_EFUSE_DUMP_V2_CONTENT_SIZE_MSK 0xfff
+#define FWCMD_C2H_EFUSE_DUMP_V2_END_BREAK_STATUS BIT(31)
+#define FWCMD_C2H_EFUSE_DUMP_V2_CONTENT_SH 0
+#define FWCMD_C2H_EFUSE_DUMP_V2_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_TXRPT_STATS_MACID_NUM_SH 0
+#define FWCMD_C2H_TXRPT_STATS_MACID_NUM_MSK 0xff
+#define FWCMD_C2H_TXRPT_STATS_C2H_MODE BIT(8)
+#define FWCMD_C2H_TXRPT_STATS_STATUS_SH 9
+#define FWCMD_C2H_TXRPT_STATS_STATUS_MSK 0xf
+#define FWCMD_C2H_TXRPT_STATS_MACID_SH 0
+#define FWCMD_C2H_TXRPT_STATS_MACID_MSK 0xffff
+#define FWCMD_C2H_TXRPT_STATS_TP_SH 16
+#define FWCMD_C2H_TXRPT_STATS_TP_MSK 0xffff
+#define FWCMD_C2H_TXRPT_STATS_TXRPT_NUM_SH 0
+#define FWCMD_C2H_TXRPT_STATS_TXRPT_NUM_MSK 0xffffffff
+#define FWCMD_C2H_TXRPT_STATS_TOTAL_PKT_SUM_SH 0
+#define FWCMD_C2H_TXRPT_STATS_TOTAL_PKT_SUM_MSK 0xffffffff
+#define FWCMD_C2H_TXRPT_STATS_PKT_OK_SUM_SH 0
+#define FWCMD_C2H_TXRPT_STATS_PKT_OK_SUM_MSK 0xffffffff
+#define FWCMD_C2H_BCN_SYNC_RPT_BAND BIT(0)
+#define FWCMD_C2H_BCN_SYNC_RPT_PORT_SH 1
+#define FWCMD_C2H_BCN_SYNC_RPT_PORT_MSK 0x7
+#define FWCMD_C2H_BCN_SYNC_RPT_RPT_ID_SH 4
+#define FWCMD_C2H_BCN_SYNC_RPT_RPT_ID_MSK 0xf
+#define FWCMD_C2H_BCN_SYNC_RPT_SEQ_NUM_SH 8
+#define FWCMD_C2H_BCN_SYNC_RPT_SEQ_NUM_MSK 0xfff
+#define FWCMD_C2H_BCN_SYNC_RPT_TSF_L_SH 0
+#define FWCMD_C2H_BCN_SYNC_RPT_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_BCN_SYNC_RPT_TSF_H_SH 0
+#define FWCMD_C2H_BCN_SYNC_RPT_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_TRX_TIME_STATS_BAND_0_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_BAND_0_MSK 0xf
+#define FWCMD_C2H_TRX_TIME_STATS_STATUS_0_SH 4
+#define FWCMD_C2H_TRX_TIME_STATS_STATUS_0_MSK 0xf
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME0_0_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME0_0_MSK 0xffffffff
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME1_0_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME1_0_MSK 0xffffffff
+#define FWCMD_C2H_TRX_TIME_STATS_IDLE_TIME1_0_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_IDLE_TIME1_0_MSK 0xffffffff
+#define FWCMD_C2H_TRX_TIME_STATS_BAND_1_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_BAND_1_MSK 0xf
+#define FWCMD_C2H_TRX_TIME_STATS_STATUS_1_SH 4
+#define FWCMD_C2H_TRX_TIME_STATS_STATUS_1_MSK 0xf
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME0_1_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME0_1_MSK 0xffffffff
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME1_1_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_BUSY_TIME1_1_MSK 0xffffffff
+#define FWCMD_C2H_TRX_TIME_STATS_IDLE_TIME1_1_SH 0
+#define FWCMD_C2H_TRX_TIME_STATS_IDLE_TIME1_1_MSK 0xffffffff
+#define FWCMD_C2H_APF_OFLD_RPT_NUM_SH 0
+#define FWCMD_C2H_APF_OFLD_RPT_NUM_MSK 0xffffffff
+#define FWCMD_C2H_APF_OFLD_RPT_LEN_SH 0
+#define FWCMD_C2H_APF_OFLD_RPT_LEN_MSK 0xffffffff
+#define FWCMD_C2H_APF_OFLD_RPT_OFFSET_LEN_SH 0
+#define FWCMD_C2H_APF_OFLD_RPT_OFFSET_LEN_MSK 0xffffffff
+#define FWCMD_C2H_APF_OFLD_RPT_STATUS_SH 0
+#define FWCMD_C2H_APF_OFLD_RPT_STATUS_MSK 0xffffffff
+#define FWCMD_C2H_APF_OFLD_RPT_APF_RAM_BUF_SH 0
+#define FWCMD_C2H_APF_OFLD_RPT_APF_RAM_BUF_MSK 0xffffffff
+#define FWCMD_C2H_APF_OFLD_RPT_RPT_PTR_SH 0
+#define FWCMD_C2H_APF_OFLD_RPT_RPT_PTR_MSK 0xffffffff
+#define FWCMD_C2H_WAIT_ANNOUNCE_WAIT_CASE_SH 0
+#define FWCMD_C2H_WAIT_ANNOUNCE_WAIT_CASE_MSK 0xf
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID0_SH 8
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID0_MSK 0xff
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID1_SH 16
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID1_MSK 0xff
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID2_SH 24
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID2_MSK 0xff
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID0_EXT_SH 0
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID0_EXT_MSK 0xff
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID1_EXT_SH 8
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID1_EXT_MSK 0xff
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID2_EXT_SH 16
+#define FWCMD_C2H_WAIT_ANNOUNCE_MACID2_EXT_MSK 0xff
+#define FWCMD_C2H_STAT_RPT_TWT_ERR_REC_SH 0
+#define FWCMD_C2H_STAT_RPT_TWT_ERR_REC_MSK 0xffffffff
+#define FWCMD_C2H_STAT_RPT_TWT_INFO_SH 0
+#define FWCMD_C2H_STAT_RPT_TWT_INFO_MSK 0xffffffff
+#define FWCMD_C2H_AOAC_REPORT_AOAC_REPORT_SH 0
+#define FWCMD_C2H_AOAC_REPORT_AOAC_REPORT_MSK 0xffffffff
+#define FWCMD_C2H_READ_WOW_CAM_VALUE_LEN_SH 0
+#define FWCMD_C2H_READ_WOW_CAM_VALUE_LEN_MSK 0x7ff
+#define FWCMD_C2H_READ_WOW_CAM_IDX_SH 24
+#define FWCMD_C2H_READ_WOW_CAM_IDX_MSK 0xff
+#define FWCMD_C2H_READ_WOW_CAM_WOW_CAM_SH 0
+#define FWCMD_C2H_READ_WOW_CAM_WOW_CAM_MSK 0xffffffff
+#define FWCMD_C2H_APF_RPT_PROGRAM_NUMBER_SH 0
+#define FWCMD_C2H_APF_RPT_PROGRAM_NUMBER_MSK 0xff
+#define FWCMD_C2H_APF_RPT_PROGRAM_LEN_SH 0
+#define FWCMD_C2H_APF_RPT_PROGRAM_LEN_MSK 0xffff
+#define FWCMD_C2H_APF_RPT_DATA_LEN_SH 16
+#define FWCMD_C2H_APF_RPT_DATA_LEN_MSK 0xffff
+#define FWCMD_C2H_APF_RPT_PROGRAM_PTR_SH 0
+#define FWCMD_C2H_APF_RPT_PROGRAM_PTR_MSK 0xffffffff
+#define FWCMD_C2H_MCC_RCV_ACK_GROUP_SH 0
+#define FWCMD_C2H_MCC_RCV_ACK_GROUP_MSK 0x3
+#define FWCMD_C2H_MCC_RCV_ACK_H2C_FUNC_SH 8
+#define FWCMD_C2H_MCC_RCV_ACK_H2C_FUNC_MSK 0xff
+#define FWCMD_C2H_MCC_REQ_ACK_GROUP_SH 0
+#define FWCMD_C2H_MCC_REQ_ACK_GROUP_MSK 0x3
+#define FWCMD_C2H_MCC_REQ_ACK_H2C_RETURN_SH 2
+#define FWCMD_C2H_MCC_REQ_ACK_H2C_RETURN_MSK 0x3f
+#define FWCMD_C2H_MCC_REQ_ACK_H2C_FUNC_SH 8
+#define FWCMD_C2H_MCC_REQ_ACK_H2C_FUNC_MSK 0xff
+#define FWCMD_C2H_MCC_TSF_RPT_MACID_X_SH 0
+#define FWCMD_C2H_MCC_TSF_RPT_MACID_X_MSK 0xff
+#define FWCMD_C2H_MCC_TSF_RPT_MACID_Y_SH 8
+#define FWCMD_C2H_MCC_TSF_RPT_MACID_Y_MSK 0xff
+#define FWCMD_C2H_MCC_TSF_RPT_GROUP_SH 16
+#define FWCMD_C2H_MCC_TSF_RPT_GROUP_MSK 0x3
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_LOW_X_SH 0
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_LOW_X_MSK 0xffffffff
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_HIGH_X_SH 0
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_HIGH_X_MSK 0xffffffff
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_LOW_Y_SH 0
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_LOW_Y_MSK 0xffffffff
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_HIGH_Y_SH 0
+#define FWCMD_C2H_MCC_TSF_RPT_TSF_HIGH_Y_MSK 0xffffffff
+#define FWCMD_C2H_MCC_STATUS_RPT_STATUS_SH 0
+#define FWCMD_C2H_MCC_STATUS_RPT_STATUS_MSK 0x3f
+#define FWCMD_C2H_MCC_STATUS_RPT_GROUP_SH 6
+#define FWCMD_C2H_MCC_STATUS_RPT_GROUP_MSK 0x3
+#define FWCMD_C2H_MCC_STATUS_RPT_MACID_SH 8
+#define FWCMD_C2H_MCC_STATUS_RPT_MACID_MSK 0xff
+#define FWCMD_C2H_MCC_STATUS_RPT_TSF_LOW_SH 0
+#define FWCMD_C2H_MCC_STATUS_RPT_TSF_LOW_MSK 0xffffffff
+#define FWCMD_C2H_MCC_STATUS_RPT_TSF_HIGH_SH 0
+#define FWCMD_C2H_MCC_STATUS_RPT_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_C2H_PLAT_FLASH_WRITE_ADDR_SH 0
+#define FWCMD_C2H_PLAT_FLASH_WRITE_ADDR_MSK 0xffffffff
+#define FWCMD_C2H_PLAT_FLASH_WRITE_LENGTH_SH 0
+#define FWCMD_C2H_PLAT_FLASH_WRITE_LENGTH_MSK 0xffffffff
+#define FWCMD_C2H_PLAT_FLASH_ERASE_ADDR_SH 0
+#define FWCMD_C2H_PLAT_FLASH_ERASE_ADDR_MSK 0xffffffff
+#define FWCMD_C2H_PLAT_FLASH_ERASE_LENGTH_SH 0
+#define FWCMD_C2H_PLAT_FLASH_ERASE_LENGTH_MSK 0xffffffff
+#define FWCMD_C2H_PLAT_FLASH_READ_ADDR_SH 0
+#define FWCMD_C2H_PLAT_FLASH_READ_ADDR_MSK 0xffffffff
+#define FWCMD_C2H_PLAT_FLASH_READ_LENGTH_SH 0
+#define FWCMD_C2H_PLAT_FLASH_READ_LENGTH_MSK 0xffffffff
+#define FWCMD_C2H_PLAT_FLASH_READ_PAYLOAD_SH 0
+#define FWCMD_C2H_PLAT_FLASH_READ_PAYLOAD_MSK 0xffffffff
+#define FWCMD_C2H_FCS_RPT_STATUS_SH 0
+#define FWCMD_C2H_FCS_RPT_STATUS_MSK 0xffffffff
+#define FWCMD_C2H_WPS_RPT_STATE_SH 0
+#define FWCMD_C2H_WPS_RPT_STATE_MSK 0xff
+#define FWCMD_C2H_CCXRPT_DWORD0_SH 0
+#define FWCMD_C2H_CCXRPT_DWORD0_MSK 0xffffffff
+#define FWCMD_C2H_CCXRPT_DWORD1_SH 0
+#define FWCMD_C2H_CCXRPT_DWORD1_MSK 0xffffffff
+#define FWCMD_C2H_CCXRPT_DWORD2_SH 0
+#define FWCMD_C2H_CCXRPT_DWORD2_MSK 0xffffffff
+#define FWCMD_C2H_CCXRPT_DWORD3_SH 0
+#define FWCMD_C2H_CCXRPT_DWORD3_MSK 0xffffffff
+#define FWCMD_C2H_CCXRPT_DWORD4_SH 0
+#define FWCMD_C2H_CCXRPT_DWORD4_MSK 0xffffffff
+#define FWCMD_C2H_CCXRPT_DWORD5_SH 0
+#define FWCMD_C2H_CCXRPT_DWORD5_MSK 0xffffffff
+#define FWCMD_C2H_BF_SENS_FEEDBACK_MACID_SH 0
+#define FWCMD_C2H_BF_SENS_FEEDBACK_MACID_MSK 0xfff
+#define FWCMD_C2H_BF_SENS_FEEDBACK_MSR_SETUP_ID_SH 12
+#define FWCMD_C2H_BF_SENS_FEEDBACK_MSR_SETUP_ID_MSK 0x7
+#define FWCMD_C2H_BF_SENS_FEEDBACK_TERMINATE BIT(15)
+#define FWCMD_C2H_BF_SENS_FEEDBACK_MSR_INST_ID_SH 16
+#define FWCMD_C2H_BF_SENS_FEEDBACK_MSR_INST_ID_MSK 0x3f
+#define FWCMD_C2H_BF_SENS_FEEDBACK_TB_MORE_TF BIT(22)
+#define FWCMD_C2H_BF_SENS_FEEDBACK_TB_COMEBACK BIT(23)
+#define FWCMD_C2H_BF_SENS_FEEDBACK_STATUS_BITMAP_SH 0
+#define FWCMD_C2H_BF_SENS_FEEDBACK_STATUS_BITMAP_MSK 0xffffffff
+#define FWCMD_C2H_PORT_INIT_STAT_BAND BIT(0)
+#define FWCMD_C2H_PORT_INIT_STAT_PORT_SH 1
+#define FWCMD_C2H_PORT_INIT_STAT_PORT_MSK 0x7
+#define FWCMD_C2H_PORT_INIT_STAT_CFG_MBID_IDX_SH 8
+#define FWCMD_C2H_PORT_INIT_STAT_CFG_MBID_IDX_MSK 0xff
+#define FWCMD_C2H_PORT_INIT_STAT_CFG_TYPE_SH 16
+#define FWCMD_C2H_PORT_INIT_STAT_CFG_TYPE_MSK 0xff
+#define FWCMD_C2H_PORT_INIT_STAT_STEP_SH 24
+#define FWCMD_C2H_PORT_INIT_STAT_STEP_MSK 0xff
+#define FWCMD_C2H_PORT_INIT_STAT_CFG_VAL_SH 0
+#define FWCMD_C2H_PORT_INIT_STAT_CFG_VAL_MSK 0xffffffff
+#define FWCMD_C2H_PORT_INIT_STAT_RET_SH 0
+#define FWCMD_C2H_PORT_INIT_STAT_RET_MSK 0xffffffff
+#define FWCMD_C2H_PORT_CFG_STAT_BAND BIT(0)
+#define FWCMD_C2H_PORT_CFG_STAT_PORT_SH 1
+#define FWCMD_C2H_PORT_CFG_STAT_PORT_MSK 0x7
+#define FWCMD_C2H_PORT_CFG_STAT_MBSSID_IDX_SH 8
+#define FWCMD_C2H_PORT_CFG_STAT_MBSSID_IDX_MSK 0xff
+#define FWCMD_C2H_PORT_CFG_STAT_TYPE_SH 16
+#define FWCMD_C2H_PORT_CFG_STAT_TYPE_MSK 0xff
+#define FWCMD_C2H_PORT_CFG_STAT_VAL_SH 0
+#define FWCMD_C2H_PORT_CFG_STAT_VAL_MSK 0xffffffff
+#define FWCMD_C2H_PORT_CFG_STAT_RET_SH 0
+#define FWCMD_C2H_PORT_CFG_STAT_RET_MSK 0xffffffff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_USER_NUM_SH 0
+#define FWCMD_C2H_OFDMA_STS_TFSTS_USER_NUM_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_RU_SU_PER_SH 8
+#define FWCMD_C2H_OFDMA_STS_TFSTS_RU_SU_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_MACID_SH 0
+#define FWCMD_C2H_OFDMA_STS_TFSTS_MACID_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_TB_RATE_SH 8
+#define FWCMD_C2H_OFDMA_STS_TFSTS_TB_RATE_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_TB_FAIL_PER_SH 16
+#define FWCMD_C2H_OFDMA_STS_TFSTS_TB_FAIL_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_AVG_TB_RSSI_SH 24
+#define FWCMD_C2H_OFDMA_STS_TFSTS_AVG_TB_RSSI_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_CCA_MISS_PER_SH 0
+#define FWCMD_C2H_OFDMA_STS_TFSTS_CCA_MISS_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_AVG_UPH_SH 8
+#define FWCMD_C2H_OFDMA_STS_TFSTS_AVG_UPH_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_MINFLAG_PER_SH 16
+#define FWCMD_C2H_OFDMA_STS_TFSTS_MINFLAG_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_AVG_TB_EVM_SH 24
+#define FWCMD_C2H_OFDMA_STS_TFSTS_AVG_TB_EVM_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_TF_NUM_SH 0
+#define FWCMD_C2H_OFDMA_STS_TFSTS_TF_NUM_MSK 0xffffffff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_BSR_LEN_SH 0
+#define FWCMD_C2H_OFDMA_STS_TFSTS_BSR_LEN_MSK 0xffff
+#define FWCMD_C2H_OFDMA_STS_TFSTS_MACID_EXT_SH 16
+#define FWCMD_C2H_OFDMA_STS_TFSTS_MACID_EXT_MSK 0xffff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_USER_NUM_SH 0
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_USER_NUM_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_TOTAL_SU_RU_RATIO_SH 8
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_TOTAL_SU_RU_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_TOTAL_RU_FAIL_RATIO_SH 16
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_TOTAL_RU_FAIL_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_TOTAL_SU_FAIL_RATIO_SH 24
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_TOTAL_SU_FAIL_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_MACID_SH 0
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_MACID_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_SU_RU_RATIO_SH 8
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_SU_RU_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_SU_FAIL_SH 16
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_SU_FAIL_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_RU_FAIL_SH 24
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_RU_FAIL_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_AVG_AGG_SH 0
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_AVG_AGG_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_RU_RATE_MCS_SH 8
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_RU_RATE_MCS_MSK 0xf
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_RU_RATE_NSS_SH 12
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_RU_RATE_NSS_MSK 0xf
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_MACID_EXT_SH 16
+#define FWCMD_C2H_OFDMA_STS_DLRUSTS_MACID_EXT_MSK 0xffff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_USER_NUM_SH 0
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_USER_NUM_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_RU_SU_PER_SH 8
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_RU_SU_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_MACID_SH 16
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_MACID_MSK 0x3ff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_TB_RATE_SH 8
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_TB_RATE_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_TB_FAIL_PER_SH 16
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_TB_FAIL_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_AVG_TB_RSSI_SH 24
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_AVG_TB_RSSI_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_CCA_MISS_PER_SH 0
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_CCA_MISS_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_AVG_UPH_SH 8
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_AVG_UPH_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_MINFLAG_PER_SH 16
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_MINFLAG_PER_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_AVG_TB_EVM_SH 24
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_AVG_TB_EVM_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_TF_NUM_SH 0
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_TF_NUM_MSK 0xffffffff
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_BSR_LEN_SH 0
+#define FWCMD_C2H_OFDMA_STS_V1_TFSTS_BSR_LEN_MSK 0xffff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_MACID_SH 16
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_MACID_MSK 0x3ff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_USER_NUM_SH 0
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_USER_NUM_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_TOTAL_SU_RU_RATIO_SH 8
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_TOTAL_SU_RU_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_TOTAL_RU_FAIL_RATIO_SH 16
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_TOTAL_RU_FAIL_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_TOTAL_SU_FAIL_RATIO_SH 24
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_TOTAL_SU_FAIL_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_SU_RU_RATIO_SH 8
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_SU_RU_RATIO_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_SU_FAIL_SH 16
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_SU_FAIL_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_RU_FAIL_SH 24
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_RU_FAIL_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_AVG_AGG_SH 0
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_AVG_AGG_MSK 0xff
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_RU_RATE_MCS_SH 8
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_RU_RATE_MCS_MSK 0xf
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_RU_RATE_NSS_SH 12
+#define FWCMD_C2H_OFDMA_STS_V1_DLRUSTS_RU_RATE_NSS_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_MACID_SH 0
+#define FWCMD_C2H_MLO_GET_TBL_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_GET_TBL_IS_DEF_VAL BIT(16)
+#define FWCMD_C2H_MLO_GET_TBL_BE_MIN_B0_SH 0
+#define FWCMD_C2H_MLO_GET_TBL_BE_MIN_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BE_MAX_B0_SH 4
+#define FWCMD_C2H_MLO_GET_TBL_BE_MAX_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BE_MIN_B1_SH 8
+#define FWCMD_C2H_MLO_GET_TBL_BE_MIN_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BE_MAX_B1_SH 12
+#define FWCMD_C2H_MLO_GET_TBL_BE_MAX_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BK_MIN_B0_SH 16
+#define FWCMD_C2H_MLO_GET_TBL_BK_MIN_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BK_MAX_B0_SH 20
+#define FWCMD_C2H_MLO_GET_TBL_BK_MAX_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BK_MIN_B1_SH 24
+#define FWCMD_C2H_MLO_GET_TBL_BK_MIN_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BK_MAX_B1_SH 28
+#define FWCMD_C2H_MLO_GET_TBL_BK_MAX_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VI_MIN_B0_SH 0
+#define FWCMD_C2H_MLO_GET_TBL_VI_MIN_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VI_MAX_B0_SH 4
+#define FWCMD_C2H_MLO_GET_TBL_VI_MAX_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VI_MIN_B1_SH 8
+#define FWCMD_C2H_MLO_GET_TBL_VI_MIN_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VI_MAX_B1_SH 12
+#define FWCMD_C2H_MLO_GET_TBL_VI_MAX_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VO_MIN_B0_SH 16
+#define FWCMD_C2H_MLO_GET_TBL_VO_MIN_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VO_MAX_B0_SH 20
+#define FWCMD_C2H_MLO_GET_TBL_VO_MAX_B0_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VO_MIN_B1_SH 24
+#define FWCMD_C2H_MLO_GET_TBL_VO_MIN_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VO_MAX_B1_SH 28
+#define FWCMD_C2H_MLO_GET_TBL_VO_MAX_B1_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BE_BA_LIMIT_SH 0
+#define FWCMD_C2H_MLO_GET_TBL_BE_BA_LIMIT_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_BK_BA_LIMIT_SH 4
+#define FWCMD_C2H_MLO_GET_TBL_BK_BA_LIMIT_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VI_BA_LIMIT_SH 8
+#define FWCMD_C2H_MLO_GET_TBL_VI_BA_LIMIT_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_VO_BA_LIMIT_SH 12
+#define FWCMD_C2H_MLO_GET_TBL_VO_BA_LIMIT_MSK 0xf
+#define FWCMD_C2H_MLO_GET_TBL_FIX_TO_B0 BIT(16)
+#define FWCMD_C2H_MLO_GET_TBL_FIX_TO_B1 BIT(17)
+#define FWCMD_C2H_MLO_GET_TBL_WAIT2C_EMPTY BIT(18)
+#define FWCMD_C2H_MLO_GET_TBL_CUR_BAND BIT(19)
+#define FWCMD_C2H_MLO_GET_TBL_B0_CHK BIT(20)
+#define FWCMD_C2H_MLO_GET_TBL_B1_CHK BIT(21)
+#define FWCMD_C2H_MLO_GET_TBL_EXT0_CHK BIT(22)
+#define FWCMD_C2H_MLO_GET_TBL_EXT1_CHK BIT(23)
+#define FWCMD_C2H_MLO_GET_TBL_WD_DROP_BE BIT(24)
+#define FWCMD_C2H_MLO_GET_TBL_WD_DROP_BK BIT(25)
+#define FWCMD_C2H_MLO_GET_TBL_WD_DROP_VI BIT(26)
+#define FWCMD_C2H_MLO_GET_TBL_WD_DROP_VO BIT(27)
+#define FWCMD_C2H_MLO_GET_TBL_PRI_BAND BIT(28)
+#define FWCMD_C2H_EMLSR_TRANS_DONE_MACID_SH 0
+#define FWCMD_C2H_EMLSR_TRANS_DONE_MACID_MSK 0xff
+#define FWCMD_C2H_EMLSR_TRANS_DONE_EXT_MACID_SH 8
+#define FWCMD_C2H_EMLSR_TRANS_DONE_EXT_MACID_MSK 0xff
+#define FWCMD_C2H_EMLSR_STA_CFG_DONE_MACID_SH 0
+#define FWCMD_C2H_EMLSR_STA_CFG_DONE_MACID_MSK 0xffff
+#define FWCMD_C2H_EMLSR_STA_CFG_DONE_ENABLE BIT(16)
+#define FWCMD_C2H_EMLSR_STA_CFG_DONE_STATUS_SH 24
+#define FWCMD_C2H_EMLSR_STA_CFG_DONE_STATUS_MSK 0xf
+#define FWCMD_C2H_MCMLO_RELINK_RPT_MACID_SH 0
+#define FWCMD_C2H_MCMLO_RELINK_RPT_MACID_MSK 0xffff
+#define FWCMD_C2H_MCMLO_RELINK_RPT_AC_BMP_SH 16
+#define FWCMD_C2H_MCMLO_RELINK_RPT_AC_BMP_MSK 0xf
+#define FWCMD_C2H_MCMLO_RELINK_RPT_OPERATION_SH 20
+#define FWCMD_C2H_MCMLO_RELINK_RPT_OPERATION_MSK 0x3
+#define FWCMD_C2H_MCMLO_RELINK_RPT_BAND BIT(22)
+#define FWCMD_C2H_MCMLO_RELINK_RPT_STATUS_SH 24
+#define FWCMD_C2H_MCMLO_RELINK_RPT_STATUS_MSK 0xff
+#define FWCMD_C2H_MCMLO_SN_SYNC_RPT_SRC_LINK_IDX_SH 0
+#define FWCMD_C2H_MCMLO_SN_SYNC_RPT_SRC_LINK_IDX_MSK 0x3
+#define FWCMD_C2H_MCMLO_SN_SYNC_RPT_MACID_START_SH 0
+#define FWCMD_C2H_MCMLO_SN_SYNC_RPT_MACID_START_MSK 0xffff
+#define FWCMD_C2H_MCMLO_SN_SYNC_RPT_MACID_NUM_SH 16
+#define FWCMD_C2H_MCMLO_SN_SYNC_RPT_MACID_NUM_MSK 0xffff
+#define FWCMD_C2H_MLO_LINK_CFG_STAT_MACID_SH 0
+#define FWCMD_C2H_MLO_LINK_CFG_STAT_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_LINK_CFG_STAT_STATUS_SH 16
+#define FWCMD_C2H_MLO_LINK_CFG_STAT_STATUS_MSK 0xf
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_MAIN_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_MAIN_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_ID_SH 16
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_ID_MSK 0xf
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_CUR_INDEX_SH 20
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_CUR_INDEX_MSK 0xf
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_DECISION_SH 24
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_DECISION_MSK 0xff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_CUR_MLO_MIN_LVL_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_CUR_MLO_MIN_LVL_MSK 0xf
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_CUR_MLO_MAX_LVL_SH 4
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_CUR_MLO_MAX_LVL_MSK 0xf
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_CUR_MLO_MIN_LVL_SH 8
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_CUR_MLO_MIN_LVL_MSK 0xf
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_CUR_MLO_MAX_LVL_SH 12
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_CUR_MLO_MAX_LVL_MSK 0xf
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_CUR_TX_LINK_SH 16
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_CUR_TX_LINK_MSK 0x3
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_TOTAL_PKT_SUM_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_TOTAL_PKT_SUM_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_TOTAL_PKT_SUM_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_TOTAL_PKT_SUM_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_PKT_OK_SUM_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_PKT_OK_SUM_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_PKT_OK_SUM_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_PKT_OK_SUM_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_TX_LEN_SUM_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_TX_LEN_SUM_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_TX_LEN_SUM_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_TX_LEN_SUM_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_EST_TP_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_EST_TP_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_EST_TP_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_EST_TP_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_TP_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_TP_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_TP_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_TP_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_Q_EMPTY_SH 0
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_Q_EMPTY_MSK 0xff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_Q_EMPTY_SH 8
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_Q_EMPTY_MSK 0xff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_UTIL_SH 16
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B0_UTIL_MSK 0xff
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_UTIL_SH 24
+#define FWCMD_C2H_MLO_DM_DBG_DUMP_B1_UTIL_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_ENABLE BIT(0)
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_RELINK_SH 1
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_RELINK_MSK 0x3
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_INTERVAL_SH 3
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_INTERVAL_MSK 0x1f
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_0_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_0_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_1_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_1_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_2_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_2_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_3_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_3_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_4_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_4_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_5_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_5_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_6_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_6_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_7_EXIST_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_7_EXIST_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_0_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_0_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_1_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_1_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_2_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_2_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_3_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_3_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_4_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_4_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_5_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_5_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_6_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_6_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_7_VALID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_STA_LST_7_VALID_MSK 0xffffffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_LST_EN_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_LST_EN_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_LST_VALID_SH 8
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_LST_VALID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_0_MLD_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_0_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_1_MLD_SH 8
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_1_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_2_MLD_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_2_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_3_MLD_SH 24
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_3_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_4_MLD_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_4_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_5_MLD_SH 8
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_5_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_6_MLD_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_6_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_7_MLD_SH 24
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_DBG_7_MLD_MSK 0xff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_0_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_0_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_1_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_1_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_2_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_2_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_3_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_3_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_4_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_4_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_5_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_5_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_6_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_6_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_7_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_7_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_8_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_8_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_9_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_9_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_10_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_10_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_11_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_11_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_12_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_12_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_13_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_13_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_14_MACID_SH 0
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_14_MACID_MSK 0xffff
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_15_MACID_SH 16
+#define FWCMD_C2H_MLO_DM_STATUS_DUMP_SND_15_MACID_MSK 0xffff
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_CAT_SH 0
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_CLASS_SH 2
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_FUNC_SH 8
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_H2C_RETURN_SH 16
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_H2C_RETURN_MSK 0xff
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_H2C_SEQ_SH 24
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_SCHEDULE_ID_SH 0
+#define FWCMD_C2H_ACT_SCHEDULE_REQ_ACK_SCHEDULE_ID_MSK 0xffffffff
+#define FWCMD_C2H_BCN_REQ_ACK_CAT_SH 0
+#define FWCMD_C2H_BCN_REQ_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_BCN_REQ_ACK_CLASS_SH 2
+#define FWCMD_C2H_BCN_REQ_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_BCN_REQ_ACK_FUNC_SH 8
+#define FWCMD_C2H_BCN_REQ_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_BCN_REQ_ACK_H2C_RETURN_SH 16
+#define FWCMD_C2H_BCN_REQ_ACK_H2C_RETURN_MSK 0xff
+#define FWCMD_C2H_BCN_REQ_ACK_H2C_SEQ_SH 24
+#define FWCMD_C2H_BCN_REQ_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_CAT_SH 0
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_CLASS_SH 2
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_FUNC_SH 8
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_H2C_RETURN_SH 16
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_H2C_RETURN_MSK 0xff
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_H2C_SEQ_SH 24
+#define FWCMD_C2H_NAN_FUNC_CTRL_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_NAN_DE_INFO_ACK_CAT_SH 0
+#define FWCMD_C2H_NAN_DE_INFO_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_NAN_DE_INFO_ACK_CLASS_SH 2
+#define FWCMD_C2H_NAN_DE_INFO_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_NAN_DE_INFO_ACK_FUNC_SH 8
+#define FWCMD_C2H_NAN_DE_INFO_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_NAN_DE_INFO_ACK_H2C_RETURN_SH 16
+#define FWCMD_C2H_NAN_DE_INFO_ACK_H2C_RETURN_MSK 0xff
+#define FWCMD_C2H_NAN_DE_INFO_ACK_H2C_SEQ_SH 24
+#define FWCMD_C2H_NAN_DE_INFO_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_CAT_SH 0
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_CLASS_SH 2
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_FUNC_SH 8
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_H2C_RETURN_SH 16
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_H2C_RETURN_MSK 0xff
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_H2C_SEQ_SH 24
+#define FWCMD_C2H_NAN_JOIN_CLUSTER_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_CAT_SH 0
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_CAT_MSK 0x3
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_CLASS_SH 2
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_CLASS_MSK 0x3f
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_FUNC_SH 8
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_FUNC_MSK 0xff
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_H2C_RETURN_SH 16
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_H2C_RETURN_MSK 0xff
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_H2C_SEQ_SH 24
+#define FWCMD_C2H_NAN_PAUSE_FAW_TX_ACK_H2C_SEQ_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_INFO1_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_INFO1_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID0_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID0_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID1_SH 8
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID1_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID2_SH 16
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID2_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID3_SH 24
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID3_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID4_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID4_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID5_SH 8
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_CLUSTER_ID5_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_MASTERPREF_SH 16
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_MASTERPREF_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_RANDOMFACTOR_SH 24
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_RANDOMFACTOR_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_AMR_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_AMR_MSK 0xffffffff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_AMBTT_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_AMBTT_MSK 0xffffffff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_HOPCOUNT_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_HOPCOUNT_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_AMR_HIGH_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_INFO_AMR_HIGH_MSK 0xffffffff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_TSF_INFO_INFO2_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_TSF_INFO_INFO2_MSK 0xff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_TSF_INFO_PORT_DWST_LOW_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_TSF_INFO_PORT_DWST_LOW_MSK 0xffffffff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_TSF_INFO_FR_DWST_LOW_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_TSF_INFO_FR_DWST_LOW_MSK 0xffffffff
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_JOIN_INFO3_SH 0
+#define FWCMD_C2H_NAN_INFO_NOTIFY_CLUSTER_JOIN_INFO3_MSK 0xff
+#define FWCMD_C2H_NAN_DEINIT_RPT_STATUS_SH 0
+#define FWCMD_C2H_NAN_DEINIT_RPT_STATUS_MSK 0xff
+#define FWCMD_C2H_MRC_TSF_RPT_REQ_TSF_NUM_SH 0
+#define FWCMD_C2H_MRC_TSF_RPT_REQ_TSF_NUM_MSK 0xff
+#define FWCMD_C2H_MRC_TSF_RPT_TSF_LOW_SH 0
+#define FWCMD_C2H_MRC_TSF_RPT_TSF_LOW_MSK 0xffffffff
+#define FWCMD_C2H_MRC_TSF_RPT_TSF_HIGH_SH 0
+#define FWCMD_C2H_MRC_TSF_RPT_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_C2H_MRC_STATUS_RPT_STATUS_SH 0
+#define FWCMD_C2H_MRC_STATUS_RPT_STATUS_MSK 0x3f
+#define FWCMD_C2H_MRC_STATUS_RPT_SCH_IDX_SH 6
+#define FWCMD_C2H_MRC_STATUS_RPT_SCH_IDX_MSK 0x3
+#define FWCMD_C2H_MRC_STATUS_RPT_TSF_LOW_SH 0
+#define FWCMD_C2H_MRC_STATUS_RPT_TSF_LOW_MSK 0xffffffff
+#define FWCMD_C2H_MRC_STATUS_RPT_TSF_HIGH_SH 0
+#define FWCMD_C2H_MRC_STATUS_RPT_TSF_HIGH_MSK 0xffffffff
+#define FWCMD_C2H_MRC_NOTIFY_EVT_STATUS_SH 0
+#define FWCMD_C2H_MRC_NOTIFY_EVT_STATUS_MSK 0xff
+#define FWCMD_C2H_MRC_NOTIFY_EVT_BAND_SH 8
+#define FWCMD_C2H_MRC_NOTIFY_EVT_BAND_MSK 0xf
+#define FWCMD_C2H_MRC_NOTIFY_EVT_PORT_SH 12
+#define FWCMD_C2H_MRC_NOTIFY_EVT_PORT_MSK 0xf
+#define FWCMD_C2H_DOZE_TIME_RPT_MACID_SH 0
+#define FWCMD_C2H_DOZE_TIME_RPT_MACID_MSK 0xff
+#define FWCMD_C2H_DOZE_TIME_RPT_DOZE_TIME_SH 0
+#define FWCMD_C2H_DOZE_TIME_RPT_DOZE_TIME_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD0_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD0_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD1_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD1_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD2_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD2_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD3_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD3_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD4_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD4_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD5_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD5_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD6_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD6_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD7_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD7_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD8_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD8_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD9_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD9_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD10_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD10_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD11_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD11_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD12_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD12_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD13_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD13_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD14_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD14_MSK 0xffffffff
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD15_SH 0
+#define FWCMD_C2H_DCTL_INFO_DWORD_DWORD15_MSK 0xffffffff
+#define FWCMD_C2H_OM_ADV_RPT_MACID_SH 0
+#define FWCMD_C2H_OM_ADV_RPT_MACID_MSK 0xffff
+#define FWCMD_C2H_OM_ADV_RPT_TYPE_SH 16
+#define FWCMD_C2H_OM_ADV_RPT_TYPE_MSK 0xf
+#define FWCMD_C2H_WOW_DIAG_RPT_TSF_H_SH 0
+#define FWCMD_C2H_WOW_DIAG_RPT_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_RPT_TSF_L_SH 0
+#define FWCMD_C2H_WOW_DIAG_RPT_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_RPT_ERR_CODE_SH 0
+#define FWCMD_C2H_WOW_DIAG_RPT_ERR_CODE_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_GTK_M1_RCV_SH 0
+#define FWCMD_C2H_WOW_DIAG_GTK_M1_RCV_MSK 0xffff
+#define FWCMD_C2H_WOW_DIAG_GTK_M2_ENQ_SH 16
+#define FWCMD_C2H_WOW_DIAG_GTK_M2_ENQ_MSK 0xffff
+#define FWCMD_C2H_WOW_DIAG_GTK_M2_SUC_SH 0
+#define FWCMD_C2H_WOW_DIAG_GTK_M2_SUC_MSK 0xffff
+#define FWCMD_C2H_WOW_DIAG_GTK_MAC_DROP_SH 16
+#define FWCMD_C2H_WOW_DIAG_GTK_MAC_DROP_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_LIFE_DROP_SH 24
+#define FWCMD_C2H_WOW_DIAG_GTK_LIFE_DROP_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_RETRY_DROP_SH 0
+#define FWCMD_C2H_WOW_DIAG_GTK_RETRY_DROP_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_OTHER_ERR_SH 8
+#define FWCMD_C2H_WOW_DIAG_GTK_OTHER_ERR_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_M1_RCV_LAST_SH 16
+#define FWCMD_C2H_WOW_DIAG_GTK_M1_RCV_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_M2_ENQ_LAST_SH 24
+#define FWCMD_C2H_WOW_DIAG_GTK_M2_ENQ_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_MAC_DROP_LAST_SH 0
+#define FWCMD_C2H_WOW_DIAG_GTK_MAC_DROP_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_LIFE_DROP_LAST_SH 8
+#define FWCMD_C2H_WOW_DIAG_GTK_LIFE_DROP_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_RETRY_DROP_LAST_SH 16
+#define FWCMD_C2H_WOW_DIAG_GTK_RETRY_DROP_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_GTK_OTHER_ERR_LAST_SH 24
+#define FWCMD_C2H_WOW_DIAG_GTK_OTHER_ERR_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_BCN_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_BCN_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_TX_SUCCESS_CNT_SH 8
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_TX_SUCCESS_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_TX_FAIL_CNT_SH 16
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_TX_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_TX_FAIL_RSN_SH 24
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_TX_FAIL_RSN_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_DISCONNECT_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_DISCONNECT_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_DISCONNECT_LIMIT_SH 8
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_DISCONNECT_LIMIT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_RETRY_CNT_SH 16
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_RETRY_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_RETRY_LIMIT_SH 24
+#define FWCMD_C2H_WOW_DIAG_AP_LOST_RETRY_LIMIT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_SER_L0_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_SER_L0_CNT_MSK 0xffff
+#define FWCMD_C2H_WOW_DIAG_SER_L1_CNT_SH 16
+#define FWCMD_C2H_WOW_DIAG_SER_L1_CNT_MSK 0xffff
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_RX_EVT_MASK_SH 0
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_RX_EVT_MASK_MSK 0xffffff
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_RX_EVT_CNT_SH 24
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_RX_EVT_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_EARLY_SH 0
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_EARLY_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_OK_SH 0
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_OK_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_NO_HIT_SH 0
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_NO_HIT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_LOST_RATE_SH 0
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_LOST_RATE_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_ROLE_IDX_SH 8
+#define FWCMD_C2H_WOW_DIAG_RX_EVT_BCN_ROLE_IDX_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_LAST_TSF_H_SH 0
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_LAST_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_LAST_TSF_L_SH 0
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_LAST_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_ENQ_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_ENQ_CNT_MSK 0xffff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_ENQ_FAIL_CNT_SH 16
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_ENQ_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_FAIL_CNT_SH 24
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_RESULT_LAST_SH 0
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_RESULT_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_RET_TYPE_LAST_SH 8
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_TX_RET_TYPE_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_DISABLED BIT(16)
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_INVALID_BANDPORT BIT(17)
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_DISCONNECT BIT(18)
+#define FWCMD_C2H_WOW_DIAG_KEEPALIVE_PKT_OFLD_FAIL BIT(19)
+#define FWCMD_C2H_WOW_DIAG_ARP_ARP_REQ_LAST_TSF_H_SH 0
+#define FWCMD_C2H_WOW_DIAG_ARP_ARP_REQ_LAST_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_ARP_ARP_REQ_LAST_TSF_L_SH 0
+#define FWCMD_C2H_WOW_DIAG_ARP_ARP_REQ_LAST_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_ARP_TX_RESULT_LAST_SH 0
+#define FWCMD_C2H_WOW_DIAG_ARP_TX_RESULT_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_ARP_TX_RET_TYPE_LAST_SH 8
+#define FWCMD_C2H_WOW_DIAG_ARP_TX_RET_TYPE_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_ARP_BSSID_MATCH_LAST BIT(16)
+#define FWCMD_C2H_WOW_DIAG_ARP_IP_MATCH_LAST BIT(17)
+#define FWCMD_C2H_WOW_DIAG_ARP_ACT_WAKEUP BIT(18)
+#define FWCMD_C2H_WOW_DIAG_ARP_TKIP_WAKEUP BIT(19)
+#define FWCMD_C2H_WOW_DIAG_ARP_PKT_OFLD_FAIL BIT(20)
+#define FWCMD_C2H_WOW_DIAG_ARP_ENQ_FAIL_LAST BIT(21)
+#define FWCMD_C2H_WOW_DIAG_PMF_ACT_LAST_TSF_H_SH 0
+#define FWCMD_C2H_WOW_DIAG_PMF_ACT_LAST_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PMF_ACT_LAST_TSF_L_SH 0
+#define FWCMD_C2H_WOW_DIAG_PMF_ACT_LAST_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PMF_DEAUTH_LAST_TSF_H_SH 0
+#define FWCMD_C2H_WOW_DIAG_PMF_DEAUTH_LAST_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PMF_DEAUTH_LAST_TSF_L_SH 0
+#define FWCMD_C2H_WOW_DIAG_PMF_DEAUTH_LAST_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PMF_TX_RESULT_LAST_SH 0
+#define FWCMD_C2H_WOW_DIAG_PMF_TX_RESULT_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_PMF_TX_RET_TYPE_LAST_SH 8
+#define FWCMD_C2H_WOW_DIAG_PMF_TX_RET_TYPE_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_PMF_NO_FCS_LAST BIT(16)
+#define FWCMD_C2H_WOW_DIAG_PMF_SA_QUERY_REQ_LAST BIT(17)
+#define FWCMD_C2H_WOW_DIAG_PMF_PKT_OFLOD_FAIL_LAST BIT(18)
+#define FWCMD_C2H_WOW_DIAG_PMF_ENQ_LAST BIT(19)
+#define FWCMD_C2H_WOW_DIAG_PMF_ENQ_FAIL_LAST BIT(20)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_NULL_PARM BIT(0)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_DISABLED BIT(1)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_WRONG_ROLE BIT(2)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_TX_TIMER_FAIL BIT(3)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_TM_TIMER_FAIL BIT(4)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_RD_PKT_OFLD_FAIL BIT(5)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_ENQ_FAIL BIT(6)
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_RETRY_CNT_SH 8
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_RETRY_CNT_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_TX_RESULT_LAST_SH 16
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_TX_RESULT_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_TX_RET_TYPE_LAST_SH 24
+#define FWCMD_C2H_WOW_DIAG_TCP_KEEPALIVE_TX_RET_TYPE_LAST_MSK 0xff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_RECV_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_RECV_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_RSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_RSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_NO_CORRSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_NO_CORRSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_NO_MEM_DROP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_MDNS_NO_MEM_DROP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_RECV_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_RECV_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_RSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_RSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_NO_CORRSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_NO_CORRSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_NO_MEM_DROP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_SNMP_NO_MEM_DROP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_RECV_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_RECV_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_RSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_RSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_NO_CORRSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_NO_CORRSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_NO_MEM_DROP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_LLMNR_NO_MEM_DROP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_RECV_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_RECV_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_RSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_RSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_NO_CORRSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_NO_CORRSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_NO_MEM_DROP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_NETBIOS_NO_MEM_DROP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_RECV_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_RECV_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_RSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_RSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_NO_CORRSP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_NO_CORRSP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_NO_MEM_DROP_CNT_SH 0
+#define FWCMD_C2H_WOW_DIAG_PROXY_OFLD_PATTERN_NO_MEM_DROP_CNT_MSK 0xffffffff
+#define FWCMD_C2H_WOW_DUMP_RX_BUF_LEN_SH 0
+#define FWCMD_C2H_WOW_DUMP_RX_BUF_LEN_MSK 0xffff
+#define FWCMD_C2H_WOW_DUMP_MEM_BUF_LEN_SH 0
+#define FWCMD_C2H_WOW_DUMP_MEM_BUF_LEN_MSK 0xffff
+#define FWCMD_C2H_WOW_DUMP_ROLE_ROLE_NUM_SH 0
+#define FWCMD_C2H_WOW_DUMP_ROLE_ROLE_NUM_MSK 0xff
+#define FWCMD_C2H_WOW_DUMP_BB_RF_REG_BUF_LEN_SH 0
+#define FWCMD_C2H_WOW_DUMP_BB_RF_REG_BUF_LEN_MSK 0xffff
+#define FWCMD_C2H_FTM_AZ_PKTID_CONTENT_FTM_AZ_PARA_TYPE_LEN_SH 0
+#define FWCMD_C2H_FTM_AZ_PKTID_CONTENT_FTM_AZ_PARA_TYPE_LEN_MSK 0xff
+#define FWCMD_C2H_FTM_AZ_PARA_DUMP_FTM_AZ_PARA_TYPE_LEN_SH 0
+#define FWCMD_C2H_FTM_AZ_PARA_DUMP_FTM_AZ_PARA_TYPE_LEN_MSK 0xff
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_FTM_AZ_PARA_TYPE_LEN_SH 0
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_FTM_AZ_PARA_TYPE_LEN_MSK 0xff
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_FUNC_NO_SH 8
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_FUNC_NO_MSK 0xff
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_STAT_CODE_SH 16
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_STAT_CODE_MSK 0xff
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_ERROR_CODE_SH 24
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_ERROR_CODE_MSK 0xff
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_FW_DEFINE_SH 0
+#define FWCMD_C2H_FTM_AZ_DBG_DUMP_FW_DEFINE_MSK 0xff
+#define FWCMD_C2H_FTM_AZ_RPT_HDL_FTM_AZ_PARA_TYPE_LEN_SH 0
+#define FWCMD_C2H_FTM_AZ_RPT_HDL_FTM_AZ_PARA_TYPE_LEN_MSK 0xff
+#define FWCMD_C2H_FTM_MC_PKTID_CONTENT_FTM_MC_PARA_TYPE_LEN_SH 0
+#define FWCMD_C2H_FTM_MC_PKTID_CONTENT_FTM_MC_PARA_TYPE_LEN_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_0_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_0_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_0 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_0 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_0_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_0_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_0_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_0_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_1_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_1_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_1 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_1 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_1_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_1_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_1_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_1_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_2_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_2_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_2 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_2 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_2_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_2_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_2_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_2_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_3_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_3_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_3 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_3 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_3_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_3_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_3_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_3_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_4_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_4_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_4 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_4 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_4_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_4_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_4_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_4_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_5_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_5_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_5 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_5 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_5_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_5_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_5_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_5_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_6_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_6_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_6 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_6 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_6_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_6_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_6_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_6_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_7_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_7_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_7 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_7 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_7_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_7_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_7_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_7_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_8_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_8_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_8 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_8 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_8_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_8_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_8_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_8_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_9_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_9_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_9 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_9 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_9_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_9_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_9_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_9_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_10_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_10_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_10 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_10 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_10_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_10_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_10_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_10_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_11_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_11_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_11 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_11 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_11_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_11_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_11_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_11_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_12_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_12_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_12 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_12 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_12_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_12_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_12_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_12_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_13_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_13_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_13 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_13 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_13_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_13_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_13_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_13_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_14_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_14_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_14 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_14 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_14_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_14_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_14_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_14_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_15_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_FEATURE_15_MSK 0xff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_LAST_15 BIT(8)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_BADN_15 BIT(9)
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_15_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_WARN_BITMAP_15_MSK 0xffffffff
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_15_SH 0
+#define FWCMD_C2H_AP_SELF_DIAG_RST_ERR_BITMAP_15_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_CHK_NUM_NORMAL_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_CHK_NUM_NORMAL_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_RPT_CHK_NUM_WOW_SH 8
+#define FWCMD_C2H_STA_DIAG_RPT_CHK_NUM_WOW_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_RPT_ERR_CODE_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_ERR_CODE_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_ERR_CODE_WOW_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_ERR_CODE_WOW_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_BCN_EARLY_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_BCN_EARLY_CNT_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_BCN_OK_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_BCN_OK_CNT_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_BCN_RCV_MASK_LAST_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_BCN_RCV_MASK_LAST_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_WOW_ENTER_TSF_H_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_WOW_ENTER_TSF_H_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_WOW_ENTER_TSF_L_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_WOW_ENTER_TSF_L_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_RPT_WOW_TOGGLE_WAKE_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_RPT_WOW_TOGGLE_WAKE_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_CB_FAIL_RTY_DROP_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_CB_FAIL_RTY_DROP_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_CB_FAIL_LIFE_DROP_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_CB_FAIL_LIFE_DROP_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_CB_FAIL_MACID_DROP_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_CB_FAIL_MACID_DROP_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_API_FAIL_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_SCAN_NULL_API_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_TX_CB_FAIL_RTY_DROP_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_SCAN_TX_CB_FAIL_RTY_DROP_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_TX_CB_FAIL_LIFE_DROP_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_SCAN_TX_CB_FAIL_LIFE_DROP_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_TX_CB_FAIL_MACID_DROP_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_SCAN_TX_CB_FAIL_MACID_DROP_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_READ_PKTOFLD_FAIL_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_SCAN_READ_PKTOFLD_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_MALFORMED_PROBE_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_SCAN_MALFORMED_PROBE_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_TXAPI_FAIL_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_SCAN_TXAPI_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_TXAPI_LAST_FAIL_RETTYPE_SH 16
+#define FWCMD_C2H_STA_DIAG_SCAN_TXAPI_LAST_FAIL_RETTYPE_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_TXAPI_LAST_FAIL_RESULT_SH 24
+#define FWCMD_C2H_STA_DIAG_SCAN_TXAPI_LAST_FAIL_RESULT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_CHSWITCH_FAIL_RET_SH 0
+#define FWCMD_C2H_STA_DIAG_SCAN_CHSWITCH_FAIL_RET_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_SCAN_BB_CTRL_MLO_FAIL_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_SCAN_BB_CTRL_MLO_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_RF_CTRL_MLO_FAIL_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_SCAN_RF_CTRL_MLO_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_RF_RELOAD_FAIL BIT(16)
+#define FWCMD_C2H_STA_DIAG_SCAN_NUM_MACID_PAUSE_SLEEP_MISMATCH_SH 17
+#define FWCMD_C2H_STA_DIAG_SCAN_NUM_MACID_PAUSE_SLEEP_MISMATCH_MSK 0x3
+#define FWCMD_C2H_STA_DIAG_SCAN_C2H_NOTIFY_FAIL_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_SCAN_C2H_NOTIFY_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_0_SH 0
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_0_MSK 0xffff
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_1_SH 16
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_1_MSK 0xffff
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_2_SH 0
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_2_MSK 0xffff
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_3_SH 16
+#define FWCMD_C2H_STA_DIAG_SCAN_MACID_PAUSE_SLEEP_MISMATCH_3_MSK 0xffff
+#define FWCMD_C2H_STA_DIAG_CSA_PRI_NOT_VALID_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_CSA_PRI_NOT_VALID_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_CSA_BW_NOT_VALID_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_CSA_BW_NOT_VALID_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_CSA_NOT_VALID_PRI_SH 16
+#define FWCMD_C2H_STA_DIAG_CSA_NOT_VALID_PRI_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_CSA_NOT_VALID_BW_SH 24
+#define FWCMD_C2H_STA_DIAG_CSA_NOT_VALID_BW_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_CSA_NEWBCN_TIMEOUT_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_CSA_NEWBCN_TIMEOUT_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_CSA_CHSW_FAILED_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_CSA_CHSW_FAILED_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_CSA_NULL_FAILED_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_CSA_NULL_FAILED_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_CSA_CHSW_FAILED_RET_SH 0
+#define FWCMD_C2H_STA_DIAG_CSA_CHSW_FAILED_RET_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_PS_TOTAL_MACID_NUM_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_TOTAL_MACID_NUM_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_NTFY_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_NTFY_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_REQ_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_REQ_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_ACT_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_ACT_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_CG_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_CG_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_PG_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_PS_RPWM_PG_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_ENTER_CG_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_PS_ENTER_CG_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_ENTER_PG_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_PS_ENTER_PG_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_MSTATUS_VAL_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_MSTATUS_VAL_MSK 0xffffffff
+#define FWCMD_C2H_STA_DIAG_PS_B0_NOR_MODE_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_B0_NOR_MODE_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_B0_CLS_RF_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_PS_B0_CLS_RF_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_B0_OPEN_RF_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_PS_B0_OPEN_RF_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_B1_NOR_MODE_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_PS_B1_NOR_MODE_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_B1_CLS_RF_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_B1_CLS_RF_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_B1_OPEN_RF_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_PS_B1_OPEN_RF_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_MACID_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_MACID_MSK 0xffff
+#define FWCMD_C2H_STA_DIAG_PS_BCN_UC_HIT_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_PS_BCN_UC_HIT_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_BCN_BC_HIT_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_PS_BCN_BC_HIT_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RX_BCN_RATE_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_RX_BCN_RATE_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RX_BCN_MODE_RATE_SH 8
+#define FWCMD_C2H_STA_DIAG_PS_RX_BCN_MODE_RATE_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RX_NOR_MODE_RATE_SH 16
+#define FWCMD_C2H_STA_DIAG_PS_RX_NOR_MODE_RATE_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_RX_KEEP_ON_RATE_SH 24
+#define FWCMD_C2H_STA_DIAG_PS_RX_KEEP_ON_RATE_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_BCN_ERLY_TIME_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_BCN_ERLY_TIME_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_BCN_TO_TIME_SH 8
+#define FWCMD_C2H_STA_DIAG_PS_BCN_TO_TIME_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_TBTT_SHIFT_TIME_SH 16
+#define FWCMD_C2H_STA_DIAG_PS_TBTT_SHIFT_TIME_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_TX_NULL1_FAIL_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_PS_TX_NULL1_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_TX_NULL0_FAIL_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_PS_TX_NULL0_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_NULL_TX_FAIL_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_PS_NULL_TX_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_TXPKTIN_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_PS_TXPKTIN_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_PS_SW_TX_OK_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_PS_SW_TX_OK_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_ADD_MCC_FAIL_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_ADD_MCC_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_START_MCC_FAIL_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_START_MCC_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_STOP_MCC_FAIL_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_STOP_MCC_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_DEL_MCC_FAIL_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_DEL_MCC_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_MCC_SET_DUR_FAIL_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_MCC_H2C_MCC_SET_DUR_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MCC_CHANGE_ROLE_FAIL_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_MCC_CHANGE_ROLE_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MCC_SW_CH_FAIL_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_MCC_SW_CH_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_EFUSE_H2C_EFUSE_ERR_CODE_SH 0
+#define FWCMD_C2H_STA_DIAG_EFUSE_H2C_EFUSE_ERR_CODE_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_ADD_MRC_FAIL_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_ADD_MRC_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_START_MRC_FAIL_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_START_MRC_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_DEL_MRC_FAIL_CNT_SH 16
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_DEL_MRC_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_MRC_UPD_DUR_FAIL_CNT_SH 24
+#define FWCMD_C2H_STA_DIAG_MRC_H2C_MRC_UPD_DUR_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MRC_CHANGE_SLOT_FAIL_CNT_SH 0
+#define FWCMD_C2H_STA_DIAG_MRC_CHANGE_SLOT_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_STA_DIAG_MRC_SW_CH_FAIL_CNT_SH 8
+#define FWCMD_C2H_STA_DIAG_MRC_SW_CH_FAIL_CNT_MSK 0xff
+#define FWCMD_C2H_PWR_INT_NOTIFY_MACID_NUM_SH 0
+#define FWCMD_C2H_PWR_INT_NOTIFY_MACID_NUM_MSK 0xffff
+#define FWCMD_C2H_PWR_INT_NOTIFY_CURR_PWR_STATUS BIT(16)
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, Phydm) ------------------*/
+#define C2H_HDR_CAT_SH 0
+#define C2H_HDR_CAT_MSK 0x3
+#define C2H_HDR_CLASS_SH 2
+#define C2H_HDR_CLASS_MSK 0x3f
+#define C2H_HDR_FUNC_SH 8
+#define C2H_HDR_FUNC_MSK 0xff
+#define C2H_HDR_DEL_TYPE_SH 16
+#define C2H_HDR_DEL_TYPE_MSK 0xf
+#define C2H_HDR_C2H_SEQ_SH 24
+#define C2H_HDR_C2H_SEQ_MSK 0xff
+#define C2H_HDR_TOTAL_LEN_SH 0
+#define C2H_HDR_TOTAL_LEN_MSK 0x3fff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, RF) ---------------------*/
+#define C2H_HDR_CAT_SH 0
+#define C2H_HDR_CAT_MSK 0x3
+#define C2H_HDR_CLASS_SH 2
+#define C2H_HDR_CLASS_MSK 0x3f
+#define C2H_HDR_FUNC_SH 8
+#define C2H_HDR_FUNC_MSK 0xff
+#define C2H_HDR_DEL_TYPE_SH 16
+#define C2H_HDR_DEL_TYPE_MSK 0xf
+#define C2H_HDR_C2H_SEQ_SH 24
+#define C2H_HDR_C2H_SEQ_MSK 0xff
+#define C2H_HDR_TOTAL_LEN_SH 0
+#define C2H_HDR_TOTAL_LEN_MSK 0x3fff
+#define FWCMD_C2H_RF_REPORT_RF_REPORT_CONTENT_SH 0
+#define FWCMD_C2H_RF_REPORT_RF_REPORT_CONTENT_MSK 0xff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, BTC) --------------------*/
+#define C2H_HDR_CAT_SH 0
+#define C2H_HDR_CAT_MSK 0x3
+#define C2H_HDR_CLASS_SH 2
+#define C2H_HDR_CLASS_MSK 0x3f
+#define C2H_HDR_FUNC_SH 8
+#define C2H_HDR_FUNC_MSK 0xff
+#define C2H_HDR_DEL_TYPE_SH 16
+#define C2H_HDR_DEL_TYPE_MSK 0xf
+#define C2H_HDR_C2H_SEQ_SH 24
+#define C2H_HDR_C2H_SEQ_MSK 0xff
+#define C2H_HDR_TOTAL_LEN_SH 0
+#define C2H_HDR_TOTAL_LEN_MSK 0x3fff
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_SH 0
+#define FWCMD_C2H_EXAMPLE_EXAMPLE_CONTENT_MSK 0xffffffff
+
+// struct for halmac//
+struct fwcmd_hdr {
+	u32 hdr0;
+	u32 hdr1;
+};
+/*----------------------------------- H2CREG ---------------------------------------*/
+struct fwcmd_h2creg_hdr {
+	u32 dword0;
+};
+
+struct fwcmd_h2creg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_h2creg_lb {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_cnsl_cmd {
+	u32 dword0;
+};
+
+struct fwcmd_fwerr {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_hidden_get {
+	u32 dword0;
+};
+
+struct fwcmd_getpkt_inform {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_sch_tx_en {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_wow_trx_stop {
+	u32 dword0;
+};
+
+struct fwcmd_aoac_rpt_1 {
+	u32 dword0;
+};
+
+struct fwcmd_aoac_rpt_2 {
+	u32 dword0;
+};
+
+struct fwcmd_aoac_rpt_3_req {
+	u32 dword0;
+};
+
+struct fwcmd_wow_cpuio_rx_ctrl {
+	u32 dword0;
+};
+
+struct fwcmd_set_gpio {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_fwdbg_addr_cfg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+/*----------------------------------- C2HREG ---------------------------------------*/
+struct fwcmd_c2hreg_hdr {
+	u32 dword0;
+};
+
+struct fwcmd_c2hreg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_c2hreg_lb {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_err_rpt {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_err_msg {
+	u32 dword0;
+};
+
+struct fwcmd_efuse_hidden {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_tx_pause_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_aoac_rpt_3 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_io_ofld_result {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_wow_cpuio_rx_ack {
+	u32 dword0;
+};
+
+struct fwcmd_fwdbg_addr_sts {
+	u32 dword0;
+};
+
+struct fwcmd_efuse_hidden_part1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+/*----------------------------------- H2CPKT - CAT(TEST) ---------------------------*/
+struct fwcmd_h2c_lb {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_log_test {
+	u32 dword0;
+};
+
+struct fwcmd_plat_spic_test {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct fwcmd_plat_fl_write {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_plat_fl_erase {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_pl_flash_read {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_long_run {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_lps_test {
+	u32 dword0;
+};
+
+struct fwcmd_lps_onoff_test {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_cpumgenq_test {
+	u32 dword0;
+};
+
+struct fwcmd_mpio_test {
+	u32 dword0;
+};
+
+struct fwcmd_mpiov2_test {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_anytestbusycall_test {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+};
+
+struct fwcmd_anyrw_test {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+};
+
+struct fwcmd_fw_status {
+	u32 dword0;
+};
+
+struct fwcmd_trigger_cpu_exception {
+	u32 dword0;
+};
+
+struct fwcmd_fw_profiling {
+	u32 dword0;
+};
+
+struct fwcmd_wow_auto_test {
+	u32 dword0;
+};
+
+/*----------------------------------- H2CPKT - CAT(MAC) ----------------------------*/
+struct fwcmd_log_cfg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_general_pkt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_c2h_rpt_cfg {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_fw_dbgreg_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_fw_redl_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_fw_self_dx {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_fw_dbg_addr_cfg {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_keep_alive {
+	u32 dword0;
+};
+
+struct fwcmd_disconnect_detect {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_wow_global {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_gtk_ofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_arp_ofld {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_ndp_ofld {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_realwow {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_nlo {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_wakeup_ctrl {
+	u32 dword0;
+};
+
+struct fwcmd_negative_pattern {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_dev2hst_gpio {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_hst2dev_ctrl {
+	u32 dword0;
+};
+
+struct fwcmd_wow_cam_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct fwcmd_aoac_report_req {
+	u32 dword0;
+};
+
+struct fwcmd_wow_stop_fw_trx {
+	u32 dword0;
+};
+
+struct fwcmd_wow_csa {
+	u32 dword0;
+};
+
+struct fwcmd_magic_waker_filter {
+	u32 dword0;
+};
+
+struct fwcmd_tcp_keepalive {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_wow_payload_cam_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+};
+
+struct fwcmd_lps_parm {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_p2p_act {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_p2p_macid_ctrl {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_ips_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_ps_advance_parm {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_periodic_wake {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_bb_rf_pwr_st {
+	u32 dword0;
+};
+
+struct fwcmd_ps_power_level {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_lps_option_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_doze_time_stats {
+	u32 dword0;
+};
+
+struct fwcmd_tbtt_tuning {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_bkp_cam_cfg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_smps_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_rtd3_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_fwhdr_dl {
+	u32 dword0;
+};
+
+struct fwcmd_fwhdr_redl {
+	u32 dword0;
+};
+
+struct fwcmd_twt_announce_upd {
+	u32 dword0;
+};
+
+struct fwcmd_twtinfo_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_twt_stansp_upd {
+	u32 dword0;
+};
+
+struct fwcmd_tblud {
+	u32 dword0;
+};
+
+struct fwcmd_dctlinfo_ud {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+};
+
+struct fwcmd_cctlinfo_ud {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+};
+
+struct fwcmd_shcut_update {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+};
+
+struct fwcmd_bcn_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_ss_ulsta_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_f2pdbg_set {
+	u32 dword0;
+};
+
+struct fwcmd_wlaninfo_get {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_dctlinfo_ud_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+};
+
+struct fwcmd_cctlinfo_ud_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+};
+
+struct fwcmd_fw_sts_para {
+	u32 dword0;
+};
+
+struct fwcmd_dctlinfo_ud_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+};
+
+struct fwcmd_bcn_upd_be {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+};
+
+struct fwcmd_bcn_ignore_edcca {
+	u32 dword0;
+};
+
+struct fwcmd_bcn_upd_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_dctlinfo_ud_v3 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+};
+
+struct fwcmd_cctlinfo_ud_g7 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+};
+
+struct fwcmd_swtx_stacfg {
+	u32 dword0;
+};
+
+struct fwcmd_txmode_switch {
+	u32 dword0;
+};
+
+struct fwcmd_bcn_dynamic_mech {
+	u32 dword0;
+};
+
+struct fwcmd_addrcam_info {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+};
+
+struct fwcmd_addrcam_info_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+};
+
+struct fwcmd_joininfo {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_joininfo_v1 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_dl_grp_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_ul_grp_upd {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mu_sta_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_fwrole_maintain {
+	u32 dword0;
+};
+
+struct fwcmd_notify_dbcc {
+	u32 dword0;
+};
+
+struct fwcmd_dl_grp_upd_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+};
+
+struct fwcmd_fw_flow_ctrl {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+};
+
+struct fwcmd_txrpt_forward {
+	u32 dword0;
+};
+
+struct fwcmd_mru_grp_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+};
+
+struct fwcmd_dump_efuse {
+	u32 dword0;
+};
+
+struct fwcmd_packet_ofld {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_read_ofld {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_write_ofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_conf_ofld {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_sys_init {
+	u32 dword0;
+};
+
+struct fwcmd_trx_init {
+	u32 dword0;
+};
+
+struct fwcmd_intf_init {
+	u32 dword0;
+};
+
+struct fwcmd_macid_pause {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_rx_fwd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_en_mac_hdr_conv {
+	u32 dword0;
+};
+
+struct fwcmd_set_hwseq_reg {
+	u32 dword0;
+};
+
+struct fwcmd_hwamsdu_reg {
+	u32 dword0;
+};
+
+struct fwcmd_amsdu_cut_reg {
+	u32 dword0;
+};
+
+struct fwcmd_tcpip_chksum_offload_reg {
+	u32 dword0;
+};
+
+struct fwcmd_usr_edca {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_tsf32_togl {
+	u32 dword0;
+};
+
+struct fwcmd_cmd_ofld_reg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_usr_tx_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_cmd_ofld_pkt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_ofld_cfg {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_h2c_agg {
+	u32 dword0;
+};
+
+struct fwcmd_add_scanofld_ch {
+	u32 dword0;
+};
+
+struct fwcmd_scanofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_tx_duty {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_disable_rf {
+	u32 dword0;
+};
+
+struct fwcmd_tx_addba_pkt {
+	u32 dword0;
+};
+
+struct fwcmd_pkt_drop {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+};
+
+struct fwcmd_ch_switch {
+	u32 dword0;
+};
+
+struct fwcmd_bcnerlyrpt {
+	u32 dword0;
+};
+
+struct fwcmd_cfg_bcnfltr {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_ofld_rssi {
+	u32 dword0;
+};
+
+struct fwcmd_ofld_tp {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_host_efuse_rec {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_scanofld_drv_ctrl {
+	u32 dword0;
+};
+
+struct fwcmd_wifi_sensing_csi {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_cmd_ofld_v1_reg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_cmd_ofld_v1_pkt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_sch_tx_en_pkt {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_macid_pause_sleep {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+	u32 dword42;
+	u32 dword43;
+	u32 dword44;
+	u32 dword45;
+	u32 dword46;
+	u32 dword47;
+	u32 dword48;
+	u32 dword49;
+	u32 dword50;
+	u32 dword51;
+	u32 dword52;
+	u32 dword53;
+	u32 dword54;
+	u32 dword55;
+	u32 dword56;
+	u32 dword57;
+	u32 dword58;
+	u32 dword59;
+	u32 dword60;
+	u32 dword61;
+	u32 dword62;
+	u32 dword63;
+};
+
+struct fwcmd_bcn_ofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_tbtt_sh_stat_ofld {
+	u32 dword0;
+};
+
+struct fwcmd_sta_csa {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_scanofld_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+};
+
+struct fwcmd_frame_to_act {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_dump_efuse_v2 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_txrpt_stats_en {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_cpumgpkt_tx {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_bcn_cnt_gpio {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_rx_fwd_be {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_trx_time_stats_en {
+	u32 dword0;
+};
+
+struct fwcmd_apf_ofld_ctrl {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_seccam_info {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+};
+
+struct fwcmd_seccam_info_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+};
+
+struct fwcmd_set_snd_para {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+	u32 dword42;
+	u32 dword43;
+	u32 dword44;
+	u32 dword45;
+	u32 dword46;
+	u32 dword47;
+	u32 dword48;
+	u32 dword49;
+	u32 dword50;
+	u32 dword51;
+	u32 dword52;
+	u32 dword53;
+	u32 dword54;
+	u32 dword55;
+	u32 dword56;
+	u32 dword57;
+	u32 dword58;
+	u32 dword59;
+	u32 dword60;
+	u32 dword61;
+	u32 dword62;
+	u32 dword63;
+	u32 dword64;
+	u32 dword65;
+	u32 dword66;
+	u32 dword67;
+	u32 dword68;
+	u32 dword69;
+	u32 dword70;
+	u32 dword71;
+};
+
+struct fwcmd_get_csi_buf {
+	u32 dword0;
+};
+
+struct fwcmd_set_csi_buf {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_get_snd_sts {
+	u32 dword0;
+};
+
+struct fwcmd_set_snd_sts {
+	u32 dword0;
+};
+
+struct fwcmd_init_snd_mer {
+	u32 dword0;
+};
+
+struct fwcmd_init_snd_mee {
+	u32 dword0;
+};
+
+struct fwcmd_csi_fix_rate {
+	u32 dword0;
+};
+
+struct fwcmd_csi_rrsc {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_set_mu_table {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_set_csi_para_reg {
+	u32 dword0;
+};
+
+struct fwcmd_hw_snd_pr {
+	u32 dword0;
+};
+
+struct fwcmd_set_snd_para_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+	u32 dword42;
+	u32 dword43;
+	u32 dword44;
+	u32 dword45;
+	u32 dword46;
+	u32 dword47;
+	u32 dword48;
+	u32 dword49;
+	u32 dword50;
+	u32 dword51;
+	u32 dword52;
+	u32 dword53;
+	u32 dword54;
+	u32 dword55;
+	u32 dword56;
+	u32 dword57;
+	u32 dword58;
+	u32 dword59;
+	u32 dword60;
+	u32 dword61;
+	u32 dword62;
+	u32 dword63;
+	u32 dword64;
+	u32 dword65;
+	u32 dword66;
+	u32 dword67;
+	u32 dword68;
+	u32 dword69;
+	u32 dword70;
+	u32 dword71;
+	u32 dword72;
+	u32 dword73;
+	u32 dword74;
+	u32 dword75;
+	u32 dword76;
+	u32 dword77;
+	u32 dword78;
+	u32 dword79;
+	u32 dword80;
+	u32 dword81;
+	u32 dword82;
+	u32 dword83;
+	u32 dword84;
+	u32 dword85;
+	u32 dword86;
+	u32 dword87;
+	u32 dword88;
+	u32 dword89;
+	u32 dword90;
+	u32 dword91;
+	u32 dword92;
+	u32 dword93;
+	u32 dword94;
+	u32 dword95;
+	u32 dword96;
+	u32 dword97;
+};
+
+struct fwcmd_set_snd_para_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+	u32 dword42;
+	u32 dword43;
+	u32 dword44;
+	u32 dword45;
+	u32 dword46;
+	u32 dword47;
+	u32 dword48;
+	u32 dword49;
+	u32 dword50;
+	u32 dword51;
+	u32 dword52;
+	u32 dword53;
+	u32 dword54;
+	u32 dword55;
+	u32 dword56;
+	u32 dword57;
+	u32 dword58;
+	u32 dword59;
+	u32 dword60;
+	u32 dword61;
+	u32 dword62;
+	u32 dword63;
+	u32 dword64;
+	u32 dword65;
+	u32 dword66;
+	u32 dword67;
+	u32 dword68;
+	u32 dword69;
+	u32 dword70;
+	u32 dword71;
+	u32 dword72;
+	u32 dword73;
+	u32 dword74;
+	u32 dword75;
+	u32 dword76;
+	u32 dword77;
+	u32 dword78;
+	u32 dword79;
+	u32 dword80;
+	u32 dword81;
+	u32 dword82;
+	u32 dword83;
+	u32 dword84;
+	u32 dword85;
+	u32 dword86;
+	u32 dword87;
+	u32 dword88;
+	u32 dword89;
+	u32 dword90;
+	u32 dword91;
+	u32 dword92;
+	u32 dword93;
+	u32 dword94;
+	u32 dword95;
+	u32 dword96;
+	u32 dword97;
+	u32 dword98;
+	u32 dword99;
+	u32 dword100;
+	u32 dword101;
+	u32 dword102;
+	u32 dword103;
+	u32 dword104;
+	u32 dword105;
+	u32 dword106;
+	u32 dword107;
+	u32 dword108;
+	u32 dword109;
+	u32 dword110;
+	u32 dword111;
+	u32 dword112;
+	u32 dword113;
+	u32 dword114;
+	u32 dword115;
+	u32 dword116;
+	u32 dword117;
+	u32 dword118;
+	u32 dword119;
+	u32 dword120;
+	u32 dword121;
+	u32 dword122;
+	u32 dword123;
+	u32 dword124;
+	u32 dword125;
+	u32 dword126;
+	u32 dword127;
+	u32 dword128;
+	u32 dword129;
+	u32 dword130;
+	u32 dword131;
+	u32 dword132;
+	u32 dword133;
+	u32 dword134;
+	u32 dword135;
+	u32 dword136;
+	u32 dword137;
+	u32 dword138;
+	u32 dword139;
+	u32 dword140;
+	u32 dword141;
+	u32 dword142;
+	u32 dword143;
+	u32 dword144;
+	u32 dword145;
+	u32 dword146;
+	u32 dword147;
+	u32 dword148;
+	u32 dword149;
+	u32 dword150;
+	u32 dword151;
+	u32 dword152;
+	u32 dword153;
+	u32 dword154;
+	u32 dword155;
+	u32 dword156;
+	u32 dword157;
+	u32 dword158;
+	u32 dword159;
+	u32 dword160;
+	u32 dword161;
+	u32 dword162;
+	u32 dword163;
+	u32 dword164;
+	u32 dword165;
+	u32 dword166;
+	u32 dword167;
+	u32 dword168;
+	u32 dword169;
+	u32 dword170;
+	u32 dword171;
+	u32 dword172;
+	u32 dword173;
+};
+
+struct fwcmd_ba_cam {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_ba_cam_v1 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_ba_cam_init {
+	u32 dword0;
+};
+
+struct fwcmd_ie_cam {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_add_mcc {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_start_mcc {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_stop_mcc {
+	u32 dword0;
+};
+
+struct fwcmd_del_mcc_group {
+	u32 dword0;
+};
+
+struct fwcmd_reset_mcc_group {
+	u32 dword0;
+};
+
+struct fwcmd_mcc_req_tsf {
+	u32 dword0;
+};
+
+struct fwcmd_mcc_macid_bitmap {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mcc_sync {
+	u32 dword0;
+};
+
+struct fwcmd_mcc_set_duration {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_mcc_dbg_notify {
+	u32 dword0;
+};
+
+struct fwcmd_scsi_tx {
+	u32 dword0;
+};
+
+struct fwcmd_usb_switch {
+	u32 dword0;
+};
+
+struct fwcmd_plat_flash_write {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_plat_flash_erase {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_plat_flash_read {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_fcs {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct fwcmd_cfg_wps {
+	u32 dword0;
+};
+
+struct fwcmd_port_init {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_port_cfg {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mlotbl_cfg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_mlo_stafunc_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_mlo_tid_to_link_map {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_mlo_dm_cfg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_emlsr_sta_cfg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_mcmlo_relink_drop {
+	u32 dword0;
+};
+
+struct fwcmd_mcmlo_sn_sync {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mlo_relink {
+	u32 dword0;
+};
+
+struct fwcmd_mlo_link_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_mlo_dm_dbg {
+	u32 dword0;
+};
+
+struct fwcmd_proxy {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+};
+
+struct fwcmd_mdns {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+	u32 dword42;
+	u32 dword43;
+	u32 dword44;
+	u32 dword45;
+	u32 dword46;
+	u32 dword47;
+	u32 dword48;
+	u32 dword49;
+	u32 dword50;
+	u32 dword51;
+	u32 dword52;
+	u32 dword53;
+	u32 dword54;
+	u32 dword55;
+	u32 dword56;
+	u32 dword57;
+	u32 dword58;
+	u32 dword59;
+	u32 dword60;
+	u32 dword61;
+	u32 dword62;
+	u32 dword63;
+	u32 dword64;
+	u32 dword65;
+	u32 dword66;
+	u32 dword67;
+	u32 dword68;
+	u32 dword69;
+	u32 dword70;
+	u32 dword71;
+	u32 dword72;
+	u32 dword73;
+	u32 dword74;
+	u32 dword75;
+	u32 dword76;
+	u32 dword77;
+	u32 dword78;
+	u32 dword79;
+	u32 dword80;
+	u32 dword81;
+	u32 dword82;
+	u32 dword83;
+};
+
+struct fwcmd_ptcl_pattern {
+	u32 dword0;
+};
+
+struct fwcmd_snmp {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+};
+
+struct fwcmd_llmnr {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+	u32 dword42;
+	u32 dword43;
+	u32 dword44;
+	u32 dword45;
+	u32 dword46;
+	u32 dword47;
+	u32 dword48;
+	u32 dword49;
+	u32 dword50;
+	u32 dword51;
+	u32 dword52;
+};
+
+struct fwcmd_mdns_ofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_apf_set {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+};
+
+struct fwcmd_apf_get {
+	u32 dword0;
+};
+
+struct fwcmd_act_schedule_req {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct fwcmd_bcn_req {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_nan_func_ctrl {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct fwcmd_nan_de_info {
+	u32 dword0;
+};
+
+struct fwcmd_nan_join_cluster {
+	u32 dword0;
+};
+
+struct fwcmd_pause_faw_tx {
+	u32 dword0;
+};
+
+struct fwcmd_nan_get_cluster_info {
+	u32 dword0;
+};
+
+struct fwcmd_nan_avail_t_bitmap {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_fw_cmdreq {
+	u32 dword0;
+};
+
+struct fwcmd_mrc_req_tsf {
+	u32 dword0;
+};
+
+struct fwcmd_add_mrc {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_start_mrc {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_del_mrc {
+	u32 dword0;
+};
+
+struct fwcmd_mrc_sync {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mrc_upd_duration {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_mrc_upd_bitmap {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mrc_dbg_dump_cfg {
+	u32 dword0;
+};
+
+struct fwcmd_sens_tbl_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_sens_ntb_tx_ofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct fwcmd_sens_invld_smr {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_sub_offload_agg {
+	u32 dword0;
+};
+
+struct fwcmd_sub_offload_agg_dctl_info_byte {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_sub_offload_agg_with_num {
+	u32 dword0;
+};
+
+struct fwcmd_om_adv_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_ser_dbg_case_set {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_nat2p5_upd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_az_en {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_az_proc {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_az_update {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_az_cancel {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_wow_diag_ctrl {
+	u32 dword0;
+};
+
+struct fwcmd_wow_req_rx_pkt {
+	u32 dword0;
+};
+
+struct fwcmd_wow_req_mem {
+	u32 dword0;
+};
+
+struct fwcmd_wow_req_bb_rf_reg {
+	u32 dword0;
+};
+
+struct fwcmd_ap_self_diag {
+	u32 dword0;
+};
+
+struct fwcmd_req_sta_diag_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_addba_ofld_req {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_sap_info {
+	u32 dword0;
+};
+
+/*----------------------------------- H2CPKT - CAT(Table) --------------------------*/
+struct fwcmd_mude_para_tblud {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_mudecision_para {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mu_fixinfo {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_ul_fixinfo_tblud {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+};
+
+struct fwcmd_ul_fixinfo {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+};
+
+struct fwcmd_ul_fixinfo_sta_info {
+	u32 dword0;
+};
+
+struct fwcmd_ul_fixinfo_ul_rua_sta_ent {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_ul_fixinfo_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+};
+
+struct fwcmd_ul_fixinfo_sta_info_v2 {
+	u32 dword0;
+};
+
+struct fwcmd_ul_fixinfo_ul_rua_sta_ent_v2 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_addr_cam {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+};
+
+struct fwcmd_bssid_cam {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_sec_cam {
+	u32 dword0;
+};
+
+struct fwcmd_fixmode_para_tblud {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_fixmode_para {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_write_ofld_req {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_read_ofld_req {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_conf_ofld_req_cmd {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_dctrl {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_cctrl {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_dctrl_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_cctrl_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_dctrl_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+};
+
+struct fwcmd_cctrl_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_cctrl_g7 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+};
+
+struct fwcmd_cctrl_g7_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+};
+
+struct fwcmd_ie_cam_info {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_plat_auto_test {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_h2creg_sch_tx_pause {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_cmd_ofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_cmd_ofld_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_h2cpkt_sch_tx_pause {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_wow_csa_chinfo {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_ba_cam_g7 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_sta_csa_chinfo {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_proxy_snmp_enterprise {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+};
+
+struct fwcmd_scanofld_chinfo_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_scanofld_chinfo_v2 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_scanofld_macc_role {
+	u32 dword0;
+};
+
+struct fwcmd_scanofld_opch {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_scanofld_chrpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+/*----------------------------------- H2CPKT - CAT(OutSrc,Phydm) -------------------*/
+struct fwcmd_example {
+	u32 dword0;
+};
+
+struct fwcmd_macid_cfg {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_rssi_ra_config {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+/*----------------------------------- H2CPKT - CAT(OutSrc, RF) ---------------------*/
+/*----------------------------------- H2CPKT - CAT(OutSrc, BTC) --------------------*/
+struct fwcmd_tdma {
+	u32 dword0;
+};
+
+/*----------------------------------- C2HPKT - CAT(TEST) ---------------------------*/
+struct fwcmd_c2h_lb {
+	u32 dword0;
+};
+
+struct fwcmd_test_phy_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_c2h_log_test {
+	u32 dword0;
+};
+
+struct fwcmd_send_null_test {
+	u32 dword0;
+};
+
+struct fwcmd_c2h_l1_2_test {
+	u32 dword0;
+};
+
+struct fwcmd_spic {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_spic2 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_sendnull_test {
+	u32 dword0;
+};
+
+struct fwcmd_mio_test {
+	u32 dword0;
+};
+
+struct fwcmd_cpuio_test {
+	u32 dword0;
+};
+
+/*----------------------------------- C2HPKT - CAT(MAC) ----------------------------*/
+struct fwcmd_rec_ack {
+	u32 dword0;
+};
+
+struct fwcmd_done_ack {
+	u32 dword0;
+};
+
+struct fwcmd_c2h_log {
+	u32 dword0;
+};
+
+struct fwcmd_bcn_cnt {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_bcn_csazero {
+	u32 dword0;
+};
+
+struct fwcmd_bcn_critical_rst {
+	u32 dword0;
+};
+
+struct fwcmd_bcn_upd_done {
+	u32 dword0;
+};
+
+struct fwcmd_bcn_bc_chg_zero {
+	u32 dword0;
+};
+
+struct fwcmd_fw_self_dx_info {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_efuse_dump {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_read_rsp {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_pkt_ofld_rsp {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_fw_getpkt_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_tsf32_togl_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_usr_tx_rpt_info {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+};
+
+struct fwcmd_cmd_ofld_rsp {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_scanofld_rsp {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_tx_duty_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_ch_switch_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_bcnerlyntfy {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_bcnfltr_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_wifi_sensing_csi_tx_result {
+	u32 dword0;
+};
+
+struct fwcmd_tbtt_sh_stat_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_bcn_ofld_rpt {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_frame_to_act_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+};
+
+struct fwcmd_efuse_dump_v2 {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_txrpt_stats {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_bcn_sync_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_trx_time_stats {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_wait_announce {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_stat_rpt {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_aoac_report {
+	u32 dword0;
+};
+
+struct fwcmd_read_wow_cam {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_apf_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_mcc_rcv_ack {
+	u32 dword0;
+};
+
+struct fwcmd_mcc_req_ack {
+	u32 dword0;
+};
+
+struct fwcmd_mcc_tsf_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_mcc_status_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_rx_dbg {
+	u32 dword0;
+};
+
+struct fwcmd_scsi_rx {
+	u32 dword0;
+};
+
+struct fwcmd_fcs_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_wps_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_ccxrpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct fwcmd_bf_sens_feedback {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_port_init_stat {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_port_cfg_stat {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_ofdma_sts {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_ofdma_sts_v1 {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+};
+
+struct fwcmd_mlo_get_tbl {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_emlsr_trans_done {
+	u32 dword0;
+};
+
+struct fwcmd_emlsr_sta_cfg_done {
+	u32 dword0;
+};
+
+struct fwcmd_mcmlo_relink_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_mcmlo_sn_sync_rpt {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_mlo_link_cfg_stat {
+	u32 dword0;
+};
+
+struct fwcmd_mlo_dm_dbg_dump {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+};
+
+struct fwcmd_mlo_dm_status_dump {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+};
+
+struct fwcmd_act_schedule_req_ack {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_bcn_req_ack {
+	u32 dword0;
+};
+
+struct fwcmd_nan_func_ctrl_ack {
+	u32 dword0;
+};
+
+struct fwcmd_nan_de_info_ack {
+	u32 dword0;
+};
+
+struct fwcmd_nan_join_cluster_ack {
+	u32 dword0;
+};
+
+struct fwcmd_nan_pause_faw_tx_ack {
+	u32 dword0;
+};
+
+struct fwcmd_nan_info_notify_cluster_info {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_nan_info_notify_tsf_info {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_nan_info_notify_cluster_join {
+	u32 dword0;
+};
+
+struct fwcmd_nan_deinit_rpt {
+	u32 dword0;
+};
+
+struct fwcmd_mrc_tsf_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_mrc_status_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_mrc_notify_evt {
+	u32 dword0;
+};
+
+struct fwcmd_doze_time_rpt {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_dctl_info_dword {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+};
+
+struct fwcmd_wow_diag_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_wow_diag_gtk {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_wow_diag_ap_lost {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_wow_diag_ser {
+	u32 dword0;
+};
+
+struct fwcmd_wow_diag_rx_evt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_wow_diag_keepalive {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+};
+
+struct fwcmd_wow_diag_arp {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_wow_diag_ns {
+	u32 dword0;
+};
+
+struct fwcmd_wow_diag_pmf {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+};
+
+struct fwcmd_wow_diag_per_wake {
+	u32 dword0;
+};
+
+struct fwcmd_wow_diag_nlo {
+	u32 dword0;
+};
+
+struct fwcmd_wow_diag_sta_csa {
+	u32 dword0;
+};
+
+struct fwcmd_wow_diag_tcp_keepalive {
+	u32 dword0;
+};
+
+struct fwcmd_wow_diag_proxy_ofld {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+};
+
+struct fwcmd_wow_dump_rx {
+	u32 dword0;
+};
+
+struct fwcmd_wow_dump_mem {
+	u32 dword0;
+};
+
+struct fwcmd_wow_dump_role {
+	u32 dword0;
+};
+
+struct fwcmd_wow_dump_bb_rf_reg {
+	u32 dword0;
+};
+
+struct fwcmd_ftm_az_pktid_content {
+	u32 dword0;
+};
+
+struct fwcmd_ftm_az_para_dump {
+	u32 dword0;
+};
+
+struct fwcmd_ftm_az_dbg_dump {
+	u32 dword0;
+};
+
+struct fwcmd_ftm_az_rpt_hdl {
+	u32 dword0;
+};
+
+struct fwcmd_ftm_mc_pktid_content {
+	u32 dword0;
+};
+
+struct fwcmd_ap_self_diag_rst {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+	u32 dword12;
+	u32 dword13;
+	u32 dword14;
+	u32 dword15;
+	u32 dword16;
+	u32 dword17;
+	u32 dword18;
+	u32 dword19;
+	u32 dword20;
+	u32 dword21;
+	u32 dword22;
+	u32 dword23;
+	u32 dword24;
+	u32 dword25;
+	u32 dword26;
+	u32 dword27;
+	u32 dword28;
+	u32 dword29;
+	u32 dword30;
+	u32 dword31;
+	u32 dword32;
+	u32 dword33;
+	u32 dword34;
+	u32 dword35;
+	u32 dword36;
+	u32 dword37;
+	u32 dword38;
+	u32 dword39;
+	u32 dword40;
+	u32 dword41;
+	u32 dword42;
+	u32 dword43;
+	u32 dword44;
+	u32 dword45;
+	u32 dword46;
+	u32 dword47;
+};
+
+struct fwcmd_sta_diag_rpt {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+};
+
+struct fwcmd_sta_diag_scan {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+};
+
+struct fwcmd_sta_diag_csa {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+};
+
+struct fwcmd_sta_diag_ps {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+};
+
+struct fwcmd_sta_diag_mcc {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_sta_diag_efuse {
+	u32 dword0;
+};
+
+struct fwcmd_sta_diag_mrc {
+	u32 dword0;
+	u32 dword1;
+};
+
+struct fwcmd_pwr_int_notify {
+	u32 dword0;
+};
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, Phydm) ------------------*/
+/*----------------------------------- C2HPKT - CAT(OutSrc, RF) ---------------------*/
+struct fwcmd_rf_report {
+	u32 dword0;
+};
+
+/*----------------------------------- C2HPKT - CAT(OutSrc, BTC) --------------------*/
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/mac_wow_reason.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/mac_wow_reason.h
new file mode 100644
index 000000000000..a245675096ef
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/mac_wow_reason.h
@@ -0,0 +1,180 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2019 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ ******************************************************************************/
+#ifndef _MAC_WOW_REASON_H_
+#define _MAC_WOW_REASON_H_
+
+/*--------------------Define -------------------------------------------*/
+/*--------------------Define MACRO--------------------------------------*/
+/*--------------------Define Enum---------------------------------------*/
+
+enum mac_ax_wow_wake_reason {
+	MAC_AX_WOW_UNKNOWN = 0x00,
+	MAC_AX_WOW_RX_PAIRWISEKEY = 0x01,
+	MAC_AX_WOW_RX_GTK = 0x02,
+	MAC_AX_WOW_RX_FOURWAY_HANDSHAKE = 0x03,
+	MAC_AX_WOW_RX_DISASSOC = 0x04,
+	MAC_AX_WOW_RX_DEAUTH = 0x08,
+	MAC_AX_WOW_RX_ARP_REQUEST = 0x09,
+	MAC_AX_WOW_RX_NS = 0x0A,
+	MAC_AX_WOW_RX_EAPREQ_IDENTIFY = 0x0B,
+	MAC_AX_WOW_FW_DECISION_DISCONNECT = 0x10,
+	MAC_AX_WOW_TX_TCP_SEND_LIMIT = 0x11,
+	MAC_AX_WOW_RX_TCP_FROM_SERVER_TO = 0x12,
+	MAC_AX_WOW_RX_TCP_FIN = 0x13,
+	MAC_AX_WOW_RX_MAGIC_PKT = 0x21,
+	MAC_AX_WOW_RX_UNICAST_PKT = 0x22,
+	MAC_AX_WOW_RX_PATTERN_PKT = 0x23,
+	MAC_AX_WOW_RTD3_SSID_MATCH = 0x24,
+	MAC_AX_WOW_RX_DATA_PKT = 0x25,
+	MAC_AX_WOW_RX_SSDP_MATCH = 0x26,
+	MAC_AX_WOW_RX_WSD_MATCH = 0x27,
+	MAC_AX_WOW_RX_SLP_MATCH = 0x28,
+	MAC_AX_WOW_RX_LLTD_MATCH = 0x29,
+	MAC_AX_WOW_RX_MDNS_MATCH = 0x2A,
+	MAC_AX_WOW_RX_MDNS_QUERY_PKT = 0x2B,
+	MAC_AX_WOW_RX_MDNS_FORWARD_ALL = 0x2C,
+	MAC_AX_WOW_RX_MDNS_PASSTHRU_LIST_MATCH = 0x2D,
+	MAC_AX_WOW_RX_REALWOW_V2_WAKEUP_PKT = 0x30,
+	MAC_AX_WOW_RX_REALWOW_V2_ACK_LOST = 0x31,
+	MAC_AX_WOW_RX_REALWOW_V2_TX_KAPKT = 0x32,
+	MAC_AX_WOW_ENABLE_FAIL_DMA_IDLE = 0x40,
+	MAC_AX_WOW_ENABLE_FAIL_DMA_PAUSE = 0x41,
+	MAC_AX_WOW_RTIME_FAIL_DMA_IDLE = 0x42,
+	MAC_AX_WOW_RTIME_FAIL_DMA_PAUSE = 0x43,
+	MAC_AX_WOW_RX_SNMP_MISMATCHED_PKT = 0x50,
+	MAC_AX_WOW_RX_DESIGNATED_MAC_PKT = 0x51,
+	MAC_AX_WOW_RX_LLMNR_MATCH = 0x52,
+	MAC_AX_WOW_RX_NETBIOS_MATCH = 0x53,
+	MAC_AX_WOW_NLO_SSID_MACH = 0x55,
+	MAC_AX_WOW_AP_OFFLOAD_WAKEUP = 0x66,
+	MAC_AX_WOW_DMAC_ERROR_OCCURRED = 0x70,
+	MAC_AX_WOW_EXCEPTION_OCCURRED = 0x71,
+	MAC_AX_WOW_L0_TO_L1_ERROR_OCCURRED = 0x72,
+	MAC_AX_WOW_ASSERT_OCCURRED = 0x73,
+	MAC_AX_WOW_L2_ERROR_OCCURRED = 0x74,
+	MAC_AX_WOW_WDT_TIMEOUT_WAKE = 0x75,
+	MAC_AX_WOW_L1_TO_L2_ERROR_OCCURED = 0x76,
+	MAC_AX_WOW_RTD3_RX_DATA = 0x77,
+	MAC_AX_WOW_RTD3_PVB_HIT = 0x78,
+	MAC_AX_WOW_RTD3_FWD_PKT_FAIL = 0x79,
+	MAC_AX_WOW_APF_PG1 = 0x80,
+	MAC_AX_WOW_APF_PG2 = 0x81,
+	MAC_AX_WOW_APF_PG3 = 0x82,
+	MAC_AX_WOW_APF_PG4 = 0x83,
+	MAC_AX_WOW_APF_PG5 = 0x84,
+	MAC_AX_WOW_APF_PG6 = 0x85,
+	MAC_AX_WOW_APF_PG7 = 0x86,
+	MAC_AX_WOW_APF_PG8 = 0x87,
+	MAC_AX_WOW_APF_PG9 = 0x88,
+	MAC_AX_WOW_APF_PG10 = 0x89,
+	MAC_AX_WOW_APF_PG11 = 0x8A,
+	MAC_AX_WOW_APF_PG12 = 0x8B,
+	MAC_AX_WOW_APF_PG13 = 0x8C,
+	MAC_AX_WOW_APF_PG14 = 0x8D,
+	MAC_AX_WOW_APF_PG15 = 0x8E,
+	MAC_AX_WOW_APF_PG16 = 0x8F,
+	MAC_AX_WOW_NO_WAKE_RX_PAIRWISEKEY = 0xB0,
+	MAC_AX_WOW_NO_WAKE_RX_GTK = 0xB1,
+	MAC_AX_WOW_NO_WAKE_RX_DISASSOC = 0xB2,
+	MAC_AX_WOW_NO_WAKE_RX_DEAUTH = 0xB3,
+	MAC_AX_WOW_NO_WAKE_RX_EAPREQ_IDENTIFY = 0xB4,
+	MAC_AX_WOW_NO_WAKE_FW_DECISION_DISCONNECT = 0xB5,
+	MAC_AX_WOW_NO_WAKE_RW_SEC_CAM_FAIL = 0xB6,
+	MAC_AX_WOW_RX_ACTION = 0xD0,
+	MAC_AX_WOW_CLK_32K_UNLOCK = 0xFD,
+	MAC_AX_WOW_CLK_32K_LOCK = 0xFE
+};
+
+enum rtw_mac_wow_wake_reason {
+	RTW_MAC_WOW_UNKNOWN = 0x00,
+	RTW_MAC_WOW_RX_PAIRWISEKEY = 0x01,
+	RTW_MAC_WOW_RX_GTK = 0x02,
+	RTW_MAC_WOW_RX_FOURWAY_HANDSHAKE = 0x03,
+	RTW_MAC_WOW_RX_DISASSOC = 0x04,
+	RTW_MAC_WOW_RX_DEAUTH = 0x08,
+	RTW_MAC_WOW_RX_ARP_REQUEST = 0x09,
+	RTW_MAC_WOW_RX_NS = 0x0A,
+	RTW_MAC_WOW_RX_EAPREQ_IDENTIFY = 0x0B,
+	RTW_MAC_WOW_FW_DECISION_DISCONNECT = 0x10,
+	RTW_MAC_WOW_TX_TCP_SEND_LIMIT = 0x11,
+	RTW_MAC_WOW_RX_TCP_FROM_SERVER_TO = 0x12,
+	RTW_MAC_WOW_RX_TCP_FIN = 0x13,
+	RTW_MAC_WOW_RX_MAGIC_PKT = 0x21,
+	RTW_MAC_WOW_RX_UNICAST_PKT = 0x22,
+	RTW_MAC_WOW_RX_PATTERN_PKT = 0x23,
+	RTW_MAC_WOW_RTD3_SSID_MATCH = 0x24,
+	RTW_MAC_WOW_RX_DATA_PKT = 0x25,
+	RTW_MAC_WOW_RX_SSDP_MATCH = 0x26,
+	RTW_MAC_WOW_RX_WSD_MATCH = 0x27,
+	RTW_MAC_WOW_RX_SLP_MATCH = 0x28,
+	RTW_MAC_WOW_RX_LLTD_MATCH = 0x29,
+	RTW_MAC_WOW_RX_MDNS_MATCH = 0x2A,
+	RTW_MAC_WOW_RX_MDNS_QUERY_PKT = 0x2B,
+	RTW_MAC_WOW_RX_MDNS_FORWARD_ALL = 0x2C,
+	RTW_MAC_WOW_RX_MDNS_PASSTHRU_LIST_MATCH = 0x2D,
+	RTW_MAC_WOW_RX_REALWOW_V2_WAKEUP_PKT = 0x30,
+	RTW_MAC_WOW_RX_REALWOW_V2_ACK_LOST = 0x31,
+	RTW_MAC_WOW_RX_REALWOW_V2_TX_KAPKT = 0x32,
+	RTW_MAC_WOW_ENABLE_FAIL_DMA_IDLE = 0x40,
+	RTW_MAC_WOW_ENABLE_FAIL_DMA_PAUSE = 0x41,
+	RTW_MAC_WOW_RTIME_FAIL_DMA_IDLE = 0x42,
+	RTW_MAC_WOW_RTIME_FAIL_DMA_PAUSE = 0x43,
+	RTW_MAC_WOW_RX_SNMP_MISMATCHED_PKT = 0x50,
+	RTW_MAC_WOW_RX_DESIGNATED_MAC_PKT = 0x51,
+	RTW_MAC_WOW_RX_LLMNR_MATCH = 0x52,
+	RTW_MAC_WOW_RX_NETBIOS_MATCH = 0x53,
+	RTW_MAC_WOW_NLO_SSID_MACH = 0x55,
+	RTW_MAC_WOW_AP_OFFLOAD_WAKEUP = 0x66,
+	RTW_MAC_WOW_DMAC_ERROR_OCCURRED = 0x70,
+	RTW_MAC_WOW_EXCEPTION_OCCURRED = 0x71,
+	RTW_MAC_WOW_L0_TO_L1_ERROR_OCCURRED = 0x72,
+	RTW_MAC_WOW_ASSERT_OCCURRED = 0x73,
+	RTW_MAC_WOW_L2_ERROR_OCCURRED = 0x74,
+	RTW_MAC_WOW_WDT_TIMEOUT_WAKE = 0x75,
+	RTW_MAC_WOW_L1_TO_L2_ERROR_OCCURED = 0x76,
+	RTW_MAC_WOW_RTD3_RX_DATA = 0x77,
+	RTW_MAC_WOW_RTD3_PVB_HIT = 0x78,
+	RTW_MAC_WOW_RTD3_FWD_PKT_FAIL = 0x79,
+	RTW_MAC_WOW_APF_PG1 = 0x80,
+	RTW_MAC_WOW_APF_PG2 = 0x81,
+	RTW_MAC_WOW_APF_PG3 = 0x82,
+	RTW_MAC_WOW_APF_PG4 = 0x83,
+	RTW_MAC_WOW_APF_PG5 = 0x84,
+	RTW_MAC_WOW_APF_PG6 = 0x85,
+	RTW_MAC_WOW_APF_PG7 = 0x86,
+	RTW_MAC_WOW_APF_PG8 = 0x87,
+	RTW_MAC_WOW_APF_PG9 = 0x88,
+	RTW_MAC_WOW_APF_PG10 = 0x89,
+	RTW_MAC_WOW_APF_PG11 = 0x8A,
+	RTW_MAC_WOW_APF_PG12 = 0x8B,
+	RTW_MAC_WOW_APF_PG13 = 0x8C,
+	RTW_MAC_WOW_APF_PG14 = 0x8D,
+	RTW_MAC_WOW_APF_PG15 = 0x8E,
+	RTW_MAC_WOW_APF_PG16 = 0x8F,
+	RTW_MAC_WOW_NO_WAKE_RX_PAIRWISEKEY = 0xB0,
+	RTW_MAC_WOW_NO_WAKE_RX_GTK = 0xB1,
+	RTW_MAC_WOW_NO_WAKE_RX_DISASSOC = 0xB2,
+	RTW_MAC_WOW_NO_WAKE_RX_DEAUTH = 0xB3,
+	RTW_MAC_WOW_NO_WAKE_RX_EAPREQ_IDENTIFY = 0xB4,
+	RTW_MAC_WOW_NO_WAKE_FW_DECISION_DISCONNECT = 0xB5,
+	RTW_MAC_WOW_NO_WAKE_RW_SEC_CAM_FAIL = 0xB6,
+	RTW_MAC_WOW_RX_ACTION = 0xD0,
+	RTW_MAC_WOW_CLK_32K_UNLOCK = 0xFD,
+	RTW_MAC_WOW_CLK_32K_LOCK = 0xFE
+};
+
+/*--------------------Define Struct-------------------------------------*/
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/wow_h2c_white_list.h b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/wow_h2c_white_list.h
new file mode 100644
index 000000000000..d2a351764b49
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/inc_hdr/wow_h2c_white_list.h
@@ -0,0 +1,93 @@
+#ifndef _MAC_WOW_H2C_WHITE_LIST_H_
+#define _MAC_WOW_H2C_WHITE_LIST_H_
+
+#define WOW_SUPPRT_H2C_NUM 78
+
+const u32 wow_h2c_list[WOW_SUPPRT_H2C_NUM] = {
+   /* CAT(TEST) */
+   /* CAT(MAC) */
+   0x00000101, //[CLASS 0 - FW_INFO] GENERAL_PKT
+   0x00000401, //[CLASS 0 - FW_INFO] FW_REDL_CFG
+   0x00000501, //[CLASS 0 - FW_INFO] FW_SELF_DX
+   0x00000601, //[CLASS 0 - FW_INFO] FW_DBG_ADDR_CFG
+   0x00000005, //[CLASS 1 - WOW] KEEP_ALIVE
+   0x00000105, //[CLASS 1 - WOW] DISCONNECT_DETECT
+   0x00000205, //[CLASS 1 - WOW] WOW_GLOBAL
+   0x00000305, //[CLASS 1 - WOW] GTK_OFLD
+   0x00000405, //[CLASS 1 - WOW] ARP_OFLD
+   0x00000505, //[CLASS 1 - WOW] NDP_OFLD
+   0x00000605, //[CLASS 1 - WOW] REALWOW
+   0x00000705, //[CLASS 1 - WOW] NLO
+   0x00000805, //[CLASS 1 - WOW] WAKEUP_CTRL
+   0x00000905, //[CLASS 1 - WOW] NEGATIVE_PATTERN
+   0x00000A05, //[CLASS 1 - WOW] DEV2HST_GPIO
+   0x00000B05, //[CLASS 1 - WOW] HST2DEV_CTRL
+   0x00000C05, //[CLASS 1 - WOW] WOW_CAM_UPD
+   0x00000D05, //[CLASS 1 - WOW] AOAC_REPORT_REQ
+   0x00000E05, //[CLASS 1 - WOW] WOW_STOP_FW_TRX
+   0x00000F05, //[CLASS 1 - WOW] WOW_CSA
+   0x00001005, //[CLASS 1 - WOW] MAGIC_WAKER_FILTER
+   0x00001105, //[CLASS 1 - WOW] TCP_KEEPALIVE
+   0x00001205, //[CLASS 1 - WOW] WOW_PAYLOAD_CAM_UPD
+   0x00000009, //[CLASS 2 - PS] LPS_PARM
+   0x00000109, //[CLASS 2 - PS] P2P_ACT
+   0x00000209, //[CLASS 2 - PS] P2P_MACID_CTRL
+   0x00000309, //[CLASS 2 - PS] IPS_CFG
+   0x00000409, //[CLASS 2 - PS] PS_ADVANCE_PARM
+   0x00000509, //[CLASS 2 - PS] PERIODIC_WAKE
+   0x00000609, //[CLASS 2 - PS] BB_RF_PWR_ST
+   0x00000709, //[CLASS 2 - PS] PS_POWER_LEVEL
+   0x00000809, //[CLASS 2 - PS] LPS_OPTION_CFG
+   0x00000909, //[CLASS 2 - PS] DOZE_TIME_STATS
+   0x00000A09, //[CLASS 2 - PS] TBTT_TUNING
+   0x00000B09, //[CLASS 2 - PS] BKP_CAM_CFG
+   0x00000C09, //[CLASS 2 - PS] SMPS_CFG
+   0x0000000C, //[CLASS 3 - FWDL] FWHDR_DL
+   0x0000010C, //[CLASS 3 - FWDL] FWHDR_REDL
+   0x00000215, //[CLASS 5 - Frame Exchange] CCTLINFO_UD
+   0x00000915, //[CLASS 5 - Frame Exchange] DCTLINFO_UD_V1
+   0x00000C15, //[CLASS 5 - Frame Exchange] DCTLINFO_UD_V2
+   0x00001115, //[CLASS 5 - Frame Exchange] CCTLINFO_UD_G7
+   0x00000019, //[CLASS 6 - Address CAM] ADDRCAM_INFO
+   0x00000019, //[CLASS 6 - Address CAM] ADDRCAM_INFO_V1
+   0x00000021, //[CLASS 8 - Media Status Report] JOININFO
+   0x00000421, //[CLASS 8 - Media Status Report] FWROLE_MAINTAIN
+   0x00000621, //[CLASS 8 - Media Status Report] FW_FLOW_CTRL
+   0x00000125, //[CLASS 9 - FW_OFLD] PACKET_OFLD
+   0x00001625, //[CLASS 9 - FW_OFLD] ADD_SCANOFLD_CH
+   0x00001725, //[CLASS 9 - FW_OFLD] SCANOFLD
+   0x00001B25, //[CLASS 9 - FW_OFLD] PKT_DROP
+   0x00002825, //[CLASS 9 - FW_OFLD] macid_pause_sleep
+   0x00002B25, //[CLASS 9 - FW_OFLD] STA_CSA
+   0x00002C25, //[CLASS 9 - FW_OFLD] SCANOFLD_V2
+   0x00003625, //[CLASS 9 - FW_OFLD] APF_OFLD_CTRL
+   0x00000051, //[CLASS 20 - MLO] MLOTBL_CFG
+   0x00000151, //[CLASS 20 - MLO] MLO_STAFUNC_CFG
+   0x00000251, //[CLASS 20 - MLO] MLO_TID_TO_LINK_MAP
+   0x00000451, //[CLASS 20 - MLO] EMLSR_STA_CFG
+   0x00000055, //[CLASS 21 - PROXY] PROXY
+   0x00000155, //[CLASS 21 - PROXY] MDNS
+   0x00000255, //[CLASS 21 - PROXY] PTCL_PATTERN
+   0x00000355, //[CLASS 21 - PROXY] SNMP
+   0x00000455, //[CLASS 21 - PROXY] LLMNR
+   0x00000855, //[CLASS 21 - PROXY] MDNS_OFLD
+   0x00001055, //[CLASS 21 - PROXY] APF_SET
+   0x00001155, //[CLASS 21 - PROXY] APF_GET
+   0x0000007D, //[CLASS 31 - WOW_DIAG] WOW_DIAG_CTRL
+   0x00000081, //[CLASS 32 - WOW_TRI_EVT] WOW_REQ_RX_PKT
+   0x00000181, //[CLASS 32 - WOW_TRI_EVT] WOW_REQ_MEM
+   0x00000281, //[CLASS 32 - WOW_TRI_EVT] WOW_REQ_ROLE_INFO
+   0x00000381, //[CLASS 32 - WOW_TRI_EVT] WOW_REQ_BB_RF_REG
+   0x00000089, //[CLASS 34 - STA_DIAG] REQ_STA_DIAG_RPT
+   0x0000008D, //[CLASS 35 - ADDBA_OFLD] ADDBA_OFLD_REQ
+   /* CAT(Table) */
+   /* CAT(OutSrc,Phydm) */
+   0x00000006, //[CLASS 1 (RA)] MACID_CFG
+   0x00000106, //[CLASS 1 (RA)] RSSI_RA_CONFIG
+   /* CAT(OutSrc, RF) */
+   0x00000022, //[CLASS 8] RADIO_INIT_0
+   /* CAT(OutSrc, BTC) */
+   0x00000042, //[CLASS 16] TDMA
+};
+#endif
+
diff --git a/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/rtl8852b/hal8852b_fw.c b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/rtl8852b/hal8852b_fw.c
new file mode 100644
index 000000000000..30dee3782041
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtl8852bs/phl/hal_g6/mac/fw_ax/rtl8852b/hal8852b_fw.c
@@ -0,0 +1,641507 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2012 - 2020 Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ ******************************************************************************/
+#include "../../type.h"
+#ifdef PHL_FEATURE_NIC
+
+#if defined(MAC_8852B_SUPPORT)
+#ifdef MAC_FW_8852B_U2
+#ifdef MAC_FW_CATEGORY_NICCE
+u32 array_length_8852b_u2_nicce = 341216;
+u8 array_8852b_u2_nicce[] = {
+0x02, 0x01, 0x52, 0x88, 0x00, 0x1D, 0x69, 0x0A,
+0x1E, 0x8A, 0xDE, 0x38, 0x20, 0x10, 0xA0, 0x00,
+0x08, 0x07, 0x09, 0x33, 0xE9, 0x07, 0x00, 0x00,
+0x00, 0x03, 0x50, 0x00, 0xE4, 0x07, 0x09, 0x00,
+0x00, 0x00, 0x97, 0xB8, 0x08, 0xF1, 0x04, 0x42,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x24, 0xE1, 0xB8, 0x38, 0x37, 0x00, 0x41,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x28, 0x19, 0xE1, 0xB8, 0x00, 0x08, 0x00, 0x09,
+0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x50, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00,
+0x2E, 0x00, 0x01, 0x00, 0x29, 0xAB, 0xFC, 0xAB,
+0xFC, 0x04, 0xAC, 0x04, 0xD3, 0x03, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+0x00, 0x00, 0x00, 0x00, 0xBB, 0x01, 0x01, 0x00,
+0x00, 0x00, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x00,
+0xBC, 0x01, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
+0x02, 0x00, 0x1A, 0xE7, 0x1D, 0x82, 0xD5, 0xD4,
+0x4D, 0x07, 0xED, 0x3E, 0xF7, 0xB6, 0x08, 0x00,
+0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
+0xD8, 0xFF, 0xBD, 0x27, 0x24, 0x00, 0xBF, 0xAF,
+0x20, 0x00, 0xB0, 0xAF, 0x4A, 0xFB, 0x26, 0x76,
+0x01, 0x00, 0x04, 0x24, 0xE1, 0xB8, 0x02, 0x3C,
+0x80, 0x9E, 0x42, 0x8C, 0xE1, 0xB8, 0x04, 0x3C,
+0xE1, 0xB8, 0x06, 0x3C, 0x38, 0x5B, 0x84, 0x24,
+0x20, 0x77, 0xC6, 0x24, 0x23, 0x30, 0xC4, 0x00,
+0x09, 0xF8, 0x40, 0x00, 0x25, 0x28, 0x00, 0x00,
+0xE1, 0xB8, 0x02, 0x3C, 0x00, 0x90, 0x42, 0x24,
+0x10, 0x00, 0xA2, 0xAF, 0xE1, 0xB8, 0x02, 0x3C,
+0xF4, 0x9E, 0x42, 0x24, 0x14, 0x00, 0xA2, 0xAF,
+0xE1, 0xB8, 0x02, 0x3C, 0x00, 0x24, 0x42, 0x24,
+0x18, 0x00, 0xA2, 0xAF, 0xE1, 0xB8, 0x02, 0x3C,
+0xF4, 0x32, 0x42, 0x24, 0x10, 0x00, 0xA4, 0x27,
+0xE1, 0xB8, 0x10, 0x3C, 0xE2, 0x84, 0x26, 0x76,
+0x1C, 0x00, 0xA2, 0xAF, 0x00, 0x1C, 0x26, 0x76,
+0x00, 0x00, 0x00, 0x00, 0xD4, 0x94, 0x02, 0x8E,
+0x00, 0x00, 0x00, 0x00, 0x09, 0xF8, 0x40, 0x00,
+0x00, 0x0C, 0x04, 0x24, 0x01, 0x00, 0x42, 0x30,
+0x10, 0x00, 0x40, 0x14, 0x02, 0x00, 0x03, 0x24,
+0xD4, 0x94, 0x02, 0x8E, 0x00, 0x00, 0x00, 0x00,
+0x09, 0xF8, 0x40, 0x00, 0x40, 0x00, 0x04, 0x24,
+0x02, 0x00, 0x03, 0x3C, 0x24, 0x10, 0x43, 0x00,
+0x08, 0x00, 0x40, 0x14, 0x02, 0x00, 0x03, 0x24,
+0xE1, 0xB8, 0x02, 0x3C, 0xC8, 0x94, 0x42, 0x8C,
+0x72, 0xB8, 0x05, 0x3C, 0x00, 0xF8, 0xA5, 0x34,
+0x09, 0xF8, 0x40, 0x00, 0x04, 0x0C, 0x04, 0x24,
+0x02, 0x00, 0x03, 0x24, 0xE1, 0xB8, 0x02, 0x3C,
+0x7E, 0xC3, 0x43, 0xA0, 0xE1, 0xB8, 0x02, 0x3C,
+0xC4, 0x94, 0x42, 0x8C, 0x09, 0x00, 0x05, 0x24,
+0x09, 0xF8, 0x40, 0x00, 0xF2, 0x83, 0x04, 0x34,
+0x24, 0x00, 0xBF, 0x8F, 0x20, 0x00, 0xB0, 0x8F,
+0x08, 0x00, 0xE0, 0x03, 0x28, 0x00, 0xBD, 0x27,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xD2, 0xF4, 0x44, 0x98, 0xF0, 0xF3, 0x12, 0x6C,
+0x40, 0xEA, 0x10, 0x6D, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF2, 0x60, 0x9A,
+0x0A, 0xD0, 0x02, 0x67, 0x03, 0x23, 0xD1, 0x18,
+0x19, 0x48, 0x00, 0x65, 0x4B, 0xF2, 0x40, 0x98,
+0x03, 0x22, 0xB1, 0x18, 0x25, 0xED, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0x27, 0x67, 0x15, 0xF0, 0x08, 0x49, 0x0B, 0xD7,
+0x49, 0xA1, 0x08, 0xA1, 0x8A, 0xA1, 0x40, 0x32,
+0x0D, 0xEA, 0x0B, 0xA1, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6E, 0x80, 0x34,
+0x60, 0x33, 0xC0, 0x36, 0x8D, 0xEA, 0x60, 0x33,
+0xC0, 0x36, 0x00, 0xF6, 0x00, 0x30, 0x4D, 0xE8,
+0x69, 0xF5, 0xB4, 0x9E, 0x72, 0xF4, 0x54, 0x9B,
+0x02, 0x6C, 0x09, 0xD3, 0x40, 0xEA, 0x08, 0xD6,
+0x42, 0x34, 0x50, 0xC0, 0x91, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x92, 0xC0, 0x53, 0xC0,
+0x89, 0xA1, 0x48, 0xA1, 0xAA, 0xA1, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA1, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x91, 0xA2, 0xB2, 0xA2, 0x10, 0xA2, 0x53, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0x8D, 0xE8, 0xA0, 0x35,
+0x0B, 0x97, 0x0D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x07, 0x67, 0x00, 0xF3, 0x1D, 0x2A,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x2A, 0xF6,
+0x18, 0x4A, 0x05, 0xD2, 0x7A, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x45, 0xA1, 0xC4, 0xA1, 0x86, 0xA1, 0x40, 0x32,
+0xCD, 0xEA, 0xC7, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEA, 0x00, 0xF6, 0xC0, 0x36, 0x4D, 0xEE,
+0x08, 0x93, 0x09, 0x92, 0x30, 0x6C, 0x69, 0xF5,
+0xB4, 0x9B, 0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x0B, 0xD6, 0x0B, 0x96, 0x42, 0x34, 0x20, 0xF0,
+0x40, 0xC6, 0x20, 0xF0, 0x81, 0xC6, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x20, 0xF0, 0x82, 0xC6,
+0x20, 0xF0, 0x43, 0xC6, 0x85, 0xA1, 0x44, 0xA1,
+0xA6, 0xA1, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA1,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0, 0x81, 0xA2,
+0x20, 0xF0, 0xA2, 0xA2, 0x20, 0xF0, 0xC0, 0xA2,
+0x20, 0xF0, 0x43, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x14, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2A, 0xF6, 0x18, 0x4A,
+0x05, 0xD2, 0x85, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0x85, 0xA1,
+0x44, 0xA1, 0xA6, 0xA1, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA1, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x81, 0xA2, 0x20, 0xF0, 0xA2, 0xA2, 0x20, 0xF0,
+0xC0, 0xA2, 0x20, 0xF0, 0x43, 0xA2, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x00, 0x6C,
+0x80, 0xC2, 0x81, 0xC2, 0x15, 0xF0, 0x49, 0xA0,
+0x15, 0xF0, 0xC8, 0xA0, 0x15, 0xF0, 0x8A, 0xA0,
+0x40, 0x32, 0xCD, 0xEA, 0x15, 0xF0, 0xCB, 0xA0,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6,
+0xC0, 0x36, 0x4D, 0xEE, 0x08, 0x93, 0x09, 0x92,
+0x3C, 0x6C, 0x69, 0xF5, 0xB4, 0x9B, 0x72, 0xF4,
+0x54, 0x9A, 0x40, 0xEA, 0x0B, 0xD6, 0x0B, 0x96,
+0x42, 0x34, 0x15, 0xF0, 0xAA, 0xA0, 0x85, 0xC6,
+0x82, 0x34, 0x44, 0xC6, 0x86, 0xC6, 0x00, 0xF6,
+0x42, 0x32, 0x15, 0xF0, 0x89, 0xA0, 0x47, 0xC6,
+0x15, 0xF0, 0x48, 0xA0, 0x80, 0x34, 0xA0, 0x35,
+0x4D, 0xEC, 0x15, 0xF0, 0x4B, 0xA0, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x85, 0xA2, 0xA6, 0xA2, 0xC4, 0xA2, 0x47, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x14, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x2A, 0xF6, 0x18, 0x4A, 0x05, 0xD2, 0x8E, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x15, 0xF0, 0x49, 0xA0, 0x15, 0xF0,
+0xC8, 0xA0, 0x15, 0xF0, 0x8A, 0xA0, 0x40, 0x32,
+0xCD, 0xEA, 0x15, 0xF0, 0xCB, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6, 0xC0, 0x36,
+0x4D, 0xEE, 0x08, 0x93, 0x09, 0x92, 0x03, 0x6C,
+0x69, 0xF5, 0xB4, 0x9B, 0x72, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x0B, 0xD6, 0x0B, 0x96, 0x42, 0x34,
+0x15, 0xF0, 0xAA, 0xA0, 0x20, 0xF0, 0x81, 0xC6,
+0x82, 0x34, 0x20, 0xF0, 0x40, 0xC6, 0x20, 0xF0,
+0x82, 0xC6, 0x00, 0xF6, 0x42, 0x32, 0x15, 0xF0,
+0x89, 0xA0, 0x20, 0xF0, 0x43, 0xC6, 0x15, 0xF0,
+0x48, 0xA0, 0x80, 0x34, 0xA0, 0x35, 0x4D, 0xEC,
+0x15, 0xF0, 0x4B, 0xA0, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x81, 0xA2, 0x20, 0xF0, 0xA2, 0xA2, 0x20, 0xF0,
+0xC0, 0xA2, 0x20, 0xF0, 0x43, 0xA2, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x14, 0x2A,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x2A, 0xF6,
+0x18, 0x4A, 0x05, 0xD2, 0x95, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x15, 0xF0, 0x89, 0xA0, 0x15, 0xF0, 0x48, 0xA0,
+0x15, 0xF0, 0xAA, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x15, 0xF0, 0x4B, 0xA0, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x20, 0xF0, 0x81, 0xA2, 0x20, 0xF0, 0xA2, 0xA2,
+0x20, 0xF0, 0xC0, 0xA2, 0x20, 0xF0, 0x43, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x00, 0x6C, 0x81, 0xC2, 0x80, 0xC2, 0x82, 0xC2,
+0x45, 0xA1, 0xC4, 0xA1, 0x86, 0xA1, 0x40, 0x32,
+0xCD, 0xEA, 0xC7, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEA, 0x00, 0xF6, 0xC0, 0x36, 0x4D, 0xEE,
+0x08, 0x93, 0x09, 0x92, 0x04, 0x6C, 0x69, 0xF5,
+0xB4, 0x9B, 0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x0B, 0xD6, 0x0B, 0x96, 0x42, 0x34, 0x58, 0xC6,
+0x99, 0xC6, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x9A, 0xC6, 0x5B, 0xC6, 0x85, 0xA1, 0x44, 0xA1,
+0xA6, 0xA1, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA1,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x99, 0xA2, 0xBA, 0xA2,
+0xD8, 0xA2, 0x5B, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x14, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2A, 0xF6, 0x18, 0x4A,
+0x05, 0xD2, 0x9E, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0x85, 0xA1,
+0x44, 0xA1, 0xA6, 0xA1, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA1, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x99, 0xA2,
+0xD8, 0xA2, 0xBA, 0xA2, 0x80, 0x34, 0xCD, 0xEC,
+0x08, 0x93, 0xDB, 0xA2, 0x09, 0x92, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x72, 0xF4, 0x54, 0x9A,
+0x69, 0xF5, 0xB4, 0x9B, 0x00, 0xF6, 0xC0, 0x36,
+0x8D, 0xEE, 0x48, 0x6C, 0x40, 0xEA, 0x0B, 0xD6,
+0x0B, 0x96, 0x42, 0x34, 0x40, 0xC6, 0x81, 0xC6,
+0x00, 0xF6, 0x42, 0x32, 0x82, 0x34, 0x82, 0xC6,
+0x43, 0xC6, 0x45, 0xA1, 0x84, 0xA1, 0xA6, 0xA1,
+0x40, 0x32, 0x8D, 0xEA, 0x87, 0xA1, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEA, 0x00, 0xF6, 0x80, 0x34,
+0x4D, 0xEC, 0xB9, 0xA4, 0x58, 0xA4, 0xDA, 0xA4,
+0xA0, 0x35, 0x4D, 0xED, 0x5B, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x81, 0xA2, 0xA2, 0xA2, 0xC0, 0xA2,
+0x43, 0xA2, 0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE,
+0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x14, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x2A, 0xF6, 0x18, 0x4A, 0x05, 0xD2,
+0xA4, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x45, 0xA1, 0x84, 0xA1,
+0xA6, 0xA1, 0x40, 0x32, 0x8D, 0xEA, 0x87, 0xA1,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEA, 0x00, 0xF6,
+0x80, 0x34, 0x4D, 0xEC, 0xB9, 0xA4, 0x58, 0xA4,
+0xDA, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x5B, 0xA4,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x81, 0xA2, 0xC0, 0xA2,
+0xA2, 0xA2, 0x80, 0x34, 0xCD, 0xEC, 0x08, 0x93,
+0xC3, 0xA2, 0x09, 0x92, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x72, 0xF4, 0x54, 0x9A, 0x69, 0xF5,
+0xB4, 0x9B, 0x00, 0xF6, 0xC0, 0x36, 0x8D, 0xEE,
+0x43, 0x6C, 0x40, 0xEA, 0x0B, 0xD6, 0x0B, 0x96,
+0x42, 0x34, 0x20, 0xF0, 0x89, 0xC6, 0x82, 0x34,
+0x20, 0xF0, 0x8A, 0xC6, 0x00, 0xF6, 0x42, 0x34,
+0x20, 0xF0, 0x48, 0xC6, 0x20, 0xF0, 0x8B, 0xC6,
+0x15, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x2A, 0xF6, 0x18, 0x4A, 0x05, 0xD2, 0xAB, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x0B, 0x96, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x0C, 0xD3, 0x20, 0xF0,
+0x89, 0xA6, 0x20, 0xF0, 0x48, 0xA6, 0x93, 0xF6,
+0xE0, 0x9B, 0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0,
+0x4A, 0xA6, 0x00, 0x6D, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x8B, 0xA6, 0x43, 0x6E,
+0x00, 0xF6, 0x80, 0x34, 0x40, 0xEF, 0x4D, 0xEC,
+0x15, 0xF0, 0x49, 0xA0, 0x15, 0xF0, 0xC8, 0xA0,
+0x15, 0xF0, 0x8A, 0xA0, 0x40, 0x32, 0xCD, 0xEA,
+0x15, 0xF0, 0xCB, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEA, 0x00, 0xF6, 0xC0, 0x36, 0x4D, 0xEE,
+0x08, 0x93, 0x09, 0x92, 0x10, 0x6C, 0x69, 0xF5,
+0xB4, 0x9B, 0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x0B, 0xD6, 0x0B, 0x96, 0x42, 0x34, 0x15, 0xF0,
+0xAA, 0xA0, 0x20, 0xF0, 0x85, 0xC6, 0x82, 0x34,
+0x20, 0xF0, 0x44, 0xC6, 0x20, 0xF0, 0x86, 0xC6,
+0x00, 0xF6, 0x42, 0x32, 0x15, 0xF0, 0x89, 0xA0,
+0x20, 0xF0, 0x47, 0xC6, 0x15, 0xF0, 0x48, 0xA0,
+0x80, 0x34, 0xA0, 0x35, 0x4D, 0xEC, 0x15, 0xF0,
+0x4B, 0xA0, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0, 0x85, 0xA2,
+0x20, 0xF0, 0xA6, 0xA2, 0x20, 0xF0, 0xC4, 0xA2,
+0x20, 0xF0, 0x47, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEE, 0xA0, 0x35, 0x0C, 0x93, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x0B, 0xD3,
+0x14, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x2A, 0xF6, 0x18, 0x4A, 0x05, 0xD2, 0xB2, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x15, 0xF0, 0xA9, 0xA0, 0x15, 0xF0,
+0x88, 0xA0, 0x15, 0xF0, 0x4A, 0xA0, 0xA0, 0x35,
+0x8D, 0xED, 0x15, 0xF0, 0x8B, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6, 0x80, 0x34,
+0xAD, 0xEC, 0x20, 0xF0, 0xA5, 0xA4, 0x20, 0xF0,
+0x44, 0xA4, 0x20, 0xF0, 0xC6, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x47, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x00, 0x6C, 0x80, 0xC2, 0x81, 0xC2,
+0x82, 0xC2, 0x83, 0xC2, 0x0A, 0x92, 0x11, 0x6D,
+0xF0, 0xF3, 0x12, 0x6C, 0xD2, 0xF4, 0x44, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x33, 0xF5, 0x40, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x1D, 0x2A, 0x07, 0xD2,
+0x7D, 0x67, 0x09, 0x6A, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x2A, 0xF6,
+0x18, 0x4A, 0x05, 0xD2, 0xD1, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x11, 0x97, 0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF,
+0x09, 0x63, 0x00, 0x6C, 0x81, 0xC2, 0xF4, 0x14,
+0xB1, 0x18, 0x7D, 0xE9, 0x00, 0x65, 0xD1, 0x18,
+0xD2, 0x31, 0x00, 0x65, 0xD1, 0x18, 0x36, 0x93,
+0x00, 0x65, 0xD1, 0x18, 0x1A, 0x20, 0x00, 0x65,
+0xA9, 0xA1, 0x48, 0xA1, 0x8A, 0xA1, 0xA0, 0x35,
+0x4D, 0xED, 0x4B, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0xA9, 0xA2, 0x8A, 0xA2, 0x68, 0xA2, 0x4B, 0xA2,
+0xA0, 0x35, 0x80, 0x34, 0xAD, 0xEB, 0x80, 0x34,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x3A, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF3, 0xEC, 0x9A, 0x02, 0x67,
+0x59, 0x67, 0x40, 0xF0, 0x89, 0xA2, 0x40, 0xF0,
+0xA8, 0xA2, 0x80, 0x34, 0xAD, 0xEC, 0x40, 0xF0,
+0xAA, 0xA2, 0x0D, 0xD2, 0xA0, 0x35, 0xA0, 0x35,
+0x8D, 0xED, 0x40, 0xF0, 0x8B, 0xA2, 0x00, 0xF6,
+0x80, 0x34, 0x40, 0xEF, 0xAD, 0xEC, 0xA5, 0xA1,
+0x84, 0xA1, 0xC6, 0xA1, 0xA0, 0x35, 0x8D, 0xED,
+0x87, 0xA1, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC, 0xC9, 0xA4,
+0xAA, 0xA4, 0x48, 0xA4, 0x6B, 0xA4, 0xC0, 0x36,
+0xA0, 0x35, 0xCD, 0xEA, 0xA0, 0x35, 0x4D, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0x81, 0xA3,
+0xA0, 0xA3, 0x12, 0xF3, 0xCC, 0x98, 0x80, 0x34,
+0xAD, 0xEC, 0xA2, 0xA3, 0x0C, 0xD3, 0xA0, 0x35,
+0xA0, 0x35, 0x8D, 0xED, 0x83, 0xA3, 0x00, 0xF6,
+0x80, 0x34, 0x40, 0xEE, 0xAD, 0xEC, 0xA5, 0xA1,
+0x84, 0xA1, 0xC6, 0xA1, 0xA0, 0x35, 0x8D, 0xED,
+0x87, 0xA1, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC, 0xAD, 0xA4,
+0xEC, 0xA4, 0xCE, 0xA4, 0xA0, 0x35, 0xED, 0xED,
+0xEF, 0xA4, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0xE0, 0x37, 0xAD, 0xEF, 0x81, 0xA7,
+0x40, 0xA7, 0x12, 0xF3, 0xAC, 0x98, 0x80, 0x34,
+0x4D, 0xEC, 0x42, 0xA7, 0x0A, 0xD7, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x83, 0xA7, 0x00, 0xF6,
+0x80, 0x34, 0x40, 0xED, 0x4D, 0xEC, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x02, 0x67,
+0x12, 0xF3, 0x50, 0x9A, 0x01, 0x6E, 0x80, 0xF2,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x0D, 0x93,
+0x42, 0x34, 0x01, 0x6E, 0x40, 0xF0, 0x48, 0xC3,
+0x00, 0xF6, 0x42, 0x32, 0x40, 0xF0, 0x4B, 0xC3,
+0x12, 0xF3, 0x50, 0x98, 0x40, 0xF0, 0x89, 0xC3,
+0x82, 0x34, 0x40, 0xF0, 0x8A, 0xC3, 0x80, 0xF1,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x0C, 0x93,
+0x42, 0x34, 0x40, 0xC3, 0x81, 0xC3, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x82, 0xC3, 0xD1, 0x18,
+0xD5, 0x62, 0x43, 0xC3, 0x08, 0x93, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x69, 0xF5,
+0xB4, 0x9B, 0x72, 0xF4, 0x58, 0x9A, 0x01, 0x6E,
+0x40, 0xEA, 0x0A, 0x6C, 0x0A, 0x97, 0x62, 0x67,
+0x4C, 0xC7, 0x42, 0x32, 0x4D, 0xC7, 0x42, 0x32,
+0x4E, 0xC7, 0x00, 0xF6, 0x62, 0x32, 0x4F, 0xC7,
+0x0C, 0x2B, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x2A, 0xF6, 0x18, 0x4A, 0x05, 0xD2, 0xFA, 0x6A,
+0x12, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF3, 0x58, 0x9A, 0x00, 0x6E,
+0x20, 0x6D, 0x55, 0x6C, 0x03, 0x67, 0x40, 0xEA,
+0x0A, 0xD7, 0x42, 0x34, 0x0A, 0x97, 0x42, 0xC0,
+0x83, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x84, 0xC0, 0x45, 0xC0, 0xAD, 0xA7, 0x4C, 0xA7,
+0x8E, 0xA7, 0xA0, 0x35, 0x4D, 0xED, 0x4F, 0xA7,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x83, 0xA2, 0xA4, 0xA2,
+0xC2, 0xA2, 0x45, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x0E, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x72, 0xF0, 0x4C, 0x9A,
+0x80, 0x34, 0x2A, 0xF6, 0x18, 0x4C, 0x40, 0xEA,
+0xFE, 0x6D, 0x09, 0x92, 0x08, 0x93, 0xFF, 0x6C,
+0x72, 0xF4, 0x54, 0x9A, 0x69, 0xF5, 0xB4, 0x9B,
+0x40, 0xEA, 0x15, 0x4C, 0x42, 0x34, 0x20, 0xF0,
+0x81, 0xC1, 0x82, 0x34, 0x20, 0xF0, 0x82, 0xC1,
+0x00, 0xF6, 0x42, 0x34, 0x20, 0xF0, 0x40, 0xC1,
+0x20, 0xF0, 0x83, 0xC1, 0x0D, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2A, 0xF6, 0x18, 0x4A,
+0x05, 0xD2, 0xFF, 0x6A, 0x07, 0x4A, 0xB3, 0x16,
+0x0B, 0x93, 0xFF, 0x6E, 0x15, 0x4E, 0x93, 0xF6,
+0xE0, 0x9B, 0x00, 0x6D, 0x40, 0xEF, 0x82, 0x67,
+0x20, 0xF0, 0x81, 0xA1, 0x20, 0xF0, 0x40, 0xA1,
+0x20, 0xF0, 0xA2, 0xA1, 0x80, 0x34, 0x4D, 0xEC,
+0x20, 0xF0, 0x43, 0xA1, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x00, 0x6B, 0x60, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF6, 0x40, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x99, 0x16, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF5, 0x44, 0x9A, 0x40, 0xEA, 0x00, 0x68,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF5, 0x44, 0x9A, 0x90, 0x67, 0x01, 0x6D,
+0x40, 0xEA, 0x01, 0x48, 0x05, 0x70, 0xF4, 0x61,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0xF7, 0xF0,
+0x01, 0x68, 0xD1, 0x18, 0xAF, 0x1C, 0x20, 0x31,
+0x00, 0x30, 0xD2, 0xF4, 0x44, 0x99, 0x00, 0x30,
+0x15, 0xF0, 0x08, 0x48, 0xF0, 0xF3, 0x12, 0x6C,
+0x40, 0xEA, 0x12, 0x6D, 0x49, 0xA0, 0x68, 0xA0,
+0x8A, 0xA0, 0x40, 0x32, 0x6D, 0xEA, 0x6B, 0xA0,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x4D, 0xEB, 0xF7, 0xF0, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x80, 0x34,
+0x80, 0x34, 0x40, 0x32, 0x69, 0xF5, 0xB8, 0x9C,
+0x72, 0xF4, 0x54, 0x9A, 0x04, 0x6C, 0x40, 0xEA,
+0x0A, 0xD3, 0x0A, 0x93, 0x42, 0x34, 0x4C, 0xC3,
+0x8D, 0xC3, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x8E, 0xC3, 0x4F, 0xC3, 0x89, 0xA0, 0x48, 0xA0,
+0x6A, 0xA0, 0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x6D, 0xA2, 0x8E, 0xA2,
+0xAC, 0xA2, 0x4F, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x15, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2A, 0xF6, 0x18, 0x4A,
+0x07, 0xD2, 0xFF, 0x6A, 0x40, 0x4A, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x89, 0xA0, 0x93, 0xF6, 0xE0, 0x9A, 0x48, 0xA0,
+0x6A, 0xA0, 0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x8D, 0xA2, 0x6C, 0xA2,
+0x04, 0x6E, 0x80, 0x34, 0x6D, 0xEC, 0x6E, 0xA2,
+0x00, 0x6D, 0x60, 0x33, 0x60, 0x33, 0x8D, 0xEB,
+0x8F, 0xA2, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xEF,
+0x6D, 0xEC, 0x01, 0x6D, 0xB1, 0x18, 0x72, 0xFE,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF4, 0x48, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x13, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x44, 0x99, 0xF0, 0xF3,
+0x12, 0x6C, 0x40, 0xEA, 0x13, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x13, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x00, 0x6C, 0xD2, 0xF4,
+0x44, 0x99, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x14, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF3, 0x58, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x44, 0x99, 0xF0, 0xF3,
+0x12, 0x6C, 0x40, 0xEA, 0x15, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF3,
+0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x44, 0x99, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x16, 0x6D, 0xD2, 0xF4, 0x44, 0x99, 0xC0, 0xF1,
+0x10, 0x6C, 0x40, 0xEA, 0xA0, 0x6D, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xE9, 0xF2, 0x60, 0x9B,
+0x40, 0x32, 0xF2, 0xF4, 0x44, 0x9A, 0x83, 0x67,
+0x40, 0xEA, 0x0A, 0xD3, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xD2, 0xF4, 0xD8, 0x9C,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x0A, 0x93, 0x69, 0xF1, 0xB4, 0x9C, 0x83, 0x67,
+0x40, 0xEE, 0x4D, 0xED, 0xD2, 0xF4, 0x44, 0x99,
+0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA, 0x24, 0x6D,
+0x89, 0xA0, 0x48, 0xA0, 0x6A, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x8D, 0xA2, 0x6C, 0xA2, 0xAE, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x6F, 0xA2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1, 0x58, 0x9A,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x1A, 0x65,
+0x00, 0xF6, 0x60, 0x33, 0x97, 0xF0, 0x18, 0x6A,
+0x8D, 0xEB, 0x40, 0x32, 0xF7, 0xF0, 0x01, 0x6C,
+0x40, 0x32, 0x80, 0x34, 0x7E, 0xF7, 0x05, 0x4A,
+0x80, 0x34, 0x04, 0xD2, 0x64, 0x6D, 0x4A, 0xF6,
+0x0C, 0x4C, 0x01, 0x6F, 0x01, 0x6E, 0x58, 0x67,
+0x40, 0xEA, 0x0A, 0xD3, 0x0A, 0x93, 0x42, 0x34,
+0x40, 0xC3, 0x81, 0xC3, 0x00, 0xF6, 0x42, 0x32,
+0x82, 0x34, 0x82, 0xC3, 0x43, 0xC3, 0x89, 0xA0,
+0x68, 0xA0, 0x4A, 0xA0, 0x80, 0x34, 0x6D, 0xEC,
+0x6B, 0xA0, 0x40, 0x32, 0x40, 0x32, 0x4D, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x8D, 0xA3,
+0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x61, 0xA2,
+0x82, 0xA2, 0xA0, 0xA2, 0x43, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x4D, 0xEC, 0x0C, 0xD4,
+0x28, 0x24, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF1, 0x54, 0x9A, 0x0A, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x92, 0xF2, 0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xC2, 0x67, 0x0C, 0x94, 0x0A, 0x92, 0x00, 0x6B,
+0x04, 0xD3, 0x00, 0x6F, 0x01, 0x6D, 0x40, 0xEA,
+0x0B, 0xD3, 0x01, 0x72, 0x0E, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0B, 0x93,
+0xF3, 0xF0, 0x50, 0x9A, 0xFF, 0x6F, 0x04, 0xD3,
+0x78, 0x4F, 0x02, 0x6E, 0x02, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF3, 0x54, 0x9A, 0x3E, 0xF4,
+0x04, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4,
+0xC4, 0x99, 0xD2, 0xF4, 0x54, 0x9B, 0x10, 0xF1,
+0x14, 0x6C, 0x0A, 0xD3, 0x40, 0xEA, 0x0B, 0xD6,
+0x0B, 0x96, 0x20, 0x6D, 0x10, 0xF1, 0x14, 0x6C,
+0x40, 0xEE, 0x4C, 0xED, 0x0A, 0x93, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0xC8, 0x9A, 0xD2, 0xF4, 0x54, 0x9B, 0x01, 0xF4,
+0x00, 0x6C, 0x40, 0xEA, 0x0B, 0xD6, 0xFF, 0x6D,
+0x0B, 0x96, 0x02, 0x4D, 0xAB, 0xED, 0x01, 0xF4,
+0x00, 0x6C, 0x40, 0xEE, 0x4C, 0xED, 0xD2, 0xF4,
+0x44, 0x99, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x17, 0x6D, 0xA9, 0xA0, 0x48, 0xA0, 0x8A, 0xA0,
+0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0xA5, 0xA2, 0x86, 0xA2, 0x0A, 0x93,
+0xC4, 0xA2, 0x27, 0xA2, 0xA0, 0x35, 0x80, 0x34,
+0xAD, 0xEE, 0xD2, 0xF4, 0x54, 0x9B, 0x80, 0x34,
+0xCD, 0xEC, 0x00, 0xF6, 0x20, 0x31, 0x8D, 0xE9,
+0x40, 0xEA, 0x80, 0x6C, 0x80, 0xF7, 0x42, 0x32,
+0x01, 0x6B, 0x6C, 0xEA, 0x49, 0xC1, 0x89, 0xA0,
+0x48, 0xA0, 0x6A, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x4B, 0xA0, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2,
+0x86, 0xA2, 0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0xD1, 0x18,
+0xAD, 0x21, 0x89, 0xA2, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF4, 0x50, 0x9A, 0xF7, 0xF0, 0x01, 0x68,
+0x40, 0xEA, 0x00, 0x30, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF4, 0x54, 0x9A,
+0x0C, 0xF1, 0x07, 0x69, 0x40, 0xEA, 0x00, 0x30,
+0x93, 0xF4, 0x4C, 0x98, 0xFF, 0x6D, 0x04, 0x4D,
+0x40, 0xEA, 0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x69, 0xF5, 0xBC, 0x9B,
+0x93, 0xF4, 0x4C, 0x98, 0x40, 0xEA, 0x01, 0x6C,
+0x93, 0xF4, 0x4C, 0x98, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x00, 0x6D, 0x40, 0xEA,
+0x02, 0x6C, 0x93, 0xF4, 0x44, 0x98, 0x2F, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x93, 0xF4,
+0x44, 0x98, 0xD1, 0x67, 0x01, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0x93, 0xF4, 0x44, 0x98, 0xD1, 0x67,
+0x02, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x93, 0xF4,
+0x44, 0x98, 0xD1, 0x67, 0x03, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0x93, 0xF4, 0x44, 0x98, 0xD1, 0x67,
+0x04, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x93, 0xF4,
+0x44, 0x98, 0xD1, 0x67, 0x05, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x89, 0xF5, 0x20, 0x9B, 0x93, 0xF4,
+0x44, 0x98, 0x06, 0x6D, 0xD1, 0x67, 0x40, 0xEA,
+0x00, 0x6C, 0x93, 0xF4, 0x44, 0x98, 0xD1, 0x67,
+0x07, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x93, 0xF4,
+0x44, 0x98, 0xD1, 0x67, 0x08, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x89, 0xF5, 0xC4, 0x9B, 0x93, 0xF4,
+0x44, 0x98, 0x09, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x73, 0xF4, 0x5C, 0x9A, 0x09, 0xF1, 0xBC, 0x9B,
+0x40, 0xEA, 0x00, 0x6C, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0xD2, 0xF4,
+0x44, 0x98, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x18, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xB3, 0xF6, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x44, 0x98, 0xF0, 0xF3,
+0x12, 0x6C, 0x40, 0xEA, 0x19, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x48, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x44, 0x98, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x1A, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x44, 0x98, 0xF0, 0xF3,
+0x12, 0x6C, 0x40, 0xEA, 0x1B, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF7,
+0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x44, 0x98, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x1C, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF7, 0x5C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x44, 0x98, 0xF0, 0xF3,
+0x12, 0x6C, 0x40, 0xEA, 0x1D, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF2, 0xF5,
+0x44, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x44, 0x98, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x1E, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF7, 0x5C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x44, 0x98, 0xF0, 0xF3,
+0x12, 0x6C, 0x40, 0xEA, 0x1F, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB2, 0xF5,
+0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x44, 0x98, 0xF0, 0xF3, 0x12, 0x6C, 0x40, 0xEA,
+0x20, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF4, 0x58, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x44, 0x98, 0xF0, 0xF3,
+0x12, 0x6C, 0x40, 0xEA, 0x21, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF2,
+0x40, 0x9A, 0x03, 0x22, 0xD1, 0x18, 0xD2, 0x41,
+0x00, 0x65, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x89, 0xA3, 0x48, 0xA3, 0xAA, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x69, 0xA2, 0x08, 0xA2, 0x8A, 0xA2, 0x60, 0x33,
+0x0D, 0xEB, 0x0B, 0xA2, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xD2, 0xF2, 0x48, 0x9A, 0x33, 0xF6, 0x9C, 0x9B,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x52, 0xF0, 0xC0, 0xAB, 0xA7, 0x40, 0x09, 0x4D,
+0xFF, 0xF7, 0x1F, 0x69, 0x05, 0xD5, 0xF7, 0xF0,
+0x01, 0x6D, 0x2C, 0xEE, 0xA0, 0x35, 0x04, 0x6B,
+0xCA, 0x36, 0xA0, 0x35, 0x04, 0xD3, 0x2C, 0xEE,
+0x4A, 0xF6, 0x18, 0x4D, 0x40, 0xEA, 0x00, 0x6F,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6D, 0x40, 0x32, 0x80, 0x34,
+0xA0, 0x35, 0xA0, 0x35, 0x40, 0x32, 0x80, 0x34,
+0xD2, 0xF2, 0x48, 0x9A, 0x53, 0xF6, 0x88, 0x9C,
+0x32, 0xF0, 0xDE, 0xAD, 0xA7, 0x40, 0x01, 0x4D,
+0x05, 0xD5, 0x06, 0x6D, 0x04, 0xD5, 0xF7, 0xF0,
+0x01, 0x6D, 0x2C, 0xEE, 0xA0, 0x35, 0xCA, 0x36,
+0xA0, 0x35, 0x2C, 0xEE, 0x6A, 0xF6, 0x04, 0x4D,
+0x40, 0xEA, 0x00, 0x6F, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6D,
+0x40, 0x32, 0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35,
+0x40, 0x32, 0x80, 0x34, 0xD2, 0xF2, 0x48, 0x9A,
+0x33, 0xF6, 0x94, 0x9C, 0x32, 0xF0, 0xDC, 0xAD,
+0xA7, 0x40, 0x11, 0x4D, 0x05, 0xD5, 0xF7, 0xF0,
+0x01, 0x6D, 0x2C, 0xEE, 0xA0, 0x35, 0x04, 0x6B,
+0xCA, 0x36, 0xA0, 0x35, 0x04, 0xD3, 0x2C, 0xEE,
+0x6A, 0xF6, 0x0C, 0x4D, 0x40, 0xEA, 0x00, 0x6F,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6D, 0x40, 0x32, 0x80, 0x34,
+0xA0, 0x35, 0xA0, 0x35, 0x40, 0x32, 0x80, 0x34,
+0xD2, 0xF2, 0x48, 0x9A, 0x33, 0xF6, 0x98, 0x9C,
+0x32, 0xF0, 0xDA, 0xAD, 0xA7, 0x40, 0x0D, 0x4D,
+0x05, 0xD5, 0xF7, 0xF0, 0x01, 0x6D, 0x2C, 0xEE,
+0xA0, 0x35, 0x04, 0x6B, 0xCA, 0x36, 0xA0, 0x35,
+0x04, 0xD3, 0x2C, 0xEE, 0x6A, 0xF6, 0x14, 0x4D,
+0x40, 0xEA, 0x00, 0x6F, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6D,
+0x40, 0x32, 0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35,
+0x40, 0x32, 0x80, 0x34, 0xD2, 0xF2, 0x48, 0x9A,
+0x33, 0xF6, 0x90, 0x9C, 0x32, 0xF0, 0xD8, 0xAD,
+0xA7, 0x40, 0x15, 0x4D, 0x05, 0xD5, 0xF7, 0xF0,
+0x01, 0x6D, 0x2C, 0xEE, 0xA0, 0x35, 0x05, 0x6B,
+0xCA, 0x36, 0xA0, 0x35, 0x04, 0xD3, 0x2C, 0xEE,
+0x00, 0x6F, 0x6A, 0xF6, 0x1C, 0x4D, 0x40, 0xEA,
+0x06, 0xD3, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6D, 0x40, 0x32,
+0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35, 0x40, 0x32,
+0x80, 0x34, 0xD2, 0xF2, 0x48, 0x9A, 0x33, 0xF6,
+0x8C, 0x9C, 0x32, 0xF0, 0xD6, 0xAD, 0xA7, 0x40,
+0x19, 0x4D, 0x05, 0xD5, 0x03, 0x6D, 0x04, 0xD5,
+0xF7, 0xF0, 0x01, 0x6D, 0x2C, 0xEE, 0xA0, 0x35,
+0xCA, 0x36, 0xA0, 0x35, 0x2C, 0xEE, 0x8A, 0xF6,
+0x08, 0x4D, 0x40, 0xEA, 0x00, 0x6F, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x6D, 0x40, 0x32, 0x80, 0x34, 0xA0, 0x35,
+0xA0, 0x35, 0x40, 0x32, 0x80, 0x34, 0xD2, 0xF2,
+0x48, 0x9A, 0x33, 0xF6, 0x88, 0x9C, 0x32, 0xF0,
+0xD4, 0xAD, 0xA7, 0x40, 0x1D, 0x4D, 0x05, 0xD5,
+0x02, 0x6D, 0x04, 0xD5, 0xF7, 0xF0, 0x01, 0x6D,
+0x2C, 0xEE, 0xA0, 0x35, 0xCA, 0x36, 0xA0, 0x35,
+0x2C, 0xEE, 0x8A, 0xF6, 0x14, 0x4D, 0x40, 0xEA,
+0x00, 0x6F, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6D, 0x40, 0x32,
+0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35, 0x40, 0x32,
+0x80, 0x34, 0xD2, 0xF2, 0x48, 0x9A, 0x53, 0xF6,
+0x84, 0x9C, 0x32, 0xF0, 0xD2, 0xAD, 0xA4, 0x40,
+0x06, 0x93, 0x05, 0xD5, 0xF7, 0xF0, 0x01, 0x6D,
+0x2C, 0xEE, 0xA0, 0x35, 0xCA, 0x36, 0xA0, 0x35,
+0x04, 0xD3, 0x2C, 0xEE, 0xAA, 0xF6, 0x00, 0x4D,
+0x40, 0xEA, 0x00, 0x6F, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6D,
+0x40, 0x32, 0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35,
+0x40, 0x32, 0x80, 0x34, 0xD2, 0xF2, 0x48, 0x9A,
+0x53, 0xF6, 0x80, 0x9C, 0x32, 0xF0, 0xD0, 0xAD,
+0xA7, 0x40, 0x05, 0x4D, 0x05, 0xD5, 0xF7, 0xF0,
+0x01, 0x6D, 0x2C, 0xEE, 0xA0, 0x35, 0x04, 0x6B,
+0xCA, 0x36, 0xA0, 0x35, 0x04, 0xD3, 0x2C, 0xEE,
+0xAA, 0xF6, 0x08, 0x4D, 0x40, 0xEA, 0x00, 0x6F,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6D, 0x40, 0x32, 0x80, 0x34,
+0xA0, 0x35, 0xA0, 0x35, 0x40, 0x32, 0x80, 0x34,
+0xD2, 0xF2, 0x48, 0x9A, 0x13, 0xF0, 0x80, 0x9C,
+0x32, 0xF0, 0xCE, 0xAD, 0xA7, 0x40, 0x21, 0x4D,
+0x05, 0xD5, 0x07, 0x6D, 0x04, 0xD5, 0xF7, 0xF0,
+0x01, 0x6D, 0x2C, 0xEE, 0xA0, 0x35, 0xCA, 0x36,
+0xA0, 0x35, 0x2C, 0xEE, 0xAA, 0xF6, 0x10, 0x4D,
+0x40, 0xEA, 0x00, 0x6F, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6D,
+0x40, 0x32, 0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35,
+0x40, 0x32, 0x80, 0x34, 0xD2, 0xF2, 0x48, 0x9A,
+0x53, 0xF6, 0x8C, 0x9C, 0x32, 0xF0, 0xCC, 0xAD,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0x2C, 0xEE,
+0x04, 0x6B, 0xCA, 0x36, 0xA0, 0x35, 0x2C, 0xEE,
+0x05, 0xD0, 0x04, 0xD3, 0xAA, 0xF6, 0x18, 0x4D,
+0x40, 0xEA, 0x00, 0x6F, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x20, 0xF0, 0x89, 0xA0, 0x40, 0x32,
+0xB2, 0xF2, 0xB0, 0x9A, 0x20, 0xF0, 0x48, 0xA0,
+0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0, 0x4A, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x8B, 0xA0, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xED,
+0x4D, 0xEC, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x00, 0xEF, 0x06, 0x63, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF7, 0x44, 0x9A, 0xF7, 0xF0,
+0x01, 0x68, 0x40, 0xEA, 0x00, 0x30, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF6,
+0x44, 0x9A, 0x00, 0x30, 0x15, 0xF0, 0x08, 0x48,
+0x40, 0xEA, 0x00, 0x65, 0xD1, 0x18, 0x45, 0x7A,
+0x00, 0x65, 0x89, 0xA0, 0x48, 0xA0, 0x6A, 0xA0,
+0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x71, 0xA2, 0x92, 0xA2, 0xB0, 0xA2,
+0x53, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xED,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x40, 0xA2, 0x02, 0x72, 0x2A, 0x61,
+0x85, 0xA0, 0x44, 0xA0, 0x66, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x7D, 0xA2, 0x9E, 0xA2, 0xBC, 0xA2, 0x5F, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x05, 0x6B, 0x61, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x05, 0x6B, 0xF3, 0xF0,
+0x50, 0x9A, 0x05, 0xD3, 0xFF, 0x6F, 0x01, 0x6B,
+0x04, 0xD3, 0x79, 0x4F, 0x02, 0x6E, 0x01, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0xB1, 0x18, 0x4F, 0xC4,
+0x00, 0x65, 0x20, 0xF0, 0x81, 0xA0, 0x20, 0xF0,
+0x40, 0xA0, 0x20, 0xF0, 0x62, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x20, 0xF0, 0x43, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x4A, 0x9A, 0x01, 0x22, 0xFF, 0x17,
+0x07, 0x97, 0x06, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xF7, 0x10, 0x6E,
+0xC0, 0xF7, 0x10, 0x6D, 0xD1, 0x18, 0x62, 0x8C,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x01, 0x6B, 0x40, 0x32, 0x47, 0xF2, 0x68, 0xC2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xB2, 0xF2, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x28, 0x9A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x1B, 0xF2, 0x10, 0x68, 0x40, 0xEA, 0x90, 0x67,
+0xFF, 0x6D, 0x02, 0x4D, 0xAB, 0xED, 0x4C, 0xED,
+0x40, 0xE9, 0x90, 0x67, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x6C, 0x9A,
+0x05, 0x6C, 0x07, 0x97, 0x81, 0xF3, 0x40, 0xA3,
+0x06, 0x91, 0x05, 0x90, 0x8B, 0xEC, 0x8C, 0xEA,
+0x81, 0xF3, 0x40, 0xC3, 0x00, 0xEF, 0x04, 0x63,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0x01, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x24, 0x67, 0x10, 0x6C, 0x81, 0xF3, 0x72, 0xA2,
+0x8C, 0xEB, 0x05, 0x23, 0xA1, 0xF3, 0x64, 0x9A,
+0x01, 0x4B, 0xA1, 0xF3, 0x64, 0xDA, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x94, 0xF2,
+0x0C, 0x9A, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x18, 0x20, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1, 0x14, 0x9C,
+0x92, 0xF2, 0x48, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x00, 0x6B, 0x04, 0xD3, 0x06, 0x07, 0xC2, 0x67,
+0x06, 0x6D, 0x40, 0xE8, 0x91, 0x67, 0x0D, 0x97,
+0x0C, 0x91, 0x0B, 0x90, 0xFF, 0x6B, 0x6C, 0xEA,
+0x00, 0xEF, 0x07, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x08, 0xD4, 0x01, 0x6B, 0x6C, 0xEA,
+0x08, 0x94, 0xDC, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1, 0x74, 0x9C,
+0x92, 0xF2, 0x4C, 0x9A, 0x40, 0xEA, 0x08, 0xD3,
+0x08, 0x93, 0x04, 0xD0, 0x00, 0x6F, 0xC2, 0x67,
+0x01, 0x6D, 0x40, 0xEB, 0x91, 0x67, 0xDB, 0x17,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x94, 0xF2, 0x2C, 0x9A, 0xF7, 0xF0, 0x01, 0x6B,
+0x01, 0x68, 0x60, 0x33, 0x06, 0xD0, 0x60, 0x33,
+0x10, 0x21, 0xD2, 0xF1, 0x54, 0x9B, 0x00, 0x6B,
+0x04, 0xD3, 0x06, 0x07, 0x00, 0x6E, 0x08, 0x6D,
+0x40, 0xEA, 0x00, 0x65, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0xFF, 0x6B, 0x6C, 0xEA, 0x00, 0xEF,
+0x07, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A, 0x09, 0xD3,
+0x40, 0xEA, 0x08, 0xD4, 0x0C, 0xEA, 0x08, 0x94,
+0x09, 0x93, 0xE3, 0x22, 0xD2, 0xF1, 0x54, 0x9B,
+0x04, 0xD1, 0x00, 0x6F, 0x00, 0x6E, 0x03, 0x6D,
+0xE3, 0x17, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x01, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0xC5, 0x67, 0x10, 0x6D,
+0x81, 0xF3, 0x72, 0xA2, 0xAC, 0xEB, 0x05, 0x23,
+0xA1, 0xF3, 0x64, 0x9A, 0x01, 0x4B, 0xA1, 0xF3,
+0x64, 0xDA, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x94, 0xF2, 0x0C, 0x9A, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x20, 0x31, 0x0F, 0x20,
+0xD2, 0xF1, 0x54, 0x99, 0x00, 0x6B, 0x04, 0xD3,
+0x06, 0x07, 0x09, 0x6D, 0x40, 0xEA, 0x00, 0x65,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0xFF, 0x6B,
+0x6C, 0xEA, 0x00, 0xEF, 0x07, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x52, 0xF0,
+0x58, 0x9A, 0x09, 0xD6, 0x40, 0xEA, 0x08, 0xD4,
+0x01, 0x6B, 0x6C, 0xEA, 0x08, 0x94, 0x09, 0x96,
+0xE3, 0x22, 0xD2, 0xF1, 0x54, 0x99, 0x04, 0xD0,
+0x00, 0x6F, 0x04, 0x6D, 0xE3, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x94, 0xF2, 0x4C, 0x9A, 0x01, 0x68, 0x04, 0xD0,
+0x0E, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF2, 0x54, 0x9A, 0x04, 0x07,
+0x40, 0xEA, 0x08, 0x6E, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x52, 0xF0,
+0x58, 0x9A, 0x06, 0xD5, 0x40, 0xEA, 0x24, 0x67,
+0x0C, 0xEA, 0x91, 0x67, 0x06, 0x95, 0xE5, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF2, 0xF2, 0x58, 0x9A, 0x00, 0x6F, 0xE4, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x67, 0x44, 0x51, 0x4B,
+0x3A, 0x65, 0x47, 0x44, 0x04, 0xD3, 0x09, 0x4A,
+0x00, 0x6B, 0xFF, 0xF7, 0x94, 0xA2, 0x28, 0x42,
+0xFC, 0x49, 0x17, 0x24, 0xF9, 0x67, 0xFF, 0xF7,
+0x96, 0xA2, 0x61, 0xF3, 0x19, 0xA7, 0x8E, 0xE8,
+0x10, 0x28, 0xFF, 0xF7, 0x1B, 0xA2, 0x09, 0x6F,
+0xEB, 0xEF, 0x0C, 0xEF, 0x01, 0x77, 0x09, 0x61,
+0x12, 0x25, 0xAE, 0xE8, 0x06, 0x28, 0x01, 0x4B,
+0xFF, 0x68, 0x0C, 0xEB, 0x0C, 0x26, 0xCA, 0xEB,
+0x0A, 0x60, 0x04, 0x94, 0x0C, 0x4A, 0x8A, 0xEA,
+0xE0, 0x61, 0xFF, 0x6A, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xA1, 0xA1,
+0x53, 0xF1, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xF1, 0x17, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0xFF, 0xF7, 0x10, 0x6B, 0x20, 0xF3, 0x56, 0xAA,
+0x63, 0xEA, 0x01, 0x6A, 0x18, 0x60, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x4B, 0xF3,
+0x88, 0x9B, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xE9, 0xF0, 0x70, 0x9B, 0xC0, 0xF0,
+0xA4, 0x9C, 0xA3, 0xEB, 0x08, 0x61, 0xC0, 0xF0,
+0xA8, 0x9C, 0xA3, 0xEB, 0x04, 0x61, 0xE0, 0xF0,
+0x44, 0x9C, 0x43, 0xEB, 0x58, 0x67, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xF3, 0x63, 0x19, 0x62,
+0x18, 0xD1, 0x17, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x4B, 0xF3, 0x0C, 0x9B,
+0xF7, 0xF0, 0x01, 0x69, 0x04, 0x04, 0xE0, 0xF3,
+0xA8, 0x40, 0x3C, 0x6E, 0x20, 0x31, 0x14, 0xD3,
+0xD1, 0x18, 0xE7, 0xF4, 0x20, 0x31, 0x00, 0x6A,
+0x20, 0xF3, 0x56, 0xC8, 0x81, 0xF3, 0x48, 0xC8,
+0x93, 0xF6, 0x40, 0x99, 0xB0, 0xF3, 0x8C, 0x40,
+0x08, 0x6E, 0x40, 0xEA, 0x00, 0x6D, 0x93, 0xF6,
+0x40, 0x99, 0x90, 0xF3, 0x88, 0x40, 0x24, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x93, 0xF6, 0x40, 0x99,
+0xD0, 0xF3, 0x84, 0x40, 0x08, 0x6E, 0x40, 0xEA,
+0x00, 0x6D, 0x93, 0xF6, 0x40, 0x99, 0xC0, 0xF3,
+0x84, 0x40, 0x08, 0x6E, 0x40, 0xEA, 0x00, 0x6D,
+0x93, 0xF6, 0x40, 0x99, 0xC0, 0xF3, 0x8C, 0x40,
+0x08, 0x6E, 0x40, 0xEA, 0x00, 0x6D, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x93, 0xF6,
+0x40, 0x99, 0x4B, 0xF3, 0x88, 0x9C, 0xEC, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x93, 0xF6, 0x40, 0x99,
+0x3C, 0x6E, 0x00, 0x6D, 0x40, 0xEA, 0x04, 0x04,
+0x14, 0x93, 0x4B, 0xF3, 0x4C, 0x9B, 0x07, 0x6B,
+0xE0, 0xF3, 0x68, 0xC2, 0x19, 0x97, 0x18, 0x91,
+0x17, 0x90, 0x00, 0xEF, 0x0D, 0x63, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x20, 0xF3, 0x76, 0xAA,
+0x32, 0x5B, 0x2C, 0x61, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x4B, 0xF3, 0x68, 0x9B,
+0xC0, 0xF3, 0xF2, 0xAA, 0xE0, 0xF0, 0xA8, 0x9B,
+0xE0, 0xF0, 0xC4, 0x9B, 0x83, 0xEF, 0x01, 0x4D,
+0xD9, 0xE4, 0xE0, 0xF0, 0xA8, 0xDB, 0xE0, 0xF0,
+0xC4, 0xDB, 0x02, 0x60, 0xC0, 0xF3, 0x92, 0xCA,
+0x06, 0x25, 0xBB, 0xEE, 0x01, 0x2D, 0xE5, 0xE8,
+0x12, 0xEC, 0xC0, 0xF3, 0x90, 0xCA, 0xE0, 0xF1,
+0x14, 0x6C, 0x9B, 0xED, 0x01, 0x2C, 0xE5, 0xE8,
+0x10, 0xEC, 0x08, 0x2C, 0xE0, 0xF0, 0x88, 0xDB,
+0xE0, 0xF0, 0x84, 0xDB, 0xC0, 0xF3, 0x92, 0xCA,
+0xC0, 0xF3, 0x90, 0xCA, 0x20, 0xE8, 0x00, 0x65,
+0xFF, 0x63, 0x01, 0xD1, 0x00, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x08, 0x9A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0xAC, 0x9A, 0x03, 0x6A,
+0xFF, 0x6F, 0x81, 0xF3, 0x72, 0xA5, 0x7F, 0x48,
+0x55, 0x48, 0x4C, 0xEB, 0xEC, 0xEB, 0x01, 0x73,
+0xD8, 0x67, 0xCB, 0xEE, 0xC0, 0xF7, 0xC2, 0x32,
+0xC0, 0xA4, 0xEC, 0xEA, 0x27, 0x76, 0x05, 0x60,
+0x39, 0x76, 0x38, 0x67, 0x27, 0x21, 0x23, 0x76,
+0x21, 0x61, 0xE7, 0xA4, 0x01, 0x6E, 0x3E, 0x65,
+0xD9, 0x67, 0x81, 0xF3, 0x30, 0xA5, 0xFA, 0x37,
+0xCC, 0xEF, 0x41, 0x6E, 0xCB, 0xEE, 0xF8, 0x37,
+0xCC, 0xE9, 0x2D, 0xEF, 0x81, 0xF3, 0xF0, 0xC5,
+0x11, 0x23, 0x63, 0xA4, 0x4C, 0x32, 0x49, 0xE0,
+0x0F, 0x6C, 0x8C, 0xEB, 0x81, 0xA2, 0x6B, 0xEB,
+0xC0, 0xF7, 0x62, 0x33, 0x8D, 0xE3, 0x61, 0xC2,
+0x81, 0xF3, 0x70, 0xA5, 0xD9, 0x67, 0x7A, 0x33,
+0x6C, 0xEE, 0xC2, 0xC2, 0x01, 0x91, 0x00, 0x90,
+0x20, 0xE8, 0x01, 0x63, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0x4C, 0xF2, 0xD8, 0x9E,
+0xEF, 0x9E, 0x01, 0x4F, 0xEF, 0xDE, 0xE2, 0xA4,
+0x02, 0x77, 0x22, 0x61, 0xF0, 0x9E, 0x01, 0x4F,
+0xF0, 0xDE, 0x01, 0x6F, 0x41, 0xF3, 0xFA, 0xC5,
+0xA3, 0xA4, 0xF1, 0x9E, 0xBD, 0xE7, 0xF1, 0xDE,
+0x24, 0xA4, 0xF2, 0x9E, 0x3D, 0xE7, 0xF2, 0xDE,
+0xE5, 0xA4, 0x93, 0x9E, 0xF1, 0xE4, 0x93, 0xDE,
+0xDD, 0x23, 0x4C, 0x32, 0x49, 0xE0, 0x60, 0xA2,
+0x84, 0xA2, 0x01, 0x4B, 0x60, 0xC2, 0x63, 0xA2,
+0x91, 0xE1, 0x84, 0xC2, 0x75, 0xE5, 0x65, 0xA2,
+0xA3, 0xC2, 0x6D, 0xE7, 0x65, 0xC2, 0xCE, 0x17,
+0x41, 0xF3, 0x3A, 0xC5, 0xE1, 0x17, 0x00, 0x65,
+0x01, 0x6A, 0x17, 0x25, 0x0B, 0x75, 0x15, 0x60,
+0x11, 0x75, 0x05, 0x61, 0x04, 0x74, 0x58, 0x67,
+0x01, 0x5A, 0x20, 0xE8, 0x58, 0x67, 0x05, 0x75,
+0x00, 0x6A, 0x0B, 0x61, 0x4E, 0x44, 0xFF, 0x6B,
+0x6C, 0xEA, 0x03, 0x5A, 0x01, 0x6A, 0x05, 0x61,
+0x09, 0x74, 0x58, 0x67, 0x01, 0x5A, 0x58, 0x67,
+0x6C, 0xEA, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0xD1, 0x0A, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x6C, 0x9A, 0x0C, 0xD4, 0x01, 0x6A, 0x41, 0xF0,
+0xC2, 0xAB, 0xE0, 0xF5, 0x9C, 0xA3, 0x02, 0xD2,
+0xC3, 0xEC, 0x06, 0x60, 0x41, 0x46, 0x8B, 0xE2,
+0xFF, 0xF7, 0x1F, 0x6C, 0x8C, 0xEA, 0x02, 0xD2,
+0x02, 0x92, 0x43, 0xEE, 0x00, 0x6A, 0xE0, 0xF0,
+0x0F, 0x61, 0xA0, 0xF7, 0x4F, 0xA3, 0x0F, 0x6D,
+0xFF, 0x6C, 0x4C, 0xED, 0xFF, 0x4D, 0x8C, 0xED,
+0x02, 0x5D, 0x00, 0x6D, 0x06, 0xD5, 0x09, 0x60,
+0xF0, 0x4D, 0xAC, 0xEA, 0x8C, 0xEA, 0x50, 0x6D,
+0xAE, 0xEA, 0x01, 0x5A, 0x58, 0x67, 0x8C, 0xEA,
+0x06, 0xD2, 0x00, 0xF6, 0x45, 0xA3, 0x00, 0x6C,
+0x00, 0x6D, 0x07, 0xD2, 0x00, 0xF6, 0x44, 0xA3,
+0x05, 0xD2, 0x00, 0xF6, 0x41, 0xA3, 0x08, 0xD2,
+0x00, 0x6A, 0x03, 0xD2, 0x3A, 0x65, 0x01, 0xD2,
+0x00, 0xD2, 0x00, 0x97, 0xFF, 0x68, 0x18, 0x65,
+0x01, 0x4F, 0x0C, 0xEF, 0x00, 0xD7, 0x20, 0x68,
+0xEF, 0x46, 0x1A, 0xEF, 0x01, 0x28, 0xE5, 0xE8,
+0x18, 0x67, 0x01, 0x69, 0x10, 0xEF, 0x0C, 0xEF,
+0x04, 0xD7, 0x04, 0x4F, 0xE8, 0x30, 0x01, 0xE3,
+0xA0, 0xF7, 0x0C, 0xA0, 0x0E, 0x30, 0x2C, 0xE8,
+0x04, 0x20, 0x20, 0x68, 0x0D, 0xEA, 0x18, 0x67,
+0x0C, 0xEA, 0xE8, 0x37, 0xFD, 0xE3, 0xA0, 0xF7,
+0xEC, 0xA7, 0x01, 0x68, 0x0C, 0xEF, 0xFF, 0x68,
+0x0C, 0xEF, 0xA0, 0xF0, 0x09, 0x27, 0x01, 0x4D,
+0x0C, 0xED, 0x04, 0x97, 0x01, 0x69, 0xFF, 0x68,
+0x04, 0x4F, 0xE8, 0x37, 0xFD, 0xE3, 0xA0, 0xF7,
+0xEC, 0xA7, 0xE6, 0x37, 0x2C, 0xEF, 0xA0, 0xF0,
+0x03, 0x27, 0xF9, 0x67, 0x01, 0x4F, 0x0C, 0xEF,
+0x3F, 0x65, 0x07, 0x97, 0x00, 0x90, 0x0A, 0xEF,
+0x08, 0x61, 0x07, 0x90, 0xF9, 0x67, 0x03, 0xEF,
+0x04, 0x61, 0x10, 0x6F, 0xED, 0xEA, 0xFF, 0x6F,
+0xEC, 0xEA, 0x04, 0x97, 0x04, 0x4F, 0xE8, 0x30,
+0x01, 0xE3, 0xA0, 0xF7, 0x0C, 0xA0, 0x0A, 0x30,
+0x30, 0x67, 0x01, 0x68, 0x18, 0x65, 0x18, 0x67,
+0x0C, 0xE9, 0x80, 0xF0, 0x0A, 0x21, 0x01, 0x4C,
+0x04, 0xD4, 0x04, 0x04, 0x80, 0xA4, 0x05, 0x90,
+0x00, 0x91, 0x2A, 0xE8, 0x06, 0x61, 0x03, 0xEC,
+0x04, 0x61, 0x08, 0x68, 0x0D, 0xEA, 0xFF, 0x68,
+0x0C, 0xEA, 0xE8, 0x37, 0xFD, 0xE3, 0xA0, 0xF7,
+0xEE, 0xAF, 0xFF, 0xF7, 0x1F, 0x68, 0x09, 0x27,
+0x03, 0x91, 0x3D, 0xE7, 0x01, 0x91, 0x0C, 0xEF,
+0x03, 0xD7, 0x01, 0x49, 0xFF, 0x6F, 0xEC, 0xE9,
+0x01, 0xD1, 0x02, 0x97, 0xFF, 0x4E, 0x0C, 0xEE,
+0xE3, 0xEE, 0x8B, 0x60, 0x06, 0x96, 0x09, 0x26,
+0x05, 0x96, 0x00, 0x97, 0xC3, 0xEF, 0x01, 0x61,
+0x04, 0x24, 0x08, 0x6C, 0x8D, 0xEA, 0xFF, 0x6C,
+0x8C, 0xEA, 0x08, 0x94, 0x83, 0xED, 0x04, 0x61,
+0x01, 0x6C, 0x8D, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA,
+0x00, 0xF6, 0xC9, 0xA3, 0x01, 0x6C, 0x40, 0x68,
+0xC3, 0xED, 0xD8, 0x67, 0x8E, 0xEE, 0x41, 0xF0,
+0x80, 0xA3, 0xFF, 0x6F, 0x0C, 0xEC, 0xEC, 0xEC,
+0x00, 0x68, 0x4B, 0x24, 0x01, 0x94, 0x06, 0x24,
+0x03, 0x94, 0x01, 0x97, 0xFA, 0xEC, 0x01, 0x2F,
+0xE5, 0xE8, 0x12, 0xE8, 0xA0, 0xF7, 0x9A, 0xAB,
+0xFF, 0xF7, 0x1F, 0x6F, 0xEC, 0xE8, 0x16, 0x24,
+0x64, 0x6F, 0xF8, 0xE8, 0x00, 0xF6, 0x22, 0xA3,
+0x12, 0xEF, 0x9A, 0xEF, 0x01, 0x2C, 0xE5, 0xE8,
+0xFF, 0x6F, 0x12, 0xEC, 0xEC, 0xEC, 0x23, 0xEC,
+0x03, 0x60, 0x02, 0x69, 0x2D, 0xEA, 0xEC, 0xEA,
+0x00, 0xF6, 0xEA, 0xA3, 0xE3, 0xEC, 0x02, 0x60,
+0x02, 0x6C, 0x8D, 0xEE, 0x00, 0xF6, 0x83, 0xA3,
+0x83, 0xE8, 0x04, 0x60, 0x04, 0x6C, 0x8D, 0xEA,
+0xFF, 0x6C, 0x8C, 0xEA, 0x0C, 0x94, 0x04, 0x24,
+0x01, 0x74, 0x46, 0x67, 0x01, 0x60, 0x00, 0x6A,
+0xA0, 0xF7, 0xB0, 0xC3, 0xA0, 0xF7, 0x12, 0xCB,
+0x0B, 0x91, 0x0A, 0x90, 0x20, 0xE8, 0x06, 0x63,
+0x5F, 0xF7, 0x17, 0x25, 0x08, 0x97, 0xA3, 0xEF,
+0x5F, 0xF7, 0x13, 0x61, 0xFF, 0x4D, 0x50, 0x17,
+0xF9, 0x67, 0x5F, 0xF7, 0x1E, 0x27, 0xFF, 0x4F,
+0x5A, 0x17, 0x7F, 0xF7, 0x18, 0x24, 0xFF, 0x4C,
+0x73, 0x17, 0x06, 0x94, 0xDF, 0x24, 0x04, 0x6C,
+0x8D, 0xEA, 0xEC, 0xEA, 0xDB, 0x17, 0x00, 0x65,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x0C, 0x9A, 0xFF, 0x6B, 0x30, 0x6D,
+0xA0, 0xF7, 0x2F, 0xA0, 0x11, 0xF1, 0x44, 0x40,
+0xFA, 0x65, 0x0F, 0x6A, 0x4C, 0xE9, 0xA0, 0xF7,
+0x4F, 0xA0, 0x6C, 0xE9, 0x04, 0xD1, 0x52, 0x32,
+0x6C, 0xEA, 0xE0, 0xF5, 0x64, 0x80, 0x05, 0xD2,
+0x00, 0xF6, 0x47, 0xA0, 0x3B, 0x65, 0x20, 0xF3,
+0x76, 0xA8, 0x06, 0xD2, 0x00, 0xF6, 0x46, 0x80,
+0xFF, 0x4B, 0xBA, 0xEB, 0x01, 0x2D, 0xE5, 0xE8,
+0x07, 0xD2, 0xE0, 0xF5, 0x5E, 0xA0, 0xE0, 0xF5,
+0x9D, 0xA0, 0xE0, 0xF5, 0xC6, 0x80, 0x08, 0xD2,
+0xE0, 0xF5, 0xE7, 0x80, 0x10, 0xEB, 0x09, 0x21,
+0x05, 0x71, 0x07, 0x60, 0xA0, 0xF7, 0xAC, 0x80,
+0xA0, 0xF7, 0xCD, 0x80, 0xA0, 0xF7, 0xEE, 0x80,
+0x3D, 0x65, 0xFF, 0x69, 0x2C, 0xEB, 0x64, 0x35,
+0x6D, 0xE5, 0x68, 0x33, 0xBF, 0x67, 0x6D, 0xE5,
+0x41, 0xF0, 0xA2, 0xA8, 0x09, 0xD3, 0xA8, 0xC3,
+0x21, 0xF0, 0x5F, 0xA0, 0xC1, 0xC3, 0xE2, 0xC3,
+0x49, 0xC3, 0x59, 0x67, 0x40, 0xC3, 0xE0, 0xF5,
+0x48, 0xA0, 0x83, 0xED, 0x98, 0x67, 0x45, 0xC3,
+0x21, 0xF0, 0x5C, 0xA0, 0x41, 0x6E, 0xCB, 0xEE,
+0x44, 0xC3, 0x01, 0x6A, 0x4E, 0xEC, 0x08, 0x92,
+0x98, 0x37, 0x41, 0xF0, 0x80, 0xA0, 0x43, 0xED,
+0x01, 0x6B, 0xCC, 0xEC, 0x58, 0x67, 0x6E, 0xEA,
+0xED, 0xEC, 0x7F, 0x6F, 0xEC, 0xEC, 0x5C, 0x32,
+0x8D, 0xEA, 0xFF, 0x4D, 0x20, 0x6C, 0x9A, 0xED,
+0x01, 0x2C, 0xE5, 0xE8, 0x41, 0xF0, 0x40, 0xC0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF6, 0x40, 0x9A, 0x04, 0x6E, 0x00, 0x6D,
+0x10, 0xEC, 0x8C, 0xE9, 0xE4, 0x41, 0xE8, 0x37,
+0xA0, 0xF7, 0x8C, 0x40, 0xF1, 0xE4, 0x40, 0xEA,
+0x08, 0xD7, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0x48, 0x9A, 0x09, 0x93,
+0xC0, 0xF0, 0x54, 0xA2, 0x08, 0x2A, 0x08, 0x97,
+0x01, 0x6C, 0xFD, 0xE0, 0xA0, 0xF7, 0x4C, 0xA7,
+0x8D, 0xEA, 0xA0, 0xF7, 0x4C, 0xC7, 0xE0, 0xF5,
+0x40, 0xA0, 0xE0, 0xF5, 0x81, 0xA0, 0xFF, 0x6D,
+0x43, 0xEC, 0x58, 0x67, 0xE0, 0xF5, 0x83, 0xA0,
+0x01, 0x72, 0xE0, 0xF5, 0x42, 0xA0, 0x90, 0x34,
+0x51, 0xE4, 0x21, 0xF0, 0x5C, 0xA0, 0xAC, 0xEC,
+0x41, 0xF0, 0xA0, 0xA0, 0x4E, 0xEC, 0x06, 0x2C,
+0x01, 0x6C, 0xB6, 0x32, 0x8C, 0xEA, 0x98, 0x67,
+0x8A, 0xEA, 0x09, 0x60, 0x44, 0x41, 0x48, 0x32,
+0x49, 0xE0, 0xA0, 0xF7, 0x8C, 0xA2, 0x02, 0x6E,
+0xCD, 0xEC, 0xA0, 0xF7, 0x8C, 0xC2, 0xE0, 0xF5,
+0x48, 0xA0, 0x06, 0x94, 0x83, 0xEA, 0x09, 0x61,
+0x44, 0x41, 0x48, 0x32, 0x49, 0xE0, 0xA0, 0xF7,
+0x8C, 0xA2, 0x04, 0x6E, 0xCD, 0xEC, 0xA0, 0xF7,
+0x8C, 0xC2, 0xE0, 0xF5, 0x49, 0x80, 0x07, 0x94,
+0x82, 0xEA, 0x09, 0x61, 0x44, 0x41, 0x48, 0x32,
+0x49, 0xE0, 0xA0, 0xF7, 0x8C, 0xA2, 0x08, 0x6E,
+0xCD, 0xEC, 0xA0, 0xF7, 0x8C, 0xC2, 0x40, 0x6A,
+0xAC, 0xEA, 0x0A, 0x22, 0x20, 0x6A, 0x4C, 0xED,
+0x44, 0x41, 0x48, 0x32, 0x49, 0xE0, 0x3E, 0x2D,
+0xE0, 0xF5, 0x90, 0xA8, 0xA0, 0xF7, 0x8E, 0xCA,
+0x00, 0x6C, 0xB1, 0x18, 0xD6, 0xC6, 0x23, 0x67,
+0x21, 0xF0, 0x5E, 0xC0, 0x71, 0x67, 0x06, 0x22,
+0x41, 0xF0, 0x40, 0xA0, 0x08, 0x6C, 0x8D, 0xEA,
+0x41, 0xF0, 0x40, 0xC0, 0x04, 0x92, 0x01, 0x72,
+0x14, 0x61, 0x41, 0xF0, 0x80, 0xA0, 0x87, 0x4A,
+0x8C, 0xEA, 0x80, 0x72, 0x0E, 0x61, 0x01, 0x6C,
+0xB1, 0x18, 0xD6, 0xC6, 0x23, 0x67, 0x21, 0xF0,
+0x5E, 0xC0, 0x71, 0x67, 0x06, 0x22, 0x41, 0xF0,
+0x40, 0xA0, 0x10, 0x6C, 0x8D, 0xEA, 0x41, 0xF0,
+0x40, 0xC0, 0x04, 0x92, 0x50, 0x31, 0x05, 0x92,
+0x25, 0xE2, 0x26, 0xC3, 0x21, 0xF0, 0x5E, 0xA0,
+0x47, 0xC3, 0xA0, 0xF7, 0x50, 0xA0, 0x43, 0xC3,
+0xA0, 0xF7, 0x52, 0xA8, 0x4A, 0xC3, 0x42, 0x32,
+0x4B, 0xC3, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0xE0, 0xF5, 0x94, 0xA8,
+0xC1, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x4B, 0xF3, 0x48, 0x99,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD0, 0xF0, 0x04, 0x42, 0x4B, 0xF3, 0x4C, 0x9B,
+0xFF, 0xF7, 0x1F, 0x6D, 0x04, 0xD3, 0xB0, 0xF1,
+0x84, 0x42, 0x05, 0xD4, 0x20, 0xF3, 0x96, 0xAA,
+0x01, 0x4C, 0xAC, 0xEC, 0x80, 0xF0, 0x13, 0x24,
+0x20, 0xF3, 0x96, 0xCA, 0x41, 0xF0, 0x80, 0xA2,
+0x01, 0x6D, 0xAC, 0xEC, 0x06, 0x24, 0x41, 0xF0,
+0x42, 0xAA, 0x03, 0x22, 0xB1, 0x18, 0x6A, 0xC7,
+0x00, 0x65, 0x00, 0x6B, 0x04, 0x92, 0xFF, 0x6C,
+0x6C, 0xEC, 0x4B, 0xF3, 0x4C, 0x9A, 0x3C, 0x65,
+0x20, 0xF3, 0x96, 0xAA, 0x17, 0x24, 0x4B, 0xF3,
+0xC8, 0x99, 0xE0, 0xF1, 0xA0, 0x43, 0xA4, 0x35,
+0xE6, 0x67, 0xC7, 0x43, 0x29, 0x4E, 0xC8, 0x36,
+0xD9, 0xE7, 0xC1, 0x9E, 0xB5, 0xE2, 0x9B, 0xEE,
+0x01, 0x2C, 0xE5, 0xE8, 0xE0, 0xF3, 0x08, 0x6E,
+0x12, 0xEF, 0xDB, 0xEF, 0x01, 0x2E, 0xE5, 0xE8,
+0x12, 0xEE, 0xC2, 0xCD, 0x68, 0x35, 0x06, 0xD5,
+0x4B, 0xF3, 0xA8, 0x99, 0x68, 0x36, 0xD5, 0xE5,
+0xA0, 0xF0, 0xF4, 0x9D, 0xE0, 0xF3, 0x08, 0x6E,
+0xFD, 0x65, 0xDB, 0xEF, 0x01, 0x2E, 0xE5, 0xE8,
+0x64, 0x36, 0xD9, 0xE2, 0xC0, 0xF3, 0xA8, 0xAE,
+0x12, 0xEF, 0xE3, 0xED, 0x02, 0x60, 0xC0, 0xF3,
+0xE8, 0xCE, 0xFF, 0x4C, 0x18, 0x6E, 0xDA, 0xEC,
+0x01, 0x2E, 0xE5, 0xE8, 0xB9, 0x67, 0xFF, 0x6E,
+0x10, 0xEC, 0x84, 0x34, 0x91, 0xE5, 0xCC, 0xEC,
+0x90, 0xF1, 0xCC, 0x44, 0xC4, 0x36, 0xD9, 0xE2,
+0xBF, 0x67, 0xE0, 0xCE, 0x7C, 0x4D, 0xB4, 0x9D,
+0x89, 0xE2, 0x00, 0xF1, 0x00, 0x5D, 0x02, 0x61,
+0x01, 0x6D, 0xAB, 0xED, 0x80, 0xF1, 0xA4, 0xC2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x07, 0xD3,
+0x40, 0x32, 0x05, 0x93, 0x73, 0xF6, 0x5C, 0x9A,
+0x8C, 0x34, 0x91, 0xE3, 0xB0, 0x67, 0x40, 0xEA,
+0x08, 0x6E, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF6, 0x40, 0x9A, 0x90, 0x67,
+0x08, 0x6E, 0x40, 0xEA, 0x00, 0x6D, 0x4B, 0xF3,
+0x48, 0x99, 0x06, 0x93, 0x01, 0x6C, 0x08, 0x48,
+0x7D, 0xE2, 0x00, 0x6A, 0xA0, 0xF0, 0x54, 0xDF,
+0x49, 0xDF, 0x04, 0x92, 0x07, 0x93, 0x4B, 0xF3,
+0x4C, 0x9A, 0x01, 0x4B, 0x02, 0x73, 0x81, 0xF3,
+0x50, 0xA2, 0x5A, 0x32, 0x8C, 0xEA, 0xFF, 0xF7,
+0x5A, 0xC0, 0x80, 0x61, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x01, 0x6C,
+0x6B, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x0C, 0x9A,
+0x01, 0x6A, 0x01, 0x74, 0x41, 0xF0, 0xE0, 0xA0,
+0x21, 0xF0, 0x7C, 0xA0, 0xB8, 0x67, 0xF6, 0x37,
+0x4C, 0xEF, 0x0F, 0x6A, 0x3A, 0x65, 0x59, 0x67,
+0x04, 0xD7, 0x72, 0x37, 0x4C, 0xEB, 0x45, 0x25,
+0x08, 0x24, 0x02, 0x74, 0x80, 0xF0, 0x10, 0x60,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x87, 0x40, 0x5D, 0x4C, 0x50, 0x67,
+0xA7, 0x42, 0x0D, 0x4D, 0x62, 0x67, 0x0F, 0x6E,
+0x41, 0xF0, 0xCC, 0xC2, 0x07, 0x6F, 0x00, 0x6E,
+0x41, 0xF0, 0xCD, 0xC2, 0x41, 0xF0, 0xEE, 0xC2,
+0x41, 0xF0, 0xCF, 0xC2, 0x00, 0xF6, 0xCF, 0x83,
+0x20, 0x26, 0x04, 0x4A, 0xAA, 0xEA, 0x04, 0x4B,
+0xEE, 0x61, 0xAA, 0xEC, 0x45, 0x67, 0xE8, 0x61,
+0x67, 0x40, 0x0D, 0x4B, 0x50, 0x67, 0x0F, 0x6D,
+0xA1, 0xF0, 0xB0, 0xC0, 0x07, 0x6E, 0x00, 0x6D,
+0xA1, 0xF0, 0xB1, 0xC0, 0xA1, 0xF0, 0xD2, 0xC0,
+0xA1, 0xF0, 0xB3, 0xC0, 0x60, 0xF6, 0xB3, 0x82,
+0x0C, 0x25, 0x04, 0x48, 0x0A, 0xEB, 0x04, 0x4A,
+0xEE, 0x61, 0x6A, 0xEC, 0x03, 0x67, 0xE8, 0x61,
+0xC7, 0x17, 0x45, 0x6E, 0x41, 0xF0, 0xCF, 0xC3,
+0xDC, 0x17, 0x45, 0x6D, 0xA1, 0xF0, 0xB3, 0xC2,
+0xF0, 0x17, 0xA0, 0xF7, 0x4F, 0xA0, 0x99, 0x67,
+0xE8, 0x31, 0x4C, 0xEC, 0x02, 0x74, 0x35, 0x61,
+0xEE, 0x4C, 0x8C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA,
+0x60, 0x72, 0x1A, 0x61, 0x04, 0x96, 0x19, 0x6D,
+0xF7, 0xF0, 0x01, 0x6A, 0xB8, 0xEE, 0x40, 0x32,
+0xF9, 0xE1, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0x06, 0xD3, 0x41, 0xF0, 0x88, 0x40, 0x05, 0xD7,
+0x12, 0xED, 0x75, 0xE5, 0x01, 0x4D, 0xD5, 0xE5,
+0xA8, 0x33, 0x71, 0xE4, 0xA0, 0xF7, 0xAC, 0x40,
+0x40, 0xEA, 0x03, 0x6E, 0x05, 0x97, 0x06, 0x93,
+0x04, 0x94, 0x19, 0x6A, 0xE5, 0xE1, 0x58, 0xEC,
+0x0F, 0x6C, 0x12, 0xEA, 0x45, 0xE1, 0xA0, 0xF7,
+0x4F, 0xA0, 0x6D, 0xE1, 0x68, 0x33, 0x8C, 0xEA,
+0xA0, 0xF7, 0x8F, 0xA0, 0x50, 0x32, 0x6D, 0xE0,
+0x92, 0x34, 0x89, 0xE2, 0x41, 0xF0, 0x4F, 0xC3,
+0x83, 0x17, 0x04, 0x96, 0x19, 0x6A, 0x41, 0xF0,
+0x88, 0x40, 0x58, 0xEE, 0xF9, 0xE1, 0x12, 0xEA,
+0x69, 0xE2, 0x01, 0x4A, 0xC9, 0xE2, 0x48, 0x32,
+0x49, 0xE4, 0x99, 0x67, 0x80, 0xC2, 0x07, 0x6C,
+0xA1, 0xC2, 0x82, 0xC2, 0xA3, 0xC2, 0xD8, 0x17,
+0x04, 0x96, 0x59, 0x67, 0x9D, 0x67, 0x58, 0xC4,
+0x19, 0x6C, 0x98, 0xEE, 0xF7, 0xF0, 0x01, 0x6A,
+0xE8, 0x31, 0x40, 0x32, 0xE5, 0xE1, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0x41, 0xF0, 0xA8, 0x40,
+0x05, 0xD3, 0x12, 0xEC, 0x04, 0xD4, 0x71, 0xE4,
+0x01, 0x4C, 0x31, 0xE4, 0x88, 0x34, 0x95, 0xE5,
+0xA0, 0xF7, 0x8C, 0x40, 0x40, 0xEA, 0x03, 0x6E,
+0x04, 0x92, 0x05, 0x93, 0x10, 0x6D, 0x45, 0xE1,
+0x6D, 0xE1, 0x68, 0x33, 0x5D, 0x67, 0x58, 0xA2,
+0x6D, 0xE0, 0x41, 0xF0, 0x6F, 0xA3, 0x3A, 0x65,
+0x59, 0x67, 0x72, 0x34, 0x4C, 0xEC, 0xA0, 0xF7,
+0x4F, 0xA0, 0xAB, 0xED, 0x70, 0x33, 0xAC, 0xEA,
+0x8D, 0xEA, 0x99, 0x67, 0x8C, 0xEA, 0x6D, 0xEA,
+0xA0, 0xF7, 0x4F, 0xC0, 0x39, 0x17, 0x00, 0x65,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x2C, 0x9A, 0x01, 0x6B, 0x15, 0xD5,
+0x41, 0xF0, 0x40, 0xA1, 0x0F, 0x6E, 0x14, 0xD4,
+0x56, 0x35, 0x6C, 0xED, 0x06, 0xD5, 0x5E, 0x32,
+0x21, 0xF0, 0xBC, 0xA1, 0x04, 0xD2, 0x06, 0x92,
+0xB2, 0x33, 0xCC, 0xED, 0x0B, 0xD5, 0xA0, 0xF7,
+0x8C, 0x81, 0xA0, 0xF7, 0x0D, 0x81, 0xF0, 0xF5,
+0xCC, 0x41, 0x68, 0x35, 0x80, 0xF0, 0x0D, 0x2A,
+0x0B, 0x92, 0x75, 0xE5, 0x04, 0x4A, 0xA9, 0xE2,
+0x48, 0x32, 0x49, 0xE6, 0xA0, 0x82, 0x08, 0xD5,
+0xA1, 0x82, 0x09, 0xD5, 0xA2, 0x82, 0x43, 0x82,
+0x07, 0xD5, 0x05, 0xD2, 0x05, 0x92, 0x02, 0x2A,
+0x07, 0x6A, 0x05, 0xD2, 0x14, 0x92, 0xA0, 0xA2,
+0x15, 0x92, 0x01, 0x75, 0x40, 0xA2, 0x80, 0xF0,
+0x02, 0x61, 0x02, 0x72, 0xF8, 0x67, 0x7F, 0x2F,
+0x04, 0x96, 0x0A, 0xD7, 0x7C, 0x26, 0xA0, 0xF7,
+0xAC, 0x41, 0xB0, 0xF7, 0x84, 0x41, 0xD1, 0x18,
+0xE7, 0xF4, 0x08, 0x6E, 0x15, 0x93, 0x0A, 0x97,
+0x03, 0x6A, 0x41, 0xF0, 0xE4, 0xC9, 0x40, 0xC3,
+0x00, 0x6B, 0x09, 0x92, 0x08, 0x94, 0x07, 0x95,
+0x91, 0xE2, 0x02, 0x6A, 0x5A, 0xEC, 0x01, 0x2A,
+0xE5, 0xE8, 0x12, 0xEC, 0x00, 0xF6, 0x80, 0x34,
+0x00, 0xF6, 0x83, 0x34, 0x5A, 0xED, 0x01, 0x2A,
+0xE5, 0xE8, 0x12, 0xE8, 0x00, 0xF6, 0x00, 0x30,
+0x00, 0xF6, 0x03, 0x30, 0x15, 0x95, 0xFF, 0x6E,
+0xA0, 0xA5, 0xFD, 0x4D, 0xCC, 0xED, 0x02, 0x5D,
+0x18, 0x60, 0xA0, 0xF7, 0xAC, 0x81, 0x8E, 0xED,
+0x04, 0x2D, 0xA0, 0xF7, 0xAD, 0x81, 0x0E, 0xED,
+0x10, 0x25, 0x02, 0x72, 0x0E, 0x60, 0x04, 0x2B,
+0xA0, 0xF7, 0x72, 0xA9, 0x41, 0xF0, 0x64, 0xC9,
+0x15, 0x93, 0x03, 0x6D, 0x60, 0xA3, 0xAE, 0xEB,
+0x01, 0x5B, 0x15, 0x95, 0x78, 0x67, 0x03, 0x4B,
+0x60, 0xC5, 0x21, 0xF0, 0x5F, 0xC1, 0x5D, 0x67,
+0x54, 0xA2, 0x00, 0x54, 0xA0, 0xF7, 0x8C, 0xC1,
+0xA0, 0xF7, 0x0D, 0xC1, 0xA0, 0xF7, 0x4E, 0xC1,
+0x08, 0x60, 0x8B, 0xEC, 0x00, 0xF6, 0x80, 0x34,
+0x80, 0x6A, 0x00, 0xF6, 0x83, 0x34, 0x4B, 0xEA,
+0x4D, 0xEC, 0x14, 0x92, 0x40, 0xA2, 0x05, 0x72,
+0x0F, 0x60, 0xFF, 0x69, 0x00, 0x6D, 0xB1, 0x18,
+0x12, 0xD5, 0x2C, 0xEC, 0x90, 0x67, 0x00, 0x6D,
+0xB1, 0x18, 0xD1, 0xD6, 0x2C, 0xEC, 0x05, 0x94,
+0x00, 0x6D, 0xB1, 0x18, 0xC4, 0xD6, 0x2C, 0xEC,
+0x13, 0x97, 0x12, 0x91, 0x11, 0x90, 0x00, 0xEF,
+0x0A, 0x63, 0x0B, 0x92, 0x75, 0xE5, 0x1D, 0x4A,
+0x55, 0xE5, 0xA8, 0x35, 0xB5, 0xE6, 0x40, 0x85,
+0x08, 0xD2, 0x41, 0x85, 0x09, 0xD2, 0x42, 0x85,
+0x07, 0xD2, 0x43, 0x85, 0x72, 0x17, 0xCD, 0x42,
+0x02, 0x5E, 0xD8, 0x67, 0x02, 0x75, 0xB8, 0x67,
+0x0C, 0xD5, 0x03, 0x2E, 0x7C, 0x2D, 0x06, 0x72,
+0x7A, 0x61, 0x41, 0xF0, 0x44, 0xA9, 0x21, 0xF0,
+0xDE, 0xA1, 0x1A, 0x65, 0x01, 0x6A, 0x3A, 0x65,
+0x59, 0x67, 0xCF, 0xEF, 0x4C, 0xEF, 0x0A, 0xD7,
+0x0A, 0x02, 0xE0, 0xA2, 0xB9, 0x67, 0xCC, 0xED,
+0x0F, 0xD7, 0x02, 0x6F, 0xEC, 0xEE, 0x0A, 0xD6,
+0xA0, 0xF7, 0xD2, 0xA9, 0x64, 0x6F, 0x40, 0xA2,
+0xF8, 0xEE, 0x00, 0xF6, 0xCB, 0xA1, 0x0D, 0xD2,
+0x12, 0xEA, 0x0A, 0xD2, 0x58, 0x67, 0xD8, 0xEA,
+0xFF, 0xF7, 0x1F, 0x6E, 0x12, 0xEA, 0xCC, 0xEA,
+0x3A, 0x65, 0x58, 0x67, 0xF8, 0xEA, 0x00, 0xF6,
+0xEA, 0xA1, 0x12, 0xEA, 0x0E, 0xD2, 0xA0, 0xF7,
+0x5A, 0xA9, 0x1A, 0x65, 0x58, 0x67, 0xF8, 0xEA,
+0x0C, 0x92, 0x12, 0xEF, 0xEC, 0xEE, 0x4A, 0x2A,
+0x14, 0x2D, 0x04, 0x92, 0x02, 0x22, 0x0D, 0x92,
+0x28, 0x2A, 0x06, 0x94, 0x68, 0x32, 0x69, 0xE2,
+0x19, 0x6B, 0x78, 0xEC, 0x12, 0xEB, 0x69, 0xE2,
+0x0B, 0x93, 0x69, 0xE2, 0x48, 0x32, 0x49, 0xE1,
+0x41, 0xF0, 0x8C, 0x82, 0x41, 0xF0, 0x0D, 0x82,
+0x0F, 0x10, 0x06, 0x92, 0x16, 0x22, 0x07, 0x93,
+0x02, 0x6A, 0x5A, 0xEB, 0x01, 0x2A, 0xE5, 0xE8,
+0x12, 0xEA, 0x02, 0xEA, 0x0E, 0x60, 0xFF, 0x48,
+0x00, 0xF6, 0x00, 0x30, 0x00, 0xF6, 0x03, 0x30,
+0x00, 0x6B, 0x14, 0x95, 0x02, 0x6A, 0x40, 0xC5,
+0x15, 0x95, 0x06, 0x6A, 0x40, 0xC5, 0x3F, 0x6A,
+0x3D, 0x17, 0x06, 0x92, 0x0F, 0x93, 0x6D, 0xEA,
+0x10, 0x2A, 0x09, 0x92, 0x08, 0x93, 0x6D, 0xE2,
+0x02, 0x6A, 0x5A, 0xEB, 0x01, 0x2A, 0xE5, 0xE8,
+0x12, 0xEA, 0x82, 0xEA, 0x06, 0x60, 0xFF, 0x4C,
+0x00, 0xF6, 0x80, 0x34, 0x00, 0xF6, 0x83, 0x34,
+0xE3, 0x17, 0x14, 0x93, 0x01, 0x6A, 0x40, 0xC3,
+0x15, 0x93, 0x02, 0x6A, 0x40, 0xC3, 0x00, 0x6B,
+0x00, 0x6A, 0x12, 0x10, 0x21, 0xF0, 0x5F, 0xA1,
+0x00, 0x6B, 0x17, 0x72, 0x7E, 0x60, 0x18, 0x5A,
+0x27, 0x60, 0x04, 0x72, 0x5C, 0x60, 0x05, 0x5A,
+0x13, 0x60, 0x02, 0x72, 0x41, 0x60, 0x03, 0x5A,
+0x4D, 0x60, 0x01, 0x72, 0xFF, 0xF6, 0x19, 0x60,
+0x14, 0x95, 0xA0, 0xA5, 0x05, 0x75, 0x1F, 0xF7,
+0x09, 0x60, 0x07, 0x6C, 0x05, 0xD4, 0x00, 0x68,
+0x0F, 0x6C, 0x04, 0x17, 0x3F, 0x6A, 0xE5, 0x17,
+0x15, 0x72, 0x48, 0x60, 0x16, 0x5A, 0x57, 0x60,
+0x14, 0x72, 0xEE, 0x61, 0x09, 0x92, 0x42, 0xEC,
+0x50, 0x60, 0x01, 0x4C, 0x00, 0xF6, 0x80, 0x34,
+0x00, 0xF6, 0x83, 0x34, 0x15, 0x6A, 0xF2, 0x16,
+0x21, 0x72, 0x80, 0xF0, 0x01, 0x60, 0x22, 0x5A,
+0x10, 0x60, 0x1F, 0x72, 0x60, 0x60, 0x20, 0x5A,
+0x70, 0x60, 0x1E, 0x72, 0xD9, 0x61, 0x07, 0x92,
+0x42, 0xE8, 0x69, 0x60, 0x01, 0x48, 0x00, 0xF6,
+0x00, 0x30, 0x00, 0xF6, 0x03, 0x30, 0x1F, 0x6A,
+0xDD, 0x16, 0x3C, 0x72, 0x7D, 0x60, 0x3F, 0x72,
+0x94, 0x60, 0x32, 0x72, 0xC9, 0x61, 0x0E, 0x92,
+0xC2, 0xEA, 0x3F, 0x6A, 0xB6, 0x60, 0x18, 0x10,
+0x0E, 0x25, 0x07, 0x94, 0x02, 0x6A, 0x5A, 0xEC,
+0x01, 0x2A, 0xE5, 0xE8, 0x08, 0x94, 0x03, 0x6A,
+0x12, 0xE8, 0x00, 0xF6, 0x00, 0x30, 0x00, 0xF6,
+0x03, 0x30, 0xC4, 0x16, 0x6D, 0x2D, 0x04, 0x97,
+0xDF, 0xF6, 0x00, 0x27, 0x06, 0x97, 0x14, 0x6A,
+0xA0, 0x27, 0x1E, 0x6A, 0x9E, 0x17, 0xF7, 0x25,
+0x3C, 0x6A, 0x9B, 0x17, 0x08, 0x2D, 0x04, 0x97,
+0xBF, 0xF6, 0x14, 0x27, 0x0A, 0x92, 0xF9, 0x67,
+0xE2, 0xEA, 0x14, 0x6A, 0x92, 0x60, 0xFF, 0x4C,
+0x00, 0xF6, 0x80, 0x34, 0x00, 0xF6, 0x83, 0x34,
+0x01, 0x6B, 0x16, 0x6A, 0x8A, 0x17, 0x07, 0x92,
+0x42, 0xE8, 0xA0, 0x60, 0x01, 0x48, 0x00, 0xF6,
+0x00, 0x30, 0x00, 0xF6, 0x03, 0x30, 0x17, 0x6A,
+0x9D, 0x16, 0x09, 0x2D, 0x04, 0x97, 0x9F, 0xF6,
+0x19, 0x27, 0x0A, 0x92, 0xF9, 0x67, 0xE2, 0xEA,
+0x16, 0x6A, 0x7F, 0xF7, 0x16, 0x60, 0xFF, 0x48,
+0x00, 0xF6, 0x00, 0x30, 0x00, 0xF6, 0x03, 0x30,
+0x01, 0x6B, 0x32, 0x6A, 0x6E, 0x17, 0x09, 0x2D,
+0x04, 0x97, 0x9F, 0xF6, 0x07, 0x27, 0x0A, 0x92,
+0xF9, 0x67, 0xE2, 0xEA, 0x1E, 0x6A, 0x7F, 0xF7,
+0x04, 0x60, 0xFF, 0x48, 0x00, 0xF6, 0x00, 0x30,
+0x00, 0xF6, 0x03, 0x30, 0x01, 0x6B, 0x20, 0x6A,
+0x5C, 0x17, 0x09, 0x92, 0x42, 0xEC, 0x7F, 0xF7,
+0x11, 0x60, 0x01, 0x4C, 0x00, 0xF6, 0x80, 0x34,
+0x00, 0xF6, 0x83, 0x34, 0x21, 0x6A, 0x6E, 0x16,
+0x09, 0x2D, 0x04, 0x97, 0x7F, 0xF6, 0x0A, 0x27,
+0x0A, 0x92, 0xF9, 0x67, 0xE2, 0xEA, 0x20, 0x6A,
+0x5F, 0xF7, 0x07, 0x60, 0xFF, 0x4C, 0x00, 0xF6,
+0x80, 0x34, 0x00, 0xF6, 0x83, 0x34, 0xD0, 0x17,
+0x14, 0x94, 0x03, 0x6A, 0x40, 0xC4, 0x15, 0x94,
+0x07, 0x6A, 0x40, 0xC4, 0x00, 0x6A, 0x4D, 0x17,
+0x08, 0x94, 0x04, 0x6A, 0x00, 0x68, 0x52, 0x16,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x0C, 0x9A, 0x0F, 0x6B, 0xFF, 0x6E,
+0xA0, 0xF7, 0x4F, 0xA0, 0x9D, 0x67, 0x6C, 0xEA,
+0xCC, 0xEA, 0x7D, 0x67, 0x51, 0xC3, 0xA0, 0xF7,
+0x6F, 0xA0, 0x72, 0x33, 0x70, 0xC4, 0x41, 0xF0,
+0x80, 0xA0, 0x08, 0x6B, 0x8C, 0xEB, 0x56, 0x23,
+0x21, 0xF0, 0x7E, 0xA0, 0x2C, 0x69, 0x01, 0x6F,
+0x72, 0x35, 0x2C, 0xEB, 0x6B, 0xEB, 0xEC, 0xED,
+0xC0, 0xF7, 0x62, 0x31, 0xCC, 0xED, 0xCC, 0xE9,
+0x00, 0x6B, 0x05, 0x22, 0x05, 0x72, 0x78, 0x67,
+0x6B, 0xEB, 0xC0, 0xF7, 0x62, 0x33, 0x05, 0x6E,
+0xCC, 0xEC, 0x01, 0x74, 0x01, 0x6E, 0x05, 0x61,
+0xD4, 0xA0, 0xFF, 0x6C, 0x01, 0x5E, 0xD8, 0x67,
+0x8C, 0xEE, 0x15, 0x23, 0x01, 0x72, 0x06, 0x61,
+0x46, 0x67, 0xAD, 0xEA, 0x03, 0x22, 0x53, 0x6A,
+0xA0, 0xF7, 0x4F, 0xC0, 0xE0, 0xF5, 0x4A, 0xA0,
+0x01, 0x6C, 0x08, 0xD3, 0x41, 0xF0, 0x41, 0xC0,
+0x07, 0xD6, 0xB1, 0x18, 0x53, 0xC8, 0x06, 0xD5,
+0x08, 0x93, 0x07, 0x96, 0x06, 0x95, 0x15, 0x25,
+0xE0, 0xF5, 0x43, 0xA0, 0xE0, 0xF5, 0x82, 0xA0,
+0x50, 0x32, 0x89, 0xE2, 0x21, 0xF0, 0x5C, 0xC0,
+0xE0, 0xF5, 0x81, 0xA0, 0xE0, 0xF5, 0x40, 0xA0,
+0x43, 0xEC, 0x41, 0xF0, 0x80, 0xA0, 0x15, 0x60,
+0x21, 0x6A, 0x4B, 0xEA, 0x8C, 0xEA, 0x41, 0xF0,
+0x40, 0xC0, 0x67, 0x26, 0x41, 0xF0, 0x40, 0xA0,
+0x01, 0x6B, 0x6C, 0xEA, 0x02, 0x22, 0x54, 0xA0,
+0x0B, 0x2A, 0x00, 0x6A, 0x7D, 0x67, 0x51, 0xC3,
+0x50, 0xC3, 0x0B, 0x10, 0x00, 0x69, 0x00, 0x6D,
+0xB3, 0x17, 0x20, 0x6A, 0x8D, 0xEA, 0xEB, 0x17,
+0x05, 0x6A, 0x9D, 0x67, 0x51, 0xC4, 0x5D, 0x67,
+0x70, 0xC2, 0x00, 0xF0, 0x11, 0x04, 0xB1, 0x18,
+0xC6, 0xC8, 0x04, 0x05, 0xA0, 0xF7, 0x4F, 0xA0,
+0x0F, 0x6B, 0x6C, 0xEA, 0xFF, 0x6B, 0x6C, 0xEA,
+0x7D, 0x67, 0x71, 0xA3, 0x6E, 0xEA, 0x07, 0x2A,
+0xA0, 0xF7, 0x4F, 0xA0, 0x7D, 0x67, 0x70, 0xA3,
+0x52, 0x32, 0x6E, 0xEA, 0x21, 0x22, 0x00, 0x6A,
+0x41, 0xF0, 0x42, 0xC8, 0x41, 0xF0, 0x40, 0xA0,
+0x41, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA, 0x7F, 0x6B,
+0x6C, 0xEA, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x41, 0xF0, 0x40, 0xC0, 0xA0, 0xF7,
+0x2C, 0x40, 0x93, 0xF6, 0x40, 0x9B, 0x87, 0x41,
+0x80, 0x6E, 0x00, 0x6D, 0x09, 0x4C, 0x40, 0xEA,
+0x06, 0xD3, 0x06, 0x93, 0x04, 0x6E, 0x00, 0x6D,
+0x93, 0xF6, 0x40, 0x9B, 0x40, 0xEA, 0x84, 0x41,
+0x5D, 0x67, 0x7D, 0x67, 0x51, 0xA2, 0x70, 0xA3,
+0x0F, 0x6C, 0x8C, 0xEA, 0x70, 0x33, 0x6D, 0xEA,
+0xA0, 0xF7, 0x4F, 0xC0, 0x41, 0xF0, 0x40, 0xA0,
+0x19, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA, 0x41, 0xF0,
+0x40, 0xC0, 0x00, 0x6A, 0x21, 0xF0, 0x5E, 0xC0,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF,
+0x07, 0x63, 0x01, 0x23, 0x1B, 0x25, 0x02, 0x6C,
+0xB1, 0x18, 0x53, 0xC8, 0x06, 0xD5, 0xA0, 0xF7,
+0x4F, 0xA0, 0x0F, 0x6B, 0x06, 0x95, 0x6C, 0xEA,
+0x7D, 0x67, 0x51, 0xC3, 0xA0, 0xF7, 0x4F, 0xA0,
+0x52, 0x32, 0x50, 0xC3, 0x00, 0x6A, 0x41, 0xF0,
+0x42, 0xC8, 0x41, 0xF0, 0x40, 0xA0, 0x41, 0x6B,
+0x6B, 0xEB, 0x6C, 0xEA, 0x7F, 0x6B, 0x6C, 0xEA,
+0x41, 0xF0, 0x40, 0xC0, 0x5D, 0x67, 0x51, 0xA2,
+0x01, 0x72, 0x0A, 0x60, 0x24, 0x22, 0x02, 0x72,
+0x15, 0x60, 0x03, 0x72, 0x1A, 0x60, 0x04, 0x6A,
+0x7D, 0x67, 0x51, 0xC3, 0x05, 0x6A, 0x78, 0x17,
+0x01, 0x6A, 0x7D, 0x67, 0x51, 0xC3, 0x03, 0x21,
+0x05, 0x6A, 0x7D, 0x67, 0x71, 0x17, 0x5D, 0x67,
+0x50, 0xA2, 0x05, 0x72, 0x7F, 0xF7, 0x19, 0x61,
+0x02, 0x6A, 0xF7, 0x17, 0x02, 0x6A, 0x7D, 0x67,
+0x51, 0xC3, 0x06, 0x6A, 0x7F, 0xF7, 0x04, 0x21,
+0xEF, 0x17, 0x5D, 0x67, 0x50, 0xA2, 0x07, 0x72,
+0x02, 0x61, 0x7F, 0xF7, 0x0A, 0x25, 0x01, 0x6A,
+0x7D, 0x67, 0x51, 0xC3, 0x02, 0x6A, 0x58, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x0C, 0x9A, 0x02, 0x6B, 0x41, 0xF0, 0x40, 0xA0,
+0x6C, 0xEA, 0x29, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF6, 0x40, 0x9A,
+0x9C, 0x6E, 0xA0, 0xF7, 0x8C, 0x40, 0x40, 0xEA,
+0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A, 0xD0, 0xF6,
+0xA4, 0x40, 0xF0, 0xF5, 0x8C, 0x40, 0x40, 0xEA,
+0xD8, 0x6E, 0xE0, 0xF5, 0x43, 0xA0, 0xE0, 0xF5,
+0x62, 0xA0, 0x50, 0x32, 0x69, 0xE2, 0x21, 0xF0,
+0x5C, 0xC0, 0xE0, 0xF5, 0x5C, 0xA0, 0x20, 0x5A,
+0x03, 0x61, 0x20, 0x6A, 0xE0, 0xF5, 0x5C, 0xC0,
+0xB1, 0x18, 0x53, 0xC8, 0x00, 0x6C, 0x7B, 0xA0,
+0x41, 0xF0, 0x40, 0xA0, 0x02, 0x6E, 0x01, 0x6D,
+0x66, 0x34, 0xCB, 0xEE, 0xAC, 0xEC, 0xCC, 0xEA,
+0x8D, 0xEA, 0x6E, 0x33, 0x05, 0x6C, 0xAC, 0xEB,
+0x8B, 0xEC, 0x68, 0x33, 0x8C, 0xEA, 0x6D, 0xEA,
+0x41, 0xF0, 0x40, 0xC0, 0x05, 0x97, 0x04, 0x90,
+0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0xE0, 0xF3, 0x61, 0xA2, 0x02, 0x2B, 0x20, 0xE8,
+0x00, 0x6A, 0x01, 0x74, 0x03, 0x61, 0xF0, 0x4D,
+0x04, 0x5D, 0xF9, 0x61, 0xE0, 0xF3, 0x64, 0x9A,
+0x01, 0x4B, 0x1C, 0x23, 0xE0, 0xF3, 0x64, 0xDA,
+0x81, 0xF3, 0x7C, 0x9A, 0xE0, 0xF3, 0x08, 0x6D,
+0x63, 0xEE, 0x6F, 0xE6, 0x15, 0x61, 0xBB, 0xEB,
+0x01, 0x2D, 0xE5, 0xE8, 0x81, 0xF3, 0xDC, 0xDA,
+0xE0, 0xF3, 0x44, 0x9A, 0x01, 0x72, 0x12, 0xEB,
+0xE2, 0x60, 0xFF, 0xF7, 0x1F, 0x6A, 0x43, 0xEB,
+0xDE, 0x61, 0x02, 0x6A, 0x4E, 0xEC, 0x01, 0x5C,
+0x20, 0xE8, 0x58, 0x67, 0x01, 0x6B, 0xE2, 0x17,
+0xFF, 0x4B, 0xE9, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0x0C, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF6,
+0x40, 0x9A, 0x08, 0x6E, 0x51, 0xF3, 0x84, 0x40,
+0x40, 0xEA, 0x00, 0x6D, 0xC1, 0xF3, 0x90, 0x98,
+0xB1, 0x18, 0xB6, 0xC5, 0x00, 0x65, 0xC1, 0xF3,
+0x88, 0x98, 0xB1, 0x18, 0xB6, 0xC5, 0x00, 0x65,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0x09, 0xD5, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0x26, 0x67, 0xE4, 0x67, 0x84, 0x36, 0xA0, 0x35,
+0x00, 0x6A, 0x3E, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x88, 0x9A,
+0xE8, 0x33, 0x6D, 0xE4, 0x49, 0x9B, 0x43, 0xE9,
+0x80, 0xF0, 0x0A, 0x61, 0x4B, 0xE1, 0x1B, 0x9B,
+0x41, 0xE0, 0x1B, 0xDB, 0xCD, 0xE4, 0x1B, 0x65,
+0x60, 0xAB, 0x18, 0x67, 0x01, 0x4B, 0x60, 0xC8,
+0xFF, 0x68, 0x6F, 0x47, 0x0C, 0xEB, 0x10, 0x5B,
+0x3B, 0x65, 0xF8, 0x65, 0x00, 0x6B, 0x0C, 0x60,
+0x19, 0x67, 0x01, 0x6B, 0x64, 0xE8, 0x14, 0xF5,
+0x0F, 0x68, 0x0C, 0xEB, 0x01, 0x5B, 0x01, 0x68,
+0x78, 0x67, 0x0E, 0xEB, 0x1F, 0x67, 0x0C, 0xEB,
+0x68, 0x33, 0x71, 0xE4, 0xA0, 0xF0, 0x74, 0x9C,
+0xA0, 0xF0, 0x3C, 0xDC, 0x4D, 0xE3, 0xA0, 0xF0,
+0x74, 0xDC, 0xC0, 0xF0, 0x64, 0x9C, 0x4D, 0xE3,
+0xC0, 0xF0, 0x64, 0xDC, 0x4B, 0xF3, 0x6C, 0x9D,
+0xE0, 0xF5, 0x78, 0x9B, 0xC0, 0xF0, 0x6C, 0xDC,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x4B, 0xF3, 0x68, 0x9B, 0x01, 0x77, 0xD1, 0xE3,
+0xC0, 0xAC, 0x51, 0x61, 0x4B, 0xF3, 0x8C, 0x9D,
+0x15, 0xA4, 0x18, 0x65, 0x0E, 0x60, 0xE0, 0xF3,
+0x0C, 0x9C, 0x43, 0xE8, 0x02, 0x60, 0xE0, 0xF3,
+0x4C, 0xDC, 0x07, 0x26, 0x5C, 0x9B, 0xDB, 0xEA,
+0x01, 0x2E, 0xE5, 0xE8, 0x12, 0xEA, 0xE0, 0xF3,
+0x54, 0xDC, 0x4B, 0xF3, 0x4C, 0x9D, 0x81, 0xF3,
+0x51, 0xA2, 0x09, 0x72, 0x12, 0x61, 0x4B, 0x47,
+0x02, 0x5A, 0x0F, 0x60, 0x4E, 0x9B, 0xE0, 0xF3,
+0x08, 0x6B, 0x43, 0xE9, 0x80, 0xF0, 0x03, 0x61,
+0x4B, 0xE1, 0x7B, 0xEA, 0x01, 0x2B, 0xE5, 0xE8,
+0x07, 0x67, 0xB1, 0x18, 0x6E, 0xC6, 0x12, 0xEC,
+0xF0, 0x67, 0x09, 0x95, 0xB1, 0x18, 0xC8, 0xC6,
+0x87, 0x67, 0x0D, 0x22, 0xB1, 0x18, 0xF9, 0xC7,
+0x00, 0x65, 0xB1, 0x18, 0x9D, 0xCA, 0x00, 0x65,
+0xB1, 0x18, 0x2D, 0xC6, 0x00, 0x65, 0x03, 0x22,
+0xB1, 0x18, 0x41, 0xC6, 0x00, 0x65, 0x09, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x48, 0x9A, 0x07, 0x43, 0x01, 0x48,
+0x08, 0x30, 0x01, 0xE2, 0x21, 0xD8, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x0F, 0x41, 0x4B, 0xE0, 0x74, 0x17, 0x04, 0x77,
+0x27, 0x61, 0x4B, 0xF3, 0x8C, 0x9D, 0x15, 0xA4,
+0x18, 0x65, 0xBB, 0x60, 0x04, 0xF7, 0x10, 0x5A,
+0x16, 0x61, 0x61, 0xF3, 0x1B, 0xA4, 0x18, 0x65,
+0x18, 0x67, 0x02, 0x48, 0x61, 0xF3, 0x1B, 0xC4,
+0xE0, 0xF3, 0x10, 0x9C, 0x43, 0xE8, 0x02, 0x60,
+0xE0, 0xF3, 0x50, 0xDC, 0xAA, 0x26, 0x5F, 0x9B,
+0xDB, 0xEA, 0x01, 0x2E, 0xE5, 0xE8, 0x12, 0xEA,
+0xE0, 0xF3, 0x58, 0xDC, 0xA2, 0x17, 0x82, 0xF3,
+0x08, 0x5A, 0xEE, 0x61, 0x61, 0xF3, 0x1B, 0xA4,
+0x18, 0x65, 0x18, 0x67, 0x01, 0x48, 0xE6, 0x17,
+0x09, 0x77, 0x97, 0x61, 0x04, 0xF7, 0x10, 0x5A,
+0x1B, 0x61, 0x4B, 0xF3, 0x8C, 0x9D, 0x61, 0xF3,
+0x1B, 0xA4, 0x18, 0x65, 0x18, 0x67, 0x02, 0x48,
+0x61, 0xF3, 0x1B, 0xC4, 0x4B, 0xF3, 0x8C, 0x9D,
+0xC0, 0xF3, 0x16, 0xAC, 0x43, 0xE8, 0x02, 0x60,
+0xC0, 0xF3, 0x56, 0xCC, 0x82, 0x26, 0x80, 0xF0,
+0x50, 0x9B, 0xDB, 0xEA, 0x01, 0x2E, 0xE5, 0xE8,
+0x12, 0xEA, 0xC0, 0xF3, 0x54, 0xCC, 0x79, 0x17,
+0xA1, 0xF7, 0x00, 0x5A, 0xEB, 0x61, 0x4B, 0xF3,
+0x8C, 0x9D, 0x61, 0xF3, 0x1B, 0xA4, 0x18, 0x65,
+0x18, 0x67, 0x01, 0x48, 0xE1, 0x17, 0x4F, 0xEA,
+0x29, 0xE2, 0x7B, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0xF7, 0xF0, 0x01, 0x6B,
+0x4B, 0xF3, 0x0C, 0x99, 0x60, 0x33, 0x60, 0x33,
+0x73, 0xF6, 0x5C, 0x9B, 0x87, 0x40, 0x08, 0x6E,
+0xB0, 0xF0, 0xA4, 0x40, 0x0D, 0x4C, 0x40, 0xEA,
+0x04, 0xD3, 0x04, 0x93, 0x87, 0x40, 0x98, 0x6E,
+0x73, 0xF6, 0x5C, 0x9B, 0x4B, 0xF3, 0x6C, 0x99,
+0xB0, 0xF0, 0xAC, 0x43, 0x40, 0xEA, 0x15, 0x4C,
+0x4B, 0xF3, 0x4C, 0x99, 0x3F, 0x6D, 0x80, 0x9A,
+0x61, 0x9A, 0x8D, 0xEB, 0x61, 0xDA, 0x81, 0xF3,
+0x72, 0xA2, 0x00, 0x6C, 0x80, 0xDA, 0xAC, 0xEB,
+0x81, 0xF3, 0x72, 0xC2, 0x81, 0xF3, 0x70, 0xA2,
+0x61, 0xF3, 0x9B, 0xC2, 0xFE, 0x4C, 0x8C, 0xEB,
+0x81, 0xF3, 0x70, 0xC2, 0xB1, 0x18, 0x9D, 0xCA,
+0x00, 0x65, 0xB1, 0x18, 0x41, 0xC6, 0x00, 0x65,
+0xB1, 0x18, 0x5C, 0xCA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF6,
+0x40, 0x9A, 0x14, 0x6E, 0x60, 0xF5, 0x88, 0x40,
+0x40, 0xEA, 0x00, 0x6D, 0x54, 0xA0, 0x07, 0x2A,
+0x59, 0xA0, 0x02, 0x72, 0x04, 0x61, 0x80, 0xF0,
+0x48, 0xA8, 0x60, 0xF5, 0x58, 0xC8, 0x95, 0xA0,
+0x07, 0x6A, 0xFF, 0x6E, 0x4C, 0xEC, 0xCC, 0xEC,
+0x1C, 0x24, 0x9A, 0xA0, 0x0F, 0x6A, 0x8C, 0xEA,
+0xCC, 0xEA, 0x60, 0xF5, 0x52, 0xC0, 0x15, 0x22,
+0x60, 0xF5, 0xB6, 0xA0, 0x01, 0x6F, 0x60, 0xF5,
+0xF3, 0xC0, 0x61, 0x45, 0x60, 0xF5, 0x76, 0xC0,
+0x92, 0x33, 0xCC, 0xEB, 0x60, 0xF5, 0x74, 0xC0,
+0x08, 0x23, 0x02, 0x4D, 0x6E, 0xEA, 0x60, 0xF5,
+0xB6, 0xC0, 0x08, 0x2A, 0x02, 0x6A, 0x60, 0xF5,
+0x55, 0xC0, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x60, 0xF5, 0xF5, 0xC0,
+0xF8, 0x17, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x7D, 0x67, 0x00, 0x6A,
+0x58, 0xC3, 0x59, 0xC3, 0x5A, 0xC3, 0x5B, 0xC3,
+0x5C, 0xC3, 0x5D, 0xC3, 0x5E, 0xC3, 0x5F, 0xC3,
+0x4B, 0xF3, 0x4C, 0x98, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x08, 0xD2, 0x08, 0x93, 0x20, 0x31,
+0x73, 0xF6, 0x5C, 0x99, 0x87, 0x43, 0x06, 0x05,
+0x08, 0x6E, 0x40, 0xEA, 0x0D, 0x4C, 0x4B, 0xF3,
+0x6C, 0x98, 0xF7, 0xF0, 0x01, 0x6D, 0x73, 0xF6,
+0x5C, 0x99, 0xA0, 0x35, 0x87, 0x43, 0xA0, 0x35,
+0x90, 0x6E, 0xCB, 0xF0, 0x00, 0x4D, 0x40, 0xEA,
+0x19, 0x4C, 0x4B, 0xF3, 0x6C, 0x98, 0x73, 0xF6,
+0x5C, 0x99, 0x98, 0x6E, 0x83, 0x67, 0x08, 0x93,
+0x7F, 0x4C, 0x3D, 0x4C, 0xA7, 0x43, 0x40, 0xEA,
+0x15, 0x4D, 0x4B, 0xF3, 0x8C, 0x98, 0x73, 0xF6,
+0x5C, 0x99, 0x08, 0x6E, 0xA7, 0x44, 0x7F, 0x4C,
+0x0D, 0x4D, 0x40, 0xEA, 0x35, 0x4C, 0xF7, 0xF0,
+0x01, 0x69, 0x4B, 0xF3, 0x6C, 0x98, 0x20, 0x31,
+0x20, 0x31, 0x12, 0x6A, 0x5D, 0xC3, 0xD2, 0xF1,
+0x58, 0x99, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x1A, 0x65, 0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x68, 0xF2, 0x19, 0x4A, 0x80, 0x34,
+0x04, 0xD2, 0x01, 0x6F, 0x58, 0x67, 0x00, 0x6E,
+0x05, 0x6D, 0xE9, 0xF6, 0x00, 0x4C, 0x40, 0xEA,
+0x09, 0xD3, 0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0xC1, 0xF3, 0x50, 0xDB, 0x08, 0x93,
+0x4B, 0xF3, 0x4C, 0x98, 0x80, 0x34, 0xB8, 0xAB,
+0x97, 0xF0, 0x17, 0x6B, 0x60, 0x33, 0x09, 0xD2,
+0x60, 0x33, 0xD2, 0xF1, 0x58, 0x99, 0xC8, 0xF2,
+0x19, 0x4B, 0x04, 0xD3, 0x01, 0x6F, 0x00, 0x6E,
+0xE9, 0xF6, 0x14, 0x4C, 0x40, 0xEA, 0x0A, 0xD3,
+0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xA1, 0xF3, 0x48, 0xDB, 0x08, 0x93, 0x4B, 0xF3,
+0x4C, 0x98, 0x80, 0x34, 0xBC, 0xAB, 0x0A, 0x93,
+0x09, 0xD2, 0xD2, 0xF1, 0x58, 0x99, 0x04, 0xD3,
+0x01, 0x6F, 0x09, 0xF7, 0x04, 0x4C, 0x40, 0xEA,
+0x00, 0x6E, 0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0xA1, 0xF3, 0x4C, 0xDB, 0x4B, 0xF3,
+0x4C, 0x98, 0xD2, 0xF1, 0x78, 0x99, 0x80, 0x34,
+0x09, 0xD2, 0x08, 0x92, 0x01, 0x6F, 0x00, 0x6E,
+0x60, 0xF0, 0xA8, 0xAA, 0x97, 0xF0, 0x17, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC8, 0xF2, 0x09, 0x4A,
+0x09, 0xF7, 0x14, 0x4C, 0x40, 0xEB, 0x04, 0xD2,
+0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xC1, 0xF3, 0x48, 0xDB, 0x4B, 0xF3, 0x4C, 0x98,
+0xD2, 0xF1, 0x78, 0x99, 0x80, 0x34, 0x09, 0xD2,
+0x08, 0x92, 0x01, 0x6F, 0x00, 0x6E, 0x40, 0xF0,
+0xA8, 0xAA, 0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xE8, 0xF2, 0x19, 0x4A, 0x29, 0xF7,
+0x04, 0x4C, 0x40, 0xEB, 0x04, 0xD2, 0x09, 0x93,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0xA1, 0xF3,
+0x50, 0xDB, 0x4B, 0xF3, 0x4C, 0x98, 0xD2, 0xF1,
+0x78, 0x99, 0x80, 0x34, 0x09, 0xD2, 0x08, 0x92,
+0x01, 0x6F, 0x00, 0x6E, 0x40, 0xF0, 0xB0, 0xAA,
+0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x08, 0xF3, 0x09, 0x4A, 0x29, 0xF7, 0x14, 0x4C,
+0x40, 0xEB, 0x04, 0xD2, 0x09, 0x93, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0xA1, 0xF3, 0x54, 0xDB,
+0x4B, 0xF3, 0x4C, 0x98, 0xD2, 0xF1, 0x78, 0x99,
+0x80, 0x34, 0x09, 0xD2, 0x08, 0x92, 0x01, 0x6F,
+0x00, 0x6E, 0x40, 0xF0, 0xB8, 0xAA, 0x97, 0xF0,
+0x17, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x08, 0xF3,
+0x19, 0x4A, 0x49, 0xF7, 0x04, 0x4C, 0x40, 0xEB,
+0x04, 0xD2, 0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0xA1, 0xF3, 0x58, 0xDB, 0x4B, 0xF3,
+0x4C, 0x98, 0xD2, 0xF1, 0x78, 0x99, 0x80, 0x34,
+0x09, 0xD2, 0x08, 0x92, 0x01, 0x6F, 0x00, 0x6E,
+0x60, 0xF0, 0xA0, 0xAA, 0x97, 0xF0, 0x17, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x28, 0xF3, 0x09, 0x4A,
+0x49, 0xF7, 0x14, 0x4C, 0x40, 0xEB, 0x04, 0xD2,
+0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xA1, 0xF3, 0x5C, 0xDB, 0x4B, 0xF3, 0x4C, 0x98,
+0xD2, 0xF1, 0x78, 0x99, 0x80, 0x34, 0x09, 0xD2,
+0x08, 0x92, 0x01, 0x6F, 0x00, 0x6E, 0x40, 0xF0,
+0xA0, 0xAA, 0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x28, 0xF3, 0x19, 0x4A, 0x69, 0xF7,
+0x04, 0x4C, 0x40, 0xEB, 0x04, 0xD2, 0x09, 0x93,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0xC1, 0xF3,
+0x40, 0xDB, 0x4B, 0xF3, 0x4C, 0x98, 0xD2, 0xF1,
+0x78, 0x99, 0x80, 0x34, 0x09, 0xD2, 0x08, 0x92,
+0x01, 0x6F, 0x00, 0x6E, 0xB4, 0xAA, 0x97, 0xF0,
+0x17, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x48, 0xF3,
+0x09, 0x4A, 0x69, 0xF7, 0x14, 0x4C, 0x40, 0xEB,
+0x04, 0xD2, 0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0xC1, 0xF3, 0x44, 0xDB, 0x4B, 0xF3,
+0x4C, 0x98, 0xD2, 0xF1, 0x78, 0x99, 0x80, 0x34,
+0x09, 0xD2, 0x08, 0x92, 0x01, 0x6F, 0x00, 0x6E,
+0x60, 0xF0, 0xB0, 0xAA, 0x97, 0xF0, 0x17, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x48, 0xF3, 0x19, 0x4A,
+0x89, 0xF7, 0x04, 0x4C, 0x40, 0xEB, 0x04, 0xD2,
+0x09, 0x93, 0x01, 0x6F, 0x00, 0x6E, 0xC1, 0xF3,
+0x4C, 0xDB, 0x4B, 0xF3, 0x4C, 0x98, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x09, 0xD2, 0xD2, 0xF1,
+0x58, 0x99, 0x60, 0x33, 0x89, 0xF7, 0x14, 0x4B,
+0x1A, 0x65, 0x08, 0x92, 0x0A, 0xD3, 0x80, 0xF0,
+0xA8, 0xAA, 0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x68, 0xF3, 0x09, 0x4A, 0x04, 0xD2,
+0x58, 0x67, 0x40, 0xEA, 0x83, 0x67, 0x09, 0x93,
+0x01, 0x6F, 0x00, 0x6E, 0xC1, 0xF3, 0x58, 0xDB,
+0x4B, 0xF3, 0x4C, 0x98, 0x0A, 0x93, 0x09, 0xD2,
+0xD2, 0xF1, 0x58, 0x99, 0x1A, 0x65, 0x08, 0x92,
+0x60, 0xF0, 0xB8, 0xAA, 0x97, 0xF0, 0x17, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x68, 0xF3, 0x19, 0x4A,
+0x04, 0xD2, 0x58, 0x67, 0x40, 0xEA, 0x83, 0x67,
+0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xC1, 0xF3, 0x5C, 0xDB, 0x4B, 0xF3, 0x4C, 0x98,
+0xD2, 0xF1, 0x78, 0x99, 0x80, 0x34, 0x09, 0xD2,
+0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x88, 0xF3, 0x09, 0x4A, 0x04, 0xD2, 0x01, 0x6F,
+0x00, 0x6E, 0xA9, 0xF7, 0x04, 0x4C, 0x40, 0xEB,
+0x03, 0x6D, 0x09, 0x93, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0xC1, 0xF3, 0x54, 0xDB, 0x4B, 0xF3,
+0x4C, 0x98, 0xD2, 0xF1, 0x78, 0x99, 0x80, 0x34,
+0x09, 0xD2, 0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xC2, 0xF5, 0x11, 0x4A, 0x04, 0xD2,
+0x01, 0x6F, 0x00, 0x6E, 0xA9, 0xF7, 0x18, 0x4C,
+0x40, 0xEB, 0x05, 0x6D, 0x09, 0x93, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0xE1, 0xF3, 0x44, 0xDB,
+0x08, 0x92, 0xD2, 0xF1, 0x78, 0x99, 0x4B, 0xF3,
+0x0C, 0x98, 0xA0, 0xF0, 0xA8, 0xAA, 0x97, 0xF0,
+0x17, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xE8, 0xF2,
+0x09, 0x4A, 0x80, 0x34, 0x04, 0xD2, 0x01, 0x6F,
+0xC9, 0xF7, 0x0C, 0x4C, 0x40, 0xEB, 0x00, 0x6E,
+0xE1, 0xF3, 0x40, 0xD8, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x5D, 0x67, 0x20, 0xF0,
+0x80, 0xC2, 0xD4, 0xCA, 0x20, 0xF0, 0x40, 0xA2,
+0x05, 0xD5, 0x14, 0x5A, 0x04, 0xD2, 0x0F, 0x60,
+0x00, 0x6E, 0x04, 0x05, 0xD1, 0x18, 0xE5, 0x6B,
+0x08, 0x6C, 0x00, 0x6E, 0x08, 0x05, 0xD1, 0x18,
+0xE5, 0x6B, 0x09, 0x6C, 0x00, 0x6E, 0x0A, 0x05,
+0xD1, 0x18, 0xE5, 0x6B, 0x0A, 0x6C, 0x07, 0x97,
+0x00, 0xEF, 0x04, 0x63, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x4B, 0xF3, 0x4C, 0x99,
+0x0A, 0x74, 0x04, 0x67, 0x3D, 0x65, 0x41, 0xF3,
+0x74, 0xA2, 0x62, 0x60, 0xC5, 0x67, 0x05, 0xD5,
+0xA4, 0x67, 0x83, 0x67, 0xB1, 0x18, 0xAE, 0xCA,
+0x04, 0xD3, 0x4B, 0xF3, 0x4C, 0x99, 0x22, 0x6B,
+0x1B, 0x65, 0x81, 0xF3, 0xB0, 0xA2, 0x78, 0x67,
+0x41, 0xF3, 0x14, 0xC2, 0xAC, 0xEB, 0x22, 0x73,
+0x07, 0x60, 0x05, 0x94, 0x01, 0x70, 0x04, 0x93,
+0x3C, 0x65, 0x47, 0x61, 0x95, 0xA2, 0x4E, 0x2C,
+0x02, 0x6B, 0xAC, 0xEB, 0x05, 0x23, 0x20, 0xF3,
+0x74, 0xAA, 0x01, 0x4B, 0x20, 0xF3, 0x74, 0xCA,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x04, 0x33, 0x6D, 0xE2, 0x80, 0xF3,
+0x98, 0xAB, 0x01, 0x4C, 0x80, 0xF3, 0x98, 0xCB,
+0x0C, 0x33, 0x6D, 0xE2, 0x29, 0x9B, 0xF1, 0xAB,
+0xFF, 0x48, 0xFF, 0x6B, 0x6C, 0xE8, 0x11, 0x58,
+0x00, 0x6B, 0x08, 0x60, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x08, 0xF1, 0x08, 0x4B,
+0x61, 0xE0, 0x60, 0xA0, 0x81, 0xF3, 0x92, 0xA2,
+0x04, 0x6E, 0xCB, 0xEE, 0x03, 0x6D, 0xCC, 0xEC,
+0x6C, 0xED, 0xAD, 0xEC, 0x01, 0x73, 0x81, 0xF3,
+0x92, 0xC2, 0xC7, 0x67, 0x06, 0x61, 0xB1, 0x67,
+0x01, 0x6C, 0xB1, 0x18, 0x68, 0xCC, 0x00, 0x65,
+0xCF, 0x17, 0x02, 0x73, 0x03, 0x61, 0xB1, 0x67,
+0x00, 0x6C, 0xF7, 0x17, 0x07, 0x67, 0xB1, 0x67,
+0xB1, 0x18, 0x68, 0xCC, 0x01, 0x6C, 0xD0, 0x67,
+0xF6, 0x17, 0x04, 0x70, 0xB7, 0x60, 0x06, 0x10,
+0x81, 0xF3, 0x50, 0xA2, 0x20, 0x6C, 0x8C, 0xEA,
+0x01, 0x2A, 0x03, 0x67, 0x4B, 0xF3, 0xEC, 0x99,
+0x02, 0x6C, 0xD9, 0x67, 0xB0, 0x67, 0xB1, 0x18,
+0x83, 0xCA, 0x04, 0xD3, 0x41, 0xF3, 0x59, 0xC7,
+0x4B, 0xF3, 0x4C, 0x99, 0x41, 0xF3, 0x99, 0xA2,
+0xB0, 0x24, 0x8B, 0x40, 0x02, 0x5C, 0x04, 0x93,
+0x02, 0x61, 0x08, 0x70, 0xAA, 0x61, 0x04, 0x73,
+0xA8, 0x60, 0x80, 0xF1, 0x62, 0xA2, 0x01, 0x4B,
+0x80, 0xF1, 0x62, 0xC2, 0x9D, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xA4, 0x67, 0xB1, 0x18,
+0x77, 0xCC, 0x00, 0x6C, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xA4, 0x67, 0xB1, 0x18, 0x77, 0xCC, 0x08, 0x6C,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0xC0, 0xF3, 0x7C, 0x9A,
+0x6D, 0xEC, 0x80, 0xF1, 0x63, 0xA2, 0xC0, 0xF3,
+0x9C, 0xDA, 0x01, 0x4B, 0x80, 0xF1, 0x63, 0xC2,
+0x00, 0x6B, 0xE0, 0xF3, 0x61, 0xC2, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0xCC, 0x9A, 0x0F, 0x5C,
+0x00, 0x6F, 0xA1, 0xF3, 0x68, 0x46, 0x08, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xE8, 0xF0, 0x18, 0x4A, 0x49, 0xE4, 0xE0, 0xA2,
+0x43, 0x47, 0x4C, 0x32, 0x49, 0xE6, 0xA4, 0xAA,
+0x09, 0x2D, 0x08, 0xF0, 0x00, 0x6C, 0xB1, 0x18,
+0xCC, 0xCC, 0x00, 0x65, 0x05, 0x97, 0x04, 0x90,
+0x00, 0xEF, 0x03, 0x63, 0x88, 0x34, 0x0F, 0x27,
+0x01, 0x9E, 0x01, 0x6A, 0x44, 0xEF, 0xF0, 0x67,
+0x4C, 0xEF, 0x09, 0x27, 0x91, 0xE3, 0x4F, 0xEA,
+0x80, 0x9C, 0x0C, 0xEA, 0xB1, 0x18, 0xD1, 0xC5,
+0x41, 0xDE, 0xEC, 0x2A, 0xE6, 0x17, 0x91, 0xE3,
+0xB1, 0x18, 0x8A, 0xC5, 0x80, 0x9C, 0xF9, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x41, 0xF3, 0x74, 0xA2,
+0x09, 0x73, 0x14, 0x61, 0xFD, 0x63, 0x05, 0x62,
+0xC0, 0xF3, 0x78, 0x9A, 0x40, 0x6C, 0x8B, 0xEC,
+0x01, 0x4B, 0xC0, 0xF3, 0x78, 0xDA, 0x81, 0xF3,
+0x72, 0xA2, 0x8D, 0xEB, 0x81, 0xF3, 0x72, 0xC2,
+0xB1, 0x18, 0xD7, 0xCC, 0x0A, 0x6C, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x6C, 0x9A, 0x61, 0xF3, 0x54, 0xA3, 0x1F, 0x2A,
+0x81, 0xF3, 0x50, 0xA3, 0x04, 0x67, 0x20, 0x6C,
+0x14, 0x6D, 0x4C, 0xEC, 0x4C, 0xED, 0x1B, 0x2C,
+0x06, 0x6C, 0x1C, 0x65, 0x98, 0x67, 0x4C, 0xEC,
+0x06, 0x74, 0x04, 0x60, 0x18, 0x6C, 0x8C, 0xEA,
+0x10, 0x72, 0x0D, 0x61, 0x81, 0xF3, 0x50, 0xA3,
+0x20, 0x6C, 0x8D, 0xEA, 0x81, 0xF3, 0x50, 0xC3,
+0xB1, 0x18, 0xD7, 0xCC, 0x09, 0x6C, 0xB0, 0x67,
+0xB1, 0x18, 0x77, 0xCC, 0x0A, 0x6C, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x06, 0x6C,
+0x1C, 0x65, 0x98, 0x67, 0x4C, 0xEC, 0x04, 0x74,
+0x0A, 0x61, 0xDB, 0x4C, 0x8C, 0xEA, 0xC1, 0xF3,
+0x8C, 0x9B, 0x81, 0xF3, 0x50, 0xC3, 0xB1, 0x18,
+0xB6, 0xC5, 0x00, 0x65, 0xE8, 0x17, 0xEB, 0x2D,
+0x21, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0x81, 0xF3,
+0x50, 0xC3, 0xE1, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0x6C, 0x9A, 0x04, 0x67,
+0x56, 0xA3, 0x01, 0x72, 0x06, 0x61, 0x61, 0xF3,
+0xB9, 0xA3, 0x0D, 0x6E, 0xB1, 0x18, 0xEB, 0xE8,
+0x00, 0x6C, 0xB0, 0x67, 0xB1, 0x18, 0x77, 0xCC,
+0x09, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x32, 0xF6, 0x5C, 0x9A, 0x40, 0xEA,
+0x00, 0x6C, 0xB1, 0x18, 0xD7, 0xCC, 0x0A, 0x6C,
+0xB1, 0x18, 0xD7, 0xCC, 0x08, 0x6C, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x6C, 0x9A, 0xA4, 0x67, 0xFF, 0x6C,
+0x54, 0xA3, 0xFE, 0x4A, 0x8C, 0xEA, 0x02, 0x5A,
+0x0A, 0x60, 0x41, 0xF3, 0x58, 0xA3, 0x04, 0x22,
+0x41, 0xF3, 0x1A, 0xA3, 0x02, 0x69, 0x05, 0x20,
+0x01, 0x68, 0x03, 0x69, 0x02, 0x10, 0x00, 0x68,
+0x02, 0x69, 0x01, 0x6C, 0xB1, 0x18, 0xEC, 0xD7,
+0x04, 0xD5, 0xB1, 0x18, 0xD7, 0xCC, 0x90, 0x67,
+0x04, 0x95, 0xB1, 0x18, 0x77, 0xCC, 0x91, 0x67,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x41, 0xF3, 0x54, 0xA2, 0x01, 0x72, 0x03, 0x60,
+0xFB, 0x4A, 0x02, 0x5A, 0x08, 0x60, 0xFD, 0x63,
+0x05, 0x62, 0xB1, 0x18, 0x3C, 0xCD, 0x00, 0x65,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x04, 0x67, 0xB1, 0x18, 0xD7, 0xCC,
+0x0E, 0x6C, 0xB0, 0x67, 0xB1, 0x18, 0x77, 0xCC,
+0x11, 0x6C, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x4B, 0xF3, 0xCC, 0x98,
+0x41, 0xF3, 0x54, 0xA6, 0xAF, 0x42, 0x03, 0x5D,
+0x2B, 0x61, 0x04, 0x72, 0x04, 0xD4, 0x61, 0xF3,
+0x39, 0xA6, 0x02, 0x60, 0x09, 0x72, 0x0B, 0x61,
+0xB1, 0x18, 0xD7, 0xCC, 0x02, 0x6C, 0x4B, 0xF3,
+0x4C, 0x98, 0x54, 0xA2, 0x03, 0x72, 0x03, 0x61,
+0xB1, 0x18, 0xD7, 0xCC, 0x05, 0x6C, 0x4B, 0xF3,
+0x4C, 0x98, 0x56, 0xA2, 0x01, 0x72, 0x05, 0x61,
+0x0D, 0x6E, 0xB1, 0x67, 0xB1, 0x18, 0xEB, 0xE8,
+0x00, 0x6C, 0xB1, 0x18, 0xEC, 0xD7, 0x00, 0x6C,
+0x04, 0x95, 0xB1, 0x18, 0x77, 0xCC, 0x05, 0x6C,
+0x4B, 0xF3, 0x6C, 0x98, 0x7F, 0x6C, 0x81, 0xF3,
+0x50, 0xA3, 0x8C, 0xEA, 0x81, 0xF3, 0x50, 0xC3,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x01, 0x6B, 0x60, 0xF5, 0x7A, 0xC2, 0x41, 0xF3,
+0x74, 0xA2, 0x04, 0x73, 0x14, 0x61, 0xFD, 0x63,
+0x05, 0x62, 0x75, 0xA2, 0x07, 0x6D, 0xAC, 0xEB,
+0x08, 0x23, 0x60, 0xF5, 0x69, 0xA2, 0x05, 0x2B,
+0x00, 0xF4, 0x78, 0x9A, 0x01, 0x4B, 0x00, 0xF4,
+0x78, 0xDA, 0xB1, 0x18, 0x67, 0xCD, 0x00, 0x65,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x04, 0x67, 0xB1, 0x18, 0xD7, 0xCC,
+0x03, 0x6C, 0xB0, 0x67, 0xB1, 0x18, 0x77, 0xCC,
+0x06, 0x6C, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x6C, 0x9A, 0x09, 0x6A, 0x4B, 0xEA,
+0x41, 0xF3, 0xB4, 0xA3, 0xAC, 0xEA, 0x02, 0x22,
+0x05, 0x72, 0x0A, 0x61, 0xC0, 0xF3, 0x40, 0xAB,
+0x41, 0xF3, 0xB5, 0xC3, 0x01, 0x4A, 0xC0, 0xF3,
+0x40, 0xCB, 0xB1, 0x18, 0x99, 0xCD, 0x00, 0x65,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x6C, 0x9A, 0x04, 0x67, 0x61, 0xF3, 0x5B, 0xA3,
+0x03, 0x22, 0xFF, 0x4A, 0x61, 0xF3, 0x5B, 0xC3,
+0xB1, 0x18, 0xD7, 0xCC, 0x04, 0x6C, 0xB0, 0x67,
+0xB1, 0x18, 0x77, 0xCC, 0x07, 0x6C, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x4B, 0xF3, 0x6C, 0x99, 0x09, 0xD5, 0x0A, 0xD6,
+0x55, 0xA3, 0x04, 0x67, 0x09, 0x22, 0x07, 0x6C,
+0x8C, 0xEA, 0x01, 0x72, 0x0A, 0x60, 0xB1, 0x18,
+0xCC, 0xCC, 0x01, 0x6C, 0x00, 0x6A, 0x28, 0x10,
+0x54, 0xA3, 0xF9, 0x2A, 0x59, 0xA3, 0x02, 0x72,
+0xF5, 0x17, 0x01, 0x70, 0x61, 0xF3, 0x99, 0xA3,
+0x24, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xB2, 0xF5, 0x5C, 0x9A, 0x02, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x4B, 0xF3, 0x4C, 0x99,
+0x0A, 0x96, 0x70, 0xF5, 0x8C, 0x42, 0xB1, 0x18,
+0x0C, 0xC6, 0x09, 0x95, 0xFF, 0x72, 0xDF, 0x60,
+0x00, 0x6F, 0x00, 0x6E, 0xB0, 0x67, 0xD1, 0x18,
+0x78, 0xCB, 0x82, 0x67, 0x13, 0x2A, 0x02, 0x67,
+0xB1, 0x18, 0xCC, 0xCC, 0x02, 0x6C, 0x50, 0x67,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0xCD, 0x28, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xB2, 0xF5, 0x5C, 0x9A,
+0x00, 0x6E, 0xDA, 0x17, 0x4B, 0xF3, 0x8C, 0x99,
+0x08, 0x33, 0x01, 0xE3, 0x08, 0x30, 0x01, 0xE4,
+0x00, 0xF4, 0x6C, 0x98, 0x01, 0x4B, 0x00, 0xF4,
+0x6C, 0xD8, 0x08, 0x6B, 0x60, 0xF5, 0x70, 0xC4,
+0x60, 0xF5, 0x71, 0xA4, 0x01, 0x4B, 0x60, 0xF5,
+0x71, 0xC4, 0x00, 0x6B, 0x60, 0xF5, 0x69, 0xC4,
+0xDB, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6D,
+0xA0, 0x35, 0xA0, 0x35, 0x4B, 0xF3, 0x2C, 0x9D,
+0x0A, 0xD4, 0x04, 0xD5, 0x60, 0xF5, 0x96, 0xA1,
+0x60, 0xF5, 0x48, 0xA1, 0x00, 0x6F, 0x02, 0x74,
+0x0A, 0x61, 0x60, 0xF5, 0xF7, 0xA1, 0x02, 0x6B,
+0xFF, 0x6C, 0x6E, 0xEF, 0x8C, 0xEF, 0xEB, 0xEF,
+0xC0, 0xF7, 0xE2, 0x37, 0x8C, 0xEF, 0x94, 0xA1,
+0x05, 0x2C, 0x99, 0xA1, 0x02, 0x74, 0x02, 0x61,
+0x95, 0xA1, 0x54, 0x2C, 0x1F, 0x2A, 0x41, 0xF3,
+0x54, 0xA1, 0x01, 0x72, 0x16, 0x61, 0x00, 0x68,
+0x60, 0xF5, 0x89, 0xA1, 0x4F, 0x2C, 0xE0, 0xF3,
+0x5C, 0x99, 0x60, 0xF5, 0x91, 0xC1, 0x01, 0x4A,
+0xE0, 0xF3, 0x5C, 0xD9, 0x0A, 0x27, 0x60, 0xF5,
+0xD5, 0xA1, 0x60, 0xF5, 0xB4, 0xA1, 0x02, 0x6A,
+0x60, 0xF5, 0x57, 0xC1, 0xB1, 0x18, 0xBE, 0xCD,
+0x00, 0x65, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x01, 0x72, 0x00, 0x68,
+0x32, 0x61, 0x41, 0xF3, 0x54, 0xA1, 0x04, 0x72,
+0xF4, 0x61, 0x60, 0xF5, 0x49, 0xA1, 0x63, 0x2A,
+0x00, 0xF4, 0x70, 0x99, 0x81, 0x43, 0x00, 0xF4,
+0x90, 0xD9, 0x60, 0xF5, 0x91, 0xA1, 0x03, 0x5C,
+0x03, 0x60, 0x60, 0xF5, 0x9A, 0xA1, 0x39, 0x24,
+0x00, 0xF4, 0x78, 0x99, 0x81, 0x43, 0x00, 0xF4,
+0x98, 0xD9, 0x00, 0x6C, 0x60, 0xF5, 0x91, 0xC1,
+0x0D, 0x27, 0x60, 0xF5, 0x9A, 0xA1, 0x0A, 0x2C,
+0x60, 0xF5, 0xD5, 0xA1, 0x60, 0xF5, 0xB4, 0xA1,
+0x02, 0x6A, 0x60, 0xF5, 0x57, 0xC1, 0xB1, 0x18,
+0xBE, 0xCD, 0x01, 0x6C, 0x0D, 0xEA, 0xFF, 0x68,
+0x0C, 0xEA, 0xCB, 0x2A, 0xB1, 0x18, 0x67, 0xCD,
+0x0A, 0x94, 0xC7, 0x17, 0x01, 0x68, 0xB0, 0x22,
+0x01, 0x72, 0xCE, 0x17, 0x00, 0xF4, 0x40, 0x99,
+0x01, 0x4A, 0x00, 0xF4, 0x40, 0xD9, 0x00, 0x6A,
+0x60, 0xF5, 0x51, 0xC1, 0x09, 0x27, 0x02, 0x6A,
+0x60, 0xF5, 0x57, 0xC1, 0x60, 0xF5, 0xD5, 0xA1,
+0x60, 0xF5, 0xB4, 0xA1, 0x00, 0x6C, 0xAE, 0x17,
+0xB0, 0x28, 0xB1, 0x18, 0x3C, 0xCD, 0x0A, 0x94,
+0xAC, 0x17, 0x60, 0xF5, 0x57, 0xA1, 0x02, 0x72,
+0x0C, 0x60, 0x60, 0xF5, 0xD3, 0xA1, 0x60, 0xF5,
+0xB2, 0xA1, 0xB1, 0x18, 0xBE, 0xCD, 0x01, 0x6C,
+0x09, 0x2A, 0x60, 0xF5, 0x51, 0xC1, 0x9D, 0x28,
+0xD1, 0x17, 0x60, 0xF5, 0xD5, 0xA1, 0x60, 0xF5,
+0xB4, 0xA1, 0xF3, 0x17, 0x04, 0x92, 0x4B, 0xF3,
+0x6C, 0x9A, 0x00, 0xF4, 0x5C, 0x9B, 0x01, 0x4A,
+0x00, 0xF4, 0x5C, 0xDB, 0x8E, 0x17, 0x00, 0xF4,
+0x54, 0x99, 0x01, 0x4A, 0x00, 0xF4, 0x54, 0xD9,
+0x00, 0x6A, 0x60, 0xF5, 0x51, 0xC1, 0x00, 0x6A,
+0x0C, 0x27, 0x60, 0xF5, 0xD5, 0xA1, 0x60, 0xF5,
+0xB4, 0xA1, 0x02, 0x6A, 0x60, 0xF5, 0x57, 0xC1,
+0x01, 0x6C, 0xB1, 0x18, 0xBE, 0xCD, 0x05, 0xD7,
+0x05, 0x97, 0x04, 0x93, 0x4B, 0xF3, 0x8C, 0x9B,
+0xB8, 0xA4, 0x28, 0x25, 0x60, 0xF5, 0x76, 0xA1,
+0x02, 0x5B, 0x24, 0x60, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0x4B, 0xF3, 0x64, 0xA6,
+0xFF, 0x69, 0x01, 0x4B, 0x2C, 0xEB, 0xA3, 0xEB,
+0x4B, 0xF3, 0x64, 0xC6, 0x81, 0xF3, 0xB2, 0xA4,
+0x02, 0x60, 0xBA, 0x33, 0x13, 0x23, 0xBA, 0x33,
+0x09, 0x23, 0x03, 0x68, 0x03, 0x4B, 0x0C, 0xEB,
+0x3F, 0x68, 0x78, 0x33, 0x0C, 0xED, 0xAD, 0xEB,
+0x81, 0xF3, 0x72, 0xC4, 0x00, 0x6B, 0x4B, 0xF3,
+0x64, 0xC6, 0x08, 0x2F, 0xB1, 0x18, 0x27, 0xCD,
+0x0A, 0x94, 0x4B, 0x17, 0xAC, 0x27, 0x5F, 0xF7,
+0x08, 0x2A, 0xA9, 0x17, 0xF7, 0x22, 0x45, 0x17,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x4B, 0xF3, 0x4C, 0x9B, 0x12, 0x74, 0x05, 0x67,
+0x61, 0xF3, 0xB9, 0xA2, 0xA0, 0xF0, 0x06, 0x60,
+0x13, 0x5C, 0x0A, 0x60, 0x10, 0x74, 0x2B, 0x60,
+0x11, 0x74, 0x80, 0xF0, 0x14, 0x60, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x17, 0x74, 0xA0, 0xF0, 0x12, 0x60, 0x18, 0x74,
+0xF6, 0x61, 0x94, 0xA2, 0xF4, 0x2C, 0x99, 0xA2,
+0x02, 0x6D, 0xAE, 0xEC, 0xF0, 0x2C, 0xB5, 0xA2,
+0xEE, 0x25, 0x60, 0xF5, 0xD3, 0xA2, 0x60, 0xF5,
+0xB2, 0xA2, 0x60, 0xF5, 0x91, 0xC2, 0x60, 0xF5,
+0x97, 0xC2, 0x01, 0x6C, 0xB1, 0x18, 0xBE, 0xCD,
+0x03, 0x67, 0xE1, 0x22, 0x4B, 0xF3, 0x6C, 0x98,
+0x04, 0x6C, 0x81, 0xF3, 0x52, 0xA3, 0x8D, 0xEA,
+0x81, 0xF3, 0x52, 0xC3, 0xD8, 0x17, 0x23, 0x67,
+0x41, 0xF3, 0x74, 0xA2, 0x0B, 0x73, 0xD3, 0x60,
+0x81, 0xF3, 0x73, 0xA2, 0x40, 0x6C, 0x8C, 0xEB,
+0x08, 0x23, 0x41, 0xF3, 0x78, 0xA2, 0x05, 0x2B,
+0xC0, 0xF3, 0x6E, 0xAA, 0x01, 0x4B, 0xC0, 0xF3,
+0x6E, 0xCA, 0x76, 0xA2, 0x01, 0x73, 0x05, 0x60,
+0x81, 0xF3, 0x52, 0xA2, 0x08, 0x6B, 0x6C, 0xEA,
+0x0D, 0x22, 0x00, 0x6C, 0xB1, 0x18, 0xFE, 0xE8,
+0x0D, 0x6E, 0x4B, 0xF3, 0x6C, 0x99, 0x09, 0x6C,
+0x8B, 0xEC, 0x81, 0xF3, 0x52, 0xA3, 0x8C, 0xEA,
+0x81, 0xF3, 0x52, 0xC3, 0x4B, 0xF3, 0x4C, 0x99,
+0x75, 0xA2, 0x05, 0x2B, 0x81, 0xF3, 0x72, 0xA2,
+0x04, 0x6C, 0x8C, 0xEB, 0x20, 0x23, 0x60, 0xF5,
+0xD3, 0xA2, 0x60, 0xF5, 0xB2, 0xA2, 0x00, 0x6B,
+0x60, 0xF5, 0x71, 0xC2, 0x60, 0xF5, 0x77, 0xC2,
+0xB1, 0x18, 0xBE, 0xCD, 0x00, 0x6C, 0x09, 0x22,
+0x4B, 0xF3, 0x6C, 0x99, 0x05, 0x6C, 0x8B, 0xEC,
+0x81, 0xF3, 0x52, 0xA3, 0x8C, 0xEA, 0x81, 0xF3,
+0x52, 0xC3, 0x4B, 0xF3, 0x4C, 0x99, 0x75, 0xA2,
+0x06, 0x23, 0x60, 0xF0, 0x58, 0xAA, 0x03, 0x22,
+0xB1, 0x18, 0xD7, 0xCC, 0x0D, 0x6C, 0x4B, 0xF3,
+0x4C, 0x99, 0x60, 0xF5, 0x78, 0xAA, 0x0B, 0x23,
+0x60, 0xF5, 0x7D, 0xA2, 0x07, 0x73, 0x10, 0x61,
+0xA0, 0xF3, 0x70, 0xAA, 0x81, 0xF3, 0x64, 0xCA,
+0xB1, 0x18, 0xD7, 0xCC, 0x0C, 0x6C, 0xB1, 0x18,
+0xEC, 0xD7, 0x01, 0x6C, 0xB0, 0x67, 0x0B, 0x6C,
+0xB1, 0x18, 0x77, 0xCC, 0x00, 0x65, 0x6F, 0x17,
+0xA0, 0xF3, 0x72, 0xAA, 0xEF, 0x17, 0x41, 0xF3,
+0x54, 0xA2, 0x0C, 0x72, 0x7F, 0xF7, 0x07, 0x60,
+0xB1, 0x18, 0xEC, 0xD7, 0x00, 0x6C, 0xB0, 0x67,
+0x0C, 0x6C, 0xEE, 0x17, 0x41, 0xF3, 0x94, 0xA2,
+0x0D, 0x74, 0x5F, 0xF7, 0x1C, 0x60, 0x56, 0xA2,
+0x01, 0x72, 0x0D, 0x61, 0x23, 0x67, 0x00, 0x6C,
+0xB1, 0x18, 0xEB, 0xE8, 0x0D, 0x6E, 0x4B, 0xF3,
+0x6C, 0x99, 0x08, 0x6C, 0x81, 0xF3, 0x52, 0xA3,
+0x8D, 0xEA, 0x81, 0xF3, 0x52, 0xC3, 0xB1, 0x18,
+0xEC, 0xD7, 0x00, 0x6C, 0xB0, 0x67, 0x0D, 0x6C,
+0xD3, 0x17, 0x74, 0xA2, 0x5F, 0xF7, 0x03, 0x2B,
+0x79, 0xA2, 0x02, 0x73, 0x3F, 0xF7, 0x1F, 0x61,
+0x60, 0xF5, 0x78, 0xAA, 0x3F, 0xF7, 0x1B, 0x23,
+0x60, 0xF5, 0x7D, 0xA2, 0x07, 0x73, 0x81, 0xF3,
+0x64, 0xAA, 0x05, 0x61, 0xA0, 0xF3, 0x90, 0xAA,
+0x6E, 0xEC, 0x3F, 0xF7, 0x10, 0x2C, 0xA0, 0xF3,
+0x52, 0xAA, 0x4E, 0xEB, 0x3F, 0xF7, 0x0B, 0x2B,
+0xB1, 0x18, 0xEC, 0xD7, 0x00, 0x6C, 0xB0, 0x67,
+0x0E, 0x6C, 0xB2, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x4B, 0xF3, 0x4C, 0x99,
+0x01, 0x6B, 0x04, 0x67, 0x81, 0xF3, 0x50, 0xA2,
+0x6C, 0xEA, 0x03, 0x22, 0xB1, 0x18, 0x2D, 0xCB,
+0x00, 0x65, 0x4B, 0xF3, 0x4C, 0x99, 0x54, 0xA2,
+0x0B, 0x2A, 0xB1, 0x18, 0xC2, 0xCC, 0x90, 0x67,
+0xB1, 0x18, 0xD0, 0xC9, 0x00, 0x65, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0xB0, 0x67, 0xB1, 0x18, 0x77, 0xCC, 0x04, 0x6C,
+0xB1, 0x18, 0xD0, 0xC9, 0x00, 0x65, 0x4B, 0xF3,
+0x4C, 0x99, 0xA0, 0xF7, 0x6F, 0xA2, 0x72, 0x33,
+0x1B, 0x23, 0x41, 0xF0, 0x82, 0xAA, 0xFF, 0xF7,
+0x1F, 0x6D, 0x01, 0x4C, 0xAC, 0xEC, 0x12, 0x24,
+0x41, 0xF0, 0x82, 0xCA, 0x8D, 0x43, 0x02, 0x5C,
+0x02, 0x61, 0x06, 0x73, 0x0D, 0x61, 0x41, 0xF0,
+0x42, 0xAA, 0x01, 0x72, 0x03, 0x61, 0xB1, 0x18,
+0xEC, 0xD7, 0x00, 0x6C, 0xB1, 0x18, 0x5F, 0xCD,
+0x90, 0x67, 0xD5, 0x17, 0x01, 0x6C, 0xEC, 0x17,
+0x75, 0xA2, 0x19, 0x23, 0x60, 0xF5, 0xD3, 0xA2,
+0x60, 0xF5, 0xB2, 0xA2, 0x00, 0x6B, 0x9D, 0x67,
+0x70, 0xC4, 0x60, 0xF5, 0x71, 0xC2, 0x60, 0xF5,
+0x77, 0xC2, 0xB1, 0x18, 0xBE, 0xCD, 0x01, 0x6C,
+0x7D, 0x67, 0x70, 0xA3, 0x08, 0x22, 0x4B, 0xF3,
+0x4C, 0x99, 0x60, 0xF5, 0x7A, 0xC2, 0xB1, 0x18,
+0xD7, 0xCC, 0x06, 0x6C, 0xB8, 0x17, 0xB1, 0x18,
+0x67, 0xCD, 0x90, 0x67, 0xB4, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x54, 0xA2, 0x06, 0x2A, 0xB1, 0x18, 0xC2, 0xCC,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xB1, 0x18, 0xE7, 0xCE, 0x00, 0x65, 0xF9, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0xE7, 0xCE,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x4B, 0xF3, 0x4C, 0x99, 0x04, 0x67, 0x41, 0xF3,
+0x74, 0xA2, 0x61, 0xF3, 0xF9, 0xA2, 0x06, 0x73,
+0x81, 0xF3, 0x70, 0xA2, 0x2E, 0x61, 0x80, 0x6C,
+0x8B, 0xEC, 0x8D, 0xEB, 0x81, 0xF3, 0x70, 0xC2,
+0xB0, 0x67, 0x01, 0x6C, 0xB1, 0x18, 0x77, 0xCC,
+0x04, 0xD7, 0x4B, 0xF3, 0x4C, 0x99, 0x04, 0x97,
+0x56, 0xA2, 0x01, 0x72, 0x05, 0x61, 0x0D, 0x6E,
+0xA7, 0x67, 0xB1, 0x18, 0xFE, 0xE8, 0x00, 0x6C,
+0x4B, 0xF3, 0x4C, 0x99, 0x75, 0xA2, 0x18, 0x23,
+0x60, 0xF5, 0xD3, 0xA2, 0x60, 0xF5, 0xB2, 0xA2,
+0x00, 0x6B, 0x60, 0xF5, 0x71, 0xC2, 0x60, 0xF5,
+0x77, 0xC2, 0xB1, 0x18, 0xBE, 0xCD, 0x00, 0x6C,
+0x0B, 0x22, 0xB1, 0x18, 0xD7, 0xCC, 0x07, 0x6C,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x7F, 0x6C, 0x8C, 0xEB, 0xD2, 0x17,
+0xB1, 0x18, 0x3C, 0xCD, 0x90, 0x67, 0xF4, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0x4B, 0xF3,
+0x4C, 0x98, 0x74, 0xA2, 0x35, 0x2B, 0x79, 0xA2,
+0x02, 0x73, 0x28, 0x61, 0x41, 0xF3, 0x74, 0xA2,
+0xA8, 0x43, 0xFC, 0x4D, 0x02, 0x5D, 0x22, 0x60,
+0xC0, 0xF3, 0xAC, 0xAA, 0x01, 0x4D, 0xC0, 0xF3,
+0xAC, 0xCA, 0x41, 0xF3, 0x58, 0xA2, 0x15, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x48, 0x9A, 0x08, 0x4B, 0x68, 0x33,
+0x6D, 0xE2, 0x41, 0x9B, 0xE0, 0xF3, 0x08, 0x6B,
+0x43, 0xEC, 0x10, 0x61, 0x53, 0xE4, 0x7B, 0xEC,
+0x01, 0x2B, 0xE5, 0xE8, 0xB1, 0x18, 0x6E, 0xC6,
+0x12, 0xEC, 0x4B, 0xF3, 0x4C, 0x98, 0x01, 0x6B,
+0x41, 0xF3, 0x78, 0xC2, 0x05, 0x97, 0x04, 0x90,
+0x00, 0xEF, 0x03, 0x63, 0xFF, 0x4C, 0x4B, 0xE4,
+0x7B, 0xEA, 0x01, 0x2B, 0xE5, 0xE8, 0xEE, 0x17,
+0x02, 0x73, 0x13, 0x61, 0x41, 0xF3, 0x74, 0xA2,
+0xFF, 0x6D, 0xFB, 0x4B, 0xAC, 0xEB, 0x02, 0x5B,
+0xED, 0x60, 0x01, 0x6B, 0x41, 0xF3, 0x78, 0xC2,
+0xC0, 0xF3, 0x6C, 0xAA, 0x01, 0x4B, 0xC0, 0xF3,
+0x6C, 0xCA, 0xB1, 0x18, 0x2A, 0xCF, 0x00, 0x65,
+0xE1, 0x17, 0x03, 0x73, 0xDF, 0x61, 0x41, 0xF3,
+0x77, 0xA2, 0xE8, 0x2B, 0x41, 0xF3, 0x74, 0xA2,
+0x05, 0x73, 0x0C, 0x61, 0x01, 0x6B, 0x41, 0xF3,
+0x78, 0xC2, 0xC0, 0xF3, 0x6C, 0xAA, 0x01, 0x4B,
+0xC0, 0xF3, 0x6C, 0xCA, 0xB1, 0x18, 0xC7, 0xCC,
+0x00, 0x65, 0xCC, 0x17, 0x06, 0x73, 0xCA, 0x61,
+0x01, 0x6B, 0x41, 0xF3, 0x78, 0xC2, 0xC0, 0xF3,
+0x6C, 0xAA, 0x01, 0x4B, 0xC0, 0xF3, 0x6C, 0xCA,
+0x08, 0x6B, 0x41, 0xF3, 0x76, 0xC2, 0xBE, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0xB4, 0xA2, 0x6E, 0x45, 0x02, 0x5B, 0x08, 0x60,
+0x41, 0xF3, 0x78, 0xA2, 0x05, 0x2B, 0xC0, 0xF3,
+0x6E, 0xAA, 0x01, 0x4B, 0xC0, 0xF3, 0x6E, 0xCA,
+0x41, 0xF3, 0x74, 0xA2, 0x05, 0x73, 0x0B, 0x61,
+0x61, 0xF3, 0x5B, 0xA2, 0x0A, 0x22, 0x01, 0x75,
+0x08, 0x61, 0xB1, 0x18, 0xB0, 0xCD, 0x00, 0x65,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x06, 0x73,
+0xFB, 0x61, 0xB1, 0x18, 0x2A, 0xCF, 0x00, 0x65,
+0xF7, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x4B, 0xF3, 0x4C, 0x98,
+0x01, 0x6E, 0x81, 0xF3, 0xB0, 0xA2, 0x61, 0xF3,
+0x39, 0xA2, 0xCD, 0xED, 0x81, 0xF3, 0xB0, 0xC2,
+0xB4, 0xA2, 0x05, 0x25, 0xA0, 0xF0, 0x5B, 0xA2,
+0x01, 0x6D, 0xAC, 0xEA, 0x27, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB2, 0xF5,
+0x5C, 0x9A, 0x04, 0xD4, 0x00, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x91, 0x67, 0x00, 0x6C, 0x0D, 0x6E,
+0xB1, 0x18, 0xFE, 0xE8, 0xB1, 0x67, 0x4B, 0xF3,
+0x4C, 0x98, 0xA0, 0xF0, 0x94, 0xA2, 0x17, 0x2C,
+0xB1, 0x18, 0x2D, 0xCB, 0x00, 0x65, 0x4B, 0xF3,
+0x6C, 0x98, 0x59, 0xA3, 0x02, 0x72, 0x04, 0x61,
+0xA0, 0xF0, 0x59, 0xA3, 0x02, 0x72, 0x03, 0x60,
+0xB1, 0x18, 0xC2, 0xCC, 0x04, 0x94, 0xB1, 0x18,
+0xD0, 0xC9, 0x00, 0x65, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x99, 0xA2,
+0x02, 0x74, 0x30, 0x61, 0x81, 0xF3, 0x52, 0xA2,
+0x08, 0x6C, 0x8C, 0xEA, 0x0E, 0x22, 0x00, 0x6C,
+0x0D, 0x6E, 0xB1, 0x18, 0xFE, 0xE8, 0xB1, 0x67,
+0x4B, 0xF3, 0x6C, 0x98, 0x09, 0x6C, 0x8B, 0xEC,
+0x81, 0xF3, 0x52, 0xA3, 0x8C, 0xEA, 0x81, 0xF3,
+0x52, 0xC3, 0x4B, 0xF3, 0x4C, 0x98, 0x04, 0x6C,
+0x81, 0xF3, 0x72, 0xA2, 0x8C, 0xEB, 0x16, 0x23,
+0x60, 0xF5, 0xD3, 0xA2, 0x60, 0xF5, 0xB2, 0xA2,
+0x00, 0x6B, 0x60, 0xF5, 0x71, 0xC2, 0x60, 0xF5,
+0x77, 0xC2, 0xB1, 0x18, 0xBE, 0xCD, 0x00, 0x6C,
+0x09, 0x22, 0x4B, 0xF3, 0x6C, 0x98, 0x05, 0x6C,
+0x8B, 0xEC, 0x81, 0xF3, 0x52, 0xA3, 0x8C, 0xEA,
+0x81, 0xF3, 0x52, 0xC3, 0xB1, 0x18, 0xE7, 0xCE,
+0x04, 0x94, 0xC4, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x74, 0xA2, 0x03, 0x73, 0x19, 0x61, 0x01, 0x6B,
+0x41, 0xF3, 0x77, 0xC2, 0x41, 0xF3, 0x78, 0xA2,
+0x13, 0x23, 0xFD, 0x63, 0x05, 0x62, 0x41, 0xF3,
+0x54, 0xA2, 0x6B, 0x42, 0x02, 0x5B, 0x08, 0x61,
+0x08, 0x72, 0x06, 0x60, 0xB1, 0x18, 0xCC, 0xCC,
+0x40, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xB1, 0x18, 0x2A, 0xCF, 0x00, 0x65, 0xF9, 0x17,
+0x20, 0xE8, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x41, 0xF3, 0x54, 0xA2,
+0x09, 0x72, 0x06, 0x61, 0xB1, 0x18, 0x67, 0xCD,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0x04, 0xF0, 0x00, 0x6C, 0xB1, 0x18, 0xCC, 0xCC,
+0x00, 0x65, 0xF7, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x6C, 0x9A, 0x54, 0xA3, 0x0D, 0x22,
+0x41, 0xF3, 0x54, 0xA3, 0xFF, 0x6D, 0xFB, 0x4A,
+0xAC, 0xEA, 0x03, 0x5A, 0x0D, 0x60, 0x61, 0xF3,
+0x5B, 0xA3, 0x06, 0x22, 0xB1, 0x18, 0xB0, 0xCD,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xB1, 0x18, 0x2A, 0xCF, 0x00, 0x65, 0xF9, 0x17,
+0xB1, 0x18, 0xCC, 0xCC, 0x20, 0x6C, 0xF5, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0x4B, 0xF3,
+0x4C, 0x98, 0x74, 0xA2, 0x06, 0x2B, 0x79, 0xA2,
+0x11, 0x2B, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x03, 0x73, 0x0B, 0x61, 0x41, 0xF3,
+0x76, 0xA2, 0x08, 0x6D, 0xAE, 0xEB, 0x06, 0x2B,
+0x41, 0xF3, 0x76, 0xC2, 0xB1, 0x18, 0xC7, 0xCC,
+0x00, 0x65, 0xEF, 0x17, 0x41, 0xF3, 0x55, 0xA2,
+0x09, 0x2A, 0xB1, 0x18, 0xC2, 0xCC, 0x00, 0x65,
+0x4B, 0xF3, 0x4C, 0x98, 0x00, 0x6B, 0x41, 0xF3,
+0x75, 0xC2, 0xE3, 0x17, 0x05, 0x72, 0x04, 0x61,
+0xB1, 0x18, 0x67, 0xCD, 0x00, 0x65, 0xF4, 0x17,
+0x08, 0x72, 0x04, 0x61, 0xB1, 0x18, 0xC7, 0xCC,
+0x00, 0x65, 0xEE, 0x17, 0x0D, 0x72, 0x05, 0x61,
+0xA4, 0x67, 0xB1, 0x18, 0x77, 0xCC, 0x0D, 0x6C,
+0xE7, 0x17, 0x02, 0xF0, 0x00, 0x6C, 0xB1, 0x18,
+0xCC, 0xCC, 0x00, 0x65, 0xE1, 0x17, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x74, 0xA2, 0x02, 0x2B,
+0x79, 0xA2, 0x2A, 0x23, 0x81, 0xF3, 0x60, 0xA2,
+0x02, 0x6D, 0x6C, 0xED, 0x0C, 0x25, 0xC0, 0xF3,
+0xC2, 0xAA, 0x81, 0xF3, 0xA8, 0xAA, 0xCE, 0xED,
+0x1B, 0x2D, 0x01, 0x6D, 0xAD, 0xEB, 0x81, 0xF3,
+0x60, 0xC2, 0x81, 0xF3, 0xC8, 0xCA, 0x41, 0xF3,
+0x74, 0xA2, 0x06, 0x73, 0x15, 0x61, 0x81, 0xF3,
+0x70, 0x82, 0x00, 0x53, 0x11, 0x61, 0xFD, 0x63,
+0x05, 0x62, 0x61, 0xF3, 0x7B, 0xA2, 0x01, 0x4B,
+0x61, 0xF3, 0x7B, 0xC2, 0xB1, 0x18, 0x0C, 0xD0,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0x02, 0x6D, 0xAB, 0xED, 0xAC, 0xEB, 0xE3, 0x17,
+0x20, 0xE8, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xB5, 0xA2, 0xD6, 0xA2, 0x14, 0xA2, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x57, 0xA2, 0x20, 0x31,
+0xA0, 0x35, 0xC0, 0x36, 0x4B, 0xF3, 0xEC, 0x99,
+0xAD, 0xE8, 0xC0, 0x36, 0x0D, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x61, 0xF3, 0x79, 0xA7,
+0x35, 0x22, 0x54, 0xA7, 0x0F, 0x2A, 0x59, 0xA7,
+0x0D, 0x2A, 0x14, 0x5C, 0x2F, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xE9, 0xF0,
+0x54, 0x9A, 0x01, 0x6D, 0x46, 0xEC, 0x4F, 0xEA,
+0xAC, 0xEA, 0x24, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF3, 0x50, 0x9A,
+0x04, 0x67, 0x00, 0x6D, 0x40, 0xEA, 0x83, 0x67,
+0xE2, 0x67, 0x4B, 0xF3, 0x4C, 0x99, 0xC7, 0x67,
+0xB0, 0x67, 0x81, 0xF3, 0x11, 0xC2, 0xB1, 0x18,
+0x83, 0xCA, 0x01, 0x6C, 0x1A, 0x58, 0x5E, 0x60,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x08, 0x32, 0x88, 0xF0, 0x10, 0x4B, 0x69, 0xE2,
+0x40, 0x9A, 0x00, 0xEA, 0x00, 0x65, 0xB1, 0x18,
+0x9F, 0xCF, 0x87, 0x67, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0xB1, 0x18,
+0x1C, 0xCF, 0x87, 0x67, 0xF7, 0x17, 0xB1, 0x18,
+0x8A, 0xCF, 0x87, 0x67, 0xF3, 0x17, 0xB1, 0x18,
+0xA1, 0xCD, 0x87, 0x67, 0xEF, 0x17, 0x4B, 0xF3,
+0x6C, 0x99, 0x03, 0x70, 0x81, 0xF3, 0x40, 0xA3,
+0x08, 0x61, 0x02, 0x6C, 0x8D, 0xEA, 0x81, 0xF3,
+0x40, 0xC3, 0xB1, 0x18, 0x2E, 0xD0, 0x87, 0x67,
+0xE1, 0x17, 0x03, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA,
+0xF6, 0x17, 0xB1, 0x18, 0xFB, 0xCF, 0x87, 0x67,
+0xD9, 0x17, 0xB1, 0x18, 0xDD, 0xCF, 0x87, 0x67,
+0xD5, 0x17, 0xB1, 0x18, 0x87, 0xCD, 0x87, 0x67,
+0xD1, 0x17, 0xB1, 0x18, 0x53, 0xCD, 0x87, 0x67,
+0xCD, 0x17, 0xB1, 0x18, 0x4E, 0xCF, 0x87, 0x67,
+0xC9, 0x17, 0xB1, 0x18, 0xEF, 0xCF, 0x87, 0x67,
+0xC5, 0x17, 0xB1, 0x18, 0xF4, 0xCC, 0x87, 0x67,
+0xC1, 0x17, 0xB1, 0x18, 0xF1, 0xCD, 0x87, 0x67,
+0xBD, 0x17, 0xB1, 0x18, 0x04, 0xCD, 0x87, 0x67,
+0xB9, 0x17, 0xA7, 0x67, 0xB1, 0x18, 0x6E, 0xCE,
+0x90, 0x67, 0xB4, 0x17, 0xB1, 0x18, 0xE7, 0xCE,
+0x87, 0x67, 0xB0, 0x17, 0x01, 0xF0, 0x00, 0x6C,
+0xB1, 0x18, 0xCC, 0xCC, 0x00, 0x65, 0xAA, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x41, 0xF3, 0x54, 0xA2,
+0x09, 0x72, 0x1B, 0x61, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x52, 0xF6, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x6C,
+0x0A, 0x6C, 0x09, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x32, 0xF6, 0x5C, 0x9A,
+0x40, 0xEA, 0x00, 0x6C, 0x0B, 0x6C, 0xB1, 0x18,
+0x49, 0xD0, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x0A, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x01, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x19, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x02, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x03, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x04, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x05, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x06, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x07, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x0E, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x17, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x49, 0xD0,
+0x18, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x4C, 0x9A, 0x04, 0x6B, 0x81, 0xF3, 0x40, 0xA2,
+0x6C, 0xEA, 0x1D, 0x22, 0xB1, 0x18, 0xD7, 0xCC,
+0x0F, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x08, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x1B, 0xF2, 0x10, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xFF, 0x6D, 0x01, 0x4D, 0x1B, 0xF2,
+0x10, 0x6C, 0x40, 0xE8, 0x4D, 0xED, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0xB1, 0x18,
+0x49, 0xD0, 0x08, 0x6C, 0xF8, 0x17, 0x00, 0x65,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69,
+0x60, 0x33, 0x20, 0x31, 0x60, 0x33, 0xE9, 0xF0,
+0x78, 0x9B, 0x20, 0x31, 0x72, 0xF4, 0x58, 0x99,
+0xF7, 0xF0, 0x01, 0x68, 0x01, 0x6E, 0xA3, 0x67,
+0xE1, 0xF3, 0x08, 0x6C, 0x00, 0x30, 0x08, 0xD3,
+0x40, 0xEA, 0x00, 0x30, 0x4B, 0xF3, 0x4C, 0xD8,
+0x1A, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xE9, 0xF7, 0x00, 0x4A, 0x05, 0xD2, 0xA1, 0xF2,
+0x1C, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x93, 0xF6,
+0xE0, 0x9B, 0xE1, 0xF3, 0x08, 0x6E, 0x00, 0x6D,
+0x82, 0x67, 0x40, 0xEF, 0x09, 0xD3, 0x72, 0xF4,
+0x58, 0x99, 0x08, 0x95, 0xEC, 0x6C, 0x40, 0xEA,
+0x01, 0x6E, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x4B, 0xF3, 0x48, 0xDC, 0x09, 0x93,
+0x0D, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xE9, 0xF7, 0x00, 0x4A, 0x05, 0xD2, 0xC1, 0xF2,
+0x08, 0x6A, 0xCB, 0x17, 0x93, 0xF6, 0x60, 0x9B,
+0xEC, 0x6E, 0x00, 0x6D, 0x40, 0xEB, 0x82, 0x67,
+0x4B, 0xF3, 0x4C, 0x98, 0x07, 0x6B, 0x08, 0x6C,
+0x80, 0xF1, 0x60, 0xC2, 0x41, 0xF0, 0x68, 0xC2,
+0xE0, 0xF3, 0x60, 0xC2, 0xE0, 0xF3, 0x68, 0xC2,
+0x7C, 0xC2, 0x68, 0xC2, 0x20, 0xF4, 0x64, 0xC2,
+0xC0, 0xF4, 0x68, 0xC2, 0x00, 0xF5, 0x7C, 0xC2,
+0x20, 0xF5, 0x6C, 0xC2, 0x20, 0xF5, 0x7C, 0xC2,
+0x40, 0xF5, 0x74, 0xC2, 0x01, 0x6B, 0x60, 0xF5,
+0x60, 0xC2, 0x81, 0xF3, 0x70, 0xA2, 0x40, 0xF1,
+0x94, 0xC2, 0x02, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB,
+0x81, 0xF3, 0x70, 0xC2, 0x7F, 0x6B, 0x61, 0xF3,
+0x7C, 0xC2, 0x61, 0xF3, 0x7D, 0xC2, 0x61, 0xF3,
+0x7E, 0xC2, 0xB1, 0x18, 0x6D, 0xCB, 0x00, 0x65,
+0x9D, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xB1, 0x18, 0x49, 0xD0, 0x00, 0x6C, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x61, 0xF3, 0x79, 0xA2, 0x6E, 0xEC, 0x13, 0x2C,
+0x75, 0xA2, 0xF4, 0xA2, 0x60, 0x33, 0xED, 0xEB,
+0x0E, 0x23, 0xFD, 0x63, 0x05, 0x62, 0x60, 0xF5,
+0x88, 0xC2, 0x60, 0xF5, 0xA9, 0xC2, 0x60, 0xF5,
+0xCC, 0xDA, 0xB1, 0x18, 0x49, 0xD0, 0x0C, 0x6C,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x61, 0xF3, 0x79, 0xA2, 0x6E, 0xEC, 0x14, 0x2C,
+0x75, 0xA2, 0x94, 0xA2, 0x60, 0x33, 0x8D, 0xEB,
+0x0F, 0x23, 0xFD, 0x63, 0x05, 0x62, 0x01, 0x6B,
+0x60, 0xF5, 0x68, 0xC2, 0x60, 0xF5, 0xA9, 0xC2,
+0x60, 0xF5, 0xCC, 0xDA, 0xB1, 0x18, 0x49, 0xD0,
+0x0D, 0x6C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0x20, 0xE8, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0x6C, 0x9A, 0x01, 0x68,
+0x8B, 0x35, 0x81, 0xF3, 0xF0, 0xA3, 0x9B, 0x34,
+0x0C, 0xED, 0xE6, 0x32, 0x0C, 0xEA, 0xC4, 0x67,
+0xAE, 0xEA, 0x0C, 0xEE, 0x2C, 0x22, 0x04, 0x6A,
+0xED, 0xEA, 0x03, 0x6F, 0xEB, 0xEF, 0xA4, 0x35,
+0xEC, 0xEA, 0xAD, 0xEA, 0x81, 0xF3, 0x50, 0xC3,
+0x81, 0xF3, 0x50, 0xA3, 0x01, 0x6F, 0x4E, 0x35,
+0xEC, 0xED, 0xAE, 0xEE, 0x20, 0x26, 0x10, 0x6D,
+0xAD, 0xEA, 0xEC, 0xEC, 0xE7, 0x4D, 0x8C, 0x34,
+0xAC, 0xEA, 0x8D, 0xEA, 0x81, 0xF3, 0x50, 0xC3,
+0x81, 0xF3, 0x50, 0xA3, 0x14, 0x6C, 0x4C, 0xEC,
+0x16, 0x2C, 0x0A, 0x6C, 0x1C, 0x65, 0x98, 0x67,
+0x4C, 0xEC, 0x08, 0x74, 0x04, 0x61, 0xD7, 0x4C,
+0x8C, 0xEA, 0x81, 0xF3, 0x50, 0xC3, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x05, 0x6A,
+0x4B, 0xEA, 0xEC, 0xEA, 0xD7, 0x17, 0x11, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEA, 0xE3, 0x17, 0xB1, 0x18,
+0x49, 0xD0, 0x0E, 0x6C, 0xF0, 0x17, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x74, 0xA2, 0x0D, 0x2B,
+0x59, 0xA2, 0x02, 0x72, 0x0A, 0x61, 0x13, 0x74,
+0x08, 0x60, 0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18,
+0x49, 0xD0, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x14, 0x74, 0x08, 0x61, 0xFD, 0x63, 0x05, 0x62,
+0xB1, 0x18, 0x49, 0xD0, 0x00, 0x65, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8, 0x00, 0x65,
+0x0F, 0x6A, 0x4C, 0xEC, 0x10, 0x24, 0xFD, 0x63,
+0x05, 0x62, 0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0xBD, 0x6B, 0x02, 0x6C, 0x10, 0x6B, 0x6C, 0xEA,
+0x03, 0x22, 0xB1, 0x18, 0x49, 0xD0, 0x0F, 0x6C,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x6C, 0x9A, 0x81, 0xF3, 0x53, 0xA3,
+0x8E, 0xEA, 0x05, 0x22, 0x81, 0xF3, 0x93, 0xC3,
+0xB1, 0x18, 0x49, 0xD0, 0x16, 0x6C, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xFD, 0x63, 0x05, 0x62,
+0xB1, 0x18, 0x49, 0xD0, 0x09, 0x6C, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x4C, 0x9A, 0x00, 0x6F, 0x80, 0xF5, 0x60, 0x42,
+0xC2, 0xA3, 0x8E, 0xEE, 0x19, 0x2E, 0xC1, 0xA3,
+0xAE, 0xEE, 0x16, 0x2E, 0xC4, 0xA3, 0x14, 0x2E,
+0xC0, 0xA3, 0x12, 0x26, 0x07, 0xA3, 0x09, 0x6E,
+0xCB, 0xEE, 0x0C, 0xEE, 0x01, 0x76, 0x14, 0x60,
+0x02, 0x76, 0x0C, 0x61, 0x81, 0xF3, 0x60, 0xA2,
+0x04, 0x6C, 0x8D, 0xEB, 0x81, 0xF3, 0x60, 0xC2,
+0xB1, 0x18, 0xD7, 0xCC, 0x0B, 0x6C, 0x32, 0x10,
+0x05, 0x77, 0x30, 0x60, 0x01, 0x4F, 0xFF, 0x6E,
+0xCC, 0xEF, 0x06, 0x77, 0x0C, 0x4B, 0xDC, 0x61,
+0xA0, 0xF3, 0x7C, 0xAA, 0x81, 0xF3, 0x92, 0xA2,
+0x10, 0x6D, 0x01, 0x4B, 0xAC, 0xEC, 0xFF, 0xF7,
+0x1F, 0x6E, 0xFF, 0x6D, 0xCC, 0xEB, 0xAC, 0xEC,
+0xA0, 0xF3, 0x7C, 0xCA, 0x1B, 0x2C, 0x94, 0xA2,
+0x11, 0x2C, 0x99, 0xA2, 0x17, 0x24, 0x01, 0x74,
+0x0D, 0x61, 0x0A, 0x6C, 0x9B, 0xEB, 0x01, 0x2C,
+0xE5, 0xE8, 0x10, 0xEC, 0xCC, 0xEC, 0x06, 0x24,
+0x81, 0xF3, 0x60, 0xA2, 0x01, 0x6C, 0x8C, 0xEB,
+0xAC, 0xEB, 0x08, 0x23, 0x41, 0xF3, 0x74, 0xA2,
+0x09, 0x6A, 0x4B, 0xEA, 0x6C, 0xEA, 0xCC, 0x22,
+0x05, 0x72, 0xCA, 0x60, 0x05, 0x97, 0x04, 0x90,
+0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x61, 0xF3, 0x79, 0xA2, 0x6E, 0xEC, 0x19, 0x2C,
+0xA0, 0xF3, 0x7E, 0xAA, 0x01, 0x4B, 0xA0, 0xF3,
+0x7E, 0xCA, 0x41, 0xF3, 0x74, 0xA2, 0x8B, 0x43,
+0x02, 0x5C, 0x02, 0x61, 0x08, 0x73, 0x0D, 0x61,
+0xFD, 0x63, 0x05, 0x62, 0xC0, 0xF3, 0x62, 0xAA,
+0x01, 0x4B, 0xC0, 0xF3, 0x62, 0xCA, 0xB1, 0x18,
+0x49, 0xD0, 0x15, 0x6C, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x4C, 0xF2, 0x58, 0x98, 0x38, 0x9A, 0x56, 0x9A,
+0x91, 0x67, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF2, 0xF2, 0x44, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x49, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x4C, 0x9A, 0x10, 0x6B, 0x81, 0xF3, 0x52, 0xA2,
+0x6C, 0xEA, 0x3E, 0x2A, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0xBD, 0x6B, 0x03, 0x6C, 0x13, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF2, 0xF2, 0x40, 0x9A, 0x40, 0xEA, 0x91, 0x67,
+0x2C, 0x22, 0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0xE5, 0x6B, 0x0C, 0x6C, 0x4C, 0xF2, 0x78, 0x98,
+0x4D, 0x9B, 0x01, 0x4A, 0x4D, 0xDB, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF2, 0xF2,
+0x50, 0x9A, 0x01, 0x6F, 0xEB, 0xEF, 0x08, 0x6E,
+0x04, 0x05, 0x40, 0xEA, 0x91, 0x67, 0x08, 0x72,
+0x1C, 0x61, 0x00, 0x6E, 0x04, 0x05, 0xD1, 0x18,
+0xE5, 0x6B, 0x0D, 0x6C, 0x00, 0x6E, 0x05, 0x05,
+0xD1, 0x18, 0xE5, 0x6B, 0x0E, 0x6C, 0x00, 0x6E,
+0x00, 0x6D, 0xD1, 0x18, 0xE5, 0x6B, 0x0B, 0x6C,
+0x4C, 0xF2, 0x78, 0x98, 0x4C, 0x9B, 0x01, 0x4A,
+0x4C, 0xDB, 0xB1, 0x18, 0x8A, 0xC5, 0x06, 0x94,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x4C, 0xF2, 0x78, 0x98, 0x4D, 0x9B,
+0x01, 0x4A, 0x4D, 0xDB, 0xF2, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4C, 0xF2, 0x78, 0x9A, 0x25, 0x67, 0x04, 0x67,
+0x4D, 0xAB, 0x00, 0x6F, 0x04, 0x06, 0x01, 0x4A,
+0x4D, 0xCB, 0x02, 0x6A, 0x42, 0xC4, 0x12, 0x6A,
+0x41, 0xC4, 0xA4, 0xA4, 0xF7, 0xF0, 0x01, 0x6A,
+0x63, 0xA4, 0x40, 0x32, 0x40, 0x32, 0xA0, 0x35,
+0xD3, 0xF5, 0x48, 0x9A, 0x6D, 0xED, 0xF8, 0x4D,
+0x40, 0xEA, 0x00, 0x6C, 0x13, 0x2A, 0x04, 0x94,
+0x11, 0x24, 0xC4, 0xA0, 0xF7, 0xF0, 0x01, 0x6A,
+0x63, 0xA0, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xC0, 0x36, 0x6D, 0xEE, 0xB1, 0x67,
+0x40, 0xEA, 0xF8, 0x4E, 0x04, 0x95, 0xD1, 0x18,
+0xA8, 0x19, 0x90, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x74, 0xA2, 0x80, 0x34, 0x0C, 0x2B,
+0x79, 0xA2, 0x0A, 0x2B, 0xE0, 0xF3, 0x61, 0xC2,
+0x4C, 0xF2, 0x78, 0x9C, 0x03, 0x6A, 0x4B, 0xEA,
+0x8B, 0x9B, 0x8C, 0xEA, 0x20, 0xE8, 0x4B, 0xDB,
+0xC0, 0xF3, 0x7C, 0x9A, 0x0A, 0x2B, 0xE0, 0xF3,
+0x61, 0xA2, 0x07, 0x2B, 0x81, 0xF3, 0x7C, 0xDA,
+0xE0, 0xF3, 0x64, 0xDA, 0x01, 0x6B, 0xE0, 0xF3,
+0x61, 0xC2, 0x4C, 0xF2, 0x78, 0x9C, 0x02, 0x6A,
+0x8B, 0x9B, 0x8D, 0xEA, 0xEB, 0x17, 0x00, 0x65,
+0x00, 0x6D, 0x00, 0x6A, 0x01, 0x6B, 0x64, 0xED,
+0x8C, 0xEB, 0x07, 0x6E, 0x67, 0xED, 0xBB, 0xE6,
+0x64, 0xEE, 0x6D, 0xEA, 0x00, 0xF6, 0x40, 0x32,
+0x01, 0x4D, 0x00, 0xF6, 0x43, 0x32, 0xFF, 0x6B,
+0x08, 0x75, 0x6C, 0xEA, 0xEF, 0x61, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x04, 0x67, 0x40, 0xEA, 0x25, 0x67, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xE9, 0xF0, 0xBC, 0x9A, 0x02, 0x6E, 0xFB, 0xF2,
+0x10, 0x6C, 0xD1, 0x18, 0x9D, 0x6A, 0x0D, 0xED,
+0xE0, 0xF3, 0x09, 0x68, 0xFB, 0xF2, 0x13, 0x6C,
+0xD1, 0x18, 0x73, 0x6A, 0x00, 0x6D, 0x20, 0x6B,
+0x6C, 0xEA, 0x14, 0x22, 0xFB, 0xF2, 0x18, 0x6C,
+0xD1, 0x18, 0x73, 0x6A, 0x02, 0x6D, 0x40, 0xD9,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x04, 0x94,
+0x01, 0x6A, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFF, 0x48, 0xF9, 0x20,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0x0A, 0x6C,
+0xD9, 0x17, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x05, 0x05, 0x40, 0x32, 0x4C, 0xF2,
+0x38, 0x9A, 0xB1, 0x18, 0x73, 0xD2, 0x38, 0x6C,
+0x06, 0x05, 0xB1, 0x18, 0x73, 0xD2, 0x54, 0x6C,
+0x05, 0x92, 0x01, 0x68, 0x9D, 0x67, 0x42, 0x32,
+0x5A, 0x33, 0x0C, 0xEB, 0x70, 0xC4, 0x06, 0x93,
+0xBD, 0x67, 0x52, 0x32, 0x6E, 0x34, 0x0C, 0xEC,
+0x91, 0xC5, 0x0C, 0xEA, 0x9D, 0x67, 0x6A, 0x33,
+0x52, 0xC4, 0x0C, 0xEB, 0x5D, 0x67, 0x73, 0xC2,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x73, 0xF6, 0x5C, 0x9B, 0x04, 0x05, 0x84, 0x41,
+0x04, 0x6E, 0x40, 0xEA, 0x08, 0xD3, 0x05, 0x94,
+0x7D, 0x67, 0x04, 0x6E, 0x80, 0xF7, 0x82, 0x32,
+0x0C, 0xEA, 0x50, 0xC3, 0x06, 0x92, 0x00, 0xF7,
+0x82, 0x34, 0x0C, 0xEC, 0x56, 0x35, 0x0C, 0xED,
+0xB1, 0xC3, 0x92, 0xC3, 0x08, 0x93, 0x52, 0x32,
+0x4C, 0xE8, 0x5D, 0x67, 0x13, 0xC2, 0x73, 0xF6,
+0x5C, 0x9B, 0x87, 0x41, 0x04, 0x05, 0x40, 0xEA,
+0x01, 0x4C, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x0C, 0x9A,
+0x42, 0xA4, 0x00, 0x6B, 0x21, 0xA4, 0x2A, 0x72,
+0x08, 0xD3, 0x07, 0xD3, 0x06, 0xD3, 0x67, 0x60,
+0x2B, 0x5A, 0x30, 0x60, 0x1E, 0x72, 0x6E, 0x60,
+0x1F, 0x5A, 0x09, 0x60, 0x48, 0x22, 0x11, 0x72,
+0xE0, 0xF0, 0x01, 0x60, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0x1F, 0x72,
+0x80, 0xF0, 0x09, 0x60, 0x20, 0x72, 0xF6, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0xA4, 0x44, 0x04, 0x6E,
+0x40, 0xEA, 0x08, 0x04, 0x20, 0xF5, 0x5D, 0xA0,
+0x40, 0xF5, 0x08, 0x48, 0x4D, 0xE9, 0xFF, 0xF7,
+0x35, 0xC0, 0x00, 0x69, 0x08, 0x02, 0x29, 0xE2,
+0x80, 0xA2, 0x01, 0x49, 0xB1, 0x18, 0x68, 0xD2,
+0x01, 0x48, 0x04, 0x71, 0xFF, 0xF7, 0x5F, 0xC0,
+0xF5, 0x61, 0xD8, 0x17, 0x2E, 0x72, 0x80, 0xF0,
+0x0F, 0x60, 0x2F, 0x5A, 0x08, 0x60, 0x2B, 0x72,
+0x23, 0x60, 0x2D, 0x72, 0xCF, 0x61, 0x44, 0xA4,
+0x20, 0xF5, 0x4D, 0xC0, 0xCB, 0x17, 0x30, 0x72,
+0x80, 0xF0, 0x16, 0x60, 0x31, 0x72, 0xC6, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA4, 0x44, 0x73, 0xF6, 0x5C, 0x9A, 0x04, 0x6E,
+0x50, 0xF5, 0x8C, 0x40, 0x0A, 0x10, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xA4, 0x44, 0x04, 0x6E, 0x20, 0xF5,
+0x84, 0x40, 0x40, 0xEA, 0x00, 0x65, 0xAE, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA4, 0x44, 0x73, 0xF6, 0x5C, 0x9A, 0x04, 0x6E,
+0x20, 0xF5, 0x80, 0x40, 0xF2, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xA4, 0x44,
+0x73, 0xF6, 0x5C, 0x9A, 0x04, 0x6E, 0x20, 0xF5,
+0x88, 0x40, 0xE7, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xA4, 0x44, 0x04, 0x6E, 0x40, 0xEA, 0x08, 0x04,
+0x20, 0xF5, 0x5D, 0xA0, 0x40, 0xF5, 0x00, 0x48,
+0x2D, 0xEA, 0xFF, 0xF7, 0x5D, 0xC0, 0x00, 0x6A,
+0x0A, 0xD2, 0x0A, 0x93, 0x08, 0x02, 0x10, 0x71,
+0x69, 0xE2, 0x80, 0xA2, 0x0B, 0x61, 0xB1, 0x18,
+0x68, 0xD2, 0x00, 0x65, 0x40, 0xC0, 0x0A, 0x92,
+0x01, 0x48, 0x01, 0x4A, 0x04, 0x72, 0x0A, 0xD2,
+0xF0, 0x61, 0x74, 0x17, 0xB1, 0x18, 0x68, 0xD2,
+0x00, 0x65, 0x4C, 0xC0, 0xF4, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xA4, 0x44, 0x04, 0x6E, 0x40, 0xEA,
+0x08, 0x04, 0x20, 0xF5, 0x5D, 0xA0, 0x40, 0xF5,
+0x04, 0x48, 0x2D, 0xEA, 0xFF, 0xF7, 0x59, 0xC0,
+0x00, 0x6A, 0x0A, 0xD2, 0x0A, 0x93, 0x08, 0x02,
+0x10, 0x71, 0x69, 0xE2, 0x80, 0xA2, 0x0B, 0x61,
+0xB1, 0x18, 0x68, 0xD2, 0x00, 0x65, 0x40, 0xC0,
+0x0A, 0x92, 0x01, 0x48, 0x01, 0x4A, 0x04, 0x72,
+0x0A, 0xD2, 0xF0, 0x61, 0x4B, 0x17, 0xB1, 0x18,
+0x68, 0xD2, 0x00, 0x65, 0x4C, 0xC0, 0xF4, 0x17,
+0x0F, 0x6A, 0x4C, 0xE9, 0x03, 0x59, 0x5F, 0xF7,
+0x01, 0x60, 0x45, 0xA4, 0x64, 0xA4, 0x28, 0x31,
+0x40, 0x32, 0x69, 0xE2, 0x21, 0xE0, 0x20, 0xF5,
+0x50, 0xC8, 0x47, 0xA4, 0x66, 0xA4, 0x40, 0x32,
+0x69, 0xE2, 0x20, 0xF5, 0x52, 0xC8, 0x32, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0xA4, 0x44, 0x04, 0x6E,
+0x40, 0xEA, 0x07, 0x04, 0x5D, 0x67, 0x5C, 0xA2,
+0x40, 0xF5, 0x56, 0xC8, 0x07, 0x92, 0x42, 0x32,
+0x40, 0xF5, 0x58, 0xD8, 0x1F, 0x17, 0xA5, 0xA4,
+0x44, 0xA4, 0x66, 0xA4, 0xA0, 0x35, 0x4D, 0xED,
+0x47, 0xA4, 0x60, 0x33, 0x60, 0x33, 0x32, 0x31,
+0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0x0F, 0x71,
+0xAD, 0xEA, 0x78, 0x67, 0x06, 0xD2, 0x0C, 0x2B,
+0x9D, 0x67, 0x03, 0x6A, 0x50, 0xC4, 0x0C, 0x6A,
+0x53, 0xC4, 0x06, 0x05, 0x5D, 0x67, 0x04, 0x04,
+0xB1, 0x18, 0x34, 0xD2, 0x74, 0xC2, 0x02, 0x17,
+0x20, 0xF1, 0x12, 0x49, 0x28, 0x31, 0x21, 0xE0,
+0x41, 0xD8, 0xFC, 0x16, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x4B, 0xF3, 0x4C, 0x99,
+0x00, 0x68, 0x04, 0xD2, 0x00, 0x6A, 0x04, 0x93,
+0x20, 0xF4, 0x65, 0xA3, 0x63, 0xE8, 0x05, 0x61,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x04, 0x93, 0x0C, 0x34, 0x91, 0xE3,
+0x20, 0xF4, 0xA8, 0xAC, 0x20, 0xF4, 0x6A, 0xAC,
+0x20, 0xF4, 0x8C, 0x9C, 0x01, 0x75, 0x24, 0x60,
+0x08, 0x25, 0x02, 0x75, 0x2C, 0x60, 0x03, 0x5D,
+0x16, 0x61, 0x01, 0x48, 0xFF, 0x6B, 0x6C, 0xE8,
+0xE2, 0x17, 0x1B, 0xF2, 0x00, 0x6A, 0x4B, 0xEA,
+0x49, 0xE4, 0x00, 0xF2, 0x00, 0x5A, 0x00, 0x6A,
+0x0A, 0x60, 0x01, 0x73, 0x00, 0x6D, 0x04, 0x60,
+0x02, 0x73, 0x01, 0x6D, 0x01, 0x60, 0x02, 0x6D,
+0xD1, 0x18, 0x73, 0x6A, 0x00, 0x65, 0x4B, 0xF3,
+0x6C, 0x99, 0x83, 0x67, 0x30, 0xF1, 0x62, 0x40,
+0x68, 0x33, 0x6D, 0xE4, 0x41, 0xDB, 0xE1, 0x17,
+0x01, 0x73, 0x00, 0x6D, 0x04, 0x60, 0x02, 0x73,
+0x01, 0x6D, 0x01, 0x60, 0x02, 0x6D, 0xD1, 0x18,
+0x14, 0x6A, 0x00, 0x65, 0xEC, 0x17, 0x03, 0x5B,
+0x43, 0x67, 0xFF, 0xF7, 0x1F, 0x6B, 0x01, 0x61,
+0x02, 0x6A, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0x6C, 0xEA, 0xA0, 0x35, 0x09, 0xF1, 0xC0, 0x9D,
+0xFF, 0x4A, 0x6C, 0xEA, 0xA4, 0x67, 0xD1, 0x18,
+0xF2, 0x6C, 0x82, 0x67, 0xD8, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x0E, 0x5C, 0x56, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x64, 0x67, 0x08, 0xF1,
+0x1C, 0x4A, 0x88, 0x34, 0x49, 0xE4, 0x40, 0x9A,
+0x00, 0xEA, 0x25, 0x67, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4C, 0xF2, 0xB8, 0x9A,
+0x50, 0x68, 0x3C, 0x2D, 0x41, 0x10, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0xF7, 0xF0,
+0x01, 0x6A, 0x4B, 0xF3, 0x8C, 0x98, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A, 0xA7, 0x44,
+0x0D, 0x4D, 0x08, 0x6E, 0x0C, 0x4C, 0x40, 0xEA,
+0x04, 0xD3, 0x4B, 0xF3, 0xAC, 0x98, 0x04, 0x93,
+0x0C, 0x68, 0x08, 0x4D, 0xE6, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0xAC, 0x9A, 0x98, 0x68, 0x1C, 0x4D, 0xDD, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0xAC, 0x9A, 0x60, 0xF2, 0x00, 0x68,
+0x80, 0xF1, 0x00, 0x4D, 0xD2, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0xAC, 0x9A, 0x00, 0xF3, 0x0C, 0x68, 0x20, 0xF3,
+0x56, 0xAD, 0x41, 0xF0, 0x08, 0x4D, 0x41, 0xC5,
+0x42, 0x32, 0x42, 0xC5, 0xC0, 0xA9, 0x43, 0x46,
+0x09, 0xE2, 0x00, 0xF7, 0x08, 0x52, 0x62, 0x61,
+0x00, 0xF2, 0x00, 0x6C, 0xB1, 0x18, 0xCC, 0xCC,
+0x00, 0x65, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0xAC, 0x9A,
+0xE0, 0xF3, 0x00, 0x4D, 0x08, 0x68, 0xA9, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0xAC, 0x9A, 0x3C, 0x68, 0xE0, 0xF3,
+0x08, 0x4D, 0x9F, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0xAC, 0x9A,
+0x54, 0x68, 0xC0, 0xF4, 0x08, 0x4D, 0x95, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0xAC, 0x9A, 0x2C, 0x68, 0x40, 0xF1,
+0x14, 0x4D, 0x8B, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0xAC, 0x9A,
+0x00, 0xF5, 0x1C, 0x4D, 0x10, 0x68, 0x81, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0xAC, 0x9A, 0x20, 0xF5, 0x0C, 0x4D,
+0xF5, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0xAC, 0x9A, 0x18, 0x68,
+0x20, 0xF5, 0x1C, 0x4D, 0x6E, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0xAC, 0x9A, 0x0C, 0x68, 0x40, 0xF5, 0x14, 0x4D,
+0x64, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4B, 0xF3, 0xAC, 0x9A, 0x60, 0xF5,
+0x00, 0x4D, 0xB0, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x10, 0x4A,
+0x59, 0xE6, 0x60, 0xC6, 0x60, 0xA9, 0x02, 0x34,
+0xD0, 0x67, 0x4D, 0xE3, 0x01, 0xC3, 0x60, 0xA9,
+0x4D, 0xE3, 0x82, 0xC3, 0xF7, 0xF0, 0x01, 0x6B,
+0x80, 0xA9, 0x60, 0x33, 0x60, 0x33, 0x73, 0xF6,
+0x7C, 0x9B, 0x03, 0x4C, 0x40, 0xEB, 0x91, 0xE2,
+0x40, 0xA9, 0x03, 0x4A, 0x41, 0xE0, 0x00, 0xC9,
+0x84, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x7D, 0x67, 0x4C, 0xF2,
+0x38, 0x9A, 0x00, 0x6A, 0x4B, 0xCB, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x4B, 0xF3,
+0x4C, 0x9B, 0x10, 0x6D, 0x06, 0xD3, 0x81, 0xF3,
+0x52, 0xA2, 0xAC, 0xEA, 0x24, 0x2A, 0x4C, 0xA9,
+0x04, 0x74, 0x0B, 0x99, 0x01, 0x4A, 0x4C, 0xC9,
+0x23, 0x61, 0x00, 0xF0, 0x16, 0x05, 0x08, 0x6C,
+0xB1, 0x18, 0x80, 0xD3, 0x00, 0x65, 0x00, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0x5D, 0x67, 0x4B, 0xAA,
+0xFF, 0xF7, 0x1F, 0x6B, 0xF7, 0xF0, 0x01, 0x6D,
+0x08, 0x4A, 0x6C, 0xEA, 0xA0, 0x35, 0x7D, 0x67,
+0x9D, 0x67, 0x53, 0xC3, 0xA0, 0x35, 0x42, 0x33,
+0x74, 0xC4, 0x4F, 0xC9, 0x4B, 0xF3, 0x10, 0x4D,
+0xB1, 0x18, 0x34, 0xD2, 0x04, 0x04, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0x0A, 0x20, 0x00, 0xF0, 0x16, 0x05, 0xB1, 0x18,
+0x80, 0xD3, 0x00, 0x6C, 0x00, 0xF0, 0x16, 0x05,
+0xB1, 0x18, 0x80, 0xD3, 0x01, 0x6C, 0x02, 0x6A,
+0x0C, 0xEA, 0x17, 0x22, 0x00, 0xF0, 0x16, 0x05,
+0xB1, 0x18, 0x80, 0xD3, 0x03, 0x6C, 0x00, 0xF0,
+0x16, 0x05, 0xB1, 0x18, 0x80, 0xD3, 0x05, 0x6C,
+0x06, 0x92, 0x0F, 0x6B, 0x4B, 0xF3, 0x4C, 0x9A,
+0xA0, 0xF7, 0x4F, 0xA2, 0x6C, 0xEA, 0x05, 0x22,
+0x00, 0xF0, 0x16, 0x05, 0xB1, 0x18, 0x80, 0xD3,
+0x06, 0x6C, 0x04, 0x6A, 0x0C, 0xEA, 0x05, 0x22,
+0x00, 0xF0, 0x16, 0x05, 0xB1, 0x18, 0x80, 0xD3,
+0x07, 0x6C, 0x08, 0x6A, 0x0C, 0xEA, 0x05, 0x22,
+0x00, 0xF0, 0x16, 0x05, 0xB1, 0x18, 0x80, 0xD3,
+0x09, 0x6C, 0x10, 0x6A, 0x0C, 0xEA, 0x05, 0x22,
+0x00, 0xF0, 0x16, 0x05, 0xB1, 0x18, 0x80, 0xD3,
+0x0A, 0x6C, 0xC0, 0x6A, 0x0C, 0xEA, 0x05, 0x22,
+0x00, 0xF0, 0x16, 0x05, 0xB1, 0x18, 0x80, 0xD3,
+0x0B, 0x6C, 0x20, 0x6A, 0x0C, 0xEA, 0x05, 0x22,
+0x00, 0xF0, 0x16, 0x05, 0xB1, 0x18, 0x80, 0xD3,
+0x0C, 0x6C, 0x00, 0xF2, 0x00, 0x6A, 0x4C, 0xE8,
+0x92, 0x20, 0x00, 0xF0, 0x16, 0x05, 0x0D, 0x6C,
+0x8B, 0x17, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x10, 0x6B, 0x81, 0xF3, 0x52, 0xA2, 0x6C, 0xEA,
+0x16, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4C, 0xF2, 0x58, 0x9A, 0x6B, 0x9A,
+0x0E, 0x23, 0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0,
+0x14, 0x9A, 0xB1, 0x18, 0xEF, 0xD3, 0x01, 0x6C,
+0xB1, 0x18, 0x8A, 0xC5, 0x90, 0x67, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x07, 0x75, 0x03, 0x6A,
+0x4F, 0x61, 0x61, 0xA4, 0x08, 0x73, 0x4C, 0x61,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xA4, 0xA4, 0x63, 0xA4, 0x45, 0xA4, 0xF7, 0xF0,
+0x01, 0x68, 0xA0, 0x35, 0x6D, 0xED, 0x00, 0x30,
+0x66, 0xA4, 0x00, 0x30, 0x40, 0x32, 0x40, 0x32,
+0x4C, 0xF2, 0xD8, 0x98, 0x4D, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x4B, 0x9E, 0x28, 0x23,
+0x02, 0x6C, 0xA4, 0x67, 0x4C, 0xED, 0x24, 0x25,
+0x8D, 0xEB, 0x94, 0x9E, 0x35, 0x9E, 0x24, 0x2A,
+0x07, 0x23, 0xB1, 0x18, 0x8A, 0xC5, 0x04, 0xD3,
+0xB1, 0x18, 0x8A, 0xC5, 0x91, 0x67, 0x04, 0x93,
+0x4C, 0xF2, 0xB8, 0x98, 0x10, 0x6A, 0x8B, 0x9D,
+0x4C, 0xEC, 0x0B, 0x2C, 0x6C, 0xEA, 0x09, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x01, 0x6C, 0x61, 0xF3,
+0x9F, 0xC2, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0x6A, 0x6B, 0xDD, 0x00, 0xEF, 0x05, 0x63,
+0x03, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB, 0xD9, 0x17,
+0xE3, 0x2B, 0xB1, 0x18, 0xB6, 0xC5, 0x04, 0xD3,
+0xB1, 0x18, 0xB6, 0xC5, 0x91, 0x67, 0xDB, 0x17,
+0x20, 0xE8, 0x00, 0x65, 0x42, 0xA4, 0x4C, 0x32,
+0x03, 0x4A, 0x4E, 0xED, 0x03, 0x6A, 0x56, 0x2D,
+0x61, 0xA4, 0x07, 0x73, 0x53, 0x61, 0xFB, 0x63,
+0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x4B, 0xF3,
+0x4C, 0x9B, 0x24, 0x67, 0x07, 0x6C, 0xA0, 0xF0,
+0x9C, 0xC2, 0x82, 0xA1, 0x00, 0x68, 0xA0, 0xF0,
+0x9D, 0xC2, 0x42, 0xA1, 0x43, 0xE8, 0x0E, 0x61,
+0xB1, 0x18, 0x45, 0xD1, 0x03, 0x67, 0x4B, 0xF3,
+0x4C, 0x98, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0x6B, 0x40, 0xF1, 0x70, 0xDA, 0x00, 0x6A,
+0x00, 0xEF, 0x05, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x4B, 0xF3, 0xCC, 0x9B, 0x40, 0x32, 0x0C, 0x37,
+0x40, 0x32, 0xFF, 0x6D, 0x87, 0x40, 0xEC, 0xED,
+0x73, 0xF6, 0x5C, 0x9A, 0x11, 0x4C, 0x04, 0xD3,
+0x03, 0x4D, 0x66, 0x67, 0x8C, 0x34, 0x91, 0xE3,
+0xB5, 0xE1, 0x08, 0x6E, 0x40, 0xEA, 0x05, 0xD7,
+0x04, 0x93, 0x05, 0x97, 0x4B, 0xF3, 0x4C, 0x9B,
+0xFD, 0xE2, 0xC0, 0xF0, 0x80, 0xAF, 0xB0, 0xAF,
+0xAE, 0xEC, 0x05, 0x24, 0xA0, 0x9A, 0x01, 0x6C,
+0x84, 0xE8, 0xAD, 0xEC, 0x80, 0xDA, 0x40, 0xF1,
+0xB0, 0x9A, 0x01, 0x6C, 0x84, 0xE8, 0xAD, 0xEC,
+0x40, 0xF1, 0x90, 0xDA, 0x01, 0x48, 0xFF, 0x6A,
+0x4C, 0xE8, 0xBF, 0x17, 0x20, 0xE8, 0x00, 0x65,
+0x62, 0xA4, 0x6C, 0x32, 0x03, 0x4A, 0x4E, 0xED,
+0x03, 0x6A, 0x3C, 0x2D, 0xA1, 0xA4, 0x07, 0x75,
+0x39, 0x61, 0x15, 0x5B, 0x37, 0x60, 0xFB, 0x63,
+0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x6C, 0x9A, 0x07, 0x6A, 0x04, 0x67, 0x20, 0xF4,
+0x44, 0xC3, 0x42, 0xA4, 0x00, 0x69, 0x20, 0xF4,
+0x45, 0xC3, 0x42, 0xA4, 0xC0, 0xF4, 0x49, 0xC3,
+0x20, 0xF4, 0x45, 0xA3, 0x43, 0xE9, 0x06, 0x61,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0x6A,
+0x00, 0xEF, 0x05, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x2C, 0x34, 0x40, 0x32, 0xFF, 0x6F, 0xA4, 0x67,
+0x40, 0x32, 0xEC, 0xED, 0x73, 0xF6, 0x5C, 0x9A,
+0x91, 0xE3, 0x03, 0x4D, 0x08, 0x6E, 0xB5, 0xE0,
+0x20, 0xF4, 0x08, 0x4C, 0x05, 0xD7, 0x40, 0xEA,
+0x04, 0xD3, 0x05, 0x97, 0x01, 0x49, 0x04, 0x93,
+0xEC, 0xE9, 0xDE, 0x17, 0x20, 0xE8, 0x00, 0x65,
+0x08, 0x2C, 0x20, 0xE8, 0x03, 0x6A, 0x03, 0x6A,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x03, 0x5D, 0xF6, 0x61, 0xFA, 0x63,
+0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0, 0x00, 0x68,
+0x05, 0xD5, 0x04, 0xD4, 0x04, 0x92, 0x05, 0xE2,
+0x42, 0xA1, 0x15, 0x2A, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x4B, 0xF3, 0x6C, 0x98,
+0x04, 0x6D, 0xA0, 0xF0, 0x9B, 0xA3, 0xAC, 0xEC,
+0x6C, 0x2C, 0xB1, 0x18, 0x45, 0xD1, 0x00, 0x65,
+0x4B, 0xF3, 0x4C, 0x98, 0x00, 0x6B, 0x40, 0xF1,
+0x70, 0xDA, 0x00, 0x6A, 0xD9, 0x17, 0x80, 0xA1,
+0x03, 0x48, 0x1E, 0x2C, 0x81, 0xA1, 0x07, 0x74,
+0xD2, 0x61, 0x05, 0x93, 0x09, 0xE2, 0x43, 0xEB,
+0xE1, 0x61, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xF7, 0xF0, 0x01, 0x6A, 0x80, 0x34, 0x4B, 0xF3,
+0x8C, 0x9C, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0x7F, 0x4C, 0x08, 0x6E, 0xA3, 0x41,
+0x40, 0xEA, 0x35, 0x4C, 0x42, 0xA1, 0x41, 0xE0,
+0x05, 0x92, 0x43, 0xE8, 0xC7, 0x61, 0xCA, 0x17,
+0x01, 0x74, 0xB5, 0x61, 0x41, 0xA1, 0x07, 0x72,
+0xB2, 0x61, 0x42, 0xA1, 0x05, 0x93, 0x09, 0xE2,
+0x43, 0xEB, 0xF2, 0x61, 0x04, 0x92, 0x09, 0xE2,
+0x60, 0xA2, 0x12, 0x5B, 0xA8, 0x60, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0x87, 0x43,
+0x07, 0xD3, 0x4B, 0xF3, 0x6C, 0x9F, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x06, 0xD7, 0x40, 0x32,
+0xE3, 0x67, 0x04, 0x93, 0x73, 0xF6, 0x5C, 0x9A,
+0x11, 0x4C, 0xA1, 0x40, 0x8C, 0x34, 0xB5, 0xE3,
+0x91, 0xE7, 0x40, 0xEA, 0x08, 0x6E, 0x06, 0x97,
+0x07, 0x93, 0x4B, 0xF3, 0x4C, 0x9F, 0x6C, 0x34,
+0x91, 0xE2, 0xC0, 0xF0, 0xA0, 0xAC, 0x90, 0xAC,
+0xAE, 0xEC, 0x05, 0x24, 0xA0, 0x9A, 0x01, 0x6C,
+0x84, 0xEB, 0xAD, 0xEC, 0x80, 0xDA, 0x01, 0x6C,
+0x84, 0xEB, 0x40, 0xF1, 0x70, 0x9A, 0x6D, 0xEC,
+0x40, 0xF1, 0x90, 0xDA, 0x42, 0xA1, 0x41, 0xE0,
+0xC4, 0x17, 0x41, 0xF0, 0x40, 0xA3, 0x02, 0x6C,
+0x8D, 0xEA, 0x41, 0xF0, 0x40, 0xC3, 0x8D, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x06, 0x2D,
+0x4B, 0xF3, 0x4C, 0x9B, 0x61, 0xF3, 0x5C, 0xA2,
+0x8E, 0xEA, 0x22, 0x22, 0x4B, 0xF3, 0x4C, 0x9B,
+0x04, 0x67, 0x61, 0xF3, 0x9C, 0xC2, 0x1A, 0xF2,
+0x00, 0x6C, 0xD1, 0x18, 0xC7, 0x6A, 0x02, 0x6D,
+0x00, 0xF4, 0x00, 0x6D, 0xAB, 0xED, 0x4C, 0xED,
+0x7F, 0x6A, 0x1A, 0x65, 0x58, 0x67, 0x0C, 0xEA,
+0x0F, 0x5A, 0x09, 0x60, 0x08, 0x32, 0xFF, 0x6B,
+0x6C, 0xEA, 0x1E, 0x30, 0x4D, 0xED, 0x0C, 0x28,
+0x00, 0xF2, 0x00, 0x6A, 0x4D, 0xED, 0x1A, 0xF2,
+0x00, 0x6C, 0xD1, 0x18, 0xF1, 0x6A, 0x02, 0x6E,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0x00, 0xF3, 0x00, 0x6A, 0xF3, 0x17, 0x00, 0x65,
+0x03, 0x6A, 0xE0, 0xF0, 0x1F, 0x24, 0xC0, 0xA4,
+0x06, 0x5E, 0xE0, 0xF0, 0x1B, 0x60, 0xFC, 0x63,
+0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0, 0x03, 0x76,
+0xA1, 0xA4, 0x62, 0xA4, 0xA0, 0xF0, 0x07, 0x60,
+0x04, 0x5E, 0x41, 0x60, 0x01, 0x76, 0x62, 0x60,
+0x02, 0x76, 0x80, 0xF0, 0x0A, 0x60, 0x18, 0x73,
+0x03, 0x6A, 0x34, 0x61, 0x07, 0x75, 0x32, 0x61,
+0xF7, 0xF0, 0x01, 0x68, 0xF7, 0xF0, 0x01, 0x6A,
+0x00, 0x30, 0x00, 0x30, 0x40, 0x32, 0x4B, 0xF3,
+0x6C, 0x98, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xA3, 0x44, 0x51, 0xF3, 0x8C, 0x43, 0x40, 0xEA,
+0x18, 0x6E, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A, 0x95, 0xA2,
+0xB4, 0xA2, 0x76, 0xA2, 0x80, 0x34, 0x8D, 0xED,
+0x97, 0xA2, 0x4B, 0xF3, 0x4C, 0x98, 0x60, 0x33,
+0x60, 0x33, 0x61, 0xF3, 0x4C, 0xA2, 0xAD, 0xEB,
+0x00, 0xF6, 0x80, 0x34, 0x01, 0x5A, 0x6D, 0xEC,
+0x58, 0x67, 0x5C, 0x33, 0x60, 0xF0, 0x4A, 0xA4,
+0x7F, 0x6D, 0xAC, 0xEA, 0x6D, 0xEA, 0x60, 0xF0,
+0x4A, 0xC4, 0x00, 0x6A, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x04, 0x76,
+0xA0, 0xF0, 0x04, 0x60, 0x05, 0x76, 0xBF, 0x61,
+0xD8, 0x73, 0xF4, 0x61, 0x07, 0x75, 0xF2, 0x61,
+0xF7, 0xF0, 0x01, 0x68, 0xF7, 0xF0, 0x01, 0x6A,
+0x00, 0x30, 0x00, 0x30, 0x40, 0x32, 0x4B, 0xF3,
+0x6C, 0x98, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xA3, 0x44, 0xD0, 0xF6, 0x84, 0x43, 0x40, 0xEA,
+0xD8, 0x6E, 0x4B, 0xF3, 0x6C, 0x98, 0x02, 0x6C,
+0x41, 0xF0, 0x40, 0xA3, 0x8D, 0xEA, 0x41, 0xF0,
+0x40, 0xC3, 0xD7, 0x17, 0x64, 0x73, 0xD6, 0x61,
+0x07, 0x75, 0xD4, 0x61, 0xF7, 0xF0, 0x01, 0x68,
+0xF7, 0xF0, 0x01, 0x6A, 0x00, 0x30, 0x00, 0x30,
+0x40, 0x32, 0x4B, 0xF3, 0x6C, 0x98, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0xA3, 0x44, 0x70, 0xF5,
+0x8C, 0x43, 0x40, 0xEA, 0x64, 0x6E, 0x4B, 0xF3,
+0x4C, 0x98, 0xC0, 0xF5, 0x74, 0x9A, 0x0F, 0x23,
+0xC0, 0xF5, 0x7C, 0x9A, 0x01, 0x73, 0x0B, 0x61,
+0x61, 0xF3, 0x79, 0xC2, 0xC0, 0xF5, 0x58, 0x9A,
+0xB4, 0x22, 0x0D, 0x6E, 0x00, 0x6D, 0xB1, 0x18,
+0xFE, 0xE8, 0x00, 0x6C, 0xAE, 0x17, 0x00, 0x6B,
+0xF3, 0x17, 0x04, 0x73, 0xAB, 0x61, 0x07, 0x75,
+0xA9, 0x61, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x40, 0x32, 0x60, 0x33,
+0x4B, 0xF3, 0x6C, 0x9B, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xA3, 0x44, 0x71, 0xF3, 0x84, 0x43,
+0x40, 0xEA, 0x04, 0x6E, 0x96, 0x17, 0x1C, 0x73,
+0x95, 0x61, 0x07, 0x75, 0x93, 0x61, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0xF7, 0xF0, 0x01, 0x6B,
+0x20, 0x31, 0x60, 0x33, 0x4B, 0xF3, 0x4C, 0x99,
+0x60, 0x33, 0x73, 0xF6, 0x7C, 0x9B, 0xA3, 0x44,
+0xE0, 0xF5, 0x80, 0x42, 0xE0, 0xF5, 0x03, 0xA2,
+0x40, 0xEB, 0x1C, 0x6E, 0x4B, 0xF3, 0x4C, 0x99,
+0x61, 0xF3, 0x7E, 0xA2, 0x05, 0x73, 0x16, 0x61,
+0xE0, 0xF5, 0x63, 0xA2, 0x02, 0x5B, 0x1A, 0x60,
+0xE0, 0xF5, 0x03, 0xC2, 0xE0, 0xF5, 0x62, 0xA2,
+0x05, 0x5B, 0x03, 0x61, 0x04, 0x6B, 0xE0, 0xF5,
+0x62, 0xC2, 0xE0, 0xF5, 0x68, 0xA2, 0x3F, 0x5B,
+0x7F, 0xF7, 0x07, 0x61, 0x00, 0x6B, 0xE0, 0xF5,
+0x68, 0xC2, 0x63, 0x17, 0x04, 0x73, 0x04, 0x61,
+0xE0, 0xF5, 0x63, 0xA2, 0x03, 0x5B, 0xE7, 0x17,
+0x04, 0x5B, 0xE5, 0x17, 0xE0, 0xF5, 0x63, 0xA2,
+0x05, 0x5B, 0xE4, 0x61, 0x04, 0x6B, 0xE0, 0xF5,
+0x63, 0xC2, 0xE0, 0x17, 0x01, 0x73, 0x5F, 0xF7,
+0x11, 0x61, 0x83, 0xA4, 0xB1, 0x18, 0x12, 0xD5,
+0x01, 0x6D, 0x4B, 0x17, 0x20, 0xE8, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4B, 0xF3, 0x4C, 0x9A, 0x10, 0x6B, 0xF7, 0xF0,
+0x01, 0x68, 0x81, 0xF3, 0x52, 0xA2, 0x00, 0x30,
+0x00, 0x30, 0x6C, 0xEA, 0x0C, 0x22, 0x4C, 0xF2,
+0x78, 0x98, 0x00, 0x68, 0x4D, 0x9B, 0x01, 0x4A,
+0x4D, 0xDB, 0x50, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x4C, 0xF2,
+0x58, 0x98, 0x05, 0xD4, 0x00, 0x6E, 0x36, 0x9A,
+0x58, 0x9A, 0x02, 0x6C, 0x00, 0x6D, 0xD1, 0x18,
+0xBD, 0x6B, 0x04, 0xD2, 0x01, 0x6C, 0x8C, 0xEA,
+0x05, 0x93, 0xE5, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF2, 0xF2, 0x58, 0x9A,
+0x04, 0x94, 0x00, 0x6F, 0x08, 0x6E, 0x40, 0xEA,
+0xA3, 0x67, 0x08, 0x72, 0x05, 0x60, 0x4C, 0xF2,
+0x78, 0x98, 0x4D, 0x9B, 0x01, 0x4A, 0x4D, 0xDB,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xB2, 0xF1, 0x44, 0x9A, 0x91, 0x67, 0x40, 0xEA,
+0x01, 0x68, 0xCF, 0x2A, 0xB1, 0x18, 0x8A, 0xC5,
+0x91, 0x67, 0xCB, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0x00, 0x6A, 0x20, 0x74, 0x04, 0xD2, 0x05, 0xD2,
+0x3D, 0x60, 0x21, 0x5C, 0x07, 0x60, 0x08, 0x74,
+0x1E, 0x60, 0x10, 0x74, 0x44, 0x60, 0x07, 0x97,
+0x00, 0xEF, 0x04, 0x63, 0x40, 0x74, 0x58, 0x67,
+0x02, 0x22, 0x80, 0x74, 0xF8, 0x61, 0x30, 0x6B,
+0x9D, 0x67, 0x70, 0xC4, 0x25, 0x2A, 0x10, 0x6A,
+0x7D, 0x67, 0x51, 0xC3, 0x04, 0x04, 0x20, 0x6A,
+0xB1, 0x18, 0xB0, 0xD5, 0x52, 0xC3, 0x1E, 0x6A,
+0x7D, 0x67, 0x04, 0x04, 0xB1, 0x18, 0xB0, 0xD5,
+0x52, 0xC3, 0x1F, 0x6A, 0x0F, 0x10, 0x7D, 0x67,
+0x30, 0x6A, 0x50, 0xC3, 0x04, 0x04, 0x10, 0x6A,
+0xB1, 0x18, 0xB0, 0xD5, 0x51, 0xC3, 0x2B, 0x6A,
+0x7D, 0x67, 0x04, 0x04, 0xB1, 0x18, 0xB0, 0xD5,
+0x52, 0xC3, 0x2A, 0x6A, 0x7D, 0x67, 0x52, 0xC3,
+0xB1, 0x18, 0xB0, 0xD5, 0x04, 0x04, 0xCF, 0x17,
+0x20, 0x6A, 0x7D, 0x67, 0x51, 0xC3, 0x01, 0x6A,
+0x54, 0xC3, 0xDD, 0x17, 0x30, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0x9D, 0x67, 0x10, 0x6B, 0x71, 0xC4,
+0x7D, 0x67, 0x04, 0x04, 0xB1, 0x18, 0xB0, 0xD5,
+0x52, 0xC3, 0x31, 0x6A, 0xE7, 0x17, 0x7D, 0x67,
+0x30, 0x6A, 0x50, 0xC3, 0x2E, 0x6A, 0x52, 0xC3,
+0x10, 0x6A, 0x51, 0xC3, 0x04, 0x04, 0x01, 0x6A,
+0xB1, 0x18, 0xB0, 0xD5, 0x53, 0xC3, 0x7D, 0x67,
+0x11, 0x6A, 0x51, 0xC3, 0x04, 0x04, 0x02, 0x6A,
+0xB1, 0x18, 0xB0, 0xD5, 0x53, 0xC3, 0x12, 0x6A,
+0x7D, 0x67, 0x51, 0xC3, 0x04, 0x6A, 0x53, 0xC3,
+0xCF, 0x17, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0xF9, 0x4C, 0x00, 0x6A, 0x0D, 0x5C, 0x04, 0xD2,
+0x05, 0xD2, 0x7A, 0x60, 0x88, 0x32, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x48, 0xF1,
+0x14, 0x4C, 0x91, 0xE2, 0x40, 0x9C, 0x00, 0xEA,
+0x00, 0x65, 0x30, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0x10, 0x6A, 0x51, 0xC3, 0x0C, 0x6A, 0x52, 0xC3,
+0x60, 0xA5, 0x5D, 0x67, 0x73, 0xC2, 0x61, 0xA5,
+0x74, 0xC2, 0x62, 0xA5, 0x75, 0xC2, 0xB1, 0x18,
+0xB0, 0xD5, 0x04, 0x04, 0x03, 0x6B, 0x01, 0x22,
+0x00, 0x6B, 0x07, 0x97, 0x43, 0x67, 0x00, 0xEF,
+0x04, 0x63, 0x30, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0x10, 0x6A, 0x51, 0xC3, 0x0D, 0x6A, 0xE7, 0x17,
+0x30, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0x10, 0x6A,
+0x4B, 0xEA, 0x51, 0xC3, 0x11, 0x6A, 0xDF, 0x17,
+0x7D, 0x67, 0x11, 0x6A, 0x50, 0xC3, 0x03, 0x6A,
+0x51, 0xC3, 0x60, 0xA5, 0x5D, 0x67, 0x72, 0xC2,
+0x61, 0xA5, 0x73, 0xC2, 0x62, 0xA5, 0x74, 0xC2,
+0xDA, 0x17, 0x23, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0x01, 0x6A, 0x51, 0xC3, 0x60, 0xA5, 0x5D, 0x67,
+0x72, 0xC2, 0xD1, 0x17, 0x1B, 0x6A, 0xF6, 0x17,
+0x7D, 0x67, 0x17, 0x6A, 0x50, 0xC3, 0x01, 0x6A,
+0x51, 0xC3, 0x40, 0xA5, 0x4B, 0xEA, 0x52, 0xC3,
+0xC6, 0x17, 0x7D, 0x67, 0x30, 0x6A, 0x50, 0xC3,
+0x10, 0x6A, 0x51, 0xC3, 0x32, 0x6A, 0x52, 0xC3,
+0x60, 0xA5, 0x5D, 0x67, 0x73, 0xC2, 0xBB, 0x17,
+0x24, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0x02, 0x6A,
+0x51, 0xC3, 0x60, 0xA5, 0x5D, 0x67, 0x72, 0xC2,
+0x61, 0xA5, 0xF4, 0x17, 0x45, 0x6A, 0xD6, 0x17,
+0x45, 0x6A, 0xF3, 0x17, 0x40, 0xA5, 0x01, 0x72,
+0xAA, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A, 0x08, 0x6E,
+0x01, 0x4D, 0x40, 0xEA, 0x04, 0x04, 0x9F, 0x17,
+0x03, 0x6B, 0xA3, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4C, 0xF2, 0x18, 0x9A,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x55, 0x98,
+0x7D, 0x67, 0x80, 0x34, 0x06, 0xD2, 0x01, 0x6A,
+0x50, 0xC3, 0x4B, 0xF3, 0x6C, 0x9C, 0x10, 0x6D,
+0x08, 0xD4, 0x81, 0xF3, 0x52, 0xA3, 0xAC, 0xEA,
+0x58, 0x2A, 0x61, 0xF3, 0x59, 0xA3, 0x2B, 0x98,
+0x07, 0xD2, 0x04, 0x6A, 0x2C, 0xEA, 0x03, 0x22,
+0xB1, 0x18, 0x4D, 0xD3, 0x00, 0x65, 0x05, 0x6B,
+0x08, 0x6A, 0x91, 0x67, 0x4C, 0xEC, 0x07, 0x24,
+0x82, 0x67, 0x0A, 0xD3, 0xB1, 0x18, 0xD7, 0xD5,
+0x09, 0xD2, 0x0A, 0x93, 0x09, 0x92, 0xFF, 0x4B,
+0x44, 0x32, 0xF3, 0x2B, 0xB1, 0x18, 0x52, 0xD2,
+0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18, 0xBD, 0x6B,
+0x08, 0x6C, 0x47, 0xC8, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0xBD, 0x6B, 0x09, 0x6C, 0x48, 0xC8,
+0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18, 0xBD, 0x6B,
+0x0A, 0x6C, 0x49, 0xC8, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0xBD, 0x6B, 0x0B, 0x6C, 0x4A, 0xC8,
+0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18, 0xE5, 0x6B,
+0x11, 0x6C, 0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0xBD, 0x6B, 0x0C, 0x6C, 0x01, 0x5A, 0x58, 0x67,
+0x01, 0x6B, 0x07, 0x96, 0x6E, 0xEA, 0x00, 0x6D,
+0x05, 0x6C, 0xD1, 0x18, 0xBD, 0x6B, 0x42, 0xC0,
+0xB1, 0x18, 0x95, 0xD2, 0x4B, 0xC8, 0x08, 0x92,
+0x4B, 0xF3, 0x4C, 0x9A, 0x61, 0xF3, 0x4C, 0xA2,
+0x01, 0x72, 0x04, 0x61, 0x04, 0x05, 0xB1, 0x18,
+0x0D, 0xD6, 0x0B, 0x6C, 0xB1, 0x18, 0x8A, 0xC5,
+0x06, 0x94, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x24, 0x67, 0x05, 0xA1,
+0x67, 0x44, 0x3F, 0x6A, 0x84, 0xA4, 0x4C, 0xE8,
+0x00, 0x30, 0x01, 0x4B, 0x8D, 0xE8, 0x00, 0x6E,
+0x00, 0x6D, 0x04, 0x6C, 0xD1, 0x18, 0xBD, 0x6B,
+0x06, 0xD3, 0xF8, 0x48, 0x06, 0x93, 0x09, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF3, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x06, 0x93, 0x81, 0xA1, 0x14, 0x5C, 0x11, 0x60,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0x88, 0x32, 0x88, 0xF1, 0x08, 0x4D, 0xA9, 0xE2,
+0x40, 0x9A, 0x00, 0xEA, 0x00, 0x65, 0xB0, 0x67,
+0xB1, 0x18, 0x4D, 0xD4, 0x83, 0x67, 0x03, 0x72,
+0x22, 0x61, 0x41, 0xA1, 0x10, 0xF0, 0x00, 0x6C,
+0x40, 0x32, 0x40, 0x32, 0xB1, 0x18, 0xCC, 0xCC,
+0x4D, 0xEC, 0x03, 0x6A, 0x18, 0x10, 0xB0, 0x67,
+0xB1, 0x18, 0x77, 0xD4, 0x83, 0x67, 0xEF, 0x17,
+0xB0, 0x67, 0xB1, 0x18, 0xA6, 0xD4, 0x83, 0x67,
+0xEA, 0x17, 0xB0, 0x67, 0xB1, 0x18, 0xC8, 0xD4,
+0x83, 0x67, 0xE5, 0x17, 0xB0, 0x67, 0xB1, 0x18,
+0x2E, 0xD5, 0x83, 0x67, 0xE0, 0x17, 0xB1, 0x18,
+0x9C, 0xD1, 0x88, 0xA1, 0x00, 0x6A, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0xB1, 0x18, 0x0D, 0xD6, 0xA3, 0x67, 0xF6, 0x17,
+0x48, 0xA1, 0x0D, 0x2A, 0x9D, 0x67, 0x09, 0x6B,
+0x70, 0xC4, 0xA7, 0x41, 0x09, 0x6B, 0x73, 0xC4,
+0x02, 0x4D, 0x7D, 0x67, 0x04, 0x04, 0xB1, 0x18,
+0x34, 0xD2, 0x54, 0xC3, 0xE7, 0x17, 0xA3, 0x67,
+0xB1, 0x18, 0x0D, 0xD6, 0x13, 0x6C, 0xBF, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x06, 0x2D, 0x4B, 0xF3,
+0x4C, 0x9B, 0x61, 0xF3, 0x5E, 0xA2, 0x8E, 0xEA,
+0x09, 0x22, 0x4B, 0xF3, 0xAC, 0x9B, 0x61, 0xF3,
+0x9E, 0xC5, 0x61, 0xF3, 0x1E, 0x4D, 0xB1, 0x18,
+0x0D, 0xD6, 0x0E, 0x6C, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x06, 0x2D, 0x4B, 0xF3, 0x4C, 0x9B, 0x61, 0xF3,
+0x5D, 0xA2, 0x8E, 0xEA, 0x09, 0x22, 0x4B, 0xF3,
+0xAC, 0x9B, 0x61, 0xF3, 0x9D, 0xC5, 0x61, 0xF3,
+0x1D, 0x4D, 0xB1, 0x18, 0x0D, 0xD6, 0x0D, 0x6C,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69,
+0x60, 0x33, 0x60, 0x33, 0x20, 0x31, 0x09, 0xF1,
+0x04, 0x9B, 0x20, 0x31, 0x72, 0xF4, 0x58, 0x99,
+0x7B, 0x6C, 0x01, 0x6E, 0x40, 0xEA, 0xB0, 0x67,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x42, 0x34, 0x95, 0xC3,
+0x82, 0x34, 0x96, 0xC3, 0x00, 0xF6, 0x42, 0x34,
+0x54, 0xC3, 0x97, 0xC3, 0x1A, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0A, 0xF0, 0x00, 0x4A,
+0x07, 0xD2, 0x00, 0xF5, 0x0A, 0x6A, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF5, 0x40, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF4, 0x5C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xB1, 0x18, 0xFA, 0xD0, 0x00, 0x65,
+0x72, 0xF4, 0x58, 0x99, 0xB0, 0x67, 0xF7, 0xF0,
+0x01, 0x68, 0x01, 0x6E, 0x64, 0x6C, 0x40, 0xEA,
+0x00, 0x30, 0x00, 0x30, 0x4C, 0xF2, 0x58, 0xD8,
+0x0D, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x58, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0A, 0xF0, 0x00, 0x4A, 0x07, 0xD2, 0x00, 0xF5,
+0x18, 0x6A, 0xC5, 0x17, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x93, 0xF6, 0x60, 0x9B,
+0x64, 0x6E, 0x00, 0x6D, 0x40, 0xEB, 0x82, 0x67,
+0x4C, 0xF2, 0x38, 0x98, 0x08, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0xC1, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x09, 0xF1, 0x48, 0x9A,
+0x60, 0x33, 0x60, 0x33, 0x49, 0xD9, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x09, 0xF1,
+0x4C, 0x9A, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x4A, 0xD9, 0xD2, 0xF1, 0x58, 0x9B, 0x80, 0x34,
+0x01, 0x6F, 0x1A, 0x65, 0x97, 0xF0, 0x17, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xEA, 0xF0, 0x0D, 0x4A,
+0x04, 0xD2, 0x00, 0x6E, 0xC0, 0xF7, 0x10, 0x6D,
+0x0A, 0xF0, 0x18, 0x4C, 0x58, 0x67, 0x40, 0xEA,
+0x0A, 0xD3, 0x0A, 0x93, 0x54, 0xD9, 0xF7, 0xF0,
+0x01, 0x6C, 0xD2, 0xF1, 0x58, 0x9B, 0x80, 0x34,
+0x4C, 0xF2, 0x38, 0x98, 0x1A, 0x65, 0x97, 0xF0,
+0x17, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x2B, 0xF1,
+0x1D, 0x4A, 0x80, 0x34, 0x04, 0xD2, 0x01, 0x6F,
+0x58, 0x67, 0xC0, 0xF7, 0x10, 0x6D, 0x2A, 0xF0,
+0x08, 0x4C, 0x40, 0xEA, 0x00, 0x6E, 0x0A, 0x93,
+0x55, 0xD9, 0x97, 0xF0, 0x17, 0x6A, 0x40, 0x32,
+0xF7, 0xF0, 0x01, 0x6C, 0xD2, 0xF1, 0x78, 0x9B,
+0x40, 0x32, 0x80, 0x34, 0x4C, 0xF2, 0x38, 0x98,
+0xE8, 0xF7, 0x19, 0x4A, 0x80, 0x34, 0x01, 0x6F,
+0x00, 0x6E, 0x01, 0x6D, 0x2A, 0xF0, 0x18, 0x4C,
+0x40, 0xEB, 0x04, 0xD2, 0x56, 0xD9, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF3,
+0x50, 0x9A, 0x4C, 0xF2, 0x18, 0x98, 0x01, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x80, 0x6C, 0x58, 0xD8,
+0x57, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x73, 0xF4, 0x5C, 0x9A, 0x09, 0xF1, 0xB0, 0x9B,
+0x40, 0xEA, 0x00, 0x6C, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30,
+0x00, 0x6E, 0x00, 0x30, 0x00, 0x6D, 0xD1, 0x18,
+0xE5, 0x6B, 0x14, 0x6C, 0x93, 0xF4, 0x4C, 0x98,
+0x02, 0x6D, 0x40, 0xEA, 0x01, 0x6C, 0x93, 0xF4,
+0x4C, 0x98, 0x20, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD0, 0x01, 0x6A,
+0x7D, 0x67, 0x00, 0x6E, 0x00, 0x6D, 0x06, 0x6C,
+0xD1, 0x18, 0xBD, 0x6B, 0x58, 0xC3, 0x01, 0x6B,
+0x6C, 0xEA, 0x14, 0x22, 0x00, 0x6A, 0x7D, 0x67,
+0x59, 0xC3, 0x5D, 0x67, 0x19, 0xA2, 0x08, 0x58,
+0x11, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF4, 0x50, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x00, 0x6E, 0x06, 0x05, 0xD1, 0x18,
+0xE5, 0x6B, 0x13, 0x6C, 0x09, 0x97, 0x08, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x00, 0x6E, 0x00, 0xF0,
+0x19, 0x05, 0xD1, 0x18, 0xBD, 0x6B, 0x07, 0x6C,
+0x04, 0x03, 0x01, 0xE3, 0x40, 0xC0, 0x5D, 0x67,
+0x59, 0xA2, 0x01, 0x4A, 0xDC, 0x17, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD0, 0x00, 0x6A,
+0x7D, 0x67, 0x54, 0xC3, 0x04, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x4C, 0x9A, 0x10, 0x6B, 0x81, 0xF3, 0x52, 0xA2,
+0x6C, 0xEA, 0xFF, 0x6B, 0x6C, 0xEA, 0x38, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x04, 0x67, 0x4C, 0xF2, 0x98, 0x9A, 0x4E, 0x9C,
+0x01, 0x4A, 0x4E, 0xDC, 0x40, 0xA0, 0x30, 0x72,
+0x3F, 0x60, 0x31, 0x5A, 0x23, 0x60, 0x24, 0x72,
+0x78, 0x67, 0x4F, 0x23, 0x27, 0x72, 0x02, 0x60,
+0x23, 0x72, 0x22, 0x61, 0x41, 0xA0, 0x06, 0x72,
+0x1F, 0x61, 0xB1, 0x18, 0x69, 0xD1, 0x85, 0xA0,
+0xB1, 0x18, 0xA2, 0xD1, 0x83, 0xA0, 0xB1, 0x18,
+0xAD, 0xD1, 0x82, 0xA0, 0xB1, 0x18, 0x8A, 0xC6,
+0x90, 0x67, 0x01, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0x41, 0xA0, 0xFF, 0xF7, 0x1F, 0x6B, 0x0A, 0x4A,
+0x6C, 0xEA, 0x7D, 0x67, 0x53, 0xC3, 0x42, 0x32,
+0x54, 0xC3, 0x1D, 0x10, 0x45, 0x72, 0x35, 0x60,
+0x46, 0x5A, 0x06, 0x60, 0x39, 0x72, 0x22, 0x60,
+0x07, 0x97, 0x06, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x48, 0x5A, 0x07, 0x6A, 0xF9, 0x60, 0x9D, 0x67,
+0x50, 0xC4, 0x41, 0xA0, 0xFF, 0xF7, 0x1F, 0x6C,
+0x0A, 0x4A, 0x8C, 0xEA, 0x4C, 0xEB, 0x9D, 0x67,
+0x73, 0xC4, 0x42, 0x32, 0x7D, 0x67, 0xE4, 0x17,
+0xB1, 0x18, 0xBD, 0xD2, 0x90, 0x67, 0x5D, 0x67,
+0x73, 0xA2, 0x54, 0xA2, 0x40, 0x32, 0x6D, 0xEA,
+0xE3, 0x22, 0xB0, 0x67, 0xB1, 0x18, 0x34, 0xD2,
+0x04, 0x04, 0xDE, 0x17, 0xB1, 0x18, 0x8A, 0xC6,
+0x90, 0x67, 0xB1, 0x18, 0xB7, 0xD1, 0x00, 0x65,
+0xEE, 0x17, 0x05, 0x6A, 0x9D, 0x67, 0x50, 0xC4,
+0x10, 0x6A, 0x53, 0xC4, 0x5D, 0x67, 0x74, 0xC2,
+0xE6, 0x17, 0x06, 0x6A, 0xD4, 0x17, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4C, 0xF2, 0x78, 0x9A, 0x4E, 0xAB, 0x01, 0x4A,
+0x4E, 0xCB, 0x40, 0xA4, 0x4A, 0x32, 0x10, 0x72,
+0x08, 0x61, 0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18,
+0x8B, 0xD6, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x02, 0x6A, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0xBD, 0x6B, 0x02, 0x6C, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x6B, 0x4B, 0xF3, 0xAC, 0x9C, 0x60, 0x33,
+0x60, 0x33, 0x09, 0xF1, 0x74, 0x9B, 0x81, 0xF3,
+0x92, 0xA5, 0x10, 0x6E, 0x4C, 0xEB, 0xCC, 0xEC,
+0x06, 0xD3, 0x17, 0x2C, 0xA1, 0xF3, 0xC0, 0x9D,
+0xCA, 0xEB, 0x13, 0x60, 0x42, 0x32, 0x4E, 0x32,
+0x01, 0x6E, 0xCC, 0xEA, 0x61, 0xF3, 0x5A, 0xC5,
+0xA1, 0xF3, 0x60, 0xDD, 0x02, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0x0C, 0x6A, 0x53, 0xC3, 0x5D, 0x67,
+0x94, 0xC2, 0x06, 0x05, 0xB1, 0x18, 0x34, 0xD2,
+0x04, 0x04, 0x09, 0x97, 0x00, 0xEF, 0x05, 0x63,
+0x01, 0x75, 0x08, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3, 0x4C, 0x9A,
+0x61, 0xF3, 0x98, 0xC2, 0x20, 0xE8, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x00, 0x6A, 0x04, 0xD2,
+0x7D, 0x67, 0x05, 0xD2, 0x42, 0x6A, 0x50, 0xC3,
+0x01, 0x6A, 0x51, 0xC3, 0x5D, 0x67, 0x92, 0xC2,
+0xB1, 0x18, 0xB0, 0xD5, 0x04, 0x04, 0x07, 0x97,
+0x00, 0xEF, 0x04, 0x63, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0xF7, 0xF0, 0x01, 0x6D, 0x20, 0x31,
+0xA0, 0x35, 0x04, 0x67, 0x4B, 0xF3, 0xCC, 0x99,
+0xA0, 0x35, 0x2C, 0x24, 0x4C, 0xF2, 0x78, 0x9D,
+0xB6, 0x9B, 0x00, 0x6B, 0xA1, 0xF3, 0x64, 0xDE,
+0xA1, 0xF3, 0x68, 0x46, 0xE1, 0xF3, 0x08, 0x4E,
+0x80, 0x9B, 0x06, 0xD6, 0x05, 0xD5, 0xB1, 0x18,
+0xB6, 0xC5, 0x04, 0xD3, 0x04, 0x93, 0x06, 0x96,
+0x05, 0x95, 0x04, 0x4B, 0xCA, 0xEB, 0xF4, 0x61,
+0xB1, 0x18, 0xB6, 0xC5, 0x85, 0x67, 0x4B, 0xF3,
+0x6C, 0x99, 0x01, 0x6A, 0x4C, 0xE8, 0x81, 0xF3,
+0x52, 0xA3, 0x11, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA,
+0x10, 0x30, 0x4D, 0xE8, 0x81, 0xF3, 0x12, 0xC3,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0x6A,
+0x00, 0xEF, 0x06, 0x63, 0x81, 0xF3, 0x72, 0xA6,
+0x10, 0x6C, 0x8C, 0xEB, 0x08, 0x23, 0xA1, 0xF3,
+0x44, 0x9E, 0x05, 0x22, 0x80, 0x6C, 0xB1, 0x18,
+0xCC, 0xCC, 0x04, 0xD5, 0x04, 0x95, 0x4C, 0xF2,
+0x58, 0x9D, 0xB1, 0x18, 0x8A, 0xC5, 0x96, 0x9A,
+0xDA, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD0, 0x00, 0x6A, 0x7D, 0x67, 0x58, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x09, 0xF1, 0x5C, 0x9A, 0x05, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x29, 0xF1,
+0x40, 0x9A, 0x04, 0xD2, 0x13, 0x24, 0x01, 0x74,
+0x2D, 0x60, 0x00, 0x6D, 0xD1, 0x18, 0xB3, 0xF4,
+0x38, 0x6C, 0x00, 0x6D, 0xD1, 0x18, 0xB3, 0xF4,
+0x3C, 0x6C, 0x00, 0x6E, 0x06, 0x05, 0xD1, 0x18,
+0xE5, 0x6B, 0x0F, 0x6C, 0x09, 0x97, 0x08, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4C, 0xF2, 0x78, 0x9A,
+0x21, 0x6C, 0x8B, 0xEC, 0x51, 0xAB, 0x00, 0x6E,
+0x05, 0x05, 0x01, 0x4A, 0x51, 0xCB, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4B, 0xF3,
+0x6C, 0x9A, 0x81, 0xF3, 0x52, 0xA3, 0x8C, 0xEA,
+0x81, 0xF3, 0x52, 0xC3, 0xD1, 0x18, 0xE5, 0x6B,
+0x15, 0x6C, 0xD3, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4C, 0xF2, 0x78, 0x9A,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x50, 0xAB,
+0x00, 0x30, 0x20, 0x6C, 0x01, 0x4A, 0x50, 0xCB,
+0x4B, 0xF3, 0x4C, 0x98, 0x81, 0xF3, 0x72, 0xA2,
+0x8D, 0xEB, 0x81, 0xF3, 0x72, 0xC2, 0x61, 0xF3,
+0x6C, 0xA2, 0x01, 0x73, 0x0C, 0x6B, 0x0E, 0x60,
+0x60, 0xF5, 0x5D, 0xA2, 0x09, 0x72, 0x0A, 0x60,
+0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18, 0xBD, 0x6B,
+0x02, 0x6C, 0x10, 0x6B, 0x6C, 0xEA, 0x0D, 0x6B,
+0x01, 0x2A, 0x0B, 0x6B, 0x4B, 0xF3, 0x4C, 0x98,
+0x6C, 0x33, 0x01, 0x6C, 0x6D, 0xE2, 0xD1, 0xAB,
+0xB1, 0x18, 0x68, 0xCC, 0xA9, 0x9B, 0x01, 0x6A,
+0x7D, 0x67, 0x58, 0xC3, 0x00, 0x6E, 0x04, 0x05,
+0xC5, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF3, 0x40, 0x9A,
+0x40, 0xEA, 0xFF, 0x69, 0x02, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x80, 0xF2, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xD2, 0xF4, 0x68, 0x9B, 0x40, 0x6D,
+0x80, 0xF2, 0x00, 0x6C, 0x40, 0xEB, 0x4D, 0xED,
+0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36,
+0xD2, 0xF4, 0x4C, 0x9E, 0x68, 0x6C, 0x40, 0xEA,
+0x05, 0xD6, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x41, 0x6D, 0xD2, 0xF4, 0xE0, 0x9B,
+0xAB, 0xED, 0x4C, 0xED, 0x2C, 0xED, 0x68, 0x6C,
+0x40, 0xEF, 0x04, 0xD3, 0x05, 0x96, 0x90, 0xF3,
+0x14, 0x6C, 0xD2, 0xF4, 0x4C, 0x9E, 0x40, 0xEA,
+0x00, 0x65, 0x04, 0x93, 0x04, 0x6D, 0x4D, 0xED,
+0xD2, 0xF4, 0x60, 0x9B, 0x90, 0xF3, 0x14, 0x6C,
+0x40, 0xEB, 0x2C, 0xED, 0x28, 0x70, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0x33, 0xF1,
+0x4C, 0x98, 0x63, 0x61, 0x60, 0xF2, 0x02, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x33, 0xF1,
+0x4C, 0x98, 0x0B, 0xF2, 0x00, 0x6E, 0x01, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x33, 0xF1, 0x4C, 0x98,
+0x04, 0x6E, 0x02, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0x33, 0xF1, 0x4C, 0x98, 0x00, 0x6E, 0x03, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x33, 0xF1, 0x4C, 0x98,
+0x0F, 0xF5, 0x00, 0x6E, 0x04, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x69, 0x33, 0xF1,
+0x4C, 0x98, 0x20, 0x31, 0x00, 0x6E, 0x05, 0x6D,
+0x00, 0x6C, 0x40, 0xEA, 0x20, 0x31, 0x33, 0xF1,
+0x48, 0x99, 0x08, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0x01, 0x6E, 0x33, 0xF1, 0x6C, 0x98, 0xCD, 0xEA,
+0xFF, 0xF7, 0x1F, 0x6E, 0x4C, 0xEE, 0x08, 0x6D,
+0x40, 0xEB, 0x00, 0x6C, 0xE0, 0xF3, 0x09, 0x6E,
+0x33, 0xF1, 0x48, 0x99, 0x08, 0x6D, 0x00, 0x6C,
+0x40, 0xEA, 0x04, 0xD6, 0x04, 0x6B, 0x6C, 0xEA,
+0x04, 0x96, 0x0B, 0x2A, 0xFF, 0x4E, 0x27, 0x2E,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x01, 0x6D, 0x40, 0xEA,
+0x01, 0x6C, 0x33, 0xF1, 0x48, 0x99, 0x08, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x33, 0xF1, 0x6C, 0x98,
+0x02, 0x6E, 0xCD, 0xEA, 0xFF, 0xF7, 0x1F, 0x6E,
+0x4C, 0xEE, 0x08, 0x6D, 0x40, 0xEB, 0x00, 0x6C,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0xC0, 0xF4, 0x04, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x33, 0xF1, 0x4C, 0x98,
+0x16, 0xF4, 0x00, 0x6E, 0x9C, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x48, 0x9A, 0x01, 0x6C, 0x40, 0xEA, 0x04, 0xD6,
+0x04, 0x96, 0xC2, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x1C, 0x2C, 0x0E, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4A, 0xF0, 0x08, 0x4A,
+0x05, 0xD2, 0x1F, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x07, 0xD4, 0x04, 0x04, 0x40, 0xEA,
+0x00, 0x65, 0x00, 0x6A, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x40, 0xEA, 0x24, 0x67, 0x02, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF7, 0xE4, 0x9A, 0x91, 0x67, 0x69, 0xA7,
+0x48, 0xA7, 0xAA, 0xA7, 0x60, 0x33, 0x4D, 0xEB,
+0x4B, 0xA7, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0x43, 0xE9,
+0x54, 0x61, 0xA1, 0xA7, 0x60, 0xA7, 0xC2, 0xA7,
+0xA0, 0x35, 0x6D, 0xED, 0x63, 0xA7, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0xA5, 0xA7, 0xC4, 0xA7, 0x26, 0xA7,
+0xA0, 0x35, 0xCD, 0xED, 0xC7, 0xA7, 0x20, 0x31,
+0x20, 0x31, 0x2D, 0xED, 0x00, 0xF6, 0xC0, 0x36,
+0xAD, 0xEE, 0xD8, 0xEB, 0x12, 0xED, 0xB5, 0xE2,
+0xA3, 0xEC, 0x37, 0x60, 0x2D, 0xA7, 0xAC, 0xA7,
+0x20, 0x31, 0xAD, 0xE9, 0x19, 0x65, 0x2E, 0xA7,
+0xB8, 0x67, 0x20, 0x31, 0x20, 0x31, 0xAD, 0xE9,
+0xAF, 0xA7, 0x00, 0x6F, 0x00, 0xF6, 0xA0, 0x35,
+0x2D, 0xED, 0x3F, 0x65, 0x5F, 0xE4, 0xFF, 0x65,
+0x00, 0x6F, 0x3F, 0x67, 0x01, 0x51, 0xA0, 0xF0,
+0x0A, 0x60, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0xDF, 0x67, 0x20, 0x31, 0xA0, 0xF0, 0x08, 0x26,
+0x00, 0x6A, 0x07, 0xD2, 0x7D, 0x67, 0x0E, 0x6A,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4A, 0xF0, 0x08, 0x4A, 0x05, 0xD2,
+0x46, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x72, 0xF0, 0x54, 0x99,
+0x88, 0x10, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF7, 0xFC, 0x9A, 0x69, 0xA7,
+0x48, 0xA7, 0xAA, 0xA7, 0x60, 0x33, 0x4D, 0xEB,
+0x4B, 0xA7, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0x43, 0xEC,
+0x2D, 0x61, 0xA1, 0xA7, 0x60, 0xA7, 0xC2, 0xA7,
+0xA0, 0x35, 0x6D, 0xED, 0x63, 0xA7, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0xA5, 0xA7, 0xC4, 0xA7, 0x26, 0xA7,
+0xA0, 0x35, 0xCD, 0xED, 0xC7, 0xA7, 0x20, 0x31,
+0x20, 0x31, 0x2D, 0xED, 0x00, 0xF6, 0xC0, 0x36,
+0xAD, 0xEE, 0xD8, 0xEB, 0x12, 0xED, 0xB5, 0xE2,
+0xA3, 0xEC, 0x10, 0x60, 0x2D, 0xA7, 0xAC, 0xA7,
+0x20, 0x31, 0xAD, 0xE9, 0x19, 0x65, 0x2E, 0xA7,
+0xB8, 0x67, 0x20, 0x31, 0x20, 0x31, 0xAD, 0xE9,
+0xAF, 0xA7, 0x01, 0x6F, 0x00, 0xF6, 0xA0, 0x35,
+0x2D, 0xED, 0x97, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF7, 0xE0, 0x9A,
+0x69, 0xA7, 0x48, 0xA7, 0xAA, 0xA7, 0x60, 0x33,
+0x4D, 0xEB, 0x4B, 0xA7, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0x43, 0xEC, 0x2D, 0x61, 0xA1, 0xA7, 0x60, 0xA7,
+0xC2, 0xA7, 0xA0, 0x35, 0x6D, 0xED, 0x63, 0xA7,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xA5, 0xA7, 0xC4, 0xA7,
+0x26, 0xA7, 0xA0, 0x35, 0xCD, 0xED, 0xC7, 0xA7,
+0x20, 0x31, 0x20, 0x31, 0x2D, 0xED, 0x00, 0xF6,
+0xC0, 0x36, 0xAD, 0xEE, 0xD8, 0xEB, 0x12, 0xED,
+0xB5, 0xE2, 0xA3, 0xEC, 0x10, 0x60, 0x2D, 0xA7,
+0xAC, 0xA7, 0x20, 0x31, 0xAD, 0xE9, 0x19, 0x65,
+0x2E, 0xA7, 0xB8, 0x67, 0x20, 0x31, 0x20, 0x31,
+0xAD, 0xE9, 0xAF, 0xA7, 0x02, 0x6F, 0x00, 0xF6,
+0xA0, 0x35, 0x2D, 0xED, 0x56, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x90, 0x67, 0xFC, 0x16, 0x3F, 0x67,
+0xC7, 0xE1, 0x01, 0x4F, 0xF9, 0x65, 0x4D, 0x17,
+0x43, 0xEC, 0x02, 0x61, 0x63, 0xEF, 0x0E, 0x61,
+0x00, 0x6A, 0x07, 0xD2, 0x7D, 0x67, 0x0E, 0x6A,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4A, 0xF0, 0x08, 0x4A, 0x05, 0xD2,
+0x50, 0x6A, 0x53, 0x17, 0xF6, 0x32, 0x48, 0x32,
+0x55, 0xE5, 0x60, 0x9D, 0x01, 0x6C, 0x44, 0x67,
+0x44, 0xEF, 0x6D, 0xEA, 0x40, 0xDD, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x53, 0xF7,
+0xD4, 0x9A, 0xA1, 0xA6, 0x40, 0xA6, 0x62, 0xA6,
+0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA6, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x79, 0x67, 0x69, 0xE2, 0x40, 0xA2,
+0x01, 0x72, 0x36, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x84, 0xEB, 0xF3, 0xF6,
+0x7C, 0xA2, 0x6D, 0xEC, 0xF3, 0xF6, 0x9C, 0xC2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x52, 0xF0, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x08, 0xD2, 0x08, 0x92, 0x00, 0xF2, 0x00, 0x6B,
+0x0A, 0xD3, 0x6D, 0xEA, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x52, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x08, 0x94, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x52, 0xF0,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x04, 0xD2,
+0x04, 0x92, 0x0A, 0x93, 0x6D, 0xEA, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x52, 0xF0, 0x4C, 0x9A, 0x40, 0xEA, 0x04, 0x94,
+0x72, 0xF0, 0x54, 0x99, 0x40, 0xEA, 0x90, 0x67,
+0x01, 0x6A, 0x8C, 0x16, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD1, 0x13, 0xD0, 0x02, 0x74, 0xE0, 0xF0,
+0x04, 0x60, 0x03, 0x5C, 0x09, 0x60, 0xE0, 0xF0,
+0x0A, 0x24, 0x01, 0x74, 0x0F, 0x60, 0x15, 0x97,
+0x14, 0x91, 0x13, 0x90, 0x00, 0xEF, 0x0B, 0x63,
+0x03, 0x74, 0xC0, 0xF0, 0x1B, 0x60, 0x04, 0x74,
+0xF6, 0x61, 0x20, 0xF6, 0x00, 0x6A, 0x00, 0xF4,
+0x10, 0x68, 0x04, 0x10, 0x00, 0xF6, 0x08, 0x6A,
+0x00, 0xF4, 0x04, 0x68, 0x09, 0xD4, 0xF7, 0xF0,
+0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6D, 0x80, 0x34,
+0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35, 0x29, 0xF1,
+0x24, 0x9C, 0xF2, 0xF4, 0x64, 0x9D, 0x08, 0xD5,
+0x40, 0xEB, 0x31, 0xE2, 0x08, 0x95, 0x0C, 0xD2,
+0x2D, 0xE0, 0xF2, 0xF4, 0x44, 0x9D, 0x83, 0x67,
+0x0B, 0xD5, 0x40, 0xEA, 0x0A, 0xD3, 0x22, 0x67,
+0x08, 0xD2, 0x42, 0x32, 0xE0, 0xF3, 0x1F, 0x6C,
+0x42, 0x32, 0x8C, 0xE9, 0x8C, 0xEA, 0x4A, 0xE9,
+0x0B, 0x95, 0xC9, 0x60, 0x43, 0xE9, 0x0C, 0x93,
+0x2B, 0xE2, 0xA0, 0xF0, 0x11, 0x60, 0xFF, 0x6C,
+0x8C, 0xEA, 0x0C, 0xD2, 0x09, 0x92, 0x01, 0x72,
+0xE0, 0xF0, 0x03, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x29, 0xF1, 0x88, 0x9A,
+0x2C, 0x30, 0x0E, 0xD2, 0x8D, 0xEB, 0x0D, 0xD3,
+0x61, 0xE0, 0x00, 0x6B, 0x0B, 0xD3, 0x0B, 0x92,
+0x0C, 0x93, 0x6A, 0xEA, 0xAC, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF5,
+0x5C, 0x9A, 0x40, 0xEA, 0x09, 0x94, 0xA3, 0x22,
+0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0xE0, 0x37, 0x53, 0xF7, 0x50, 0xA7, 0x43, 0xED,
+0x6F, 0x60, 0xC5, 0xA0, 0x44, 0xA0, 0x86, 0xA0,
+0x0E, 0x93, 0xC0, 0x36, 0x4D, 0xEE, 0x80, 0x34,
+0x47, 0xA0, 0x80, 0x34, 0x8D, 0xEE, 0x29, 0xF1,
+0x88, 0x9B, 0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA,
+0x8D, 0xEA, 0xC2, 0xA2, 0x43, 0xA2, 0x1F, 0x6C,
+0x09, 0x93, 0x8C, 0xEA, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x40, 0x32, 0x80, 0x34, 0xCD, 0xEA,
+0x73, 0xF7, 0x08, 0x4C, 0x64, 0x36, 0x99, 0xE6,
+0x80, 0xAE, 0x8E, 0xEA, 0x69, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF5,
+0x58, 0x9A, 0xB0, 0x67, 0x83, 0x67, 0x40, 0xEA,
+0x0F, 0xD6, 0x01, 0x72, 0x7F, 0xF7, 0x0B, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x09, 0x94, 0xF3, 0xF5, 0x5C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x80, 0x6C, 0x8B, 0xEC, 0x80, 0xC0,
+0x81, 0xA0, 0x40, 0x6D, 0xAB, 0xED, 0xAC, 0xEC,
+0x81, 0xC0, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x29, 0xF1, 0x8C, 0x9C, 0x0F, 0x96,
+0xFF, 0xF7, 0x1F, 0x6B, 0x8C, 0xEA, 0x42, 0x34,
+0x44, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x47, 0xC0,
+0x40, 0xAE, 0x85, 0xC0, 0x82, 0x34, 0x01, 0x4A,
+0x6C, 0xEA, 0x04, 0xF0, 0x00, 0x5A, 0x86, 0xC0,
+0x31, 0x60, 0x01, 0x49, 0x40, 0xCE, 0x14, 0x6A,
+0x5B, 0xE9, 0x01, 0x2A, 0xE5, 0xE8, 0x08, 0x93,
+0xFF, 0xF3, 0x0C, 0x4A, 0x0A, 0x94, 0x4C, 0xEB,
+0x43, 0x67, 0x10, 0xE9, 0x2D, 0xEA, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x58, 0x9A, 0x40, 0xEA, 0x08, 0x95,
+0x0D, 0x92, 0x0B, 0x94, 0x2C, 0x30, 0x41, 0xE0,
+0x01, 0x4C, 0xFF, 0x6A, 0x4C, 0xEC, 0x0B, 0xD4,
+0x72, 0x17, 0x00, 0xF6, 0x10, 0x6A, 0x00, 0xF4,
+0x08, 0x68, 0x30, 0x17, 0x00, 0xF6, 0x18, 0x6A,
+0x00, 0xF4, 0x0C, 0x68, 0x2B, 0x17, 0x00, 0xF6,
+0x00, 0x6A, 0x00, 0xF4, 0x00, 0x68, 0x26, 0x17,
+0x14, 0x4A, 0x4D, 0x17, 0x01, 0x6A, 0xCD, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0xE0, 0xF3, 0x08, 0x6C,
+0x10, 0xD7, 0x40, 0xEA, 0x0F, 0xD5, 0x0F, 0x95,
+0x10, 0x97, 0xFF, 0x6A, 0x01, 0x4D, 0x4C, 0xED,
+0x53, 0xF7, 0x50, 0xA7, 0x42, 0xED, 0x5F, 0xF7,
+0x18, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xB3, 0xF5, 0x48, 0x9A, 0x40, 0xEA,
+0x0F, 0xD5, 0x00, 0x6A, 0x07, 0xD2, 0x7D, 0x67,
+0x0E, 0x6A, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4A, 0xF0, 0x08, 0x4A,
+0x05, 0xD2, 0xEE, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0x0F, 0x95,
+0x38, 0x17, 0xF2, 0xF4, 0x44, 0x9D, 0x40, 0xEA,
+0x0A, 0x94, 0x42, 0x33, 0xE0, 0xF3, 0x1F, 0x6C,
+0xA2, 0x67, 0x62, 0x33, 0x8C, 0xED, 0x8C, 0xEB,
+0x6A, 0xED, 0xDF, 0xF6, 0x0C, 0x60, 0x63, 0xED,
+0xAF, 0xE3, 0x13, 0x60, 0xFF, 0x6C, 0x8C, 0xEB,
+0xAD, 0xE3, 0x00, 0xF4, 0x00, 0x6D, 0xAB, 0xED,
+0x4C, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x03, 0x6C, 0x40, 0x32, 0x8C, 0xEB, 0xD2, 0xF4,
+0x58, 0x9A, 0x0A, 0x94, 0x40, 0xEA, 0x6D, 0xED,
+0xB6, 0x16, 0x04, 0x4B, 0xEB, 0x17, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0x00, 0x68, 0x02, 0x70, 0x78, 0x67, 0x01, 0x70,
+0x58, 0x67, 0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x1B, 0x65, 0x40, 0x32, 0xE0, 0xF0,
+0x19, 0x60, 0x03, 0x58, 0x80, 0xF0, 0x10, 0x60,
+0x09, 0x93, 0xC0, 0xF0, 0x03, 0x23, 0x15, 0xF0,
+0x89, 0xA2, 0x15, 0xF0, 0x68, 0xA2, 0x15, 0xF0,
+0xAA, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x15, 0xF0,
+0x6B, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3,
+0x40, 0xA3, 0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x43, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x85, 0xA2,
+0x66, 0xA2, 0xA4, 0xA2, 0x47, 0xA2, 0x80, 0x34,
+0x60, 0x33, 0x8D, 0xED, 0x60, 0x33, 0xAD, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0x08, 0xD2,
+0x00, 0xF3, 0x00, 0x6F, 0x00, 0xF6, 0x00, 0x6C,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0xD2, 0xF4, 0x58, 0x99, 0xF7, 0xF0, 0x01, 0x6E,
+0xF7, 0xF0, 0x01, 0x6B, 0xC0, 0x36, 0x60, 0x33,
+0xC0, 0x36, 0x60, 0x33, 0x1A, 0x65, 0x29, 0xF1,
+0xAC, 0x9E, 0x29, 0xF1, 0x44, 0x9B, 0x0B, 0xD3,
+0x08, 0x93, 0x51, 0xE4, 0x0C, 0xD2, 0x6C, 0xED,
+0x58, 0x67, 0x0D, 0xD7, 0x40, 0xEA, 0x0A, 0xD6,
+0x0C, 0x92, 0x0D, 0x97, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x51, 0xE7, 0x09, 0x92, 0x60, 0x33,
+0x00, 0xF1, 0x00, 0x22, 0xD2, 0xF4, 0x5C, 0x9B,
+0x14, 0x6D, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x29, 0xF1,
+0x90, 0x9B, 0xD2, 0xF4, 0x58, 0x99, 0x40, 0xEA,
+0x3F, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF5, 0x44, 0x9A, 0x01, 0x6D,
+0x40, 0xEA, 0x90, 0x67, 0x09, 0x92, 0xE0, 0xF0,
+0x09, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF5, 0x48, 0x9A, 0x40, 0xEA,
+0x90, 0x67, 0x01, 0x48, 0x05, 0x70, 0x7F, 0xF7,
+0x14, 0x61, 0x0B, 0x93, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x29, 0xF1, 0x04, 0x9B, 0x40, 0x32,
+0xF2, 0xF4, 0x44, 0x9A, 0x40, 0xEA, 0x90, 0x67,
+0xD2, 0xF4, 0x78, 0x99, 0x04, 0xF0, 0x00, 0x6D,
+0x4D, 0xED, 0x40, 0xEB, 0x90, 0x67, 0xF6, 0x10,
+0x03, 0x70, 0x80, 0xF0, 0x13, 0x60, 0x04, 0x70,
+0x7F, 0xF7, 0x0D, 0x61, 0x15, 0xF0, 0x89, 0xA2,
+0x15, 0xF0, 0x68, 0xA2, 0x15, 0xF0, 0xAA, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x15, 0xF0, 0x6B, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2, 0x86, 0xA2,
+0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x80, 0xF2, 0x00, 0x4A,
+0x08, 0xD2, 0x00, 0xF3, 0x08, 0x6F, 0x20, 0xF6,
+0x00, 0x6C, 0x6A, 0x17, 0x15, 0xF0, 0x89, 0xA2,
+0x15, 0xF0, 0x68, 0xA2, 0x15, 0xF0, 0xAA, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x15, 0xF0, 0x6B, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2, 0x86, 0xA2,
+0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x7F, 0x4A, 0x21, 0x4A,
+0x08, 0xD2, 0x00, 0xF3, 0x02, 0x6F, 0x00, 0xF6,
+0x08, 0x6C, 0x3A, 0x17, 0x15, 0xF0, 0x89, 0xA2,
+0x15, 0xF0, 0x68, 0xA2, 0x15, 0xF0, 0xAA, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x15, 0xF0, 0x6B, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2, 0x86, 0xA2,
+0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0xF1, 0x00, 0x4A,
+0x08, 0xD2, 0x00, 0xF3, 0x04, 0x6F, 0x00, 0xF6,
+0x10, 0x6C, 0x0A, 0x17, 0x15, 0xF0, 0x89, 0xA2,
+0x15, 0xF0, 0x68, 0xA2, 0x15, 0xF0, 0xAA, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x15, 0xF0, 0x6B, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2, 0x86, 0xA2,
+0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0xE0, 0xF1, 0x00, 0x4A,
+0x08, 0xD2, 0x00, 0xF3, 0x06, 0x6F, 0x00, 0xF6,
+0x18, 0x6C, 0xDA, 0x16, 0xD2, 0xF4, 0x5C, 0x9B,
+0x04, 0x6D, 0xFF, 0x16, 0x08, 0x93, 0xE7, 0x43,
+0x19, 0x4F, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6C, 0x40, 0x32, 0x80, 0x34, 0x80, 0x34,
+0x40, 0x32, 0x29, 0xF1, 0xB4, 0x9C, 0x72, 0xF4,
+0x58, 0x9A, 0x00, 0x6E, 0x80, 0x6C, 0x09, 0xD3,
+0x40, 0xEA, 0x08, 0xD7, 0x08, 0x97, 0x09, 0x93,
+0x1A, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4A, 0xF0, 0x08, 0x4A, 0x05, 0xD2, 0xFF, 0x6A,
+0x67, 0x4A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x80, 0x6C,
+0x8B, 0xEC, 0x80, 0xC3, 0x81, 0xA3, 0x40, 0x6D,
+0xAB, 0xED, 0xAC, 0xEC, 0x81, 0xC3, 0x0A, 0x94,
+0x08, 0x4B, 0x29, 0xF1, 0x8C, 0x9C, 0x8C, 0xEA,
+0x42, 0x34, 0xFF, 0xF7, 0x5C, 0xC3, 0xFF, 0xF7,
+0x9D, 0xC3, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0xFF, 0xF7, 0x9E, 0xC3, 0xFF, 0xF7, 0x5F, 0xC3,
+0x6A, 0xEF, 0xB7, 0x61, 0xD2, 0x16, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x04, 0x6B,
+0x40, 0x32, 0x53, 0xF7, 0x78, 0xC2, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x15, 0xF0,
+0xA9, 0xA3, 0x15, 0xF0, 0x88, 0xA3, 0x15, 0xF0,
+0x4A, 0xA3, 0xA0, 0x35, 0x8D, 0xED, 0x15, 0xF0,
+0x8B, 0xA3, 0x40, 0x32, 0x40, 0x32, 0x4D, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC, 0xA1, 0xA4,
+0x40, 0xA4, 0xC2, 0xA4, 0xA0, 0x35, 0x4D, 0xED,
+0x43, 0xA4, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0x0A, 0xD3, 0xAD, 0xEA,
+0x80, 0xF3, 0xD9, 0xA2, 0x80, 0xF3, 0x98, 0xA2,
+0x80, 0xF3, 0xBA, 0xA2, 0xC0, 0x36, 0x8D, 0xEE,
+0x80, 0xF3, 0x9B, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xF7, 0xF0, 0x01, 0x6B, 0xAD, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0x60, 0x33, 0xCD, 0xEC, 0x60, 0x33,
+0x73, 0xF7, 0x84, 0xDB, 0x80, 0xF3, 0xDD, 0xA2,
+0x80, 0xF3, 0x9C, 0xA2, 0x80, 0xF3, 0xBE, 0xA2,
+0xC0, 0x36, 0x8D, 0xEE, 0x80, 0xF3, 0x9F, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xF7, 0xF0, 0x01, 0x69,
+0xAD, 0xEE, 0x20, 0x31, 0x00, 0xF6, 0x80, 0x34,
+0xCD, 0xEC, 0x20, 0x31, 0x73, 0xF7, 0x80, 0xD9,
+0xA0, 0xF3, 0xC1, 0xA2, 0xA0, 0xF3, 0x80, 0xA2,
+0xA0, 0xF3, 0xA2, 0xA2, 0xC0, 0x36, 0x8D, 0xEE,
+0xA0, 0xF3, 0x83, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xF7, 0xF0, 0x01, 0x68, 0xAD, 0xEE, 0x00, 0x30,
+0x00, 0xF6, 0x80, 0x34, 0xCD, 0xEC, 0x00, 0x30,
+0x53, 0xF7, 0x9C, 0xD8, 0xA0, 0xF3, 0xA5, 0xA2,
+0xA0, 0xF3, 0x86, 0xA2, 0xA0, 0xF3, 0x64, 0xA2,
+0xA0, 0xF3, 0xE7, 0xA2, 0xA0, 0x35, 0x80, 0x34,
+0xAD, 0xEB, 0x80, 0x34, 0xF7, 0xF0, 0x01, 0x6A,
+0x6D, 0xEC, 0x00, 0xF6, 0xE0, 0x37, 0x40, 0x32,
+0x8D, 0xEF, 0x40, 0x32, 0x53, 0xF7, 0xF4, 0xDA,
+0x03, 0x6A, 0x49, 0xC7, 0xF7, 0xF0, 0x01, 0x6C,
+0x0D, 0x6A, 0x48, 0xC7, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x6A, 0x80, 0x34, 0x40, 0x32, 0x08, 0xD7,
+0x40, 0x32, 0x29, 0xF1, 0x74, 0x9C, 0x72, 0xF4,
+0x58, 0x9A, 0x0D, 0x6C, 0xA3, 0x67, 0x00, 0x6E,
+0x40, 0xEA, 0x09, 0xD3, 0x08, 0x97, 0x42, 0x34,
+0x0A, 0x93, 0x44, 0xC7, 0x00, 0xF6, 0x42, 0x32,
+0x47, 0xC7, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x85, 0xC7, 0x40, 0x32, 0x82, 0x34, 0x86, 0xC7,
+0x53, 0xF7, 0xF4, 0x9A, 0xA5, 0xA7, 0x44, 0xA7,
+0x86, 0xA7, 0xA0, 0x35, 0x4D, 0xED, 0x80, 0x34,
+0x47, 0xA7, 0x80, 0x34, 0x8D, 0xED, 0xF7, 0xF0,
+0x01, 0x6C, 0x00, 0xF6, 0x40, 0x32, 0x80, 0x34,
+0x80, 0x34, 0xAD, 0xEA, 0x08, 0xD3, 0x0A, 0xD4,
+0x1C, 0x2A, 0x07, 0xD2, 0x7D, 0x67, 0x12, 0x6A,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4A, 0xF0, 0x08, 0x4A, 0x05, 0xD2,
+0x80, 0xF1, 0x0D, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0x00, 0x6A,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x09, 0x95, 0x89, 0xA7, 0xF7, 0xF0,
+0x01, 0x6B, 0x72, 0xF4, 0x58, 0x9A, 0x60, 0x33,
+0x60, 0x33, 0x00, 0x6E, 0x0B, 0xD3, 0x40, 0xEA,
+0x09, 0xD7, 0x09, 0x97, 0x42, 0x34, 0x0B, 0x93,
+0x40, 0xC7, 0x00, 0xF6, 0x42, 0x32, 0x43, 0xC7,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF7, 0xD4, 0x9A, 0x81, 0xC7, 0x82, 0x34,
+0x82, 0xC7, 0xA1, 0xA6, 0x40, 0xA6, 0x82, 0xA6,
+0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA6, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x0E, 0x2A, 0x07, 0xD2, 0x7D, 0x67,
+0x12, 0x6A, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4A, 0xF0, 0x08, 0x4A,
+0x05, 0xD2, 0x80, 0xF1, 0x14, 0x6A, 0xB6, 0x17,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0x13, 0xF6, 0x44, 0x9F, 0x73, 0xF7, 0x84, 0x9B,
+0x40, 0x6E, 0x84, 0x6D, 0x0B, 0xD7, 0x40, 0xEA,
+0x09, 0xD3, 0x09, 0x93, 0x0B, 0x97, 0xAF, 0x22,
+0x13, 0xF6, 0x44, 0x9F, 0x53, 0xF7, 0x9C, 0x98,
+0x40, 0x6E, 0x84, 0x6D, 0x0B, 0xD3, 0x40, 0xEA,
+0x09, 0xD7, 0x09, 0x97, 0x0B, 0x93, 0xA3, 0x22,
+0x13, 0xF6, 0x44, 0x9F, 0x73, 0xF7, 0x80, 0x99,
+0x40, 0x6E, 0x84, 0x6D, 0x40, 0xEA, 0x09, 0xD3,
+0x9A, 0x22, 0x09, 0x93, 0x00, 0x6D, 0x0A, 0x92,
+0x53, 0xF7, 0x94, 0x9A, 0x49, 0xA4, 0x43, 0xED,
+0x5C, 0x61, 0x08, 0x92, 0x15, 0xF0, 0xA8, 0xA2,
+0x15, 0xF0, 0x49, 0xA2, 0x40, 0x32, 0xAD, 0xEA,
+0x08, 0x95, 0x15, 0xF0, 0xCA, 0xA5, 0x15, 0xF0,
+0xAB, 0xA5, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEA,
+0x00, 0xF6, 0xA0, 0x35, 0x4D, 0xED, 0xC1, 0xA5,
+0x40, 0xA5, 0xE2, 0xA5, 0xC0, 0x36, 0x4D, 0xEE,
+0x43, 0xA5, 0xE0, 0x37, 0x73, 0xF7, 0xA4, 0xA3,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x40, 0x32,
+0xCD, 0xEA, 0x80, 0xF3, 0xB8, 0xC2, 0x73, 0xF7,
+0xA5, 0xA3, 0x80, 0xF3, 0xB9, 0xC2, 0x73, 0xF7,
+0xA6, 0xA3, 0x73, 0xF7, 0x67, 0xA3, 0x80, 0xF3,
+0xBA, 0xC2, 0x80, 0xF3, 0x7B, 0xC2, 0x73, 0xF7,
+0x60, 0xA1, 0x80, 0xF3, 0x7C, 0xC2, 0x73, 0xF7,
+0x61, 0xA1, 0x80, 0xF3, 0x7D, 0xC2, 0x73, 0xF7,
+0x62, 0xA1, 0x80, 0xF3, 0x7E, 0xC2, 0x73, 0xF7,
+0x63, 0xA1, 0x80, 0xF3, 0x7F, 0xC2, 0x53, 0xF7,
+0x7C, 0xA0, 0xA0, 0xF3, 0x60, 0xC2, 0x53, 0xF7,
+0x7D, 0xA0, 0xA0, 0xF3, 0x61, 0xC2, 0x53, 0xF7,
+0x7E, 0xA0, 0xA0, 0xF3, 0x62, 0xC2, 0x53, 0xF7,
+0x7F, 0xA0, 0xA0, 0xF3, 0x84, 0xC2, 0xA0, 0xF3,
+0x63, 0xC2, 0x82, 0x33, 0xA0, 0xF3, 0x65, 0xC2,
+0x00, 0xF6, 0x82, 0x34, 0x62, 0x33, 0xA0, 0xF3,
+0x66, 0xC2, 0xA0, 0xF3, 0x87, 0xC2, 0x01, 0x6A,
+0x37, 0x17, 0xC1, 0xA4, 0x40, 0xA4, 0xE2, 0xA4,
+0xC0, 0x36, 0x4D, 0xEE, 0x43, 0xA4, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x40, 0x32,
+0xCD, 0xEA, 0xA9, 0xE2, 0x00, 0x6C, 0x80, 0xC2,
+0x01, 0x4D, 0xFF, 0x6A, 0x4C, 0xED, 0x8B, 0x17,
+0x20, 0xE8, 0x01, 0x6A, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD0, 0x20, 0x24, 0x01, 0x74, 0x3E, 0x60,
+0x02, 0x74, 0x2C, 0x60, 0x00, 0x6A, 0x07, 0xD2,
+0x7D, 0x67, 0x07, 0x6A, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4A, 0xF0,
+0x08, 0x4A, 0x05, 0xD2, 0xC0, 0xF1, 0x12, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x00, 0x6A, 0x09, 0x97, 0x08, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x00, 0x68, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x40, 0x32, 0x60, 0x33, 0xD2, 0xF4,
+0x58, 0x9A, 0x29, 0xF1, 0x98, 0x9B, 0x40, 0xEA,
+0x0F, 0x6D, 0x0E, 0x28, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD2, 0xF4, 0x58, 0x9A,
+0x29, 0xF1, 0x90, 0x9B, 0x40, 0xEA, 0x3F, 0x6D,
+0x01, 0x6A, 0xDC, 0x17, 0x01, 0x68, 0xDF, 0x17,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0x15, 0xF0, 0x89, 0xA5, 0x15, 0xF0, 0x48, 0xA5,
+0x15, 0xF0, 0x6A, 0xA5, 0x80, 0x34, 0x4D, 0xEC,
+0x15, 0xF0, 0x4B, 0xA5, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x81, 0xA2, 0x60, 0xA2, 0xA2, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x63, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0xA0, 0xF3, 0x8D, 0xA3, 0xA0, 0xF3, 0x4C, 0xA3,
+0xA0, 0xF3, 0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0xA0, 0xF3, 0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x00, 0x6B, 0x60, 0xC2, 0x61, 0xC2, 0x62, 0xC2,
+0x63, 0xC2, 0x64, 0xC2, 0x65, 0xC2, 0x66, 0xC2,
+0x67, 0xC2, 0x68, 0xC2, 0x69, 0xC2, 0x6A, 0xC2,
+0x6B, 0xC2, 0x6C, 0xC2, 0x6D, 0xC2, 0x6E, 0xC2,
+0x20, 0xE8, 0x6F, 0xC2, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x33, 0xF5, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xB1, 0x18, 0x0E, 0xDC, 0x00, 0x65, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x11, 0x2D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0x04, 0xF0, 0x00, 0x6D, 0x40, 0xEA,
+0x01, 0x6C, 0x06, 0x68, 0x50, 0x67, 0x0D, 0x97,
+0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63,
+0x00, 0x6A, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5, 0x58, 0x9A,
+0x25, 0x67, 0x00, 0x6E, 0x40, 0xEA, 0x04, 0x05,
+0x02, 0x67, 0xEC, 0x2A, 0x04, 0x92, 0x71, 0x67,
+0xA1, 0xA2, 0x80, 0xA2, 0xC2, 0xA2, 0xA0, 0x35,
+0x8D, 0xED, 0x83, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC,
+0x83, 0xE9, 0x0F, 0x61, 0xE5, 0xA2, 0xA4, 0xA2,
+0xC6, 0xA2, 0xE0, 0x37, 0xAD, 0xEF, 0xA7, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEF, 0x00, 0xF6,
+0xA0, 0x35, 0xED, 0xED, 0xB5, 0xE4, 0xA3, 0xE9,
+0x0B, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x40, 0x6D,
+0x40, 0xEA, 0x02, 0x6C, 0x07, 0x68, 0xC2, 0x17,
+0x83, 0xE1, 0x96, 0xA2, 0x32, 0xA2, 0xFF, 0x6D,
+0x06, 0xEC, 0x93, 0xA2, 0xFF, 0xF7, 0x1F, 0x6E,
+0xF0, 0x67, 0x80, 0x34, 0x2D, 0xEC, 0xAC, 0xEF,
+0xCC, 0xEC, 0x83, 0xEF, 0x0D, 0x60, 0x90, 0xA2,
+0x07, 0x67, 0x2C, 0xA2, 0xCC, 0xEC, 0xE7, 0xEC,
+0xEC, 0xED, 0x87, 0x67, 0xED, 0xA2, 0xCC, 0xED,
+0xE0, 0x37, 0x2D, 0xEF, 0xE3, 0xED, 0x0B, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x10, 0x6D, 0x40, 0xEA,
+0x02, 0x6C, 0x08, 0x68, 0x9B, 0x17, 0xAF, 0xA2,
+0xF7, 0xF0, 0x01, 0x69, 0xEE, 0xA2, 0x20, 0x31,
+0x20, 0x31, 0xA0, 0x35, 0x8D, 0xF6, 0x58, 0xA1,
+0xED, 0xED, 0xCC, 0xED, 0x00, 0x6E, 0x06, 0xD6,
+0x0E, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x09, 0xD3,
+0x08, 0xD5, 0x40, 0xEA, 0x07, 0xD4, 0x09, 0x93,
+0x08, 0x95, 0x07, 0x94, 0x06, 0xD2, 0xBA, 0xE8,
+0x01, 0x2D, 0xE5, 0xE8, 0x04, 0x92, 0x01, 0x6D,
+0xC9, 0xA2, 0xE8, 0xA2, 0xC0, 0x36, 0xED, 0xEE,
+0x1E, 0x65, 0xCA, 0xA2, 0xF8, 0x67, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xEF, 0xCB, 0xA2, 0x07, 0xD4,
+0x07, 0x04, 0x80, 0xA4, 0x00, 0xF6, 0xC0, 0x36,
+0xED, 0xEE, 0x88, 0x34, 0x9D, 0xE6, 0x00, 0x9F,
+0x10, 0xEC, 0xA4, 0xEC, 0xAC, 0xE8, 0x07, 0xD5,
+0x17, 0x20, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x80, 0x6D,
+0x40, 0xEA, 0x02, 0x6C, 0x8D, 0xF6, 0x58, 0xA1,
+0x09, 0x68, 0x5F, 0xF7, 0x13, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x06, 0x94, 0x4A, 0x17,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x08, 0xD7,
+0x80, 0x34, 0x93, 0xF6, 0x80, 0x9C, 0xD4, 0xA2,
+0x55, 0xA2, 0x3C, 0x65, 0x00, 0x6D, 0x40, 0x32,
+0x4D, 0xEE, 0x59, 0x67, 0x40, 0xEA, 0x83, 0x67,
+0x08, 0x97, 0x07, 0x92, 0x04, 0x94, 0x60, 0x9F,
+0x6D, 0xEA, 0x40, 0xDF, 0x59, 0xA4, 0xB8, 0xA4,
+0xFF, 0xF7, 0x1F, 0x6B, 0x40, 0x32, 0xAD, 0xEA,
+0x01, 0x4A, 0x6C, 0xEA, 0x07, 0xD2, 0x42, 0x32,
+0x59, 0xC4, 0xF7, 0xF0, 0x01, 0x6A, 0x07, 0x03,
+0x40, 0x32, 0x60, 0xA3, 0x40, 0x32, 0xB3, 0xF5,
+0x5C, 0x9A, 0x78, 0xC4, 0x40, 0xEA, 0x00, 0x6C,
+0x8D, 0xF6, 0x58, 0xA1, 0xC6, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0x00, 0x6A, 0xF7, 0xF0, 0x01, 0x68, 0x07, 0xD2,
+0x00, 0x30, 0xF7, 0xF0, 0x01, 0x6A, 0x00, 0x30,
+0x40, 0x32, 0x29, 0xF1, 0xDC, 0x98, 0x40, 0x32,
+0xF2, 0xF4, 0x44, 0x9A, 0x11, 0xD5, 0x24, 0x67,
+0x86, 0x67, 0x40, 0xEA, 0x09, 0xD6, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x09, 0x96,
+0xD2, 0xF4, 0xF8, 0x9B, 0x04, 0x6D, 0xAB, 0xED,
+0x4C, 0xED, 0x86, 0x67, 0x40, 0xEF, 0x08, 0xD3,
+0x09, 0xD0, 0x08, 0x93, 0x62, 0x21, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x20, 0x31, 0xD2, 0xF4,
+0x54, 0x99, 0x52, 0xF0, 0x08, 0x6C, 0x03, 0x67,
+0x40, 0xEA, 0x0B, 0xD3, 0x08, 0xD2, 0xD2, 0xF4,
+0x54, 0x99, 0x12, 0xF0, 0x08, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x08, 0x96, 0xE1, 0xF7, 0x1F, 0x6B,
+0xF7, 0xF0, 0x01, 0x6C, 0x6C, 0xEE, 0x03, 0x6B,
+0x6C, 0xEA, 0xC4, 0xEA, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x6A, 0x80, 0x34, 0x40, 0x32, 0x49, 0xF1,
+0x60, 0x9C, 0x40, 0x32, 0xF2, 0xF4, 0x44, 0x9A,
+0x83, 0x67, 0x0A, 0xD6, 0x40, 0xEA, 0x08, 0xD3,
+0x0B, 0x93, 0x0A, 0x96, 0x00, 0xF2, 0x00, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0xD2, 0xF4, 0x78, 0x9B,
+0x08, 0x94, 0xE0, 0xF1, 0x1F, 0x6D, 0xCC, 0xED,
+0x40, 0xEB, 0x4D, 0xED, 0xD2, 0xF4, 0x54, 0x99,
+0x51, 0xF4, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x0A, 0xD2, 0xD2, 0xF4, 0x54, 0x99, 0x11, 0xF4,
+0x08, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x0A, 0x93,
+0xE1, 0xF7, 0x1F, 0x6C, 0x8C, 0xEB, 0x03, 0x6C,
+0x8C, 0xEA, 0x64, 0xEA, 0x01, 0x73, 0x1A, 0x61,
+0x07, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4A, 0xF0,
+0x08, 0x4A, 0x05, 0xD2, 0x60, 0xF2, 0x06, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x08, 0x94, 0xF2, 0xF4,
+0x44, 0x9A, 0x40, 0xEA, 0x0A, 0xD3, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x49, 0xF1,
+0xA4, 0x9C, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xAC, 0xEA, 0x49, 0xF1, 0xA8, 0x9C,
+0xD2, 0xF4, 0xD8, 0x98, 0x08, 0x94, 0x40, 0xEE,
+0x4D, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x0A, 0x93,
+0x40, 0x32, 0x40, 0x32, 0x49, 0xF1, 0x4C, 0x9A,
+0x60, 0x35, 0xF7, 0xF0, 0x01, 0x6C, 0xA0, 0x35,
+0x80, 0x34, 0x4C, 0xED, 0x80, 0x34, 0xE3, 0xF7,
+0x1F, 0x6A, 0x4C, 0xEB, 0x49, 0xF1, 0x90, 0x9C,
+0xD2, 0xF4, 0x58, 0x98, 0x6D, 0xED, 0x40, 0xEA,
+0x08, 0xD3, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xD2, 0xF4, 0x58, 0x98, 0x49, 0xF1,
+0x94, 0x9C, 0x40, 0xEA, 0x00, 0x6D, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x08, 0x93, 0x80, 0x34,
+0xD2, 0xF4, 0x58, 0x98, 0x49, 0xF1, 0x98, 0x9C,
+0x40, 0xEA, 0xA3, 0x67, 0x08, 0x93, 0xD2, 0xF4,
+0x58, 0x98, 0xA3, 0x67, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x49, 0xF1, 0x9C, 0x9B,
+0x40, 0xEA, 0x00, 0x65, 0x11, 0x92, 0x1F, 0x2A,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x69, 0xF1, 0x80, 0x9B, 0xD2, 0xF4, 0x58, 0x98,
+0x40, 0xEA, 0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x58, 0x98,
+0x69, 0xF1, 0x84, 0x9B, 0x40, 0xEA, 0x00, 0x6D,
+0x00, 0x6A, 0x03, 0x6D, 0x4D, 0xED, 0x09, 0x92,
+0xD2, 0xF4, 0x78, 0x98, 0x29, 0xF1, 0x9C, 0x9A,
+0x40, 0xEB, 0x00, 0x65, 0x86, 0x17, 0x11, 0x92,
+0x01, 0x72, 0x32, 0x61, 0xD2, 0xF4, 0x54, 0x99,
+0x52, 0xF0, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xE1, 0xF7, 0x1F, 0x6D, 0x4C, 0xED, 0xD2, 0xF4,
+0x54, 0x99, 0x12, 0xF0, 0x08, 0x6C, 0x40, 0xEA,
+0x08, 0xD5, 0x03, 0x6B, 0x6C, 0xEA, 0x08, 0x95,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x69, 0xF1, 0x68, 0x9B, 0xA4, 0xEA, 0xA0, 0x35,
+0xA0, 0x35, 0x6C, 0xED, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x69, 0xF1, 0x80, 0x9B,
+0xD2, 0xF4, 0x58, 0x98, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x58, 0x98, 0x69, 0xF1, 0x84, 0x9B,
+0x40, 0xEA, 0x00, 0x6D, 0x04, 0x6A, 0xC1, 0x17,
+0x07, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4A, 0xF0,
+0x08, 0x4A, 0x05, 0xD2, 0x80, 0xF2, 0x0E, 0x6A,
+0x3B, 0x17, 0x00, 0x65, 0xF4, 0x63, 0x17, 0x62,
+0x16, 0xD1, 0x15, 0xD0, 0x0E, 0x2C, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6,
+0x50, 0x9A, 0x69, 0xF1, 0xAC, 0x9B, 0x00, 0x6C,
+0x5D, 0x10, 0x20, 0xF0, 0x59, 0xA4, 0x64, 0x67,
+0x20, 0xF0, 0x38, 0xA4, 0x0E, 0xD2, 0x20, 0xF0,
+0x55, 0xA4, 0x20, 0xF0, 0x14, 0xA4, 0x20, 0xF0,
+0x96, 0xA4, 0x40, 0x32, 0x0D, 0xEA, 0x20, 0xF0,
+0x17, 0xA3, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEA,
+0x00, 0xF6, 0x00, 0x30, 0x01, 0x71, 0x4D, 0xE8,
+0x15, 0x61, 0x14, 0x28, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6, 0x50, 0x9A,
+0x69, 0xF1, 0xAC, 0x9B, 0x40, 0xEA, 0x00, 0x6C,
+0x51, 0x67, 0x17, 0x97, 0x16, 0x91, 0x15, 0x90,
+0x00, 0xEF, 0x0C, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF5, 0x40, 0x9A,
+0x04, 0x06, 0xB1, 0x67, 0x00, 0x6C, 0x40, 0xEA,
+0x0F, 0xD3, 0x0F, 0x93, 0x26, 0x22, 0x20, 0xF0,
+0x58, 0xA3, 0x01, 0x72, 0x17, 0x61, 0x40, 0xA0,
+0x01, 0x6B, 0x6C, 0xEA, 0x13, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x85, 0xA0,
+0x72, 0xF4, 0xB0, 0x9A, 0x44, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x46, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x87, 0xA0, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xED, 0x4D, 0xEC, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A,
+0x08, 0x6D, 0x02, 0x6C, 0x40, 0xEA, 0x01, 0x69,
+0xC7, 0x17, 0x0E, 0x92, 0xFF, 0x4A, 0x08, 0x5A,
+0x1C, 0x61, 0x20, 0xF0, 0x58, 0xA3, 0x01, 0x72,
+0x8A, 0x61, 0x40, 0xA0, 0x01, 0x6B, 0x6C, 0xEA,
+0x86, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x85, 0xA0, 0x72, 0xF4, 0xB0, 0x9A,
+0x44, 0xA0, 0x80, 0x34, 0x4D, 0xEC, 0x46, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x87, 0xA0,
+0x00, 0xF6, 0x80, 0x34, 0x40, 0xED, 0x4D, 0xEC,
+0x72, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x11, 0xD3,
+0x40, 0xEA, 0x28, 0x31, 0xF7, 0xF0, 0x01, 0x6B,
+0x0F, 0xD2, 0x60, 0x33, 0xF7, 0xF0, 0x01, 0x6A,
+0x60, 0x33, 0x40, 0x32, 0x29, 0xF1, 0x84, 0x9B,
+0x40, 0x32, 0x06, 0x93, 0xF2, 0xF4, 0x44, 0x9A,
+0x40, 0xEA, 0x91, 0xE3, 0x07, 0x93, 0x08, 0x97,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0x46, 0xEB, 0x4C, 0xEF, 0x15, 0xF0, 0x49, 0xA5,
+0x15, 0xF0, 0x88, 0xA5, 0x15, 0xF0, 0xCA, 0xA5,
+0x40, 0x32, 0x8D, 0xEA, 0x15, 0xF0, 0x8B, 0xA5,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEA, 0x00, 0xF6,
+0x80, 0x34, 0x4D, 0xEC, 0xC1, 0xA4, 0x40, 0xA4,
+0xC0, 0x36, 0x4D, 0xEE, 0x1E, 0x65, 0xC2, 0xA4,
+0x58, 0x67, 0xC0, 0x36, 0xC0, 0x36, 0x4D, 0xEE,
+0x43, 0xA4, 0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA,
+0xA0, 0xF3, 0x8D, 0xA2, 0xA0, 0xF3, 0xCE, 0xA2,
+0xA0, 0xF3, 0x6C, 0xA2, 0xA0, 0xF3, 0x4F, 0xA2,
+0x80, 0x34, 0xC0, 0x36, 0x8D, 0xEB, 0xC0, 0x36,
+0x6D, 0xEE, 0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA,
+0x45, 0xE1, 0xC1, 0xA1, 0x40, 0xA1, 0x82, 0xA1,
+0xC0, 0x36, 0xCD, 0xEA, 0xC3, 0xA1, 0x80, 0x34,
+0x80, 0x34, 0x4D, 0xEC, 0x00, 0xF6, 0xC0, 0x36,
+0xF7, 0xF0, 0x01, 0x6A, 0xCD, 0xEC, 0x40, 0x32,
+0x40, 0x32, 0xE3, 0xEC, 0x10, 0xD2, 0x11, 0x93,
+0xEB, 0xE4, 0x04, 0x60, 0x5D, 0x67, 0x57, 0xAA,
+0x89, 0xE2, 0xEB, 0xE2, 0x0E, 0x94, 0xFF, 0x4A,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x83, 0xEA,
+0x20, 0x31, 0x2D, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xB3, 0xF5, 0x58, 0x9A,
+0x00, 0x6C, 0x40, 0xEA, 0x0E, 0xD3, 0x72, 0xF0,
+0x54, 0x99, 0x0F, 0x94, 0x40, 0xEA, 0x02, 0x69,
+0x0E, 0x93, 0x20, 0xF0, 0x58, 0xA3, 0x01, 0x72,
+0x3F, 0xF7, 0x02, 0x61, 0x40, 0xA0, 0x01, 0x6B,
+0x6C, 0xEA, 0x1F, 0xF7, 0x1D, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x85, 0xA0,
+0x72, 0xF4, 0xB0, 0x9A, 0x44, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x46, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x87, 0xA0, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xED, 0x4D, 0xEC, 0x09, 0x17, 0x20, 0xF0,
+0x58, 0xA3, 0x01, 0x72, 0x26, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF5,
+0x5C, 0x9A, 0x83, 0x67, 0x00, 0x6E, 0x00, 0x6D,
+0x12, 0xD7, 0x40, 0xEA, 0x11, 0xD3, 0x11, 0x93,
+0x12, 0x97, 0x17, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0x69, 0xF1, 0xAC, 0x9B,
+0xD3, 0xF6, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x6C,
+0x72, 0xF0, 0x54, 0x99, 0x0F, 0x94, 0x40, 0xEA,
+0x04, 0x69, 0x40, 0xA0, 0x01, 0x6B, 0x6C, 0xEA,
+0xC0, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF5, 0x58, 0x9A, 0xA7, 0x67,
+0x83, 0x67, 0x40, 0xEA, 0x07, 0x67, 0x5D, 0x67,
+0x77, 0xAA, 0x0E, 0x92, 0x4B, 0xE3, 0x43, 0xE8,
+0x0E, 0x92, 0x15, 0xE2, 0x02, 0x61, 0x77, 0xE2,
+0x15, 0xE5, 0x08, 0x93, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x6C, 0xED, 0x07, 0x93, 0x40, 0x32,
+0xD2, 0xF4, 0x5C, 0x9A, 0xA4, 0xEB, 0x10, 0x93,
+0x29, 0xF1, 0x84, 0x9B, 0xFF, 0xF7, 0x1F, 0x6B,
+0x6C, 0xED, 0x06, 0x93, 0x40, 0xEA, 0x91, 0xE3,
+0x72, 0xF0, 0x54, 0x99, 0x0F, 0x94, 0x40, 0xEA,
+0x00, 0x69, 0xB2, 0x16, 0x20, 0xE8, 0x01, 0x6A,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x02, 0x74, 0xC0, 0xF0, 0x06, 0x60, 0x03, 0x5C,
+0x09, 0x60, 0x3C, 0x24, 0x01, 0x74, 0x80, 0xF0,
+0x11, 0x60, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x03, 0x74, 0xE0, 0xF0,
+0x07, 0x60, 0x04, 0x74, 0xF6, 0x61, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0x15, 0xF0,
+0xA9, 0xA6, 0x15, 0xF0, 0x68, 0xA6, 0x15, 0xF0,
+0x4A, 0xA6, 0xA0, 0x35, 0x6D, 0xED, 0x15, 0xF0,
+0x6B, 0xA6, 0x40, 0x32, 0x40, 0x32, 0x4D, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0xA1, 0xA3,
+0x40, 0xA3, 0xC2, 0xA3, 0xA0, 0x35, 0x4D, 0xED,
+0x43, 0xA3, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x65, 0xA2,
+0x04, 0xA2, 0xA6, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x07, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB,
+0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8, 0x80, 0xF2,
+0x00, 0x48, 0x2C, 0x10, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0x15, 0xF0, 0xA9, 0xA6,
+0x15, 0xF0, 0x68, 0xA6, 0x15, 0xF0, 0x4A, 0xA6,
+0xA0, 0x35, 0x6D, 0xED, 0x15, 0xF0, 0x6B, 0xA6,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xA1, 0xA3, 0x40, 0xA3,
+0xC2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x65, 0xA2, 0x04, 0xA2,
+0xA6, 0xA2, 0x60, 0x33, 0x0D, 0xEB, 0x07, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x24, 0x67, 0xA0, 0xF0,
+0xA0, 0x40, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF5, 0x5C, 0x9A, 0x91, 0x67,
+0x40, 0xEA, 0x04, 0xD5, 0x80, 0x6B, 0x6B, 0xEB,
+0x60, 0xC0, 0x61, 0xA0, 0x40, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEB, 0x61, 0xC0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x29, 0xF1, 0x6C, 0x9B,
+0x04, 0x95, 0x08, 0x48, 0x6C, 0xEA, 0x42, 0x33,
+0xFF, 0xF7, 0x5C, 0xC0, 0xFF, 0xF7, 0x7D, 0xC0,
+0x00, 0xF6, 0x42, 0x32, 0x62, 0x33, 0xFF, 0xF7,
+0x7E, 0xC0, 0xFF, 0xF7, 0x5F, 0xC0, 0x0A, 0xED,
+0xD8, 0x61, 0x6F, 0x17, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0x15, 0xF0, 0xA9, 0xA6,
+0x15, 0xF0, 0x68, 0xA6, 0x15, 0xF0, 0x4A, 0xA6,
+0xA0, 0x35, 0x6D, 0xED, 0x15, 0xF0, 0x6B, 0xA6,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xA1, 0xA3, 0x40, 0xA3,
+0xC2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x65, 0xA2, 0x04, 0xA2,
+0xA6, 0xA2, 0x60, 0x33, 0x0D, 0xEB, 0x07, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x7F, 0x48, 0x21, 0x48,
+0xA5, 0x17, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xC0, 0x36, 0x15, 0xF0, 0xA9, 0xA6, 0x15, 0xF0,
+0x68, 0xA6, 0x15, 0xF0, 0x4A, 0xA6, 0xA0, 0x35,
+0x6D, 0xED, 0x15, 0xF0, 0x6B, 0xA6, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0xA1, 0xA3, 0x40, 0xA3, 0xC2, 0xA3,
+0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA3, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x65, 0xA2, 0x04, 0xA2, 0xA6, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x07, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6, 0x00, 0x30,
+0x6D, 0xE8, 0x40, 0xF1, 0x00, 0x48, 0x76, 0x17,
+0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36,
+0x15, 0xF0, 0xA9, 0xA6, 0x15, 0xF0, 0x68, 0xA6,
+0x15, 0xF0, 0x4A, 0xA6, 0xA0, 0x35, 0x6D, 0xED,
+0x15, 0xF0, 0x6B, 0xA6, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0xA1, 0xA3, 0x40, 0xA3, 0xC2, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x65, 0xA2, 0x04, 0xA2, 0xA6, 0xA2, 0x60, 0x33,
+0x0D, 0xEB, 0x07, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEB, 0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8,
+0xE0, 0xF1, 0x00, 0x48, 0x47, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF6, 0x4C, 0x9A, 0x40, 0xEA, 0x04, 0x67,
+0x04, 0xD2, 0x04, 0x93, 0x00, 0x6A, 0x22, 0x23,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x22, 0x67, 0xF0, 0x67, 0x00, 0x6E, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB3, 0xF5,
+0x44, 0x9A, 0xA7, 0x67, 0x90, 0x67, 0x06, 0xD6,
+0x40, 0xEA, 0x05, 0xD7, 0x05, 0x97, 0x06, 0x96,
+0x0E, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0x00, 0x6A, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x05, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF5, 0x58, 0x9A, 0x87, 0x67, 0x40, 0xEA,
+0x06, 0xD6, 0x05, 0x95, 0x06, 0x96, 0x0B, 0x2A,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x72, 0xF0, 0xB4, 0x9C, 0x02, 0x67, 0x40, 0xED,
+0x91, 0x67, 0x50, 0x67, 0xE3, 0x17, 0x04, 0x92,
+0x01, 0x4E, 0xE5, 0x67, 0xCA, 0xEA, 0xC7, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x91, 0x67,
+0x01, 0x6A, 0xD4, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x40, 0xA4, 0x36, 0x22,
+0x01, 0x72, 0x01, 0x6A, 0x2E, 0x61, 0x33, 0xF0,
+0x04, 0x6A, 0x04, 0xD2, 0x24, 0x67, 0xA2, 0xA1,
+0x81, 0xA4, 0xF7, 0xF0, 0x01, 0x6B, 0xA0, 0x35,
+0x8D, 0xED, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x60, 0x33, 0x80, 0x34, 0x09, 0xF1, 0x9C, 0x9C,
+0x60, 0x33, 0xD2, 0xF4, 0x68, 0x9B, 0xFC, 0x4A,
+0x8D, 0xED, 0x40, 0xEB, 0x82, 0x67, 0x64, 0x68,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x04, 0x94,
+0x00, 0x52, 0x13, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x0A, 0x6C, 0x40, 0xEA, 0xFF, 0x48, 0xEC, 0x28,
+0x02, 0x6A, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x13, 0xF0, 0x04, 0x6A,
+0xCC, 0x17, 0xE1, 0xF7, 0x1F, 0x6B, 0x6C, 0xEA,
+0x42, 0x33, 0xE1, 0xF7, 0x1F, 0x72, 0x43, 0xC1,
+0x64, 0xC1, 0x03, 0x6A, 0xEE, 0x60, 0x00, 0x6A,
+0xEC, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x40, 0xA5, 0x79, 0x22,
+0x01, 0x72, 0x01, 0x6A, 0x71, 0x61, 0x33, 0xF0,
+0x1C, 0x6A, 0x05, 0xD2, 0x33, 0xF0, 0x18, 0x6B,
+0x33, 0xF0, 0x14, 0x6F, 0xF4, 0x4A, 0x05, 0x67,
+0x04, 0xD2, 0x07, 0xD7, 0x06, 0xD4, 0x85, 0xA0,
+0xC3, 0xA5, 0xA4, 0xA5, 0xFC, 0x65, 0x86, 0xA0,
+0xF7, 0xF0, 0x01, 0x69, 0xA0, 0x35, 0xCD, 0xED,
+0x20, 0x31, 0xE1, 0xF7, 0x1F, 0x6E, 0xFF, 0x67,
+0x20, 0x31, 0xCC, 0xED, 0x80, 0x34, 0xD2, 0xF4,
+0x48, 0x99, 0xED, 0xEC, 0xA0, 0x35, 0xCC, 0xEC,
+0xA0, 0x35, 0x8D, 0xED, 0x40, 0xEA, 0x83, 0x67,
+0x68, 0xA0, 0x07, 0x6A, 0xFF, 0x6D, 0x1B, 0x65,
+0x78, 0x67, 0x4C, 0xEB, 0xAC, 0xEB, 0x89, 0xA0,
+0x80, 0xF5, 0x60, 0x33, 0x1B, 0x65, 0x3F, 0x6B,
+0x6C, 0xEC, 0xAC, 0xEC, 0x80, 0x34, 0xD8, 0x67,
+0x80, 0x34, 0x8D, 0xEE, 0x8A, 0xA0, 0x07, 0x97,
+0x4C, 0xEC, 0xAC, 0xEC, 0x98, 0x34, 0xCD, 0xEC,
+0xCB, 0xA0, 0xD2, 0xF4, 0x48, 0x99, 0x6C, 0xEE,
+0xCC, 0xED, 0x8D, 0xED, 0x40, 0xEA, 0x87, 0x67,
+0x06, 0x92, 0xA7, 0xA0, 0x0F, 0x6C, 0xF7, 0xF0,
+0x01, 0x6F, 0x4C, 0xEC, 0xA0, 0x35, 0xE0, 0x37,
+0xA0, 0x35, 0x00, 0xF6, 0x80, 0x34, 0xE0, 0x37,
+0xAD, 0xEC, 0x09, 0xF1, 0xFC, 0x9F, 0xA1, 0xA0,
+0xD2, 0xF4, 0x48, 0x99, 0x64, 0x69, 0xED, 0xED,
+0x8D, 0xED, 0x40, 0xEA, 0x04, 0x94, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x40, 0xEA, 0x05, 0x94, 0x00, 0x52,
+0x18, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A, 0x0A, 0x6C,
+0x40, 0xEA, 0xFF, 0x49, 0xEC, 0x29, 0x02, 0x6A,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x13, 0xF0, 0x1C, 0x6A, 0x05, 0xD2,
+0x13, 0xF0, 0x18, 0x6B, 0x13, 0xF0, 0x14, 0x6F,
+0x89, 0x17, 0x06, 0x93, 0x02, 0x5B, 0x06, 0x60,
+0xE1, 0xF7, 0x1F, 0x6C, 0x8C, 0xEA, 0x4C, 0xC0,
+0x42, 0x32, 0x4D, 0xC0, 0x00, 0x6A, 0xE8, 0x17,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xB1, 0xE4, 0x40, 0xA4, 0xA6, 0xA4, 0x7A, 0x5A,
+0x27, 0x60, 0x74, 0x5A, 0x0A, 0x60, 0x2D, 0x5A,
+0x11, 0x60, 0x2B, 0x5A, 0x06, 0x60, 0x21, 0x5A,
+0x0A, 0x60, 0x1F, 0x5A, 0x02, 0x60, 0x0F, 0x72,
+0x10, 0x2A, 0x01, 0x6A, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0xD9, 0x4A,
+0x01, 0x6B, 0x06, 0x10, 0x51, 0x5A, 0x08, 0x60,
+0x4F, 0x5A, 0xF3, 0x60, 0xD1, 0x4A, 0x11, 0x6B,
+0x43, 0xEB, 0xEF, 0x60, 0x00, 0x6A, 0xEE, 0x17,
+0x71, 0x72, 0x33, 0x60, 0x72, 0x5A, 0x02, 0x60,
+0xA1, 0x4A, 0xEE, 0x17, 0x72, 0x72, 0xF5, 0x17,
+0xA7, 0x72, 0x6C, 0x60, 0xA8, 0x5A, 0x17, 0x60,
+0xA1, 0x72, 0x5B, 0x60, 0xA2, 0x5A, 0x06, 0x60,
+0x80, 0x5A, 0xEC, 0x61, 0x8F, 0x5A, 0xD9, 0x61,
+0xA0, 0x72, 0xE7, 0x17, 0xA6, 0x5A, 0xD5, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x40, 0x9A, 0xF9, 0xF6, 0x1C, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xCA, 0x17, 0xC0, 0x72,
+0xC8, 0x60, 0xC1, 0x5A, 0x08, 0x60, 0xB0, 0x72,
+0xC4, 0x60, 0xB0, 0x5A, 0xD3, 0x61, 0x80, 0x4A,
+0xCE, 0x4A, 0x03, 0x6B, 0xCD, 0x17, 0xF2, 0x72,
+0xBC, 0x60, 0xFF, 0x72, 0xBA, 0x60, 0xF0, 0x72,
+0xC8, 0x17, 0x07, 0x75, 0x2C, 0x60, 0x08, 0x5D,
+0x25, 0x60, 0x2D, 0x25, 0x04, 0x75, 0x40, 0x68,
+0xC1, 0x61, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x69, 0xF1, 0x30, 0x9B, 0x40, 0x32, 0xF2, 0xF4,
+0x44, 0x9A, 0x91, 0x67, 0x40, 0xEA, 0x00, 0x30,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x49, 0xF1, 0xA4, 0x9B, 0x00, 0x30, 0x91, 0x67,
+0x4C, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x58, 0x9A, 0x40, 0xEA,
+0x0D, 0xED, 0x8F, 0x17, 0x0A, 0x75, 0x05, 0x60,
+0x0B, 0x75, 0x80, 0x68, 0xD9, 0x17, 0x5B, 0x68,
+0xD8, 0x17, 0x76, 0x68, 0xD6, 0x17, 0x00, 0x68,
+0xD4, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x44, 0x9A, 0xFF, 0x6B,
+0x01, 0x4B, 0xF8, 0xF6, 0x14, 0x6C, 0x40, 0xEA,
+0x6D, 0xED, 0x77, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x40, 0x9A,
+0xF9, 0xF4, 0x14, 0x6C, 0xA1, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA, 0x04, 0x67,
+0x05, 0xD2, 0x48, 0xA0, 0x01, 0x72, 0x14, 0x60,
+0x80, 0xF0, 0x1D, 0x22, 0x02, 0x72, 0x80, 0xF0,
+0x01, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x05, 0x94,
+0x40, 0xEA, 0x00, 0x69, 0x51, 0x67, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x58, 0x6C,
+0x04, 0xD2, 0x00, 0x6F, 0x00, 0x6B, 0xE9, 0xE0,
+0xFF, 0x6D, 0x80, 0xA2, 0xC2, 0xA2, 0xEC, 0xED,
+0x44, 0xA2, 0x7B, 0x2F, 0xFF, 0x69, 0x01, 0x49,
+0x2B, 0xE9, 0x2C, 0xEB, 0x8D, 0xEB, 0x00, 0x6C,
+0x02, 0x26, 0x01, 0xF0, 0x00, 0x6C, 0x6D, 0xEC,
+0x18, 0xF0, 0x01, 0x6B, 0x6B, 0xEB, 0x6C, 0xEC,
+0x40, 0x33, 0x78, 0x33, 0xFF, 0xF7, 0x1F, 0x6A,
+0x4C, 0xEB, 0x8D, 0xEB, 0x90, 0x67, 0x07, 0xD7,
+0xB1, 0x18, 0x4E, 0xDF, 0x06, 0xD3, 0x01, 0x72,
+0x22, 0x67, 0x06, 0x93, 0x07, 0x97, 0xC1, 0x61,
+0x01, 0x4F, 0x02, 0x77, 0xD8, 0x61, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0xD2, 0xF4,
+0x48, 0x9F, 0xA3, 0x67, 0x58, 0x6C, 0x40, 0xEA,
+0x06, 0xD7, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xD2, 0xF4, 0x54, 0x9B, 0xF4, 0x6C,
+0x40, 0xEA, 0x07, 0xD3, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x89, 0xF1, 0xA4, 0x9C,
+0xF7, 0xF0, 0x01, 0x6C, 0x06, 0x97, 0x80, 0x34,
+0x80, 0x34, 0xAC, 0xEA, 0x09, 0xF1, 0xB0, 0x9C,
+0xD2, 0xF4, 0xC8, 0x9F, 0xF4, 0x6C, 0x40, 0xEE,
+0x4D, 0xED, 0x07, 0x93, 0xC0, 0x6C, 0xD2, 0xF4,
+0x54, 0x9B, 0x40, 0xEA, 0x00, 0x65, 0x43, 0xD8,
+0x48, 0xA0, 0x06, 0x97, 0x01, 0x72, 0x05, 0x61,
+0xD2, 0xF4, 0x48, 0x9F, 0x04, 0x95, 0x40, 0xEA,
+0x58, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x05, 0x94, 0x88, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x69, 0xF1, 0x38, 0x9A,
+0xD2, 0xF4, 0xC8, 0x9B, 0xC0, 0xF2, 0x10, 0x6C,
+0xB1, 0x67, 0x40, 0xEE, 0x04, 0xD3, 0x04, 0x93,
+0xB1, 0x67, 0xC0, 0xF2, 0x14, 0x6C, 0xD2, 0xF4,
+0x68, 0x9B, 0x40, 0xEB, 0x00, 0x65, 0x00, 0x6A,
+0x7B, 0x17, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0x69, 0xF1, 0x3C, 0x99, 0x6C, 0xE9,
+0x80, 0x33, 0x60, 0x33, 0x2D, 0xEB, 0x00, 0x6C,
+0x06, 0x26, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x69, 0xF1, 0x94, 0x9C, 0x8D, 0xEB,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x89, 0xF1, 0x80, 0x9C, 0x80, 0xF7, 0x40, 0x32,
+0x8C, 0xEB, 0x4D, 0xEB, 0x7B, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD0, 0x5D, 0x67,
+0x7D, 0x67, 0x90, 0xC2, 0xB6, 0xC2, 0x01, 0x76,
+0x00, 0x6A, 0x07, 0x67, 0x52, 0xC3, 0x54, 0xC3,
+0x91, 0xC3, 0xB7, 0xC3, 0x53, 0xC3, 0x55, 0xC3,
+0x0D, 0x61, 0x01, 0x6A, 0x04, 0x04, 0xB1, 0x18,
+0xA2, 0xDF, 0x58, 0xC3, 0x01, 0x72, 0x02, 0x61,
+0x07, 0x93, 0x60, 0xD8, 0x09, 0x97, 0x08, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x7D, 0x67, 0xF2, 0x17,
+0xED, 0x63, 0x25, 0x62, 0x24, 0xD1, 0x23, 0xD0,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6B,
+0x80, 0x34, 0x80, 0x34, 0x60, 0x33, 0x60, 0x33,
+0x89, 0xF1, 0x28, 0x9C, 0x72, 0xF4, 0x54, 0x9B,
+0x0C, 0x6C, 0xB1, 0x67, 0x40, 0xEA, 0x20, 0xD3,
+0x20, 0x93, 0x19, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6A, 0xF0, 0x00, 0x4A, 0x05, 0xD2,
+0x91, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x25, 0x97, 0x24, 0x91,
+0x23, 0x90, 0x00, 0xEF, 0x13, 0x63, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x15, 0xF0,
+0x08, 0x4C, 0xAD, 0xA4, 0x02, 0x67, 0x4C, 0xA4,
+0xCE, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x4F, 0xA4,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x40, 0xF0, 0x91, 0xA2,
+0x40, 0xF0, 0xB2, 0xA2, 0x40, 0xF0, 0xD0, 0xA2,
+0x40, 0xF0, 0x53, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x01, 0xDA, 0x0E, 0x6A,
+0x40, 0xC0, 0x72, 0xF4, 0x54, 0x9B, 0xB1, 0x67,
+0x38, 0x6C, 0x40, 0xEA, 0x20, 0xD3, 0x41, 0xD8,
+0x20, 0x93, 0x0C, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6A, 0xF0, 0x00, 0x4A, 0x05, 0xD2,
+0xA0, 0x6A, 0xB7, 0x17, 0x72, 0xF4, 0x54, 0x9B,
+0xB1, 0x67, 0x40, 0xEA, 0x38, 0x6C, 0x42, 0xD8,
+0x0C, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x6A, 0xF0, 0x00, 0x4A, 0x05, 0xD2, 0xAA, 0x6A,
+0xA4, 0x17, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0xF7, 0xF0, 0x01, 0x69, 0xA0, 0x35, 0x20, 0x31,
+0x12, 0x04, 0x20, 0x31, 0xC8, 0xF1, 0x18, 0x4D,
+0xD1, 0x18, 0xE7, 0xF4, 0x38, 0x6E, 0x81, 0x98,
+0x73, 0xF6, 0x5C, 0x99, 0x12, 0x05, 0x40, 0xEA,
+0x38, 0x6E, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0xA0, 0x35, 0x04, 0x04, 0x08, 0xF2, 0x10, 0x4D,
+0xD1, 0x18, 0xE7, 0xF4, 0x38, 0x6E, 0x73, 0xF6,
+0x5C, 0x99, 0x82, 0x98, 0x38, 0x6E, 0x40, 0xEA,
+0x04, 0x05, 0x88, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x05, 0x24, 0xD2, 0xF4,
+0x40, 0x98, 0x69, 0x6D, 0x40, 0xEA, 0xCF, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x4C, 0x9A, 0xF7, 0xF0, 0x01, 0x69,
+0x40, 0xEA, 0xCC, 0x6C, 0xD2, 0xF4, 0x60, 0x98,
+0x04, 0x6D, 0xAD, 0xEA, 0xFF, 0x6D, 0x4C, 0xED,
+0x40, 0xEB, 0xCC, 0x6C, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x50, 0x9B,
+0x20, 0x31, 0x20, 0x31, 0x00, 0x6C, 0x40, 0xEA,
+0x04, 0xD3, 0xD2, 0xF4, 0xC4, 0x99, 0x08, 0xF0,
+0x00, 0x6D, 0xFF, 0xF7, 0x1F, 0x68, 0x4D, 0xED,
+0x0C, 0xED, 0x40, 0xEE, 0x00, 0x6C, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x4C, 0x9A, 0x40, 0xEA, 0x01, 0x6C, 0x04, 0x93,
+0x00, 0x6C, 0xD2, 0xF4, 0x50, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x10, 0xF0, 0x00, 0x6D, 0xD2, 0xF4,
+0xC4, 0x99, 0xAB, 0xED, 0x4D, 0xED, 0x0C, 0xED,
+0x40, 0xEE, 0x00, 0x6C, 0x04, 0x93, 0x00, 0x6C,
+0xD2, 0xF4, 0x50, 0x9B, 0x40, 0xEA, 0x00, 0x65,
+0xFF, 0x6D, 0x02, 0x4D, 0xD2, 0xF4, 0x64, 0x99,
+0xAB, 0xED, 0x4C, 0xED, 0x0C, 0xED, 0x40, 0xEB,
+0x00, 0x6C, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x80, 0x34, 0x80, 0x34,
+0x40, 0x32, 0x40, 0x32, 0x89, 0xF1, 0x2C, 0x9C,
+0xD2, 0xF4, 0x68, 0x9A, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0xB1, 0x67, 0x00, 0x30, 0x21, 0xF4,
+0x10, 0x6C, 0x40, 0xEB, 0x04, 0xD2, 0x73, 0xF6,
+0x68, 0x98, 0x40, 0xEB, 0xC8, 0x6C, 0x04, 0x92,
+0xB1, 0x67, 0x30, 0x6C, 0xD2, 0xF4, 0x48, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x73, 0xF6, 0x48, 0x98,
+0x40, 0xEA, 0xC8, 0x6C, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0xB1, 0x18, 0x9B, 0xE0, 0x05, 0xD4, 0xD2, 0xF4,
+0x50, 0x99, 0xF7, 0xF0, 0x01, 0x68, 0x40, 0xEA,
+0x00, 0x6C, 0xFF, 0x6D, 0x01, 0x4D, 0x4D, 0xED,
+0x00, 0x30, 0x00, 0x30, 0x04, 0x02, 0x04, 0xD5,
+0xD2, 0xF4, 0xE4, 0x98, 0xA0, 0xAA, 0x40, 0xEF,
+0x00, 0x6C, 0xD2, 0xF4, 0x50, 0x99, 0x40, 0xEA,
+0x00, 0x6C, 0xD2, 0xF4, 0xE4, 0x98, 0xEF, 0xF7,
+0x1F, 0x6D, 0x4C, 0xED, 0x40, 0xEF, 0x00, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x4C, 0x9A, 0x40, 0xEA, 0x01, 0x6C,
+0xD2, 0xF4, 0x50, 0x99, 0x40, 0xEA, 0x00, 0x6C,
+0x08, 0xF0, 0x01, 0x6D, 0xAB, 0xED, 0x4C, 0xED,
+0x04, 0xD5, 0x04, 0x02, 0xD2, 0xF4, 0xC4, 0x98,
+0xA0, 0xAA, 0xF7, 0xF0, 0x01, 0x68, 0x40, 0xEE,
+0x00, 0x6C, 0x05, 0x93, 0x00, 0x30, 0x00, 0x30,
+0x05, 0x23, 0xD2, 0xF4, 0x40, 0x98, 0x00, 0x6D,
+0x40, 0xEA, 0xCF, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0x40, 0xEA, 0xCC, 0x6C, 0xD2, 0xF4, 0x60, 0x98,
+0xFB, 0x6D, 0x4C, 0xED, 0x40, 0xEB, 0xCC, 0x6C,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33, 0x40, 0x32,
+0x60, 0x33, 0x89, 0xF1, 0x70, 0x9B, 0x40, 0x32,
+0x25, 0x67, 0xD2, 0xF4, 0x48, 0x9A, 0x80, 0x35,
+0xA0, 0x35, 0x21, 0xF4, 0x10, 0x6C, 0x40, 0xEA,
+0x6C, 0xED, 0x53, 0xF4, 0x00, 0x68, 0xFF, 0x48,
+0xFF, 0xF7, 0x1F, 0x6A, 0x4C, 0xE8, 0x0C, 0x28,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0xFF, 0x6D, 0x01, 0x4D,
+0x40, 0xEA, 0x00, 0x6C, 0x00, 0x6A, 0x1E, 0x10,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x21, 0xF4, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x89, 0xF1, 0x74, 0x9B,
+0x6C, 0xEA, 0xDD, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0x21, 0xF4, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x40, 0xC1, 0x01, 0x6A, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x0A, 0xD4, 0x0B, 0xD5, 0x10, 0x6C, 0x01, 0x6D,
+0x06, 0x67, 0xB1, 0x18, 0xDD, 0xE7, 0x01, 0x6E,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x6C, 0x9A, 0x01, 0x6C, 0x40, 0xEB,
+0x04, 0xD2, 0x02, 0x6E, 0x02, 0x6D, 0xB1, 0x18,
+0xDD, 0xE7, 0x10, 0x6C, 0x0A, 0x91, 0x0A, 0x92,
+0x0B, 0x93, 0x69, 0xE2, 0x43, 0xE9, 0x16, 0x61,
+0x02, 0x6E, 0x00, 0x6D, 0xB1, 0x18, 0xDD, 0xE7,
+0x10, 0x6C, 0x04, 0x92, 0x01, 0x6C, 0x73, 0xF6,
+0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x01, 0x6E,
+0x00, 0x6D, 0xB1, 0x18, 0xDD, 0xE7, 0x10, 0x6C,
+0x01, 0x6A, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFF, 0x6E, 0x40, 0x6D,
+0xB1, 0x18, 0xDD, 0xE7, 0x63, 0x6C, 0xFF, 0x6A,
+0xB1, 0x67, 0x4C, 0xED, 0xFF, 0x6E, 0x62, 0x6C,
+0xB1, 0x18, 0xDD, 0xE7, 0x05, 0xD2, 0x05, 0x92,
+0x22, 0x35, 0x07, 0x6E, 0x4C, 0xED, 0xB1, 0x18,
+0xDD, 0xE7, 0x63, 0x6C, 0xC0, 0x6E, 0x00, 0x6D,
+0xB1, 0x18, 0xDD, 0xE7, 0x63, 0x6C, 0x04, 0xF7,
+0x10, 0x6B, 0xFF, 0x4B, 0x0C, 0x2B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0xFF, 0x6D, 0x01, 0x4D, 0x40, 0xEA,
+0x00, 0x6C, 0x00, 0x6A, 0xD2, 0x17, 0x63, 0x6C,
+0xB1, 0x18, 0xF8, 0xE7, 0x05, 0xD3, 0x20, 0x6C,
+0x8C, 0xEA, 0x0A, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0x05, 0x93, 0xE1, 0x17,
+0xB1, 0x18, 0xF8, 0xE7, 0x7A, 0x6C, 0x40, 0xC0,
+0x01, 0x49, 0x01, 0x48, 0xA4, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x96, 0xF6, 0x0E, 0xAA, 0x00, 0x6A, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0x00, 0x6F, 0x08, 0x06,
+0xB0, 0x67, 0x40, 0xEA, 0x00, 0x6C, 0x22, 0x67,
+0x04, 0x6A, 0x21, 0x29, 0x08, 0x96, 0x24, 0x2E,
+0x02, 0x67, 0x06, 0x6B, 0x5D, 0x67, 0x70, 0xC2,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x6A, 0xF0, 0x00, 0x4B, 0x05, 0xD3, 0x20, 0x6B,
+0x06, 0xD3, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x12, 0xF1, 0x78, 0x9B, 0x40, 0xEB,
+0x04, 0x04, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xD3, 0xF6, 0x70, 0x9B, 0x10, 0x6D,
+0x40, 0xEB, 0x00, 0x6C, 0x50, 0x67, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0x96, 0xF6, 0x8C, 0xA5, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x0A, 0xD2, 0x60, 0x33, 0x17, 0x24,
+0x8C, 0xF2, 0x91, 0xA3, 0x14, 0x24, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xD3, 0xF6,
+0x70, 0x9B, 0xFF, 0x6D, 0x01, 0x4D, 0x40, 0xEB,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x72, 0xF4, 0x70, 0x9B, 0x40, 0xEB,
+0x08, 0x94, 0x0A, 0x92, 0xD8, 0x17, 0x8C, 0xF2,
+0x51, 0xA3, 0x0A, 0xD5, 0x77, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x33, 0xF5,
+0x48, 0x9A, 0x40, 0xEA, 0x00, 0x6C, 0x0A, 0x95,
+0x96, 0xF6, 0x4C, 0xA5, 0x68, 0x22, 0xCF, 0x40,
+0xE0, 0xF5, 0x00, 0x6B, 0x2F, 0xF2, 0x40, 0x43,
+0x03, 0xEA, 0x1C, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x33, 0xF5, 0x44, 0x9A,
+0x40, 0xEA, 0x00, 0x6C, 0x7D, 0x67, 0x01, 0x6A,
+0x52, 0xC3, 0x51, 0xC3, 0x00, 0x6A, 0x50, 0xC3,
+0x08, 0x48, 0xFF, 0xF7, 0x1F, 0x6A, 0x08, 0x95,
+0x4C, 0xE8, 0x5D, 0x67, 0x13, 0xC2, 0x04, 0x04,
+0x02, 0x30, 0xD1, 0x18, 0xA8, 0x19, 0x14, 0xC2,
+0x00, 0x6A, 0xA5, 0x17, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x6F, 0x8C, 0xEA, 0xE0, 0x37, 0x08, 0x95,
+0xE0, 0x37, 0xFF, 0xF7, 0x1F, 0x6C, 0x21, 0x2A,
+0x33, 0xF5, 0x50, 0x9F, 0x6C, 0xEC, 0x35, 0xE5,
+0x0B, 0xD6, 0x40, 0xEA, 0x0A, 0xD3, 0x0A, 0x93,
+0x0B, 0x96, 0x14, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A,
+0xFF, 0x6D, 0x01, 0x4D, 0x40, 0xEA, 0x00, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF4, 0x50, 0x9A, 0x40, 0xEA, 0x08, 0x94,
+0x04, 0x6A, 0x7D, 0x17, 0x01, 0x49, 0x01, 0x4B,
+0xB5, 0x17, 0x33, 0xF5, 0x50, 0x9F, 0xD5, 0xE5,
+0x6C, 0xEC, 0x0B, 0xD6, 0x40, 0xEA, 0x0A, 0xD3,
+0x0A, 0x93, 0x0B, 0x96, 0xDF, 0x22, 0xFF, 0x4E,
+0xF2, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x08, 0x93, 0x33, 0xF5, 0x50, 0x9A,
+0xFF, 0xF7, 0x1F, 0x6C, 0x35, 0xE3, 0x40, 0xEA,
+0x2C, 0xEC, 0xD0, 0x22, 0x01, 0x49, 0x03, 0xE9,
+0xF0, 0x61, 0x9C, 0x17, 0x60, 0x6D, 0xB1, 0x18,
+0x08, 0xE1, 0x00, 0x6C, 0x9F, 0x2A, 0xC6, 0x17,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x04, 0x67, 0x40, 0xEA,
+0x00, 0x30, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF3, 0x50, 0x9A, 0x3E, 0xF4,
+0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xD1, 0x18,
+0x73, 0x81, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x28, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0xE0, 0xF1, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x89, 0xF1, 0xB8, 0x9B,
+0xE0, 0xF1, 0x10, 0x6C, 0x4C, 0xED, 0x40, 0xE9,
+0x0D, 0xED, 0xD7, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x4D, 0xA3, 0x8E, 0xA3, 0x2C, 0xA3, 0x40, 0x32,
+0x80, 0x34, 0x2D, 0xEA, 0x80, 0x34, 0x2F, 0xA3,
+0x8D, 0xEA, 0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x68, 0x80, 0x34, 0x80, 0x34, 0x00, 0x30,
+0x09, 0xD3, 0x00, 0xF6, 0x20, 0x31, 0x89, 0xF1,
+0x7C, 0x9C, 0x00, 0x30, 0x4D, 0xE9, 0x72, 0xF4,
+0x54, 0x98, 0xA3, 0x67, 0x08, 0x6C, 0x40, 0xEA,
+0x08, 0xD3, 0x42, 0x34, 0x09, 0x93, 0x40, 0xF0,
+0x50, 0xC1, 0x40, 0xF0, 0x91, 0xC1, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x40, 0xF0, 0x92, 0xC1,
+0x40, 0xF0, 0x53, 0xC1, 0x8D, 0xA3, 0x4C, 0xA3,
+0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0xF0, 0xB1, 0xA2,
+0x40, 0xF0, 0x92, 0xA2, 0x40, 0xF0, 0xD0, 0xA2,
+0x40, 0xF0, 0x33, 0xA2, 0xA0, 0x35, 0x80, 0x34,
+0xAD, 0xEE, 0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6,
+0x20, 0x31, 0x8D, 0xE9, 0x1B, 0x29, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x6A, 0xF0, 0x00, 0x4A,
+0x05, 0xD2, 0xC0, 0xF1, 0x13, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x00, 0x6A, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0x08, 0x95, 0x72, 0xF4,
+0x54, 0x98, 0x06, 0x6C, 0x40, 0xEA, 0x09, 0xD3,
+0x09, 0x93, 0x40, 0xD9, 0x4D, 0xA3, 0xCC, 0xA3,
+0x8E, 0xA3, 0x40, 0x32, 0xCD, 0xEA, 0xCF, 0xA3,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6,
+0xC0, 0x36, 0x4D, 0xEE, 0x40, 0xF0, 0xB1, 0xA6,
+0x40, 0xF0, 0x50, 0xA6, 0x40, 0xF0, 0x92, 0xA6,
+0xA0, 0x35, 0x4D, 0xED, 0x40, 0xF0, 0x53, 0xA6,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0x9A, 0x0D, 0x29,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6A, 0xF0,
+0x00, 0x4A, 0x05, 0xD2, 0xC0, 0xF1, 0x1C, 0x6A,
+0xBE, 0x17, 0x08, 0x95, 0x72, 0xF4, 0x54, 0x98,
+0x04, 0x6C, 0x0A, 0xD6, 0x40, 0xEA, 0x09, 0xD3,
+0x0A, 0x96, 0x09, 0x93, 0x42, 0x34, 0x20, 0xF0,
+0x44, 0xC6, 0x20, 0xF0, 0x85, 0xC6, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x20, 0xF0, 0x86, 0xC6,
+0x20, 0xF0, 0x47, 0xC6, 0x8D, 0xA3, 0x4C, 0xA3,
+0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0, 0x85, 0xA2,
+0x20, 0xF0, 0xC4, 0xA2, 0x20, 0xF0, 0xA6, 0xA2,
+0x80, 0x34, 0xCD, 0xEC, 0x20, 0xF0, 0xC7, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0xC0, 0x36, 0x8D, 0xEE, 0x0D, 0x2E, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x6A, 0xF0, 0x00, 0x4A,
+0x05, 0xD2, 0xE0, 0xF1, 0x09, 0x6A, 0x7F, 0x17,
+0x08, 0x95, 0x72, 0xF4, 0x54, 0x98, 0x04, 0x6C,
+0x0A, 0xD6, 0x40, 0xEA, 0x09, 0xD3, 0x0A, 0x96,
+0x09, 0x93, 0x42, 0x34, 0x40, 0xC6, 0x81, 0xC6,
+0x00, 0xF6, 0x42, 0x32, 0x82, 0x34, 0x82, 0xC6,
+0x43, 0xC6, 0xAD, 0xA3, 0x8C, 0xA3, 0x4E, 0xA3,
+0xA0, 0x35, 0x8D, 0xED, 0x8F, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6, 0x80, 0x34,
+0xAD, 0xEC, 0x20, 0xF0, 0xA5, 0xA4, 0x20, 0xF0,
+0x44, 0xA4, 0x20, 0xF0, 0xC6, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x47, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x81, 0xA2, 0xA2, 0xA2, 0xC0, 0xA2,
+0x43, 0xA2, 0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE,
+0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x0D, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6A, 0xF0, 0x00, 0x4A, 0x05, 0xD2,
+0xE0, 0xF1, 0x12, 0x6A, 0x38, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x33, 0xF5,
+0x48, 0x9A, 0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30,
+0x00, 0x6C, 0x00, 0x30, 0x40, 0xEA, 0x08, 0xD3,
+0x33, 0xF5, 0x50, 0x98, 0xE0, 0xF5, 0x16, 0x6C,
+0x40, 0xEA, 0x04, 0x05, 0x08, 0x93, 0x3F, 0xF7,
+0x0B, 0x22, 0x5D, 0x67, 0x50, 0xA2, 0x07, 0x6E,
+0x9D, 0x67, 0x52, 0x32, 0xCC, 0xEA, 0x08, 0xD2,
+0x08, 0x02, 0x40, 0xA2, 0x20, 0xF0, 0xC8, 0xC4,
+0x50, 0xC4, 0x1B, 0x2A, 0x33, 0xF5, 0x50, 0x98,
+0x04, 0x05, 0xE0, 0xF5, 0x16, 0x6C, 0x40, 0xEA,
+0x09, 0xD3, 0x1F, 0xF7, 0x15, 0x22, 0x7D, 0x67,
+0x5D, 0x67, 0x50, 0xA2, 0x20, 0xF0, 0xC8, 0xA3,
+0x09, 0x93, 0x4C, 0xEE, 0x08, 0xD6, 0x08, 0x02,
+0xC0, 0xA2, 0x5D, 0x67, 0xD0, 0xC2, 0x05, 0x2E,
+0x1D, 0xF2, 0x0A, 0x6C, 0xB1, 0x18, 0xB6, 0xE1,
+0x00, 0x65, 0x5D, 0x67, 0xD0, 0xA2, 0xC0, 0xC1,
+0x4D, 0xA3, 0x8C, 0xA3, 0xAE, 0xA3, 0x40, 0x32,
+0x8D, 0xEA, 0x8F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEA, 0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC,
+0x08, 0xD3, 0x20, 0xF0, 0xA5, 0xA4, 0x20, 0xF0,
+0x44, 0xA4, 0x20, 0xF0, 0xE6, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x47, 0xA4, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x81, 0xA2, 0xA2, 0xA2, 0xE0, 0xA2,
+0x43, 0xA2, 0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEF,
+0xA0, 0x35, 0xED, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0xC0, 0xC2, 0x33, 0xF5, 0x50, 0x98,
+0xE0, 0xF5, 0x1A, 0x6C, 0x40, 0xEA, 0x04, 0x05,
+0xDF, 0xF6, 0x0E, 0x22, 0x5D, 0x67, 0x50, 0xA2,
+0x07, 0x6F, 0xFF, 0x6E, 0xEC, 0xEA, 0x7D, 0x67,
+0xCC, 0xEA, 0x20, 0xF0, 0xE8, 0xC3, 0x50, 0xC3,
+0x09, 0xD6, 0x08, 0x93, 0x13, 0x2A, 0x33, 0xF5,
+0x50, 0x98, 0xE0, 0xF5, 0x19, 0x6C, 0x40, 0xEA,
+0x04, 0x05, 0xBF, 0xF6, 0x19, 0x22, 0x7D, 0x67,
+0x5D, 0x67, 0x50, 0xA2, 0x20, 0xF0, 0xE8, 0xA3,
+0x09, 0x96, 0xEC, 0xEA, 0xCC, 0xEA, 0x50, 0xC3,
+0x08, 0x93, 0xA6, 0x22, 0x5D, 0x67, 0xD0, 0xA2,
+0xC1, 0xC1, 0x4D, 0xA3, 0x8C, 0xA3, 0xAE, 0xA3,
+0x40, 0x32, 0x8D, 0xEA, 0x8F, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEA, 0x00, 0xF6, 0x80, 0x34,
+0x4D, 0xEC, 0x08, 0xD3, 0x20, 0xF0, 0xA5, 0xA4,
+0x20, 0xF0, 0x44, 0xA4, 0x20, 0xF0, 0xE6, 0xA4,
+0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0, 0x47, 0xA4,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x81, 0xA2, 0xA2, 0xA2,
+0xE0, 0xA2, 0x43, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEF, 0xA0, 0x35, 0xED, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xC1, 0xC2, 0x33, 0xF5,
+0x50, 0x98, 0xE0, 0xF5, 0x1A, 0x6C, 0x40, 0xEA,
+0x04, 0x05, 0x7F, 0xF6, 0x19, 0x22, 0x5D, 0x67,
+0x50, 0xA2, 0x07, 0x6E, 0x7D, 0x67, 0x52, 0x32,
+0xCC, 0xEA, 0x20, 0xF0, 0xC4, 0xC3, 0x50, 0xC3,
+0x08, 0x93, 0x15, 0x2A, 0x33, 0xF5, 0x50, 0x98,
+0xE0, 0xF5, 0x19, 0x6C, 0x40, 0xEA, 0x04, 0x05,
+0x7F, 0xF6, 0x06, 0x22, 0x5D, 0x67, 0x7D, 0x67,
+0x50, 0xA2, 0x20, 0xF0, 0xC4, 0xA3, 0xFF, 0x6C,
+0x52, 0x32, 0xCC, 0xEA, 0x8C, 0xEA, 0x50, 0xC3,
+0x08, 0x93, 0x5F, 0xF7, 0x11, 0x22, 0x5D, 0x67,
+0xD0, 0xA2, 0xC2, 0xC1, 0x4D, 0xA3, 0x8C, 0xA3,
+0xAE, 0xA3, 0x40, 0x32, 0x8D, 0xEA, 0x8F, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEA, 0x00, 0xF6,
+0x80, 0x34, 0x4D, 0xEC, 0x08, 0xD3, 0x20, 0xF0,
+0xA5, 0xA4, 0x20, 0xF0, 0x44, 0xA4, 0x20, 0xF0,
+0xE6, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0,
+0x47, 0xA4, 0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0xA2, 0xA2,
+0x81, 0xA2, 0xE0, 0xA2, 0x43, 0xA2, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEF, 0xA0, 0x35, 0xED, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0xC2, 0xC2,
+0x33, 0xF5, 0x50, 0x98, 0xE0, 0xF5, 0x1D, 0x6C,
+0x40, 0xEA, 0x04, 0x05, 0x3F, 0xF6, 0x04, 0x22,
+0x5D, 0x67, 0x50, 0xA2, 0x03, 0x6E, 0x7D, 0x67,
+0x4A, 0x32, 0xCC, 0xEA, 0x20, 0xF0, 0xC4, 0xC3,
+0x50, 0xC3, 0x08, 0x93, 0x14, 0x2A, 0x33, 0xF5,
+0x50, 0x98, 0xE0, 0xF5, 0x1D, 0x6C, 0x40, 0xEA,
+0x04, 0x05, 0x1F, 0xF6, 0x11, 0x22, 0x7D, 0x67,
+0x5D, 0x67, 0x50, 0xA2, 0x20, 0xF0, 0xC4, 0xA3,
+0xFF, 0x6C, 0xCC, 0xEA, 0x8C, 0xEA, 0x50, 0xC3,
+0x08, 0x93, 0xFF, 0xF6, 0x1D, 0x22, 0x5D, 0x67,
+0x90, 0xA2, 0x83, 0xC1, 0x4D, 0xA3, 0xAE, 0xA3,
+0xCC, 0xA3, 0x6F, 0xA3, 0x40, 0x32, 0xA0, 0x35,
+0x4D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x20, 0xF0, 0xA5, 0xA3,
+0x20, 0xF0, 0x44, 0xA3, 0x20, 0xF0, 0xC6, 0xA3,
+0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0, 0x47, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x61, 0xA2, 0xA2, 0xA2,
+0xC0, 0xA2, 0x43, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x83, 0xC2, 0x33, 0xF5,
+0x50, 0x98, 0xE0, 0xF5, 0x15, 0x6C, 0x40, 0xEA,
+0x04, 0x05, 0xDF, 0xF5, 0x11, 0x22, 0x5D, 0x67,
+0x50, 0xA2, 0x7D, 0x67, 0x52, 0x32, 0x50, 0xC3,
+0x12, 0x2A, 0x33, 0xF5, 0x50, 0x98, 0xE0, 0xF5,
+0x15, 0x6C, 0x40, 0xEA, 0x04, 0x05, 0xDF, 0xF5,
+0x03, 0x22, 0x5D, 0x67, 0x50, 0xA2, 0x0F, 0x6B,
+0x6C, 0xEA, 0xFF, 0x6B, 0x6C, 0xEA, 0x7D, 0x67,
+0x50, 0xC3, 0xBF, 0xF6, 0x11, 0x22, 0x5D, 0x67,
+0x50, 0xA2, 0x04, 0x05, 0xE0, 0xF5, 0x17, 0x6C,
+0x44, 0xC1, 0x33, 0xF5, 0x50, 0x98, 0x40, 0xEA,
+0x00, 0x65, 0xBF, 0xF5, 0x0D, 0x22, 0x5D, 0x67,
+0x50, 0xA2, 0x7D, 0x67, 0x52, 0x32, 0x50, 0xC3,
+0x11, 0x2A, 0x33, 0xF5, 0x50, 0x98, 0xE0, 0xF5,
+0x0F, 0x6C, 0x40, 0xEA, 0x04, 0x05, 0x9F, 0xF5,
+0x1F, 0x22, 0x5D, 0x67, 0x50, 0xA2, 0xFF, 0x6B,
+0x52, 0x32, 0x6C, 0xEA, 0x7D, 0x67, 0x50, 0xC3,
+0x9F, 0xF6, 0x0E, 0x22, 0x5D, 0x67, 0x50, 0xA2,
+0x00, 0x6C, 0x45, 0xC1, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x33, 0xF5, 0x44, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6A, 0x89, 0x15,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0x09, 0xD4, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x15, 0xF0, 0x08, 0x4C, 0x25, 0x67,
+0xAD, 0xA4, 0x6C, 0xA4, 0xCE, 0xA4, 0xA0, 0x35,
+0x6D, 0xED, 0x6F, 0xA4, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x40, 0xF0, 0x91, 0xA3, 0x40, 0xF0, 0xB2, 0xA3,
+0x40, 0xF0, 0xD0, 0xA3, 0x40, 0xF0, 0x73, 0xA3,
+0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x60, 0x9B, 0x01, 0x6C, 0x80, 0xF3, 0x0B, 0x6F,
+0x01, 0xA3, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xF3, 0xF0, 0x50, 0x9B, 0x04, 0xD4,
+0x05, 0xD0, 0x05, 0x6E, 0x02, 0x6C, 0x00, 0x6D,
+0x40, 0xEA, 0x08, 0xD3, 0x09, 0x92, 0x08, 0x93,
+0x2C, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A, 0x18, 0xF0,
+0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x95,
+0x03, 0x6B, 0x6C, 0xEA, 0x02, 0x58, 0x32, 0x61,
+0x23, 0xEA, 0x62, 0x67, 0x01, 0x60, 0x71, 0x67,
+0xFF, 0x6C, 0x04, 0x70, 0x8C, 0xEB, 0x1A, 0x61,
+0x00, 0x68, 0x02, 0x23, 0x1D, 0xF2, 0x04, 0x68,
+0xF3, 0xF0, 0x70, 0x9D, 0x06, 0xD2, 0x03, 0x6A,
+0x07, 0xD0, 0x05, 0xD1, 0x04, 0xD2, 0x80, 0xF3,
+0x0C, 0x6F, 0x05, 0x6E, 0x02, 0x6C, 0x40, 0xEB,
+0x00, 0x6D, 0x1B, 0x20, 0xB1, 0x18, 0xB6, 0xE1,
+0x90, 0x67, 0x01, 0x72, 0x16, 0x60, 0xA3, 0x67,
+0xFF, 0x6A, 0xDC, 0x17, 0x05, 0x70, 0x06, 0x61,
+0x02, 0x5B, 0x00, 0x68, 0xE5, 0x61, 0x1D, 0xF2,
+0x05, 0x68, 0xE2, 0x17, 0x06, 0x70, 0x00, 0x68,
+0xDF, 0x61, 0x03, 0x5B, 0xDD, 0x61, 0x1D, 0xF2,
+0x06, 0x68, 0xDA, 0x17, 0x1D, 0xF2, 0x0A, 0x68,
+0xD7, 0x17, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x8D, 0xA3, 0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x40, 0xF0, 0x71, 0xA2, 0x40, 0xF0, 0x92, 0xA2,
+0x40, 0xF0, 0xB0, 0xA2, 0x40, 0xF0, 0x53, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x40, 0x9A, 0x07, 0x6B, 0x1B, 0x65, 0x44, 0xA2,
+0x78, 0x67, 0x4C, 0xEB, 0x07, 0x73, 0x31, 0x60,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x26, 0x22,
+0x04, 0x6B, 0x4C, 0xEB, 0xFF, 0x68, 0x09, 0x2B,
+0xD1, 0x18, 0xEE, 0x80, 0x00, 0x6C, 0x0C, 0xEA,
+0x02, 0x72, 0x1F, 0x61, 0x1D, 0xF2, 0x12, 0x6C,
+0x16, 0x10, 0x02, 0x6B, 0x4C, 0xEB, 0x09, 0x2B,
+0xD1, 0x18, 0xEE, 0x80, 0x00, 0x6C, 0x0C, 0xEA,
+0x01, 0x72, 0x13, 0x61, 0x1D, 0xF2, 0x11, 0x6C,
+0x0A, 0x10, 0x01, 0x6B, 0x6C, 0xEA, 0x0D, 0x2A,
+0xD1, 0x18, 0xEE, 0x80, 0x00, 0x6C, 0x0C, 0xEA,
+0x08, 0x2A, 0x1D, 0xF2, 0x10, 0x6C, 0xB1, 0x18,
+0xB6, 0xE1, 0x00, 0x65, 0x1D, 0xF2, 0x0A, 0x6C,
+0xFA, 0x17, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x8D, 0xA3, 0x4C, 0xA3,
+0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0xF0, 0x71, 0xA2,
+0x40, 0xF0, 0x92, 0xA2, 0x40, 0xF0, 0xB0, 0xA2,
+0x40, 0xF0, 0x53, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0x9A, 0x20, 0xE8,
+0x42, 0xA2, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x8D, 0xA3, 0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x40, 0xF0, 0x71, 0xA2, 0x40, 0xF0, 0x92, 0xA2,
+0x40, 0xF0, 0xB0, 0xA2, 0x40, 0xF0, 0x53, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x40, 0x9A, 0x20, 0xE8, 0x43, 0xA2, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x8D, 0xA3, 0x4C, 0xA3,
+0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0xF0, 0x71, 0xA2,
+0x40, 0xF0, 0x92, 0xA2, 0x40, 0xF0, 0xB0, 0xA2,
+0x40, 0xF0, 0x53, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0x9A, 0x20, 0xE8,
+0x45, 0xA2, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x04, 0x67, 0x60, 0xA0, 0x22, 0x67,
+0x01, 0x6A, 0x4C, 0xEB, 0x08, 0x23, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF3,
+0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x60, 0xA0,
+0x02, 0x6A, 0x4C, 0xEB, 0x04, 0x23, 0xD1, 0x18,
+0x0E, 0xFB, 0x00, 0x6C, 0xFF, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x74, 0x9A, 0x40, 0xEB, 0x91, 0x67, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0x6A, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x15, 0xF0, 0x89, 0xA1,
+0x15, 0xF0, 0x68, 0xA1, 0x15, 0xF0, 0x4A, 0xA1,
+0x80, 0x34, 0x6D, 0xEC, 0x15, 0xF0, 0x6B, 0xA1,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x20, 0xF0, 0x85, 0xA3,
+0x20, 0xF0, 0x44, 0xA3, 0x20, 0xF0, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x0D, 0x22, 0x61, 0xA2,
+0x00, 0xA2, 0x82, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x03, 0xA2, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8, 0x1B, 0x28,
+0x00, 0x6A, 0x07, 0xD2, 0x7D, 0x67, 0x06, 0x6A,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6A, 0xF0, 0x18, 0x4A, 0x05, 0xD2,
+0xAC, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63, 0xD1, 0x18,
+0x9D, 0x17, 0x80, 0xA0, 0x62, 0x67, 0x41, 0xA0,
+0x6E, 0xEA, 0xF4, 0x22, 0x42, 0xA0, 0x61, 0xC0,
+0xF1, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x15, 0xF0,
+0x08, 0x49, 0x40, 0xEA, 0x08, 0xD3, 0x08, 0x93,
+0x00, 0x6C, 0x82, 0xC0, 0x63, 0xC0, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x72, 0xF0,
+0x74, 0x9B, 0xFF, 0x6C, 0x40, 0xEB, 0x4C, 0xEC,
+0xF7, 0xF0, 0x01, 0x6A, 0x89, 0xA1, 0x40, 0x32,
+0x68, 0xA1, 0x40, 0x32, 0xF2, 0xF1, 0x10, 0x9A,
+0x4A, 0xA1, 0x80, 0x34, 0x6D, 0xEC, 0x6B, 0xA1,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x89, 0xA3, 0x48, 0xA3,
+0xAA, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x95, 0xA2, 0x74, 0xA2,
+0x00, 0x6F, 0x80, 0x34, 0x6D, 0xEC, 0x76, 0xA2,
+0x01, 0x6E, 0x10, 0x6D, 0x60, 0x33, 0x60, 0x33,
+0x8D, 0xEB, 0x97, 0xA2, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xE8, 0x6D, 0xEC, 0xAB, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x08, 0x5C, 0x04, 0x67, 0xC0, 0xF0, 0x0C, 0x60,
+0xB0, 0x75, 0x74, 0x61, 0xB2, 0x76, 0x3C, 0x61,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69,
+0x60, 0x33, 0x20, 0x31, 0x60, 0x33, 0x20, 0x31,
+0xD2, 0xF4, 0xC8, 0x9B, 0xD2, 0xF4, 0x54, 0x99,
+0x44, 0x6C, 0x05, 0xD3, 0x40, 0xEA, 0x04, 0xD6,
+0x01, 0x6C, 0x84, 0xE8, 0x04, 0x96, 0x00, 0xF6,
+0x80, 0x35, 0x4D, 0xED, 0x04, 0x67, 0x40, 0xEE,
+0x44, 0x6C, 0x05, 0x93, 0x44, 0x6C, 0xD2, 0xF4,
+0xC8, 0x9B, 0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA,
+0x04, 0xD6, 0x00, 0x35, 0x04, 0x96, 0xA0, 0x35,
+0xAF, 0xED, 0x4C, 0xED, 0x40, 0xEE, 0x44, 0x6C,
+0x05, 0x93, 0x48, 0x6C, 0xD2, 0xF4, 0x68, 0x9B,
+0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA, 0x04, 0xD3,
+0xB0, 0x67, 0x4D, 0xED, 0x48, 0x6C, 0x04, 0x93,
+0x40, 0xEB, 0x00, 0x65, 0xF0, 0x6A, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0xB1, 0x76, 0xE2, 0x6A, 0xF8, 0x61, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69, 0x60, 0x33,
+0x20, 0x31, 0x60, 0x33, 0x20, 0x31, 0xD2, 0xF4,
+0xC8, 0x9B, 0xD2, 0xF4, 0x54, 0x99, 0x44, 0x6C,
+0x05, 0xD3, 0x40, 0xEA, 0x04, 0xD6, 0x01, 0x6D,
+0xA4, 0xE8, 0x04, 0x96, 0x05, 0x67, 0x00, 0xF6,
+0xA0, 0x35, 0x4D, 0xED, 0x40, 0xEE, 0x44, 0x6C,
+0x05, 0x93, 0x44, 0x6C, 0xD2, 0xF4, 0xC8, 0x9B,
+0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA, 0x04, 0xD6,
+0x00, 0x35, 0x04, 0x96, 0xA0, 0x35, 0xAF, 0xED,
+0x4C, 0xED, 0x40, 0xEE, 0x44, 0x6C, 0x05, 0x93,
+0x48, 0x6C, 0xD2, 0xF4, 0x68, 0x9B, 0xD2, 0xF4,
+0x54, 0x99, 0x40, 0xEA, 0x04, 0xD3, 0x0F, 0xED,
+0x4C, 0xED, 0xC0, 0x17, 0xA0, 0x75, 0xE1, 0x6A,
+0xC2, 0x61, 0xA1, 0x76, 0x27, 0x61, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69, 0x60, 0x33,
+0x20, 0x31, 0x60, 0x33, 0x20, 0x31, 0xD2, 0xF4,
+0xC8, 0x9B, 0xD2, 0xF4, 0x54, 0x99, 0x44, 0x6C,
+0x05, 0xD3, 0x40, 0xEA, 0x04, 0xD6, 0x01, 0x6D,
+0xA4, 0xE8, 0x05, 0x67, 0x04, 0x96, 0x00, 0xF6,
+0xA0, 0x35, 0xAF, 0xED, 0x4C, 0xED, 0x40, 0xEE,
+0x44, 0x6C, 0x05, 0x93, 0x44, 0x6C, 0xD2, 0xF4,
+0x68, 0x9B, 0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA,
+0x04, 0xD3, 0x00, 0x35, 0xA0, 0x35, 0x4D, 0xED,
+0x44, 0x6C, 0x95, 0x17, 0xA2, 0x76, 0xE2, 0x6A,
+0x96, 0x61, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x69, 0x60, 0x33, 0x20, 0x31, 0x60, 0x33,
+0x20, 0x31, 0xD2, 0xF4, 0xC8, 0x9B, 0xD2, 0xF4,
+0x54, 0x99, 0x44, 0x6C, 0x05, 0xD3, 0x40, 0xEA,
+0x04, 0xD6, 0x01, 0x6D, 0xA4, 0xE8, 0x05, 0x67,
+0x04, 0x96, 0x00, 0xF6, 0xA0, 0x35, 0xAF, 0xED,
+0x4C, 0xED, 0x40, 0xEE, 0x44, 0x6C, 0x05, 0x93,
+0x44, 0x6C, 0xD2, 0xF4, 0x68, 0x9B, 0xD2, 0xF4,
+0x54, 0x99, 0x40, 0xEA, 0x04, 0xD3, 0x00, 0x35,
+0xA0, 0x35, 0xAF, 0xED, 0x4C, 0xED, 0xD4, 0x17,
+0xFF, 0x6A, 0x28, 0x44, 0x4C, 0xE9, 0x08, 0x59,
+0xE0, 0x6A, 0x7F, 0xF7, 0x08, 0x60, 0xB0, 0x75,
+0x7F, 0x61, 0xB2, 0x76, 0x3E, 0x61, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0xD2, 0xF4,
+0x48, 0x9F, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xC0, 0x36, 0x04, 0xD2, 0xD2, 0xF4, 0x54, 0x9E,
+0x60, 0x6C, 0x06, 0xD7, 0x40, 0xEA, 0x05, 0xD6,
+0x01, 0x6B, 0x83, 0x67, 0x84, 0xE9, 0x00, 0xF6,
+0x80, 0x35, 0x4D, 0xED, 0x04, 0x92, 0x24, 0x67,
+0x60, 0x6C, 0x40, 0xEA, 0x07, 0xD3, 0x06, 0x97,
+0x05, 0x96, 0x60, 0x6C, 0xD2, 0xF4, 0x48, 0x9F,
+0x04, 0xD2, 0xD2, 0xF4, 0x54, 0x9E, 0x40, 0xEA,
+0x00, 0x65, 0x20, 0x35, 0xA0, 0x35, 0xAF, 0xED,
+0x4C, 0xED, 0x04, 0x92, 0x40, 0xEA, 0x60, 0x6C,
+0x06, 0x97, 0x05, 0x96, 0x48, 0x6C, 0xD2, 0xF4,
+0x28, 0x9F, 0xD2, 0xF4, 0x54, 0x9E, 0x40, 0xEA,
+0x00, 0x65, 0x07, 0x93, 0x64, 0xE8, 0xA3, 0x67,
+0x4D, 0xED, 0x48, 0x6C, 0x40, 0xE9, 0x00, 0x65,
+0x25, 0x17, 0xB1, 0x76, 0xE2, 0x6A, 0x3F, 0xF7,
+0x02, 0x61, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0xE0, 0x37, 0xD2, 0xF4, 0x48, 0x9F, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0x04, 0xD2,
+0xD2, 0xF4, 0x54, 0x9E, 0x60, 0x6C, 0x06, 0xD7,
+0x40, 0xEA, 0x05, 0xD6, 0x01, 0x6B, 0x83, 0x67,
+0x84, 0xE9, 0x00, 0xF6, 0x80, 0x35, 0x4D, 0xED,
+0x04, 0x92, 0x24, 0x67, 0x60, 0x6C, 0x40, 0xEA,
+0x07, 0xD3, 0x06, 0x97, 0x05, 0x96, 0x60, 0x6C,
+0xD2, 0xF4, 0x48, 0x9F, 0x04, 0xD2, 0xD2, 0xF4,
+0x54, 0x9E, 0x40, 0xEA, 0x00, 0x65, 0x20, 0x35,
+0xA0, 0x35, 0xAF, 0xED, 0x4C, 0xED, 0x04, 0x92,
+0x40, 0xEA, 0x60, 0x6C, 0x06, 0x97, 0x05, 0x96,
+0x48, 0x6C, 0xD2, 0xF4, 0x28, 0x9F, 0xD2, 0xF4,
+0x54, 0x9E, 0x40, 0xEA, 0x00, 0x65, 0x07, 0x93,
+0x64, 0xE8, 0x6F, 0xED, 0x4C, 0xED, 0xBD, 0x17,
+0xA0, 0x75, 0xE1, 0x6A, 0xFF, 0xF6, 0x03, 0x61,
+0xA1, 0x76, 0x28, 0x61, 0xF7, 0xF0, 0x01, 0x6E,
+0xF7, 0xF0, 0x01, 0x6B, 0xC0, 0x36, 0x60, 0x33,
+0xC0, 0x36, 0x60, 0x33, 0xD2, 0xF4, 0xE8, 0x9E,
+0xD2, 0xF4, 0x54, 0x9B, 0x01, 0x68, 0x04, 0xE9,
+0x60, 0x6C, 0x06, 0xD6, 0x04, 0xD3, 0x40, 0xEA,
+0x05, 0xD7, 0x05, 0x97, 0x00, 0xF6, 0x00, 0x35,
+0xAF, 0xED, 0x4C, 0xED, 0x40, 0xEF, 0x60, 0x6C,
+0x06, 0x96, 0x04, 0x93, 0x60, 0x6C, 0xD2, 0xF4,
+0x28, 0x9E, 0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x00, 0x35, 0xA0, 0x35, 0x4D, 0xED,
+0x60, 0x6C, 0x90, 0x17, 0xA2, 0x76, 0xE2, 0x6A,
+0xBF, 0xF6, 0x15, 0x61, 0xF7, 0xF0, 0x01, 0x6E,
+0xF7, 0xF0, 0x01, 0x6B, 0xC0, 0x36, 0x60, 0x33,
+0xC0, 0x36, 0x60, 0x33, 0xD2, 0xF4, 0xE8, 0x9E,
+0xD2, 0xF4, 0x54, 0x9B, 0x01, 0x68, 0x04, 0xE9,
+0x60, 0x6C, 0x06, 0xD6, 0x04, 0xD3, 0x40, 0xEA,
+0x05, 0xD7, 0x05, 0x97, 0x00, 0xF6, 0x00, 0x35,
+0xAF, 0xED, 0x4C, 0xED, 0x40, 0xEF, 0x60, 0x6C,
+0x06, 0x96, 0x04, 0x93, 0x60, 0x6C, 0xD2, 0xF4,
+0x28, 0x9E, 0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x00, 0x35, 0xA0, 0x35, 0xAF, 0xED,
+0x4C, 0xED, 0xD2, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6D,
+0xA0, 0x35, 0xA0, 0x35, 0x15, 0xF0, 0x89, 0xA5,
+0x15, 0xF0, 0x68, 0xA5, 0x00, 0x6A, 0x08, 0xD2,
+0x15, 0xF0, 0x4A, 0xA5, 0x80, 0x34, 0x6D, 0xEC,
+0x15, 0xF0, 0x6B, 0xA5, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x20, 0xF0, 0x85, 0xA3, 0x20, 0xF0, 0x44, 0xA3,
+0x20, 0xF0, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x20, 0xF0, 0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x0D, 0x22, 0x61, 0xA2, 0x00, 0xA2, 0x82, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x03, 0xA2, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30,
+0x6D, 0xE8, 0x1D, 0x28, 0x00, 0x6A, 0x7D, 0x67,
+0x07, 0xD2, 0x06, 0x6A, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6A, 0xF0,
+0x18, 0x4A, 0x05, 0xD2, 0x7A, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x01, 0x6B, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x43, 0x67, 0x00, 0xEF, 0x08, 0x63, 0x42, 0xA0,
+0x00, 0x6B, 0xF7, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A,
+0x00, 0x6F, 0x08, 0x06, 0x04, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0x04, 0x6B, 0xEA, 0x2A, 0x08, 0x95,
+0x02, 0x6B, 0xE7, 0x25, 0x7D, 0x67, 0x01, 0x69,
+0x32, 0xC3, 0x9D, 0x67, 0x09, 0x6B, 0x71, 0xC4,
+0x7D, 0x67, 0x50, 0xC3, 0x0C, 0x6B, 0x73, 0xC4,
+0x7D, 0x67, 0x54, 0xC3, 0x43, 0xA0, 0x04, 0x04,
+0xD1, 0x18, 0xA8, 0x19, 0x40, 0xC5, 0x62, 0x67,
+0xD4, 0x2A, 0x0A, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x22, 0xC0, 0x80, 0x34, 0x72, 0xF0,
+0xB4, 0x9C, 0xFF, 0x6C, 0x40, 0xED, 0x4C, 0xEC,
+0x0A, 0x93, 0xBF, 0x17, 0x41, 0xA4, 0x02, 0x6B,
+0x10, 0x5A, 0x00, 0xF1, 0x02, 0x60, 0x42, 0xA4,
+0x03, 0x6B, 0xE0, 0xF0, 0x1E, 0x22, 0xF8, 0x63,
+0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0x15, 0xF0,
+0xA9, 0xA6, 0x15, 0xF0, 0x48, 0xA6, 0x0A, 0xD4,
+0x15, 0xF0, 0x8A, 0xA6, 0xA0, 0x35, 0x4D, 0xED,
+0x15, 0xF0, 0x4B, 0xA6, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xF0, 0x85, 0xA2, 0x20, 0xF0, 0x04, 0xA2,
+0x20, 0xF0, 0xA6, 0xA2, 0x80, 0x34, 0x0D, 0xEC,
+0x20, 0xF0, 0x07, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8,
+0x1B, 0x28, 0x06, 0x6A, 0x7D, 0x67, 0x58, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x6A, 0xF0, 0x18, 0x4A, 0x07, 0xD2, 0xD8, 0x6A,
+0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x06, 0x04, 0x01, 0x6B, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x43, 0x67, 0x00, 0xEF, 0x08, 0x63,
+0x41, 0xA0, 0x20, 0xA0, 0x82, 0xA0, 0x40, 0x32,
+0x2D, 0xEA, 0x23, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0x2C, 0x29, 0x0A, 0x92, 0x01, 0x6C, 0x40, 0xA2,
+0x8C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA, 0x25, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF4, 0x54, 0x9A, 0x00, 0x6D, 0x40, 0xEA,
+0x04, 0x6C, 0x22, 0x67, 0x0C, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x6A, 0xF0, 0x18, 0x4A,
+0x07, 0xD2, 0xE4, 0x6A, 0xC5, 0x17, 0x40, 0xC0,
+0x42, 0x32, 0x41, 0xC0, 0x42, 0x32, 0x42, 0xC0,
+0x00, 0xF6, 0x22, 0x32, 0x43, 0xC0, 0x0A, 0x92,
+0x81, 0xA2, 0xD1, 0x18, 0x9D, 0x17, 0x80, 0xC1,
+0x41, 0xC1, 0x0A, 0x92, 0x01, 0x6C, 0x40, 0xA2,
+0x8C, 0xEA, 0x57, 0x22, 0x0A, 0x92, 0x0F, 0x6D,
+0xD1, 0x18, 0xFB, 0x17, 0x81, 0xA2, 0x62, 0x67,
+0xB5, 0x2A, 0x0A, 0x92, 0x00, 0x6D, 0xD1, 0x18,
+0xAC, 0x17, 0x81, 0xA2, 0x62, 0x67, 0xAE, 0x2A,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x4C, 0xF2, 0x5C, 0x98, 0x1C, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1,
+0x38, 0x9A, 0x0A, 0x92, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0xA2, 0xA2, 0x97, 0xF0, 0x18, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF0, 0x0D, 0x4A,
+0x80, 0x34, 0x04, 0xD2, 0x01, 0x6F, 0x01, 0x6E,
+0x8A, 0xF0, 0x0C, 0x4C, 0x40, 0xE9, 0x0B, 0xD3,
+0x0B, 0x93, 0x4C, 0xF2, 0x5C, 0xD8, 0x4C, 0xF2,
+0x1C, 0x98, 0x0C, 0x28, 0x06, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6A, 0xF0, 0x18, 0x4A, 0x07, 0xD2,
+0xFE, 0x6A, 0x72, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1, 0x34, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x92, 0xF2, 0x4C, 0x9A, 0x40, 0xEA, 0x0A, 0xD3,
+0x00, 0x6C, 0x04, 0xD4, 0x00, 0x6F, 0xC2, 0x67,
+0x01, 0x6D, 0x40, 0xE9, 0x90, 0x67, 0x0A, 0x93,
+0x65, 0x17, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x4C, 0xF2, 0x9C, 0x9C, 0x0B, 0x24,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0xD2, 0xF1, 0x74, 0x9D, 0x04, 0xD2, 0x00, 0x6F,
+0x00, 0x6E, 0x40, 0xEB, 0x03, 0x6D, 0x00, 0x6B,
+0x5F, 0xF7, 0x10, 0x21, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A,
+0x40, 0xEA, 0x0A, 0x94, 0x00, 0x6A, 0x40, 0xC0,
+0x41, 0xC0, 0x42, 0xC0, 0x43, 0xC0, 0x00, 0x6B,
+0x41, 0x17, 0x20, 0xE8, 0x43, 0x67, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xFF, 0x69, 0x8C, 0xE9, 0x03, 0x6E, 0x51, 0x67,
+0xCC, 0xEA, 0x80, 0xF0, 0x05, 0x2A, 0xF7, 0xF0,
+0x01, 0x68, 0xF7, 0xF0, 0x01, 0x6B, 0x00, 0x30,
+0x60, 0x33, 0x60, 0x33, 0x00, 0x30, 0x5D, 0x67,
+0xDC, 0xC2, 0xD2, 0xF4, 0xE8, 0x98, 0xD2, 0xF4,
+0x54, 0x9B, 0x04, 0xD4, 0x70, 0x6C, 0x05, 0xD3,
+0x40, 0xEA, 0x06, 0xD7, 0x06, 0x97, 0x01, 0xF0,
+0x00, 0x6D, 0x4D, 0xED, 0x40, 0xEF, 0x70, 0x6C,
+0x04, 0x92, 0x02, 0xF0, 0x00, 0x6D, 0x42, 0x34,
+0x5D, 0x67, 0xDC, 0xA2, 0xD2, 0xF4, 0x48, 0x98,
+0x8C, 0xEE, 0xC0, 0x36, 0xC0, 0x36, 0x2D, 0xEE,
+0xCD, 0xED, 0xE4, 0x6C, 0x40, 0xEA, 0x04, 0xD6,
+0x04, 0x93, 0xD2, 0xF4, 0x48, 0x98, 0x06, 0xF0,
+0x00, 0x6D, 0x6D, 0xED, 0x40, 0xEA, 0xE4, 0x6C,
+0x05, 0x93, 0xE0, 0xF3, 0x08, 0x6D, 0x23, 0x67,
+0xD2, 0xF4, 0x54, 0x99, 0xE4, 0x6C, 0x40, 0xEA,
+0x05, 0xD5, 0x04, 0xF0, 0x00, 0x6B, 0x6C, 0xEA,
+0x05, 0x95, 0x18, 0x2A, 0xD2, 0xF4, 0x54, 0x99,
+0x40, 0xEA, 0xE8, 0x6C, 0x04, 0xD2, 0xD2, 0xF4,
+0x08, 0x98, 0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA,
+0x70, 0x6C, 0x01, 0xF0, 0x01, 0x6D, 0xAB, 0xED,
+0x4C, 0xED, 0x40, 0xE8, 0x70, 0x6C, 0x04, 0x93,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x43, 0x67,
+0x00, 0xEF, 0x06, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x01, 0x6C, 0x40, 0xEA, 0x05, 0xD5, 0x05, 0x95,
+0xFF, 0xF7, 0x1F, 0x6A, 0xFF, 0x4D, 0x4C, 0xED,
+0xCF, 0x2D, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0x89, 0xF1, 0xAC, 0x9B, 0xD3, 0xF6,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x6C, 0xD2, 0xF4,
+0x48, 0x98, 0x04, 0x95, 0x40, 0xEA, 0xE4, 0x6C,
+0xD2, 0xF4, 0x08, 0x98, 0xD2, 0xF4, 0x54, 0x99,
+0x40, 0xEA, 0x70, 0x6C, 0x01, 0xF0, 0x01, 0x6D,
+0xAB, 0xED, 0x4C, 0xED, 0x40, 0xE8, 0x70, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA9, 0xF1, 0x60, 0x9A, 0xC5, 0x17, 0x00, 0x65,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xFF, 0x6B, 0xE4, 0x67, 0x6C, 0xEF, 0x03, 0x68,
+0xEC, 0xE8, 0x46, 0x67, 0x00, 0x6E, 0x80, 0xF0,
+0x0B, 0x28, 0x0F, 0x6E, 0x4C, 0xEE, 0x6C, 0xEE,
+0x80, 0xF0, 0x06, 0x26, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0xD2, 0xF4, 0x48, 0x98,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x04, 0xD2, 0xD2, 0xF4, 0x54, 0x99, 0x05, 0xD4,
+0x70, 0x6C, 0x08, 0xD6, 0x07, 0xD7, 0x40, 0xEA,
+0x06, 0xD5, 0x01, 0xF0, 0x00, 0x6D, 0x4D, 0xED,
+0x04, 0x92, 0x40, 0xEA, 0x70, 0x6C, 0x05, 0x92,
+0x07, 0x97, 0x08, 0x96, 0x42, 0x34, 0x03, 0x6A,
+0x4C, 0xEC, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEF,
+0xD2, 0xF4, 0x48, 0x98, 0xC0, 0x36, 0xED, 0xEE,
+0xA6, 0x67, 0xE4, 0x6C, 0x40, 0xEA, 0x04, 0xD6,
+0x06, 0x93, 0xD2, 0xF4, 0x48, 0x98, 0xE8, 0x6C,
+0x40, 0xEA, 0xA3, 0x67, 0x04, 0x93, 0xD2, 0xF4,
+0x48, 0x98, 0x04, 0xF0, 0x00, 0x6D, 0x6D, 0xED,
+0x40, 0xEA, 0xE4, 0x6C, 0xE0, 0xF3, 0x08, 0x6D,
+0xD2, 0xF4, 0x54, 0x99, 0xE4, 0x6C, 0x40, 0xEA,
+0x05, 0xD5, 0x04, 0xF0, 0x00, 0x6B, 0x6C, 0xEA,
+0x05, 0x95, 0x0E, 0x2A, 0xD2, 0xF4, 0x08, 0x98,
+0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA, 0x70, 0x6C,
+0x01, 0xF0, 0x01, 0x6D, 0xAB, 0xED, 0x4C, 0xED,
+0x40, 0xE8, 0x70, 0x6C, 0x01, 0x6E, 0x34, 0x10,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0x01, 0x6C, 0x40, 0xEA,
+0x05, 0xD5, 0x05, 0x95, 0xFF, 0xF7, 0x1F, 0x6A,
+0xFF, 0x4D, 0x4C, 0xED, 0xD9, 0x2D, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xA9, 0xF1,
+0xA4, 0x9B, 0xD3, 0xF6, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x6C, 0x04, 0x93, 0x05, 0xF7, 0x01, 0x6D,
+0xD2, 0xF4, 0x48, 0x98, 0xAB, 0xED, 0xAC, 0xEB,
+0xA3, 0x67, 0x40, 0xEA, 0xE4, 0x6C, 0xD2, 0xF4,
+0x08, 0x98, 0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA,
+0x70, 0x6C, 0x01, 0xF0, 0x01, 0x6D, 0xAB, 0xED,
+0x4C, 0xED, 0x40, 0xE8, 0x70, 0x6C, 0x00, 0x6E,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x46, 0x67,
+0x00, 0xEF, 0x07, 0x63, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x00, 0x6A, 0x60, 0x33, 0x40, 0xC4,
+0x15, 0xF0, 0x08, 0x4B, 0xA9, 0xA3, 0x48, 0xA3,
+0xCA, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x65, 0xA2, 0xA6, 0xA2,
+0xC4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x48, 0xA2, 0x03, 0x72,
+0x07, 0x60, 0x20, 0xE8, 0x00, 0x6A, 0x00, 0x6A,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF4,
+0x44, 0x9A, 0x04, 0x67, 0xC2, 0xF1, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xA9, 0xF1, 0x60, 0x9B,
+0x4E, 0xEB, 0xE5, 0x23, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xA9, 0xF1, 0x68, 0x9B,
+0x4C, 0xEB, 0x01, 0x6A, 0xDD, 0x23, 0x40, 0xC0,
+0xDB, 0x17, 0x00, 0x65, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0x05, 0x67, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0x24, 0x67,
+0x20, 0x6E, 0x48, 0xF2, 0x08, 0x4D, 0xD1, 0x18,
+0xE7, 0xF4, 0x06, 0x04, 0x14, 0x29, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x04, 0xD1, 0x40, 0xF2, 0x15, 0x6F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x00, 0x6A, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF4,
+0x44, 0x9A, 0x02, 0xF0, 0x10, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xA9, 0xF1, 0x80, 0x9B, 0x00, 0x6B,
+0x4E, 0xEC, 0x0A, 0x2C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x04, 0xD4, 0x40, 0xF2, 0x16, 0x6F, 0xD8, 0x17,
+0x91, 0x67, 0x87, 0xEB, 0x01, 0x6D, 0xAC, 0xEC,
+0x06, 0x24, 0x20, 0x28, 0x68, 0x34, 0x06, 0x05,
+0x91, 0xE5, 0x80, 0x9C, 0x8D, 0xEA, 0x01, 0x4B,
+0x08, 0x73, 0xF2, 0x61, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xF3, 0xF4, 0x60, 0x9B,
+0x0F, 0x6E, 0x02, 0xF0, 0x10, 0x6C, 0x40, 0xEB,
+0xA2, 0x67, 0x62, 0x67, 0x01, 0x6A, 0xC2, 0x2B,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0x04, 0xD3, 0x40, 0xF2,
+0x17, 0x6F, 0xB2, 0x17, 0x01, 0x70, 0xE3, 0x61,
+0x68, 0x34, 0x06, 0x05, 0x91, 0xE5, 0x80, 0x9C,
+0x8F, 0xEC, 0x8C, 0xEA, 0xDC, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x04, 0x67, 0x89, 0xA3,
+0x48, 0xA3, 0xAA, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x4B, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2,
+0x86, 0xA2, 0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x28, 0xA2,
+0x03, 0x6A, 0x4E, 0xE9, 0x07, 0x21, 0x00, 0x68,
+0x50, 0x67, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x10, 0xF4, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x01, 0x70, 0x3A, 0x60, 0x1A, 0x20, 0x02, 0x70,
+0x6F, 0x60, 0x07, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xAA, 0xF0, 0x00, 0x4A, 0x05, 0xD2, 0x20, 0xF2,
+0x1E, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x07, 0xD1, 0x40, 0xEA, 0x04, 0x04, 0x01, 0x68,
+0xD3, 0x17, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0x06, 0xF3, 0x60, 0x99, 0x01, 0x6E,
+0xCC, 0xEB, 0xC9, 0x2B, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x68, 0x9B,
+0x04, 0x6D, 0xAB, 0xED, 0x4C, 0xED, 0x10, 0xF4,
+0x10, 0x6C, 0x40, 0xEB, 0x08, 0xD6, 0x06, 0xF3,
+0x40, 0x99, 0x08, 0x96, 0x62, 0x67, 0x07, 0x6A,
+0x4B, 0xEA, 0x6C, 0xEA, 0xCD, 0xEA, 0x35, 0x10,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x06, 0xF3, 0x60, 0x99, 0x06, 0x68, 0x6C, 0xE8,
+0xAA, 0x28, 0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x6B, 0x80, 0x34, 0x80, 0x34, 0x60, 0x33,
+0xA9, 0xF1, 0xCC, 0x9C, 0x60, 0x33, 0xD2, 0xF4,
+0xE8, 0x9B, 0xA6, 0x67, 0x10, 0xF4, 0x18, 0x6C,
+0x0A, 0xD2, 0x08, 0xD3, 0x40, 0xEF, 0x09, 0xD6,
+0x08, 0x93, 0x09, 0x96, 0x10, 0xF4, 0x1C, 0x6C,
+0xD2, 0xF4, 0xE8, 0x9B, 0x40, 0xEF, 0xA6, 0x67,
+0x08, 0x93, 0x0A, 0x92, 0x23, 0x6D, 0xD2, 0xF4,
+0x68, 0x9B, 0x10, 0xF4, 0x10, 0x6C, 0x40, 0xEB,
+0x4D, 0xED, 0x06, 0xF3, 0x40, 0x99, 0x62, 0x67,
+0x02, 0x6A, 0x4B, 0xEA, 0x6C, 0xEA, 0x02, 0x6B,
+0x6D, 0xEA, 0x06, 0xF3, 0x40, 0xD9, 0x7C, 0x17,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x06, 0xF3, 0x60, 0x99, 0x04, 0x6E, 0x03, 0x67,
+0xCC, 0xE8, 0x7F, 0xF7, 0x10, 0x28, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6C, 0x60, 0x33,
+0x80, 0x34, 0x60, 0x33, 0x80, 0x34, 0xD2, 0xF4,
+0xE8, 0x9B, 0xA9, 0xF1, 0xB0, 0x9C, 0x10, 0xF4,
+0x18, 0x6C, 0x0A, 0xD6, 0x09, 0xD2, 0x40, 0xEF,
+0x08, 0xD3, 0x08, 0x93, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xD2, 0xF4, 0xE8, 0x9B,
+0xA9, 0xF1, 0xB4, 0x9C, 0x10, 0xF4, 0x1C, 0x6C,
+0x40, 0xEF, 0x00, 0x65, 0x08, 0x93, 0x09, 0x92,
+0x23, 0x6D, 0xD2, 0xF4, 0x68, 0x9B, 0x10, 0xF4,
+0x10, 0x6C, 0x40, 0xEB, 0x4D, 0xED, 0x06, 0xF3,
+0x40, 0x99, 0x0A, 0x96, 0x62, 0x67, 0x02, 0x6A,
+0x4B, 0xEA, 0x6C, 0xEA, 0x8B, 0x17, 0x00, 0x65,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0x08, 0x02, 0x40, 0xF0, 0x40, 0xA2, 0x02, 0x5C,
+0x17, 0xD7, 0x0E, 0xD2, 0x08, 0x02, 0x40, 0xF0,
+0x24, 0xA2, 0x15, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6, 0x50, 0x9A,
+0x89, 0xF1, 0xAC, 0x9B, 0x40, 0xEA, 0x01, 0x6C,
+0x00, 0x68, 0x50, 0x67, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x29, 0xF1,
+0xEC, 0x9A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xA9, 0xF1, 0x58, 0x9A, 0xEC, 0xEE,
+0xEC, 0xED, 0x4D, 0xE5, 0x0C, 0xD6, 0x49, 0xE6,
+0x10, 0xF0, 0x00, 0x6E, 0xC3, 0xEA, 0x02, 0x61,
+0xC3, 0xEB, 0x17, 0x60, 0x00, 0x6A, 0x07, 0xD2,
+0x7D, 0x67, 0x13, 0x6A, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xAA, 0xF0,
+0x14, 0x4A, 0x05, 0xD2, 0x2F, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0xCB, 0x17, 0x01, 0x74, 0x42, 0x60, 0x08, 0x6A,
+0x0B, 0xD2, 0x04, 0x6A, 0x0A, 0xD2, 0x00, 0x6A,
+0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xF6, 0x78, 0xA2, 0x0D, 0xD2,
+0x00, 0x6A, 0x08, 0xD2, 0x0C, 0x2B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x0F, 0xD4, 0x40, 0xEA, 0x05, 0x67,
+0x0F, 0x94, 0x08, 0xD2, 0xB0, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF4,
+0x40, 0x9A, 0x40, 0xEA, 0x0F, 0xD5, 0x02, 0x67,
+0x0F, 0x95, 0x21, 0x2A, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x6C, 0x60, 0x33, 0x80, 0x34,
+0x80, 0x34, 0x60, 0x33, 0xD3, 0xF6, 0x70, 0x9B,
+0x89, 0xF1, 0xAC, 0x9C, 0x40, 0xEB, 0x01, 0x6C,
+0x0D, 0x92, 0x8D, 0xF6, 0x78, 0xA2, 0x91, 0x2B,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x72, 0xF0, 0x74, 0x9B, 0x40, 0xEB, 0x08, 0x94,
+0x88, 0x17, 0x18, 0x6A, 0x0B, 0xD2, 0x14, 0x6A,
+0x0A, 0xD2, 0x10, 0x6A, 0xBD, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xA9, 0xF1,
+0x1C, 0x9A, 0x17, 0x92, 0x4C, 0xE8, 0x0E, 0x92,
+0x07, 0x2A, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xC9, 0xF1, 0x60, 0x9B, 0x6D, 0xE8,
+0x07, 0x21, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x89, 0xF1, 0x74, 0x9B, 0x6D, 0xE8,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69,
+0x60, 0x33, 0x20, 0x31, 0x60, 0x33, 0x20, 0x31,
+0xC9, 0xF1, 0x64, 0x9B, 0x09, 0x92, 0xD2, 0xF4,
+0xF8, 0x99, 0x09, 0xD3, 0x40, 0xEF, 0x71, 0xE2,
+0x0A, 0x92, 0x09, 0x93, 0xD2, 0xF4, 0xF8, 0x99,
+0x0C, 0x95, 0x40, 0xEF, 0x71, 0xE2, 0x09, 0x93,
+0x0B, 0x92, 0xD2, 0xF4, 0xF8, 0x99, 0xB0, 0x67,
+0x69, 0xE2, 0x82, 0x67, 0x40, 0xEF, 0x09, 0xD2,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x09, 0xF1, 0xBC, 0x9C, 0x09, 0x92, 0xD2, 0xF4,
+0x78, 0x99, 0x0D, 0xED, 0x40, 0xEB, 0x82, 0x67,
+0x0D, 0x92, 0x01, 0x68, 0x8D, 0xF6, 0x78, 0xA2,
+0xA6, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x68, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x09, 0xF1,
+0x5C, 0x9A, 0xC0, 0x36, 0xC0, 0x36, 0x4D, 0xEC,
+0x8D, 0xEE, 0xA0, 0x35, 0x60, 0xF2, 0x10, 0x6C,
+0x40, 0xEB, 0xCD, 0xED, 0xE0, 0xF3, 0x09, 0x68,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x60, 0xF2, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x00, 0x52, 0x04, 0x61,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xFF, 0x48, 0xFA, 0x20, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0xE5, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xC9, 0xF1, 0xA8, 0x9B,
+0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A, 0x8D, 0xED,
+0x60, 0xF2, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xE0, 0xF3, 0x09, 0x68, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x60, 0xF2, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x00, 0x52, 0x0E, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0x60, 0xF2, 0x11, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xFF, 0x48, 0x09, 0x20, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0xDB, 0x17, 0xFF, 0x6A,
+0xEF, 0x17, 0x00, 0x65, 0x40, 0xA4, 0x01, 0x6B,
+0x6C, 0xEA, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x06, 0xF3, 0x44, 0xC3, 0x20, 0xE8,
+0x00, 0x6A, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x04, 0x67, 0x25, 0x67,
+0x0C, 0xD6, 0x22, 0x27, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x89, 0xA3, 0x48, 0xA3, 0xAA, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x65, 0xA2, 0x86, 0xA2, 0xA4, 0xA2, 0x47, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x49, 0xA2, 0x02, 0x2A, 0x00, 0x6A, 0x13, 0x10,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x4C, 0x9B, 0xF8, 0xF6, 0x0C, 0x6C,
+0x05, 0xD7, 0x40, 0xEA, 0x04, 0xD3, 0x00, 0xF6,
+0x40, 0x32, 0x00, 0xF6, 0x43, 0x32, 0x00, 0x52,
+0x05, 0x97, 0x06, 0x61, 0x01, 0x6A, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF5, 0x50, 0x9A, 0x40, 0xEA, 0x87, 0x67,
+0x01, 0x72, 0xDC, 0x61, 0x01, 0x70, 0x07, 0x60,
+0x26, 0x20, 0x02, 0x70, 0x22, 0x60, 0x03, 0x70,
+0xD5, 0x61, 0x0A, 0x68, 0x01, 0x10, 0x08, 0x68,
+0xFF, 0x49, 0xD0, 0x21, 0x04, 0x92, 0xF8, 0xF6,
+0x0C, 0x6C, 0xD2, 0xF4, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x00, 0xF6, 0x40, 0x33, 0x00, 0xF6,
+0x63, 0x33, 0x00, 0x53, 0xD7, 0x60, 0x20, 0x6B,
+0x4C, 0xEB, 0xFF, 0x6C, 0x8C, 0xEB, 0x0D, 0x23,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0x0C, 0x94,
+0xE3, 0x17, 0x09, 0x68, 0xE1, 0x17, 0x0B, 0x68,
+0xDF, 0x17, 0x1F, 0x6B, 0x6C, 0xEA, 0x8C, 0xEA,
+0x0E, 0xEA, 0xEE, 0x22, 0xBF, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x0A, 0xD4, 0x0B, 0xD5,
+0x0C, 0xD6, 0x40, 0xEA, 0x0D, 0xD7, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x02, 0x67,
+0x93, 0xF1, 0x54, 0x9B, 0x0A, 0x05, 0x00, 0x6C,
+0xF7, 0xF0, 0x01, 0x69, 0x04, 0xD3, 0x40, 0xEA,
+0x20, 0x31, 0x20, 0x31, 0x04, 0x93, 0x0A, 0x22,
+0x72, 0xF0, 0x54, 0x99, 0x40, 0xEA, 0x90, 0x67,
+0x00, 0x6A, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x02, 0x02, 0x56, 0xAA,
+0xE1, 0xF7, 0x1F, 0x72, 0x06, 0x60, 0x93, 0xF1,
+0x54, 0x9B, 0x0A, 0x05, 0x40, 0xEA, 0x09, 0x6C,
+0xEB, 0x2A, 0x72, 0xF0, 0x54, 0x99, 0x40, 0xEA,
+0x90, 0x67, 0x01, 0x6A, 0xEA, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x00, 0x6A, 0x07, 0xD2,
+0x7D, 0x67, 0x15, 0x6A, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xCA, 0xF0,
+0x0C, 0x4A, 0x05, 0xD2, 0xB8, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x09, 0x97, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF1, 0x50, 0x9A, 0x04, 0x67, 0x13, 0xD5,
+0x0C, 0xD6, 0x04, 0x04, 0x20, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x27, 0x67, 0x16, 0x32, 0x84, 0x42,
+0x04, 0x03, 0x88, 0x34, 0x91, 0xE3, 0x0C, 0x93,
+0x01, 0x6E, 0xC4, 0xE8, 0xC0, 0xDC, 0x86, 0x67,
+0x01, 0x2B, 0x00, 0x6C, 0x48, 0x30, 0x04, 0x02,
+0x09, 0xE2, 0x80, 0xDA, 0xB1, 0x67, 0x04, 0x04,
+0xD1, 0x18, 0xAA, 0x2E, 0x0C, 0xD6, 0x13, 0x92,
+0x0C, 0x96, 0x01, 0x2A, 0x00, 0x6E, 0x04, 0x02,
+0x01, 0xE2, 0xB1, 0x67, 0x82, 0x67, 0xD1, 0x18,
+0xC7, 0x2E, 0xC0, 0xD8, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0xC4, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xAD, 0xA2, 0x6C, 0xA2,
+0xEE, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x6F, 0xA2,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x20, 0xF0, 0xA9, 0xA3,
+0x20, 0xF0, 0x48, 0xA3, 0x20, 0xF0, 0xEA, 0xA3,
+0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0, 0x4B, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x84, 0x34, 0x89, 0xE2,
+0x61, 0xAA, 0xA0, 0xAA, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x06, 0xF3, 0x08, 0x4A,
+0x51, 0xE4, 0x80, 0xAC, 0x40, 0xEE, 0x6C, 0xED,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0x64, 0xA4, 0xC0, 0xAC, 0xE1, 0xAC, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x15, 0xF0,
+0x08, 0x4C, 0x0D, 0xA4, 0x4C, 0xA4, 0x2E, 0xA4,
+0x00, 0x30, 0x4D, 0xE8, 0x4F, 0xA4, 0x20, 0x31,
+0x20, 0x31, 0x2D, 0xE8, 0x00, 0xF6, 0x40, 0x32,
+0x0D, 0xEA, 0x20, 0xF0, 0x89, 0xA2, 0x20, 0xF0,
+0x08, 0xA2, 0x20, 0xF0, 0x2A, 0xA2, 0x80, 0x34,
+0x0D, 0xEC, 0x20, 0xF0, 0x0B, 0xA2, 0x20, 0x31,
+0x20, 0x31, 0x2D, 0xEC, 0x00, 0xF6, 0x00, 0x30,
+0x64, 0x32, 0x8D, 0xE8, 0x41, 0xE0, 0x80, 0xA8,
+0xEF, 0xE9, 0xCD, 0xE9, 0x46, 0x67, 0xEC, 0xEA,
+0x8C, 0xE9, 0x4D, 0xE9, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x09, 0xD4, 0x06, 0x6C, 0x0A, 0xD1, 0x08, 0xD7,
+0x07, 0xD6, 0x06, 0xD3, 0x05, 0xD5, 0x04, 0xD4,
+0x69, 0xF6, 0x07, 0x6F, 0x04, 0x6E, 0x00, 0x6D,
+0x40, 0x6C, 0x40, 0xEA, 0x0C, 0xD3, 0x0C, 0x93,
+0x20, 0xC8, 0xB1, 0x18, 0xA4, 0xE8, 0x83, 0x67,
+0x11, 0x97, 0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF,
+0x09, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x5D, 0x67, 0xB4, 0xC2, 0x7D, 0x67, 0x00, 0x6A,
+0x01, 0x74, 0x48, 0xCB, 0x0A, 0x60, 0x06, 0x24,
+0x02, 0x74, 0x13, 0x60, 0x03, 0x74, 0x14, 0x60,
+0x00, 0x6A, 0x0C, 0x10, 0x01, 0x6A, 0x4B, 0xEA,
+0x02, 0x10, 0x01, 0xF0, 0x00, 0x6A, 0x7D, 0x67,
+0xA6, 0x67, 0x04, 0x04, 0xB1, 0x18, 0xC2, 0xE8,
+0x49, 0xCB, 0x01, 0x6A, 0x07, 0x97, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0xF1, 0x00, 0x6A, 0xF3, 0x17,
+0x03, 0xF3, 0x01, 0x6A, 0xEC, 0x17, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x5D, 0x67, 0x01, 0x74,
+0xB4, 0xC2, 0x14, 0x60, 0x06, 0x24, 0x02, 0x74,
+0x14, 0x60, 0x03, 0x74, 0x15, 0x60, 0x00, 0x6A,
+0x0A, 0x10, 0x01, 0x6A, 0x4B, 0xEA, 0x7D, 0x67,
+0x48, 0xCB, 0xA6, 0x67, 0x04, 0x04, 0xB1, 0x18,
+0xC2, 0xE8, 0x49, 0xCB, 0x01, 0x6A, 0x07, 0x97,
+0x00, 0xEF, 0x04, 0x63, 0x01, 0xF0, 0x00, 0x6A,
+0xF2, 0x17, 0x00, 0xF1, 0x00, 0x6A, 0xEF, 0x17,
+0x01, 0x6A, 0x4B, 0xEA, 0x7D, 0x67, 0x48, 0xCB,
+0x03, 0xF7, 0x01, 0x6A, 0x4B, 0xEA, 0xE9, 0x17,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0x0F, 0x74, 0x38, 0x67, 0x0E, 0xD4, 0x0F, 0xD5,
+0x10, 0xD6, 0x00, 0x6A, 0x47, 0x29, 0x5D, 0x67,
+0x20, 0xF0, 0x1D, 0xA2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF1, 0x50, 0x9A,
+0x0E, 0x6E, 0x00, 0x6D, 0x40, 0xEA, 0x04, 0x04,
+0x5D, 0x67, 0x7D, 0x67, 0x20, 0xF0, 0x04, 0xC2,
+0x30, 0xCA, 0x02, 0x6D, 0x00, 0xF4, 0x00, 0x6A,
+0x08, 0x04, 0xB1, 0x18, 0xC2, 0xE8, 0x51, 0xCB,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF5, 0x4C, 0x9A, 0xF0, 0x67, 0x08, 0x6E,
+0x04, 0xF7, 0x10, 0x6D, 0x40, 0xEA, 0x03, 0x6C,
+0x01, 0x72, 0x20, 0x61, 0x07, 0x6A, 0x7D, 0x67,
+0x5A, 0xC3, 0x5D, 0x67, 0x40, 0xF0, 0x40, 0xA2,
+0x1E, 0x22, 0x03, 0x6A, 0x5B, 0xC3, 0x7D, 0x67,
+0x03, 0x6A, 0x58, 0xC3, 0x14, 0x6A, 0x59, 0xC3,
+0x04, 0x94, 0x05, 0x95, 0x06, 0x96, 0xB1, 0x18,
+0x5C, 0xE8, 0x07, 0x97, 0x01, 0x72, 0x0A, 0x61,
+0x02, 0x67, 0x00, 0xF4, 0x00, 0x6B, 0x5D, 0x67,
+0x02, 0x6D, 0x08, 0x04, 0xB1, 0x18, 0xC2, 0xE8,
+0x70, 0xCA, 0x50, 0x67, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63, 0x04, 0x6A,
+0x7D, 0x67, 0xE0, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0x07, 0xD5, 0x5D, 0x67, 0x5D, 0xA2, 0x06, 0xD4,
+0x08, 0xD6, 0x20, 0x22, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x89, 0xA3, 0x48, 0xA3, 0xAA, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x65, 0xA2, 0x86, 0xA2, 0xA4, 0xA2, 0x47, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x49, 0xA2, 0x09, 0x22, 0x06, 0x92, 0x0F, 0x72,
+0x00, 0x6A, 0x05, 0x61, 0x07, 0x95, 0x08, 0x96,
+0xB1, 0x18, 0x12, 0xE9, 0x0F, 0x6C, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x44, 0x67, 0xC1, 0xAA, 0xA0, 0xAA,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xED, 0xA2, 0x6C, 0xA2,
+0x0E, 0xA2, 0xE0, 0x37, 0x6D, 0xEF, 0x6F, 0xA2,
+0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF, 0x00, 0xF6,
+0x60, 0x33, 0xED, 0xEB, 0x20, 0xF0, 0xE9, 0xA3,
+0x20, 0xF0, 0x48, 0xA3, 0x20, 0xF0, 0x0A, 0xA3,
+0xE0, 0x37, 0x4D, 0xEF, 0x20, 0xF0, 0x4B, 0xA3,
+0x84, 0xA4, 0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF,
+0x00, 0xF6, 0x40, 0x32, 0x84, 0x30, 0xED, 0xEA,
+0x09, 0xE2, 0xE1, 0xAA, 0xCF, 0xEB, 0xAD, 0xEB,
+0xEC, 0xEB, 0xCC, 0xED, 0xAD, 0xEB, 0xB1, 0x18,
+0xA4, 0xE8, 0x61, 0xCA, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x70, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x06, 0xF3, 0x08, 0x4A, 0x41, 0xE0, 0x40, 0xEB,
+0x80, 0xA8, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x15, 0xF0, 0x08, 0x48,
+0x4D, 0xA0, 0x2C, 0xA0, 0x6E, 0xA0, 0x40, 0x32,
+0x2D, 0xEA, 0x2F, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xC9, 0xF1, 0xAC, 0x9B, 0x72, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x04, 0x6C, 0x42, 0x33, 0x20, 0xF0,
+0x48, 0xC1, 0x20, 0xF0, 0x69, 0xC1, 0x00, 0xF6,
+0x42, 0x32, 0x62, 0x33, 0x20, 0xF0, 0x6A, 0xC1,
+0x20, 0xF0, 0x4B, 0xC1, 0x8D, 0xA0, 0x4C, 0xA0,
+0x6E, 0xA0, 0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0, 0x69, 0xA2,
+0x20, 0xF0, 0x8A, 0xA2, 0x20, 0xF0, 0xA8, 0xA2,
+0x20, 0xF0, 0x4B, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x1A, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xCA, 0xF0, 0x0C, 0x4A,
+0x05, 0xD2, 0xFF, 0x6A, 0x1C, 0x4A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x01, 0x6B, 0x6B, 0xEB, 0x60, 0xCA,
+0x61, 0xCA, 0xF6, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0x46, 0xA4, 0x01, 0x6B, 0x0E, 0x6D, 0x6C, 0xEA,
+0x7D, 0x67, 0x54, 0xC3, 0x41, 0xA4, 0x60, 0xA4,
+0x40, 0x32, 0x6D, 0xEA, 0x7D, 0x67, 0x48, 0xCB,
+0x45, 0xA4, 0x64, 0xA4, 0x04, 0x04, 0x40, 0x32,
+0x6D, 0xEA, 0x7D, 0x67, 0xB1, 0x18, 0x59, 0xE9,
+0x49, 0xCB, 0x07, 0x97, 0x00, 0x6A, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0x40, 0xA4, 0x21, 0x22,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xA9, 0xA3, 0x48, 0xA3,
+0xCA, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x65, 0xA2, 0xA6, 0xA2,
+0xC4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x49, 0xA2, 0xA0, 0xF0,
+0x1F, 0x22, 0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1,
+0x05, 0xD0, 0x61, 0xA4, 0x42, 0xA4, 0x01, 0x6D,
+0xAC, 0xEB, 0x04, 0x67, 0x4C, 0xEB, 0xFF, 0x6C,
+0x8C, 0xEB, 0x0C, 0x2B, 0xC1, 0xA0, 0x62, 0x67,
+0xCC, 0xEB, 0x66, 0x33, 0xAC, 0xEB, 0x8C, 0xEB,
+0x05, 0x2B, 0xCC, 0xEA, 0x4A, 0x32, 0xAC, 0xEA,
+0x8C, 0xEA, 0x17, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x20, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x4C, 0x9A, 0x11, 0xF0, 0x1C, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6D, 0xAD, 0xEA,
+0xFF, 0x6D, 0x11, 0xF0, 0x1C, 0x6C, 0x40, 0xE9,
+0x4C, 0xED, 0x50, 0xA0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x6C, 0xEA, 0x20, 0x31,
+0x06, 0x22, 0xD2, 0xF4, 0x44, 0x99, 0x38, 0xF6,
+0x10, 0x6C, 0x40, 0xEA, 0xA2, 0xA8, 0x50, 0xA0,
+0x02, 0x6B, 0x6C, 0xEA, 0x06, 0x22, 0xD2, 0xF4,
+0x44, 0x99, 0x38, 0xF6, 0x12, 0x6C, 0x40, 0xEA,
+0xA3, 0xA8, 0x50, 0xA0, 0x04, 0x6B, 0x6C, 0xEA,
+0x06, 0x22, 0xD2, 0xF4, 0x44, 0x99, 0x38, 0xF6,
+0x14, 0x6C, 0x40, 0xEA, 0xA4, 0xA8, 0x50, 0xA0,
+0x08, 0x6B, 0x6C, 0xEA, 0x06, 0x22, 0xD2, 0xF4,
+0x44, 0x99, 0x38, 0xF6, 0x16, 0x6C, 0x40, 0xEA,
+0xA5, 0xA8, 0x50, 0xA0, 0x10, 0x6B, 0x6C, 0xEA,
+0x06, 0x22, 0xD2, 0xF4, 0x44, 0x99, 0x38, 0xF6,
+0x18, 0x6C, 0x40, 0xEA, 0xA6, 0xA8, 0x50, 0xA0,
+0x20, 0x6B, 0x6C, 0xEA, 0x06, 0x22, 0xD2, 0xF4,
+0x44, 0x99, 0x38, 0xF6, 0x1A, 0x6C, 0x40, 0xEA,
+0xA7, 0xA8, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A, 0x18, 0xF6,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x62, 0xA0,
+0xA2, 0x67, 0x01, 0x6C, 0x43, 0x67, 0x8C, 0xEA,
+0x0A, 0x22, 0x41, 0xA0, 0x8C, 0xEA, 0xFF, 0x6C,
+0x8C, 0xEA, 0xFF, 0xF7, 0x1F, 0x6C, 0x2B, 0x22,
+0x40, 0x6A, 0x4D, 0xED, 0x8C, 0xED, 0x02, 0x6C,
+0x43, 0x67, 0x8C, 0xEA, 0x0A, 0x22, 0x41, 0xA0,
+0x8C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA, 0xFF, 0xF7,
+0x1F, 0x6C, 0x21, 0x22, 0x80, 0x6A, 0x4D, 0xED,
+0x8C, 0xED, 0x04, 0x6C, 0x8C, 0xEB, 0x0B, 0x23,
+0x41, 0xA0, 0xFF, 0x6B, 0x8C, 0xEA, 0x6C, 0xEA,
+0xFF, 0xF7, 0x1F, 0x6B, 0x18, 0x22, 0xFF, 0x6A,
+0x01, 0x4A, 0x4D, 0xED, 0x6C, 0xED, 0xD2, 0xF4,
+0x44, 0x99, 0x18, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x07, 0x97, 0x06, 0x91, 0x05, 0x90,
+0x01, 0x6A, 0x00, 0xEF, 0x04, 0x63, 0x41, 0x6A,
+0x4B, 0xEA, 0x4C, 0xED, 0xD3, 0x17, 0x81, 0x6A,
+0x4B, 0xEA, 0x4C, 0xED, 0xDD, 0x17, 0xFF, 0x6A,
+0x02, 0x4A, 0x4B, 0xEA, 0x4C, 0xED, 0xE6, 0x17,
+0x20, 0xE8, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x62, 0xA4, 0x5D, 0x67, 0x04, 0x04, 0x74, 0xC2,
+0x7D, 0x67, 0x00, 0x6A, 0x48, 0xCB, 0x03, 0xF7,
+0x01, 0x6A, 0x4B, 0xEA, 0xB1, 0x18, 0xC2, 0xE8,
+0x49, 0xCB, 0x07, 0x97, 0x01, 0x6A, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A, 0x04, 0x67,
+0x18, 0xF6, 0x0C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x68, 0xA0, 0x42, 0x34, 0x02, 0x6F, 0x01, 0x6D,
+0x9A, 0x36, 0xEB, 0xEF, 0xAC, 0xEE, 0xEC, 0xEB,
+0xCD, 0xEB, 0xFF, 0xF7, 0x1F, 0x6E, 0x4C, 0xEE,
+0xC2, 0x36, 0xDE, 0x36, 0xAC, 0xEE, 0xFF, 0x4F,
+0xC4, 0x36, 0xEC, 0xEB, 0xCD, 0xEB, 0x96, 0x36,
+0xAC, 0xEE, 0xFE, 0x4F, 0x92, 0x34, 0xAC, 0xEC,
+0xC8, 0x36, 0xEC, 0xEB, 0x09, 0x6D, 0xCD, 0xEB,
+0xAB, 0xED, 0x8C, 0x34, 0xAC, 0xEB, 0x8D, 0xEB,
+0x68, 0xC0, 0x03, 0x6B, 0x6C, 0xEA, 0x02, 0x72,
+0x0C, 0x60, 0x03, 0x72, 0x0C, 0x60, 0x01, 0x72,
+0x00, 0x6A, 0x01, 0x61, 0x01, 0x6A, 0x41, 0xD8,
+0x05, 0x97, 0x04, 0x90, 0x01, 0x6A, 0x00, 0xEF,
+0x03, 0x63, 0x02, 0x6A, 0xF8, 0x17, 0x03, 0x6A,
+0xF6, 0x17, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0x41, 0x9C, 0x01, 0x72,
+0x07, 0x60, 0x4E, 0x22, 0x02, 0x72, 0x48, 0x60,
+0x03, 0x72, 0x48, 0x60, 0x00, 0x6A, 0x3F, 0x10,
+0x01, 0x68, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A, 0x24, 0x67,
+0x18, 0xF6, 0x0C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x88, 0xA1, 0x01, 0x6B, 0xE1, 0xF7, 0x1F, 0x6F,
+0x8C, 0xEB, 0xEC, 0xEA, 0x00, 0x6F, 0x02, 0x23,
+0x08, 0xF0, 0x00, 0x6F, 0x02, 0x6B, 0x8C, 0xEB,
+0x00, 0x6E, 0x03, 0x23, 0x10, 0xF0, 0x00, 0x6E,
+0xCB, 0xEE, 0x04, 0x6B, 0x8C, 0xEB, 0x00, 0x6D,
+0x02, 0x23, 0x04, 0xF0, 0x00, 0x6D, 0x08, 0x6B,
+0x6C, 0xEC, 0x00, 0x6B, 0x02, 0x24, 0x02, 0xF0,
+0x00, 0x6B, 0x4D, 0xEF, 0xED, 0xEE, 0xCD, 0xED,
+0xAD, 0xEB, 0xF7, 0xF0, 0x01, 0x6A, 0x04, 0x6D,
+0xAB, 0xED, 0x40, 0x32, 0xAC, 0xEB, 0x40, 0x32,
+0xFF, 0xF7, 0x1F, 0x6D, 0xAC, 0xEB, 0xD2, 0xF4,
+0x44, 0x9A, 0x0D, 0xEB, 0x18, 0xF6, 0x0C, 0x6C,
+0x40, 0xEA, 0x6C, 0xED, 0x01, 0x6A, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x02, 0x68, 0xBB, 0x17, 0x03, 0x68, 0xB9, 0x17,
+0x00, 0x68, 0xB7, 0x17, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x04, 0x67, 0x85, 0x67,
+0xB1, 0x18, 0x3D, 0xEA, 0x25, 0x67, 0x01, 0x72,
+0x07, 0x60, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0xFF, 0x6B, 0x6C, 0xEA, 0x00, 0xEF, 0x06, 0x63,
+0x60, 0xA1, 0x5D, 0x67, 0x02, 0x70, 0x70, 0xC2,
+0x41, 0x99, 0x05, 0xD2, 0x5D, 0x67, 0x78, 0xA2,
+0x10, 0x6A, 0x4B, 0xEA, 0x6C, 0xEA, 0x7D, 0x67,
+0x58, 0xC3, 0x15, 0x60, 0x03, 0x58, 0x08, 0x60,
+0x10, 0x20, 0x01, 0x70, 0x02, 0x6B, 0x09, 0x60,
+0xB1, 0x18, 0x5F, 0xEA, 0x04, 0x04, 0xE1, 0x17,
+0x03, 0x70, 0x0B, 0x60, 0x04, 0x70, 0x0F, 0x6B,
+0xF7, 0x61, 0x6D, 0xEA, 0x7D, 0x67, 0x58, 0xC3,
+0xF3, 0x17, 0x01, 0x6B, 0xFA, 0x17, 0x04, 0x6B,
+0xF8, 0x17, 0x08, 0x6B, 0xF6, 0x17, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0x45, 0xA4, 0x04, 0x67, 0x27, 0x67, 0x0F, 0x2A,
+0x81, 0xA0, 0x0C, 0x97, 0x01, 0x6D, 0xB1, 0x18,
+0x88, 0xE8, 0x01, 0x6E, 0xB1, 0x67, 0xB1, 0x18,
+0x33, 0xEA, 0x90, 0x67, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x42, 0xA0,
+0x85, 0x67, 0xA7, 0x46, 0x48, 0xC6, 0xB1, 0x18,
+0x8B, 0xEA, 0x01, 0x4D, 0x01, 0x72, 0xF2, 0x61,
+0xE7, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x62, 0xA4, 0x5D, 0x67,
+0x04, 0x67, 0x74, 0xC2, 0x01, 0x6B, 0x6B, 0xEB,
+0x68, 0xCA, 0x03, 0xF7, 0x01, 0x6B, 0x6B, 0xEB,
+0x06, 0xD5, 0x04, 0x04, 0xA6, 0x67, 0x69, 0xCA,
+0xB1, 0x18, 0xC2, 0xE8, 0x27, 0x67, 0x65, 0xA0,
+0x0E, 0x23, 0x06, 0x92, 0x87, 0x42, 0xB1, 0x18,
+0x5F, 0xEA, 0x01, 0x4C, 0x01, 0x72, 0x07, 0x60,
+0xFF, 0x6B, 0x6C, 0xEA, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x81, 0xA0,
+0xF1, 0x67, 0x00, 0x6E, 0xB1, 0x18, 0x88, 0xE8,
+0x00, 0x6D, 0x01, 0x6A, 0xF3, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x13, 0xF6, 0x0C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x69, 0xF1, 0xAC, 0x9B, 0x62, 0x67, 0xAC, 0xEB,
+0x11, 0x23, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xF7, 0xF0, 0x01, 0x6B, 0x80, 0x34, 0xC9, 0xF1,
+0x90, 0x9C, 0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4,
+0x68, 0x9B, 0x8C, 0xEA, 0x13, 0xF6, 0x0C, 0x6C,
+0x40, 0xEB, 0x4D, 0xED, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xF4, 0x63, 0x17, 0x62,
+0x16, 0xD1, 0x15, 0xD0, 0x01, 0x75, 0x07, 0x60,
+0x63, 0x25, 0x02, 0x75, 0x5D, 0x60, 0x03, 0x75,
+0x5D, 0x60, 0x00, 0x68, 0x53, 0x10, 0x01, 0x6A,
+0x24, 0x67, 0x1D, 0x6C, 0x12, 0xD5, 0x04, 0xD4,
+0x06, 0x6F, 0x06, 0x06, 0xA2, 0x67, 0xB1, 0x18,
+0xA8, 0xEA, 0x91, 0x67, 0x01, 0x72, 0x10, 0xD2,
+0x58, 0x67, 0x11, 0xD2, 0x4F, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF5,
+0x58, 0x9A, 0xE2, 0xA1, 0x81, 0xA1, 0x81, 0xF0,
+0x18, 0x6D, 0x40, 0xEA, 0x1E, 0x6E, 0x01, 0x72,
+0x02, 0x67, 0x01, 0x6B, 0x23, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF1,
+0x50, 0x9A, 0x0E, 0x6E, 0x00, 0x6D, 0x40, 0xEA,
+0x0B, 0x04, 0x61, 0xA1, 0x9D, 0x67, 0x40, 0xF0,
+0x88, 0xA4, 0x5D, 0x67, 0x20, 0xF0, 0x73, 0xC2,
+0x7D, 0x67, 0x20, 0xF0, 0x95, 0xC3, 0x03, 0x6A,
+0x07, 0x6C, 0x20, 0xF0, 0x96, 0xC3, 0x20, 0xF0,
+0x54, 0xC3, 0x20, 0xF0, 0x57, 0xC3, 0x0B, 0x94,
+0x0C, 0x95, 0x0D, 0x96, 0xB1, 0x18, 0x5C, 0xE8,
+0x0E, 0x97, 0x62, 0x67, 0x91, 0x67, 0x1D, 0x6F,
+0x06, 0x6E, 0x06, 0x05, 0xB1, 0x18, 0xB9, 0xEA,
+0x12, 0xD3, 0x11, 0x94, 0x12, 0x93, 0x15, 0x2C,
+0x01, 0x70, 0x04, 0x61, 0x01, 0x73, 0x03, 0x67,
+0x01, 0x61, 0x02, 0x67, 0x50, 0x67, 0x17, 0x97,
+0x16, 0x91, 0x15, 0x90, 0x00, 0xEF, 0x0C, 0x63,
+0x02, 0x6A, 0xA6, 0x17, 0x03, 0x6A, 0xA4, 0x17,
+0x00, 0x6A, 0xA2, 0x17, 0x01, 0x6B, 0x01, 0x68,
+0xE1, 0x17, 0x10, 0x90, 0xEF, 0x17, 0x00, 0x65,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0x1E, 0x6A, 0x04, 0xD2, 0x07, 0x6F, 0x06, 0x06,
+0x04, 0x6D, 0xB1, 0x18, 0xA8, 0xEA, 0x24, 0x67,
+0x01, 0x72, 0x10, 0xD2, 0x58, 0x67, 0x11, 0xD2,
+0x1B, 0x22, 0x01, 0x6A, 0x82, 0x67, 0x01, 0x68,
+0x12, 0xD4, 0x1E, 0x6F, 0x91, 0x67, 0x07, 0x6E,
+0xB1, 0x18, 0xB9, 0xEA, 0x06, 0x05, 0x11, 0x93,
+0x12, 0x94, 0x4D, 0x2B, 0x01, 0x70, 0x78, 0x67,
+0x05, 0x2B, 0x01, 0x74, 0x78, 0x67, 0x04, 0x67,
+0x01, 0x2B, 0x02, 0x67, 0x50, 0x67, 0x17, 0x97,
+0x16, 0x91, 0x15, 0x90, 0x00, 0xEF, 0x0C, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x81, 0xA1, 0x12, 0xF5, 0x58, 0x9A, 0xE2, 0xA1,
+0x81, 0xF0, 0x18, 0x6D, 0x40, 0xEA, 0x1E, 0x6E,
+0x01, 0x72, 0x02, 0x67, 0x01, 0x6B, 0x58, 0x67,
+0x83, 0x67, 0xD6, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF1, 0x50, 0x9A,
+0x0B, 0x04, 0x0E, 0x6E, 0x40, 0xEA, 0x00, 0x6D,
+0x61, 0xA1, 0x5D, 0x67, 0x9D, 0x67, 0x20, 0xF0,
+0x73, 0xC2, 0x03, 0x6A, 0x7D, 0x67, 0x20, 0xF0,
+0x54, 0xC3, 0x07, 0x6B, 0x20, 0xF0, 0x76, 0xC4,
+0x7D, 0x67, 0x20, 0xF0, 0x57, 0xC3, 0x00, 0x6B,
+0x5D, 0x67, 0x20, 0xF0, 0x75, 0xC2, 0x0B, 0x94,
+0x0C, 0x95, 0x0D, 0x96, 0x0E, 0x97, 0xB1, 0x18,
+0x5C, 0xE8, 0x12, 0xD3, 0x01, 0x72, 0x82, 0x67,
+0x58, 0x67, 0xAE, 0x2A, 0x12, 0x93, 0x01, 0x4B,
+0x04, 0x73, 0xEE, 0x61, 0xA6, 0x17, 0x10, 0x90,
+0xB9, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF1, 0x50, 0x9A, 0x04, 0x04, 0x0E, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x7D, 0x67, 0x07, 0x6A,
+0x5A, 0xC3, 0x03, 0x6A, 0x5B, 0xC3, 0x58, 0xC3,
+0x11, 0x6A, 0x59, 0xC3, 0x07, 0x97, 0x04, 0x94,
+0x05, 0x95, 0xB1, 0x18, 0x5C, 0xE8, 0x06, 0x96,
+0x09, 0x97, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x72, 0xF1, 0x50, 0x99, 0x0D, 0xD4, 0x13, 0xD5,
+0x06, 0x67, 0x04, 0x04, 0x20, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x0C, 0xD7, 0x0D, 0x93, 0x72, 0xF1,
+0x50, 0x99, 0x0B, 0xED, 0xC0, 0xF7, 0xA3, 0x35,
+0xFF, 0x68, 0x08, 0xD3, 0x10, 0x6E, 0x0C, 0xED,
+0x40, 0xEA, 0x04, 0x04, 0x0C, 0x97, 0x04, 0x04,
+0xD1, 0x18, 0xAA, 0x2E, 0xA7, 0x67, 0x13, 0x93,
+0x72, 0xF1, 0x50, 0x99, 0x10, 0x6E, 0x6B, 0xED,
+0xC0, 0xF7, 0xA3, 0x35, 0x0C, 0xED, 0x40, 0xEA,
+0x04, 0x04, 0x0C, 0x97, 0x04, 0x04, 0xD1, 0x18,
+0xC7, 0x2E, 0xA7, 0x67, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x00, 0x65,
+0xF1, 0x63, 0x1D, 0x62, 0x1C, 0xD1, 0x1B, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x42, 0xA4, 0x00, 0x30,
+0x00, 0x30, 0x15, 0xD2, 0x72, 0xF1, 0x50, 0x98,
+0x24, 0x67, 0x20, 0x6E, 0x00, 0x6D, 0x40, 0xEA,
+0x04, 0x04, 0x72, 0xF1, 0x50, 0x98, 0x0E, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x0C, 0x04, 0x42, 0x99,
+0x1F, 0x6F, 0x01, 0x6E, 0x4F, 0xEA, 0x82, 0x67,
+0x01, 0x6D, 0xB1, 0x18, 0x64, 0xEB, 0x18, 0xD2,
+0x15, 0x95, 0x08, 0x6E, 0xB1, 0x18, 0xEB, 0xE8,
+0x00, 0x6C, 0x01, 0x72, 0x08, 0x60, 0xFF, 0x68,
+0x4C, 0xE8, 0x50, 0x67, 0x1D, 0x97, 0x1C, 0x91,
+0x1B, 0x90, 0x00, 0xEF, 0x0F, 0x63, 0x7D, 0x67,
+0x40, 0xF0, 0x74, 0xA3, 0x5D, 0x67, 0x10, 0x05,
+0x40, 0xF0, 0x60, 0xC2, 0xB1, 0x18, 0x8B, 0xEA,
+0x04, 0x6C, 0x01, 0x72, 0x17, 0xD2, 0xA0, 0xF0,
+0x10, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xC2, 0xA1, 0x12, 0xF5, 0x54, 0x9A,
+0x81, 0xF0, 0x18, 0x6C, 0x40, 0xEA, 0x1E, 0x6D,
+0x01, 0x72, 0x02, 0x67, 0x01, 0x69, 0x7E, 0x61,
+0x7D, 0x67, 0x07, 0x6A, 0x20, 0xF0, 0x5A, 0xC3,
+0x03, 0x6A, 0x20, 0xF0, 0x5B, 0xC3, 0x15, 0x93,
+0x04, 0x6A, 0x01, 0x2B, 0x03, 0x6A, 0x7D, 0x67,
+0x20, 0xF0, 0x58, 0xC3, 0x10, 0x6B, 0x5D, 0x67,
+0x20, 0xF0, 0x79, 0xC2, 0x0C, 0x94, 0x0D, 0x95,
+0x0E, 0x96, 0x0F, 0x97, 0xB1, 0x18, 0x5C, 0xE8,
+0x14, 0xD3, 0x01, 0x72, 0x22, 0x67, 0x57, 0x61,
+0x14, 0x93, 0x01, 0x4B, 0x15, 0x73, 0xEF, 0x61,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x54, 0x9B, 0x91, 0xF5, 0x00, 0x6C,
+0x40, 0xEA, 0x14, 0xD3, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xD2, 0xF4, 0xC8, 0x9C,
+0x08, 0x6D, 0xAB, 0xED, 0x91, 0xF5, 0x00, 0x6C,
+0x40, 0xEE, 0x4C, 0xED, 0x14, 0x93, 0x91, 0xF5,
+0x04, 0x6C, 0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x4F, 0xEA, 0x14, 0xD2, 0x00, 0x6A,
+0x16, 0xD2, 0x2F, 0x10, 0x14, 0x93, 0x01, 0x6A,
+0x6C, 0xEA, 0x23, 0x22, 0x7D, 0x67, 0x40, 0xF0,
+0x78, 0xA3, 0x5D, 0x67, 0x9D, 0x67, 0x20, 0xF0,
+0x77, 0xC2, 0x03, 0x6A, 0x7D, 0x67, 0x20, 0xF0,
+0x58, 0xC3, 0x07, 0x6B, 0x20, 0xF0, 0x7A, 0xC4,
+0x7D, 0x67, 0x20, 0xF0, 0x5B, 0xC3, 0x00, 0x6B,
+0x5D, 0x67, 0x20, 0xF0, 0x79, 0xC2, 0x0C, 0x94,
+0x0D, 0x95, 0x0E, 0x96, 0x0F, 0x97, 0xB1, 0x18,
+0x5C, 0xE8, 0x19, 0xD3, 0x01, 0x72, 0x22, 0x67,
+0x04, 0x61, 0x19, 0x93, 0x01, 0x4B, 0x04, 0x73,
+0xEF, 0x61, 0x16, 0x93, 0xFF, 0x6A, 0x01, 0x4B,
+0x4C, 0xEB, 0x14, 0x92, 0x16, 0xD3, 0x46, 0x32,
+0x14, 0xD2, 0x14, 0x92, 0xCF, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x4C, 0x9A, 0x40, 0xEA, 0x01, 0x6C, 0xB1, 0x18,
+0xD0, 0xEA, 0x00, 0x65, 0x18, 0x94, 0x01, 0x6F,
+0x00, 0x6E, 0xB1, 0x18, 0x64, 0xEB, 0x00, 0x6D,
+0x15, 0x95, 0x08, 0x6E, 0xB1, 0x18, 0xFE, 0xE8,
+0x00, 0x6C, 0x10, 0x04, 0xB1, 0x18, 0x5F, 0xEA,
+0x14, 0xD2, 0x17, 0x93, 0x14, 0x95, 0x01, 0x73,
+0x13, 0x61, 0x01, 0x70, 0x5F, 0xF7, 0x09, 0x61,
+0x01, 0x71, 0x11, 0x67, 0x5F, 0xF7, 0x05, 0x61,
+0xFF, 0x6B, 0x6C, 0xED, 0x01, 0x75, 0x05, 0x67,
+0x3F, 0xF7, 0x1F, 0x61, 0x6C, 0xEA, 0x02, 0x67,
+0x3C, 0x17, 0x01, 0x69, 0x01, 0x68, 0xDA, 0x17,
+0x17, 0x90, 0x37, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF1, 0x50, 0x9A, 0x04, 0x04, 0x0E, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x7D, 0x67, 0x07, 0x6A,
+0x5A, 0xC3, 0x03, 0x6A, 0x5B, 0xC3, 0x58, 0xC3,
+0x12, 0x6A, 0x59, 0xC3, 0x07, 0x97, 0x04, 0x94,
+0x05, 0x95, 0xB1, 0x18, 0x5C, 0xE8, 0x06, 0x96,
+0x09, 0x97, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x40, 0xA4, 0x05, 0x72, 0x2A, 0x60, 0x06, 0x72,
+0x26, 0x61, 0x01, 0x69, 0x00, 0x6A, 0x02, 0xA4,
+0x05, 0xD4, 0x09, 0x6E, 0xB0, 0x67, 0x91, 0x67,
+0xB1, 0x18, 0xEB, 0xE8, 0x04, 0xD2, 0x01, 0x72,
+0x05, 0x93, 0x2D, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF5, 0x4C, 0x9A,
+0x04, 0x94, 0xF0, 0x67, 0x1E, 0x6E, 0x81, 0xF0,
+0x18, 0x6D, 0x40, 0xEA, 0x06, 0xD3, 0x01, 0x72,
+0x04, 0xD2, 0x58, 0x67, 0x05, 0xD2, 0x01, 0x6F,
+0x10, 0x2A, 0x06, 0x93, 0x40, 0xA3, 0x05, 0x72,
+0x07, 0x60, 0x06, 0x72, 0x19, 0x60, 0x00, 0x6A,
+0x12, 0x10, 0x02, 0x69, 0x01, 0x6A, 0xD7, 0x17,
+0xB1, 0x18, 0xF9, 0xEB, 0x90, 0x67, 0xFF, 0x6F,
+0x4C, 0xEF, 0x09, 0x6E, 0xB0, 0x67, 0x91, 0x67,
+0xB1, 0x18, 0xFE, 0xE8, 0x06, 0xD7, 0x05, 0x93,
+0x06, 0x97, 0x0A, 0x23, 0x04, 0x92, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0xB1, 0x18, 0x55, 0xEB, 0x83, 0x67, 0xEB, 0x17,
+0x01, 0x77, 0xF5, 0x60, 0x47, 0x67, 0xF3, 0x17,
+0x20, 0xE8, 0x01, 0x6A, 0xFD, 0x63, 0x05, 0x62,
+0x60, 0xA4, 0x05, 0x6A, 0x0B, 0x5B, 0x12, 0x60,
+0x68, 0x32, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x68, 0xF2, 0x08, 0x4B, 0x6D, 0xE2,
+0x40, 0x9B, 0x00, 0xEA, 0x00, 0x65, 0x00, 0x6D,
+0xB1, 0x18, 0xE5, 0xEA, 0x00, 0x65, 0x01, 0x72,
+0x01, 0x61, 0x00, 0x6A, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x01, 0x6D, 0xF5, 0x17, 0x02, 0x6D,
+0xF3, 0x17, 0x03, 0x6D, 0xF1, 0x17, 0xB1, 0x18,
+0x1E, 0xEB, 0x00, 0x65, 0xF0, 0x17, 0xB1, 0x18,
+0x80, 0xEB, 0x00, 0x65, 0xEC, 0x17, 0xB1, 0x18,
+0x08, 0xEC, 0x00, 0x65, 0xFF, 0x6B, 0x6C, 0xEA,
+0xE6, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0xD2, 0xF4, 0x50, 0x98,
+0x59, 0xF6, 0x02, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x04, 0x6B, 0x4C, 0xEB, 0x00, 0x6A, 0x26, 0x23,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x4C, 0x9A, 0xF7, 0xF0, 0x01, 0x69,
+0xF9, 0xF6, 0x10, 0x6C, 0x40, 0xEA, 0x20, 0x31,
+0x20, 0x31, 0x04, 0xD2, 0xD2, 0xF4, 0x40, 0x99,
+0xF9, 0xF6, 0x10, 0x6C, 0x40, 0xEA, 0x06, 0x6D,
+0xD2, 0xF4, 0x50, 0x98, 0xF9, 0xF6, 0x12, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x04, 0x93, 0x02, 0x67,
+0xD2, 0xF4, 0x40, 0x99, 0xF9, 0xF6, 0x10, 0x6C,
+0x40, 0xEA, 0xA3, 0x67, 0x09, 0x70, 0x58, 0x67,
+0x01, 0x5A, 0x58, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0xF7, 0xF0, 0x01, 0x6A,
+0x20, 0x31, 0x40, 0x32, 0x20, 0x31, 0x40, 0x32,
+0xD2, 0xF4, 0x60, 0x99, 0xD2, 0xF4, 0x50, 0x9A,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEA, 0x04, 0xD3,
+0x04, 0x93, 0xA1, 0x42, 0xFF, 0x68, 0xE0, 0xF1,
+0x1C, 0x6C, 0x40, 0xEB, 0x0C, 0xED, 0xB1, 0x18,
+0x49, 0xEC, 0x00, 0x6C, 0x01, 0x72, 0x4D, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x68, 0x9A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x19, 0xF0, 0x00, 0x6C, 0x40, 0xEA, 0x04, 0xD3,
+0x04, 0x93, 0x10, 0xF0, 0x00, 0x6D, 0x19, 0xF0,
+0x00, 0x6C, 0x40, 0xEB, 0x4D, 0xED, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0x15, 0xF0,
+0x89, 0xA5, 0x15, 0xF0, 0x48, 0xA5, 0x15, 0xF0,
+0x6A, 0xA5, 0x80, 0x34, 0x4D, 0xEC, 0x15, 0xF0,
+0x4B, 0xA5, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x61, 0xA2, 0x20, 0xF0, 0x82, 0xA2, 0x20, 0xF0,
+0xA0, 0xA2, 0x20, 0xF0, 0x43, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0xA2, 0xA2,
+0x30, 0xF4, 0x06, 0x6C, 0x01, 0x4D, 0x0C, 0xED,
+0xA2, 0xC2, 0xD2, 0xF4, 0x40, 0x99, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x13, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFB, 0x63, 0x09, 0x62,
+0x00, 0x6A, 0x07, 0xD2, 0x1D, 0x24, 0x07, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xEA, 0xF0, 0x04, 0x4A,
+0x05, 0xD2, 0x1D, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xC9, 0xF1,
+0x54, 0x9A, 0x09, 0x97, 0x00, 0xEF, 0x05, 0x63,
+0x02, 0x5D, 0x0C, 0x61, 0x07, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xEA, 0xF0, 0x04, 0x4A, 0x05, 0xD2,
+0x22, 0x6A, 0xE0, 0x17, 0x02, 0x5E, 0x0C, 0x61,
+0x07, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xEA, 0xF0,
+0x04, 0x4A, 0x05, 0xD2, 0x27, 0x6A, 0xD2, 0x17,
+0x0B, 0x5F, 0x54, 0x60, 0xE8, 0x32, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0x88, 0xF2,
+0x14, 0x4F, 0xFD, 0xE2, 0x40, 0x9F, 0x00, 0xEA,
+0x00, 0x65, 0x54, 0x25, 0x58, 0xF2, 0x00, 0x6A,
+0xD0, 0x17, 0x53, 0x25, 0x58, 0xF2, 0x06, 0x6A,
+0xCC, 0x17, 0x52, 0x25, 0x58, 0xF2, 0x08, 0x6A,
+0xC8, 0x17, 0x51, 0x25, 0x58, 0xF2, 0x0C, 0x6A,
+0xC4, 0x17, 0x04, 0x2D, 0x4F, 0x26, 0x38, 0xF2,
+0x00, 0x6A, 0xBF, 0x17, 0x4E, 0x26, 0x78, 0xF2,
+0x00, 0x6A, 0xBB, 0x17, 0x04, 0x2D, 0x4C, 0x26,
+0x38, 0xF2, 0x04, 0x6A, 0xB6, 0x17, 0x4B, 0x26,
+0x78, 0xF2, 0x04, 0x6A, 0xB2, 0x17, 0x04, 0x2D,
+0x49, 0x26, 0x38, 0xF2, 0x08, 0x6A, 0xAD, 0x17,
+0x48, 0x26, 0x78, 0xF2, 0x08, 0x6A, 0xA9, 0x17,
+0x04, 0x2D, 0x46, 0x26, 0x38, 0xF2, 0x0C, 0x6A,
+0xA4, 0x17, 0x45, 0x26, 0x78, 0xF2, 0x0C, 0x6A,
+0xA0, 0x17, 0x04, 0x2D, 0x43, 0x26, 0x38, 0xF2,
+0x14, 0x6A, 0x9B, 0x17, 0x42, 0x26, 0x78, 0xF2,
+0x14, 0x6A, 0x97, 0x17, 0x02, 0x6A, 0x95, 0x26,
+0x04, 0x6A, 0x93, 0x17, 0x3D, 0x26, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x09, 0xF1,
+0x5C, 0x9A, 0x8B, 0x17, 0x07, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xEA, 0xF0, 0x04, 0x4A, 0x05, 0xD2,
+0x6D, 0x6A, 0x70, 0x17, 0x18, 0xF2, 0x00, 0x6A,
+0x7C, 0x17, 0x18, 0xF2, 0x06, 0x6A, 0x79, 0x17,
+0x18, 0xF2, 0x08, 0x6A, 0x76, 0x17, 0x18, 0xF2,
+0x0C, 0x6A, 0x73, 0x17, 0x18, 0xF2, 0x10, 0x6A,
+0x70, 0x17, 0x58, 0xF2, 0x10, 0x6A, 0x6D, 0x17,
+0x18, 0xF2, 0x14, 0x6A, 0x6A, 0x17, 0x58, 0xF2,
+0x14, 0x6A, 0x67, 0x17, 0x18, 0xF2, 0x18, 0x6A,
+0x64, 0x17, 0x58, 0xF2, 0x18, 0x6A, 0x61, 0x17,
+0x18, 0xF2, 0x1C, 0x6A, 0x5E, 0x17, 0x58, 0xF2,
+0x1C, 0x6A, 0x5B, 0x17, 0x38, 0xF2, 0x10, 0x6A,
+0x58, 0x17, 0x78, 0xF2, 0x10, 0x6A, 0x55, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC9, 0xF1, 0x58, 0x9A, 0x4E, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x84, 0x34,
+0xB1, 0xE4, 0x6C, 0xF2, 0x04, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x88, 0x35, 0x40, 0x32, 0x95, 0xE5,
+0x40, 0x32, 0xAC, 0x35, 0x93, 0xF6, 0x40, 0x9A,
+0xA1, 0xE0, 0x90, 0x67, 0x28, 0x6E, 0x40, 0xEA,
+0x00, 0x6D, 0x40, 0xA0, 0x04, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x01, 0x6B, 0x6D, 0xEA, 0xE2, 0x4B,
+0x6C, 0xEA, 0x14, 0x6B, 0x6D, 0xEA, 0x40, 0xC0,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x00, 0x6D, 0xB1, 0x18,
+0x0A, 0xED, 0x00, 0x6C, 0x01, 0x6D, 0xB1, 0x18,
+0x0A, 0xED, 0x00, 0x6C, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x69, 0x60, 0x33, 0x60, 0x33,
+0x20, 0x31, 0xC9, 0xF1, 0x1C, 0x9B, 0x20, 0x31,
+0x72, 0xF4, 0x54, 0x99, 0xB0, 0x67, 0x40, 0xEA,
+0x50, 0x6C, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x6C, 0xF2, 0x44, 0xDB, 0x19, 0x2A,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xEA, 0xF0,
+0x04, 0x4A, 0x05, 0xD2, 0x82, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x72, 0xF4, 0x54, 0x99, 0xB0, 0x67,
+0x40, 0xEA, 0x18, 0x6C, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x6C, 0xF2, 0x40, 0xDB,
+0x0C, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xEA, 0xF0, 0x04, 0x4A, 0x05, 0xD2, 0x8A, 0x6A,
+0xDA, 0x17, 0xB1, 0x18, 0x1E, 0xED, 0x00, 0x65,
+0xDF, 0x17, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x6C, 0xF2, 0xA4, 0x9A,
+0x8C, 0x33, 0x84, 0x32, 0x4D, 0xE3, 0x6C, 0x33,
+0x6D, 0xE5, 0x80, 0xA3, 0x20, 0x6F, 0xEB, 0xEF,
+0xEC, 0xEC, 0xFF, 0x6E, 0xCC, 0xEC, 0x13, 0x24,
+0x01, 0x4A, 0x48, 0x33, 0x4D, 0xE3, 0x6C, 0x33,
+0x6D, 0xE5, 0x80, 0xA3, 0x02, 0x6A, 0xEC, 0xEC,
+0xCC, 0xEC, 0x07, 0x2C, 0x01, 0x6A, 0x80, 0xA3,
+0x1F, 0x6D, 0xAC, 0xEC, 0x20, 0x6D, 0xAD, 0xEC,
+0x80, 0xC3, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x6A,
+0xF6, 0x17, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x24, 0x67, 0x2C, 0x33,
+0x84, 0x34, 0x8D, 0xE3, 0x6C, 0x32, 0x08, 0xD2,
+0x28, 0x32, 0x89, 0xE2, 0x48, 0x32, 0x09, 0xD2,
+0x00, 0x6A, 0x11, 0xD5, 0x04, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x08, 0x93,
+0x6C, 0xF2, 0x44, 0x9A, 0x69, 0xE2, 0x80, 0xA2,
+0x20, 0x6B, 0x6B, 0xEB, 0x8C, 0xEB, 0x13, 0x2B,
+0x04, 0x93, 0xFF, 0x6A, 0x01, 0x4B, 0x4C, 0xEB,
+0x08, 0x92, 0x02, 0x73, 0x04, 0xD3, 0x28, 0x4A,
+0x08, 0xD2, 0x09, 0x92, 0x0C, 0x4A, 0x09, 0xD2,
+0xE6, 0x61, 0x00, 0x6A, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x04, 0x4A, 0x60, 0x33,
+0x6C, 0xF2, 0x60, 0x9B, 0x07, 0xD2, 0x09, 0x92,
+0x00, 0x68, 0x49, 0xE3, 0x05, 0xD2, 0x08, 0x4A,
+0x06, 0xD2, 0x07, 0x92, 0x40, 0xA2, 0x01, 0x72,
+0x3D, 0x61, 0x11, 0x92, 0x4A, 0x2A, 0x04, 0x95,
+0xD0, 0x67, 0x07, 0x6F, 0xB1, 0x18, 0xA1, 0xEC,
+0x91, 0x67, 0xC2, 0x67, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC9, 0xF1, 0x74, 0x9A,
+0x6A, 0xEE, 0x02, 0x61, 0x11, 0x6A, 0xD6, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x50, 0x9A, 0x86, 0x67, 0x0B, 0xD3,
+0x40, 0xEA, 0x0A, 0xD6, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x06, 0x93, 0x0A, 0x96, 0x80, 0x34,
+0xD2, 0xF4, 0xE4, 0x9C, 0xFF, 0x6D, 0x40, 0xCB,
+0x4C, 0xED, 0x40, 0xEF, 0x86, 0x67, 0x04, 0x95,
+0x0A, 0x6F, 0xD0, 0x67, 0xB1, 0x18, 0xA1, 0xEC,
+0x91, 0x67, 0x0B, 0x93, 0x4E, 0xEB, 0xDE, 0x23,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x74, 0x9B, 0x40, 0xEB, 0x82, 0x67,
+0x05, 0x93, 0x40, 0xDB, 0xFF, 0x6A, 0x01, 0x48,
+0x4C, 0xE8, 0x07, 0x92, 0x02, 0x70, 0x01, 0x4A,
+0x07, 0xD2, 0x05, 0x92, 0x04, 0x4A, 0x05, 0xD2,
+0x06, 0x92, 0x02, 0x4A, 0x06, 0xD2, 0x90, 0x60,
+0xB0, 0x17, 0x11, 0x92, 0x01, 0x72, 0xEE, 0x61,
+0x04, 0x95, 0x0A, 0x6F, 0xD0, 0x67, 0xB1, 0x18,
+0xA1, 0xEC, 0x91, 0x67, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xC9, 0xF1, 0x74, 0x9B,
+0x6A, 0xEA, 0x0A, 0xD3, 0xB3, 0x60, 0x05, 0x93,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xA0, 0x9B, 0xD2, 0xF4, 0xC8, 0x9C, 0x40, 0xEE,
+0x82, 0x67, 0x04, 0x95, 0x07, 0x6F, 0xD0, 0x67,
+0xB1, 0x18, 0xA1, 0xEC, 0x91, 0x67, 0x0A, 0x93,
+0x4E, 0xEB, 0xA0, 0x23, 0x06, 0x94, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xA0, 0xAC,
+0xD2, 0xF4, 0x64, 0x9B, 0x40, 0xEB, 0x82, 0x67,
+0xC1, 0x17, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x16, 0x24, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x01, 0x6B, 0x05, 0xD4, 0x04, 0xD3,
+0x20, 0xF2, 0x1D, 0x6F, 0x02, 0x6E, 0x01, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x09, 0x6A,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0xA8, 0x33, 0xAD, 0xE3, 0x6C, 0x32,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x6C, 0xF2, 0x64, 0x9B, 0x05, 0x67, 0x26, 0x67,
+0x4D, 0xE3, 0xA0, 0xA3, 0x40, 0x6A, 0x4B, 0xEA,
+0xAC, 0xEA, 0x16, 0x26, 0xFF, 0x6D, 0xAC, 0xEA,
+0x29, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x02, 0x6B,
+0x05, 0xD4, 0x06, 0xD0, 0x04, 0xD3, 0x20, 0xF2,
+0x19, 0x6F, 0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x01, 0x6A, 0xD4, 0x17,
+0xFF, 0x6C, 0x8C, 0xEA, 0x13, 0x2A, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xF3, 0xF0,
+0x70, 0x9B, 0x05, 0xD2, 0x02, 0x6A, 0x04, 0xD2,
+0x06, 0xD0, 0x20, 0xF2, 0x1A, 0x6F, 0x02, 0x6E,
+0x01, 0xF0, 0x00, 0x6C, 0x40, 0xEB, 0x00, 0x6D,
+0x02, 0x6A, 0xBE, 0x17, 0x00, 0x6F, 0x00, 0x6E,
+0xB0, 0x67, 0x00, 0x6C, 0xB1, 0x18, 0xA1, 0xEC,
+0x0A, 0xD3, 0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC9, 0xF1, 0x54, 0x9A,
+0x09, 0xD2, 0x09, 0x93, 0x08, 0x92, 0x6A, 0xEA,
+0x0A, 0x93, 0x02, 0x61, 0x11, 0x6A, 0xA8, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x08, 0x94, 0x40, 0xEA,
+0x0A, 0xD3, 0x19, 0x21, 0x01, 0x6C, 0xA2, 0x67,
+0x8C, 0xED, 0x0A, 0x93, 0x54, 0x25, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x00, 0x6B, 0x05, 0xD3, 0x02, 0x6B,
+0x06, 0xD0, 0x04, 0xD3, 0x20, 0xF2, 0x1B, 0x6F,
+0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x03, 0x6A, 0x85, 0x17, 0x01, 0x6B,
+0x4C, 0xEB, 0x13, 0x2B, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x02, 0x6B, 0x06, 0xD0, 0x05, 0xD1, 0x04, 0xD3,
+0x20, 0xF2, 0x1C, 0x6F, 0x02, 0x6E, 0x01, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x04, 0x6A,
+0x6F, 0x17, 0x02, 0x69, 0x2B, 0xE9, 0x08, 0x6F,
+0x00, 0x6E, 0xB0, 0x67, 0x00, 0x6C, 0xB1, 0x18,
+0xA1, 0xEC, 0x4C, 0xE9, 0x09, 0x93, 0x6A, 0xEA,
+0xB9, 0x60, 0x4F, 0xEA, 0x08, 0x6F, 0x01, 0x6E,
+0xB0, 0x67, 0x00, 0x6C, 0xB1, 0x18, 0xA1, 0xEC,
+0x4C, 0xE9, 0x09, 0x93, 0x4E, 0xEB, 0xAE, 0x23,
+0x4F, 0xEA, 0x2C, 0xEA, 0x22, 0x67, 0xB0, 0x67,
+0xB1, 0x18, 0x0A, 0xED, 0x00, 0x6C, 0x51, 0x67,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x68, 0x9B, 0x08, 0x94, 0x40, 0xEB,
+0xA2, 0x67, 0x00, 0x6A, 0x45, 0x17, 0x8D, 0xEA,
+0x34, 0x34, 0x20, 0xA3, 0x1F, 0x6D, 0xAC, 0xE9,
+0x8D, 0xE9, 0x20, 0xC3, 0xED, 0x17, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0x15, 0xD7, 0x16, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x01, 0x6B, 0x05, 0xD4, 0x04, 0xD3, 0x20, 0xF2,
+0x1D, 0x6F, 0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x09, 0x6A, 0x11, 0x97,
+0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xA8, 0x32, 0x6C, 0xF2, 0x84, 0x9C, 0xA9, 0xE2,
+0x4C, 0x32, 0x49, 0xE4, 0x08, 0xD2, 0x15, 0x92,
+0x25, 0x67, 0x66, 0x67, 0x33, 0x22, 0x08, 0x92,
+0x80, 0xA2, 0x40, 0x6A, 0x4B, 0xEA, 0x8C, 0xEA,
+0x13, 0x2A, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xF3, 0xF0, 0x70, 0x9B, 0x05, 0xD2,
+0x02, 0x6A, 0x06, 0xD5, 0x04, 0xD2, 0x20, 0xF2,
+0x1E, 0x6F, 0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C,
+0x40, 0xEB, 0x00, 0x6D, 0x0D, 0x6A, 0xD3, 0x17,
+0x08, 0x92, 0x01, 0x6C, 0xC9, 0xE2, 0x44, 0xA2,
+0x8E, 0xEA, 0x2C, 0x2A, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xF3, 0xF0, 0x10, 0x9C,
+0x05, 0xD2, 0x03, 0x6A, 0x07, 0xD6, 0x06, 0xD5,
+0x04, 0xD2, 0x20, 0xF2, 0x1F, 0x6F, 0x02, 0x6E,
+0x01, 0xF0, 0x00, 0x6C, 0x40, 0xE8, 0x00, 0x6D,
+0x05, 0x6A, 0xB9, 0x17, 0x08, 0x92, 0xC9, 0xE2,
+0x44, 0xA2, 0x14, 0x2A, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xF3, 0xF0, 0x10, 0x9C,
+0x05, 0xD2, 0x03, 0x6A, 0x07, 0xD6, 0x06, 0xD5,
+0x04, 0xD2, 0x40, 0xF2, 0x00, 0x6F, 0x02, 0x6E,
+0x01, 0xF0, 0x00, 0x6C, 0x40, 0xE8, 0x00, 0x6D,
+0x06, 0x6A, 0xA1, 0x17, 0x00, 0x6C, 0x00, 0x6F,
+0x00, 0x6E, 0xB1, 0x67, 0xB1, 0x18, 0xA1, 0xEC,
+0x03, 0x67, 0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC9, 0xF1, 0x74, 0x9A,
+0x0B, 0xD2, 0x0A, 0xD3, 0x0A, 0x94, 0x09, 0x93,
+0x8A, 0xEB, 0x02, 0x61, 0x11, 0x6A, 0x8B, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x09, 0x94, 0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x0C, 0xD0, 0x0C, 0x93, 0x00, 0x6C, 0x08, 0x6F,
+0xC3, 0x67, 0xB1, 0x67, 0xB1, 0x18, 0xA1, 0xEC,
+0x02, 0x67, 0x0A, 0x94, 0x4E, 0xEC, 0xEA, 0x24,
+0x15, 0x95, 0x90, 0x67, 0x4C, 0xEC, 0x0C, 0x93,
+0x16, 0x25, 0x2B, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x07, 0xD3, 0x00, 0x6B, 0x05, 0xD3, 0x03, 0x6B,
+0x06, 0xD1, 0x04, 0xD3, 0x40, 0xF2, 0x01, 0x6F,
+0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x07, 0x6A, 0x5C, 0x17, 0x3C, 0x2C,
+0xF7, 0xF0, 0x01, 0x6A, 0x07, 0xD3, 0x40, 0x32,
+0x15, 0x93, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x05, 0xD3, 0x03, 0x6B, 0x06, 0xD1, 0x04, 0xD3,
+0x40, 0xF2, 0x02, 0x6F, 0x02, 0x6E, 0x01, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x08, 0x6A,
+0x46, 0x17, 0x4D, 0xE8, 0x08, 0x92, 0x01, 0x6C,
+0x69, 0xE2, 0x84, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A,
+0x09, 0x94, 0xB0, 0x67, 0x40, 0xEA, 0x08, 0xD3,
+0x08, 0x93, 0x09, 0x6F, 0xB1, 0x67, 0xC3, 0x67,
+0xB1, 0x18, 0xA1, 0xEC, 0x00, 0x6C, 0x0B, 0x93,
+0xC9, 0xF1, 0x74, 0x9B, 0x4E, 0xEB, 0x9E, 0x23,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x68, 0x9B, 0x09, 0x94, 0xB0, 0x67,
+0x40, 0xEB, 0x4D, 0xED, 0x00, 0x6A, 0x1F, 0x17,
+0x4F, 0xEA, 0x4C, 0xE8, 0x08, 0x92, 0x06, 0x04,
+0x20, 0xF0, 0x9C, 0xA4, 0x69, 0xE2, 0xD5, 0x17,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x04, 0x67, 0x40, 0x32,
+0xA4, 0x34, 0xD1, 0xE4, 0x40, 0x32, 0x6C, 0xF2,
+0x44, 0x9A, 0x88, 0x33, 0x8D, 0xE3, 0x6C, 0x33,
+0x69, 0xE2, 0x11, 0xD5, 0x12, 0xD6, 0xE9, 0xE2,
+0x44, 0xA2, 0x27, 0x67, 0x01, 0x72, 0x18, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0x03, 0x6B, 0x07, 0xD7,
+0x06, 0xD6, 0x05, 0xD5, 0x04, 0xD3, 0x40, 0xF2,
+0x03, 0x6F, 0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x0C, 0x6A, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0x10, 0x28, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x04, 0xD0,
+0x40, 0xF2, 0x04, 0x6F, 0x02, 0x6E, 0x01, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x0E, 0x6A,
+0xEA, 0x17, 0x12, 0x95, 0x11, 0x94, 0xD1, 0x67,
+0xB1, 0x18, 0xA1, 0xEC, 0x04, 0x6F, 0xC2, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC9, 0xF1, 0x54, 0x9A, 0x4A, 0xEE, 0x08, 0xD2,
+0x02, 0x61, 0x11, 0x6A, 0xD8, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4,
+0x54, 0x9B, 0x86, 0x67, 0x0A, 0xD3, 0x40, 0xEA,
+0x09, 0xD6, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x09, 0x96, 0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A,
+0xA1, 0x98, 0x40, 0xEA, 0x86, 0x67, 0x12, 0x95,
+0x11, 0x94, 0xD1, 0x67, 0xB1, 0x18, 0xA1, 0xEC,
+0x05, 0x6F, 0xC2, 0x67, 0x08, 0x92, 0x0A, 0x93,
+0x4A, 0xEE, 0xDF, 0x60, 0xD2, 0xF4, 0x54, 0x9B,
+0x86, 0x67, 0x40, 0xEA, 0x09, 0xD6, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x09, 0x96, 0x40, 0x32,
+0xD2, 0xF4, 0x48, 0x9A, 0xA2, 0x98, 0x40, 0xEA,
+0x86, 0x67, 0x12, 0x95, 0x11, 0x94, 0xD1, 0x67,
+0xB1, 0x18, 0xA1, 0xEC, 0x06, 0x6F, 0xC2, 0x67,
+0x08, 0x92, 0x0A, 0x93, 0x4A, 0xEE, 0xC5, 0x60,
+0xD2, 0xF4, 0x54, 0x9B, 0x86, 0x67, 0x40, 0xEA,
+0x09, 0xD6, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x09, 0x96, 0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A,
+0xA3, 0x98, 0x40, 0xEA, 0x86, 0x67, 0x12, 0x95,
+0x11, 0x94, 0xD1, 0x67, 0xB1, 0x18, 0xA1, 0xEC,
+0x07, 0x6F, 0x22, 0x67, 0x08, 0x92, 0x0A, 0x93,
+0x2E, 0xEA, 0xAB, 0x22, 0xD2, 0xF4, 0x54, 0x9B,
+0x40, 0xEA, 0x91, 0x67, 0xFF, 0x6D, 0x01, 0x4D,
+0xF7, 0xF0, 0x01, 0x6B, 0xAB, 0xED, 0x60, 0x33,
+0xAC, 0xEA, 0x60, 0x33, 0xA1, 0xA0, 0xD2, 0xF4,
+0x68, 0x9B, 0x91, 0x67, 0x40, 0xEB, 0x4D, 0xED,
+0x00, 0x6A, 0x71, 0x17, 0xF6, 0x63, 0x13, 0x62,
+0x12, 0xD1, 0x11, 0xD0, 0x04, 0x67, 0x14, 0x2C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0x04, 0xD4, 0x40, 0xF2,
+0x04, 0x6F, 0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x0E, 0x6A, 0x13, 0x97,
+0x12, 0x91, 0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63,
+0x45, 0xA4, 0x01, 0x69, 0xFF, 0x6B, 0x2C, 0xEA,
+0x6C, 0xEA, 0x12, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x70, 0x9A,
+0x01, 0x6A, 0x05, 0xD2, 0x04, 0xD2, 0x20, 0xF2,
+0x1D, 0x6F, 0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C,
+0x40, 0xEB, 0x00, 0x6D, 0x09, 0x6A, 0xE3, 0x17,
+0x45, 0xA4, 0x07, 0x6C, 0x46, 0x32, 0x8C, 0xEA,
+0x6C, 0xEA, 0x05, 0x52, 0x12, 0x61, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xF3, 0xF0,
+0x70, 0x9B, 0x05, 0xD2, 0x01, 0x6A, 0x04, 0xD2,
+0x40, 0xF2, 0x05, 0x6F, 0x02, 0x6E, 0x01, 0xF0,
+0x00, 0x6C, 0x40, 0xEB, 0x00, 0x6D, 0x0A, 0x6A,
+0xCA, 0x17, 0x00, 0x6C, 0xB1, 0x18, 0x4D, 0xED,
+0x06, 0xD3, 0x06, 0x93, 0x0F, 0x6C, 0x8C, 0xEA,
+0x6C, 0xEA, 0x08, 0xD2, 0x50, 0x35, 0x45, 0xA0,
+0x8C, 0xEA, 0x08, 0x94, 0xAD, 0xEA, 0x45, 0xC0,
+0x02, 0x5C, 0x11, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x00, 0x6B, 0x04, 0xD3, 0x40, 0xF2, 0x06, 0x6F,
+0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x0B, 0x6A, 0xA8, 0x17, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x6C, 0xF2,
+0x84, 0x9C, 0x2C, 0xEA, 0x6C, 0xEA, 0x07, 0xD2,
+0x08, 0x92, 0x06, 0xD4, 0x07, 0x94, 0x00, 0x6E,
+0x00, 0x6F, 0xA2, 0x67, 0x0B, 0xD3, 0xB1, 0x18,
+0xA1, 0xEC, 0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xC2, 0x67, 0xC9, 0xF1,
+0x54, 0x9C, 0x0A, 0xD4, 0x0B, 0x93, 0xCE, 0xEA,
+0x02, 0x2A, 0x11, 0x6A, 0x88, 0x17, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0xD2, 0xF4,
+0x54, 0x9D, 0x86, 0x67, 0x0D, 0xD3, 0x0C, 0xD6,
+0x40, 0xEA, 0x0B, 0xD5, 0x01, 0x6C, 0x4C, 0xEC,
+0x0C, 0x96, 0x0D, 0x93, 0x11, 0x24, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x00, 0x6B, 0x04, 0xD3, 0x40, 0xF2,
+0x07, 0x6F, 0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x03, 0x6A, 0x67, 0x17,
+0x07, 0x94, 0x84, 0x35, 0x08, 0x94, 0x95, 0xE5,
+0xA8, 0x34, 0xB1, 0xE4, 0x06, 0x95, 0x8C, 0x34,
+0x91, 0xE5, 0xA5, 0xA0, 0x06, 0xD4, 0x2C, 0xED,
+0x20, 0xA4, 0x04, 0x6C, 0x8B, 0xEC, 0x8C, 0xE9,
+0x85, 0xA0, 0xAD, 0xE9, 0x07, 0x6D, 0x86, 0x34,
+0xAC, 0xEC, 0x8C, 0xEB, 0xDC, 0x4D, 0x07, 0x6C,
+0xAC, 0xE9, 0x6C, 0xEC, 0x06, 0x95, 0x88, 0x34,
+0x2D, 0xEC, 0x80, 0xC5, 0x88, 0xA0, 0x02, 0x6D,
+0x8C, 0xED, 0x20, 0xF1, 0x12, 0x25, 0x08, 0x6D,
+0xAD, 0xEA, 0x01, 0x6D, 0x8C, 0xED, 0x20, 0xF1,
+0x10, 0x25, 0x10, 0x6D, 0xAD, 0xEA, 0x04, 0x6D,
+0x8C, 0xED, 0x20, 0xF1, 0x0E, 0x25, 0x20, 0x6D,
+0xAD, 0xEA, 0x9E, 0x35, 0x20, 0xF1, 0x0D, 0x25,
+0x40, 0x6D, 0xAD, 0xEA, 0x08, 0x6D, 0x8C, 0xED,
+0x20, 0xF1, 0x0B, 0x25, 0x80, 0x6D, 0xAD, 0xEA,
+0x00, 0xF7, 0x01, 0x6D, 0xAB, 0xED, 0x60, 0x33,
+0xAC, 0xEA, 0x6D, 0xEA, 0x10, 0x6B, 0x8C, 0xEB,
+0x20, 0xF1, 0x03, 0x23, 0x02, 0xF0, 0x00, 0x6B,
+0x6D, 0xEA, 0x64, 0xA0, 0x03, 0x73, 0x20, 0xF1,
+0x01, 0x61, 0x04, 0xF0, 0x00, 0x6D, 0xAD, 0xEA,
+0x20, 0x6B, 0x8C, 0xEB, 0x00, 0xF1, 0x1F, 0x23,
+0x08, 0xF0, 0x00, 0x6D, 0xAD, 0xEA, 0x40, 0x6B,
+0x6C, 0xEC, 0x00, 0xF1, 0x1D, 0x24, 0x10, 0xF0,
+0x00, 0x6D, 0xAD, 0xEA, 0xF7, 0xF0, 0x01, 0x6C,
+0x65, 0xA8, 0x80, 0x34, 0x80, 0x34, 0xE9, 0xF1,
+0xA0, 0x9C, 0xF7, 0xF0, 0x01, 0x6C, 0x60, 0x33,
+0x80, 0x34, 0x80, 0x34, 0xF7, 0xF0, 0x01, 0x69,
+0x60, 0x33, 0xAC, 0xEB, 0x20, 0x31, 0xE9, 0xF1,
+0xA4, 0x9C, 0x20, 0x31, 0xD2, 0xF4, 0xE8, 0x99,
+0x4C, 0xED, 0x6D, 0xED, 0x40, 0xEF, 0x86, 0x67,
+0x09, 0x95, 0x07, 0x94, 0x00, 0x6E, 0xB1, 0x18,
+0xA1, 0xEC, 0x01, 0x6F, 0xC2, 0x67, 0x0A, 0x92,
+0xC9, 0xF1, 0x54, 0x9A, 0xCE, 0xEA, 0x5F, 0xF7,
+0x18, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A, 0x86, 0x67,
+0x40, 0xEA, 0x0C, 0xD6, 0xFF, 0x6B, 0x01, 0x4B,
+0x6B, 0xEB, 0x4C, 0xEB, 0x47, 0xA0, 0xFF, 0xF7,
+0x1F, 0x6D, 0xAC, 0xEB, 0x6D, 0xEA, 0x69, 0xA0,
+0x01, 0x6C, 0xAC, 0xEA, 0x6C, 0xEC, 0x0C, 0x96,
+0xC0, 0xF0, 0x1F, 0x24, 0xFF, 0x6C, 0x01, 0x4C,
+0x8D, 0xEA, 0x02, 0x6C, 0x8C, 0xEB, 0xAC, 0xEA,
+0xFF, 0xF7, 0x1F, 0x6D, 0xC0, 0xF0, 0x1A, 0x23,
+0x01, 0xF0, 0x00, 0x6B, 0x6D, 0xEA, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4,
+0x64, 0x9B, 0x4C, 0xED, 0x40, 0xEB, 0x86, 0x67,
+0x09, 0x95, 0x07, 0x94, 0x00, 0x6E, 0xB1, 0x18,
+0xA1, 0xEC, 0x02, 0x6F, 0xC2, 0x67, 0x0A, 0x92,
+0xC9, 0xF1, 0x74, 0x9A, 0x6A, 0xEE, 0x1F, 0xF7,
+0x1C, 0x60, 0x0B, 0x92, 0x86, 0x67, 0x0E, 0xD3,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x0D, 0xD6,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xE9, 0xF1, 0x68, 0x9C, 0x0D, 0x96, 0xD2, 0xF4,
+0xE8, 0x99, 0xA3, 0x67, 0x4C, 0xED, 0x46, 0xA8,
+0x86, 0x67, 0x0A, 0xD3, 0xAD, 0xEA, 0xA8, 0xA8,
+0x0C, 0xD2, 0x0C, 0x02, 0x40, 0xAA, 0xA0, 0x35,
+0xA0, 0x35, 0x40, 0xEF, 0x4D, 0xED, 0x09, 0x95,
+0x07, 0x94, 0x00, 0x6E, 0xB1, 0x18, 0xA1, 0xEC,
+0x03, 0x6F, 0x0E, 0x93, 0xC2, 0x67, 0x4E, 0xEB,
+0xFF, 0xF6, 0x13, 0x23, 0x0B, 0x92, 0x86, 0x67,
+0x0B, 0xD6, 0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xA7, 0xA8, 0xFF, 0xF7, 0x1F, 0x6B,
+0x6C, 0xEA, 0xA0, 0x35, 0xA0, 0x35, 0x4D, 0xED,
+0x0A, 0x92, 0x69, 0xA8, 0x0B, 0x96, 0x4C, 0xED,
+0xD2, 0xF4, 0x48, 0x99, 0x6D, 0xED, 0x40, 0xEA,
+0x86, 0x67, 0x00, 0x69, 0x40, 0x98, 0x30, 0x34,
+0x01, 0x6B, 0x91, 0xE2, 0x40, 0xA4, 0xFF, 0x6F,
+0x6C, 0xEA, 0xEC, 0xEA, 0x7C, 0x2A, 0x01, 0x49,
+0x02, 0x71, 0xF4, 0x61, 0x06, 0x92, 0x65, 0x98,
+0x08, 0x91, 0x3F, 0x6E, 0x62, 0xDA, 0x66, 0x98,
+0x63, 0xDA, 0x67, 0x98, 0x64, 0xDA, 0x68, 0x98,
+0x65, 0xDA, 0x69, 0x98, 0x66, 0xDA, 0x6A, 0x98,
+0x67, 0xDA, 0x6C, 0x98, 0x69, 0xDA, 0x01, 0x6A,
+0x4E, 0xE9, 0x02, 0x29, 0x1F, 0xF4, 0x00, 0x6E,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF4, 0x40, 0x9A, 0x07, 0x94, 0x40, 0xEA,
+0x0A, 0x6D, 0x46, 0xA0, 0x60, 0x21, 0x01, 0x6B,
+0x4A, 0x36, 0x6C, 0xEE, 0x4C, 0xEB, 0x02, 0x23,
+0x02, 0x6B, 0x6D, 0xEE, 0x08, 0x6B, 0x4C, 0xEB,
+0x02, 0x23, 0x04, 0x6B, 0x6D, 0xEE, 0x02, 0x6B,
+0x4C, 0xEB, 0x02, 0x23, 0x08, 0x6B, 0x6D, 0xEE,
+0x10, 0x6B, 0x4C, 0xEB, 0x02, 0x23, 0x10, 0x6B,
+0x6D, 0xEE, 0x20, 0x6B, 0x6C, 0xEA, 0x02, 0x22,
+0x20, 0x6A, 0x4D, 0xEE, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF4, 0x44, 0x9A,
+0x07, 0x94, 0x40, 0xEA, 0x0A, 0x6D, 0x00, 0x6A,
+0x12, 0x16, 0x09, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0xCC, 0x16, 0x11, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0xCE, 0x16, 0x21, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0xD0, 0x16, 0x41, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0xD1, 0x16, 0x81, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0xD3, 0x16, 0x02, 0xF0, 0x01, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0xDB, 0x16, 0x04, 0xF0, 0x01, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0xDD, 0x16, 0x08, 0xF0,
+0x01, 0x6D, 0xAB, 0xED, 0xAC, 0xEA, 0xDF, 0x16,
+0x10, 0xF0, 0x01, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0xE1, 0x16, 0xFF, 0x6C, 0x02, 0x4C, 0x8B, 0xEC,
+0x8C, 0xEA, 0x1F, 0x17, 0x01, 0xF0, 0x01, 0x6B,
+0x6B, 0xEB, 0x6C, 0xEA, 0x24, 0x17, 0x09, 0x96,
+0x07, 0x95, 0xB1, 0x18, 0xAE, 0xEE, 0x2C, 0xEF,
+0x7F, 0xF7, 0x1D, 0x22, 0xD8, 0x15, 0x04, 0x6B,
+0x4C, 0xEB, 0x00, 0x6E, 0x02, 0x23, 0x00, 0xF4,
+0x00, 0x6E, 0x01, 0x6B, 0x4C, 0xEB, 0x03, 0x23,
+0x01, 0xF0, 0x00, 0x6B, 0x6D, 0xEE, 0x08, 0x6B,
+0x4C, 0xEB, 0x03, 0x23, 0x02, 0xF0, 0x00, 0x6B,
+0x6D, 0xEE, 0x02, 0x6B, 0x4C, 0xEB, 0x03, 0x23,
+0x04, 0xF0, 0x00, 0x6B, 0x6D, 0xEE, 0x10, 0x6B,
+0x4C, 0xEB, 0x03, 0x23, 0x08, 0xF0, 0x00, 0x6B,
+0x6D, 0xEE, 0x20, 0x6B, 0x6C, 0xEA, 0x9A, 0x22,
+0x10, 0xF0, 0x00, 0x6A, 0x96, 0x17, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x84, 0x33, 0xAD, 0xE3, 0x40, 0x32, 0x68, 0x36,
+0x40, 0x32, 0x6D, 0xE6, 0x6C, 0xF2, 0xC4, 0x9A,
+0x6C, 0x33, 0x79, 0xE6, 0x65, 0x9E, 0x1E, 0x23,
+0x45, 0x67, 0x03, 0x6D, 0xAC, 0xEC, 0xBD, 0x67,
+0xB0, 0xA5, 0x04, 0x6F, 0xEB, 0xEF, 0xEC, 0xED,
+0x8D, 0xED, 0x07, 0x6C, 0x8C, 0xEA, 0x1D, 0x6C,
+0x8B, 0xEC, 0x48, 0x32, 0x8C, 0xED, 0x4D, 0xED,
+0x5D, 0x67, 0xB0, 0xC2, 0x48, 0xAA, 0x3F, 0xF6,
+0x1C, 0x4C, 0x8C, 0xEA, 0x60, 0x6C, 0x8D, 0xEA,
+0x9D, 0x67, 0x48, 0xCC, 0x49, 0x9E, 0x04, 0x04,
+0x40, 0xEB, 0x05, 0xD2, 0x07, 0x97, 0x00, 0x6A,
+0x00, 0xEF, 0x04, 0x63, 0xFC, 0x63, 0x07, 0x62,
+0x84, 0x33, 0xAD, 0xE3, 0x68, 0x32, 0x69, 0xE2,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x6C, 0xF2, 0x64, 0x9B, 0x4C, 0x32, 0x49, 0xE3,
+0x00, 0x6B, 0x66, 0xC2, 0x64, 0x9A, 0x1F, 0x23,
+0xC5, 0x67, 0x03, 0x6D, 0xAC, 0xEC, 0xBD, 0x67,
+0xB0, 0xA5, 0x04, 0x6F, 0xEB, 0xEF, 0xEC, 0xED,
+0x8D, 0xED, 0x07, 0x6C, 0xCC, 0xEC, 0x1D, 0x6E,
+0xCB, 0xEE, 0x88, 0x34, 0xCC, 0xED, 0x8D, 0xED,
+0x9D, 0x67, 0xB0, 0xC4, 0x88, 0xAC, 0xE0, 0xF1,
+0x01, 0x6D, 0xAB, 0xED, 0xAC, 0xEC, 0x40, 0x6D,
+0xAD, 0xEC, 0xBD, 0x67, 0x88, 0xCD, 0x49, 0x9A,
+0x04, 0x04, 0x40, 0xEB, 0x05, 0xD2, 0x07, 0x97,
+0x00, 0x6A, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x84, 0x33, 0xAD, 0xE3, 0x40, 0x32, 0x68, 0x36,
+0x40, 0x32, 0x6D, 0xE6, 0x6C, 0xF2, 0xC4, 0x9A,
+0x6C, 0x33, 0x79, 0xE6, 0x63, 0x9E, 0x1E, 0x23,
+0x45, 0x67, 0x03, 0x6D, 0xAC, 0xEC, 0xBD, 0x67,
+0xB0, 0xA5, 0x04, 0x6F, 0xEB, 0xEF, 0xEC, 0xED,
+0x8D, 0xED, 0x07, 0x6C, 0x8C, 0xEA, 0x1D, 0x6C,
+0x8B, 0xEC, 0x48, 0x32, 0x8C, 0xED, 0x4D, 0xED,
+0x5D, 0x67, 0xB0, 0xC2, 0x48, 0xAA, 0x3F, 0xF6,
+0x1C, 0x4C, 0x8C, 0xEA, 0x20, 0x6C, 0x8D, 0xEA,
+0x9D, 0x67, 0x48, 0xCC, 0x49, 0x9E, 0x04, 0x04,
+0x40, 0xEB, 0x05, 0xD2, 0x07, 0x97, 0x00, 0x6A,
+0x00, 0xEF, 0x04, 0x63, 0xFC, 0x63, 0x07, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x84, 0x33, 0xAD, 0xE3,
+0x40, 0x32, 0x68, 0x36, 0x40, 0x32, 0x6D, 0xE6,
+0x6C, 0xF2, 0xC4, 0x9A, 0x6C, 0x33, 0x01, 0x6A,
+0x79, 0xE6, 0x62, 0x9E, 0x46, 0xC6, 0x1C, 0x23,
+0x45, 0x67, 0x03, 0x6D, 0xAC, 0xEC, 0xBD, 0x67,
+0xB0, 0xA5, 0x04, 0x6F, 0xEB, 0xEF, 0xEC, 0xED,
+0x8D, 0xED, 0x07, 0x6C, 0x8C, 0xEA, 0x1D, 0x6C,
+0x8B, 0xEC, 0x48, 0x32, 0x8C, 0xED, 0x4D, 0xED,
+0x5D, 0x67, 0xB0, 0xC2, 0x48, 0xAA, 0x3F, 0xF6,
+0x1C, 0x4C, 0x8C, 0xEA, 0x9D, 0x67, 0x48, 0xCC,
+0x49, 0x9E, 0x04, 0x04, 0x40, 0xEB, 0x05, 0xD2,
+0x07, 0x97, 0x00, 0x6A, 0x00, 0xEF, 0x04, 0x63,
+0xFC, 0x63, 0x07, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x84, 0x33, 0xAD, 0xE3, 0x40, 0x32, 0x68, 0x36,
+0x40, 0x32, 0x6D, 0xE6, 0x6C, 0xF2, 0xC4, 0x9A,
+0x6C, 0x33, 0x79, 0xE6, 0x66, 0x9E, 0x1E, 0x23,
+0x45, 0x67, 0x03, 0x6D, 0xAC, 0xEC, 0xBD, 0x67,
+0xB0, 0xA5, 0x04, 0x6F, 0xEB, 0xEF, 0xEC, 0xED,
+0x8D, 0xED, 0x07, 0x6C, 0x8C, 0xEA, 0x1D, 0x6C,
+0x8B, 0xEC, 0x48, 0x32, 0x8C, 0xED, 0x4D, 0xED,
+0x5D, 0x67, 0xB0, 0xC2, 0x48, 0xAA, 0x3F, 0xF6,
+0x1C, 0x4C, 0x8C, 0xEA, 0x80, 0x6C, 0x8D, 0xEA,
+0x9D, 0x67, 0x48, 0xCC, 0x49, 0x9E, 0x04, 0x04,
+0x40, 0xEB, 0x05, 0xD2, 0x07, 0x97, 0x00, 0x6A,
+0x00, 0xEF, 0x04, 0x63, 0xFC, 0x63, 0x07, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x84, 0x33, 0xAD, 0xE3,
+0x40, 0x32, 0x68, 0x36, 0x40, 0x32, 0x6D, 0xE6,
+0x6C, 0xF2, 0xC4, 0x9A, 0x6C, 0x33, 0x79, 0xE6,
+0x67, 0x9E, 0x1E, 0x23, 0x45, 0x67, 0x03, 0x6D,
+0xAC, 0xEC, 0xBD, 0x67, 0xB0, 0xA5, 0x04, 0x6F,
+0xEB, 0xEF, 0xEC, 0xED, 0x8D, 0xED, 0x07, 0x6C,
+0x8C, 0xEA, 0x1D, 0x6C, 0x8B, 0xEC, 0x48, 0x32,
+0x8C, 0xED, 0x4D, 0xED, 0x5D, 0x67, 0xB0, 0xC2,
+0x48, 0xAA, 0x3F, 0xF6, 0x1C, 0x4C, 0x8C, 0xEA,
+0xA0, 0x6C, 0x8D, 0xEA, 0x9D, 0x67, 0x48, 0xCC,
+0x49, 0x9E, 0x04, 0x04, 0x40, 0xEB, 0x05, 0xD2,
+0x07, 0x97, 0x00, 0x6A, 0x00, 0xEF, 0x04, 0x63,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF1, 0x40, 0x9A, 0x04, 0x67, 0x40, 0xEA,
+0x25, 0x67, 0xFF, 0x72, 0x63, 0x60, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x15, 0xF0,
+0x08, 0x4C, 0xA5, 0xA4, 0x64, 0xA4, 0xC6, 0xA4,
+0xA0, 0x35, 0x6D, 0xED, 0x67, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0x20, 0xF0, 0xA1, 0xA3, 0x20, 0xF0,
+0xC0, 0xA3, 0x20, 0xF0, 0x82, 0xA3, 0xA0, 0x35,
+0xAD, 0xEE, 0x20, 0xF0, 0xA3, 0xA3, 0x80, 0x34,
+0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6, 0xA0, 0x35,
+0xAD, 0xEC, 0x48, 0x33, 0x8D, 0xE3, 0x85, 0xA3,
+0xA6, 0xA3, 0xC4, 0xA3, 0x67, 0xA3, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0x63, 0xA3,
+0x01, 0x73, 0x28, 0x61, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x92, 0xF6, 0x74, 0x9B,
+0x82, 0x67, 0x40, 0xEB, 0x04, 0xD2, 0x04, 0x92,
+0xD1, 0x18, 0x35, 0x61, 0x82, 0x67, 0x01, 0x6B,
+0x6E, 0xEA, 0x18, 0x2A, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x6C, 0xF2, 0x48, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x6C, 0xF2, 0x70, 0x9A, 0x60, 0x6C, 0xA1, 0x43,
+0x6C, 0xF2, 0xB0, 0xDA, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x13, 0xF1, 0x4C, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xB1, 0x67, 0xD1, 0x18,
+0x76, 0x32, 0x90, 0x67, 0xB1, 0x67, 0xB1, 0x18,
+0xBC, 0xD1, 0x90, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF1, 0x40, 0x9A, 0x40, 0xEA, 0x04, 0x67,
+0xFF, 0x72, 0x62, 0x60, 0x22, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x15, 0xF0,
+0x08, 0x4A, 0xA5, 0xA2, 0x64, 0xA2, 0x86, 0xA2,
+0xA0, 0x35, 0x6D, 0xED, 0x67, 0xA2, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0x20, 0xF0, 0x81, 0xA3, 0x20, 0xF0,
+0xA0, 0xA3, 0x20, 0xF0, 0x42, 0xA3, 0x80, 0x34,
+0x8D, 0xED, 0x20, 0xF0, 0x83, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x00, 0xF6, 0x80, 0x34,
+0x8D, 0xEA, 0x28, 0x33, 0x4D, 0xE3, 0x85, 0xA3,
+0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x91, 0x67,
+0xD1, 0x18, 0x3E, 0x32, 0x04, 0xD2, 0x04, 0x92,
+0x43, 0xA2, 0x01, 0x72, 0x26, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x92, 0xF6,
+0x50, 0x9A, 0x40, 0xEA, 0x91, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6C, 0xF2,
+0x48, 0xA2, 0x17, 0x2A, 0xD1, 0x18, 0x35, 0x61,
+0x91, 0x67, 0x01, 0x72, 0x12, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6C, 0xF2,
+0x6C, 0x9A, 0x88, 0x6C, 0xA1, 0x43, 0x6C, 0xF2,
+0xAC, 0xDA, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x13, 0xF1, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xB1, 0x18, 0xEB, 0xD1, 0x90, 0x67,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF1, 0x40, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xFF, 0x72, 0x54, 0x60, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x02, 0x67, 0x60, 0x33,
+0xF7, 0xF0, 0x01, 0x6A, 0x15, 0xF0, 0x08, 0x4B,
+0x40, 0x32, 0x85, 0xA3, 0x40, 0x32, 0xD2, 0xF6,
+0xA0, 0x9A, 0x44, 0xA3, 0xC6, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x20, 0xF0, 0x81, 0xA2, 0x20, 0xF0, 0xC0, 0xA2,
+0x20, 0xF0, 0x62, 0xA2, 0x80, 0x34, 0x8D, 0xEE,
+0x20, 0xF0, 0x83, 0xA2, 0x60, 0x33, 0x60, 0x33,
+0xCD, 0xEB, 0x00, 0xF6, 0x80, 0x34, 0x8D, 0xEB,
+0x08, 0x32, 0x69, 0xE2, 0x65, 0xA2, 0x86, 0xA2,
+0xC4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xEE, 0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0xED, 0x80, 0xA2,
+0x01, 0x72, 0x19, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x32, 0xF6, 0x58, 0x9A,
+0x40, 0xEA, 0x90, 0x67, 0x10, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF6,
+0x5C, 0x9A, 0x40, 0xEA, 0x90, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF6,
+0x58, 0x9A, 0x40, 0xEA, 0x90, 0x67, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF1, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xFF, 0x72, 0x5C, 0x60, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x85, 0xA3, 0x22, 0x67, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x81, 0xA2, 0x20, 0xF0,
+0xA0, 0xA2, 0x20, 0xF0, 0x62, 0xA2, 0x80, 0x34,
+0x8D, 0xED, 0x20, 0xF0, 0x83, 0xA2, 0x60, 0x33,
+0x60, 0x33, 0xAD, 0xEB, 0x00, 0xF6, 0x80, 0x34,
+0x8D, 0xEB, 0x28, 0x32, 0x69, 0xE2, 0x65, 0xA2,
+0x04, 0xA2, 0x86, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x07, 0xA2, 0x80, 0x34, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x6A, 0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30,
+0x40, 0x32, 0x6D, 0xE8, 0x40, 0x32, 0xD2, 0xF6,
+0x40, 0x9A, 0x40, 0xEA, 0x80, 0xA0, 0x01, 0x72,
+0x21, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x32, 0xF6, 0x58, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0x18, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF6, 0x5C, 0x9A,
+0x40, 0xEA, 0x91, 0x67, 0x56, 0xA0, 0x03, 0x6B,
+0x6B, 0xEB, 0x4C, 0xEB, 0x76, 0xC0, 0x05, 0x6B,
+0x6C, 0xEA, 0x08, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF6, 0x5C, 0x9A,
+0x40, 0xEA, 0x91, 0x67, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF1, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xFF, 0x72, 0x5A, 0x60, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x85, 0xA3, 0x22, 0x67, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x81, 0xA2, 0x20, 0xF0,
+0xA0, 0xA2, 0x20, 0xF0, 0x62, 0xA2, 0x80, 0x34,
+0x8D, 0xED, 0x20, 0xF0, 0x83, 0xA2, 0x60, 0x33,
+0x60, 0x33, 0xAD, 0xEB, 0x00, 0xF6, 0x80, 0x34,
+0x8D, 0xEB, 0x28, 0x32, 0x69, 0xE2, 0x65, 0xA2,
+0x04, 0xA2, 0x86, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x07, 0xA2, 0x80, 0x34, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x6A, 0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30,
+0x40, 0x32, 0x6D, 0xE8, 0x40, 0x32, 0xA0, 0xA0,
+0x32, 0xF6, 0x58, 0x9A, 0x91, 0x67, 0x40, 0xEA,
+0x05, 0xD5, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x05, 0x95, 0x40, 0x32, 0xD2, 0xF6,
+0x40, 0x9A, 0x40, 0xEA, 0x85, 0x67, 0x01, 0x72,
+0x04, 0x93, 0x0F, 0x61, 0x0E, 0x23, 0x02, 0x73,
+0x04, 0x60, 0x6A, 0xA0, 0xEF, 0x4A, 0x6C, 0xEA,
+0x10, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF5, 0x48, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0xB1, 0x18, 0xBB, 0xD1, 0x00, 0x65,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x52, 0xF6, 0x4C, 0x9A, 0xEF, 0x17,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF1, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xFF, 0x72, 0x5E, 0x60, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x85, 0xA3, 0x22, 0x67, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x81, 0xA2, 0x20, 0xF0,
+0xA0, 0xA2, 0x20, 0xF0, 0x62, 0xA2, 0x80, 0x34,
+0x8D, 0xED, 0x20, 0xF0, 0x83, 0xA2, 0x60, 0x33,
+0x60, 0x33, 0xAD, 0xEB, 0x00, 0xF6, 0x80, 0x34,
+0x8D, 0xEB, 0x28, 0x32, 0x69, 0xE2, 0x65, 0xA2,
+0x04, 0xA2, 0x86, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x07, 0xA2, 0x80, 0x34, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x6A, 0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30,
+0x40, 0x32, 0x6D, 0xE8, 0x40, 0x32, 0xD2, 0xF6,
+0x40, 0x9A, 0x40, 0xEA, 0x80, 0xA0, 0x01, 0x72,
+0x20, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x32, 0xF6, 0x58, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0x17, 0x22, 0x50, 0xA0, 0x10, 0x6B,
+0x4C, 0xEB, 0x03, 0x23, 0x20, 0x6B, 0x6D, 0xEA,
+0x50, 0xC0, 0x56, 0xA0, 0x02, 0x6B, 0x6B, 0xEB,
+0x4C, 0xEB, 0x76, 0xC0, 0x06, 0x6B, 0x6C, 0xEA,
+0x08, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF6, 0x5C, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0xB1, 0x18, 0xBB, 0xD1, 0x00, 0x65,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x53, 0xF1, 0x40, 0x9A,
+0x04, 0x67, 0x40, 0xEA, 0x25, 0x67, 0xFF, 0x72,
+0x05, 0xD2, 0x64, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xB2, 0xF5, 0x58, 0x9A,
+0xB1, 0x67, 0x90, 0x67, 0x40, 0xEA, 0x00, 0x6E,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x04, 0xD2, 0x15, 0xF0, 0x08, 0x4B, 0x85, 0xA3,
+0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x81, 0xA2, 0x20, 0xF0, 0x60, 0xA2, 0x20, 0xF0,
+0xA2, 0xA2, 0x80, 0x34, 0x8D, 0xEB, 0x20, 0xF0,
+0x83, 0xA2, 0xA0, 0x35, 0x05, 0x97, 0xA0, 0x35,
+0x6D, 0xED, 0x00, 0xF6, 0x80, 0x34, 0x8D, 0xED,
+0xE8, 0x33, 0xAD, 0xE3, 0x85, 0xA3, 0x44, 0xA3,
+0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x71, 0xA2, 0x01, 0x6C,
+0x8E, 0xEB, 0x0D, 0x2B, 0x71, 0xC2, 0x7D, 0x67,
+0x70, 0xA3, 0x87, 0x67, 0x73, 0xC2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF6,
+0x5C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF3,
+0x50, 0x9A, 0x00, 0x6D, 0x40, 0xEA, 0x90, 0x67,
+0x04, 0x95, 0xC2, 0x67, 0xB1, 0x18, 0x49, 0xD1,
+0x90, 0x67, 0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65,
+0x05, 0x22, 0x04, 0x96, 0xB1, 0x67, 0xD1, 0x18,
+0x22, 0x3D, 0x90, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF1, 0x40, 0x9A, 0x04, 0x67, 0x40, 0xEA,
+0x25, 0x67, 0xFF, 0x72, 0x05, 0xD2, 0x64, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xB2, 0xF5, 0x58, 0x9A, 0xB1, 0x67, 0x90, 0x67,
+0x40, 0xEA, 0x01, 0x6E, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x04, 0xD2, 0x15, 0xF0,
+0x08, 0x4B, 0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x81, 0xA2, 0x20, 0xF0,
+0x60, 0xA2, 0x20, 0xF0, 0xA2, 0xA2, 0x80, 0x34,
+0x8D, 0xEB, 0x20, 0xF0, 0x83, 0xA2, 0xA0, 0x35,
+0x05, 0x97, 0xA0, 0x35, 0x6D, 0xED, 0x00, 0xF6,
+0x80, 0x34, 0x8D, 0xED, 0xE8, 0x33, 0xAD, 0xE3,
+0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x72, 0xA2, 0x01, 0x6C, 0x8E, 0xEB, 0x0D, 0x2B,
+0x72, 0xC2, 0x7D, 0x67, 0x70, 0xA3, 0x87, 0x67,
+0x74, 0xC2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF6, 0x5C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF3, 0x50, 0x9A, 0x00, 0x6D,
+0x40, 0xEA, 0x90, 0x67, 0x04, 0x95, 0xC2, 0x67,
+0xB1, 0x18, 0x59, 0xD1, 0x90, 0x67, 0xD1, 0x18,
+0xCD, 0x8A, 0x00, 0x65, 0x05, 0x22, 0x04, 0x96,
+0xB1, 0x67, 0xD1, 0x18, 0xDC, 0x3C, 0x90, 0x67,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF1, 0x40, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xFF, 0x72, 0x5A, 0x60, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x15, 0xF0,
+0x08, 0x4C, 0x02, 0x67, 0x45, 0xA4, 0x64, 0xA4,
+0xA6, 0xA4, 0x40, 0x32, 0x6D, 0xEA, 0x67, 0xA4,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x4D, 0xEB, 0x20, 0xF0, 0x81, 0xA3,
+0x20, 0xF0, 0xA0, 0xA3, 0x20, 0xF0, 0x42, 0xA3,
+0x80, 0x34, 0x8D, 0xED, 0x20, 0xF0, 0x83, 0xA3,
+0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA, 0x00, 0xF6,
+0x80, 0x34, 0x8D, 0xEA, 0x08, 0x33, 0x4D, 0xE3,
+0x45, 0xA3, 0x86, 0xA3, 0xA4, 0xA3, 0x67, 0xA3,
+0x40, 0x32, 0x80, 0x34, 0x4D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x43, 0xA3, 0x01, 0x72, 0x26, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x92, 0xF6,
+0x4C, 0x9A, 0x40, 0xEA, 0x90, 0x67, 0xD1, 0x18,
+0x35, 0x61, 0x90, 0x67, 0x01, 0x72, 0x19, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x01, 0x6B,
+0x40, 0x32, 0x6C, 0xF2, 0x68, 0xC2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6C, 0xF2,
+0x6C, 0x9A, 0x88, 0x6C, 0xA1, 0x43, 0x6C, 0xF2,
+0xAC, 0xDA, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x13, 0xF1, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0x0F, 0x76, 0x04, 0x67,
+0x25, 0x67, 0x2D, 0x60, 0x10, 0x5E, 0x0A, 0x60,
+0x0D, 0x76, 0x17, 0x60, 0x0E, 0x76, 0x1E, 0x60,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0x6A,
+0x00, 0xEF, 0x04, 0x63, 0x11, 0x76, 0x31, 0x60,
+0x11, 0x5E, 0x26, 0x61, 0x12, 0x76, 0xF4, 0x61,
+0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65, 0xF0, 0x22,
+0xB1, 0x67, 0xB1, 0x18, 0xBB, 0xF0, 0x90, 0x67,
+0xEB, 0x17, 0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65,
+0xE7, 0x22, 0xB1, 0x67, 0xB1, 0x18, 0x70, 0xF0,
+0x90, 0x67, 0xE2, 0x17, 0xD1, 0x18, 0xCD, 0x8A,
+0x00, 0x65, 0xDE, 0x22, 0xB1, 0x67, 0xB1, 0x18,
+0x3C, 0xF0, 0x90, 0x67, 0xD9, 0x17, 0xD1, 0x18,
+0xCD, 0x8A, 0x00, 0x65, 0xD5, 0x22, 0xB1, 0x67,
+0xB1, 0x18, 0x89, 0xF0, 0x90, 0x67, 0xD0, 0x17,
+0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65, 0xCC, 0x22,
+0xB1, 0x67, 0xB1, 0x18, 0x55, 0xF0, 0x90, 0x67,
+0xC7, 0x17, 0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65,
+0xC3, 0x22, 0xB1, 0x67, 0xB1, 0x18, 0xA2, 0xF0,
+0x90, 0x67, 0xBE, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x24, 0x67, 0x13, 0xF4, 0x04, 0x6C, 0x40, 0xEA,
+0x05, 0x67, 0x0D, 0x59, 0xA0, 0xF0, 0x07, 0x60,
+0x28, 0x33, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0xC8, 0xF2, 0x00, 0x49, 0x25, 0xE3,
+0x60, 0x99, 0x00, 0xEB, 0x00, 0x65, 0x04, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0x03, 0x6D, 0x0C, 0xED,
+0x4D, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A, 0x13, 0xF4,
+0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x01, 0x6A,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0x0D, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0x0C, 0x6B, 0x08, 0x35, 0x6C, 0xED, 0xE8, 0x17,
+0x31, 0x6D, 0xAB, 0xED, 0xAC, 0xEA, 0x30, 0x6B,
+0x10, 0x35, 0xF8, 0x17, 0xC1, 0x6D, 0xAB, 0xED,
+0xAC, 0xEA, 0xFF, 0x6B, 0x18, 0x35, 0xF2, 0x17,
+0x00, 0xF3, 0x01, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0x00, 0xF3, 0x00, 0x6B, 0x00, 0x35, 0xEA, 0x17,
+0x01, 0xF4, 0x01, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0x00, 0x35, 0xA8, 0x35, 0x01, 0xF4, 0x00, 0x6B,
+0xE1, 0x17, 0x06, 0xF0, 0x01, 0x6D, 0xAB, 0xED,
+0xAC, 0xEA, 0x00, 0x35, 0xB0, 0x35, 0x06, 0xF0,
+0x00, 0x6B, 0xD8, 0x17, 0x18, 0xF0, 0x01, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0x00, 0x35, 0xB8, 0x35,
+0xFF, 0xF7, 0x1F, 0x6B, 0xCF, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x89, 0xF1,
+0xA4, 0x9B, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0xAC, 0xEA, 0x60, 0x33, 0x00, 0x35, 0xA0, 0x35,
+0xE9, 0xF1, 0x6C, 0x9B, 0xBF, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xE9, 0xF1,
+0xB0, 0x9B, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xAC, 0xEA, 0xE9, 0xF1, 0x74, 0x9B,
+0x80, 0xF4, 0x00, 0x35, 0xAF, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xE9, 0xF1,
+0xB8, 0x9B, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xAC, 0xEA, 0xE9, 0xF1, 0x7C, 0x9B,
+0x00, 0xF5, 0x00, 0x35, 0x9F, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x09, 0xF2,
+0xA0, 0x9B, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xAC, 0xEA, 0x09, 0xF2, 0x64, 0x9B,
+0x80, 0xF5, 0x00, 0x35, 0x8F, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x09, 0xF2,
+0xA8, 0x9B, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xAC, 0xEA, 0x09, 0xF2, 0x6C, 0x9B,
+0x00, 0xF6, 0x00, 0x35, 0x7F, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0xA9, 0xF1, 0xA4, 0x9B, 0x40, 0xEA,
+0x01, 0x6C, 0x00, 0x6A, 0x65, 0x17, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x04, 0x67, 0x13, 0xF4,
+0x08, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x22, 0x67,
+0x40, 0xA0, 0x01, 0x72, 0x46, 0x60, 0x15, 0x22,
+0x02, 0x72, 0x64, 0x60, 0x03, 0x72, 0x80, 0xF0,
+0x02, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0xA9, 0xF1,
+0xA4, 0x9B, 0x40, 0xEA, 0x01, 0x6C, 0x00, 0x6A,
+0x2B, 0x10, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A, 0x13, 0xF4,
+0x0C, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x64, 0x9A, 0x43, 0xA0, 0xA2, 0xA0, 0x13, 0xF4,
+0x0C, 0x6C, 0x40, 0x32, 0x4D, 0xED, 0xFF, 0xF7,
+0x1F, 0x6A, 0x40, 0xEB, 0x4C, 0xED, 0x04, 0x6D,
+0xAB, 0xED, 0xAC, 0xE9, 0xA1, 0xA0, 0x03, 0x6A,
+0x4C, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A, 0x13, 0xF4,
+0x08, 0x6C, 0x40, 0xEA, 0x2D, 0xED, 0x01, 0x6A,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A, 0x13, 0xF4,
+0x0E, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x64, 0x9A, 0x43, 0xA0, 0xA2, 0xA0, 0x13, 0xF4,
+0x0E, 0x6C, 0x40, 0x32, 0x4D, 0xED, 0xFF, 0xF7,
+0x1F, 0x6A, 0x40, 0xEB, 0x4C, 0xED, 0x0D, 0x6D,
+0xAB, 0xED, 0xAC, 0xE9, 0xA1, 0xA0, 0x0C, 0x6A,
+0xA8, 0x35, 0xCE, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A,
+0x13, 0xF4, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x64, 0x9A, 0x43, 0xA0, 0xA2, 0xA0,
+0x13, 0xF4, 0x10, 0x6C, 0x40, 0x32, 0x4D, 0xED,
+0xFF, 0xF7, 0x1F, 0x6A, 0x40, 0xEB, 0x4C, 0xED,
+0x31, 0x6D, 0xAB, 0xED, 0xAC, 0xE9, 0xA1, 0xA0,
+0x30, 0x6A, 0xB0, 0x35, 0xAD, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x50, 0x9A, 0x13, 0xF4, 0x12, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x64, 0x9A, 0x43, 0xA0,
+0xA2, 0xA0, 0x13, 0xF4, 0x12, 0x6C, 0x40, 0x32,
+0x4D, 0xED, 0xFF, 0xF7, 0x1F, 0x6A, 0x40, 0xEB,
+0x4C, 0xED, 0xC1, 0x6D, 0xAB, 0xED, 0xAC, 0xE9,
+0xA1, 0xA0, 0xFF, 0x6A, 0xB8, 0x35, 0x8C, 0x17,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x04, 0x67, 0x13, 0xF4,
+0x14, 0x6C, 0x40, 0xEA, 0x25, 0x67, 0x03, 0x70,
+0x41, 0x60, 0x04, 0x58, 0x0A, 0x60, 0x01, 0x70,
+0x36, 0x60, 0x02, 0x58, 0x1E, 0x61, 0x31, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0x30, 0x6B, 0x30, 0x35,
+0x33, 0x10, 0x05, 0x70, 0x41, 0x60, 0x05, 0x58,
+0x37, 0x61, 0x06, 0x70, 0x46, 0x60, 0x07, 0x70,
+0x4D, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0xA9, 0xF1,
+0xA4, 0x9B, 0x40, 0xEA, 0x01, 0x6C, 0x00, 0x6A,
+0x11, 0x10, 0x04, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0x03, 0x6D, 0x2C, 0xED, 0x4D, 0xED, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x48, 0x9A, 0x13, 0xF4, 0x14, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x01, 0x6A, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x0D, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0x0C, 0x6B, 0x28, 0x35,
+0x6C, 0xED, 0xE8, 0x17, 0xC1, 0x6D, 0xAB, 0xED,
+0xAC, 0xEA, 0xFF, 0x6B, 0x38, 0x35, 0xF8, 0x17,
+0x00, 0xF3, 0x01, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0x00, 0xF3, 0x00, 0x6B, 0x20, 0x35, 0xF0, 0x17,
+0x01, 0xF4, 0x01, 0x6D, 0xAB, 0xED, 0xAC, 0xEA,
+0x20, 0x35, 0xA8, 0x35, 0x01, 0xF4, 0x00, 0x6B,
+0xE7, 0x17, 0x06, 0xF0, 0x01, 0x6D, 0xAB, 0xED,
+0xAC, 0xEA, 0x20, 0x35, 0xB0, 0x35, 0x06, 0xF0,
+0x00, 0x6B, 0xDE, 0x17, 0x18, 0xF0, 0x01, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0x20, 0x35, 0xB8, 0x35,
+0xFF, 0xF7, 0x1F, 0x6B, 0xD5, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x14, 0x2C, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0xA9, 0xF1,
+0xA4, 0x9B, 0x40, 0xEA, 0x01, 0x6C, 0x00, 0x6A,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x50, 0xA4, 0x04, 0x67, 0x10, 0x5A,
+0x0F, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x00, 0x6B,
+0xFF, 0x6F, 0x04, 0xD3, 0x15, 0x4F, 0x02, 0x6E,
+0x01, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0xD9, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x13, 0xF4, 0x1C, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x11, 0x6D, 0x60, 0x33, 0xAB, 0xED,
+0xD2, 0xF4, 0x68, 0x9B, 0xAC, 0xEA, 0x8F, 0x6D,
+0x13, 0xF4, 0x1C, 0x6C, 0x40, 0xEB, 0x4D, 0xED,
+0x50, 0xA0, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x48, 0x31, 0x45, 0xE1, 0x28, 0x32, 0xF7, 0xF0,
+0x01, 0x69, 0x06, 0xD2, 0x20, 0x31, 0x80, 0x34,
+0x20, 0x31, 0x06, 0x93, 0x09, 0xF2, 0x90, 0x9C,
+0xD2, 0xF4, 0x58, 0x99, 0xA0, 0x98, 0x40, 0xEA,
+0x91, 0xE3, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x06, 0x93, 0x09, 0xF2, 0x94, 0x9C,
+0xD2, 0xF4, 0x58, 0x99, 0xA1, 0x98, 0x40, 0xEA,
+0x91, 0xE3, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x06, 0x93, 0x09, 0xF2, 0x98, 0x9C,
+0xD2, 0xF4, 0x58, 0x99, 0xA2, 0x98, 0x40, 0xEA,
+0x91, 0xE3, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x06, 0x93, 0x09, 0xF2, 0x9C, 0x9C,
+0xD2, 0xF4, 0x58, 0x99, 0xA3, 0x98, 0x40, 0xEA,
+0x91, 0xE3, 0x51, 0xA0, 0xB3, 0xA0, 0x06, 0x6F,
+0x4B, 0xEA, 0xC0, 0xF7, 0x42, 0x32, 0x1A, 0x65,
+0x52, 0xA0, 0xAC, 0x35, 0x71, 0x67, 0x44, 0x32,
+0x4C, 0xEF, 0x78, 0x6A, 0x4C, 0xED, 0x54, 0xA0,
+0x80, 0x6E, 0x01, 0x2A, 0x00, 0x6E, 0x4B, 0xA8,
+0x98, 0x67, 0xD2, 0xF4, 0x78, 0x9B, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0xED, 0xEA, 0xAD, 0xEA,
+0xB5, 0xA0, 0x00, 0xF7, 0x00, 0x6C, 0xA0, 0x35,
+0x8C, 0xED, 0x4D, 0xED, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x29, 0xF2, 0x80, 0x9A,
+0x06, 0x92, 0xCD, 0xED, 0x40, 0xEB, 0x91, 0xE2,
+0x01, 0x6A, 0x6A, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0x5C, 0xA4, 0x01, 0x72, 0x0E, 0x60, 0x06, 0x22,
+0x02, 0x72, 0x0F, 0x60, 0x03, 0x72, 0x12, 0x60,
+0x00, 0x6A, 0x04, 0x10, 0xBE, 0xA4, 0xB1, 0x18,
+0x01, 0xF3, 0x9D, 0xA4, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0xB1, 0x18, 0x66, 0xF3, 0x18, 0x4C,
+0xF9, 0x17, 0xBE, 0xA4, 0xB1, 0x18, 0xC4, 0xF3,
+0x9D, 0xA4, 0xF4, 0x17, 0xB1, 0x18, 0x02, 0xF4,
+0x00, 0x65, 0xF0, 0x17, 0xF4, 0x63, 0x17, 0x62,
+0x16, 0xD1, 0x15, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0xF7, 0xF0, 0x01, 0x6B, 0x00, 0x30, 0x00, 0x30,
+0x60, 0x33, 0x60, 0x33, 0xF3, 0xF0, 0x30, 0x98,
+0xB1, 0x1C, 0x70, 0xF4, 0x00, 0x65, 0x00, 0x65,
+0x00, 0x70, 0x02, 0x40, 0x73, 0xF4, 0x25, 0x76,
+0x00, 0x00, 0x00, 0x00, 0x94, 0xF1, 0x40, 0xDB,
+0x05, 0xD2, 0x01, 0x6A, 0xA0, 0xF1, 0x0F, 0x6F,
+0x00, 0x6D, 0x18, 0xD4, 0x04, 0xD2, 0x02, 0x6E,
+0x02, 0xF0, 0x00, 0x6C, 0x40, 0xE9, 0x0A, 0xD3,
+0xF7, 0xF0, 0x01, 0x69, 0xF7, 0xF0, 0x01, 0x6E,
+0x20, 0x31, 0xC0, 0x36, 0xC0, 0x36, 0x20, 0x31,
+0x29, 0xF2, 0x84, 0x9E, 0xF2, 0xF4, 0x44, 0x99,
+0x40, 0xEA, 0x0B, 0xD6, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x08, 0xD2, 0x29, 0xF2,
+0x88, 0x9C, 0xF2, 0xF4, 0x44, 0x99, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x09, 0xD2, 0x29, 0xF2, 0x8C, 0x9C,
+0xF2, 0xF4, 0x44, 0x99, 0x40, 0xEA, 0x00, 0x65,
+0x08, 0x97, 0x0A, 0x93, 0xFF, 0x6C, 0x0B, 0x96,
+0x8C, 0xEF, 0x01, 0x77, 0x0C, 0xD3, 0x08, 0xD0,
+0x0A, 0xD1, 0x0D, 0xD6, 0x13, 0x60, 0x04, 0x27,
+0x68, 0x47, 0xE7, 0x4B, 0x02, 0x5B, 0x72, 0x61,
+0x08, 0x92, 0x01, 0x6B, 0x05, 0xD7, 0xF3, 0xF0,
+0x50, 0x9A, 0x04, 0xD3, 0xA0, 0xF1, 0x11, 0x6F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x27, 0x10, 0xFF, 0xF7, 0x1F, 0x6C,
+0x4C, 0xEC, 0xE0, 0xF1, 0x00, 0x6B, 0x8C, 0xEB,
+0x80, 0xF1, 0x00, 0x73, 0x02, 0x68, 0x2B, 0x60,
+0x80, 0xF1, 0x01, 0x5B, 0x1C, 0x60, 0x00, 0xF1,
+0x00, 0x73, 0x00, 0x68, 0x24, 0x60, 0x60, 0xF1,
+0x00, 0x73, 0x01, 0x68, 0x20, 0x60, 0xFF, 0x6B,
+0x01, 0x4B, 0x6C, 0xEC, 0x06, 0x68, 0x1B, 0x24,
+0x08, 0x93, 0x05, 0xD2, 0x01, 0x6A, 0xF3, 0xF0,
+0x70, 0x9B, 0x04, 0xD2, 0xA0, 0xF1, 0x10, 0x6F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEB,
+0x00, 0x6D, 0x07, 0x68, 0x0C, 0x10, 0xC0, 0xF1,
+0x00, 0x73, 0x04, 0x68, 0x08, 0x60, 0xE0, 0xF1,
+0x00, 0x73, 0x05, 0x68, 0x04, 0x60, 0xA0, 0xF1,
+0x00, 0x73, 0xE1, 0x61, 0x03, 0x68, 0x00, 0x6E,
+0xF7, 0xF0, 0x01, 0x6B, 0x0A, 0x92, 0x60, 0x33,
+0x60, 0x33, 0x29, 0xF2, 0x70, 0x9B, 0xF2, 0xF4,
+0x44, 0x9A, 0x0E, 0xD6, 0x83, 0x67, 0x40, 0xEA,
+0x0B, 0xD3, 0x09, 0x93, 0xFF, 0x6C, 0xF7, 0xF0,
+0x01, 0x69, 0x8C, 0xEB, 0x01, 0x73, 0x20, 0x31,
+0x83, 0x67, 0x20, 0x31, 0x0B, 0x93, 0x0E, 0x96,
+0x52, 0x60, 0x44, 0x24, 0x02, 0x74, 0xB8, 0x67,
+0x6F, 0x25, 0x03, 0x74, 0x20, 0xF2, 0x17, 0x60,
+0x08, 0x92, 0x01, 0x6B, 0x05, 0xD4, 0xF3, 0xF0,
+0x50, 0x9A, 0x04, 0xD3, 0xA0, 0xF1, 0x14, 0x6F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x3B, 0x10, 0xFF, 0xF7, 0x1F, 0x6D,
+0x82, 0x67, 0xAC, 0xEC, 0xF0, 0x6B, 0x8C, 0xEB,
+0xAC, 0xEB, 0xC0, 0x73, 0x02, 0x68, 0x24, 0x60,
+0xC1, 0x5B, 0x19, 0x60, 0x80, 0x73, 0x00, 0x68,
+0x1F, 0x60, 0xB0, 0x73, 0x01, 0x68, 0x1C, 0x60,
+0x80, 0x6B, 0x8C, 0xEB, 0x06, 0x68, 0x18, 0x23,
+0x08, 0x93, 0x05, 0xD2, 0x01, 0x6A, 0xF3, 0xF0,
+0x70, 0x9B, 0x04, 0xD2, 0xA0, 0xF1, 0x10, 0x6F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEB,
+0x00, 0x6D, 0x07, 0x68, 0x09, 0x10, 0xE0, 0x73,
+0x04, 0x68, 0x06, 0x60, 0xF0, 0x73, 0x05, 0x68,
+0x03, 0x60, 0xD0, 0x73, 0xE5, 0x61, 0x03, 0x68,
+0x01, 0x6E, 0xA2, 0x17, 0x08, 0x92, 0x04, 0xD4,
+0xA0, 0xF1, 0x12, 0x6F, 0xF3, 0xF0, 0x50, 0x9A,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x00, 0x6E, 0x36, 0x10, 0x01, 0x76,
+0x00, 0xF2, 0x05, 0x61, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x29, 0xF2, 0x94, 0x9C,
+0xD2, 0xF4, 0xF8, 0x99, 0x0E, 0xD6, 0x8C, 0xEA,
+0xA2, 0x67, 0x83, 0x67, 0x0B, 0xD2, 0x40, 0xEF,
+0x09, 0xD3, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xD2, 0xF4, 0xF8, 0x99, 0x69, 0xF1,
+0xAC, 0x9C, 0x0B, 0x92, 0x09, 0x93, 0x4D, 0xED,
+0x40, 0xEF, 0x83, 0x67, 0x0E, 0x96, 0x15, 0x10,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x6C, 0xF2, 0x54, 0x9B, 0x04, 0xD5, 0x02, 0x6E,
+0x01, 0x4A, 0x6C, 0xF2, 0x54, 0xDB, 0x08, 0x92,
+0xA0, 0xF1, 0x13, 0x6F, 0x02, 0xF0, 0x00, 0x6C,
+0xF3, 0xF0, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x6D,
+0x02, 0x6E, 0x0A, 0x92, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x49, 0xF2, 0x80, 0x9B,
+0xF2, 0xF4, 0x44, 0x9A, 0x40, 0xEA, 0x09, 0xD6,
+0x09, 0x96, 0x62, 0x67, 0x00, 0x6A, 0x02, 0x76,
+0x80, 0xF0, 0x07, 0x61, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x49, 0xF2, 0x84, 0x9C,
+0x63, 0xEC, 0x7F, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x49, 0xF2, 0x48, 0x9A,
+0x63, 0xEA, 0x0A, 0x6A, 0x76, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x49, 0xF2,
+0x4C, 0x9A, 0x63, 0xEA, 0x01, 0x6A, 0x6D, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x49, 0xF2, 0x50, 0x9A, 0x63, 0xEA, 0x02, 0x6A,
+0x64, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x49, 0xF2, 0x54, 0x9A, 0x63, 0xEA,
+0x0A, 0x6A, 0x5B, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x49, 0xF2, 0x58, 0x9A,
+0x63, 0xEA, 0x03, 0x6A, 0x52, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x49, 0xF2,
+0x5C, 0x9A, 0x63, 0xEA, 0x04, 0x6A, 0x49, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x69, 0xF2, 0x40, 0x9A, 0x63, 0xEA, 0x05, 0x6A,
+0x40, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x69, 0xF2, 0x44, 0x9A, 0x63, 0xEA,
+0x0A, 0x6A, 0x37, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x69, 0xF2, 0x48, 0x9A,
+0x63, 0xEA, 0x06, 0x6A, 0x2E, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x69, 0xF2,
+0x4C, 0x9A, 0x63, 0xEA, 0x0A, 0x6A, 0x25, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x69, 0xF2, 0x50, 0x9A, 0x63, 0xEA, 0x07, 0x6A,
+0x1C, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x69, 0xF2, 0x54, 0x9A, 0x63, 0xEA,
+0x0A, 0x6A, 0x13, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x69, 0xF2, 0x58, 0x9A,
+0x63, 0xEA, 0x08, 0x6A, 0x0A, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x69, 0xF2,
+0x5C, 0x9A, 0x63, 0xEA, 0x09, 0x6A, 0x01, 0x60,
+0x0A, 0x6A, 0xC0, 0x36, 0x10, 0x30, 0xCD, 0xE8,
+0x4D, 0xE8, 0x04, 0xF0, 0x00, 0x6A, 0x0D, 0xEA,
+0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30,
+0x0B, 0xD2, 0x00, 0x30, 0x13, 0xF1, 0x4C, 0x98,
+0x0B, 0x95, 0x30, 0x6C, 0x40, 0xEA, 0x0E, 0xD3,
+0x0E, 0x93, 0x13, 0xF1, 0x4C, 0x98, 0x34, 0x6C,
+0x40, 0xEA, 0xA3, 0x67, 0x0C, 0x93, 0x13, 0xF1,
+0x4C, 0x98, 0x94, 0xF1, 0xA0, 0x9B, 0x40, 0xEA,
+0x38, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x06, 0xF3, 0x44, 0xA2, 0x0E, 0x93,
+0x14, 0x2A, 0xF7, 0xF0, 0x01, 0x6C, 0x0A, 0x92,
+0x80, 0x34, 0x80, 0x34, 0x89, 0xF2, 0x00, 0x9C,
+0xD2, 0xF4, 0xD8, 0x99, 0xF2, 0xF4, 0x44, 0x9A,
+0x90, 0x67, 0x0F, 0xD3, 0x40, 0xEA, 0x0E, 0xD6,
+0x0E, 0x96, 0xA1, 0x42, 0x40, 0xEE, 0x90, 0x67,
+0x0F, 0x93, 0x08, 0x92, 0x06, 0xD3, 0x03, 0x67,
+0x0B, 0x93, 0xF3, 0xF0, 0x50, 0x9A, 0x02, 0x6C,
+0x05, 0xD3, 0xA0, 0xF1, 0x15, 0x6F, 0x02, 0x6E,
+0x04, 0xD4, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xD2, 0xF4, 0x58, 0x99, 0x89, 0xF2,
+0x84, 0x9C, 0x40, 0xEA, 0x01, 0x6D, 0x0D, 0x93,
+0x0A, 0x92, 0x29, 0xF2, 0x84, 0x9B, 0xF2, 0xF4,
+0x44, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xFF, 0x6C,
+0x8C, 0xEA, 0x70, 0x67, 0x0D, 0x22, 0x08, 0x92,
+0x00, 0x6C, 0x04, 0xD4, 0xF3, 0xF0, 0x50, 0x9A,
+0xA0, 0xF1, 0x16, 0x6F, 0x02, 0x6E, 0x02, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x70, 0x67,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x75, 0xF3, 0x44, 0x98, 0xC0, 0xF0, 0x05, 0x2A,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0xD2, 0xF4, 0x48, 0x99, 0xFF, 0x6C, 0x00, 0x6D,
+0x6D, 0x4C, 0x40, 0xEA, 0x0A, 0xD3, 0x0A, 0x93,
+0xD2, 0xF4, 0x48, 0x99, 0xE0, 0xF1, 0x14, 0x6C,
+0x40, 0xEA, 0xA3, 0x67, 0x0C, 0x93, 0xD2, 0xF4,
+0x48, 0x99, 0xE0, 0xF1, 0x18, 0x6C, 0x94, 0xF1,
+0xA0, 0x9B, 0x40, 0xEA, 0x00, 0x65, 0x09, 0x93,
+0xFF, 0xF7, 0x10, 0x6A, 0x6C, 0xEA, 0x04, 0xF0,
+0x00, 0x6B, 0x6D, 0xEA, 0x24, 0xF0, 0x00, 0x72,
+0x71, 0x67, 0x15, 0x69, 0x09, 0x60, 0x04, 0xF0,
+0x10, 0x72, 0x14, 0x69, 0x05, 0x60, 0x04, 0xF2,
+0x10, 0x72, 0x16, 0x69, 0x01, 0x60, 0x17, 0x69,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x68, 0x9B, 0xD2, 0xF4, 0x54, 0x9A,
+0xE0, 0xF1, 0x10, 0x6C, 0x40, 0xEA, 0x0A, 0xD3,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x89, 0xF1, 0xB8, 0x9C, 0x0A, 0x93, 0x20, 0x31,
+0x4C, 0xED, 0xE0, 0xF1, 0x10, 0x6C, 0x40, 0xEB,
+0x2D, 0xED, 0x75, 0xF3, 0x44, 0x98, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x01, 0x4A, 0x75, 0xF3,
+0x44, 0xD8, 0x0C, 0x92, 0x20, 0x31, 0xF4, 0xF0,
+0x14, 0x49, 0x94, 0xF1, 0x40, 0x9A, 0x08, 0x93,
+0xA0, 0xF1, 0x17, 0x6F, 0x80, 0xF0, 0x48, 0xD9,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x2B, 0xF2, 0x9C, 0x9A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2B, 0xF2, 0x58, 0x9A,
+0xF3, 0xF0, 0x70, 0x9B, 0x9A, 0xD9, 0x5C, 0xD9,
+0x05, 0xD2, 0x02, 0x6A, 0x06, 0xD4, 0x04, 0xD2,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEB,
+0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF6, 0x44, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0x09, 0x93, 0xFF, 0xF7, 0x10, 0x6A,
+0x6C, 0xEA, 0x04, 0xF0, 0x00, 0x6B, 0x6D, 0xEA,
+0x04, 0xF0, 0x10, 0x72, 0x80, 0xF0, 0x0A, 0x60,
+0x0B, 0x92, 0x04, 0xF2, 0x19, 0x72, 0xA0, 0xF0,
+0x08, 0x61, 0x75, 0xF3, 0x44, 0x98, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0xFF, 0x4A, 0x75, 0xF3,
+0x44, 0xD8, 0x60, 0x33, 0xF7, 0xF0, 0x01, 0x6A,
+0x89, 0xF2, 0x88, 0x9B, 0x40, 0x32, 0x09, 0x93,
+0x40, 0x32, 0x13, 0xF0, 0x48, 0x9A, 0x8D, 0xEB,
+0x40, 0xEA, 0x83, 0x67, 0x17, 0x97, 0x16, 0x91,
+0x15, 0x90, 0x00, 0xEF, 0x0C, 0x63, 0x01, 0x76,
+0x03, 0x6E, 0x5F, 0xF6, 0x06, 0x61, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x29, 0xF2,
+0x98, 0x9C, 0xD2, 0xF4, 0xF8, 0x99, 0x0E, 0xD6,
+0x8C, 0xEA, 0xA2, 0x67, 0x83, 0x67, 0x0B, 0xD2,
+0x40, 0xEF, 0x09, 0xD3, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xD2, 0xF4, 0xF8, 0x99,
+0x29, 0xF2, 0xBC, 0x9C, 0x12, 0x16, 0x01, 0x6E,
+0x2C, 0x16, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0x94, 0xF2, 0xA4, 0x99, 0x41, 0x5D,
+0x7F, 0xF7, 0x1B, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x92, 0xF1, 0x40, 0x9A,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x0A, 0xD2, 0xF7, 0xF0, 0x01, 0x6E, 0x94, 0xF2,
+0x48, 0x9B, 0xC0, 0x36, 0xC0, 0x36, 0x0D, 0xD2,
+0xD2, 0xF4, 0x50, 0x9E, 0xE0, 0xF1, 0x10, 0x6C,
+0x11, 0xD3, 0x12, 0xD5, 0x40, 0xEA, 0x0E, 0xD6,
+0x0E, 0x96, 0x10, 0xD2, 0xE0, 0xF1, 0x14, 0x6C,
+0xD2, 0xF4, 0x50, 0x9E, 0x40, 0xEA, 0x0F, 0xD6,
+0x0F, 0x96, 0x0E, 0xD2, 0xE0, 0xF1, 0x18, 0x6C,
+0xD2, 0xF4, 0x50, 0x9E, 0x40, 0xEA, 0x00, 0x65,
+0x05, 0xD2, 0x0E, 0x92, 0xF7, 0xF0, 0x01, 0x6E,
+0x0D, 0x94, 0x04, 0xD2, 0x10, 0x97, 0xC0, 0x36,
+0x0A, 0x92, 0xC0, 0x36, 0xEA, 0xF0, 0x18, 0x4E,
+0x40, 0xEA, 0x12, 0x95, 0x11, 0x93, 0x94, 0xF2,
+0x88, 0x9B, 0x51, 0xE4, 0x94, 0xF2, 0x88, 0xDB,
+0x94, 0xF2, 0x64, 0x99, 0x4B, 0xE3, 0x94, 0xF2,
+0x44, 0xD9, 0x37, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A,
+0xE0, 0xF1, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x02, 0x6B, 0x6C, 0xEA, 0x16, 0x22, 0xD1, 0x18,
+0xAD, 0xF4, 0x74, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x4C, 0x9A,
+0x40, 0xEA, 0xC8, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF3, 0x50, 0x9A,
+0x3E, 0xF4, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xED, 0x17, 0x18, 0x92, 0xE8, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x89, 0xF2,
+0x08, 0x9A, 0x09, 0x92, 0x0D, 0xEA, 0x02, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF0, 0x48, 0x9A, 0x40, 0xEA, 0x90, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0xFF, 0x6C, 0x40, 0xEA,
+0x6D, 0x4C, 0x0E, 0xEA, 0xCC, 0x22, 0xFF, 0x17,
+0xFD, 0x63, 0x7F, 0x67, 0x05, 0x62, 0x04, 0xD0,
+0x1F, 0x67, 0xD1, 0x18, 0xE9, 0xF7, 0x83, 0x67,
+0xB1, 0x18, 0x67, 0xF4, 0x01, 0x6C, 0xD1, 0x18,
+0x10, 0xF8, 0x90, 0x67, 0x05, 0x97, 0x04, 0x90,
+0x00, 0xEF, 0x03, 0x63, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x29, 0xF2, 0x10, 0x9B, 0x40, 0x32, 0xF2, 0xF4,
+0x44, 0x9A, 0x40, 0xEA, 0x90, 0x67, 0xF7, 0xF0,
+0x01, 0x6C, 0x02, 0xF0, 0x00, 0x6D, 0xF7, 0xF0,
+0x01, 0x6B, 0x80, 0x34, 0xAB, 0xED, 0x80, 0x34,
+0x60, 0x33, 0xAC, 0xEA, 0x60, 0x33, 0x89, 0xF2,
+0xAC, 0x9C, 0xD2, 0xF4, 0x78, 0x9B, 0x90, 0x67,
+0x40, 0xEB, 0x4D, 0xED, 0x05, 0x97, 0x04, 0x90,
+0x00, 0x6A, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0x60, 0xA4, 0x0F, 0x6A, 0xFF, 0x6C, 0x4C, 0xEB,
+0x8C, 0xEB, 0x0B, 0x53, 0x78, 0x67, 0x4F, 0x43,
+0x20, 0xE8, 0x8C, 0xEA, 0x01, 0x74, 0x0D, 0x60,
+0x06, 0x24, 0x02, 0x74, 0x0D, 0x60, 0x03, 0x74,
+0x24, 0x60, 0x01, 0x6C, 0x07, 0x10, 0xFD, 0x2D,
+0x07, 0x5E, 0xD8, 0x67, 0x01, 0x76, 0x98, 0x67,
+0x01, 0x10, 0xFA, 0x25, 0x20, 0xE8, 0x44, 0x67,
+0x01, 0x75, 0x0E, 0x60, 0x0B, 0x25, 0x02, 0x75,
+0xF0, 0x61, 0xFF, 0x6C, 0xFB, 0x4E, 0x8C, 0xEE,
+0x02, 0x5E, 0xB8, 0x67, 0x01, 0x6A, 0x4E, 0xED,
+0xAC, 0xEC, 0xF0, 0x17, 0x02, 0x5E, 0xE9, 0x17,
+0xFF, 0x6C, 0xFE, 0x4E, 0x8C, 0xEE, 0x03, 0x5E,
+0xF8, 0x67, 0x01, 0x6A, 0x4E, 0xEF, 0xEC, 0xEC,
+0xE5, 0x17, 0x01, 0x75, 0x09, 0x60, 0xF2, 0x25,
+0x02, 0x75, 0xD7, 0x61, 0x06, 0x6A, 0x4E, 0xEE,
+0xCB, 0xEE, 0xC0, 0xF7, 0xC2, 0x34, 0xDA, 0x17,
+0xFF, 0x6C, 0xFE, 0x4E, 0x8C, 0xEE, 0x04, 0x5E,
+0x78, 0x67, 0x01, 0x6A, 0x4E, 0xEB, 0x6C, 0xEC,
+0xD1, 0x17, 0x00, 0x65, 0x01, 0x74, 0x45, 0x67,
+0x0E, 0x60, 0x06, 0x24, 0x02, 0x74, 0x0E, 0x60,
+0x03, 0x74, 0x14, 0x60, 0x20, 0xE8, 0x00, 0x6A,
+0x1A, 0x25, 0xFF, 0x4A, 0x02, 0x5A, 0x00, 0x6A,
+0x16, 0x60, 0x20, 0xE8, 0xF1, 0x6A, 0xF9, 0x2D,
+0x20, 0xE8, 0x00, 0x65, 0x10, 0x25, 0x01, 0x75,
+0x0D, 0x60, 0x02, 0x75, 0x00, 0x6A, 0x0B, 0x61,
+0x20, 0xE8, 0x05, 0x6A, 0x08, 0x25, 0x01, 0x75,
+0x05, 0x60, 0x02, 0x75, 0x00, 0x6A, 0x03, 0x61,
+0x20, 0xE8, 0x06, 0x6A, 0x02, 0x6A, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0x00, 0x6A, 0x12, 0xD4,
+0x9D, 0x67, 0x00, 0x6B, 0x08, 0xD2, 0x52, 0xCC,
+0x06, 0xD2, 0x4E, 0xCC, 0x04, 0xD2, 0x4A, 0xCC,
+0x5D, 0x67, 0x76, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x89, 0xF2, 0x30, 0x9A,
+0x13, 0xD5, 0x14, 0xD6, 0x15, 0xD7, 0x20, 0xF0,
+0x66, 0xC4, 0x7E, 0xC4, 0x0B, 0xD2, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0xF2, 0xF4,
+0x44, 0x98, 0x40, 0xEA, 0x91, 0x67, 0x01, 0x6C,
+0x8C, 0xEA, 0x80, 0xF0, 0x12, 0x22, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0xF2, 0xF4,
+0x4C, 0x9E, 0x87, 0x41, 0x11, 0x4C, 0x40, 0xEA,
+0x0A, 0xD6, 0x12, 0x93, 0x6A, 0xEA, 0x80, 0xF0,
+0x04, 0x61, 0xF2, 0xF4, 0x44, 0x98, 0x87, 0x41,
+0x40, 0xEA, 0x15, 0x4C, 0x42, 0x30, 0x03, 0x6C,
+0x02, 0x30, 0x8C, 0xE8, 0x00, 0x6C, 0xA7, 0x44,
+0x7D, 0x67, 0x02, 0x4D, 0x99, 0xE3, 0xA4, 0x35,
+0x62, 0x67, 0x66, 0xED, 0xA3, 0x67, 0x03, 0x6F,
+0x01, 0x4C, 0xEC, 0xED, 0x07, 0x74, 0xB8, 0xC6,
+0xF2, 0x61, 0x0A, 0x92, 0x87, 0x41, 0x19, 0x4C,
+0xF2, 0xF4, 0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x00, 0x6C, 0x7D, 0x67, 0x01, 0x6D, 0x4C, 0xED,
+0x99, 0xE3, 0x01, 0x4C, 0x20, 0xF0, 0xA0, 0xC6,
+0x46, 0x32, 0xFF, 0x6D, 0x07, 0x74, 0xAC, 0xEA,
+0xF4, 0x61, 0xC7, 0x41, 0xA7, 0x41, 0x1A, 0x4E,
+0x21, 0x4D, 0x0A, 0x92, 0x86, 0x67, 0x0D, 0xD5,
+0xF2, 0xF4, 0x4C, 0x9A, 0x40, 0xEA, 0x0C, 0xD6,
+0x0B, 0x93, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0x89, 0xF2, 0x90, 0x9B, 0x0C, 0x96, 0xE0, 0x37,
+0x89, 0xF2, 0xF4, 0x9F, 0x33, 0xE4, 0x0D, 0x95,
+0xD1, 0xE4, 0xF1, 0xE4, 0x04, 0x03, 0x01, 0x4E,
+0x91, 0xE3, 0xCA, 0xED, 0x40, 0xC4, 0xE5, 0x61,
+0x14, 0x95, 0xB1, 0x18, 0x7B, 0xF6, 0x90, 0x67,
+0xF1, 0x72, 0x22, 0x67, 0xF1, 0x6A, 0x2C, 0x60,
+0x13, 0x92, 0xFF, 0x72, 0x19, 0x61, 0x07, 0x59,
+0x02, 0x61, 0xF0, 0x6A, 0x25, 0x10, 0x5D, 0x67,
+0x29, 0xE2, 0x20, 0xF0, 0x40, 0xA2, 0x01, 0x72,
+0x12, 0x60, 0x01, 0x49, 0xF4, 0x17, 0x5D, 0x67,
+0x29, 0xE2, 0x58, 0xA2, 0x13, 0x93, 0x6E, 0xEA,
+0x06, 0x2A, 0x5D, 0x67, 0x29, 0xE2, 0x20, 0xF0,
+0x40, 0xA2, 0x01, 0x72, 0x04, 0x60, 0x01, 0x49,
+0x07, 0x59, 0xF1, 0x61, 0xE6, 0x17, 0x14, 0x95,
+0xFF, 0x6E, 0x2C, 0xEE, 0xB1, 0x18, 0x5B, 0xF6,
+0x90, 0x67, 0xDF, 0x2A, 0x04, 0x03, 0x25, 0xE3,
+0x60, 0xA1, 0x83, 0x67, 0x15, 0x93, 0x80, 0xC3,
+0x11, 0x97, 0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF,
+0x09, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x89, 0xF2, 0x58, 0x9A, 0x40, 0x49,
+0x2E, 0xEA, 0x5F, 0xF7, 0x18, 0x2A, 0xF0, 0x17,
+0xF5, 0x63, 0x15, 0x62, 0x14, 0xD1, 0x13, 0xD0,
+0x00, 0x6A, 0x7D, 0x67, 0x20, 0xF0, 0x44, 0xC3,
+0x02, 0x76, 0x00, 0x6A, 0x19, 0xD7, 0x1A, 0x91,
+0x1B, 0x90, 0x04, 0xD2, 0x05, 0xD2, 0x06, 0xD2,
+0x07, 0xD2, 0x08, 0xD2, 0x06, 0x61, 0x4C, 0x45,
+0xFF, 0x6F, 0xEC, 0xEA, 0x02, 0x5A, 0x01, 0x60,
+0xA2, 0x67, 0xB1, 0x18, 0x8F, 0xF6, 0x09, 0x07,
+0x09, 0x2A, 0x0D, 0x21, 0x80, 0xA1, 0x0F, 0x6D,
+0xFF, 0x6E, 0x8C, 0xED, 0xCC, 0xED, 0x0B, 0x55,
+0x0A, 0x61, 0xF1, 0x6A, 0x15, 0x97, 0x14, 0x91,
+0x13, 0x90, 0x00, 0xEF, 0x0B, 0x63, 0xF0, 0x6A,
+0xF9, 0x17, 0x01, 0x6A, 0xF7, 0x17, 0x61, 0x99,
+0x30, 0x6F, 0xEC, 0xEC, 0x05, 0xD3, 0x62, 0x99,
+0xCC, 0xEC, 0xAD, 0xEC, 0x06, 0xD3, 0x63, 0x99,
+0x04, 0xD4, 0xF7, 0xF0, 0x01, 0x6C, 0x07, 0xD3,
+0x80, 0x34, 0x64, 0x99, 0x80, 0x34, 0xA9, 0xF2,
+0x20, 0x9C, 0xF7, 0xF0, 0x01, 0x6C, 0x08, 0xD3,
+0x80, 0x34, 0x7D, 0x67, 0x80, 0x34, 0x20, 0xF0,
+0x64, 0xA3, 0xA9, 0xF2, 0xA4, 0x9C, 0x0C, 0xD4,
+0x74, 0x33, 0xB5, 0xE3, 0x25, 0xE3, 0x0B, 0xD5,
+0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36,
+0x10, 0xD2, 0xF7, 0xF0, 0x01, 0x6F, 0xD2, 0xF4,
+0x58, 0x9E, 0xE0, 0x37, 0xE0, 0x37, 0x3A, 0x65,
+0x89, 0xF2, 0x5C, 0x9F, 0x73, 0xE1, 0x0F, 0xD6,
+0x0A, 0xD2, 0x51, 0xE4, 0x04, 0x02, 0x91, 0xE2,
+0xA0, 0x9C, 0x59, 0x67, 0x91, 0x67, 0x0E, 0xD3,
+0x40, 0xEA, 0x0D, 0xD7, 0x0B, 0x92, 0x04, 0x49,
+0x0F, 0x96, 0x4A, 0xE9, 0x0D, 0x97, 0x0E, 0x93,
+0x10, 0x92, 0xDE, 0x61, 0x19, 0x93, 0x01, 0x73,
+0xB1, 0x61, 0xB7, 0x20, 0x60, 0xA0, 0x0F, 0x6C,
+0xFF, 0x6D, 0x6C, 0xEC, 0xAC, 0xEC, 0x0B, 0x54,
+0xA8, 0x60, 0x30, 0x69, 0x2C, 0xEB, 0xAC, 0xEB,
+0x8D, 0xEB, 0x04, 0xD3, 0x61, 0x98, 0x05, 0xD3,
+0x62, 0x98, 0x06, 0xD3, 0x63, 0x98, 0x07, 0xD3,
+0x64, 0x98, 0x08, 0xD3, 0x7D, 0x67, 0x20, 0xF0,
+0x04, 0xA3, 0x0A, 0x93, 0x01, 0x48, 0x14, 0x30,
+0x67, 0xE0, 0x0C, 0x93, 0xA9, 0xF2, 0x64, 0x9B,
+0x6D, 0xE0, 0x0A, 0xD3, 0x89, 0xF2, 0xBC, 0x9F,
+0x13, 0xE1, 0x0D, 0xD2, 0xB1, 0xE4, 0x04, 0x02,
+0x91, 0xE2, 0xD2, 0xF4, 0x78, 0x9E, 0xA0, 0x9C,
+0x91, 0x67, 0x0C, 0xD6, 0x40, 0xEB, 0x0B, 0xD7,
+0x0A, 0x92, 0x04, 0x49, 0x0B, 0x97, 0x4A, 0xE9,
+0x0C, 0x96, 0x0D, 0x92, 0xEB, 0x61, 0x7A, 0x17,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0x00, 0x6A, 0x7D, 0x67, 0x17, 0xD7, 0x20, 0xF0,
+0x44, 0xC3, 0x09, 0x07, 0x00, 0x6A, 0x18, 0x91,
+0x19, 0x90, 0x04, 0xD2, 0x05, 0xD2, 0x06, 0xD2,
+0x07, 0xD2, 0xB1, 0x18, 0x8F, 0xF6, 0x08, 0xD2,
+0x0B, 0xD2, 0x80, 0xF0, 0x18, 0x2A, 0x5D, 0x67,
+0x20, 0xF0, 0xA4, 0xA2, 0xB4, 0x32, 0x0A, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA9, 0xF2, 0xA0, 0x9A, 0x0A, 0x92, 0x0A, 0x93,
+0xB5, 0xE2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xA9, 0xF2, 0xE4, 0x9A, 0x0C, 0xD2,
+0xFD, 0xE3, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xC0, 0x36, 0xF2, 0xF4, 0x44, 0x9E, 0x85, 0x67,
+0x0F, 0xD7, 0x0E, 0xD6, 0x40, 0xEA, 0x0D, 0xD5,
+0x0D, 0x95, 0x0A, 0x93, 0x0F, 0x97, 0x73, 0xE5,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x89, 0xF2, 0xDC, 0x9B, 0x04, 0x4D, 0xEA, 0xED,
+0xD1, 0xE4, 0x3E, 0x65, 0xDD, 0x67, 0x91, 0xE6,
+0x44, 0xDC, 0x0E, 0x96, 0x0D, 0xD3, 0xE1, 0x61,
+0x04, 0x94, 0x40, 0xA1, 0x10, 0x6F, 0xEB, 0xEF,
+0x0F, 0x6D, 0x8C, 0xED, 0xEC, 0xEA, 0xAD, 0xEA,
+0x01, 0x6F, 0x92, 0x35, 0x11, 0x6B, 0xEC, 0xED,
+0x6B, 0xEB, 0xB0, 0x35, 0x6C, 0xEA, 0xAD, 0xEA,
+0x96, 0x34, 0x21, 0x6D, 0x8C, 0xEF, 0xAB, 0xED,
+0xAC, 0xEA, 0xF4, 0x37, 0x4D, 0xEF, 0x05, 0x92,
+0xE0, 0xC1, 0x41, 0xD9, 0x06, 0x92, 0x42, 0xD9,
+0x07, 0x92, 0x43, 0xD9, 0x08, 0x92, 0x44, 0xD9,
+0x17, 0x92, 0x01, 0x72, 0x40, 0x61, 0x5D, 0x67,
+0x20, 0xF0, 0x64, 0xA2, 0x59, 0x67, 0x01, 0x4B,
+0x74, 0x33, 0x5F, 0xE3, 0x0C, 0x92, 0xA9, 0xF2,
+0xA4, 0x9A, 0xA5, 0xE3, 0xF2, 0xF4, 0x44, 0x9E,
+0x87, 0x67, 0x0E, 0xD3, 0x0C, 0xD6, 0x40, 0xEA,
+0x0A, 0xD7, 0x0D, 0x95, 0x0A, 0x97, 0x0E, 0x93,
+0x89, 0xF2, 0xBC, 0x9D, 0x0C, 0x96, 0x73, 0xE7,
+0xB1, 0xE4, 0x04, 0x4F, 0xBD, 0x67, 0x91, 0xE5,
+0x2A, 0xEF, 0x44, 0xDC, 0xEB, 0x61, 0x04, 0x93,
+0x40, 0xA0, 0x10, 0x6D, 0xAB, 0xED, 0x0F, 0x6C,
+0x6C, 0xEC, 0xAC, 0xEA, 0x8D, 0xEA, 0x01, 0x6D,
+0x72, 0x34, 0x11, 0x6E, 0xAC, 0xEC, 0xCB, 0xEE,
+0x90, 0x34, 0xCC, 0xEA, 0x8D, 0xEA, 0x76, 0x33,
+0x21, 0x6C, 0xAC, 0xEB, 0x8B, 0xEC, 0x74, 0x33,
+0x8C, 0xEA, 0x6D, 0xEA, 0x40, 0xC0, 0x05, 0x92,
+0x41, 0xD8, 0x06, 0x92, 0x42, 0xD8, 0x07, 0x92,
+0x43, 0xD8, 0x08, 0x92, 0x44, 0xD8, 0x0B, 0x92,
+0x13, 0x97, 0x12, 0x91, 0x11, 0x90, 0x00, 0xEF,
+0x0A, 0x63, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x00, 0x6A, 0x7D, 0x67,
+0x06, 0x67, 0x20, 0xF0, 0x44, 0xC3, 0xC5, 0x67,
+0x00, 0x6A, 0x09, 0x07, 0xFF, 0x6D, 0x04, 0xD2,
+0x05, 0xD2, 0x06, 0xD2, 0x07, 0xD2, 0xB1, 0x18,
+0x8F, 0xF6, 0x08, 0xD2, 0x4F, 0x2A, 0x5D, 0x67,
+0x20, 0xF0, 0xA4, 0xA2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xA9, 0xF2, 0x20, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA9, 0xF2, 0xC4, 0x9A, 0xB4, 0x35, 0x25, 0xE5,
+0xD9, 0xE5, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF4, 0x44, 0x9A, 0x91, 0x67,
+0x0B, 0xD6, 0x40, 0xEA, 0x0A, 0xD5, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x0A, 0x95, 0x80, 0x34,
+0x89, 0xF2, 0x9C, 0x9C, 0x0B, 0x96, 0xAF, 0xE1,
+0x8D, 0xE3, 0x04, 0x49, 0x9D, 0x67, 0x6D, 0xE4,
+0xCA, 0xE9, 0x44, 0xDB, 0xE6, 0x61, 0x04, 0x93,
+0x40, 0xA0, 0x10, 0x6D, 0xAB, 0xED, 0x0F, 0x6C,
+0x6C, 0xEC, 0xAC, 0xEA, 0x8D, 0xEA, 0x01, 0x6D,
+0x72, 0x34, 0x11, 0x6E, 0xAC, 0xEC, 0xCB, 0xEE,
+0x90, 0x34, 0xCC, 0xEA, 0x8D, 0xEA, 0x76, 0x33,
+0x21, 0x6C, 0xAC, 0xEB, 0x8B, 0xEC, 0x74, 0x33,
+0x8C, 0xEA, 0x6D, 0xEA, 0x40, 0xC0, 0x05, 0x92,
+0x41, 0xD8, 0x06, 0x92, 0x42, 0xD8, 0x07, 0x92,
+0x43, 0xD8, 0x08, 0x92, 0x44, 0xD8, 0x5D, 0x67,
+0x20, 0xF0, 0x44, 0xA2, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0x00, 0x6A, 0x7D, 0x67, 0x04, 0xD2, 0x4A, 0xCB,
+0x56, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x89, 0xF2, 0x30, 0x9A, 0x12, 0xD4,
+0x13, 0xD5, 0x14, 0xD6, 0x15, 0xD7, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0xF2, 0xF4,
+0x44, 0x98, 0x40, 0xEA, 0x91, 0x67, 0x01, 0x6C,
+0x8C, 0xEA, 0x0B, 0xD0, 0x5F, 0x22, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0xF2, 0xF4,
+0x4C, 0x98, 0x87, 0x41, 0x40, 0xEA, 0x11, 0x4C,
+0x12, 0x93, 0x6A, 0xEA, 0x53, 0x61, 0x0B, 0x92,
+0x67, 0x41, 0x15, 0x4B, 0xF2, 0xF4, 0x44, 0x9A,
+0x83, 0x67, 0x40, 0xEA, 0x07, 0xD3, 0x42, 0x34,
+0x82, 0x34, 0x03, 0x6A, 0x4C, 0xEC, 0x67, 0x41,
+0xF2, 0xF4, 0x4C, 0x98, 0x19, 0x4B, 0x08, 0xD4,
+0x83, 0x67, 0x40, 0xEA, 0x0C, 0xD3, 0x09, 0xD2,
+0x00, 0x6C, 0x7D, 0x67, 0x01, 0x6D, 0x4C, 0xED,
+0x99, 0xE3, 0x01, 0x4C, 0xB0, 0xC6, 0x46, 0x32,
+0xFF, 0x6D, 0x07, 0x74, 0xAC, 0xEA, 0xF5, 0x61,
+0x47, 0x41, 0x1A, 0x4A, 0x67, 0x41, 0x0A, 0xD2,
+0x21, 0x4B, 0x22, 0x67, 0xF2, 0xF4, 0x4C, 0x98,
+0x91, 0x67, 0x40, 0xEA, 0x06, 0xD3, 0x06, 0x93,
+0x01, 0x49, 0x2A, 0xEB, 0xF7, 0x61, 0x14, 0x95,
+0xB1, 0x18, 0x7B, 0xF6, 0x08, 0x94, 0x22, 0x67,
+0xF1, 0x72, 0xF1, 0x6A, 0x06, 0xD2, 0x0C, 0x61,
+0x06, 0x92, 0x11, 0x97, 0x10, 0x91, 0x0F, 0x90,
+0x00, 0xEF, 0x09, 0x63, 0x5D, 0x67, 0x29, 0xE2,
+0x50, 0xA2, 0x01, 0x72, 0x19, 0x61, 0x01, 0x49,
+0x07, 0x59, 0xF8, 0x61, 0xF2, 0x6A, 0x08, 0x10,
+0x01, 0x4C, 0x07, 0x74, 0x28, 0x61, 0x3F, 0x10,
+0x01, 0x4A, 0x07, 0x72, 0x48, 0x61, 0x00, 0x6A,
+0x06, 0xD2, 0xE6, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x89, 0xF2, 0x58, 0x9A,
+0x40, 0x49, 0x2E, 0xEA, 0x8C, 0x2A, 0xF3, 0x17,
+0xFF, 0x6A, 0x14, 0x95, 0x08, 0x94, 0xC2, 0x67,
+0x2C, 0xEE, 0xB1, 0x18, 0x5B, 0xF6, 0x3A, 0x65,
+0x06, 0xD2, 0xE0, 0x2A, 0x09, 0x92, 0x01, 0x68,
+0x04, 0xE9, 0x4D, 0xE8, 0x59, 0x67, 0x4C, 0xE8,
+0x0B, 0x92, 0x07, 0x94, 0xF2, 0xF4, 0x44, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x00, 0x6C, 0x8A, 0xE9,
+0xD3, 0x61, 0x87, 0x41, 0x02, 0x4C, 0xFF, 0x6D,
+0x84, 0x34, 0xAC, 0xEC, 0x03, 0x6D, 0xA4, 0xEC,
+0xAF, 0xED, 0xAC, 0xEA, 0xF7, 0xF0, 0x01, 0x6D,
+0xA0, 0x35, 0x13, 0x93, 0xA0, 0x35, 0xD2, 0xF4,
+0xD8, 0x9D, 0x03, 0x6D, 0x6C, 0xED, 0xA4, 0xEC,
+0x07, 0x94, 0x40, 0xEE, 0x4D, 0xED, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0xF2, 0xF4,
+0x40, 0x9E, 0x0C, 0x94, 0xB0, 0x67, 0x40, 0xEA,
+0x07, 0xD6, 0x07, 0x96, 0x00, 0x6A, 0x4A, 0xE9,
+0xB3, 0x61, 0x0A, 0x93, 0xF2, 0xF4, 0x40, 0x9E,
+0x15, 0x95, 0x40, 0xEA, 0x71, 0xE1, 0x98, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0xF7, 0xF0, 0x01, 0x6F,
+0x20, 0x31, 0xE0, 0x37, 0x20, 0x31, 0xE0, 0x37,
+0x1B, 0x2C, 0x00, 0x6B, 0xA9, 0xF2, 0x88, 0x9F,
+0xF2, 0xF4, 0x44, 0x99, 0xFF, 0x68, 0x91, 0xE3,
+0x94, 0x34, 0x6C, 0xE8, 0x05, 0xD7, 0x40, 0xEA,
+0x04, 0xD3, 0x0F, 0x6C, 0x8C, 0xEA, 0x06, 0x22,
+0x04, 0x93, 0x05, 0x97, 0x01, 0x4B, 0x10, 0x73,
+0xED, 0x61, 0xF3, 0x68, 0x50, 0x67, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0x00, 0x6D, 0xA9, 0xF2, 0x68, 0x9F, 0xF2, 0xF4,
+0x44, 0x99, 0xFF, 0x68, 0x6D, 0xE5, 0x74, 0x33,
+0x83, 0x67, 0xAC, 0xE8, 0x07, 0xD7, 0x06, 0xD5,
+0x40, 0xEA, 0x05, 0xD3, 0x05, 0x93, 0x04, 0xD2,
+0xF2, 0xF4, 0x44, 0x99, 0x87, 0x43, 0x40, 0xEA,
+0x19, 0x4C, 0x04, 0x96, 0xCD, 0xEA, 0x0F, 0x6E,
+0xCC, 0xEA, 0xE0, 0x22, 0x06, 0x95, 0x07, 0x97,
+0x01, 0x4D, 0x0F, 0x75, 0xE2, 0x61, 0xD9, 0x17,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0x00, 0x6A, 0x10, 0xD4, 0x87, 0x67, 0x0F, 0xD5,
+0x0E, 0xD6, 0x1B, 0xD7, 0x1C, 0x91, 0x1D, 0x90,
+0x09, 0xD2, 0x0A, 0xD2, 0x0B, 0xD2, 0x0C, 0xD2,
+0x0D, 0xD2, 0x04, 0xD2, 0x05, 0xD2, 0x06, 0xD2,
+0x07, 0xD2, 0xB1, 0x18, 0x3E, 0xF8, 0x08, 0xD2,
+0xF3, 0x72, 0x3A, 0x65, 0x0E, 0x96, 0xF3, 0x6A,
+0x0F, 0x95, 0x10, 0x93, 0x0A, 0x60, 0xF9, 0x67,
+0x83, 0x67, 0xB1, 0x18, 0xD8, 0xF7, 0x0E, 0xD7,
+0x04, 0x2A, 0x0E, 0x93, 0x3B, 0x65, 0x06, 0x29,
+0xF0, 0x6A, 0x17, 0x97, 0x16, 0x91, 0x15, 0x90,
+0x00, 0xEF, 0x0C, 0x63, 0x60, 0xA1, 0x0F, 0x6C,
+0xFF, 0x6D, 0x6C, 0xEC, 0xAC, 0xEC, 0x0B, 0x54,
+0x02, 0x61, 0xF1, 0x6A, 0xF2, 0x17, 0x30, 0x6E,
+0xCC, 0xEB, 0xAC, 0xEB, 0x8D, 0xEB, 0x09, 0xD3,
+0x61, 0x99, 0x0A, 0xD3, 0x62, 0x99, 0x0B, 0xD3,
+0x63, 0x99, 0x0C, 0xD3, 0x64, 0x99, 0x0D, 0xD3,
+0x79, 0x67, 0x74, 0x31, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xA9, 0xF2, 0xE0, 0x9B,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xA9, 0xF2, 0x64, 0x9B, 0xFD, 0xE1, 0x6D, 0xE1,
+0x0E, 0xD3, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6E, 0x60, 0x33, 0xC0, 0x36, 0x60, 0x33,
+0xC0, 0x36, 0x89, 0xF2, 0xBC, 0x9E, 0x12, 0xD2,
+0xD2, 0xF4, 0x58, 0x9B, 0x33, 0xE7, 0xB1, 0xE4,
+0x3A, 0x65, 0x5D, 0x67, 0x91, 0xE2, 0xA9, 0x9C,
+0x59, 0x67, 0x87, 0x67, 0x11, 0xD3, 0x10, 0xD6,
+0x40, 0xEA, 0x0F, 0xD7, 0x0F, 0x97, 0x0E, 0x92,
+0x11, 0x93, 0x04, 0x4F, 0x4A, 0xEF, 0x10, 0x96,
+0x12, 0x92, 0xDF, 0x61, 0x1B, 0x94, 0x01, 0x74,
+0xB0, 0x61, 0xAE, 0x20, 0x80, 0xA0, 0x0F, 0x6D,
+0xFF, 0x6F, 0x8C, 0xED, 0xEC, 0xED, 0x0B, 0x55,
+0x3F, 0x65, 0xB3, 0x60, 0x30, 0x6F, 0x1F, 0x65,
+0xF8, 0x67, 0xEC, 0xEC, 0xF9, 0x67, 0xEC, 0xEC,
+0xAD, 0xEC, 0x04, 0xD4, 0x81, 0x98, 0x05, 0xD4,
+0x82, 0x98, 0x06, 0xD4, 0x83, 0x98, 0x07, 0xD4,
+0x84, 0x98, 0x08, 0xD4, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xA9, 0xF2, 0x0C, 0x9C,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xA9, 0xF2, 0xF0, 0x9C, 0x01, 0xE1, 0xF1, 0xE1,
+0x0E, 0xD4, 0x89, 0xF2, 0xBC, 0x9E, 0x33, 0xE0,
+0xE0, 0x4C, 0x11, 0xD2, 0xB1, 0xE4, 0x5D, 0x67,
+0x91, 0xE2, 0xD2, 0xF4, 0xF8, 0x9B, 0xA4, 0x9C,
+0x90, 0x67, 0x10, 0xD3, 0x40, 0xEF, 0x0F, 0xD6,
+0x0E, 0x92, 0x04, 0x48, 0x0F, 0x96, 0x4A, 0xE8,
+0x10, 0x93, 0x11, 0x92, 0xEA, 0x61, 0x71, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x01, 0x74, 0x20, 0x61, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x89, 0xA3, 0x48, 0xA3, 0x0A, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA3, 0x00, 0x30, 0x00, 0x30,
+0x0D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x65, 0xA2, 0x86, 0xA2, 0x04, 0xA2, 0x47, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xE8, 0x80, 0x34,
+0x0D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x49, 0xA2, 0x2A, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x99, 0xF5, 0x08, 0x6C, 0x26, 0x67, 0x07, 0x67,
+0x40, 0xEA, 0x05, 0xD5, 0xF7, 0xF0, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6B, 0x80, 0x34, 0x80, 0x34,
+0x60, 0x33, 0x60, 0x33, 0xA9, 0xF2, 0xB4, 0x9C,
+0xD2, 0xF4, 0xE8, 0x9B, 0x99, 0xF5, 0x08, 0x6C,
+0x4D, 0xED, 0x40, 0xEF, 0x04, 0xD3, 0x04, 0x93,
+0x05, 0x96, 0x20, 0x35, 0xD2, 0xF4, 0x48, 0x9B,
+0x00, 0x30, 0xCD, 0xED, 0x00, 0x30, 0x99, 0xF5,
+0x10, 0x6C, 0x40, 0xEA, 0x0D, 0xED, 0x01, 0x6A,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0x01, 0x74, 0x21, 0x61,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x89, 0xA3, 0x48, 0xA3,
+0xCA, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2, 0x86, 0xA2,
+0xC4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xEE, 0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x69, 0xA2, 0x00, 0x6A,
+0x10, 0x23, 0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x85, 0x67,
+0xD2, 0xF4, 0x50, 0x9A, 0x8C, 0xF6, 0x00, 0x4C,
+0x40, 0xEA, 0x84, 0x34, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x20, 0xE8, 0x01, 0x6A, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x6A, 0x20, 0xE8, 0x00, 0x6A,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x72, 0xF0, 0x50, 0x99, 0x10, 0xD4, 0x40, 0xEA,
+0x05, 0x67, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0xE0, 0x37, 0x15, 0xF0, 0xC9, 0xA7, 0x15, 0xF0,
+0x68, 0xA7, 0x15, 0xF0, 0x8A, 0xA7, 0xC0, 0x36,
+0x04, 0xD2, 0x6D, 0xEE, 0x10, 0x92, 0x15, 0xF0,
+0x6B, 0xA7, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEE,
+0x00, 0xF6, 0x60, 0x33, 0x82, 0x42, 0xCD, 0xEB,
+0x88, 0x34, 0x6D, 0xE4, 0xC5, 0xA3, 0x84, 0xA3,
+0xE6, 0xA3, 0xC0, 0x36, 0x8D, 0xEE, 0x87, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0x08, 0x32, 0xCD, 0xEC, 0x05, 0xD2,
+0x91, 0xE2, 0xC1, 0xA4, 0x60, 0xA4, 0xE2, 0xA4,
+0xC0, 0x36, 0x6D, 0xEE, 0x63, 0xA4, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x60, 0x33,
+0xCD, 0xEB, 0xE0, 0x9B, 0xB0, 0x67, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0xEA, 0xEB, 0x00, 0x30,
+0x17, 0x61, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6C, 0x60, 0x33, 0x80, 0x34, 0x80, 0x34,
+0x60, 0x33, 0xD3, 0xF6, 0x70, 0x9B, 0x69, 0xF1,
+0xAC, 0x9C, 0x40, 0xEB, 0x01, 0x6C, 0x72, 0xF0,
+0x74, 0x98, 0x40, 0xEB, 0x04, 0x94, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0x0A, 0xD3, 0x09, 0xD5, 0xFF, 0xF7, 0xD9, 0xA7,
+0xFF, 0xF7, 0x58, 0xA7, 0xFF, 0xF7, 0x9A, 0xA7,
+0xC0, 0x36, 0xCD, 0xEA, 0xFF, 0xF7, 0xDB, 0xA7,
+0x80, 0x34, 0x80, 0x34, 0x4D, 0xEC, 0x00, 0xF6,
+0xC0, 0x36, 0x8D, 0xEE, 0x06, 0xD6, 0xFF, 0xF7,
+0x9D, 0xA7, 0xFF, 0xF7, 0x5C, 0xA7, 0xFF, 0xF7,
+0xDF, 0xA7, 0x80, 0x34, 0x8D, 0xEA, 0xFF, 0xF7,
+0x9E, 0xA7, 0x00, 0xF6, 0xC0, 0x36, 0x07, 0xD7,
+0x80, 0x34, 0x80, 0x34, 0x4D, 0xEC, 0x8D, 0xEE,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x13, 0xF6, 0x5C, 0x9C, 0x87, 0x67, 0x40, 0xEA,
+0x08, 0xD6, 0x72, 0xF0, 0x54, 0x98, 0x40, 0xEA,
+0x04, 0x94, 0xF7, 0xF0, 0x01, 0x6A, 0x07, 0x97,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A,
+0x88, 0x47, 0x40, 0xEA, 0xF8, 0x4C, 0x08, 0x96,
+0x06, 0x92, 0x40, 0xEA, 0x86, 0x67, 0x72, 0xF0,
+0x50, 0x99, 0x40, 0xEA, 0x00, 0x65, 0x0A, 0x93,
+0x22, 0x67, 0x40, 0x9B, 0x4E, 0xEB, 0x32, 0x23,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD3, 0xF3, 0x68, 0x9B, 0xFF, 0xF7, 0xD1, 0xA2,
+0xFF, 0xF7, 0x90, 0xA2, 0x1B, 0x65, 0xFF, 0xF7,
+0x72, 0xA2, 0xC0, 0x36, 0x8D, 0xEE, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEE, 0xFF, 0xF7, 0x73, 0xA2,
+0xFF, 0xF7, 0xF5, 0xA2, 0x09, 0x95, 0x00, 0xF6,
+0x60, 0x33, 0x3B, 0x65, 0xFF, 0xF7, 0x74, 0xA2,
+0xE0, 0x37, 0x10, 0x94, 0x6D, 0xEF, 0xFF, 0xF7,
+0x76, 0xA2, 0x60, 0x33, 0x60, 0x33, 0xED, 0xEB,
+0xFF, 0xF7, 0xF7, 0xA2, 0x59, 0x67, 0x4D, 0xEE,
+0x00, 0xF6, 0xE0, 0x37, 0x58, 0x67, 0x40, 0xEA,
+0x6D, 0xEF, 0x72, 0xF0, 0x54, 0x98, 0x40, 0xEA,
+0x91, 0x67, 0x81, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0xC8, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0xF8, 0xF5, 0x04, 0x6C,
+0x40, 0xEA, 0x04, 0xD6, 0xF7, 0xF0, 0x01, 0x6B,
+0x05, 0x94, 0x60, 0x33, 0x60, 0x33, 0xB3, 0xF7,
+0x10, 0x4B, 0x6D, 0xE4, 0xA0, 0x9B, 0x04, 0x96,
+0xF8, 0xF5, 0x04, 0x6C, 0xAF, 0xED, 0x40, 0xEE,
+0x4C, 0xED, 0xDB, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6F,
+0xE0, 0x37, 0xE0, 0x37, 0x15, 0xF0, 0xC9, 0xA7,
+0x15, 0xF0, 0x48, 0xA7, 0x15, 0xF0, 0x6A, 0xA7,
+0xC0, 0x36, 0x4D, 0xEE, 0x15, 0xF0, 0x4B, 0xA7,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0x02, 0x4C, 0xCD, 0xEA, 0x88, 0x34,
+0x51, 0xE4, 0xC5, 0xA4, 0x44, 0xA4, 0x66, 0xA4,
+0xC0, 0x36, 0x4D, 0xEE, 0x47, 0xA4, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEE, 0x00, 0xF6, 0x40, 0x32,
+0xA8, 0x33, 0xCD, 0xEA, 0x05, 0xD3, 0x49, 0xE3,
+0x81, 0xA2, 0xA2, 0xA2, 0x60, 0xA2, 0x03, 0xA2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x80, 0x34,
+0xA0, 0x35, 0x40, 0x32, 0x6D, 0xEC, 0xA0, 0x35,
+0x72, 0xF0, 0x50, 0x9A, 0xAD, 0xEC, 0x00, 0xF6,
+0x00, 0x30, 0x40, 0xEA, 0x8D, 0xE8, 0x20, 0x98,
+0x04, 0xD2, 0x2A, 0xE8, 0x1B, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6,
+0x50, 0x9A, 0x69, 0xF1, 0xAC, 0x9B, 0x40, 0xEA,
+0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x04, 0x94, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x13, 0xF6, 0xBC, 0x9C,
+0x40, 0xED, 0x91, 0x67, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x72, 0xF4, 0xB0, 0x9C,
+0x88, 0x41, 0x40, 0xED, 0xF8, 0x4C, 0x20, 0x99,
+0x2A, 0xE8, 0xEC, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0xC8, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0xF8, 0xF5, 0x04, 0x6C,
+0x40, 0xEA, 0x26, 0x67, 0xF7, 0xF0, 0x01, 0x6B,
+0x05, 0x94, 0x60, 0x33, 0x60, 0x33, 0xB3, 0xF7,
+0x10, 0x4B, 0x81, 0xE3, 0xA0, 0x98, 0xF8, 0xF5,
+0x04, 0x6C, 0xAF, 0xED, 0x40, 0xE9, 0x4C, 0xED,
+0xC0, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x0E, 0xD6, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0x0C, 0xD4,
+0x15, 0xF0, 0x4A, 0xA6, 0x15, 0xF0, 0x89, 0xA6,
+0x15, 0xF0, 0x68, 0xA6, 0x40, 0x32, 0x80, 0x34,
+0x8D, 0xEB, 0x40, 0x32, 0x15, 0xF0, 0x8B, 0xA6,
+0x6D, 0xEA, 0x0C, 0x93, 0x00, 0xF6, 0x80, 0x34,
+0x8D, 0xEA, 0x02, 0x4B, 0x68, 0x33, 0x4D, 0xE3,
+0x85, 0xA3, 0x44, 0xA3, 0xC6, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0xA8, 0x33,
+0x8D, 0xEA, 0x06, 0xD5, 0x05, 0xD3, 0x49, 0xE3,
+0x81, 0xA2, 0xC2, 0xA2, 0x60, 0xA2, 0x03, 0xA2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x80, 0x34,
+0xC0, 0x36, 0x40, 0x32, 0x6D, 0xEC, 0xC0, 0x36,
+0x72, 0xF0, 0x50, 0x9A, 0xCD, 0xEC, 0x00, 0xF6,
+0x00, 0x30, 0x40, 0xEA, 0x8D, 0xE8, 0x20, 0x98,
+0x04, 0xD2, 0x06, 0x95, 0x2A, 0xE8, 0x1B, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x40, 0x32, 0x60, 0x33,
+0xD3, 0xF6, 0x50, 0x9A, 0x69, 0xF1, 0xAC, 0x9B,
+0x40, 0xEA, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x04, 0x94, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0xFF, 0xF7,
+0xF9, 0xA1, 0xFF, 0xF7, 0x98, 0xA1, 0xFF, 0xF7,
+0xDA, 0xA1, 0xE0, 0x37, 0x8D, 0xEF, 0xFF, 0xF7,
+0x9B, 0xA1, 0xC0, 0x36, 0xC0, 0x36, 0x0E, 0x92,
+0xCD, 0xEF, 0x00, 0xF6, 0x80, 0x34, 0xED, 0xEC,
+0x4E, 0xEC, 0x13, 0x2C, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x13, 0xF6, 0xDC, 0x9C,
+0x91, 0x67, 0x40, 0xEE, 0x06, 0xD5, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x72, 0xF4,
+0xD0, 0x9C, 0x88, 0x41, 0x40, 0xEE, 0xF8, 0x4C,
+0x06, 0x95, 0x20, 0x99, 0x2A, 0xE8, 0xD7, 0x61,
+0x40, 0x98, 0x70, 0x67, 0x4E, 0xEB, 0x2D, 0x23,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD3, 0xF3, 0x68, 0x9B, 0xFF, 0xF7, 0xD1, 0xA2,
+0xFF, 0xF7, 0x90, 0xA2, 0x1B, 0x65, 0xFF, 0xF7,
+0x72, 0xA2, 0xC0, 0x36, 0x8D, 0xEE, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEE, 0xFF, 0xF7, 0x73, 0xA2,
+0xFF, 0xF7, 0xF5, 0xA2, 0x0C, 0x94, 0x00, 0xF6,
+0x60, 0x33, 0x3B, 0x65, 0xFF, 0xF7, 0x74, 0xA2,
+0xE0, 0x37, 0x6D, 0xEF, 0xFF, 0xF7, 0x76, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0xED, 0xEB, 0xFF, 0xF7,
+0xF7, 0xA2, 0x59, 0x67, 0x4D, 0xEE, 0x00, 0xF6,
+0xE0, 0x37, 0x58, 0x67, 0x40, 0xEA, 0x6D, 0xEF,
+0x99, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x28, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0xF8, 0xF5, 0x04, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x05, 0x94,
+0x60, 0x33, 0x60, 0x33, 0xB3, 0xF7, 0x10, 0x4B,
+0x8D, 0xE3, 0xA0, 0x9B, 0xF8, 0xF5, 0x04, 0x6C,
+0xAF, 0xED, 0x40, 0xE9, 0x4C, 0xED, 0x7A, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0x24, 0x67, 0x15, 0xF0, 0x89, 0xA7, 0x15, 0xF0,
+0x68, 0xA7, 0x15, 0xF0, 0x4A, 0xA7, 0x80, 0x34,
+0x8D, 0xEB, 0x15, 0xF0, 0x8B, 0xA7, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xEA, 0x00, 0xF6, 0x80, 0x34,
+0x62, 0x41, 0x8D, 0xEA, 0x68, 0x33, 0x07, 0xD6,
+0x4D, 0xE3, 0x85, 0xA3, 0x44, 0xA3, 0xE6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0xA8, 0x33, 0x8D, 0xEA, 0x06, 0xD5, 0x05, 0xD3,
+0x49, 0xE3, 0x60, 0xA2, 0x81, 0xA2, 0xE2, 0xA2,
+0x03, 0xA2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x80, 0x34, 0xE0, 0x37, 0x72, 0xF0,
+0x50, 0x9A, 0x6D, 0xEC, 0xE0, 0x37, 0xED, 0xEC,
+0x00, 0xF6, 0x00, 0x30, 0x40, 0xEA, 0x8D, 0xE8,
+0x04, 0xD2, 0x40, 0x98, 0x06, 0x95, 0x07, 0x96,
+0x4A, 0xE8, 0x1E, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6, 0x50, 0x9A,
+0x69, 0xF1, 0xAC, 0x9B, 0x40, 0xEA, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x04, 0x94,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x40, 0x9A, 0x4A, 0xE8, 0x29, 0x60,
+0xFF, 0xF7, 0xF9, 0xA2, 0xFF, 0xF7, 0x98, 0xA2,
+0xE0, 0x37, 0x8D, 0xEF, 0x1F, 0x65, 0xFF, 0xF7,
+0xFA, 0xA2, 0xFF, 0xF7, 0x9B, 0xA2, 0x78, 0x67,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEB, 0x00, 0xF6,
+0x80, 0x34, 0x6D, 0xEC, 0xCE, 0xEC, 0xE9, 0x2C,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x13, 0xF6, 0xDC, 0x9C, 0x82, 0x67, 0x07, 0xD5,
+0x40, 0xEE, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6C,
+0x06, 0x92, 0x80, 0x34, 0x80, 0x34, 0x72, 0xF4,
+0xD0, 0x9C, 0x88, 0x42, 0x40, 0xEE, 0xF8, 0x4C,
+0x07, 0x95, 0x40, 0x98, 0x70, 0x67, 0x4E, 0xEB,
+0x2D, 0x23, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xD3, 0xF3, 0x68, 0x9B, 0xFF, 0xF7,
+0xD1, 0xA2, 0xFF, 0xF7, 0x90, 0xA2, 0x1B, 0x65,
+0xFF, 0xF7, 0x72, 0xA2, 0xC0, 0x36, 0x8D, 0xEE,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEE, 0xFF, 0xF7,
+0x73, 0xA2, 0xFF, 0xF7, 0xF5, 0xA2, 0x91, 0x67,
+0x00, 0xF6, 0x60, 0x33, 0x3B, 0x65, 0xFF, 0xF7,
+0x74, 0xA2, 0xE0, 0x37, 0x6D, 0xEF, 0xFF, 0xF7,
+0x76, 0xA2, 0x60, 0x33, 0x60, 0x33, 0xED, 0xEB,
+0xFF, 0xF7, 0xF7, 0xA2, 0x59, 0x67, 0x4D, 0xEE,
+0x00, 0xF6, 0xE0, 0x37, 0x58, 0x67, 0x40, 0xEA,
+0x6D, 0xEF, 0x96, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x28, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0xF8, 0xF5, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x05, 0x94, 0x60, 0x33, 0x60, 0x33, 0xB3, 0xF7,
+0x10, 0x4B, 0x8D, 0xE3, 0xA0, 0x9B, 0xF8, 0xF5,
+0x04, 0x6C, 0xAF, 0xED, 0x40, 0xE9, 0x4C, 0xED,
+0x77, 0x17, 0x00, 0x65, 0x00, 0x6A, 0x60, 0xF1,
+0x0E, 0x2C, 0x06, 0x5D, 0x44, 0x67, 0x60, 0xF1,
+0x0A, 0x60, 0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1,
+0x0D, 0xD0, 0x05, 0x75, 0x07, 0x67, 0x26, 0x67,
+0x08, 0xD5, 0x48, 0x60, 0xF7, 0xF0, 0x01, 0x6F,
+0xE0, 0x37, 0xE0, 0x37, 0xF3, 0xF3, 0x4C, 0x9F,
+0x01, 0x6E, 0x40, 0xEA, 0x0A, 0xD7, 0x0A, 0x97,
+0x09, 0xD2, 0x08, 0x95, 0xF3, 0xF3, 0x4C, 0x9F,
+0x00, 0x6E, 0x40, 0xEA, 0x00, 0x6C, 0x09, 0x93,
+0x63, 0xE8, 0x04, 0x61, 0x0A, 0xEB, 0x10, 0x61,
+0x43, 0xE9, 0x0E, 0x60, 0x33, 0xE2, 0x83, 0xEA,
+0xD8, 0x67, 0x17, 0xE3, 0xD7, 0xE5, 0x16, 0x2D,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0xA9, 0xF2, 0xB8, 0x9D, 0x83, 0xED, 0x0E, 0x61,
+0x03, 0xEB, 0x04, 0x61, 0x6A, 0xE8, 0x27, 0x61,
+0x23, 0xEA, 0x25, 0x60, 0x4B, 0xE1, 0x43, 0xE9,
+0x98, 0x67, 0x6F, 0xE0, 0x8F, 0xE3, 0x02, 0x2B,
+0x00, 0x52, 0x1D, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6, 0x50, 0x9A,
+0xA9, 0xF2, 0xBC, 0x9B, 0x40, 0xEA, 0x01, 0x6C,
+0x00, 0x6A, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF3, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x6C, 0xC1, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0x72, 0xF4,
+0x54, 0x9A, 0xC9, 0xF2, 0xA0, 0x9B, 0x40, 0xEA,
+0x18, 0x6C, 0x82, 0x67, 0x23, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xC9, 0xF2,
+0xA4, 0x9B, 0xD3, 0xF6, 0x50, 0x9A, 0x40, 0xEA,
+0x01, 0x6C, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0A, 0xF1, 0x18, 0x4A, 0x05, 0xD2, 0xED, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0xBE, 0x17, 0x20, 0xC2, 0x22, 0x32,
+0x41, 0xC4, 0x42, 0x32, 0x42, 0xC4, 0x00, 0xF6,
+0x22, 0x32, 0x43, 0xC4, 0x02, 0x32, 0x45, 0xC4,
+0x42, 0x32, 0x46, 0xC4, 0x00, 0xF6, 0x02, 0x32,
+0x47, 0xC4, 0x04, 0x02, 0x40, 0xF0, 0x40, 0xA2,
+0x04, 0xC4, 0x0A, 0xD4, 0x48, 0xC4, 0x14, 0x92,
+0x42, 0x32, 0x49, 0xC4, 0x42, 0x32, 0x4A, 0xC4,
+0x14, 0x92, 0x00, 0xF6, 0x42, 0x32, 0x4B, 0xC4,
+0x04, 0x02, 0x40, 0xF0, 0x44, 0xA2, 0x4C, 0xC4,
+0x15, 0x92, 0x42, 0x32, 0x4D, 0xC4, 0x42, 0x32,
+0x4E, 0xC4, 0x15, 0x92, 0x00, 0xF6, 0x42, 0x32,
+0x4F, 0xC4, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xC0, 0x36, 0x15, 0xF0, 0x69, 0xA6, 0x0B, 0xD2,
+0x15, 0xF0, 0x48, 0xA6, 0x15, 0xF0, 0xAA, 0xA6,
+0x60, 0x33, 0x4D, 0xEB, 0x15, 0xF0, 0x4B, 0xA6,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6,
+0x40, 0x32, 0x6D, 0xEA, 0x6D, 0xA2, 0xCC, 0xA2,
+0xAE, 0xA2, 0x60, 0x33, 0x4F, 0xA2, 0x6D, 0xEE,
+0xA0, 0x35, 0x08, 0x93, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x68, 0x36,
+0x59, 0xE6, 0x61, 0xA6, 0x40, 0xA6, 0xA2, 0xA6,
+0x60, 0x33, 0x4D, 0xEB, 0x43, 0xA6, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6, 0x40, 0x32,
+0x6D, 0xEA, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x6C, 0xF2, 0xF8, 0x9B, 0x09, 0xD2,
+0x0A, 0x94, 0x01, 0x2F, 0xE2, 0x67, 0x09, 0x92,
+0xA0, 0x9A, 0x01, 0x6A, 0xEA, 0xED, 0x0A, 0xD2,
+0x28, 0x60, 0xFF, 0xF7, 0x71, 0xA5, 0xFF, 0xF7,
+0x50, 0xA5, 0xFF, 0xF7, 0xD3, 0xA5, 0x60, 0x33,
+0x6D, 0xEA, 0xFF, 0xF7, 0x72, 0xA5, 0x00, 0xF6,
+0xC0, 0x36, 0x60, 0x33, 0x60, 0x33, 0x4D, 0xEB,
+0x6D, 0xEE, 0xFF, 0xF7, 0x75, 0xA5, 0xFF, 0xF7,
+0x54, 0xA5, 0x60, 0x33, 0x4D, 0xEB, 0x1B, 0x65,
+0xFF, 0xF7, 0x76, 0xA5, 0x58, 0x67, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEA, 0xFF, 0xF7, 0x77, 0xA5,
+0x00, 0xF6, 0x60, 0x33, 0x4D, 0xEB, 0x03, 0xEB,
+0x43, 0x61, 0x0E, 0xEB, 0x02, 0x2B, 0x23, 0xEE,
+0x3F, 0x61, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x33, 0xF6, 0x64, 0x9B, 0x40, 0xEB,
+0x10, 0x4C, 0x0A, 0x92, 0x2B, 0x22, 0x09, 0x92,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x40, 0x9A,
+0x60, 0x33, 0xD3, 0xF3, 0x28, 0x9B, 0xFF, 0xF7,
+0xD1, 0xA2, 0xFF, 0xF7, 0xB0, 0xA2, 0xFF, 0xF7,
+0xF5, 0xA2, 0xFF, 0xF7, 0x74, 0xA2, 0xC0, 0x36,
+0xAD, 0xEE, 0xFF, 0xF7, 0xB2, 0xA2, 0xE0, 0x37,
+0x6D, 0xEF, 0xFF, 0xF7, 0x76, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xCD, 0xED, 0x60, 0x33, 0xFF, 0xF7,
+0xD3, 0xA2, 0x60, 0x33, 0xED, 0xEB, 0xFF, 0xF7,
+0xF7, 0xA2, 0x00, 0xF6, 0xC0, 0x36, 0xAD, 0xEE,
+0x08, 0x95, 0x00, 0xF6, 0xE0, 0x37, 0x6D, 0xEF,
+0x40, 0xE9, 0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x0B, 0x94, 0x01, 0x6A, 0xE5, 0x16,
+0xA0, 0x9D, 0x00, 0x6A, 0x93, 0x17, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x15, 0xF0, 0x49, 0xA0,
+0x15, 0xF0, 0x6A, 0xA0, 0x15, 0xF0, 0xC8, 0xA0,
+0x40, 0x32, 0x60, 0x33, 0xCD, 0xEA, 0x60, 0x33,
+0x15, 0xF0, 0xCB, 0xA0, 0x6D, 0xEA, 0xF7, 0xF0,
+0x01, 0x69, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x20, 0x31, 0x60, 0x33, 0x20, 0x31, 0x00, 0xF6,
+0xC0, 0x36, 0x4D, 0xEE, 0xC9, 0xF2, 0xA8, 0x99,
+0x72, 0xF4, 0x54, 0x9B, 0x90, 0x6C, 0x09, 0xD6,
+0x40, 0xEA, 0x08, 0xD3, 0x09, 0x96, 0x42, 0x34,
+0x15, 0xF0, 0xA9, 0xA0, 0x4C, 0xC6, 0x00, 0xF6,
+0x42, 0x32, 0x8D, 0xC6, 0x4F, 0xC6, 0x82, 0x34,
+0x15, 0xF0, 0x48, 0xA0, 0x8E, 0xC6, 0x15, 0xF0,
+0x8A, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x15, 0xF0,
+0x4B, 0xA0, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x8D, 0xA2,
+0xAE, 0xA2, 0xCC, 0xA2, 0x4F, 0xA2, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35, 0x08, 0x93,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x09, 0xD3, 0x0A, 0xD1, 0x15, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0A, 0xF1, 0x18, 0x4A,
+0x05, 0xD2, 0xFF, 0x6A, 0x5A, 0x4A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x15, 0xF0, 0x89, 0xA0, 0x15, 0xF0, 0x48, 0xA0,
+0x15, 0xF0, 0x6A, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x15, 0xF0, 0x4B, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x8D, 0xA2, 0x6E, 0xA2, 0xAC, 0xA2, 0x4F, 0xA2,
+0x80, 0x34, 0x60, 0x33, 0x8D, 0xED, 0x60, 0x33,
+0xAD, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0x08, 0xD2, 0x00, 0x69, 0x00, 0x68, 0x0A, 0x93,
+0x09, 0x92, 0x08, 0x6C, 0xC9, 0xF2, 0xA8, 0x9B,
+0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x08, 0x93, 0x28, 0x35, 0x42, 0x34, 0xB9, 0xE3,
+0x81, 0xC6, 0x82, 0x34, 0x82, 0xC6, 0x00, 0xF6,
+0x42, 0x34, 0x40, 0xC6, 0x83, 0xC6, 0x19, 0x2A,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0A, 0xF1,
+0x18, 0x4A, 0x05, 0xD2, 0xFF, 0x6A, 0x65, 0x4A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x04, 0x04,
+0x0C, 0xD5, 0x40, 0xEA, 0x0B, 0xD6, 0x0C, 0x95,
+0x0B, 0x96, 0xE1, 0xA6, 0x40, 0xA6, 0x82, 0xA6,
+0xE0, 0x37, 0x4D, 0xEF, 0x43, 0xA6, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEF, 0x00, 0xF6, 0x40, 0x32,
+0xED, 0xEA, 0x40, 0xDA, 0x41, 0xDA, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF7,
+0x08, 0x4A, 0xA9, 0xE2, 0x80, 0x9A, 0x07, 0x6A,
+0x44, 0xEC, 0x4F, 0xEA, 0x4C, 0xE8, 0x51, 0x67,
+0x44, 0xEC, 0x4D, 0xE8, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF4, 0xE4, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF7, 0x00, 0x4A, 0xB5, 0xE2, 0xC0, 0x9D,
+0x00, 0x6C, 0x00, 0x6D, 0x40, 0xEF, 0x01, 0x49,
+0x06, 0x71, 0xA1, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A,
+0xF8, 0xF5, 0x04, 0x6C, 0x40, 0xEA, 0xB0, 0x67,
+0x11, 0x97, 0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF,
+0x09, 0x63, 0x00, 0x65, 0x02, 0x5D, 0x02, 0x61,
+0x20, 0xE8, 0x00, 0x6A, 0x4E, 0x44, 0xFF, 0x6B,
+0x6C, 0xEA, 0x02, 0x5A, 0xF9, 0x60, 0xF6, 0x63,
+0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x20, 0x31, 0x15, 0xF0,
+0x69, 0xA1, 0x15, 0xF0, 0x48, 0xA1, 0x04, 0x67,
+0x15, 0xF0, 0x8A, 0xA1, 0x60, 0x33, 0x4D, 0xEB,
+0x15, 0xF0, 0x4B, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x0C, 0xD7,
+0x0B, 0xD6, 0x09, 0xD5, 0x6D, 0xEA, 0x91, 0xA2,
+0x70, 0xA2, 0xB2, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x73, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x4F, 0x2B,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xC9, 0xF2, 0xAC, 0x9B, 0x72, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x40, 0x6C, 0x62, 0x67, 0x1B, 0x2A,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0A, 0xF1,
+0x18, 0x4A, 0x05, 0xD2, 0x80, 0xF1, 0x14, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x00, 0x6A, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF6,
+0x40, 0x9A, 0x83, 0x67, 0x00, 0x6D, 0x40, 0x6E,
+0x40, 0xEA, 0x08, 0xD3, 0x15, 0xF0, 0xA9, 0xA1,
+0x15, 0xF0, 0x48, 0xA1, 0x15, 0xF0, 0x8A, 0xA1,
+0xA0, 0x35, 0x4D, 0xED, 0x15, 0xF0, 0x4B, 0xA1,
+0x08, 0x93, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x62, 0x34,
+0x91, 0xC2, 0x82, 0x34, 0x92, 0xC2, 0x00, 0xF6,
+0x62, 0x34, 0x70, 0xC2, 0x93, 0xC2, 0x10, 0x32,
+0x5D, 0xE3, 0xA1, 0xA7, 0x0A, 0xD2, 0x40, 0xA7,
+0x82, 0xA7, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA7,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xC6, 0x2A, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0xD2, 0xF4,
+0x48, 0x9E, 0x0D, 0xD3, 0x00, 0x6D, 0x3A, 0x65,
+0x04, 0x32, 0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x08, 0x93, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF7,
+0x18, 0x4A, 0x51, 0xE3, 0x80, 0xAC, 0x02, 0x67,
+0x59, 0x67, 0x0E, 0xD7, 0x40, 0xEA, 0x26, 0x67,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x49, 0xF2, 0xA4, 0x9C, 0x0C, 0x92, 0x09, 0x94,
+0xD1, 0x67, 0xAC, 0xEA, 0xA2, 0x67, 0x0D, 0x93,
+0x50, 0x67, 0x0E, 0x97, 0x38, 0x2C, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x29, 0xF1,
+0x88, 0x9C, 0x8D, 0xED, 0x0B, 0x94, 0x07, 0x24,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xC9, 0xF1, 0x80, 0x9C, 0x8D, 0xED, 0x08, 0x04,
+0x40, 0xF0, 0x80, 0xA4, 0x80, 0xC7, 0x18, 0x94,
+0x82, 0x34, 0x81, 0xC7, 0x82, 0x34, 0x82, 0xC7,
+0x18, 0x94, 0x00, 0xF6, 0x82, 0x31, 0x08, 0x04,
+0x40, 0xF0, 0x84, 0xA4, 0x23, 0xC7, 0x84, 0xC7,
+0x19, 0x94, 0x82, 0x34, 0x85, 0xC7, 0x82, 0x34,
+0x86, 0xC7, 0x19, 0x94, 0x00, 0xF6, 0x82, 0x30,
+0x0A, 0x94, 0x07, 0xC7, 0x8D, 0xE3, 0x9D, 0x67,
+0x20, 0xF0, 0x84, 0xA4, 0x88, 0xC3, 0x08, 0x94,
+0xD2, 0xF4, 0x68, 0x9E, 0x89, 0xE2, 0x40, 0xEB,
+0x80, 0xAA, 0x01, 0x6A, 0x67, 0x17, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xC9, 0xF2,
+0x90, 0x9C, 0xC7, 0x17, 0x02, 0x5D, 0x02, 0x61,
+0x20, 0xE8, 0x00, 0x6A, 0x4E, 0x44, 0xFF, 0x6B,
+0x6C, 0xEA, 0x02, 0x5A, 0xF9, 0x60, 0xF6, 0x63,
+0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x20, 0x31, 0x15, 0xF0,
+0x69, 0xA1, 0x15, 0xF0, 0x48, 0xA1, 0x04, 0x67,
+0x15, 0xF0, 0x8A, 0xA1, 0x60, 0x33, 0x4D, 0xEB,
+0x15, 0xF0, 0x4B, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x0C, 0xD7,
+0x0B, 0xD6, 0x09, 0xD5, 0x6D, 0xEA, 0x91, 0xA2,
+0x70, 0xA2, 0xB2, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x73, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x4F, 0x2B,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xC9, 0xF2, 0xB4, 0x9B, 0x72, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x40, 0x6C, 0x62, 0x67, 0x1B, 0x2A,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0A, 0xF1,
+0x18, 0x4A, 0x05, 0xD2, 0xC0, 0xF1, 0x16, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x00, 0x6A, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF6,
+0x40, 0x9A, 0x83, 0x67, 0x00, 0x6D, 0x40, 0x6E,
+0x40, 0xEA, 0x08, 0xD3, 0x15, 0xF0, 0xA9, 0xA1,
+0x15, 0xF0, 0x48, 0xA1, 0x15, 0xF0, 0x8A, 0xA1,
+0xA0, 0x35, 0x4D, 0xED, 0x15, 0xF0, 0x4B, 0xA1,
+0x08, 0x93, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x62, 0x34,
+0x91, 0xC2, 0x82, 0x34, 0x92, 0xC2, 0x00, 0xF6,
+0x62, 0x34, 0x70, 0xC2, 0x93, 0xC2, 0x10, 0x32,
+0x5D, 0xE3, 0xA1, 0xA7, 0x0A, 0xD2, 0x40, 0xA7,
+0x82, 0xA7, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA7,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xC6, 0x2A, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0xD2, 0xF4,
+0x48, 0x9E, 0x0D, 0xD3, 0x00, 0x6D, 0x3A, 0x65,
+0x04, 0x32, 0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x08, 0x93, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF7,
+0x18, 0x4A, 0x51, 0xE3, 0x80, 0xAC, 0x02, 0x67,
+0x59, 0x67, 0x0E, 0xD7, 0x40, 0xEA, 0x26, 0x67,
+0xF7, 0xF0, 0x01, 0x6C, 0x0C, 0x92, 0x80, 0x34,
+0x80, 0x34, 0x49, 0xF2, 0x84, 0x9C, 0x40, 0x35,
+0xA8, 0x35, 0x8C, 0xED, 0x09, 0x94, 0xD1, 0x67,
+0x50, 0x67, 0x0D, 0x93, 0x0E, 0x97, 0x38, 0x2C,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x29, 0xF1, 0x88, 0x9C, 0x8D, 0xED, 0x0B, 0x94,
+0x07, 0x24, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xC9, 0xF1, 0x80, 0x9C, 0x8D, 0xED,
+0x08, 0x04, 0x40, 0xF0, 0x80, 0xA4, 0x80, 0xC7,
+0x18, 0x94, 0x82, 0x34, 0x81, 0xC7, 0x82, 0x34,
+0x82, 0xC7, 0x18, 0x94, 0x00, 0xF6, 0x82, 0x31,
+0x08, 0x04, 0x40, 0xF0, 0x84, 0xA4, 0x23, 0xC7,
+0x84, 0xC7, 0x19, 0x94, 0x82, 0x34, 0x85, 0xC7,
+0x82, 0x34, 0x86, 0xC7, 0x19, 0x94, 0x00, 0xF6,
+0x82, 0x30, 0x0A, 0x94, 0x07, 0xC7, 0x8D, 0xE3,
+0x9D, 0x67, 0x20, 0xF0, 0x84, 0xA4, 0x88, 0xC3,
+0x08, 0x94, 0xD2, 0xF4, 0x68, 0x9E, 0x89, 0xE2,
+0x40, 0xEB, 0x80, 0xAA, 0x01, 0x6A, 0x66, 0x17,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xC9, 0xF2, 0x90, 0x9C, 0xC7, 0x17, 0x00, 0x65,
+0x4E, 0x44, 0xFF, 0x6B, 0x6C, 0xEA, 0x02, 0x5A,
+0x02, 0x61, 0x20, 0xE8, 0x00, 0x6A, 0x02, 0x5D,
+0xFC, 0x60, 0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1,
+0x11, 0xD0, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0x15, 0xF0, 0x69, 0xA1, 0x15, 0xF0,
+0x48, 0xA1, 0x04, 0x67, 0x15, 0xF0, 0x8A, 0xA1,
+0x60, 0x33, 0x4D, 0xEB, 0x15, 0xF0, 0x4B, 0xA1,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6,
+0x40, 0x32, 0x0C, 0xD7, 0x0B, 0xD6, 0x09, 0xD5,
+0x6D, 0xEA, 0x91, 0xA2, 0x70, 0xA2, 0xB2, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x73, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x4F, 0x2B, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xC9, 0xF2, 0xB8, 0x9B,
+0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x40, 0x6C,
+0x62, 0x67, 0x1B, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0A, 0xF1, 0x18, 0x4A, 0x05, 0xD2,
+0x00, 0xF2, 0x1B, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0x00, 0x6A,
+0x13, 0x97, 0x12, 0x91, 0x11, 0x90, 0x00, 0xEF,
+0x0A, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF6, 0x40, 0x9A, 0x83, 0x67,
+0x00, 0x6D, 0x40, 0x6E, 0x40, 0xEA, 0x08, 0xD3,
+0x15, 0xF0, 0xA9, 0xA1, 0x15, 0xF0, 0x48, 0xA1,
+0x15, 0xF0, 0x8A, 0xA1, 0xA0, 0x35, 0x4D, 0xED,
+0x15, 0xF0, 0x4B, 0xA1, 0x08, 0x93, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x62, 0x34, 0x91, 0xC2, 0x82, 0x34,
+0x92, 0xC2, 0x00, 0xF6, 0x62, 0x34, 0x70, 0xC2,
+0x93, 0xC2, 0x10, 0x32, 0x5D, 0xE3, 0xA1, 0xA7,
+0x0A, 0xD2, 0x40, 0xA7, 0x82, 0xA7, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA7, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0xC6, 0x2A, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xC0, 0x36, 0xD2, 0xF4, 0x48, 0x9E, 0x0D, 0xD3,
+0x00, 0x6D, 0x3A, 0x65, 0x04, 0x32, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x08, 0x93, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF7, 0x18, 0x4A, 0x51, 0xE3,
+0x80, 0xAC, 0x02, 0x67, 0x59, 0x67, 0x0E, 0xD7,
+0x40, 0xEA, 0x26, 0x67, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x0C, 0x92, 0x80, 0x34, 0x49, 0xF2,
+0x84, 0x9C, 0x00, 0xF5, 0x40, 0x35, 0xD1, 0x67,
+0x8C, 0xED, 0x09, 0x94, 0x50, 0x67, 0x0D, 0x93,
+0x0E, 0x97, 0x38, 0x2C, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x29, 0xF1, 0x88, 0x9C,
+0x8D, 0xED, 0x0B, 0x94, 0x07, 0x24, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xC9, 0xF1,
+0x80, 0x9C, 0x8D, 0xED, 0x08, 0x04, 0x40, 0xF0,
+0x80, 0xA4, 0x80, 0xC7, 0x18, 0x94, 0x82, 0x34,
+0x81, 0xC7, 0x82, 0x34, 0x82, 0xC7, 0x18, 0x94,
+0x00, 0xF6, 0x82, 0x31, 0x08, 0x04, 0x40, 0xF0,
+0x84, 0xA4, 0x23, 0xC7, 0x84, 0xC7, 0x19, 0x94,
+0x82, 0x34, 0x85, 0xC7, 0x82, 0x34, 0x86, 0xC7,
+0x19, 0x94, 0x00, 0xF6, 0x82, 0x30, 0x0A, 0x94,
+0x07, 0xC7, 0x8D, 0xE3, 0x9D, 0x67, 0x20, 0xF0,
+0x84, 0xA4, 0x88, 0xC3, 0x08, 0x94, 0xD2, 0xF4,
+0x68, 0x9E, 0x89, 0xE2, 0x40, 0xEB, 0x80, 0xAA,
+0x01, 0x6A, 0x66, 0x17, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xC9, 0xF2, 0x90, 0x9C,
+0xC7, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF3, 0x50, 0x9A,
+0x3E, 0xF4, 0x04, 0x6C, 0xF7, 0xF0, 0x01, 0x68,
+0x40, 0xEA, 0x00, 0x30, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xC9, 0xF2, 0x7C, 0x9B,
+0x00, 0x30, 0xF2, 0xF4, 0x44, 0x98, 0xF7, 0xF0,
+0x01, 0x69, 0x83, 0x67, 0x20, 0x31, 0x04, 0xD3,
+0x40, 0xEA, 0x20, 0x31, 0x04, 0x93, 0xD2, 0xF4,
+0xD8, 0x99, 0xFF, 0x6D, 0x01, 0x4D, 0x83, 0x67,
+0x40, 0xEE, 0x4D, 0xED, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0xD2, 0xF4, 0x48, 0x9E,
+0x20, 0x6D, 0x10, 0xF1, 0x14, 0x6C, 0x40, 0xEA,
+0x05, 0xD6, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A, 0x10, 0xF1,
+0x10, 0x6B, 0x83, 0x67, 0x40, 0xEA, 0x04, 0xD3,
+0x05, 0x96, 0x04, 0x93, 0x21, 0x6D, 0xD2, 0xF4,
+0xC8, 0x9E, 0xAB, 0xED, 0x4C, 0xED, 0x40, 0xEE,
+0x83, 0x67, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xF2, 0xF4, 0x44, 0x98, 0xE9, 0xF2,
+0x00, 0x9B, 0x40, 0xEA, 0x90, 0x67, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x09, 0xF1,
+0xB4, 0x9C, 0xD2, 0xF4, 0x78, 0x99, 0x90, 0x67,
+0x40, 0xEB, 0x4C, 0xED, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xE9, 0xF2,
+0x20, 0x9B, 0x40, 0x32, 0xF2, 0xF4, 0x44, 0x9A,
+0x04, 0x67, 0x40, 0xEA, 0x91, 0x67, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xE9, 0xF1,
+0x68, 0x9B, 0x91, 0x67, 0x6C, 0xEA, 0x4D, 0xE8,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x78, 0x9A, 0xB0, 0x67, 0x40, 0xEB,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6B, 0x04, 0x92,
+0x60, 0x33, 0x60, 0x33, 0xE9, 0xF2, 0xA4, 0x9B,
+0xD2, 0xF4, 0x58, 0x9A, 0x91, 0x67, 0x40, 0xEA,
+0x0D, 0xED, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6F,
+0xE0, 0x37, 0xE0, 0x37, 0xD2, 0xF4, 0x54, 0x9F,
+0x04, 0x67, 0x10, 0xF1, 0x10, 0x6C, 0x06, 0xD7,
+0x40, 0xEA, 0x07, 0xD5, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x28, 0x9B,
+0x20, 0x6D, 0x10, 0xF1, 0x10, 0x6C, 0x40, 0xE9,
+0x4D, 0xED, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0xD2, 0xF4, 0x58, 0x99, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0x1A, 0x65,
+0xE9, 0xF2, 0x40, 0x9E, 0x04, 0xD6, 0x82, 0x67,
+0x05, 0xD2, 0x58, 0x67, 0x40, 0xEA, 0xB0, 0x67,
+0x07, 0x93, 0x05, 0x92, 0x01, 0x73, 0x23, 0x60,
+0x02, 0x73, 0x06, 0x97, 0x2E, 0x60, 0x04, 0x92,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xE9, 0xF2, 0x40, 0x9A, 0x09, 0xF1, 0xBC, 0x9C,
+0xD2, 0xF4, 0x78, 0x99, 0x82, 0x67, 0x0D, 0xED,
+0x40, 0xEB, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xE9, 0xF2, 0xA4, 0x9C,
+0x04, 0x92, 0xD2, 0xF4, 0x78, 0x99, 0x0D, 0xED,
+0x40, 0xEB, 0x82, 0x67, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x89, 0xF1,
+0x74, 0x9B, 0x82, 0x67, 0x6D, 0xE8, 0xD2, 0xF4,
+0x78, 0x99, 0xB0, 0x67, 0x40, 0xEB, 0x00, 0x65,
+0xD2, 0x17, 0xD2, 0xF4, 0x54, 0x9F, 0xFF, 0x6C,
+0x40, 0xEA, 0x71, 0x4C, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x00, 0xF6, 0x00, 0x6D,
+0xFF, 0x6C, 0xD2, 0xF4, 0x68, 0x9B, 0x4D, 0xED,
+0x71, 0x4C, 0xEC, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF3, 0x50, 0x9A, 0x3E, 0xF4, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x96, 0x32, 0x1F, 0x68, 0x04, 0x5A,
+0x8C, 0xE8, 0x00, 0x6C, 0x09, 0x60, 0x44, 0x34,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xE8, 0xF2, 0x1C, 0x4A, 0x49, 0xE4, 0x80, 0xAA,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x01, 0x6B, 0x64, 0xE8, 0x6C, 0xEA, 0x01, 0x5A,
+0x05, 0x97, 0x04, 0x90, 0x58, 0x67, 0x01, 0x6B,
+0x6E, 0xEA, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x96, 0x32,
+0x1F, 0x68, 0x04, 0x5A, 0x8C, 0xE8, 0x00, 0x6C,
+0x09, 0x60, 0x44, 0x34, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xE8, 0xF2, 0x14, 0x4A,
+0x49, 0xE4, 0x80, 0xAA, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6B, 0x64, 0xE8,
+0x6C, 0xEA, 0x01, 0x5A, 0x05, 0x97, 0x04, 0x90,
+0x58, 0x67, 0x01, 0x6B, 0x6E, 0xEA, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x26, 0x67, 0x04, 0x67,
+0xB1, 0x18, 0xF4, 0xFD, 0x04, 0xD5, 0x04, 0x95,
+0xF1, 0x67, 0x90, 0x67, 0xB1, 0x18, 0x88, 0xE8,
+0xC2, 0x67, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x25, 0x67, 0x04, 0x67,
+0xB1, 0x18, 0xE1, 0xFD, 0x04, 0xD6, 0x04, 0x96,
+0x90, 0x67, 0xA2, 0x67, 0xE6, 0x67, 0xB1, 0x18,
+0x88, 0xE8, 0xD1, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0x18, 0xF6, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x03, 0x6B, 0xFF, 0x6C, 0x4C, 0xEB, 0x8C, 0xEB,
+0x6B, 0xEB, 0x05, 0x97, 0xC0, 0xF7, 0x62, 0x32,
+0x8C, 0xEA, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD0, 0xB1, 0x18,
+0x1C, 0xFE, 0x04, 0x67, 0x2D, 0x2A, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0x09, 0xF1, 0xBC, 0x9B,
+0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A, 0x0D, 0xED,
+0x53, 0xF6, 0x08, 0x6C, 0x40, 0xEA, 0xFF, 0x68,
+0x2E, 0x48, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A, 0x53, 0xF6,
+0x08, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x00, 0x52,
+0x0F, 0x60, 0xFF, 0x48, 0xF2, 0x28, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0xFF, 0x6F, 0x04, 0xD0, 0x16, 0x4F,
+0x02, 0x6E, 0x40, 0x6C, 0x40, 0xEA, 0x00, 0x6D,
+0x07, 0x97, 0x06, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x59, 0x2C, 0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1,
+0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0xF7, 0xF0, 0x01, 0x68, 0x20, 0x31, 0x00, 0x30,
+0x00, 0x30, 0xD2, 0xF4, 0x54, 0x99, 0x28, 0x25,
+0x18, 0xF0, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xE9, 0xF2, 0xA8, 0x9C, 0xD2, 0xF4, 0x68, 0x98,
+0x18, 0xF0, 0x04, 0x6C, 0x40, 0xEB, 0x4D, 0xED,
+0xD2, 0xF4, 0x54, 0x99, 0x18, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xE9, 0xF2, 0xAC, 0x9C,
+0xD2, 0xF4, 0x68, 0x98, 0x18, 0xF0, 0x00, 0x6C,
+0x4D, 0xED, 0x40, 0xEB, 0x00, 0x65, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x18, 0xF0, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xE9, 0xF2, 0xB0, 0x9C, 0xD2, 0xF4, 0x68, 0x98,
+0x18, 0xF0, 0x00, 0x6C, 0x40, 0xEB, 0x4C, 0xED,
+0xD2, 0xF4, 0x54, 0x99, 0x18, 0xF0, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xE9, 0xF2, 0xB4, 0x9C,
+0xD2, 0xF4, 0x68, 0x98, 0x18, 0xF0, 0x04, 0x6C,
+0x4C, 0xED, 0xD7, 0x17, 0x20, 0xE8, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xB1, 0x18, 0x44, 0xFE,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0x20, 0xE8, 0x00, 0x6A, 0x20, 0xE8, 0x01, 0x6A,
+0xFD, 0x63, 0x05, 0x62, 0xE3, 0xA4, 0xC2, 0xA4,
+0xA1, 0xA4, 0xD1, 0x18, 0xBE, 0x70, 0x80, 0xA4,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xD1, 0x18, 0xCC, 0x70,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xB1, 0xA3, 0x50, 0xA3,
+0xD2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x53, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x61, 0xA2, 0xA2, 0xA2,
+0xC0, 0xA2, 0x43, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x88, 0x34, 0x51, 0xE4,
+0xA1, 0xA4, 0x40, 0xA4, 0x62, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA4, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x00, 0x6B, 0x20, 0xF0, 0x6C, 0xC2, 0x00, 0x6B,
+0x20, 0xF0, 0x6D, 0xC2, 0x20, 0xF0, 0x6E, 0xC2,
+0x20, 0xF0, 0x6F, 0xC2, 0x20, 0xF0, 0x70, 0xC2,
+0x20, 0xF0, 0x77, 0xC2, 0x20, 0xF0, 0x78, 0xC2,
+0x20, 0xF0, 0x71, 0xC2, 0x20, 0xF0, 0x72, 0xC2,
+0x20, 0xF0, 0x79, 0xC2, 0x20, 0xF0, 0x7A, 0xC2,
+0x20, 0xF0, 0x73, 0xC2, 0x20, 0xF0, 0x74, 0xC2,
+0x20, 0xF0, 0x7B, 0xC2, 0x20, 0xF0, 0x7C, 0xC2,
+0x20, 0xF0, 0x75, 0xC2, 0x20, 0xF0, 0x76, 0xC2,
+0x20, 0xF0, 0x7D, 0xC2, 0x20, 0xF0, 0x7E, 0xC2,
+0x20, 0xE8, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xF7, 0xF0, 0x01, 0x6A,
+0xE9, 0xF2, 0x38, 0x9B, 0x40, 0x32, 0x40, 0x32,
+0x94, 0x34, 0xF2, 0xF4, 0x44, 0x9A, 0x25, 0xE4,
+0x91, 0x67, 0x40, 0xEA, 0x05, 0x67, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x0F, 0x6D,
+0xAC, 0xE8, 0xE9, 0xF2, 0xBC, 0x9B, 0x00, 0x30,
+0x00, 0x30, 0x4C, 0xED, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x58, 0x9A,
+0x0D, 0xED, 0x40, 0xEA, 0x91, 0x67, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xD1, 0xA3, 0x50, 0xA3,
+0xF2, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x53, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x61, 0xA2, 0xC2, 0xA2,
+0xE0, 0xA2, 0x43, 0xA2, 0x60, 0x33, 0xC0, 0x36,
+0x6D, 0xEF, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x88, 0x34, 0x49, 0xE4,
+0x61, 0xA2, 0x80, 0xA2, 0xC2, 0xA2, 0x60, 0x33,
+0x8D, 0xEB, 0x83, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEB, 0x00, 0xF6, 0x80, 0x34, 0x6D, 0xEC,
+0x41, 0xA5, 0x60, 0xF0, 0x76, 0xA4, 0x02, 0x6E,
+0x01, 0x6F, 0xCB, 0xEE, 0xEC, 0xEA, 0xCC, 0xEB,
+0x4D, 0xEB, 0x60, 0xF0, 0x76, 0xC4, 0x41, 0xA5,
+0xFF, 0x4E, 0xCC, 0xEB, 0x47, 0x32, 0xEC, 0xEA,
+0x44, 0x32, 0x6D, 0xEA, 0x60, 0xF0, 0x56, 0xC4,
+0x61, 0xA5, 0xFE, 0x4E, 0xCC, 0xEA, 0x6B, 0x33,
+0xEC, 0xEB, 0x68, 0x33, 0x6D, 0xEA, 0x60, 0xF0,
+0x56, 0xC4, 0x40, 0xA5, 0x60, 0xF0, 0x57, 0xC4,
+0x42, 0xA5, 0x60, 0xF0, 0x58, 0xC4, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x01, 0x6A, 0x07, 0x24,
+0xFF, 0x6A, 0x01, 0x4D, 0x4C, 0xED, 0xA3, 0xEC,
+0x00, 0x6A, 0x01, 0x60, 0x02, 0x6A, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x02, 0x5C, 0x45, 0x67,
+0x15, 0x61, 0x04, 0x74, 0x03, 0x61, 0x15, 0x25,
+0x20, 0xE8, 0x20, 0x6A, 0x05, 0x74, 0x06, 0x61,
+0x02, 0x5D, 0x58, 0x67, 0x01, 0x5A, 0x58, 0x67,
+0x20, 0xE8, 0x58, 0x32, 0x06, 0x74, 0x08, 0x61,
+0x03, 0x5D, 0x58, 0x67, 0x01, 0x5A, 0x58, 0x67,
+0x20, 0xE8, 0x5C, 0x32, 0x20, 0xE8, 0x01, 0x6A,
+0x00, 0x6A, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x01, 0x6A, 0x07, 0x24, 0x01, 0x74, 0x07, 0x61,
+0x18, 0x6A, 0x4C, 0xED, 0x00, 0x6A, 0x01, 0x25,
+0x04, 0x6A, 0x20, 0xE8, 0x00, 0x65, 0x02, 0x74,
+0x00, 0x6A, 0xFB, 0x61, 0x10, 0x6B, 0x6C, 0xED,
+0xF6, 0x17, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xE4, 0x67, 0x8D, 0xA2, 0x6C, 0xA2, 0x0E, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x6F, 0xA2, 0x00, 0x30,
+0x00, 0x30, 0x0D, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x20, 0xF0, 0x85, 0xA3, 0x20, 0xF0,
+0x44, 0xA3, 0x20, 0xF0, 0x06, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x20, 0xF0, 0x47, 0xA3, 0x00, 0x30,
+0x00, 0x30, 0x0D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x61, 0xA2, 0x00, 0xA2, 0x82, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x03, 0xA2, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30,
+0x6D, 0xE8, 0x82, 0xA0, 0xB1, 0x18, 0xED, 0xFE,
+0xFF, 0x69, 0x81, 0xA0, 0xA6, 0x67, 0xB1, 0x18,
+0xF3, 0xFE, 0x62, 0x67, 0x83, 0xA0, 0x6D, 0xEA,
+0x2C, 0xEA, 0xA7, 0x67, 0xB1, 0x18, 0x02, 0xFF,
+0xC2, 0x67, 0xCD, 0xEA, 0x2C, 0xEA, 0x1B, 0x22,
+0x01, 0x6B, 0x4C, 0xEB, 0x29, 0x2B, 0x02, 0x6B,
+0x4C, 0xEB, 0x1A, 0x2B, 0x04, 0x6B, 0x4C, 0xEB,
+0x1A, 0x2B, 0x20, 0x6B, 0x4C, 0xEB, 0x1A, 0x2B,
+0x40, 0x6B, 0x4C, 0xEB, 0x1A, 0x2B, 0x5E, 0x32,
+0x1B, 0x22, 0x1D, 0xF2, 0x06, 0x6C, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x13, 0xF5,
+0x5C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x1D, 0xF2, 0x08, 0x6C, 0xF0, 0x17, 0x1D, 0xF2,
+0x09, 0x6C, 0xED, 0x17, 0x1D, 0xF2, 0x04, 0x6C,
+0xEA, 0x17, 0x1D, 0xF2, 0x05, 0x6C, 0xE7, 0x17,
+0x1D, 0xF2, 0x0A, 0x6C, 0xE4, 0x17, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0x41, 0xA4, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x05, 0xD2, 0x43, 0xA4, 0x01, 0x6E, 0x20, 0x31,
+0x5E, 0x33, 0x5A, 0x32, 0x0C, 0xD2, 0x42, 0xA4,
+0x15, 0xF0, 0x08, 0x49, 0x0B, 0xD3, 0x52, 0x32,
+0x07, 0xD2, 0xFF, 0x6A, 0x3A, 0x65, 0x07, 0x92,
+0x04, 0xD1, 0xCC, 0xEA, 0x08, 0xD2, 0x71, 0xA1,
+0x52, 0xA1, 0xB0, 0xA1, 0x60, 0x33, 0x40, 0x32,
+0x6D, 0xED, 0x40, 0x32, 0x4D, 0xED, 0x53, 0xA1,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0xA1, 0xA2,
+0x60, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x62, 0xA2,
+0x43, 0xA2, 0x60, 0x33, 0x60, 0x33, 0xAD, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x4D, 0xEB, 0x05, 0x92,
+0x48, 0x32, 0x69, 0xE2, 0xA1, 0xA2, 0x60, 0xA2,
+0x23, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x62, 0xA2,
+0x00, 0xF6, 0x20, 0x32, 0x60, 0x33, 0x60, 0x33,
+0xAD, 0xEB, 0x6D, 0xEA, 0xA0, 0xF2, 0x06, 0x22,
+0x22, 0x67, 0x40, 0xA4, 0x04, 0x67, 0x41, 0x6C,
+0xCC, 0xEA, 0x58, 0x33, 0x40, 0xF0, 0x5D, 0xA1,
+0x8B, 0xEC, 0x7F, 0x6D, 0x8C, 0xEA, 0x6D, 0xEA,
+0x40, 0xF0, 0x5D, 0xC1, 0x60, 0xF0, 0x64, 0xA1,
+0x8F, 0xA0, 0xAC, 0xEB, 0x9C, 0x34, 0x8D, 0xEB,
+0x0A, 0xD3, 0x7D, 0x67, 0x20, 0xF0, 0x68, 0xA3,
+0x60, 0xF0, 0x64, 0xC1, 0x00, 0x6B, 0x06, 0xD3,
+0x08, 0x93, 0xA0, 0xF0, 0x1F, 0x23, 0xA0, 0xA0,
+0x03, 0x6F, 0xA7, 0x33, 0x09, 0xD3, 0x09, 0x94,
+0x1F, 0x6B, 0x03, 0x6D, 0x8C, 0xEB, 0x20, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEA, 0x6D, 0xEA, 0x40, 0xF0,
+0x5D, 0xC1, 0x42, 0xA0, 0x60, 0xF0, 0x63, 0xA1,
+0x1B, 0x4C, 0x57, 0x32, 0xCC, 0xEA, 0x8C, 0xEB,
+0x48, 0x32, 0x6D, 0xEA, 0x60, 0xF0, 0x43, 0xC1,
+0x43, 0xA0, 0x07, 0x6B, 0xFC, 0x65, 0x4F, 0x32,
+0x6C, 0xEA, 0x60, 0xF0, 0x65, 0xA1, 0xF6, 0x4C,
+0x44, 0x32, 0x8C, 0xEB, 0x4D, 0xEB, 0x60, 0xF0,
+0x65, 0xC1, 0x62, 0xA0, 0x60, 0xF0, 0x46, 0xA1,
+0x0D, 0x4C, 0xCC, 0xEB, 0x8C, 0xEA, 0x6D, 0xEA,
+0x60, 0xF0, 0x46, 0xC1, 0x62, 0xA0, 0xAB, 0xED,
+0xAC, 0xEA, 0x67, 0x33, 0xCC, 0xEB, 0x64, 0x33,
+0x6D, 0xEA, 0x60, 0xF0, 0x46, 0xC1, 0x62, 0xA0,
+0x6B, 0x35, 0xEC, 0xED, 0x7B, 0x33, 0xF9, 0x67,
+0xEC, 0xEB, 0xEC, 0xED, 0xCC, 0xEB, 0x06, 0xD5,
+0x68, 0x35, 0x1D, 0x65, 0xBF, 0x67, 0xAC, 0xEA,
+0xB8, 0x67, 0xAD, 0xEA, 0xA2, 0xA1, 0x60, 0xF0,
+0x46, 0xC1, 0x03, 0x6F, 0x1D, 0x65, 0xB8, 0x67,
+0x8C, 0xED, 0xAD, 0xEB, 0x62, 0xC1, 0x62, 0xA0,
+0x0F, 0x6D, 0x7E, 0x33, 0x6C, 0xEE, 0x09, 0x6B,
+0x6B, 0xEB, 0xCC, 0x36, 0x6C, 0xEA, 0xCD, 0xEA,
+0x60, 0xF0, 0x46, 0xC1, 0x4F, 0xA0, 0xD8, 0x4B,
+0x57, 0x32, 0x4C, 0xEF, 0x40, 0xA1, 0xF0, 0x37,
+0x6C, 0xEA, 0xED, 0xEA, 0x40, 0xC1, 0x0A, 0x92,
+0x01, 0x6B, 0x8C, 0xEA, 0x60, 0xF0, 0x44, 0xC1,
+0x60, 0xF0, 0x54, 0xA1, 0x05, 0x94, 0x6D, 0xEA,
+0xFC, 0x4B, 0x6C, 0xEA, 0x7F, 0x67, 0x6C, 0xEA,
+0x07, 0x6B, 0x6C, 0xEA, 0x60, 0xF0, 0x54, 0xC1,
+0x09, 0x92, 0x4C, 0xED, 0x59, 0x67, 0xB1, 0x18,
+0xAD, 0xFE, 0x4C, 0xED, 0x42, 0xA0, 0xA1, 0xA0,
+0x02, 0x6E, 0x40, 0x32, 0x40, 0x32, 0xA0, 0x35,
+0x55, 0xE5, 0x40, 0xA0, 0xC0, 0xF1, 0x10, 0x6C,
+0x55, 0xE5, 0x43, 0xA0, 0x00, 0xF6, 0x40, 0x32,
+0xD1, 0x18, 0x40, 0x6A, 0x55, 0xE5, 0x46, 0xA0,
+0xA5, 0xA0, 0x02, 0x6E, 0x40, 0x32, 0x40, 0x32,
+0xA0, 0x35, 0x55, 0xE5, 0x44, 0xA0, 0xC0, 0xF1,
+0x14, 0x6C, 0x55, 0xE5, 0x47, 0xA0, 0x00, 0xF6,
+0x40, 0x32, 0xD1, 0x18, 0x40, 0x6A, 0x55, 0xE5,
+0x4A, 0xA0, 0xA9, 0xA0, 0x02, 0x6E, 0x40, 0x32,
+0x40, 0x32, 0xA0, 0x35, 0x55, 0xE5, 0x48, 0xA0,
+0xC0, 0xF1, 0x18, 0x6C, 0x55, 0xE5, 0x4B, 0xA0,
+0x00, 0xF6, 0x40, 0x32, 0xD1, 0x18, 0x40, 0x6A,
+0x55, 0xE5, 0x4E, 0xA0, 0xAD, 0xA0, 0x02, 0x6E,
+0x40, 0x32, 0x40, 0x32, 0xA0, 0x35, 0x55, 0xE5,
+0x4C, 0xA0, 0xC0, 0xF1, 0x1C, 0x6C, 0x55, 0xE5,
+0x4F, 0xA0, 0x00, 0xF6, 0x40, 0x32, 0xD1, 0x18,
+0x40, 0x6A, 0x55, 0xE5, 0x07, 0x93, 0x0C, 0x92,
+0x6D, 0xEA, 0x01, 0x6B, 0x6C, 0xEA, 0x15, 0x22,
+0x80, 0xA0, 0x40, 0xF0, 0x7C, 0xA1, 0x04, 0x6D,
+0xAB, 0xED, 0x9A, 0x34, 0xAC, 0xEB, 0x8D, 0xEB,
+0x40, 0xF0, 0x7C, 0xC1, 0x63, 0xA0, 0x07, 0x6C,
+0x93, 0x4D, 0x6C, 0xEC, 0x60, 0xF0, 0x65, 0xA1,
+0x90, 0x34, 0xAC, 0xEB, 0x8D, 0xEB, 0x60, 0xF0,
+0x65, 0xC1, 0x0B, 0x93, 0xFF, 0x6D, 0x4D, 0xEB,
+0x51, 0x23, 0x43, 0xA0, 0x07, 0x6B, 0x0F, 0x6C,
+0x4F, 0x32, 0x6C, 0xEA, 0x60, 0xF0, 0x65, 0xA1,
+0x8B, 0xEC, 0x44, 0x32, 0x8C, 0xEB, 0x6D, 0xEA,
+0x60, 0xF0, 0x45, 0xC1, 0x4D, 0xA0, 0x01, 0x6B,
+0xFE, 0x4C, 0x4F, 0x32, 0x4C, 0xEB, 0x60, 0xF0,
+0x46, 0xA1, 0x70, 0x33, 0x40, 0xF0, 0xDD, 0xA1,
+0x8C, 0xEA, 0x6D, 0xEA, 0x60, 0xF0, 0x46, 0xC1,
+0x6D, 0xA0, 0x1F, 0x6C, 0x8C, 0xEA, 0x73, 0x33,
+0x74, 0x33, 0x6D, 0xEA, 0x60, 0xF0, 0x46, 0xC1,
+0x44, 0xA0, 0xCC, 0xEC, 0xAC, 0xEC, 0x10, 0x54,
+0x3A, 0x65, 0x65, 0xA0, 0x80, 0xF1, 0x03, 0x61,
+0xA6, 0xA0, 0x60, 0x33, 0x49, 0xE3, 0xA0, 0x35,
+0xA0, 0x35, 0xA9, 0xE2, 0xA7, 0xA0, 0xC9, 0xA0,
+0xF7, 0xF0, 0x01, 0x6B, 0x00, 0xF6, 0xA0, 0x35,
+0xB5, 0xE2, 0x48, 0xA0, 0xC0, 0x36, 0x60, 0x33,
+0x59, 0xE6, 0x4A, 0xA0, 0x60, 0x33, 0x09, 0xF3,
+0x60, 0x9B, 0x40, 0x32, 0x40, 0x32, 0x49, 0xE6,
+0xCB, 0xA0, 0x00, 0xF6, 0xC0, 0x36, 0x6C, 0xEE,
+0xD9, 0xE2, 0x60, 0xF0, 0x5D, 0xA1, 0x04, 0x6B,
+0x6C, 0xEA, 0x04, 0x2A, 0x05, 0x94, 0xD1, 0x18,
+0xAF, 0x16, 0xF1, 0x67, 0x05, 0x94, 0xA7, 0x40,
+0xB1, 0x18, 0xC4, 0xFE, 0x05, 0x4D, 0x08, 0x92,
+0x20, 0xF1, 0x1E, 0x22, 0x04, 0x92, 0x71, 0xA2,
+0x90, 0xA2, 0x52, 0xA2, 0x60, 0x33, 0x6D, 0xEC,
+0x04, 0x93, 0x40, 0x32, 0x40, 0x32, 0xB3, 0xA3,
+0x8D, 0xEA, 0x00, 0xF6, 0xA0, 0x35, 0x4D, 0xED,
+0x61, 0xA5, 0x42, 0xA5, 0x80, 0xA5, 0x60, 0x33,
+0x40, 0x32, 0x6D, 0xEC, 0x40, 0x32, 0x8D, 0xEA,
+0x83, 0xA5, 0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC,
+0x80, 0xF0, 0x45, 0xA4, 0x80, 0xF0, 0x04, 0xA4,
+0x80, 0xF0, 0x66, 0xA4, 0x40, 0x32, 0x0D, 0xEA,
+0x80, 0xF0, 0x07, 0xA4, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEA, 0x00, 0xF6, 0x00, 0x30, 0xD1, 0x18,
+0x20, 0x11, 0x4D, 0xE8, 0xD1, 0x18, 0x76, 0x17,
+0x91, 0x67, 0x06, 0x92, 0x12, 0x22, 0x06, 0x93,
+0x04, 0x6A, 0x09, 0x6D, 0x6B, 0xE2, 0xA4, 0xEA,
+0x40, 0xF0, 0x5C, 0xA1, 0xFF, 0x6B, 0x03, 0x6E,
+0x05, 0x94, 0x6C, 0xED, 0x4C, 0xEE, 0x60, 0xF0,
+0xAD, 0xC1, 0xF1, 0x67, 0xD1, 0x18, 0x9D, 0x12,
+0x6C, 0xEE, 0x01, 0x6A, 0x4B, 0xEA, 0x80, 0xF0,
+0x40, 0xC1, 0x80, 0xF0, 0x41, 0xC1, 0x60, 0xF0,
+0x42, 0xA1, 0x10, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x60, 0xF0, 0x42, 0xC1, 0x60, 0xF0, 0x43, 0xA1,
+0x7F, 0x6B, 0x80, 0x6C, 0x6C, 0xEA, 0x60, 0xF0,
+0x43, 0xC1, 0x00, 0x6A, 0x60, 0xF0, 0x40, 0xC1,
+0x40, 0xF0, 0x5F, 0xC1, 0x4B, 0xA1, 0x08, 0x6B,
+0x6B, 0xEB, 0x6C, 0xEA, 0x03, 0x6B, 0x60, 0xF0,
+0x61, 0xC1, 0x6D, 0xEA, 0xA0, 0xF0, 0x76, 0xA1,
+0x4B, 0xC1, 0x8B, 0xEC, 0x00, 0x6A, 0x60, 0xF0,
+0x5B, 0xC1, 0x8D, 0xEB, 0x14, 0x6A, 0xA0, 0xF0,
+0x4E, 0xC1, 0xA0, 0xF0, 0x4F, 0xC1, 0xA0, 0xF0,
+0x76, 0xC1, 0x00, 0x6A, 0x60, 0xA1, 0xA0, 0xF0,
+0x50, 0xC1, 0xA0, 0xF0, 0x51, 0xC1, 0xA0, 0xF0,
+0x52, 0xC1, 0xA0, 0xF0, 0x53, 0xC1, 0xA0, 0xF0,
+0x54, 0xC1, 0xA0, 0xF0, 0x4A, 0xC1, 0xA0, 0xF0,
+0x4B, 0xC1, 0xA0, 0xF0, 0x4C, 0xC1, 0xA0, 0xF0,
+0x4D, 0xC1, 0x60, 0xF0, 0x51, 0xA0, 0x3F, 0x4C,
+0x8C, 0xEB, 0x60, 0xC1, 0x08, 0x6B, 0x6D, 0xEA,
+0x60, 0xF0, 0x51, 0xC0, 0x80, 0xF0, 0x50, 0xA1,
+0xEB, 0x4B, 0x6C, 0x6E, 0x6C, 0xEA, 0x80, 0xF0,
+0x50, 0xC1, 0x04, 0x92, 0x70, 0xA2, 0x91, 0xA2,
+0x52, 0xA2, 0x80, 0x34, 0x40, 0x32, 0x6D, 0xEC,
+0x40, 0x32, 0x4D, 0xEC, 0x04, 0x92, 0x73, 0xA2,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3,
+0x40, 0xA3, 0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x43, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x80, 0xF0,
+0x65, 0xA2, 0x80, 0xF0, 0x84, 0xA2, 0xF7, 0xF0,
+0x01, 0x6D, 0x60, 0x33, 0x8D, 0xEB, 0x80, 0xF0,
+0x86, 0xA2, 0x80, 0xF0, 0x47, 0xA2, 0xA0, 0x35,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6,
+0x40, 0x32, 0x6D, 0xEA, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x04, 0xD2, 0x60, 0x33, 0x73, 0xF6,
+0x5C, 0x9B, 0x05, 0xD3, 0x04, 0x93, 0xA0, 0x35,
+0x6B, 0xF1, 0x00, 0x4D, 0x60, 0xF1, 0xE5, 0x43,
+0x87, 0x67, 0x40, 0xEA, 0x06, 0xD7, 0x05, 0x93,
+0x06, 0x97, 0xF7, 0xF0, 0x01, 0x6D, 0x73, 0xF6,
+0x5C, 0x9B, 0xA0, 0x35, 0x87, 0x47, 0xA0, 0x35,
+0x6C, 0x6E, 0xCB, 0xF1, 0x0C, 0x4D, 0x40, 0xEA,
+0x65, 0x4C, 0x05, 0x93, 0xF7, 0xF0, 0x01, 0x6D,
+0xA0, 0x35, 0x73, 0xF6, 0x5C, 0x9B, 0x04, 0x93,
+0xA0, 0x35, 0x40, 0xF7, 0x8D, 0x43, 0x4B, 0xF1,
+0x14, 0x4D, 0x40, 0xEA, 0x0A, 0x6E, 0x00, 0x6C,
+0x04, 0x93, 0x84, 0x32, 0x01, 0x4C, 0x49, 0xE3,
+0x0A, 0x6D, 0x00, 0x6B, 0x6C, 0x74, 0x20, 0xF2,
+0xBD, 0xC2, 0x20, 0xF2, 0x7E, 0xC2, 0xC0, 0xF4,
+0x65, 0xC2, 0xC0, 0xF4, 0x66, 0xC2, 0x00, 0xF3,
+0xB5, 0xC2, 0x00, 0xF3, 0x76, 0xC2, 0x80, 0xF5,
+0x7D, 0xC2, 0x80, 0xF5, 0x7E, 0xC2, 0xE0, 0xF3,
+0xAD, 0xC2, 0xE0, 0xF3, 0x6E, 0xC2, 0x60, 0xF6,
+0x75, 0xC2, 0x60, 0xF6, 0x76, 0xC2, 0xE0, 0x61,
+0x01, 0x6A, 0x4B, 0xEA, 0x80, 0xF0, 0x46, 0xC1,
+0x80, 0xF0, 0x47, 0xC1, 0x40, 0xA1, 0xF7, 0x4B,
+0x6C, 0xEA, 0x40, 0xC1, 0x00, 0x6A, 0x80, 0xF0,
+0x4D, 0xC1, 0x80, 0xF0, 0x4E, 0xC1, 0x80, 0xF0,
+0x40, 0xA0, 0x01, 0x6B, 0x6C, 0xEA, 0x0D, 0x2A,
+0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18, 0xBD, 0x6B,
+0x0D, 0x6C, 0x60, 0xF0, 0x5F, 0xC0, 0x80, 0xF0,
+0x40, 0xA0, 0x01, 0x6B, 0x6D, 0xEA, 0x80, 0xF0,
+0x40, 0xC0, 0x40, 0xA1, 0x05, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x40, 0xC1, 0x05, 0x6A, 0x80, 0xF0,
+0x52, 0xC1, 0x00, 0x6A, 0x80, 0xF0, 0x53, 0xC1,
+0x40, 0xF0, 0xDC, 0xA1, 0x03, 0x6A, 0x60, 0xF0,
+0x65, 0xA1, 0x4C, 0xEE, 0x40, 0xF0, 0x5D, 0xA1,
+0x1F, 0x6C, 0xFF, 0x6F, 0x72, 0x33, 0x4C, 0xEC,
+0x07, 0x6D, 0xEC, 0xEE, 0x6C, 0xED, 0xB1, 0x18,
+0x0B, 0xFF, 0xEC, 0xEC, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x0F, 0x6E,
+0x4C, 0xEE, 0xFF, 0x6A, 0xFA, 0x65, 0x5F, 0x67,
+0x70, 0x37, 0x4C, 0xEF, 0x59, 0x67, 0x52, 0x32,
+0xAC, 0xEA, 0x4D, 0xEF, 0x08, 0x54, 0xAC, 0xEE,
+0xF0, 0x37, 0x14, 0x61, 0x86, 0xA0, 0xE0, 0xF3,
+0x10, 0x6A, 0x72, 0x33, 0x90, 0x34, 0x4C, 0xEC,
+0x6C, 0xED, 0x8D, 0xED, 0xA0, 0x35, 0xB0, 0x35,
+0xED, 0xED, 0xAD, 0xEE, 0xA8, 0xA0, 0xE7, 0xA0,
+0xA0, 0x35, 0xED, 0xED, 0x80, 0xF5, 0xA0, 0x35,
+0xCD, 0xED, 0x11, 0x10, 0x04, 0x54, 0x11, 0x61,
+0x86, 0xA0, 0x5F, 0x67, 0x72, 0x33, 0x90, 0x34,
+0x4C, 0xEC, 0x6C, 0xED, 0x8D, 0xED, 0xA0, 0x32,
+0xA7, 0xA0, 0x50, 0x32, 0x00, 0xF5, 0xA0, 0x35,
+0xCD, 0xED, 0xED, 0xED, 0x4D, 0xED, 0x00, 0x6E,
+0x64, 0x16, 0xA7, 0x67, 0xE9, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xD1, 0xA3, 0x50, 0xA3,
+0xF2, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x53, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0xC1, 0xA2, 0xE0, 0xA2,
+0x62, 0xA2, 0xC0, 0x36, 0xCD, 0xEF, 0xC3, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0xED, 0xEB, 0x00, 0xF6,
+0xC0, 0x36, 0xCD, 0xEB, 0x88, 0x32, 0x69, 0xE2,
+0xC1, 0xA2, 0x60, 0xA2, 0xE2, 0xA2, 0xC0, 0x36,
+0x6D, 0xEE, 0x63, 0xA2, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0x60, 0x33, 0xCD, 0xEB,
+0x5F, 0x23, 0x41, 0xA5, 0x5E, 0x36, 0x61, 0x2E,
+0x60, 0xF0, 0x4D, 0xC3, 0x42, 0xA5, 0x5E, 0x36,
+0x65, 0x26, 0x7F, 0x6E, 0xCC, 0xEA, 0x60, 0xF0,
+0x4F, 0xC3, 0x43, 0xA5, 0xCC, 0xEA, 0x60, 0xF0,
+0x50, 0xC3, 0xE5, 0xA5, 0x60, 0xF0, 0x59, 0xA3,
+0x02, 0x68, 0x01, 0x6E, 0x0B, 0xE8, 0xCC, 0xEF,
+0x0C, 0xEA, 0xED, 0xEA, 0x4C, 0xEE, 0xFF, 0x68,
+0x0C, 0xEE, 0x60, 0xF0, 0x59, 0xC3, 0x40, 0x26,
+0x25, 0xA5, 0xC5, 0x67, 0x3F, 0x6D, 0x1D, 0x65,
+0x06, 0xD4, 0xB8, 0x67, 0x26, 0x31, 0xAC, 0xE9,
+0x60, 0xF0, 0xBA, 0xA3, 0x80, 0x6F, 0xEB, 0xEF,
+0x0C, 0xE9, 0xEC, 0xED, 0x2D, 0xED, 0x60, 0xF0,
+0xBA, 0xC3, 0xE6, 0xA6, 0x98, 0x67, 0x8C, 0xEA,
+0xF8, 0x35, 0xAD, 0xEA, 0x60, 0xF0, 0x59, 0xC3,
+0xA6, 0xA6, 0x07, 0x6C, 0x1C, 0x65, 0x98, 0x67,
+0xAA, 0x35, 0x8C, 0xED, 0xA4, 0x34, 0x1C, 0x65,
+0x0F, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0x98, 0x67,
+0x8D, 0xEA, 0x60, 0xF0, 0x59, 0xC3, 0xC6, 0xA6,
+0x03, 0x6C, 0x1C, 0x65, 0x98, 0x67, 0xD6, 0x36,
+0x8C, 0xEE, 0xD0, 0x34, 0x3C, 0x65, 0x31, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEA, 0x99, 0x67, 0x8D, 0xEA,
+0x60, 0xF0, 0x59, 0xC3, 0x06, 0x94, 0x58, 0x67,
+0x4C, 0xEF, 0x05, 0xD3, 0x04, 0xD1, 0x0C, 0xEF,
+0x0C, 0xEE, 0xD1, 0x18, 0x61, 0x11, 0x0C, 0xED,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x7F, 0x6E, 0xCC, 0xEA, 0x60, 0xF0,
+0x4D, 0xC3, 0x47, 0xA5, 0xCC, 0xEA, 0x60, 0xF0,
+0x4E, 0xC3, 0x98, 0x17, 0x00, 0x6A, 0x60, 0xF0,
+0x4F, 0xC3, 0x9D, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x47, 0x44, 0x04, 0x67,
+0x39, 0x4A, 0x80, 0xA0, 0x27, 0xA0, 0x04, 0xD2,
+0xFF, 0x74, 0x3E, 0x31, 0x08, 0x60, 0xD1, 0x18,
+0xDA, 0x00, 0xB0, 0x67, 0x04, 0x29, 0x04, 0x92,
+0x08, 0x48, 0x4A, 0xE8, 0xF2, 0x61, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xB1, 0xA3, 0x50, 0xA3,
+0xD2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x53, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xA1, 0xA2, 0xC0, 0xA2,
+0x62, 0xA2, 0xA0, 0x35, 0xAD, 0xEE, 0xA3, 0xA2,
+0x20, 0xA4, 0x60, 0x33, 0x60, 0x33, 0xCD, 0xEB,
+0x00, 0xF6, 0xA0, 0x35, 0xAD, 0xEB, 0x28, 0x32,
+0x69, 0xE2, 0x01, 0xA2, 0xE0, 0xA2, 0x00, 0x30,
+0xED, 0xE8, 0xE2, 0xA2, 0xE0, 0x37, 0xE0, 0x37,
+0x0D, 0xEF, 0x03, 0xA2, 0x00, 0xF6, 0x00, 0x30,
+0xED, 0xE8, 0x68, 0x20, 0x62, 0xA4, 0x01, 0x6E,
+0x05, 0x6A, 0x6B, 0x33, 0xCC, 0xEB, 0x68, 0x35,
+0x60, 0xF0, 0x7D, 0xA0, 0x4B, 0xEA, 0x02, 0x6F,
+0x4C, 0xEB, 0xAD, 0xEB, 0x60, 0xF0, 0x7D, 0xC0,
+0xA1, 0xA4, 0x60, 0xF0, 0x5C, 0xA0, 0xEB, 0xEF,
+0xCC, 0xED, 0xEC, 0xEA, 0xAD, 0xEA, 0x60, 0xF0,
+0x5C, 0xC0, 0xE2, 0xA4, 0x03, 0x6D, 0x3D, 0x65,
+0xB9, 0x67, 0xAC, 0xEF, 0x04, 0x6D, 0xAB, 0xED,
+0xAC, 0xEB, 0xED, 0xEB, 0x60, 0xF0, 0x7D, 0xC0,
+0x61, 0xA4, 0xFF, 0x6D, 0xCC, 0xEA, 0x66, 0x33,
+0x64, 0x34, 0xAC, 0xEF, 0x8D, 0xEA, 0x01, 0x77,
+0x60, 0xF0, 0x5C, 0xC0, 0x58, 0x67, 0x3F, 0x2A,
+0x59, 0x67, 0x6F, 0x36, 0x4C, 0xEE, 0xAC, 0xEE,
+0x07, 0x6A, 0x4C, 0xEB, 0x6C, 0xED, 0x87, 0x67,
+0xD1, 0x18, 0x71, 0x06, 0x04, 0xD7, 0x04, 0x97,
+0x82, 0x67, 0xD0, 0x67, 0xD1, 0x18, 0x1A, 0x0A,
+0xA7, 0x67, 0x60, 0xF0, 0x5D, 0xA0, 0x04, 0x6B,
+0x6C, 0xEA, 0x24, 0x22, 0x40, 0xF0, 0xCC, 0xA0,
+0x40, 0xF0, 0xA8, 0xA0, 0x40, 0xF0, 0x8B, 0xA0,
+0x40, 0xF0, 0x47, 0xA0, 0xC0, 0x36, 0xA0, 0x35,
+0x8D, 0xEE, 0x4D, 0xED, 0x40, 0xF0, 0x8D, 0xA0,
+0x40, 0xF0, 0x49, 0xA0, 0xF0, 0x67, 0x80, 0x34,
+0x40, 0x32, 0x80, 0x34, 0x40, 0x32, 0xCD, 0xEC,
+0xAD, 0xEA, 0x40, 0xF0, 0xCE, 0xA0, 0x40, 0xF0,
+0xAA, 0xA0, 0x00, 0xF6, 0xC0, 0x36, 0x00, 0xF6,
+0xA0, 0x35, 0x8D, 0xEE, 0x4D, 0xED, 0xD1, 0x18,
+0xAF, 0x16, 0x91, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x73, 0x36,
+0xAC, 0xEE, 0x0F, 0x6A, 0xC2, 0x17, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xD1, 0xA3, 0x50, 0xA3,
+0xF2, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x53, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x61, 0xA2, 0xC2, 0xA2,
+0xE0, 0xA2, 0x43, 0xA2, 0x60, 0x33, 0xC0, 0x36,
+0x6D, 0xEF, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x88, 0x34, 0x49, 0xE4,
+0x61, 0xA2, 0x80, 0xA2, 0xC2, 0xA2, 0x60, 0x33,
+0x8D, 0xEB, 0x83, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEB, 0x00, 0xF6, 0x80, 0x34, 0x6D, 0xEC,
+0x12, 0x24, 0x60, 0xF0, 0x7B, 0xA4, 0xC1, 0xA5,
+0x80, 0x6A, 0x7F, 0x6F, 0x4B, 0xEA, 0x6C, 0xEA,
+0xEC, 0xEE, 0xCD, 0xEA, 0x60, 0xF0, 0x5B, 0xC4,
+0x61, 0xA5, 0xEC, 0xEA, 0x7E, 0x33, 0x7C, 0x33,
+0x6D, 0xEA, 0x60, 0xF0, 0x5B, 0xC4, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xA4, 0x67, 0x80, 0xA4,
+0xFF, 0x74, 0x08, 0x60, 0xFD, 0x63, 0x05, 0x62,
+0xD1, 0x18, 0x8A, 0x01, 0x00, 0x65, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xB1, 0xA2, 0x70, 0xA2,
+0xD2, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x73, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xA1, 0xA3, 0xC2, 0xA3,
+0x40, 0xA3, 0xA0, 0x35, 0xC0, 0x36, 0xAD, 0xEA,
+0xC0, 0x36, 0xA3, 0xA3, 0x4D, 0xEE, 0x40, 0xA4,
+0x00, 0xF6, 0xA0, 0x35, 0xAD, 0xEE, 0x48, 0x32,
+0xC9, 0xE2, 0xA1, 0xA2, 0x60, 0xA2, 0xC2, 0xA2,
+0xA0, 0x35, 0x6D, 0xED, 0x63, 0xA2, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0x17, 0x23, 0xA1, 0xA4, 0x7F, 0x6E,
+0xA0, 0xF0, 0x56, 0xA3, 0xCC, 0xED, 0xA0, 0xF0,
+0xAE, 0xC3, 0x81, 0xA4, 0xCC, 0xEA, 0x9E, 0x34,
+0x9C, 0x34, 0x8D, 0xEA, 0xA0, 0xF0, 0x56, 0xC3,
+0x00, 0xF6, 0x40, 0x32, 0x00, 0xF6, 0x43, 0x32,
+0x00, 0x52, 0x03, 0x61, 0x00, 0x6A, 0xA0, 0xF0,
+0x54, 0xC3, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xB1, 0xA2, 0x70, 0xA2,
+0xD2, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x73, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xA1, 0xA3, 0x40, 0xA3,
+0xC2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x80, 0xF0, 0xA5, 0xA2,
+0x80, 0xF0, 0x64, 0xA2, 0x03, 0x6E, 0xA0, 0x35,
+0x6D, 0xED, 0x80, 0xF0, 0x66, 0xA2, 0x80, 0xF0,
+0x47, 0xA2, 0xCB, 0xEE, 0x60, 0x33, 0x60, 0x33,
+0xAD, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x4D, 0xEB,
+0x40, 0xA4, 0x01, 0x6D, 0x4C, 0xED, 0x60, 0xF0,
+0x59, 0xA3, 0xA4, 0x35, 0xCC, 0xEA, 0xAD, 0xEA,
+0x02, 0x6D, 0x60, 0xF0, 0x59, 0xC3, 0xAC, 0xEA,
+0x0C, 0x22, 0xA1, 0x44, 0x50, 0xF7, 0x47, 0x43,
+0x19, 0x4C, 0x60, 0xA5, 0x01, 0x4D, 0x8A, 0xED,
+0x60, 0xC2, 0x01, 0x4A, 0xFA, 0x61, 0x20, 0xE8,
+0x00, 0x65, 0xFD, 0x63, 0x05, 0x62, 0xD1, 0x18,
+0x79, 0x75, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0x48, 0xA4, 0x01, 0x6B,
+0xFF, 0x68, 0x6C, 0xEA, 0x0C, 0xEA, 0x48, 0x22,
+0xCA, 0xA4, 0xA9, 0xA4, 0xF7, 0xF0, 0x01, 0x6A,
+0x7F, 0x6B, 0x40, 0x32, 0x40, 0x32, 0x6C, 0xEE,
+0x73, 0xF6, 0xFC, 0x9A, 0xCC, 0x36, 0xB6, 0x32,
+0x4D, 0xEE, 0x1F, 0x6A, 0x4C, 0xED, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x15, 0xF0,
+0x08, 0x4A, 0x70, 0xA2, 0xFD, 0x65, 0x27, 0x44,
+0x1B, 0x65, 0x71, 0xA2, 0xB8, 0x67, 0x05, 0x49,
+0x60, 0x33, 0x6D, 0xED, 0x72, 0xA2, 0x53, 0xA2,
+0x0C, 0xEE, 0x60, 0x33, 0x60, 0x33, 0xAD, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x4D, 0xEB, 0x40, 0xA3,
+0x3A, 0x65, 0x41, 0xA3, 0xB9, 0x67, 0x40, 0x32,
+0x4D, 0xED, 0x42, 0xA3, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x43, 0xA3, 0x68, 0xA4, 0x00, 0xF6,
+0x40, 0x32, 0x66, 0x33, 0xAD, 0xEA, 0x68, 0x33,
+0x49, 0xE3, 0x81, 0xA2, 0x60, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x62, 0xA2, 0x60, 0x33, 0x60, 0x33,
+0x8D, 0xEB, 0x83, 0xA2, 0x5F, 0x67, 0x55, 0xE1,
+0x00, 0xF6, 0x80, 0x34, 0x40, 0xEF, 0x6D, 0xEC,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x01, 0x6A,
+0x00, 0xEF, 0x04, 0x63, 0xFE, 0x63, 0x03, 0xD1,
+0x02, 0xD0, 0x45, 0xA4, 0xA4, 0xA4, 0x3F, 0x6B,
+0x6C, 0xEA, 0x40, 0x32, 0xAD, 0xEA, 0xF8, 0x4A,
+0x0C, 0x6B, 0x7A, 0xEA, 0x01, 0x2B, 0xE5, 0xE8,
+0xFF, 0xF7, 0x1F, 0x6A, 0x08, 0x4C, 0x00, 0x6F,
+0x12, 0xE8, 0x4C, 0xE8, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x00, 0xD0, 0x15, 0xF0,
+0x08, 0x4A, 0x71, 0xA2, 0xD0, 0xA2, 0xB2, 0xA2,
+0x60, 0x33, 0xCD, 0xEB, 0xD3, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6, 0xC0, 0x36,
+0x6D, 0xEE, 0x00, 0x93, 0xFF, 0xF7, 0x1F, 0x6A,
+0xEC, 0xEA, 0x63, 0xEA, 0x04, 0x61, 0x03, 0x91,
+0x02, 0x90, 0x20, 0xE8, 0x02, 0x63, 0xA1, 0xA4,
+0x01, 0x6A, 0x1A, 0x65, 0x58, 0x67, 0xAA, 0x31,
+0x4C, 0xE9, 0x41, 0xA6, 0x60, 0xA6, 0x40, 0x32,
+0x6D, 0xEA, 0x3A, 0x65, 0x42, 0xA6, 0x79, 0x67,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEB, 0x3B, 0x65,
+0x63, 0xA6, 0x59, 0x67, 0x00, 0xF6, 0x60, 0x33,
+0x4D, 0xEB, 0x40, 0xA4, 0x48, 0x32, 0x6D, 0xE2,
+0x40, 0xA3, 0x3A, 0x65, 0x41, 0xA3, 0x19, 0x67,
+0x40, 0x32, 0x0D, 0xEA, 0x02, 0xA3, 0x63, 0xA3,
+0x00, 0x30, 0x00, 0x30, 0x0D, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x6D, 0xEA, 0x34, 0x22, 0x78, 0x67,
+0xAC, 0xEB, 0x16, 0x23, 0x64, 0xA4, 0x3B, 0x65,
+0x65, 0xA4, 0x19, 0x67, 0x60, 0x33, 0x6D, 0xE8,
+0x66, 0xA4, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xE8,
+0x67, 0xA4, 0x00, 0xF6, 0x60, 0x33, 0x0D, 0xEB,
+0x62, 0x30, 0x64, 0xC2, 0x05, 0xC2, 0x00, 0xF6,
+0x62, 0x33, 0x02, 0x30, 0x06, 0xC2, 0x67, 0xC2,
+0x02, 0x6B, 0x6C, 0xED, 0x18, 0x25, 0xA9, 0xA4,
+0x68, 0xA4, 0xA0, 0x35, 0x6D, 0xED, 0x1D, 0x65,
+0xAA, 0xA4, 0x78, 0x67, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEB, 0x1B, 0x65, 0x6B, 0xA4, 0xB8, 0x67,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0x62, 0x35,
+0x70, 0xC2, 0xB1, 0xC2, 0x00, 0xF6, 0x62, 0x33,
+0xA2, 0x35, 0xB2, 0xC2, 0x73, 0xC2, 0x9F, 0x29,
+0x01, 0x4F, 0x09, 0x77, 0x9C, 0x60, 0x0C, 0x4C,
+0x94, 0x17, 0x00, 0x65, 0xFE, 0x63, 0x03, 0xD1,
+0x02, 0xD0, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B, 0xB1, 0xA3,
+0x50, 0xA3, 0xD2, 0xA3, 0xA0, 0x35, 0x4D, 0xED,
+0x53, 0xA3, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0xA1, 0xA2,
+0xC2, 0xA2, 0x60, 0xA2, 0xA0, 0x35, 0xC0, 0x36,
+0xAD, 0xEB, 0xC0, 0x36, 0xA3, 0xA2, 0x6D, 0xEE,
+0x68, 0xA4, 0x00, 0xF6, 0xA0, 0x35, 0xAD, 0xEE,
+0x68, 0x33, 0xCD, 0xE3, 0xA1, 0xA3, 0x40, 0xA3,
+0xC2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xAD, 0xA4, 0x01, 0x68,
+0x7D, 0x67, 0x0C, 0xED, 0x00, 0xD5, 0xA0, 0xA3,
+0xE1, 0xA2, 0x08, 0x2D, 0x02, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEF, 0xE1, 0xC2, 0x03, 0x91, 0x02, 0x90,
+0x20, 0xE8, 0x02, 0x63, 0x01, 0x6B, 0xAC, 0xA4,
+0x6D, 0xEF, 0x1B, 0x65, 0x60, 0xA2, 0x02, 0x6E,
+0xCB, 0xEE, 0x0C, 0xED, 0xCC, 0xEB, 0xAD, 0xEB,
+0xAC, 0xA4, 0xA6, 0x35, 0x0C, 0xED, 0xA4, 0x35,
+0x3D, 0x65, 0x03, 0x6D, 0xAB, 0xED, 0x39, 0x67,
+0xAC, 0xEB, 0x2D, 0xEB, 0x60, 0xC2, 0x6D, 0xA4,
+0x3B, 0x65, 0x39, 0x67, 0x38, 0x6B, 0x2C, 0xEB,
+0x39, 0x69, 0x2B, 0xE9, 0x2C, 0xEF, 0xED, 0xEB,
+0x3F, 0x69, 0xED, 0xA4, 0x39, 0x65, 0x39, 0x67,
+0x2C, 0xEB, 0xFA, 0x37, 0x22, 0xA2, 0xF8, 0x37,
+0xED, 0xEB, 0xEE, 0xA4, 0x39, 0x65, 0x39, 0x67,
+0x0C, 0xEF, 0xCC, 0xE9, 0xED, 0xE9, 0xEE, 0xA4,
+0x2C, 0xED, 0x05, 0x69, 0xE6, 0x37, 0x0C, 0xEF,
+0xE4, 0x37, 0xED, 0xED, 0xEE, 0xA4, 0x2B, 0xE9,
+0x2C, 0xED, 0xEA, 0x37, 0x0C, 0xEF, 0xE8, 0x37,
+0xAD, 0xEF, 0xAE, 0xA4, 0xE2, 0xC2, 0xE8, 0xA2,
+0xAE, 0x35, 0x0C, 0xED, 0xEC, 0xEE, 0xAD, 0xEE,
+0xA9, 0xA4, 0x61, 0xC2, 0xC8, 0xC2, 0xAC, 0xE8,
+0x00, 0xD0, 0xBD, 0x67, 0x00, 0xA5, 0x1D, 0x20,
+0xAA, 0xA4, 0xE9, 0xA4, 0x03, 0x68, 0x0C, 0xED,
+0xE6, 0x37, 0xBC, 0x35, 0xED, 0xED, 0xF8, 0x67,
+0xAC, 0xEF, 0xA6, 0x35, 0x00, 0xD5, 0xBD, 0x67,
+0xA0, 0xA5, 0x7F, 0x68, 0xFC, 0x37, 0xA9, 0xC2,
+0xAA, 0xA4, 0x0C, 0xEE, 0xED, 0xEE, 0xC8, 0xC2,
+0xAA, 0x35, 0x03, 0x6E, 0xCC, 0xED, 0x07, 0x6E,
+0xCB, 0xEE, 0xA4, 0x35, 0xCC, 0xEB, 0x6D, 0xED,
+0xA1, 0xC2, 0x6A, 0xA4, 0x10, 0x6D, 0xAC, 0xEB,
+0xFF, 0x6D, 0xAC, 0xEB, 0x8F, 0x23, 0x6B, 0xA4,
+0xAA, 0xA4, 0x07, 0x6C, 0x8C, 0xEB, 0xB6, 0x35,
+0x8A, 0xA2, 0x6C, 0x33, 0xAD, 0xEB, 0x40, 0x6D,
+0xAB, 0xED, 0xAC, 0xEC, 0x8D, 0xEB, 0x6A, 0xC2,
+0x81, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0xD1,
+0x04, 0xD0, 0x45, 0xA4, 0xA4, 0xA4, 0x3F, 0x6B,
+0x6C, 0xEA, 0x40, 0x32, 0xAD, 0xEA, 0xF8, 0x4A,
+0x08, 0x6B, 0x7A, 0xEA, 0x01, 0x2B, 0xE5, 0xE8,
+0xFF, 0xF7, 0x1F, 0x6A, 0x08, 0x4C, 0x00, 0x69,
+0x12, 0xEB, 0x4C, 0xEB, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x00, 0xD3, 0x15, 0xF0,
+0x08, 0x4A, 0x71, 0xA2, 0xD0, 0xA2, 0xB2, 0xA2,
+0x60, 0x33, 0xCD, 0xEB, 0xD3, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6, 0xC0, 0x36,
+0x6D, 0xEE, 0x01, 0xD1, 0x01, 0x02, 0x40, 0xAA,
+0x00, 0x93, 0x63, 0xEA, 0x04, 0x61, 0x05, 0x91,
+0x04, 0x90, 0x20, 0xE8, 0x03, 0x63, 0x41, 0xA4,
+0x01, 0x68, 0x61, 0xA6, 0x0C, 0xEA, 0x01, 0xD2,
+0x01, 0x02, 0x40, 0xA2, 0xA0, 0xA6, 0x60, 0x33,
+0x01, 0xD2, 0x42, 0xA6, 0x6D, 0xED, 0x60, 0xA4,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xED, 0x43, 0xA6,
+0x68, 0x33, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x49, 0xE3, 0x61, 0xA2, 0xA0, 0xA2, 0x60, 0x33,
+0xAD, 0xEB, 0xA2, 0xA2, 0x43, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0x4D, 0xED, 0x62, 0x25, 0x44, 0xA4, 0xEC, 0xA5,
+0x02, 0x6B, 0x6B, 0xEB, 0x0C, 0xEA, 0x6C, 0xEF,
+0x4D, 0xEF, 0xEC, 0xC5, 0x44, 0xA4, 0x3B, 0x65,
+0x06, 0x6B, 0x6C, 0xEA, 0xF3, 0x4B, 0xEC, 0xEB,
+0x4D, 0xEB, 0x6C, 0xC5, 0x44, 0xA4, 0x07, 0x6F,
+0x1F, 0x65, 0xF8, 0x67, 0x4E, 0x32, 0xEC, 0xEA,
+0x4C, 0x37, 0x39, 0x6A, 0x4B, 0xEA, 0x6C, 0xEA,
+0xED, 0xEA, 0x4C, 0xC5, 0x64, 0xA4, 0x3F, 0x6F,
+0xEC, 0xEA, 0x7A, 0x33, 0x78, 0x33, 0x6D, 0xEA,
+0x4C, 0xC5, 0x65, 0xA4, 0x4D, 0xA5, 0xF9, 0x67,
+0x0C, 0xEB, 0xEC, 0xEA, 0x4D, 0xEB, 0x6D, 0xC5,
+0xE5, 0xA4, 0x58, 0x67, 0xE6, 0x37, 0x4C, 0xEF,
+0x0F, 0x6A, 0x4B, 0xEA, 0xE4, 0x37, 0x4C, 0xEB,
+0xED, 0xEB, 0x6D, 0xC5, 0xE5, 0xA4, 0x0F, 0x6A,
+0x4C, 0xEB, 0xF2, 0x37, 0xF0, 0x37, 0xED, 0xEB,
+0x6D, 0xC5, 0x66, 0xA4, 0x4E, 0xA5, 0x21, 0x6F,
+0x76, 0x33, 0x0C, 0xEB, 0xEB, 0xEF, 0xEC, 0xEA,
+0x74, 0x33, 0x4D, 0xEB, 0x6E, 0xC5, 0x46, 0xA4,
+0x5A, 0x32, 0x4C, 0xE8, 0x41, 0x6A, 0x4B, 0xEA,
+0x18, 0x30, 0x4C, 0xEB, 0x0D, 0xEB, 0x6E, 0xC5,
+0xE6, 0xA4, 0x47, 0xA4, 0x7F, 0x68, 0xFE, 0x37,
+0x44, 0x32, 0xED, 0xEA, 0x02, 0xD2, 0x02, 0x02,
+0x40, 0xAA, 0x01, 0x6F, 0x0C, 0xEB, 0x4C, 0xEF,
+0xFC, 0x37, 0x46, 0x32, 0x6D, 0xEF, 0xEE, 0xC5,
+0x02, 0xD2, 0x02, 0x02, 0x40, 0xA2, 0x4F, 0xC5,
+0x01, 0x92, 0x7F, 0xF7, 0x14, 0x2A, 0x01, 0x49,
+0x09, 0x71, 0x7F, 0xF7, 0x10, 0x60, 0x08, 0x4C,
+0x68, 0x17, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xB1, 0xA2, 0x70, 0xA2, 0xD2, 0xA2, 0xA0, 0x35,
+0x6D, 0xED, 0x73, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0xA1, 0xA3, 0xC2, 0xA3, 0x40, 0xA3, 0xA0, 0x35,
+0xC0, 0x36, 0xAD, 0xEA, 0xC0, 0x36, 0xA3, 0xA3,
+0x4D, 0xEE, 0x48, 0xA4, 0x00, 0xF6, 0xA0, 0x35,
+0xAD, 0xEE, 0x48, 0x32, 0xC9, 0xE2, 0x61, 0xA2,
+0xA2, 0xA2, 0xC0, 0xA2, 0x43, 0xA2, 0x60, 0x33,
+0xA0, 0x35, 0x6D, 0xEE, 0xA0, 0x35, 0x69, 0xA4,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xE8, 0x63, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xB1, 0xA2, 0x70, 0xA2, 0xD2, 0xA2, 0xA0, 0x35,
+0x6D, 0xED, 0x73, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0xA1, 0xA3, 0x40, 0xA3, 0xC2, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x80, 0xF0, 0x65, 0xA2, 0x80, 0xF0, 0xA6, 0xA2,
+0x80, 0xF0, 0xC4, 0xA2, 0x80, 0xF0, 0x47, 0xA2,
+0x60, 0x33, 0xA0, 0x35, 0x6D, 0xEE, 0xA0, 0x35,
+0x68, 0xA4, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x60, 0xF0, 0x70, 0xC2, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xB1, 0xA2, 0x70, 0xA2, 0xD2, 0xA2, 0xA0, 0x35,
+0x6D, 0xED, 0x73, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0xA1, 0xA3, 0x40, 0xA3, 0xC2, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x80, 0xF0, 0xA5, 0xA2, 0x80, 0xF0, 0x64, 0xA2,
+0x80, 0xF0, 0xC6, 0xA2, 0xA0, 0x35, 0x6D, 0xED,
+0x80, 0xF0, 0x67, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x00, 0x6D, 0xA4, 0x32, 0x59, 0xE3, 0x49, 0xE4,
+0xE8, 0xA2, 0x49, 0xA2, 0x01, 0x4D, 0x20, 0x75,
+0xE0, 0xC6, 0x41, 0xC6, 0xF6, 0x61, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0xB1, 0xA3, 0x50, 0xA3, 0xD2, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x53, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0xA1, 0xA2, 0x60, 0xA2, 0xC2, 0xA2, 0xA0, 0x35,
+0x6D, 0xED, 0x63, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x80, 0xF0, 0xA5, 0xA3, 0x80, 0xF0, 0x44, 0xA3,
+0x80, 0xF0, 0xC6, 0xA3, 0xA0, 0x35, 0x4D, 0xED,
+0x80, 0xF0, 0x47, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0xC0, 0xA4, 0x60, 0xF0, 0xFB, 0xA2, 0x60, 0xF0,
+0x7A, 0xA2, 0x01, 0x6D, 0xA4, 0xEE, 0x60, 0xF0,
+0xDC, 0xA2, 0xE0, 0x37, 0x6D, 0xEF, 0x60, 0xF0,
+0x7D, 0xA2, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEF,
+0x00, 0xF6, 0x60, 0x33, 0xED, 0xEB, 0xAE, 0xEB,
+0x62, 0x35, 0x60, 0xF0, 0x7A, 0xC2, 0x60, 0xF0,
+0xBB, 0xC2, 0x00, 0xF6, 0x62, 0x33, 0xA2, 0x35,
+0x60, 0xF0, 0xBC, 0xC2, 0x60, 0xF0, 0x7D, 0xC2,
+0x64, 0xA4, 0x60, 0xF0, 0x7E, 0xC2, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x44, 0x67, 0x06, 0x72,
+0x86, 0x67, 0x12, 0x60, 0x07, 0x5A, 0x04, 0x60,
+0x0B, 0x22, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0x09, 0x72, 0x0E, 0x60, 0x0D, 0x72, 0xF9, 0x61,
+0xD1, 0x18, 0xCC, 0x70, 0x00, 0x65, 0xF5, 0x17,
+0xD1, 0x18, 0x83, 0x03, 0x00, 0x65, 0xF1, 0x17,
+0xD1, 0x18, 0x38, 0x05, 0x00, 0x65, 0xED, 0x17,
+0xB1, 0x18, 0x78, 0xFE, 0x00, 0x65, 0xE9, 0x17,
+0x20, 0xE8, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x00, 0x6A, 0x7D, 0x67,
+0x06, 0xD2, 0x02, 0x6A, 0x52, 0xC3, 0x5D, 0x67,
+0x91, 0xC2, 0xB0, 0xC2, 0xD3, 0xC2, 0xC2, 0x32,
+0x54, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A, 0x08, 0x46,
+0x27, 0x67, 0x06, 0x06, 0x00, 0x6F, 0xB0, 0x67,
+0x40, 0xEA, 0x00, 0x6C, 0x0F, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF7, 0xF0,
+0x01, 0x6C, 0x13, 0xF1, 0x50, 0x9A, 0x80, 0x34,
+0x80, 0x34, 0x2A, 0xF1, 0x10, 0x4C, 0x40, 0xEA,
+0x00, 0x65, 0x10, 0x10, 0x06, 0x94, 0xEF, 0x24,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0xB1, 0x67, 0x40, 0xEA,
+0xD0, 0x67, 0x06, 0x95, 0xD1, 0x18, 0xA8, 0x19,
+0x04, 0x04, 0xE1, 0x2A, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x91, 0xA3, 0x50, 0xA3, 0xB2, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x53, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x81, 0xA2, 0x60, 0xA2, 0xA2, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x63, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x80, 0xF0, 0x85, 0xA3, 0x80, 0xF0, 0x44, 0xA3,
+0x80, 0xF0, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x80, 0xF0, 0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x60, 0xF0, 0x77, 0xA2, 0x60, 0xF0, 0x96, 0xA2,
+0x04, 0x07, 0x60, 0x33, 0x8D, 0xEB, 0x9D, 0x67,
+0x68, 0xCC, 0x00, 0x6B, 0x60, 0xF0, 0x76, 0xC2,
+0x60, 0xF0, 0x77, 0xC2, 0x0C, 0x6E, 0x03, 0x6D,
+0xD1, 0x18, 0xBF, 0x03, 0x02, 0x6C, 0x07, 0x97,
+0x00, 0xEF, 0x04, 0x63, 0xF2, 0x63, 0x1B, 0x62,
+0x1A, 0xD1, 0x19, 0xD0, 0x60, 0xA4, 0x5D, 0x67,
+0x00, 0x68, 0x40, 0xF0, 0x70, 0xC2, 0x62, 0xA4,
+0x40, 0xF0, 0x71, 0xC2, 0x05, 0x02, 0x16, 0xD2,
+0x5D, 0x67, 0x09, 0xE2, 0x40, 0xF0, 0x70, 0xA2,
+0x16, 0x92, 0x20, 0x5B, 0x83, 0x67, 0xFF, 0xF7,
+0x7C, 0xCA, 0x5A, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xD1, 0xA2, 0xB0, 0xA2, 0x32, 0xA2, 0xC0, 0x36,
+0xAD, 0xEE, 0xB3, 0xA2, 0x20, 0x31, 0x20, 0x31,
+0x2D, 0xEE, 0x00, 0xF6, 0xA0, 0x35, 0xCD, 0xED,
+0xC1, 0xA5, 0x40, 0xA5, 0x22, 0xA5, 0xC0, 0x36,
+0x4D, 0xEE, 0x43, 0xA5, 0x20, 0x31, 0x20, 0x31,
+0x2D, 0xEE, 0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA,
+0x68, 0x33, 0x49, 0xE3, 0xA1, 0xA2, 0x62, 0xA2,
+0xC0, 0xA2, 0x43, 0xA2, 0xA0, 0x35, 0x60, 0x33,
+0xAD, 0xEE, 0x60, 0x33, 0xCD, 0xEB, 0x00, 0xF6,
+0x40, 0x32, 0x6D, 0xEA, 0x2D, 0x22, 0x16, 0x95,
+0x20, 0xF0, 0x6C, 0xA2, 0xC5, 0x67, 0xFF, 0xF7,
+0x7E, 0xC5, 0x00, 0x6D, 0xA4, 0x31, 0x25, 0xE2,
+0x20, 0xF0, 0x6F, 0xA1, 0x01, 0x4D, 0x02, 0x4E,
+0x1B, 0x65, 0x20, 0xF0, 0x70, 0xA1, 0xF8, 0x67,
+0x60, 0x33, 0xED, 0xEB, 0xFF, 0xF7, 0x7E, 0xCE,
+0x20, 0xF0, 0x77, 0xA1, 0x1B, 0x65, 0x20, 0xF0,
+0x78, 0xA1, 0xF8, 0x67, 0x04, 0x75, 0x60, 0x33,
+0xED, 0xEB, 0x63, 0xCE, 0xE7, 0x61, 0x20, 0xF0,
+0xAD, 0xA2, 0x20, 0xF0, 0x4E, 0xA2, 0xFF, 0xF7,
+0x1F, 0x6B, 0x40, 0x32, 0xAD, 0xEA, 0x6C, 0xEA,
+0x16, 0x93, 0xB1, 0x18, 0x82, 0xFE, 0x4A, 0xCB,
+0x16, 0x92, 0x01, 0x48, 0x02, 0x70, 0x20, 0x4A,
+0x16, 0xD2, 0x96, 0x61, 0x04, 0x07, 0x48, 0x6E,
+0x02, 0x6D, 0xD1, 0x18, 0xBF, 0x03, 0x01, 0x6C,
+0x1B, 0x97, 0x1A, 0x91, 0x19, 0x90, 0x00, 0xEF,
+0x0E, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x44, 0x67, 0x03, 0x72, 0x86, 0x67, 0x1B, 0x60,
+0x04, 0x5A, 0x08, 0x60, 0x01, 0x72, 0x13, 0x60,
+0x02, 0x5A, 0x0D, 0x61, 0xD1, 0x18, 0x03, 0x04,
+0x00, 0x65, 0x06, 0x10, 0x05, 0x72, 0x17, 0x60,
+0x05, 0x5A, 0x11, 0x61, 0x07, 0x72, 0x17, 0x60,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0xB1, 0x18,
+0x46, 0xFF, 0x00, 0x65, 0xF9, 0x17, 0xD1, 0x18,
+0x2B, 0x01, 0x00, 0x65, 0xF5, 0x17, 0xD1, 0x18,
+0xAB, 0x01, 0x00, 0x65, 0xF1, 0x17, 0xD1, 0x18,
+0x38, 0x01, 0x00, 0x65, 0xED, 0x17, 0xD1, 0x18,
+0xB2, 0x01, 0x00, 0x65, 0xE9, 0x17, 0xD1, 0x18,
+0xD6, 0x01, 0x00, 0x65, 0xE5, 0x17, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xA9, 0xA3, 0x48, 0xA3,
+0xCA, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x71, 0xA2, 0xB2, 0xA2,
+0xD0, 0xA2, 0x53, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x61, 0xA2, 0x05, 0x6A,
+0x22, 0x2B, 0xFD, 0x63, 0x05, 0x62, 0x44, 0x67,
+0x60, 0xA2, 0xC7, 0x42, 0x81, 0xA4, 0x6A, 0x33,
+0x01, 0x73, 0x01, 0x4E, 0x06, 0x60, 0x02, 0x73,
+0x0D, 0x60, 0x05, 0x97, 0x00, 0x6A, 0x00, 0xEF,
+0x03, 0x63, 0xA4, 0xA2, 0x45, 0xA2, 0x3F, 0x6B,
+0x6C, 0xEA, 0x40, 0x32, 0xD1, 0x18, 0x45, 0x04,
+0x4D, 0xED, 0xF3, 0x17, 0xA4, 0xA2, 0x45, 0xA2,
+0x3F, 0x6B, 0x6C, 0xEA, 0x40, 0x32, 0xD1, 0x18,
+0xAE, 0x03, 0x4D, 0xED, 0xEA, 0x17, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFF, 0x63, 0x01, 0xD1,
+0x00, 0xD0, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B, 0x31, 0xA3,
+0x5D, 0x67, 0x0C, 0xAA, 0x50, 0xA3, 0x20, 0x31,
+0xA4, 0x35, 0x4D, 0xE9, 0x19, 0x65, 0x32, 0xA3,
+0x58, 0x67, 0x91, 0xE5, 0x20, 0x31, 0x20, 0x31,
+0x4D, 0xE9, 0x53, 0xA3, 0x84, 0x34, 0x00, 0xF6,
+0x40, 0x32, 0x2D, 0xEA, 0x21, 0xA2, 0x60, 0xA2,
+0x20, 0x31, 0x6D, 0xE9, 0x19, 0x65, 0x22, 0xA2,
+0x78, 0x67, 0x20, 0x31, 0x20, 0x31, 0x6D, 0xE9,
+0x63, 0xA2, 0x00, 0xF6, 0x60, 0x33, 0x2D, 0xEB,
+0x80, 0xF0, 0x39, 0xA3, 0x80, 0xF0, 0x58, 0xA3,
+0x20, 0x31, 0x4D, 0xE9, 0x19, 0x65, 0x80, 0xF0,
+0x3A, 0xA3, 0x58, 0x67, 0x20, 0x31, 0x20, 0x31,
+0x4D, 0xE9, 0x80, 0xF0, 0x5B, 0xA3, 0x00, 0xF6,
+0x40, 0x32, 0x2D, 0xEA, 0x91, 0xE2, 0x00, 0xF1,
+0x04, 0xC4, 0x02, 0x30, 0x00, 0xF1, 0x05, 0xC4,
+0x01, 0x91, 0x00, 0x90, 0xC8, 0xC4, 0x00, 0xF2,
+0xE0, 0xC4, 0xC2, 0x36, 0xE2, 0x37, 0xC9, 0xC4,
+0x00, 0xF2, 0xE1, 0xC4, 0x20, 0xE8, 0x01, 0x63,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xD1, 0xA3, 0x50, 0xA3,
+0xF2, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x53, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0xC1, 0xA2, 0x60, 0xA2,
+0xE2, 0xA2, 0xC0, 0x36, 0x6D, 0xEE, 0x63, 0xA2,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x60, 0x33, 0xCD, 0xEB, 0x80, 0xF0, 0xD9, 0xA3,
+0x80, 0xF0, 0x58, 0xA3, 0x80, 0xF0, 0xFA, 0xA3,
+0xC0, 0x36, 0x4D, 0xEE, 0x80, 0xF0, 0x5B, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0x84, 0x34, 0xED, 0xEE,
+0x00, 0xF6, 0x40, 0x32, 0xB5, 0xE4, 0xCD, 0xEA,
+0xA4, 0x35, 0xB5, 0xE2, 0x01, 0x6A, 0x4B, 0xEA,
+0x48, 0xC5, 0x49, 0xC5, 0x00, 0xF1, 0x44, 0xC5,
+0x00, 0xF1, 0x45, 0xC5, 0x00, 0xF2, 0x40, 0xC5,
+0x00, 0xF2, 0x41, 0xC5, 0x20, 0xE8, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x00, 0x68,
+0x3C, 0x65, 0x90, 0x67, 0xD1, 0x18, 0xA8, 0x04,
+0xB9, 0x67, 0x01, 0x48, 0xFF, 0x6A, 0x4C, 0xE8,
+0x3F, 0x70, 0xF7, 0x61, 0x05, 0x97, 0x04, 0x90,
+0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xD1, 0xA2, 0x70, 0xA2, 0xF2, 0xA2, 0xC0, 0x36,
+0x6D, 0xEE, 0x73, 0xA2, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0x60, 0x33, 0xCD, 0xEB,
+0xC1, 0xA3, 0x40, 0xA3, 0xE2, 0xA3, 0xC0, 0x36,
+0x4D, 0xEE, 0x43, 0xA3, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA,
+0x80, 0xF0, 0x79, 0xA2, 0x80, 0xF0, 0xDA, 0xA2,
+0x80, 0xF0, 0xF8, 0xA2, 0x80, 0xF0, 0x5B, 0xA2,
+0x60, 0x33, 0xC0, 0x36, 0x6D, 0xEF, 0xC0, 0x36,
+0xED, 0xEE, 0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA,
+0xA4, 0x35, 0x49, 0xE5, 0x84, 0xC2, 0x82, 0x34,
+0x20, 0xE8, 0x85, 0xC2, 0xA0, 0x35, 0xB1, 0xE4,
+0xFF, 0xF7, 0x1F, 0x6A, 0x20, 0xE8, 0x8C, 0xEA,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x91, 0xA3, 0x50, 0xA3, 0xB2, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x53, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x81, 0xA2, 0x60, 0xA2, 0xA2, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x63, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x80, 0xF0, 0x99, 0xA3, 0x80, 0xF0, 0x58, 0xA3,
+0x80, 0xF0, 0xBA, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x80, 0xF0, 0x5B, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x62, 0xA2, 0x3E, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB,
+0x62, 0xC2, 0x00, 0x6B, 0x60, 0xC2, 0x61, 0xC2,
+0xD1, 0x18, 0xC8, 0x04, 0x00, 0x6C, 0xD1, 0x18,
+0xC8, 0x04, 0x01, 0x6C, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0x91, 0xA2, 0x70, 0xA2,
+0xD2, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x73, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xC2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x80, 0xF0, 0x99, 0xA2,
+0x80, 0xF0, 0x78, 0xA2, 0x80, 0xF0, 0xDA, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x80, 0xF0, 0x7B, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x42, 0xA3, 0x01, 0x6C,
+0x02, 0x6E, 0x4C, 0xEC, 0x5D, 0x67, 0x50, 0xA2,
+0xCB, 0xEE, 0x04, 0x07, 0xCC, 0xEA, 0x8D, 0xEA,
+0x82, 0xA3, 0x0F, 0x6E, 0x63, 0xA3, 0x8A, 0x34,
+0xCC, 0xEC, 0x1F, 0x6E, 0xCB, 0xEE, 0x84, 0x34,
+0xCC, 0xEA, 0x8D, 0xEA, 0x9D, 0x67, 0x50, 0xC4,
+0x5D, 0x67, 0xB2, 0xC2, 0x0C, 0x6E, 0x04, 0x6D,
+0x02, 0x6C, 0xD1, 0x18, 0xBF, 0x03, 0x71, 0xC2,
+0x07, 0x97, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36,
+0x15, 0xF0, 0x08, 0x4E, 0xE4, 0x67, 0x91, 0xA6,
+0x50, 0xA6, 0x72, 0xA6, 0x80, 0x34, 0x4D, 0xEC,
+0x53, 0xA6, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x81, 0xA2,
+0x60, 0xA2, 0xA2, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x63, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x80, 0xF0,
+0x99, 0xA3, 0x80, 0xF0, 0x58, 0xA3, 0x80, 0xF0,
+0xBA, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x80, 0xF0,
+0x5B, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x81, 0xA7,
+0x01, 0x69, 0x62, 0xA2, 0x2C, 0xEC, 0x3C, 0x65,
+0x02, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB, 0x99, 0x67,
+0x8D, 0xEB, 0x81, 0xA7, 0x62, 0xC2, 0x03, 0x6B,
+0x86, 0x30, 0x6C, 0xE8, 0x8E, 0x34, 0x2C, 0xEC,
+0x15, 0xE2, 0x80, 0xC5, 0x80, 0xA7, 0xFF, 0x69,
+0x04, 0xD4, 0x9D, 0x67, 0x90, 0xA4, 0x83, 0xC2,
+0x82, 0xA2, 0x8C, 0xEB, 0x2C, 0xEB, 0x02, 0x73,
+0x07, 0x61, 0x03, 0x69, 0x79, 0x67, 0x2B, 0xE9,
+0x64, 0x33, 0x2C, 0xEC, 0x6D, 0xEC, 0x82, 0xC2,
+0x62, 0xA2, 0x01, 0x6C, 0x6C, 0xEC, 0x0F, 0x2C,
+0xC3, 0x4C, 0x8C, 0xEB, 0x62, 0xC2, 0xA2, 0xA7,
+0xD1, 0x18, 0x0F, 0x05, 0x00, 0x6C, 0xD1, 0x18,
+0xEE, 0x04, 0x00, 0x65, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x04, 0x94,
+0x08, 0x5C, 0x0B, 0x61, 0x3D, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEB, 0x04, 0x6C, 0x8D, 0xEB, 0x62, 0xC2,
+0xA2, 0xA7, 0xD1, 0x18, 0x0F, 0x05, 0x00, 0x6C,
+0xED, 0x17, 0x90, 0x67, 0x07, 0xD5, 0x06, 0xD6,
+0x05, 0xD7, 0xD1, 0x18, 0xC8, 0x04, 0x22, 0x67,
+0x07, 0x95, 0x51, 0x67, 0x05, 0x97, 0x60, 0xA5,
+0x06, 0x96, 0x06, 0x2B, 0x62, 0xA1, 0x3D, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEB, 0x08, 0x6C, 0xE6, 0x17,
+0x83, 0xA7, 0x42, 0xA7, 0xB0, 0x67, 0x80, 0x34,
+0x89, 0xE2, 0xE0, 0xF3, 0x1F, 0x6C, 0x4C, 0xEC,
+0x05, 0xD6, 0xD1, 0x18, 0xD1, 0x04, 0x27, 0x67,
+0x05, 0x96, 0x51, 0xA6, 0x70, 0xA6, 0x92, 0xA6,
+0x40, 0x32, 0x6D, 0xEA, 0x73, 0xA6, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6, 0x60, 0x33,
+0x4D, 0xEB, 0x84, 0x41, 0x00, 0x6E, 0x04, 0x97,
+0xFF, 0x6D, 0x46, 0x67, 0xAC, 0xEA, 0xEE, 0xEA,
+0xB9, 0x22, 0x40, 0xA4, 0xE3, 0xA4, 0x08, 0x4C,
+0x07, 0xD2, 0xFF, 0xF7, 0x1F, 0x6A, 0x3A, 0x65,
+0xFF, 0xF7, 0x5A, 0xA4, 0xE0, 0x37, 0xE9, 0xE2,
+0xF9, 0x67, 0xEC, 0xEA, 0x06, 0xD2, 0xFF, 0xF7,
+0x5C, 0xA4, 0xEC, 0xEA, 0xFF, 0xF7, 0xFD, 0xA4,
+0xE0, 0x37, 0xE9, 0xE2, 0xF9, 0x67, 0xEC, 0xEA,
+0xE1, 0xA3, 0x05, 0xD2, 0x40, 0xA3, 0xE0, 0x37,
+0x4D, 0xEF, 0x42, 0xA3, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEF, 0x43, 0xA3, 0x00, 0xF6, 0x40, 0x32,
+0xED, 0xEA, 0x80, 0xF0, 0xF8, 0xA2, 0x1F, 0x65,
+0x80, 0xF0, 0xF9, 0xA2, 0x38, 0x67, 0xE0, 0x37,
+0xED, 0xE9, 0x80, 0xF0, 0xFA, 0xA2, 0x80, 0xF0,
+0x5B, 0xA2, 0xE0, 0x37, 0xE0, 0x37, 0x2D, 0xEF,
+0x00, 0xF6, 0x40, 0x32, 0x4D, 0xEF, 0xC4, 0x32,
+0x09, 0xE2, 0x44, 0x32, 0x5D, 0xE7, 0xFF, 0xF7,
+0x59, 0xA4, 0x07, 0x91, 0x01, 0x4E, 0x40, 0x32,
+0x49, 0xE1, 0x39, 0x67, 0x2C, 0xEA, 0x22, 0x67,
+0x42, 0x32, 0x49, 0xC7, 0x06, 0x92, 0xAC, 0xE9,
+0x07, 0x76, 0xAC, 0xEA, 0x00, 0xF1, 0x44, 0xC7,
+0x06, 0x92, 0x28, 0xC7, 0x42, 0x32, 0x00, 0xF1,
+0x45, 0xC7, 0x05, 0x92, 0x4C, 0xED, 0x42, 0x32,
+0x00, 0xF2, 0xA0, 0xC7, 0x00, 0xF2, 0x41, 0xC7,
+0xA2, 0x61, 0x60, 0x17, 0x00, 0x6B, 0xA4, 0x67,
+0xA7, 0xEB, 0x01, 0x6E, 0xFF, 0x6A, 0xCC, 0xED,
+0x6C, 0xEA, 0x04, 0x2D, 0x01, 0x4B, 0x08, 0x73,
+0xF6, 0x61, 0x08, 0x6A, 0x20, 0xE8, 0x00, 0x65,
+0x00, 0x6B, 0xA4, 0x67, 0xA7, 0xEB, 0x01, 0x6E,
+0xFF, 0x6A, 0xCC, 0xED, 0x6C, 0xEA, 0x04, 0x2D,
+0x01, 0x4B, 0x10, 0x73, 0xF6, 0x61, 0x10, 0x6A,
+0x20, 0xE8, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x06, 0x67, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xFF, 0x75, 0x24, 0x67,
+0x04, 0xD5, 0xC0, 0x36, 0x20, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x09, 0xF3, 0x84, 0x9E,
+0x40, 0x32, 0xF2, 0xF4, 0x4C, 0x9A, 0x91, 0xE1,
+0x40, 0xEA, 0x05, 0xD6, 0x04, 0x95, 0xE2, 0x67,
+0xD1, 0x18, 0xBB, 0x05, 0x85, 0x67, 0x00, 0xF6,
+0xE0, 0x37, 0x04, 0x95, 0x00, 0xF6, 0xE3, 0x37,
+0x04, 0xEA, 0xEE, 0xE8, 0x0C, 0xED, 0xEE, 0xED,
+0x00, 0xF6, 0xA0, 0x35, 0x05, 0x96, 0x00, 0xF6,
+0xA3, 0x35, 0xFF, 0x68, 0xAC, 0xE8, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x09, 0xF3, 0x84, 0x9E,
+0x40, 0x32, 0xF2, 0xF4, 0x40, 0x9A, 0xB0, 0x67,
+0x40, 0xEA, 0x91, 0xE1, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x06, 0x67, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xFF, 0xF7, 0x1F, 0x75, 0x24, 0x67, 0x04, 0xD5,
+0xC0, 0x36, 0x21, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x09, 0xF3, 0x84, 0x9E, 0x40, 0x32,
+0xF2, 0xF4, 0x48, 0x9A, 0x91, 0xE1, 0x40, 0xEA,
+0x05, 0xD6, 0x04, 0x95, 0xE2, 0x67, 0xD1, 0x18,
+0xC2, 0x05, 0x85, 0x67, 0xE0, 0x37, 0xE0, 0x37,
+0xE3, 0x37, 0x04, 0x95, 0xE3, 0x37, 0x04, 0xEA,
+0xEE, 0xE8, 0x0C, 0xED, 0xEE, 0xED, 0xA0, 0x35,
+0xA0, 0x35, 0xA3, 0x35, 0x05, 0x96, 0xA3, 0x35,
+0xFF, 0xF7, 0x1F, 0x68, 0xAC, 0xE8, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x09, 0xF3, 0x84, 0x9E,
+0x40, 0x32, 0xD2, 0xF4, 0x5C, 0x9A, 0xB0, 0x67,
+0x40, 0xEA, 0x91, 0xE1, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x15, 0xF0, 0x08, 0x4C, 0xA9, 0xA4, 0x48, 0xA4,
+0x6A, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA4,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x71, 0xA2, 0xB2, 0xA2,
+0xD0, 0xA2, 0x53, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x41, 0xA2, 0xA0, 0xF0,
+0x0E, 0x2A, 0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1,
+0x07, 0xD0, 0xB1, 0xA4, 0x70, 0xA4, 0x52, 0xA4,
+0xA0, 0x35, 0x6D, 0xED, 0x73, 0xA4, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0x81, 0xA3, 0x40, 0xA3, 0xA2, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x80, 0xF0, 0x79, 0xA2, 0x80, 0xF0,
+0x18, 0xA2, 0x80, 0xF0, 0x9A, 0xA2, 0x60, 0x33,
+0x0D, 0xEB, 0x80, 0xF0, 0x1B, 0xA2, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEB, 0x18, 0x6D, 0x00, 0xF6,
+0x00, 0x30, 0xE0, 0xF3, 0x1F, 0x6E, 0x00, 0x6C,
+0xD1, 0x18, 0xF2, 0x6C, 0x6D, 0xE8, 0x22, 0x67,
+0x62, 0xA0, 0x03, 0x6A, 0x1A, 0x65, 0x58, 0x67,
+0x6C, 0xEA, 0xFF, 0x6D, 0xAC, 0xEA, 0x01, 0x72,
+0x12, 0x61, 0xC2, 0x4A, 0x4C, 0xEB, 0x18, 0x6A,
+0x4D, 0xEB, 0x62, 0xC0, 0x00, 0x6C, 0xD1, 0x18,
+0x0F, 0x05, 0x2C, 0xED, 0x42, 0xA0, 0x01, 0x6B,
+0x03, 0x6C, 0x4C, 0xEB, 0x8B, 0xEC, 0x64, 0x33,
+0x8C, 0xEA, 0x6D, 0xEA, 0x42, 0xC0, 0x42, 0xA0,
+0x01, 0x6B, 0xFF, 0x6D, 0x6C, 0xEA, 0xAC, 0xEA,
+0x28, 0x22, 0x65, 0xA0, 0xC4, 0xA0, 0xFF, 0xF7,
+0x1F, 0x6C, 0x60, 0x33, 0xCD, 0xEB, 0xF1, 0x67,
+0x47, 0xA0, 0x8C, 0xEF, 0x6C, 0xEC, 0x66, 0xA0,
+0x40, 0x32, 0x6D, 0xEA, 0xEE, 0xEA, 0x01, 0x6B,
+0x06, 0x22, 0x67, 0x67, 0x8E, 0xEB, 0x6B, 0xEB,
+0xC0, 0xF7, 0x63, 0x33, 0xAC, 0xEB, 0x40, 0xA0,
+0x01, 0x72, 0x03, 0x61, 0x41, 0xA0, 0x01, 0x72,
+0x11, 0x60, 0x42, 0xA0, 0x3D, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x10, 0x6B, 0x6D, 0xEA, 0xFF, 0x6D,
+0x42, 0xC0, 0x2C, 0xED, 0xD1, 0x18, 0x0F, 0x05,
+0x00, 0x6C, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFF, 0x73, 0x00, 0x6F,
+0x06, 0x61, 0x42, 0xA0, 0x3D, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x0C, 0x6B, 0xEB, 0x17, 0xE4, 0x32,
+0x69, 0xE2, 0x44, 0x32, 0x49, 0xE0, 0x89, 0xA2,
+0xA8, 0xA2, 0xFF, 0xF7, 0x1F, 0x69, 0x80, 0x34,
+0xAD, 0xEC, 0x2A, 0xEC, 0x14, 0x60, 0x00, 0xF2,
+0xC1, 0xA2, 0x00, 0xF2, 0xA0, 0xA2, 0x05, 0xD3,
+0xC0, 0x36, 0xAD, 0xEE, 0x00, 0xF1, 0xA4, 0xA2,
+0x00, 0xF1, 0x45, 0xA2, 0x2C, 0xEE, 0x04, 0xD7,
+0x40, 0x32, 0x4D, 0xED, 0xD1, 0x18, 0xE8, 0x05,
+0x2C, 0xED, 0x05, 0x93, 0x04, 0x97, 0x01, 0x4F,
+0x3F, 0x77, 0xDD, 0x61, 0xCE, 0x17, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x03, 0x74, 0x0B, 0x61,
+0xC4, 0x32, 0x59, 0xE6, 0xC8, 0x36, 0x47, 0x46,
+0x4D, 0x4A, 0xFF, 0x6E, 0xCC, 0xEA, 0xA9, 0xE2,
+0xFF, 0x6B, 0x20, 0xE8, 0x6C, 0xEA, 0x02, 0x74,
+0x06, 0x61, 0xC8, 0x32, 0x59, 0xE6, 0xC4, 0x36,
+0x47, 0x46, 0x25, 0x4A, 0xF2, 0x17, 0x01, 0x74,
+0x00, 0x6A, 0xF1, 0x61, 0xCC, 0x36, 0x0C, 0x4E,
+0xFF, 0x6A, 0xEC, 0x17, 0x0A, 0x24, 0x0C, 0x5D,
+0x04, 0x60, 0x00, 0x6A, 0xA0, 0xC6, 0x20, 0xE8,
+0x40, 0xC7, 0x03, 0x5C, 0x0C, 0x6A, 0x54, 0x6B,
+0x10, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x08, 0xF3, 0x08, 0x4A, 0x49, 0xE4,
+0x60, 0xA2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x08, 0xF3, 0x04, 0x4A, 0x51, 0xE4,
+0x40, 0xA4, 0x77, 0xE5, 0xFF, 0x6B, 0x6C, 0xED,
+0x00, 0x6B, 0x43, 0xED, 0x60, 0xC7, 0x02, 0x60,
+0x20, 0xE8, 0xA0, 0xC6, 0xFF, 0x6B, 0x57, 0xE5,
+0x6C, 0xED, 0x60, 0xA7, 0x01, 0x4B, 0xF5, 0x17,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD0, 0x04, 0x07,
+0x00, 0xF0, 0x11, 0x06, 0xD1, 0x18, 0x7F, 0x06,
+0x04, 0x67, 0x03, 0x6A, 0x4E, 0xE8, 0x5D, 0x67,
+0x51, 0xA2, 0x01, 0x58, 0x18, 0x67, 0x0F, 0x6B,
+0x6C, 0xEA, 0x18, 0x30, 0x4D, 0xE8, 0xFF, 0x6A,
+0x0C, 0xEA, 0x07, 0x97, 0x06, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0x9E, 0x32, 0x03, 0x6B,
+0x6C, 0xEA, 0x01, 0x72, 0x40, 0xC5, 0x09, 0x60,
+0x0E, 0x22, 0x0F, 0x6A, 0x8C, 0xEA, 0x40, 0xC6,
+0x92, 0x34, 0x07, 0x6A, 0x4C, 0xEC, 0x20, 0xE8,
+0x80, 0xC7, 0x64, 0x67, 0x07, 0x6A, 0x4C, 0xEB,
+0x60, 0xC6, 0x8E, 0x34, 0xF7, 0x17, 0x0F, 0x6A,
+0x4C, 0xEC, 0x00, 0x6A, 0x80, 0xC6, 0x20, 0xE8,
+0x40, 0xC7, 0x00, 0x65, 0x10, 0x5C, 0x03, 0x6A,
+0x07, 0x60, 0x08, 0x5C, 0x02, 0x6A, 0x04, 0x60,
+0x04, 0x5C, 0x98, 0x67, 0x01, 0x74, 0x58, 0x67,
+0x20, 0xE8, 0x00, 0x65, 0x0C, 0x5C, 0x00, 0x6A,
+0x08, 0x61, 0x2C, 0x5C, 0x01, 0x6A, 0x05, 0x61,
+0x54, 0x5C, 0x58, 0x67, 0x01, 0x5A, 0x58, 0x67,
+0x02, 0x4A, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xAA, 0xEC, 0x64, 0x6A, 0x2B, 0x60, 0x00, 0x6A,
+0x29, 0x24, 0x83, 0xED, 0x27, 0x61, 0x02, 0xF0,
+0x00, 0x5D, 0x04, 0x6A, 0x0C, 0x61, 0x04, 0xF0,
+0x00, 0x5D, 0x03, 0x6A, 0x08, 0x61, 0x08, 0xF0,
+0x00, 0x5D, 0x02, 0x6A, 0x04, 0x61, 0x10, 0xF0,
+0x00, 0x6A, 0x43, 0xED, 0x58, 0x67, 0xA4, 0xEA,
+0x84, 0xEA, 0x00, 0x6E, 0x01, 0x6A, 0xA6, 0x35,
+0x83, 0xED, 0x14, 0x60, 0x81, 0xF4, 0x00, 0x6B,
+0x67, 0xEE, 0x69, 0xE2, 0x01, 0x75, 0xFF, 0xF7,
+0x1F, 0x6B, 0x6C, 0xEA, 0xB3, 0xE4, 0x04, 0x61,
+0x02, 0x5C, 0x07, 0x60, 0x01, 0x74, 0x01, 0x60,
+0x05, 0x2C, 0x5A, 0x32, 0xFF, 0x6B, 0x20, 0xE8,
+0x6C, 0xEA, 0x02, 0x6D, 0x01, 0x4E, 0x0C, 0x76,
+0xE6, 0x61, 0xF7, 0x17, 0xFE, 0x63, 0x7D, 0x67,
+0x60, 0x6A, 0x40, 0xC3, 0x54, 0x6A, 0x41, 0xC3,
+0x4A, 0x6A, 0x42, 0xC3, 0x40, 0x6A, 0x43, 0xC3,
+0x36, 0x6A, 0x44, 0xC3, 0x2C, 0x6A, 0x45, 0xC3,
+0x24, 0x6A, 0x46, 0xC3, 0x1C, 0x6A, 0x47, 0xC3,
+0x14, 0x6A, 0x48, 0xC3, 0x0C, 0x6A, 0x49, 0xC3,
+0x04, 0x6A, 0x4A, 0xC3, 0x00, 0x6A, 0x4B, 0xC3,
+0x00, 0x6B, 0x5D, 0x67, 0x69, 0xE2, 0xA0, 0xA2,
+0xA3, 0xEC, 0x06, 0x61, 0x41, 0x44, 0xAB, 0xE2,
+0xFF, 0x6C, 0x8C, 0xEA, 0x20, 0xE8, 0x02, 0x63,
+0x01, 0x4B, 0x0C, 0x73, 0xF2, 0x61, 0x00, 0x6A,
+0xF9, 0x17, 0x00, 0x65, 0x40, 0xA4, 0x60, 0xA5,
+0x60, 0xC4, 0x20, 0xE8, 0x40, 0xC5, 0x00, 0x65,
+0x00, 0x6A, 0x20, 0xF0, 0x4C, 0xC4, 0x00, 0x6A,
+0x20, 0xF0, 0x4D, 0xC4, 0x20, 0xF0, 0x4E, 0xC4,
+0x20, 0xF0, 0x4F, 0xC4, 0x20, 0xF0, 0x50, 0xC4,
+0x20, 0xF0, 0x57, 0xC4, 0x20, 0xF0, 0x58, 0xC4,
+0x20, 0xF0, 0x51, 0xC4, 0x20, 0xF0, 0x52, 0xC4,
+0x20, 0xF0, 0x59, 0xC4, 0x20, 0xF0, 0x5A, 0xC4,
+0x20, 0xF0, 0x53, 0xC4, 0x20, 0xF0, 0x54, 0xC4,
+0x20, 0xF0, 0x5B, 0xC4, 0x20, 0xF0, 0x5C, 0xC4,
+0x20, 0xF0, 0x55, 0xC4, 0x20, 0xF0, 0x56, 0xC4,
+0x20, 0xF0, 0x5D, 0xC4, 0x20, 0xF0, 0x5E, 0xC4,
+0x20, 0xE8, 0x00, 0x65, 0x00, 0x6A, 0x5C, 0xC4,
+0x5D, 0xC4, 0x5E, 0xC4, 0x5F, 0xC4, 0x5A, 0xC4,
+0x5B, 0xC4, 0x58, 0xC4, 0x59, 0xC4, 0x20, 0xF0,
+0x40, 0xC4, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x00, 0x6A, 0x20, 0xF0, 0x43, 0xC4, 0x20, 0xF0,
+0x44, 0xC4, 0x20, 0xF0, 0x4A, 0xC4, 0x20, 0xF0,
+0x4B, 0xC4, 0x20, 0xF0, 0x45, 0xC4, 0x20, 0xF0,
+0x46, 0xC4, 0x20, 0xF0, 0x47, 0xC4, 0x20, 0xF0,
+0x48, 0xC4, 0x20, 0xF0, 0x49, 0xC4, 0x60, 0xF0,
+0x5E, 0xA4, 0x20, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x60, 0xF0, 0x5E, 0xC4, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0x5C, 0x00, 0x6A, 0x10, 0x60, 0x80, 0xF0,
+0x95, 0xA5, 0x80, 0xF0, 0x54, 0xA5, 0x80, 0xF0,
+0x76, 0xA5, 0x80, 0x34, 0x4D, 0xEC, 0x80, 0xF0,
+0x57, 0xA5, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD0, 0x40, 0xF0, 0xB5, 0xA4, 0x04, 0x67,
+0xD1, 0x18, 0xB5, 0x06, 0x85, 0x67, 0x82, 0x67,
+0x00, 0xF0, 0x11, 0x06, 0xD1, 0x18, 0x7F, 0x06,
+0x04, 0x07, 0x80, 0xF0, 0x93, 0xA0, 0x80, 0xF0,
+0x52, 0xA0, 0x80, 0x34, 0x4D, 0xEC, 0x80, 0xF0,
+0x40, 0xA0, 0xFF, 0x72, 0x16, 0x60, 0x80, 0xF0,
+0x43, 0xA0, 0x80, 0xF0, 0xC2, 0xA0, 0xFF, 0x6B,
+0x1B, 0x65, 0x40, 0x32, 0x78, 0x67, 0xFF, 0xF7,
+0x1F, 0x6D, 0xCD, 0xEA, 0xAC, 0xEA, 0x8C, 0xEB,
+0x63, 0xEA, 0x22, 0x60, 0x01, 0x4A, 0xAC, 0xEA,
+0x80, 0xF0, 0x42, 0xC0, 0x42, 0x32, 0x80, 0xF0,
+0x43, 0xC0, 0x80, 0xF0, 0x41, 0xA0, 0xFF, 0x72,
+0x13, 0x60, 0x80, 0xF0, 0x45, 0xA0, 0x80, 0xF0,
+0x64, 0xA0, 0x40, 0x32, 0x6D, 0xEA, 0xFF, 0x6B,
+0x6C, 0xEC, 0x83, 0xEA, 0x17, 0x61, 0x00, 0x6A,
+0x80, 0xF0, 0x44, 0xC0, 0x80, 0xF0, 0x45, 0xC0,
+0x01, 0x6A, 0x4B, 0xEA, 0x80, 0xF0, 0x41, 0xC0,
+0x07, 0x97, 0x06, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x00, 0x6A, 0x80, 0xF0, 0x42, 0xC0, 0x80, 0xF0,
+0x43, 0xC0, 0x01, 0x6A, 0x4B, 0xEA, 0x80, 0xF0,
+0x40, 0xC0, 0xDB, 0x17, 0x01, 0x4A, 0xFF, 0xF7,
+0x1F, 0x6B, 0x6C, 0xEA, 0x80, 0xF0, 0x44, 0xC0,
+0x42, 0x32, 0x80, 0xF0, 0x45, 0xC0, 0xE8, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x00, 0x6A, 0x7D, 0x67, 0x52, 0xC3, 0x51, 0xC3,
+0x50, 0xC3, 0x60, 0xF0, 0x66, 0xA4, 0x08, 0x6A,
+0x4C, 0xEB, 0xFF, 0x6A, 0x4C, 0xEB, 0x2F, 0x23,
+0x07, 0xD2, 0x06, 0xD7, 0x80, 0xF0, 0x6D, 0xA4,
+0x04, 0x67, 0x80, 0xF0, 0x8E, 0xA4, 0x3E, 0x65,
+0x25, 0x67, 0x80, 0x34, 0x6D, 0xEC, 0x00, 0xF0,
+0x11, 0x06, 0x00, 0xF0, 0x12, 0x05, 0xD1, 0x18,
+0xA1, 0x06, 0x04, 0x07, 0x4E, 0x41, 0x62, 0x67,
+0x07, 0x92, 0x01, 0x6C, 0x4C, 0xEB, 0x02, 0x5B,
+0x0A, 0x61, 0x01, 0x71, 0x00, 0x6C, 0x07, 0x61,
+0x79, 0x67, 0x05, 0x2B, 0x7D, 0x67, 0x90, 0xA3,
+0x01, 0x5C, 0x98, 0x67, 0x4C, 0xEC, 0x60, 0xF0,
+0x54, 0xA0, 0x01, 0x6D, 0x4C, 0xED, 0x08, 0x25,
+0x79, 0x67, 0x02, 0x2B, 0x06, 0x93, 0x04, 0x23,
+0x04, 0x6B, 0x6C, 0xEA, 0x01, 0x2A, 0x00, 0x6C,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x44, 0x67,
+0x00, 0xEF, 0x06, 0x63, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xFF, 0x69, 0x04, 0x67,
+0x85, 0x67, 0x2C, 0xEC, 0x00, 0x6D, 0xD1, 0x18,
+0x1B, 0x6B, 0x04, 0x06, 0x5D, 0x67, 0xA8, 0xAA,
+0xE0, 0xF1, 0x1F, 0x6A, 0x7D, 0x67, 0x4C, 0xED,
+0x00, 0x6A, 0x4A, 0xCB, 0xBE, 0x32, 0x82, 0x67,
+0x5D, 0x67, 0xF1, 0xA2, 0x03, 0x6B, 0x2C, 0xEC,
+0xEA, 0x32, 0x6C, 0xEA, 0x2C, 0xEA, 0x02, 0x74,
+0x3C, 0x65, 0x06, 0xD2, 0x30, 0x60, 0x03, 0x5C,
+0x3D, 0x60, 0x01, 0x74, 0x21, 0x60, 0x0F, 0x6A,
+0xAC, 0xEA, 0x2C, 0xEA, 0x00, 0x69, 0x9D, 0x67,
+0x06, 0x95, 0x94, 0xA4, 0x04, 0x6E, 0x03, 0x6B,
+0xCB, 0xEE, 0x6C, 0xED, 0xCC, 0xEC, 0xAD, 0xEC,
+0xB9, 0x67, 0x6C, 0xED, 0xA8, 0x33, 0x0D, 0x6D,
+0xAB, 0xED, 0xAC, 0xEC, 0x6D, 0xEC, 0x30, 0x33,
+0x71, 0x69, 0x2B, 0xE9, 0x8C, 0xE9, 0x6D, 0xE9,
+0x20, 0xC0, 0x41, 0xC0, 0x0B, 0x97, 0x50, 0x67,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0x1F, 0x6A, 0x4C, 0xED, 0x00, 0x6E, 0xD1, 0x18,
+0x71, 0x06, 0x2C, 0xED, 0xF2, 0x33, 0x07, 0x6F,
+0xEC, 0xEB, 0x6C, 0xE9, 0xD8, 0x17, 0x07, 0x6B,
+0x5D, 0x67, 0xB2, 0x36, 0x7C, 0xC2, 0x0F, 0x6A,
+0x6C, 0xEE, 0x4C, 0xED, 0x2C, 0xEE, 0x2C, 0xED,
+0xD1, 0x18, 0x71, 0x06, 0xF2, 0x37, 0x7D, 0x67,
+0x7C, 0xA3, 0xEE, 0x17, 0x07, 0x6B, 0x5D, 0x67,
+0xB2, 0x36, 0x7C, 0xC2, 0x0F, 0x6A, 0x6C, 0xEE,
+0x4C, 0xED, 0x2C, 0xEE, 0x2C, 0xED, 0x03, 0x6C,
+0xEF, 0x17, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD0, 0x20, 0x5D, 0x04, 0x67, 0x0E, 0x61,
+0x00, 0x6A, 0x7D, 0x67, 0x48, 0xCB, 0x5D, 0x67,
+0x50, 0xA2, 0x07, 0x97, 0x40, 0xC0, 0x5D, 0x67,
+0x51, 0xA2, 0x41, 0xC0, 0x50, 0x67, 0x06, 0x90,
+0x00, 0xEF, 0x04, 0x63, 0xD1, 0x18, 0x77, 0x07,
+0x04, 0x04, 0xF1, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0x05, 0x24, 0x09, 0x5C, 0x16, 0x60, 0xFF, 0x4C,
+0xFF, 0x6A, 0x4C, 0xEC, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x09, 0xF3, 0x48, 0x9A,
+0x88, 0x34, 0x00, 0x6E, 0x62, 0x67, 0x64, 0xEC,
+0x04, 0x05, 0x02, 0x6C, 0x04, 0xD3, 0xD1, 0x18,
+0xE5, 0x6B, 0x05, 0xD2, 0x07, 0x97, 0x00, 0xEF,
+0x04, 0x63, 0x07, 0x6C, 0xEB, 0x17, 0x00, 0x65,
+0xF3, 0x63, 0x19, 0x62, 0x18, 0xD1, 0x17, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x1C, 0xD6, 0x1B, 0xD5, 0x12, 0xD4, 0x15, 0xF0,
+0x08, 0x4B, 0xB1, 0xA3, 0x90, 0xA3, 0xD2, 0xA3,
+0xA0, 0x35, 0x8D, 0xED, 0x93, 0xA3, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34,
+0xAD, 0xEC, 0xA1, 0xA4, 0x60, 0xA4, 0xC2, 0xA4,
+0xA0, 0x35, 0x6D, 0xED, 0x63, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0x80, 0xF0, 0xA5, 0xA3, 0x80, 0xF0,
+0x86, 0xA3, 0x80, 0xF0, 0xC4, 0xA3, 0x80, 0xF0,
+0x67, 0xA3, 0xA0, 0x35, 0x80, 0x34, 0xAD, 0xEE,
+0x80, 0x34, 0xCD, 0xEC, 0x1B, 0x92, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x11, 0xD3, 0x00, 0x6B,
+0x80, 0xA2, 0x0D, 0xD3, 0x0C, 0xD3, 0x03, 0x6A,
+0x7D, 0x67, 0x40, 0xF0, 0x40, 0xC3, 0x1B, 0x92,
+0x8A, 0x34, 0x27, 0x67, 0xA1, 0xA2, 0x10, 0x92,
+0x0F, 0x07, 0x20, 0xF0, 0x1D, 0x06, 0x4C, 0xEC,
+0xD1, 0x18, 0x7F, 0x06, 0x1E, 0x90, 0x1B, 0x92,
+0x9D, 0x67, 0xFF, 0xF7, 0x1F, 0x6B, 0xA0, 0xA2,
+0x10, 0x92, 0xAA, 0x35, 0x4C, 0xED, 0x02, 0x75,
+0x12, 0x92, 0x38, 0x60, 0x03, 0x5D, 0x40, 0x60,
+0x01, 0x75, 0x28, 0x60, 0x20, 0xF0, 0x9D, 0xA4,
+0x8C, 0xEB, 0x00, 0x6C, 0x0B, 0xD4, 0xE0, 0xF1,
+0x1F, 0x6C, 0x8C, 0xEB, 0x83, 0x67, 0x12, 0xD3,
+0x7D, 0x67, 0x96, 0xCB, 0x1B, 0x93, 0x03, 0x6E,
+0x60, 0xA3, 0x6C, 0xEE, 0x82, 0x33, 0x0D, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEB, 0xC8, 0x36, 0xCD, 0xEB,
+0x9D, 0x67, 0x20, 0xF0, 0x6D, 0xC4, 0x3E, 0x25,
+0x03, 0x5D, 0x2B, 0x60, 0x1B, 0x94, 0xA0, 0xA4,
+0x70, 0x6C, 0x8C, 0xED, 0x71, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEB, 0xAD, 0xEB, 0x9D, 0x67, 0x20, 0xF0,
+0x6D, 0xC4, 0x30, 0x10, 0x20, 0xF0, 0xDC, 0xA4,
+0x20, 0xF0, 0x9D, 0xA4, 0x80, 0x6F, 0xEB, 0xEF,
+0xED, 0xEC, 0xFF, 0x6F, 0xCC, 0x36, 0xEC, 0xEC,
+0xCD, 0xEC, 0xCE, 0x17, 0x20, 0xF0, 0x9C, 0xA4,
+0xDD, 0x67, 0x20, 0xF0, 0xDD, 0xA6, 0xFF, 0x6F,
+0x90, 0x34, 0x01, 0x4F, 0xED, 0xEE, 0xF4, 0x17,
+0x20, 0xF0, 0x9C, 0xA4, 0xDD, 0x67, 0x20, 0xF0,
+0xDD, 0xA6, 0x90, 0x34, 0x80, 0xF1, 0x00, 0x6F,
+0xF5, 0x17, 0x60, 0xF0, 0x86, 0xA0, 0x10, 0x6D,
+0xAC, 0xEC, 0x71, 0x6D, 0xAB, 0xED, 0xC0, 0xF0,
+0x01, 0x24, 0x60, 0xF0, 0x86, 0xA0, 0xAC, 0xEB,
+0x96, 0x34, 0x90, 0x34, 0x6D, 0xEC, 0x7D, 0x67,
+0x20, 0xF0, 0x8D, 0xC3, 0x1C, 0x94, 0x01, 0x6B,
+0x09, 0x6D, 0x8C, 0xEB, 0x6C, 0x34, 0x7D, 0x67,
+0x20, 0xF0, 0x6F, 0xA3, 0xAB, 0xED, 0x0A, 0x07,
+0xAC, 0xEB, 0x8D, 0xEB, 0x9D, 0x67, 0x20, 0xF0,
+0x6F, 0xC4, 0x7D, 0x67, 0x20, 0xF0, 0x6D, 0xA3,
+0x3C, 0x34, 0x7F, 0x69, 0x2C, 0xEB, 0x8D, 0xEB,
+0x9D, 0x67, 0x20, 0xF0, 0x6D, 0xC4, 0x00, 0x6B,
+0x0A, 0xD3, 0xE0, 0xF1, 0x1F, 0x6B, 0x1B, 0x65,
+0x98, 0x67, 0x7D, 0x67, 0x94, 0xCB, 0x78, 0x67,
+0x04, 0x6C, 0x14, 0xD3, 0x8B, 0xEC, 0x62, 0x33,
+0x8D, 0xEB, 0x9D, 0x67, 0x20, 0xF0, 0x69, 0xC4,
+0x08, 0x6B, 0x20, 0xF0, 0x6B, 0xC4, 0xFF, 0x6B,
+0x6C, 0xEA, 0x82, 0x67, 0x0B, 0x06, 0x00, 0x6D,
+0x13, 0xD3, 0xD1, 0x18, 0x83, 0x6B, 0x10, 0xD2,
+0x09, 0x06, 0x00, 0x6D, 0xD1, 0x18, 0x1B, 0x6B,
+0x00, 0x6C, 0x08, 0x06, 0x00, 0x6D, 0xD1, 0x18,
+0x1B, 0x6B, 0x01, 0x6C, 0x09, 0x95, 0xFF, 0x6C,
+0x02, 0x6E, 0xD1, 0x18, 0x40, 0x6A, 0x61, 0x4C,
+0x08, 0x95, 0xFF, 0x6C, 0x02, 0x6E, 0xD1, 0x18,
+0x40, 0x6A, 0x65, 0x4C, 0x5D, 0x67, 0x20, 0xF0,
+0xCD, 0xA2, 0x03, 0x6C, 0x07, 0x6A, 0xCA, 0x35,
+0x8C, 0xED, 0xA4, 0x37, 0xA1, 0xA0, 0x4B, 0xEA,
+0x14, 0x93, 0x4C, 0xED, 0xD2, 0x32, 0x07, 0x6E,
+0xCC, 0xEA, 0x39, 0x6E, 0xCB, 0xEE, 0xED, 0xED,
+0xCC, 0xED, 0x4C, 0x32, 0xAD, 0xEA, 0xC8, 0xA0,
+0x41, 0xC0, 0x02, 0x6A, 0x4B, 0xEA, 0x4C, 0xEE,
+0x5D, 0x67, 0x56, 0xAA, 0x1B, 0x65, 0x01, 0x6D,
+0x6C, 0xEA, 0x13, 0x93, 0x4C, 0xED, 0x46, 0x32,
+0x4C, 0xEB, 0x1B, 0x92, 0xBC, 0x35, 0xCC, 0xE9,
+0x40, 0xA2, 0xAD, 0xE9, 0x69, 0xC0, 0x4A, 0x32,
+0x4C, 0xEC, 0x28, 0xC0, 0x78, 0x67, 0x45, 0x24,
+0x5D, 0x67, 0x20, 0xF0, 0x9D, 0xA2, 0x08, 0x6A,
+0x03, 0x5C, 0x03, 0x60, 0x06, 0x6A, 0x01, 0x2C,
+0x04, 0x6A, 0x00, 0x6C, 0x6C, 0xEA, 0x07, 0xD4,
+0x06, 0xD4, 0x9D, 0x67, 0x4F, 0xCC, 0x10, 0x94,
+0x5D, 0x67, 0x06, 0x07, 0x07, 0x06, 0x01, 0x6D,
+0xD1, 0x18, 0x83, 0x6B, 0x6D, 0xCA, 0x1B, 0x92,
+0x7D, 0x67, 0x20, 0xF0, 0xDC, 0xA3, 0x40, 0xA2,
+0x1C, 0x97, 0x03, 0x6D, 0x4A, 0x32, 0x4C, 0xED,
+0xD1, 0x18, 0x54, 0x07, 0x90, 0x67, 0x22, 0x67,
+0x60, 0xF0, 0x54, 0xA0, 0x01, 0x6B, 0x4C, 0xEB,
+0x34, 0x23, 0x11, 0x93, 0x80, 0xF0, 0x61, 0xA3,
+0x06, 0x23, 0x06, 0x6B, 0x1B, 0x65, 0x78, 0x67,
+0x4C, 0xEB, 0x04, 0x73, 0x09, 0x61, 0x10, 0x5A,
+0x19, 0x61, 0x04, 0x6B, 0x4C, 0xEB, 0x16, 0x2B,
+0x02, 0x6B, 0x6D, 0xEA, 0x60, 0xF0, 0x54, 0xC0,
+0x00, 0x69, 0x1F, 0x10, 0x1B, 0x94, 0x70, 0x6E,
+0xAC, 0xEB, 0x80, 0xA4, 0xCC, 0xEC, 0x8D, 0xEB,
+0x0D, 0x17, 0x1B, 0x92, 0x41, 0xA2, 0x09, 0x5A,
+0x08, 0x6A, 0xBF, 0x60, 0x5D, 0x67, 0x20, 0xF0,
+0x5D, 0xA2, 0xBB, 0x17, 0x7D, 0x67, 0x20, 0xF0,
+0x7C, 0xA3, 0x01, 0x73, 0xA0, 0xF0, 0x07, 0x61,
+0x1C, 0x93, 0xA0, 0xF0, 0x04, 0x2B, 0x60, 0xF0,
+0x54, 0xA0, 0xFD, 0x4B, 0x6C, 0xEA, 0x60, 0xF0,
+0x54, 0xC0, 0x60, 0xF0, 0x94, 0xA0, 0x01, 0x6B,
+0x60, 0xF0, 0xD4, 0xA0, 0x6C, 0xEC, 0xFF, 0x6A,
+0x4C, 0xEC, 0xCE, 0x35, 0x9C, 0x34, 0xAD, 0xEC,
+0xCA, 0x35, 0x6C, 0xED, 0xB8, 0x35, 0xAD, 0xEC,
+0xC6, 0x35, 0x6C, 0xED, 0xB4, 0x35, 0x8D, 0xED,
+0x14, 0xD2, 0x5D, 0x67, 0xE0, 0xF1, 0x0F, 0x6C,
+0x40, 0xF0, 0x6C, 0xC2, 0xD1, 0x18, 0x40, 0x6A,
+0x00, 0x6E, 0x5D, 0x67, 0x20, 0xF0, 0x95, 0xA2,
+0x7F, 0x6E, 0x3C, 0x35, 0xCC, 0xEC, 0xAD, 0xEC,
+0x20, 0xF0, 0x95, 0xC2, 0x20, 0xF0, 0x91, 0xA2,
+0x80, 0x6D, 0xAB, 0xED, 0xAD, 0xEC, 0x20, 0xF0,
+0x91, 0xC2, 0x10, 0x94, 0x04, 0x6D, 0x0C, 0x07,
+0xD1, 0x18, 0x83, 0x6B, 0x0D, 0x06, 0x60, 0xF0,
+0x96, 0xA0, 0x14, 0x92, 0x03, 0x6D, 0xAC, 0xEC,
+0x7D, 0x67, 0x8C, 0xEA, 0x40, 0xF0, 0x6C, 0xA3,
+0x2A, 0x22, 0x5D, 0x67, 0x40, 0xF0, 0x6C, 0xC2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x99, 0xF5, 0x08, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x04, 0xF0, 0x01, 0x6C,
+0x8B, 0xEC, 0x7D, 0x67, 0x8C, 0xEA, 0x40, 0xF0,
+0x6C, 0xA3, 0x60, 0xF0, 0x94, 0xA0, 0x8C, 0xEB,
+0x4F, 0x23, 0x06, 0x6B, 0x6C, 0xEC, 0x04, 0x74,
+0x4B, 0x60, 0x20, 0x31, 0x34, 0x31, 0x2D, 0xEA,
+0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A, 0x99, 0xF5,
+0x08, 0x6C, 0x40, 0xEA, 0x09, 0x95, 0x12, 0x92,
+0xE0, 0xF1, 0x1F, 0x6B, 0x00, 0x69, 0x6C, 0xEA,
+0x9D, 0x67, 0x05, 0xD1, 0x4A, 0xCC, 0x10, 0x94,
+0x5D, 0x67, 0x04, 0x07, 0x05, 0x06, 0x05, 0x6D,
+0x04, 0xD1, 0xD1, 0x18, 0x83, 0x6B, 0x68, 0xCA,
+0x5D, 0x67, 0x20, 0xF0, 0x3B, 0xC2, 0x20, 0xF0,
+0x3A, 0xC2, 0x20, 0xF0, 0x39, 0xC2, 0x20, 0xF0,
+0x38, 0xC2, 0x1B, 0x92, 0x60, 0xA2, 0x03, 0x6A,
+0x6A, 0x33, 0x4C, 0xEB, 0x02, 0x73, 0x20, 0xF0,
+0x1A, 0x02, 0x0B, 0x60, 0x03, 0x5B, 0x20, 0xF0,
+0x1B, 0x02, 0x07, 0x60, 0x01, 0x73, 0x58, 0x67,
+0x01, 0x5A, 0x58, 0x67, 0x9D, 0x67, 0x89, 0xE2,
+0x38, 0x4A, 0x60, 0xF0, 0xB6, 0xA0, 0x02, 0x6C,
+0xAC, 0xEC, 0x13, 0x24, 0x02, 0x23, 0x60, 0xF0,
+0x78, 0xA0, 0x60, 0xC2, 0x20, 0x10, 0x02, 0x6B,
+0x6C, 0xEA, 0x3F, 0xF7, 0x1D, 0x2A, 0x5D, 0x17,
+0x60, 0xF0, 0x66, 0xA0, 0x01, 0x6C, 0x6E, 0x33,
+0x8C, 0xEB, 0x60, 0x33, 0x74, 0x33, 0x6D, 0xEA,
+0xAF, 0x17, 0x01, 0x6E, 0xCC, 0xED, 0x1B, 0x25,
+0xEC, 0x23, 0x7D, 0x67, 0x20, 0xF0, 0x7D, 0xA3,
+0x08, 0x5B, 0x02, 0x61, 0x05, 0x6B, 0xE5, 0x17,
+0x05, 0x5B, 0x02, 0x61, 0x03, 0x6B, 0xE1, 0x17,
+0x03, 0x5B, 0x32, 0x61, 0xC0, 0xC2, 0x5D, 0x67,
+0x20, 0xF0, 0xFB, 0xA2, 0x20, 0xF0, 0xDA, 0xA2,
+0x20, 0xF0, 0xB9, 0xA2, 0x60, 0xF0, 0x97, 0xA0,
+0xB1, 0x18, 0xBE, 0xF8, 0x00, 0x65, 0x11, 0x92,
+0x02, 0x6B, 0xFF, 0x6C, 0x60, 0xF0, 0x59, 0xA2,
+0x6C, 0xEA, 0x8C, 0xEA, 0x18, 0x22, 0x1B, 0x92,
+0x60, 0xA2, 0x03, 0x6A, 0x6A, 0x33, 0x4C, 0xEB,
+0x1A, 0x23, 0x5D, 0x67, 0x20, 0xF0, 0x5C, 0xA2,
+0x44, 0x33, 0x4D, 0xE3, 0x5D, 0x67, 0x20, 0xF0,
+0x5D, 0xA2, 0x68, 0x33, 0x49, 0xE3, 0x11, 0x93,
+0x8C, 0xEA, 0x49, 0xE3, 0x40, 0xF7, 0x97, 0xA2,
+0xD1, 0x18, 0xB9, 0x07, 0x00, 0x65, 0x19, 0x97,
+0x18, 0x91, 0x17, 0x90, 0x00, 0xEF, 0x0D, 0x63,
+0xAC, 0x23, 0x80, 0xC2, 0xCC, 0x17, 0xD1, 0x18,
+0x79, 0x75, 0x00, 0x65, 0xF4, 0x17, 0x00, 0x65,
+0x20, 0x5C, 0x09, 0x60, 0xFC, 0x63, 0x07, 0x62,
+0x0C, 0x92, 0xD1, 0x18, 0xC8, 0x07, 0x04, 0xD2,
+0x07, 0x97, 0x00, 0xEF, 0x04, 0x63, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x20, 0x5C, 0x64, 0x60,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF6, 0x40, 0x9A, 0xFF, 0x69, 0x08, 0xD4,
+0x2C, 0xEC, 0x09, 0xD6, 0x40, 0xEA, 0x05, 0x67,
+0x08, 0x93, 0x09, 0x96, 0x4A, 0x22, 0x00, 0x6A,
+0x04, 0xD2, 0x05, 0xD2, 0x5D, 0x67, 0x68, 0xCA,
+0x60, 0xF0, 0x49, 0xA6, 0x80, 0xA0, 0x7D, 0x67,
+0x52, 0xC3, 0x8A, 0x34, 0x03, 0x6A, 0xA1, 0xA0,
+0x4C, 0xEC, 0x2C, 0xEC, 0x00, 0xF0, 0x19, 0x06,
+0xD1, 0x18, 0x7F, 0x06, 0x06, 0x07, 0x40, 0xA0,
+0x0C, 0x6B, 0x9D, 0x67, 0x6C, 0xEA, 0x4C, 0xE9,
+0x7D, 0x67, 0x5D, 0x67, 0x04, 0x71, 0x58, 0xA2,
+0x79, 0xA3, 0x94, 0xA4, 0x2F, 0x61, 0x4C, 0x32,
+0x6D, 0xEA, 0x7F, 0x6B, 0x6C, 0xEA, 0x80, 0x6B,
+0x6B, 0xEB, 0x8C, 0xEB, 0x6D, 0xEA, 0x7D, 0x67,
+0x54, 0xC3, 0x60, 0xA0, 0x5D, 0x67, 0x55, 0xA2,
+0x04, 0x6E, 0x03, 0x6D, 0x6A, 0x34, 0xCB, 0xEE,
+0xAC, 0xEC, 0xCC, 0xEA, 0x8D, 0xEA, 0x07, 0x6E,
+0x72, 0x34, 0xCC, 0xEC, 0x1D, 0x6E, 0xCB, 0xEE,
+0x88, 0x34, 0xCC, 0xEA, 0x8D, 0xEA, 0x61, 0x6C,
+0xAC, 0xEB, 0x8B, 0xEC, 0x74, 0x33, 0x8C, 0xEA,
+0x6D, 0xEA, 0x7D, 0x67, 0x55, 0xC3, 0x04, 0x07,
+0x10, 0x6E, 0x00, 0x6D, 0xD1, 0x18, 0xBF, 0x03,
+0x01, 0x6C, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0x50, 0x32, 0xD0, 0x17,
+0x20, 0xE8, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0x01, 0xA4, 0x00, 0x6E,
+0xFF, 0x70, 0x3E, 0x60, 0xE0, 0xA4, 0x03, 0x6A,
+0x3A, 0x65, 0xEA, 0x31, 0x59, 0x67, 0x4C, 0xE9,
+0x64, 0x67, 0x14, 0x21, 0x40, 0xF0, 0x9D, 0xA5,
+0x1F, 0x6A, 0x4C, 0xEC, 0xFF, 0x6A, 0xD1, 0x18,
+0xAF, 0x06, 0x4C, 0xEC, 0x4E, 0xE9, 0x00, 0x6E,
+0x09, 0x21, 0x99, 0x67, 0x8C, 0xEA, 0x0D, 0x6C,
+0x8B, 0xEC, 0x48, 0x32, 0x8C, 0xEF, 0x4D, 0xEF,
+0xE0, 0xC3, 0x03, 0x6E, 0x40, 0xF0, 0x93, 0xA5,
+0x03, 0xEC, 0x24, 0x60, 0xC0, 0xA3, 0x03, 0x6D,
+0x81, 0xC3, 0xCA, 0x32, 0xAC, 0xEA, 0x00, 0x6D,
+0x05, 0x22, 0x07, 0x6A, 0xD2, 0x35, 0x4C, 0xED,
+0xFF, 0x6A, 0x4C, 0xED, 0xB0, 0x32, 0xA0, 0xA3,
+0x71, 0x6E, 0xCB, 0xEE, 0xCC, 0xED, 0x4D, 0xED,
+0xD1, 0x18, 0xB5, 0x06, 0xA0, 0xC3, 0x03, 0x6C,
+0x8C, 0xEA, 0x0D, 0x6C, 0x8B, 0xEC, 0x48, 0x32,
+0x8C, 0xED, 0x4D, 0xED, 0xA0, 0xC3, 0x01, 0x6E,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x46, 0x67,
+0x00, 0xEF, 0x04, 0x63, 0x40, 0xF0, 0x94, 0xA5,
+0x83, 0xE8, 0xF6, 0x60, 0xC0, 0xA3, 0x03, 0x6D,
+0x81, 0xC3, 0xCA, 0x32, 0xAC, 0xEA, 0x00, 0x6D,
+0x05, 0x22, 0x07, 0x6A, 0xD2, 0x35, 0x4C, 0xED,
+0xFF, 0x6A, 0x4C, 0xED, 0xB0, 0x32, 0xA0, 0xA3,
+0x71, 0x6E, 0xCB, 0xEE, 0xCC, 0xED, 0x4D, 0xED,
+0xD1, 0x18, 0xB5, 0x06, 0xA0, 0xC3, 0x03, 0x6C,
+0x8C, 0xEA, 0x0D, 0x6C, 0x8B, 0xEC, 0x48, 0x32,
+0x8C, 0xED, 0x4D, 0xED, 0xA0, 0xC3, 0x02, 0x6E,
+0xD7, 0x17, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x26, 0x67, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0x10, 0xD4, 0x03, 0x6A,
+0x9D, 0x67, 0xC0, 0x36, 0x58, 0xC4, 0x15, 0xF0,
+0x08, 0x4E, 0x51, 0xA6, 0x90, 0xA6, 0x65, 0x67,
+0xB2, 0xA6, 0x40, 0x32, 0x8D, 0xEA, 0x93, 0xA6,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEA, 0x00, 0xF6,
+0x80, 0x34, 0x4D, 0xEC, 0xA1, 0xA4, 0x40, 0xA4,
+0x07, 0x67, 0xE2, 0xA4, 0xA0, 0x35, 0x4D, 0xED,
+0x43, 0xA4, 0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0x0A, 0xD6, 0xAD, 0xEA,
+0x80, 0xF0, 0xA5, 0xA2, 0x80, 0xF0, 0x86, 0xA2,
+0x80, 0xF0, 0xE4, 0xA2, 0x80, 0xF0, 0x47, 0xA2,
+0xA0, 0x35, 0x80, 0x34, 0xAD, 0xEF, 0x80, 0x34,
+0xED, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0xB0, 0x67, 0x83, 0x67, 0x09, 0xD3, 0xD1, 0x18,
+0x43, 0x09, 0x08, 0xD2, 0x09, 0x93, 0xA1, 0xA3,
+0xFF, 0x75, 0x66, 0x60, 0x40, 0xF0, 0xB5, 0xC0,
+0xE0, 0xA3, 0x0C, 0x6C, 0xFF, 0x6A, 0xEC, 0xEC,
+0x4C, 0xEC, 0x1A, 0x65, 0x0A, 0x96, 0x09, 0x2C,
+0x40, 0xA0, 0x40, 0x6C, 0x8C, 0xEA, 0x98, 0x67,
+0x8C, 0xEA, 0x03, 0x2A, 0xFC, 0x4A, 0x4C, 0xEF,
+0xE0, 0xC3, 0x40, 0xA3, 0x03, 0x6C, 0x0D, 0x6F,
+0x4C, 0xEC, 0x40, 0xF0, 0x5C, 0xA0, 0xEB, 0xEF,
+0x88, 0x34, 0xEC, 0xEA, 0x8D, 0xEA, 0x40, 0xF0,
+0x5C, 0xC0, 0x4B, 0x29, 0x40, 0xF0, 0x57, 0xA0,
+0x4E, 0xED, 0x08, 0x2D, 0x40, 0xA3, 0x07, 0x6C,
+0x52, 0x32, 0x8C, 0xEA, 0x40, 0xF0, 0x98, 0xA0,
+0x8E, 0xEA, 0x05, 0x22, 0x00, 0x6A, 0x80, 0xF0,
+0x4A, 0xC0, 0x80, 0xF0, 0x4B, 0xC0, 0x60, 0xF0,
+0x43, 0xA0, 0x7F, 0x6C, 0x8C, 0xEA, 0x60, 0xF0,
+0x43, 0xC0, 0x80, 0xF0, 0x50, 0xA0, 0x05, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEA, 0x80, 0xF0, 0x50, 0xC0,
+0xA0, 0xF0, 0x54, 0xA0, 0x02, 0x72, 0x0F, 0x61,
+0x00, 0x6E, 0x00, 0x6D, 0x0D, 0x6C, 0xD1, 0x18,
+0xBD, 0x6B, 0x09, 0xD3, 0x08, 0x93, 0x00, 0x6E,
+0x06, 0x05, 0x60, 0xF0, 0x5F, 0xC3, 0xD1, 0x18,
+0xE5, 0x6B, 0x01, 0x6C, 0x09, 0x93, 0x10, 0x94,
+0xD0, 0x67, 0xA3, 0x67, 0xD1, 0x18, 0x0F, 0x09,
+0x08, 0xD3, 0x08, 0x93, 0x10, 0x94, 0x04, 0xD0,
+0xF1, 0x67, 0xD1, 0x67, 0xD1, 0x18, 0x08, 0x09,
+0xA3, 0x67, 0x10, 0x92, 0x20, 0x5A, 0x04, 0x60,
+0x40, 0xA0, 0x08, 0x6B, 0x6D, 0xEA, 0x40, 0xC0,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0x60, 0xF0, 0x43, 0x80, 0x00, 0x52,
+0x47, 0x61, 0xB1, 0xA6, 0x50, 0xA6, 0x92, 0xA6,
+0xA0, 0x35, 0x4D, 0xED, 0x53, 0xA6, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x81, 0xA2, 0xC0, 0xA2, 0xA2, 0xA2,
+0x80, 0x34, 0xCD, 0xEC, 0xC3, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0xC0, 0x36,
+0x8D, 0xEE, 0x00, 0x6F, 0x00, 0x6D, 0xE8, 0x34,
+0x91, 0xE6, 0x21, 0xA4, 0x40, 0xA4, 0x20, 0x31,
+0x4D, 0xE9, 0x19, 0x65, 0x22, 0xA4, 0x58, 0x67,
+0x20, 0x31, 0x20, 0x31, 0x4D, 0xE9, 0x43, 0xA4,
+0x00, 0xF6, 0x40, 0x32, 0x2D, 0xEA, 0x08, 0x22,
+0x40, 0xA2, 0x04, 0x6C, 0x8C, 0xEA, 0xFF, 0x6C,
+0x8C, 0xEA, 0x02, 0x22, 0x01, 0x4D, 0x8C, 0xED,
+0x01, 0x4F, 0x20, 0x77, 0xE4, 0x61, 0x09, 0x2D,
+0x00, 0x6E, 0x0D, 0x6C, 0xD1, 0x18, 0xBD, 0x6B,
+0x23, 0x67, 0x08, 0x93, 0x60, 0xF0, 0x5F, 0xC3,
+0x71, 0x67, 0x10, 0x92, 0x20, 0x5A, 0x04, 0x60,
+0x40, 0xA0, 0x04, 0x6C, 0x8D, 0xEA, 0x40, 0xC0,
+0x60, 0xF0, 0x43, 0xA0, 0x80, 0x6C, 0x8B, 0xEC,
+0x8D, 0xEA, 0x60, 0xF0, 0x43, 0xC0, 0xF7, 0xF0,
+0x01, 0x6A, 0x81, 0xA3, 0x40, 0x32, 0x40, 0x32,
+0x08, 0xF0, 0x0C, 0x4A, 0x49, 0xE4, 0x40, 0xA2,
+0x60, 0xF0, 0x88, 0xA0, 0x60, 0xF0, 0x47, 0xC0,
+0x43, 0xEC, 0x02, 0x60, 0x60, 0xF0, 0x87, 0xC0,
+0x00, 0x6E, 0x06, 0x05, 0x01, 0x6C, 0x08, 0xD3,
+0xD1, 0x18, 0xE5, 0x6B, 0x01, 0x69, 0x81, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xFF, 0x75, 0x05, 0x67, 0x14, 0x60, 0x26, 0x67,
+0x00, 0xF0, 0x11, 0x06, 0xD1, 0x18, 0x7F, 0x06,
+0x04, 0x07, 0x5D, 0x67, 0x50, 0xA2, 0x11, 0x2A,
+0x80, 0xF0, 0x60, 0xA1, 0xFF, 0x73, 0x07, 0x60,
+0x5D, 0x67, 0x51, 0xA2, 0x0F, 0x6C, 0x8C, 0xEB,
+0x63, 0xEA, 0x01, 0x61, 0xFF, 0x68, 0x50, 0x67,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x80, 0xF0, 0x61, 0xA1, 0xEE, 0x17,
+0xFF, 0x63, 0x01, 0xD1, 0x00, 0xD0, 0x60, 0xF0,
+0x1D, 0xA6, 0x60, 0xF0, 0x3C, 0xA6, 0x01, 0x6B,
+0x0A, 0x32, 0x6C, 0xE9, 0xFF, 0x6F, 0x6C, 0xEA,
+0xEC, 0xE9, 0x38, 0x2A, 0x40, 0xF0, 0x90, 0xA6,
+0x40, 0xF0, 0x4F, 0xA6, 0x40, 0xF0, 0x71, 0xA6,
+0x80, 0x34, 0x4D, 0xEC, 0x40, 0xF0, 0x52, 0xA6,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x1A, 0x22, 0x08, 0x6B,
+0x6C, 0xE8, 0x42, 0x33, 0x4C, 0xEF, 0x42, 0x34,
+0x62, 0x33, 0x00, 0xF6, 0x42, 0x32, 0x15, 0x20,
+0x40, 0xF0, 0xEB, 0xC6, 0x40, 0xF0, 0x8C, 0xC6,
+0x40, 0xF0, 0x6D, 0xC6, 0x40, 0xF0, 0x4E, 0xC6,
+0x00, 0x6A, 0x40, 0xF0, 0x4F, 0xC6, 0x40, 0xF0,
+0x50, 0xC6, 0x40, 0xF0, 0x51, 0xC6, 0x40, 0xF0,
+0x52, 0xC6, 0x01, 0x91, 0x00, 0x90, 0x20, 0xE8,
+0x01, 0x63, 0x40, 0xF0, 0xE7, 0xC6, 0x40, 0xF0,
+0x88, 0xC6, 0x40, 0xF0, 0x69, 0xC6, 0x40, 0xF0,
+0x4A, 0xC6, 0xEA, 0x17, 0x03, 0x75, 0x1F, 0x61,
+0x54, 0x5C, 0xC0, 0xF0, 0x13, 0x61, 0x68, 0x44,
+0xC0, 0x4B, 0xEC, 0xEB, 0x20, 0x5B, 0x35, 0x61,
+0x40, 0xF0, 0x6C, 0xA6, 0x40, 0xF0, 0x4B, 0xA6,
+0x40, 0xF0, 0xAD, 0xA6, 0x60, 0x33, 0x4D, 0xEB,
+0x40, 0xF0, 0x4E, 0xA6, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0x68, 0x44, 0xA0, 0x4B, 0xEC, 0xEB, 0x20, 0x5B,
+0x69, 0x60, 0x01, 0x68, 0x2F, 0x10, 0x02, 0x75,
+0x80, 0xF0, 0x1D, 0x61, 0x2C, 0x5C, 0xA0, 0xF0,
+0x11, 0x61, 0x68, 0x44, 0xE8, 0x4B, 0xEC, 0xEB,
+0x20, 0x5B, 0x13, 0x61, 0x40, 0xF0, 0x6C, 0xA6,
+0x40, 0xF0, 0x4B, 0xA6, 0x40, 0xF0, 0xAD, 0xA6,
+0x60, 0x33, 0x4D, 0xEB, 0x40, 0xF0, 0x4E, 0xA6,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6,
+0x40, 0x32, 0x6D, 0xEA, 0x68, 0x44, 0xC8, 0x4B,
+0xDD, 0x17, 0x40, 0xF0, 0x88, 0xA6, 0x40, 0xF0,
+0x47, 0xA6, 0x40, 0xF0, 0xA9, 0xA6, 0x80, 0x34,
+0x4D, 0xEC, 0x40, 0xF0, 0x4A, 0xA6, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x00, 0x68, 0xC0, 0xF0, 0x09, 0x29,
+0x01, 0x6C, 0x84, 0xEB, 0x8C, 0xEA, 0x32, 0x22,
+0x60, 0xF0, 0xBD, 0xA6, 0x8F, 0xEC, 0x80, 0xF0,
+0x11, 0x20, 0x40, 0xF0, 0xEC, 0xA6, 0x40, 0xF0,
+0x4B, 0xA6, 0x40, 0xF0, 0x6D, 0xA6, 0xE0, 0x37,
+0x4D, 0xEF, 0x40, 0xF0, 0x4E, 0xA6, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEF, 0x00, 0xF6, 0x40, 0x32,
+0xED, 0xEA, 0x42, 0x33, 0x40, 0xF0, 0x70, 0xC6,
+0x62, 0x33, 0x40, 0xF0, 0x71, 0xC6, 0x00, 0xF6,
+0x42, 0x33, 0x40, 0xF0, 0x72, 0xC6, 0x08, 0x6B,
+0x6D, 0xED, 0x40, 0xF0, 0x4F, 0xC6, 0x60, 0xF0,
+0xBD, 0xC6, 0x8C, 0xEA, 0x42, 0x33, 0x40, 0xF0,
+0x4B, 0xC6, 0x40, 0xF0, 0x6C, 0xC6, 0x00, 0xF6,
+0x42, 0x32, 0x62, 0x33, 0x40, 0xF0, 0x6D, 0xC6,
+0x40, 0xF0, 0x4E, 0xC6, 0x40, 0xF0, 0x48, 0xA6,
+0x40, 0xF0, 0x87, 0xA6, 0x40, 0xF0, 0x69, 0xA6,
+0x40, 0x32, 0x8D, 0xEA, 0x40, 0xF0, 0x8A, 0xA6,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6,
+0x80, 0x34, 0x40, 0xF0, 0xAC, 0xA6, 0x4D, 0xEC,
+0x40, 0xF0, 0x4B, 0xA6, 0x40, 0xF0, 0x6D, 0xA6,
+0xA0, 0x35, 0x4D, 0xED, 0x40, 0xF0, 0x4E, 0xA6,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0x82, 0x33, 0xAD, 0xEA, 0x40, 0xF0,
+0x60, 0xC6, 0x62, 0x33, 0x40, 0xF0, 0x61, 0xC6,
+0x42, 0x33, 0x20, 0xF0, 0x9F, 0xC6, 0x40, 0xF0,
+0x43, 0xC6, 0x00, 0xF6, 0x82, 0x34, 0x40, 0xF0,
+0x64, 0xC6, 0x00, 0xF6, 0x42, 0x32, 0x62, 0x33,
+0x40, 0xF0, 0x82, 0xC6, 0x40, 0xF0, 0x65, 0xC6,
+0x40, 0xF0, 0x46, 0xC6, 0x32, 0x17, 0x01, 0x75,
+0x15, 0x61, 0x20, 0x5C, 0x15, 0x61, 0x40, 0xF0,
+0x6C, 0xA6, 0x40, 0xF0, 0x4B, 0xA6, 0x40, 0xF0,
+0xAD, 0xA6, 0x60, 0x33, 0x4D, 0xEB, 0x40, 0xF0,
+0x4E, 0xA6, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0x68, 0x44,
+0xE8, 0x4B, 0x44, 0x17, 0x0C, 0x5C, 0xAE, 0x60,
+0x40, 0xF0, 0x68, 0xA6, 0x40, 0xF0, 0x47, 0xA6,
+0x40, 0xF0, 0xA9, 0xA6, 0x60, 0x33, 0x4D, 0xEB,
+0x40, 0xF0, 0x4A, 0xA6, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0x64, 0x67, 0x63, 0x17, 0x40, 0xF0, 0xE8, 0xA6,
+0x40, 0xF0, 0x47, 0xA6, 0x40, 0xF0, 0x69, 0xA6,
+0xE0, 0x37, 0x4D, 0xEF, 0x40, 0xF0, 0x4A, 0xA6,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEF, 0x00, 0xF6,
+0x40, 0x32, 0xED, 0xEA, 0x42, 0x33, 0x40, 0xF0,
+0x70, 0xC6, 0x62, 0x33, 0x40, 0xF0, 0x71, 0xC6,
+0x00, 0xF6, 0x42, 0x33, 0x40, 0xF0, 0x72, 0xC6,
+0x09, 0x6B, 0x6B, 0xEB, 0x40, 0xF0, 0x4F, 0xC6,
+0x8C, 0xEA, 0x6C, 0xED, 0x42, 0x33, 0x40, 0xF0,
+0x68, 0xC6, 0x62, 0x33, 0x40, 0xF0, 0x47, 0xC6,
+0x60, 0xF0, 0xBD, 0xC6, 0x40, 0xF0, 0x69, 0xC6,
+0x00, 0xF6, 0x42, 0x32, 0x40, 0xF0, 0x4A, 0xC6,
+0x6D, 0x17, 0x01, 0x6D, 0xA4, 0xEB, 0xAC, 0xEA,
+0x7F, 0xF7, 0x08, 0x2A, 0x60, 0xF0, 0xFD, 0xA6,
+0x22, 0x20, 0x40, 0xF0, 0x8C, 0xA6, 0x40, 0xF0,
+0x4B, 0xA6, 0x40, 0xF0, 0x6D, 0xA6, 0x80, 0x34,
+0x4D, 0xEC, 0x40, 0xF0, 0x4E, 0xA6, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x42, 0x33, 0x40, 0xF0, 0x70, 0xC6,
+0x62, 0x33, 0x40, 0xF0, 0x71, 0xC6, 0x00, 0xF6,
+0x42, 0x33, 0x40, 0xF0, 0x72, 0xC6, 0x08, 0x6B,
+0x6D, 0xEF, 0x40, 0xF0, 0x4F, 0xC6, 0x60, 0xF0,
+0xFD, 0xC6, 0xAD, 0xEA, 0x37, 0x17, 0x40, 0xF0,
+0x68, 0xA6, 0x40, 0xF0, 0x47, 0xA6, 0x60, 0x33,
+0x4D, 0xEB, 0x40, 0xF0, 0x49, 0xA6, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xEA, 0x40, 0xF0, 0x6A, 0xA6,
+0x00, 0xF6, 0x60, 0x33, 0x4D, 0xEB, 0x62, 0x32,
+0x40, 0xF0, 0x50, 0xC6, 0x42, 0x32, 0x40, 0xF0,
+0x51, 0xC6, 0x00, 0xF6, 0x62, 0x32, 0x6D, 0xED,
+0x40, 0xF0, 0x52, 0xC6, 0x09, 0x6A, 0x4B, 0xEA,
+0x40, 0xF0, 0x6F, 0xC6, 0xA2, 0x33, 0x4C, 0xEF,
+0x40, 0xF0, 0x68, 0xC6, 0x62, 0x33, 0x60, 0xF0,
+0xFD, 0xC6, 0x40, 0xF0, 0xA7, 0xC6, 0x40, 0xF0,
+0x69, 0xC6, 0x00, 0xF6, 0xA2, 0x32, 0xA6, 0x17,
+0xFF, 0x74, 0xC4, 0x67, 0x80, 0xF0, 0x0A, 0x60,
+0xFD, 0x63, 0x05, 0x62, 0x40, 0xF0, 0x9D, 0xA5,
+0x1F, 0x6A, 0xFF, 0x6F, 0x4C, 0xEC, 0xD1, 0x18,
+0xAF, 0x06, 0xEC, 0xEC, 0x03, 0x72, 0x24, 0x61,
+0x54, 0x5E, 0x78, 0x61, 0x68, 0x46, 0xC0, 0x4B,
+0xEC, 0xEB, 0x20, 0x5B, 0x39, 0x61, 0x40, 0xF0,
+0x6C, 0xA5, 0x40, 0xF0, 0x4B, 0xA5, 0x40, 0xF0,
+0x8D, 0xA5, 0x60, 0x33, 0x4D, 0xEB, 0x40, 0xF0,
+0x4E, 0xA5, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0x68, 0x46,
+0xA0, 0x4B, 0xEC, 0xEB, 0x20, 0x5B, 0x60, 0x60,
+0x01, 0x6C, 0x84, 0xEB, 0x8C, 0xEA, 0x5C, 0x22,
+0x05, 0x97, 0x46, 0x67, 0x00, 0xEF, 0x03, 0x63,
+0x02, 0x72, 0x2B, 0x61, 0x2C, 0x5E, 0x52, 0x61,
+0x68, 0x46, 0xE8, 0x4B, 0xEC, 0xEB, 0x20, 0x5B,
+0x13, 0x61, 0x40, 0xF0, 0x6C, 0xA5, 0x40, 0xF0,
+0x4B, 0xA5, 0x40, 0xF0, 0x8D, 0xA5, 0x60, 0x33,
+0x4D, 0xEB, 0x40, 0xF0, 0x4E, 0xA5, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6, 0x40, 0x32,
+0x6D, 0xEA, 0x68, 0x46, 0xC8, 0x4B, 0xD9, 0x17,
+0x40, 0xF0, 0x88, 0xA5, 0x40, 0xF0, 0x47, 0xA5,
+0x40, 0xF0, 0xE9, 0xA5, 0x80, 0x34, 0x4D, 0xEC,
+0x40, 0xF0, 0x4A, 0xA5, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0xCB, 0x17, 0x01, 0x72, 0x27, 0x61, 0x20, 0x5E,
+0x13, 0x61, 0x40, 0xF0, 0x6C, 0xA5, 0x40, 0xF0,
+0x4B, 0xA5, 0x40, 0xF0, 0x8D, 0xA5, 0x60, 0x33,
+0x4D, 0xEB, 0x40, 0xF0, 0x4E, 0xA5, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6, 0x40, 0x32,
+0x6D, 0xEA, 0x68, 0x46, 0xE8, 0x4B, 0xB1, 0x17,
+0x40, 0xF0, 0x68, 0xA5, 0x40, 0xF0, 0x47, 0xA5,
+0x40, 0xF0, 0x89, 0xA5, 0x60, 0x33, 0x4D, 0xEB,
+0x40, 0xF0, 0x4A, 0xA5, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0x66, 0x67, 0xA2, 0x17, 0x0C, 0x5E, 0xEC, 0x61,
+0xFF, 0x6E, 0xA2, 0x17, 0x20, 0xE8, 0x44, 0x67,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x26, 0x67, 0xC0, 0xA4, 0xE4, 0x67, 0x61, 0xA4,
+0x03, 0x6C, 0x3C, 0x65, 0xCA, 0x32, 0x99, 0x67,
+0x01, 0x75, 0xFF, 0x68, 0x8C, 0xEA, 0x80, 0xF0,
+0x1A, 0x61, 0x44, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x87, 0xF7, 0x14, 0x4A,
+0x0B, 0x73, 0x49, 0xE3, 0x80, 0xF0, 0x17, 0x61,
+0x40, 0xF0, 0x9D, 0xA1, 0x1F, 0x6A, 0x06, 0xD7,
+0x4C, 0xEC, 0xD1, 0x18, 0xAF, 0x06, 0x0C, 0xEC,
+0x79, 0x67, 0x0D, 0x6C, 0x06, 0x97, 0x6C, 0xEA,
+0x8B, 0xEC, 0x48, 0x33, 0x8C, 0xEE, 0x6D, 0xEE,
+0x0C, 0xEA, 0xC0, 0xC7, 0x00, 0x6D, 0x00, 0x6E,
+0xD1, 0x18, 0x71, 0x06, 0x82, 0x67, 0x06, 0x97,
+0xB1, 0x67, 0x82, 0x67, 0xD1, 0x18, 0xF6, 0x0A,
+0x07, 0x67, 0x41, 0xC0, 0x40, 0xA0, 0x03, 0x6B,
+0xF0, 0x67, 0x4A, 0x32, 0x6C, 0xEA, 0x03, 0x72,
+0x00, 0x6B, 0x01, 0x61, 0x03, 0x6B, 0x40, 0xA7,
+0x71, 0x6C, 0x8B, 0xEC, 0x70, 0x33, 0x8C, 0xEA,
+0x6D, 0xEA, 0x40, 0xC7, 0x60, 0xF0, 0x43, 0xA1,
+0x19, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA, 0x60, 0xF0,
+0x43, 0xC1, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x00, 0xEF, 0x06, 0x63, 0x01, 0x72, 0x3B, 0x61,
+0xF4, 0x4B, 0x0C, 0xEB, 0x64, 0x32, 0x69, 0xE2,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x67, 0xF6, 0x08, 0x4B, 0x61, 0xE2, 0x01, 0x6A,
+0x4B, 0xEA, 0x7D, 0x67, 0x50, 0xC3, 0x51, 0xC3,
+0x52, 0xC3, 0x43, 0x40, 0x3A, 0x65, 0x00, 0x6B,
+0x80, 0xA0, 0x2C, 0x74, 0x06, 0x61, 0x40, 0xA7,
+0x0C, 0x6D, 0xAC, 0xEA, 0x04, 0x72, 0x01, 0x61,
+0x0C, 0x6C, 0xB1, 0x67, 0x07, 0xD7, 0xD1, 0x18,
+0xF6, 0x0A, 0x06, 0xD3, 0xFF, 0x72, 0x06, 0x93,
+0x07, 0x97, 0x06, 0x60, 0x9D, 0x67, 0x71, 0xE4,
+0x50, 0xC4, 0x01, 0x4B, 0xFF, 0x6A, 0x4C, 0xEB,
+0x01, 0x48, 0x59, 0x67, 0x0A, 0xEA, 0xE4, 0x61,
+0x60, 0xF0, 0x43, 0xA1, 0x55, 0x2B, 0xE7, 0x4B,
+0x6C, 0xEA, 0x60, 0xF0, 0x43, 0xC1, 0x01, 0x6A,
+0x4B, 0xEA, 0x41, 0xC7, 0xBE, 0x17, 0x02, 0x72,
+0x0B, 0x61, 0xD4, 0x4B, 0x0C, 0xEB, 0x64, 0x32,
+0x69, 0xE2, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xC7, 0xF6, 0x08, 0x4B, 0xC2, 0x17,
+0xAC, 0x4B, 0x0C, 0xEB, 0x64, 0x32, 0x69, 0xE2,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x47, 0xF7, 0x00, 0x4B, 0xB7, 0x17, 0x0E, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x87, 0xF7, 0x08, 0x4A, 0x49, 0xE3, 0x60, 0xA7,
+0x0D, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB, 0x60, 0xC7,
+0x40, 0xA2, 0x7A, 0x17, 0x01, 0x72, 0x0B, 0x61,
+0xF4, 0x4B, 0x0C, 0xEB, 0x64, 0x32, 0x69, 0xE2,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x47, 0xF5, 0x08, 0x4B, 0x9B, 0x17, 0x02, 0x72,
+0x0C, 0x61, 0xD4, 0x4B, 0x0C, 0xEB, 0x64, 0x34,
+0x71, 0xE4, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xA7, 0xF5, 0x08, 0x4B, 0x61, 0xE4,
+0x8E, 0x17, 0xAC, 0x4B, 0x6C, 0xE8, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x04, 0x34, 0x60, 0x33,
+0x11, 0xE4, 0x27, 0xF6, 0x00, 0x4B, 0xF3, 0x17,
+0x60, 0xF0, 0x66, 0xA1, 0x08, 0x6C, 0xFF, 0x6D,
+0x8C, 0xEB, 0xAC, 0xEB, 0x34, 0x23, 0x70, 0x6B,
+0x60, 0xF0, 0x85, 0xA1, 0x1B, 0x65, 0x78, 0x67,
+0x8C, 0xEB, 0xAC, 0xEB, 0x10, 0x73, 0x2B, 0x61,
+0xA0, 0xF0, 0x74, 0xA1, 0x01, 0x73, 0x27, 0x60,
+0x60, 0xF0, 0x94, 0xA1, 0x02, 0x6B, 0x6C, 0xEC,
+0xAC, 0xEC, 0x21, 0x2C, 0x60, 0xA7, 0x0C, 0x6C,
+0x8C, 0xEB, 0xAC, 0xEB, 0x04, 0x73, 0x43, 0x60,
+0x08, 0x73, 0x48, 0x60, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x68, 0xF0, 0x18, 0x4C,
+0xBD, 0x67, 0xB1, 0xA5, 0xFF, 0x75, 0xC5, 0x67,
+0x0E, 0x60, 0x0C, 0x73, 0x42, 0x61, 0xC8, 0x45,
+0xC0, 0x4E, 0x80, 0x9C, 0x01, 0x6B, 0x64, 0xEE,
+0x8C, 0xEB, 0x05, 0x23, 0x7D, 0x67, 0x70, 0xA3,
+0x9D, 0x67, 0xB0, 0xC4, 0x71, 0xC4, 0x19, 0x6D,
+0xAB, 0xED, 0xAC, 0xEA, 0x60, 0xF0, 0x43, 0xC1,
+0x7D, 0x67, 0x71, 0xA3, 0x9D, 0x67, 0x92, 0xA4,
+0xFF, 0x73, 0x39, 0x60, 0xFF, 0x74, 0xAC, 0xEA,
+0x2D, 0x60, 0x40, 0xF0, 0x9A, 0xC1, 0x40, 0xF0,
+0x79, 0xC1, 0x60, 0xA7, 0x03, 0x6C, 0x8C, 0xEB,
+0x80, 0xF0, 0x67, 0xC1, 0x80, 0xF0, 0x66, 0xC1,
+0x10, 0x6B, 0x6D, 0xEA, 0x60, 0xF0, 0x43, 0xC1,
+0x5D, 0x67, 0x50, 0xA2, 0xA0, 0xA7, 0x0C, 0x5A,
+0x41, 0xC7, 0x26, 0x60, 0x0D, 0x6A, 0x4B, 0xEA,
+0xAC, 0xEA, 0x40, 0xC7, 0x0E, 0x17, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x88, 0xF0,
+0x08, 0x4C, 0xBE, 0x17, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x88, 0xF0, 0x00, 0x4C,
+0xB7, 0x17, 0x08, 0x73, 0xBE, 0x61, 0xC8, 0x45,
+0xE8, 0x4E, 0xBB, 0x17, 0x40, 0xF0, 0x79, 0xC1,
+0x60, 0xA7, 0x03, 0x6C, 0x8C, 0xEB, 0x80, 0xF0,
+0x66, 0xC1, 0x08, 0x6B, 0xD6, 0x17, 0xFF, 0x74,
+0xD7, 0x60, 0x40, 0xF0, 0x99, 0xC1, 0xF4, 0x17,
+0x40, 0xF0, 0x5D, 0xA1, 0x1F, 0x6C, 0xD1, 0x18,
+0xAF, 0x06, 0x4C, 0xEC, 0x03, 0x6B, 0x4C, 0xEB,
+0x0D, 0x6A, 0x4B, 0xEA, 0x68, 0x33, 0xAC, 0xEA,
+0x6D, 0xEA, 0xCF, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x11, 0xD5, 0x24, 0x67,
+0x40, 0xF0, 0x9D, 0xA6, 0x1F, 0x6A, 0xFF, 0x6D,
+0x4C, 0xEC, 0xD1, 0x18, 0xAF, 0x06, 0xAC, 0xEC,
+0x06, 0xD2, 0x56, 0xA6, 0x03, 0x22, 0x00, 0x6A,
+0x3A, 0x65, 0x18, 0x10, 0x60, 0xA1, 0x3A, 0x65,
+0x03, 0x6A, 0x83, 0x67, 0x4C, 0xEC, 0xAC, 0xEC,
+0x09, 0xD4, 0x40, 0xF0, 0x9C, 0xA6, 0x4C, 0xEC,
+0xAC, 0xEC, 0x05, 0xD4, 0x05, 0x97, 0x09, 0x94,
+0x82, 0xEF, 0x0E, 0x60, 0x87, 0x67, 0x4C, 0xEC,
+0x44, 0x67, 0x04, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB,
+0x6D, 0xEA, 0x40, 0xC1, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x59, 0x67, 0x00, 0xEF, 0x08, 0x63,
+0x05, 0x93, 0xDD, 0x23, 0x06, 0x93, 0xDB, 0x23,
+0x60, 0xA1, 0x6A, 0x33, 0x6C, 0xEA, 0xAC, 0xEA,
+0x61, 0xA1, 0x79, 0x2A, 0x04, 0x73, 0xD3, 0x61,
+0x01, 0x5A, 0x58, 0x67, 0x04, 0xD2, 0x11, 0x92,
+0x01, 0x72, 0x80, 0xF0, 0x1E, 0x60, 0x09, 0x6A,
+0x08, 0xD2, 0x01, 0x6C, 0x84, 0x32, 0x89, 0xE2,
+0x44, 0x32, 0x00, 0x6D, 0x0A, 0xD2, 0x07, 0xD5,
+0x00, 0x6A, 0x06, 0x95, 0x07, 0x97, 0x01, 0x75,
+0x0A, 0x95, 0xF5, 0xE5, 0x80, 0xF0, 0x11, 0x61,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x27, 0xF4, 0x0C, 0x48, 0xA1, 0xE0, 0xA0, 0xA8,
+0xFF, 0x6F, 0xAC, 0xEF, 0xEA, 0xEB, 0xA0, 0xF0,
+0x0E, 0x61, 0x09, 0x97, 0xA2, 0x35, 0xEE, 0xED,
+0xA0, 0xF0, 0x09, 0x2D, 0x11, 0x93, 0x4B, 0xEA,
+0x44, 0x32, 0x01, 0x73, 0x80, 0xF0, 0x0A, 0x61,
+0x06, 0x4A, 0x41, 0xE0, 0x44, 0x40, 0x06, 0xD2,
+0x80, 0xA8, 0xFF, 0x6F, 0xA6, 0x67, 0x82, 0x33,
+0xEC, 0xEC, 0x09, 0xD3, 0x08, 0xD6, 0xD1, 0x18,
+0xF6, 0x0A, 0x07, 0xD7, 0x05, 0x94, 0x09, 0x93,
+0x08, 0x96, 0x62, 0xEC, 0x0F, 0x61, 0xFF, 0x72,
+0x0D, 0x60, 0x99, 0x67, 0x07, 0x97, 0x74, 0x2C,
+0x41, 0xC1, 0x03, 0x6A, 0x6C, 0xEA, 0x60, 0xA1,
+0xFC, 0x4C, 0x8C, 0xEB, 0x4D, 0xEB, 0x01, 0x6A,
+0x60, 0xC1, 0x3A, 0x65, 0x06, 0x92, 0x02, 0x48,
+0x0A, 0xEA, 0xDE, 0x61, 0x59, 0x67, 0x7F, 0xF7,
+0x1A, 0x22, 0xD1, 0x18, 0xB5, 0x06, 0x81, 0xA1,
+0x03, 0x6B, 0x6C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA,
+0x80, 0xA1, 0x0D, 0x6D, 0x4C, 0xEB, 0xAB, 0xED,
+0x68, 0x33, 0xAC, 0xEC, 0x8D, 0xEB, 0x60, 0xC1,
+0x04, 0x93, 0x84, 0x23, 0x03, 0x72, 0x00, 0x6B,
+0x01, 0x61, 0x03, 0x6B, 0x40, 0xA1, 0x71, 0x6C,
+0x8B, 0xEC, 0x70, 0x33, 0x8C, 0xEA, 0x6D, 0xEA,
+0x40, 0xC1, 0x04, 0x92, 0x5D, 0x17, 0x01, 0x72,
+0x0A, 0x61, 0x88, 0x43, 0xF6, 0x4C, 0x8C, 0xED,
+0x02, 0x5D, 0x5F, 0xF7, 0x14, 0x61, 0x1A, 0x5B,
+0x7F, 0xF7, 0x1E, 0x61, 0x50, 0x17, 0x02, 0x72,
+0x08, 0x61, 0x88, 0x43, 0xD6, 0x4C, 0x8C, 0xED,
+0x04, 0x5D, 0x5F, 0xF7, 0x08, 0x61, 0x3C, 0x5B,
+0xF3, 0x17, 0x48, 0x43, 0xAE, 0x4A, 0xAC, 0xEA,
+0x06, 0x5A, 0x5F, 0xF7, 0x00, 0x61, 0x66, 0x5B,
+0x3F, 0xF7, 0x1D, 0x60, 0x40, 0xA6, 0x40, 0x6C,
+0x8C, 0xEA, 0x4C, 0xED, 0x08, 0x25, 0x48, 0x43,
+0xB3, 0x4A, 0x0B, 0x5A, 0x3F, 0xF7, 0x13, 0x61,
+0x61, 0x5B, 0x3F, 0xF7, 0x10, 0x60, 0x00, 0x6A,
+0x5D, 0x17, 0x08, 0x6A, 0x00, 0x6C, 0x08, 0xD2,
+0x61, 0x17, 0x06, 0x97, 0x02, 0x77, 0x07, 0x61,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xE7, 0xF3, 0x14, 0x48, 0x6B, 0x17, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0xA7, 0xF3,
+0x1C, 0x48, 0x64, 0x17, 0xFA, 0x4A, 0x75, 0x17,
+0x60, 0xF0, 0x83, 0xA6, 0x03, 0x6D, 0x8E, 0x34,
+0xAC, 0xEC, 0xEC, 0xEC, 0x9D, 0xE6, 0x80, 0xF0,
+0x66, 0xC7, 0x40, 0xF0, 0x59, 0xC7, 0x60, 0xF0,
+0x43, 0xA6, 0x01, 0x4C, 0x19, 0x6B, 0xAC, 0xEC,
+0x6B, 0xEB, 0x8C, 0x34, 0x6C, 0xEA, 0x4D, 0xEC,
+0x60, 0xF0, 0x83, 0xC6, 0x7F, 0x17, 0x07, 0x97,
+0x01, 0x4A, 0xFF, 0x6D, 0xAC, 0xEA, 0x02, 0x4F,
+0x03, 0x72, 0x07, 0xD7, 0x3F, 0xF7, 0x15, 0x61,
+0x08, 0x92, 0x01, 0x4C, 0xAC, 0xEC, 0x8A, 0xEA,
+0x3F, 0xF7, 0x08, 0x61, 0xF0, 0x16, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0xD1, 0x04, 0xD0, 0x40, 0xA4,
+0x0C, 0x6B, 0xFF, 0x6F, 0x6C, 0xEA, 0xEC, 0xEA,
+0x06, 0xD4, 0x05, 0x2A, 0x00, 0x6A, 0x05, 0x91,
+0x04, 0x90, 0x20, 0xE8, 0x03, 0x63, 0x60, 0xF0,
+0x66, 0xA6, 0x10, 0x68, 0x0C, 0xEB, 0xEC, 0xEB,
+0x02, 0x23, 0x0C, 0x72, 0xF3, 0x60, 0x60, 0xF0,
+0x63, 0xA6, 0x04, 0x6F, 0xEC, 0xEB, 0x02, 0x2B,
+0x0C, 0x72, 0xEC, 0x61, 0xA0, 0xF0, 0x74, 0xA6,
+0x01, 0x73, 0xE8, 0x60, 0xEB, 0xA6, 0xFF, 0x6C,
+0x07, 0x6B, 0x27, 0x67, 0x04, 0x67, 0x6C, 0xE9,
+0x2C, 0xE8, 0x38, 0x65, 0x1C, 0x65, 0x08, 0x68,
+0x99, 0x67, 0x0B, 0xE8, 0x06, 0x24, 0xFF, 0x49,
+0x6C, 0xE9, 0x0C, 0xEF, 0xED, 0xE9, 0x2B, 0xC6,
+0xD5, 0x17, 0x06, 0x94, 0x0C, 0xEF, 0x03, 0x68,
+0x20, 0xA4, 0x0D, 0xEF, 0x98, 0x67, 0x32, 0x31,
+0x2C, 0xEB, 0x08, 0x72, 0x8C, 0xEB, 0xEB, 0xC6,
+0x05, 0x61, 0x02, 0x75, 0x30, 0x61, 0xC6, 0x23,
+0x00, 0x6A, 0x31, 0x10, 0x04, 0x72, 0xF9, 0x60,
+0x0C, 0x72, 0xC0, 0x61, 0x02, 0x5D, 0x38, 0x61,
+0x05, 0x69, 0x01, 0x6F, 0x04, 0x6A, 0xEB, 0xE2,
+0xFF, 0x68, 0x0C, 0xEA, 0x3A, 0x65, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0x67, 0xF4,
+0x04, 0x48, 0xE4, 0x32, 0x09, 0xE2, 0x40, 0xAA,
+0x02, 0xD0, 0x6E, 0xEA, 0x0A, 0x2A, 0x02, 0x5D,
+0x39, 0x67, 0x01, 0x61, 0x27, 0x67, 0x00, 0x6A,
+0x07, 0x67, 0x01, 0xD2, 0x01, 0x92, 0x2A, 0xEA,
+0x1E, 0x61, 0xFF, 0x6A, 0x99, 0x67, 0x01, 0x4F,
+0x4C, 0xEF, 0xFF, 0x4C, 0x4C, 0xEC, 0x23, 0xEF,
+0x3C, 0x65, 0xE1, 0x61, 0x97, 0x17, 0x02, 0x5D,
+0x95, 0x60, 0x01, 0x6A, 0x93, 0x2B, 0x07, 0x6B,
+0x6C, 0xEA, 0x06, 0x93, 0x71, 0x6D, 0xAB, 0xED,
+0x60, 0xA3, 0x50, 0x32, 0xAC, 0xEB, 0x6D, 0xEA,
+0x06, 0x93, 0x40, 0xC3, 0x01, 0x6A, 0x87, 0x17,
+0x04, 0x69, 0x00, 0x6F, 0xC7, 0x17, 0x02, 0x5D,
+0x19, 0x60, 0x01, 0x48, 0xFF, 0x6A, 0x02, 0x94,
+0x4C, 0xE8, 0x04, 0x32, 0x49, 0xE4, 0x40, 0xAA,
+0x05, 0x72, 0x00, 0xD2, 0x11, 0x61, 0x60, 0xF0,
+0x85, 0xA6, 0x02, 0x6A, 0x86, 0x34, 0x4C, 0xEC,
+0x1C, 0x65, 0x16, 0x61, 0x01, 0x92, 0x01, 0x4A,
+0x1A, 0x65, 0x98, 0x67, 0xFF, 0x6A, 0x4C, 0xEC,
+0x01, 0xD4, 0xC4, 0x17, 0xFF, 0x48, 0xE6, 0x17,
+0x00, 0x92, 0x01, 0x72, 0x06, 0x61, 0x60, 0xF0,
+0x45, 0xA6, 0x46, 0x32, 0x82, 0x67, 0x01, 0x6A,
+0xEA, 0x17, 0x00, 0x92, 0xFF, 0x72, 0xE9, 0x17,
+0x00, 0x92, 0xFF, 0x6B, 0x6C, 0xEA, 0xC3, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x0C, 0xD4, 0x0E, 0xD6, 0x40, 0xF0, 0x53, 0xA7,
+0x05, 0x67, 0x05, 0x6D, 0x04, 0xD2, 0x01, 0x6A,
+0xCC, 0xEA, 0x48, 0x34, 0x80, 0xF0, 0x50, 0xA7,
+0xAB, 0xED, 0x27, 0x67, 0xAC, 0xEA, 0x8D, 0xEA,
+0x80, 0xF0, 0x50, 0xC7, 0x01, 0x6A, 0x4B, 0xEA,
+0x40, 0xF0, 0x59, 0xC7, 0x40, 0xF0, 0x5A, 0xC7,
+0x80, 0xF0, 0x46, 0xC7, 0x80, 0xF0, 0x47, 0xC7,
+0x60, 0xF0, 0x43, 0xA7, 0x19, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEA, 0x60, 0xF0, 0x43, 0xC7, 0x41, 0xA0,
+0x04, 0x93, 0x4A, 0xEB, 0x69, 0x61, 0xC7, 0x67,
+0x01, 0x6D, 0xD1, 0x18, 0x88, 0x0C, 0x90, 0x67,
+0x00, 0xF1, 0x02, 0x22, 0x00, 0x6A, 0x01, 0x6B,
+0x0E, 0xD2, 0x80, 0xA0, 0xA1, 0xA0, 0x03, 0x6A,
+0x8A, 0x34, 0x4C, 0xEC, 0xD1, 0x67, 0xD1, 0x18,
+0x08, 0x0A, 0x04, 0xD3, 0x41, 0xC0, 0x60, 0xF0,
+0x43, 0xA1, 0x18, 0x6C, 0xFF, 0x6B, 0x8C, 0xEA,
+0x6C, 0xEA, 0x04, 0x93, 0xA0, 0xF0, 0x06, 0x2A,
+0x40, 0xF0, 0xBA, 0xA1, 0x00, 0x6A, 0xFF, 0x75,
+0x05, 0x61, 0x01, 0x6A, 0x4B, 0xEA, 0x80, 0xF0,
+0x47, 0xC1, 0x01, 0x6A, 0x40, 0xF0, 0x99, 0xA1,
+0xFF, 0x74, 0x0F, 0x61, 0x80, 0xF0, 0x87, 0xA1,
+0x01, 0x4A, 0x40, 0xF0, 0xB9, 0xC1, 0x80, 0xF0,
+0x86, 0xC1, 0x01, 0x6C, 0x8B, 0xEC, 0x40, 0xF0,
+0x9A, 0xC1, 0x80, 0xF0, 0x87, 0xC1, 0xFF, 0x6C,
+0x8C, 0xEA, 0x81, 0xA0, 0xFF, 0x74, 0xA0, 0xF0,
+0x02, 0x61, 0x40, 0xF0, 0x99, 0xA1, 0x03, 0x6D,
+0x04, 0x6E, 0x81, 0xC0, 0x80, 0xF0, 0x86, 0xA1,
+0xCB, 0xEE, 0x01, 0x4A, 0x8C, 0xED, 0x80, 0xA0,
+0xCC, 0xEC, 0xAD, 0xEC, 0x80, 0xC0, 0x40, 0xF0,
+0x9A, 0xA1, 0x40, 0xF0, 0x99, 0xC1, 0x80, 0xF0,
+0x87, 0xA1, 0x80, 0xF0, 0x86, 0xC1, 0x01, 0x6C,
+0x8B, 0xEC, 0x40, 0xF0, 0x9A, 0xC1, 0x80, 0xF0,
+0x87, 0xC1, 0xFF, 0x6C, 0x8C, 0xEA, 0x60, 0xF0,
+0x83, 0xA1, 0x19, 0x6D, 0xAB, 0xED, 0x01, 0x72,
+0xAC, 0xEC, 0xA0, 0xF0, 0x01, 0x61, 0x08, 0x6A,
+0x4D, 0xEC, 0x60, 0xF0, 0x83, 0xC1, 0x82, 0x10,
+0x04, 0x93, 0x43, 0xEB, 0x05, 0x60, 0x5D, 0x67,
+0x50, 0xA2, 0x00, 0x6B, 0x41, 0xC0, 0x99, 0x17,
+0xC7, 0x67, 0x01, 0x6D, 0xD1, 0x18, 0x88, 0x0C,
+0x90, 0x67, 0x90, 0x2A, 0xD1, 0x67, 0x01, 0x6D,
+0xD1, 0x18, 0xF7, 0x0B, 0x90, 0x67, 0x02, 0x22,
+0x01, 0x6B, 0x8B, 0x17, 0xA0, 0xA0, 0x40, 0xF0,
+0x9C, 0xA1, 0x03, 0x6E, 0x45, 0x67, 0xFF, 0x6B,
+0xCC, 0xEA, 0xCC, 0xEC, 0x6C, 0xEA, 0x6C, 0xEC,
+0x82, 0xEA, 0x0B, 0x60, 0x81, 0xA0, 0x0C, 0x5C,
+0x08, 0x61, 0x01, 0x4A, 0x4C, 0xEE, 0x04, 0x6A,
+0x4B, 0xEA, 0x4C, 0xED, 0xCD, 0xED, 0xA0, 0xC0,
+0xE7, 0x17, 0x6C, 0x6B, 0x41, 0xA0, 0x90, 0x67,
+0xD1, 0x67, 0x01, 0x6D, 0x06, 0xD3, 0xD1, 0x18,
+0x3E, 0x0B, 0x05, 0xD2, 0x81, 0xA0, 0xFF, 0x74,
+0xDB, 0x61, 0x04, 0x93, 0x05, 0x92, 0x63, 0xEA,
+0x06, 0x93, 0x04, 0x61, 0x5D, 0x67, 0x50, 0xA2,
+0x41, 0xC0, 0xD2, 0x17, 0x81, 0x42, 0x06, 0xD4,
+0x06, 0x02, 0x80, 0xA2, 0x05, 0xD3, 0xD1, 0x18,
+0xB5, 0x06, 0x81, 0xC0, 0x03, 0x6C, 0x4C, 0xEC,
+0x40, 0xA0, 0x05, 0x93, 0x0D, 0x6E, 0xCB, 0xEE,
+0x88, 0x34, 0xCC, 0xEA, 0x8D, 0xEA, 0xFF, 0x4B,
+0x40, 0xC0, 0x05, 0xD3, 0x05, 0x02, 0x60, 0xA2,
+0xD5, 0x2B, 0xBA, 0x17, 0x80, 0xA0, 0x03, 0x6A,
+0x40, 0xF0, 0xB9, 0xA1, 0x8A, 0x34, 0x4C, 0xEC,
+0xFF, 0x6A, 0xD1, 0x67, 0x4C, 0xEC, 0xD1, 0x18,
+0x08, 0x0A, 0x04, 0xD3, 0x40, 0xF0, 0x59, 0xC1,
+0x80, 0xA0, 0x03, 0x6A, 0x40, 0xF0, 0xBA, 0xA1,
+0x8A, 0x34, 0x4C, 0xEC, 0xFF, 0x6A, 0xD1, 0x67,
+0xD1, 0x18, 0x08, 0x0A, 0x4C, 0xEC, 0x40, 0xF0,
+0x5A, 0xC1, 0x04, 0x93, 0x3D, 0x17, 0x7F, 0xF7,
+0x1A, 0x2A, 0x60, 0xF0, 0x43, 0xA1, 0x19, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEA, 0x10, 0x6C, 0x8D, 0xEA,
+0x60, 0xF0, 0x43, 0xC1, 0x0E, 0x92, 0x01, 0x72,
+0x08, 0x61, 0x60, 0xF0, 0x43, 0xA1, 0x19, 0x6C,
+0x8B, 0xEC, 0x8C, 0xEA, 0x60, 0xF0, 0x43, 0xC1,
+0x00, 0x6B, 0x60, 0xF0, 0x42, 0xA1, 0x0F, 0x6C,
+0xF1, 0x67, 0x8C, 0xEA, 0x10, 0x6C, 0x8D, 0xEA,
+0x60, 0xF0, 0x42, 0xC1, 0x0C, 0x94, 0xC3, 0x67,
+0xD1, 0x18, 0x7B, 0x09, 0xB0, 0x67, 0x04, 0x10,
+0x02, 0x72, 0x60, 0xF0, 0x83, 0xC1, 0xE2, 0x60,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x10, 0xD6, 0x0E, 0xD4,
+0x15, 0xF0, 0x08, 0x4A, 0x25, 0x67, 0xB1, 0xA2,
+0x90, 0xA2, 0xD2, 0xA2, 0xA0, 0x35, 0x8D, 0xED,
+0x93, 0xA2, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC, 0xA1, 0xA4,
+0x40, 0xA4, 0xC2, 0xA4, 0xA0, 0x35, 0x4D, 0xED,
+0x43, 0xA4, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x80, 0xF0,
+0xA5, 0xA2, 0x80, 0xF0, 0x86, 0xA2, 0x80, 0xF0,
+0xC4, 0xA2, 0x80, 0xF0, 0x47, 0xA2, 0xA0, 0x35,
+0x80, 0x34, 0xAD, 0xEE, 0x80, 0x34, 0xCD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x09, 0xD2,
+0x40, 0xF0, 0x54, 0xA7, 0x10, 0x93, 0x05, 0x6E,
+0x06, 0xD2, 0x01, 0x6A, 0x6C, 0xEA, 0x48, 0x34,
+0x80, 0xF0, 0x50, 0xA7, 0xCB, 0xEE, 0x40, 0xF0,
+0xB3, 0xA7, 0xCC, 0xEA, 0x8D, 0xEA, 0x80, 0xF0,
+0x50, 0xC7, 0x01, 0x6A, 0x4B, 0xEA, 0x40, 0xF0,
+0x59, 0xC7, 0x40, 0xF0, 0x5A, 0xC7, 0x80, 0xF0,
+0x46, 0xC7, 0x80, 0xF0, 0x47, 0xC7, 0x60, 0xF0,
+0x43, 0xA7, 0x19, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA,
+0x60, 0xF0, 0x43, 0xC7, 0x41, 0xA1, 0x07, 0x67,
+0x43, 0xED, 0x01, 0x60, 0xA1, 0xC1, 0x41, 0xA1,
+0x06, 0x93, 0x4A, 0xEB, 0x34, 0x61, 0x40, 0xA1,
+0x03, 0x6C, 0x8C, 0xEA, 0x48, 0x2A, 0xD0, 0x67,
+0x02, 0x6D, 0xD1, 0x18, 0x88, 0x0C, 0x91, 0x67,
+0x25, 0x22, 0x01, 0x6E, 0x09, 0x92, 0xFF, 0x6D,
+0x60, 0xF0, 0x91, 0xA2, 0x08, 0x6A, 0x4C, 0xEC,
+0xAC, 0xEC, 0x0F, 0x24, 0x80, 0xF0, 0x50, 0xA0,
+0x04, 0x6C, 0x8C, 0xEA, 0xAC, 0xEA, 0x09, 0x22,
+0x41, 0xA1, 0x03, 0x6C, 0x01, 0x6E, 0x80, 0xF0,
+0x51, 0xC0, 0x40, 0xA1, 0x8C, 0xEA, 0x80, 0xF0,
+0x4F, 0xC0, 0x60, 0xF0, 0x42, 0xA0, 0x0F, 0x6C,
+0xF0, 0x67, 0x8C, 0xEA, 0x20, 0x6C, 0x8D, 0xEA,
+0x0E, 0x94, 0x60, 0xF0, 0x42, 0xC0, 0xD1, 0x18,
+0x7B, 0x09, 0xB1, 0x67, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63, 0x06, 0x93,
+0x63, 0xEA, 0x15, 0x60, 0x5D, 0x67, 0x58, 0xA2,
+0x00, 0x6E, 0x41, 0xC1, 0x10, 0x92, 0x01, 0x72,
+0xCD, 0x61, 0x09, 0x92, 0x08, 0x6C, 0x60, 0xF0,
+0x51, 0xA2, 0x8C, 0xEA, 0xC7, 0x2A, 0x60, 0xF0,
+0x43, 0xA0, 0xDF, 0x4C, 0x00, 0x6E, 0x8C, 0xEA,
+0x60, 0xF0, 0x43, 0xC0, 0xBF, 0x17, 0xD0, 0x67,
+0x02, 0x6D, 0xD1, 0x18, 0x88, 0x0C, 0x91, 0x67,
+0xB8, 0x2A, 0xD0, 0x67, 0x02, 0x6D, 0xD1, 0x18,
+0xF7, 0x0B, 0x91, 0x67, 0x02, 0x22, 0x01, 0x6E,
+0xE1, 0x17, 0x6C, 0x6A, 0x08, 0xD2, 0x41, 0xA1,
+0xD0, 0x67, 0x02, 0x6D, 0x91, 0x67, 0xD1, 0x18,
+0x3E, 0x0B, 0x07, 0xD2, 0x41, 0xA1, 0x06, 0x93,
+0x43, 0xEB, 0x04, 0x61, 0x5D, 0x67, 0x58, 0xA2,
+0x41, 0xC1, 0xED, 0x17, 0xFF, 0x72, 0xEB, 0x61,
+0xD1, 0x18, 0xB5, 0x06, 0x07, 0x94, 0x07, 0x95,
+0x82, 0x67, 0x00, 0xF0, 0x11, 0x06, 0xD1, 0x18,
+0x7F, 0x06, 0x04, 0x07, 0x5D, 0x67, 0x51, 0xA2,
+0xED, 0x22, 0x07, 0x94, 0x07, 0x02, 0xFF, 0x4C,
+0x07, 0xD4, 0x80, 0xA2, 0xD1, 0x18, 0xB5, 0x06,
+0x81, 0xC1, 0x03, 0x6C, 0x4C, 0xEC, 0x40, 0xA1,
+0x0D, 0x6E, 0xCB, 0xEE, 0x88, 0x34, 0xCC, 0xEA,
+0x8D, 0xEA, 0x40, 0xC1, 0x08, 0x92, 0xFF, 0x4A,
+0x07, 0xD2, 0x07, 0x02, 0x40, 0xA2, 0x08, 0xD2,
+0xCA, 0x2A, 0xC5, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0x08, 0xD4, 0x60, 0xF0,
+0x43, 0xA6, 0x19, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x25, 0x67, 0x91, 0x67, 0x60, 0xF0, 0x43, 0xC6,
+0xD1, 0x18, 0x88, 0x0C, 0x00, 0x6D, 0x60, 0xF0,
+0x62, 0xA6, 0x0F, 0x6C, 0x8C, 0xEB, 0x60, 0xF0,
+0x62, 0xC6, 0x06, 0x22, 0x08, 0x94, 0xE6, 0x67,
+0xB1, 0x67, 0xD1, 0x18, 0x7B, 0x09, 0x01, 0x6E,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x0B, 0xD5, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0x0D, 0xD7,
+0x15, 0xF0, 0x08, 0x4D, 0x11, 0xA5, 0x70, 0xA5,
+0xF3, 0xA5, 0x00, 0x30, 0x0D, 0xEB, 0x12, 0xA5,
+0x00, 0xF6, 0xE0, 0x37, 0x0F, 0x92, 0x00, 0x30,
+0x00, 0x30, 0x6D, 0xE8, 0x0D, 0xEF, 0x01, 0xA7,
+0x60, 0xA7, 0xA3, 0xA7, 0x00, 0x30, 0x0D, 0xEB,
+0x02, 0xA7, 0x00, 0xF6, 0xA0, 0x35, 0xFF, 0xF7,
+0x1F, 0x69, 0x00, 0x30, 0x00, 0x30, 0x6D, 0xE8,
+0x0D, 0xED, 0x80, 0xF0, 0x05, 0xA5, 0x80, 0xF0,
+0xE6, 0xA5, 0x80, 0xF0, 0x64, 0xA5, 0x00, 0x30,
+0xE0, 0x37, 0x0D, 0xEB, 0xE0, 0x37, 0x6D, 0xEF,
+0x80, 0xF0, 0x67, 0xA5, 0x6C, 0x6D, 0xB8, 0xEE,
+0x00, 0xF6, 0x60, 0x33, 0xED, 0xEB, 0x04, 0xD3,
+0x0B, 0x93, 0x12, 0xEF, 0x7D, 0xE7, 0x04, 0x93,
+0xE4, 0x37, 0xFD, 0xE3, 0x20, 0xF2, 0xBE, 0xA7,
+0x20, 0xF2, 0xDD, 0xA7, 0xA0, 0x35, 0xCD, 0xED,
+0xA3, 0xEC, 0xBB, 0xE4, 0x01, 0x60, 0x9B, 0xE5,
+0xC0, 0xF4, 0x06, 0xA7, 0x2C, 0xEE, 0xC0, 0xF4,
+0x65, 0xA7, 0xFF, 0x4E, 0x00, 0x30, 0x2C, 0xEE,
+0x6D, 0xE8, 0x07, 0x5E, 0x2C, 0xE8, 0x74, 0x60,
+0xA3, 0xEC, 0xCF, 0x45, 0x01, 0x61, 0xC1, 0x45,
+0x2C, 0xEE, 0x20, 0xF2, 0xDD, 0xC7, 0xC2, 0x36,
+0x20, 0xF2, 0xDE, 0xC7, 0x20, 0xF2, 0xDE, 0xA7,
+0x20, 0xF2, 0x3D, 0xA7, 0x83, 0xED, 0xC0, 0x33,
+0x6D, 0xE9, 0x40, 0xF0, 0x7E, 0xA2, 0xFF, 0xF7,
+0x1F, 0x6E, 0x3B, 0x65, 0x62, 0x61, 0x97, 0xE5,
+0xAC, 0xEE, 0x6A, 0x35, 0x0F, 0x6B, 0x1B, 0x65,
+0x78, 0x67, 0x6C, 0xED, 0xA4, 0x35, 0x01, 0x6B,
+0x6D, 0xED, 0x78, 0x67, 0x6C, 0xED, 0xA8, 0x33,
+0xFB, 0x65, 0x3D, 0x6B, 0x6B, 0xEB, 0x1B, 0x65,
+0xB8, 0x67, 0x79, 0x67, 0xAC, 0xEB, 0xBF, 0x67,
+0x6D, 0xED, 0x40, 0xF0, 0xBE, 0xC2, 0x40, 0xF0,
+0xBE, 0xA2, 0x03, 0x6B, 0x1B, 0x65, 0x78, 0x67,
+0xAA, 0x35, 0x6C, 0xED, 0xFF, 0x4D, 0x02, 0x5D,
+0x06, 0x61, 0x03, 0x5E, 0x0F, 0x61, 0xA0, 0xF0,
+0xB4, 0xA2, 0x01, 0x75, 0x43, 0x61, 0x09, 0x6B,
+0x3B, 0x65, 0xFF, 0xF7, 0x1F, 0x6D, 0x79, 0x67,
+0xAC, 0xEB, 0x63, 0xEE, 0x01, 0x60, 0x3E, 0x65,
+0xD9, 0x67, 0xAC, 0xEE, 0x04, 0x93, 0xD9, 0xE3,
+0x40, 0xF7, 0xAD, 0xA6, 0xFF, 0xF7, 0x1F, 0x6E,
+0xB5, 0xE0, 0xCC, 0xED, 0x0A, 0x5D, 0x00, 0x68,
+0x07, 0x61, 0xF6, 0x4D, 0x05, 0x67, 0xCC, 0xED,
+0x1F, 0x5D, 0x01, 0x61, 0x1E, 0x68, 0xCC, 0xE8,
+0x02, 0x35, 0x0B, 0x59, 0xC0, 0xF4, 0x05, 0xC7,
+0xC0, 0xF4, 0xA6, 0xC7, 0xD0, 0x67, 0x28, 0x61,
+0x15, 0x58, 0x01, 0x6E, 0x04, 0x60, 0x0B, 0x58,
+0x02, 0x6E, 0x01, 0x60, 0x03, 0x6E, 0xA8, 0x41,
+0xFE, 0x4D, 0xA7, 0xEE, 0xB5, 0xE0, 0xD0, 0x67,
+0xFF, 0xF7, 0x1F, 0x68, 0xAC, 0xE8, 0x11, 0x10,
+0xAC, 0x36, 0x99, 0xE6, 0xBB, 0xE6, 0xCF, 0x36,
+0x8B, 0x17, 0xB7, 0xE4, 0xAC, 0xEE, 0x6A, 0x35,
+0x0F, 0x6B, 0x1B, 0x65, 0x78, 0x67, 0x6C, 0xED,
+0xA4, 0x35, 0xA0, 0x17, 0x0B, 0x59, 0xD0, 0x67,
+0xDF, 0x60, 0x29, 0x58, 0xB0, 0x67, 0x01, 0x61,
+0x28, 0x6D, 0xFF, 0xF7, 0x1F, 0x68, 0xAC, 0xE8,
+0x04, 0x5E, 0x80, 0xF0, 0x04, 0x60, 0x8A, 0x31,
+0x14, 0x6D, 0x00, 0x6C, 0x80, 0xF0, 0x93, 0xC2,
+0xFF, 0x6C, 0x80, 0xF0, 0xB2, 0xC2, 0x69, 0x4C,
+0x00, 0x6E, 0x05, 0xD7, 0xD1, 0x18, 0x40, 0x6A,
+0x04, 0xD2, 0x04, 0x92, 0x05, 0x97, 0xA0, 0xF0,
+0x96, 0x82, 0x00, 0x54, 0x6B, 0x60, 0xA0, 0xF0,
+0x94, 0xA2, 0x3F, 0x2C, 0x40, 0xF0, 0x94, 0xA2,
+0x0B, 0x93, 0x63, 0xEC, 0x38, 0x61, 0x04, 0x6B,
+0xA0, 0xF0, 0x74, 0xC2, 0xA0, 0xF0, 0x74, 0xA2,
+0x02, 0x73, 0x47, 0x60, 0x03, 0x5B, 0x41, 0x60,
+0x01, 0x6C, 0x8E, 0xEB, 0x54, 0x23, 0xA0, 0xF0,
+0x73, 0xA2, 0xA0, 0xF0, 0x92, 0xA2, 0x60, 0x33,
+0x8D, 0xEB, 0x0E, 0x94, 0x60, 0xCC, 0xA0, 0xF0,
+0x70, 0xA2, 0xA0, 0xF0, 0x51, 0xA2, 0x40, 0x32,
+0x6D, 0xEA, 0x0D, 0x93, 0x40, 0xCB, 0x20, 0xF2,
+0x5E, 0xA7, 0x20, 0xF2, 0xBD, 0xA7, 0xFF, 0x6C,
+0x40, 0x32, 0x07, 0x67, 0x4D, 0xED, 0x01, 0x6E,
+0xD1, 0x18, 0x40, 0x6A, 0x6D, 0x4C, 0xC0, 0xF4,
+0x46, 0xA0, 0xC0, 0xF4, 0xA5, 0xA0, 0xFF, 0x6C,
+0x40, 0x32, 0x01, 0x6E, 0x4D, 0xED, 0xD1, 0x18,
+0x40, 0x6A, 0x6F, 0x4C, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x01, 0x6B,
+0xC7, 0x17, 0x01, 0x74, 0x02, 0x6B, 0xC4, 0x60,
+0x40, 0xF0, 0x94, 0xA2, 0x0B, 0x93, 0x63, 0xEC,
+0xBE, 0x60, 0x70, 0x67, 0x2D, 0xEB, 0xBB, 0x23,
+0xA0, 0xF0, 0x6F, 0xA2, 0xF0, 0x23, 0x03, 0x6B,
+0xB7, 0x17, 0x03, 0x73, 0x11, 0x60, 0x04, 0x73,
+0xBE, 0x61, 0xA0, 0xF0, 0x6E, 0xA2, 0xA0, 0xF0,
+0x10, 0xC2, 0xA0, 0xF0, 0x32, 0xC2, 0x02, 0x30,
+0x22, 0x31, 0xA0, 0xF0, 0x6F, 0xC2, 0xA0, 0xF0,
+0x11, 0xC2, 0xA0, 0xF0, 0x33, 0xC2, 0xAF, 0x17,
+0xA0, 0xF0, 0x6F, 0xA2, 0xFF, 0x4B, 0xA0, 0xF0,
+0x6F, 0xC2, 0xA9, 0x17, 0x0E, 0x92, 0x20, 0xCA,
+0x0D, 0x92, 0x00, 0xCA, 0xB4, 0x17, 0x07, 0x5E,
+0x00, 0x69, 0x05, 0x6D, 0x7F, 0xF7, 0x19, 0x60,
+0x0A, 0x6D, 0x77, 0x17, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x13, 0xD5, 0x14, 0xD6,
+0x15, 0xF0, 0x08, 0x4B, 0xD1, 0xA3, 0xB0, 0xA3,
+0x04, 0x67, 0x87, 0x67, 0xF2, 0xA3, 0xC0, 0x36,
+0xAD, 0xEE, 0xB3, 0xA3, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0xA0, 0x35, 0xCD, 0xED,
+0xC1, 0xA5, 0x60, 0xA5, 0xE2, 0xA5, 0xC0, 0x36,
+0x6D, 0xEE, 0x63, 0xA5, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0x60, 0x33, 0xCD, 0xEB,
+0x80, 0xF0, 0xC5, 0xA3, 0x80, 0xF0, 0xA6, 0xA3,
+0x80, 0xF0, 0xE4, 0xA3, 0x80, 0xF0, 0x67, 0xA3,
+0xC0, 0x36, 0xA0, 0x35, 0xCD, 0xEF, 0xA0, 0x35,
+0xED, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0xA0, 0xA0, 0x41, 0xA0, 0x0A, 0xD3, 0x03, 0x6E,
+0xAA, 0x33, 0xCC, 0xEB, 0x09, 0xD2, 0x08, 0xD3,
+0x5D, 0x67, 0x07, 0x6B, 0x20, 0xF0, 0x70, 0xC2,
+0x16, 0x91, 0xFF, 0x6A, 0x00, 0x6F, 0xB2, 0x30,
+0x0B, 0xD2, 0xAC, 0xEE, 0x5D, 0x67, 0x09, 0x95,
+0xED, 0xCA, 0xEC, 0xCA, 0x06, 0x07, 0x04, 0xD7,
+0x05, 0xD1, 0x00, 0xF0, 0x1A, 0x07, 0xD1, 0x18,
+0x0B, 0x0E, 0x6C, 0xE8, 0x0A, 0x92, 0x09, 0x93,
+0xA0, 0xF0, 0x96, 0x81, 0x7D, 0xE2, 0x00, 0x54,
+0x7D, 0x67, 0x0B, 0x92, 0x20, 0xF0, 0x70, 0xA3,
+0x80, 0xF0, 0x0E, 0x60, 0x08, 0x94, 0x03, 0x2C,
+0xC0, 0xF1, 0xD1, 0xA7, 0x10, 0x10, 0x08, 0x94,
+0x03, 0x74, 0x6C, 0x60, 0x60, 0xF0, 0x83, 0xA1,
+0x04, 0x6D, 0xAC, 0xEC, 0x4C, 0xEC, 0xF4, 0x24,
+0x8B, 0xA1, 0x8C, 0xEB, 0x4C, 0xEB, 0xF0, 0x2B,
+0x01, 0x70, 0x0A, 0x6E, 0xED, 0x61, 0x5D, 0x67,
+0xAD, 0xAA, 0x0C, 0xAA, 0x00, 0x6C, 0xAD, 0xE6,
+0x02, 0x4B, 0x02, 0xEB, 0x05, 0x61, 0x83, 0x45,
+0x13, 0xE4, 0x99, 0xE6, 0xFF, 0x6C, 0xCC, 0xEC,
+0x60, 0xF1, 0xC5, 0xA7, 0xAD, 0xE6, 0x62, 0xE8,
+0x00, 0x6B, 0x04, 0x60, 0x03, 0xE5, 0x01, 0xE6,
+0xFF, 0x6B, 0x0C, 0xEB, 0x60, 0xF0, 0xDB, 0xA1,
+0xDE, 0x37, 0x63, 0x27, 0x7F, 0x6F, 0xEC, 0xEE,
+0xC2, 0xEC, 0x5D, 0x61, 0xD3, 0xE4, 0xFF, 0x6F,
+0xEC, 0xEC, 0xC2, 0xEB, 0x63, 0x61, 0xCF, 0xE3,
+0xFF, 0x6E, 0xCC, 0xEB, 0x5B, 0x5C, 0xC4, 0x67,
+0x01, 0x61, 0x5A, 0x6E, 0xFF, 0x6F, 0x86, 0x67,
+0xC3, 0x67, 0xEC, 0xEB, 0x42, 0x5B, 0xEC, 0xEC,
+0x01, 0x61, 0x41, 0x6E, 0x13, 0x92, 0xC0, 0xC2,
+0x14, 0x92, 0x80, 0xC2, 0x13, 0x92, 0xC0, 0xA2,
+0xFF, 0x6A, 0xCC, 0xEA, 0x43, 0xEC, 0x01, 0x60,
+0xC4, 0x67, 0x13, 0x92, 0xFF, 0x6C, 0x71, 0x4C,
+0xC0, 0xC2, 0xD1, 0x18, 0x40, 0x6A, 0x01, 0x6E,
+0x5D, 0x67, 0xAC, 0xAA, 0xFF, 0x6C, 0x01, 0x6E,
+0xD1, 0x18, 0x40, 0x6A, 0x73, 0x4C, 0x14, 0x92,
+0xFF, 0x6C, 0x00, 0x6E, 0xA0, 0xA2, 0xD1, 0x18,
+0x40, 0x6A, 0x6A, 0x4C, 0x13, 0x92, 0xFF, 0x6C,
+0x00, 0x6E, 0xA0, 0xA2, 0xD1, 0x18, 0x40, 0x6A,
+0x6B, 0x4C, 0x11, 0x97, 0x10, 0x91, 0x0F, 0x90,
+0x00, 0xEF, 0x09, 0x63, 0x60, 0xF0, 0x86, 0xA1,
+0x10, 0x6D, 0xAC, 0xEC, 0x4C, 0xEC, 0x88, 0x2C,
+0x8B, 0xA1, 0x6C, 0xEC, 0x4C, 0xEC, 0x84, 0x2C,
+0x05, 0x6C, 0xB0, 0x67, 0x8C, 0xED, 0x01, 0x75,
+0x06, 0x6E, 0x91, 0x60, 0x05, 0x70, 0x8F, 0x60,
+0x06, 0x48, 0x0C, 0xEB, 0x8C, 0xEB, 0x4C, 0xEB,
+0x0A, 0x6E, 0x7F, 0xF7, 0x15, 0x2B, 0x87, 0x17,
+0x0A, 0x6E, 0x85, 0x28, 0x71, 0x17, 0x00, 0x6C,
+0xA4, 0x17, 0xA8, 0x26, 0x7F, 0x68, 0x0C, 0xEE,
+0xFF, 0x6F, 0xD1, 0xE4, 0xCD, 0xE3, 0xEC, 0xEC,
+0xEC, 0xEB, 0xA0, 0x17, 0x00, 0x6B, 0x9E, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x7F, 0xA7, 0x5E, 0xA7, 0x06, 0x67, 0x60, 0x33,
+0x4D, 0xEB, 0x04, 0xD3, 0xD9, 0xA7, 0x58, 0xA7,
+0x3C, 0x65, 0xC0, 0x36, 0x4D, 0xEE, 0x05, 0xD6,
+0x9B, 0xA7, 0x5A, 0xA7, 0x25, 0x67, 0x80, 0x34,
+0x4D, 0xEC, 0x5C, 0xA7, 0xFD, 0xA7, 0xA4, 0x67,
+0xE0, 0x37, 0x4D, 0xEF, 0xD1, 0x18, 0xBC, 0x06,
+0xF3, 0xE4, 0x79, 0x67, 0x40, 0xCB, 0x04, 0x93,
+0xF3, 0xE3, 0xD1, 0x18, 0xBC, 0x06, 0xA3, 0x67,
+0x05, 0x96, 0x04, 0x93, 0x40, 0xC9, 0xA6, 0x67,
+0xD1, 0x18, 0xBC, 0x06, 0x73, 0xE6, 0x40, 0xC8,
+0x59, 0x67, 0xA0, 0xA2, 0xFF, 0x6C, 0x00, 0x6E,
+0xD1, 0x18, 0x40, 0x6A, 0x6C, 0x4C, 0xA0, 0xA1,
+0xFF, 0x6C, 0x00, 0x6E, 0xD1, 0x18, 0x40, 0x6A,
+0x75, 0x4C, 0xA0, 0xA0, 0xFF, 0x6C, 0x00, 0x6E,
+0xD1, 0x18, 0x40, 0x6A, 0x76, 0x4C, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0x40, 0xF2, 0x0D, 0x25, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD1, 0x13, 0xD0, 0x40, 0xF0, 0x5D, 0xA5,
+0x40, 0x6B, 0x05, 0x67, 0x6C, 0xEA, 0xFF, 0x6B,
+0x6C, 0xEA, 0x24, 0x67, 0x09, 0x2A, 0x60, 0xF0,
+0x83, 0x85, 0x00, 0x54, 0x05, 0x61, 0x99, 0xA0,
+0xB8, 0xA5, 0x80, 0x34, 0xAD, 0xEC, 0x13, 0x2C,
+0xD1, 0x18, 0x05, 0x07, 0x90, 0x67, 0xFF, 0x6C,
+0x00, 0x6D, 0xD1, 0x18, 0x14, 0x6A, 0x77, 0x4C,
+0xFF, 0x6C, 0x00, 0x6E, 0xA1, 0x42, 0xD1, 0x18,
+0x40, 0x6A, 0x77, 0x4C, 0x15, 0x97, 0x14, 0x91,
+0x13, 0x90, 0x00, 0xEF, 0x0B, 0x63, 0xB1, 0x67,
+0x0C, 0x04, 0x0F, 0xD3, 0xD1, 0x18, 0xAD, 0x07,
+0x0E, 0xD2, 0x7D, 0x67, 0x20, 0xF0, 0x70, 0xA3,
+0x5D, 0x67, 0x20, 0xF0, 0x07, 0x06, 0x20, 0xF0,
+0x64, 0xC2, 0x7D, 0x67, 0x20, 0xF0, 0x71, 0xA3,
+0x20, 0xF0, 0x06, 0x07, 0x20, 0xF0, 0x65, 0xC2,
+0x0E, 0x92, 0x0F, 0x93, 0x06, 0xD2, 0x5D, 0x67,
+0x20, 0xF0, 0x84, 0xA2, 0x03, 0x6A, 0x0E, 0xD3,
+0x8A, 0x34, 0x4C, 0xEC, 0x5D, 0x67, 0x20, 0xF0,
+0xA5, 0xA2, 0xD1, 0x18, 0x7F, 0x06, 0x6C, 0xEC,
+0xB0, 0x67, 0xD1, 0x18, 0x43, 0x09, 0x09, 0x04,
+0x60, 0xF0, 0xC5, 0xA0, 0x3A, 0x65, 0x07, 0x6A,
+0xD2, 0x36, 0x4C, 0xEE, 0x0E, 0x93, 0x5D, 0x67,
+0x20, 0xF0, 0x46, 0xA2, 0x6C, 0xEE, 0x42, 0xEE,
+0x11, 0x60, 0x5D, 0x67, 0x20, 0xF0, 0x84, 0xA2,
+0x03, 0x6A, 0x8A, 0x34, 0x4C, 0xEC, 0x5D, 0x67,
+0x20, 0xF0, 0xA7, 0xA2, 0xD1, 0x18, 0x71, 0x06,
+0x6C, 0xEC, 0x7D, 0x67, 0x20, 0xF0, 0x45, 0xC3,
+0x03, 0x6A, 0x3A, 0x65, 0x7D, 0x67, 0x20, 0xF0,
+0x64, 0xA3, 0x40, 0xF0, 0x5C, 0xA0, 0x03, 0x6C,
+0xA3, 0x67, 0xFF, 0x6E, 0x8C, 0xEA, 0x8C, 0xED,
+0xCC, 0xEA, 0xCC, 0xED, 0xA2, 0xEA, 0x0A, 0x60,
+0x04, 0x6D, 0xAB, 0xED, 0x4C, 0xEC, 0xAC, 0xEB,
+0x8D, 0xEB, 0x9D, 0x67, 0x20, 0xF0, 0x64, 0xC4,
+0x04, 0x6B, 0x3B, 0x65, 0x7D, 0x67, 0x20, 0xF0,
+0x84, 0xA3, 0x03, 0x6D, 0x8A, 0x33, 0xAC, 0xEB,
+0x05, 0x23, 0x7D, 0x67, 0x20, 0xF0, 0x67, 0xA3,
+0x06, 0x5B, 0x0F, 0x60, 0xA0, 0xA0, 0x40, 0x6B,
+0xFF, 0x6E, 0x6C, 0xED, 0xCC, 0xED, 0x1A, 0x25,
+0x0C, 0x6B, 0x8C, 0xEB, 0xCC, 0xEB, 0x0C, 0x73,
+0x15, 0x61, 0x7D, 0x67, 0x20, 0xF0, 0x67, 0xA3,
+0x11, 0x23, 0x03, 0x6D, 0x64, 0x67, 0xAC, 0xEB,
+0xFF, 0x6E, 0xCC, 0xEB, 0x42, 0xEB, 0x0A, 0x60,
+0x04, 0x6B, 0x6B, 0xEB, 0x6C, 0xEC, 0xAC, 0xEA,
+0x8D, 0xEA, 0x7D, 0x67, 0x20, 0xF0, 0x44, 0xC3,
+0x05, 0x6A, 0x3A, 0x65, 0x5D, 0x67, 0x20, 0xF0,
+0x44, 0xA2, 0x0C, 0x6B, 0x1B, 0x65, 0x78, 0x67,
+0x4C, 0xEB, 0xFF, 0x6C, 0x43, 0x67, 0x8C, 0xEA,
+0x0C, 0x72, 0x11, 0x61, 0x5D, 0x67, 0x20, 0xF0,
+0x44, 0xA2, 0x60, 0xF0, 0xA6, 0xA0, 0x07, 0x6B,
+0x52, 0x32, 0x6C, 0xEA, 0xB6, 0x33, 0x6E, 0xEA,
+0x8C, 0xEA, 0x05, 0x22, 0x10, 0x6A, 0x4C, 0xED,
+0x02, 0x25, 0x06, 0x6A, 0x3A, 0x65, 0xFF, 0x6B,
+0x91, 0x67, 0x59, 0x67, 0x6C, 0xEC, 0x06, 0x06,
+0x04, 0x6D, 0x10, 0xD2, 0xD1, 0x18, 0x1B, 0x6B,
+0x0E, 0xD3, 0x5D, 0x67, 0x59, 0xA2, 0x0E, 0x93,
+0x90, 0x67, 0x5E, 0x32, 0x6C, 0xEA, 0x0E, 0xD2,
+0x5D, 0x67, 0x20, 0xF0, 0xA4, 0xA2, 0x03, 0x6A,
+0x00, 0x6F, 0xAA, 0x35, 0x4C, 0xED, 0x5D, 0x67,
+0x20, 0xF0, 0xC6, 0xA2, 0x6C, 0xED, 0xD1, 0x18,
+0x54, 0x07, 0x0F, 0xD3, 0x60, 0xF0, 0x94, 0xA0,
+0x10, 0x93, 0x01, 0x6D, 0x8C, 0xED, 0x3B, 0x65,
+0x0E, 0x93, 0x34, 0x25, 0x34, 0x23, 0x5D, 0x67,
+0x20, 0xF0, 0x46, 0xA2, 0x21, 0x2A, 0x5D, 0x67,
+0x20, 0xF0, 0x44, 0xA2, 0x03, 0x6B, 0x4A, 0x32,
+0x6C, 0xEA, 0x1A, 0x22, 0x5D, 0xA0, 0xBC, 0xA0,
+0x40, 0x32, 0xAD, 0xEA, 0x1C, 0x2A, 0x04, 0x6A,
+0x8C, 0xEA, 0x19, 0x2A, 0x02, 0x6A, 0x4D, 0xEC,
+0x60, 0xF0, 0x94, 0xC0, 0x60, 0xF0, 0x94, 0xA0,
+0x0F, 0x93, 0x8E, 0x32, 0x4C, 0xEB, 0x1F, 0x73,
+0x07, 0x60, 0x01, 0x4B, 0x07, 0x6A, 0x6C, 0x33,
+0x4C, 0xEC, 0x8D, 0xEB, 0x60, 0xF0, 0x74, 0xC0,
+0x60, 0xF0, 0x54, 0xA0, 0x03, 0x6B, 0x6C, 0xEA,
+0x03, 0x72, 0x0A, 0x61, 0x0B, 0x10, 0x60, 0xF0,
+0x54, 0xA0, 0x04, 0x6B, 0x6D, 0xEA, 0x60, 0xF0,
+0x54, 0xC0, 0xF2, 0x17, 0xF1, 0x2B, 0x09, 0x2A,
+0x59, 0x67, 0x0C, 0x22, 0xF0, 0x67, 0x00, 0x6E,
+0x09, 0x05, 0xD1, 0x18, 0x7B, 0x09, 0x91, 0x67,
+0x54, 0x10, 0x03, 0x6A, 0x4C, 0xEC, 0x03, 0x74,
+0xF5, 0x61, 0xF2, 0x17, 0x5D, 0x67, 0x20, 0xF0,
+0x44, 0xA2, 0x03, 0x6C, 0x07, 0x6D, 0x4A, 0x33,
+0x8C, 0xEB, 0x40, 0xF0, 0x76, 0xC0, 0x7D, 0x67,
+0x20, 0xF0, 0x65, 0xA3, 0xF0, 0x67, 0x07, 0x06,
+0x40, 0xF0, 0x77, 0xC0, 0x52, 0x33, 0xAC, 0xEB,
+0x40, 0xF0, 0x78, 0xC0, 0x40, 0xF0, 0x7C, 0xA0,
+0x8C, 0xEA, 0x31, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB,
+0x50, 0x32, 0x6D, 0xEA, 0x40, 0xF0, 0x5C, 0xC0,
+0x00, 0xF0, 0x1E, 0x05, 0xD1, 0x18, 0x42, 0x0F,
+0x08, 0x04, 0x5D, 0x67, 0xF0, 0xAA, 0x98, 0xA0,
+0x40, 0x6B, 0xE2, 0x32, 0x60, 0xF0, 0x4A, 0xC0,
+0x59, 0xA0, 0x60, 0xF0, 0xE9, 0xC0, 0x40, 0x32,
+0x8D, 0xEA, 0x46, 0x32, 0x41, 0x5A, 0x03, 0x60,
+0x61, 0x42, 0xFF, 0x6A, 0x4C, 0xEB, 0x60, 0xF0,
+0x68, 0xC0, 0x0A, 0x06, 0x04, 0xD0, 0x20, 0xF0,
+0x09, 0x05, 0xD1, 0x18, 0xC3, 0x0E, 0x09, 0x04,
+0x20, 0x59, 0x13, 0x60, 0x5D, 0x67, 0x4E, 0xAA,
+0x64, 0x5A, 0x0F, 0x61, 0xFF, 0x6C, 0x00, 0x6D,
+0xD1, 0x18, 0x14, 0x6A, 0x78, 0x4C, 0xFF, 0x6C,
+0x00, 0x6E, 0xA1, 0x42, 0xD1, 0x18, 0x40, 0x6A,
+0x78, 0x4C, 0xD1, 0x18, 0x05, 0x07, 0x90, 0x67,
+0x95, 0x16, 0xA0, 0xF0, 0x56, 0x80, 0x00, 0x52,
+0x7F, 0x60, 0xA0, 0xF0, 0x54, 0xA0, 0x02, 0x72,
+0x23, 0x61, 0x5D, 0x67, 0x20, 0xF0, 0x44, 0xA2,
+0xA0, 0xF0, 0x96, 0xA0, 0x04, 0x6B, 0x03, 0x6D,
+0x6B, 0xEB, 0xAC, 0xEC, 0x6C, 0xEA, 0x8D, 0xEA,
+0xA0, 0xF0, 0x95, 0xA0, 0x7D, 0x67, 0x20, 0xF0,
+0x85, 0xC3, 0xA0, 0xF0, 0x96, 0xA0, 0x8A, 0x33,
+0xAC, 0xEB, 0x0D, 0x6D, 0xAB, 0xED, 0x68, 0x33,
+0xAC, 0xEA, 0x6D, 0xEA, 0x70, 0x6B, 0x6C, 0xEC,
+0x71, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA, 0x8D, 0xEA,
+0x7D, 0x67, 0x20, 0xF0, 0x44, 0xC3, 0x76, 0x17,
+0x01, 0x72, 0x56, 0x61, 0x5D, 0x67, 0x20, 0xF0,
+0x84, 0xA2, 0xA0, 0xF0, 0x56, 0xA0, 0x04, 0x6B,
+0x6B, 0xEB, 0x6C, 0xEA, 0x7D, 0x67, 0x20, 0xF0,
+0x65, 0xA3, 0x03, 0x6D, 0xAC, 0xEC, 0xA0, 0xF0,
+0x75, 0xC0, 0x7D, 0x67, 0x8D, 0xEA, 0x20, 0xF0,
+0x84, 0xA3, 0x8A, 0x33, 0xAC, 0xEB, 0x0D, 0x6D,
+0xAB, 0xED, 0x68, 0x33, 0xAC, 0xEA, 0x6D, 0xEA,
+0x70, 0x6B, 0x6C, 0xEC, 0x71, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x8D, 0xEA, 0xA0, 0xF0, 0x56, 0xC0,
+0x01, 0x6A, 0x7D, 0x67, 0x90, 0xAB, 0x20, 0xF0,
+0x68, 0xA3, 0x63, 0xEC, 0x34, 0x60, 0x7D, 0x67,
+0x6F, 0xAB, 0x1E, 0x5B, 0x30, 0x60, 0x30, 0x2A,
+0x7D, 0x67, 0x20, 0xF0, 0x69, 0xA3, 0xFF, 0xF7,
+0x1F, 0x6D, 0xAC, 0xEB, 0x63, 0xEC, 0x49, 0x60,
+0x60, 0xF0, 0x60, 0xA0, 0x60, 0xF0, 0xA1, 0xA0,
+0xFF, 0x6C, 0x01, 0x4B, 0x60, 0xF0, 0x60, 0xC0,
+0x79, 0x4C, 0x00, 0x6E, 0xD1, 0x18, 0x40, 0x6A,
+0x0E, 0xD2, 0x60, 0xF0, 0x60, 0xA0, 0x60, 0xF0,
+0x81, 0xA0, 0x83, 0xEB, 0x09, 0x61, 0xF0, 0x67,
+0x00, 0x6E, 0x09, 0x05, 0xD1, 0x18, 0xE2, 0x0C,
+0x91, 0x67, 0x0E, 0x92, 0x60, 0xF0, 0x40, 0xC0,
+0x00, 0x6A, 0x40, 0xF0, 0x5F, 0xC0, 0x79, 0x17,
+0x5D, 0x67, 0x4F, 0xAA, 0x01, 0x6B, 0x50, 0x5A,
+0x58, 0x67, 0x6E, 0xEA, 0xC6, 0x17, 0x0C, 0x22,
+0xF0, 0x67, 0x01, 0x6E, 0x09, 0x05, 0xD1, 0x18,
+0x7F, 0x0D, 0x91, 0x67, 0x00, 0x6A, 0x40, 0xF0,
+0x5F, 0xC0, 0x60, 0xF0, 0x40, 0xC0, 0x65, 0x17,
+0x40, 0xF0, 0x7F, 0xA0, 0xFF, 0x6C, 0x0E, 0xD2,
+0x01, 0x4B, 0x8C, 0xEB, 0x02, 0x5B, 0x40, 0xF0,
+0x7F, 0xC0, 0x09, 0x61, 0xF0, 0x67, 0x00, 0x6E,
+0x09, 0x05, 0xD1, 0x18, 0x7F, 0x0D, 0x91, 0x67,
+0x0E, 0x92, 0x40, 0xF0, 0x5F, 0xC0, 0x00, 0x6A,
+0xE8, 0x17, 0x60, 0xF0, 0x40, 0xA0, 0x03, 0x22,
+0xFF, 0x4A, 0x60, 0xF0, 0x40, 0xC0, 0x40, 0xF0,
+0x5F, 0xA0, 0x03, 0x22, 0xFF, 0x4A, 0x40, 0xF0,
+0x5F, 0xC0, 0xD0, 0x67, 0x09, 0x05, 0xD1, 0x18,
+0xF9, 0x0D, 0x91, 0x67, 0x3E, 0x17, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x15, 0xF0, 0x08, 0x49,
+0xB1, 0xA1, 0x70, 0xA1, 0x52, 0xA1, 0xA0, 0x35,
+0x6D, 0xED, 0x73, 0xA1, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0xA1, 0xA3, 0x40, 0xA3, 0xC2, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x80, 0xF0, 0x65, 0xA2, 0x80, 0xF0, 0x04, 0xA2,
+0x80, 0xF0, 0xA6, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x80, 0xF0, 0x07, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEB, 0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8,
+0x00, 0x6B, 0x07, 0xD3, 0x06, 0xD3, 0x08, 0xD3,
+0x42, 0x24, 0x40, 0xF1, 0x5E, 0xA0, 0x40, 0xF1,
+0xBD, 0xA0, 0x40, 0xF1, 0x9F, 0xA0, 0x40, 0x32,
+0x4D, 0xED, 0x60, 0xF1, 0x40, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x4D, 0xEC, 0x09, 0xD4, 0x28, 0x24, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1,
+0x34, 0x9A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x92, 0xF2, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x08, 0x93, 0x09, 0x94, 0x00, 0x6F,
+0x04, 0xD3, 0xC2, 0x67, 0x40, 0xE9, 0x01, 0x6D,
+0x01, 0x72, 0x11, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x08, 0x93, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3,
+0xC0, 0xF1, 0x0D, 0x6F, 0x02, 0x6E, 0x01, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x00, 0x6A,
+0x80, 0xF0, 0x41, 0xC0, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0x00, 0x6A,
+0x08, 0xD2, 0x91, 0xA1, 0x50, 0xA1, 0xB2, 0xA1,
+0x80, 0x34, 0x4D, 0xEC, 0x53, 0xA1, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x81, 0xA2, 0xA2, 0xA2, 0xC0, 0xA2,
+0x43, 0xA2, 0x80, 0x34, 0xA0, 0x35, 0x08, 0x93,
+0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x68, 0x36, 0x59, 0xE6,
+0x81, 0xA6, 0x40, 0xA6, 0xA2, 0xA6, 0x80, 0x34,
+0x4D, 0xEC, 0x43, 0xA6, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x49, 0x22, 0x60, 0xF0, 0x94, 0xA2, 0x01, 0x6D,
+0x0A, 0xD2, 0x8C, 0xED, 0x43, 0x25, 0x06, 0x6B,
+0x6C, 0xEC, 0x04, 0x74, 0x3F, 0x60, 0x08, 0x02,
+0x60, 0xA2, 0x07, 0x06, 0x04, 0x6D, 0x83, 0x67,
+0xD1, 0x18, 0x1B, 0x6B, 0x09, 0xD3, 0x5D, 0x67,
+0x9D, 0x82, 0x09, 0x93, 0x0A, 0x92, 0x00, 0x54,
+0x31, 0x60, 0x09, 0xD2, 0x5D, 0x67, 0x9D, 0xA2,
+0x7F, 0x6D, 0x06, 0x07, 0xAC, 0xEC, 0x9D, 0xC2,
+0x99, 0xA2, 0x80, 0x6D, 0xAB, 0xED, 0xAD, 0xEC,
+0x99, 0xC2, 0x07, 0x06, 0x83, 0x67, 0xD1, 0x18,
+0x83, 0x6B, 0x04, 0x6D, 0x09, 0x92, 0x03, 0x6C,
+0x60, 0xF0, 0x56, 0xA2, 0x8C, 0xEA, 0x09, 0xD2,
+0x09, 0x02, 0x40, 0xA2, 0x17, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x99, 0xF5, 0x08, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xD2, 0xF4, 0xC8, 0x9C, 0x04, 0xF0,
+0x01, 0x6D, 0xAB, 0xED, 0x99, 0xF5, 0x08, 0x6C,
+0x40, 0xEE, 0x4C, 0xED, 0x08, 0x92, 0x01, 0x4A,
+0x20, 0x72, 0x08, 0xD2, 0x58, 0x67, 0x09, 0xD2,
+0x88, 0x2A, 0x40, 0xF1, 0x7E, 0xA0, 0x40, 0xF1,
+0x9D, 0xA0, 0x40, 0xF1, 0x5F, 0xA0, 0x60, 0x33,
+0x8D, 0xEB, 0x60, 0xF1, 0x80, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xEB, 0x00, 0xF6, 0x80, 0x34,
+0x6D, 0xEC, 0x1F, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1, 0x54, 0x9A,
+0x78, 0x67, 0x04, 0xD3, 0x00, 0x6F, 0x00, 0x6E,
+0x40, 0xEA, 0x03, 0x6D, 0x01, 0x72, 0x11, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x09, 0x93, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x05, 0xD3,
+0x01, 0x6B, 0x04, 0xD3, 0xC0, 0xF1, 0x0C, 0x6F,
+0x02, 0x6E, 0x01, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x01, 0x6A, 0x4D, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0x91, 0xA2, 0x70, 0xA2,
+0xB2, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x73, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x80, 0xF0, 0x65, 0xA2,
+0x80, 0xF0, 0x04, 0xA2, 0x80, 0xF0, 0x86, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x80, 0xF0, 0x07, 0xA2,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x40, 0xF1, 0x5E, 0xA0,
+0x40, 0xF1, 0x3D, 0xA0, 0x40, 0xF1, 0x7F, 0xA0,
+0x40, 0x32, 0x2D, 0xEA, 0x60, 0xF1, 0x20, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6,
+0x20, 0x31, 0x4D, 0xE9, 0x3A, 0x29, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1,
+0x78, 0x9A, 0x97, 0xF0, 0x18, 0x6A, 0x40, 0x32,
+0xF7, 0xF0, 0x01, 0x6C, 0x40, 0x32, 0x80, 0x34,
+0x29, 0xF0, 0x1D, 0x4A, 0x80, 0x34, 0x4A, 0xF1,
+0x10, 0x4C, 0x04, 0xD2, 0x00, 0x6F, 0x01, 0x6E,
+0x40, 0xEB, 0x64, 0x6D, 0x82, 0x67, 0x40, 0xF1,
+0x5D, 0xC0, 0x42, 0x32, 0x40, 0xF1, 0x5E, 0xC0,
+0x42, 0x32, 0x40, 0xF1, 0x5F, 0xC0, 0x00, 0xF6,
+0x82, 0x32, 0x60, 0xF1, 0x40, 0xC0, 0x06, 0xD4,
+0x14, 0x24, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF1, 0x14, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x92, 0xF2,
+0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x06, 0x94,
+0x04, 0xD1, 0x00, 0x6F, 0xC2, 0x67, 0x40, 0xE8,
+0x01, 0x6D, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x00, 0xEF, 0x06, 0x63, 0xF4, 0x63, 0x17, 0x62,
+0x16, 0xD1, 0x15, 0xD0, 0x06, 0x02, 0x40, 0xF0,
+0x58, 0xA2, 0x1B, 0xD7, 0x03, 0x69, 0x3A, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xF1, 0xA2, 0x70, 0xA2,
+0x12, 0xA2, 0xE0, 0x37, 0x6D, 0xEF, 0x73, 0xA2,
+0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF, 0x00, 0xF6,
+0x60, 0x33, 0xED, 0xEB, 0xE1, 0xA3, 0x40, 0xA3,
+0x02, 0xA3, 0xE0, 0x37, 0x4D, 0xEF, 0x43, 0xA3,
+0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF, 0x00, 0xF6,
+0x40, 0x32, 0xED, 0xEA, 0x80, 0xF0, 0x65, 0xA2,
+0x80, 0xF0, 0x06, 0xA2, 0x80, 0xF0, 0xE4, 0xA2,
+0x80, 0xF0, 0x47, 0xA2, 0x60, 0x33, 0x00, 0x30,
+0x6D, 0xEF, 0x00, 0x30, 0xED, 0xE8, 0x00, 0xF6,
+0x40, 0x32, 0x0D, 0xEA, 0x0F, 0xD2, 0x00, 0x6A,
+0x0B, 0xD2, 0x0A, 0xD2, 0x1B, 0x92, 0x79, 0x67,
+0xE0, 0xF1, 0x1F, 0x68, 0x5C, 0x32, 0x6D, 0xEA,
+0x12, 0xD3, 0x62, 0x67, 0x0C, 0xEB, 0x2C, 0xEE,
+0xFD, 0x67, 0xC8, 0x36, 0x62, 0x32, 0x76, 0xCF,
+0xCD, 0xEA, 0x07, 0x6B, 0xDD, 0x67, 0x40, 0xF0,
+0x64, 0xC6, 0x71, 0x6E, 0x6C, 0xED, 0xCB, 0xEE,
+0xB0, 0x35, 0xCC, 0xEA, 0x7D, 0x67, 0xAD, 0xEA,
+0x20, 0xF0, 0x4D, 0xC3, 0x5D, 0x67, 0x14, 0xCA,
+0x7C, 0x6D, 0x02, 0x32, 0xAD, 0xEA, 0x20, 0xF0,
+0x49, 0xC3, 0xFF, 0x6A, 0x4C, 0xEC, 0x0A, 0x07,
+0x0B, 0x06, 0x00, 0x6D, 0x10, 0xD2, 0xD1, 0x18,
+0x83, 0x6B, 0x0E, 0xD4, 0x5D, 0x67, 0x20, 0xF0,
+0xED, 0xA2, 0x1D, 0x92, 0x07, 0x6C, 0xEA, 0x36,
+0xA1, 0xA2, 0x5D, 0x67, 0x8B, 0xEC, 0x40, 0xF0,
+0x64, 0xA2, 0x2C, 0xEE, 0xC4, 0x36, 0x8C, 0xED,
+0x1D, 0x92, 0xCD, 0xED, 0xF2, 0x34, 0x39, 0x6E,
+0xCB, 0xEE, 0x6C, 0xEC, 0xCC, 0xED, 0x8C, 0x34,
+0xAD, 0xEC, 0xA8, 0xA2, 0x81, 0xC2, 0x02, 0x6C,
+0x5D, 0x67, 0x8B, 0xEC, 0xAC, 0xEC, 0xB6, 0xAA,
+0x7F, 0x6E, 0x1D, 0x92, 0xAC, 0xE8, 0x01, 0x6D,
+0x0C, 0xED, 0xBC, 0x35, 0xCC, 0xEC, 0xAD, 0xEC,
+0x88, 0xC2, 0x10, 0x92, 0x1D, 0x94, 0x06, 0x30,
+0x4C, 0xE8, 0x09, 0xC4, 0x1B, 0x94, 0x12, 0x97,
+0x01, 0x74, 0x3F, 0x65, 0x14, 0x61, 0x07, 0x67,
+0x6C, 0xE8, 0xEF, 0x33, 0x6C, 0xE9, 0x4C, 0xE8,
+0x4C, 0xE9, 0x0A, 0x6A, 0x05, 0x58, 0x3A, 0x65,
+0x18, 0x60, 0x08, 0x6A, 0x03, 0x58, 0x3A, 0x65,
+0x14, 0x60, 0x06, 0x6A, 0x3A, 0x65, 0x11, 0x28,
+0x04, 0x6A, 0x3A, 0x65, 0x0E, 0x10, 0x1B, 0x94,
+0x0F, 0x68, 0xEC, 0xE8, 0x02, 0x5C, 0x4C, 0xE8,
+0x00, 0x69, 0x07, 0x61, 0xF3, 0x31, 0x6C, 0xE9,
+0x4C, 0xE9, 0x08, 0x58, 0x0B, 0x6A, 0x3A, 0x65,
+0xE4, 0x61, 0x00, 0x6A, 0x99, 0x67, 0x09, 0xD2,
+0x08, 0xD2, 0x10, 0xD2, 0x5D, 0x67, 0x93, 0xCA,
+0x0E, 0x94, 0xE0, 0xF1, 0x1F, 0x6B, 0x08, 0x07,
+0x09, 0x06, 0x01, 0x6D, 0x71, 0xCA, 0xD1, 0x18,
+0x83, 0x6B, 0x11, 0xD3, 0x10, 0x92, 0x11, 0x93,
+0x06, 0x07, 0x07, 0xD2, 0x06, 0xD2, 0x5D, 0x67,
+0x96, 0xAA, 0x07, 0x06, 0x05, 0x6D, 0x6C, 0xEC,
+0x8E, 0xCA, 0x0E, 0x94, 0xD1, 0x18, 0x83, 0x6B,
+0x6C, 0xCA, 0x05, 0x06, 0x00, 0x6D, 0xD1, 0x18,
+0x1B, 0x6B, 0x00, 0x6C, 0x04, 0x06, 0x00, 0x6D,
+0xD1, 0x18, 0x1B, 0x6B, 0x01, 0x6C, 0x05, 0x95,
+0xFF, 0x6C, 0x02, 0x6E, 0xD1, 0x18, 0x40, 0x6A,
+0x61, 0x4C, 0x04, 0x95, 0xFF, 0x6C, 0x02, 0x6E,
+0xD1, 0x18, 0x40, 0x6A, 0x65, 0x4C, 0x10, 0x92,
+0x7D, 0x67, 0x20, 0xF0, 0x53, 0xC3, 0x20, 0xF0,
+0x52, 0xC3, 0x20, 0xF0, 0x51, 0xC3, 0x20, 0xF0,
+0x50, 0xC3, 0x1B, 0x92, 0x02, 0x72, 0x20, 0xF0,
+0x12, 0x02, 0x0C, 0x60, 0x1B, 0x92, 0x03, 0x72,
+0x20, 0xF0, 0x13, 0x02, 0x07, 0x60, 0x1B, 0x92,
+0x01, 0x72, 0x58, 0x67, 0x01, 0x5A, 0x58, 0x67,
+0x69, 0xE2, 0x30, 0x4A, 0x1D, 0x93, 0x02, 0x6C,
+0x60, 0xF0, 0x76, 0xA3, 0x6C, 0xEC, 0x07, 0x24,
+0x1B, 0x93, 0x40, 0x23, 0x1D, 0x93, 0x60, 0xF0,
+0x78, 0xA3, 0x60, 0xC2, 0x0E, 0x10, 0x01, 0x6D,
+0xAC, 0xEB, 0x18, 0x23, 0x1B, 0x93, 0x36, 0x23,
+0x08, 0x58, 0x05, 0x6B, 0xF6, 0x60, 0x05, 0x58,
+0x03, 0x6B, 0xF3, 0x60, 0x03, 0x58, 0x29, 0x61,
+0xA0, 0xC2, 0x5D, 0x67, 0x20, 0xF0, 0xF3, 0xA2,
+0x20, 0xF0, 0xD2, 0xA2, 0x20, 0xF0, 0xB1, 0xA2,
+0x1D, 0x92, 0x60, 0xF0, 0x97, 0xA2, 0xB1, 0x18,
+0xBE, 0xF8, 0x00, 0x65, 0x0F, 0x92, 0x02, 0x6B,
+0x60, 0xF0, 0x59, 0xA2, 0x6C, 0xEA, 0xFF, 0x6B,
+0x6C, 0xEA, 0x0E, 0x22, 0x1B, 0x92, 0x1A, 0x22,
+0x24, 0x32, 0x45, 0xE1, 0x28, 0x31, 0x0F, 0x92,
+0x21, 0xE0, 0x6C, 0xE8, 0x01, 0xE2, 0x40, 0xF7,
+0x97, 0xA0, 0xD1, 0x18, 0xB9, 0x07, 0x00, 0x65,
+0x17, 0x97, 0x16, 0x91, 0x15, 0x90, 0x00, 0xEF,
+0x0C, 0x63, 0x02, 0x20, 0x80, 0xC2, 0xD5, 0x17,
+0x00, 0xC2, 0xD3, 0x17, 0x06, 0x03, 0x40, 0xF0,
+0x74, 0xA3, 0xBF, 0x17, 0xD1, 0x18, 0x79, 0x75,
+0x00, 0x65, 0xEE, 0x17, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x00, 0x6A, 0x7D, 0x67, 0x20, 0x31,
+0x4E, 0xCB, 0x4D, 0xCB, 0x4C, 0xCB, 0x15, 0xF0,
+0x08, 0x49, 0xA9, 0xA1, 0x48, 0xA1, 0x8A, 0xA1,
+0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA1, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x91, 0xA2, 0xB2, 0xA2, 0xD0, 0xA2,
+0x53, 0xA2, 0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE,
+0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x41, 0xA2, 0x80, 0xF0, 0x0D, 0x2A,
+0xB1, 0xA1, 0x90, 0xA1, 0x52, 0xA1, 0xA0, 0x35,
+0x8D, 0xED, 0x93, 0xA1, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC,
+0xA1, 0xA4, 0x40, 0xA4, 0xC2, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA4, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x80, 0xF0, 0x85, 0xA2, 0x80, 0xF0, 0xA6, 0xA2,
+0x80, 0xF0, 0xC4, 0xA2, 0x80, 0xF0, 0x47, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x80, 0xF0, 0x41, 0xA2, 0x62, 0x2A, 0x0A, 0xD2,
+0x0A, 0x93, 0xFF, 0xF7, 0x1F, 0x6A, 0x6C, 0xEA,
+0x0B, 0xD2, 0xB1, 0xA1, 0x50, 0xA1, 0x92, 0xA1,
+0xA0, 0x35, 0x4D, 0xED, 0x53, 0xA1, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0xA1, 0xA2, 0xC0, 0xA2, 0x82, 0xA2,
+0xA0, 0x35, 0xAD, 0xEE, 0xA3, 0xA2, 0x80, 0x34,
+0x80, 0x34, 0x00, 0xF6, 0xA0, 0x35, 0xCD, 0xEC,
+0xAD, 0xEC, 0x68, 0x32, 0x89, 0xE2, 0x81, 0xA2,
+0x00, 0xA2, 0xA2, 0xA2, 0x80, 0x34, 0x0D, 0xEC,
+0x03, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8, 0x2D, 0x20,
+0x60, 0xF0, 0x59, 0xA0, 0x01, 0x6C, 0x8C, 0xEA,
+0x0C, 0xD2, 0x0C, 0x02, 0x40, 0xA2, 0x32, 0x22,
+0x59, 0xA0, 0x98, 0xA0, 0x40, 0x32, 0x8D, 0xEA,
+0x07, 0x22, 0xF0, 0x67, 0x06, 0x06, 0x00, 0xF0,
+0x1A, 0x05, 0xD1, 0x18, 0x42, 0x0F, 0x07, 0x04,
+0x60, 0xF0, 0xF9, 0xA0, 0x05, 0xD0, 0x60, 0xF0,
+0x5A, 0xA0, 0x7F, 0x6C, 0xF2, 0x36, 0x8C, 0xEA,
+0x04, 0xD2, 0x0B, 0x94, 0x03, 0x6A, 0x4C, 0xEE,
+0xE6, 0x35, 0x07, 0x6A, 0x4C, 0xED, 0xD1, 0x18,
+0x61, 0x11, 0xFA, 0x37, 0x0B, 0x95, 0xD1, 0x18,
+0x77, 0x07, 0x08, 0x04, 0xD1, 0x18, 0x05, 0x07,
+0x90, 0x67, 0x0A, 0x92, 0x01, 0x4A, 0x20, 0x72,
+0x0A, 0xD2, 0xA2, 0x61, 0xD1, 0x18, 0xE0, 0x03,
+0x00, 0x65, 0x11, 0x97, 0x10, 0x91, 0x0F, 0x90,
+0x00, 0xEF, 0x09, 0x63, 0xFF, 0x6C, 0x00, 0x6D,
+0xD1, 0x18, 0x14, 0x6A, 0x7A, 0x4C, 0xFF, 0x6C,
+0xA1, 0x42, 0x00, 0x6E, 0xD1, 0x18, 0x40, 0x6A,
+0x7A, 0x4C, 0x0B, 0x94, 0xD1, 0x18, 0x64, 0x0F,
+0xB0, 0x67, 0x60, 0xF0, 0x43, 0x80, 0x00, 0x52,
+0x36, 0x61, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xF7, 0xF0, 0x01, 0x6A, 0x80, 0x34, 0x09, 0xF3,
+0x8C, 0x9C, 0x0A, 0x93, 0x40, 0x32, 0x40, 0x32,
+0xF2, 0xF4, 0x44, 0x9A, 0x91, 0xE3, 0x40, 0xEA,
+0x94, 0x34, 0x42, 0x35, 0xAA, 0x34, 0x03, 0x6E,
+0x8C, 0xEE, 0x81, 0xA0, 0x07, 0x6F, 0xEB, 0xEF,
+0xC4, 0x36, 0xEC, 0xEC, 0xCD, 0xEC, 0xB2, 0x35,
+0x07, 0x6E, 0xCC, 0xED, 0x39, 0x6E, 0xCB, 0xEE,
+0xAC, 0x35, 0xCC, 0xEC, 0xAD, 0xEC, 0x81, 0xC0,
+0x88, 0xA0, 0x02, 0x6D, 0xAB, 0xED, 0xAC, 0xEC,
+0xE0, 0xF1, 0x1F, 0x6D, 0xAC, 0xEA, 0x01, 0x6D,
+0x4C, 0xED, 0x46, 0x32, 0x0B, 0xD2, 0x0B, 0x02,
+0x7F, 0x6E, 0x40, 0xA2, 0xBC, 0x35, 0xCC, 0xEC,
+0xAD, 0xEC, 0x88, 0xC0, 0x49, 0xC0, 0xD1, 0x18,
+0x25, 0x07, 0x90, 0x67, 0xA6, 0x17, 0x00, 0x65,
+0x38, 0x5C, 0x40, 0xF0, 0x5B, 0xA5, 0x40, 0xF0,
+0x74, 0xA5, 0x09, 0x60, 0x24, 0x5C, 0xFE, 0x4A,
+0x04, 0x60, 0x15, 0x5C, 0x43, 0x67, 0x03, 0x61,
+0x02, 0x4A, 0xFF, 0x6B, 0x6C, 0xEA, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD0, 0x3C, 0x65, 0x85, 0x67, 0xD1, 0x18,
+0x94, 0x12, 0xA7, 0x67, 0x82, 0x67, 0x7D, 0x67,
+0xD1, 0x18, 0xB5, 0x06, 0x51, 0xC3, 0x03, 0x6C,
+0x8C, 0xEA, 0xFF, 0x6D, 0xAC, 0xEA, 0x62, 0x67,
+0x8C, 0xEB, 0x68, 0x33, 0x1D, 0x67, 0x70, 0xC0,
+0x06, 0x22, 0x03, 0x5A, 0x0D, 0x60, 0x8C, 0xEE,
+0xCD, 0xEB, 0x5D, 0x67, 0x70, 0xC2, 0x00, 0x6E,
+0x04, 0x05, 0xD1, 0x18, 0x7B, 0x09, 0x99, 0x67,
+0x07, 0x97, 0x06, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x60, 0xF0, 0x46, 0xA7, 0x10, 0x6C, 0x8C, 0xEA,
+0xAC, 0xEA, 0x12, 0x22, 0x60, 0xF0, 0x46, 0xA7,
+0x56, 0x32, 0x50, 0x32, 0x4D, 0xEB, 0x5D, 0x67,
+0x70, 0xC2, 0x5D, 0x67, 0x03, 0x6B, 0xCC, 0xEB,
+0xD0, 0xA2, 0x04, 0x6A, 0x4B, 0xEA, 0x4C, 0xEE,
+0x6D, 0xEE, 0x5D, 0x67, 0xD0, 0xC2, 0xDF, 0x17,
+0x30, 0x6A, 0xF0, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xB1, 0xA3, 0x50, 0xA3,
+0xD2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x53, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xA1, 0xA2, 0xC0, 0xA2,
+0x62, 0xA2, 0xA0, 0x35, 0xAD, 0xEE, 0xA3, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0xCD, 0xEB, 0x00, 0xF6,
+0xA0, 0x35, 0xAD, 0xEB, 0x88, 0x32, 0x69, 0xE2,
+0xA1, 0xA2, 0x60, 0xA2, 0xA0, 0x35, 0x6D, 0xED,
+0x62, 0xA2, 0x60, 0x33, 0x60, 0x33, 0xAD, 0xEB,
+0xA3, 0xA2, 0x00, 0xF6, 0xA0, 0x35, 0xD1, 0x18,
+0x64, 0x0F, 0x6D, 0xED, 0x05, 0x97, 0x00, 0x6A,
+0x00, 0xEF, 0x03, 0x63, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x00, 0x6D, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0x15, 0xF0,
+0x08, 0x48, 0x51, 0xA0, 0x70, 0xA0, 0xD2, 0xA0,
+0x40, 0x32, 0x6D, 0xEA, 0x73, 0xA0, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xEA, 0x00, 0xF6, 0x60, 0x33,
+0x4D, 0xEB, 0xC1, 0xA3, 0x40, 0xA3, 0xE2, 0xA3,
+0xC0, 0x36, 0x4D, 0xEE, 0x43, 0xA3, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x40, 0x32,
+0xCD, 0xEA, 0xA8, 0x31, 0x49, 0xE1, 0x61, 0xA2,
+0xC2, 0xA2, 0xE0, 0xA2, 0x43, 0xA2, 0x60, 0x33,
+0xC0, 0x36, 0x6D, 0xEF, 0xC0, 0x36, 0xED, 0xEE,
+0x00, 0xF6, 0x40, 0x32, 0xFF, 0xF7, 0x1F, 0x6C,
+0xCD, 0xEA, 0xAC, 0xEC, 0x52, 0x22, 0xD1, 0x18,
+0xBD, 0x12, 0x04, 0xD5, 0x91, 0xA0, 0x70, 0xA0,
+0x52, 0xA0, 0x80, 0x34, 0x6D, 0xEC, 0x73, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xC2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x49, 0xE1, 0x81, 0xA2,
+0x60, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x62, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0x8D, 0xEB, 0x83, 0xA2,
+0x00, 0xF6, 0x80, 0x34, 0xD1, 0x18, 0x05, 0x07,
+0x6D, 0xEC, 0x91, 0xA0, 0x50, 0xA0, 0x72, 0xA0,
+0x80, 0x34, 0x4D, 0xEC, 0x53, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x61, 0xA2, 0x82, 0xA2, 0xC0, 0xA2,
+0x43, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xEE,
+0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x45, 0xE1, 0x81, 0xA1, 0x40, 0xA1,
+0x80, 0x34, 0x4D, 0xEC, 0x42, 0xA1, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x83, 0xA1, 0x00, 0xF6,
+0x80, 0x34, 0xD1, 0x18, 0x25, 0x07, 0x4D, 0xEC,
+0x04, 0x95, 0x01, 0x4D, 0x20, 0x75, 0x7F, 0xF7,
+0x1A, 0x61, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0A, 0xD4, 0x15, 0xF0,
+0x08, 0x4A, 0x91, 0xA2, 0x70, 0xA2, 0xF2, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x73, 0xA2, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3, 0xE2, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x80, 0xF0, 0x65, 0xA2, 0x80, 0xF0,
+0x84, 0xA2, 0x80, 0xF0, 0xE6, 0xA2, 0x60, 0x33,
+0x8D, 0xEB, 0x80, 0xF0, 0x87, 0xA2, 0xE0, 0x37,
+0x40, 0xF0, 0x5C, 0xA6, 0xE0, 0x37, 0xED, 0xEB,
+0x00, 0xF6, 0x80, 0x34, 0x6D, 0xEC, 0x52, 0x32,
+0x03, 0x6B, 0x4C, 0xEB, 0x40, 0xA5, 0x04, 0x6F,
+0xEB, 0xEF, 0xEC, 0xEA, 0x6D, 0xEA, 0x40, 0xC5,
+0x40, 0xF0, 0x57, 0xA6, 0x40, 0xF0, 0x73, 0xA6,
+0xFC, 0x65, 0x43, 0xEB, 0x47, 0x60, 0x61, 0xC5,
+0x40, 0xF0, 0x56, 0xA6, 0x03, 0x6F, 0x0D, 0x6B,
+0x02, 0x67, 0x3A, 0x65, 0x40, 0xA5, 0xEC, 0xE8,
+0x6B, 0xEB, 0x08, 0x30, 0x6C, 0xEA, 0x0D, 0xEA,
+0x40, 0xC5, 0x40, 0xF0, 0x98, 0xA6, 0x07, 0x68,
+0x1C, 0x65, 0x38, 0x67, 0x0C, 0xE9, 0x19, 0x65,
+0x98, 0x67, 0xFF, 0x69, 0x2C, 0xEC, 0x8C, 0xE8,
+0x04, 0xD4, 0x71, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA,
+0x10, 0x30, 0x0D, 0xEA, 0x40, 0xC5, 0x5F, 0x67,
+0x60, 0xF0, 0x51, 0xA2, 0x08, 0x6C, 0x8C, 0xEA,
+0x2C, 0xEA, 0x41, 0x22, 0x80, 0xF0, 0x50, 0xA6,
+0x0C, 0x6C, 0x8C, 0xEA, 0x2C, 0xEA, 0x04, 0x6C,
+0x8E, 0xEA, 0x39, 0x2A, 0x80, 0xF0, 0x91, 0xA6,
+0x00, 0xA5, 0x0C, 0x5C, 0x81, 0xC5, 0xA0, 0xF0,
+0x4A, 0xC6, 0xA0, 0xF0, 0x4B, 0xC6, 0xA0, 0xF0,
+0x4C, 0xC6, 0xA0, 0xF0, 0x4D, 0xC6, 0x0C, 0x60,
+0x0C, 0xEB, 0x59, 0x67, 0x60, 0xC5, 0x31, 0x2A,
+0x40, 0xA5, 0x71, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x40, 0xC5, 0x15, 0x10, 0x41, 0xC5, 0xB8, 0x17,
+0xD1, 0x18, 0xB5, 0x06, 0x0C, 0xEB, 0xEC, 0xEA,
+0x4C, 0xE9, 0x2C, 0xEF, 0xE8, 0x37, 0xED, 0xEB,
+0x59, 0x67, 0x60, 0xC5, 0x1E, 0x2A, 0x03, 0x71,
+0xEB, 0x61, 0x71, 0x6A, 0x4B, 0xEA, 0x4C, 0xEB,
+0x30, 0x68, 0x0D, 0xEB, 0x60, 0xC5, 0x80, 0xF0,
+0x4F, 0xA6, 0xFF, 0x72, 0x08, 0x60, 0x03, 0x6B,
+0x4C, 0xEB, 0x40, 0xA5, 0x04, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEA, 0x6D, 0xEA, 0x40, 0xC5, 0x0A, 0x94,
+0xE6, 0x67, 0xD1, 0x18, 0x7B, 0x09, 0x00, 0x6E,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x40, 0xA5, 0x0C, 0x6B, 0x4C, 0xEB,
+0xCB, 0x23, 0x04, 0x93, 0x71, 0x6C, 0x8B, 0xEC,
+0x70, 0x33, 0x8C, 0xEA, 0x6D, 0xEA, 0xC8, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0C, 0xD4, 0x0D, 0xD5, 0x15, 0xF0, 0x08, 0x4A,
+0x91, 0xA2, 0x70, 0xA2, 0x07, 0x67, 0xF2, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x73, 0xA2, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3, 0xE2, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x80, 0xF0, 0x85, 0xA2, 0x80, 0xF0,
+0x64, 0xA2, 0x80, 0xF0, 0xE6, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x80, 0xF0, 0x67, 0xA2, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x01, 0x76, 0x8D, 0xEB, 0x05, 0x61, 0x60, 0xF0,
+0x51, 0xA3, 0x08, 0x6C, 0x8C, 0xEA, 0x12, 0x22,
+0x80, 0xF0, 0x90, 0xA0, 0x08, 0x69, 0xFF, 0x6F,
+0x2C, 0xEC, 0xEC, 0xEC, 0x60, 0xF0, 0x43, 0xA0,
+0x61, 0x24, 0x60, 0xF0, 0xD1, 0xA3, 0x2C, 0xEE,
+0xEC, 0xEE, 0x5C, 0x26, 0x18, 0x6C, 0x4C, 0xEC,
+0xEC, 0xEC, 0x58, 0x2C, 0x60, 0xF0, 0x42, 0xA0,
+0x10, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0x60, 0xF0,
+0x42, 0xC0, 0x60, 0xF0, 0x43, 0xA0, 0xAF, 0x4C,
+0x08, 0x6F, 0x8C, 0xEA, 0x03, 0x6C, 0x60, 0xF0,
+0x43, 0xC0, 0x60, 0xF0, 0x81, 0xC0, 0x60, 0xF0,
+0x71, 0xA3, 0xFF, 0x6E, 0xEC, 0xEB, 0xCC, 0xEB,
+0x29, 0x23, 0x80, 0xF0, 0x50, 0xA0, 0xEC, 0xEA,
+0xCC, 0xEA, 0x24, 0x22, 0x40, 0xF0, 0x57, 0xA0,
+0x0D, 0x93, 0x41, 0xC3, 0x40, 0xF0, 0xDC, 0xA0,
+0x60, 0xA3, 0x04, 0x6A, 0x4B, 0xEA, 0x4C, 0xEB,
+0xD2, 0x36, 0x0D, 0x92, 0x8C, 0xEE, 0xCD, 0xEB,
+0x60, 0xC2, 0x40, 0xF0, 0x56, 0xA0, 0x4C, 0xEC,
+0x0D, 0x6A, 0x4B, 0xEA, 0x6C, 0xEA, 0x0D, 0x93,
+0x88, 0x34, 0x8D, 0xEA, 0x40, 0xC3, 0x40, 0xF0,
+0x78, 0xA0, 0x07, 0x6C, 0x8C, 0xEB, 0x71, 0x6C,
+0x8B, 0xEC, 0x70, 0x33, 0x8C, 0xEA, 0x6D, 0xEA,
+0x0D, 0x93, 0x40, 0xC3, 0x0D, 0x95, 0x0C, 0x94,
+0xF0, 0x67, 0xD1, 0x18, 0x7B, 0x09, 0x00, 0x6E,
+0x60, 0xF0, 0x43, 0xA0, 0x19, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x60, 0xF0, 0x43, 0xC0, 0x80, 0xF0,
+0x50, 0xA0, 0x10, 0x4B, 0x6C, 0xEA, 0x80, 0xF0,
+0x50, 0xC0, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x00, 0xEF, 0x06, 0x63, 0x18, 0x6B, 0x6C, 0xEA,
+0xFF, 0x69, 0x2C, 0xEA, 0x31, 0x2A, 0x0C, 0x94,
+0x0D, 0x95, 0xD1, 0x18, 0x1F, 0x13, 0xD0, 0x67,
+0x60, 0xF0, 0x43, 0xA0, 0x61, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x60, 0xF0, 0x62, 0xA0, 0x10, 0x6C,
+0x8B, 0xEC, 0x6C, 0xEC, 0x60, 0xF0, 0x43, 0xC0,
+0x44, 0x67, 0x2C, 0xEA, 0x10, 0x72, 0x18, 0x61,
+0x4B, 0xA0, 0x07, 0x6D, 0x1D, 0x65, 0xB8, 0x67,
+0x4C, 0xED, 0x45, 0x67, 0x2C, 0xEA, 0x03, 0x72,
+0x0F, 0x60, 0x0F, 0x6D, 0xAC, 0xEB, 0x01, 0x4B,
+0xAC, 0xEB, 0x43, 0x67, 0x2C, 0xEA, 0x04, 0x5A,
+0x05, 0x60, 0xAC, 0xEA, 0x8D, 0xEA, 0x60, 0xF0,
+0x42, 0xC0, 0xCB, 0x17, 0x60, 0xF0, 0x82, 0xC0,
+0x03, 0x6A, 0x60, 0xF0, 0x41, 0xC0, 0xC5, 0x17,
+0x60, 0xF0, 0x43, 0xA0, 0x03, 0x6B, 0x56, 0x32,
+0x6C, 0xEA, 0x2C, 0xEA, 0x45, 0xE0, 0x40, 0xF0,
+0x99, 0xA1, 0x04, 0xD2, 0x0D, 0x92, 0x7D, 0x67,
+0x0C, 0x5C, 0x81, 0xC2, 0x40, 0xA2, 0x58, 0xC3,
+0x46, 0x60, 0x06, 0x93, 0x0D, 0x6A, 0x4B, 0xEA,
+0x4C, 0xEB, 0x0D, 0x92, 0x60, 0xC2, 0x40, 0xF0,
+0x56, 0xA0, 0x4E, 0x2A, 0x0D, 0x92, 0x03, 0x6C,
+0x40, 0xA2, 0x4A, 0x33, 0x8C, 0xEB, 0x03, 0x73,
+0x41, 0x60, 0x0D, 0x92, 0x71, 0x6B, 0x6B, 0xEB,
+0x40, 0xA2, 0x6C, 0xEA, 0x0D, 0x93, 0x40, 0xC3,
+0x80, 0xF0, 0x46, 0xA1, 0xFF, 0x72, 0x0A, 0x60,
+0x03, 0x6B, 0x4C, 0xEB, 0x0D, 0x92, 0x04, 0x6C,
+0x8B, 0xEC, 0x40, 0xA2, 0x8C, 0xEA, 0x6D, 0xEA,
+0x0D, 0x93, 0x40, 0xC3, 0x04, 0x93, 0x60, 0xF0,
+0x43, 0xA0, 0x03, 0x6E, 0x01, 0x4B, 0x61, 0x6C,
+0xCC, 0xEB, 0x8B, 0xEC, 0x8C, 0xEA, 0x74, 0x33,
+0x4D, 0xEB, 0x60, 0xF0, 0x63, 0xC0, 0x60, 0xF0,
+0x83, 0xA0, 0x19, 0x6B, 0x6B, 0xEB, 0x8E, 0x32,
+0xCC, 0xEA, 0x03, 0x4A, 0xCC, 0xEA, 0x6C, 0xEC,
+0x4C, 0x32, 0x8D, 0xEA, 0x0D, 0x95, 0x0C, 0x94,
+0x60, 0xF0, 0x43, 0xC0, 0xF0, 0x67, 0xD1, 0x18,
+0x7B, 0x09, 0x01, 0x6E, 0x6E, 0x17, 0xD1, 0x18,
+0xB5, 0x06, 0x03, 0x6B, 0x6C, 0xEA, 0x06, 0x93,
+0x0D, 0x6C, 0x8B, 0xEC, 0x48, 0x32, 0x8C, 0xEB,
+0x4D, 0xEB, 0xB3, 0x17, 0x71, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEA, 0x30, 0x6B, 0x6D, 0xEA, 0xBE, 0x17,
+0x0D, 0x92, 0x0C, 0x6B, 0x40, 0xA2, 0x4C, 0xEB,
+0xB4, 0x23, 0x40, 0xF0, 0x78, 0xA0, 0x07, 0x6C,
+0x8C, 0xEB, 0x71, 0x6C, 0x8B, 0xEC, 0x70, 0x33,
+0x8C, 0xEA, 0xF0, 0x17, 0x60, 0xF0, 0x43, 0x85,
+0x00, 0x52, 0xE0, 0xF1, 0x0C, 0x60, 0xF6, 0x63,
+0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0C, 0xD4,
+0x15, 0xF0, 0x08, 0x4A, 0x91, 0xA2, 0x70, 0xA2,
+0x05, 0x67, 0xB2, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x73, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3,
+0x40, 0xA3, 0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x43, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x80, 0xF0,
+0x85, 0xA2, 0x80, 0xF0, 0x66, 0xA2, 0x80, 0xF0,
+0xA4, 0xA2, 0x80, 0x34, 0x80, 0xF0, 0x47, 0xA2,
+0x60, 0x33, 0x8D, 0xED, 0x60, 0x33, 0xAD, 0xEB,
+0x0C, 0x95, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0x06, 0x04, 0xD1, 0x18, 0xAD, 0x07, 0x09, 0xD2,
+0x7D, 0x67, 0x78, 0xA3, 0x5D, 0x67, 0x70, 0xC2,
+0x7D, 0x67, 0x79, 0xA3, 0x71, 0xC2, 0x60, 0xF0,
+0x49, 0xA0, 0x0D, 0xD2, 0x20, 0xF0, 0x89, 0xA0,
+0x20, 0xF0, 0x48, 0xA0, 0x20, 0xF0, 0x24, 0xA0,
+0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0, 0x43, 0xA0,
+0x20, 0x31, 0xA4, 0x67, 0x4D, 0xE9, 0xD1, 0x18,
+0xBC, 0x06, 0x33, 0xE4, 0x08, 0xD2, 0x20, 0xF0,
+0xEB, 0xA0, 0x20, 0xF0, 0x4A, 0xA0, 0xE0, 0x37,
+0x4D, 0xEF, 0x33, 0xE7, 0xD1, 0x18, 0xBC, 0x06,
+0xA7, 0x67, 0x20, 0xF0, 0x86, 0xA0, 0x22, 0x67,
+0x20, 0xF0, 0x45, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0xA4, 0x67, 0xD1, 0x18, 0xBC, 0x06, 0xF3, 0xE4,
+0x0E, 0xD2, 0x40, 0xF0, 0x57, 0xA0, 0x40, 0xF0,
+0xF6, 0xA0, 0x40, 0xF0, 0xB8, 0xA0, 0x0F, 0xD2,
+0x00, 0x6A, 0x0B, 0xD2, 0x08, 0x27, 0x40, 0xF0,
+0x5C, 0xA0, 0x03, 0x6C, 0x52, 0x32, 0x8C, 0xEA,
+0xFF, 0x6C, 0x8C, 0xEA, 0x0B, 0xD2, 0x5D, 0x67,
+0x50, 0xA2, 0x0C, 0x6C, 0xFA, 0x65, 0x4C, 0xEC,
+0x00, 0x6A, 0x0A, 0xD2, 0x06, 0x24, 0x7F, 0x67,
+0x03, 0x6A, 0x6C, 0xEA, 0xFF, 0x6B, 0x4C, 0xEB,
+0x0A, 0xD3, 0x5D, 0x67, 0x51, 0xA2, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0x44, 0x32,
+0x67, 0xF4, 0x10, 0x4E, 0xC9, 0xE2, 0x0C, 0x74,
+0x40, 0xAA, 0xE0, 0xF0, 0x07, 0x61, 0xFF, 0x6B,
+0x3B, 0x65, 0x50, 0x6B, 0x1B, 0x65, 0x98, 0x67,
+0x7F, 0x67, 0x6C, 0xEC, 0x10, 0x74, 0x09, 0x60,
+0x70, 0x6C, 0x7F, 0x67, 0x8C, 0xEB, 0x83, 0x67,
+0x79, 0x67, 0x6C, 0xEC, 0x50, 0x74, 0xC0, 0xF0,
+0x0B, 0x61, 0x4E, 0x34, 0x56, 0x33, 0x71, 0xE4,
+0x51, 0xE4, 0x5A, 0x32, 0x49, 0xE4, 0x0F, 0x93,
+0x03, 0x77, 0x64, 0x34, 0x99, 0xE6, 0x80, 0xAE,
+0xC0, 0xF0, 0x1B, 0x61, 0x03, 0x6E, 0xCB, 0xEE,
+0xE5, 0x67, 0xCC, 0xEF, 0x01, 0x77, 0x03, 0x60,
+0x05, 0x75, 0xC0, 0xF0, 0x0A, 0x61, 0x8E, 0x35,
+0x96, 0x36, 0xD5, 0xE5, 0x95, 0xE5, 0x9A, 0x34,
+0x91, 0xE5, 0x60, 0xF0, 0x62, 0xA0, 0x10, 0x6D,
+0xAB, 0xED, 0x1B, 0x65, 0x78, 0x67, 0xAC, 0xEB,
+0xFF, 0x6F, 0xEC, 0xEB, 0x10, 0x73, 0x00, 0x6E,
+0x09, 0x61, 0x08, 0x6B, 0x80, 0xF0, 0xB0, 0xA0,
+0x1B, 0x65, 0x78, 0x67, 0x6C, 0xED, 0xEC, 0xED,
+0x01, 0x2D, 0x05, 0x6E, 0x0D, 0x93, 0x64, 0x6D,
+0x7F, 0xE5, 0xD9, 0xE7, 0x98, 0xEE, 0x0B, 0x93,
+0xFF, 0x6C, 0x00, 0x6E, 0x7B, 0x4C, 0x12, 0xEF,
+0xE4, 0xEB, 0x08, 0x93, 0x0B, 0xD7, 0x77, 0xE5,
+0x58, 0xED, 0x0A, 0x93, 0x08, 0x95, 0x12, 0xEA,
+0x44, 0xEB, 0xD1, 0x18, 0x40, 0x6A, 0x0A, 0xD2,
+0x00, 0x6E, 0x80, 0xF1, 0x07, 0x6C, 0xD1, 0x18,
+0x40, 0x6A, 0xB1, 0x67, 0x0E, 0x95, 0x80, 0xF1,
+0x08, 0x6C, 0xD1, 0x18, 0x40, 0x6A, 0x00, 0x6E,
+0x0A, 0x92, 0x0B, 0x97, 0x60, 0xF0, 0x82, 0xA0,
+0xE3, 0xEA, 0x03, 0x61, 0x1E, 0x59, 0x80, 0xF0,
+0x17, 0x61, 0x10, 0x6B, 0x6B, 0xEB, 0x8C, 0xEB,
+0x1B, 0x65, 0xFF, 0x6B, 0x3B, 0x65, 0x99, 0x67,
+0x78, 0x67, 0x8C, 0xEB, 0x10, 0x73, 0x1F, 0x61,
+0x1E, 0x59, 0x1D, 0x61, 0x22, 0x67, 0x5D, 0x67,
+0x90, 0xA2, 0x03, 0x6B, 0xB1, 0xA2, 0x8A, 0x34,
+0x6C, 0xEC, 0x59, 0x67, 0x00, 0xF0, 0x12, 0x07,
+0x00, 0xF0, 0x13, 0x06, 0xD1, 0x18, 0x7F, 0x06,
+0x4C, 0xEC, 0x5D, 0x67, 0x70, 0xA2, 0x70, 0x6C,
+0x8C, 0xEB, 0x93, 0xA2, 0x59, 0x67, 0x8D, 0xEB,
+0x4C, 0xEB, 0x5D, 0x67, 0x92, 0xA2, 0x51, 0x67,
+0x6C, 0x2C, 0x80, 0xF0, 0x60, 0xC0, 0x09, 0x93,
+0xFF, 0x6D, 0x00, 0x6E, 0x60, 0xF0, 0x91, 0xA3,
+0x08, 0x6B, 0x6C, 0xEC, 0xAC, 0xEC, 0x32, 0x24,
+0x80, 0xF0, 0x70, 0xA0, 0x04, 0x6C, 0x8C, 0xEB,
+0xAC, 0xEB, 0x2C, 0x23, 0xA0, 0xF0, 0x8B, 0xA0,
+0xA0, 0xF0, 0x6A, 0xA0, 0x80, 0x34, 0x6D, 0xEC,
+0x1C, 0x65, 0xA0, 0xF0, 0x8C, 0xA0, 0x78, 0x67,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB, 0x1B, 0x65,
+0xA0, 0xF0, 0x6D, 0xA0, 0x98, 0x67, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x43, 0xEB, 0x16, 0x60,
+0x7D, 0x67, 0x71, 0xA3, 0x03, 0x6C, 0xA0, 0xF0,
+0x4A, 0xC0, 0x80, 0xF0, 0x71, 0xC0, 0x7D, 0x67,
+0x70, 0xA3, 0x8C, 0xEB, 0x80, 0xF0, 0x6F, 0xC0,
+0x42, 0x33, 0xA0, 0xF0, 0x6B, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x62, 0x33, 0xA0, 0xF0, 0x6C, 0xC0,
+0xA0, 0xF0, 0x4D, 0xC0, 0x0C, 0x94, 0xF0, 0x67,
+0xD1, 0x18, 0x80, 0x13, 0x04, 0x05, 0x13, 0x97,
+0x12, 0x91, 0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63,
+0x21, 0x6B, 0x6B, 0xEB, 0xE0, 0x4C, 0x6C, 0xEC,
+0x79, 0x67, 0x6C, 0xEC, 0x3F, 0xF7, 0x13, 0x2C,
+0x52, 0x34, 0x2C, 0x17, 0x70, 0x6C, 0x7F, 0x67,
+0x8C, 0xEB, 0x3F, 0xF7, 0x0C, 0x23, 0x52, 0x34,
+0x56, 0x33, 0x71, 0xE4, 0x51, 0xE4, 0x5E, 0x32,
+0x25, 0x17, 0xFE, 0x4D, 0xAC, 0xEE, 0xFF, 0x6D,
+0xAC, 0xEE, 0x3F, 0xF7, 0x16, 0x2E, 0x92, 0x35,
+0x2F, 0x17, 0x3F, 0xF7, 0x12, 0x25, 0x92, 0x35,
+0x96, 0x36, 0xD5, 0xE5, 0x95, 0xE5, 0x9E, 0x34,
+0x2B, 0x17, 0x80, 0xF0, 0x61, 0xC0, 0x93, 0x17,
+0x09, 0x92, 0x08, 0x6D, 0xFF, 0x6E, 0x60, 0xF0,
+0x51, 0xA2, 0xAC, 0xEA, 0xCC, 0xEA, 0x2C, 0x22,
+0x5D, 0x67, 0x51, 0xA2, 0x03, 0x6F, 0x31, 0x69,
+0x40, 0xF0, 0x57, 0xC0, 0x5D, 0x67, 0x50, 0xA2,
+0x2B, 0xE9, 0x08, 0x93, 0xEC, 0xEA, 0x50, 0x35,
+0x40, 0xF0, 0x5C, 0xA0, 0x2C, 0xEA, 0xAD, 0xEA,
+0x40, 0xF0, 0x5C, 0xC0, 0x5D, 0x67, 0x50, 0xA2,
+0x4A, 0x35, 0xAC, 0xEF, 0xFF, 0xF7, 0x1F, 0x6D,
+0xAC, 0xEB, 0x6C, 0xEE, 0x62, 0x33, 0x60, 0xF0,
+0x6A, 0xC0, 0x52, 0x32, 0x07, 0x6B, 0x6C, 0xEA,
+0x40, 0xF0, 0x58, 0xC0, 0x80, 0xF0, 0x50, 0xA0,
+0x08, 0x6B, 0x40, 0xF0, 0xF6, 0xC0, 0x6D, 0xEA,
+0x60, 0xF0, 0xC9, 0xC0, 0x80, 0xF0, 0x50, 0xC0,
+0x10, 0x6A, 0x4B, 0xEA, 0x8C, 0xEA, 0xFF, 0x6C,
+0x8C, 0xEA, 0x10, 0x72, 0x02, 0x60, 0x01, 0x6E,
+0x91, 0x17, 0x01, 0x6A, 0x40, 0xF0, 0x5F, 0xC0,
+0x4B, 0xA0, 0x07, 0x6D, 0x62, 0x67, 0xAC, 0xEB,
+0x6C, 0xEC, 0x03, 0x74, 0xF4, 0x60, 0x08, 0x6C,
+0x01, 0x4B, 0x8B, 0xEC, 0xAC, 0xEB, 0x8C, 0xEA,
+0x6D, 0xEA, 0x4B, 0xC0, 0xEC, 0x17, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xC0, 0xF2, 0x07, 0x24,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xC9, 0xA3, 0x48, 0xA3,
+0xAA, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x4B, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0xB1, 0xA2, 0xD2, 0xA2,
+0xF0, 0xA2, 0x53, 0xA2, 0xA0, 0x35, 0xC0, 0x36,
+0xAD, 0xEF, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x41, 0xA2, 0x60, 0xF2,
+0x07, 0x2A, 0xD1, 0xA3, 0xB0, 0xA3, 0x52, 0xA3,
+0xC0, 0x36, 0xAD, 0xEE, 0xB3, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xEE, 0x00, 0xF6, 0xA0, 0x35,
+0xCD, 0xED, 0xC1, 0xA5, 0x40, 0xA5, 0xE2, 0xA5,
+0xC0, 0x36, 0x4D, 0xEE, 0x43, 0xA5, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x40, 0x32,
+0xCD, 0xEA, 0x80, 0xF0, 0xC5, 0xA2, 0x80, 0xF0,
+0xA6, 0xA2, 0x80, 0xF0, 0xE4, 0xA2, 0x80, 0xF0,
+0x47, 0xA2, 0xC0, 0x36, 0xA0, 0x35, 0xCD, 0xEF,
+0xA0, 0x35, 0xED, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x07, 0xD2, 0x80, 0xF0, 0x41, 0xA2,
+0x06, 0xD2, 0x20, 0xF2, 0x19, 0x2A, 0xBD, 0x67,
+0xB8, 0xA5, 0x5D, 0x67, 0xFF, 0x6E, 0xB4, 0xC2,
+0x40, 0xA4, 0x1F, 0x6D, 0x4C, 0xED, 0x20, 0xF2,
+0x0F, 0x2D, 0xA1, 0x84, 0x00, 0x55, 0x20, 0xF2,
+0x0B, 0x61, 0x20, 0x6D, 0xAC, 0xEA, 0x20, 0xF2,
+0x07, 0x2A, 0x4C, 0xA4, 0x09, 0xD2, 0x20, 0xF2,
+0x03, 0x22, 0x4D, 0xA4, 0x3F, 0x6D, 0xAC, 0xEA,
+0xCC, 0xEA, 0x00, 0xF2, 0x1D, 0x22, 0xB1, 0xA4,
+0x40, 0x68, 0x0C, 0xED, 0xCC, 0xED, 0x00, 0xF2,
+0x17, 0x2D, 0x49, 0xA4, 0x0C, 0xEA, 0xCC, 0xEA,
+0x00, 0xF2, 0x12, 0x2A, 0x42, 0xA4, 0x7F, 0x6D,
+0xAC, 0xEA, 0xFF, 0x6D, 0xAC, 0xEA, 0x20, 0x5A,
+0x08, 0xD2, 0x00, 0xF2, 0x09, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x24, 0x67,
+0xD2, 0xF6, 0x40, 0x9A, 0x08, 0x94, 0x0A, 0xD5,
+0x40, 0xEA, 0x03, 0x67, 0xE0, 0xF1, 0x1C, 0x22,
+0x91, 0xA0, 0x50, 0xA0, 0xD2, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x53, 0xA0, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x81, 0xA2, 0xC2, 0xA2, 0x70, 0x67, 0x00, 0xA2,
+0x80, 0x34, 0x43, 0xA2, 0x8D, 0xE8, 0xC0, 0x36,
+0x08, 0x94, 0xC0, 0x36, 0x0D, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x88, 0x34, 0x49, 0xE4,
+0x81, 0xA2, 0x00, 0xA2, 0xC2, 0xA2, 0x80, 0x34,
+0x0D, 0xEC, 0x03, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8,
+0xC0, 0xF1, 0x12, 0x20, 0x51, 0xA1, 0x0A, 0x95,
+0x30, 0x6C, 0x8C, 0xEA, 0xAC, 0xEA, 0xC0, 0xF1,
+0x0B, 0x2A, 0x4D, 0xA1, 0x3F, 0x6C, 0x1C, 0x65,
+0x98, 0x67, 0x4C, 0xEC, 0x44, 0x67, 0xAC, 0xEA,
+0x01, 0x72, 0x0F, 0x61, 0x51, 0xA1, 0xD0, 0xA1,
+0x01, 0x6C, 0x8C, 0xEA, 0x40, 0x32, 0xCD, 0xEA,
+0xFF, 0xF7, 0x1F, 0x6C, 0x8C, 0xEA, 0x4C, 0xED,
+0x42, 0x32, 0x80, 0xF0, 0xAD, 0xC0, 0x80, 0xF0,
+0x4E, 0xC0, 0x40, 0xA0, 0x08, 0x6C, 0xFF, 0x6F,
+0x8C, 0xEA, 0xEC, 0xEA, 0xA0, 0xF0, 0x02, 0x22,
+0x08, 0x94, 0x04, 0x06, 0x00, 0x6D, 0x0B, 0xD7,
+0xD1, 0x18, 0x1B, 0x6B, 0x0A, 0xD3, 0x4D, 0xA1,
+0x3F, 0x6B, 0x1B, 0x65, 0x78, 0x67, 0x0B, 0x97,
+0x4C, 0xEB, 0x43, 0x67, 0xEC, 0xEA, 0x01, 0x72,
+0x0A, 0x93, 0x80, 0xF0, 0x0A, 0x61, 0x5D, 0x67,
+0x48, 0xAA, 0x01, 0x6C, 0xD0, 0xA1, 0x1A, 0x65,
+0x51, 0xA1, 0xE0, 0xF1, 0x1F, 0x6D, 0x4C, 0xEC,
+0x80, 0x34, 0x3C, 0x65, 0x98, 0x67, 0x8C, 0xED,
+0x99, 0x67, 0xCD, 0xEC, 0xAE, 0xEC, 0x79, 0x2C,
+0x9D, 0x67, 0xB1, 0xA4, 0x89, 0xA1, 0xAA, 0x36,
+0x92, 0x34, 0xCE, 0xEC, 0x03, 0x6E, 0xCC, 0xEC,
+0xEC, 0xEC, 0x6F, 0x2C, 0xB2, 0x35, 0x46, 0x32,
+0xAE, 0xEA, 0x07, 0x6D, 0xAC, 0xEA, 0xEC, 0xEA,
+0x68, 0x2A, 0x5D, 0x67, 0x93, 0xA2, 0x41, 0xA1,
+0x8E, 0x34, 0x5A, 0x32, 0x8E, 0xEA, 0x01, 0x6C,
+0x8C, 0xEA, 0xEC, 0xEA, 0x5E, 0x2A, 0x80, 0xA0,
+0xF7, 0x4A, 0x40, 0x6D, 0x8C, 0xEA, 0x81, 0xA1,
+0x40, 0xC0, 0xAC, 0xEC, 0xEC, 0xEC, 0x55, 0x2C,
+0x04, 0x6C, 0x8C, 0xEA, 0xEC, 0xEA, 0x04, 0x2A,
+0xA0, 0xF0, 0x54, 0xA0, 0x02, 0x72, 0x4D, 0x61,
+0x91, 0xA3, 0x50, 0xA3, 0xB2, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x53, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x61, 0xA2, 0xA0, 0xA2, 0x82, 0xA2, 0x60, 0x33,
+0xAD, 0xEB, 0xA3, 0xA2, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEB, 0x00, 0xF6, 0xA0, 0x35, 0x6D, 0xED,
+0x00, 0x6C, 0x00, 0x6B, 0x88, 0x32, 0x49, 0xE5,
+0xC0, 0xA2, 0x1E, 0x65, 0xC1, 0xA2, 0xF8, 0x67,
+0xC0, 0x36, 0xCD, 0xEF, 0xC2, 0xA2, 0x43, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0x4D, 0xEE, 0x0E, 0x26, 0x08, 0x97,
+0xFF, 0xF7, 0x1F, 0x6A, 0x8C, 0xEA, 0xEE, 0xEA,
+0x08, 0x22, 0xC0, 0xA6, 0x04, 0x6A, 0xCC, 0xEA,
+0xFF, 0x6E, 0xCC, 0xEA, 0x02, 0x22, 0x01, 0x4B,
+0xCC, 0xEB, 0x01, 0x4C, 0x20, 0x74, 0xDE, 0x61,
+0x0B, 0x2B, 0x07, 0x92, 0x00, 0x6E, 0x05, 0x05,
+0x60, 0xF0, 0x5F, 0xA2, 0x01, 0x6C, 0x62, 0x67,
+0x5D, 0x67, 0xD1, 0x18, 0xE5, 0x6B, 0x74, 0xC2,
+0x40, 0xA0, 0x05, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x40, 0xC0, 0x40, 0xA0, 0x08, 0x6B, 0x6C, 0xEA,
+0x00, 0xF1, 0x0A, 0x2A, 0x41, 0xA1, 0x40, 0x6B,
+0x00, 0x6C, 0x6C, 0xEA, 0xFF, 0x6B, 0x6C, 0xEA,
+0x0B, 0xD4, 0x05, 0x22, 0x60, 0xF0, 0x43, 0xA0,
+0x5E, 0x32, 0x6C, 0xEA, 0x0B, 0xD2, 0x49, 0xA1,
+0x01, 0x6C, 0x68, 0xA1, 0x8C, 0xEA, 0x40, 0x32,
+0x6D, 0xEA, 0x71, 0xA1, 0xB0, 0xA1, 0x8C, 0xEB,
+0x60, 0x33, 0xAD, 0xEB, 0x4E, 0xEB, 0xFF, 0xF7,
+0x1F, 0x6C, 0x80, 0xF0, 0x0E, 0x2B, 0x6E, 0xA1,
+0x8C, 0xEB, 0x0A, 0xD3, 0x6C, 0xA1, 0x8C, 0xEB,
+0x3B, 0x65, 0xA3, 0xA1, 0x3F, 0x6B, 0xFF, 0x6C,
+0x6C, 0xED, 0x8C, 0xED, 0x10, 0x5D, 0x17, 0x60,
+0x01, 0x6B, 0x64, 0xED, 0x91, 0xF0, 0x08, 0x6D,
+0x03, 0x6E, 0x6C, 0xED, 0x06, 0xD6, 0x0F, 0x2D,
+0x48, 0xF4, 0x04, 0x6D, 0x02, 0x6E, 0x6C, 0xED,
+0x06, 0xD6, 0x09, 0x2D, 0x24, 0xF2, 0x02, 0x6F,
+0x6C, 0xEF, 0x01, 0x5F, 0xF8, 0x67, 0x01, 0x6B,
+0x6E, 0xEF, 0x8C, 0xEF, 0x06, 0xD7, 0x20, 0xF0,
+0x6E, 0xA0, 0x20, 0xF0, 0x8D, 0xA0, 0xFF, 0xF7,
+0x1F, 0x6D, 0x60, 0x33, 0x8D, 0xEB, 0x0C, 0xD3,
+0x0C, 0x94, 0xFF, 0xF5, 0x09, 0x6B, 0x63, 0xEC,
+0x39, 0x60, 0x06, 0x93, 0x64, 0x37, 0xFD, 0xE0,
+0x20, 0xF0, 0x90, 0xA7, 0x6E, 0xA1, 0x20, 0xF0,
+0xCF, 0xA7, 0x80, 0x34, 0xAC, 0xEB, 0xCD, 0xEC,
+0x91, 0xE3, 0xAC, 0xEC, 0x20, 0xF0, 0x8F, 0xC7,
+0x82, 0x34, 0x3F, 0x6E, 0x20, 0xF0, 0x90, 0xC7,
+0x8D, 0xA1, 0x1E, 0x65, 0xD8, 0x67, 0x8C, 0xEE,
+0x06, 0xD6, 0x06, 0x04, 0x80, 0xA4, 0x02, 0x54,
+0x12, 0x61, 0x20, 0xF0, 0x97, 0xA7, 0x1C, 0x65,
+0x20, 0xF0, 0x98, 0xA7, 0xD8, 0x67, 0x80, 0x34,
+0xCD, 0xEC, 0x8D, 0xE3, 0x6C, 0xED, 0x06, 0xD5,
+0x06, 0x03, 0xC0, 0xA3, 0xA2, 0x35, 0x20, 0xF0,
+0xB8, 0xC7, 0x20, 0xF0, 0xD7, 0xC7, 0x09, 0x94,
+0x0C, 0x93, 0x8D, 0xE3, 0xFF, 0xF7, 0x1F, 0x6C,
+0x8C, 0xEB, 0x20, 0xF0, 0x6D, 0xC0, 0x62, 0x33,
+0x20, 0xF0, 0x6E, 0xC0, 0x80, 0xA1, 0x40, 0x6B,
+0x6B, 0xEB, 0x8C, 0xEB, 0x40, 0x73, 0x29, 0x61,
+0x06, 0x5A, 0x1F, 0x60, 0x04, 0x5A, 0x02, 0x60,
+0x03, 0x5A, 0x23, 0x60, 0x07, 0x92, 0x60, 0xF0,
+0x76, 0xA2, 0x60, 0xF0, 0x57, 0xA2, 0x40, 0x32,
+0x6D, 0xEA, 0xFF, 0xF5, 0x08, 0x6B, 0x63, 0xEA,
+0x18, 0x60, 0x09, 0x93, 0x69, 0xE2, 0xFF, 0xF7,
+0x1F, 0x6B, 0x6C, 0xEA, 0x07, 0x93, 0x60, 0xF0,
+0x56, 0xC3, 0x42, 0x32, 0x60, 0xF0, 0x57, 0xC3,
+0x0C, 0x10, 0x00, 0x6B, 0x3B, 0x65, 0x0A, 0xD3,
+0x74, 0x17, 0x00, 0xF1, 0x00, 0x72, 0xE2, 0x60,
+0x80, 0xF1, 0x00, 0x72, 0xDF, 0x60, 0x80, 0x72,
+0xDD, 0x60, 0x0B, 0x92, 0xFF, 0xF7, 0x1F, 0x6B,
+0x8E, 0xA1, 0x53, 0x22, 0x20, 0xF0, 0x44, 0xA0,
+0x20, 0xF0, 0xC3, 0xA0, 0x0A, 0x95, 0x40, 0x32,
+0xCD, 0xEA, 0x49, 0xE5, 0x6C, 0xEA, 0x42, 0x35,
+0x20, 0xF0, 0xA4, 0xC0, 0x20, 0xF0, 0xA9, 0xA0,
+0x20, 0xF0, 0xE8, 0xA0, 0xD9, 0x67, 0xA0, 0x35,
+0xED, 0xED, 0xB5, 0xE6, 0x6C, 0xED, 0x20, 0xF0,
+0xA8, 0xC0, 0xA2, 0x35, 0x20, 0xF0, 0xA9, 0xC0,
+0x20, 0xF0, 0xA6, 0xA0, 0x20, 0xF0, 0xE5, 0xA0,
+0x09, 0x96, 0xA0, 0x35, 0xED, 0xED, 0xD5, 0xE5,
+0x6C, 0xED, 0xA2, 0x36, 0x20, 0xF0, 0xC6, 0xC0,
+0x20, 0xF0, 0xCB, 0xA0, 0x20, 0xF0, 0xEA, 0xA0,
+0x20, 0xF0, 0x43, 0xC0, 0xC0, 0x36, 0xED, 0xEE,
+0x91, 0xE6, 0x6C, 0xEC, 0x20, 0xF0, 0x8A, 0xC0,
+0x82, 0x34, 0x20, 0xF0, 0x8B, 0xC0, 0x20, 0xF0,
+0x87, 0xA0, 0xFF, 0x6E, 0x20, 0xF0, 0xA5, 0xC0,
+0x01, 0x4C, 0xCC, 0xEC, 0x60, 0xF0, 0xC7, 0xA0,
+0x20, 0xF0, 0x87, 0xC0, 0xCC, 0xEB, 0x63, 0xED,
+0x0B, 0x61, 0xAE, 0xEA, 0x02, 0x22, 0x02, 0x5C,
+0x07, 0x61, 0x08, 0x94, 0xD1, 0x18, 0x29, 0x14,
+0xB0, 0x67, 0xD1, 0x18, 0x0C, 0x07, 0x90, 0x67,
+0x11, 0x97, 0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF,
+0x09, 0x63, 0x5D, 0xA0, 0xDC, 0xA0, 0x0A, 0x95,
+0x40, 0x32, 0xCD, 0xEA, 0x49, 0xE5, 0x6C, 0xEA,
+0x5C, 0xC0, 0x42, 0x32, 0x5D, 0xC0, 0x5B, 0xA0,
+0xDA, 0xA0, 0xB9, 0x67, 0x40, 0x32, 0xCD, 0xEA,
+0x49, 0xE5, 0x6C, 0xEA, 0x5A, 0xC0, 0x42, 0x32,
+0x5B, 0xC0, 0x59, 0xA0, 0xD8, 0xA0, 0x09, 0x95,
+0x40, 0x32, 0xCD, 0xEA, 0xA9, 0xE2, 0x6C, 0xEA,
+0xA2, 0x67, 0x42, 0x32, 0x59, 0xC0, 0x5F, 0xA0,
+0x3E, 0xA0, 0xFF, 0x6F, 0x40, 0x32, 0x2D, 0xEA,
+0x89, 0xE2, 0x6C, 0xEA, 0xEC, 0xED, 0x4C, 0xEF,
+0x42, 0x32, 0x5F, 0xC0, 0x20, 0xF0, 0x40, 0xA0,
+0xFF, 0x6C, 0xB8, 0xC0, 0x01, 0x4A, 0xFE, 0xC0,
+0x20, 0xF0, 0x40, 0xC0, 0x00, 0x6E, 0xD1, 0x18,
+0x40, 0x6A, 0x7C, 0x4C, 0xC5, 0x17, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFF, 0x63, 0x01, 0xD0,
+0x33, 0x2D, 0x04, 0x67, 0x00, 0x6B, 0x1F, 0x6A,
+0x01, 0x6F, 0xE4, 0xEA, 0x0C, 0xEF, 0x33, 0x27,
+0x4D, 0xE3, 0xFF, 0x6F, 0xEC, 0xEB, 0x0C, 0x5B,
+0x09, 0x61, 0x40, 0xF0, 0x5D, 0xA6, 0x1F, 0x68,
+0x0C, 0xEA, 0xEC, 0xEA, 0x10, 0x52, 0x23, 0x61,
+0x48, 0x4B, 0xEC, 0xEB, 0x00, 0x6F, 0x02, 0x2C,
+0x85, 0x67, 0x20, 0x6F, 0x00, 0x6A, 0x01, 0x6D,
+0xA4, 0xEA, 0x8C, 0xED, 0x25, 0x25, 0x49, 0xE7,
+0xFF, 0x6D, 0xAC, 0xEA, 0x0C, 0x5A, 0x09, 0x61,
+0x40, 0xF0, 0x9D, 0xA6, 0x1F, 0x6F, 0xEC, 0xEC,
+0xAC, 0xEC, 0x10, 0x54, 0x15, 0x61, 0x48, 0x4A,
+0xAC, 0xEA, 0x01, 0x90, 0x40, 0xF0, 0x73, 0xC6,
+0x40, 0xF0, 0x54, 0xC6, 0x20, 0xE8, 0x01, 0x63,
+0x05, 0x67, 0x20, 0x6B, 0xCC, 0x17, 0x08, 0x52,
+0xDD, 0x61, 0x20, 0x4B, 0xDA, 0x17, 0xFF, 0x4A,
+0xE1, 0x42, 0xC6, 0x2F, 0x00, 0x6B, 0xD6, 0x17,
+0x08, 0x54, 0xEB, 0x61, 0x20, 0x4A, 0xE8, 0x17,
+0x01, 0x4A, 0x20, 0x72, 0xD4, 0x61, 0x00, 0x6A,
+0xE4, 0x17, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x00, 0x6A, 0x04, 0xD2,
+0x20, 0xF1, 0x17, 0x27, 0xC2, 0x32, 0xFF, 0x69,
+0x08, 0xD4, 0x82, 0x67, 0x42, 0x32, 0x07, 0x67,
+0x2C, 0xEA, 0xFA, 0x65, 0x40, 0xF0, 0x4D, 0xC0,
+0x00, 0xF6, 0xC2, 0x32, 0x06, 0xD2, 0x5D, 0x67,
+0x58, 0xA2, 0x65, 0x67, 0xE6, 0x67, 0x40, 0xF0,
+0x4E, 0xC0, 0x43, 0x67, 0x2C, 0xEA, 0x3A, 0x65,
+0x40, 0xF0, 0x47, 0xC0, 0x62, 0x32, 0xA6, 0x67,
+0xC2, 0x67, 0x42, 0x32, 0x2C, 0xEA, 0x07, 0xD2,
+0x5D, 0x67, 0x5C, 0xA2, 0x2C, 0xEE, 0x1E, 0x65,
+0x40, 0xF0, 0x49, 0xC0, 0x59, 0x67, 0x20, 0xF0,
+0x5F, 0xC0, 0x58, 0x67, 0x40, 0xF0, 0x40, 0xC0,
+0x5D, 0x67, 0x5C, 0xA2, 0x2C, 0xED, 0x2C, 0xEC,
+0x40, 0xF0, 0x41, 0xC0, 0x5F, 0x67, 0x40, 0xF0,
+0x45, 0xC0, 0x5D, 0x67, 0x58, 0xA2, 0x40, 0xF0,
+0xC8, 0xC0, 0x00, 0xF6, 0x62, 0x36, 0x40, 0xF0,
+0xAB, 0xC0, 0x40, 0xF0, 0x8C, 0xC0, 0x40, 0xF0,
+0xCA, 0xC0, 0x40, 0xF0, 0xC2, 0xC0, 0x40, 0xF0,
+0xA3, 0xC0, 0x40, 0xF0, 0x84, 0xC0, 0x40, 0xF0,
+0x46, 0xC0, 0x83, 0x67, 0xD0, 0x67, 0xD1, 0x18,
+0x89, 0x16, 0xA7, 0x67, 0x40, 0xF0, 0x5D, 0xA0,
+0x1F, 0x6B, 0x07, 0x6C, 0x6C, 0xEA, 0x4C, 0xE9,
+0x60, 0xF0, 0x45, 0xA0, 0x10, 0x51, 0x52, 0x33,
+0x8C, 0xEB, 0xE0, 0xF0, 0x03, 0x61, 0x64, 0x34,
+0x8D, 0xE3, 0x68, 0x33, 0x5D, 0x4B, 0x40, 0xF0,
+0x7B, 0xC0, 0x40, 0xF0, 0x93, 0xA0, 0x40, 0xF0,
+0x7B, 0xA0, 0x63, 0xEC, 0x02, 0x60, 0x40, 0xF0,
+0x9B, 0xC0, 0x04, 0x51, 0x60, 0xF0, 0x9E, 0xA0,
+0xC0, 0xF0, 0x1A, 0x61, 0x07, 0x6B, 0x52, 0x32,
+0x6C, 0xEA, 0x01, 0x4A, 0x1F, 0x6B, 0x54, 0x32,
+0x8C, 0xEB, 0x6D, 0xEA, 0x60, 0xF0, 0x5E, 0xC0,
+0x40, 0xF0, 0xA8, 0xA0, 0x40, 0xF0, 0x47, 0xA0,
+0xFF, 0x6C, 0xA0, 0x35, 0x4D, 0xED, 0x40, 0xF0,
+0x49, 0xA0, 0x02, 0x6E, 0x7D, 0x4C, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x40, 0xF0, 0xAA, 0xA0,
+0x00, 0xF6, 0xA0, 0x35, 0xD1, 0x18, 0x40, 0x6A,
+0x4D, 0xED, 0x40, 0xF0, 0xAC, 0xA0, 0x40, 0xF0,
+0x4B, 0xA0, 0x02, 0x6E, 0xA0, 0x35, 0x4D, 0xED,
+0x40, 0xF0, 0x4D, 0xA0, 0x80, 0xF1, 0x00, 0x6C,
+0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA, 0x40, 0xF0,
+0xAE, 0xA0, 0x00, 0xF6, 0xA0, 0x35, 0xD1, 0x18,
+0x40, 0x6A, 0x4D, 0xED, 0x40, 0xF0, 0xB3, 0xA0,
+0x80, 0xF1, 0x04, 0x6C, 0xD1, 0x18, 0x40, 0x6A,
+0x00, 0x6E, 0x40, 0xF0, 0xB4, 0xA0, 0x80, 0xF1,
+0x05, 0x6C, 0xD1, 0x18, 0x40, 0x6A, 0x00, 0x6E,
+0x08, 0x92, 0x05, 0x04, 0xA2, 0x67, 0xD1, 0x18,
+0x77, 0x07, 0x06, 0xD2, 0x5D, 0x67, 0x94, 0xA2,
+0xB5, 0xA2, 0x03, 0x6B, 0x8A, 0x34, 0x00, 0xF0,
+0x17, 0x06, 0x6C, 0xEC, 0x00, 0xF0, 0x16, 0x07,
+0xD1, 0x18, 0x7F, 0x06, 0x7C, 0xC2, 0xB0, 0x67,
+0xD1, 0x18, 0x43, 0x09, 0x05, 0x04, 0x60, 0xF0,
+0xC5, 0xA0, 0x22, 0x67, 0x07, 0x6A, 0xD2, 0x36,
+0x4C, 0xEE, 0x5D, 0x67, 0x56, 0xA2, 0x42, 0xEE,
+0x0D, 0x60, 0x5D, 0x67, 0x94, 0xA2, 0x7C, 0xA2,
+0xB7, 0xA2, 0x8A, 0x34, 0x8C, 0xEB, 0xFF, 0x6C,
+0xD1, 0x18, 0x71, 0x06, 0x6C, 0xEC, 0x7D, 0x67,
+0x55, 0xC3, 0x03, 0x69, 0x7D, 0x67, 0x74, 0xA3,
+0x40, 0xF0, 0x5C, 0xA0, 0x03, 0x6C, 0xA3, 0x67,
+0xFF, 0x6E, 0x8C, 0xEA, 0x8C, 0xED, 0xCC, 0xEA,
+0xCC, 0xED, 0xA2, 0xEA, 0x08, 0x60, 0x04, 0x6D,
+0xAB, 0xED, 0x4C, 0xEC, 0xAC, 0xEB, 0x8D, 0xEB,
+0x9D, 0x67, 0x74, 0xC4, 0x04, 0x69, 0x7D, 0x67,
+0x94, 0xA3, 0x03, 0x6D, 0x8A, 0x33, 0xAC, 0xEB,
+0x04, 0x23, 0x7D, 0x67, 0x77, 0xA3, 0x06, 0x5B,
+0x0E, 0x60, 0xA0, 0xA0, 0x40, 0x6B, 0xFF, 0x6E,
+0x6C, 0xED, 0xCC, 0xED, 0x17, 0x25, 0x0C, 0x6B,
+0x8C, 0xEB, 0xCC, 0xEB, 0x0C, 0x73, 0x12, 0x61,
+0x7D, 0x67, 0x77, 0xA3, 0x0F, 0x23, 0x03, 0x6D,
+0x64, 0x67, 0xAC, 0xEB, 0xFF, 0x6E, 0xCC, 0xEB,
+0x42, 0xEB, 0x08, 0x60, 0x04, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEC, 0xAC, 0xEA, 0x8D, 0xEA, 0x7D, 0x67,
+0x54, 0xC3, 0x05, 0x69, 0x5D, 0x67, 0x54, 0xA2,
+0x0C, 0x6B, 0x1B, 0x65, 0x78, 0x67, 0x4C, 0xEB,
+0xFF, 0x6C, 0x43, 0x67, 0x8C, 0xEA, 0x0C, 0x72,
+0x0F, 0x61, 0x5D, 0x67, 0x54, 0xA2, 0x60, 0xF0,
+0xA6, 0xA0, 0x07, 0x6B, 0x52, 0x32, 0x6C, 0xEA,
+0xB6, 0x33, 0x6E, 0xEA, 0x8C, 0xEA, 0x04, 0x22,
+0x10, 0x6A, 0x4C, 0xED, 0x01, 0x25, 0x06, 0x69,
+0x60, 0xF0, 0x43, 0x80, 0x00, 0x52, 0x15, 0x60,
+0x37, 0x29, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0x08, 0x51, 0x05, 0x61,
+0x68, 0x34, 0x8D, 0xE3, 0x64, 0x33, 0x33, 0x4B,
+0x1A, 0x17, 0x6C, 0x33, 0x11, 0x4B, 0x17, 0x17,
+0x1F, 0x6B, 0x8C, 0xEB, 0x60, 0xF0, 0x7E, 0xC0,
+0x2B, 0x17, 0x08, 0x94, 0x04, 0x06, 0xD1, 0x18,
+0x1B, 0x6B, 0x04, 0x6D, 0x5D, 0x67, 0x51, 0xA2,
+0x03, 0x6B, 0x90, 0x67, 0x5E, 0x32, 0x07, 0xD2,
+0x5D, 0x67, 0xB4, 0xA2, 0xD6, 0xA2, 0x00, 0x6F,
+0xAA, 0x35, 0x20, 0xF0, 0x60, 0xC2, 0xD1, 0x18,
+0x54, 0x07, 0x6C, 0xED, 0x07, 0x94, 0x7D, 0x67,
+0x20, 0xF0, 0x60, 0xA3, 0x18, 0x2C, 0xD0, 0x22,
+0x60, 0xF0, 0x54, 0xA0, 0x4C, 0xEB, 0x07, 0xD3,
+0x07, 0x02, 0x60, 0xA2, 0x03, 0x73, 0xC8, 0x60,
+0x60, 0xF0, 0x59, 0xA0, 0x01, 0x6B, 0x6C, 0xEA,
+0xC4, 0x2A, 0x06, 0x94, 0xF0, 0x67, 0x00, 0x6E,
+0xD1, 0x18, 0x7B, 0x09, 0x05, 0x05, 0xD1, 0x18,
+0x05, 0x07, 0x90, 0x67, 0xBA, 0x17, 0x60, 0xF0,
+0x54, 0xA0, 0x4C, 0xEB, 0x07, 0xD3, 0x07, 0x02,
+0x60, 0xA2, 0x03, 0x73, 0xB1, 0x61, 0xE8, 0x17,
+0x00, 0x6A, 0x58, 0xC4, 0x59, 0xC4, 0x5C, 0xC4,
+0x5D, 0xC4, 0x5E, 0xC4, 0x5F, 0xC4, 0x5A, 0xC4,
+0x5B, 0xC4, 0x20, 0xF0, 0x43, 0xC4, 0x20, 0xF0,
+0x44, 0xC4, 0x20, 0xF0, 0x4A, 0xC4, 0x20, 0xF0,
+0x4B, 0xC4, 0x20, 0xF0, 0x40, 0xC4, 0x20, 0xF0,
+0x45, 0xC4, 0x20, 0xF0, 0x46, 0xC4, 0x20, 0xF0,
+0x47, 0xC4, 0x20, 0xF0, 0x48, 0xC4, 0x20, 0xF0,
+0x49, 0xC4, 0x20, 0xF0, 0x4D, 0xC4, 0x20, 0xF0,
+0x4E, 0xC4, 0x20, 0xF0, 0x4F, 0xC4, 0x20, 0xF0,
+0x50, 0xC4, 0x20, 0xF0, 0x57, 0xC4, 0x20, 0xF0,
+0x58, 0xC4, 0x20, 0xF0, 0x51, 0xC4, 0x20, 0xF0,
+0x52, 0xC4, 0x20, 0xF0, 0x59, 0xC4, 0x20, 0xF0,
+0x5A, 0xC4, 0x20, 0xF0, 0x53, 0xC4, 0x20, 0xF0,
+0x54, 0xC4, 0x20, 0xF0, 0x5B, 0xC4, 0x20, 0xF0,
+0x5C, 0xC4, 0x20, 0xF0, 0x55, 0xC4, 0x20, 0xF0,
+0x56, 0xC4, 0x20, 0xF0, 0x5D, 0xC4, 0x20, 0xF0,
+0x5E, 0xC4, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x08, 0x5C, 0x04, 0x67, 0x0A, 0x61,
+0xF8, 0x48, 0xFF, 0x6A, 0x4C, 0xE8, 0x08, 0x58,
+0x60, 0x6C, 0x05, 0x61, 0x05, 0x97, 0x04, 0x90,
+0x00, 0xEF, 0x03, 0x63, 0x44, 0x6C, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x47, 0xE8,
+0x01, 0x6B, 0x6C, 0xEA, 0xEF, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x08, 0x5C, 0x24, 0x67, 0x0C, 0x61, 0xFF, 0x6A,
+0xF8, 0x49, 0x4C, 0xE9, 0x08, 0x59, 0x02, 0x6A,
+0x62, 0x68, 0x06, 0x61, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x46, 0x68,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x50, 0x9A, 0x90, 0x67, 0x40, 0xEA,
+0x04, 0xD5, 0x04, 0x95, 0x01, 0x6C, 0xC4, 0x67,
+0xAB, 0xEB, 0xA7, 0x41, 0x01, 0x4D, 0xC4, 0xED,
+0xA6, 0x67, 0x84, 0xE9, 0x8D, 0xED, 0xC0, 0xF7,
+0x62, 0x33, 0x64, 0xE9, 0xAF, 0xED, 0xAC, 0xEA,
+0x60, 0x35, 0xAD, 0xEB, 0x6D, 0xEA, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x40, 0x32,
+0x60, 0x33, 0x43, 0x32, 0xD2, 0xF4, 0x64, 0x9B,
+0x43, 0x32, 0xFF, 0xF7, 0x1F, 0x6D, 0x4C, 0xED,
+0x40, 0xEB, 0x90, 0x67, 0x00, 0x6A, 0xCE, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x08, 0x5C, 0x04, 0x67, 0x0C, 0x61, 0xFF, 0x6A,
+0xF8, 0x48, 0x4C, 0xE8, 0x08, 0x58, 0x02, 0x6A,
+0x61, 0x69, 0x06, 0x61, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x45, 0x69,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x4C, 0x9A, 0x91, 0x67, 0x40, 0xEA,
+0x04, 0xD5, 0x04, 0x95, 0x01, 0x6B, 0x64, 0xE8,
+0x6F, 0xEB, 0xAB, 0xED, 0x6C, 0xEA, 0xC0, 0xF7,
+0xA2, 0x35, 0xF7, 0xF0, 0x01, 0x6B, 0xA4, 0xE8,
+0x60, 0x33, 0xAD, 0xEA, 0x60, 0x33, 0x00, 0xF6,
+0x40, 0x32, 0xD2, 0xF4, 0x60, 0x9B, 0x00, 0xF6,
+0x43, 0x32, 0xFF, 0x6D, 0x4C, 0xED, 0x40, 0xEB,
+0x91, 0x67, 0x00, 0x6A, 0xD7, 0x17, 0x00, 0x65,
+0x08, 0x5C, 0x1B, 0x61, 0x10, 0x5C, 0x1C, 0x61,
+0x13, 0x5C, 0xFF, 0x6A, 0x1C, 0x60, 0xC0, 0xF2,
+0x18, 0x6A, 0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x74, 0x9B, 0x04, 0x67, 0x40, 0xEB,
+0x82, 0x67, 0x08, 0x30, 0x46, 0xE8, 0x05, 0x97,
+0x04, 0x90, 0x0F, 0x6B, 0x6C, 0xEA, 0x00, 0xEF,
+0x03, 0x63, 0xC0, 0xF2, 0x10, 0x6A, 0xE9, 0x17,
+0xC0, 0xF2, 0x14, 0x6A, 0xE6, 0x17, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x08, 0x5C, 0x2B, 0x61,
+0x10, 0x5C, 0x2C, 0x61, 0x13, 0x5C, 0x02, 0x6A,
+0x21, 0x60, 0xC0, 0xF2, 0x18, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x07, 0x68, 0x8C, 0xE8, 0x83, 0x67,
+0x25, 0x67, 0x08, 0x30, 0x40, 0xEA, 0x04, 0xD3,
+0x0F, 0x6D, 0xA4, 0xE8, 0xF7, 0xF0, 0x01, 0x6C,
+0xAF, 0xED, 0x80, 0x34, 0xAC, 0xEA, 0x80, 0x34,
+0x04, 0x93, 0x0F, 0x6D, 0xD2, 0xF4, 0xC8, 0x9C,
+0x2C, 0xED, 0xA4, 0xE8, 0x4D, 0xED, 0x40, 0xEE,
+0x83, 0x67, 0x00, 0x6A, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0xC0, 0xF2,
+0x10, 0x6B, 0xD9, 0x17, 0xC0, 0xF2, 0x14, 0x6B,
+0xD6, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x4C, 0x9A, 0x59, 0xF6, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x05, 0x97, 0x01, 0x6B,
+0x6C, 0xEA, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0xD2, 0xF4, 0x4C, 0x99, 0x59, 0xF6, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x02, 0x67, 0xD2, 0xF4,
+0x4C, 0x99, 0x32, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x0D, 0xEA, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x01, 0x6B, 0x6C, 0xEA, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A, 0x04, 0x67,
+0x59, 0xF6, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x60, 0x9B, 0xFE, 0x6D, 0x03, 0x20,
+0x01, 0x6D, 0xAD, 0xEA, 0xFF, 0x6D, 0x59, 0xF6,
+0x00, 0x6C, 0x40, 0xEB, 0x4C, 0xED, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0x08, 0x2C, 0xFD, 0x63, 0x05, 0x62, 0xD1, 0x18,
+0x17, 0x18, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x00, 0x6A, 0x00, 0x65,
+0x08, 0x2C, 0xFD, 0x63, 0x05, 0x62, 0xD1, 0x18,
+0x20, 0x18, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x20, 0xE8, 0x00, 0x6A, 0x00, 0x65,
+0x09, 0x2D, 0xFD, 0x63, 0x05, 0x62, 0xD1, 0x18,
+0x2F, 0x18, 0x00, 0x65, 0x05, 0x97, 0x01, 0x6A,
+0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8, 0x01, 0x6A,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x54, 0x9B, 0x24, 0x67, 0x10, 0xF4,
+0x00, 0x6C, 0x05, 0x67, 0x40, 0xEA, 0x05, 0xD3,
+0x05, 0x93, 0x04, 0xD2, 0x10, 0xF4, 0x04, 0x6C,
+0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA, 0x00, 0x65,
+0x05, 0xD2, 0x62, 0x21, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x49, 0xF1, 0x68, 0x9A,
+0x04, 0x94, 0x00, 0x6A, 0x6C, 0xEC, 0x03, 0x24,
+0x05, 0x94, 0x8C, 0xEB, 0x70, 0x2B, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x49, 0xF1, 0x68, 0x9A, 0x04, 0x94, 0x01, 0x6A,
+0x6C, 0xEC, 0xF1, 0x24, 0x05, 0x94, 0x6C, 0xEC,
+0xEE, 0x24, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0xD2, 0xF4, 0x40, 0x99, 0x10, 0x6D,
+0xFF, 0x6B, 0x0D, 0xED, 0x32, 0xF6, 0x00, 0x6C,
+0x40, 0xEA, 0x6C, 0xED, 0x0A, 0x6E, 0xFF, 0x4E,
+0x10, 0x2E, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x09, 0xF3,
+0xB0, 0x9B, 0x40, 0xEA, 0x01, 0x6C, 0x00, 0x6A,
+0xCE, 0x17, 0x06, 0x92, 0x32, 0xF6, 0x00, 0x6C,
+0x04, 0xD6, 0xD2, 0xF4, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x20, 0x6B, 0x6C, 0xEA, 0xFF, 0x6D,
+0xAC, 0xEA, 0x0C, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0xE0, 0xF3, 0x08, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x04, 0x96, 0xD5, 0x17, 0x12, 0x6A, 0x4B, 0xEA,
+0xD2, 0xF4, 0x60, 0x99, 0x4C, 0xE8, 0x32, 0xF6,
+0x00, 0x6C, 0x40, 0xEB, 0x0C, 0xED, 0x4D, 0x10,
+0x32, 0x28, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A, 0x59, 0xF6,
+0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4,
+0x60, 0x9B, 0xFE, 0x6D, 0x1C, 0x10, 0xD2, 0xF4,
+0x4C, 0x9A, 0x32, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x02, 0x67, 0x25, 0x10, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0x28,
+0xD2, 0xF4, 0x4C, 0x9A, 0x59, 0xF6, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x60, 0x9B,
+0x01, 0x6D, 0xAD, 0xEA, 0xFF, 0x6D, 0x59, 0xF6,
+0x04, 0x6C, 0x40, 0xEB, 0x4C, 0xED, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4,
+0x4C, 0x9B, 0x32, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x06, 0xD3, 0x02, 0x67, 0x7F, 0xF7, 0x10, 0x21,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x01, 0x6D, 0xD2, 0xF4, 0x40, 0x9A, 0xAD, 0xE8,
+0xFF, 0x6D, 0x32, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x0C, 0xED, 0x01, 0x6A, 0x5C, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xD1, 0x18,
+0x17, 0x18, 0x00, 0x65, 0x05, 0x22, 0x02, 0x67,
+0xD1, 0x18, 0x2F, 0x18, 0x00, 0x6C, 0x50, 0x67,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0x01, 0x6A, 0x4C, 0xEC, 0x09, 0x24, 0xFD, 0x63,
+0x05, 0x62, 0xD1, 0x18, 0x2F, 0x18, 0x01, 0x6C,
+0x05, 0x97, 0x00, 0x6A, 0x00, 0xEF, 0x03, 0x63,
+0x20, 0xE8, 0x00, 0x6A, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xD1, 0x18, 0x20, 0x18, 0x00, 0x65,
+0x02, 0x67, 0x04, 0x22, 0x00, 0x6D, 0xD1, 0x18,
+0x52, 0x18, 0x00, 0x6C, 0x50, 0x67, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x01, 0x6A,
+0x8C, 0xEA, 0x04, 0x67, 0x04, 0x22, 0x00, 0x6D,
+0xD1, 0x18, 0x52, 0x18, 0x01, 0x6C, 0x02, 0x6A,
+0x4C, 0xE8, 0x04, 0x20, 0x01, 0x6D, 0xD1, 0x18,
+0x52, 0x18, 0x01, 0x6C, 0x05, 0x97, 0x04, 0x90,
+0x00, 0x6A, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x6A, 0x81, 0xA4, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0x40, 0x32, 0xD3, 0xF6,
+0x70, 0x9B, 0x89, 0xF1, 0xB4, 0x9A, 0x06, 0x2C,
+0x40, 0xEB, 0x00, 0x65, 0x05, 0x97, 0x03, 0x6A,
+0x00, 0xEF, 0x03, 0x63, 0x00, 0x6C, 0xF8, 0x17,
+0x63, 0xA4, 0x20, 0x5B, 0x4A, 0x60, 0xFF, 0x63,
+0x01, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A, 0xF1, 0xA2,
+0xD0, 0xA2, 0x12, 0xA2, 0xE0, 0x37, 0xCD, 0xEF,
+0xD3, 0xA2, 0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF,
+0x00, 0xF6, 0xC0, 0x36, 0xED, 0xEE, 0xE1, 0xA6,
+0x40, 0xA6, 0x02, 0xA6, 0xE0, 0x37, 0x4D, 0xEF,
+0x43, 0xA6, 0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF,
+0x00, 0xF6, 0x40, 0x32, 0xED, 0xEA, 0x68, 0x33,
+0x49, 0xE3, 0xC1, 0xA2, 0x60, 0xA2, 0xE2, 0xA2,
+0xC0, 0x36, 0x6D, 0xEE, 0x63, 0xA2, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x60, 0x33,
+0xCD, 0xEB, 0x19, 0x23, 0x49, 0xA4, 0x08, 0x6C,
+0xFF, 0x6E, 0x8C, 0xEA, 0xCC, 0xEA, 0x13, 0x22,
+0x60, 0xF0, 0x93, 0xA3, 0x60, 0xF0, 0x52, 0xA3,
+0xFF, 0xF7, 0x1F, 0x6F, 0x80, 0x34, 0x4D, 0xEC,
+0x0E, 0x2C, 0xB0, 0x32, 0xEC, 0xEA, 0x4C, 0xEE,
+0x42, 0x32, 0x60, 0xF0, 0xD2, 0xC3, 0x60, 0xF0,
+0x53, 0xC3, 0x60, 0xF0, 0xB1, 0xC3, 0x01, 0x90,
+0x01, 0x63, 0x20, 0xE8, 0x00, 0x65, 0xA8, 0x32,
+0x49, 0xE4, 0x8A, 0x34, 0x97, 0xE2, 0xEC, 0xED,
+0xAC, 0xEE, 0xA2, 0x32, 0x08, 0x4D, 0x60, 0xF0,
+0xD2, 0xC3, 0x60, 0xF0, 0x53, 0xC3, 0xB3, 0x35,
+0xEC, 0x17, 0x00, 0x65, 0x60, 0xA4, 0x1F, 0x6A,
+0xFF, 0x6F, 0x4C, 0xEB, 0xEC, 0xEB, 0x60, 0xDE,
+0x41, 0xA4, 0x40, 0xCD, 0x40, 0xA4, 0x5E, 0x32,
+0x20, 0xE8, 0xEC, 0xEA, 0x41, 0xA4, 0x40, 0xC5,
+0x20, 0xE8, 0x01, 0x6A, 0x42, 0xA6, 0x61, 0xA6,
+0x40, 0x32, 0x6D, 0xEA, 0x04, 0x5A, 0x02, 0x61,
+0x41, 0xA4, 0x40, 0xC5, 0x20, 0xE8, 0x01, 0x6A,
+0xF3, 0x63, 0x19, 0x62, 0x18, 0xD1, 0x17, 0xD0,
+0x0F, 0xD5, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0xA0, 0x35, 0x06, 0x67, 0x24, 0x67, 0x20, 0x6E,
+0x04, 0x04, 0x08, 0xF3, 0x0C, 0x4D, 0xD1, 0x18,
+0xE7, 0xF4, 0x0E, 0xD7, 0x00, 0x6A, 0x7D, 0x67,
+0x20, 0xF0, 0x50, 0xC3, 0x40, 0xA0, 0x01, 0x6C,
+0x0E, 0x97, 0x8C, 0xEA, 0x0F, 0x93, 0x02, 0x22,
+0x48, 0xA0, 0x43, 0xC0, 0xC0, 0xA1, 0xDE, 0x32,
+0x07, 0x2A, 0x00, 0x69, 0x51, 0x67, 0x19, 0x97,
+0x18, 0x91, 0x17, 0x90, 0x00, 0xEF, 0x0D, 0x63,
+0x1F, 0x6A, 0x4C, 0xEE, 0xFF, 0x6A, 0x4C, 0xEE,
+0x41, 0xA1, 0x0E, 0xD6, 0x4C, 0x32, 0x6E, 0xEA,
+0x27, 0x22, 0x00, 0x69, 0x5D, 0x67, 0x20, 0xF0,
+0xB0, 0xA2, 0x20, 0xF0, 0x58, 0xA2, 0xFF, 0x6C,
+0x20, 0xC7, 0x42, 0xC7, 0x0E, 0x92, 0xA1, 0xC7,
+0x42, 0x32, 0x8C, 0xEA, 0x43, 0xC7, 0x00, 0x6A,
+0x44, 0xC7, 0x0E, 0x92, 0x00, 0xF6, 0x42, 0x36,
+0xC5, 0xC7, 0xDB, 0x21, 0x60, 0xA0, 0x08, 0x6E,
+0x43, 0x67, 0xCC, 0xEA, 0xD7, 0x22, 0x46, 0xA7,
+0xD5, 0x22, 0x01, 0x6A, 0x4C, 0xEB, 0xD2, 0x23,
+0x49, 0xA0, 0xCC, 0xEA, 0x8C, 0xEA, 0xCE, 0x22,
+0xD1, 0x18, 0xE8, 0x18, 0x90, 0x67, 0xCA, 0x17,
+0x0E, 0x92, 0x06, 0x5A, 0x02, 0x61, 0x08, 0x72,
+0x26, 0x61, 0x40, 0xA0, 0x09, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEA, 0x40, 0xC0, 0x00, 0x6A, 0x46, 0xC7,
+0xA0, 0xA0, 0x03, 0x6A, 0xA6, 0x35, 0x4C, 0xED,
+0x0F, 0xD5, 0x00, 0x6A, 0x0F, 0x94, 0x82, 0xEA,
+0x06, 0x61, 0x40, 0xA0, 0x08, 0x6C, 0x8C, 0xEA,
+0x15, 0x2A, 0x01, 0x69, 0xBF, 0x17, 0x84, 0x42,
+0x84, 0x34, 0x91, 0xE0, 0xC1, 0xA4, 0x01, 0x4A,
+0x1E, 0x65, 0x10, 0x6E, 0xCB, 0xEE, 0x3E, 0x65,
+0xB9, 0x67, 0xD8, 0x67, 0xAC, 0xEE, 0xC1, 0xC4,
+0xFF, 0x6C, 0x8C, 0xEA, 0xE7, 0x17, 0x01, 0x6A,
+0x46, 0xC7, 0xE7, 0x17, 0xF8, 0x4B, 0xFF, 0xF7,
+0x1F, 0x6A, 0x4C, 0xEB, 0x11, 0xD3, 0xE5, 0x23,
+0x20, 0x6A, 0x13, 0xD2, 0x00, 0x6A, 0x08, 0x49,
+0x0F, 0xD2, 0x12, 0xD2, 0x80, 0xA1, 0x1F, 0x6A,
+0x01, 0x6B, 0x8C, 0xEA, 0x64, 0xEA, 0x14, 0xD3,
+0x14, 0x95, 0x12, 0x93, 0xAC, 0xEB, 0x95, 0x2B,
+0x7D, 0x67, 0x4D, 0xE3, 0x70, 0xA3, 0xFF, 0x73,
+0x26, 0x60, 0x6C, 0x33, 0x10, 0xD3, 0x2C, 0x2A,
+0x0C, 0x05, 0x91, 0x67, 0xD1, 0x18, 0x1D, 0x19,
+0x15, 0xD7, 0x15, 0x97, 0x10, 0x92, 0x10, 0x93,
+0x45, 0xE1, 0x0F, 0x92, 0x49, 0xE3, 0x3A, 0x65,
+0x79, 0x67, 0xFF, 0xF7, 0x1F, 0x6A, 0x4C, 0xEB,
+0x0F, 0xD3, 0x12, 0x92, 0x14, 0x93, 0x6D, 0xEA,
+0x12, 0xD2, 0x0F, 0x93, 0x11, 0x92, 0x6A, 0xEA,
+0xB4, 0x60, 0x63, 0xEA, 0x7F, 0xF7, 0x11, 0x61,
+0x13, 0x93, 0xFF, 0x6A, 0xFF, 0x4B, 0x4C, 0xEB,
+0x13, 0xD3, 0xCC, 0x2B, 0xAA, 0x17, 0x61, 0xA1,
+0x78, 0x6D, 0x96, 0x34, 0x6C, 0x33, 0xAC, 0xEB,
+0xFF, 0x6D, 0xAC, 0xEC, 0x8D, 0xEB, 0xD1, 0x17,
+0x01, 0x72, 0x9F, 0x61, 0x42, 0xA0, 0x81, 0xA0,
+0x40, 0x32, 0x8D, 0xEA, 0x04, 0x5A, 0xD2, 0x61,
+0x61, 0xA1, 0x5D, 0x67, 0x20, 0xF0, 0x70, 0xC2,
+0xCD, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x18, 0x6A, 0x44, 0xC4, 0x00, 0x6A,
+0x45, 0xC4, 0x46, 0xC4, 0x47, 0xC4, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x04, 0x67, 0xC4, 0x67, 0x00, 0x6F,
+0x18, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x0F, 0x2A,
+0xA1, 0xA0, 0x60, 0xA0, 0x82, 0xA0, 0xA0, 0x35,
+0x6D, 0xED, 0x63, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x03, 0x2B, 0x02, 0x6A, 0x01, 0x10, 0x04, 0x6A,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0xE4, 0x67, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x15, 0xF0, 0x08, 0x4C, 0xC5, 0x67,
+0xA9, 0xA4, 0x48, 0xA4, 0x6A, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x4B, 0xA4, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x69, 0xA2, 0x08, 0xA2, 0xAA, 0xA2, 0x60, 0x33,
+0x0D, 0xEB, 0x0B, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEB, 0x00, 0xF6, 0x00, 0x30, 0xA5, 0xA4,
+0x6D, 0xE8, 0x64, 0xA4, 0x46, 0xA4, 0xA0, 0x35,
+0x6D, 0xED, 0x67, 0xA4, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x8D, 0xA3, 0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x8D, 0xA2, 0xAE, 0xA2, 0x6C, 0xA2, 0x2F, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0xA0, 0x35, 0x6D, 0xEC,
+0xAD, 0xEC, 0x00, 0xF6, 0x20, 0x31, 0x8D, 0xE9,
+0x83, 0xA1, 0xA2, 0xA1, 0x44, 0xA1, 0x80, 0x34,
+0x8D, 0xED, 0x85, 0xA1, 0x40, 0x32, 0x40, 0x32,
+0xAD, 0xEA, 0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC,
+0x11, 0xD4, 0x16, 0x28, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6, 0x50, 0x9A,
+0x09, 0xF3, 0xB4, 0x9B, 0x40, 0xEA, 0x00, 0x6C,
+0x01, 0x6A, 0x10, 0xD2, 0x10, 0x92, 0x17, 0x97,
+0x16, 0x91, 0x15, 0x90, 0x00, 0xEF, 0x0C, 0x63,
+0x16, 0x2F, 0x06, 0x6A, 0x7D, 0x67, 0x58, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4A, 0xF1, 0x18, 0x4A, 0x07, 0xD2, 0xFF, 0x6A,
+0x66, 0x4A, 0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x06, 0x04, 0xE1, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x13, 0xD6, 0x40, 0xEA, 0x12, 0xD7,
+0x80, 0xA1, 0x01, 0x4C, 0x80, 0xC1, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x72, 0xF0,
+0xB4, 0x9C, 0x40, 0xED, 0x82, 0x67, 0xD1, 0x18,
+0x93, 0x19, 0x06, 0x04, 0x10, 0xD2, 0x12, 0x97,
+0x10, 0x93, 0x13, 0x96, 0xA3, 0xA7, 0x44, 0xA7,
+0x17, 0x23, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xD3, 0xF5, 0x60, 0x9B,
+0x4D, 0xED, 0x00, 0x6F, 0xF8, 0x4D, 0x40, 0xEB,
+0x00, 0x6C, 0xB8, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A,
+0x20, 0x6D, 0x40, 0xEA, 0x02, 0x6C, 0xAE, 0x17,
+0x40, 0x32, 0xAD, 0xEA, 0xF8, 0x4A, 0x0B, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0A, 0xD6, 0x06, 0x94, 0x07, 0x96, 0x93, 0xF6,
+0x40, 0x9A, 0x00, 0x6D, 0x40, 0xEA, 0x12, 0xD7,
+0x12, 0x97, 0xE7, 0xF7, 0x1F, 0x6C, 0x06, 0x95,
+0x44, 0xA7, 0xC3, 0xA7, 0x40, 0x32, 0xCD, 0xEA,
+0x8C, 0xEA, 0xFF, 0x6E, 0x4C, 0xEE, 0xFF, 0xF7,
+0x1F, 0x6C, 0x42, 0x32, 0x8C, 0xEA, 0x0A, 0x6C,
+0xC0, 0xC5, 0x41, 0xC5, 0x83, 0xC5, 0x60, 0xA7,
+0x71, 0xC5, 0x81, 0xA7, 0x88, 0x33, 0x1B, 0x65,
+0x70, 0xA5, 0x3B, 0x65, 0x03, 0x6B, 0xFB, 0x65,
+0x7F, 0x67, 0x99, 0x67, 0x6C, 0xEC, 0x78, 0x67,
+0x8D, 0xEB, 0x70, 0xC5, 0xE2, 0xA7, 0x83, 0x67,
+0x7F, 0x67, 0x6C, 0xEF, 0x04, 0x6B, 0x6B, 0xEB,
+0x6C, 0xEC, 0xED, 0xEC, 0x90, 0xC5, 0x95, 0xA5,
+0xD4, 0xC5, 0x40, 0x6E, 0xCB, 0xEE, 0xCC, 0xEC,
+0x8D, 0xEA, 0x92, 0xA5, 0x55, 0xC5, 0x10, 0x6A,
+0x4B, 0xEA, 0x4C, 0xEC, 0x01, 0x6E, 0xCD, 0xEC,
+0x92, 0xC5, 0x40, 0xA1, 0x53, 0xC5, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x94, 0xF2,
+0x6C, 0x9A, 0x22, 0x67, 0x2C, 0x23, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF3,
+0x48, 0x9A, 0x11, 0x94, 0x00, 0x6F, 0x00, 0x6E,
+0x40, 0xEA, 0x06, 0x05, 0x56, 0x22, 0x00, 0x6A,
+0x0E, 0xD2, 0x94, 0xF2, 0x4C, 0x99, 0x2D, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF2, 0xF1, 0x4C, 0x9A, 0x95, 0xA0, 0x0E, 0x03,
+0x1A, 0x65, 0x54, 0xA0, 0x80, 0x34, 0x00, 0x6F,
+0x4D, 0xEC, 0x56, 0xA0, 0x01, 0x6E, 0x01, 0x6D,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x97, 0xA0,
+0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC, 0x58, 0x67,
+0x40, 0xEA, 0x04, 0xD3, 0x2F, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x52, 0xF0,
+0x58, 0x9A, 0x40, 0xEA, 0x12, 0xD6, 0x12, 0x96,
+0xCC, 0xEA, 0xC9, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF3, 0x4C, 0x9A,
+0xC8, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x01, 0x6B, 0x6C, 0xEA, 0xC8, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF2, 0xF1, 0x50, 0x9A, 0x95, 0xA0, 0x00, 0x6F,
+0x1A, 0x65, 0x54, 0xA0, 0x80, 0x34, 0x01, 0x6E,
+0x4D, 0xEC, 0x56, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x97, 0xA0, 0x00, 0xF6, 0x80, 0x34,
+0x4D, 0xEC, 0x58, 0x67, 0x40, 0xEA, 0x01, 0x6D,
+0xF9, 0x16, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF7, 0xF0, 0x01, 0x6C, 0x72, 0xF0,
+0x4C, 0x9A, 0x80, 0x34, 0x80, 0x34, 0xA0, 0xF1,
+0x06, 0x6D, 0x4A, 0xF1, 0x18, 0x4C, 0x40, 0xEA,
+0x00, 0x65, 0xE8, 0x16, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x00, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0x08, 0xD4, 0x25, 0x67,
+0x00, 0x6F, 0x06, 0x06, 0x0A, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0x04, 0x68, 0x08, 0x93, 0x1C, 0x2A,
+0x06, 0x94, 0x08, 0xD2, 0x02, 0x68, 0x18, 0x24,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0xA3, 0x67, 0x40, 0xEA,
+0x0A, 0x6E, 0x08, 0x97, 0x5D, 0x67, 0x06, 0x95,
+0x11, 0xC2, 0xF2, 0xC2, 0x30, 0xC2, 0x7D, 0x67,
+0x12, 0x6A, 0x53, 0xC3, 0x04, 0x04, 0x5D, 0x67,
+0xD1, 0x18, 0xA8, 0x19, 0xF4, 0xC2, 0x02, 0x67,
+0x50, 0x67, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x01, 0x6B, 0x24, 0x67,
+0x9D, 0x67, 0x00, 0x6A, 0x72, 0xC4, 0x7D, 0x67,
+0x51, 0xC3, 0x06, 0xD2, 0x02, 0x6A, 0x50, 0xC3,
+0x47, 0x41, 0xFF, 0xF7, 0x1F, 0x6B, 0x01, 0x4A,
+0x6C, 0xEA, 0x7D, 0x67, 0x53, 0xC3, 0x42, 0x32,
+0x54, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A, 0x05, 0x67,
+0x00, 0x6F, 0x06, 0x06, 0xB1, 0x67, 0x40, 0xEA,
+0x00, 0x6C, 0x04, 0x6B, 0x11, 0x2A, 0x06, 0x94,
+0x02, 0x6B, 0x0E, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xD1, 0x67, 0x40, 0xEA, 0xB0, 0x67, 0x06, 0x95,
+0xD1, 0x18, 0xA8, 0x19, 0x04, 0x04, 0x62, 0x67,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x43, 0x67,
+0x00, 0xEF, 0x06, 0x63, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x01, 0x6B, 0x24, 0x67,
+0x9D, 0x67, 0x72, 0xC4, 0x71, 0xC4, 0x05, 0x6B,
+0x70, 0xC4, 0x0C, 0x6B, 0x00, 0x6A, 0x73, 0xC4,
+0x7D, 0x67, 0x54, 0xC3, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x05, 0x67, 0x00, 0x6F, 0x06, 0x06,
+0x04, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x04, 0x6B,
+0x0F, 0x2A, 0x06, 0x95, 0x02, 0x6B, 0x0C, 0x25,
+0x0F, 0x6A, 0x4C, 0xE8, 0x41, 0xA5, 0xEE, 0x4B,
+0x20, 0xC5, 0x6C, 0xEA, 0x0D, 0xEA, 0x41, 0xC5,
+0xD1, 0x18, 0xA8, 0x19, 0x04, 0x04, 0x62, 0x67,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x43, 0x67,
+0x00, 0xEF, 0x06, 0x63, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x00, 0x6A, 0x00, 0xF2,
+0x00, 0x5C, 0x65, 0x67, 0x06, 0xD2, 0x04, 0x67,
+0x02, 0x61, 0xE0, 0xF1, 0x1F, 0x68, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x00, 0x6F, 0x06, 0x06, 0xA1, 0x40,
+0x00, 0x6C, 0x40, 0xEA, 0x08, 0xD3, 0x22, 0x67,
+0x08, 0x93, 0x04, 0x6A, 0x25, 0x29, 0x06, 0x94,
+0x02, 0x6A, 0x22, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xA3, 0x67, 0x40, 0xEA, 0xD0, 0x67, 0x06, 0x92,
+0x09, 0xE2, 0x60, 0xA2, 0x01, 0x23, 0x20, 0xC2,
+0x7D, 0x67, 0x01, 0x6A, 0x52, 0xC3, 0x00, 0x6A,
+0x51, 0xC3, 0x02, 0x6A, 0x50, 0xC3, 0x09, 0x48,
+0xFF, 0xF7, 0x1F, 0x6A, 0x06, 0x95, 0x4C, 0xE8,
+0x5D, 0x67, 0x13, 0xC2, 0x04, 0x04, 0x02, 0x30,
+0xD1, 0x18, 0xA8, 0x19, 0x14, 0xC2, 0x00, 0x6A,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF,
+0x07, 0x63, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x00, 0x6A, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x06, 0x67, 0xD3, 0xF5, 0x48, 0x9A, 0x25, 0x67,
+0xA7, 0x40, 0x0A, 0xD4, 0x13, 0xD7, 0x08, 0x06,
+0x00, 0x6F, 0x01, 0x4D, 0x40, 0xEA, 0x00, 0x6C,
+0x0A, 0x93, 0x24, 0x22, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x4A, 0xF1, 0x18, 0x4A, 0x05, 0xD2,
+0xAC, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A,
+0x10, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x01, 0x6A,
+0x4B, 0xEA, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0x08, 0x92, 0x0C, 0x2A,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4A, 0xF1,
+0x18, 0x4A, 0x05, 0xD2, 0xB2, 0x6A, 0xD9, 0x17,
+0x22, 0x34, 0x81, 0xC2, 0x82, 0x34, 0x82, 0xC2,
+0x02, 0x34, 0x85, 0xC2, 0x82, 0x34, 0x86, 0xC2,
+0x00, 0xF6, 0x02, 0x34, 0x87, 0xC2, 0x87, 0x42,
+0x01, 0x4C, 0x82, 0x35, 0xA9, 0xC2, 0xA2, 0x35,
+0x20, 0xC2, 0xAA, 0xC2, 0x00, 0xF6, 0x22, 0x31,
+0x00, 0xF6, 0x82, 0x35, 0x23, 0xC2, 0x04, 0xC2,
+0x88, 0xC2, 0xAB, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xD0, 0x67, 0x40, 0xEA, 0xA3, 0x67, 0x7D, 0x67,
+0x01, 0x6A, 0x52, 0xC3, 0x06, 0x6A, 0x51, 0xC3,
+0x02, 0x03, 0x40, 0xF0, 0x64, 0xA3, 0x5D, 0x67,
+0x10, 0x48, 0x70, 0xC2, 0x08, 0x95, 0xFF, 0xF7,
+0x1F, 0x6A, 0x4C, 0xE8, 0x5D, 0x67, 0x13, 0xC2,
+0x04, 0x04, 0x02, 0x30, 0xD1, 0x18, 0xA8, 0x19,
+0x14, 0xC2, 0xB3, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x00, 0x6A, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0x0A, 0xD4, 0x00, 0x6F,
+0x25, 0x67, 0x06, 0x67, 0x08, 0x6D, 0x08, 0x06,
+0x40, 0xEA, 0x00, 0x6C, 0xE2, 0x67, 0x0A, 0x93,
+0x04, 0x6A, 0x1F, 0x2F, 0x08, 0x94, 0x0A, 0xD7,
+0x02, 0x6A, 0x1B, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0x04, 0xD3, 0x04, 0x05, 0x05, 0xD1, 0x40, 0xEA,
+0x08, 0x6E, 0x7D, 0x67, 0x01, 0x6A, 0x5A, 0xC3,
+0x06, 0x6A, 0x59, 0xC3, 0x0A, 0x97, 0x5D, 0x67,
+0x08, 0x95, 0x18, 0xC2, 0x10, 0x6A, 0x5B, 0xC3,
+0x06, 0x04, 0x5D, 0x67, 0xD1, 0x18, 0xA8, 0x19,
+0xFC, 0xC2, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x00, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0x26, 0x67, 0x00, 0x6F,
+0x06, 0x06, 0x00, 0x6C, 0x40, 0xEA, 0x05, 0x67,
+0x62, 0x67, 0x04, 0x6A, 0x21, 0x2B, 0x06, 0x94,
+0x08, 0xD3, 0x02, 0x6A, 0x1D, 0x24, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xD0, 0x67, 0x40, 0xEA, 0xB1, 0x67,
+0x08, 0x93, 0x5D, 0x67, 0x9D, 0x67, 0x72, 0xC2,
+0x04, 0x6A, 0x51, 0xC4, 0x5D, 0x67, 0x70, 0xC2,
+0x08, 0x48, 0xFF, 0xF7, 0x1F, 0x6A, 0x06, 0x95,
+0x4C, 0xE8, 0x5D, 0x67, 0x13, 0xC2, 0x04, 0x04,
+0x02, 0x30, 0xD1, 0x18, 0xA8, 0x19, 0x14, 0xC2,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF,
+0x07, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x01, 0x6A, 0x7D, 0x67, 0x52, 0xC3, 0x05, 0x6A,
+0x51, 0xC3, 0x00, 0x6A, 0x50, 0xC3, 0x08, 0x6A,
+0x53, 0xC3, 0x00, 0x6D, 0x00, 0x6A, 0x04, 0x04,
+0xD1, 0x18, 0xA8, 0x19, 0x54, 0xC3, 0x07, 0x97,
+0x00, 0xEF, 0x04, 0x63, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD0, 0xA5, 0xA4, 0x44, 0xA4, 0x66, 0xA4,
+0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA4, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x2D, 0x22, 0x7D, 0x67, 0xA1, 0xA4,
+0x58, 0xCB, 0x40, 0xA4, 0x62, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA4, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x04, 0xD2, 0xB5, 0xA4, 0x74, 0xA4, 0x56, 0xA4,
+0xA0, 0x35, 0x6D, 0xED, 0x77, 0xA4, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0x01, 0x6A, 0x10, 0x23, 0x5D, 0x67,
+0xB1, 0xA4, 0x79, 0xCA, 0x50, 0xA4, 0x72, 0xA4,
+0xA0, 0x35, 0x4D, 0xED, 0x53, 0xA4, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x05, 0xD2, 0x02, 0x6A, 0x9D, 0x67,
+0x00, 0x6B, 0x40, 0xF0, 0x68, 0xC4, 0x7D, 0x67,
+0x40, 0xF0, 0x49, 0xC3, 0x01, 0x6A, 0x40, 0xF0,
+0x4A, 0xC3, 0x00, 0x6A, 0x11, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x15, 0xF0,
+0x08, 0x4A, 0x85, 0xA2, 0x64, 0xA2, 0xA6, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x67, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x8D, 0xA3, 0x4C, 0xA3, 0xAE, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x6D, 0xA2, 0x0C, 0xA2, 0x8E, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x0F, 0xA2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x80, 0x34,
+0x73, 0xF5, 0x50, 0x9A, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x00, 0x30, 0x04, 0x04, 0x40, 0xEA,
+0x6D, 0xE8, 0x41, 0xC0, 0x15, 0x97, 0x14, 0x90,
+0x00, 0xEF, 0x0B, 0x63, 0xF6, 0x63, 0x13, 0x62,
+0x12, 0xD1, 0x11, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0x85, 0xA2, 0x64, 0xA2, 0xA6, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x67, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x8D, 0xA3, 0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x8D, 0xA2, 0xAE, 0xA2, 0x6C, 0xA2, 0x2F, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0x6D, 0xEC, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x20, 0x31, 0x8D, 0xE9,
+0x43, 0xA1, 0x02, 0xA1, 0x84, 0xA1, 0x40, 0x32,
+0x0D, 0xEA, 0x05, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0xF7, 0xF0, 0x01, 0x6B, 0x8D, 0xEA, 0x00, 0xF6,
+0x00, 0x30, 0x60, 0x33, 0x60, 0x33, 0x4D, 0xE8,
+0x00, 0x6A, 0x0C, 0xD2, 0x94, 0xF2, 0x4C, 0x9B,
+0x0E, 0xD3, 0x36, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF3, 0x54, 0x9A,
+0x00, 0x6E, 0x04, 0x05, 0x40, 0xEA, 0x90, 0x67,
+0x51, 0x22, 0xD1, 0x18, 0x67, 0x1B, 0x04, 0x04,
+0x41, 0xA1, 0x20, 0x22, 0x0E, 0x92, 0x94, 0xF2,
+0x4C, 0x9A, 0x34, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF3, 0x48, 0x9A,
+0x01, 0x6F, 0x0C, 0x06, 0x04, 0x05, 0x40, 0xEA,
+0x90, 0x67, 0x10, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x4C, 0x9A,
+0xE0, 0xF1, 0x0C, 0x6D, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x4A, 0xF1, 0x18, 0x4C,
+0x40, 0xEA, 0x00, 0x65, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0x6A, 0x00, 0xEF, 0x0A, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x52, 0xF0, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x01, 0x6C, 0x8C, 0xEA, 0xBF, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF3,
+0x40, 0x9A, 0xBE, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6B, 0x6C, 0xEA,
+0xC1, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF3, 0x4C, 0x9A, 0x01, 0x6F,
+0x00, 0x6E, 0xC0, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x4C, 0x9A,
+0xE0, 0xF1, 0x0F, 0x6D, 0xC3, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0xD1, 0x18, 0xA7, 0x1B,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0xA6, 0xF4,
+0x0C, 0x4A, 0x60, 0x33, 0xF4, 0xF0, 0x50, 0xDB,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x02, 0x6B,
+0x40, 0x32, 0xF4, 0xF0, 0x6F, 0xC2, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0x00, 0x6E, 0x04, 0x67,
+0x04, 0x6D, 0xD1, 0x18, 0xD4, 0xD9, 0x00, 0x6C,
+0x42, 0xA0, 0x2D, 0x2A, 0x43, 0xA0, 0x02, 0x5A,
+0x2A, 0x61, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x09, 0xF3, 0x38, 0x9B, 0x40, 0x32, 0xF2, 0xF4,
+0x44, 0x9A, 0x40, 0xEA, 0x91, 0x67, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x06, 0xF3,
+0x64, 0xA3, 0x15, 0x2B, 0xFF, 0xF7, 0x1F, 0x6D,
+0xAC, 0xEA, 0x01, 0x4A, 0x83, 0xA0, 0xAC, 0xEA,
+0xF7, 0xF0, 0x01, 0x6B, 0xA2, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x80, 0x34, 0x80, 0x34, 0x00, 0xF6,
+0xA0, 0x35, 0xD2, 0xF4, 0x78, 0x9B, 0x8D, 0xED,
+0x4D, 0xED, 0x40, 0xEB, 0x91, 0x67, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0x20, 0xE8, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0xF3, 0x63, 0x19, 0x62, 0x18, 0xD1, 0x17, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x47, 0xF2, 0x48, 0xA2, 0x00, 0x6F, 0x04, 0xD7,
+0x05, 0xD7, 0x06, 0xD7, 0x07, 0xD7, 0x08, 0xD7,
+0x0D, 0xD7, 0x0E, 0xD7, 0x0F, 0xD7, 0x10, 0xD7,
+0x2B, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0x13, 0xD5, 0x27, 0x44, 0x72, 0xF4,
+0x54, 0x9A, 0x09, 0xF3, 0xBC, 0x9B, 0x11, 0x49,
+0x12, 0xD4, 0x91, 0x67, 0x40, 0xEA, 0x14, 0xD7,
+0x02, 0x67, 0x14, 0x97, 0x1A, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x20, 0xF0, 0x44, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xCA, 0xF1,
+0x0C, 0x4A, 0x0A, 0xD2, 0x54, 0x6A, 0x0B, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x09, 0x04,
+0x19, 0x97, 0x18, 0x91, 0x17, 0x90, 0x00, 0xEF,
+0x0D, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF6, 0x40, 0x9A, 0x90, 0x67,
+0xD1, 0x67, 0x00, 0x6D, 0x40, 0xEA, 0x14, 0xD7,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0x96, 0x13, 0x95, 0x73, 0xF6, 0x5C, 0x9A,
+0x87, 0x40, 0x40, 0xEA, 0x11, 0x4C, 0x01, 0x6E,
+0x04, 0x6D, 0xD1, 0x18, 0xD4, 0xD9, 0x00, 0x6C,
+0x14, 0x97, 0x5D, 0x67, 0x7D, 0x67, 0x20, 0xF0,
+0xFC, 0xC2, 0x97, 0xF0, 0x18, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x2E, 0xF0, 0x0D, 0x4A, 0x10, 0xD2,
+0x0B, 0x6A, 0x51, 0xC3, 0x5D, 0x67, 0xF0, 0xC2,
+0x52, 0xA2, 0x7F, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA,
+0x29, 0x6C, 0x8D, 0xEA, 0x52, 0xC3, 0x04, 0x05,
+0x0D, 0x02, 0x09, 0x04, 0x05, 0xD0, 0x07, 0xD2,
+0x0E, 0xD7, 0xD1, 0x18, 0x66, 0xCA, 0x06, 0xD1,
+0x5D, 0x67, 0x20, 0xF0, 0x04, 0xA2, 0x20, 0xF0,
+0x65, 0xA2, 0x01, 0x70, 0x01, 0x61, 0xB8, 0x23,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x80, 0x34, 0x80, 0x34, 0x40, 0x32, 0x29, 0xF3,
+0x20, 0x9C, 0x40, 0x32, 0xF2, 0xF4, 0x44, 0x9A,
+0x91, 0x67, 0x40, 0xEA, 0x12, 0xD3, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x06, 0xF3,
+0x84, 0xA4, 0x12, 0x93, 0x13, 0x2C, 0xFF, 0xF7,
+0x1F, 0x6D, 0xF7, 0xF0, 0x01, 0x6C, 0xAC, 0xEA,
+0x80, 0x34, 0x80, 0x34, 0x01, 0x4A, 0x60, 0x33,
+0xAC, 0xEA, 0xD2, 0xF4, 0xD8, 0x9C, 0x00, 0xF6,
+0x00, 0x35, 0x60, 0x33, 0x6D, 0xED, 0x4D, 0xED,
+0x40, 0xEE, 0x91, 0x67, 0x00, 0x6E, 0x04, 0x6D,
+0xD1, 0x18, 0xD4, 0xD9, 0x00, 0x6C, 0x88, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x0F, 0x02, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x0F, 0xD7, 0xF7, 0xF0, 0x01, 0x68, 0x06, 0xD6,
+0x40, 0xEA, 0x00, 0x30, 0x22, 0x67, 0x00, 0x30,
+0xF7, 0xF0, 0x01, 0x6A, 0x98, 0xF3, 0x08, 0x48,
+0x40, 0x32, 0x8B, 0xA0, 0x40, 0x32, 0x92, 0xF1,
+0x6C, 0x9A, 0x4A, 0xA0, 0x80, 0x34, 0x04, 0x97,
+0x4D, 0xEC, 0x4C, 0xA0, 0x06, 0x96, 0x00, 0xF2,
+0x00, 0x6D, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x8D, 0xA0, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xEB,
+0x4D, 0xEC, 0xAB, 0xA0, 0x6A, 0xA0, 0x82, 0x67,
+0xA0, 0x35, 0x6D, 0xED, 0x6C, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0xAD, 0xEB, 0xAD, 0xA0, 0x00, 0xF6,
+0xA0, 0x35, 0xD1, 0x18, 0x2E, 0x1C, 0x6D, 0xED,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x91, 0x67,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xA5, 0xA2, 0x64, 0xA2,
+0x86, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x67, 0xA2,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xBD, 0xA3, 0x9C, 0xA3,
+0xDE, 0xA3, 0xA0, 0x35, 0x8D, 0xED, 0x9F, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x80, 0x34, 0xAD, 0xEC, 0x69, 0xA2, 0xAA, 0xA2,
+0xC8, 0xA2, 0x4B, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x71, 0xA2, 0xB2, 0xA2,
+0xD0, 0xA2, 0x53, 0xA2, 0x60, 0x33, 0xA0, 0x35,
+0x6D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x00, 0x6B, 0x60, 0xC2,
+0x40, 0xA4, 0x01, 0x6B, 0x6C, 0xEA, 0x0E, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x40, 0x32, 0x60, 0x33,
+0xB3, 0xF3, 0x54, 0x9A, 0x29, 0xF3, 0x84, 0x9B,
+0x40, 0xEA, 0x00, 0x65, 0x04, 0x07, 0x00, 0x6E,
+0xF2, 0x6D, 0xB1, 0x18, 0x0A, 0xE0, 0xF2, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x48, 0x9A, 0xE0, 0xF1, 0x1C, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0x13, 0xF1, 0x4C, 0x9A,
+0x29, 0xF3, 0xA8, 0x9B, 0x40, 0xEA, 0x00, 0x6C,
+0x07, 0x97, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xA5, 0xA2, 0x64, 0xA2,
+0xC6, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x67, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xBD, 0xA3, 0x5C, 0xA3,
+0xDE, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x5F, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x43, 0x22, 0x68, 0xA4,
+0x61, 0xC2, 0x6D, 0xA4, 0xCC, 0xA4, 0xAE, 0xA4,
+0x60, 0x33, 0xCD, 0xEB, 0xCF, 0xA4, 0xA0, 0x35,
+0xA0, 0x35, 0xF1, 0xA4, 0xAD, 0xEB, 0x00, 0xF6,
+0xC0, 0x36, 0x6D, 0xEE, 0x70, 0xA4, 0xB2, 0xA4,
+0xE0, 0x37, 0x6D, 0xEF, 0x73, 0xA4, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEF, 0x00, 0xF6, 0x60, 0x33,
+0xC2, 0x35, 0xED, 0xEB, 0xA3, 0xC2, 0xA2, 0x35,
+0xA4, 0xC2, 0x62, 0x35, 0xC2, 0xC2, 0x66, 0xC2,
+0xA7, 0xC2, 0x00, 0xF6, 0x62, 0x33, 0x00, 0xF6,
+0xC2, 0x36, 0xA2, 0x35, 0xC5, 0xC2, 0xA8, 0xC2,
+0x69, 0xC2, 0x69, 0xA4, 0x60, 0xC2, 0x01, 0x6A,
+0x4C, 0xEB, 0x14, 0x23, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xB3, 0xF3, 0x54, 0x9A, 0x29, 0xF3, 0x84, 0x9B,
+0x40, 0xEA, 0x00, 0x65, 0x05, 0x97, 0x00, 0x6A,
+0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8, 0x00, 0x6A,
+0x01, 0xD5, 0x02, 0xD6, 0x20, 0xE8, 0x03, 0xD7,
+0x20, 0xE8, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xF7, 0xF0, 0x01, 0x69,
+0x75, 0xF3, 0x84, 0x9B, 0x20, 0x31, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6F, 0x20, 0x31,
+0x40, 0x32, 0xE0, 0x37, 0x07, 0xD3, 0xF4, 0xF0,
+0x14, 0x49, 0x40, 0x32, 0xE0, 0x37, 0x65, 0x2C,
+0x80, 0xF0, 0x04, 0x99, 0x89, 0xF1, 0x54, 0x9A,
+0x1F, 0x6C, 0x0A, 0x30, 0x8C, 0xE8, 0x4D, 0xE8,
+0x06, 0xD0, 0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30,
+0x00, 0x30, 0x06, 0x95, 0xD2, 0xF4, 0x48, 0x98,
+0xFF, 0x6C, 0x6D, 0x4C, 0x40, 0xEA, 0x09, 0xD7,
+0xD2, 0xF4, 0x48, 0x98, 0xE0, 0xF1, 0x10, 0x6C,
+0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x89, 0xF1, 0xB8, 0x9C, 0xE0, 0xF1,
+0x10, 0x6C, 0xAC, 0xEA, 0x02, 0xF3, 0x00, 0x6D,
+0x4D, 0xED, 0x08, 0x92, 0x40, 0xEA, 0x00, 0x65,
+0xD2, 0xF4, 0x48, 0x98, 0x80, 0xF0, 0xA0, 0x99,
+0xE0, 0xF1, 0x14, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x09, 0x97, 0xD2, 0xF4, 0x48, 0x98, 0xE0, 0xF1,
+0x18, 0x6C, 0x94, 0xF1, 0xA0, 0x9F, 0x40, 0xEA,
+0x00, 0x65, 0x07, 0x92, 0x07, 0x93, 0x91, 0x67,
+0x75, 0xF3, 0x44, 0x9A, 0x01, 0x4A, 0x75, 0xF3,
+0x44, 0xDB, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF6, 0x44, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD1, 0x18, 0xAD, 0xF4, 0x71, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF0, 0x48, 0x9A, 0x40, 0xEA, 0x06, 0x94,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF3, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xFF, 0x17, 0x80, 0xF0, 0x04, 0x99, 0x89, 0xF1,
+0x54, 0x9A, 0x1F, 0x6C, 0x0A, 0x30, 0x8C, 0xE8,
+0x4D, 0xE8, 0x06, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x68, 0x60, 0x33, 0x00, 0x30,
+0x60, 0x33, 0x00, 0x30, 0x94, 0xF2, 0x84, 0x9B,
+0x08, 0xD3, 0x94, 0xF2, 0x68, 0x98, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x1B, 0x65, 0x40, 0x32,
+0x94, 0xF1, 0x60, 0x9F, 0xF7, 0xF0, 0x01, 0x6E,
+0x92, 0xF1, 0x40, 0x9A, 0x06, 0x97, 0xC0, 0x36,
+0xA4, 0x67, 0xC0, 0x36, 0x12, 0x6C, 0x05, 0xD3,
+0x04, 0xD4, 0xEA, 0xF1, 0x04, 0x4E, 0x40, 0xEA,
+0x98, 0x67, 0x94, 0xF2, 0x68, 0x98, 0x51, 0xE3,
+0x08, 0x93, 0x94, 0xF2, 0x88, 0xD8, 0x94, 0xF2,
+0x84, 0x9B, 0x4B, 0xE4, 0x94, 0xF2, 0x44, 0xDB,
+0xA4, 0x17, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x06, 0x03, 0x20, 0xF0,
+0x78, 0xA3, 0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30,
+0x00, 0x30, 0x06, 0xD3, 0x98, 0xF3, 0x6B, 0xA0,
+0x98, 0xF3, 0x2A, 0xA0, 0x44, 0x67, 0x60, 0x33,
+0x2D, 0xEB, 0x1B, 0x65, 0x98, 0xF3, 0x6C, 0xA0,
+0x98, 0xF3, 0x2D, 0xA0, 0x98, 0x67, 0x60, 0x33,
+0x60, 0x33, 0x8D, 0xEB, 0x00, 0xF6, 0x20, 0x31,
+0x6D, 0xE9, 0x98, 0xF3, 0x6E, 0xA0, 0x13, 0xD7,
+0x4C, 0xE9, 0x1B, 0x65, 0x98, 0xF3, 0x6F, 0xA0,
+0x98, 0x67, 0x60, 0x33, 0x6D, 0xEC, 0x70, 0x67,
+0x98, 0xF3, 0x08, 0x4B, 0x3C, 0x65, 0x88, 0xA3,
+0x80, 0x34, 0x80, 0x34, 0x1C, 0x65, 0xF8, 0x67,
+0x99, 0x67, 0xED, 0xEC, 0x3C, 0x65, 0x89, 0xA3,
+0x00, 0xF6, 0x80, 0x34, 0x1C, 0x65, 0xF8, 0x67,
+0x99, 0x67, 0xED, 0xEC, 0x8C, 0xED, 0xAD, 0xE9,
+0x65, 0x21, 0xA1, 0xA3, 0xC3, 0xED, 0x62, 0x61,
+0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x13, 0x97, 0x72, 0xF0, 0x50, 0x9A,
+0x23, 0x67, 0x0A, 0xD7, 0x40, 0xEA, 0x09, 0xD6,
+0x07, 0xD2, 0x4A, 0xA1, 0x0E, 0x6B, 0x0A, 0x97,
+0x1A, 0x65, 0x4B, 0xA1, 0x98, 0x67, 0x08, 0x95,
+0x40, 0x32, 0x4D, 0xEC, 0x4C, 0xA1, 0x2D, 0xA1,
+0x09, 0x96, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9, 0x5B, 0x6A,
+0x4B, 0xEA, 0x40, 0xC1, 0x41, 0xC1, 0x06, 0x92,
+0xE4, 0xC1, 0xA9, 0xC1, 0x4B, 0xEA, 0xC0, 0xF7,
+0x42, 0x32, 0x6D, 0xEA, 0x44, 0x32, 0x42, 0xC1,
+0x5A, 0x6A, 0x43, 0xC1, 0xE2, 0x32, 0x45, 0xC1,
+0x42, 0x32, 0x46, 0xC1, 0x5D, 0x67, 0x58, 0x82,
+0x06, 0x93, 0x00, 0xF6, 0xE2, 0x37, 0xE7, 0xC1,
+0x48, 0xC1, 0xCA, 0xC1, 0x1B, 0x2B, 0x47, 0x41,
+0x04, 0x4A, 0x00, 0x6C, 0xA1, 0x42, 0x80, 0xC2,
+0x2B, 0xE5, 0x06, 0xD2, 0x98, 0xF3, 0x48, 0xA0,
+0x01, 0x6C, 0x8C, 0xEA, 0x3E, 0x2A, 0x98, 0xF3,
+0x48, 0xA0, 0x02, 0x6C, 0x8C, 0xEA, 0x4A, 0x2A,
+0x98, 0xF3, 0x48, 0xA0, 0x04, 0x6C, 0x8C, 0xEA,
+0x09, 0x22, 0x06, 0x94, 0xD1, 0x18, 0x2E, 0x1C,
+0xB1, 0x67, 0x04, 0x10, 0x4B, 0xC1, 0x06, 0x92,
+0x21, 0x5A, 0x0D, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x07, 0x94, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0xA7, 0x41,
+0x05, 0x4D, 0x15, 0x06, 0x04, 0xD6, 0xE5, 0x67,
+0x00, 0x6A, 0x60, 0x9E, 0x04, 0x4F, 0x04, 0x4E,
+0x62, 0x34, 0xFF, 0xF7, 0x9D, 0xC7, 0x82, 0x34,
+0xFF, 0xF7, 0x7C, 0xC7, 0xFF, 0xF7, 0x9E, 0xC7,
+0x00, 0xF6, 0x62, 0x33, 0x06, 0x94, 0xFF, 0xF7,
+0x7F, 0xC7, 0x61, 0x42, 0xFF, 0x6A, 0x6C, 0xEA,
+0x4A, 0xEC, 0xEB, 0x61, 0x68, 0x32, 0x49, 0xE5,
+0xB8, 0x17, 0x51, 0x67, 0xA3, 0xEA, 0xBF, 0x60,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xB3, 0xF3, 0xC4, 0x9C, 0x80, 0x82, 0x09, 0xD5,
+0x40, 0xEE, 0x08, 0xD2, 0x08, 0x92, 0x09, 0x95,
+0x01, 0x4A, 0xF0, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x98, 0xF3, 0x58, 0x9A,
+0xAF, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x06, 0x94, 0x33, 0xF0, 0x48, 0x9A,
+0x40, 0xEA, 0xB1, 0x67, 0xA5, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x06, 0x03, 0x44, 0x67, 0x20, 0xF0, 0x98, 0xA3,
+0x98, 0xF3, 0x6B, 0xA1, 0x98, 0xF3, 0x0A, 0xA1,
+0x13, 0xD7, 0x60, 0x33, 0x0D, 0xEB, 0x1B, 0x65,
+0x98, 0xF3, 0x6C, 0xA1, 0x98, 0xF3, 0x0D, 0xA1,
+0xF8, 0x67, 0x60, 0x33, 0x60, 0x33, 0xED, 0xEB,
+0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8, 0x98, 0xF3,
+0x6E, 0xA1, 0x06, 0xD0, 0x1B, 0x65, 0x98, 0xF3,
+0x6F, 0xA1, 0xF8, 0x67, 0x60, 0x33, 0x6D, 0xEF,
+0x71, 0x67, 0x98, 0xF3, 0x08, 0x4B, 0x3F, 0x65,
+0xE8, 0xA3, 0xE0, 0x37, 0xE0, 0x37, 0x1F, 0x65,
+0x18, 0x67, 0xF9, 0x67, 0x0D, 0xEF, 0x3F, 0x65,
+0xE9, 0xA3, 0x00, 0xF6, 0xE0, 0x37, 0x1F, 0x65,
+0x18, 0x67, 0xF9, 0x67, 0x0D, 0xEF, 0x06, 0x90,
+0xEC, 0xED, 0x4C, 0xE8, 0x0D, 0xED, 0x61, 0x25,
+0xA1, 0xA3, 0xC3, 0xED, 0x5E, 0x61, 0x07, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x0A, 0xD3, 0x09, 0xD4,
+0x08, 0xD6, 0x11, 0x67, 0x40, 0xEA, 0x13, 0x91,
+0x0A, 0x93, 0x06, 0xD2, 0x07, 0x95, 0xEB, 0xA3,
+0x4A, 0xA3, 0x08, 0x96, 0xE0, 0x37, 0xED, 0xEA,
+0x1A, 0x65, 0x4C, 0xA3, 0x6D, 0xA3, 0x98, 0x67,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x09, 0x94, 0x4D, 0xEB, 0x5B, 0x6A,
+0x4B, 0xEA, 0x40, 0xC3, 0x41, 0xC3, 0x8B, 0xEA,
+0x0C, 0x6F, 0xC0, 0xF7, 0x42, 0x32, 0xED, 0xEA,
+0x44, 0x32, 0x42, 0xC3, 0x5A, 0x6A, 0x43, 0xC3,
+0x00, 0xF6, 0x80, 0x37, 0x22, 0x32, 0x24, 0xC3,
+0x45, 0xC3, 0x00, 0xF6, 0x22, 0x31, 0x42, 0x32,
+0x00, 0xF6, 0xE3, 0x37, 0x46, 0xC3, 0x27, 0xC3,
+0xE8, 0xC3, 0xA9, 0xC3, 0xCA, 0xC3, 0x47, 0x43,
+0x25, 0x2C, 0x04, 0x4A, 0x00, 0x6C, 0x80, 0xC2,
+0xA1, 0x42, 0x98, 0xF3, 0x48, 0xA0, 0x01, 0x6C,
+0x67, 0xE5, 0x8C, 0xEA, 0x32, 0x2A, 0x98, 0xF3,
+0x48, 0xA0, 0x02, 0x6C, 0x8C, 0xEA, 0x40, 0x2A,
+0x98, 0xF3, 0x48, 0xA0, 0x04, 0x6C, 0x8C, 0xEA,
+0x04, 0x22, 0xA3, 0x67, 0xD1, 0x18, 0x2E, 0x1C,
+0x91, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x06, 0x94, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0x15, 0x06, 0x05, 0x4A,
+0xEB, 0xC3, 0x04, 0xD6, 0x00, 0x6D, 0x83, 0xED,
+0xD5, 0x60, 0xE0, 0x9E, 0xA2, 0x67, 0x89, 0xE2,
+0xAA, 0xEA, 0x05, 0x61, 0xA1, 0x44, 0xFF, 0x6F,
+0xEC, 0xED, 0x04, 0x4E, 0xF4, 0x17, 0x20, 0x87,
+0x01, 0x4D, 0x01, 0x4F, 0xFF, 0xF7, 0x3F, 0xC5,
+0xF3, 0x17, 0x43, 0x67, 0xA3, 0xEA, 0xCB, 0x60,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xB3, 0xF3, 0xC4, 0x9C, 0x80, 0x82, 0x09, 0xD5,
+0x08, 0xD3, 0x40, 0xEE, 0x07, 0xD2, 0x07, 0x92,
+0x09, 0x95, 0x08, 0x93, 0x01, 0x4A, 0xEE, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x98, 0xF3, 0x58, 0x9A, 0xB9, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x33, 0xF0,
+0x48, 0x9A, 0xA3, 0x67, 0x91, 0x67, 0x40, 0xEA,
+0x07, 0xD3, 0x07, 0x93, 0xAD, 0x17, 0x00, 0x65,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x06, 0xF3, 0x44, 0xA0, 0x61, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xF6,
+0x78, 0xA2, 0x05, 0xD2, 0x00, 0x6A, 0x08, 0xD5,
+0x07, 0xD4, 0x04, 0xD2, 0x09, 0x2B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6D, 0xF7, 0xF0, 0x01, 0x6A,
+0xA0, 0x35, 0xA0, 0x35, 0x40, 0x32, 0x29, 0xF3,
+0x2C, 0x9D, 0x40, 0x32, 0xF2, 0xF4, 0x44, 0x9A,
+0x91, 0x67, 0x40, 0xEA, 0x06, 0xD5, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x29, 0xF3,
+0xE8, 0x9C, 0xEA, 0xEA, 0x09, 0xD7, 0x18, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF6, 0x40, 0x9A, 0x00, 0xF4, 0x00, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x91, 0x67, 0x06, 0xF3,
+0x44, 0xA0, 0x09, 0x97, 0x19, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x58, 0x9A, 0xA7, 0x67, 0x40, 0xEA, 0x91, 0x67,
+0x06, 0xF3, 0x44, 0xA0, 0x0D, 0x2A, 0x06, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x29, 0xF3,
+0x8C, 0x9B, 0x40, 0x32, 0x07, 0x93, 0xD2, 0xF4,
+0x58, 0x9A, 0x08, 0x95, 0x40, 0xEA, 0x91, 0xE3,
+0x05, 0x92, 0x8D, 0xF6, 0x58, 0xA2, 0x08, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x04, 0x94,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF,
+0x07, 0x63, 0x00, 0x65, 0x20, 0xE8, 0x05, 0x6A,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x4C, 0x9A, 0x9A, 0xF2, 0x08, 0x68, 0x40, 0xEA,
+0x90, 0x67, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x02, 0x6D, 0xAB, 0xED, 0x60, 0x33, 0xD2, 0xF4,
+0x60, 0x9B, 0xAC, 0xEA, 0x02, 0x6D, 0xAD, 0xEA,
+0xFF, 0x6D, 0x90, 0x67, 0x40, 0xEB, 0x4C, 0xED,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A,
+0x29, 0xF3, 0xB0, 0x9B, 0x18, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xD1, 0x18, 0x9A, 0x1E,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x54, 0x9B, 0xB1, 0xF0, 0x08, 0x69,
+0x91, 0x67, 0x40, 0xEA, 0x04, 0xD3, 0x04, 0x93,
+0x02, 0x67, 0x91, 0x67, 0xD2, 0xF4, 0x54, 0x9B,
+0x00, 0xF5, 0x02, 0x30, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x32, 0x42, 0x32, 0x4D, 0xE8, 0x07, 0x6A,
+0x0C, 0xEA, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x01, 0x5A, 0x58, 0x67, 0x00, 0xEF, 0x05, 0x63,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF1, 0x50, 0x9A, 0x04, 0x67, 0x09, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x04, 0x04, 0x01, 0x69,
+0x5D, 0x67, 0x34, 0xC2, 0x52, 0xA2, 0x10, 0x6B,
+0x6B, 0xEB, 0x6C, 0xEA, 0x07, 0x6B, 0x6D, 0xEA,
+0x7D, 0x67, 0x52, 0xC3, 0x5D, 0x67, 0x53, 0xA2,
+0x80, 0x6B, 0x6B, 0xEB, 0x6D, 0xEA, 0x7D, 0x67,
+0x53, 0xC3, 0x5D, 0x67, 0x50, 0xA2, 0x20, 0x6B,
+0x6B, 0xEB, 0x6C, 0xEA, 0x7D, 0x67, 0x50, 0xC3,
+0x00, 0x6A, 0x40, 0xC0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x33, 0xF1, 0x50, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x00, 0x6B, 0x09, 0x2A,
+0x05, 0x92, 0x03, 0xF0, 0x00, 0x6C, 0x42, 0x32,
+0x8C, 0xEA, 0x8A, 0xEA, 0x08, 0x61, 0x20, 0xC0,
+0x01, 0x6B, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x43, 0x67, 0x00, 0xEF, 0x06, 0x63, 0x60, 0xC0,
+0xF7, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF1, 0x50, 0x9A,
+0x04, 0x67, 0x09, 0x6E, 0x00, 0x6D, 0x40, 0xEA,
+0x04, 0x04, 0x01, 0x69, 0x5D, 0x67, 0x34, 0xC2,
+0x52, 0xA2, 0x10, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x07, 0x6B, 0x6D, 0xEA, 0x7D, 0x67, 0x52, 0xC3,
+0x5D, 0x67, 0x53, 0xA2, 0x80, 0x6B, 0x6B, 0xEB,
+0x6D, 0xEA, 0x7D, 0x67, 0x53, 0xC3, 0x5D, 0x67,
+0x50, 0xA2, 0x20, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x7D, 0x67, 0x50, 0xC3, 0x00, 0x6A, 0x40, 0xC0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x33, 0xF1, 0x50, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x00, 0x6B, 0x06, 0x2A, 0x5D, 0x67, 0x55, 0xA2,
+0xFF, 0x72, 0x08, 0x61, 0x20, 0xC0, 0x01, 0x6B,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x43, 0x67,
+0x00, 0xEF, 0x06, 0x63, 0x60, 0xC0, 0xF7, 0x17,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x11, 0xD5, 0x15, 0xF0, 0x08, 0x4B, 0x85, 0xA3,
+0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x99, 0xA2,
+0x78, 0xA2, 0xBA, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x7B, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3,
+0x40, 0xA3, 0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x43, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x69, 0xA2, 0x20, 0xF0, 0x08, 0xA2, 0x20, 0xF0,
+0x8A, 0xA2, 0x60, 0x33, 0x0D, 0xEB, 0x20, 0xF0,
+0x0B, 0xA2, 0x00, 0x6A, 0x40, 0xC6, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x80, 0x34,
+0x72, 0xF1, 0x50, 0x9A, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0x6D, 0x04, 0x04, 0x26, 0x67, 0x00, 0xF6,
+0x00, 0x30, 0x09, 0x6E, 0x40, 0xEA, 0x6D, 0xE8,
+0x01, 0x6A, 0x7D, 0x67, 0x54, 0xC3, 0x5D, 0x67,
+0x72, 0xA2, 0x10, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB,
+0x01, 0x6A, 0x4D, 0xEB, 0x5D, 0x67, 0x72, 0xC2,
+0x73, 0xA2, 0x80, 0x6A, 0x4B, 0xEA, 0x4D, 0xEB,
+0x5D, 0x67, 0x73, 0xC2, 0x50, 0xA2, 0x06, 0x6B,
+0x8C, 0xEA, 0x6D, 0xEA, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x12, 0xF1, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x09, 0xD2, 0x0B, 0x6A,
+0x08, 0xD2, 0x08, 0x92, 0xFF, 0x4A, 0x08, 0xD2,
+0x02, 0x2A, 0x02, 0x6A, 0x36, 0x10, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0x29, 0xF3,
+0x94, 0x9B, 0xF2, 0xF4, 0x44, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x89, 0xF1, 0x74, 0x9B, 0x6C, 0xEA,
+0x4A, 0x2A, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x33, 0xF1, 0x50, 0x9B, 0x04, 0x04,
+0x40, 0xEA, 0x0A, 0xD3, 0x15, 0x2A, 0x5D, 0x67,
+0x05, 0x95, 0x58, 0xA2, 0xE1, 0xF7, 0x1F, 0x6B,
+0xA2, 0x35, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x6C, 0xEA, 0x11, 0x93, 0x63, 0xEA, 0x33, 0x60,
+0x00, 0xF0, 0x1A, 0x04, 0xD1, 0x18, 0xEB, 0x1E,
+0x00, 0x65, 0x01, 0x72, 0x0A, 0x93, 0x03, 0x60,
+0x03, 0x6A, 0x40, 0xC1, 0x28, 0x10, 0x5D, 0x67,
+0x5A, 0xA2, 0x01, 0x72, 0x4B, 0x61, 0x33, 0xF1,
+0x50, 0x9B, 0x40, 0xEA, 0x04, 0x04, 0xF4, 0x2A,
+0x5D, 0x67, 0x05, 0x94, 0x58, 0xA2, 0xE1, 0xF7,
+0x1F, 0x6B, 0x82, 0x34, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x6C, 0xEA, 0x1A, 0x65, 0x11, 0x92,
+0x78, 0x67, 0x43, 0xEB, 0x10, 0x60, 0x00, 0xF0,
+0x19, 0x04, 0xD1, 0x18, 0xC8, 0x1E, 0x00, 0x65,
+0x01, 0x72, 0xDE, 0x61, 0x5D, 0x67, 0x59, 0xA2,
+0x01, 0x72, 0x1C, 0x61, 0xD1, 0x18, 0xB6, 0x1E,
+0x00, 0x65, 0x01, 0x72, 0xD6, 0x60, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xF7, 0xF0,
+0x01, 0x6A, 0x09, 0xF1, 0xBC, 0x9B, 0x40, 0x32,
+0x09, 0x93, 0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A,
+0xAD, 0xEB, 0x12, 0xF1, 0x10, 0x6C, 0x40, 0xEA,
+0xA3, 0x67, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0x40, 0xF0, 0x42, 0xA0,
+0x40, 0xF0, 0x81, 0xA0, 0xFF, 0xF7, 0x1F, 0x6B,
+0x40, 0x32, 0x8D, 0xEA, 0x01, 0x4A, 0x6C, 0xEA,
+0x40, 0xF0, 0x41, 0xC0, 0x42, 0x32, 0x40, 0xF0,
+0x42, 0xC0, 0xD9, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0x6A, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xAD, 0xA2, 0x6C, 0xA2,
+0xCE, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x6F, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x20, 0xF0, 0xB1, 0xA3,
+0x20, 0xF0, 0x50, 0xA3, 0x20, 0xF0, 0xD2, 0xA3,
+0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0, 0x53, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x61, 0xA2, 0x20, 0xA2,
+0xA2, 0xA2, 0x60, 0x33, 0x2D, 0xEB, 0x23, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6,
+0x20, 0x31, 0x6D, 0xE9, 0x65, 0xA2, 0x04, 0xA2,
+0xA6, 0xA2, 0x60, 0x33, 0x0D, 0xEB, 0x07, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x48, 0x2C, 0x62, 0xA1,
+0x08, 0x6A, 0xFF, 0x6C, 0x6C, 0xEA, 0x8C, 0xEA,
+0x3D, 0x2A, 0x08, 0x6A, 0x4D, 0xEB, 0x62, 0xC1,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6D,
+0x60, 0x33, 0x60, 0x33, 0xA0, 0x35, 0xA0, 0x35,
+0x29, 0xF3, 0x38, 0x9B, 0xF2, 0xF4, 0x44, 0x9D,
+0x04, 0xD5, 0x40, 0xEA, 0x91, 0x67, 0x42, 0x33,
+0x79, 0xC0, 0x62, 0x33, 0x7A, 0xC0, 0xF7, 0xF0,
+0x01, 0x6B, 0x04, 0x95, 0x60, 0x33, 0x60, 0x33,
+0x29, 0xF3, 0x7C, 0x9B, 0x58, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x5B, 0xC0, 0xF2, 0xF4, 0x44, 0x9D,
+0x83, 0x67, 0x40, 0xEA, 0x04, 0xD3, 0x42, 0x34,
+0x5C, 0xC0, 0x9D, 0xC0, 0x00, 0xF6, 0x42, 0x32,
+0x82, 0x34, 0x9E, 0xC0, 0x5F, 0xC0, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0xD2, 0xF4,
+0x58, 0x98, 0x00, 0x6D, 0x40, 0xEA, 0x91, 0x67,
+0x04, 0x93, 0xD2, 0xF4, 0x58, 0x98, 0x00, 0x6D,
+0x40, 0xEA, 0x83, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x01, 0x74,
+0xF9, 0x61, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xC0, 0x36, 0x04, 0xD6, 0xB9, 0xA0, 0x58, 0xA0,
+0xD2, 0xF4, 0xF8, 0x9E, 0xA0, 0x35, 0x4D, 0xED,
+0x5A, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0xBB, 0xA0, 0x00, 0xF6, 0xA0, 0x35, 0x4D, 0xED,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x29, 0xF3, 0x98, 0x9A, 0x40, 0xEF, 0x00, 0x65,
+0xBD, 0xA0, 0x5C, 0xA0, 0x04, 0x96, 0xA0, 0x35,
+0x4D, 0xED, 0x5E, 0xA0, 0xD2, 0xF4, 0xD8, 0x9E,
+0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA, 0xBF, 0xA0,
+0x00, 0xF6, 0xA0, 0x35, 0x4D, 0xED, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x29, 0xF3,
+0x9C, 0x9A, 0x40, 0xEE, 0x00, 0x65, 0x42, 0xA1,
+0x09, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA, 0x42, 0xC1,
+0xC1, 0x17, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0x30, 0xF4, 0x07, 0x69, 0x04, 0x67, 0x40, 0xEA,
+0x91, 0x67, 0x0F, 0x6D, 0xAC, 0xEA, 0x10, 0x30,
+0x4D, 0xE8, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x40, 0x9A, 0x00, 0xF6,
+0x00, 0x30, 0x00, 0xF6, 0x03, 0x30, 0xFF, 0x6D,
+0x0C, 0xED, 0x40, 0xEA, 0x91, 0x67, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x40, 0x9A,
+0xA4, 0x67, 0x30, 0xF4, 0x06, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xD2, 0xF4, 0x48, 0x98, 0x25, 0x67, 0xA4, 0x67,
+0x30, 0xF5, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xD2, 0xF4, 0x48, 0x98, 0x78, 0xF1, 0x00, 0x6C,
+0x40, 0xEA, 0xB1, 0x67, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x15, 0xF0, 0x08, 0x49, 0x4D, 0xA1, 0x6E, 0xA1,
+0x0C, 0xA1, 0x40, 0x32, 0x60, 0x33, 0x0D, 0xEA,
+0x60, 0x33, 0x0F, 0xA1, 0x6D, 0xEA, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6E, 0x60, 0x33,
+0xC0, 0x36, 0x60, 0x33, 0xC0, 0x36, 0x49, 0xF3,
+0x60, 0x9B, 0x00, 0xF6, 0x00, 0x30, 0x4D, 0xE8,
+0x72, 0xF4, 0x54, 0x9E, 0xA3, 0x67, 0x0C, 0x6C,
+0x0B, 0xD6, 0x40, 0xEA, 0x0A, 0xD3, 0x42, 0x34,
+0x20, 0xF0, 0x50, 0xC0, 0x20, 0xF0, 0x91, 0xC0,
+0x00, 0xF6, 0x42, 0x32, 0x82, 0x34, 0x20, 0xF0,
+0x92, 0xC0, 0x20, 0xF0, 0x53, 0xC0, 0xAD, 0xA1,
+0x4C, 0xA1, 0x8E, 0xA1, 0xA0, 0x35, 0x4D, 0xED,
+0x4F, 0xA1, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0,
+0x91, 0xA2, 0x20, 0xF0, 0x10, 0xA2, 0x20, 0xF0,
+0xB2, 0xA2, 0x80, 0x34, 0x0D, 0xEC, 0x20, 0xF0,
+0x13, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8, 0x0A, 0x93,
+0x0B, 0x96, 0x1A, 0x28, 0x06, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0A, 0xF2, 0x14, 0x4A, 0x07, 0xD2,
+0x80, 0xF2, 0x1B, 0x6A, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0x72, 0xF4, 0x54, 0x9E, 0xA3, 0x67, 0x4C, 0x6C,
+0x0B, 0xD6, 0x40, 0xEA, 0x0A, 0xD3, 0x42, 0x34,
+0x44, 0xC0, 0x85, 0xC0, 0x00, 0xF6, 0x42, 0x32,
+0x82, 0x34, 0x86, 0xC0, 0x47, 0xC0, 0x4D, 0xA1,
+0x8C, 0xA1, 0xAE, 0xA1, 0x40, 0x32, 0x8D, 0xEA,
+0x8F, 0xA1, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEA,
+0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0,
+0xB1, 0xA4, 0x20, 0xF0, 0x50, 0xA4, 0x20, 0xF0,
+0xF2, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0,
+0x53, 0xA4, 0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x85, 0xA2,
+0xA6, 0xA2, 0xE4, 0xA2, 0x47, 0xA2, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEF, 0xA0, 0x35, 0xED, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x0A, 0x93,
+0x0B, 0x96, 0x0D, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0A, 0xF2, 0x14, 0x4A, 0x07, 0xD2,
+0xA0, 0xF2, 0x06, 0x6A, 0xAB, 0x17, 0x72, 0xF4,
+0x54, 0x9E, 0xA3, 0x67, 0x40, 0xEA, 0x09, 0x6C,
+0x40, 0xC0, 0x62, 0x67, 0x42, 0x32, 0x41, 0xC0,
+0x42, 0x32, 0x42, 0xC0, 0x00, 0xF6, 0x62, 0x32,
+0x43, 0xC0, 0xAD, 0xA1, 0x8C, 0xA1, 0x4E, 0xA1,
+0xA0, 0x35, 0x8D, 0xED, 0x8F, 0xA1, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xED, 0x00, 0xF6, 0x80, 0x34,
+0xAD, 0xEC, 0x20, 0xF0, 0xB1, 0xA4, 0x20, 0xF0,
+0x50, 0xA4, 0x20, 0xF0, 0xD2, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x53, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x81, 0xA2, 0xA2, 0xA2, 0xC0, 0xA2,
+0x43, 0xA2, 0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE,
+0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x0D, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0A, 0xF2, 0x14, 0x4A, 0x07, 0xD2,
+0xA0, 0xF2, 0x10, 0x6A, 0x67, 0x17, 0x0B, 0xD3,
+0x45, 0xA0, 0x24, 0xA0, 0x86, 0xA0, 0x40, 0x32,
+0x2D, 0xEA, 0x27, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0x0A, 0xD1, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0x93, 0xF6, 0x40, 0x99, 0x0A, 0x94,
+0x10, 0x6E, 0x40, 0xEA, 0x00, 0x6D, 0x0A, 0x93,
+0x93, 0xF6, 0x40, 0x99, 0x04, 0x6E, 0x87, 0x43,
+0xFF, 0x6D, 0x40, 0xEA, 0x09, 0x4C, 0x0A, 0x93,
+0x93, 0xF6, 0x40, 0x99, 0x02, 0x6E, 0x87, 0x43,
+0x00, 0x6D, 0x40, 0xEA, 0x0D, 0x4C, 0x0A, 0x93,
+0x93, 0xF6, 0x40, 0x99, 0x08, 0x6E, 0x87, 0x43,
+0x00, 0x6D, 0x40, 0xEA, 0x11, 0x4C, 0x0A, 0x93,
+0x93, 0xF6, 0x40, 0x99, 0x0C, 0x6E, 0x87, 0x43,
+0x00, 0x6D, 0x40, 0xEA, 0x19, 0x4C, 0x0A, 0x93,
+0x93, 0xF6, 0x40, 0x99, 0x20, 0x6E, 0x87, 0x43,
+0x00, 0x6D, 0x40, 0xEA, 0x25, 0x4C, 0x0B, 0x93,
+0x01, 0x6A, 0x10, 0x6C, 0x40, 0xC3, 0x00, 0x6A,
+0x41, 0xC3, 0x42, 0xA3, 0x8B, 0xEC, 0x00, 0x69,
+0x8C, 0xEA, 0x42, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x24, 0xC3, 0x25, 0xC3,
+0x26, 0xC3, 0x27, 0xC3, 0xD2, 0xF1, 0x78, 0x9A,
+0x97, 0xF0, 0x18, 0x6A, 0x40, 0x32, 0xF7, 0xF0,
+0x01, 0x6C, 0x40, 0x32, 0x80, 0x34, 0xB6, 0xF1,
+0x01, 0x4A, 0x80, 0x34, 0x2A, 0xF2, 0x10, 0x4C,
+0x04, 0xD2, 0x01, 0x6F, 0xE0, 0xF1, 0x14, 0x6D,
+0x40, 0xEB, 0x01, 0x6E, 0x82, 0x67, 0x48, 0xC0,
+0x42, 0x32, 0x49, 0xC0, 0x42, 0x32, 0x4A, 0xC0,
+0x00, 0xF6, 0x82, 0x32, 0x4B, 0xC0, 0x0D, 0x2C,
+0x06, 0x6A, 0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0A, 0xF2,
+0x14, 0x4A, 0x07, 0xD2, 0xC0, 0xF2, 0x0C, 0x6A,
+0xE9, 0x16, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF1, 0x14, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x92, 0xF2,
+0x4C, 0x9A, 0x40, 0xEA, 0x0A, 0xD4, 0x0A, 0x94,
+0x04, 0xD1, 0x00, 0x6F, 0xC2, 0x67, 0x40, 0xE8,
+0x01, 0x6D, 0xDD, 0x16, 0xFD, 0x63, 0x05, 0x62,
+0x06, 0x24, 0xD1, 0x18, 0x9A, 0x1E, 0x00, 0x65,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x4C, 0x9A, 0x9A, 0xF2, 0x08, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x03, 0x6D, 0xD2, 0xF4, 0x60, 0x9B,
+0xAD, 0xEA, 0xFF, 0x6D, 0x9A, 0xF2, 0x08, 0x6C,
+0x40, 0xEB, 0x4C, 0xED, 0xE5, 0x17, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69,
+0x60, 0x33, 0x20, 0x31, 0x60, 0x33, 0x20, 0x31,
+0xD2, 0xF4, 0xE8, 0x9B, 0xD2, 0xF4, 0x54, 0x99,
+0x18, 0xF0, 0x00, 0x68, 0x90, 0x67, 0x05, 0xD3,
+0x40, 0xEA, 0x04, 0xD7, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x09, 0xF1, 0xDC, 0x9C,
+0x04, 0x97, 0x90, 0x67, 0xCD, 0xEA, 0xA2, 0x67,
+0x40, 0xEF, 0x06, 0xD6, 0x05, 0x93, 0x10, 0xF4,
+0x00, 0x68, 0x90, 0x67, 0xD2, 0xF4, 0xE8, 0x9B,
+0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA, 0x04, 0xD7,
+0x06, 0x96, 0x04, 0x97, 0x90, 0x67, 0xA6, 0x67,
+0x40, 0xEF, 0x4D, 0xED, 0xD1, 0x18, 0xDF, 0x20,
+0x00, 0x6C, 0x05, 0x93, 0x90, 0x67, 0xD2, 0xF4,
+0x68, 0x9B, 0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA,
+0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x49, 0xF3, 0xA4, 0x9C, 0x04, 0x93,
+0x90, 0x67, 0x40, 0xEB, 0x4C, 0xED, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x07, 0x24,
+0xD1, 0x18, 0xAA, 0x1E, 0x00, 0x65, 0x05, 0x97,
+0x04, 0x90, 0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0,
+0x01, 0x68, 0xF7, 0xF0, 0x01, 0x6B, 0x00, 0x30,
+0x60, 0x33, 0x60, 0x33, 0x00, 0x30, 0xD2, 0xF4,
+0x48, 0x98, 0x29, 0xF3, 0xB0, 0x9B, 0x18, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xD1, 0x18,
+0xDF, 0x20, 0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x48, 0x98,
+0x09, 0xF1, 0xBC, 0x9B, 0x18, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xDC, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x21, 0x24,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x08, 0x9A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x10, 0xF4, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xC9, 0xF1, 0xB8, 0x9B, 0x10, 0xF4, 0x00, 0x6C,
+0x40, 0xE8, 0x4D, 0xED, 0xD1, 0x18, 0x9A, 0x1E,
+0x00, 0x65, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0xD1, 0x18, 0xF0, 0x20, 0x00, 0x65,
+0xF8, 0x17, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0x8D, 0xA2, 0x6C, 0xA2, 0xAE, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x6F, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x20, 0xF0, 0x91, 0xA3, 0x20, 0xF0, 0x50, 0xA3,
+0x20, 0xF0, 0xB2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x20, 0xF0, 0x53, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x81, 0xA2, 0x60, 0xA2, 0xA2, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x63, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0xC2, 0xA3, 0x04, 0x6C, 0xFF, 0x6D, 0xCC, 0xEC,
+0xAC, 0xEC, 0x80, 0xF0, 0x19, 0x2C, 0x85, 0xA2,
+0x04, 0xA2, 0xA6, 0xA2, 0x80, 0x34, 0xF7, 0xF0,
+0x01, 0x69, 0x0D, 0xEC, 0x20, 0x31, 0x07, 0xA2,
+0x20, 0x31, 0x04, 0x6A, 0xA0, 0x35, 0x4D, 0xEE,
+0xA0, 0x35, 0xD2, 0xF4, 0x54, 0x99, 0xAD, 0xEC,
+0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8, 0xB3, 0xF6,
+0x10, 0x6C, 0x40, 0xEA, 0xC2, 0xC3, 0x40, 0xC0,
+0x42, 0x33, 0x00, 0xF6, 0x42, 0x32, 0x43, 0xC0,
+0xD2, 0xF4, 0x54, 0x99, 0x61, 0xC0, 0x62, 0x33,
+0xB3, 0xF6, 0x14, 0x6C, 0x40, 0xEA, 0x62, 0xC0,
+0x44, 0xC0, 0x42, 0x33, 0x00, 0xF6, 0x42, 0x32,
+0x47, 0xC0, 0xD2, 0xF4, 0x54, 0x99, 0x65, 0xC0,
+0x62, 0x33, 0xB3, 0xF6, 0x18, 0x6C, 0x40, 0xEA,
+0x66, 0xC0, 0x48, 0xC0, 0x42, 0x33, 0x00, 0xF6,
+0x42, 0x32, 0x4B, 0xC0, 0xD2, 0xF4, 0x54, 0x99,
+0x69, 0xC0, 0x62, 0x33, 0xB3, 0xF6, 0x1C, 0x6C,
+0xF7, 0xF0, 0x01, 0x69, 0x40, 0xEA, 0x6A, 0xC0,
+0x20, 0x31, 0x4C, 0xC0, 0x42, 0x33, 0x20, 0x31,
+0x00, 0xF6, 0x42, 0x32, 0x4F, 0xC0, 0xD2, 0xF4,
+0x50, 0x99, 0x6D, 0xC0, 0x62, 0x33, 0x58, 0xF3,
+0x08, 0x6C, 0x40, 0xEA, 0x6E, 0xC0, 0x50, 0xC0,
+0x42, 0x32, 0x51, 0xC0, 0xD2, 0xF4, 0x50, 0x99,
+0x19, 0xF6, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x32, 0x01, 0x6B, 0x4E, 0x32, 0x6C, 0xEA,
+0x54, 0xC0, 0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30,
+0x00, 0x30, 0xD2, 0xF4, 0x48, 0x98, 0x01, 0x6D,
+0xB3, 0xF6, 0x10, 0x6C, 0x40, 0xEA, 0xAB, 0xED,
+0xD2, 0xF4, 0x48, 0x98, 0x01, 0x6D, 0xB3, 0xF6,
+0x14, 0x6C, 0x40, 0xEA, 0xAB, 0xED, 0xD2, 0xF4,
+0x48, 0x98, 0x01, 0x6D, 0xB3, 0xF6, 0x18, 0x6C,
+0x40, 0xEA, 0xAB, 0xED, 0xD2, 0xF4, 0x48, 0x98,
+0xF7, 0xF0, 0x01, 0x68, 0x01, 0x6D, 0x00, 0x30,
+0xAB, 0xED, 0xB3, 0xF6, 0x1C, 0x6C, 0x40, 0xEA,
+0x00, 0x30, 0xD2, 0xF4, 0x44, 0x98, 0x58, 0xF3,
+0x08, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0xD2, 0xF4,
+0x50, 0x99, 0x19, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x01, 0xF0, 0x00, 0x6D, 0xD2, 0xF4,
+0x64, 0x98, 0xAD, 0xEA, 0xFF, 0xF7, 0x1F, 0x6D,
+0x19, 0xF6, 0x00, 0x6C, 0x40, 0xEB, 0x4C, 0xED,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF7, 0xF0, 0x01, 0x68,
+0xD2, 0xF4, 0x50, 0x9A, 0x00, 0x30, 0x00, 0x30,
+0x15, 0xF0, 0x08, 0x48, 0x19, 0xF6, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xAD, 0xA0, 0x8C, 0xA0,
+0x6E, 0xA0, 0xA0, 0x35, 0x8D, 0xED, 0x8F, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6,
+0x80, 0x34, 0xAD, 0xEC, 0x20, 0xF0, 0xB1, 0xA4,
+0x20, 0xF0, 0x70, 0xA4, 0x20, 0xF0, 0xD2, 0xA4,
+0xA0, 0x35, 0x6D, 0xED, 0x20, 0xF0, 0x73, 0xA4,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x85, 0xA3, 0xA6, 0xA3,
+0xC4, 0xA3, 0x67, 0xA3, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x74, 0xA3, 0xFF, 0xF7,
+0x1F, 0x6D, 0x40, 0xF1, 0x1B, 0x23, 0x01, 0xF0,
+0x00, 0x6B, 0x4D, 0xEB, 0x6C, 0xED, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0xAE, 0xEA, 0x20, 0x31,
+0x06, 0x22, 0xD2, 0xF4, 0x44, 0x99, 0x19, 0xF6,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x8D, 0xA0,
+0x6C, 0xA0, 0x4E, 0xA0, 0x80, 0x34, 0x6D, 0xEC,
+0x6F, 0xA0, 0x40, 0x32, 0x40, 0x32, 0x4D, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x20, 0xF0,
+0x91, 0xA3, 0x20, 0xF0, 0x50, 0xA3, 0x20, 0xF0,
+0xB2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0,
+0x53, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2,
+0x86, 0xA2, 0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0xB0, 0xA2,
+0x51, 0xA2, 0xD2, 0xF4, 0xC4, 0x99, 0x58, 0xF3,
+0x08, 0x6C, 0x40, 0x32, 0x40, 0xEE, 0x4D, 0xED,
+0x8D, 0xA0, 0x4C, 0xA0, 0x6E, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x4F, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x20, 0xF0, 0x91, 0xA2, 0x20, 0xF0, 0x70, 0xA2,
+0x20, 0xF0, 0xB2, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x20, 0xF0, 0x73, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0xA1, 0xA2, 0x60, 0xA2, 0xF7, 0xF0, 0x01, 0x69,
+0xA0, 0x35, 0x6D, 0xED, 0x62, 0xA2, 0x20, 0x31,
+0x20, 0x31, 0x60, 0x33, 0x60, 0x33, 0xAD, 0xEB,
+0xA3, 0xA2, 0xD2, 0xF4, 0xC8, 0x99, 0xB3, 0xF6,
+0x10, 0x6C, 0x00, 0xF6, 0xA0, 0x35, 0x40, 0xEE,
+0x6D, 0xED, 0x8D, 0xA0, 0x4C, 0xA0, 0x6E, 0xA0,
+0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x91, 0xA2, 0x20, 0xF0,
+0x70, 0xA2, 0x20, 0xF0, 0xB2, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x20, 0xF0, 0x73, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0xA5, 0xA2, 0x64, 0xA2, 0xD2, 0xF4,
+0xC8, 0x99, 0xA0, 0x35, 0x6D, 0xED, 0x66, 0xA2,
+0xB3, 0xF6, 0x14, 0x6C, 0x60, 0x33, 0x60, 0x33,
+0xAD, 0xEB, 0xA7, 0xA2, 0x00, 0xF6, 0xA0, 0x35,
+0x40, 0xEE, 0x6D, 0xED, 0x8D, 0xA0, 0x4C, 0xA0,
+0x6E, 0xA0, 0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0, 0x91, 0xA2,
+0x20, 0xF0, 0x70, 0xA2, 0x20, 0xF0, 0xB2, 0xA2,
+0x80, 0x34, 0x6D, 0xEC, 0x20, 0xF0, 0x73, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x85, 0xA3, 0x44, 0xA3,
+0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0xA9, 0xA2, 0x68, 0xA2,
+0xD2, 0xF4, 0xC8, 0x99, 0xA0, 0x35, 0x6D, 0xED,
+0x6A, 0xA2, 0xB3, 0xF6, 0x18, 0x6C, 0x60, 0x33,
+0x60, 0x33, 0xAD, 0xEB, 0xAB, 0xA2, 0x00, 0xF6,
+0xA0, 0x35, 0x40, 0xEE, 0x6D, 0xED, 0x8D, 0xA0,
+0x4C, 0xA0, 0x6E, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x4F, 0xA0, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x91, 0xA2, 0x20, 0xF0, 0x70, 0xA2, 0x20, 0xF0,
+0xB2, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x20, 0xF0,
+0x73, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x85, 0xA3,
+0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0xAD, 0xA2,
+0x6C, 0xA2, 0xD2, 0xF4, 0xC8, 0x99, 0xA0, 0x35,
+0x6D, 0xED, 0x6E, 0xA2, 0xB3, 0xF6, 0x1C, 0x6C,
+0x60, 0x33, 0x60, 0x33, 0xAD, 0xEB, 0xAF, 0xA2,
+0x00, 0xF6, 0xA0, 0x35, 0x40, 0xEE, 0x6D, 0xED,
+0x8D, 0xA0, 0x6C, 0xA0, 0x4E, 0xA0, 0x80, 0x34,
+0x6D, 0xEC, 0x6F, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x20, 0xF0, 0x91, 0xA3, 0x20, 0xF0, 0x50, 0xA3,
+0x20, 0xF0, 0xB2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x20, 0xF0, 0x53, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x82, 0xA2, 0x61, 0xA2, 0xA0, 0xA2, 0x43, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x62, 0xA2, 0x05, 0x6C, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x8B, 0xEC, 0x8C, 0xEB, 0x62, 0xC2,
+0x00, 0xEF, 0x04, 0x63, 0x01, 0xF0, 0x01, 0x6B,
+0x6B, 0xEB, 0x4C, 0xEB, 0xA3, 0x16, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xD2, 0xF4, 0x48, 0x98, 0x25, 0x67, 0xA4, 0x67,
+0x33, 0xF0, 0x18, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x44, 0x9A, 0xC0, 0xF1, 0x00, 0x6D,
+0x33, 0xF0, 0x14, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xD2, 0xF4, 0x48, 0x98, 0x49, 0xF3, 0xA8, 0x9B,
+0x33, 0xF0, 0x10, 0x6C, 0xE0, 0xF3, 0x09, 0x68,
+0x40, 0xEA, 0x00, 0x65, 0xFF, 0x48, 0xFF, 0xF7,
+0x1F, 0x6A, 0x4C, 0xE8, 0x13, 0x28, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x01, 0x6B, 0xFF, 0x6F, 0x05, 0xD1,
+0x04, 0xD3, 0x2A, 0x4F, 0x02, 0x6E, 0x02, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x01, 0x6A,
+0x4B, 0xEA, 0x16, 0x10, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x33, 0xF0, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x00, 0x52, 0x09, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0xD3, 0x17, 0x00, 0x6A,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x05, 0xD5, 0x06, 0xD4,
+0x15, 0xF0, 0x08, 0x49, 0xCD, 0xA1, 0x8C, 0xA1,
+0x4E, 0xA1, 0xC0, 0x36, 0x8D, 0xEE, 0x8F, 0xA1,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0xCD, 0xEC, 0x20, 0xF0, 0xD1, 0xA4,
+0x20, 0xF0, 0x50, 0xA4, 0x20, 0xF0, 0xF2, 0xA4,
+0xC0, 0x36, 0x4D, 0xEE, 0x20, 0xF0, 0x53, 0xA4,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x80, 0xA2, 0xC1, 0xA2,
+0xE2, 0xA2, 0x03, 0xA2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC0, 0x36, 0xE0, 0x37,
+0x72, 0xF0, 0x50, 0x9A, 0x8D, 0xEE, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0x00, 0x30, 0x40, 0xEA,
+0xCD, 0xE8, 0x04, 0xD2, 0x4D, 0xA1, 0xCC, 0xA1,
+0xEE, 0xA1, 0x40, 0x32, 0xCD, 0xEA, 0xCF, 0xA1,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEA, 0x00, 0xF6,
+0xC0, 0x36, 0x4D, 0xEE, 0x20, 0xF0, 0xF1, 0xA6,
+0x20, 0xF0, 0x90, 0xA6, 0x20, 0xF0, 0x53, 0xA6,
+0xE0, 0x37, 0xED, 0xEC, 0x20, 0xF0, 0xF2, 0xA6,
+0x00, 0xF6, 0x40, 0x32, 0x05, 0x95, 0xE0, 0x37,
+0xE0, 0x37, 0x8D, 0xEF, 0xED, 0xEA, 0xE1, 0xA2,
+0x80, 0xA2, 0xC3, 0xA2, 0xE0, 0x37, 0xED, 0xEC,
+0xE2, 0xA2, 0x00, 0xF6, 0xC0, 0x36, 0x01, 0x6A,
+0xE0, 0x37, 0xE0, 0x37, 0x8D, 0xEF, 0x81, 0x98,
+0xED, 0xEE, 0x06, 0x93, 0x01, 0x5C, 0xF8, 0x67,
+0x4E, 0xEF, 0x1B, 0x2D, 0x44, 0xEB, 0x4F, 0xEA,
+0x8C, 0xEA, 0x41, 0xD8, 0x81, 0x98, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x01, 0x5C, 0x58, 0x67,
+0x01, 0x72, 0x58, 0x67, 0x4A, 0xEF, 0x00, 0x30,
+0x10, 0x61, 0x01, 0x2B, 0x0E, 0x25, 0x72, 0xF0,
+0x54, 0x98, 0x40, 0xEA, 0x04, 0x94, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0x6A, 0x00, 0xEF,
+0x06, 0x63, 0x01, 0x6A, 0x44, 0xEB, 0x8D, 0xEA,
+0xE4, 0x17, 0x05, 0xD3, 0xA9, 0xA1, 0x48, 0xA1,
+0xEA, 0xA1, 0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA1,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xA5, 0xA2, 0xE6, 0xA2,
+0x24, 0xA2, 0x47, 0xA2, 0xA0, 0x35, 0xE0, 0x37,
+0xAD, 0xE9, 0xE0, 0x37, 0x2D, 0xEF, 0x00, 0xF6,
+0x40, 0x32, 0xF7, 0xF0, 0x01, 0x6D, 0xED, 0xEA,
+0xA0, 0x35, 0x29, 0xA2, 0xE2, 0xA6, 0xA0, 0x35,
+0x22, 0x24, 0x02, 0x6A, 0xED, 0xEA, 0x42, 0xC6,
+0xD2, 0xF4, 0x54, 0x9D, 0xE0, 0xF1, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x40, 0x32, 0x42, 0x32,
+0x42, 0x32, 0x11, 0x72, 0xC4, 0x60, 0x05, 0x93,
+0x09, 0x2B, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x13, 0xF0, 0x44, 0x9A, 0x40, 0xEA,
+0x02, 0x6C, 0xB9, 0x17, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0x0C, 0x20, 0x00, 0x6C, 0xD1, 0x18,
+0x41, 0x21, 0x91, 0x67, 0xB0, 0x17, 0x03, 0x6A,
+0x4B, 0xEA, 0xEC, 0xEA, 0x42, 0xC6, 0xD2, 0xF4,
+0x54, 0x9D, 0xE0, 0xF1, 0x10, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x40, 0x32, 0x42, 0x32, 0x42, 0x32,
+0x11, 0x72, 0xA1, 0x60, 0x01, 0x6E, 0xCB, 0xEE,
+0x86, 0x67, 0xD1, 0x18, 0x0C, 0x20, 0xA6, 0x67,
+0xD1, 0x18, 0xAD, 0x21, 0x91, 0x67, 0x97, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xD2, 0xF4, 0x54, 0x98, 0x18, 0xF4, 0x00, 0x6E,
+0xF7, 0xF0, 0x01, 0x69, 0x86, 0x67, 0x20, 0x31,
+0x20, 0x31, 0x40, 0xEA, 0x04, 0xD6, 0x04, 0x96,
+0xD2, 0xF4, 0xE8, 0x99, 0x04, 0x6D, 0xAB, 0xED,
+0x4C, 0xED, 0x40, 0xEF, 0x86, 0x67, 0xD2, 0xF4,
+0x54, 0x98, 0x58, 0xF4, 0x00, 0x6E, 0x86, 0x67,
+0x40, 0xEA, 0x04, 0xD6, 0x04, 0x96, 0xD2, 0xF4,
+0xE8, 0x99, 0x04, 0x6D, 0xAB, 0xED, 0x4C, 0xED,
+0x40, 0xEF, 0x86, 0x67, 0xD2, 0xF4, 0x54, 0x98,
+0x98, 0xF4, 0x00, 0x6E, 0x86, 0x67, 0x40, 0xEA,
+0x04, 0xD6, 0x04, 0x96, 0xD2, 0xF4, 0xE8, 0x99,
+0x04, 0x6D, 0xAB, 0xED, 0x4C, 0xED, 0x40, 0xEF,
+0x86, 0x67, 0xD2, 0xF4, 0x54, 0x98, 0xD8, 0xF4,
+0x00, 0x6E, 0x86, 0x67, 0x40, 0xEA, 0x04, 0xD6,
+0x04, 0x96, 0xD2, 0xF4, 0xE8, 0x99, 0x04, 0x6D,
+0xAB, 0xED, 0x4C, 0xED, 0x40, 0xEF, 0x86, 0x67,
+0xD2, 0xF4, 0x54, 0x98, 0x18, 0xF5, 0x00, 0x6B,
+0x83, 0x67, 0x40, 0xEA, 0x03, 0x67, 0xD2, 0xF4,
+0xC8, 0x99, 0x04, 0x6D, 0xAB, 0xED, 0x90, 0x67,
+0x40, 0xEE, 0x4C, 0xED, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x85, 0xA3, 0x44, 0xA3,
+0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x99, 0xA2, 0x78, 0xA2,
+0xBA, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x7B, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0, 0x69, 0xA2,
+0x20, 0xF0, 0x08, 0xA2, 0x20, 0xF0, 0x8A, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x20, 0xF0, 0x0B, 0xA2,
+0x80, 0x34, 0x80, 0x34, 0xF7, 0xF0, 0x01, 0x69,
+0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30, 0x20, 0x31,
+0x6D, 0xE8, 0x20, 0x31, 0xD2, 0xF4, 0x40, 0x99,
+0xC0, 0xF1, 0x00, 0x6C, 0x40, 0xEA, 0xA2, 0xA0,
+0xD2, 0xF4, 0x40, 0x99, 0xC0, 0xF1, 0x01, 0x6C,
+0x40, 0xEA, 0xA3, 0xA0, 0xD2, 0xF4, 0x40, 0x99,
+0xC0, 0xF1, 0x02, 0x6C, 0x40, 0xEA, 0xA4, 0xA0,
+0xD2, 0xF4, 0x40, 0x99, 0xC0, 0xF1, 0x03, 0x6C,
+0x40, 0xEA, 0xA5, 0xA0, 0xD2, 0xF4, 0x40, 0x99,
+0xC0, 0xF1, 0x04, 0x6C, 0x40, 0xEA, 0xA6, 0xA0,
+0xD2, 0xF4, 0x40, 0x99, 0xC0, 0xF1, 0x05, 0x6C,
+0x40, 0xEA, 0xA7, 0xA0, 0xD2, 0xF4, 0x40, 0x99,
+0xC0, 0xF1, 0x06, 0x6C, 0x40, 0xEA, 0xA8, 0xA0,
+0xD2, 0xF4, 0x40, 0x99, 0xC0, 0xF1, 0x07, 0x6C,
+0x40, 0xEA, 0xA0, 0xA0, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0x14, 0xD4, 0x15, 0xF0, 0x08, 0x4F, 0x8D, 0xA7,
+0x6C, 0xA7, 0x4E, 0xA7, 0x80, 0x34, 0x6D, 0xEC,
+0x6F, 0xA7, 0x40, 0x32, 0x40, 0x32, 0x4D, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x20, 0xF0,
+0x91, 0xA3, 0x20, 0xF0, 0x50, 0xA3, 0x20, 0xF0,
+0xB2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0,
+0x53, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x81, 0xA2,
+0xA0, 0xA2, 0x62, 0xA2, 0x80, 0x34, 0x8D, 0xED,
+0x83, 0xA2, 0x60, 0x33, 0x60, 0x33, 0xAD, 0xEB,
+0x00, 0xF6, 0x80, 0x34, 0x6D, 0xEC, 0x0B, 0xD4,
+0x85, 0xA2, 0xA6, 0xA2, 0x64, 0xA2, 0x07, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0x6D, 0xEC, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8,
+0x89, 0xA7, 0x48, 0xA7, 0xAA, 0xA7, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA7, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x0E, 0xD7,
+0x8D, 0xEA, 0xA5, 0xA2, 0x86, 0xA2, 0xC4, 0xA2,
+0xA0, 0x35, 0x47, 0xA2, 0x80, 0x34, 0xAD, 0xEE,
+0x80, 0x34, 0xCD, 0xEC, 0xF7, 0xF0, 0x01, 0x6E,
+0x00, 0xF6, 0x40, 0x32, 0xC0, 0x36, 0xC0, 0x36,
+0x8D, 0xEA, 0x0A, 0xD2, 0xF7, 0xF0, 0x01, 0x69,
+0x93, 0xF3, 0x50, 0x9E, 0x20, 0x31, 0x20, 0x31,
+0x3E, 0xF4, 0x04, 0x6C, 0x40, 0xEA, 0x08, 0xD6,
+0xD2, 0xF4, 0x48, 0x99, 0xE0, 0xF1, 0x1C, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x54, 0x9B,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEA, 0x0C, 0xD3,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x89, 0xF1, 0xAC, 0x9C, 0xD2, 0xF4, 0x68, 0x99,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEB, 0x4D, 0xED,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x01, 0x6C,
+0x40, 0x32, 0x8C, 0xF2, 0x88, 0xC2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x4C, 0x9A, 0x30, 0xF4, 0x07, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x0F, 0x6B, 0x1B, 0x65, 0x78, 0x67,
+0x4C, 0xEB, 0xFF, 0x6D, 0xAC, 0xEB, 0x0F, 0x73,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x08, 0x96, 0x09, 0xD3, 0x0C, 0x93, 0x08, 0xD1,
+0x0D, 0xD6, 0x23, 0x67, 0x0E, 0x97, 0x7F, 0x61,
+0x10, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x4C, 0xED,
+0xD2, 0xF4, 0x40, 0x9B, 0x30, 0xF4, 0x07, 0x6C,
+0x0E, 0xD7, 0x40, 0xEA, 0x0C, 0xD3, 0x0A, 0x92,
+0xFF, 0x6C, 0x0C, 0x93, 0x48, 0xA2, 0x0E, 0x97,
+0x0A, 0xD3, 0xFF, 0x4A, 0x8C, 0xEA, 0x03, 0x5A,
+0x77, 0x60, 0xD2, 0xF4, 0x54, 0x99, 0x30, 0xF5,
+0x04, 0x6C, 0x40, 0xEA, 0x0C, 0xD7, 0x40, 0x72,
+0x0C, 0x97, 0x6E, 0x61, 0xD2, 0xF4, 0x54, 0x99,
+0x32, 0xF0, 0x1C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x49, 0xF3, 0x8C, 0x9C, 0x0C, 0x97, 0x8E, 0xEA,
+0x50, 0x2A, 0xD2, 0xF4, 0x54, 0x99, 0x32, 0xF0,
+0x14, 0x6C, 0x40, 0xEA, 0x0C, 0xD7, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xE9, 0xF1,
+0x88, 0x9C, 0x0C, 0x97, 0x8C, 0xEA, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x49, 0xF3,
+0x90, 0x9C, 0x8E, 0xEA, 0x49, 0x2A, 0x08, 0x92,
+0xE3, 0xF7, 0x1F, 0x6D, 0x53, 0xF0, 0x04, 0x6C,
+0xD2, 0xF4, 0x48, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xC9, 0xF2, 0xB0, 0x9B, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0x29, 0xF3, 0x94, 0x9B,
+0xD2, 0xF4, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x08, 0x92, 0xE0, 0xF1, 0x10, 0x6C, 0xD2, 0xF4,
+0x08, 0x9A, 0xD2, 0xF4, 0x54, 0x99, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x89, 0xF1, 0xB8, 0x9B, 0xE0, 0xF1,
+0x10, 0x6C, 0xAC, 0xEA, 0x10, 0xF1, 0x00, 0x6D,
+0x40, 0xE8, 0x4D, 0xED, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0x01, 0x4A,
+0x82, 0x17, 0xD2, 0xF4, 0x54, 0x99, 0x32, 0xF0,
+0x1C, 0x6C, 0x40, 0xEA, 0x0C, 0xD7, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xA9, 0xF2,
+0x9C, 0x9C, 0x0C, 0x97, 0x8E, 0xEA, 0xA1, 0x22,
+0x00, 0x6E, 0x00, 0x6D, 0x00, 0x6C, 0xD1, 0x18,
+0x0C, 0x20, 0x0C, 0xD7, 0x0C, 0x97, 0x89, 0xA7,
+0x48, 0xA7, 0xAA, 0xA7, 0x80, 0x34, 0x4D, 0xEC,
+0x4B, 0xA7, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x85, 0xA2,
+0xA6, 0xA2, 0xC4, 0xA2, 0x47, 0xA2, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x8B, 0xA2,
+0x20, 0xF2, 0x06, 0x2C, 0x02, 0x6C, 0x8B, 0xC2,
+0x0A, 0x92, 0x02, 0x6D, 0xE0, 0xF1, 0x1F, 0x6C,
+0xD2, 0xF4, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x0C, 0x97, 0xA9, 0xA7, 0x48, 0xA7, 0x8A, 0xA7,
+0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA7, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x85, 0xA2, 0xA6, 0xA2, 0xC4, 0xA2,
+0x47, 0xA2, 0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE,
+0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x49, 0xA2, 0x00, 0x6C, 0xD1, 0x18,
+0x90, 0x1F, 0x0C, 0xD2, 0xD1, 0x18, 0xCD, 0x8A,
+0x00, 0x65, 0x04, 0x22, 0x00, 0x6D, 0xB1, 0x18,
+0x61, 0xED, 0x00, 0x6C, 0x14, 0x92, 0x10, 0x72,
+0x58, 0x67, 0x0A, 0xD2, 0x03, 0x22, 0xD1, 0x18,
+0x41, 0x21, 0x0C, 0x94, 0x0B, 0x92, 0x01, 0x6C,
+0x42, 0xA2, 0x8C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA,
+0x60, 0xF1, 0x04, 0x22, 0xD1, 0x18, 0x4E, 0x23,
+0x00, 0x65, 0xD2, 0xF4, 0x54, 0x99, 0x31, 0xF1,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x93,
+0x03, 0x6D, 0x4D, 0xED, 0xD2, 0xF4, 0xC8, 0x9B,
+0x31, 0xF1, 0x00, 0x6C, 0x40, 0xEE, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x80, 0x34, 0x80, 0x34, 0x40, 0x32, 0x29, 0xF1,
+0x64, 0x9C, 0x40, 0x32, 0xF2, 0xF4, 0x44, 0x9A,
+0x83, 0x67, 0x40, 0xEA, 0x0B, 0xD3, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xD2, 0xF4,
+0xD8, 0x9C, 0xF7, 0xF0, 0x01, 0x6C, 0x05, 0xF0,
+0x01, 0x6D, 0x80, 0x34, 0xAB, 0xED, 0x80, 0x34,
+0x0B, 0x93, 0xAC, 0xEA, 0x49, 0xF3, 0xB4, 0x9C,
+0x83, 0x67, 0x40, 0xEE, 0x4D, 0xED, 0xD1, 0x18,
+0x20, 0x23, 0x00, 0x65, 0xD2, 0xF4, 0x54, 0x99,
+0x39, 0xF6, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x34, 0x20, 0xF0, 0x40, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x20, 0xF0, 0x43, 0xC0, 0xD2, 0xF4,
+0x54, 0x99, 0x20, 0xF0, 0x81, 0xC0, 0x82, 0x34,
+0x20, 0xF0, 0x82, 0xC0, 0x39, 0xF6, 0x08, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x42, 0x34, 0x20, 0xF0,
+0x44, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x20, 0xF0,
+0x47, 0xC0, 0xD2, 0xF4, 0x54, 0x99, 0x20, 0xF0,
+0x85, 0xC0, 0x82, 0x34, 0x20, 0xF0, 0x86, 0xC0,
+0x39, 0xF6, 0x0C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x34, 0x20, 0xF0, 0x48, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x20, 0xF0, 0x4B, 0xC0, 0xD2, 0xF4,
+0x54, 0x99, 0x20, 0xF0, 0x89, 0xC0, 0x82, 0x34,
+0x20, 0xF0, 0x8A, 0xC0, 0xD8, 0xF2, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x42, 0x34, 0x20, 0xF0,
+0x4C, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x20, 0xF0,
+0x4F, 0xC0, 0xD2, 0xF4, 0x54, 0x99, 0x20, 0xF0,
+0x8D, 0xC0, 0x82, 0x34, 0x20, 0xF0, 0x8E, 0xC0,
+0xD8, 0xF2, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x34, 0x20, 0xF0, 0x50, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x20, 0xF0, 0x53, 0xC0, 0xD2, 0xF4,
+0x54, 0x99, 0x20, 0xF0, 0x91, 0xC0, 0x82, 0x34,
+0x20, 0xF0, 0x92, 0xC0, 0xD8, 0xF2, 0x08, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x42, 0x34, 0x20, 0xF0,
+0x54, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x20, 0xF0,
+0x57, 0xC0, 0xD2, 0xF4, 0x54, 0x99, 0x20, 0xF0,
+0x95, 0xC0, 0x82, 0x34, 0x20, 0xF0, 0x96, 0xC0,
+0xD8, 0xF2, 0x0C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x34, 0x20, 0xF0, 0x58, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x20, 0xF0, 0x5B, 0xC0, 0xD2, 0xF4,
+0x54, 0x99, 0x20, 0xF0, 0x99, 0xC0, 0x82, 0x34,
+0x20, 0xF0, 0x9A, 0xC0, 0xD8, 0xF2, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x42, 0x34, 0x20, 0xF0,
+0x5C, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x20, 0xF0,
+0x5F, 0xC0, 0xD2, 0xF4, 0x54, 0x99, 0x20, 0xF0,
+0x9D, 0xC0, 0x82, 0x34, 0x20, 0xF0, 0x9E, 0xC0,
+0xD8, 0xF2, 0x14, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x34, 0x40, 0xF0, 0x40, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x40, 0xF0, 0x43, 0xC0, 0xD2, 0xF4,
+0x54, 0x99, 0x40, 0xF0, 0x81, 0xC0, 0x82, 0x34,
+0x40, 0xF0, 0x82, 0xC0, 0xD8, 0xF2, 0x18, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x42, 0x34, 0x40, 0xF0,
+0x44, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x40, 0xF0,
+0x47, 0xC0, 0xD2, 0xF4, 0x54, 0x99, 0x40, 0xF0,
+0x85, 0xC0, 0x82, 0x34, 0x40, 0xF0, 0x86, 0xC0,
+0xD8, 0xF2, 0x1C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x34, 0x40, 0xF0, 0x48, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x40, 0xF0, 0x4B, 0xC0, 0x08, 0x92,
+0x40, 0xF0, 0x89, 0xC0, 0x82, 0x34, 0xD2, 0xF4,
+0x48, 0x9A, 0x40, 0xF0, 0x8A, 0xC0, 0x39, 0xF6,
+0x04, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x08, 0x92,
+0x00, 0x6D, 0x39, 0xF6, 0x08, 0x6C, 0xD2, 0xF4,
+0x48, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x92,
+0x00, 0x6D, 0x39, 0xF6, 0x0C, 0x6C, 0xD2, 0xF4,
+0x48, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x92,
+0x01, 0x6D, 0xAB, 0xED, 0xD2, 0xF4, 0x48, 0x9A,
+0xD8, 0xF2, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x08, 0x92, 0x01, 0x6D, 0xAB, 0xED, 0xD2, 0xF4,
+0x48, 0x9A, 0xD8, 0xF2, 0x04, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x08, 0x92, 0x01, 0x6D, 0xAB, 0xED,
+0xD2, 0xF4, 0x48, 0x9A, 0xD8, 0xF2, 0x08, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x08, 0x92, 0x01, 0x6D,
+0xAB, 0xED, 0xD2, 0xF4, 0x48, 0x9A, 0xD8, 0xF2,
+0x0C, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x92,
+0x01, 0x6D, 0xAB, 0xED, 0xD2, 0xF4, 0x48, 0x9A,
+0xD8, 0xF2, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x08, 0x92, 0x01, 0x6D, 0xAB, 0xED, 0xD2, 0xF4,
+0x48, 0x9A, 0xD8, 0xF2, 0x14, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x08, 0x92, 0x01, 0x6D, 0xAB, 0xED,
+0xD2, 0xF4, 0x48, 0x9A, 0xD8, 0xF2, 0x18, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x08, 0x92, 0x01, 0x6D,
+0xAB, 0xED, 0xD2, 0xF4, 0x48, 0x9A, 0xD8, 0xF2,
+0x1C, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x11, 0x6A,
+0x14, 0xD2, 0x0A, 0x92, 0x03, 0x22, 0x02, 0xF0,
+0x12, 0x6A, 0x14, 0xD2, 0x09, 0x93, 0x09, 0x92,
+0x15, 0xF0, 0x89, 0xA3, 0x15, 0xF0, 0x48, 0xA2,
+0x80, 0x34, 0x4D, 0xEC, 0x43, 0x67, 0x15, 0xF0,
+0x6A, 0xA3, 0x15, 0xF0, 0x4B, 0xA2, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x61, 0xA2, 0x20, 0xF0,
+0x00, 0xA2, 0x20, 0xF0, 0x82, 0xA2, 0x60, 0x33,
+0x0D, 0xEB, 0x20, 0xF0, 0x03, 0xA2, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30,
+0xD1, 0x18, 0xC7, 0x18, 0x6D, 0xE8, 0x09, 0x93,
+0x41, 0xC0, 0x09, 0x92, 0x15, 0xF0, 0x89, 0xA3,
+0x15, 0xF0, 0x48, 0xA2, 0x80, 0x34, 0x4D, 0xEC,
+0x43, 0x67, 0x15, 0xF0, 0x6A, 0xA3, 0x15, 0xF0,
+0x4B, 0xA2, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x61, 0xA2, 0x20, 0xF0, 0x00, 0xA2, 0x20, 0xF0,
+0x82, 0xA2, 0x60, 0x33, 0x0D, 0xEB, 0x20, 0xF0,
+0x03, 0xA2, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x00, 0x30, 0xD1, 0x18, 0xB8, 0x18,
+0x6D, 0xE8, 0x40, 0xC0, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0x8A, 0x6C, 0x00, 0x6C, 0x00, 0x6C,
+0x01, 0x6E, 0xD1, 0x18, 0x8A, 0x6C, 0x00, 0x6D,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF0, 0x48, 0x9A, 0x14, 0x94, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x54, 0x99, 0xFF, 0x6C,
+0x40, 0xEA, 0x69, 0x4C, 0x01, 0x72, 0x34, 0x61,
+0xD2, 0xF4, 0x54, 0x99, 0xE0, 0xF1, 0x1C, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C,
+0x08, 0x93, 0x80, 0x34, 0x80, 0x34, 0xA9, 0xF1,
+0xA4, 0x9C, 0xD2, 0xF4, 0x68, 0x9B, 0xE0, 0xF1,
+0x1C, 0x6C, 0x40, 0xEB, 0x4D, 0xED, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x13, 0xF0,
+0x5C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xA2, 0x15,
+0x24, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0A, 0xF2,
+0x14, 0x4A, 0x05, 0xD2, 0x40, 0xF4, 0x1C, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x04, 0x04,
+0x40, 0xEA, 0x0A, 0xD7, 0x0A, 0x97, 0xCD, 0x15,
+0x0D, 0x92, 0x3E, 0xF4, 0x04, 0x6C, 0x93, 0xF3,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x4C, 0x9A, 0x01, 0x6C, 0xB4, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x89, 0xA3, 0x48, 0xA3,
+0xAA, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x85, 0xA2, 0x64, 0xA2,
+0xA6, 0xA2, 0x80, 0x34, 0xF7, 0xF0, 0x01, 0x68,
+0x6D, 0xEC, 0x00, 0x30, 0x67, 0xA2, 0xA0, 0x35,
+0x00, 0x30, 0xA0, 0x35, 0xD2, 0xF4, 0x54, 0x98,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEA, 0x0A, 0xD3,
+0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x69,
+0x80, 0x34, 0x80, 0x34, 0x20, 0x31, 0x49, 0xF3,
+0xB8, 0x9C, 0x20, 0x31, 0xD2, 0xF4, 0xC8, 0x99,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEE, 0x4D, 0xED,
+0x0A, 0x93, 0x0A, 0xD1, 0x4B, 0xA3, 0x02, 0x72,
+0x36, 0x61, 0x03, 0x6A, 0x4B, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x40, 0x9A, 0xE0, 0xF1, 0x1F, 0x6C, 0x40, 0xEA,
+0x03, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF5, 0x40, 0x9A, 0x00, 0x6D,
+0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x69, 0x40, 0xEA,
+0x20, 0x31, 0x20, 0x31, 0x32, 0x2A, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xF3, 0xF0,
+0x10, 0x9C, 0xFF, 0x6F, 0x04, 0xD2, 0x2B, 0x4F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xE8,
+0x00, 0x6D, 0xD1, 0x18, 0xF1, 0x1F, 0x00, 0x6C,
+0x13, 0xF0, 0x48, 0x99, 0x02, 0xF0, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0x24, 0x6A,
+0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0A, 0xF2, 0x14, 0x4A,
+0x07, 0xD2, 0xE0, 0xF4, 0x04, 0x6A, 0x08, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04,
+0xC0, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x33, 0xF5, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD1, 0x18, 0xF0, 0x20, 0x00, 0x65,
+0x13, 0xF0, 0x48, 0x99, 0x02, 0xF0, 0x01, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4, 0x54, 0x98,
+0xFF, 0x6C, 0x40, 0xEA, 0x69, 0x4C, 0x02, 0x72,
+0x1C, 0x61, 0xD2, 0xF4, 0x54, 0x98, 0xE0, 0xF1,
+0x1C, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6C, 0x0A, 0x93, 0x80, 0x34, 0x80, 0x34,
+0x09, 0xF3, 0xB4, 0x9C, 0xD2, 0xF4, 0x68, 0x9B,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEB, 0x4D, 0xED,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF0, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xB1, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF3, 0x50, 0x9A, 0x3E, 0xF4,
+0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x4C, 0x9A, 0x01, 0x6C, 0xC9, 0x17, 0x00, 0x65,
+0xF5, 0x63, 0x15, 0x62, 0x14, 0xD1, 0x13, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x15, 0xF0, 0x08, 0x49, 0x8D, 0xA1, 0x6C, 0xA1,
+0x4E, 0xA1, 0x80, 0x34, 0x6D, 0xEC, 0x6F, 0xA1,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x20, 0xF0, 0x91, 0xA3,
+0x20, 0xF0, 0x50, 0xA3, 0x20, 0xF0, 0xB2, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x20, 0xF0, 0x53, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x81, 0xA2, 0xA0, 0xA2,
+0x62, 0xA2, 0x80, 0x34, 0x8D, 0xED, 0x83, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0xAD, 0xEB, 0x00, 0xF6,
+0x80, 0x34, 0x6D, 0xEC, 0x0F, 0xD4, 0x65, 0xA2,
+0x04, 0xA2, 0x86, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x07, 0xA2, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8, 0x69, 0xA1,
+0x48, 0xA1, 0x8A, 0xA1, 0x60, 0x33, 0x4D, 0xEB,
+0x4B, 0xA1, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0x65, 0xA2,
+0x84, 0xA2, 0x60, 0x33, 0x8D, 0xEB, 0x86, 0xA2,
+0x47, 0xA2, 0x80, 0x34, 0x80, 0x34, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEB, 0x4D, 0xEB, 0x01, 0x6C,
+0xD1, 0x18, 0x2C, 0x21, 0x0B, 0xD3, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0xD2, 0xF4,
+0x54, 0x9E, 0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEA,
+0x0C, 0xD6, 0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x6F, 0x80, 0x34, 0x80, 0x34, 0xE0, 0x37,
+0xE0, 0x37, 0x09, 0xF3, 0xB0, 0x9C, 0xD2, 0xF4,
+0x68, 0x9F, 0xE0, 0xF1, 0x1C, 0x6C, 0x4D, 0xED,
+0x40, 0xEB, 0x0A, 0xD7, 0x0B, 0x93, 0x03, 0x6C,
+0x4B, 0xA3, 0x8E, 0xEA, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x0B, 0xD4, 0x36, 0x2A,
+0x4B, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x40, 0x9A, 0xE0, 0xF1,
+0x1F, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x33, 0xF5,
+0x5C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x01, 0x72,
+0x37, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x00, 0x6B,
+0xFF, 0x6F, 0x04, 0xD3, 0x2B, 0x4F, 0x02, 0x6E,
+0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D,
+0xD1, 0x18, 0xF1, 0x1F, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x13, 0xF0,
+0x48, 0x9A, 0x02, 0xF0, 0x10, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x15, 0x97, 0x14, 0x91, 0x13, 0x90,
+0x00, 0xEF, 0x0B, 0x63, 0x24, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0A, 0xF2, 0x14, 0x4A, 0x07, 0xD2,
+0x20, 0xF5, 0x05, 0x6A, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04, 0xBF, 0x17,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0xD2, 0xF4, 0x40, 0x9F, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0x0D, 0xD2, 0xD2, 0xF4,
+0x4C, 0x9E, 0x11, 0xF2, 0x00, 0x6C, 0x0E, 0xD7,
+0x40, 0xEA, 0x11, 0xD6, 0x02, 0x6D, 0xAB, 0xED,
+0x4C, 0xED, 0x0D, 0x92, 0xFF, 0x6B, 0x6C, 0xED,
+0x11, 0xF2, 0x00, 0x6C, 0x40, 0xEA, 0x10, 0xD3,
+0x0E, 0x97, 0x11, 0x96, 0x11, 0xF2, 0x00, 0x6C,
+0xD2, 0xF4, 0x40, 0x9F, 0x0D, 0xD2, 0xD2, 0xF4,
+0x4C, 0x9E, 0x40, 0xEA, 0x00, 0x65, 0x10, 0x93,
+0x01, 0x6D, 0x4D, 0xED, 0x0D, 0x92, 0x11, 0xF2,
+0x00, 0x6C, 0x40, 0xEA, 0x6C, 0xED, 0x0E, 0x97,
+0xC0, 0xF7, 0x11, 0x6B, 0x10, 0xD7, 0xFF, 0x4B,
+0xFF, 0xF7, 0x1F, 0x6A, 0x4C, 0xEB, 0x12, 0x2B,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xFF, 0x6F, 0x04, 0xD3,
+0x2C, 0x4F, 0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x02, 0x6A, 0x0E, 0xD2,
+0x01, 0x6A, 0x17, 0x10, 0x0C, 0x92, 0x13, 0xF6,
+0x10, 0x6C, 0x0D, 0xD3, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x00, 0x52, 0x0A, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0x01, 0x6C,
+0x0D, 0x93, 0xD5, 0x17, 0x0F, 0x6A, 0x0E, 0xD2,
+0x00, 0x6A, 0x0D, 0xD2, 0xC0, 0xF7, 0x11, 0x6B,
+0xFF, 0x4B, 0xFF, 0xF7, 0x1F, 0x6A, 0x4C, 0xEB,
+0x40, 0xF2, 0x12, 0x2B, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0xFF, 0x6F, 0x04, 0xD3, 0x2D, 0x4F, 0x02, 0x6E,
+0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D,
+0x03, 0x6A, 0x0E, 0xD2, 0x01, 0x6A, 0x0D, 0xD2,
+0xC0, 0xF7, 0x11, 0x6B, 0xFF, 0x4B, 0xFF, 0xF7,
+0x1F, 0x6A, 0x4C, 0xEB, 0x40, 0xF2, 0x0F, 0x2B,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xFF, 0x6F, 0x04, 0xD3,
+0x2E, 0x4F, 0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x04, 0x6A, 0x0E, 0xD2,
+0x01, 0x6A, 0x0D, 0xD2, 0x0F, 0x92, 0x01, 0x6B,
+0x42, 0xA2, 0x6C, 0xEA, 0xE0, 0xF0, 0x12, 0x22,
+0x0A, 0x92, 0x20, 0xF0, 0xA1, 0xA0, 0x39, 0xF6,
+0x04, 0x6C, 0xD2, 0xF4, 0xC8, 0x9A, 0x20, 0xF0,
+0x40, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0,
+0x42, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xF0, 0xA3, 0xA0, 0x00, 0xF6, 0xA0, 0x35,
+0x40, 0xEE, 0x4D, 0xED, 0x0A, 0x92, 0x20, 0xF0,
+0xA5, 0xA0, 0x39, 0xF6, 0x08, 0x6C, 0xD2, 0xF4,
+0xC8, 0x9A, 0x20, 0xF0, 0x44, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x46, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0, 0xA7, 0xA0,
+0x00, 0xF6, 0xA0, 0x35, 0x40, 0xEE, 0x4D, 0xED,
+0x0A, 0x92, 0x20, 0xF0, 0xA9, 0xA0, 0x39, 0xF6,
+0x0C, 0x6C, 0xD2, 0xF4, 0xC8, 0x9A, 0x20, 0xF0,
+0x48, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0,
+0x4A, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xF0, 0xAB, 0xA0, 0x00, 0xF6, 0xA0, 0x35,
+0x40, 0xEE, 0x4D, 0xED, 0x0A, 0x92, 0x20, 0xF0,
+0xAD, 0xA0, 0xD8, 0xF2, 0x00, 0x6C, 0xD2, 0xF4,
+0xC8, 0x9A, 0x20, 0xF0, 0x4C, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x4E, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0, 0xAF, 0xA0,
+0x00, 0xF6, 0xA0, 0x35, 0x40, 0xEE, 0x4D, 0xED,
+0x0A, 0x92, 0x20, 0xF0, 0xB1, 0xA0, 0xD8, 0xF2,
+0x04, 0x6C, 0xD2, 0xF4, 0xC8, 0x9A, 0x20, 0xF0,
+0x50, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0,
+0x52, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xF0, 0xB3, 0xA0, 0x00, 0xF6, 0xA0, 0x35,
+0x40, 0xEE, 0x4D, 0xED, 0x0A, 0x92, 0x20, 0xF0,
+0xB5, 0xA0, 0xD8, 0xF2, 0x08, 0x6C, 0xD2, 0xF4,
+0xC8, 0x9A, 0x20, 0xF0, 0x54, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x56, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0, 0xB7, 0xA0,
+0x00, 0xF6, 0xA0, 0x35, 0x40, 0xEE, 0x4D, 0xED,
+0x0A, 0x92, 0x20, 0xF0, 0xB9, 0xA0, 0xD8, 0xF2,
+0x0C, 0x6C, 0xD2, 0xF4, 0xC8, 0x9A, 0x20, 0xF0,
+0x58, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0,
+0x5A, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xF0, 0xBB, 0xA0, 0x00, 0xF6, 0xA0, 0x35,
+0x40, 0xEE, 0x4D, 0xED, 0x0A, 0x92, 0x20, 0xF0,
+0xBD, 0xA0, 0xD8, 0xF2, 0x10, 0x6C, 0xD2, 0xF4,
+0xC8, 0x9A, 0x20, 0xF0, 0x5C, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x20, 0xF0, 0x5E, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0, 0xBF, 0xA0,
+0x00, 0xF6, 0xA0, 0x35, 0x40, 0xEE, 0x4D, 0xED,
+0x0A, 0x92, 0x40, 0xF0, 0xA1, 0xA0, 0xD8, 0xF2,
+0x14, 0x6C, 0xD2, 0xF4, 0xC8, 0x9A, 0x40, 0xF0,
+0x40, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x40, 0xF0,
+0x42, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0x40, 0xF0, 0xA3, 0xA0, 0x00, 0xF6, 0xA0, 0x35,
+0x40, 0xEE, 0x4D, 0xED, 0x0A, 0x92, 0x40, 0xF0,
+0xA5, 0xA0, 0xD8, 0xF2, 0x18, 0x6C, 0xD2, 0xF4,
+0xC8, 0x9A, 0x40, 0xF0, 0x44, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x40, 0xF0, 0x46, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x40, 0xF0, 0xA7, 0xA0,
+0x00, 0xF6, 0xA0, 0x35, 0x40, 0xEE, 0x4D, 0xED,
+0x0A, 0x92, 0x40, 0xF0, 0xA9, 0xA0, 0xD8, 0xF2,
+0x1C, 0x6C, 0xD2, 0xF4, 0xC8, 0x9A, 0x40, 0xF0,
+0x48, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x40, 0xF0,
+0x4A, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0x40, 0xF0, 0xAB, 0xA0, 0x00, 0xF6, 0xA0, 0x35,
+0x40, 0xEE, 0x4D, 0xED, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0x8A, 0x6C, 0x01, 0x6C, 0x01, 0x6E,
+0x00, 0x6D, 0xD1, 0x18, 0x8A, 0x6C, 0x01, 0x6C,
+0x0B, 0x93, 0x0B, 0x92, 0x15, 0xF0, 0x89, 0xA3,
+0x15, 0xF0, 0x48, 0xA2, 0x80, 0x34, 0x4D, 0xEC,
+0x43, 0x67, 0x15, 0xF0, 0x6A, 0xA3, 0x15, 0xF0,
+0x4B, 0xA2, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x61, 0xA2, 0x20, 0xF0, 0x82, 0xA2, 0x20, 0xF0,
+0xA0, 0xA2, 0x20, 0xF0, 0x43, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0xD1, 0x18,
+0xD0, 0x18, 0x81, 0xA2, 0x0B, 0x93, 0x0B, 0x92,
+0x15, 0xF0, 0x89, 0xA3, 0x15, 0xF0, 0x48, 0xA2,
+0x80, 0x34, 0x4D, 0xEC, 0x43, 0x67, 0x15, 0xF0,
+0x6A, 0xA3, 0x15, 0xF0, 0x4B, 0xA2, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x61, 0xA2, 0x20, 0xF0,
+0x82, 0xA2, 0x20, 0xF0, 0xA0, 0xA2, 0x20, 0xF0,
+0x43, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xED,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0xD1, 0x18, 0xC0, 0x18, 0x80, 0xA2,
+0x0A, 0x92, 0xE3, 0xF7, 0x1F, 0x6D, 0x53, 0xF0,
+0x04, 0x6C, 0xD2, 0xF4, 0x48, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xC9, 0xF2, 0xB0, 0x9B, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0x29, 0xF3,
+0x94, 0x9B, 0xD2, 0xF4, 0x58, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xD1, 0x18, 0x90, 0x1F, 0x01, 0x6C,
+0xD1, 0x18, 0xF1, 0x1F, 0x0E, 0x94, 0x10, 0x92,
+0x0B, 0x93, 0xD2, 0xF4, 0xC0, 0x9A, 0x0B, 0x92,
+0x15, 0xF0, 0x89, 0xA3, 0x15, 0xF0, 0x48, 0xA2,
+0x80, 0x34, 0x4D, 0xEC, 0x43, 0x67, 0x15, 0xF0,
+0x6A, 0xA3, 0x15, 0xF0, 0x4B, 0xA2, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x61, 0xA2, 0x20, 0xF0,
+0x82, 0xA2, 0x20, 0xF0, 0xA0, 0xA2, 0x20, 0xF0,
+0x43, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xED,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x30, 0xF4, 0x06, 0x6C, 0x40, 0xEE,
+0xA2, 0xA2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xF1, 0x48, 0x9A, 0x13, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF5, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x6C,
+0x0A, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x04, 0xF0,
+0x00, 0x6D, 0x40, 0xEA, 0x02, 0x6C, 0xD1, 0x18,
+0x52, 0x95, 0x00, 0x65, 0x8D, 0xA1, 0x4C, 0xA1,
+0x6E, 0xA1, 0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA1,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x79, 0xA2, 0x9A, 0xA2,
+0xB8, 0xA2, 0x5B, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x01, 0x6B, 0xE0, 0xF0,
+0x70, 0xC2, 0xF7, 0xF0, 0x01, 0x6A, 0x0D, 0x93,
+0x40, 0x32, 0x40, 0x32, 0x13, 0xF0, 0x48, 0x9A,
+0x79, 0x2B, 0x02, 0xF0, 0x02, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x89, 0xA1, 0x48, 0xA1, 0x6A, 0xA1,
+0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA1, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x65, 0xA2, 0x86, 0xA2, 0xA4, 0xA2,
+0x47, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xED,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0xD1, 0x18, 0xAD, 0x21, 0x89, 0xA2,
+0x0A, 0x92, 0xE0, 0xF1, 0x10, 0x6C, 0xD2, 0xF4,
+0x08, 0x9A, 0x0C, 0x92, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x89, 0xF1, 0xB8, 0x9B,
+0xE0, 0xF1, 0x10, 0x6C, 0xAC, 0xEA, 0x10, 0xF1,
+0x00, 0x6D, 0x40, 0xE8, 0x4D, 0xED, 0x0C, 0x92,
+0xE0, 0xF1, 0x1C, 0x6C, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C,
+0x0A, 0x93, 0x80, 0x34, 0x80, 0x34, 0xC9, 0xF2,
+0xA4, 0x9C, 0xD2, 0xF4, 0x68, 0x9B, 0xE0, 0xF1,
+0x1C, 0x6C, 0x40, 0xEB, 0x4D, 0xED, 0x29, 0x15,
+0x0C, 0x92, 0x11, 0xF5, 0x00, 0x6C, 0x11, 0xD3,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x03, 0x6C, 0x8C, 0xEA, 0x03, 0x72, 0xBF, 0xF5,
+0x13, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA,
+0x01, 0x6C, 0x11, 0x93, 0x91, 0x15, 0x0C, 0x92,
+0x12, 0xF1, 0x00, 0x6C, 0x11, 0xD3, 0xD2, 0xF4,
+0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x03, 0x6C,
+0x8C, 0xEA, 0x03, 0x72, 0xBF, 0xF5, 0x16, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0x01, 0x6C,
+0x11, 0x93, 0x94, 0x15, 0x02, 0xF0, 0x10, 0x6C,
+0x86, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x00, 0x68, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x53, 0xF1, 0x40, 0x9A,
+0xB0, 0x67, 0x40, 0xEA, 0x00, 0x6C, 0xFF, 0x72,
+0x37, 0x60, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x15, 0xF0, 0x08, 0x4C, 0xA5, 0xA4,
+0x64, 0xA4, 0xC6, 0xA4, 0xA0, 0x35, 0x6D, 0xED,
+0x67, 0xA4, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0x20, 0xF0,
+0x81, 0xA3, 0x20, 0xF0, 0xA2, 0xA3, 0x20, 0xF0,
+0xC0, 0xA3, 0x20, 0xF0, 0x63, 0xA3, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0x48, 0x32,
+0x69, 0xE2, 0x65, 0xA2, 0x86, 0xA2, 0xA4, 0xA2,
+0x47, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xED,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x64, 0xA2, 0x03, 0x6C, 0x8C, 0xEB,
+0x03, 0x23, 0x00, 0x6B, 0x71, 0xC2, 0x72, 0xC2,
+0x01, 0x48, 0xFF, 0x6A, 0x4C, 0xE8, 0x05, 0x70,
+0xB9, 0x61, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A, 0xAD, 0xA2,
+0x6C, 0xA2, 0xCE, 0xA2, 0xA0, 0x35, 0x6D, 0xED,
+0x6F, 0xA2, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0x20, 0xF0,
+0xB1, 0xA3, 0x20, 0xF0, 0x50, 0xA3, 0x20, 0xF0,
+0xD2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x20, 0xF0,
+0x53, 0xA3, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0xA1, 0xA2,
+0x60, 0xA2, 0xC2, 0xA2, 0xA0, 0x35, 0x6D, 0xED,
+0x63, 0xA2, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0x00, 0x6A, 0xAD, 0xEB,
+0x05, 0x5C, 0xBD, 0x67, 0x08, 0xD2, 0x20, 0xF0,
+0x44, 0xC5, 0x04, 0xD2, 0x05, 0xD2, 0x06, 0xD2,
+0x4E, 0xCD, 0x16, 0x60, 0x03, 0x2C, 0x48, 0xA3,
+0x01, 0x4A, 0x48, 0xC3, 0x10, 0x6A, 0x7D, 0x67,
+0x20, 0xF0, 0x41, 0xC3, 0x00, 0x6A, 0x20, 0xF0,
+0x42, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x08, 0x04, 0x06, 0x22, 0xFF, 0x6B, 0x6C, 0xEA,
+0x0B, 0x97, 0x0A, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0x7D, 0x67, 0x20, 0xF0, 0xA3, 0xA3, 0x50, 0xC3,
+0x20, 0xF0, 0x64, 0xA3, 0xFF, 0x68, 0x60, 0x33,
+0xAD, 0xEB, 0x83, 0x67, 0xBD, 0x67, 0x0C, 0xEC,
+0x93, 0xC5, 0x62, 0x33, 0x95, 0xC5, 0x9D, 0x67,
+0x74, 0xC5, 0x76, 0xC4, 0x01, 0x6B, 0x71, 0xC4,
+0x7D, 0x67, 0x52, 0xC3, 0x07, 0x6A, 0x5A, 0xC3,
+0x04, 0x6A, 0x5B, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF1, 0x54, 0x9A,
+0x04, 0x05, 0x40, 0xEA, 0x05, 0x6C, 0x62, 0x67,
+0x01, 0x6A, 0xD6, 0x23, 0x43, 0x67, 0x0C, 0xEA,
+0xD3, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x60, 0x9A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x4C, 0x9A, 0x30, 0xF4, 0x05, 0x68,
+0x90, 0x67, 0x40, 0xEA, 0x04, 0xD3, 0x04, 0x93,
+0xA1, 0x42, 0xFF, 0x69, 0x2C, 0xED, 0x40, 0xEB,
+0x90, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A, 0x89, 0xA2,
+0x68, 0xA2, 0xAA, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x6B, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x85, 0xA3,
+0x46, 0xA3, 0xA4, 0xA3, 0x67, 0xA3, 0x80, 0x34,
+0x40, 0x32, 0x8D, 0xED, 0x40, 0x32, 0xAD, 0xEA,
+0x00, 0xF6, 0x60, 0x33, 0x4D, 0xEB, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x80, 0x6C, 0x40, 0xEA, 0x04, 0xD3,
+0x80, 0xF7, 0x42, 0x30, 0x04, 0x93, 0x01, 0x6A,
+0x4C, 0xE8, 0x2C, 0xE8, 0x09, 0xC3, 0xD1, 0x18,
+0x41, 0x21, 0x90, 0x67, 0xD1, 0x18, 0xB8, 0x18,
+0x00, 0x65, 0xD1, 0x18, 0xC7, 0x18, 0x04, 0xD2,
+0x00, 0x6E, 0x00, 0x6D, 0x00, 0x6C, 0xD1, 0x18,
+0x8A, 0x6C, 0x22, 0x67, 0xD1, 0x18, 0x17, 0x8E,
+0x20, 0x6C, 0x01, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0x8A, 0x6C, 0x00, 0x6C, 0xD1, 0x18, 0x17, 0x8E,
+0x20, 0x6C, 0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0x8A, 0x6C, 0x01, 0x6C, 0xD1, 0x18, 0x17, 0x8E,
+0x20, 0x6C, 0x01, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0x8A, 0x6C, 0x01, 0x6C, 0xD1, 0x18, 0x17, 0x8E,
+0x20, 0x6C, 0xD1, 0x18, 0xAD, 0x21, 0x90, 0x67,
+0x04, 0x93, 0xD1, 0x18, 0xC0, 0x18, 0x83, 0x67,
+0xD1, 0x18, 0xD0, 0x18, 0x91, 0x67, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x49, 0xF3,
+0xBC, 0x9B, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD2, 0xF4, 0x58, 0x9A, 0x29, 0xF3,
+0x94, 0x9B, 0x40, 0xEA, 0x00, 0x65, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0xF2, 0x63, 0x1B, 0x62, 0x1A, 0xD1, 0x19, 0xD0,
+0x00, 0x6A, 0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x15, 0xF0,
+0x08, 0x4A, 0x09, 0xD2, 0x09, 0x93, 0x48, 0xA2,
+0xF7, 0xF0, 0x01, 0x68, 0x89, 0xA3, 0x00, 0x30,
+0x00, 0x30, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0x67,
+0x6A, 0xA3, 0x4B, 0xA2, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x65, 0xA2, 0x24, 0xA2, 0x86, 0xA2, 0x60, 0x33,
+0x2D, 0xEB, 0x27, 0xA2, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEB, 0x00, 0xF6, 0x20, 0x31, 0x6D, 0xE9,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x97, 0xF0,
+0x19, 0x6C, 0x60, 0x33, 0x80, 0x34, 0xD2, 0xF4,
+0x48, 0x9B, 0x80, 0x34, 0x74, 0xF1, 0x01, 0x4C,
+0xA4, 0x67, 0x11, 0xD4, 0xE0, 0xF1, 0x1C, 0x6C,
+0x40, 0xEA, 0x0A, 0xD3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x01, 0x6D, 0x40, 0x32, 0x8C, 0xF2,
+0xA8, 0xC2, 0x5D, 0x67, 0x20, 0xF0, 0xA0, 0xC2,
+0xD2, 0xF4, 0x54, 0x98, 0x40, 0xEA, 0x80, 0x6C,
+0x7D, 0x67, 0x20, 0xF0, 0xA0, 0xA3, 0x80, 0xF7,
+0x42, 0x32, 0x3E, 0xF4, 0x04, 0x6C, 0xAC, 0xEA,
+0x10, 0xD2, 0x5D, 0x67, 0x40, 0xF0, 0x40, 0xA2,
+0x49, 0xC1, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF3, 0x50, 0x9A, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x40, 0xEA, 0x20, 0x31,
+0xD2, 0xF4, 0x4C, 0x99, 0x78, 0xF1, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x12, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x70, 0x9A, 0x00, 0x6C, 0xFF, 0x6F, 0x04, 0xD4,
+0x32, 0x4F, 0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C,
+0x00, 0x6D, 0x40, 0xEB, 0x08, 0xD2, 0x12, 0x92,
+0x10, 0x72, 0x08, 0x92, 0x0F, 0xD2, 0x74, 0x61,
+0xD2, 0xF4, 0x54, 0x98, 0xF9, 0xF4, 0x1C, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x69, 0xF1, 0x6C, 0x9B,
+0x6C, 0xEA, 0x66, 0x22, 0xD2, 0xF4, 0x54, 0x98,
+0xF8, 0xF6, 0x14, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x0A, 0x93, 0xFF, 0x6D, 0x01, 0x4D, 0xD2, 0xF4,
+0x68, 0x9B, 0xF8, 0xF6, 0x14, 0x6C, 0x40, 0xEB,
+0x4D, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x40, 0x9A, 0xF8, 0xF6,
+0x14, 0x6C, 0x40, 0xEA, 0x10, 0x6D, 0xD2, 0xF4,
+0x54, 0x98, 0xF8, 0xF6, 0x10, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x01, 0x6B, 0x6C, 0xEA, 0x40, 0x22,
+0xD2, 0xF4, 0x54, 0x98, 0xF9, 0xF2, 0x08, 0x6C,
+0x40, 0xEA, 0x0F, 0x69, 0x2C, 0xEA, 0x2F, 0x2A,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xF7, 0xF0, 0x01, 0x6A, 0x49, 0xF3, 0xBC, 0x9B,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD2, 0xF4, 0x58, 0x9A,
+0x29, 0xF3, 0x94, 0x9B, 0x40, 0xEA, 0x00, 0x65,
+0x0A, 0x92, 0xE0, 0xF1, 0x10, 0x6C, 0xD2, 0xF4,
+0x28, 0x9A, 0xD2, 0xF4, 0x54, 0x98, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x89, 0xF1, 0xB8, 0x9B, 0xE0, 0xF1,
+0x10, 0x6C, 0xAC, 0xEA, 0x10, 0xF1, 0x00, 0x6D,
+0x40, 0xE9, 0x4D, 0xED, 0x1B, 0x97, 0x1A, 0x91,
+0x19, 0x90, 0x00, 0xEF, 0x0E, 0x63, 0xD2, 0xF4,
+0x54, 0x98, 0xF9, 0xF2, 0x08, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x2C, 0xEA, 0x01, 0x72, 0xC8, 0x60,
+0xD1, 0x18, 0x0D, 0x28, 0x00, 0x65, 0xD8, 0x17,
+0xD1, 0x18, 0xA5, 0x27, 0x00, 0x65, 0x00, 0x6E,
+0x00, 0x6D, 0xD1, 0x18, 0x0C, 0x20, 0x00, 0x6C,
+0xD2, 0xF4, 0x4C, 0x99, 0xD8, 0xF6, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6B, 0x4C, 0xEB,
+0xD1, 0x18, 0xCD, 0x8A, 0x13, 0xD3, 0x04, 0x22,
+0x00, 0x6D, 0xB1, 0x18, 0x61, 0xED, 0x00, 0x6C,
+0x10, 0x94, 0xF7, 0xF0, 0x01, 0x69, 0xD1, 0x18,
+0x41, 0x21, 0x20, 0x31, 0xD1, 0x18, 0xB8, 0x18,
+0x20, 0x31, 0xD1, 0x18, 0xC7, 0x18, 0x14, 0xD2,
+0x00, 0x6E, 0x00, 0x6D, 0x00, 0x6C, 0xD1, 0x18,
+0x8A, 0x6C, 0x15, 0xD2, 0x00, 0x6D, 0x01, 0x6E,
+0xD1, 0x18, 0x8A, 0x6C, 0x00, 0x6C, 0xD2, 0xF4,
+0x50, 0x99, 0x32, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6C,
+0x4C, 0xEB, 0xFF, 0xF7, 0x1F, 0x6D, 0x80, 0x34,
+0x80, 0x34, 0xAC, 0xEB, 0x0C, 0xD1, 0x0D, 0xD4,
+0x2C, 0x23, 0xD2, 0xF4, 0x64, 0x9C, 0x10, 0x6C,
+0x8D, 0xEA, 0x32, 0xF6, 0x00, 0x6C, 0x40, 0xEB,
+0x4C, 0xED, 0xC9, 0x69, 0xFF, 0x49, 0x0F, 0x29,
+0x0F, 0x92, 0xFF, 0x6F, 0x04, 0xD1, 0xF3, 0xF0,
+0x50, 0x9A, 0x33, 0x4F, 0x02, 0x6E, 0x02, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x00, 0x6A,
+0x08, 0xD2, 0x01, 0x6A, 0x15, 0x10, 0xD2, 0xF4,
+0x54, 0x98, 0x32, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x20, 0x6B, 0x6C, 0xEA, 0x09, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0x01, 0x6C,
+0xDD, 0x17, 0x0F, 0x6A, 0x08, 0xD2, 0x00, 0x6A,
+0x0B, 0xD2, 0x0C, 0x92, 0xF8, 0xF6, 0x00, 0x6C,
+0xFF, 0xF7, 0x1F, 0x69, 0xD2, 0xF4, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x0D, 0x93, 0x03, 0x6D,
+0xAB, 0xED, 0xD2, 0xF4, 0x64, 0x9B, 0x4C, 0xED,
+0xF8, 0xF6, 0x00, 0x6C, 0x40, 0xEB, 0x2C, 0xED,
+0x0C, 0x92, 0x19, 0xF0, 0x00, 0x6C, 0xD2, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x0D, 0x93,
+0x20, 0x6D, 0x4D, 0xED, 0xD2, 0xF4, 0x64, 0x9B,
+0x19, 0xF0, 0x00, 0x6C, 0x40, 0xEB, 0x2C, 0xED,
+0xC9, 0x69, 0xFF, 0x49, 0x27, 0x29, 0x0F, 0x92,
+0xFF, 0x6F, 0x04, 0xD1, 0xF3, 0xF0, 0x50, 0x9A,
+0x34, 0x4F, 0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x6D, 0x08, 0x92, 0x0F, 0x72,
+0x01, 0x6A, 0x0B, 0xD2, 0x02, 0x61, 0x01, 0x6A,
+0x08, 0xD2, 0x00, 0x6B, 0x04, 0x73, 0x5C, 0x60,
+0x05, 0x5B, 0x24, 0x60, 0x02, 0x73, 0x33, 0x60,
+0x03, 0x5B, 0x4E, 0x60, 0x01, 0x73, 0x40, 0xF2,
+0x1A, 0x60, 0x02, 0xF0, 0x00, 0x6A, 0x00, 0x69,
+0x0E, 0xD2, 0x18, 0xF0, 0x00, 0x6D, 0x19, 0xF0,
+0x14, 0x6C, 0x2A, 0x10, 0x0C, 0x92, 0x19, 0xF0,
+0x00, 0x6C, 0xD2, 0xF4, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xC3, 0xF7, 0x00, 0x6B, 0x6C, 0xEA,
+0xE0, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA,
+0x01, 0x6C, 0xC3, 0x17, 0x06, 0x73, 0x4C, 0x60,
+0x06, 0x5B, 0x3B, 0x61, 0x07, 0x73, 0x00, 0xF2,
+0x12, 0x60, 0x08, 0x73, 0xDA, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x10, 0x6D,
+0x09, 0xF1, 0x3C, 0x9A, 0x0B, 0x12, 0x00, 0x69,
+0x18, 0xF0, 0x00, 0x6D, 0x19, 0xF0, 0x18, 0x6C,
+0xD2, 0xF4, 0x54, 0x98, 0x17, 0xD5, 0x40, 0xEA,
+0x16, 0xD3, 0x17, 0x95, 0xC2, 0x67, 0x16, 0x93,
+0xAC, 0xEE, 0xCE, 0xED, 0x05, 0x25, 0x0E, 0x94,
+0x0E, 0x95, 0x4C, 0xEC, 0xAE, 0xEC, 0x3F, 0x2C,
+0xE1, 0xF7, 0x1F, 0x6C, 0x84, 0xE9, 0x4C, 0xEC,
+0x29, 0x21, 0x82, 0x32, 0x42, 0x32, 0x28, 0x10,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x10, 0x69, 0x49, 0xF3, 0xBC, 0x9A, 0xDE, 0x17,
+0x02, 0xF0, 0x00, 0x6A, 0x00, 0x69, 0x0E, 0xD2,
+0x18, 0xF0, 0x00, 0x6D, 0x19, 0xF0, 0x1C, 0x6C,
+0xD7, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xC9, 0xF1, 0x40, 0x9A, 0x10, 0x69,
+0x0E, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x49, 0xF3, 0xBC, 0x9A, 0xEE, 0x17,
+0x00, 0x69, 0x18, 0xF0, 0x00, 0x6D, 0x39, 0xF0,
+0x00, 0x6C, 0xC2, 0x17, 0x80, 0x32, 0x40, 0x32,
+0xA3, 0x67, 0x4D, 0xEC, 0xD1, 0x18, 0x7E, 0x22,
+0x23, 0x67, 0x01, 0x4A, 0x71, 0x67, 0x07, 0x2A,
+0x08, 0x92, 0x0F, 0x72, 0x01, 0x6A, 0x0B, 0xD2,
+0x02, 0x61, 0x02, 0x6A, 0x08, 0xD2, 0x01, 0x4B,
+0xFF, 0xF7, 0x1F, 0x69, 0x2C, 0xEB, 0x09, 0x73,
+0x7F, 0xF7, 0x0C, 0x61, 0xD1, 0x18, 0x16, 0x21,
+0x00, 0x6C, 0xD1, 0x18, 0xAA, 0x1E, 0x00, 0x65,
+0x0C, 0x92, 0xF8, 0xF6, 0x00, 0x6C, 0xD2, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x0D, 0x93,
+0x02, 0x6D, 0x4D, 0xED, 0xD2, 0xF4, 0x64, 0x9B,
+0xF8, 0xF6, 0x00, 0x6C, 0x40, 0xEB, 0x2C, 0xED,
+0x0C, 0x92, 0xF8, 0xF6, 0x00, 0x6C, 0xD2, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x0D, 0x93,
+0x01, 0x6D, 0x4D, 0xED, 0xD2, 0xF4, 0x64, 0x9B,
+0xF8, 0xF6, 0x00, 0x6C, 0x40, 0xEB, 0x2C, 0xED,
+0xD2, 0xF4, 0x54, 0x98, 0x18, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x0A, 0x93, 0x10, 0x6D,
+0x4D, 0xED, 0xD2, 0xF4, 0x68, 0x9B, 0x18, 0xF0,
+0x00, 0x6C, 0x40, 0xEB, 0x00, 0x65, 0x0C, 0x92,
+0x19, 0xF0, 0x00, 0x6C, 0xD2, 0xF4, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x0D, 0x93, 0x21, 0x6D,
+0xAB, 0xED, 0xD2, 0xF4, 0x64, 0x9B, 0x4C, 0xED,
+0x19, 0xF0, 0x00, 0x6C, 0x40, 0xEB, 0x2C, 0xED,
+0xC9, 0x69, 0xFF, 0x49, 0xA0, 0xF0, 0x18, 0x29,
+0x0F, 0x92, 0xFF, 0x6F, 0x04, 0xD1, 0xF3, 0xF0,
+0x50, 0x9A, 0x35, 0x4F, 0x02, 0x6E, 0x02, 0xF0,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0x13, 0x92,
+0x07, 0x22, 0x08, 0x92, 0x0F, 0x72, 0x01, 0x6A,
+0x0B, 0xD2, 0x02, 0x61, 0x03, 0x6A, 0x08, 0xD2,
+0xD2, 0xF4, 0x54, 0x98, 0x52, 0xF0, 0x18, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xE1, 0xF7, 0x1F, 0x6D,
+0x06, 0x06, 0x4C, 0xED, 0xD1, 0x18, 0x0C, 0x1F,
+0x00, 0x6C, 0x0C, 0x92, 0x32, 0xF6, 0x00, 0x6C,
+0xD2, 0xF4, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x0D, 0x93, 0xFF, 0xF7, 0x0F, 0x6D, 0x4C, 0xED,
+0xD2, 0xF4, 0x64, 0x9B, 0x32, 0xF6, 0x00, 0x6C,
+0x40, 0xEB, 0x00, 0x65, 0x00, 0x6E, 0x00, 0x6D,
+0xD1, 0x18, 0x8A, 0x6C, 0x01, 0x6C, 0x01, 0x6E,
+0x00, 0x6D, 0xD1, 0x18, 0x8A, 0x6C, 0x01, 0x6C,
+0xD1, 0x18, 0xC0, 0x18, 0x14, 0x94, 0xD1, 0x18,
+0xD0, 0x18, 0x15, 0x94, 0x0A, 0x92, 0xE3, 0xF7,
+0x1F, 0x6D, 0x53, 0xF0, 0x04, 0x6C, 0xD2, 0xF4,
+0x48, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x54, 0x98, 0x13, 0xF7, 0x1C, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xA9, 0xF2, 0xBC, 0x9B, 0x82, 0x67,
+0xAC, 0xEC, 0x08, 0x24, 0x0A, 0x93, 0x4D, 0xED,
+0x13, 0xF7, 0x1C, 0x6C, 0xD2, 0xF4, 0xC8, 0x9B,
+0x40, 0xEE, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x49, 0xF3, 0xBC, 0x9B,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x29, 0xF3, 0x94, 0x9B, 0xD2, 0xF4, 0x58, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x5D, 0x67, 0x58, 0xA2,
+0x08, 0x93, 0x01, 0x72, 0x98, 0x67, 0x0F, 0x73,
+0x78, 0x67, 0x52, 0x2C, 0x04, 0x6A, 0x54, 0x23,
+0xD1, 0x18, 0xF1, 0x1F, 0x08, 0x94, 0x0B, 0x91,
+0x01, 0x6A, 0x4E, 0xE9, 0x0E, 0x29, 0x0F, 0x92,
+0x08, 0x93, 0xFF, 0x6F, 0xF3, 0xF0, 0x50, 0x9A,
+0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3, 0x36, 0x4F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x0A, 0x92, 0x11, 0x95, 0xE0, 0xF1,
+0x1C, 0x6C, 0xD2, 0xF4, 0x48, 0x9A, 0x40, 0xEA,
+0x02, 0x4D, 0x5D, 0x67, 0x58, 0xA2, 0x03, 0x6C,
+0x8B, 0xEC, 0x1A, 0x65, 0x58, 0x67, 0x8C, 0xEA,
+0xFF, 0x6D, 0xAC, 0xEA, 0x01, 0x72, 0x01, 0x60,
+0x5F, 0x29, 0xD1, 0x18, 0x04, 0x20, 0x12, 0x94,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x33, 0xF0, 0x44, 0x9A, 0x40, 0xEA, 0x10, 0x6C,
+0xB1, 0x15, 0x42, 0x34, 0x82, 0x34, 0xE8, 0x16,
+0xD2, 0xF4, 0x54, 0x98, 0xF8, 0xF6, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x69, 0xF1, 0x6C, 0x9B,
+0x6C, 0xEA, 0x5F, 0xF7, 0x0D, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x48, 0x9A, 0x40, 0xEA, 0x01, 0x6C, 0x2D, 0x17,
+0x03, 0x72, 0x04, 0x61, 0xAD, 0x2B, 0x05, 0x6A,
+0x08, 0xD2, 0xAA, 0x17, 0xA9, 0x2B, 0x09, 0x93,
+0x09, 0x92, 0x89, 0xA3, 0x48, 0xA2, 0x80, 0x34,
+0x4D, 0xEC, 0x43, 0x67, 0x6A, 0xA3, 0x4B, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2, 0x86, 0xA2,
+0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x48, 0xA2, 0x03, 0x6B,
+0x6E, 0xEA, 0x8A, 0x2A, 0x7D, 0x67, 0x00, 0xF0,
+0x19, 0x04, 0xB1, 0x18, 0x97, 0xE6, 0x59, 0xC3,
+0x7D, 0x67, 0x79, 0xA3, 0x01, 0x73, 0x80, 0x61,
+0x01, 0x72, 0x7F, 0xF7, 0x1D, 0x61, 0x06, 0x6A,
+0x08, 0xD2, 0x01, 0x6A, 0x0B, 0xD2, 0x78, 0x17,
+0x09, 0x92, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x6C, 0xA2, 0x4D, 0xA2, 0x20, 0x31, 0x40, 0x32,
+0x6D, 0xEA, 0x09, 0x93, 0xCE, 0xA3, 0x6F, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x4D, 0xEB, 0x20, 0xF0, 0xD1, 0xA3,
+0x20, 0xF0, 0x50, 0xA3, 0x20, 0xF0, 0xF2, 0xA3,
+0xC0, 0x36, 0x4D, 0xEE, 0x20, 0xF0, 0x53, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x61, 0xA2, 0xC2, 0xA2,
+0xE0, 0xA2, 0x43, 0xA2, 0x60, 0x33, 0xC0, 0x36,
+0x6D, 0xEF, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0xC2, 0xA2, 0x02, 0x6B,
+0xCC, 0xEB, 0x6C, 0xED, 0x16, 0x25, 0xCC, 0xEC,
+0x82, 0xC2, 0x0A, 0x92, 0xE0, 0xF1, 0x10, 0x6C,
+0xD2, 0xF4, 0x68, 0x9A, 0xD2, 0xF4, 0x54, 0x98,
+0x40, 0xEA, 0x08, 0xD3, 0x89, 0xF1, 0xB8, 0x99,
+0x08, 0x93, 0xE0, 0xF1, 0x10, 0x6C, 0xAC, 0xEA,
+0x10, 0xF1, 0x00, 0x6D, 0x40, 0xEB, 0x4D, 0xED,
+0x15, 0x15, 0xD1, 0x18, 0xAD, 0x21, 0x10, 0x94,
+0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65, 0x04, 0x22,
+0x01, 0x6D, 0xB1, 0x18, 0x61, 0xED, 0x00, 0x6C,
+0x01, 0x6E, 0xCB, 0xEE, 0xA6, 0x67, 0xD1, 0x18,
+0x0C, 0x20, 0x86, 0x67, 0xDA, 0x17, 0x00, 0x6D,
+0x10, 0xF0, 0x00, 0x69, 0xD2, 0xF4, 0x54, 0x98,
+0x39, 0xF0, 0x00, 0x6C, 0x17, 0xD5, 0x40, 0xEA,
+0x16, 0xD3, 0x00, 0x52, 0x16, 0x93, 0x17, 0x95,
+0x3F, 0xF6, 0x1D, 0x61, 0xD2, 0xF4, 0x54, 0x98,
+0x39, 0xF0, 0x04, 0x6C, 0x16, 0xD3, 0x40, 0xEA,
+0x17, 0xD5, 0x4C, 0xE9, 0x16, 0x93, 0x3F, 0xF6,
+0x12, 0x21, 0x17, 0x95, 0xE1, 0xF7, 0x1F, 0x6C,
+0x84, 0xED, 0x8C, 0xEA, 0x3F, 0xF7, 0x11, 0x2D,
+0x40, 0x34, 0x80, 0x34, 0x19, 0x16, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xC9, 0xF1,
+0x40, 0x9A, 0x10, 0x69, 0x0E, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x49, 0xF3,
+0xBC, 0x9A, 0x9D, 0x15, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0xD2, 0xF4, 0x54, 0x98,
+0x19, 0xF0, 0x00, 0x6C, 0x27, 0x67, 0x05, 0xD5,
+0x40, 0xEA, 0x04, 0xD6, 0xD2, 0xF4, 0x54, 0x98,
+0x19, 0xF0, 0x1C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xD2, 0xF4, 0x54, 0x98, 0x19, 0xF0, 0x14, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x05, 0x95, 0x04, 0x96,
+0x00, 0x6A, 0x40, 0xC5, 0x40, 0xC6, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x8C, 0xF2,
+0x44, 0x9A, 0x40, 0xC1, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x8C, 0xF2, 0x60, 0x9A,
+0x0E, 0x92, 0x60, 0xC2, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xC8, 0x68, 0x24, 0x67, 0xFF, 0x6A, 0xFF, 0x48,
+0x4C, 0xE8, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x1B, 0x28, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xF3, 0xF0, 0x10, 0x9B,
+0xD2, 0xF4, 0x54, 0x9A, 0xFF, 0x6C, 0x40, 0xEA,
+0x6D, 0x4C, 0x05, 0xD2, 0xFF, 0x6F, 0x02, 0x6A,
+0x06, 0xD1, 0x04, 0xD2, 0x37, 0x4F, 0x02, 0x6E,
+0x02, 0xF0, 0x00, 0x6C, 0x40, 0xE8, 0x00, 0x6D,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0xD2, 0xF4, 0x54, 0x9A, 0xFF, 0x6C,
+0x40, 0xEA, 0x65, 0x4C, 0x09, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x48, 0x9A, 0x40, 0xEA, 0x0A, 0x6C, 0xCE, 0x17,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xD2, 0xF4, 0x48, 0x98, 0xFF, 0x6C, 0xB1, 0x67,
+0x40, 0xEA, 0x6D, 0x4C, 0xD2, 0xF4, 0x48, 0x98,
+0xFF, 0x6C, 0x01, 0x6D, 0x40, 0xEA, 0x65, 0x4C,
+0xDB, 0x17, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x15, 0xF0, 0x08, 0x49,
+0xA9, 0xA1, 0x48, 0xA1, 0x8A, 0xA1, 0xA0, 0x35,
+0x4D, 0xED, 0x4B, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x85, 0xA2, 0x04, 0xA2, 0xA6, 0xA2, 0x80, 0x34,
+0x0D, 0xEC, 0x07, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x00, 0x30, 0xAD, 0xA1,
+0x8D, 0xE8, 0x8C, 0xA1, 0x4E, 0xA1, 0xA0, 0x35,
+0x8D, 0xED, 0x8F, 0xA1, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC,
+0x20, 0xF0, 0xB1, 0xA4, 0x20, 0xF0, 0x50, 0xA4,
+0x20, 0xF0, 0xD2, 0xA4, 0xA0, 0x35, 0x4D, 0xED,
+0x20, 0xF0, 0x53, 0xA4, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x81, 0xA2, 0x60, 0xA2, 0xA2, 0xA2, 0x80, 0x34,
+0x43, 0xA2, 0x6D, 0xEC, 0xA0, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x06, 0xD2, 0x8B, 0xA0, 0x01, 0x6A, 0x4E, 0xEC,
+0x32, 0x2C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x08, 0xD4, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0xA0, 0x35, 0x12, 0xF2, 0xA8, 0x9D, 0x40, 0xED,
+0x07, 0xD2, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0x08, 0x94, 0xA0, 0x35, 0xF3, 0xF0, 0x70, 0x9D,
+0xFF, 0x6F, 0x00, 0x6D, 0x04, 0xD4, 0x38, 0x4F,
+0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEB, 0x02, 0x6E,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x92, 0xF2, 0x98, 0x9C, 0x40, 0xEC, 0x00, 0x65,
+0x02, 0x6C, 0x8B, 0xC0, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x07, 0x92, 0x72, 0xF0,
+0xB4, 0x9C, 0x40, 0xED, 0x82, 0x67, 0x4B, 0xA0,
+0x03, 0x72, 0xC7, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x07, 0xD2, 0x06, 0x92,
+0x02, 0x6C, 0x42, 0xA2, 0x4C, 0xEC, 0x5A, 0x24,
+0x03, 0x6C, 0x06, 0x93, 0x8B, 0xEC, 0x8C, 0xEA,
+0x42, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF2, 0x48, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x70, 0x9A, 0xFF, 0x6F,
+0x00, 0x6A, 0x04, 0xD2, 0x39, 0x4F, 0x02, 0x6E,
+0x02, 0xF0, 0x00, 0x6C, 0x00, 0x6D, 0x40, 0xEB,
+0x08, 0xD2, 0xA9, 0xA1, 0x88, 0xA1, 0xCA, 0xA1,
+0xA0, 0x35, 0x8D, 0xED, 0x8B, 0xA1, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34,
+0xAD, 0xEC, 0xA5, 0xA4, 0xC6, 0xA4, 0xE4, 0xA4,
+0x87, 0xA4, 0xA0, 0x35, 0xC0, 0x36, 0x08, 0x92,
+0xAD, 0xEF, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0xCD, 0xEC, 0x4B, 0xC4, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x92, 0xF2,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x44, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x07, 0x94, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xB2, 0xF2, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x6C, 0x5F, 0x17, 0x01, 0x6E, 0xCB, 0xEE,
+0xA6, 0x67, 0xD1, 0x18, 0x0C, 0x20, 0x86, 0x67,
+0xA9, 0xA1, 0x48, 0xA1, 0x8A, 0xA1, 0xA0, 0x35,
+0x4D, 0xED, 0x4B, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x85, 0xA2, 0xA6, 0xA2, 0xC4, 0xA2, 0x47, 0xA2,
+0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0xD1, 0x18, 0xAD, 0x21, 0x89, 0xA2, 0xD1, 0x18,
+0xCD, 0x8A, 0x00, 0x65, 0x86, 0x22, 0x01, 0x6D,
+0xB1, 0x18, 0x61, 0xED, 0x00, 0x6C, 0x81, 0x17,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x24, 0x67, 0x15, 0xF0, 0x08, 0x48, 0x9F, 0x67,
+0x07, 0x62, 0xD1, 0x18, 0xE9, 0xF7, 0x00, 0x65,
+0x85, 0xA0, 0x44, 0xA0, 0x66, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x99, 0xA2, 0x78, 0xA2, 0xDA, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x7B, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x81, 0xA3, 0x40, 0xA3, 0xC2, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x43, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x20, 0xF0, 0x89, 0xA2, 0x20, 0xF0, 0x68, 0xA2,
+0x20, 0xF0, 0xCA, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x20, 0xF0, 0x6B, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0xD1, 0x18, 0x6C, 0x5D, 0x06, 0xD3, 0x51, 0x67,
+0x01, 0x6E, 0xB1, 0x67, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0xCC, 0xEA, 0x20, 0x31, 0x06, 0x93,
+0x48, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x68, 0x40, 0x32, 0x00, 0x30, 0x40, 0x32,
+0x00, 0x30, 0xD2, 0xF4, 0x68, 0x9A, 0xD2, 0xF4,
+0x54, 0x98, 0xE0, 0xF1, 0x10, 0x6C, 0x07, 0xD6,
+0x40, 0xEA, 0x06, 0xD3, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x89, 0xF1, 0xB8, 0x9C,
+0x06, 0x93, 0xE0, 0xF1, 0x10, 0x6C, 0xAC, 0xEA,
+0x02, 0xF1, 0x00, 0x6D, 0x40, 0xEB, 0x4D, 0xED,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x54, 0x9A, 0x40, 0xEA, 0x1C, 0x6C,
+0xF3, 0xF0, 0x30, 0x99, 0xD2, 0xF4, 0x54, 0x98,
+0x30, 0xF5, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x07, 0x96, 0xFF, 0x6F, 0x05, 0xD2, 0x04, 0xD6,
+0x02, 0xF0, 0x00, 0x6C, 0x3A, 0x4F, 0x02, 0x6E,
+0x40, 0xE9, 0x00, 0x6D, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x33, 0xF0, 0x44, 0x9A,
+0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0x02, 0x6A, 0x4C, 0xED, 0x00, 0x6A,
+0x06, 0xD2, 0x5A, 0x25, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x48, 0x9A,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0x06, 0xD2, 0xD2, 0xF4, 0x54, 0x9F, 0xE0, 0xF1,
+0x10, 0x6C, 0x09, 0xD3, 0x0A, 0xD6, 0x40, 0xEA,
+0x08, 0xD7, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x89, 0xF1, 0xB8, 0x9C, 0xE0, 0xF1,
+0x10, 0x6C, 0xAC, 0xEA, 0x02, 0xF1, 0x00, 0x6D,
+0x4D, 0xED, 0x06, 0x92, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x54, 0x9A, 0x40, 0xEA, 0x1D, 0x6C,
+0xF3, 0xF0, 0x50, 0x99, 0x08, 0x97, 0x78, 0xF1,
+0x04, 0x6C, 0x06, 0xD2, 0xD2, 0xF4, 0x54, 0x9F,
+0x40, 0xEA, 0x00, 0x65, 0x0A, 0x96, 0x05, 0xD2,
+0x06, 0x92, 0xFF, 0x6F, 0x3B, 0x4F, 0x00, 0x6D,
+0x04, 0xD6, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA,
+0x02, 0x6E, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0xC0, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x4C, 0x9A, 0x30, 0xF4, 0x04, 0x6C, 0x40, 0xEA,
+0x06, 0xD6, 0x06, 0x96, 0xA1, 0x42, 0xFF, 0x6C,
+0x8C, 0xED, 0x30, 0xF4, 0x04, 0x6C, 0x40, 0xEE,
+0x00, 0x65, 0x09, 0x93, 0x01, 0x6A, 0x06, 0xD2,
+0x8D, 0xA0, 0xAC, 0xA0, 0xCE, 0xA0, 0x80, 0x34,
+0xAD, 0xEC, 0xAF, 0xA0, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0xA0, 0x35, 0x8D, 0xED,
+0x20, 0xF0, 0xD1, 0xA5, 0x20, 0xF0, 0x90, 0xA5,
+0x20, 0xF0, 0xF2, 0xA5, 0xC0, 0x36, 0x8D, 0xEE,
+0x20, 0xF0, 0x93, 0xA5, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0x80, 0x34, 0xCD, 0xEC,
+0xC1, 0xA4, 0xA2, 0xA4, 0xE0, 0xA4, 0x83, 0xA4,
+0xC0, 0x36, 0xA0, 0x35, 0xCD, 0xEF, 0xA0, 0x35,
+0x00, 0xF6, 0x80, 0x34, 0xED, 0xED, 0x8D, 0xED,
+0x80, 0xA5, 0x02, 0x74, 0x0E, 0x60, 0x03, 0x5C,
+0x05, 0x60, 0x01, 0x6A, 0x01, 0x24, 0x00, 0x6A,
+0x01, 0x68, 0x0B, 0x10, 0x03, 0x74, 0x66, 0x60,
+0x04, 0x74, 0xF9, 0x61, 0x02, 0x6C, 0x80, 0xC5,
+0xF6, 0x17, 0x03, 0x6C, 0x80, 0xC5, 0x01, 0x6A,
+0x00, 0x68, 0x40, 0xF0, 0x82, 0xA3, 0x40, 0xF0,
+0xC1, 0xA3, 0xA1, 0xF3, 0x18, 0x6D, 0x80, 0x34,
+0xCD, 0xEC, 0xAE, 0xEC, 0x05, 0x2C, 0x40, 0xF0,
+0x81, 0xC3, 0x40, 0xF0, 0x82, 0xC3, 0x01, 0x6A,
+0xF3, 0xF0, 0x70, 0x99, 0x01, 0x6C, 0xFF, 0x6F,
+0x22, 0x67, 0x04, 0xD4, 0x05, 0xD2, 0x3D, 0x4F,
+0x02, 0x6E, 0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEB,
+0x00, 0x6D, 0x33, 0x21, 0x06, 0x92, 0x31, 0x22,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0xD1, 0x18,
+0x4E, 0x23, 0x20, 0x31, 0xD2, 0xF4, 0x40, 0x99,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEA, 0x00, 0x6D,
+0xD2, 0xF4, 0x40, 0x99, 0xE0, 0xF1, 0x1D, 0x6C,
+0x40, 0xEA, 0x01, 0x6D, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x13, 0xF0, 0x48, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x01, 0x6B, 0x6C, 0xEA, 0x19, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x48, 0x9A, 0xE0, 0xF1, 0x1C, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x08, 0x20, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x13, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xD1, 0x18, 0x10, 0xF8,
+0x07, 0x94, 0x02, 0x17, 0x00, 0x6A, 0xA0, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x4C, 0x9A, 0x40, 0xEA, 0xC8, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF3, 0x50, 0x9A, 0x3E, 0xF4, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xC7, 0x17, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xA5, 0xA3, 0x44, 0xA3,
+0xC6, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0, 0xA1, 0xA2,
+0x20, 0xF0, 0x62, 0xA2, 0x20, 0xF0, 0xC0, 0xA2,
+0x20, 0xF0, 0x43, 0xA2, 0xA0, 0x35, 0x60, 0x33,
+0xAD, 0xEE, 0x60, 0x33, 0xCD, 0xEB, 0x00, 0xF6,
+0x40, 0x32, 0x6D, 0xEA, 0x60, 0xA4, 0x20, 0xA2,
+0x00, 0x6F, 0x3B, 0x65, 0x71, 0x67, 0x67, 0xEF,
+0x01, 0x6D, 0xAC, 0xEB, 0x23, 0x23, 0xE8, 0x36,
+0xD9, 0xE2, 0xA5, 0xA6, 0x64, 0xA6, 0x06, 0xA6,
+0xA0, 0x35, 0x6D, 0xED, 0x67, 0xA6, 0x00, 0x30,
+0x00, 0x30, 0x0D, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0xA0, 0xA3, 0xD9, 0x67, 0xCE, 0xED,
+0x11, 0x2D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A, 0xA4, 0x67,
+0x87, 0x43, 0x08, 0x6E, 0x40, 0xEA, 0x3D, 0x4C,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0x6A,
+0x00, 0xEF, 0x04, 0x63, 0x01, 0x4F, 0x08, 0x77,
+0x78, 0x67, 0xD4, 0x2B, 0xF5, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x88, 0xA4, 0x00, 0x6E,
+0xD1, 0x18, 0xD4, 0xD9, 0x00, 0x6D, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x21, 0x9C, 0xAD, 0xF0,
+0x94, 0xA0, 0x04, 0x06, 0xB1, 0x18, 0xA3, 0xF7,
+0x00, 0x6D, 0x67, 0x42, 0x09, 0x4B, 0xFF, 0x6C,
+0x8C, 0xEB, 0x02, 0x5B, 0x1D, 0x61, 0x20, 0xF0,
+0x61, 0xA1, 0x20, 0xF0, 0x40, 0xC1, 0x5D, 0x67,
+0x50, 0xA2, 0x01, 0x6C, 0x8D, 0xEB, 0x0F, 0x6C,
+0x4C, 0xEC, 0x1F, 0x6A, 0x4B, 0xEA, 0x84, 0x34,
+0x6C, 0xEA, 0x8D, 0xEA, 0xAD, 0xF0, 0x14, 0x48,
+0x20, 0xF0, 0x41, 0xC1, 0x58, 0xA0, 0x01, 0x6B,
+0x41, 0x6C, 0x4C, 0xEB, 0x40, 0xA1, 0x8B, 0xEC,
+0x78, 0x33, 0x8C, 0xEA, 0x6D, 0xEA, 0x40, 0xC1,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF,
+0x07, 0x63, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x6A,
+0x8F, 0x63, 0xE1, 0x62, 0xE0, 0xD1, 0xDF, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x68,
+0x60, 0x33, 0x00, 0x30, 0x60, 0x33, 0x69, 0xF3,
+0x60, 0x9B, 0x00, 0x30, 0x72, 0xF4, 0x54, 0x98,
+0xA3, 0x67, 0x14, 0x6C, 0x40, 0xEA, 0xDC, 0xD3,
+0x19, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4A, 0xF2, 0x00, 0x4A, 0x05, 0xD2, 0xA7, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0xE1, 0x97, 0xE0, 0x91, 0xDF, 0x90,
+0x00, 0xEF, 0x71, 0x63, 0x22, 0x67, 0xDC, 0x95,
+0x72, 0xF4, 0x54, 0x98, 0x40, 0xEA, 0x08, 0x6C,
+0x42, 0x34, 0x81, 0xC1, 0x82, 0x34, 0x82, 0xC1,
+0x00, 0xF6, 0x42, 0x34, 0x40, 0xC1, 0x83, 0xC1,
+0x0C, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4A, 0xF2, 0x00, 0x4A, 0x05, 0xD2, 0xB3, 0x6A,
+0xD7, 0x17, 0x40, 0xDA, 0x41, 0xDA, 0xDC, 0x95,
+0x72, 0xF4, 0x54, 0x98, 0x40, 0xF3, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x42, 0x34, 0x89, 0xC1,
+0x82, 0x34, 0x8A, 0xC1, 0x00, 0xF6, 0x42, 0x34,
+0x48, 0xC1, 0x8B, 0xC1, 0x44, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4A, 0xF2, 0x00, 0x4A,
+0x05, 0xD2, 0xC2, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0xF7, 0xF0,
+0x01, 0x68, 0x00, 0x30, 0x00, 0x30, 0x0D, 0xF1,
+0x58, 0x98, 0x0A, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF3, 0x54, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0x0D, 0xF1, 0x58, 0xD8,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x15, 0xF0, 0x08, 0x4C, 0xA5, 0xA4, 0x44, 0xA4,
+0xC6, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA4,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x22, 0x34, 0x20, 0xF0,
+0x89, 0xC2, 0x00, 0xF6, 0x22, 0x33, 0x82, 0x34,
+0x20, 0xF0, 0x28, 0xC2, 0x20, 0xF0, 0x8A, 0xC2,
+0x20, 0xF0, 0x6B, 0xC2, 0x8A, 0x17, 0x40, 0xF3,
+0x10, 0x6E, 0x00, 0x6D, 0xD1, 0x18, 0xD2, 0xF4,
+0x04, 0x04, 0x72, 0xF4, 0x54, 0x98, 0xDC, 0x95,
+0x40, 0xEA, 0x1C, 0x6C, 0x02, 0x67, 0x28, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x89, 0xA1, 0x72, 0xF4, 0xD0, 0x9A, 0x48, 0xA1,
+0x80, 0x34, 0x4D, 0xEC, 0x4A, 0xA1, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x8B, 0xA1, 0x00, 0xF6,
+0x80, 0x34, 0x40, 0xEE, 0x4D, 0xEC, 0x06, 0x6A,
+0x7D, 0x67, 0x60, 0xF3, 0x40, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x4A, 0xF2,
+0x00, 0x4A, 0xD9, 0xD2, 0xD4, 0x6A, 0xDA, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0xD8, 0x04,
+0xA9, 0xA1, 0xE8, 0xA1, 0x4A, 0xA1, 0xA0, 0x35,
+0xED, 0xED, 0xEB, 0xA1, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x00, 0xF6, 0xE0, 0x37, 0xAD, 0xEF,
+0x87, 0x67, 0x40, 0xF3, 0x10, 0x6E, 0x04, 0x05,
+0xD1, 0x18, 0xE7, 0xF4, 0x04, 0xD0, 0xE2, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x6D, 0xF1, 0xEC, 0xDA, 0x80, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x40, 0xA4, 0x64, 0x67, 0xF7, 0xF0, 0x01, 0x69,
+0x02, 0x6C, 0xF7, 0xF0, 0x01, 0x68, 0x20, 0x31,
+0x00, 0x30, 0x8C, 0xEA, 0x20, 0x31, 0x00, 0x30,
+0x04, 0xD3, 0x27, 0x22, 0xD2, 0xF4, 0x54, 0x99,
+0x11, 0xF0, 0x14, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xD2, 0xF4, 0xC8, 0x98, 0x01, 0x6D, 0x4D, 0xED,
+0x11, 0xF0, 0x14, 0x6C, 0x40, 0xEE, 0x00, 0x65,
+0x04, 0x93, 0x40, 0xA3, 0x02, 0x6B, 0x6C, 0xEA,
+0x20, 0x22, 0xD2, 0xF4, 0x54, 0x99, 0x11, 0xF1,
+0x18, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x68, 0x98, 0x01, 0x6D, 0x4D, 0xED, 0x11, 0xF1,
+0x18, 0x6C, 0x40, 0xEB, 0x00, 0x65, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0x6A, 0x00, 0xEF,
+0x05, 0x63, 0xD2, 0xF4, 0x54, 0x99, 0x11, 0xF0,
+0x14, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x02, 0x6D,
+0xAB, 0xED, 0xD2, 0xF4, 0xC8, 0x98, 0x4C, 0xED,
+0xD7, 0x17, 0xD2, 0xF4, 0x54, 0x99, 0x11, 0xF1,
+0x18, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x02, 0x6D,
+0xAB, 0xED, 0xD2, 0xF4, 0x68, 0x98, 0x4C, 0xED,
+0xDE, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x40, 0xA4, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x68, 0x6C, 0xEA, 0xFF, 0x6B,
+0x6C, 0xEA, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x00, 0x30, 0x24, 0x67, 0x60, 0x33, 0x00, 0x30,
+0x5C, 0x22, 0xD2, 0xF4, 0x54, 0x9B, 0x53, 0xF4,
+0x00, 0x6C, 0x40, 0xEA, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x69, 0xF1,
+0xDC, 0x9C, 0x81, 0xA1, 0xE0, 0xA1, 0x4C, 0xEE,
+0x03, 0x6D, 0x44, 0x67, 0xF7, 0xF0, 0x01, 0x6B,
+0xAC, 0xEA, 0x60, 0x33, 0xE6, 0x37, 0x60, 0x33,
+0x5C, 0x32, 0xED, 0xEA, 0x69, 0xF3, 0xE4, 0x9B,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xEC, 0xEA, 0xE9, 0xF1,
+0xE8, 0x9B, 0x8A, 0x33, 0x82, 0xA1, 0xCD, 0xEA,
+0xEC, 0xEA, 0x98, 0x34, 0x8D, 0xEB, 0x83, 0xA1,
+0xD2, 0xF4, 0xE8, 0x98, 0x8C, 0xED, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xA0, 0x35,
+0x69, 0xF3, 0x88, 0x9C, 0xB8, 0x35, 0x6D, 0xED,
+0x8D, 0xED, 0x53, 0xF4, 0x00, 0x6C, 0x40, 0xEF,
+0x4D, 0xED, 0x04, 0x93, 0x53, 0xF4, 0x04, 0x6C,
+0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x69, 0xF3, 0xAC, 0x9B, 0x63, 0xA1, 0x03, 0x6C,
+0x4C, 0xED, 0x6A, 0x33, 0xD2, 0xF4, 0x48, 0x98,
+0x8C, 0xEB, 0x40, 0xF4, 0x60, 0x33, 0x53, 0xF4,
+0x04, 0x6C, 0x40, 0xEA, 0x6D, 0xED, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0x6A, 0x00, 0xEF,
+0x05, 0x63, 0xD2, 0xF4, 0x54, 0x9B, 0x53, 0xF4,
+0x00, 0x6C, 0x40, 0xEA, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x69, 0xF3,
+0xD0, 0x9C, 0x81, 0xA1, 0xE0, 0xA1, 0x4C, 0xEE,
+0x03, 0x6D, 0x44, 0x67, 0xF7, 0xF0, 0x01, 0x6B,
+0xAC, 0xEA, 0x60, 0x33, 0xE6, 0x37, 0x60, 0x33,
+0x5C, 0x32, 0xED, 0xEA, 0x69, 0xF3, 0xE4, 0x9B,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xEC, 0xEA, 0xE9, 0xF1,
+0xE8, 0x9B, 0x8A, 0x33, 0x82, 0xA1, 0xCD, 0xEA,
+0xEC, 0xEA, 0x98, 0x34, 0x8D, 0xEB, 0x83, 0xA1,
+0xD2, 0xF4, 0xE8, 0x98, 0x8C, 0xED, 0xA0, 0x35,
+0xB8, 0x35, 0x6D, 0xED, 0xAA, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x40, 0xA4, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x69,
+0x6C, 0xEA, 0xF7, 0xF0, 0x01, 0x68, 0xFF, 0x6B,
+0x20, 0x31, 0x00, 0x30, 0x6C, 0xEA, 0x04, 0xD4,
+0x20, 0x31, 0x00, 0x30, 0x47, 0x22, 0xD2, 0xF4,
+0x54, 0x99, 0x13, 0xF2, 0x04, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x04, 0x96, 0x03, 0x6F, 0x80, 0xA6,
+0x86, 0x35, 0xEC, 0xED, 0x19, 0x6F, 0xEB, 0xEF,
+0xEC, 0xEA, 0xAC, 0x35, 0x4D, 0xED, 0x05, 0x6A,
+0x4B, 0xEA, 0x4C, 0xED, 0x01, 0x6A, 0x4D, 0xED,
+0x08, 0x6A, 0x4C, 0xEC, 0x04, 0x6A, 0x01, 0x2C,
+0x00, 0x6A, 0xD2, 0xF4, 0xE8, 0x98, 0x4D, 0xED,
+0x13, 0xF2, 0x04, 0x6C, 0x40, 0xEF, 0x04, 0xD6,
+0xD2, 0xF4, 0x08, 0x98, 0xD2, 0xF4, 0x54, 0x99,
+0x30, 0xF4, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x04, 0x96, 0x60, 0x33,
+0x60, 0x33, 0x09, 0xF1, 0xB4, 0x9B, 0x60, 0xA6,
+0x10, 0x6C, 0xAC, 0xEA, 0x8C, 0xEB, 0x00, 0x6D,
+0x06, 0x23, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x09, 0xF1, 0xBC, 0x9B, 0x4D, 0xED,
+0x30, 0xF4, 0x00, 0x6C, 0x40, 0xE8, 0x00, 0x65,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0x6A,
+0x00, 0xEF, 0x05, 0x63, 0xD2, 0xF4, 0x54, 0x99,
+0x13, 0xF2, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xD2, 0xF4, 0xC8, 0x98, 0x06, 0x6D, 0xAB, 0xED,
+0x13, 0xF2, 0x04, 0x6C, 0x40, 0xEE, 0x4C, 0xED,
+0xD2, 0xF4, 0x08, 0x98, 0xD2, 0xF4, 0x54, 0x99,
+0x30, 0xF4, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x09, 0xF1, 0xB4, 0x9B, 0x4C, 0xED, 0xD8, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x40, 0xA4,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x68, 0x6C, 0xEA,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x00, 0x30,
+0x60, 0x33, 0x00, 0x30, 0x12, 0x22, 0xD2, 0xF4,
+0x54, 0x9B, 0x13, 0xF3, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xD2, 0xF4, 0x48, 0x98, 0x07, 0x6D,
+0x13, 0xF3, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x05, 0x97, 0x04, 0x90, 0x00, 0x6A, 0x00, 0xEF,
+0x03, 0x63, 0xD2, 0xF4, 0x54, 0x9B, 0x13, 0xF3,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x6D,
+0xD2, 0xF4, 0x48, 0x98, 0xAB, 0xED, 0xEC, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x40, 0xA4, 0x03, 0x6B, 0x6C, 0xEA, 0x01, 0x72,
+0x07, 0x60, 0x35, 0x22, 0x02, 0x72, 0x08, 0x60,
+0x03, 0x72, 0x2F, 0x60, 0x05, 0x6A, 0x28, 0x10,
+0x10, 0x69, 0x13, 0xF2, 0x0C, 0x68, 0x03, 0x10,
+0x00, 0x69, 0x13, 0xF2, 0x10, 0x68, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x04, 0xD4, 0x40, 0xEA, 0x90, 0x67,
+0xE1, 0xF7, 0x1F, 0x6D, 0xA4, 0xE9, 0x04, 0x97,
+0xAF, 0xED, 0x4C, 0xED, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x61, 0xA7, 0xD2, 0xF4,
+0xC8, 0x9A, 0x40, 0xA7, 0x3F, 0x6C, 0x8C, 0xEB,
+0x4A, 0x32, 0x78, 0x33, 0x4D, 0xEB, 0x64, 0xE9,
+0x6D, 0xED, 0x40, 0xEE, 0x90, 0x67, 0x00, 0x6A,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x10, 0x69, 0xD6, 0x17, 0x00, 0x69,
+0xD0, 0x17, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x60, 0xA4, 0x5D, 0x67, 0xBD, 0x67, 0x20, 0xF0,
+0x6C, 0xC2, 0x61, 0xA4, 0x03, 0x6E, 0x20, 0xF0,
+0x6D, 0xC2, 0x62, 0xA4, 0x20, 0xF0, 0x6E, 0xC2,
+0x64, 0xA4, 0x20, 0xF0, 0x68, 0xC2, 0x65, 0xA4,
+0x20, 0xF0, 0x69, 0xC2, 0x66, 0xA4, 0x20, 0xF0,
+0x6A, 0xC2, 0x67, 0xA4, 0x20, 0xF0, 0x6B, 0xC2,
+0x69, 0xA4, 0x72, 0x32, 0x20, 0xF0, 0x40, 0xC5,
+0x48, 0xA4, 0x01, 0x6D, 0xAC, 0xEA, 0xBD, 0x67,
+0x20, 0xF0, 0x41, 0xC5, 0x48, 0xA4, 0x46, 0x35,
+0xCC, 0xED, 0xDD, 0x67, 0x20, 0xF0, 0xA2, 0xC6,
+0x4E, 0x35, 0x0F, 0x6E, 0xCC, 0xED, 0xDD, 0x67,
+0x20, 0xF0, 0xA3, 0xC6, 0x5E, 0x32, 0xBD, 0x67,
+0x20, 0xF0, 0x44, 0xC5, 0x07, 0x6A, 0x4C, 0xEB,
+0x5D, 0x67, 0x20, 0xF0, 0x65, 0xC2, 0x4B, 0xA4,
+0x6A, 0xA4, 0x40, 0x32, 0x6D, 0xEA, 0x7D, 0x67,
+0x53, 0xCB, 0xAD, 0xA4, 0x4C, 0xA4, 0x6E, 0xA4,
+0xA0, 0x35, 0x4D, 0xED, 0x4F, 0xA4, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x04, 0xD2, 0x50, 0xA4, 0xB1, 0xA4,
+0x72, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x53, 0xA4,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x05, 0xD2, 0xB5, 0xA4,
+0x54, 0xA4, 0x76, 0xA4, 0xA0, 0x35, 0x4D, 0xED,
+0x57, 0xA4, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x06, 0xD2,
+0xB9, 0xA4, 0x58, 0xA4, 0x7A, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x5B, 0xA4, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x04, 0x04, 0xB1, 0x18, 0x57, 0xF4, 0x07, 0xD2,
+0x0D, 0x97, 0x00, 0x6A, 0x00, 0xEF, 0x07, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x40, 0xA4, 0x0F, 0x6B,
+0x6C, 0xEA, 0x0E, 0x22, 0x1D, 0xF2, 0x00, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF5, 0x5C, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x05, 0x97, 0x00, 0x6A, 0x00, 0xEF, 0x03, 0x63,
+0x1D, 0xF2, 0x0B, 0x6C, 0xF1, 0x17, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x4F, 0x44, 0x03, 0x5A, 0xE4, 0x67, 0x3A, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0xF7, 0xF0,
+0x01, 0x68, 0x40, 0x32, 0xF7, 0xF0, 0x01, 0x69,
+0x00, 0x30, 0x2D, 0xF1, 0x6C, 0x9A, 0x00, 0x30,
+0x20, 0x31, 0x20, 0x31, 0x04, 0xD2, 0x4D, 0xF1,
+0x4C, 0x98, 0xD2, 0xF4, 0xC8, 0x99, 0xA3, 0x67,
+0x05, 0xD4, 0xB3, 0xF6, 0x10, 0x6C, 0x40, 0xEE,
+0x4D, 0xED, 0x04, 0x92, 0x4D, 0xF1, 0x0C, 0x48,
+0xD2, 0xF4, 0xC8, 0x99, 0x2D, 0xF1, 0x0C, 0x4A,
+0xA1, 0x9A, 0x04, 0xD2, 0x41, 0x98, 0xB3, 0xF6,
+0x14, 0x6C, 0x40, 0xEE, 0x4D, 0xED, 0x04, 0x92,
+0xD2, 0xF4, 0xC8, 0x99, 0xB3, 0xF6, 0x18, 0x6C,
+0xA2, 0x9A, 0x42, 0x98, 0x40, 0xEE, 0x4D, 0xED,
+0x04, 0x92, 0xD2, 0xF4, 0x68, 0x99, 0xB3, 0xF6,
+0x1C, 0x6C, 0xA3, 0x9A, 0x43, 0x98, 0x40, 0xEB,
+0x4D, 0xED, 0x05, 0x97, 0x4E, 0x47, 0x02, 0x5A,
+0x01, 0x61, 0x38, 0x2F, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x69, 0x40, 0x32, 0x20, 0x31,
+0x40, 0x32, 0x20, 0x31, 0xD2, 0xF4, 0x68, 0x9A,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x68, 0x0D, 0xF1,
+0x5C, 0x99, 0x00, 0x30, 0x00, 0x30, 0x82, 0x67,
+0x2D, 0xF1, 0x5C, 0x98, 0xA4, 0x67, 0xD8, 0xF2,
+0x00, 0x6C, 0x40, 0xEB, 0x4D, 0xED, 0x04, 0x92,
+0x0D, 0xF1, 0x1C, 0x49, 0x2D, 0xF1, 0x1C, 0x48,
+0xD2, 0xF4, 0x68, 0x9A, 0xA1, 0x99, 0x41, 0x98,
+0xD8, 0xF2, 0x04, 0x6C, 0x40, 0xEB, 0x4D, 0xED,
+0x04, 0x92, 0xA2, 0x99, 0xD8, 0xF2, 0x08, 0x6C,
+0xD2, 0xF4, 0x68, 0x9A, 0x42, 0x98, 0x40, 0xEB,
+0x4D, 0xED, 0x04, 0x92, 0xA3, 0x99, 0x63, 0x98,
+0xD2, 0xF4, 0x48, 0x9A, 0xD8, 0xF2, 0x0C, 0x6C,
+0x40, 0xEA, 0x6D, 0xED, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x73, 0xF6, 0x5C, 0x98, 0xA4, 0x67, 0x24, 0x67,
+0x10, 0x6E, 0x40, 0xEA, 0x0C, 0x04, 0x73, 0xF6,
+0x5C, 0x98, 0xA7, 0x41, 0x10, 0x6E, 0x09, 0x4D,
+0x40, 0xEA, 0x08, 0x04, 0x00, 0x6B, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x20, 0x31, 0x4D, 0xF1,
+0x0C, 0x49, 0x0C, 0x02, 0x31, 0xE3, 0x69, 0xE2,
+0x08, 0x06, 0xA0, 0x9C, 0x79, 0xE6, 0x40, 0x9A,
+0xC0, 0x9E, 0xAE, 0xEA, 0xCC, 0xEA, 0xAE, 0xEA,
+0x40, 0xDC, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x2D, 0xF1, 0x1C, 0x4C, 0x91, 0xE3,
+0x04, 0x4B, 0x10, 0x73, 0x40, 0xDC, 0xE3, 0x61,
+0xD1, 0x18, 0xFC, 0x2D, 0x02, 0x6C, 0x00, 0x6A,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x04, 0x03, 0x2D, 0xF1, 0x0C, 0x4C, 0x55, 0xE3,
+0x91, 0xE2, 0x4D, 0xE1, 0x80, 0x9C, 0x60, 0x9B,
+0x8D, 0xEB, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x0D, 0xF1, 0x1C, 0x4C, 0x91, 0xE2,
+0x80, 0x9C, 0x04, 0x4A, 0x10, 0x72, 0x8D, 0xEB,
+0x60, 0xDD, 0x78, 0x67, 0xE5, 0x2B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x12, 0xD3, 0x00, 0x6F, 0x12, 0x06,
+0x10, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x06, 0x22,
+0x04, 0x6A, 0x17, 0x97, 0x16, 0x91, 0x15, 0x90,
+0x00, 0xEF, 0x0C, 0x63, 0x12, 0x94, 0xF8, 0x24,
+0xBD, 0x67, 0x01, 0x6B, 0x40, 0xF0, 0x62, 0xC5,
+0x40, 0xF0, 0x61, 0xC5, 0x04, 0x6B, 0x40, 0xF0,
+0x60, 0xC5, 0x18, 0x6B, 0x40, 0xF0, 0x63, 0xC5,
+0x7D, 0x67, 0x40, 0xF0, 0x44, 0xC3, 0x73, 0xF6,
+0x5C, 0x98, 0x04, 0x05, 0x40, 0xEA, 0x10, 0x6E,
+0x12, 0x95, 0xD1, 0x18, 0xA8, 0x19, 0x10, 0x04,
+0xDF, 0x2A, 0xDF, 0x17, 0xF4, 0x63, 0x17, 0x62,
+0x16, 0xD1, 0x15, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x73, 0xF6, 0x5C, 0x99,
+0x04, 0x67, 0xA4, 0x67, 0x10, 0x6E, 0x40, 0xEA,
+0x10, 0x04, 0x73, 0xF6, 0x5C, 0x99, 0xA7, 0x40,
+0x10, 0x6E, 0x09, 0x4D, 0x40, 0xEA, 0x0C, 0x04,
+0x73, 0xF6, 0x5C, 0x99, 0xA7, 0x40, 0x10, 0x6E,
+0x19, 0x4D, 0x40, 0xEA, 0x08, 0x04, 0x73, 0xF6,
+0x5C, 0x99, 0xA7, 0x40, 0x10, 0x6E, 0x29, 0x4D,
+0x40, 0xEA, 0x04, 0x04, 0x00, 0x6A, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x4D, 0xF1,
+0x0C, 0x4C, 0x10, 0x03, 0x91, 0xE2, 0x4D, 0xE3,
+0x0C, 0x06, 0xA0, 0x9C, 0x59, 0xE6, 0x60, 0x9B,
+0xC0, 0x9E, 0xAE, 0xEB, 0xCC, 0xEB, 0xAE, 0xEB,
+0x60, 0xDC, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x2D, 0xF1, 0x1C, 0x4C, 0x08, 0x03,
+0x91, 0xE2, 0x4D, 0xE3, 0x04, 0x06, 0xA0, 0x9C,
+0x59, 0xE6, 0x60, 0x9B, 0xC0, 0x9E, 0x04, 0x4A,
+0xAE, 0xEB, 0xCC, 0xEB, 0xAE, 0xEB, 0x10, 0x72,
+0x60, 0xDC, 0xD9, 0x61, 0xD1, 0x18, 0xFC, 0x2D,
+0x03, 0x6C, 0x17, 0x97, 0x16, 0x91, 0x15, 0x90,
+0x00, 0x6A, 0x00, 0xEF, 0x0C, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0x00, 0x68, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0xA0, 0x35, 0x08, 0x32, 0x0D, 0xF1, 0x1C, 0x4D,
+0xB5, 0xE2, 0x49, 0xE4, 0x21, 0xA2, 0xC2, 0xA2,
+0x60, 0xA2, 0x20, 0x31, 0xC0, 0x36, 0x6D, 0xE9,
+0xC0, 0x36, 0xCD, 0xE9, 0xD0, 0xA2, 0x63, 0xA2,
+0xE0, 0x9D, 0x1E, 0x65, 0xD1, 0xA2, 0x00, 0xF6,
+0x60, 0x33, 0x2D, 0xEB, 0xC0, 0x36, 0x38, 0x67,
+0xCD, 0xE9, 0x19, 0x65, 0x32, 0xA2, 0x53, 0xA2,
+0xD8, 0x67, 0x20, 0x31, 0x20, 0x31, 0xCD, 0xE9,
+0x00, 0xF6, 0x40, 0x32, 0xEE, 0xEB, 0x2D, 0xEA,
+0x6C, 0xEA, 0x01, 0x48, 0xEE, 0xEA, 0x04, 0x70,
+0x40, 0xDD, 0xD3, 0x61, 0xD1, 0x18, 0xFC, 0x2D,
+0x00, 0x6C, 0x07, 0x97, 0x06, 0x91, 0x05, 0x90,
+0x00, 0xEF, 0x04, 0x63, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD1, 0x13, 0xD0, 0x00, 0x6A, 0x04, 0x67,
+0x0C, 0xD2, 0x0D, 0xD2, 0x0E, 0xD2, 0x0F, 0xD2,
+0x00, 0x6F, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x23, 0x67, 0xE8, 0x32, 0x2D, 0xF1,
+0x0C, 0x49, 0x25, 0xE2, 0x80, 0x99, 0x01, 0x4F,
+0x04, 0x77, 0x10, 0xD4, 0x10, 0x96, 0x9D, 0x67,
+0x51, 0xE4, 0x49, 0xE0, 0xCC, 0xDC, 0x80, 0xA2,
+0x3C, 0x65, 0x81, 0xA2, 0xD9, 0x67, 0x80, 0x34,
+0x8D, 0xEE, 0x82, 0xA2, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEE, 0x83, 0xA2, 0x00, 0xF6, 0x80, 0x34,
+0xCD, 0xEC, 0x10, 0x96, 0xCE, 0xEC, 0xD0, 0xA2,
+0x11, 0xD4, 0xFE, 0x65, 0xD1, 0xA2, 0xC0, 0x36,
+0x3E, 0x65, 0x99, 0x67, 0xDF, 0x67, 0x8D, 0xEE,
+0x92, 0xA2, 0x53, 0xA2, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEE, 0x11, 0x94, 0x00, 0xF6, 0x40, 0x32,
+0xCD, 0xEA, 0x8C, 0xEA, 0x10, 0x94, 0x8E, 0xEA,
+0x40, 0xD9, 0xC7, 0x61, 0x01, 0x6C, 0x11, 0xD3,
+0xD1, 0x18, 0xFC, 0x2D, 0x10, 0xD5, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x30, 0x9A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0D, 0xF1, 0x5C, 0x9A, 0x11, 0x93,
+0x91, 0xA0, 0x0B, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xF1, 0x4C, 0x9A,
+0x80, 0x34, 0x10, 0x95, 0x0A, 0xD2, 0x2D, 0xF1,
+0x4C, 0x9B, 0x72, 0xA0, 0x05, 0xD5, 0x09, 0xD2,
+0x0C, 0x92, 0x60, 0x33, 0x60, 0x33, 0x08, 0xD2,
+0x50, 0xA0, 0x69, 0xF6, 0x0B, 0x6F, 0x04, 0x6E,
+0x4D, 0xEC, 0x53, 0xA0, 0x6D, 0xEC, 0x62, 0xA0,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x81, 0xA0,
+0x07, 0xD2, 0x40, 0xA0, 0x80, 0x34, 0x60, 0x33,
+0x4D, 0xEC, 0x43, 0xA0, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x06, 0xD2,
+0x07, 0x6A, 0x04, 0xD2, 0x40, 0x6C, 0x40, 0xE9,
+0x00, 0x6D, 0x15, 0x97, 0x14, 0x91, 0x13, 0x90,
+0x00, 0xEF, 0x0B, 0x63, 0x00, 0x6B, 0xA4, 0x67,
+0xA6, 0xEB, 0x01, 0x6E, 0xFF, 0x6A, 0xCC, 0xED,
+0x6C, 0xEA, 0x04, 0x2D, 0x01, 0x4B, 0x20, 0x73,
+0xF6, 0x61, 0x20, 0x6A, 0x20, 0xE8, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x41, 0x45, 0x24, 0x67, 0x06, 0x67, 0x13, 0x22,
+0x85, 0x67, 0xD1, 0x18, 0x0D, 0x2F, 0x05, 0xD5,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0x04, 0x93, 0x05, 0x95, 0x04, 0xEB,
+0x4E, 0xE8, 0xAC, 0xE8, 0x4E, 0xE8, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x48, 0x9A, 0xB0, 0x67, 0x40, 0xEA, 0x91, 0x67,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x05, 0x67, 0x0C, 0xEA, 0x90, 0x67, 0xD1, 0x18,
+0x0D, 0x2F, 0xE2, 0x67, 0xE6, 0xEA, 0x47, 0x67,
+0x04, 0x90, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0x00, 0x6A, 0x7D, 0x67, 0x04, 0xD2, 0x05, 0xD2,
+0x06, 0xD2, 0x4E, 0xCB, 0x09, 0x6A, 0x20, 0xF0,
+0x40, 0xC3, 0x0C, 0x6A, 0x20, 0xF0, 0x41, 0xC3,
+0x00, 0x6A, 0x19, 0xD5, 0x1A, 0xD6, 0x04, 0x67,
+0x0F, 0xD2, 0x00, 0x69, 0x19, 0x92, 0x43, 0xE8,
+0xA0, 0xF0, 0x1D, 0x60, 0x49, 0xA0, 0x68, 0xA0,
+0xEA, 0xA0, 0x40, 0x32, 0x4D, 0xEB, 0xE0, 0x37,
+0x4B, 0xA0, 0xE0, 0x37, 0x6D, 0xEF, 0x6D, 0xA0,
+0x00, 0xF6, 0x40, 0x32, 0x8C, 0xA0, 0x4D, 0xEF,
+0x4E, 0xA0, 0x60, 0x33, 0x6D, 0xEC, 0x6F, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x4D, 0xEB, 0x0E, 0xD3, 0x63, 0xA0,
+0x42, 0xA0, 0x03, 0x6E, 0x60, 0x33, 0x4D, 0xEB,
+0x40, 0xA0, 0xFF, 0x6C, 0x3C, 0x65, 0x4A, 0x35,
+0xCC, 0xED, 0x01, 0x75, 0x30, 0x60, 0x0A, 0x25,
+0x02, 0x75, 0x80, 0xF0, 0x09, 0x60, 0x40, 0xA0,
+0x10, 0x6B, 0x6C, 0xEA, 0x80, 0xF0, 0x0F, 0x2A,
+0x10, 0x48, 0xCC, 0x17, 0xA2, 0x67, 0xCC, 0xED,
+0x8C, 0xED, 0x01, 0x75, 0x0F, 0x60, 0x08, 0x25,
+0x02, 0x75, 0xF1, 0x61, 0x0E, 0x95, 0xC7, 0x67,
+0xD1, 0x18, 0x14, 0x2F, 0x83, 0x67, 0xEB, 0x17,
+0x0E, 0x95, 0xC7, 0x67, 0xD1, 0x18, 0x45, 0x6C,
+0x83, 0x67, 0xE5, 0x17, 0xA6, 0xA0, 0x56, 0x32,
+0xCC, 0xEA, 0x1D, 0x65, 0xA7, 0xA0, 0x98, 0x67,
+0x0E, 0x96, 0xA0, 0x35, 0x8D, 0xED, 0xA0, 0x35,
+0x99, 0x67, 0xA0, 0x35, 0x6D, 0xED, 0xD1, 0x18,
+0x10, 0xF5, 0x4C, 0xEC, 0xD4, 0x17, 0x32, 0x6A,
+0x10, 0xD2, 0x40, 0xA0, 0x03, 0x6D, 0xFF, 0x6C,
+0xAC, 0xEA, 0x8C, 0xEA, 0x01, 0x72, 0x38, 0x60,
+0x2D, 0x22, 0x02, 0x72, 0x47, 0x60, 0xEA, 0xE9,
+0x12, 0xD3, 0x57, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x48, 0x9A,
+0x64, 0x6C, 0x40, 0xEA, 0x11, 0xD7, 0x10, 0x92,
+0x11, 0x97, 0x12, 0x93, 0xFF, 0x4A, 0x10, 0xD2,
+0xE4, 0x2A, 0x01, 0x6A, 0x9D, 0x67, 0x50, 0xC4,
+0x41, 0xA0, 0x7F, 0x6C, 0x8C, 0xEA, 0x9D, 0x67,
+0x51, 0xC4, 0x5D, 0x67, 0x69, 0xCA, 0xEB, 0xCA,
+0x62, 0x33, 0xE2, 0x37, 0x62, 0x33, 0xE2, 0x37,
+0x6A, 0xCA, 0xEC, 0xCA, 0x2D, 0xCA, 0x22, 0x32,
+0x42, 0x32, 0x7D, 0x67, 0x4E, 0xCB, 0x04, 0x6A,
+0x0F, 0xD2, 0x9D, 0x17, 0x0E, 0x95, 0x83, 0x67,
+0x12, 0xD7, 0xD1, 0x18, 0x7B, 0x6C, 0x11, 0xD3,
+0x22, 0x67, 0x11, 0x93, 0x12, 0x97, 0xCB, 0x17,
+0x12, 0xD7, 0x47, 0xA0, 0x26, 0xA0, 0x0E, 0x96,
+0x40, 0x32, 0x2D, 0xEA, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEB, 0x40, 0xA0, 0x11, 0xD3, 0x56, 0x32,
+0xAC, 0xEA, 0x4C, 0xEC, 0xD1, 0x18, 0xF2, 0x6C,
+0xA3, 0x67, 0xEA, 0x17, 0x0E, 0x95, 0x83, 0x67,
+0x12, 0xD7, 0xD1, 0x18, 0x29, 0x2F, 0x11, 0xD3,
+0x22, 0x67, 0x12, 0x97, 0x11, 0x93, 0xAF, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0x87, 0x67,
+0x6E, 0x17, 0x27, 0x67, 0x6C, 0x17, 0x1A, 0x92,
+0x59, 0x22, 0x00, 0x6A, 0x7D, 0x67, 0x0C, 0xD2,
+0x01, 0x6A, 0x20, 0xF0, 0x4A, 0xC3, 0x20, 0xF0,
+0x49, 0xC3, 0x08, 0x6A, 0x20, 0xF0, 0x48, 0xC3,
+0x0F, 0x93, 0x0E, 0x6A, 0x01, 0x2B, 0x04, 0x6A,
+0x08, 0x4A, 0x7D, 0x67, 0x20, 0xF0, 0x4B, 0xC3,
+0x00, 0x6A, 0x20, 0xF0, 0x4C, 0xC3, 0x06, 0x68,
+0xFF, 0x48, 0xFF, 0x6A, 0x4C, 0xE8, 0x16, 0x20,
+0x9D, 0x67, 0x20, 0xF0, 0xAC, 0xA4, 0xF7, 0xF0,
+0x01, 0x6A, 0x7D, 0x67, 0x40, 0x32, 0x20, 0xF0,
+0x6B, 0xA3, 0x40, 0x32, 0xA0, 0x35, 0xD3, 0xF5,
+0x48, 0x9A, 0x6D, 0xED, 0x00, 0x6F, 0x0C, 0x06,
+0xF8, 0x4D, 0x40, 0xEA, 0x00, 0x6C, 0x08, 0x22,
+0x04, 0x6A, 0x0F, 0xD2, 0x0F, 0x92, 0x17, 0x97,
+0x16, 0x91, 0x15, 0x90, 0x00, 0xEF, 0x0C, 0x63,
+0x0C, 0x94, 0x09, 0x2C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xB2, 0xF2, 0x5C, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0xD5, 0x17, 0xBD, 0x67,
+0x20, 0xF0, 0xCC, 0xA5, 0xF7, 0xF0, 0x01, 0x6A,
+0x7D, 0x67, 0x20, 0xF0, 0x6B, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A, 0xC0, 0x36,
+0x6D, 0xEE, 0x04, 0x05, 0x40, 0xEA, 0xF8, 0x4E,
+0x0C, 0x95, 0xD1, 0x18, 0xA8, 0x19, 0x0A, 0x04,
+0xE1, 0x2A, 0xD8, 0x17, 0x04, 0x02, 0x7D, 0x67,
+0x51, 0xCB, 0x42, 0x32, 0x42, 0x32, 0x52, 0xCB,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF0, 0x4C, 0x9A, 0x40, 0xEA, 0x08, 0x04,
+0xC9, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x69, 0xF3,
+0x44, 0x9A, 0x60, 0x33, 0x69, 0xF3, 0x74, 0x9B,
+0x8C, 0xEA, 0x4E, 0xEB, 0x08, 0x2B, 0xFF, 0xF7,
+0x1F, 0x6A, 0xD1, 0x18, 0x45, 0x6C, 0x4C, 0xEC,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x69, 0xF3,
+0x78, 0x9B, 0x4E, 0xEB, 0x10, 0x2B, 0x1E, 0xF0,
+0x00, 0x6A, 0x8C, 0xEA, 0x02, 0xF0, 0x00, 0x6B,
+0x6E, 0xEA, 0xE6, 0x67, 0x01, 0x5A, 0xC5, 0x67,
+0xFF, 0xF7, 0x1F, 0x6D, 0x8C, 0xED, 0xD1, 0x18,
+0x10, 0xF5, 0x98, 0x67, 0xE5, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x69, 0xF3,
+0x7C, 0x9B, 0x6E, 0xEA, 0xDD, 0x2A, 0xFF, 0xF7,
+0x1F, 0x6A, 0xD1, 0x18, 0x14, 0x2F, 0x4C, 0xEC,
+0xD7, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x69, 0xF3,
+0x44, 0x9A, 0x60, 0x33, 0x69, 0xF3, 0x74, 0x9B,
+0x8C, 0xEA, 0x4E, 0xEB, 0x08, 0x2B, 0xFF, 0xF7,
+0x1F, 0x6A, 0xD1, 0x18, 0x7B, 0x6C, 0x4C, 0xEC,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x69, 0xF3,
+0x78, 0x9B, 0x4E, 0xEB, 0x0F, 0x2B, 0x1E, 0xF0,
+0x00, 0x6A, 0x8C, 0xEA, 0x02, 0xF0, 0x00, 0x6B,
+0x6E, 0xEA, 0xC5, 0x67, 0x01, 0x5A, 0xFF, 0xF7,
+0x1F, 0x6D, 0x8C, 0xED, 0xD1, 0x18, 0xF2, 0x6C,
+0x98, 0x67, 0xE6, 0x17, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x69, 0xF3, 0x7C, 0x9B,
+0x4E, 0xEB, 0x00, 0x6A, 0xDD, 0x2B, 0xFF, 0xF7,
+0x1F, 0x6A, 0xD1, 0x18, 0x29, 0x2F, 0x4C, 0xEC,
+0xD7, 0x17, 0x00, 0x65, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD1, 0x13, 0xD0, 0x00, 0x6A, 0x7D, 0x67,
+0x04, 0xD2, 0x05, 0xD2, 0x06, 0xD2, 0x4E, 0xCB,
+0x09, 0x6A, 0x20, 0xF0, 0x40, 0xC3, 0x0C, 0x6A,
+0x17, 0xD5, 0x18, 0xD6, 0x20, 0xF0, 0x41, 0xC3,
+0x24, 0x67, 0x00, 0x68, 0x17, 0x92, 0x43, 0xE9,
+0x00, 0xF1, 0x01, 0x60, 0x40, 0xA1, 0x07, 0x6C,
+0x8C, 0xEA, 0x01, 0x72, 0x31, 0x60, 0x0D, 0x22,
+0x02, 0x72, 0x80, 0xF0, 0x07, 0x60, 0x03, 0x72,
+0x80, 0xF0, 0x18, 0x60, 0x40, 0xA1, 0x08, 0x6C,
+0x8C, 0xEA, 0xE0, 0xF0, 0x10, 0x2A, 0x10, 0x49,
+0xE9, 0x17, 0xCD, 0xA1, 0x4E, 0xA1, 0x8C, 0xA1,
+0xC0, 0x36, 0x40, 0x32, 0x8D, 0xEE, 0x40, 0x32,
+0x4D, 0xEE, 0xA9, 0xA1, 0x4A, 0xA1, 0x88, 0xA1,
+0xA0, 0x35, 0x40, 0x32, 0x8D, 0xED, 0x40, 0x32,
+0x4D, 0xED, 0x42, 0xA1, 0x81, 0xA1, 0xEF, 0xA1,
+0x40, 0x32, 0x6B, 0xA1, 0x8D, 0xEA, 0x83, 0xA1,
+0x00, 0xF6, 0xE0, 0x37, 0x00, 0xF6, 0x60, 0x33,
+0x80, 0x34, 0x80, 0x34, 0xED, 0xEE, 0x6D, 0xED,
+0xD1, 0x18, 0xD5, 0x2F, 0x4D, 0xEC, 0xD6, 0x17,
+0x32, 0x6E, 0x0F, 0xD6, 0xA9, 0xA1, 0xE8, 0xA1,
+0x42, 0xA1, 0xA0, 0x35, 0xED, 0xED, 0xEA, 0xA1,
+0x81, 0xA1, 0x40, 0x32, 0xE0, 0x37, 0x8D, 0xEA,
+0xE0, 0x37, 0x83, 0xA1, 0xAD, 0xEF, 0xAB, 0xA1,
+0x80, 0x34, 0x80, 0x34, 0x00, 0xF6, 0xA0, 0x35,
+0xED, 0xED, 0xD1, 0x18, 0xF5, 0x2F, 0x4D, 0xEC,
+0xED, 0xA1, 0x8C, 0xA1, 0xAE, 0xA1, 0xE0, 0x37,
+0x8D, 0xEF, 0x8F, 0xA1, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEF, 0x00, 0xF6, 0x80, 0x34, 0xED, 0xEC,
+0x4E, 0xEC, 0x0E, 0xD2, 0xAF, 0x24, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x73, 0xF6,
+0xA8, 0x9C, 0x40, 0xED, 0x64, 0x6C, 0x0F, 0x96,
+0x0E, 0x92, 0xFF, 0x4E, 0xCE, 0x2E, 0x7D, 0x67,
+0x01, 0x6C, 0x90, 0xC3, 0x87, 0xA1, 0xA2, 0xA1,
+0xC1, 0xA1, 0x91, 0xC3, 0x83, 0xA1, 0xA0, 0x35,
+0xAD, 0xEE, 0x80, 0x34, 0x80, 0x34, 0xCD, 0xEC,
+0x89, 0xCB, 0x82, 0x34, 0x82, 0x34, 0x8A, 0xCB,
+0xCD, 0xA1, 0x8C, 0xA1, 0xAE, 0xA1, 0xC0, 0x36,
+0x8D, 0xEE, 0x8F, 0xA1, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEE, 0x00, 0xF6, 0x80, 0x34, 0xCD, 0xEC,
+0x8B, 0xCB, 0x4D, 0xCB, 0x82, 0x34, 0x42, 0x32,
+0x82, 0x34, 0x42, 0x32, 0x8C, 0xCB, 0x4E, 0xCB,
+0x04, 0x68, 0x7C, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xA1, 0x73, 0xF6,
+0xC8, 0x9A, 0x4C, 0xA1, 0x80, 0x34, 0x4D, 0xEC,
+0x4E, 0xA1, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x8F, 0xA1, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xEE,
+0x4D, 0xEC, 0x68, 0x17, 0xA9, 0xA1, 0xC8, 0xA1,
+0x42, 0xA1, 0xA0, 0x35, 0xCD, 0xED, 0xCA, 0xA1,
+0x81, 0xA1, 0x40, 0x32, 0xC0, 0x36, 0x8D, 0xEA,
+0xC0, 0x36, 0x83, 0xA1, 0xAD, 0xEE, 0xAB, 0xA1,
+0x80, 0x34, 0x80, 0x34, 0x00, 0xF6, 0xA0, 0x35,
+0xCD, 0xED, 0xD1, 0x18, 0xF5, 0x2F, 0x4D, 0xEC,
+0x0F, 0xD2, 0x89, 0xA1, 0xE8, 0xA1, 0x4A, 0xA1,
+0x80, 0x34, 0xED, 0xEC, 0xEB, 0xA1, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xEC, 0x00, 0xF6, 0xE0, 0x37,
+0x8D, 0xEF, 0x87, 0x67, 0xD1, 0x18, 0x0D, 0x2F,
+0x0E, 0xD7, 0x0E, 0x93, 0x66, 0xEA, 0x0E, 0xD3,
+0x4D, 0xA1, 0x8C, 0xA1, 0xAE, 0xA1, 0x40, 0x32,
+0x4D, 0xEC, 0x4F, 0xA1, 0xA0, 0x35, 0xA0, 0x35,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0x4D, 0xED,
+0x85, 0x67, 0xD1, 0x18, 0x0D, 0x2F, 0x10, 0xD5,
+0x10, 0x95, 0x00, 0x6C, 0xC5, 0x67, 0xC6, 0xEA,
+0x0E, 0x92, 0x43, 0xEE, 0x10, 0x61, 0x00, 0x6A,
+0x0E, 0x93, 0xC3, 0xEB, 0x13, 0x61, 0x0F, 0x96,
+0xE4, 0xA1, 0xC4, 0xEA, 0x45, 0xA1, 0xC6, 0xEC,
+0x40, 0x32, 0xED, 0xEA, 0xE6, 0xA1, 0xE0, 0x37,
+0xE0, 0x37, 0x87, 0x67, 0x3D, 0x17, 0xFF, 0x6A,
+0x01, 0x4C, 0x4C, 0xEC, 0x0E, 0x92, 0x46, 0x32,
+0x0E, 0xD2, 0xE6, 0x17, 0x01, 0x4A, 0xFF, 0x6B,
+0x6C, 0xEA, 0xC6, 0x36, 0xE5, 0x17, 0x18, 0x92,
+0x57, 0x22, 0x00, 0x6A, 0x7D, 0x67, 0x0C, 0xD2,
+0x01, 0x6A, 0x20, 0xF0, 0x4A, 0xC3, 0x20, 0xF0,
+0x49, 0xC3, 0x08, 0x6A, 0x20, 0xF0, 0x48, 0xC3,
+0x0E, 0x6A, 0x01, 0x28, 0x04, 0x6A, 0x08, 0x4A,
+0x7D, 0x67, 0x20, 0xF0, 0x4B, 0xC3, 0x00, 0x6A,
+0x20, 0xF0, 0x4C, 0xC3, 0x06, 0x69, 0xFF, 0x49,
+0xFF, 0x6A, 0x4C, 0xE9, 0x15, 0x21, 0x9D, 0x67,
+0x20, 0xF0, 0xAC, 0xA4, 0xF7, 0xF0, 0x01, 0x6A,
+0x7D, 0x67, 0x40, 0x32, 0x20, 0xF0, 0x6B, 0xA3,
+0x40, 0x32, 0xA0, 0x35, 0xD3, 0xF5, 0x48, 0x9A,
+0x6D, 0xED, 0x00, 0x6F, 0x0C, 0x06, 0xF8, 0x4D,
+0x40, 0xEA, 0x00, 0x6C, 0x07, 0x22, 0x04, 0x68,
+0x50, 0x67, 0x15, 0x97, 0x14, 0x91, 0x13, 0x90,
+0x00, 0xEF, 0x0B, 0x63, 0x0C, 0x94, 0x09, 0x2C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xB2, 0xF2, 0x5C, 0x9A, 0x40, 0xEA, 0x01, 0x6C,
+0xD6, 0x17, 0xBD, 0x67, 0x20, 0xF0, 0xCC, 0xA5,
+0xF7, 0xF0, 0x01, 0x6A, 0x7D, 0x67, 0x20, 0xF0,
+0x6B, 0xA3, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xC0, 0x36, 0x6D, 0xEE, 0x04, 0x05,
+0x40, 0xEA, 0xF8, 0x4E, 0x0C, 0x95, 0xD1, 0x18,
+0xA8, 0x19, 0x0A, 0x04, 0xE1, 0x2A, 0xD8, 0x17,
+0x04, 0x02, 0x7D, 0x67, 0x51, 0xCB, 0x42, 0x32,
+0x42, 0x32, 0x52, 0xCB, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF0, 0x4C, 0x9A,
+0x40, 0xEA, 0x08, 0x04, 0xC9, 0x17, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0xFF, 0x6A, 0x7D, 0x67,
+0x49, 0xCB, 0xFF, 0x6A, 0x01, 0x24, 0x00, 0x6A,
+0x7D, 0x67, 0x48, 0xCB, 0x05, 0x6D, 0x00, 0x6A,
+0x04, 0x04, 0xB1, 0x18, 0xC2, 0xE8, 0x54, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x80, 0x6C,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xC9, 0xF1, 0x78, 0x9B, 0x6C, 0xEA, 0x07, 0x22,
+0x01, 0x6A, 0x7D, 0x67, 0x05, 0x6D, 0x04, 0x04,
+0xB1, 0x18, 0xC2, 0xE8, 0x54, 0xC3, 0x07, 0x97,
+0x00, 0xEF, 0x04, 0x63, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x8D, 0xA3, 0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x40, 0xF0, 0x61, 0xA2, 0x40, 0xF0, 0x00, 0xA2,
+0x40, 0xF0, 0x82, 0xA2, 0x60, 0x33, 0x0D, 0xEB,
+0x40, 0xF0, 0x03, 0xA2, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEB, 0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8,
+0x61, 0xA0, 0x20, 0xA0, 0x42, 0xA0, 0x60, 0x33,
+0x2D, 0xEB, 0x23, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEB, 0x00, 0xF6, 0x20, 0x31, 0x6D, 0xE9,
+0x01, 0x71, 0xB8, 0x67, 0x2C, 0x25, 0x02, 0x6A,
+0x4E, 0xE9, 0x24, 0x29, 0xCB, 0xA0, 0x4A, 0xA0,
+0x00, 0x6C, 0xC0, 0x36, 0x4D, 0xEE, 0x01, 0x6A,
+0x40, 0xC0, 0x21, 0xC0, 0x22, 0xC0, 0x23, 0xC0,
+0xD1, 0x18, 0xD6, 0x30, 0x06, 0xD6, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x85, 0xA0,
+0xD2, 0xF1, 0x74, 0x9A, 0x44, 0xA0, 0x80, 0x34,
+0x06, 0x96, 0x4D, 0xEC, 0x46, 0xA0, 0x00, 0x6F,
+0x04, 0x6D, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x87, 0xA0, 0x04, 0xD1, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xEB, 0x4D, 0xEC, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x29, 0xA0,
+0xC8, 0xA0, 0x02, 0x6A, 0x40, 0xC0, 0xA1, 0xC0,
+0xA2, 0xC0, 0xA3, 0xC0, 0x01, 0x6C, 0x20, 0x31,
+0xCD, 0xE9, 0xD1, 0x18, 0xD6, 0x30, 0x06, 0xD5,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF1, 0x54, 0x9A, 0x85, 0xA0, 0x06, 0x95,
+0x1A, 0x65, 0x44, 0xA0, 0x80, 0x34, 0x00, 0x6F,
+0x4D, 0xEC, 0x46, 0xA0, 0xD1, 0x67, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x87, 0xA0, 0x04, 0xD5,
+0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC, 0x58, 0x67,
+0x40, 0xEA, 0x04, 0x6D, 0xD3, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x10, 0xD4, 0x15, 0xF0, 0x08, 0x4B, 0x8D, 0xA3,
+0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x40, 0xF0,
+0x81, 0xA2, 0x40, 0xF0, 0x60, 0xA2, 0x40, 0xF0,
+0x23, 0xA2, 0x80, 0x34, 0x8D, 0xEB, 0x1B, 0x65,
+0x40, 0xF0, 0x62, 0xA2, 0x98, 0x67, 0x00, 0xF6,
+0x20, 0x31, 0x60, 0x33, 0x60, 0x33, 0x8D, 0xEB,
+0x6D, 0xE9, 0x9D, 0x67, 0x01, 0x6B, 0x00, 0x6A,
+0x7A, 0xC4, 0x79, 0xC4, 0x03, 0x67, 0x7D, 0x67,
+0x08, 0xD2, 0x5C, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0A, 0x6C, 0xD3, 0xF5,
+0x48, 0x9A, 0x98, 0xC3, 0x0C, 0x6C, 0x9B, 0xC3,
+0x00, 0x6F, 0x08, 0x06, 0x04, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0x0A, 0xD2, 0x0A, 0x02, 0xE0, 0xA2,
+0x06, 0x27, 0x04, 0x6A, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0x5D, 0x67,
+0x20, 0xF0, 0x08, 0xC2, 0x08, 0x90, 0x0B, 0xD7,
+0xF4, 0x20, 0x40, 0xA0, 0x08, 0x6B, 0x6B, 0xEB,
+0x10, 0x95, 0x6C, 0xEA, 0x87, 0x41, 0x40, 0xC0,
+0x08, 0x6E, 0xD1, 0x18, 0xE7, 0xF4, 0x01, 0x4C,
+0x7D, 0x67, 0x4C, 0xA1, 0x20, 0xF0, 0x68, 0xA3,
+0x4C, 0xEB, 0x0A, 0xD3, 0x0A, 0x02, 0x60, 0xA2,
+0x41, 0x23, 0x65, 0xA1, 0x84, 0xA1, 0x46, 0xA1,
+0x60, 0x33, 0x8D, 0xEB, 0x87, 0xA1, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xEB, 0x00, 0xF6, 0x80, 0x34,
+0x6D, 0xEC, 0x2C, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x0B, 0x97, 0x40, 0x32, 0xD2, 0xF1,
+0x54, 0x9A, 0x04, 0xD7, 0x00, 0x6E, 0x00, 0x6F,
+0x40, 0xEA, 0x03, 0x6D, 0x81, 0xA1, 0x40, 0xA1,
+0x62, 0xA1, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA1,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x02, 0x6B, 0x6E, 0xEA,
+0x08, 0x2A, 0x01, 0x6B, 0x60, 0xC1, 0x41, 0xC1,
+0x42, 0xC1, 0x43, 0xC1, 0xD1, 0x18, 0xD6, 0x30,
+0x00, 0x6C, 0x08, 0x95, 0xD1, 0x18, 0xA8, 0x19,
+0x06, 0x04, 0xFF, 0x6B, 0x6C, 0xEA, 0xA9, 0x2A,
+0x00, 0x6A, 0xA8, 0x17, 0x20, 0xA0, 0x08, 0x6A,
+0x4B, 0xEA, 0x4C, 0xE9, 0x04, 0x6A, 0x4D, 0xE9,
+0x20, 0xC0, 0xEF, 0x17, 0xAB, 0xA1, 0x4A, 0xA1,
+0x60, 0xA1, 0xA0, 0x35, 0x4D, 0xED, 0x41, 0xA1,
+0x82, 0xA1, 0x40, 0x32, 0x6D, 0xEA, 0x63, 0xA1,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x4D, 0xEB, 0x01, 0x73, 0xF8, 0x67,
+0x6D, 0x27, 0x28, 0x23, 0x02, 0x73, 0xF8, 0x67,
+0xD8, 0x2F, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0xC9, 0xA1, 0x40, 0x32, 0xD2, 0xF1, 0x74, 0x9A,
+0x48, 0xA1, 0x85, 0xA1, 0xC0, 0x36, 0x4D, 0xEE,
+0x44, 0xA1, 0x80, 0x34, 0xFF, 0xF7, 0x1F, 0x6D,
+0x4D, 0xEC, 0x46, 0xA1, 0xAC, 0xEE, 0x04, 0x6D,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x87, 0xA1,
+0x04, 0xD7, 0x00, 0x6F, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xEB, 0x4D, 0xEC, 0x01, 0x72, 0xB9, 0x60,
+0x20, 0xA0, 0x08, 0x6A, 0x4B, 0xEA, 0x4C, 0xE9,
+0x03, 0x6A, 0xC1, 0x17, 0x97, 0xF0, 0x19, 0x6C,
+0x80, 0x34, 0xF7, 0xF0, 0x01, 0x6A, 0x80, 0x34,
+0xB8, 0xF3, 0x0D, 0x4C, 0x40, 0x32, 0x04, 0xD4,
+0x40, 0x32, 0xF7, 0xF0, 0x01, 0x6C, 0xD2, 0xF1,
+0x58, 0x9A, 0x80, 0x34, 0x80, 0x34, 0x00, 0x6F,
+0x00, 0x6E, 0x4A, 0xF2, 0x18, 0x4C, 0x40, 0xEA,
+0x0A, 0xD3, 0x82, 0x67, 0x44, 0xC1, 0x42, 0x32,
+0x45, 0xC1, 0x42, 0x32, 0x46, 0xC1, 0x00, 0xF6,
+0x82, 0x32, 0x47, 0xC1, 0x01, 0x6A, 0x0A, 0x93,
+0x40, 0xC1, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x61, 0xC1, 0x62, 0xC1, 0x63, 0xC1,
+0xD2, 0xF1, 0x34, 0x9A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x92, 0xF2, 0x4C, 0x9A,
+0x40, 0xEA, 0x0B, 0xD4, 0x0A, 0x93, 0x0B, 0x94,
+0x00, 0x6F, 0x04, 0xD3, 0xC2, 0x67, 0x40, 0xE9,
+0x01, 0x6D, 0x01, 0x72, 0x7F, 0xF7, 0x15, 0x60,
+0x20, 0xA0, 0x08, 0x6A, 0x4B, 0xEA, 0x4C, 0xE9,
+0x02, 0x6A, 0x7D, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x85, 0xA1, 0x40, 0x32, 0xD2, 0xF1,
+0x74, 0x9A, 0x44, 0xA1, 0x80, 0x34, 0xC5, 0x67,
+0x4D, 0xEC, 0x46, 0xA1, 0x04, 0x6D, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x87, 0xA1, 0x04, 0xD7,
+0x00, 0x6F, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xEB,
+0x4D, 0xEC, 0x01, 0x72, 0x5F, 0xF7, 0x15, 0x60,
+0x08, 0x69, 0x40, 0xA0, 0x2B, 0xE9, 0x9B, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x15, 0xF0, 0x08, 0x48, 0x4D, 0xA0, 0x2C, 0xA0,
+0x6E, 0xA0, 0x40, 0x32, 0x2D, 0xEA, 0x2F, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6,
+0x20, 0x31, 0x4D, 0xE9, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0x89, 0xF3, 0xA0, 0x9B,
+0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x10, 0x6C,
+0x42, 0x33, 0x40, 0xF0, 0x40, 0xC1, 0x40, 0xF0,
+0x61, 0xC1, 0x00, 0xF6, 0x42, 0x32, 0x62, 0x33,
+0x40, 0xF0, 0x62, 0xC1, 0x40, 0xF0, 0x43, 0xC1,
+0x8D, 0xA0, 0x6C, 0xA0, 0x4E, 0xA0, 0x80, 0x34,
+0x6D, 0xEC, 0x6F, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x40, 0xF0, 0x81, 0xA3, 0x40, 0xF0, 0x40, 0xA3,
+0x40, 0xF0, 0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x40, 0xF0, 0x43, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x1A, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x4A, 0xF2, 0x00, 0x4A, 0x05, 0xD2, 0x20, 0xF5,
+0x14, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x00, 0x6B,
+0x60, 0xC2, 0x61, 0xC2, 0x62, 0xC2, 0x63, 0xC2,
+0x64, 0xC2, 0x65, 0xC2, 0x66, 0xC2, 0x67, 0xC2,
+0x6A, 0xC2, 0x6B, 0xC2, 0x68, 0xC2, 0x69, 0xC2,
+0x6C, 0xA2, 0x02, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB,
+0x6C, 0xC2, 0xE8, 0x17, 0x20, 0xE8, 0x00, 0x6A,
+0x20, 0xE8, 0x04, 0x6A, 0x20, 0xE8, 0x04, 0x6A,
+0x20, 0xE8, 0x04, 0x6A, 0x20, 0xE8, 0x00, 0x65,
+0xE0, 0xA4, 0x01, 0x6E, 0x04, 0x6A, 0xEE, 0x35,
+0xCC, 0xED, 0x4B, 0x2D, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x07, 0x6D, 0xFF, 0x69,
+0xEC, 0xED, 0x2C, 0xED, 0x05, 0x55, 0x38, 0x60,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xF2, 0x32, 0x8C, 0xF2, 0x0C, 0x48, 0xCC, 0xEA,
+0x15, 0xE5, 0x40, 0xC5, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0x7D, 0x67, 0x05, 0xD4, 0x99, 0xF6, 0x00, 0x6C,
+0x40, 0xEA, 0xD0, 0xC3, 0x05, 0x93, 0x9D, 0x67,
+0xD0, 0xA4, 0x60, 0xA3, 0x6E, 0x33, 0xCC, 0xEB,
+0x2C, 0xEB, 0x68, 0x34, 0x6D, 0xE4, 0x00, 0x6E,
+0x00, 0x6C, 0x75, 0xE4, 0xB5, 0xE0, 0xE0, 0xA5,
+0x01, 0x4C, 0x05, 0x74, 0xFF, 0x6D, 0xED, 0xEE,
+0xF8, 0x61, 0x13, 0x26, 0x01, 0x6B, 0x6D, 0xEA,
+0x4C, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x40, 0x9A, 0x99, 0xF6,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x00, 0x6A,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x02, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0xEB, 0x17, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xA5, 0xA3, 0x44, 0xA3,
+0xC6, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0, 0xA1, 0xA2,
+0x20, 0xF0, 0x60, 0xA2, 0x20, 0xF0, 0xC2, 0xA2,
+0xA0, 0x35, 0xAD, 0xEB, 0x20, 0xF0, 0xA3, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0x6D, 0xEE, 0x00, 0xF6,
+0xA0, 0x35, 0xAD, 0xEE, 0x88, 0x33, 0xCD, 0xE3,
+0xA5, 0xA3, 0x44, 0xA3, 0xC6, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x47, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x61, 0xA2, 0x22, 0xA2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x68, 0x30, 0x40, 0x32, 0x61, 0xE0,
+0x8C, 0xF2, 0x0C, 0x4A, 0x49, 0xE0, 0x29, 0xE2,
+0x40, 0xA2, 0x29, 0x22, 0xF7, 0xF0, 0x01, 0x6D,
+0xA0, 0x35, 0xA0, 0x35, 0x0A, 0xD4, 0x14, 0x6E,
+0x28, 0xF3, 0x0C, 0x4D, 0xD1, 0x18, 0xE7, 0xF4,
+0x04, 0x04, 0x21, 0xE0, 0xF7, 0xF0, 0x01, 0x6A,
+0x04, 0x03, 0x40, 0x32, 0x08, 0x30, 0x01, 0xE3,
+0x40, 0x32, 0x80, 0x98, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x00, 0xF6, 0x42, 0x34,
+0x01, 0x74, 0x0A, 0x93, 0x0C, 0x61, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x12, 0xF6,
+0xC0, 0x9C, 0x42, 0x32, 0x42, 0x32, 0xFF, 0x6D,
+0x4C, 0xED, 0x40, 0xEE, 0x83, 0x67, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0x36, 0x2C, 0x05, 0x5D, 0x34, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x8C, 0xF2,
+0x0C, 0x4A, 0x49, 0xE5, 0x40, 0xA2, 0x2B, 0x22,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x05, 0x67, 0x06, 0xD4, 0x00, 0x6F,
+0x06, 0x06, 0x40, 0xEA, 0x14, 0x6D, 0x17, 0x2A,
+0x06, 0x95, 0x07, 0x6C, 0x0C, 0xEC, 0x60, 0xA5,
+0x10, 0x68, 0x0B, 0xE8, 0x84, 0x34, 0x6C, 0xE8,
+0x8D, 0xE8, 0x01, 0x6B, 0x9D, 0x67, 0x00, 0xC5,
+0x72, 0xC4, 0x71, 0xC4, 0x0C, 0x6B, 0x70, 0xC4,
+0x1C, 0x6B, 0x73, 0xC4, 0x7D, 0x67, 0x04, 0x04,
+0xD1, 0x18, 0xA8, 0x19, 0x54, 0xC3, 0x09, 0x97,
+0x08, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x25, 0x24, 0xFD, 0x63,
+0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x39, 0xF6, 0x0C, 0x6C, 0x40, 0xEA, 0x05, 0x67,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x89, 0xF3, 0xA4, 0x9B, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0xAC, 0xEA, 0x60, 0x33, 0x00, 0x35,
+0xD2, 0xF4, 0x68, 0x9B, 0xA0, 0x35, 0x0D, 0xED,
+0x39, 0xF6, 0x0C, 0x6C, 0x40, 0xEB, 0x4D, 0xED,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0x20, 0xE8, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x00, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xAD, 0xF0, 0x14, 0x4A, 0x3C, 0x65, 0x94, 0x9A,
+0x04, 0x75, 0x08, 0xD4, 0x2E, 0x60, 0x06, 0xAA,
+0x59, 0x67, 0x40, 0x31, 0x28, 0x31, 0x23, 0xE8,
+0x65, 0x67, 0x08, 0x61, 0x00, 0xF4, 0x40, 0x41,
+0x43, 0xE8, 0x23, 0xE0, 0x05, 0x61, 0x00, 0xF4,
+0x00, 0x68, 0x02, 0x10, 0x30, 0x67, 0x00, 0x68,
+0x59, 0x67, 0x03, 0x72, 0x01, 0x60, 0x02, 0x6B,
+0x59, 0x67, 0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A,
+0xA7, 0x40, 0x00, 0x6F, 0x06, 0x06, 0x11, 0x4D,
+0x00, 0x6C, 0x40, 0xEA, 0x0A, 0xD3, 0x09, 0x93,
+0x3B, 0x65, 0x0A, 0x93, 0x0A, 0x22, 0x14, 0x6A,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0x00, 0x69, 0x00, 0x68, 0x03, 0x6B,
+0xE3, 0x17, 0x06, 0x94, 0xF4, 0x24, 0x59, 0x67,
+0x40, 0xC4, 0x59, 0x67, 0xFF, 0x6F, 0x42, 0x32,
+0xEC, 0xEA, 0x41, 0xC4, 0x00, 0x6A, 0x42, 0xC4,
+0x59, 0x67, 0x00, 0xF6, 0x42, 0x32, 0x43, 0xC4,
+0x02, 0x32, 0x45, 0xC4, 0x42, 0x32, 0x46, 0xC4,
+0x00, 0xF6, 0x02, 0x32, 0x47, 0xC4, 0x22, 0x32,
+0x49, 0xC4, 0x42, 0x32, 0x4A, 0xC4, 0x00, 0xF6,
+0x22, 0x32, 0x4B, 0xC4, 0x62, 0x32, 0x4D, 0xC4,
+0x42, 0x32, 0x4E, 0xC4, 0xF7, 0xF0, 0x01, 0x6A,
+0x6C, 0xC4, 0x40, 0x32, 0x00, 0xF6, 0x62, 0x33,
+0x6F, 0xC4, 0x40, 0x32, 0x08, 0x93, 0x73, 0xF6,
+0x5C, 0x9A, 0x04, 0xC4, 0x28, 0xC4, 0x35, 0xE3,
+0xD0, 0x67, 0x14, 0x4C, 0x40, 0xEA, 0x09, 0xD7,
+0x01, 0x6A, 0x7D, 0x67, 0x52, 0xC3, 0x51, 0xC3,
+0x09, 0x97, 0x16, 0x6A, 0x50, 0xC3, 0x20, 0x48,
+0xFF, 0xF7, 0x1F, 0x6A, 0x06, 0x95, 0x4C, 0xE8,
+0x0C, 0xEF, 0x5D, 0x67, 0x02, 0x30, 0x04, 0x04,
+0xF3, 0xC2, 0xD1, 0x18, 0xA8, 0x19, 0x14, 0xC2,
+0xAE, 0x2A, 0xAE, 0x17, 0xEE, 0x63, 0x23, 0x62,
+0x22, 0xD1, 0x21, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x24, 0xD4, 0x25, 0xD5, 0x08, 0x04,
+0x00, 0x6D, 0x20, 0x31, 0x06, 0x67, 0xD1, 0x18,
+0xD2, 0xF4, 0x40, 0x6E, 0xD2, 0xF4, 0x54, 0x99,
+0xD8, 0xF5, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x64, 0x67, 0xAD, 0xF0, 0x00, 0x4B, 0x41, 0xDB,
+0x19, 0xD3, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xAD, 0xF0, 0x14, 0x4B, 0x40, 0xF0,
+0x54, 0xA3, 0x1A, 0xD1, 0x1B, 0xD4, 0x01, 0x72,
+0x15, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF7, 0xF0, 0x01, 0x6C, 0x72, 0xF1,
+0x5C, 0x9A, 0x80, 0x34, 0x80, 0x34, 0x6A, 0xF2,
+0x08, 0x4C, 0x40, 0xEA, 0x00, 0x65, 0x00, 0x6E,
+0x23, 0x97, 0x22, 0x91, 0x21, 0x90, 0x46, 0x67,
+0x00, 0xEF, 0x12, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x69, 0x40, 0x32, 0x20, 0x31,
+0x20, 0x31, 0x40, 0x32, 0x52, 0xF3, 0x5C, 0x9A,
+0x0D, 0xF1, 0x94, 0x99, 0x00, 0x6D, 0x40, 0xEA,
+0x18, 0xD3, 0x01, 0x72, 0x1D, 0xD1, 0x00, 0x6E,
+0x18, 0x93, 0xE6, 0x61, 0x1E, 0xD3, 0x24, 0x93,
+0xF7, 0xF0, 0x01, 0x69, 0x4A, 0x40, 0x20, 0x31,
+0x20, 0x31, 0x69, 0xE2, 0x18, 0xD2, 0x67, 0x40,
+0x24, 0x95, 0x73, 0xF6, 0x5C, 0x99, 0x01, 0x4B,
+0xC3, 0x67, 0x08, 0x04, 0x40, 0xEA, 0x1C, 0xD3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x18, 0x94, 0x93, 0xF6, 0x40, 0x9A, 0x0E, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x18, 0x94, 0x73, 0xF6,
+0x5C, 0x99, 0x06, 0x6E, 0x40, 0xEA, 0x09, 0x05,
+0xF7, 0xF0, 0x01, 0x6C, 0x73, 0xF6, 0x5C, 0x99,
+0x80, 0x34, 0x80, 0x34, 0x20, 0xF0, 0x0A, 0x05,
+0x8D, 0xF0, 0x14, 0x4C, 0x40, 0xEA, 0x06, 0x6E,
+0x18, 0x93, 0x73, 0xF6, 0x5C, 0x99, 0x06, 0x6E,
+0x86, 0x43, 0x40, 0xEA, 0x0C, 0x05, 0xA6, 0x40,
+0x08, 0x03, 0xB5, 0xE3, 0x18, 0x93, 0x73, 0xF6,
+0x5C, 0x99, 0x02, 0x6E, 0x87, 0x43, 0x40, 0xEA,
+0x05, 0x4C, 0x25, 0x92, 0x40, 0x6C, 0x1E, 0x93,
+0x03, 0xE2, 0x5D, 0x67, 0x20, 0xF0, 0x41, 0xA2,
+0xAA, 0x40, 0x8C, 0xEA, 0x01, 0x2A, 0xA2, 0x40,
+0x1A, 0x92, 0xD8, 0xF5, 0x00, 0x6C, 0x1F, 0xD3,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x1E, 0xD5,
+0x02, 0x67, 0x1A, 0x92, 0xD8, 0xF5, 0x04, 0x6C,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x1F, 0x93, 0x1E, 0x95, 0x40, 0xF0, 0xD6, 0xA3,
+0x40, 0xF0, 0x55, 0xA3, 0x40, 0xF0, 0x97, 0xA3,
+0xC0, 0x36, 0x4D, 0xEE, 0x40, 0xF0, 0x58, 0xA3,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x4B, 0xE0, 0xE0, 0xF3,
+0x08, 0x6C, 0x9B, 0xEA, 0x01, 0x2C, 0xE5, 0xE8,
+0x7D, 0x6C, 0xE6, 0xAB, 0xC7, 0xAB, 0x05, 0xD5,
+0x12, 0xEA, 0x50, 0x32, 0x9B, 0xEA, 0x01, 0x2C,
+0xE5, 0xE8, 0x00, 0x6C, 0x12, 0xEA, 0x5C, 0x32,
+0x06, 0xD2, 0x18, 0x92, 0x04, 0xD2, 0xB4, 0x9B,
+0xD1, 0x18, 0x43, 0xE1, 0x18, 0xD3, 0x02, 0x67,
+0x1D, 0x93, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0D, 0xF1, 0x94, 0x9B, 0x72, 0xF3,
+0x4C, 0x9A, 0x00, 0x6E, 0x00, 0x6F, 0x40, 0xEA,
+0x00, 0x6D, 0x00, 0x6E, 0x18, 0x93, 0x09, 0x20,
+0x1C, 0x96, 0x73, 0xF6, 0x5C, 0x99, 0x24, 0x94,
+0x03, 0x67, 0x40, 0xEA, 0x08, 0x05, 0x01, 0x6E,
+0x70, 0x67, 0x9A, 0xA3, 0x5F, 0xF7, 0x04, 0x24,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x8D, 0xF0, 0x5C, 0x98, 0x25, 0x95, 0xA9, 0xE2,
+0x8D, 0xF0, 0x5C, 0xD8, 0x19, 0x92, 0x41, 0x9A,
+0xA2, 0x67, 0x1B, 0x92, 0xAD, 0xF0, 0x40, 0x9A,
+0xE2, 0x67, 0x45, 0x67, 0xF7, 0xF0, 0x01, 0x6D,
+0xA0, 0x35, 0xA0, 0x35, 0x89, 0xF3, 0xA8, 0x9D,
+0xEB, 0xE2, 0xB8, 0xEC, 0x12, 0xEC, 0x83, 0xEA,
+0x3F, 0xF7, 0x06, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x50, 0x9A,
+0xF7, 0xF0, 0x01, 0x69, 0xF9, 0xF6, 0x02, 0x6C,
+0x20, 0x31, 0x1C, 0xD3, 0x18, 0xD6, 0x40, 0xEA,
+0x20, 0x31, 0xAD, 0xF0, 0x48, 0xD9, 0x1A, 0x92,
+0xF9, 0xF6, 0x00, 0x6C, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xD2, 0xF4, 0xE8, 0x9C,
+0xFF, 0x6D, 0x01, 0x4D, 0xF9, 0xF6, 0x00, 0x6C,
+0x40, 0xEF, 0x4D, 0xED, 0xAD, 0xF0, 0x48, 0x99,
+0xF1, 0x67, 0xAD, 0xF0, 0x08, 0x4F, 0x18, 0x96,
+0x1C, 0x93, 0x02, 0x2A, 0x41, 0x9F, 0x16, 0x22,
+0x42, 0x9F, 0x99, 0xA3, 0x01, 0x6D, 0x01, 0x4A,
+0x1C, 0xD6, 0x42, 0xDF, 0x1A, 0xD7, 0xD1, 0x18,
+0x93, 0x32, 0x18, 0xD3, 0x04, 0x6D, 0xD1, 0x18,
+0xA7, 0x32, 0x00, 0x6C, 0x18, 0x93, 0x03, 0x6A,
+0x1C, 0x96, 0x1A, 0x97, 0x40, 0xF0, 0x54, 0xC3,
+0x00, 0x6A, 0x42, 0xC3, 0x19, 0x93, 0x00, 0x6A,
+0xAD, 0xF0, 0x48, 0xD9, 0x61, 0x9B, 0x41, 0xDF,
+0x8D, 0xF0, 0x5C, 0xD8, 0x83, 0x67, 0x1B, 0x93,
+0xAD, 0xF0, 0x80, 0xDB, 0xD5, 0x16, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x00, 0x6B,
+0x40, 0x32, 0xAD, 0xF0, 0x68, 0xDA, 0xAD, 0xF0,
+0x08, 0x4A, 0x61, 0xDA, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x54, 0x9B,
+0x24, 0x67, 0xF9, 0xF6, 0x00, 0x6C, 0x40, 0xEA,
+0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6C, 0x40, 0x6D,
+0x80, 0x34, 0xAB, 0xED, 0x80, 0x34, 0xAC, 0xEA,
+0xD2, 0xF4, 0xC8, 0x9C, 0xFF, 0x6D, 0xF7, 0xF0,
+0x01, 0x68, 0x22, 0x4D, 0x00, 0x30, 0x4D, 0xED,
+0xF9, 0xF6, 0x00, 0x6C, 0x40, 0xEE, 0x00, 0x30,
+0x04, 0x93, 0x0D, 0xF1, 0x54, 0x98, 0x04, 0xD0,
+0x07, 0xD3, 0x0B, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF3, 0x54, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0x04, 0x93, 0x0D, 0xF1,
+0x54, 0xDB, 0x04, 0x93, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF3, 0x5C, 0x9A,
+0x0D, 0xF1, 0x94, 0x9B, 0x40, 0xEA, 0x00, 0x6D,
+0x01, 0x72, 0x04, 0x6A, 0x43, 0x61, 0xF7, 0xF0,
+0x01, 0x68, 0x59, 0xA1, 0x00, 0x30, 0x00, 0x30,
+0xAD, 0xF0, 0x14, 0x48, 0x59, 0xC0, 0x4F, 0xA1,
+0x8E, 0xA1, 0x00, 0x6B, 0x40, 0x32, 0x8D, 0xEA,
+0x05, 0xD3, 0x0C, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0x20, 0x6E, 0xB1, 0x67, 0x40, 0xEA, 0x90, 0x67,
+0x01, 0x6A, 0x05, 0xD2, 0x40, 0xF0, 0x54, 0xA0,
+0x3F, 0x2A, 0x62, 0xA1, 0xFD, 0x4A, 0x6C, 0xEA,
+0x01, 0x72, 0x3A, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6C, 0x40, 0x32, 0x80, 0x34,
+0x80, 0x34, 0x40, 0x32, 0x89, 0xF3, 0xAC, 0x9C,
+0x72, 0xF4, 0x54, 0x9A, 0x02, 0xF0, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x54, 0xD8, 0x13, 0x2A,
+0x04, 0x93, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF3, 0x4C, 0x9A, 0x0D, 0xF1,
+0x94, 0x9B, 0x00, 0x6F, 0x00, 0x6E, 0x40, 0xEA,
+0x00, 0x6D, 0x04, 0x6A, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x93, 0xF6,
+0xE0, 0x9C, 0x02, 0xF0, 0x00, 0x6E, 0x00, 0x6D,
+0x40, 0xEF, 0x82, 0x67, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0x20, 0x6E, 0xB1, 0x67, 0x40, 0xEA, 0x90, 0x67,
+0x42, 0xA1, 0x01, 0x72, 0x69, 0x60, 0x02, 0x72,
+0x20, 0x61, 0x40, 0xF0, 0x74, 0xA0, 0xFB, 0x4A,
+0x6C, 0xEA, 0x01, 0x72, 0x0B, 0x61, 0x00, 0x69,
+0x40, 0xF0, 0xB4, 0xA0, 0x91, 0x67, 0xD1, 0x18,
+0xA7, 0x32, 0x01, 0x49, 0xFF, 0x6A, 0x4C, 0xE9,
+0x04, 0x71, 0xF6, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x04, 0x93, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF3,
+0x4C, 0x9A, 0x0D, 0xF1, 0x94, 0x9B, 0x00, 0x6F,
+0x00, 0x6E, 0x40, 0xEA, 0x00, 0x6D, 0x00, 0x6A,
+0xC1, 0x17, 0x03, 0x72, 0x09, 0x61, 0x99, 0xA0,
+0xD1, 0x18, 0x93, 0x32, 0x01, 0x6D, 0x03, 0x6A,
+0x40, 0xF0, 0x54, 0xC0, 0x42, 0xC0, 0xE6, 0x17,
+0x04, 0x72, 0x0C, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x94, 0x98, 0x93, 0xF6,
+0x40, 0x9A, 0x02, 0xF0, 0x00, 0x6E, 0x40, 0xEA,
+0x00, 0x6D, 0xD8, 0x17, 0x05, 0x72, 0x79, 0x61,
+0x4F, 0xA1, 0x8E, 0xA1, 0x40, 0x32, 0x8D, 0xEA,
+0x93, 0x22, 0x47, 0xC8, 0x07, 0x92, 0xD8, 0xF5,
+0x00, 0x6C, 0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x42, 0x34, 0x40, 0xF0, 0x55, 0xC0,
+0x00, 0xF6, 0x42, 0x32, 0x40, 0xF0, 0x58, 0xC0,
+0x07, 0x92, 0x40, 0xF0, 0x96, 0xC0, 0x82, 0x34,
+0xD2, 0xF4, 0x54, 0x9A, 0x40, 0xF0, 0x97, 0xC0,
+0xD8, 0xF5, 0x04, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x42, 0x34, 0x40, 0xF0, 0x59, 0xC0, 0x40, 0xF0,
+0x9A, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x40, 0xF0, 0x9B, 0xC0, 0x40, 0xF0, 0x5C, 0xC0,
+0x44, 0xA1, 0xFF, 0x72, 0x1B, 0x60, 0x8C, 0x98,
+0x0A, 0x24, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x00, 0x6A, 0x4C, 0xD8, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x84, 0xA1, 0x40, 0x32,
+0xF2, 0xF7, 0x44, 0x9A, 0xC7, 0x40, 0xA7, 0x40,
+0x84, 0xC0, 0x29, 0x4E, 0x40, 0xEA, 0x19, 0x4D,
+0x5F, 0xF7, 0x0E, 0x22, 0x05, 0x92, 0x01, 0x72,
+0x65, 0x60, 0x42, 0xA1, 0x01, 0x72, 0x07, 0x61,
+0x99, 0xA0, 0xD1, 0x18, 0x93, 0x32, 0x02, 0x6D,
+0x01, 0x6A, 0x40, 0xF0, 0x54, 0xC0, 0x04, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0D, 0xF1, 0x94, 0x9B, 0x72, 0xF3, 0x4C, 0x9A,
+0x00, 0x6F, 0x00, 0x6E, 0x40, 0xEA, 0x00, 0x6D,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x00, 0x6B,
+0x40, 0x32, 0x8D, 0xF0, 0x7C, 0xDA, 0x07, 0x92,
+0xD8, 0xF5, 0x00, 0x6C, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xAD, 0xF0, 0x40, 0xDB,
+0x6A, 0x17, 0xB2, 0x2A, 0x40, 0xF0, 0x94, 0xA0,
+0x1F, 0xF7, 0x1A, 0x24, 0x99, 0xA0, 0x22, 0x67,
+0xD1, 0x18, 0x93, 0x32, 0x01, 0x6D, 0x40, 0xF0,
+0x34, 0xC0, 0x00, 0x69, 0x91, 0x67, 0xD1, 0x18,
+0xA7, 0x32, 0x00, 0x6D, 0x01, 0x49, 0xFF, 0x6A,
+0x4C, 0xE9, 0x04, 0x71, 0xF7, 0x61, 0x94, 0x98,
+0x08, 0x24, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x00, 0x6A, 0x54, 0xD8, 0x00, 0x69,
+0x24, 0x32, 0x49, 0xE0, 0x8C, 0x9A, 0x09, 0x24,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0x72, 0xF4, 0xB0, 0x9D, 0x40, 0xED, 0x05, 0xD2,
+0x05, 0x92, 0x00, 0x6C, 0x8C, 0xDA, 0x29, 0xE0,
+0x02, 0x49, 0x10, 0x71, 0x90, 0xCA, 0xEC, 0x61,
+0x00, 0x6A, 0x3C, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0x10, 0x6E, 0xB1, 0x67, 0x40, 0xEA, 0x90, 0x67,
+0x54, 0x98, 0x67, 0x40, 0xE7, 0x40, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x2D, 0x4B, 0x1B, 0x4F, 0x08, 0xD2, 0x43, 0xA0,
+0x05, 0x94, 0x82, 0xEA, 0x82, 0x61, 0x05, 0x92,
+0x49, 0xE0, 0x84, 0xA2, 0xFF, 0x74, 0xDF, 0xF6,
+0x07, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF7, 0x44, 0x9A, 0xC3, 0x67,
+0xA7, 0x67, 0x0A, 0xD3, 0x40, 0xEA, 0x09, 0xD7,
+0x09, 0x97, 0x0A, 0x93, 0xBF, 0xF6, 0x18, 0x22,
+0x08, 0x92, 0xC0, 0xAF, 0xA0, 0x9B, 0x06, 0x94,
+0x73, 0xF6, 0x5C, 0x9A, 0x0A, 0xD7, 0x40, 0xEA,
+0x09, 0xD3, 0x0A, 0x97, 0x06, 0x93, 0x40, 0xAF,
+0x49, 0xE3, 0x06, 0xD2, 0x09, 0x93, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x80, 0x9B, 0x0A, 0x97,
+0x09, 0x93, 0x00, 0x6A, 0x40, 0xCF, 0x00, 0x6A,
+0x40, 0xDB, 0x05, 0x92, 0x04, 0x4B, 0x02, 0x4F,
+0x01, 0x4A, 0x05, 0xD2, 0xC4, 0x17, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x04, 0x67,
+0x61, 0xA0, 0xF7, 0xF0, 0x01, 0x6E, 0xF7, 0xF0,
+0x01, 0x6D, 0xF7, 0xF0, 0x01, 0x6C, 0xC0, 0x36,
+0xA0, 0x35, 0x80, 0x34, 0x00, 0x6A, 0xC0, 0x36,
+0xA0, 0x35, 0x80, 0x34, 0x37, 0x5B, 0x96, 0xF6,
+0x4E, 0xCE, 0x96, 0xF6, 0x4C, 0xC5, 0x8C, 0xF2,
+0x51, 0xC4, 0xE0, 0xF0, 0x11, 0x60, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x68, 0x33,
+0x48, 0xF3, 0x00, 0x4A, 0x49, 0xE3, 0x40, 0x9A,
+0x00, 0xEA, 0x00, 0x65, 0x49, 0xA0, 0x00, 0xF7,
+0x00, 0x6F, 0x40, 0x33, 0xEC, 0xEB, 0xE8, 0xA0,
+0xED, 0xEB, 0x96, 0xF6, 0x6E, 0xCE, 0x08, 0x6B,
+0x4C, 0xEB, 0x96, 0xF6, 0x6C, 0xC5, 0x10, 0x6B,
+0x6C, 0xEA, 0x8C, 0xF2, 0x51, 0xC4, 0xD1, 0x18,
+0xCD, 0x8A, 0x00, 0x65, 0x02, 0x2A, 0x04, 0x6A,
+0x08, 0x10, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF7, 0x58, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x05, 0x97, 0x04, 0x90, 0x00, 0xEF,
+0x03, 0x63, 0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65,
+0xEE, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF7, 0x48, 0x9A, 0x87, 0x40,
+0x40, 0xEA, 0x01, 0x4C, 0xEE, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF2, 0xF7,
+0xD8, 0x9A, 0x45, 0xA0, 0xA4, 0xA0, 0x3F, 0x6B,
+0x6C, 0xEA, 0x40, 0x32, 0xAD, 0xEA, 0x87, 0x40,
+0xF8, 0x4A, 0xFF, 0xF7, 0x1F, 0x6D, 0x4C, 0xED,
+0x40, 0xEE, 0x01, 0x4C, 0xDA, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF2, 0xF7,
+0xD4, 0x9A, 0xEB, 0x17, 0x87, 0x40, 0x00, 0x6D,
+0xD1, 0x18, 0x3E, 0x2E, 0x01, 0x4C, 0xCD, 0x17,
+0x87, 0x40, 0x22, 0x6D, 0xD1, 0x18, 0x7F, 0x2E,
+0x01, 0x4C, 0xC7, 0x17, 0x87, 0x40, 0xD1, 0x18,
+0xB3, 0x2D, 0x01, 0x4C, 0xC2, 0x17, 0x87, 0x40,
+0xD1, 0x18, 0x92, 0x2D, 0x01, 0x4C, 0xBD, 0x17,
+0x87, 0x40, 0xD1, 0x18, 0x7C, 0x2D, 0x01, 0x4C,
+0xB8, 0x17, 0x87, 0x40, 0xD1, 0x18, 0x40, 0x2D,
+0x01, 0x4C, 0xB3, 0x17, 0x87, 0x40, 0xD1, 0x18,
+0xEF, 0x2C, 0x01, 0x4C, 0xAE, 0x17, 0x87, 0x40,
+0xD1, 0x18, 0xC6, 0x2C, 0x01, 0x4C, 0xA9, 0x17,
+0x87, 0x40, 0xD1, 0x18, 0xF5, 0x93, 0x01, 0x4C,
+0xA4, 0x17, 0x87, 0x40, 0xD1, 0x18, 0xBA, 0x93,
+0x01, 0x4C, 0x9F, 0x17, 0xA5, 0xA0, 0x44, 0xA0,
+0x3F, 0x6B, 0x6C, 0xED, 0xA0, 0x35, 0x4D, 0xED,
+0x00, 0x6E, 0x87, 0x40, 0xB5, 0xE0, 0xD1, 0x18,
+0x34, 0x2F, 0x01, 0x4C, 0x92, 0x17, 0xA5, 0xA0,
+0x44, 0xA0, 0x3F, 0x6B, 0x6C, 0xED, 0xA0, 0x35,
+0x4D, 0xED, 0x01, 0x6E, 0xF2, 0x17, 0xA5, 0xA0,
+0x44, 0xA0, 0x3F, 0x6B, 0x6C, 0xED, 0xA0, 0x35,
+0x4D, 0xED, 0x00, 0x6E, 0x87, 0x40, 0xB5, 0xE0,
+0xD1, 0x18, 0x15, 0x30, 0x01, 0x4C, 0x7D, 0x17,
+0xA5, 0xA0, 0x44, 0xA0, 0x3F, 0x6B, 0x6C, 0xED,
+0xA0, 0x35, 0x4D, 0xED, 0x01, 0x6E, 0xF2, 0x17,
+0x87, 0x40, 0xD1, 0x18, 0x35, 0x53, 0x01, 0x4C,
+0x70, 0x17, 0x87, 0x40, 0xD1, 0x18, 0x14, 0x58,
+0x01, 0x4C, 0x6B, 0x17, 0x87, 0x40, 0xD1, 0x18,
+0xC5, 0x52, 0x01, 0x4C, 0x66, 0x17, 0x87, 0x40,
+0xD1, 0x18, 0x30, 0x31, 0x01, 0x4C, 0x61, 0x17,
+0x87, 0x40, 0xD1, 0x18, 0xF0, 0x2D, 0x01, 0x4C,
+0x5C, 0x17, 0x87, 0x40, 0xB1, 0x18, 0x31, 0xEC,
+0x01, 0x4C, 0x57, 0x17, 0xD1, 0x18, 0xCD, 0x8A,
+0x00, 0x65, 0x5F, 0xF7, 0x08, 0x22, 0x87, 0x40,
+0xD1, 0x18, 0x9C, 0x91, 0x01, 0x4C, 0x4D, 0x17,
+0x87, 0x40, 0xD1, 0x18, 0xF5, 0x9C, 0x01, 0x4C,
+0x48, 0x17, 0x87, 0x40, 0xB1, 0x18, 0xB3, 0xE9,
+0x01, 0x4C, 0x43, 0x17, 0x87, 0x40, 0xD1, 0x18,
+0x14, 0x32, 0x01, 0x4C, 0x3E, 0x17, 0x87, 0x40,
+0xD1, 0x18, 0xA0, 0x33, 0x01, 0x4C, 0x39, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x89, 0xF1, 0xB4, 0x9B,
+0x40, 0xEA, 0x00, 0x6C, 0x20, 0x17, 0x00, 0x6A,
+0x28, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x85, 0x67, 0x72, 0xF4, 0x54, 0x9A, 0x89, 0xF3,
+0xAC, 0x9B, 0x40, 0xEA, 0x00, 0x65, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xF1, 0x63, 0x1D, 0x62,
+0x1C, 0xD1, 0x1B, 0xD0, 0xF7, 0xF0, 0x01, 0x6F,
+0xE0, 0x37, 0xE0, 0x37, 0x67, 0x67, 0xAD, 0xF0,
+0x14, 0x4B, 0x40, 0xF0, 0x54, 0xA3, 0xA0, 0xF0,
+0x08, 0x22, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xF7, 0xF0, 0x01, 0x69, 0x80, 0x34, 0x18, 0xD7,
+0x16, 0xD6, 0x15, 0xD5, 0x8C, 0xF2, 0x14, 0x4C,
+0x20, 0x31, 0x10, 0xAB, 0x20, 0x31, 0x14, 0xD4,
+0xAC, 0x9B, 0x73, 0xF6, 0x5C, 0x99, 0xD0, 0x67,
+0x40, 0xEA, 0x19, 0xD3, 0x16, 0x93, 0x73, 0xF6,
+0x5C, 0x99, 0xC8, 0x43, 0x15, 0x93, 0xFA, 0x4E,
+0xA7, 0x43, 0x14, 0x93, 0x07, 0x4D, 0x40, 0xEA,
+0x11, 0xE3, 0x14, 0x93, 0xF7, 0xF0, 0x01, 0x6D,
+0x73, 0xF6, 0x5C, 0x99, 0xA0, 0x35, 0xA0, 0x35,
+0x84, 0x43, 0x8D, 0xF0, 0x14, 0x4D, 0x40, 0xEA,
+0x06, 0x6E, 0x14, 0x93, 0x15, 0x95, 0x73, 0xF6,
+0x5C, 0x99, 0x87, 0x43, 0x06, 0x6E, 0x40, 0xEA,
+0x09, 0x4C, 0x15, 0x93, 0x73, 0xF6, 0x5C, 0x99,
+0x06, 0x6E, 0xA6, 0x43, 0x14, 0x93, 0xF2, 0x48,
+0x87, 0x43, 0x40, 0xEA, 0x03, 0x4C, 0x14, 0x92,
+0xC4, 0xA2, 0xA1, 0xA2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x8C, 0xF2, 0x94, 0xA2,
+0xD1, 0x18, 0x9A, 0xF4, 0x00, 0x65, 0x73, 0xF6,
+0x7C, 0x99, 0x06, 0x4A, 0x02, 0x6E, 0x1B, 0x65,
+0x15, 0x93, 0xA7, 0x43, 0x14, 0x93, 0x51, 0xE3,
+0x58, 0x67, 0x40, 0xEA, 0x05, 0x4D, 0x18, 0x97,
+0x16, 0x92, 0xAD, 0xF0, 0x94, 0xA7, 0x41, 0xE0,
+0xFF, 0xF7, 0x1F, 0x6A, 0x4C, 0xE8, 0xD1, 0x18,
+0xDE, 0x60, 0x17, 0xD0, 0x02, 0x67, 0xF7, 0xF0,
+0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x80, 0x34, 0x80, 0x34, 0x40, 0x32, 0x89, 0xF3,
+0xB0, 0x9C, 0x72, 0xF4, 0x54, 0x9A, 0x17, 0x94,
+0x00, 0x6F, 0x06, 0xD7, 0x07, 0xD7, 0x08, 0xD7,
+0x09, 0xD7, 0x0A, 0xD7, 0x0F, 0xD7, 0x10, 0xD7,
+0x11, 0xD7, 0x12, 0xD7, 0x40, 0xEA, 0x18, 0xD7,
+0x16, 0xD2, 0x18, 0x97, 0x19, 0x93, 0x2F, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xA1, 0xF0, 0x16, 0x6B,
+0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3, 0x02, 0xF0,
+0x00, 0x6C, 0x20, 0xF3, 0x03, 0x6F, 0x02, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x06, 0x6A, 0x7D, 0x67,
+0x20, 0xF0, 0x4C, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x4A, 0xF2, 0x00, 0x4A,
+0x0C, 0xD2, 0xA1, 0xF0, 0x18, 0x6A, 0x0D, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x0B, 0x04, 0x40, 0xEA,
+0x00, 0x65, 0x1D, 0x97, 0x1C, 0x91, 0x1B, 0x90,
+0x00, 0x6A, 0x00, 0xEF, 0x0F, 0x63, 0x17, 0x96,
+0x14, 0x95, 0x16, 0x94, 0x73, 0xF6, 0x5C, 0x99,
+0x19, 0xD7, 0x40, 0xEA, 0x18, 0xD3, 0x90, 0x67,
+0x01, 0x6E, 0xD1, 0x18, 0xD4, 0xD9, 0x00, 0x6D,
+0x5D, 0x67, 0x40, 0xF0, 0x04, 0xC2, 0x97, 0xF0,
+0x19, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB6, 0xF0,
+0x11, 0x4A, 0x7D, 0x67, 0x12, 0xD2, 0x04, 0x6A,
+0x59, 0xC3, 0x5D, 0x67, 0x18, 0xC2, 0x5A, 0xA2,
+0x7F, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0x29, 0x6C,
+0x8D, 0xEA, 0x5A, 0xC3, 0x16, 0x92, 0x18, 0x93,
+0x19, 0x97, 0x07, 0xD2, 0x17, 0x92, 0x10, 0xD7,
+0x08, 0xD2, 0x0F, 0x02, 0x09, 0xD2, 0x58, 0xA3,
+0x01, 0x72, 0x07, 0x61, 0x97, 0xF0, 0x19, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD6, 0xF0, 0x05, 0x4A,
+0x0A, 0xD2, 0x06, 0x05, 0xD1, 0x18, 0x66, 0xCA,
+0x0B, 0x04, 0x5D, 0x67, 0x20, 0xF0, 0x4C, 0xA2,
+0x01, 0x72, 0x04, 0x61, 0x5D, 0x67, 0x20, 0xF0,
+0x4D, 0xA2, 0x38, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0xC1, 0xF0, 0x10, 0x6B, 0x05, 0xD3, 0x01, 0x6B,
+0x02, 0x6E, 0x20, 0xF3, 0x04, 0x6F, 0x00, 0x6D,
+0x02, 0xF0, 0x00, 0x6C, 0x40, 0xEA, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF4, 0x50, 0x9A, 0x40, 0xEA, 0x16, 0x94,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF1, 0x2C, 0x9A, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0x89, 0xF3, 0x94, 0x9B,
+0xF2, 0xF4, 0x44, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x10, 0xF0, 0x00, 0x6D, 0x4D, 0xED, 0x40, 0xE9,
+0xD4, 0x6C, 0x00, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0xD4, 0xD9, 0x90, 0x67, 0x15, 0x92, 0x7F, 0xF7,
+0x1C, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A, 0x15, 0x94,
+0x72, 0x17, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x6A,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xB1, 0x18, 0x1A, 0xE0, 0x04, 0x01, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x15, 0xF0,
+0x08, 0x4B, 0x8D, 0xA3, 0x4C, 0xA3, 0xAE, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x40, 0xF0, 0x71, 0xA2, 0x40, 0xF0,
+0x92, 0xA2, 0x40, 0xF0, 0xB0, 0xA2, 0x40, 0xF0,
+0x53, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xED,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x01, 0x9A, 0x00, 0x6C, 0xA0, 0xA0,
+0x41, 0x98, 0x62, 0x98, 0xA1, 0xC1, 0x43, 0xD9,
+0x47, 0x45, 0x4E, 0x32, 0x65, 0xD9, 0x4C, 0x32,
+0x7D, 0x67, 0x4B, 0xE3, 0xBA, 0x65, 0x04, 0x02,
+0x42, 0xD9, 0x03, 0x6A, 0x40, 0xC1, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x33, 0xF5,
+0x48, 0x9A, 0x40, 0xEA, 0xA4, 0xD9, 0xA4, 0x99,
+0xC2, 0x99, 0xE3, 0x99, 0x65, 0x99, 0xA9, 0xE6,
+0x44, 0xD9, 0x44, 0x99, 0x4A, 0xEE, 0x28, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x33, 0xF5, 0x44, 0x9A, 0x40, 0xEA, 0x00, 0x6C,
+0x44, 0xA9, 0x91, 0x67, 0x41, 0xC9, 0x42, 0x99,
+0x42, 0x32, 0x42, 0x32, 0x42, 0xC9, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF0,
+0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x63, 0x99,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x06, 0x23, 0x72, 0xF4, 0x70, 0x9A, 0x83, 0x99,
+0x40, 0xEB, 0x42, 0xD9, 0x42, 0x99, 0x72, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x90, 0x67, 0x1A, 0x10,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x80, 0xAF, 0x33, 0xF5, 0x50, 0x9A, 0xA6, 0x41,
+0x67, 0xD9, 0xC6, 0xD9, 0x40, 0xEA, 0xE5, 0xD9,
+0xE5, 0x99, 0xC6, 0x99, 0x67, 0x99, 0x10, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0xFF, 0x6D, 0x01, 0x4D,
+0x40, 0xEA, 0x00, 0x6C, 0xB9, 0x65, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0x82, 0xA7, 0x46, 0xA1, 0x47, 0xEC, 0x83, 0xA7,
+0x8C, 0xEA, 0x40, 0xC6, 0x16, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x80, 0xAB,
+0x33, 0xF5, 0x50, 0x9A, 0xA6, 0x41, 0xC7, 0xD9,
+0xE6, 0xD9, 0x40, 0xEA, 0x65, 0xD9, 0x65, 0x99,
+0xE6, 0x99, 0xC7, 0x99, 0xD9, 0x22, 0x82, 0xA3,
+0x46, 0xA1, 0x47, 0xEC, 0x83, 0xA3, 0x8C, 0xEA,
+0x40, 0xC6, 0x04, 0x4F, 0x01, 0x4E, 0x04, 0x4B,
+0x94, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x04, 0x6A, 0x7D, 0x67, 0x58, 0xC3, 0x03, 0x6A,
+0x59, 0xC3, 0x60, 0xAC, 0x5D, 0x67, 0x00, 0x6D,
+0x68, 0xCA, 0x61, 0xAC, 0x69, 0xCA, 0x62, 0xAC,
+0x04, 0x04, 0xB1, 0x18, 0x59, 0xE9, 0x6A, 0xCA,
+0x7D, 0x67, 0x4F, 0xCB, 0x00, 0xF0, 0x1E, 0x02,
+0x4D, 0xCB, 0x42, 0x32, 0x42, 0x32, 0x4E, 0xCB,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF0, 0x4C, 0x9A, 0x40, 0xEA, 0x06, 0x04,
+0x09, 0x97, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x5D, 0x67, 0xB1, 0xC2,
+0x7D, 0x67, 0x00, 0x6A, 0x50, 0xC3, 0x5D, 0x67,
+0x89, 0xCA, 0x82, 0x34, 0x82, 0x34, 0x8A, 0xCA,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF0, 0x4C, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0x07, 0x97, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x40, 0xA4, 0x7F, 0x6B,
+0xFF, 0x6D, 0x6C, 0xEA, 0xAC, 0xEA, 0x03, 0x72,
+0x0D, 0x60, 0x05, 0x72, 0x11, 0x60, 0x0D, 0x2A,
+0x41, 0xA4, 0x0F, 0x6B, 0x02, 0x4C, 0x6C, 0xEA,
+0x48, 0x32, 0xFE, 0x4A, 0xD1, 0x18, 0x40, 0x36,
+0x4C, 0xED, 0x03, 0x10, 0xD1, 0x18, 0xD2, 0x35,
+0x00, 0x65, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xD1, 0x18, 0x2D, 0x36, 0x02, 0x4C, 0xF9, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xD2, 0xF4, 0x4C, 0x98, 0x50, 0xF1, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x7F, 0x6B, 0x6C, 0xEA,
+0x06, 0xD2, 0x06, 0x02, 0x40, 0xA2, 0x06, 0x72,
+0x1B, 0x61, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0xD2, 0xF4, 0x48, 0x99, 0x70, 0xF1,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D, 0xB1, 0x18,
+0x19, 0xDE, 0x00, 0x65, 0x12, 0x2A, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xF3, 0xF0,
+0x70, 0x9B, 0x04, 0xD2, 0x60, 0xF2, 0x12, 0x6F,
+0x02, 0x6E, 0x10, 0x6C, 0x40, 0xEB, 0x00, 0x6D,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36,
+0xC0, 0x36, 0xD2, 0xF4, 0x40, 0x9E, 0x08, 0x6D,
+0x50, 0xF1, 0x10, 0x6C, 0x40, 0xEA, 0x06, 0xD6,
+0xD2, 0xF4, 0x4C, 0x98, 0x70, 0xF1, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x06, 0x96, 0x01, 0x6D,
+0x4D, 0xED, 0x06, 0xD5, 0x06, 0x02, 0xA0, 0xA2,
+0xD2, 0xF4, 0xC0, 0x9E, 0x70, 0xF1, 0x04, 0x6C,
+0x40, 0xEE, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x08, 0x99,
+0xD2, 0xF4, 0x54, 0x9A, 0xE0, 0xF1, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x89, 0xF1, 0xB8, 0x9B,
+0xE0, 0xF1, 0x10, 0x6C, 0xAC, 0xEA, 0x03, 0xF0,
+0x00, 0x6D, 0x40, 0xE8, 0x4D, 0xED, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x4C, 0x9A, 0x40, 0xEA, 0xC8, 0x6C, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF3,
+0x50, 0x9A, 0x3E, 0xF4, 0x04, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xED, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x04, 0x67, 0x22, 0x67, 0x50, 0xF1,
+0x10, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xA0, 0x98, 0xD2, 0xF4, 0x68, 0x9B,
+0x82, 0x67, 0x40, 0xEB, 0x04, 0xD2, 0x04, 0x92,
+0x04, 0x48, 0x04, 0x4A, 0x70, 0xF1, 0x00, 0x72,
+0xF0, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x44, 0x9A, 0x70, 0xF1,
+0x04, 0x6C, 0x40, 0xEA, 0x01, 0x6D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x91, 0x67, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xA5, 0xA2, 0x84, 0xA2,
+0x66, 0xA2, 0xA0, 0x35, 0x8D, 0xED, 0x87, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6,
+0x80, 0x34, 0xAD, 0xEC, 0xA1, 0xA4, 0x60, 0xA4,
+0xC2, 0xA4, 0xA0, 0x35, 0x6D, 0xED, 0x63, 0xA4,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x00, 0x6C, 0x0C, 0xD3,
+0x0A, 0xD4, 0xA9, 0xA2, 0x88, 0xA2, 0xCA, 0xA2,
+0xA0, 0x35, 0x8D, 0xED, 0x8B, 0xA2, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34,
+0xAD, 0xEC, 0xA9, 0xA4, 0x48, 0xA4, 0xCA, 0xA4,
+0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x91, 0xA2, 0x10, 0xA2, 0xB2, 0xA2,
+0x80, 0x34, 0x0D, 0xEC, 0x13, 0xA2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0xA0, 0x35, 0x40, 0x32,
+0xA0, 0x35, 0xD3, 0xF0, 0x54, 0x9A, 0xF7, 0xF0,
+0x01, 0x69, 0xAD, 0xEC, 0x00, 0xF6, 0x00, 0x30,
+0x20, 0x31, 0x8D, 0xE8, 0x20, 0x31, 0x40, 0xEA,
+0x06, 0x04, 0x94, 0xF2, 0x4C, 0x99, 0x0C, 0x93,
+0x45, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF2, 0x54, 0x9A, 0x81, 0xA3,
+0x0A, 0x07, 0x1A, 0x65, 0x40, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x42, 0xA3, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x83, 0xA3, 0x00, 0xF6, 0x80, 0x34,
+0x4D, 0xEC, 0x10, 0x6E, 0x58, 0x67, 0x40, 0xEA,
+0x06, 0x05, 0x10, 0x6B, 0x6E, 0xEA, 0x61, 0x2A,
+0x0A, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x32, 0xF2, 0x44, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x01, 0x72, 0x10, 0x60, 0x94, 0xF2,
+0x4C, 0x99, 0x3C, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF2, 0xF1, 0x4C, 0x9A,
+0x0A, 0x03, 0x04, 0xD3, 0x00, 0x6F, 0x01, 0x6E,
+0x04, 0x6D, 0x40, 0xEA, 0x90, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x40, 0x9A, 0x70, 0xF1, 0x00, 0x6C, 0x40, 0xEA,
+0x00, 0x6D, 0x11, 0x97, 0x10, 0x91, 0x0F, 0x90,
+0x00, 0xEF, 0x09, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x0C, 0xD3, 0x01, 0x6C, 0x8C, 0xEA,
+0x0C, 0x93, 0xAF, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF2, 0xF2, 0x58, 0x9A,
+0x81, 0xA3, 0x00, 0x6F, 0x1A, 0x65, 0x40, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x42, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x83, 0xA3, 0x00, 0xF6,
+0x80, 0x34, 0xAE, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6B, 0x6C, 0xEA,
+0xB9, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF1, 0x50, 0x9A, 0x00, 0x6F,
+0x01, 0x6E, 0x04, 0x6D, 0x40, 0xEA, 0x90, 0x67,
+0xBA, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x4C, 0x9A, 0x29, 0xF2,
+0xBC, 0x9B, 0x03, 0x6E, 0x40, 0xEA, 0x00, 0x6C,
+0xAA, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0x70, 0xF1, 0x04, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x05, 0x97, 0x01, 0x5A,
+0x58, 0x67, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x85, 0xA3, 0x44, 0xA3,
+0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2, 0x04, 0xA2,
+0x86, 0xA2, 0x60, 0x33, 0x0D, 0xEB, 0x07, 0xA2,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0xD1, 0x18, 0x29, 0x37, 0x6D, 0xE8,
+0x01, 0x72, 0x0E, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6, 0x50, 0x9A,
+0x69, 0xF1, 0xB4, 0x9B, 0x40, 0xEA, 0x01, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x81, 0xA0, 0xF2, 0xF2, 0x30, 0x9A, 0x40, 0xA0,
+0x80, 0x34, 0x01, 0x6F, 0x4D, 0xEC, 0x42, 0xA0,
+0xEB, 0xEF, 0x10, 0x6E, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x83, 0xA0, 0x04, 0x05, 0x00, 0xF6,
+0x80, 0x34, 0x40, 0xE9, 0x4D, 0xEC, 0x10, 0x72,
+0x0E, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF0, 0x44, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x01, 0x6A, 0x00, 0xEF, 0x06, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0x09, 0xF1, 0xB0, 0x9B, 0x40, 0xEA,
+0x01, 0x6C, 0xEB, 0x17, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x15, 0xF0, 0x08, 0x48,
+0x45, 0xA0, 0x24, 0xA0, 0x66, 0xA0, 0x40, 0x32,
+0x2D, 0xEA, 0x27, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x89, 0xF3, 0xB8, 0x9B, 0x72, 0xF4, 0x58, 0x9A,
+0x10, 0x6C, 0x40, 0xEA, 0x03, 0x6E, 0x42, 0x33,
+0x40, 0xC1, 0x61, 0xC1, 0x00, 0xF6, 0x42, 0x32,
+0x62, 0x33, 0x62, 0xC1, 0x43, 0xC1, 0x85, 0xA0,
+0x44, 0xA0, 0x66, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA0, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x61, 0xA2,
+0x20, 0xA2, 0x82, 0xA2, 0x60, 0x33, 0x2D, 0xEB,
+0x23, 0xA2, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x20, 0x31, 0x6D, 0xE9, 0x1A, 0x29,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6A, 0xF2,
+0x18, 0x4A, 0x05, 0xD2, 0xFF, 0x6A, 0x42, 0x4A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x00, 0xEF, 0x06, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF3, 0x50, 0x9A,
+0x00, 0x6D, 0x14, 0x6C, 0x40, 0xEA, 0x01, 0x6E,
+0x42, 0x33, 0x40, 0xC1, 0x61, 0xC1, 0x00, 0xF6,
+0x42, 0x32, 0x62, 0x33, 0x62, 0xC1, 0x43, 0xC1,
+0x85, 0xA0, 0x64, 0xA0, 0x46, 0xA0, 0x80, 0x34,
+0x6D, 0xEC, 0x67, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x81, 0xA3, 0x40, 0xA3, 0xA2, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x43, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x61, 0xA2, 0x82, 0xA2, 0xA0, 0xA2, 0x43, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x0D, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x6A, 0xF2, 0x18, 0x4A, 0x05, 0xD2, 0xFF, 0x6A,
+0x49, 0x4A, 0xAE, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF4, 0x4C, 0x9A,
+0xFF, 0x6D, 0x01, 0x4D, 0x40, 0xEA, 0x00, 0x6C,
+0xAC, 0x17, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0x15, 0xF0, 0x08, 0x48,
+0x45, 0xA0, 0x24, 0xA0, 0x66, 0xA0, 0x40, 0x32,
+0x2D, 0xEA, 0x27, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x89, 0xF3, 0xBC, 0x9B, 0x72, 0xF4, 0x58, 0x9A,
+0x10, 0x6C, 0x40, 0xEA, 0x03, 0x6E, 0x42, 0x33,
+0x44, 0xC1, 0x65, 0xC1, 0x00, 0xF6, 0x42, 0x32,
+0x62, 0x33, 0x66, 0xC1, 0x47, 0xC1, 0x85, 0xA0,
+0x44, 0xA0, 0x66, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA0, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x65, 0xA2,
+0x24, 0xA2, 0x86, 0xA2, 0x60, 0x33, 0x2D, 0xEB,
+0x27, 0xA2, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB,
+0x00, 0xF6, 0x20, 0x31, 0x6D, 0xE9, 0x1A, 0x29,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6A, 0xF2,
+0x18, 0x4A, 0x05, 0xD2, 0xFF, 0x6A, 0x5B, 0x4A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x00, 0xEF, 0x06, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF3, 0x50, 0x9A,
+0x00, 0x6D, 0x14, 0x6C, 0x40, 0xEA, 0x01, 0x6E,
+0x42, 0x33, 0x40, 0xC1, 0x61, 0xC1, 0x00, 0xF6,
+0x42, 0x32, 0x62, 0x33, 0x62, 0xC1, 0x43, 0xC1,
+0x85, 0xA0, 0x64, 0xA0, 0x46, 0xA0, 0x80, 0x34,
+0x6D, 0xEC, 0x67, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x61, 0xA2, 0x82, 0xA2, 0xA0, 0xA2, 0x43, 0xA2,
+0x60, 0x33, 0x80, 0x34, 0x6D, 0xED, 0x80, 0x34,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0xC4, 0x2A, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x6A, 0xF2, 0x18, 0x4A, 0x05, 0xD2, 0xFF, 0x6A,
+0x62, 0x4A, 0xAE, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0xD2, 0xF4, 0x54, 0x99,
+0xE0, 0xF1, 0x14, 0x6C, 0x40, 0xEA, 0x0F, 0x68,
+0x4C, 0xE8, 0x01, 0x48, 0x10, 0x70, 0x51, 0x67,
+0x01, 0x61, 0x00, 0x68, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x20, 0x9B,
+0xD2, 0xF4, 0x54, 0x9A, 0xE0, 0xF1, 0x14, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x10, 0x6D, 0xAB, 0xED,
+0xAC, 0xEA, 0x4D, 0xE8, 0xFF, 0x6D, 0xE0, 0xF1,
+0x14, 0x6C, 0x40, 0xE9, 0x0C, 0xED, 0x07, 0x97,
+0x06, 0x91, 0x05, 0x90, 0x00, 0xEF, 0x04, 0x63,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF2, 0xF2,
+0x10, 0x9A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A, 0x85, 0xA2,
+0x64, 0xA2, 0xA6, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x67, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3,
+0x40, 0xA3, 0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x43, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x81, 0xA2,
+0x60, 0xA2, 0x01, 0x6F, 0x80, 0x34, 0x6D, 0xEC,
+0x62, 0xA2, 0xEB, 0xEF, 0x10, 0x6E, 0x60, 0x33,
+0x60, 0x33, 0x8D, 0xEB, 0x83, 0xA2, 0x04, 0x05,
+0x00, 0xF6, 0x80, 0x34, 0x40, 0xE8, 0x6D, 0xEC,
+0x10, 0x72, 0x0F, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF0, 0x5C, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0xD1, 0x18, 0x11, 0x38,
+0x00, 0x65, 0x09, 0x97, 0x08, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x09, 0xF1,
+0xB0, 0x9B, 0x40, 0xEA, 0x01, 0x6C, 0xED, 0x17,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0xD2, 0xF4, 0x54, 0x99, 0xE0, 0xF1, 0x14, 0x6C,
+0x40, 0xEA, 0x0F, 0x68, 0x52, 0x32, 0x4C, 0xE8,
+0x01, 0x48, 0x10, 0x70, 0x01, 0x61, 0x00, 0x68,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x60, 0x9A, 0xD2, 0xF4, 0x54, 0x99,
+0xE0, 0xF1, 0x14, 0x6C, 0x40, 0xEA, 0x04, 0xD3,
+0x10, 0x35, 0x0F, 0x68, 0x4C, 0xE8, 0x04, 0x93,
+0xFF, 0x6A, 0x0D, 0xED, 0xE0, 0xF1, 0x14, 0x6C,
+0x40, 0xEB, 0x4C, 0xED, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xA5, 0xA2, 0x24, 0x67,
+0x84, 0xA2, 0x66, 0xA2, 0xA0, 0x35, 0x8D, 0xED,
+0x87, 0xA2, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC, 0xA5, 0xA4,
+0x64, 0xA4, 0xC6, 0xA4, 0xA0, 0x35, 0x6D, 0xED,
+0x67, 0xA4, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0x00, 0x6D,
+0x0A, 0xD5, 0xC9, 0xA2, 0x88, 0xA2, 0xEA, 0xA2,
+0xC0, 0x36, 0x8D, 0xEE, 0x8B, 0xA2, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x80, 0x34,
+0xCD, 0xEC, 0xC9, 0xA4, 0x48, 0xA4, 0xEA, 0xA4,
+0xC0, 0x36, 0x4D, 0xEE, 0x4B, 0xA4, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x40, 0x32,
+0xCD, 0xEA, 0x95, 0xA2, 0x14, 0xA2, 0xD6, 0xA2,
+0x80, 0x34, 0x0D, 0xEC, 0x17, 0xA2, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xEC, 0x00, 0xF6, 0x00, 0x30,
+0x8D, 0xE8, 0x81, 0xA1, 0x06, 0xD5, 0x07, 0xD5,
+0x08, 0xD5, 0x09, 0xD5, 0x42, 0x44, 0xFF, 0x6D,
+0x0F, 0x5C, 0xAC, 0xEA, 0x13, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x40, 0x32, 0x60, 0x33, 0xD3, 0xF6,
+0x50, 0x9A, 0x09, 0xF1, 0xB0, 0x9B, 0x40, 0xEA,
+0x01, 0x6C, 0x11, 0x97, 0x10, 0x91, 0x0F, 0x90,
+0x00, 0xEF, 0x09, 0x63, 0x80, 0xA1, 0x7F, 0x6E,
+0x80, 0x6F, 0x8C, 0xEE, 0x9D, 0x67, 0x98, 0xA4,
+0xEB, 0xEF, 0xEC, 0xEC, 0xCD, 0xEC, 0xDD, 0x67,
+0x98, 0xC6, 0x03, 0x6C, 0x4C, 0xEC, 0x4A, 0x32,
+0x7B, 0x24, 0x81, 0x42, 0x0F, 0x6A, 0x0D, 0xD3,
+0x8C, 0xEA, 0x7D, 0x67, 0xAC, 0xEA, 0xB9, 0xA3,
+0x10, 0x6C, 0x8B, 0xEC, 0xF7, 0xF0, 0x01, 0x6F,
+0x8C, 0xED, 0xE0, 0x37, 0xE0, 0x37, 0xAD, 0xEA,
+0x59, 0xC3, 0x73, 0xF6, 0x5C, 0x9F, 0xA3, 0xA1,
+0x0C, 0xD7, 0x1A, 0x65, 0x42, 0xA1, 0xA0, 0x35,
+0x4D, 0xED, 0x44, 0xA1, 0x00, 0xF0, 0x1A, 0x04,
+0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA, 0xA5, 0xA1,
+0x00, 0xF6, 0xA0, 0x35, 0x4D, 0xED, 0x58, 0x67,
+0x40, 0xEA, 0x02, 0x6E, 0x83, 0xA1, 0xA2, 0xA1,
+0x44, 0xA1, 0x80, 0x34, 0x0C, 0x97, 0xAD, 0xEC,
+0xA5, 0xA1, 0x40, 0x32, 0xC1, 0xA1, 0x40, 0x32,
+0x4D, 0xEC, 0x73, 0xF6, 0xFC, 0x9F, 0x00, 0xF6,
+0xA0, 0x35, 0xF7, 0xF0, 0x01, 0x69, 0x8D, 0xED,
+0x20, 0x31, 0xFE, 0x4E, 0x02, 0x4D, 0x07, 0x04,
+0x40, 0xEF, 0x20, 0x31, 0x94, 0xF2, 0x4C, 0x99,
+0x0D, 0x93, 0x3C, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF2, 0xF2, 0x54, 0x9A,
+0x81, 0xA3, 0x0A, 0x07, 0x1A, 0x65, 0x40, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x42, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x83, 0xA3, 0x00, 0xF6,
+0x80, 0x34, 0x4D, 0xEC, 0x10, 0x6E, 0x58, 0x67,
+0x40, 0xEA, 0x06, 0x05, 0x10, 0x6B, 0x6E, 0xEA,
+0x58, 0x2A, 0x0A, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x32, 0xF2, 0x44, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x72, 0x85, 0x60,
+0x94, 0xF2, 0x4C, 0x99, 0x33, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF2, 0xF1,
+0x4C, 0x9A, 0x0A, 0x03, 0x04, 0xD3, 0x00, 0x6F,
+0x01, 0x6E, 0x08, 0x6D, 0x40, 0xEA, 0x90, 0x67,
+0xD1, 0x18, 0x54, 0x38, 0x00, 0x65, 0x71, 0x17,
+0x0F, 0x6C, 0x85, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x0C, 0xD3, 0x01, 0x6C, 0x8C, 0xEA,
+0x0C, 0x93, 0xB8, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF2, 0xF2, 0x58, 0x9A,
+0x81, 0xA3, 0x00, 0x6F, 0x1A, 0x65, 0x40, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x42, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x83, 0xA3, 0x00, 0xF6,
+0x80, 0x34, 0xB7, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6B, 0x6C, 0xEA,
+0xC2, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF2, 0xF1, 0x50, 0x9A, 0x00, 0x6F,
+0x01, 0x6E, 0x08, 0x6D, 0x40, 0xEA, 0x90, 0x67,
+0xC3, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x4C, 0x9A, 0x29, 0xF2,
+0xBC, 0x9B, 0x04, 0x6E, 0x40, 0xEA, 0x00, 0x6C,
+0x28, 0x17, 0x00, 0x65, 0x01, 0x5C, 0x20, 0xE8,
+0x58, 0x67, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x41, 0xA4, 0x02, 0x72, 0x05, 0x60, 0x03, 0x5A,
+0x13, 0x60, 0x1C, 0x22, 0x01, 0x72, 0x25, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x89, 0xF1, 0xB4, 0x9B,
+0x40, 0xEA, 0x00, 0x6C, 0x03, 0x6A, 0x12, 0x10,
+0x04, 0x72, 0x1B, 0x60, 0x04, 0x5A, 0x15, 0x61,
+0x05, 0x72, 0xEA, 0x61, 0xD1, 0x18, 0x99, 0x1E,
+0x08, 0x4C, 0x08, 0x10, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0xD1, 0x18, 0x00, 0x2C, 0x08, 0x4C,
+0xF9, 0x17, 0xB1, 0x18, 0x17, 0xE8, 0x08, 0x4C,
+0xF5, 0x17, 0xB1, 0x18, 0xEB, 0xE3, 0x08, 0x4C,
+0xF1, 0x17, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x41, 0xA4, 0x06, 0x2A, 0xB1, 0x18, 0x77, 0xE5,
+0x08, 0x4C, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x89, 0xF1, 0xB4, 0x9B,
+0x40, 0xEA, 0x00, 0x6C, 0x03, 0x6A, 0xED, 0x17,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0x40, 0xA4,
+0x4A, 0x32, 0x17, 0x5A, 0x80, 0xF0, 0x05, 0x60,
+0x48, 0x33, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x08, 0xF4, 0x1C, 0x4A, 0x49, 0xE3,
+0x40, 0x9A, 0x00, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF0,
+0x44, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x03, 0x10,
+0xB1, 0x18, 0x76, 0xFE, 0x00, 0x65, 0x62, 0x67,
+0x05, 0x97, 0x04, 0x90, 0x43, 0x67, 0x00, 0xEF,
+0x03, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF0, 0x40, 0x9A, 0xED, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF0, 0x5C, 0x9A, 0xE6, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF0,
+0x58, 0x9A, 0xDF, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF0, 0x54, 0x9A,
+0xD8, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF0, 0x50, 0x9A, 0xD1, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF2, 0xF7, 0x40, 0x9A, 0xCA, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF0,
+0x4C, 0x9A, 0xC3, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF0, 0x48, 0x9A,
+0xBC, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF0, 0x44, 0x9A, 0xB5, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF0, 0x40, 0x9A, 0xAE, 0x17, 0xD1, 0x18,
+0x12, 0x9B, 0x00, 0x65, 0xB0, 0x17, 0xD1, 0x18,
+0xCD, 0x8A, 0x04, 0x67, 0x04, 0x6B, 0x90, 0x67,
+0xAB, 0x22, 0xD1, 0x18, 0xDD, 0x46, 0x00, 0x65,
+0xA6, 0x17, 0xD1, 0x18, 0xDC, 0x18, 0x00, 0x65,
+0xA2, 0x17, 0xB1, 0x18, 0x07, 0xF9, 0x00, 0x65,
+0x9E, 0x17, 0xD1, 0x18, 0x29, 0x39, 0x00, 0x65,
+0x9A, 0x17, 0xD1, 0x18, 0x24, 0x47, 0x00, 0x65,
+0x96, 0x17, 0xD1, 0x18, 0x2F, 0xC4, 0x00, 0x65,
+0x92, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF0, 0x58, 0x9A, 0x85, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x20, 0x9A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x4C, 0x9A,
+0xE0, 0xF1, 0x10, 0x68, 0x40, 0xEA, 0x90, 0x67,
+0x01, 0x6D, 0xAD, 0xEA, 0xFF, 0x6D, 0x4C, 0xED,
+0x40, 0xE9, 0x90, 0x67, 0x7D, 0x67, 0x06, 0x6A,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x8A, 0xF2, 0x14, 0x4A, 0x05, 0xD2,
+0xE4, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0x6A, 0x00, 0xEF, 0x06, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x41, 0xA4, 0x04, 0x22,
+0x01, 0x72, 0x08, 0x60, 0x00, 0x6A, 0x03, 0x10,
+0xD1, 0x18, 0xD1, 0x35, 0x00, 0x65, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xD1, 0x18, 0x80, 0x39,
+0x00, 0x65, 0xF9, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0x40, 0xA4, 0x4A, 0x32, 0x02, 0x72, 0x24, 0x60,
+0x03, 0x5A, 0x0A, 0x60, 0x11, 0x22, 0x01, 0x72,
+0x18, 0x60, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF0, 0x5C, 0x9A, 0x0E, 0x10,
+0x04, 0x72, 0x1D, 0x60, 0x05, 0x72, 0xF5, 0x61,
+0xD1, 0x18, 0x9A, 0x39, 0x00, 0x65, 0x1A, 0x10,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x11, 0x10, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF0, 0x50, 0x9A, 0xF6, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x53, 0xF0, 0x4C, 0x9A, 0xEF, 0x17, 0xD1, 0x18,
+0x40, 0x62, 0x00, 0x65, 0x05, 0x97, 0x00, 0xEF,
+0x03, 0x63, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x41, 0xA4, 0x00, 0xA4,
+0x62, 0xA4, 0x40, 0x32, 0x0D, 0xEA, 0x03, 0xA4,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6,
+0x00, 0x30, 0x4D, 0xE8, 0x40, 0xA0, 0x03, 0x6B,
+0x6C, 0xEA, 0x01, 0x72, 0x45, 0x60, 0x12, 0x22,
+0x02, 0x72, 0x49, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0x69, 0xF1, 0xB4, 0x9B,
+0xD3, 0xF6, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x6C,
+0x01, 0x6B, 0x09, 0x10, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x90, 0x67, 0x62, 0x67, 0x45, 0x80,
+0x01, 0x69, 0x00, 0x52, 0x1F, 0x60, 0x00, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A, 0x00, 0x6F,
+0x06, 0x06, 0x04, 0x6D, 0x00, 0x6C, 0x40, 0xEA,
+0x08, 0xD3, 0x08, 0x93, 0x23, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0x09, 0xF3, 0xB4, 0x9B, 0x01, 0x6C,
+0x40, 0xEA, 0x00, 0x69, 0x51, 0x67, 0x0D, 0x97,
+0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF0, 0x5C, 0x9A, 0xCD, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB3, 0xF0,
+0x40, 0x9A, 0xC6, 0x17, 0x06, 0x95, 0xDB, 0x25,
+0x9D, 0x67, 0x32, 0xC4, 0x51, 0xC4, 0x30, 0xC4,
+0xDD, 0x67, 0x0C, 0x6C, 0x93, 0xC6, 0x9D, 0x67,
+0x54, 0xC4, 0x40, 0xA0, 0x03, 0x6E, 0x4A, 0x34,
+0x88, 0x34, 0xCC, 0xEA, 0x8D, 0xEA, 0x40, 0xC5,
+0x41, 0xA0, 0x04, 0x04, 0x62, 0xC5, 0x41, 0xC5,
+0x43, 0xA0, 0xD1, 0x18, 0xA8, 0x19, 0x43, 0xC5,
+0x0E, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x40, 0x32,
+0x60, 0x33, 0xD3, 0xF6, 0x50, 0x9A, 0x09, 0xF3,
+0xB4, 0x9B, 0x40, 0xEA, 0x01, 0x6C, 0x01, 0x69,
+0xC1, 0x17, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x00, 0x69, 0x04, 0x67, 0x04, 0xD1, 0x15, 0x6F,
+0x04, 0x6E, 0x10, 0x6C, 0x40, 0xEA, 0x00, 0x6D,
+0xA5, 0xA0, 0x5D, 0x67, 0x3A, 0xC2, 0x39, 0xC2,
+0x38, 0xC2, 0x44, 0xA0, 0x3F, 0x69, 0x2C, 0xED,
+0xA0, 0x35, 0x7D, 0x67, 0x4D, 0xED, 0x5B, 0xC3,
+0xA2, 0x32, 0x5C, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A,
+0x00, 0x6F, 0x08, 0x06, 0xF8, 0x4D, 0x40, 0xEA,
+0x00, 0x6C, 0x04, 0x6B, 0x17, 0x2A, 0x08, 0x94,
+0x02, 0x6B, 0x14, 0x24, 0xC5, 0xA0, 0xF7, 0xF0,
+0x01, 0x6A, 0x64, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x2C, 0xEE, 0x73, 0xF6, 0x5C, 0x9A, 0xC0, 0x36,
+0x6D, 0xEE, 0xA7, 0x40, 0xF8, 0x4E, 0x40, 0xEA,
+0x01, 0x4D, 0x08, 0x95, 0xD1, 0x18, 0xA8, 0x19,
+0x06, 0x04, 0x62, 0x67, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x43, 0x67, 0x00, 0xEF, 0x07, 0x63,
+0xFD, 0x63, 0x05, 0x62, 0x04, 0xD0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4,
+0x54, 0x9A, 0x04, 0x67, 0x11, 0xF1, 0x10, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6B, 0x4C, 0xEB,
+0x00, 0x6A, 0x06, 0x2B, 0x66, 0xA0, 0x40, 0x6A,
+0x4C, 0xEB, 0x38, 0x6A, 0x01, 0x2B, 0x18, 0x6A,
+0x05, 0x97, 0x04, 0x90, 0x00, 0xEF, 0x03, 0x63,
+0xA1, 0xA4, 0x60, 0xA4, 0x42, 0xA4, 0xA0, 0x35,
+0x6D, 0xED, 0x63, 0xA4, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x40, 0xA3, 0x25, 0x72, 0x00, 0x6A, 0x05, 0x61,
+0x41, 0xA3, 0x15, 0x6B, 0x6E, 0xEA, 0x01, 0x5A,
+0x58, 0x67, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0x65, 0xA4, 0x04, 0xA4, 0x46, 0xA4, 0x60, 0x33,
+0x0D, 0xEB, 0x07, 0xA4, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEB, 0x00, 0xF6, 0x00, 0x30, 0x6D, 0xE8,
+0x61, 0xA4, 0x20, 0xA4, 0x42, 0xA4, 0x60, 0x33,
+0x2D, 0xEB, 0x23, 0xA4, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEB, 0x00, 0xF6, 0x20, 0x31, 0x6D, 0xE9,
+0xF8, 0x48, 0x08, 0x49, 0x00, 0x6B, 0x37, 0x20,
+0x65, 0xE1, 0x40, 0x99, 0x64, 0x42, 0x63, 0xE8,
+0x0E, 0xD3, 0x0E, 0x61, 0x0B, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB3, 0xF0,
+0x58, 0x9A, 0x84, 0x41, 0x0A, 0xD4, 0x40, 0xEA,
+0x0A, 0x04, 0x0E, 0x93, 0x63, 0xE0, 0xEB, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0x00, 0x6B, 0x04, 0xD3,
+0x10, 0x6C, 0x40, 0xF2, 0x19, 0x6F, 0x02, 0x6E,
+0x40, 0xEA, 0x00, 0x6D, 0x17, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x8A, 0xF2, 0x14, 0x4A, 0x07, 0xD2,
+0xA0, 0xF1, 0x1E, 0x6A, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04, 0x13, 0x97,
+0x12, 0x91, 0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63,
+0x69, 0xA4, 0x48, 0xA4, 0x60, 0x33, 0x4D, 0xEB,
+0x1B, 0x65, 0x6A, 0xA4, 0x58, 0x67, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEA, 0x1A, 0x65, 0x4B, 0xA4,
+0x78, 0x67, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0x21, 0x5A, 0x3A, 0x60, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x93, 0xF6, 0x40, 0x99,
+0x04, 0x67, 0x20, 0x6E, 0x04, 0x04, 0x40, 0xEA,
+0x00, 0x6D, 0xC9, 0xA0, 0x48, 0xA0, 0x93, 0xF6,
+0xE0, 0x99, 0xC0, 0x36, 0x4D, 0xEE, 0x4A, 0xA0,
+0x04, 0x04, 0x59, 0x6D, 0x40, 0x32, 0x40, 0x32,
+0xCD, 0xEA, 0xCB, 0xA0, 0x00, 0xF6, 0xC0, 0x36,
+0x40, 0xEF, 0x4D, 0xEE, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x89, 0xA0, 0x40, 0x32, 0x33, 0xF0,
+0xC8, 0x9A, 0x48, 0xA0, 0x80, 0x34, 0x04, 0x05,
+0x4D, 0xEC, 0x4A, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x8B, 0xA0, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xEE, 0x4D, 0xEC, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0x6A, 0x00, 0xEF, 0x08, 0x63,
+0x20, 0xE8, 0x01, 0x6A, 0xFD, 0x63, 0x05, 0x62,
+0x41, 0xA4, 0x12, 0x22, 0x01, 0x72, 0x1C, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x89, 0xF1, 0xB4, 0x9B,
+0x40, 0xEA, 0x00, 0x6C, 0x03, 0x6A, 0x09, 0x10,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xB3, 0xF0, 0x44, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x00, 0x6A, 0x05, 0x97, 0x00, 0xEF, 0x03, 0x63,
+0xD1, 0x18, 0x76, 0x3A, 0x00, 0x65, 0xF8, 0x17,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0xA5, 0xA4, 0x44, 0xA4, 0x66, 0xA4, 0xA0, 0x35,
+0x4D, 0xED, 0x47, 0xA4, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x06, 0xD2, 0x42, 0xA4, 0x35, 0x2A, 0x5D, 0x67,
+0xB9, 0xA2, 0x43, 0xA4, 0x01, 0x72, 0x1A, 0x60,
+0x2F, 0x22, 0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x6B, 0x80, 0x34, 0x60, 0x33, 0x02, 0x72,
+0x80, 0x34, 0x60, 0x33, 0x2A, 0x60, 0x03, 0x72,
+0x36, 0x60, 0xF3, 0xF0, 0x10, 0x9C, 0xA9, 0xF6,
+0xA4, 0x9B, 0xA9, 0xF6, 0x80, 0x9B, 0x05, 0xD2,
+0x01, 0x6A, 0x04, 0xD2, 0x6E, 0x6F, 0x40, 0xE8,
+0x02, 0x6E, 0x16, 0x10, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0x05, 0xD5, 0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6,
+0x80, 0x9B, 0xA9, 0xF6, 0xA4, 0x9B, 0xC0, 0xF3,
+0x06, 0x6F, 0x02, 0x6E, 0x40, 0xEA, 0x00, 0x65,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0xF3, 0xF0, 0x50, 0x9C, 0xA9, 0xF6,
+0x00, 0x9B, 0xA9, 0xF6, 0x24, 0x9B, 0x01, 0x6B,
+0x05, 0xD5, 0x04, 0xD3, 0x6D, 0x6F, 0x02, 0x6E,
+0x90, 0x67, 0xB1, 0x67, 0xEB, 0x17, 0xA9, 0xF6,
+0x00, 0x9B, 0xA9, 0xF6, 0x24, 0x9B, 0x01, 0x6B,
+0xF3, 0xF0, 0x50, 0x9C, 0x05, 0xD5, 0x04, 0xD3,
+0x6C, 0x6F, 0xF1, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC6, 0xF4, 0x4C, 0x9A, 0x16, 0x2A, 0xF7, 0xF0,
+0x01, 0x6B, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x60, 0x33, 0x40, 0x32, 0x40, 0x32, 0x60, 0x33,
+0xF3, 0xF0, 0x70, 0x9B, 0xA9, 0xF6, 0x80, 0x9A,
+0xA9, 0xF6, 0xA4, 0x9A, 0x5E, 0x6F, 0x40, 0xEB,
+0x01, 0x6E, 0x00, 0x6A, 0x07, 0x97, 0x00, 0xEF,
+0x04, 0x63, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x94, 0x32, 0xC6, 0xF4, 0x88, 0x9B,
+0x51, 0xE4, 0x40, 0x9C, 0x64, 0x67, 0x4E, 0xEB,
+0x07, 0x23, 0x68, 0xA2, 0xAE, 0xEB, 0xEE, 0x23,
+0x40, 0x9A, 0x4A, 0xEC, 0xFA, 0x61, 0xE9, 0x17,
+0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6,
+0x80, 0x9B, 0xA9, 0xF6, 0xA4, 0x9B, 0x5F, 0x6F,
+0x40, 0xEA, 0x01, 0x6E, 0xD6, 0x17, 0x00, 0x65,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0x01, 0x6B, 0x0E, 0xD4, 0x9D, 0x67, 0x72, 0xC4,
+0x04, 0x6B, 0x00, 0x6A, 0x71, 0xC4, 0x7D, 0x67,
+0x50, 0xC3, 0x0C, 0x6B, 0x73, 0xC4, 0x7D, 0x67,
+0x06, 0xD2, 0x54, 0xC3, 0x25, 0x67, 0x05, 0x68,
+0x01, 0x6A, 0x06, 0x95, 0x35, 0x2D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x00, 0x6F, 0x06, 0x06, 0x04, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x11, 0x2A, 0x06, 0x95,
+0x0F, 0x25, 0x0E, 0x92, 0x03, 0x6B, 0x04, 0x6C,
+0x4C, 0xEB, 0x40, 0xA5, 0x8B, 0xEC, 0x21, 0xC5,
+0x8C, 0xEA, 0x6D, 0xEA, 0x40, 0xC5, 0xD1, 0x18,
+0xA8, 0x19, 0x04, 0x04, 0x17, 0x22, 0x00, 0x6A,
+0x00, 0x6B, 0x06, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x73, 0xF6, 0x68, 0x9B,
+0x64, 0x6C, 0x08, 0xD2, 0x40, 0xEB, 0xFF, 0x48,
+0xFF, 0x6B, 0x6C, 0xE8, 0x08, 0x92, 0xD1, 0x28,
+0x00, 0x6A, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0x01, 0x6A, 0xF9, 0x17,
+0xD8, 0x22, 0xE6, 0x17, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x01, 0x6B, 0x0E, 0xD4,
+0x9D, 0x67, 0x72, 0xC4, 0x0F, 0xD5, 0x04, 0x6C,
+0xBD, 0x67, 0x91, 0xC5, 0x9D, 0x67, 0x70, 0xC4,
+0x0C, 0x6B, 0x00, 0x6A, 0x73, 0xC4, 0x7D, 0x67,
+0x06, 0xD2, 0x54, 0xC3, 0x26, 0x67, 0x05, 0x68,
+0x01, 0x6A, 0x06, 0x95, 0x35, 0x2D, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x00, 0x6F, 0x06, 0x06, 0x04, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x11, 0x2A, 0x06, 0x95,
+0x0F, 0x25, 0x02, 0x02, 0x20, 0xF0, 0x54, 0xA2,
+0x28, 0x33, 0x41, 0xC5, 0x0E, 0x94, 0x03, 0x6A,
+0x8C, 0xEA, 0x6D, 0xEA, 0x40, 0xC5, 0xD1, 0x18,
+0xA8, 0x19, 0x04, 0x04, 0x17, 0x22, 0x00, 0x6A,
+0x00, 0x6B, 0x06, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x73, 0xF6, 0x68, 0x9B,
+0x64, 0x6C, 0x08, 0xD2, 0x40, 0xEB, 0xFF, 0x48,
+0xFF, 0x6B, 0x6C, 0xE8, 0x08, 0x92, 0xD1, 0x28,
+0x00, 0x6A, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0x01, 0x6A, 0xF9, 0x17,
+0xD8, 0x22, 0xE6, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x00, 0x6A, 0x10, 0xD4,
+0x11, 0xD5, 0x12, 0xD6, 0xD1, 0x18, 0xE3, 0x3A,
+0x06, 0xD2, 0x02, 0x67, 0x09, 0x2A, 0x12, 0x92,
+0x05, 0x5A, 0x06, 0x60, 0x00, 0x6A, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0x7D, 0x67, 0x01, 0x6A, 0x52, 0xC3, 0x04, 0x6A,
+0x51, 0xC3, 0x03, 0x6A, 0x50, 0xC3, 0x14, 0x6A,
+0x53, 0xC3, 0x00, 0x6A, 0x54, 0xC3, 0x05, 0x6A,
+0x08, 0xD2, 0x01, 0x6A, 0x06, 0x91, 0x71, 0x29,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0x00, 0x6F, 0x06, 0x06,
+0x0C, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x22, 0x2A,
+0x06, 0x91, 0x20, 0x21, 0x32, 0x28, 0x04, 0xC1,
+0x05, 0xC1, 0x06, 0xC1, 0x07, 0xC1, 0x08, 0xC1,
+0x09, 0xC1, 0x0A, 0xC1, 0x0B, 0xC1, 0x04, 0x02,
+0x20, 0xF0, 0x54, 0xA2, 0x3F, 0x6B, 0x06, 0x95,
+0x41, 0xC1, 0x10, 0x92, 0x58, 0x34, 0x40, 0xA1,
+0x6C, 0xEA, 0x8D, 0xEA, 0x12, 0x94, 0x8C, 0xEB,
+0x40, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0x6D, 0xEA,
+0x40, 0xC1, 0xD1, 0x18, 0xA8, 0x19, 0x04, 0x04,
+0x42, 0x22, 0x00, 0x6A, 0x00, 0x6B, 0x06, 0xD3,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x73, 0xF6, 0x68, 0x9B, 0x64, 0x6C, 0x40, 0xEB,
+0x22, 0x67, 0x08, 0x94, 0xFF, 0x6B, 0x51, 0x67,
+0xFF, 0x4C, 0x6C, 0xEC, 0x08, 0xD4, 0xBE, 0x2C,
+0xA9, 0x17, 0x7B, 0xA0, 0x0F, 0x6C, 0xFF, 0x6A,
+0x8C, 0xEB, 0x4C, 0xEB, 0xF8, 0xA0, 0x09, 0xD3,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x8C, 0xEF, 0x4C, 0xEF, 0x09, 0x94, 0xF3, 0xF3,
+0x4C, 0x9B, 0xA7, 0x67, 0x00, 0x6E, 0x0B, 0xD3,
+0x40, 0xEA, 0x0A, 0xD7, 0x0B, 0x93, 0x42, 0x34,
+0x44, 0xC1, 0x85, 0xC1, 0x00, 0xF6, 0x42, 0x32,
+0x82, 0x34, 0x0A, 0x97, 0x86, 0xC1, 0x47, 0xC1,
+0x09, 0x94, 0xF3, 0xF3, 0x4C, 0x9B, 0x01, 0x6E,
+0x40, 0xEA, 0xA7, 0x67, 0x42, 0x33, 0x48, 0xC1,
+0x69, 0xC1, 0x00, 0xF6, 0x42, 0x32, 0x62, 0x33,
+0x6A, 0xC1, 0x4B, 0xC1, 0xA8, 0x17, 0x01, 0x6A,
+0x7A, 0x17, 0x9C, 0x22, 0xBB, 0x17, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC6, 0xF4, 0x48, 0x9A, 0xC7, 0x42, 0x79, 0x4E,
+0x70, 0xA2, 0x01, 0x6F, 0xEC, 0xEB, 0x05, 0x2B,
+0x20, 0x4A, 0xCA, 0xEA, 0xF9, 0x61, 0x20, 0xE8,
+0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1,
+0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x70, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF3,
+0x4C, 0x9A, 0x00, 0x6E, 0x25, 0x67, 0x04, 0x67,
+0x40, 0xEA, 0x08, 0xD3, 0x07, 0xD2, 0x03, 0x6A,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xA9, 0xF6, 0x80, 0x9A, 0xA9, 0xF6,
+0xA4, 0x9A, 0x08, 0x93, 0x06, 0xD1, 0x05, 0xD0,
+0x62, 0x6F, 0x40, 0xEB, 0x02, 0x6E, 0x0D, 0x97,
+0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63,
+0x58, 0x24, 0xF2, 0x63, 0x1B, 0x62, 0x1A, 0xD1,
+0x19, 0xD0, 0x4E, 0xA4, 0x04, 0x6B, 0x6C, 0xEA,
+0xFF, 0x6B, 0x6C, 0xEA, 0x43, 0x2A, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0x93, 0xF6,
+0x40, 0x9F, 0x04, 0x67, 0x14, 0xD6, 0x04, 0x04,
+0x20, 0x6E, 0x25, 0x67, 0x00, 0x6D, 0x16, 0xD3,
+0x40, 0xEA, 0x15, 0xD7, 0x15, 0x97, 0x20, 0x6E,
+0x00, 0x6D, 0x93, 0xF6, 0x40, 0x9F, 0x40, 0xEA,
+0x0C, 0x04, 0xBD, 0xA0, 0x9C, 0xA0, 0xDE, 0xA0,
+0x5A, 0xA0, 0xA0, 0x35, 0x8D, 0xED, 0x9F, 0xA0,
+0xC0, 0x36, 0x16, 0x93, 0xC0, 0x36, 0xE8, 0x42,
+0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34, 0xF8, 0x4F,
+0xAD, 0xEC, 0x0C, 0x06, 0x04, 0x05, 0xEC, 0xEB,
+0xE0, 0x9C, 0x01, 0x71, 0xE4, 0xDE, 0xE4, 0xDD,
+0x1D, 0x60, 0x19, 0x21, 0x02, 0x71, 0xF8, 0x67,
+0x19, 0x27, 0xFC, 0x4A, 0xFF, 0x6F, 0xEC, 0xEA,
+0x05, 0x22, 0x6A, 0xEA, 0x04, 0x4C, 0x04, 0x4E,
+0x04, 0x4D, 0xEE, 0x61, 0x14, 0x95, 0xD1, 0x18,
+0xC7, 0x2E, 0x0C, 0x04, 0x14, 0x95, 0xD1, 0x18,
+0xAA, 0x2E, 0x04, 0x04, 0x1B, 0x97, 0x1A, 0x91,
+0x19, 0x90, 0x00, 0xEF, 0x0E, 0x63, 0xE0, 0xDE,
+0x20, 0xDD, 0xE7, 0x17, 0xE0, 0xDE, 0xE0, 0xDD,
+0xE4, 0x17, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF3, 0x63, 0x19, 0x62, 0x18, 0xD1, 0x17, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x93, 0xF6, 0x40, 0x9B, 0x04, 0x67, 0x25, 0x67,
+0x20, 0x6E, 0x00, 0x6D, 0x04, 0x04, 0x40, 0xEA,
+0x14, 0xD3, 0x14, 0x93, 0x00, 0x6D, 0x0C, 0x04,
+0x93, 0xF6, 0x40, 0x9B, 0x40, 0xEA, 0x20, 0x6E,
+0x16, 0x34, 0x64, 0x44, 0x01, 0x6A, 0x68, 0x33,
+0x04, 0x05, 0x44, 0xE8, 0x6D, 0xE5, 0x01, 0x71,
+0x48, 0xDB, 0x40, 0xDB, 0x17, 0x60, 0x10, 0x21,
+0x02, 0x6A, 0x4E, 0xE9, 0x19, 0x21, 0x0C, 0x04,
+0xD1, 0x18, 0xC7, 0x2E, 0x16, 0x6D, 0x16, 0x6D,
+0xD1, 0x18, 0xAA, 0x2E, 0x04, 0x04, 0x19, 0x97,
+0x18, 0x91, 0x17, 0x90, 0x00, 0xEF, 0x0D, 0x63,
+0x88, 0x34, 0x04, 0x03, 0x91, 0xE3, 0x48, 0xDC,
+0x20, 0xDC, 0xED, 0x17, 0x88, 0x34, 0x04, 0x03,
+0x91, 0xE3, 0x48, 0xDC, 0x40, 0xDC, 0xE7, 0x17,
+0x88, 0x34, 0x04, 0x02, 0x91, 0xE2, 0x28, 0xDC,
+0xF3, 0x17, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x0F, 0xD5, 0x10, 0xD6,
+0x15, 0xF0, 0x08, 0x4B, 0x04, 0x67, 0x85, 0xA3,
+0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0x81, 0xA2, 0x20, 0xF0, 0x60, 0xA2, 0x20, 0xF0,
+0xA2, 0xA2, 0x80, 0x34, 0x8D, 0xEB, 0x20, 0xF0,
+0x83, 0xA2, 0xA0, 0x35, 0x0F, 0x92, 0xA0, 0x35,
+0x6D, 0xED, 0x00, 0xF6, 0x80, 0x34, 0x8D, 0xED,
+0x48, 0x33, 0xAD, 0xE3, 0x85, 0xA3, 0x44, 0xA3,
+0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x61, 0xA2, 0xE2, 0xA2,
+0x00, 0x6A, 0x06, 0xD2, 0x5D, 0xA0, 0x9E, 0xA0,
+0x08, 0xD3, 0x7C, 0xA0, 0x3F, 0xA0, 0x40, 0x32,
+0x6D, 0xEA, 0x80, 0x34, 0x02, 0x03, 0x80, 0x34,
+0x20, 0xF0, 0x78, 0xA3, 0x8D, 0xEA, 0x00, 0xF6,
+0x20, 0x31, 0x4D, 0xE9, 0x5D, 0x67, 0x78, 0xC2,
+0x50, 0xA0, 0x02, 0x6C, 0x8C, 0xEA, 0x36, 0x22,
+0x71, 0xA0, 0x5D, 0x67, 0x79, 0xC2, 0x87, 0x41,
+0x04, 0x6E, 0x06, 0x05, 0x05, 0x4C, 0xD1, 0x18,
+0xE7, 0xF4, 0x09, 0xD7, 0x10, 0x92, 0x09, 0x97,
+0x01, 0x72, 0x12, 0x61, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xB2, 0xF5, 0x7C, 0x9A,
+0x08, 0x94, 0x02, 0x6E, 0xA7, 0x67, 0x40, 0xEB,
+0x09, 0xD2, 0x09, 0x92, 0x08, 0x94, 0x02, 0x6E,
+0xB2, 0xF5, 0x5C, 0x9A, 0x40, 0xEA, 0x00, 0x6D,
+0x90, 0xA0, 0x08, 0x6A, 0x8C, 0xEA, 0x49, 0x22,
+0x02, 0x6A, 0x8C, 0xEA, 0x1F, 0x22, 0x5D, 0x67,
+0xB9, 0xA2, 0x92, 0x34, 0xFF, 0x6A, 0xD1, 0x18,
+0xE3, 0x3A, 0x4C, 0xEC, 0x00, 0x6C, 0x22, 0x2A,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x44, 0x67,
+0x00, 0xEF, 0x07, 0x63, 0xA9, 0xA0, 0x48, 0xA0,
+0x8A, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA0,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x48, 0xA2, 0x7D, 0x67,
+0x59, 0xC3, 0xBD, 0x17, 0xA9, 0xA0, 0x48, 0xA0,
+0x8A, 0xA0, 0xA0, 0x35, 0x4D, 0xED, 0x4B, 0xA0,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x4B, 0xA2, 0x22, 0x5A,
+0x18, 0x60, 0x92, 0xA1, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x05, 0xD4, 0x01, 0x6C, 0x04, 0xD4,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x40, 0x32, 0xA9, 0xF6, 0xA4, 0x9C, 0xF3, 0xF0,
+0x50, 0x9A, 0xA9, 0xF6, 0x80, 0x9C, 0x64, 0x6F,
+0x40, 0xEA, 0x02, 0x6E, 0xB1, 0x18, 0x90, 0xD1,
+0x13, 0x6C, 0xED, 0xA1, 0x4C, 0xA1, 0x97, 0xF0,
+0x19, 0x6E, 0xE0, 0x37, 0x4D, 0xEF, 0x4E, 0xA1,
+0xC0, 0x36, 0x10, 0x95, 0x40, 0x32, 0x40, 0x32,
+0xED, 0xEA, 0xEF, 0xA1, 0x0F, 0x94, 0xC0, 0x36,
+0x00, 0xF6, 0xE0, 0x37, 0xDD, 0xF2, 0x01, 0x4E,
+0xD1, 0x18, 0x78, 0xCB, 0x4D, 0xEF, 0x17, 0x2A,
+0x10, 0x93, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xA9, 0xF6, 0x80, 0x9B, 0xF3, 0xF0, 0x50, 0x9A,
+0xA9, 0xF6, 0xA4, 0x9B, 0x65, 0x6F, 0x40, 0xEA,
+0x01, 0x6E, 0x00, 0x6C, 0x95, 0x17, 0x10, 0x92,
+0x08, 0x2A, 0x01, 0x6A, 0x52, 0xC0, 0x02, 0x02,
+0x20, 0xF0, 0x58, 0xA2, 0x54, 0xC0, 0x01, 0x6C,
+0x8B, 0x17, 0x01, 0x6A, 0x53, 0xC0, 0x00, 0x6A,
+0x55, 0xC0, 0xF9, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x11, 0xD5, 0x04, 0x67,
+0xE8, 0xA4, 0x49, 0xA4, 0x8A, 0xA4, 0x2B, 0xA0,
+0x40, 0x32, 0x80, 0x34, 0x80, 0x34, 0xED, 0xEA,
+0x8D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0x50, 0xA0, 0x02, 0x6C, 0x8C, 0xEA, 0x23, 0x22,
+0x51, 0xA0, 0x82, 0x67, 0xD1, 0x18, 0xDE, 0x60,
+0x0A, 0xD2, 0xFF, 0x72, 0xA2, 0x67, 0x1F, 0x61,
+0x0A, 0x93, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6, 0x80, 0x9B,
+0xA9, 0xF6, 0xA4, 0x9B, 0x66, 0x6F, 0x40, 0xEA,
+0x01, 0x6E, 0x00, 0x6A, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0x48, 0xA1,
+0x20, 0xF0, 0xA0, 0xA1, 0x0A, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x32, 0xF6,
+0x58, 0x9A, 0x85, 0x67, 0x40, 0xEA, 0x0B, 0xD5,
+0x0B, 0x95, 0x08, 0x22, 0x50, 0xA0, 0x02, 0x6C,
+0x8C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA, 0x08, 0x22,
+0x01, 0x6A, 0xE4, 0x17, 0x11, 0x96, 0xD1, 0x18,
+0x11, 0x3C, 0x90, 0x67, 0xF3, 0x2A, 0xDE, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0x0B, 0xD2, 0xBD, 0xA0,
+0x5C, 0xA0, 0xDE, 0xA0, 0xA0, 0x35, 0x4D, 0xED,
+0x5F, 0xA0, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x12, 0xA2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF3, 0x6C, 0x9A, 0xB8, 0xA1, 0x5B, 0xA1,
+0x0F, 0x6E, 0xCC, 0xED, 0xCC, 0xEA, 0x8C, 0xED,
+0x00, 0x6E, 0x40, 0xEB, 0x4C, 0xEC, 0x22, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x54, 0x9A, 0xD8, 0xF5, 0x00, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x09, 0xD2, 0x11, 0x92,
+0x08, 0xD1, 0x05, 0xD0, 0x07, 0xD2, 0x0A, 0x92,
+0x67, 0x6F, 0x06, 0xD2, 0x05, 0x6A, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA9, 0xF6, 0x80, 0x9A, 0xA9, 0xF6, 0xA4, 0x9A,
+0x0B, 0x92, 0x40, 0xEA, 0x02, 0x6E, 0xB4, 0x17,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0x00, 0x68, 0x18, 0xD4, 0x19, 0xD5, 0x26, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC6, 0xF4, 0xE8, 0x9A, 0x0A, 0x04, 0x20, 0x6E,
+0x1D, 0xE7, 0xA7, 0x67, 0xD1, 0x18, 0xE7, 0xF4,
+0x12, 0xD7, 0x12, 0x97, 0x89, 0xA7, 0x48, 0xA7,
+0xAA, 0xA7, 0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA7,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x0E, 0x22, 0xDB, 0xA2,
+0x0F, 0x6B, 0x18, 0x94, 0x6C, 0xEE, 0xFF, 0x6D,
+0xAC, 0xEE, 0x8E, 0xEE, 0x06, 0x2E, 0x98, 0xA2,
+0x6C, 0xEC, 0x19, 0x93, 0xAC, 0xEC, 0x6E, 0xEC,
+0x19, 0x24, 0x20, 0x48, 0x80, 0x70, 0xD4, 0x61,
+0x17, 0x97, 0x16, 0x91, 0x15, 0x90, 0x00, 0xEF,
+0x0C, 0x63, 0x40, 0xEA, 0x00, 0x65, 0x08, 0xD2,
+0x01, 0x6A, 0x07, 0xD2, 0x12, 0x92, 0xA9, 0xF6,
+0x80, 0x98, 0xA9, 0xF6, 0xA4, 0x98, 0x05, 0xD2,
+0x04, 0x6A, 0x06, 0xD1, 0x04, 0xD2, 0x6B, 0x6F,
+0x03, 0x6E, 0x3E, 0x10, 0x7D, 0x67, 0x20, 0xF0,
+0x78, 0xA3, 0x01, 0x6C, 0x8C, 0xEB, 0x6C, 0xED,
+0xE3, 0x25, 0x73, 0xA7, 0x01, 0x73, 0xE0, 0x61,
+0xBD, 0xA7, 0x7C, 0xA7, 0x9E, 0xA7, 0xA0, 0x35,
+0x6D, 0xED, 0x7F, 0xA7, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x72, 0xA3, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x01, 0x71, 0x12, 0xD3, 0x28, 0xA2,
+0xF3, 0xF0, 0x70, 0x9C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0xF7, 0xF0, 0x01, 0x68, 0x40, 0x32,
+0x00, 0x30, 0x00, 0x30, 0x13, 0xD3, 0xD2, 0xF4,
+0x54, 0x9A, 0xD8, 0xF5, 0x00, 0x6C, 0xC1, 0x61,
+0x40, 0xEA, 0x00, 0x65, 0x08, 0xD2, 0x01, 0x6A,
+0x07, 0xD2, 0x12, 0x92, 0xA9, 0xF6, 0x80, 0x98,
+0xA9, 0xF6, 0xA4, 0x98, 0x05, 0xD2, 0x04, 0x6A,
+0x06, 0xD1, 0x04, 0xD2, 0x6A, 0x6F, 0x02, 0x6E,
+0x13, 0x93, 0x40, 0xEB, 0x00, 0x65, 0xA8, 0x17,
+0xF1, 0x63, 0x1D, 0x62, 0x1C, 0xD1, 0x1B, 0xD0,
+0x00, 0x69, 0x64, 0x67, 0x1F, 0xD5, 0x20, 0xD6,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC6, 0xF4, 0xE8, 0x9A, 0x0A, 0x04, 0x20, 0x6E,
+0x3D, 0xE7, 0x07, 0x67, 0xA7, 0x67, 0xD1, 0x18,
+0xE7, 0xF4, 0x12, 0xD3, 0xF0, 0x67, 0x49, 0xA7,
+0x08, 0xA0, 0x8A, 0xA7, 0x40, 0x32, 0x0D, 0xEA,
+0x0B, 0xA7, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xEA,
+0x00, 0xF6, 0x00, 0x30, 0x4D, 0xE8, 0x12, 0x93,
+0x0F, 0x20, 0x9B, 0xA0, 0x0F, 0x6D, 0xFF, 0x6A,
+0xAC, 0xEC, 0x4C, 0xEC, 0x6E, 0xEC, 0x3A, 0x65,
+0x07, 0x2C, 0x58, 0xA0, 0x99, 0x67, 0xAC, 0xEA,
+0x8C, 0xEA, 0x1F, 0x94, 0x8E, 0xEA, 0x50, 0x22,
+0x20, 0x49, 0x80, 0x71, 0xD1, 0x61, 0x1D, 0x97,
+0x1C, 0x91, 0x1B, 0x90, 0x00, 0xEF, 0x0F, 0x63,
+0x17, 0xD3, 0x79, 0x67, 0x16, 0xD3, 0x74, 0xA7,
+0x18, 0xD5, 0x15, 0xD3, 0xF3, 0xF0, 0x70, 0x9C,
+0xD2, 0xF4, 0x54, 0x9A, 0xD8, 0xF5, 0x00, 0x6C,
+0x40, 0xEA, 0x14, 0xD3, 0x08, 0xD2, 0x13, 0x92,
+0x18, 0x95, 0x17, 0x93, 0x06, 0xD2, 0x12, 0x92,
+0xA9, 0xF6, 0x80, 0x9D, 0xA9, 0xF6, 0xA4, 0x9D,
+0x05, 0xD2, 0x04, 0x6A, 0x04, 0xD2, 0x14, 0x92,
+0x07, 0xD3, 0x6B, 0x6F, 0x40, 0xEA, 0x03, 0x6E,
+0x4D, 0xA0, 0x02, 0x6B, 0x6C, 0xEA, 0x16, 0x93,
+0x6C, 0xEA, 0x3B, 0x65, 0xD4, 0x2A, 0x4D, 0xA0,
+0x07, 0x6B, 0x4E, 0x32, 0x6C, 0xEA, 0x79, 0x67,
+0x6C, 0xEA, 0x15, 0x93, 0x42, 0xEB, 0xCB, 0x60,
+0xED, 0xA1, 0x4C, 0xA1, 0x97, 0xF0, 0x19, 0x6E,
+0xE0, 0x37, 0x4D, 0xEF, 0x4E, 0xA1, 0xC0, 0x36,
+0x20, 0xF0, 0x80, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0xED, 0xEA, 0xEF, 0xA1, 0xC0, 0x36, 0xDD, 0xF2,
+0x01, 0x4E, 0x00, 0xF6, 0xE0, 0x37, 0x4D, 0xEF,
+0xD1, 0x18, 0x78, 0xCB, 0x00, 0x6D, 0xB3, 0x17,
+0x5D, 0x67, 0x20, 0xF0, 0x58, 0xA2, 0x01, 0x6B,
+0x6C, 0xEA, 0x79, 0x67, 0x6C, 0xEA, 0xAB, 0x22,
+0x72, 0xA7, 0x01, 0x6A, 0x4E, 0xEB, 0xA7, 0x2B,
+0x5D, 0xA7, 0x3C, 0xA7, 0x9E, 0xA7, 0x40, 0x32,
+0x2D, 0xEA, 0x3F, 0xA7, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0x20, 0x92, 0xF7, 0xF0, 0x01, 0x6C, 0xF7, 0xF0,
+0x01, 0x6D, 0x01, 0x72, 0x52, 0xA1, 0x80, 0x34,
+0xA0, 0x35, 0x12, 0xD2, 0x48, 0xA0, 0xD8, 0x67,
+0x80, 0x34, 0x13, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xA0, 0x35, 0x8C, 0x2E,
+0xF3, 0xF0, 0x10, 0x9C, 0xD2, 0xF4, 0x54, 0x9A,
+0xD8, 0xF5, 0x00, 0x6C, 0x14, 0xD5, 0x40, 0xEA,
+0x38, 0x67, 0x08, 0xD2, 0x13, 0x92, 0x14, 0x95,
+0x07, 0xD1, 0x06, 0xD2, 0x12, 0x92, 0xA9, 0xF6,
+0x80, 0x9D, 0xA9, 0xF6, 0xA4, 0x9D, 0x05, 0xD2,
+0x04, 0x6A, 0x04, 0xD2, 0x6A, 0x6F, 0x40, 0xE8,
+0x02, 0x6E, 0x6D, 0x17, 0x70, 0xA4, 0x02, 0x6C,
+0x01, 0x6A, 0x6C, 0xEC, 0x06, 0x2C, 0x6A, 0x32,
+0x01, 0x6B, 0x6E, 0xEA, 0x6C, 0xEA, 0xFF, 0x6B,
+0x6C, 0xEA, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x04, 0xD4, 0x04, 0x94,
+0xD1, 0x18, 0x87, 0x3D, 0x00, 0x65, 0x01, 0x72,
+0x11, 0x60, 0x04, 0x95, 0x17, 0x6E, 0x89, 0xA5,
+0x48, 0xA5, 0x80, 0x34, 0x4D, 0xEC, 0x4A, 0xA5,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x8B, 0xA5,
+0x00, 0x6D, 0x00, 0xF6, 0x80, 0x34, 0xD1, 0x18,
+0xC0, 0x3B, 0x4D, 0xEC, 0x07, 0x97, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0x00, 0x6A, 0x1D, 0x24,
+0x1C, 0x25, 0xAA, 0xEC, 0x01, 0x6A, 0x19, 0x60,
+0x4B, 0xA5, 0x6B, 0xA4, 0x4E, 0xEB, 0x00, 0x6A,
+0x14, 0x2B, 0x69, 0xA4, 0xC9, 0xA5, 0xCE, 0xEB,
+0x10, 0x2B, 0xDB, 0xA5, 0x7B, 0xA4, 0x0F, 0x6F,
+0xCE, 0xEB, 0xEC, 0xEB, 0xFF, 0x6E, 0xCC, 0xEB,
+0x08, 0x2B, 0x6C, 0xA4, 0x4C, 0xA5, 0x6E, 0xEA,
+0xEC, 0xEA, 0xCC, 0xEA, 0x01, 0x5A, 0x58, 0x67,
+0xCC, 0xEA, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x7B, 0xA4, 0x0F, 0x6E, 0xFF, 0x6D, 0xCC, 0xEB,
+0xAC, 0xEB, 0x57, 0x2B, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0x58, 0xA4, 0xCC, 0xEA,
+0xFF, 0x4A, 0xAC, 0xEA, 0x04, 0x5A, 0x48, 0x60,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x44, 0x32, 0x88, 0xF4, 0x14, 0x4B, 0x6D, 0xE2,
+0x20, 0xAB, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x88, 0xF4, 0x0C, 0x4B, 0x69, 0xE2,
+0x60, 0xAA, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0x04, 0x67, 0xE0, 0x37, 0xB8, 0xA4, 0x9B, 0xA4,
+0xF3, 0xF3, 0x4C, 0x9F, 0x01, 0x6E, 0xB2, 0x35,
+0x92, 0x34, 0x06, 0xD3, 0x40, 0xEA, 0x05, 0xD7,
+0x05, 0x97, 0xB8, 0xA0, 0x9B, 0xA0, 0x04, 0xD2,
+0xF3, 0xF3, 0x4C, 0x9F, 0x00, 0x6E, 0xB2, 0x35,
+0x40, 0xEA, 0x92, 0x34, 0x99, 0xA0, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0x80, 0x34, 0x88, 0x34,
+0x55, 0xE4, 0x04, 0x92, 0x83, 0xED, 0x98, 0x67,
+0xC0, 0x36, 0x06, 0x93, 0x41, 0xE4, 0xD2, 0xF4,
+0x48, 0x9E, 0x83, 0x67, 0x40, 0xEA, 0x04, 0xD6,
+0x04, 0x96, 0xB0, 0x67, 0x91, 0x67, 0xD2, 0xF4,
+0x48, 0x9E, 0x40, 0xEA, 0x00, 0x65, 0x0B, 0x97,
+0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF, 0x06, 0x63,
+0x38, 0xF4, 0x18, 0x6B, 0x38, 0xF4, 0x1C, 0x69,
+0xC4, 0x17, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF5, 0x63, 0x15, 0x62, 0x14, 0xD1, 0x13, 0xD0,
+0x4B, 0xA4, 0x69, 0xA4, 0xBB, 0xA4, 0x04, 0x67,
+0x8C, 0xA4, 0xF7, 0xF0, 0x01, 0x69, 0x0F, 0x6E,
+0x20, 0x31, 0x0C, 0xD2, 0xCC, 0xED, 0xFF, 0x6A,
+0xCC, 0xEC, 0x20, 0x31, 0x4C, 0xED, 0x4C, 0xEC,
+0x10, 0xD2, 0xF3, 0xF0, 0x50, 0x99, 0x0D, 0xD5,
+0x0E, 0xD4, 0x1A, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xA9, 0xF6, 0x80, 0x9A,
+0xA9, 0xF6, 0xA4, 0x9A, 0x01, 0x6E, 0x58, 0x67,
+0x05, 0xD3, 0x04, 0xD6, 0x70, 0x6F, 0x04, 0x6E,
+0x40, 0xEA, 0x0F, 0xD3, 0x7D, 0x67, 0x20, 0xF0,
+0x70, 0xA3, 0x5D, 0x67, 0x00, 0x6C, 0x78, 0xC2,
+0x0F, 0x93, 0x20, 0xF0, 0x8C, 0xC2, 0x08, 0xD4,
+0x79, 0xC2, 0x7D, 0x67, 0x7C, 0xAB, 0x09, 0xD4,
+0x0A, 0xD4, 0x6D, 0xCA, 0x00, 0x6B, 0x6E, 0xCA,
+0x0C, 0x92, 0x10, 0x6C, 0x22, 0x5A, 0x58, 0x67,
+0x01, 0x72, 0x78, 0x67, 0x5D, 0x67, 0x6F, 0xCA,
+0x00, 0x6B, 0x70, 0xCA, 0x7D, 0x67, 0x20, 0xF0,
+0x74, 0xA3, 0x20, 0xF0, 0x66, 0xC2, 0x6E, 0xA0,
+0x10, 0x92, 0xF7, 0xF0, 0x01, 0x68, 0x8C, 0xEB,
+0x00, 0x30, 0x6C, 0xEA, 0x00, 0x30, 0x2A, 0x2A,
+0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65, 0x1C, 0x2A,
+0x00, 0x6A, 0xD1, 0x18, 0x08, 0x06, 0x0C, 0xD2,
+0x0C, 0x92, 0x01, 0x6C, 0x0F, 0x22, 0x85, 0x6B,
+0x4C, 0xEB, 0x0C, 0x23, 0xA9, 0xF6, 0x80, 0x98,
+0xF3, 0xF0, 0x50, 0x99, 0xA9, 0xF6, 0xA4, 0x98,
+0x00, 0x6B, 0x04, 0xD3, 0x6F, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x00, 0x6C, 0x15, 0x97, 0x14, 0x91,
+0x13, 0x90, 0x44, 0x67, 0x00, 0xEF, 0x0B, 0x63,
+0x97, 0xF0, 0x1A, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0xC8, 0xF0, 0x01, 0x4D, 0xD1, 0x18, 0x9A, 0x90,
+0x06, 0x04, 0xDB, 0x17, 0xD1, 0x18, 0xCD, 0x8A,
+0x00, 0x65, 0xD6, 0x22, 0x00, 0x6D, 0xF6, 0x17,
+0xF5, 0x63, 0x15, 0x62, 0x14, 0xD1, 0x13, 0xD0,
+0x04, 0x67, 0x69, 0xA0, 0x88, 0xA4, 0x4A, 0xA0,
+0x60, 0x33, 0x6D, 0xEC, 0x6B, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x00, 0xF6, 0x60, 0x33,
+0x4D, 0xEB, 0x0D, 0xD3, 0x5D, 0xA0, 0x3C, 0xA0,
+0x7E, 0xA0, 0x40, 0x32, 0x2D, 0xEA, 0x3F, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6,
+0x20, 0x31, 0x4D, 0xE9, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x0D, 0x94, 0x0A, 0x6E,
+0x01, 0x6D, 0xD1, 0x18, 0xC0, 0x3B, 0x0B, 0xD2,
+0x0D, 0x92, 0x2B, 0x22, 0x6E, 0xA2, 0x04, 0x6A,
+0x4C, 0xEB, 0xFF, 0x6A, 0x4C, 0xEB, 0x25, 0x2B,
+0xF7, 0xF0, 0x01, 0x6F, 0x0D, 0x94, 0xE0, 0x37,
+0xE0, 0x37, 0xB2, 0xF5, 0x7C, 0x9F, 0xB8, 0xA4,
+0x9B, 0xA4, 0x1B, 0x65, 0x0F, 0x6B, 0x6C, 0xED,
+0x6C, 0xEC, 0xDD, 0x67, 0x20, 0xF0, 0x70, 0xC6,
+0x4C, 0xED, 0x4C, 0xEC, 0x00, 0x6E, 0x0A, 0xD2,
+0x58, 0x67, 0x40, 0xEA, 0x0E, 0xD7, 0x0D, 0x92,
+0x0E, 0x97, 0x00, 0x6E, 0x9B, 0xA2, 0x5D, 0x67,
+0x20, 0xF0, 0x70, 0xA2, 0x0A, 0x92, 0xB2, 0xF5,
+0xFC, 0x9F, 0x6C, 0xEC, 0x00, 0x6D, 0x40, 0xEF,
+0x4C, 0xEC, 0x4D, 0xA0, 0x6C, 0xA0, 0x8E, 0xA0,
+0x40, 0x32, 0x6D, 0xEA, 0x6F, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6, 0x60, 0x33,
+0x4D, 0xEB, 0x62, 0x32, 0x49, 0xC0, 0x42, 0x32,
+0x4A, 0xC0, 0xA1, 0xA3, 0x00, 0xF6, 0x62, 0x32,
+0x4B, 0xC0, 0x40, 0xA3, 0x82, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA3, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x0A, 0xEA, 0x68, 0xC0, 0x80, 0xF0, 0x08, 0x60,
+0x42, 0x34, 0x4C, 0xC0, 0x8D, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x4F, 0xC0, 0x8E, 0xC0,
+0x00, 0x6A, 0x98, 0xA3, 0x0E, 0xD2, 0x10, 0x6A,
+0x4B, 0xEA, 0x8C, 0xEA, 0xF0, 0x72, 0x05, 0x60,
+0x83, 0x67, 0xD1, 0x18, 0xAE, 0x3D, 0x0A, 0xD3,
+0x0A, 0x93, 0x41, 0xA1, 0xC0, 0xA1, 0x82, 0xA1,
+0x40, 0x32, 0xCD, 0xEA, 0xC3, 0xA1, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6, 0xC0, 0x36,
+0x4D, 0xEE, 0x45, 0xA1, 0xA4, 0xA1, 0x86, 0xA1,
+0x40, 0x32, 0xAD, 0xEA, 0xA7, 0xA1, 0x80, 0x34,
+0x80, 0x34, 0xF1, 0xA3, 0x8D, 0xEA, 0x00, 0xF6,
+0xA0, 0x35, 0x4D, 0xED, 0x50, 0xA3, 0x92, 0xA3,
+0xE0, 0x37, 0x4D, 0xEF, 0x53, 0xA3, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEF, 0x00, 0xF6, 0x40, 0x32,
+0xED, 0xEA, 0x80, 0xF5, 0x42, 0x34, 0x40, 0x32,
+0x48, 0x32, 0x11, 0xD3, 0xCD, 0xE2, 0x43, 0xEB,
+0xF7, 0xF0, 0x01, 0x6F, 0x58, 0x67, 0xB5, 0xE4,
+0xE0, 0x37, 0xE0, 0x37, 0xA9, 0xE2, 0x0C, 0xD2,
+0xA9, 0xA1, 0xF3, 0xF3, 0x4C, 0x9F, 0x88, 0xA1,
+0x00, 0x6E, 0x0A, 0xD3, 0x40, 0xEA, 0x10, 0xD7,
+0x10, 0x97, 0x0F, 0xD2, 0xA9, 0xA1, 0xF3, 0xF3,
+0x4C, 0x9F, 0x88, 0xA1, 0x40, 0xEA, 0x01, 0x6E,
+0x0C, 0x93, 0x43, 0xEB, 0x08, 0x61, 0x0C, 0x93,
+0x6E, 0xEA, 0x11, 0x93, 0x3F, 0x2A, 0x0F, 0x92,
+0x0A, 0x94, 0x43, 0xEC, 0x3B, 0x60, 0x0A, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x06, 0xD3,
+0x0F, 0x93, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x05, 0xD3, 0x02, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xA9, 0xF6,
+0x80, 0x9B, 0xA9, 0xF6, 0xA4, 0x9B, 0x20, 0xF3,
+0x1B, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x0B, 0x94, 0x15, 0x97,
+0x14, 0x91, 0x13, 0x90, 0x00, 0xEF, 0x0B, 0x63,
+0xA1, 0xA0, 0x40, 0xA0, 0x82, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x42, 0x34, 0x4C, 0xC0, 0x8D, 0xC0, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x4F, 0xC0, 0x8E, 0xC0,
+0x01, 0x6A, 0x6B, 0x17, 0x54, 0xA3, 0x01, 0x6C,
+0xFF, 0x6E, 0x8C, 0xEA, 0xCC, 0xEA, 0x1B, 0x22,
+0x20, 0xF0, 0x41, 0xA3, 0x95, 0xA3, 0x01, 0x4A,
+0xCC, 0xEA, 0x20, 0xF0, 0x41, 0xC3, 0x8E, 0xEA,
+0x12, 0x2A, 0x20, 0xF0, 0x41, 0xC3, 0x90, 0xA0,
+0xB6, 0xA3, 0x92, 0x34, 0xD1, 0x18, 0xE3, 0x3A,
+0xCC, 0xEC, 0x62, 0x67, 0x48, 0xC0, 0x42, 0x32,
+0x49, 0xC0, 0x42, 0x32, 0x4A, 0xC0, 0x00, 0xF6,
+0x62, 0x32, 0x4B, 0xC0, 0xBC, 0x23, 0x4D, 0xA3,
+0x01, 0x6C, 0x8C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x47, 0x2A, 0xCE, 0xA3, 0xAD, 0xA3, 0xD3, 0xF3,
+0x44, 0x9C, 0x03, 0x6C, 0x8C, 0xEE, 0xBA, 0x35,
+0xC8, 0x36, 0x0F, 0xD3, 0xAD, 0xEE, 0x0A, 0x93,
+0xC0, 0x36, 0xC8, 0x36, 0x97, 0xF0, 0x19, 0x6F,
+0xDB, 0xE3, 0xE0, 0x37, 0xC3, 0xEB, 0xE0, 0x37,
+0x0C, 0x93, 0xA9, 0xA1, 0x88, 0xA1, 0xA0, 0xF0,
+0x19, 0x4F, 0x04, 0xD7, 0xF8, 0x67, 0xFF, 0xE3,
+0x40, 0xEA, 0x05, 0xD0, 0x0F, 0x93, 0x0A, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x10, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0x89, 0x17, 0x0F, 0xD3, 0x0A, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x0F, 0xF0,
+0xC0, 0x43, 0x63, 0xEE, 0x0C, 0x93, 0x97, 0xF0,
+0x19, 0x6C, 0x40, 0x32, 0xEF, 0x43, 0x68, 0xA1,
+0x80, 0x34, 0xA9, 0xA1, 0xD3, 0xF3, 0x44, 0x9A,
+0x80, 0x34, 0x3B, 0x65, 0x3E, 0xF6, 0x19, 0x4C,
+0x78, 0x67, 0x04, 0xD4, 0xFD, 0xE3, 0x05, 0xD0,
+0x40, 0xEA, 0x99, 0x67, 0x0F, 0x93, 0xD8, 0x22,
+0x97, 0xF0, 0x19, 0x6E, 0xF7, 0xF0, 0x01, 0x6A,
+0xC0, 0x36, 0x40, 0x32, 0xC0, 0x36, 0x3F, 0xF6,
+0x1D, 0x4E, 0x40, 0x32, 0xA9, 0xA1, 0x88, 0xA1,
+0x0F, 0xD6, 0x04, 0xD6, 0xD3, 0xF3, 0x44, 0x9A,
+0x0A, 0x96, 0x0C, 0x97, 0x05, 0xD0, 0x40, 0xEA,
+0x11, 0xD3, 0x11, 0x93, 0xC1, 0x22, 0x5D, 0x67,
+0x20, 0xF0, 0x48, 0xA2, 0xFF, 0x6C, 0x3C, 0x65,
+0x40, 0xC1, 0x0A, 0x92, 0x0D, 0x94, 0xA3, 0x67,
+0x42, 0x32, 0x41, 0xC1, 0x42, 0x32, 0x42, 0xC1,
+0x0A, 0x92, 0x0A, 0xD3, 0x00, 0xF6, 0x42, 0x32,
+0x43, 0xC1, 0x5D, 0x67, 0x20, 0xF0, 0x50, 0xA2,
+0x44, 0xC1, 0x0C, 0x92, 0x42, 0x32, 0x45, 0xC1,
+0x42, 0x32, 0x46, 0xC1, 0x0C, 0x92, 0x00, 0xF6,
+0x42, 0x32, 0xD1, 0x18, 0x9D, 0x3D, 0x47, 0xC1,
+0x0A, 0x93, 0x6A, 0x2A, 0x50, 0xA0, 0x08, 0x6C,
+0x8C, 0xEA, 0x99, 0x67, 0x8C, 0xEA, 0x21, 0x22,
+0x4E, 0xA3, 0x04, 0x6C, 0xF7, 0xF0, 0x01, 0x6D,
+0x8C, 0xEA, 0x99, 0x67, 0x8C, 0xEA, 0xF7, 0xF0,
+0x01, 0x6C, 0xA0, 0x35, 0x80, 0x34, 0xA0, 0x35,
+0x80, 0x34, 0xD2, 0xA1, 0x80, 0xF0, 0x1C, 0x22,
+0xF3, 0xF0, 0x50, 0x9D, 0xA9, 0xF6, 0xA4, 0x9C,
+0xA9, 0xF6, 0x80, 0x9C, 0x05, 0xD6, 0x01, 0x6E,
+0x04, 0xD6, 0x75, 0x6F, 0x40, 0xEA, 0x02, 0x6E,
+0x12, 0x6C, 0xB1, 0x18, 0x90, 0xD1, 0x00, 0x65,
+0x0A, 0x93, 0x4E, 0xA3, 0x04, 0x6C, 0xFF, 0x6F,
+0x8C, 0xEA, 0xEC, 0xEA, 0x80, 0xF0, 0x1B, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xF2, 0xA1, 0xA9, 0xA1,
+0x0A, 0xD2, 0x10, 0x92, 0x88, 0xA1, 0x00, 0x6E,
+0xF3, 0xF3, 0x4C, 0x9A, 0x0D, 0xD7, 0x40, 0xEA,
+0x11, 0xD3, 0x0C, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0xD8, 0xF5, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65,
+0x07, 0xD2, 0x03, 0x6A, 0x0C, 0x95, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0D, 0x97, 0x06, 0xD5, 0xA9, 0xF6, 0x80, 0x9A,
+0xA9, 0xF6, 0xA4, 0x9A, 0x0A, 0x92, 0x05, 0xD7,
+0x02, 0x6E, 0x40, 0xEA, 0x78, 0x6F, 0x57, 0xA0,
+0x01, 0x6C, 0x11, 0x93, 0x8C, 0xEA, 0x08, 0x22,
+0x56, 0xA0, 0xFF, 0x72, 0xA0, 0xF0, 0x05, 0x61,
+0xD1, 0x18, 0x5D, 0x1B, 0x0A, 0xD3, 0x0A, 0x93,
+0x4D, 0xA3, 0x01, 0x6C, 0x8C, 0xEA, 0x0D, 0x2A,
+0x83, 0x67, 0x0A, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0xC0, 0x3B, 0x0A, 0xD3, 0x00, 0x6D, 0xD1, 0x18,
+0x8F, 0x3C, 0x90, 0x67, 0x0A, 0x93, 0xBF, 0xF6,
+0x16, 0x22, 0x83, 0x67, 0x0A, 0x6E, 0xD1, 0x18,
+0xC0, 0x3B, 0x02, 0x6D, 0xD1, 0x18, 0x87, 0x3D,
+0x90, 0x67, 0x01, 0x72, 0xBF, 0xF6, 0x0B, 0x61,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0xD3, 0xF3, 0x40, 0x98, 0x0F, 0x96, 0xA9, 0xA1,
+0x40, 0xEA, 0x88, 0xA1, 0x97, 0xF0, 0x19, 0x6E,
+0xD3, 0xF3, 0x40, 0x98, 0xA9, 0xA1, 0xC0, 0x36,
+0xC0, 0x36, 0xA0, 0xF0, 0x19, 0x4E, 0x40, 0xEA,
+0x88, 0xA1, 0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3,
+0x40, 0x98, 0xA9, 0xA1, 0xC0, 0x36, 0xC0, 0x36,
+0x3E, 0xF6, 0x19, 0x4E, 0x40, 0xEA, 0x88, 0xA1,
+0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3, 0x40, 0x98,
+0xC0, 0x36, 0xA9, 0xA1, 0xC0, 0x36, 0x5F, 0xF6,
+0x1D, 0x4E, 0x40, 0xEA, 0x88, 0xA1, 0x7F, 0x16,
+0x4B, 0xA3, 0x05, 0xD6, 0x01, 0x6E, 0x22, 0x5A,
+0x0A, 0xD3, 0xF3, 0xF0, 0x50, 0x9D, 0x04, 0xD6,
+0xA9, 0xF6, 0xA4, 0x9C, 0xA9, 0xF6, 0x80, 0x9C,
+0x05, 0x61, 0x76, 0x6F, 0x40, 0xEA, 0x02, 0x6E,
+0x11, 0x6C, 0x5F, 0x17, 0x77, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x10, 0x6C, 0x5A, 0x17, 0x83, 0x67,
+0x0C, 0xD7, 0xD1, 0x18, 0xDE, 0x3D, 0x0A, 0xD3,
+0x0A, 0x93, 0x0C, 0x97, 0x5F, 0xF6, 0x1F, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xB8, 0xA3, 0x9B, 0xA3,
+0x0A, 0xD2, 0x52, 0xA1, 0x0F, 0x6E, 0xCC, 0xED,
+0x0C, 0xD2, 0x48, 0xA3, 0xCC, 0xEC, 0xEC, 0xED,
+0x0D, 0xD2, 0x10, 0x92, 0xEC, 0xEC, 0x00, 0x6E,
+0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA, 0x11, 0xD3,
+0x10, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A, 0xD8, 0xF5,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0x08, 0xD2,
+0x0D, 0x92, 0x10, 0x95, 0x79, 0x6F, 0x06, 0xD2,
+0x0C, 0x92, 0x07, 0xD5, 0x05, 0xD2, 0x04, 0x6A,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xA9, 0xF6, 0x80, 0x9A, 0xA9, 0xF6,
+0xA4, 0x9A, 0x0A, 0x92, 0x40, 0xEA, 0x02, 0x6E,
+0x52, 0x17, 0x5F, 0xF7, 0x1D, 0x22, 0xD1, 0x18,
+0x5D, 0x1B, 0x0A, 0xD3, 0x0E, 0x92, 0x0A, 0x93,
+0x5F, 0xF7, 0x16, 0x22, 0x56, 0xA0, 0xFF, 0x4A,
+0x56, 0xC0, 0x52, 0x17, 0xFC, 0x63, 0x07, 0x62,
+0x04, 0xD4, 0x04, 0x94, 0xD1, 0x18, 0x87, 0x3D,
+0x00, 0x65, 0x01, 0x72, 0x03, 0x60, 0xD1, 0x18,
+0x20, 0x3E, 0x04, 0x94, 0x07, 0x97, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xF9, 0x63, 0x0D, 0x62,
+0x0C, 0xD1, 0x0B, 0xD0, 0x06, 0xD4, 0x06, 0x94,
+0x81, 0x6F, 0x02, 0x6E, 0x5D, 0xA4, 0x1C, 0xA4,
+0x7E, 0xA4, 0x40, 0x32, 0x0D, 0xEA, 0x1F, 0xA4,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6,
+0x00, 0x30, 0x00, 0x6B, 0x4D, 0xE8, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xA9, 0xF6, 0x80, 0x9B, 0xA9, 0xF6, 0xA4, 0x9B,
+0xF3, 0xF0, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x28, 0xA0,
+0x49, 0xA0, 0x60, 0x33, 0x97, 0xF0, 0x19, 0x6E,
+0xD3, 0xF3, 0xE0, 0x9B, 0xC0, 0x36, 0xC0, 0x36,
+0xA2, 0x67, 0x91, 0x67, 0x3F, 0xF6, 0x1D, 0x4E,
+0x09, 0xD3, 0x40, 0xEF, 0x08, 0xD2, 0x09, 0x93,
+0x08, 0x92, 0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3,
+0xE0, 0x9B, 0xC0, 0x36, 0xC0, 0x36, 0xA2, 0x67,
+0xA0, 0xF0, 0x19, 0x4E, 0x40, 0xEF, 0x91, 0x67,
+0x09, 0x93, 0x08, 0x92, 0x97, 0xF0, 0x19, 0x6E,
+0xD3, 0xF3, 0x60, 0x9B, 0xC0, 0x36, 0xC0, 0x36,
+0xA2, 0x67, 0x3E, 0xF6, 0x19, 0x4E, 0x40, 0xEB,
+0x91, 0x67, 0x20, 0xF0, 0x45, 0xA0, 0x20, 0xF0,
+0xA4, 0xA0, 0x20, 0xF0, 0x66, 0xA0, 0x40, 0x32,
+0xAD, 0xEA, 0x20, 0xF0, 0xA7, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6, 0xA0, 0x35,
+0x20, 0xF0, 0x69, 0xA0, 0x4D, 0xED, 0x20, 0xF0,
+0x48, 0xA0, 0x20, 0xF0, 0x8A, 0xA0, 0x60, 0x33,
+0x4D, 0xEB, 0x20, 0xF0, 0x4B, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6, 0x40, 0x32,
+0x6D, 0xEA, 0x20, 0xF0, 0x61, 0xA0, 0x20, 0xF0,
+0xC0, 0xA0, 0x20, 0xF0, 0x82, 0xA0, 0x60, 0x33,
+0xCD, 0xEB, 0x20, 0xF0, 0xC3, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0x20, 0xF0, 0xED, 0xA0, 0x8D, 0xEB,
+0x00, 0xF6, 0xC0, 0x36, 0x6D, 0xEE, 0x20, 0xF0,
+0x6C, 0xA0, 0x20, 0xF0, 0x8E, 0xA0, 0xE0, 0x37,
+0x6D, 0xEF, 0x20, 0xF0, 0x6F, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEF, 0x00, 0xF6, 0x60, 0x33,
+0xED, 0xEB, 0x62, 0x34, 0x70, 0xC6, 0x91, 0xC6,
+0x00, 0xF6, 0x62, 0x33, 0x82, 0x34, 0x92, 0xC6,
+0x73, 0xC6, 0x20, 0xF0, 0xD1, 0xA0, 0x20, 0xF0,
+0x70, 0xA0, 0x20, 0xF0, 0x92, 0xA0, 0xC0, 0x36,
+0x6D, 0xEE, 0x20, 0xF0, 0x73, 0xA0, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEE, 0x00, 0xF6, 0x60, 0x33,
+0xCD, 0xEB, 0x70, 0xC5, 0x62, 0x34, 0x00, 0xF6,
+0x62, 0x33, 0x73, 0xC5, 0x06, 0x93, 0x91, 0xC5,
+0x82, 0x34, 0x92, 0xC5, 0x42, 0x34, 0x8D, 0xC3,
+0x82, 0x34, 0x8E, 0xC3, 0x00, 0xF6, 0x42, 0x34,
+0x4C, 0xC3, 0x8F, 0xC3, 0x75, 0xA0, 0xB4, 0xA0,
+0x96, 0xA0, 0x60, 0x33, 0xAD, 0xEB, 0xB7, 0xA0,
+0x80, 0x34, 0x80, 0x34, 0xD9, 0xA0, 0x8D, 0xEB,
+0x00, 0xF6, 0xA0, 0x35, 0x6D, 0xED, 0x78, 0xA0,
+0x9A, 0xA0, 0xC0, 0x36, 0x6D, 0xEE, 0x7B, 0xA0,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEE, 0x00, 0xF6,
+0x60, 0x33, 0xA2, 0x34, 0xCD, 0xEB, 0x81, 0xC0,
+0x82, 0x34, 0x82, 0xC0, 0x64, 0xC0, 0x62, 0x34,
+0x00, 0xF6, 0x62, 0x33, 0x67, 0xC0, 0x7B, 0xA2,
+0x85, 0xC0, 0x82, 0x34, 0x86, 0xC0, 0x0F, 0x6C,
+0x8C, 0xEB, 0x68, 0xC0, 0x58, 0xA2, 0x01, 0x6B,
+0xA0, 0xC0, 0x8C, 0xEA, 0x49, 0xC0, 0x06, 0x94,
+0x20, 0xF0, 0x54, 0xA0, 0x00, 0xF6, 0xA2, 0x35,
+0xA3, 0xC0, 0x4C, 0xEB, 0x50, 0xA4, 0x09, 0x6D,
+0xAB, 0xED, 0x6C, 0x33, 0xAC, 0xEA, 0x6D, 0xEA,
+0x50, 0xC4, 0xD1, 0x18, 0x20, 0x3E, 0x06, 0x94,
+0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90, 0x00, 0xEF,
+0x07, 0x63, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0x49, 0xA4, 0x08, 0xA4,
+0x6A, 0xA4, 0x40, 0x32, 0x0D, 0xEA, 0x0B, 0xA4,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA, 0x00, 0xF6,
+0x00, 0x30, 0x4D, 0xE8, 0x24, 0x67, 0x00, 0x6D,
+0x90, 0x67, 0xD1, 0x18, 0xC0, 0x3B, 0x0B, 0x6E,
+0x01, 0x6D, 0xD1, 0x18, 0x8F, 0x3C, 0x91, 0x67,
+0x0D, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x73, 0xF6, 0x4C, 0x9A, 0x40, 0xEA,
+0x01, 0x6C, 0x0B, 0x6E, 0x02, 0x6D, 0xD1, 0x18,
+0xC0, 0x3B, 0x90, 0x67, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x04, 0xD4, 0x04, 0x94,
+0xD1, 0x18, 0x87, 0x3D, 0x00, 0x65, 0x01, 0x72,
+0x03, 0x60, 0xD1, 0x18, 0x17, 0x40, 0x04, 0x94,
+0x07, 0x97, 0x00, 0xEF, 0x04, 0x63, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xC6, 0xF4, 0x68, 0x9B, 0x94, 0x30, 0x0A, 0xD4,
+0x0C, 0xD6, 0x01, 0xE3, 0x50, 0xA0, 0x01, 0x6C,
+0x25, 0x67, 0x8C, 0xEA, 0x09, 0x2A, 0x0A, 0x94,
+0x02, 0x6E, 0xD1, 0x18, 0x55, 0x3B, 0xB1, 0x67,
+0x4B, 0xEA, 0xC0, 0xF7, 0x42, 0x37, 0x9E, 0x10,
+0x40, 0x98, 0x4A, 0xE8, 0xF4, 0x60, 0x9D, 0xA0,
+0x5C, 0xA0, 0xBE, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x5F, 0xA0, 0xA0, 0x35, 0x0C, 0x93, 0xA0, 0x35,
+0xF7, 0xF0, 0x01, 0x6F, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0xE0, 0x37, 0x8D, 0xEA, 0xE0, 0x37,
+0x0F, 0x23, 0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3,
+0x60, 0x9F, 0xA9, 0xA2, 0x88, 0xA2, 0xC0, 0x36,
+0xC0, 0x36, 0x60, 0xF5, 0x0D, 0x4E, 0x05, 0xD7,
+0x40, 0xEB, 0x04, 0xD2, 0x05, 0x97, 0x04, 0x92,
+0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3, 0x60, 0x9F,
+0xA9, 0xA2, 0x88, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0x3F, 0xF6, 0x1D, 0x4E, 0x05, 0xD7, 0x40, 0xEB,
+0x04, 0xD2, 0x04, 0x92, 0x05, 0x97, 0x97, 0xF0,
+0x19, 0x6E, 0xA9, 0xA2, 0xD3, 0xF3, 0x60, 0x9F,
+0xC0, 0x36, 0xC0, 0x36, 0xA0, 0xF0, 0x19, 0x4E,
+0x40, 0xEB, 0x88, 0xA2, 0x04, 0x92, 0x05, 0x97,
+0x97, 0xF0, 0x19, 0x6E, 0xA9, 0xA2, 0xD3, 0xF3,
+0x60, 0x9F, 0xC0, 0x36, 0xC0, 0x36, 0x3E, 0xF6,
+0x19, 0x4E, 0x40, 0xEB, 0x88, 0xA2, 0x04, 0x92,
+0x05, 0x97, 0x97, 0xF0, 0x19, 0x6E, 0xA9, 0xA2,
+0xD3, 0xF3, 0xE0, 0x9F, 0xC0, 0x36, 0xC0, 0x36,
+0x5F, 0xF6, 0x1D, 0x4E, 0x40, 0xEF, 0x88, 0xA2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x4C, 0x9A, 0x40, 0xEA, 0x01, 0x6C,
+0x0A, 0x94, 0xD1, 0x18, 0xE3, 0x3A, 0xB1, 0x67,
+0x04, 0xD2, 0x12, 0x6E, 0x32, 0x22, 0x50, 0xA0,
+0x02, 0x6C, 0x31, 0xC0, 0x8D, 0xEA, 0x08, 0x6C,
+0x50, 0xC0, 0x8C, 0xEA, 0x03, 0x22, 0xB1, 0x18,
+0x90, 0xD1, 0x10, 0x6C, 0x89, 0xA0, 0x48, 0xA0,
+0x18, 0x6E, 0x80, 0x34, 0x4D, 0xEC, 0x4A, 0xA0,
+0x01, 0x6D, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x8B, 0xA0, 0x00, 0xF6, 0x80, 0x34, 0xD1, 0x18,
+0xC0, 0x3B, 0x4D, 0xEC, 0x0C, 0x92, 0x3C, 0x22,
+0x89, 0xA0, 0x48, 0xA0, 0x04, 0x95, 0x80, 0x34,
+0x4D, 0xEC, 0x4A, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x8B, 0xA0, 0x00, 0xF6, 0x80, 0x34,
+0xD1, 0x18, 0x9D, 0x3D, 0x4D, 0xEC, 0x10, 0x2A,
+0xD1, 0x18, 0xDE, 0x3D, 0x04, 0x94, 0x0C, 0x2A,
+0x17, 0x6E, 0x0A, 0x94, 0xD1, 0x18, 0x55, 0x3B,
+0xB1, 0x67, 0x00, 0x6F, 0x47, 0x67, 0x08, 0x91,
+0x09, 0x97, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0x04, 0x92, 0x01, 0x6C, 0x4D, 0xA2, 0x8C, 0xEA,
+0x12, 0x2A, 0x04, 0x94, 0x00, 0x6D, 0xD1, 0x18,
+0xC0, 0x3B, 0x18, 0x6E, 0x00, 0x6D, 0xD1, 0x18,
+0x8F, 0x3C, 0x90, 0x67, 0x08, 0x2A, 0x0A, 0x94,
+0x02, 0x67, 0x18, 0x6E, 0xD1, 0x18, 0x55, 0x3B,
+0xB1, 0x67, 0xF0, 0x67, 0xE3, 0x17, 0x04, 0x94,
+0x18, 0x6E, 0xD1, 0x18, 0xC0, 0x3B, 0x02, 0x6D,
+0x50, 0xA0, 0x02, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA,
+0x50, 0xC0, 0x31, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xC6, 0xF4, 0x08, 0x9B,
+0x94, 0x32, 0x03, 0x6E, 0x41, 0xE0, 0xD1, 0x18,
+0x55, 0x3B, 0xB1, 0xA0, 0x04, 0x22, 0x40, 0x98,
+0x4A, 0xE8, 0x01, 0x6A, 0x44, 0x61, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0x4E, 0xA1, 0x04, 0x6B, 0x6C, 0xEA, 0xFF, 0x6B,
+0x6C, 0xEA, 0x0F, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0xB8, 0xA1, 0x9B, 0xA1, 0x40, 0x32,
+0x0F, 0x6E, 0xB2, 0xF5, 0x5C, 0x9A, 0xCC, 0xED,
+0xCC, 0xEC, 0x6C, 0xED, 0x00, 0x6E, 0x40, 0xEA,
+0x6C, 0xEC, 0x91, 0x67, 0x19, 0x6E, 0xD1, 0x18,
+0xC0, 0x3B, 0x02, 0x6D, 0x7D, 0xA1, 0x9C, 0xA1,
+0x5E, 0xA1, 0x60, 0x33, 0x8D, 0xEB, 0x9F, 0xA1,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEB, 0x00, 0xF6,
+0x80, 0x34, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x6D, 0xEC, 0x40, 0x32, 0x05, 0x24, 0x72, 0xF4,
+0x70, 0x9A, 0x40, 0xEB, 0x04, 0xD2, 0x04, 0x92,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x80, 0x98, 0x13, 0xF6, 0x7C, 0x9B, 0x40, 0xEB,
+0x04, 0xD2, 0x04, 0x92, 0x91, 0x67, 0x72, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x20, 0x98,
+0x50, 0x67, 0x2E, 0xEA, 0xBD, 0x2A, 0x70, 0xA0,
+0x10, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB, 0x48, 0xC0,
+0x49, 0xC0, 0x4A, 0xC0, 0x4B, 0xC0, 0x4C, 0xC0,
+0x4D, 0xC0, 0x4E, 0xC0, 0x4F, 0xC0, 0x51, 0xC0,
+0x52, 0xC0, 0x53, 0xC0, 0x70, 0xC0, 0x01, 0x6A,
+0xA6, 0x17, 0x00, 0x65, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0x50, 0xA4, 0x04, 0x6B,
+0xF7, 0xF0, 0x01, 0x69, 0x6C, 0xEA, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x20, 0x31, 0x60, 0x33,
+0x05, 0x67, 0x14, 0xD6, 0x20, 0x31, 0x08, 0xD3,
+0xC0, 0xF0, 0x1A, 0x2A, 0xBD, 0xA4, 0x5C, 0xA4,
+0xDE, 0xA4, 0xA0, 0x35, 0x4D, 0xED, 0x5F, 0xA4,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x0A, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0xC0, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0xA9, 0xF6, 0x80, 0x9A, 0xA9, 0xF6, 0xA4, 0x9A,
+0xF3, 0xF0, 0x70, 0x99, 0x05, 0xD6, 0x01, 0x6E,
+0x04, 0xD6, 0x00, 0xF3, 0x19, 0x6F, 0x40, 0xEB,
+0x04, 0x6E, 0x0A, 0x92, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x97, 0xF0, 0x19, 0x6E,
+0xD3, 0xF3, 0xE0, 0x9C, 0xC0, 0x36, 0xA9, 0xA2,
+0xC0, 0x36, 0x60, 0xF5, 0x0D, 0x4E, 0x40, 0xEF,
+0x88, 0xA2, 0x60, 0xA0, 0x04, 0x6E, 0x74, 0x32,
+0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xC6, 0xF4, 0x88, 0x9A, 0x74, 0x35,
+0x0A, 0xD2, 0xB1, 0xE4, 0xB0, 0xA4, 0xCC, 0xED,
+0x28, 0x25, 0x01, 0x6A, 0x09, 0xD2, 0xF3, 0xF0,
+0x50, 0x99, 0x01, 0x6E, 0x04, 0xD6, 0x1A, 0x65,
+0x08, 0x92, 0x05, 0xD3, 0x00, 0xF3, 0x1C, 0x6F,
+0xA9, 0xF6, 0x80, 0x9A, 0xA9, 0xF6, 0xA4, 0x9A,
+0x58, 0x67, 0x40, 0xEA, 0x04, 0x6E, 0x09, 0x92,
+0x6C, 0x22, 0x0A, 0x92, 0x80, 0xA0, 0xC6, 0xF4,
+0x48, 0x9A, 0x94, 0x34, 0x89, 0xE2, 0x89, 0xA2,
+0xAA, 0xA2, 0xC8, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x4B, 0xA2, 0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x48, 0xA2,
+0x5A, 0x10, 0x0D, 0xD2, 0xBD, 0xA4, 0xDE, 0xA4,
+0xFC, 0xA4, 0x9F, 0xA4, 0xA0, 0x35, 0xC0, 0x36,
+0xAD, 0xEF, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0xCD, 0xEC, 0x90, 0xA4, 0x01, 0x74,
+0x46, 0x61, 0xF3, 0xF0, 0x50, 0x99, 0x05, 0xD3,
+0x00, 0xF3, 0x1A, 0x6F, 0x1A, 0x65, 0x08, 0x92,
+0x04, 0x6E, 0x0C, 0xD3, 0xA9, 0xF6, 0x80, 0x9A,
+0xA9, 0xF6, 0xA4, 0x9A, 0x58, 0x67, 0x0A, 0xD4,
+0x0B, 0xD5, 0x01, 0x6C, 0x04, 0xD4, 0x0B, 0x95,
+0x40, 0xEA, 0x0A, 0x94, 0x0D, 0x92, 0x09, 0x93,
+0xC6, 0xF4, 0x48, 0x9A, 0x69, 0xE2, 0x9D, 0xA2,
+0xBE, 0xA2, 0xDC, 0xA2, 0x5F, 0xA2, 0x80, 0x34,
+0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x51, 0xA2,
+0x0C, 0x93, 0x40, 0xC0, 0xD1, 0x18, 0xA7, 0x40,
+0x83, 0x67, 0x82, 0x67, 0x08, 0x2A, 0x04, 0xD4,
+0xF3, 0xF0, 0x50, 0x99, 0x0A, 0x94, 0x0B, 0x95,
+0x7E, 0x6F, 0x40, 0xEA, 0x01, 0x6E, 0x08, 0x93,
+0xC0, 0xA0, 0xF3, 0xF0, 0x50, 0x99, 0xA9, 0xF6,
+0x80, 0x9B, 0xA9, 0xF6, 0xA4, 0x9B, 0x05, 0xD6,
+0x01, 0x6E, 0x04, 0xD6, 0x00, 0xF3, 0x1B, 0x6F,
+0x40, 0xEA, 0x04, 0x6E, 0x72, 0x17, 0x00, 0x6A,
+0x81, 0x17, 0x01, 0x6A, 0x4B, 0xEA, 0x14, 0x93,
+0x00, 0xF3, 0x1D, 0x6F, 0x04, 0x6E, 0x40, 0xC3,
+0x08, 0x93, 0xF3, 0xF0, 0x50, 0x99, 0xA9, 0xF6,
+0x80, 0x9B, 0xA9, 0xF6, 0xA4, 0x9B, 0x14, 0x93,
+0x60, 0xA3, 0x06, 0xD3, 0x60, 0xA0, 0x05, 0xD3,
+0x02, 0x6B, 0x40, 0xEA, 0x04, 0xD3, 0x11, 0x97,
+0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63,
+0xA9, 0xA4, 0xCA, 0xA4, 0x48, 0xA4, 0xA0, 0x35,
+0xC0, 0x36, 0x4D, 0xED, 0xC0, 0x36, 0xCD, 0xED,
+0x4B, 0xA4, 0x7E, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x06, 0xD4, 0x06, 0x95, 0x06, 0x92, 0xFF, 0x6C,
+0x70, 0xA2, 0x04, 0x6A, 0x4C, 0xEB, 0x8C, 0xEB,
+0x5E, 0x2B, 0x06, 0x92, 0x02, 0x6B, 0x50, 0xA2,
+0x6C, 0xEA, 0x8C, 0xEA, 0x58, 0x2A, 0x9D, 0xA5,
+0x5C, 0xA5, 0x7E, 0xA5, 0x06, 0x96, 0x80, 0x34,
+0x4D, 0xEC, 0x60, 0x33, 0x5F, 0xA5, 0x60, 0x33,
+0xB0, 0xA6, 0x6D, 0xEC, 0xF7, 0xF0, 0x01, 0x6B,
+0x00, 0xF6, 0x40, 0x32, 0x60, 0x33, 0x04, 0x6F,
+0x8D, 0xEA, 0x60, 0x33, 0xED, 0xED, 0xC6, 0xF4,
+0x88, 0x9B, 0x71, 0xA2, 0xB0, 0xC6, 0x50, 0xA2,
+0x3B, 0x22, 0xBD, 0x67, 0x01, 0x72, 0x7D, 0xC5,
+0x37, 0x61, 0x74, 0x33, 0x07, 0x06, 0x00, 0xF0,
+0x1D, 0x05, 0xD1, 0x18, 0xDF, 0x40, 0x71, 0xE4,
+0x5D, 0x67, 0xBC, 0xA2, 0x9D, 0xA2, 0xD1, 0x18,
+0x36, 0x40, 0x00, 0x6E, 0x12, 0x2A, 0xF7, 0xF0,
+0x01, 0x6B, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x60, 0x33, 0x40, 0x32, 0x40, 0x32, 0x60, 0x33,
+0xF3, 0xF0, 0x70, 0x9B, 0xA9, 0xF6, 0x80, 0x9A,
+0xA9, 0xF6, 0xA4, 0x9A, 0x7D, 0x6F, 0x40, 0xEB,
+0x01, 0x6E, 0x5D, 0x67, 0xD1, 0x18, 0xA7, 0x40,
+0x9D, 0xA2, 0x12, 0x2A, 0xF7, 0xF0, 0x01, 0x6B,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x40, 0x32, 0x40, 0x32, 0x60, 0x33, 0xF3, 0xF0,
+0x70, 0x9B, 0xA9, 0xF6, 0x80, 0x9A, 0xA9, 0xF6,
+0xA4, 0x9A, 0x7E, 0x6F, 0x40, 0xEB, 0x01, 0x6E,
+0xD1, 0x18, 0x20, 0x3E, 0x06, 0x94, 0x09, 0x97,
+0x00, 0xEF, 0x05, 0x63, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC6, 0xF4, 0x48, 0x9A,
+0xB4, 0x35, 0x64, 0x67, 0xB5, 0xE2, 0x89, 0xA5,
+0x06, 0x67, 0xC8, 0xA5, 0x4A, 0xA5, 0x80, 0x34,
+0x8D, 0xEE, 0x8B, 0xA5, 0x40, 0x32, 0x40, 0x32,
+0xCD, 0xEA, 0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC,
+0x07, 0xD4, 0x9D, 0xA3, 0x5C, 0xA3, 0xBE, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x5F, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xDA, 0xA3, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x06, 0xD7, 0x00, 0x69,
+0x54, 0x26, 0x80, 0x9A, 0xFC, 0x4E, 0x05, 0xD4,
+0xFF, 0x6C, 0x8C, 0xEE, 0x00, 0x6C, 0x04, 0xD4,
+0x06, 0x20, 0x06, 0x94, 0xFC, 0x48, 0x80, 0x9C,
+0x04, 0xD4, 0xFF, 0x6C, 0x8C, 0xE8, 0x04, 0x94,
+0x05, 0x95, 0xAA, 0xEC, 0x1D, 0x60, 0x00, 0x6F,
+0x04, 0x95, 0x01, 0x6C, 0x84, 0xEF, 0x8C, 0xED,
+0x1D, 0x65, 0x05, 0x95, 0xAC, 0xEC, 0xB8, 0x67,
+0xAE, 0xEC, 0x0F, 0x24, 0x22, 0x61, 0xF1, 0xE1,
+0x09, 0xD2, 0xFF, 0x6A, 0x4C, 0xEC, 0x02, 0x6D,
+0x0B, 0xD3, 0x0A, 0xD6, 0xD1, 0x18, 0xEE, 0x3B,
+0x08, 0xD7, 0x08, 0x97, 0x09, 0x92, 0x0A, 0x96,
+0x0B, 0x93, 0x01, 0x4F, 0x20, 0x77, 0xE4, 0x61,
+0xFF, 0x6C, 0x20, 0x49, 0x8C, 0xE9, 0x06, 0x94,
+0x80, 0x71, 0x04, 0x4A, 0x04, 0x4C, 0x06, 0xD4,
+0xC7, 0x61, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0x00, 0x6C, 0x05, 0xD4,
+0xC8, 0x17, 0x07, 0x95, 0x88, 0xA3, 0xA8, 0xA5,
+0xAE, 0xEC, 0xE7, 0x24, 0xF1, 0xE1, 0x09, 0xD2,
+0xFF, 0x6A, 0x4C, 0xEC, 0x01, 0x6D, 0x0B, 0xD3,
+0x0A, 0xD6, 0xD1, 0x18, 0xEE, 0x3B, 0x08, 0xD7,
+0x0B, 0x93, 0x0A, 0x96, 0x09, 0x92, 0x08, 0x97,
+0xD8, 0x17, 0xE8, 0x28, 0xD0, 0x67, 0xD8, 0x17,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x15, 0xF0, 0x08, 0x48, 0x45, 0xA0, 0x66, 0xA0,
+0xC4, 0xA0, 0x40, 0x32, 0x60, 0x33, 0xCD, 0xEA,
+0x60, 0x33, 0xC7, 0xA0, 0x6D, 0xEA, 0xF7, 0xF0,
+0x01, 0x69, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x20, 0x31, 0x60, 0x33, 0x20, 0x31, 0x00, 0xF6,
+0xC0, 0x36, 0x4D, 0xEE, 0xA9, 0xF3, 0xA0, 0x99,
+0x72, 0xF4, 0x54, 0x9B, 0x8C, 0x6C, 0x09, 0xD6,
+0x40, 0xEA, 0x08, 0xD3, 0x09, 0x96, 0x42, 0x34,
+0x08, 0x93, 0x20, 0xF0, 0x54, 0xC6, 0x20, 0xF0,
+0x95, 0xC6, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x20, 0xF0, 0x96, 0xC6, 0x20, 0xF0, 0x57, 0xC6,
+0xA5, 0xA0, 0x44, 0xA0, 0x86, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x47, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xF0, 0xB5, 0xA2, 0x20, 0xF0, 0x94, 0xA2,
+0x20, 0xF0, 0xD6, 0xA2, 0xA0, 0x35, 0x8D, 0xED,
+0x20, 0xF0, 0x97, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xF7, 0xF0, 0x01, 0x68, 0xCD, 0xED, 0x00, 0xF6,
+0x80, 0x34, 0x00, 0x30, 0xAD, 0xEC, 0x00, 0x30,
+0xC6, 0xF4, 0x8C, 0xD8, 0x0A, 0xD3, 0x0C, 0xD1,
+0x1F, 0x2C, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xAA, 0xF2, 0x0C, 0x4A, 0x05, 0xD2, 0x20, 0xF6,
+0x04, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x04, 0x04, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A,
+0x10, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x91, 0x10,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x93, 0xF6, 0x40, 0x99, 0x8C, 0x6E, 0x40, 0xEA,
+0x00, 0x6D, 0xC6, 0xF4, 0x4C, 0x98, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0xC6, 0xF4,
+0x48, 0xDF, 0x00, 0x6A, 0x08, 0xD2, 0x0B, 0xD1,
+0x08, 0x92, 0x0F, 0x6D, 0x0D, 0xD7, 0x54, 0x31,
+0xC6, 0xF4, 0x48, 0x9F, 0x2D, 0xE2, 0x08, 0x92,
+0x09, 0xD3, 0x50, 0x34, 0x50, 0xA3, 0xAC, 0xEA,
+0x8D, 0xEA, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF3, 0x54, 0x9A,
+0x40, 0xEA, 0x01, 0x6C, 0x09, 0x93, 0x0D, 0x97,
+0x42, 0x34, 0x58, 0xC3, 0x00, 0xF6, 0x42, 0x32,
+0x5B, 0xC3, 0xC6, 0xF4, 0x48, 0x9F, 0x99, 0xC3,
+0x82, 0x34, 0x9A, 0xC3, 0x0C, 0x93, 0x39, 0xE2,
+0x0A, 0x92, 0xA9, 0xF3, 0x60, 0x9B, 0x38, 0x6C,
+0x72, 0xF4, 0x54, 0x9A, 0xA3, 0x67, 0x0E, 0xD7,
+0x0D, 0xD6, 0x40, 0xEA, 0x09, 0xD3, 0x0D, 0x96,
+0x0E, 0x97, 0x42, 0x34, 0x5C, 0xC6, 0x00, 0xF6,
+0x42, 0x32, 0x5F, 0xC6, 0xC6, 0xF4, 0x48, 0x9F,
+0x9D, 0xC6, 0x82, 0x34, 0x9E, 0xC6, 0x29, 0xE2,
+0xBD, 0xA2, 0x9C, 0xA2, 0xDE, 0xA2, 0xA0, 0x35,
+0x8D, 0xED, 0x9F, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC,
+0xFF, 0x6B, 0x4E, 0x2C, 0x09, 0xD3, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xAA, 0xF2, 0x0C, 0x4A,
+0x05, 0xD2, 0x20, 0xF6, 0x17, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x04, 0x04, 0x40, 0xEA,
+0x0A, 0xD7, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x10, 0x6D, 0x00, 0x6C,
+0x40, 0xEA, 0x20, 0x31, 0x72, 0xF4, 0x50, 0x99,
+0xC6, 0xF4, 0x8C, 0x98, 0x40, 0xEA, 0x00, 0x65,
+0x08, 0x92, 0x09, 0x93, 0x0A, 0x97, 0x6C, 0xEA,
+0x54, 0x30, 0xE0, 0x48, 0x47, 0x40, 0x19, 0x4A,
+0x05, 0x2A, 0x13, 0x97, 0x12, 0x91, 0x11, 0x90,
+0x00, 0xEF, 0x0A, 0x63, 0xC6, 0xF4, 0x48, 0x9F,
+0x08, 0xD7, 0x72, 0xF4, 0xD0, 0x99, 0x09, 0xE2,
+0x9D, 0xA2, 0x7C, 0xA2, 0xE0, 0x48, 0x80, 0x34,
+0x6D, 0xEC, 0x7E, 0xA2, 0x60, 0x33, 0x60, 0x33,
+0x8D, 0xEB, 0x9F, 0xA2, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xEE, 0x6D, 0xEC, 0x08, 0x97, 0xE2, 0x17,
+0x0B, 0x92, 0x38, 0x6E, 0x00, 0x6D, 0x93, 0xF6,
+0x40, 0x9A, 0x0E, 0xD7, 0x40, 0xEA, 0x0D, 0xD3,
+0x0E, 0x97, 0x0D, 0x93, 0xC6, 0xF4, 0x48, 0x9F,
+0x25, 0xE2, 0x22, 0x32, 0xA2, 0x67, 0x42, 0x32,
+0x6C, 0xEA, 0x42, 0xC1, 0x46, 0xC1, 0x08, 0x92,
+0xD1, 0x67, 0x6C, 0xEE, 0x01, 0x4A, 0x6C, 0xED,
+0x00, 0xF6, 0x22, 0x34, 0x04, 0x72, 0xC0, 0xC1,
+0xA1, 0xC1, 0x83, 0xC1, 0xC4, 0xC1, 0xA5, 0xC1,
+0x87, 0xC1, 0x08, 0xD2, 0x5F, 0xF7, 0x04, 0x61,
+0x0A, 0x92, 0x09, 0x95, 0x08, 0x6C, 0x72, 0xF4,
+0x54, 0x9A, 0xC6, 0xF4, 0x2C, 0x98, 0x40, 0xEA,
+0x08, 0xD3, 0x08, 0x93, 0x82, 0x67, 0x6C, 0xEC,
+0x80, 0xF0, 0x80, 0xC1, 0x42, 0x34, 0xA4, 0x67,
+0x6C, 0xED, 0x80, 0xF0, 0xA1, 0xC1, 0x82, 0x34,
+0xC6, 0xF4, 0xAC, 0x98, 0x8C, 0xEB, 0x00, 0xF6,
+0x42, 0x32, 0x80, 0xF0, 0x62, 0xC1, 0x80, 0xF0,
+0x43, 0xC1, 0x80, 0xF0, 0x61, 0xA5, 0x80, 0xF0,
+0x80, 0xA5, 0x80, 0xF0, 0x42, 0xA5, 0x60, 0x33,
+0x8D, 0xEB, 0x80, 0xF0, 0x83, 0xA5, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xEB, 0x00, 0xF6, 0x80, 0x34,
+0x6D, 0xEC, 0x0D, 0x2C, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xAA, 0xF2, 0x0C, 0x4A, 0x05, 0xD2,
+0x40, 0xF6, 0x0A, 0x6A, 0xE2, 0x16, 0x0B, 0x92,
+0x08, 0x6E, 0x00, 0x6D, 0x93, 0xF6, 0x40, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x7E, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0x04, 0xA4, 0x03, 0x6A, 0x20, 0xA4, 0x12, 0x30,
+0x4C, 0xE8, 0x0A, 0xD4, 0x00, 0x6D, 0xD1, 0x18,
+0x06, 0x3B, 0x90, 0x67, 0x0A, 0x97, 0x06, 0x2A,
+0x04, 0x6A, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x0B, 0xD7, 0x0A, 0xD2, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xA9, 0xF3,
+0xA4, 0x9B, 0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x30, 0x6C, 0x62, 0x67, 0x0B, 0x97, 0x2C, 0x2A,
+0x7D, 0x67, 0x06, 0x6A, 0x58, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xAA, 0xF2,
+0x0C, 0x4A, 0x07, 0xD2, 0x60, 0xF6, 0x13, 0x6A,
+0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x06, 0x04, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x10, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x0A, 0x94, 0x16, 0x6E, 0xB1, 0x67,
+0xD1, 0x18, 0x55, 0x3B, 0x90, 0x67, 0xB4, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0x87, 0x43, 0xA7, 0x67,
+0x01, 0x4C, 0x10, 0x6E, 0x40, 0xEA, 0x0B, 0xD3,
+0x0B, 0x93, 0x04, 0x6A, 0xFF, 0x6D, 0x8E, 0xA3,
+0x8C, 0xEA, 0xAC, 0xEA, 0x47, 0x22, 0x5B, 0xA3,
+0x10, 0x6E, 0xCB, 0xEE, 0xCC, 0xEA, 0xB8, 0xA3,
+0x5B, 0xC3, 0x01, 0x6A, 0x4B, 0xEA, 0x48, 0xC3,
+0x4D, 0xA3, 0xCC, 0xED, 0xB8, 0xC3, 0x01, 0x6D,
+0xAD, 0xEA, 0x4D, 0xC3, 0x10, 0x6A, 0x8D, 0xEA,
+0x4E, 0xC3, 0x09, 0x6A, 0x98, 0xA3, 0x10, 0x6D,
+0xAB, 0xED, 0xAD, 0xEC, 0xF7, 0xF0, 0x01, 0x6D,
+0x20, 0xF0, 0x40, 0xC3, 0xA0, 0x35, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0xA0, 0x35, 0x98, 0xC3,
+0xC6, 0xF4, 0xA8, 0x9D, 0x00, 0x6C, 0x40, 0x32,
+0x99, 0xC3, 0x9A, 0xC3, 0x33, 0xF6, 0x44, 0x9A,
+0x00, 0x6C, 0x9C, 0xC3, 0x9D, 0xC3, 0x9E, 0xC3,
+0x9F, 0xC3, 0x20, 0xF0, 0x81, 0xC3, 0x14, 0x34,
+0x95, 0xE5, 0x83, 0x67, 0x40, 0xEA, 0x23, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0A, 0x94, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xA8, 0xA1, 0x00, 0x6E, 0xD1, 0x18,
+0x55, 0x3B, 0x90, 0x67, 0x5F, 0xF7, 0x1C, 0x22,
+0x00, 0x6A, 0x5B, 0x17, 0x91, 0x67, 0xD1, 0x18,
+0xDE, 0x60, 0x0B, 0xD3, 0xFF, 0x72, 0x0B, 0x93,
+0x1E, 0x61, 0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xA9, 0xF6,
+0x80, 0x9B, 0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6,
+0xA4, 0x9B, 0x01, 0x6E, 0x66, 0x6F, 0x40, 0xEA,
+0x05, 0xD1, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x0A, 0x94, 0x11, 0x6E, 0x7C, 0x17, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0x15, 0xF0,
+0x08, 0x4D, 0xC5, 0xA5, 0x84, 0xA5, 0xE6, 0xA5,
+0xC0, 0x36, 0x8D, 0xEE, 0x87, 0xA5, 0xE0, 0x37,
+0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6, 0x80, 0x34,
+0xCD, 0xEC, 0x20, 0xF0, 0xA1, 0xA4, 0x20, 0xF0,
+0xC2, 0xA4, 0x20, 0xF0, 0xE0, 0xA4, 0x20, 0xF0,
+0x83, 0xA4, 0xA0, 0x35, 0xC0, 0x36, 0xAD, 0xEF,
+0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6, 0x80, 0x34,
+0xCD, 0xEC, 0x48, 0x37, 0x9D, 0xE7, 0xA5, 0xA7,
+0x84, 0xA7, 0xC6, 0xA7, 0xA0, 0x35, 0x8D, 0xED,
+0x87, 0xA7, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC, 0x22, 0xA4,
+0xB8, 0xA3, 0x10, 0x6E, 0x0F, 0x6F, 0xCB, 0xEE,
+0xEC, 0xE9, 0xCC, 0xED, 0x2D, 0xED, 0xB8, 0xC3,
+0x81, 0xA4, 0x8C, 0xEF, 0x9B, 0xA3, 0xCC, 0xEC,
+0xED, 0xEC, 0x9B, 0xC3, 0x67, 0x17, 0x00, 0x65,
+0xF1, 0x63, 0x1D, 0x62, 0x1C, 0xD1, 0x1B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0x04, 0x67, 0x14, 0x6E, 0x68, 0xF4, 0x18, 0x4D,
+0xD1, 0x18, 0xE7, 0xF4, 0x06, 0x04, 0xF0, 0x67,
+0x00, 0xA0, 0x00, 0x6E, 0x5D, 0x67, 0xDE, 0xCA,
+0x03, 0x6A, 0x4C, 0xE8, 0x10, 0xD0, 0x10, 0x02,
+0x40, 0xA2, 0x01, 0x6D, 0x0B, 0xD6, 0x10, 0xD2,
+0x10, 0x94, 0x43, 0xA7, 0x0C, 0xD6, 0x0D, 0xD6,
+0x0E, 0xD6, 0x12, 0xD6, 0x07, 0x67, 0xD1, 0x18,
+0x06, 0x3B, 0x11, 0xD2, 0x04, 0x69, 0x12, 0x96,
+0x23, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x13, 0xD6, 0x12, 0xD2, 0x10, 0x92, 0x13, 0x96,
+0x54, 0x33, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xC6, 0xF4, 0x48, 0x9A, 0x6D, 0xE2,
+0x40, 0x9B, 0x4A, 0xEB, 0x13, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x12, 0x94, 0x10, 0x6E,
+0x11, 0x95, 0xD1, 0x18, 0x55, 0x3B, 0x10, 0x94,
+0x51, 0x67, 0x1D, 0x97, 0x1C, 0x91, 0x1B, 0x90,
+0x00, 0xEF, 0x0F, 0x63, 0x15, 0xD6, 0x14, 0xD0,
+0x9D, 0xA3, 0x5C, 0xA3, 0x1F, 0xA3, 0x80, 0x34,
+0x8D, 0xEA, 0x1A, 0x65, 0x5E, 0xA3, 0x98, 0x67,
+0x11, 0x95, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x10, 0x94, 0x00, 0xF6, 0x00, 0x30, 0x13, 0xD3,
+0xD1, 0x18, 0xE3, 0x3A, 0x4D, 0xE8, 0xA2, 0x67,
+0x13, 0x93, 0x14, 0x97, 0x15, 0x96, 0x0A, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x12, 0x94,
+0x12, 0x6E, 0xD2, 0x17, 0x9B, 0xA5, 0x0F, 0x6A,
+0x3A, 0x65, 0x59, 0x67, 0x4C, 0xEC, 0x13, 0xD4,
+0x13, 0x02, 0x40, 0xA2, 0x16, 0xD3, 0x79, 0x67,
+0x13, 0xD2, 0x58, 0xA5, 0x18, 0xD6, 0x17, 0xD7,
+0x6C, 0xEA, 0x14, 0xD2, 0x14, 0x02, 0x13, 0x93,
+0x40, 0xA2, 0x15, 0xD5, 0x68, 0x34, 0x14, 0xD2,
+0x71, 0xE4, 0x14, 0x93, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x71, 0xE4, 0x88, 0x34, 0x06, 0x03,
+0x91, 0xE3, 0x40, 0x32, 0xD2, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x80, 0x9C, 0x2C, 0xEA, 0x15, 0x95,
+0x16, 0x93, 0x17, 0x97, 0x18, 0x96, 0x02, 0x2A,
+0x19, 0x6E, 0xA6, 0x17, 0x89, 0xA7, 0x48, 0xA7,
+0x24, 0xA7, 0x80, 0x34, 0x8D, 0xEA, 0xFA, 0x65,
+0x4A, 0xA7, 0x9F, 0x67, 0x40, 0x32, 0x40, 0x32,
+0x8D, 0xEA, 0x8B, 0xA7, 0x00, 0xF6, 0x80, 0x34,
+0x4D, 0xEC, 0xFC, 0x65, 0x45, 0xA7, 0x86, 0xA7,
+0x40, 0x32, 0x80, 0x34, 0x4D, 0xE9, 0x80, 0x34,
+0x50, 0xA3, 0x8D, 0xE9, 0x87, 0xA7, 0xC8, 0xC3,
+0xC9, 0xC3, 0xCA, 0xC3, 0xCB, 0xC3, 0xAC, 0xC3,
+0xA2, 0x36, 0x00, 0xF6, 0xA2, 0x35, 0xAF, 0xC3,
+0xA0, 0xA7, 0x3A, 0x65, 0x01, 0x6A, 0x1A, 0x65,
+0xCD, 0xC3, 0xC2, 0x36, 0x00, 0xF6, 0x80, 0x34,
+0xCE, 0xC3, 0xAA, 0x35, 0xD8, 0x67, 0xCC, 0xED,
+0x2D, 0xEC, 0x01, 0x6A, 0x39, 0x67, 0x0F, 0x6E,
+0xCB, 0xEE, 0x2D, 0xEA, 0xAC, 0x35, 0xCC, 0xEA,
+0xAD, 0xEA, 0xA2, 0xA7, 0x50, 0xC3, 0x58, 0x67,
+0xB6, 0x35, 0x4C, 0xED, 0x57, 0xA3, 0x0D, 0x4E,
+0xCC, 0xEA, 0xAD, 0xEA, 0xB8, 0x67, 0x57, 0xC3,
+0xAC, 0xEA, 0x15, 0xD2, 0x15, 0x02, 0x40, 0xA2,
+0x08, 0x22, 0x42, 0xA7, 0x07, 0x6D, 0xAC, 0xEA,
+0x15, 0xD2, 0x15, 0x02, 0x40, 0xA2, 0x40, 0x22,
+0x56, 0xC3, 0x82, 0x32, 0x41, 0xC0, 0x42, 0x32,
+0x42, 0xC0, 0x00, 0xF6, 0x82, 0x32, 0x43, 0xC0,
+0x5F, 0x67, 0x44, 0xC0, 0x5F, 0x67, 0x42, 0x32,
+0x45, 0xC0, 0x42, 0x32, 0x46, 0xC0, 0x5F, 0x67,
+0x00, 0xF6, 0x42, 0x32, 0x47, 0xC0, 0x5D, 0x67,
+0x40, 0xF0, 0x4C, 0xA2, 0x03, 0x6E, 0xF7, 0xF0,
+0x01, 0x69, 0x48, 0xC0, 0x5D, 0x67, 0x40, 0xF0,
+0x50, 0xA2, 0x20, 0x31, 0x80, 0xC0, 0x49, 0xC0,
+0x40, 0xA7, 0x20, 0x31, 0x56, 0x32, 0xCC, 0xEA,
+0x51, 0xC0, 0xA0, 0xA7, 0xAE, 0x35, 0xCC, 0xED,
+0xDD, 0x67, 0x40, 0xF0, 0xC0, 0xA6, 0xB0, 0xC0,
+0xD2, 0xC0, 0xDD, 0x67, 0x40, 0xF0, 0xC4, 0xA6,
+0xD3, 0xC0, 0x11, 0x25, 0x10, 0x95, 0xAE, 0xEA,
+0x0E, 0x2A, 0x72, 0xF0, 0x54, 0x99, 0x40, 0xEA,
+0x12, 0x94, 0x13, 0x6E, 0x11, 0x95, 0xD1, 0x18,
+0x55, 0x3B, 0x10, 0x94, 0x04, 0x69, 0x1C, 0x17,
+0x01, 0x6A, 0x4B, 0xEA, 0xBD, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x05, 0xD3, 0x97, 0xF0,
+0x19, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0xC4, 0x67, 0xD3, 0xF3, 0x44, 0x9A, 0x14, 0x95,
+0x13, 0x94, 0x60, 0xF5, 0x0D, 0x4B, 0xFF, 0x67,
+0x40, 0xEA, 0x04, 0xD3, 0x0F, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0x10, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0x72, 0xF0, 0x54, 0x99, 0x40, 0xEA, 0x12, 0x94,
+0x15, 0x6E, 0xD4, 0x17, 0x11, 0x95, 0x10, 0x94,
+0xD1, 0x18, 0x55, 0x3B, 0x01, 0x6E, 0x05, 0x2A,
+0x72, 0xF0, 0x54, 0x99, 0x40, 0xEA, 0x12, 0x94,
+0xCD, 0x17, 0x7D, 0x67, 0x40, 0xF0, 0x6C, 0xA3,
+0x5D, 0x67, 0x04, 0x6C, 0x20, 0xF0, 0x6C, 0xC2,
+0x20, 0xF0, 0x6D, 0xA2, 0x8D, 0xEB, 0x20, 0xF0,
+0x6D, 0xC2, 0x20, 0xF0, 0x4E, 0xA2, 0x7D, 0x67,
+0x8D, 0xEA, 0x20, 0xF0, 0x4E, 0xC3, 0x3C, 0x6A,
+0x5D, 0xCB, 0x5D, 0x67, 0x20, 0xF0, 0x5C, 0xA2,
+0x20, 0x6B, 0x6D, 0xEA, 0x7D, 0x67, 0x20, 0xF0,
+0x5C, 0xC3, 0xB1, 0x18, 0xC1, 0xE9, 0x0B, 0x04,
+0x72, 0xF0, 0x54, 0x99, 0x12, 0x94, 0x40, 0xEA,
+0x00, 0x69, 0xC6, 0x16, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0x61, 0xA4, 0x01, 0xA4,
+0xA0, 0xA4, 0x03, 0x6A, 0x4C, 0xEB, 0xFF, 0x6A,
+0x4C, 0xEB, 0x01, 0x6C, 0x0A, 0x30, 0x07, 0xD5,
+0x8C, 0xE8, 0x02, 0x6D, 0x83, 0x67, 0x4C, 0xE8,
+0xD1, 0x18, 0x06, 0x3B, 0x06, 0xD3, 0x04, 0x69,
+0x6C, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x09, 0xD2, 0x06, 0x93, 0xA0, 0xF0,
+0x16, 0x20, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0xA0, 0x35, 0xC6, 0xF4, 0x48, 0x9D, 0x74, 0x33,
+0x6D, 0xE2, 0x9D, 0xA3, 0x5C, 0xA3, 0xDE, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x5F, 0xA3, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x48, 0xA2, 0x00, 0x6B, 0x0B, 0xD5,
+0x0A, 0xD2, 0x0B, 0x94, 0xFF, 0x6A, 0x6C, 0xEA,
+0xC6, 0xF4, 0x88, 0x9C, 0x08, 0xD2, 0x74, 0x32,
+0x49, 0xE4, 0xDD, 0xA2, 0xBE, 0xA2, 0x9C, 0xA2,
+0xC0, 0x36, 0xA0, 0x35, 0x8D, 0xEE, 0xA0, 0x35,
+0x9F, 0xA2, 0xAD, 0xEE, 0xA0, 0x9A, 0x00, 0xF6,
+0x80, 0x34, 0xCD, 0xEC, 0xAE, 0xEA, 0x08, 0xA4,
+0x78, 0x22, 0x0A, 0x92, 0x0A, 0xEA, 0x75, 0x61,
+0x49, 0xA4, 0x07, 0x95, 0x08, 0x94, 0x23, 0x67,
+0xD1, 0x18, 0xE3, 0x3A, 0x06, 0xD2, 0x2B, 0x22,
+0x07, 0x95, 0x08, 0x94, 0xD1, 0x18, 0x36, 0x40,
+0x01, 0x6E, 0x71, 0x67, 0x66, 0x2A, 0x08, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x05, 0xD3,
+0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xA9, 0xF6,
+0x80, 0x9B, 0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6,
+0xA4, 0x9B, 0x8A, 0x6F, 0x40, 0xEA, 0x01, 0x6E,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x09, 0x94, 0x40, 0xEA,
+0x04, 0x69, 0x51, 0x67, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x0C, 0xD1,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3, 0xE0, 0x99,
+0x06, 0x95, 0xC0, 0x36, 0xC0, 0x36, 0x60, 0xF5,
+0x0D, 0x4E, 0x40, 0xEF, 0x90, 0x67, 0x97, 0xF0,
+0x19, 0x6E, 0xD3, 0xF3, 0xE0, 0x99, 0x06, 0x95,
+0xC0, 0x36, 0xC0, 0x36, 0x3F, 0xF6, 0x1D, 0x4E,
+0x40, 0xEF, 0x90, 0x67, 0x97, 0xF0, 0x19, 0x6E,
+0xD3, 0xF3, 0xE0, 0x99, 0x06, 0x95, 0xC0, 0x36,
+0xC0, 0x36, 0xA0, 0xF0, 0x19, 0x4E, 0x40, 0xEF,
+0x90, 0x67, 0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3,
+0xE0, 0x99, 0x06, 0x95, 0xC0, 0x36, 0xC0, 0x36,
+0x3E, 0xF6, 0x19, 0x4E, 0x40, 0xEF, 0x90, 0x67,
+0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3, 0x40, 0x99,
+0x06, 0x95, 0xC0, 0x36, 0xC0, 0x36, 0x5F, 0xF6,
+0x1D, 0x4E, 0x40, 0xEA, 0x90, 0x67, 0x07, 0x95,
+0x08, 0x94, 0xD1, 0x18, 0x55, 0x3B, 0x12, 0x6E,
+0x0C, 0x93, 0x01, 0x4B, 0x04, 0x73, 0x7F, 0xF7,
+0x0C, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x09, 0x94,
+0x40, 0xEA, 0x00, 0x69, 0xAA, 0x17, 0x07, 0x95,
+0x01, 0x6E, 0xD1, 0x18, 0x36, 0x40, 0x83, 0x67,
+0xF0, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x09, 0x94, 0x9B, 0x17, 0xF6, 0x63, 0x13, 0x62,
+0x12, 0xD1, 0x11, 0xD0, 0x00, 0x6A, 0x7D, 0x67,
+0x54, 0xCB, 0x60, 0xA4, 0x06, 0xD2, 0x07, 0xD2,
+0x08, 0xD2, 0x09, 0xD2, 0x03, 0x6A, 0x4C, 0xEB,
+0x40, 0xA4, 0xFF, 0x6D, 0x01, 0x6C, 0x4A, 0x32,
+0xAC, 0xEB, 0x8C, 0xEA, 0xAC, 0xEA, 0x83, 0x67,
+0x03, 0x6D, 0x0C, 0xD2, 0xD1, 0x18, 0x06, 0x3B,
+0x23, 0x67, 0x04, 0x68, 0x77, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x40, 0xEA, 0x0D, 0xD1, 0x22, 0x67,
+0x0C, 0x92, 0x0D, 0x93, 0x7E, 0x22, 0xF7, 0xF0,
+0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36, 0xC6, 0xF4,
+0x48, 0x9E, 0x74, 0x33, 0x0D, 0xD6, 0x6D, 0xE2,
+0x9D, 0xA3, 0x5C, 0xA3, 0xBE, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x5F, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x48, 0xA2, 0x7D, 0x67, 0x06, 0x04, 0x0C, 0xD2,
+0x20, 0xF0, 0x70, 0xA3, 0x5D, 0x67, 0x78, 0xC2,
+0x59, 0xA2, 0x05, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x7D, 0x67, 0x59, 0xC3, 0x5D, 0x67, 0x5A, 0xA2,
+0x4D, 0xE8, 0x5D, 0x67, 0xB1, 0x18, 0xC1, 0xE9,
+0x1A, 0xC2, 0x0D, 0x96, 0x00, 0x6D, 0x06, 0x67,
+0xC6, 0xF4, 0x88, 0x98, 0xB4, 0x32, 0xFF, 0x6B,
+0x49, 0xE4, 0xFD, 0xA2, 0x9C, 0xA2, 0xDE, 0xA2,
+0xE0, 0x37, 0x8D, 0xEF, 0x9F, 0xA2, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xEF, 0x00, 0xF6, 0x80, 0x34,
+0xED, 0xEC, 0x88, 0xA4, 0x0C, 0x96, 0xAC, 0xEB,
+0xCE, 0xEC, 0x2E, 0x2C, 0x80, 0x9A, 0x8E, 0xEA,
+0x2B, 0x22, 0x83, 0x67, 0x0E, 0xD5, 0xD1, 0x18,
+0xA7, 0x40, 0x0D, 0xD3, 0x0D, 0x93, 0x0E, 0x95,
+0x23, 0x2A, 0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xA9, 0xF6, 0x80, 0x9B, 0xF3, 0xF0, 0x50, 0x9A,
+0xA9, 0xF6, 0xA4, 0x9B, 0x8D, 0x6F, 0x40, 0xEA,
+0x01, 0x6E, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x91, 0x67,
+0x40, 0xEA, 0x04, 0x68, 0x50, 0x67, 0x13, 0x97,
+0x12, 0x91, 0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63,
+0x01, 0x4D, 0x04, 0x75, 0xB9, 0x61, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x91, 0x67, 0x40, 0xEA, 0x00, 0x68,
+0xED, 0x17, 0xD1, 0x18, 0xA7, 0x40, 0x83, 0x67,
+0xF2, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x91, 0x67, 0xE0, 0x17, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x20, 0xA4, 0x03, 0x6A,
+0xFF, 0x6C, 0x4C, 0xE9, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x8C, 0xE9, 0x40, 0xEA, 0x04, 0xD4, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0xC6, 0xF4,
+0x08, 0x9D, 0x62, 0x67, 0x34, 0x32, 0x41, 0xE0,
+0x50, 0xA0, 0x04, 0x94, 0x01, 0x6D, 0xAC, 0xEA,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0x8C, 0xEA,
+0xE0, 0x37, 0x20, 0x22, 0x05, 0xD7, 0x04, 0xD3,
+0xA9, 0xA0, 0x48, 0xA0, 0x8A, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x4B, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0xA8, 0xA2, 0x01, 0x6E, 0xD1, 0x18, 0x36, 0x40,
+0x91, 0x67, 0x04, 0x93, 0x05, 0x97, 0x0A, 0x2A,
+0x72, 0xF0, 0x54, 0x9F, 0x40, 0xEA, 0x83, 0x67,
+0x04, 0x6A, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x40, 0x98, 0x4E, 0xE8,
+0x08, 0x20, 0x91, 0x67, 0x04, 0xD7, 0xD1, 0x18,
+0xA7, 0x40, 0x03, 0x67, 0x70, 0x67, 0x04, 0x97,
+0xEB, 0x22, 0x72, 0xF0, 0x54, 0x9F, 0x40, 0xEA,
+0x83, 0x67, 0x00, 0x6A, 0xEA, 0x17, 0x00, 0x65,
+0xF5, 0x63, 0x15, 0x62, 0x14, 0xD1, 0x13, 0xD0,
+0x60, 0xA4, 0x03, 0x6A, 0x00, 0x69, 0x4C, 0xEB,
+0x41, 0xA4, 0x05, 0x6D, 0x08, 0xD1, 0x0E, 0xD2,
+0x42, 0xA4, 0x83, 0x67, 0x0A, 0xD3, 0xD1, 0x18,
+0x06, 0x3B, 0x0F, 0xD2, 0x06, 0x2A, 0x04, 0x6A,
+0x15, 0x97, 0x14, 0x91, 0x13, 0x90, 0x00, 0xEF,
+0x0B, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x0B, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0x85, 0xA2, 0x64, 0xA2, 0xA6, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x67, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x20, 0xF0, 0x41, 0xA3, 0x20, 0xF0, 0x82, 0xA3,
+0x20, 0xF0, 0x00, 0xA3, 0x40, 0x32, 0x80, 0x34,
+0x0D, 0xEA, 0x80, 0x34, 0x20, 0xF0, 0x03, 0xA3,
+0x8D, 0xEA, 0x0E, 0x94, 0x00, 0xF6, 0x00, 0x30,
+0xD1, 0x18, 0xDE, 0x60, 0x4D, 0xE8, 0xFF, 0x72,
+0x1C, 0x60, 0x48, 0x32, 0x49, 0xE0, 0x85, 0xA2,
+0x66, 0xA2, 0xA4, 0xA2, 0x47, 0xA2, 0x80, 0x34,
+0x60, 0x33, 0x8D, 0xED, 0x60, 0x33, 0x0F, 0x94,
+0xAD, 0xEB, 0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA,
+0xD1, 0x18, 0xDE, 0x60, 0x0C, 0xD2, 0xFF, 0x72,
+0x2C, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x0F, 0x93,
+0x07, 0x10, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x0E, 0x93,
+0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xA9, 0xF6,
+0x80, 0x9B, 0xA9, 0xF6, 0xA4, 0x9B, 0x8F, 0x6F,
+0x40, 0xEA, 0x01, 0x6E, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0B, 0x94, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x0A, 0x94,
+0x11, 0x6E, 0xD1, 0x18, 0x2D, 0x3B, 0x05, 0x6D,
+0x8A, 0x17, 0x48, 0x32, 0x49, 0xE0, 0x65, 0xA2,
+0x06, 0xA2, 0x84, 0xA2, 0x47, 0xA2, 0x60, 0x33,
+0x00, 0x30, 0x6D, 0xEC, 0x00, 0x30, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xE8, 0x7D, 0x67, 0x4D, 0xE8,
+0x01, 0x6A, 0x5A, 0xC3, 0x04, 0x6A, 0x59, 0xC3,
+0x02, 0x6A, 0x58, 0xC3, 0x1C, 0x6A, 0x5B, 0xC3,
+0x5D, 0x67, 0x3C, 0xC2, 0x05, 0x6A, 0x0D, 0xD2,
+0x01, 0x6A, 0x08, 0x91, 0x80, 0xF0, 0x12, 0x29,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0x00, 0x6F, 0x08, 0x06,
+0x14, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x59, 0x2A,
+0x08, 0x91, 0x57, 0x21, 0x5D, 0x67, 0x20, 0xF0,
+0x58, 0xA2, 0x0A, 0x93, 0x04, 0x6C, 0x40, 0xC1,
+0x5D, 0x67, 0x20, 0xF0, 0x5C, 0xA2, 0x8B, 0xEC,
+0xF7, 0xF0, 0x01, 0x6F, 0x41, 0xC1, 0x42, 0xA1,
+0xE0, 0x37, 0xE0, 0x37, 0x8C, 0xEA, 0x6D, 0xEA,
+0x0C, 0x93, 0x42, 0xC1, 0xF3, 0xF3, 0x4C, 0x9F,
+0xA2, 0xA3, 0x81, 0xA3, 0x00, 0x6E, 0x40, 0xEA,
+0x10, 0xD7, 0x42, 0x34, 0x0C, 0x93, 0x10, 0x97,
+0x44, 0xC1, 0x85, 0xC1, 0x00, 0xF6, 0x42, 0x32,
+0x82, 0x34, 0x47, 0xC1, 0x86, 0xC1, 0xA2, 0xA3,
+0x81, 0xA3, 0xF3, 0xF3, 0x4C, 0x9F, 0x40, 0xEA,
+0x01, 0x6E, 0x42, 0x34, 0x10, 0x97, 0x48, 0xC1,
+0x89, 0xC1, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x4B, 0xC1, 0x8A, 0xC1, 0xF3, 0xF3, 0x4C, 0x9F,
+0xA2, 0xA0, 0x81, 0xA0, 0x40, 0xEA, 0x00, 0x6E,
+0x42, 0x34, 0x10, 0x97, 0x4C, 0xC1, 0x8D, 0xC1,
+0x00, 0xF6, 0x42, 0x32, 0x82, 0x34, 0x4F, 0xC1,
+0x8E, 0xC1, 0xA2, 0xA0, 0x81, 0xA0, 0xF3, 0xF3,
+0x4C, 0x9F, 0x40, 0xEA, 0x01, 0x6E, 0x42, 0x34,
+0x08, 0x95, 0x50, 0xC1, 0x91, 0xC1, 0x00, 0xF6,
+0x42, 0x32, 0x82, 0x34, 0x92, 0xC1, 0x53, 0xC1,
+0xD1, 0x18, 0xA8, 0x19, 0x06, 0x04, 0x1C, 0x22,
+0x00, 0x6A, 0x00, 0x6B, 0x08, 0xD3, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x73, 0xF6,
+0x68, 0x9B, 0x64, 0x6C, 0x40, 0xEB, 0x10, 0xD2,
+0x0D, 0x93, 0xFF, 0x69, 0x10, 0x92, 0xFF, 0x4B,
+0x2C, 0xEB, 0x0D, 0xD3, 0x86, 0x2B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x0B, 0x94, 0xEB, 0x16,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0B, 0x94, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x0A, 0x94, 0x00, 0x6E, 0xD1, 0x18,
+0x2D, 0x3B, 0x05, 0x6D, 0xDF, 0xF6, 0x1B, 0x22,
+0x00, 0x6A, 0xDA, 0x16, 0x7F, 0xF7, 0x1A, 0x22,
+0xD0, 0x17, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x00, 0xA4, 0x03, 0x6A,
+0x21, 0xA4, 0x4C, 0xE8, 0x42, 0xA4, 0x09, 0xD4,
+0x06, 0x6D, 0x90, 0x67, 0xD1, 0x18, 0x06, 0x3B,
+0x08, 0xD2, 0x09, 0x96, 0x06, 0x2A, 0x04, 0x6A,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x0A, 0xD6, 0xB1, 0x67, 0x90, 0x67, 0xD1, 0x18,
+0xE3, 0x3A, 0x09, 0xD2, 0x22, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0A, 0x96,
+0x0A, 0x29, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x09, 0x94, 0x12, 0x6E, 0x06, 0x6D, 0xD1, 0x18,
+0x2D, 0x3B, 0x90, 0x67, 0xDC, 0x17, 0x44, 0x46,
+0x0A, 0xD2, 0xBD, 0xA1, 0x5C, 0xA1, 0x9E, 0xA1,
+0xA0, 0x35, 0x4D, 0xED, 0x5F, 0xA1, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x2E, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC6, 0xF4, 0x48, 0x9A,
+0x14, 0x34, 0x89, 0xE2, 0x89, 0xA2, 0xAA, 0xA2,
+0xC8, 0xA2, 0x4B, 0xA2, 0x80, 0x34, 0xA0, 0x35,
+0x8D, 0xEE, 0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x06, 0x22, 0x0A, 0x97,
+0x08, 0x96, 0xB0, 0x67, 0xD1, 0x18, 0x91, 0x41,
+0x91, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x9D, 0xA1, 0x72, 0xF4, 0xD0, 0x9A,
+0x5C, 0xA1, 0x80, 0x34, 0x4D, 0xEC, 0x5E, 0xA1,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x9F, 0xA1,
+0x00, 0xF6, 0x80, 0x34, 0x40, 0xEE, 0x4D, 0xEC,
+0x5D, 0x67, 0x20, 0xF0, 0x40, 0xA2, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x5A, 0xC1, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x80, 0x34, 0x40, 0x32,
+0xA9, 0xF3, 0xA8, 0x9C, 0x72, 0xF4, 0x54, 0x9A,
+0x40, 0xEA, 0x08, 0x94, 0x42, 0x34, 0x9D, 0xC1,
+0x82, 0x34, 0x9E, 0xC1, 0x00, 0xF6, 0x42, 0x34,
+0x5C, 0xC1, 0x9F, 0xC1, 0x28, 0x2A, 0x7D, 0x67,
+0x06, 0x6A, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xAA, 0xF2, 0x0C, 0x4A,
+0x05, 0xD2, 0xE1, 0xF0, 0x17, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x10, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x09, 0x94, 0x16, 0x6E, 0x7B, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x08, 0x96,
+0x0A, 0x95, 0x73, 0xF6, 0x7C, 0x9B, 0x40, 0xEB,
+0x82, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x09, 0x94, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x00, 0x6E, 0x06, 0x6D,
+0xD1, 0x18, 0x2D, 0x3B, 0x90, 0x67, 0x5F, 0xF7,
+0x02, 0x22, 0x00, 0x6A, 0x41, 0x17, 0x00, 0x65,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0x00, 0xA4, 0x03, 0x6A, 0x24, 0x67, 0x4C, 0xE8,
+0x41, 0xA4, 0x07, 0x6D, 0x07, 0xD2, 0x42, 0xA4,
+0x90, 0x67, 0xD1, 0x18, 0x06, 0x3B, 0x08, 0xD2,
+0x06, 0x2A, 0x04, 0x6A, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x08, 0x95,
+0x90, 0x67, 0xD1, 0x18, 0xE3, 0x3A, 0x06, 0xD2,
+0xA2, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0A, 0x2D, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x06, 0x94, 0x12, 0x6E, 0x07, 0x6D,
+0xD1, 0x18, 0x2D, 0x3B, 0x90, 0x67, 0xDD, 0x17,
+0x07, 0x94, 0xD1, 0x18, 0xDE, 0x60, 0x08, 0xD5,
+0xFF, 0x72, 0x08, 0x95, 0x1F, 0x61, 0x07, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x05, 0xD3,
+0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xA9, 0xF6,
+0x80, 0x9B, 0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6,
+0xA4, 0x9B, 0x01, 0x6E, 0x40, 0xEA, 0x8F, 0x6F,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x06, 0x94,
+0x11, 0x6E, 0xD5, 0x17, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0x15, 0xF0, 0x08, 0x4E,
+0xE5, 0xA6, 0x64, 0xA6, 0x87, 0xA6, 0xE0, 0x37,
+0xED, 0xEB, 0xE6, 0xA6, 0x00, 0xF6, 0x80, 0x34,
+0x48, 0x32, 0xE0, 0x37, 0xE0, 0x37, 0x6D, 0xEF,
+0xED, 0xEC, 0x20, 0xF0, 0xC1, 0xA4, 0x20, 0xF0,
+0xE2, 0xA4, 0x20, 0xF0, 0x60, 0xA4, 0x20, 0xF0,
+0x83, 0xA4, 0xC0, 0x36, 0xE0, 0x37, 0xCD, 0xEB,
+0xE0, 0x37, 0x6D, 0xEF, 0x00, 0xF6, 0x80, 0x34,
+0xED, 0xEC, 0x89, 0xE2, 0x85, 0xA2, 0xC6, 0xA2,
+0xE4, 0xA2, 0x47, 0xA2, 0x80, 0x34, 0xC0, 0x36,
+0x8D, 0xEF, 0xC0, 0x36, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0xE1, 0xA2, 0x9B, 0xA5,
+0x0F, 0x6E, 0xF0, 0x37, 0xCC, 0xEC, 0xED, 0xEC,
+0x9B, 0xC5, 0x82, 0xA2, 0x58, 0xA5, 0x90, 0x34,
+0xCC, 0xEA, 0x8D, 0xEA, 0x58, 0xC5, 0x43, 0xA1,
+0x85, 0x67, 0xD1, 0x18, 0xAE, 0x3D, 0x59, 0xC5,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x06, 0x94, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x00, 0x6E, 0x07, 0x6D, 0xD1, 0x18,
+0x2D, 0x3B, 0x90, 0x67, 0x7F, 0xF7, 0x05, 0x22,
+0x00, 0x6A, 0x64, 0x17, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD1, 0x13, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xA4, 0x67, 0x14, 0x6E, 0x40, 0xEA, 0x06, 0x04,
+0x5D, 0x67, 0x98, 0xA2, 0x03, 0x6A, 0xFF, 0x69,
+0x4C, 0xEC, 0x08, 0x6D, 0xD1, 0x18, 0x06, 0x3B,
+0x2C, 0xEC, 0x06, 0x2A, 0x04, 0x6A, 0x15, 0x97,
+0x14, 0x91, 0x13, 0x90, 0x00, 0xEF, 0x0B, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x0D, 0xD2, 0x5D, 0x67, 0x98, 0xA2, 0x03, 0x6A,
+0x4C, 0xEC, 0x5D, 0x67, 0xBA, 0xA2, 0xD1, 0x18,
+0xE3, 0x3A, 0x2C, 0xEC, 0x0C, 0xD2, 0x5D, 0x67,
+0x98, 0xA2, 0x03, 0x6A, 0x4C, 0xEC, 0x5D, 0x67,
+0xBB, 0xA2, 0xD1, 0x18, 0xE3, 0x3A, 0x2C, 0xEC,
+0x02, 0x67, 0x5D, 0x67, 0x98, 0xA2, 0x03, 0x6A,
+0x4C, 0xEC, 0x5D, 0x67, 0xB9, 0xA2, 0xD1, 0x18,
+0xE3, 0x3A, 0x2C, 0xEC, 0x0C, 0x93, 0x02, 0x23,
+0x01, 0x20, 0x12, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x0D, 0x94, 0x5D, 0x67, 0x58, 0xA2,
+0x03, 0x6C, 0x12, 0x6E, 0x07, 0x6D, 0x4C, 0xEC,
+0xD1, 0x18, 0x2D, 0x3B, 0x00, 0x65, 0xBE, 0x17,
+0x11, 0xD2, 0x5D, 0x67, 0x98, 0xA2, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0x03, 0x6A, 0xA0, 0x35,
+0xC6, 0xF4, 0xA8, 0x9D, 0x4C, 0xEC, 0x2C, 0xEC,
+0x94, 0x34, 0x89, 0xE5, 0x10, 0xD0, 0x9D, 0xA2,
+0x1C, 0xA2, 0xBE, 0xA2, 0x80, 0x34, 0x0F, 0xD2,
+0x0D, 0xEC, 0x1F, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xD3, 0xF3, 0xE0, 0x9C, 0x97, 0xF0, 0x19, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x5F, 0xF6, 0x1D, 0x4C,
+0xC4, 0x67, 0x0E, 0xD4, 0xBD, 0xA0, 0x40, 0xEF,
+0x9C, 0xA0, 0x11, 0x92, 0x0F, 0x6E, 0x08, 0x97,
+0x9B, 0xA2, 0xB8, 0xA2, 0xCC, 0xEC, 0x2C, 0xEC,
+0xFC, 0x65, 0xCC, 0xED, 0x07, 0x96, 0x7F, 0x67,
+0x7C, 0xC0, 0x7D, 0x67, 0x20, 0xF0, 0x70, 0xA3,
+0xC2, 0x34, 0x95, 0xC0, 0x82, 0x34, 0x96, 0xC0,
+0x00, 0xF6, 0xC2, 0x34, 0x97, 0xC0, 0x20, 0xF0,
+0x60, 0xC0, 0xE2, 0x34, 0x0C, 0x93, 0x99, 0xC0,
+0x82, 0x34, 0x9A, 0xC0, 0x00, 0xF6, 0xE2, 0x34,
+0x9B, 0xC0, 0x62, 0x34, 0x20, 0xF0, 0x81, 0xC0,
+0x82, 0x34, 0x20, 0xF0, 0x82, 0xC0, 0x00, 0xF6,
+0x62, 0x34, 0x10, 0x93, 0x20, 0xF0, 0x83, 0xC0,
+0x20, 0xF0, 0x48, 0xC0, 0x20, 0xF0, 0x64, 0xC0,
+0x62, 0x34, 0x00, 0xF6, 0x62, 0x33, 0x20, 0xF0,
+0x67, 0xC0, 0x42, 0x33, 0x20, 0xF0, 0x85, 0xC0,
+0x20, 0xF0, 0x69, 0xC0, 0x82, 0x34, 0x62, 0x33,
+0x00, 0xF6, 0x42, 0x32, 0x2C, 0xED, 0x20, 0xF0,
+0x86, 0xC0, 0x20, 0xF0, 0x6A, 0xC0, 0x20, 0xF0,
+0x4B, 0xC0, 0xD4, 0xC0, 0xF8, 0xC0, 0xBD, 0xC0,
+0x5D, 0x67, 0x20, 0xF0, 0x44, 0xA2, 0x01, 0x6B,
+0x9F, 0x67, 0x20, 0xF0, 0x4C, 0xC0, 0x5D, 0x67,
+0x20, 0xF0, 0x45, 0xA2, 0x20, 0xF0, 0x4D, 0xC0,
+0x5D, 0x67, 0x20, 0xF0, 0x46, 0xA2, 0x20, 0xF0,
+0x4E, 0xC0, 0x5D, 0x67, 0x20, 0xF0, 0x47, 0xA2,
+0x20, 0xF0, 0x4F, 0xC0, 0x5D, 0x67, 0x20, 0xF0,
+0x48, 0xA2, 0x20, 0xF0, 0x50, 0xC0, 0x5D, 0x67,
+0x20, 0xF0, 0x49, 0xA2, 0x20, 0xF0, 0x51, 0xC0,
+0x5D, 0x67, 0x20, 0xF0, 0x4A, 0xA2, 0x20, 0xF0,
+0x52, 0xC0, 0x5D, 0x67, 0x20, 0xF0, 0x4B, 0xA2,
+0x20, 0xF0, 0x53, 0xC0, 0x5D, 0x67, 0x58, 0xA2,
+0x4A, 0x32, 0x6C, 0xEA, 0x0F, 0x93, 0x20, 0xF0,
+0x54, 0xC0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x05, 0xD3, 0x0E, 0x93, 0xD3, 0xF3,
+0x44, 0x9A, 0x40, 0xEA, 0x04, 0xD3, 0x19, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x10, 0x6D, 0x40, 0xEA,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x0D, 0x94, 0x5D, 0x67, 0x98, 0xA2, 0x03, 0x6A,
+0x15, 0x6E, 0x4C, 0xEC, 0x07, 0x6D, 0x2C, 0xEC,
+0x33, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x0D, 0x94, 0x5D, 0x67, 0x98, 0xA2, 0x03, 0x6A,
+0x00, 0x6E, 0x4C, 0xEC, 0x08, 0x6D, 0xD1, 0x18,
+0x2D, 0x3B, 0x2C, 0xEC, 0xFF, 0xF6, 0x02, 0x22,
+0x00, 0x6A, 0xE1, 0x16, 0xFD, 0x63, 0x05, 0x62,
+0x41, 0xA4, 0x04, 0x72, 0x37, 0x60, 0x05, 0x5A,
+0x17, 0x60, 0x01, 0x72, 0x27, 0x60, 0x20, 0x22,
+0x02, 0x72, 0x28, 0x60, 0x03, 0x72, 0x2A, 0x60,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x89, 0xF1, 0xB4, 0x9B,
+0x40, 0xEA, 0x00, 0x6C, 0x04, 0x6A, 0x0F, 0x10,
+0x06, 0x72, 0x24, 0x60, 0x06, 0x5A, 0x1E, 0x61,
+0x07, 0x72, 0x24, 0x60, 0x08, 0x72, 0xE8, 0x61,
+0xD1, 0x18, 0x41, 0x46, 0x08, 0x4C, 0x03, 0x10,
+0xD1, 0x18, 0x98, 0x42, 0x08, 0x4C, 0x05, 0x97,
+0x00, 0xEF, 0x03, 0x63, 0xD1, 0x18, 0x26, 0x43,
+0x08, 0x4C, 0xF9, 0x17, 0xD1, 0x18, 0xE9, 0x43,
+0x08, 0x4C, 0xF5, 0x17, 0xD1, 0x18, 0x5D, 0x44,
+0x08, 0x4C, 0xF1, 0x17, 0xD1, 0x18, 0xB7, 0x44,
+0x08, 0x4C, 0xED, 0x17, 0xD1, 0x18, 0xE2, 0x44,
+0x08, 0x4C, 0xE9, 0x17, 0xD1, 0x18, 0x81, 0x45,
+0x08, 0x4C, 0xE5, 0x17, 0xD1, 0x18, 0xEA, 0x45,
+0x08, 0x4C, 0xE1, 0x17, 0x01, 0x6A, 0x20, 0xE8,
+0x4B, 0xEA, 0x00, 0x65, 0x01, 0x6A, 0x20, 0xE8,
+0x4B, 0xEA, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x6A, 0x20, 0xE8, 0x00, 0x6A,
+0x20, 0xE8, 0x00, 0x65, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x20, 0xE8, 0x18, 0x6A, 0x20, 0xE8, 0x18, 0x6A,
+0x41, 0xA4, 0x02, 0x5A, 0x14, 0x61, 0xFD, 0x63,
+0x05, 0x62, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x89, 0xF1,
+0xB4, 0x9B, 0x40, 0xEA, 0x00, 0x6C, 0x05, 0x97,
+0x03, 0x6A, 0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8,
+0x18, 0x6A, 0x00, 0x65, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0x1C, 0x2C, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xCA, 0xF2, 0x00, 0x4A,
+0x05, 0xD2, 0x07, 0xD4, 0x40, 0xF2, 0x0C, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x04, 0x04,
+0x40, 0xEA, 0x00, 0x65, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x81, 0x9C,
+0x04, 0x5C, 0x0F, 0x61, 0x00, 0x6A, 0x07, 0xD2,
+0x7D, 0x67, 0x07, 0x6A, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xCA, 0xF2,
+0x00, 0x4A, 0x05, 0xD2, 0x40, 0xF2, 0x13, 0x6A,
+0xDF, 0x17, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B, 0xA5, 0xA3,
+0x44, 0xA3, 0xC6, 0xA3, 0xA0, 0x35, 0x4D, 0xED,
+0x47, 0xA3, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0,
+0xBD, 0xA2, 0x20, 0xF0, 0x1C, 0xA2, 0x20, 0xF0,
+0x7E, 0xA2, 0xA0, 0x35, 0x0D, 0xED, 0x20, 0xF0,
+0x1F, 0xA2, 0x60, 0x33, 0x60, 0x33, 0x84, 0x32,
+0x6D, 0xED, 0x89, 0xE2, 0x00, 0xF6, 0x00, 0x30,
+0xAD, 0xE8, 0x48, 0x32, 0x41, 0xE0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x3A, 0x65,
+0x46, 0xA0, 0x08, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A,
+0x99, 0x67, 0xAE, 0x17, 0x69, 0xA0, 0x88, 0xA0,
+0x4A, 0xA0, 0x60, 0x33, 0x6D, 0xEC, 0x6B, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x4D, 0xEB, 0x08, 0xD3, 0x00, 0x69,
+0x45, 0xA0, 0x43, 0xE9, 0xE7, 0x60, 0x08, 0x93,
+0x28, 0x32, 0x49, 0xE3, 0x40, 0x9A, 0x04, 0x2A,
+0x01, 0x49, 0xFF, 0x6A, 0x4C, 0xE9, 0xF4, 0x17,
+0xFF, 0x6B, 0x34, 0x34, 0x6C, 0xEC, 0x09, 0xD4,
+0x00, 0x6B, 0x01, 0x6C, 0x84, 0xEB, 0x4C, 0xEC,
+0x11, 0x24, 0x0C, 0xD2, 0x59, 0x67, 0x0B, 0xD2,
+0x09, 0x92, 0x1C, 0x6F, 0x01, 0x6E, 0x71, 0xE2,
+0xFF, 0x6A, 0x4C, 0xEC, 0x01, 0x6D, 0xB1, 0x18,
+0x88, 0xE8, 0x0A, 0xD3, 0x0B, 0x93, 0x0C, 0x92,
+0x3B, 0x65, 0x0A, 0x93, 0x01, 0x4B, 0x20, 0x73,
+0xE8, 0x61, 0xDE, 0x17, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0x1C, 0x2C, 0x06, 0x6A,
+0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xCA, 0xF2, 0x00, 0x4A,
+0x05, 0xD2, 0x07, 0xD4, 0x00, 0xF2, 0x0A, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x04, 0x04,
+0x40, 0xEA, 0x00, 0x65, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0xC1, 0x9C,
+0x04, 0x5E, 0x0F, 0x61, 0x00, 0x6A, 0x07, 0xD2,
+0x7D, 0x67, 0x07, 0x6A, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xCA, 0xF2,
+0x00, 0x4A, 0x05, 0xD2, 0x00, 0xF2, 0x11, 0x6A,
+0xDF, 0x17, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B, 0x85, 0xA3,
+0x44, 0xA3, 0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0,
+0xBD, 0xA2, 0x20, 0xF0, 0x1C, 0xA2, 0x20, 0xF0,
+0x7E, 0xA2, 0xA0, 0x35, 0x0D, 0xED, 0x60, 0x33,
+0x20, 0xF0, 0x1F, 0xA2, 0x60, 0x33, 0x6D, 0xED,
+0xC4, 0x33, 0x00, 0xF6, 0x00, 0x30, 0xCD, 0xE3,
+0xAD, 0xE8, 0x68, 0x33, 0x20, 0xF0, 0x95, 0xA2,
+0x61, 0xE0, 0x20, 0xF0, 0x74, 0xA2, 0x20, 0xF0,
+0xB6, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x20, 0xF0,
+0x77, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x00, 0x6D,
+0x00, 0x6A, 0xB4, 0x34, 0x91, 0xE3, 0x90, 0xA4,
+0x01, 0x6E, 0xCC, 0xEC, 0xFF, 0x6E, 0xCC, 0xEC,
+0x02, 0x24, 0x01, 0x4A, 0xCC, 0xEA, 0x01, 0x4D,
+0x04, 0x75, 0xF3, 0x61, 0x03, 0x2A, 0xB1, 0x18,
+0x90, 0xD1, 0x10, 0x6C, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x3A, 0x65, 0x46, 0xA0,
+0x08, 0x2A, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x99, 0x67,
+0x8B, 0x17, 0x69, 0xA0, 0x88, 0xA0, 0x4A, 0xA0,
+0x60, 0x33, 0x6D, 0xEC, 0x6B, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x00, 0xF6, 0x60, 0x33,
+0x4D, 0xEB, 0x08, 0xD3, 0x00, 0x69, 0x45, 0xA0,
+0x43, 0xE9, 0xE7, 0x60, 0x08, 0x93, 0x28, 0x32,
+0x49, 0xE3, 0x40, 0x9A, 0x04, 0x2A, 0x01, 0x49,
+0xFF, 0x6A, 0x4C, 0xE9, 0xF4, 0x17, 0xFF, 0x6B,
+0x34, 0x34, 0x6C, 0xEC, 0x09, 0xD4, 0x00, 0x6B,
+0x01, 0x6C, 0x84, 0xEB, 0x4C, 0xEC, 0x11, 0x24,
+0x0C, 0xD2, 0x59, 0x67, 0x0B, 0xD2, 0x09, 0x92,
+0x1B, 0x6F, 0x00, 0x6E, 0x71, 0xE2, 0xFF, 0x6A,
+0x4C, 0xEC, 0x00, 0x6D, 0xB1, 0x18, 0x88, 0xE8,
+0x0A, 0xD3, 0x0B, 0x93, 0x0C, 0x92, 0x3B, 0x65,
+0x0A, 0x93, 0x01, 0x4B, 0x20, 0x73, 0xE8, 0x61,
+0xDE, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x17, 0x2C, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xCA, 0xF2, 0x00, 0x4A, 0x05, 0xD2, 0x07, 0xD4,
+0x60, 0xF2, 0x1E, 0x6A, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x04, 0x04, 0x38, 0x10,
+0x41, 0x9C, 0x04, 0x5A, 0x38, 0x60, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x15, 0xF0,
+0x08, 0x4B, 0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x95, 0xA2, 0x20, 0xF0,
+0x74, 0xA2, 0x20, 0xF0, 0xB6, 0xA2, 0x80, 0x34,
+0x6D, 0xEC, 0x20, 0xF0, 0x77, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x00, 0x6D, 0x00, 0x6A, 0xB4, 0x34,
+0x91, 0xE3, 0x90, 0xA4, 0x01, 0x6E, 0xCC, 0xEC,
+0xFF, 0x6E, 0xCC, 0xEC, 0x02, 0x24, 0x01, 0x4A,
+0xCC, 0xEA, 0x01, 0x4D, 0x04, 0x75, 0xF3, 0x61,
+0x03, 0x2A, 0xB1, 0x18, 0x90, 0xD1, 0x12, 0x6C,
+0x09, 0x97, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x6A,
+0x07, 0xD2, 0x7D, 0x67, 0x07, 0x6A, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xCA, 0xF2, 0x00, 0x4A, 0x05, 0xD2, 0x80, 0xF2,
+0x05, 0x6A, 0xAC, 0x17, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x00, 0x6A, 0x60, 0x33, 0x15, 0xF0,
+0x08, 0x4B, 0x07, 0xD2, 0x45, 0xA3, 0xC4, 0xA3,
+0x86, 0xA3, 0x40, 0x32, 0xCD, 0xEA, 0xC7, 0xA3,
+0x80, 0x34, 0xF7, 0xF0, 0x01, 0x68, 0xF7, 0xF0,
+0x01, 0x69, 0x80, 0x34, 0x00, 0x30, 0x20, 0x31,
+0x8D, 0xEA, 0x00, 0xF6, 0xC0, 0x36, 0x00, 0x30,
+0x20, 0x31, 0x4D, 0xEE, 0xA9, 0xF3, 0xAC, 0x99,
+0x72, 0xF4, 0x54, 0x98, 0x30, 0x6C, 0x09, 0xD3,
+0x40, 0xEA, 0x08, 0xD6, 0x08, 0x96, 0x09, 0x93,
+0x42, 0x34, 0x20, 0xF0, 0x5C, 0xC6, 0x20, 0xF0,
+0x9D, 0xC6, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x20, 0xF0, 0x9E, 0xC6, 0x20, 0xF0, 0x5F, 0xC6,
+0xA5, 0xA3, 0x44, 0xA3, 0x86, 0xA3, 0xA0, 0x35,
+0x4D, 0xED, 0x47, 0xA3, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x20, 0xF0, 0xBD, 0xA2, 0x20, 0xF0, 0x9C, 0xA2,
+0x20, 0xF0, 0xDE, 0xA2, 0xA0, 0x35, 0x8D, 0xED,
+0x20, 0xF0, 0x9F, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC,
+0x19, 0x2C, 0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xCA, 0xF2, 0x00, 0x4A, 0x05, 0xD2, 0x19, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x40, 0xEA,
+0x04, 0x04, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x93, 0xF6, 0x40, 0x9A,
+0x30, 0x6E, 0x00, 0x6D, 0x40, 0xEA, 0x08, 0xD3,
+0x08, 0x93, 0x00, 0x6F, 0x85, 0xA3, 0x44, 0xA3,
+0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x0A, 0xD3, 0x8D, 0xEA, 0x20, 0xF0,
+0x9D, 0xA2, 0x20, 0xF0, 0xDC, 0xA2, 0x20, 0xF0,
+0xBE, 0xA2, 0x80, 0x34, 0xCD, 0xEC, 0x20, 0xF0,
+0xDF, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0xC0, 0x36, 0x8D, 0xEE, 0xF9, 0xE6,
+0x08, 0x6A, 0x45, 0xC6, 0xA9, 0xF3, 0xAC, 0x99,
+0x72, 0xF4, 0x54, 0x98, 0x40, 0x6C, 0x09, 0xD7,
+0x40, 0xEA, 0x08, 0xD6, 0x08, 0x96, 0x42, 0x34,
+0x09, 0x97, 0x89, 0xC6, 0x82, 0x34, 0x8A, 0xC6,
+0x00, 0xF6, 0x42, 0x34, 0x48, 0xC6, 0x8B, 0xC6,
+0x0A, 0x93, 0x0C, 0x2A, 0x06, 0x6A, 0x7D, 0x67,
+0x50, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xCA, 0xF2, 0x00, 0x4A, 0x05, 0xD2,
+0x26, 0x6A, 0xA6, 0x17, 0x0C, 0x4F, 0x30, 0x77,
+0xBD, 0x61, 0xAB, 0x17, 0xFD, 0x63, 0x05, 0x62,
+0x41, 0xA4, 0x02, 0x6B, 0x6B, 0xEB, 0x6C, 0xEA,
+0x41, 0xC4, 0x00, 0x6A, 0x44, 0xC4, 0x46, 0xC4,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x93, 0xF6, 0xE0, 0x9A, 0x69, 0xA4, 0x4A, 0xA4,
+0xA8, 0xA4, 0xC5, 0xA4, 0x8B, 0xA4, 0x60, 0x33,
+0x40, 0x32, 0x6D, 0xED, 0x40, 0x32, 0xAD, 0xEA,
+0xFF, 0x6B, 0xCC, 0x36, 0x00, 0xF6, 0x80, 0x34,
+0x6C, 0xEE, 0x00, 0x6D, 0x40, 0xEF, 0x4D, 0xEC,
+0x05, 0x97, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0x24, 0x67, 0x19, 0x2C, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x04, 0xD4, 0xA9, 0xF6, 0xAC, 0x9B, 0xA9, 0xF6,
+0x88, 0x9B, 0x80, 0xF2, 0x18, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x0D, 0x6A, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x15, 0xF0,
+0x08, 0x4B, 0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x7D, 0xA2, 0x20, 0xF0,
+0x1C, 0xA2, 0x20, 0xF0, 0x9E, 0xA2, 0x60, 0x33,
+0x0D, 0xEB, 0x20, 0xF0, 0x1F, 0xA2, 0x80, 0x34,
+0x80, 0x34, 0x41, 0xA1, 0x8D, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x0F, 0x6B, 0x6C, 0xEA,
+0xFF, 0x6B, 0x6C, 0xEA, 0x44, 0x34, 0x49, 0xE4,
+0x48, 0x32, 0xC1, 0xA1, 0x41, 0xE0, 0x44, 0xA0,
+0xD2, 0x36, 0x6C, 0xEE, 0x47, 0xEE, 0x01, 0x6C,
+0x8C, 0xEA, 0x3E, 0x22, 0xA0, 0xA0, 0x80, 0xA0,
+0x0F, 0x6A, 0x4C, 0xED, 0x92, 0x34, 0x01, 0x6A,
+0x4C, 0xEC, 0x6C, 0xED, 0x6C, 0xEC, 0x00, 0x6F,
+0xB1, 0x18, 0x2E, 0xEE, 0x0E, 0xD3, 0xA2, 0x67,
+0x81, 0xA1, 0x27, 0x22, 0x09, 0xD5, 0x92, 0x34,
+0x00, 0x6D, 0x08, 0xD5, 0x07, 0xD4, 0x20, 0xA0,
+0x0F, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x6C, 0xE9,
+0x0E, 0x93, 0x40, 0x32, 0x40, 0x32, 0x6C, 0xE9,
+0x06, 0xD1, 0x80, 0xA0, 0x01, 0x6B, 0xF3, 0xF0,
+0x50, 0x9A, 0x92, 0x34, 0x8C, 0xEB, 0x05, 0xD3,
+0x05, 0x6B, 0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xA9, 0xF6, 0x88, 0x9B,
+0xA9, 0xF6, 0xAC, 0x9B, 0x80, 0xF2, 0x1C, 0x6F,
+0x02, 0x6E, 0x40, 0xEA, 0x00, 0x65, 0x07, 0x6A,
+0x91, 0x17, 0x64, 0xA0, 0x92, 0x34, 0x01, 0x6A,
+0x44, 0xEC, 0x4F, 0xEA, 0x6C, 0xEA, 0x44, 0xC0,
+0x5D, 0x67, 0x20, 0xF0, 0x48, 0xA2, 0x01, 0x6B,
+0xC0, 0xA0, 0x6D, 0xEA, 0x7D, 0x67, 0x20, 0xF0,
+0x48, 0xC3, 0x8D, 0xA1, 0x4C, 0xA1, 0x6E, 0xA1,
+0x80, 0x34, 0x4D, 0xEC, 0x4F, 0xA1, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x0B, 0xD2, 0x89, 0xA1, 0x48, 0xA1,
+0x6A, 0xA1, 0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA1,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x0C, 0xD2, 0x85, 0xA1,
+0x44, 0xA1, 0x66, 0xA1, 0x80, 0x34, 0x4D, 0xEC,
+0x47, 0xA1, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x70, 0xA1, 0x8D, 0xEA,
+0xA0, 0xA0, 0x0D, 0xD2, 0xE1, 0xA1, 0x5D, 0x67,
+0x20, 0xF0, 0x69, 0xC2, 0x0F, 0x6A, 0xFF, 0x6B,
+0x4C, 0xEE, 0xB2, 0x35, 0x01, 0x6A, 0x6C, 0xEE,
+0xF2, 0x37, 0x4C, 0xED, 0x0A, 0x04, 0xB1, 0x18,
+0xAE, 0xEE, 0x0E, 0xD3, 0x0E, 0x93, 0x17, 0x22,
+0x05, 0xD2, 0x01, 0x6A, 0x04, 0xD2, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x40, 0x32, 0x40, 0x32, 0x60, 0x33, 0xA9, 0xF6,
+0x88, 0x9A, 0xA9, 0xF6, 0xAC, 0x9A, 0xF3, 0xF0,
+0x70, 0x9B, 0x80, 0xF2, 0x1D, 0x6F, 0x40, 0xEB,
+0x02, 0x6E, 0x05, 0x6A, 0x2F, 0x17, 0xA0, 0xA0,
+0x80, 0xA0, 0xC1, 0xA1, 0x0F, 0x6A, 0x4C, 0xED,
+0x92, 0x34, 0x01, 0x6A, 0x4C, 0xEC, 0xD2, 0x36,
+0x6C, 0xEC, 0x01, 0x6F, 0x6C, 0xEE, 0x6C, 0xED,
+0xB1, 0x18, 0x2E, 0xEE, 0x0E, 0xD3, 0x81, 0xA1,
+0x26, 0x22, 0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35,
+0x09, 0xD2, 0xA0, 0x35, 0x01, 0x6A, 0x92, 0x34,
+0xF3, 0xF0, 0x70, 0x9D, 0x07, 0xD4, 0x08, 0xD2,
+0x40, 0xA0, 0x1B, 0x65, 0x0F, 0x6B, 0x6C, 0xEA,
+0x0E, 0x93, 0x80, 0xF2, 0x1C, 0x6F, 0x02, 0x6E,
+0x4C, 0xEB, 0x06, 0xD3, 0x40, 0xA0, 0x01, 0x6B,
+0x52, 0x32, 0x6C, 0xEA, 0x05, 0xD2, 0x05, 0x6A,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xA9, 0xF6, 0x88, 0x9A, 0xA9, 0xF6,
+0xAC, 0x9A, 0x58, 0x67, 0x62, 0x17, 0x64, 0xA0,
+0x92, 0x34, 0x01, 0x69, 0x24, 0xEC, 0x6D, 0xE9,
+0x24, 0xC0, 0xF0, 0x16, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x24, 0x67, 0x19, 0x2C,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0x04, 0xD4, 0xA9, 0xF6,
+0xAC, 0x9B, 0xA9, 0xF6, 0x88, 0x9B, 0x80, 0xF2,
+0x18, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0x0D, 0x6A,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0x61, 0xA4, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x15, 0xF0, 0x08, 0x4C,
+0x0F, 0x6A, 0x4C, 0xEB, 0xA5, 0xA4, 0xFF, 0x6A,
+0x4C, 0xEB, 0x1A, 0x65, 0x44, 0xA4, 0xC6, 0xA4,
+0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x20, 0xF0, 0x9D, 0xA2, 0x20, 0xF0,
+0x1C, 0xA2, 0x20, 0xF0, 0xBE, 0xA2, 0x80, 0x34,
+0x0D, 0xEC, 0x20, 0xF0, 0x1F, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0x64, 0x32, 0xC1, 0xA1, 0xAD, 0xEC,
+0x00, 0xF6, 0x00, 0x30, 0x69, 0xE2, 0x8D, 0xE8,
+0x48, 0x32, 0x41, 0xE0, 0xD2, 0x36, 0x58, 0x67,
+0x4C, 0xEE, 0x44, 0xA0, 0x01, 0x6C, 0x47, 0xEE,
+0x8C, 0xEA, 0x69, 0x22, 0xA0, 0xA0, 0x80, 0xA0,
+0x0F, 0x6A, 0x01, 0x6B, 0x4C, 0xED, 0x92, 0x34,
+0x5D, 0x67, 0x6C, 0xEC, 0x20, 0xF0, 0x6C, 0xC2,
+0x58, 0x67, 0x4C, 0xEC, 0x00, 0x6F, 0x4C, 0xED,
+0xB1, 0x18, 0x2E, 0xEE, 0x0A, 0xD2, 0x0A, 0x93,
+0x81, 0xA1, 0x1B, 0x65, 0x7D, 0x67, 0x20, 0xF0,
+0x6C, 0xA3, 0x27, 0x22, 0x09, 0xD2, 0x92, 0x34,
+0x00, 0x6A, 0x08, 0xD2, 0x07, 0xD4, 0x20, 0xA0,
+0x0F, 0x6A, 0xF7, 0xF0, 0x01, 0x6D, 0x4C, 0xE9,
+0x58, 0x67, 0x4C, 0xE9, 0x06, 0xD1, 0x40, 0xA0,
+0xA0, 0x35, 0xA0, 0x35, 0x52, 0x32, 0x4C, 0xEB,
+0x05, 0x6A, 0xF3, 0xF0, 0xB0, 0x9D, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x3D, 0x65, 0xA9, 0xF6, 0x88, 0x9A, 0xA9, 0xF6,
+0xAC, 0x9A, 0x05, 0xD3, 0x59, 0x67, 0x80, 0xF2,
+0x1C, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0x07, 0x6A,
+0x87, 0x17, 0x92, 0x34, 0x01, 0x6D, 0xA4, 0xEC,
+0xAF, 0xEC, 0xA4, 0xA0, 0xAC, 0xEC, 0xB8, 0x67,
+0xAC, 0xEC, 0x84, 0xC0, 0x7F, 0xF7, 0x1C, 0x2C,
+0x81, 0xA1, 0x0F, 0x69, 0xF7, 0xF0, 0x01, 0x6A,
+0x8C, 0xE9, 0xAC, 0xE9, 0x06, 0xD1, 0x80, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x92, 0x34, 0x8C, 0xEB,
+0x05, 0xD3, 0x02, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xF3, 0xF0,
+0x50, 0x9A, 0xA9, 0xF6, 0x88, 0x9B, 0xA9, 0xF6,
+0xAC, 0x9B, 0x80, 0xF2, 0x1F, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x0B, 0x6A, 0x5D, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x07, 0xD6, 0x06, 0xD3,
+0x40, 0x32, 0xF3, 0xF0, 0x30, 0x9A, 0x40, 0xA0,
+0x01, 0x6B, 0x80, 0xF2, 0x1E, 0x6F, 0x52, 0x32,
+0x6C, 0xEA, 0x05, 0xD2, 0x03, 0x6A, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA9, 0xF6, 0x88, 0x9A, 0xA9, 0xF6, 0xAC, 0x9A,
+0x40, 0xE9, 0x02, 0x6E, 0x0A, 0x6A, 0x40, 0x17,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0x04, 0x67, 0x19, 0x2C, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x04, 0xD4, 0xA9, 0xF6, 0xAC, 0x9B, 0xA9, 0xF6,
+0x88, 0x9B, 0xA0, 0xF2, 0x0C, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x0D, 0x6A, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0x69, 0xA0,
+0x88, 0xA4, 0x4A, 0xA0, 0x60, 0x33, 0x6D, 0xEC,
+0x6B, 0xA0, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x00, 0xF6, 0x60, 0x33, 0x4D, 0xEB, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x0B, 0xD3, 0x40, 0xEA, 0x00, 0x69,
+0x0A, 0xD2, 0x65, 0xA0, 0x63, 0xE9, 0x0A, 0x61,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x72, 0xF0, 0x74, 0x9B, 0x40, 0xEB, 0x0A, 0x94,
+0x00, 0x6A, 0xD8, 0x17, 0x0B, 0x93, 0x28, 0x32,
+0x49, 0xE3, 0x60, 0x9A, 0x04, 0x2B, 0x01, 0x49,
+0xFF, 0x6B, 0x6C, 0xE9, 0xEA, 0x17, 0xFF, 0x6B,
+0x34, 0x34, 0x6C, 0xEC, 0x00, 0x6B, 0x0C, 0xD4,
+0xA3, 0x67, 0x80, 0x9A, 0x01, 0x6B, 0x64, 0xED,
+0x6C, 0xEC, 0x17, 0x24, 0x0E, 0xD2, 0x0C, 0x92,
+0x0D, 0xD5, 0x1A, 0x6F, 0xB1, 0xE2, 0xFF, 0x6A,
+0x4C, 0xEC, 0x00, 0x6E, 0x00, 0x6D, 0xB1, 0x18,
+0x88, 0xE8, 0x0F, 0xD3, 0x0E, 0x92, 0x0F, 0x93,
+0x80, 0x9A, 0x6F, 0xEB, 0x8C, 0xEB, 0x60, 0xDA,
+0x66, 0xA0, 0x0A, 0x23, 0x0D, 0x95, 0xFF, 0x4B,
+0x66, 0xC0, 0x65, 0x67, 0x01, 0x4B, 0x20, 0x73,
+0xA3, 0x67, 0x78, 0x67, 0xDE, 0x2B, 0xD3, 0x17,
+0x5D, 0x67, 0x09, 0xD3, 0x07, 0x6B, 0x78, 0xC2,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xCA, 0xF2, 0x00, 0x4B, 0x07, 0xD3, 0xFF, 0x6B,
+0x34, 0x4B, 0x08, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x12, 0xF1, 0x78, 0x9B,
+0x40, 0xEB, 0x06, 0x04, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x72, 0xF0, 0x74, 0x9B,
+0x40, 0xEB, 0x0A, 0x94, 0x11, 0x6A, 0x86, 0x17,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xE4, 0x67, 0x19, 0x2C, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x04, 0xD4, 0xA9, 0xF6, 0xAC, 0x9B, 0xA9, 0xF6,
+0x88, 0x9B, 0x80, 0xF2, 0x18, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x0D, 0x6A, 0x0F, 0x97, 0x0E, 0x91,
+0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x15, 0xF0,
+0x08, 0x4B, 0x85, 0xA3, 0x44, 0xA3, 0xA6, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x7D, 0xA2, 0x20, 0xF0,
+0x1C, 0xA2, 0x20, 0xF0, 0x9E, 0xA2, 0x60, 0x33,
+0x0D, 0xEB, 0x20, 0xF0, 0x1F, 0xA2, 0x80, 0x34,
+0x41, 0xA7, 0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x0F, 0x6B, 0x6C, 0xEA,
+0xFF, 0x69, 0x2C, 0xEA, 0x44, 0x34, 0x49, 0xE4,
+0x48, 0x32, 0x41, 0xE0, 0x80, 0xA0, 0xA0, 0xA0,
+0x5D, 0x67, 0x20, 0xF0, 0x68, 0xC2, 0x92, 0x34,
+0x01, 0x6A, 0x6C, 0xED, 0x4C, 0xEC, 0x00, 0x6E,
+0x2C, 0xED, 0x2C, 0xEC, 0xB1, 0x18, 0xC1, 0xED,
+0x0B, 0xD7, 0x7D, 0x67, 0x20, 0xF0, 0x68, 0xA3,
+0x0B, 0x97, 0x24, 0x22, 0x08, 0xD2, 0x00, 0x6A,
+0x07, 0xD2, 0x40, 0xA0, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x4C, 0xEB, 0x6C, 0xE9, 0x06, 0xD1,
+0x40, 0xA0, 0x01, 0x6B, 0x80, 0x34, 0x52, 0x32,
+0x6C, 0xEA, 0x05, 0xD2, 0x04, 0x6A, 0xF3, 0xF0,
+0x90, 0x9C, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x1C, 0x65, 0xA9, 0xF6,
+0xAC, 0x9A, 0xA9, 0xF6, 0x88, 0x9A, 0x80, 0xF2,
+0x1B, 0x6F, 0x58, 0x67, 0x40, 0xEA, 0x02, 0x6E,
+0x06, 0x6A, 0x98, 0x17, 0x5D, 0x67, 0x90, 0x67,
+0x20, 0xF0, 0x6C, 0xC2, 0xD1, 0x18, 0x8C, 0x49,
+0x0A, 0xD7, 0x7D, 0x67, 0x0A, 0x97, 0x20, 0xF0,
+0x6C, 0xA3, 0x1C, 0x22, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x06, 0xD2, 0x80, 0x34, 0xF3, 0xF0,
+0x10, 0x9C, 0x81, 0xA7, 0x02, 0x6E, 0xA0, 0xF2,
+0x0D, 0x6F, 0x8C, 0xEB, 0x6C, 0xE9, 0x02, 0x6B,
+0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xA9, 0xF6, 0x88, 0x9B, 0xA9, 0xF6,
+0xAC, 0x9B, 0x05, 0xD1, 0x40, 0xE8, 0x0A, 0xD2,
+0x0A, 0x92, 0x70, 0x17, 0x22, 0x67, 0xD1, 0x18,
+0x77, 0x48, 0x90, 0x67, 0x51, 0x67, 0x6A, 0x17,
+0xED, 0x63, 0x25, 0x62, 0x24, 0xD1, 0x23, 0xD0,
+0x64, 0x67, 0x19, 0x2C, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x04, 0xD4, 0xA9, 0xF6, 0xAC, 0x9B, 0xA9, 0xF6,
+0x88, 0x9B, 0x80, 0xF2, 0x18, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x0D, 0x6A, 0x25, 0x97, 0x24, 0x91,
+0x23, 0x90, 0x00, 0xEF, 0x13, 0x63, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x20, 0x31, 0x15, 0xF0,
+0x08, 0x49, 0xA5, 0xA1, 0x44, 0xA1, 0x86, 0xA1,
+0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA1, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x20, 0xF0, 0x9D, 0xA2, 0x20, 0xF0,
+0x1C, 0xA2, 0x20, 0xF0, 0xBE, 0xA2, 0x80, 0x34,
+0x0D, 0xEC, 0x20, 0xF0, 0x1F, 0xA2, 0x41, 0xA3,
+0x21, 0xD3, 0x0F, 0x6B, 0x6C, 0xEA, 0x20, 0xD2,
+0x20, 0x02, 0x40, 0xA2, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x00, 0x30, 0x8D, 0xE8,
+0x44, 0x34, 0x49, 0xE4, 0x48, 0x32, 0x41, 0xE0,
+0xD1, 0x18, 0x77, 0x48, 0x90, 0x67, 0x21, 0x93,
+0x80, 0xA3, 0xD1, 0x18, 0xDE, 0x60, 0x20, 0xD3,
+0xFF, 0x72, 0x20, 0x93, 0x18, 0x61, 0x60, 0xA3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x05, 0xD3,
+0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0xA9, 0xF6, 0x88, 0x9B, 0xA9, 0xF6,
+0xAC, 0x9B, 0x80, 0xF2, 0x19, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x03, 0x6A, 0xAB, 0x17, 0x43, 0xC0,
+0x85, 0xA1, 0xA4, 0xA1, 0xC6, 0xA1, 0x80, 0x34,
+0xAD, 0xEC, 0xA7, 0xA1, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0xA0, 0x35, 0x8D, 0xED,
+0x20, 0xF0, 0xC1, 0xA5, 0x20, 0xF0, 0x80, 0xA5,
+0x20, 0xF0, 0xE2, 0xA5, 0xC0, 0x36, 0x8D, 0xEE,
+0x20, 0xF0, 0x83, 0xA5, 0xE0, 0x37, 0xE0, 0x37,
+0xED, 0xEE, 0x00, 0xF6, 0x80, 0x34, 0xCD, 0xEC,
+0x48, 0x32, 0x89, 0xE2, 0x85, 0xA2, 0x24, 0xA2,
+0xA6, 0xA2, 0x80, 0x34, 0x2D, 0xEC, 0x27, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x20, 0x31, 0x8D, 0xE9, 0x02, 0x6A, 0x9D, 0x67,
+0x20, 0xF0, 0x4C, 0xC4, 0xA1, 0xA1, 0x01, 0x6A,
+0x02, 0x6C, 0x4C, 0xED, 0x5D, 0x67, 0x20, 0xF0,
+0x4D, 0xA2, 0x8B, 0xEC, 0x07, 0x6E, 0x8C, 0xEA,
+0xAD, 0xEA, 0xBD, 0x67, 0x20, 0xF0, 0x4D, 0xC5,
+0xA2, 0xA1, 0xFC, 0x65, 0xCC, 0xED, 0x0F, 0x6E,
+0xCB, 0xEE, 0xA4, 0x35, 0xCC, 0xEA, 0xAD, 0xEA,
+0xBD, 0x67, 0x20, 0xF0, 0x4D, 0xC5, 0x5D, 0x67,
+0x20, 0xF0, 0x4E, 0xA2, 0x01, 0x6D, 0x04, 0x6E,
+0xAD, 0xEA, 0xFC, 0x4D, 0xAC, 0xEA, 0xCD, 0xEA,
+0x08, 0x6E, 0xCD, 0xEA, 0xE7, 0x4E, 0xCC, 0xEA,
+0xF0, 0x4E, 0xCC, 0xEA, 0xDD, 0x67, 0x20, 0xF0,
+0x4E, 0xC6, 0xC0, 0xA1, 0x5D, 0x67, 0xF1, 0xA3,
+0x20, 0xF0, 0xCF, 0xC2, 0xD2, 0xA3, 0x20, 0xF0,
+0x50, 0xA2, 0xC0, 0x36, 0xED, 0xEE, 0xCB, 0xEF,
+0x8C, 0xEA, 0xC0, 0xF7, 0xE2, 0x37, 0x01, 0x6C,
+0xAC, 0xEA, 0x8C, 0xEF, 0xFA, 0x4C, 0xE8, 0x37,
+0x8C, 0xEA, 0xED, 0xEA, 0xE2, 0xA3, 0x1C, 0x65,
+0x01, 0x6C, 0xF6, 0x37, 0x8C, 0xEF, 0xEC, 0x34,
+0x09, 0x6F, 0xEB, 0xEF, 0xEC, 0xEA, 0x8D, 0xEA,
+0x11, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0xF1, 0x4C,
+0x8D, 0xEA, 0x9D, 0x67, 0x20, 0xF0, 0x50, 0xC4,
+0x5D, 0x67, 0x20, 0xF0, 0x51, 0xA2, 0x01, 0x6C,
+0x8D, 0xEA, 0x4C, 0xED, 0x58, 0x67, 0x4C, 0xED,
+0xEC, 0xED, 0x5D, 0x67, 0x9D, 0x67, 0x20, 0xF0,
+0xB1, 0xC2, 0xD9, 0xCA, 0x60, 0x6D, 0x00, 0x6A,
+0x5A, 0xCC, 0xBB, 0xCC, 0x5C, 0xCC, 0x5D, 0xCC,
+0x97, 0xF0, 0x19, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x03, 0xF6, 0x0D, 0x4A, 0x0F, 0xD2, 0x00, 0x6A,
+0x10, 0xD2, 0x13, 0xD2, 0x14, 0xD2, 0x41, 0xA3,
+0x0F, 0x6C, 0x97, 0xF0, 0x19, 0x6D, 0x8C, 0xEA,
+0x20, 0xD2, 0x20, 0x02, 0x40, 0xA2, 0xA0, 0x35,
+0xA0, 0x35, 0x16, 0xD2, 0x17, 0x02, 0x0A, 0xD2,
+0x83, 0xF7, 0x1D, 0x4D, 0x5D, 0x67, 0x11, 0xD5,
+0x40, 0xF0, 0x5C, 0xA2, 0x97, 0xF0, 0x19, 0x6D,
+0xA0, 0x35, 0x01, 0x6C, 0xA0, 0x35, 0x8D, 0xEA,
+0xC3, 0xF4, 0x05, 0x4D, 0x9D, 0x67, 0x40, 0xF0,
+0x5C, 0xC4, 0x12, 0xD5, 0xCD, 0xA3, 0x4C, 0xA3,
+0xAE, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x4F, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x18, 0xD2, 0xC9, 0xA3,
+0x48, 0xA3, 0xAA, 0xA3, 0xC0, 0x36, 0x4D, 0xEE,
+0x4B, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEE,
+0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA, 0x19, 0xD2,
+0xC5, 0xA3, 0x44, 0xA3, 0xA6, 0xA3, 0xC0, 0x36,
+0x4D, 0xEE, 0x47, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0x70, 0xA3, 0xAD, 0xEE, 0x00, 0xF6, 0x40, 0x32,
+0xCD, 0xEA, 0x1A, 0xD2, 0x5D, 0x67, 0x40, 0xF0,
+0x7D, 0xC2, 0x60, 0xF0, 0x4C, 0xA2, 0x9F, 0x67,
+0x4C, 0xEC, 0x5D, 0x67, 0x60, 0xF0, 0x8C, 0xC2,
+0xB1, 0x18, 0x0B, 0xEF, 0x0A, 0x04, 0x17, 0x22,
+0x05, 0xD2, 0x01, 0x6A, 0x04, 0xD2, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x40, 0x32, 0x40, 0x32, 0x60, 0x33, 0xA9, 0xF6,
+0x88, 0x9A, 0xA9, 0xF6, 0xAC, 0x9A, 0xF3, 0xF0,
+0x70, 0x9B, 0x80, 0xF2, 0x1A, 0x6F, 0x40, 0xEB,
+0x02, 0x6E, 0x04, 0x6A, 0x9F, 0x16, 0x5D, 0x67,
+0x20, 0xF0, 0x8D, 0xA2, 0x01, 0x6A, 0x02, 0x6E,
+0x92, 0x35, 0x20, 0xD5, 0x4C, 0xEC, 0x20, 0x02,
+0xA0, 0xA2, 0x20, 0xD4, 0xB1, 0x18, 0xC1, 0xED,
+0x80, 0xA2, 0x62, 0x67, 0x5D, 0x67, 0x20, 0xF0,
+0x8D, 0xA2, 0x1E, 0x23, 0x08, 0xD3, 0x02, 0x6B,
+0x07, 0xD3, 0x92, 0x33, 0x06, 0xD3, 0x01, 0x6B,
+0x6C, 0xEC, 0x04, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x05, 0xD4, 0xA9, 0xF6, 0xAC, 0x9B,
+0xA9, 0xF6, 0x88, 0x9B, 0x80, 0xF2, 0x1B, 0x6F,
+0x40, 0xEA, 0x02, 0x6E, 0x06, 0x6A, 0x6E, 0x16,
+0x92, 0x35, 0x01, 0x6A, 0x20, 0xD5, 0x4C, 0xEC,
+0x20, 0x02, 0xA0, 0xA2, 0x20, 0xD4, 0x80, 0xA2,
+0x01, 0x6F, 0x00, 0x6E, 0xB1, 0x18, 0x2E, 0xEE,
+0x21, 0xD3, 0x7D, 0x67, 0x20, 0xF0, 0x8D, 0xA3,
+0x20, 0x22, 0x09, 0xD2, 0x01, 0x6A, 0x08, 0xD2,
+0x92, 0x32, 0x06, 0xD2, 0x01, 0x6A, 0x4C, 0xEC,
+0x05, 0x6A, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6D,
+0xF7, 0xF0, 0x01, 0x6A, 0xA0, 0x35, 0x40, 0x32,
+0x40, 0x32, 0xA0, 0x35, 0x21, 0x93, 0xF3, 0xF0,
+0x10, 0x9D, 0x05, 0xD4, 0xA9, 0xF6, 0xAC, 0x9A,
+0xA9, 0xF6, 0x88, 0x9A, 0x07, 0xD3, 0x80, 0xF2,
+0x1C, 0x6F, 0x40, 0xE8, 0x02, 0x6E, 0x07, 0x6A,
+0x3D, 0x16, 0x0F, 0x6B, 0x92, 0x34, 0x6C, 0xEC,
+0x60, 0xA0, 0x10, 0x6D, 0xAB, 0xED, 0xAC, 0xEB,
+0x8D, 0xEB, 0x60, 0xC0, 0x81, 0xA1, 0x01, 0x6D,
+0xAC, 0xEC, 0xEE, 0x4D, 0x90, 0x34, 0xAC, 0xEB,
+0x8D, 0xEB, 0x60, 0xC0, 0x82, 0xA1, 0x1F, 0x6D,
+0xAC, 0xEB, 0x94, 0x34, 0x8D, 0xEB, 0x60, 0xC0,
+0x60, 0xA1, 0x01, 0x6C, 0x62, 0xC0, 0x61, 0xA0,
+0x8D, 0xEB, 0x61, 0xC0, 0x64, 0xA0, 0x8D, 0xEB,
+0x64, 0xC0, 0x1C, 0x16, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x1A, 0x2C, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0x04, 0xD4,
+0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6, 0x88, 0x9B,
+0xA9, 0xF6, 0xAC, 0x9B, 0xA0, 0xF2, 0x00, 0x6F,
+0x40, 0xEA, 0x02, 0x6E, 0x0D, 0x68, 0x50, 0x67,
+0x09, 0x97, 0x08, 0x91, 0x07, 0x90, 0x00, 0xEF,
+0x05, 0x63, 0x49, 0xA4, 0x0F, 0x68, 0xFF, 0x6F,
+0x0C, 0xEA, 0xEC, 0xEA, 0x04, 0x52, 0x17, 0x61,
+0x05, 0xD2, 0x01, 0x6A, 0x04, 0xD2, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x40, 0x32, 0x40, 0x32, 0x60, 0x33, 0xF3, 0xF0,
+0x70, 0x9B, 0xA9, 0xF6, 0x88, 0x9A, 0xA9, 0xF6,
+0xAC, 0x9A, 0xA0, 0xF2, 0x01, 0x6F, 0x40, 0xEB,
+0x02, 0x6E, 0x01, 0x68, 0xDC, 0x17, 0x69, 0xA4,
+0x72, 0x33, 0xEC, 0xEB, 0x02, 0x53, 0x17, 0x61,
+0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0xA9, 0xF6, 0x88, 0x9B, 0xA9, 0xF6,
+0xAC, 0x9B, 0xA0, 0xF2, 0x02, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x02, 0x68, 0xC0, 0x17, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x15, 0xF0,
+0x08, 0x4B, 0xC5, 0xA3, 0xA4, 0xA3, 0x26, 0xA3,
+0xC0, 0x36, 0xAD, 0xEE, 0xA7, 0xA3, 0x20, 0x31,
+0x20, 0x31, 0x2D, 0xEE, 0x00, 0xF6, 0xA0, 0x35,
+0xCD, 0xED, 0x20, 0xF0, 0x3D, 0xA5, 0x20, 0xF0,
+0xDC, 0xA5, 0x20, 0xF0, 0x7E, 0xA5, 0x20, 0x31,
+0xCD, 0xE9, 0x60, 0x33, 0x20, 0xF0, 0xDF, 0xA5,
+0x60, 0x33, 0x6D, 0xE9, 0x44, 0x33, 0x00, 0xF6,
+0xC0, 0x36, 0x4D, 0xE3, 0x68, 0x33, 0x2D, 0xEE,
+0x79, 0xE6, 0xAA, 0xA4, 0x61, 0xA6, 0x01, 0x6E,
+0xAC, 0xE8, 0xCC, 0xEB, 0xEC, 0xE8, 0xEC, 0xEB,
+0x18, 0x28, 0x2F, 0x23, 0x05, 0xD2, 0x01, 0x6A,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x40, 0x32, 0x40, 0x32,
+0x60, 0x33, 0xF3, 0xF0, 0x70, 0x9B, 0xA9, 0xF6,
+0x88, 0x9A, 0xA9, 0xF6, 0xAC, 0x9A, 0xA0, 0xF2,
+0x03, 0x6F, 0x40, 0xEB, 0x02, 0x6E, 0x09, 0x68,
+0x7A, 0x17, 0x17, 0x2B, 0x05, 0xD2, 0x01, 0x6A,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x40, 0x32, 0x40, 0x32,
+0x60, 0x33, 0xF3, 0xF0, 0x70, 0x9B, 0xA9, 0xF6,
+0x88, 0x9A, 0xA9, 0xF6, 0xAC, 0x9A, 0xA0, 0xF2,
+0x04, 0x6F, 0x40, 0xEB, 0x02, 0x6E, 0x08, 0x68,
+0x62, 0x17, 0x0F, 0x6A, 0xAC, 0xEA, 0x01, 0x72,
+0x08, 0x4C, 0x38, 0x60, 0x1B, 0x22, 0x02, 0x72,
+0x47, 0x60, 0x03, 0x72, 0x57, 0x60, 0x05, 0xD2,
+0x01, 0x6A, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33, 0x40, 0x32,
+0x40, 0x32, 0x60, 0x33, 0xF3, 0xF0, 0x70, 0x9B,
+0xA9, 0xF6, 0x88, 0x9A, 0xA9, 0xF6, 0xAC, 0x9A,
+0xA0, 0xF2, 0x09, 0x6F, 0x40, 0xEB, 0x02, 0x6E,
+0x0C, 0x68, 0x41, 0x17, 0xD1, 0x18, 0x2E, 0x4A,
+0x00, 0x65, 0x02, 0x67, 0x3F, 0xF7, 0x1B, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0x01, 0x6B, 0x05, 0xD0,
+0x04, 0xD3, 0xA0, 0xF2, 0x05, 0x6F, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xA9, 0xF6,
+0x88, 0x9B, 0xA9, 0xF6, 0xAC, 0x9B, 0x40, 0xEA,
+0x02, 0x6E, 0x25, 0x17, 0xD1, 0x18, 0x8A, 0x48,
+0x00, 0x65, 0x02, 0x67, 0x1F, 0xF7, 0x1F, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x01, 0x6B, 0xF3, 0xF0, 0x50, 0x9A, 0x05, 0xD0,
+0x04, 0xD3, 0xA0, 0xF2, 0x06, 0x6F, 0xE3, 0x17,
+0xD1, 0x18, 0x1F, 0x49, 0x00, 0x65, 0x02, 0x67,
+0x1F, 0xF7, 0x0D, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x01, 0x6B, 0xF3, 0xF0,
+0x50, 0x9A, 0x05, 0xD0, 0x04, 0xD3, 0xA0, 0xF2,
+0x07, 0x6F, 0xD1, 0x17, 0xD1, 0x18, 0xD6, 0x49,
+0x00, 0x65, 0x02, 0x67, 0xFF, 0xF6, 0x1B, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x01, 0x6B, 0xF3, 0xF0, 0x50, 0x9A, 0x05, 0xD0,
+0x04, 0xD3, 0xA0, 0xF2, 0x08, 0x6F, 0xBF, 0x17,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0x64, 0x67, 0x19, 0x2C, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32, 0x60, 0x33,
+0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0x04, 0xD4, 0xA9, 0xF6, 0xAC, 0x9B, 0xA9, 0xF6,
+0x88, 0x9B, 0xA0, 0xF2, 0x00, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x0D, 0x6A, 0x17, 0x97, 0x16, 0x91,
+0x15, 0x90, 0x00, 0xEF, 0x0C, 0x63, 0x48, 0xA4,
+0x0F, 0x69, 0xFF, 0x6D, 0x2C, 0xEA, 0xAC, 0xEA,
+0x04, 0x52, 0x17, 0x61, 0x05, 0xD2, 0x01, 0x6A,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x40, 0x32, 0x40, 0x32,
+0x60, 0x33, 0xA9, 0xF6, 0x88, 0x9A, 0xA9, 0xF6,
+0xAC, 0x9A, 0xF3, 0xF0, 0x70, 0x9B, 0xA0, 0xF2,
+0x0A, 0x6F, 0x40, 0xEB, 0x02, 0x6E, 0x01, 0x6A,
+0xDD, 0x17, 0x88, 0xA4, 0x92, 0x36, 0xAC, 0xEE,
+0x03, 0x56, 0x17, 0x61, 0x01, 0x6B, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6, 0x88, 0x9B,
+0xA9, 0xF6, 0xAC, 0x9B, 0x05, 0xD6, 0xA0, 0xF2,
+0x0B, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0x0E, 0x6A,
+0xC1, 0x17, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0xE0, 0x37, 0x15, 0xF0, 0x08, 0x4F, 0x05, 0xA7,
+0xC4, 0xA7, 0x00, 0x30, 0xCD, 0xE8, 0x18, 0x65,
+0x06, 0xA7, 0xD8, 0x67, 0x00, 0x30, 0x00, 0x30,
+0xCD, 0xE8, 0xC7, 0xA7, 0x00, 0xF6, 0xC0, 0x36,
+0x0D, 0xEE, 0x20, 0xF0, 0xFD, 0xA6, 0x20, 0xF0,
+0x1C, 0xA6, 0xE0, 0x37, 0x0D, 0xEF, 0x1F, 0x65,
+0x20, 0xF0, 0xFE, 0xA6, 0x18, 0x67, 0xE0, 0x37,
+0xE0, 0x37, 0x0D, 0xEF, 0x20, 0xF0, 0x1F, 0xA6,
+0x44, 0x36, 0x59, 0xE6, 0x00, 0xF6, 0x00, 0x30,
+0xED, 0xE8, 0xC8, 0x36, 0xC1, 0xE0, 0xC1, 0xA0,
+0x01, 0x6F, 0xEC, 0xEE, 0xAC, 0xEE, 0x17, 0x2E,
+0x05, 0xD2, 0x01, 0x6A, 0x04, 0xD2, 0xF7, 0xF0,
+0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A, 0x60, 0x33,
+0x40, 0x32, 0x40, 0x32, 0x60, 0x33, 0xA9, 0xF6,
+0x88, 0x9A, 0xA9, 0xF6, 0xAC, 0x9A, 0xF3, 0xF0,
+0x70, 0x9B, 0xA0, 0xF2, 0x04, 0x6F, 0x40, 0xEB,
+0x02, 0x6E, 0x08, 0x6A, 0x7B, 0x17, 0x10, 0x6A,
+0x4B, 0xEA, 0x4C, 0xEC, 0xAC, 0xEC, 0x20, 0x74,
+0x25, 0x61, 0x90, 0x67, 0x0D, 0xD5, 0xD1, 0x18,
+0x8C, 0x49, 0x0C, 0xD3, 0x0C, 0x93, 0x0D, 0x95,
+0x7F, 0xF7, 0x0C, 0x22, 0x06, 0xD2, 0x68, 0xA3,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x6C, 0xE9,
+0x02, 0x6B, 0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x80, 0x34, 0x2C, 0xED,
+0xF3, 0xF0, 0x10, 0x9C, 0x05, 0xD5, 0xA9, 0xF6,
+0x88, 0x9B, 0xA9, 0xF6, 0xAC, 0x9B, 0xA0, 0xF2,
+0x0D, 0x6F, 0x02, 0x6E, 0x40, 0xE8, 0x0C, 0xD2,
+0x0C, 0x92, 0x50, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x23, 0x67, 0x0F, 0xD2, 0x47, 0x41,
+0x05, 0x4A, 0x10, 0xD2, 0x00, 0x6A, 0x71, 0x67,
+0x0C, 0xD2, 0x4A, 0xA3, 0x0C, 0x94, 0x43, 0xEC,
+0x0A, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA,
+0x0F, 0x94, 0x00, 0x6A, 0x33, 0x17, 0x0C, 0x92,
+0x10, 0x94, 0x48, 0x32, 0x49, 0xE4, 0x0E, 0xD2,
+0x40, 0x9A, 0x06, 0x2A, 0x0C, 0x94, 0xFF, 0x6A,
+0x01, 0x4C, 0x4C, 0xEC, 0x0C, 0xD4, 0xE5, 0x17,
+0x0C, 0x92, 0xFF, 0x6C, 0x54, 0x35, 0x8C, 0xED,
+0x00, 0x6A, 0x11, 0xD5, 0x0D, 0xD2, 0x0D, 0x92,
+0x01, 0x69, 0xB1, 0x67, 0xA4, 0xEA, 0x0E, 0x92,
+0x0D, 0x94, 0xFF, 0x6F, 0x40, 0x9A, 0xEC, 0xEC,
+0x4C, 0xED, 0x2E, 0x25, 0x11, 0x92, 0xA9, 0xA3,
+0x1F, 0x6E, 0xB5, 0xE2, 0xB1, 0xE4, 0xA9, 0xA0,
+0x48, 0xA0, 0xEC, 0xEC, 0xA0, 0x35, 0x4D, 0xED,
+0x3D, 0x65, 0xAA, 0xA0, 0x59, 0x67, 0x8C, 0xEE,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEA, 0xAB, 0xA0,
+0xEC, 0xEE, 0xFE, 0x65, 0x00, 0xF6, 0xA0, 0x35,
+0x4D, 0xED, 0x96, 0x32, 0x48, 0x32, 0x49, 0xE5,
+0xA8, 0xA3, 0x3D, 0x65, 0x10, 0x6D, 0xAB, 0xED,
+0xD9, 0x67, 0xAC, 0xEE, 0xA6, 0x67, 0xEC, 0xED,
+0x11, 0x2D, 0x80, 0x9A, 0xBF, 0x67, 0x24, 0xED,
+0xA4, 0x67, 0x2C, 0xED, 0x05, 0x2D, 0x2D, 0xEC,
+0x80, 0xDA, 0x86, 0xA0, 0x01, 0x4C, 0x86, 0xC0,
+0x0D, 0x92, 0x01, 0x4A, 0x20, 0x72, 0x0D, 0xD2,
+0xC2, 0x61, 0xB4, 0x17, 0x10, 0x75, 0xF8, 0x61,
+0xBF, 0x67, 0x24, 0xED, 0xA0, 0x9A, 0x2C, 0xED,
+0xF3, 0x25, 0x1A, 0x6F, 0x00, 0x6E, 0x00, 0x6D,
+0x13, 0xD3, 0xB1, 0x18, 0x88, 0xE8, 0x12, 0xD2,
+0x12, 0x92, 0x2F, 0xE9, 0x60, 0x9A, 0x6C, 0xE9,
+0x20, 0xDA, 0x86, 0xA0, 0x13, 0x93, 0x02, 0x24,
+0xFF, 0x4C, 0xE1, 0x17, 0x0B, 0xD4, 0x5D, 0x67,
+0x07, 0x6C, 0x20, 0xF0, 0x80, 0xC2, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0xCA, 0xF2,
+0x00, 0x4C, 0x09, 0xD4, 0xE0, 0xF1, 0x13, 0x6C,
+0x0A, 0xD4, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x12, 0xF1, 0xB8, 0x9C, 0x23, 0x67,
+0x40, 0xED, 0x08, 0x04, 0x71, 0x67, 0xC8, 0x17,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x00, 0x6D, 0xF7, 0xF0,
+0x01, 0x68, 0x08, 0xD5, 0x00, 0x30, 0x0B, 0xD5,
+0x40, 0xEA, 0x00, 0x30, 0x0A, 0xD2, 0x6D, 0xF1,
+0x40, 0x98, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0xF0, 0x62, 0xA2, 0x20, 0x31, 0x0B, 0x95,
+0x09, 0x2B, 0x72, 0xF0, 0x54, 0x99, 0x40, 0xEA,
+0x0A, 0x94, 0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90,
+0x00, 0xEF, 0x08, 0x63, 0x20, 0xF0, 0x60, 0xA2,
+0x00, 0x6F, 0x68, 0x33, 0x69, 0xE2, 0x81, 0xA2,
+0x60, 0xA2, 0xC2, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x63, 0xA2, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x01, 0x6A,
+0x0B, 0xD3, 0x7D, 0x67, 0x5A, 0xC3, 0x59, 0xC3,
+0x02, 0x6A, 0x58, 0xC3, 0x0C, 0x6A, 0x5B, 0xC3,
+0x5D, 0x67, 0xBC, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A,
+0x08, 0x06, 0x04, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0x0B, 0x93, 0x1E, 0x22, 0x00, 0x6B, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xF3, 0xF0, 0x50, 0x9A, 0xA9, 0xF6, 0x90, 0x9B,
+0xA9, 0xF6, 0xB4, 0x9B, 0x00, 0xF3, 0x10, 0x6F,
+0x40, 0xEA, 0x02, 0x6E, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x4D, 0xF1, 0x5C, 0xA3,
+0xFF, 0x4A, 0x4D, 0xF1, 0x5C, 0xC3, 0xB1, 0x17,
+0x08, 0x94, 0x0E, 0x24, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x73, 0xF6, 0x5C, 0x9A,
+0xA3, 0x67, 0x40, 0xEA, 0x04, 0x6E, 0x08, 0x95,
+0xD1, 0x18, 0xA8, 0x19, 0x06, 0x04, 0xD2, 0x2A,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF4, 0xD0, 0x9A, 0x6D, 0xF1, 0x40, 0x98,
+0x20, 0xF0, 0x60, 0xA2, 0x68, 0x33, 0x69, 0xE2,
+0x81, 0xA2, 0x60, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x62, 0xA2, 0x60, 0x33, 0x60, 0x33, 0x8D, 0xEB,
+0x83, 0xA2, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xEE,
+0x6D, 0xEC, 0x6D, 0xF1, 0x60, 0x98, 0x07, 0x6C,
+0x20, 0xF0, 0x40, 0xA3, 0x01, 0x4A, 0x8C, 0xEA,
+0x20, 0xF0, 0x40, 0xC3, 0x20, 0xF0, 0x42, 0xA3,
+0xFF, 0x6C, 0xFF, 0x4A, 0x8C, 0xEA, 0x20, 0xF0,
+0x42, 0xC3, 0x08, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x0A, 0x6B, 0x40, 0x32, 0x4D, 0xF1,
+0x7C, 0xC2, 0x6F, 0x17, 0xF7, 0xF0, 0x01, 0x6B,
+0xF7, 0xF0, 0x01, 0x6C, 0x60, 0x33, 0x80, 0x34,
+0x60, 0x33, 0x80, 0x34, 0xD2, 0xF1, 0x74, 0x9B,
+0x6D, 0xF1, 0x84, 0x9C, 0x04, 0xD2, 0x00, 0x6F,
+0x00, 0x6E, 0x40, 0xEB, 0x03, 0x6D, 0x5D, 0x17,
+0xF3, 0x63, 0x19, 0x62, 0x18, 0xD1, 0x17, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0, 0x01, 0x6A,
+0x60, 0x33, 0x40, 0x32, 0x60, 0x33, 0xA9, 0xF3,
+0x70, 0x9B, 0x40, 0x32, 0x72, 0xF4, 0x54, 0x9A,
+0x00, 0x69, 0x0E, 0xD4, 0x1B, 0xD5, 0x0C, 0xD1,
+0xA3, 0x67, 0x04, 0x6C, 0x40, 0xEA, 0x11, 0xD3,
+0x02, 0x67, 0x0E, 0x97, 0x37, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0xA9, 0xF6, 0x90, 0x9B, 0xA9, 0xF6,
+0xB4, 0x9B, 0x04, 0xD0, 0x00, 0xF3, 0x13, 0x6F,
+0x40, 0xEA, 0x02, 0x6E, 0x06, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xCA, 0xF2, 0x14, 0x4A, 0x07, 0xD2,
+0x09, 0xD0, 0x5A, 0x6A, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0x10, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0x04, 0x6A, 0x19, 0x97, 0x18, 0x91, 0x17, 0x90,
+0x00, 0xEF, 0x0D, 0x63, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x73, 0xF6, 0x5C, 0x9B,
+0xA7, 0x67, 0x04, 0x6E, 0x90, 0x67, 0x40, 0xEA,
+0x0E, 0xD3, 0x7D, 0x67, 0x01, 0x6A, 0x20, 0xF0,
+0x4A, 0xC3, 0x20, 0xF0, 0x49, 0xC3, 0x02, 0x6A,
+0x20, 0xF0, 0x48, 0xC3, 0x0C, 0x6A, 0x20, 0xF0,
+0x4B, 0xC3, 0x5D, 0x67, 0x20, 0xF0, 0x2C, 0xC2,
+0x08, 0x02, 0x40, 0xF0, 0x4C, 0xA2, 0x00, 0x6F,
+0x0C, 0x06, 0x42, 0xC0, 0x1B, 0x92, 0x04, 0x6D,
+0x00, 0x6C, 0x42, 0x32, 0x43, 0xC0, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x20, 0xF1,
+0x00, 0x2A, 0x0C, 0x94, 0x00, 0xF1, 0x1D, 0x24,
+0x0E, 0x93, 0x04, 0x6E, 0xB0, 0x67, 0x73, 0xF6,
+0x5C, 0x9B, 0x40, 0xEA, 0x00, 0x65, 0x0C, 0x95,
+0xD1, 0x18, 0xA8, 0x19, 0x0A, 0x04, 0x0F, 0xD2,
+0x00, 0xF1, 0x0F, 0x22, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xF7, 0xF0, 0x01, 0x69,
+0xA9, 0xF6, 0x90, 0x9B, 0xA9, 0xF6, 0xB4, 0x9B,
+0x20, 0x31, 0x0F, 0x93, 0x20, 0x31, 0xF3, 0xF0,
+0x50, 0x99, 0x05, 0xD3, 0x01, 0x6B, 0x04, 0xD3,
+0x02, 0x6E, 0x00, 0xF3, 0x11, 0x6F, 0x12, 0xD4,
+0x40, 0xEA, 0x13, 0xD5, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6E,
+0xC0, 0x36, 0xC0, 0x36, 0x6D, 0xF1, 0x60, 0x9E,
+0x0E, 0xD2, 0x10, 0xD6, 0x3E, 0x2B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF4,
+0x54, 0x9A, 0x11, 0x95, 0x23, 0x6C, 0x15, 0xD3,
+0x40, 0xEA, 0x14, 0xD6, 0x14, 0x96, 0x15, 0x93,
+0x6D, 0xF1, 0x40, 0xDE, 0x27, 0x2A, 0xF3, 0xF0,
+0x70, 0x99, 0x13, 0x95, 0x12, 0x94, 0x00, 0xF3,
+0x13, 0x6F, 0x02, 0x6E, 0x04, 0xD2, 0x40, 0xEB,
+0x22, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x74, 0x9A, 0x40, 0xEB,
+0x0E, 0x94, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A, 0x09, 0xD1,
+0x40, 0xEA, 0x90, 0x67, 0x06, 0x6A, 0x7D, 0x67,
+0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xCA, 0xF2, 0x14, 0x4A, 0x07, 0xD2,
+0x80, 0x6A, 0x48, 0x17, 0x07, 0x6C, 0x20, 0xF0,
+0x60, 0xC2, 0x20, 0xF0, 0x81, 0xC2, 0x20, 0xF0,
+0x62, 0xC2, 0x10, 0x92, 0x6D, 0xF1, 0x60, 0x9A,
+0x20, 0xF0, 0x82, 0xA3, 0x08, 0x74, 0x11, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF4, 0x50, 0x9A, 0x40, 0xEA, 0x90, 0x67,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x0E, 0x94,
+0x3B, 0x17, 0x20, 0xF0, 0x41, 0xA3, 0x07, 0x6D,
+0xFF, 0x69, 0x01, 0x4A, 0xAC, 0xEA, 0x2C, 0xEA,
+0x20, 0xF0, 0x41, 0xC3, 0x48, 0x32, 0x49, 0xE3,
+0x02, 0x35, 0xA1, 0xC2, 0xA2, 0x35, 0xA2, 0xC2,
+0x01, 0x4C, 0x00, 0xF6, 0x02, 0x35, 0x00, 0xC2,
+0xA3, 0xC2, 0x20, 0xF0, 0x82, 0xC3, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xF1,
+0x44, 0x9B, 0x11, 0xD3, 0x2F, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1,
+0x58, 0x9A, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x3A, 0x65, 0x97, 0xF0, 0x19, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xE6, 0xF1, 0x01, 0x4A, 0x80, 0x34,
+0x04, 0xD2, 0x01, 0x6F, 0x01, 0x6E, 0x0A, 0x6D,
+0xEA, 0xF2, 0x0C, 0x4C, 0x59, 0x67, 0x40, 0xEA,
+0x12, 0xD3, 0x12, 0x93, 0x6D, 0xF1, 0x44, 0xDB,
+0x11, 0x2A, 0x10, 0x92, 0x07, 0x6B, 0x6D, 0xF1,
+0x40, 0x9A, 0x20, 0xF0, 0x81, 0xA2, 0x02, 0x24,
+0x6F, 0x44, 0x2C, 0xEB, 0x20, 0xF0, 0x61, 0xC2,
+0x20, 0xF0, 0x62, 0xA2, 0xFF, 0x4B, 0x20, 0xF0,
+0x62, 0xC2, 0xA2, 0x17, 0x11, 0x93, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB2, 0xF1,
+0x44, 0x9A, 0x6D, 0xF1, 0x84, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x12, 0xD2, 0x2C, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x0A, 0x6C, 0x40, 0x32,
+0x4D, 0xF1, 0x9C, 0xC2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD2, 0xF1, 0x34, 0x9A,
+0x11, 0x92, 0x6D, 0xF1, 0x84, 0x9A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x92, 0xF2,
+0x4C, 0x9A, 0x40, 0xEA, 0x11, 0xD4, 0x12, 0x95,
+0x11, 0x94, 0x00, 0x6F, 0x04, 0xD5, 0xC2, 0x67,
+0x40, 0xE9, 0x01, 0x6D, 0x01, 0x72, 0x0B, 0x60,
+0x10, 0x92, 0x07, 0x6B, 0x6D, 0xF1, 0x40, 0x9A,
+0x20, 0xF0, 0x81, 0xA2, 0xC3, 0x24, 0x6F, 0x44,
+0xFF, 0x6C, 0x8C, 0xEB, 0xBF, 0x17, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x0E, 0x94, 0x0F, 0x92,
+0xAC, 0x16, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A, 0x40, 0xEA,
+0x90, 0x67, 0x00, 0x6A, 0xA2, 0x16, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x04, 0x67, 0x40, 0xEA,
+0x05, 0xD5, 0x20, 0x98, 0x90, 0x67, 0x04, 0xD2,
+0x2A, 0xE8, 0x05, 0x95, 0x00, 0x68, 0x2C, 0x60,
+0x6E, 0xA1, 0xAE, 0xEB, 0x37, 0x2B, 0x0D, 0xA1,
+0x6C, 0xA1, 0xA9, 0xA1, 0x88, 0xA1, 0x00, 0x30,
+0x6D, 0xE8, 0x6A, 0xA1, 0xA0, 0x35, 0x8D, 0xED,
+0x8B, 0xA1, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0xAD, 0xEC, 0x60, 0x33, 0x05, 0x24,
+0x72, 0xF4, 0xB0, 0x9B, 0x40, 0xED, 0x05, 0xD3,
+0x05, 0x93, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x13, 0xF6, 0xBC, 0x9C, 0x91, 0x67,
+0x40, 0xED, 0x05, 0xD3, 0x05, 0x93, 0x91, 0x67,
+0x72, 0xF4, 0x70, 0x9B, 0x40, 0xEB, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x04, 0x94,
+0x50, 0x67, 0x09, 0x97, 0x08, 0x91, 0x07, 0x90,
+0x00, 0xEF, 0x05, 0x63, 0x20, 0x99, 0x2A, 0xEC,
+0xC3, 0x61, 0x00, 0x68, 0xED, 0x17, 0x00, 0x65,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD1, 0x11, 0xD0,
+0x40, 0xA4, 0x07, 0x6B, 0x04, 0x67, 0x0D, 0xD2,
+0x41, 0xA4, 0x6C, 0xEA, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x0C, 0xD2, 0x15, 0xF0,
+0x08, 0x4B, 0xA5, 0xA3, 0x84, 0xA3, 0xC6, 0xA3,
+0xA0, 0x35, 0x8D, 0xED, 0x87, 0xA3, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x80, 0x34,
+0xAD, 0xEC, 0x20, 0xF0, 0xA9, 0xA4, 0x20, 0xF0,
+0x68, 0xA4, 0x20, 0xF0, 0xCA, 0xA4, 0xA0, 0x35,
+0x6D, 0xED, 0x20, 0xF0, 0x6B, 0xA4, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0xA1, 0xA3, 0x82, 0xA3, 0xC0, 0xA3,
+0x63, 0xA3, 0x0C, 0x02, 0xA0, 0x35, 0x80, 0x34,
+0x40, 0xA2, 0xAD, 0xEE, 0x80, 0x34, 0xCD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x0C, 0xD3,
+0x01, 0x72, 0x00, 0x6B, 0x0A, 0xD3, 0x09, 0x60,
+0x1A, 0x22, 0x02, 0x72, 0xE0, 0xF0, 0x0C, 0x60,
+0x00, 0x6D, 0xD1, 0x18, 0xD8, 0x4C, 0x90, 0x67,
+0x14, 0x11, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x0D, 0x95, 0xF2, 0xF7, 0x50, 0x9A,
+0x40, 0xEA, 0x0C, 0x94, 0xA2, 0x67, 0xD1, 0x18,
+0xD8, 0x4C, 0x90, 0x67, 0x13, 0x97, 0x12, 0x91,
+0x11, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0D, 0x95,
+0x0C, 0x94, 0xF2, 0xF7, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6F, 0x60, 0x33, 0x60, 0x33, 0xE0, 0x37,
+0xE0, 0x37, 0xA9, 0xF3, 0xD4, 0x9B, 0x72, 0xF4,
+0x54, 0x9F, 0x0F, 0x6C, 0xA6, 0x67, 0x0F, 0xD7,
+0x40, 0xEA, 0x0E, 0xD6, 0x22, 0x67, 0x0E, 0x96,
+0x0F, 0x97, 0x28, 0x2A, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6C, 0x40, 0x32, 0x80, 0x34,
+0x80, 0x34, 0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A,
+0xA9, 0xF6, 0xB4, 0x9C, 0xA9, 0xF6, 0x90, 0x9C,
+0x04, 0xD1, 0x00, 0xF3, 0x12, 0x6F, 0x40, 0xEA,
+0x02, 0x6E, 0x06, 0x6A, 0x7D, 0x67, 0x58, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xCA, 0xF2, 0x14, 0x4A, 0x07, 0xD2, 0x09, 0xD1,
+0xFD, 0x6A, 0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x06, 0x04, 0x4B, 0x10, 0x83, 0xA0, 0x42, 0xA0,
+0xA6, 0x67, 0x80, 0x34, 0x4D, 0xEC, 0x0E, 0xD4,
+0x0E, 0x02, 0x40, 0xA2, 0x4C, 0xC1, 0x82, 0x32,
+0x4D, 0xC1, 0x72, 0xF4, 0x54, 0x9F, 0x40, 0xEA,
+0x00, 0x65, 0x42, 0x34, 0x89, 0xC1, 0x82, 0x34,
+0x8A, 0xC1, 0x00, 0xF6, 0x42, 0x34, 0x48, 0xC1,
+0x8B, 0xC1, 0x3F, 0x2A, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xF3, 0xF0, 0x10, 0x9C,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xA9, 0xF6, 0xB4, 0x9C, 0xA9, 0xF6, 0x90, 0x9C,
+0x04, 0xD2, 0x00, 0xF3, 0x12, 0x6F, 0x02, 0x6E,
+0x40, 0xE8, 0x0C, 0xD2, 0x0C, 0x92, 0x7D, 0x67,
+0x06, 0x04, 0x09, 0xD2, 0x06, 0x6A, 0x58, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xCA, 0xF2, 0x14, 0x4A, 0x07, 0xD2, 0xFF, 0x6A,
+0x0F, 0x4A, 0x08, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A,
+0x91, 0x67, 0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6,
+0x50, 0x9A, 0x10, 0x6D, 0x40, 0xEA, 0x00, 0x6C,
+0x64, 0x10, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0xCD, 0xA1, 0x80, 0x34, 0x73, 0xF6, 0xFC, 0x9C,
+0x7D, 0x67, 0x8C, 0xA1, 0x20, 0xF0, 0x74, 0xA3,
+0xC0, 0x36, 0x8D, 0xEE, 0x6E, 0xC1, 0x0D, 0xD6,
+0x0D, 0x03, 0xC0, 0xAB, 0xA4, 0x40, 0x40, 0xEF,
+0x82, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x33, 0xF6, 0xC4, 0x9C, 0x0C, 0x95,
+0x91, 0x67, 0x40, 0xEE, 0x0D, 0xD2, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x0D, 0x92, 0x80, 0x34,
+0x72, 0xF0, 0xB4, 0x9C, 0x40, 0xED, 0x82, 0x67,
+0xAD, 0xA1, 0x4C, 0xA1, 0xA0, 0x35, 0x4D, 0xED,
+0x0C, 0xD5, 0x0C, 0x02, 0xA0, 0xAA, 0x23, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x0E, 0xD2, 0x0C, 0x92, 0x20, 0x9A, 0x0C, 0x92,
+0x4A, 0xE9, 0x02, 0x61, 0x00, 0x6A, 0x93, 0x10,
+0x4E, 0xA1, 0x0D, 0x93, 0x6E, 0xEA, 0x80, 0xF0,
+0x1A, 0x2A, 0xAD, 0xA1, 0xF7, 0xF0, 0x01, 0x6A,
+0x8C, 0xA1, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0xA0, 0x35, 0x8D, 0xED, 0x00, 0x6F,
+0x0A, 0x06, 0x04, 0x4D, 0x40, 0xEA, 0x00, 0x6C,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x06, 0x22, 0x72, 0xF0, 0x54, 0x9B, 0x40, 0xEA,
+0x0E, 0x94, 0x04, 0x6A, 0xF7, 0x16, 0x0A, 0x92,
+0x2A, 0x2A, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6C, 0x60, 0x33, 0x80, 0x34, 0x80, 0x34,
+0x60, 0x33, 0xA9, 0xF6, 0xB4, 0x9C, 0xF3, 0xF0,
+0x70, 0x9B, 0xA9, 0xF6, 0x90, 0x9C, 0x04, 0xD2,
+0x00, 0xF3, 0x12, 0x6F, 0x02, 0x6E, 0x40, 0xEB,
+0x02, 0x67, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x74, 0x9A, 0x40, 0xEB,
+0x0E, 0x94, 0x06, 0x6A, 0x7D, 0x67, 0x58, 0xC3,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xCA, 0xF2, 0x14, 0x4A, 0x07, 0xD2, 0xFF, 0x6A,
+0x09, 0xD0, 0x33, 0x4A, 0x0E, 0x17, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0x73, 0xF6,
+0x7C, 0x9F, 0x04, 0x6E, 0xB0, 0x67, 0x82, 0x67,
+0x40, 0xEB, 0x0C, 0xD7, 0xCD, 0xA1, 0x4C, 0xA1,
+0xA9, 0xA1, 0xC0, 0x36, 0x4D, 0xEE, 0x48, 0xA1,
+0xA0, 0x35, 0x0C, 0x97, 0x4D, 0xED, 0x4A, 0xA1,
+0x0C, 0x03, 0x73, 0xF6, 0xFC, 0x9F, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0xAB, 0xA1, 0x0C, 0xD6,
+0xC0, 0xAB, 0x00, 0xF6, 0xA0, 0x35, 0x4D, 0xED,
+0x0A, 0x92, 0x40, 0xEF, 0x84, 0x42, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x0E, 0x94, 0x4D, 0xA1,
+0x8C, 0xA1, 0x0A, 0x95, 0x40, 0x32, 0x8D, 0xEA,
+0x0C, 0xD2, 0x0C, 0x02, 0x40, 0xAA, 0x9D, 0x67,
+0x42, 0xC5, 0x42, 0x33, 0x0C, 0x4A, 0x0C, 0xD2,
+0x0C, 0x02, 0x63, 0xC5, 0x40, 0xAA, 0x01, 0x6B,
+0x7A, 0xC4, 0x79, 0xC4, 0x02, 0x6B, 0x78, 0xC4,
+0x7D, 0x67, 0x5B, 0xC3, 0x06, 0x04, 0x42, 0x32,
+0xD1, 0x18, 0xA8, 0x19, 0x5C, 0xC3, 0x0A, 0x93,
+0x9F, 0xF6, 0x00, 0x2B, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x54, 0x9A,
+0x40, 0xEA, 0x0E, 0x94, 0x65, 0x16, 0x20, 0x99,
+0x5A, 0x17, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0A, 0xD4, 0x15, 0xF0,
+0x08, 0x4A, 0xE5, 0xA2, 0x65, 0x67, 0xA4, 0xA2,
+0x06, 0xA2, 0xE0, 0x37, 0xAD, 0xEF, 0xA7, 0xA2,
+0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF, 0x00, 0xF6,
+0xA0, 0x35, 0xED, 0xED, 0x20, 0xF0, 0xE9, 0xA5,
+0x20, 0xF0, 0x48, 0xA5, 0x20, 0xF0, 0x0A, 0xA5,
+0xE0, 0x37, 0x4D, 0xEF, 0x20, 0xF0, 0x4B, 0xA5,
+0x00, 0x30, 0x00, 0x30, 0x0D, 0xEF, 0x00, 0xF6,
+0x40, 0x32, 0xED, 0xEA, 0xE1, 0xA2, 0xA0, 0xA2,
+0x02, 0xA2, 0xE0, 0x37, 0xAD, 0xEF, 0xA3, 0xA2,
+0x00, 0x6A, 0x40, 0xCB, 0x00, 0x6A, 0x40, 0xDE,
+0xF7, 0xF0, 0x01, 0x6A, 0x00, 0x30, 0x40, 0x32,
+0x00, 0x30, 0x40, 0x32, 0x0D, 0xEF, 0x00, 0xF6,
+0xA0, 0x35, 0x72, 0xF0, 0x50, 0x9A, 0xED, 0xED,
+0x05, 0x67, 0x09, 0xD3, 0x40, 0xEA, 0x08, 0xD6,
+0xB0, 0x67, 0x00, 0x98, 0x08, 0x96, 0x09, 0x93,
+0x0A, 0x94, 0x22, 0x67, 0x0A, 0xED, 0x6C, 0x60,
+0x4E, 0xA0, 0x8E, 0xEA, 0x66, 0x2A, 0x09, 0xD6,
+0x8D, 0xA0, 0x4C, 0xA0, 0xF7, 0xF0, 0x01, 0x6D,
+0x80, 0x34, 0x4D, 0xEC, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0xA0, 0x35, 0x40, 0x32, 0xA0, 0x35,
+0x72, 0xF4, 0x54, 0x9A, 0xA9, 0xF3, 0xB8, 0x9D,
+0x80, 0xCB, 0x40, 0xEA, 0x08, 0xD3, 0x09, 0x96,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0x40, 0xDE,
+0xE0, 0x37, 0x08, 0x93, 0x26, 0x2A, 0x72, 0xF0,
+0x74, 0x9F, 0x91, 0x67, 0x40, 0xEB, 0x02, 0x67,
+0x06, 0x6A, 0x7D, 0x67, 0x50, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xCA, 0xF2,
+0x14, 0x4A, 0x05, 0xD2, 0x80, 0xF1, 0x08, 0x6A,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x04, 0x04,
+0x40, 0xEA, 0x07, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A,
+0x10, 0x6D, 0x40, 0xEA, 0x00, 0x6C, 0x00, 0x6A,
+0x1F, 0x10, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0x73, 0xF6, 0x9C, 0x9C, 0x08, 0xD7,
+0xA9, 0xA0, 0x1C, 0x65, 0x88, 0xA0, 0xA0, 0x35,
+0x8D, 0xED, 0x8A, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0xAD, 0xEC, 0xAB, 0xA0, 0x00, 0xF6, 0xA0, 0x35,
+0x8D, 0xED, 0x82, 0x67, 0x58, 0x67, 0x40, 0xEA,
+0xC0, 0xAB, 0x08, 0x97, 0x91, 0x67, 0x72, 0xF0,
+0x54, 0x9F, 0x40, 0xEA, 0x00, 0x65, 0x01, 0x6A,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0x00, 0x98, 0x0A, 0xED, 0x93, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x91, 0x67,
+0xCE, 0x17, 0x00, 0x65, 0xFB, 0x63, 0x09, 0x62,
+0x08, 0xD1, 0x07, 0xD0, 0x61, 0xA4, 0x00, 0xA4,
+0x42, 0xA4, 0x60, 0x33, 0x0D, 0xEB, 0x03, 0xA4,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x03, 0x6B, 0x0C, 0xEB,
+0xA0, 0xF0, 0x18, 0x23, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A,
+0x08, 0xF0, 0x00, 0x6D, 0x40, 0xEA, 0x02, 0x6C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0x6D, 0xF1, 0x68, 0x9C, 0xFF, 0x4B, 0x6D, 0xF1,
+0x68, 0xDC, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x72, 0xF0, 0x74, 0x9B, 0x40, 0xEB,
+0x82, 0x67, 0x00, 0x6A, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0xA5, 0xA1,
+0x84, 0xA1, 0x46, 0xA1, 0xA0, 0x35, 0x8D, 0xED,
+0x87, 0xA1, 0x40, 0x32, 0x40, 0x32, 0x4D, 0xED,
+0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC, 0x68, 0x32,
+0x51, 0xE4, 0x41, 0xA4, 0x17, 0x6D, 0xAC, 0xEA,
+0x10, 0x72, 0x13, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x53, 0xF5, 0x54, 0x9A,
+0x40, 0xEA, 0x04, 0xD3, 0x04, 0x93, 0x09, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x02, 0xF0, 0x00, 0x6D,
+0xB9, 0x17, 0x01, 0x4B, 0x0A, 0xEB, 0xD7, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x50, 0x9A, 0xF7, 0xF0, 0x01, 0x69,
+0xF7, 0xF0, 0x01, 0x68, 0x40, 0xEA, 0x20, 0x31,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0x15, 0xF0, 0x89, 0xA5, 0x15, 0xF0, 0x68, 0xA5,
+0x04, 0xD2, 0x15, 0xF0, 0x4A, 0xA5, 0x80, 0x34,
+0x6D, 0xEC, 0x15, 0xF0, 0x6B, 0xA5, 0x40, 0x32,
+0x40, 0x32, 0x4D, 0xEC, 0x00, 0xF6, 0x60, 0x33,
+0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3, 0xA2, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0xA0, 0xF3, 0x6D, 0xA2, 0xA0, 0xF3,
+0x8E, 0xA2, 0xA0, 0xF3, 0xAC, 0xA2, 0xA0, 0xF3,
+0x4F, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xED,
+0x80, 0x34, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x50, 0xA2, 0x01, 0x6B, 0x00, 0x30,
+0x6C, 0xEA, 0x20, 0x31, 0x00, 0x30, 0x25, 0x22,
+0x6D, 0xF1, 0x48, 0x99, 0x01, 0x72, 0x21, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF5, 0x40, 0x9A, 0x40, 0xEA, 0x00, 0x6C,
+0x18, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xD3, 0xF6, 0x50, 0x9A, 0x04, 0xF0,
+0x00, 0x6D, 0x40, 0xEA, 0x02, 0x6C, 0x6D, 0xF1,
+0x48, 0x99, 0x04, 0x94, 0xFF, 0x4A, 0x6D, 0xF1,
+0x48, 0xD9, 0x72, 0xF0, 0x54, 0x98, 0x40, 0xEA,
+0x00, 0x65, 0x6B, 0x17, 0x24, 0x67, 0x0A, 0x30,
+0x95, 0x17, 0x6D, 0xF1, 0x48, 0x99, 0x04, 0x94,
+0xFF, 0x4A, 0x6D, 0xF1, 0x48, 0xD9, 0x72, 0xF0,
+0x54, 0x98, 0x40, 0xEA, 0x00, 0x65, 0x01, 0x6A,
+0x5D, 0x17, 0x00, 0x65, 0x20, 0xE8, 0x01, 0x6A,
+0x20, 0xE8, 0x01, 0x6A, 0x20, 0xE8, 0x01, 0x6A,
+0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1, 0x09, 0xD0,
+0x00, 0x6A, 0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5, 0x48, 0x9A,
+0x24, 0x67, 0x05, 0x67, 0x00, 0x6F, 0xA4, 0x67,
+0x06, 0x06, 0x40, 0xEA, 0x00, 0x6C, 0x01, 0x6B,
+0x20, 0x2A, 0x06, 0x94, 0x1E, 0x24, 0x5D, 0x67,
+0x72, 0xC2, 0xBD, 0x67, 0x09, 0x6A, 0x51, 0xC5,
+0x5D, 0x67, 0x70, 0xC2, 0x47, 0x41, 0xFF, 0xF7,
+0x1F, 0x6B, 0x01, 0x4A, 0x6C, 0xEA, 0x7D, 0x67,
+0x53, 0xC3, 0x42, 0x32, 0x54, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xD1, 0x67, 0x40, 0xEA, 0xB0, 0x67,
+0x06, 0x95, 0xD1, 0x18, 0xA8, 0x19, 0x04, 0x04,
+0x62, 0x67, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x43, 0x67, 0x00, 0xEF, 0x06, 0x63, 0x00, 0x65,
+0x45, 0xA4, 0x64, 0xA4, 0xA6, 0xA4, 0x40, 0x32,
+0x4D, 0xEB, 0x47, 0xA4, 0xA0, 0x35, 0xA0, 0x35,
+0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0x4D, 0xED,
+0x3F, 0x6B, 0x43, 0xA5, 0x1B, 0x65, 0x78, 0x67,
+0x4C, 0xEB, 0xFF, 0x6C, 0x8C, 0xEB, 0x10, 0x53,
+0x05, 0x61, 0x37, 0x6B, 0x6C, 0xEA, 0x8C, 0xEA,
+0x12, 0x72, 0x09, 0x61, 0xFD, 0x63, 0x05, 0x62,
+0xD1, 0x18, 0x9A, 0x4F, 0x18, 0x6C, 0x05, 0x97,
+0x01, 0x6A, 0x00, 0xEF, 0x03, 0x63, 0x20, 0xE8,
+0x01, 0x6A, 0x00, 0x65, 0xFD, 0x63, 0x05, 0x62,
+0x04, 0xD0, 0x04, 0x67, 0x44, 0xA4, 0x85, 0xA4,
+0x80, 0x34, 0x4D, 0xEC, 0x46, 0xA0, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x87, 0xA0, 0x00, 0xF6,
+0x80, 0x34, 0xD1, 0x18, 0xB1, 0xDA, 0x4D, 0xEC,
+0x85, 0xA0, 0x44, 0xA0, 0x80, 0x34, 0x4D, 0xEC,
+0x46, 0xA0, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x87, 0xA0, 0x00, 0xF6, 0x80, 0x34, 0xD1, 0x18,
+0x23, 0x15, 0x4D, 0xEC, 0x05, 0x97, 0x04, 0x90,
+0x01, 0x6A, 0x00, 0xEF, 0x03, 0x63, 0x00, 0x65,
+0x20, 0xE8, 0x01, 0x6A, 0x20, 0xE8, 0x00, 0x65,
+0xF0, 0x63, 0x1F, 0x62, 0x1E, 0xD1, 0x1D, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xA9, 0xA2, 0x68, 0xA2,
+0x24, 0x67, 0x8A, 0xA2, 0xA0, 0x35, 0x6D, 0xED,
+0x6B, 0xA2, 0x80, 0x34, 0x80, 0x34, 0x8D, 0xED,
+0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB, 0xA9, 0xA3,
+0x8A, 0xA3, 0xC8, 0xA3, 0x6B, 0xA3, 0xA0, 0x35,
+0x80, 0x34, 0xAD, 0xEE, 0x80, 0x34, 0xCD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x12, 0xD3,
+0x00, 0x6B, 0x0F, 0xD3, 0x0E, 0xD3, 0x65, 0xA2,
+0x86, 0xA2, 0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x8D, 0xA2,
+0x6C, 0xA2, 0xAE, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x6F, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB, 0x8D, 0xA3,
+0x4C, 0xA3, 0xAE, 0xA3, 0x80, 0x34, 0x4D, 0xEC,
+0x4F, 0xA3, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x64, 0xA2,
+0x83, 0xA2, 0xA2, 0xA2, 0x45, 0xA2, 0x80, 0x34,
+0x60, 0x33, 0x8D, 0xED, 0x60, 0x33, 0xAD, 0xEB,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0x16, 0xD2,
+0x45, 0xA1, 0x04, 0xA1, 0x66, 0xA1, 0x40, 0x32,
+0x0D, 0xEA, 0x07, 0xA1, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEA, 0x00, 0xF6, 0x00, 0x30, 0x4D, 0xE8,
+0x4C, 0x40, 0x13, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x13, 0xF6, 0x4C, 0x9A,
+0x40, 0xEA, 0x8C, 0x40, 0x18, 0xD2, 0x43, 0x80,
+0x20, 0x6B, 0x00, 0x52, 0x01, 0x61, 0x10, 0x6B,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0xA3, 0x67, 0x00, 0x6F,
+0x0F, 0x06, 0x00, 0x6C, 0x40, 0xEA, 0x15, 0xD3,
+0x04, 0x6B, 0x14, 0xD3, 0x15, 0x93, 0x37, 0x2A,
+0x0F, 0x94, 0x02, 0x6A, 0x14, 0xD2, 0x33, 0x24,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x73, 0xF6, 0x5C, 0x9A, 0xC3, 0x67, 0x06, 0xD4,
+0x07, 0xD3, 0x40, 0xEA, 0xB0, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x3F, 0x6B, 0xA0, 0xA0, 0x1A, 0x65,
+0x41, 0xA0, 0x00, 0x6F, 0x0E, 0x06, 0x6C, 0xEA,
+0x40, 0x32, 0x4D, 0xED, 0x58, 0x67, 0x40, 0xEA,
+0x00, 0x6C, 0x14, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x17, 0xD2, 0x14, 0x92,
+0x15, 0x93, 0x13, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5, 0x40, 0x9A,
+0x0F, 0x96, 0x00, 0x6F, 0xA3, 0x67, 0x40, 0xEA,
+0x00, 0x6C, 0x04, 0x6A, 0x14, 0xD2, 0x14, 0x92,
+0x1F, 0x97, 0x1E, 0x91, 0x1D, 0x90, 0x00, 0xEF,
+0x10, 0x63, 0x0E, 0x97, 0x0C, 0x2F, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x40, 0x9A, 0x0F, 0x96, 0xA3, 0x67, 0x40, 0xEA,
+0x00, 0x6C, 0x02, 0x6A, 0xEB, 0x17, 0x41, 0xA0,
+0x80, 0xA0, 0x3F, 0x6D, 0xAC, 0xEA, 0x13, 0x95,
+0x40, 0x32, 0x8D, 0xEA, 0x13, 0x94, 0xC1, 0xA5,
+0x3F, 0x6D, 0x80, 0xA4, 0x0A, 0xD7, 0x0B, 0xD2,
+0xAC, 0xEE, 0xC0, 0x36, 0xA5, 0xA1, 0x8D, 0xEE,
+0x84, 0xA1, 0xA0, 0x35, 0x17, 0x92, 0x8D, 0xED,
+0x86, 0xA1, 0x73, 0xF6, 0x5C, 0x9A, 0x04, 0x43,
+0x80, 0x34, 0x80, 0x34, 0xAD, 0xEC, 0xA7, 0xA1,
+0x1B, 0xE6, 0x27, 0x67, 0x00, 0xF6, 0xA0, 0x35,
+0x8D, 0xED, 0x75, 0xE5, 0x40, 0xEA, 0x87, 0x67,
+0x13, 0x92, 0x3F, 0x6B, 0x13, 0x95, 0x80, 0xA2,
+0x41, 0xA2, 0x4C, 0xEB, 0x60, 0x33, 0x8D, 0xEB,
+0x03, 0xE3, 0x09, 0xE1, 0xF7, 0xF0, 0x01, 0x69,
+0x20, 0x31, 0x20, 0x31, 0x15, 0xD2, 0xB3, 0xF5,
+0x44, 0x99, 0x40, 0xEA, 0x85, 0x67, 0x02, 0x67,
+0x01, 0x6B, 0x18, 0x92, 0x43, 0xEB, 0x33, 0x61,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x94, 0xF2, 0x4C, 0x98, 0x51, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF3,
+0x48, 0x9A, 0x16, 0x94, 0x00, 0x6F, 0x00, 0x6E,
+0x40, 0xEA, 0x06, 0x05, 0x7B, 0x22, 0x00, 0x6A,
+0x10, 0xD2, 0x94, 0xF2, 0x4C, 0x98, 0x52, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x12, 0x93,
+0x40, 0x32, 0xF2, 0xF1, 0x0C, 0x9A, 0x12, 0x92,
+0x95, 0xA3, 0x00, 0x6F, 0x54, 0xA2, 0x80, 0x34,
+0x01, 0x6E, 0x4D, 0xEC, 0x56, 0xA3, 0x01, 0x6D,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x97, 0xA3,
+0x10, 0x03, 0x04, 0xD3, 0x00, 0xF6, 0x80, 0x34,
+0x40, 0xE8, 0x4D, 0xEC, 0x7C, 0x17, 0xC1, 0xA0,
+0x80, 0xA0, 0x3F, 0x6F, 0x17, 0x92, 0xEC, 0xEE,
+0xC0, 0x36, 0x8D, 0xEE, 0x73, 0xF6, 0x5C, 0x9A,
+0x15, 0x94, 0xA4, 0x40, 0xFC, 0x4E, 0x1A, 0xD3,
+0x40, 0xEA, 0x19, 0xD7, 0x41, 0xA0, 0x19, 0x97,
+0x80, 0xA0, 0xB0, 0x67, 0x4C, 0xEF, 0xE0, 0x37,
+0x15, 0x92, 0x8D, 0xEF, 0xFC, 0x4F, 0xE9, 0xE2,
+0x15, 0xD2, 0x13, 0x94, 0xB3, 0xF5, 0x44, 0x99,
+0x40, 0xEA, 0x00, 0x65, 0x1A, 0x93, 0x02, 0x67,
+0xFF, 0x6A, 0x01, 0x4B, 0x4C, 0xEB, 0xA5, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x52, 0xF0, 0x58, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x01, 0x6B, 0x6C, 0xEA, 0xA4, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF3,
+0x4C, 0x9A, 0xA3, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x52, 0xF0, 0x58, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x01, 0x6B, 0x6C, 0xEA,
+0xA3, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x12, 0x93, 0x40, 0x32, 0xF2, 0xF1, 0x10, 0x9A,
+0x12, 0x92, 0x95, 0xA3, 0x00, 0x6F, 0x54, 0xA2,
+0x80, 0x34, 0x01, 0x6E, 0x4D, 0xEC, 0x56, 0xA3,
+0x01, 0x6D, 0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA,
+0x97, 0xA3, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xE8,
+0x4D, 0xEC, 0x21, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xF7, 0xF0, 0x01, 0x6C,
+0x72, 0xF0, 0x4C, 0x9A, 0x80, 0x34, 0x80, 0x34,
+0x20, 0xF4, 0x06, 0x6D, 0x0A, 0xF3, 0x00, 0x4C,
+0x40, 0xEA, 0x00, 0x65, 0x10, 0x17, 0x00, 0x65,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0xF7, 0xF0, 0x01, 0x68, 0x00, 0x30, 0x00, 0x30,
+0x15, 0xF0, 0x08, 0x48, 0xF7, 0xF0, 0x01, 0x6A,
+0x89, 0xA0, 0x40, 0x32, 0x68, 0xA0, 0x40, 0x32,
+0xF2, 0xF2, 0x30, 0x9A, 0x4A, 0xA0, 0x80, 0x34,
+0x6D, 0xEC, 0x6B, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0x4D, 0xEC, 0x00, 0xF6, 0x60, 0x33, 0x8D, 0xEB,
+0x89, 0xA3, 0x48, 0xA3, 0xAA, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x4B, 0xA3, 0xA0, 0x35, 0xA0, 0x35,
+0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x40, 0xF0, 0x89, 0xA2, 0x40, 0xF0, 0x68, 0xA2,
+0x01, 0x6F, 0x80, 0x34, 0x6D, 0xEC, 0x40, 0xF0,
+0x6A, 0xA2, 0xEB, 0xEF, 0x10, 0x6E, 0x60, 0x33,
+0x60, 0x33, 0x8D, 0xEB, 0x40, 0xF0, 0x8B, 0xA2,
+0x06, 0x05, 0x00, 0xF6, 0x80, 0x34, 0x40, 0xE9,
+0x6D, 0xEC, 0x10, 0x72, 0xE0, 0xF1, 0x14, 0x61,
+0x07, 0x93, 0x0E, 0xD3, 0x63, 0x83, 0x00, 0x53,
+0x01, 0x60, 0x20, 0x6A, 0x0E, 0x93, 0x0E, 0x95,
+0x06, 0x94, 0x63, 0xA3, 0x0F, 0x69, 0xFF, 0x6E,
+0x2C, 0xEB, 0x55, 0xE5, 0xCC, 0xEB, 0x07, 0xD5,
+0x05, 0x73, 0x57, 0xE4, 0x06, 0xD5, 0x5A, 0x60,
+0x06, 0x5B, 0x1F, 0x60, 0x01, 0x73, 0x2B, 0x60,
+0x04, 0x73, 0x4B, 0x60, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x02, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x0E, 0x94, 0x40, 0x32,
+0x13, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0xFC, 0x4C,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF0, 0x54, 0x9A, 0x40, 0xEA, 0x90, 0x67,
+0x1F, 0x10, 0x06, 0x73, 0x41, 0x60, 0x09, 0x73,
+0xE1, 0x61, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x92, 0xF7, 0x48, 0x9A, 0xCC, 0xED,
+0x40, 0xEA, 0x06, 0x04, 0xD7, 0x17, 0x0E, 0x92,
+0x06, 0xD4, 0x06, 0x04, 0xD1, 0x18, 0xE0, 0x4F,
+0x07, 0xD2, 0x0E, 0x93, 0x0C, 0x43, 0x0D, 0x22,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x13, 0xF6, 0x48, 0x9A, 0x40, 0xEA, 0x90, 0x67,
+0x17, 0x97, 0x16, 0x91, 0x15, 0x90, 0x00, 0xEF,
+0x0C, 0x63, 0xD1, 0x18, 0x93, 0xAC, 0x06, 0x04,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x09, 0xF1, 0x70, 0x9B, 0x6E, 0xEA, 0xE8, 0x2A,
+0xB5, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x92, 0xF7, 0x40, 0x9A, 0x40, 0xEA,
+0x06, 0x04, 0xAC, 0x17, 0x0E, 0x92, 0x06, 0xD4,
+0x07, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x92, 0xF7, 0x58, 0x9A, 0xF3, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD2, 0xF4, 0x4C, 0x9A, 0xE0, 0xF1, 0x13, 0x6C,
+0x40, 0xEA, 0x0F, 0xD6, 0xA7, 0x42, 0x10, 0x6B,
+0x6B, 0xEB, 0x09, 0x4D, 0x6C, 0xED, 0x2C, 0xEA,
+0x4D, 0xED, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x0F, 0x96, 0x40, 0x32, 0xD2, 0xF4, 0x40, 0x9A,
+0x00, 0xF6, 0xA0, 0x35, 0x00, 0xF6, 0xA3, 0x35,
+0xE0, 0xF1, 0x13, 0x6C, 0x40, 0xEA, 0xCC, 0xED,
+0x07, 0x92, 0x0F, 0x96, 0x1F, 0x6C, 0x60, 0xA2,
+0x8C, 0xEB, 0xCC, 0xEB, 0x08, 0x73, 0x07, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x72, 0xF7, 0x50, 0x9A, 0xC4, 0x17, 0x04, 0x73,
+0x20, 0xF1, 0x1D, 0x61, 0x6D, 0xA0, 0x8C, 0xA0,
+0xAE, 0xA0, 0x60, 0x33, 0x8D, 0xEB, 0x8F, 0xA0,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEB, 0x00, 0xF6,
+0x80, 0x34, 0x6D, 0xEC, 0xB9, 0xA4, 0x78, 0xA4,
+0xFA, 0xA4, 0xA0, 0x35, 0x6D, 0xED, 0x7B, 0xA4,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xB1, 0xA2, 0x07, 0x6C,
+0xAC, 0xEC, 0xCC, 0xEC, 0x0F, 0xD4, 0x90, 0xA2,
+0x8C, 0xE9, 0x0F, 0x94, 0xCC, 0xE9, 0x10, 0xD1,
+0x02, 0x24, 0x23, 0x44, 0xCC, 0xE9, 0x0E, 0x94,
+0x02, 0xA4, 0x01, 0x6C, 0x1A, 0x30, 0x8C, 0xE8,
+0x10, 0x6C, 0x8B, 0xEC, 0xAC, 0xEC, 0xFF, 0x6D,
+0xAC, 0xEC, 0x1B, 0x24, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x12, 0xD3, 0xF3, 0xF0,
+0x70, 0x9C, 0x91, 0xA2, 0x40, 0xF3, 0x16, 0x6F,
+0x02, 0x6E, 0x92, 0x34, 0x05, 0xD4, 0x01, 0x6C,
+0x04, 0xD4, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xA9, 0xF6, 0xBC, 0x9C, 0xA9, 0xF6,
+0x98, 0x9C, 0x40, 0xEB, 0x11, 0xD2, 0x12, 0x93,
+0x11, 0x92, 0x0F, 0x94, 0x12, 0xD3, 0xD1, 0x18,
+0xF2, 0xAE, 0x11, 0xD2, 0x11, 0x92, 0x12, 0x93,
+0x51, 0xA2, 0x52, 0x32, 0x0B, 0x5A, 0x20, 0x60,
+0x48, 0x34, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x88, 0xF4, 0x1C, 0x4A, 0x49, 0xE4,
+0xA0, 0x9A, 0x04, 0x34, 0x24, 0x32, 0x29, 0xE2,
+0x11, 0xE4, 0x44, 0x32, 0x90, 0x34, 0x89, 0xE2,
+0x00, 0xED, 0x49, 0xE3, 0xA0, 0xF0, 0x9D, 0xA2,
+0x0F, 0x6E, 0xA4, 0x67, 0xCC, 0xED, 0x01, 0x4D,
+0xCC, 0xED, 0x10, 0x6E, 0xCB, 0xEE, 0xCC, 0xEC,
+0xAD, 0xEC, 0x10, 0x4C, 0xA0, 0xF0, 0x9D, 0xC2,
+0x24, 0x32, 0x04, 0x34, 0x29, 0xE2, 0x11, 0xE4,
+0x44, 0x32, 0x90, 0x34, 0x89, 0xE2, 0x49, 0xE3,
+0xA0, 0xF0, 0x5D, 0xA2, 0xA0, 0x5A, 0xFF, 0xF6,
+0x0D, 0x61, 0x00, 0x6C, 0xD1, 0x18, 0x5D, 0x9A,
+0x11, 0xD3, 0x00, 0x6A, 0x0C, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF5,
+0x48, 0x9A, 0x00, 0x6F, 0x0C, 0x06, 0x07, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x24, 0x37, 0x11, 0x93,
+0x37, 0x2A, 0x0C, 0x94, 0x35, 0x24, 0xDD, 0x67,
+0x01, 0x6D, 0x20, 0xF0, 0xAA, 0xC6, 0xBD, 0x67,
+0x20, 0xF0, 0x49, 0xC5, 0x03, 0x6D, 0x20, 0xF0,
+0xA8, 0xC6, 0x0F, 0x6D, 0x20, 0xF0, 0xAB, 0xC6,
+0xBD, 0x67, 0x20, 0xF0, 0x4C, 0xC5, 0x0F, 0x92,
+0x04, 0x22, 0x0F, 0x4A, 0xFF, 0x6D, 0xAC, 0xEA,
+0x10, 0xD2, 0x10, 0x95, 0x1C, 0x32, 0x0F, 0xD3,
+0xAD, 0xEA, 0x40, 0xC4, 0x04, 0x34, 0x15, 0xE4,
+0xB0, 0x35, 0x31, 0xE7, 0x7F, 0x4D, 0x84, 0x34,
+0xF7, 0xF0, 0x01, 0x6A, 0x39, 0x4D, 0x95, 0xE5,
+0x40, 0x32, 0xB5, 0xE3, 0x40, 0x32, 0x0C, 0x93,
+0x73, 0xF6, 0x5C, 0x9A, 0x06, 0x6E, 0x81, 0x43,
+0x40, 0xEA, 0x10, 0xD7, 0x0C, 0x95, 0xD1, 0x18,
+0xA8, 0x19, 0x0A, 0x04, 0x10, 0x97, 0x0F, 0x93,
+0x04, 0x34, 0x11, 0xE4, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x90, 0x34, 0x25, 0xE7, 0x40, 0x32,
+0x7F, 0x4C, 0x24, 0x31, 0x39, 0x4C, 0x93, 0xF6,
+0x40, 0x9A, 0x31, 0xE4, 0x06, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x91, 0xE3, 0x8F, 0x16, 0xA0, 0xF0,
+0x98, 0xA2, 0x0F, 0x6E, 0xA4, 0x67, 0xCC, 0xED,
+0x01, 0x4D, 0xCC, 0xED, 0x10, 0x6E, 0xCB, 0xEE,
+0xCC, 0xEC, 0xAD, 0xEC, 0xA0, 0xF0, 0x98, 0xC2,
+0xA0, 0xF0, 0x9D, 0xA2, 0x82, 0x17, 0xA0, 0xF0,
+0x98, 0xA2, 0x10, 0x4C, 0xF7, 0x17, 0xA0, 0xF0,
+0x99, 0xA2, 0x0F, 0x6E, 0xA4, 0x67, 0xCC, 0xED,
+0x01, 0x4D, 0xCC, 0xED, 0x10, 0x6E, 0xCB, 0xEE,
+0xCC, 0xEC, 0xAD, 0xEC, 0xA0, 0xF0, 0x99, 0xC2,
+0xEB, 0x17, 0xA0, 0xF0, 0x99, 0xA2, 0x10, 0x4C,
+0xF9, 0x17, 0xA0, 0xF0, 0x9A, 0xA2, 0x0F, 0x6E,
+0xA4, 0x67, 0xCC, 0xED, 0x01, 0x4D, 0xCC, 0xED,
+0x10, 0x6E, 0xCB, 0xEE, 0xCC, 0xEC, 0xAD, 0xEC,
+0xA0, 0xF0, 0x9A, 0xC2, 0xD9, 0x17, 0xA0, 0xF0,
+0x9A, 0xA2, 0x10, 0x4C, 0xF9, 0x17, 0xA0, 0xF0,
+0x9B, 0xA2, 0x0F, 0x6E, 0xA4, 0x67, 0xCC, 0xED,
+0x01, 0x4D, 0xCC, 0xED, 0x10, 0x6E, 0xCB, 0xEE,
+0xCC, 0xEC, 0xAD, 0xEC, 0xA0, 0xF0, 0x9B, 0xC2,
+0xC7, 0x17, 0xA0, 0xF0, 0x9B, 0xA2, 0x10, 0x4C,
+0xF9, 0x17, 0xA0, 0xF0, 0x9C, 0xA2, 0x0F, 0x6E,
+0xA4, 0x67, 0xCC, 0xED, 0x01, 0x4D, 0xCC, 0xED,
+0x10, 0x6E, 0xCB, 0xEE, 0xCC, 0xEC, 0xAD, 0xEC,
+0xA0, 0xF0, 0x9C, 0xC2, 0xB5, 0x17, 0xA0, 0xF0,
+0x9C, 0xA2, 0x10, 0x4C, 0xF9, 0x17, 0x02, 0x73,
+0x04, 0x61, 0xD1, 0x18, 0xB8, 0x4F, 0x06, 0x04,
+0x2D, 0x16, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF7, 0x58, 0x9A, 0x77, 0x16,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF6, 0x50, 0x9A, 0x08, 0xF0, 0x00, 0x6D,
+0x40, 0xEA, 0x00, 0x6C, 0x55, 0x16, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x15, 0xF0, 0x08, 0x4A, 0xA5, 0xA2, 0x64, 0xA2,
+0xC6, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x67, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0x20, 0xF0, 0xA1, 0xA3,
+0x20, 0xF0, 0xC2, 0xA3, 0x20, 0xF0, 0x40, 0xA3,
+0xA0, 0x35, 0xC0, 0x36, 0xAD, 0xEA, 0xC0, 0x36,
+0x20, 0xF0, 0xA3, 0xA3, 0x4D, 0xEE, 0x48, 0xA4,
+0x00, 0xF6, 0xA0, 0x35, 0xAD, 0xEE, 0x48, 0x32,
+0xC9, 0xE2, 0x65, 0xA2, 0xA6, 0xA2, 0xC4, 0xA2,
+0x47, 0xA2, 0x60, 0x33, 0xA0, 0x35, 0x6D, 0xEE,
+0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x61, 0xA2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x68, 0x33, 0x6D, 0xF1,
+0x0C, 0x4A, 0x49, 0xE3, 0xA0, 0x9A, 0x42, 0xA4,
+0x17, 0x2A, 0x43, 0xA4, 0x15, 0x22, 0x20, 0xF0,
+0x59, 0xA5, 0x0F, 0x6B, 0xFF, 0x6C, 0x4C, 0xEB,
+0x8C, 0xEB, 0x0F, 0x73, 0x0F, 0x6A, 0x02, 0x60,
+0x41, 0x43, 0x8C, 0xEA, 0x0F, 0x6B, 0x6C, 0xEA,
+0x20, 0xF0, 0x79, 0xA5, 0x10, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEB, 0x6D, 0xEA, 0x20, 0xF0, 0x59, 0xC5,
+0x00, 0x6A, 0x20, 0xF0, 0x5E, 0xC5, 0x20, 0xE8,
+0x00, 0x65, 0x00, 0x65, 0x42, 0xA4, 0x52, 0x2A,
+0x43, 0xA4, 0x50, 0x22, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A,
+0xA5, 0xA2, 0x64, 0xA2, 0xC6, 0xA2, 0xA0, 0x35,
+0x6D, 0xED, 0x67, 0xA2, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x20, 0xF0, 0xA1, 0xA3, 0x20, 0xF0, 0xC2, 0xA3,
+0x20, 0xF0, 0x40, 0xA3, 0xA0, 0x35, 0xC0, 0x36,
+0xAD, 0xEA, 0xC0, 0x36, 0x20, 0xF0, 0xA3, 0xA3,
+0x4D, 0xEE, 0x48, 0xA4, 0x00, 0xF6, 0xA0, 0x35,
+0xAD, 0xEE, 0x48, 0x32, 0xC9, 0xE2, 0x65, 0xA2,
+0x86, 0xA2, 0xA4, 0xA2, 0x47, 0xA2, 0x60, 0x33,
+0x80, 0x34, 0x6D, 0xED, 0x80, 0x34, 0xAD, 0xEC,
+0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA, 0x61, 0xA2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x68, 0x33, 0x6D, 0xF1, 0x0C, 0x4A, 0x49, 0xE3,
+0x80, 0x9A, 0x0F, 0x6B, 0xFF, 0x6D, 0x20, 0xF0,
+0x59, 0xA4, 0x4C, 0xEB, 0xAC, 0xEB, 0x0F, 0x73,
+0x0F, 0x6A, 0x02, 0x60, 0x41, 0x43, 0xAC, 0xEA,
+0x0F, 0x6B, 0x6C, 0xEA, 0x20, 0xF0, 0x79, 0xA4,
+0x10, 0x6D, 0xAB, 0xED, 0xAC, 0xEB, 0x6D, 0xEA,
+0x20, 0xF0, 0x59, 0xC4, 0x20, 0xE8, 0x00, 0x65,
+0xFD, 0x63, 0x05, 0x62, 0x71, 0xA4, 0x02, 0x6A,
+0x4C, 0xEB, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x15, 0xF0, 0x08, 0x4A, 0x12, 0x23,
+0x9D, 0xA2, 0x7C, 0xA2, 0x80, 0x34, 0x6D, 0xEC,
+0x7E, 0xA2, 0x60, 0x33, 0x60, 0x33, 0x8D, 0xEB,
+0x9F, 0xA2, 0x00, 0xF6, 0x80, 0x34, 0xD1, 0x18,
+0xED, 0x7D, 0x6D, 0xEC, 0x05, 0x97, 0x01, 0x6A,
+0x00, 0xEF, 0x03, 0x63, 0xBD, 0xA2, 0x7E, 0xA2,
+0xDC, 0xA2, 0x5F, 0xA2, 0xA0, 0x35, 0x60, 0x33,
+0xAD, 0xEE, 0x60, 0x33, 0xCD, 0xEB, 0xA1, 0xA4,
+0x00, 0xF6, 0x40, 0x32, 0x6D, 0xEA, 0xD1, 0x18,
+0xA6, 0x7D, 0x82, 0x67, 0xEB, 0x17, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0x06, 0x02, 0x40, 0xF0, 0x40, 0xA2, 0x48, 0x31,
+0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xF1, 0x0C, 0x4A, 0x49, 0xE1,
+0x60, 0x9A, 0x20, 0xF0, 0x3C, 0xA3, 0xE0, 0xF0,
+0x03, 0x21, 0x20, 0xF0, 0x5D, 0xA3, 0x0A, 0xD7,
+0x06, 0x67, 0x0B, 0xD2, 0x00, 0x6A, 0x06, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xD3, 0xF5, 0x48, 0x9A, 0x0C, 0xD5, 0x08, 0xD4,
+0x00, 0x6F, 0x06, 0x06, 0x1C, 0x6D, 0x00, 0x6C,
+0x40, 0xEA, 0x0D, 0xD3, 0xC0, 0xF0, 0x0C, 0x2A,
+0x06, 0x94, 0xC0, 0xF0, 0x09, 0x24, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x93, 0xF6,
+0x40, 0x9A, 0x1C, 0x6E, 0x40, 0xEA, 0x00, 0x6D,
+0x0D, 0x93, 0x9D, 0xA3, 0x5C, 0xA3, 0xBE, 0xA3,
+0x80, 0x34, 0x4D, 0xEC, 0x5F, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x20, 0xF0, 0x81, 0xA3, 0x20, 0xF0,
+0xE0, 0xA3, 0x20, 0xF0, 0xA2, 0xA3, 0x80, 0x34,
+0xED, 0xEC, 0x20, 0xF0, 0xE3, 0xA3, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6, 0xE0, 0x37,
+0x06, 0x95, 0x8D, 0xEF, 0x0B, 0x94, 0x20, 0xC5,
+0x03, 0x69, 0x2C, 0xEC, 0xCF, 0xA5, 0x24, 0x67,
+0x04, 0x6C, 0x8B, 0xEC, 0x8C, 0xEE, 0x2D, 0xEE,
+0x09, 0x91, 0x01, 0x6C, 0x3F, 0x65, 0x8C, 0xE9,
+0x28, 0x34, 0x05, 0x69, 0x2B, 0xE9, 0x2C, 0xEE,
+0xCD, 0xEC, 0x8F, 0xC5, 0x08, 0x96, 0x0F, 0x6C,
+0xF5, 0x49, 0x8C, 0xEE, 0x1E, 0x65, 0xC2, 0xA5,
+0xF8, 0x67, 0x2C, 0xEE, 0xED, 0xEE, 0x0C, 0x97,
+0x8C, 0xEE, 0xF0, 0x37, 0xED, 0xEE, 0xC2, 0xC5,
+0x08, 0x96, 0x05, 0x76, 0xD8, 0x67, 0x23, 0x26,
+0x08, 0x96, 0x06, 0x76, 0xD8, 0x67, 0x1F, 0x26,
+0x08, 0x92, 0x04, 0x72, 0x5D, 0x61, 0x20, 0xF0,
+0xED, 0xA3, 0x20, 0xF0, 0x4E, 0xA3, 0x20, 0xF0,
+0xCC, 0xA3, 0xE0, 0x37, 0x40, 0x32, 0xCD, 0xEF,
+0x40, 0x32, 0x20, 0xF0, 0xCF, 0xA3, 0x4D, 0xEF,
+0x20, 0xF0, 0x59, 0xA3, 0x00, 0xF6, 0xC0, 0x36,
+0xED, 0xEE, 0x8C, 0xEA, 0x8C, 0xA5, 0xC3, 0xE0,
+0x02, 0x30, 0x8C, 0xE9, 0x0A, 0x30, 0x2D, 0xEA,
+0x03, 0xC5, 0x4C, 0xC5, 0x41, 0x10, 0xFB, 0xA3,
+0x4B, 0xE0, 0x44, 0xC5, 0xE1, 0xC5, 0x42, 0x37,
+0xE5, 0xC5, 0xE2, 0x37, 0xE6, 0xC5, 0x00, 0xF6,
+0x42, 0x37, 0xE7, 0xC5, 0x0A, 0x97, 0x43, 0xE8,
+0x59, 0x67, 0x5F, 0xE7, 0x58, 0x67, 0x5F, 0xE7,
+0xE2, 0x32, 0x49, 0xC5, 0x42, 0x32, 0x4A, 0xC5,
+0x20, 0xF0, 0x59, 0xA3, 0xE8, 0xC5, 0x00, 0xF6,
+0xE2, 0x37, 0x52, 0x32, 0xEB, 0xC5, 0x50, 0x37,
+0x4C, 0xA5, 0xCD, 0xC5, 0xCE, 0xC5, 0x8C, 0xEA,
+0xED, 0xEA, 0x4C, 0xC5, 0x20, 0xF0, 0xED, 0xA3,
+0x20, 0xF0, 0xCC, 0xA3, 0x4C, 0xE9, 0xE0, 0x37,
+0xCD, 0xEF, 0x1F, 0x65, 0x20, 0xF0, 0xEE, 0xA3,
+0xD8, 0x67, 0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE,
+0x1E, 0x65, 0x20, 0xF0, 0xCF, 0xA3, 0xF8, 0x67,
+0x00, 0xF6, 0xC0, 0x36, 0xED, 0xEE, 0xC3, 0xE0,
+0x20, 0xF0, 0xD9, 0xA3, 0x02, 0x30, 0x0A, 0x30,
+0xCC, 0xEC, 0x8D, 0xE9, 0x03, 0xC5, 0x2C, 0xC5,
+0x7D, 0x67, 0x01, 0x6A, 0x52, 0xC3, 0x51, 0xC3,
+0x09, 0x6A, 0x50, 0xC3, 0x24, 0x6A, 0x53, 0xC3,
+0x04, 0x04, 0x00, 0x6A, 0xD1, 0x18, 0xA8, 0x19,
+0x54, 0xC3, 0x08, 0x92, 0x03, 0x72, 0x08, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xB2, 0xF2, 0x5C, 0x9A, 0x40, 0xEA, 0x01, 0x6C,
+0x11, 0x97, 0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF,
+0x09, 0x63, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0x01, 0xA4, 0x01, 0x6B,
+0x16, 0x32, 0x6C, 0xEA, 0x1D, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x70, 0x9A, 0x01, 0x6A, 0x06, 0xD2, 0x05, 0xD2,
+0x02, 0x6A, 0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC9, 0xF6, 0x80, 0x9A,
+0xC9, 0xF6, 0xA4, 0x9A, 0x60, 0xF3, 0x19, 0x6F,
+0x40, 0xEB, 0x02, 0x6E, 0x44, 0x6A, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x6D, 0xF1, 0x4C, 0x9A, 0x74, 0xA2, 0x15, 0x2B,
+0xF7, 0xF0, 0x01, 0x6A, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0xC9, 0xF6,
+0x80, 0x9B, 0xC9, 0xF6, 0xA4, 0x9B, 0x60, 0xF3,
+0x1A, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0x04, 0x6A,
+0xDE, 0x17, 0xE9, 0xA2, 0xC0, 0xA4, 0x88, 0xA2,
+0x6A, 0xA2, 0xE0, 0x37, 0x8D, 0xEF, 0x8B, 0xA2,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEF, 0x00, 0xF6,
+0x80, 0x34, 0xED, 0xEC, 0x23, 0xA4, 0x2A, 0xEE,
+0x08, 0x61, 0x65, 0xA4, 0x03, 0x6F, 0x0C, 0xEF,
+0x7A, 0x33, 0xFF, 0x6D, 0xEE, 0xEB, 0xAC, 0xEB,
+0x17, 0x23, 0x02, 0x6B, 0x04, 0xD3, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B, 0x40, 0x32,
+0x60, 0x33, 0x60, 0x33, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0xC9, 0xF6, 0x80, 0x9B, 0xC9, 0xF6,
+0xA4, 0x9B, 0x06, 0xD6, 0x05, 0xD1, 0x60, 0xF3,
+0x1B, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0xCF, 0x17,
+0x0A, 0x30, 0x07, 0x69, 0x2C, 0xE8, 0xAC, 0xE8,
+0x52, 0x28, 0x61, 0xA2, 0x82, 0xA2, 0xC0, 0xA2,
+0x43, 0xA2, 0x60, 0x33, 0x80, 0x34, 0x6D, 0xEE,
+0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x42, 0xA2, 0x97, 0xF0, 0x19, 0x6E,
+0xC0, 0x36, 0x4C, 0xE9, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xD3, 0xF3, 0x60, 0x9A,
+0xAC, 0xE9, 0xC0, 0x36, 0xAE, 0xF0, 0x19, 0x4E,
+0xB1, 0x67, 0x00, 0x6C, 0x40, 0xEB, 0x08, 0xD2,
+0x08, 0x92, 0x97, 0xF0, 0x19, 0x6B, 0x60, 0x33,
+0xD3, 0xF3, 0x40, 0x9A, 0x60, 0x33, 0xED, 0xF0,
+0x19, 0x4B, 0xC3, 0x67, 0xB1, 0x67, 0x00, 0x6C,
+0x40, 0xEA, 0x0A, 0xD3, 0xF7, 0xF0, 0x01, 0x6F,
+0xE0, 0x37, 0xE0, 0x37, 0xF3, 0xF3, 0x4C, 0x9F,
+0x01, 0x6E, 0xB1, 0x67, 0x00, 0x6C, 0x40, 0xEA,
+0x09, 0xD7, 0x09, 0x97, 0x08, 0xD2, 0x00, 0x6E,
+0xF3, 0xF3, 0x4C, 0x9F, 0xB1, 0x67, 0x40, 0xEA,
+0x00, 0x6C, 0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34,
+0x80, 0x34, 0xD3, 0xF3, 0x64, 0x9C, 0x08, 0x97,
+0xC2, 0x67, 0x1B, 0x65, 0x0A, 0x93, 0x05, 0xD0,
+0xB1, 0x67, 0x04, 0xD3, 0x58, 0x67, 0x40, 0xEA,
+0x00, 0x6C, 0x00, 0x6A, 0x58, 0x17, 0x01, 0x6B,
+0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0xC9, 0xF6,
+0x80, 0x9B, 0xC9, 0xF6, 0xA4, 0x9B, 0x05, 0xD0,
+0x60, 0xF3, 0x1C, 0x6F, 0x40, 0xEA, 0x02, 0x6E,
+0x62, 0x17, 0x00, 0x65, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD1, 0x13, 0xD0, 0x42, 0xA4, 0x01, 0x6B,
+0x04, 0x67, 0x6C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA,
+0x1D, 0x22, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x70, 0x9A, 0x01, 0x6A,
+0x06, 0xD2, 0x05, 0xD2, 0x02, 0x6A, 0x04, 0xD2,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC9, 0xF6, 0x80, 0x9A, 0xC9, 0xF6, 0xA4, 0x9A,
+0x60, 0xF3, 0x1D, 0x6F, 0x40, 0xEB, 0x02, 0x6E,
+0x44, 0x6A, 0x15, 0x97, 0x14, 0x91, 0x13, 0x90,
+0x00, 0xEF, 0x0B, 0x63, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xF1, 0x2C, 0x9B,
+0xB4, 0xA1, 0x17, 0x25, 0x05, 0xD2, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x6B, 0x04, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x40, 0x32, 0x40, 0x32,
+0x60, 0x33, 0xF3, 0xF0, 0x70, 0x9B, 0xC9, 0xF6,
+0x80, 0x9A, 0xC9, 0xF6, 0xA4, 0x9A, 0x60, 0xF3,
+0x1E, 0x6F, 0x40, 0xEB, 0x02, 0x6E, 0x04, 0x6A,
+0xDC, 0x17, 0x41, 0xA0, 0x48, 0x32, 0x8C, 0xEA,
+0x1C, 0x5A, 0x0D, 0xD2, 0x19, 0x60, 0x1C, 0x6B,
+0x06, 0xD3, 0x0D, 0x93, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x05, 0xD3, 0x02, 0x6B, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0xC9, 0xF6,
+0x80, 0x9B, 0xC9, 0xF6, 0xA4, 0x9B, 0x60, 0xF3,
+0x1F, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0xDF, 0x17,
+0x40, 0xA0, 0x0C, 0xD2, 0x15, 0x2A, 0x0C, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0xC9, 0xF6,
+0x80, 0x9B, 0xC9, 0xF6, 0xA4, 0x9B, 0x80, 0xF3,
+0x00, 0x6F, 0x40, 0xEA, 0x02, 0x6E, 0xC7, 0x17,
+0x0C, 0x92, 0x10, 0xD5, 0x81, 0x42, 0x8C, 0x32,
+0x8B, 0xE2, 0x48, 0x32, 0x0E, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0xE5, 0xA1,
+0x84, 0xA1, 0xC6, 0xA1, 0xE0, 0x37, 0x8D, 0xEF,
+0x87, 0xA1, 0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEF,
+0x00, 0xF6, 0x80, 0x34, 0xED, 0xEC, 0x13, 0x24,
+0xF7, 0xF0, 0x01, 0x6E, 0xC0, 0x36, 0xC0, 0x36,
+0x72, 0xF4, 0xD0, 0x9E, 0x40, 0xEE, 0x0F, 0xD2,
+0x10, 0x95, 0x0F, 0x92, 0xA4, 0xC1, 0xA5, 0xC1,
+0xA6, 0xC1, 0xA7, 0xC1, 0xA8, 0xC1, 0xA9, 0xC1,
+0xAA, 0xC1, 0xAB, 0xC1, 0xB5, 0xC1, 0xF7, 0xF0,
+0x01, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x72, 0xF0,
+0xB4, 0x9C, 0x40, 0xED, 0x82, 0x67, 0xF7, 0xF0,
+0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6C, 0x40, 0x32,
+0x80, 0x34, 0x80, 0x34, 0x40, 0x32, 0xA9, 0xF3,
+0xBC, 0x9C, 0x72, 0xF4, 0x54, 0x9A, 0x40, 0xEA,
+0x0E, 0x94, 0x62, 0x67, 0x29, 0x2A, 0x04, 0xD3,
+0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0, 0x01, 0x6B,
+0x40, 0x32, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xC9, 0xF6, 0x80, 0x9B, 0xC9, 0xF6, 0xA4, 0x9B,
+0xF3, 0xF0, 0x50, 0x9A, 0x80, 0xF3, 0x01, 0x6F,
+0x40, 0xEA, 0x02, 0x6E, 0x06, 0x6A, 0x7D, 0x67,
+0x20, 0xF0, 0x40, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0A, 0xF3, 0x1C, 0x4A,
+0x09, 0xD2, 0x68, 0x6A, 0x0A, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x08, 0x04, 0x57, 0x17,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x0E, 0x96, 0x93, 0xF6, 0x40, 0x9A, 0x83, 0x67,
+0x00, 0x6D, 0x40, 0xEA, 0x0E, 0xD3, 0x0E, 0x93,
+0x04, 0x48, 0x62, 0x32, 0x45, 0xC1, 0x42, 0x32,
+0x46, 0xC1, 0x00, 0xF6, 0x62, 0x32, 0x47, 0xC1,
+0x0C, 0x92, 0x64, 0xC1, 0x4C, 0x37, 0x5F, 0xE7,
+0xE8, 0x37, 0xFD, 0xE3, 0x43, 0x67, 0xEE, 0xEA,
+0x26, 0x2A, 0xCD, 0xA1, 0x8C, 0xA1, 0xAE, 0xA1,
+0xC0, 0x36, 0x8D, 0xEE, 0x8F, 0xA1, 0xA0, 0x35,
+0xA0, 0x35, 0xAD, 0xEE, 0x00, 0xF6, 0x80, 0x34,
+0xCD, 0xEC, 0x02, 0x67, 0x10, 0x24, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0x72, 0xF4,
+0xB0, 0x9D, 0x40, 0xED, 0x00, 0x65, 0x0C, 0xC1,
+0x0D, 0xC1, 0x0E, 0xC1, 0x0F, 0xC1, 0x10, 0xC1,
+0x11, 0xC1, 0x12, 0xC1, 0x13, 0xC1, 0x00, 0x6A,
+0x54, 0xC1, 0x5D, 0x67, 0x20, 0xF0, 0x50, 0xA2,
+0x55, 0xC1, 0x00, 0x6A, 0xF2, 0x16, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x73, 0xF6,
+0x5C, 0x9A, 0xB0, 0x67, 0x83, 0x67, 0x1C, 0x6E,
+0x0F, 0xD7, 0x40, 0xEA, 0x0E, 0xD3, 0x0E, 0x93,
+0x0D, 0x92, 0x0F, 0x97, 0x1C, 0x4B, 0x41, 0xE0,
+0xC5, 0x17, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x5D, 0x67, 0x20, 0xF0, 0x50, 0xA2, 0x06, 0x22,
+0x5D, 0x67, 0x20, 0xF0, 0x54, 0xA2, 0xD1, 0x18,
+0x46, 0x52, 0x04, 0xD2, 0x07, 0x97, 0x00, 0xEF,
+0x04, 0x63, 0x00, 0x65, 0xF3, 0x63, 0x19, 0x62,
+0x18, 0xD1, 0x17, 0xD0, 0x1E, 0x92, 0x24, 0x67,
+0x1C, 0xD6, 0x11, 0xD2, 0x1F, 0x92, 0xFF, 0x6B,
+0x1B, 0xD5, 0x12, 0xD2, 0x06, 0x02, 0x60, 0xF0,
+0x48, 0xA2, 0x48, 0x34, 0x10, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x6D, 0xF1,
+0x0C, 0x4A, 0x49, 0xE4, 0x00, 0x9A, 0x47, 0xA1,
+0x04, 0x6C, 0xDA, 0xA0, 0x8C, 0xEA, 0x6C, 0xEA,
+0x2B, 0x22, 0xA1, 0xA0, 0x80, 0xA0, 0xF7, 0xF0,
+0x01, 0x6A, 0xA0, 0x35, 0x8D, 0xED, 0x82, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x83, 0xA0, 0xF3, 0xF3, 0x50, 0x9A,
+0x14, 0xD7, 0x00, 0xF6, 0x80, 0x34, 0xAD, 0xEC,
+0x82, 0xA4, 0x01, 0x6D, 0x13, 0xD6, 0x8E, 0x34,
+0xAC, 0xEC, 0x6C, 0xEC, 0x40, 0xEA, 0x00, 0x6D,
+0x11, 0x93, 0x14, 0x97, 0x13, 0x96, 0x6E, 0xEA,
+0x42, 0x34, 0x20, 0xF0, 0x54, 0xC0, 0x20, 0xF0,
+0x95, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x82, 0x34,
+0x20, 0xF0, 0x96, 0xC0, 0x20, 0xF0, 0x57, 0xC0,
+0x00, 0x6A, 0x06, 0xD2, 0x0D, 0xD2, 0x0A, 0xD2,
+0x0B, 0xD2, 0x0C, 0xD2, 0x5D, 0x67, 0x20, 0xF0,
+0xD4, 0xC2, 0x97, 0xF0, 0x19, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x09, 0xF0, 0x0D, 0x4A, 0x0E, 0xD2,
+0x5D, 0x67, 0xF9, 0xC2, 0xD8, 0xC2, 0x7D, 0x67,
+0x29, 0x6A, 0x5A, 0xC3, 0x1B, 0x92, 0x09, 0x77,
+0x07, 0xD2, 0x1C, 0x92, 0x08, 0xD2, 0x0B, 0x02,
+0x09, 0xD2, 0x07, 0x61, 0x97, 0xF0, 0x1B, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x06, 0xF3, 0x15, 0x4A,
+0x0A, 0xD2, 0x06, 0x05, 0xD1, 0x18, 0x66, 0xCA,
+0x0F, 0x04, 0x5D, 0x67, 0x20, 0xF0, 0x7D, 0xA2,
+0x20, 0xF0, 0x5C, 0xA2, 0x01, 0x72, 0x02, 0x61,
+0x01, 0x6D, 0x02, 0x23, 0x00, 0x6A, 0x02, 0x6D,
+0x13, 0xD2, 0x10, 0x92, 0x01, 0x6C, 0x11, 0x96,
+0x05, 0xD2, 0x64, 0xA1, 0x12, 0x97, 0xFF, 0x69,
+0x76, 0x33, 0x8C, 0xEB, 0x04, 0xD3, 0xD1, 0x18,
+0xD7, 0x53, 0x02, 0x6C, 0x13, 0x92, 0x14, 0x2A,
+0x20, 0xF0, 0x59, 0xA0, 0x0F, 0x6B, 0x4C, 0xEB,
+0x2C, 0xEB, 0x0F, 0x73, 0x0F, 0x6A, 0x02, 0x60,
+0x41, 0x43, 0x2C, 0xEA, 0x0F, 0x6B, 0x6C, 0xEA,
+0x20, 0xF0, 0x79, 0xA0, 0x10, 0x6C, 0x8B, 0xEC,
+0x8C, 0xEB, 0x6D, 0xEA, 0x20, 0xF0, 0x59, 0xC0,
+0x19, 0x97, 0x18, 0x91, 0x17, 0x90, 0x00, 0xEF,
+0x0D, 0x63, 0x00, 0x65, 0xEF, 0x63, 0x21, 0x62,
+0x20, 0xD1, 0x1F, 0xD0, 0x04, 0x02, 0x80, 0xF0,
+0x4C, 0xA2, 0x22, 0xD4, 0x00, 0x6D, 0x1A, 0xD2,
+0x48, 0x33, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xF1, 0x0C, 0x4A, 0x49, 0xE3,
+0x20, 0x9A, 0x1A, 0x92, 0x01, 0x6B, 0x24, 0xD6,
+0x05, 0xD2, 0x44, 0xA4, 0x01, 0x6C, 0x25, 0xD7,
+0x52, 0x32, 0x6C, 0xEA, 0xD1, 0x18, 0xD7, 0x53,
+0x04, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x26, 0x93,
+0x40, 0x32, 0x40, 0x32, 0xF2, 0xF7, 0x44, 0x9A,
+0x81, 0xA3, 0x18, 0x06, 0x40, 0xEA, 0x19, 0x05,
+0x1D, 0x2A, 0x1A, 0x92, 0x01, 0x6B, 0x24, 0x96,
+0x05, 0xD2, 0x22, 0x92, 0x25, 0x97, 0x02, 0x6C,
+0x44, 0xA2, 0x02, 0x6D, 0x56, 0x32, 0x6C, 0xEA,
+0xD1, 0x18, 0xD7, 0x53, 0x04, 0xD2, 0x20, 0xF0,
+0x59, 0xA1, 0x0F, 0x6C, 0x4C, 0xEC, 0x1A, 0xD4,
+0x1A, 0x02, 0x80, 0xA2, 0x0F, 0x6A, 0x0F, 0x74,
+0x32, 0x60, 0x41, 0x44, 0x1A, 0xD2, 0x1A, 0x02,
+0x40, 0xA2, 0x2D, 0x10, 0x18, 0x94, 0xFC, 0x6B,
+0x40, 0xAC, 0x6C, 0xEA, 0x40, 0x72, 0x07, 0x61,
+0x41, 0xA4, 0x40, 0x6D, 0xAC, 0xEA, 0x1B, 0xD2,
+0x1B, 0x02, 0x40, 0xA2, 0x2F, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0xFF, 0x68, 0x1A, 0x92,
+0x01, 0x6C, 0x24, 0x96, 0x05, 0xD2, 0x22, 0x92,
+0x25, 0x97, 0x02, 0x6D, 0x44, 0xA2, 0x56, 0x32,
+0x8C, 0xEA, 0x04, 0xD2, 0xD1, 0x18, 0xD7, 0x53,
+0x02, 0x6C, 0x20, 0xF0, 0x59, 0xA1, 0x0F, 0x6C,
+0x4C, 0xEC, 0x0C, 0xEC, 0x0F, 0x74, 0x0F, 0x6A,
+0x02, 0x60, 0x41, 0x44, 0x0C, 0xEA, 0x0F, 0x6C,
+0x8C, 0xEA, 0x20, 0xF0, 0x99, 0xA1, 0x10, 0x6D,
+0xAB, 0xED, 0xAC, 0xEC, 0x8D, 0xEA, 0x20, 0xF0,
+0x59, 0xC1, 0x21, 0x97, 0x20, 0x91, 0x1F, 0x90,
+0x00, 0xEF, 0x11, 0x63, 0xF7, 0xF0, 0x01, 0x68,
+0x00, 0x30, 0x00, 0x30, 0xF2, 0xF4, 0x4C, 0x98,
+0x40, 0xEA, 0x00, 0x65, 0x18, 0x93, 0x1C, 0xD2,
+0xF2, 0xF4, 0x4C, 0x98, 0x40, 0xEA, 0x81, 0x43,
+0x18, 0x93, 0x1B, 0xD2, 0xF2, 0xF4, 0x4C, 0x98,
+0x40, 0xEA, 0x84, 0x43, 0x1C, 0x93, 0x1B, 0x95,
+0xC2, 0x67, 0xD1, 0x18, 0x9A, 0xF4, 0x83, 0x67,
+0x18, 0x97, 0x02, 0x67, 0x49, 0xE7, 0x80, 0xA2,
+0x0B, 0x24, 0x41, 0xA2, 0x09, 0x22, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x87, 0x67, 0x7D, 0x17,
+0x21, 0x6E, 0x00, 0x6D, 0x08, 0x04, 0xD1, 0x18,
+0xD2, 0xF4, 0x1B, 0xD7, 0x00, 0x6A, 0x17, 0xD2,
+0x15, 0xD2, 0x16, 0xD2, 0x26, 0x92, 0x42, 0xA2,
+0x02, 0x72, 0x4B, 0x60, 0x03, 0x72, 0x58, 0x60,
+0x01, 0x72, 0x1B, 0x97, 0x62, 0x61, 0x26, 0x92,
+0x21, 0x6E, 0x08, 0x04, 0x40, 0xA2, 0x54, 0x35,
+0x55, 0xE5, 0xB5, 0xE1, 0x7F, 0x4D, 0xD1, 0x18,
+0xE7, 0xF4, 0x01, 0x4D, 0x5D, 0x67, 0x7D, 0x67,
+0x20, 0xF0, 0x40, 0xA2, 0x60, 0xF0, 0x64, 0xAB,
+0x69, 0xE2, 0xFF, 0xF7, 0x1F, 0x6B, 0x6C, 0xEA,
+0x1C, 0xD2, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0x1C, 0x94, 0x72, 0xF4, 0x54, 0x9A,
+0xC9, 0xF3, 0xA0, 0x9B, 0x40, 0xEA, 0x00, 0x65,
+0x22, 0x67, 0x18, 0x94, 0x44, 0x2A, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF4,
+0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65, 0x06, 0x6A,
+0x7D, 0x67, 0x40, 0xF0, 0x44, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x0A, 0xF3,
+0x1C, 0x4A, 0x12, 0xD2, 0xC0, 0xF1, 0x0D, 0x6A,
+0x13, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x12, 0xF1, 0x58, 0x9A, 0x11, 0x04,
+0x23, 0x10, 0x26, 0x92, 0x04, 0x6E, 0x17, 0x04,
+0xA0, 0xA2, 0x7F, 0x4D, 0x25, 0x4D, 0xA8, 0x35,
+0xD1, 0x18, 0xE7, 0xF4, 0xB5, 0xE1, 0x5D, 0x67,
+0x60, 0xF0, 0x44, 0xAA, 0x08, 0x4A, 0xBD, 0x17,
+0x26, 0x92, 0x08, 0x6E, 0x15, 0x04, 0xA0, 0xA2,
+0x5A, 0x4D, 0xAC, 0x35, 0xD1, 0x18, 0xE7, 0xF4,
+0xB5, 0xE1, 0x5D, 0x67, 0x60, 0xF0, 0x44, 0xAA,
+0xB3, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A, 0x87, 0x67,
+0x40, 0xEA, 0x00, 0x65, 0x52, 0x17, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0x73, 0xF6,
+0x5C, 0x9F, 0xA4, 0x67, 0xD0, 0x67, 0x91, 0x67,
+0x40, 0xEA, 0x1D, 0xD7, 0x61, 0x40, 0x1B, 0xD3,
+0x1B, 0x02, 0x60, 0xA2, 0x00, 0x6C, 0x09, 0xE1,
+0x80, 0xC2, 0x26, 0x92, 0x1D, 0x97, 0x42, 0xA2,
+0x1B, 0xD7, 0x02, 0x72, 0x4A, 0x60, 0x03, 0x72,
+0x63, 0x60, 0x01, 0x72, 0x77, 0x61, 0xE2, 0x40,
+0x1D, 0xD7, 0x1D, 0x02, 0xE0, 0xA2, 0x5D, 0x67,
+0x20, 0xF0, 0x80, 0xA2, 0x1B, 0x92, 0x6D, 0xE1,
+0x80, 0xC3, 0x7D, 0x67, 0x20, 0xF0, 0xC0, 0xA3,
+0x73, 0xF6, 0x5C, 0x9A, 0xF1, 0xE1, 0x20, 0xF0,
+0x01, 0x05, 0x40, 0xEA, 0x1D, 0xD7, 0x5D, 0x67,
+0x20, 0xF0, 0x60, 0xA2, 0x1D, 0x97, 0x6D, 0xE7,
+0x1D, 0xD3, 0x1D, 0x02, 0x60, 0xA2, 0x26, 0x92,
+0xFF, 0x6C, 0x42, 0xA2, 0xFF, 0x4A, 0x8C, 0xEA,
+0x02, 0x5A, 0x0D, 0x60, 0x9D, 0x67, 0x1B, 0x92,
+0x60, 0xF0, 0xC4, 0xAC, 0x18, 0x94, 0x73, 0xF6,
+0x5C, 0x9A, 0xFE, 0x4E, 0xA2, 0x40, 0xB5, 0xE4,
+0x1B, 0xE6, 0x40, 0xEA, 0x71, 0xE1, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x18, 0x94,
+0x72, 0xF4, 0x50, 0x9A, 0x40, 0xEA, 0x00, 0x65,
+0x1A, 0x92, 0x25, 0x93, 0x1C, 0x96, 0x06, 0xD2,
+0x24, 0x92, 0x22, 0x94, 0x05, 0xD3, 0x04, 0xD2,
+0x09, 0x6F, 0xD1, 0x18, 0xDF, 0x53, 0xB1, 0x67,
+0xF0, 0x16, 0x1B, 0x92, 0xF7, 0xF0, 0x01, 0x6D,
+0xA0, 0x35, 0x73, 0xF6, 0x5C, 0x9A, 0xA0, 0x35,
+0x71, 0xE1, 0xC6, 0xF4, 0x10, 0x4D, 0x40, 0xEA,
+0x09, 0x6E, 0x87, 0x40, 0x1B, 0x92, 0x03, 0x4C,
+0x1D, 0x03, 0x1D, 0xD4, 0x80, 0xA3, 0x73, 0xF6,
+0x5C, 0x9A, 0x04, 0x6E, 0x17, 0x05, 0x40, 0xEA,
+0x91, 0xE1, 0x67, 0x40, 0x07, 0x4B, 0xB8, 0x17,
+0x1B, 0x92, 0x9D, 0x67, 0x60, 0xF0, 0xC4, 0xAC,
+0x18, 0x94, 0x73, 0xF6, 0x5C, 0x9A, 0x1B, 0xE6,
+0x15, 0xE4, 0x71, 0xE1, 0x40, 0xEA, 0x1D, 0xD3,
+0x1B, 0x92, 0x87, 0x41, 0x06, 0x6E, 0x73, 0xF6,
+0x5C, 0x9A, 0x15, 0x05, 0x40, 0xEA, 0x09, 0x4C,
+0x1D, 0x93, 0xA5, 0x17, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A,
+0x40, 0xEA, 0x18, 0x94, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A,
+0x91, 0x67, 0x5A, 0x17, 0xF7, 0x63, 0x11, 0x62,
+0x10, 0xD1, 0x0F, 0xD0, 0x04, 0x02, 0x40, 0xF0,
+0x10, 0xA2, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x1C, 0x65, 0x6D, 0xF1, 0x0C, 0x4B,
+0x08, 0x34, 0x6D, 0xE4, 0x58, 0x67, 0x20, 0x9B,
+0x14, 0xD6, 0x05, 0xD0, 0x84, 0xA2, 0x01, 0x6B,
+0x16, 0x96, 0x17, 0x97, 0x92, 0x34, 0x6C, 0xEC,
+0x0B, 0xD2, 0x5D, 0x67, 0x20, 0xF0, 0x68, 0xC2,
+0x0C, 0xD5, 0x04, 0xD4, 0x00, 0x6D, 0xD1, 0x18,
+0xD7, 0x53, 0x01, 0x6C, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0x0C, 0x92, 0xF2, 0xF7,
+0xE4, 0x9C, 0x08, 0x06, 0x09, 0x05, 0x40, 0xEF,
+0x82, 0x67, 0x0B, 0x93, 0x1B, 0x65, 0x2B, 0x2A,
+0x05, 0xD0, 0x44, 0xA3, 0x7D, 0x67, 0x20, 0xF0,
+0x68, 0xA3, 0x56, 0x32, 0x16, 0x96, 0x17, 0x97,
+0x4C, 0xEB, 0x04, 0xD3, 0x02, 0x6D, 0xD1, 0x18,
+0xD7, 0x53, 0x02, 0x6C, 0x20, 0xF0, 0x59, 0xA1,
+0x0F, 0x6B, 0x4C, 0xEB, 0x0A, 0xD3, 0x0A, 0x02,
+0x60, 0xA2, 0x0F, 0x6A, 0x0F, 0x73, 0x04, 0x60,
+0x41, 0x43, 0x0A, 0xD2, 0x0A, 0x02, 0x40, 0xA2,
+0x0F, 0x6B, 0x6C, 0xEA, 0x20, 0xF0, 0x79, 0xA1,
+0x10, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB, 0x6D, 0xEA,
+0x20, 0xF0, 0x59, 0xC1, 0x11, 0x97, 0x10, 0x91,
+0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63, 0x5D, 0x67,
+0xD2, 0xAA, 0x17, 0x93, 0x16, 0x92, 0x14, 0x97,
+0x08, 0x95, 0x06, 0xD0, 0x04, 0xD2, 0x05, 0xD3,
+0xD1, 0x18, 0xDF, 0x53, 0x98, 0x67, 0xEE, 0x17,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0x5D, 0x67, 0x40, 0xF0, 0x08, 0xA2, 0x24, 0x67,
+0x0F, 0xD5, 0x10, 0xD6, 0x11, 0xD7, 0x11, 0x6C,
+0x01, 0x28, 0x10, 0x6C, 0xB1, 0x18, 0x90, 0xD1,
+0x00, 0x65, 0x7D, 0x67, 0x20, 0xF0, 0x7C, 0xA3,
+0x00, 0x6A, 0x08, 0xD2, 0x06, 0xD2, 0x07, 0xD2,
+0x5D, 0x67, 0x71, 0xC2, 0x7D, 0x67, 0x40, 0xF0,
+0x64, 0xAB, 0x30, 0xC2, 0x69, 0xCA, 0x7D, 0x67,
+0x00, 0x6A, 0x4A, 0xCB, 0x5D, 0x67, 0x0B, 0xCA,
+0x00, 0x6A, 0x4C, 0xCB, 0x40, 0xF0, 0x60, 0xA3,
+0x5D, 0x67, 0x7E, 0xC2, 0x7D, 0x67, 0x40, 0xF0,
+0x6C, 0xA3, 0x20, 0xF0, 0x61, 0xC2, 0x13, 0x92,
+0x7D, 0x67, 0x42, 0x32, 0x20, 0xF0, 0x42, 0xC3,
+0x42, 0x32, 0x20, 0xF0, 0x43, 0xC3, 0x13, 0x92,
+0x00, 0xF6, 0x42, 0x30, 0x5D, 0x67, 0x20, 0xF0,
+0x04, 0xC2, 0xD1, 0x18, 0xCD, 0x8A, 0x00, 0x65,
+0x01, 0x6B, 0x0A, 0x22, 0x97, 0xF0, 0x19, 0x6D,
+0xA0, 0x35, 0xA0, 0x35, 0xA9, 0xF0, 0x19, 0x4D,
+0xD1, 0x18, 0x9A, 0x90, 0x04, 0x04, 0x62, 0x67,
+0x7B, 0x6A, 0x4B, 0xEA, 0x4C, 0xEB, 0xFF, 0x6A,
+0x4C, 0xEB, 0x01, 0x5B, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x78, 0x67, 0x6C, 0xEA, 0x00, 0xEF,
+0x07, 0x63, 0x00, 0x65, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x0C, 0xD4, 0x6D, 0xF1,
+0x0C, 0x4A, 0x88, 0x34, 0x49, 0xE4, 0x60, 0x9A,
+0xFF, 0x69, 0x41, 0xA3, 0xA0, 0xA3, 0x82, 0xA3,
+0x40, 0x32, 0xAD, 0xEA, 0xA3, 0xA3, 0x80, 0x34,
+0x80, 0x34, 0x8D, 0xEA, 0x00, 0xF6, 0xA0, 0x35,
+0x4D, 0xED, 0x55, 0xA3, 0xC5, 0xA3, 0x86, 0xA3,
+0x01, 0x4A, 0x56, 0xC3, 0x44, 0xA3, 0xC0, 0x36,
+0x80, 0x34, 0x4D, 0xEE, 0x47, 0xA3, 0x80, 0x34,
+0x8D, 0xEE, 0x00, 0xF6, 0x40, 0x32, 0xCD, 0xEA,
+0xED, 0xA3, 0x48, 0xC3, 0x42, 0x34, 0x00, 0xF6,
+0x42, 0x32, 0x89, 0xC3, 0x4B, 0xC3, 0x82, 0x34,
+0x4C, 0xA3, 0x8A, 0xC3, 0x8E, 0xA3, 0xE0, 0x37,
+0x4D, 0xEF, 0x4F, 0xA3, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEF, 0x00, 0xF6, 0x40, 0x32, 0xED, 0xEA,
+0x50, 0xC3, 0x42, 0x34, 0x00, 0xF6, 0x42, 0x32,
+0x00, 0x6E, 0x53, 0xC3, 0x20, 0xF0, 0x59, 0xA3,
+0xD7, 0xC3, 0xDB, 0xC3, 0x20, 0xF0, 0xDA, 0xC3,
+0x20, 0xF0, 0xDC, 0xC3, 0x91, 0xC3, 0x82, 0x34,
+0x92, 0xC3, 0x02, 0xA5, 0x0F, 0x6C, 0xF7, 0xF0,
+0x01, 0x6F, 0x8C, 0xEA, 0xE0, 0x37, 0x20, 0xF0,
+0x59, 0xC3, 0xE0, 0x37, 0x07, 0x6A, 0x06, 0xD3,
+0x4C, 0xE8, 0xF3, 0xF3, 0x4C, 0x9F, 0x0C, 0x94,
+0x2C, 0xE8, 0xB0, 0x67, 0x01, 0x6E, 0x40, 0xEA,
+0x05, 0xD7, 0x05, 0x97, 0x04, 0xD2, 0x0C, 0x94,
+0xF3, 0xF3, 0x4C, 0x9F, 0xB0, 0x67, 0x40, 0xEA,
+0x00, 0x6E, 0x71, 0x67, 0x4C, 0xEB, 0xFB, 0x65,
+0x83, 0x67, 0x06, 0x93, 0x42, 0x35, 0x00, 0xF6,
+0x42, 0x32, 0x3A, 0x65, 0x5F, 0xC3, 0x04, 0x92,
+0x9C, 0xC3, 0x04, 0x96, 0x42, 0x34, 0x44, 0x67,
+0x2C, 0xEA, 0x20, 0xF0, 0x41, 0xC3, 0x1A, 0x65,
+0x04, 0x92, 0xE5, 0x67, 0x82, 0x34, 0xA2, 0x35,
+0x2C, 0xEF, 0x2C, 0xEE, 0x2C, 0xED, 0x2C, 0xEC,
+0x00, 0xF6, 0x42, 0x31, 0x5F, 0x67, 0x20, 0xF0,
+0x44, 0xC3, 0x59, 0x67, 0x20, 0xF0, 0x47, 0xC3,
+0x58, 0x67, 0x20, 0xF0, 0x49, 0xC3, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xFD, 0xC3,
+0x20, 0xF0, 0xE5, 0xC3, 0x20, 0xF0, 0xC0, 0xC3,
+0x20, 0xF0, 0x23, 0xC3, 0x20, 0xF0, 0xC8, 0xC3,
+0x20, 0xF0, 0x2B, 0xC3, 0xBE, 0xC3, 0x20, 0xF0,
+0x82, 0xC3, 0x20, 0xF0, 0xA6, 0xC3, 0x20, 0xF0,
+0x8A, 0xC3, 0x15, 0xF0, 0x08, 0x4A, 0x9D, 0xA2,
+0xBC, 0xA2, 0xF7, 0xF0, 0x01, 0x69, 0x80, 0x34,
+0xAD, 0xEC, 0xBE, 0xA2, 0x20, 0x31, 0x20, 0x31,
+0xA0, 0x35, 0xA0, 0x35, 0x8D, 0xED, 0x9F, 0xA2,
+0x00, 0xF6, 0x80, 0x34, 0xD1, 0x18, 0xCE, 0x7D,
+0xAD, 0xEC, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x97, 0xF0, 0x19, 0x6E, 0x0C, 0x94,
+0xD3, 0xF3, 0x40, 0x9A, 0xC0, 0x36, 0xC0, 0x36,
+0xAE, 0xF0, 0x19, 0x4E, 0x40, 0xEA, 0xB0, 0x67,
+0xD2, 0xF4, 0x50, 0x99, 0x19, 0xF6, 0x06, 0x68,
+0x40, 0xEA, 0x90, 0x67, 0xF7, 0xF0, 0x01, 0x6E,
+0x01, 0x6D, 0x4D, 0xED, 0xC0, 0x36, 0xC0, 0x36,
+0x04, 0x02, 0x04, 0xD5, 0xD2, 0xF4, 0x64, 0x9E,
+0xA0, 0xAA, 0x90, 0x67, 0x40, 0xEB, 0x07, 0xD6,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0xD2, 0xF4, 0x40, 0x9F, 0x2F, 0x6D, 0xF9, 0xF6,
+0x00, 0x6C, 0x40, 0xEA, 0x06, 0xD7, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0xF7, 0xF0,
+0x01, 0x6C, 0xD2, 0xF4, 0x48, 0x9B, 0x80, 0x34,
+0x80, 0x34, 0xC9, 0xF3, 0xA4, 0x9C, 0x1A, 0x65,
+0xF9, 0xF6, 0x04, 0x6A, 0x82, 0x67, 0x04, 0xD2,
+0x58, 0x67, 0x40, 0xEA, 0x05, 0xD3, 0x06, 0x97,
+0x2E, 0x6D, 0xF9, 0xF6, 0x00, 0x6C, 0xD2, 0xF4,
+0xE0, 0x9F, 0x40, 0xEF, 0x00, 0x65, 0x05, 0x93,
+0xF7, 0xF0, 0x01, 0x6C, 0x80, 0x34, 0x04, 0x92,
+0x80, 0x34, 0xC9, 0xF3, 0xA8, 0x9C, 0xD2, 0xF4,
+0x68, 0x9B, 0x40, 0xEB, 0x82, 0x67, 0xD2, 0xF4,
+0x50, 0x99, 0x40, 0xEA, 0x90, 0x67, 0x02, 0x6D,
+0x07, 0x96, 0xAB, 0xED, 0x4C, 0xED, 0x04, 0xD5,
+0x04, 0x02, 0xD2, 0xF4, 0x64, 0x9E, 0xA0, 0xAA,
+0x40, 0xEB, 0x90, 0x67, 0x0B, 0x97, 0x0A, 0x91,
+0x09, 0x90, 0x00, 0xEF, 0x06, 0x63, 0x00, 0x65,
+0x68, 0x2C, 0xFA, 0x63, 0x0B, 0x62, 0x0A, 0xD1,
+0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xF1, 0x0C, 0x9A, 0x55, 0xA0,
+0x57, 0x22, 0xA5, 0xA0, 0x44, 0xA0, 0x66, 0xA0,
+0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xED, 0x00, 0xF6, 0x40, 0x32,
+0xAD, 0xEA, 0x4A, 0x22, 0x20, 0xF0, 0x5B, 0xA0,
+0x47, 0x22, 0x54, 0xA0, 0x01, 0x72, 0x44, 0x60,
+0x41, 0xA0, 0x20, 0xA0, 0x62, 0xA0, 0x40, 0x32,
+0x2D, 0xEA, 0x23, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEA, 0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9,
+0xD1, 0x18, 0x7D, 0x55, 0x06, 0xD4, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xD3, 0xF3,
+0x44, 0x9A, 0x20, 0xF0, 0xE1, 0xA0, 0xDD, 0xA0,
+0x1A, 0x65, 0x20, 0xF0, 0x40, 0xA0, 0x7C, 0xA0,
+0xE0, 0x37, 0x4D, 0xEF, 0xC0, 0x36, 0x20, 0xF0,
+0x42, 0xA0, 0x6D, 0xEE, 0x01, 0x6C, 0x7E, 0xA0,
+0x94, 0xC0, 0xA2, 0xA1, 0x40, 0x32, 0x06, 0x91,
+0x60, 0x33, 0x40, 0x32, 0xED, 0xEA, 0x60, 0x33,
+0x20, 0xF0, 0xE3, 0xA0, 0xCD, 0xEB, 0xDF, 0xA0,
+0x05, 0xD1, 0x97, 0xF0, 0x19, 0x69, 0x20, 0x31,
+0x00, 0xF6, 0xE0, 0x37, 0x20, 0x31, 0xAE, 0x30,
+0x00, 0xF6, 0xC0, 0x36, 0xED, 0xF0, 0x19, 0x49,
+0x4D, 0xEF, 0x07, 0x6A, 0x4C, 0xED, 0x04, 0xD1,
+0x6D, 0xEE, 0x58, 0x67, 0x40, 0xEA, 0x0C, 0xEC,
+0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90, 0x00, 0xEF,
+0x06, 0x63, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x65, 0xA4, 0x10, 0x6A, 0xE5, 0x67, 0x4C, 0xEB,
+0xA6, 0x67, 0xFF, 0x6E, 0xCC, 0xEB, 0x80, 0xF0,
+0x00, 0x23, 0x47, 0xA4, 0x02, 0x6B, 0x6C, 0xEA,
+0xCC, 0xEA, 0x7B, 0x22, 0xFC, 0x63, 0x07, 0x62,
+0x06, 0xD1, 0x05, 0xD0, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x15, 0xF0, 0x08, 0x4B,
+0x85, 0xA3, 0x44, 0xA3, 0xC6, 0xA3, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA3, 0xC0, 0x36, 0xC0, 0x36,
+0xCD, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x20, 0xF0, 0x81, 0xA2, 0x20, 0xF0, 0xC0, 0xA2,
+0x20, 0xF0, 0x62, 0xA2, 0x80, 0x34, 0x8D, 0xEE,
+0x20, 0xF0, 0x83, 0xA2, 0x60, 0x33, 0x60, 0x33,
+0xCD, 0xEB, 0x00, 0xF6, 0x80, 0x34, 0x8D, 0xEB,
+0xE8, 0x32, 0x69, 0xE2, 0x65, 0xA2, 0x86, 0xA2,
+0xC4, 0xA2, 0x47, 0xA2, 0x60, 0x33, 0x80, 0x34,
+0x6D, 0xEE, 0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x61, 0xA2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x68, 0x33,
+0x6D, 0xF1, 0x0C, 0x4A, 0x49, 0xE3, 0x00, 0x9A,
+0x00, 0x6E, 0x20, 0xF0, 0xBE, 0xC0, 0x06, 0x25,
+0x97, 0xF0, 0x19, 0x6E, 0xC0, 0x36, 0xC0, 0x36,
+0x48, 0xF7, 0x19, 0x4E, 0x87, 0x67, 0xD1, 0x18,
+0x78, 0xCB, 0x00, 0x6F, 0x2C, 0x2A, 0x20, 0xF0,
+0x5E, 0xC0, 0x20, 0xF0, 0x59, 0xA0, 0x0F, 0x6B,
+0xFF, 0x6C, 0x4C, 0xEB, 0x8C, 0xEB, 0x0F, 0x73,
+0x0F, 0x6A, 0x02, 0x60, 0x41, 0x43, 0x8C, 0xEA,
+0x0F, 0x6B, 0x6C, 0xEA, 0x20, 0xF0, 0x79, 0xA0,
+0x10, 0x6C, 0x8B, 0xEC, 0x8C, 0xEB, 0x6D, 0xEA,
+0x20, 0xF0, 0x59, 0xC0, 0x07, 0x97, 0x06, 0x91,
+0x05, 0x90, 0x00, 0xEF, 0x04, 0x63, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xB2, 0xF2,
+0x5C, 0x9A, 0x01, 0x6C, 0x40, 0xEA, 0xFF, 0x49,
+0xFF, 0x6A, 0x4C, 0xE9, 0xEF, 0x21, 0x20, 0xF0,
+0x5E, 0xA0, 0xF1, 0x2A, 0xEB, 0x17, 0x0A, 0x69,
+0xFA, 0x17, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF6, 0x63, 0x13, 0x62, 0x12, 0xD0, 0x34, 0x6E,
+0x04, 0x67, 0x00, 0x6D, 0xD1, 0x18, 0xD2, 0xF4,
+0x04, 0x04, 0x90, 0x67, 0x34, 0x6E, 0xD1, 0x18,
+0xE7, 0xF4, 0x04, 0x05, 0x50, 0x67, 0x13, 0x97,
+0x12, 0x90, 0x00, 0xEF, 0x0A, 0x63, 0x00, 0x65,
+0xF7, 0x63, 0x11, 0x62, 0x10, 0xD1, 0x0F, 0xD0,
+0x06, 0xD6, 0x5D, 0x67, 0x58, 0xA2, 0x04, 0x67,
+0x07, 0xD7, 0x20, 0xF0, 0x4C, 0xC0, 0xC2, 0x32,
+0x20, 0xF0, 0x4D, 0xC0, 0x42, 0x32, 0x20, 0xF0,
+0x4E, 0xC0, 0x00, 0xF6, 0xC2, 0x32, 0x20, 0xF0,
+0x4F, 0xC0, 0x5D, 0x67, 0x5C, 0xA2, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x20, 0xF0, 0x50, 0xC0,
+0xE2, 0x32, 0x20, 0xF0, 0x51, 0xC0, 0x42, 0x32,
+0x20, 0x31, 0x20, 0xF0, 0x52, 0xC0, 0x00, 0xF6,
+0xE2, 0x32, 0x00, 0x6C, 0x20, 0xF0, 0x53, 0xC0,
+0x93, 0xF6, 0x40, 0x99, 0x99, 0xC0, 0x98, 0xC0,
+0x20, 0xF0, 0x98, 0xC0, 0x87, 0x40, 0x13, 0xD5,
+0x08, 0x6E, 0x00, 0x6D, 0x40, 0xEA, 0x41, 0x4C,
+0x93, 0xF6, 0x40, 0x99, 0x87, 0x40, 0x10, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x49, 0x4C, 0x93, 0xF6,
+0x40, 0x99, 0x87, 0x40, 0x10, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x59, 0x4C, 0x93, 0xF6, 0x40, 0x99,
+0x87, 0x40, 0x00, 0x6D, 0x69, 0x4C, 0x40, 0xEA,
+0x10, 0x6E, 0x00, 0x6A, 0x20, 0xF0, 0x54, 0xC0,
+0x20, 0xF0, 0x55, 0xC0, 0x20, 0xF0, 0x56, 0xC0,
+0x20, 0xF0, 0x57, 0xC0, 0x20, 0xF0, 0x59, 0xA0,
+0x10, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0xA1, 0xA0,
+0x20, 0xF0, 0x59, 0xC0, 0x40, 0xA0, 0x82, 0xA0,
+0x13, 0x93, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA0,
+0x80, 0x34, 0x72, 0xA3, 0x80, 0x34, 0x8D, 0xED,
+0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA, 0x01, 0x2B,
+0x04, 0x6B, 0x42, 0xA2, 0x01, 0x6D, 0x4E, 0x34,
+0xAC, 0xEC, 0x08, 0xD4, 0xA0, 0xF0, 0x1F, 0x2C,
+0x07, 0x6C, 0x8C, 0xEA, 0xFF, 0x6C, 0x8C, 0xEA,
+0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x97, 0xF0, 0x19, 0x6C, 0x40, 0x32, 0x80, 0x34,
+0x09, 0x95, 0xD3, 0xF3, 0x40, 0x9A, 0x80, 0x34,
+0xF7, 0xF0, 0x01, 0x69, 0xAE, 0xF0, 0x19, 0x4C,
+0x20, 0x31, 0xC4, 0x67, 0x20, 0x31, 0x0A, 0xD4,
+0x00, 0x6C, 0x40, 0xEA, 0x0D, 0xD3, 0xD2, 0xF4,
+0x50, 0x99, 0x19, 0xF6, 0x06, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6E, 0x01, 0x6D,
+0x4D, 0xED, 0xC0, 0x36, 0xC0, 0x36, 0x0B, 0x02,
+0x0B, 0xD5, 0xD2, 0xF4, 0xE4, 0x9E, 0xA0, 0xAA,
+0x19, 0xF6, 0x06, 0x6C, 0x40, 0xEF, 0x0C, 0xD6,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0xD2, 0xF4, 0x40, 0x9F, 0x2F, 0x6D, 0xF9, 0xF6,
+0x00, 0x6C, 0x40, 0xEA, 0x0B, 0xD7, 0xD2, 0xF4,
+0x50, 0x99, 0xF9, 0xF6, 0x02, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x0B, 0x97, 0x42, 0x34, 0x40, 0xF0,
+0x40, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x40, 0xF0,
+0x43, 0xC0, 0xD2, 0xF4, 0x40, 0x9F, 0x40, 0xF0,
+0x81, 0xC0, 0x82, 0x34, 0x40, 0xF0, 0x82, 0xC0,
+0xF9, 0xF6, 0x00, 0x6C, 0x40, 0xEA, 0x2E, 0x6D,
+0xD2, 0xF4, 0x50, 0x99, 0xF9, 0xF6, 0x02, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x40, 0xF0, 0xE1, 0xA0,
+0x40, 0xF0, 0x80, 0xA0, 0x40, 0xF0, 0xA2, 0xA0,
+0xE0, 0x37, 0x8D, 0xEF, 0x40, 0xF0, 0x83, 0xA0,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEF, 0x00, 0xF6,
+0x80, 0x34, 0xED, 0xEC, 0x49, 0xE4, 0x42, 0x34,
+0x40, 0xF0, 0x40, 0xC0, 0x00, 0xF6, 0x42, 0x32,
+0x40, 0xF0, 0x43, 0xC0, 0xD2, 0xF4, 0x50, 0x99,
+0x40, 0xF0, 0x81, 0xC0, 0x82, 0x34, 0x40, 0xF0,
+0x82, 0xC0, 0x19, 0xF6, 0x06, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x02, 0x6D, 0x0C, 0x96, 0xAB, 0xED,
+0x4C, 0xED, 0x0B, 0xD5, 0x0B, 0x02, 0xA0, 0xAA,
+0xD2, 0xF4, 0xC4, 0x9E, 0x19, 0xF6, 0x06, 0x6C,
+0x40, 0xEE, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x0D, 0x93, 0x40, 0x32, 0xD3, 0xF3,
+0x44, 0x9A, 0x68, 0x34, 0x3A, 0x65, 0x69, 0xE4,
+0x06, 0x93, 0x83, 0xEA, 0x80, 0xF5, 0x42, 0x35,
+0x40, 0x32, 0x48, 0x32, 0x98, 0x67, 0x79, 0xE2,
+0x80, 0x34, 0x07, 0x93, 0x88, 0x37, 0xAD, 0xEF,
+0x7D, 0xE7, 0x08, 0x93, 0x43, 0xEE, 0x09, 0x95,
+0x05, 0xD3, 0x0A, 0x93, 0x58, 0x67, 0xFD, 0xE2,
+0x04, 0xD3, 0x59, 0x67, 0x40, 0xEA, 0x00, 0x6C,
+0x08, 0x92, 0x01, 0x6B, 0x06, 0x96, 0x05, 0xD2,
+0x13, 0x92, 0x07, 0x97, 0x01, 0x6D, 0x44, 0xA2,
+0x03, 0x6C, 0x5A, 0x32, 0x6C, 0xEA, 0xD1, 0x18,
+0xD7, 0x53, 0x04, 0xD2, 0xBA, 0xA0, 0x13, 0x94,
+0xD1, 0x18, 0x46, 0x56, 0x00, 0x6E, 0x11, 0x97,
+0x10, 0x91, 0x0F, 0x90, 0x00, 0xEF, 0x09, 0x63,
+0x55, 0xA4, 0x76, 0xA4, 0x62, 0xEA, 0x28, 0x61,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0x40, 0xA4, 0x04, 0x67, 0x81, 0xA4, 0x62, 0xA0,
+0x25, 0x67, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA0,
+0x60, 0x33, 0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x42, 0xA2, 0x01, 0x6B,
+0x04, 0x6C, 0x4E, 0x32, 0x6C, 0xEA, 0x05, 0xD2,
+0x44, 0xA5, 0x00, 0x6D, 0x5E, 0x32, 0xD1, 0x18,
+0xD7, 0x53, 0x04, 0xD2, 0xBA, 0xA0, 0x91, 0x67,
+0xD1, 0x18, 0x46, 0x56, 0x01, 0x6E, 0x09, 0x97,
+0x08, 0x91, 0x07, 0x90, 0x00, 0xEF, 0x05, 0x63,
+0x20, 0xE8, 0x00, 0x65, 0xFC, 0x63, 0x07, 0x62,
+0x5D, 0x67, 0x01, 0x75, 0x94, 0xC2, 0x58, 0x67,
+0x12, 0x22, 0x09, 0x25, 0x02, 0x75, 0x0D, 0x60,
+0x0B, 0x6D, 0xB1, 0x18, 0xC2, 0xE8, 0x04, 0x04,
+0x07, 0x97, 0x00, 0xEF, 0x04, 0x63, 0x01, 0x6A,
+0x4B, 0xEA, 0x7D, 0x67, 0x48, 0xCB, 0x49, 0xCB,
+0xF3, 0x17, 0x00, 0xF7, 0x00, 0x6A, 0x7D, 0x67,
+0x48, 0xCB, 0x01, 0x6A, 0x4B, 0xEA, 0xF7, 0x17,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xA8, 0x33, 0x6D, 0xF1, 0x0C, 0x4A, 0x49, 0xE3,
+0x00, 0x9A, 0x0F, 0xD5, 0x0E, 0xD4, 0x81, 0xA0,
+0xA0, 0xA0, 0x42, 0xA0, 0x80, 0x34, 0x8D, 0xED,
+0x83, 0xA0, 0x40, 0x32, 0x40, 0x32, 0xAD, 0xEA,
+0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC, 0x06, 0xD4,
+0x89, 0xA0, 0xA8, 0xA0, 0x4A, 0xA0, 0x80, 0x34,
+0x8D, 0xED, 0x8B, 0xA0, 0x40, 0x32, 0x40, 0x32,
+0xAD, 0xEA, 0x00, 0xF6, 0x80, 0x34, 0x4D, 0xEC,
+0x06, 0x92, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x42, 0xA2, 0x60, 0x33, 0x07, 0x69, 0x97, 0xF0,
+0x19, 0x6E, 0x4C, 0xE9, 0x08, 0xD4, 0xD3, 0xF3,
+0x40, 0x9B, 0x0F, 0x94, 0xC0, 0x36, 0xC0, 0x36,
+0xED, 0xF0, 0x19, 0x4E, 0xB1, 0x67, 0x40, 0xEA,
+0x07, 0xD3, 0x07, 0x93, 0x97, 0xF0, 0x19, 0x6E,
+0xC0, 0x36, 0xD3, 0xF3, 0x40, 0x9B, 0x0F, 0x94,
+0xC0, 0x36, 0xAE, 0xF0, 0x19, 0x4E, 0x40, 0xEA,
+0xB1, 0x67, 0x20, 0xF0, 0x5C, 0xA0, 0x59, 0x22,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0xF3, 0xF3, 0x4C, 0x9F, 0x0F, 0x94, 0xB1, 0x67,
+0x01, 0x6E, 0x40, 0xEA, 0x09, 0xD7, 0x09, 0x97,
+0x07, 0xD2, 0x0F, 0x94, 0xF3, 0xF3, 0x4C, 0x9F,
+0xB1, 0x67, 0x40, 0xEA, 0x00, 0x6E, 0x22, 0x67,
+0x55, 0xA0, 0x56, 0xC0, 0x0E, 0x92, 0x02, 0x72,
+0x37, 0x61, 0x0F, 0x94, 0xD1, 0x18, 0x47, 0x57,
+0x00, 0x6D, 0x00, 0x6A, 0x54, 0xC0, 0x0F, 0x92,
+0x07, 0x97, 0x01, 0x6C, 0x05, 0xD2, 0x06, 0x92,
+0x0E, 0x95, 0xD1, 0x67, 0x44, 0xA2, 0x8C, 0xEA,
+0x04, 0xD2, 0xD1, 0x18, 0xD7, 0x53, 0x05, 0x6C,
+0x9A, 0xA0, 0x00, 0x6E, 0xD1, 0x18, 0xD4, 0xD9,
+0x02, 0x6D, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF0, 0x50, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xC5, 0xA0, 0x84, 0xA0, 0x22, 0x67,
+0x46, 0xA0, 0xC0, 0x36, 0x8D, 0xEE, 0x87, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0xCD, 0xEC, 0x17, 0x2C, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x72, 0xF0,
+0x54, 0x9A, 0x40, 0xEA, 0x91, 0x67, 0x09, 0x10,
+0x07, 0x92, 0x08, 0x96, 0x06, 0x95, 0x04, 0xD1,
+0x05, 0xD2, 0xD1, 0x18, 0xB5, 0x57, 0x90, 0x67,
+0xC0, 0x2A, 0x0D, 0x97, 0x0C, 0x91, 0x0B, 0x90,
+0x00, 0xEF, 0x07, 0x63, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF4, 0x50, 0x9A,
+0x40, 0xEA, 0x00, 0x65, 0x00, 0x6C, 0x84, 0xC0,
+0x85, 0xC0, 0x86, 0xC0, 0x87, 0xC0, 0x88, 0xC0,
+0x89, 0xC0, 0x8A, 0xC0, 0x8B, 0xC0, 0x95, 0xC0,
+0xD6, 0x17, 0x00, 0x65, 0xF5, 0x63, 0x15, 0x62,
+0x14, 0xD1, 0x13, 0xD0, 0x04, 0xA5, 0x02, 0x6A,
+0xFF, 0x6F, 0x4C, 0xE8, 0xEC, 0xE8, 0x01, 0x6A,
+0x77, 0x20, 0x46, 0xA5, 0x65, 0x67, 0x04, 0x67,
+0x0D, 0xD2, 0x45, 0xA5, 0x1E, 0x65, 0x1A, 0x96,
+0x0B, 0xD2, 0x44, 0xA5, 0xA2, 0xA5, 0x10, 0xD3,
+0x56, 0x32, 0xEC, 0xEA, 0x0E, 0xD2, 0xBA, 0x32,
+0xEC, 0xEA, 0x0C, 0xD2, 0xAE, 0x35, 0x55, 0xA4,
+0x01, 0x6C, 0xAC, 0xEC, 0xEC, 0xEC, 0x1B, 0x97,
+0x0A, 0xD4, 0xB8, 0x67, 0x90, 0x67, 0xD1, 0x18,
+0x30, 0x57, 0x0F, 0xD2, 0xA5, 0xA0, 0x24, 0xA0,
+0x86, 0xA0, 0x0F, 0x92, 0xA0, 0x35, 0x2D, 0xED,
+0x80, 0x34, 0x27, 0xA0, 0x80, 0x34, 0x8D, 0xED,
+0x4C, 0x34, 0x00, 0xF6, 0x20, 0x31, 0x53, 0xE4,
+0xAD, 0xE9, 0x88, 0x34, 0x85, 0xE1, 0x22, 0x34,
+0x89, 0xC0, 0x82, 0x34, 0x8A, 0xC0, 0xCD, 0xA0,
+0x00, 0xF6, 0x22, 0x34, 0x8B, 0xC0, 0x8C, 0xA0,
+0xAE, 0xA0, 0xC0, 0x36, 0x8D, 0xEE, 0x8F, 0xA0,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0xCD, 0xEC, 0x28, 0xC0, 0x10, 0x93,
+0x02, 0x24, 0x48, 0x32, 0x51, 0xE4, 0x82, 0x32,
+0x51, 0xC0, 0x42, 0x32, 0x52, 0xC0, 0x00, 0x6A,
+0x59, 0xC0, 0x20, 0xF0, 0x5C, 0xA0, 0x90, 0xC0,
+0x00, 0xF6, 0x82, 0x34, 0x01, 0x5A, 0x58, 0x67,
+0x02, 0x4A, 0x0F, 0xD2, 0x5D, 0x67, 0x20, 0xF0,
+0x4C, 0xA2, 0x93, 0xC0, 0x0A, 0x94, 0x20, 0xF0,
+0x5C, 0xC0, 0x5D, 0x67, 0x20, 0xF0, 0x50, 0xA2,
+0x10, 0xD3, 0x20, 0xF0, 0x5D, 0xC0, 0xD1, 0x18,
+0x47, 0x57, 0x01, 0x6D, 0x0F, 0x92, 0x0E, 0x97,
+0x0A, 0x96, 0x05, 0xD2, 0x0C, 0x92, 0x0D, 0x95,
+0x0B, 0x94, 0xD1, 0x18, 0x50, 0x55, 0x04, 0xD2,
+0x10, 0x93, 0x0B, 0x2A, 0x0A, 0x95, 0x54, 0xC0,
+0xD1, 0x18, 0x56, 0x57, 0x02, 0x6C, 0x00, 0x6A,
+0x15, 0x97, 0x14, 0x91, 0x13, 0x90, 0x00, 0xEF,
+0x0B, 0x63, 0x82, 0xA3, 0x01, 0x6A, 0x00, 0x6D,
+0x8E, 0x34, 0x4C, 0xEC, 0xFF, 0x6A, 0x4C, 0xEC,
+0xD1, 0x18, 0x47, 0x57, 0x0B, 0xD3, 0x55, 0xA0,
+0x0B, 0x93, 0x01, 0x4A, 0x56, 0xC0, 0x94, 0xA3,
+0x5D, 0x67, 0x20, 0xF0, 0x80, 0xC2, 0x95, 0xA3,
+0x20, 0xF0, 0x81, 0xC2, 0x96, 0xA3, 0x20, 0xF0,
+0x82, 0xC2, 0x97, 0xA3, 0x20, 0xF0, 0x83, 0xC2,
+0x00, 0x6A, 0x87, 0xA3, 0x83, 0xEA, 0x02, 0x61,
+0x01, 0x6A, 0xDA, 0x17, 0x0B, 0xD3, 0x7D, 0x67,
+0x51, 0xE3, 0x02, 0x67, 0x0A, 0x92, 0x20, 0xF0,
+0xA0, 0xA4, 0x1B, 0x93, 0x06, 0xD2, 0x1A, 0x92,
+0x05, 0xD3, 0x91, 0x67, 0x04, 0xD2, 0xD1, 0x18,
+0x1B, 0x55, 0x04, 0x6E, 0x50, 0x67, 0x01, 0x4A,
+0xFF, 0x6C, 0x8C, 0xEA, 0x0B, 0x93, 0xE5, 0x17,
+0xF4, 0x63, 0x17, 0x62, 0x16, 0xD1, 0x15, 0xD0,
+0x62, 0xA4, 0x01, 0x6A, 0x6E, 0x36, 0x4C, 0xEE,
+0x44, 0x6A, 0x58, 0x2E, 0x24, 0x67, 0x72, 0x32,
+0x03, 0x6C, 0xFF, 0x6D, 0x8C, 0xEA, 0xAC, 0xEA,
+0x0A, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xF1, 0x0C, 0x9A, 0x0A, 0x92,
+0x03, 0x72, 0xF8, 0x67, 0x24, 0x2F, 0x20, 0xF0,
+0x5B, 0xA0, 0x02, 0x2A, 0x00, 0x6A, 0x42, 0x10,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0x07, 0x68, 0x6C, 0xE8, 0xF3, 0xF3, 0x4C, 0x99,
+0xAC, 0xE8, 0xB0, 0x67, 0x01, 0x6E, 0x00, 0x6C,
+0x40, 0xEA, 0x09, 0xD7, 0x08, 0xD2, 0xF3, 0xF3,
+0x4C, 0x99, 0xB0, 0x67, 0x00, 0x6E, 0x40, 0xEA,
+0x00, 0x6C, 0x09, 0x97, 0x08, 0x93, 0xC2, 0x67,
+0x04, 0xD7, 0x00, 0x6D, 0xE3, 0x67, 0xD1, 0x18,
+0x46, 0x52, 0x06, 0x6C, 0xDF, 0x17, 0x0A, 0x93,
+0x54, 0xA0, 0x09, 0x2B, 0x01, 0x72, 0x04, 0x61,
+0x00, 0x6D, 0xD1, 0x18, 0x56, 0x57, 0x01, 0x6C,
+0x00, 0x6A, 0x54, 0xC0, 0xD3, 0x17, 0x01, 0x6B,
+0x6E, 0xEA, 0x19, 0x2A, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xF3, 0xF0, 0x70, 0x9B,
+0x04, 0xD2, 0x80, 0xF3, 0x03, 0x6F, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xC9, 0xF6,
+0x80, 0x9A, 0xC9, 0xF6, 0xA4, 0x9A, 0x40, 0xEB,
+0x02, 0x6E, 0x04, 0x6A, 0x17, 0x97, 0x16, 0x91,
+0x15, 0x90, 0x00, 0xEF, 0x0C, 0x63, 0x55, 0xA0,
+0x0A, 0x2A, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xF3, 0xF0, 0x70, 0x9B, 0x04, 0xD2,
+0x80, 0xF3, 0x04, 0x6F, 0xE4, 0x17, 0x44, 0xA1,
+0x18, 0x6B, 0x4C, 0xEB, 0xAC, 0xEB, 0x18, 0x73,
+0x03, 0x61, 0xE5, 0x4B, 0x6C, 0xEA, 0x44, 0xC1,
+0x58, 0xA1, 0x11, 0x5A, 0x0B, 0xD2, 0x17, 0x61,
+0xF7, 0xF0, 0x01, 0x6A, 0x0B, 0x93, 0x40, 0x32,
+0x40, 0x32, 0xF3, 0xF0, 0x50, 0x9A, 0x05, 0xD3,
+0x01, 0x6B, 0x04, 0xD3, 0x80, 0xF3, 0x05, 0x6F,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xC9, 0xF6, 0x80, 0x9B, 0xC9, 0xF6, 0xA4, 0x9B,
+0x40, 0xEA, 0x02, 0x6E, 0xCA, 0x17, 0x59, 0xA1,
+0x11, 0x5A, 0x0D, 0xD2, 0x0D, 0x61, 0x0D, 0x93,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x05, 0xD3, 0x01, 0x6B, 0xF3, 0xF0, 0x50, 0x9A,
+0x04, 0xD3, 0xA0, 0xF3, 0x1D, 0x6F, 0xE4, 0x17,
+0x5A, 0xA1, 0x11, 0x5A, 0x0E, 0xD2, 0x0D, 0x61,
+0x0E, 0x93, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x05, 0xD3, 0x01, 0x6B, 0xF3, 0xF0,
+0x50, 0x9A, 0x04, 0xD3, 0xA0, 0xF3, 0x1E, 0x6F,
+0xD3, 0x17, 0x62, 0xA1, 0x07, 0x6A, 0x00, 0x6C,
+0x4C, 0xEB, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x53, 0xF1, 0x40, 0x9A, 0xA3, 0x67,
+0x40, 0xEA, 0x09, 0xD3, 0xFF, 0x72, 0x78, 0x67,
+0x18, 0x2B, 0x05, 0xD3, 0x02, 0x6B, 0x09, 0x94,
+0x04, 0xD3, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6B, 0x40, 0x32, 0x60, 0x33, 0x60, 0x33,
+0x40, 0x32, 0x06, 0xD4, 0xF3, 0xF0, 0x50, 0x9A,
+0xC9, 0xF6, 0x80, 0x9B, 0xC9, 0xF6, 0xA4, 0x9B,
+0x80, 0xF3, 0x06, 0x6F, 0x40, 0xEA, 0x02, 0x6E,
+0x80, 0x17, 0x81, 0xA0, 0x5A, 0xC0, 0x40, 0xA0,
+0x80, 0x34, 0xF7, 0xF0, 0x01, 0x6F, 0x4D, 0xEC,
+0x42, 0xA0, 0xE0, 0x37, 0xE0, 0x37, 0x40, 0x32,
+0x40, 0x32, 0x8D, 0xEA, 0x83, 0xA0, 0x73, 0xF6,
+0x7C, 0x9F, 0x1C, 0x6E, 0x00, 0xF6, 0x80, 0x34,
+0xB1, 0x67, 0x4D, 0xEC, 0x40, 0xEB, 0x11, 0xD7,
+0xD1, 0x18, 0x7D, 0x55, 0x00, 0x6C, 0x90, 0xF2,
+0x40, 0x40, 0x0F, 0xD2, 0xD0, 0xF2, 0x40, 0x40,
+0x67, 0x40, 0x08, 0xD2, 0x10, 0xD2, 0x90, 0xF2,
+0x40, 0x40, 0x79, 0x4B, 0x0C, 0xD2, 0xF7, 0xF0,
+0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37, 0x93, 0xF6,
+0x40, 0x9F, 0x83, 0x67, 0x21, 0x6E, 0x00, 0x6D,
+0x13, 0xD3, 0x40, 0xEA, 0x12, 0xD7, 0x12, 0x97,
+0x0C, 0x94, 0x04, 0x6E, 0x93, 0xF6, 0x40, 0x9F,
+0x40, 0xEA, 0x00, 0x6D, 0x12, 0x97, 0x10, 0x94,
+0x08, 0x6E, 0x93, 0xF6, 0x40, 0x9F, 0x40, 0xEA,
+0x00, 0x6D, 0x0C, 0x92, 0x13, 0x93, 0x04, 0x4A,
+0x0C, 0xD2, 0x10, 0x92, 0x0C, 0x94, 0x21, 0x4B,
+0x08, 0x4A, 0x10, 0xD2, 0x08, 0x92, 0x8A, 0xEA,
+0xDA, 0x61, 0x0B, 0x92, 0x0B, 0x95, 0x81, 0x6C,
+0x54, 0x32, 0xA9, 0xE2, 0x89, 0xE2, 0x67, 0x41,
+0x49, 0xE0, 0x15, 0x4B, 0x9D, 0xE0, 0x0C, 0xD2,
+0x0C, 0x92, 0xEA, 0xEA, 0x80, 0xF0, 0x19, 0x61,
+0x0D, 0x92, 0x48, 0x37, 0xFD, 0xE3, 0xEA, 0xEB,
+0xA0, 0xF0, 0x07, 0x61, 0x0E, 0x97, 0x5A, 0x4F,
+0xEC, 0x37, 0xFD, 0xE0, 0x08, 0x92, 0xEA, 0xEA,
+0xA0, 0xF0, 0x0F, 0x61, 0x5D, 0x67, 0x20, 0xF0,
+0x48, 0xA2, 0x54, 0xC0, 0x01, 0x6A, 0x20, 0xF0,
+0x5B, 0xC0, 0x0A, 0x92, 0x02, 0x72, 0xDF, 0xF6,
+0x09, 0x60, 0x44, 0xA1, 0x04, 0x6B, 0xFF, 0x6E,
+0x6C, 0xEA, 0xCC, 0xEA, 0xA0, 0xF0, 0x0D, 0x22,
+0x6D, 0xA1, 0x4C, 0xA1, 0x8E, 0xA1, 0x60, 0x33,
+0x4D, 0xEB, 0x4F, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0xB1, 0xA1, 0x8D, 0xEB, 0x00, 0xF6, 0x40, 0x32,
+0x6D, 0xEA, 0x70, 0xA1, 0x92, 0xA1, 0xA0, 0x35,
+0x6D, 0xED, 0x73, 0xA1, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x60, 0x33, 0xAD, 0xEB,
+0x86, 0x67, 0x6C, 0xEC, 0x62, 0x35, 0x3C, 0x65,
+0x9C, 0xC0, 0x85, 0x67, 0xCC, 0xEC, 0x1C, 0x65,
+0x9D, 0xC0, 0x42, 0x34, 0xE4, 0x67, 0xA2, 0x35,
+0x82, 0x34, 0xCC, 0xED, 0xCC, 0xEF, 0xCC, 0xEC,
+0x26, 0x67, 0xD9, 0x67, 0x4C, 0xE9, 0x00, 0xF6,
+0x62, 0x33, 0x00, 0xF6, 0x42, 0x32, 0x20, 0xF0,
+0xC4, 0xC0, 0xD8, 0x67, 0xBE, 0xC0, 0x7F, 0xC0,
+0x20, 0xF0, 0x20, 0xC0, 0x20, 0xF0, 0xE1, 0xC0,
+0x20, 0xF0, 0x82, 0xC0, 0x20, 0xF0, 0x43, 0xC0,
+0x20, 0xF0, 0xC5, 0xC0, 0x20, 0xF0, 0xA6, 0xC0,
+0x20, 0xF0, 0x67, 0xC0, 0x20, 0xF0, 0x28, 0xC0,
+0x20, 0xF0, 0xE9, 0xC0, 0x20, 0xF0, 0x8A, 0xC0,
+0x20, 0xF0, 0x4B, 0xC0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xDD, 0xA0, 0xD3, 0xF3,
+0x64, 0x9A, 0x5E, 0xA0, 0x9C, 0xA0, 0xC0, 0x36,
+0x40, 0x32, 0x8D, 0xEE, 0x40, 0x32, 0x20, 0xF0,
+0xE1, 0xA0, 0x4D, 0xEE, 0x20, 0xF0, 0x42, 0xA0,
+0x20, 0xF0, 0xA0, 0xA0, 0xE0, 0x37, 0x40, 0x32,
+0xAD, 0xEF, 0x40, 0x32, 0x00, 0x6D, 0x9F, 0xA0,
+0x4D, 0xEF, 0x20, 0xF0, 0x43, 0xA0, 0x05, 0xD5,
+0x97, 0xF0, 0x19, 0x6D, 0xA0, 0x35, 0xA0, 0x35,
+0xED, 0xF0, 0x19, 0x4D, 0x04, 0xD5, 0x09, 0x95,
+0x00, 0xF6, 0x80, 0x34, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEE, 0x4D, 0xEF, 0x40, 0xEB, 0x00, 0x6C,
+0x49, 0x16, 0x41, 0x43, 0x0B, 0xD2, 0x11, 0x92,
+0x60, 0xA3, 0x0B, 0x95, 0x73, 0xF6, 0x5C, 0x9A,
+0xFF, 0xF7, 0x7F, 0xC7, 0xC3, 0x67, 0x87, 0x67,
+0x12, 0xD3, 0x40, 0xEA, 0x10, 0xD7, 0x0B, 0x92,
+0x12, 0x93, 0x10, 0x97, 0x6D, 0xE2, 0x21, 0x4F,
+0x4F, 0x17, 0x11, 0x92, 0x0F, 0x94, 0xA3, 0x67,
+0x73, 0xF6, 0x5C, 0x9A, 0x04, 0x6E, 0x0C, 0xD7,
+0x40, 0xEA, 0x0B, 0xD3, 0x0F, 0x92, 0x0B, 0x93,
+0x0C, 0x97, 0x04, 0x4A, 0x04, 0x4B, 0x0F, 0xD2,
+0x46, 0x17, 0x11, 0x92, 0x08, 0x94, 0xA3, 0x67,
+0x73, 0xF6, 0x5C, 0x9A, 0x06, 0x6E, 0x0C, 0xD7,
+0x40, 0xEA, 0x0B, 0xD3, 0x08, 0x92, 0x0B, 0x93,
+0x0C, 0x97, 0x08, 0x4A, 0x06, 0x4B, 0x08, 0xD2,
+0x3D, 0x17, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0xF3, 0xF3, 0x4C, 0x9B, 0x09, 0x95,
+0x01, 0x6E, 0x00, 0x6C, 0x40, 0xEA, 0x08, 0xD3,
+0x08, 0x93, 0x22, 0x67, 0x09, 0x95, 0xF3, 0xF3,
+0x4C, 0x9B, 0x00, 0x6E, 0x40, 0xEA, 0x00, 0x6C,
+0x42, 0x33, 0x5C, 0xC0, 0x00, 0xF6, 0x42, 0x32,
+0x5F, 0xC0, 0x22, 0x32, 0x7D, 0xC0, 0x20, 0xF0,
+0x20, 0xC0, 0x62, 0x33, 0x20, 0xF0, 0x41, 0xC0,
+0x00, 0xF6, 0x22, 0x31, 0x42, 0x32, 0x7E, 0xC0,
+0x20, 0xF0, 0x42, 0xC0, 0x20, 0xF0, 0x23, 0xC0,
+0x75, 0x17, 0x00, 0x65, 0xF3, 0x63, 0x19, 0x62,
+0x18, 0xD1, 0x17, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x88, 0x33, 0x6D, 0xF1,
+0x0C, 0x4A, 0x49, 0xE3, 0x00, 0x9A, 0x1A, 0xD4,
+0x54, 0xA0, 0xC0, 0xF0, 0x1A, 0x22, 0x02, 0x72,
+0xC0, 0xF0, 0x1C, 0x60, 0x61, 0xA0, 0x80, 0xA0,
+0x42, 0xA0, 0x60, 0x33, 0x6D, 0xEC, 0x63, 0xA0,
+0x40, 0x32, 0x40, 0x32, 0x8D, 0xEA, 0x00, 0xF6,
+0x60, 0x33, 0x4D, 0xEB, 0x0D, 0xD3, 0x49, 0xA0,
+0x28, 0xA0, 0x6A, 0xA0, 0x40, 0x32, 0x2D, 0xEA,
+0x2B, 0xA0, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA,
+0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9, 0x0D, 0x92,
+0x07, 0x6B, 0x0E, 0xD1, 0x42, 0xA2, 0xF7, 0xF0,
+0x01, 0x69, 0x20, 0x31, 0x6C, 0xEA, 0xFF, 0x6B,
+0x6C, 0xEA, 0x0A, 0xD2, 0x20, 0x31, 0x0A, 0x95,
+0x1A, 0x94, 0xF3, 0xF3, 0x4C, 0x99, 0x40, 0xEA,
+0x01, 0x6E, 0x0B, 0xD2, 0x0A, 0x95, 0x1A, 0x94,
+0xF3, 0xF3, 0x4C, 0x99, 0x40, 0xEA, 0x00, 0x6E,
+0x0B, 0x97, 0x0E, 0x95, 0xC2, 0x67, 0x90, 0x67,
+0xD1, 0x18, 0x30, 0x57, 0x0C, 0xD2, 0x55, 0xA0,
+0x96, 0xA0, 0x0E, 0xD1, 0xFF, 0x4A, 0x42, 0xEC,
+0xA0, 0xF0, 0x01, 0x61, 0x00, 0x6A, 0x85, 0xA0,
+0x56, 0xC0, 0x44, 0xA0, 0x66, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x42, 0x33, 0x48, 0xC0, 0x8D, 0xA0, 0x00, 0xF6,
+0x42, 0x32, 0x69, 0xC0, 0x4B, 0xC0, 0x62, 0x33,
+0x4C, 0xA0, 0x6A, 0xC0, 0x6E, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x4F, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x42, 0x33, 0x50, 0xC0, 0x00, 0xF6, 0x42, 0x32,
+0x53, 0xC0, 0x5D, 0x67, 0x20, 0xF0, 0x50, 0xA2,
+0x9A, 0xA0, 0x71, 0xC0, 0x20, 0xF0, 0x44, 0xC0,
+0x0C, 0x92, 0x62, 0x33, 0x72, 0xC0, 0x42, 0x32,
+0x20, 0xF0, 0x45, 0xC0, 0x42, 0x32, 0x20, 0xF0,
+0x46, 0xC0, 0x0C, 0x92, 0x01, 0x6E, 0x00, 0xF6,
+0x42, 0x32, 0x20, 0xF0, 0x47, 0xC0, 0x5D, 0x67,
+0x20, 0xF0, 0x4C, 0xA2, 0x20, 0xF0, 0x48, 0xC0,
+0x0B, 0x92, 0x42, 0x32, 0x20, 0xF0, 0x49, 0xC0,
+0x42, 0x32, 0x20, 0xF0, 0x4A, 0xC0, 0x0B, 0x92,
+0x00, 0xF6, 0x42, 0x32, 0x20, 0xF0, 0x4B, 0xC0,
+0xD1, 0x18, 0xD4, 0xD9, 0x02, 0x6D, 0x49, 0xA0,
+0x28, 0xA0, 0x6A, 0xA0, 0x40, 0x32, 0x2D, 0xEA,
+0x2B, 0xA0, 0x60, 0x33, 0x60, 0x33, 0x6D, 0xEA,
+0x00, 0xF6, 0x20, 0x31, 0x4D, 0xE9, 0x62, 0xA1,
+0xE4, 0xA1, 0xC5, 0xA1, 0x0F, 0xD3, 0x07, 0x6B,
+0x43, 0xA1, 0x6C, 0xEF, 0xFF, 0x6B, 0x6C, 0xEF,
+0xDA, 0x36, 0x6C, 0xEE, 0x10, 0xD7, 0x20, 0xF0,
+0xFC, 0xA0, 0x20, 0xF0, 0xDD, 0xC0, 0x20, 0xF0,
+0x5C, 0xC0, 0x1A, 0x94, 0x01, 0x5F, 0xF8, 0x67,
+0x01, 0x6D, 0x14, 0xD3, 0x13, 0xD7, 0x12, 0xD2,
+0xD1, 0x18, 0x47, 0x57, 0x11, 0xD6, 0x13, 0x97,
+0x11, 0x96, 0x12, 0x92, 0x05, 0xD7, 0x04, 0xD6,
+0x10, 0x97, 0x1A, 0x96, 0x0F, 0x95, 0xD1, 0x18,
+0x50, 0x55, 0x82, 0x67, 0x14, 0x93, 0x33, 0x2A,
+0x54, 0xC0, 0x1A, 0x92, 0x01, 0x6B, 0x0C, 0x96,
+0x05, 0xD2, 0x44, 0xA1, 0x0B, 0x97, 0x02, 0x6D,
+0x5A, 0x32, 0x6C, 0xEA, 0x03, 0x6C, 0xD1, 0x18,
+0xD7, 0x53, 0x04, 0xD2, 0x1A, 0x95, 0x02, 0x6C,
+0x02, 0x10, 0x1A, 0x95, 0x01, 0x6C, 0xD1, 0x18,
+0x56, 0x57, 0x00, 0x65, 0x19, 0x97, 0x18, 0x91,
+0x17, 0x90, 0x00, 0xEF, 0x0D, 0x63, 0x01, 0x4C,
+0x96, 0xC0, 0x89, 0xA0, 0x48, 0xA0, 0x6A, 0xA0,
+0x80, 0x34, 0x4D, 0xEC, 0x4B, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32,
+0x8D, 0xEA, 0x1C, 0x4A, 0x42, 0x33, 0x48, 0xC0,
+0x69, 0xC0, 0x00, 0xF6, 0x42, 0x32, 0x62, 0x33,
+0x6A, 0xC0, 0x4B, 0xC0, 0x98, 0x17, 0x45, 0xA1,
+0x20, 0x6C, 0x02, 0x6D, 0x8C, 0xEA, 0x4C, 0xEB,
+0x01, 0x2B, 0x00, 0x6D, 0xD1, 0x18, 0x47, 0x57,
+0x1A, 0x94, 0x0E, 0x92, 0x0A, 0x95, 0x1A, 0x94,
+0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA, 0x01, 0x6E,
+0x0B, 0xD2, 0x0E, 0x92, 0x0A, 0x95, 0x1A, 0x94,
+0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x6E,
+0x0B, 0x97, 0xC2, 0x67, 0x90, 0x67, 0xB1, 0x67,
+0xD1, 0x18, 0x96, 0x56, 0x0C, 0xD2, 0x45, 0xA1,
+0x10, 0x6B, 0xFF, 0x6C, 0x4C, 0xEB, 0x39, 0x23,
+0x67, 0xA1, 0x01, 0x6D, 0xAC, 0xEB, 0x8C, 0xEB,
+0x5B, 0x2B, 0xC0, 0x4B, 0x6C, 0xEA, 0x8C, 0xEA,
+0x80, 0x72, 0x56, 0x60, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x73, 0xF6, 0x4C, 0x9B,
+0x01, 0x6C, 0x40, 0xEA, 0x0F, 0xD3, 0x1A, 0x92,
+0xA6, 0xA1, 0x09, 0x6E, 0x06, 0xD2, 0x0C, 0x92,
+0x91, 0x67, 0x04, 0xD2, 0x0B, 0x92, 0xD1, 0x18,
+0x1B, 0x55, 0x05, 0xD2, 0x00, 0x6A, 0x65, 0xA1,
+0x0F, 0x6C, 0x8C, 0xEB, 0x62, 0xEA, 0x72, 0x61,
+0x76, 0xA1, 0x00, 0x6A, 0x08, 0xD2, 0x5D, 0x67,
+0x20, 0xF0, 0x61, 0xC2, 0x01, 0x6A, 0x7D, 0x67,
+0x20, 0xF0, 0x42, 0xC3, 0x5D, 0x67, 0x20, 0xF0,
+0x42, 0xA2, 0x04, 0x5A, 0x7B, 0x61, 0x0F, 0x92,
+0x01, 0x6C, 0x73, 0xF6, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x01, 0x6A, 0x20, 0xF0, 0x58, 0xC0,
+0x93, 0xF6, 0x40, 0x9B, 0x87, 0x40, 0x08, 0x6E,
+0x01, 0x6D, 0x41, 0x4C, 0x40, 0xEA, 0x0D, 0xD3,
+0x0D, 0x93, 0x87, 0x40, 0x10, 0x6E, 0x93, 0xF6,
+0x40, 0x9B, 0x01, 0x6D, 0x40, 0xEA, 0x49, 0x4C,
+0x0D, 0x93, 0x87, 0x40, 0x10, 0x6E, 0x93, 0xF6,
+0x40, 0x9B, 0x01, 0x6D, 0x40, 0xEA, 0x59, 0x4C,
+0x0D, 0x93, 0x87, 0x40, 0x10, 0x6E, 0x93, 0xF6,
+0x40, 0x9B, 0x01, 0x6D, 0x40, 0xEA, 0x69, 0x4C,
+0x00, 0xA1, 0x0C, 0x92, 0x0A, 0x95, 0x00, 0x30,
+0x08, 0x30, 0x45, 0xE0, 0x0B, 0x92, 0x03, 0xE9,
+0x18, 0x67, 0x41, 0xE0, 0x0E, 0x92, 0x1A, 0x94,
+0x01, 0x6E, 0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x65, 0x0B, 0xD2, 0x0E, 0x92, 0x0A, 0x95,
+0x1A, 0x94, 0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA,
+0x00, 0x6E, 0x0B, 0x97, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0x03, 0xEF, 0xD3, 0xF3,
+0x64, 0x9B, 0x04, 0x61, 0xEA, 0xE8, 0x04, 0x61,
+0x23, 0xEA, 0x02, 0x60, 0x51, 0x67, 0xF0, 0x67,
+0x1A, 0x94, 0x0A, 0x95, 0xC2, 0x67, 0x05, 0xD4,
+0x97, 0xF0, 0x19, 0x6C, 0x80, 0x34, 0x80, 0x34,
+0xED, 0xF0, 0x19, 0x4C, 0x04, 0xD4, 0x40, 0xEB,
+0x1A, 0x94, 0x28, 0x17, 0x73, 0xA1, 0x01, 0x6C,
+0x04, 0x6E, 0x67, 0xEA, 0x8C, 0xEB, 0x01, 0x2B,
+0x09, 0x6E, 0x4D, 0xE1, 0x10, 0xD2, 0x1A, 0x92,
+0xA8, 0xA3, 0x91, 0x67, 0x06, 0xD2, 0x0C, 0x92,
+0x04, 0xD2, 0x0B, 0x92, 0xD1, 0x18, 0x1B, 0x55,
+0x05, 0xD2, 0x10, 0x92, 0xFF, 0x6B, 0x01, 0x4A,
+0x6C, 0xEA, 0x71, 0x17, 0x02, 0x72, 0x11, 0x60,
+0x03, 0x72, 0x16, 0x60, 0x01, 0x72, 0x08, 0x61,
+0x55, 0xA1, 0x74, 0xA1, 0x40, 0x32, 0x6D, 0xEA,
+0x0D, 0x93, 0x78, 0xA3, 0x00, 0x6C, 0x2A, 0x2A,
+0x5D, 0x67, 0x20, 0xF0, 0x42, 0xA2, 0x01, 0x4A,
+0x6A, 0x17, 0x59, 0xA1, 0x78, 0xA1, 0x40, 0x32,
+0x6D, 0xEA, 0x0D, 0x93, 0x79, 0xA3, 0xF2, 0x17,
+0x5B, 0xA1, 0x7A, 0xA1, 0x40, 0x32, 0x6D, 0xEA,
+0x0D, 0x93, 0x7A, 0xA3, 0xEB, 0x17, 0xA2, 0x67,
+0xA7, 0xEC, 0x85, 0x67, 0x01, 0x6D, 0xAC, 0xEC,
+0x0D, 0x24, 0x10, 0xD2, 0x1A, 0x92, 0x0C, 0x96,
+0x0B, 0x97, 0x08, 0x04, 0x05, 0xD2, 0x04, 0xD4,
+0x91, 0x67, 0xD1, 0x18, 0x33, 0x54, 0x11, 0xD3,
+0x11, 0x93, 0x10, 0x92, 0x9D, 0x67, 0x20, 0xF0,
+0x80, 0xA4, 0x01, 0x4C, 0xBD, 0x67, 0x20, 0xF0,
+0x80, 0xC5, 0x9D, 0x67, 0x20, 0xF0, 0x80, 0xA4,
+0x63, 0xEC, 0xE1, 0x61, 0xCD, 0x17, 0x00, 0x65,
+0xF8, 0x63, 0x0F, 0x62, 0x0E, 0xD1, 0x0D, 0xD0,
+0x06, 0xD4, 0x06, 0x02, 0x40, 0xA2, 0x48, 0x33,
+0x06, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x6D, 0xF1, 0x0C, 0x4A, 0x49, 0xE3,
+0x00, 0x9A, 0xD5, 0xA0, 0x80, 0xF0, 0x0B, 0x26,
+0x85, 0xA0, 0x44, 0xA0, 0x66, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x47, 0xA0, 0x60, 0x33, 0x60, 0x33,
+0x6D, 0xEC, 0x00, 0xF6, 0x40, 0x32, 0x8D, 0xEA,
+0x7E, 0x22, 0x20, 0xF0, 0x5B, 0xA0, 0x7B, 0x22,
+0x54, 0xA0, 0x75, 0x22, 0x02, 0x72, 0x77, 0x60,
+0x49, 0xA0, 0x68, 0xA0, 0x8A, 0xA0, 0x40, 0x32,
+0x6D, 0xEA, 0x6B, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xEA, 0x00, 0xF6, 0x60, 0x33, 0xA1, 0xA0,
+0x4D, 0xEB, 0x40, 0xA0, 0x82, 0xA0, 0xA0, 0x35,
+0x4D, 0xED, 0x43, 0xA0, 0x80, 0x34, 0x80, 0x34,
+0x8D, 0xED, 0x00, 0xF6, 0x40, 0x32, 0xAD, 0xEA,
+0x22, 0xA2, 0x07, 0x6A, 0x09, 0xD3, 0x4C, 0xE9,
+0x07, 0xD1, 0x07, 0x02, 0x20, 0xA2, 0x41, 0xA3,
+0x5B, 0x22, 0x59, 0xA0, 0x59, 0x22, 0x58, 0xA0,
+0x57, 0x2A, 0x56, 0xA0, 0x01, 0x4E, 0x4E, 0xEE,
+0x53, 0x26, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0xE0, 0x37, 0x06, 0x94, 0xF3, 0xF3, 0x4C, 0x9F,
+0xB1, 0x67, 0x01, 0x6E, 0x40, 0xEA, 0x08, 0xD7,
+0x08, 0x97, 0x07, 0xD2, 0x06, 0x94, 0xF3, 0xF3,
+0x4C, 0x9F, 0xB1, 0x67, 0x40, 0xEA, 0x00, 0x6E,
+0x08, 0xD2, 0x06, 0x92, 0x09, 0x93, 0x08, 0x96,
+0x05, 0xD2, 0x84, 0xA3, 0x01, 0x6A, 0x07, 0x97,
+0x8E, 0x34, 0x4C, 0xEC, 0x0A, 0xD3, 0x7D, 0x67,
+0x20, 0xF0, 0x44, 0xC3, 0x00, 0x6D, 0x04, 0xD4,
+0xD1, 0x18, 0xD7, 0x53, 0x00, 0x6C, 0x5D, 0x67,
+0x20, 0xF0, 0x44, 0xA2, 0x0A, 0x93, 0x06, 0x94,
+0x58, 0xC0, 0xE1, 0xA3, 0x09, 0x03, 0xF7, 0xF0,
+0x01, 0x6A, 0x09, 0xD7, 0xE0, 0xA3, 0x08, 0x93,
+0x40, 0x32, 0xE0, 0x37, 0xE8, 0x37, 0x79, 0xE7,
+0x06, 0x93, 0x40, 0x32, 0xE3, 0xEE, 0x05, 0xD3,
+0x97, 0xF0, 0x19, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xED, 0xF0, 0x19, 0x4B, 0x04, 0xD3, 0x07, 0x93,
+0xD3, 0xF3, 0x44, 0x9A, 0xF8, 0x67, 0x7D, 0xE7,
+0x40, 0xEA, 0xB1, 0x67, 0x04, 0x10, 0x06, 0x95,
+0xD1, 0x18, 0x56, 0x57, 0x01, 0x6C, 0x0F, 0x97,
+0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF, 0x08, 0x63,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x97, 0xF0, 0x19, 0x6E, 0xD3, 0xF3, 0x40, 0x9A,
+0x06, 0x94, 0xC0, 0x36, 0xC0, 0x36, 0xAE, 0xF0,
+0x19, 0x4E, 0x40, 0xEA, 0xB1, 0x67, 0x75, 0xA0,
+0x56, 0xA0, 0x06, 0x94, 0xFF, 0x4B, 0x6E, 0xEA,
+0x04, 0x2A, 0xD1, 0x18, 0x9F, 0x5A, 0x00, 0x65,
+0xE2, 0x17, 0xD1, 0x18, 0x2D, 0x59, 0x00, 0x65,
+0xDE, 0x17, 0x00, 0x65, 0xF8, 0x63, 0x0F, 0x62,
+0x0E, 0xD1, 0x0D, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x88, 0x33, 0x6D, 0xF1,
+0x0C, 0x4A, 0x49, 0xE3, 0x60, 0x9A, 0x04, 0x67,
+0xFF, 0x69, 0x41, 0xA3, 0xE0, 0xA3, 0x82, 0xA3,
+0x40, 0x32, 0xED, 0xEA, 0x80, 0x34, 0xE3, 0xA3,
+0x80, 0x34, 0x8D, 0xEA, 0x89, 0xA3, 0xA8, 0xA3,
+0x00, 0xF6, 0xE0, 0x37, 0x4D, 0xEF, 0x4A, 0xA3,
+0x80, 0x34, 0x8D, 0xED, 0x8B, 0xA3, 0x40, 0x32,
+0x40, 0x32, 0xAD, 0xEA, 0x00, 0xF6, 0x80, 0x34,
+0x4D, 0xEC, 0x5B, 0xA3, 0x08, 0xD4, 0x07, 0x6C,
+0x01, 0x4A, 0x5B, 0xC3, 0x55, 0xA3, 0x01, 0x6E,
+0x0A, 0xD3, 0x56, 0xC3, 0x42, 0xA7, 0x07, 0xD7,
+0x8C, 0xEA, 0x2C, 0xEA, 0x06, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x06, 0x95,
+0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA, 0x90, 0x67,
+0x09, 0xD2, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x06, 0x95, 0xF3, 0xF3, 0x4C, 0x9A,
+0x90, 0x67, 0x40, 0xEA, 0x00, 0x6E, 0x0A, 0x93,
+0xF7, 0xF0, 0x01, 0x6D, 0xA0, 0x35, 0x94, 0xA3,
+0xA0, 0x35, 0x0A, 0xD5, 0x16, 0x24, 0x02, 0x74,
+0x0F, 0x60, 0x07, 0x97, 0x03, 0x6D, 0x84, 0xA7,
+0x8E, 0x34, 0xAC, 0xEC, 0x2C, 0xEC, 0x01, 0x74,
+0x11, 0x60, 0x0B, 0x24, 0x02, 0x74, 0x80, 0xF0,
+0x07, 0x60, 0x03, 0x74, 0xE0, 0xF0, 0x1A, 0x60,
+0x0F, 0x97, 0x0E, 0x91, 0x0D, 0x90, 0x00, 0xEF,
+0x08, 0x63, 0xB0, 0x67, 0xD1, 0x18, 0x56, 0x57,
+0x01, 0x6C, 0xF6, 0x17, 0x20, 0xF0, 0xA5, 0xA3,
+0x20, 0xF0, 0xC4, 0xA3, 0x20, 0xF0, 0x86, 0xA3,
+0xA0, 0x35, 0xAD, 0xEE, 0x20, 0xF0, 0xA7, 0xA3,
+0x80, 0x34, 0x80, 0x34, 0xCD, 0xEC, 0x00, 0xF6,
+0xA0, 0x35, 0x8D, 0xED, 0x07, 0xD5, 0x20, 0xF0,
+0xA9, 0xA3, 0x20, 0xF0, 0xC8, 0xA3, 0x20, 0xF0,
+0x8A, 0xA3, 0xA0, 0x35, 0xAD, 0xEE, 0x20, 0xF0,
+0xAB, 0xA3, 0x80, 0x34, 0x80, 0x34, 0xCD, 0xEC,
+0x00, 0xF6, 0xA0, 0x35, 0x8D, 0xED, 0x0A, 0xD5,
+0x29, 0xA7, 0x88, 0xA7, 0x04, 0xD2, 0x09, 0x92,
+0x08, 0x96, 0x20, 0x31, 0x8D, 0xE9, 0x05, 0xD2,
+0x83, 0x67, 0xA7, 0x67, 0xD1, 0x18, 0xB5, 0x57,
+0x08, 0xD3, 0x08, 0x93, 0xC5, 0x22, 0x9A, 0xA3,
+0x00, 0x6E, 0xD1, 0x18, 0xD4, 0xD9, 0x02, 0x6D,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x06, 0x95, 0xF3, 0xF3, 0x4C, 0x9A, 0x01, 0x6E,
+0x40, 0xEA, 0x90, 0x67, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x06, 0x95,
+0xF3, 0xF3, 0x4C, 0x9A, 0x00, 0x6E, 0x40, 0xEA,
+0x90, 0x67, 0x64, 0x6B, 0x79, 0xE9, 0x0A, 0x96,
+0x08, 0x97, 0x10, 0xED, 0xA0, 0x35, 0xA8, 0x35,
+0x12, 0xEC, 0x80, 0xF5, 0x82, 0x34, 0xAD, 0xEC,
+0x12, 0xEB, 0x07, 0x95, 0x60, 0x33, 0x68, 0x33,
+0xB5, 0xE3, 0x63, 0xED, 0x78, 0x67, 0xD1, 0xE4,
+0x91, 0xE3, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x83, 0xEF, 0xD3, 0xF3, 0x64, 0x9B,
+0x04, 0x61, 0xEA, 0xEC, 0x04, 0x61, 0xA3, 0xEA,
+0x02, 0x60, 0x45, 0x67, 0xE4, 0x67, 0x97, 0xF0,
+0x19, 0x6C, 0x80, 0x34, 0x80, 0x34, 0x06, 0x95,
+0xED, 0xF0, 0x19, 0x4C, 0x04, 0xD4, 0x05, 0xD0,
+0xC2, 0x67, 0x40, 0xEB, 0x90, 0x67, 0x7C, 0x17,
+0x97, 0xA3, 0xFF, 0xF7, 0x1F, 0x6D, 0x01, 0x4C,
+0x8C, 0xE9, 0x37, 0xC3, 0x81, 0xA7, 0x23, 0xEC,
+0x6A, 0x61, 0xC8, 0xA7, 0x89, 0xA7, 0x80, 0x34,
+0xCD, 0xEC, 0xAC, 0xEC, 0x07, 0xD4, 0x20, 0xF0,
+0xA5, 0xA3, 0x20, 0xF0, 0xC4, 0xA3, 0x20, 0xF0,
+0x86, 0xA3, 0xA0, 0x35, 0xAD, 0xEE, 0x20, 0xF0,
+0xA7, 0xA3, 0x80, 0x34, 0x80, 0x34, 0xCD, 0xEC,
+0x00, 0xF6, 0xA0, 0x35, 0x8D, 0xED, 0x0B, 0xD5,
+0x20, 0xF0, 0x89, 0xA3, 0x20, 0xF0, 0x28, 0xA3,
+0x20, 0xF0, 0xAA, 0xA3, 0x80, 0x34, 0x2D, 0xEC,
+0x20, 0xF0, 0x2B, 0xA3, 0xA0, 0x35, 0x04, 0xD2,
+0x09, 0x92, 0xA0, 0x35, 0x08, 0x96, 0xAD, 0xEC,
+0x00, 0xF6, 0x20, 0x31, 0x8D, 0xE9, 0x05, 0xD2,
+0x83, 0x67, 0xA7, 0x67, 0xD1, 0x18, 0xB5, 0x57,
+0x08, 0xD3, 0x08, 0x93, 0x5F, 0xF7, 0x00, 0x22,
+0x9A, 0xA3, 0x00, 0x6E, 0xD1, 0x18, 0xD4, 0xD9,
+0x02, 0x6D, 0x0A, 0x92, 0x06, 0x95, 0x01, 0x6E,
+0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA, 0x90, 0x67,
+0x08, 0xD2, 0x0A, 0x92, 0x06, 0x95, 0x90, 0x67,
+0xF3, 0xF3, 0x4C, 0x9A, 0x40, 0xEA, 0x00, 0x6E,
+0x07, 0x94, 0x64, 0x6B, 0x08, 0x97, 0x79, 0xEC,
+0x10, 0xED, 0xA0, 0x35, 0xA8, 0x35, 0x12, 0xEC,
+0x80, 0xF5, 0x82, 0x34, 0xAD, 0xEC, 0x12, 0xEB,
+0x0B, 0x95, 0x60, 0x33, 0x68, 0x33, 0xB5, 0xE3,
+0x63, 0xED, 0x78, 0x67, 0x25, 0xE4, 0x25, 0xE3,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x23, 0xEF, 0xD3, 0xF3, 0x64, 0x9B, 0x04, 0x61,
+0xEA, 0xE9, 0x85, 0x61, 0xA3, 0xEA, 0x83, 0x60,
+0x45, 0x67, 0xF1, 0x67, 0x80, 0x17, 0xCA, 0xA7,
+0x8B, 0xA7, 0x95, 0x17, 0x9A, 0xA3, 0x00, 0x6E,
+0xD1, 0x18, 0xD4, 0xD9, 0x02, 0x6D, 0xD1, 0x18,
+0x3E, 0x5A, 0x90, 0x67, 0xFD, 0x16, 0x00, 0x65,
+0x20, 0xE8, 0x00, 0x65, 0x20, 0xE8, 0x00, 0x65,
+0xF5, 0x63, 0x15, 0x62, 0x14, 0xD1, 0x13, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x88, 0x33, 0x6D, 0xF1, 0x0C, 0x4A, 0x49, 0xE3,
+0x20, 0x9A, 0x49, 0xA1, 0x08, 0xA1, 0xCA, 0xA1,
+0x40, 0x32, 0x0D, 0xEA, 0x0B, 0xA1, 0xC0, 0x36,
+0xC0, 0x36, 0xCD, 0xEA, 0x00, 0xF6, 0x00, 0x30,
+0x4D, 0xE8, 0x62, 0x20, 0x0A, 0xD4, 0x81, 0xA1,
+0x65, 0x67, 0xA0, 0xA1, 0x42, 0xA1, 0x80, 0x34,
+0x8D, 0xED, 0x83, 0xA1, 0x40, 0x32, 0x40, 0x32,
+0xF7, 0xF0, 0x01, 0x6F, 0xAD, 0xEA, 0x00, 0xF6,
+0x80, 0x34, 0xE0, 0x37, 0xE0, 0x37, 0x4D, 0xEC,
+0x0B, 0xD4, 0xF3, 0xF3, 0x4C, 0x9F, 0x0A, 0x94,
+0xA3, 0x67, 0x01, 0x6E, 0x0E, 0xD7, 0x40, 0xEA,
+0x0D, 0xD3, 0x0E, 0x97, 0x0D, 0x93, 0x0C, 0xD2,
+0x0A, 0x94, 0xF3, 0xF3, 0x4C, 0x9F, 0xA3, 0x67,
+0x40, 0xEA, 0x00, 0x6E, 0x85, 0xA0, 0x0D, 0xD2,
+0x40, 0x6A, 0x4B, 0xEA, 0x4C, 0xEC, 0x20, 0xF0,
+0x58, 0xA1, 0xFF, 0x6D, 0xAC, 0xEC, 0x0E, 0xD4,
+0x1D, 0x22, 0x76, 0xA0, 0x00, 0x6A, 0x08, 0xD2,
+0x5D, 0x67, 0x20, 0xF0, 0x61, 0xC2, 0x01, 0x6A,
+0x7D, 0x67, 0x20, 0xF0, 0x42, 0xC3, 0x5D, 0x67,
+0x20, 0xF0, 0x42, 0xA2, 0x04, 0x5A, 0x29, 0x61,
+0x00, 0x6A, 0x85, 0xA0, 0x0F, 0x6D, 0xAC, 0xEC,
+0x82, 0xEA, 0x1E, 0x60, 0x51, 0xE1, 0x40, 0xF0,
+0xA8, 0xA4, 0x74, 0x25, 0x01, 0x4A, 0xFF, 0x6C,
+0x8C, 0xEA, 0xF3, 0x17, 0x01, 0x6A, 0x20, 0xF0,
+0x58, 0xC1, 0x47, 0xA0, 0x10, 0x6C, 0x8C, 0xEA,
+0xAC, 0xEA, 0xDB, 0x22, 0x0A, 0x92, 0xA6, 0xA0,
+0x09, 0x6E, 0x06, 0xD2, 0x0D, 0x92, 0x90, 0x67,
+0x04, 0xD2, 0x0C, 0x92, 0xD1, 0x18, 0x1B, 0x55,
+0x05, 0xD2, 0x0E, 0x92, 0x80, 0x72, 0xCD, 0x61,
+0x15, 0x97, 0x14, 0x91, 0x13, 0x90, 0x00, 0xEF,
+0x0B, 0x63, 0x02, 0x72, 0x13, 0x60, 0x03, 0x72,
+0x1A, 0x60, 0x01, 0x72, 0x0A, 0x61, 0x0B, 0x93,
+0x55, 0xA0, 0x94, 0xA0, 0x78, 0xA3, 0x40, 0x32,
+0xA7, 0x41, 0x8D, 0xEA, 0x49, 0x4D, 0x00, 0x6C,
+0x1E, 0x2A, 0x5D, 0x67, 0x20, 0xF0, 0x42, 0xA2,
+0x01, 0x4A, 0xBA, 0x17, 0x59, 0xA0, 0x98, 0xA0,
+0x0B, 0x93, 0x40, 0x32, 0xA7, 0x41, 0x8D, 0xEA,
+0x59, 0x4D, 0x79, 0xA3, 0xF0, 0x17, 0x5B, 0xA0,
+0x9A, 0xA0, 0x0B, 0x93, 0x40, 0x32, 0xA7, 0x41,
+0x8D, 0xEA, 0x69, 0x4D, 0x7A, 0xA3, 0xE7, 0x17,
+0x91, 0xE5, 0xC0, 0xA4, 0x0D, 0x26, 0x9D, 0x67,
+0x20, 0xF0, 0x80, 0xA4, 0x01, 0x4C, 0xDD, 0x67,
+0x20, 0xF0, 0x80, 0xC6, 0x9D, 0x67, 0x20, 0xF0,
+0x80, 0xA4, 0x63, 0xEC, 0xF1, 0x61, 0xD9, 0x17,
+0x01, 0x6E, 0xC0, 0xC4, 0x9D, 0x67, 0x20, 0xF0,
+0x80, 0xA4, 0xC2, 0x67, 0xC7, 0xEC, 0x86, 0x67,
+0x01, 0x6E, 0xCC, 0xEC, 0xE8, 0x24, 0x10, 0xD2,
+0x0A, 0x92, 0x0D, 0x96, 0x0C, 0x97, 0x08, 0x04,
+0x05, 0xD2, 0x04, 0xD4, 0x90, 0x67, 0x11, 0xD3,
+0xD1, 0x18, 0x33, 0x54, 0x0F, 0xD5, 0x0E, 0x92,
+0x0F, 0x95, 0x11, 0x93, 0x80, 0x72, 0x10, 0x92,
+0xD6, 0x61, 0xA6, 0x17, 0x01, 0x6D, 0x40, 0xF0,
+0xA8, 0xC4, 0x93, 0xA0, 0x01, 0x6D, 0x04, 0x6E,
+0x87, 0xEA, 0xAC, 0xEC, 0x01, 0x2C, 0x09, 0x6E,
+0x51, 0xE0, 0x0B, 0xD2, 0x0A, 0x92, 0xA8, 0xA4,
+0x90, 0x67, 0x06, 0xD2, 0x0D, 0x92, 0x04, 0xD2,
+0x0C, 0x92, 0xD1, 0x18, 0x1B, 0x55, 0x05, 0xD2,
+0x0E, 0x92, 0x80, 0x72, 0x0B, 0x92, 0x7F, 0xF7,
+0x11, 0x61, 0x8A, 0x17, 0xA8, 0x32, 0xF7, 0xF0,
+0x01, 0x6D, 0xA0, 0x35, 0xA0, 0x35, 0x6D, 0xF1,
+0x0C, 0x4D, 0xB5, 0xE2, 0x40, 0x9D, 0xC9, 0xA2,
+0x68, 0xA2, 0xAA, 0xA2, 0xC0, 0x36, 0x6D, 0xEE,
+0x6B, 0xA2, 0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEE,
+0x00, 0xF6, 0x60, 0x33, 0xCD, 0xEB, 0x1D, 0x23,
+0x01, 0x6D, 0xB9, 0xC2, 0x67, 0xA3, 0x02, 0x6D,
+0xAC, 0xEB, 0x11, 0x2B, 0x20, 0xF0, 0x79, 0xA2,
+0x72, 0x33, 0x91, 0xE3, 0x10, 0x54, 0x01, 0x61,
+0x0F, 0x6C, 0x20, 0xF0, 0x79, 0xA2, 0x0F, 0x6D,
+0x90, 0x34, 0xAC, 0xEB, 0x6D, 0xEC, 0x20, 0xF0,
+0x99, 0xC2, 0x20, 0xE8, 0x00, 0x65, 0x20, 0xF0,
+0x7A, 0xA2, 0xA1, 0x43, 0x20, 0xF0, 0xBA, 0xC2,
+0xE9, 0x23, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0x80, 0xF0, 0x07, 0x2C, 0xFA, 0x63, 0x0B, 0x62,
+0x0A, 0xD1, 0x09, 0xD0, 0xF7, 0xF0, 0x01, 0x6A,
+0xF7, 0xF0, 0x01, 0x69, 0x40, 0x32, 0x20, 0x31,
+0x20, 0x31, 0x40, 0x32, 0x6D, 0xF1, 0x0C, 0x9A,
+0xD2, 0xF4, 0x50, 0x99, 0x19, 0xF6, 0x06, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0xF7, 0xF0, 0x01, 0x6E,
+0x01, 0x6D, 0x4D, 0xED, 0xC0, 0x36, 0xC0, 0x36,
+0x04, 0x02, 0x04, 0xD5, 0xD2, 0xF4, 0xE4, 0x9E,
+0xA0, 0xAA, 0x19, 0xF6, 0x06, 0x6C, 0x40, 0xEF,
+0x06, 0xD6, 0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37,
+0xE0, 0x37, 0xD2, 0xF4, 0x40, 0x9F, 0x2F, 0x6D,
+0xF9, 0xF6, 0x00, 0x6C, 0x40, 0xEA, 0x05, 0xD7,
+0xD2, 0xF4, 0x50, 0x99, 0xF9, 0xF6, 0x02, 0x6C,
+0x40, 0xEA, 0x00, 0x65, 0x05, 0x97, 0x04, 0xD2,
+0x2E, 0x6D, 0xD2, 0xF4, 0x40, 0x9F, 0xF9, 0xF6,
+0x00, 0x6C, 0x40, 0xEA, 0x00, 0x65, 0xD2, 0xF4,
+0x50, 0x99, 0xF9, 0xF6, 0x02, 0x6C, 0x40, 0xEA,
+0x00, 0x65, 0x04, 0x93, 0x19, 0xF6, 0x06, 0x6C,
+0x7D, 0xE2, 0xD2, 0xF4, 0x50, 0x99, 0x40, 0xEA,
+0x05, 0xD7, 0x02, 0x6D, 0x06, 0x96, 0xAB, 0xED,
+0x4C, 0xED, 0x04, 0xD5, 0x04, 0x02, 0xD2, 0xF4,
+0x64, 0x9E, 0x19, 0xF6, 0x06, 0x6C, 0x40, 0xEB,
+0xA0, 0xAA, 0x40, 0xF0, 0x81, 0xA0, 0x40, 0xF0,
+0x40, 0xA0, 0x40, 0xF0, 0x62, 0xA0, 0x80, 0x34,
+0x4D, 0xEC, 0x40, 0xF0, 0x43, 0xA0, 0x60, 0x33,
+0x60, 0x33, 0x05, 0x97, 0x6D, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x4A, 0xEF, 0xFF, 0xF7,
+0x1F, 0x6C, 0x13, 0x60, 0xE3, 0xEA, 0x16, 0x60,
+0x4B, 0xE7, 0x4C, 0xEC, 0xD1, 0x18, 0xCD, 0x5B,
+0x00, 0x6D, 0xE2, 0x32, 0x40, 0xF0, 0xE0, 0xC0,
+0x40, 0xF0, 0x41, 0xC0, 0x00, 0xF6, 0xE2, 0x37,
+0x42, 0x32, 0x40, 0xF0, 0x42, 0xC0, 0x40, 0xF0,
+0xE3, 0xC0, 0x0B, 0x97, 0x0A, 0x91, 0x09, 0x90,
+0x00, 0xEF, 0x06, 0x63, 0x4F, 0xEA, 0xE9, 0xE2,
+0xE8, 0x17, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF9, 0x63, 0x0D, 0x62, 0x0C, 0xD1, 0x0B, 0xD0,
+0xF7, 0xF0, 0x01, 0x6A, 0xFF, 0x6D, 0x40, 0x32,
+0xAC, 0xEC, 0x40, 0x32, 0x88, 0x33, 0x6D, 0xF1,
+0x0C, 0x4A, 0x49, 0xE3, 0x60, 0x9A, 0x54, 0xA3,
+0x80, 0xF0, 0x08, 0x22, 0x02, 0x72, 0x80, 0xF0,
+0x05, 0x60, 0xC1, 0xA3, 0x40, 0xA3, 0x04, 0x67,
+0x82, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x43, 0xA3,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x22, 0xA2, 0x07, 0x6A,
+0x90, 0x67, 0x4C, 0xE9, 0x07, 0xD3, 0xAC, 0xE9,
+0xD1, 0x18, 0xE8, 0x5B, 0x06, 0xD5, 0x07, 0x93,
+0x59, 0xA3, 0x14, 0x2A, 0xC9, 0xA3, 0x88, 0xA3,
+0x4A, 0xA3, 0xC0, 0x36, 0x8D, 0xEE, 0x8B, 0xA3,
+0x40, 0x32, 0x40, 0x32, 0x4D, 0xEE, 0x00, 0xF6,
+0x80, 0x34, 0xCD, 0xEC, 0x45, 0xA4, 0x06, 0x95,
+0x40, 0x6C, 0x8B, 0xEC, 0x8C, 0xEA, 0x4C, 0xED,
+0x80, 0x75, 0x06, 0x61, 0xB1, 0x67, 0x90, 0x67,
+0xD1, 0x18, 0x54, 0x5B, 0x06, 0xD3, 0x06, 0x93,
+0xF7, 0xF0, 0x01, 0x6F, 0xE0, 0x37, 0xE0, 0x37,
+0xF3, 0xF3, 0x4C, 0x9F, 0x01, 0x6E, 0xB1, 0x67,
+0x90, 0x67, 0x08, 0xD3, 0x40, 0xEA, 0x07, 0xD7,
+0x07, 0x97, 0x06, 0xD2, 0x00, 0x6E, 0xF3, 0xF3,
+0x4C, 0x9F, 0xB1, 0x67, 0x40, 0xEA, 0x90, 0x67,
+0x08, 0x93, 0x89, 0xA3, 0xAA, 0xA3, 0xC8, 0xA3,
+0x6B, 0xA3, 0x80, 0x34, 0xA0, 0x35, 0x8D, 0xEE,
+0xA0, 0x35, 0xCD, 0xED, 0x00, 0xF6, 0x60, 0x33,
+0xAD, 0xEB, 0x72, 0xA3, 0x34, 0x23, 0x06, 0x97,
+0xC2, 0x67, 0x04, 0xD0, 0x00, 0x6D, 0x07, 0x6C,
+0x08, 0xD3, 0xD1, 0x18, 0x46, 0x52, 0x07, 0xD2,
+0x07, 0x92, 0x08, 0x93, 0xF7, 0xF0, 0x01, 0x6C,
+0x80, 0x34, 0x80, 0x34, 0xD3, 0xF3, 0x84, 0x9C,
+0x05, 0xD0, 0x3C, 0x65, 0x68, 0x34, 0x6D, 0xE4,
+0x83, 0xEB, 0x80, 0xF5, 0x62, 0x35, 0x60, 0x33,
+0x68, 0x33, 0x98, 0x67, 0x59, 0xE3, 0x80, 0x34,
+0x06, 0x92, 0x88, 0x37, 0xAD, 0xEF, 0x5D, 0xE7,
+0x97, 0xF0, 0x19, 0x6A, 0x40, 0x32, 0x63, 0xEE,
+0x40, 0x32, 0xAE, 0xF0, 0x19, 0x4A, 0x78, 0x67,
+0x04, 0xD2, 0xFD, 0xE3, 0xB1, 0x67, 0x59, 0x67,
+0x40, 0xEA, 0x90, 0x67, 0x0D, 0x97, 0x0C, 0x91,
+0x0B, 0x90, 0x00, 0xEF, 0x07, 0x63, 0x04, 0x6B,
+0xD5, 0x17, 0x00, 0x65, 0x00, 0x6B, 0x01, 0x6A,
+0x82, 0xEB, 0x4B, 0xEA, 0x0F, 0x61, 0x40, 0xF7,
+0xC0, 0x34, 0x4F, 0xEA, 0xA4, 0x46, 0xD3, 0xE4,
+0xCD, 0xE4, 0x6C, 0x33, 0xE2, 0x67, 0xE6, 0xEB,
+0xE0, 0xC6, 0x01, 0x4E, 0xCA, 0xED, 0xF8, 0x61,
+0x20, 0xE8, 0x00, 0x65, 0xFF, 0x63, 0x01, 0xD1,
+0x00, 0xD0, 0x7D, 0xE5, 0xE0, 0xA7, 0xEE, 0xEA,
+0x08, 0x6F, 0x01, 0x69, 0x46, 0x30, 0x2C, 0xEA,
+0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31, 0x20, 0x31,
+0xC9, 0xF3, 0x2C, 0x99, 0x4B, 0xEA, 0xFF, 0x4F,
+0x2C, 0xEA, 0x0E, 0xEA, 0xFF, 0xF7, 0x1F, 0x68,
+0x0C, 0xEF, 0xEF, 0x2F, 0x01, 0x4B, 0x60, 0x33,
+0x60, 0x33, 0x63, 0x33, 0x63, 0x33, 0x82, 0xEB,
+0xE4, 0x61, 0x40, 0xF7, 0xC0, 0x34, 0x4F, 0xEA,
+0xA4, 0x46, 0xD3, 0xE4, 0xCD, 0xE4, 0x6C, 0x33,
+0xE2, 0x67, 0xE6, 0xEB, 0xE0, 0xC6, 0x01, 0x4E,
+0xCA, 0xED, 0xF8, 0x61, 0x01, 0x91, 0x00, 0x90,
+0x20, 0xE8, 0x01, 0x63, 0x20, 0xE8, 0x00, 0x6A,
+0x01, 0x6A, 0x8C, 0xEA, 0x03, 0x22, 0x57, 0xA5,
+0x01, 0x4A, 0x57, 0xC5, 0x02, 0x6A, 0x4C, 0xEC,
+0x03, 0x24, 0x58, 0xA5, 0x01, 0x4A, 0x58, 0xC5,
+0x20, 0xE8, 0x00, 0x65, 0x01, 0x6A, 0x8C, 0xEA,
+0x03, 0x22, 0x59, 0xA5, 0x01, 0x4A, 0x59, 0xC5,
+0xFF, 0x6A, 0x01, 0x4A, 0x8C, 0xEA, 0x03, 0x22,
+0x5A, 0xA5, 0x01, 0x4A, 0x5A, 0xC5, 0x00, 0xF2,
+0x00, 0x6A, 0x8C, 0xEA, 0x03, 0x22, 0x5B, 0xA5,
+0x01, 0x4A, 0x5B, 0xC5, 0x00, 0xF4, 0x00, 0x6A,
+0x8C, 0xEA, 0x03, 0x22, 0x5C, 0xA5, 0x01, 0x4A,
+0x5C, 0xC5, 0x01, 0xF0, 0x00, 0x6A, 0x8C, 0xEA,
+0x03, 0x22, 0x5D, 0xA5, 0x01, 0x4A, 0x5D, 0xC5,
+0x02, 0xF0, 0x00, 0x6A, 0x8C, 0xEA, 0x03, 0x22,
+0x5E, 0xA5, 0x01, 0x4A, 0x5E, 0xC5, 0x08, 0xF0,
+0x00, 0x6A, 0x8C, 0xEA, 0x03, 0x22, 0x5F, 0xA5,
+0x01, 0x4A, 0x5F, 0xC5, 0x10, 0xF0, 0x00, 0x6A,
+0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0, 0x40, 0xA5,
+0x01, 0x4A, 0x20, 0xF0, 0x40, 0xC5, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xC9, 0xF2,
+0x44, 0x9A, 0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0,
+0x41, 0xA5, 0x01, 0x4A, 0x20, 0xF0, 0x41, 0xC5,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x69, 0xF1, 0x4C, 0x9A, 0x8C, 0xEA, 0x05, 0x22,
+0x20, 0xF0, 0x42, 0xA5, 0x01, 0x4A, 0x20, 0xF0,
+0x42, 0xC5, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xA9, 0xF2, 0x5C, 0x9A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x43, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x43, 0xC5, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x29, 0xF2, 0x5C, 0x9A,
+0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0, 0x44, 0xA5,
+0x01, 0x4A, 0x20, 0xF0, 0x44, 0xC5, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x69, 0xF1,
+0x54, 0x9A, 0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0,
+0x45, 0xA5, 0x01, 0x4A, 0x20, 0xF0, 0x45, 0xC5,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC9, 0xF1, 0x40, 0x9A, 0x8C, 0xEA, 0x05, 0x22,
+0x20, 0xF0, 0x46, 0xA5, 0x01, 0x4A, 0x20, 0xF0,
+0x46, 0xC5, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x89, 0xF1, 0x54, 0x9A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x47, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x47, 0xC5, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0xC9, 0xF1, 0x58, 0x9A,
+0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0, 0x48, 0xA5,
+0x01, 0x4A, 0x20, 0xF0, 0x48, 0xC5, 0x00, 0x54,
+0x05, 0x60, 0x20, 0xF0, 0x49, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x49, 0xC5, 0x20, 0xE8, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x49, 0xF1, 0x48, 0x9A, 0x8C, 0xEA, 0x05, 0x22,
+0x20, 0xF0, 0x4A, 0xA5, 0x01, 0x4A, 0x20, 0xF0,
+0x4A, 0xC5, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x89, 0xF1, 0x4C, 0x9A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x4B, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x4B, 0xC5, 0x00, 0x54, 0x05, 0x60,
+0x20, 0xF0, 0x4C, 0xA5, 0x01, 0x4A, 0x20, 0xF0,
+0x4C, 0xC5, 0x20, 0xE8, 0x00, 0x65, 0x00, 0x65,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0x69, 0xF1, 0x4C, 0x9A, 0x8C, 0xEA, 0x05, 0x22,
+0x20, 0xF0, 0x4E, 0xA5, 0x01, 0x4A, 0x20, 0xF0,
+0x4E, 0xC5, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0xA9, 0xF2, 0x5C, 0x9A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x4F, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x4F, 0xC5, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x29, 0xF2, 0x5C, 0x9A,
+0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0, 0x50, 0xA5,
+0x01, 0x4A, 0x20, 0xF0, 0x50, 0xC5, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x69, 0xF1,
+0x54, 0x9A, 0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0,
+0x51, 0xA5, 0x01, 0x4A, 0x20, 0xF0, 0x51, 0xC5,
+0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32, 0x40, 0x32,
+0xC9, 0xF1, 0x40, 0x9A, 0x8C, 0xEA, 0x05, 0x22,
+0x20, 0xF0, 0x52, 0xA5, 0x01, 0x4A, 0x20, 0xF0,
+0x52, 0xC5, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x89, 0xF1, 0x54, 0x9A, 0x4C, 0xEC,
+0x05, 0x24, 0x20, 0xF0, 0x53, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x53, 0xC5, 0x20, 0xE8, 0x00, 0x65,
+0x01, 0xF0, 0x00, 0x6A, 0x8C, 0xEA, 0x05, 0x22,
+0x20, 0xF0, 0x5D, 0xA5, 0x01, 0x4A, 0x20, 0xF0,
+0x5D, 0xC5, 0x02, 0xF0, 0x00, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x5E, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x5E, 0xC5, 0x04, 0xF0, 0x00, 0x6A,
+0x8C, 0xEA, 0x05, 0x22, 0x20, 0xF0, 0x5F, 0xA5,
+0x01, 0x4A, 0x20, 0xF0, 0x5F, 0xC5, 0x08, 0xF0,
+0x00, 0x6A, 0x4C, 0xEC, 0x05, 0x24, 0x40, 0xF0,
+0x40, 0xA5, 0x01, 0x4A, 0x40, 0xF0, 0x40, 0xC5,
+0x20, 0xE8, 0x00, 0x65, 0x01, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x55, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x55, 0xC5, 0x02, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x56, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x56, 0xC5, 0x04, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x57, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x57, 0xC5, 0x08, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x58, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x58, 0xC5, 0x10, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x59, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x59, 0xC5, 0x20, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x5A, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x5A, 0xC5, 0x40, 0x6A, 0x8C, 0xEA,
+0x05, 0x22, 0x20, 0xF0, 0x5B, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x5B, 0xC5, 0x80, 0x6A, 0x4C, 0xEC,
+0x05, 0x24, 0x20, 0xF0, 0x5C, 0xA5, 0x01, 0x4A,
+0x20, 0xF0, 0x5C, 0xC5, 0x20, 0xE8, 0x00, 0x65,
+0xFB, 0x63, 0x09, 0x62, 0x08, 0xD1, 0x07, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0x85, 0xA3, 0x44, 0xA3,
+0xA6, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x47, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x99, 0xA2, 0x78, 0xA2,
+0xBA, 0xA2, 0x80, 0x34, 0x6D, 0xEC, 0x7B, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x60, 0x33, 0x8D, 0xEB, 0x81, 0xA3, 0x40, 0xA3,
+0xA2, 0xA3, 0x80, 0x34, 0x4D, 0xEC, 0x43, 0xA3,
+0xA0, 0x35, 0xA0, 0x35, 0xAD, 0xEC, 0x00, 0xF6,
+0x40, 0x32, 0x8D, 0xEA, 0x20, 0xF0, 0x69, 0xA2,
+0x20, 0xF0, 0x08, 0xA2, 0x20, 0xF0, 0x8A, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x20, 0xF0, 0x0B, 0xA2,
+0x80, 0x34, 0x80, 0x34, 0x8D, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0xF7, 0xF0, 0x01, 0x6B,
+0x60, 0x33, 0x60, 0x33, 0xD2, 0xF4, 0x54, 0x9B,
+0x78, 0xF1, 0x04, 0x6C, 0x40, 0xEA, 0x04, 0xD3,
+0x04, 0x93, 0x22, 0x67, 0x30, 0xF5, 0x04, 0x6C,
+0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA, 0x00, 0x65,
+0x04, 0x93, 0x48, 0x22, 0x81, 0xA0, 0x01, 0x4C,
+0x81, 0xC0, 0x01, 0x6C, 0x4C, 0xEC, 0x03, 0x24,
+0x89, 0xA0, 0x01, 0x4C, 0x89, 0xC0, 0x02, 0x6C,
+0x4C, 0xEC, 0x03, 0x24, 0x8A, 0xA0, 0x01, 0x4C,
+0x8A, 0xC0, 0x04, 0x6C, 0x4C, 0xEC, 0x03, 0x24,
+0x8B, 0xA0, 0x01, 0x4C, 0x8B, 0xC0, 0x08, 0x6C,
+0x4C, 0xEC, 0x03, 0x24, 0x8D, 0xA0, 0x01, 0x4C,
+0x8D, 0xC0, 0x10, 0x6C, 0x4C, 0xEC, 0x03, 0x24,
+0x8F, 0xA0, 0x01, 0x4C, 0x8F, 0xC0, 0x20, 0x6C,
+0x4C, 0xEC, 0x03, 0x24, 0x8E, 0xA0, 0x01, 0x4C,
+0x8E, 0xC0, 0x40, 0x6C, 0x4C, 0xEC, 0x03, 0x24,
+0x90, 0xA0, 0x01, 0x4C, 0x90, 0xC0, 0x80, 0x6C,
+0x4C, 0xEC, 0x03, 0x24, 0x91, 0xA0, 0x01, 0x4C,
+0x91, 0xC0, 0xFF, 0x6C, 0x01, 0x4C, 0x4C, 0xEC,
+0x03, 0x24, 0x92, 0xA0, 0x01, 0x4C, 0x92, 0xC0,
+0x00, 0xF4, 0x00, 0x6C, 0x4C, 0xEC, 0x03, 0x24,
+0x95, 0xA0, 0x01, 0x4C, 0x95, 0xC0, 0x01, 0xF0,
+0x00, 0x6C, 0x8C, 0xEA, 0x03, 0x22, 0x56, 0xA0,
+0x01, 0x4A, 0x56, 0xC0, 0x73, 0x21, 0x40, 0xA0,
+0x01, 0x4A, 0x40, 0xC0, 0x01, 0x6A, 0x2C, 0xEA,
+0x0F, 0x22, 0x42, 0xA0, 0xF8, 0xF3, 0x0C, 0x6C,
+0x04, 0xD3, 0x01, 0x4A, 0x42, 0xC0, 0xD2, 0xF4,
+0x54, 0x9B, 0x40, 0xEA, 0x00, 0x65, 0x82, 0x67,
+0xD1, 0x18, 0xA2, 0x5C, 0xB0, 0x67, 0x04, 0x93,
+0x02, 0x6A, 0x2C, 0xEA, 0x0F, 0x22, 0x43, 0xA0,
+0xD8, 0xF6, 0x04, 0x6C, 0x04, 0xD3, 0x01, 0x4A,
+0x43, 0xC0, 0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x82, 0x67, 0xD1, 0x18, 0xA9, 0x5C,
+0xB0, 0x67, 0x04, 0x93, 0x08, 0x6A, 0x2C, 0xEA,
+0x0F, 0x22, 0x44, 0xA0, 0x19, 0xF0, 0x00, 0x6C,
+0x04, 0xD3, 0x01, 0x4A, 0x44, 0xC0, 0xD2, 0xF4,
+0x54, 0x9B, 0x40, 0xEA, 0x00, 0x65, 0x82, 0x67,
+0xD1, 0x18, 0xFE, 0x5C, 0xB0, 0x67, 0x04, 0x93,
+0x10, 0x6A, 0x2C, 0xEA, 0x0F, 0x22, 0x45, 0xA0,
+0xF9, 0xF4, 0x1C, 0x6C, 0x04, 0xD3, 0x01, 0x4A,
+0x45, 0xC0, 0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x82, 0x67, 0xD1, 0x18, 0x10, 0x5D,
+0xB0, 0x67, 0x04, 0x93, 0x20, 0x6A, 0x2C, 0xEA,
+0x03, 0x22, 0x46, 0xA0, 0x01, 0x4A, 0x46, 0xC0,
+0x40, 0x6A, 0x2C, 0xEA, 0x0F, 0x22, 0x47, 0xA0,
+0xF9, 0xF6, 0x14, 0x6C, 0x04, 0xD3, 0x01, 0x4A,
+0x47, 0xC0, 0xD2, 0xF4, 0x54, 0x9B, 0x40, 0xEA,
+0x00, 0x65, 0x82, 0x67, 0xD1, 0x18, 0x10, 0x5D,
+0xB0, 0x67, 0x04, 0x93, 0x80, 0x6A, 0x4C, 0xE9,
+0x0D, 0x21, 0x48, 0xA0, 0xF9, 0xF4, 0x0C, 0x6C,
+0x01, 0x4A, 0x48, 0xC0, 0xD2, 0xF4, 0x54, 0x9B,
+0x40, 0xEA, 0x00, 0x65, 0x82, 0x67, 0xD1, 0x18,
+0x10, 0x5D, 0xB0, 0x67, 0x09, 0x97, 0x08, 0x91,
+0x07, 0x90, 0x00, 0xEF, 0x05, 0x63, 0x00, 0x65,
+0xFC, 0x63, 0x07, 0x62, 0x06, 0xD1, 0x05, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xA5, 0xA3, 0x44, 0xA3,
+0xC6, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x47, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0xB9, 0xA2, 0x78, 0xA2,
+0xDA, 0xA2, 0xA0, 0x35, 0x6D, 0xED, 0x7B, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x60, 0x33, 0xAD, 0xEB, 0xA1, 0xA3, 0x40, 0xA3,
+0xC2, 0xA3, 0xA0, 0x35, 0x4D, 0xED, 0x43, 0xA3,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xED, 0x00, 0xF6,
+0x40, 0x32, 0xAD, 0xEA, 0x20, 0xF0, 0x69, 0xA2,
+0x20, 0xF0, 0x08, 0xA2, 0x20, 0xF0, 0xAA, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x20, 0xF0, 0x0B, 0xA2,
+0xA0, 0x35, 0xA0, 0x35, 0x15, 0x74, 0xAD, 0xEB,
+0x00, 0xF6, 0x00, 0x30, 0x58, 0x67, 0x6D, 0xE8,
+0x44, 0x2A, 0xF7, 0xF0, 0x01, 0x69, 0x20, 0x31,
+0x20, 0x31, 0x40, 0xC0, 0x42, 0xC0, 0x43, 0xC0,
+0x44, 0xC0, 0x45, 0xC0, 0x46, 0xC0, 0x47, 0xC0,
+0x48, 0xC0, 0x49, 0xC0, 0x4A, 0xC0, 0x93, 0xF6,
+0x40, 0x99, 0x87, 0x40, 0x02, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x10, 0x4C, 0x93, 0xF6, 0x40, 0x99,
+0x87, 0x40, 0x11, 0x6E, 0x00, 0x6D, 0x40, 0xEA,
+0x12, 0x4C, 0x93, 0xF6, 0x40, 0x99, 0x87, 0x40,
+0x04, 0x6E, 0x00, 0x6D, 0x40, 0xEA, 0x23, 0x4C,
+0x93, 0xF6, 0x40, 0x99, 0x87, 0x40, 0x06, 0x6E,
+0x00, 0x6D, 0x40, 0xEA, 0x27, 0x4C, 0x93, 0xF6,
+0x40, 0x99, 0x87, 0x40, 0x01, 0x6E, 0x00, 0x6D,
+0x40, 0xEA, 0x2D, 0x4C, 0x93, 0xF6, 0x40, 0x99,
+0x87, 0x40, 0x08, 0x6E, 0x00, 0x6D, 0x40, 0xEA,
+0x2E, 0x4C, 0x93, 0xF6, 0x40, 0x99, 0x87, 0x40,
+0x04, 0x6E, 0x00, 0x6D, 0x40, 0xEA, 0x36, 0x4C,
+0x07, 0x97, 0x06, 0x91, 0x05, 0x90, 0x00, 0xEF,
+0x04, 0x63, 0x16, 0x6A, 0x4E, 0xEC, 0xF8, 0x2C,
+0x81, 0xC0, 0x89, 0xC0, 0x8A, 0xC0, 0x8B, 0xC0,
+0x8C, 0xC0, 0x8D, 0xC0, 0x8E, 0xC0, 0x8F, 0xC0,
+0x90, 0xC0, 0x91, 0xC0, 0x92, 0xC0, 0x93, 0xC0,
+0x94, 0xC0, 0x95, 0xC0, 0x96, 0xC0, 0xE8, 0x17,
+0xF5, 0x63, 0x15, 0x62, 0x14, 0xD1, 0x13, 0xD0,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x15, 0xF0, 0x08, 0x4B, 0xC5, 0xA3, 0x44, 0xA3,
+0xE6, 0xA3, 0xC0, 0x36, 0x4D, 0xEE, 0x47, 0xA3,
+0xE0, 0x37, 0xE0, 0x37, 0xED, 0xEE, 0x00, 0xF6,
+0x40, 0x32, 0xCD, 0xEA, 0x20, 0xF0, 0x61, 0xA2,
+0x20, 0xF0, 0x00, 0xA2, 0x20, 0xF0, 0xC2, 0xA2,
+0x60, 0x33, 0x0D, 0xEB, 0x20, 0xF0, 0x03, 0xA2,
+0xC0, 0x36, 0xC0, 0x36, 0xCD, 0xEB, 0x00, 0xF6,
+0x00, 0x30, 0x6D, 0xE8, 0x40, 0xA0, 0x00, 0x6B,
+0x0A, 0xD3, 0x0A, 0x93, 0x0A, 0x96, 0xFF, 0x6F,
+0xEC, 0xEB, 0x0B, 0xD3, 0x62, 0x67, 0x67, 0xEE,
+0x01, 0x6E, 0xCC, 0xEB, 0x0C, 0xD3, 0x40, 0xF1,
+0x15, 0x2B, 0xF7, 0xF0, 0x01, 0x6B, 0xF7, 0xF0,
+0x01, 0x6A, 0x60, 0x33, 0x60, 0x33, 0x40, 0x32,
+0xC9, 0xF3, 0x30, 0x9B, 0x40, 0x32, 0x72, 0xF4,
+0x54, 0x9A, 0x0E, 0xD5, 0x0D, 0xD4, 0xB1, 0x67,
+0x84, 0x6C, 0x40, 0xEA, 0x0F, 0xD7, 0x62, 0x67,
+0x0F, 0x97, 0x23, 0x2A, 0x02, 0x67, 0x7D, 0x67,
+0x06, 0x6A, 0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2A, 0xF3, 0x18, 0x4A,
+0x07, 0xD2, 0x1F, 0x6A, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x04, 0xD0, 0xC8, 0x6F, 0x02, 0x6E,
+0x00, 0xF4, 0x00, 0x6C, 0x40, 0xEA, 0x00, 0x6D,
+0x4C, 0x10, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF6, 0x40, 0x9A, 0x83, 0x67,
+0x84, 0x6E, 0x00, 0x6D, 0x10, 0xD7, 0x40, 0xEA,
+0x0F, 0xD3, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x72, 0xF4, 0x54, 0x9A, 0xFF, 0x6C,
+0xB1, 0x67, 0x40, 0xEA, 0x09, 0x4C, 0x10, 0x97,
+0x0F, 0x93, 0x82, 0x67, 0xEC, 0xEC, 0x40, 0xF0,
+0x80, 0xC3, 0x42, 0x34, 0xA4, 0x67, 0x82, 0x34,
+0xEC, 0xEC, 0xEC, 0xED, 0x40, 0xF0, 0x82, 0xC3,
+0x00, 0xF6, 0x42, 0x34, 0x40, 0xF0, 0xA1, 0xC3,
+0x40, 0xF0, 0x83, 0xC3, 0x25, 0x2A, 0x02, 0x67,
+0x06, 0x6B, 0x5D, 0x67, 0x78, 0xC2, 0xF7, 0xF0,
+0x01, 0x6B, 0x60, 0x33, 0x60, 0x33, 0x2A, 0xF3,
+0x18, 0x4B, 0x07, 0xD3, 0x2B, 0x6B, 0x08, 0xD3,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0x12, 0xF1, 0x78, 0x9B, 0x40, 0xEB, 0x06, 0x04,
+0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33, 0x60, 0x33,
+0xF3, 0xF0, 0x70, 0x9B, 0x04, 0xD0, 0xC9, 0x6F,
+0x02, 0x6E, 0x00, 0xF4, 0x00, 0x6C, 0x40, 0xEB,
+0x00, 0x6D, 0xFF, 0x6A, 0x0B, 0xD2, 0xC8, 0x10,
+0x0F, 0xD3, 0xF7, 0xF0, 0x01, 0x6B, 0x60, 0x33,
+0x60, 0x33, 0x93, 0xF6, 0x60, 0x9B, 0xFF, 0x6E,
+0x82, 0x67, 0x09, 0x4E, 0x43, 0x67, 0x00, 0x6D,
+0x40, 0xEA, 0x10, 0xD7, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x72, 0xF4, 0x54, 0x9A,
+0xB1, 0x67, 0x40, 0xEA, 0x1C, 0x6C, 0x10, 0x97,
+0x0F, 0x93, 0x22, 0x67, 0xEC, 0xEA, 0x40, 0xF0,
+0x50, 0xC3, 0x22, 0x32, 0x82, 0x67, 0x42, 0x32,
+0xEC, 0xEA, 0xEC, 0xEC, 0x40, 0xF0, 0x52, 0xC3,
+0x00, 0xF6, 0x22, 0x32, 0x40, 0xF0, 0x91, 0xC3,
+0x40, 0xF0, 0x53, 0xC3, 0x1E, 0x29, 0x06, 0x6A,
+0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2A, 0xF3, 0x18, 0x4A,
+0x07, 0xD2, 0x51, 0x6A, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 0x9A, 0x40, 0xEA, 0x06, 0x04, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0xF3, 0xF0,
+0x50, 0x9A, 0x04, 0xD1, 0x00, 0xF3, 0x07, 0x6F,
+0x62, 0x17, 0xF7, 0xF0, 0x01, 0x6A, 0x40, 0x32,
+0x40, 0x32, 0x93, 0xF6, 0x40, 0x9A, 0x1C, 0x6E,
+0x00, 0x6D, 0x91, 0x67, 0x10, 0xD7, 0x40, 0xEA,
+0x0F, 0xD3, 0xF7, 0xF0, 0x01, 0x6A, 0xF7, 0xF0,
+0x01, 0x6C, 0x40, 0x32, 0x80, 0x34, 0x80, 0x34,
+0x40, 0x32, 0xC9, 0xF3, 0xB4, 0x9C, 0x72, 0xF4,
+0x54, 0x9A, 0x40, 0xEA, 0x1C, 0x6C, 0x10, 0x97,
+0x0F, 0x93, 0x82, 0x67, 0xEC, 0xEC, 0x42, 0x35,
+0x40, 0xF0, 0x9C, 0xC3, 0x85, 0x67, 0xEC, 0xEC,
+0xA2, 0x35, 0x40, 0xF0, 0x9D, 0xC3, 0xEC, 0xED,
+0x00, 0xF6, 0x42, 0x34, 0x40, 0xF0, 0xBE, 0xC3,
+0x40, 0xF0, 0x9F, 0xC3, 0x15, 0x2A, 0x06, 0x6A,
+0x7D, 0x67, 0x58, 0xC3, 0xF7, 0xF0, 0x01, 0x6A,
+0x40, 0x32, 0x40, 0x32, 0x2A, 0xF3, 0x18, 0x4A,
+0x07, 0xD2, 0x6C, 0x6A, 0x08, 0xD2, 0xF7, 0xF0,
+0x01, 0x6A, 0x40, 0x32, 0x40, 0x32, 0x12, 0xF1,
+0x58, 